From 4df4e2d71aca4544160f5e71ea8a75fc45ac5bc8 Mon Sep 17 00:00:00 2001 From: Ping Date: Sat, 9 Aug 2025 12:40:51 +0800 Subject: [PATCH 001/634] [Core] Cover cpplint for ray/src/ray/common/task (#55413) Signed-off-by: 400Ping --- src/ray/common/task/task.cc | 4 ++++ src/ray/common/task/task.h | 3 +++ src/ray/common/task/task_spec.cc | 4 ++++ src/ray/common/task/task_spec.h | 1 + src/ray/common/task/task_util.h | 6 ++++++ 5 files changed, 18 insertions(+) diff --git a/src/ray/common/task/task.cc b/src/ray/common/task/task.cc index 8be3a423c1b5..28d28d6a8a12 100644 --- a/src/ray/common/task/task.cc +++ b/src/ray/common/task/task.cc @@ -14,6 +14,10 @@ #include "ray/common/task/task.h" +#include +#include +#include + #include "absl/strings/str_format.h" namespace ray { diff --git a/src/ray/common/task/task.h b/src/ray/common/task/task.h index fa9f4db14b3e..9408990fb1de 100644 --- a/src/ray/common/task/task.h +++ b/src/ray/common/task/task.h @@ -16,6 +16,9 @@ #include +#include +#include + #include "ray/common/task/task_common.h" #include "ray/common/task/task_spec.h" diff --git a/src/ray/common/task/task_spec.cc b/src/ray/common/task/task_spec.cc index 1101d0f7c953..76cf5ae8574b 100644 --- a/src/ray/common/task/task_spec.cc +++ b/src/ray/common/task/task_spec.cc @@ -15,7 +15,11 @@ #include "ray/common/task/task_spec.h" #include +#include #include +#include +#include +#include #include "ray/common/ray_config.h" #include "ray/common/runtime_env_common.h" diff --git a/src/ray/common/task/task_spec.h b/src/ray/common/task/task_spec.h index dca4db743701..a076309d4b07 100644 --- a/src/ray/common/task/task_spec.h +++ b/src/ray/common/task/task_spec.h @@ -17,6 +17,7 @@ #include #include +#include #include #include #include diff --git a/src/ray/common/task/task_util.h b/src/ray/common/task/task_util.h index bf9e35eb0c38..1665f9e96298 100644 --- a/src/ray/common/task/task_util.h +++ b/src/ray/common/task/task_util.h @@ -14,6 +14,12 @@ #pragma once +#include +#include +#include +#include +#include + #include "ray/common/buffer.h" #include "ray/common/ray_object.h" #include "ray/common/task/task_spec.h" From ea69b7854e426a8be9e54e9c5e1af3fcae74493e Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Fri, 8 Aug 2025 23:11:12 -0700 Subject: [PATCH 002/634] [image] add cuda 12.6 for aarch64 (#55444) adding 12.6.3 Signed-off-by: Lonnie Liu --- .buildkite/linux_aarch64.rayci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/linux_aarch64.rayci.yml b/.buildkite/linux_aarch64.rayci.yml index 8cf1b12ef59a..0ba034ec1064 100644 --- a/.buildkite/linux_aarch64.rayci.yml +++ b/.buildkite/linux_aarch64.rayci.yml @@ -34,6 +34,7 @@ steps: - "12.3.2-cudnn9" - "12.4.1-cudnn" - "12.5.1-cudnn" + - "12.6.3-cudnn" - "12.8.1-cudnn" instance_type: builder-arm64 env: @@ -88,7 +89,8 @@ steps: --platform cu11.7.1-cudnn8 --platform cu11.8.0-cudnn8 --platform cu12.1.1-cudnn8 --platform cu12.3.2-cudnn9 --platform cu12.4.1-cudnn --platform cu12.5.1-cudnn - --platform cu12.8.1-cudnn --platform cpu + --platform cu12.6.3-cudnn --platform cu12.8.1-cudnn + --platform cpu --image-type ray --architecture aarch64 --upload depends_on: - manylinux-aarch64 From 495863ddd6c2613d47356fdf99c728aca5beaef9 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Sat, 9 Aug 2025 01:07:56 -0700 Subject: [PATCH 003/634] [ci] shorten test name of `concurrency_group_manager_test` (#55445) fixes #55441 Signed-off-by: Lonnie Liu --- src/ray/core_worker/task_execution/test/BUILD.bazel | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ray/core_worker/task_execution/test/BUILD.bazel b/src/ray/core_worker/task_execution/test/BUILD.bazel index 48666ca86ce0..731ce60abcdb 100644 --- a/src/ray/core_worker/task_execution/test/BUILD.bazel +++ b/src/ray/core_worker/task_execution/test/BUILD.bazel @@ -23,8 +23,9 @@ ray_cc_test( ], ) +# Test name is shortened for running on Windows. ray_cc_test( - name = "concurrency_group_manager_test", + name = "concurrency_grp_mgr_test", srcs = ["concurrency_group_manager_test.cc"], tags = ["team:core"], deps = [ From 8bdc75c09318dfb319df44265ffadbcb4c806e4f Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Sat, 9 Aug 2025 01:52:52 -0700 Subject: [PATCH 004/634] Revert "[release] Move logic to check whether to build custom BYOD image" (#55447) Reverts ray-project/ray#55397 --- release/ray_release/byod/build.py | 20 +++++++++++--------- release/ray_release/scripts/ray_bisect.py | 7 +------ release/ray_release/tests/test_byod_build.py | 6 +----- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/release/ray_release/byod/build.py b/release/ray_release/byod/build.py index 62ae1d62f3b3..14e9bb43311a 100644 --- a/release/ray_release/byod/build.py +++ b/release/ray_release/byod/build.py @@ -30,11 +30,13 @@ REQUIREMENTS_ML_BYOD = "requirements_ml_byod" -def build_anyscale_custom_byod_image( - image: str, base_image: str, post_build_script: str -) -> None: - if _image_exist(image): - logger.info(f"Image {image} already exists") +def build_anyscale_custom_byod_image(test: Test) -> None: + if not test.require_custom_byod_image(): + logger.info(f"Test {test.get_name()} does not require a custom byod image") + return + byod_image = test.get_anyscale_byod_image() + if _image_exist(byod_image): + logger.info(f"Image {byod_image} already exists") return env = os.environ.copy() @@ -45,11 +47,11 @@ def build_anyscale_custom_byod_image( "build", "--progress=plain", "--build-arg", - f"BASE_IMAGE={base_image}", + f"BASE_IMAGE={test.get_anyscale_base_byod_image()}", "--build-arg", - f"POST_BUILD_SCRIPT={post_build_script}", + f"POST_BUILD_SCRIPT={test.get_byod_post_build_script()}", "-t", - image, + byod_image, "-f", os.path.join(RELEASE_BYOD_DIR, "byod.custom.Dockerfile"), RELEASE_BYOD_DIR, @@ -57,7 +59,7 @@ def build_anyscale_custom_byod_image( stdout=sys.stderr, env=env, ) - _validate_and_push(image) + _validate_and_push(byod_image) def build_anyscale_base_byod_images(tests: List[Test]) -> None: diff --git a/release/ray_release/scripts/ray_bisect.py b/release/ray_release/scripts/ray_bisect.py index 257519d620e4..15ebd4c9d396 100644 --- a/release/ray_release/scripts/ray_bisect.py +++ b/release/ray_release/scripts/ray_bisect.py @@ -178,12 +178,7 @@ def _trigger_test_run( ) -> None: os.environ["COMMIT_TO_TEST"] = commit build_anyscale_base_byod_images([test]) - if test.require_custom_byod_image(): - build_anyscale_custom_byod_image( - test.get_anyscale_byod_image(), - test.get_anyscale_base_byod_image(), - test.get_byod_post_build_script(), - ) + build_anyscale_custom_byod_image(test) for run in range(run_per_commit): step = get_step( copy.deepcopy(test), # avoid mutating the original test diff --git a/release/ray_release/tests/test_byod_build.py b/release/ray_release/tests/test_byod_build.py index 920b2c826e36..7da9d5449a69 100644 --- a/release/ray_release/tests/test_byod_build.py +++ b/release/ray_release/tests/test_byod_build.py @@ -68,11 +68,7 @@ def _mock_check_call( name="name", cluster={"byod": {"post_build_script": "foo.sh"}}, ) - build_anyscale_custom_byod_image( - test.get_anyscale_byod_image(), - test.get_anyscale_base_byod_image(), - test.get_byod_post_build_script(), - ) + build_anyscale_custom_byod_image(test) assert "docker build --build-arg BASE_IMAGE=029272617770.dkr.ecr.us-west-2." "amazonaws.com/anyscale/ray:abc123-py37 -t 029272617770.dkr.ecr.us-west-2." "amazonaws.com/anyscale/ray:abc123-py37-c3fc5fc6d84cea4d7ab885c6cdc966542e" From 9b41002c87916c9dfee05b54a8e6294767299e57 Mon Sep 17 00:00:00 2001 From: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Date: Sat, 9 Aug 2025 12:39:00 -0700 Subject: [PATCH 005/634] [Train] Have user faced APIs use TrainFnUtils (#55226) There are some user-faced APIs calling internal TrainContext. We add one `TrainFnUtils` as the layer between user facing APIs and public context and internal context. This makes patching the user facing APIs behavior much easier. --------- Signed-off-by: xgui --- python/ray/train/collective/collectives.py | 16 ++-- .../train/v2/_internal/execution/context.py | 12 ++- .../v2/_internal/execution/train_fn_utils.py | 81 +++++++++++++++++++ .../execution/worker_group/worker.py | 7 ++ python/ray/train/v2/api/train_fn_utils.py | 10 +-- python/ray/train/v2/tests/test_persistence.py | 13 +-- 6 files changed, 122 insertions(+), 17 deletions(-) create mode 100644 python/ray/train/v2/_internal/execution/train_fn_utils.py diff --git a/python/ray/train/collective/collectives.py b/python/ray/train/collective/collectives.py index c35c43564bbe..3b06fc369e32 100644 --- a/python/ray/train/collective/collectives.py +++ b/python/ray/train/collective/collectives.py @@ -3,7 +3,10 @@ import ray import ray.cloudpickle as pickle -from ray.train.v2._internal.execution.context import get_train_context +from ray.train.v2._internal.execution.context import ( + get_train_context as get_internal_train_context, +) +from ray.train.v2._internal.execution.train_fn_utils import get_train_fn_utils from ray.util.annotations import PublicAPI # For reference, {1:1} is 19 bytes, {"1":"1"} is 21 bytes, @@ -66,8 +69,11 @@ def train_func(): ) # Send data to all workers. - train_context = get_train_context() - sync_actor = train_context.get_synchronization_actor() + # TODO (xgui): We should not expose get_synchronization_actor() from internal_context here. + # Maybe create one public barrier API inside `TrainFnUtils` + sync_actor = get_internal_train_context().get_synchronization_actor() + train_context = get_train_fn_utils().get_context() + return ray.get( sync_actor.broadcast_from_rank_zero.remote( world_rank=train_context.get_world_rank(), @@ -103,8 +109,8 @@ def train_func(): trainer = TorchTrainer(train_func) trainer.fit() """ - train_context = get_train_context() - sync_actor = train_context.get_synchronization_actor() + train_context = get_train_fn_utils().get_context() + sync_actor = get_internal_train_context().get_synchronization_actor() return ray.get( sync_actor.broadcast_from_rank_zero.remote( world_rank=train_context.get_world_rank(), diff --git a/python/ray/train/v2/_internal/execution/context.py b/python/ray/train/v2/_internal/execution/context.py index cf76ec4f8484..fd2f0df8e23e 100644 --- a/python/ray/train/v2/_internal/execution/context.py +++ b/python/ray/train/v2/_internal/execution/context.py @@ -216,7 +216,7 @@ def report( metrics: Dict[str, Any], checkpoint: Optional[Checkpoint] = None, checkpoint_dir_name: Optional[str] = None, - ): + ) -> None: """ Upload checkpoint to remote storage and put a training result on the result queue of this worker process. @@ -277,6 +277,16 @@ def report( def get_train_context() -> TrainContext: + """Get the internal train context. + + Note: + This should not be used directly by user-facing APIs. User-facing APIs should + call :class:`~ray.train.v2._internal.execution.train_fn_utils.TrainFnUtils` + or use :class:`~ray.train.v2.api.context.TrainContext` instead. + + Returns: + The internal TrainContext for this worker. + """ with _context_lock: if _train_context is None: raise RuntimeError("TrainContext has not been initialized.") diff --git a/python/ray/train/v2/_internal/execution/train_fn_utils.py b/python/ray/train/v2/_internal/execution/train_fn_utils.py new file mode 100644 index 000000000000..28bf683fda2d --- /dev/null +++ b/python/ray/train/v2/_internal/execution/train_fn_utils.py @@ -0,0 +1,81 @@ +import threading +from typing import Any, Dict, Optional + +from ray.data import DataIterator +from ray.train import Checkpoint +from ray.train.v2._internal.execution.context import ( + get_train_context as get_internal_train_context, +) +from ray.train.v2.api.context import TrainContext as ExternalTrainContext + + +class TrainFnUtils: + """Utility class providing an abstraction layer between user-facing APIs + and :class:`~ray.train.v2._internal.execution.context.TrainContext`. + + It should be set before the users' training function is called, like training workers initialization. + This class can be patched if new user APIs behaviors is wanted. + """ + + def report( + self, + metrics: Dict[str, Any], + checkpoint: Optional[Checkpoint] = None, + checkpoint_dir_name: Optional[str] = None, + ) -> None: + """Upload checkpoint to remote storage and put a training result on the result queue. + + Args: + metrics: The metrics to report. + checkpoint: The checkpoint to report. + checkpoint_dir_name: The name of the checkpoint dir + in this iteration. Note: If not set, the checkpoint will + be stored in the default storage path. If set, make sure + this value is unique for each iteration. + """ + return get_internal_train_context().report( + metrics, checkpoint, checkpoint_dir_name + ) + + def get_checkpoint(self): + """Get the latest checkpoint to resume training from. + + Returns: + The latest checkpoint if available, None otherwise. + """ + return get_internal_train_context().get_checkpoint() + + def get_dataset_shard(self, dataset_name: str) -> DataIterator: + """Get the dataset shard for this worker. + + This method is used by the public API function :func:`ray.train.get_dataset_shard`. + Users should typically call ``ray.train.get_dataset_shard()`` instead of calling this method directly. + + Args: + dataset_name: The name of the dataset to get the shard for. + + Returns: + The DataIterator shard for this worker. + """ + return get_internal_train_context().get_dataset_shard(dataset_name) + + def get_context(self) -> ExternalTrainContext: + return ExternalTrainContext() + + +_train_fn_utils: Optional[TrainFnUtils] = None +_train_fn_utils_lock = threading.Lock() + + +def get_train_fn_utils() -> TrainFnUtils: + global _train_fn_utils + with _train_fn_utils_lock: + if _train_fn_utils is None: + raise RuntimeError("TrainFnUtils has not been initialized.") + return _train_fn_utils + + +def set_train_fn_utils(train_fn_utils) -> None: + global _train_fn_utils + with _train_fn_utils_lock: + _train_fn_utils = train_fn_utils diff --git a/python/ray/train/v2/_internal/execution/worker_group/worker.py b/python/ray/train/v2/_internal/execution/worker_group/worker.py index 1a3ea2e7a554..667ab318296b 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/worker.py +++ b/python/ray/train/v2/_internal/execution/worker_group/worker.py @@ -30,6 +30,10 @@ set_train_context, ) from ray.train.v2._internal.execution.storage import StorageContext +from ray.train.v2._internal.execution.train_fn_utils import ( + TrainFnUtils, + set_train_fn_utils, +) from ray.train.v2._internal.execution.worker_group.poll import WorkerStatus from ray.train.v2._internal.logging.logging import LoggingManager from ray.train.v2._internal.logging.patch_print import patch_print_function @@ -219,5 +223,8 @@ def init_train_context( # Set the train context global variable for the worker. set_train_context(context) + # user facing train fn utils + set_train_fn_utils(TrainFnUtils()) + for callback in self._callbacks: callback.after_init_train_context() diff --git a/python/ray/train/v2/api/train_fn_utils.py b/python/ray/train/v2/api/train_fn_utils.py index 6266f9441b42..efd3e6575ef4 100644 --- a/python/ray/train/v2/api/train_fn_utils.py +++ b/python/ray/train/v2/api/train_fn_utils.py @@ -1,7 +1,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional from ray.train import Checkpoint -from ray.train.v2._internal.execution.context import get_train_context +from ray.train.v2._internal.execution.train_fn_utils import get_train_fn_utils from ray.train.v2.api.context import TrainContext from ray.util.annotations import PublicAPI @@ -88,7 +88,7 @@ def train_func(config): index in the name. """ - get_train_context().report( + get_train_fn_utils().report( metrics=metrics, checkpoint=checkpoint, checkpoint_dir_name=checkpoint_dir_name ) @@ -103,7 +103,7 @@ def get_context() -> TrainContext: """ # TODO: Return a dummy train context on the controller and driver process # instead of raising an exception if the train context does not exist. - return TrainContext() + return get_train_fn_utils().get_context() @PublicAPI(stability="stable") @@ -148,7 +148,7 @@ def train_func(config): Checkpoint object if the session is currently being resumed. Otherwise, return None. """ - return get_train_context().get_checkpoint() + return get_train_fn_utils().get_checkpoint() @PublicAPI(stability="stable") @@ -195,4 +195,4 @@ def train_loop_per_worker(config): The ``DataIterator`` shard to use for this worker. If no dataset is passed into Trainer, then return None. """ - return get_train_context().get_dataset_shard(dataset_name) + return get_train_fn_utils().get_dataset_shard(dataset_name) diff --git a/python/ray/train/v2/tests/test_persistence.py b/python/ray/train/v2/tests/test_persistence.py index cc231722a651..84468f1ea57a 100644 --- a/python/ray/train/v2/tests/test_persistence.py +++ b/python/ray/train/v2/tests/test_persistence.py @@ -24,8 +24,11 @@ ScalingConfig, ) from ray.train.v2._internal.constants import HEALTH_CHECK_INTERVAL_S_ENV_VAR -from ray.train.v2._internal.execution.context import get_train_context +from ray.train.v2._internal.execution.context import ( + get_train_context as get_internal_train_context, +) from ray.train.v2._internal.execution.storage import _download_from_fs_path +from ray.train.v2._internal.execution.train_fn_utils import get_train_fn_utils from ray.train.v2.api.data_parallel_trainer import DataParallelTrainer @@ -212,11 +215,9 @@ def train_fn(config): # which will cause the test assertions to fail. # This should be fixed by forcing a queue flush on all workers before # executing the failure decisions. - # Note: this `get_train_context` is not a public API. - # TODO (hpguo): Think about expose `get_synchronization_actor` as a - # public API, which will be a useful collection of communication utils. - train_context = get_train_context() - sync_actor = train_context.get_synchronization_actor() + sync_actor = get_internal_train_context().get_synchronization_actor() + train_context = get_train_fn_utils().get_context() + ray.get( sync_actor.broadcast_from_rank_zero.remote( world_rank=train_context.get_world_rank(), From 93f765d04ee055ebbd8437825ca197ab54bace4a Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Sat, 9 Aug 2025 13:33:39 -0700 Subject: [PATCH 006/634] Revert "[Data] Deprecate `with_columns` API in favor of `with_column`" (#55451) Reverts ray-project/ray#55322 --- python/ray/data/dataset.py | 29 ++++++------- python/ray/data/expressions.py | 8 ++-- python/ray/data/tests/test_map.py | 71 ++++++++++++++++--------------- 3 files changed, 55 insertions(+), 53 deletions(-) diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 8766434c425a..c2a3f0184854 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -783,31 +783,31 @@ def _map_batches_without_batch_size_validation( return Dataset(plan, logical_plan) @PublicAPI(api_group=EXPRESSION_API_GROUP, stability="alpha") - def with_column(self, column_name: str, expr: Expr, **ray_remote_args) -> "Dataset": + def with_columns(self, exprs: Dict[str, Expr]) -> "Dataset": """ - Add a new column to the dataset via an expression. + Add new columns to the dataset. Examples: >>> import ray >>> from ray.data.expressions import col >>> ds = ray.data.range(100) - >>> ds.with_column("id_2", (col("id") * 2)).schema() - Column Type - ------ ---- - id int64 - id_2 int64 + >>> ds.with_columns({"new_id": col("id") * 2, "new_id_2": col("id") * 3}).schema() + Column Type + ------ ---- + id int64 + new_id int64 + new_id_2 int64 Args: - column_name: The name of the new column. - expr: An expression that defines the new column values. - **ray_remote_args: Additional resource requirements to request from - Ray (e.g., num_gpus=1 to request GPUs for the map tasks). See - :func:`ray.remote` for details. + exprs: A dictionary mapping column names to expressions that define the new column values. Returns: - A new dataset with the added column evaluated via the expression. + A new dataset with the added columns evaluated via expressions. """ + if not exprs: + raise ValueError("at least one expression is required") + from ray.data._internal.logical.operators.map_operator import Project plan = self._plan.copy() @@ -815,8 +815,7 @@ def with_column(self, column_name: str, expr: Expr, **ray_remote_args) -> "Datas self._logical_plan.dag, cols=None, cols_rename=None, - exprs={column_name: expr}, - ray_remote_args=ray_remote_args, + exprs=exprs, ) logical_plan = LogicalPlan(project_op, self.context) return Dataset(plan, logical_plan) diff --git a/python/ray/data/expressions.py b/python/ray/data/expressions.py index 3ba8f48356da..cf59aa30b5e0 100644 --- a/python/ray/data/expressions.py +++ b/python/ray/data/expressions.py @@ -259,10 +259,10 @@ def col(name: str) -> ColumnExpr: >>> # Reference columns in an expression >>> expr = col("price") * col("quantity") >>> - >>> # Use with Dataset.with_column() + >>> # Use with Dataset.with_columns() >>> import ray >>> ds = ray.data.from_items([{"price": 10, "quantity": 2}]) - >>> ds = ds.with_column("total", col("price") * col("quantity")) + >>> ds = ds.with_columns({"total": col("price") * col("quantity")}) """ return ColumnExpr(name) @@ -293,10 +293,10 @@ def lit(value: Any) -> LiteralExpr: >>> # Use in expressions >>> expr = col("age") + lit(1) # Add 1 to age column >>> - >>> # Use with Dataset.with_column() + >>> # Use with Dataset.with_columns() >>> import ray >>> ds = ray.data.from_items([{"age": 25}, {"age": 30}]) - >>> ds = ds.with_column("age_plus_one", col("age") + lit(1)) + >>> ds = ds.with_columns({"age_plus_one": col("age") + lit(1)}) """ return LiteralExpr(value) diff --git a/python/ray/data/tests/test_map.py b/python/ray/data/tests/test_map.py index 0bfb35f68f17..6c3f9c6d4a37 100644 --- a/python/ray/data/tests/test_map.py +++ b/python/ray/data/tests/test_map.py @@ -2286,53 +2286,52 @@ def func(x, y): @pytest.mark.skipif( get_pyarrow_version() < parse_version("20.0.0"), - reason="with_column requires PyArrow >= 20.0.0", + reason="with_columns requires PyArrow >= 20.0.0", ) @pytest.mark.parametrize( - "column_name, expr, expected_value", + "exprs, expected_value", [ # Arithmetic operations - ("result", col("id") + 1, 1), # 0 + 1 = 1 - ("result", col("id") + 5, 5), # 0 + 5 = 5 - ("result", col("id") - 1, -1), # 0 - 1 = -1 - ("result", col("id") * 2, 0), # 0 * 2 = 0 - ("result", col("id") * 3, 0), # 0 * 3 = 0 - ("result", col("id") / 2, 0.0), # 0 / 2 = 0.0 + ({"result": col("id") + 1}, 1), # 0 + 1 = 1 + ({"result": col("id") + 5}, 5), # 0 + 5 = 5 + ({"result": col("id") - 1}, -1), # 0 - 1 = -1 + ({"result": col("id") * 2}, 0), # 0 * 2 = 0 + ({"result": col("id") * 3}, 0), # 0 * 3 = 0 + ({"result": col("id") / 2}, 0.0), # 0 / 2 = 0.0 # More complex arithmetic - ("result", (col("id") + 1) * 2, 2), # (0 + 1) * 2 = 2 - ("result", (col("id") * 2) + 3, 3), # 0 * 2 + 3 = 3 + ({"result": (col("id") + 1) * 2}, 2), # (0 + 1) * 2 = 2 + ({"result": (col("id") * 2) + 3}, 3), # 0 * 2 + 3 = 3 # Comparison operations - ("result", col("id") > 0, False), # 0 > 0 = False - ("result", col("id") >= 0, True), # 0 >= 0 = True - ("result", col("id") < 1, True), # 0 < 1 = True - ("result", col("id") <= 0, True), # 0 <= 0 = True - ("result", col("id") == 0, True), # 0 == 0 = True + ({"result": col("id") > 0}, False), # 0 > 0 = False + ({"result": col("id") >= 0}, True), # 0 >= 0 = True + ({"result": col("id") < 1}, True), # 0 < 1 = True + ({"result": col("id") <= 0}, True), # 0 <= 0 = True + ({"result": col("id") == 0}, True), # 0 == 0 = True # Operations with literals - ("result", col("id") + lit(10), 10), # 0 + 10 = 10 - ("result", col("id") * lit(5), 0), # 0 * 5 = 0 - ("result", lit(2) + col("id"), 2), # 2 + 0 = 2 - ("result", lit(10) / (col("id") + 1), 10.0), # 10 / (0 + 1) = 10.0 + ({"result": col("id") + lit(10)}, 10), # 0 + 10 = 10 + ({"result": col("id") * lit(5)}, 0), # 0 * 5 = 0 + ({"result": lit(2) + col("id")}, 2), # 2 + 0 = 2 + ({"result": lit(10) / (col("id") + 1)}, 10.0), # 10 / (0 + 1) = 10.0 ], ) -def test_with_column( +def test_with_columns( ray_start_regular_shared, - column_name, - expr, + exprs, expected_value, target_max_block_size_infinite_or_default, ): - """Verify that `with_column` works with various operations.""" - ds = ray.data.range(5).with_column(column_name, expr) + """Verify that `with_columns` works with various operations.""" + ds = ray.data.range(5).with_columns(exprs) result = ds.take(1)[0] assert result["id"] == 0 - assert result[column_name] == expected_value + assert result["result"] == expected_value @pytest.mark.skipif( get_pyarrow_version() < parse_version("20.0.0"), - reason="with_column requires PyArrow >= 20.0.0", + reason="with_columns requires PyArrow >= 20.0.0", ) -def test_with_column_nonexistent_column( +def test_with_columns_nonexistent_column( ray_start_regular_shared, target_max_block_size_infinite_or_default ): """Verify that referencing a non-existent column with col() raises an exception.""" @@ -2341,22 +2340,26 @@ def test_with_column_nonexistent_column( # Try to reference a non-existent column - this should raise an exception with pytest.raises(UserCodeException): - ds.with_column("result", col("nonexistent_column") + 1).materialize() + ds.with_columns({"result": col("nonexistent_column") + 1}).materialize() @pytest.mark.skipif( get_pyarrow_version() < parse_version("20.0.0"), - reason="with_column requires PyArrow >= 20.0.0", + reason="with_columns requires PyArrow >= 20.0.0", ) -def test_with_column_multiple_expressions( +def test_with_columns_multiple_expressions( ray_start_regular_shared, target_max_block_size_infinite_or_default ): - """Verify that `with_column` correctly handles multiple expressions at once.""" + """Verify that `with_columns` correctly handles multiple expressions at once.""" ds = ray.data.range(5) - ds = ds.with_column("plus_one", col("id") + 1) - ds = ds.with_column("times_two", col("id") * 2) - ds = ds.with_column("ten_minus_id", 10 - col("id")) + exprs = { + "plus_one": col("id") + 1, + "times_two": col("id") * 2, + "ten_minus_id": 10 - col("id"), + } + + ds = ds.with_columns(exprs) first_row = ds.take(1)[0] assert first_row["id"] == 0 From 0703eca08b16f5ee3a6109894d86618315bae432 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Sat, 9 Aug 2025 13:38:13 -0700 Subject: [PATCH 007/634] [core] shorten more on the test name (#55450) the name was still not sort enough Signed-off-by: Lonnie Liu --- src/ray/core_worker/task_execution/test/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ray/core_worker/task_execution/test/BUILD.bazel b/src/ray/core_worker/task_execution/test/BUILD.bazel index 731ce60abcdb..261846d87063 100644 --- a/src/ray/core_worker/task_execution/test/BUILD.bazel +++ b/src/ray/core_worker/task_execution/test/BUILD.bazel @@ -25,7 +25,7 @@ ray_cc_test( # Test name is shortened for running on Windows. ray_cc_test( - name = "concurrency_grp_mgr_test", + name = "concrncy_grp_mgr_test", srcs = ["concurrency_group_manager_test.cc"], tags = ["team:core"], deps = [ From 2ecd577e95e01ce80faad90d12b00ab2bed2e3ec Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Sat, 9 Aug 2025 15:31:13 -0700 Subject: [PATCH 008/634] [ci] raydepsets: enable isort (#55455) - enabling isort for raydepsets - sorting imports for raydepsets --------- Signed-off-by: elliot-barn --- ci/raydepsets/cli.py | 13 +++++++------ ci/raydepsets/test_cli.py | 28 ++++++++++++++-------------- ci/raydepsets/testing_utils.py | 1 + ci/raydepsets/workspace.py | 5 +++-- pyproject.toml | 7 ++++++- 5 files changed, 31 insertions(+), 23 deletions(-) diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index 41f4e6382685..6ecadca48917 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -1,13 +1,14 @@ -import click -from pathlib import Path -from ci.raydepsets.workspace import Workspace, Depset -from typing import List -import subprocess import platform +import subprocess +from pathlib import Path +from typing import List, Optional + +import click import runfiles -from typing import Optional from networkx import DiGraph, topological_sort +from ci.raydepsets.workspace import Depset, Workspace + DEFAULT_UV_FLAGS = """ --generate-hashes --strip-extras diff --git a/ci/raydepsets/test_cli.py b/ci/raydepsets/test_cli.py index b7cb3af62b8d..1db2887d31fe 100644 --- a/ci/raydepsets/test_cli.py +++ b/ci/raydepsets/test_cli.py @@ -1,34 +1,34 @@ -import pytest -import sys -from typing import Optional -from pathlib import Path -import subprocess import shutil +import subprocess +import sys import tempfile import unittest +from pathlib import Path +from typing import Optional +import pytest import runfiles +from click.testing import CliRunner from networkx import topological_sort from ci.raydepsets.cli import ( - load, + DEFAULT_UV_FLAGS, DependencySetManager, - _uv_binary, - _override_uv_flags, + Depset, _append_uv_flags, _flatten_flags, - Depset, - DEFAULT_UV_FLAGS, + _override_uv_flags, + _uv_binary, + load, ) -from ci.raydepsets.workspace import Workspace -from click.testing import CliRunner from ci.raydepsets.testing_utils import ( + append_to_file, copy_data_to_tmpdir, replace_in_file, - save_packages_to_file, save_file_as, - append_to_file, + save_packages_to_file, ) +from ci.raydepsets.workspace import Workspace _REPO_NAME = "com_github_ray_project_ray" _runfiles = runfiles.Create() diff --git a/ci/raydepsets/testing_utils.py b/ci/raydepsets/testing_utils.py index 13d6d9b373a0..ee6040df5e41 100644 --- a/ci/raydepsets/testing_utils.py +++ b/ci/raydepsets/testing_utils.py @@ -1,6 +1,7 @@ """Shared test utilities for raydepsets tests.""" import shutil + import runfiles _REPO_NAME = "com_github_ray_project_ray" diff --git a/ci/raydepsets/workspace.py b/ci/raydepsets/workspace.py index ed3f8652ca40..647b2127e1ca 100644 --- a/ci/raydepsets/workspace.py +++ b/ci/raydepsets/workspace.py @@ -1,7 +1,8 @@ -import yaml +import os from dataclasses import dataclass, field from typing import List, Optional -import os + +import yaml @dataclass diff --git a/pyproject.toml b/pyproject.toml index a8287a17047f..8ae06c326372 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,12 @@ afterray = ["psutil", "setproctitle"] "python/ray/setup-dev.py" = ["I"] "python/ray/cloudpickle/*" = ["I"] "python/ray/dag/*.py" = ["I"] -"ci/*" = ["I"] +"ci/lint/*" = ["I"] +"ci/pipeline/*" = ["I"] +"ci/ray_ci/*" = ["I"] +"ci/build/*" = ["I"] +"ci/env/*" = ["I"] +"ci/repro-ci.py" = ["I"] "python/ray/includes/*" = ["I"] "python/ray/internal/*" = ["I"] "python/ray/ray_operator/*" = ["I"] From 97c0e24f553b9882e80464959785e55bbe095862 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Sat, 9 Aug 2025 16:26:16 -0700 Subject: [PATCH 009/634] [core] give `should_fetch_cluster_id_` an init value (#55456) fixes core worker's ubsan test, makes sure it is a bool value. `CoreWorkOptions` contains a `GcsClientOptions` which is initialized with default constructor. Signed-off-by: Lonnie Liu --- src/ray/gcs/gcs_client/gcs_client.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ray/gcs/gcs_client/gcs_client.h b/src/ray/gcs/gcs_client/gcs_client.h index eb088f3f7564..3c339bc8ff50 100644 --- a/src/ray/gcs/gcs_client/gcs_client.h +++ b/src/ray/gcs/gcs_client/gcs_client.h @@ -87,7 +87,7 @@ class GcsClientOptions { std::string gcs_address_; int gcs_port_ = 0; ClusterID cluster_id_; - bool should_fetch_cluster_id_; + bool should_fetch_cluster_id_ = false; }; /// \class GcsClient From 96036f7b31a55f0948925fea215d57ba1d69adc7 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Sun, 10 Aug 2025 05:47:09 -0400 Subject: [PATCH 010/634] [core] Congratulate the user for a ray check failure and ask them to file an issue (#55276) Signed-off-by: dayshah Signed-off-by: Dhyey Shah Co-authored-by: Ibrahim Rabbani Co-authored-by: Edward Oakes --- src/ray/util/logging.h | 13 ++++++++----- src/ray/util/tests/event_test.cc | 8 ++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/ray/util/logging.h b/src/ray/util/logging.h index 4a010ae9b8e1..040a609f9722 100644 --- a/src/ray/util/logging.h +++ b/src/ray/util/logging.h @@ -139,11 +139,14 @@ enum class RayLogLevel { #define RAY_IGNORE_EXPR(expr) ((void)(expr)) -#define RAY_CHECK_WITH_DISPLAY(condition, display) \ - RAY_PREDICT_TRUE((condition)) \ - ? RAY_IGNORE_EXPR(0) \ - : ::ray::Voidify() & (::ray::RayLog(__FILE__, __LINE__, ray::RayLogLevel::FATAL) \ - << " Check failed: " display " ") +#define RAY_CHECK_WITH_DISPLAY(condition, display) \ + RAY_PREDICT_TRUE((condition)) \ + ? RAY_IGNORE_EXPR(0) \ + : ::ray::Voidify() & (::ray::RayLog(__FILE__, __LINE__, ray::RayLogLevel::FATAL) \ + << " An unexpected system state has occurred. You have likely " \ + "discovered a bug in Ray. Please report this issue at " \ + "https://github.com/ray-project/ray/issues and we'll work " \ + "with you to fix it. Check failed: " display " ") #define RAY_CHECK(condition) RAY_CHECK_WITH_DISPLAY(condition, #condition) diff --git a/src/ray/util/tests/event_test.cc b/src/ray/util/tests/event_test.cc index 60821bcbc699..6f0b00320a7c 100644 --- a/src/ray/util/tests/event_test.cc +++ b/src/ray/util/tests/event_test.cc @@ -594,8 +594,12 @@ TEST_F(EventTest, TestRayCheckAbort) { "FATAL", "RAY_FATAL_CHECK_FAILED", "NULL"); - EXPECT_THAT(ele_1.message(), - testing::HasSubstr("Check failed: 1 < 0 incorrect test case")); + EXPECT_THAT( + ele_1.message(), + testing::HasSubstr( + "An unexpected system state has occurred. You have likely discovered a bug in " + "Ray. Please report this issue at https://github.com/ray-project/ray/issues " + "and we'll work with you to fix it. Check failed: 1 < 0 incorrect test case")); EXPECT_THAT(ele_1.message(), testing::HasSubstr("*** StackTrace Information ***")); EXPECT_THAT(ele_1.message(), testing::HasSubstr("ray::RayLog::~RayLog()")); } From 46e0066fa862513a3bc1f8879f3e5720dfdff875 Mon Sep 17 00:00:00 2001 From: harshit-anyscale Date: Sun, 10 Aug 2025 19:03:15 +0530 Subject: [PATCH 011/634] add _common directory to the packages to link (#55351) - add the `_common` dierctory to work with the dev-setup --------- Signed-off-by: harshit --- python/ray/setup-dev.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ray/setup-dev.py b/python/ray/setup-dev.py index 2585da4e4f81..372ee95912ef 100755 --- a/python/ray/setup-dev.py +++ b/python/ray/setup-dev.py @@ -165,6 +165,7 @@ def do_link(package, force=False, skip_list=None, allow_list=None, local_path=No "widgets": None, "cluster_utils.py": None, "_private": None, + "_common": None, "dashboard": None, } From 8985f0386705113d2d2438980aca060074e7ef86 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Sun, 10 Aug 2025 16:50:38 -0700 Subject: [PATCH 012/634] [ci] isorting ci/* (#55465) - removing ci directories from the isort exclude list - ran isort on the ci directory Signed-off-by: elliot-barn --- ci/build/get_build_info.py | 2 +- ci/env/check_minimal_install.py | 2 +- ci/env/setup_credentials.py | 3 +- ci/lint/check_cpp_files_inclusion.py | 2 +- ci/pipeline/determine_tests_to_run.py | 3 +- ci/pipeline/test_conditional_testing.py | 2 +- ci/ray_ci/anyscale_docker_container.py | 2 +- .../determine_microcheck_step_ids.py | 9 +++-- .../automation/determine_microcheck_tests.py | 9 +++-- ci/ray_ci/automation/docker_tags_lib.py | 22 +++++------ ci/ray_ci/automation/filter_tests.py | 3 +- ci/ray_ci/automation/generate_index.py | 6 +-- ci/ray_ci/automation/get_contributors.py | 2 +- ci/ray_ci/automation/list_docker_tags.py | 7 ++-- ci/ray_ci/automation/pypi_lib.py | 4 +- ci/ray_ci/automation/ray_wheels_lib.py | 5 ++- ci/ray_ci/automation/test_db_bot.py | 4 +- .../test_determine_microcheck_tests.py | 8 ++-- ci/ray_ci/automation/test_docker_tags_lib.py | 37 ++++++++++--------- ci/ray_ci/automation/test_pypi_lib.py | 13 ++++--- ci/ray_ci/automation/test_ray_wheels_lib.py | 19 +++++----- .../automation/test_update_version_lib.py | 6 +-- ci/ray_ci/automation/update_version.py | 3 +- ci/ray_ci/automation/upload_wheels_pypi.py | 8 ++-- ci/ray_ci/automation/weekly_green_metric.py | 5 +-- ci/ray_ci/bazel_sharding.py | 6 +-- ci/ray_ci/bisect/bisect_test.py | 10 ++--- ci/ray_ci/bisect/bisector.py | 5 ++- ci/ray_ci/bisect/generic_validator.py | 6 +-- ci/ray_ci/bisect/macos_validator.py | 2 +- ci/ray_ci/bisect/test_bisector.py | 7 ++-- ci/ray_ci/bisect/test_generic_validator.py | 6 +-- ci/ray_ci/builder.py | 12 +++--- ci/ray_ci/container.py | 6 +-- ci/ray_ci/doc/api.py | 8 ++-- ci/ray_ci/doc/autodoc.py | 5 +-- ci/ray_ci/doc/build_cache.py | 7 ++-- ci/ray_ci/doc/cmd_build.py | 7 ++-- ci/ray_ci/doc/cmd_check_api_discrepancy.py | 4 +- ci/ray_ci/doc/mock/__init__.py | 3 +- ci/ray_ci/doc/test_api.py | 5 ++- ci/ray_ci/doc/test_autodoc.py | 5 ++- ci/ray_ci/doc/test_build_cache.py | 5 ++- ci/ray_ci/doc/test_module.py | 3 +- ci/ray_ci/doc/test_update_cache_env.py | 8 ++-- ci/ray_ci/docker_container.py | 5 +-- ci/ray_ci/linux_container.py | 2 +- ci/ray_ci/pipeline/gap_filling_scheduler.py | 3 +- ci/ray_ci/pipeline/scheduler.py | 6 +-- .../pipeline/test_gap_filling_scheduler.py | 2 +- ci/ray_ci/ray_docker_container.py | 7 ++-- ci/ray_ci/test_anyscale_docker_container.py | 4 +- ci/ray_ci/test_base.py | 2 +- ci/ray_ci/test_bazel_sharding.py | 5 ++- ci/ray_ci/test_builder_container.py | 5 ++- ci/ray_ci/test_linux_container.py | 1 + ci/ray_ci/test_linux_tester_container.py | 10 ++--- ci/ray_ci/test_privileged.py | 4 +- ci/ray_ci/test_ray_docker_container.py | 5 ++- ci/ray_ci/test_tester.py | 10 ++--- ci/ray_ci/test_utils.py | 7 ++-- ci/ray_ci/test_windows_container.py | 7 ++-- ci/ray_ci/test_windows_tester_container.py | 2 +- ci/ray_ci/tester.py | 16 ++++---- ci/ray_ci/tester_container.py | 13 +++---- ci/ray_ci/utils.py | 10 ++--- ci/ray_ci/windows_builder_container.py | 2 +- ci/ray_ci/windows_container.py | 3 +- ci/ray_ci/windows_tester_container.py | 2 +- ci/repro-ci.py | 2 +- pyproject.toml | 6 --- 71 files changed, 227 insertions(+), 220 deletions(-) diff --git a/ci/build/get_build_info.py b/ci/build/get_build_info.py index b22918551788..ae0758382b26 100755 --- a/ci/build/get_build_info.py +++ b/ci/build/get_build_info.py @@ -10,9 +10,9 @@ } """ +import json import os import platform -import json def gha_get_self_url(): diff --git a/ci/env/check_minimal_install.py b/ci/env/check_minimal_install.py index 8bf4630ee210..c9ec2255aed6 100644 --- a/ci/env/check_minimal_install.py +++ b/ci/env/check_minimal_install.py @@ -8,9 +8,9 @@ It also ensures the correct Python version. """ -from typing import List import argparse import sys +from typing import List # These are taken from `setup.py` for ray[default] DEFAULT_BLACKLIST = [ diff --git a/ci/env/setup_credentials.py b/ci/env/setup_credentials.py index 2f03b1c766a4..86a886cf75aa 100644 --- a/ci/env/setup_credentials.py +++ b/ci/env/setup_credentials.py @@ -7,10 +7,11 @@ export WANDB_API_KEY=abcd export COMET_API_KEY=efgh """ -import boto3 import json import sys +import boto3 + AWS_AIR_SECRETS_ARN = ( "arn:aws:secretsmanager:us-west-2:029272617770:secret:" "oss-ci/ray-air-test-secrets20221014164754935800000002-UONblX" diff --git a/ci/lint/check_cpp_files_inclusion.py b/ci/lint/check_cpp_files_inclusion.py index b1e4df83d19f..d849b0f765eb 100755 --- a/ci/lint/check_cpp_files_inclusion.py +++ b/ci/lint/check_cpp_files_inclusion.py @@ -2,8 +2,8 @@ """This script checks whether header file inclusion for ray core C++ code is correct. """ -import sys import re +import sys def check_ray_core_inclusion(fname: str): diff --git a/ci/pipeline/determine_tests_to_run.py b/ci/pipeline/determine_tests_to_run.py index 2b7e394345f5..61939c5adacb 100644 --- a/ci/pipeline/determine_tests_to_run.py +++ b/ci/pipeline/determine_tests_to_run.py @@ -5,9 +5,8 @@ import os import subprocess import sys -from typing import List, Optional, Set, Tuple from pprint import pformat - +from typing import List, Optional, Set, Tuple _ALL_TAGS = set( """ diff --git a/ci/pipeline/test_conditional_testing.py b/ci/pipeline/test_conditional_testing.py index 0e8dd6ba8420..b7b577dfa59a 100644 --- a/ci/pipeline/test_conditional_testing.py +++ b/ci/pipeline/test_conditional_testing.py @@ -4,8 +4,8 @@ import tempfile from typing import List, Set -import runfiles import pytest +import runfiles import yaml from ci.pipeline.determine_tests_to_run import TagRule, TagRuleSet diff --git a/ci/ray_ci/anyscale_docker_container.py b/ci/ray_ci/anyscale_docker_container.py index b2b5aa1bd169..4025951268b7 100644 --- a/ci/ray_ci/anyscale_docker_container.py +++ b/ci/ray_ci/anyscale_docker_container.py @@ -1,5 +1,5 @@ -from ci.ray_ci.docker_container import DockerContainer from ci.ray_ci.container import _DOCKER_ECR_REPO, _DOCKER_GCP_REGISTRY +from ci.ray_ci.docker_container import DockerContainer class AnyscaleDockerContainer(DockerContainer): diff --git a/ci/ray_ci/automation/determine_microcheck_step_ids.py b/ci/ray_ci/automation/determine_microcheck_step_ids.py index 4d15da5f8e6e..097afee56aca 100644 --- a/ci/ray_ci/automation/determine_microcheck_step_ids.py +++ b/ci/ray_ci/automation/determine_microcheck_step_ids.py @@ -1,14 +1,15 @@ -import click import os -from ci.ray_ci.utils import ci_init +import click from ray_release.test import ( - Test, LINUX_TEST_PREFIX, - WINDOWS_TEST_PREFIX, MACOS_TEST_PREFIX, + WINDOWS_TEST_PREFIX, + Test, ) +from ci.ray_ci.utils import ci_init + BAZEL_WORKSPACE_DIR = os.environ.get("BUILD_WORKSPACE_DIRECTORY", "") diff --git a/ci/ray_ci/automation/determine_microcheck_tests.py b/ci/ray_ci/automation/determine_microcheck_tests.py index ba40521860bb..67f914d4dff4 100644 --- a/ci/ray_ci/automation/determine_microcheck_tests.py +++ b/ci/ray_ci/automation/determine_microcheck_tests.py @@ -1,12 +1,13 @@ -import click -from typing import List, Set, Dict +from typing import Dict, List, Set -from ci.ray_ci.utils import logger, ci_init +import click from ray_release.configs.global_config import get_global_config -from ray_release.test import Test from ray_release.result import ResultStatus +from ray_release.test import Test from ray_release.test_automation.ci_state_machine import CITestStateMachine +from ci.ray_ci.utils import ci_init, logger + # The s3 prefix for the tests that run on Linux. It comes from the bazel prefix rule # linux:// with the character "/" replaced by "_" for s3 compatibility LINUX_TEST_PREFIX = "linux:__" diff --git a/ci/ray_ci/automation/docker_tags_lib.py b/ci/ray_ci/automation/docker_tags_lib.py index 99d3b1f3f0a0..6e780b230c4c 100644 --- a/ci/ray_ci/automation/docker_tags_lib.py +++ b/ci/ray_ci/automation/docker_tags_lib.py @@ -1,25 +1,25 @@ -import subprocess -import re -from datetime import datetime -from typing import List, Optional, Callable, Tuple import os -import sys -from dateutil import parser import platform +import re +import subprocess +import sys +from datetime import datetime +from typing import Callable, List, Optional, Tuple -import docker import requests import runfiles +from dateutil import parser +import docker from ci.ray_ci.builder_container import DEFAULT_ARCHITECTURE, DEFAULT_PYTHON_VERSION from ci.ray_ci.docker_container import ( + ARCHITECTURES_RAY, + ARCHITECTURES_RAY_ML, GPU_PLATFORM, - PYTHON_VERSIONS_RAY, - PYTHON_VERSIONS_RAY_ML, PLATFORMS_RAY, PLATFORMS_RAY_ML, - ARCHITECTURES_RAY, - ARCHITECTURES_RAY_ML, + PYTHON_VERSIONS_RAY, + PYTHON_VERSIONS_RAY_ML, RayType, ) from ci.ray_ci.utils import logger diff --git a/ci/ray_ci/automation/filter_tests.py b/ci/ray_ci/automation/filter_tests.py index 2444902515ab..84c053498e89 100644 --- a/ci/ray_ci/automation/filter_tests.py +++ b/ci/ray_ci/automation/filter_tests.py @@ -1,7 +1,8 @@ import sys + import click -from ci.ray_ci.utils import filter_tests, ci_init +from ci.ray_ci.utils import ci_init, filter_tests @click.command() diff --git a/ci/ray_ci/automation/generate_index.py b/ci/ray_ci/automation/generate_index.py index 1cea2c5901ff..ad6844b62bec 100644 --- a/ci/ray_ci/automation/generate_index.py +++ b/ci/ray_ci/automation/generate_index.py @@ -1,11 +1,11 @@ import click -from ci.ray_ci.automation.docker_tags_lib import list_image_tags, generate_index +from ci.ray_ci.automation.docker_tags_lib import generate_index, list_image_tags from ci.ray_ci.docker_container import ( - RayType, + ARCHITECTURES_RAY, PLATFORMS_RAY, PYTHON_VERSIONS_RAY, - ARCHITECTURES_RAY, + RayType, ) diff --git a/ci/ray_ci/automation/get_contributors.py b/ci/ray_ci/automation/get_contributors.py index 5f78ac0b47ab..f2c251a1b834 100644 --- a/ci/ray_ci/automation/get_contributors.py +++ b/ci/ray_ci/automation/get_contributors.py @@ -1,7 +1,7 @@ import os import sys -from subprocess import check_output from collections import defaultdict +from subprocess import check_output import click from github import Github diff --git a/ci/ray_ci/automation/list_docker_tags.py b/ci/ray_ci/automation/list_docker_tags.py index 1876a18f4aa6..8a95dc84b223 100644 --- a/ci/ray_ci/automation/list_docker_tags.py +++ b/ci/ray_ci/automation/list_docker_tags.py @@ -1,14 +1,15 @@ -import click import sys +import click + from ci.ray_ci.automation.docker_tags_lib import list_image_tags from ci.ray_ci.docker_container import ( + ARCHITECTURES_RAY, + ARCHITECTURES_RAY_ML, PLATFORMS_RAY, PLATFORMS_RAY_ML, PYTHON_VERSIONS_RAY, PYTHON_VERSIONS_RAY_ML, - ARCHITECTURES_RAY, - ARCHITECTURES_RAY_ML, RayType, ) diff --git a/ci/ray_ci/automation/pypi_lib.py b/ci/ray_ci/automation/pypi_lib.py index 31aeea2aae66..df60d8759928 100644 --- a/ci/ray_ci/automation/pypi_lib.py +++ b/ci/ray_ci/automation/pypi_lib.py @@ -1,7 +1,7 @@ -import subprocess import os -from typing import List +import subprocess import sys +from typing import List from ray_release.aws import get_secret_token diff --git a/ci/ray_ci/automation/ray_wheels_lib.py b/ci/ray_ci/automation/ray_wheels_lib.py index 2e16002a105f..aa46ac4f7769 100644 --- a/ci/ray_ci/automation/ray_wheels_lib.py +++ b/ci/ray_ci/automation/ray_wheels_lib.py @@ -1,6 +1,7 @@ -import boto3 -from typing import List import os +from typing import List + +import boto3 from ci.ray_ci.utils import logger diff --git a/ci/ray_ci/automation/test_db_bot.py b/ci/ray_ci/automation/test_db_bot.py index 186d706a739d..5d562309ac63 100644 --- a/ci/ray_ci/automation/test_db_bot.py +++ b/ci/ray_ci/automation/test_db_bot.py @@ -1,10 +1,10 @@ import os import click +from ray_release.configs.global_config import get_global_config -from ci.ray_ci.utils import logger, ci_init from ci.ray_ci.tester_container import TesterContainer -from ray_release.configs.global_config import get_global_config +from ci.ray_ci.utils import ci_init, logger @click.command() diff --git a/ci/ray_ci/automation/test_determine_microcheck_tests.py b/ci/ray_ci/automation/test_determine_microcheck_tests.py index 71f055591902..75083482ca3a 100644 --- a/ci/ray_ci/automation/test_determine_microcheck_tests.py +++ b/ci/ray_ci/automation/test_determine_microcheck_tests.py @@ -1,19 +1,19 @@ -import sys import json +import sys from typing import List import pytest +from ray_release.result import ResultStatus +from ray_release.test import Test, TestResult from ci.ray_ci.automation.determine_microcheck_tests import ( _get_failed_commits, + _get_failed_tests_from_master_branch, _get_flaky_tests, _get_test_with_minimal_coverage, - _get_failed_tests_from_master_branch, _update_high_impact_tests, ) from ci.ray_ci.utils import ci_init -from ray_release.result import ResultStatus -from ray_release.test import TestResult, Test ci_init() diff --git a/ci/ray_ci/automation/test_docker_tags_lib.py b/ci/ray_ci/automation/test_docker_tags_lib.py index e36b9fba1b18..fea87eca42ce 100644 --- a/ci/ray_ci/automation/test_docker_tags_lib.py +++ b/ci/ray_ci/automation/test_docker_tags_lib.py @@ -1,36 +1,37 @@ -from unittest import mock +import platform +import random +import shutil +import subprocess import sys +import tempfile +import threading +import time from datetime import datetime, timezone +from unittest import mock + import pytest import requests -import subprocess -import tempfile import runfiles -import platform -import time -import threading -import shutil -import random from ci.ray_ci.automation.docker_tags_lib import ( + AuthTokenException, + DockerHubRateLimitException, + RetrieveImageConfigException, _get_docker_auth_token, _get_docker_hub_auth_token, _get_image_creation_time, + _is_release_tag, + _list_recent_commit_short_shas, backup_release_tags, + call_crane_copy, + check_image_ray_commit, copy_tag_to_aws_ecr, delete_tag, - _list_recent_commit_short_shas, + generate_index, + get_ray_commit, + list_image_tags, query_tags_from_docker_hub, query_tags_from_docker_with_oci, - _is_release_tag, - list_image_tags, - get_ray_commit, - check_image_ray_commit, - generate_index, - AuthTokenException, - RetrieveImageConfigException, - DockerHubRateLimitException, - call_crane_copy, ) diff --git a/ci/ray_ci/automation/test_pypi_lib.py b/ci/ray_ci/automation/test_pypi_lib.py index 9cb23f27c3b4..17e787369024 100644 --- a/ci/ray_ci/automation/test_pypi_lib.py +++ b/ci/ray_ci/automation/test_pypi_lib.py @@ -1,14 +1,15 @@ -import pytest -from unittest import mock -import tempfile import os -import sys import subprocess +import sys +import tempfile +from unittest import mock + +import pytest from ci.ray_ci.automation.pypi_lib import ( - upload_wheels_to_pypi, - _get_pypi_url, _get_pypi_token, + _get_pypi_url, + upload_wheels_to_pypi, ) diff --git a/ci/ray_ci/automation/test_ray_wheels_lib.py b/ci/ray_ci/automation/test_ray_wheels_lib.py index aef36be327ae..0b8d46bcdf8d 100644 --- a/ci/ray_ci/automation/test_ray_wheels_lib.py +++ b/ci/ray_ci/automation/test_ray_wheels_lib.py @@ -1,20 +1,21 @@ -from unittest import mock +import os import sys import tempfile -import os -from botocore.exceptions import ClientError +from unittest import mock + import pytest +from botocore.exceptions import ClientError from ci.ray_ci.automation.ray_wheels_lib import ( - _get_wheel_names, - download_wheel_from_s3, - download_ray_wheels_from_s3, - _check_downloaded_wheels, - PYTHON_VERSIONS, ALL_PLATFORMS, + PYTHON_VERSIONS, RAY_TYPES, - add_build_tag_to_wheels, + _check_downloaded_wheels, + _get_wheel_names, add_build_tag_to_wheel, + add_build_tag_to_wheels, + download_ray_wheels_from_s3, + download_wheel_from_s3, ) SAMPLE_WHEELS = [ diff --git a/ci/ray_ci/automation/test_update_version_lib.py b/ci/ray_ci/automation/test_update_version_lib.py index cd43d3142fde..ca51369fd084 100644 --- a/ci/ray_ci/automation/test_update_version_lib.py +++ b/ci/ray_ci/automation/test_update_version_lib.py @@ -1,13 +1,13 @@ -from unittest import mock +import os import sys import tempfile -import os +from unittest import mock import pytest from ci.ray_ci.automation.update_version_lib import ( - list_java_files, get_current_version, + list_java_files, update_file_version, ) diff --git a/ci/ray_ci/automation/update_version.py b/ci/ray_ci/automation/update_version.py index eec6aed47bfe..221e49ca59e2 100644 --- a/ci/ray_ci/automation/update_version.py +++ b/ci/ray_ci/automation/update_version.py @@ -1,7 +1,8 @@ -import click import os from typing import Optional +import click + from ci.ray_ci.automation.update_version_lib import ( get_current_version, update_file_version, diff --git a/ci/ray_ci/automation/upload_wheels_pypi.py b/ci/ray_ci/automation/upload_wheels_pypi.py index 784f57a52453..48859cacb7d9 100644 --- a/ci/ray_ci/automation/upload_wheels_pypi.py +++ b/ci/ray_ci/automation/upload_wheels_pypi.py @@ -1,11 +1,13 @@ -import click import tempfile from typing import Optional + +import click + +from ci.ray_ci.automation.pypi_lib import upload_wheels_to_pypi from ci.ray_ci.automation.ray_wheels_lib import ( - download_ray_wheels_from_s3, add_build_tag_to_wheels, + download_ray_wheels_from_s3, ) -from ci.ray_ci.automation.pypi_lib import upload_wheels_to_pypi @click.command() diff --git a/ci/ray_ci/automation/weekly_green_metric.py b/ci/ray_ci/automation/weekly_green_metric.py index ab66bd893662..230a14691dea 100644 --- a/ci/ray_ci/automation/weekly_green_metric.py +++ b/ci/ray_ci/automation/weekly_green_metric.py @@ -1,14 +1,13 @@ import json -import time import sys +import time import boto3 import click - -from ci.ray_ci.utils import logger, ci_init from ray_release.test_automation.state_machine import TestStateMachine from ray_release.util import get_write_state_machine_aws_bucket +from ci.ray_ci.utils import ci_init, logger AWS_WEEKLY_GREEN_METRIC = "ray_weekly_green_metric" diff --git a/ci/ray_ci/bazel_sharding.py b/ci/ray_ci/bazel_sharding.py index 93be178fd5a9..d40683c45f6d 100644 --- a/ci/ray_ci/bazel_sharding.py +++ b/ci/ray_ci/bazel_sharding.py @@ -16,9 +16,6 @@ # BASED ON https://github.com/philwo/bazel-utils/blob/main/sharding/sharding.py -from collections import defaultdict -from dataclasses import dataclass -from typing import Iterable, List, Optional, Set, Tuple import argparse import os import re @@ -26,6 +23,9 @@ import subprocess import sys import xml.etree.ElementTree as ET +from collections import defaultdict +from dataclasses import dataclass +from typing import Iterable, List, Optional, Set, Tuple @dataclass diff --git a/ci/ray_ci/bisect/bisect_test.py b/ci/ray_ci/bisect/bisect_test.py index 6defa6f250b3..2fa8466572dd 100644 --- a/ci/ray_ci/bisect/bisect_test.py +++ b/ci/ray_ci/bisect/bisect_test.py @@ -1,17 +1,17 @@ -import click import json import os -from ci.ray_ci.utils import logger, ci_init -from ci.ray_ci.bisect.macos_validator import MacOSValidator -from ci.ray_ci.bisect.generic_validator import GenericValidator -from ci.ray_ci.bisect.bisector import Bisector +import click from ray_release.test import ( Test, TestType, ) from ray_release.test_automation.ci_state_machine import CITestStateMachine +from ci.ray_ci.bisect.bisector import Bisector +from ci.ray_ci.bisect.generic_validator import GenericValidator +from ci.ray_ci.bisect.macos_validator import MacOSValidator +from ci.ray_ci.utils import ci_init, logger # This is the directory where the ray repository is mounted in the container RAYCI_CHECKOUT_DIR_MOUNT = "/ray" diff --git a/ci/ray_ci/bisect/bisector.py b/ci/ray_ci/bisect/bisector.py index 822c391ce5cd..095f48d09876 100644 --- a/ci/ray_ci/bisect/bisector.py +++ b/ci/ray_ci/bisect/bisector.py @@ -1,10 +1,11 @@ import subprocess from typing import List, Optional -from ci.ray_ci.utils import logger -from ci.ray_ci.bisect.validator import Validator from ray_release.test import Test +from ci.ray_ci.bisect.validator import Validator +from ci.ray_ci.utils import logger + class Bisector: def __init__( diff --git a/ci/ray_ci/bisect/generic_validator.py b/ci/ray_ci/bisect/generic_validator.py index 636ed9246dda..3142a0eed463 100644 --- a/ci/ray_ci/bisect/generic_validator.py +++ b/ci/ray_ci/bisect/generic_validator.py @@ -1,12 +1,12 @@ import time from pybuildkite.buildkite import Buildkite +from ray_release.aws import get_secret_token +from ray_release.configs.global_config import get_global_config +from ray_release.test import Test from ci.ray_ci.bisect.validator import Validator from ci.ray_ci.utils import logger -from ray_release.test import Test -from ray_release.aws import get_secret_token -from ray_release.configs.global_config import get_global_config BUILDKITE_ORGANIZATION = "ray-project" BUILDKITE_POSTMERGE_PIPELINE = "postmerge" diff --git a/ci/ray_ci/bisect/macos_validator.py b/ci/ray_ci/bisect/macos_validator.py index 2112b9db0704..e1f7beb4e71f 100644 --- a/ci/ray_ci/bisect/macos_validator.py +++ b/ci/ray_ci/bisect/macos_validator.py @@ -1,10 +1,10 @@ import os import subprocess -from ci.ray_ci.bisect.validator import Validator from ray_release.bazel import bazel_runfile from ray_release.test import Test +from ci.ray_ci.bisect.validator import Validator TEST_SCRIPT = "ci/ray_ci/bisect/macos_validator.sh" diff --git a/ci/ray_ci/bisect/test_bisector.py b/ci/ray_ci/bisect/test_bisector.py index 0928be55e2dd..975cda7e18ce 100644 --- a/ci/ray_ci/bisect/test_bisector.py +++ b/ci/ray_ci/bisect/test_bisector.py @@ -1,11 +1,12 @@ import sys -import pytest from unittest import mock +import pytest +from ray_release.test import Test + from ci.ray_ci.bisect.bisector import Bisector -from ci.ray_ci.bisect.validator import Validator from ci.ray_ci.bisect.macos_validator import MacOSValidator -from ray_release.test import Test +from ci.ray_ci.bisect.validator import Validator class MockValidator(Validator): diff --git a/ci/ray_ci/bisect/test_generic_validator.py b/ci/ray_ci/bisect/test_generic_validator.py index e5314a6d68d3..3c3d4ea857c7 100644 --- a/ci/ray_ci/bisect/test_generic_validator.py +++ b/ci/ray_ci/bisect/test_generic_validator.py @@ -1,11 +1,11 @@ -import time import sys -import pytest +import time from unittest import mock +import pytest +from ray_release.test import Test from ci.ray_ci.bisect.generic_validator import WAIT, GenericValidator -from ray_release.test import Test START = time.time() diff --git a/ci/ray_ci/builder.py b/ci/ray_ci/builder.py index 3f3ba27f8a96..62edca1171c2 100644 --- a/ci/ray_ci/builder.py +++ b/ci/ray_ci/builder.py @@ -2,19 +2,19 @@ import click +from ci.ray_ci.anyscale_docker_container import AnyscaleDockerContainer from ci.ray_ci.builder_container import ( + ARCHITECTURE, + BUILD_TYPES, DEFAULT_PYTHON_VERSION, PYTHON_VERSIONS, - BUILD_TYPES, - ARCHITECTURE, BuilderContainer, ) -from ci.ray_ci.windows_builder_container import WindowsBuilderContainer +from ci.ray_ci.container import _DOCKER_ECR_REPO from ci.ray_ci.docker_container import PLATFORMS_RAY from ci.ray_ci.ray_docker_container import RayDockerContainer -from ci.ray_ci.anyscale_docker_container import AnyscaleDockerContainer -from ci.ray_ci.container import _DOCKER_ECR_REPO -from ci.ray_ci.utils import logger, docker_login, ci_init +from ci.ray_ci.utils import ci_init, docker_login, logger +from ci.ray_ci.windows_builder_container import WindowsBuilderContainer @click.command() diff --git a/ci/ray_ci/container.py b/ci/ray_ci/container.py index 44bda3117273..d8ec6a37a6bb 100644 --- a/ci/ray_ci/container.py +++ b/ci/ray_ci/container.py @@ -1,11 +1,9 @@ import abc import os +import re import subprocess import sys -import re - -from typing import List, Tuple, Optional - +from typing import List, Optional, Tuple # Regex pattern to match CUDA copyright header with any version _CUDA_COPYRIGHT_PATTERN = r"""========== diff --git a/ci/ray_ci/doc/api.py b/ci/ray_ci/doc/api.py index 265bad77b2a0..f570b15be0e2 100644 --- a/ci/ray_ci/doc/api.py +++ b/ci/ray_ci/doc/api.py @@ -1,11 +1,9 @@ -import re import importlib import inspect - -from enum import Enum +import re from dataclasses import dataclass -from typing import Optional, List, Tuple, Set, Dict - +from enum import Enum +from typing import Dict, List, Optional, Set, Tuple _SPHINX_AUTOSUMMARY_HEADER = ".. autosummary::" _SPHINX_AUTOCLASS_HEADER = ".. autoclass::" diff --git a/ci/ray_ci/doc/autodoc.py b/ci/ray_ci/doc/autodoc.py index 9d2f18b8dd78..2f875488d5e8 100644 --- a/ci/ray_ci/doc/autodoc.py +++ b/ci/ray_ci/doc/autodoc.py @@ -3,12 +3,11 @@ from typing import List, Set from ci.ray_ci.doc.api import ( - API, - _SPHINX_AUTOSUMMARY_HEADER, _SPHINX_AUTOCLASS_HEADER, + _SPHINX_AUTOSUMMARY_HEADER, + API, ) - _SPHINX_CURRENTMODULE_HEADER = ".. currentmodule::" _SPHINX_TOCTREE_HEADER = ".. toctree::" _SPHINX_INCLUDE_HEADER = ".. include::" diff --git a/ci/ray_ci/doc/build_cache.py b/ci/ray_ci/doc/build_cache.py index 4301dbc1204a..9a45d98496ae 100644 --- a/ci/ray_ci/doc/build_cache.py +++ b/ci/ray_ci/doc/build_cache.py @@ -1,14 +1,13 @@ -import tempfile -import subprocess import os import pickle +import subprocess +import tempfile from typing import Set import boto3 - -from ci.ray_ci.utils import logger from ray_release.util import get_write_state_machine_aws_bucket +from ci.ray_ci.utils import logger AWS_CACHE_KEY = "doc_build" ENVIRONMENT_PICKLE = "_build/doctrees/environment.pickle" diff --git a/ci/ray_ci/doc/cmd_build.py b/ci/ray_ci/doc/cmd_build.py index fd89bdf32854..8740980910d1 100644 --- a/ci/ray_ci/doc/cmd_build.py +++ b/ci/ray_ci/doc/cmd_build.py @@ -1,12 +1,11 @@ -import subprocess import os +import subprocess import click +from ray_release.configs.global_config import get_global_config -from ci.ray_ci.utils import logger, ci_init from ci.ray_ci.doc.build_cache import BuildCache - -from ray_release.configs.global_config import get_global_config +from ci.ray_ci.utils import ci_init, logger @click.command() diff --git a/ci/ray_ci/doc/cmd_check_api_discrepancy.py b/ci/ray_ci/doc/cmd_check_api_discrepancy.py index ffdbce7792b0..0112a1a3e121 100644 --- a/ci/ray_ci/doc/cmd_check_api_discrepancy.py +++ b/ci/ray_ci/doc/cmd_check_api_discrepancy.py @@ -1,8 +1,8 @@ import click -from ci.ray_ci.doc.module import Module -from ci.ray_ci.doc.autodoc import Autodoc from ci.ray_ci.doc.api import API +from ci.ray_ci.doc.autodoc import Autodoc +from ci.ray_ci.doc.module import Module from ci.ray_ci.utils import logger TEAM_API_CONFIGS = { diff --git a/ci/ray_ci/doc/mock/__init__.py b/ci/ray_ci/doc/mock/__init__.py index 8491bdf4eb10..8692093685ca 100644 --- a/ci/ray_ci/doc/mock/__init__.py +++ b/ci/ray_ci/doc/mock/__init__.py @@ -1,5 +1,4 @@ -from ci.ray_ci.doc.mock.mock_module import MockClass -from ci.ray_ci.doc.mock.mock_module import mock_function +from ci.ray_ci.doc.mock.mock_module import MockClass, mock_function # classes and functions __all__ = [ diff --git a/ci/ray_ci/doc/test_api.py b/ci/ray_ci/doc/test_api.py index 490d517ffac3..d95417987ba5 100644 --- a/ci/ray_ci/doc/test_api.py +++ b/ci/ray_ci/doc/test_api.py @@ -1,12 +1,13 @@ import sys + import pytest from ci.ray_ci.doc.api import ( + _SPHINX_AUTOCLASS_HEADER, + _SPHINX_AUTOSUMMARY_HEADER, API, AnnotationType, CodeType, - _SPHINX_AUTOCLASS_HEADER, - _SPHINX_AUTOSUMMARY_HEADER, ) from ci.ray_ci.doc.mock.mock_module import mock_function diff --git a/ci/ray_ci/doc/test_autodoc.py b/ci/ray_ci/doc/test_autodoc.py index e340889e8255..cbd7d54eb4f6 100644 --- a/ci/ray_ci/doc/test_autodoc.py +++ b/ci/ray_ci/doc/test_autodoc.py @@ -1,11 +1,12 @@ import os -import tempfile import sys +import tempfile + import pytest +from ci.ray_ci.doc.api import API, AnnotationType, CodeType from ci.ray_ci.doc.autodoc import Autodoc from ci.ray_ci.doc.mock.mock_module import MockClass, mock_function, mock_w00t -from ci.ray_ci.doc.api import API, AnnotationType, CodeType def test_walk(): diff --git a/ci/ray_ci/doc/test_build_cache.py b/ci/ray_ci/doc/test_build_cache.py index 8c45bc97d932..b1070f03b6f2 100644 --- a/ci/ray_ci/doc/test_build_cache.py +++ b/ci/ray_ci/doc/test_build_cache.py @@ -1,10 +1,11 @@ -import sys import os import pickle -import pytest +import sys import tempfile from unittest import mock +import pytest + from ci.ray_ci.doc.build_cache import BuildCache diff --git a/ci/ray_ci/doc/test_module.py b/ci/ray_ci/doc/test_module.py index 3407cfce5fc5..ead02afb7157 100644 --- a/ci/ray_ci/doc/test_module.py +++ b/ci/ray_ci/doc/test_module.py @@ -1,8 +1,9 @@ import sys + import pytest -from ci.ray_ci.doc.module import Module from ci.ray_ci.doc.api import AnnotationType, CodeType +from ci.ray_ci.doc.module import Module def test_walk(): diff --git a/ci/ray_ci/doc/test_update_cache_env.py b/ci/ray_ci/doc/test_update_cache_env.py index a7d2592793d2..88ce8c6894da 100644 --- a/ci/ray_ci/doc/test_update_cache_env.py +++ b/ci/ray_ci/doc/test_update_cache_env.py @@ -1,11 +1,13 @@ -import sys import os import pickle +import sys +import tempfile + import pytest from sphinx.project import Project -import tempfile -from ci.ray_ci.doc.cmd_update_cache_env import update_environment_pickle + from ci.ray_ci.doc.build_cache import ENVIRONMENT_PICKLE +from ci.ray_ci.doc.cmd_update_cache_env import update_environment_pickle class FakeBuildEnv: diff --git a/ci/ray_ci/docker_container.py b/ci/ray_ci/docker_container.py index 6e30335b73e1..73a6f3d2879c 100644 --- a/ci/ray_ci/docker_container.py +++ b/ci/ray_ci/docker_container.py @@ -1,11 +1,10 @@ import os -from typing import List from datetime import datetime from enum import Enum +from typing import List -from ci.ray_ci.linux_container import LinuxContainer from ci.ray_ci.builder_container import DEFAULT_ARCHITECTURE, DEFAULT_PYTHON_VERSION - +from ci.ray_ci.linux_container import LinuxContainer PLATFORMS_RAY = [ "cpu", diff --git a/ci/ray_ci/linux_container.py b/ci/ray_ci/linux_container.py index 1e865269d25c..44c6d1971d2d 100644 --- a/ci/ray_ci/linux_container.py +++ b/ci/ray_ci/linux_container.py @@ -1,7 +1,7 @@ import os import subprocess import sys -from typing import List, Tuple, Optional +from typing import List, Optional, Tuple from ci.ray_ci.container import Container diff --git a/ci/ray_ci/pipeline/gap_filling_scheduler.py b/ci/ray_ci/pipeline/gap_filling_scheduler.py index 4bcf65cd6b38..52b1d507a977 100644 --- a/ci/ray_ci/pipeline/gap_filling_scheduler.py +++ b/ci/ray_ci/pipeline/gap_filling_scheduler.py @@ -1,10 +1,9 @@ import subprocess from datetime import datetime, timedelta -from typing import List, Dict, Optional, Any, Tuple +from typing import Any, Dict, List, Optional, Tuple from pybuildkite.buildkite import Buildkite - BRANCH = "master" BLOCK_STEP_KEY = "unblock-me" diff --git a/ci/ray_ci/pipeline/scheduler.py b/ci/ray_ci/pipeline/scheduler.py index 75b498fc88ab..a23e271817d6 100644 --- a/ci/ray_ci/pipeline/scheduler.py +++ b/ci/ray_ci/pipeline/scheduler.py @@ -1,10 +1,10 @@ import click - -from ci.ray_ci.utils import ci_init, logger -from ci.ray_ci.pipeline.gap_filling_scheduler import GapFillingScheduler from ray_release.aws import get_secret_token from ray_release.configs.global_config import get_global_config +from ci.ray_ci.pipeline.gap_filling_scheduler import GapFillingScheduler +from ci.ray_ci.utils import ci_init, logger + @click.command() @click.argument("buildkite_organization", type=str) diff --git a/ci/ray_ci/pipeline/test_gap_filling_scheduler.py b/ci/ray_ci/pipeline/test_gap_filling_scheduler.py index 669899275aec..c4e667ee6679 100644 --- a/ci/ray_ci/pipeline/test_gap_filling_scheduler.py +++ b/ci/ray_ci/pipeline/test_gap_filling_scheduler.py @@ -3,7 +3,7 @@ import pytest -from ci.ray_ci.pipeline.gap_filling_scheduler import GapFillingScheduler, BLOCK_STEP_KEY +from ci.ray_ci.pipeline.gap_filling_scheduler import BLOCK_STEP_KEY, GapFillingScheduler @mock.patch( diff --git a/ci/ray_ci/ray_docker_container.py b/ci/ray_ci/ray_docker_container.py index 71d67b72e502..5bc41a1c5cdb 100644 --- a/ci/ray_ci/ray_docker_container.py +++ b/ci/ray_ci/ray_docker_container.py @@ -1,11 +1,12 @@ import os from typing import List +from ray_release.configs.global_config import get_global_config + +from ci.ray_ci.builder_container import DEFAULT_ARCHITECTURE, PYTHON_VERSIONS from ci.ray_ci.container import _DOCKER_ECR_REPO from ci.ray_ci.docker_container import DockerContainer -from ci.ray_ci.builder_container import PYTHON_VERSIONS, DEFAULT_ARCHITECTURE -from ci.ray_ci.utils import docker_pull, RAY_VERSION -from ray_release.configs.global_config import get_global_config +from ci.ray_ci.utils import RAY_VERSION, docker_pull class RayDockerContainer(DockerContainer): diff --git a/ci/ray_ci/test_anyscale_docker_container.py b/ci/ray_ci/test_anyscale_docker_container.py index 7f09036aa366..777a98447d6f 100644 --- a/ci/ray_ci/test_anyscale_docker_container.py +++ b/ci/ray_ci/test_anyscale_docker_container.py @@ -1,13 +1,13 @@ -import sys import os +import sys from typing import List from unittest import mock import pytest from ci.ray_ci.anyscale_docker_container import AnyscaleDockerContainer +from ci.ray_ci.container import _DOCKER_ECR_REPO, _DOCKER_GCP_REGISTRY from ci.ray_ci.test_base import RayCITestBase -from ci.ray_ci.container import _DOCKER_GCP_REGISTRY, _DOCKER_ECR_REPO class TestAnyscaleDockerContainer(RayCITestBase): diff --git a/ci/ray_ci/test_base.py b/ci/ray_ci/test_base.py index e5c5d0b76679..d0cbc4c253bc 100644 --- a/ci/ray_ci/test_base.py +++ b/ci/ray_ci/test_base.py @@ -2,8 +2,8 @@ import unittest from unittest.mock import patch -from ci.ray_ci.builder_container import PYTHON_VERSIONS from ci.ray_ci.builder import DEFAULT_PYTHON_VERSION +from ci.ray_ci.builder_container import PYTHON_VERSIONS from ci.ray_ci.utils import ci_init diff --git a/ci/ray_ci/test_bazel_sharding.py b/ci/ray_ci/test_bazel_sharding.py index 927eb59afe58..74d0e320b3c7 100644 --- a/ci/ray_ci/test_bazel_sharding.py +++ b/ci/ray_ci/test_bazel_sharding.py @@ -1,9 +1,10 @@ -from typing import List -import pytest import os import shutil import sys import tempfile +from typing import List + +import pytest # Required for bazel file_parent = os.path.dirname(__file__) diff --git a/ci/ray_ci/test_builder_container.py b/ci/ray_ci/test_builder_container.py index 47abef000116..eb2e6dc23600 100644 --- a/ci/ray_ci/test_builder_container.py +++ b/ci/ray_ci/test_builder_container.py @@ -1,7 +1,8 @@ import sys -import pytest -from unittest import mock from typing import List +from unittest import mock + +import pytest from ci.ray_ci.builder_container import BuilderContainer diff --git a/ci/ray_ci/test_linux_container.py b/ci/ray_ci/test_linux_container.py index e6c7d693b4bc..3e6e32e1bba3 100644 --- a/ci/ray_ci/test_linux_container.py +++ b/ci/ray_ci/test_linux_container.py @@ -1,4 +1,5 @@ import sys + import pytest from ci.ray_ci.linux_container import LinuxContainer diff --git a/ci/ray_ci/test_linux_tester_container.py b/ci/ray_ci/test_linux_tester_container.py index 831eb5e7f594..bf2003bff15e 100644 --- a/ci/ray_ci/test_linux_tester_container.py +++ b/ci/ray_ci/test_linux_tester_container.py @@ -2,17 +2,17 @@ import os import platform import sys -import pytest import tempfile -from unittest import mock from typing import List, Optional +from unittest import mock + +import pytest +from ray_release.configs.global_config import get_global_config +from ci.ray_ci.container import _DOCKER_ECR_REPO, _RAYCI_BUILD_ID from ci.ray_ci.linux_tester_container import LinuxTesterContainer from ci.ray_ci.tester_container import RUN_PER_FLAKY_TEST from ci.ray_ci.utils import chunk_into_n, ci_init -from ci.ray_ci.container import _DOCKER_ECR_REPO, _RAYCI_BUILD_ID -from ray_release.configs.global_config import get_global_config - ci_init() diff --git a/ci/ray_ci/test_privileged.py b/ci/ray_ci/test_privileged.py index c1f71dfe7056..e69d6ad78367 100644 --- a/ci/ray_ci/test_privileged.py +++ b/ci/ray_ci/test_privileged.py @@ -1,9 +1,9 @@ import os -import pytest import sys - from pathlib import Path +import pytest + # In privileged containers, we expect the following # cgroupv1 is disabled # cgroupv2 is enabled and mounted on /sys/fs/cgroup diff --git a/ci/ray_ci/test_ray_docker_container.py b/ci/ray_ci/test_ray_docker_container.py index 6d474129f722..ce2bc3021e58 100644 --- a/ci/ray_ci/test_ray_docker_container.py +++ b/ci/ray_ci/test_ray_docker_container.py @@ -1,9 +1,11 @@ import os import sys +from datetime import datetime from typing import List from unittest import mock -from datetime import datetime + import pytest +from ray_release.configs.global_config import get_global_config from ci.ray_ci.builder_container import DEFAULT_PYTHON_VERSION from ci.ray_ci.container import _DOCKER_ECR_REPO @@ -11,7 +13,6 @@ from ci.ray_ci.ray_docker_container import RayDockerContainer from ci.ray_ci.test_base import RayCITestBase from ci.ray_ci.utils import RAY_VERSION -from ray_release.configs.global_config import get_global_config class TestRayDockerContainer(RayCITestBase): diff --git a/ci/ray_ci/test_tester.py b/ci/ray_ci/test_tester.py index 21861e250dad..4f3e8ce06fb6 100644 --- a/ci/ray_ci/test_tester.py +++ b/ci/ray_ci/test_tester.py @@ -5,19 +5,19 @@ from unittest import mock import pytest +from ray_release.test import Test, TestState from ci.ray_ci.linux_tester_container import LinuxTesterContainer -from ci.ray_ci.windows_tester_container import WindowsTesterContainer from ci.ray_ci.tester import ( _add_default_except_tags, - _get_container, _get_all_test_query, - _get_test_targets, - _get_new_tests, + _get_container, _get_flaky_test_targets, + _get_new_tests, _get_tag_matcher, + _get_test_targets, ) -from ray_release.test import Test, TestState +from ci.ray_ci.windows_tester_container import WindowsTesterContainer def _stub_test(val: dict) -> Test: diff --git a/ci/ray_ci/test_utils.py b/ci/ray_ci/test_utils.py index f97566d00d19..cd946229612a 100644 --- a/ci/ray_ci/test_utils.py +++ b/ci/ray_ci/test_utils.py @@ -1,16 +1,17 @@ import base64 import io import sys -import pytest -from unittest import mock from typing import List +from unittest import mock +import pytest from ray_release.test import Test + from ci.ray_ci.utils import ( chunk_into_n, docker_login, - get_flaky_test_names, filter_tests, + get_flaky_test_names, ) diff --git a/ci/ray_ci/test_windows_container.py b/ci/ray_ci/test_windows_container.py index 9ea95d212c23..d7527b97b234 100644 --- a/ci/ray_ci/test_windows_container.py +++ b/ci/ray_ci/test_windows_container.py @@ -1,10 +1,11 @@ import sys -import pytest -from unittest import mock from typing import List +from unittest import mock + +import pytest -from ci.ray_ci.windows_container import WindowsContainer from ci.ray_ci.container import _DOCKER_ENV +from ci.ray_ci.windows_container import WindowsContainer def test_install_ray() -> None: diff --git a/ci/ray_ci/test_windows_tester_container.py b/ci/ray_ci/test_windows_tester_container.py index 48667b9265a5..753ffd373d6c 100644 --- a/ci/ray_ci/test_windows_tester_container.py +++ b/ci/ray_ci/test_windows_tester_container.py @@ -1,5 +1,5 @@ -from unittest import mock from typing import List +from unittest import mock from ci.ray_ci.windows_tester_container import WindowsTesterContainer diff --git a/ci/ray_ci/tester.py b/ci/ray_ci/tester.py index c2b700f34db8..339c6aea5844 100644 --- a/ci/ray_ci/tester.py +++ b/ci/ray_ci/tester.py @@ -1,23 +1,23 @@ import os import sys -from typing import List, Set, Tuple, Optional +from typing import List, Optional, Set, Tuple -import yaml import click +import yaml +from ray_release.test import Test, TestState -from ci.ray_ci.container import _DOCKER_ECR_REPO from ci.ray_ci.builder_container import ( - BuilderContainer, + DEFAULT_ARCHITECTURE, DEFAULT_BUILD_TYPE, DEFAULT_PYTHON_VERSION, - DEFAULT_ARCHITECTURE, PYTHON_VERSIONS, + BuilderContainer, ) +from ci.ray_ci.container import _DOCKER_ECR_REPO from ci.ray_ci.linux_tester_container import LinuxTesterContainer -from ci.ray_ci.windows_tester_container import WindowsTesterContainer from ci.ray_ci.tester_container import TesterContainer -from ci.ray_ci.utils import docker_login, ci_init -from ray_release.test import Test, TestState +from ci.ray_ci.utils import ci_init, docker_login +from ci.ray_ci.windows_tester_container import WindowsTesterContainer CUDA_COPYRIGHT = """ ========== diff --git a/ci/ray_ci/tester_container.py b/ci/ray_ci/tester_container.py index 5c0078e323d5..0e91ee58a5d0 100644 --- a/ci/ray_ci/tester_container.py +++ b/ci/ray_ci/tester_container.py @@ -5,16 +5,15 @@ import shutil import string import subprocess -from typing import List, Tuple, Optional -from os import path, listdir +from os import listdir, path +from typing import List, Optional, Tuple -from ci.ray_ci.utils import shard_tests, chunk_into_n -from ci.ray_ci.utils import logger -from ci.ray_ci.container import Container -from ray_release.test import TestResult, Test -from ray_release.test_automation.ci_state_machine import CITestStateMachine from ray_release.configs.global_config import get_global_config +from ray_release.test import Test, TestResult +from ray_release.test_automation.ci_state_machine import CITestStateMachine +from ci.ray_ci.container import Container +from ci.ray_ci.utils import chunk_into_n, logger, shard_tests # We will run each flaky test this number of times per CI job independent of pass/fail. RUN_PER_FLAKY_TEST = 1 diff --git a/ci/ray_ci/utils.py b/ci/ray_ci/utils.py index a9d6159aa8a3..90b510fd5379 100644 --- a/ci/ray_ci/utils.py +++ b/ci/ray_ci/utils.py @@ -5,15 +5,15 @@ import subprocess import sys import tempfile - -import boto3 -from typing import List from math import ceil +from typing import List -import ci.ray_ci.bazel_sharding as bazel_sharding +import boto3 from ray_release.bazel import bazel_runfile -from ray_release.test import Test, TestState from ray_release.configs.global_config import init_global_config +from ray_release.test import Test, TestState + +import ci.ray_ci.bazel_sharding as bazel_sharding GLOBAL_CONFIG_FILE = ( os.environ.get("RAYCI_GLOBAL_CONFIG") or "ci/ray_ci/oss_config.yaml" diff --git a/ci/ray_ci/windows_builder_container.py b/ci/ray_ci/windows_builder_container.py index 0c4f0cf214ee..1ccf2c5078c0 100644 --- a/ci/ray_ci/windows_builder_container.py +++ b/ci/ray_ci/windows_builder_container.py @@ -1,6 +1,6 @@ import os -from ci.ray_ci.windows_container import WindowsContainer, WORKDIR +from ci.ray_ci.windows_container import WORKDIR, WindowsContainer class WindowsBuilderContainer(WindowsContainer): diff --git a/ci/ray_ci/windows_container.py b/ci/ray_ci/windows_container.py index 838c6491b05c..0e9f4b79e0e1 100644 --- a/ci/ray_ci/windows_container.py +++ b/ci/ray_ci/windows_container.py @@ -1,11 +1,10 @@ import os import subprocess import sys -from typing import List, Tuple, Optional +from typing import List, Optional, Tuple from ci.ray_ci.container import Container - WORKDIR = "C:\\rayci" diff --git a/ci/ray_ci/windows_tester_container.py b/ci/ray_ci/windows_tester_container.py index 37ea14f645e6..750f4da112dc 100644 --- a/ci/ray_ci/windows_tester_container.py +++ b/ci/ray_ci/windows_tester_container.py @@ -1,7 +1,7 @@ from typing import List, Optional -from ci.ray_ci.windows_container import WindowsContainer from ci.ray_ci.tester_container import TesterContainer +from ci.ray_ci.windows_container import WindowsContainer class WindowsTesterContainer(TesterContainer, WindowsContainer): diff --git a/ci/repro-ci.py b/ci/repro-ci.py index 7800e71bc1ec..c5b6537ecbbe 100644 --- a/ci/repro-ci.py +++ b/ci/repro-ci.py @@ -37,7 +37,7 @@ import threading import time from numbers import Number -from typing import Any, Dict, List, Optional, Callable +from typing import Any, Callable, Dict, List, Optional import boto3 import click diff --git a/pyproject.toml b/pyproject.toml index 8ae06c326372..5d803303c12a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,12 +65,6 @@ afterray = ["psutil", "setproctitle"] "python/ray/setup-dev.py" = ["I"] "python/ray/cloudpickle/*" = ["I"] "python/ray/dag/*.py" = ["I"] -"ci/lint/*" = ["I"] -"ci/pipeline/*" = ["I"] -"ci/ray_ci/*" = ["I"] -"ci/build/*" = ["I"] -"ci/env/*" = ["I"] -"ci/repro-ci.py" = ["I"] "python/ray/includes/*" = ["I"] "python/ray/internal/*" = ["I"] "python/ray/ray_operator/*" = ["I"] From 1895aa1c4329f18a803e9a4386d50e8b1a4f1f1e Mon Sep 17 00:00:00 2001 From: Ping Date: Mon, 11 Aug 2025 11:06:21 +0800 Subject: [PATCH 013/634] [Core] Cover cpplint for ray/src/ray/common/test (#55414) Signed-off-by: 400Ping --- .pre-commit-config.yaml | 2 +- .../common/test/bundle_location_index_test.cc | 3 + src/ray/common/test/event_stats_test.cc | 3 + src/ray/common/test/grpc_util_test.cc | 2 + src/ray/common/test/id_test.cc | 10 ++-- src/ray/common/test/label_selector_test.cc | 2 + src/ray/common/test/memory_monitor_test.cc | 3 + src/ray/common/test/postable_test.cc | 2 + src/ray/common/test/ray_config_test.cc | 3 + src/ray/common/test/ray_syncer_test.cc | 56 +++++++++++-------- .../common/test/resource_instance_set_test.cc | 3 + src/ray/common/test/resource_request_test.cc | 2 + src/ray/common/test/resource_set_test.cc | 3 + src/ray/common/test/scheduling_ids_test.cc | 49 ++++++++-------- src/ray/common/test/status_test.cc | 2 + .../common/test/syncer_service_e2e_test.cc | 23 ++++---- src/ray/common/test/task_spec_test.cc | 4 ++ 17 files changed, 110 insertions(+), 62 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a586e806b6bc..7a19234ed517 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -73,7 +73,7 @@ repos: hooks: - id: cpplint args: ["--filter=-whitespace/braces,-whitespace/line_length,-build/c++11,-build/c++14,-build/c++17,-readability/braces,-whitespace/indent_namespace,-runtime/int,-runtime/references,-build/include_order"] - files: ^src/ray/(common/cgroup2|common/scheduling|common/ray_syncer|util|raylet_client|internal|scheduling|pubsub|object_manager|rpc(?:/.*)?|raylet|core_worker)/.*\.(h|cc)$ + files: ^src/ray/(common/cgroup2|common/scheduling|common/ray_syncer|common/test|util|raylet_client|internal|scheduling|pubsub|object_manager|rpc(?:/.*)?|raylet|core_worker)/.*\.(h|cc)$ exclude: | (?x)^( src/ray/raylet/scheduling/.*\.(h|cc)$ | diff --git a/src/ray/common/test/bundle_location_index_test.cc b/src/ray/common/test/bundle_location_index_test.cc index 49468d978274..9e3bba19c8ee 100644 --- a/src/ray/common/test/bundle_location_index_test.cc +++ b/src/ray/common/test/bundle_location_index_test.cc @@ -15,6 +15,9 @@ #include "ray/common/bundle_location_index.h" +#include +#include + #include "gtest/gtest.h" namespace ray { diff --git a/src/ray/common/test/event_stats_test.cc b/src/ray/common/test/event_stats_test.cc index 9d88065ea8ff..3b1bf67c1e56 100644 --- a/src/ray/common/test/event_stats_test.cc +++ b/src/ray/common/test/event_stats_test.cc @@ -14,6 +14,9 @@ #include "ray/common/event_stats.h" +#include +#include + #include "gtest/gtest.h" TEST(EventStatsTest, TestRecordEnd) { diff --git a/src/ray/common/test/grpc_util_test.cc b/src/ray/common/test/grpc_util_test.cc index 17c50faf28dc..5170446c123b 100644 --- a/src/ray/common/test/grpc_util_test.cc +++ b/src/ray/common/test/grpc_util_test.cc @@ -14,6 +14,8 @@ #include "ray/common/grpc_util.h" +#include + #include "gtest/gtest.h" #include "src/ray/protobuf/common.pb.h" diff --git a/src/ray/common/test/id_test.cc b/src/ray/common/test/id_test.cc index 2c11d7681a1e..1264dcf0db30 100644 --- a/src/ray/common/test/id_test.cc +++ b/src/ray/common/test/id_test.cc @@ -14,6 +14,8 @@ #include +#include + #include "absl/container/flat_hash_set.h" #include "ray/common/common_protocol.h" #include "ray/common/task/task_spec.h" @@ -35,9 +37,9 @@ void TestRandomObjectId() { ASSERT_EQ(random_object_id.ObjectIndex(), 0); } -const static JobID kDefaultJobId = JobID::FromInt(199); +static const JobID kDefaultJobId = JobID::FromInt(199); -const static TaskID kDefaultDriverTaskId = TaskID::ForDriverTask(kDefaultJobId); +static const TaskID kDefaultDriverTaskId = TaskID::ForDriverTask(kDefaultJobId); TEST(JobIDTest, TestJobID) { uint32_t id = 100; @@ -104,9 +106,9 @@ TEST(TaskIDTest, TestTaskIDForExecution) { } TEST(ObjectIDTest, TestObjectID) { - const static ActorID default_actor_id = + static const ActorID default_actor_id = ActorID::Of(kDefaultJobId, kDefaultDriverTaskId, 1); - const static TaskID default_task_id = + static const TaskID default_task_id = TaskID::ForActorTask(kDefaultJobId, kDefaultDriverTaskId, 1, default_actor_id); { diff --git a/src/ray/common/test/label_selector_test.cc b/src/ray/common/test/label_selector_test.cc index f30ad26caa05..89c1fe20aaea 100644 --- a/src/ray/common/test/label_selector_test.cc +++ b/src/ray/common/test/label_selector_test.cc @@ -14,6 +14,8 @@ #include "ray/common/scheduling/label_selector.h" +#include + #include "gtest/gtest.h" namespace ray { diff --git a/src/ray/common/test/memory_monitor_test.cc b/src/ray/common/test/memory_monitor_test.cc index 3f9e8f0071b6..03ccb4d693c5 100644 --- a/src/ray/common/test/memory_monitor_test.cc +++ b/src/ray/common/test/memory_monitor_test.cc @@ -20,6 +20,9 @@ #include #include #include +#include +#include +#include #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" diff --git a/src/ray/common/test/postable_test.cc b/src/ray/common/test/postable_test.cc index 60f8d18571ff..1490d18f8684 100644 --- a/src/ray/common/test/postable_test.cc +++ b/src/ray/common/test/postable_test.cc @@ -16,6 +16,8 @@ #include +#include + namespace ray { TEST(PostableTest, TestPostable) { diff --git a/src/ray/common/test/ray_config_test.cc b/src/ray/common/test/ray_config_test.cc index 6522640a499d..5584ab647eb8 100644 --- a/src/ray/common/test/ray_config_test.cc +++ b/src/ray/common/test/ray_config_test.cc @@ -14,6 +14,9 @@ #include "ray/common/ray_config.h" +#include +#include + #include "gtest/gtest.h" #include "ray/common/grpc_util.h" diff --git a/src/ray/common/test/ray_syncer_test.cc b/src/ray/common/test/ray_syncer_test.cc index 6b8757f5e080..6982980efedf 100644 --- a/src/ray/common/test/ray_syncer_test.cc +++ b/src/ray/common/test/ray_syncer_test.cc @@ -25,7 +25,13 @@ #include #include +#include #include +#include +#include +#include +#include +#include #include "ray/common/ray_syncer/node_state.h" #include "ray/common/ray_syncer/ray_syncer.h" @@ -35,8 +41,6 @@ #include "ray/util/network_util.h" #include "ray/util/path_utils.h" -using namespace std::chrono; -using namespace ray::syncer; using ray::NodeID; using ::testing::_; using ::testing::Eq; @@ -204,7 +208,7 @@ TEST_F(RaySyncerTest, RaySyncerBidiReactorBase) { } struct SyncerServerTest { - SyncerServerTest(std::string port) + explicit SyncerServerTest(std::string port) : SyncerServerTest( std::move(port), /*node_id=*/NodeID::FromRandom(), /*ray_sync_observer=*/{}) { } @@ -306,22 +310,24 @@ struct SyncerServerTest { if (f.get()) { return; } else { - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(std::chrono::seconds(1)); } } } bool WaitUntil(std::function predicate, int64_t time_s) { - auto start = steady_clock::now(); + auto start = std::chrono::steady_clock::now(); - while (duration_cast(steady_clock::now() - start).count() <= time_s) { + while (std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count() <= time_s) { std::promise p; auto f = p.get_future(); io_context.post([&p, predicate]() mutable { p.set_value(predicate()); }, "TEST"); if (f.get()) { return true; } else { - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(std::chrono::seconds(1)); } } return false; @@ -445,7 +451,7 @@ class SyncerTest : public ::testing::Test { s->Stop(); } - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(std::chrono::seconds(1)); } std::vector> servers; }; @@ -526,10 +532,10 @@ TEST_F(SyncerTest, Test1To1) { // Make sure no new messages are sent s2.local_versions[0] = 0; - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(std::chrono::seconds(1)); - ASSERT_TRUE(s1.GetNumConsumedMessages(s2.syncer->GetLocalNodeID()) == 2); - ASSERT_TRUE(s2.GetNumConsumedMessages(s1.syncer->GetLocalNodeID()) == 1); + ASSERT_EQ(s1.GetNumConsumedMessages(s2.syncer->GetLocalNodeID()), 2); + ASSERT_EQ(s2.GetNumConsumedMessages(s1.syncer->GetLocalNodeID()), 1); // Change it back s2.local_versions[0] = 1; @@ -539,7 +545,7 @@ TEST_F(SyncerTest, Test1To1) { std::uniform_int_distribution<> rand_sleep(0, 10000); std::uniform_int_distribution<> choose_component(0, kTestComponents - 1); - auto start = steady_clock::now(); + auto start = std::chrono::steady_clock::now(); for (int i = 0; i < 10000; ++i) { if (choose_component(gen) == 0) { s1.local_versions[0]++; @@ -547,16 +553,16 @@ TEST_F(SyncerTest, Test1To1) { s2.local_versions[choose_component(gen)]++; } if (rand_sleep(gen) < 5) { - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(std::chrono::seconds(1)); } } - auto end = steady_clock::now(); + auto end = std::chrono::steady_clock::now(); // Max messages can be send during this period of time. // +1 is for corner cases. auto max_sends = - duration_cast(end - start).count() / + std::chrono::duration_cast(end - start).count() / RayConfig::instance().raylet_report_resources_period_milliseconds() + 1; @@ -721,7 +727,7 @@ bool TestCorrectness(std::function get_cluster_ for (size_t i = 0; i < 10; ++i) { if (!check()) { - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(std::chrono::seconds(1)); } else { break; } @@ -747,7 +753,7 @@ bool TestCorrectness(std::function get_cluster_ servers[server_idx]->local_versions[message_type]++; // expect to sleep for 100 times for the whole loop. if (rand_sleep(gen) < 100) { - std::this_thread::sleep_for(100ms); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); } } @@ -757,7 +763,7 @@ bool TestCorrectness(std::function get_cluster_ // Make sure everything is synced. for (size_t i = 0; i < 10; ++i) { if (!check()) { - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(std::chrono::seconds(1)); } else { break; } @@ -881,14 +887,18 @@ class SyncerReactorTest : public ::testing::Test { work_guard_ = std::make_unique(io_context_.get_executor()); thread_ = std::make_unique([this]() { io_context_.run(); }); - auto start = steady_clock::now(); - while (duration_cast(steady_clock::now() - start).count() <= 5) { + auto start = std::chrono::steady_clock::now(); + while (std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count() <= 5) { RAY_LOG(INFO) << "Waiting: " - << duration_cast(steady_clock::now() - start).count(); + << std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); if (rpc_service_->reactor != nullptr) { break; }; - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(std::chrono::seconds(1)); } } @@ -991,6 +1001,6 @@ int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); auto ret = RUN_ALL_TESTS(); // Sleep for gRPC to gracefully shutdown. - std::this_thread::sleep_for(2s); + std::this_thread::sleep_for(std::chrono::seconds(2)); return ret; } diff --git a/src/ray/common/test/resource_instance_set_test.cc b/src/ray/common/test/resource_instance_set_test.cc index ba969f54509c..b5745caabf60 100644 --- a/src/ray/common/test/resource_instance_set_test.cc +++ b/src/ray/common/test/resource_instance_set_test.cc @@ -14,6 +14,9 @@ #include "ray/common/scheduling/resource_instance_set.h" +#include +#include + #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "gtest/gtest.h" diff --git a/src/ray/common/test/resource_request_test.cc b/src/ray/common/test/resource_request_test.cc index 6b58e63e2757..50d9b14223ef 100644 --- a/src/ray/common/test/resource_request_test.cc +++ b/src/ray/common/test/resource_request_test.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "gtest/gtest.h" diff --git a/src/ray/common/test/resource_set_test.cc b/src/ray/common/test/resource_set_test.cc index 00ae8343853f..5eb5ae1eb822 100644 --- a/src/ray/common/test/resource_set_test.cc +++ b/src/ray/common/test/resource_set_test.cc @@ -14,6 +14,9 @@ #include "ray/common/scheduling/resource_set.h" +#include +#include + #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "gtest/gtest.h" diff --git a/src/ray/common/test/scheduling_ids_test.cc b/src/ray/common/test/scheduling_ids_test.cc index f06a5cd10544..762436910b26 100644 --- a/src/ray/common/test/scheduling_ids_test.cc +++ b/src/ray/common/test/scheduling_ids_test.cc @@ -14,40 +14,43 @@ #include "ray/common/scheduling/scheduling_ids.h" +#include +#include + #include "gtest/gtest.h" namespace ray { -using namespace ray::scheduling; struct SchedulingIDsTest : public ::testing::Test {}; TEST_F(SchedulingIDsTest, BasicTest) { std::vector string_ids = {"hello", "whaaat", "yes"}; - std::vector node_ids; + std::vector node_ids; for (auto &string_id : string_ids) { - node_ids.emplace_back(NodeID(string_id)); + node_ids.emplace_back(scheduling::NodeID(string_id)); ASSERT_EQ(node_ids.back().Binary(), string_id); } - ASSERT_EQ(node_ids[0], NodeID(string_ids[0])); - ASSERT_EQ(node_ids[0], NodeID(node_ids[0].ToInt())); + ASSERT_EQ(node_ids[0], scheduling::NodeID(string_ids[0])); + ASSERT_EQ(node_ids[0], scheduling::NodeID(node_ids[0].ToInt())); - ASSERT_TRUE(NodeID::Nil().IsNil()); - ASSERT_EQ(NodeID::Nil().ToInt(), -1); - ASSERT_EQ(NodeID::Nil().Binary(), "-1"); + ASSERT_TRUE(scheduling::NodeID::Nil().IsNil()); + ASSERT_EQ(scheduling::NodeID::Nil().ToInt(), -1); + ASSERT_EQ(scheduling::NodeID::Nil().Binary(), "-1"); - ASSERT_EQ(NodeID(13), NodeID(13)); - ASSERT_NE(NodeID(1), NodeID(2)); - ASSERT_TRUE(NodeID(1) < NodeID(2)); + ASSERT_EQ(scheduling::NodeID(13), scheduling::NodeID(13)); + ASSERT_NE(scheduling::NodeID(1), scheduling::NodeID(2)); + ASSERT_TRUE(scheduling::NodeID(1) < scheduling::NodeID(2)); } TEST_F(SchedulingIDsTest, PrepopulateResourceIDTest) { - ASSERT_EQ(kCPU_ResourceLabel, ResourceID(CPU).Binary()); - ASSERT_EQ(kGPU_ResourceLabel, ResourceID(GPU).Binary()); - ASSERT_EQ(kObjectStoreMemory_ResourceLabel, ResourceID(OBJECT_STORE_MEM).Binary()); - ASSERT_EQ(kMemory_ResourceLabel, ResourceID(MEM).Binary()); + ASSERT_EQ(kCPU_ResourceLabel, scheduling::ResourceID(CPU).Binary()); + ASSERT_EQ(kGPU_ResourceLabel, scheduling::ResourceID(GPU).Binary()); + ASSERT_EQ(kObjectStoreMemory_ResourceLabel, + scheduling::ResourceID(OBJECT_STORE_MEM).Binary()); + ASSERT_EQ(kMemory_ResourceLabel, scheduling::ResourceID(MEM).Binary()); // mean while NodeID is not populated. - ASSERT_NE(kCPU_ResourceLabel, NodeID(CPU).Binary()); + ASSERT_NE(kCPU_ResourceLabel, scheduling::NodeID(CPU).Binary()); } TEST_F(SchedulingIDsTest, UnitInstanceResourceTest) { @@ -58,13 +61,13 @@ TEST_F(SchedulingIDsTest, UnitInstanceResourceTest) { "custom_unit_instance_resources": "neuron_cores,TPU,custom1" } )"); - ASSERT_TRUE(ResourceID::CPU().IsUnitInstanceResource()); - ASSERT_TRUE(ResourceID::GPU().IsUnitInstanceResource()); - ASSERT_TRUE(ResourceID("custom1").IsUnitInstanceResource()); - ASSERT_TRUE(ResourceID("neuron_cores").IsUnitInstanceResource()); - ASSERT_TRUE(ResourceID("TPU").IsUnitInstanceResource()); + ASSERT_TRUE(scheduling::ResourceID::CPU().IsUnitInstanceResource()); + ASSERT_TRUE(scheduling::ResourceID::GPU().IsUnitInstanceResource()); + ASSERT_TRUE(scheduling::ResourceID("custom1").IsUnitInstanceResource()); + ASSERT_TRUE(scheduling::ResourceID("neuron_cores").IsUnitInstanceResource()); + ASSERT_TRUE(scheduling::ResourceID("TPU").IsUnitInstanceResource()); - ASSERT_FALSE(ResourceID::Memory().IsUnitInstanceResource()); - ASSERT_FALSE(ResourceID("custom2").IsUnitInstanceResource()); + ASSERT_FALSE(scheduling::ResourceID::Memory().IsUnitInstanceResource()); + ASSERT_FALSE(scheduling::ResourceID("custom2").IsUnitInstanceResource()); } } // namespace ray diff --git a/src/ray/common/test/status_test.cc b/src/ray/common/test/status_test.cc index aa3597193d25..84166968c5b4 100644 --- a/src/ray/common/test/status_test.cc +++ b/src/ray/common/test/status_test.cc @@ -14,6 +14,8 @@ #include "ray/common/status.h" +#include + #include "gtest/gtest.h" #include "ray/common/grpc_util.h" diff --git a/src/ray/common/test/syncer_service_e2e_test.cc b/src/ray/common/test/syncer_service_e2e_test.cc index 650a6fed0e47..90dbf97619c4 100644 --- a/src/ray/common/test/syncer_service_e2e_test.cc +++ b/src/ray/common/test/syncer_service_e2e_test.cc @@ -22,16 +22,17 @@ #include #include #include +#include +#include +#include #include "ray/common/asio/periodical_runner.h" #include "ray/common/id.h" #include "ray/common/ray_syncer/ray_syncer.h" #include "ray/util/network_util.h" -using namespace std; -using namespace ray::syncer; using ray::PeriodicalRunner; -class LocalNode : public ReporterInterface { +class LocalNode : public ray::syncer::ReporterInterface { public: LocalNode(instrumented_io_context &io_context, ray::NodeID node_id) : node_id_(node_id), timer_(PeriodicalRunner::Create(io_context)) { @@ -51,8 +52,8 @@ class LocalNode : public ReporterInterface { "LocalNodeStateUpdate"); } - std::optional CreateSyncMessage(int64_t current_version, - MessageType) const override { + std::optional CreateSyncMessage( + int64_t current_version, ray::syncer::MessageType) const override { if (current_version > version_) { return std::nullopt; } @@ -72,7 +73,7 @@ class LocalNode : public ReporterInterface { std::shared_ptr timer_; }; -class RemoteNodes : public ReceiverInterface { +class RemoteNodes : public ray::syncer::ReceiverInterface { public: RemoteNodes() {} void ConsumeSyncMessage( @@ -100,18 +101,18 @@ int main(int argc, char *argv[]) { auto leader_port = std::string(argv[2]); auto local_node = std::make_unique(io_context, node_id); auto remote_node = std::make_unique(); - RaySyncer syncer(io_context, node_id.Binary()); + ray::syncer::RaySyncer syncer(io_context, node_id.Binary()); // RPC related field grpc::ServerBuilder builder; - std::unique_ptr service; + std::unique_ptr service; std::unique_ptr server; std::shared_ptr channel; syncer.Register( ray::rpc::syncer::MessageType::RESOURCE_VIEW, local_node.get(), remote_node.get()); if (server_port != ".") { RAY_LOG(INFO) << "Start server on port " << server_port; - auto server_address = BuildAddress("0.0.0.0", server_port); - service = std::make_unique(syncer); + auto server_address = ray::BuildAddress("0.0.0.0", server_port); + service = std::make_unique(syncer); builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); builder.RegisterService(service.get()); server = builder.BuildAndStart(); @@ -123,7 +124,7 @@ int main(int argc, char *argv[]) { argument.SetMaxSendMessageSize(::RayConfig::instance().max_grpc_message_size()); argument.SetMaxReceiveMessageSize(::RayConfig::instance().max_grpc_message_size()); - channel = grpc::CreateCustomChannel(BuildAddress("localhost", leader_port), + channel = grpc::CreateCustomChannel(ray::BuildAddress("localhost", leader_port), grpc::InsecureChannelCredentials(), argument); diff --git a/src/ray/common/test/task_spec_test.cc b/src/ray/common/test/task_spec_test.cc index 5383adcd707f..67ed76daaf4b 100644 --- a/src/ray/common/test/task_spec_test.cc +++ b/src/ray/common/test/task_spec_test.cc @@ -14,6 +14,10 @@ #include "ray/common/task/task_spec.h" +#include +#include +#include + #include "gtest/gtest.h" #include "ray/common/task/task_util.h" From ff6b4855ce52f002e2513aaf645452e8d4f90af6 Mon Sep 17 00:00:00 2001 From: Anmol Singh Date: Mon, 11 Aug 2025 22:00:23 +0530 Subject: [PATCH 014/634] Add RayNodeType label to Prometheus metrics for nodes (#55192) ## Why are these changes needed? We currently don't have an easy way to filter head node metrics in the Ray Grafana dashboard, and this feature can be useful for surfacing head node hardware issues quickly. Also, the current `IsHeadNode` Prometheus label surfaces the underlying boolean conditional rather than the actual node type, making the dashboard filter name and legends look a bit awkward. #### This PR: Replaces `IsHeadNode` with a `RayNodeType` label that uses descriptive values (`"head"/"worker"`), allowing us to * Filter metrics by node type using a Grafana templating variable, so that users can look at just head node metrics when needed * Provide intuitive legends in the time series to help mark the head vs worker nodes * Remove the duplicated queries for head/worker nodes from our base template Note - We use `RayNodeType` instead of more general names like `NodeType` to help distinguish it from labels representing e.g. EC2 node types. #### Changes: * Update node metrics to use `RayNodeType` instead of `IsHeadNode` * Update dashboard Prometheus queries to use the new label and add a new Grafana templating variable * Minor touchups to make the Grafana filter descriptions consistent #### Dashboard screenshots (post changes): **Default dashboard** image **Train dashboard** image ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: anmol Co-authored-by: anmol --- python/ray/dashboard/consts.py | 4 +- .../dashboards/default_dashboard_panels.py | 82 ++++++------------- .../default_grafana_dashboard_base.json | 39 ++++++++- .../dashboards/train_dashboard_panels.py | 70 ++++++++-------- .../train_grafana_dashboard_base.json | 35 +++++++- .../modules/reporter/reporter_agent.py | 6 +- .../modules/reporter/tests/test_reporter.py | 12 ++- python/ray/tests/test_metrics_head.py | 3 + 8 files changed, 152 insertions(+), 99 deletions(-) diff --git a/python/ray/dashboard/consts.py b/python/ray/dashboard/consts.py index 1c5fdb9386b2..30505878cb80 100644 --- a/python/ray/dashboard/consts.py +++ b/python/ray/dashboard/consts.py @@ -65,7 +65,9 @@ # Port that dashboard prometheus metrics will be exported to DASHBOARD_METRIC_PORT = env_integer("DASHBOARD_METRIC_PORT", 44227) -NODE_TAG_KEYS = ["ip", "Version", "SessionName", "IsHeadNode"] +# We use RayNodeType to mark head/worker nodes. IsHeadNode is retained +# for backward compatibility for user-customized dashboards that might rely on it +NODE_TAG_KEYS = ["ip", "Version", "SessionName", "IsHeadNode", "RayNodeType"] GPU_TAG_KEYS = NODE_TAG_KEYS + ["GpuDeviceName", "GpuIndex"] # TpuDeviceName and TpuIndex are expected to be equal to the number of TPU diff --git a/python/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py index dd4e702b34d7..00cfc8194ac8 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py @@ -200,15 +200,11 @@ def max_plus_pending(max_resource, pending_resource): unit="cores", targets=[ Target( - expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", IsHeadNode="false", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance",{global_filters}}} / 100) by (instance)', - legend="CPU Usage: {{instance}}", - ), - Target( - expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", IsHeadNode="true", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance",{global_filters}}} / 100) by (instance)', - legend="CPU Usage: {{instance}} (head)", + expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType",{global_filters}}} / 100) by (instance, RayNodeType)', + legend="CPU Usage: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_cpu_count{{instance=~"$Instance",{global_filters}}})', + expr='sum(ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', legend="MAX", ), ], @@ -220,15 +216,11 @@ def max_plus_pending(max_resource, pending_resource): unit="GPUs", targets=[ Target( - expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", IsHeadNode="false", {global_filters}}} / 100) by (instance, GpuIndex, GpuDeviceName)', - legend="GPU Usage: {{instance}}, gpu.{{GpuIndex}}, {{GpuDeviceName}}", + expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} / 100) by (instance, RayNodeType, GpuIndex, GpuDeviceName)', + legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}", ), Target( - expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", IsHeadNode="true", {global_filters}}} / 100) by (instance, GpuIndex, GpuDeviceName)', - legend="GPU Usage: {{instance}} (head), gpu.{{GpuIndex}}, {{GpuDeviceName}}", - ), - Target( - expr='sum(ray_node_gpus_available{{instance=~"$Instance",{global_filters}}})', + expr='sum(ray_node_gpus_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', legend="MAX", ), ], @@ -240,15 +232,11 @@ def max_plus_pending(max_resource, pending_resource): unit="bytes", targets=[ Target( - expr='sum(ray_node_disk_usage{{instance=~"$Instance", IsHeadNode="false", {global_filters}}}) by (instance)', - legend="Disk Used: {{instance}}", - ), - Target( - expr='sum(ray_node_disk_usage{{instance=~"$Instance", IsHeadNode="true", {global_filters}}}) by (instance)', - legend="Disk Used: {{instance}} (head)", + expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Disk Used: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_disk_free{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}})', + expr='sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', legend="MAX", ), ], @@ -260,20 +248,12 @@ def max_plus_pending(max_resource, pending_resource): unit="Bps", targets=[ Target( - expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", IsHeadNode="false", {global_filters}}}) by (instance)', - legend="Write: {{instance}}", + expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Write: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", IsHeadNode="true", {global_filters}}}) by (instance)', - legend="Write: {{instance}} (head)", - ), - Target( - expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", IsHeadNode="false", {global_filters}}}) by (instance)', - legend="Read: {{instance}}", - ), - Target( - expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", IsHeadNode="true", {global_filters}}}) by (instance)', - legend="Read: {{instance}} (head)", + expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Read: {{instance}} ({{RayNodeType}})", ), ], ), @@ -284,15 +264,11 @@ def max_plus_pending(max_resource, pending_resource): unit="bytes", targets=[ Target( - expr='sum(ray_node_mem_used{{instance=~"$Instance", IsHeadNode="false", {global_filters}}}) by (instance)', - legend="Memory Used: {{instance}}", - ), - Target( - expr='sum(ray_node_mem_used{{instance=~"$Instance", IsHeadNode="true", {global_filters}}}) by (instance)', - legend="Memory Used: {{instance}} (head)", + expr='sum(ray_node_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Memory Used: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_mem_total{{instance=~"$Instance",{global_filters}}})', + expr='sum(ray_node_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', legend="MAX", ), ], @@ -304,12 +280,8 @@ def max_plus_pending(max_resource, pending_resource): unit="%", targets=[ Target( - expr='sum(ray_node_mem_used{{instance=~"$Instance", IsHeadNode="false", {global_filters}}}/ray_node_mem_total{{instance=~"$Instance", IsHeadNode="false", {global_filters}}} * 100) by (instance)', - legend="Memory Used: {{instance}}", - ), - Target( - expr='sum(ray_node_mem_used{{instance=~"$Instance", IsHeadNode="true", {global_filters}}}/ray_node_mem_total{{instance=~"$Instance", IsHeadNode="true", {global_filters}}} * 100) by (instance)', - legend="Memory Used: {{instance}} (head)", + expr='sum(ray_node_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}/ray_node_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * 100) by (instance, RayNodeType)', + legend="Memory Used: {{instance}} ({{RayNodeType}})", ), ], fill=0, @@ -322,8 +294,8 @@ def max_plus_pending(max_resource, pending_resource): unit="failures", targets=[ Target( - expr='sum(ray_memory_manager_worker_eviction_total{{instance=~"$Instance",{global_filters}}}) by (Name, instance)', - legend="OOM Killed: {{Name}}, {{instance}}", + expr='sum(ray_memory_manager_worker_eviction_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (Name, instance, RayNodeType)', + legend="OOM Killed: {{Name}}, {{instance}} ({{RayNodeType}})", ), ], ), @@ -371,11 +343,11 @@ def max_plus_pending(max_resource, pending_resource): unit="bytes", targets=[ Target( - expr='sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}} * 1024 * 1024) by (instance, GpuIndex, GpuDeviceName)', - legend="Used GRAM: {{instance}}, gpu.{{GpuIndex}}, {{GpuDeviceName}}", + expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * 1024 * 1024) by (instance, RayNodeType, GpuIndex, GpuDeviceName)', + legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}", ), Target( - expr='(sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 1024 * 1024', + expr='(sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})) * 1024 * 1024', legend="MAX", ), ], @@ -439,12 +411,12 @@ def max_plus_pending(max_resource, pending_resource): unit="Bps", targets=[ Target( - expr='sum(ray_node_network_receive_speed{{instance=~"$Instance",{global_filters}}}) by (instance)', - legend="Recv: {{instance}}", + expr='sum(ray_node_network_receive_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Recv: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_network_send_speed{{instance=~"$Instance",{global_filters}}}) by (instance)', - legend="Send: {{instance}}", + expr='sum(ray_node_network_send_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Send: {{instance}} ({{RayNodeType}})", ), ], ), diff --git a/python/ray/dashboard/modules/metrics/dashboards/default_grafana_dashboard_base.json b/python/ray/dashboard/modules/metrics/dashboards/default_grafana_dashboard_base.json index 76cf304f21b0..0ad52d3fb128 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/default_grafana_dashboard_base.json +++ b/python/ray/dashboard/modules/metrics/dashboards/default_grafana_dashboard_base.json @@ -47,7 +47,7 @@ }, "datasource": "${datasource}", "definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, SessionName)", - "description": "Filter queries to specific ray sessions.", + "description": "Filter queries to specific Ray sessions.", "error": null, "hide": 0, "includeAll": true, @@ -78,7 +78,7 @@ }, "datasource": "${datasource}", "definition": "label_values(ray_node_network_receive_speed{{SessionName=~\"$SessionName\",{global_filters}}}, instance)", - "description": null, + "description": "Filter queries to specific Ray nodes by their IP address.", "error": null, "hide": 0, "includeAll": true, @@ -106,7 +106,7 @@ }, "datasource": "${datasource}", "definition": "label_values(ray_node_network_receive_speed{{{global_filters}}}, ray_io_cluster)", - "description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automaticaly with Prometheus PodMonitor.", + "description": "Filter queries to specific Ray clusters for KubeRay. When ingesting metrics across multiple Ray clusters, the ray_io_cluster label should be set per cluster. For KubeRay users, this is done automatically with Prometheus PodMonitor.", "error": null, "hide": 0, "includeAll": true, @@ -127,6 +127,39 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "description": "Filter queries to specific Ray node types (head or worker).", + "includeAll": true, + "multi": true, + "name": "RayNodeType", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "Head Node", + "value": "head" + }, + { + "selected": false, + "text": "Worker Node", + "value": "worker" + } + ], + "query": "head, worker", + "type": "custom" } ] }, diff --git a/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py index ed53c44ee4bd..e003c27c82a0 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py @@ -77,11 +77,11 @@ def next(): unit="cores", targets=[ Target( - expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance", {global_filters}}} / 100) by (instance)', - legend="CPU Usage: {{instance}}", + expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} / 100) by (instance, RayNodeType)', + legend="CPU Usage: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_cpu_count{{instance=~"$Instance", {global_filters}}})', + expr='sum(ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', legend="MAX", ), ], @@ -94,11 +94,11 @@ def next(): unit="bytes", targets=[ Target( - expr='sum(ray_node_mem_used{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Memory Used: {{instance}}", + expr='sum(ray_node_mem_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Memory Used: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_mem_total{{instance=~"$Instance", {global_filters}}})', + expr='sum(ray_node_mem_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', legend="MAX", ), ], @@ -111,12 +111,12 @@ def next(): unit="bytes", targets=[ Target( - expr='sum(ray_node_mem_available{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Available Memory: {{instance}}", + expr='sum(ray_node_mem_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Available Memory: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_mem_shared_bytes{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Shared Memory: {{instance}}", + expr='sum(ray_node_mem_shared_bytes{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Shared Memory: {{instance}} ({{RayNodeType}})", ), ], ) @@ -130,11 +130,11 @@ def next(): unit="GPUs", targets=[ Target( - expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}} / 100) by (instance, GpuIndex, GpuDeviceName)', - legend="GPU Usage: {{instance}}, gpu.{{GpuIndex}}, {{GpuDeviceName}}", + expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}} / 100) by (instance, RayNodeType, GpuIndex, GpuDeviceName)', + legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}", ), Target( - expr='sum(ray_node_gpus_available{{instance=~"$Instance", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}})', + expr='sum(ray_node_gpus_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}})', legend="MAX", ), ], @@ -147,11 +147,11 @@ def next(): unit="bytes", targets=[ Target( - expr='sum(ray_node_gram_used{{instance=~"$Instance", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}} * 1024 * 1024) by (instance, GpuIndex, GpuDeviceName)', - legend="Used GRAM: {{instance}}, gpu.{{GpuIndex}}, {{GpuDeviceName}}", + expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}} * 1024 * 1024) by (instance, RayNodeType, GpuIndex, GpuDeviceName)', + legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}", ), Target( - expr='(sum(ray_node_gram_available{{instance=~"$Instance", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}})) * 1024 * 1024', + expr='(sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", GpuIndex=~"$GpuIndex", GpuDeviceName=~"$GpuDeviceName", {global_filters}}})) * 1024 * 1024', legend="MAX", ), ], @@ -165,11 +165,11 @@ def next(): unit="bytes", targets=[ Target( - expr='sum(ray_node_disk_usage{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Disk Used: {{instance}}", + expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Disk Used: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_disk_free{{instance=~"$Instance", {global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance", {global_filters}}})', + expr='sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', legend="MAX", ), ], @@ -182,12 +182,12 @@ def next(): unit="Bps", targets=[ Target( - expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Read Speed: {{instance}}", + expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Read Speed: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Write Speed: {{instance}}", + expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Write Speed: {{instance}} ({{RayNodeType}})", ), ], ) @@ -199,12 +199,12 @@ def next(): unit="ops/s", targets=[ Target( - expr='sum(ray_node_disk_read_iops{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Read IOPS: {{instance}}", + expr='sum(ray_node_disk_read_iops{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Read IOPS: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_disk_write_iops{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Write IOPS: {{instance}}", + expr='sum(ray_node_disk_write_iops{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Write IOPS: {{instance}} ({{RayNodeType}})", ), ], ) @@ -217,12 +217,12 @@ def next(): unit="Bps", targets=[ Target( - expr='sum(ray_node_network_receive_speed{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Receive Speed: {{instance}}", + expr='sum(ray_node_network_receive_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Receive Speed: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_network_send_speed{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Send Speed: {{instance}}", + expr='sum(ray_node_network_send_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Send Speed: {{instance}} ({{RayNodeType}})", ), ], ) @@ -234,12 +234,12 @@ def next(): unit="bytes", targets=[ Target( - expr='sum(ray_node_network_sent{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Total Sent: {{instance}}", + expr='sum(ray_node_network_sent{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Total Sent: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_network_received{{instance=~"$Instance", {global_filters}}}) by (instance)', - legend="Total Received: {{instance}}", + expr='sum(ray_node_network_received{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Total Received: {{instance}} ({{RayNodeType}})", ), ], ) diff --git a/python/ray/dashboard/modules/metrics/dashboards/train_grafana_dashboard_base.json b/python/ray/dashboard/modules/metrics/dashboards/train_grafana_dashboard_base.json index d94d8816ad99..82570ed428e8 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/train_grafana_dashboard_base.json +++ b/python/ray/dashboard/modules/metrics/dashboards/train_grafana_dashboard_base.json @@ -227,8 +227,41 @@ "text": ["All"], "value": ["$__all"] } - } + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "description": "Filter queries to specific Ray node types (head or worker).", + "includeAll": true, + "multi": true, + "name": "RayNodeType", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "Head Node", + "value": "head" + }, + { + "selected": false, + "text": "Worker Node", + "value": "worker" + } + ], + "query": "head, worker", + "type": "custom" + } ] } } diff --git a/python/ray/dashboard/modules/reporter/reporter_agent.py b/python/ray/dashboard/modules/reporter/reporter_agent.py index 877257b78b1e..a94f7d0a5b96 100644 --- a/python/ray/dashboard/modules/reporter/reporter_agent.py +++ b/python/ray/dashboard/modules/reporter/reporter_agent.py @@ -1320,10 +1320,12 @@ def generate_worker_stats_record(self, worker_stats: List[dict]) -> List[Record] def _to_records(self, stats, cluster_stats) -> List[Record]: records_reported = [] ip = stats["ip"] - is_head_node = str(self._is_head_node).lower() + ray_node_type = "head" if self._is_head_node else "worker" + is_head_node = "true" if self._is_head_node else "false" # Common tags for node-level metrics - node_tags = {"ip": ip, "IsHeadNode": is_head_node} + # We use RayNodeType to mark head/worker node, IsHeadNode is retained for backward compatibility + node_tags = {"ip": ip, "RayNodeType": ray_node_type, "IsHeadNode": is_head_node} # -- Instance count of cluster -- # Only report cluster stats on head node diff --git a/python/ray/dashboard/modules/reporter/tests/test_reporter.py b/python/ray/dashboard/modules/reporter/tests/test_reporter.py index 4ea6b4070f2f..28820f370965 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_reporter.py +++ b/python/ray/dashboard/modules/reporter/tests/test_reporter.py @@ -335,9 +335,11 @@ def test_report_stats(): print(record.gauge.name) print(record) assert len(records) == 41 - # Verify IsHeadNode tag + # Verify RayNodeType and IsHeadNode tags for record in records: if record.gauge.name.startswith("node_"): + assert "RayNodeType" in record.tags + assert record.tags["RayNodeType"] == "head" assert "IsHeadNode" in record.tags assert record.tags["IsHeadNode"] == "true" # Test stats without raylets @@ -458,13 +460,19 @@ def test_report_stats_gpu(): index = 0 for record in records: if record.tags["GpuIndex"] == "3": - assert record.tags == {"ip": ip, "GpuIndex": "3", "IsHeadNode": "true"} + assert record.tags == { + "ip": ip, + "GpuIndex": "3", + "IsHeadNode": "true", + "RayNodeType": "head", + } else: assert record.tags == { "ip": ip, # The tag value must be string for prometheus. "GpuIndex": str(index), "GpuDeviceName": "NVIDIA A10G", + "RayNodeType": "head", "IsHeadNode": "true", } diff --git a/python/ray/tests/test_metrics_head.py b/python/ray/tests/test_metrics_head.py index 9ac48c5bb131..6fdcb3306404 100644 --- a/python/ray/tests/test_metrics_head.py +++ b/python/ray/tests/test_metrics_head.py @@ -140,6 +140,9 @@ def test_metrics_folder_with_dashboard_override( for variable in contents["templating"]["list"]: if variable["name"] == "datasource": continue + if variable["name"] == "RayNodeType": + # RayNodeType uses hardcoded values instead of a query + continue assert global_filters in variable["definition"] assert global_filters in variable["query"]["query"] assert "supportsGlobalFilterOverride" in contents["rayMeta"] From 7c7ce12618ca7c94c27c39ef28aeebffd5707dd5 Mon Sep 17 00:00:00 2001 From: Kit Lee <7000003+wingkitlee0@users.noreply.github.com> Date: Mon, 11 Aug 2025 12:56:50 -0400 Subject: [PATCH 015/634] [core] Add as_completed and map_unordered APIs (#53461) ## Why are these changes needed? This PR adds two public APIs: - `as_completed` for "unordered ray.get" - `map_unordered` for "unordered map" As described in #52696 , many new users learned about the `@ray.remote` decorator but failed to scale up their applications. Therefore, it will be helpful to provide some helper functions that follow the design patterns on Ray's documentation: https://docs.ray.io/en/latest/ray-core/patterns/index.html These two functions are marked as alpha Public APIs because ideally they should be public and used by new users. Closes #52696 Signed-off-by: Kit Lee <7000003+wingkitlee0@users.noreply.github.com> --- doc/source/ray-core/api/core.rst | 2 + python/ray/tests/BUILD | 1 + python/ray/tests/test_util_helpers.py | 188 +++++++++++++++++++ python/ray/util/__init__.py | 3 + python/ray/util/helpers.py | 255 ++++++++++++++++++++++++++ 5 files changed, 449 insertions(+) create mode 100644 python/ray/tests/test_util_helpers.py create mode 100644 python/ray/util/helpers.py diff --git a/doc/source/ray-core/api/core.rst b/doc/source/ray-core/api/core.rst index 1a428adbd8c3..b860c87fa6df 100644 --- a/doc/source/ray-core/api/core.rst +++ b/doc/source/ray-core/api/core.rst @@ -50,6 +50,8 @@ Objects ray.get ray.wait ray.put + ray.util.as_completed + ray.util.map_unordered .. _runtime-context-apis: diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD index a2b2f3b002a9..46dccaa12c0e 100644 --- a/python/ray/tests/BUILD +++ b/python/ray/tests/BUILD @@ -67,6 +67,7 @@ py_test_module_list( "test_reference_counting_2.py", "test_reference_counting_standalone.py", "test_runtime_env_agent.py", + "test_util_helpers.py", ], tags = [ "exclusive", diff --git a/python/ray/tests/test_util_helpers.py b/python/ray/tests/test_util_helpers.py new file mode 100644 index 000000000000..265b6931f528 --- /dev/null +++ b/python/ray/tests/test_util_helpers.py @@ -0,0 +1,188 @@ +import pytest +import sys +import ray +from ray.util import as_completed, map_unordered +from ray._common.test_utils import SignalActor + + +@pytest.fixture(scope="module") +def ray_init_4_cpu_shared(): + ray.init(num_cpus=4) + yield + ray.shutdown() + + +@pytest.mark.parametrize("yield_obj_refs", [True, False]) +def test_as_completed_chunk_size_1(ray_init_4_cpu_shared, yield_obj_refs): + """Test as_completed with chunk_size=1. + + Use SignalActor to control task completion order and mimic time.sleep(x) behavior. + + """ + inputs = [10, 8, 6, 4, 2] + + # Create signals for each task + signals = [SignalActor.remote() for _ in range(len(inputs))] + + # Create tasks + @ray.remote + def f(x, signal): + ray.get(signal.wait.remote()) + return x + + # Submit tasks with their corresponding signals in the original order + refs = [f.remote(x, signal) for x, signal in zip(inputs, signals)] + + # Use as_completed() lazily + it = as_completed(refs, chunk_size=1, yield_obj_refs=yield_obj_refs) + + # Send signal in reverse order to mimic time.sleep(x), i.e., + # smallest value releases first. At the same time, collect results + + results = [] + for signal in reversed(signals): + ray.get(signal.send.remote()) + results.append(next(it)) + + if yield_obj_refs: + results = ray.get(results) + + assert results == [2, 4, 6, 8, 10] + + +@pytest.mark.parametrize("yield_obj_refs", [True, False]) +def test_as_completed_chunk_size_2(ray_init_4_cpu_shared, yield_obj_refs): + """Test as_completed with chunk_size=2. + + Use SignalActor to control task completion order and mimic time.sleep(x) behavior. + + """ + inputs = [10, 8, 6, 4, 2] + + # Create signals for each task + signals = [SignalActor.remote() for _ in range(len(inputs))] + + # Create tasks + @ray.remote + def f(x, signal): + ray.get(signal.wait.remote()) + return x + + # Submit tasks with their corresponding signals in the original order + refs = [f.remote(x, signal) for x, signal in zip(inputs, signals)] + + # Use as_completed() lazily + it = as_completed(refs, chunk_size=2, yield_obj_refs=yield_obj_refs) + + # Send signal in reverse order to mimic time.sleep(x), i.e., + # smallest value releases first. At the same time, collect results + + results = [] + + ray.get(signals[4].send.remote()) + ray.get(signals[3].send.remote()) + results.append(next(it)) + results.append(next(it)) + + ray.get(signals[2].send.remote()) + ray.get(signals[1].send.remote()) + results.append(next(it)) + results.append(next(it)) + + ray.get(signals[0].send.remote()) + results.append(next(it)) + + if yield_obj_refs: + results = ray.get(results) + + assert results == [4, 2, 8, 6, 10] + + +@pytest.mark.parametrize("yield_obj_refs", [True, False]) +def test_map_unordered_chunk_size_1(ray_init_4_cpu_shared, yield_obj_refs): + """Test map_unordered with chunk_size=1. + + Use SignalActor to control task completion order and mimic time.sleep(x) behavior. + + """ + inputs = [10, 8, 6, 4, 2] + + # Create signals for each task + signals = [SignalActor.remote() for _ in range(len(inputs))] + + # Create tasks + @ray.remote + def f(args): + x, signal = args + ray.get(signal.wait.remote()) + return x + + # Submit tasks with their corresponding signals in the original order + it = map_unordered( + f, zip(inputs, signals), chunk_size=1, yield_obj_refs=yield_obj_refs + ) + + # Send signal in reverse order to mimic time.sleep(x), i.e., + # smallest value releases first. At the same time, collect results + + results = [] + for signal in reversed(signals): + ray.get(signal.send.remote()) + results.append(next(it)) + + if yield_obj_refs: + results = ray.get(results) + + assert results == [2, 4, 6, 8, 10] + + +@pytest.mark.parametrize("yield_obj_refs", [True, False]) +def test_map_unordered_chunk_size_2(ray_init_4_cpu_shared, yield_obj_refs): + """Test map_unordered with chunk_size=2. + + Use SignalActor to control task completion order and mimic time.sleep(x) behavior. + + """ + inputs = [10, 8, 6, 4, 2] + + # Create signals for each task + signals = [SignalActor.remote() for _ in range(len(inputs))] + + # Create tasks + @ray.remote + def f(args): + x, signal = args + ray.get(signal.wait.remote()) + return x + + # Submit tasks with their corresponding signals in the original order + it = map_unordered( + f, zip(inputs, signals), chunk_size=2, yield_obj_refs=yield_obj_refs + ) + + # Send signal in reverse order to mimic time.sleep(x), i.e., + # smallest value releases first. At the same time, collect results + + results = [] + + ray.get(signals[4].send.remote()) + ray.get(signals[3].send.remote()) + results.append(next(it)) + results.append(next(it)) + + ray.get(signals[2].send.remote()) + ray.get(signals[1].send.remote()) + results.append(next(it)) + results.append(next(it)) + + ray.get(signals[0].send.remote()) + results.append(next(it)) + + if yield_obj_refs: + results = ray.get(results) + + assert results == [4, 2, 8, 6, 10] + + +if __name__ == "__main__": + sys.exit(pytest.main(["-sv", __file__])) diff --git a/python/ray/util/__init__.py b/python/ray/util/__init__.py index bc8b6eae909a..81feec19bcda 100644 --- a/python/ray/util/__init__.py +++ b/python/ray/util/__init__.py @@ -13,6 +13,7 @@ from ray.util.check_serialize import inspect_serializability from ray.util.client_connect import connect, disconnect from ray.util.debug import disable_log_once_globally, enable_periodic_logging, log_once +from ray.util.helpers import as_completed, map_unordered from ray.util.placement_group import ( get_current_placement_group, get_placement_group, @@ -52,6 +53,7 @@ def list_named_actors(all_namespaces: bool = False) -> List[str]: __all__ = [ "accelerators", "ActorPool", + "as_completed", "disable_log_once_globally", "enable_periodic_logging", "iter", @@ -63,6 +65,7 @@ def list_named_actors(all_namespaces: bool = False) -> List[str]: "get_current_placement_group", "get_node_instance_id", "get_node_ip_address", + "map_unordered", "remove_placement_group", "ray_debugpy", "inspect_serializability", diff --git a/python/ray/util/helpers.py b/python/ray/util/helpers.py new file mode 100644 index 000000000000..b53c3c970a74 --- /dev/null +++ b/python/ray/util/helpers.py @@ -0,0 +1,255 @@ +from typing import TYPE_CHECKING, Any, Iterable, Iterator, Optional, Sequence, Union +import ray +from ray.util.annotations import PublicAPI + +if TYPE_CHECKING: + from ray import ObjectRef + from ray.remote_function import RemoteFunction + + +# ray.wait() has a default num_returns of 1. +# Using a slightly larger batch until the optimization is fully implemented, see +# https://github.com/ray-project/ray/issues/49905 +DEFAULT_CHUNK_SIZE = 10 +DEFAULT_BACKPRESSURE_SIZE = 100 + + +def _wait_and_get_single_batch( + refs: "Sequence[ObjectRef]", + *, + chunk_size: int, + yield_obj_refs: bool = False, + **kwargs, +) -> tuple[list[Union[Any, "ObjectRef"]], "list[ObjectRef]"]: + """Call ray.wait and explicitly return the ready objects/results + and remaining Ray remote refs. + + Args: + refs: A list of Ray object refs. + chunk_size: The `num_returns` parameter to pass to `ray.wait()`. + yield_obj_refs: If True, return Ray remote refs instead of results (by calling :meth:`~ray.get`). + **kwargs: Additional keyword arguments to pass to `ray.wait()`. + + Returns: + A tuple of two lists, ready and not ready. This is the same as the return value of `ray.wait()`. + """ + + if chunk_size < 1: + raise ValueError("`chunk_size` must be >= 1") + + kwargs = kwargs or {} + + # num_returns must be <= len(refs) + ready, refs = ray.wait( + refs, + num_returns=min(chunk_size, len(refs)), + **kwargs, + ) + + if not yield_obj_refs: + return ray.get(ready), refs + + return ready, refs + + +@PublicAPI(stability="alpha") +def as_completed( + refs: "Sequence[ObjectRef]", + *, + chunk_size: int = DEFAULT_CHUNK_SIZE, + yield_obj_refs: bool = False, + **kwargs, +) -> Iterator[Union[Any, "ObjectRef"]]: + """Given a list of Ray task references, yield results as they become available. + + Unlike calling :meth:`~ray.get` on a list of references (i.e., `ray.get(refs)`) which + waits for all results to be ready, this function begins to yield result as soon as + a batch of `chunk_size` results are ready. + + .. note:: + Generally there is no guarantee on the order of results. For example, the first result + is not necessarily the first one completed, but rather the first one submitted in the + first available batch (See :meth:`~ray.wait` for more details about + preservation of submission order). + + .. note:: + Use this function instead of calling :meth:`~ray.get` inside a for loop. See + https://docs.ray.io/en/latest/ray-core/patterns/ray-get-loop.html for more details. + + Example: + Suppose we have a function that sleeps for x seconds depending on the input. + We expect to obtain a partially sorted list of results. + + .. testcode:: python + import ray + import time + + @ray.remote + def f(x): + time.sleep(x) + return x + + refs = [f.remote(i) for i in [10, 4, 6, 8, 2]] + for x in ray.util.as_completed(refs, chunk_size=2): + print(x) + + .. testoutput:: + :options: +MOCK + + # Output: + 4 + 2 + 6 + 8 + 10 + + Args: + refs: A list of Ray object refs. + chunk_size: The number of tasks to wait for in each iteration (default 10). + The parameter is passed as `num_returns` to :meth:`~ray.wait` internally. + yield_obj_refs: If True, return Ray remote refs instead of results (by calling :meth:`~ray.get`). + **kwargs: Additional keyword arguments to pass to :meth:`~ray.wait`, e.g., + `timeout` and `fetch_local`. + + Yields: + Union[Any, ObjectRef]: The results (or optionally their Ray references) of the Ray tasks as they complete. + """ + if chunk_size < 1: + raise ValueError("`chunk_size` must be >= 1") + + if "num_returns" in kwargs: + raise ValueError("Use the `chunksize` argument instead of `num_returns`.") + + while refs: + results, refs = _wait_and_get_single_batch( + refs, + chunk_size=chunk_size, + yield_obj_refs=yield_obj_refs, + **kwargs, + ) + yield from results + + +@PublicAPI(stability="alpha") +def map_unordered( + fn: "RemoteFunction", + items: Iterable[Any], + *, + backpressure_size: Optional[int] = DEFAULT_BACKPRESSURE_SIZE, + chunk_size: int = DEFAULT_CHUNK_SIZE, + yield_obj_refs: bool = False, + **kwargs, +) -> Iterator[Union[Any, "ObjectRef"]]: + """Apply a Ray remote function to a list of items and return an iterator that yields + the completed results as they become available. + + This helper function applies backpressure to control the number of pending tasks, following the + design pattern described in + https://docs.ray.io/en/latest/ray-core/patterns/limit-pending-tasks.html. + + .. note:: + There is generally no guarantee on the order of results. + + Example: + Suppose we have a function that sleeps for x seconds depending on the input. + We expect to obtain a partially sorted list of results. + + .. testcode:: python + + import ray + import time + + @ray.remote + def f(x): + time.sleep(x) + return x + + # Example 1: chunk_size=2 + for x in ray.util.map_unordered(f, [10, 4, 6, 8, 2], chunk_size=2): + print(x) + + .. testoutput:: + :options: +MOCK + + 4 + 2 + 6 + 8 + 10 + + .. testcode:: python + + # Example 2: backpressure_size=2, chunk_size=1 + for x in ray.util.map_unordered(f, [10, 4, 6, 8, 2], backpressure_size=2, chunk_size=1): + print(x) + + .. testoutput:: + :options: +MOCK + + 4 + 10 + 6 + 8 + 2 + + Args: + fn: A remote function to apply to the list of items. For more complex use cases, use Ray Data's + :meth:`~ray.data.Dataset.map` / :meth:`~ray.data.Dataset.map_batches` instead. + items: An iterable of items to apply the function to. + backpressure_size: Maximum number of in-flight tasks allowed before + calling a blocking :meth:`~ray.wait` (default 100). If None, no backpressure is applied. + chunk_size: The number of tasks to wait for when the number of in-flight tasks exceeds + `backpressure_size`. The parameter is passed as `num_returns` to :meth:`~ray.wait` internally. + yield_obj_refs: If True, return Ray remote refs instead of results (by calling :meth:`~ray.get`). + **kwargs: Additional keyword arguments to pass to :meth:`~ray.wait`, e.g., + `timeout` and `fetch_local`. + + Yields: + Union[Any, ObjectRef]: The results (or optionally their Ray references) of the Ray tasks as they complete. + + .. seealso:: + + :meth:`~ray.util.as_completed` + Call this method for an existing list of Ray object refs. + + :meth:`~ray.data.Dataset.map` + Use Ray Data APIs (e.g., :meth:`~ray.data.Dataset.map` and :meth:`~ray.data.Dataset.map_batches`) + for better control and complex use cases, e.g., functions with multiple arguments. + + .. note:: + + This is an altenative to `pool.imap_unordered()` in Ray's Actor-based `multiprocessing.Pool`. + See https://docs.ray.io/en/latest/ray-more-libs/multiprocessing.html for more details. + + """ + + if backpressure_size is None: + backpressure_size: float = float("inf") + elif backpressure_size <= 0: + raise ValueError("backpressure_size must be positive.") + + if chunk_size < 1: + raise ValueError("`chunk_size` must be >= 1") + + if "num_returns" in kwargs: + raise ValueError("Use the `chunk_size` argument instead of `num_returns`.") + + refs = [] + for item in items: + refs.append(fn.remote(item)) + + if len(refs) >= backpressure_size: + results, refs = _wait_and_get_single_batch( + refs, + chunk_size=chunk_size, + yield_obj_refs=yield_obj_refs, + **kwargs, + ) + yield from results + else: + yield from as_completed( + refs, + chunk_size=chunk_size, + yield_obj_refs=yield_obj_refs, + **kwargs, + ) From c374ef93782c2a569b1da910143e57fa7957617c Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 11 Aug 2025 12:28:09 -0500 Subject: [PATCH 016/634] [core] Rename `transport/` to `task_submission/` and clean up BUILD targets (#55449) Follow up from: https://github.com/ray-project/ray/pull/55421 This revealed a lot of oddities in our dependency structure in these files. The most glaring is that we have excessive dependencies between actor manager / actor handle / actor creator / actor task submitter. Will try to clean that up sometime. --------- Signed-off-by: Edward Oakes --- src/ray/core_worker/BUILD.bazel | 106 +----------------- src/ray/core_worker/actor_manager.h | 2 +- src/ray/core_worker/core_worker.cc | 24 ---- src/ray/core_worker/core_worker.h | 15 +-- .../core_worker/task_submission/BUILD.bazel | 99 ++++++++++++++++ .../actor_submit_queue.h | 1 - .../actor_task_submitter.cc | 2 +- .../actor_task_submitter.h | 13 +-- .../dependency_resolver.cc | 2 +- .../dependency_resolver.h | 0 .../normal_task_submitter.cc | 27 ++++- .../normal_task_submitter.h | 15 ++- .../out_of_order_actor_submit_queue.cc | 2 +- .../out_of_order_actor_submit_queue.h | 2 +- .../sequential_actor_submit_queue.cc | 2 +- .../sequential_actor_submit_queue.h | 2 +- .../task_submission/test/BUILD.bazel | 77 +++++++++++++ .../test/actor_task_submitter_test.cc | 2 +- .../test/dependency_resolver_test.cc | 5 +- .../test/direct_actor_transport_test.cc} | 2 +- .../test/normal_task_submitter_test.cc | 3 +- .../out_of_order_actor_submit_queue_test.cc} | 4 +- src/ray/core_worker/test/BUILD.bazel | 85 -------------- src/ray/core_worker/test/core_worker_test.cc | 4 +- .../test/object_recovery_manager_test.cc | 1 - 25 files changed, 240 insertions(+), 257 deletions(-) create mode 100644 src/ray/core_worker/task_submission/BUILD.bazel rename src/ray/core_worker/{transport => task_submission}/actor_submit_queue.h (98%) rename src/ray/core_worker/{transport => task_submission}/actor_task_submitter.cc (99%) rename src/ray/core_worker/{transport => task_submission}/actor_task_submitter.h (97%) rename src/ray/core_worker/{transport => task_submission}/dependency_resolver.cc (99%) rename src/ray/core_worker/{transport => task_submission}/dependency_resolver.h (100%) rename src/ray/core_worker/{transport => task_submission}/normal_task_submitter.cc (97%) rename src/ray/core_worker/{transport => task_submission}/normal_task_submitter.h (96%) rename src/ray/core_worker/{transport => task_submission}/out_of_order_actor_submit_queue.cc (97%) rename src/ray/core_worker/{transport => task_submission}/out_of_order_actor_submit_queue.h (97%) rename src/ray/core_worker/{transport => task_submission}/sequential_actor_submit_queue.cc (98%) rename src/ray/core_worker/{transport => task_submission}/sequential_actor_submit_queue.h (97%) create mode 100644 src/ray/core_worker/task_submission/test/BUILD.bazel rename src/ray/core_worker/{ => task_submission}/test/actor_task_submitter_test.cc (99%) rename src/ray/core_worker/{ => task_submission}/test/dependency_resolver_test.cc (98%) rename src/ray/core_worker/{test/direct_actor_transport_mock_test.cc => task_submission/test/direct_actor_transport_test.cc} (99%) rename src/ray/core_worker/{ => task_submission}/test/normal_task_submitter_test.cc (99%) rename src/ray/core_worker/{test/actor_submit_queue_test.cc => task_submission/test/out_of_order_actor_submit_queue_test.cc} (96%) diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 931ad1959ad0..a47df3870715 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -22,7 +22,6 @@ ray_cc_library( ":future_resolver", ":generator_waiter", ":memory_store", - ":normal_task_submitter", ":object_recovery_manager", ":plasma_store_provider", ":profile_event", @@ -32,6 +31,7 @@ ray_cc_library( "//src/ray/common/cgroup:cgroup_manager", "//src/ray/common/cgroup:constants", "//src/ray/core_worker/task_execution:task_receiver", + "//src/ray/core_worker/task_submission:normal_task_submitter", "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/ipc:raylet_ipc_client", @@ -73,25 +73,6 @@ ray_cc_library( ], ) -ray_cc_library( - name = "core_worker_fiber", - hdrs = ["fiber.h"], - deps = [ - "//src/ray/util:logging", - "@boost//:fiber", - ], -) - -ray_cc_library( - name = "actor_submit_queue", - hdrs = ["transport/actor_submit_queue.h"], - deps = [ - "//src/ray/common:id", - "//src/ray/common:task_common", - "@com_google_absl//absl/types:optional", - ], -) - ray_cc_library( name = "common", srcs = ["common.cc"], @@ -150,12 +131,12 @@ ray_cc_library( deps = [ ":actor_creator", ":actor_handle", - ":actor_task_submitter", ":common", ":core_worker_context", ":reference_count", "//src/ray/common:id", "//src/ray/common:task_common", + "//src/ray/core_worker/task_submission:actor_task_submitter", "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/protobuf:core_worker_cc_proto", @@ -169,8 +150,8 @@ ray_cc_library( srcs = ["reference_count.cc"], hdrs = ["reference_count.h"], deps = [ - ":lease_policy", "//src/ray/common:id", + "//src/ray/core_worker:lease_policy", "//src/ray/protobuf:common_cc_proto", "//src/ray/pubsub:publisher", "//src/ray/pubsub:subscriber", @@ -187,6 +168,7 @@ ray_cc_library( name = "lease_policy", srcs = ["lease_policy.cc"], hdrs = ["lease_policy.h"], + visibility = [":__subpackages__"], deps = [ "//src/ray/common:id", "//src/ray/common:task_common", @@ -219,29 +201,6 @@ ray_cc_library( ], ) -ray_cc_library( - name = "out_of_order_actor_submit_queue", - srcs = ["transport/out_of_order_actor_submit_queue.cc"], - hdrs = ["transport/out_of_order_actor_submit_queue.h"], - deps = [ - ":actor_submit_queue", - "//src/ray/common:id", - "@com_google_absl//absl/container:btree", - "@com_google_absl//absl/types:optional", - ], -) - -ray_cc_library( - name = "sequential_actor_submit_queue", - srcs = ["transport/sequential_actor_submit_queue.cc"], - hdrs = ["transport/sequential_actor_submit_queue.h"], - deps = [ - "actor_submit_queue", - "//src/ray/common:id", - "@com_google_absl//absl/types:optional", - ], -) - ray_cc_library( name = "memory_store", srcs = ["store_provider/memory_store/memory_store.cc"], @@ -298,44 +257,6 @@ ray_cc_library( ], ) -ray_cc_library( - name = "dependency_resolver", - srcs = ["transport/dependency_resolver.cc"], - hdrs = ["transport/dependency_resolver.h"], - deps = [ - ":actor_creator", - ":memory_store", - ":task_manager_interface", - "//src/ray/common:id", - "//src/ray/common:task_common", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - ], -) - -ray_cc_library( - name = "actor_task_submitter", - srcs = ["transport/actor_task_submitter.cc"], - hdrs = ["transport/actor_task_submitter.h"], - deps = [ - ":actor_creator", - ":actor_submit_queue", - ":core_worker_context", - ":dependency_resolver", - ":out_of_order_actor_submit_queue", - ":sequential_actor_submit_queue", - "//src/ray/common:asio", - "//src/ray/common:id", - "//src/ray/common:ray_object", - "//src/ray/gcs:gcs_pb_util", - "//src/ray/gcs/gcs_client:gcs_client_lib", - "//src/ray/rpc:core_worker_client", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - ], -) - ray_cc_library( name = "experimental_mutable_object_manager", srcs = ["experimental_mutable_object_manager.cc"], @@ -432,22 +353,3 @@ ray_cc_library( "@com_google_absl//absl/container:flat_hash_set", ], ) - -ray_cc_library( - name = "normal_task_submitter", - srcs = ["transport/normal_task_submitter.cc"], - hdrs = ["transport/normal_task_submitter.h"], - deps = [ - ":actor_manager", - ":core_worker_context", - ":dependency_resolver", - ":lease_policy", - ":memory_store", - ":task_manager", - "//src/ray/common:id", - "//src/ray/gcs:gcs_pb_util", - "//src/ray/raylet_client:raylet_client_lib", - "//src/ray/rpc:core_worker_client", - "@com_google_absl//absl/base:core_headers", - ], -) diff --git a/src/ray/core_worker/actor_manager.h b/src/ray/core_worker/actor_manager.h index 897aa45e5b5b..3d91ca155dbc 100644 --- a/src/ray/core_worker/actor_manager.h +++ b/src/ray/core_worker/actor_manager.h @@ -25,7 +25,7 @@ #include "ray/core_worker/actor_creator.h" #include "ray/core_worker/actor_handle.h" #include "ray/core_worker/reference_count.h" -#include "ray/core_worker/transport/actor_task_submitter.h" +#include "ray/core_worker/task_submission/actor_task_submitter.h" #include "ray/gcs/gcs_client/gcs_client.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 6cebf980861a..634c82975e3b 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -4655,28 +4655,4 @@ void CoreWorker::TaskManagerRetryTask(TaskSpecification &spec, } } -ClusterSizeBasedLeaseRequestRateLimiter::ClusterSizeBasedLeaseRequestRateLimiter( - size_t min_concurrent_lease_limit) - : min_concurrent_lease_cap_(min_concurrent_lease_limit), num_alive_nodes_(0) {} - -size_t ClusterSizeBasedLeaseRequestRateLimiter:: - GetMaxPendingLeaseRequestsPerSchedulingCategory() { - return std::max(min_concurrent_lease_cap_, num_alive_nodes_.load()); -} - -void ClusterSizeBasedLeaseRequestRateLimiter::OnNodeChanges( - const rpc::GcsNodeInfo &data) { - if (data.state() == rpc::GcsNodeInfo::DEAD) { - if (num_alive_nodes_ != 0) { - num_alive_nodes_--; - } else { - RAY_LOG(WARNING) << "Node" << data.node_manager_address() - << " change state to DEAD but num_alive_node is 0."; - } - } else { - num_alive_nodes_++; - } - RAY_LOG_EVERY_MS(INFO, 60000) << "Number of alive nodes:" << num_alive_nodes_.load(); -} - } // namespace ray::core diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index 003a9954b683..d2dc9610cac7 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -46,7 +46,7 @@ #include "ray/core_worker/store_provider/plasma_store_provider.h" #include "ray/core_worker/task_event_buffer.h" #include "ray/core_worker/task_execution/task_receiver.h" -#include "ray/core_worker/transport/normal_task_submitter.h" +#include "ray/core_worker/task_submission/normal_task_submitter.h" #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/ipc/raylet_ipc_client.h" #include "ray/pubsub/publisher.h" @@ -1928,17 +1928,4 @@ class CoreWorker { std::condition_variable gcs_client_node_cache_populated_cv_; bool gcs_client_node_cache_populated_ = false; }; - -// Lease request rate-limiter based on cluster node size. -// It returns max(num_nodes_in_cluster, min_concurrent_lease_limit) -class ClusterSizeBasedLeaseRequestRateLimiter : public LeaseRequestRateLimiter { - public: - explicit ClusterSizeBasedLeaseRequestRateLimiter(size_t min_concurrent_lease_limit); - size_t GetMaxPendingLeaseRequestsPerSchedulingCategory() override; - void OnNodeChanges(const rpc::GcsNodeInfo &data); - - private: - const size_t min_concurrent_lease_cap_; - std::atomic num_alive_nodes_; -}; } // namespace ray::core diff --git a/src/ray/core_worker/task_submission/BUILD.bazel b/src/ray/core_worker/task_submission/BUILD.bazel new file mode 100644 index 000000000000..6cd702f312f6 --- /dev/null +++ b/src/ray/core_worker/task_submission/BUILD.bazel @@ -0,0 +1,99 @@ +load("//bazel:ray.bzl", "ray_cc_library") + +ray_cc_library( + name = "dependency_resolver", + srcs = ["dependency_resolver.cc"], + hdrs = ["dependency_resolver.h"], + visibility = [":__subpackages__"], + deps = [ + "//src/ray/common:id", + "//src/ray/common:task_common", + "//src/ray/core_worker:actor_creator", + "//src/ray/core_worker:lease_policy", + "//src/ray/core_worker:memory_store", + "//src/ray/core_worker:task_manager_interface", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + ], +) + +ray_cc_library( + name = "actor_submit_queue", + hdrs = ["actor_submit_queue.h"], + visibility = ["//visibility:private"], + deps = [ + "//src/ray/common:id", + "//src/ray/common:task_common", + "@com_google_absl//absl/types:optional", + ], +) + +ray_cc_library( + name = "out_of_order_actor_submit_queue", + srcs = ["out_of_order_actor_submit_queue.cc"], + hdrs = ["out_of_order_actor_submit_queue.h"], + visibility = [":__subpackages__"], + deps = [ + ":actor_submit_queue", + "//src/ray/common:id", + "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/types:optional", + ], +) + +ray_cc_library( + name = "sequential_actor_submit_queue", + srcs = ["sequential_actor_submit_queue.cc"], + hdrs = ["sequential_actor_submit_queue.h"], + visibility = [":__subpackages__"], + deps = [ + ":actor_submit_queue", + "//src/ray/common:id", + "@com_google_absl//absl/types:optional", + ], +) + +ray_cc_library( + name = "actor_task_submitter", + srcs = ["actor_task_submitter.cc"], + hdrs = ["actor_task_submitter.h"], + visibility = [ + ":__subpackages__", + "//src/ray/core_worker:__pkg__", + ], + deps = [ + ":actor_submit_queue", + ":dependency_resolver", + ":out_of_order_actor_submit_queue", + ":sequential_actor_submit_queue", + "//src/ray/common:asio", + "//src/ray/common:id", + "//src/ray/core_worker:actor_creator", + "//src/ray/gcs:gcs_pb_util", + "//src/ray/rpc:core_worker_client", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + ], +) + +ray_cc_library( + name = "normal_task_submitter", + srcs = ["normal_task_submitter.cc"], + hdrs = ["normal_task_submitter.h"], + visibility = [ + ":__subpackages__", + "//src/ray/core_worker:__pkg__", + ], + deps = [ + ":dependency_resolver", + "//src/ray/common:id", + "//src/ray/core_worker:lease_policy", + "//src/ray/core_worker:memory_store", + "//src/ray/core_worker:task_manager", + "//src/ray/gcs:gcs_pb_util", + "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/rpc:core_worker_client", + "@com_google_absl//absl/base:core_headers", + ], +) diff --git a/src/ray/core_worker/transport/actor_submit_queue.h b/src/ray/core_worker/task_submission/actor_submit_queue.h similarity index 98% rename from src/ray/core_worker/transport/actor_submit_queue.h rename to src/ray/core_worker/task_submission/actor_submit_queue.h index 0f3dbd6c4182..e84f662a380f 100644 --- a/src/ray/core_worker/transport/actor_submit_queue.h +++ b/src/ray/core_worker/task_submission/actor_submit_queue.h @@ -37,7 +37,6 @@ namespace core { * to know the actual sequence_no to send over the network. * * This class is not thread safe. - * TODO(scv119): the protocol could be improved. */ class IActorSubmitQueue { public: diff --git a/src/ray/core_worker/transport/actor_task_submitter.cc b/src/ray/core_worker/task_submission/actor_task_submitter.cc similarity index 99% rename from src/ray/core_worker/transport/actor_task_submitter.cc rename to src/ray/core_worker/task_submission/actor_task_submitter.cc index 7de64c9e1330..8ff56cd4f150 100644 --- a/src/ray/core_worker/transport/actor_task_submitter.cc +++ b/src/ray/core_worker/task_submission/actor_task_submitter.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/core_worker/transport/actor_task_submitter.h" +#include "ray/core_worker/task_submission/actor_task_submitter.h" #include #include diff --git a/src/ray/core_worker/transport/actor_task_submitter.h b/src/ray/core_worker/task_submission/actor_task_submitter.h similarity index 97% rename from src/ray/core_worker/transport/actor_task_submitter.h rename to src/ray/core_worker/task_submission/actor_task_submitter.h index 8a8350a9f64c..1d699bd3c8ca 100644 --- a/src/ray/core_worker/transport/actor_task_submitter.h +++ b/src/ray/core_worker/task_submission/actor_task_submitter.h @@ -14,8 +14,6 @@ #pragma once -#include -#include #include #include #include @@ -31,15 +29,12 @@ #include "absl/synchronization/mutex.h" #include "ray/common/asio/asio_util.h" #include "ray/common/id.h" -#include "ray/common/ray_object.h" #include "ray/core_worker/actor_creator.h" -#include "ray/core_worker/context.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" -#include "ray/core_worker/transport/actor_submit_queue.h" -#include "ray/core_worker/transport/dependency_resolver.h" -#include "ray/core_worker/transport/out_of_order_actor_submit_queue.h" -#include "ray/core_worker/transport/sequential_actor_submit_queue.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/core_worker/task_submission/actor_submit_queue.h" +#include "ray/core_worker/task_submission/dependency_resolver.h" +#include "ray/core_worker/task_submission/out_of_order_actor_submit_queue.h" +#include "ray/core_worker/task_submission/sequential_actor_submit_queue.h" #include "ray/rpc/worker/core_worker_client.h" namespace ray { diff --git a/src/ray/core_worker/transport/dependency_resolver.cc b/src/ray/core_worker/task_submission/dependency_resolver.cc similarity index 99% rename from src/ray/core_worker/transport/dependency_resolver.cc rename to src/ray/core_worker/task_submission/dependency_resolver.cc index f35fd39175e0..a9600f535465 100644 --- a/src/ray/core_worker/transport/dependency_resolver.cc +++ b/src/ray/core_worker/task_submission/dependency_resolver.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/core_worker/transport/dependency_resolver.h" +#include "ray/core_worker/task_submission/dependency_resolver.h" #include #include diff --git a/src/ray/core_worker/transport/dependency_resolver.h b/src/ray/core_worker/task_submission/dependency_resolver.h similarity index 100% rename from src/ray/core_worker/transport/dependency_resolver.h rename to src/ray/core_worker/task_submission/dependency_resolver.h diff --git a/src/ray/core_worker/transport/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc similarity index 97% rename from src/ray/core_worker/transport/normal_task_submitter.cc rename to src/ray/core_worker/task_submission/normal_task_submitter.cc index b95d61224e0f..47dcd21fe3a7 100644 --- a/src/ray/core_worker/transport/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/core_worker/transport/normal_task_submitter.h" +#include "ray/core_worker/task_submission/normal_task_submitter.h" +#include #include #include #include @@ -831,5 +832,29 @@ bool NormalTaskSubmitter::QueueGeneratorForResubmit(const TaskSpecification &spe return true; } +ClusterSizeBasedLeaseRequestRateLimiter::ClusterSizeBasedLeaseRequestRateLimiter( + size_t min_concurrent_lease_limit) + : min_concurrent_lease_cap_(min_concurrent_lease_limit), num_alive_nodes_(0) {} + +size_t ClusterSizeBasedLeaseRequestRateLimiter:: + GetMaxPendingLeaseRequestsPerSchedulingCategory() { + return std::max(min_concurrent_lease_cap_, num_alive_nodes_.load()); +} + +void ClusterSizeBasedLeaseRequestRateLimiter::OnNodeChanges( + const rpc::GcsNodeInfo &data) { + if (data.state() == rpc::GcsNodeInfo::DEAD) { + if (num_alive_nodes_ != 0) { + num_alive_nodes_--; + } else { + RAY_LOG(WARNING) << "Node" << data.node_manager_address() + << " change state to DEAD but num_alive_node is 0."; + } + } else { + num_alive_nodes_++; + } + RAY_LOG_EVERY_MS(INFO, 60000) << "Number of alive nodes:" << num_alive_nodes_.load(); +} + } // namespace core } // namespace ray diff --git a/src/ray/core_worker/transport/normal_task_submitter.h b/src/ray/core_worker/task_submission/normal_task_submitter.h similarity index 96% rename from src/ray/core_worker/transport/normal_task_submitter.h rename to src/ray/core_worker/task_submission/normal_task_submitter.h index f7fb0bcbe691..b882f9b93d88 100644 --- a/src/ray/core_worker/transport/normal_task_submitter.h +++ b/src/ray/core_worker/task_submission/normal_task_submitter.h @@ -30,7 +30,7 @@ #include "ray/core_worker/lease_policy.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/core_worker/task_manager.h" -#include "ray/core_worker/transport/dependency_resolver.h" +#include "ray/core_worker/task_submission/dependency_resolver.h" #include "ray/raylet_client/raylet_client.h" #include "ray/rpc/node_manager/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client.h" @@ -67,6 +67,19 @@ class StaticLeaseRequestRateLimiter : public LeaseRequestRateLimiter { const size_t kLimit; }; +// Lease request rate-limiter based on cluster node size. +// It returns max(num_nodes_in_cluster, min_concurrent_lease_limit) +class ClusterSizeBasedLeaseRequestRateLimiter : public LeaseRequestRateLimiter { + public: + explicit ClusterSizeBasedLeaseRequestRateLimiter(size_t min_concurrent_lease_limit); + size_t GetMaxPendingLeaseRequestsPerSchedulingCategory() override; + void OnNodeChanges(const rpc::GcsNodeInfo &data); + + private: + const size_t min_concurrent_lease_cap_; + std::atomic num_alive_nodes_; +}; + // This class is thread-safe. class NormalTaskSubmitter { public: diff --git a/src/ray/core_worker/transport/out_of_order_actor_submit_queue.cc b/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.cc similarity index 97% rename from src/ray/core_worker/transport/out_of_order_actor_submit_queue.cc rename to src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.cc index 32a8712e1682..75afa6274b71 100644 --- a/src/ray/core_worker/transport/out_of_order_actor_submit_queue.cc +++ b/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/core_worker/transport/out_of_order_actor_submit_queue.h" +#include "ray/core_worker/task_submission/out_of_order_actor_submit_queue.h" #include #include diff --git a/src/ray/core_worker/transport/out_of_order_actor_submit_queue.h b/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.h similarity index 97% rename from src/ray/core_worker/transport/out_of_order_actor_submit_queue.h rename to src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.h index facbb456775a..26284014a300 100644 --- a/src/ray/core_worker/transport/out_of_order_actor_submit_queue.h +++ b/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.h @@ -20,7 +20,7 @@ #include "absl/container/btree_map.h" #include "absl/types/optional.h" #include "ray/common/id.h" -#include "ray/core_worker/transport/actor_submit_queue.h" +#include "ray/core_worker/task_submission/actor_submit_queue.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/transport/sequential_actor_submit_queue.cc b/src/ray/core_worker/task_submission/sequential_actor_submit_queue.cc similarity index 98% rename from src/ray/core_worker/transport/sequential_actor_submit_queue.cc rename to src/ray/core_worker/task_submission/sequential_actor_submit_queue.cc index e5c676e21258..dc81606fd79f 100644 --- a/src/ray/core_worker/transport/sequential_actor_submit_queue.cc +++ b/src/ray/core_worker/task_submission/sequential_actor_submit_queue.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/core_worker/transport/sequential_actor_submit_queue.h" +#include "ray/core_worker/task_submission/sequential_actor_submit_queue.h" #include #include diff --git a/src/ray/core_worker/transport/sequential_actor_submit_queue.h b/src/ray/core_worker/task_submission/sequential_actor_submit_queue.h similarity index 97% rename from src/ray/core_worker/transport/sequential_actor_submit_queue.h rename to src/ray/core_worker/task_submission/sequential_actor_submit_queue.h index 5559bed185d5..a971bdd9125a 100644 --- a/src/ray/core_worker/transport/sequential_actor_submit_queue.h +++ b/src/ray/core_worker/task_submission/sequential_actor_submit_queue.h @@ -20,7 +20,7 @@ #include "absl/container/btree_map.h" #include "absl/types/optional.h" #include "ray/common/id.h" -#include "ray/core_worker/transport/actor_submit_queue.h" +#include "ray/core_worker/task_submission/actor_submit_queue.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/task_submission/test/BUILD.bazel b/src/ray/core_worker/task_submission/test/BUILD.bazel new file mode 100644 index 000000000000..0c96c0d3c0ae --- /dev/null +++ b/src/ray/core_worker/task_submission/test/BUILD.bazel @@ -0,0 +1,77 @@ +load("//bazel:ray.bzl", "ray_cc_test") + +ray_cc_test( + name = "dep_res_test", + size = "small", + srcs = ["dependency_resolver_test.cc"], + tags = ["team:core"], + deps = [ + "//:ray_mock", + "//src/ray/common:task_common", + "//src/ray/common:test_util", + "//src/ray/core_worker/task_submission:dependency_resolver", + "@com_google_googletest//:gtest", + ], +) + +ray_cc_test( + name = "ooo_submit_queue_test", + size = "small", + srcs = ["out_of_order_actor_submit_queue_test.cc"], + tags = ["team:core"], + deps = [ + "//src/ray/common:asio", + "//src/ray/core_worker/task_submission:out_of_order_actor_submit_queue", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "da_transport_test", + srcs = ["direct_actor_transport_test.cc"], + tags = ["team:core"], + deps = [ + "//:ray_mock", + "//src/ray/core_worker/task_submission:actor_task_submitter", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "at_submitter_test", + srcs = ["actor_task_submitter_test.cc"], + tags = ["team:core"], + deps = [ + "//:ray_mock", + "//src/ray/common:asio", + "//src/ray/common:task_common", + "//src/ray/common:test_util", + "//src/ray/core_worker:actor_creator", + "//src/ray/core_worker:reference_count", + "//src/ray/core_worker:task_manager", + "//src/ray/rpc:core_worker_client", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "nt_submitter_test", + size = "small", + srcs = ["normal_task_submitter_test.cc"], + tags = ["team:core"], + deps = [ + "//:ray_fakes", + "//:ray_mock", + "//src/ray/common:task_common", + "//src/ray/common:test_util", + "//src/ray/core_worker:core_worker_lib", + "//src/ray/core_worker:memory_store", + "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/rpc:core_worker_client", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/core_worker/test/actor_task_submitter_test.cc b/src/ray/core_worker/task_submission/test/actor_task_submitter_test.cc similarity index 99% rename from src/ray/core_worker/test/actor_task_submitter_test.cc rename to src/ray/core_worker/task_submission/test/actor_task_submitter_test.cc index d4bd062a551e..6aad5b7fc53a 100644 --- a/src/ray/core_worker/test/actor_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/test/actor_task_submitter_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/core_worker/transport/actor_task_submitter.h" +#include "ray/core_worker/task_submission/actor_task_submitter.h" #include #include diff --git a/src/ray/core_worker/test/dependency_resolver_test.cc b/src/ray/core_worker/task_submission/test/dependency_resolver_test.cc similarity index 98% rename from src/ray/core_worker/test/dependency_resolver_test.cc rename to src/ray/core_worker/task_submission/test/dependency_resolver_test.cc index e36d721f53d4..011a72e99859 100644 --- a/src/ray/core_worker/test/dependency_resolver_test.cc +++ b/src/ray/core_worker/task_submission/test/dependency_resolver_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/core_worker/transport/dependency_resolver.h" +#include "ray/core_worker/task_submission/dependency_resolver.h" #include #include @@ -27,9 +27,6 @@ #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" #include "ray/common/test_util.h" -#include "ray/core_worker/store_provider/memory_store/memory_store.h" -#include "ray/raylet_client/raylet_client.h" -#include "ray/rpc/worker/core_worker_client.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/test/direct_actor_transport_mock_test.cc b/src/ray/core_worker/task_submission/test/direct_actor_transport_test.cc similarity index 99% rename from src/ray/core_worker/test/direct_actor_transport_mock_test.cc rename to src/ray/core_worker/task_submission/test/direct_actor_transport_test.cc index 8ea082cfa779..b5a7bf44bb77 100644 --- a/src/ray/core_worker/test/direct_actor_transport_mock_test.cc +++ b/src/ray/core_worker/task_submission/test/direct_actor_transport_test.cc @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/core_worker/transport/actor_task_submitter.h" +#include "ray/core_worker/task_submission/actor_task_submitter.h" // clang-format off #include diff --git a/src/ray/core_worker/test/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc similarity index 99% rename from src/ray/core_worker/test/normal_task_submitter_test.cc rename to src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc index cb1ae15f7791..7e5a8b96bc2a 100644 --- a/src/ray/core_worker/test/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/core_worker/transport/normal_task_submitter.h" +#include "ray/core_worker/task_submission/normal_task_submitter.h" #include #include @@ -29,7 +29,6 @@ #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" #include "ray/common/test_util.h" -#include "ray/core_worker/core_worker.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/raylet_client/raylet_client.h" #include "ray/rpc/worker/core_worker_client.h" diff --git a/src/ray/core_worker/test/actor_submit_queue_test.cc b/src/ray/core_worker/task_submission/test/out_of_order_actor_submit_queue_test.cc similarity index 96% rename from src/ray/core_worker/test/actor_submit_queue_test.cc rename to src/ray/core_worker/task_submission/test/out_of_order_actor_submit_queue_test.cc index d12f38ebd0a9..eabbab200fa0 100644 --- a/src/ray/core_worker/test/actor_submit_queue_test.cc +++ b/src/ray/core_worker/task_submission/test/out_of_order_actor_submit_queue_test.cc @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "ray/core_worker/task_submission/out_of_order_actor_submit_queue.h" + #include #include #include #include "gtest/gtest.h" -#include "ray/common/test_util.h" -#include "ray/core_worker/transport/out_of_order_actor_submit_queue.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/test/BUILD.bazel b/src/ray/core_worker/test/BUILD.bazel index c2b327301160..f94145237eb9 100644 --- a/src/ray/core_worker/test/BUILD.bazel +++ b/src/ray/core_worker/test/BUILD.bazel @@ -28,76 +28,6 @@ ray_cc_test( ], ) -ray_cc_test( - name = "actor_task_submitter_test", - srcs = ["actor_task_submitter_test.cc"], - tags = ["team:core"], - deps = [ - "//:ray_mock", - "//src/ray/common:asio", - "//src/ray/common:task_common", - "//src/ray/common:test_util", - "//src/ray/core_worker:actor_creator", - "//src/ray/core_worker:reference_count", - "//src/ray/core_worker:task_manager", - "//src/ray/rpc:core_worker_client", - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "direct_actor_transport_mock_test", - srcs = ["direct_actor_transport_mock_test.cc"], - tags = ["team:core"], - deps = [ - "//:ray_mock", - "//src/ray/core_worker:memory_store", - "//src/ray/core_worker:reference_count", - "//src/ray/core_worker:task_manager", - "//src/ray/gcs/gcs_client:gcs_client_lib", - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "dependency_resolver_test", - size = "small", - srcs = ["dependency_resolver_test.cc"], - tags = ["team:core"], - deps = [ - "//:ray_mock", - "//src/ray/common:task_common", - "//src/ray/common:test_util", - "//src/ray/core_worker:dependency_resolver", - "//src/ray/core_worker:memory_store", - "//src/ray/raylet_client:raylet_client_lib", - "//src/ray/rpc:core_worker_client", - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "normal_task_submitter_test", - size = "small", - srcs = ["normal_task_submitter_test.cc"], - tags = ["team:core"], - deps = [ - "//:ray_fakes", - "//:ray_mock", - "//src/ray/common:task_common", - "//src/ray/common:test_util", - "//src/ray/core_worker:core_worker_lib", - "//src/ray/core_worker:memory_store", - "//src/ray/raylet_client:raylet_client_lib", - "//src/ray/rpc:core_worker_client", - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - ], -) - ray_cc_test( name = "reference_count_test", size = "small", @@ -127,7 +57,6 @@ ray_cc_test( "//src/ray/common:task_common", "//src/ray/common:test_util", "//src/ray/core_worker:memory_store", - "//src/ray/core_worker:normal_task_submitter", "//src/ray/core_worker:object_recovery_manager", "//src/ray/object_manager:object_manager_common", "//src/ray/raylet_client:raylet_client_lib", @@ -136,20 +65,6 @@ ray_cc_test( ], ) -ray_cc_test( - name = "actor_submit_queue_test", - size = "small", - srcs = ["actor_submit_queue_test.cc"], - tags = ["team:core"], - deps = [ - "//src/ray/common:asio", - "//src/ray/common:test_util", - "//src/ray/core_worker:out_of_order_actor_submit_queue", - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - ], -) - ray_cc_test( name = "task_manager_test", size = "small", diff --git a/src/ray/core_worker/test/core_worker_test.cc b/src/ray/core_worker/test/core_worker_test.cc index 1979d217aeac..1a667638c64d 100644 --- a/src/ray/core_worker/test/core_worker_test.cc +++ b/src/ray/core_worker/test/core_worker_test.cc @@ -38,8 +38,8 @@ #include "ray/core_worker/object_recovery_manager.h" #include "ray/core_worker/reference_count.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" -#include "ray/core_worker/transport/actor_task_submitter.h" -#include "ray/core_worker/transport/normal_task_submitter.h" +#include "ray/core_worker/task_submission/actor_task_submitter.h" +#include "ray/core_worker/task_submission/normal_task_submitter.h" #include "ray/rpc/worker/core_worker_client_pool.h" namespace ray { diff --git a/src/ray/core_worker/test/object_recovery_manager_test.cc b/src/ray/core_worker/test/object_recovery_manager_test.cc index 12317e359692..44e6f277a6e0 100644 --- a/src/ray/core_worker/test/object_recovery_manager_test.cc +++ b/src/ray/core_worker/test/object_recovery_manager_test.cc @@ -30,7 +30,6 @@ #include "ray/common/task/task_util.h" #include "ray/common/test_util.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" -#include "ray/core_worker/transport/normal_task_submitter.h" #include "ray/raylet_client/raylet_client.h" namespace ray { From de10267e2eb1943469c230df440d77ead9cc9cce Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Mon, 11 Aug 2025 10:28:25 -0700 Subject: [PATCH 017/634] [core][raycheck/02] core_worker_process already initialized (#54906) This PR fixes RAY_CHECK failures with the message !core_worker_process: The process is already initialized for core worker. These failures can occur when ray.init is called concurrently with itself or when ray.init and ray.shutdown are invoked concurrently from separate threads. **Case 1:** When ray.shutdown is called, it disconnects and begins destructing the driver. If ray.init/connect is triggered before the driver is fully destructed (and it might because the driver already disconnected from the cluster), it can lead to re-initialization of core_worker_process while it's still in a partially destructed state. **Case 2:** When ray.init/connect is called concurrently from two threads, both can attempt to initialize core_worker_process, resulting in a double initialization. To prevent these race conditions, this PR introduces a lock around ray.connect and ray.shutdown, ensuring they cannot interleave. Since these functions are meant to be called only once during the cluster lifecycle and should not overlap, this locking mechanism does not introduce performance overhead. Test: - CI Signed-off-by: Cuong Nguyen --- python/ray/_private/worker.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python/ray/_private/worker.py b/python/ray/_private/worker.py index e7fa2ef7782d..37879b15ba25 100644 --- a/python/ray/_private/worker.py +++ b/python/ray/_private/worker.py @@ -16,6 +16,7 @@ from collections.abc import Mapping from contextlib import contextmanager from dataclasses import dataclass +from functools import wraps from typing import ( TYPE_CHECKING, Any, @@ -1086,6 +1087,18 @@ def get_accelerator_ids_for_accelerator_resource( return list(assigned_ids) +_connect_or_shutdown_lock = threading.RLock() + + +def with_connect_or_shutdown_lock(func: Callable) -> Callable: + @wraps(func) + def wrapper(*args, **kwargs): + with _connect_or_shutdown_lock: + return func(*args, **kwargs) + + return wrapper + + @PublicAPI @client_mode_hook def get_gpu_ids() -> Union[List[int], List[str]]: @@ -1979,6 +1992,7 @@ def sigterm_handler(signum, frame): @PublicAPI @client_mode_hook +@with_connect_or_shutdown_lock def shutdown(_exiting_interpreter: bool = False): """Disconnect the worker, and terminate processes started by ray.init(). @@ -2356,6 +2370,7 @@ def is_initialized() -> bool: # TODO(hjiang): Add cgroup path along with [enable_resource_isolation]. +@with_connect_or_shutdown_lock def connect( node, session_name: str, From e35657ad5938a117d9456ae9e70a522b253e7159 Mon Sep 17 00:00:00 2001 From: Jason Li <57246540+JasonLi1909@users.noreply.github.com> Date: Mon, 11 Aug 2025 10:46:44 -0700 Subject: [PATCH 018/634] Fix Ray Train Docs Code Snippet Mis-highlighting (#54909) The Ray Train code snippets for pytorch getting started and xgboost getting started are mis-highlighted, drawing the readers' attention to the incorrect sections of code. This PR is to clean the mis-highlights and to add any that are missing. I also change the ordering of the tabs for all four getting started pages (pytorch, pytorch lightning, hugging face transformers, xgboost) so that the "+ Ray" tabs show first. --------- Signed-off-by: JasonLi1909 Signed-off-by: Jason Li <57246540+JasonLi1909@users.noreply.github.com> Co-authored-by: matthewdeng Co-authored-by: matthewdeng --- .../getting-started-pytorch-lightning.rst | 95 +++++++------- doc/source/train/getting-started-pytorch.rst | 100 ++++++++------- .../train/getting-started-transformers.rst | 119 +++++++++--------- doc/source/train/getting-started-xgboost.rst | 2 +- .../data-loading-preprocessing.rst | 4 +- .../train/user-guides/experiment-tracking.rst | 4 +- 6 files changed, 160 insertions(+), 164 deletions(-) diff --git a/doc/source/train/getting-started-pytorch-lightning.rst b/doc/source/train/getting-started-pytorch-lightning.rst index 9a24b8d77b82..96768a1ad9d2 100644 --- a/doc/source/train/getting-started-pytorch-lightning.rst +++ b/doc/source/train/getting-started-pytorch-lightning.rst @@ -38,54 +38,6 @@ Compare a PyTorch Lightning training script with and without Ray Train. .. tab-set:: - .. tab-item:: PyTorch Lightning - - .. This snippet isn't tested because it doesn't use any Ray code. - - .. testcode:: - :skipif: True - - import torch - from torchvision.models import resnet18 - from torchvision.datasets import FashionMNIST - from torchvision.transforms import ToTensor, Normalize, Compose - from torch.utils.data import DataLoader - import lightning.pytorch as pl - - # Model, Loss, Optimizer - class ImageClassifier(pl.LightningModule): - def __init__(self): - super(ImageClassifier, self).__init__() - self.model = resnet18(num_classes=10) - self.model.conv1 = torch.nn.Conv2d( - 1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False - ) - self.criterion = torch.nn.CrossEntropyLoss() - - def forward(self, x): - return self.model(x) - - def training_step(self, batch, batch_idx): - x, y = batch - outputs = self.forward(x) - loss = self.criterion(outputs, y) - self.log("loss", loss, on_step=True, prog_bar=True) - return loss - - def configure_optimizers(self): - return torch.optim.Adam(self.model.parameters(), lr=0.001) - - # Data - transform = Compose([ToTensor(), Normalize((0.28604,), (0.32025,))]) - train_data = FashionMNIST(root='./data', train=True, download=True, transform=transform) - train_dataloader = DataLoader(train_data, batch_size=128, shuffle=True) - - # Training - model = ImageClassifier() - trainer = pl.Trainer(max_epochs=10) - trainer.fit(model, train_dataloaders=train_dataloader) - - .. tab-item:: PyTorch Lightning + Ray Train .. code-block:: python @@ -175,6 +127,53 @@ Compare a PyTorch Lightning training script with and without Ray Train. ), ) + .. tab-item:: PyTorch Lightning + + .. This snippet isn't tested because it doesn't use any Ray code. + + .. testcode:: + :skipif: True + + import torch + from torchvision.models import resnet18 + from torchvision.datasets import FashionMNIST + from torchvision.transforms import ToTensor, Normalize, Compose + from torch.utils.data import DataLoader + import lightning.pytorch as pl + + # Model, Loss, Optimizer + class ImageClassifier(pl.LightningModule): + def __init__(self): + super(ImageClassifier, self).__init__() + self.model = resnet18(num_classes=10) + self.model.conv1 = torch.nn.Conv2d( + 1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False + ) + self.criterion = torch.nn.CrossEntropyLoss() + + def forward(self, x): + return self.model(x) + + def training_step(self, batch, batch_idx): + x, y = batch + outputs = self.forward(x) + loss = self.criterion(outputs, y) + self.log("loss", loss, on_step=True, prog_bar=True) + return loss + + def configure_optimizers(self): + return torch.optim.Adam(self.model.parameters(), lr=0.001) + + # Data + transform = Compose([ToTensor(), Normalize((0.28604,), (0.32025,))]) + train_data = FashionMNIST(root='./data', train=True, download=True, transform=transform) + train_dataloader = DataLoader(train_data, batch_size=128, shuffle=True) + + # Training + model = ImageClassifier() + trainer = pl.Trainer(max_epochs=10) + trainer.fit(model, train_dataloaders=train_dataloader) + Set up a training function -------------------------- diff --git a/doc/source/train/getting-started-pytorch.rst b/doc/source/train/getting-started-pytorch.rst index 8a225d34f9d0..6d28c5df3309 100644 --- a/doc/source/train/getting-started-pytorch.rst +++ b/doc/source/train/getting-started-pytorch.rst @@ -40,60 +40,10 @@ Compare a PyTorch training script with and without Ray Train. .. tab-set:: - .. tab-item:: PyTorch - - .. This snippet isn't tested because it doesn't use any Ray code. - - .. testcode:: - :skipif: True - - import os - import tempfile - - import torch - from torch.nn import CrossEntropyLoss - from torch.optim import Adam - from torch.utils.data import DataLoader - from torchvision.models import resnet18 - from torchvision.datasets import FashionMNIST - from torchvision.transforms import ToTensor, Normalize, Compose - - # Model, Loss, Optimizer - model = resnet18(num_classes=10) - model.conv1 = torch.nn.Conv2d( - 1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False - ) - model.to("cuda") - criterion = CrossEntropyLoss() - optimizer = Adam(model.parameters(), lr=0.001) - - # Data - transform = Compose([ToTensor(), Normalize((0.28604,), (0.32025,))]) - train_data = FashionMNIST(root='./data', train=True, download=True, transform=transform) - train_loader = DataLoader(train_data, batch_size=128, shuffle=True) - - # Training - for epoch in range(10): - for images, labels in train_loader: - images, labels = images.to("cuda"), labels.to("cuda") - outputs = model(images) - loss = criterion(outputs, labels) - optimizer.zero_grad() - loss.backward() - optimizer.step() - - metrics = {"loss": loss.item(), "epoch": epoch} - checkpoint_dir = tempfile.mkdtemp() - checkpoint_path = os.path.join(checkpoint_dir, "model.pt") - torch.save(model.state_dict(), checkpoint_path) - print(metrics) - - - .. tab-item:: PyTorch + Ray Train .. code-block:: python - :emphasize-lines: 12, 14, 21, 55-58, 59, 63, 66-68, 72-73, 76 + :emphasize-lines: 12, 14, 21, 32, 36-37, 55-58, 59, 63, 66-73 import os import tempfile @@ -179,6 +129,54 @@ Compare a PyTorch training script with and without Ray Train. ) model.load_state_dict(model_state_dict) + .. tab-item:: PyTorch + + .. This snippet isn't tested because it doesn't use any Ray code. + + .. testcode:: + :skipif: True + + import os + import tempfile + + import torch + from torch.nn import CrossEntropyLoss + from torch.optim import Adam + from torch.utils.data import DataLoader + from torchvision.models import resnet18 + from torchvision.datasets import FashionMNIST + from torchvision.transforms import ToTensor, Normalize, Compose + + # Model, Loss, Optimizer + model = resnet18(num_classes=10) + model.conv1 = torch.nn.Conv2d( + 1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False + ) + model.to("cuda") + criterion = CrossEntropyLoss() + optimizer = Adam(model.parameters(), lr=0.001) + + # Data + transform = Compose([ToTensor(), Normalize((0.28604,), (0.32025,))]) + train_data = FashionMNIST(root='./data', train=True, download=True, transform=transform) + train_loader = DataLoader(train_data, batch_size=128, shuffle=True) + + # Training + for epoch in range(10): + for images, labels in train_loader: + images, labels = images.to("cuda"), labels.to("cuda") + outputs = model(images) + loss = criterion(outputs, labels) + optimizer.zero_grad() + loss.backward() + optimizer.step() + + metrics = {"loss": loss.item(), "epoch": epoch} + checkpoint_dir = tempfile.mkdtemp() + checkpoint_path = os.path.join(checkpoint_dir, "model.pt") + torch.save(model.state_dict(), checkpoint_path) + print(metrics) + Set up a training function -------------------------- diff --git a/doc/source/train/getting-started-transformers.rst b/doc/source/train/getting-started-transformers.rst index c07215e58ef8..a7beae254d13 100644 --- a/doc/source/train/getting-started-transformers.rst +++ b/doc/source/train/getting-started-transformers.rst @@ -54,66 +54,6 @@ Compare a standard Hugging Face Transformers script with its Ray Train equivalen .. tab-set:: - .. tab-item:: Hugging Face Transformers - - .. This snippet isn't tested because it doesn't use any Ray code. - - .. testcode:: - :skipif: True - - # Adapted from Hugging Face tutorial: https://huggingface.co/docs/transformers/training - - import numpy as np - import evaluate - from datasets import load_dataset - from transformers import ( - Trainer, - TrainingArguments, - AutoTokenizer, - AutoModelForSequenceClassification, - ) - - # Datasets - dataset = load_dataset("yelp_review_full") - tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") - - def tokenize_function(examples): - return tokenizer(examples["text"], padding="max_length", truncation=True) - - small_train_dataset = dataset["train"].select(range(100)).map(tokenize_function, batched=True) - small_eval_dataset = dataset["test"].select(range(100)).map(tokenize_function, batched=True) - - # Model - model = AutoModelForSequenceClassification.from_pretrained( - "bert-base-cased", num_labels=5 - ) - - # Metrics - metric = evaluate.load("accuracy") - - def compute_metrics(eval_pred): - logits, labels = eval_pred - predictions = np.argmax(logits, axis=-1) - return metric.compute(predictions=predictions, references=labels) - - # Hugging Face Trainer - training_args = TrainingArguments( - output_dir="test_trainer", evaluation_strategy="epoch", report_to="none" - ) - - trainer = Trainer( - model=model, - args=training_args, - train_dataset=small_train_dataset, - eval_dataset=small_eval_dataset, - compute_metrics=compute_metrics, - ) - - # Start Training - trainer.train() - - - .. tab-item:: Hugging Face Transformers + Ray Train .. code-block:: python @@ -216,6 +156,65 @@ Compare a standard Hugging Face Transformers script with its Ray Train equivalen model = AutoModelForSequenceClassification.from_pretrained(checkpoint_path) + .. tab-item:: Hugging Face Transformers + + .. This snippet isn't tested because it doesn't use any Ray code. + + .. testcode:: + :skipif: True + + # Adapted from Hugging Face tutorial: https://huggingface.co/docs/transformers/training + + import numpy as np + import evaluate + from datasets import load_dataset + from transformers import ( + Trainer, + TrainingArguments, + AutoTokenizer, + AutoModelForSequenceClassification, + ) + + # Datasets + dataset = load_dataset("yelp_review_full") + tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + + def tokenize_function(examples): + return tokenizer(examples["text"], padding="max_length", truncation=True) + + small_train_dataset = dataset["train"].select(range(100)).map(tokenize_function, batched=True) + small_eval_dataset = dataset["test"].select(range(100)).map(tokenize_function, batched=True) + + # Model + model = AutoModelForSequenceClassification.from_pretrained( + "bert-base-cased", num_labels=5 + ) + + # Metrics + metric = evaluate.load("accuracy") + + def compute_metrics(eval_pred): + logits, labels = eval_pred + predictions = np.argmax(logits, axis=-1) + return metric.compute(predictions=predictions, references=labels) + + # Hugging Face Trainer + training_args = TrainingArguments( + output_dir="test_trainer", evaluation_strategy="epoch", report_to="none" + ) + + trainer = Trainer( + model=model, + args=training_args, + train_dataset=small_train_dataset, + eval_dataset=small_eval_dataset, + compute_metrics=compute_metrics, + ) + + # Start Training + trainer.train() + + Set up a training function -------------------------- diff --git a/doc/source/train/getting-started-xgboost.rst b/doc/source/train/getting-started-xgboost.rst index f4568f221ba6..983dc5138648 100644 --- a/doc/source/train/getting-started-xgboost.rst +++ b/doc/source/train/getting-started-xgboost.rst @@ -41,6 +41,7 @@ Compare a XGBoost training script with and without Ray Train. .. tab-item:: XGBoost + Ray Train .. literalinclude:: ./doc_code/xgboost_quickstart.py + :emphasize-lines: 3-4, 7-8, 11, 15-16, 19-20, 48, 53, 56-64 :language: python :start-after: __xgboost_ray_start__ :end-before: __xgboost_ray_end__ @@ -53,7 +54,6 @@ Compare a XGBoost training script with and without Ray Train. :end-before: __xgboost_end__ - Set up a training function -------------------------- diff --git a/doc/source/train/user-guides/data-loading-preprocessing.rst b/doc/source/train/user-guides/data-loading-preprocessing.rst index e82ea87c9bb8..5db4669578ad 100644 --- a/doc/source/train/user-guides/data-loading-preprocessing.rst +++ b/doc/source/train/user-guides/data-loading-preprocessing.rst @@ -45,7 +45,7 @@ Data ingestion can be set up with four basic steps: .. tab-item:: PyTorch .. code-block:: python - :emphasize-lines: 14,21,29,31-33,53 + :emphasize-lines: 14,21,29,33-35,53 import torch import ray @@ -149,7 +149,7 @@ Data ingestion can be set up with four basic steps: .. tab-item:: HuggingFace Transformers .. code-block:: python - :emphasize-lines: 7-8,13-14,17-18,30-31,41 + :emphasize-lines: 7-8,13-14,17-18,24,30-31,41 import ray import ray.train diff --git a/doc/source/train/user-guides/experiment-tracking.rst b/doc/source/train/user-guides/experiment-tracking.rst index 5feb082a43e9..e80d67bb79c5 100644 --- a/doc/source/train/user-guides/experiment-tracking.rst +++ b/doc/source/train/user-guides/experiment-tracking.rst @@ -304,14 +304,14 @@ PyTorch .. dropdown:: Log to W&B .. literalinclude:: ../../../../python/ray/train/examples/experiment_tracking//torch_exp_tracking_wandb.py - :emphasize-lines: 15, 16, 17, 21, 22, 51, 52, 54, 55 + :emphasize-lines: 16, 19-21, 59-60, 62-63 :language: python :start-after: __start__ .. dropdown:: Log to file-based MLflow .. literalinclude:: ../../../../python/ray/train/examples/experiment_tracking/torch_exp_tracking_mlflow.py - :emphasize-lines: 22, 23, 24, 25, 54, 55, 57, 58, 64 + :emphasize-lines: 22-25, 58-59, 61-62, 68 :language: python :start-after: __start__ :end-before: __end__ From f9373d2f9eca5659f326eb6ea971fdd34a706d3b Mon Sep 17 00:00:00 2001 From: Mengqing Cao Date: Tue, 12 Aug 2025 01:59:59 +0800 Subject: [PATCH 019/634] [Build][Bugfix] Fix protobuf version (#54910) This pr fixes the `AttributeError: 'str' object has no attribute 'DESCRIPTOR'` when packaging message to dict using `protobuf` Closes #54849 Signed-off-by: MengqingCao Co-authored-by: Edward Oakes --- python/requirements.txt | 2 +- python/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/requirements.txt b/python/requirements.txt index 709e744bb0da..eb70dadc5f4e 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -13,7 +13,7 @@ filelock jsonschema msgpack<2.0.0,>=1.0.0 packaging -protobuf!=3.19.5,>=3.15.3 +protobuf>=3.20.3 pyyaml requests watchfiles diff --git a/python/setup.py b/python/setup.py index 7044291c4edd..71bd4a54a810 100644 --- a/python/setup.py +++ b/python/setup.py @@ -383,7 +383,7 @@ def get_packages(self): "jsonschema", "msgpack >= 1.0.0, < 2.0.0", "packaging", - "protobuf >= 3.15.3, != 3.19.5", + "protobuf>=3.20.3", "pyyaml", "requests", ] From a58eaf7474c0e56ffa08230d44a5d27b7084e3fd Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Mon, 11 Aug 2025 11:00:25 -0700 Subject: [PATCH 020/634] [ci] moving workspace tests to test_workspace file (#55403) descendant of https://github.com/ray-project/ray/pull/55404 - refactoring unit tests - moving workspace tests into test_workspace - Updating bazel build file --------- Signed-off-by: elliot-barn --- ci/raydepsets/BUILD.bazel | 37 ++++++++++----- ci/raydepsets/{ => tests}/test_cli.py | 28 ++---------- .../requirement_constraints_test.txt | 0 .../test_data/requirements_compiled_test.txt | 0 .../requirements_compiled_test_expand.txt | 0 .../requirements_compiled_test_update.txt | 0 .../test_data/requirements_test.txt | 0 .../{ => tests}/test_data/test.depsets.yaml | 0 ci/raydepsets/tests/test_workspace.py | 35 +++++++++++++++ ci/raydepsets/tests/utils.py | 45 +++++++++++++++++++ 10 files changed, 110 insertions(+), 35 deletions(-) rename ci/raydepsets/{ => tests}/test_cli.py (93%) rename ci/raydepsets/{ => tests}/test_data/requirement_constraints_test.txt (100%) rename ci/raydepsets/{ => tests}/test_data/requirements_compiled_test.txt (100%) rename ci/raydepsets/{ => tests}/test_data/requirements_compiled_test_expand.txt (100%) rename ci/raydepsets/{ => tests}/test_data/requirements_compiled_test_update.txt (100%) rename ci/raydepsets/{ => tests}/test_data/requirements_test.txt (100%) rename ci/raydepsets/{ => tests}/test_data/test.depsets.yaml (100%) create mode 100644 ci/raydepsets/tests/test_workspace.py create mode 100644 ci/raydepsets/tests/utils.py diff --git a/ci/raydepsets/BUILD.bazel b/ci/raydepsets/BUILD.bazel index d69d976b5d1f..eb4d56c06a20 100644 --- a/ci/raydepsets/BUILD.bazel +++ b/ci/raydepsets/BUILD.bazel @@ -33,14 +33,14 @@ py_binary( py_test( name = "test_cli", - srcs = ["test_cli.py"], + srcs = ["tests/test_cli.py"], data = [ - "test_data/requirement_constraints_test.txt", - "test_data/requirements_compiled_test.txt", - "test_data/requirements_compiled_test_expand.txt", - "test_data/requirements_compiled_test_update.txt", - "test_data/requirements_test.txt", - "test_data/test.depsets.yaml", + "tests/test_data/requirement_constraints_test.txt", + "tests/test_data/requirements_compiled_test.txt", + "tests/test_data/requirements_compiled_test_expand.txt", + "tests/test_data/requirements_compiled_test_update.txt", + "tests/test_data/requirements_test.txt", + "tests/test_data/test.depsets.yaml", ], exec_compatible_with = ["//:hermetic_python"], tags = [ @@ -50,15 +50,32 @@ py_test( deps = [ ci_require("pytest"), ":raydepsets_lib", - ":testing_utils", + ":utils", ], ) py_library( - name = "testing_utils", + name = "utils", testonly = True, - srcs = ["testing_utils.py"], + srcs = ["tests/utils.py"], deps = [ ci_require("bazel-runfiles"), ], ) + +py_test( + name = "test_workspace", + srcs = ["tests/test_workspace.py"], + data = [ + "tests/test_data/test.depsets.yaml", + ], + tags = [ + "ci_unit", + "team:ci", + ], + deps = [ + ci_require("pytest"), + ":utils", + ":workspace", + ], +) diff --git a/ci/raydepsets/test_cli.py b/ci/raydepsets/tests/test_cli.py similarity index 93% rename from ci/raydepsets/test_cli.py rename to ci/raydepsets/tests/test_cli.py index 1db2887d31fe..2aac8e132af3 100644 --- a/ci/raydepsets/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -21,14 +21,13 @@ _uv_binary, load, ) -from ci.raydepsets.testing_utils import ( +from ci.raydepsets.tests.utils import ( append_to_file, copy_data_to_tmpdir, replace_in_file, save_file_as, save_packages_to_file, ) -from ci.raydepsets.workspace import Workspace _REPO_NAME = "com_github_ray_project_ray" _runfiles = runfiles.Create() @@ -48,11 +47,6 @@ def _create_test_manager( class TestCli(unittest.TestCase): - def test_workspace_init(self): - with tempfile.TemporaryDirectory() as tmpdir: - workspace = Workspace(tmpdir) - assert workspace.dir is not None - def test_cli_load_fail_no_config(self): result = CliRunner().invoke( load, @@ -103,12 +97,12 @@ def test_uv_version(self): def test_compile(self): compiled_file = Path( _runfiles.Rlocation( - f"{_REPO_NAME}/ci/raydepsets/test_data/requirements_compiled_test.txt" + f"{_REPO_NAME}/ci/raydepsets/tests/test_data/requirements_compiled_test.txt" ) ) output_file = Path( _runfiles.Rlocation( - f"{_REPO_NAME}/ci/raydepsets/test_data/requirements_compiled.txt" + f"{_REPO_NAME}/ci/raydepsets/tests/test_data/requirements_compiled.txt" ) ) shutil.copy(compiled_file, output_file) @@ -458,22 +452,6 @@ def test_expand_with_requirements(self): output_text_valid = output_file_valid.read_text() assert output_text == output_text_valid - def test_parse_build_arg_sets(self): - with tempfile.TemporaryDirectory() as tmpdir: - copy_data_to_tmpdir(tmpdir) - workspace = Workspace(dir=tmpdir) - config = workspace.load_config(path=Path(tmpdir) / "test.depsets.yaml") - assert config.build_arg_sets[0].name == "py311_cpu" - assert config.build_arg_sets[0].build_args == { - "CUDA_VERSION": "cpu", - "PYTHON_VERSION": "py311", - } - assert config.build_arg_sets[1].name == "py311_cuda128" - assert config.build_arg_sets[1].build_args == { - "CUDA_VERSION": 128, - "PYTHON_VERSION": "py311", - } - if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/ci/raydepsets/test_data/requirement_constraints_test.txt b/ci/raydepsets/tests/test_data/requirement_constraints_test.txt similarity index 100% rename from ci/raydepsets/test_data/requirement_constraints_test.txt rename to ci/raydepsets/tests/test_data/requirement_constraints_test.txt diff --git a/ci/raydepsets/test_data/requirements_compiled_test.txt b/ci/raydepsets/tests/test_data/requirements_compiled_test.txt similarity index 100% rename from ci/raydepsets/test_data/requirements_compiled_test.txt rename to ci/raydepsets/tests/test_data/requirements_compiled_test.txt diff --git a/ci/raydepsets/test_data/requirements_compiled_test_expand.txt b/ci/raydepsets/tests/test_data/requirements_compiled_test_expand.txt similarity index 100% rename from ci/raydepsets/test_data/requirements_compiled_test_expand.txt rename to ci/raydepsets/tests/test_data/requirements_compiled_test_expand.txt diff --git a/ci/raydepsets/test_data/requirements_compiled_test_update.txt b/ci/raydepsets/tests/test_data/requirements_compiled_test_update.txt similarity index 100% rename from ci/raydepsets/test_data/requirements_compiled_test_update.txt rename to ci/raydepsets/tests/test_data/requirements_compiled_test_update.txt diff --git a/ci/raydepsets/test_data/requirements_test.txt b/ci/raydepsets/tests/test_data/requirements_test.txt similarity index 100% rename from ci/raydepsets/test_data/requirements_test.txt rename to ci/raydepsets/tests/test_data/requirements_test.txt diff --git a/ci/raydepsets/test_data/test.depsets.yaml b/ci/raydepsets/tests/test_data/test.depsets.yaml similarity index 100% rename from ci/raydepsets/test_data/test.depsets.yaml rename to ci/raydepsets/tests/test_data/test.depsets.yaml diff --git a/ci/raydepsets/tests/test_workspace.py b/ci/raydepsets/tests/test_workspace.py new file mode 100644 index 000000000000..cc3ded45abef --- /dev/null +++ b/ci/raydepsets/tests/test_workspace.py @@ -0,0 +1,35 @@ +import sys +import tempfile +from pathlib import Path + +import pytest + +from ci.raydepsets.tests.utils import copy_data_to_tmpdir +from ci.raydepsets.workspace import Workspace + + +def test_workspace_init(): + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Workspace(tmpdir) + assert workspace.dir is not None + + +def test_parse_build_arg_sets(): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + workspace = Workspace(dir=tmpdir) + config = workspace.load_config(path=Path(tmpdir) / "test.depsets.yaml") + assert config.build_arg_sets[0].name == "py311_cpu" + assert config.build_arg_sets[0].build_args == { + "CUDA_VERSION": "cpu", + "PYTHON_VERSION": "py311", + } + assert config.build_arg_sets[1].name == "py311_cuda128" + assert config.build_arg_sets[1].build_args == { + "CUDA_VERSION": 128, + "PYTHON_VERSION": "py311", + } + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", __file__])) diff --git a/ci/raydepsets/tests/utils.py b/ci/raydepsets/tests/utils.py new file mode 100644 index 000000000000..8dae5634a524 --- /dev/null +++ b/ci/raydepsets/tests/utils.py @@ -0,0 +1,45 @@ +"""Shared test utilities for raydepsets tests.""" + +import shutil + +import runfiles + +_REPO_NAME = "com_github_ray_project_ray" +_runfiles = runfiles.Create() + + +def copy_data_to_tmpdir(tmpdir): + """Copy test data to a temporary directory.""" + shutil.copytree( + _runfiles.Rlocation(f"{_REPO_NAME}/ci/raydepsets/tests/test_data"), + tmpdir, + dirs_exist_ok=True, + ) + + +def replace_in_file(filepath, old, new): + with open(filepath, "r") as f: + contents = f.read() + + contents = contents.replace(old, new) + + with open(filepath, "w") as f: + f.write(contents) + + +def save_packages_to_file(filepath, packages): + with open(filepath, "w") as f: + for package in packages: + f.write(package + "\n") + + +def save_file_as(input_file, output_file): + with open(input_file, "rb") as f: + contents = f.read() + with open(output_file, "wb") as f: + f.write(contents) + + +def append_to_file(filepath, new): + with open(filepath, "a") as f: + f.write(new + "\n") From e620cd5e20a884795cde2824d28d8ff88da73787 Mon Sep 17 00:00:00 2001 From: Sampan S Nayak Date: Mon, 11 Aug 2025 23:32:38 +0530 Subject: [PATCH 021/634] [core] Fallback unserializable exceptions to their string representation (#55476) Falls back to printing the user exception's string representation instead of simply throwing a generic (non debugable) ray internal exception when the user's exception is not serializable. eg for: ```python import openai import ray from openai import AuthenticationError def call_openai_and_error_out(): client = openai.OpenAI( base_url="https://api.endpoints.anyscale.com/v1", api_key="test", ) try: client.chat.completions.create( model="gpt-3.5-turbo", messages=[ {"role": "system", "content": "You are a chatbot."}, {"role": "user", "content": "What is the capital of France?"}, ], ) except AuthenticationError as e: print("Errored as expected given API key is invalid.") raise e remote_fn = ray.remote(call_openai_and_error_out) ray.get(remote_fn.remote()) ``` we previously threw ``` 2025-08-02 14:19:36,726 ERROR serialization.py:462 -- Failed to unpickle serialized exception Traceback (most recent call last): File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/exceptions.py", line 51, in from_ray_exception return pickle.loads(ray_exception.serialized_exception) TypeError: APIStatusError.__init__() missing 2 required keyword-only arguments: 'response' and 'body' The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/_private/serialization.py", line 460, in deserialize_objects obj = self._deserialize_object(data, metadata, object_ref) File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/_private/serialization.py", line 342, in _deserialize_object return RayError.from_bytes(obj) File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/exceptions.py", line 45, in from_bytes return RayError.from_ray_exception(ray_exception) File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/exceptions.py", line 54, in from_ray_exception raise RuntimeError(msg) from e RuntimeError: Failed to unpickle serialized exception Traceback (most recent call last): File "/Users/rliaw/dev/proteins/_test.py", line 31, in ray.get(remote_fn.remote()) File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/_private/auto_init_hook.py", line 21, in auto_init_wrapper return fn(*args, **kwargs) File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper return func(*args, **kwargs) File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/_private/worker.py", line 2782, in get values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout) File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/_private/worker.py", line 931, in get_objects raise value ray.exceptions.RaySystemError: System error: Failed to unpickle serialized exception traceback: Traceback (most recent call last): File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/exceptions.py", line 51, in from_ray_exception return pickle.loads(ray_exception.serialized_exception) TypeError: APIStatusError.__init__() missing 2 required keyword-only arguments: 'response' and 'body' The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/_private/serialization.py", line 460, in deserialize_objects obj = self._deserialize_object(data, metadata, object_ref) File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/_private/serialization.py", line 342, in _deserialize_object return RayError.from_bytes(obj) File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/exceptions.py", line 45, in from_bytes return RayError.from_ray_exception(ray_exception) File "/Users/rliaw/miniconda3/lib/python3.10/site-packages/ray/exceptions.py", line 54, in from_ray_exception raise RuntimeError(msg) from e RuntimeError: Failed to unpickle serialized exception ``` but with this change we instead throw: ``` ERROR serialization.py:539 -- Failed to unpickle serialized exception Original exception (string repr): ray.exceptions.RayTaskError: ray::call_openai_and_error_out() (pid=2177610, ip=172.31.5.49) exc_info=True) File "/home/ubuntu/clone/ray/test.py", line 12, in call_openai_and_error_out client.chat.completions.create( File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_utils/_utils.py", line 287, in wrapper return func(*args, **kwargs) File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/resources/chat/completions/completions.py", line 925, in create return self._post( File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_base_client.py", line 1249, in post return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)) File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_base_client.py", line 1037, in request raise self._make_status_error_from_response(err.response) from None openai.NotFoundError:

Thank you for using Anyscale's Public Endpoints API.

Effective August 1, 2024 Anyscale Endpoints API is available exclusively through the fully Hosted Anyscale Platform. Multi-tenant access to LLM models has been removed.

With the Hosted Anyscale Platform, you can access the latest GPUs billed by the second, and deploy models on your own dedicated instances. Enjoy full customization to build your end-to-end applications with Anyscale. Get started today.

Traceback (most recent call last): File "/home/ubuntu/clone/ray/python/ray/exceptions.py", line 50, in from_ray_exception return pickle.loads(ray_exception.serialized_exception) TypeError: __init__() missing 2 required keyword-only arguments: 'response' and 'body' The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/home/ubuntu/clone/ray/python/ray/_private/serialization.py", line 532, in deserialize_objects obj = self._deserialize_object( File "/home/ubuntu/clone/ray/python/ray/_private/serialization.py", line 396, in _deserialize_object return RayError.from_bytes(obj) File "/home/ubuntu/clone/ray/python/ray/exceptions.py", line 44, in from_bytes return RayError.from_ray_exception(ray_exception) File "/home/ubuntu/clone/ray/python/ray/exceptions.py", line 60, in from_ray_exception raise RuntimeError(msg) from e RuntimeError: Failed to unpickle serialized exception Original exception (string repr): ray.exceptions.RayTaskError: ray::call_openai_and_error_out() (pid=2177610, ip=172.31.5.49) exc_info=True) File "/home/ubuntu/clone/ray/test.py", line 12, in call_openai_and_error_out client.chat.completions.create( File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_utils/_utils.py", line 287, in wrapper return func(*args, **kwargs) File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/resources/chat/completions/completions.py", line 925, in create return self._post( File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_base_client.py", line 1249, in post return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)) File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_base_client.py", line 1037, in request raise self._make_status_error_from_response(err.response) from None openai.NotFoundError:

Thank you for using Anyscale's Public Endpoints API.

Effective August 1, 2024 Anyscale Endpoints API is available exclusively through the fully Hosted Anyscale Platform. Multi-tenant access to LLM models has been removed.

With the Hosted Anyscale Platform, you can access the latest GPUs billed by the second, and deploy models on your own dedicated instances. Enjoy full customization to build your end-to-end applications with Anyscale. Get started today.

Traceback (most recent call last): File "/home/ubuntu/clone/ray/test.py", line 24, in ray.get(remote_fn.remote()) File "/home/ubuntu/clone/ray/python/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper return fn(*args, **kwargs) File "/home/ubuntu/clone/ray/python/ray/_private/client_mode_hook.py", line 104, in wrapper return func(*args, **kwargs) File "/home/ubuntu/clone/ray/python/ray/_private/worker.py", line 2869, in get values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout) File "/home/ubuntu/clone/ray/python/ray/_private/worker.py", line 970, in get_objects raise value ray.exceptions.RaySystemError: System error: Failed to unpickle serialized exception Original exception (string repr): ray.exceptions.RayTaskError: ray::call_openai_and_error_out() (pid=2177610, ip=172.31.5.49) exc_info=True) File "/home/ubuntu/clone/ray/test.py", line 12, in call_openai_and_error_out client.chat.completions.create( File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_utils/_utils.py", line 287, in wrapper return func(*args, **kwargs) File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/resources/chat/completions/completions.py", line 925, in create return self._post( File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_base_client.py", line 1249, in post return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)) File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_base_client.py", line 1037, in request raise self._make_status_error_from_response(err.response) from None openai.NotFoundError:

Thank you for using Anyscale's Public Endpoints API.

Effective August 1, 2024 Anyscale Endpoints API is available exclusively through the fully Hosted Anyscale Platform. Multi-tenant access to LLM models has been removed.

With the Hosted Anyscale Platform, you can access the latest GPUs billed by the second, and deploy models on your own dedicated instances. Enjoy full customization to build your end-to-end applications with Anyscale. Get started today.

traceback: Traceback (most recent call last): File "/home/ubuntu/clone/ray/python/ray/exceptions.py", line 50, in from_ray_exception return pickle.loads(ray_exception.serialized_exception) TypeError: __init__() missing 2 required keyword-only arguments: 'response' and 'body' The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/home/ubuntu/clone/ray/python/ray/_private/serialization.py", line 532, in deserialize_objects obj = self._deserialize_object( File "/home/ubuntu/clone/ray/python/ray/_private/serialization.py", line 396, in _deserialize_object return RayError.from_bytes(obj) File "/home/ubuntu/clone/ray/python/ray/exceptions.py", line 44, in from_bytes return RayError.from_ray_exception(ray_exception) File "/home/ubuntu/clone/ray/python/ray/exceptions.py", line 60, in from_ray_exception raise RuntimeError(msg) from e RuntimeError: Failed to unpickle serialized exception Original exception (string repr): ray.exceptions.RayTaskError: ray::call_openai_and_error_out() (pid=2177610, ip=172.31.5.49) exc_info=True) File "/home/ubuntu/clone/ray/test.py", line 12, in call_openai_and_error_out client.chat.completions.create( File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_utils/_utils.py", line 287, in wrapper return func(*args, **kwargs) File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/resources/chat/completions/completions.py", line 925, in create return self._post( File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_base_client.py", line 1249, in post return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)) File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_base_client.py", line 1037, in request raise self._make_status_error_from_response(err.response) from None openai.NotFoundError:

Thank you for using Anyscale's Public Endpoints API.

Effective August 1, 2024 Anyscale Endpoints API is available exclusively through the fully Hosted Anyscale Platform. Multi-tenant access to LLM models has been removed.

With the Hosted Anyscale Platform, you can access the latest GPUs billed by the second, and deploy models on your own dedicated instances. Enjoy full customization to build your end-to-end applications with Anyscale. Get started today.

``` ## Related issue number https://github.com/ray-project/ray/issues/55171 https://github.com/ray-project/ray/issues/43428 https://github.com/ray-project/ray/issues/50138 https://github.com/ray-project/ray/issues/49885 https://github.com/ray-project/ray/issues/49970 https://github.com/ray-project/ray/issues/54341 --------- Signed-off-by: sampan Co-authored-by: sampan --- python/ray/exceptions.py | 10 ++++++++++ python/ray/tests/test_traceback.py | 20 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/python/ray/exceptions.py b/python/ray/exceptions.py index a91b010ff93b..451deb295955 100644 --- a/python/ray/exceptions.py +++ b/python/ray/exceptions.py @@ -50,6 +50,16 @@ def from_ray_exception(ray_exception): return pickle.loads(ray_exception.serialized_exception) except Exception as e: msg = "Failed to unpickle serialized exception" + # Include a fallback string/stacktrace to aid debugging. + # formatted_exception_string is set in to_bytes() above by calling + # traceback.format_exception() on the original exception. It contains + # the string representation and stack trace of the original error. + formatted = getattr( + ray_exception, + "formatted_exception_string", + "No formatted exception string available.", + ) + msg += f"\nOriginal exception (string repr):\n{formatted}" raise RuntimeError(msg) from e else: return CrossLanguageError(ray_exception) diff --git a/python/ray/tests/test_traceback.py b/python/ray/tests/test_traceback.py index 93cb47b3d158..bb1f37755714 100644 --- a/python/ray/tests/test_traceback.py +++ b/python/ray/tests/test_traceback.py @@ -295,6 +295,14 @@ def __repr__(self): def test_unpickleable_stacktrace(shutdown_only): expected_output = """System error: Failed to unpickle serialized exception +Original exception (string repr): +ray.exceptions.RayTaskError: ray::f() (pid=XXX, ip=YYY) + File "FILE", line ZZ, in f + return g(c) + File "FILE", line ZZ, in g + raise NoPickleError("FILE") +test_traceback.NoPickleError + traceback: Traceback (most recent call last): File "FILE", line ZZ, in from_ray_exception return pickle.loads(ray_exception.serialized_exception) @@ -311,7 +319,14 @@ def test_unpickleable_stacktrace(shutdown_only): return RayError.from_ray_exception(ray_exception) File "FILE", line ZZ, in from_ray_exception raise RuntimeError(msg) from e -RuntimeError: Failed to unpickle serialized exception""" +RuntimeError: Failed to unpickle serialized exception +Original exception (string repr): +ray.exceptions.RayTaskError: ray::f() (pid=XXX, ip=YYY) + File "FILE", line ZZ, in f + return g(c) + File "FILE", line ZZ, in g + raise NoPickleError("FILE") +test_traceback.NoPickleError""" class NoPickleError(OSError): def __init__(self, arg): @@ -331,9 +346,10 @@ def f(): ray.get(f.remote()) except Exception as ex: python310_extra_exc_msg = "test_unpickleable_stacktrace..NoPickleError." - assert clean_noqa(expected_output) == scrub_traceback(str(ex)).replace( + cleaned = scrub_traceback(str(ex)).replace( f"TypeError: {python310_extra_exc_msg}", "TypeError: " ) + assert clean_noqa(expected_output) == cleaned def test_serialization_error_message(shutdown_only): From c266574a1bee430456007023f279fe6c89b32abd Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Mon, 11 Aug 2025 12:18:26 -0700 Subject: [PATCH 022/634] [data] external buffer num blocks metrics (#55022) ## Why are these changes needed? external1 external2 ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- .../dashboards/data_dashboard_panels.py | 28 +++++++++++++++++++ .../interfaces/op_runtime_metrics.py | 10 +++++++ .../execution/streaming_executor_state.py | 2 ++ python/ray/data/tests/test_stats.py | 10 +++++++ 4 files changed, 50 insertions(+) diff --git a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py index 47c52d65c57d..8bedac2db259 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py @@ -615,6 +615,34 @@ fill=0, stack=True, ), + Panel( + id=2, + title="Operator External OutQueue Size (Blocks)", + description="Number of blocks in operator's external output queue", + unit="blocks", + targets=[ + Target( + expr="sum(ray_data_num_output_queue_blocks{{{global_filters}}}) by (dataset, operator)", + legend="Number of Blocks: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, + ), + Panel( + id=27, + title="Operator External OutQueue Size (bytes)", + description="Byte size of blocks in operator's external output queue", + unit="bytes", + targets=[ + Target( + expr="sum(ray_data_num_output_queue_bytes{{{global_filters}}}) by (dataset, operator)", + legend="Number of Bytes: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, + ), Panel( id=34, title="Size of Blocks used in Pending Tasks (Bytes)", diff --git a/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py b/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py index e9c048fd8e84..b3b6bfb3445f 100644 --- a/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py +++ b/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py @@ -323,6 +323,16 @@ class OpRuntimeMetrics(metaclass=OpRuntimesMetricsMeta): description=("Number of rows generated by finished tasks."), metrics_group=MetricsGroup.OUTPUTS, ) + num_output_queue_blocks: int = metric_field( + default=0, + description="Number of blocks in the output queue", + metrics_group=MetricsGroup.OUTPUTS, + ) + num_output_queue_bytes: int = metric_field( + default=0, + description="Byte size of blocks in the output queue", + metrics_group=MetricsGroup.OUTPUTS, + ) # === Tasks-related metrics === num_tasks_submitted: int = metric_field( diff --git a/python/ray/data/_internal/execution/streaming_executor_state.py b/python/ray/data/_internal/execution/streaming_executor_state.py index 6475b0b72631..717ad190cc5a 100644 --- a/python/ray/data/_internal/execution/streaming_executor_state.py +++ b/python/ray/data/_internal/execution/streaming_executor_state.py @@ -303,6 +303,8 @@ def add_output(self, ref: RefBundle) -> None: self.op.metrics.num_alive_actors = actor_info.running self.op.metrics.num_restarting_actors = actor_info.restarting self.op.metrics.num_pending_actors = actor_info.pending + self.op.metrics.num_output_queue_blocks = self.output_queue.num_blocks + self.op.metrics.num_output_queue_bytes = self.output_queue.memory_usage def refresh_progress_bar(self, resource_manager: ResourceManager) -> None: """Update the console with the latest operator progress.""" diff --git a/python/ray/data/tests/test_stats.py b/python/ray/data/tests/test_stats.py index 23953efe059b..25deb5b1c0a5 100644 --- a/python/ray/data/tests/test_stats.py +++ b/python/ray/data/tests/test_stats.py @@ -99,6 +99,8 @@ def gen_expected_metrics( "'num_outputs_of_finished_tasks': N", "'bytes_outputs_of_finished_tasks': N", "'rows_outputs_of_finished_tasks': N", + "'num_output_queue_blocks': N", + "'num_output_queue_bytes': N", "'num_tasks_submitted': N", "'num_tasks_running': Z", "'num_tasks_have_outputs': N", @@ -154,6 +156,8 @@ def gen_expected_metrics( "'num_outputs_of_finished_tasks': Z", "'bytes_outputs_of_finished_tasks': Z", "'rows_outputs_of_finished_tasks': Z", + "'num_output_queue_blocks': N", + "'num_output_queue_bytes': N", "'num_tasks_submitted': Z", "'num_tasks_running': Z", "'num_tasks_have_outputs': Z", @@ -663,6 +667,8 @@ def test_dataset__repr__(ray_start_regular_shared, restore_data_context): " num_outputs_of_finished_tasks: N,\n" " bytes_outputs_of_finished_tasks: N,\n" " rows_outputs_of_finished_tasks: N,\n" + " num_output_queue_blocks: N,\n" + " num_output_queue_bytes: N,\n" " num_tasks_submitted: N,\n" " num_tasks_running: Z,\n" " num_tasks_have_outputs: N,\n" @@ -790,6 +796,8 @@ def check_stats(): " num_outputs_of_finished_tasks: N,\n" " bytes_outputs_of_finished_tasks: N,\n" " rows_outputs_of_finished_tasks: N,\n" + " num_output_queue_blocks: N,\n" + " num_output_queue_bytes: N,\n" " num_tasks_submitted: N,\n" " num_tasks_running: Z,\n" " num_tasks_have_outputs: N,\n" @@ -872,6 +880,8 @@ def check_stats(): " num_outputs_of_finished_tasks: N,\n" " bytes_outputs_of_finished_tasks: N,\n" " rows_outputs_of_finished_tasks: N,\n" + " num_output_queue_blocks: N,\n" + " num_output_queue_bytes: N,\n" " num_tasks_submitted: N,\n" " num_tasks_running: Z,\n" " num_tasks_have_outputs: N,\n" From eeebcbd2bf664fc43ca58a4d44c991b03fefe09d Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 11 Aug 2025 14:19:24 -0500 Subject: [PATCH 023/634] [core] Use passed report interval in Raylet instead of `RayConfig` (#55481) We were [passing the interval into the constructor](https://github.com/ray-project/ray/blob/1895aa1c4329f18a803e9a4386d50e8b1a4f1f1e/src/ray/raylet/node_manager.cc#L149) but then still pulling the value from `RayConfig`. Signed-off-by: Edward Oakes --- src/ray/raylet/node_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 0d726f782f79..a9ad42a4bf27 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -251,7 +251,7 @@ void NodeManager::RegisterGcs() { /* reporter */ &cluster_resource_scheduler_.GetLocalResourceManager(), /* receiver */ this, /* pull_from_reporter_interval_ms */ - RayConfig::instance().raylet_report_resources_period_milliseconds()); + report_resources_period_ms_); // Register a commands channel. // It's only used for GC right now. From 45727bb8660795584b4679c76688a5e03c6aff24 Mon Sep 17 00:00:00 2001 From: Zac Policzer Date: Mon, 11 Aug 2025 13:10:40 -0700 Subject: [PATCH 024/634] [metrics][gcs] Make CPU metric on gcs more reliable (#55471) gcs cpu metric consistenly reports zero. This is because psutil.as_dict() seems to return zero very frequently. cpu_percent() takes an interval argument which mitigates this, but psutil provides no utility for providing this argument in the as_dict() call. This is a 'minimal effort' patch to mitigate this behavior, but it remains somewhat of a mystery for why the other processes are unaffected by this the same way gcs metrics are. Signed-off-by: zac --- python/ray/dashboard/modules/reporter/reporter_agent.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/ray/dashboard/modules/reporter/reporter_agent.py b/python/ray/dashboard/modules/reporter/reporter_agent.py index a94f7d0a5b96..835a5b1be6a8 100644 --- a/python/ray/dashboard/modules/reporter/reporter_agent.py +++ b/python/ray/dashboard/modules/reporter/reporter_agent.py @@ -990,7 +990,9 @@ def _get_gcs(self): if self._gcs_pid: gcs_proc = psutil.Process(self._gcs_pid) if gcs_proc: - return gcs_proc.as_dict(attrs=PSUTIL_PROCESS_ATTRS) + dictionary = gcs_proc.as_dict(attrs=PSUTIL_PROCESS_ATTRS) + dictionary["cpu_percent"] = gcs_proc.cpu_percent(interval=1) + return dictionary return {} def _get_raylet(self): From 85f504ea86032c61ecf41f569717de482eba9ee9 Mon Sep 17 00:00:00 2001 From: Justin Yu Date: Mon, 11 Aug 2025 13:34:25 -0700 Subject: [PATCH 025/634] [data][train] Package `iter_batches` as a class (#55410) Lightweight refactor of `iter_batches` into a class. This will allow individual pipeline stages to store state on the `BatchIterator` instance and pass context to other stages. --------- Signed-off-by: Justin Yu --- .../_internal/block_batching/iter_batches.py | 233 +++++++++++------- python/ray/data/iterator.py | 38 ++- .../tests/block_batching/test_iter_batches.py | 8 +- 3 files changed, 171 insertions(+), 108 deletions(-) diff --git a/python/ray/data/_internal/block_batching/iter_batches.py b/python/ray/data/_internal/block_batching/iter_batches.py index 824f17aecbcb..cfad92a7acd7 100644 --- a/python/ray/data/_internal/block_batching/iter_batches.py +++ b/python/ray/data/_internal/block_batching/iter_batches.py @@ -1,5 +1,5 @@ import collections -from contextlib import nullcontext +from contextlib import contextmanager, nullcontext from typing import Any, Callable, Dict, Iterator, Optional import ray @@ -9,42 +9,28 @@ WaitBlockPrefetcher, blocks_to_batches, collate, - extract_data_from_batch, finalize_batches, format_batches, resolve_block_refs, ) from ray.data._internal.execution.interfaces.ref_bundle import RefBundle from ray.data._internal.memory_tracing import trace_deallocation -from ray.data._internal.stats import DatasetStats +from ray.data._internal.stats import DatasetStats, StatsManager from ray.data._internal.util import make_async_gen from ray.data.block import Block, DataBatch from ray.data.context import DataContext from ray.types import ObjectRef -def iter_batches( - ref_bundles: Iterator[RefBundle], - *, - stats: Optional[DatasetStats] = None, - clear_block_after_read: bool = False, - batch_size: Optional[int] = None, - batch_format: Optional[str] = "default", - drop_last: bool = False, - collate_fn: Optional[Callable[[DataBatch], Any]] = None, - finalize_fn: Optional[Callable[[Any], Any]] = None, - shuffle_buffer_min_size: Optional[int] = None, - shuffle_seed: Optional[int] = None, - ensure_copy: bool = False, - prefetch_batches: int = 1, -) -> Iterator[DataBatch]: - """Create formatted batches of data from an iterator of block object references and - corresponding metadata. +class BatchIterator: + """Defines an iterator pipeline to convert a stream of block object references + into a stream of formatted batches ready to be consumed by the user. This takes a block iterator and creates batch_size batches, slicing, unioning, shuffling, prefetching, and formatting blocks as needed. - The algorithm uses both pipeline parallelism and data parallelism: + This involves both pipeline parallelism (e.g. prefetching) + and data parallelism (e.g. threadpool operations): If prefetch_batches=2, these are all the batches in flight: @@ -74,6 +60,7 @@ def iter_batches( Args: ref_bundles: An iterator over RefBundles. stats: DatasetStats object to record timing and other statistics. + dataset_tag: The tag of the dataset to record timing and other statistics. clear_block_after_read: Whether to clear the block from object store manually (i.e. without waiting for Python's automatic GC) after it is read. Doing so will reclaim memory faster and hence reduce the @@ -103,86 +90,166 @@ def iter_batches( the specified amount of formatted batches from blocks. This improves performance for non-CPU bound UDFs, allowing batch fetching compute and formatting to be overlapped with the UDF. Defaults to 1. - - Returns: - An iterator over record batches. """ - context = DataContext.get_current() - if ( - prefetch_batches > 0 - and context.actor_prefetcher_enabled - and not ray.util.client.ray.is_connected() + def __init__( + self, + ref_bundles: Iterator[RefBundle], + *, + stats: Optional[DatasetStats] = None, + dataset_tag: Optional[str] = None, + clear_block_after_read: bool = False, + batch_size: Optional[int] = None, + batch_format: Optional[str] = "default", + drop_last: bool = False, + collate_fn: Optional[Callable[[DataBatch], Any]] = None, + finalize_fn: Optional[Callable[[Any], Any]] = None, + shuffle_buffer_min_size: Optional[int] = None, + shuffle_seed: Optional[int] = None, + ensure_copy: bool = False, + prefetch_batches: int = 1, ): - prefetcher = ActorBlockPrefetcher() - else: - prefetcher = WaitBlockPrefetcher() + self._ref_bundles = ref_bundles + self._stats = stats + self._dataset_tag = dataset_tag + self._batch_size = batch_size + self._batch_format = batch_format + self._drop_last = drop_last + self._collate_fn = collate_fn + self._finalize_fn = finalize_fn + self._shuffle_buffer_min_size = shuffle_buffer_min_size + self._shuffle_seed = shuffle_seed + self._ensure_copy = ensure_copy + self._prefetch_batches = prefetch_batches + self._eager_free = ( + clear_block_after_read and DataContext.get_current().eager_free + ) - eager_free = clear_block_after_read and DataContext.get_current().eager_free + actor_prefetcher_enabled = ( + prefetch_batches > 0 + and DataContext.get_current().actor_prefetcher_enabled + and not ray.util.client.ray.is_connected() + ) + self._prefetcher = ( + ActorBlockPrefetcher() + if actor_prefetcher_enabled + else WaitBlockPrefetcher() + ) - def _async_iter_batches( - ref_bundles: Iterator[RefBundle], - ) -> Iterator[DataBatch]: - # Step 1: Prefetch logical batches locally. - block_iter = prefetch_batches_locally( + def _prefetch_blocks( + self, ref_bundles: Iterator[RefBundle] + ) -> Iterator[ObjectRef[Block]]: + return prefetch_batches_locally( ref_bundles=ref_bundles, - prefetcher=prefetcher, - num_batches_to_prefetch=prefetch_batches, - batch_size=batch_size, - eager_free=eager_free, + prefetcher=self._prefetcher, + num_batches_to_prefetch=self._prefetch_batches, + batch_size=self._batch_size, + eager_free=self._eager_free, + ) + + def _resolve_block_refs( + self, block_refs: Iterator[ObjectRef[Block]] + ) -> Iterator[Block]: + return resolve_block_refs(block_ref_iter=block_refs, stats=self._stats) + + def _blocks_to_batches(self, blocks: Iterator[Block]) -> Iterator[Batch]: + return blocks_to_batches( + block_iter=blocks, + stats=self._stats, + batch_size=self._batch_size, + drop_last=self._drop_last, + shuffle_buffer_min_size=self._shuffle_buffer_min_size, + shuffle_seed=self._shuffle_seed, + ensure_copy=self._ensure_copy, + ) + + def _format_batches(self, batches: Iterator[Batch]) -> Iterator[Batch]: + return _format_in_threadpool( + batch_iter=batches, + stats=self._stats, + batch_format=self._batch_format, + collate_fn=self._collate_fn, + num_threadpool_workers=self._prefetch_batches, + ) + + def _finalize_batches( + self, + batch_iter: Iterator[Batch], + ) -> Iterator[Batch]: + if self._finalize_fn is None: + return batch_iter + + return finalize_batches( + batch_iter, finalize_fn=self._finalize_fn, stats=self._stats ) + def _restore_original_batch_order( + self, batches: Iterator[Batch] + ) -> Iterator[Batch]: + return restore_original_order(batches) + + def _pipeline(self, ref_bundles: Iterator[RefBundle]) -> Iterator[Batch]: + # Step 1: Prefetch logical batches locally. + block_iter = self._prefetch_blocks(ref_bundles) + # Step 2: Resolve the blocks. - block_iter = resolve_block_refs(block_ref_iter=block_iter, stats=stats) + block_iter = self._resolve_block_refs(block_iter) # Step 3: Batch and shuffle the resolved blocks. - batch_iter = blocks_to_batches( - block_iter=block_iter, - stats=stats, - batch_size=batch_size, - drop_last=drop_last, - shuffle_buffer_min_size=shuffle_buffer_min_size, - shuffle_seed=shuffle_seed, - ensure_copy=ensure_copy, - ) + batch_iter = self._blocks_to_batches(block_iter) + + # Step 4: Format and collate the batches in a threadpool. + batch_iter = self._format_batches(batch_iter) + + # Step 5: Finalize the batches (e.g., move to GPU). + batch_iter = self._finalize_batches(batch_iter) + + # Step 6: Restore the original order of the batches, as the prior + # threadpool operations may have reordered the batches non-deterministically. + batch_iter = self._restore_original_batch_order(batch_iter) - # Step 4: Use a threadpool for formatting and collation. - batch_iter = _format_in_threadpool( - batch_iter, - stats=stats, - batch_format=batch_format, - collate_fn=collate_fn, - num_threadpool_workers=prefetch_batches, + yield from batch_iter + + def _iter_batches(self) -> Iterator[DataBatch]: + async_batch_iter = make_async_gen( + self._ref_bundles, + fn=self._pipeline, + num_workers=1, + preserve_ordering=False, ) - # Step 5: Finalize each batch. - if finalize_fn is not None: - batch_iter = finalize_batches( - batch_iter, finalize_fn=finalize_fn, stats=stats - ) + self.before_epoch_start() - # Step 6: Restore original order. - batch_iter: Iterator[Batch] = restore_original_order(batch_iter) + while True: + with self.get_next_batch_context(): + try: + batch = next(async_batch_iter) + except StopIteration: + break + with self.yield_batch_context(batch): + yield batch.data - yield from extract_data_from_batch(batch_iter) + self.after_epoch_end() - # Run everything in a separate thread to not block the main thread when waiting - # for streaming results. - async_batch_iter = make_async_gen( - ref_bundles, - fn=_async_iter_batches, - num_workers=1, - preserve_ordering=False, - ) + def __iter__(self) -> Iterator[DataBatch]: + return self._iter_batches() - while True: - with stats.iter_total_blocked_s.timer() if stats else nullcontext(): - try: - next_batch = next(async_batch_iter) - except StopIteration: - break - with stats.iter_user_s.timer() if stats else nullcontext(): - yield next_batch + def before_epoch_start(self): + pass + + def after_epoch_end(self): + StatsManager.clear_iteration_metrics(self._dataset_tag) + + @contextmanager + def get_next_batch_context(self): + with self._stats.iter_total_blocked_s.timer() if self._stats else nullcontext(): + yield + + @contextmanager + def yield_batch_context(self, batch: Batch): + with self._stats.iter_user_s.timer() if self._stats else nullcontext(): + yield + StatsManager.update_iteration_metrics(self._stats, self._dataset_tag) def _format_in_threadpool( diff --git a/python/ray/data/iterator.py b/python/ray/data/iterator.py index 8602baf10f0a..70144586a80b 100644 --- a/python/ray/data/iterator.py +++ b/python/ray/data/iterator.py @@ -17,7 +17,7 @@ import numpy as np -from ray.data._internal.block_batching.iter_batches import iter_batches +from ray.data._internal.block_batching.iter_batches import BatchIterator from ray.data._internal.execution.interfaces import RefBundle from ray.data._internal.logical.interfaces import LogicalPlan from ray.data._internal.logical.operators.input_data_operator import InputData @@ -184,31 +184,27 @@ def _create_iterator() -> Iterator[DataBatch]: blocks_owned_by_consumer, ) = self._to_ref_bundle_iterator() - iterator = iter( - iter_batches( - ref_bundles_iterator, - stats=stats, - clear_block_after_read=blocks_owned_by_consumer, - batch_size=batch_size, - batch_format=batch_format, - drop_last=drop_last, - collate_fn=_collate_fn, - finalize_fn=_finalize_fn, - shuffle_buffer_min_size=local_shuffle_buffer_size, - shuffle_seed=local_shuffle_seed, - prefetch_batches=prefetch_batches, - ) - ) - dataset_tag = self._get_dataset_tag() + batch_iterator = BatchIterator( + ref_bundles_iterator, + stats=stats, + dataset_tag=dataset_tag, + clear_block_after_read=blocks_owned_by_consumer, + batch_size=batch_size, + batch_format=batch_format, + drop_last=drop_last, + collate_fn=_collate_fn, + finalize_fn=_finalize_fn, + shuffle_buffer_min_size=local_shuffle_buffer_size, + shuffle_seed=local_shuffle_seed, + prefetch_batches=prefetch_batches, + ) + if stats: stats.iter_initialize_s.add(time.perf_counter() - time_start) - for batch in iterator: - yield batch - StatsManager.update_iteration_metrics(stats, dataset_tag) - StatsManager.clear_iteration_metrics(dataset_tag) + yield from batch_iterator if stats: stats.iter_total_s.add(time.perf_counter() - time_start) diff --git a/python/ray/data/tests/block_batching/test_iter_batches.py b/python/ray/data/tests/block_batching/test_iter_batches.py index ecbdd7e16173..36f4b8a5005c 100644 --- a/python/ray/data/tests/block_batching/test_iter_batches.py +++ b/python/ray/data/tests/block_batching/test_iter_batches.py @@ -10,7 +10,7 @@ import ray from ray.data._internal.block_batching.interfaces import Batch, BlockPrefetcher from ray.data._internal.block_batching.iter_batches import ( - iter_batches, + BatchIterator, prefetch_batches_locally, restore_original_order, ) @@ -123,7 +123,7 @@ def finalize_enforce_single_thread(batch): # Test that finalize_fn is called in a single thread, # even if prefetch_batches is set. - output_batches = iter_batches( + output_batches = BatchIterator( ref_bundles_iter, collate_fn=lambda batch: batch, finalize_fn=finalize_enforce_single_thread, @@ -156,7 +156,7 @@ def collate_fn(batch: pd.DataFrame): ref_bundles_iter = ref_bundle_generator(num_blocks=4, num_rows=2) - output_batches = iter_batches( + output_batches = BatchIterator( ref_bundles_iter, batch_size=batch_size, prefetch_batches=prefetch_batches, @@ -198,7 +198,7 @@ def collate_fn(batch): ref_bundles = ref_bundle_generator(num_blocks=20, num_rows=2) start_time = time.time() - output_batches = iter_batches( + output_batches = BatchIterator( ref_bundles, batch_size=None, collate_fn=collate_fn, From 63c41687c5d0f19252e760820044991440818c5f Mon Sep 17 00:00:00 2001 From: Sven Mika Date: Mon, 11 Aug 2025 22:50:26 +0200 Subject: [PATCH 026/634] [RLlib] Upgrade RLlink protocol for external env/simulator training. (#53550) --- doc/source/rllib/external-envs.rst | 2 +- doc/source/rllib/package_ref/env.rst | 7 +- doc/source/rllib/package_ref/env/external.rst | 22 + doc/source/rllib/package_ref/env/utils.rst | 1 - rllib/BUILD | 1 + rllib/env/external/__init__.py | 12 + ...nv_runner_server_for_external_inference.py | 368 +++++++++++ rllib/env/external/rllink.py | 109 ++++ rllib/env/policy_client.py | 90 +-- rllib/env/policy_server_input.py | 77 +-- rllib/env/tcp_client_inference_env_runner.py | 591 +----------------- rllib/env/utils/external_env_protocol.py | 53 +- rllib/env/utils/infinite_lookback_buffer.py | 2 + rllib/env/wrappers/unity3d_env.py | 67 +- .../classes/utils/dummy_external_client.py | 126 ++++ .../env_connecting_to_rllib_w_tcp_client.py | 43 +- .../envs/external_envs/cartpole_client.py | 133 ---- .../envs/external_envs/cartpole_server.py | 278 -------- .../dummy_client_with_two_episodes.py | 95 --- .../envs/external_envs/unity3d_client.py | 133 ---- .../external_envs/unity3d_dummy_client.py | 160 ----- .../envs/external_envs/unity3d_server.py | 178 ------ rllib/utils/checkpoints.py | 3 +- 23 files changed, 691 insertions(+), 1860 deletions(-) create mode 100644 doc/source/rllib/package_ref/env/external.rst create mode 100644 rllib/env/external/__init__.py create mode 100644 rllib/env/external/env_runner_server_for_external_inference.py create mode 100644 rllib/env/external/rllink.py create mode 100644 rllib/examples/envs/classes/utils/dummy_external_client.py delete mode 100755 rllib/examples/envs/external_envs/cartpole_client.py delete mode 100755 rllib/examples/envs/external_envs/cartpole_server.py delete mode 100644 rllib/examples/envs/external_envs/dummy_client_with_two_episodes.py delete mode 100644 rllib/examples/envs/external_envs/unity3d_client.py delete mode 100644 rllib/examples/envs/external_envs/unity3d_dummy_client.py delete mode 100755 rllib/examples/envs/external_envs/unity3d_server.py diff --git a/doc/source/rllib/external-envs.rst b/doc/source/rllib/external-envs.rst index 7730a17117c6..2307457447a8 100644 --- a/doc/source/rllib/external-envs.rst +++ b/doc/source/rllib/external-envs.rst @@ -30,7 +30,7 @@ should step. .. scale: 75 % .. A Unity3D soccer game being learnt by RLlib via the ExternalEnv API. -RLlib provides an `external messaging protocol `__ +RLlib provides an `external messaging protocol `__ called :ref:`RLlink ` for this purpose as well as the option to customize your :py:class:`~ray.rllib.env.env_runner.EnvRunner` class toward communicating through :ref:`RLlink ` with one or more clients. An example, `tcp-based EnvRunner implementation with RLlink is available here `__. diff --git a/doc/source/rllib/package_ref/env.rst b/doc/source/rllib/package_ref/env.rst index b8a49f196508..aa9bfcc483c0 100644 --- a/doc/source/rllib/package_ref/env.rst +++ b/doc/source/rllib/package_ref/env.rst @@ -21,12 +21,6 @@ gymnasium's own `vectorization feature = 1.x` custom vectorization feature. - External Envs ------------- @@ -55,4 +49,5 @@ Environment API Reference env/multi_agent_env.rst env/multi_agent_env_runner.rst env/multi_agent_episode.rst + env/external.rst env/utils.rst diff --git a/doc/source/rllib/package_ref/env/external.rst b/doc/source/rllib/package_ref/env/external.rst new file mode 100644 index 000000000000..4dce2def1646 --- /dev/null +++ b/doc/source/rllib/package_ref/env/external.rst @@ -0,0 +1,22 @@ +.. include:: /_includes/rllib/we_are_hiring.rst + +.. _env-external-reference-docs: + +External Envs +============= + +.. include:: /_includes/rllib/new_api_stack.rst + +ray.rllib.env.external.rllink.RLlink +------------------------------------ + +.. currentmodule:: ray.rllib.env.external.rllink + +.. autoclass:: ray.rllib.env.external.rllink.RLlink + +.. autosummary:: + :nosignatures: + :toctree: doc/ + + ~get_rllink_message + ~send_rllink_message diff --git a/doc/source/rllib/package_ref/env/utils.rst b/doc/source/rllib/package_ref/env/utils.rst index 49a884bd6bc4..99717102ef34 100644 --- a/doc/source/rllib/package_ref/env/utils.rst +++ b/doc/source/rllib/package_ref/env/utils.rst @@ -16,6 +16,5 @@ rllib.env.utils :nosignatures: :toctree: env/ - ~external_env_protocol.RLlink ~try_import_open_spiel ~try_import_pyspiel diff --git a/rllib/BUILD b/rllib/BUILD index 8ed94ce32d53..41fcc6f16146 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -4112,6 +4112,7 @@ py_test( args = [ "--as-test", "--port=12346", + "--use-dummy-client", ], main = "examples/envs/env_connecting_to_rllib_w_tcp_client.py", tags = [ diff --git a/rllib/env/external/__init__.py b/rllib/env/external/__init__.py new file mode 100644 index 000000000000..343adb18b3c5 --- /dev/null +++ b/rllib/env/external/__init__.py @@ -0,0 +1,12 @@ +from ray.rllib.env.external.rllink import ( + get_rllink_message, + send_rllink_message, + RLlink, +) + + +__all__ = [ + "get_rllink_message", + "send_rllink_message", + "RLlink", +] diff --git a/rllib/env/external/env_runner_server_for_external_inference.py b/rllib/env/external/env_runner_server_for_external_inference.py new file mode 100644 index 000000000000..36bb2723c27b --- /dev/null +++ b/rllib/env/external/env_runner_server_for_external_inference.py @@ -0,0 +1,368 @@ +from collections import defaultdict +import pickle +import socket +import threading +import time +from typing import Collection, DefaultDict, List, Optional, Union + +from ray.rllib.core import ( + COMPONENT_RL_MODULE, + DEFAULT_AGENT_ID, + DEFAULT_MODULE_ID, +) +from ray.rllib.env import INPUT_ENV_SPACES +from ray.rllib.env.env_runner import EnvRunner +from ray.rllib.env.single_agent_env_runner import SingleAgentEnvRunner +from ray.rllib.env.single_agent_episode import SingleAgentEpisode +from ray.rllib.env.external.rllink import ( + get_rllink_message, + send_rllink_message, + RLlink, +) +from ray.rllib.utils.annotations import override +from ray.rllib.utils.checkpoints import Checkpointable +from ray.rllib.utils.framework import try_import_torch +from ray.rllib.utils.metrics import ( + EPISODE_DURATION_SEC_MEAN, + EPISODE_LEN_MAX, + EPISODE_LEN_MEAN, + EPISODE_LEN_MIN, + EPISODE_RETURN_MAX, + EPISODE_RETURN_MEAN, + EPISODE_RETURN_MIN, + WEIGHTS_SEQ_NO, +) +from ray.rllib.utils.metrics.metrics_logger import MetricsLogger +from ray.rllib.utils.typing import EpisodeID, StateDict +from ray.util.annotations import DeveloperAPI + +torch, _ = try_import_torch() + + +@DeveloperAPI +class EnvRunnerServerForExternalInference(EnvRunner, Checkpointable): + """An EnvRunner communicating with an external env through a TCP socket. + + This implementation assumes: + - Only one external client ever connects to this env runner. + - The external client owns the connector pipelines (env-to-module and module-to-env) + as well as the RLModule and thus performs inference locally. Samples are sent in + bulk as lists of RLlib episodes once a certain number of timesteps has been executed + on the client's side. + - A copy of the RLModule is kept at all times on this EnvRunner, but is never used + for inference, only as a weights container. + TODO (sven): The above might be inefficient as we have to store basically two + models, one in this EnvRunner, one in the env (as ONNX). + - As a consequence, there are no environment and no connectors on this env runner. + The external env is responsible for generating all the data to create episodes. + """ + + @override(EnvRunner) + def __init__(self, *, config, **kwargs): + """ + Initializes an EnvRunnerServerForExternalInference instance. + + Args: + config: The AlgorithmConfig to use for setup. + + Keyword Args: + port: The base port number. The server socket is then actually bound to + `port` + self.worker_index. + """ + super().__init__(config=config, **kwargs) + + self.worker_index: int = kwargs.get("worker_index", 0) + + self._weights_seq_no = 0 + + # Build the module from its spec. + module_spec = self.config.get_rl_module_spec( + spaces=self.get_spaces(), inference_only=True + ) + self.module = module_spec.build() + + self.host = "localhost" + self.port = int(self.config.env_config.get("port", 5555)) + self.worker_index + self.server_socket = None + self.client_socket = None + self.address = None + + self.metrics = MetricsLogger() + + self._episode_chunks_to_return: Optional[List[SingleAgentEpisode]] = None + self._done_episodes_for_metrics: List[SingleAgentEpisode] = [] + self._ongoing_episodes_for_metrics: DefaultDict[ + EpisodeID, List[SingleAgentEpisode] + ] = defaultdict(list) + + self._sample_lock = threading.Lock() + self._on_policy_lock = threading.Lock() + self._blocked_on_state = False + + # Start a background thread for client communication. + self.thread = threading.Thread( + target=self._client_message_listener, daemon=True + ) + self.thread.start() + + @override(EnvRunner) + def assert_healthy(self): + """Checks that the server socket is open and listening.""" + assert ( + self.server_socket is not None + ), "Server socket is None (not connected, not listening)." + + @override(EnvRunner) + def sample(self, **kwargs): + """Waits for the client to send episodes.""" + while True: + with self._sample_lock: + if self._episode_chunks_to_return is not None: + num_env_steps = 0 + num_episodes_completed = 0 + for eps in self._episode_chunks_to_return: + if eps.is_done: + self._done_episodes_for_metrics.append(eps) + num_episodes_completed += 1 + else: + self._ongoing_episodes_for_metrics[eps.id_].append(eps) + num_env_steps += len(eps) + + ret = self._episode_chunks_to_return + self._episode_chunks_to_return = None + + SingleAgentEnvRunner._increase_sampled_metrics( + self, num_env_steps, num_episodes_completed + ) + + return ret + time.sleep(0.01) + + @override(EnvRunner) + def get_metrics(self): + # TODO (sven): We should probably make this a utility function to be called + # from within Single/MultiAgentEnvRunner and other EnvRunner subclasses, as + # needed. + # Compute per-episode metrics (only on already completed episodes). + for eps in self._done_episodes_for_metrics: + assert eps.is_done + episode_length = len(eps) + episode_return = eps.get_return() + episode_duration_s = eps.get_duration_s() + # Don't forget about the already returned chunks of this episode. + if eps.id_ in self._ongoing_episodes_for_metrics: + for eps2 in self._ongoing_episodes_for_metrics[eps.id_]: + episode_length += len(eps2) + episode_return += eps2.get_return() + episode_duration_s += eps2.get_duration_s() + del self._ongoing_episodes_for_metrics[eps.id_] + + self._log_episode_metrics( + episode_length, episode_return, episode_duration_s + ) + + # Now that we have logged everything, clear cache of done episodes. + self._done_episodes_for_metrics.clear() + + # Return reduced metrics. + return self.metrics.reduce() + + def get_spaces(self): + return { + INPUT_ENV_SPACES: (self.config.observation_space, self.config.action_space), + DEFAULT_MODULE_ID: ( + self.config.observation_space, + self.config.action_space, + ), + } + + @override(EnvRunner) + def stop(self): + """Closes the client and server sockets.""" + self._close_sockets_if_necessary() + + @override(Checkpointable) + def get_ctor_args_and_kwargs(self): + return ( + (), # *args + {"config": self.config}, # **kwargs + ) + + @override(Checkpointable) + def get_checkpointable_components(self): + return [ + (COMPONENT_RL_MODULE, self.module), + ] + + @override(Checkpointable) + def get_state( + self, + components: Optional[Union[str, Collection[str]]] = None, + *, + not_components: Optional[Union[str, Collection[str]]] = None, + **kwargs, + ) -> StateDict: + return { + COMPONENT_RL_MODULE: self.module.get_state(), + WEIGHTS_SEQ_NO: self._weights_seq_no, + } + + @override(Checkpointable) + def set_state(self, state: StateDict) -> None: + # Update the RLModule state. + if COMPONENT_RL_MODULE in state: + # A missing value for WEIGHTS_SEQ_NO or a value of 0 means: Force the + # update. + weights_seq_no = state.get(WEIGHTS_SEQ_NO, 0) + + # Only update the weigths, if this is the first synchronization or + # if the weights of this `EnvRunner` lacks behind the actual ones. + if weights_seq_no == 0 or self._weights_seq_no < weights_seq_no: + rl_module_state = state[COMPONENT_RL_MODULE] + if ( + isinstance(rl_module_state, dict) + and DEFAULT_MODULE_ID in rl_module_state + ): + rl_module_state = rl_module_state[DEFAULT_MODULE_ID] + self.module.set_state(rl_module_state) + + # Update our weights_seq_no, if the new one is > 0. + if weights_seq_no > 0: + self._weights_seq_no = weights_seq_no + + if self._blocked_on_state is True: + self._send_set_state_message() + self._blocked_on_state = False + + def _client_message_listener(self): + """Entry point for the listener thread.""" + + # Set up the server socket and bind to the specified host and port. + self._recycle_sockets() + + # Enter an endless message receival- and processing loop. + while True: + # As long as we are blocked on a new state, sleep a bit and continue. + # Do NOT process any incoming messages (until we send out the new state + # back to the client). + if self._blocked_on_state is True: + time.sleep(0.01) + continue + + try: + # Blocking call to get next message. + msg_type, msg_body = get_rllink_message(self.client_socket) + + # Process the message received based on its type. + # Initial handshake. + if msg_type == RLlink.PING: + self._send_pong_message() + + # Episode data from the client. + elif msg_type in [ + RLlink.EPISODES, + RLlink.EPISODES_AND_GET_STATE, + ]: + self._process_episodes_message(msg_type, msg_body) + + # Client requests the state (model weights). + elif msg_type == RLlink.GET_STATE: + self._send_set_state_message() + + # Clients requests config information. + elif msg_type == RLlink.GET_CONFIG: + self._send_set_config_message() + + except ConnectionError as e: + print(f"Messaging/connection error {e}! Recycling sockets ...") + self._recycle_sockets(5.0) + continue + + def _recycle_sockets(self, sleep: float = 0.0): + # Close all old sockets, if they exist. + self._close_sockets_if_necessary() + + time.sleep(sleep) + + # Start listening on the configured port. + self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Allow reuse of the address. + self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.server_socket.bind((self.host, self.port)) + # Listen for a single connection. + self.server_socket.listen(1) + print(f"Waiting for client to connect to port {self.port}...") + + self.client_socket, self.address = self.server_socket.accept() + print(f"Connected to client at {self.address}") + + def _close_sockets_if_necessary(self): + if self.client_socket: + self.client_socket.close() + if self.server_socket: + self.server_socket.close() + + def _send_pong_message(self): + send_rllink_message(self.client_socket, {"type": RLlink.PONG.name}) + + def _process_episodes_message(self, msg_type, msg_body): + # On-policy training -> we have to block until we get a new `set_state` call + # (b/c the learning step is done and we can send new weights back to all + # clients). + if msg_type == RLlink.EPISODES_AND_GET_STATE: + self._blocked_on_state = True + + episodes = [] + for episode_state in msg_body["episodes"]: + episode = SingleAgentEpisode.from_state(episode_state) + episodes.append(episode.to_numpy()) + + # Push episodes into the to-be-returned list (for `sample()` requests). + with self._sample_lock: + if isinstance(self._episode_chunks_to_return, list): + self._episode_chunks_to_return.extend(episodes) + else: + self._episode_chunks_to_return = episodes + + def _send_set_state_message(self): + send_rllink_message( + self.client_socket, + { + "type": RLlink.SET_STATE.name, + "state": self.get_state(inference_only=True), + }, + ) + + def _send_set_config_message(self): + send_rllink_message( + self.client_socket, + { + "type": RLlink.SET_CONFIG.name, + # TODO (sven): We need AlgorithmConfig to be a `Checkpointable` with a + # msgpack'able state. + "config": pickle.dumps(self.config), + }, + ) + + def _log_episode_metrics(self, length, ret, sec): + # Log general episode metrics. + # To mimic the old API stack behavior, we'll use `window` here for + # these particular stats (instead of the default EMA). + win = self.config.metrics_num_episodes_for_smoothing + self.metrics.log_value(EPISODE_LEN_MEAN, length, window=win) + self.metrics.log_value(EPISODE_RETURN_MEAN, ret, window=win) + self.metrics.log_value(EPISODE_DURATION_SEC_MEAN, sec, window=win) + # Per-agent returns. + self.metrics.log_value( + ("agent_episode_returns_mean", DEFAULT_AGENT_ID), ret, window=win + ) + # Per-RLModule returns. + self.metrics.log_value( + ("module_episode_returns_mean", DEFAULT_MODULE_ID), ret, window=win + ) + + # For some metrics, log min/max as well. + self.metrics.log_value(EPISODE_LEN_MIN, length, reduce="min", window=win) + self.metrics.log_value(EPISODE_RETURN_MIN, ret, reduce="min", window=win) + self.metrics.log_value(EPISODE_LEN_MAX, length, reduce="max", window=win) + self.metrics.log_value(EPISODE_RETURN_MAX, ret, reduce="max", window=win) diff --git a/rllib/env/external/rllink.py b/rllib/env/external/rllink.py new file mode 100644 index 000000000000..dfb72bda97b6 --- /dev/null +++ b/rllib/env/external/rllink.py @@ -0,0 +1,109 @@ +from enum import Enum +from packaging.version import Version + +from ray.rllib.utils.checkpoints import try_import_msgpack +from ray.util.annotations import DeveloperAPI + + +msgpack = None + + +@DeveloperAPI +class RLlink(Enum): + PROTOCOL_VERSION = Version("0.0.1") + + # Requests: Client (external env) -> Server (RLlib). + # ---- + # Ping command (initial handshake). + PING = "PING" + # List of episodes (similar to what an EnvRunner.sample() call would return). + EPISODES = "EPISODES" + # Request state (e.g. model weights). + GET_STATE = "GET_STATE" + # Request Algorithm config. + GET_CONFIG = "GET_CONFIG" + # Send episodes and request the next state update right after that. + # Clients sending this message should wait for a SET_STATE message as an immediate + # response. Useful for external samplers that must collect on-policy data. + EPISODES_AND_GET_STATE = "EPISODES_AND_GET_STATE" + + # Responses: Server (RLlib) -> Client (external env). + # ---- + # Pong response (initial handshake). + PONG = "PONG" + # Set state (e.g. model weights). + SET_STATE = "SET_STATE" + # Set Algorithm config. + SET_CONFIG = "SET_CONFIG" + + # @OldAPIStack (to be deprecated soon). + ACTION_SPACE = "ACTION_SPACE" + OBSERVATION_SPACE = "OBSERVATION_SPACE" + GET_WORKER_ARGS = "GET_WORKER_ARGS" + GET_WEIGHTS = "GET_WEIGHTS" + REPORT_SAMPLES = "REPORT_SAMPLES" + START_EPISODE = "START_EPISODE" + GET_ACTION = "GET_ACTION" + LOG_ACTION = "LOG_ACTION" + LOG_RETURNS = "LOG_RETURNS" + END_EPISODE = "END_EPISODE" + + def __str__(self): + return self.name + + +@DeveloperAPI +def send_rllink_message(sock_, message: dict): + """Sends a message to the client with a length header.""" + global msgpack + if msgpack is None: + msgpack = try_import_msgpack(error=True) + + body = msgpack.packb(message, use_bin_type=True) # .encode("utf-8") + header = str(len(body)).zfill(8).encode("utf-8") + try: + sock_.sendall(header + body) + except Exception as e: + raise ConnectionError( + f"Error sending message {message} to server on socket {sock_}! " + f"Original error was: {e}" + ) + + +@DeveloperAPI +def get_rllink_message(sock_): + """Receives a message from the client following the length-header protocol.""" + global msgpack + if msgpack is None: + msgpack = try_import_msgpack(error=True) + + try: + # Read the length header (8 bytes) + header = _get_num_bytes(sock_, 8) + msg_length = int(header.decode("utf-8")) + # Read the message body + body = _get_num_bytes(sock_, msg_length) + # Decode JSON. + message = msgpack.unpackb(body, raw=False) # .loads(body.decode("utf-8")) + # Check for proper protocol. + if "type" not in message: + raise ConnectionError( + "Protocol Error! Message from peer does not contain `type` field." + ) + return RLlink(message.pop("type")), message + except Exception as e: + raise ConnectionError( + f"Error receiving message from peer on socket {sock_}! " + f"Original error was: {e}" + ) + + +def _get_num_bytes(sock_, num_bytes): + """Helper function to receive a specific number of bytes.""" + data = b"" + while len(data) < num_bytes: + packet = sock_.recv(num_bytes - len(data)) + if not packet: + raise ConnectionError(f"No data received from socket {sock_}!") + data += packet + return data diff --git a/rllib/env/policy_client.py b/rllib/env/policy_client.py index 2f3791226077..e4e2e9ad8a62 100644 --- a/rllib/env/policy_client.py +++ b/rllib/env/policy_client.py @@ -1,9 +1,3 @@ -"""REST client to interact with a policy server. - -This client supports both local and remote policy inference modes. Local -inference is faster but causes more compute to be done on the client. -""" - import logging import threading import time @@ -23,7 +17,7 @@ ) # Backward compatibility. -from ray.rllib.env.utils.external_env_protocol import RLlink as Commands +from ray.rllib.env.external.rllink import RLlink as Commands logger = logging.getLogger(__name__) @@ -48,20 +42,6 @@ def __init__( update_interval: float = 10.0, session: Optional[requests.Session] = None, ): - """Create a PolicyClient instance. - - Args: - address: Server to connect to (e.g., "localhost:9090"). - inference_mode: Whether to use 'local' or 'remote' policy - inference for computing actions. - update_interval (float or None): If using 'local' inference mode, - the policy is refreshed after this many seconds have passed, - or None for manual control via client. - session (requests.Session or None): If available the session object - is used to communicate with the policy server. Using a session - can lead to speedups as connections are reused. It is the - responsibility of the creator of the session to close it. - """ self.address = address self.session = session self.env: ExternalEnv = None @@ -76,18 +56,6 @@ def __init__( def start_episode( self, episode_id: Optional[str] = None, training_enabled: bool = True ) -> str: - """Record the start of one or more episode(s). - - Args: - episode_id (Optional[str]): Unique string id for the episode or - None for it to be auto-assigned. - training_enabled: Whether to use experiences for this - episode to improve the policy. - - Returns: - episode_id: Unique string id for the episode. - """ - if self.local: self._update_local_policy() return self.env.start_episode(episode_id, training_enabled) @@ -103,16 +71,6 @@ def start_episode( def get_action( self, episode_id: str, observation: Union[EnvObsType, MultiAgentDict] ) -> Union[EnvActionType, MultiAgentDict]: - """Record an observation and get the on-policy action. - - Args: - episode_id: Episode id returned from start_episode(). - observation: Current environment observation. - - Returns: - action: Action from the env action space. - """ - if self.local: self._update_local_policy() if isinstance(episode_id, (list, tuple)): @@ -138,14 +96,6 @@ def log_action( observation: Union[EnvObsType, MultiAgentDict], action: Union[EnvActionType, MultiAgentDict], ) -> None: - """Record an observation and (off-policy) action taken. - - Args: - episode_id: Episode id returned from start_episode(). - observation: Current environment observation. - action: Action for the observation. - """ - if self.local: self._update_local_policy() return self.env.log_action(episode_id, observation, action) @@ -166,19 +116,6 @@ def log_returns( info: Union[EnvInfoDict, MultiAgentDict] = None, multiagent_done_dict: Optional[MultiAgentDict] = None, ) -> None: - """Record returns from the environment. - - The reward will be attributed to the previous action taken by the - episode. Rewards accumulate until the next action. If no reward is - logged before the next action, a reward of 0.0 is assumed. - - Args: - episode_id: Episode id returned from start_episode(). - reward: Reward from the environment. - info: Extra info dict. - multiagent_done_dict: Multi-agent done information. - """ - if self.local: self._update_local_policy() if multiagent_done_dict is not None: @@ -201,13 +138,6 @@ def log_returns( def end_episode( self, episode_id: str, observation: Union[EnvObsType, MultiAgentDict] ) -> None: - """Record the end of an episode. - - Args: - episode_id: Episode id returned from start_episode(). - observation: Current environment observation. - """ - if self.local: self._update_local_policy() return self.env.end_episode(episode_id, observation) @@ -276,9 +206,8 @@ def _update_local_policy(self, force=False): self.last_updated = time.time() +@OldAPIStack class _LocalInferenceThread(threading.Thread): - """Thread that handles experience generation (worker.sample() loop).""" - def __init__(self, rollout_worker, send_fn): super().__init__() self.daemon = True @@ -313,13 +242,8 @@ def run(self): logger.error("Error: inference worker thread died!", e) +@OldAPIStack def _auto_wrap_external(real_env_creator): - """Wrap an environment in the ExternalEnv interface if needed. - - Args: - real_env_creator: Create an env given the env_config. - """ - def wrapped_creator(env_config): real_env = real_env_creator(env_config) if not isinstance(real_env, (ExternalEnv, ExternalMultiAgentEnv)): @@ -352,14 +276,8 @@ def run(self): return wrapped_creator +@OldAPIStack def _create_embedded_rollout_worker(kwargs, send_fn): - """Create a local rollout worker and a thread that samples from it. - - Args: - kwargs: Args for the RolloutWorker constructor. - send_fn: Function to send a JSON request to the server. - """ - # Since the server acts as an input datasource, we have to reset the # input config to the default, which runs env rollouts. kwargs = kwargs.copy() diff --git a/rllib/env/policy_server_input.py b/rllib/env/policy_server_input.py index 70bc2d130757..48598a51d9eb 100644 --- a/rllib/env/policy_server_input.py +++ b/rllib/env/policy_server_input.py @@ -16,7 +16,7 @@ from ray.rllib.offline.input_reader import InputReader from ray.rllib.offline.io_context import IOContext from ray.rllib.policy.sample_batch import SampleBatch -from ray.rllib.utils.annotations import override, PublicAPI +from ray.rllib.utils.annotations import OldAPIStack, override from ray.rllib.evaluation.metrics import RolloutMetrics from ray.rllib.evaluation.sampler import SamplerInput from ray.rllib.utils.typing import SampleBatchType @@ -25,58 +25,8 @@ logger = logging.getLogger(__name__) -@PublicAPI +@OldAPIStack class PolicyServerInput(ThreadingMixIn, HTTPServer, InputReader): - """REST policy server that acts as an offline data source. - - This launches a multi-threaded server that listens on the specified host - and port to serve policy requests and forward experiences to RLlib. For - high performance experience collection, it implements InputReader. - - For an example, run `examples/envs/external_envs/cartpole_server.py` along - with `examples/envs/external_envs/cartpole_client.py --inference-mode=local|remote`. - - WARNING: This class is not meant to be publicly exposed. Anyone that can - communicate with this server can execute arbitary code on the machine. Use - this with caution, in isolated environments, and at your own risk. - - .. testcode:: - :skipif: True - - import gymnasium as gym - from ray.rllib.algorithms.ppo import PPOConfig - from ray.rllib.env.policy_client import PolicyClient - from ray.rllib.env.policy_server_input import PolicyServerInput - addr, port = ... - config = ( - PPOConfig() - .api_stack( - enable_rl_module_and_learner=False, - enable_env_runner_and_connector_v2=False, - ) - .environment("CartPole-v1") - .offline_data( - input_=lambda ioctx: PolicyServerInput(ioctx, addr, port) - ) - # Run just 1 server (in the Algorithm's EnvRunnerGroup). - .env_runners(num_env_runners=0) - ) - algo = config.build() - while True: - algo.train() - client = PolicyClient( - "localhost:9900", inference_mode="local") - eps_id = client.start_episode() - env = gym.make("CartPole-v1") - obs, info = env.reset() - action = client.get_action(eps_id, obs) - _, reward, _, _, _ = env.step(action) - client.log_returns(eps_id, reward) - client.log_returns(eps_id, reward) - algo.stop() - """ - - @PublicAPI def __init__( self, ioctx: IOContext, @@ -85,29 +35,6 @@ def __init__( idle_timeout: float = 3.0, max_sample_queue_size: int = 20, ): - """Create a PolicyServerInput. - - This class implements rllib.offline.InputReader, and can be used with - any Algorithm by configuring - - [AlgorithmConfig object] - .env_runners(num_env_runners=0) - .offline_data(input_=lambda ioctx: PolicyServerInput(ioctx, addr, port)) - - Note that by setting num_env_runners: 0, the algorithm will only create one - rollout worker / PolicyServerInput. Clients can connect to the launched - server using rllib.env.PolicyClient. You can increase the number of available - connections (ports) by setting num_env_runners to a larger number. The ports - used will then be `port` + the worker's index. - - Args: - ioctx: IOContext provided by RLlib. - address: Server addr (e.g., "localhost"). - port: Server port (e.g., 9900). - max_queue_size: The maximum size for the sample queue. Once full, will - purge (throw away) 50% of all samples, oldest first, and continue. - """ - self.rollout_worker = ioctx.worker # Protect ourselves from having a bottleneck on the server (learning) side. # Once the queue (deque) is full, we throw away 50% (oldest diff --git a/rllib/env/tcp_client_inference_env_runner.py b/rllib/env/tcp_client_inference_env_runner.py index 8aaf29749a28..09f8f4a2e715 100644 --- a/rllib/env/tcp_client_inference_env_runner.py +++ b/rllib/env/tcp_client_inference_env_runner.py @@ -1,589 +1,6 @@ -import base64 -from collections import defaultdict -import gzip -import json -import pathlib -import socket -import tempfile -import threading -import time -from typing import Collection, DefaultDict, List, Optional, Union - -import gymnasium as gym -import numpy as np -import onnxruntime - -from ray.rllib.core import ( - Columns, - COMPONENT_RL_MODULE, - DEFAULT_AGENT_ID, - DEFAULT_MODULE_ID, -) -from ray.rllib.env import INPUT_ENV_SPACES -from ray.rllib.env.env_runner import EnvRunner -from ray.rllib.env.single_agent_env_runner import SingleAgentEnvRunner -from ray.rllib.env.single_agent_episode import SingleAgentEpisode -from ray.rllib.env.utils.external_env_protocol import RLlink as rllink -from ray.rllib.utils.annotations import ExperimentalAPI, override -from ray.rllib.utils.checkpoints import Checkpointable -from ray.rllib.utils.framework import try_import_torch -from ray.rllib.utils.metrics import ( - EPISODE_DURATION_SEC_MEAN, - EPISODE_LEN_MAX, - EPISODE_LEN_MEAN, - EPISODE_LEN_MIN, - EPISODE_RETURN_MAX, - EPISODE_RETURN_MEAN, - EPISODE_RETURN_MIN, - WEIGHTS_SEQ_NO, +from ray.rllib.env.external.env_runner_server_for_external_inference import ( + EnvRunnerServerForExternalInference, ) -from ray.rllib.utils.metrics.metrics_logger import MetricsLogger -from ray.rllib.utils.numpy import softmax -from ray.rllib.utils.typing import EpisodeID, StateDict - -torch, _ = try_import_torch() - - -@ExperimentalAPI -class TcpClientInferenceEnvRunner(EnvRunner, Checkpointable): - """An EnvRunner communicating with an external env through a TCP socket. - - This implementation assumes: - - Only one external client ever connects to this env runner. - - The external client performs inference locally through an ONNX model. Thus, - samples are sent in bulk once a certain number of timesteps has been executed on the - client's side (no individual action requests). - - A copy of the RLModule is kept at all times on the env runner, but never used - for inference, only as a data (weights) container. - TODO (sven): The above might be inefficient as we have to store basically two - models, one in this EnvRunner, one in the env (as ONNX). - - There is no environment and no connectors on this env runner. The external env - is responsible for generating all the data to create episodes. - """ - - @override(EnvRunner) - def __init__(self, *, config, **kwargs): - """ - Initializes a TcpClientInferenceEnvRunner instance. - - Args: - config: The AlgorithmConfig to use for setup. - - Keyword Args: - port: The base port number. The server socket is then actually bound to - `port` + self.worker_index. - """ - super().__init__(config=config, **kwargs) - - self.worker_index: int = kwargs.get("worker_index", 0) - - self._weights_seq_no = 0 - - # Build the module from its spec. - module_spec = self.config.get_rl_module_spec( - spaces=self.get_spaces(), inference_only=True - ) - self.module = module_spec.build() - - self.host = "localhost" - self.port = int(self.config.env_config.get("port", 5555)) + self.worker_index - self.server_socket = None - self.client_socket = None - self.address = None - - self.metrics = MetricsLogger() - - self._episode_chunks_to_return: Optional[List[SingleAgentEpisode]] = None - self._done_episodes_for_metrics: List[SingleAgentEpisode] = [] - self._ongoing_episodes_for_metrics: DefaultDict[ - EpisodeID, List[SingleAgentEpisode] - ] = defaultdict(list) - - self._sample_lock = threading.Lock() - self._on_policy_lock = threading.Lock() - self._blocked_on_state = False - - # Start a background thread for client communication. - self.thread = threading.Thread( - target=self._client_message_listener, daemon=True - ) - self.thread.start() - - @override(EnvRunner) - def assert_healthy(self): - """Checks that the server socket is open and listening.""" - assert ( - self.server_socket is not None - ), "Server socket is None (not connected, not listening)." - - @override(EnvRunner) - def sample(self, **kwargs): - """Waits for the client to send episodes.""" - while True: - with self._sample_lock: - if self._episode_chunks_to_return is not None: - num_env_steps = 0 - num_episodes_completed = 0 - for eps in self._episode_chunks_to_return: - if eps.is_done: - self._done_episodes_for_metrics.append(eps) - num_episodes_completed += 1 - else: - self._ongoing_episodes_for_metrics[eps.id_].append(eps) - num_env_steps += len(eps) - - ret = self._episode_chunks_to_return - self._episode_chunks_to_return = None - - SingleAgentEnvRunner._increase_sampled_metrics( - self, num_env_steps, num_episodes_completed - ) - - return ret - time.sleep(0.01) - - @override(EnvRunner) - def get_metrics(self): - # TODO (sven): We should probably make this a utility function to be called - # from within Single/MultiAgentEnvRunner and other EnvRunner subclasses, as - # needed. - # Compute per-episode metrics (only on already completed episodes). - for eps in self._done_episodes_for_metrics: - assert eps.is_done - episode_length = len(eps) - episode_return = eps.get_return() - episode_duration_s = eps.get_duration_s() - # Don't forget about the already returned chunks of this episode. - if eps.id_ in self._ongoing_episodes_for_metrics: - for eps2 in self._ongoing_episodes_for_metrics[eps.id_]: - episode_length += len(eps2) - episode_return += eps2.get_return() - episode_duration_s += eps2.get_duration_s() - del self._ongoing_episodes_for_metrics[eps.id_] - - self._log_episode_metrics( - episode_length, episode_return, episode_duration_s - ) - - # Now that we have logged everything, clear cache of done episodes. - self._done_episodes_for_metrics.clear() - - # Return reduced metrics. - return self.metrics.reduce() - - def get_spaces(self): - return { - INPUT_ENV_SPACES: (self.config.observation_space, self.config.action_space), - DEFAULT_MODULE_ID: ( - self.config.observation_space, - self.config.action_space, - ), - } - - @override(EnvRunner) - def stop(self): - """Closes the client and server sockets.""" - self._close_sockets_if_necessary() - - @override(Checkpointable) - def get_ctor_args_and_kwargs(self): - return ( - (), # *args - {"config": self.config}, # **kwargs - ) - - @override(Checkpointable) - def get_checkpointable_components(self): - return [ - (COMPONENT_RL_MODULE, self.module), - ] - - @override(Checkpointable) - def get_state( - self, - components: Optional[Union[str, Collection[str]]] = None, - *, - not_components: Optional[Union[str, Collection[str]]] = None, - **kwargs, - ) -> StateDict: - return {} - - @override(Checkpointable) - def set_state(self, state: StateDict) -> None: - # Update the RLModule state. - if COMPONENT_RL_MODULE in state: - # A missing value for WEIGHTS_SEQ_NO or a value of 0 means: Force the - # update. - weights_seq_no = state.get(WEIGHTS_SEQ_NO, 0) - - # Only update the weigths, if this is the first synchronization or - # if the weights of this `EnvRunner` lacks behind the actual ones. - if weights_seq_no == 0 or self._weights_seq_no < weights_seq_no: - rl_module_state = state[COMPONENT_RL_MODULE] - if ( - isinstance(rl_module_state, dict) - and DEFAULT_MODULE_ID in rl_module_state - ): - rl_module_state = rl_module_state[DEFAULT_MODULE_ID] - self.module.set_state(rl_module_state) - - # Update our weights_seq_no, if the new one is > 0. - if weights_seq_no > 0: - self._weights_seq_no = weights_seq_no - - if self._blocked_on_state is True: - self._send_set_state_message() - self._blocked_on_state = False - - def _client_message_listener(self): - """Entry point for the listener thread.""" - - # Set up the server socket and bind to the specified host and port. - self._recycle_sockets() - - # Enter an endless message receival- and processing loop. - while True: - # As long as we are blocked on a new state, sleep a bit and continue. - # Do NOT process any incoming messages (until we send out the new state - # back to the client). - if self._blocked_on_state is True: - time.sleep(0.01) - continue - - try: - # Blocking call to get next message. - msg_type, msg_body = _get_message(self.client_socket) - - # Process the message received based on its type. - # Initial handshake. - if msg_type == rllink.PING: - self._send_pong_message() - - # Episode data from the client. - elif msg_type in [ - rllink.EPISODES, - rllink.EPISODES_AND_GET_STATE, - ]: - self._process_episodes_message(msg_type, msg_body) - - # Client requests the state (model weights). - elif msg_type == rllink.GET_STATE: - self._send_set_state_message() - - # Clients requests some (relevant) config information. - elif msg_type == rllink.GET_CONFIG: - self._send_set_config_message() - - except ConnectionError as e: - print(f"Messaging/connection error {e}! Recycling sockets ...") - self._recycle_sockets(5.0) - continue - - def _recycle_sockets(self, sleep: float = 0.0): - # Close all old sockets, if they exist. - self._close_sockets_if_necessary() - - time.sleep(sleep) - - # Start listening on the configured port. - self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # Allow reuse of the address. - self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.server_socket.bind((self.host, self.port)) - # Listen for a single connection. - self.server_socket.listen(1) - print(f"Waiting for client to connect to port {self.port}...") - - self.client_socket, self.address = self.server_socket.accept() - print(f"Connected to client at {self.address}") - - def _close_sockets_if_necessary(self): - if self.client_socket: - self.client_socket.close() - if self.server_socket: - self.server_socket.close() - - def _send_pong_message(self): - _send_message(self.client_socket, {"type": rllink.PONG.name}) - - def _process_episodes_message(self, msg_type, msg_body): - # On-policy training -> we have to block until we get a new `set_state` call - # (b/c the learning step is done and we can sent new weights back to all - # clients). - if msg_type == rllink.EPISODES_AND_GET_STATE: - self._blocked_on_state = True - - episodes = [] - for episode_data in msg_body["episodes"]: - episode = SingleAgentEpisode( - observation_space=self.config.observation_space, - observations=[np.array(o) for o in episode_data[Columns.OBS]], - action_space=self.config.action_space, - actions=episode_data[Columns.ACTIONS], - rewards=episode_data[Columns.REWARDS], - extra_model_outputs={ - Columns.ACTION_DIST_INPUTS: [ - np.array(a) for a in episode_data[Columns.ACTION_DIST_INPUTS] - ], - Columns.ACTION_LOGP: episode_data[Columns.ACTION_LOGP], - }, - terminated=episode_data["is_terminated"], - truncated=episode_data["is_truncated"], - len_lookback_buffer=0, - ) - episodes.append(episode.to_numpy()) - - # Push episodes into the to-be-returned list (for `sample()` requests). - with self._sample_lock: - if isinstance(self._episode_chunks_to_return, list): - self._episode_chunks_to_return.extend(episodes) - else: - self._episode_chunks_to_return = episodes - - def _send_set_state_message(self): - with tempfile.TemporaryDirectory() as dir: - onnx_file = pathlib.Path(dir) / "_temp_model.onnx" - torch.onnx.export( - self.module, - { - "batch": { - "obs": torch.randn(1, *self.config.observation_space.shape) - } - }, - onnx_file, - export_params=True, - ) - with open(onnx_file, "rb") as f: - compressed = gzip.compress(f.read()) - onnx_binary = base64.b64encode(compressed).decode("utf-8") - _send_message( - self.client_socket, - { - "type": rllink.SET_STATE.name, - "onnx_file": onnx_binary, - WEIGHTS_SEQ_NO: self._weights_seq_no, - }, - ) - - def _send_set_config_message(self): - _send_message( - self.client_socket, - { - "type": rllink.SET_CONFIG.name, - "env_steps_per_sample": self.config.get_rollout_fragment_length( - worker_index=self.worker_index - ), - "force_on_policy": True, - }, - ) - - def _log_episode_metrics(self, length, ret, sec): - # Log general episode metrics. - # To mimic the old API stack behavior, we'll use `window` here for - # these particular stats (instead of the default EMA). - win = self.config.metrics_num_episodes_for_smoothing - self.metrics.log_value(EPISODE_LEN_MEAN, length, window=win) - self.metrics.log_value(EPISODE_RETURN_MEAN, ret, window=win) - self.metrics.log_value(EPISODE_DURATION_SEC_MEAN, sec, window=win) - # Per-agent returns. - self.metrics.log_value( - ("agent_episode_returns_mean", DEFAULT_AGENT_ID), ret, window=win - ) - # Per-RLModule returns. - self.metrics.log_value( - ("module_episode_returns_mean", DEFAULT_MODULE_ID), ret, window=win - ) - - # For some metrics, log min/max as well. - self.metrics.log_value(EPISODE_LEN_MIN, length, reduce="min", window=win) - self.metrics.log_value(EPISODE_RETURN_MIN, ret, reduce="min", window=win) - self.metrics.log_value(EPISODE_LEN_MAX, length, reduce="max", window=win) - self.metrics.log_value(EPISODE_RETURN_MAX, ret, reduce="max", window=win) - - -def _send_message(sock_, message: dict): - """Sends a message to the client with a length header.""" - body = json.dumps(message).encode("utf-8") - header = str(len(body)).zfill(8).encode("utf-8") - try: - sock_.sendall(header + body) - except Exception as e: - raise ConnectionError( - f"Error sending message {message} to server on socket {sock_}! " - f"Original error was: {e}" - ) - - -def _get_message(sock_): - """Receives a message from the client following the length-header protocol.""" - try: - # Read the length header (8 bytes) - header = _get_num_bytes(sock_, 8) - msg_length = int(header.decode("utf-8")) - # Read the message body - body = _get_num_bytes(sock_, msg_length) - # Decode JSON. - message = json.loads(body.decode("utf-8")) - # Check for proper protocol. - if "type" not in message: - raise ConnectionError( - "Protocol Error! Message from peer does not contain `type` field." - ) - return rllink(message.pop("type")), message - except Exception as e: - raise ConnectionError( - f"Error receiving message from peer on socket {sock_}! " - f"Original error was: {e}" - ) - - -def _get_num_bytes(sock_, num_bytes): - """Helper function to receive a specific number of bytes.""" - data = b"" - while len(data) < num_bytes: - packet = sock_.recv(num_bytes - len(data)) - if not packet: - raise ConnectionError(f"No data received from socket {sock_}!") - data += packet - return data - - -def _dummy_client(port: int = 5556): - """A dummy client that runs CartPole and acts as a testing external env.""" - - def _set_state(msg_body): - with tempfile.TemporaryDirectory(): - with open("_temp_onnx", "wb") as f: - f.write( - gzip.decompress( - base64.b64decode(msg_body["onnx_file"].encode("utf-8")) - ) - ) - onnx_session = onnxruntime.InferenceSession("_temp_onnx") - output_names = [o.name for o in onnx_session.get_outputs()] - return onnx_session, output_names - - # Connect to server. - while True: - try: - print(f"Trying to connect to localhost:{port} ...") - sock_ = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock_.connect(("localhost", port)) - break - except ConnectionRefusedError: - time.sleep(5) - - # Send ping-pong. - _send_message(sock_, {"type": rllink.PING.name}) - msg_type, msg_body = _get_message(sock_) - assert msg_type == rllink.PONG - - # Request config. - _send_message(sock_, {"type": rllink.GET_CONFIG.name}) - msg_type, msg_body = _get_message(sock_) - assert msg_type == rllink.SET_CONFIG - env_steps_per_sample = msg_body["env_steps_per_sample"] - force_on_policy = msg_body["force_on_policy"] - - # Request ONNX weights. - _send_message(sock_, {"type": rllink.GET_STATE.name}) - msg_type, msg_body = _get_message(sock_) - assert msg_type == rllink.SET_STATE - onnx_session, output_names = _set_state(msg_body) - - # Episode collection buckets. - episodes = [] - observations = [] - actions = [] - action_dist_inputs = [] - action_logps = [] - rewards = [] - - timesteps = 0 - episode_return = 0.0 - - # Start actual env loop. - env = gym.make("CartPole-v1") - obs, info = env.reset() - observations.append(obs.tolist()) - - while True: - timesteps += 1 - # Perform action inference using the ONNX model. - logits = onnx_session.run( - output_names, - {"onnx::Gemm_0": np.array([obs], np.float32)}, - )[0][ - 0 - ] # [0]=first return item, [0]=batch size 1 - - # Stochastic sample. - action_probs = softmax(logits) - action = int(np.random.choice(list(range(env.action_space.n)), p=action_probs)) - logp = float(np.log(action_probs[action])) - - # Perform the env step. - obs, reward, terminated, truncated, info = env.step(action) - - # Collect step data. - observations.append(obs.tolist()) - actions.append(action) - action_dist_inputs.append(logits.tolist()) - action_logps.append(logp) - rewards.append(reward) - episode_return += reward - - # We have to create a new episode record. - if timesteps == env_steps_per_sample or terminated or truncated: - episodes.append( - { - Columns.OBS: observations, - Columns.ACTIONS: actions, - Columns.ACTION_DIST_INPUTS: action_dist_inputs, - Columns.ACTION_LOGP: action_logps, - Columns.REWARDS: rewards, - "is_terminated": terminated, - "is_truncated": truncated, - } - ) - # We collected enough samples -> Send them to server. - if timesteps == env_steps_per_sample: - # Make sure the amount of data we collected is correct. - assert sum(len(e["actions"]) for e in episodes) == env_steps_per_sample - - # Send the data to the server. - if force_on_policy: - _send_message( - sock_, - { - "type": rllink.EPISODES_AND_GET_STATE.name, - "episodes": episodes, - "timesteps": timesteps, - }, - ) - # We are forced to sample on-policy. Have to wait for a response - # with the state (weights) in it. - msg_type, msg_body = _get_message(sock_) - assert msg_type == rllink.SET_STATE - onnx_session, output_names = _set_state(msg_body) - - # Sampling doesn't have to be on-policy -> continue collecting - # samples. - else: - raise NotImplementedError - - episodes = [] - timesteps = 0 - - # Set new buckets to empty lists (for next episode). - observations = [observations[-1]] - actions = [] - action_dist_inputs = [] - action_logps = [] - rewards = [] - # The episode is done -> Reset. - if terminated or truncated: - obs, _ = env.reset() - observations = [obs.tolist()] - episode_return = 0.0 +# @Deprecated +TcpClientInferenceEnvRunner = EnvRunnerServerForExternalInference diff --git a/rllib/env/utils/external_env_protocol.py b/rllib/env/utils/external_env_protocol.py index 0234d273470f..7fedebedcd26 100644 --- a/rllib/env/utils/external_env_protocol.py +++ b/rllib/env/utils/external_env_protocol.py @@ -1,45 +1,8 @@ -from enum import Enum - -from ray.util.annotations import PublicAPI - - -@PublicAPI(stability="alpha") -class RLlink(Enum): - # Requests: Client (external env) -> Server (RLlib). - # ---- - # Ping command (initial handshake). - PING = "PING" - # List of episodes (similar to what an EnvRunner.sample() call would return). - EPISODES = "EPISODES" - # Request state (e.g. model weights). - GET_STATE = "GET_STATE" - # Request (relevant) config. - GET_CONFIG = "GET_CONFIG" - # Send episodes and request the next state update right after that. - # Clients sending this message should wait for a SET_STATE message as an immediate - # response. Useful for external samplers that must collect on-policy data. - EPISODES_AND_GET_STATE = "EPISODES_AND_GET_STATE" - - # Responses: Server (RLlib) -> Client (external env). - # ---- - # Pong response (initial handshake). - PONG = "PONG" - # Set state (e.g. model weights). - SET_STATE = "SET_STATE" - # Set (relevant) config. - SET_CONFIG = "SET_CONFIG" - - # @OldAPIStack (to be deprecated soon). - ACTION_SPACE = "ACTION_SPACE" - OBSERVATION_SPACE = "OBSERVATION_SPACE" - GET_WORKER_ARGS = "GET_WORKER_ARGS" - GET_WEIGHTS = "GET_WEIGHTS" - REPORT_SAMPLES = "REPORT_SAMPLES" - START_EPISODE = "START_EPISODE" - GET_ACTION = "GET_ACTION" - LOG_ACTION = "LOG_ACTION" - LOG_RETURNS = "LOG_RETURNS" - END_EPISODE = "END_EPISODE" - - def __str__(self): - return self.name +from ray.rllib.env.external.rllink import RLlink # noqa +from ray.rllib.utils.deprecation import deprecation_warning + +deprecation_warning( + old="ray.rllib.env.utils.external_env_protocol", + new="ray.rllib.env.external.rllink", + error=False, +) diff --git a/rllib/env/utils/infinite_lookback_buffer.py b/rllib/env/utils/infinite_lookback_buffer.py index 26f76fbc31ae..76004f0200fa 100644 --- a/rllib/env/utils/infinite_lookback_buffer.py +++ b/rllib/env/utils/infinite_lookback_buffer.py @@ -13,8 +13,10 @@ get_base_struct_from_space, to_jsonable_if_needed, ) +from ray.util.annotations import DeveloperAPI +@DeveloperAPI class InfiniteLookbackBuffer: @property def space(self): diff --git a/rllib/env/wrappers/unity3d_env.py b/rllib/env/wrappers/unity3d_env.py index 45f0f910af92..82babd666741 100644 --- a/rllib/env/wrappers/unity3d_env.py +++ b/rllib/env/wrappers/unity3d_env.py @@ -7,28 +7,14 @@ from ray.rllib.env.multi_agent_env import MultiAgentEnv from ray.rllib.policy.policy import PolicySpec -from ray.rllib.utils.annotations import PublicAPI +from ray.rllib.utils.annotations import OldAPIStack from ray.rllib.utils.typing import MultiAgentDict, PolicyID, AgentID logger = logging.getLogger(__name__) -@PublicAPI +@OldAPIStack class Unity3DEnv(MultiAgentEnv): - """A MultiAgentEnv representing a single Unity3D game instance. - - For an example on how to use this Env with a running Unity3D editor - or with a compiled game, see: - `rllib/examples/unity3d_env_local.py` - For an example on how to use it inside a Unity game client, which - connects to an RLlib Policy server, see: - `rllib/examples/envs/external_envs/unity3d_[client|server].py` - - Supports all Unity3D (MLAgents) examples, multi- or single-agent and - gets converted automatically into an ExternalMultiAgentEnv, when used - inside an RLlib PolicyClient for cloud/distributed training of Unity games. - """ - # Default base port when connecting directly to the Editor _BASE_PORT_EDITOR = 5004 # Default base port when connecting to a compiled environment @@ -45,25 +31,6 @@ def __init__( timeout_wait: int = 300, episode_horizon: int = 1000, ): - """Initializes a Unity3DEnv object. - - Args: - file_name (Optional[str]): Name of the Unity game binary. - If None, will assume a locally running Unity3D editor - to be used, instead. - port (Optional[int]): Port number to connect to Unity environment. - seed: A random seed value to use for the Unity3D game. - no_graphics: Whether to run the Unity3D simulator in - no-graphics mode. Default: False. - timeout_wait: Time (in seconds) to wait for connection from - the Unity3D instance. - episode_horizon: A hard horizon to abide to. After at most - this many steps (per-agent episode `step()` calls), the - Unity3D game is reset and will start again (finishing the - multi-agent episode that the game represents). - Note: The game itself may contain its own episode length - limits, which are always obeyed (on top of this value here). - """ super().__init__() if file_name is None: @@ -120,24 +87,6 @@ def step( ) -> Tuple[ MultiAgentDict, MultiAgentDict, MultiAgentDict, MultiAgentDict, MultiAgentDict ]: - """Performs one multi-agent step through the game. - - Args: - action_dict: Multi-agent action dict with: - keys=agent identifier consisting of - [MLagents behavior name, e.g. "Goalie?team=1"] + "_" + - [Agent index, a unique MLAgent-assigned index per single agent] - - Returns: - tuple: - - obs: Multi-agent observation dict. - Only those observations for which to get new actions are - returned. - - rewards: Rewards dict matching `obs`. - - dones: Done dict with only an __all__ multi-agent entry in - it. __all__=True, if episode is done for all agents. - - infos: An (empty) info dict. - """ from mlagents_envs.base_env import ActionTuple # Set only the required actions (from the DecisionSteps) in Unity3D. @@ -199,18 +148,6 @@ def reset( return obs, infos def _get_step_results(self): - """Collects those agents' obs/rewards that have to act in next `step`. - - Returns: - Tuple: - obs: Multi-agent observation dict. - Only those observations for which to get new actions are - returned. - rewards: Rewards dict matching `obs`. - dones: Done dict with only an __all__ multi-agent entry in it. - __all__=True, if episode is done for all agents. - infos: An (empty) info dict. - """ obs = {} rewards = {} infos = {} diff --git a/rllib/examples/envs/classes/utils/dummy_external_client.py b/rllib/examples/envs/classes/utils/dummy_external_client.py new file mode 100644 index 000000000000..8cc1bf0af6f7 --- /dev/null +++ b/rllib/examples/envs/classes/utils/dummy_external_client.py @@ -0,0 +1,126 @@ +import pickle +import socket +import time + +import gymnasium as gym +import numpy as np + +from ray.rllib.core import ( + Columns, + COMPONENT_RL_MODULE, +) +from ray.rllib.env.external.rllink import ( + get_rllink_message, + send_rllink_message, + RLlink, +) +from ray.rllib.env.single_agent_episode import SingleAgentEpisode +from ray.rllib.utils.framework import try_import_torch +from ray.rllib.utils.numpy import softmax + +torch, _ = try_import_torch() + + +def _dummy_external_client(port: int = 5556): + """A dummy client that runs CartPole and acts as a testing external env.""" + + def _set_state(msg_body, rl_module): + rl_module.set_state(msg_body[COMPONENT_RL_MODULE]) + # return msg_body[WEIGHTS_SEQ_NO] + + # Connect to server. + while True: + try: + print(f"Trying to connect to localhost:{port} ...") + sock_ = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock_.connect(("localhost", port)) + break + except ConnectionRefusedError: + time.sleep(5) + + # Send ping-pong. + send_rllink_message(sock_, {"type": RLlink.PING.name}) + msg_type, msg_body = get_rllink_message(sock_) + assert msg_type == RLlink.PONG + + # Request config. + send_rllink_message(sock_, {"type": RLlink.GET_CONFIG.name}) + msg_type, msg_body = get_rllink_message(sock_) + assert msg_type == RLlink.SET_CONFIG + + config = pickle.loads(msg_body["config"]) + # Create the RLModule. + rl_module = config.get_rl_module_spec().build() + + # Request state/weights. + send_rllink_message(sock_, {"type": RLlink.GET_STATE.name}) + msg_type, msg_body = get_rllink_message(sock_) + assert msg_type == RLlink.SET_STATE + _set_state(msg_body["state"], rl_module) + + env_steps_per_sample = config.get_rollout_fragment_length() + + # Start actual env loop. + env = gym.make("CartPole-v1") + obs, _ = env.reset() + episode = SingleAgentEpisode(observations=[obs]) + episodes = [episode] + + while True: + # Perform action inference using the RLModule. + logits = rl_module.forward_exploration( + batch={ + Columns.OBS: torch.tensor(np.array([obs], np.float32)), + } + )[Columns.ACTION_DIST_INPUTS][ + 0 + ].numpy() # [0]=batch size 1 + + # Stochastic sample. + action_probs = softmax(logits) + action = int(np.random.choice(list(range(env.action_space.n)), p=action_probs)) + logp = float(np.log(action_probs[action])) + + # Perform the env step. + obs, reward, terminated, truncated, _ = env.step(action) + + # Collect step data. + episode.add_env_step( + action=action, + reward=reward, + observation=obs, + terminated=terminated, + truncated=truncated, + extra_model_outputs={ + Columns.ACTION_DIST_INPUTS: logits, + Columns.ACTION_LOGP: logp, + }, + ) + + # We collected enough samples -> Send them to server. + if sum(map(len, episodes)) == env_steps_per_sample: + # Send the data to the server. + send_rllink_message( + sock_, + { + "type": RLlink.EPISODES_AND_GET_STATE.name, + "episodes": [e.get_state() for e in episodes], + "timesteps": env_steps_per_sample, + }, + ) + # We are forced to sample on-policy. Have to wait for a response + # with the state (weights) in it. + msg_type, msg_body = get_rllink_message(sock_) + assert msg_type == RLlink.SET_STATE + _set_state(msg_body["state"], rl_module) + + episodes = [] + if not episode.is_done: + episode = episode.cut() + episodes.append(episode) + + # If episode is done, reset env and create a new episode. + if episode.is_done: + obs, _ = env.reset() + episode = SingleAgentEpisode(observations=[obs]) + episodes.append(episode) diff --git a/rllib/examples/envs/env_connecting_to_rllib_w_tcp_client.py b/rllib/examples/envs/env_connecting_to_rllib_w_tcp_client.py index 3a757bab5993..aedd97237ce0 100644 --- a/rllib/examples/envs/env_connecting_to_rllib_w_tcp_client.py +++ b/rllib/examples/envs/env_connecting_to_rllib_w_tcp_client.py @@ -1,6 +1,6 @@ """Example of running against a TCP-connected external env performing its own inference. -The example uses a custom EnvRunner (TcpClientInferenceEnvRunner) to allow +The example uses a custom EnvRunner (EnvRunnerServerForExternalInference) to allow connections from one or more TCP clients to RLlib's EnvRunner actors, which act as RL servers. In this example, action inference for stepping the env is performed on the client's @@ -60,16 +60,17 @@ ConnectionError: Error receiving message from peer on socket ... ``` """ -from functools import partial import threading import gymnasium as gym import numpy as np from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig -from ray.rllib.env.tcp_client_inference_env_runner import ( - _dummy_client, - TcpClientInferenceEnvRunner, +from ray.rllib.env.external.env_runner_server_for_external_inference import ( + EnvRunnerServerForExternalInference, +) +from ray.rllib.examples.envs.classes.utils.dummy_external_client import ( + _dummy_external_client, ) from ray.rllib.utils.test_utils import ( add_rllib_example_script_args, @@ -90,34 +91,44 @@ help="The port for RLlib's EnvRunner to listen to for incoming UE5 connections. " "You need to specify the same port inside your UE5 `RLlibClient` plugin.", ) +parser.add_argument( + "--use-dummy-client", + action="store_true", + help="If set, the script runs with its own external client acting as a " + "simulator. Otherwise connect on your own from your C++ application.", +) if __name__ == "__main__": args = parser.parse_args() - # Start the dummy CartPole client in a thread (and do its thing in parallel). - client_thread = threading.Thread( - target=partial( - _dummy_client, - port=args.port - + (args.num_env_runners if args.num_env_runners is not None else 1), - ), - ) - client_thread.start() + # Start the dummy CartPole "simulation". + if args.use_dummy_client: + threading.Thread( + target=_dummy_external_client, + args=( + # Connect to the first remote EnvRunner, of - if there is no remote one - + # to the local EnvRunner. + args.port + + (args.num_env_runners if args.num_env_runners is not None else 1), + ), + ).start() # Define the RLlib (server) config. base_config = ( get_trainable_cls(args.algo) .get_default_config() .environment( - observation_space=gym.spaces.Box(-1.0, 1.0, (4,), np.float32), + observation_space=gym.spaces.Box( + float("-inf"), float("-inf"), (4,), np.float32 + ), action_space=gym.spaces.Discrete(2), # EnvRunners listen on `port` + their worker index. env_config={"port": args.port}, ) .env_runners( # Point RLlib to the custom EnvRunner to be used here. - env_runner_cls=TcpClientInferenceEnvRunner, + env_runner_cls=EnvRunnerServerForExternalInference, ) .training( num_epochs=10, diff --git a/rllib/examples/envs/external_envs/cartpole_client.py b/rllib/examples/envs/external_envs/cartpole_client.py deleted file mode 100755 index d1ed0345f0a5..000000000000 --- a/rllib/examples/envs/external_envs/cartpole_client.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python - -# TODO (sven): Move this example script into the new API stack. - -""" -Example of running an external simulator (a simple CartPole env -in this case) against an RLlib policy server listening on one or more -HTTP-speaking port(s). See `cartpole_server.py` in this same directory for -how to start this server. - -This script will only create one single env altogether to illustrate -that RLlib can run w/o needing an internalized environment. - -Setup: -1) Start the policy server: - See `cartpole_server.py` on how to do this. -2) Run this client: - $ python cartpole_client.py --inference-mode=local|remote --[other options] - Use --help for help. - -In "local" inference-mode, the action computations are performed -inside the PolicyClient used in this script w/o sending an HTTP request -to the server. This reduces network communication overhead, but requires -the PolicyClient to create its own RolloutWorker (+Policy) based on -the server's config. The PolicyClient will retrieve this config automatically. -You do not need to define the RLlib config dict here! - -In "remote" inference mode, the PolicyClient will send action requests to the -server and not compute its own actions locally. The server then performs the -inference forward pass and returns the action to the client. - -In either case, the user of PolicyClient must: -- Declare new episodes and finished episodes to the PolicyClient. -- Log rewards to the PolicyClient. -- Call `get_action` to receive an action from the PolicyClient (whether it'd be - computed locally or remotely). -- Besides `get_action`, the user may let the PolicyClient know about - off-policy actions having been taken via `log_action`. This can be used in - combination with `get_action`, but will only work, if the connected server - runs an off-policy RL algorithm (such as DQN, SAC, or DDPG). -""" - -import argparse -import gymnasium as gym - -from ray.rllib.env.policy_client import PolicyClient -from ray._common.network_utils import build_address - -parser = argparse.ArgumentParser() -parser.add_argument( - "--no-train", action="store_true", help="Whether to disable training." -) -parser.add_argument( - "--inference-mode", type=str, default="local", choices=["local", "remote"] -) -parser.add_argument( - "--off-policy", - action="store_true", - help="Whether to compute random actions instead of on-policy " - "(Policy-computed) ones.", -) -parser.add_argument( - "--stop-reward", - type=float, - default=9999, - help="Stop once the specified reward is reached.", -) -parser.add_argument( - "--port", type=int, default=9900, help="The port to use (on localhost)." -) - -if __name__ == "__main__": - args = parser.parse_args() - - # The following line is the only instance, where an actual env will - # be created in this entire example (including the server side!). - # This is to demonstrate that RLlib does not require you to create - # unnecessary env objects within the PolicyClient/Server objects, but - # that only this following env and the loop below runs the entire - # training process. - env = gym.make("CartPole-v1") - - # If server has n workers, all ports between 9900 and 990[n-1] should - # be listened on. E.g. if server has num_env_runners=2, try 9900 or 9901. - # Note that no config is needed in this script as it will be defined - # on and sent from the server. - client = PolicyClient( - f"http://{build_address('localhost', args.port)}", - inference_mode=args.inference_mode, - ) - - # In the following, we will use our external environment (the CartPole - # env we created above) in connection with the PolicyClient to query - # actions (from the server if "remote"; if "local" we'll compute them - # on this client side), and send back observations and rewards. - - # Start a new episode. - obs, info = env.reset() - eid = client.start_episode(training_enabled=not args.no_train) - - rewards = 0.0 - while True: - # Compute an action randomly (off-policy) and log it. - if args.off_policy: - action = env.action_space.sample() - client.log_action(eid, obs, action) - # Compute an action locally or remotely (on server). - # No need to log it here as the action - else: - action = client.get_action(eid, obs) - - # Perform a step in the external simulator (env). - obs, reward, terminated, truncated, info = env.step(action) - rewards += reward - - # Log next-obs, rewards, and infos. - client.log_returns(eid, reward, info=info) - - # Reset the episode if done. - if terminated or truncated: - print("Total reward:", rewards) - if rewards >= args.stop_reward: - print("Target reward achieved, exiting") - exit(0) - - rewards = 0.0 - - # End the old episode. - client.end_episode(eid, obs) - - # Start a new episode. - obs, info = env.reset() - eid = client.start_episode(training_enabled=not args.no_train) diff --git a/rllib/examples/envs/external_envs/cartpole_server.py b/rllib/examples/envs/external_envs/cartpole_server.py deleted file mode 100755 index 65d86b14ef3e..000000000000 --- a/rllib/examples/envs/external_envs/cartpole_server.py +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env python - -# TODO (sven): Move this example script into the new API stack. - -""" -Example of running an RLlib policy server, allowing connections from -external environment running clients. The server listens on -(a simple CartPole env -in this case) against an RLlib policy server listening on one or more -HTTP-speaking ports. See `cartpole_client.py` in this same directory for how -to start any number of clients (after this server has been started). - -This script will not create any actual env to illustrate that RLlib can -run w/o needing an internalized environment. - -Setup: -1) Start this server: - $ python cartpole_server.py --num-workers --[other options] - Use --help for help. -2) Run n policy clients: - See `cartpole_client.py` on how to do this. - -The `num-workers` setting will allow you to distribute the incoming feed over n -listen sockets (in this example, between 9900 and 990n with n=worker_idx-1). -You may connect more than one policy client to any open listen port. -""" - -import argparse -import gymnasium as gym -import os - -import ray -from ray import tune -from ray.rllib.env.policy_server_input import PolicyServerInput -from ray.rllib.utils.metrics import ( - ENV_RUNNER_RESULTS, - EPISODE_RETURN_MEAN, - NUM_ENV_STEPS_SAMPLED_LIFETIME, -) -from ray.tune.logger import pretty_print -from ray.tune.registry import get_trainable_cls -from ray.tune.result import TRAINING_ITERATION - -SERVER_ADDRESS = "localhost" -# In this example, the user can run the policy server with -# n workers, opening up listen ports 9900 - 990n (n = num_env_runners - 1) -# to each of which different clients may connect. -SERVER_BASE_PORT = 9900 # + worker-idx - 1 - -CHECKPOINT_FILE = "last_checkpoint_{}.out" - - -def get_cli_args(): - """Create CLI parser and return parsed arguments""" - parser = argparse.ArgumentParser() - - # Example-specific args. - parser.add_argument( - "--port", - type=int, - default=SERVER_BASE_PORT, - help="The base-port to use (on localhost). " f"Default is {SERVER_BASE_PORT}.", - ) - parser.add_argument( - "--callbacks-verbose", - action="store_true", - help="Activates info-messages for different events on " - "server/client (episode steps, postprocessing, etc..).", - ) - parser.add_argument( - "--num-workers", - type=int, - default=2, - help="The number of workers to use. Each worker will create " - "its own listening socket for incoming experiences.", - ) - parser.add_argument( - "--no-restore", - action="store_true", - help="Do not restore from a previously saved checkpoint (location of " - "which is saved in `last_checkpoint_[algo-name].out`).", - ) - - # General args. - parser.add_argument( - "--run", - default="PPO", - choices=["APEX", "DQN", "IMPALA", "PPO", "R2D2"], - help="The RLlib-registered algorithm to use.", - ) - parser.add_argument("--num-cpus", type=int, default=3) - parser.add_argument( - "--framework", - choices=["tf", "tf2", "torch"], - default="torch", - help="The DL framework specifier.", - ) - parser.add_argument( - "--use-lstm", - action="store_true", - help="Whether to auto-wrap the model with an LSTM. Only valid option for " - "--run=[IMPALA|PPO|R2D2]", - ) - parser.add_argument( - "--stop-iters", type=int, default=200, help="Number of iterations to train." - ) - parser.add_argument( - "--stop-timesteps", - type=int, - default=500000, - help="Number of timesteps to train.", - ) - parser.add_argument( - "--stop-reward", - type=float, - default=80.0, - help="Reward at which we stop training.", - ) - parser.add_argument( - "--as-test", - action="store_true", - help="Whether this script should be run as a test: --stop-reward must " - "be achieved within --stop-timesteps AND --stop-iters.", - ) - parser.add_argument( - "--no-tune", - action="store_true", - help="Run without Tune using a manual train loop instead. Here," - "there is no TensorBoard support.", - ) - parser.add_argument( - "--local-mode", - action="store_true", - help="Init Ray in local mode for easier debugging.", - ) - - args = parser.parse_args() - print(f"Running with following CLI args: {args}") - return args - - -if __name__ == "__main__": - args = get_cli_args() - ray.init() - - # `InputReader` generator (returns None if no input reader is needed on - # the respective worker). - def _input(ioctx): - # We are remote worker or we are local worker with num_env_runners=0: - # Create a PolicyServerInput. - if ioctx.worker_index > 0 or ioctx.worker.num_workers == 0: - return PolicyServerInput( - ioctx, - SERVER_ADDRESS, - args.port + ioctx.worker_index - (1 if ioctx.worker_index > 0 else 0), - ) - # No InputReader (PolicyServerInput) needed. - else: - return None - - # Algorithm config. Note that this config is sent to the client only in case - # the client needs to create its own policy copy for local inference. - config = ( - get_trainable_cls(args.run).get_default_config() - # Indicate that the Algorithm we setup here doesn't need an actual env. - # Allow spaces to be determined by user (see below). - .environment( - env=None, - # TODO: (sven) make these settings unnecessary and get the information - # about the env spaces from the client. - observation_space=gym.spaces.Box(float("-inf"), float("inf"), (4,)), - action_space=gym.spaces.Discrete(2), - ) - # DL framework to use. - .framework(args.framework) - # Use the `PolicyServerInput` to generate experiences. - .offline_data(input_=_input) - # Use n worker processes to listen on different ports. - .env_runners( - num_env_runners=args.num_workers, - # Connectors are not compatible with the external env. - enable_connectors=False, - ) - # Disable OPE, since the rollouts are coming from online clients. - .evaluation(off_policy_estimation_methods={}) - # Set to INFO so we'll see the server's actual address:port. - .debugging(log_level="INFO") - ) - # Disable RLModules because they need connectors - - # DQN. - if args.run == "DQN" or args.run == "APEX" or args.run == "R2D2": - # Example of using DQN (supports off-policy actions). - config.update_from_dict( - { - "num_steps_sampled_before_learning_starts": 100, - "min_sample_timesteps_per_iteration": 200, - "n_step": 3, - "rollout_fragment_length": 4, - "train_batch_size": 8, - } - ) - config.model.update( - { - "fcnet_hiddens": [64], - "fcnet_activation": "linear", - } - ) - if args.run == "R2D2": - config.model["use_lstm"] = args.use_lstm - - elif args.run == "IMPALA": - config.update_from_dict( - { - "num_gpus": 0, - "model": {"use_lstm": args.use_lstm}, - } - ) - - # PPO. - else: - # Example of using PPO (does NOT support off-policy actions). - config.update_from_dict( - { - "rollout_fragment_length": 1000, - "train_batch_size": 4000, - "model": {"use_lstm": args.use_lstm}, - } - ) - - checkpoint_path = CHECKPOINT_FILE.format(args.run) - # Attempt to restore from checkpoint, if possible. - if not args.no_restore and os.path.exists(checkpoint_path): - checkpoint_path = open(checkpoint_path).read() - else: - checkpoint_path = None - - # Manual training loop (no Ray tune). - if args.no_tune: - algo = config.build() - - if checkpoint_path: - print("Restoring from checkpoint path", checkpoint_path) - algo.restore(checkpoint_path) - - # Serving and training loop. - ts = 0 - for _ in range(args.stop_iters): - results = algo.train() - print(pretty_print(results)) - checkpoint = algo.save().checkpoint - print("Last checkpoint", checkpoint) - with open(checkpoint_path, "w") as f: - f.write(checkpoint.path) - if ( - results[ENV_RUNNER_RESULTS][EPISODE_RETURN_MEAN] >= args.stop_reward - or ts >= args.stop_timesteps - ): - break - ts += results[f"{NUM_ENV_STEPS_SAMPLED_LIFETIME}"] - - algo.stop() - - # Run with Tune for auto env and algo creation and TensorBoard. - else: - print("Ignoring restore even if previous checkpoint is provided...") - - stop = { - TRAINING_ITERATION: args.stop_iters, - NUM_ENV_STEPS_SAMPLED_LIFETIME: args.stop_timesteps, - f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": args.stop_reward, - } - - tune.Tuner( - args.run, - param_space=config, - run_config=tune.RunConfig(stop=stop, verbose=2), - ).fit() diff --git a/rllib/examples/envs/external_envs/dummy_client_with_two_episodes.py b/rllib/examples/envs/external_envs/dummy_client_with_two_episodes.py deleted file mode 100644 index 8f201d5e01c0..000000000000 --- a/rllib/examples/envs/external_envs/dummy_client_with_two_episodes.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python - -# TODO (sven): Move this example script into the new API stack. - -""" -For testing purposes only. -Runs a policy client that starts two episodes, uses one for calculating actions -("action episode") and the other for logging those actions ("logging episode"). -Terminates the "logging episode" before computing a few more actions -from the "action episode". -The action episode is also started with the training_enabled=False flag so no -batches should be produced by this episode for training inside the -SampleCollector's `postprocess_trajectory` method. -""" - -import argparse -import gymnasium as gym -import ray - -from ray.rllib.env.policy_client import PolicyClient -from ray._common.network_utils import build_address - -parser = argparse.ArgumentParser() -parser.add_argument( - "--inference-mode", type=str, default="local", choices=["local", "remote"] -) -parser.add_argument( - "--off-policy", - action="store_true", - help="Whether to compute random actions instead of on-policy " - "(Policy-computed) ones.", -) -parser.add_argument( - "--port", type=int, default=9900, help="The port to use (on localhost)." -) -parser.add_argument("--dummy-arg", type=str, default="") - - -if __name__ == "__main__": - args = parser.parse_args() - - ray.init() - - # Use a CartPole-v1 env so this plays nicely with our cartpole server script. - env = gym.make("CartPole-v1") - - # Note that the RolloutWorker that is generated inside the client (in case - # of local inference) will contain only a RandomEnv dummy env to step through. - # The actual env we care about is the above generated CartPole one. - client = PolicyClient( - f"http://{build_address('localhost', args.port)}", - inference_mode=args.inference_mode, - ) - - # Get a dummy obs - dummy_obs, dummy_infos = env.reset() - dummy_reward = 1.3 - - # Start an episode to only compute actions (do NOT record this episode's - # trajectories in any returned SampleBatches sent to the server for learning). - action_eid = client.start_episode(training_enabled=False) - print(f"Starting action episode: {action_eid}.") - # Get some actions using the action episode - dummy_action = client.get_action(action_eid, dummy_obs) - print(f"Computing action 1 in action episode: {dummy_action}.") - dummy_action = client.get_action(action_eid, dummy_obs) - print(f"Computing action 2 in action episode: {dummy_action}.") - - # Start a log episode to log action and log rewards for learning. - log_eid = client.start_episode(training_enabled=True) - print(f"Starting logging episode: {log_eid}.") - # Produce an action, just for testing. - garbage_action = client.get_action(log_eid, dummy_obs) - # Log 1 action and 1 reward. - client.log_action(log_eid, dummy_obs, dummy_action) - client.log_returns(log_eid, dummy_reward) - print(f".. logged action + reward: {dummy_action} + {dummy_reward}") - - # Log 2 actions (w/o reward in the middle) and then one reward. - # The reward after the 1st of these actions should be considered 0.0. - client.log_action(log_eid, dummy_obs, dummy_action) - client.log_action(log_eid, dummy_obs, dummy_action) - client.log_returns(log_eid, dummy_reward) - print(f".. logged actions + reward: 2x {dummy_action} + {dummy_reward}") - - # End the log episode - client.end_episode(log_eid, dummy_obs) - print(".. ended logging episode") - - # Continue getting actions using the action episode - # The bug happens when executing the following line - dummy_action = client.get_action(action_eid, dummy_obs) - print(f"Computing action 3 in action episode: {dummy_action}.") - dummy_action = client.get_action(action_eid, dummy_obs) - print(f"Computing action 4 in action episode: {dummy_action}.") diff --git a/rllib/examples/envs/external_envs/unity3d_client.py b/rllib/examples/envs/external_envs/unity3d_client.py deleted file mode 100644 index 4160836d8f2c..000000000000 --- a/rllib/examples/envs/external_envs/unity3d_client.py +++ /dev/null @@ -1,133 +0,0 @@ -# TODO (sven): Move this example script into the new API stack. - -""" -Example of running a Unity3D client instance against an RLlib Policy server. -Unity3D clients can be run in distributed fashion on n nodes in the cloud -and all connect to the same RLlib server for faster sample collection. -For a locally running Unity3D example, see: -`examples/unity3d_env_local.py` - -To run this script on possibly different machines -against a central Policy server: -1) Install Unity3D and `pip install mlagents`. - -2) Compile a Unity3D example game with MLAgents support (e.g. 3DBall or any - other one that you created yourself) and place the compiled binary - somewhere, where your RLlib client script (see below) can access it. - -2.1) To find Unity3D MLAgent examples, first `pip install mlagents`, - then check out the `.../ml-agents/Project/Assets/ML-Agents/Examples/` - folder. - -3) Change your RLlib Policy server code so it knows the observation- and - action Spaces, the different Policies (called "behaviors" in Unity3D - MLAgents), and Agent-to-Policy mappings for your particular game. - Alternatively, use one of the two already existing setups (3DBall or - SoccerStrikersVsGoalie). - -4) Then run (two separate shells/machines): -$ python unity3d_server.py --env 3DBall -$ python unity3d_client.py --inference-mode=local --game [path to game binary] -""" - -import argparse - -from ray.rllib.env.policy_client import PolicyClient -from ray.rllib.env.wrappers.unity3d_env import Unity3DEnv -from ray._common.network_utils import build_address - -SERVER_ADDRESS = "localhost" -SERVER_PORT = 9900 - -parser = argparse.ArgumentParser() -parser.add_argument( - "--game", - type=str, - default=None, - help="The game executable to run as RL env. If not provided, uses local " - "Unity3D editor instance.", -) -parser.add_argument( - "--horizon", - type=int, - default=200, - help="The max. number of `step()`s for any episode (per agent) before " - "it'll be reset again automatically.", -) -parser.add_argument( - "--server", - type=str, - default=SERVER_ADDRESS, - help="The Policy server's address to connect to from this client.", -) -parser.add_argument( - "--port", type=int, default=SERVER_PORT, help="The port to use (on --server)." -) -parser.add_argument( - "--no-train", - action="store_true", - help="Whether to disable training (on the server side).", -) -parser.add_argument( - "--inference-mode", - type=str, - default="local", - choices=["local", "remote"], - help="Whether to compute actions `local`ly or `remote`ly. Note that " - "`local` is much faster b/c observations/actions do not have to be " - "sent via the network.", -) -parser.add_argument( - "--update-interval-local-mode", - type=float, - default=10.0, - help="For `inference-mode=local`, every how many seconds do we update " - "learnt policy weights from the server?", -) -parser.add_argument( - "--stop-reward", - type=float, - default=9999, - help="Stop once the specified reward is reached.", -) - -if __name__ == "__main__": - args = parser.parse_args() - - # Start the client for sending environment information (e.g. observations, - # actions) to a policy server (listening on port 9900). - client = PolicyClient( - f"http://{build_address(args.server, args.port)}", - inference_mode=args.inference_mode, - update_interval=args.update_interval_local_mode, - ) - - # Start and reset the actual Unity3DEnv (either already running Unity3D - # editor or a binary (game) to be started automatically). - env = Unity3DEnv(file_name=args.game, episode_horizon=args.horizon) - obs, info = env.reset() - eid = client.start_episode(training_enabled=not args.no_train) - - # Keep track of the total reward per episode. - total_rewards_this_episode = 0.0 - - # Loop infinitely through the env. - while True: - # Get actions from the Policy server given our current obs. - actions = client.get_action(eid, obs) - # Apply actions to our env. - obs, rewards, terminateds, truncateds, infos = env.step(actions) - total_rewards_this_episode += sum(rewards.values()) - # Log rewards and single-agent terminateds. - client.log_returns(eid, rewards, infos, multiagent_done_dict=terminateds) - # Check whether all agents are done and end the episode, if necessary. - if terminateds["__all__"] or truncateds["__all__"]: - print("Episode done: Reward={}".format(total_rewards_this_episode)) - if total_rewards_this_episode >= args.stop_reward: - quit(0) - # End the episode and reset Unity Env. - total_rewards_this_episode = 0.0 - client.end_episode(eid, obs) - obs, info = env.reset() - # Start a new episode. - eid = client.start_episode(training_enabled=not args.no_train) diff --git a/rllib/examples/envs/external_envs/unity3d_dummy_client.py b/rllib/examples/envs/external_envs/unity3d_dummy_client.py deleted file mode 100644 index 58b723c61349..000000000000 --- a/rllib/examples/envs/external_envs/unity3d_dummy_client.py +++ /dev/null @@ -1,160 +0,0 @@ -# TODO (sven): Move this example script into the new API stack. - -""" -Dummy in-place replacement for the unity3d_client.py script -in case you don't have an actual Unity3D engine installed or just want -to test client/server connectivity with the unity3d_server.py script. - -This client script simply uses RLlib's RandomMultiAgentEnv to mimic -one of the ML Agents (Unity3D) example games (e.g. "3DBall"). - -To run this script on possibly different machines -against a central Policy server: - -1) Run (two separate shells/machines): -$ python unity3d_server.py --env 3DBall -$ python unity3d_dummy_client.py --env 3DBall --inference-mode=local -""" - -import argparse - -from ray.rllib.env.policy_client import PolicyClient -from ray.rllib.env.wrappers.unity3d_env import Unity3DEnv -from ray.rllib.examples.envs.classes.random_env import RandomMultiAgentEnv -from ray._common.network_utils import build_address - -SERVER_ADDRESS = "localhost" -SERVER_PORT = 9900 - -parser = argparse.ArgumentParser() -parser.add_argument( - "--env", - type=str, - default="3DBall", - choices=[ - "3DBall", - "3DBallHard", - "FoodCollector", - "GridFoodCollector", - "Pyramids", - "Sorter", - "Tennis", - "VisualHallway", - "Walker", - ], - help="The name of the Env to mimic. Only those examples supported so " - "far for which all agents have the same " - "observation- and action spaces (feel free to add more to this script!)", -) -parser.add_argument( - "--horizon", - type=int, - default=200, - help="The max. number of `step()`s for any episode (per agent) before " - "it'll be reset again automatically.", -) -parser.add_argument( - "--server", - type=str, - default=SERVER_ADDRESS, - help="The Policy server's address to connect to from this client.", -) -parser.add_argument( - "--port", type=int, default=SERVER_PORT, help="The port to use (on --server)." -) -parser.add_argument( - "--no-train", - action="store_true", - help="Whether to disable training (on the server side).", -) -parser.add_argument( - "--inference-mode", - type=str, - default="local", - choices=["local", "remote"], - help="Whether to compute actions `local`ly or `remote`ly. Note that " - "`local` is much faster b/c observations/actions do not have to be " - "sent via the network.", -) -parser.add_argument( - "--update-interval-local-mode", - type=float, - default=10.0, - help="For `inference-mode=local`, every how many seconds do we update " - "learnt policy weights from the server?", -) -parser.add_argument( - "--num-episodes", - type=int, - default=10, - help="Stop once the specified number of episodes have been played.", -) - -if __name__ == "__main__": - args = parser.parse_args() - - # Start the client for sending environment information (e.g. observations, - # actions) to a policy server (listening on port 9900). - client = PolicyClient( - f"http://{build_address(args.server, args.port)}", - inference_mode=args.inference_mode, - update_interval=args.update_interval_local_mode, - ) - - # Get the multi-agent policies dict and agent->policy - # mapping-fn. - policies, policy_mapping_fn = Unity3DEnv.get_policy_configs_for_game(args.env) - - # Make sure all policies' obs- and action spaces are the same. - # If not, we won't be able to mimic the Unity3D env using RLlib's - # RandomMultiAgentEnv. - first_policy_spec = next(iter(policies.values())) - for pid, policy_spec in policies.items(): - assert policy_spec.observation_space == first_policy_spec.observation_space - assert policy_spec.action_space == first_policy_spec.action_space - - # Start and reset the actual Unity3DEnv (either already running Unity3D - # editor or a binary (game) to be started automatically). - env = RandomMultiAgentEnv( - { - # Same number of agents as the actual Unity3D game would have. - "num_agents": len(policies), - # Make sure we stick to the user given horizons using our - # RandomMultiAgentEnv options. - "max_episode_len": args.horizon, - "p_terminated": 0.0, - # Same obs- action spaces as the actual Unity3D game would have. - "observation_space": first_policy_spec.observation_space, - "action_space": first_policy_spec.action_space, - } - ) - obs, info = env.reset() - eid = client.start_episode(training_enabled=not args.no_train) - - # Keep track of the total reward per episode. - total_rewards_this_episode = 0.0 - - # Loop through the env until n episodes completed. - num_episodes = 0 - while True: - # Get actions from the Policy server given our current obs. - actions = client.get_action(eid, obs) - # Apply actions to our env. - obs, rewards, terminateds, truncateds, infos = env.step(actions) - total_rewards_this_episode += sum(rewards.values()) - # Log rewards and single-agent terminateds. - client.log_returns(eid, rewards, infos, multiagent_done_dict=terminateds) - # Check whether all agents are done and end the episode, if necessary. - if terminateds["__all__"] or truncateds["__all__"]: - print("Episode done: Reward={}".format(total_rewards_this_episode)) - - num_episodes += 1 - if num_episodes >= args.num_episodes: - quit(0) - - # End the episode and reset dummy Env. - total_rewards_this_episode = 0.0 - client.end_episode(eid, obs) - obs, info = env.reset() - # Start a new episode. - eid = client.start_episode(training_enabled=not args.no_train) diff --git a/rllib/examples/envs/external_envs/unity3d_server.py b/rllib/examples/envs/external_envs/unity3d_server.py deleted file mode 100755 index 4457102877e1..000000000000 --- a/rllib/examples/envs/external_envs/unity3d_server.py +++ /dev/null @@ -1,178 +0,0 @@ -# TODO (sven): Move this example script into the new API stack. - -""" -Example of running a Unity3D (MLAgents) Policy server that can learn -Policies via sampling inside many connected Unity game clients (possibly -running in the cloud on n nodes). -For a locally running Unity3D example, see: -`examples/unity3d_env_local.py` - -To run this script against one or more possibly cloud-based clients: -1) Install Unity3D and `pip install mlagents`. - -2) Compile a Unity3D example game with MLAgents support (e.g. 3DBall or any - other one that you created yourself) and place the compiled binary - somewhere, where your RLlib client script (see below) can access it. - -2.1) To find Unity3D MLAgent examples, first `pip install mlagents`, - then check out the `.../ml-agents/Project/Assets/ML-Agents/Examples/` - folder. - -3) Change this RLlib Policy server code so it knows the observation- and - action Spaces, the different Policies (called "behaviors" in Unity3D - MLAgents), and Agent-to-Policy mappings for your particular game. - Alternatively, use one of the two already existing setups (3DBall or - SoccerStrikersVsGoalie). - -4) Then run (two separate shells/machines): -$ python unity3d_server.py --env 3DBall -$ python unity3d_client.py --inference-mode=local --game [path to game binary] -""" - -import argparse -import gymnasium as gym -import os - -import ray -from ray.rllib.env.policy_server_input import PolicyServerInput -from ray.rllib.env.wrappers.unity3d_env import Unity3DEnv -from ray.tune.registry import get_trainable_cls - -SERVER_ADDRESS = "localhost" -SERVER_PORT = 9900 -CHECKPOINT_FILE = "last_checkpoint_{}.out" - -parser = argparse.ArgumentParser() -parser.add_argument( - "--run", - default="PPO", - choices=["DQN", "PPO"], - help="The RLlib-registered algorithm to use.", -) -parser.add_argument( - "--framework", - choices=["tf", "tf2", "torch"], - default="torch", - help="The DL framework specifier.", -) -parser.add_argument( - "--num-workers", - type=int, - default=2, - help="The number of workers to use. Each worker will create " - "its own listening socket for incoming experiences.", -) -parser.add_argument( - "--env", - type=str, - default="3DBall", - choices=[ - "3DBall", - "3DBallHard", - "FoodCollector", - "GridFoodCollector", - "Pyramids", - "SoccerStrikersVsGoalie", - "Sorter", - "Tennis", - "VisualHallway", - "Walker", - ], - help="The name of the Env to run in the Unity3D editor " - "(feel free to add more to this script!)", -) -parser.add_argument( - "--port", - type=int, - default=SERVER_PORT, - help="The Policy server's port to listen on for ExternalEnv client conections.", -) -parser.add_argument( - "--checkpoint-freq", - type=int, - default=10, - help="The frequency with which to create checkpoint files of the learnt " - "Policies.", -) -parser.add_argument( - "--no-restore", - action="store_true", - help="Whether to load the Policy weights from a previous checkpoint", -) - -if __name__ == "__main__": - args = parser.parse_args() - ray.init() - - # `InputReader` generator (returns None if no input reader is needed on - # the respective worker). - def _input(ioctx): - # We are remote worker or we are local worker with num_env_runners=0: - # Create a PolicyServerInput. - if ioctx.worker_index > 0 or ioctx.worker.num_workers == 0: - return PolicyServerInput( - ioctx, - SERVER_ADDRESS, - args.port + ioctx.worker_index - (1 if ioctx.worker_index > 0 else 0), - ) - # No InputReader (PolicyServerInput) needed. - else: - return None - - # Get the multi-agent policies dict and agent->policy - # mapping-fn. - policies, policy_mapping_fn = Unity3DEnv.get_policy_configs_for_game(args.env) - - # The entire config will be sent to connecting clients so they can - # build their own samplers (and also Policy objects iff - # `inference_mode=local` on clients' command line). - config = ( - get_trainable_cls(args.run) - .get_default_config() - # DL framework to use. - .framework(args.framework) - # Use n worker processes to listen on different ports. - .env_runners( - num_env_runners=args.num_workers, - rollout_fragment_length=20, - ) - .environment( - env=None, - # TODO: (sven) make these settings unnecessary and get the information - # about the env spaces from the client. - observation_space=gym.spaces.Box(float("-inf"), float("inf"), (8,)), - action_space=gym.spaces.Box(-1.0, 1.0, (2,)), - ) - .training(train_batch_size=256) - # Multi-agent setup for the given env. - .multi_agent(policies=policies, policy_mapping_fn=policy_mapping_fn) - # Use the `PolicyServerInput` to generate experiences. - .offline_data(input_=_input) - # Disable OPE, since the rollouts are coming from online clients. - .evaluation(off_policy_estimation_methods={}) - ) - - # Create the Algorithm used for Policy serving. - algo = config.build() - - # Attempt to restore from checkpoint if possible. - checkpoint_path = CHECKPOINT_FILE.format(args.env) - if not args.no_restore and os.path.exists(checkpoint_path): - checkpoint_path = open(checkpoint_path).read() - print("Restoring from checkpoint path", checkpoint_path) - algo.restore(checkpoint_path) - - # Serving and training loop. - count = 0 - while True: - # Calls to train() will block on the configured `input` in the Algorithm - # config above (PolicyServerInput). - print(algo.train()) - if count % args.checkpoint_freq == 0: - print("Saving learning progress to checkpoint file.") - checkpoint = algo.save().checkpoint - # Write the latest checkpoint location to CHECKPOINT_FILE, - # so we can pick up from the latest one after a server re-start. - with open(checkpoint_path, "w") as f: - f.write(checkpoint.path) - count += 1 diff --git a/rllib/utils/checkpoints.py b/rllib/utils/checkpoints.py index 43c522cfc565..99cd5ef3846f 100644 --- a/rllib/utils/checkpoints.py +++ b/rllib/utils/checkpoints.py @@ -1036,7 +1036,8 @@ def try_import_msgpack(error: bool = False): error: Whether to raise an error if msgpack/msgpack_numpy cannot be imported. Returns: - The `msgpack` module. + The `msgpack` module, with the msgpack_numpy module already patched in. This + means you can already encde and decode numpy arrays with the returned module. Raises: ImportError: If error=True and msgpack/msgpack_numpy is not installed. From f973fe59032e20a80a7ed5cbc75b87eee37a2b45 Mon Sep 17 00:00:00 2001 From: goutamvenkat-anyscale Date: Mon, 11 Aug 2025 14:08:54 -0700 Subject: [PATCH 027/634] [Data] Deprecate `with_columns` API in favor of `with_column` (#55475) Applies changes again reverted by ray-project/ray#55451 --- python/ray/data/dataset.py | 29 +++++++------ python/ray/data/expressions.py | 8 ++-- python/ray/data/tests/test_map.py | 71 +++++++++++++++---------------- 3 files changed, 53 insertions(+), 55 deletions(-) diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index c2a3f0184854..8766434c425a 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -783,31 +783,31 @@ def _map_batches_without_batch_size_validation( return Dataset(plan, logical_plan) @PublicAPI(api_group=EXPRESSION_API_GROUP, stability="alpha") - def with_columns(self, exprs: Dict[str, Expr]) -> "Dataset": + def with_column(self, column_name: str, expr: Expr, **ray_remote_args) -> "Dataset": """ - Add new columns to the dataset. + Add a new column to the dataset via an expression. Examples: >>> import ray >>> from ray.data.expressions import col >>> ds = ray.data.range(100) - >>> ds.with_columns({"new_id": col("id") * 2, "new_id_2": col("id") * 3}).schema() - Column Type - ------ ---- - id int64 - new_id int64 - new_id_2 int64 + >>> ds.with_column("id_2", (col("id") * 2)).schema() + Column Type + ------ ---- + id int64 + id_2 int64 Args: - exprs: A dictionary mapping column names to expressions that define the new column values. + column_name: The name of the new column. + expr: An expression that defines the new column values. + **ray_remote_args: Additional resource requirements to request from + Ray (e.g., num_gpus=1 to request GPUs for the map tasks). See + :func:`ray.remote` for details. Returns: - A new dataset with the added columns evaluated via expressions. + A new dataset with the added column evaluated via the expression. """ - if not exprs: - raise ValueError("at least one expression is required") - from ray.data._internal.logical.operators.map_operator import Project plan = self._plan.copy() @@ -815,7 +815,8 @@ def with_columns(self, exprs: Dict[str, Expr]) -> "Dataset": self._logical_plan.dag, cols=None, cols_rename=None, - exprs=exprs, + exprs={column_name: expr}, + ray_remote_args=ray_remote_args, ) logical_plan = LogicalPlan(project_op, self.context) return Dataset(plan, logical_plan) diff --git a/python/ray/data/expressions.py b/python/ray/data/expressions.py index cf59aa30b5e0..3ba8f48356da 100644 --- a/python/ray/data/expressions.py +++ b/python/ray/data/expressions.py @@ -259,10 +259,10 @@ def col(name: str) -> ColumnExpr: >>> # Reference columns in an expression >>> expr = col("price") * col("quantity") >>> - >>> # Use with Dataset.with_columns() + >>> # Use with Dataset.with_column() >>> import ray >>> ds = ray.data.from_items([{"price": 10, "quantity": 2}]) - >>> ds = ds.with_columns({"total": col("price") * col("quantity")}) + >>> ds = ds.with_column("total", col("price") * col("quantity")) """ return ColumnExpr(name) @@ -293,10 +293,10 @@ def lit(value: Any) -> LiteralExpr: >>> # Use in expressions >>> expr = col("age") + lit(1) # Add 1 to age column >>> - >>> # Use with Dataset.with_columns() + >>> # Use with Dataset.with_column() >>> import ray >>> ds = ray.data.from_items([{"age": 25}, {"age": 30}]) - >>> ds = ds.with_columns({"age_plus_one": col("age") + lit(1)}) + >>> ds = ds.with_column("age_plus_one", col("age") + lit(1)) """ return LiteralExpr(value) diff --git a/python/ray/data/tests/test_map.py b/python/ray/data/tests/test_map.py index 6c3f9c6d4a37..0bfb35f68f17 100644 --- a/python/ray/data/tests/test_map.py +++ b/python/ray/data/tests/test_map.py @@ -2286,52 +2286,53 @@ def func(x, y): @pytest.mark.skipif( get_pyarrow_version() < parse_version("20.0.0"), - reason="with_columns requires PyArrow >= 20.0.0", + reason="with_column requires PyArrow >= 20.0.0", ) @pytest.mark.parametrize( - "exprs, expected_value", + "column_name, expr, expected_value", [ # Arithmetic operations - ({"result": col("id") + 1}, 1), # 0 + 1 = 1 - ({"result": col("id") + 5}, 5), # 0 + 5 = 5 - ({"result": col("id") - 1}, -1), # 0 - 1 = -1 - ({"result": col("id") * 2}, 0), # 0 * 2 = 0 - ({"result": col("id") * 3}, 0), # 0 * 3 = 0 - ({"result": col("id") / 2}, 0.0), # 0 / 2 = 0.0 + ("result", col("id") + 1, 1), # 0 + 1 = 1 + ("result", col("id") + 5, 5), # 0 + 5 = 5 + ("result", col("id") - 1, -1), # 0 - 1 = -1 + ("result", col("id") * 2, 0), # 0 * 2 = 0 + ("result", col("id") * 3, 0), # 0 * 3 = 0 + ("result", col("id") / 2, 0.0), # 0 / 2 = 0.0 # More complex arithmetic - ({"result": (col("id") + 1) * 2}, 2), # (0 + 1) * 2 = 2 - ({"result": (col("id") * 2) + 3}, 3), # 0 * 2 + 3 = 3 + ("result", (col("id") + 1) * 2, 2), # (0 + 1) * 2 = 2 + ("result", (col("id") * 2) + 3, 3), # 0 * 2 + 3 = 3 # Comparison operations - ({"result": col("id") > 0}, False), # 0 > 0 = False - ({"result": col("id") >= 0}, True), # 0 >= 0 = True - ({"result": col("id") < 1}, True), # 0 < 1 = True - ({"result": col("id") <= 0}, True), # 0 <= 0 = True - ({"result": col("id") == 0}, True), # 0 == 0 = True + ("result", col("id") > 0, False), # 0 > 0 = False + ("result", col("id") >= 0, True), # 0 >= 0 = True + ("result", col("id") < 1, True), # 0 < 1 = True + ("result", col("id") <= 0, True), # 0 <= 0 = True + ("result", col("id") == 0, True), # 0 == 0 = True # Operations with literals - ({"result": col("id") + lit(10)}, 10), # 0 + 10 = 10 - ({"result": col("id") * lit(5)}, 0), # 0 * 5 = 0 - ({"result": lit(2) + col("id")}, 2), # 2 + 0 = 2 - ({"result": lit(10) / (col("id") + 1)}, 10.0), # 10 / (0 + 1) = 10.0 + ("result", col("id") + lit(10), 10), # 0 + 10 = 10 + ("result", col("id") * lit(5), 0), # 0 * 5 = 0 + ("result", lit(2) + col("id"), 2), # 2 + 0 = 2 + ("result", lit(10) / (col("id") + 1), 10.0), # 10 / (0 + 1) = 10.0 ], ) -def test_with_columns( +def test_with_column( ray_start_regular_shared, - exprs, + column_name, + expr, expected_value, target_max_block_size_infinite_or_default, ): - """Verify that `with_columns` works with various operations.""" - ds = ray.data.range(5).with_columns(exprs) + """Verify that `with_column` works with various operations.""" + ds = ray.data.range(5).with_column(column_name, expr) result = ds.take(1)[0] assert result["id"] == 0 - assert result["result"] == expected_value + assert result[column_name] == expected_value @pytest.mark.skipif( get_pyarrow_version() < parse_version("20.0.0"), - reason="with_columns requires PyArrow >= 20.0.0", + reason="with_column requires PyArrow >= 20.0.0", ) -def test_with_columns_nonexistent_column( +def test_with_column_nonexistent_column( ray_start_regular_shared, target_max_block_size_infinite_or_default ): """Verify that referencing a non-existent column with col() raises an exception.""" @@ -2340,26 +2341,22 @@ def test_with_columns_nonexistent_column( # Try to reference a non-existent column - this should raise an exception with pytest.raises(UserCodeException): - ds.with_columns({"result": col("nonexistent_column") + 1}).materialize() + ds.with_column("result", col("nonexistent_column") + 1).materialize() @pytest.mark.skipif( get_pyarrow_version() < parse_version("20.0.0"), - reason="with_columns requires PyArrow >= 20.0.0", + reason="with_column requires PyArrow >= 20.0.0", ) -def test_with_columns_multiple_expressions( +def test_with_column_multiple_expressions( ray_start_regular_shared, target_max_block_size_infinite_or_default ): - """Verify that `with_columns` correctly handles multiple expressions at once.""" + """Verify that `with_column` correctly handles multiple expressions at once.""" ds = ray.data.range(5) - exprs = { - "plus_one": col("id") + 1, - "times_two": col("id") * 2, - "ten_minus_id": 10 - col("id"), - } - - ds = ds.with_columns(exprs) + ds = ds.with_column("plus_one", col("id") + 1) + ds = ds.with_column("times_two", col("id") * 2) + ds = ds.with_column("ten_minus_id", 10 - col("id")) first_row = ds.take(1)[0] assert first_row["id"] == 0 From 42ce235932c58a611bb2ea77cde485886b1a170d Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 11 Aug 2025 16:17:38 -0500 Subject: [PATCH 028/634] Disable gemini code review summary (#55488) Copied [default config](https://developers.google.com/gemini-code-assist/docs/customize-gemini-behavior-github#default-configuration), but disabled summary (not useful). --------- Signed-off-by: Edward Oakes --- .gemini/config.yaml | 10 ++++++++++ .github/CODEOWNERS | 2 ++ 2 files changed, 12 insertions(+) create mode 100644 .gemini/config.yaml diff --git a/.gemini/config.yaml b/.gemini/config.yaml new file mode 100644 index 000000000000..9add3a6c8058 --- /dev/null +++ b/.gemini/config.yaml @@ -0,0 +1,10 @@ +have_fun: false +code_review: + disable: false + comment_severity_threshold: MEDIUM + max_review_comments: -1 + pull_request_opened: + help: false + summary: false + code_review: true +ignore_patterns: [] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f692cfbe2ef4..723c50deedda 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -108,3 +108,5 @@ /.github/ISSUE_TEMPLATE/ @aslonnie /.github/workflows/ @ray-project/ray-ci + +/.gemini/ @edoakes @aslonnie From 2a558d38cf958d05bae0b4d9342c7d1a894cdf88 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Mon, 11 Aug 2025 14:52:09 -0700 Subject: [PATCH 029/634] [ci] raydepsets: first step of build arg substitution for depsets (#55399) Branched from : https://github.com/ray-project/ray/pull/55390 performing substitutions on depsets with build arg set matrices **Top level build arg sets** build_arg_sets: - name: py311_cpu build_args: CUDA_VERSION: cpu PYTHON_VERSION: py311 **build arg set matrix defined on depset** - name: build_args_test_depset_${PYTHON_VERSION} operation: compile requirements: - requirements_test.txt output: requirements_compiled_general_${PYTHON_VERSION}_${CUDA_VERSION}.txt build_arg_sets: - py311_cpu --------- Signed-off-by: elliot-barn Signed-off-by: Elliot Barnwell Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- ci/raydepsets/tests/test_cli.py | 35 +++++++-- .../tests/test_data/test.depsets.yaml | 7 ++ ci/raydepsets/tests/test_workspace.py | 30 +++++++- ci/raydepsets/workspace.py | 73 ++++++++++++++----- 4 files changed, 121 insertions(+), 24 deletions(-) diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 2aac8e132af3..5ee194f96e88 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -333,8 +333,9 @@ def test_build_graph(self): copy_data_to_tmpdir(tmpdir) manager = _create_test_manager(tmpdir) assert manager.build_graph is not None - assert len(manager.build_graph.nodes()) == 5 + assert len(manager.build_graph.nodes()) == 6 assert len(manager.build_graph.edges()) == 3 + # assert that the compile depsets are first assert manager.build_graph.nodes["general_depset"]["operation"] == "compile" assert ( manager.build_graph.nodes["subset_general_depset"]["operation"] @@ -344,11 +345,11 @@ def test_build_graph(self): manager.build_graph.nodes["expand_general_depset"]["operation"] == "expand" ) - sorted_nodes = list(topological_sort(manager.build_graph)) - assert sorted_nodes[0] == "ray_base_test_depset" - assert sorted_nodes[1] == "general_depset" - assert sorted_nodes[2] == "expanded_depset" + # assert that the root nodes are the compile depsets + assert "ray_base_test_depset" in sorted_nodes[:3] + assert "general_depset" in sorted_nodes[:3] + assert "build_args_test_depset_py311" in sorted_nodes[:3] def test_build_graph_bad_operation(self): with tempfile.TemporaryDirectory() as tmpdir: @@ -452,6 +453,30 @@ def test_expand_with_requirements(self): output_text_valid = output_file_valid.read_text() assert output_text == output_text_valid + def test_get_depset_with_build_arg_set(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = DependencySetManager( + config_path="test.depsets.yaml", + workspace_dir=tmpdir, + ) + depset = manager.get_depset("build_args_test_depset_py311") + assert depset.name == "build_args_test_depset_py311" + assert depset.build_arg_set.name == "py311_cpu" + assert depset.build_arg_set.build_args["PYTHON_VERSION"] == "py311" + assert depset.build_arg_set.build_args["CUDA_VERSION"] == "cpu" + + def test_get_depset_without_build_arg_set(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = DependencySetManager( + config_path="test.depsets.yaml", + workspace_dir=tmpdir, + ) + depset = manager.get_depset("ray_base_test_depset") + assert depset.name == "ray_base_test_depset" + assert depset.build_arg_set is None + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/ci/raydepsets/tests/test_data/test.depsets.yaml b/ci/raydepsets/tests/test_data/test.depsets.yaml index 5ebd0485746e..c5385b541887 100644 --- a/ci/raydepsets/tests/test_data/test.depsets.yaml +++ b/ci/raydepsets/tests/test_data/test.depsets.yaml @@ -21,6 +21,13 @@ depsets: requirements: - requirements_test.txt output: requirements_compiled_general.txt + - name: build_args_test_depset_${PYTHON_VERSION} + operation: compile + requirements: + - requirements_test.txt + output: requirements_compiled_general_${PYTHON_VERSION}_${CUDA_VERSION}.txt + build_arg_sets: + - py311_cpu - name: subset_general_depset operation: subset source_depset: general_depset diff --git a/ci/raydepsets/tests/test_workspace.py b/ci/raydepsets/tests/test_workspace.py index cc3ded45abef..2f2429e44a64 100644 --- a/ci/raydepsets/tests/test_workspace.py +++ b/ci/raydepsets/tests/test_workspace.py @@ -5,7 +5,7 @@ import pytest from ci.raydepsets.tests.utils import copy_data_to_tmpdir -from ci.raydepsets.workspace import Workspace +from ci.raydepsets.workspace import BuildArgSet, Workspace, _substitute_build_args def test_workspace_init(): @@ -31,5 +31,33 @@ def test_parse_build_arg_sets(): } +def test_from_dict_build_arg_set_matrix(): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + workspace = Workspace(dir=tmpdir) + config = workspace.load_config(path=Path(tmpdir) / "test.depsets.yaml") + config.build_arg_sets[0].build_args["PYTHON_VERSION"] = "py312" + config.build_arg_sets[0].build_args["CUDA_VERSION"] = "cu128" + + +def test_substitute_build_args(): + build_arg_set = BuildArgSet( + name="py311_cpu", + build_args={ + "PYTHON_VERSION": "py311", + "CUDA_VERSION": "cu128", + }, + ) + depset_dict = { + "name": "test_depset_${PYTHON_VERSION}_${CUDA_VERSION}", + "operation": "compile", + "requirements": ["requirements_test.txt"], + "output": "requirements_compiled_test_${PYTHON_VERSION}_${CUDA_VERSION}.txt", + } + substituted_depset = _substitute_build_args(depset_dict, build_arg_set) + assert substituted_depset["output"] == "requirements_compiled_test_py311_cu128.txt" + assert substituted_depset["name"] == "test_depset_py311_cu128" + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/ci/raydepsets/workspace.py b/ci/raydepsets/workspace.py index 647b2127e1ca..ccfced7b9052 100644 --- a/ci/raydepsets/workspace.py +++ b/ci/raydepsets/workspace.py @@ -1,6 +1,7 @@ import os from dataclasses import dataclass, field -from typing import List, Optional +from string import Template +from typing import Any, Dict, List, Optional import yaml @@ -8,7 +9,7 @@ @dataclass class BuildArgSet: name: str - build_args: List[str] + build_args: Dict[str, str] @dataclass @@ -22,6 +23,38 @@ class Depset: append_flags: List[str] source_depset: Optional[str] = None depsets: Optional[List[str]] = None + build_arg_set: Optional[BuildArgSet] = None + + +def _substitute_build_args(obj: Any, build_arg_set: BuildArgSet): + if isinstance(obj, str): + return Template(obj).substitute(build_arg_set.build_args) + elif isinstance(obj, dict): + return { + key: _substitute_build_args(value, build_arg_set) + for key, value in obj.items() + } + elif isinstance(obj, list): + return [_substitute_build_args(item, build_arg_set) for item in obj] + else: + return obj + + +def _dict_to_depset( + depset: dict, build_arg_set: Optional[BuildArgSet] = None +) -> Depset: + return Depset( + name=depset.get("name"), + requirements=depset.get("requirements", []), + constraints=depset.get("constraints", []), + operation=depset.get("operation", None), + output=depset.get("output"), + source_depset=depset.get("source_depset"), + depsets=depset.get("depsets", []), + build_arg_set=build_arg_set, + override_flags=depset.get("override_flags", []), + append_flags=depset.get("append_flags", []), + ) @dataclass @@ -31,23 +64,27 @@ class Config: @staticmethod def from_dict(data: dict) -> "Config": - raw_depsets = data.get("depsets", []) - depsets = [ - Depset( - name=values.get("name"), - requirements=values.get("requirements", []), - constraints=values.get("constraints", []), - operation=values.get("operation", "compile"), - output=values.get("output"), - source_depset=values.get("source_depset"), - override_flags=values.get("override_flags", []), - append_flags=values.get("append_flags", []), - depsets=values.get("depsets", []), - ) - for values in raw_depsets - ] - build_arg_sets = Config.parse_build_arg_sets(data.get("build_arg_sets", [])) + raw_depsets = data.get("depsets", []) + depsets = [] + for depset in raw_depsets: + build_arg_set_matrix = depset.get("build_arg_sets", []) + if build_arg_set_matrix: + for build_arg_set_name in build_arg_set_matrix: + build_arg_set = next( + ( + build_arg_set + for build_arg_set in build_arg_sets + if build_arg_set.name == build_arg_set_name + ), + None, + ) + if build_arg_set is None: + raise KeyError(f"Build arg set {build_arg_set_name} not found") + depset_yaml = _substitute_build_args(depset, build_arg_set) + depsets.append(_dict_to_depset(depset_yaml, build_arg_set)) + else: + depsets.append(_dict_to_depset(depset=depset)) return Config(depsets=depsets, build_arg_sets=build_arg_sets) @staticmethod From 54c5d132969e7cd8abb2fd7168bcdf86b70c6dd5 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Mon, 11 Aug 2025 14:55:15 -0700 Subject: [PATCH 030/634] [release] Move logic to check whether to build custom BYOD image (#55470) The logic should be outside of this function. This PR: - Moves the logic to outside of the func for every place that calls it. - Turns the arg to be 3 separate ones so they can be called without a Test object (in order to do things like https://github.com/ray-project/ray/pull/55398 where Test object doesn't exist) - Change the logic in another spot that calls `build_anyscale_custom_byod_image` that was missed last time in https://github.com/ray-project/ray/pull/55397 which caused release test pipeline to fail Signed-off-by: kevin --- release/ray_release/byod/build.py | 20 +++++++++---------- release/ray_release/scripts/build_pipeline.py | 7 ++++++- release/ray_release/scripts/ray_bisect.py | 7 ++++++- release/ray_release/tests/test_byod_build.py | 6 +++++- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/release/ray_release/byod/build.py b/release/ray_release/byod/build.py index 14e9bb43311a..62ae1d62f3b3 100644 --- a/release/ray_release/byod/build.py +++ b/release/ray_release/byod/build.py @@ -30,13 +30,11 @@ REQUIREMENTS_ML_BYOD = "requirements_ml_byod" -def build_anyscale_custom_byod_image(test: Test) -> None: - if not test.require_custom_byod_image(): - logger.info(f"Test {test.get_name()} does not require a custom byod image") - return - byod_image = test.get_anyscale_byod_image() - if _image_exist(byod_image): - logger.info(f"Image {byod_image} already exists") +def build_anyscale_custom_byod_image( + image: str, base_image: str, post_build_script: str +) -> None: + if _image_exist(image): + logger.info(f"Image {image} already exists") return env = os.environ.copy() @@ -47,11 +45,11 @@ def build_anyscale_custom_byod_image(test: Test) -> None: "build", "--progress=plain", "--build-arg", - f"BASE_IMAGE={test.get_anyscale_base_byod_image()}", + f"BASE_IMAGE={base_image}", "--build-arg", - f"POST_BUILD_SCRIPT={test.get_byod_post_build_script()}", + f"POST_BUILD_SCRIPT={post_build_script}", "-t", - byod_image, + image, "-f", os.path.join(RELEASE_BYOD_DIR, "byod.custom.Dockerfile"), RELEASE_BYOD_DIR, @@ -59,7 +57,7 @@ def build_anyscale_custom_byod_image(test: Test) -> None: stdout=sys.stderr, env=env, ) - _validate_and_push(byod_image) + _validate_and_push(image) def build_anyscale_base_byod_images(tests: List[Test]) -> None: diff --git a/release/ray_release/scripts/build_pipeline.py b/release/ray_release/scripts/build_pipeline.py index 29e448d8f4fd..9b6ff71adc1f 100644 --- a/release/ray_release/scripts/build_pipeline.py +++ b/release/ray_release/scripts/build_pipeline.py @@ -127,7 +127,12 @@ def main( build_anyscale_base_byod_images(tests) logger.info("Build anyscale custom BYOD images") for test in tests: - build_anyscale_custom_byod_image(test) + if test.require_custom_byod_image(): + build_anyscale_custom_byod_image( + test.get_anyscale_byod_image(), + test.get_anyscale_base_byod_image(), + test.get_byod_post_build_script(), + ) grouped_tests = group_tests(filtered_tests) group_str = "" diff --git a/release/ray_release/scripts/ray_bisect.py b/release/ray_release/scripts/ray_bisect.py index 15ebd4c9d396..257519d620e4 100644 --- a/release/ray_release/scripts/ray_bisect.py +++ b/release/ray_release/scripts/ray_bisect.py @@ -178,7 +178,12 @@ def _trigger_test_run( ) -> None: os.environ["COMMIT_TO_TEST"] = commit build_anyscale_base_byod_images([test]) - build_anyscale_custom_byod_image(test) + if test.require_custom_byod_image(): + build_anyscale_custom_byod_image( + test.get_anyscale_byod_image(), + test.get_anyscale_base_byod_image(), + test.get_byod_post_build_script(), + ) for run in range(run_per_commit): step = get_step( copy.deepcopy(test), # avoid mutating the original test diff --git a/release/ray_release/tests/test_byod_build.py b/release/ray_release/tests/test_byod_build.py index 7da9d5449a69..920b2c826e36 100644 --- a/release/ray_release/tests/test_byod_build.py +++ b/release/ray_release/tests/test_byod_build.py @@ -68,7 +68,11 @@ def _mock_check_call( name="name", cluster={"byod": {"post_build_script": "foo.sh"}}, ) - build_anyscale_custom_byod_image(test) + build_anyscale_custom_byod_image( + test.get_anyscale_byod_image(), + test.get_anyscale_base_byod_image(), + test.get_byod_post_build_script(), + ) assert "docker build --build-arg BASE_IMAGE=029272617770.dkr.ecr.us-west-2." "amazonaws.com/anyscale/ray:abc123-py37 -t 029272617770.dkr.ecr.us-west-2." "amazonaws.com/anyscale/ray:abc123-py37-c3fc5fc6d84cea4d7ab885c6cdc966542e" From 359818a2563d3466471f71b37b697da96f98a6d9 Mon Sep 17 00:00:00 2001 From: Mengjin Yan Date: Mon, 11 Aug 2025 15:17:14 -0700 Subject: [PATCH 031/634] [Core][TaskEventFollowup/02] Fix Feedback for the Task Event Content & Configuration (#55300) This PR fixed feedback from internal teams about the content of the event & update configuration of the feature, to help better use the feature. To be specific: Event content changes: Add session name to the base events Add the following fields to the definition event: language, actor task name only for actor definition event, task type only for task definition event Remove the user error info from the execution events. This is because today the RayErrorInfo already covers the user error information when the RayErrorInfo is with type TASK_EXECUTION_EXCEPTION Remove the actor task execution event and use task execution event for all task execution. This is because with the above changes, the fields in task execution event and the actor task execution event are exactly the same. Configuration: Improve the coordination between enabling the current export API and the new event sending mechanism. The logic updated to: if ray event to event aggregator is set, ignore the old export event env var. The old export event mechanism will only be enabled if the ray event to event aggregator config is set to false. Signed-off-by: Mengjin Yan Signed-off-by: myan Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python/ray/_private/node.py | 2 +- python/ray/_private/parameter.py | 2 +- python/ray/_private/services.py | 4 +- python/ray/_private/worker.py | 2 +- python/ray/_private/workers/default_worker.py | 4 +- python/ray/autoscaler/_private/autoscaler.py | 2 +- python/ray/autoscaler/v2/autoscaler.py | 2 +- python/ray/dashboard/agent.py | 2 +- .../modules/aggregator/aggregator_agent.py | 1 + .../aggregator/tests/test_aggregator_agent.py | 98 ++++++++----------- .../modules/event/tests/test_export_task.py | 1 + python/ray/tests/test_metrics_agent.py | 3 +- src/ray/core_worker/core_worker.cc | 6 +- src/ray/core_worker/core_worker_options.h | 2 +- src/ray/core_worker/core_worker_process.cc | 3 +- src/ray/core_worker/task_event_buffer.cc | 50 +++++----- src/ray/core_worker/task_event_buffer.h | 33 +++++-- .../test/scheduling_queue_test.cc | 22 +++++ .../task_execution/test/task_receiver_test.cc | 12 +++ src/ray/core_worker/test/core_worker_test.cc | 3 +- .../task_event_buffer_export_event_test.cc | 7 +- .../test/task_event_buffer_test.cc | 5 +- src/ray/core_worker/test/task_manager_test.cc | 12 +++ .../gcs_server/gcs_autoscaler_state_manager.h | 2 +- src/ray/gcs/gcs_server/gcs_server_main.cc | 4 +- src/ray/protobuf/BUILD.bazel | 15 --- src/ray/protobuf/autoscaler.proto | 2 +- .../events_actor_task_definition_event.proto | 18 ++-- .../events_actor_task_execution_event.proto | 43 -------- src/ray/protobuf/events_base_event.proto | 13 ++- .../events_task_definition_event.proto | 19 ++-- .../events_task_execution_event.proto | 10 +- src/ray/raylet/main.cc | 2 +- src/ray/rpc/event_aggregator_client.h | 1 - 34 files changed, 203 insertions(+), 204 deletions(-) delete mode 100644 src/ray/protobuf/events_actor_task_execution_event.proto diff --git a/python/ray/_private/node.py b/python/ray/_private/node.py index b11a4664400a..3465f995529b 100644 --- a/python/ray/_private/node.py +++ b/python/ray/_private/node.py @@ -553,7 +553,7 @@ def node_id(self): @property def session_name(self): - """Get the session name (cluster ID).""" + """Get the current Ray session name.""" return self._session_name @property diff --git a/python/ray/_private/parameter.py b/python/ray/_private/parameter.py index 7f415c391b0d..eb78bd86f510 100644 --- a/python/ray/_private/parameter.py +++ b/python/ray/_private/parameter.py @@ -114,7 +114,7 @@ class RayParams: worker available externally to the node it is running on. This will bind on 0.0.0.0 instead of localhost. env_vars: Override environment variables for the raylet. - session_name: The name of the session of the ray cluster. + session_name: The current Ray session name. webui: The url of the UI. cluster_id: The cluster ID in hex string. resource_isolation_config: settings for cgroupv2 based isolation of ray diff --git a/python/ray/_private/services.py b/python/ray/_private/services.py index 6785a78c571f..3abe99bf23b4 100644 --- a/python/ray/_private/services.py +++ b/python/ray/_private/services.py @@ -1469,7 +1469,7 @@ def start_gcs_server( If None, stdout is not redirected. stderr_filepath: The file path to dump gcs server stderr. If None, stderr is not redirected. - session_name: The session name (cluster id) of this cluster. + session_name: The current Ray session name. redis_username: The username of the Redis server. redis_password: The password of the Redis server. config: Optional configuration that will @@ -1606,7 +1606,7 @@ def start_raylet( fallback_directory: A directory where the Object store fallback files will be created. object_store_memory: The amount of memory (in bytes) to start the object store with. - session_name: The session name (cluster id) of this cluster. + session_name: The current Ray session name. resource_isolation_config: Resource isolation configuration for reserving memory and cpu resources for ray system processes through cgroupv2 is_head_node: whether this node is the head node. diff --git a/python/ray/_private/worker.py b/python/ray/_private/worker.py index 37879b15ba25..e3eeee3f84ad 100644 --- a/python/ray/_private/worker.py +++ b/python/ray/_private/worker.py @@ -2394,7 +2394,7 @@ def connect( Args: node (ray._private.node.Node): The node to connect. - session_name: The session name (cluster id) of this cluster. + session_name: The current Ray session name. mode: The mode of the worker. One of SCRIPT_MODE, WORKER_MODE, and LOCAL_MODE. log_to_driver: If true, then output from all of the worker processes on all nodes will be directed to the driver. diff --git a/python/ray/_private/workers/default_worker.py b/python/ray/_private/workers/default_worker.py index 03ea6e456e24..9859d9d15f7e 100644 --- a/python/ray/_private/workers/default_worker.py +++ b/python/ray/_private/workers/default_worker.py @@ -165,7 +165,9 @@ action="store_true", help="True if Ray debugger is made available externally.", ) -parser.add_argument("--session-name", required=False, help="The current session name") +parser.add_argument( + "--session-name", required=False, help="The current Ray session name" +) parser.add_argument( "--webui", required=False, diff --git a/python/ray/autoscaler/_private/autoscaler.py b/python/ray/autoscaler/_private/autoscaler.py index e352479b9308..8fbd8fc60960 100644 --- a/python/ray/autoscaler/_private/autoscaler.py +++ b/python/ray/autoscaler/_private/autoscaler.py @@ -207,7 +207,7 @@ def __init__( config_reader: Path to a Ray Autoscaler YAML, or a function to read and return the latest config. load_metrics: Provides metrics for the Ray cluster. - session_name: The session name of the cluster this autoscaler + session_name: The current Ray session name when this autoscaler is deployed. max_launch_batch: Max number of nodes to launch in one request. max_concurrent_launches: Max number of nodes that can be diff --git a/python/ray/autoscaler/v2/autoscaler.py b/python/ray/autoscaler/v2/autoscaler.py index c55646a46d8e..82de748a1f5f 100644 --- a/python/ray/autoscaler/v2/autoscaler.py +++ b/python/ray/autoscaler/v2/autoscaler.py @@ -55,7 +55,7 @@ def __init__( ) -> None: """ Args: - session_name: The name of the ray session. + session_name: The current Ray session name. config_reader: The config reader. gcs_client: The GCS client. event_logger: The event logger for emitting cluster events. diff --git a/python/ray/dashboard/agent.py b/python/ray/dashboard/agent.py index 9302f020e898..85ac9f05f503 100644 --- a/python/ray/dashboard/agent.py +++ b/python/ray/dashboard/agent.py @@ -371,7 +371,7 @@ async def wait_forever(): required=False, type=str, default=None, - help="The session name (cluster id) of this cluster.", + help="The current Ray session name.", ) parser.add_argument( "--stdout-filepath", diff --git a/python/ray/dashboard/modules/aggregator/aggregator_agent.py b/python/ray/dashboard/modules/aggregator/aggregator_agent.py index 97ba70bfabe8..4499116a6b2a 100644 --- a/python/ray/dashboard/modules/aggregator/aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/aggregator_agent.py @@ -202,6 +202,7 @@ def _receive_events(self, request): try: self._event_buffer.put_nowait(event) except queue.Full: + # Remove the oldest event to make room for the new event. self._event_buffer.get_nowait() self._event_buffer.put_nowait(event) with self._lock: diff --git a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py index 94d29ab0840c..3020813227fd 100644 --- a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py @@ -1,6 +1,5 @@ import sys import json -import time import base64 import pytest @@ -40,6 +39,16 @@ def httpserver_listen_address(): return ("127.0.0.1", _EVENT_AGGREGATOR_AGENT_TARGET_PORT) +@pytest.fixture +def fake_timestamp(): + """ + Returns a fake proto timestamp and the expected timestamp string in the event JSON. + """ + test_time = 1751302230130457542 + seconds, nanos = divmod(test_time, 10**9) + return Timestamp(seconds=seconds, nanos=nanos), "2025-06-30T16:50:30.130457542Z" + + _with_aggregator_port = pytest.mark.parametrize( "ray_start_cluster_head_with_env_vars", [ @@ -79,7 +88,7 @@ def get_event_aggregator_grpc_stub(webui_url, gcs_address, head_node_id): @_with_aggregator_port def test_aggregator_agent_receive_publish_events_normally( - ray_start_cluster_head_with_env_vars, httpserver + ray_start_cluster_head_with_env_vars, httpserver, fake_timestamp ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( @@ -88,10 +97,6 @@ def test_aggregator_agent_receive_publish_events_normally( httpserver.expect_request("/", method="POST").respond_with_data("", status=200) - test_time = 1751302230130457542 - seconds, nanos = divmod(test_time, 10**9) - timestamp = Timestamp(seconds=seconds, nanos=nanos) - request = AddEventsRequest( events_data=RayEventsData( events=[ @@ -99,7 +104,7 @@ def test_aggregator_agent_receive_publish_events_normally( event_id=b"1", source_type=RayEvent.SourceType.CORE_WORKER, event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, - timestamp=timestamp, + timestamp=fake_timestamp[0], severity=RayEvent.Severity.INFO, message="hello", ), @@ -110,8 +115,7 @@ def test_aggregator_agent_receive_publish_events_normally( ) ) - reply = stub.AddEvents(request) - assert reply is not None + stub.AddEvents(request) wait_for_condition(lambda: len(httpserver.log) == 1) req, _ = httpserver.log[0] @@ -123,7 +127,7 @@ def test_aggregator_agent_receive_publish_events_normally( assert req_json[0]["eventType"] == "TASK_DEFINITION_EVENT" assert req_json[0]["severity"] == "INFO" assert req_json[0]["message"] == "hello" - assert req_json[0]["timestamp"] == "2025-06-30T16:50:30.130457542Z" + assert req_json[0]["timestamp"] == fake_timestamp[1] @pytest.mark.parametrize( @@ -141,7 +145,7 @@ def test_aggregator_agent_receive_publish_events_normally( indirect=True, ) def test_aggregator_agent_receive_event_full( - ray_start_cluster_head_with_env_vars, httpserver + ray_start_cluster_head_with_env_vars, httpserver, fake_timestamp ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( @@ -150,10 +154,6 @@ def test_aggregator_agent_receive_event_full( httpserver.expect_request("/", method="POST").respond_with_data("", status=200) - test_time = 1751302230130457542 - seconds, nanos = divmod(test_time, 10**9) - timestamp = Timestamp(seconds=seconds, nanos=nanos) - request = AddEventsRequest( events_data=RayEventsData( events=[ @@ -161,7 +161,7 @@ def test_aggregator_agent_receive_event_full( event_id=b"2", source_type=RayEvent.SourceType.CORE_WORKER, event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, - timestamp=timestamp, + timestamp=fake_timestamp[0], severity=RayEvent.Severity.INFO, message="hello", ), @@ -169,7 +169,7 @@ def test_aggregator_agent_receive_event_full( event_id=b"3", source_type=RayEvent.SourceType.CORE_WORKER, event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, - timestamp=timestamp, + timestamp=fake_timestamp[0], severity=RayEvent.Severity.INFO, message="hello", ), @@ -180,8 +180,7 @@ def test_aggregator_agent_receive_event_full( ) ) - reply = stub.AddEvents(request) - assert reply is not None + stub.AddEvents(request) wait_for_condition(lambda: len(httpserver.log) == 1) req, _ = httpserver.log[0] @@ -193,7 +192,7 @@ def test_aggregator_agent_receive_event_full( @_with_aggregator_port def test_aggregator_agent_receive_multiple_events( - ray_start_cluster_head_with_env_vars, httpserver + ray_start_cluster_head_with_env_vars, httpserver, fake_timestamp ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( @@ -201,9 +200,6 @@ def test_aggregator_agent_receive_multiple_events( ) httpserver.expect_request("/", method="POST").respond_with_data("", status=200) - now = time.time_ns() - seconds, nanos = divmod(now, 10**9) - timestamp = Timestamp(seconds=seconds, nanos=nanos) request = AddEventsRequest( events_data=RayEventsData( events=[ @@ -211,7 +207,7 @@ def test_aggregator_agent_receive_multiple_events( event_id=b"4", source_type=RayEvent.SourceType.CORE_WORKER, event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, - timestamp=timestamp, + timestamp=fake_timestamp[0], severity=RayEvent.Severity.INFO, message="event1", ), @@ -219,7 +215,7 @@ def test_aggregator_agent_receive_multiple_events( event_id=b"5", source_type=RayEvent.SourceType.CORE_WORKER, event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, - timestamp=timestamp, + timestamp=fake_timestamp[0], severity=RayEvent.Severity.INFO, message="event2", ), @@ -229,8 +225,7 @@ def test_aggregator_agent_receive_multiple_events( ), ) ) - reply = stub.AddEvents(request) - assert reply is not None + stub.AddEvents(request) wait_for_condition(lambda: len(httpserver.log) == 1) req, _ = httpserver.log[0] req_json = json.loads(req.data) @@ -256,16 +251,13 @@ def test_aggregator_agent_receive_multiple_events( indirect=True, ) def test_aggregator_agent_receive_multiple_events_failures( - ray_start_cluster_head_with_env_vars, httpserver + ray_start_cluster_head_with_env_vars, httpserver, fake_timestamp ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( cluster.webui_url, cluster.gcs_address, cluster.head_node.node_id ) httpserver.expect_request("/", method="POST").respond_with_data("", status=200) - now = time.time_ns() - seconds, nanos = divmod(now, 10**9) - timestamp = Timestamp(seconds=seconds, nanos=nanos) request = AddEventsRequest( events_data=RayEventsData( events=[ @@ -273,7 +265,7 @@ def test_aggregator_agent_receive_multiple_events_failures( event_id=b"1", source_type=RayEvent.SourceType.CORE_WORKER, event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, - timestamp=timestamp, + timestamp=fake_timestamp[0], severity=RayEvent.Severity.INFO, message="event1", ), @@ -281,7 +273,7 @@ def test_aggregator_agent_receive_multiple_events_failures( event_id=b"2", source_type=RayEvent.SourceType.CORE_WORKER, event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, - timestamp=timestamp, + timestamp=fake_timestamp[0], severity=RayEvent.Severity.INFO, message="event2", ), @@ -289,15 +281,14 @@ def test_aggregator_agent_receive_multiple_events_failures( event_id=b"3", source_type=RayEvent.SourceType.CORE_WORKER, event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, - timestamp=timestamp, + timestamp=fake_timestamp[0], severity=RayEvent.Severity.INFO, message="event3", ), ], ) ) - reply = stub.AddEvents(request) - assert reply is not None + stub.AddEvents(request) wait_for_condition(lambda: len(httpserver.log) == 1) req, _ = httpserver.log[0] req_json = json.loads(req.data) @@ -322,13 +313,12 @@ def test_aggregator_agent_receive_empty_events( ), ) ) - reply = stub.AddEvents(request) - assert reply is not None + stub.AddEvents(request) @_with_aggregator_port def test_aggregator_agent_profile_events_not_exposed( - ray_start_cluster_head_with_env_vars, httpserver + ray_start_cluster_head_with_env_vars, httpserver, fake_timestamp ): """Test that profile events are not sent when not in exposable event types.""" cluster = ray_start_cluster_head_with_env_vars @@ -337,20 +327,15 @@ def test_aggregator_agent_profile_events_not_exposed( ) httpserver.expect_request("/", method="POST").respond_with_data("", status=200) - - now = time.time_ns() - seconds, nanos = divmod(now, 10**9) - timestamp = Timestamp(seconds=seconds, nanos=nanos) - request = AddEventsRequest( events_data=RayEventsData( events=[ - _create_profile_event_request(), + _create_profile_event_request(fake_timestamp[0]), RayEvent( event_id=b"1", source_type=RayEvent.SourceType.CORE_WORKER, event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, - timestamp=timestamp, + timestamp=fake_timestamp[0], severity=RayEvent.Severity.INFO, message="event1", ), @@ -361,8 +346,7 @@ def test_aggregator_agent_profile_events_not_exposed( ) ) - reply = stub.AddEvents(request) - assert reply is not None + stub.AddEvents(request) # Wait for exactly one event to be received (the TASK_DEFINITION_EVENT) wait_for_condition(lambda: len(httpserver.log) == 1) @@ -391,7 +375,7 @@ def test_aggregator_agent_profile_events_not_exposed( indirect=True, ) def test_aggregator_agent_receive_profile_events( - ray_start_cluster_head_with_env_vars, httpserver + ray_start_cluster_head_with_env_vars, httpserver, fake_timestamp ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( @@ -402,29 +386,25 @@ def test_aggregator_agent_receive_profile_events( request = AddEventsRequest( events_data=RayEventsData( - events=[_create_profile_event_request()], + events=[_create_profile_event_request(fake_timestamp[0])], task_events_metadata=TaskEventsMetadata( dropped_task_attempts=[], ), ) ) - reply = stub.AddEvents(request) - assert reply is not None + stub.AddEvents(request) wait_for_condition(lambda: len(httpserver.log) == 1) req, _ = httpserver.log[0] req_json = json.loads(req.data) - _verify_profile_event_json(req_json) + _verify_profile_event_json(req_json, fake_timestamp[1]) -def _create_profile_event_request(): +def _create_profile_event_request(timestamp): """Helper function to create a profile event request.""" - test_time = 1751302230130457542 - seconds, nanos = (test_time // 10**9, test_time % 10**9) - timestamp = Timestamp(seconds=seconds, nanos=nanos) return RayEvent( event_id=b"1", @@ -454,7 +434,7 @@ def _create_profile_event_request(): ) -def _verify_profile_event_json(req_json): +def _verify_profile_event_json(req_json, expected_timestamp): """Helper function to verify profile event JSON structure.""" assert len(req_json) == 1 assert req_json[0]["eventId"] == base64.b64encode(b"1").decode() @@ -462,7 +442,7 @@ def _verify_profile_event_json(req_json): assert req_json[0]["eventType"] == "TASK_PROFILE_EVENT" assert req_json[0]["severity"] == "INFO" assert req_json[0]["message"] == "profile event test" - assert req_json[0]["timestamp"] == "2025-06-30T16:50:30.130457542Z" + assert req_json[0]["timestamp"] == expected_timestamp # Verify task profile event specific fields assert "taskProfileEvents" in req_json[0] diff --git a/python/ray/dashboard/modules/event/tests/test_export_task.py b/python/ray/dashboard/modules/event/tests/test_export_task.py index 50b47601090c..6698ffa9703f 100644 --- a/python/ray/dashboard/modules/event/tests/test_export_task.py +++ b/python/ray/dashboard/modules/event/tests/test_export_task.py @@ -10,6 +10,7 @@ from ray.dashboard.tests.conftest import * # noqa os.environ["RAY_enable_export_api_write"] = "1" +os.environ["RAY_enable_core_worker_ray_event_to_aggregator"] = "0" @pytest.mark.asyncio diff --git a/python/ray/tests/test_metrics_agent.py b/python/ray/tests/test_metrics_agent.py index ad7d717619d0..b7494f52dc43 100644 --- a/python/ray/tests/test_metrics_agent.py +++ b/python/ray/tests/test_metrics_agent.py @@ -560,8 +560,7 @@ def test_case_value_correct(): ) ) - reply = stub.AddEvents(request) - assert reply is not None + stub.AddEvents(request) wait_for_condition(lambda: len(httpserver.log) == 1) wait_for_condition(test_case_value_correct, timeout=30, retry_interval_ms=1000) diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 634c82975e3b..d9b8fcbd292d 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -396,7 +396,8 @@ CoreWorker::CoreWorker( /*attempt_number=*/0, rpc::TaskStatus::RUNNING, /*timestamp=*/absl::GetCurrentTimeNanos(), - /*is_actor_task=*/false, + /*is_actor_task_event=*/false, + options_.session_name, std::make_shared(std::move(spec))); task_event_buffer_->AddTaskEvent(std::move(task_event)); } @@ -569,7 +570,8 @@ void CoreWorker::Disconnect( /*attempt_number=*/0, rpc::TaskStatus::FINISHED, /*timestamp=*/absl::GetCurrentTimeNanos(), - /*is_actor_task_event=*/worker_context_->GetCurrentActorID().IsNil()); + /*is_actor_task_event=*/worker_context_->GetCurrentActorID().IsNil(), + options_.session_name); task_event_buffer_->AddTaskEvent(std::move(task_event)); } diff --git a/src/ray/core_worker/core_worker_options.h b/src/ray/core_worker/core_worker_options.h index 5c89fb56db49..72c108d759bb 100644 --- a/src/ray/core_worker/core_worker_options.h +++ b/src/ray/core_worker/core_worker_options.h @@ -203,7 +203,7 @@ struct CoreWorkerOptions { std::function(const ray::RayObject &object, const ObjectID &object_id)> object_allocator; - /// Session name (Cluster ID) of the cluster. + /// The current Ray session name. std::string session_name; std::string entrypoint; int64_t worker_launch_time_ms; diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index a0b5109fc8da..348140770fab 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -175,7 +175,8 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( auto task_event_buffer = std::make_unique( std::make_unique(options.gcs_options), std::make_unique(options.metrics_agent_port, - *client_call_manager)); + *client_call_manager), + options.session_name); // Start the IO thread first to make sure the checker is working. boost::thread::attributes io_thread_attrs; diff --git a/src/ray/core_worker/task_event_buffer.cc b/src/ray/core_worker/task_event_buffer.cc index 48348add2176..bfc85faad1f7 100644 --- a/src/ray/core_worker/task_event_buffer.cc +++ b/src/ray/core_worker/task_event_buffer.cc @@ -37,12 +37,14 @@ TaskStatusEvent::TaskStatusEvent( const rpc::TaskStatus &task_status, int64_t timestamp, bool is_actor_task_event, + std::string session_name, const std::shared_ptr &task_spec, std::optional state_update) : TaskEvent(task_id, job_id, attempt_number), task_status_(task_status), timestamp_(timestamp), is_actor_task_event_(is_actor_task_event), + session_name_(session_name), task_spec_(task_spec), state_update_(std::move(state_update)) {} @@ -181,6 +183,7 @@ void TaskStatusEvent::PopulateRpcRayTaskDefinitionEvent(T &definition_event_data definition_event_data.set_task_attempt(attempt_number_); // Common fields + definition_event_data.set_language(task_spec_->GetLanguage()); const auto &required_resources = task_spec_->GetRequiredResources().GetResourceMap(); definition_event_data.mutable_required_resources()->insert( std::make_move_iterator(required_resources.begin()), @@ -199,16 +202,18 @@ void TaskStatusEvent::PopulateRpcRayTaskDefinitionEvent(T &definition_event_data definition_event_data.mutable_actor_func()->CopyFrom( task_spec_->FunctionDescriptor()->GetMessage()); definition_event_data.set_actor_id(task_spec_->ActorId().Binary()); + definition_event_data.set_actor_task_name(task_spec_->GetName()); } else { definition_event_data.mutable_task_func()->CopyFrom( task_spec_->FunctionDescriptor()->GetMessage()); + definition_event_data.set_task_type(task_spec_->GetMessage().type()); definition_event_data.set_task_name(task_spec_->GetName()); } } -template void TaskStatusEvent::PopulateRpcRayTaskExecutionEvent( - T &execution_event_data, google::protobuf::Timestamp timestamp) { + rpc::events::TaskExecutionEvent &execution_event_data, + google::protobuf::Timestamp timestamp) { // Task identifier execution_event_data.set_task_id(task_id_.Binary()); execution_event_data.set_task_attempt(attempt_number_); @@ -257,19 +262,16 @@ void TaskStatusEvent::PopulateRpcRayEventBaseFields( ray_event.set_source_type(rpc::events::RayEvent::CORE_WORKER); ray_event.mutable_timestamp()->CopyFrom(timestamp); ray_event.set_severity(rpc::events::RayEvent::INFO); + ray_event.set_session_name(session_name_); - if (is_actor_task_event_) { - if (is_definition_event) { + if (is_definition_event) { + if (is_actor_task_event_) { ray_event.set_event_type(rpc::events::RayEvent::ACTOR_TASK_DEFINITION_EVENT); } else { - ray_event.set_event_type(rpc::events::RayEvent::ACTOR_TASK_EXECUTION_EVENT); - } - } else { - if (is_definition_event) { ray_event.set_event_type(rpc::events::RayEvent::TASK_DEFINITION_EVENT); - } else { - ray_event.set_event_type(rpc::events::RayEvent::TASK_EXECUTION_EVENT); } + } else { + ray_event.set_event_type(rpc::events::RayEvent::TASK_EXECUTION_EVENT); } } @@ -298,15 +300,9 @@ void TaskStatusEvent::ToRpcRayEvents(RayEventsPair &ray_events_pair) { : task_execution_event_rpc.emplace(), false, timestamp); - if (is_actor_task_event_) { - auto actor_task_execution_event = - task_execution_event_rpc.value().mutable_actor_task_execution_event(); - PopulateRpcRayTaskExecutionEvent(*actor_task_execution_event, timestamp); - } else { - auto task_execution_event = - task_execution_event_rpc.value().mutable_task_execution_event(); - PopulateRpcRayTaskExecutionEvent(*task_execution_event, timestamp); - } + auto task_execution_event = + task_execution_event_rpc.value().mutable_task_execution_event(); + PopulateRpcRayTaskExecutionEvent(*task_execution_event, timestamp); } void TaskProfileEvent::ToRpcTaskEvents(rpc::TaskEvents *rpc_task_events) { @@ -351,7 +347,7 @@ void TaskProfileEvent::ToRpcRayEvents(RayEventsPair &ray_events_pair) { // to the new ray event format. } -bool TaskEventBuffer::RecordTaskStatusEventIfNeeded( +bool TaskEventBufferImpl::RecordTaskStatusEventIfNeeded( const TaskID &task_id, const JobID &job_id, int32_t attempt_number, @@ -373,6 +369,7 @@ bool TaskEventBuffer::RecordTaskStatusEventIfNeeded( status, /* timestamp */ absl::GetCurrentTimeNanos(), /*is_actor_task_event=*/spec.IsActorTask(), + session_name_, include_task_info ? std::make_shared(spec) : nullptr, std::move(state_update)); @@ -382,21 +379,28 @@ bool TaskEventBuffer::RecordTaskStatusEventIfNeeded( TaskEventBufferImpl::TaskEventBufferImpl( std::unique_ptr gcs_client, - std::unique_ptr event_aggregator_client) + std::unique_ptr event_aggregator_client, + std::string session_name) : work_guard_(boost::asio::make_work_guard(io_service_)), periodical_runner_(PeriodicalRunner::Create(io_service_)), gcs_client_(std::move(gcs_client)), - event_aggregator_client_(std::move(event_aggregator_client)) {} + event_aggregator_client_(std::move(event_aggregator_client)), + session_name_(session_name) {} TaskEventBufferImpl::~TaskEventBufferImpl() { Stop(); } Status TaskEventBufferImpl::Start(bool auto_flush) { absl::MutexLock lock(&mutex_); - export_event_write_enabled_ = TaskEventBufferImpl::IsExportAPIEnabledTask(); send_task_events_to_gcs_enabled_ = RayConfig::instance().enable_core_worker_task_event_to_gcs(); send_ray_events_to_aggregator_enabled_ = RayConfig::instance().enable_core_worker_ray_event_to_aggregator(); + + // We want to make sure that only one of the event export mechanism is enabled. And + // if both are enabled, we will use the event aggregator instead of the export API. + // This code will be removed when we deprecate the export API implementation. + export_event_write_enabled_ = !send_ray_events_to_aggregator_enabled_ && + TaskEventBufferImpl::IsExportAPIEnabledTask(); auto report_interval_ms = RayConfig::instance().task_events_report_interval_ms(); RAY_CHECK(report_interval_ms > 0) << "RAY_task_events_report_interval_ms should be > 0 to use TaskEventBuffer."; diff --git a/src/ray/core_worker/task_event_buffer.h b/src/ray/core_worker/task_event_buffer.h index 131c905a35d1..5d68e1dc50db 100644 --- a/src/ray/core_worker/task_event_buffer.h +++ b/src/ray/core_worker/task_event_buffer.h @@ -150,6 +150,7 @@ class TaskStatusEvent : public TaskEvent { const rpc::TaskStatus &task_status, int64_t timestamp, bool is_actor_task_event, + std::string session_name, const std::shared_ptr &task_spec = nullptr, std::optional state_update = std::nullopt); @@ -175,17 +176,13 @@ class TaskStatusEvent : public TaskEvent { private: // Helper functions to populate the task definition event of rpc::events::RayEvent // This function assumes task_spec_ is not null. - // This function also checks T must be one of rpc::events::ActorTaskDefinitionEvent or - // rpc::events::TaskDefinitionEvent template void PopulateRpcRayTaskDefinitionEvent(T &definition_event_data); // Helper functions to populate the task execution event of rpc::events::RayEvent - // This function checks T must be one of rpc::events::ActorTaskExecutionEvent or - // rpc::events::TaskExecutionEvent - template - void PopulateRpcRayTaskExecutionEvent(T &execution_event_data, - google::protobuf::Timestamp timestamp); + void PopulateRpcRayTaskExecutionEvent( + rpc::events::TaskExecutionEvent &execution_event_data, + google::protobuf::Timestamp timestamp); // Helper functions to populate the base fields of rpc::events::RayEvent void PopulateRpcRayEventBaseFields(rpc::events::RayEvent &ray_event, @@ -198,6 +195,8 @@ class TaskStatusEvent : public TaskEvent { int64_t timestamp_ = -1; /// Whether the task is an actor task. bool is_actor_task_event_ = false; + /// The current Ray session name. + std::string session_name_; /// Pointer to the task spec. std::shared_ptr task_spec_ = nullptr; /// Optional task state update @@ -300,14 +299,15 @@ class TaskEventBuffer { /// \param status the changed status. /// \param state_update optional task state updates. /// \return true if the event is recorded, false otherwise. - bool RecordTaskStatusEventIfNeeded( + virtual bool RecordTaskStatusEventIfNeeded( const TaskID &task_id, const JobID &job_id, int32_t attempt_number, const TaskSpecification &spec, rpc::TaskStatus status, bool include_task_info = false, - std::optional state_update = absl::nullopt); + std::optional state_update = + absl::nullopt) = 0; /// Add a task event to be reported. /// @@ -367,13 +367,23 @@ class TaskEventBufferImpl : public TaskEventBuffer { /// \param event_aggregator_client Event aggregator client explicit TaskEventBufferImpl( std::unique_ptr gcs_client, - std::unique_ptr event_aggregator_client); + std::unique_ptr event_aggregator_client, + std::string session_name); TaskEventBufferImpl(const TaskEventBufferImpl &) = delete; TaskEventBufferImpl &operator=(const TaskEventBufferImpl &) = delete; ~TaskEventBufferImpl() override; + bool RecordTaskStatusEventIfNeeded(const TaskID &task_id, + const JobID &job_id, + int32_t attempt_number, + const TaskSpecification &spec, + rpc::TaskStatus status, + bool include_task_info = false, + std::optional + state_update = absl::nullopt) override; + void AddTaskEvent(std::unique_ptr task_event) ABSL_LOCKS_EXCLUDED(mutex_) override; @@ -566,6 +576,9 @@ class TaskEventBufferImpl : public TaskEventBuffer { /// If true, ray events from the event buffer are sent to the event aggregator bool send_ray_events_to_aggregator_enabled_ = false; + /// The current Ray session name. Passed in from the core worker + std::string session_name_ = ""; + FRIEND_TEST(TaskEventBufferTestManualStart, TestGcsClientFail); FRIEND_TEST(TaskEventBufferTestBatchSendDifferentDestination, TestBatchedSend); FRIEND_TEST(TaskEventBufferTest, TestAddEvents); diff --git a/src/ray/core_worker/task_execution/test/scheduling_queue_test.cc b/src/ray/core_worker/task_execution/test/scheduling_queue_test.cc index 057fde36ec89..32752abc4ee3 100644 --- a/src/ray/core_worker/task_execution/test/scheduling_queue_test.cc +++ b/src/ray/core_worker/task_execution/test/scheduling_queue_test.cc @@ -61,6 +61,28 @@ class MockTaskEventBuffer : public worker::TaskEventBuffer { std::string DebugString() override { return ""; } + bool RecordTaskStatusEventIfNeeded( + const TaskID &task_id, + const JobID &job_id, + int32_t attempt_number, + const TaskSpecification &spec, + rpc::TaskStatus status, + bool include_task_info, + std::optional state_update) + override { + AddTaskEvent(std::make_unique( + task_id, + job_id, + attempt_number, + status, + /* timestamp */ absl::GetCurrentTimeNanos(), + /*is_actor_task_event=*/spec.IsActorTask(), + "test-session-name", + include_task_info ? std::make_shared(spec) : nullptr, + std::move(state_update))); + return true; + } + std::vector> task_events; }; diff --git a/src/ray/core_worker/task_execution/test/task_receiver_test.cc b/src/ray/core_worker/task_execution/test/task_receiver_test.cc index 915299250fc8..ee1d2bfa7d8b 100644 --- a/src/ray/core_worker/task_execution/test/task_receiver_test.cc +++ b/src/ray/core_worker/task_execution/test/task_receiver_test.cc @@ -107,6 +107,18 @@ class MockTaskEventBuffer : public worker::TaskEventBuffer { bool Enabled() const override { return true; } + bool RecordTaskStatusEventIfNeeded( + const TaskID &task_id, + const JobID &job_id, + int32_t attempt_number, + const TaskSpecification &spec, + rpc::TaskStatus status, + bool include_task_info, + std::optional state_update) + override { + return true; + } + std::string DebugString() override { return ""; } }; diff --git a/src/ray/core_worker/test/core_worker_test.cc b/src/ray/core_worker/test/core_worker_test.cc index 1a667638c64d..78e4fef68a24 100644 --- a/src/ray/core_worker/test/core_worker_test.cc +++ b/src/ray/core_worker/test/core_worker_test.cc @@ -142,7 +142,8 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { auto task_event_buffer = std::make_unique( std::make_unique(), - std::make_unique(0, *client_call_manager)); + std::make_unique(0, *client_call_manager), + "test_session"); auto task_manager = std::make_shared( *memory_store_, diff --git a/src/ray/core_worker/test/task_event_buffer_export_event_test.cc b/src/ray/core_worker/test/task_event_buffer_export_event_test.cc index e3f76d3a1e83..9e6345bd9582 100644 --- a/src/ray/core_worker/test/task_event_buffer_export_event_test.cc +++ b/src/ray/core_worker/test/task_event_buffer_export_event_test.cc @@ -63,13 +63,15 @@ class TaskEventTestWriteExport : public ::testing::Test { "task_events_send_batch_size": 100, "export_task_events_write_batch_size": 1, "task_events_max_num_export_status_events_buffer_on_worker": 15, - "enable_export_api_write": true + "enable_export_api_write": true, + "enable_core_worker_ray_event_to_aggregator": false } )"); task_event_buffer_ = std::make_unique( std::make_unique(), - std::make_unique()); + std::make_unique(), + "test_session_name"); } virtual void SetUp() { RAY_CHECK_OK(task_event_buffer_->Start(/*auto_flush*/ false)); } @@ -99,6 +101,7 @@ class TaskEventTestWriteExport : public ::testing::Test { rpc::TaskStatus::RUNNING, running_ts, /*is_actor_task_event=*/false, + "test_session_name", nullptr, state_update); } diff --git a/src/ray/core_worker/test/task_event_buffer_test.cc b/src/ray/core_worker/test/task_event_buffer_test.cc index 462a940c3dab..b57467f5e3d2 100644 --- a/src/ray/core_worker/test/task_event_buffer_test.cc +++ b/src/ray/core_worker/test/task_event_buffer_test.cc @@ -91,7 +91,8 @@ class TaskEventBufferTest : public ::testing::Test { task_event_buffer_ = std::make_unique( std::make_unique(), - std::make_unique()); + std::make_unique(), + "test_session_name"); } virtual void SetUp() { RAY_CHECK_OK(task_event_buffer_->Start(/*auto_flush*/ false)); } @@ -156,6 +157,7 @@ class TaskEventBufferTest : public ::testing::Test { rpc::TaskStatus::RUNNING, 1, /*is_actor_task_event=*/false, + "test_session_name", std::make_shared(task_spec), status_update); } @@ -172,6 +174,7 @@ class TaskEventBufferTest : public ::testing::Test { rpc::TaskStatus::RUNNING, running_ts, /*is_actor_task_event=*/false, + "test_session_name", nullptr, state_update); } diff --git a/src/ray/core_worker/test/task_manager_test.cc b/src/ray/core_worker/test/task_manager_test.cc index 9be48562a797..64e5ef03d29c 100644 --- a/src/ray/core_worker/test/task_manager_test.cc +++ b/src/ray/core_worker/test/task_manager_test.cc @@ -120,6 +120,18 @@ class MockTaskEventBuffer : public worker::TaskEventBuffer { MOCK_METHOD(bool, Enabled, (), (const, override)); MOCK_METHOD(std::string, DebugString, (), (override)); + + MOCK_METHOD( + bool, + RecordTaskStatusEventIfNeeded, + (const TaskID &task_id, + const JobID &job_id, + int32_t attempt_number, + const TaskSpecification &spec, + rpc::TaskStatus status, + bool include_task_info, + std::optional state_update), + (override)); }; class TaskManagerTest : public ::testing::Test { diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h index e37d582fb630..b57cc5c46626 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h @@ -172,7 +172,7 @@ class GcsAutoscalerStateManager : public rpc::autoscaler::AutoscalerStateHandler /// TODO: Implement the function void CancelInfeasibleRequests() const; - // Ray cluster session name. + // The current Ray session name. const std::string session_name_; /// Gcs node manager that provides node status information. diff --git a/src/ray/gcs/gcs_server/gcs_server_main.cc b/src/ray/gcs/gcs_server/gcs_server_main.cc index 2bf2366a2626..74c371cb4201 100644 --- a/src/ray/gcs/gcs_server/gcs_server_main.cc +++ b/src/ray/gcs/gcs_server/gcs_server_main.cc @@ -42,9 +42,7 @@ DEFINE_string(redis_username, "", "The username of Redis."); DEFINE_string(redis_password, "", "The password of Redis."); DEFINE_bool(retry_redis, false, "Whether to retry to connect to Redis."); DEFINE_string(node_ip_address, "", "The IP address of the node."); -DEFINE_string(session_name, - "", - "session_name: The session name (ClusterID) of the cluster."); +DEFINE_string(session_name, "", "session_name: The current Ray session name."); DEFINE_string(ray_commit, "", "The commit hash of Ray."); int main(int argc, char *argv[]) { diff --git a/src/ray/protobuf/BUILD.bazel b/src/ray/protobuf/BUILD.bazel index d1bf2f6f2b66..a4b1f3c61ff4 100644 --- a/src/ray/protobuf/BUILD.bazel +++ b/src/ray/protobuf/BUILD.bazel @@ -451,20 +451,6 @@ cc_proto_library( deps = [":events_actor_task_definition_event_proto"], ) -proto_library( - name = "events_actor_task_execution_event_proto", - srcs = ["events_actor_task_execution_event.proto"], - deps = [ - ":common_proto", - "@com_google_protobuf//:timestamp_proto", - ], -) - -cc_proto_library( - name = "events_actor_task_execution_event_cc_proto", - deps = [":events_actor_task_execution_event_proto"], -) - proto_library( name = "events_task_definition_event_proto", srcs = ["events_task_definition_event.proto"], @@ -511,7 +497,6 @@ proto_library( srcs = ["events_base_event.proto"], deps = [ ":events_actor_task_definition_event_proto", - ":events_actor_task_execution_event_proto", ":events_task_definition_event_proto", ":events_task_execution_event_proto", ":events_task_profile_events_proto", diff --git a/src/ray/protobuf/autoscaler.proto b/src/ray/protobuf/autoscaler.proto index 9ad2ef7b191c..6666e131828c 100644 --- a/src/ray/protobuf/autoscaler.proto +++ b/src/ray/protobuf/autoscaler.proto @@ -214,7 +214,7 @@ message ClusterResourceState { // There could be multiple constraints issued by different // jobs. Autoscaler to make sure all constraints are satisfied. repeated ClusterResourceConstraint cluster_resource_constraints = 6; - // The cluster session name. + // The current Ray session name. string cluster_session_name = 7; } diff --git a/src/ray/protobuf/events_actor_task_definition_event.proto b/src/ray/protobuf/events_actor_task_definition_event.proto index 4aa26e561fd9..c201c1602804 100644 --- a/src/ray/protobuf/events_actor_task_definition_event.proto +++ b/src/ray/protobuf/events_actor_task_definition_event.proto @@ -27,15 +27,17 @@ message ActorTaskDefinitionEvent { int32 task_attempt = 2; // The actor task definition information. - FunctionDescriptor actor_func = 3; - map required_resources = 5; - RuntimeEnvInfo runtime_env_info = 6; + Language language = 3; + FunctionDescriptor actor_func = 4; + string actor_task_name = 5; + map required_resources = 6; + RuntimeEnvInfo runtime_env_info = 7; // The correlation ids of the task that can be used to correlate the task with // other events. - bytes job_id = 7; - bytes actor_id = 8; - bytes parent_task_id = 9; - bytes placement_group_id = 10; - map ref_ids = 11; + bytes job_id = 8; + bytes actor_id = 9; + bytes parent_task_id = 10; + bytes placement_group_id = 11; + map ref_ids = 12; } diff --git a/src/ray/protobuf/events_actor_task_execution_event.proto b/src/ray/protobuf/events_actor_task_execution_event.proto deleted file mode 100644 index 3e7ae892c769..000000000000 --- a/src/ray/protobuf/events_actor_task_execution_event.proto +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2025 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -import "src/ray/protobuf/common.proto"; -import "google/protobuf/timestamp.proto"; - -package ray.rpc.events; - -// Message containing the execution information of an actor task. -message ActorTaskExecutionEvent { - // task_id and task_attempt form the unique identifier of a task. - bytes task_id = 1; - int32 task_attempt = 2; - - // The actor task execution information - - // The map of task state to the time when the state was last updated. - - // Key is the integer value of TaskStatus enum (protobuf doesn't support Enum as key). - // Value is the timestamp when status changes to the target status indicated by the key. - map task_state = 3; - UserErrorInfo user_error_info = 4; - RayErrorInfo ray_error_info = 5; - - // The correlation ids of the task that can be used to correlate the task with - // other events. - bytes node_id = 6; - bytes worker_id = 7; - int32 worker_pid = 8; -} diff --git a/src/ray/protobuf/events_base_event.proto b/src/ray/protobuf/events_base_event.proto index aac704ef6e25..a68a9e02c112 100644 --- a/src/ray/protobuf/events_base_event.proto +++ b/src/ray/protobuf/events_base_event.proto @@ -18,7 +18,6 @@ package ray.rpc.events; import "google/protobuf/timestamp.proto"; import "src/ray/protobuf/events_actor_task_definition_event.proto"; -import "src/ray/protobuf/events_actor_task_execution_event.proto"; import "src/ray/protobuf/events_task_definition_event.proto"; import "src/ray/protobuf/events_task_execution_event.proto"; import "src/ray/protobuf/events_task_profile_events.proto"; @@ -46,8 +45,7 @@ message RayEvent { TASK_DEFINITION_EVENT = 1; TASK_EXECUTION_EVENT = 2; ACTOR_TASK_DEFINITION_EVENT = 3; - ACTOR_TASK_EXECUTION_EVENT = 4; - TASK_PROFILE_EVENT = 5; + TASK_PROFILE_EVENT = 4; } // The severities of events that can be generated. @@ -78,12 +76,13 @@ message RayEvent { Severity severity = 5; // A string message associated with the event. string message = 6; + // The current Ray session name. + string session_name = 7; // Nested event messages containing the specific fields for each event type. // One of the following fields is expected to be set for each RayEvent message. - TaskDefinitionEvent task_definition_event = 7; - TaskExecutionEvent task_execution_event = 8; - ActorTaskDefinitionEvent actor_task_definition_event = 9; - ActorTaskExecutionEvent actor_task_execution_event = 10; + TaskDefinitionEvent task_definition_event = 8; + TaskExecutionEvent task_execution_event = 9; + ActorTaskDefinitionEvent actor_task_definition_event = 10; TaskProfileEvents task_profile_events = 11; } diff --git a/src/ray/protobuf/events_task_definition_event.proto b/src/ray/protobuf/events_task_definition_event.proto index c47b2b0503de..2c63c7559b34 100644 --- a/src/ray/protobuf/events_task_definition_event.proto +++ b/src/ray/protobuf/events_task_definition_event.proto @@ -27,15 +27,18 @@ message TaskDefinitionEvent { int32 task_attempt = 2; // The task definition information. - FunctionDescriptor task_func = 3; - string task_name = 4; - map required_resources = 5; - RuntimeEnvInfo runtime_env_info = 6; + // Valid values are NORMAL_TASK, ACTOR_CREATION_TASK, DRIVER_TASK + TaskType task_type = 3; + Language language = 4; + FunctionDescriptor task_func = 5; + string task_name = 6; + map required_resources = 7; + RuntimeEnvInfo runtime_env_info = 8; // The correlation ids of the task that can be used to correlate the task with // other events. - bytes job_id = 7; - bytes parent_task_id = 8; - bytes placement_group_id = 9; - map ref_ids = 10; + bytes job_id = 9; + bytes parent_task_id = 10; + bytes placement_group_id = 11; + map ref_ids = 12; } diff --git a/src/ray/protobuf/events_task_execution_event.proto b/src/ray/protobuf/events_task_execution_event.proto index 7418f9354064..49e3ee37d569 100644 --- a/src/ray/protobuf/events_task_execution_event.proto +++ b/src/ray/protobuf/events_task_execution_event.proto @@ -28,15 +28,15 @@ message TaskExecutionEvent { // The task execution information + // The map of task state to the time when the state was last updated. // Key is the integer value of TaskStatus enum (protobuf doesn't support Enum as key). // Value is the timestamp when status changes to the target status indicated by the key. map task_state = 3; - UserErrorInfo user_error_info = 4; - RayErrorInfo ray_error_info = 5; + RayErrorInfo ray_error_info = 4; // The correlation ids of the task that can be used to correlate the task with // other events. - bytes node_id = 6; - bytes worker_id = 7; - int32 worker_pid = 8; + bytes node_id = 5; + bytes worker_id = 6; + int32 worker_pid = 7; } diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 676b7d0bc2ef..7193230c0729 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -90,7 +90,7 @@ DEFINE_int32(ray_debugger_external, 0, "Make Ray debugger externally accessible. // store options DEFINE_int64(object_store_memory, -1, "The initial memory of the object store."); DEFINE_string(node_name, "", "The user-provided identifier or name for this node."); -DEFINE_string(session_name, "", "Session name (ClusterID) of the cluster."); +DEFINE_string(session_name, "", "The current Ray session name."); DEFINE_string(cluster_id, "", "ID of the cluster, separate from observability."); // TODO(hjiang): At the moment only enablement flag is added, I will add other flags for // CPU and memory resource reservation in the followup PR. diff --git a/src/ray/rpc/event_aggregator_client.h b/src/ray/rpc/event_aggregator_client.h index b8b0f2fe3dd1..83746f02faa7 100644 --- a/src/ray/rpc/event_aggregator_client.h +++ b/src/ray/rpc/event_aggregator_client.h @@ -20,7 +20,6 @@ #include #include -#include "ray/common/status.h" #include "ray/rpc/grpc_client.h" #include "ray/util/logging.h" #include "src/ray/protobuf/events_event_aggregator_service.grpc.pb.h" From 458c83b90231e220a03a41da6ed58960871b131c Mon Sep 17 00:00:00 2001 From: Jason Li <57246540+JasonLi1909@users.noreply.github.com> Date: Mon, 11 Aug 2025 16:06:24 -0700 Subject: [PATCH 032/634] Ray train improve worker error logging (#55222) Improved worker error logging by making them less verbose. Previously, all workers errors were appended together to produce a large string. This caused the training runs to show up as `Running` on the dashboard even if they already `Errored` and prevented the final error from being logged on the Train Dashboard as it was too large to export. This PR dedupes similar errors, indicates the corresponding workers, and truncates long error messages to be displayed to users. It also fixes the issue of the train run status not being updated. --------- Signed-off-by: JasonLi1909 Signed-off-by: Jason Li <57246540+JasonLi1909@users.noreply.github.com> --- python/ray/train/v2/BUILD | 16 ++++ .../_internal/execution/worker_group/poll.py | 74 ++++++++++++++- .../v2/tests/test_worker_group_poll_status.py | 92 +++++++++++++++++++ 3 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 python/ray/train/v2/tests/test_worker_group_poll_status.py diff --git a/python/ray/train/v2/BUILD b/python/ray/train/v2/BUILD index 502056cb9885..31bab21afc44 100644 --- a/python/ray/train/v2/BUILD +++ b/python/ray/train/v2/BUILD @@ -469,6 +469,22 @@ py_test( ], ) +py_test( + name = "test_worker_group_poll_status", + size = "small", + srcs = ["tests/test_worker_group_poll_status.py"], + env = {"RAY_TRAIN_V2_ENABLED": "1"}, + tags = [ + "exclusive", + "team:ml", + "train_v2", + ], + deps = [ + ":conftest", + "//:ray_lib", + ], +) + py_test( name = "test_xgboost_trainer", size = "small", diff --git a/python/ray/train/v2/_internal/execution/worker_group/poll.py b/python/ray/train/v2/_internal/execution/worker_group/poll.py index e7f47d68da46..dffe2eb5d892 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/poll.py +++ b/python/ray/train/v2/_internal/execution/worker_group/poll.py @@ -1,10 +1,36 @@ +import re +from collections import defaultdict from dataclasses import dataclass from typing import Dict, Optional +from ray._private.ray_logging import NUMBERS from ray.train._internal.session import _TrainingResult +from ray.train.v2._internal.exceptions import WorkerHealthCheckFailedError from ray.train.v2.api.exceptions import WorkerGroupError from ray.types import ObjectRef +ERR_CHAR_LIMIT = 1000 + + +def _normalize_error_string(error_str: str) -> str: + # Replace numbers with based on NUMBERS regex + normalized = re.sub(NUMBERS, "", error_str) + return normalized + + +def _truncate_error_string(error_str: str) -> str: + """ + Truncates error strings to include the first ERR_CHAR_LIMIT // 2 + characters and the last ERR_CHAR_LIMIT // 2 characters. + """ + if len(error_str) >= ERR_CHAR_LIMIT: + return ( + error_str[: ERR_CHAR_LIMIT // 2] + + "...\n... (Output truncated. See individual worker logs for full details) ...\n" + + error_str[len(error_str) - ERR_CHAR_LIMIT // 2 :] + ) + return error_str + @dataclass class WorkerStatus: @@ -38,9 +64,51 @@ def finished(self) -> bool: ) def get_error_string(self) -> str: - return "\n".join( - f"[Rank {world_rank}]\n{error}" for world_rank, error in self.errors.items() - ) + """ + Returns a string representation of worker group errors. + Groups similar errors (ignoring numbers) and shows original error examples. + """ + # Group errors by normalized strings (ignoring numbers) + normalized_error_to_ranks = defaultdict(list) + normalized_error_to_original = {} + show_full_error = set() + + for world_rank, status in self.worker_statuses.items(): + if status.error: + error_str = str(status.error) + normalized_error = _normalize_error_string(error_str) + + normalized_error_to_ranks[normalized_error].append(str(world_rank)) + + # Store the first original error for this normalized group + if normalized_error not in normalized_error_to_original: + normalized_error_to_original[normalized_error] = error_str + + # Fully show errors for non-graceful worker failures or running workers + if ( + isinstance(status.error, WorkerHealthCheckFailedError) + or status.running + ): + show_full_error.add(normalized_error) + + errors = [] + for normalized_error, ranks in normalized_error_to_ranks.items(): + # Show the original error + orig_error = normalized_error_to_original[normalized_error] + + # Convert rank list to comma-separated strings + ranks_str = ",".join(ranks) + + if normalized_error in show_full_error: + errors.append(f"[Rank {ranks_str} Error Snippet]:\n{orig_error}") + else: + errors.append( + f"[Rank {ranks_str} Error Snippet]:\n{_truncate_error_string(orig_error)}" + ) + + error_str = "\n".join(errors) + + return error_str @dataclass(frozen=True) diff --git a/python/ray/train/v2/tests/test_worker_group_poll_status.py b/python/ray/train/v2/tests/test_worker_group_poll_status.py new file mode 100644 index 000000000000..03c394a961a6 --- /dev/null +++ b/python/ray/train/v2/tests/test_worker_group_poll_status.py @@ -0,0 +1,92 @@ +import pytest + +from ray.train.v2._internal.execution.worker_group.poll import ( + ERR_CHAR_LIMIT, + WorkerGroupPollStatus, + WorkerStatus, + _normalize_error_string, +) + + +def test_get_error_string_basic(): + """ + Simulate four workers, two with the same error, one with a different error, + and one without an error. + """ + + statuses = { + 0: WorkerStatus(running=False, error=ValueError("An error")), + 1: WorkerStatus(running=False, error=None), + 2: WorkerStatus(running=False, error=RuntimeError("Different error")), + 3: WorkerStatus(running=False, error=ValueError("An error")), + } + poll_status = WorkerGroupPollStatus(worker_statuses=statuses) + error_str = poll_status.get_error_string() + + expected_error_str = ( + "[Rank 0,3 Error Snippet]:\nAn error\n[Rank 2 Error Snippet]:\nDifferent error" + ) + assert error_str == expected_error_str + + +def test_get_error_string_with_numbers(): + """ + Simulate workers with similar errors that differ only by numbers. + These should be grouped together. + """ + statuses = { + 0: WorkerStatus( + running=False, error=ValueError("Error parsing object at 0x7f8b12345678") + ), + 1: WorkerStatus( + running=False, error=ValueError("Error parsing object at 0x7f8b12345679") + ), + } + poll_status = WorkerGroupPollStatus(worker_statuses=statuses) + error_str = poll_status.get_error_string() + + assert ( + error_str == "[Rank 0,1 Error Snippet]:\nError parsing object at 0x7f8b12345678" + ) + + +def test_get_error_string_long_error(): + """ + Simulate two workers with identical long error string. + """ + long_error_str = "test string" * 200 + statuses = { + 0: WorkerStatus(running=False, error=long_error_str), + 1: WorkerStatus(running=False, error=long_error_str), + } + poll_status = WorkerGroupPollStatus(worker_statuses=statuses) + error_str = poll_status.get_error_string() + + expected_error_str = ( + "[Rank 0,1 Error Snippet]:\n" + + long_error_str[: ERR_CHAR_LIMIT // 2] + + "...\n... (Output truncated. See individual worker logs for full details) ...\n" + + long_error_str[len(long_error_str) - ERR_CHAR_LIMIT // 2 :] + ) + assert error_str == expected_error_str + + +def test_normalize_error_string(): + """Test that _normalize_error_string properly handles all types of numbers.""" + error = """Traceback (most recent call last): +File "/home/ray/default/train_benchmark.py", line 35, in train_fn_per_worker +File "/tmp/ray/session_2025-08-07_23-49-55_617067_2585/runtime_resources/working_dir_files/_ray_pkg_5abd79ca51ba0ed4/runner.py", line 282, in run""" + result = _normalize_error_string(error) + + assert ( + result + == """Traceback (most recent call last): +File "/home/ray/default/train_benchmark.py", line , in train_fn_per_worker +File "/tmp/ray/session_--_--__/runtime_resources/working_dir_files/_ray_pkg_abdcabaed/runner.py", line , in run""" + ) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", "-x", __file__])) From da64afb59fa449cfaa6174bd3b03cc621b5be33a Mon Sep 17 00:00:00 2001 From: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Date: Mon, 11 Aug 2025 16:54:29 -0700 Subject: [PATCH 033/634] [docs][serve.llm] Add Ray Serve LLM docs examples to test (#54763) Signed-off-by: Seiji Eicher --- doc/BUILD.bazel | 26 ++++++ .../serve/qwen/llm_config_example.yaml | 29 +++++++ .../serve/qwen/llm_yaml_config_example.py | 47 +++++++++++ .../llm/doc_code/serve/qwen/qwen_example.py | 84 +++++++++++++++++++ doc/source/serve/llm/serving-llms.rst | 56 ++----------- 5 files changed, 192 insertions(+), 50 deletions(-) create mode 100644 doc/source/llm/doc_code/serve/qwen/llm_config_example.yaml create mode 100644 doc/source/llm/doc_code/serve/qwen/llm_yaml_config_example.py create mode 100644 doc/source/llm/doc_code/serve/qwen/qwen_example.py diff --git a/doc/BUILD.bazel b/doc/BUILD.bazel index 8e0cf7adde4e..2d0ac0634687 100644 --- a/doc/BUILD.bazel +++ b/doc/BUILD.bazel @@ -239,6 +239,8 @@ py_test_run_all_subdirectory( "source/serve/doc_code/stable_diffusion.py", "source/serve/doc_code/object_detection.py", "source/serve/doc_code/vllm_example.py", + "source/serve/doc_code/llm/llm_yaml_config_example.py", + "source/serve/doc_code/llm/qwen_example.py", ], extra_srcs = [], tags = [ @@ -270,6 +272,30 @@ py_test_run_all_subdirectory( ], ) +# -------------------------------------------------------------------- +# Test all doc/source/llm/doc_code/serve code included in rst/md files. +# -------------------------------------------------------------------- + +filegroup( + name = "serve_llm_examples", + srcs = glob(["source/llm/doc_code/serve/**/*.py"]), + visibility = ["//doc:__subpackages__"], +) + +# GPU Tests +py_test_run_all_subdirectory( + size = "large", + include = ["source/llm/doc_code/serve/**/*.py"], + exclude = [], + extra_srcs = [], + data = ["source/llm/doc_code/serve/qwen/llm_config_example.yaml"], + tags = [ + "exclusive", + "gpu", + "team:llm", + ], +) + # -------------------------------------------------------------------- # Test all doc/source/tune/doc_code code included in rst/md files. # -------------------------------------------------------------------- diff --git a/doc/source/llm/doc_code/serve/qwen/llm_config_example.yaml b/doc/source/llm/doc_code/serve/qwen/llm_config_example.yaml new file mode 100644 index 000000000000..cd5302b6f637 --- /dev/null +++ b/doc/source/llm/doc_code/serve/qwen/llm_config_example.yaml @@ -0,0 +1,29 @@ +# config.yaml +applications: +- args: + llm_configs: + - model_loading_config: + model_id: qwen-0.5b + model_source: Qwen/Qwen2.5-0.5B-Instruct + accelerator_type: A10G + deployment_config: + autoscaling_config: + min_replicas: 1 + max_replicas: 2 + runtime_env: + env_vars: + VLLM_USE_V1: "1" + - model_loading_config: + model_id: qwen-1.5b + model_source: Qwen/Qwen2.5-1.5B-Instruct + accelerator_type: A10G + deployment_config: + autoscaling_config: + min_replicas: 1 + max_replicas: 2 + runtime_env: + env_vars: + VLLM_USE_V1: "1" + import_path: ray.serve.llm:build_openai_app + name: llm_app + route_prefix: "/" \ No newline at end of file diff --git a/doc/source/llm/doc_code/serve/qwen/llm_yaml_config_example.py b/doc/source/llm/doc_code/serve/qwen/llm_yaml_config_example.py new file mode 100644 index 000000000000..1f921f886716 --- /dev/null +++ b/doc/source/llm/doc_code/serve/qwen/llm_yaml_config_example.py @@ -0,0 +1,47 @@ +""" +This file serves as a documentation example and CI test for YAML config deployment. + +Structure: +1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing. +2. Load YAML config and convert to Python using build_openai_app +3. Test validation (deployment status polling + cleanup) +""" + +import time +import os +import yaml +from ray import serve +from ray.serve.schema import ApplicationStatus +from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME +from ray.serve import llm + + +config_path = os.path.join(os.path.dirname(__file__), "llm_config_example.yaml") +with open(config_path, "r") as f: + config_dict = yaml.safe_load(f) + +llm_configs = config_dict["applications"][0]["args"]["llm_configs"] +for config in llm_configs: + config.pop("accelerator_type", None) + +app = llm.build_openai_app({"llm_configs": llm_configs}) +serve.run(app, blocking=False) + +status = ApplicationStatus.NOT_STARTED +timeout_seconds = 180 +start_time = time.time() + +while ( + status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds +): + status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status + + if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]: + raise AssertionError(f"Deployment failed with status: {status}") + + time.sleep(1) + +if status != ApplicationStatus.RUNNING: + raise AssertionError( + f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}" + ) diff --git a/doc/source/llm/doc_code/serve/qwen/qwen_example.py b/doc/source/llm/doc_code/serve/qwen/qwen_example.py new file mode 100644 index 000000000000..791405940351 --- /dev/null +++ b/doc/source/llm/doc_code/serve/qwen/qwen_example.py @@ -0,0 +1,84 @@ +""" +This file serves as a documentation example and CI test. + +Structure: +1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing. +2. Docs example (between __qwen_example_start/end__): Embedded in Sphinx docs via literalinclude. +3. Test validation (deployment status polling + cleanup) +""" + +import time +from ray import serve +from ray.serve.schema import ApplicationStatus +from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME +from ray.serve import llm + +_original_serve_run = serve.run +_original_build_openai_app = llm.build_openai_app + + +def _non_blocking_serve_run(app, **kwargs): + """Forces blocking=False for testing""" + kwargs["blocking"] = False + return _original_serve_run(app, **kwargs) + + +def _testing_build_openai_app(llm_serving_args): + """Removes accelerator requirements for testing""" + for config in llm_serving_args["llm_configs"]: + config.accelerator_type = None + + return _original_build_openai_app(llm_serving_args) + + +serve.run = _non_blocking_serve_run +llm.build_openai_app = _testing_build_openai_app + +# __qwen_example_start__ +from ray import serve +from ray.serve.llm import LLMConfig, build_openai_app + +llm_config = LLMConfig( + model_loading_config={ + "model_id": "qwen-0.5b", + "model_source": "Qwen/Qwen2.5-0.5B-Instruct", + }, + deployment_config={ + "autoscaling_config": { + "min_replicas": 1, + "max_replicas": 2, + } + }, + # Pass the desired accelerator type (e.g. A10G, L4, etc.) + accelerator_type="A10G", + # You can customize the engine arguments (e.g. vLLM engine kwargs) + engine_kwargs={ + "tensor_parallel_size": 2, + }, + runtime_env={"env_vars": {"VLLM_USE_V1": "1"}}, +) + +app = build_openai_app({"llm_configs": [llm_config]}) +serve.run(app, blocking=True) +# __qwen_example_end__ + +status = ApplicationStatus.NOT_STARTED +timeout_seconds = 180 +start_time = time.time() + +while ( + status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds +): + status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status + + if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]: + raise AssertionError(f"Deployment failed with status: {status}") + + time.sleep(1) + +if status != ApplicationStatus.RUNNING: + raise AssertionError( + f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}" + ) + +serve.shutdown() diff --git a/doc/source/serve/llm/serving-llms.rst b/doc/source/serve/llm/serving-llms.rst index 9e5048e423b1..25657d6a9c9e 100644 --- a/doc/source/serve/llm/serving-llms.rst +++ b/doc/source/serve/llm/serving-llms.rst @@ -68,31 +68,10 @@ Deployment through :class:`OpenAiIngress ` .. tab-item:: Builder Pattern :sync: builder - .. code-block:: python - - from ray import serve - from ray.serve.llm import LLMConfig, build_openai_app - - llm_config = LLMConfig( - model_loading_config=dict( - model_id="qwen-0.5b", - model_source="Qwen/Qwen2.5-0.5B-Instruct", - ), - deployment_config=dict( - autoscaling_config=dict( - min_replicas=1, max_replicas=2, - ) - ), - # Pass the desired accelerator type (e.g. A10G, L4, etc.) - accelerator_type="A10G", - # You can customize the engine arguments (e.g. vLLM engine kwargs) - engine_kwargs=dict( - tensor_parallel_size=2, - ), - ) - - app = build_openai_app({"llm_configs": [llm_config]}) - serve.run(app, blocking=True) + .. literalinclude:: ../../llm/doc_code/serve/qwen/qwen_example.py + :language: python + :start-after: __qwen_example_start__ + :end-before: __qwen_example_end__ .. tab-item:: Bind Pattern :sync: bind @@ -263,31 +242,8 @@ For production deployments, Ray Serve LLM provides utilities for config-driven d .. tab-item:: Inline Config :sync: inline - .. code-block:: yaml - - # config.yaml - applications: - - args: - llm_configs: - - model_loading_config: - model_id: qwen-0.5b - model_source: Qwen/Qwen2.5-0.5B-Instruct - accelerator_type: A10G - deployment_config: - autoscaling_config: - min_replicas: 1 - max_replicas: 2 - - model_loading_config: - model_id: qwen-1.5b - model_source: Qwen/Qwen2.5-1.5B-Instruct - accelerator_type: A10G - deployment_config: - autoscaling_config: - min_replicas: 1 - max_replicas: 2 - import_path: ray.serve.llm:build_openai_app - name: llm_app - route_prefix: "/" + .. literalinclude:: ../../llm/doc_code/serve/qwen/llm_config_example.yaml + :language: yaml .. tab-item:: Standalone Config From 551a31b5041a4b4cd5799846c6f66788ed44a0ef Mon Sep 17 00:00:00 2001 From: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Date: Mon, 11 Aug 2025 16:56:00 -0700 Subject: [PATCH 034/634] [Serve.llm] Forbid extra fields in ModelLoadingConfig (#55440) Signed-off-by: Seiji Eicher Signed-off-by: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python/ray/llm/_internal/common/base_pydantic.py | 5 ++++- .../llm/_internal/serve/configs/server_models.py | 9 ++------- .../serve/deployments/llm/vllm/vllm_models.py | 1 - .../prefill_decode_disagg.py | 5 +++-- .../llm/tests/serve/cpu/configs/test_models.py | 16 ++++++++++++++++ 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/python/ray/llm/_internal/common/base_pydantic.py b/python/ray/llm/_internal/common/base_pydantic.py index 7add5baee6d8..ce4d49d8f955 100644 --- a/python/ray/llm/_internal/common/base_pydantic.py +++ b/python/ray/llm/_internal/common/base_pydantic.py @@ -13,7 +13,10 @@ class BaseModelExtended(BaseModel): # namespace as not protected. This means we need to be careful about overriding # internal attributes starting with `model_`. # See: https://github.com/anyscale/ray-llm/issues/1425 - model_config = ConfigDict(protected_namespaces=tuple()) + model_config = ConfigDict( + protected_namespaces=tuple(), + extra="forbid", + ) @classmethod def parse_yaml(cls: Type[ModelT], file, **kwargs) -> ModelT: diff --git a/python/ray/llm/_internal/serve/configs/server_models.py b/python/ray/llm/_internal/serve/configs/server_models.py index cacdf4aeeba3..ca046794cfa8 100644 --- a/python/ray/llm/_internal/serve/configs/server_models.py +++ b/python/ray/llm/_internal/serve/configs/server_models.py @@ -13,7 +13,6 @@ import pydantic from pydantic import ( BaseModel, - ConfigDict, Field, PositiveInt, PrivateAttr, @@ -108,6 +107,7 @@ def validate_dynamic_lora_loading_path(cls, value: Optional[str]): class ModelLoadingConfig(BaseModelExtended): + model_id: str = Field( description="The ID that should be used by end users to access this model.", ) @@ -134,11 +134,6 @@ class ModelLoadingConfig(BaseModelExtended): class LLMConfig(BaseModelExtended): - # model_config is a Pydantic setting. This setting merges with - # model_configs in parent classes. - model_config = ConfigDict( - extra="forbid", - ) runtime_env: Optional[Dict[str, Any]] = Field( None, @@ -537,7 +532,7 @@ def parse_args( return models -class LLMServingArgs(BaseModel): +class LLMServingArgs(BaseModelExtended): llm_configs: List[Union[str, LLMConfig]] = Field( description="A list of LLMConfigs, or paths to LLMConfigs, to run.", ) diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py index 77e7237f3b56..a2b0ec2b8d92 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py @@ -34,7 +34,6 @@ class VLLMEngineConfig(BaseModelExtended): model_config = ConfigDict( use_enum_values=True, - extra="forbid", ) model_id: str = Field( diff --git a/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py b/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py index ff11971dad74..2d66fac44c53 100644 --- a/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py +++ b/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py @@ -4,10 +4,11 @@ import uuid from typing import Any, AsyncGenerator, Dict, Union -from pydantic import BaseModel, Field +from pydantic import Field from vllm.config import KVTransferConfig from ray import serve +from ray.llm._internal.common.base_pydantic import BaseModelExtended from ray.llm._internal.serve.configs.openai_api_models import ( ChatCompletionRequest, ChatCompletionResponse, @@ -33,7 +34,7 @@ RequestType = Union[ChatCompletionRequest, CompletionRequest] -class PDServingArgs(BaseModel): +class PDServingArgs(BaseModelExtended): """Schema for P/D serving args.""" prefill_config: Union[str, LLMConfig] diff --git a/python/ray/llm/tests/serve/cpu/configs/test_models.py b/python/ray/llm/tests/serve/cpu/configs/test_models.py index 886cf7430e31..057e27b8c65a 100644 --- a/python/ray/llm/tests/serve/cpu/configs/test_models.py +++ b/python/ray/llm/tests/serve/cpu/configs/test_models.py @@ -68,6 +68,22 @@ def test_invalid_accelerator_type(self): accelerator_type="A100_40G", # Should use A100-40G instead ) + def test_model_loading_config_forbids_extra_fields(self): + """Test that ModelLoadingConfig rejects extra fields.""" + + with pytest.raises(pydantic.ValidationError, match="engine_kwargs"): + ModelLoadingConfig( + model_id="test_model", + model_source="test_source", + engine_kwargs={"max_model_len": 8000}, # This should be rejected + ) + + valid_config = ModelLoadingConfig( + model_id="test_model", model_source="test_source" + ) + assert valid_config.model_id == "test_model" + assert valid_config.model_source == "test_source" + def test_invalid_generation_config(self, disable_placement_bundles): """Test that passing an invalid generation_config raises an error.""" with pytest.raises( From 300318067df9b36936992f80245fcddeda70e275 Mon Sep 17 00:00:00 2001 From: Justin Yu Date: Mon, 11 Aug 2025 17:05:18 -0700 Subject: [PATCH 035/634] [train] Make `ray.train.get_dataset_shard` lazily configure the dataset sharding (#55230) Instead of pre-sharding the dataset before creating the worker group, configure the dataset on the fly, when the user calls `ray.train.get_dataset_shard`. In the future, this can be extended to allow dynamic configuration of the dataset configuration at runtime, rather than having a static configuration defined in the `ray.train.DataConfig`. --------- Signed-off-by: Justin Yu --- doc/source/tune/examples/index.rst | 2 +- .../train/v2/_internal/callbacks/datasets.py | 183 +++++++++++++-- .../train/v2/_internal/execution/context.py | 40 +++- .../v2/_internal/execution/train_fn_utils.py | 8 +- .../execution/worker_group/worker.py | 8 +- .../train/v2/tests/test_data_integration.py | 218 +++++++++++++++++- 6 files changed, 422 insertions(+), 37 deletions(-) diff --git a/doc/source/tune/examples/index.rst b/doc/source/tune/examples/index.rst index 38c334ab4717..121e3e47d77e 100644 --- a/doc/source/tune/examples/index.rst +++ b/doc/source/tune/examples/index.rst @@ -6,7 +6,7 @@ Ray Tune Examples ================= .. tip:: - See :ref:`overview` to learn more about Tune features. + See :ref:`tune-main` to learn more about Tune features. Below are examples for using Ray Tune for a variety use cases and sorted by categories: diff --git a/python/ray/train/v2/_internal/callbacks/datasets.py b/python/ray/train/v2/_internal/callbacks/datasets.py index a51b633d457a..3b31dcb96644 100644 --- a/python/ray/train/v2/_internal/callbacks/datasets.py +++ b/python/ray/train/v2/_internal/callbacks/datasets.py @@ -1,20 +1,162 @@ +import asyncio import copy +from dataclasses import dataclass from typing import Any, Callable, Dict, List, Union import ray.train -from ray.data import Dataset +from ray.data import DataIterator, Dataset, NodeIdStr from ray.data.context import DataContext from ray.train.v2._internal.execution.callback import WorkerGroupCallback from ray.train.v2._internal.execution.worker_group.worker_group import ( Worker, WorkerGroup, ) +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy # A type representing either a ray.data.Dataset or a function that returns a # ray.data.Dataset and accepts no arguments. GenDataset = Union[Dataset, Callable[[], Dataset]] +@dataclass +class DatasetShardMetadata: + """Metadata about a dataset shard used for lookup and configuration.""" + + dataset_name: str + world_rank: int + + +class DatasetManager: + """Manages the dataset shards for datasets configured in the trainer.""" + + def __init__( + self, + datasets: Dict[str, GenDataset], + data_config: ray.train.DataConfig, + data_context: DataContext, + world_size: int, + worker_node_ids: List[NodeIdStr], + ): + self._datasets = {k: v() if callable(v) else v for k, v in datasets.items()} + self._data_config = data_config + self._datasets_to_split = ( + set(self._datasets.keys()) + if data_config._datasets_to_split == "all" + else set(data_config._datasets_to_split) + ) + self._world_size = world_size + self._worker_node_ids = worker_node_ids + + # Maps dataset name to a list of cached `DataIterator`s corresponding to + # Train worker ranks. + self._dataset_iterators: Dict[str, List[DataIterator]] = {} + + # A condition variable to synchronize the calls to the async `get_dataset_shard` method. + self._condition = asyncio.Condition() + + DataContext._set_current(data_context) + + def _create_dataset_iterators( + self, dataset_info: DatasetShardMetadata, base_dataset: Dataset + ) -> List[DataIterator]: + dataset_name = dataset_info.dataset_name + + iterators_per_rank = self._data_config.configure( + datasets={dataset_name: base_dataset}, + world_size=self._world_size, + worker_handles=None, + worker_node_ids=self._worker_node_ids, + ) + assert len(iterators_per_rank) == self._world_size + # TODO: Update DataConfig to return a List[DataIterator] directly + # for configuring a single dataset. + # Convert the List[Dict[str, DataIterator]] to a List[DataIterator], + # since we only configured one dataset. + return [iterators_per_rank[i][dataset_name] for i in range(self._world_size)] + + def _get_unsharded_dataset_iterator( + self, dataset_info: DatasetShardMetadata + ) -> DataIterator: + """Returns the dataset iterator for a dataset that is excluded + from `DataConfig.datasets_to_split`. + + Note that this method is NOT a barrier across workers and can be called + by any subset of workers and will return immediately. + """ + dataset_name = dataset_info.dataset_name + world_rank = dataset_info.world_rank + + if dataset_name not in self._dataset_iterators: + self._dataset_iterators[dataset_name] = self._create_dataset_iterators( + dataset_info, self._datasets[dataset_name] + ) + + return self._dataset_iterators[dataset_name][world_rank] + + async def _get_sharded_dataset_iterator( + self, dataset_info: DatasetShardMetadata + ) -> DataIterator: + """Returns the dataset iterator for a dataset that is included + in `DataConfig.datasets_to_split`. + + Note that this method is a barrier across workers, + and all workers must call this method before training. + """ + dataset_name = dataset_info.dataset_name + world_rank = dataset_info.world_rank + + async with self._condition: + if dataset_name in self._dataset_iterators: + # If the dataset iterators have already been created, return the + # existing one. + iterator = self._dataset_iterators[dataset_name][world_rank] + elif world_rank == 0: + # In this case, the dataset iterators have not been created yet. + # The dataset only needs to be configured once globally for all workers. + # Do it only when the rank 0 worker calls this method. + iterators = self._create_dataset_iterators( + dataset_info, self._datasets[dataset_name] + ) + iterator = iterators[world_rank] + + # Cache the dataset iterators for future use. + self._dataset_iterators[dataset_name] = iterators + self._condition.notify_all() + else: + # Wait for the dataset iterators to be created by the rank 0 worker. + await self._condition.wait_for( + lambda: dataset_name in self._dataset_iterators + ) + iterator = self._dataset_iterators[dataset_name][world_rank] + return iterator + + async def get_dataset_shard( + self, + dataset_info: DatasetShardMetadata, + ) -> DataIterator: + """Create and return the dataset shard iterator for a Ray Train worker's + call to `ray.train.get_dataset_shard`. + + This method is a barrier that should be called by all Ray Train workers at once. + If the dataset iterators have already been created, return the existing ones. + Otherwise, create the dataset iterators and cache them. + + Here's an example of how this method is used with 4 workers: + + Rank 2 calls get_dataset_shard, waits on the condition variable. + Rank 1 calls get_dataset_shard, waits on the condition variable. + Rank 0 calls get_dataset_shard, creates the dataset iterators, caches them, + and notifies all workers hanging on the condition variable. + Rank 3 calls get_dataset_shard, returns the cached iterator. + """ + dataset_name = dataset_info.dataset_name + + if dataset_name in self._datasets_to_split: + return await self._get_sharded_dataset_iterator(dataset_info) + else: + return self._get_unsharded_dataset_iterator(dataset_info) + + class DatasetsSetupCallback(WorkerGroupCallback): """The callback to setup Ray Datasets for the worker group.""" @@ -31,7 +173,7 @@ def __init__( # Capture the current DataContext to propagate it to # the Train workers later. # The propagation works in the following way: - # 1. This callback is created when user create the Trainer. + # 1. This callback is created when user creates the Trainer. # 2. Then this callback will be passed to the Controller actor. # 3. Lastly, when the worker group is initialized, the Controller # will call the `after_worker_group_start` callback to propagate @@ -45,26 +187,39 @@ def get_train_total_resources( these resources logically from its available pool.""" return scaling_config.total_resources + # -------------------------- + # WorkerGroupCallback + # -------------------------- + def before_init_train_context(self, workers: List[Worker]) -> Dict[str, List[Any]]: - # Configure dataset shards - datasets = {k: v() if callable(v) else v for k, v in self._datasets.items()} - node_ids = [worker.metadata.node_id for worker in workers] + if not self._datasets: + return {"dataset_manager": [None] * len(workers)} + + world_size = len(workers) + worker_node_ids = [worker.metadata.node_id for worker in workers] - # Notify the DataConfig about the total resources reserved for training. total_train_resources = self.get_train_total_resources(self._scaling_config) self._data_config.set_train_total_resources( total_train_resources.get("CPU", 0), total_train_resources.get("GPU", 0) ) - dataset_shards = self._data_config.configure( - datasets, - world_size=len(workers), - worker_handles=None, - worker_node_ids=node_ids, + dataset_manager = ( + ray.remote(DatasetManager) + .options( + num_cpus=0, + scheduling_strategy=NodeAffinitySchedulingStrategy( + ray.get_runtime_context().get_node_id(), soft=False + ), + ) + .remote( + datasets=self._datasets, + data_config=self._data_config, + data_context=self._data_context, + world_size=world_size, + worker_node_ids=worker_node_ids, + ) ) - assert len(dataset_shards) == len(workers) - - return {"dataset_shards": dataset_shards} + return {"dataset_manager": [dataset_manager] * len(workers)} def after_worker_group_start(self, worker_group: WorkerGroup): # Propagate DataContext diff --git a/python/ray/train/v2/_internal/execution/context.py b/python/ray/train/v2/_internal/execution/context.py index fd2f0df8e23e..ecb3ae3f9cd8 100644 --- a/python/ray/train/v2/_internal/execution/context.py +++ b/python/ray/train/v2/_internal/execution/context.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional import ray +from ray.actor import ActorHandle from ray.data import DataIterator, Dataset from ray.train import BackendConfig, Checkpoint, DataConfig from ray.train._internal import session @@ -17,6 +18,10 @@ from ray.train.v2.api.config import RunConfig, ScalingConfig if TYPE_CHECKING: + from ray.train.v2._internal.callbacks.datasets import ( + DatasetManager, + DatasetShardMetadata, + ) from ray.train.v2._internal.execution.callback import TrainContextCallback from ray.train.v2._internal.execution.worker_group.thread_runner import ThreadRunner @@ -92,9 +97,11 @@ class TrainContext: distributed_context: DistributedContext execution_context: ExecutionContext storage_context: StorageContext - dataset_shards: Dict[str, DataIterator] checkpoint: Optional[Checkpoint] = None + dataset_manager: Optional[ActorHandle["DatasetManager"]] = None + _cached_dataset_shards: Dict[str, DataIterator] = field(default_factory=dict) + @_copy_doc(session.get_experiment_name) def get_experiment_name(self) -> str: return self.train_run_context.run_config.name @@ -133,7 +140,7 @@ def get_synchronization_actor(self): def get_checkpoint(self): return self.checkpoint - def get_dataset_shard(self, dataset_name: str) -> DataIterator: + def get_dataset_shard(self, dataset_info: "DatasetShardMetadata") -> DataIterator: """Returns the :class:`ray.data.DataIterator` shard for this worker. Call :meth:`~ray.data.DataIterator.iter_torch_batches` or @@ -141,19 +148,34 @@ def get_dataset_shard(self, dataset_name: str) -> DataIterator: appropriate framework-specific data type. Args: - dataset_name: Name of the dataset shard. + dataset_info: The shard metadata, including the dataset name and worker rank. + Returns: The ``DataIterator`` shard with the given name for this worker. + Raises: KeyError: If the dataset shard with the given name is not found. """ + dataset_name = dataset_info.dataset_name + error = KeyError( + f"Dataset shard for '{dataset_name}' not found. " + "Please ensure that the dataset is passed through the Trainer `datasets` " + "argument." + ) + + if self.dataset_manager is None: + raise error + + if dataset_info.dataset_name in self._cached_dataset_shards: + return self._cached_dataset_shards[dataset_info.dataset_name] + try: - return self.dataset_shards[dataset_name] - except KeyError: - raise KeyError( - f"Dataset {dataset_name} not found. Available datasets: " - f"{list(self.dataset_shards.keys())}." - ) + shard = ray.get(self.dataset_manager.get_dataset_shard.remote(dataset_info)) + except KeyError as e: + raise error from e + + self._cached_dataset_shards[dataset_info.dataset_name] = shard + return shard def get_context_callbacks(self) -> List["TrainContextCallback"]: return self.execution_context.train_context_callbacks diff --git a/python/ray/train/v2/_internal/execution/train_fn_utils.py b/python/ray/train/v2/_internal/execution/train_fn_utils.py index 28bf683fda2d..b6b784de163e 100644 --- a/python/ray/train/v2/_internal/execution/train_fn_utils.py +++ b/python/ray/train/v2/_internal/execution/train_fn_utils.py @@ -57,7 +57,13 @@ def get_dataset_shard(self, dataset_name: str) -> DataIterator: Returns: The DataIterator shard for this worker. """ - return get_internal_train_context().get_dataset_shard(dataset_name) + from ray.train.v2._internal.callbacks.datasets import DatasetShardMetadata + + dataset_info = DatasetShardMetadata( + dataset_name=dataset_name, + world_rank=get_internal_train_context().get_world_rank(), + ) + return get_internal_train_context().get_dataset_shard(dataset_info) def get_context(self) -> ExternalTrainContext: return ExternalTrainContext() diff --git a/python/ray/train/v2/_internal/execution/worker_group/worker.py b/python/ray/train/v2/_internal/execution/worker_group/worker.py index 667ab318296b..b3fec846128d 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/worker.py +++ b/python/ray/train/v2/_internal/execution/worker_group/worker.py @@ -10,7 +10,6 @@ import ray._private.ray_constants as ray_constants from .thread_runner import ThreadRunner from ray.actor import ActorHandle -from ray.data.iterator import DataIterator from ray.train import Checkpoint from ray.train.v2._internal.constants import ( DEFAULT_ENABLE_WORKER_LOGGING, @@ -20,7 +19,6 @@ TrainContextCallback, WorkerCallback, ) -from ray.train.v2._internal.execution.checkpoint.sync_actor import SynchronizationActor from ray.train.v2._internal.execution.context import ( DistributedContext, ExecutionContext, @@ -189,10 +187,10 @@ def init_train_context( self, train_run_context: TrainRunContext, distributed_context: DistributedContext, - synchronization_actor: SynchronizationActor, + synchronization_actor: ActorHandle, storage_context: StorageContext, worker_callbacks: List[Union[WorkerCallback, TrainContextCallback]], - dataset_shards: Dict[str, DataIterator] = None, + dataset_manager: Optional[ActorHandle] = None, checkpoint: Optional[Checkpoint] = None, ): self._callbacks = [c for c in worker_callbacks if isinstance(c, WorkerCallback)] @@ -211,8 +209,8 @@ def init_train_context( train_context_callbacks=context_callbacks_to_propagate, ), storage_context=storage_context, - dataset_shards=dataset_shards or {}, checkpoint=checkpoint, + dataset_manager=dataset_manager, ) # Configure the train and root logger for the worker processes. if ray_constants.env_bool( diff --git a/python/ray/train/v2/tests/test_data_integration.py b/python/ray/train/v2/tests/test_data_integration.py index fe8159d5190f..ff5d5acf9b0f 100644 --- a/python/ray/train/v2/tests/test_data_integration.py +++ b/python/ray/train/v2/tests/test_data_integration.py @@ -1,3 +1,4 @@ +import asyncio from unittest.mock import MagicMock import pytest @@ -7,7 +8,11 @@ from ray.data import DataContext, ExecutionResources from ray.data._internal.iterator.stream_split_iterator import StreamSplitDataIterator from ray.data.tests.conftest import restore_data_context # noqa: F401 -from ray.train.v2._internal.callbacks import DatasetsSetupCallback +from ray.train.v2._internal.callbacks.datasets import ( + DatasetManager, + DatasetShardMetadata, + DatasetsSetupCallback, +) from ray.train.v2._internal.execution.context import TrainRunContext from ray.train.v2._internal.execution.worker_group.worker_group import ( WorkerGroupContext, @@ -87,13 +92,31 @@ def test_dataset_setup_callback(ray_start_4_cpus): data_config=data_config, scaling_config=scaling_config, ) - dataset_shards = callback.before_init_train_context(worker_group.get_workers())[ - "dataset_shards" - ] - assert len(dataset_shards) == NUM_WORKERS + dataset_manager_for_each_worker = callback.before_init_train_context( + worker_group.get_workers() + )["dataset_manager"] + assert len(dataset_manager_for_each_worker) == NUM_WORKERS + + # We should send the same dataset manager to all workers. + dataset_manager = dataset_manager_for_each_worker[0] + assert all( + manager == dataset_manager for manager in dataset_manager_for_each_worker + ) - processed_train_ds = dataset_shards[0]["train"] - processed_valid_ds = dataset_shards[0]["valid"] + def get_rank_0_shard(dataset_name: str): + for i in range(1, NUM_WORKERS): + dataset_manager.get_dataset_shard.remote( + DatasetShardMetadata(dataset_name=dataset_name, world_rank=i) + ) + + return ray.get( + dataset_manager.get_dataset_shard.remote( + DatasetShardMetadata(dataset_name=dataset_name, world_rank=0) + ) + ) + + processed_train_ds = get_rank_0_shard("train") + processed_valid_ds = get_rank_0_shard("valid") assert isinstance(processed_train_ds, StreamSplitDataIterator) assert not isinstance(processed_valid_ds, StreamSplitDataIterator) @@ -109,6 +132,187 @@ def test_dataset_setup_callback(ray_start_4_cpus): ) +async def get_dataset_shard_for_worker( + dataset_manager: DatasetManager, + dataset_name: str, + num_workers: int, + worker_rank: int, +): + return await asyncio.create_task( + dataset_manager.get_dataset_shard( + DatasetShardMetadata(dataset_name=dataset_name, world_rank=worker_rank) + ) + ) + + +async def get_dataset_shard_for_all_workers( + dataset_manager: DatasetManager, + dataset_name: str, + num_workers: int, +): + return await asyncio.gather( + *[ + get_dataset_shard_for_worker(dataset_manager, dataset_name, num_workers, i) + for i in range(num_workers) + ] + ) + + +@pytest.mark.asyncio +async def test_get_multiple_datasets_serially(ray_start_4_cpus): + """Tests DatasetManager.get_dataset_shard for multiple datasets, + called serially by each worker. This is the typical case. + + Workers 0, 1: + ray.train.get_dataset_shard("sharded_1") + ray.train.get_dataset_shard("sharded_2") + ray.train.get_dataset_shard("unsharded") + """ + + NUM_ROWS = 100 + NUM_TRAIN_WORKERS = 2 + + sharded_ds_1 = ray.data.range(NUM_ROWS) + sharded_ds_2 = ray.data.range(NUM_ROWS) + unsharded_ds = ray.data.range(NUM_ROWS) + + dataset_manager = DatasetManager( + datasets={ + "sharded_1": sharded_ds_1, + "sharded_2": sharded_ds_2, + "unsharded": unsharded_ds, + }, + data_config=ray.train.DataConfig(datasets_to_split=["sharded_1", "sharded_2"]), + data_context=DataContext.get_current(), + world_size=NUM_TRAIN_WORKERS, + worker_node_ids=None, + ) + + shards = await get_dataset_shard_for_all_workers( + dataset_manager, "sharded_1", NUM_TRAIN_WORKERS + ) + assert all(isinstance(shard, StreamSplitDataIterator) for shard in shards) + assert [shard._base_dataset.name for shard in shards] == [ + "sharded_1" + ] * NUM_TRAIN_WORKERS + + shards = await get_dataset_shard_for_all_workers( + dataset_manager, "sharded_2", NUM_TRAIN_WORKERS + ) + assert all(isinstance(shard, StreamSplitDataIterator) for shard in shards) + assert [shard._base_dataset.name for shard in shards] == [ + "sharded_2" + ] * NUM_TRAIN_WORKERS + + shards = await get_dataset_shard_for_all_workers( + dataset_manager, "unsharded", NUM_TRAIN_WORKERS + ) + assert not any(isinstance(shard, StreamSplitDataIterator) for shard in shards) + assert [shard._base_dataset.name for shard in shards] == [ + "unsharded" + ] * NUM_TRAIN_WORKERS + + +@pytest.mark.asyncio +async def test_get_multiple_datasets_interleaved(ray_start_4_cpus): + """Tests DatasetManager.get_dataset_shard for multiple datasets, + called in an interleaved order by workers. + + Worker 0: + ray.train.get_dataset_shard("train") + ray.train.get_dataset_shard("valid") + + Worker 1: + ray.train.get_dataset_shard("valid") + ray.train.get_dataset_shard("train") + """ + + NUM_ROWS = 100 + NUM_TRAIN_WORKERS = 2 + + train_ds = ray.data.range(NUM_ROWS) + valid_ds = ray.data.range(NUM_ROWS) + + dataset_manager = DatasetManager( + datasets={"train": train_ds, "valid": valid_ds}, + data_config=ray.train.DataConfig(datasets_to_split="all"), + data_context=DataContext.get_current(), + world_size=NUM_TRAIN_WORKERS, + worker_node_ids=None, + ) + + tasks = [ + get_dataset_shard_for_worker(dataset_manager, "train", NUM_TRAIN_WORKERS, 0), + get_dataset_shard_for_worker(dataset_manager, "valid", NUM_TRAIN_WORKERS, 1), + get_dataset_shard_for_worker(dataset_manager, "train", NUM_TRAIN_WORKERS, 1), + get_dataset_shard_for_worker(dataset_manager, "valid", NUM_TRAIN_WORKERS, 0), + ] + iterators = await asyncio.gather(*tasks) + assert all(isinstance(iterator, StreamSplitDataIterator) for iterator in iterators) + assert [iterator._base_dataset.name for iterator in iterators] == [ + "train", + "valid", + "train", + "valid", + ] + + +@pytest.mark.asyncio +async def test_get_multiple_datasets_rank_specific(ray_start_4_cpus): + """Tests rank-specific DatasetManager.get_dataset_shard calls. + + # Epoch 1 + ray.train.get_dataset_shard("train") + + # Validation, which only happens on worker 0. + if world_rank == 0: + ray.train.get_dataset_shard("valid") + + # Epoch 2 + ray.train.get_dataset_shard("train") + """ + + NUM_ROWS = 100 + NUM_TRAIN_WORKERS = 2 + + train_ds = ray.data.range(NUM_ROWS) + valid_ds = ray.data.range(NUM_ROWS) + + dataset_manager = DatasetManager( + datasets={"train": train_ds, "valid": valid_ds}, + data_config=ray.train.DataConfig(datasets_to_split=["train"]), + data_context=DataContext.get_current(), + world_size=NUM_TRAIN_WORKERS, + worker_node_ids=None, + ) + + # ray.train.get_dataset_shard("train") + iterators = await get_dataset_shard_for_all_workers( + dataset_manager, "train", NUM_TRAIN_WORKERS + ) + assert all(isinstance(iterator, StreamSplitDataIterator) for iterator in iterators) + assert [iterator._base_dataset.name for iterator in iterators] == [ + "train" + ] * NUM_TRAIN_WORKERS + + # if world_rank == 0: + # ray.train.get_dataset_shard("valid") + iterator = await get_dataset_shard_for_worker( + dataset_manager, "valid", NUM_TRAIN_WORKERS, 0 + ) + assert not isinstance(iterator, StreamSplitDataIterator) + assert iterator._base_dataset.name == "valid" + + # ray.train.get_dataset_shard("train") + iterators = await get_dataset_shard_for_all_workers( + dataset_manager, "train", NUM_TRAIN_WORKERS + ) + assert all(isinstance(iterator, StreamSplitDataIterator) for iterator in iterators) + assert [iterator._base_dataset.name for iterator in iterators] == [ + "train" + ] * NUM_TRAIN_WORKERS + + if __name__ == "__main__": import sys From 026723186a805e4a08b409e761fee85028e7cd9e Mon Sep 17 00:00:00 2001 From: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Date: Mon, 11 Aug 2025 17:06:52 -0700 Subject: [PATCH 036/634] [Serve.llm] Support Data Parallel Attention Deployment (#55438) Signed-off-by: Rui Qiao --- .../serve/builders/application_builders.py | 2 +- .../data_parallel/dp_rank_assigner.py | 30 +++++++ .../deployments/data_parallel/dp_server.py | 82 +++++++++++++++++++ .../serve/deployments/llm/llm_server.py | 4 +- .../serve/deployments/llm/vllm/vllm_engine.py | 2 +- 5 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py create mode 100644 python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py diff --git a/python/ray/llm/_internal/serve/builders/application_builders.py b/python/ray/llm/_internal/serve/builders/application_builders.py index a0f7607e14fa..201974b36514 100644 --- a/python/ray/llm/_internal/serve/builders/application_builders.py +++ b/python/ray/llm/_internal/serve/builders/application_builders.py @@ -22,7 +22,7 @@ def build_llm_deployment( name_prefix: Optional[str] = None, deployment_kwargs: Optional[dict] = None, ) -> Application: - name_prefix = name_prefix or "LLMDeployment" + name_prefix = name_prefix or "LLMDeployment:" deployment_kwargs = deployment_kwargs or {} deployment_options = llm_config.get_serve_options( diff --git a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py new file mode 100644 index 000000000000..b9b23065609e --- /dev/null +++ b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py @@ -0,0 +1,30 @@ +import asyncio + +from ray import serve + + +@serve.deployment(num_replicas=1) +class DPRankAssigner: + """ + Data Parallel Rank Assigner. + + This class is used to assign a rank to each replica in the data parallel + deployment. + """ + + def __init__(self, dp_size: int): + self.dp_size = dp_size + self.lock = asyncio.Lock() + self.next_rank = 0 + + async def register(self, replica_ctx: "serve.context.ReplicaContext"): + async with self.lock: + if self.next_rank >= self.dp_size: + raise ValueError( + f"Attempted to assign rank {self.next_rank} but dp_size is {self.dp_size}" + ) + # TODO(rui): instead of using the naive increment approach, + # we should use the Ray Serve Replica Rank API to assign ranks. + rank = self.next_rank + self.next_rank += 1 + return rank diff --git a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py new file mode 100644 index 000000000000..0c3cce80e090 --- /dev/null +++ b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py @@ -0,0 +1,82 @@ +import logging +from typing import Optional + +from ray import serve +from ray.llm._internal.serve.configs.server_models import LLMConfig +from ray.llm._internal.serve.deployments.data_parallel.dp_rank_assigner import ( + DPRankAssigner, +) +from ray.llm._internal.serve.deployments.llm.llm_server import LLMServer +from ray.serve.deployment import Application +from ray.serve.handle import DeploymentHandle + +logger = logging.getLogger(__name__) + + +class DPServer(LLMServer): + """ + Data Parallel LLM Server. + + This class is used to serve data parallel attention (DP Attention) + deployment paradigm, where the attention layers are replicated and + the MoE layers are sharded. DP Attention is typically used for models + like DeepSeek-V3. + """ + + async def __init__(self, llm_config: LLMConfig, dp_rank_assigner: DeploymentHandle): + self.dp_rank_assigner = dp_rank_assigner + + replica_ctx = serve.get_replica_context() + self.dp_rank = await self.dp_rank_assigner.register.remote(replica_ctx) + logger.info(f"DP rank: {self.dp_rank}") + + # override the engine_kwargs to assign the DP rank. + llm_config.engine_kwargs["data_parallel_rank"] = self.dp_rank + + await super().__init__(llm_config) + + def _push_telemetry_report(self): + # Only push telemetry report for the first DP replica. + if self.dp_rank == 0: + # TODO(rui): refine the telemetry report for DP deployment. + super()._push_telemetry_report() + + @classmethod + def as_deployment(cls, deployment_options: dict) -> serve.Deployment: + return serve.deployment(cls).options(**deployment_options) + + +def build_dp_deployment( + llm_config: LLMConfig, + *, + name_prefix: Optional[str] = None, +) -> Application: + """Build a data parallel LLM deployment.""" + dp_size = llm_config.engine_kwargs.get("data_parallel_size", 1) + if dp_size == 1: + raise ValueError( + "data_parallel_size should be greater than 1 for DP deployment." + ) + dp_rank_assigner = DPRankAssigner.bind(dp_size=dp_size) + name_prefix = name_prefix or "DPLLMDeployment:" + name = name_prefix + llm_config._get_deployment_name() + if "num_replicas" in llm_config.deployment_config: + raise ValueError( + "num_replicas should not be specified for DP deployment, " + "use engine_kwargs.data_parallel_size instead." + ) + if "autoscaling_config" in llm_config.deployment_config: + raise ValueError( + "autoscaling_config is not supported for DP deployment, " + "use engine_kwargs.data_parallel_size to set a fixed number " + "of replicas instead." + ) + # TODO(rui): support data_parallel_backend=ray and unify + # deployment_options handling with LLMDeployment. + deployment_options = { + "name": name, + "num_replicas": dp_size, + } + return DPServer.as_deployment(deployment_options).bind( + llm_config=llm_config, dp_rank_assigner=dp_rank_assigner + ) diff --git a/python/ray/llm/_internal/serve/deployments/llm/llm_server.py b/python/ray/llm/_internal/serve/deployments/llm/llm_server.py index 551329753526..da20c9b805a8 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/llm_server.py +++ b/python/ray/llm/_internal/serve/deployments/llm/llm_server.py @@ -204,6 +204,7 @@ async def start(self): if self._engine_cls is not None: self.engine = self._engine_cls(self._llm_config) await asyncio.wait_for(self._start_engine(), timeout=ENGINE_START_TIMEOUT_S) + self._push_telemetry_report() def _init_multiplex_loader( self, model_downloader_cls: Optional[Type[LoraModelLoader]] = None @@ -251,7 +252,8 @@ async def _start_engine(self): await self.engine.start() - # Push telemetry reports for the model in the current deployment. + def _push_telemetry_report(self): + """Push telemetry reports for the model in the current deployment.""" push_telemetry_report_for_all_models(all_models=[self._llm_config]) def _get_batch_interval_ms(self, stream: bool = True) -> int: diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py index b55c27120179..e3ba947266ad 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py @@ -328,7 +328,7 @@ def _start_async_llm_engine( """Creates an async LLM engine from the engine arguments.""" from vllm import envs as vllm_envs - # NOTE: This is a temporary solution untill vLLM v1 supports embeddings. + # NOTE: This is a temporary solution until vLLM v1 supports embeddings. if not vllm_envs.VLLM_USE_V1: return self._start_async_llm_engine_v0( vllm_engine_args, vllm_engine_config, placement_group From f4feaa34791b13f33da4aad7ef810fce4785d2f6 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 11 Aug 2025 17:20:11 -0700 Subject: [PATCH 037/634] [image] remove extra image build tags (#55494) rayci has proper dependency tracking now, and does not require explicit tagging for downstream deps Signed-off-by: Lonnie Liu --- .buildkite/_forge.rayci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml index fcc4a3e770d9..662f7824178a 100644 --- a/.buildkite/_forge.rayci.yml +++ b/.buildkite/_forge.rayci.yml @@ -32,15 +32,11 @@ steps: PYTHON_VERSION: "{{matrix.python}}" CUDA_VERSION: "{{matrix.cuda}}" - - name: raycpubase label: "wanda: ray.py{{matrix}}.cpu.base" tags: - python_dependencies - - python - docker - - tune - - serve wanda: ci/docker/ray.cpu.base.wanda.yaml matrix: - "3.9" From 4dcab96e0d380de086052efdbfec8d5abc85192f Mon Sep 17 00:00:00 2001 From: Alexey Kudinkin Date: Mon, 11 Aug 2025 20:41:11 -0400 Subject: [PATCH 038/634] [Data] Restore handling of `PyExtensionType` to maintain compatibility w/ previously written datasets (#55498) ## Why are these changes needed? Batch inference tests started to fail after https://github.com/ray-project/ray/pull/55426 due to us eliminating handling of the auto-loading of `PyExtensionType` This change restores handling of the auto-loading while also fixing it for Arrow >= 21.0 that deleted PyExtensionType support completely. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Alexey Kudinkin --- python/ray/data/__init__.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/python/ray/data/__init__.py b/python/ray/data/__init__.py index c4962fd7db54..96a774f5a63f 100644 --- a/python/ray/data/__init__.py +++ b/python/ray/data/__init__.py @@ -1,6 +1,9 @@ # Short term workaround for https://github.com/ray-project/ray/issues/32435 # Dataset has a hard dependency on pandas, so it doesn't need to be delayed. import pandas # noqa +from packaging.version import parse as parse_version + +from ray._private.arrow_utils import get_pyarrow_version from ray.data._internal.compute import ActorPoolStrategy from ray.data._internal.datasource.tfrecords_datasource import TFXReadOptions @@ -75,6 +78,40 @@ configure_logging() +try: + import pyarrow as pa + + # Import these arrow extension types to ensure that they are registered. + from ray.air.util.tensor_extensions.arrow import ( # noqa + ArrowTensorType, + ArrowVariableShapedTensorType, + ) + + # https://github.com/apache/arrow/pull/38608 deprecated `PyExtensionType`, and + # disabled it's deserialization by default. To ensure that users can load data + # written with earlier version of Ray Data, we enable auto-loading of serialized + # tensor extensions. + # + # NOTE: `PyExtensionType` is deleted from Arrow >= 21.0 + pyarrow_version = get_pyarrow_version() + if pyarrow_version is None or pyarrow_version >= parse_version("21.0.0"): + pass + else: + from ray._private.ray_constants import env_bool + + RAY_DATA_AUTOLOAD_PYEXTENSIONTYPE = env_bool( + "RAY_DATA_AUTOLOAD_PYEXTENSIONTYPE", False + ) + + if ( + pyarrow_version >= parse_version("14.0.1") + and RAY_DATA_AUTOLOAD_PYEXTENSIONTYPE + ): + pa.PyExtensionType.set_auto_load(True) + +except ModuleNotFoundError: + pass + __all__ = [ "ActorPoolStrategy", From 3a8e4239eb2df8a429ac84e69cfdb499091e54ad Mon Sep 17 00:00:00 2001 From: Nary Yeh <60069744+machichima@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:43:02 +0800 Subject: [PATCH 039/634] [Core]Separate Environment Variables for ray.init() and ray ctl to Reflect Different Protocols and Ports (#55189) Currently, both `ray.init()` and ray ctl rely on the same environment variable RAY_ADDRESS to determine the default address to connect to a Ray cluster. However, in practice: `ray.init()` uses the `ray:// protocol` and connects via the GCS gRPC port (default `10001`) ray ctl uses the HTTP protocol (`http://`) and connects via the Dashboard port (default `8265`) This leads to potential confusion when setting the `RAY_ADDRESS` environment variable, as it may not be valid for both tools simultaneously. For example, setting `RAY_ADDRESS=http://localhost:8265/` would work for ray ctl, but not for `ray.init()`, which expects a `ray://` URI with the GCS port. In this PR, we do: - Keep `RAY_ADDRESS` for `ray.init()` - Use `RAY_API_SERVER_ADDRESS` for ray job ctl - Update docs: https://anyscale-ray--55189.com.readthedocs.build/en/55189/cluster/running-applications/job-submission/quickstart.html Closes #53226 --- .../job-submission/quickstart.rst | 8 ++--- python/ray/_private/ray_constants.py | 1 + python/ray/dashboard/modules/job/cli.py | 12 +++---- python/ray/dashboard/modules/job/sdk.py | 2 +- .../modules/job/tests/test_cli_integration.py | 33 +++++++++++++++++-- python/ray/dashboard/utils.py | 12 +++++-- python/ray/scripts/scripts.py | 2 +- .../test_cli_patterns/test_ray_start.txt | 2 +- .../test_ray_start_windows_osx.txt | 2 +- 9 files changed, 54 insertions(+), 20 deletions(-) diff --git a/doc/source/cluster/running-applications/job-submission/quickstart.rst b/doc/source/cluster/running-applications/job-submission/quickstart.rst index c40e344e3b38..3a9efd1043ca 100644 --- a/doc/source/cluster/running-applications/job-submission/quickstart.rst +++ b/doc/source/cluster/running-applications/job-submission/quickstart.rst @@ -68,13 +68,13 @@ If you are using a local Ray Cluster (``ray start --head``), connect directly at If you are using a Ray Cluster started on VMs or Kubernetes, follow the instructions there for setting up network access from a client. See :ref:`Using a Remote Cluster ` for tips. -To tell the Ray Jobs CLI how to find your Ray Cluster, pass the Ray Dashboard address. Set the ``RAY_ADDRESS`` environment variable: +To tell the Ray Jobs CLI how to find your Ray Cluster, pass the Ray Dashboard address. Set the ``RAY_API_SERVER_ADDRESS`` environment variable: .. code-block:: bash - $ export RAY_ADDRESS="http://127.0.0.1:8265" + $ export RAY_API_SERVER_ADDRESS="http://127.0.0.1:8265" -Alternatively, you can also pass the ``--address=http://127.0.0.1:8265`` flag explicitly to each Ray Jobs CLI command, or prepend each command with ``RAY_ADDRESS=http://127.0.0.1:8265``. +Alternatively, you can also pass the ``--address=http://127.0.0.1:8265`` flag explicitly to each Ray Jobs CLI command, or prepend each command with ``RAY_API_SERVER_ADDRESS=http://127.0.0.1:8265``. Additionally, if you wish to pass headers per HTTP request to the Cluster, use the `RAY_JOB_HEADERS` environment variable. This environment variable must be in JSON form. @@ -217,7 +217,7 @@ Run the following command on your local machine, where ``cluster.yaml`` is the c ray dashboard cluster.yaml Once this command is running, verify that you can view the Ray Dashboard in your local browser at ``http://127.0.0.1:8265``. -Also, verify that you set the environment variable ``RAY_ADDRESS`` to ``"http://127.0.0.1:8265"``. After this setup, you can use the Jobs CLI on the local machine as in the preceding example to interact with the remote Ray cluster. +Also, verify that you set the environment variable ``RAY_API_SERVER_ADDRESS`` to ``"http://127.0.0.1:8265"``. After this setup, you can use the Jobs CLI on the local machine as in the preceding example to interact with the remote Ray cluster. Using the CLI on Kubernetes ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/python/ray/_private/ray_constants.py b/python/ray/_private/ray_constants.py index 35b8ea4e43ac..bad2a4585938 100644 --- a/python/ray/_private/ray_constants.py +++ b/python/ray/_private/ray_constants.py @@ -126,6 +126,7 @@ def env_set_by_user(key): DEFAULT_PORT = 6379 RAY_ADDRESS_ENVIRONMENT_VARIABLE = "RAY_ADDRESS" +RAY_API_SERVER_ADDRESS_ENVIRONMENT_VARIABLE = "RAY_API_SERVER_ADDRESS" RAY_NAMESPACE_ENVIRONMENT_VARIABLE = "RAY_NAMESPACE" RAY_RUNTIME_ENV_ENVIRONMENT_VARIABLE = "RAY_RUNTIME_ENV" RAY_RUNTIME_ENV_URI_PIN_EXPIRATION_S_ENV_VAR = ( diff --git a/python/ray/dashboard/modules/job/cli.py b/python/ray/dashboard/modules/job/cli.py index c90c30c30789..37e93c6cc50b 100644 --- a/python/ray/dashboard/modules/job/cli.py +++ b/python/ray/dashboard/modules/job/cli.py @@ -115,7 +115,7 @@ def job_cli_group(): required=False, help=( "Address of the Ray cluster to connect to. Can also be specified " - "using the RAY_ADDRESS environment variable." + "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)." ), ) @click.option( @@ -333,7 +333,7 @@ def submit( required=False, help=( "Address of the Ray cluster to connect to. Can also be specified " - "using the `RAY_ADDRESS` environment variable." + "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)." ), ) @click.argument("job-id", type=str) @@ -363,7 +363,7 @@ def status( required=False, help=( "Address of the Ray cluster to connect to. Can also be specified " - "using the `RAY_ADDRESS` environment variable." + "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)." ), ) @click.option( @@ -418,7 +418,7 @@ def stop( required=False, help=( "Address of the Ray cluster to connect to. Can also be specified " - "using the RAY_ADDRESS environment variable." + "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)." ), ) @click.argument("job-id", type=str) @@ -455,7 +455,7 @@ def delete( required=False, help=( "Address of the Ray cluster to connect to. Can also be specified " - "using the RAY_ADDRESS environment variable." + "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)." ), ) @click.argument("job-id", type=str) @@ -508,7 +508,7 @@ def logs( required=False, help=( "Address of the Ray cluster to connect to. Can also be specified " - "using the RAY_ADDRESS environment variable." + "using the RAY_API_SERVER_ADDRESS environment variable (falls back to RAY_ADDRESS)." ), ) @add_common_job_options diff --git a/python/ray/dashboard/modules/job/sdk.py b/python/ray/dashboard/modules/job/sdk.py index f8442e09dbc8..e01c880a0e87 100644 --- a/python/ray/dashboard/modules/job/sdk.py +++ b/python/ray/dashboard/modules/job/sdk.py @@ -46,7 +46,7 @@ class JobSubmissionClient(SubmissionClient): ray.init(), e.g. a Ray Client address (ray://:10001), or "auto", or "localhost:". If unspecified, will try to connect to a running local Ray cluster. This argument is always overridden by the - RAY_ADDRESS environment variable. + RAY_API_SERVER_ADDRESS or RAY_ADDRESS environment variable. create_cluster_if_needed: Indicates whether the cluster at the specified address needs to already be running. Ray doesn't start a cluster before interacting with jobs, but third-party job managers may do so. diff --git a/python/ray/dashboard/modules/job/tests/test_cli_integration.py b/python/ray/dashboard/modules/job/tests/test_cli_integration.py index 883671af0415..872a1c823d57 100644 --- a/python/ray/dashboard/modules/job/tests/test_cli_integration.py +++ b/python/ray/dashboard/modules/job/tests/test_cli_integration.py @@ -142,11 +142,38 @@ def test_empty_ray_address(self, ray_start_stop): assert "succeeded" in stdout @pytest.mark.parametrize( - "ray_client_address", ["127.0.0.1:8265", "ray://127.0.0.1:8265"] + "ray_api_server_address,should_fail", + [ + ("http://127.0.0.1:8265", False), # correct API server + ("127.0.0.1:8265", True), # wrong format without http + ("http://127.0.0.1:9999", True), # wrong port + ], ) - def test_ray_client_address(self, ray_start_stop, ray_client_address: str): + def test_ray_api_server_address( + self, + ray_start_stop, + ray_api_server_address: str, + should_fail: bool, + ): + # Set a `RAY_ADDRESS` that would not work with the `ray job submit` CLI because it uses the `ray://` prefix. + # This verifies that the `RAY_API_SERVER_ADDRESS` env var takes precedence. + with set_env_var("RAY_ADDRESS", "ray://127.0.0.1:8265"): + with set_env_var("RAY_API_SERVER_ADDRESS", ray_api_server_address): + _run_cmd("ray job submit -- echo hello", should_fail=should_fail) + + @pytest.mark.parametrize( + "ray_client_address,should_fail", + [ + ("127.0.0.1:8265", True), + ("ray://127.0.0.1:8265", True), + ("http://127.0.0.1:8265", False), + ], + ) + def test_ray_client_address( + self, ray_start_stop, ray_client_address: str, should_fail: bool + ): with set_env_var("RAY_ADDRESS", ray_client_address): - _run_cmd("ray job submit -- echo hello", should_fail=True) + _run_cmd("ray job submit -- echo hello", should_fail=should_fail) def test_valid_http_ray_address(self, ray_start_stop): stdout, _ = _run_cmd("ray job submit -- echo hello") diff --git a/python/ray/dashboard/utils.py b/python/ray/dashboard/utils.py index 47c6340133d4..237992b0d2f0 100644 --- a/python/ray/dashboard/utils.py +++ b/python/ray/dashboard/utils.py @@ -709,9 +709,15 @@ def get_address_for_submission_client(address: Optional[str]) -> str: Returns: API server HTTP URL, e.g. "http://:8265". """ - if os.environ.get("RAY_ADDRESS"): - logger.debug(f"Using RAY_ADDRESS={os.environ['RAY_ADDRESS']}") - address = os.environ["RAY_ADDRESS"] + if api_server_address := os.environ.get( + ray_constants.RAY_API_SERVER_ADDRESS_ENVIRONMENT_VARIABLE + ): + address = api_server_address + logger.debug(f"Using RAY_API_SERVER_ADDRESS={address}") + # Fall back to RAY_ADDRESS if RAY_API_SERVER_ADDRESS not set + elif ray_address := os.environ.get(ray_constants.RAY_ADDRESS_ENVIRONMENT_VARIABLE): + address = ray_address + logger.debug(f"Using RAY_ADDRESS={address}") if address and "://" in address: module_string, _ = split_address(address) diff --git a/python/ray/scripts/scripts.py b/python/ray/scripts/scripts.py index 1e28c74efd0b..9818eb308604 100644 --- a/python/ray/scripts/scripts.py +++ b/python/ray/scripts/scripts.py @@ -983,7 +983,7 @@ def start( cli_logger.print("To submit a Ray job using the Ray Jobs CLI:") cli_logger.print( cf.bold( - " RAY_ADDRESS='http://{}' ray job submit " + " RAY_API_SERVER_ADDRESS='http://{}' ray job submit " "--working-dir . " "-- python my_script.py" ), diff --git a/python/ray/tests/test_cli_patterns/test_ray_start.txt b/python/ray/tests/test_cli_patterns/test_ray_start.txt index 55d250d62f21..6d6df437a1ca 100644 --- a/python/ray/tests/test_cli_patterns/test_ray_start.txt +++ b/python/ray/tests/test_cli_patterns/test_ray_start.txt @@ -14,7 +14,7 @@ Next steps ray\.init\(\) To submit a Ray job using the Ray Jobs CLI: - RAY_ADDRESS='http://.+:8265' ray job submit --working-dir \. -- python my_script\.py + RAY_API_SERVER_ADDRESS='http://.+:8265' ray job submit --working-dir \. -- python my_script\.py See https://docs\.ray\.io/en/latest/cluster/running-applications/job-submission/index\.html for more information on submitting Ray jobs to the Ray cluster. diff --git a/python/ray/tests/test_cli_patterns/test_ray_start_windows_osx.txt b/python/ray/tests/test_cli_patterns/test_ray_start_windows_osx.txt index b6ea1348f10f..b11b51a275e0 100644 --- a/python/ray/tests/test_cli_patterns/test_ray_start_windows_osx.txt +++ b/python/ray/tests/test_cli_patterns/test_ray_start_windows_osx.txt @@ -15,7 +15,7 @@ Next steps ray\.init\(\) To submit a Ray job using the Ray Jobs CLI: - RAY_ADDRESS='http://.+:8265' ray job submit --working-dir \. -- python my_script\.py + RAY_API_SERVER_ADDRESS='http://.+:8265' ray job submit --working-dir \. -- python my_script\.py See https://docs\.ray\.io/en/latest/cluster/running-applications/job-submission/index\.html for more information on submitting Ray jobs to the Ray cluster. From ecf81072d8abbac1b2f3c65a6d6a0de4f612c693 Mon Sep 17 00:00:00 2001 From: Alexey Kudinkin Date: Mon, 11 Aug 2025 20:49:30 -0400 Subject: [PATCH 040/634] [Data] Cleaning up `StatsManager` (#55400) ## Why are these changes needed? Just cleaning up, merging 2 overlapping metrics ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Alexey Kudinkin --- python/ray/data/_internal/stats.py | 53 ++++++++++++++---------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/python/ray/data/_internal/stats.py b/python/ray/data/_internal/stats.py index 1b129f81c5c0..869aa49b84ca 100644 --- a/python/ray/data/_internal/stats.py +++ b/python/ray/data/_internal/stats.py @@ -633,7 +633,9 @@ def __init__(self): self._update_thread: Optional[threading.Thread] = None self._update_thread_lock: threading.Lock = threading.Lock() - def _get_stats_actor(self, skip_cache: bool = False) -> Optional[ActorHandle]: + def _get_or_create_stats_actor( + self, skip_cache: bool = False + ) -> Optional[ActorHandle]: if ray._private.worker._global_node is None: raise RuntimeError( "Global node is not initialized. Driver might be not connected to Ray." @@ -650,27 +652,13 @@ def _get_stats_actor(self, skip_cache: bool = False) -> Optional[ActorHandle]: self._stats_actor_handle = ray.get_actor( name=STATS_ACTOR_NAME, namespace=STATS_ACTOR_NAMESPACE ) + self._stats_actor_cluster_id = current_cluster_id except ValueError: - return None - self._stats_actor_cluster_id = current_cluster_id - - return self._stats_actor_handle - - def _get_or_create_stats_actor(self) -> Optional[ActorHandle]: - if ray._private.worker._global_node is None: - raise RuntimeError( - "Global node is not initialized. Driver might be not connected to Ray." - ) - - # NOTE: In some cases (for ex, when registering dataset) actor might be gone - # (for ex, when prior driver disconnects) and therefore to avoid using - # stale handle we force looking up the actor with Ray to determine if - # we should create a new one. - actor = self._get_stats_actor(skip_cache=True) - - if actor is None: - self._stats_actor_handle = _get_or_create_stats_actor() - self._stats_actor_cluster_id = ray._private.worker._global_node.cluster_id + # Create an actor if it doesn't exist + self._stats_actor_handle = _get_or_create_stats_actor() + self._stats_actor_cluster_id = ( + ray._private.worker._global_node.cluster_id + ) return self._stats_actor_handle @@ -684,11 +672,7 @@ def _run_update_loop(): while True: if self._last_iteration_stats or self._last_execution_stats: try: - # Do not create _StatsActor if it doesn't exist because - # this thread can be running even after the cluster is - # shutdown. Creating an actor will automatically start - # a new cluster. - stats_actor = self._get_stats_actor() + stats_actor = self._get_or_create_stats_actor() if stats_actor is None: continue stats_actor.update_metrics.remote( @@ -806,7 +790,14 @@ def register_dataset_to_stats_actor( topology: Optional Topology representing the DAG structure to export data_context: The DataContext attached to the dataset """ - self._get_or_create_stats_actor().register_dataset.remote( + + # NOTE: In some cases (for ex, when registering dataset) actor might be gone + # (for ex, when prior driver disconnects) and therefore to avoid using + # stale handle we force looking up the actor with Ray to determine if + # we should create a new one. + stats_actor = self._get_or_create_stats_actor(skip_cache=True) + + stats_actor.register_dataset.remote( ray.get_runtime_context().get_job_id(), dataset_tag, operator_tags, @@ -816,7 +807,13 @@ def register_dataset_to_stats_actor( def get_dataset_id_from_stats_actor(self) -> str: try: - return ray.get(self._get_or_create_stats_actor().get_dataset_id.remote()) + # NOTE: In some cases (for ex, when registering dataset) actor might be gone + # (for ex, when prior driver disconnects) and therefore to avoid using + # stale handle we force looking up the actor with Ray to determine if + # we should create a new one. + stats_actor = self._get_or_create_stats_actor(skip_cache=True) + + return ray.get(stats_actor.get_dataset_id.remote()) except Exception: # Getting dataset id from _StatsActor may fail, in this case # fall back to uuid4 From de4412af12122fc7c76a1102893f0cbce2495178 Mon Sep 17 00:00:00 2001 From: lkchen Date: Mon, 11 Aug 2025 18:25:03 -0700 Subject: [PATCH 041/634] [Serve.llm] Adapt vLLM change, use `enable_log_requests` if `disable_log_requests` is missing (#55336) Signed-off-by: Linkun --- .../serve/deployments/llm/vllm/vllm_models.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py index a2b0ec2b8d92..12b7635243ad 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py @@ -92,9 +92,24 @@ def get_initialization_kwargs(self) -> dict: else: engine_kwargs["distributed_executor_backend"] = "ray" - if "disable_log_requests" not in engine_kwargs: + # TODO(lk-chen): Remove the logic once we require vllm>=0.10.1 + # vLLM 0.10.1 replaces `disable_log_requests` with + # `enable_log_requests`. Here we are trying to be compatible with both. + if hasattr(AsyncEngineArgs, "enable_log_requests"): + if "disable_log_requests" in engine_kwargs: + logger.warning( + "disable_log_requests is set in engine_kwargs, but vLLM " + "does not support it. Converting to enable_log_requests." + ) + engine_kwargs["enable_log_requests"] = not engine_kwargs.pop( + "disable_log_requests" + ) + else: + engine_kwargs["enable_log_requests"] = False + elif "disable_log_requests" not in engine_kwargs: logger.info( - "Disabling request logging by default. To enable, set to False in engine_kwargs." + "Disabling request logging by default. To enable, set to False" + " in engine_kwargs." ) engine_kwargs["disable_log_requests"] = True From 5f624859ab51c5d6759d8d4f3ab503ecfd0bc07c Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 11 Aug 2025 19:01:49 -0700 Subject: [PATCH 042/634] [image] check gpu image with cuda directory (#55497) instead of checking on base image. also removes g++ for arm cpu images; arm does not need g++ for future package building any more. Signed-off-by: Lonnie Liu --- docker/base-deps/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index 94259767c134..4d183a6ba892 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -5,8 +5,6 @@ # The GPU options are NVIDIA CUDA developer images. ARG BASE_IMAGE="ubuntu:22.04" FROM ${BASE_IMAGE} -# FROM directive resets ARG -ARG BASE_IMAGE # If this arg is not "autoscaler" then no autoscaler requirements will be included ENV TZ=America/Los_Angeles ENV LC_ALL=C.UTF-8 @@ -122,7 +120,7 @@ uv pip uninstall --system dask sudo apt-get autoremove -y cmake zlib1g-dev # We keep g++ on GPU images, because uninstalling removes CUDA Devel tooling -if [[ "$BASE_IMAGE" == "ubuntu:22.04" && "$HOSTTYPE" == "x86_64" ]]; then +if [[ ! -d /usr/local/cuda ]]; then sudo apt-get autoremove -y g++ fi From 8f38de4192a922aa8696ac80ab96034385f898ab Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Mon, 11 Aug 2025 21:24:36 -0700 Subject: [PATCH 043/634] Add `orjson` as a dependency for release test images (#55313) ## Why are these changes needed? This PR adds `orjson` to our release test images because it's a common package that will be used in testing. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Balaji Veeramani --- release/ray_release/byod/requirements_byod_3.9.in | 1 + 1 file changed, 1 insertion(+) diff --git a/release/ray_release/byod/requirements_byod_3.9.in b/release/ray_release/byod/requirements_byod_3.9.in index 3f45139e034c..a4ce891981f8 100644 --- a/release/ray_release/byod/requirements_byod_3.9.in +++ b/release/ray_release/byod/requirements_byod_3.9.in @@ -39,3 +39,4 @@ xarray xgboost zarr pyyaml +orjson From 5696f6a0460868d5ea59b49e6474e4be7c473387 Mon Sep 17 00:00:00 2001 From: Sampan S Nayak Date: Tue, 12 Aug 2025 11:15:17 +0530 Subject: [PATCH 044/634] [core] deflake test_actors::test_kill_actor_gcs (#55479) Signed-off-by: sampan Co-authored-by: sampan --- .../modules/reporter/tests/test_actors.py | 37 ++++++++----------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/python/ray/dashboard/modules/reporter/tests/test_actors.py b/python/ray/dashboard/modules/reporter/tests/test_actors.py index 47eea650b2c0..763c66d866b4 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_actors.py +++ b/python/ray/dashboard/modules/reporter/tests/test_actors.py @@ -3,12 +3,15 @@ import sys import time +import psutil import pytest import requests import ray from ray._private.test_utils import format_web_url, wait_until_server_available from ray.dashboard.tests.conftest import * # noqa +from ray._common.test_utils import wait_for_condition +from ray._private.state_api_test_utils import _is_actor_task_running logger = logging.getLogger(__name__) @@ -16,23 +19,8 @@ def _actor_killed(pid: str) -> bool: - """Check For the existence of a unix pid.""" - try: - os.kill(pid, 0) - except OSError: - return True - else: - return False - - -def _actor_killed_loop(worker_pid: str, timeout_secs=3) -> bool: - dead = False - for _ in range(timeout_secs): - time.sleep(1) - if _actor_killed(worker_pid): - dead = True - break - return dead + """Check if a process with given pid is running.""" + return not psutil.pid_exists(int(pid)) def _kill_actor_using_dashboard_gcs( @@ -44,6 +32,7 @@ def _kill_actor_using_dashboard_gcs( "actor_id": actor_id, "force_kill": force_kill, }, + timeout=5, ) assert resp.status_code == expected_status_code resp_json = resp.json() @@ -78,7 +67,7 @@ def loop(self): OK = 200 NOT_FOUND = 404 - # Kill an non-existent actor + # Kill a non-existent actor resp = _kill_actor_using_dashboard_gcs( webui_url, "non-existent-actor-id", NOT_FOUND ) @@ -87,7 +76,7 @@ def loop(self): # Kill the actor resp = _kill_actor_using_dashboard_gcs(webui_url, actor_id, OK, force_kill=False) assert "It will exit once running tasks complete" in resp["msg"] - assert _actor_killed_loop(worker_pid) + wait_for_condition(lambda: _actor_killed(worker_pid)) # Create an actor and have it loop a = Actor.remote() @@ -95,15 +84,21 @@ def loop(self): actor_id = a._ray_actor_id.hex() a.loop.remote() + # wait for loop() to start + wait_for_condition(lambda: _is_actor_task_running(worker_pid, "Actor.loop")) + # Try to kill the actor, it should not die since a task is running resp = _kill_actor_using_dashboard_gcs(webui_url, actor_id, OK, force_kill=False) assert "It will exit once running tasks complete" in resp["msg"] - assert not _actor_killed_loop(worker_pid, timeout_secs=1) + with pytest.raises( + RuntimeError, match="The condition wasn't met before the timeout expired." + ): + wait_for_condition(lambda: _actor_killed(worker_pid), 1) # Force kill the actor resp = _kill_actor_using_dashboard_gcs(webui_url, actor_id, OK, force_kill=True) assert "Force killed actor with id" in resp["msg"] - assert _actor_killed_loop(worker_pid) + wait_for_condition(lambda: _actor_killed(worker_pid)) if __name__ == "__main__": From 930d27619de1bd14504fd005324981c1e0e0d170 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 11 Aug 2025 23:06:16 -0700 Subject: [PATCH 045/634] [ci] add orjson to release test images (#55518) and update lock files also tries to sort the requirements list Signed-off-by: Lonnie Liu --- .../ray_release/byod/requirements_byod_3.9.in | 4 +- .../byod/requirements_byod_3.9.txt | 57 ++++++++++++++++++- .../byod/requirements_ml_byod_3.9.in | 1 + .../byod/requirements_ml_byod_3.9.txt | 54 ++++++++++++++++++ 4 files changed, 112 insertions(+), 4 deletions(-) diff --git a/release/ray_release/byod/requirements_byod_3.9.in b/release/ray_release/byod/requirements_byod_3.9.in index a4ce891981f8..248863c233c5 100644 --- a/release/ray_release/byod/requirements_byod_3.9.in +++ b/release/ray_release/byod/requirements_byod_3.9.in @@ -17,11 +17,13 @@ lightgbm locust==2.18.0 memray openskill +orjson petastorm protobuf pyarrow pydantic>=2.5.0 pytest +pyyaml requests>=2.31.0 semidbm s3fs @@ -38,5 +40,3 @@ typing-extensions xarray xgboost zarr -pyyaml -orjson diff --git a/release/ray_release/byod/requirements_byod_3.9.txt b/release/ray_release/byod/requirements_byod_3.9.txt index 7a28fcc4d03c..3613efcea7f5 100644 --- a/release/ray_release/byod/requirements_byod_3.9.txt +++ b/release/ray_release/byod/requirements_byod_3.9.txt @@ -1199,7 +1199,7 @@ greenlet==3.0.1 \ # via # -c release/ray_release/byod/requirements_compiled.txt # gevent -grpcio==1.66.2 ; sys_platform != "darwin" \ +grpcio==1.66.2 \ --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ @@ -1256,7 +1256,6 @@ grpcio==1.66.2 ; sys_platform != "darwin" \ --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 # via - # -c release/ray_release/byod/requirements_compiled.txt # tensorboard # tensorflow gsutil==5.27 \ @@ -1816,6 +1815,60 @@ opt-einsum==3.3.0 \ # via # -c release/ray_release/byod/requirements_compiled.txt # tensorflow +orjson==3.9.10 \ + --hash=sha256:06ad5543217e0e46fd7ab7ea45d506c76f878b87b1b4e369006bdb01acc05a83 \ + --hash=sha256:0a73160e823151f33cdc05fe2cea557c5ef12fdf276ce29bb4f1c571c8368a60 \ + --hash=sha256:1234dc92d011d3554d929b6cf058ac4a24d188d97be5e04355f1b9223e98bbe9 \ + --hash=sha256:1d0dc4310da8b5f6415949bd5ef937e60aeb0eb6b16f95041b5e43e6200821fb \ + --hash=sha256:2a11b4b1a8415f105d989876a19b173f6cdc89ca13855ccc67c18efbd7cbd1f8 \ + --hash=sha256:2e2ecd1d349e62e3960695214f40939bbfdcaeaaa62ccc638f8e651cf0970e5f \ + --hash=sha256:3a2ce5ea4f71681623f04e2b7dadede3c7435dfb5e5e2d1d0ec25b35530e277b \ + --hash=sha256:3e892621434392199efb54e69edfff9f699f6cc36dd9553c5bf796058b14b20d \ + --hash=sha256:3fb205ab52a2e30354640780ce4587157a9563a68c9beaf52153e1cea9aa0921 \ + --hash=sha256:4689270c35d4bb3102e103ac43c3f0b76b169760aff8bcf2d401a3e0e58cdb7f \ + --hash=sha256:49f8ad582da6e8d2cf663c4ba5bf9f83cc052570a3a767487fec6af839b0e777 \ + --hash=sha256:4bd176f528a8151a6efc5359b853ba3cc0e82d4cd1fab9c1300c5d957dc8f48c \ + --hash=sha256:4cf7837c3b11a2dfb589f8530b3cff2bd0307ace4c301e8997e95c7468c1378e \ + --hash=sha256:4fd72fab7bddce46c6826994ce1e7de145ae1e9e106ebb8eb9ce1393ca01444d \ + --hash=sha256:5148bab4d71f58948c7c39d12b14a9005b6ab35a0bdf317a8ade9a9e4d9d0bd5 \ + --hash=sha256:5869e8e130e99687d9e4be835116c4ebd83ca92e52e55810962446d841aba8de \ + --hash=sha256:602a8001bdf60e1a7d544be29c82560a7b49319a0b31d62586548835bbe2c862 \ + --hash=sha256:61804231099214e2f84998316f3238c4c2c4aaec302df12b21a64d72e2a135c7 \ + --hash=sha256:666c6fdcaac1f13eb982b649e1c311c08d7097cbda24f32612dae43648d8db8d \ + --hash=sha256:674eb520f02422546c40401f4efaf8207b5e29e420c17051cddf6c02783ff5ca \ + --hash=sha256:7ec960b1b942ee3c69323b8721df2a3ce28ff40e7ca47873ae35bfafeb4555ca \ + --hash=sha256:7f433be3b3f4c66016d5a20e5b4444ef833a1f802ced13a2d852c637f69729c1 \ + --hash=sha256:7f8fb7f5ecf4f6355683ac6881fd64b5bb2b8a60e3ccde6ff799e48791d8f864 \ + --hash=sha256:81a3a3a72c9811b56adf8bcc829b010163bb2fc308877e50e9910c9357e78521 \ + --hash=sha256:858379cbb08d84fe7583231077d9a36a1a20eb72f8c9076a45df8b083724ad1d \ + --hash=sha256:8b9ba0ccd5a7f4219e67fbbe25e6b4a46ceef783c42af7dbc1da548eb28b6531 \ + --hash=sha256:92af0d00091e744587221e79f68d617b432425a7e59328ca4c496f774a356071 \ + --hash=sha256:9ebbdbd6a046c304b1845e96fbcc5559cd296b4dfd3ad2509e33c4d9ce07d6a1 \ + --hash=sha256:9edd2856611e5050004f4722922b7b1cd6268da34102667bd49d2a2b18bafb81 \ + --hash=sha256:a353bf1f565ed27ba71a419b2cd3db9d6151da426b61b289b6ba1422a702e643 \ + --hash=sha256:b5b7d4a44cc0e6ff98da5d56cde794385bdd212a86563ac321ca64d7f80c80d1 \ + --hash=sha256:b90f340cb6397ec7a854157fac03f0c82b744abdd1c0941a024c3c29d1340aff \ + --hash=sha256:c18a4da2f50050a03d1da5317388ef84a16013302a5281d6f64e4a3f406aabc4 \ + --hash=sha256:c338ed69ad0b8f8f8920c13f529889fe0771abbb46550013e3c3d01e5174deef \ + --hash=sha256:c5a02360e73e7208a872bf65a7554c9f15df5fe063dc047f79738998b0506a14 \ + --hash=sha256:c62b6fa2961a1dcc51ebe88771be5319a93fd89bd247c9ddf732bc250507bc2b \ + --hash=sha256:c812312847867b6335cfb264772f2a7e85b3b502d3a6b0586aa35e1858528ab1 \ + --hash=sha256:c943b35ecdf7123b2d81d225397efddf0bce2e81db2f3ae633ead38e85cd5ade \ + --hash=sha256:ce0a29c28dfb8eccd0f16219360530bc3cfdf6bf70ca384dacd36e6c650ef8e8 \ + --hash=sha256:cf80b550092cc480a0cbd0750e8189247ff45457e5a023305f7ef1bcec811616 \ + --hash=sha256:cff7570d492bcf4b64cc862a6e2fb77edd5e5748ad715f487628f102815165e9 \ + --hash=sha256:d2c1e559d96a7f94a4f581e2a32d6d610df5840881a8cba8f25e446f4d792df3 \ + --hash=sha256:deeb3922a7a804755bbe6b5be9b312e746137a03600f488290318936c1a2d4dc \ + --hash=sha256:e28a50b5be854e18d54f75ef1bb13e1abf4bc650ab9d635e4258c58e71eb6ad5 \ + --hash=sha256:e99c625b8c95d7741fe057585176b1b8783d46ed4b8932cf98ee145c4facf499 \ + --hash=sha256:ec6f18f96b47299c11203edfbdc34e1b69085070d9a3d1f302810cc23ad36bf3 \ + --hash=sha256:ed8bc367f725dfc5cabeed1ae079d00369900231fbb5a5280cf0736c30e2adf7 \ + --hash=sha256:ee5926746232f627a3be1cc175b2cfad24d0170d520361f4ce3fa2fd83f09e1d \ + --hash=sha256:f295efcd47b6124b01255d1491f9e46f17ef40d3d7eabf7364099e463fb45f0f \ + --hash=sha256:fb0b361d73f6b8eeceba47cd37070b5e6c9de5beaeaa63a1cb35c7e1a73ef088 + # via + # -c release/ray_release/byod/requirements_compiled.txt + # -r release/ray_release/byod/requirements_byod_3.9.in packaging==23.0 \ --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 diff --git a/release/ray_release/byod/requirements_ml_byod_3.9.in b/release/ray_release/byod/requirements_ml_byod_3.9.in index 6e93e852e7ed..1a7ce561af71 100644 --- a/release/ray_release/byod/requirements_ml_byod_3.9.in +++ b/release/ray_release/byod/requirements_ml_byod_3.9.in @@ -27,6 +27,7 @@ modin numpy openai-whisper openskill +orjson petastorm protobuf pyarrow diff --git a/release/ray_release/byod/requirements_ml_byod_3.9.txt b/release/ray_release/byod/requirements_ml_byod_3.9.txt index 84b41c4c5447..e907ad35cae5 100644 --- a/release/ray_release/byod/requirements_ml_byod_3.9.txt +++ b/release/ray_release/byod/requirements_ml_byod_3.9.txt @@ -2310,6 +2310,60 @@ openskill==6.0.0 \ --hash=sha256:eee2d0b3c1648663a480cf4680654dfd12bdc749a96d611b1904e191f2632f62 \ --hash=sha256:f89b18930c2befd580407e7cf80a480bc69c3b25d2841346be6d875c8c4bc92e # via -r release/ray_release/byod/requirements_ml_byod_3.9.in +orjson==3.9.10 \ + --hash=sha256:06ad5543217e0e46fd7ab7ea45d506c76f878b87b1b4e369006bdb01acc05a83 \ + --hash=sha256:0a73160e823151f33cdc05fe2cea557c5ef12fdf276ce29bb4f1c571c8368a60 \ + --hash=sha256:1234dc92d011d3554d929b6cf058ac4a24d188d97be5e04355f1b9223e98bbe9 \ + --hash=sha256:1d0dc4310da8b5f6415949bd5ef937e60aeb0eb6b16f95041b5e43e6200821fb \ + --hash=sha256:2a11b4b1a8415f105d989876a19b173f6cdc89ca13855ccc67c18efbd7cbd1f8 \ + --hash=sha256:2e2ecd1d349e62e3960695214f40939bbfdcaeaaa62ccc638f8e651cf0970e5f \ + --hash=sha256:3a2ce5ea4f71681623f04e2b7dadede3c7435dfb5e5e2d1d0ec25b35530e277b \ + --hash=sha256:3e892621434392199efb54e69edfff9f699f6cc36dd9553c5bf796058b14b20d \ + --hash=sha256:3fb205ab52a2e30354640780ce4587157a9563a68c9beaf52153e1cea9aa0921 \ + --hash=sha256:4689270c35d4bb3102e103ac43c3f0b76b169760aff8bcf2d401a3e0e58cdb7f \ + --hash=sha256:49f8ad582da6e8d2cf663c4ba5bf9f83cc052570a3a767487fec6af839b0e777 \ + --hash=sha256:4bd176f528a8151a6efc5359b853ba3cc0e82d4cd1fab9c1300c5d957dc8f48c \ + --hash=sha256:4cf7837c3b11a2dfb589f8530b3cff2bd0307ace4c301e8997e95c7468c1378e \ + --hash=sha256:4fd72fab7bddce46c6826994ce1e7de145ae1e9e106ebb8eb9ce1393ca01444d \ + --hash=sha256:5148bab4d71f58948c7c39d12b14a9005b6ab35a0bdf317a8ade9a9e4d9d0bd5 \ + --hash=sha256:5869e8e130e99687d9e4be835116c4ebd83ca92e52e55810962446d841aba8de \ + --hash=sha256:602a8001bdf60e1a7d544be29c82560a7b49319a0b31d62586548835bbe2c862 \ + --hash=sha256:61804231099214e2f84998316f3238c4c2c4aaec302df12b21a64d72e2a135c7 \ + --hash=sha256:666c6fdcaac1f13eb982b649e1c311c08d7097cbda24f32612dae43648d8db8d \ + --hash=sha256:674eb520f02422546c40401f4efaf8207b5e29e420c17051cddf6c02783ff5ca \ + --hash=sha256:7ec960b1b942ee3c69323b8721df2a3ce28ff40e7ca47873ae35bfafeb4555ca \ + --hash=sha256:7f433be3b3f4c66016d5a20e5b4444ef833a1f802ced13a2d852c637f69729c1 \ + --hash=sha256:7f8fb7f5ecf4f6355683ac6881fd64b5bb2b8a60e3ccde6ff799e48791d8f864 \ + --hash=sha256:81a3a3a72c9811b56adf8bcc829b010163bb2fc308877e50e9910c9357e78521 \ + --hash=sha256:858379cbb08d84fe7583231077d9a36a1a20eb72f8c9076a45df8b083724ad1d \ + --hash=sha256:8b9ba0ccd5a7f4219e67fbbe25e6b4a46ceef783c42af7dbc1da548eb28b6531 \ + --hash=sha256:92af0d00091e744587221e79f68d617b432425a7e59328ca4c496f774a356071 \ + --hash=sha256:9ebbdbd6a046c304b1845e96fbcc5559cd296b4dfd3ad2509e33c4d9ce07d6a1 \ + --hash=sha256:9edd2856611e5050004f4722922b7b1cd6268da34102667bd49d2a2b18bafb81 \ + --hash=sha256:a353bf1f565ed27ba71a419b2cd3db9d6151da426b61b289b6ba1422a702e643 \ + --hash=sha256:b5b7d4a44cc0e6ff98da5d56cde794385bdd212a86563ac321ca64d7f80c80d1 \ + --hash=sha256:b90f340cb6397ec7a854157fac03f0c82b744abdd1c0941a024c3c29d1340aff \ + --hash=sha256:c18a4da2f50050a03d1da5317388ef84a16013302a5281d6f64e4a3f406aabc4 \ + --hash=sha256:c338ed69ad0b8f8f8920c13f529889fe0771abbb46550013e3c3d01e5174deef \ + --hash=sha256:c5a02360e73e7208a872bf65a7554c9f15df5fe063dc047f79738998b0506a14 \ + --hash=sha256:c62b6fa2961a1dcc51ebe88771be5319a93fd89bd247c9ddf732bc250507bc2b \ + --hash=sha256:c812312847867b6335cfb264772f2a7e85b3b502d3a6b0586aa35e1858528ab1 \ + --hash=sha256:c943b35ecdf7123b2d81d225397efddf0bce2e81db2f3ae633ead38e85cd5ade \ + --hash=sha256:ce0a29c28dfb8eccd0f16219360530bc3cfdf6bf70ca384dacd36e6c650ef8e8 \ + --hash=sha256:cf80b550092cc480a0cbd0750e8189247ff45457e5a023305f7ef1bcec811616 \ + --hash=sha256:cff7570d492bcf4b64cc862a6e2fb77edd5e5748ad715f487628f102815165e9 \ + --hash=sha256:d2c1e559d96a7f94a4f581e2a32d6d610df5840881a8cba8f25e446f4d792df3 \ + --hash=sha256:deeb3922a7a804755bbe6b5be9b312e746137a03600f488290318936c1a2d4dc \ + --hash=sha256:e28a50b5be854e18d54f75ef1bb13e1abf4bc650ab9d635e4258c58e71eb6ad5 \ + --hash=sha256:e99c625b8c95d7741fe057585176b1b8783d46ed4b8932cf98ee145c4facf499 \ + --hash=sha256:ec6f18f96b47299c11203edfbdc34e1b69085070d9a3d1f302810cc23ad36bf3 \ + --hash=sha256:ed8bc367f725dfc5cabeed1ae079d00369900231fbb5a5280cf0736c30e2adf7 \ + --hash=sha256:ee5926746232f627a3be1cc175b2cfad24d0170d520361f4ce3fa2fd83f09e1d \ + --hash=sha256:f295efcd47b6124b01255d1491f9e46f17ef40d3d7eabf7364099e463fb45f0f \ + --hash=sha256:fb0b361d73f6b8eeceba47cd37070b5e6c9de5beaeaa63a1cb35c7e1a73ef088 + # via + # -c release/ray_release/byod/requirements_compiled.txt + # -r release/ray_release/byod/requirements_ml_byod_3.9.in packaging==23.0 \ --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 From 36db71dde3ab9138ecfbd0470d58c1b8b35152d2 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 11 Aug 2025 23:07:02 -0700 Subject: [PATCH 046/634] [llm] add grpcio-tools into the depset (#55514) protobuf is upgraded to 4, and `grpcio-tool` compiles now, so we no longer need to ignore it. Signed-off-by: Lonnie Liu --- ci/compile_llm_requirements.sh | 2 - .../requirements_compiled_ray_py311_cpu.txt | 2 +- .../requirements_compiled_ray_py311_cu121.txt | 2 +- .../requirements_compiled_ray_py311_cu128.txt | 2 +- ...quirements_compiled_ray_test_py311_cpu.txt | 55 ++++++++++++++++++- ...irements_compiled_ray_test_py311_cu121.txt | 55 ++++++++++++++++++- ...irements_compiled_ray_test_py311_cu128.txt | 55 ++++++++++++++++++- ...requirements_compiled_rayllm_py311_cpu.txt | 2 +- ...quirements_compiled_rayllm_py311_cu121.txt | 2 +- ...quirements_compiled_rayllm_py311_cu128.txt | 2 +- ...rements_compiled_rayllm_test_py311_cpu.txt | 55 ++++++++++++++++++- ...ments_compiled_rayllm_test_py311_cu121.txt | 55 ++++++++++++++++++- ...ments_compiled_rayllm_test_py311_cu128.txt | 55 ++++++++++++++++++- 13 files changed, 324 insertions(+), 20 deletions(-) diff --git a/ci/compile_llm_requirements.sh b/ci/compile_llm_requirements.sh index b12e7df6c07d..5932d1e005b9 100755 --- a/ci/compile_llm_requirements.sh +++ b/ci/compile_llm_requirements.sh @@ -17,8 +17,6 @@ for CUDA_CODE in cpu cu121 cu128; do UV_PIP_COMPILE=( uv pip compile --generate-hashes --strip-extras --unsafe-package ray - # The version we use on python 3.9 is not installable on python 3.11 - --unsafe-package grpcio-tools # setuptools should not be pinned. --unsafe-package setuptools --index-url "https://pypi.org/simple" diff --git a/python/requirements_compiled_ray_py311_cpu.txt b/python/requirements_compiled_ray_py311_cpu.txt index 1398bbae57df..5cac2c637eaf 100644 --- a/python/requirements_compiled_ray_py311_cpu.txt +++ b/python/requirements_compiled_ray_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/requirements_compiled_ray_py311_cu121.txt b/python/requirements_compiled_ray_py311_cu121.txt index e3e854a33f4d..f50394202b62 100644 --- a/python/requirements_compiled_ray_py311_cu121.txt +++ b/python/requirements_compiled_ray_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/requirements_compiled_ray_py311_cu128.txt b/python/requirements_compiled_ray_py311_cu128.txt index f1b0a3107207..28c3d535db38 100644 --- a/python/requirements_compiled_ray_py311_cu128.txt +++ b/python/requirements_compiled_ray_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/python/requirements_compiled_ray_test_py311_cpu.txt b/python/requirements_compiled_ray_test_py311_cpu.txt index d8bae7252f61..989c6654e83d 100644 --- a/python/requirements_compiled_ray_test_py311_cpu.txt +++ b/python/requirements_compiled_ray_test_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu @@ -1008,6 +1008,58 @@ grpcio==1.66.2 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # grpcio-tools +grpcio-tools==1.62.3 \ + --hash=sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133 \ + --hash=sha256:0a8c0c4724ae9c2181b7dbc9b186df46e4f62cb18dc184e46d06c0ebeccf569e \ + --hash=sha256:0cb3a3436ac119cbd37a7d3331d9bdf85dad21a6ac233a3411dff716dcbf401e \ + --hash=sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7 \ + --hash=sha256:11f363570dea661dde99e04a51bd108a5807b5df32a6f8bdf4860e34e94a4dbf \ + --hash=sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa \ + --hash=sha256:1c989246c2aebc13253f08be32538a4039a64e12d9c18f6d662d7aee641dc8b5 \ + --hash=sha256:1da38070738da53556a4b35ab67c1b9884a5dd48fa2f243db35dc14079ea3d0c \ + --hash=sha256:27cd9ef5c5d68d5ed104b6dcb96fe9c66b82050e546c9e255716903c3d8f0373 \ + --hash=sha256:2e02d3b96f2d0e4bab9ceaa30f37d4f75571e40c6272e95364bff3125a64d184 \ + --hash=sha256:2f968b049c2849540751ec2100ab05e8086c24bead769ca734fdab58698408c1 \ + --hash=sha256:350a80485e302daaa95d335a931f97b693e170e02d43767ab06552c708808950 \ + --hash=sha256:3eae6ea76d62fcac091e1f15c2dcedf1dc3f114f8df1a972a8a0745e89f4cf61 \ + --hash=sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0 \ + --hash=sha256:5782883a27d3fae8c425b29a9d3dcf5f47d992848a1b76970da3b5a28d424b26 \ + --hash=sha256:6a56d344b0bab30bf342a67e33d386b0b3c4e65868ffe93c341c51e1a8853ca5 \ + --hash=sha256:6c3064610826f50bd69410c63101954676edc703e03f9e8f978a135f1aaf97c1 \ + --hash=sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23 \ + --hash=sha256:710fecf6a171dcbfa263a0a3e7070e0df65ba73158d4c539cec50978f11dad5d \ + --hash=sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833 \ + --hash=sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492 \ + --hash=sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43 \ + --hash=sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7 \ + --hash=sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3 \ + --hash=sha256:8c5d22b252dcef11dd1e0fbbe5bbfb9b4ae048e8880d33338215e8ccbdb03edc \ + --hash=sha256:8e62cc7164b0b7c5128e637e394eb2ef3db0e61fc798e80c301de3b2379203ed \ + --hash=sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a \ + --hash=sha256:ace43b26d88a58dcff16c20d23ff72b04d0a415f64d2820f4ff06b1166f50557 \ + --hash=sha256:b47d0dda1bdb0a0ba7a9a6de88e5a1ed61f07fad613964879954961e36d49193 \ + --hash=sha256:b77f9f9cee87cd798f0fe26b7024344d1b03a7cd2d2cba7035f8433b13986325 \ + --hash=sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b \ + --hash=sha256:bfda6ee8990997a9df95c5606f3096dae65f09af7ca03a1e9ca28f088caca5cf \ + --hash=sha256:c3a1ac9d394f8e229eb28eec2e04b9a6f5433fa19c9d32f1cb6066e3c5114a1d \ + --hash=sha256:c8ad5cce554e2fcaf8842dee5d9462583b601a3a78f8b76a153c38c963f58c10 \ + --hash=sha256:ca246dffeca0498be9b4e1ee169b62e64694b0f92e6d0be2573e65522f39eea9 \ + --hash=sha256:ca4f5eeadbb57cf03317d6a2857823239a63a59cc935f5bd6cf6e8b7af7a7ecc \ + --hash=sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5 \ + --hash=sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5 \ + --hash=sha256:dc9ad9950119d8ae27634e68b7663cc8d340ae535a0f80d85a55e56a6973ab1f \ + --hash=sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc \ + --hash=sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f \ + --hash=sha256:e18e15287c31baf574fcdf8251fb7f997d64e96c6ecf467906e576da0a079af6 \ + --hash=sha256:ec279dcf3518201fc592c65002754f58a6b542798cd7f3ecd4af086422f33f29 \ + --hash=sha256:ec6fbded0c61afe6f84e3c2a43e6d656791d95747d6d28b73eff1af64108c434 \ + --hash=sha256:eec73a005443061f4759b71a056f745e3b000dc0dc125c9f20560232dfbcbd14 \ + --hash=sha256:f3d812daffd0c2d2794756bd45a353f89e55dc8f91eb2fc840c51b9f6be62667 \ + --hash=sha256:f4b1615adf67bd8bb71f3464146a6f9949972d06d21a4f5e87e73f6464d97f57 \ + --hash=sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # -r python/requirements/cloud-requirements.txt gymnasium==1.0.0 \ --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad @@ -3399,5 +3451,4 @@ zipp==3.19.2 \ # importlib-metadata # The following packages were excluded from the output: -# grpcio-tools # setuptools diff --git a/python/requirements_compiled_ray_test_py311_cu121.txt b/python/requirements_compiled_ray_test_py311_cu121.txt index ab15c20d3ec4..f9fb02769003 100644 --- a/python/requirements_compiled_ray_test_py311_cu121.txt +++ b/python/requirements_compiled_ray_test_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 @@ -1008,6 +1008,58 @@ grpcio==1.66.2 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # grpcio-tools +grpcio-tools==1.62.3 \ + --hash=sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133 \ + --hash=sha256:0a8c0c4724ae9c2181b7dbc9b186df46e4f62cb18dc184e46d06c0ebeccf569e \ + --hash=sha256:0cb3a3436ac119cbd37a7d3331d9bdf85dad21a6ac233a3411dff716dcbf401e \ + --hash=sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7 \ + --hash=sha256:11f363570dea661dde99e04a51bd108a5807b5df32a6f8bdf4860e34e94a4dbf \ + --hash=sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa \ + --hash=sha256:1c989246c2aebc13253f08be32538a4039a64e12d9c18f6d662d7aee641dc8b5 \ + --hash=sha256:1da38070738da53556a4b35ab67c1b9884a5dd48fa2f243db35dc14079ea3d0c \ + --hash=sha256:27cd9ef5c5d68d5ed104b6dcb96fe9c66b82050e546c9e255716903c3d8f0373 \ + --hash=sha256:2e02d3b96f2d0e4bab9ceaa30f37d4f75571e40c6272e95364bff3125a64d184 \ + --hash=sha256:2f968b049c2849540751ec2100ab05e8086c24bead769ca734fdab58698408c1 \ + --hash=sha256:350a80485e302daaa95d335a931f97b693e170e02d43767ab06552c708808950 \ + --hash=sha256:3eae6ea76d62fcac091e1f15c2dcedf1dc3f114f8df1a972a8a0745e89f4cf61 \ + --hash=sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0 \ + --hash=sha256:5782883a27d3fae8c425b29a9d3dcf5f47d992848a1b76970da3b5a28d424b26 \ + --hash=sha256:6a56d344b0bab30bf342a67e33d386b0b3c4e65868ffe93c341c51e1a8853ca5 \ + --hash=sha256:6c3064610826f50bd69410c63101954676edc703e03f9e8f978a135f1aaf97c1 \ + --hash=sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23 \ + --hash=sha256:710fecf6a171dcbfa263a0a3e7070e0df65ba73158d4c539cec50978f11dad5d \ + --hash=sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833 \ + --hash=sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492 \ + --hash=sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43 \ + --hash=sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7 \ + --hash=sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3 \ + --hash=sha256:8c5d22b252dcef11dd1e0fbbe5bbfb9b4ae048e8880d33338215e8ccbdb03edc \ + --hash=sha256:8e62cc7164b0b7c5128e637e394eb2ef3db0e61fc798e80c301de3b2379203ed \ + --hash=sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a \ + --hash=sha256:ace43b26d88a58dcff16c20d23ff72b04d0a415f64d2820f4ff06b1166f50557 \ + --hash=sha256:b47d0dda1bdb0a0ba7a9a6de88e5a1ed61f07fad613964879954961e36d49193 \ + --hash=sha256:b77f9f9cee87cd798f0fe26b7024344d1b03a7cd2d2cba7035f8433b13986325 \ + --hash=sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b \ + --hash=sha256:bfda6ee8990997a9df95c5606f3096dae65f09af7ca03a1e9ca28f088caca5cf \ + --hash=sha256:c3a1ac9d394f8e229eb28eec2e04b9a6f5433fa19c9d32f1cb6066e3c5114a1d \ + --hash=sha256:c8ad5cce554e2fcaf8842dee5d9462583b601a3a78f8b76a153c38c963f58c10 \ + --hash=sha256:ca246dffeca0498be9b4e1ee169b62e64694b0f92e6d0be2573e65522f39eea9 \ + --hash=sha256:ca4f5eeadbb57cf03317d6a2857823239a63a59cc935f5bd6cf6e8b7af7a7ecc \ + --hash=sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5 \ + --hash=sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5 \ + --hash=sha256:dc9ad9950119d8ae27634e68b7663cc8d340ae535a0f80d85a55e56a6973ab1f \ + --hash=sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc \ + --hash=sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f \ + --hash=sha256:e18e15287c31baf574fcdf8251fb7f997d64e96c6ecf467906e576da0a079af6 \ + --hash=sha256:ec279dcf3518201fc592c65002754f58a6b542798cd7f3ecd4af086422f33f29 \ + --hash=sha256:ec6fbded0c61afe6f84e3c2a43e6d656791d95747d6d28b73eff1af64108c434 \ + --hash=sha256:eec73a005443061f4759b71a056f745e3b000dc0dc125c9f20560232dfbcbd14 \ + --hash=sha256:f3d812daffd0c2d2794756bd45a353f89e55dc8f91eb2fc840c51b9f6be62667 \ + --hash=sha256:f4b1615adf67bd8bb71f3464146a6f9949972d06d21a4f5e87e73f6464d97f57 \ + --hash=sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # -r python/requirements/cloud-requirements.txt gymnasium==1.0.0 \ --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad @@ -3399,5 +3451,4 @@ zipp==3.19.2 \ # importlib-metadata # The following packages were excluded from the output: -# grpcio-tools # setuptools diff --git a/python/requirements_compiled_ray_test_py311_cu128.txt b/python/requirements_compiled_ray_test_py311_cu128.txt index ff57573edfbe..21697b75b978 100644 --- a/python/requirements_compiled_ray_test_py311_cu128.txt +++ b/python/requirements_compiled_ray_test_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 @@ -1008,6 +1008,58 @@ grpcio==1.66.2 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # grpcio-tools +grpcio-tools==1.62.3 \ + --hash=sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133 \ + --hash=sha256:0a8c0c4724ae9c2181b7dbc9b186df46e4f62cb18dc184e46d06c0ebeccf569e \ + --hash=sha256:0cb3a3436ac119cbd37a7d3331d9bdf85dad21a6ac233a3411dff716dcbf401e \ + --hash=sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7 \ + --hash=sha256:11f363570dea661dde99e04a51bd108a5807b5df32a6f8bdf4860e34e94a4dbf \ + --hash=sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa \ + --hash=sha256:1c989246c2aebc13253f08be32538a4039a64e12d9c18f6d662d7aee641dc8b5 \ + --hash=sha256:1da38070738da53556a4b35ab67c1b9884a5dd48fa2f243db35dc14079ea3d0c \ + --hash=sha256:27cd9ef5c5d68d5ed104b6dcb96fe9c66b82050e546c9e255716903c3d8f0373 \ + --hash=sha256:2e02d3b96f2d0e4bab9ceaa30f37d4f75571e40c6272e95364bff3125a64d184 \ + --hash=sha256:2f968b049c2849540751ec2100ab05e8086c24bead769ca734fdab58698408c1 \ + --hash=sha256:350a80485e302daaa95d335a931f97b693e170e02d43767ab06552c708808950 \ + --hash=sha256:3eae6ea76d62fcac091e1f15c2dcedf1dc3f114f8df1a972a8a0745e89f4cf61 \ + --hash=sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0 \ + --hash=sha256:5782883a27d3fae8c425b29a9d3dcf5f47d992848a1b76970da3b5a28d424b26 \ + --hash=sha256:6a56d344b0bab30bf342a67e33d386b0b3c4e65868ffe93c341c51e1a8853ca5 \ + --hash=sha256:6c3064610826f50bd69410c63101954676edc703e03f9e8f978a135f1aaf97c1 \ + --hash=sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23 \ + --hash=sha256:710fecf6a171dcbfa263a0a3e7070e0df65ba73158d4c539cec50978f11dad5d \ + --hash=sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833 \ + --hash=sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492 \ + --hash=sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43 \ + --hash=sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7 \ + --hash=sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3 \ + --hash=sha256:8c5d22b252dcef11dd1e0fbbe5bbfb9b4ae048e8880d33338215e8ccbdb03edc \ + --hash=sha256:8e62cc7164b0b7c5128e637e394eb2ef3db0e61fc798e80c301de3b2379203ed \ + --hash=sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a \ + --hash=sha256:ace43b26d88a58dcff16c20d23ff72b04d0a415f64d2820f4ff06b1166f50557 \ + --hash=sha256:b47d0dda1bdb0a0ba7a9a6de88e5a1ed61f07fad613964879954961e36d49193 \ + --hash=sha256:b77f9f9cee87cd798f0fe26b7024344d1b03a7cd2d2cba7035f8433b13986325 \ + --hash=sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b \ + --hash=sha256:bfda6ee8990997a9df95c5606f3096dae65f09af7ca03a1e9ca28f088caca5cf \ + --hash=sha256:c3a1ac9d394f8e229eb28eec2e04b9a6f5433fa19c9d32f1cb6066e3c5114a1d \ + --hash=sha256:c8ad5cce554e2fcaf8842dee5d9462583b601a3a78f8b76a153c38c963f58c10 \ + --hash=sha256:ca246dffeca0498be9b4e1ee169b62e64694b0f92e6d0be2573e65522f39eea9 \ + --hash=sha256:ca4f5eeadbb57cf03317d6a2857823239a63a59cc935f5bd6cf6e8b7af7a7ecc \ + --hash=sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5 \ + --hash=sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5 \ + --hash=sha256:dc9ad9950119d8ae27634e68b7663cc8d340ae535a0f80d85a55e56a6973ab1f \ + --hash=sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc \ + --hash=sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f \ + --hash=sha256:e18e15287c31baf574fcdf8251fb7f997d64e96c6ecf467906e576da0a079af6 \ + --hash=sha256:ec279dcf3518201fc592c65002754f58a6b542798cd7f3ecd4af086422f33f29 \ + --hash=sha256:ec6fbded0c61afe6f84e3c2a43e6d656791d95747d6d28b73eff1af64108c434 \ + --hash=sha256:eec73a005443061f4759b71a056f745e3b000dc0dc125c9f20560232dfbcbd14 \ + --hash=sha256:f3d812daffd0c2d2794756bd45a353f89e55dc8f91eb2fc840c51b9f6be62667 \ + --hash=sha256:f4b1615adf67bd8bb71f3464146a6f9949972d06d21a4f5e87e73f6464d97f57 \ + --hash=sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # -r python/requirements/cloud-requirements.txt gymnasium==1.0.0 \ --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad @@ -3399,5 +3451,4 @@ zipp==3.19.2 \ # importlib-metadata # The following packages were excluded from the output: -# grpcio-tools # setuptools diff --git a/python/requirements_compiled_rayllm_py311_cpu.txt b/python/requirements_compiled_rayllm_py311_cpu.txt index 96179efeaef6..f98d7fb01c4c 100644 --- a/python/requirements_compiled_rayllm_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_rayllm_test_py311_cpu.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_rayllm_test_py311_cpu.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/requirements_compiled_rayllm_py311_cu121.txt b/python/requirements_compiled_rayllm_py311_cu121.txt index 61dfd0f354e5..b28613cf00e4 100644 --- a/python/requirements_compiled_rayllm_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_rayllm_test_py311_cu121.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_rayllm_test_py311_cu121.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/requirements_compiled_rayllm_py311_cu128.txt b/python/requirements_compiled_rayllm_py311_cu128.txt index 0435a7e31115..a944365fb566 100644 --- a/python/requirements_compiled_rayllm_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_rayllm_test_py311_cu128.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_rayllm_test_py311_cu128.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/python/requirements_compiled_rayllm_test_py311_cpu.txt b/python/requirements_compiled_rayllm_test_py311_cpu.txt index 5c59d6a2d967..e96f530c2310 100644 --- a/python/requirements_compiled_rayllm_test_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_test_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu @@ -1218,6 +1218,58 @@ grpcio==1.66.2 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # grpcio-tools +grpcio-tools==1.62.3 \ + --hash=sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133 \ + --hash=sha256:0a8c0c4724ae9c2181b7dbc9b186df46e4f62cb18dc184e46d06c0ebeccf569e \ + --hash=sha256:0cb3a3436ac119cbd37a7d3331d9bdf85dad21a6ac233a3411dff716dcbf401e \ + --hash=sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7 \ + --hash=sha256:11f363570dea661dde99e04a51bd108a5807b5df32a6f8bdf4860e34e94a4dbf \ + --hash=sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa \ + --hash=sha256:1c989246c2aebc13253f08be32538a4039a64e12d9c18f6d662d7aee641dc8b5 \ + --hash=sha256:1da38070738da53556a4b35ab67c1b9884a5dd48fa2f243db35dc14079ea3d0c \ + --hash=sha256:27cd9ef5c5d68d5ed104b6dcb96fe9c66b82050e546c9e255716903c3d8f0373 \ + --hash=sha256:2e02d3b96f2d0e4bab9ceaa30f37d4f75571e40c6272e95364bff3125a64d184 \ + --hash=sha256:2f968b049c2849540751ec2100ab05e8086c24bead769ca734fdab58698408c1 \ + --hash=sha256:350a80485e302daaa95d335a931f97b693e170e02d43767ab06552c708808950 \ + --hash=sha256:3eae6ea76d62fcac091e1f15c2dcedf1dc3f114f8df1a972a8a0745e89f4cf61 \ + --hash=sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0 \ + --hash=sha256:5782883a27d3fae8c425b29a9d3dcf5f47d992848a1b76970da3b5a28d424b26 \ + --hash=sha256:6a56d344b0bab30bf342a67e33d386b0b3c4e65868ffe93c341c51e1a8853ca5 \ + --hash=sha256:6c3064610826f50bd69410c63101954676edc703e03f9e8f978a135f1aaf97c1 \ + --hash=sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23 \ + --hash=sha256:710fecf6a171dcbfa263a0a3e7070e0df65ba73158d4c539cec50978f11dad5d \ + --hash=sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833 \ + --hash=sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492 \ + --hash=sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43 \ + --hash=sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7 \ + --hash=sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3 \ + --hash=sha256:8c5d22b252dcef11dd1e0fbbe5bbfb9b4ae048e8880d33338215e8ccbdb03edc \ + --hash=sha256:8e62cc7164b0b7c5128e637e394eb2ef3db0e61fc798e80c301de3b2379203ed \ + --hash=sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a \ + --hash=sha256:ace43b26d88a58dcff16c20d23ff72b04d0a415f64d2820f4ff06b1166f50557 \ + --hash=sha256:b47d0dda1bdb0a0ba7a9a6de88e5a1ed61f07fad613964879954961e36d49193 \ + --hash=sha256:b77f9f9cee87cd798f0fe26b7024344d1b03a7cd2d2cba7035f8433b13986325 \ + --hash=sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b \ + --hash=sha256:bfda6ee8990997a9df95c5606f3096dae65f09af7ca03a1e9ca28f088caca5cf \ + --hash=sha256:c3a1ac9d394f8e229eb28eec2e04b9a6f5433fa19c9d32f1cb6066e3c5114a1d \ + --hash=sha256:c8ad5cce554e2fcaf8842dee5d9462583b601a3a78f8b76a153c38c963f58c10 \ + --hash=sha256:ca246dffeca0498be9b4e1ee169b62e64694b0f92e6d0be2573e65522f39eea9 \ + --hash=sha256:ca4f5eeadbb57cf03317d6a2857823239a63a59cc935f5bd6cf6e8b7af7a7ecc \ + --hash=sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5 \ + --hash=sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5 \ + --hash=sha256:dc9ad9950119d8ae27634e68b7663cc8d340ae535a0f80d85a55e56a6973ab1f \ + --hash=sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc \ + --hash=sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f \ + --hash=sha256:e18e15287c31baf574fcdf8251fb7f997d64e96c6ecf467906e576da0a079af6 \ + --hash=sha256:ec279dcf3518201fc592c65002754f58a6b542798cd7f3ecd4af086422f33f29 \ + --hash=sha256:ec6fbded0c61afe6f84e3c2a43e6d656791d95747d6d28b73eff1af64108c434 \ + --hash=sha256:eec73a005443061f4759b71a056f745e3b000dc0dc125c9f20560232dfbcbd14 \ + --hash=sha256:f3d812daffd0c2d2794756bd45a353f89e55dc8f91eb2fc840c51b9f6be62667 \ + --hash=sha256:f4b1615adf67bd8bb71f3464146a6f9949972d06d21a4f5e87e73f6464d97f57 \ + --hash=sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -r python/requirements/cloud-requirements.txt gymnasium==1.0.0 \ --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad @@ -4838,5 +4890,4 @@ zipp==3.19.2 \ # The following packages were excluded from the output: # ray -# grpcio-tools # setuptools diff --git a/python/requirements_compiled_rayllm_test_py311_cu121.txt b/python/requirements_compiled_rayllm_test_py311_cu121.txt index b33cc1e6eb75..b7987154666c 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 @@ -1218,6 +1218,58 @@ grpcio==1.66.2 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # grpcio-tools +grpcio-tools==1.62.3 \ + --hash=sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133 \ + --hash=sha256:0a8c0c4724ae9c2181b7dbc9b186df46e4f62cb18dc184e46d06c0ebeccf569e \ + --hash=sha256:0cb3a3436ac119cbd37a7d3331d9bdf85dad21a6ac233a3411dff716dcbf401e \ + --hash=sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7 \ + --hash=sha256:11f363570dea661dde99e04a51bd108a5807b5df32a6f8bdf4860e34e94a4dbf \ + --hash=sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa \ + --hash=sha256:1c989246c2aebc13253f08be32538a4039a64e12d9c18f6d662d7aee641dc8b5 \ + --hash=sha256:1da38070738da53556a4b35ab67c1b9884a5dd48fa2f243db35dc14079ea3d0c \ + --hash=sha256:27cd9ef5c5d68d5ed104b6dcb96fe9c66b82050e546c9e255716903c3d8f0373 \ + --hash=sha256:2e02d3b96f2d0e4bab9ceaa30f37d4f75571e40c6272e95364bff3125a64d184 \ + --hash=sha256:2f968b049c2849540751ec2100ab05e8086c24bead769ca734fdab58698408c1 \ + --hash=sha256:350a80485e302daaa95d335a931f97b693e170e02d43767ab06552c708808950 \ + --hash=sha256:3eae6ea76d62fcac091e1f15c2dcedf1dc3f114f8df1a972a8a0745e89f4cf61 \ + --hash=sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0 \ + --hash=sha256:5782883a27d3fae8c425b29a9d3dcf5f47d992848a1b76970da3b5a28d424b26 \ + --hash=sha256:6a56d344b0bab30bf342a67e33d386b0b3c4e65868ffe93c341c51e1a8853ca5 \ + --hash=sha256:6c3064610826f50bd69410c63101954676edc703e03f9e8f978a135f1aaf97c1 \ + --hash=sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23 \ + --hash=sha256:710fecf6a171dcbfa263a0a3e7070e0df65ba73158d4c539cec50978f11dad5d \ + --hash=sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833 \ + --hash=sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492 \ + --hash=sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43 \ + --hash=sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7 \ + --hash=sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3 \ + --hash=sha256:8c5d22b252dcef11dd1e0fbbe5bbfb9b4ae048e8880d33338215e8ccbdb03edc \ + --hash=sha256:8e62cc7164b0b7c5128e637e394eb2ef3db0e61fc798e80c301de3b2379203ed \ + --hash=sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a \ + --hash=sha256:ace43b26d88a58dcff16c20d23ff72b04d0a415f64d2820f4ff06b1166f50557 \ + --hash=sha256:b47d0dda1bdb0a0ba7a9a6de88e5a1ed61f07fad613964879954961e36d49193 \ + --hash=sha256:b77f9f9cee87cd798f0fe26b7024344d1b03a7cd2d2cba7035f8433b13986325 \ + --hash=sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b \ + --hash=sha256:bfda6ee8990997a9df95c5606f3096dae65f09af7ca03a1e9ca28f088caca5cf \ + --hash=sha256:c3a1ac9d394f8e229eb28eec2e04b9a6f5433fa19c9d32f1cb6066e3c5114a1d \ + --hash=sha256:c8ad5cce554e2fcaf8842dee5d9462583b601a3a78f8b76a153c38c963f58c10 \ + --hash=sha256:ca246dffeca0498be9b4e1ee169b62e64694b0f92e6d0be2573e65522f39eea9 \ + --hash=sha256:ca4f5eeadbb57cf03317d6a2857823239a63a59cc935f5bd6cf6e8b7af7a7ecc \ + --hash=sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5 \ + --hash=sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5 \ + --hash=sha256:dc9ad9950119d8ae27634e68b7663cc8d340ae535a0f80d85a55e56a6973ab1f \ + --hash=sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc \ + --hash=sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f \ + --hash=sha256:e18e15287c31baf574fcdf8251fb7f997d64e96c6ecf467906e576da0a079af6 \ + --hash=sha256:ec279dcf3518201fc592c65002754f58a6b542798cd7f3ecd4af086422f33f29 \ + --hash=sha256:ec6fbded0c61afe6f84e3c2a43e6d656791d95747d6d28b73eff1af64108c434 \ + --hash=sha256:eec73a005443061f4759b71a056f745e3b000dc0dc125c9f20560232dfbcbd14 \ + --hash=sha256:f3d812daffd0c2d2794756bd45a353f89e55dc8f91eb2fc840c51b9f6be62667 \ + --hash=sha256:f4b1615adf67bd8bb71f3464146a6f9949972d06d21a4f5e87e73f6464d97f57 \ + --hash=sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -r python/requirements/cloud-requirements.txt gymnasium==1.0.0 \ --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad @@ -4949,5 +5001,4 @@ zipp==3.19.2 \ # The following packages were excluded from the output: # ray -# grpcio-tools # setuptools diff --git a/python/requirements_compiled_rayllm_test_py311_cu128.txt b/python/requirements_compiled_rayllm_test_py311_cu128.txt index 56592053ec4f..b50bdfd3c41b 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 @@ -1217,6 +1217,58 @@ grpcio==1.66.2 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # grpcio-tools +grpcio-tools==1.62.3 \ + --hash=sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133 \ + --hash=sha256:0a8c0c4724ae9c2181b7dbc9b186df46e4f62cb18dc184e46d06c0ebeccf569e \ + --hash=sha256:0cb3a3436ac119cbd37a7d3331d9bdf85dad21a6ac233a3411dff716dcbf401e \ + --hash=sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7 \ + --hash=sha256:11f363570dea661dde99e04a51bd108a5807b5df32a6f8bdf4860e34e94a4dbf \ + --hash=sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa \ + --hash=sha256:1c989246c2aebc13253f08be32538a4039a64e12d9c18f6d662d7aee641dc8b5 \ + --hash=sha256:1da38070738da53556a4b35ab67c1b9884a5dd48fa2f243db35dc14079ea3d0c \ + --hash=sha256:27cd9ef5c5d68d5ed104b6dcb96fe9c66b82050e546c9e255716903c3d8f0373 \ + --hash=sha256:2e02d3b96f2d0e4bab9ceaa30f37d4f75571e40c6272e95364bff3125a64d184 \ + --hash=sha256:2f968b049c2849540751ec2100ab05e8086c24bead769ca734fdab58698408c1 \ + --hash=sha256:350a80485e302daaa95d335a931f97b693e170e02d43767ab06552c708808950 \ + --hash=sha256:3eae6ea76d62fcac091e1f15c2dcedf1dc3f114f8df1a972a8a0745e89f4cf61 \ + --hash=sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0 \ + --hash=sha256:5782883a27d3fae8c425b29a9d3dcf5f47d992848a1b76970da3b5a28d424b26 \ + --hash=sha256:6a56d344b0bab30bf342a67e33d386b0b3c4e65868ffe93c341c51e1a8853ca5 \ + --hash=sha256:6c3064610826f50bd69410c63101954676edc703e03f9e8f978a135f1aaf97c1 \ + --hash=sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23 \ + --hash=sha256:710fecf6a171dcbfa263a0a3e7070e0df65ba73158d4c539cec50978f11dad5d \ + --hash=sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833 \ + --hash=sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492 \ + --hash=sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43 \ + --hash=sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7 \ + --hash=sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3 \ + --hash=sha256:8c5d22b252dcef11dd1e0fbbe5bbfb9b4ae048e8880d33338215e8ccbdb03edc \ + --hash=sha256:8e62cc7164b0b7c5128e637e394eb2ef3db0e61fc798e80c301de3b2379203ed \ + --hash=sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a \ + --hash=sha256:ace43b26d88a58dcff16c20d23ff72b04d0a415f64d2820f4ff06b1166f50557 \ + --hash=sha256:b47d0dda1bdb0a0ba7a9a6de88e5a1ed61f07fad613964879954961e36d49193 \ + --hash=sha256:b77f9f9cee87cd798f0fe26b7024344d1b03a7cd2d2cba7035f8433b13986325 \ + --hash=sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b \ + --hash=sha256:bfda6ee8990997a9df95c5606f3096dae65f09af7ca03a1e9ca28f088caca5cf \ + --hash=sha256:c3a1ac9d394f8e229eb28eec2e04b9a6f5433fa19c9d32f1cb6066e3c5114a1d \ + --hash=sha256:c8ad5cce554e2fcaf8842dee5d9462583b601a3a78f8b76a153c38c963f58c10 \ + --hash=sha256:ca246dffeca0498be9b4e1ee169b62e64694b0f92e6d0be2573e65522f39eea9 \ + --hash=sha256:ca4f5eeadbb57cf03317d6a2857823239a63a59cc935f5bd6cf6e8b7af7a7ecc \ + --hash=sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5 \ + --hash=sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5 \ + --hash=sha256:dc9ad9950119d8ae27634e68b7663cc8d340ae535a0f80d85a55e56a6973ab1f \ + --hash=sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc \ + --hash=sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f \ + --hash=sha256:e18e15287c31baf574fcdf8251fb7f997d64e96c6ecf467906e576da0a079af6 \ + --hash=sha256:ec279dcf3518201fc592c65002754f58a6b542798cd7f3ecd4af086422f33f29 \ + --hash=sha256:ec6fbded0c61afe6f84e3c2a43e6d656791d95747d6d28b73eff1af64108c434 \ + --hash=sha256:eec73a005443061f4759b71a056f745e3b000dc0dc125c9f20560232dfbcbd14 \ + --hash=sha256:f3d812daffd0c2d2794756bd45a353f89e55dc8f91eb2fc840c51b9f6be62667 \ + --hash=sha256:f4b1615adf67bd8bb71f3464146a6f9949972d06d21a4f5e87e73f6464d97f57 \ + --hash=sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -r python/requirements/cloud-requirements.txt gymnasium==1.0.0 \ --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad @@ -4839,5 +4891,4 @@ zipp==3.19.2 \ # The following packages were excluded from the output: # ray -# grpcio-tools # setuptools From 7b7b01f8a61ef85863f7ab09de13b583e405b7ec Mon Sep 17 00:00:00 2001 From: Sampan S Nayak Date: Tue, 12 Aug 2025 18:45:16 +0530 Subject: [PATCH 047/634] [core] fix test_unpickleable_stacktrace test on windows (#55523) fix `test_traceback::test_unpickleable_stacktrace()` test case on windows by scrubbing logs related to `ray._raylet` before assertion ci run with failure: https://buildkite.com/ray-project/postmerge/builds/12165#01989bf5-4008-4098-a6ff-1963047047a6/3056-3537 --------- Signed-off-by: sampan Co-authored-by: sampan --- python/ray/tests/test_traceback.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/ray/tests/test_traceback.py b/python/ray/tests/test_traceback.py index bb1f37755714..4d8c20bc24ce 100644 --- a/python/ray/tests/test_traceback.py +++ b/python/ray/tests/test_traceback.py @@ -54,6 +54,13 @@ def scrub_traceback(ex): ) # Clean up underscore in stack trace, which is new in python 3.12 ex = re.sub("^\\s+~*\\^+~*\n", "", ex, flags=re.MULTILINE) + # Remove internal Cython frames from ray._raylet that can appear on Windows. + ex = re.sub( + r"^\s*File \"FILE\", line ZZ, in ray\._raylet\.[^\n]+\n", + "", + ex, + flags=re.MULTILINE, + ) return ex From 3e0b804643afa9e98d8a8655e0676c3ef01533a7 Mon Sep 17 00:00:00 2001 From: Markus <44006014+minosvasilias@users.noreply.github.com> Date: Tue, 12 Aug 2025 16:16:58 +0200 Subject: [PATCH 048/634] Upgrade commons-lang3 to fix CVE-2025-48924 (#55524) Fixes [CVE-2025-48924](https://nvd.nist.gov/vuln/detail/CVE-2025-48924) by upgrading the `commons-lang3` dependency. Vulnerabilities such as this one, even if not severe, can block deployments of ray within certain enterprise environments. Closes #55068 Signed-off-by: Markus --- java/dependencies.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/dependencies.bzl b/java/dependencies.bzl index 21c621af9b07..c19e82bb757f 100644 --- a/java/dependencies.bzl +++ b/java/dependencies.bzl @@ -18,7 +18,7 @@ def gen_java_deps(): "de.ruedigermoeller:fst:2.57", "javax.xml.bind:jaxb-api:2.3.0", "javax.activation:activation:1.1.1", - "org.apache.commons:commons-lang3:3.13.0", + "org.apache.commons:commons-lang3:3.18.0", "org.msgpack:msgpack-core:0.8.20", "org.ow2.asm:asm:6.0", "org.apache.logging.log4j:log4j-api:2.17.1", From a8fdb50e72013c5479f5c89f93e2dd4c26ef6098 Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Tue, 12 Aug 2025 08:44:56 -0700 Subject: [PATCH 049/634] [core] Refactoring rayletID to nodeID (#55474) Signed-off-by: joshlee --- .../java/io/ray/runtime/gcs/GcsClient.java | 4 +- python/ray/_private/state.py | 4 +- python/ray/autoscaler/_private/autoscaler.py | 26 +++---- .../ray/autoscaler/_private/load_metrics.py | 10 +-- .../components/ActorTable.component.test.tsx | 6 +- .../client/src/components/ActorTable.tsx | 12 +-- .../client/src/pages/actor/ActorDetail.tsx | 10 +-- .../client/src/pages/actor/ActorLogs.tsx | 4 +- .../pages/actor/hook/mockedUseActorList.ts | 10 +-- .../serve/ServeSystemActorDetailPage.tsx | 4 +- python/ray/dashboard/client/src/type/actor.ts | 2 +- .../ray/dashboard/modules/log/log_manager.py | 2 +- .../ray/dashboard/modules/node/datacenter.py | 6 +- .../ray/dashboard/modules/node/node_head.py | 8 +- .../modules/node/tests/test_actor.py | 4 +- python/ray/dashboard/state_aggregator.py | 4 +- python/ray/dashboard/utils.py | 2 +- python/ray/tests/test_advanced_2.py | 12 +-- python/ray/tests/test_autoscaler.py | 76 +++++++++---------- python/ray/tests/test_autoscaling_policy.py | 6 +- .../tests/test_resource_demand_scheduler.py | 58 +++++++------- python/ray/tests/test_state_api.py | 4 +- python/ray/tests/test_state_api_log.py | 2 +- src/fakes/ray/ipc/raylet_ipc_client.h | 2 +- src/mock/ray/core_worker/reference_count.h | 2 +- src/ray/common/bundle_spec.h | 2 +- src/ray/common/task/task_spec.cc | 2 +- src/ray/common/test/task_spec_test.cc | 2 +- src/ray/core_worker/actor_manager.cc | 4 +- src/ray/core_worker/core_worker.cc | 8 +- src/ray/core_worker/core_worker.h | 2 +- src/ray/core_worker/core_worker_process.cc | 20 ++--- .../core_worker/object_recovery_manager.cc | 2 +- src/ray/core_worker/reference_count.cc | 44 +++++------ src/ray/core_worker/reference_count.h | 22 +++--- src/ray/core_worker/task_manager.cc | 14 ++-- src/ray/core_worker/task_manager.h | 2 +- .../task_submission/actor_task_submitter.cc | 2 +- .../task_submission/normal_task_submitter.cc | 38 +++++----- .../task_submission/normal_task_submitter.h | 8 +- .../test/normal_task_submitter_test.cc | 32 ++++---- src/ray/core_worker/test/core_worker_test.cc | 2 +- src/ray/core_worker/test/lease_policy_test.cc | 20 ++--- .../test/object_recovery_manager_test.cc | 12 +-- .../core_worker/test/reference_count_test.cc | 2 +- src/ray/core_worker/test/task_manager_test.cc | 2 +- src/ray/flatbuffers/node_manager.fbs | 4 +- src/ray/gcs/gcs_server/gcs_actor_manager.cc | 14 ++-- src/ray/gcs/gcs_server/gcs_actor_manager.h | 2 +- src/ray/gcs/gcs_server/gcs_actor_scheduler.cc | 26 +++---- src/ray/gcs/gcs_server/gcs_actor_scheduler.h | 2 +- src/ray/gcs/gcs_server/gcs_job_manager.cc | 2 +- .../gcs_placement_group_scheduler.cc | 2 +- src/ray/gcs/gcs_server/gcs_server.cc | 6 +- src/ray/gcs/gcs_server/gcs_worker_manager.cc | 7 +- .../gcs_actor_manager_export_event_test.cc | 2 +- .../gcs_server/test/gcs_actor_manager_test.cc | 44 +++++------ .../test/gcs_actor_scheduler_mock_test.cc | 4 +- .../test/gcs_actor_scheduler_test.cc | 6 +- .../gcs_server/test/gcs_job_manager_test.cc | 8 +- .../gcs_server/test/gcs_server_test_util.h | 13 ++-- src/ray/gcs/pb_util.h | 2 +- src/ray/gcs/test/gcs_test_util.h | 8 +- src/ray/ipc/raylet_ipc_client.cc | 6 +- src/ray/ipc/raylet_ipc_client.h | 6 +- src/ray/object_manager/common.h | 6 +- src/ray/object_manager/object_manager.cc | 2 +- .../ownership_object_directory.cc | 2 +- src/ray/object_manager/plasma/plasma.fbs | 4 +- src/ray/object_manager/plasma/protocol.cc | 8 +- .../plasma/test/object_store_test.cc | 2 +- .../test/ownership_object_directory_test.cc | 2 +- .../test/spilled_object_test.cc | 28 +++---- src/ray/protobuf/common.proto | 2 +- src/ray/protobuf/gcs.proto | 2 +- src/ray/pubsub/test/subscriber_test.cc | 2 +- src/ray/raylet/local_object_manager.cc | 2 +- src/ray/raylet/local_task_manager.cc | 4 +- src/ray/raylet/node_manager.cc | 14 ++-- .../raylet/scheduling/cluster_task_manager.cc | 2 +- .../scheduling/cluster_task_manager_test.cc | 43 +++++------ src/ray/raylet/test/node_manager_test.cc | 2 +- .../rpc/node_manager/raylet_client_pool.cc | 34 ++++----- .../test/raylet_client_pool_test.cc | 6 +- .../rpc/test/core_worker_client_pool_test.cc | 6 +- src/ray/rpc/worker/core_worker_client.h | 2 +- src/ray/rpc/worker/core_worker_client_pool.cc | 4 +- 87 files changed, 428 insertions(+), 437 deletions(-) diff --git a/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClient.java b/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClient.java index 5a8d11f84bcf..65c7c629f388 100644 --- a/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClient.java +++ b/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClient.java @@ -122,10 +122,10 @@ public List getAllActorInfo(JobId jobId, ActorState actorState) { try { Gcs.ActorTableData info = Gcs.ActorTableData.parseFrom(result); UniqueId nodeId = UniqueId.NIL; - if (!info.getAddress().getRayletId().isEmpty()) { + if (!info.getAddress().getNodeId().isEmpty()) { nodeId = UniqueId.fromByteBuffer( - ByteBuffer.wrap(info.getAddress().getRayletId().toByteArray())); + ByteBuffer.wrap(info.getAddress().getNodeId().toByteArray())); } actorInfos.add( new ActorInfo( diff --git a/python/ray/_private/state.py b/python/ray/_private/state.py index 8c17054b5730..331ba4a1fea6 100644 --- a/python/ray/_private/state.py +++ b/python/ray/_private/state.py @@ -138,12 +138,12 @@ def _gen_actor_info(self, actor_table_data): "Address": { "IPAddress": actor_table_data.address.ip_address, "Port": actor_table_data.address.port, - "NodeID": binary_to_hex(actor_table_data.address.raylet_id), + "NodeID": binary_to_hex(actor_table_data.address.node_id), }, "OwnerAddress": { "IPAddress": actor_table_data.owner_address.ip_address, "Port": actor_table_data.owner_address.port, - "NodeID": binary_to_hex(actor_table_data.owner_address.raylet_id), + "NodeID": binary_to_hex(actor_table_data.owner_address.node_id), }, "State": gcs_pb2.ActorTableData.ActorState.DESCRIPTOR.values_by_number[ actor_table_data.state diff --git a/python/ray/autoscaler/_private/autoscaler.py b/python/ray/autoscaler/_private/autoscaler.py index 8fbd8fc60960..2051977bf655 100644 --- a/python/ray/autoscaler/_private/autoscaler.py +++ b/python/ray/autoscaler/_private/autoscaler.py @@ -635,10 +635,10 @@ def drain_nodes_via_gcs(self, provider_node_ids_to_drain: List[NodeID]): # For type checking, assert that this object has been instantitiated. assert self.provider - # The GCS expects Raylet ids in the request, rather than NodeProvider - # ids. To get the Raylet ids of the nodes to we're draining, we make + # The GCS expects Node ids in the request, rather than NodeProvider + # ids. To get the Node ids of the nodes to we're draining, we make # the following translations of identifiers: - # node provider node id -> ip -> raylet id + # node provider node id -> ip -> node id # Convert node provider node ids to ips. node_ips = set() @@ -660,29 +660,29 @@ def drain_nodes_via_gcs(self, provider_node_ids_to_drain: List[NodeID]): # Only attempt to drain connected nodes, i.e. nodes with ips in # LoadMetrics. - connected_node_ips = node_ips & self.load_metrics.raylet_id_by_ip.keys() + connected_node_ips = node_ips & self.load_metrics.node_id_by_ip.keys() - # Convert ips to Raylet ids. - # (The assignment ip->raylet_id is well-defined under current + # Convert ips to Node ids. + # (The assignment ip->node_id is well-defined under current # assumptions. See "use_node_id_as_ip" in monitor.py) - raylet_ids_to_drain = { - self.load_metrics.raylet_id_by_ip[ip] for ip in connected_node_ips + node_ids_to_drain = { + self.load_metrics.node_id_by_ip[ip] for ip in connected_node_ips } - if not raylet_ids_to_drain: + if not node_ids_to_drain: return - logger.info(f"Draining {len(raylet_ids_to_drain)} raylet(s).") + logger.info(f"Draining {len(node_ids_to_drain)} raylet(s).") try: # A successful response indicates that the GCS has marked the # desired nodes as "drained." The cloud provider can then terminate # the nodes without the GCS printing an error. # Check if we succeeded in draining all of the intended nodes by # looking at the RPC response. - drained_raylet_ids = set( - self.gcs_client.drain_nodes(raylet_ids_to_drain, timeout=5) + drained_node_ids = set( + self.gcs_client.drain_nodes(node_ids_to_drain, timeout=5) ) - failed_to_drain = raylet_ids_to_drain - drained_raylet_ids + failed_to_drain = node_ids_to_drain - drained_node_ids if failed_to_drain: self.prom_metrics.drain_node_exceptions.inc() logger.error(f"Failed to drain {len(failed_to_drain)} raylet(s).") diff --git a/python/ray/autoscaler/_private/load_metrics.py b/python/ray/autoscaler/_private/load_metrics.py index 2e083730aa30..ec94647bda9c 100644 --- a/python/ray/autoscaler/_private/load_metrics.py +++ b/python/ray/autoscaler/_private/load_metrics.py @@ -73,7 +73,7 @@ def __init__(self): self.last_heartbeat_time_by_ip = {} self.static_resources_by_ip = {} self.dynamic_resources_by_ip = {} - self.raylet_id_by_ip = {} + self.node_id_by_ip = {} self.waiting_bundles = [] self.infeasible_bundles = [] self.pending_placement_groups = [] @@ -85,12 +85,12 @@ def __bool__(self): """A load metrics instance is Falsey iff the autoscaler process has not received a resource message from the GCS. """ - return bool(self.raylet_id_by_ip) + return bool(self.node_id_by_ip) def update( self, ip: str, - raylet_id: bytes, + node_id: bytes, static_resources: Dict[str, Dict], dynamic_resources: Dict[str, Dict], node_idle_duration_s: float, @@ -100,7 +100,7 @@ def update( cluster_full_of_actors_detected: bool = False, ): self.static_resources_by_ip[ip] = static_resources - self.raylet_id_by_ip[ip] = raylet_id + self.node_id_by_ip[ip] = node_id self.cluster_full_of_actors_detected = cluster_full_of_actors_detected if not waiting_bundles: @@ -163,7 +163,7 @@ def prune(mapping, should_log): prune(self.ray_nodes_last_used_time_by_ip, should_log=True) prune(self.static_resources_by_ip, should_log=False) - prune(self.raylet_id_by_ip, should_log=False) + prune(self.node_id_by_ip, should_log=False) prune(self.dynamic_resources_by_ip, should_log=False) prune(self.last_heartbeat_time_by_ip, should_log=False) diff --git a/python/ray/dashboard/client/src/components/ActorTable.component.test.tsx b/python/ray/dashboard/client/src/components/ActorTable.component.test.tsx index cebe9825ed7a..ec58ad0f568c 100644 --- a/python/ray/dashboard/client/src/components/ActorTable.component.test.tsx +++ b/python/ray/dashboard/client/src/components/ActorTable.component.test.tsx @@ -10,7 +10,7 @@ const MOCK_ACTORS: { [actorId: string]: ActorDetail } = { actorId: "ACTOR_1", jobId: "01000000", address: { - rayletId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", + nodeId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", ipAddress: "172.31.11.178", port: 10003, workerId: "b8b276a03612644098ed7a929c3b0e50f5bde894eb0d8cab288fbb6d", @@ -61,7 +61,7 @@ const MOCK_ACTORS: { [actorId: string]: ActorDetail } = { actorId: "ACTOR_2", jobId: "01000000", address: { - rayletId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", + nodeId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", ipAddress: "172.31.11.178", port: 10003, workerId: "b8b276a03612644098ed7a929c3b0e50f5bde894eb0d8cab288fbb6d", @@ -120,7 +120,7 @@ describe("ActorTable", () => { ACTOR_2: { ...MOCK_ACTORS.ACTOR_2, address: { - rayletId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e2", + nodeId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e2", ipAddress: "172.31.11.178", port: 10003, workerId: "b8b276a03612644098ed7a929c3b0e50f5bde894eb0d8cab288fbb6e", diff --git a/python/ray/dashboard/client/src/components/ActorTable.tsx b/python/ray/dashboard/client/src/components/ActorTable.tsx index 5c8ebebfa8ed..ec4f1d745234 100644 --- a/python/ray/dashboard/client/src/components/ActorTable.tsx +++ b/python/ray/dashboard/client/src/components/ActorTable.tsx @@ -385,10 +385,10 @@ const ActorTable = ({ data-testid="nodeIdFilter" style={{ margin: 8, width: 150 }} options={Array.from( - new Set(Object.values(actors).map((e) => e.address?.rayletId)), + new Set(Object.values(actors).map((e) => e.address?.nodeId)), )} onInputChange={(_: any, value: string) => { - changeFilter("address.rayletId", value.trim()); + changeFilter("address.nodeId", value.trim()); }} renderInput={(params: TextFieldProps) => ( @@ -684,14 +684,14 @@ const ActorTable = ({ {address?.ipAddress ? address?.ipAddress : "-"} - {address?.rayletId ? ( - + {address?.nodeId ? ( + - {address?.rayletId} + {address?.nodeId} diff --git a/python/ray/dashboard/client/src/pages/actor/ActorDetail.tsx b/python/ray/dashboard/client/src/pages/actor/ActorDetail.tsx index 2e42384d1151..6fb16c1d6cdf 100644 --- a/python/ray/dashboard/client/src/pages/actor/ActorDetail.tsx +++ b/python/ray/dashboard/client/src/pages/actor/ActorDetail.tsx @@ -118,12 +118,12 @@ const ActorDetailPage = () => { }, { label: "Node ID", - content: actorDetail.address?.rayletId + content: actorDetail.address?.nodeId ? { - value: actorDetail.address?.rayletId, - copyableValue: actorDetail.address?.rayletId, - link: actorDetail.address.rayletId - ? generateNodeLink(actorDetail.address.rayletId) + value: actorDetail.address?.nodeId, + copyableValue: actorDetail.address?.nodeId, + link: actorDetail.address.nodeId + ? generateNodeLink(actorDetail.address.nodeId) : undefined, } : { value: "-" }, diff --git a/python/ray/dashboard/client/src/pages/actor/ActorLogs.tsx b/python/ray/dashboard/client/src/pages/actor/ActorLogs.tsx index 5191001d955f..23d9fe2ae438 100644 --- a/python/ray/dashboard/client/src/pages/actor/ActorLogs.tsx +++ b/python/ray/dashboard/client/src/pages/actor/ActorLogs.tsx @@ -13,7 +13,7 @@ export const ActorLogs = ({ actor: { actorId, pid, - address: { workerId, rayletId }, + address: { workerId, nodeId }, }, }: ActorLogsProps) => { const tabs: MultiTabLogViewerTabDetails[] = [ @@ -29,7 +29,7 @@ export const ActorLogs = ({ }, { title: "system", - nodeId: rayletId, + nodeId: nodeId, // TODO(aguo): Have API return the log file name. filename: `python-core-worker-${workerId}_${pid}.log`, }, diff --git a/python/ray/dashboard/client/src/pages/actor/hook/mockedUseActorList.ts b/python/ray/dashboard/client/src/pages/actor/hook/mockedUseActorList.ts index d4bb6061bd7d..384387228c4b 100644 --- a/python/ray/dashboard/client/src/pages/actor/hook/mockedUseActorList.ts +++ b/python/ray/dashboard/client/src/pages/actor/hook/mockedUseActorList.ts @@ -5,7 +5,7 @@ const MOCK_ACTORS: { [actorId: string]: Actor } = { actorId: "ACTOR_1", jobId: "01000000", address: { - rayletId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", + nodeId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", ipAddress: "172.31.11.178", port: 10003, workerId: "b8b276a03612644098ed7a929c3b0e50f5bde894eb0d8cab288fbb6d", @@ -28,7 +28,7 @@ const MOCK_ACTORS: { [actorId: string]: Actor } = { actorId: "ACTOR_2", jobId: "01000000", address: { - rayletId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", + nodeId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", ipAddress: "172.31.11.178", port: 10003, workerId: "b8b276a03612644098ed7a929c3b0e50f5bde894eb0d8cab288fbb6d", @@ -51,7 +51,7 @@ const MOCK_ACTORS: { [actorId: string]: Actor } = { actorId: "ACTOR_3", jobId: "01000000", address: { - rayletId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", + nodeId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", ipAddress: "172.31.11.178", port: 10003, workerId: "b8b276a03612644098ed7a929c3b0e50f5bde894eb0d8cab288fbb6d", @@ -74,7 +74,7 @@ const MOCK_ACTORS: { [actorId: string]: Actor } = { actorId: "ACTOR_4", jobId: "01000000", address: { - rayletId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", + nodeId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", ipAddress: "172.31.11.178", port: 10003, workerId: "b8b276a03612644098ed7a929c3b0e50f5bde894eb0d8cab288fbb6d", @@ -97,7 +97,7 @@ const MOCK_ACTORS: { [actorId: string]: Actor } = { actorId: "ACTOR_5", jobId: "01000000", address: { - rayletId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", + nodeId: "426854e68e4225b3941deaf03c8dcfcb1daacc69a92711d370dbb0e1", ipAddress: "172.31.11.178", port: 10003, workerId: "b8b276a03612644098ed7a929c3b0e50f5bde894eb0d8cab288fbb6d", diff --git a/python/ray/dashboard/client/src/pages/serve/ServeSystemActorDetailPage.tsx b/python/ray/dashboard/client/src/pages/serve/ServeSystemActorDetailPage.tsx index c326e2c86cd3..3e7c1ecdab5c 100644 --- a/python/ray/dashboard/client/src/pages/serve/ServeSystemActorDetailPage.tsx +++ b/python/ray/dashboard/client/src/pages/serve/ServeSystemActorDetailPage.tsx @@ -237,14 +237,14 @@ const ServeSystemActorLogs = ({ actor: { actorId, pid, - address: { workerId, rayletId }, + address: { workerId, nodeId }, }, systemLogFilePath, }: ServeSystemActorLogsProps) => { const tabs: MultiTabLogViewerTabDetails[] = [ { title: type === "controller" ? "Controller logs" : "proxy logs", - nodeId: rayletId, + nodeId: nodeId, filename: systemLogFilePath.startsWith("/") ? systemLogFilePath.substring(1) : systemLogFilePath, diff --git a/python/ray/dashboard/client/src/type/actor.ts b/python/ray/dashboard/client/src/type/actor.ts index 52c8527ab94e..b1242ed86c74 100644 --- a/python/ray/dashboard/client/src/type/actor.ts +++ b/python/ray/dashboard/client/src/type/actor.ts @@ -9,7 +9,7 @@ export enum ActorEnum { } export type Address = { - rayletId: string; + nodeId: string; ipAddress: string; port: number; workerId: string; diff --git a/python/ray/dashboard/modules/log/log_manager.py b/python/ray/dashboard/modules/log/log_manager.py index 783ee1c3a760..60f503888ffb 100644 --- a/python/ray/dashboard/modules/log/log_manager.py +++ b/python/ray/dashboard/modules/log/log_manager.py @@ -231,7 +231,7 @@ async def _resolve_actor_filename( "Actor is not scheduled yet." ) worker_id = WorkerID(worker_id_binary) - node_id_binary = actor_data.address.raylet_id + node_id_binary = actor_data.address.node_id if not node_id_binary: raise ValueError( f"Node ID for Actor ID {actor_id} not found. " diff --git a/python/ray/dashboard/modules/node/datacenter.py b/python/ray/dashboard/modules/node/datacenter.py index dcdd0c286060..6b32e4c545d5 100644 --- a/python/ray/dashboard/modules/node/datacenter.py +++ b/python/ray/dashboard/modules/node/datacenter.py @@ -198,11 +198,11 @@ async def get_actor_infos(cls, actor_ids: Optional[List[str]] = None): } @staticmethod - async def _get_actor_info(actor): + async def _get_actor_info(actor: Optional[dict]) -> Optional[dict]: if actor is None: return None - actor = dict(actor) + actor = actor.copy() worker_id = actor["address"]["workerId"] core_worker_stats = DataSource.core_worker_stats.get(worker_id, {}) actor_constructor = core_worker_stats.get( @@ -213,7 +213,7 @@ async def _get_actor_info(actor): # TODO(fyrestone): remove this, give a link from actor # info to worker info in front-end. - node_id = actor["address"]["rayletId"] + node_id = actor["address"]["nodeId"] pid = core_worker_stats.get("pid") node_physical_stats = DataSource.node_physical_stats.get(node_id, {}) actor_process_stats = None diff --git a/python/ray/dashboard/modules/node/node_head.py b/python/ray/dashboard/modules/node/node_head.py index bc696afc343f..d774ca7cac36 100644 --- a/python/ray/dashboard/modules/node/node_head.py +++ b/python/ray/dashboard/modules/node/node_head.py @@ -87,7 +87,7 @@ def _actor_table_data_to_dict(message): "parentId", "jobId", "workerId", - "rayletId", + "nodeId", "callerId", "taskId", "parentTaskId", @@ -576,7 +576,7 @@ async def _update_actors(self): # Update node actors and job actors. node_actors = defaultdict(dict) for actor_id_bytes, updated_actor_table in actor_dicts.items(): - node_id = updated_actor_table["address"]["rayletId"] + node_id = updated_actor_table["address"]["nodeId"] # Update only when node_id is not Nil. if node_id != actor_consts.NIL_NODE_ID: node_actors[node_id][actor_id_bytes] = updated_actor_table @@ -653,7 +653,7 @@ def _process_updated_actor_table( actor_table_data = actor actor_id = actor_table_data["actorId"] - node_id = actor_table_data["address"]["rayletId"] + node_id = actor_table_data["address"]["nodeId"] if actor_table_data["state"] == "DEAD": self._destroyed_actors_queue.append(actor_id) @@ -688,7 +688,7 @@ async def _cleanup_actors(self): actor_id = self._destroyed_actors_queue.popleft() if actor_id in DataSource.actors: actor = DataSource.actors.pop(actor_id) - node_id = actor["address"].get("rayletId") + node_id = actor["address"].get("nodeId") if node_id and node_id != actor_consts.NIL_NODE_ID: del DataSource.node_actors[node_id][actor_id] await asyncio.sleep(ACTOR_CLEANUP_FREQUENCY) diff --git a/python/ray/dashboard/modules/node/tests/test_actor.py b/python/ray/dashboard/modules/node/tests/test_actor.py index 3b8c4cbaf888..e4afeb0b818d 100644 --- a/python/ray/dashboard/modules/node/tests/test_actor.py +++ b/python/ray/dashboard/modules/node/tests/test_actor.py @@ -102,7 +102,7 @@ def get_placement_group_id(self): assert "Foo" in actor_response["className"] assert "address" in actor_response assert type(actor_response["address"]) is dict - assert actor_response["address"]["rayletId"] == node_id + assert actor_response["address"]["nodeId"] == node_id assert actor_response["state"] == "ALIVE" assert actor_response["name"] == "first" assert actor_response["numRestarts"] == "0" @@ -285,7 +285,7 @@ def actor_table_data_to_dict(message): "parentId", "jobId", "workerId", - "rayletId", + "nodeId", "callerId", "taskId", "parentTaskId", diff --git a/python/ray/dashboard/state_aggregator.py b/python/ray/dashboard/state_aggregator.py index 70b939ddde4f..a33dd4e7b3be 100644 --- a/python/ray/dashboard/state_aggregator.py +++ b/python/ray/dashboard/state_aggregator.py @@ -243,10 +243,10 @@ def transform(reply) -> ListApiResponse: result = [] for message in reply.worker_table_data: data = protobuf_message_to_dict( - message=message, fields_to_decode=["worker_id", "raylet_id"] + message=message, fields_to_decode=["worker_id", "node_id"] ) data["worker_id"] = data["worker_address"]["worker_id"] - data["node_id"] = data["worker_address"]["raylet_id"] + data["node_id"] = data["worker_address"]["node_id"] data["ip"] = data["worker_address"]["ip_address"] data["start_time_ms"] = int(data["start_time_ms"]) data["end_time_ms"] = int(data["end_time_ms"]) diff --git a/python/ray/dashboard/utils.py b/python/ray/dashboard/utils.py index 237992b0d2f0..73cfcd298ec8 100644 --- a/python/ray/dashboard/utils.py +++ b/python/ray/dashboard/utils.py @@ -360,7 +360,7 @@ def node_stats_to_dict( "parentTaskId", "sourceActorId", "callerId", - "rayletId", + "nodeId", "workerId", "placementGroupId", } diff --git a/python/ray/tests/test_advanced_2.py b/python/ray/tests/test_advanced_2.py index 20647e54169b..24714270a7fd 100644 --- a/python/ray/tests/test_advanced_2.py +++ b/python/ray/tests/test_advanced_2.py @@ -380,9 +380,9 @@ def h(): return ray._private.worker.global_worker.node.unique_id # The g tasks should be scheduled only on the second raylet. - raylet_ids = set(ray.get([g.remote() for _ in range(50)])) - assert len(raylet_ids) == 1 - assert list(raylet_ids)[0] == custom_resource_node.unique_id + node_ids = set(ray.get([g.remote() for _ in range(50)])) + assert len(node_ids) == 1 + assert list(node_ids)[0] == custom_resource_node.unique_id # Make sure that resource bookkeeping works when a task that uses a # custom resources gets blocked. @@ -460,9 +460,9 @@ def k(): assert len(set(ray.get([g.remote() for _ in range(500)]))) == 2 # The h tasks should be scheduled only on the second raylet. - raylet_ids = set(ray.get([h.remote() for _ in range(50)])) - assert len(raylet_ids) == 1 - assert list(raylet_ids)[0] == custom_resource_node.unique_id + node_ids = set(ray.get([h.remote() for _ in range(50)])) + assert len(node_ids) == 1 + assert list(node_ids)[0] == custom_resource_node.unique_id # Make sure that tasks with unsatisfied custom resource requirements do # not get scheduled. diff --git a/python/ray/tests/test_autoscaler.py b/python/ray/tests/test_autoscaler.py index 312c7fdcb17f..a728071528c2 100644 --- a/python/ray/tests/test_autoscaler.py +++ b/python/ray/tests/test_autoscaler.py @@ -107,7 +107,7 @@ def __init__(self, drain_node_outcome=DrainNodeOutcome.Succeeded): # Tracks how many times DrainNode returned a successful RPC response. self.drain_node_reply_success = 0 - def drain_nodes(self, raylet_ids_to_drain, timeout: int): + def drain_nodes(self, node_ids_to_drain, timeout: int): """Simulate NodeInfo stub's DrainNode call. Outcome determined by self.drain_outcome. @@ -132,28 +132,28 @@ def drain_nodes(self, raylet_ids_to_drain, timeout: int): DrainNodeOutcome.Succeeded, DrainNodeOutcome.FailedToFindIp, ]: - return raylet_ids_to_drain + return node_ids_to_drain elif self.drain_node_outcome == DrainNodeOutcome.NotAllDrained: # All but the last. - return raylet_ids_to_drain[:-1] + return node_ids_to_drain[:-1] else: # Shouldn't land here. assert False, "Possible drain node outcomes exhausted." -def mock_raylet_id() -> bytes: - """Random raylet id to pass to load_metrics.update.""" +def mock_node_id() -> bytes: + """Random node id to pass to load_metrics.update.""" return os.urandom(10) -def fill_in_raylet_ids(provider, load_metrics) -> None: - """Raylet ids for each ip are usually obtained by polling the GCS +def fill_in_node_ids(provider, load_metrics) -> None: + """Node ids for each ip are usually obtained by polling the GCS in monitor.py. For test purposes, we sometimes need to manually fill these fields with mocks. """ for node in provider.non_terminated_nodes({}): ip = provider.internal_ip(node) - load_metrics.raylet_id_by_ip[ip] = mock_raylet_id() + load_metrics.node_id_by_ip[ip] = mock_node_id() class MockAutoscaler(StandardAutoscaler): @@ -336,7 +336,7 @@ def update_nodes(self): class LoadMetricsTest(unittest.TestCase): def testHeartbeat(self): lm = LoadMetrics() - lm.update("1.1.1.1", mock_raylet_id(), {"CPU": 2}, {"CPU": 1}, 0) + lm.update("1.1.1.1", mock_node_id(), {"CPU": 2}, {"CPU": 1}, 0) lm.mark_active("2.2.2.2") assert "1.1.1.1" in lm.last_heartbeat_time_by_ip assert "2.2.2.2" in lm.last_heartbeat_time_by_ip @@ -344,13 +344,13 @@ def testHeartbeat(self): def testDebugString(self): lm = LoadMetrics() - lm.update("1.1.1.1", mock_raylet_id(), {"CPU": 2}, {"CPU": 0}, 0) + lm.update("1.1.1.1", mock_node_id(), {"CPU": 2}, {"CPU": 0}, 0) lm.update( - "2.2.2.2", mock_raylet_id(), {"CPU": 2, "GPU": 16}, {"CPU": 2, "GPU": 2}, 0 + "2.2.2.2", mock_node_id(), {"CPU": 2, "GPU": 16}, {"CPU": 2, "GPU": 2}, 0 ) lm.update( "3.3.3.3", - mock_raylet_id(), + mock_node_id(), { "memory": 1.05 * 1024 * 1024 * 1024, "object_store_memory": 2.1 * 1024 * 1024 * 1024, @@ -695,7 +695,7 @@ def testNodeTypeNameChange(self): == "ray.worker.old" ) - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) autoscaler.update() self.waitForNodes(2) events = autoscaler.event_summarizer.summary() @@ -1383,7 +1383,7 @@ def testTerminateOutdatedNodesGracefully(self): ) self.waitForNodes(10, tag_filters=WORKER_FILTER) - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) # Gradually scales down to meet target size, never going too low for _ in range(10): autoscaler.update() @@ -1546,7 +1546,7 @@ def _helperDynamicScaling( }, 1, ) - lm.update("172.0.0.0", mock_raylet_id(), {"CPU": 1}, {"CPU": 0}, 0) + lm.update("172.0.0.0", mock_node_id(), {"CPU": 1}, {"CPU": 0}, 0) autoscaler = MockAutoscaler( config_path, lm, @@ -1586,7 +1586,7 @@ def _helperDynamicScaling( new_config["available_node_types"]["worker"]["max_workers"] = 1 new_config["available_node_types"]["worker"]["min_workers"] = 1 self.write_config(new_config) - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) autoscaler.update() self.waitForNodes(1, tag_filters={TAG_RAY_NODE_KIND: NODE_KIND_WORKER}) @@ -1610,7 +1610,7 @@ def _helperDynamicScaling( tag_filters={TAG_RAY_NODE_KIND: NODE_KIND_WORKER}, )[0] lm.update( - worker_ip, mock_raylet_id(), {"CPU": 1}, {"CPU": 1}, DUMMY_IDLE_DURATION_S + worker_ip, mock_node_id(), {"CPU": 1}, {"CPU": 1}, DUMMY_IDLE_DURATION_S ) autoscaler.update() @@ -1682,7 +1682,7 @@ def _helperDynamicScaling( # self.waitForNodes(1) # lm.update( # head_ip, - # mock_raylet_id(), + # mock_node_id(), # {"CPU": 1}, # {"CPU": 0}, # waiting_bundles=[{"CPU": 1}] * 7, @@ -1708,7 +1708,7 @@ def _helperDynamicScaling( # # for being idle and instantly re-created due to resource demand! # lm.update( # head_ip, - # mock_raylet_id(), + # mock_node_id(), # {}, # {}, # waiting_bundles=[], @@ -1772,10 +1772,10 @@ def testUnmanagedNodes(self): autoscaler.update() self.waitForNodes(2) # This node has num_cpus=0 - lm.update(head_ip, mock_raylet_id(), {"CPU": 1}, {"CPU": 0}, 0) + lm.update(head_ip, mock_node_id(), {"CPU": 1}, {"CPU": 0}, 0) lm.update( unmanaged_ip, - mock_raylet_id(), + mock_node_id(), {"CPU": 0}, {"CPU": 0}, DUMMY_IDLE_DURATION_S, @@ -1785,7 +1785,7 @@ def testUnmanagedNodes(self): # 1 CPU task cannot be scheduled. lm.update( unmanaged_ip, - mock_raylet_id(), + mock_node_id(), {"CPU": 0}, {"CPU": 0}, DUMMY_IDLE_DURATION_S, @@ -1838,10 +1838,10 @@ def testUnmanagedNodes2(self): update_interval_s=0, ) - lm.update(head_ip, mock_raylet_id(), {"CPU": 1}, {"CPU": 0}, 0) + lm.update(head_ip, mock_node_id(), {"CPU": 1}, {"CPU": 0}, 0) lm.update( unmanaged_ip, - mock_raylet_id(), + mock_node_id(), {"CPU": 0}, {"CPU": 0}, DUMMY_IDLE_DURATION_S, @@ -1896,7 +1896,7 @@ def testDelayedLaunch(self): self.provider.ready_to_create.clear() lm.update( head_ip, - mock_raylet_id(), + mock_node_id(), {"CPU": 1}, {"CPU": 0}, 0, @@ -1922,7 +1922,7 @@ def testDelayedLaunch(self): new_config = copy.deepcopy(SMALL_CLUSTER) new_config["available_node_types"]["worker"]["max_workers"] = 1 self.write_config(new_config) - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) autoscaler.update() assert ( len( @@ -2076,7 +2076,7 @@ def testLaunchConfigChange(self): ] = "updated" self.write_config(new_config) self.provider.ready_to_create.clear() - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) for _ in range(5): autoscaler.update() self.waitForNodes(0, tag_filters=WORKER_FILTER) @@ -2100,7 +2100,7 @@ def testIgnoresCorruptedConfig(self): 1, ) lm = LoadMetrics() - lm.update("172.0.0.0", mock_raylet_id(), {"CPU": 1}, {"CPU": 0}, 0) + lm.update("172.0.0.0", mock_node_id(), {"CPU": 1}, {"CPU": 0}, 0) mock_metrics = Mock(spec=AutoscalerPrometheusMetrics()) autoscaler = MockAutoscaler( config_path, @@ -2146,7 +2146,7 @@ def testIgnoresCorruptedConfig(self): # Because one worker already started, the scheduler waits for its # resources to be updated before it launches the remaining min_workers. lm.update( - worker_ip, mock_raylet_id(), {"CPU": 1}, {"CPU": 1}, DUMMY_IDLE_DURATION_S + worker_ip, mock_node_id(), {"CPU": 1}, {"CPU": 1}, DUMMY_IDLE_DURATION_S ) autoscaler.update() self.waitForNodes(10, tag_filters={TAG_RAY_NODE_KIND: NODE_KIND_WORKER}) @@ -2279,7 +2279,7 @@ def testReportsConfigFailures(self): autoscaler.update() self.waitForNodes(2, tag_filters=WORKER_FILTER) self.provider.finish_starting_nodes() - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) autoscaler.update() try: self.waitForNodes( @@ -2412,7 +2412,7 @@ def testScaleDownMaxWorkers(self): config["available_node_types"]["p2.xlarge"]["min_workers"] = 6 # 5 config["available_node_types"]["p2.xlarge"]["max_workers"] = 6 self.write_config(config) - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) autoscaler.update() events = autoscaler.event_summarizer.summary() self.waitFor(lambda: autoscaler.pending_launches.value == 0) @@ -2437,7 +2437,7 @@ def testScaleDownMaxWorkers(self): def testFalseyLoadMetrics(self): lm = LoadMetrics() assert not lm - lm.update("172.0.0.0", mock_raylet_id(), {"CPU": 1}, {"CPU": 0}, 0) + lm.update("172.0.0.0", mock_node_id(), {"CPU": 1}, {"CPU": 0}, 0) assert lm def testRecoverUnhealthyWorkers(self): @@ -2571,7 +2571,7 @@ def unhealthyWorkerHelper(self, disable_liveness_check: bool): autoscaler.disable_node_updaters = True # Reduce min_workers to 1 autoscaler.config["available_node_types"]["worker"]["min_workers"] = 1 - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) if disable_liveness_check: # We've disabled the liveness check, so the unhealthy node should stick @@ -2673,7 +2673,7 @@ def testTerminateUnhealthyWorkers2(self): # Mark nodes unhealthy. for ip in ips: lm.last_heartbeat_time_by_ip[ip] = 0 - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) autoscaler.update() # Unhealthy nodes are gone. self.waitForNodes(0, tag_filters=WORKER_FILTER) @@ -3408,7 +3408,7 @@ def terminate_worker_zero(): ), "Node zero still non-terminated." assert not self.provider.is_terminated("1"), "Node one terminated prematurely." - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) autoscaler.update() # Failed updates processed are now processed. assert ( @@ -3436,7 +3436,7 @@ def terminate_worker_zero(): ), events # Should get two new nodes after the next update. - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) autoscaler.update() self.waitForNodes(2) assert set(NonTerminatedNodes(self.provider).worker_ids) == { @@ -3619,7 +3619,7 @@ def testScaleDownIdleTimeOut(self): worker_ip = self.provider.non_terminated_node_ips(WORKER_FILTER)[0] # Mark the node as idle - lm.update(worker_ip, mock_raylet_id(), {"CPU": 1}, {"CPU": 1}, 20) + lm.update(worker_ip, mock_node_id(), {"CPU": 1}, {"CPU": 1}, 20) autoscaler.update() assert self.provider.internal_ip("1") == worker_ip events = autoscaler.event_summarizer.summary() @@ -3691,7 +3691,7 @@ def testDontScaleDownIdleTimeOutForPlacementGroups(self): worker_ip = self.provider.non_terminated_node_ips(WORKER_FILTER)[0] lm.update( worker_ip, - mock_raylet_id(), + mock_node_id(), {"CPU": 1}, {"CPU": 1}, 20, # idle for 20 seconds, which is longer than the idle_timeout_minutes. diff --git a/python/ray/tests/test_autoscaling_policy.py b/python/ray/tests/test_autoscaling_policy.py index 34a7a5944e2b..32f44e217555 100644 --- a/python/ray/tests/test_autoscaling_policy.py +++ b/python/ray/tests/test_autoscaling_policy.py @@ -15,7 +15,7 @@ MockProvider, MockProcessRunner, MockGcsClient, - mock_raylet_id, + mock_node_id, MockAutoscaler, ) from ray.tests.test_resource_demand_scheduler import MULTI_WORKER_CLUSTER @@ -83,7 +83,7 @@ def __init__(self, resources, in_cluster, node_type, start_time): self.in_cluster = in_cluster self.node_type = node_type self.start_time = start_time - self.raylet_id = mock_raylet_id() + self.node_id = mock_node_id() def bundle_fits(self, bundle): if not self.in_cluster: @@ -370,7 +370,7 @@ def run_autoscaler(self): continue self.load_metrics.update( ip=ip, - raylet_id=node.raylet_id, + node_id=node.node_id, static_resources=node.total_resources, dynamic_resources=node.available_resources, node_idle_duration_s=0, diff --git a/python/ray/tests/test_resource_demand_scheduler.py b/python/ray/tests/test_resource_demand_scheduler.py index 69e4fc478ac0..99f77f2a812b 100644 --- a/python/ray/tests/test_resource_demand_scheduler.py +++ b/python/ray/tests/test_resource_demand_scheduler.py @@ -62,8 +62,8 @@ MockGcsClient, MockProcessRunner, MockProvider, - fill_in_raylet_ids, - mock_raylet_id, + fill_in_node_ids, + mock_node_id, ) from functools import partial @@ -1775,7 +1775,7 @@ def testResourceDemandVector(self): lm = LoadMetrics() lm.update( "1.1.1.1", - mock_raylet_id(), + mock_node_id(), {"CPU": 2}, {"CPU": 1}, 0, @@ -1800,7 +1800,7 @@ def testPlacementGroupLoad(self): ] lm.update( "1.1.1.1", - mock_raylet_id(), + mock_node_id(), {}, {}, DUMMY_IDLE_DURATION_S, @@ -1825,7 +1825,7 @@ def testSummary(self): ] lm.update( "1.1.1.1", - mock_raylet_id(), + mock_node_id(), { "CPU": 64, "memory": 1000 * 1024 * 1024, @@ -1840,7 +1840,7 @@ def testSummary(self): ) lm.update( "1.1.1.2", - mock_raylet_id(), + mock_node_id(), { "CPU": 64, "GPU": 8, @@ -1855,14 +1855,14 @@ def testSummary(self): ) lm.update( "1.1.1.3", - mock_raylet_id(), + mock_node_id(), {"CPU": 64, "GPU": 8, "accelerator_type:V100": 1}, {"CPU": 0, "GPU": 0, "accelerator_type:V100": 0.92}, 0, ) lm.update( "1.1.1.4", - mock_raylet_id(), + mock_node_id(), {"CPU": 2}, {"CPU": 2}, DUMMY_IDLE_DURATION_S, @@ -2077,9 +2077,9 @@ def testSummary(self): self.waitForNodes(3) for ip in self.provider.non_terminated_node_ips({}): - lm.update(ip, mock_raylet_id(), {"CPU": 2}, {"CPU": 0}, 0) + lm.update(ip, mock_node_id(), {"CPU": 2}, {"CPU": 0}, 0) - lm.update(head_ip, mock_raylet_id(), {"CPU": 16}, {"CPU": 1}, 0) + lm.update(head_ip, mock_node_id(), {"CPU": 16}, {"CPU": 1}, 0) autoscaler.update() while True: @@ -2098,7 +2098,7 @@ def testSummary(self): lm.update( head_ip, - mock_raylet_id(), + mock_node_id(), {"CPU": 16}, {"CPU": 1}, 0, @@ -2281,7 +2281,7 @@ def testPlacementGroup(self): ] lm.update( head_ip, - mock_raylet_id(), + mock_node_id(), {"CPU": 16}, {"CPU": 16}, DUMMY_IDLE_DURATION_S, @@ -2366,7 +2366,7 @@ def testScaleUpMinWorkers(self): # min workers. for node_id in self.provider.non_terminated_nodes({}): lm.ray_nodes_last_used_time_by_ip[self.provider.internal_ip(node_id)] = -60 - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) autoscaler.update() self.waitForNodes(3) @@ -2415,12 +2415,12 @@ def testScaleUpIgnoreUsed(self): ) autoscaler.update() self.waitForNodes(1) - lm.update(head_ip, mock_raylet_id(), {"CPU": 4, "GPU": 1}, {}, 0) + lm.update(head_ip, mock_node_id(), {"CPU": 4, "GPU": 1}, {}, 0) self.waitForNodes(1) lm.update( head_ip, - mock_raylet_id(), + mock_node_id(), {"CPU": 4, "GPU": 1}, {"GPU": 0}, 0, @@ -2600,7 +2600,7 @@ def testScaleUpLoadMetrics(self): autoscaler.update() lm.update( "1.2.3.4", - mock_raylet_id(), + mock_node_id(), {}, {}, DUMMY_IDLE_DURATION_S, @@ -2641,7 +2641,7 @@ def testCommandPassing(self): 1, ) lm = LoadMetrics() - lm.update("172.0.0.0", mock_raylet_id(), {"CPU": 0}, {"CPU": 0}, 0) + lm.update("172.0.0.0", mock_node_id(), {"CPU": 0}, {"CPU": 0}, 0) autoscaler = MockAutoscaler( config_path, lm, @@ -2821,7 +2821,7 @@ def testUpdateConfig(self): config["available_node_types"]["m4.large"]["min_workers"] = 0 config["available_node_types"]["m4.large"]["node_config"]["field_changed"] = 1 config_path = self.write_config(config) - fill_in_raylet_ids(self.provider, lm) + fill_in_node_ids(self.provider, lm) autoscaler.update() self.waitForNodes(0, tag_filters={TAG_RAY_NODE_KIND: NODE_KIND_WORKER}) @@ -2915,7 +2915,7 @@ def testRequestResourcesIdleTimeout(self): autoscaler.provider.mock_nodes[node_id].state = "unterminatable" lm.update( node_ip, - mock_raylet_id(), + mock_node_id(), config["available_node_types"]["def_worker"]["resources"], config["available_node_types"]["def_worker"]["resources"], DUMMY_IDLE_DURATION_S, @@ -2930,7 +2930,7 @@ def testRequestResourcesIdleTimeout(self): autoscaler.load_metrics.set_resource_requests([{"CPU": 0.2, "WORKER": 1.0}]) lm.update( node_ip, - mock_raylet_id(), + mock_node_id(), config["available_node_types"]["def_worker"]["resources"], {}, 0, @@ -2940,7 +2940,7 @@ def testRequestResourcesIdleTimeout(self): self.waitForNodes(2, tag_filters={TAG_RAY_NODE_KIND: NODE_KIND_WORKER}) lm.update( node_ip, - mock_raylet_id(), + mock_node_id(), config["available_node_types"]["def_worker"]["resources"], config["available_node_types"]["def_worker"]["resources"], DUMMY_IDLE_DURATION_S, @@ -2954,7 +2954,7 @@ def testRequestResourcesIdleTimeout(self): assert autoscaler.provider.mock_nodes[node_id].state == "unterminatable" lm.update( "172.0.0.2", - mock_raylet_id(), + mock_node_id(), config["available_node_types"]["def_worker"]["resources"], config["available_node_types"]["def_worker"]["resources"], DUMMY_IDLE_DURATION_S, @@ -3023,7 +3023,7 @@ def testRequestResourcesRaceConditionsLong(self): autoscaler.provider.mock_nodes[node_id].state = "unterminatable" lm.update( node_ip, - mock_raylet_id(), + mock_node_id(), config["available_node_types"]["def_worker"]["resources"], config["available_node_types"]["def_worker"]["resources"], DUMMY_IDLE_DURATION_S, @@ -3041,7 +3041,7 @@ def testRequestResourcesRaceConditionsLong(self): autoscaler.load_metrics.set_resource_requests([{"CPU": 0.2, "WORKER": 1.0}] * 3) lm.update( node_ip, - mock_raylet_id(), + mock_node_id(), config["available_node_types"]["def_worker"]["resources"], {}, 0, @@ -3053,21 +3053,21 @@ def testRequestResourcesRaceConditionsLong(self): lm.update( "172.0.0.2", - mock_raylet_id(), + mock_node_id(), config["available_node_types"]["def_worker"]["resources"], config["available_node_types"]["def_worker"]["resources"], DUMMY_IDLE_DURATION_S, ) lm.update( "172.0.0.3", - mock_raylet_id(), + mock_node_id(), config["available_node_types"]["def_worker"]["resources"], config["available_node_types"]["def_worker"]["resources"], DUMMY_IDLE_DURATION_S, ) lm.update( node_ip, - mock_raylet_id(), + mock_node_id(), config["available_node_types"]["def_worker"]["resources"], {}, 0, @@ -3174,7 +3174,7 @@ def testRequestResourcesRaceConditionWithResourceDemands(self): ) lm.update( "127.0.0.0", - mock_raylet_id(), + mock_node_id(), {"CPU": 2, "GPU": 1}, {"CPU": 2}, 0, @@ -3186,7 +3186,7 @@ def testRequestResourcesRaceConditionWithResourceDemands(self): self.waitForNodes(2) lm.update( "127.0.0.0", - mock_raylet_id(), + mock_node_id(), {"CPU": 2, "GPU": 1}, {"CPU": 2}, 0, diff --git a/python/ray/tests/test_state_api.py b/python/ray/tests/test_state_api.py index 9d9396ac0552..8c451f72b587 100644 --- a/python/ray/tests/test_state_api.py +++ b/python/ray/tests/test_state_api.py @@ -171,7 +171,7 @@ def generate_actor_data(id, state=ActorTableData.ActorState.ALIVE, class_name="c name="abc", pid=1234, class_name=class_name, - address=Address(raylet_id=id, ip_address="127.0.0.1", port=124, worker_id=id), + address=Address(node_id=id, ip_address="127.0.0.1", port=124, worker_id=id), job_id=b"123", node_id=None, ray_namespace="", @@ -208,7 +208,7 @@ def generate_worker_data( ): return WorkerTableData( worker_address=Address( - raylet_id=id, ip_address="127.0.0.1", port=124, worker_id=id + node_id=id, ip_address="127.0.0.1", port=124, worker_id=id ), is_alive=True, timestamp=1234, diff --git a/python/ray/tests/test_state_api_log.py b/python/ray/tests/test_state_api_log.py index 3a874fe7dd76..61554b3b90a3 100644 --- a/python/ray/tests/test_state_api_log.py +++ b/python/ray/tests/test_state_api_log.py @@ -90,7 +90,7 @@ async def generate_actor_data(id, node_id, worker_id): pid=1234, class_name="class", address=Address( - raylet_id=node_id.binary(), + node_id=node_id.binary(), ip_address="127.0.0.1", port=1234, worker_id=worker_id, diff --git a/src/fakes/ray/ipc/raylet_ipc_client.h b/src/fakes/ray/ipc/raylet_ipc_client.h index 29fb02135bff..dc7b0ca92bc0 100644 --- a/src/fakes/ray/ipc/raylet_ipc_client.h +++ b/src/fakes/ray/ipc/raylet_ipc_client.h @@ -29,7 +29,7 @@ class FakeRayletIpcClient : public RayletIpcClientInterface { const std::string &ip_address, const std::string &serialized_job_config, const StartupToken &startup_token, - NodeID *raylet_id, + NodeID *node_id, int *assigned_port) override { return Status::OK(); } diff --git a/src/mock/ray/core_worker/reference_count.h b/src/mock/ray/core_worker/reference_count.h index 9efc65afc25d..bf02c1bc987a 100644 --- a/src/mock/ray/core_worker/reference_count.h +++ b/src/mock/ray/core_worker/reference_count.h @@ -39,7 +39,7 @@ class MockReferenceCounter : public ReferenceCounterInterface { const int64_t object_size, bool is_reconstructable, bool add_local_ref, - const std::optional &pinned_at_raylet_id, + const std::optional &pinned_at_node_id, rpc::TensorTransport tensor_transport)); MOCK_METHOD2(AddObjectOutOfScopeOrFreedCallback, diff --git a/src/ray/common/bundle_spec.h b/src/ray/common/bundle_spec.h index 5f77cbb7650d..63c9eea454b9 100644 --- a/src/ray/common/bundle_spec.h +++ b/src/ray/common/bundle_spec.h @@ -28,7 +28,7 @@ namespace ray { -/// Arguments are the raylet ID to spill back to, the raylet's +/// Arguments are the node ID to spill back to, the raylet's /// address and the raylet's port. typedef std::function SpillbackBundleCallback; diff --git a/src/ray/common/task/task_spec.cc b/src/ray/common/task/task_spec.cc index 76cf5ae8574b..d231470f174a 100644 --- a/src/ray/common/task/task_spec.cc +++ b/src/ray/common/task/task_spec.cc @@ -495,7 +495,7 @@ std::string TaskSpecification::CallerWorkerIdBinary() const { } NodeID TaskSpecification::CallerNodeId() const { - return NodeID::FromBinary(message_->caller_address().raylet_id()); + return NodeID::FromBinary(message_->caller_address().node_id()); } // === Below are getter methods specific to actor tasks. diff --git a/src/ray/common/test/task_spec_test.cc b/src/ray/common/test/task_spec_test.cc index 67ed76daaf4b..a3b0cb5e05d1 100644 --- a/src/ray/common/test/task_spec_test.cc +++ b/src/ray/common/test/task_spec_test.cc @@ -238,7 +238,7 @@ TEST(TaskSpecTest, TestCallerAddress) { rpc::Address caller_address; NodeID caller_node_id = NodeID::FromRandom(); WorkerID caller_worker_id = WorkerID::FromRandom(); - caller_address.set_raylet_id(caller_node_id.Binary()); + caller_address.set_node_id(caller_node_id.Binary()); caller_address.set_worker_id(caller_worker_id.Binary()); TaskSpecBuilder task_spec_builder; task_spec_builder.SetCommonTaskSpec( diff --git a/src/ray/core_worker/actor_manager.cc b/src/ray/core_worker/actor_manager.cc index 4e6586dc10d1..418539515a25 100644 --- a/src/ray/core_worker/actor_manager.cc +++ b/src/ray/core_worker/actor_manager.cc @@ -214,8 +214,8 @@ void ActorManager::HandleActorStateNotification(const ActorID &actor_id, const rpc::ActorTableData &actor_data) { const auto &actor_state = rpc::ActorTableData::ActorState_Name(actor_data.state()); const auto worker_id = WorkerID::FromBinary(actor_data.address().worker_id()); - const auto raylet_id = NodeID::FromBinary(actor_data.address().raylet_id()); - RAY_LOG(INFO).WithField(actor_id).WithField(worker_id).WithField(raylet_id) + const auto node_id = NodeID::FromBinary(actor_data.address().node_id()); + RAY_LOG(INFO).WithField(actor_id).WithField(worker_id).WithField(node_id) << "received notification on actor, state: " << actor_state << ", ip address: " << actor_data.address().ip_address() << ", port: " << actor_data.address().port() diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index d9b8fcbd292d..22f3f7fd3513 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -790,7 +790,7 @@ void CoreWorker::RegisterToGcs(int64_t worker_launch_time_ms, } auto worker_data = std::make_shared(); - worker_data->mutable_worker_address()->set_raylet_id(rpc_address_.raylet_id()); + worker_data->mutable_worker_address()->set_node_id(rpc_address_.node_id()); worker_data->mutable_worker_address()->set_ip_address(rpc_address_.ip_address()); worker_data->mutable_worker_address()->set_port(rpc_address_.port()); worker_data->mutable_worker_address()->set_worker_id(worker_id.Binary()); @@ -1042,7 +1042,7 @@ Status CoreWorker::Put(const RayObject &object, object.GetSize(), /*is_reconstructable=*/false, /*add_local_ref=*/true, - NodeID::FromBinary(rpc_address_.raylet_id())); + NodeID::FromBinary(rpc_address_.node_id())); auto status = Put(object, contained_object_ids, *object_id, /*pin_object=*/true); if (!status.ok()) { RemoveLocalReference(*object_id); @@ -1126,7 +1126,7 @@ Status CoreWorker::CreateOwnedAndIncrementLocalRef( data_size + metadata->Size(), /*is_reconstructable=*/false, /*add_local_ref=*/true, - NodeID::FromBinary(rpc_address_.raylet_id())); + NodeID::FromBinary(rpc_address_.node_id())); } else { // Because in the remote worker's `HandleAssignObjectOwner`, // a `WaitForRefRemoved` RPC request will be sent back to @@ -4348,7 +4348,7 @@ void CoreWorker::HandleAssignObjectOwner(rpc::AssignObjectOwnerRequest request, request.object_size(), /*is_reconstructable=*/false, /*add_local_ref=*/false, - /*pinned_at_raylet_id=*/NodeID::FromBinary(borrower_address.raylet_id())); + /*pinned_at_node_id=*/NodeID::FromBinary(borrower_address.node_id())); reference_counter_->AddBorrowerAddress(object_id, borrower_address); memory_store_->Put(RayObject(rpc::ErrorType::OBJECT_IN_PLASMA), object_id); send_reply_callback(Status::OK(), nullptr, nullptr); diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index d2dc9610cac7..839b8a96f177 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -288,7 +288,7 @@ class CoreWorker { int64_t GetTaskDepth() const { return worker_context_->GetTaskDepth(); } - NodeID GetCurrentNodeId() const { return NodeID::FromBinary(rpc_address_.raylet_id()); } + NodeID GetCurrentNodeId() const { return NodeID::FromBinary(rpc_address_.node_id()); } /// Read the next index of a ObjectRefStream of generator_id. /// This API always return immediately. diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index 348140770fab..6ea8b140350e 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -215,7 +215,7 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( auto raylet_ipc_client = std::make_shared( io_service_, options.raylet_socket, /*num_retries=*/-1, /*timeout=*/-1); - NodeID local_raylet_id; + NodeID local_node_id; int assigned_port = 0; Status status = raylet_ipc_client->RegisterClient(worker_context->GetWorkerID(), options.worker_type, @@ -225,7 +225,7 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( options.node_ip_address, options.serialized_job_config, options.startup_token, - &local_raylet_id, + &local_node_id, &assigned_port); if (!status.ok()) { // Avoid using FATAL log or RAY_CHECK here because they may create a core dump file. @@ -243,7 +243,7 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( // so that the worker (java/python .etc) can retrieve and handle the error // instead of crashing. auto raylet_address = rpc::RayletClientPool::GenerateRayletAddress( - local_raylet_id, options.node_ip_address, options.node_manager_port); + local_node_id, options.node_ip_address, options.node_manager_port); auto local_raylet_rpc_client = std::make_shared( std::move(raylet_address), *client_call_manager, @@ -260,13 +260,13 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( core_worker_server->Run(); // Set our own address. - RAY_CHECK(!local_raylet_id.IsNil()); + RAY_CHECK(!local_node_id.IsNil()); rpc::Address rpc_address; rpc_address.set_ip_address(options.node_ip_address); rpc_address.set_port(core_worker_server->GetPort()); - rpc_address.set_raylet_id(local_raylet_id.Binary()); + rpc_address.set_node_id(local_node_id.Binary()); rpc_address.set_worker_id(worker_context->GetWorkerID().Binary()); - RAY_LOG(INFO).WithField(worker_context->GetWorkerID()).WithField(local_raylet_id) + RAY_LOG(INFO).WithField(worker_context->GetWorkerID()).WithField(local_node_id) << "Initializing worker at address: " << BuildAddress(rpc_address.ip_address(), rpc_address.port()); @@ -488,7 +488,7 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( std::optional address_opt; if (auto node_info = core_worker->gcs_client_->Nodes().Get(node_id)) { auto &address = address_opt.emplace(); - address.set_raylet_id(node_info->node_id()); + address.set_node_id(node_info->node_id()); address.set_ip_address(node_info->node_manager_address()); address.set_port(node_info->node_manager_port()); } @@ -510,7 +510,7 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( std::move(lease_policy), memory_store, *task_manager, - local_raylet_id, + local_node_id, options.worker_type, RayConfig::instance().worker_lease_timeout_milliseconds(), actor_creator, @@ -571,7 +571,7 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( continue; } rpc::Address addr; - addr.set_raylet_id(node_info->node_id()); + addr.set_node_id(node_info->node_id()); addr.set_ip_address(node_info->node_manager_address()); addr.set_port(node_info->node_manager_port()); locations.push_back(std::move(addr)); @@ -587,7 +587,7 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( for (const auto &node_info : node_infos) { if (node_info.state() != rpc::GcsNodeInfo::DEAD) { rpc::Address addr; - addr.set_raylet_id(node_info.node_id()); + addr.set_node_id(node_info.node_id()); addr.set_ip_address(node_info.node_manager_address()); addr.set_port(node_info.node_manager_port()); locations.push_back(std::move(addr)); diff --git a/src/ray/core_worker/object_recovery_manager.cc b/src/ray/core_worker/object_recovery_manager.cc index 8aa30697d53e..238dd3634268 100644 --- a/src/ray/core_worker/object_recovery_manager.cc +++ b/src/ray/core_worker/object_recovery_manager.cc @@ -110,7 +110,7 @@ void ObjectRecoveryManager::PinExistingObjectCopy( std::vector other_locations) { // If a copy still exists, pin the object by sending a // PinObjectIDs RPC. - const auto node_id = NodeID::FromBinary(raylet_address.raylet_id()); + const auto node_id = NodeID::FromBinary(raylet_address.node_id()); RAY_LOG(DEBUG).WithField(object_id).WithField(node_id) << "Trying to pin copy of lost object at node"; diff --git a/src/ray/core_worker/reference_count.cc b/src/ray/core_worker/reference_count.cc index 0e8e842ffc8d..6f15095e3e24 100644 --- a/src/ray/core_worker/reference_count.cc +++ b/src/ray/core_worker/reference_count.cc @@ -195,7 +195,7 @@ void ReferenceCounter::AddOwnedObject(const ObjectID &object_id, const int64_t object_size, bool is_reconstructable, bool add_local_ref, - const std::optional &pinned_at_raylet_id, + const std::optional &pinned_at_node_id, rpc::TensorTransport tensor_transport) { absl::MutexLock lock(&mutex_); RAY_CHECK(AddOwnedObjectInternal(object_id, @@ -205,7 +205,7 @@ void ReferenceCounter::AddOwnedObject(const ObjectID &object_id, object_size, is_reconstructable, add_local_ref, - pinned_at_raylet_id, + pinned_at_node_id, tensor_transport)) << "Tried to create an owned object that already exists: " << object_id; } @@ -319,7 +319,7 @@ bool ReferenceCounter::AddOwnedObjectInternal( const int64_t object_size, bool is_reconstructable, bool add_local_ref, - const std::optional &pinned_at_raylet_id, + const std::optional &pinned_at_node_id, rpc::TensorTransport tensor_transport) { if (object_id_refs_.contains(object_id)) { return false; @@ -341,7 +341,7 @@ bool ReferenceCounter::AddOwnedObjectInternal( call_site, object_size, is_reconstructable, - pinned_at_raylet_id, + pinned_at_node_id, tensor_transport)) .first; if (!inner_ids.empty()) { @@ -349,9 +349,9 @@ bool ReferenceCounter::AddOwnedObjectInternal( // the inner objects until the outer object ID goes out of scope. AddNestedObjectIdsInternal(object_id, inner_ids, rpc_address_); } - if (pinned_at_raylet_id.has_value()) { + if (pinned_at_node_id.has_value()) { // We eagerly add the pinned location to the set of object locations. - AddObjectLocationInternal(it, pinned_at_raylet_id.value()); + AddObjectLocationInternal(it, pinned_at_node_id.value()); } reconstructable_owned_objects_.emplace_back(object_id); @@ -787,7 +787,7 @@ void ReferenceCounter::OnObjectOutOfScopeOrFreed(ReferenceTable::iterator it) { } void ReferenceCounter::UnsetObjectPrimaryCopy(ReferenceTable::iterator it) { - it->second.pinned_at_raylet_id.reset(); + it->second.pinned_at_node_id.reset(); if (it->second.spilled && !it->second.spilled_node_id.IsNil()) { it->second.spilled = false; it->second.spilled_url = ""; @@ -827,18 +827,18 @@ bool ReferenceCounter::AddObjectOutOfScopeOrFreedCallback( return true; } -void ReferenceCounter::ResetObjectsOnRemovedNode(const NodeID &raylet_id) { +void ReferenceCounter::ResetObjectsOnRemovedNode(const NodeID &node_id) { absl::MutexLock lock(&mutex_); for (auto it = object_id_refs_.begin(); it != object_id_refs_.end(); it++) { const auto &object_id = it->first; - if (it->second.pinned_at_raylet_id.value_or(NodeID::Nil()) == raylet_id || - it->second.spilled_node_id == raylet_id) { + if (it->second.pinned_at_node_id.value_or(NodeID::Nil()) == node_id || + it->second.spilled_node_id == node_id) { UnsetObjectPrimaryCopy(it); if (!it->second.OutOfScope(lineage_pinning_enabled_)) { objects_to_recover_.push_back(object_id); } } - RemoveObjectLocationInternal(it, raylet_id); + RemoveObjectLocationInternal(it, node_id); } } @@ -850,7 +850,7 @@ std::vector ReferenceCounter::FlushObjectsToRecover() { } void ReferenceCounter::UpdateObjectPinnedAtRaylet(const ObjectID &object_id, - const NodeID &raylet_id) { + const NodeID &node_id) { absl::MutexLock lock(&mutex_); auto it = object_id_refs_.find(object_id); if (it != object_id_refs_.end()) { @@ -861,17 +861,17 @@ void ReferenceCounter::UpdateObjectPinnedAtRaylet(const ObjectID &object_id, // The object is still in scope. Track the raylet location until the object // has gone out of scope or the raylet fails, whichever happens first. - if (it->second.pinned_at_raylet_id.has_value()) { + if (it->second.pinned_at_node_id.has_value()) { RAY_LOG(INFO).WithField(object_id) - << "Updating primary location for object to node " << raylet_id - << ", but it already has a primary location " << *it->second.pinned_at_raylet_id + << "Updating primary location for object to node " << node_id + << ", but it already has a primary location " << *it->second.pinned_at_node_id << ". This should only happen during reconstruction"; } // Only the owner tracks the location. RAY_CHECK(it->second.owned_by_us); if (!it->second.OutOfScope(lineage_pinning_enabled_)) { - if (!is_node_dead_(raylet_id)) { - it->second.pinned_at_raylet_id = raylet_id; + if (!is_node_dead_(node_id)) { + it->second.pinned_at_node_id = node_id; } else { UnsetObjectPrimaryCopy(it); objects_to_recover_.push_back(object_id); @@ -890,7 +890,7 @@ bool ReferenceCounter::IsPlasmaObjectPinnedOrSpilled(const ObjectID &object_id, if (it->second.owned_by_us) { *owned_by_us = true; *spilled = it->second.spilled; - *pinned_at = it->second.pinned_at_raylet_id.value_or(NodeID::Nil()); + *pinned_at = it->second.pinned_at_node_id.value_or(NodeID::Nil()); } return true; } @@ -1479,8 +1479,8 @@ std::optional ReferenceCounter::GetLocalityData( auto node_ids = it->second.locations; // Add location of the primary copy since the object must be there: either in memory or // spilled. - if (it->second.pinned_at_raylet_id.has_value()) { - node_ids.emplace(it->second.pinned_at_raylet_id.value()); + if (it->second.pinned_at_node_id.has_value()) { + node_ids.emplace(it->second.pinned_at_node_id.value()); } // We should only reach here if we have valid locality data to return. @@ -1566,7 +1566,7 @@ void ReferenceCounter::PushToLocationSubscribers(ReferenceTable::iterator it) { auto object_size = it->second.object_size; const auto &spilled_url = it->second.spilled_url; const auto &spilled_node_id = it->second.spilled_node_id; - const auto &optional_primary_node_id = it->second.pinned_at_raylet_id; + const auto &optional_primary_node_id = it->second.pinned_at_node_id; const auto &primary_node_id = optional_primary_node_id.value_or(NodeID::Nil()); RAY_LOG(DEBUG).WithField(object_id) << "Published message for object, " << locations.size() @@ -1610,7 +1610,7 @@ void ReferenceCounter::FillObjectInformationInternal( } object_info->set_spilled_url(it->second.spilled_url); object_info->set_spilled_node_id(it->second.spilled_node_id.Binary()); - auto primary_node_id = it->second.pinned_at_raylet_id.value_or(NodeID::Nil()); + auto primary_node_id = it->second.pinned_at_node_id.value_or(NodeID::Nil()); object_info->set_primary_node_id(primary_node_id.Binary()); object_info->set_pending_creation(it->second.pending_creation); object_info->set_did_spill(it->second.did_spill); diff --git a/src/ray/core_worker/reference_count.h b/src/ray/core_worker/reference_count.h index 1ede49834e94..c58cbe5af91a 100644 --- a/src/ray/core_worker/reference_count.h +++ b/src/ray/core_worker/reference_count.h @@ -56,7 +56,7 @@ class ReferenceCounterInterface { const int64_t object_size, bool is_reconstructable, bool add_local_ref, - const std::optional &pinned_at_raylet_id = std::optional(), + const std::optional &pinned_at_node_id = std::optional(), rpc::TensorTransport tensor_transport = rpc::TensorTransport::OBJECT_STORE) = 0; virtual bool AddObjectOutOfScopeOrFreedCallback( const ObjectID &object_id, @@ -188,7 +188,7 @@ class ReferenceCounter : public ReferenceCounterInterface, /// \param[in] add_local_ref Whether to initialize the local ref count to 1. /// This is used to ensure that the ref is considered in scope before the /// corresponding ObjectRef has been returned to the language frontend. - /// \param[in] pinned_at_raylet_id The primary location for the object, if it + /// \param[in] pinned_at_node_id The primary location for the object, if it /// is already known. This is only used for ray.put calls. /// \param[in] tensor_transport The transport used for the object. void AddOwnedObject( @@ -199,7 +199,7 @@ class ReferenceCounter : public ReferenceCounterInterface, const int64_t object_size, bool is_reconstructable, bool add_local_ref, - const std::optional &pinned_at_raylet_id = std::optional(), + const std::optional &pinned_at_node_id = std::optional(), rpc::TensorTransport tensor_transport = rpc::TensorTransport::OBJECT_STORE) override ABSL_LOCKS_EXCLUDED(mutex_); @@ -444,8 +444,8 @@ class ReferenceCounter : public ReferenceCounterInterface, /// Update the pinned location of an object stored in plasma. /// /// \param[in] object_id The object to update. - /// \param[in] raylet_id The raylet that is now pinning the object ID. - void UpdateObjectPinnedAtRaylet(const ObjectID &object_id, const NodeID &raylet_id) + /// \param[in] node_id The raylet that is now pinning the object ID. + void UpdateObjectPinnedAtRaylet(const ObjectID &object_id, const NodeID &node_id) ABSL_LOCKS_EXCLUDED(mutex_); /// Check whether the object is pinned at a remote plasma store node or @@ -473,7 +473,7 @@ class ReferenceCounter : public ReferenceCounterInterface, /// /// \param[in] node_id The node whose object store has been removed. /// \return The set of objects that were pinned on the given node. - void ResetObjectsOnRemovedNode(const NodeID &raylet_id); + void ResetObjectsOnRemovedNode(const NodeID &node_id); std::vector FlushObjectsToRecover(); @@ -652,16 +652,16 @@ class ReferenceCounter : public ReferenceCounterInterface, std::string call_site, int64_t object_size, bool is_reconstructable, - std::optional pinned_at_raylet_id, + std::optional pinned_at_node_id, rpc::TensorTransport tensor_transport) : call_site(std::move(call_site)), object_size(object_size), owner_address(std::move(owner_address)), - pinned_at_raylet_id(std::move(pinned_at_raylet_id)), + pinned_at_node_id(std::move(pinned_at_node_id)), tensor_transport(tensor_transport), owned_by_us(true), is_reconstructable(is_reconstructable), - pending_creation(!pinned_at_raylet_id.has_value()) {} + pending_creation(!pinned_at_node_id.has_value()) {} /// Constructor from a protobuf. This is assumed to be a message from /// another process, so the object defaults to not being owned by us. @@ -770,7 +770,7 @@ class ReferenceCounter : public ReferenceCounterInterface, /// If this object is owned by us and stored in plasma, and reference /// counting is enabled, then some raylet must be pinning the object value. /// This is the address of that raylet. - std::optional pinned_at_raylet_id; + std::optional pinned_at_node_id; /// TODO(kevin85421): Make tensor_transport a required field for all constructors. /// /// The transport used for the object. @@ -860,7 +860,7 @@ class ReferenceCounter : public ReferenceCounterInterface, const int64_t object_size, bool is_reconstructable, bool add_local_ref, - const std::optional &pinned_at_raylet_id, + const std::optional &pinned_at_node_id, rpc::TensorTransport tensor_transport = rpc::TensorTransport::OBJECT_STORE) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); diff --git a/src/ray/core_worker/task_manager.cc b/src/ray/core_worker/task_manager.cc index b41e0616ba29..0992f15ff2f9 100644 --- a/src/ray/core_worker/task_manager.cc +++ b/src/ray/core_worker/task_manager.cc @@ -267,7 +267,7 @@ std::vector TaskManager::AddPendingTask( -1, is_reconstructable, /*add_local_ref=*/true, - /*pinned_at_raylet_id=*/std::optional(), + /*pinned_at_node_id=*/std::optional(), /*tensor_transport=*/spec.TensorTransport()); } @@ -535,7 +535,7 @@ size_t TaskManager::NumPendingTasks() const { bool TaskManager::HandleTaskReturn(const ObjectID &object_id, const rpc::ReturnObject &return_object, - const NodeID &worker_raylet_id, + const NodeID &worker_node_id, bool store_in_plasma) { bool direct_return = false; reference_counter_.UpdateObjectSize(object_id, return_object.size()); @@ -548,7 +548,7 @@ bool TaskManager::HandleTaskReturn(const ObjectID &object_id, // NOTE(swang): We need to add the location of the object before marking // it as local in the in-memory store so that the data locality policy // will choose the right raylet for any queued dependent tasks. - reference_counter_.UpdateObjectPinnedAtRaylet(object_id, worker_raylet_id); + reference_counter_.UpdateObjectPinnedAtRaylet(object_id, worker_node_id); // Mark it as in plasma with a dummy object. in_memory_store_.Put(RayObject(rpc::ErrorType::OBJECT_IN_PLASMA), object_id); } else { @@ -815,7 +815,7 @@ bool TaskManager::HandleReportGeneratorItemReturns( reference_counter_.UpdateObjectPendingCreation(object_id, false); HandleTaskReturn(object_id, return_object, - NodeID::FromBinary(request.worker_addr().raylet_id()), + NodeID::FromBinary(request.worker_addr().node_id()), /*store_in_plasma=*/store_in_plasma_ids.contains(object_id)); } @@ -902,7 +902,7 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, } if (!HandleTaskReturn(object_id, return_object, - NodeID::FromBinary(worker_addr.raylet_id()), + NodeID::FromBinary(worker_addr.node_id()), store_in_plasma_ids.contains(object_id))) { if (first_execution) { dynamic_returns_in_plasma.push_back(object_id); @@ -915,7 +915,7 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, const auto object_id = ObjectID::FromBinary(return_object.object_id()); if (HandleTaskReturn(object_id, return_object, - NodeID::FromBinary(worker_addr.raylet_id()), + NodeID::FromBinary(worker_addr.node_id()), store_in_plasma_ids.contains(object_id))) { direct_return_ids.push_back(object_id); } @@ -1042,7 +1042,7 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, const auto &return_object = reply.return_objects(0); HandleTaskReturn(generator_return_id, return_object, - NodeID::FromBinary(worker_addr.raylet_id()), + NodeID::FromBinary(worker_addr.node_id()), store_in_plasma_ids.contains(generator_return_id)); } } diff --git a/src/ray/core_worker/task_manager.h b/src/ray/core_worker/task_manager.h index 1d841f21f284..25f5c90717c9 100644 --- a/src/ray/core_worker/task_manager.h +++ b/src/ray/core_worker/task_manager.h @@ -612,7 +612,7 @@ class TaskManager : public TaskManagerInterface { /// directly by value. bool HandleTaskReturn(const ObjectID &object_id, const rpc::ReturnObject &return_object, - const NodeID &worker_raylet_id, + const NodeID &worker_node_id, bool store_in_plasma) ABSL_LOCKS_EXCLUDED(mu_); /// Remove a lineage reference to this object ID. This should be called diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.cc b/src/ray/core_worker/task_submission/actor_task_submitter.cc index 8ff56cd4f150..506f07feb3b3 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.cc +++ b/src/ray/core_worker/task_submission/actor_task_submitter.cc @@ -621,7 +621,7 @@ void ActorTaskSubmitter::PushActorTask(ClientQueue &queue, }; task_manager_.MarkTaskWaitingForExecution(task_id, - NodeID::FromBinary(addr.raylet_id()), + NodeID::FromBinary(addr.node_id()), WorkerID::FromBinary(addr.worker_id())); queue.rpc_client->PushActorTask( std::move(request), skip_queue, std::move(wrapped_callback)); diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 47dcd21fe3a7..6ee82372d73c 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -112,7 +112,7 @@ void NormalTaskSubmitter::ReturnWorker(const rpc::Address &addr, bool worker_exiting, const SchedulingKey &scheduling_key) { RAY_LOG(DEBUG) << "Returning worker " << WorkerID::FromBinary(addr.worker_id()) - << " to raylet " << NodeID::FromBinary(addr.raylet_id()); + << " to raylet " << NodeID::FromBinary(addr.node_id()); auto &scheduling_key_entry = scheduling_key_entries_[scheduling_key]; RAY_CHECK(scheduling_key_entry.active_workers.size() >= 1); auto &lease_entry = worker_to_lease_entry_[addr]; @@ -232,14 +232,14 @@ std::shared_ptr NormalTaskSubmitter::GetOrConnectRayletCl const rpc::Address *raylet_address) { std::shared_ptr raylet_client; RAY_CHECK(raylet_address != nullptr); - if (NodeID::FromBinary(raylet_address->raylet_id()) != local_raylet_id_) { + if (NodeID::FromBinary(raylet_address->node_id()) != local_node_id_) { // A remote raylet was specified. Connect to the raylet if needed. - NodeID raylet_id = NodeID::FromBinary(raylet_address->raylet_id()); - auto it = remote_raylet_clients_.find(raylet_id); + NodeID node_id = NodeID::FromBinary(raylet_address->node_id()); + auto it = remote_raylet_clients_.find(node_id); if (it == remote_raylet_clients_.end()) { - RAY_LOG(INFO) << "Connecting to raylet " << raylet_id; + RAY_LOG(INFO) << "Connecting to raylet " << node_id; it = remote_raylet_clients_ - .emplace(raylet_id, + .emplace(node_id, raylet_client_pool_->GetOrConnectByAddress(*raylet_address)) .first; } @@ -345,7 +345,7 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli const TaskID task_id = resource_spec.TaskId(); const std::string task_name = resource_spec.GetName(); RAY_LOG(DEBUG) << "Requesting lease from raylet " - << NodeID::FromBinary(raylet_address->raylet_id()) << " for task " + << NodeID::FromBinary(raylet_address->node_id()) << " for task " << task_id; raylet_client->RequestWorkerLease( @@ -426,11 +426,11 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli // refreshed. RAY_CHECK(is_spillback); RequestNewWorkerIfNeeded(scheduling_key); - } else if (!reply.worker_address().raylet_id().empty()) { + } else if (!reply.worker_address().node_id().empty()) { // We got a lease for a worker. Add the lease client state and try to // assign work to the worker. RAY_LOG(DEBUG) << "Lease granted to task " << task_id << " from raylet " - << NodeID::FromBinary(reply.worker_address().raylet_id()) + << NodeID::FromBinary(reply.worker_address().node_id()) << " with worker " << WorkerID::FromBinary(reply.worker_address().worker_id()); @@ -450,10 +450,10 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli // The raylet redirected us to a different raylet to retry at. RAY_CHECK(!is_spillback); RAY_LOG(DEBUG) << "Redirect lease for task " << task_id << " from raylet " - << NodeID::FromBinary(raylet_address.raylet_id()) + << NodeID::FromBinary(raylet_address.node_id()) << " to raylet " << NodeID::FromBinary( - reply.retry_at_raylet_address().raylet_id()); + reply.retry_at_raylet_address().node_id()); RequestNewWorkerIfNeeded(scheduling_key, &reply.retry_at_raylet_address()); } @@ -464,7 +464,7 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli RAY_LOG_EVERY_MS(INFO, 30 * 1000) << "Retrying attempt to schedule task (id: " << task_id << " name: " << task_name - << ") at remote node (id: " << raylet_address.raylet_id() + << ") at remote node (id: " << raylet_address.node_id() << " ip: " << raylet_address.ip_address() << "). Try again " "on a local node. Error: " @@ -490,7 +490,7 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli // Grpc errors are not helpful at all. So we are overwriting it. std::stringstream ss; ss << "The worker failed to receive a response from the local raylet" - << "(id: " << NodeID::FromBinary(raylet_address.raylet_id()).Hex() + << "(id: " << NodeID::FromBinary(raylet_address.node_id()).Hex() << " ,ip: " << raylet_address.ip_address() << ") " << "because the raylet is " "unavailable (crashed)."; @@ -542,7 +542,7 @@ void NormalTaskSubmitter::PushNormalTask( const google::protobuf::RepeatedPtrField &assigned_resources) { RAY_LOG(DEBUG) << "Pushing task " << task_spec.TaskId() << " to worker " << WorkerID::FromBinary(addr.worker_id()) << " of raylet " - << NodeID::FromBinary(addr.raylet_id()); + << NodeID::FromBinary(addr.node_id()); auto task_id = task_spec.TaskId(); auto request = std::make_unique(); // NOTE(swang): CopyFrom is needed because if we use Swap here and the task @@ -552,7 +552,7 @@ void NormalTaskSubmitter::PushNormalTask( request->mutable_resource_mapping()->CopyFrom(assigned_resources); request->set_intended_worker_id(addr.worker_id()); task_manager_.MarkTaskWaitingForExecution(task_id, - NodeID::FromBinary(addr.raylet_id()), + NodeID::FromBinary(addr.node_id()), WorkerID::FromBinary(addr.worker_id())); client->PushNormalTask( std::move(request), @@ -566,7 +566,7 @@ void NormalTaskSubmitter::PushNormalTask( { RAY_LOG(DEBUG) << "Task " << task_id << " finished from worker " << WorkerID::FromBinary(addr.worker_id()) << " of raylet " - << NodeID::FromBinary(addr.raylet_id()); + << NodeID::FromBinary(addr.node_id()); absl::MutexLock lock(&mu_); executing_tasks_.erase(task_id); @@ -666,18 +666,18 @@ bool NormalTaskSubmitter::HandleGetTaskFailureCause( } else { RAY_LOG(WARNING) << "Failed to fetch task result with status " << get_task_failure_cause_reply_status.ToString() - << " node id: " << NodeID::FromBinary(addr.raylet_id()) + << " node id: " << NodeID::FromBinary(addr.node_id()) << " ip: " << addr.ip_address(); task_error_type = rpc::ErrorType::NODE_DIED; std::stringstream buffer; buffer << "Task failed due to the node (where this task was running) " << " was dead or unavailable.\n\nThe node IP: " << addr.ip_address() - << ", node ID: " << NodeID::FromBinary(addr.raylet_id()) << "\n\n" + << ", node ID: " << NodeID::FromBinary(addr.node_id()) << "\n\n" << "This can happen if the instance where the node was running failed, " << "the node was preempted, or raylet crashed unexpectedly " << "(e.g., due to OOM) etc.\n\n" << "To see node death information, use `ray list nodes --filter \"node_id=" - << NodeID::FromBinary(addr.raylet_id()) << "\"`, " + << NodeID::FromBinary(addr.node_id()) << "\"`, " << "or check Ray dashboard cluster page, or search the node ID in GCS log, " << "or use `ray logs raylet.out -ip " << addr.ip_address() << "`"; error_info = std::make_unique(); diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.h b/src/ray/core_worker/task_submission/normal_task_submitter.h index b882f9b93d88..f72628319465 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.h +++ b/src/ray/core_worker/task_submission/normal_task_submitter.h @@ -91,7 +91,7 @@ class NormalTaskSubmitter { std::unique_ptr lease_policy, std::shared_ptr store, TaskManagerInterface &task_manager, - NodeID local_raylet_id, + NodeID local_node_id, WorkerType worker_type, int64_t lease_timeout_ms, std::shared_ptr actor_creator, @@ -106,7 +106,7 @@ class NormalTaskSubmitter { resolver_(*store, task_manager, *actor_creator, tensor_transport_getter), task_manager_(task_manager), lease_timeout_ms_(lease_timeout_ms), - local_raylet_id_(local_raylet_id), + local_node_id_(local_node_id), worker_type_(worker_type), core_worker_client_pool_(std::move(core_worker_client_pool)), job_id_(job_id), @@ -270,9 +270,9 @@ class NormalTaskSubmitter { /// to the raylet. int64_t lease_timeout_ms_; - /// The local raylet ID. Used to make sure that we use the local lease client + /// The local node ID. Used to make sure that we use the local lease client /// if a remote raylet tells us to spill the task back to the local raylet. - const NodeID local_raylet_id_; + const NodeID local_node_id_; /// The type of this core worker process. const WorkerType worker_type_; diff --git a/src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc index 7e5a8b96bc2a..43821ecd5f94 100644 --- a/src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc @@ -313,7 +313,7 @@ class MockRayletClient : public FakeRayletClient { bool GrantWorkerLease( const std::string &address, int port, - const NodeID &retry_at_raylet_id, + const NodeID &retry_at_node_id, bool cancel = false, std::string worker_id = WorkerID::FromRandom().Binary(), bool reject = false, @@ -325,14 +325,14 @@ class MockRayletClient : public FakeRayletClient { reply.set_failure_type(failure_type); } else if (reject) { reply.set_rejected(true); - } else if (!retry_at_raylet_id.IsNil()) { + } else if (!retry_at_node_id.IsNil()) { reply.mutable_retry_at_raylet_address()->set_ip_address(address); reply.mutable_retry_at_raylet_address()->set_port(port); - reply.mutable_retry_at_raylet_address()->set_raylet_id(retry_at_raylet_id.Binary()); + reply.mutable_retry_at_raylet_address()->set_node_id(retry_at_node_id.Binary()); } else { reply.mutable_worker_address()->set_ip_address(address); reply.mutable_worker_address()->set_port(port); - reply.mutable_worker_address()->set_raylet_id(retry_at_raylet_id.Binary()); + reply.mutable_worker_address()->set_node_id(retry_at_node_id.Binary()); reply.mutable_worker_address()->set_worker_id(worker_id); } rpc::ClientCallback callback = PopCallbackInLock(); @@ -446,9 +446,7 @@ class MockActorCreator : public ActorCreatorInterface { class MockLeasePolicy : public LeasePolicyInterface { public: - void SetNodeID(NodeID node_id) { - fallback_rpc_address_.set_raylet_id(node_id.Binary()); - } + void SetNodeID(NodeID node_id) { fallback_rpc_address_.set_node_id(node_id.Binary()); } std::pair GetBestNodeForTask(const TaskSpecification &spec) { num_lease_policy_consults++; @@ -497,7 +495,7 @@ class NormalTaskSubmitterTest : public testing::Test { raylet_client_factory = nullptr, std::shared_ptr custom_memory_store = nullptr, int64_t lease_timeout_ms = kLongTimeout, - NodeID local_raylet_id = NodeID::Nil()) { + NodeID local_node_id = NodeID::Nil()) { if (custom_memory_store != nullptr) { store = custom_memory_store; } @@ -515,7 +513,7 @@ class NormalTaskSubmitterTest : public testing::Test { std::move(lease_policy), store, *task_manager, - local_raylet_id, + local_node_id, worker_type, lease_timeout_ms, actor_creator, @@ -1337,8 +1335,8 @@ TEST_F(NormalTaskSubmitterTest, TestSpillback) { ASSERT_EQ(remote_raylet_clients.size(), 0); // Spillback to a remote node. - auto remote_raylet_id = NodeID::FromRandom(); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 7777, remote_raylet_id)); + auto remote_node_id = NodeID::FromRandom(); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 7777, remote_node_id)); ASSERT_EQ(remote_raylet_clients.count(7777), 1); // Confirm that lease policy is not consulted on spillback. ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 1); @@ -1377,8 +1375,8 @@ TEST_F(NormalTaskSubmitterTest, TestSpillbackRoundTrip) { remote_raylet_clients[addr.port()] = client; return client; }; - auto local_raylet_id = NodeID::FromRandom(); - lease_policy_ptr->SetNodeID(local_raylet_id); + auto local_node_id = NodeID::FromRandom(); + lease_policy_ptr->SetNodeID(local_node_id); auto store = DefaultCoreWorkerMemoryStoreWithThread::CreateShared(); auto submitter = CreateNormalTaskSubmitter(std::make_shared(1), @@ -1386,7 +1384,7 @@ TEST_F(NormalTaskSubmitterTest, TestSpillbackRoundTrip) { raylet_client_factory, store, kLongTimeout, - local_raylet_id); + local_node_id); TaskSpecification task = BuildEmptyTaskSpec(); ASSERT_TRUE(submitter.SubmitTask(task).ok()); @@ -1397,8 +1395,8 @@ TEST_F(NormalTaskSubmitterTest, TestSpillbackRoundTrip) { ASSERT_EQ(remote_raylet_clients.size(), 0); // Spillback to a remote node. - auto remote_raylet_id = NodeID::FromRandom(); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 7777, remote_raylet_id)); + auto remote_node_id = NodeID::FromRandom(); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 7777, remote_node_id)); ASSERT_EQ(remote_raylet_clients.count(7777), 1); ASSERT_EQ(remote_raylet_clients[7777]->num_workers_requested, 1); // Confirm that the spillback lease request has grant_or_reject set to true. @@ -1408,7 +1406,7 @@ TEST_F(NormalTaskSubmitterTest, TestSpillbackRoundTrip) { ASSERT_FALSE(raylet_client->GrantWorkerLease("remote", 1234, NodeID::Nil())); // Trigger a rejection back to the local node. ASSERT_TRUE(remote_raylet_clients[7777]->GrantWorkerLease( - "local", 1234, local_raylet_id, false, "", /*reject=*/true)); + "local", 1234, local_node_id, false, "", /*reject=*/true)); // We should not have created another lease client to the local raylet. ASSERT_EQ(remote_raylet_clients.size(), 1); // There should be no more callbacks on the remote node. diff --git a/src/ray/core_worker/test/core_worker_test.cc b/src/ray/core_worker/test/core_worker_test.cc index 78e4fef68a24..d364a6b006a5 100644 --- a/src/ray/core_worker/test/core_worker_test.cc +++ b/src/ray/core_worker/test/core_worker_test.cc @@ -115,7 +115,7 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { rpc::Address rpc_address; rpc_address.set_ip_address(options.node_ip_address); rpc_address.set_port(core_worker_server->GetPort()); - rpc_address.set_raylet_id(NodeID::FromRandom().Binary()); + rpc_address.set_node_id(NodeID::FromRandom().Binary()); rpc_address.set_worker_id(worker_context->GetWorkerID().Binary()); auto fake_object_info_publisher = std::make_unique(); diff --git a/src/ray/core_worker/test/lease_policy_test.cc b/src/ray/core_worker/test/lease_policy_test.cc index 3bdd17bb5000..37e1a584cd2c 100644 --- a/src/ray/core_worker/test/lease_policy_test.cc +++ b/src/ray/core_worker/test/lease_policy_test.cc @@ -57,7 +57,7 @@ class MockLocalityDataProvider : public LocalityDataProviderInterface { std::optional MockNodeAddrFactory(const NodeID &node_id) { rpc::Address mock_rpc_address; - mock_rpc_address.set_raylet_id(node_id.Binary()); + mock_rpc_address.set_node_id(node_id.Binary()); std::optional opt_mock_rpc_address = mock_rpc_address; return opt_mock_rpc_address; } @@ -77,7 +77,7 @@ TEST(LocalLeasePolicyTest, TestReturnFallback) { auto [best_node_address, is_selected_based_on_locality] = local_lease_policy.GetBestNodeForTask(task_spec); // Test that fallback node was chosen. - ASSERT_EQ(NodeID::FromBinary(best_node_address.raylet_id()), fallback_node); + ASSERT_EQ(NodeID::FromBinary(best_node_address.node_id()), fallback_node); ASSERT_FALSE(is_selected_based_on_locality); } @@ -105,7 +105,7 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityFallbackSpreadSchedulingStrat // Locality logic is not run since it's a spread scheduling strategy. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, 0); // Test that fallback node was chosen. - ASSERT_EQ(NodeID::FromBinary(best_node_address.raylet_id()), fallback_node); + ASSERT_EQ(NodeID::FromBinary(best_node_address.node_id()), fallback_node); ASSERT_FALSE(is_selected_based_on_locality); } @@ -136,7 +136,7 @@ TEST(LocalityAwareLeasePolicyTest, // Locality logic is not run since it's a node affinity scheduling strategy. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, 0); // Test that node affinity node was chosen. - ASSERT_EQ(NodeID::FromBinary(best_node_address.raylet_id()), node_affinity_node); + ASSERT_EQ(NodeID::FromBinary(best_node_address.node_id()), node_affinity_node); ASSERT_FALSE(is_selected_based_on_locality); } @@ -161,7 +161,7 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityDominatingNode) { // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that best node was chosen. - ASSERT_EQ(NodeID::FromBinary(best_node_address.raylet_id()), best_node); + ASSERT_EQ(NodeID::FromBinary(best_node_address.node_id()), best_node); ASSERT_TRUE(is_selected_based_on_locality); } @@ -187,7 +187,7 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityBiggerObject) { // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that best node was chosen. - ASSERT_EQ(NodeID::FromBinary(best_node_address.raylet_id()), best_node); + ASSERT_EQ(NodeID::FromBinary(best_node_address.node_id()), best_node); ASSERT_TRUE(is_selected_based_on_locality); } @@ -217,7 +217,7 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityBetterNode) { // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that best node was chosen. - ASSERT_EQ(NodeID::FromBinary(best_node_address.raylet_id()), best_node); + ASSERT_EQ(NodeID::FromBinary(best_node_address.node_id()), best_node); ASSERT_TRUE(is_selected_based_on_locality); } @@ -241,7 +241,7 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityFallbackNoLocations) { // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that fallback node was chosen. - ASSERT_EQ(NodeID::FromBinary(best_node_address.raylet_id()), fallback_node); + ASSERT_EQ(NodeID::FromBinary(best_node_address.node_id()), fallback_node); ASSERT_FALSE(is_selected_based_on_locality); } @@ -260,7 +260,7 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityFallbackNoDeps) { // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that fallback node was chosen. - ASSERT_EQ(NodeID::FromBinary(best_node_address.raylet_id()), fallback_node); + ASSERT_EQ(NodeID::FromBinary(best_node_address.node_id()), fallback_node); ASSERT_FALSE(is_selected_based_on_locality); } @@ -285,7 +285,7 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityFallbackAddrFetchFail) { // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that fallback node was chosen. - ASSERT_EQ(NodeID::FromBinary(best_node_address.raylet_id()), fallback_node); + ASSERT_EQ(NodeID::FromBinary(best_node_address.node_id()), fallback_node); ASSERT_FALSE(is_selected_based_on_locality); } diff --git a/src/ray/core_worker/test/object_recovery_manager_test.cc b/src/ray/core_worker/test/object_recovery_manager_test.cc index 44e6f277a6e0..55eeaebf706d 100644 --- a/src/ray/core_worker/test/object_recovery_manager_test.cc +++ b/src/ray/core_worker/test/object_recovery_manager_test.cc @@ -121,7 +121,7 @@ class MockObjectDirectory { class ObjectRecoveryManagerTestBase : public ::testing::Test { public: explicit ObjectRecoveryManagerTestBase(bool lineage_enabled) - : local_raylet_id_(NodeID::FromRandom()), + : local_node_id_(NodeID::FromRandom()), io_context_("TestOnly.ObjectRecoveryManagerTestBase"), publisher_(std::make_shared()), subscriber_(std::make_shared()), @@ -171,7 +171,7 @@ class ObjectRecoveryManagerTestBase : public ::testing::Test { io_context_.Stop(); } - NodeID local_raylet_id_; + NodeID local_node_id_; absl::flat_hash_map failed_reconstructions_; // Used by memory_store_. @@ -236,7 +236,7 @@ TEST_F(ObjectRecoveryLineageDisabledTest, TestPinNewCopy) { true, /*add_local_ref=*/true); rpc::Address address; - address.set_raylet_id(NodeID::FromRandom().Binary()); + address.set_node_id(NodeID::FromRandom().Binary()); object_directory_->SetLocations(object_id, {address}); ASSERT_TRUE(manager_.RecoverObject(object_id)); @@ -256,9 +256,9 @@ TEST_F(ObjectRecoveryManagerTest, TestPinNewCopy) { true, /*add_local_ref=*/true); rpc::Address address1; - address1.set_raylet_id(NodeID::FromRandom().Binary()); + address1.set_node_id(NodeID::FromRandom().Binary()); rpc::Address address2; - address2.set_raylet_id(NodeID::FromRandom().Binary()); + address2.set_node_id(NodeID::FromRandom().Binary()); object_directory_->SetLocations(object_id, {address1, address2}); ASSERT_TRUE(manager_.RecoverObject(object_id)); @@ -308,7 +308,7 @@ TEST_F(ObjectRecoveryManagerTest, TestReconstructionSuppression) { // A new copy of the object is pinned. NodeID remote_node_id = NodeID::FromRandom(); rpc::Address address; - address.set_raylet_id(remote_node_id.Binary()); + address.set_node_id(remote_node_id.Binary()); object_directory_->SetLocations(object_id, {address}); ASSERT_EQ(object_directory_->Flush(), 1); ASSERT_EQ(raylet_client_->Flush(), 1); diff --git a/src/ray/core_worker/test/reference_count_test.cc b/src/ray/core_worker/test/reference_count_test.cc index 31fb503fab9e..206a86263fa1 100644 --- a/src/ray/core_worker/test/reference_count_test.cc +++ b/src/ray/core_worker/test/reference_count_test.cc @@ -293,7 +293,7 @@ class MockWorkerClient : public MockCoreWorkerClientInterface { static rpc::Address CreateRandomAddress(const std::string &addr) { rpc::Address address; address.set_ip_address(addr); - address.set_raylet_id(NodeID::FromRandom().Binary()); + address.set_node_id(NodeID::FromRandom().Binary()); address.set_worker_id(WorkerID::FromRandom().Binary()); return address; } diff --git a/src/ray/core_worker/test/task_manager_test.cc b/src/ray/core_worker/test/task_manager_test.cc index 64e5ef03d29c..3830d9821474 100644 --- a/src/ray/core_worker/test/task_manager_test.cc +++ b/src/ray/core_worker/test/task_manager_test.cc @@ -737,7 +737,7 @@ TEST_F(TaskManagerTest, TestLocalityDataAdded) { return_object->set_in_plasma(true); return_object->set_size(object_size); rpc::Address worker_addr; - worker_addr.set_raylet_id(node_id.Binary()); + worker_addr.set_node_id(node_id.Binary()); manager_.AddPendingTask(rpc::Address(), spec, "", 0); manager_.CompletePendingTask(spec.TaskId(), reply, worker_addr, false); } diff --git a/src/ray/flatbuffers/node_manager.fbs b/src/ray/flatbuffers/node_manager.fbs index dde5dfac89d6..c6399c0d2aa2 100644 --- a/src/ray/flatbuffers/node_manager.fbs +++ b/src/ray/flatbuffers/node_manager.fbs @@ -103,7 +103,7 @@ table RegisterClientRequest { table RegisterClientReply { success: bool; failure_reason: string; - raylet_id: string; + node_id: string; port: int; } @@ -122,7 +122,7 @@ table AnnounceWorkerPortReply { // Mimics the Address protobuf. table Address { - raylet_id: string; + node_id: string; ip_address: string; port: int; // Optional unique id for the worker. diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index c1955578fedf..8683fed03eb5 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -180,11 +180,11 @@ bool is_uuid(const std::string &str) { } NodeID GcsActor::GetNodeID() const { - const auto &raylet_id_binary = actor_table_data_.address().raylet_id(); - if (raylet_id_binary.empty()) { + const auto &node_id_binary = actor_table_data_.address().node_id(); + if (node_id_binary.empty()) { return NodeID::Nil(); } - return NodeID::FromBinary(raylet_id_binary); + return NodeID::FromBinary(node_id_binary); } void GcsActor::UpdateAddress(const rpc::Address &address) { @@ -206,7 +206,7 @@ WorkerID GcsActor::GetOwnerID() const { } NodeID GcsActor::GetOwnerNodeID() const { - return NodeID::FromBinary(GetOwnerAddress().raylet_id()); + return NodeID::FromBinary(GetOwnerAddress().node_id()); } const rpc::Address &GcsActor::GetOwnerAddress() const { @@ -819,7 +819,7 @@ Status GcsActorManager::RegisterActor(const ray::rpc::RegisterActorRequest &requ function_manager_.AddJobReference(actor_id.JobId()); const auto &owner_address = actor->GetOwnerAddress(); - auto node_id = NodeID::FromBinary(owner_address.raylet_id()); + auto node_id = NodeID::FromBinary(owner_address.node_id()); auto worker_id = WorkerID::FromBinary(owner_address.worker_id()); RAY_CHECK(unresolved_actors_[node_id][worker_id].emplace(actor->GetActorID()).second); @@ -1669,7 +1669,7 @@ void GcsActorManager::Initialize(const GcsInitData &gcs_init_data) { if (actor_table_data.state() == ray::rpc::ActorTableData::DEPENDENCIES_UNREADY) { const auto &owner = actor->GetOwnerAddress(); - const auto &owner_node = NodeID::FromBinary(owner.raylet_id()); + const auto &owner_node = NodeID::FromBinary(owner.node_id()); const auto &owner_worker = WorkerID::FromBinary(owner.worker_id()); RAY_CHECK(unresolved_actors_[owner_node][owner_worker] .emplace(actor->GetActorID()) @@ -1738,7 +1738,7 @@ const absl::flat_hash_map> void GcsActorManager::RemoveUnresolvedActor(const std::shared_ptr &actor) { const auto &owner_address = actor->GetOwnerAddress(); - auto node_id = NodeID::FromBinary(owner_address.raylet_id()); + auto node_id = NodeID::FromBinary(owner_address.node_id()); auto worker_id = WorkerID::FromBinary(owner_address.worker_id()); auto iter = unresolved_actors_.find(node_id); if (iter != unresolved_actors_.end()) { diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.h b/src/ray/gcs/gcs_server/gcs_actor_manager.h index ddd788579ea7..263683ece7ab 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.h +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.h @@ -109,7 +109,7 @@ class GcsActor { actor_table_data_.set_state(rpc::ActorTableData::DEPENDENCIES_UNREADY); - actor_table_data_.mutable_address()->set_raylet_id(NodeID::Nil().Binary()); + actor_table_data_.mutable_address()->set_node_id(NodeID::Nil().Binary()); actor_table_data_.mutable_address()->set_worker_id(WorkerID::Nil().Binary()); actor_table_data_.set_ray_namespace(ray_namespace); diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc index 5eb2948d11f1..35b9c03dd146 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc @@ -77,14 +77,14 @@ void GcsActorScheduler::ScheduleByGcs(std::shared_ptr actor) { return; } const auto &retry_at_raylet_address = reply->retry_at_raylet_address(); - RAY_CHECK(!retry_at_raylet_address.raylet_id().empty()); - auto node_id = NodeID::FromBinary(retry_at_raylet_address.raylet_id()); + RAY_CHECK(!retry_at_raylet_address.node_id().empty()); + auto node_id = NodeID::FromBinary(retry_at_raylet_address.node_id()); auto node = gcs_node_manager_.GetAliveNode(node_id); RAY_CHECK(node.has_value()); // Update the address of the actor as it is tied to a node. rpc::Address address; - address.set_raylet_id(node.value()->node_id()); + address.set_node_id(node.value()->node_id()); actor->UpdateAddress(address); RAY_CHECK(node_to_actors_when_leasing_[actor->GetNodeID()] @@ -126,7 +126,7 @@ void GcsActorScheduler::ScheduleByRaylet(std::shared_ptr actor) { // Update the address of the actor as it is tied to a node. rpc::Address address; - address.set_raylet_id(node.value()->node_id()); + address.set_node_id(node.value()->node_id()); actor->UpdateAddress(address); RAY_CHECK(node_to_actors_when_leasing_[actor->GetNodeID()] @@ -245,7 +245,7 @@ void GcsActorScheduler::CancelOnLeasing(const NodeID &node_id, if (iter != alive_nodes.end()) { const auto &node_info = iter->second; rpc::Address address; - address.set_raylet_id(node_info->node_id()); + address.set_node_id(node_info->node_id()); address.set_ip_address(node_info->node_manager_address()); address.set_port(node_info->node_manager_port()); auto raylet_client = GetOrConnectRayletClient(address); @@ -287,7 +287,7 @@ void GcsActorScheduler::ReleaseUnusedActorWorkers( nodes_of_releasing_unused_workers_.insert(node_id); rpc::Address address; - address.set_raylet_id(alive_node.second->node_id()); + address.set_node_id(alive_node.second->node_id()); address.set_ip_address(alive_node.second->node_manager_address()); address.set_port(alive_node.second->node_manager_port()); auto raylet_client = GetOrConnectRayletClient(address); @@ -323,7 +323,7 @@ void GcsActorScheduler::LeaseWorkerFromNode(std::shared_ptr actor, } rpc::Address remote_address; - remote_address.set_raylet_id(node->node_id()); + remote_address.set_node_id(node->node_id()); remote_address.set_ip_address(node->node_manager_address()); remote_address.set_port(node->node_manager_port()); auto raylet_client = GetOrConnectRayletClient(remote_address); @@ -369,11 +369,11 @@ void GcsActorScheduler::HandleWorkerLeaseGrantedReply( std::shared_ptr actor, const ray::rpc::RequestWorkerLeaseReply &reply) { const auto &retry_at_raylet_address = reply.retry_at_raylet_address(); const auto &worker_address = reply.worker_address(); - if (worker_address.raylet_id().empty()) { + if (worker_address.node_id().empty()) { // The worker did not succeed in the lease, but the specified node returned a new // node, and then try again on the new node. - RAY_CHECK(!retry_at_raylet_address.raylet_id().empty()); - auto spill_back_node_id = NodeID::FromBinary(retry_at_raylet_address.raylet_id()); + RAY_CHECK(!retry_at_raylet_address.node_id().empty()); + auto spill_back_node_id = NodeID::FromBinary(retry_at_raylet_address.node_id()); auto maybe_spill_back_node = gcs_node_manager_.GetAliveNode(spill_back_node_id); if (maybe_spill_back_node.has_value()) { auto spill_back_node = maybe_spill_back_node.value(); @@ -537,7 +537,7 @@ std::shared_ptr GcsActorScheduler::GetOrConnectRayletClie bool GcsActorScheduler::KillActorOnWorker(const rpc::Address &worker_address, ActorID actor_id) { - if (worker_address.raylet_id().empty()) { + if (worker_address.node_id().empty()) { RAY_LOG(DEBUG) << "Invalid worker address, skip the killing of actor " << actor_id; return false; } @@ -607,8 +607,8 @@ void GcsActorScheduler::HandleWorkerLeaseReply( return; } - if (reply.worker_address().raylet_id().empty() && - reply.retry_at_raylet_address().raylet_id().empty() && !reply.rejected()) { + if (reply.worker_address().node_id().empty() && + reply.retry_at_raylet_address().node_id().empty() && !reply.rejected()) { // Actor creation task has been cancelled. It is triggered by `ray.kill`. If // the number of remaining restarts of the actor is not equal to 0, GCS will // reschedule the actor, so it return directly here. diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h index 1cd9a0907e05..b3a3c708debc 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h @@ -236,7 +236,7 @@ class GcsActorScheduler : public GcsActorSchedulerInterface { WorkerID GetWorkerID() const { return WorkerID::FromBinary(address_.worker_id()); } /// Get the NodeID of this leased worker. - NodeID GetNodeID() const { return NodeID::FromBinary(address_.raylet_id()); } + NodeID GetNodeID() const { return NodeID::FromBinary(address_.node_id()); } /// Get the id of the actor which is assigned to this leased worker. ActorID GetAssignedActorID() const { return assigned_actor_id_; } diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.cc b/src/ray/gcs/gcs_server/gcs_job_manager.cc index 5400b353dc06..8211d1c77ba2 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_job_manager.cc @@ -489,7 +489,7 @@ void GcsJobManager::OnNodeDead(const NodeID &node_id) { // - (1) are not already dead. // - (2) have their driver running on the dead node. for (auto &data : result) { - auto driver_node_id = NodeID::FromBinary(data.second.driver_address().raylet_id()); + auto driver_node_id = NodeID::FromBinary(data.second.driver_address().node_id()); if (!data.second.is_dead() && driver_node_id == node_id) { MarkJobAsFinished(data.second, [data](Status status) { if (!status.ok()) { diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc index 0ca35f1765ed..6a981400adf5 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc @@ -292,7 +292,7 @@ std::shared_ptr GcsPlacementGroupScheduler::GetRayletClientFromNode( const std::shared_ptr &node) { rpc::Address remote_address; - remote_address.set_raylet_id(node->node_id()); + remote_address.set_node_id(node->node_id()); remote_address.set_ip_address(node->node_manager_address()); remote_address.set_port(node->node_manager_port()); return GetOrConnectRayletClient(remote_address); diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 81058297b98c..f5fff4004c96 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -70,7 +70,7 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, addr, this->client_call_manager_, /*raylet_unavailable_timeout_callback=*/[this, addr]() { - const NodeID node_id = NodeID::FromBinary(addr.raylet_id()); + const NodeID node_id = NodeID::FromBinary(addr.node_id()); auto alive_node = this->gcs_node_manager_->GetAliveNode(node_id); if (!alive_node.has_value()) { this->raylet_client_pool_.Disconnect(node_id); @@ -82,7 +82,7 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, addr, this->client_call_manager_, /*core_worker_unavailable_timeout_callback*/ [this, addr]() { - const NodeID node_id = NodeID::FromBinary(addr.raylet_id()); + const NodeID node_id = NodeID::FromBinary(addr.node_id()); const WorkerID worker_id = WorkerID::FromBinary(addr.worker_id()); auto alive_node = this->gcs_node_manager_->GetAliveNode(node_id); if (!alive_node.has_value()) { @@ -797,7 +797,7 @@ void GcsServer::InstallEventListeners() { auto &worker_address = worker_failure_data->worker_address(); auto worker_id = WorkerID::FromBinary(worker_address.worker_id()); worker_client_pool_.Disconnect(worker_id); - auto node_id = NodeID::FromBinary(worker_address.raylet_id()); + auto node_id = NodeID::FromBinary(worker_address.node_id()); auto worker_ip = worker_address.ip_address(); const rpc::RayException *creation_task_exception = nullptr; if (worker_failure_data->has_creation_task_exception()) { diff --git a/src/ray/gcs/gcs_server/gcs_worker_manager.cc b/src/ray/gcs/gcs_server/gcs_worker_manager.cc index 42e8709d5f8d..bd3b16ff9bdc 100644 --- a/src/ray/gcs/gcs_server/gcs_worker_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_worker_manager.cc @@ -42,7 +42,7 @@ void GcsWorkerManager::HandleReportWorkerFailure( {[this, reply, send_reply_callback, worker_id, request = std::move(request)]( const std::optional &result) { const auto &worker_address = request.worker_failure().worker_address(); - const auto node_id = NodeID::FromBinary(worker_address.raylet_id()); + const auto node_id = NodeID::FromBinary(worker_address.node_id()); std::string message = absl::StrCat("Reporting worker exit, worker id = ", worker_id.Hex(), @@ -89,13 +89,12 @@ void GcsWorkerManager::HandleReportWorkerFailure( if (!IsIntentionalWorkerFailure(worker_failure_data->exit_type())) { stats::UnintentionalWorkerFailures.Record(1); } - // Only publish worker_id and raylet_id in address as they are the only + // Only publish worker_id and node_id in address as they are the only // fields used by sub clients. rpc::WorkerDeltaData worker_failure; worker_failure.set_worker_id( worker_failure_data->worker_address().worker_id()); - worker_failure.set_raylet_id( - worker_failure_data->worker_address().raylet_id()); + worker_failure.set_node_id(worker_failure_data->worker_address().node_id()); RAY_CHECK_OK( gcs_publisher_.PublishWorkerFailure(worker_id, worker_failure, nullptr)); } diff --git a/src/ray/gcs/gcs_server/test/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/gcs_server/test/export_api/gcs_actor_manager_export_event_test.cc index 7f552fc41571..e5942c4da032 100644 --- a/src/ray/gcs/gcs_server/test/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/test/export_api/gcs_actor_manager_export_event_test.cc @@ -212,7 +212,7 @@ class GcsActorManagerTest : public ::testing::Test { rpc::Address address; auto node_id = NodeID::FromRandom(); auto worker_id = WorkerID::FromRandom(); - address.set_raylet_id(node_id.Binary()); + address.set_node_id(node_id.Binary()); address.set_worker_id(worker_id.Binary()); return address; } diff --git a/src/ray/gcs/gcs_server/test/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/test/gcs_actor_manager_test.cc index 5b41f25778ab..ba3e54770746 100644 --- a/src/ray/gcs/gcs_server/test/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/test/gcs_actor_manager_test.cc @@ -157,7 +157,7 @@ class GcsActorManagerTest : public ::testing::Test { rpc::Address address; auto node_id = NodeID::FromRandom(); auto worker_id = WorkerID::FromRandom(); - address.set_raylet_id(node_id.Binary()); + address.set_node_id(node_id.Binary()); address.set_worker_id(worker_id.Binary()); return address; } @@ -350,7 +350,7 @@ TEST_F(GcsActorManagerTest, TestWorkerFailure) { // Check that the actor is in state `ALIVE`. auto address = RandomAddress(); - auto node_id = NodeID::FromBinary(address.raylet_id()); + auto node_id = NodeID::FromBinary(address.node_id()); auto worker_id = WorkerID::FromBinary(address.worker_id()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); @@ -408,7 +408,7 @@ TEST_F(GcsActorManagerTest, TestNodeFailure) { ASSERT_EQ(actor->GetState(), rpc::ActorTableData::ALIVE); // Remove node and then check that the actor is dead. - auto node_id = NodeID::FromBinary(address.raylet_id()); + auto node_id = NodeID::FromBinary(address.node_id()); EXPECT_CALL(*mock_actor_scheduler_, CancelOnNode(node_id)); OnNodeDead(node_id); @@ -447,7 +447,7 @@ TEST_F(GcsActorManagerTest, TestActorReconstruction) { // Check that the actor is in state `ALIVE`. auto address = RandomAddress(); - auto node_id = NodeID::FromBinary(address.raylet_id()); + auto node_id = NodeID::FromBinary(address.node_id()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); io_service_.run_one(); @@ -463,7 +463,7 @@ TEST_F(GcsActorManagerTest, TestActorReconstruction) { mock_actor_scheduler_->actors.clear(); ASSERT_EQ(finished_actors.size(), 1); auto node_id2 = NodeID::FromRandom(); - address.set_raylet_id(node_id2.Binary()); + address.set_node_id(node_id2.Binary()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); io_service_.run_one(); @@ -515,7 +515,7 @@ TEST_F(GcsActorManagerTest, TestActorRestartWhenOwnerDead) { // Check that the actor is in state `ALIVE`. auto address = RandomAddress(); - auto node_id = NodeID::FromBinary(address.raylet_id()); + auto node_id = NodeID::FromBinary(address.node_id()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); io_service_.run_one(); @@ -689,7 +689,7 @@ TEST_F(GcsActorManagerTest, TestNamedActorDeletionWorkerFailure) { // Check that the actor is in state `ALIVE`. auto address = RandomAddress(); - auto node_id = NodeID::FromBinary(address.raylet_id()); + auto node_id = NodeID::FromBinary(address.node_id()); auto worker_id = WorkerID::FromBinary(address.worker_id()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); @@ -747,7 +747,7 @@ TEST_F(GcsActorManagerTest, TestNamedActorDeletionNodeFailure) { // Check that the actor is in state `ALIVE`. auto address = RandomAddress(); - auto node_id = NodeID::FromBinary(address.raylet_id()); + auto node_id = NodeID::FromBinary(address.node_id()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); io_service_.run_one(); @@ -805,7 +805,7 @@ TEST_F(GcsActorManagerTest, TestNamedActorDeletionNotHappendWhenReconstructed) { // Check that the actor is in state `ALIVE`. auto address = RandomAddress(); - auto node_id = NodeID::FromBinary(address.raylet_id()); + auto node_id = NodeID::FromBinary(address.node_id()); auto worker_id = WorkerID::FromBinary(address.worker_id()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); @@ -891,7 +891,7 @@ TEST_F(GcsActorManagerTest, TestRaceConditionCancelLease) { rpc::Address address; auto node_id = NodeID::FromRandom(); auto worker_id = WorkerID::FromRandom(); - address.set_raylet_id(node_id.Binary()); + address.set_node_id(node_id.Binary()); address.set_worker_id(worker_id.Binary()); actor->UpdateAddress(address); const auto &actor_id = actor->GetActorID(); @@ -942,7 +942,7 @@ TEST_F(GcsActorManagerTest, TestOwnerWorkerDieBeforeActorDependenciesResolved) { auto job_id = JobID::FromInt(1); auto registered_actor = RegisterActor(job_id); const auto &owner_address = registered_actor->GetOwnerAddress(); - auto node_id = NodeID::FromBinary(owner_address.raylet_id()); + auto node_id = NodeID::FromBinary(owner_address.node_id()); auto worker_id = WorkerID::FromBinary(owner_address.worker_id()); gcs_actor_manager_->OnWorkerDead(node_id, worker_id); io_service_.run_one(); @@ -968,7 +968,7 @@ TEST_F(GcsActorManagerTest, TestOwnerWorkerDieBeforeDetachedActorDependenciesRes auto job_id = JobID::FromInt(1); auto registered_actor = RegisterActor(job_id, /*max_restarts=*/1, /*detached=*/true); const auto &owner_address = registered_actor->GetOwnerAddress(); - auto node_id = NodeID::FromBinary(owner_address.raylet_id()); + auto node_id = NodeID::FromBinary(owner_address.node_id()); auto worker_id = WorkerID::FromBinary(owner_address.worker_id()); gcs_actor_manager_->OnWorkerDead(node_id, worker_id); io_service_.run_one(); @@ -993,7 +993,7 @@ TEST_F(GcsActorManagerTest, TestOwnerNodeDieBeforeActorDependenciesResolved) { auto job_id = JobID::FromInt(1); auto registered_actor = RegisterActor(job_id); const auto &owner_address = registered_actor->GetOwnerAddress(); - auto node_id = NodeID::FromBinary(owner_address.raylet_id()); + auto node_id = NodeID::FromBinary(owner_address.node_id()); OnNodeDead(node_id); ASSERT_EQ(registered_actor->GetState(), rpc::ActorTableData::DEAD); ASSERT_TRUE( @@ -1015,7 +1015,7 @@ TEST_F(GcsActorManagerTest, TestOwnerNodeDieBeforeDetachedActorDependenciesResol auto job_id = JobID::FromInt(1); auto registered_actor = RegisterActor(job_id, /*max_restarts=*/1, /*detached=*/true); const auto &owner_address = registered_actor->GetOwnerAddress(); - auto node_id = NodeID::FromBinary(owner_address.raylet_id()); + auto node_id = NodeID::FromBinary(owner_address.node_id()); OnNodeDead(node_id); ASSERT_EQ(registered_actor->GetState(), rpc::ActorTableData::DEAD); ASSERT_TRUE( @@ -1135,7 +1135,7 @@ TEST_F(GcsActorManagerTest, TestReuseActorNameInNamespace) { auto owner_address = request_1.task_spec().caller_address(); auto node_info = std::make_shared(); - node_info->set_node_id(owner_address.raylet_id()); + node_info->set_node_id(owner_address.node_id()); gcs_actor_manager_->OnNodeDead(node_info, ""); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName(actor_name, ray_namespace).Binary(), ActorID::Nil().Binary()); @@ -1358,7 +1358,7 @@ TEST_F(GcsActorManagerTest, TestRestartActorForLineageReconstruction) { // Check that the actor is in state `ALIVE`. auto address = RandomAddress(); - auto node_id = NodeID::FromBinary(address.raylet_id()); + auto node_id = NodeID::FromBinary(address.node_id()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); io_service_.run_one(); @@ -1375,7 +1375,7 @@ TEST_F(GcsActorManagerTest, TestRestartActorForLineageReconstruction) { mock_actor_scheduler_->actors.clear(); ASSERT_EQ(created_actors.size(), 1); auto node_id2 = NodeID::FromRandom(); - address.set_raylet_id(node_id2.Binary()); + address.set_node_id(node_id2.Binary()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); io_service_.run_one(); @@ -1407,7 +1407,7 @@ TEST_F(GcsActorManagerTest, TestRestartActorForLineageReconstruction) { mock_actor_scheduler_->actors.clear(); ASSERT_EQ(created_actors.size(), 1); auto node_id3 = NodeID::FromRandom(); - address.set_raylet_id(node_id3.Binary()); + address.set_node_id(node_id3.Binary()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); io_service_.run_one(); @@ -1538,7 +1538,7 @@ TEST_F(GcsActorManagerTest, TestIdempotencyOfRestartActorForLineageReconstructio mock_actor_scheduler_->actors.clear(); ASSERT_EQ(created_actors.size(), 1); auto node_id = NodeID::FromRandom(); - address.set_raylet_id(node_id.Binary()); + address.set_node_id(node_id.Binary()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); io_service_.run_one(); @@ -1667,7 +1667,7 @@ TEST_F(GcsActorManagerTest, TestRestartPreemptedActor) { // Make the actor alive on a specific node auto address = RandomAddress(); - auto node_id = NodeID::FromBinary(address.raylet_id()); + auto node_id = NodeID::FromBinary(address.node_id()); auto worker_id = WorkerID::FromBinary(address.worker_id()); actor->UpdateAddress(address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); @@ -1686,7 +1686,7 @@ TEST_F(GcsActorManagerTest, TestRestartPreemptedActor) { // Make the actor alive on a specific node again. auto new_address = RandomAddress(); - auto new_node_id = NodeID::FromBinary(new_address.raylet_id()); + auto new_node_id = NodeID::FromBinary(new_address.node_id()); auto new_worker_id = WorkerID::FromBinary(new_address.worker_id()); actor->UpdateAddress(new_address); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); @@ -1709,7 +1709,7 @@ TEST_F(GcsActorManagerTest, TestRestartPreemptedActor) { // Make the actor alive on another node again auto new_address_2 = RandomAddress(); - auto new_node_id_2 = NodeID::FromBinary(new_address_2.raylet_id()); + auto new_node_id_2 = NodeID::FromBinary(new_address_2.node_id()); auto new_worker_id_2 = WorkerID::FromBinary(new_address_2.worker_id()); actor->UpdateAddress(new_address_2); gcs_actor_manager_->OnActorCreationSuccess(actor, rpc::PushTaskReply()); diff --git a/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_mock_test.cc b/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_mock_test.cc index 4493b8257bc1..2671f360ec09 100644 --- a/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_mock_test.cc +++ b/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_mock_test.cc @@ -130,7 +130,7 @@ TEST_F(GcsActorSchedulerMockTest, KillWorkerLeak1) { actor->GetMutableActorTableData()->set_state(rpc::ActorTableData::DEAD); actor_scheduler->CancelOnNode(node_id); ray::rpc::RequestWorkerLeaseReply reply; - reply.mutable_worker_address()->set_raylet_id(node_id.Binary()); + reply.mutable_worker_address()->set_node_id(node_id.Binary()); reply.mutable_worker_address()->set_worker_id(worker_id.Binary()); cb(Status::OK(), std::move(reply)); } @@ -162,7 +162,7 @@ TEST_F(GcsActorSchedulerMockTest, KillWorkerLeak2) { DoAll(SaveArgToUniquePtr<4>(&async_put_with_index_cb), Return(Status::OK()))); actor_scheduler->ScheduleByRaylet(actor); rpc::RequestWorkerLeaseReply reply; - reply.mutable_worker_address()->set_raylet_id(node_id.Binary()); + reply.mutable_worker_address()->set_node_id(node_id.Binary()); reply.mutable_worker_address()->set_worker_id(worker_id.Binary()); request_worker_lease_cb(Status::OK(), std::move(reply)); diff --git a/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc b/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc index 1834056bbd62..128d5df7124b 100644 --- a/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc @@ -120,7 +120,7 @@ class GcsActorSchedulerTest : public ::testing::Test { std::shared_ptr NewGcsActor( const std::unordered_map &required_placement_resources) { rpc::Address owner_address; - owner_address.set_raylet_id(NodeID::FromRandom().Binary()); + owner_address.set_node_id(NodeID::FromRandom().Binary()); owner_address.set_ip_address("127.0.0.1"); owner_address.set_port(5678); owner_address.set_worker_id(WorkerID::FromRandom().Binary()); @@ -578,7 +578,7 @@ TEST_F(GcsActorSchedulerTest, TestReschedule) { std::make_shared(create_actor_request.task_spec(), "", counter); rpc::Address address; WorkerID worker_id = WorkerID::FromRandom(); - address.set_raylet_id(node_id_1.Binary()); + address.set_node_id(node_id_1.Binary()); address.set_worker_id(worker_id.Binary()); actor->UpdateAddress(address); @@ -1129,7 +1129,7 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestRescheduleByGcs) { // 1.Actor is already tied to a leased worker. rpc::Address address; WorkerID worker_id = WorkerID::FromRandom(); - address.set_raylet_id(node_id_1.Binary()); + address.set_node_id(node_id_1.Binary()); address.set_worker_id(worker_id.Binary()); actor->UpdateAddress(address); diff --git a/src/ray/gcs/gcs_server/test/gcs_job_manager_test.cc b/src/ray/gcs/gcs_server/test/gcs_job_manager_test.cc index 683016b5d801..eab0a5333a08 100644 --- a/src/ray/gcs/gcs_server/test/gcs_job_manager_test.cc +++ b/src/ray/gcs/gcs_server/test/gcs_job_manager_test.cc @@ -129,7 +129,7 @@ TEST_F(GcsJobManagerTest, TestIsRunningTasks) { address.set_port(num_running_tasks); // Populate other fields, the value is not important. - address.set_raylet_id(NodeID::FromRandom().Binary()); + address.set_node_id(NodeID::FromRandom().Binary()); address.set_ip_address("123.456.7.8"); address.set_worker_id(WorkerID::FromRandom().Binary()); @@ -557,7 +557,7 @@ TEST_F(GcsJobManagerTest, TestPreserveDriverInfo) { rpc::Address address; address.set_ip_address("10.0.0.1"); address.set_port(8264); - address.set_raylet_id(NodeID::FromRandom().Binary()); + address.set_node_id(NodeID::FromRandom().Binary()); address.set_worker_id(WorkerID::FromRandom().Binary()); add_job_request->mutable_data()->set_driver_ip_address("10.0.0.1"); add_job_request->mutable_data()->mutable_driver_address()->CopyFrom(address); @@ -744,7 +744,7 @@ TEST_F(GcsJobManagerTest, TestNodeFailure) { // Remove node and then check that the job is dead. auto address = all_job_info_reply.job_info_list().Get(0).driver_address(); - auto node_id = NodeID::FromBinary(address.raylet_id()); + auto node_id = NodeID::FromBinary(address.node_id()); gcs_job_manager_->OnNodeDead(node_id); // Test get all jobs and check if killed node jobs marked as finished @@ -763,7 +763,7 @@ TEST_F(GcsJobManagerTest, TestNodeFailure) { bool job_condition = true; // job1 from the current node should dead, while job2 is still alive for (auto job_info : all_job_info_reply2.job_info_list()) { - auto job_node_id = NodeID::FromBinary(job_info.driver_address().raylet_id()); + auto job_node_id = NodeID::FromBinary(job_info.driver_address().node_id()); job_condition = job_condition && (job_info.is_dead() == (job_node_id == node_id)); } return job_condition; diff --git a/src/ray/gcs/gcs_server/test/gcs_server_test_util.h b/src/ray/gcs/gcs_server/test/gcs_server_test_util.h index c530d76f285e..93ad48a844d2 100644 --- a/src/ray/gcs/gcs_server/test/gcs_server_test_util.h +++ b/src/ray/gcs/gcs_server/test/gcs_server_test_util.h @@ -137,26 +137,25 @@ struct GcsServerMocker { bool GrantWorkerLease(const std::string &address, int port, const WorkerID &worker_id, - const NodeID &raylet_id, - const NodeID &retry_at_raylet_id, + const NodeID &node_id, + const NodeID &retry_at_node_id, Status status = Status::OK(), bool rejected = false) { rpc::RequestWorkerLeaseReply reply; - if (!retry_at_raylet_id.IsNil()) { + if (!retry_at_node_id.IsNil()) { reply.mutable_retry_at_raylet_address()->set_ip_address(address); reply.mutable_retry_at_raylet_address()->set_port(port); - reply.mutable_retry_at_raylet_address()->set_raylet_id( - retry_at_raylet_id.Binary()); + reply.mutable_retry_at_raylet_address()->set_node_id(retry_at_node_id.Binary()); } else { reply.mutable_worker_address()->set_ip_address(address); reply.mutable_worker_address()->set_port(port); - reply.mutable_worker_address()->set_raylet_id(raylet_id.Binary()); + reply.mutable_worker_address()->set_node_id(node_id.Binary()); reply.mutable_worker_address()->set_worker_id(worker_id.Binary()); } if (rejected) { reply.set_rejected(true); auto resources_data = reply.mutable_resources_data(); - resources_data->set_node_id(raylet_id.Binary()); + resources_data->set_node_id(node_id.Binary()); resources_data->set_resources_normal_task_changed(true); auto &normal_task_map = *(resources_data->mutable_resources_normal_task()); normal_task_map[kMemory_ResourceLabel] = diff --git a/src/ray/gcs/pb_util.h b/src/ray/gcs/pb_util.h index 2733cf470e86..2bb0c6ace6d4 100644 --- a/src/ray/gcs/pb_util.h +++ b/src/ray/gcs/pb_util.h @@ -84,7 +84,7 @@ inline std::shared_ptr CreateWorkerFailureData( // Only report the worker id + delta (new data upon worker failures). // GCS will merge the data with original worker data. worker_failure_info_ptr->mutable_worker_address()->set_worker_id(worker_id.Binary()); - worker_failure_info_ptr->mutable_worker_address()->set_raylet_id(node_id.Binary()); + worker_failure_info_ptr->mutable_worker_address()->set_node_id(node_id.Binary()); worker_failure_info_ptr->mutable_worker_address()->set_ip_address(ip_address); worker_failure_info_ptr->set_timestamp(timestamp); worker_failure_info_ptr->set_exit_type(disconnect_type); diff --git a/src/ray/gcs/test/gcs_test_util.h b/src/ray/gcs/test/gcs_test_util.h index 56379b5a1154..4b5b125a97a7 100644 --- a/src/ray/gcs/test/gcs_test_util.h +++ b/src/ray/gcs/test/gcs_test_util.h @@ -95,7 +95,7 @@ struct Mocker { const std::string &name = "", const std::string &ray_namespace = "") { rpc::Address owner_address; - owner_address.set_raylet_id(NodeID::FromRandom().Binary()); + owner_address.set_node_id(NodeID::FromRandom().Binary()); owner_address.set_ip_address("1234"); owner_address.set_port(5678); owner_address.set_worker_id(WorkerID::FromRandom().Binary()); @@ -113,7 +113,7 @@ struct Mocker { const std::string &name = "", const std::string &ray_namespace = "test") { rpc::Address owner_address; - owner_address.set_raylet_id(NodeID::FromRandom().Binary()); + owner_address.set_node_id(NodeID::FromRandom().Binary()); owner_address.set_ip_address("1234"); owner_address.set_port(5678); owner_address.set_worker_id(WorkerID::FromRandom().Binary()); @@ -225,7 +225,7 @@ struct Mocker { rpc::Address address; address.set_ip_address("127.0.0.1"); address.set_port(1234); - address.set_raylet_id(UniqueID::FromRandom().Binary()); + address.set_node_id(UniqueID::FromRandom().Binary()); address.set_worker_id(UniqueID::FromRandom().Binary()); job_table_data->mutable_driver_address()->CopyFrom(address); job_table_data->set_driver_pid(5667L); @@ -271,7 +271,7 @@ struct Mocker { } else { rpc::Address dummy_address; dummy_address.set_port(1234); - dummy_address.set_raylet_id(NodeID::FromRandom().Binary()); + dummy_address.set_node_id(NodeID::FromRandom().Binary()); dummy_address.set_ip_address("123.456.7.8"); dummy_address.set_worker_id(WorkerID::FromRandom().Binary()); job_table_data->mutable_driver_address()->CopyFrom(dummy_address); diff --git a/src/ray/ipc/raylet_ipc_client.cc b/src/ray/ipc/raylet_ipc_client.cc index bcf3e4409367..dc693d281978 100644 --- a/src/ray/ipc/raylet_ipc_client.cc +++ b/src/ray/ipc/raylet_ipc_client.cc @@ -32,7 +32,7 @@ namespace { flatbuffers::Offset to_flatbuf( flatbuffers::FlatBufferBuilder &fbb, const ray::rpc::Address &address) { return ray::protocol::CreateAddress(fbb, - fbb.CreateString(address.raylet_id()), + fbb.CreateString(address.node_id()), fbb.CreateString(address.ip_address()), address.port(), fbb.CreateString(address.worker_id())); @@ -74,7 +74,7 @@ ray::Status RayletIpcClient::RegisterClient(const WorkerID &worker_id, const std::string &ip_address, const std::string &serialized_job_config, const StartupToken &startup_token, - NodeID *raylet_id, + NodeID *node_id, int *assigned_port) { flatbuffers::FlatBufferBuilder fbb; auto message = @@ -101,7 +101,7 @@ ray::Status RayletIpcClient::RegisterClient(const WorkerID &worker_id, return Status::Invalid(string_from_flatbuf(*reply_message->failure_reason())); } - *raylet_id = NodeID::FromBinary(reply_message->raylet_id()->str()); + *node_id = NodeID::FromBinary(reply_message->node_id()->str()); *assigned_port = reply_message->port(); return Status::OK(); } diff --git a/src/ray/ipc/raylet_ipc_client.h b/src/ray/ipc/raylet_ipc_client.h index 47a1132d6a29..15ee56670513 100644 --- a/src/ray/ipc/raylet_ipc_client.h +++ b/src/ray/ipc/raylet_ipc_client.h @@ -49,7 +49,7 @@ class RayletIpcClientInterface { /// \param ip_address The ip_address of the connecting worker. /// \param serialized_job_config The serialized job config of the connecting worker. /// \param startup_token The token that was passed to this worker at startup. - /// \param[out] raylet_id The node ID for the local Raylet. + /// \param[out] node_id The node ID for the local Raylet. /// \param[out] assigned_port The assigned port for the worker to listen on. If zero, /// the worker should pick a port randomly. virtual ray::Status RegisterClient(const WorkerID &worker_id, @@ -60,7 +60,7 @@ class RayletIpcClientInterface { const std::string &ip_address, const std::string &serialized_job_config, const StartupToken &startup_token, - NodeID *raylet_id, + NodeID *node_id, int *assigned_port) = 0; /// Notify the raylet that this client is disconnecting gracefully. This @@ -210,7 +210,7 @@ class RayletIpcClient : public RayletIpcClientInterface { const std::string &ip_address, const std::string &serialized_job_config, const StartupToken &startup_token, - NodeID *raylet_id, + NodeID *node_id, int *assigned_port) override; ray::Status Disconnect(const rpc::WorkerExitType &exit_type, diff --git a/src/ray/object_manager/common.h b/src/ray/object_manager/common.h index 7a790bc91275..709671026e11 100644 --- a/src/ray/object_manager/common.h +++ b/src/ray/object_manager/common.h @@ -216,8 +216,8 @@ struct ObjectInfo { bool is_mutable = false; int64_t data_size = 0; int64_t metadata_size = 0; - /// Owner's raylet ID. - NodeID owner_raylet_id; + /// Owner's node ID. + NodeID owner_node_id; /// Owner's IP address. std::string owner_ip_address; /// Owner's port. @@ -232,7 +232,7 @@ struct ObjectInfo { bool operator==(const ObjectInfo &other) const { return ((object_id == other.object_id) && (data_size == other.data_size) && (metadata_size == other.metadata_size) && - (owner_raylet_id == other.owner_raylet_id) && + (owner_node_id == other.owner_node_id) && (owner_ip_address == other.owner_ip_address) && (owner_port == other.owner_port) && (owner_worker_id == other.owner_worker_id)); diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index a6bdead01cfe..e2040c3aa297 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -401,7 +401,7 @@ void ObjectManager::PushLocalObject(const ObjectID &object_id, const NodeID &nod uint64_t metadata_size = static_cast(object_info.metadata_size); rpc::Address owner_address; - owner_address.set_raylet_id(object_info.owner_raylet_id.Binary()); + owner_address.set_node_id(object_info.owner_node_id.Binary()); owner_address.set_ip_address(object_info.owner_ip_address); owner_address.set_port(object_info.owner_port); owner_address.set_worker_id(object_info.owner_worker_id.Binary()); diff --git a/src/ray/object_manager/ownership_object_directory.cc b/src/ray/object_manager/ownership_object_directory.cc index bc17c10c8396..53d9ba0eed0c 100644 --- a/src/ray/object_manager/ownership_object_directory.cc +++ b/src/ray/object_manager/ownership_object_directory.cc @@ -104,7 +104,7 @@ bool UpdateObjectLocations(const rpc::WorkerObjectLocationsPubMessage &location_ rpc::Address GetOwnerAddressFromObjectInfo(const ObjectInfo &object_info) { rpc::Address owner_address; - owner_address.set_raylet_id(object_info.owner_raylet_id.Binary()); + owner_address.set_node_id(object_info.owner_node_id.Binary()); owner_address.set_ip_address(object_info.owner_ip_address); owner_address.set_port(object_info.owner_port); owner_address.set_worker_id(object_info.owner_worker_id.Binary()); diff --git a/src/ray/object_manager/plasma/plasma.fbs b/src/ray/object_manager/plasma/plasma.fbs index f64e450da7bc..0211916f0513 100644 --- a/src/ray/object_manager/plasma/plasma.fbs +++ b/src/ray/object_manager/plasma/plasma.fbs @@ -127,8 +127,8 @@ table PlasmaGetDebugStringReply { table PlasmaCreateRequest { // ID of the object to be created. object_id: string; - // Owner raylet ID of this object. - owner_raylet_id: string; + // Owner node ID of this object. + owner_node_id: string; // Owner IP address of this object. owner_ip_address: string; // Owner port address of this object. diff --git a/src/ray/object_manager/plasma/protocol.cc b/src/ray/object_manager/plasma/protocol.cc index 153de7181d28..8d589efeea01 100644 --- a/src/ray/object_manager/plasma/protocol.cc +++ b/src/ray/object_manager/plasma/protocol.cc @@ -40,7 +40,7 @@ using flatbuffers::uoffset_t; inline constexpr std::string_view kDebugString = "debug_string"; inline constexpr std::string_view kObjectId = "object_id"; inline constexpr std::string_view kObjectIds = "object_ids"; -inline constexpr std::string_view kOwnerRayletId = "owner_raylet_id"; +inline constexpr std::string_view kOwnerNodeId = "owner_node_id"; inline constexpr std::string_view kOwnerIpAddress = "owner_ip_address"; inline constexpr std::string_view kOnwerWorkerId = "owner_worker_id"; @@ -222,7 +222,7 @@ Status SendCreateRequest(const std::shared_ptr &store_conn, auto message = fb::CreatePlasmaCreateRequest(fbb, fbb.CreateString(object_id.Binary()), - fbb.CreateString(owner_address.raylet_id()), + fbb.CreateString(owner_address.node_id()), fbb.CreateString(owner_address.ip_address()), owner_address.port(), fbb.CreateString(owner_address.worker_id()), @@ -249,8 +249,8 @@ void ReadCreateRequest(const uint8_t *data, VerifyNotNullPtr(message->object_id(), kObjectId, MessageType::PlasmaCreateRequest); object_info->object_id = ObjectID::FromBinary(message->object_id()->str()); VerifyNotNullPtr( - message->owner_raylet_id(), kOwnerRayletId, MessageType::PlasmaCreateRequest); - object_info->owner_raylet_id = NodeID::FromBinary(message->owner_raylet_id()->str()); + message->owner_node_id(), kOwnerNodeId, MessageType::PlasmaCreateRequest); + object_info->owner_node_id = NodeID::FromBinary(message->owner_node_id()->str()); VerifyNotNullPtr( message->owner_ip_address(), kOwnerIpAddress, MessageType::PlasmaCreateRequest); object_info->owner_ip_address = message->owner_ip_address()->str(); diff --git a/src/ray/object_manager/plasma/test/object_store_test.cc b/src/ray/object_manager/plasma/test/object_store_test.cc index bbe73175313e..0d5704c9a484 100644 --- a/src/ray/object_manager/plasma/test/object_store_test.cc +++ b/src/ray/object_manager/plasma/test/object_store_test.cc @@ -65,7 +65,7 @@ ObjectInfo CreateObjectInfo(ObjectID object_id, int64_t object_size) { info.object_id = object_id; info.data_size = Random(object_size); info.metadata_size = object_size - info.data_size; - info.owner_raylet_id = NodeID::FromRandom(); + info.owner_node_id = NodeID::FromRandom(); info.owner_ip_address = "random_ip"; info.owner_port = Random(); info.owner_worker_id = WorkerID::FromRandom(); diff --git a/src/ray/object_manager/test/ownership_object_directory_test.cc b/src/ray/object_manager/test/ownership_object_directory_test.cc index 0b082e6bd2ac..9fc8964fdfb7 100644 --- a/src/ray/object_manager/test/ownership_object_directory_test.cc +++ b/src/ray/object_manager/test/ownership_object_directory_test.cc @@ -155,7 +155,7 @@ class OwnershipBasedObjectDirectoryTest : public ::testing::Test { ray::ObjectInfo info; info.object_id = id; info.data_size = 12; - info.owner_raylet_id = NodeID::FromRandom(); + info.owner_node_id = NodeID::FromRandom(); info.owner_ip_address = "124.2.3.4"; info.owner_port = 6739; info.owner_worker_id = worker_id; diff --git a/src/ray/object_manager/test/spilled_object_test.cc b/src/ray/object_manager/test/spilled_object_test.cc index 15f54365ea29..643596a106e9 100644 --- a/src/ray/object_manager/test/spilled_object_test.cc +++ b/src/ray/object_manager/test/spilled_object_test.cc @@ -138,9 +138,9 @@ TEST(SpilledObjectReaderTest, ParseObjectHeader) { auto assert_parse_success = [](uint64_t object_offset, std::string data, std::string metadata, - std::string raylet_id) { + std::string node_id) { rpc::Address owner_address; - owner_address.set_raylet_id(raylet_id); + owner_address.set_node_id(node_id); auto str = ContructObjectString(object_offset, data, metadata, owner_address); uint64_t actual_data_offset = 0; uint64_t actual_data_size = 0; @@ -162,7 +162,7 @@ TEST(SpilledObjectReaderTest, ParseObjectHeader) { actual_data_offset); ASSERT_EQ(data.size(), actual_data_size); ASSERT_EQ(metadata.size(), actual_metadata_size); - ASSERT_EQ(owner_address.raylet_id(), actual_owner_address.raylet_id()); + ASSERT_EQ(owner_address.node_id(), actual_owner_address.node_id()); ASSERT_EQ(data, str.substr(actual_data_offset, actual_data_size)); ASSERT_EQ(metadata, str.substr(actual_metadata_offset, actual_metadata_size)); }; @@ -171,13 +171,13 @@ TEST(SpilledObjectReaderTest, ParseObjectHeader) { std::vector data_list{"", "somedata", large_data}; std::string large_metadata(10000, 'm'); std::vector metadata_list{"", "somemetadata", large_metadata}; - std::vector raylet_ids{"", "yes", "laaaaaaaarrrrrggge"}; + std::vector node_ids{"", "yes", "laaaaaaaarrrrrggge"}; for (auto offset : offsets) { for (auto &data : data_list) { for (auto &metadata : metadata_list) { - for (auto &raylet_id : raylet_ids) { - assert_parse_success(offset, data, metadata, raylet_id); + for (auto &node_id : node_ids) { + assert_parse_success(offset, data, metadata, node_id); } } } @@ -249,7 +249,7 @@ TEST(ChunkObjectReaderTest, GetNumChunks) { auto assert_get_num_chunks = [](uint64_t data_size, uint64_t chunk_size, uint64_t expected_num_chunks) { rpc::Address owner_address; - owner_address.set_raylet_id("nonsense"); + owner_address.set_node_id("nonsense"); ChunkObjectReader reader(std::make_shared( SpilledObjectReader("path", 100 /* object_size */, @@ -334,12 +334,12 @@ TYPED_TEST(ObjectReaderTest, Getters) { std::string data("data"); std::string metadata("metadata"); rpc::Address owner_address; - owner_address.set_raylet_id("nonsense"); + owner_address.set_node_id("nonsense"); auto obj_reader = this->CreateObjectReader_(data, metadata, owner_address); ASSERT_EQ(data.size(), obj_reader->GetDataSize()); ASSERT_EQ(metadata.size(), obj_reader->GetMetadataSize()); ASSERT_EQ(data.size() + metadata.size(), obj_reader->GetObjectSize()); - ASSERT_EQ(owner_address.raylet_id(), obj_reader->GetOwnerAddress().raylet_id()); + ASSERT_EQ(owner_address.node_id(), obj_reader->GetOwnerAddress().node_id()); } TYPED_TEST(ObjectReaderTest, GetDataAndMetadata) { @@ -386,7 +386,7 @@ TYPED_TEST(ObjectReaderTest, GetChunk) { for (auto &metadata : list_metadata) { std::vector chunk_sizes{1, 2, 3, 5, 100}; rpc::Address owner_address; - owner_address.set_raylet_id("nonsense"); + owner_address.set_node_id("nonsense"); std::string expected_output = data + metadata; if (expected_output.size() != 0) { @@ -421,8 +421,8 @@ TEST(StringAllocationTest, TestNoCopyWhenStringMoved) { std::string s(1000, '\0'); auto allocation_address = s.c_str(); rpc::Address address; - address.set_raylet_id(std::move(s)); - EXPECT_EQ(allocation_address, address.raylet_id().c_str()); + address.set_node_id(std::move(s)); + EXPECT_EQ(allocation_address, address.node_id().c_str()); } TEST(StringAllocationTest, TestCopyWhenPassByPointer) { @@ -431,8 +431,8 @@ TEST(StringAllocationTest, TestCopyWhenPassByPointer) { char arr[1000]; auto allocation_address = &arr[0]; rpc::Address address; - address.set_raylet_id(allocation_address, 1000); - EXPECT_NE(allocation_address, address.raylet_id().c_str()); + address.set_node_id(allocation_address, 1000); + EXPECT_NE(allocation_address, address.node_id().c_str()); } } // namespace ray diff --git a/src/ray/protobuf/common.proto b/src/ray/protobuf/common.proto index d03d9286e7ff..9b3f04f2e59a 100644 --- a/src/ray/protobuf/common.proto +++ b/src/ray/protobuf/common.proto @@ -124,7 +124,7 @@ message SchedulingStrategy { // Address of a worker or node manager. message Address { - bytes raylet_id = 1; + bytes node_id = 1; string ip_address = 2; int32 port = 3; // Optional unique id for the worker. diff --git a/src/ray/protobuf/gcs.proto b/src/ray/protobuf/gcs.proto index 97a8e1ae1af8..972c80489b25 100644 --- a/src/ray/protobuf/gcs.proto +++ b/src/ray/protobuf/gcs.proto @@ -440,7 +440,7 @@ message WorkerTableData { // Fields to publish when worker fails. message WorkerDeltaData { - bytes raylet_id = 1; + bytes node_id = 1; bytes worker_id = 2; } diff --git a/src/ray/pubsub/test/subscriber_test.cc b/src/ray/pubsub/test/subscriber_test.cc index 1453ec9409da..6d0b55ae8438 100644 --- a/src/ray/pubsub/test/subscriber_test.cc +++ b/src/ray/pubsub/test/subscriber_test.cc @@ -165,7 +165,7 @@ class SubscriberTest : public ::testing::Test { const std::string address = "abc", const int port = 1234) { rpc::Address addr; - addr.set_raylet_id(node_id); + addr.set_node_id(node_id); addr.set_ip_address(address); addr.set_port(port); addr.set_worker_id(worker_id); diff --git a/src/ray/raylet/local_object_manager.cc b/src/ray/raylet/local_object_manager.cc index 51aff1d5c824..02c280e8c9cb 100644 --- a/src/ray/raylet/local_object_manager.cc +++ b/src/ray/raylet/local_object_manager.cc @@ -72,7 +72,7 @@ void LocalObjectManager::PinObjectsAndWaitForFree( wait_request->set_generator_id(generator_id.Binary()); } rpc::Address subscriber_address; - subscriber_address.set_raylet_id(self_node_id_.Binary()); + subscriber_address.set_node_id(self_node_id_.Binary()); subscriber_address.set_ip_address(self_node_address_); subscriber_address.set_port(self_node_port_); wait_request->mutable_subscriber_address()->CopyFrom(subscriber_address); diff --git a/src/ray/raylet/local_task_manager.cc b/src/ray/raylet/local_task_manager.cc index 44be7658d3d9..d37571f3d2f2 100644 --- a/src/ray/raylet/local_task_manager.cc +++ b/src/ray/raylet/local_task_manager.cc @@ -681,7 +681,7 @@ void LocalTaskManager::Spillback(const NodeID &spillback_to, reply->mutable_retry_at_raylet_address()->set_ip_address( node_info_ptr->node_manager_address()); reply->mutable_retry_at_raylet_address()->set_port(node_info_ptr->node_manager_port()); - reply->mutable_retry_at_raylet_address()->set_raylet_id(spillback_to.Binary()); + reply->mutable_retry_at_raylet_address()->set_node_id(spillback_to.Binary()); send_reply_callback(); } @@ -969,7 +969,7 @@ void LocalTaskManager::Dispatch( reply->mutable_worker_address()->set_ip_address(worker->IpAddress()); reply->mutable_worker_address()->set_port(worker->Port()); reply->mutable_worker_address()->set_worker_id(worker->WorkerId().Binary()); - reply->mutable_worker_address()->set_raylet_id(self_node_id_.Binary()); + reply->mutable_worker_address()->set_node_id(self_node_id_.Binary()); RAY_CHECK(leased_workers.find(worker->WorkerId()) == leased_workers.end()); leased_workers[worker->WorkerId()] = worker; diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index a9ad42a4bf27..7dd636db78e2 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -68,7 +68,7 @@ inline ray::rpc::ObjectReference FlatbufferToSingleObjectReference( const flatbuffers::String &object_id, const ray::protocol::Address &address) { ray::rpc::ObjectReference ref; ref.set_object_id(object_id.str()); - ref.mutable_owner_address()->set_raylet_id(address.raylet_id()->str()); + ref.mutable_owner_address()->set_node_id(address.node_id()->str()); ref.mutable_owner_address()->set_ip_address(address.ip_address()->str()); ref.mutable_owner_address()->set_port(address.port()); ref.mutable_owner_address()->set_worker_id(address.worker_id()->str()); @@ -85,7 +85,7 @@ std::vector FlatbufferToObjectReference( ray::rpc::ObjectReference ref; ref.set_object_id(object_ids.Get(i)->str()); const auto &addr = owner_addresses.Get(i); - ref.mutable_owner_address()->set_raylet_id(addr->raylet_id()->str()); + ref.mutable_owner_address()->set_node_id(addr->node_id()->str()); ref.mutable_owner_address()->set_ip_address(addr->ip_address()->str()); ref.mutable_owner_address()->set_port(addr->port()); ref.mutable_owner_address()->set_worker_id(addr->worker_id()->str()); @@ -281,7 +281,7 @@ void NodeManager::RegisterGcs() { // Subscribe to all unexpected failure notifications from the local and // remote raylets. Note that this does not include workers that failed due to - // node failure. These workers can be identified by comparing the raylet_id + // node failure. These workers can be identified by comparing the node_id // in their rpc::Address to the ID of a failed raylet. const auto &worker_failure_handler = [this](const rpc::WorkerDeltaData &worker_failure_data) { @@ -836,7 +836,7 @@ void NodeManager::NodeRemoved(const NodeID &node_id) { // Clean up workers that were owned by processes that were on the failed // node. for (const auto &[_, worker] : leased_workers_) { - const auto owner_node_id = NodeID::FromBinary(worker->GetOwnerAddress().raylet_id()); + const auto owner_node_id = NodeID::FromBinary(worker->GetOwnerAddress().node_id()); RAY_CHECK(!owner_node_id.IsNil()); if (worker->IsDetachedActor() || owner_node_id != node_id) { continue; @@ -1212,8 +1212,8 @@ void NodeManager::ProcessAnnounceWorkerPortMessageImpl( RAY_CHECK(job_config.has_value()); rpc::Address driver_address; - // Assume raylet ID is the same as the node ID. - driver_address.set_raylet_id(self_node_id_.Binary()); + // Assume node ID is the same as the node ID. + driver_address.set_node_id(self_node_id_.Binary()); driver_address.set_ip_address(worker->IpAddress()); driver_address.set_port(port); driver_address.set_worker_id(worker->WorkerId().Binary()); @@ -1663,7 +1663,7 @@ void NodeManager::HandleRequestWorkerLease(rpc::RequestWorkerLeaseRequest reques const auto caller_worker = WorkerID::FromBinary(task.GetTaskSpecification().CallerAddress().worker_id()); const auto caller_node = - NodeID::FromBinary(task.GetTaskSpecification().CallerAddress().raylet_id()); + NodeID::FromBinary(task.GetTaskSpecification().CallerAddress().node_id()); if (!task.GetTaskSpecification().IsDetachedActor() && (failed_workers_cache_.contains(caller_worker) || failed_nodes_cache_.contains(caller_node))) { diff --git a/src/ray/raylet/scheduling/cluster_task_manager.cc b/src/ray/raylet/scheduling/cluster_task_manager.cc index d3b11fb8022b..18ddbcf5d36d 100644 --- a/src/ray/raylet/scheduling/cluster_task_manager.cc +++ b/src/ray/raylet/scheduling/cluster_task_manager.cc @@ -455,7 +455,7 @@ void ClusterTaskManager::ScheduleOnNode(const NodeID &spillback_to, reply->mutable_retry_at_raylet_address()->set_ip_address( node_info_ptr->node_manager_address()); reply->mutable_retry_at_raylet_address()->set_port(node_info_ptr->node_manager_port()); - reply->mutable_retry_at_raylet_address()->set_raylet_id(spillback_to.Binary()); + reply->mutable_retry_at_raylet_address()->set_node_id(spillback_to.Binary()); send_reply_callback(); } diff --git a/src/ray/raylet/scheduling/cluster_task_manager_test.cc b/src/ray/raylet/scheduling/cluster_task_manager_test.cc index 27bc2bfa3c14..88fa3a320dc1 100644 --- a/src/ray/raylet/scheduling/cluster_task_manager_test.cc +++ b/src/ray/raylet/scheduling/cluster_task_manager_test.cc @@ -281,7 +281,7 @@ RayTask CreateTask( TaskID id = RandomTaskId(); JobID job_id = RandomJobId(); rpc::Address address; - address.set_raylet_id(NodeID::FromRandom().Binary()); + address.set_node_id(NodeID::FromRandom().Binary()); address.set_worker_id(WorkerID::FromRandom().Binary()); spec_builder.SetCommonTaskSpec(id, "dummy_task", @@ -874,8 +874,7 @@ TEST_F(ClusterTaskManagerTest, DrainingWhileResolving) { missing_objects_.erase(missing_arg); std::vector unblocked = {resolving_args_task.GetTaskSpecification().TaskId()}; local_task_manager_->TasksUnblocked(unblocked); - ASSERT_EQ(spillback_reply.retry_at_raylet_address().raylet_id(), - remote_node_id.Binary()); + ASSERT_EQ(spillback_reply.retry_at_raylet_address().node_id(), remote_node_id.Binary()); } TEST_F(ClusterTaskManagerTest, ResourceTakenWhileResolving) { @@ -994,8 +993,7 @@ TEST_F(ClusterTaskManagerTest, TestIsSelectedBasedOnLocality) { pool_.TriggerCallbacks(); // The second task was spilled. ASSERT_EQ(num_callbacks, 2); - ASSERT_EQ(spillback_reply.retry_at_raylet_address().raylet_id(), - remote_node_id.Binary()); + ASSERT_EQ(spillback_reply.retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); @@ -1049,8 +1047,7 @@ TEST_F(ClusterTaskManagerTest, TestGrantOrReject) { pool_.TriggerCallbacks(); // The second task was spilled. ASSERT_EQ(num_callbacks, 2); - ASSERT_EQ(spillback_reply.retry_at_raylet_address().raylet_id(), - remote_node_id.Binary()); + ASSERT_EQ(spillback_reply.retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); @@ -1117,8 +1114,7 @@ TEST_F(ClusterTaskManagerTest, TestSpillAfterAssigned) { // The third task was spilled. ASSERT_EQ(num_callbacks, 2); - ASSERT_EQ(spillback_reply.retry_at_raylet_address().raylet_id(), - remote_node_id.Binary()); + ASSERT_EQ(spillback_reply.retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_EQ(leased_workers_.size(), 0); // Two workers start. First task was dispatched now. @@ -1212,7 +1208,7 @@ TEST_F(ClusterTaskManagerTest, NotOKPopWorkerAfterDrainingTest) { pool_.callbacks.clear(); task_manager_.ScheduleAndDispatchTasks(); // task1 is spilled and task2 is cancelled. - ASSERT_EQ(reply1.retry_at_raylet_address().raylet_id(), remote_node_id.Binary()); + ASSERT_EQ(reply1.retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_TRUE(reply2.canceled()); ASSERT_EQ(reply2.scheduling_failure_message(), "runtime env setup error"); } @@ -1811,7 +1807,7 @@ TEST_F(ClusterTaskManagerTest, TestInfeasibleTaskWarning) { ASSERT_EQ(leased_workers_.size(), 0); ASSERT_EQ(pool_.workers.size(), 1); // Make sure the spillback callback is called. - ASSERT_EQ(reply.retry_at_raylet_address().raylet_id(), remote_node_id.Binary()); + ASSERT_EQ(reply.retry_at_raylet_address().node_id(), remote_node_id.Binary()); AssertNoLeaks(); } @@ -2192,14 +2188,14 @@ TEST_F(ClusterTaskManagerTest, TestSpillWaitingTasks) { task_manager_.ScheduleAndDispatchTasks(); ASSERT_EQ(num_callbacks, 2); // Spill from the back of the waiting queue. - ASSERT_EQ(replies[0]->retry_at_raylet_address().raylet_id(), ""); - ASSERT_EQ(replies[1]->retry_at_raylet_address().raylet_id(), ""); - ASSERT_EQ(replies[2]->retry_at_raylet_address().raylet_id(), remote_node_id.Binary()); - ASSERT_EQ(replies[3]->retry_at_raylet_address().raylet_id(), remote_node_id.Binary()); + ASSERT_EQ(replies[0]->retry_at_raylet_address().node_id(), ""); + ASSERT_EQ(replies[1]->retry_at_raylet_address().node_id(), ""); + ASSERT_EQ(replies[2]->retry_at_raylet_address().node_id(), remote_node_id.Binary()); + ASSERT_EQ(replies[3]->retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_FALSE(task_manager_.CancelTask(tasks[2].GetTaskSpecification().TaskId())); ASSERT_FALSE(task_manager_.CancelTask(tasks[3].GetTaskSpecification().TaskId())); // Do not spill back tasks ready to dispatch. - ASSERT_EQ(replies[4]->retry_at_raylet_address().raylet_id(), ""); + ASSERT_EQ(replies[4]->retry_at_raylet_address().node_id(), ""); AddNode(remote_node_id, 8); // Dispatch the ready task. @@ -2210,8 +2206,8 @@ TEST_F(ClusterTaskManagerTest, TestSpillWaitingTasks) { pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 4); // One waiting task spilled. - ASSERT_EQ(replies[0]->retry_at_raylet_address().raylet_id(), ""); - ASSERT_EQ(replies[1]->retry_at_raylet_address().raylet_id(), remote_node_id.Binary()); + ASSERT_EQ(replies[0]->retry_at_raylet_address().node_id(), ""); + ASSERT_EQ(replies[1]->retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_FALSE(task_manager_.CancelTask(tasks[1].GetTaskSpecification().TaskId())); // One task dispatched. ASSERT_EQ(replies[4]->worker_address().port(), 1234); @@ -2221,8 +2217,8 @@ TEST_F(ClusterTaskManagerTest, TestSpillWaitingTasks) { pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 4); // One waiting task spilled. - ASSERT_EQ(replies[0]->retry_at_raylet_address().raylet_id(), ""); - ASSERT_EQ(replies[1]->retry_at_raylet_address().raylet_id(), remote_node_id.Binary()); + ASSERT_EQ(replies[0]->retry_at_raylet_address().node_id(), ""); + ASSERT_EQ(replies[1]->retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_FALSE(task_manager_.CancelTask(tasks[1].GetTaskSpecification().TaskId())); // One task dispatched. ASSERT_EQ(replies[4]->worker_address().port(), 1234); @@ -2231,7 +2227,7 @@ TEST_F(ClusterTaskManagerTest, TestSpillWaitingTasks) { AddNode(remote_node_id, 8); task_manager_.ScheduleAndDispatchTasks(); ASSERT_EQ(num_callbacks, 4); - ASSERT_EQ(replies[0]->retry_at_raylet_address().raylet_id(), ""); + ASSERT_EQ(replies[0]->retry_at_raylet_address().node_id(), ""); RayTask finished_task; local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); @@ -2576,7 +2572,7 @@ TEST_F(ClusterTaskManagerTest, SchedulingClassCapSpillback) { AddNode(remote_node_id, 8); task_manager_.ScheduleAndDispatchTasks(); ASSERT_EQ(num_callbacks, 2); - ASSERT_EQ(replies[1]->retry_at_raylet_address().raylet_id(), remote_node_id.Binary()); + ASSERT_EQ(replies[1]->retry_at_raylet_address().node_id(), remote_node_id.Binary()); } /// Test that we exponentially increase the amount of time it takes to increase @@ -2927,8 +2923,7 @@ TEST_F(ClusterTaskManagerTest, UnscheduleableWhileDraining) { pool_.TriggerCallbacks(); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); - ASSERT_EQ(spillback_reply.retry_at_raylet_address().raylet_id(), - remote_node_id.Binary()); + ASSERT_EQ(spillback_reply.retry_at_raylet_address().node_id(), remote_node_id.Binary()); } // Regression test for https://github.com/ray-project/ray/issues/16935: diff --git a/src/ray/raylet/test/node_manager_test.cc b/src/ray/raylet/test/node_manager_test.cc index 9a42fcac04d7..3ef9df384f98 100644 --- a/src/ray/raylet/test/node_manager_test.cc +++ b/src/ray/raylet/test/node_manager_test.cc @@ -675,7 +675,7 @@ TEST_F(NodeManagerTest, TestDetachedWorkerIsKilledByFailedNode) { // Preparing a detached actor creation task spec for the later RequestWorkerLease rpc. const auto owner_node_id = NodeID::FromRandom(); rpc::Address owner_address; - owner_address.set_raylet_id(owner_node_id.Binary()); + owner_address.set_node_id(owner_node_id.Binary()); const auto actor_id = ActorID::Of(JobID::FromInt(1), TaskID::FromRandom(JobID::FromInt(1)), 0); const auto task_spec_builder = diff --git a/src/ray/rpc/node_manager/raylet_client_pool.cc b/src/ray/rpc/node_manager/raylet_client_pool.cc index bd37718b5e55..013a69055eae 100644 --- a/src/ray/rpc/node_manager/raylet_client_pool.cc +++ b/src/ray/rpc/node_manager/raylet_client_pool.cc @@ -28,12 +28,12 @@ std::function RayletClientPool::GetDefaultUnavailableTimeoutCallback( rpc::RayletClientPool *raylet_client_pool, const rpc::Address &addr) { return [addr, gcs_client, raylet_client_pool]() { - const NodeID raylet_id = NodeID::FromBinary(addr.raylet_id()); + const NodeID node_id = NodeID::FromBinary(addr.node_id()); - auto gcs_check_node_alive = [raylet_id, addr, raylet_client_pool, gcs_client]() { + auto gcs_check_node_alive = [node_id, addr, raylet_client_pool, gcs_client]() { gcs_client->Nodes().AsyncGetAll( - [addr, raylet_id, raylet_client_pool](const Status &status, - std::vector &&nodes) { + [addr, node_id, raylet_client_pool](const Status &status, + std::vector &&nodes) { if (!status.ok()) { // Will try again when unavailable timeout callback is retried. RAY_LOG(INFO) << "Failed to get node info from GCS"; @@ -47,18 +47,18 @@ std::function RayletClientPool::GetDefaultUnavailableTimeoutCallback( // maximum_gcs_dead_node_cached_count. // In this case, it must be 2 since there's no way for a component to // know about a remote node id until the gcs has registered it. - RAY_LOG(INFO).WithField(raylet_id) + RAY_LOG(INFO).WithField(node_id) << "Disconnecting raylet client because its node is dead"; - raylet_client_pool->Disconnect(raylet_id); + raylet_client_pool->Disconnect(node_id); return; } }, -1, - {raylet_id}); + {node_id}); }; if (gcs_client->Nodes().IsSubscribedToNodeChange()) { - auto *node_info = gcs_client->Nodes().Get(raylet_id, /*filter_dead_nodes=*/false); + auto *node_info = gcs_client->Nodes().Get(node_id, /*filter_dead_nodes=*/false); if (node_info == nullptr) { // Node could be dead or info may have not made it to the subscriber cache yet. // Check with the GCS to confirm if the node is dead. @@ -66,9 +66,9 @@ std::function RayletClientPool::GetDefaultUnavailableTimeoutCallback( return; } if (node_info->state() == rpc::GcsNodeInfo::DEAD) { - RAY_LOG(INFO).WithField(raylet_id) + RAY_LOG(INFO).WithField(node_id) << "Disconnecting raylet client because its node is dead."; - raylet_client_pool->Disconnect(raylet_id); + raylet_client_pool->Disconnect(node_id); return; } // Node is alive so raylet client is alive. @@ -81,18 +81,18 @@ std::function RayletClientPool::GetDefaultUnavailableTimeoutCallback( std::shared_ptr RayletClientPool::GetOrConnectByAddress( const rpc::Address &address) { - RAY_CHECK(address.raylet_id() != ""); + RAY_CHECK(address.node_id() != ""); absl::MutexLock lock(&mu_); - auto raylet_id = NodeID::FromBinary(address.raylet_id()); - auto it = client_map_.find(raylet_id); + auto node_id = NodeID::FromBinary(address.node_id()); + auto it = client_map_.find(node_id); if (it != client_map_.end()) { RAY_CHECK(it->second != nullptr); return it->second; } auto connection = client_factory_(address); - client_map_[raylet_id] = connection; + client_map_[node_id] = connection; - RAY_LOG(DEBUG) << "Connected to raylet " << raylet_id << " at " + RAY_LOG(DEBUG) << "Connected to raylet " << node_id << " at " << BuildAddress(address.ip_address(), address.port()); RAY_CHECK(connection != nullptr); return connection; @@ -116,13 +116,13 @@ void RayletClientPool::Disconnect(ray::NodeID id) { client_map_.erase(it); } -rpc::Address RayletClientPool::GenerateRayletAddress(const NodeID &raylet_id, +rpc::Address RayletClientPool::GenerateRayletAddress(const NodeID &node_id, const std::string &ip_address, int port) { rpc::Address address; address.set_ip_address(ip_address); address.set_port(port); - address.set_raylet_id(raylet_id.Binary()); + address.set_node_id(node_id.Binary()); return address; } diff --git a/src/ray/rpc/node_manager/test/raylet_client_pool_test.cc b/src/ray/rpc/node_manager/test/raylet_client_pool_test.cc index 15d81cccf7a2..6b031229354b 100644 --- a/src/ray/rpc/node_manager/test/raylet_client_pool_test.cc +++ b/src/ray/rpc/node_manager/test/raylet_client_pool_test.cc @@ -44,7 +44,7 @@ namespace { rpc::Address CreateRandomAddress(const std::string &addr) { rpc::Address address; address.set_ip_address(addr); - address.set_raylet_id(NodeID::FromRandom().Binary()); + address.set_node_id(NodeID::FromRandom().Binary()); address.set_worker_id(WorkerID::FromRandom().Binary()); return address; } @@ -123,8 +123,8 @@ TEST_P(DefaultUnavailableTimeoutCallbackTest, NodeDeath) { auto raylet_client_1_address = CreateRandomAddress("1"); auto raylet_client_2_address = CreateRandomAddress("2"); - auto raylet_client_1_node_id = NodeID::FromBinary(raylet_client_1_address.raylet_id()); - auto raylet_client_2_node_id = NodeID::FromBinary(raylet_client_2_address.raylet_id()); + auto raylet_client_1_node_id = NodeID::FromBinary(raylet_client_1_address.node_id()); + auto raylet_client_2_node_id = NodeID::FromBinary(raylet_client_2_address.node_id()); auto raylet_client_1 = dynamic_cast( raylet_client_pool_->GetOrConnectByAddress(raylet_client_1_address).get()); diff --git a/src/ray/rpc/test/core_worker_client_pool_test.cc b/src/ray/rpc/test/core_worker_client_pool_test.cc index 7f06a7c5b192..f57eb3b29430 100644 --- a/src/ray/rpc/test/core_worker_client_pool_test.cc +++ b/src/ray/rpc/test/core_worker_client_pool_test.cc @@ -49,7 +49,7 @@ namespace { rpc::Address CreateRandomAddress(const std::string &addr) { rpc::Address address; address.set_ip_address(addr); - address.set_raylet_id(NodeID::FromRandom().Binary()); + address.set_node_id(NodeID::FromRandom().Binary()); address.set_worker_id(WorkerID::FromRandom().Binary()); return address; } @@ -181,8 +181,8 @@ TEST_P(DefaultUnavailableTimeoutCallbackTest, NodeDeath) { client_pool_->GetOrConnect(worker_2_address).get()); AssertID(worker_id2, *client_pool_, true); - auto worker_1_node_id = NodeID::FromBinary(worker_1_address.raylet_id()); - auto worker_2_node_id = NodeID::FromBinary(worker_2_address.raylet_id()); + auto worker_1_node_id = NodeID::FromBinary(worker_1_address.node_id()); + auto worker_2_node_id = NodeID::FromBinary(worker_2_address.node_id()); rpc::GcsNodeInfo node_info_alive; node_info_alive.set_state(rpc::GcsNodeInfo::ALIVE); diff --git a/src/ray/rpc/worker/core_worker_client.h b/src/ray/rpc/worker/core_worker_client.h index 51a06cdb7166..cab4480b0c98 100644 --- a/src/ray/rpc/worker/core_worker_client.h +++ b/src/ray/rpc/worker/core_worker_client.h @@ -40,7 +40,7 @@ struct hash { size_t hash = std::hash()(addr.port()); hash ^= std::hash()(addr.ip_address()); hash ^= std::hash()(addr.worker_id()); - hash ^= std::hash()(addr.raylet_id()); + hash ^= std::hash()(addr.node_id()); return hash; } }; diff --git a/src/ray/rpc/worker/core_worker_client_pool.cc b/src/ray/rpc/worker/core_worker_client_pool.cc index 99df4075a900..96f46cc3cbe8 100644 --- a/src/ray/rpc/worker/core_worker_client_pool.cc +++ b/src/ray/rpc/worker/core_worker_client_pool.cc @@ -30,7 +30,7 @@ std::function CoreWorkerClientPool::GetDefaultUnavailableTimeoutCallback rpc::RayletClientPool *raylet_client_pool, const rpc::Address &addr) { return [addr, gcs_client, worker_client_pool, raylet_client_pool]() { - const NodeID node_id = NodeID::FromBinary(addr.raylet_id()); + const NodeID node_id = NodeID::FromBinary(addr.node_id()); const WorkerID worker_id = WorkerID::FromBinary(addr.worker_id()); auto check_worker_alive = [raylet_client_pool, @@ -122,7 +122,7 @@ std::shared_ptr CoreWorkerClientPool::GetOrConnect( RemoveIdleClients(); CoreWorkerClientEntry entry; - auto node_id = NodeID::FromBinary(addr_proto.raylet_id()); + auto node_id = NodeID::FromBinary(addr_proto.node_id()); auto worker_id = WorkerID::FromBinary(addr_proto.worker_id()); auto it = worker_client_map_.find(worker_id); if (it != worker_client_map_.end()) { From ea304fa8ba4c72bd6636799049b79c4383c2a744 Mon Sep 17 00:00:00 2001 From: Anmol Singh Date: Tue, 12 Aug 2025 21:34:38 +0530 Subject: [PATCH 050/634] Remove ref to deleted panel in Ray Data metrics (#55478) ## Why are these changes needed? The "Bytes Allocated" Grafana panel was removed in https://github.com/ray-project/ray/pull/52943. However, the reference to it in the Metrics UI code was not removed, and this leads to a "Panel not found" error showing up on the UI. This PR just cleans up the reference. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: anmol Co-authored-by: anmol --- python/ray/dashboard/client/src/pages/metrics/Metrics.tsx | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/ray/dashboard/client/src/pages/metrics/Metrics.tsx b/python/ray/dashboard/client/src/pages/metrics/Metrics.tsx index 07e7af851266..ea393c6e9c1a 100644 --- a/python/ray/dashboard/client/src/pages/metrics/Metrics.tsx +++ b/python/ray/dashboard/client/src/pages/metrics/Metrics.tsx @@ -221,10 +221,6 @@ const DATA_METRICS_CONFIG: MetricsSectionConfig[] = [ title: "Bytes Spilled", pathParams: "theme=light&panelId=1", }, - { - title: "Bytes Allocated", - pathParams: "theme=light&panelId=2", - }, { title: "Bytes Freed", pathParams: "theme=light&panelId=3", From 646fafed0a076344c3623cda822a07d1cf8f5ffc Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Tue, 12 Aug 2025 09:09:29 -0700 Subject: [PATCH 051/634] [core][obs-clean/01] de-static object_store_available_memory metric (#55515) Ray core currently offers two APIs for defining internal metrics: a static object-oriented (OO) API and a template/extern-based API. The OO API is also used for defining custom metrics at the Ray application level, and I personally find it easier to read. This series of PRs aims to unify all metric definitions under the OO API. ---- This PR migrates one metric from static to runtime definition, as part of the effort to eliminate all statically defined metrics. Currently, the OO interface attempts to register a metric at the same time its first value is recorded, due to the [C++ static initialization order fiasco](https://en.cppreference.com/w/cpp/language/siof.html), which is awkward and potentially inefficient. We can fix this by removing all statically defined metrics. Test: - CI Signed-off-by: Cuong Nguyen --- src/ray/object_manager/object_manager.cc | 2 +- src/ray/object_manager/object_manager.h | 7 +++++++ src/ray/stats/metric_defs.h | 5 ----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index e2040c3aa297..405559f58b55 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -767,7 +767,7 @@ void ObjectManager::RecordMetrics() { push_manager_->RecordMetrics(); // used_memory_ includes the fallback allocation, so we should add it again here // to calculate the exact available memory. - stats::ObjectStoreAvailableMemory().Record( + ray_metric_object_store_available_memory_.Record( config_.object_store_memory - used_memory_ + plasma::plasma_store_runner->GetFallbackAllocated()); // Subtract fallback allocated memory. It is tracked separately by diff --git a/src/ray/object_manager/object_manager.h b/src/ray/object_manager/object_manager.h index 8f20893d1d46..534ac3fcb7b6 100644 --- a/src/ray/object_manager/object_manager.h +++ b/src/ray/object_manager/object_manager.h @@ -32,6 +32,7 @@ #include "ray/object_manager/push_manager.h" #include "ray/rpc/object_manager/object_manager_client.h" #include "ray/rpc/object_manager/object_manager_server.h" +#include "ray/stats/metric.h" #include "src/ray/protobuf/common.pb.h" #include "src/ray/protobuf/node_manager.pb.h" @@ -497,6 +498,12 @@ class ObjectManager : public ObjectManagerInterface, /// create the object in plasma. This is usually due to out-of-memory in /// plasma. size_t num_chunks_received_failed_due_to_plasma_ = 0; + + /// Metrics + ray::stats::Gauge ray_metric_object_store_available_memory_{ + /*name=*/"object_store_available_memory", + /*description=*/"Amount of memory currently available in the object store.", + /*unit=*/"bytes"}; }; } // namespace ray diff --git a/src/ray/stats/metric_defs.h b/src/ray/stats/metric_defs.h index ebd44994d3a0..ac07a5de106e 100644 --- a/src/ray/stats/metric_defs.h +++ b/src/ray/stats/metric_defs.h @@ -164,11 +164,6 @@ static Gauge LocalTotalResource("local_total_resource", {kResourceNameKey}); /// Object Manager. -static Gauge ObjectStoreAvailableMemory( - "object_store_available_memory", - "Amount of memory currently available in the object store.", - "bytes"); - static Gauge ObjectStoreUsedMemory( "object_store_used_memory", "Amount of memory currently occupied in the object store.", From fea78b7b09a47f0f10cc07224a45da4c2e731d15 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Tue, 12 Aug 2025 09:54:41 -0700 Subject: [PATCH 052/634] [ci] raydepsets: storing build arg set name in depset (#55504) instead of storing an entire build arg set object in a depset, only store the name for reference Signed-off-by: elliot-barn --- ci/raydepsets/tests/test_cli.py | 6 ++---- ci/raydepsets/workspace.py | 10 ++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 5ee194f96e88..a3f017ece473 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -462,9 +462,7 @@ def test_get_depset_with_build_arg_set(self): ) depset = manager.get_depset("build_args_test_depset_py311") assert depset.name == "build_args_test_depset_py311" - assert depset.build_arg_set.name == "py311_cpu" - assert depset.build_arg_set.build_args["PYTHON_VERSION"] == "py311" - assert depset.build_arg_set.build_args["CUDA_VERSION"] == "cpu" + assert depset.build_arg_set_name == "py311_cpu" def test_get_depset_without_build_arg_set(self): with tempfile.TemporaryDirectory() as tmpdir: @@ -475,7 +473,7 @@ def test_get_depset_without_build_arg_set(self): ) depset = manager.get_depset("ray_base_test_depset") assert depset.name == "ray_base_test_depset" - assert depset.build_arg_set is None + assert depset.build_arg_set_name is None if __name__ == "__main__": diff --git a/ci/raydepsets/workspace.py b/ci/raydepsets/workspace.py index ccfced7b9052..afc5fc3618b0 100644 --- a/ci/raydepsets/workspace.py +++ b/ci/raydepsets/workspace.py @@ -23,7 +23,7 @@ class Depset: append_flags: List[str] source_depset: Optional[str] = None depsets: Optional[List[str]] = None - build_arg_set: Optional[BuildArgSet] = None + build_arg_set_name: Optional[str] = None def _substitute_build_args(obj: Any, build_arg_set: BuildArgSet): @@ -40,9 +40,7 @@ def _substitute_build_args(obj: Any, build_arg_set: BuildArgSet): return obj -def _dict_to_depset( - depset: dict, build_arg_set: Optional[BuildArgSet] = None -) -> Depset: +def _dict_to_depset(depset: dict, build_arg_set_name: Optional[str] = None) -> Depset: return Depset( name=depset.get("name"), requirements=depset.get("requirements", []), @@ -51,7 +49,7 @@ def _dict_to_depset( output=depset.get("output"), source_depset=depset.get("source_depset"), depsets=depset.get("depsets", []), - build_arg_set=build_arg_set, + build_arg_set_name=build_arg_set_name, override_flags=depset.get("override_flags", []), append_flags=depset.get("append_flags", []), ) @@ -82,7 +80,7 @@ def from_dict(data: dict) -> "Config": if build_arg_set is None: raise KeyError(f"Build arg set {build_arg_set_name} not found") depset_yaml = _substitute_build_args(depset, build_arg_set) - depsets.append(_dict_to_depset(depset_yaml, build_arg_set)) + depsets.append(_dict_to_depset(depset_yaml, build_arg_set_name)) else: depsets.append(_dict_to_depset(depset=depset)) return Config(depsets=depsets, build_arg_sets=build_arg_sets) From 0eb2bfe758eec10559cf94eae525a44719a1194a Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 12 Aug 2025 22:26:41 +0530 Subject: [PATCH 053/634] [wheel] switch to installing ``bazelisk`` binary instead of installing from ``npm`` (#55375) this allows separately build the dashboard in another build environment other than the manylinux container that is used for building the c/c++ parts, which not only makes the build more parallel, but also make it possible to upgrade npm to a much newer version. --------- Signed-off-by: Gagandeep Singh --- ci/build/build-manylinux-forge.sh | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/ci/build/build-manylinux-forge.sh b/ci/build/build-manylinux-forge.sh index 483aa167ae3e..29a52e3651a6 100755 --- a/ci/build/build-manylinux-forge.sh +++ b/ci/build/build-manylinux-forge.sh @@ -3,6 +3,12 @@ set -exuo pipefail +BAZELISK_VERSION="v1.26.0" + +platform="linux" + +echo "Architecture(HOSTTYPE) is ${HOSTTYPE}" + if [[ ! -e /usr/bin/nproc ]]; then echo -e '#!/bin/bash\necho 10' > "/usr/bin/nproc" chmod +x /usr/bin/nproc @@ -33,12 +39,22 @@ nvm install "$NODE_VERSION" nvm use "$NODE_VERSION" # Install bazel -npm install -g @bazel/bazelisk mkdir -p "$HOME"/bin -ln -sf "$(which bazelisk)" "$HOME"/bin/bazel +if [[ "${HOSTTYPE}" == "aarch64" || "${HOSTTYPE}" = "arm64" ]]; then + # architecture is "aarch64", but the bazel tag is "arm64" + BAZELISK_URL="https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-${platform}-arm64" +elif [[ "${HOSTTYPE}" == "x86_64" ]]; then + BAZELISK_URL="https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-${platform}-amd64" +else + echo "Could not found matching bazelisk URL for platform ${platform} and architecture ${HOSTTYPE}" + exit 1 +fi +curl -sSfL -o "$HOME"/bin/bazelisk "${BAZELISK_URL}" +chmod +x "$HOME"/bin/bazelisk +sudo ln -sf "$HOME"/bin/bazelisk /usr/local/bin/bazel # Use python3.9 as default python3 -ln -sf /usr/local/bin/python3.9 /usr/local/bin/python3 +sudo ln -sf /usr/local/bin/python3.9 /usr/local/bin/python3 { echo "build --config=ci" From 64f32074a34b5bf7c7879be664957482f17a9de8 Mon Sep 17 00:00:00 2001 From: Kai-Hsun Chen Date: Tue, 12 Aug 2025 10:37:27 -0700 Subject: [PATCH 054/634] [core][gpu-objects] Exception handling for application errors (#55442) https://github.com/ray-project/ray/pull/55427 and https://github.com/ray-project/ray/pull/55433 fixed exception handling for application-level errors. After #55433, the logic will look like: * The sender actor task throws an exception. * `serialize_and_store_gpu_objects` calls `GPUObjectStore.add_object(...)` to add a new object, ensuring that `_get_tensor_meta` does not hang. * `__ray_send__` and `__ray_recv__` will not actually transfer any tensor out of band, since `tensor_meta` is empty. * The exception will be sent through the normal object store to its consumers. ## Related issue number #51275 Signed-off-by: Kai-Hsun Chen --- python/ray/tests/test_gpu_objects_gloo.py | 52 +++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/python/ray/tests/test_gpu_objects_gloo.py b/python/ray/tests/test_gpu_objects_gloo.py index a1b10fc4cb47..ead9c93461a1 100644 --- a/python/ray/tests/test_gpu_objects_gloo.py +++ b/python/ray/tests/test_gpu_objects_gloo.py @@ -43,6 +43,9 @@ def get_num_gpu_objects(self): gpu_object_manager = ray._private.worker.global_worker.gpu_object_manager return gpu_object_manager.gpu_object_store.get_num_objects() + def fail(self, error_message): + raise Exception(error_message) + @pytest.mark.parametrize("data_size_bytes", [100]) def test_gc_gpu_object(ray_start_regular, data_size_bytes): @@ -490,5 +493,54 @@ def test_tensor_extracted_from_tensordict_in_gpu_object_store(ray_start_regular) assert torch.equal(ret_val_src[1], td["reward"]) +def test_app_error_inter_actor(ray_start_regular): + world_size = 2 + actors = [GPUTestActor.remote() for _ in range(world_size)] + create_collective_group(actors, backend="torch_gloo") + + src_actor, dst_actor = actors[0], actors[1] + + # Make sure the receiver can receive an exception from the sender. + ref = src_actor.fail.options(tensor_transport="gloo").remote("test_app_error") + with pytest.raises(Exception, match="test_app_error"): + ray.get(dst_actor.double.remote(ref)) + + # Make sure the sender and receiver do not hang. + small_tensor = torch.randn((1,)) + ref = src_actor.echo.remote(small_tensor) + result = dst_actor.double.remote(ref) + assert ray.get(result) == pytest.approx(small_tensor * 2) + + +def test_app_error_intra_actor(ray_start_regular): + actor = GPUTestActor.remote() + create_collective_group([actor], backend="torch_gloo") + + # Make sure the receiver can receive an exception from the sender. + ref = actor.fail.options(tensor_transport="gloo").remote("test_app_error") + with pytest.raises(Exception, match="test_app_error"): + ray.get(actor.double.remote(ref)) + + # Make sure the sender and receiver do not hang. + small_tensor = torch.randn((1,)) + ref = actor.echo.remote(small_tensor) + result = actor.double.remote(ref) + assert ray.get(result) == pytest.approx(small_tensor * 2) + + +def test_app_error_fetch_to_driver(ray_start_regular): + actor = GPUTestActor.remote() + create_collective_group([actor], backend="torch_gloo") + + ref = actor.fail.options(tensor_transport="gloo").remote("test_app_error") + with pytest.raises(Exception, match="test_app_error"): + ray.get(ref) + + # Make sure the driver can receive an exception from the actor. + small_tensor = torch.tensor([1, 2, 3]) + ref = actor.echo.remote(small_tensor) + assert torch.equal(ray.get(ref), small_tensor) + + if __name__ == "__main__": sys.exit(pytest.main(["-sv", __file__])) From ad184b085da4c452559fa9bf73f6a59e9aeb8641 Mon Sep 17 00:00:00 2001 From: goutamvenkat-anyscale Date: Tue, 12 Aug 2025 11:04:58 -0700 Subject: [PATCH 055/634] [Data] Update streaming_exec test by removing upper bound on assertion (#55489) ## Why are these changes needed? After introducing stats to the streaming executor, the `streaming_exec_schedule_s` value seems to fluctuate, so we will remove the upper bound in the test assertion. ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Goutam V --- python/ray/data/tests/test_streaming_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/data/tests/test_streaming_executor.py b/python/ray/data/tests/test_streaming_executor.py index ae9f0e4ef286..a8989c2bd409 100644 --- a/python/ray/data/tests/test_streaming_executor.py +++ b/python/ray/data/tests/test_streaming_executor.py @@ -579,7 +579,7 @@ def test_streaming_exec_schedule_s(): continue ds_stats = ds._plan.stats() - assert 0 < ds_stats.streaming_exec_schedule_s.get() < 1 + assert ds_stats.streaming_exec_schedule_s.get() > 0 def test_execution_callbacks(): From bd3807072d94ec71fdf46d181b277ba19efa9505 Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Tue, 12 Aug 2025 11:14:48 -0700 Subject: [PATCH 056/634] [serve] Update compute type for serve microbenchmarks (#55520) ## Why are these changes needed? We've been using m7a.4xlarge for lots of perf benchmarks. We should update these nightly benchmarks with the same hw type. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: akyang-anyscale --- release/release_tests.yaml | 2 +- .../compute_tpl_single_node_16_cpu.yaml | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 release/serve_tests/compute_tpl_single_node_16_cpu.yaml diff --git a/release/release_tests.yaml b/release/release_tests.yaml index 9edc6f66a9ac..8d096b1f2d44 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -2058,7 +2058,7 @@ cluster: byod: {} - cluster_compute: compute_tpl_single_node_32_cpu.yaml + cluster_compute: compute_tpl_single_node_16_cpu.yaml cloud_id: cld_wy5a6nhazplvu32526ams61d98 project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq diff --git a/release/serve_tests/compute_tpl_single_node_16_cpu.yaml b/release/serve_tests/compute_tpl_single_node_16_cpu.yaml new file mode 100644 index 000000000000..d4684d799118 --- /dev/null +++ b/release/serve_tests/compute_tpl_single_node_16_cpu.yaml @@ -0,0 +1,18 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +max_workers: 0 + +head_node_type: + name: head_node + # 16 cpus, arm, 64G mem, 12.5Gb NIC + instance_type: m7a.4xlarge + +worker_node_types: [] + +advanced_configurations_json: + TagSpecifications: + - ResourceType: "instance" + Tags: + - Key: ttl-hours + Value: '24' From 961efc45b71d0d5e95cd8d553f1785febd33ce55 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:14:58 -0700 Subject: [PATCH 057/634] [depset] remove unsafe treatment for grpcio-tools (#55516) not required any more. `grpcio-tools` are now resolving. Signed-off-by: Lonnie Liu --- ci/raydepsets/cli.py | 1 - ci/raydepsets/tests/test_cli.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index 6ecadca48917..a6e046c2d2a2 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -16,7 +16,6 @@ --emit-index-url --emit-find-links --unsafe-package ray - --unsafe-package grpcio-tools --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index a3f017ece473..9bb61764c2d9 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -298,8 +298,6 @@ def test_override_uv_flag_multiple_flags(self): expected_flags.remove("--unsafe-package") expected_flags.remove("ray") expected_flags.remove("--unsafe-package") - expected_flags.remove("grpcio-tools") - expected_flags.remove("--unsafe-package") expected_flags.remove("setuptools") expected_flags.extend(["--unsafe-package", "dummy"]) assert ( From 36f5f498f421e7b1799985abc271ae084981a2c2 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:32:58 -0700 Subject: [PATCH 058/634] [image] auto detect architecture type (#55509) do not depends on the build arg Signed-off-by: Lonnie Liu --- ci/docker/ray.cpu.base.aarch64.wanda.yaml | 1 - ci/docker/ray.cuda.base.aarch64.wanda.yaml | 1 - docker/base-deps/Dockerfile | 13 +++++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ci/docker/ray.cpu.base.aarch64.wanda.yaml b/ci/docker/ray.cpu.base.aarch64.wanda.yaml index 1726fb261825..70d81359ad31 100644 --- a/ci/docker/ray.cpu.base.aarch64.wanda.yaml +++ b/ci/docker/ray.cpu.base.aarch64.wanda.yaml @@ -6,6 +6,5 @@ srcs: build_args: - PYTHON_VERSION - BASE_IMAGE=ubuntu:22.04 - - HOSTTYPE=aarch64 tags: - cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base-aarch64 diff --git a/ci/docker/ray.cuda.base.aarch64.wanda.yaml b/ci/docker/ray.cuda.base.aarch64.wanda.yaml index 1d1d6df12787..325525355b44 100644 --- a/ci/docker/ray.cuda.base.aarch64.wanda.yaml +++ b/ci/docker/ray.cuda.base.aarch64.wanda.yaml @@ -6,6 +6,5 @@ srcs: build_args: - PYTHON_VERSION - BASE_IMAGE=nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04 - - HOSTTYPE=aarch64 tags: - cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base-aarch64 diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index 4d183a6ba892..06d1fa73e3eb 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -13,7 +13,6 @@ ENV LANG=C.UTF-8 ENV PATH "/home/ray/anaconda3/bin:$PATH" ARG DEBIAN_FRONTEND=noninteractive ARG PYTHON_VERSION=3.9 -ARG HOSTTYPE=${HOSTTYPE:-x86_64} ARG RAY_UID=1000 ARG RAY_GID=100 @@ -65,9 +64,19 @@ RUN </dev/stderr + exit 1 +fi + # Install miniforge wget --quiet \ - "https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-24.11.3-0-Linux-${HOSTTYPE}.sh" \ + "https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-24.11.3-0-Linux-${ARCH}.sh" \ -O /tmp/miniforge.sh /bin/bash /tmp/miniforge.sh -b -u -p $HOME/anaconda3 From 491908a0c0f422a3989b891bba9acaf73074672c Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:33:25 -0700 Subject: [PATCH 059/634] [image] remove uninstall dask in base image (#55508) dask is not installed in base image anymore. it is not a dependency of any of the installed required python packages. Signed-off-by: Lonnie Liu --- docker/base-deps/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index 06d1fa73e3eb..1320aed23d53 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -121,10 +121,6 @@ uv pip install --system --no-cache-dir --index-strategy unsafe-best-match \ -c $HOME/requirements_compiled.txt \ "${PIP_PKGS[@]}" -# To avoid the following error on Jenkins: -# AttributeError: 'numpy.ufunc' object has no attribute '__module__' -uv pip uninstall --system dask - # We install cmake temporarily to get psutil sudo apt-get autoremove -y cmake zlib1g-dev From 5d05513c9ac8abb36ab81c1d91b9ee2b5a688fb9 Mon Sep 17 00:00:00 2001 From: Abrar Sheikh Date: Tue, 12 Aug 2025 11:38:41 -0700 Subject: [PATCH 060/634] make get_current_servable_instance async (#55457) This PR updates the injected dependency used in FastAPI routes to be asynchronous. Specifically, it modifies the implementation of get_current_servable_instance to be an async def so that FastAPI awaits it instead of offloading it to a threadpool. This change avoids unnecessary threadpool overhead and improves request performance. ### Performance Comparison Table (With fix vs Master) | Metric | With fix | Master | |--------------------------|----------------|----------------| | # Requests | 22,429 | 12,970 | | # Fails | 0 | 0 | | Median (ms) | 250 | 340 | | 95%ile (ms) | 310 | 430 | | 99%ile (ms) | 350 | 480 | | Average (ms) | 248.04 | 338.4 | | Min (ms) | 119 | 156 | | Max (ms) | 392 | 662 | | Average size (bytes) | 15 | 15 | | Current RPS | 400.2 | 295.7 | | Current Failures/s | 0 | 0 | evidence that fastapi dependency injection is using threadpool ### master image ### after fix image ### app used for benchmark ```python app = FastAPI() @serve.deployment(max_ongoing_requests=10000) @serve.ingress(app) class MyDeployment: @app.get("/") async def request(self): return "Hello, world!" app = MyDeployment.bind() ``` run using `serve run app:app` device used: `m6a.2xlarge` Concurrency: `100` Signed-off-by: abrar --- python/ray/serve/_private/http_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/serve/_private/http_util.py b/python/ray/serve/_private/http_util.py index ee233d7c354d..2da5b8223964 100644 --- a/python/ray/serve/_private/http_util.py +++ b/python/ray/serve/_private/http_util.py @@ -432,7 +432,7 @@ def make_fastapi_class_based_view(fastapi_app, cls: Type) -> None: from fastapi import APIRouter, Depends from fastapi.routing import APIRoute, APIWebSocketRoute - def get_current_servable_instance(): + async def get_current_servable_instance(): from ray import serve return serve.get_replica_context().servable_object From 6030ee7d293ed62c5b1253f52a30ea67ff4ee9b1 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 12 Aug 2025 12:10:12 -0700 Subject: [PATCH 061/634] [Core] Install uv from test-requirements.txt (#55483) Signed-off-by: Jiajun Yao --- python/ray/tests/test_runtime_env_uv_run.py | 245 +++++++++----------- python/requirements/test-requirements.txt | 1 + python/requirements_compiled.txt | 2 + 3 files changed, 116 insertions(+), 132 deletions(-) diff --git a/python/ray/tests/test_runtime_env_uv_run.py b/python/ray/tests/test_runtime_env_uv_run.py index 138ff87e81bf..1db6424432d3 100644 --- a/python/ray/tests/test_runtime_env_uv_run.py +++ b/python/ray/tests/test_runtime_env_uv_run.py @@ -1,15 +1,12 @@ import json import os from pathlib import Path -import platform -import stat import subprocess import sys -import tarfile import tempfile -from urllib import request import pytest +from uv import find_uv_bin import ray from ray._private.test_utils import ( @@ -18,21 +15,6 @@ ) -@pytest.fixture(scope="function") -def with_uv(): - arch = "aarch64" if platform.machine() in ["aarch64", "arm64"] else "x86_64" - system = "unknown-linux-gnu" if platform.system() == "Linux" else "apple-darwin" - name = f"uv-{arch}-{system}" - url = f"https://github.com/astral-sh/uv/releases/download/0.5.27/{name}.tar.gz" - with tempfile.TemporaryDirectory() as tmp_dir: - with request.urlopen(request.Request(url), timeout=15.0) as response: - with tarfile.open(fileobj=response, mode="r|*") as tar: - tar.extractall(tmp_dir) - uv = Path(tmp_dir) / name / "uv" - uv.chmod(uv.stat().st_mode | stat.S_IEXEC) - yield uv - - PYPROJECT_TOML = """ [project] name = "test" @@ -58,11 +40,9 @@ def tmp_working_dir(): @pytest.mark.skipif(sys.platform == "win32", reason="Not ported to Windows yet.") -def test_uv_run_simple(shutdown_only, with_uv): - uv = with_uv - +def test_uv_run_simple(shutdown_only): runtime_env = { - "py_executable": f"{uv} run --with emoji --no-project", + "py_executable": f"{find_uv_bin()} run --with emoji --no-project", } ray.init(runtime_env=runtime_env) @@ -76,15 +56,14 @@ def emojize(): @pytest.mark.skipif(sys.platform == "win32", reason="Not ported to Windows yet.") -def test_uv_run_pyproject(shutdown_only, with_uv, tmp_working_dir): - uv = with_uv +def test_uv_run_pyproject(shutdown_only, tmp_working_dir): tmp_dir = tmp_working_dir ray.init( runtime_env={ "working_dir": tmp_dir, # We want to run in the system environment so the current installation of Ray can be found here - "py_executable": f"env PYTHONPATH={':'.join(sys.path)} {uv} run --python-preference=only-system", + "py_executable": f"env PYTHONPATH={':'.join(sys.path)} {find_uv_bin()} run --python-preference=only-system", } ) @@ -98,8 +77,7 @@ def emojize(): @pytest.mark.skipif(sys.platform == "win32", reason="Not ported to Windows yet.") -def test_uv_run_editable(shutdown_only, with_uv, tmp_working_dir): - uv = with_uv +def test_uv_run_editable(shutdown_only, tmp_working_dir): tmp_dir = tmp_working_dir subprocess.run( @@ -113,7 +91,7 @@ def test_uv_run_editable(shutdown_only, with_uv, tmp_working_dir): ) subprocess.run( - [uv, "add", "--editable", "./emoji_copy"], + [find_uv_bin(), "add", "--editable", "./emoji_copy"], cwd=tmp_dir, ) @@ -133,7 +111,7 @@ def test_uv_run_editable(shutdown_only, with_uv, tmp_working_dir): runtime_env={ "working_dir": tmp_dir, # We want to run in the system environment so the current installation of Ray can be found here - "py_executable": f"env PYTHONPATH={':'.join(sys.path)} {uv} run --python-preference=only-system", + "py_executable": f"env PYTHONPATH={':'.join(sys.path)} {find_uv_bin()} run --python-preference=only-system", } ) @@ -147,12 +125,10 @@ def emojize(): @pytest.mark.skipif(sys.platform == "win32", reason="Not ported to Windows yet.") -def test_uv_run_runtime_env_hook(with_uv): +def test_uv_run_runtime_env_hook(): import ray._private.runtime_env.uv_runtime_env_hook - uv = with_uv - def check_uv_run( cmd, runtime_env, expected_output, subprocess_kwargs=None, expected_error=None ): @@ -171,25 +147,28 @@ def check_uv_run( script = ray._private.runtime_env.uv_runtime_env_hook.__file__ check_uv_run( - cmd=[uv, "run", "--no-project", script], + cmd=[find_uv_bin(), "run", "--no-project", script], runtime_env={}, expected_output={ - "py_executable": f"{uv} run --no-project", + "py_executable": f"{find_uv_bin()} run --no-project", "working_dir": os.getcwd(), }, ) check_uv_run( - cmd=[uv, "run", "--no-project", "--directory", "/tmp", script], + cmd=[find_uv_bin(), "run", "--no-project", "--directory", "/tmp", script], runtime_env={}, expected_output={ - "py_executable": f"{uv} run --no-project", + "py_executable": f"{find_uv_bin()} run --no-project", "working_dir": os.path.realpath("/tmp"), }, ) check_uv_run( - [uv, "run", "--no-project", script], + [find_uv_bin(), "run", "--no-project", script], {"working_dir": "/some/path"}, - {"py_executable": f"{uv} run --no-project", "working_dir": "/some/path"}, + { + "py_executable": f"{find_uv_bin()} run --no-project", + "working_dir": "/some/path", + }, ) with tempfile.TemporaryDirectory() as tmp_dir: @@ -200,9 +179,12 @@ def check_uv_run( file.write('version = "0.1"\n') file.write('dependencies = ["psutil"]\n') check_uv_run( - cmd=[uv, "run", script], + cmd=[find_uv_bin(), "run", script], runtime_env={}, - expected_output={"py_executable": f"{uv} run", "working_dir": f"{tmp_dir}"}, + expected_output={ + "py_executable": f"{find_uv_bin()} run", + "working_dir": f"{tmp_dir}", + }, subprocess_kwargs={"cwd": tmp_dir}, ) @@ -213,10 +195,10 @@ def check_uv_run( with open(requirements, "w") as file: file.write("psutil\n") check_uv_run( - cmd=[uv, "run", "--with-requirements", requirements, script], + cmd=[find_uv_bin(), "run", "--with-requirements", requirements, script], runtime_env={}, expected_output={ - "py_executable": f"{uv} run --with-requirements {requirements}", + "py_executable": f"{find_uv_bin()} run --with-requirements {requirements}", "working_dir": f"{tmp_dir}", }, subprocess_kwargs={"cwd": tmp_dir}, @@ -232,7 +214,7 @@ def check_uv_run( file.write('version = "0.1"\n') file.write('dependencies = ["psutil"]\n') check_uv_run( - cmd=[uv, "run", script], + cmd=[find_uv_bin(), "run", script], runtime_env={}, expected_output=None, subprocess_kwargs={"cwd": tmp_dir / "cwd"}, @@ -247,7 +229,7 @@ def check_uv_run( file.write("psutil\n") check_uv_run( cmd=[ - uv, + find_uv_bin(), "run", "--with-requirements", tmp_dir / "requirements.txt", @@ -263,7 +245,7 @@ def check_uv_run( # when combined with the 'pip' or 'uv' environment. for runtime_env in [{"uv": ["emoji"]}, {"pip": ["emoji"]}]: check_uv_run( - cmd=[uv, "run", "--no-project", script], + cmd=[find_uv_bin(), "run", "--no-project", script], runtime_env=runtime_env, expected_output=None, expected_error="You are using the 'pip' or 'uv' runtime environments together with 'uv run'.", @@ -275,10 +257,10 @@ def check_uv_run( # Check in the case that there is one more level of subprocess indirection between # the "uv run" process and the process that checks the environment check_uv_run( - cmd=[uv, "run", "--no-project", script], + cmd=[find_uv_bin(), "run", "--no-project", script], runtime_env={}, expected_output={ - "py_executable": f"{uv} run --no-project", + "py_executable": f"{find_uv_bin()} run --no-project", "working_dir": os.getcwd(), }, subprocess_kwargs={ @@ -288,10 +270,10 @@ def check_uv_run( # Check in the case that the script is started with multiprocessing spawn check_uv_run( - cmd=[uv, "run", "--no-project", script], + cmd=[find_uv_bin(), "run", "--no-project", script], runtime_env={}, expected_output={ - "py_executable": f"{uv} run --no-project", + "py_executable": f"{find_uv_bin()} run --no-project", "working_dir": os.getcwd(), }, subprocess_kwargs={ @@ -302,7 +284,7 @@ def check_uv_run( # Check in the case that a module is used for "uv run" (-m or --module) check_uv_run( cmd=[ - uv, + find_uv_bin(), "run", "--no-project", "-m", @@ -310,7 +292,7 @@ def check_uv_run( ], runtime_env={}, expected_output={ - "py_executable": f"{uv} run --no-project", + "py_executable": f"{find_uv_bin()} run --no-project", "working_dir": os.getcwd(), }, ) @@ -319,7 +301,7 @@ def check_uv_run( # an argument immediately behind it check_uv_run( cmd=[ - uv, + find_uv_bin(), "run", "--no-project", "-m", @@ -328,7 +310,7 @@ def check_uv_run( ], runtime_env={}, expected_output={ - "py_executable": f"{uv} run --no-project", + "py_executable": f"{find_uv_bin()} run --no-project", "working_dir": os.getcwd(), }, ) @@ -385,10 +367,9 @@ def test_uv_run_parser(): @pytest.mark.skipif(sys.platform == "win32", reason="Not ported to Windows yet.") -def test_uv_run_runtime_env_hook_e2e(shutdown_only, with_uv, temp_dir): +def test_uv_run_runtime_env_hook_e2e(shutdown_only, temp_dir): - uv = with_uv - tmp_out_dir = Path(temp_dir) + tmp_dir = Path(temp_dir) script = f""" import json @@ -400,39 +381,41 @@ def f(): import emoji return {{"working_dir_files": os.listdir(os.getcwd())}} -with open("{tmp_out_dir / "output.txt"}", "w") as out: +with open("{tmp_dir / "output.txt"}", "w") as out: json.dump(ray.get(f.remote()), out) """ - with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f: + working_dir = tmp_dir / "working_dir" + working_dir.mkdir(parents=True, exist_ok=True) + + script_file = working_dir / "script.py" + with open(script_file, "w") as f: f.write(script) f.close() - subprocess.run( - [ - uv, - "run", - # We want to run in the system environment so the current installation of Ray can be found here - "--python-preference=only-system", - "--with", - "emoji", - "--no-project", - f.name, - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env={ - "RAY_RUNTIME_ENV_HOOK": "ray._private.runtime_env.uv_runtime_env_hook.hook", - "PYTHONPATH": ":".join(sys.path), - "PATH": os.environ["PATH"], - }, - cwd=os.path.dirname(uv), - check=True, - ) - with open(tmp_out_dir / "output.txt") as f: - assert json.load(f) == { - "working_dir_files": os.listdir(os.path.dirname(uv)) - } + + subprocess.run( + [ + find_uv_bin(), + "run", + # We want to run in the system environment so the current installation of Ray can be found here + "--python-preference=only-system", + "--with", + "emoji", + "--no-project", + str(script_file), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env={ + "PYTHONPATH": ":".join(sys.path), + "PATH": os.environ["PATH"], + }, + cwd=working_dir, + check=True, + ) + with open(tmp_dir / "output.txt") as f: + assert json.load(f) == {"working_dir_files": os.listdir(working_dir)} @pytest.mark.skipif(sys.platform == "win32", reason="Not ported to Windows yet.") @@ -440,23 +423,19 @@ def f(): "ray_start_cluster_head_with_env_vars", [ { - "env_vars": { - "RAY_RUNTIME_ENV_HOOK": "ray._private.runtime_env.uv_runtime_env_hook.hook" - }, "include_dashboard": True, } ], indirect=True, ) def test_uv_run_runtime_env_hook_e2e_job( - ray_start_cluster_head_with_env_vars, with_uv, temp_dir + ray_start_cluster_head_with_env_vars, temp_dir ): cluster = ray_start_cluster_head_with_env_vars assert wait_until_server_available(cluster.webui_url) is True webui_url = format_web_url(cluster.webui_url) - uv = with_uv - tmp_out_dir = Path(temp_dir) + tmp_dir = Path(temp_dir) script = f""" import json @@ -468,52 +447,54 @@ def f(): import emoji return {{"working_dir_files": os.listdir(os.getcwd())}} -with open("{tmp_out_dir / "output.txt"}", "w") as out: +with open("{tmp_dir / "output.txt"}", "w") as out: json.dump(ray.get(f.remote()), out) """ - with tempfile.NamedTemporaryFile( - "w", suffix=".py", delete=False - ) as f, tempfile.NamedTemporaryFile("w", delete=False) as requirements: + working_dir = tmp_dir / "working_dir" + working_dir.mkdir(parents=True, exist_ok=True) + + script_file = working_dir / "script.py" + with open(script_file, "w") as f: f.write(script) f.close() - requirements.write("emoji\n") - requirements.close() - # Test job submission - runtime_env_json = ( - '{"env_vars": {"PYTHONPATH": "' - + ":".join(sys.path) - + '"}, "working_dir": "."}' - ) - subprocess.run( - [ - "ray", - "job", - "submit", - "--runtime-env-json", - runtime_env_json, - "--", - uv, - "run", - "--with-requirements", - requirements.name, - "--no-project", - f.name, - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env={ - "PATH": os.environ["PATH"], - "RAY_ADDRESS": webui_url, - }, - cwd=os.path.dirname(uv), - check=True, - ) - with open(tmp_out_dir / "output.txt") as f: - assert json.load(f) == { - "working_dir_files": os.listdir(os.path.dirname(uv)) - } + + requirements_file = working_dir / "requirements.txt" + with open(requirements_file, "w") as f: + f.write("emoji\n") + f.close() + + # Test job submission + runtime_env_json = ( + '{"env_vars": {"PYTHONPATH": "' + ":".join(sys.path) + '"}, "working_dir": "."}' + ) + subprocess.run( + [ + "ray", + "job", + "submit", + "--runtime-env-json", + runtime_env_json, + "--", + find_uv_bin(), + "run", + "--with-requirements", + str(requirements_file), + "--no-project", + str(script_file), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env={ + "PATH": os.environ["PATH"], + "RAY_ADDRESS": webui_url, + }, + cwd=working_dir, + check=True, + ) + with open(tmp_dir / "output.txt") as f: + assert json.load(f) == {"working_dir_files": os.listdir(working_dir)} if __name__ == "__main__": diff --git a/python/requirements/test-requirements.txt b/python/requirements/test-requirements.txt index 7932603cab7a..d08dc4ad4215 100644 --- a/python/requirements/test-requirements.txt +++ b/python/requirements/test-requirements.txt @@ -64,6 +64,7 @@ smart_open[s3]==6.2.0 tqdm==4.67.1 trustme==0.9.0 testfixtures==7.0.0 +uv==0.8.9 uvicorn==0.22.0 vsphere-automation-sdk @ git+https://github.com/vmware/vsphere-automation-sdk-python.git@v8.0.1.0 werkzeug==2.3.8 diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt index b43938ccf1d0..d4cc152a26c0 100644 --- a/python/requirements_compiled.txt +++ b/python/requirements_compiled.txt @@ -2405,6 +2405,8 @@ urllib3==1.26.19 # sentry-sdk utilsforecast==0.2.0 # via statsforecast +uv==0.8.9 + # via -r python/requirements/test-requirements.txt uvicorn==0.22.0 # via # -r python/requirements.txt From 83ddc96e7412f5b949027cb94ed0dd942d86fa48 Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Tue, 12 Aug 2025 13:01:35 -0700 Subject: [PATCH 062/634] [data] Task metric improvements (#55429) image ## Why are these changes needed? - Adds pXX filters to reduce number of panels - Adds Task output backpressure time - Fixes metrics for task completion time - wall time for task completion without backpressure Reference PRs: - https://github.com/iamjustinhsu/ray/pull/1 - https://github.com/ray-project/ray/pull/55025 ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- .../client/src/pages/metrics/Metrics.tsx | 16 +--- .../dashboards/data_dashboard_panels.py | 80 +++---------------- .../interfaces/op_runtime_metrics.py | 24 +++--- python/ray/data/tests/test_stats.py | 21 +++-- 4 files changed, 45 insertions(+), 96 deletions(-) diff --git a/python/ray/dashboard/client/src/pages/metrics/Metrics.tsx b/python/ray/dashboard/client/src/pages/metrics/Metrics.tsx index ea393c6e9c1a..f446e79e0605 100644 --- a/python/ray/dashboard/client/src/pages/metrics/Metrics.tsx +++ b/python/ray/dashboard/client/src/pages/metrics/Metrics.tsx @@ -325,20 +325,8 @@ const DATA_METRICS_CONFIG: MetricsSectionConfig[] = [ pathParams: "theme=light&panelId=37", }, { - title: "(p50) Task Completion Time", - pathParams: "theme=light&panelId=40", - }, - { - title: "(p75) Task Completion Time", - pathParams: "theme=light&panelId=41", - }, - { - title: "(p99) Task Completion Time", - pathParams: "theme=light&panelId=44", - }, - { - title: "(p100) Task Completion Time", - pathParams: "theme=light&panelId=45", + title: "Task Completion Time", + pathParams: "theme=light&panelId=38", }, ], }, diff --git a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py index 8bedac2db259..a900f57fabdd 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py @@ -460,13 +460,13 @@ ), Panel( id=38, - title="(p00) Task Completion Time", - description="Time spent running tasks to completion.", + title="Task Completion Time", + description="Time spent running tasks to completion w/ backpressure.", unit="seconds", targets=[ Target( - expr="histogram_quantile(0, sum by (dataset, operator, le) (rate(ray_data_task_completion_time_bucket{{{global_filters}}}[5m])))", - legend="(p00) Completion Time: {{dataset}}, {{operator}}", + expr="increase(ray_data_task_completion_time{{{global_filters}}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}}}[5m])", + legend="Task Completion Time: {{dataset}}, {{operator}}", ), ], fill=0, @@ -474,13 +474,13 @@ ), Panel( id=39, - title="(p05) Task Completion Time", - description="Time spent running tasks to completion.", + title="Task Output Backpressure Time", + description="Time spent in output backpressure.", unit="seconds", targets=[ Target( - expr="histogram_quantile(0.05, sum by (dataset, operator, le) (rate(ray_data_task_completion_time_bucket{{{global_filters}}}[5m])))", - legend="(p05) Completion Time: {{dataset}}, {{operator}}", + expr="increase(ray_data_task_output_backpressure_time{{{global_filters}}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}}}[5m])", + legend="Task Output Backpressure Time: {{dataset}}, {{operator}}", ), ], fill=0, @@ -488,69 +488,13 @@ ), Panel( id=40, - title="(p50) Task Completion Time", - description="Time spent running tasks to completion.", + title="Task Completion Time Without Backpressure", + description="Time spent running tasks to completion w/o backpressure.", unit="seconds", targets=[ Target( - expr="histogram_quantile(0.50, sum by (dataset, operator, le) (rate(ray_data_task_completion_time_bucket{{{global_filters}}}[5m])))", - legend="(p50) Completion Time: {{dataset}}, {{operator}}", - ), - ], - fill=0, - stack=False, - ), - Panel( - id=41, - title="(p75) Task Completion Time", - description="Time spent running tasks to completion.", - unit="seconds", - targets=[ - Target( - expr="histogram_quantile(0.75, sum by (dataset, operator, le) (rate(ray_data_task_completion_time_bucket{{{global_filters}}}[5m])))", - legend="(p75) Completion Time: {{dataset}}, {{operator}}", - ), - ], - fill=0, - stack=False, - ), - Panel( - id=42, - title="(p90) Task Completion Time", - description="Time spent running tasks to completion.", - unit="seconds", - targets=[ - Target( - expr="histogram_quantile(0.9, sum by (dataset, operator, le) (rate(ray_data_task_completion_time_bucket{{{global_filters}}}[5m])))", - legend="(p90) Completion Time: {{dataset}}, {{operator}}", - ), - ], - fill=0, - stack=False, - ), - Panel( - id=44, - title="p(99) Task Completion Time", - description="Time spent running tasks to completion.", - unit="seconds", - targets=[ - Target( - expr="histogram_quantile(0.99, sum by (dataset, operator, le) (rate(ray_data_task_completion_time_bucket{{{global_filters}}}[5m])))", - legend="(p99) Completion Time: {{dataset}}, {{operator}}", - ), - ], - fill=0, - stack=False, - ), - Panel( - id=45, - title="p(100) Task Completion Time", - description="Time spent running tasks to completion.", - unit="seconds", - targets=[ - Target( - expr="histogram_quantile(1, sum by (dataset, operator, le) (rate(ray_data_task_completion_time_bucket{{{global_filters}}}[5m])))", - legend="(p100) Completion Time: {{dataset}}, {{operator}}", + expr="increase(ray_data_task_completion_time_without_backpressure{{{global_filters}}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}}}[5m])", + legend="Task Completion Time w/o Backpressure: {{dataset}}, {{operator}}", ), ], fill=0, diff --git a/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py b/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py index b3b6bfb3445f..47436d5f329b 100644 --- a/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py +++ b/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py @@ -130,6 +130,7 @@ class RunningTaskInfo: bytes_outputs: int num_rows_produced: int start_time: float + cum_block_gen_time: float @dataclass @@ -396,13 +397,15 @@ class OpRuntimeMetrics(metaclass=OpRuntimesMetricsMeta): 2500.0, 5000.0, ] - - mean_task_completion_time: float = metric_field( + task_completion_time: float = metric_field( default=0, description="Time spent running tasks to completion.", metrics_group=MetricsGroup.TASKS, - metrics_type=MetricsType.Histogram, - metrics_args={"boundaries": histogram_buckets_s}, + ) + task_completion_time_without_backpressure: float = metric_field( + default=0, + description="Time spent running tasks to completion without backpressure.", + metrics_group=MetricsGroup.TASKS, ) # === Actor-related metrics === @@ -731,9 +734,8 @@ def on_toggle_task_output_backpressure(self, in_backpressure): self._task_output_backpressure_start_time = time.perf_counter() elif self._task_output_backpressure_start_time != -1: # backpressure stopping, stop timer - self.task_output_backpressure_time += ( - time.perf_counter() - self._task_output_backpressure_start_time - ) + delta = time.perf_counter() - self._task_output_backpressure_start_time + self.task_output_backpressure_time += delta self._task_output_backpressure_start_time = -1 def on_output_taken(self, output: RefBundle): @@ -756,6 +758,7 @@ def on_task_submitted(self, task_index: int, inputs: RefBundle): bytes_outputs=0, num_rows_produced=0, start_time=time.perf_counter(), + cum_block_gen_time=0, ) def on_task_output_generated(self, task_index: int, output: RefBundle): @@ -781,6 +784,7 @@ def on_task_output_generated(self, task_index: int, output: RefBundle): meta.exec_stats is not None and meta.exec_stats.wall_time_s is not None ) self.block_generation_time += meta.exec_stats.wall_time_s + task_info.cum_block_gen_time += meta.exec_stats.wall_time_s assert meta.num_rows is not None trace_allocation(block_ref, "operator_output") if meta.exec_stats.max_uss_bytes is not None: @@ -812,8 +816,10 @@ def on_task_finished(self, task_index: int, exception: Optional[Exception]): self.rows_outputs_of_finished_tasks += task_info.num_rows_produced task_time_delta = time.perf_counter() - task_info.start_time - self._op_task_duration_stats.add_duration(task_time_delta) - self.mean_task_completion_time = self._op_task_duration_stats.mean() + self.task_completion_time += task_time_delta + + assert task_info.cum_block_gen_time is not None + self.task_completion_time_without_backpressure += task_info.cum_block_gen_time inputs = self._running_tasks[task_index].inputs self.num_task_inputs_processed += len(inputs) total_input_size = inputs.size_bytes() diff --git a/python/ray/data/tests/test_stats.py b/python/ray/data/tests/test_stats.py index 25deb5b1c0a5..0c93e23bcbd0 100644 --- a/python/ray/data/tests/test_stats.py +++ b/python/ray/data/tests/test_stats.py @@ -115,7 +115,11 @@ def gen_expected_metrics( "'task_output_backpressure_time': " f"{'N' if task_output_backpressure else 'Z'}" ), - ("'mean_task_completion_time': " f"{'N' if task_backpressure else 'Z'}"), + ("'task_completion_time': " f"{'N' if task_backpressure else 'Z'}"), + ( + "'task_completion_time_without_backpressure': " + f"{'N' if task_backpressure else 'Z'}" + ), "'num_alive_actors': Z", "'num_restarting_actors': Z", "'num_pending_actors': Z", @@ -172,7 +176,11 @@ def gen_expected_metrics( "'task_output_backpressure_time': " f"{'N' if task_output_backpressure else 'Z'}" ), - ("'mean_task_completion_time': " f"{'N' if task_backpressure else 'Z'}"), + ("'task_completion_time': " f"{'N' if task_backpressure else 'Z'}"), + ( + "'task_completion_time_without_backpressure': " + f"{'N' if task_backpressure else 'Z'}" + ), "'num_alive_actors': Z", "'num_restarting_actors': Z", "'num_pending_actors': Z", @@ -677,7 +685,8 @@ def test_dataset__repr__(ray_start_regular_shared, restore_data_context): " block_generation_time: N,\n" " task_submission_backpressure_time: N,\n" " task_output_backpressure_time: Z,\n" - " mean_task_completion_time: N,\n" + " task_completion_time: N,\n" + " task_completion_time_without_backpressure: N,\n" " num_alive_actors: Z,\n" " num_restarting_actors: Z,\n" " num_pending_actors: Z,\n" @@ -806,7 +815,8 @@ def check_stats(): " block_generation_time: N,\n" " task_submission_backpressure_time: N,\n" " task_output_backpressure_time: Z,\n" - " mean_task_completion_time: N,\n" + " task_completion_time: N,\n" + " task_completion_time_without_backpressure: N,\n" " num_alive_actors: Z,\n" " num_restarting_actors: Z,\n" " num_pending_actors: Z,\n" @@ -890,7 +900,8 @@ def check_stats(): " block_generation_time: N,\n" " task_submission_backpressure_time: N,\n" " task_output_backpressure_time: Z,\n" - " mean_task_completion_time: N,\n" + " task_completion_time: N,\n" + " task_completion_time_without_backpressure: N,\n" " num_alive_actors: Z,\n" " num_restarting_actors: Z,\n" " num_pending_actors: Z,\n" From 0d11235a2d72bd551475cfb515dc39b1f1a40c06 Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Tue, 12 Aug 2025 14:07:54 -0700 Subject: [PATCH 063/634] [data] make shuffle pb total more accurate (#55543) ## Why are these changes needed? https://github.com/user-attachments/assets/402efd79-cda2-467e-b84a-774ccd69efa5 ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: iamjustinhsu --- .../data/_internal/execution/operators/hash_shuffle.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/ray/data/_internal/execution/operators/hash_shuffle.py b/python/ray/data/_internal/execution/operators/hash_shuffle.py index 50cb0136228e..2ef8453b2fe3 100644 --- a/python/ray/data/_internal/execution/operators/hash_shuffle.py +++ b/python/ray/data/_internal/execution/operators/hash_shuffle.py @@ -634,7 +634,13 @@ def _on_partitioning_done(cur_shuffle_task_idx: int): ) # Update Shuffle progress bar - self.shuffle_bar.update(total=self.shuffle_metrics.num_row_inputs_received) + _, _, num_rows = estimate_total_num_of_blocks( + cur_shuffle_task_idx + 1, + self.upstream_op_num_outputs(), + self.shuffle_metrics, + total_num_tasks=None, + ) + self.shuffle_bar.update(total=num_rows) def has_next(self) -> bool: self._try_finalize() From 5d7ff2d554987c91f531ada21ff59a41a90b6dd4 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 12 Aug 2025 14:28:21 -0700 Subject: [PATCH 064/634] [ci] base image: move cpu base up (#55536) cpu bases above cuda bases. simpler case should be listed first. Signed-off-by: Lonnie Liu --- .buildkite/_forge.rayci.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml index 662f7824178a..5af4f0a67b08 100644 --- a/.buildkite/_forge.rayci.yml +++ b/.buildkite/_forge.rayci.yml @@ -6,6 +6,20 @@ steps: - name: manylinux wanda: ci/docker/manylinux.wanda.yaml + - name: raycpubase + label: "wanda: ray.py{{matrix}}.cpu.base" + tags: + - python_dependencies + - docker + wanda: ci/docker/ray.cpu.base.wanda.yaml + matrix: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + env: + PYTHON_VERSION: "{{matrix}}" + - name: raycudabase label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base" tags: @@ -32,20 +46,6 @@ steps: PYTHON_VERSION: "{{matrix.python}}" CUDA_VERSION: "{{matrix.cuda}}" - - name: raycpubase - label: "wanda: ray.py{{matrix}}.cpu.base" - tags: - - python_dependencies - - docker - wanda: ci/docker/ray.cpu.base.wanda.yaml - matrix: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - env: - PYTHON_VERSION: "{{matrix}}" - - name: ray-llmbase label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base" tags: From 461e68537a51bc2597b3a18da39e0d70b9939d61 Mon Sep 17 00:00:00 2001 From: Alan Guo Date: Tue, 12 Aug 2025 15:08:58 -0700 Subject: [PATCH 065/634] Add Operator as filter to data grafana dashboard (#55493) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? Filtering by Operator will be useful to see only a single operator at a time Screenshot 2025-08-11 at 3 32 32 PM ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Alan Guo --- .../dashboards/data_dashboard_panels.py | 90 +++++++++---------- .../data_grafana_dashboard_base.json | 35 ++++++++ 2 files changed, 80 insertions(+), 45 deletions(-) diff --git a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py index a900f57fabdd..b763d30ca950 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py @@ -33,7 +33,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_spilled_bytes{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_spilled_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Spilled: {{dataset}}, {{operator}}", ) ], @@ -47,7 +47,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_freed_bytes{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_freed_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Freed: {{dataset}}, {{operator}}", ) ], @@ -61,7 +61,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_current_bytes{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_current_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Current Usage: {{dataset}}, {{operator}}", ) ], @@ -75,7 +75,7 @@ unit="cores", targets=[ Target( - expr="sum(ray_data_cpu_usage_cores{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_cpu_usage_cores{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="CPU Usage: {{dataset}}, {{operator}}", ) ], @@ -89,7 +89,7 @@ unit="cores", targets=[ Target( - expr="sum(ray_data_gpu_usage_cores{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_gpu_usage_cores{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="GPU Usage: {{dataset}}, {{operator}}", ) ], @@ -103,7 +103,7 @@ unit="Bps", targets=[ Target( - expr="sum(rate(ray_data_output_bytes{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_output_bytes{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Output / Second: {{dataset}}, {{operator}}", ) ], @@ -117,7 +117,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_max_bytes_to_read{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_max_bytes_to_read{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Max Bytes to Read: {{dataset}}, {{operator}}", ) ], @@ -126,12 +126,12 @@ ), Panel( id=11, - title="Rows Output / Second", + title="Throughput (Rows Output / Second)", description="Total rows output per second by dataset operators.", unit="rows/sec", targets=[ Target( - expr="sum(rate(ray_data_output_rows{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_output_rows{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Rows Output / Second: {{dataset}}, {{operator}}", ) ], @@ -146,7 +146,7 @@ unit="blocks/sec", targets=[ Target( - expr="sum(rate(ray_data_num_inputs_received{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_num_inputs_received{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Blocks Received / Second: {{dataset}}, {{operator}}", ) ], @@ -160,7 +160,7 @@ unit="Bps", targets=[ Target( - expr="sum(rate(ray_data_bytes_inputs_received{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_bytes_inputs_received{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Received / Second: {{dataset}}, {{operator}}", ) ], @@ -176,7 +176,7 @@ unit="blocks/sec", targets=[ Target( - expr="sum(rate(ray_data_num_task_inputs_processed{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_num_task_inputs_processed{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Blocks Processed / Second: {{dataset}}, {{operator}}", ) ], @@ -192,7 +192,7 @@ unit="Bps", targets=[ Target( - expr="sum(rate(ray_data_bytes_task_inputs_processed{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_bytes_task_inputs_processed{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Processed / Second: {{dataset}}, {{operator}}", ) ], @@ -206,7 +206,7 @@ unit="Bps", targets=[ Target( - expr="sum(rate(ray_data_bytes_inputs_of_submitted_tasks{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_bytes_inputs_of_submitted_tasks{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Submitted / Second: {{dataset}}, {{operator}}", ) ], @@ -220,7 +220,7 @@ unit="blocks/sec", targets=[ Target( - expr="sum(rate(ray_data_num_task_outputs_generated{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_num_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Blocks Generated / Second: {{dataset}}, {{operator}}", ) ], @@ -234,7 +234,7 @@ unit="Bps", targets=[ Target( - expr="sum(rate(ray_data_bytes_task_outputs_generated{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_bytes_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Generated / Second: {{dataset}}, {{operator}}", ) ], @@ -248,7 +248,7 @@ unit="bytes", targets=[ Target( - expr="increase(ray_data_bytes_task_outputs_generated{{{global_filters}}}[5m]) / increase(ray_data_num_task_outputs_generated{{{global_filters}}}[5m])", + expr='increase(ray_data_bytes_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[5m])', legend="Average Bytes Generated / Output Block: {{dataset}}, {{operator}}", ) ], @@ -262,7 +262,7 @@ unit="blocks", targets=[ Target( - expr="increase(ray_data_num_task_outputs_generated{{{global_filters}}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}}}[5m])", + expr='increase(ray_data_num_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', legend="Average Number of Output Blocks / Task: {{dataset}}, {{operator}}", ) ], @@ -276,7 +276,7 @@ unit="rows/sec", targets=[ Target( - expr="sum(rate(ray_data_rows_task_outputs_generated{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_rows_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Rows Generated / Second: {{dataset}}, {{operator}}", ) ], @@ -290,7 +290,7 @@ unit="blocks/sec", targets=[ Target( - expr="sum(rate(ray_data_num_outputs_taken{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_num_outputs_taken{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Blocks Taken / Second: {{dataset}}, {{operator}}", ) ], @@ -306,7 +306,7 @@ unit="Bps", targets=[ Target( - expr="sum(rate(ray_data_bytes_outputs_taken{{{global_filters}}}[1m])) by (dataset, operator)", + expr='sum(rate(ray_data_bytes_outputs_taken{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Taken / Second: {{dataset}}, {{operator}}", ) ], @@ -353,7 +353,7 @@ unit="tasks", targets=[ Target( - expr="sum(ray_data_num_tasks_submitted{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_num_tasks_submitted{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Submitted Tasks: {{dataset}}, {{operator}}", ) ], @@ -367,7 +367,7 @@ unit="tasks", targets=[ Target( - expr="sum(ray_data_num_tasks_running{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_num_tasks_running{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Running Tasks: {{dataset}}, {{operator}}", ) ], @@ -381,7 +381,7 @@ unit="tasks", targets=[ Target( - expr="sum(ray_data_num_tasks_have_outputs{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_num_tasks_have_outputs{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Tasks with output blocks: {{dataset}}, {{operator}}", ) ], @@ -395,7 +395,7 @@ unit="tasks", targets=[ Target( - expr="sum(ray_data_num_tasks_finished{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Finished Tasks: {{dataset}}, {{operator}}", ) ], @@ -423,7 +423,7 @@ unit="tasks", targets=[ Target( - expr="sum(ray_data_num_tasks_failed{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_num_tasks_failed{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Failed Tasks: {{dataset}}, {{operator}}", ) ], @@ -437,7 +437,7 @@ unit="seconds", targets=[ Target( - expr="sum(ray_data_block_generation_time{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_block_generation_time{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Block Generation Time: {{dataset}}, {{operator}}", ) ], @@ -451,7 +451,7 @@ unit="seconds", targets=[ Target( - expr="sum(ray_data_task_submission_backpressure_time{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_task_submission_backpressure_time{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Backpressure Time: {{dataset}}, {{operator}}", ) ], @@ -465,7 +465,7 @@ unit="seconds", targets=[ Target( - expr="increase(ray_data_task_completion_time{{{global_filters}}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}}}[5m])", + expr='increase(ray_data_task_completion_time{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', legend="Task Completion Time: {{dataset}}, {{operator}}", ), ], @@ -479,7 +479,7 @@ unit="seconds", targets=[ Target( - expr="increase(ray_data_task_output_backpressure_time{{{global_filters}}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}}}[5m])", + expr='increase(ray_data_task_output_backpressure_time{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', legend="Task Output Backpressure Time: {{dataset}}, {{operator}}", ), ], @@ -493,7 +493,7 @@ unit="seconds", targets=[ Target( - expr="increase(ray_data_task_completion_time_without_backpressure{{{global_filters}}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}}}[5m])", + expr='increase(ray_data_task_completion_time_without_backpressure{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', legend="Task Completion Time w/o Backpressure: {{dataset}}, {{operator}}", ), ], @@ -508,7 +508,7 @@ unit="blocks", targets=[ Target( - expr="sum(ray_data_obj_store_mem_internal_inqueue_blocks{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_obj_store_mem_internal_inqueue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Number of Blocks: {{dataset}}, {{operator}}", ) ], @@ -522,7 +522,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_obj_store_mem_internal_inqueue{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_obj_store_mem_internal_inqueue{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Size: {{dataset}}, {{operator}}", ) ], @@ -536,7 +536,7 @@ unit="blocks", targets=[ Target( - expr="sum(ray_data_obj_store_mem_internal_outqueue_blocks{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_obj_store_mem_internal_outqueue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Number of Blocks: {{dataset}}, {{operator}}", ) ], @@ -552,7 +552,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_obj_store_mem_internal_outqueue{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_obj_store_mem_internal_outqueue{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Size: {{dataset}}, {{operator}}", ) ], @@ -566,7 +566,7 @@ unit="blocks", targets=[ Target( - expr="sum(ray_data_num_output_queue_blocks{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_num_output_queue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Number of Blocks: {{dataset}}, {{operator}}", ) ], @@ -580,7 +580,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_num_output_queue_bytes{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_num_output_queue_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Number of Bytes: {{dataset}}, {{operator}}", ) ], @@ -594,7 +594,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_obj_store_mem_pending_task_inputs{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_obj_store_mem_pending_task_inputs{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Size: {{dataset}}, {{operator}}", ) ], @@ -608,7 +608,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_obj_store_mem_freed{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_obj_store_mem_freed{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Size: {{dataset}}, {{operator}}", ) ], @@ -622,7 +622,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_obj_store_mem_spilled{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_obj_store_mem_spilled{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Size: {{dataset}}, {{operator}}", ) ], @@ -638,7 +638,7 @@ targets=[ Target( expr="sum(ray_data_iter_initialize_seconds{{{global_filters}}}) by (dataset)", - legend="Seconds: {{dataset}}, {{operator}}", + legend="Seconds: {{dataset}}", ) ], fill=0, @@ -694,7 +694,7 @@ unit="cpu", targets=[ Target( - expr="sum(ray_data_cpu_budget{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_cpu_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Budget (CPU): {{dataset}}, {{operator}}", ) ], @@ -708,7 +708,7 @@ unit="gpu", targets=[ Target( - expr="sum(ray_data_gpu_budget{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_gpu_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Budget (GPU): {{dataset}}, {{operator}}", ) ], @@ -722,7 +722,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_memory_budget{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_memory_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Budget (Memory): {{dataset}}, {{operator}}", ) ], @@ -736,7 +736,7 @@ unit="bytes", targets=[ Target( - expr="sum(ray_data_object_store_memory_budget{{{global_filters}}}) by (dataset, operator)", + expr='sum(ray_data_object_store_memory_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Budget (Object Store Memory): {{dataset}}, {{operator}}", ) ], diff --git a/python/ray/dashboard/modules/metrics/dashboards/data_grafana_dashboard_base.json b/python/ray/dashboard/modules/metrics/dashboards/data_grafana_dashboard_base.json index dea96d4513b2..23606bb9120f 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/data_grafana_dashboard_base.json +++ b/python/ray/dashboard/modules/metrics/dashboards/data_grafana_dashboard_base.json @@ -104,6 +104,41 @@ "type": "query", "useTags": false }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "${datasource}", + "definition": "query_result(count by (operator)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Operator", + "options": [], + "query": { + "query": "query_result(count by (operator)(last_over_time(ray_data_output_bytes{{SessionName=~\"$SessionName\",{global_filters}}}[$__range])))", + "refId": "Prometheus-Dataset-Variable-Query" + }, + "refresh": 2, + "regex": "{operator=\"(?.*)\".*", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "current": { "selected": false From 6afcdfa3cd235c9c5ce1d61ce2fdf2d767c08e05 Mon Sep 17 00:00:00 2001 From: Howie Tien Date: Wed, 13 Aug 2025 06:30:12 +0800 Subject: [PATCH 066/634] [Data] decrease parquet metadata storage usage (#54821) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? When working with search and recommendation systems, datasets often contain numerous columns, resulting in large metadata overhead in Parquet files (sometimes a few MBs or more for each file). Currently, the driver fetches first all metadata, then simplifies and merges them to reduce memory usage. However, this process can cause memory peaks proportional to the number of fragments multiplied by their metadata size, potentially leading to OOM issues. This PR addresses the problem by **simplifying** and **merging** the dataset metadata within each `_fetch_metadata` task before sending it back to the driver. This change helps lower memory consumption and reduces the risk of OOM errors. Test script: ``` from pyarrow._fs import FileSystem from ray.data.datasource.file_meta_provider import _get_file_infos import os import ray import psutil hdfs_path = "hdfs://path/to/dataset" def list_files(remote_path): filesystem, remote_path = FileSystem.from_uri(remote_path) from ray.data.datasource.file_meta_provider import _get_file_infos files = _get_file_infos(remote_path, filesystem) return filesystem, files filesystem, files = list_files(hdfs_path) files = [f"{fs[0]}" for fs in files if fs[0].endswith(".parquet")] process = psutil.Process() print(f"total file_num: {len(files)}") start_mem = process.memory_info().rss / 1024**2 dataset = ray.data.read_parquet(paths=files[:500], filesystem=filesystem) end_mem = process.memory_info().rss / 1024**2 print(f"datasize: {dataset.count()}, col number: {len(dataset.columns())}") print(f"mem diff {end_mem - start_mem:.3f}MiB [start: {start_mem:.3f}MiB, end: {end_mem:.3f}MiB]") ``` Output before this PR: ``` total file_num: 2358 Metadata Fetch Progress 0: 100%|███████████████████████|500/500 [02:13<00:00, 3.75 task/s] Parquet Files Sample 0: 100%|███████████████████████| 5.00/5.00 [00:13<00:00, 2.62s/ file] datasize: 22630452, col number: 1200 mem diff 13727.605MiB [start: 570.617MiB, end: 14298.223MiB] ``` Output after this PR: ``` total file_num: 2358 Metadata Fetch Progress 0: 100%|███████████████████████| 500/500 [01:55<00:00, 4.35 task/s] Parquet Files Sample 0: 100%|███████████████████████| 5.00/5.00 [00:03<00:00, 1.56 file/s] datasize: 22630452, col number: 1200 mem diff 69.113MiB [start: 575.820MiB, end: 644.934MiB] ``` We can see the memory usage reduce from 13GBs to 69MBs. Note: This approach is most effective for large-scale datasets. If `len(fragments) < PARALLELIZE_META_FETCH_THRESHOLD`, there will be no performance improvements. ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: haotian Signed-off-by: Howie Tien Signed-off-by: Balaji Veeramani Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Balaji Veeramani --- .../data/datasource/parquet_meta_provider.py | 78 ++++++++++++------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/python/ray/data/datasource/parquet_meta_provider.py b/python/ray/data/datasource/parquet_meta_provider.py index c8484574da18..73a3c41ef6e2 100644 --- a/python/ray/data/datasource/parquet_meta_provider.py +++ b/python/ray/data/datasource/parquet_meta_provider.py @@ -41,10 +41,10 @@ def __init__(self, fragment_metadata: "pyarrow.parquet.FileMetaData"): self.num_row_groups = fragment_metadata.num_row_groups self.num_rows = fragment_metadata.num_rows self.serialized_size = fragment_metadata.serialized_size - # This is a pickled schema object, to be set later with - # `self.set_schema_pickled()`. To get the underlying schema, use - # `cloudpickle.loads(self.schema_pickled)`. - self.schema_pickled = None + + # Serialize the schema directly in the constructor + schema_ser = cloudpickle.dumps(fragment_metadata.schema.to_arrow_schema()) + self.schema_pickled = schema_ser # Calculate the total byte size of the file fragment using the original # object, as it is not possible to access row groups from this class. @@ -150,10 +150,16 @@ def fetch_func(fragments): **ray_remote_args, ) ) + + return _dedupe_schemas(raw_metadata) + else: + # We don't deduplicate schemas in this branch because they're already + # deduplicated in `_fetch_metadata`. See + # https://github.com/ray-project/ray/pull/54821/files#r2265140929 for + # related discussion. raw_metadata = _fetch_metadata(fragments) - - return _dedupe_metadata(raw_metadata) + return raw_metadata def _fetch_metadata_serialization_wrapper( @@ -161,7 +167,7 @@ def _fetch_metadata_serialization_wrapper( retry_match: Optional[List[str]], retry_max_attempts: int, retry_max_interval: int, -) -> List["pyarrow.parquet.FileMetaData"]: +) -> List["_ParquetFileFragmentMetaData"]: from ray.data._internal.datasource.parquet_datasource import ( _deserialize_fragments_with_retry, ) @@ -209,39 +215,53 @@ def _fetch_metadata_serialization_wrapper( def _fetch_metadata( fragments: List["pyarrow.dataset.ParquetFileFragment"], -) -> List["pyarrow.parquet.FileMetaData"]: - fragment_metadata = [] +) -> List[_ParquetFileFragmentMetaData]: + fragment_metadatas = [] for f in fragments: try: - fragment_metadata.append(f.metadata) + # Convert directly to _ParquetFileFragmentMetaData + fragment_metadatas.append(_ParquetFileFragmentMetaData(f.metadata)) except AttributeError: break - return fragment_metadata + # Deduplicate schemas to reduce memory usage + return _dedupe_schemas(fragment_metadatas) -def _dedupe_metadata( - raw_metadatas: List["pyarrow.parquet.FileMetaData"], +def _dedupe_schemas( + metadatas: List[_ParquetFileFragmentMetaData], ) -> List[_ParquetFileFragmentMetaData]: - """For datasets with a large number of columns, the FileMetaData - (in particular the schema) can be very large. We can reduce the - memory usage by only keeping unique schema objects across all - file fragments. This method deduplicates the schemas and returns - a list of `_ParquetFileFragmentMetaData` objects.""" - schema_to_id = {} # schema_id -> serialized_schema - id_to_schema = {} # serialized_schema -> schema_id - stripped_metadatas = [] - for fragment_metadata in raw_metadatas: - stripped_md = _ParquetFileFragmentMetaData(fragment_metadata) + """Deduplicates schema objects across existing _ParquetFileFragmentMetaData objects. + + For datasets with a large number of columns, the pickled schema can be very large. + This function reduces memory usage by ensuring that identical schemas across multiple + fragment metadata objects reference the same underlying pickled schema object, + rather than each fragment maintaining its own copy. + + Args: + metadatas: List of _ParquetFileFragmentMetaData objects that already have + pickled schemas set. + + Returns: + The same list of _ParquetFileFragmentMetaData objects, but with duplicate + schemas deduplicated to reference the same object in memory. + """ + schema_to_id = {} # schema_ser -> schema_id + id_to_schema = {} # schema_id -> schema_ser + + for metadata in metadatas: + # Get the current schema serialization + schema_ser = metadata.schema_pickled - schema_ser = cloudpickle.dumps(fragment_metadata.schema.to_arrow_schema()) if schema_ser not in schema_to_id: + # This is a new unique schema schema_id = len(schema_to_id) schema_to_id[schema_ser] = schema_id id_to_schema[schema_id] = schema_ser - stripped_md.set_schema_pickled(schema_ser) + # No need to set schema_pickled - it already has the correct value else: - schema_id = schema_to_id.get(schema_ser) + # This schema already exists, reuse the existing one + schema_id = schema_to_id[schema_ser] existing_schema_ser = id_to_schema[schema_id] - stripped_md.set_schema_pickled(existing_schema_ser) - stripped_metadatas.append(stripped_md) - return stripped_metadatas + metadata.set_schema_pickled(existing_schema_ser) + + return metadatas From f70c283d500a9700e136b426c50587a4f7c76258 Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Tue, 12 Aug 2025 16:18:00 -0700 Subject: [PATCH 067/634] [serve] Add concurrencies option to serve microbenchmarks (#55522) ## Why are these changes needed? Perf can differ at various concurrencies. This makes it easy to configure / run at various concurrencies + max_ongoing_request combinations ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: akyang-anyscale --- .../serve_tests/workloads/microbenchmarks.py | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/release/serve_tests/workloads/microbenchmarks.py b/release/serve_tests/workloads/microbenchmarks.py index 6ccde24344b9..4cf10ab9e615 100644 --- a/release/serve_tests/workloads/microbenchmarks.py +++ b/release/serve_tests/workloads/microbenchmarks.py @@ -133,6 +133,7 @@ async def _main( run_throughput: bool, run_streaming: bool, throughput_max_ongoing_requests: List[int], + concurrencies: List[int], ): perf_metrics = [] payload_1mb = generate_payload(1000000) @@ -157,14 +158,16 @@ async def _main( if run_throughput: # Microbenchmark: HTTP throughput - for max_ongoing_requests in throughput_max_ongoing_requests: + for max_ongoing_requests, concurrency in zip( + throughput_max_ongoing_requests, concurrencies + ): serve.run( Noop.options(max_ongoing_requests=max_ongoing_requests).bind() ) url = get_application_url(use_localhost=True) mean, std, _ = await run_throughput_benchmark( - fn=partial(do_single_http_batch, batch_size=BATCH_SIZE, url=url), - multiplier=BATCH_SIZE, + fn=partial(do_single_http_batch, batch_size=concurrency, url=url), + multiplier=concurrency, num_trials=NUM_TRIALS, trial_runtime=TRIAL_RUNTIME_S, ) @@ -279,7 +282,9 @@ async def _main( if run_throughput: # Microbenchmark: GRPC throughput - for max_ongoing_requests in throughput_max_ongoing_requests: + for max_ongoing_requests, concurrency in zip( + throughput_max_ongoing_requests, concurrencies + ): serve.start(grpc_options=serve_grpc_options) serve.run( GrpcDeployment.options( @@ -291,9 +296,9 @@ async def _main( ) mean, std, _ = await run_throughput_benchmark( fn=partial( - do_single_grpc_batch, batch_size=BATCH_SIZE, target=target + do_single_grpc_batch, batch_size=concurrency, target=target ), - multiplier=BATCH_SIZE, + multiplier=concurrency, num_trials=NUM_TRIALS, trial_runtime=TRIAL_RUNTIME_S, ) @@ -320,14 +325,16 @@ async def _main( if run_throughput: # Microbenchmark: Handle throughput - for max_ongoing_requests in throughput_max_ongoing_requests: + for max_ongoing_requests, concurrency in zip( + throughput_max_ongoing_requests, concurrencies + ): h: DeploymentHandle = serve.run( Benchmarker.options(max_ongoing_requests=max_ongoing_requests).bind( Noop.options(max_ongoing_requests=max_ongoing_requests).bind() ) ) mean, std, _ = await h.run_throughput_benchmark.remote( - batch_size=BATCH_SIZE, + batch_size=concurrency, num_trials=NUM_TRIALS, trial_runtime=TRIAL_RUNTIME_S, ) @@ -383,8 +390,16 @@ async def _main( "-t", multiple=True, type=int, - default=[5, 100], - help="Max ongoing requests for throughput benchmarks. Default: [5, 100]", + default=[5, 100, 800], + help="Max ongoing requests for throughput benchmarks. Must be in the same order as --concurrencies. Default: [5, 100, 800]", +) +@click.option( + "--concurrencies", + "-c", + multiple=True, + type=int, + default=[100, 100, 800], + help="User concurrency for throughput benchmarks. Must be in the same order as --throughput-max-ongoing-requests. Default: [100, 100, 800]", ) def main( output_path: Optional[str], @@ -396,7 +411,12 @@ def main( run_throughput: bool, run_streaming: bool, throughput_max_ongoing_requests: List[int], + concurrencies: List[int], ): + assert len(throughput_max_ongoing_requests) == len( + concurrencies + ), "Must have the same number of --throughput-max-ongoing-requests and --concurrencies" + # If none of the flags are set, default to run all if not ( run_http @@ -426,6 +446,7 @@ def main( run_throughput, run_streaming, throughput_max_ongoing_requests, + concurrencies, ) ) From b4b2a89e93b4b849c1bb5a0e43f968a932bbbe2a Mon Sep 17 00:00:00 2001 From: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Date: Tue, 12 Aug 2025 16:24:16 -0700 Subject: [PATCH 068/634] [serve.llm] Serialize ChatCompletionRequest tool_calls ValidatorIterator object (#55538) Signed-off-by: Seiji Eicher --- .../serve/deployments/routers/router.py | 30 ++++++++++++ .../cpu/deployments/routers/test_router.py | 48 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/python/ray/llm/_internal/serve/deployments/routers/router.py b/python/ray/llm/_internal/serve/deployments/routers/router.py index ac79d7e22a8c..0d22410c6ef8 100644 --- a/python/ray/llm/_internal/serve/deployments/routers/router.py +++ b/python/ray/llm/_internal/serve/deployments/routers/router.py @@ -77,6 +77,31 @@ logger = get_logger(__name__) T = TypeVar("T") + + +def _sanitize_chat_completion_request( + request: ChatCompletionRequest, +) -> ChatCompletionRequest: + """Sanitize ChatCompletionRequest to fix Pydantic ValidatorIterator serialization issue. + + This addresses a known Pydantic bug where tool_calls fields become ValidatorIterator + objects that cannot be pickled for Ray remote calls. + + References: + - vLLM PR that introduces the workaround: https://github.com/vllm-project/vllm/pull/9951 + - Pydantic Issue: https://github.com/pydantic/pydantic/issues/9467 + - Related Issue: https://github.com/pydantic/pydantic/issues/9541 + - Official Workaround: https://github.com/pydantic/pydantic/issues/9467#issuecomment-2442097291 + + TODO(seiji): Remove when we update to Pydantic v2.11+ with the fix. + """ + from vllm.transformers_utils.tokenizers.mistral import maybe_serialize_tool_calls + + maybe_serialize_tool_calls(request) + + return request + + StreamResponseType = Union[ ChatCompletionStreamResponse, CompletionStreamResponse, @@ -302,6 +327,11 @@ async def _get_response( model_handle = self._get_configured_serve_handle(model) + # TODO(seiji): Remove when we update to Pydantic v2.11+ with the fix + # for tool calling ValidatorIterator serialization issue. + if isinstance(body, ChatCompletionRequest): + body = _sanitize_chat_completion_request(body) + async for response in getattr(model_handle, call_method).remote(body): yield response diff --git a/python/ray/llm/tests/serve/cpu/deployments/routers/test_router.py b/python/ray/llm/tests/serve/cpu/deployments/routers/test_router.py index 4204231fd069..032e1e655c00 100644 --- a/python/ray/llm/tests/serve/cpu/deployments/routers/test_router.py +++ b/python/ray/llm/tests/serve/cpu/deployments/routers/test_router.py @@ -115,6 +115,54 @@ async def test_completion(self, stream_batching_interval_ms, client, stream): expected_text = " ".join([f"test_{i}" for i in range(n_tokens)]) assert text.strip() == expected_text + @pytest.mark.asyncio + @pytest.mark.parametrize("stream", [True, False]) + async def test_tool_call(self, client, stream): + response = client.chat.completions.create( + model="llm_model_id", + messages=[ + { + "role": "user", + "content": "Can you tell me what the temperate will be in Dallas, in fahrenheit?", + }, + { + "content": None, + "role": "assistant", + "tool_calls": [ + { + "id": "RBS92VTjJ", + "function": { + "arguments": '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}', + "name": "get_current_weather", + }, + "type": "function", + } + ], + }, + { + "role": "tool", + "content": "The weather in Dallas, TX is 85 degrees fahrenheit. It is partly cloudly, with highs in the 90's.", + "tool_call_id": "n3OMUpydP", + }, + ], + stream=stream, + max_tokens=200, + ) + + if stream: + text = "" + role = None + for chunk in response: + if chunk.choices[0].delta.role is not None and role is None: + role = chunk.choices[0].delta.role + if chunk.choices[0].delta.content: + text += chunk.choices[0].delta.content + else: + text = response.choices[0].message.content + role = response.choices[0].message.role + + assert text + def test_router_with_num_router_replicas_config(self): """Test the router with num_router_replicas config.""" # Test with no num_router_replicas config. From 4e5f03e7a1d06b9da8f3a9329400d426055f8ea4 Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Tue, 12 Aug 2025 16:55:07 -0700 Subject: [PATCH 069/634] [data] make hash shuffling resource usage calculation faster (#55503) ## Why are these changes needed? Currently, we use sum(shuffle_task.cpu_usage for all shuffling tasks). This can be very slow for large number of shuffling tasks. We can mitigate this by calculating the usage on every task submission ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- .../execution/operators/hash_shuffle.py | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/python/ray/data/_internal/execution/operators/hash_shuffle.py b/python/ray/data/_internal/execution/operators/hash_shuffle.py index 2ef8453b2fe3..400bd9e04573 100644 --- a/python/ray/data/_internal/execution/operators/hash_shuffle.py +++ b/python/ray/data/_internal/execution/operators/hash_shuffle.py @@ -469,6 +469,11 @@ def __init__( data_context=data_context, ) + # We track the running usage total because iterating + # and summing over all shuffling tasks can be expensive + # if the # of shuffling tasks is large + self._shuffling_resource_usage = ExecutionResources.zero() + self._input_block_transformer = input_block_transformer self._next_shuffle_tasks_idx: int = 0 @@ -585,6 +590,11 @@ def _do_add_input_inner(self, input_bundle: RefBundle, input_index: int): def _on_partitioning_done(cur_shuffle_task_idx: int): task = self._shuffling_tasks[input_index].pop(cur_shuffle_task_idx) + self._shuffling_resource_usage = ( + self._shuffling_resource_usage.subtract( + task.get_requested_resource_bundle() + ) + ) # Fetch input block and resulting partition shards block metadata and # handle obtained metadata # @@ -614,16 +624,22 @@ def _on_partitioning_done(cur_shuffle_task_idx: int): self.shuffle_bar.update(i=input_block_metadata.num_rows) # TODO update metrics - self._shuffling_tasks[input_index][cur_shuffle_task_idx] = MetadataOpTask( + task = self._shuffling_tasks[input_index][ + cur_shuffle_task_idx + ] = MetadataOpTask( task_index=cur_shuffle_task_idx, object_ref=input_block_partition_shards_metadata_tuple_ref, task_done_callback=functools.partial( _on_partitioning_done, cur_shuffle_task_idx ), - task_resource_bundle=( - ExecutionResources.from_resource_dict(shuffle_task_resource_bundle) + task_resource_bundle=ExecutionResources.from_resource_dict( + shuffle_task_resource_bundle ), ) + if task.get_requested_resource_bundle() is not None: + self._shuffling_resource_usage = self._shuffling_resource_usage.add( + task.get_requested_resource_bundle() + ) # Update Shuffle Metrics on task submission self.shuffle_metrics.on_task_submitted( @@ -856,19 +872,13 @@ def current_processor_usage(self) -> ExecutionResources: # `base_resource_usage` method) # - Active shuffling tasks # - Active finalizing tasks (actor tasks) - base_usage = self.base_resource_usage() - - shuffling_tasks = self._get_active_shuffling_tasks() - shuffling_tasks_cpus_used = sum( - [t.get_requested_resource_bundle().cpu for t in shuffling_tasks] - ) + base_usage = self.base_resource_usage + running_usage = self._shuffling_resource_usage # TODO add memory to resources being tracked - return ExecutionResources( - cpu=base_usage.cpu + shuffling_tasks_cpus_used, - gpu=0, - ) + return base_usage.add(running_usage) + @property def base_resource_usage(self) -> ExecutionResources: # TODO add memory to resources being tracked return ExecutionResources( From 07c3ed0a0290c090128757bffce29e79e448614b Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 12 Aug 2025 21:17:48 -0400 Subject: [PATCH 070/634] [ci] Shorten bazel directory paths to avoid failures on Windows (#55525) - Shortening bazel workspace name: `s/com_github_ray_project_ray/io_ray` - Set `--output_base=c:/bzl` instead of `--output_user_root=c:/raytmp`, which saves ~20 characters in the path. --------- Signed-off-by: Edward Oakes --- WORKSPACE | 2 +- bazel/gen_extract.py | 2 +- bazel/jemalloc.BUILD | 2 +- bazel/ray_deps_build_all.bzl | 2 +- bazel/ray_deps_setup.bzl | 40 +++++++++---------- ci/pipeline/test_conditional_testing.py | 2 +- ci/ray_ci/windows/build_ray.sh | 3 +- ci/raydepsets/testing_utils.py | 2 +- ci/raydepsets/tests/test_cli.py | 2 +- ci/raydepsets/tests/utils.py | 2 +- doc/external/test_hashes.py | 2 +- java/BUILD.bazel | 8 ++-- python/ray/serve/tests/unit/test_config.py | 4 +- python/ray/tests/test_traceback.py | 4 +- release/ray_release/bazel.py | 2 +- .../task_execution/test/BUILD.bazel | 3 +- .../task_submission/test/BUILD.bazel | 10 ++--- thirdparty/patches/grpc-zlib-fdopen.patch | 4 +- .../patches/prometheus-zlib-fdopen.patch | 4 +- 19 files changed, 51 insertions(+), 49 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index be7a05ef4371..73158be8e560 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,4 +1,4 @@ -workspace(name = "com_github_ray_project_ray") +workspace(name = "io_ray") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") diff --git a/bazel/gen_extract.py b/bazel/gen_extract.py index ce89c7e49a3b..1930440a4235 100644 --- a/bazel/gen_extract.py +++ b/bazel/gen_extract.py @@ -12,7 +12,7 @@ def gen_extract( sub_dir: str = "python", ): r = runfiles.Create() - _repo_name = "com_github_ray_project_ray" + _repo_name = "io_ray" root_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY") if not root_dir: diff --git a/bazel/jemalloc.BUILD b/bazel/jemalloc.BUILD index e0be47fd4446..545a557293a2 100644 --- a/bazel/jemalloc.BUILD +++ b/bazel/jemalloc.BUILD @@ -1,5 +1,5 @@ load("@rules_foreign_cc//foreign_cc:configure.bzl", "configure_make") -load("@com_github_ray_project_ray//bazel:ray.bzl", "filter_files_with_suffix") +load("@io_ray//bazel:ray.bzl", "filter_files_with_suffix") filegroup( name = "all", diff --git a/bazel/ray_deps_build_all.bzl b/bazel/ray_deps_build_all.bzl index a8597dd1840f..8d59beab3263 100644 --- a/bazel/ray_deps_build_all.bzl +++ b/bazel/ray_deps_build_all.bzl @@ -1,5 +1,5 @@ load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") -load("@com_github_ray_project_ray//java:dependencies.bzl", "gen_java_deps") +load("@io_ray//java:dependencies.bzl", "gen_java_deps") load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps") load("@com_github_jupp0r_prometheus_cpp//bazel:repositories.bzl", "prometheus_cpp_repositories") load("@com_github_grpc_grpc//third_party/py:python_configure.bzl", "python_configure") diff --git a/bazel/ray_deps_setup.bzl b/bazel/ray_deps_setup.bzl index 210f71c8b1da..5ece022cdb29 100644 --- a/bazel/ray_deps_setup.bzl +++ b/bazel/ray_deps_setup.bzl @@ -53,7 +53,7 @@ def auto_http_archive( # auto appending ray project namespace prefix for 3rd party library reusing. if build_file == True: - build_file = "@com_github_ray_project_ray//%s:%s" % ("bazel", name + ".BUILD") + build_file = "@io_ray//%s:%s" % ("bazel", name + ".BUILD") if urls == True: prefer_url_over_mirrors = is_github @@ -106,41 +106,41 @@ def ray_deps_setup(): # all of http/git_repository should add prefix for patches defined in ray directory. auto_http_archive( name = "com_github_antirez_redis", - build_file = "@com_github_ray_project_ray//bazel:redis.BUILD", + build_file = "@io_ray//bazel:redis.BUILD", patch_args = ["-p1"], url = "https://github.com/redis/redis/archive/refs/tags/7.2.3.tar.gz", sha256 = "afd656dbc18a886f9a1cc08a550bf5eb89de0d431e713eba3ae243391fb008a6", patches = [ - "@com_github_ray_project_ray//thirdparty/patches:redis-quiet.patch", + "@io_ray//thirdparty/patches:redis-quiet.patch", ], workspace_file_content = 'workspace(name = "com_github_antirez_redis")', ) auto_http_archive( name = "com_github_redis_hiredis", - build_file = "@com_github_ray_project_ray//bazel:hiredis.BUILD", + build_file = "@io_ray//bazel:hiredis.BUILD", url = "https://github.com/redis/hiredis/archive/60e5075d4ac77424809f855ba3e398df7aacefe8.tar.gz", sha256 = "b6d6f799b7714d85316f9ebfb76a35a78744f42ea3b6774289d882d13a2f0383", patches = [ - "@com_github_ray_project_ray//thirdparty/patches:hiredis-windows-msvc.patch", + "@io_ray//thirdparty/patches:hiredis-windows-msvc.patch", ], ) auto_http_archive( name = "com_github_spdlog", - build_file = "@com_github_ray_project_ray//bazel:spdlog.BUILD", + build_file = "@io_ray//bazel:spdlog.BUILD", urls = ["https://github.com/gabime/spdlog/archive/v1.12.0.zip"], sha256 = "6174bf8885287422a6c6a0312eb8a30e8d22bcfcee7c48a6d02d1835d7769232", # spdlog rotation filename format conflict with ray, update the format. patches = [ - "@com_github_ray_project_ray//thirdparty/patches:spdlog-rotation-file-format.patch", + "@io_ray//thirdparty/patches:spdlog-rotation-file-format.patch", ], patch_args = ["-p1"], ) auto_http_archive( name = "com_github_tporadowski_redis_bin", - build_file = "@com_github_ray_project_ray//bazel:redis.BUILD", + build_file = "@io_ray//bazel:redis.BUILD", strip_prefix = None, url = "https://github.com/tporadowski/redis/releases/download/v5.0.9/Redis-x64-5.0.9.zip", sha256 = "b09565b22b50c505a5faa86a7e40b6683afb22f3c17c5e6a5e35fc9b7c03f4c2", @@ -224,8 +224,8 @@ def ray_deps_setup(): url = "https://github.com/census-instrumentation/opencensus-cpp/archive/5e5f2632c84e2230fb7ccb8e336f603d2ec6aa1b.zip", sha256 = "1b88d6663f05c6a56c1604eb2afad22831d5f28a76f6fab8f37187f1e4ace425", patches = [ - "@com_github_ray_project_ray//thirdparty/patches:opencensus-cpp-harvest-interval.patch", - "@com_github_ray_project_ray//thirdparty/patches:opencensus-cpp-shutdown-api.patch", + "@io_ray//thirdparty/patches:opencensus-cpp-harvest-interval.patch", + "@io_ray//thirdparty/patches:opencensus-cpp-shutdown-api.patch", ], patch_args = ["-p1"], ) @@ -263,11 +263,11 @@ def ray_deps_setup(): url = "https://github.com/jupp0r/prometheus-cpp/archive/60eaa4ea47b16751a8e8740b05fe70914c68a480.tar.gz", sha256 = "ec825b802487ac18b0d98e2e8b7961487b12562f8f82e424521d0a891d9e1373", patches = [ - "@com_github_ray_project_ray//thirdparty/patches:prometheus-windows-headers.patch", + "@io_ray//thirdparty/patches:prometheus-windows-headers.patch", # https://github.com/jupp0r/prometheus-cpp/pull/225 - "@com_github_ray_project_ray//thirdparty/patches:prometheus-windows-zlib.patch", - "@com_github_ray_project_ray//thirdparty/patches:prometheus-windows-pollfd.patch", - "@com_github_ray_project_ray//thirdparty/patches:prometheus-zlib-fdopen.patch", + "@io_ray//thirdparty/patches:prometheus-windows-zlib.patch", + "@io_ray//thirdparty/patches:prometheus-windows-pollfd.patch", + "@io_ray//thirdparty/patches:prometheus-zlib-fdopen.patch", ], ) @@ -277,9 +277,9 @@ def ray_deps_setup(): url = "https://github.com/grpc/grpc/archive/refs/tags/v1.57.1.tar.gz", sha256 = "0762f809b9de845e6a7c809cabccad6aa4143479fd43b396611fe5a086c0aeeb", patches = [ - "@com_github_ray_project_ray//thirdparty/patches:grpc-cython-copts.patch", - "@com_github_ray_project_ray//thirdparty/patches:grpc-zlib-fdopen.patch", - "@com_github_ray_project_ray//thirdparty/patches:grpc-configurable-thread-count.patch", + "@io_ray//thirdparty/patches:grpc-cython-copts.patch", + "@io_ray//thirdparty/patches:grpc-zlib-fdopen.patch", + "@io_ray//thirdparty/patches:grpc-configurable-thread-count.patch", ], ) @@ -356,7 +356,7 @@ def ray_deps_setup(): url = "https://github.com/msgpack/msgpack-c/archive/8085ab8721090a447cf98bb802d1406ad7afe420.tar.gz", sha256 = "83c37c9ad926bbee68d564d9f53c6cbb057c1f755c264043ddd87d89e36d15bb", patches = [ - "@com_github_ray_project_ray//thirdparty/patches:msgpack-windows-iovec.patch", + "@io_ray//thirdparty/patches:msgpack-windows-iovec.patch", ], ) @@ -372,7 +372,7 @@ def ray_deps_setup(): strip_prefix = "json-3.9.1", urls = ["https://github.com/nlohmann/json/archive/v3.9.1.tar.gz"], sha256 = "4cf0df69731494668bdd6460ed8cb269b68de9c19ad8c27abc24cd72605b2d5b", - build_file = "@com_github_ray_project_ray//bazel:nlohmann_json.BUILD", + build_file = "@io_ray//bazel:nlohmann_json.BUILD", ) auto_http_archive( @@ -398,7 +398,7 @@ def ray_deps_setup(): http_archive( name = "jemalloc", urls = ["https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"], - build_file = "@com_github_ray_project_ray//bazel:jemalloc.BUILD", + build_file = "@io_ray//bazel:jemalloc.BUILD", sha256 = "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa", strip_prefix = "jemalloc-5.3.0", ) diff --git a/ci/pipeline/test_conditional_testing.py b/ci/pipeline/test_conditional_testing.py index b7b577dfa59a..91e59c462a94 100644 --- a/ci/pipeline/test_conditional_testing.py +++ b/ci/pipeline/test_conditional_testing.py @@ -10,7 +10,7 @@ from ci.pipeline.determine_tests_to_run import TagRule, TagRuleSet -_REPO_NAME = "com_github_ray_project_ray" +_REPO_NAME = "io_ray" _runfiles = runfiles.Create() diff --git a/ci/ray_ci/windows/build_ray.sh b/ci/ray_ci/windows/build_ray.sh index 0966becbf3d1..2731118742ce 100644 --- a/ci/ray_ci/windows/build_ray.sh +++ b/ci/ray_ci/windows/build_ray.sh @@ -11,7 +11,8 @@ cd /c/rayci { echo "build --announce_rc"; echo "build --config=ci"; - echo "startup --output_user_root=c:/raytmp"; + # Set a shorter output_base to avoid long file paths that Windows can't handle. + echo "startup --output_base=c:/bzl"; echo "build --remote_cache=${BUILDKITE_BAZEL_CACHE_URL}"; } >> ~/.bazelrc diff --git a/ci/raydepsets/testing_utils.py b/ci/raydepsets/testing_utils.py index ee6040df5e41..38f595ce7003 100644 --- a/ci/raydepsets/testing_utils.py +++ b/ci/raydepsets/testing_utils.py @@ -4,7 +4,7 @@ import runfiles -_REPO_NAME = "com_github_ray_project_ray" +_REPO_NAME = "io_ray" _runfiles = runfiles.Create() diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 9bb61764c2d9..b623a7db3cfa 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -29,7 +29,7 @@ save_packages_to_file, ) -_REPO_NAME = "com_github_ray_project_ray" +_REPO_NAME = "io_ray" _runfiles = runfiles.Create() diff --git a/ci/raydepsets/tests/utils.py b/ci/raydepsets/tests/utils.py index 8dae5634a524..6e3ca310b7d7 100644 --- a/ci/raydepsets/tests/utils.py +++ b/ci/raydepsets/tests/utils.py @@ -4,7 +4,7 @@ import runfiles -_REPO_NAME = "com_github_ray_project_ray" +_REPO_NAME = "io_ray" _runfiles = runfiles.Create() diff --git a/doc/external/test_hashes.py b/doc/external/test_hashes.py index 7e98fcf6bc4b..4bd5a62aa203 100644 --- a/doc/external/test_hashes.py +++ b/doc/external/test_hashes.py @@ -6,7 +6,7 @@ import runfiles import pytest -_REPO_NAME = "com_github_ray_project_ray" +_REPO_NAME = "io_ray" _runfiles = runfiles.Create() diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 357673725614..df4c0d1aceed 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -249,22 +249,22 @@ java_test( # More detail please see https://github.com/ray-project/ray/pull/21641. java_proto_compile( name = "common_java_proto", - deps = ["@com_github_ray_project_ray//src/ray/protobuf:common_proto"], + deps = ["@io_ray//src/ray/protobuf:common_proto"], ) java_proto_compile( name = "runtime_env_common_java_proto", - deps = ["@com_github_ray_project_ray//src/ray/protobuf:runtime_env_common_proto"], + deps = ["@io_ray//src/ray/protobuf:runtime_env_common_proto"], ) java_proto_compile( name = "gcs_java_proto", - deps = ["@com_github_ray_project_ray//src/ray/protobuf:gcs_proto"], + deps = ["@io_ray//src/ray/protobuf:gcs_proto"], ) java_proto_compile( name = "serve_java_proto", - deps = ["@com_github_ray_project_ray//src/ray/protobuf:serve_proto"], + deps = ["@io_ray//src/ray/protobuf:serve_proto"], ) filegroup( diff --git a/python/ray/serve/tests/unit/test_config.py b/python/ray/serve/tests/unit/test_config.py index ee462a42ba63..eb5eb7017061 100644 --- a/python/ray/serve/tests/unit/test_config.py +++ b/python/ray/serve/tests/unit/test_config.py @@ -168,7 +168,9 @@ def test_setting_and_getting_request_router_class(self): "python.ray.serve.tests.unit.test_config.FakeRequestRouter" ) if sys.platform == "win32": - request_router_path = "com_github_ray_project_ray.python.ray.serve.tests.unit.test_config.FakeRequestRouter" + request_router_path = ( + "io_ray.python.ray.serve.tests.unit.test_config.FakeRequestRouter" + ) # Passing request_router_class as a class. deployment_config = DeploymentConfig.from_default( diff --git a/python/ray/tests/test_traceback.py b/python/ray/tests/test_traceback.py index 4d8c20bc24ce..0d3de67cda18 100644 --- a/python/ray/tests/test_traceback.py +++ b/python/ray/tests/test_traceback.py @@ -40,9 +40,9 @@ def scrub_traceback(ex): ex = re.sub(r"\x1b\[39m", "", ex) # When running bazel test with pytest 6.x, the module name becomes # "python.ray.tests.test_traceback" instead of just "test_traceback" - # Also remove the "com_github_ray_project_ray" prefix, which may appear on Windows. + # Also remove the "io_ray" prefix, which may appear on Windows. ex = re.sub( - r"(com_github_ray_project_ray.)?python\.ray\.tests\.test_traceback", + r"(io_ray.)?python\.ray\.tests\.test_traceback", "test_traceback", ex, ) diff --git a/release/ray_release/bazel.py b/release/ray_release/bazel.py index 895fefc5cc7d..aa82b3a5acf7 100644 --- a/release/ray_release/bazel.py +++ b/release/ray_release/bazel.py @@ -2,7 +2,7 @@ import runfiles -REPO_NAME = "com_github_ray_project_ray" +REPO_NAME = "io_ray" _LEGACY_REPO_ROOT = os.path.abspath( os.path.join(os.path.dirname(__file__), "../.."), ) diff --git a/src/ray/core_worker/task_execution/test/BUILD.bazel b/src/ray/core_worker/task_execution/test/BUILD.bazel index 261846d87063..48666ca86ce0 100644 --- a/src/ray/core_worker/task_execution/test/BUILD.bazel +++ b/src/ray/core_worker/task_execution/test/BUILD.bazel @@ -23,9 +23,8 @@ ray_cc_test( ], ) -# Test name is shortened for running on Windows. ray_cc_test( - name = "concrncy_grp_mgr_test", + name = "concurrency_group_manager_test", srcs = ["concurrency_group_manager_test.cc"], tags = ["team:core"], deps = [ diff --git a/src/ray/core_worker/task_submission/test/BUILD.bazel b/src/ray/core_worker/task_submission/test/BUILD.bazel index 0c96c0d3c0ae..036e4a297df1 100644 --- a/src/ray/core_worker/task_submission/test/BUILD.bazel +++ b/src/ray/core_worker/task_submission/test/BUILD.bazel @@ -1,7 +1,7 @@ load("//bazel:ray.bzl", "ray_cc_test") ray_cc_test( - name = "dep_res_test", + name = "dependency_resolver_test", size = "small", srcs = ["dependency_resolver_test.cc"], tags = ["team:core"], @@ -15,7 +15,7 @@ ray_cc_test( ) ray_cc_test( - name = "ooo_submit_queue_test", + name = "out_of_order_actor_submit_queue_test", size = "small", srcs = ["out_of_order_actor_submit_queue_test.cc"], tags = ["team:core"], @@ -28,7 +28,7 @@ ray_cc_test( ) ray_cc_test( - name = "da_transport_test", + name = "direct_actor_transport_test", srcs = ["direct_actor_transport_test.cc"], tags = ["team:core"], deps = [ @@ -40,7 +40,7 @@ ray_cc_test( ) ray_cc_test( - name = "at_submitter_test", + name = "actor_task_submitter_test", srcs = ["actor_task_submitter_test.cc"], tags = ["team:core"], deps = [ @@ -58,7 +58,7 @@ ray_cc_test( ) ray_cc_test( - name = "nt_submitter_test", + name = "normal_task_submitter_test", size = "small", srcs = ["normal_task_submitter_test.cc"], tags = ["team:core"], diff --git a/thirdparty/patches/grpc-zlib-fdopen.patch b/thirdparty/patches/grpc-zlib-fdopen.patch index c48a35bc4ec5..83dfba2b95ff 100644 --- a/thirdparty/patches/grpc-zlib-fdopen.patch +++ b/thirdparty/patches/grpc-zlib-fdopen.patch @@ -6,8 +6,8 @@ diff -u bazel/grpc_deps.bzl "https://github.com/madler/zlib/archive/04f42ceca40f73e2978b50e93806c2a18c1281fc.tar.gz", ], + patches = [ -+ "@com_github_ray_project_ray//thirdparty/patches:zlib-fdopen.patch", ++ "@io_ray//thirdparty/patches:zlib-fdopen.patch", + ] ) - if "com_google_protobuf" not in native.existing_rules(): \ No newline at end of file + if "com_google_protobuf" not in native.existing_rules(): diff --git a/thirdparty/patches/prometheus-zlib-fdopen.patch b/thirdparty/patches/prometheus-zlib-fdopen.patch index e8ef276d1d14..6d0a112f0891 100644 --- a/thirdparty/patches/prometheus-zlib-fdopen.patch +++ b/thirdparty/patches/prometheus-zlib-fdopen.patch @@ -6,6 +6,6 @@ diff -u bazel/repositories.bzl /tmp/repositories.bzl ], build_file = "@com_github_jupp0r_prometheus_cpp//bazel:zlib.BUILD", + patches = [ -+ "@com_github_ray_project_ray//thirdparty/patches:zlib-fdopen.patch", ++ "@io_ray//thirdparty/patches:zlib-fdopen.patch", + ] - ) \ No newline at end of file + ) From ade722a935099a8700dba8b2436eeffba90a207f Mon Sep 17 00:00:00 2001 From: Alexey Kudinkin Date: Tue, 12 Aug 2025 21:27:09 -0400 Subject: [PATCH 071/634] [Data] Fixing flaky test (#55485) ## Why are these changes needed? Currently, the test is allocated only 100Mb, while it's trying to store 2Gb worth of data in the Object Store. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Alexey Kudinkin --- python/ray/data/BUILD | 4 ++-- ...k_scaling.py => test_jumbo_arrow_block.py} | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) rename python/ray/data/tests/{test_arrow_block_scaling.py => test_jumbo_arrow_block.py} (82%) diff --git a/python/ray/data/BUILD b/python/ray/data/BUILD index 6b0083700fd0..59bc8b57c741 100644 --- a/python/ray/data/BUILD +++ b/python/ray/data/BUILD @@ -214,9 +214,9 @@ py_test( ) py_test( - name = "test_arrow_block_scaling", + name = "test_jumbo_arrow_block", size = "large", - srcs = ["tests/test_arrow_block_scaling.py"], + srcs = ["tests/test_jumbo_arrow_block.py"], tags = [ "data_non_parallel", "exclusive", diff --git a/python/ray/data/tests/test_arrow_block_scaling.py b/python/ray/data/tests/test_jumbo_arrow_block.py similarity index 82% rename from python/ray/data/tests/test_arrow_block_scaling.py rename to python/ray/data/tests/test_jumbo_arrow_block.py index 75282dcdb085..ba7fd0b586d3 100644 --- a/python/ray/data/tests/test_arrow_block_scaling.py +++ b/python/ray/data/tests/test_jumbo_arrow_block.py @@ -10,6 +10,7 @@ import ray from ray.data import DataContext from ray.data._internal.util import GiB, MiB +from ray.tests.conftest import _ray_start @pytest.fixture(scope="module") @@ -45,6 +46,18 @@ def parquet_dataset_single_column_gt_2gb(): print(f">>> Cleaning up dataset at {dataset_path}") +@pytest.fixture(scope="module") +def ray_cluster_3gb_object_store(): + original_limit = ray._private.ray_constants.MAC_DEGRADED_PERF_MMAP_SIZE_LIMIT + + ray._private.ray_constants.MAC_DEGRADED_PERF_MMAP_SIZE_LIMIT = 3 * GiB + + with _ray_start(object_store_memory=3 * GiB) as res: + yield res + + ray._private.ray_constants.MAC_DEGRADED_PERF_MMAP_SIZE_LIMIT = original_limit + + @pytest.mark.parametrize( "op", [ @@ -54,7 +67,7 @@ def parquet_dataset_single_column_gt_2gb(): ) @pytest.mark.timeout(300) def test_arrow_batch_gt_2gb( - ray_start_regular, + ray_cluster_3gb_object_store, parquet_dataset_single_column_gt_2gb, restore_data_context, op, @@ -76,9 +89,9 @@ def _id(x): # numpy format ds = ds.map_batches( _id, - batch_format="numpy", + batch_format="pyarrow", batch_size=num_rows, - zero_copy_batch=False, + zero_copy_batch=True, ) batch = ds.take_batch() From 15d33bfd32415279703300bf50a7f26d5a3102c9 Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Wed, 13 Aug 2025 15:58:42 +0200 Subject: [PATCH 072/634] [RLlib] Add another curriculum example using Atari Pong and frameskip tasks. (#54966) --- doc/source/rllib/rllib-examples.rst | 5 + rllib/BUILD | 20 ++ .../curriculum/curriculum_learning.py | 9 + .../curriculum/pong_curriculum_learning.py | 281 ++++++++++++++++++ 4 files changed, 315 insertions(+) create mode 100644 rllib/examples/curriculum/pong_curriculum_learning.py diff --git a/doc/source/rllib/rllib-examples.rst b/doc/source/rllib/rllib-examples.rst index f32307de0bfa..16d49f4917d0 100644 --- a/doc/source/rllib/rllib-examples.rst +++ b/doc/source/rllib/rllib-examples.rst @@ -177,6 +177,11 @@ Curriculum learning This approach enables gradual learning, allowing agents to master simpler tasks before progressing to more challenging ones, ideal for environments with hierarchical or staged difficulties. Also see the :doc:`curriculum learning how-to ` from the documentation. +- `Curriculum learning for Atari Pong `__: + Demonstrates curriculum learning for Atari Pong using the `frameskip` to increase difficulty of the task. + This approach enables gradual learning, allowing agents to master slower reactions (lower `frameskip`) before progressing to more faster ones (higher `frameskip`). + Also see the :doc:`curriculum learning how-to ` from the documentation. + Debugging +++++++++ diff --git a/rllib/BUILD b/rllib/BUILD index 41fcc6f16146..a71722bd0607 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -3973,6 +3973,26 @@ py_test( ], ) +py_test( + name = "examples/curriculum/pong_curriculum_learning", + size = "large", + srcs = ["examples/curriculum/pong_curriculum_learning.py"], + args = [ + "--as-test", + "--num-env-runners=10", + "--num-cpus=11", + "--num-envs-per-env-runner=5", + "--stop-iters=20", + "--stop-reward=-21.0", + ], + main = "examples/curriculum/pong_curriculum_learning.py", + tags = [ + "examples", + "exclusive", + "team:rllib", + ], +) + # subdirectory: debugging/ # .................................... py_test( diff --git a/rllib/examples/curriculum/curriculum_learning.py b/rllib/examples/curriculum/curriculum_learning.py index 252a468698be..a3a71de6789f 100644 --- a/rllib/examples/curriculum/curriculum_learning.py +++ b/rllib/examples/curriculum/curriculum_learning.py @@ -148,6 +148,15 @@ def _remote_fn(env_runner, new_task: int): class EnvTaskCallback(RLlibCallback): """Custom callback implementing `on_train_result()` for changing the envs' maps.""" + def on_algorithm_init( + self, + *, + algorithm: "Algorithm", + **kwargs, + ) -> None: + # Set the initial task to 0. + algorithm._counters["current_env_task"] = 0 + def on_train_result( self, *, diff --git a/rllib/examples/curriculum/pong_curriculum_learning.py b/rllib/examples/curriculum/pong_curriculum_learning.py new file mode 100644 index 000000000000..07881d3275d5 --- /dev/null +++ b/rllib/examples/curriculum/pong_curriculum_learning.py @@ -0,0 +1,281 @@ +"""Example of using curriculum learning for Atari Pong by implementing a custom callback. + +This example: + - demonstrates how to define a curriculum for an agent playing gymnasium's Atari + Pong. + - defines a custom callback that gets called once per iteration and - if the agent + performs well enough - increases the task difficulty, i.e. the `frameskip` for all + environments on all EnvRunners (the agent must act now faster). + - also demonstrates how to provide the callback with varying curriculum parameters + (like threshold maps, returns at which the curriculum ends, etc.). + - uses Ray Tune and RLlib to curriculum-learn Atari Pong with a high frameskip. + +We use Atari Pong with a framestack of 4 images (i.e. observation dimensions of 64x64x4) +and start with a frameskip of 1. At a return of 15.0 we increase the frameskip to 2, at +a return of 17.0 to 3, at 19.0 to 4, and the task is solved at a frameskip of 21.0. + +How to run this script +---------------------- +`python [script file name].py` + +Use the `--solved-return` flag to define the threshold at which curriculum learning ends. +Note that a PPO agent on Atari Pong will need a long time to learn. + +To ensure the agent has not collapsed, but rather made had a bad seed, we only decrease +the frameskip when the agent performed worse than the next lower threshold. The margin by +which the agent has to be worse is defined by the `--demotion-margin` argument and defaults +to 2.0. + +For debugging, use the following additional command line options +`--no-tune --num-env-runners=0` +which should allow you to set breakpoints anywhere in the RLlib code and +have the execution stop there for inspection and debugging. + +For logging to your WandB account, use: +`--wandb-key=[your WandB API key] --wandb-project=[some project name] +--wandb-run-name=[optional: WandB run name (within the defined project)]` + + +""" + +import functools +import gymnasium as gym +from typing import Callable + +from ray import tune +from ray.rllib.algorithms.algorithm import Algorithm +from ray.rllib.algorithms.ppo import PPOConfig +from ray.rllib.callbacks.callbacks import RLlibCallback +from ray.rllib.connectors.env_to_module.frame_stacking import FrameStackingEnvToModule +from ray.rllib.connectors.learner.frame_stacking import FrameStackingLearner +from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig +from ray.rllib.env.wrappers.atari_wrappers import wrap_atari_for_new_api_stack +from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS, EPISODE_RETURN_MEAN +from ray.rllib.utils.test_utils import add_rllib_example_script_args + + +parser = add_rllib_example_script_args( + default_reward=float("inf"), + default_timesteps=3000000, + default_iters=100000000000, +) +parser.set_defaults( + env="ale_py:ALE/Pong-v5", +) +parser.add_argument( + "--solved-return", + type=float, + default=21.0, + help=("The mean episode return at which we consider the task to be fully solved."), +) +parser.add_argument( + "--demotion-margin", + type=float, + default=2.0, + help=( + "The margin below the next lower task threshold, beneath which the agent " + " is considered to have collapsed, prompting a downgrade of the task." + ), +) +# Use `parser` to add your own custom command line options to this script +# and (if needed) use their values to set up `config` below. +args = parser.parse_args() + +NUM_LEARNERS = args.num_learners or 1 +ENV = args.env + + +class PongEnvTaskCallback(RLlibCallback): + """Custom callback changing the frameskip in Atari Pong dependent on return.""" + + def __init__( + self, + task_threshold_map: dict, + remote_fn: Callable, + demotion_margin: float = 0.0, + solved_return: float = float("inf"), + ): + self.task_threshold_map = task_threshold_map + self.remote_fn = remote_fn + self.demotion_margin = demotion_margin + self.solved_return = solved_return + + def on_algorithm_init( + self, + *, + algorithm: "Algorithm", + **kwargs, + ) -> None: + # Set the initial task to 1, which corresponds to a frameskip of 1. + algorithm.metrics.log_value("current_env_task", 1, reduce="sum") + + def on_train_result( + self, + *, + algorithm: Algorithm, + metrics_logger=None, + result: dict, + **kwargs, + ) -> None: + # Store the current task inside the metrics logger in our Algorithm. + current_task = metrics_logger.peek("current_env_task") + + # If episode return is consistently above `task_threshold_map[current_task]`, + # we switch to a more difficult task (i.e. higher `frameskip`` if possible). + # If we already mastered the most difficult task, we publish our victory in + # the result dict. + result["task_solved"] = 0.0 + + # Note, in the first callback executions there may be no completed episode + # (and therefore no episode return) reported. In this case we will skip the + # the logic to manage task difficulty. + if EPISODE_RETURN_MEAN in result[ENV_RUNNER_RESULTS]: + current_return = result[ENV_RUNNER_RESULTS][EPISODE_RETURN_MEAN] + else: + return + + # Get the threshold of the current task from the threshold map. + threshold = self.task_threshold_map.get(current_task, float("inf")) + + # Check, if curriculum is solved. + final_task = max(self.task_threshold_map.keys()) + if current_task == final_task and current_return >= self.solved_return: + # Hardest task was solved -> report this in the results dict. + result["task_solved"] = 1.0 + + # Check promotion (increasing task). Note, we could use here also a promotion_patience + # that ensures that the return is collected in a stable manner instead of a lucky shot. + if ( + current_return >= threshold + ): # & result[ENV_RUNNER_RESULTS][NUM_EPISODES] > promotion_patience. + next_task = current_task + 1 + if next_task in self.task_threshold_map: + print( + f"Switching task on all EnvRunners up to #{next_task} (1=easiest, " + f"4=hardest), b/c R={current_return} on current task." + ) + # Increase task. + algorithm.env_runner_group.foreach_env_runner( + func=functools.partial(self.remote_fn, new_task=next_task) + ) + metrics_logger.log_value("current_env_task", next_task, window=1) + + # Check demotion (decreasing task). The demotion is used to avoid decreasing the task + # in case of an unlucky episode run. Only if the return is singificantly lower we + # decrease the task. + previous_task = current_task - 1 + if previous_task in self.task_threshold_map: + previous_threshold = self.task_threshold_map[previous_task] + if current_return < previous_threshold - self.demotion_margin: + print( + f"Switching task on all EnvRunners back to #{previous_task} (1=easiest, " + f"4=hardest), b/c R={current_return} on current task." + ) + # Decrease to previous level. + algorithm.env_runner_group.foreach_env_runner( + func=functools.partial(self.remote_fn, new_task=previous_task) + ) + metrics_logger.log_value("current_env_task", previous_task, window=1) + + +# These tags allow extracting portions of this script on Anyscale. +# ws-template-code-start +def _make_env_to_module_connector(env, spaces, device): + return FrameStackingEnvToModule(num_frames=4) + + +def _make_learner_connector(input_observation_space, input_action_space): + return FrameStackingLearner(num_frames=4) + + +# Create a custom Atari setup (w/o the usual RLlib-hard-coded framestacking in it). +# We would like our frame stacking connector to do this job. +def _env_creator(cfg): + return wrap_atari_for_new_api_stack( + gym.make(ENV, **cfg, render_mode="rgb_array"), + # Perform frame-stacking through ConnectorV2 API. + framestack=None, + ) + + +# Simple function sent to an EnvRunner to change the map of all its gym. Envs from +# the current one to a new (tougher) one, in which the frameskip is higher +# and the agent must therefore act faster. +def _remote_fn(env_runner, new_task: int): + # Override the env_config with the new setting. + env_runner.config.env_config.update( + { + "frameskip": new_task, + } + ) + # We recreate the entire env object by changing the env_config on the worker, + # then calling its `make_env()` method. + env_runner.make_env() + + +# Task threshold map keeps track of thresholds for each task. If the threshold has +# been surpassed the task difficulty is increased. +task_threshold_map = { + # Frameskip: Return. + 1: 15.0, + 2: 17.0, + 3: 19.0, + 4: float("inf"), +} + +tune.register_env("env", _env_creator) + +config = ( + PPOConfig() + .environment( + "env", + env_config={ + # Make analogous to old v4 + NoFrameskip. + "frameskip": 1, + "full_action_space": False, + "repeat_action_probability": 0.0, + }, + clip_rewards=True, + ) + .env_runners( + env_to_module_connector=_make_env_to_module_connector, + ) + .training( + learner_connector=_make_learner_connector, + train_batch_size_per_learner=4000, + minibatch_size=128, + lambda_=0.95, + kl_coeff=0.5, + clip_param=0.1, + vf_clip_param=10.0, + entropy_coeff=0.01, + num_epochs=10, + lr=0.00015 * NUM_LEARNERS, + grad_clip=100.0, + grad_clip_by="global_norm", + ) + .rl_module( + model_config=DefaultModelConfig( + conv_filters=[[16, 4, 2], [32, 4, 2], [64, 4, 2], [128, 4, 2]], + conv_activation="relu", + head_fcnet_hiddens=[256], + vf_share_layers=True, + ), + ) + .callbacks( + functools.partial( + PongEnvTaskCallback, + task_threshold_map=task_threshold_map, + remote_fn=_remote_fn, + # Avoids downgrading the task to early when the agent had an unlucky run. + demotion_margin=args.demotion_margin, + # The return at which the task is learned. + solved_return=args.solved_return, + ) + ) +) + +if __name__ == "__main__": + from ray.rllib.utils.test_utils import run_rllib_example_script_experiment + + run_rllib_example_script_experiment(config, args=args) From 0ee95c8e66ee4db39eeb3d623382a4cf39147d71 Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Wed, 13 Aug 2025 15:59:06 +0200 Subject: [PATCH 073/634] [RLlib] Implicit Q-Learning. (#55304) --- rllib/BUILD | 51 ++++ rllib/algorithms/iql/__init__.py | 6 + rllib/algorithms/iql/default_iql_rl_module.py | 35 +++ rllib/algorithms/iql/iql.py | 202 +++++++++++++++ rllib/algorithms/iql/iql_learner.py | 84 ++++++ rllib/algorithms/iql/torch/__init__.py | 0 .../iql/torch/default_iql_torch_rl_module.py | 78 ++++++ .../algorithms/iql/torch/iql_torch_learner.py | 245 ++++++++++++++++++ rllib/algorithms/registry.py | 8 + rllib/algorithms/sac/sac_learner.py | 69 +++-- .../sac/torch/default_sac_torch_rl_module.py | 7 +- rllib/tuned_examples/bc/cartpole_bc.py | 2 +- .../bc/cartpole_bc_with_offline_evaluation.py | 2 +- rllib/tuned_examples/cql/pendulum_cql.py | 2 +- rllib/tuned_examples/iql/pendulum_iql.py | 90 +++++++ .../tuned_examples/marwil/cartpole_marwil.py | 2 +- 16 files changed, 856 insertions(+), 27 deletions(-) create mode 100644 rllib/algorithms/iql/__init__.py create mode 100644 rllib/algorithms/iql/default_iql_rl_module.py create mode 100644 rllib/algorithms/iql/iql.py create mode 100644 rllib/algorithms/iql/iql_learner.py create mode 100644 rllib/algorithms/iql/torch/__init__.py create mode 100644 rllib/algorithms/iql/torch/default_iql_torch_rl_module.py create mode 100644 rllib/algorithms/iql/torch/iql_torch_learner.py create mode 100644 rllib/tuned_examples/iql/pendulum_iql.py diff --git a/rllib/BUILD b/rllib/BUILD index a71722bd0607..6c061cedb07f 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -1095,6 +1095,57 @@ py_test( # args = ["--as-test", "--num-learners=2", "--num-gpus-per-learner=1"] # ) +# IQL +# Pendulum-v1 (enormous) +py_test( + name = "learning_tests_pendulum_iql", + size = "large", + srcs = ["tuned_examples/iql/pendulum_iql.py"], + args = [ + "--as-test", + "--num-cpus=32", + ], + # Include the offline data files. + data = [ + "tests/data/pendulum/pendulum-v1_enormous", + ], + main = "tuned_examples/iql/pendulum_iql.py", + tags = [ + "exclusive", + "learning_tests", + "learning_tests_continuous", + "learning_tests_pytorch_use_all_core", + "team:rllib", + "torch_only", + ], +) + +# GPU training. +py_test( + name = "learning_tests_pendulum_iql_gpu", + size = "large", + srcs = ["tuned_examples/iql/pendulum_iql.py"], + args = [ + "--as-test", + "--num-cpus=32", + "--num-gpus-per-learner=1", + ], + # Include the offline data files. + data = [ + "tests/data/pendulum/pendulum-v1_enormous", + ], + main = "tuned_examples/iql/pendulum_iql.py", + tags = [ + "exclusive", + "gpu", + "learning_tests", + "learning_tests_continuous", + "learning_tests_pytorch_use_all_core", + "team:rllib", + "torch_only", + ], +) + # MARWIL # CartPole py_test( diff --git a/rllib/algorithms/iql/__init__.py b/rllib/algorithms/iql/__init__.py new file mode 100644 index 000000000000..404fb83b6aac --- /dev/null +++ b/rllib/algorithms/iql/__init__.py @@ -0,0 +1,6 @@ +from ray.rllib.algorithms.iql.iql import IQL, IQLConfig + +__all__ = [ + "IQL", + "IQLConfig", +] diff --git a/rllib/algorithms/iql/default_iql_rl_module.py b/rllib/algorithms/iql/default_iql_rl_module.py new file mode 100644 index 000000000000..e6e3b2279ac5 --- /dev/null +++ b/rllib/algorithms/iql/default_iql_rl_module.py @@ -0,0 +1,35 @@ +from ray.rllib.algorithms.sac.default_sac_rl_module import DefaultSACRLModule +from ray.rllib.core.models.configs import MLPHeadConfig +from ray.rllib.core.rl_module.apis.value_function_api import ValueFunctionAPI +from ray.rllib.utils.annotations import ( + override, + OverrideToImplementCustomLogic_CallToSuperRecommended, +) + + +class DefaultIQLRLModule(DefaultSACRLModule, ValueFunctionAPI): + @override(DefaultSACRLModule) + def setup(self): + # Setup the `DefaultSACRLModule` to get the catalog. + super().setup() + + # Only, if the `RLModule` is used on a `Learner` we build the value network. + if not self.inference_only: + # Build the encoder for the value function. + self.vf_encoder = self.catalog.build_encoder(framework=self.framework) + + # Build the vf head. + self.vf = MLPHeadConfig( + input_dims=self.catalog.latent_dims, + # Note, we use the same layers as for the policy and Q-network. + hidden_layer_dims=self.catalog.pi_and_qf_head_hiddens, + hidden_layer_activation=self.catalog.pi_and_qf_head_activation, + output_layer_activation="linear", + output_layer_dim=1, + ).build(framework=self.framework) + + @override(DefaultSACRLModule) + @OverrideToImplementCustomLogic_CallToSuperRecommended + def get_non_inference_attributes(self): + # Use all of `super`'s attributes and add the value function attributes. + return super().get_non_inference_attributes() + ["vf_encoder", "vf"] diff --git a/rllib/algorithms/iql/iql.py b/rllib/algorithms/iql/iql.py new file mode 100644 index 000000000000..c7175f6d1a7f --- /dev/null +++ b/rllib/algorithms/iql/iql.py @@ -0,0 +1,202 @@ +from typing import Optional, Type, Union + +from ray.rllib.algorithms.algorithm_config import AlgorithmConfig, NotProvided +from ray.rllib.algorithms.marwil.marwil import MARWIL, MARWILConfig +from ray.rllib.connectors.common.add_observations_from_episodes_to_batch import ( + AddObservationsFromEpisodesToBatch, +) +from ray.rllib.connectors.learner.add_next_observations_from_episodes_to_train_batch import ( # noqa + AddNextObservationsFromEpisodesToTrainBatch, +) +from ray.rllib.core.learner.learner import Learner +from ray.rllib.core.rl_module.rl_module import RLModuleSpec +from ray.rllib.utils.annotations import override +from ray.rllib.utils.typing import LearningRateOrSchedule, RLModuleSpecType + + +class IQLConfig(MARWILConfig): + """Defines a configuration class from which a new IQL Algorithm can be built + + .. testcode:: + :skipif: True + + from ray.rllib.algorithms.iql import IQLConfig + # Run this from the ray directory root. + config = IQLConfig().training(lr=0.00001, gamma=0.99) + config = config.offline_data( + input_="./rllib/tests/data/pendulum/pendulum-v1_enormous") + + # Build an Algorithm object from the config and run 1 training iteration. + algo = config.build() + algo.train() + + .. testcode:: + :skipif: True + + from ray.rllib.algorithms.iql import IQLConfig + from ray import tune + config = IQLConfig() + # Print out some default values. + print(config.beta) + # Update the config object. + config.training( + lr=tune.grid_search([0.001, 0.0001]), beta=0.75 + ) + # Set the config object's data path. + # Run this from the ray directory root. + config.offline_data( + input_="./rllib/tests/data/pendulum-v1_enormous" + ) + # Set the config object's env, used for evaluation. + config.environment(env="Pendulum-v1") + # Use to_dict() to get the old-style python config dict + # when running with tune. + tune.Tuner( + "IQL", + param_space=config.to_dict(), + ).fit() + """ + + def __init__(self, algo_class=None): + super().__init__(algo_class=algo_class or IQL) + + # fmt: off + # __sphinx_doc_begin__ + # The temperature for the actor loss. + self.beta = 0.1 + + # The expectile to use in expectile regression. + self.expectile = 0.8 + + # The learning rates for the actor, critic and value network(s). + self.actor_lr = 3e-4 + self.critic_lr = 3e-4 + self.value_lr = 3e-4 + # Set `lr` parameter to `None` and ensure it is not used. + self.lr = None + + # If a twin-Q architecture should be used (advisable). + self.twin_q = True + + # How often the target network should be updated. + self.target_network_update_freq = 0 + # The weight for Polyak averaging. + self.tau = 1.0 + + # __sphinx_doc_end__ + # fmt: on + + @override(MARWILConfig) + def training( + self, + *, + twin_q: Optional[bool] = NotProvided, + expectile: Optional[float] = NotProvided, + actor_lr: Optional[LearningRateOrSchedule] = NotProvided, + critic_lr: Optional[LearningRateOrSchedule] = NotProvided, + value_lr: Optional[LearningRateOrSchedule] = NotProvided, + target_network_update_freq: Optional[int] = NotProvided, + tau: Optional[float] = NotProvided, + **kwargs, + ) -> "IQLConfig": + super().training(**kwargs) + + if twin_q is not NotProvided: + self.twin_q = twin_q + if expectile is not NotProvided: + self.expectile = expectile + if actor_lr is not NotProvided: + self.actor_lr = actor_lr + if critic_lr is not NotProvided: + self.critic_lr = critic_lr + if value_lr is not NotProvided: + self.value_lr = value_lr + if target_network_update_freq is not NotProvided: + self.target_network_update_freq = target_network_update_freq + if tau is not NotProvided: + self.tau = tau + + return self + + @override(MARWILConfig) + def get_default_learner_class(self) -> Union[Type["Learner"], str]: + if self.framework_str == "torch": + from ray.rllib.algorithms.iql.torch.iql_torch_learner import IQLTorchLearner + + return IQLTorchLearner + else: + raise ValueError( + f"The framework {self.framework_str} is not supported. " + "Use `'torch'` instead." + ) + + @override(MARWILConfig) + def get_default_rl_module_spec(self) -> RLModuleSpecType: + if self.framework_str == "torch": + from ray.rllib.algorithms.iql.torch.default_iql_torch_rl_module import ( + DefaultIQLTorchRLModule, + ) + + return RLModuleSpec(module_class=DefaultIQLTorchRLModule) + else: + raise ValueError( + f"The framework {self.framework_str} is not supported. " + "Use `torch` instead." + ) + + @override(MARWILConfig) + def build_learner_connector( + self, + input_observation_space, + input_action_space, + device=None, + ): + pipeline = super().build_learner_connector( + input_observation_space=input_observation_space, + input_action_space=input_action_space, + device=device, + ) + + # Remove unneeded connectors from the MARWIL connector pipeline. + pipeline.remove("AddOneTsToEpisodesAndTruncate") + pipeline.remove("GeneralAdvantageEstimation") + + # Prepend the "add-NEXT_OBS-from-episodes-to-train-batch" connector piece (right + # after the corresponding "add-OBS-..." default piece). + pipeline.insert_after( + AddObservationsFromEpisodesToBatch, + AddNextObservationsFromEpisodesToTrainBatch(), + ) + + return pipeline + + @override(MARWILConfig) + def validate(self) -> None: + # Call super's validation method. + super().validate() + + # Ensure hyperparameters are meaningful. + if self.beta <= 0.0: + self._value_error( + "For meaningful results, `beta` (temperature) parameter must be >> 0.0!" + ) + if not 0.0 < self.expectile < 1.0: + self._value_error( + "For meaningful results, `expectile` parameter must be in (0, 1)." + ) + + @property + def _model_config_auto_includes(self): + return super()._model_config_auto_includes | {"twin_q": self.twin_q} + + +class IQL(MARWIL): + """Implicit Q-learning (derived from MARWIL). + + Uses MARWIL training step. + """ + + @classmethod + @override(MARWIL) + def get_default_config(cls) -> AlgorithmConfig: + return IQLConfig() diff --git a/rllib/algorithms/iql/iql_learner.py b/rllib/algorithms/iql/iql_learner.py new file mode 100644 index 000000000000..5821f2ccb5e0 --- /dev/null +++ b/rllib/algorithms/iql/iql_learner.py @@ -0,0 +1,84 @@ +from typing import Dict + +from ray.rllib.algorithms.dqn.dqn_learner import DQNLearner +from ray.rllib.utils.annotations import ( + override, + OverrideToImplementCustomLogic_CallToSuperRecommended, +) +from ray.rllib.utils.lambda_defaultdict import LambdaDefaultDict +from ray.rllib.utils.typing import ModuleID, TensorType + +QF_TARGET_PREDS = "qf_target_preds" +VF_PREDS_NEXT = "vf_preds_next" +VF_LOSS = "value_loss" + + +class IQLLearner(DQNLearner): + @OverrideToImplementCustomLogic_CallToSuperRecommended + @override(DQNLearner) + def build(self) -> None: + # Build the `DQNLearner` (builds the target network). + super().build() + + # Define the expectile parameter(s). + self.expectile: Dict[ModuleID, TensorType] = LambdaDefaultDict( + lambda module_id: self._get_tensor_variable( + # Note, we want to train with a certain expectile. + [self.config.get_config_for_module(module_id).expectile], + trainable=False, + ) + ) + + # Define the temperature for the actor advantage loss. + self.temperature: Dict[ModuleID, TensorType] = LambdaDefaultDict( + lambda module_id: self._get_tensor_variable( + # Note, we want to train with a certain expectile. + [self.config.get_config_for_module(module_id).beta], + trainable=False, + ) + ) + + # Store loss tensors here temporarily inside the loss function for (exact) + # consumption later by the compute gradients function. + # Keys=(module_id, optimizer_name), values=loss tensors (in-graph). + self._temp_losses = {} + + @override(DQNLearner) + def remove_module(self, module_id: ModuleID) -> None: + """Removes the expectile and temperature for removed modules.""" + # First call `super`'s `remove_module` method. + super().remove_module(module_id) + # Remove the expectile from the mapping. + self.expectile.pop(module_id, None) + # Remove the temperature from the mapping. + self.temperature.pop(module_id, None) + + @override(DQNLearner) + def add_module( + self, + *, + module_id, + module_spec, + config_overrides=None, + new_should_module_be_updated=None + ): + """Adds the expectile and temperature for new modules.""" + # First call `super`'s `add_module` method. + super().add_module( + module_id=module_id, + module_spec=module_spec, + config_overrides=config_overrides, + new_should_module_be_updated=new_should_module_be_updated, + ) + # Add the expectile to the mapping. + self.expectile[module_id] = self._get_tensor_variable( + # Note, we want to train with a certain expectile. + [self.config.get_config_for_module(module_id).beta], + trainable=False, + ) + # Add the temperature to the mapping. + self.temperature[module_id] = self._get_tensor_variable( + # Note, we want to train with a certain expectile. + [self.config.get_config_for_module(module_id).beta], + trainable=False, + ) diff --git a/rllib/algorithms/iql/torch/__init__.py b/rllib/algorithms/iql/torch/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/rllib/algorithms/iql/torch/default_iql_torch_rl_module.py b/rllib/algorithms/iql/torch/default_iql_torch_rl_module.py new file mode 100644 index 000000000000..00d7fc821e49 --- /dev/null +++ b/rllib/algorithms/iql/torch/default_iql_torch_rl_module.py @@ -0,0 +1,78 @@ +import gymnasium as gym +from typing import Any, Dict, Optional + +from ray.rllib.algorithms.iql.default_iql_rl_module import DefaultIQLRLModule +from ray.rllib.algorithms.iql.iql_learner import VF_PREDS_NEXT, QF_TARGET_PREDS +from ray.rllib.algorithms.sac.torch.default_sac_torch_rl_module import ( + DefaultSACTorchRLModule, +) +from ray.rllib.core.columns import Columns +from ray.rllib.core.models.base import ENCODER_OUT +from ray.rllib.core.rl_module.apis.value_function_api import ValueFunctionAPI +from ray.rllib.utils.annotations import override +from ray.rllib.utils.framework import try_import_torch +from ray.rllib.utils.typing import TensorType + +torch, nn = try_import_torch() + + +class DefaultIQLTorchRLModule(DefaultSACTorchRLModule, DefaultIQLRLModule): + + framework: str = "torch" + + @override(DefaultSACTorchRLModule) + def _forward_train(self, batch: Dict, **kwargs) -> Dict[str, Any]: + + # Right now, IQL runs only with continuous action spaces. + # TODO (simon): Implement it also for discrete action spaces. + if not isinstance(self.action_space, gym.spaces.Box): + raise ValueError( + f"Unsupported action space type: {type(self.action_space)}. " + "Only continuous action spaces are supported." + ) + + # Call the forward pass of the SAC module. + output = super()._forward_train(batch, **kwargs) + + # Create batches for the forward passes of the target Q-networks and the + # value function. + batch_curr = { + Columns.OBS: batch[Columns.OBS], + Columns.ACTIONS: batch[Columns.ACTIONS], + } + batch_next = {Columns.OBS: batch[Columns.NEXT_OBS]} + + # These target q-values are needed for the value loss and actor loss. + output[QF_TARGET_PREDS] = self._qf_forward_train_helper( + batch_curr, encoder=self.target_qf_encoder, head=self.target_qf + ) + # If a twin-Q architecture is used run its target Q-network. + if self.twin_q: + output[QF_TARGET_PREDS] = torch.min( + output[QF_TARGET_PREDS], + self._qf_forward_train_helper( + batch_curr, encoder=self.target_qf_twin_encoder, head=self.qf_twin + ), + ) + + # Compute values for the current observations. + output[Columns.VF_PREDS] = self.compute_values(batch_curr) + # The values of the next observations are needed for the critic loss. + output[VF_PREDS_NEXT] = self.compute_values(batch_next) + + return output + + @override(ValueFunctionAPI) + def compute_values( + self, + batch: Dict[str, Any], + embeddings: Optional[Any] = None, + ) -> TensorType: + # If no embeddings are provided make a forward pass on the encoder. + if embeddings is None: + embeddings = self.vf_encoder(batch)[ENCODER_OUT] + + # Value head. + vf_out = self.vf(embeddings) + # Squeeze out last dimension (single node value head). + return vf_out.squeeze(-1) diff --git a/rllib/algorithms/iql/torch/iql_torch_learner.py b/rllib/algorithms/iql/torch/iql_torch_learner.py new file mode 100644 index 000000000000..85dc68e86fb2 --- /dev/null +++ b/rllib/algorithms/iql/torch/iql_torch_learner.py @@ -0,0 +1,245 @@ +from typing import Dict + +from ray.rllib.algorithms.algorithm_config import AlgorithmConfig +from ray.rllib.algorithms.dqn.dqn_learner import QF_PREDS, QF_LOSS_KEY +from ray.rllib.algorithms.iql.iql_learner import ( + IQLLearner, + QF_TARGET_PREDS, + VF_PREDS_NEXT, + VF_LOSS, +) +from ray.rllib.algorithms.sac.sac_learner import QF_TWIN_PREDS, QF_TWIN_LOSS_KEY +from ray.rllib.core import ALL_MODULES +from ray.rllib.core.columns import Columns +from ray.rllib.core.learner.learner import ( + POLICY_LOSS_KEY, +) +from ray.rllib.core.learner.torch.torch_learner import TorchLearner +from ray.rllib.utils.annotations import override +from ray.rllib.utils.framework import try_import_torch +from ray.rllib.utils.typing import ModuleID, ParamDict, TensorType + +torch, nn = try_import_torch() + + +class IQLTorchLearner(TorchLearner, IQLLearner): + """Implements the IQL loss on top of `IQLLearner`. + + This Learner implements configure_optimizers_for_module to define + separate optimizers for the policy, Q-, and value networks. When + using a twin-Q network architecture, each Q-network is assigned its + own optimizer—consistent with the SAC algorithm. + + The IQL loss is defined in compute_loss_for_module and consists of + three components: value loss, Q-loss (TD error), and actor (policy) + loss. + + Note that the original IQL implementation performs separate backward + passes for each network. However, due to RLlib's reliance on TorchDDP, + all backward passes must be executed within a single update step. This + constraint can lead to parameter lag and cyclical loss behavior, though + it does not hinder convergence. + """ + + @override(TorchLearner) + def configure_optimizers_for_module( + self, module_id: ModuleID, config: AlgorithmConfig = None + ) -> None: + + # Note, we could have derived directly from SACTorchLearner to + # inherit the setup of optimizers, but that learner comes with + # additional parameters which we do not need. + # Receive the module. + module = self._module[module_id] + + # Define the optimizer for the critic. + # TODO (sven): Maybe we change here naming to `qf` for unification. + params_critic = self.get_parameters(module.qf_encoder) + self.get_parameters( + module.qf + ) + optim_critic = torch.optim.Adam(params_critic, eps=1e-7) + self.register_optimizer( + module_id=module_id, + optimizer_name="qf", + optimizer=optim_critic, + params=params_critic, + lr_or_lr_schedule=config.critic_lr, + ) + # If necessary register also an optimizer for a twin Q network. + if config.twin_q: + params_twin_critic = self.get_parameters( + module.qf_twin_encoder + ) + self.get_parameters(module.qf_twin) + optim_twin_critic = torch.optim.Adam(params_twin_critic, eps=1e-7) + self.register_optimizer( + module_id=module_id, + optimizer_name="qf_twin", + optimizer=optim_twin_critic, + params=params_twin_critic, + lr_or_lr_schedule=config.critic_lr, + ) + + # Define the optimizer for the actor. + params_actor = self.get_parameters(module.pi_encoder) + self.get_parameters( + module.pi + ) + optim_actor = torch.optim.Adam(params_actor, eps=1e-7) + self.register_optimizer( + module_id=module_id, + optimizer_name="policy", + optimizer=optim_actor, + params=params_actor, + lr_or_lr_schedule=config.actor_lr, + ) + + # Define the optimizer for the value function. + params_value = self.get_parameters(module.vf_encoder) + self.get_parameters( + module.vf + ) + optim_value = torch.optim.Adam(params_value, eps=1e-7) + self.register_optimizer( + module_id=module_id, + optimizer_name="value", + optimizer=optim_value, + params=params_value, + lr_or_lr_schedule=config.value_lr, + ) + + @override(TorchLearner) + def compute_loss_for_module( + self, + *, + module_id: ModuleID, + config: AlgorithmConfig, + batch: Dict, + fwd_out: Dict + ): + + # Get the module and hyperparameters. + module = self._module[module_id] + expectile = self.expectile[module_id] + temperature = self.temperature[module_id] + + # Get the action distribution for the actor loss. + action_train_dist_class = module.get_train_action_dist_cls() + action_train_dist = action_train_dist_class.from_logits( + fwd_out[Columns.ACTION_DIST_INPUTS] + ) + + # First, compute the value loss via the target Q-network and current observations. + value_loss = torch.mean( + self._expectile_loss( + fwd_out[QF_TARGET_PREDS] - fwd_out[Columns.VF_PREDS], expectile + ) + ) + + # Second, compute the actor loss using the target-Q network and values. + exp_advantages = torch.minimum( + torch.exp( + temperature * (fwd_out[QF_TARGET_PREDS] - fwd_out[Columns.VF_PREDS]) + ), + torch.Tensor([100.0]).to(self.device), + ) + # Note, we are using here the actions from the data sample. + action_logps = action_train_dist.logp(batch[Columns.ACTIONS]) + # Compute the actor loss. + actor_loss = -torch.mean(exp_advantages.detach() * action_logps) + + # Third, compute the critic loss. + target_critic = ( + batch[Columns.REWARDS] + + config.gamma + * (1 - batch[Columns.TERMINATEDS].float()) + * fwd_out[VF_PREDS_NEXT].detach() + ) + + critic_loss = torch.mean( + torch.nn.MSELoss(reduction="none")(target_critic, fwd_out[QF_PREDS]) + ) + + # If we have a twin-Q architecture, calculate the its loss, too. + if config.twin_q: + critic_twin_loss = ( + torch.mean( + torch.nn.MSELoss(reduction="none")( + target_critic, fwd_out[QF_TWIN_PREDS] + ) + ) + * 0.5 + ) + critic_loss *= 0.5 + + # Compute the total loss. + total_loss = value_loss + actor_loss + critic_loss + + # If we have a twin-Q architecture, add its loss. + if config.twin_q: + total_loss += critic_twin_loss + + # Log metrics. + self.metrics.log_dict( + { + POLICY_LOSS_KEY: actor_loss, + QF_LOSS_KEY: critic_loss, + }, + key=module_id, + window=1, # <- single items (should not be mean/ema-reduced over time). + ) + + # Log the losses also in the temporary containers for gradient computation. + self._temp_losses[(module_id, POLICY_LOSS_KEY)] = actor_loss + self._temp_losses[(module_id, QF_LOSS_KEY)] = critic_loss + self._temp_losses[(module_id, VF_LOSS)] = value_loss + + # If a twin-Q architecture is used add metrics and loss. + if config.twin_q: + self.metrics.log_value( + key=(module_id, QF_TWIN_LOSS_KEY), + value=critic_twin_loss, + window=1, # <- single items (should not be mean/ema-reduced over time). + ) + self._temp_losses[(module_id, QF_TWIN_LOSS_KEY)] = critic_twin_loss + + return total_loss + + @override(TorchLearner) + def compute_gradients( + self, loss_per_module: Dict[ModuleID, TensorType], **kwargs + ) -> ParamDict: + grads = {} + for module_id in set(loss_per_module.keys()) - {ALL_MODULES}: + # Loop through optimizers registered for this module. + for optim_name, optim in self.get_optimizers_for_module(module_id): + # Zero the gradients. Note, we need to reset the gradients b/c + # each component for a module operates on the same graph. + optim.zero_grad(set_to_none=True) + + # Compute the gradients for the component and module. + loss_tensor = self._temp_losses.pop((module_id, optim_name + "_loss")) + loss_tensor.backward(retain_graph=True) + # Store the gradients for the component and module. + grads.update( + { + pid: p.grad + for pid, p in self.filter_param_dict_for_optimizer( + self._params, optim + ).items() + } + ) + + # Make sure we updated on all loss terms. + assert not self._temp_losses + return grads + + def _expectile_loss(self, diff: TensorType, expectile: TensorType) -> TensorType: + """Computes the expectile loss. + + Args: + diff: A tensor containing a difference loss. + expectile: The expectile to use for the expectile loss. + + Returns: + The expectile loss of `diff` using `expectile`. + """ + weight = torch.where(diff > 0, expectile, 1 - expectile) + return weight * torch.pow(diff, 2) diff --git a/rllib/algorithms/registry.py b/rllib/algorithms/registry.py index 77f0581a69dc..c349489d165c 100644 --- a/rllib/algorithms/registry.py +++ b/rllib/algorithms/registry.py @@ -40,6 +40,12 @@ def _import_impala(): return impala.IMPALA, impala.IMPALA.get_default_config() +def _import_iql(): + import ray.rllib.algorithms.iql as iql + + return iql.IQL, iql.IQL.get_default_config() + + def _import_marwil(): import ray.rllib.algorithms.marwil as marwil @@ -65,6 +71,7 @@ def _import_sac(): "DQN": _import_dqn, "DreamerV3": _import_dreamerv3, "IMPALA": _import_impala, + "IQL": _import_iql, "MARWIL": _import_marwil, "PPO": _import_ppo, "SAC": _import_sac, @@ -78,6 +85,7 @@ def _import_sac(): "DQN": "DQN", "DreamerV3": "DreamerV3", "Impala": "IMPALA", + "IQL": "IQL", "IMPALA": "IMPALA", "MARWIL": "MARWIL", "PPO": "PPO", diff --git a/rllib/algorithms/sac/sac_learner.py b/rllib/algorithms/sac/sac_learner.py index 8046c4c07892..f4108943ad04 100644 --- a/rllib/algorithms/sac/sac_learner.py +++ b/rllib/algorithms/sac/sac_learner.py @@ -49,25 +49,10 @@ def build(self) -> None: # for the alpha already defined. super().build() - def get_target_entropy(module_id): - """Returns the target entropy to use for the loss. - - Args: - module_id: Module ID for which the target entropy should be - returned. - - Returns: - Target entropy. - """ - target_entropy = self.config.get_config_for_module(module_id).target_entropy - if target_entropy is None or target_entropy == "auto": - target_entropy = -np.prod( - self._module_spec.module_specs[module_id].action_space.shape - ) - return target_entropy - self.target_entropy: Dict[ModuleID, TensorType] = LambdaDefaultDict( - lambda module_id: self._get_tensor_variable(get_target_entropy(module_id)) + lambda module_id: self._get_tensor_variable( + self._get_target_entropy(module_id) + ) ) @override(Learner) @@ -80,3 +65,51 @@ def remove_module(self, module_id: ModuleID) -> None: super().remove_module(module_id) self.curr_log_alpha.pop(module_id, None) self.target_entropy.pop(module_id, None) + + @override(Learner) + def add_module( + self, + *, + module_id, + module_spec, + config_overrides=None, + new_should_module_be_updated=None + ): + # First call `super`'s `add_module` method. + super().add_module( + module_id=module_id, + module_spec=module_spec, + config_overrides=config_overrides, + new_should_module_be_updated=new_should_module_be_updated, + ) + # Now add the log alpha. + self.curr_log_alpha[module_id] = self._get_tensor_variable( + # Note, we want to train the temperature parameter. + [ + np.log( + self.config.get_config_for_module(module_id).initial_alpha + ).astype(np.float32) + ], + trainable=True, + ) + # Add also the target entropy for the new module. + self.target_entropy[module_id] = self._get_tensor_variable( + self._get_target_entropy(module_id) + ) + + def _get_target_entropy(self, module_id): + """Returns the target entropy to use for the loss. + + Args: + module_id: Module ID for which the target entropy should be + returned. + + Returns: + Target entropy. + """ + target_entropy = self.config.get_config_for_module(module_id).target_entropy + if target_entropy is None or target_entropy == "auto": + target_entropy = -np.prod( + self._module_spec.module_specs[module_id].action_space.shape + ) + return target_entropy diff --git a/rllib/algorithms/sac/torch/default_sac_torch_rl_module.py b/rllib/algorithms/sac/torch/default_sac_torch_rl_module.py index 3b62e949a9cf..0612dce7c391 100644 --- a/rllib/algorithms/sac/torch/default_sac_torch_rl_module.py +++ b/rllib/algorithms/sac/torch/default_sac_torch_rl_module.py @@ -54,11 +54,8 @@ def _forward_exploration(self, batch: Dict, **kwargs) -> Dict[str, Any]: @override(RLModule) def _forward_train(self, batch: Dict) -> Dict[str, Any]: - if self.inference_only: - raise RuntimeError( - "Trying to train a module that is not a learner module. Set the " - "flag `inference_only=False` when building the module." - ) + # Call the `super`'s `forward_train` + super()._forward_train(batch) if isinstance(self.action_space, gym.spaces.Discrete): return self._forward_train_discrete(batch) elif isinstance(self.action_space, gym.spaces.Box): diff --git a/rllib/tuned_examples/bc/cartpole_bc.py b/rllib/tuned_examples/bc/cartpole_bc.py index 327dbb32fb44..0c0b3630a642 100644 --- a/rllib/tuned_examples/bc/cartpole_bc.py +++ b/rllib/tuned_examples/bc/cartpole_bc.py @@ -51,7 +51,7 @@ # The number of iterations to be run per learner when in multi-learner # mode in a single RLlib training iteration. Leave this to `None` to # run an entire epoch on the dataset during a single RLlib training - # iteration. For single-learner mode, 1 is the only option. + # iteration. dataset_num_iters_per_learner=5, ) .training( diff --git a/rllib/tuned_examples/bc/cartpole_bc_with_offline_evaluation.py b/rllib/tuned_examples/bc/cartpole_bc_with_offline_evaluation.py index 326f7712936b..25a6eec32b1e 100644 --- a/rllib/tuned_examples/bc/cartpole_bc_with_offline_evaluation.py +++ b/rllib/tuned_examples/bc/cartpole_bc_with_offline_evaluation.py @@ -79,7 +79,7 @@ # The number of iterations to be run per learner when in multi-learner # mode in a single RLlib training iteration. Leave this to `None` to # run an entire epoch on the dataset during a single RLlib training - # iteration. For single-learner mode, 1 is the only option. + # iteration. dataset_num_iters_per_learner=5, ) .training( diff --git a/rllib/tuned_examples/cql/pendulum_cql.py b/rllib/tuned_examples/cql/pendulum_cql.py index 74526cd23153..391e7a7376d0 100644 --- a/rllib/tuned_examples/cql/pendulum_cql.py +++ b/rllib/tuned_examples/cql/pendulum_cql.py @@ -44,7 +44,7 @@ # The number of iterations to be run per learner when in multi-learner # mode in a single RLlib training iteration. Leave this to `None` to # run an entire epoch on the dataset during a single RLlib training - # iteration. For single-learner mode 1 is the only option. + # iteration. dataset_num_iters_per_learner=5, # TODO (sven): Has this any influence in the connectors? actions_in_input_normalized=True, diff --git a/rllib/tuned_examples/iql/pendulum_iql.py b/rllib/tuned_examples/iql/pendulum_iql.py new file mode 100644 index 000000000000..cb56bb6faed5 --- /dev/null +++ b/rllib/tuned_examples/iql/pendulum_iql.py @@ -0,0 +1,90 @@ +from pathlib import Path + +from ray.tune.result import TRAINING_ITERATION +from ray.rllib.algorithms.iql.iql import IQLConfig +from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig +from ray.rllib.utils.metrics import ( + ENV_RUNNER_RESULTS, + EPISODE_RETURN_MEAN, + EVALUATION_RESULTS, +) +from ray.rllib.utils.test_utils import ( + add_rllib_example_script_args, + run_rllib_example_script_experiment, +) + + +parser = add_rllib_example_script_args() +# Use `parser` to add your own custom command line options to this script +# and (if needed) use their values to set up `config` below. +args = parser.parse_args() + +assert ( + args.env == "Pendulum-v1" or args.env is None +), "This tuned example works only with `Pendulum-v1`." + +# Define the data paths. +data_path = "tests/data/pendulum/pendulum-v1_enormous" +base_path = Path(__file__).parents[2] +print(f"base_path={base_path}") +data_path = "local://" / base_path / data_path +print(f"data_path={data_path}") + +# Define the IQL config. +config = ( + IQLConfig() + .environment(env="Pendulum-v1") + .evaluation( + evaluation_interval=3, + evaluation_num_env_runners=1, + evaluation_duration=5, + evaluation_parallel_to_training=True, + ) + # Note, the `input_` argument is the major argument for the + # new offline API. Via the `input_read_method_kwargs` the + # arguments for the `ray.data.Dataset` read method can be + # configured. The read method needs at least as many blocks + # as remote learners. + .offline_data( + input_=[data_path.as_posix()], + # Concurrency defines the number of processes that run the + # `map_batches` transformations. This should be aligned with the + # 'prefetch_batches' argument in 'iter_batches_kwargs'. + map_batches_kwargs={"concurrency": 2, "num_cpus": 2}, + # This data set is small so do not prefetch too many batches and use no + # local shuffle. + iter_batches_kwargs={ + "prefetch_batches": 1, + }, + # The number of iterations to be run per learner when in multi-learner + # mode in a single RLlib training iteration. Leave this to `None` to + # run an entire epoch on the dataset during a single RLlib training + # iteration. + dataset_num_iters_per_learner=5, + ) + .training( + # To increase learning speed with multiple learners, + # increase the learning rates correspondingly. + actor_lr=2.59e-4 * (args.num_learners or 1) ** 0.5, + critic_lr=2.14e-4 * (args.num_learners or 1) ** 0.5, + value_lr=3.7e-5 * (args.num_learners or 1) ** 0.5, + # Smooth Polyak-averaging for the target network. + tau=6e-4, + # Update the target network each training iteration. + target_network_update_freq=1, + train_batch_size_per_learner=1024, + ) + .rl_module( + model_config=DefaultModelConfig( + fcnet_activation="relu", + ), + ) +) + +stop = { + f"{EVALUATION_RESULTS}/{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": -200.0, + TRAINING_ITERATION: 1250, +} + +if __name__ == "__main__": + run_rllib_example_script_experiment(config, args, stop=stop) diff --git a/rllib/tuned_examples/marwil/cartpole_marwil.py b/rllib/tuned_examples/marwil/cartpole_marwil.py index d31836b93960..c758bae0f238 100644 --- a/rllib/tuned_examples/marwil/cartpole_marwil.py +++ b/rllib/tuned_examples/marwil/cartpole_marwil.py @@ -57,7 +57,7 @@ # The number of iterations to be run per learner when in multi-learner # mode in a single RLlib training iteration. Leave this to `None` to # run an entire epoch on the dataset during a single RLlib training - # iteration. For single-learner mode 1 is the only option. + # iteration. dataset_num_iters_per_learner=5, ) .training( From 7a307de253884e9d7de1370dbfda08a2514d5c16 Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Wed, 13 Aug 2025 08:09:45 -0700 Subject: [PATCH 074/634] [core] Remove raylet ip address (#55492) Removing raylet_ip_address as it was never used (so it defaulted to node_ip_address) and is only use in private APIs. Originally introduced here: https://github.com/ray-project/ray/pull/7985/files --------- Signed-off-by: joshlee --- cpp/src/ray/util/process_helper.cc | 1 - python/ray/_private/node.py | 27 +++---------------- python/ray/_private/parameter.py | 4 --- python/ray/_private/worker.py | 1 - python/ray/_private/workers/default_worker.py | 4 --- python/ray/_raylet.pyx | 3 +-- .../modules/reporter/tests/test_reporter.py | 4 +-- python/ray/dashboard/tests/test_dashboard.py | 2 +- python/ray/includes/libcoreworker.pxd | 1 - python/ray/tests/test_metrics_agent.py | 4 +-- .../many_nodes_tests/dashboard_test.py | 2 +- src/ray/core_worker/core_worker_options.h | 3 --- .../java/io_ray_runtime_RayNativeRuntime.cc | 1 - src/ray/core_worker/test/core_worker_test.cc | 1 - 14 files changed, 10 insertions(+), 48 deletions(-) diff --git a/cpp/src/ray/util/process_helper.cc b/cpp/src/ray/util/process_helper.cc index 25c691894d78..63c75ffaf24f 100644 --- a/cpp/src/ray/util/process_helper.cc +++ b/cpp/src/ray/util/process_helper.cc @@ -150,7 +150,6 @@ void ProcessHelper::RayStart(CoreWorkerOptions::TaskExecutionCallback callback) options.install_failure_signal_handler = true; options.node_ip_address = node_ip; options.node_manager_port = ConfigInternal::Instance().node_manager_port; - options.raylet_ip_address = node_ip; options.driver_name = "cpp_worker"; options.metrics_agent_port = -1; options.task_execution_callback = callback; diff --git a/python/ray/_private/node.py b/python/ray/_private/node.py index 3465f995529b..74e13ae883f3 100644 --- a/python/ray/_private/node.py +++ b/python/ray/_private/node.py @@ -211,28 +211,13 @@ def __init__( node_ip_address = ray.util.get_node_ip_address() assert node_ip_address is not None - ray_params.update_if_absent( - node_ip_address=node_ip_address, raylet_ip_address=node_ip_address - ) + ray_params.update_if_absent(node_ip_address=node_ip_address) self._node_ip_address = node_ip_address if not connect_only: ray._private.services.write_node_ip_address( self.get_session_dir_path(), node_ip_address ) - if ray_params.raylet_ip_address: - raylet_ip_address = ray_params.raylet_ip_address - else: - raylet_ip_address = node_ip_address - - if raylet_ip_address != node_ip_address and (not connect_only or head): - raise ValueError( - "The raylet IP address should only be different than the node " - "IP address when connecting to an existing raylet; i.e., when " - "head=False and connect_only=True." - ) - self._raylet_ip_address = raylet_ip_address - self._object_spilling_config = self._get_object_spilling_config() logger.debug( f"Starting node with object spilling config: {self._object_spilling_config}" @@ -272,7 +257,7 @@ def __init__( # from Redis or GCS. node_info = ray._private.services.get_node_to_connect_for_driver( self.gcs_address, - self._raylet_ip_address, + self._node_ip_address, ) self._plasma_store_socket_name = node_info["object_store_socket_name"] self._raylet_socket_name = node_info["raylet_socket_name"] @@ -561,11 +546,6 @@ def node_ip_address(self): """Get the IP address of this node.""" return self._node_ip_address - @property - def raylet_ip_address(self): - """Get the IP address of the raylet that this node connects to.""" - return self._raylet_ip_address - @property def address(self): """Get the address for bootstrapping, e.g. the address to pass to @@ -633,7 +613,7 @@ def runtime_env_agent_port(self): @property def runtime_env_agent_address(self): """Get the address that exposes runtime env agent as http""" - return f"http://{build_address(self._raylet_ip_address, self._runtime_env_agent_port)}" + return f"http://{build_address(self._node_ip_address, self._runtime_env_agent_port)}" @property def dashboard_agent_listen_port(self): @@ -653,7 +633,6 @@ def address_info(self): """Get a dictionary of addresses.""" return { "node_ip_address": self._node_ip_address, - "raylet_ip_address": self._raylet_ip_address, "redis_address": self.redis_address, "object_store_address": self._plasma_store_socket_name, "raylet_socket_name": self._raylet_socket_name, diff --git a/python/ray/_private/parameter.py b/python/ray/_private/parameter.py index eb78bd86f510..60ab5ab9aa81 100644 --- a/python/ray/_private/parameter.py +++ b/python/ray/_private/parameter.py @@ -36,8 +36,6 @@ class RayParams: node_manager_port: The port to use for the node manager. gcs_server_port: The port to use for the GCS server. node_ip_address: The IP address of the node that we are on. - raylet_ip_address: The IP address of the raylet that this node - connects to. min_worker_port: The lowest port number that workers will bind on. If not set or set to 0, random ports will be chosen. max_worker_port: The highest port number that workers will bind @@ -138,7 +136,6 @@ def __init__( gcs_server_port: Optional[int] = None, node_ip_address: Optional[str] = None, node_name: Optional[str] = None, - raylet_ip_address: Optional[str] = None, min_worker_port: Optional[int] = None, max_worker_port: Optional[int] = None, worker_port_list: Optional[List[int]] = None, @@ -196,7 +193,6 @@ def __init__( self.gcs_server_port = gcs_server_port self.node_ip_address = node_ip_address self.node_name = node_name - self.raylet_ip_address = raylet_ip_address self.min_worker_port = min_worker_port self.max_worker_port = max_worker_port self.worker_port_list = worker_port_list diff --git a/python/ray/_private/worker.py b/python/ray/_private/worker.py index e3eeee3f84ad..dacaa9e2ff7f 100644 --- a/python/ray/_private/worker.py +++ b/python/ray/_private/worker.py @@ -2584,7 +2584,6 @@ def connect( logs_dir, node.node_ip_address, node.node_manager_port, - node.raylet_ip_address, (mode == LOCAL_MODE), driver_name, serialized_job_config, diff --git a/python/ray/_private/workers/default_worker.py b/python/ray/_private/workers/default_worker.py index 9859d9d15f7e..cb6bce0043f6 100644 --- a/python/ray/_private/workers/default_worker.py +++ b/python/ray/_private/workers/default_worker.py @@ -220,12 +220,8 @@ # for asyncio try_install_uvloop() - raylet_ip_address = args.raylet_ip_address - if raylet_ip_address is None: - raylet_ip_address = args.node_ip_address ray_params = RayParams( node_ip_address=args.node_ip_address, - raylet_ip_address=raylet_ip_address, node_manager_port=args.node_manager_port, redis_address=args.redis_address, redis_username=args.redis_username, diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index e51d12958700..db0abd516ed7 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -2973,7 +2973,7 @@ cdef class CoreWorker: def __cinit__(self, worker_type, store_socket, raylet_socket, JobID job_id, GcsClientOptions gcs_options, log_dir, - node_ip_address, node_manager_port, raylet_ip_address, + node_ip_address, node_manager_port, local_mode, driver_name, serialized_job_config, metrics_agent_port, runtime_env_hash, startup_token, session_name, cluster_id, entrypoint, @@ -3007,7 +3007,6 @@ cdef class CoreWorker: options.interactive = hasattr(sys, "ps1") options.node_ip_address = node_ip_address.encode("utf-8") options.node_manager_port = node_manager_port - options.raylet_ip_address = raylet_ip_address.encode("utf-8") options.driver_name = driver_name options.initialize_thread_callback = initialize_pygilstate_for_thread options.task_execution_callback = task_execution_handler diff --git a/python/ray/dashboard/modules/reporter/tests/test_reporter.py b/python/ray/dashboard/modules/reporter/tests/test_reporter.py index 28820f370965..1c4ee8a0f8d0 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_reporter.py +++ b/python/ray/dashboard/modules/reporter/tests/test_reporter.py @@ -217,7 +217,7 @@ def test_prometheus_physical_stats_record( ): addresses = ray.init(include_dashboard=True, num_cpus=1) metrics_export_port = addresses["metrics_export_port"] - addr = addresses["raylet_ip_address"] + addr = addresses["node_ip_address"] prom_addresses = [build_address(addr, metrics_export_port)] def test_case_stats_exist(): @@ -284,7 +284,7 @@ def test_case_ip_correct(): def test_prometheus_export_worker_and_memory_stats(enable_test_module, shutdown_only): addresses = ray.init(include_dashboard=True, num_cpus=1) metrics_export_port = addresses["metrics_export_port"] - addr = addresses["raylet_ip_address"] + addr = addresses["node_ip_address"] prom_addresses = [build_address(addr, metrics_export_port)] @ray.remote diff --git a/python/ray/dashboard/tests/test_dashboard.py b/python/ray/dashboard/tests/test_dashboard.py index 43afbe941b26..1f7036651465 100644 --- a/python/ray/dashboard/tests/test_dashboard.py +++ b/python/ray/dashboard/tests/test_dashboard.py @@ -1315,7 +1315,7 @@ async def make_blocking_call(): await asyncio.gather(*tasks) # Fetch the metrics from the dashboard. - addr = ray_context["raylet_ip_address"] + addr = ray_context["node_ip_address"] prom_addresses = [build_address(addr, dashboard_consts.DASHBOARD_METRIC_PORT)] def check_lag_metrics(): diff --git a/python/ray/includes/libcoreworker.pxd b/python/ray/includes/libcoreworker.pxd index 0d7b4fe68559..88f7252a3a8a 100644 --- a/python/ray/includes/libcoreworker.pxd +++ b/python/ray/includes/libcoreworker.pxd @@ -379,7 +379,6 @@ cdef extern from "ray/core_worker/core_worker.h" nogil: c_bool interactive c_string node_ip_address int node_manager_port - c_string raylet_ip_address c_string driver_name (CRayStatus( const CAddress &caller_address, diff --git a/python/ray/tests/test_metrics_agent.py b/python/ray/tests/test_metrics_agent.py index b7494f52dc43..2021c97ef36c 100644 --- a/python/ray/tests/test_metrics_agent.py +++ b/python/ray/tests/test_metrics_agent.py @@ -422,7 +422,7 @@ def test_metrics_export_node_metrics(shutdown_only): # Verify node metrics are available. addr = ray.init() dashboard_export_addr = build_address( - addr["raylet_ip_address"], DASHBOARD_METRIC_PORT + addr["node_ip_address"], DASHBOARD_METRIC_PORT ) def verify_node_metrics(): @@ -494,7 +494,7 @@ def test_metrics_export_event_aggregator_agent( httpserver.expect_request("/", method="POST").respond_with_data("", status=200) metrics_export_port = cluster.head_node.metrics_export_port - addr = cluster.head_node.raylet_ip_address + addr = cluster.head_node.node_ip_address prom_addresses = [build_address(addr, metrics_export_port)] def test_case_stats_exist(): diff --git a/release/benchmarks/distributed/many_nodes_tests/dashboard_test.py b/release/benchmarks/distributed/many_nodes_tests/dashboard_test.py index 98323ed95c45..03a1fb287d1a 100644 --- a/release/benchmarks/distributed/many_nodes_tests/dashboard_test.py +++ b/release/benchmarks/distributed/many_nodes_tests/dashboard_test.py @@ -126,7 +126,7 @@ def get_result(self): # Get the memory usage. dashboard_export_addr = build_address( - self.addr["raylet_ip_address"], DASHBOARD_METRIC_PORT + self.addr["node_ip_address"], DASHBOARD_METRIC_PORT ) metrics = fetch_prometheus_metrics([dashboard_export_addr]) memories = [] diff --git a/src/ray/core_worker/core_worker_options.h b/src/ray/core_worker/core_worker_options.h index 72c108d759bb..4eee61e4d75a 100644 --- a/src/ray/core_worker/core_worker_options.h +++ b/src/ray/core_worker/core_worker_options.h @@ -83,7 +83,6 @@ struct CoreWorkerOptions { interactive(false), node_ip_address(""), node_manager_port(0), - raylet_ip_address(""), driver_name(""), task_execution_callback(nullptr), free_actor_object_callback(nullptr), @@ -135,8 +134,6 @@ struct CoreWorkerOptions { std::string node_ip_address; /// Port of the local raylet. int node_manager_port; - /// IP address of the raylet. - std::string raylet_ip_address; /// The name of the driver. std::string driver_name; /// Application-language worker callback to execute tasks. diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc b/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc index 40cac1dc9ecc..739bc6e5f63e 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc @@ -299,7 +299,6 @@ Java_io_ray_runtime_RayNativeRuntime_nativeInitialize(JNIEnv *env, options.install_failure_signal_handler = false; options.node_ip_address = JavaStringToNativeString(env, nodeIpAddress); options.node_manager_port = static_cast(nodeManagerPort); - options.raylet_ip_address = JavaStringToNativeString(env, nodeIpAddress); options.driver_name = JavaStringToNativeString(env, driverName); options.task_execution_callback = task_execution_callback; options.gc_collect = gc_collect; diff --git a/src/ray/core_worker/test/core_worker_test.cc b/src/ray/core_worker/test/core_worker_test.cc index d364a6b006a5..bf45ffa279ef 100644 --- a/src/ray/core_worker/test/core_worker_test.cc +++ b/src/ray/core_worker/test/core_worker_test.cc @@ -58,7 +58,6 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { options.worker_type = WorkerType::WORKER; options.language = Language::PYTHON; options.node_ip_address = "127.0.0.1"; - options.raylet_ip_address = "127.0.0.1"; options.task_execution_callback = [](const rpc::Address &caller_address, TaskType task_type, From dda42b2d97768dbebdbaf766a7ed2e2e2372cc8b Mon Sep 17 00:00:00 2001 From: William Lin Date: Wed, 13 Aug 2025 08:36:19 -0700 Subject: [PATCH 075/634] [core] Add return type to ActorClass.options (#55563) Currently the following pattern throws many lint errors as `ActorDemoRay.options(name="demo_ray")` returns an instance of `ActorOptionWrapper` which messes with the IDE's static type checker: ```python import ray from ray import ObjectRef from ray.actor import ActorProxy, ActorClass class DemoRay: def __init__(self, init: int): self.init = init @ray.method def calculate(self, v1: int, v2: int) -> int: return self.init + v1 + v2 ActorDemoRay: ActorClass[DemoRay] = ray.remote(DemoRay) def main(): p: ActorProxy[DemoRay] = ActorDemoRay.options(name="demo_ray").remote(1) actor: ActorProxy[DemoRay] = ray.get_actor("demo_ray") a = actor.calculate.remote(1, 2) print(ray.get(a)) return if __name__ == "__main__": main() ``` This PR changes ActorClass[T].options(...) to return a new instance of ActorClass[T] instead, allow IDEs to correct infer the type of subsequent `.remote(...)` calls https://github.com/ray-project/ray/issues/54149 --------- Signed-off-by: will.lin --- python/ray/actor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/actor.py b/python/ray/actor.py index f2a00efe5deb..fba4a4d39bb2 100644 --- a/python/ray/actor.py +++ b/python/ray/actor.py @@ -1296,7 +1296,7 @@ def remote(self, *args, **kwargs) -> ActorProxy[T]: """ return self._remote(args=args, kwargs=kwargs, **self._default_options) - def options(self, **actor_options): + def options(self, **actor_options) -> "ActorClass[T]": """Configures and overrides the actor instantiation parameters. The arguments are the same as those that can be passed From a24defd4c4773879a834762ba414d3c0cea9b1e9 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Wed, 13 Aug 2025 10:51:07 -0700 Subject: [PATCH 076/634] [release test] remove release image build step from postmerge (#55564) they should be always building from release test pipeline directly we used to run release tests on postmerge; we are no longer doing it any more. also add oss tag for those steps. Signed-off-by: Lonnie Liu --- .buildkite/release/build.rayci.yml | 5 ++--- .buildkite/releasebuild.rayci.yml | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) delete mode 120000 .buildkite/releasebuild.rayci.yml diff --git a/.buildkite/release/build.rayci.yml b/.buildkite/release/build.rayci.yml index f7fdc95375a9..f734ac593361 100644 --- a/.buildkite/release/build.rayci.yml +++ b/.buildkite/release/build.rayci.yml @@ -1,7 +1,8 @@ group: release build +tags: + - oss steps: - label: ":tapioca: build: anyscale py{{matrix.python}}-{{matrix.platform}} docker" - tags: skip-on-premerge key: anyscalebuild instance_type: release-medium commands: @@ -26,7 +27,6 @@ steps: - cpu - label: ":tapioca: build: anyscale-llm py{{matrix}} docker" - tags: skip-on-premerge key: anyscalellmbuild instance_type: release-medium commands: @@ -40,7 +40,6 @@ steps: - "3.11" - label: ":tapioca: build: anyscale-ml py{{matrix}} docker" - tags: skip-on-premerge key: anyscalemlbuild instance_type: release-medium commands: diff --git a/.buildkite/releasebuild.rayci.yml b/.buildkite/releasebuild.rayci.yml deleted file mode 120000 index d0497f6db89d..000000000000 --- a/.buildkite/releasebuild.rayci.yml +++ /dev/null @@ -1 +0,0 @@ -release/build.rayci.yml \ No newline at end of file From 3c1314afb82128f30e5a445462c7277717e62863 Mon Sep 17 00:00:00 2001 From: William Lin Date: Wed, 13 Aug 2025 10:55:47 -0700 Subject: [PATCH 077/634] [docs] Add documentation for using type hints in Ray Core (#55013) ## Why are these changes needed? ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: will.lin Signed-off-by: Richard Liaw Co-authored-by: Richard Liaw --- doc/source/ray-core/advanced-topics.rst | 1 + doc/source/ray-core/type-hint.md | 90 +++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 doc/source/ray-core/type-hint.md diff --git a/doc/source/ray-core/advanced-topics.rst b/doc/source/ray-core/advanced-topics.rst index 7d1344856507..a1e310b96ca5 100644 --- a/doc/source/ray-core/advanced-topics.rst +++ b/doc/source/ray-core/advanced-topics.rst @@ -7,6 +7,7 @@ This section covers extended topics on how to use Ray. :maxdepth: -1 tips-for-first-time + type-hint starting-ray ray-generator namespaces diff --git a/doc/source/ray-core/type-hint.md b/doc/source/ray-core/type-hint.md new file mode 100644 index 000000000000..536271860a61 --- /dev/null +++ b/doc/source/ray-core/type-hint.md @@ -0,0 +1,90 @@ +# Type hints in Ray + +As of Ray 2.48, Ray provides comprehensive support for Python type hints with both remote functions and actors. This enables better IDE support, static type checking, and improved code maintainability in distributed Ray applications. + +## Overview + +In most cases, Ray applications can use type hints without any modifications to existing code. Ray automatically handles type inference for standard remote functions and basic actor usage patterns. For example, remote functions support standard Python type annotations without additional configuration. The `@ray.remote` decorator preserves the original function signature and type information. + +```python +import ray + +@ray.remote +def add_numbers(x: int, y: int) -> int: + return x + y + +# Type hints work seamlessly with remote function calls +a = add_numbers.remote(5, 3) +print(ray.get(a)) +``` + +However, certain patterns, especially when working with actors, require specific approaches to ensure proper type annotation. + +## Pattern 1: Use `ray.remote` as a function to build an actor + +Use the `ray.remote` function directly to create an actor class, instead of using the `@ray.remote` decorator. This will preserve the original class type and allow type inference to work correctly. For example, in this case, the original class type is `DemoRay`, and the actor class type is `ActorClass[DemoRay]`. + +```python +import ray +from ray.actor import ActorClass + +class DemoRay: + def __init__(self, init: int): + self.init = init + + @ray.method + def calculate(self, v1: int, v2: int) -> int: + return self.init + v1 + v2 + +ActorDemoRay: ActorClass[DemoRay] = ray.remote(DemoRay) +# DemoRay is the original class type, ActorDemoRay is the ActorClass[DemoRay] type +``` + +After creating the `ActorClass[DemoRay]` type, we can use it to instantiate an actor by calling `ActorDemoRay.remote(1)`. It returns an `ActorProxy[DemoRay]` type, which represents an actor handle. + +This handle will provide type hints for the actor methods, including their arguments and return types. + +```python + +actor: ActorProxy[DemoRay] = ActorDemoRay.remote(1) + +def func(actor: ActorProxy[DemoRay]) -> int: + b: ObjectRef[int] = actor.calculate.remote(1, 2) + return ray.get(b) + +a = func.remote() +print(ray.get(a)) +``` + +**Why do we need to do this?** + +In Ray, the `@ray.remote` decorator indicates that instances of the class `T` are actors, with each actor running in its own Python process. However, the `@ray.remote` decorator will transform the class `T` into a `ActorClass[T]` type, which is not the original class type. + +Unfortunately, IDE and static type checkers will not be able to infer the original type `T` of the `ActorClass[T]`. To solve this problem, using `ray.remote(T)` will explicitly return a new generic class `ActorClass[T]` type while preserving the original class type. + +## Pattern 2: Use `@ray.method` decorator for remote methods + +Add the `@ray.method` decorator to the actor methods in order to obtain type hints for the remote methods of the actor through `ActorProxy[T]` type, including their arguments and return types. + +```python +from ray.actor import ActorClass, ActorProxy + +class DemoRay: + def __init__(self, init: int): + self.init = init + + @ray.method + def calculate(self, v1: int, v2: int) -> int: + return self.init + v1 + v2 + +ActorDemoRay: ActorClass[DemoRay] = ray.remote(DemoRay) +actor: ActorProxy[DemoRay] = ActorDemoRay.remote(1) +# IDEs will be able to correctly list the remote methods of the actor +# and provide type hints for the arguments and return values of the remote methods +a: ObjectRef[int] = actor.calculate.remote(1, 2) +print(ray.get(a)) +``` + +:::{note} +We would love to make the typing of remote methods work without `@ray.method` decorator. If any community member has an idea, we welcome PRs. +::: From 6d318ce84ddeacf67dc0c66f6e2fb6f6a8fef2e4 Mon Sep 17 00:00:00 2001 From: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Date: Wed, 13 Aug 2025 10:57:54 -0700 Subject: [PATCH 078/634] [Serve.llm] Add missing data_parallel/__init__.py (#55573) Signed-off-by: Rui Qiao --- .../ray/llm/_internal/serve/deployments/data_parallel/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 python/ray/llm/_internal/serve/deployments/data_parallel/__init__.py diff --git a/python/ray/llm/_internal/serve/deployments/data_parallel/__init__.py b/python/ray/llm/_internal/serve/deployments/data_parallel/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From 64feab4b01583023cec89bc2d199b0ff0de4c3cd Mon Sep 17 00:00:00 2001 From: Ryan O'Leary <113500783+ryanaoleary@users.noreply.github.com> Date: Wed, 13 Aug 2025 18:09:39 +0000 Subject: [PATCH 079/634] [Train] Implement a JaxTrainer to support SPMD with TPUs (#55207) This PR builds off previous efforts to add a `JaxTrainer` and the [ray-tpu package](https://github.com/AI-Hypercomputer/ray-tpu/tree/main) to implement support for a `JaxTrainer` in RayTrain that supports SPMD workloads with TPUs. Support for more types of workloads (i.e. better support for CPU and GPU) can be added incrementally. In order to support SPMD locality-aware scheduling at the TPU slice level, we alter the `WorkerGroup` construction in V2 Ray Train to optionally accept multiple placement groups specs to apply to a range of workers. This enables us to reserve the "TPU head" using a placement group with label selectors, retrieve its unique `ray.io/tpu-slice-name`, and then schedule the remaining workers on that slice in a separate placement group. --------- Signed-off-by: Ryan O'Leary Signed-off-by: Andrew Sy Kim Co-authored-by: Andrew Sy Kim --- python/ray/_private/accelerators/tpu.py | 86 ++++++++++ .../ray/_private/resource_and_label_spec.py | 8 +- python/ray/tests/accelerators/test_tpu.py | 72 ++++++++ python/ray/train/v2/BUILD | 16 ++ .../train/v2/_internal/callbacks/__init__.py | 2 + .../callbacks/tpu_reservation_callback.py | 45 +++++ .../train/v2/_internal/execution/callback.py | 23 +++ .../execution/controller/controller.py | 16 ++ .../execution/worker_group/worker_group.py | 10 ++ python/ray/train/v2/api/config.py | 58 ++++++- .../ray/train/v2/api/data_parallel_trainer.py | 3 + python/ray/train/v2/jax/__init__.py | 15 ++ python/ray/train/v2/jax/config.py | 59 +++++++ python/ray/train/v2/jax/jax_trainer.py | 162 ++++++++++++++++++ python/ray/train/v2/tests/test_jax_trainer.py | 137 +++++++++++++++ 15 files changed, 706 insertions(+), 6 deletions(-) create mode 100644 python/ray/train/v2/_internal/callbacks/tpu_reservation_callback.py create mode 100644 python/ray/train/v2/jax/__init__.py create mode 100644 python/ray/train/v2/jax/config.py create mode 100644 python/ray/train/v2/jax/jax_trainer.py create mode 100644 python/ray/train/v2/tests/test_jax_trainer.py diff --git a/python/ray/_private/accelerators/tpu.py b/python/ray/_private/accelerators/tpu.py index c6df2c858779..83da22475879 100644 --- a/python/ray/_private/accelerators/tpu.py +++ b/python/ray/_private/accelerators/tpu.py @@ -9,6 +9,7 @@ import ray from ray._private.accelerators.accelerator import AcceleratorManager +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy logger = logging.getLogger(__name__) @@ -110,6 +111,91 @@ def get_tpu_cores_per_chip(accelerator_type: str) -> int: return DEFAULT_TPU_NUM_CORES_PER_CHIP +def infer_tpu_pod_type_from_topology( + topology: str, accelerator_type: str +) -> Optional[str]: + """Infer the TPU pod type (e.g. v4-32) from topology and accelerator type.""" + try: + num_chips = 1 + for value in topology.strip().lower().split("x"): + num_chips *= int(value) + generation = accelerator_type.lower().replace("tpu-", "") + return f"{generation}-{num_chips}" + except Exception as e: + logger.warning( + f"Failed to infer pod type from topology {topology} and type {accelerator_type}: {e}" + ) + return None + + +def fetch_tpu_slice_name_from_pg(pg): + @ray.remote(num_cpus=0) + def _get_tpu_slice_name(): + return TPUAcceleratorManager.get_current_node_tpu_name() + + tpu_name_ref = _get_tpu_slice_name.options( + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=pg, placement_group_bundle_index=0 + ) + ).remote() + + return ray.get(tpu_name_ref) + + +def reserve_tpu_slice( + topology: str, + accelerator_type: str, +) -> Optional[str]: + """Reserves a TPU slice using its head resource and returns the slice name. + This enables gang scheduling of training workers with multi-host TPUs. + This is used by JaxTrainer with TPUs in Ray Train. + + Args: + topology: The TPU topology string (e.g. "2x2x2"). + accelerator_type: The accelerator type of the node (e.g. "TPU-V4"). + + Returns: + A string representing a unique TPU slice name. + """ + pod_type = infer_tpu_pod_type_from_topology(topology, accelerator_type) + if pod_type is None: + return None + + # Reserve a slice by creating a placement group on the TPU head. + head_label_selector = { + "ray.io/tpu-worker-id": "0", + "ray.io/tpu-pod-type": pod_type, + } + head_placement_group = ray.util.placement_group( + bundles=[{f"TPU-{pod_type}-head": 1}], + bundle_label_selector=[head_label_selector], + ) + + logger.debug("Waiting to reserve multi-host slice head.") + timeout = 100 + ready, _ = ray.wait([head_placement_group.ready()], timeout=timeout) + + if not ready: + raise TimeoutError( + "Failed to reserve TPU head for slice with shape: {}. " + "Ensure your cluster has sufficient resources. Requesting TPU " + "head node with labels: {}. Current resources: {}".format( + pod_type, head_label_selector, ray.available_resources() + ) + ) + + # Retrieve the unique slice ID. + slice_name = fetch_tpu_slice_name_from_pg(head_placement_group) + if slice_name is None: + raise RuntimeError( + "Failed to retrieve TPU slice name after reserving head placement group. " + "Ensure that TPU slice metadata is available and correctly configured on multi-host nodes." + ) + + # TODO: return both the slice name and reference to the PG reservation. + return slice_name + + class TPUAcceleratorManager(AcceleratorManager): """Google TPU accelerators.""" diff --git a/python/ray/_private/resource_and_label_spec.py b/python/ray/_private/resource_and_label_spec.py index 03c8efd0e119..d737b70961da 100644 --- a/python/ray/_private/resource_and_label_spec.py +++ b/python/ray/_private/resource_and_label_spec.py @@ -10,7 +10,6 @@ from ray._common.utils import RESOURCE_CONSTRAINT_PREFIX from ray._private import accelerators from ray._private.accelerators import AcceleratorManager -from ray._private.accelerators.tpu import TPUAcceleratorManager logger = logging.getLogger(__name__) @@ -292,10 +291,11 @@ def _get_default_labels( ray._raylet.RAY_NODE_ACCELERATOR_TYPE_KEY ] = accelerator_type - # Set TPU specific default labels to enable SPMD scheduling. - if isinstance(accelerator_manager, TPUAcceleratorManager): + # Set TPU specific default labels to enable multi-host scheduling. + if accelerator_manager.get_resource_name() == "TPU": tpu_labels = accelerator_manager.get_current_node_accelerator_labels() - default_labels.update(tpu_labels) + if tpu_labels: + default_labels.update(tpu_labels) return default_labels diff --git a/python/ray/tests/accelerators/test_tpu.py b/python/ray/tests/accelerators/test_tpu.py index e0d405f60efd..f13f27ffea80 100644 --- a/python/ray/tests/accelerators/test_tpu.py +++ b/python/ray/tests/accelerators/test_tpu.py @@ -8,6 +8,7 @@ import ray from ray._private.accelerators import TPUAcceleratorManager from ray._private.accelerators import tpu +from ray.tests.conftest import _ray_start_cluster @patch("glob.glob") @@ -353,5 +354,76 @@ def test_get_current_node_tpu_topology_from_metadata(): assert topology == "2x2x4" +@pytest.mark.parametrize( + "topology, accelerator_type, expected_pod_type", + [ + ("2x4", "TPU-V6E", "v6e-8"), + ("2x2x2", "TPU-V4", "v4-8"), + ("2x4x4", "TPU-V3", "v3-32"), + ("4x4", "TPU-V5P", "v5p-16"), + ("8x16", "TPU-V6E", "v6e-128"), + ("", "TPU-V3", None), + ("4x", "TPU-V3", None), + ], +) +def test_infer_tpu_pod_type_from_topology( + topology, accelerator_type, expected_pod_type +): + assert ( + tpu.infer_tpu_pod_type_from_topology(topology, accelerator_type) + == expected_pod_type + ) + + +@pytest.fixture +def ray_start_cpu(): + address_info = ray.init(num_cpus=1) + yield address_info + ray.shutdown() + + +@pytest.fixture +def ray_tpu_cluster(monkeypatch): + """Start a mock TPU Ray cluster.""" + with _ray_start_cluster() as cluster: + monkeypatch.setenv("TPU_NAME", "test-slice-0") + monkeypatch.setenv("TPU_WORKER_ID", "0") + monkeypatch.setenv("TPU_ACCELERATOR_TYPE", "v4-8") + monkeypatch.setenv("TPU_TOPOLOGY", "2x2x2") + + cluster.add_node( + num_cpus=2, + resources={"TPU": 4, "TPU-v4-8-head": 1}, + ) + monkeypatch.setenv("TPU_WORKER_ID", "1") + cluster.add_node( + num_cpus=2, + resources={"TPU": 4}, + ) + ray.init(address=cluster.address) + + yield cluster + ray.shutdown() + + +def test_fetch_tpu_slice_name_from_pg(ray_tpu_cluster): + """Tests that the slice name can be fetched from a PG.""" + tpu_head_pg = ray.util.placement_group(bundles=[{"TPU-v4-8-head": 1}]) + ray.get(tpu_head_pg.ready()) + + tpu_slice_name = "test-slice-0" + slice_name = tpu.fetch_tpu_slice_name_from_pg(tpu_head_pg) + assert slice_name == tpu_slice_name + + ray.util.remove_placement_group(tpu_head_pg) + + +def test_reserve_tpu_slice(ray_tpu_cluster): + """Tests that a TPU slice can be successfully reserved.""" + tpu_slice_name = "test-slice-0" + reserved_name = tpu.reserve_tpu_slice(topology="2x2x2", accelerator_type="TPU-V4") + assert reserved_name == tpu_slice_name + + if __name__ == "__main__": sys.exit(pytest.main(["-sv", __file__])) diff --git a/python/ray/train/v2/BUILD b/python/ray/train/v2/BUILD index 31bab21afc44..8b0128a3a15d 100644 --- a/python/ray/train/v2/BUILD +++ b/python/ray/train/v2/BUILD @@ -133,6 +133,22 @@ py_test( ], ) +py_test( + name = "test_jax_trainer", + size = "small", + srcs = ["tests/test_jax_trainer.py"], + env = {"RAY_TRAIN_V2_ENABLED": "1"}, + tags = [ + "exclusive", + "team:ml", + "train_v2", + ], + deps = [ + ":conftest", + "//:ray_lib", + ], +) + py_test( name = "test_lightgbm_trainer", size = "small", diff --git a/python/ray/train/v2/_internal/callbacks/__init__.py b/python/ray/train/v2/_internal/callbacks/__init__.py index 5c5b204acdcf..3db8d835fba3 100644 --- a/python/ray/train/v2/_internal/callbacks/__init__.py +++ b/python/ray/train/v2/_internal/callbacks/__init__.py @@ -2,6 +2,7 @@ from .backend_setup import BackendSetupCallback from .datasets import DatasetsSetupCallback from .state_manager import StateManagerCallback +from .tpu_reservation_callback import TPUReservationCallback from .working_dir_setup import WorkingDirectorySetupCallback __all__ = [ @@ -9,6 +10,7 @@ "BackendSetupCallback", "DatasetsSetupCallback", "StateManagerCallback", + "TPUReservationCallback", "WorkingDirectorySetupCallback", ] diff --git a/python/ray/train/v2/_internal/callbacks/tpu_reservation_callback.py b/python/ray/train/v2/_internal/callbacks/tpu_reservation_callback.py new file mode 100644 index 000000000000..acb7b70847ea --- /dev/null +++ b/python/ray/train/v2/_internal/callbacks/tpu_reservation_callback.py @@ -0,0 +1,45 @@ +from typing import Dict, Optional + +import ray +from ray._private.accelerators.tpu import reserve_tpu_slice +from ray.train.v2._internal.execution.callback import ControllerCallback +from ray.train.v2.api.config import ScalingConfig + + +class TPUReservationCallback(ControllerCallback): + """A callback to handle TPU slice reservation for multi-host training.""" + + def on_controller_start_worker_group( + self, *, scaling_config: ScalingConfig, num_workers: int + ) -> Optional[Dict[str, str]]: + """Reserves a multi-host TPU slice before the worker group starts. + + This hook is called by the TrainController. It checks if multi-host + TPUs are being used and, if so, reserves a slice. + + Args: + scaling_config: The scaling configuration for the run. + num_workers: The number of workers to be started. + + Returns: + A dictionary defining a `bundle_label_selector` to gang schedule + the worker group on the reserved TPU slice. + """ + bundle_label_selector = None + + if scaling_config.use_tpu and num_workers > 1: + assert scaling_config.accelerator_type is not None + assert scaling_config.topology is not None + + slice_name = reserve_tpu_slice( + topology=scaling_config.topology, + accelerator_type=scaling_config.accelerator_type, + ) + if not slice_name: + raise RuntimeError("Failed to reserve TPU slice.") + + bundle_label_selector = { + ray._raylet.RAY_NODE_TPU_SLICE_NAME_KEY: slice_name + } + + return bundle_label_selector diff --git a/python/ray/train/v2/_internal/execution/callback.py b/python/ray/train/v2/_internal/execution/callback.py index 50796a0700c8..f5cfd3584f79 100644 --- a/python/ray/train/v2/_internal/execution/callback.py +++ b/python/ray/train/v2/_internal/execution/callback.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional from ray.train.v2.api.callback import RayTrainCallback +from ray.train.v2.api.config import ScalingConfig from ray.train.v2.api.result import Result from ray.util.annotations import DeveloperAPI @@ -78,6 +79,28 @@ def after_controller_start(self, train_run_context: "TrainRunContext"): before the control loop starts executing.""" pass + # TODO(matthewdeng): Revisit this callback interface for better extensibility. + # This hook was added for the specific use case of setting a `bundle_label_selector` + # for new worker groups (e.g., for TPU reservations). The current interface is + # tightly coupled to this purpose and limits its reuse for other use-cases. + def on_controller_start_worker_group( + self, *, scaling_config: ScalingConfig, num_workers: int + ) -> Optional[Dict[str, str]]: + """Called by the TrainController before the worker group is started. + + This hook can be used to perform setup that modifies the worker group's + placement, such as reserving an accelerator slice. + + Args: + scaling_config: The scaling configuration for the run. + num_workers: The number of workers to be started. + + Returns: + An optional dictionary defining a `bundle_label_selector` + to gang schedule the worker group on the reserved TPU slice. + """ + return None + def before_controller_shutdown(self): """Called before `TrainController.run` exits, after the control loop has exited.""" diff --git a/python/ray/train/v2/_internal/execution/controller/controller.py b/python/ray/train/v2/_internal/execution/controller/controller.py index e2f916d140a4..59d3760bb503 100644 --- a/python/ray/train/v2/_internal/execution/controller/controller.py +++ b/python/ray/train/v2/_internal/execution/controller/controller.py @@ -280,12 +280,28 @@ def _start_worker_group( ControllerError if the worker group failed to start. """ placement_strategy = self._scaling_policy.scaling_config.placement_strategy + scaling_config = self._train_run_context.scaling_config + + # Check for `bundle_label_selector` to influence WorkerGroup scheduling. + bundle_label_selector = None + try: + for callback in self._controller_callbacks: + selector = callback.on_controller_start_worker_group( + scaling_config=scaling_config, num_workers=num_workers + ) + if selector: + bundle_label_selector = selector + break + except Exception as e: + return ControllerError(e) + worker_group_context = WorkerGroupContext( run_attempt_id=self._get_run_attempt_id(), train_fn_ref=self._train_fn_ref, num_workers=num_workers, resources_per_worker=resources_per_worker, placement_strategy=placement_strategy, + bundle_label_selector=bundle_label_selector, ) try: self._worker_group = self.worker_group_cls.create( diff --git a/python/ray/train/v2/_internal/execution/worker_group/worker_group.py b/python/ray/train/v2/_internal/execution/worker_group/worker_group.py index 8931145ecdbb..81087c0f91c7 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/worker_group.py +++ b/python/ray/train/v2/_internal/execution/worker_group/worker_group.py @@ -89,6 +89,7 @@ class WorkerGroupContext: num_workers: The number of workers in the worker group. resources_per_worker: The resources per worker. placement_strategy: Strategy for placing workers. + bundle_label_selector: Optional label selectors to apply per-bundle for workers. """ run_attempt_id: str @@ -96,6 +97,7 @@ class WorkerGroupContext: num_workers: int resources_per_worker: Dict[str, float] placement_strategy: str = "PACK" + bundle_label_selector: Optional[Dict[str, str]] = None class WorkerGroup: @@ -268,10 +270,18 @@ def _start_impl( for callback in self._callbacks: callback.before_worker_group_start(worker_group_context) + bundle_label_selector = ( + [worker_group_context.bundle_label_selector.copy()] + * worker_group_context.num_workers + if worker_group_context.bundle_label_selector + else None + ) + pg = placement_group( bundles=[worker_group_context.resources_per_worker] * worker_group_context.num_workers, strategy=worker_group_context.placement_strategy, + bundle_label_selector=bundle_label_selector, ) logger.info( f"Attempting to start training worker group of size {worker_group_context.num_workers} with " diff --git a/python/ray/train/v2/api/config.py b/python/ray/train/v2/api/config.py index 4efc25a2960c..665b3998cf70 100644 --- a/python/ray/train/v2/api/config.py +++ b/python/ray/train/v2/api/config.py @@ -22,7 +22,6 @@ if TYPE_CHECKING: from ray.train import UserCallback - logger = logging.getLogger(__name__) @@ -51,7 +50,17 @@ class ScalingConfig(ScalingConfigV1): of accelerators. See :ref:`the available accelerator types `. Ensure that your cluster has instances with the specified accelerator type - or is able to autoscale to fulfill the request. + or is able to autoscale to fulfill the request. This field is required + when `use_tpu` is True and `num_workers` is greater than 1. + use_tpu: [Experimental] If True, training will be done on TPUs (1 TPU VM + per worker). Defaults to False. The number of TPUs reserved by each + worker can be overridden with the ``resources_per_worker`` + argument. This arg enables SPMD execution of the training workload. + topology: [Experimental] If specified, Ray Train will launch the training + coordinator and workers on nodes with the specified topology. Topology is + auto-detected for TPUs and added as Ray node labels. This arg enables + SPMD execution of the training workload. This field is required + when `use_tpu` is True and `num_workers` is greater than 1. Example: @@ -73,17 +82,62 @@ class ScalingConfig(ScalingConfigV1): """ trainer_resources: Optional[dict] = None + use_tpu: Union[bool] = False + topology: Optional[str] = None def __post_init__(self): if self.trainer_resources is not None: raise DeprecationWarning(TRAINER_RESOURCES_DEPRECATION_MESSAGE) + if self.use_gpu and self.use_tpu: + raise ValueError("Cannot specify both `use_gpu=True` and `use_tpu=True`.") + + if not self.use_tpu and self.num_tpus_per_worker > 0: + raise ValueError( + "`use_tpu` is False but `TPU` was found in " + "`resources_per_worker`. Either set `use_tpu` to True or " + "remove `TPU` from `resources_per_worker." + ) + + if self.use_tpu and self.num_tpus_per_worker == 0: + raise ValueError( + "`use_tpu` is True but `TPU` is set to 0 in " + "`resources_per_worker`. Either set `use_tpu` to False or " + "request a positive number of `TPU` in " + "`resources_per_worker." + ) + + if self.use_tpu and self.num_workers > 1: + if not self.topology: + raise ValueError( + "`topology` must be specified in ScalingConfig when `use_tpu=True` " + " and `num_workers` > 1." + ) + if not self.accelerator_type: + raise ValueError( + "`accelerator_type` must be specified in ScalingConfig when " + "`use_tpu=True` and `num_workers` > 1." + ) + super().__post_init__() + @property + def _resources_per_worker_not_none(self): + if self.resources_per_worker is None: + if self.use_tpu: + return {"TPU": 1} + + return super()._resources_per_worker_not_none + @property def _trainer_resources_not_none(self): return {} + @property + def num_tpus_per_worker(self): + """The number of TPUs to set per worker.""" + return self._resources_per_worker_not_none.get("TPU", 0) + @dataclass class FailureConfig(FailureConfigV1): diff --git a/python/ray/train/v2/api/data_parallel_trainer.py b/python/ray/train/v2/api/data_parallel_trainer.py index 40fede922b90..369d8762e87d 100644 --- a/python/ray/train/v2/api/data_parallel_trainer.py +++ b/python/ray/train/v2/api/data_parallel_trainer.py @@ -27,6 +27,7 @@ AcceleratorSetupCallback, BackendSetupCallback, DatasetsSetupCallback, + TPUReservationCallback, WorkingDirectorySetupCallback, ) from ray.train.v2._internal.callbacks.datasets import GenDataset @@ -154,9 +155,11 @@ def _create_default_callbacks(self) -> List[RayTrainCallback]: data_config=self.data_config, scaling_config=self.scaling_config, ) + tpu_reservation_setup_callback = TPUReservationCallback() callbacks.extend( [ accelerator_setup_callback, + tpu_reservation_setup_callback, backend_setup_callback, datasets_setup_callback, ] diff --git a/python/ray/train/v2/jax/__init__.py b/python/ray/train/v2/jax/__init__.py new file mode 100644 index 000000000000..097ee852b783 --- /dev/null +++ b/python/ray/train/v2/jax/__init__.py @@ -0,0 +1,15 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + try: + import jax # noqa: F401 + except ModuleNotFoundError as exception: + raise ModuleNotFoundError( + "Jax isn't installed. To install Jax, please check" + " `https://github.com/google/jax#installation` for the instructions." + ) from exception + +from ray.train.v2.jax.config import JaxConfig +from ray.train.v2.jax.jax_trainer import JaxTrainer + +__all__ = ["JaxConfig", "JaxTrainer"] diff --git a/python/ray/train/v2/jax/config.py b/python/ray/train/v2/jax/config.py new file mode 100644 index 000000000000..5e8dc5ba33e4 --- /dev/null +++ b/python/ray/train/v2/jax/config.py @@ -0,0 +1,59 @@ +import logging +import os +from dataclasses import dataclass + +import ray +from ray.train._internal.utils import get_address_and_port +from ray.train._internal.worker_group import WorkerGroup +from ray.train.backend import Backend, BackendConfig +from ray.util import PublicAPI + +logger = logging.getLogger(__name__) + + +@PublicAPI(stability="alpha") +@dataclass +class JaxConfig(BackendConfig): + use_tpu: bool = False + + @property + def backend_cls(self): + return _JaxBackend + + +def _setup_jax_tpu_environment( + master_addr_with_port: str, num_workers: int, index: int +): + """Set up distributed Jax training information. + + This function should be called on each worker. + """ + import jax + + jax_platforms = os.environ.get("JAX_PLATFORMS", "").lower() + + if "tpu" in jax_platforms.split(","): + jax.distributed.initialize(master_addr_with_port, num_workers, index) + + +class _JaxBackend(Backend): + def on_start(self, worker_group: WorkerGroup, backend_config: JaxConfig): + if not backend_config.use_tpu: + return + + master_addr, master_port = worker_group.execute_single(0, get_address_and_port) + master_addr_with_port = f"{master_addr}:{master_port}" + + # Get setup tasks in order to throw errors on failure. + setup_futures = [] + for i in range(len(worker_group)): + setup_futures.append( + worker_group.execute_single_async( + i, + _setup_jax_tpu_environment, + master_addr_with_port=master_addr_with_port, + num_workers=len(worker_group), + index=i, + ) + ) + ray.get(setup_futures) diff --git a/python/ray/train/v2/jax/jax_trainer.py b/python/ray/train/v2/jax/jax_trainer.py new file mode 100644 index 000000000000..f1845d8d50ff --- /dev/null +++ b/python/ray/train/v2/jax/jax_trainer.py @@ -0,0 +1,162 @@ +import logging +from typing import TYPE_CHECKING, Callable, Dict, Optional, Union + +from ray.air._internal.config import ensure_only_allowed_dataclass_keys_updated +from ray.train import Checkpoint, DataConfig +from ray.train.trainer import GenDataset +from ray.train.v2.api.config import RunConfig, ScalingConfig +from ray.train.v2.api.data_parallel_trainer import DataParallelTrainer +from ray.train.v2.jax.config import JaxConfig +from ray.util import PublicAPI + +if TYPE_CHECKING: + pass + +logger = logging.getLogger(__name__) + + +@PublicAPI(stability="alpha") +class JaxTrainer(DataParallelTrainer): + """A Trainer for Single-Program Multi-Data (SPMD) JAX training. + Currently only supports TPUs. GPUs will be supported in a future version. + + This Trainer runs the function ``train_loop_per_worker`` on multiple Ray + Actors. These actors are expected to be scheduled on TPU VMs within the same + TPU slice, connected via inter-chip interconnects (ICI). The ``train_loop_per_worker`` + function is expected to take in either 0 or 1 arguments: + + .. testcode:: + + import os + from absl import app + import logging + from typing import Sequence + + import ray + from ray.train.v2.api.config import ScalingConfig, RunConfig + from ray.train.v2.jax import JaxTrainer + from MaxText.train import main as maxtext_main + + def train_loop_per_worker(config): + argv = config["argv"] + maxtext_main(argv) + + def main(argv: Sequence[str]): + ray.init() + + trainer = JaxTrainer( + train_loop_per_worker=train_loop_per_worker, + train_loop_config={"argv": absolute_argv}, + scaling_config=ScalingConfig( + use_tpu=True, + num_workers=4, + topology="4x4", + accelerator_type="TPU-V6E", + resources_per_worker={"TPU": 4}, + placement_strategy="SPREAD", + ), + run_config=RunConfig( + name="maxtext_jaxtrainer", + worker_runtime_env={ + "env_vars": { + "JAX_PLATFORMS": "tpu", + "ENABLE_PJRT_COMPATIBILITY": "true", + "TPU_SLICE_BUILDER_DUMP_CHIP_FORCE": "true", + "TPU_SLICE_BUILDER_DUMP_ICI": "true", + "XLA_FLAGS": "--xla_dump_to=/tmp/xla_dump_file --xla_dump_hlo_as_proto", + } + }, + ), + ) + + result = trainer.fit() + + .. testoutput:: + :options: +ELLIPSIS + :hide: + + If ``train_loop_per_worker`` accepts an argument, then + ``train_loop_config`` will be passed in as the argument. + + If the ``datasets`` dict contains a training dataset (denoted by + the "train" key), then it will be split into multiple dataset + shards that can then be accessed by ``session.get_dataset_shard("train")``. + + Note: + * Only TPU-based distributed training is supported. + * Each worker must be assigned one TPU device via + ``resources_per_worker={"TPU": 1}``. + * Placement strategy is automatically set to ``SPREAD`` to ensure + TPU workers are placed on separate VMs. + * Importing `jax` should occur within `train_loop_per_worker` to + avoid driver-side TPU lock issues. + + Args: + train_loop_per_worker: The training function to execute on each worker. + This function can either take in zero arguments or a single ``Dict`` + argument which is set by defining ``train_loop_config``. + Within this function you can use any of the + :ref:`Ray Train Loop utilities `. + train_loop_config: A configuration ``Dict`` to pass in as an argument to + ``train_loop_per_worker``. + This is typically used for specifying hyperparameters. Passing large + datasets via `train_loop_config` is not recommended and may introduce + large overhead and unknown issues with serialization and deserialization. + jax_config: The configuration for setting up the JAX backend. + If set to None, a default configuration with TPUs will be used. + scaling_config: Configuration for how to scale data parallel training + with SPMD. ``num_workers`` should be set to the number of TPU hosts + and ``topology`` should be set to the TPU topology. + See :class:`~ray.train.ScalingConfig` for more info. + dataset_config: The configuration for ingesting the input ``datasets``. + By default, all the Ray Dataset are split equally across workers. + See :class:`~ray.train.DataConfig` for more details. + run_config: The configuration for the execution of the training run. + See :class:`~ray.train.RunConfig` for more info. + datasets: The Ray Datasets to ingest for training. + Datasets are keyed by name (``{name: dataset}``). + Each dataset can be accessed from within the ``train_loop_per_worker`` + by calling ``ray.train.get_dataset_shard(name)``. + Sharding and additional configuration can be done by + passing in a ``dataset_config``. + resume_from_checkpoint: A checkpoint to resume training from. + This checkpoint can be accessed from within ``train_loop_per_worker`` + by calling ``ray.train.get_checkpoint()``. + """ + + def __init__( + self, + train_loop_per_worker: Union[Callable[[], None], Callable[[Dict], None]], + *, + train_loop_config: Optional[Dict] = None, + jax_config: Optional[JaxConfig] = None, + scaling_config: Optional[ScalingConfig] = None, + dataset_config: Optional[Dict[str, DataConfig]] = None, + run_config: Optional[RunConfig] = None, + datasets: Optional[Dict[str, GenDataset]] = None, + resume_from_checkpoint: Optional[Checkpoint] = None, + ): + if not jax_config: + jax_config = JaxConfig( + use_tpu=scaling_config.use_tpu, + ) + super(JaxTrainer, self).__init__( + train_loop_per_worker=train_loop_per_worker, + train_loop_config=train_loop_config, + backend_config=jax_config, + scaling_config=scaling_config, + dataset_config=dataset_config, + run_config=run_config, + datasets=datasets, + resume_from_checkpoint=resume_from_checkpoint, + ) + + @classmethod + def _validate_scaling_config(cls, scaling_config: ScalingConfig) -> ScalingConfig: + """Return scaling config dataclass after validating updated keys.""" + ensure_only_allowed_dataclass_keys_updated( + dataclass=scaling_config, + allowed_keys=cls._scaling_config_allowed_keys, + ) + + return scaling_config diff --git a/python/ray/train/v2/tests/test_jax_trainer.py b/python/ray/train/v2/tests/test_jax_trainer.py new file mode 100644 index 000000000000..a6449577181b --- /dev/null +++ b/python/ray/train/v2/tests/test_jax_trainer.py @@ -0,0 +1,137 @@ +import pytest + +import ray +from ray.tests.conftest import _ray_start_cluster +from ray.train.v2._internal.constants import HEALTH_CHECK_INTERVAL_S_ENV_VAR +from ray.train.v2.api.config import RunConfig, ScalingConfig +from ray.train.v2.jax import JaxTrainer + + +@pytest.fixture +def ray_tpu_single_host(monkeypatch): + """Start a mock single-host TPU Ray cluster with 2x4 v6e (8 chips per host).""" + with _ray_start_cluster() as cluster: + monkeypatch.setenv("TPU_ACCELERATOR_TYPE", "v6e-8") + + # Simulate one node with 8 TPU chips. + cluster.add_node( + num_cpus=4, + resources={"TPU": 8}, + ) + + ray.init(address=cluster.address) + + yield cluster + ray.shutdown() + + +@pytest.fixture +def ray_tpu_multi_host(monkeypatch): + """Start a simulated multi-host TPU Ray cluster.""" + with _ray_start_cluster() as cluster: + monkeypatch.setenv("TPU_NAME", "test-slice-1") + monkeypatch.setenv("TPU_WORKER_ID", "0") + monkeypatch.setenv("TPU_ACCELERATOR_TYPE", "v4-8") + monkeypatch.setenv("TPU_TOPOLOGY", "2x2x2") + + cluster.add_node( + num_cpus=2, + resources={"TPU": 4, "TPU-v4-8-head": 1}, + ) + monkeypatch.setenv("TPU_WORKER_ID", "1") + cluster.add_node( + num_cpus=2, + resources={"TPU": 4}, + ) + + ray.init(address=cluster.address) + + yield cluster + ray.shutdown() + + +@pytest.fixture(autouse=True) +def reduce_health_check_interval(monkeypatch): + monkeypatch.setenv(HEALTH_CHECK_INTERVAL_S_ENV_VAR, "0.2") + yield + + +def train_func(): + import jax + + from ray import train + + devices = jax.devices() + print(f"Devices on this worker: {devices}") + train.report({"result": [str(d) for d in devices]}) + + +def test_minimal_singlehost(ray_tpu_single_host, tmp_path): + trainer = JaxTrainer( + train_loop_per_worker=train_func, + # Topology can be omitted for single-host. + scaling_config=ScalingConfig( + num_workers=1, + resources_per_worker={"TPU": 8}, + use_tpu=True, + accelerator_type="TPU-V6E", + ), + run_config=RunConfig( + storage_path=str(tmp_path), + worker_runtime_env={ + "pip": ["jax"], + "env_vars": { + "JAX_PLATFORMS": "cpu", + }, + }, + ), + ) + result = trainer.fit() + assert result.error is None + + # Check that exactly 1 TPU node was used. + nodes = ray.nodes() + labeled_nodes = [ + node for node in nodes if node["Alive"] and node["Resources"].get("TPU") == 8 + ] + assert len(labeled_nodes) == 1 + + +def test_minimal_multihost(ray_tpu_multi_host, tmp_path): + trainer = JaxTrainer( + train_loop_per_worker=train_func, + scaling_config=ScalingConfig( + num_workers=2, + resources_per_worker={"TPU": 4}, + use_tpu=True, + topology="2x2x2", + accelerator_type="TPU-V4", + ), + run_config=RunConfig( + storage_path=str(tmp_path), + worker_runtime_env={ + "pip": ["jax"], + "env_vars": { + "JAX_PLATFORMS": "cpu", + }, + }, + ), + ) + result = trainer.fit() + assert result.error is None + + # Check that multi-host slice was scheduled atomically. + nodes = ray.nodes() + slice_label = "test-slice-1" + labeled_nodes = [ + node + for node in nodes + if node["Alive"] and node["Labels"].get("ray.io/tpu-slice-name") == slice_label + ] + assert len(labeled_nodes) == 2 + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", "-x", __file__])) From 8d810e2667fc728e45ca990ff7d7dc8547eae99b Mon Sep 17 00:00:00 2001 From: Alexey Kudinkin Date: Wed, 13 Aug 2025 14:32:25 -0400 Subject: [PATCH 080/634] [Data] Fixing `AutoscalingActorPool` to properly downscale upon completion of the execution (#55565) ## Why are these changes needed? In 2.48 change introduced debouncing handling that disallows downscaling for Actor Pool for 30s after latest upscaling to give AP Operator enough time to start utilizing upscaled actor. However, that affected ability of the Actor Pool to downscale upon completion of the execution: when operator completes execution it should start downscaling immediately. This change addresses that. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Alexey Kudinkin --- .../autoscaler/autoscaling_actor_pool.py | 7 +++-- .../autoscaler/default_autoscaler.py | 2 +- .../operators/actor_pool_map_operator.py | 26 +++++++++---------- .../tests/test_actor_pool_map_operator.py | 7 ++++- python/ray/data/tests/test_autoscaler.py | 8 ++++-- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/python/ray/data/_internal/execution/autoscaler/autoscaling_actor_pool.py b/python/ray/data/_internal/execution/autoscaler/autoscaling_actor_pool.py index 57bd47932b20..97d4dc589a18 100644 --- a/python/ray/data/_internal/execution/autoscaler/autoscaling_actor_pool.py +++ b/python/ray/data/_internal/execution/autoscaler/autoscaling_actor_pool.py @@ -10,6 +10,7 @@ class ActorPoolScalingRequest: delta: int + force: bool = field(default=False) reason: Optional[str] = field(default=None) @classmethod @@ -22,9 +23,11 @@ def upscale(cls, *, delta: int, reason: Optional[str] = None): return ActorPoolScalingRequest(delta=delta, reason=reason) @classmethod - def downscale(cls, *, delta: int, reason: Optional[str] = None): + def downscale( + cls, *, delta: int, force: bool = False, reason: Optional[str] = None + ): assert delta < 0, "For scale down delta is expected to be negative!" - return ActorPoolScalingRequest(delta=delta, reason=reason) + return ActorPoolScalingRequest(delta=delta, force=force, reason=reason) @DeveloperAPI diff --git a/python/ray/data/_internal/execution/autoscaler/default_autoscaler.py b/python/ray/data/_internal/execution/autoscaler/default_autoscaler.py index fd385f97ba33..86ba8ec5e771 100644 --- a/python/ray/data/_internal/execution/autoscaler/default_autoscaler.py +++ b/python/ray/data/_internal/execution/autoscaler/default_autoscaler.py @@ -64,7 +64,7 @@ def _derive_target_scaling_config( op._inputs_complete and op_state.total_enqueued_input_bundles() == 0 ): return ActorPoolScalingRequest.downscale( - delta=-1, reason="consumed all inputs" + delta=-1, force=True, reason="consumed all inputs" ) if actor_pool.current_size() < actor_pool.min_size(): diff --git a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py index 818ee7ac7fe7..9cc21cfb512c 100644 --- a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py @@ -700,7 +700,7 @@ class _ActorPool(AutoscalingActorPool): actors when the operator is done submitting work to the pool. """ - _ACTOR_POOL_SCALE_DOWN_DEBOUNCE_PERIOD_S = 30 + _ACTOR_POOL_SCALE_DOWN_DEBOUNCE_PERIOD_S = 10 _ACTOR_POOL_GRACEFUL_SHUTDOWN_TIMEOUT_S = 30 _LOGICAL_ACTOR_ID_LABEL_KEY = "__ray_data_logical_actor_id" @@ -750,7 +750,7 @@ def __init__( assert self._create_actor_fn is not None # Timestamp of the last scale up action - self._last_upscaling_ts: Optional[float] = None + self._last_upscaled_at: Optional[float] = None self._last_downscaling_debounce_warning_ts: Optional[float] = None # Actors that have started running, including alive and restarting actors. self._running_actors: Dict[ray.actor.ActorHandle, _ActorState] = {} @@ -815,21 +815,21 @@ def _can_apply(self, config: ActorPoolScalingRequest) -> bool: # scaling up, ie if actor pool just scaled down, it'd still be able # to scale back up immediately. if ( - self._last_upscaling_ts is not None - and time.time() - <= self._last_upscaling_ts - + self._ACTOR_POOL_SCALE_DOWN_DEBOUNCE_PERIOD_S + not config.force + and self._last_upscaled_at is not None + and ( + time.time() + <= self._last_upscaled_at + + self._ACTOR_POOL_SCALE_DOWN_DEBOUNCE_PERIOD_S + ) ): # NOTE: To avoid spamming logs unnecessarily, debounce log is produced once # per upscaling event - if ( - self._last_upscaling_ts - != self._last_downscaling_debounce_warning_ts - ): + if self._last_upscaled_at != self._last_downscaling_debounce_warning_ts: logger.debug( - f"Ignoring scaling down request (request={config}; reason=debounced from scaling up at {self._last_upscaling_ts})" + f"Ignoring scaling down request (request={config}; reason=debounced from scaling up at {self._last_upscaled_at})" ) - self._last_downscaling_debounce_warning_ts = self._last_upscaling_ts + self._last_downscaling_debounce_warning_ts = self._last_upscaled_at return False @@ -853,7 +853,7 @@ def scale(self, req: ActorPoolScalingRequest) -> Optional[int]: self.add_pending_actor(actor, ready_ref) # Capture last scale up timestamp - self._last_upscaling_ts = time.time() + self._last_upscaled_at = time.time() return target_num_actors diff --git a/python/ray/data/tests/test_actor_pool_map_operator.py b/python/ray/data/tests/test_actor_pool_map_operator.py index cf68f0ee05c8..e2627c29a187 100644 --- a/python/ray/data/tests/test_actor_pool_map_operator.py +++ b/python/ray/data/tests/test_actor_pool_map_operator.py @@ -4,6 +4,7 @@ import threading import time import unittest +from dataclasses import replace from typing import Any, Dict, Optional, Tuple from unittest.mock import MagicMock @@ -161,7 +162,11 @@ def test_can_scale_down(self): pool.scale(ActorPoolScalingRequest(delta=1, reason="scaling up")) # Assert we can't scale down immediately after scale up assert not pool._can_apply(downscaling_request) - assert pool._last_upscaling_ts == time.time() + assert pool._last_upscaled_at == time.time() + + # Check that we can still scale down if downscaling request + # is a forced one + assert pool._can_apply(replace(downscaling_request, force=True)) # Advance clock f.tick( diff --git a/python/ray/data/tests/test_autoscaler.py b/python/ray/data/tests/test_autoscaler.py index 736fff47fb58..f2d0d2369ead 100644 --- a/python/ray/data/tests/test_autoscaler.py +++ b/python/ray/data/tests/test_autoscaler.py @@ -80,14 +80,16 @@ def patch(mock, attr, value, is_method=True): yield setattr(mock, attr, original) - def assert_autoscaling_action(*, delta: int, expected_reason: Optional[str]): + def assert_autoscaling_action( + *, delta: int, expected_reason: Optional[str], force: bool = False + ): nonlocal actor_pool, op, op_state assert autoscaler._derive_target_scaling_config( actor_pool=actor_pool, op=op, op_state=op_state, - ) == ActorPoolScalingRequest(delta=delta, reason=expected_reason) + ) == ActorPoolScalingRequest(delta=delta, force=force, reason=expected_reason) # Should scale up since the util above the threshold. assert actor_pool.get_pool_util() == 1.5 @@ -141,6 +143,7 @@ def assert_autoscaling_action(*, delta: int, expected_reason: Optional[str]): assert_autoscaling_action( delta=-1, expected_reason="consumed all inputs", + force=True, ) # Should scale down only once all inputs have been already dispatched AND @@ -150,6 +153,7 @@ def assert_autoscaling_action(*, delta: int, expected_reason: Optional[str]): with patch(op, "_inputs_complete", True, is_method=False): assert_autoscaling_action( delta=-1, + force=True, expected_reason="consumed all inputs", ) From 3d44e3d17b56e993f1fd7407bdf1288c852c8c41 Mon Sep 17 00:00:00 2001 From: Mengjin Yan Date: Wed, 13 Aug 2025 11:57:54 -0700 Subject: [PATCH 081/634] [Core][TaskEventFollowup/03] Improve the Target Http Endpoint in Aggregator Agent (#55529) This PR improves the target http endpoint in the aggregator_agent.py: Merge the address and port as one env var to specify the target http endpoint Set the default value of the endpoint to be empty. And only when the endpoint is specified, we send the events out to the endpoint Update corresponding tests ----------- Signed-off-by: Mengjin Yan Signed-off-by: myan --- python/ray/dashboard/agent.py | 2 + .../modules/aggregator/aggregator_agent.py | 45 +++++++++++------ .../aggregator/tests/test_aggregator_agent.py | 48 ++++++++++++++----- python/ray/tests/test_metrics_agent.py | 26 ++++++++-- 4 files changed, 90 insertions(+), 31 deletions(-) diff --git a/python/ray/dashboard/agent.py b/python/ray/dashboard/agent.py index 85ac9f05f503..1c46afee5372 100644 --- a/python/ray/dashboard/agent.py +++ b/python/ray/dashboard/agent.py @@ -30,6 +30,7 @@ def __init__( minimal, metrics_export_port=None, node_manager_port=None, + events_export_addr=None, listen_port=ray_constants.DEFAULT_DASHBOARD_AGENT_LISTEN_PORT, disable_metrics_collection: bool = False, *, # the following are required kwargs @@ -56,6 +57,7 @@ def __init__( self.dashboard_agent_port = dashboard_agent_port self.metrics_export_port = metrics_export_port self.node_manager_port = node_manager_port + self.events_export_addr = events_export_addr self.listen_port = listen_port self.object_store_name = object_store_name self.raylet_name = raylet_name diff --git a/python/ray/dashboard/modules/aggregator/aggregator_agent.py b/python/ray/dashboard/modules/aggregator/aggregator_agent.py index 4499116a6b2a..d970623633a9 100644 --- a/python/ray/dashboard/modules/aggregator/aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/aggregator_agent.py @@ -66,12 +66,8 @@ REQUEST_BACKOFF_FACTOR = ray_constants.env_float( f"{env_var_prefix}_REQUEST_BACKOFF_FACTOR", 1.0 ) -# Address of the external service to send events -EVENT_SEND_ADDR = os.environ.get( - f"{env_var_prefix}_EVENT_SEND_ADDR", "http://127.0.0.1" -) -# Port of the external service to send events -EVENT_SEND_PORT = ray_constants.env_integer(f"{env_var_prefix}_EVENT_SEND_PORT", 12345) +# Address of the external service to send events with format of "http://:" +EVENTS_EXPORT_ADDR = os.environ.get(f"{env_var_prefix}_EVENTS_EXPORT_ADDR", "") # Interval to update metrics METRICS_UPDATE_INTERVAL_SECONDS = ray_constants.env_float( f"{env_var_prefix}_METRICS_UPDATE_INTERVAL_SECONDS", 0.1 @@ -119,7 +115,9 @@ ) events_filtered_out = Counter( f"{metrics_prefix}_events_filtered_out", - "Total number of events filtered out before publishing to external server.", + "Total number of events filtered out before publishing to external server. The " + "metric counts the events that are received by the aggregator agent but are " + "not part of the public API yet.", tuple(dashboard_consts.COMPONENT_METRICS_TAG_KEYS), namespace="ray", ) @@ -164,13 +162,22 @@ def __init__(self, dashboard_agent) -> None: self._events_dropped_at_event_aggregator_since_last_metrics_update = 0 self._events_published_since_last_metrics_update = 0 self._events_filtered_out_since_last_metrics_update = 0 - - self._orig_sigterm_handler = signal.signal( - signal.SIGTERM, self._sigterm_handler + self._events_export_addr = ( + dashboard_agent.events_export_addr or EVENTS_EXPORT_ADDR ) - self._is_cleanup = False - self._cleanup_finished_event = threading.Event() + self._event_http_target_enabled = bool(self._events_export_addr) + if not self._event_http_target_enabled: + logger.info( + "Event HTTP target not set, skipping sending events to " + f"external http service. events_export_addr: {self._events_export_addr}" + ) + + self._event_processing_enabled = self._event_http_target_enabled + if self._event_processing_enabled: + logger.info("Event processing enabled") + else: + logger.info("Event processing disabled") self._exposable_event_types = { event_type.strip() @@ -178,6 +185,13 @@ def __init__(self, dashboard_agent) -> None: if event_type.strip() } + self._orig_sigterm_handler = signal.signal( + signal.SIGTERM, self._sigterm_handler + ) + + self._is_cleanup = False + self._cleanup_finished_event = threading.Event() + async def AddEvents(self, request, context) -> None: """ gRPC handler for adding events to the event aggregator @@ -192,6 +206,9 @@ def _receive_events(self, request): """ Receives events from the request, adds them to the event buffer, """ + if not self._event_processing_enabled: + return events_event_aggregator_service_pb2.AddEventReply() + # TODO(myan) #54515: Considering adding a mechanism to also send out the events # metadata (e.g. dropped task attempts) to help with event processing at the # downstream @@ -235,7 +252,7 @@ def _send_events_to_external_service(self, event_batch) -> None: """ Sends a batch of events to the external service via HTTP POST request """ - if not event_batch: + if not event_batch or not self._event_http_target_enabled: return filtered_event_batch = [ @@ -255,7 +272,7 @@ def _send_events_to_external_service(self, event_batch) -> None: try: response = self._http_session.post( - f"{EVENT_SEND_ADDR}:{EVENT_SEND_PORT}", json=filtered_event_batch_json + f"{self._events_export_addr}", json=filtered_event_batch_json ) response.raise_for_status() with self._lock: diff --git a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py index 3020813227fd..f7f1735110ba 100644 --- a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py @@ -1,6 +1,7 @@ import sys import json import base64 +from unittest.mock import MagicMock import pytest from google.protobuf.timestamp_pb2 import Timestamp @@ -30,13 +31,19 @@ from ray.core.generated.profile_events_pb2 import ProfileEvents, ProfileEventEntry from ray.core.generated.events_task_profile_events_pb2 import TaskProfileEvents +from ray.dashboard.modules.aggregator.aggregator_agent import AggregatorAgent + _EVENT_AGGREGATOR_AGENT_TARGET_PORT = find_free_port() +_EVENT_AGGREGATOR_AGENT_TARGET_IP = "127.0.0.1" +_EVENT_AGGREGATOR_AGENT_TARGET_ADDR = ( + f"http://{_EVENT_AGGREGATOR_AGENT_TARGET_IP}:{_EVENT_AGGREGATOR_AGENT_TARGET_PORT}" +) @pytest.fixture(scope="module") def httpserver_listen_address(): - return ("127.0.0.1", _EVENT_AGGREGATOR_AGENT_TARGET_PORT) + return (_EVENT_AGGREGATOR_AGENT_TARGET_IP, _EVENT_AGGREGATOR_AGENT_TARGET_PORT) @pytest.fixture @@ -54,9 +61,7 @@ def fake_timestamp(): [ { "env_vars": { - "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENT_SEND_PORT": str( - _EVENT_AGGREGATOR_AGENT_TARGET_PORT - ), + "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": _EVENT_AGGREGATOR_AGENT_TARGET_ADDR, }, }, ], @@ -86,6 +91,29 @@ def get_event_aggregator_grpc_stub(webui_url, gcs_address, head_node_id): return EventAggregatorServiceStub(channel) +@pytest.mark.parametrize( + ( + "export_addr", + "expected_http_target_enabled", + "expected_event_processing_enabled", + ), + [ + ("", False, False), + ("http://127.0.0.1:" + str(_EVENT_AGGREGATOR_AGENT_TARGET_PORT), True, True), + ], +) +def test_aggregator_agent_http_target_not_enabled( + export_addr, + expected_http_target_enabled, + expected_event_processing_enabled, +): + dashboard_agent = MagicMock() + dashboard_agent.events_export_addr = export_addr + agent = AggregatorAgent(dashboard_agent) + assert agent._event_http_target_enabled == expected_http_target_enabled + assert agent._event_processing_enabled == expected_event_processing_enabled + + @_with_aggregator_port def test_aggregator_agent_receive_publish_events_normally( ray_start_cluster_head_with_env_vars, httpserver, fake_timestamp @@ -136,9 +164,7 @@ def test_aggregator_agent_receive_publish_events_normally( { "env_vars": { "RAY_DASHBOARD_AGGREGATOR_AGENT_MAX_EVENT_BUFFER_SIZE": 1, - "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENT_SEND_PORT": str( - _EVENT_AGGREGATOR_AGENT_TARGET_PORT - ), + "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": _EVENT_AGGREGATOR_AGENT_TARGET_ADDR, }, }, ], @@ -242,9 +268,7 @@ def test_aggregator_agent_receive_multiple_events( { "env_vars": { "RAY_DASHBOARD_AGGREGATOR_AGENT_MAX_EVENT_BUFFER_SIZE": 1, - "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENT_SEND_PORT": str( - _EVENT_AGGREGATOR_AGENT_TARGET_PORT - ), + "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": _EVENT_AGGREGATOR_AGENT_TARGET_ADDR, }, }, ], @@ -365,9 +389,7 @@ def test_aggregator_agent_profile_events_not_exposed( [ { "env_vars": { - "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENT_SEND_PORT": str( - _EVENT_AGGREGATOR_AGENT_TARGET_PORT - ), + "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": _EVENT_AGGREGATOR_AGENT_TARGET_ADDR, "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "TASK_DEFINITION_EVENT,TASK_EXECUTION_EVENT,ACTOR_TASK_DEFINITION_EVENT,ACTOR_TASK_EXECUTION_EVENT,TASK_PROFILE_EVENT", }, }, diff --git a/python/ray/tests/test_metrics_agent.py b/python/ray/tests/test_metrics_agent.py index 2021c97ef36c..a167b701012b 100644 --- a/python/ray/tests/test_metrics_agent.py +++ b/python/ray/tests/test_metrics_agent.py @@ -144,6 +144,7 @@ "ray_event_aggregator_agent_events_failed_to_add_to_aggregator_total", "ray_event_aggregator_agent_events_dropped_at_event_aggregator_total", "ray_event_aggregator_agent_events_published_total", + "ray_event_aggregator_agent_events_filtered_out_total", ] _NODE_METRICS = [ @@ -465,6 +466,10 @@ def verify_dashboard_metrics(): _EVENT_AGGREGATOR_AGENT_TARGET_PORT = find_free_port() +_EVENT_AGGREGATOR_AGENT_TARGET_IP = "127.0.0.1" +_EVENT_AGGREGATOR_AGENT_TARGET_ADDR = ( + f"http://{_EVENT_AGGREGATOR_AGENT_TARGET_IP}:{_EVENT_AGGREGATOR_AGENT_TARGET_PORT}" +) @pytest.fixture(scope="module") @@ -477,8 +482,11 @@ def httpserver_listen_address(): [ { "env_vars": { - "RAY_DASHBOARD_AGGREGATOR_AGENT_MAX_EVENT_BUFFER_SIZE": 1, - "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENT_SEND_PORT": _EVENT_AGGREGATOR_AGENT_TARGET_PORT, + "RAY_DASHBOARD_AGGREGATOR_AGENT_MAX_EVENT_BUFFER_SIZE": 2, + "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": _EVENT_AGGREGATOR_AGENT_TARGET_ADDR, + # Turn off task events generation to avoid the task events from the + # cluster impacting the test result + "RAY_task_events_report_interval_ms": 0, }, }, ], @@ -505,16 +513,18 @@ def test_case_stats_exist(): "ray_event_aggregator_agent_events_failed_to_add_to_aggregator_total", "ray_event_aggregator_agent_events_dropped_at_event_aggregator_total", "ray_event_aggregator_agent_events_published_total", + "ray_event_aggregator_agent_events_filtered_out_total", ] return all(metric in metrics_names for metric in event_aggregator_metrics) def test_case_value_correct(): _, _, metric_samples = fetch_prometheus(prom_addresses) expected_metrics_values = { - "ray_event_aggregator_agent_events_received_total": 2.0, + "ray_event_aggregator_agent_events_received_total": 3.0, "ray_event_aggregator_agent_events_failed_to_add_to_aggregator_total": 0.0, "ray_event_aggregator_agent_events_dropped_at_event_aggregator_total": 1.0, "ray_event_aggregator_agent_events_published_total": 1.0, + "ray_event_aggregator_agent_events_filtered_out_total": 1.0, } for descriptor, expected_value in expected_metrics_values.items(): samples = [m for m in metric_samples if m.name == descriptor] @@ -543,11 +553,19 @@ def test_case_value_correct(): RayEvent( event_id=b"2", source_type=RayEvent.SourceType.CORE_WORKER, - event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, + event_type=RayEvent.EventType.TASK_PROFILE_EVENT, timestamp=timestamp, severity=RayEvent.Severity.INFO, message="hello 2", ), + RayEvent( + event_id=b"3", + source_type=RayEvent.SourceType.CORE_WORKER, + event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, + timestamp=timestamp, + severity=RayEvent.Severity.INFO, + message="hello 3", + ), ], task_events_metadata=TaskEventsMetadata( dropped_task_attempts=[ From 6ebd7d013933dfa990b11ffcad63cfd6f78db6cd Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Wed, 13 Aug 2025 12:22:56 -0700 Subject: [PATCH 082/634] [data] Sanitization of Dataset Metadata Export (#55379) ## Why are these changes needed? A couple of things that have been improved - updating structs should have string keys - More tests for bytes, bytearrays, dataclasses ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- .../issue_detector_configuration.py | 4 +- .../ray/data/_internal/metadata_exporter.py | 38 +- python/ray/data/tests/test_state_export.py | 335 ++++++++++++++++-- 3 files changed, 330 insertions(+), 47 deletions(-) diff --git a/python/ray/data/_internal/issue_detection/issue_detector_configuration.py b/python/ray/data/_internal/issue_detection/issue_detector_configuration.py index 53aea74dbd63..6b59a7318d23 100644 --- a/python/ray/data/_internal/issue_detection/issue_detector_configuration.py +++ b/python/ray/data/_internal/issue_detection/issue_detector_configuration.py @@ -13,10 +13,10 @@ @dataclass class IssueDetectorsConfiguration: hanging_detector_config: HangingExecutionIssueDetectorConfig = field( - default=HangingExecutionIssueDetectorConfig + default_factory=HangingExecutionIssueDetectorConfig ) high_memory_detector_config: HighMemoryIssueDetectorConfig = field( - default=HighMemoryIssueDetectorConfig + default_factory=HighMemoryIssueDetectorConfig ) detectors: List[Type[IssueDetector]] = field( default_factory=lambda: [HangingExecutionIssueDetector, HighMemoryIssueDetector] diff --git a/python/ray/data/_internal/metadata_exporter.py b/python/ray/data/_internal/metadata_exporter.py index dfc2a60bcffc..5ff8242cfc5c 100644 --- a/python/ray/data/_internal/metadata_exporter.py +++ b/python/ray/data/_internal/metadata_exporter.py @@ -1,10 +1,9 @@ """Metadata exporter API for Ray Data datasets.""" -import json import logging import os from abc import ABC, abstractmethod -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field, is_dataclass from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Sequence import ray @@ -142,30 +141,36 @@ class DatasetMetadata: data_context: DataContext -def _add_ellipsis(s, truncate_length): +def _add_ellipsis(s: str, truncate_length: int) -> str: if len(s) > truncate_length: return s[:truncate_length] + "..." return s def sanitize_for_struct(obj, truncate_length=DEFAULT_TRUNCATION_LENGTH): + """Prepares the obj for Struct Protobuf format by recursively + going through dictionaries, lists, etc... + + - Dataclasses will be converted to dicts + - Dictionary keys will be converted to strings + - Lists, tuples, sets, bytes, bytearrays will be converted to lists + """ if isinstance(obj, Mapping): - return {k: sanitize_for_struct(v, truncate_length) for k, v in obj.items()} - elif isinstance(obj, (int, float, bool)) or obj is None: - return obj + # protobuf Struct key names must be strings. + return {str(k): sanitize_for_struct(v, truncate_length) for k, v in obj.items()} elif isinstance(obj, str): return _add_ellipsis(obj, truncate_length) - elif isinstance(obj, Sequence): - return [sanitize_for_struct(v, truncate_length) for v in obj] + elif isinstance(obj, (Sequence, set)): + # Convert all sequence-like types (lists, tuples, sets, bytes, other sequences) to lists + return [sanitize_for_struct(v, truncate_length=truncate_length) for v in obj] else: - # Convert unhandled types to string try: - return _add_ellipsis(json.dumps(obj), truncate_length) - except (TypeError, OverflowError): - try: - return _add_ellipsis(str(obj), truncate_length) - except Exception: - return UNKNOWN + if is_dataclass(obj): + return sanitize_for_struct(asdict(obj), truncate_length) + return _add_ellipsis(str(obj), truncate_length) + except Exception: + unk_name = f"{UNKNOWN}: {type(obj).__name__}" + return _add_ellipsis(unk_name, truncate_length) def dataset_metadata_to_proto(dataset_metadata: DatasetMetadata) -> Any: @@ -178,7 +183,6 @@ def dataset_metadata_to_proto(dataset_metadata: DatasetMetadata) -> Any: Returns: The protobuf message representing the dataset metadata. """ - from dataclasses import asdict from google.protobuf.struct_pb2 import Struct @@ -221,7 +225,7 @@ def dataset_metadata_to_proto(dataset_metadata: DatasetMetadata) -> Any: # Populate the data metadata proto data_context = Struct() - data_context.update(sanitize_for_struct(asdict(dataset_metadata.data_context))) + data_context.update(sanitize_for_struct(dataset_metadata.data_context)) proto_dataset_metadata = ProtoDatasetMetadata( dataset_id=dataset_metadata.dataset_id, job_id=dataset_metadata.job_id, diff --git a/python/ray/data/tests/test_state_export.py b/python/ray/data/tests/test_state_export.py index 58beeff858a9..cfa815b32294 100644 --- a/python/ray/data/tests/test_state_export.py +++ b/python/ray/data/tests/test_state_export.py @@ -1,11 +1,13 @@ import json import os -from dataclasses import asdict +from dataclasses import asdict, dataclass +from typing import Tuple import pytest import ray from ray.data import DataContext +from ray.data._internal.logical.interfaces import LogicalOperator from ray.data._internal.metadata_exporter import ( UNKNOWN, Operator, @@ -62,9 +64,81 @@ def ray_start_cluster_with_export_api_write(shutdown_only): yield res +@dataclass +class TestDataclass: + """A test dataclass for testing dataclass serialization.""" + + list_field: list = None + dict_field: dict = None + string_field: str = "test" + int_field: int = 1 + float_field: float = 1.0 + set_field: set = None + tuple_field: Tuple[int] = None + bool_field: bool = True + none_field: None = None + + def __post_init__(self): + self.list_field = [1, 2, 3] + self.dict_field = {1: 2, "3": "4"} + self.set_field = {1, 2, 3} + self.tuple_field = (1, 2, 3) + + +class DummyLogicalOperator(LogicalOperator): + """A dummy logical operator for testing _get_logical_args with various data types.""" + + def __init__(self, input_op=None): + super().__init__("DummyOperator", []) + + # Test various data types that might be returned by _get_logical_args + self._string_value = "test_string" + self._int_value = 42 + self._float_value = 3.14 + self._bool_value = True + self._none_value = None + self._list_value = [1, 2, 3, "string", None] + self._dict_value = {"key1": "value1", "key2": 123, "key3": None} + self._nested_dict = { + "level1": { + "level2": { + "level3": "deep_value", + "numbers": [1, 2, 3], + "mixed": {"a": 1, "b": "string", "c": None}, + } + } + } + self._tuple_value = (1, "string", None, 3.14) + self._set_value = {1} + self._bytes_value = b"binary_data" + self._complex_dict = { + "string_keys": {"a": 1, "b": 2}, + "int_keys": {1: "one", 2: "two"}, # This should cause issues if not handled + "mixed_keys": {"str": "value", 1: "int_key", None: "none_key"}, + } + self._empty_containers = { + "empty_list": [], + "empty_dict": {}, + "empty_tuple": (), + "empty_set": set(), + } + self._special_values = { + "zero": 0, + "negative": -1, + "large_int": 999999999999999999, + "small_float": 0.0000001, + "inf": float("inf"), + "neg_inf": float("-inf"), + "nan": float("nan"), + } + + self._data_class = TestDataclass() + + @pytest.fixture def dummy_dataset_topology(): """Create a dummy Topology.""" + dummy_operator = DummyLogicalOperator() dummy_topology = Topology( operators=[ Operator( @@ -73,6 +147,7 @@ def dummy_dataset_topology(): uuid="uuid_0", input_dependencies=[], sub_stages=[], + args=sanitize_for_struct(dummy_operator._get_args()), ), Operator( name="ReadRange->Map()->Filter()", @@ -80,12 +155,179 @@ def dummy_dataset_topology(): uuid="uuid_1", input_dependencies=["Input_0"], sub_stages=[], + args=sanitize_for_struct(dummy_operator._get_args()), ), ], ) return dummy_topology +@pytest.fixture +def dummy_dataset_topology_expected_output(): + return { + "operators": [ + { + "name": "Input", + "id": "Input_0", + "uuid": "uuid_0", + "args": { + "_num_outputs": "None", + "_int_value": "42", + "_special_values": { + "negative": "-1", + "inf": "inf", + "zero": "0", + "large_int": "999999999999999999", + "small_float": "1e-07", + "neg_inf": "-inf", + "nan": "nan", + }, + "_none_value": "None", + "_name": "DummyOperator", + "_output_dependencies": [], + "_float_value": "3.14", + "_list_value": ["1", "2", "3", "string", "None"], + "_dict_value": {"key1": "value1", "key3": "None", "key2": "123"}, + "_set_value": ["1"], + "_tuple_value": ["1", "string", "None", "3.14"], + "_bytes_value": [ + "98", + "105", + "110", + "97", + "114", + "121", + "95", + "100", + "97", + "116", + "97", + ], + "_input_dependencies": [], + "_empty_containers": { + "empty_set": [], + "empty_tuple": [], + "empty_dict": {}, + "empty_list": [], + }, + "_bool_value": "True", + "_nested_dict": { + "level1": { + "level2": { + "mixed": {"a": "1", "b": "string", "c": "None"}, + "numbers": ["1", "2", "3"], + "level3": "deep_value", + } + } + }, + "_string_value": "test_string", + "_complex_dict": { + "string_keys": {"a": "1", "b": "2"}, + "mixed_keys": { + "None": "none_key", + "str": "value", + "1": "int_key", + }, + "int_keys": {"1": "one", "2": "two"}, + }, + "_data_class": { + "list_field": ["1", "2", "3"], + "dict_field": {"3": "4", "1": "2"}, + "tuple_field": ["1", "2", "3"], + "set_field": ["1", "2", "3"], + "int_field": "1", + "none_field": "None", + "bool_field": "True", + "string_field": "test", + "float_field": "1.0", + }, + }, + "input_dependencies": [], + "sub_stages": [], + }, + { + "name": "ReadRange->Map()->Filter()", + "id": "ReadRange->Map()->Filter()_1", + "uuid": "uuid_1", + "input_dependencies": ["Input_0"], + "args": { + "_num_outputs": "None", + "_int_value": "42", + "_special_values": { + "negative": "-1", + "inf": "inf", + "zero": "0", + "large_int": "999999999999999999", + "small_float": "1e-07", + "neg_inf": "-inf", + "nan": "nan", + }, + "_none_value": "None", + "_name": "DummyOperator", + "_output_dependencies": [], + "_float_value": "3.14", + "_list_value": ["1", "2", "3", "string", "None"], + "_dict_value": {"key1": "value1", "key3": "None", "key2": "123"}, + "_set_value": ["1"], + "_tuple_value": ["1", "string", "None", "3.14"], + "_bytes_value": [ + "98", + "105", + "110", + "97", + "114", + "121", + "95", + "100", + "97", + "116", + "97", + ], + "_input_dependencies": [], + "_empty_containers": { + "empty_set": [], + "empty_tuple": [], + "empty_dict": {}, + "empty_list": [], + }, + "_bool_value": "True", + "_nested_dict": { + "level1": { + "level2": { + "mixed": {"a": "1", "b": "string", "c": "None"}, + "numbers": ["1", "2", "3"], + "level3": "deep_value", + } + } + }, + "_string_value": "test_string", + "_complex_dict": { + "string_keys": {"a": "1", "b": "2"}, + "mixed_keys": { + "None": "none_key", + "str": "value", + "1": "int_key", + }, + "int_keys": {"1": "one", "2": "two"}, + }, + "_data_class": { + "list_field": ["1", "2", "3"], + "dict_field": {"3": "4", "1": "2"}, + "tuple_field": ["1", "2", "3"], + "set_field": ["1", "2", "3"], + "int_field": "1", + "none_field": "None", + "bool_field": "True", + "string_field": "test", + "float_field": "1.0", + }, + }, + "sub_stages": [], + }, + ] + } + + def test_export_disabled(ray_start_regular, dummy_dataset_topology): """Test that no export files are created when export API is disabled.""" stats_actor = _get_or_create_stats_actor() @@ -105,7 +347,7 @@ def test_export_disabled(ray_start_regular, dummy_dataset_topology): assert not os.path.exists(_get_export_file_path()) -def _test_dataset_metadata_export(topology): +def _test_dataset_metadata_export(topology, dummy_dataset_topology_expected_output): """Test that dataset metadata export events are written when export API is enabled.""" stats_actor = _get_or_create_stats_actor() @@ -124,22 +366,30 @@ def _test_dataset_metadata_export(topology): data = _get_exported_data() assert len(data) == 1 assert data[0]["source_type"] == "EXPORT_DATASET_METADATA" - assert data[0]["event_data"]["topology"] == sanitize_for_struct(asdict(topology)) + assert data[0]["event_data"]["topology"] == dummy_dataset_topology_expected_output assert data[0]["event_data"]["dataset_id"] == STUB_DATASET_ID assert data[0]["event_data"]["job_id"] == STUB_JOB_ID assert data[0]["event_data"]["start_time"] is not None def test_export_dataset_metadata_enabled_by_config( - ray_start_cluster_with_export_api_config, dummy_dataset_topology + ray_start_cluster_with_export_api_config, + dummy_dataset_topology, + dummy_dataset_topology_expected_output, ): - _test_dataset_metadata_export(dummy_dataset_topology) + _test_dataset_metadata_export( + dummy_dataset_topology, dummy_dataset_topology_expected_output + ) def test_export_dataset_metadata( - ray_start_cluster_with_export_api_write, dummy_dataset_topology + ray_start_cluster_with_export_api_write, + dummy_dataset_topology, + dummy_dataset_topology_expected_output, ): - _test_dataset_metadata_export(dummy_dataset_topology) + _test_dataset_metadata_export( + dummy_dataset_topology, dummy_dataset_topology_expected_output + ) @pytest.mark.parametrize( @@ -181,7 +431,9 @@ def __call__(self, x): def test_export_multiple_datasets( - ray_start_cluster_with_export_api_write, dummy_dataset_topology + ray_start_cluster_with_export_api_write, + dummy_dataset_topology, + dummy_dataset_topology_expected_output, ): """Test that multiple datasets can be exported when export API is enabled.""" stats_actor = _get_or_create_stats_actor() @@ -245,8 +497,8 @@ def test_export_multiple_datasets( ), f"First dataset {first_dataset_id} not found in exported data" first_entry = datasets_by_id[first_dataset_id] assert first_entry["source_type"] == "EXPORT_DATASET_METADATA" - assert first_entry["event_data"]["topology"] == sanitize_for_struct( - asdict(dummy_dataset_topology) + assert ( + first_entry["event_data"]["topology"] == dummy_dataset_topology_expected_output ) assert first_entry["event_data"]["job_id"] == STUB_JOB_ID assert first_entry["event_data"]["start_time"] is not None @@ -257,9 +509,7 @@ def test_export_multiple_datasets( ), f"Second dataset {second_dataset_id} not found in exported data" second_entry = datasets_by_id[second_dataset_id] assert second_entry["source_type"] == "EXPORT_DATASET_METADATA" - assert second_entry["event_data"]["topology"] == sanitize_for_struct( - asdict(second_topology) - ) + assert second_entry["event_data"]["topology"] == asdict(second_topology) assert second_entry["event_data"]["job_id"] == STUB_JOB_ID assert second_entry["event_data"]["start_time"] is not None @@ -287,12 +537,12 @@ def __str__(self): @pytest.mark.parametrize( "input_obj,expected_output,truncate_length", [ - # Basic types - should return as-is - (42, 42, 100), - (3.14, 3.14, 100), - (True, True, 100), - (False, False, 100), - (None, None, 100), + # Basic types - should return as strings + (42, "42", 100), + (3.14, "3.14", 100), + (True, "True", 100), + (False, "False", 100), + (None, "None", 100), # Strings - short strings return as-is ("hello", "hello", 100), # Strings - long strings get truncated @@ -302,28 +552,57 @@ def __str__(self): ({"key": "value"}, {"key": "value"}, 100), ({"long_key": "a" * 150}, {"long_key": "a" * 100 + "..."}, 100), ({"nested": {"inner": "value"}}, {"nested": {"inner": "value"}}, 100), - # Sequences - should recursively sanitize elements - ([1, 2, 3], [1, 2, 3], 100), + # Sequences - should recursively sanitize elements (convert to strings) + ([1, 2, 3], ["1", "2", "3"], 100), (["short", "a" * 150], ["short", "a" * 100 + "..."], 100), # Complex nested structures ( {"list": [1, "a" * 150], "dict": {"key": "a" * 150}}, - {"list": [1, "a" * 100 + "..."], "dict": {"key": "a" * 100 + "..."}}, + {"list": ["1", "a" * 100 + "..."], "dict": {"key": "a" * 100 + "..."}}, 100, ), # Objects that can be converted to string (BasicObject("test"), "BasicObject(test)", 100), # Falls back to str() - # Objects that can't be JSON serialized but can be stringified - ({1, 2, 3}, "{1, 2, 3}", 100), # Falls back to str() + # Sets can be converted to Lists of strings + ({1, 2, 3}, ["1", "2", "3"], 100), + ((1, 2, 3), ["1", "2", "3"], 100), # Objects that can't be serialized or stringified - (UnserializableObject(), UNKNOWN, 100), + (UnserializableObject(), f"{UNKNOWN}: {UnserializableObject.__name__}", 100), # Empty containers ({}, {}, 100), ([], [], 100), - # Mixed type sequences + # Mixed type sequences - all converted to strings ( [1, "hello", {"key": "value"}, None], - [1, "hello", {"key": "value"}, None], + ["1", "hello", {"key": "value"}, "None"], + 100, + ), + # Bytearrays/bytes - should be converted to lists of string representations + (bytearray(b"hello"), ["104", "101", "108", "108", "111"], 100), + (bytearray([1, 2, 3, 4, 5]), ["1", "2", "3", "4", "5"], 100), + (bytes(b"test"), ["116", "101", "115", "116"], 100), + # Dataclass + ( + TestDataclass(), + { + "list_field": ["1", "2", "3"], + "dict_field": {"1": "2", "3": "4"}, # key should be strings + "string_field": "test", + "int_field": "1", + "float_field": "1.0", + "set_field": [ + "1", + "2", + "3", + ], # sets will be converted to Lists of strings + "tuple_field": [ + "1", + "2", + "3", + ], # tuples will be converted to Lists of strings + "bool_field": "True", + "none_field": "None", + }, 100, ), ], @@ -331,7 +610,7 @@ def __str__(self): def test_sanitize_for_struct(input_obj, expected_output, truncate_length): """Test sanitize_for_struct with various input types and truncation lengths.""" result = sanitize_for_struct(input_obj, truncate_length) - assert result == expected_output + assert result == expected_output, f"Expected {expected_output}, got {result}" if __name__ == "__main__": From c47048e6ebf1b7a705cdb1be18b027889623e1a4 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Wed, 13 Aug 2025 12:56:01 -0700 Subject: [PATCH 083/634] [core][obsclean/02] de-static more internal ray metrics (#55537) Ray core currently offers two APIs for defining internal metrics: a static object-oriented (OO) API and a template/extern-based API. The OO API is also used for defining custom metrics at the Ray application level, and I personally find it easier to read. This series of PRs aims to unify all metric definitions under the OO API. --------- This PR migrates **all** metric from static to runtime definition, as part of the effort to eliminate all statically defined metrics. Currently, the OO interface attempts to register a metric at the same time its first value is recorded, due to the [C++ static initialization order fiasco](https://en.cppreference.com/w/cpp/language/siof.html), which is awkward and potentially inefficient. We can fix this by removing all statically defined metrics. Test: - CI --------- Signed-off-by: Cuong Nguyen Signed-off-by: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- src/ray/gcs/gcs_server/gcs_node_manager.cc | 2 +- src/ray/gcs/gcs_server/gcs_node_manager.h | 6 + src/ray/gcs/gcs_server/gcs_worker_manager.cc | 2 +- src/ray/gcs/gcs_server/gcs_worker_manager.h | 8 + src/ray/gcs/redis_context.cc | 2 +- src/ray/gcs/redis_context.h | 10 ++ src/ray/object_manager/object_manager.cc | 8 +- src/ray/object_manager/object_manager.h | 20 +++ .../ownership_object_directory.cc | 10 +- .../ownership_object_directory.h | 40 +++++ src/ray/raylet/scheduling/scheduler_stats.cc | 8 +- src/ray/raylet/scheduling/scheduler_stats.h | 16 +- src/ray/raylet/worker_pool.cc | 20 +-- src/ray/raylet/worker_pool.h | 29 ++++ src/ray/stats/metric_defs.h | 150 ------------------ src/ray/stats/stats_test.cc | 11 +- 16 files changed, 163 insertions(+), 179 deletions(-) diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.cc b/src/ray/gcs/gcs_server/gcs_node_manager.cc index a9a54756cf85..112e45a594d4 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_node_manager.cc @@ -406,7 +406,7 @@ std::shared_ptr GcsNodeManager::RemoveNode( << ", death reason = " << rpc::NodeDeathInfo_Reason_Name(death_info->reason()) << ", death message = " << death_info->reason_message(); // Record stats that there's a new removed node. - stats::NodeFailureTotal.Record(1); + ray_metric_node_failures_total_.Record(1); // Remove from alive nodes. alive_nodes_.erase(iter); // Remove from draining nodes if present. diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.h b/src/ray/gcs/gcs_server/gcs_node_manager.h index 972ecba9c4e6..eb3107fcc4ba 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/ray/gcs/gcs_server/gcs_node_manager.h @@ -286,6 +286,12 @@ class GcsNodeManager : public rpc::NodeInfoHandler { /// If true, node events are exported for Export API bool export_event_write_enabled_ = false; + /// Ray metrics + ray::stats::Count ray_metric_node_failures_total_{ + /*name=*/"node_failure_total", + /*description=*/"Number of node failures that have happened in the cluster.", + /*unit=*/""}; + friend GcsAutoscalerStateManagerTest; friend GcsStateTest; }; diff --git a/src/ray/gcs/gcs_server/gcs_worker_manager.cc b/src/ray/gcs/gcs_server/gcs_worker_manager.cc index bd3b16ff9bdc..3e6962595874 100644 --- a/src/ray/gcs/gcs_server/gcs_worker_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_worker_manager.cc @@ -87,7 +87,7 @@ void GcsWorkerManager::HandleReportWorkerFailure( << "Failed to report worker failure"; } else { if (!IsIntentionalWorkerFailure(worker_failure_data->exit_type())) { - stats::UnintentionalWorkerFailures.Record(1); + ray_metric_unintentional_worker_failures_.Record(1); } // Only publish worker_id and node_id in address as they are the only // fields used by sub clients. diff --git a/src/ray/gcs/gcs_server/gcs_worker_manager.h b/src/ray/gcs/gcs_server/gcs_worker_manager.h index d4858efcbd6e..fb33d80ddb6c 100644 --- a/src/ray/gcs/gcs_server/gcs_worker_manager.h +++ b/src/ray/gcs/gcs_server/gcs_worker_manager.h @@ -86,6 +86,14 @@ class GcsWorkerManager : public rpc::WorkerInfoHandler { /// Tracks the number of occurences of worker crash due to OOM int32_t worker_crash_oom_count_ = 0; + + /// Ray metrics + ray::stats::Count ray_metric_unintentional_worker_failures_{ + /*name=*/"unintentional_worker_failures_total", + /*description=*/ + "Number of worker failures that are not intentional. For example, worker failures " + "due to system related errors.", + /*unit=*/""}; }; } // namespace gcs diff --git a/src/ray/gcs/redis_context.cc b/src/ray/gcs/redis_context.cc index fb684bbd8db6..3ee8c564e978 100644 --- a/src/ray/gcs/redis_context.cc +++ b/src/ray/gcs/redis_context.cc @@ -205,7 +205,7 @@ void RedisRequestContext::RedisResponseFn(redisAsyncContext *async_context, }, "RedisRequestContext.Callback"); auto end_time = absl::Now(); - ray::stats::GcsLatency().Record( + request_cxt->ray_metric_gcs_latency_.Record( absl::ToDoubleMilliseconds(end_time - request_cxt->start_time_)); delete request_cxt; } diff --git a/src/ray/gcs/redis_context.h b/src/ray/gcs/redis_context.h index d21331aac550..cc3ec46f3c43 100644 --- a/src/ray/gcs/redis_context.h +++ b/src/ray/gcs/redis_context.h @@ -24,6 +24,8 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/status.h" #include "ray/gcs/redis_async_context.h" +#include "ray/stats/metric.h" +#include "ray/stats/tag_defs.h" #include "ray/util/exponential_backoff.h" #include "src/ray/protobuf/gcs.pb.h" @@ -127,6 +129,14 @@ struct RedisRequestContext { std::vector redis_cmds_; std::vector argv_; std::vector argc_; + + // Ray metrics + ray::stats::Histogram ray_metric_gcs_latency_{ + "gcs_latency", + "The latency of a GCS (by default Redis) operation.", + "us", + {100, 200, 300, 400, 500, 600, 700, 800, 900, 1000}, + {stats::kCustomKey}}; }; class RedisContext { diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index 405559f58b55..90d8cbe5d93b 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -772,12 +772,12 @@ void ObjectManager::RecordMetrics() { plasma::plasma_store_runner->GetFallbackAllocated()); // Subtract fallback allocated memory. It is tracked separately by // `ObjectStoreFallbackMemory`. - stats::ObjectStoreUsedMemory().Record( + ray_metric_object_store_used_memory_.Record( used_memory_ - plasma::plasma_store_runner->GetFallbackAllocated()); - stats::ObjectStoreFallbackMemory().Record( + ray_metric_object_store_fallback_memory_.Record( plasma::plasma_store_runner->GetFallbackAllocated()); - stats::ObjectStoreLocalObjects().Record(local_objects_.size()); - stats::ObjectManagerPullRequests().Record(pull_manager_->NumObjectPullRequests()); + ray_metric_object_store_local_objects_.Record(local_objects_.size()); + ray_metric_object_manager_pull_requests_.Record(pull_manager_->NumObjectPullRequests()); ray::stats::STATS_object_manager_bytes.Record(num_bytes_pushed_from_plasma_, "PushedFromLocalPlasma"); diff --git a/src/ray/object_manager/object_manager.h b/src/ray/object_manager/object_manager.h index 534ac3fcb7b6..593f2d2b1455 100644 --- a/src/ray/object_manager/object_manager.h +++ b/src/ray/object_manager/object_manager.h @@ -504,6 +504,26 @@ class ObjectManager : public ObjectManagerInterface, /*name=*/"object_store_available_memory", /*description=*/"Amount of memory currently available in the object store.", /*unit=*/"bytes"}; + + ray::stats::Gauge ray_metric_object_store_used_memory_{ + /*name=*/"object_store_used_memory", + /*description=*/"Amount of memory currently occupied in the object store.", + /*unit=*/"bytes"}; + + ray::stats::Gauge ray_metric_object_store_fallback_memory_{ + /*name=*/"object_store_fallback_memory", + /*description=*/"Amount of memory in fallback allocations in the filesystem.", + /*unit=*/"bytes"}; + + ray::stats::Gauge ray_metric_object_store_local_objects_{ + /*name=*/"object_store_num_local_objects", + /*description=*/"Number of objects currently in the object store.", + /*unit=*/"objects"}; + + ray::stats::Gauge ray_metric_object_manager_pull_requests_{ + /*name=*/"object_manager_num_pull_requests", + /*description=*/"Number of active pull requests for objects.", + /*unit=*/"requests"}; }; } // namespace ray diff --git a/src/ray/object_manager/ownership_object_directory.cc b/src/ray/object_manager/ownership_object_directory.cc index 53d9ba0eed0c..0abe8b4edd8e 100644 --- a/src/ray/object_manager/ownership_object_directory.cc +++ b/src/ray/object_manager/ownership_object_directory.cc @@ -472,34 +472,34 @@ void OwnershipBasedObjectDirectory::HandleNodeRemoved(const NodeID &node_id) { } void OwnershipBasedObjectDirectory::RecordMetrics(uint64_t duration_ms) { - stats::ObjectDirectoryLocationSubscriptions.Record(listeners_.size()); + ray_metric_object_directory_location_subscriptions_.Record(listeners_.size()); // Record number of object location updates per second. metrics_num_object_location_updates_per_second_ = static_cast(metrics_num_object_location_updates_) * (1000.0 / static_cast(duration_ms)); - stats::ObjectDirectoryLocationUpdates.Record( + ray_metric_object_directory_location_updates_.Record( metrics_num_object_location_updates_per_second_); metrics_num_object_location_updates_ = 0; // Record number of object location lookups per second. metrics_num_object_location_lookups_per_second_ = static_cast(metrics_num_object_location_lookups_) * (1000.0 / static_cast(duration_ms)); - stats::ObjectDirectoryLocationLookups.Record( + ray_metric_object_directory_location_lookups_.Record( metrics_num_object_location_lookups_per_second_); metrics_num_object_location_lookups_ = 0; // Record number of object locations added per second. metrics_num_object_locations_added_per_second_ = static_cast(metrics_num_object_locations_added_) * (1000.0 / static_cast(duration_ms)); - stats::ObjectDirectoryAddedLocations.Record( + ray_metric_object_directory_location_added_.Record( metrics_num_object_locations_added_per_second_); metrics_num_object_locations_added_ = 0; // Record number of object locations removed per second. metrics_num_object_locations_removed_per_second_ = static_cast(metrics_num_object_locations_removed_) * (1000.0 / static_cast(duration_ms)); - stats::ObjectDirectoryRemovedLocations.Record( + ray_metric_object_directory_location_removed_.Record( metrics_num_object_locations_removed_per_second_); metrics_num_object_locations_removed_ = 0; } diff --git a/src/ray/object_manager/ownership_object_directory.h b/src/ray/object_manager/ownership_object_directory.h index 556544ff109d..a054a7ef68f5 100644 --- a/src/ray/object_manager/ownership_object_directory.h +++ b/src/ray/object_manager/ownership_object_directory.h @@ -28,6 +28,7 @@ #include "ray/pubsub/subscriber.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" +#include "ray/stats/metric.h" namespace ray { @@ -174,6 +175,45 @@ class OwnershipBasedObjectDirectory : public IObjectDirectory { uint64_t cum_metrics_num_object_location_updates_ = 0; + /// Ray metrics + ray::stats::Gauge ray_metric_object_directory_location_subscriptions_{ + /*name=*/"object_directory_subscriptions", + /*description=*/ + "Number of object location subscriptions. If this is high, the raylet is " + "attempting " + "to pull a lot of objects.", + /*unit=*/"subscriptions"}; + + ray::stats::Gauge ray_metric_object_directory_location_updates_{ + /*name=*/"object_directory_updates", + /*description=*/ + "Number of object location updates per second. If this is high, the raylet is " + "attempting to pull a lot of objects and/or the locations for objects are " + "frequently " + "changing (e.g. due to many object copies or evictions).", + /*unit=*/"updates"}; + + ray::stats::Gauge ray_metric_object_directory_location_lookups_{ + /*name=*/"object_directory_lookups", + /*description=*/ + "Number of object location lookups per second. If this is high, the raylet is " + "waiting on a lot of objects.", + /*unit=*/"lookups"}; + + ray::stats::Gauge ray_metric_object_directory_location_added_{ + /*name=*/"object_directory_added_locations", + /*description=*/ + "Number of object locations added per second. If this is high, a lot of objects " + "have been added on this node.", + /*unit=*/"additions"}; + + ray::stats::Gauge ray_metric_object_directory_location_removed_{ + /*name=*/"object_directory_removed_locations", + /*description=*/ + "Number of object locations removed per second. If this is high, a lot of objects " + "have been removed from this node.", + /*unit=*/"removals"}; + friend class OwnershipBasedObjectDirectoryTest; }; diff --git a/src/ray/raylet/scheduling/scheduler_stats.cc b/src/ray/raylet/scheduling/scheduler_stats.cc index c80a44fa8b2b..4b62d8e5fae4 100644 --- a/src/ray/raylet/scheduling/scheduler_stats.cc +++ b/src/ray/raylet/scheduling/scheduler_stats.cc @@ -116,14 +116,14 @@ void SchedulerStats::ComputeStats() { num_tasks_to_dispatch_ = num_tasks_to_dispatch; } -void SchedulerStats::RecordMetrics() const { +void SchedulerStats::RecordMetrics() { /// This method intentionally doesn't call ComputeStats() because /// that function is expensive. ComputeStats is called by ComputeAndReportDebugStr /// method and they are always periodically called by node manager. - stats::NumSpilledTasks.Record(metric_tasks_spilled_ + - local_task_manager_.GetNumTaskSpilled()); + ray_metric_num_spilled_tasks_.Record(metric_tasks_spilled_ + + local_task_manager_.GetNumTaskSpilled()); local_task_manager_.RecordMetrics(); - stats::NumInfeasibleSchedulingClasses.Record( + ray_metric_num_infeasible_scheduling_classes_.Record( cluster_task_manager_.infeasible_tasks_.size()); /// Worker startup failure ray::stats::STATS_scheduler_failed_worker_startup_total.Record( diff --git a/src/ray/raylet/scheduling/scheduler_stats.h b/src/ray/raylet/scheduling/scheduler_stats.h index c71f1fb8cab4..97dc0f9a1858 100644 --- a/src/ray/raylet/scheduling/scheduler_stats.h +++ b/src/ray/raylet/scheduling/scheduler_stats.h @@ -21,6 +21,7 @@ #include "ray/common/task/task_spec.h" #include "ray/raylet/scheduling/internal.h" #include "ray/raylet/scheduling/local_task_manager_interface.h" +#include "ray/stats/metric.h" namespace ray { namespace raylet { @@ -34,7 +35,7 @@ class SchedulerStats { const ILocalTaskManager &local_task_manager); // Report metrics doesn't recompute the stats. - void RecordMetrics() const; + void RecordMetrics(); // Recompute the stats and report the result as string. std::string ComputeAndReportDebugStr(); @@ -78,6 +79,19 @@ class SchedulerStats { int64_t num_tasks_to_schedule_ = 0; /// Number of tasks to dispatch. int64_t num_tasks_to_dispatch_ = 0; + + /// Ray metrics + ray::stats::Gauge ray_metric_num_spilled_tasks_{ + /*name=*/"internal_num_spilled_tasks", + /*description=*/ + "The cumulative number of lease requeusts that this raylet has spilled to other " + "raylets.", + /*unit=*/"tasks"}; + + ray::stats::Gauge ray_metric_num_infeasible_scheduling_classes_{ + /*name=*/"internal_num_infeasible_scheduling_classes", + /*description=*/"The number of unique scheduling classes that are infeasible.", + /*unit=*/"tasks"}; }; } // namespace raylet diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index 7058b96847b3..6162847962f3 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -129,11 +129,11 @@ WorkerPool::WorkerPool(instrumented_io_context &io_service, // We need to record so that the metric exists. This way, we report that 0 // processes have started before a task runs on the node (as opposed to the // metric not existing at all). - stats::NumWorkersStarted.Record(0); - stats::NumWorkersStartedFromCache.Record(0); - stats::NumCachedWorkersSkippedJobMismatch.Record(0); - stats::NumCachedWorkersSkippedDynamicOptionsMismatch.Record(0); - stats::NumCachedWorkersSkippedRuntimeEnvironmentMismatch.Record(0); + ray_metric_num_workers_started_.Record(0); + ray_metric_num_workers_started_from_cache_.Record(0); + ray_metric_num_cached_workers_skipped_job_mismatch_.Record(0); + ray_metric_num_cached_workers_skipped_dynamic_options_mismatch_.Record(0); + ray_metric_num_cached_workers_skipped_runtime_environment_mismatch_.Record(0); // We used to ignore SIGCHLD here. The code is moved to raylet main.cc to support the // subreaper feature. for (const auto &entry : worker_commands) { @@ -529,7 +529,7 @@ std::tuple WorkerPool::StartWorkerProcess( auto start = std::chrono::high_resolution_clock::now(); // Start a process and measure the startup time. Process proc = StartProcess(worker_command_args, env); - stats::NumWorkersStarted.Record(1); + ray_metric_num_workers_started_.Record(1); RAY_LOG(INFO) << "Started worker process with pid " << proc.GetId() << ", the token is " << worker_startup_token_counter_; if (!IsIOWorkerType(worker_type)) { @@ -1420,11 +1420,11 @@ std::shared_ptr WorkerPool::FindAndPopIdleWorker( } skip_reason_count[reason]++; if (reason == WorkerUnfitForTaskReason::DYNAMIC_OPTIONS_MISMATCH) { - stats::NumCachedWorkersSkippedDynamicOptionsMismatch.Record(1); + ray_metric_num_cached_workers_skipped_dynamic_options_mismatch_.Record(1); } else if (reason == WorkerUnfitForTaskReason::RUNTIME_ENV_MISMATCH) { - stats::NumCachedWorkersSkippedRuntimeEnvironmentMismatch.Record(1); + ray_metric_num_cached_workers_skipped_runtime_environment_mismatch_.Record(1); } else if (reason == WorkerUnfitForTaskReason::ROOT_MISMATCH) { - stats::NumCachedWorkersSkippedJobMismatch.Record(1); + ray_metric_num_cached_workers_skipped_job_mismatch_.Record(1); } return false; }; @@ -1463,7 +1463,7 @@ void WorkerPool::PopWorker(std::shared_ptr pop_worker_request) } RAY_CHECK(worker->GetAssignedJobId().IsNil() || worker->GetAssignedJobId() == pop_worker_request->job_id); - stats::NumWorkersStartedFromCache.Record(1); + ray_metric_num_workers_started_from_cache_.Record(1); PopWorkerCallbackAsync(pop_worker_request->callback, worker, PopWorkerStatus::OK); } diff --git a/src/ray/raylet/worker_pool.h b/src/ray/raylet/worker_pool.h index 36ba3601168a..2f8b02e3b588 100644 --- a/src/ray/raylet/worker_pool.h +++ b/src/ray/raylet/worker_pool.h @@ -41,6 +41,7 @@ #include "ray/ipc/client_connection.h" #include "ray/raylet/runtime_env_agent_client.h" #include "ray/raylet/worker.h" +#include "ray/stats/metric.h" namespace ray { @@ -916,6 +917,34 @@ class WorkerPool : public WorkerPoolInterface { // cgroup after it is created. bool enable_resource_isolation_ = false; + /// Ray metrics + ray::stats::Sum ray_metric_num_workers_started_{ + /*name=*/"internal_num_processes_started", + /*description=*/"The total number of worker processes the worker pool has created.", + /*unit=*/"processes"}; + + ray::stats::Sum ray_metric_num_cached_workers_skipped_job_mismatch_{ + /*name=*/"internal_num_processes_skipped_job_mismatch", + /*description=*/"The total number of cached workers skipped due to job mismatch.", + /*unit=*/"workers"}; + + ray::stats::Sum ray_metric_num_cached_workers_skipped_runtime_environment_mismatch_{ + /*name=*/"internal_num_processes_skipped_runtime_environment_mismatch", + /*description=*/ + "The total number of cached workers skipped due to runtime environment mismatch.", + /*unit=*/"workers"}; + + ray::stats::Sum ray_metric_num_cached_workers_skipped_dynamic_options_mismatch_{ + /*name=*/"internal_num_processes_skipped_dynamic_options_mismatch", + /*description=*/ + "The total number of cached workers skipped due to dynamic options mismatch.", + /*unit=*/"workers"}; + + ray::stats::Sum ray_metric_num_workers_started_from_cache_{ + /*name=*/"internal_num_processes_started_from_cache", + /*description=*/"The total number of workers started from a cached worker process.", + /*unit=*/"workers"}; + friend class WorkerPoolTest; friend class WorkerPoolDriverRegisteredTest; }; diff --git a/src/ray/stats/metric_defs.h b/src/ray/stats/metric_defs.h index ac07a5de106e..dfe22aa345c4 100644 --- a/src/ray/stats/metric_defs.h +++ b/src/ray/stats/metric_defs.h @@ -135,156 +135,6 @@ DECLARE_stats(memory_manager_worker_eviction_total); /// Core Worker Task Manager DECLARE_stats(total_lineage_bytes); -/// The below items are legacy implementation of metrics. -/// TODO(sang): Use DEFINE_stats instead. - -/// -/// Common -/// -/// RPC -static Histogram GcsLatency("gcs_latency", - "The latency of a GCS (by default Redis) operation.", - "us", - {100, 200, 300, 400, 500, 600, 700, 800, 900, 1000}, - {kCustomKey}); - -/// -/// Raylet Metrics -/// - -/// Raylet Resource Manager -static Gauge TestMetrics("local_available_resource", - "The available resources on this node.", - "", - {kResourceNameKey}); - -static Gauge LocalTotalResource("local_total_resource", - "The total resources on this node.", - "", - {kResourceNameKey}); - -/// Object Manager. -static Gauge ObjectStoreUsedMemory( - "object_store_used_memory", - "Amount of memory currently occupied in the object store.", - "bytes"); - -static Gauge ObjectStoreFallbackMemory( - "object_store_fallback_memory", - "Amount of memory in fallback allocations in the filesystem.", - "bytes"); - -static Gauge ObjectStoreLocalObjects("object_store_num_local_objects", - "Number of objects currently in the object store.", - "objects"); - -static Gauge ObjectManagerPullRequests("object_manager_num_pull_requests", - "Number of active pull requests for objects.", - "requests"); - -/// Object Directory. -static Gauge ObjectDirectoryLocationSubscriptions( - "object_directory_subscriptions", - "Number of object location subscriptions. If this is high, the raylet is attempting " - "to pull a lot of objects.", - "subscriptions"); - -static Gauge ObjectDirectoryLocationUpdates( - "object_directory_updates", - "Number of object location updates per second., If this is high, the raylet is " - "attempting to pull a lot of objects and/or the locations for objects are frequently " - "changing (e.g. due to many object copies or evictions).", - "updates"); - -static Gauge ObjectDirectoryLocationLookups( - "object_directory_lookups", - "Number of object location lookups per second. If this is high, the raylet is " - "waiting on a lot of objects.", - "lookups"); - -static Gauge ObjectDirectoryAddedLocations( - "object_directory_added_locations", - "Number of object locations added per second., If this is high, a lot of objects " - "have been added on this node.", - "additions"); - -static Gauge ObjectDirectoryRemovedLocations( - "object_directory_removed_locations", - "Number of object locations removed per second. If this is high, a lot of objects " - "have been removed from this node.", - "removals"); - -static Sum NumWorkersStarted( - "internal_num_processes_started", - "The total number of worker processes the worker pool has created.", - "processes"); - -static Sum NumCachedWorkersSkippedJobMismatch( - "internal_num_processes_skipped_job_mismatch", - "The total number of cached workers skipped due to job mismatch.", - "workers"); - -static Sum NumCachedWorkersSkippedRuntimeEnvironmentMismatch( - "internal_num_processes_skipped_runtime_environment_mismatch", - "The total number of cached workers skipped due to runtime environment mismatch.", - "workers"); - -static Sum NumCachedWorkersSkippedDynamicOptionsMismatch( - "internal_num_processes_skipped_dynamic_options_mismatch", - "The total number of cached workers skipped due to dynamic options mismatch.", - "workers"); - -static Sum NumWorkersStartedFromCache( - "internal_num_processes_started_from_cache", - "The total number of workers started from a cached worker process.", - "workers"); - -static Gauge NumSpilledTasks("internal_num_spilled_tasks", - "The cumulative number of lease requeusts that this raylet " - "has spilled to other raylets.", - "tasks"); - -static Gauge NumInfeasibleSchedulingClasses( - "internal_num_infeasible_scheduling_classes", - "The number of unique scheduling classes that are infeasible.", - "tasks"); - -/// -/// GCS Server Metrics -/// - -/// Workers -static Count UnintentionalWorkerFailures( - "unintentional_worker_failures_total", - "Number of worker failures that are not intentional. For example, worker failures " - "due to system related errors.", - ""); - -/// Nodes -static Count NodeFailureTotal( - "node_failure_total", - "Number of node failures that have happened in the cluster.", - ""); - -/// Resources -static Histogram OutboundHeartbeatSizeKB("outbound_heartbeat_size_kb", - "Outbound heartbeat payload size", - "kb", - {10, 50, 100, 1000, 10000, 100000}); - -static Histogram GcsUpdateResourceUsageTime( - "gcs_update_resource_usage_time", - "The average RTT of a UpdateResourceUsage RPC.", - "ms", - {1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000}, - {kCustomKey}); - -/// Testing -static Gauge LiveActors("live_actors", "Number of live actors.", "actors"); -static Gauge RestartingActors("restarting_actors", - "Number of restarting actors.", - "actors"); - } // namespace stats } // namespace ray diff --git a/src/ray/stats/stats_test.cc b/src/ray/stats/stats_test.cc index a333ba335313..47d2c22c7663 100644 --- a/src/ray/stats/stats_test.cc +++ b/src/ray/stats/stats_test.cc @@ -22,7 +22,8 @@ #include "absl/memory/memory.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "ray/stats/metric_defs.h" +#include "ray/stats/metric.h" +#include "ray/stats/tag_defs.h" DEFINE_stats(test_hist, "TestStats", @@ -93,12 +94,18 @@ class StatsTest : public ::testing::Test { virtual void TearDown() override { Shutdown(); } void Shutdown() { ray::stats::Shutdown(); } + + protected: + ray::stats::Gauge ray_metric_test_metrics_{"local_available_resource", + "The available resources on this node.", + "", + {stats::kResourceNameKey}}; }; TEST_F(StatsTest, F) { for (size_t i = 0; i < 20; ++i) { std::this_thread::sleep_for(std::chrono::milliseconds(50)); - stats::TestMetrics().Record(2345); + ray_metric_test_metrics_.Record(2345); } } From 32304ab50a5f1c94504d2610a338fef1e84ecef7 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Wed, 13 Aug 2025 13:21:41 -0700 Subject: [PATCH 084/634] [release test] remove "multi" test frequency (#55561) not used anywhere any more Signed-off-by: Lonnie Liu --- release/ray_release/buildkite/settings.py | 2 -- release/ray_release/schema.json | 2 -- release/ray_release/tests/test_buildkite.py | 4 ++-- release/ray_release/tests/test_config.py | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/release/ray_release/buildkite/settings.py b/release/ray_release/buildkite/settings.py index da8e88245443..17ab6f9db657 100644 --- a/release/ray_release/buildkite/settings.py +++ b/release/ray_release/buildkite/settings.py @@ -11,7 +11,6 @@ class Frequency(enum.Enum): MANUAL = enum.auto() ANY = enum.auto() - MULTI = enum.auto() NIGHTLY = enum.auto() NIGHTLY_3x = enum.auto() WEEKLY = enum.auto() @@ -22,7 +21,6 @@ class Frequency(enum.Enum): "manual": Frequency.MANUAL, "any": Frequency.ANY, "any-smoke": Frequency.ANY, - "multi": Frequency.MULTI, "nightly": Frequency.NIGHTLY, "nightly-3x": Frequency.NIGHTLY_3x, "weekly": Frequency.WEEKLY, diff --git a/release/ray_release/schema.json b/release/ray_release/schema.json index 7be825405dea..7ad30d9c3ce1 100644 --- a/release/ray_release/schema.json +++ b/release/ray_release/schema.json @@ -36,7 +36,6 @@ "type": "string", "enum": [ "manual", - "multi", "nightly", "nightly-3x", "weekly", @@ -201,7 +200,6 @@ "type": "string", "enum": [ "manual", - "multi", "nightly", "nightly-3x", "weekly", diff --git a/release/ray_release/tests/test_buildkite.py b/release/ray_release/tests/test_buildkite.py index f8fbf964a8bb..79e709cfb434 100644 --- a/release/ray_release/tests/test_buildkite.py +++ b/release/ray_release/tests/test_buildkite.py @@ -411,7 +411,7 @@ def testFilterTests(self, *args): { "name": "other_2", "frequency": "nightly", - "smoke_test": {"frequency": "multi"}, + "smoke_test": {"frequency": "manual"}, "team": "team_2", "run": {"type": "job"}, } @@ -586,7 +586,7 @@ def testGetStep(self): "name": "test", "frequency": "nightly", "run": {"script": "test_script.py"}, - "smoke_test": {"frequency": "multi"}, + "smoke_test": {"frequency": "nightly"}, } ) diff --git a/release/ray_release/tests/test_config.py b/release/ray_release/tests/test_config.py index b0ec1c4b1a2c..c815dfc1bba6 100644 --- a/release/ray_release/tests/test_config.py +++ b/release/ray_release/tests/test_config.py @@ -37,7 +37,7 @@ "wait_for_nodes": {"num_nodes": 2, "timeout": 100}, "type": "client", }, - "smoke_test": {"run": {"timeout": 20}, "frequency": "multi"}, + "smoke_test": {"run": {"timeout": 20}, "frequency": "nightly"}, "alert": "default", } From 52bef607fd4349e70a1874fb2d6a8a9f6d447111 Mon Sep 17 00:00:00 2001 From: Matvei Pashkovskii Date: Thu, 14 Aug 2025 00:10:21 +0300 Subject: [PATCH 085/634] [Serve.llm] Add LMCacheConnectorV1 support for kv_transfer_config (#54579) Signed-off-by: Matvei Pashkovskii Signed-off-by: Kourosh Hakhamaneshi Co-authored-by: Kourosh Hakhamaneshi --- .../examples/rayserve-deepseek-example.md | 4 +- .../examples/rayserve-llm-example.md | 4 +- .../pd_dissagregation/lmcache/mooncake.yaml | 17 ++ .../lmcache/nixl/decoder.yaml | 12 ++ .../lmcache/nixl/prefiller.yaml | 12 ++ .../lmcache_mooncake_example.yaml | 34 ++++ .../lmcache_nixl_example.yaml | 45 +++++ .../pd_dissagregation/nixl_example.yaml | 33 ++++ doc/source/serve/index.md | 2 +- doc/source/serve/llm/index.md | 61 ++++++ doc/source/serve/llm/pd-dissagregation.md | 179 ++++++++++++++++++ .../llm/{serving-llms.rst => quick-start.rst} | 61 +----- .../_internal/serve/configs/server_models.py | 25 +++ .../llm/vllm/kv_transfer_backends/__init__.py | 16 ++ .../llm/vllm/kv_transfer_backends/base.py | 32 ++++ .../lmcache_connector_v1.py | 61 ++++++ .../kv_transfer_backends/nixl_connector.py | 45 +++++ .../serve/deployments/llm/vllm/vllm_engine.py | 32 +--- .../prefill_decode_disagg.py | 3 +- .../llm/vllm/kv_transfer_backends/__init__.py | 1 + .../test_lmcache_connector_v1.py | 86 +++++++++ .../test_nixl_connector.py | 48 +++++ .../test_prefill_decode_disagg.py | 3 +- .../test_prefill_decode_disagg_gpu.py | 12 +- .../serve/configs/lmcache/decoder.yaml | 12 ++ .../serve/configs/lmcache/prefiller.yaml | 12 ++ ...a_3dot1_8b_quantized_tp1_2p6d_lmcache.yaml | 52 +++++ .../ray_release/byod/byod_llm_lmcache_test.sh | 7 + release/release_tests.yaml | 20 ++ 29 files changed, 831 insertions(+), 100 deletions(-) create mode 100644 doc/source/serve/doc_code/pd_dissagregation/lmcache/mooncake.yaml create mode 100644 doc/source/serve/doc_code/pd_dissagregation/lmcache/nixl/decoder.yaml create mode 100644 doc/source/serve/doc_code/pd_dissagregation/lmcache/nixl/prefiller.yaml create mode 100644 doc/source/serve/doc_code/pd_dissagregation/lmcache_mooncake_example.yaml create mode 100644 doc/source/serve/doc_code/pd_dissagregation/lmcache_nixl_example.yaml create mode 100644 doc/source/serve/doc_code/pd_dissagregation/nixl_example.yaml create mode 100644 doc/source/serve/llm/index.md create mode 100644 doc/source/serve/llm/pd-dissagregation.md rename doc/source/serve/llm/{serving-llms.rst => quick-start.rst} (94%) create mode 100644 python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/__init__.py create mode 100644 python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/base.py create mode 100644 python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/lmcache_connector_v1.py create mode 100644 python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/nixl_connector.py create mode 100644 python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/__init__.py create mode 100644 python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/test_lmcache_connector_v1.py create mode 100644 python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/test_nixl_connector.py create mode 100644 release/llm_tests/serve/configs/lmcache/decoder.yaml create mode 100644 release/llm_tests/serve/configs/lmcache/prefiller.yaml create mode 100644 release/llm_tests/serve/configs/serve_llama_3dot1_8b_quantized_tp1_2p6d_lmcache.yaml create mode 100755 release/ray_release/byod/byod_llm_lmcache_test.sh diff --git a/doc/source/cluster/kubernetes/examples/rayserve-deepseek-example.md b/doc/source/cluster/kubernetes/examples/rayserve-deepseek-example.md index 3b9fff613fd3..8f147de21742 100644 --- a/doc/source/cluster/kubernetes/examples/rayserve-deepseek-example.md +++ b/doc/source/cluster/kubernetes/examples/rayserve-deepseek-example.md @@ -2,7 +2,7 @@ # Serve Deepseek R1 using Ray Serve LLM -This guide provides a step-by-step guide for deploying a Large Language Model (LLM) using Ray Serve LLM on Kubernetes. Leveraging KubeRay, Ray Serve, and vLLM, this guide deploys the `deepseek-ai/DeepSeek-R1` model from Hugging Face, enabling scalable, efficient, and OpenAI-compatible LLM serving within a Kubernetes environment. See [Serving LLMs](serving_llms) for information on Ray Serve LLM. +This guide provides a step-by-step guide for deploying a Large Language Model (LLM) using Ray Serve LLM on Kubernetes. Leveraging KubeRay, Ray Serve, and vLLM, this guide deploys the `deepseek-ai/DeepSeek-R1` model from Hugging Face, enabling scalable, efficient, and OpenAI-compatible LLM serving within a Kubernetes environment. See [Serving LLMs](serving-llms) for information on Ray Serve LLM. ## Prerequisites A DeepSeek model requires 2 nodes, each equipped with 8 H100 80 GB GPUs. @@ -108,7 +108,7 @@ In particular, this configuration loads the model from `deepseek-ai/DeepSeek-R1` This setting enables pipeline parallelism, dividing the model's entire set of layers into 2 sequential stages. Adjust this variable according to cluster worker node numbers. -The `deployment_config` section sets the desired number of engine replicas. See [Serving LLMs](serving_llms) and the [Ray Serve config documentation](serve-in-production-config-file) for more information. +The `deployment_config` section sets the desired number of engine replicas. See [Serving LLMs](serving-llms) and the [Ray Serve config documentation](serve-in-production-config-file) for more information. Wait for the RayService resource to become healthy. You can confirm its status by running the following command: ```sh diff --git a/doc/source/cluster/kubernetes/examples/rayserve-llm-example.md b/doc/source/cluster/kubernetes/examples/rayserve-llm-example.md index b3aa86b24997..77e48dc45064 100644 --- a/doc/source/cluster/kubernetes/examples/rayserve-llm-example.md +++ b/doc/source/cluster/kubernetes/examples/rayserve-llm-example.md @@ -2,7 +2,7 @@ # Serve a Large Language Model using Ray Serve LLM on Kubernetes -This guide provides a step-by-step guide for deploying a Large Language Model (LLM) using Ray Serve LLM on Kubernetes. Leveraging KubeRay, Ray Serve, and vLLM, this guide deploys the `Qwen/Qwen2.5-7B-Instruct` model from Hugging Face, enabling scalable, efficient, and OpenAI-compatible LLM serving within a Kubernetes environment. See [Serving LLMs](serving_llms) for information on Ray Serve LLM. +This guide provides a step-by-step guide for deploying a Large Language Model (LLM) using Ray Serve LLM on Kubernetes. Leveraging KubeRay, Ray Serve, and vLLM, this guide deploys the `Qwen/Qwen2.5-7B-Instruct` model from Hugging Face, enabling scalable, efficient, and OpenAI-compatible LLM serving within a Kubernetes environment. See [Serving LLMs](serving-llms) for information on Ray Serve LLM. ## Prerequisites @@ -72,7 +72,7 @@ serveConfigV2: | max_ongoing_requests: 128 ``` -In particular, this configuration loads the model from `Qwen/Qwen2.5-7B-Instruct` and sets its `model_id` to `qwen2.5-7b-instruct`. The `LLMDeployment` initializes the underlying LLM engine using the `engine_kwargs` field. The `deployment_config` section sets the desired number of engine replicas. By default, each replica requires one GPU. See [Serving LLMs](serving_llms) and the [Ray Serve config documentation](serve-in-production-config-file) for more information. +In particular, this configuration loads the model from `Qwen/Qwen2.5-7B-Instruct` and sets its `model_id` to `qwen2.5-7b-instruct`. The `LLMDeployment` initializes the underlying LLM engine using the `engine_kwargs` field. The `deployment_config` section sets the desired number of engine replicas. By default, each replica requires one GPU. See [Serving LLMs](serving-llms) and the [Ray Serve config documentation](serve-in-production-config-file) for more information. Wait for the RayService resource to become healthy. You can confirm its status by running the following command: ```sh diff --git a/doc/source/serve/doc_code/pd_dissagregation/lmcache/mooncake.yaml b/doc/source/serve/doc_code/pd_dissagregation/lmcache/mooncake.yaml new file mode 100644 index 000000000000..e6430eff7549 --- /dev/null +++ b/doc/source/serve/doc_code/pd_dissagregation/lmcache/mooncake.yaml @@ -0,0 +1,17 @@ +# LMCache configuration for Mooncake store backend +chunk_size: 256 +local_device: "cpu" +remote_url: "mooncakestore://storage-server:49999/" +remote_serde: "naive" +pipelined_backend: false +local_cpu: false +max_local_cpu_size: 5 +extra_config: + local_hostname: "compute-node-001" + metadata_server: "etcd://metadata-server:2379" + protocol: "rdma" + device_name: "rdma0" + master_server_address: "storage-server:49999" + global_segment_size: 3355443200 # 3.125 GB + local_buffer_size: 1073741824 # 1 GB + transfer_timeout: 1 diff --git a/doc/source/serve/doc_code/pd_dissagregation/lmcache/nixl/decoder.yaml b/doc/source/serve/doc_code/pd_dissagregation/lmcache/nixl/decoder.yaml new file mode 100644 index 000000000000..34e22d421997 --- /dev/null +++ b/doc/source/serve/doc_code/pd_dissagregation/lmcache/nixl/decoder.yaml @@ -0,0 +1,12 @@ +local_cpu: False +max_local_cpu_size: 0 +max_local_disk_size: 0 +remote_serde: NULL + +enable_nixl: True +nixl_role: "receiver" +nixl_receiver_host: "localhost" +nixl_receiver_port: 55555 +nixl_buffer_size: 1073741824 # 1GB +nixl_buffer_device: "cuda" +nixl_enable_gc: True diff --git a/doc/source/serve/doc_code/pd_dissagregation/lmcache/nixl/prefiller.yaml b/doc/source/serve/doc_code/pd_dissagregation/lmcache/nixl/prefiller.yaml new file mode 100644 index 000000000000..544551b78a78 --- /dev/null +++ b/doc/source/serve/doc_code/pd_dissagregation/lmcache/nixl/prefiller.yaml @@ -0,0 +1,12 @@ +local_cpu: False +max_local_cpu_size: 0 +max_local_disk_size: 0 +remote_serde: NULL + +enable_nixl: True +nixl_role: "sender" +nixl_receiver_host: "localhost" +nixl_receiver_port: 55555 +nixl_buffer_size: 1073741824 # 1GB +nixl_buffer_device: "cuda" +nixl_enable_gc: True diff --git a/doc/source/serve/doc_code/pd_dissagregation/lmcache_mooncake_example.yaml b/doc/source/serve/doc_code/pd_dissagregation/lmcache_mooncake_example.yaml new file mode 100644 index 000000000000..d7702cbf5d5b --- /dev/null +++ b/doc/source/serve/doc_code/pd_dissagregation/lmcache_mooncake_example.yaml @@ -0,0 +1,34 @@ +# Example: LMCacheConnectorV1 with Mooncake store configuration + +applications: + - args: + prefill_config: + model_loading_config: + model_id: meta-llama/Llama-3.1-8B-Instruct + engine_kwargs: + kv_transfer_config: &kv_transfer_config + kv_connector: LMCacheConnectorV1 + kv_role: kv_both + deployment_config: + autoscaling_config: + min_replicas: 2 + max_replicas: 2 + runtime_env: &runtime_env + env_vars: + LMCACHE_CONFIG_FILE: lmcache_mooncake.yaml + LMCACHE_USE_EXPERIMENTAL: "True" + + decode_config: + model_loading_config: + model_id: meta-llama/Llama-3.1-8B-Instruct + engine_kwargs: + kv_transfer_config: *kv_transfer_config + deployment_config: + autoscaling_config: + min_replicas: 1 + max_replicas: 1 + runtime_env: *runtime_env + + import_path: ray.serve.llm:build_pd_openai_app + name: pd-disaggregation-lmcache-mooncake + route_prefix: "/" diff --git a/doc/source/serve/doc_code/pd_dissagregation/lmcache_nixl_example.yaml b/doc/source/serve/doc_code/pd_dissagregation/lmcache_nixl_example.yaml new file mode 100644 index 000000000000..4284627055ae --- /dev/null +++ b/doc/source/serve/doc_code/pd_dissagregation/lmcache_nixl_example.yaml @@ -0,0 +1,45 @@ +# Example: LMCacheConnectorV1 with NIXL backend configuration + +applications: + - args: + prefill_config: + model_loading_config: + model_id: meta-llama/Llama-3.1-8B-Instruct + engine_kwargs: + kv_transfer_config: + kv_connector: LMCacheConnectorV1 + kv_role: kv_producer + kv_connector_extra_config: + discard_partial_chunks: false + lmcache_rpc_port: producer1 + deployment_config: + autoscaling_config: + min_replicas: 2 + max_replicas: 2 + runtime_env: + env_vars: + LMCACHE_CONFIG_FILE: lmcache_prefiller.yaml + LMCACHE_USE_EXPERIMENTAL: "True" + + decode_config: + model_loading_config: + model_id: meta-llama/Llama-3.1-8B-Instruct + engine_kwargs: + kv_transfer_config: + kv_connector: LMCacheConnectorV1 + kv_role: kv_consumer + kv_connector_extra_config: + discard_partial_chunks: false + lmcache_rpc_port: consumer1 + deployment_config: + autoscaling_config: + min_replicas: 6 + max_replicas: 6 + runtime_env: + env_vars: + LMCACHE_CONFIG_FILE: lmcache_decoder.yaml + LMCACHE_USE_EXPERIMENTAL: "True" + + import_path: ray.serve.llm:build_pd_openai_app + name: pd-disaggregation-lmcache-nixl + route_prefix: "/" diff --git a/doc/source/serve/doc_code/pd_dissagregation/nixl_example.yaml b/doc/source/serve/doc_code/pd_dissagregation/nixl_example.yaml new file mode 100644 index 000000000000..ac30e0e7f8ef --- /dev/null +++ b/doc/source/serve/doc_code/pd_dissagregation/nixl_example.yaml @@ -0,0 +1,33 @@ +# Example: Basic NIXLConnector configuration for prefill/decode disaggregation + +applications: + - args: + prefill_config: + model_loading_config: + model_id: meta-llama/Llama-3.1-8B-Instruct + engine_kwargs: + kv_transfer_config: + kv_connector: NixlConnector + kv_role: kv_producer + engine_id: engine1 + deployment_config: + autoscaling_config: + min_replicas: 2 + max_replicas: 4 + + decode_config: + model_loading_config: + model_id: meta-llama/Llama-3.1-8B-Instruct + engine_kwargs: + kv_transfer_config: + kv_connector: NixlConnector + kv_role: kv_consumer + engine_id: engine2 + deployment_config: + autoscaling_config: + min_replicas: 6 + max_replicas: 10 + + import_path: ray.serve.llm:build_pd_openai_app + name: pd-disaggregation-nixl + route_prefix: "/" diff --git a/doc/source/serve/index.md b/doc/source/serve/index.md index 7c581ab947d4..5cc34ad54f61 100644 --- a/doc/source/serve/index.md +++ b/doc/source/serve/index.md @@ -13,7 +13,7 @@ multi-app model-multiplexing configure-serve-deployment http-guide -Serving LLMs +Serving LLMs Production Guide monitoring resource-allocation diff --git a/doc/source/serve/llm/index.md b/doc/source/serve/llm/index.md new file mode 100644 index 000000000000..358704b9d101 --- /dev/null +++ b/doc/source/serve/llm/index.md @@ -0,0 +1,61 @@ +(serving-llms)= + +# Serving LLMs + +Ray Serve LLM APIs allow users to deploy multiple LLM models together with a familiar Ray Serve API, while providing compatibility with the OpenAI API. + +## Features + +- ⚡️ Automatic scaling and load balancing +- 🌐 Unified multi-node multi-model deployment +- 🔌 OpenAI compatible +- 🔄 Multi-LoRA support with shared base models +- 🚀 Engine agnostic architecture (i.e. vLLM, SGLang, etc) + +## Requirements + +```bash +pip install ray[serve,llm]>=2.43.0 vllm>=0.7.2 + +# Suggested dependencies when using vllm 0.7.2: +pip install xgrammar==0.1.11 pynvml==12.0.0 +``` + +## Key Components + +The ray.serve.llm module provides two key deployment types for serving LLMs: + +### LLMServer + +The LLMServer sets up and manages the vLLM engine for model serving. It can be used standalone or combined with your own custom Ray Serve deployments. + +### OpenAiIngress + +This deployment provides an OpenAI-compatible FastAPI ingress and routes traffic to the appropriate model for multi-model services. The following endpoints are supported: + +- `/v1/chat/completions`: Chat interface (ChatGPT-style) +- `/v1/completions`: Text completion +- `/v1/embeddings`: Text embeddings +- `/v1/models`: List available models +- `/v1/models/{model}`: Model information + +## Configuration + +### LLMConfig + +The LLMConfig class specifies model details such as: + +- Model loading sources (HuggingFace or cloud storage) +- Hardware requirements (accelerator type) +- Engine arguments (e.g. vLLM engine kwargs) +- LoRA multiplexing configuration +- Serve auto-scaling parameters + +```{toctree} +:hidden: + +Quickstart +Prefill/Decode Disaggregation +``` + + diff --git a/doc/source/serve/llm/pd-dissagregation.md b/doc/source/serve/llm/pd-dissagregation.md new file mode 100644 index 000000000000..96f8859ef8e6 --- /dev/null +++ b/doc/source/serve/llm/pd-dissagregation.md @@ -0,0 +1,179 @@ +(serve-pd-dissagregation)= +# Prefill/Decode Disaggregation with KV Transfer Backends + +## Overview + +Prefill/decode disaggregation is a technique that separates the prefill phase (processing input prompts) from the decode phase (generating tokens). This separation allows for: + +- **Better resource utilization**: Prefill operations can use high-memory, high-compute nodes while decode operations can use optimized inference nodes +- **Improved scalability**: Each phase can be scaled independently based on demand +- **Cost optimization**: Different node types can be used for different workloads + +vLLM v1 supports two main KV transfer backends: +1. **NIXLConnector**: Network-based KV cache transfer using NIXL (Network Interface for eXtended LLM). Simple setup with automatic network configuration. +2. **LMCacheConnectorV1**: Advanced caching solution with support for various storage backends. **Requires etcd server** for metadata coordination between prefill and decode instances. + +## Prerequisites + +Make sure that you are using vLLM v1 by setting `VLLM_USE_V1=1` environment variable. + +For NixlConnector make sure nixl is installed. If you use [ray-project/ray-llm](https://hub.docker.com/r/rayproject/ray-llm/tags) images you automatically get the dependency installed. + +For LMCacheConnectorV1, also install LMCache: + +```bash +pip install lmcache +``` + +## NIXLConnector Backend + +The NIXLConnector provides network-based KV cache transfer between prefill and decode servers using a side channel communication mechanism. + +### Basic Configuration + +```python +from ray.serve.llm import LLMConfig, build_pd_openai_app + +# Prefill configuration +prefill_config = LLMConfig( + model_loading_config={ + "model_id": "meta-llama/Llama-3.1-8B-Instruct" + }, + engine_kwargs={ + "kv_transfer_config": { + "kv_connector": "NixlConnector", + "kv_role": "kv_both", + "engine_id": "engine1" + } + } +) + +# Decode configuration +decode_config = LLMConfig( + model_loading_config={ + "model_id": "meta-llama/Llama-3.1-8B-Instruct" + }, + engine_kwargs={ + "kv_transfer_config": { + "kv_connector": "NixlConnector", + "kv_role": "kv_both", + "engine_id": "engine2" + } + } +) + +pd_config = dict( + prefill_config=prefill_config, + decode_config=decode_config, +) + +app = build_pd_openai_app(pd_config) +serve.run(app) +``` + +### Complete YAML Configuration Example + +Here's a complete configuration file for NIXLConnector: + +```{literalinclude} ../doc_code/pd_dissagregation/nixl_example.yaml +:language: yaml +``` + +## LMCacheConnectorV1 Backend + +LMCacheConnectorV1 provides a more advanced caching solution with support for multiple storage backends and enhanced performance features. + +### Scenario 1: LMCache with NIXL Backend + +This configuration uses LMCache with a NIXL-based storage backend for network communication. + +```{literalinclude} ../doc_code/pd_dissagregation/lmcache_nixl_example.yaml +:language: yaml +``` + +#### LMCache Configuration for NIXL Backend + +Create `lmcache_prefiller.yaml`: + +```{literalinclude} ../doc_code/pd_dissagregation/lmcache/nixl/prefiller.yaml +:language: yaml +``` + +Create `lmcache_decoder.yaml`: + +```{literalinclude} ../doc_code/pd_dissagregation/lmcache/nixl/decoder.yaml +:language: yaml +``` + +**Important**: The `LMCACHE_CONFIG_FILE` environment variable must point to an existing configuration file that is accessible within the Ray Serve container or worker environment. Ensure these configuration files are properly mounted or available in your deployment environment. + +### Scenario 2: LMCache with Mooncake Store Backend + +This configuration uses LMCache with Mooncake store, a high-performance distributed storage system. + +```{literalinclude} ../doc_code/pd_dissagregation/lmcache_mooncake_example.yaml +:language: yaml +``` + +#### LMCache Configuration for Mooncake Store + +Create `lmcache_mooncake.yaml`: + +```{literalinclude} ../doc_code/pd_dissagregation/lmcache/mooncake.yaml +:language: yaml +``` + +**Important Notes**: +- The `LMCACHE_CONFIG_FILE` environment variable must point to an existing configuration file that is accessible within the Ray Serve container or worker environment. +- For Mooncake store backend, ensure the etcd metadata server is running and accessible at the specified address. +- Verify that RDMA devices and storage servers are properly configured and accessible. +- In containerized deployments, mount configuration files with appropriate read permissions (e.g., `chmod 644`). +- Ensure all referenced hostnames and IP addresses in configuration files are resolvable from the deployment environment. + +## Deployment and Testing + +### Deploy the Application + +1. **Start required services** (for LMCacheConnectorV1): + + ```bash + # Start etcd server if not already running + docker run -d --name etcd-server \ + -p 2379:2379 -p 2380:2380 \ + quay.io/coreos/etcd:latest \ + etcd --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://localhost:2379 + + # See https://docs.lmcache.ai/kv_cache/mooncake.html for more details. + mooncake_master --port 49999 + ``` + +2. **Save your configuration** to a YAML file (e.g., `mooncake.yaml`) + +3. **Deploy using Ray Serve CLI**: + ```bash + serve deploy pd_config.yaml + ``` + +### Test the Deployment + +Test with a simple request: + +```bash +curl -X POST "http://localhost:8000/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "meta-llama/Llama-3.1-8B-Instruct", + "messages": [ + {"role": "user", "content": "Explain the benefits of prefill/decode disaggregation"} + ], + "max_tokens": 100, + "temperature": 0.7 + }' +``` + +## Related Resources + +LMCache prefill/decode dissagregation official guide: + +- [dissagregated serving](https://docs.lmcache.ai/disaggregated_prefill/nixl/index.html) diff --git a/doc/source/serve/llm/serving-llms.rst b/doc/source/serve/llm/quick-start.rst similarity index 94% rename from doc/source/serve/llm/serving-llms.rst rename to doc/source/serve/llm/quick-start.rst index 25657d6a9c9e..438dc948c751 100644 --- a/doc/source/serve/llm/serving-llms.rst +++ b/doc/source/serve/llm/quick-start.rst @@ -1,61 +1,4 @@ -.. _serving_llms: - -Serving LLMs -============ - -Ray Serve LLM APIs allow users to deploy multiple LLM models together with a familiar Ray Serve API, while providing compatibility with the OpenAI API. - -Features --------- -- ⚡️ Automatic scaling and load balancing -- 🌐 Unified multi-node multi-model deployment -- 🔌 OpenAI compatible -- 🔄 Multi-LoRA support with shared base models -- 🚀 Engine agnostic architecture (i.e. vLLM, SGLang, etc) - -Requirements --------------- - -.. code-block:: bash - - pip install ray[serve,llm]>=2.43.0 vllm>=0.7.2 - - # Suggested dependencies when using vllm 0.7.2: - pip install xgrammar==0.1.11 pynvml==12.0.0 - - -Key Components --------------- - -The :ref:`ray.serve.llm ` module provides two key deployment types for serving LLMs: - -LLMServer -~~~~~~~~~~~~~~~~~~ - -The LLMServer sets up and manages the vLLM engine for model serving. It can be used standalone or combined with your own custom Ray Serve deployments. - -OpenAiIngress -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This deployment provides an OpenAI-compatible FastAPI ingress and routes traffic to the appropriate model for multi-model services. The following endpoints are supported: - -- ``/v1/chat/completions``: Chat interface (ChatGPT-style) -- ``/v1/completions``: Text completion -- ``/v1/embeddings``: Text embeddings -- ``/v1/models``: List available models -- ``/v1/models/{model}``: Model information - -Configuration -------------- - -LLMConfig -~~~~~~~~~ -The :class:`LLMConfig ` class specifies model details such as: - -- Model loading sources (HuggingFace or cloud storage) -- Hardware requirements (accelerator type) -- Engine arguments (e.g. vLLM engine kwargs) -- LoRA multiplexing configuration -- Serve auto-scaling parameters +.. _quick-start: Quickstart Examples ------------------- @@ -894,4 +837,4 @@ We collect data about the following features and attributes: - GPU type used and number of GPUs used If you would like to opt-out from usage data collection, you can follow :ref:`Ray usage stats ` -to disable it. +to disable it. \ No newline at end of file diff --git a/python/ray/llm/_internal/serve/configs/server_models.py b/python/ray/llm/_internal/serve/configs/server_models.py index ca046794cfa8..d7f3b27a1944 100644 --- a/python/ray/llm/_internal/serve/configs/server_models.py +++ b/python/ray/llm/_internal/serve/configs/server_models.py @@ -34,6 +34,9 @@ ENABLE_WORKER_PROCESS_SETUP_HOOK, MODEL_RESPONSE_BATCH_TIMEOUT_MS, ) +from ray.llm._internal.serve.deployments.llm.vllm.kv_transfer_backends import ( + SUPPORTED_BACKENDS as SUPPORTED_KV_CONNECTOR_BACKENDS, +) from ray.llm._internal.serve.observability.logging import get_logger from ray.serve._private.config import DeploymentConfig @@ -456,6 +459,28 @@ def get_serve_options( return deployment_config + def setup_engine_backend(self): + self._setup_kv_connector_backend() + + def _setup_kv_connector_backend(self): + """Private method to setup kv connector dependning on the local deployment state""" + # 1. validate that the backend is one of the backends supported (Nixl or LMCache) + kv_transfer_config = self.engine_kwargs.get("kv_transfer_config") + if not kv_transfer_config: + return + + kv_connector = kv_transfer_config.get("kv_connector") + if not kv_connector: + raise ValueError("Connector type is not specified.") + + kv_connector_backend_class = SUPPORTED_KV_CONNECTOR_BACKENDS.get(kv_connector) + if not kv_connector_backend_class: + raise ValueError(f"Unsupported connector type: {kv_connector}") + + # 2. Setup the backend + kv_connector_backend = kv_connector_backend_class(kv_transfer_config) + kv_connector_backend.setup() + def _is_yaml_file(filename: str) -> bool: yaml_extensions = [".yml", ".yaml", ".json"] diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/__init__.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/__init__.py new file mode 100644 index 000000000000..72e24a18bad5 --- /dev/null +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/__init__.py @@ -0,0 +1,16 @@ +from typing import Dict + +from ray.llm._internal.serve.deployments.llm.vllm.kv_transfer_backends.base import ( + BaseConnectorBackend, +) +from ray.llm._internal.serve.deployments.llm.vllm.kv_transfer_backends.lmcache_connector_v1 import ( + LMCacheConnectorV1Backend, +) +from ray.llm._internal.serve.deployments.llm.vllm.kv_transfer_backends.nixl_connector import ( + NixlConnectorBackend, +) + +SUPPORTED_BACKENDS: Dict[str, BaseConnectorBackend] = { + "LMCacheConnectorV1": LMCacheConnectorV1Backend, + "NixlConnector": NixlConnectorBackend, +} diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/base.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/base.py new file mode 100644 index 000000000000..420f7f44a06f --- /dev/null +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/base.py @@ -0,0 +1,32 @@ +import abc +import random +import string +from typing import Any, Dict + + +class BaseConnectorBackend(abc.ABC): + def __init__(self, kv_transfer_config: Dict[str, Any]): + """Base class for connector backends. + + Args: + kv_transfer_config: Configuration for the KV transfer. + """ + self.kv_transfer_config = kv_transfer_config + + def _get_unique_suffix(self, len: int = 6) -> str: + """Generates unique alphanumeric suffix. + + Args: + len: Length of the suffix to generate. + Returns: + A unique alphanumeric suffix string of specified length. + """ + return "".join(random.choices(string.ascii_letters + string.digits, k=len)) + + @abc.abstractmethod + def setup(self) -> None: + """Setup the connector backend. + + This method is called to setup the connector backend. + """ + pass diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/lmcache_connector_v1.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/lmcache_connector_v1.py new file mode 100644 index 000000000000..85945c30eb14 --- /dev/null +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/lmcache_connector_v1.py @@ -0,0 +1,61 @@ +from ray.llm._internal.serve.deployments.llm.vllm.kv_transfer_backends.base import ( + BaseConnectorBackend, +) +from ray.llm._internal.serve.observability.logging import get_logger + +logger = get_logger(__name__) + + +def _check_lmcache_installed(): + try: + import lmcache # noqa: F401 + except ImportError: + raise ImportError( + "LMCache is not installed. Please install it with `pip install lmcache`." + ) + + +class LMCacheConnectorV1Backend(BaseConnectorBackend): + + KV_CONNECTOR_EXTRA_CONFIG_FIELD_NAME = "kv_connector_extra_config" + LMCACHE_RPC_PORT_FIELD_NAME = "lmcache_rpc_port" + DEFAULT_LMCACHE_RPC_PORT_NAME = "lmcache_rpc_port" + + def setup(self) -> None: + """Initialize the LMCache connector backend. + This method sets up the LMCache connector by: + 1. Checking if LMCache is installed. + 2. Configuring the LMCache RPC port if not already set. + 3. Creating a unique LMCache RPC port across replicas. + Raises: + ImportError: If LMCache is not installed. + """ + _check_lmcache_installed() + + if ( + LMCacheConnectorV1Backend.KV_CONNECTOR_EXTRA_CONFIG_FIELD_NAME + not in self.kv_transfer_config + ): + return + + kv_connector_extra_config = self.kv_transfer_config[ + LMCacheConnectorV1Backend.KV_CONNECTOR_EXTRA_CONFIG_FIELD_NAME + ] + lmcache_rpc_port = ( + kv_connector_extra_config.get( + LMCacheConnectorV1Backend.LMCACHE_RPC_PORT_FIELD_NAME, + LMCacheConnectorV1Backend.DEFAULT_LMCACHE_RPC_PORT_NAME, + ) + + self._get_unique_suffix() + ) + if ( + LMCacheConnectorV1Backend.LMCACHE_RPC_PORT_FIELD_NAME + in kv_connector_extra_config + ): + logger.info( + f"Setting unique {lmcache_rpc_port=} for current replica LMCacheConnectorV1." + ) + + kv_connector_extra_config[ + LMCacheConnectorV1Backend.LMCACHE_RPC_PORT_FIELD_NAME + ] = lmcache_rpc_port diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/nixl_connector.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/nixl_connector.py new file mode 100644 index 000000000000..5cdbb37f744e --- /dev/null +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/nixl_connector.py @@ -0,0 +1,45 @@ +import os + +from ray.llm._internal.serve.deployments.llm.vllm.kv_transfer_backends.base import ( + BaseConnectorBackend, +) + + +class NixlConnectorBackend(BaseConnectorBackend): + def setup(self) -> None: + """Initialize the NIXL connector backend. + + This method sets up the NIXL (Network Interface for eXtended LLM) connector by: + 1. Verifying that the required vLLM environment variables are supported + 2. Configuring the side channel port and host if not already set + 3. Creating a unique engine ID across replicas + + The side channel is used for KV cache transfer between vLLM instances. + + Raises: + ValueError: If the current vLLM version doesn't support the required + NIXL environment variables. + """ + from vllm import envs as vllm_envs, utils as vllm_utils + + if ( + "VLLM_NIXL_SIDE_CHANNEL_PORT" not in vllm_envs.environment_variables + or "VLLM_NIXL_SIDE_CHANNEL_HOST" not in vllm_envs.environment_variables + ): + raise ValueError( + "This vLLM version does not support VLLM_NIXL_SIDE_CHANNEL_PORT" + "or VLLM_NIXL_SIDE_CHANNEL_HOST environment variable. It's likely" + "that you are using an older version of vLLM." + ) + + if not vllm_envs.is_set("VLLM_NIXL_SIDE_CHANNEL_PORT"): + port: int = vllm_utils.get_open_port() + os.environ["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(port) + if not vllm_envs.is_set("VLLM_NIXL_SIDE_CHANNEL_HOST"): + os.environ["VLLM_NIXL_SIDE_CHANNEL_HOST"] = vllm_utils.get_ip() + + # We need to overwrite the engine_id to make it unique across replicas. + engine_id = self.kv_transfer_config.get("engine_id", self._get_unique_suffix()) + host = vllm_envs.VLLM_NIXL_SIDE_CHANNEL_HOST + port = vllm_envs.VLLM_NIXL_SIDE_CHANNEL_PORT + self.kv_transfer_config["engine_id"] = "-".join([engine_id, host, str(port)]) diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py index e3ba947266ad..a6654ca89c4e 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py @@ -1,6 +1,5 @@ import argparse import os -import uuid from typing import TYPE_CHECKING, AsyncGenerator, Optional, Tuple, Union from starlette.datastructures import State @@ -118,41 +117,14 @@ def __init__( raise ImportError( "vLLM is not installed. Please install it with `pip install ray[llm]`." ) - from vllm import envs as vllm_envs, utils as vllm_utils + from vllm import envs as vllm_envs if not vllm_envs.VLLM_USE_V1: logger.warning( "vLLM v0 is getting fully deprecated. As a result in Ray Serve LLM only v1 is supported. Only when you know what you are doing, you can set VLLM_USE_V1=0" ) - # TODO (Kourosh): This validation logic belongs to the PDProxy module. - # Pick a random port in P/D case. - kv_transfer_config = llm_config.engine_kwargs.get("kv_transfer_config", None) - if kv_transfer_config is not None: - connector_type = getattr(kv_transfer_config, "kv_connector", "") - if connector_type != "NixlConnector": - raise ValueError("Only NixlConnector is supported for kv transfer.") - if ( - "VLLM_NIXL_SIDE_CHANNEL_PORT" not in vllm_envs.environment_variables - or "VLLM_NIXL_SIDE_CHANNEL_HOST" not in vllm_envs.environment_variables - ): - raise ValueError( - "This vLLM version does not support VLLM_NIXL_SIDE_CHANNEL_PORT" - "or VLLM_NIXL_SIDE_CHANNEL_HOST environment variable. It's likely" - "that you are using an older version of vLLM." - ) - - if not vllm_envs.is_set("VLLM_NIXL_SIDE_CHANNEL_PORT"): - port: int = vllm_utils.get_open_port() - os.environ["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(port) - if not vllm_envs.is_set("VLLM_NIXL_SIDE_CHANNEL_HOST"): - os.environ["VLLM_NIXL_SIDE_CHANNEL_HOST"] = vllm_utils.get_ip() - - # We need to overwrite the engine_id to make it unique across replicas. - engine_id = getattr(kv_transfer_config, "engine_id", str(uuid.uuid4())) - host = vllm_envs.VLLM_NIXL_SIDE_CHANNEL_HOST - port = vllm_envs.VLLM_NIXL_SIDE_CHANNEL_PORT - kv_transfer_config.engine_id = "-".join([engine_id, host, str(port)]) + self.llm_config.setup_engine_backend() self._running = False diff --git a/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py b/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py index 2d66fac44c53..d825a30a2161 100644 --- a/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py +++ b/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py @@ -5,7 +5,6 @@ from typing import Any, AsyncGenerator, Dict, Union from pydantic import Field -from vllm.config import KVTransferConfig from ray import serve from ray.llm._internal.common.base_pydantic import BaseModelExtended @@ -187,7 +186,7 @@ def build_pd_openai_app(pd_serving_args: dict) -> Application: if "kv_transfer_config" not in config.engine_kwargs: config.engine_kwargs.update( { - "kv_transfer_config": KVTransferConfig( + "kv_transfer_config": dict( kv_connector="NixlConnector", kv_role="kv_both", engine_id=str(uuid.uuid4()), diff --git a/python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/__init__.py b/python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/__init__.py new file mode 100644 index 000000000000..0a39e777cc97 --- /dev/null +++ b/python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/__init__.py @@ -0,0 +1 @@ +# Test package for KV transfer backends diff --git a/python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/test_lmcache_connector_v1.py b/python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/test_lmcache_connector_v1.py new file mode 100644 index 000000000000..e6da3303343f --- /dev/null +++ b/python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/test_lmcache_connector_v1.py @@ -0,0 +1,86 @@ +import pytest + +from ray.llm._internal.serve.deployments.llm.vllm.kv_transfer_backends.lmcache_connector_v1 import ( + LMCacheConnectorV1Backend, +) + + +class TestLMCacheConnectorV1Backend: + @pytest.fixture + def lmcache_backend_basic(self): + """Fixture for basic LMCacheConnectorV1Backend.""" + return LMCacheConnectorV1Backend( + { + "kv_connector": "LMCacheConnectorV1", + "kv_role": "kv_both", + } + ) + + @pytest.fixture + def lmcache_backend_with_extra(self): + """Fixture for LMCacheConnectorV1Backend with extra config.""" + return LMCacheConnectorV1Backend( + { + "kv_connector": "LMCacheConnectorV1", + "kv_role": "kv_both", + "kv_connector_extra_config": {}, + } + ) + + @pytest.fixture + def lmcache_backend_with_port(self): + """Fixture for LMCacheConnectorV1Backend with port config.""" + return LMCacheConnectorV1Backend( + { + "kv_connector": "LMCacheConnectorV1", + "kv_role": "kv_both", + "kv_connector_extra_config": { + "lmcache_rpc_port": LMCacheConnectorV1Backend.DEFAULT_LMCACHE_RPC_PORT_NAME, + }, + } + ) + + def test_setup_basic_config(self, lmcache_backend_basic): + """Test setup with basic configuration (no kv_connector_extra_config).""" + lmcache_backend_basic.setup() + + # Configuration should remain unchanged + assert ( + "kv_connector_extra_config" not in lmcache_backend_basic.kv_transfer_config + ) + + def test_setup_with_extra_config_no_port(self, lmcache_backend_with_extra): + """Test setup with extra config but no lmcache_rpc_port.""" + lmcache_backend_with_extra.setup() + + # Should add lmcache_rpc_port with default DEFAULT_LMCACHE_RPC_PORT_NAME + random string + assert ( + "lmcache_rpc_port" + in lmcache_backend_with_extra.kv_transfer_config[ + "kv_connector_extra_config" + ] + ) + port_value = lmcache_backend_with_extra.kv_transfer_config[ + "kv_connector_extra_config" + ]["lmcache_rpc_port"] + assert port_value.startswith( + LMCacheConnectorV1Backend.DEFAULT_LMCACHE_RPC_PORT_NAME + ) + assert len(port_value) > len( + LMCacheConnectorV1Backend.DEFAULT_LMCACHE_RPC_PORT_NAME + ) # Should have random string appended + + def test_setup_with_existing_port(self, lmcache_backend_with_port): + """Test setup with existing lmcache_rpc_port configuration.""" + original_port = lmcache_backend_with_port.kv_transfer_config[ + "kv_connector_extra_config" + ]["lmcache_rpc_port"] + + lmcache_backend_with_port.setup() + + # Should modify the existing port by appending random string + new_port = lmcache_backend_with_port.kv_transfer_config[ + "kv_connector_extra_config" + ]["lmcache_rpc_port"] + assert new_port.startswith(original_port) + assert len(new_port) > len(original_port) # Should have random string appended diff --git a/python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/test_nixl_connector.py b/python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/test_nixl_connector.py new file mode 100644 index 000000000000..aabe766c4a5a --- /dev/null +++ b/python/ray/llm/tests/serve/cpu/deployments/llm/vllm/kv_transfer_backends/test_nixl_connector.py @@ -0,0 +1,48 @@ +import os +import uuid +from unittest.mock import patch + +import pytest + +from ray.llm._internal.serve.deployments.llm.vllm.kv_transfer_backends.nixl_connector import ( + NixlConnectorBackend, +) + + +@pytest.fixture +def engine_id(): + """Fixture for the engine ID.""" + return str(uuid.uuid4()) + + +class TestNixlConnectorBackend: + @pytest.fixture + def nixl_backend(self, engine_id: str): + """Fixture for the NixlConnectorBackend.""" + return NixlConnectorBackend( + dict( + kv_connector="NixlConnector", + kv_role="kv_both", + engine_id=engine_id, + ) + ) + + @pytest.mark.parametrize( + "env_vars", + [ + {}, + {"VLLM_NIXL_SIDE_CHANNEL_PORT": "8080"}, + {"VLLM_NIXL_SIDE_CHANNEL_HOST": "127.0.0.1"}, + { + "VLLM_NIXL_SIDE_CHANNEL_PORT": "8080", + "VLLM_NIXL_SIDE_CHANNEL_HOST": "127.0.0.1", + }, + ], + ) + def test_setup_environment_variables(self, nixl_backend, env_vars, engine_id: str): + """Test that setup configures environment variables and overrides engine_id correctly.""" + with patch.dict("os.environ", env_vars, clear=True): + nixl_backend.setup() + assert "VLLM_NIXL_SIDE_CHANNEL_PORT" in os.environ + assert "VLLM_NIXL_SIDE_CHANNEL_HOST" in os.environ + assert engine_id in nixl_backend.kv_transfer_config["engine_id"] diff --git a/python/ray/llm/tests/serve/cpu/deployments/prefill_decode_disagg/test_prefill_decode_disagg.py b/python/ray/llm/tests/serve/cpu/deployments/prefill_decode_disagg/test_prefill_decode_disagg.py index c6444a8b7152..75c50a0ba443 100644 --- a/python/ray/llm/tests/serve/cpu/deployments/prefill_decode_disagg/test_prefill_decode_disagg.py +++ b/python/ray/llm/tests/serve/cpu/deployments/prefill_decode_disagg/test_prefill_decode_disagg.py @@ -9,7 +9,8 @@ class TestServingArgsParsing: - def test_parse_dict(self): + @pytest.mark.parametrize("kv_connector", ["NixlConnector", "LMCacheConnectorV1"]) + def test_parse_dict(self, kv_connector: str): prefill_config = LLMConfig( model_loading_config=dict( model_id="qwen-0.5b", diff --git a/python/ray/llm/tests/serve/gpu/deployments/llm/prefill_decode_disagg/test_prefill_decode_disagg_gpu.py b/python/ray/llm/tests/serve/gpu/deployments/llm/prefill_decode_disagg/test_prefill_decode_disagg_gpu.py index 6083ae772ea2..b4da68d79087 100644 --- a/python/ray/llm/tests/serve/gpu/deployments/llm/prefill_decode_disagg/test_prefill_decode_disagg_gpu.py +++ b/python/ray/llm/tests/serve/gpu/deployments/llm/prefill_decode_disagg/test_prefill_decode_disagg_gpu.py @@ -1,7 +1,7 @@ import sys +from unittest.mock import MagicMock import pytest -from vllm.config import KVTransferConfig from ray.llm._internal.serve.configs.server_models import ( LLMConfig, @@ -15,17 +15,23 @@ class TestPDDisaggVLLMEngine: """Test vLLM engine under PD disagg.""" @pytest.mark.asyncio + @pytest.mark.parametrize("kv_connector", ["NixlConnector", "LMCacheConnectorV1"]) async def test_pd_disagg_vllm_engine( self, # llm_config is a fixture defined in serve.tests.conftest.py llm_config: LLMConfig, + kv_connector: str, + monkeypatch, ): """Test vLLM engine under PD disagg.""" + if kv_connector == "LMCacheConnectorV1": + lmcache_mock = MagicMock() + monkeypatch.setitem(sys.modules, "lmcache", lmcache_mock) llm_config = llm_config.model_copy(deep=True) llm_config.engine_kwargs.update( { - "kv_transfer_config": KVTransferConfig( - kv_connector="NixlConnector", + "kv_transfer_config": dict( + kv_connector=kv_connector, kv_role="kv_both", ), } diff --git a/release/llm_tests/serve/configs/lmcache/decoder.yaml b/release/llm_tests/serve/configs/lmcache/decoder.yaml new file mode 100644 index 000000000000..34e22d421997 --- /dev/null +++ b/release/llm_tests/serve/configs/lmcache/decoder.yaml @@ -0,0 +1,12 @@ +local_cpu: False +max_local_cpu_size: 0 +max_local_disk_size: 0 +remote_serde: NULL + +enable_nixl: True +nixl_role: "receiver" +nixl_receiver_host: "localhost" +nixl_receiver_port: 55555 +nixl_buffer_size: 1073741824 # 1GB +nixl_buffer_device: "cuda" +nixl_enable_gc: True diff --git a/release/llm_tests/serve/configs/lmcache/prefiller.yaml b/release/llm_tests/serve/configs/lmcache/prefiller.yaml new file mode 100644 index 000000000000..544551b78a78 --- /dev/null +++ b/release/llm_tests/serve/configs/lmcache/prefiller.yaml @@ -0,0 +1,12 @@ +local_cpu: False +max_local_cpu_size: 0 +max_local_disk_size: 0 +remote_serde: NULL + +enable_nixl: True +nixl_role: "sender" +nixl_receiver_host: "localhost" +nixl_receiver_port: 55555 +nixl_buffer_size: 1073741824 # 1GB +nixl_buffer_device: "cuda" +nixl_enable_gc: True diff --git a/release/llm_tests/serve/configs/serve_llama_3dot1_8b_quantized_tp1_2p6d_lmcache.yaml b/release/llm_tests/serve/configs/serve_llama_3dot1_8b_quantized_tp1_2p6d_lmcache.yaml new file mode 100644 index 000000000000..87636bb790b5 --- /dev/null +++ b/release/llm_tests/serve/configs/serve_llama_3dot1_8b_quantized_tp1_2p6d_lmcache.yaml @@ -0,0 +1,52 @@ +applications: + - args: + + prefill_config: + model_loading_config: + model_id: neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16 + accelerator_type: A10G + engine_kwargs: + max_model_len: 8192 + tensor_parallel_size: 1 + enforce_eager: true + kv_transfer_config: + kv_connector: LMCacheConnectorV1 + kv_role: kv_producer + kv_connector_extra_config: + discard_partial_chunks: false + lmcache_rpc_port: producer1 + deployment_config: + autoscaling_config: + min_replicas: 2 + max_replicas: 2 + runtime_env: + env_vars: + LMCACHE_CONFIG_FILE: configs/lmcache/prefiller.yaml + LMCACHE_USE_EXPERIMENTAL: "True" + + decode_config: + model_loading_config: + model_id: neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16 + accelerator_type: A10G + engine_kwargs: + max_model_len: 8192 + tensor_parallel_size: 1 + enforce_eager: true + kv_transfer_config: + kv_connector: LMCacheConnectorV1 + kv_role: kv_consumer + kv_connector_extra_config: + discard_partial_chunks: false + lmcache_rpc_port: consumer1 + deployment_config: + autoscaling_config: + min_replicas: 6 + max_replicas: 6 + runtime_env: + env_vars: + LMCACHE_CONFIG_FILE: configs/lmcache/decoder.yaml + LMCACHE_USE_EXPERIMENTAL: "True" + + import_path: ray.serve.llm:build_pd_openai_app + name: llm-endpoint + route_prefix: / diff --git a/release/ray_release/byod/byod_llm_lmcache_test.sh b/release/ray_release/byod/byod_llm_lmcache_test.sh new file mode 100755 index 000000000000..f3a1a7d77497 --- /dev/null +++ b/release/ray_release/byod/byod_llm_lmcache_test.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# This script is used to build an extra layer on top of the base llm image +# to run the llm sglang release tests + +set -exo pipefail + +pip3 install "lmcache>=0.3.2" diff --git a/release/release_tests.yaml b/release/release_tests.yaml index 8d096b1f2d44..a854d73267af 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -4553,6 +4553,26 @@ long_running: false script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_quantized_tp1_2p6d.yaml --skip-hf-token true +- name: llm_serve_llama_3dot1_8B_quantized_tp1_2p6d_lmcache + frequency: nightly + python: "3.11" + group: llm-serve + team: llm + working_dir: llm_tests/serve + + cluster: + byod: + type: llm-cu128 + post_build_script: byod_llm_lmcache_test.sh + cluster_compute: llm_auto_select_worker.yaml + # NOTE: Important for getting the correct secrets + cloud_id: cld_wy5a6nhazplvu32526ams61d98 + project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq + + run: + timeout: 3600 + long_running: false + script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_quantized_tp1_2p6d_lmcache.yaml --skip-hf-token true ############## # LLM Batch From 2c7bd7d06930e5cc302a01c5baedef43911e3582 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Wed, 13 Aug 2025 14:35:25 -0700 Subject: [PATCH 086/634] [core][ci] Kill debug wheel step (#55571) Signed-off-by: dayshah --- .buildkite/build.rayci.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.buildkite/build.rayci.yml b/.buildkite/build.rayci.yml index dcd69269f6a7..01256a3b63c8 100644 --- a/.buildkite/build.rayci.yml +++ b/.buildkite/build.rayci.yml @@ -19,17 +19,6 @@ steps: - manylinux - forge - - label: ":tapioca: build: debug wheel" - tags: - - linux_wheels - - oss - instance_type: large - commands: - - bazel run //ci/ray_ci:build_in_docker -- wheel --build-type debug --upload - depends_on: - - manylinux - - forge - - label: ":tapioca: build: jar" key: java_wheels tags: From 3e34885814e4da9a83123e22b042a7ee684074ad Mon Sep 17 00:00:00 2001 From: Kishanthan Thangarajah Date: Wed, 13 Aug 2025 19:21:05 -0400 Subject: [PATCH 087/634] [serve] Support custom autoscaling at deployment level for ray serve (#55253) ## Why are these changes needed? This PR adds initial changes to support custom auto scaling with ray serve. Two new classes (AutoscalingContext and AutoscalingPolicy) have been introduced as per discussions in https://docs.google.com/document/d/1KtMUDz1O3koihG6eh-QcUqudZjNAX3NsqqOMYh3BoWA/edit?usp=sharing. Related RFC https://github.com/ray-project/ray/issues/41135#issuecomment-3156717488 The changes will have two phases. Phase1 is to add required changes to support custom autoscaling at deployment level. Phase2 is to extend the changes to support custom autoscaling at application level. This PR is part of Phase1 (deployment level custom autoscaling). ## Related issue number Related to #41135 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Kishanthan Thangarajah --- doc/source/serve/api/index.md | 1 + .../ray/serve/_private/autoscaling_state.py | 64 ++- python/ray/serve/_private/constants.py | 4 +- python/ray/serve/autoscaling_policy.py | 34 +- python/ray/serve/config.py | 42 +- python/ray/serve/tests/test_controller.py | 10 +- python/ray/serve/tests/test_deploy_app_2.py | 5 +- .../tests/unit/test_autoscaling_policy.py | 366 +++++++++--------- python/ray/serve/tests/unit/test_config.py | 7 +- src/ray/protobuf/serve.proto | 13 +- 10 files changed, 310 insertions(+), 236 deletions(-) diff --git a/doc/source/serve/api/index.md b/doc/source/serve/api/index.md index 37ae947210cb..52d8c502eb3f 100644 --- a/doc/source/serve/api/index.md +++ b/doc/source/serve/api/index.md @@ -83,6 +83,7 @@ See the [model composition guide](serve-model-composition) for how to update cod serve.config.gRPCOptions serve.config.HTTPOptions serve.config.AutoscalingConfig + serve.config.AutoscalingPolicy serve.config.RequestRouterConfig ``` diff --git a/python/ray/serve/_private/autoscaling_state.py b/python/ray/serve/_private/autoscaling_state.py index e8adb9562998..c95abfa6cb5c 100644 --- a/python/ray/serve/_private/autoscaling_state.py +++ b/python/ray/serve/_private/autoscaling_state.py @@ -1,7 +1,7 @@ import logging import time from dataclasses import dataclass -from typing import Dict, List, Optional, Set +from typing import Any, Dict, List, Optional, Set from ray.serve._private.common import ( DeploymentHandleSource, @@ -78,6 +78,45 @@ class ReplicaMetricReport: timestamp: float +@dataclass +class AutoscalingContext: + """Rich context provided to custom autoscaling policies.""" + + # Deployment information + deployment_id: DeploymentID + deployment_name: str + app_name: Optional[str] + + # Current state + current_num_replicas: int + target_num_replicas: int + running_replicas: List[ReplicaID] + + # Built-in metrics + total_num_requests: float + queued_requests: Optional[float] + requests_per_replica: Dict[ReplicaID, float] + + # Custom metrics + aggregated_metrics: Dict[str, Dict[ReplicaID, float]] + raw_metrics: Dict[str, Dict[ReplicaID, List[float]]] + + # Capacity and bounds + capacity_adjusted_min_replicas: int + capacity_adjusted_max_replicas: int + + # Policy state + policy_state: Dict[str, Any] + + # Timing + last_scale_up_time: Optional[float] + last_scale_down_time: Optional[float] + current_time: Optional[float] + + # Config + config: Optional[Any] + + class AutoscalingState: """Manages autoscaling for a single deployment.""" @@ -270,16 +309,29 @@ def get_decision_num_replicas( `_skip_bound_check` is True, then the bounds are not applied. """ - decision_num_replicas = self._policy( - curr_target_num_replicas=curr_target_num_replicas, + autoscaling_context: AutoscalingContext = AutoscalingContext( + deployment_id=self._deployment_id, + deployment_name=self._deployment_id.name, + app_name=self._deployment_id.app_name, + current_num_replicas=len(self._running_replicas), + target_num_replicas=curr_target_num_replicas, + running_replicas=self._running_replicas, total_num_requests=self.get_total_num_requests(), - num_running_replicas=len(self._running_replicas), - config=self._config, capacity_adjusted_min_replicas=self.get_num_replicas_lower_bound(), capacity_adjusted_max_replicas=self.get_num_replicas_upper_bound(), - policy_state=self._policy_state, + policy_state=self._policy_state.copy(), + current_time=time.time(), + config=self._config, + queued_requests=None, + requests_per_replica=None, + aggregated_metrics=None, + raw_metrics=None, + last_scale_up_time=None, + last_scale_down_time=None, ) + decision_num_replicas, self._policy_state = self._policy(autoscaling_context) + if _skip_bound_check: return decision_num_replicas diff --git a/python/ray/serve/_private/constants.py b/python/ray/serve/_private/constants.py index 166abcae2246..e6aaac1f62ad 100644 --- a/python/ray/serve/_private/constants.py +++ b/python/ray/serve/_private/constants.py @@ -343,7 +343,9 @@ ) # The default autoscaling policy to use if none is specified. -DEFAULT_AUTOSCALING_POLICY = "ray.serve.autoscaling_policy:default_autoscaling_policy" +DEFAULT_AUTOSCALING_POLICY_NAME = ( + "ray.serve.autoscaling_policy:default_autoscaling_policy" +) # Feature flag to enable collecting all queued and ongoing request # metrics at handles instead of replicas. ON by default. diff --git a/python/ray/serve/autoscaling_policy.py b/python/ray/serve/autoscaling_policy.py index 2cabe736a870..eb443a46c03a 100644 --- a/python/ray/serve/autoscaling_policy.py +++ b/python/ray/serve/autoscaling_policy.py @@ -1,7 +1,8 @@ import logging import math -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple +from ray.serve._private.autoscaling_state import AutoscalingContext from ray.serve._private.constants import CONTROL_LOOP_INTERVAL_S, SERVE_LOGGER_NAME from ray.serve.config import AutoscalingConfig from ray.util.annotations import PublicAPI @@ -83,14 +84,8 @@ def _calculate_desired_num_replicas( @PublicAPI(stability="alpha") def replica_queue_length_autoscaling_policy( - curr_target_num_replicas: int, - total_num_requests: int, - num_running_replicas: int, - config: Optional[AutoscalingConfig], - capacity_adjusted_min_replicas: int, - capacity_adjusted_max_replicas: int, - policy_state: Dict[str, Any], -) -> int: + ctx: AutoscalingContext, +) -> Tuple[int, Dict[str, Any]]: """The default autoscaling policy based on basic thresholds for scaling. There is a minimum threshold for the average queue length in the cluster to scale up and a maximum threshold to scale down. Each period, a 'scale @@ -100,15 +95,26 @@ def replica_queue_length_autoscaling_policy( `get_decision_num_replicas` is called once every CONTROL_LOOP_PERIOD_S seconds. """ + + curr_target_num_replicas: int = ctx.target_num_replicas + total_num_requests: int = ctx.total_num_requests + num_running_replicas: int = ctx.current_num_replicas + config: Optional[AutoscalingConfig] = ctx.config + capacity_adjusted_min_replicas: int = ctx.capacity_adjusted_min_replicas + capacity_adjusted_max_replicas: int = ctx.capacity_adjusted_max_replicas + policy_state: Dict[str, Any] = ctx.policy_state decision_counter = policy_state.get("decision_counter", 0) if num_running_replicas == 0: # When 0 replicas and queries are queued, scale up the replicas if total_num_requests > 0: - return max( - math.ceil(1 * config.get_upscaling_factor()), - curr_target_num_replicas, + return ( + max( + math.ceil(1 * config.get_upscaling_factor()), + curr_target_num_replicas, + ), + policy_state, ) - return curr_target_num_replicas + return curr_target_num_replicas, policy_state decision_num_replicas = curr_target_num_replicas @@ -153,7 +159,7 @@ def replica_queue_length_autoscaling_policy( decision_counter = 0 policy_state["decision_counter"] = decision_counter - return decision_num_replicas + return decision_num_replicas, policy_state default_autoscaling_policy = replica_queue_length_autoscaling_policy diff --git a/python/ray/serve/config.py b/python/ray/serve/config.py index d06c8f8cddf4..c58570eff39b 100644 --- a/python/ray/serve/config.py +++ b/python/ray/serve/config.py @@ -17,7 +17,7 @@ ) from ray._common.utils import import_attr from ray.serve._private.constants import ( - DEFAULT_AUTOSCALING_POLICY, + DEFAULT_AUTOSCALING_POLICY_NAME, DEFAULT_GRPC_PORT, DEFAULT_HTTP_HOST, DEFAULT_HTTP_PORT, @@ -162,6 +162,15 @@ def get_request_router_class(self) -> Callable: DEFAULT_METRICS_INTERVAL_S = 10.0 +@PublicAPI(stability="alpha") +class AutoscalingPolicy(BaseModel): + name: Union[str, Callable] = Field( + default=DEFAULT_AUTOSCALING_POLICY_NAME, + description="Name of the policy function or the import path of the policy. " + "Will be the concatenation of the policy module and the policy name if user passed a callable.", + ) + + @PublicAPI(stability="stable") class AutoscalingConfig(BaseModel): """Config for the Serve Autoscaler.""" @@ -174,7 +183,7 @@ class AutoscalingConfig(BaseModel): initial_replicas: Optional[NonNegativeInt] = None max_replicas: PositiveInt = 1 - target_ongoing_requests: PositiveFloat = DEFAULT_TARGET_ONGOING_REQUESTS + target_ongoing_requests: Optional[PositiveFloat] = DEFAULT_TARGET_ONGOING_REQUESTS metrics_interval_s: PositiveFloat = Field( default=DEFAULT_METRICS_INTERVAL_S, @@ -222,8 +231,12 @@ class AutoscalingConfig(BaseModel): # Cloudpickled policy definition. _serialized_policy_def: bytes = PrivateAttr(default=b"") - # Custom autoscaling config. Defaults to the request-based autoscaler. - _policy: Union[str, Callable] = PrivateAttr(default=DEFAULT_AUTOSCALING_POLICY) + # Autoscaling policy. This policy is deployment scoped. Defaults to the request-based autoscaler. + _policy: AutoscalingPolicy = Field(default_factory=AutoscalingPolicy) + + # This is to make `_policy` a normal field until its GA ready. + class Config: + underscore_attrs_are_private = True @validator("max_replicas", always=True) def replicas_settings_valid(cls, max_replicas, values): @@ -273,18 +286,21 @@ def serialize_policy(self) -> None: """ values = self.dict() policy = values.get("_policy") - if isinstance(policy, Callable): - policy = f"{policy.__module__}.{policy.__name__}" - if not policy: - policy = DEFAULT_AUTOSCALING_POLICY + policy_name = None + if isinstance(policy, dict): + policy_name = policy.get("name") + + if isinstance(policy_name, Callable): + policy_name = f"{policy_name.__module__}.{policy_name.__name__}" + + if not policy_name: + policy_name = DEFAULT_AUTOSCALING_POLICY_NAME - policy_path = policy - policy = import_attr(policy) + if not self._serialized_policy_def: + self._serialized_policy_def = cloudpickle.dumps(import_attr(policy_name)) - if not values.get("_serialized_policy_def"): - self._serialized_policy_def = cloudpickle.dumps(policy) - self._policy = policy_path + self._policy = AutoscalingPolicy(name=policy_name) @classmethod def default(cls): diff --git a/python/ray/serve/tests/test_controller.py b/python/ray/serve/tests/test_controller.py index d3696ca521fb..18ca6ce16ab2 100644 --- a/python/ray/serve/tests/test_controller.py +++ b/python/ray/serve/tests/test_controller.py @@ -9,7 +9,7 @@ from ray.serve._private.common import DeploymentID from ray.serve._private.config import DeploymentConfig from ray.serve._private.constants import ( - DEFAULT_AUTOSCALING_POLICY, + DEFAULT_AUTOSCALING_POLICY_NAME, SERVE_DEFAULT_APP_NAME, ) from ray.serve._private.deployment_info import DeploymentInfo @@ -79,9 +79,9 @@ def check_custom_exception() -> bool: @pytest.mark.parametrize( - "policy", [None, DEFAULT_AUTOSCALING_POLICY, default_autoscaling_policy] + "policy_name", [None, DEFAULT_AUTOSCALING_POLICY_NAME, default_autoscaling_policy] ) -def test_get_serve_instance_details_json_serializable(serve_instance, policy): +def test_get_serve_instance_details_json_serializable(serve_instance, policy_name): """Test the result from get_serve_instance_details is json serializable.""" controller = _get_global_client()._controller @@ -89,9 +89,9 @@ def test_get_serve_instance_details_json_serializable(serve_instance, policy): autoscaling_config = { "min_replicas": 1, "max_replicas": 10, - "_policy": policy, + "_policy": {"name": policy_name}, } - if policy is None: + if policy_name is None: autoscaling_config.pop("_policy") @serve.deployment(autoscaling_config=autoscaling_config) diff --git a/python/ray/serve/tests/test_deploy_app_2.py b/python/ray/serve/tests/test_deploy_app_2.py index 0c379082e77b..98eff882586b 100644 --- a/python/ray/serve/tests/test_deploy_app_2.py +++ b/python/ray/serve/tests/test_deploy_app_2.py @@ -14,7 +14,10 @@ from ray import serve from ray._common.test_utils import SignalActor, wait_for_condition from ray.serve._private.common import DeploymentID, ReplicaID -from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME, SERVE_NAMESPACE +from ray.serve._private.constants import ( + SERVE_DEFAULT_APP_NAME, + SERVE_NAMESPACE, +) from ray.serve._private.test_utils import ( check_num_replicas_eq, ) diff --git a/python/ray/serve/tests/unit/test_autoscaling_policy.py b/python/ray/serve/tests/unit/test_autoscaling_policy.py index ac3960103f9d..93678aa00f29 100644 --- a/python/ray/serve/tests/unit/test_autoscaling_policy.py +++ b/python/ray/serve/tests/unit/test_autoscaling_policy.py @@ -2,6 +2,7 @@ import pytest +from ray.serve._private.autoscaling_state import AutoscalingContext from ray.serve._private.constants import CONTROL_LOOP_INTERVAL_S from ray.serve.autoscaling_policy import ( _calculate_desired_num_replicas, @@ -218,15 +219,27 @@ def test_scaling_factor_scale_up_from_0_replicas( upscale_smoothing_factor=10 if use_upscale_smoothing_factor else None, upscaling_factor=10 if use_upscaling_factor else None, ) - new_num_replicas = replica_queue_length_autoscaling_policy( - curr_target_num_replicas=0, + ctx = AutoscalingContext( + target_num_replicas=0, total_num_requests=1, - num_running_replicas=0, + current_num_replicas=0, config=config, capacity_adjusted_min_replicas=min_replicas, capacity_adjusted_max_replicas=max_replicas, policy_state={}, - ) + deployment_id=None, + deployment_name=None, + app_name=None, + running_replicas=None, + current_time=None, + queued_requests=None, + requests_per_replica=None, + aggregated_metrics=None, + raw_metrics=None, + last_scale_up_time=None, + last_scale_down_time=None, + ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) # 1 * 10 assert new_num_replicas == 10 @@ -236,15 +249,7 @@ def test_scaling_factor_scale_up_from_0_replicas( if use_upscaling_factor: config.upscaling_factor = 0.5 - new_num_replicas = replica_queue_length_autoscaling_policy( - curr_target_num_replicas=0, - total_num_requests=1, - num_running_replicas=0, - config=config, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state={}, - ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) # math.ceil(1 * 0.5) assert new_num_replicas == 1 @@ -271,15 +276,27 @@ def test_scaling_factor_scale_down_to_0_replicas( upscale_delay_s=0, downscale_delay_s=0, ) - new_num_replicas = replica_queue_length_autoscaling_policy( + ctx = AutoscalingContext( config=config, total_num_requests=0, - num_running_replicas=5, - curr_target_num_replicas=5, + current_num_replicas=5, + target_num_replicas=5, capacity_adjusted_min_replicas=min_replicas, capacity_adjusted_max_replicas=max_replicas, policy_state=policy_state, - ) + deployment_id=None, + deployment_name=None, + app_name=None, + running_replicas=None, + current_time=None, + queued_requests=None, + requests_per_replica=None, + aggregated_metrics=None, + raw_metrics=None, + last_scale_up_time=None, + last_scale_down_time=None, + ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 0 @@ -292,17 +309,12 @@ def test_scaling_factor_scale_down_to_0_replicas( config.downscaling_factor = 0.2 # policy_manager = AutoscalingPolicyManager(config) + ctx.total_num_requests = 0 num_replicas = 5 for _ in range(5): - num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=0, - num_running_replicas=num_replicas, - curr_target_num_replicas=num_replicas, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + ctx.current_num_replicas = num_replicas + ctx.target_num_replicas = num_replicas + num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert num_replicas == 0 @@ -328,164 +340,113 @@ def test_upscale_downscale_delay(self): overload_requests = 100 - # Scale up when there are 0 replicas and current_handle_queued_queries > 0 - new_num_replicas = replica_queue_length_autoscaling_policy( + ctx = AutoscalingContext( config=config, total_num_requests=1, - num_running_replicas=0, - curr_target_num_replicas=0, + current_num_replicas=0, + target_num_replicas=0, capacity_adjusted_min_replicas=min_replicas, capacity_adjusted_max_replicas=max_replicas, policy_state=policy_state, + deployment_id=None, + deployment_name=None, + app_name=None, + running_replicas=None, + current_time=None, + queued_requests=None, + requests_per_replica=None, + aggregated_metrics=None, + raw_metrics=None, + last_scale_up_time=None, + last_scale_down_time=None, ) + + # Scale up when there are 0 replicas and current_handle_queued_queries > 0 + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 1 + ctx.total_num_requests = overload_requests + ctx.current_num_replicas = 1 + ctx.target_num_replicas = 1 + # We should scale up only after enough consecutive scale-up decisions. for i in range(upscale_wait_periods): - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=overload_requests, - num_running_replicas=1, - curr_target_num_replicas=1, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 1, i - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=overload_requests, - num_running_replicas=1, - curr_target_num_replicas=1, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 2 no_requests = 0 + ctx.total_num_requests = no_requests + ctx.current_num_replicas = 2 + ctx.target_num_replicas = 2 + # We should scale down only after enough consecutive scale-down decisions. for i in range(downscale_wait_periods): - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=no_requests, - num_running_replicas=2, - curr_target_num_replicas=2, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 2, i - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=no_requests, - num_running_replicas=2, - curr_target_num_replicas=2, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 0 # Get some scale-up decisions, but not enough to trigger a scale up. + ctx.total_num_requests = overload_requests + ctx.current_num_replicas = 1 + ctx.target_num_replicas = 1 + for i in range(int(upscale_wait_periods / 2)): - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=overload_requests, - num_running_replicas=1, - curr_target_num_replicas=1, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 1, i + ctx.total_num_requests = 0 + ctx.current_num_replicas = 1 + ctx.target_num_replicas = 1 + # Interrupt with a scale-down decision. - replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=0, - num_running_replicas=1, - curr_target_num_replicas=1, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + replica_queue_length_autoscaling_policy(ctx=ctx) # The counter should be reset, so it should require `upscale_wait_periods` # more periods before we actually scale up. + + ctx.total_num_requests = overload_requests + ctx.current_num_replicas = 1 + ctx.target_num_replicas = 1 + for i in range(upscale_wait_periods): - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=overload_requests, - num_running_replicas=1, - curr_target_num_replicas=1, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 1, i - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=overload_requests, - num_running_replicas=1, - curr_target_num_replicas=1, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 2 + ctx.total_num_requests = no_requests + ctx.current_num_replicas = 2 + ctx.target_num_replicas = 2 + # Get some scale-down decisions, but not enough to trigger a scale down. for i in range(int(downscale_wait_periods / 2)): - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=no_requests, - num_running_replicas=2, - curr_target_num_replicas=2, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 2, i + ctx.total_num_requests = 200 + ctx.current_num_replicas = 2 + ctx.target_num_replicas = 2 + # Interrupt with a scale-up decision. - replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=200, - num_running_replicas=2, - curr_target_num_replicas=2, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + replica_queue_length_autoscaling_policy(ctx=ctx) # The counter should be reset so it should require `downscale_wait_periods` # more periods before we actually scale down. + ctx.total_num_requests = no_requests + ctx.current_num_replicas = 2 + ctx.target_num_replicas = 2 for i in range(downscale_wait_periods): - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=no_requests, - num_running_replicas=2, - curr_target_num_replicas=2, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 2, i - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=no_requests, - num_running_replicas=2, - curr_target_num_replicas=2, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 0 def test_replicas_delayed_startup(self): @@ -502,54 +463,53 @@ def test_replicas_delayed_startup(self): } config = AutoscalingConfig(**config) - # new_num_replicas = policy_manager.get_decision_num_replicas(1, 100, 1) - new_num_replicas = replica_queue_length_autoscaling_policy( + ctx = AutoscalingContext( config=config, - curr_target_num_replicas=1, + target_num_replicas=1, total_num_requests=100, - num_running_replicas=1, + current_num_replicas=1, capacity_adjusted_min_replicas=min_replicas, capacity_adjusted_max_replicas=max_replicas, policy_state=policy_state, + deployment_id=None, + deployment_name=None, + app_name=None, + running_replicas=None, + current_time=None, + queued_requests=None, + requests_per_replica=None, + aggregated_metrics=None, + raw_metrics=None, + last_scale_up_time=None, + last_scale_down_time=None, ) + + # new_num_replicas = policy_manager.get_decision_num_replicas(1, 100, 1) + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 100 # New target is 100, but no new replicas finished spinning up during this # timestep. - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - curr_target_num_replicas=100, - total_num_requests=100, - num_running_replicas=1, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + ctx.total_num_requests = 100 + ctx.current_num_replicas = 1 + ctx.target_num_replicas = 100 + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 100 # Two new replicas spun up during this timestep. - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - curr_target_num_replicas=100, - total_num_requests=123, - num_running_replicas=3, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + ctx.total_num_requests = 123 + ctx.current_num_replicas = 3 + ctx.target_num_replicas = 100 + + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 123 # A lot of queries got drained and a lot of replicas started up, but # new_num_replicas should not decrease, because of the downscale delay. - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - curr_target_num_replicas=123, - total_num_requests=10, - num_running_replicas=4, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + ctx.total_num_requests = 10 + ctx.current_num_replicas = 4 + ctx.target_num_replicas = 123 + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == 123 @pytest.mark.parametrize("delay_s", [30.0, 0.0]) @@ -578,32 +538,43 @@ def test_fluctuating_ongoing_requests(self, delay_s): underload_requests, overload_requests = 2 * 20, 100 trials = 1000 + ctx = AutoscalingContext( + config=config, + capacity_adjusted_min_replicas=min_replicas, + capacity_adjusted_max_replicas=max_replicas, + policy_state=policy_state, + target_num_replicas=None, + total_num_requests=None, + current_num_replicas=None, + deployment_id=None, + deployment_name=None, + app_name=None, + running_replicas=None, + current_time=None, + queued_requests=None, + requests_per_replica=None, + aggregated_metrics=None, + raw_metrics=None, + last_scale_up_time=None, + last_scale_down_time=None, + ) + new_num_replicas = None for trial in range(trials): if trial % 2 == 0: - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=overload_requests, - num_running_replicas=1, - curr_target_num_replicas=1, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + ctx.target_num_replicas = 1 + ctx.total_num_requests = overload_requests + ctx.current_num_replicas = 1 + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) if delay_s > 0: assert new_num_replicas == 1, trial else: assert new_num_replicas == 2, trial else: - new_num_replicas = replica_queue_length_autoscaling_policy( - config=config, - total_num_requests=underload_requests, - num_running_replicas=2, - curr_target_num_replicas=2, - capacity_adjusted_min_replicas=min_replicas, - capacity_adjusted_max_replicas=max_replicas, - policy_state=policy_state, - ) + ctx.target_num_replicas = 2 + ctx.total_num_requests = underload_requests + ctx.current_num_replicas = 2 + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) if delay_s > 0: assert new_num_replicas == 2, trial else: @@ -624,15 +595,28 @@ def test_single_replica_receives_all_requests(self, ongoing_requests): downscale_delay_s=0.0, ) - new_num_replicas = replica_queue_length_autoscaling_policy( + ctx = AutoscalingContext( config=config, total_num_requests=ongoing_requests, - num_running_replicas=4, - curr_target_num_replicas=4, + current_num_replicas=4, + target_num_replicas=4, capacity_adjusted_min_replicas=min_replicas, capacity_adjusted_max_replicas=max_replicas, policy_state=policy_state, - ) + deployment_id=None, + deployment_name=None, + app_name=None, + running_replicas=None, + current_time=None, + queued_requests=None, + requests_per_replica=None, + aggregated_metrics=None, + raw_metrics=None, + last_scale_up_time=None, + last_scale_down_time=None, + ) + + new_num_replicas, _ = replica_queue_length_autoscaling_policy(ctx=ctx) assert new_num_replicas == ongoing_requests / target_requests diff --git a/python/ray/serve/tests/unit/test_config.py b/python/ray/serve/tests/unit/test_config.py index eb5eb7017061..abf9eb51bc3d 100644 --- a/python/ray/serve/tests/unit/test_config.py +++ b/python/ray/serve/tests/unit/test_config.py @@ -7,7 +7,10 @@ from ray._common.pydantic_compat import ValidationError from ray._common.utils import import_attr from ray.serve._private.config import DeploymentConfig, ReplicaConfig, _proto_to_dict -from ray.serve._private.constants import DEFAULT_AUTOSCALING_POLICY, DEFAULT_GRPC_PORT +from ray.serve._private.constants import ( + DEFAULT_AUTOSCALING_POLICY_NAME, + DEFAULT_GRPC_PORT, +) from ray.serve._private.request_router import PowerOfTwoChoicesRequestRouter from ray.serve._private.utils import DEFAULT from ray.serve.autoscaling_policy import default_autoscaling_policy @@ -801,7 +804,7 @@ def test_autoscaling_policy_import_fails_for_non_existing_policy(): def test_default_autoscaling_policy_import_path(): """Test that default autoscaling policy can be imported.""" - policy = import_attr(DEFAULT_AUTOSCALING_POLICY) + policy = import_attr(DEFAULT_AUTOSCALING_POLICY_NAME) assert policy == default_autoscaling_policy diff --git a/src/ray/protobuf/serve.proto b/src/ray/protobuf/serve.proto index d26aa1e9ad81..a4d7202fb866 100644 --- a/src/ray/protobuf/serve.proto +++ b/src/ray/protobuf/serve.proto @@ -22,6 +22,14 @@ option java_outer_classname = "ServeProtos"; option java_multiple_files = true; +// Configuration options for Serve's autoscaling policy +message AutoscalingPolicy { + // Name of the policy function or the import path of the policy if user passed a string. + // Will be the concatenation of the policy module and the policy name if user passed a + // callable. + string name = 1; +} + // Configuration options for Serve's replica autoscaler. message AutoscalingConfig { // Minimal number of replicas, must be a non-negative integer. @@ -61,9 +69,8 @@ message AutoscalingConfig { // The cloudpickled policy definition. bytes _serialized_policy_def = 11; - // The import path of the policy if user passed a string. Will be the concatenation - // of the policy module and the policy name if user passed a callable. - string _policy = 12; + // The autoscaling policy definition. + AutoscalingPolicy _policy = 12; // Target number of in flight requests per replica. This is the primary configuration // knob for replica autoscaler. Lower the number, the more rapidly the replicas From ecc4c93af0308ccf4b5e08135865766e9a1fbd30 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Wed, 13 Aug 2025 16:35:17 -0700 Subject: [PATCH 088/634] [image] add base-extra layer (#55513) this the layer required to run on anyscale cloud and for running in ray release tests. we have been sourcing this layer from a tarball in s3; this change builds it from the source. Signed-off-by: Lonnie Liu --- .buildkite/_forge.rayci.yml | 82 ++++++++++ .buildkite/release/build.rayci.yml | 8 +- ci/build/build-anyscale-docker.sh | 16 -- ci/docker/ray.cpu.base-extra.wanda.yaml | 7 + ci/docker/ray.cuda.base-extra.wanda.yaml | 7 + ci/ray_ci/anyscale_docker_container.py | 4 +- ci/ray_ci/builder.py | 2 +- ci/ray_ci/docker_container.py | 1 + ci/ray_ci/ray_docker_container.py | 7 +- docker/base-extra/Dockerfile | 189 +++++++++++++++++++++++ 10 files changed, 297 insertions(+), 26 deletions(-) create mode 100644 ci/docker/ray.cpu.base-extra.wanda.yaml create mode 100644 ci/docker/ray.cuda.base-extra.wanda.yaml create mode 100644 docker/base-extra/Dockerfile diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml index 5af4f0a67b08..667d64c36cc8 100644 --- a/.buildkite/_forge.rayci.yml +++ b/.buildkite/_forge.rayci.yml @@ -20,6 +20,19 @@ steps: env: PYTHON_VERSION: "{{matrix}}" + - name: raycpubaseextra + label: "wanda: ray.py{{matrix}}.cpu.base-extra" + wanda: ci/docker/ray.cpu.base-extra.wanda.yaml + matrix: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + env: + PYTHON_VERSION: "{{matrix}}" + IMAGE_TYPE: "ray" + depends_on: raycpubase + - name: raycudabase label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base" tags: @@ -46,6 +59,31 @@ steps: PYTHON_VERSION: "{{matrix.python}}" CUDA_VERSION: "{{matrix.cuda}}" + - name: raycudabaseextra + label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" + wanda: ci/docker/ray.cuda.base-extra.wanda.yaml + matrix: + setup: + python: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + cuda: + - "11.7.1-cudnn8" + - "11.8.0-cudnn8" + - "12.1.1-cudnn8" + - "12.3.2-cudnn9" + - "12.4.1-cudnn" + - "12.5.1-cudnn" + - "12.6.3-cudnn" + - "12.8.1-cudnn" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + IMAGE_TYPE: "ray" + depends_on: raycudabase + - name: ray-llmbase label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base" tags: @@ -63,6 +101,21 @@ steps: PYTHON_VERSION: "{{matrix.python}}" CUDA_VERSION: "{{matrix.cuda}}" + - name: ray-llmbaseextra + label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" + wanda: ci/docker/ray.cuda.base-extra.wanda.yaml + matrix: + setup: + python: + - "3.11" + cuda: + - "12.8.1-cudnn" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + IMAGE_TYPE: "ray-llm" + depends_on: ray-llmbase + - name: ray-mlcudabase label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base" tags: @@ -82,6 +135,23 @@ steps: PYTHON_VERSION: "{{matrix.python}}" CUDA_VERSION: "{{matrix.cuda}}" + - name: ray-mlcudabaseextra + label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" + wanda: ci/docker/ray.cuda.base-extra.wanda.yaml + matrix: + setup: + python: + - "3.9" + - "3.10" + - "3.11" + cuda: + - "12.1.1-cudnn8" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + IMAGE_TYPE: "ray-ml" + depends_on: raycudabaseextra + - name: ray-mlcpubase label: "wanda: ray-ml.py{{matrix}}.cpu.base" tags: @@ -95,3 +165,15 @@ steps: - "3.11" env: PYTHON_VERSION: "{{matrix}}" + + - name: ray-mlcpubaseextra + label: "wanda: ray-ml.py{{matrix}}.cpu.base-extra" + wanda: ci/docker/ray.cpu.base-extra.wanda.yaml + matrix: + - "3.9" + - "3.10" + - "3.11" + env: + PYTHON_VERSION: "{{matrix}}" + IMAGE_TYPE: "ray-ml" + depends_on: raycpubaseextra diff --git a/.buildkite/release/build.rayci.yml b/.buildkite/release/build.rayci.yml index f734ac593361..d3e04eb91848 100644 --- a/.buildkite/release/build.rayci.yml +++ b/.buildkite/release/build.rayci.yml @@ -12,8 +12,8 @@ steps: depends_on: - manylinux - forge - - raycudabase - - raycpubase + - raycudabaseextra + - raycpubaseextra matrix: setup: python: @@ -35,7 +35,7 @@ steps: depends_on: - manylinux - forge - - ray-llmbase + - ray-llmbaseextra matrix: - "3.11" @@ -48,7 +48,7 @@ steps: depends_on: - manylinux - forge - - ray-mlcudabase + - ray-mlcudabaseextra matrix: # This list should be kept in sync with the list of supported Python in # release test suite. We don't have ray-ml release tests for Python 3.10 and 3.11 diff --git a/ci/build/build-anyscale-docker.sh b/ci/build/build-anyscale-docker.sh index 3e4e74ef8a0b..2e6db88d15f3 100755 --- a/ci/build/build-anyscale-docker.sh +++ b/ci/build/build-anyscale-docker.sh @@ -6,24 +6,8 @@ DEST_IMAGE="$2" REQUIREMENTS="$3" ECR="$4" -DATAPLANE_S3_BUCKET="ray-release-automation-results" -DATAPLANE_FILENAME="dataplane_20250624.tar.gz" -DATAPLANE_DIGEST="3cffb55f1a56f0bc6256cbf1a38bf1e764e202a647a4272b80531760f1250059" - -# download dataplane build file -aws s3api get-object --bucket "${DATAPLANE_S3_BUCKET}" \ - --key "${DATAPLANE_FILENAME}" "${DATAPLANE_FILENAME}" - -# check dataplane build file digest -echo "${DATAPLANE_DIGEST} ${DATAPLANE_FILENAME}" | sha256sum -c - -# build anyscale image DOCKER_BUILDKIT=1 docker build \ --build-arg BASE_IMAGE="$SOURCE_IMAGE" \ - -t "$DEST_IMAGE" - < "${DATAPLANE_FILENAME}" - -DOCKER_BUILDKIT=1 docker build \ - --build-arg BASE_IMAGE="$DEST_IMAGE" \ --build-arg PIP_REQUIREMENTS="$REQUIREMENTS" \ -t "$DEST_IMAGE" \ -f release/ray_release/byod/byod.Dockerfile \ diff --git a/ci/docker/ray.cpu.base-extra.wanda.yaml b/ci/docker/ray.cpu.base-extra.wanda.yaml new file mode 100644 index 000000000000..0792fba51ec9 --- /dev/null +++ b/ci/docker/ray.cpu.base-extra.wanda.yaml @@ -0,0 +1,7 @@ +name: "$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra" +froms: ["cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base"] +dockerfile: docker/base-extra/Dockerfile +build_args: + - BASE_IMAGE=cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base +tags: + - cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra diff --git a/ci/docker/ray.cuda.base-extra.wanda.yaml b/ci/docker/ray.cuda.base-extra.wanda.yaml new file mode 100644 index 000000000000..ff1fab0a919f --- /dev/null +++ b/ci/docker/ray.cuda.base-extra.wanda.yaml @@ -0,0 +1,7 @@ +name: "$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra" +froms: ["cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base"] +dockerfile: docker/base-extra/Dockerfile +build_args: + - BASE_IMAGE=cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base +tags: + - cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra diff --git a/ci/ray_ci/anyscale_docker_container.py b/ci/ray_ci/anyscale_docker_container.py index 4025951268b7..7144d065a1e1 100644 --- a/ci/ray_ci/anyscale_docker_container.py +++ b/ci/ray_ci/anyscale_docker_container.py @@ -20,8 +20,8 @@ def run(self) -> None: cmds = [ # build docker image - f"./ci/build/build-anyscale-docker.sh " - f"{ray_image} {anyscale_image} {requirement} {aws_registry}", + "./ci/build/build-anyscale-docker.sh " + + f"{ray_image} {anyscale_image} {requirement} {aws_registry}", # gcloud login "./release/gcloud_docker_login.sh release/aws2gce_iam.json", "export PATH=$(pwd)/google-cloud-sdk/bin:$PATH", diff --git a/ci/ray_ci/builder.py b/ci/ray_ci/builder.py index 62edca1171c2..66fbe735f4ae 100644 --- a/ci/ray_ci/builder.py +++ b/ci/ray_ci/builder.py @@ -172,7 +172,7 @@ def build_anyscale( for p in platform: RayDockerContainer( python_version, p, image_type, architecture, canonical_tag, upload=False - ).run() + ).run(use_base_extra=True) AnyscaleDockerContainer( python_version, p, image_type, architecture, canonical_tag, upload ).run() diff --git a/ci/ray_ci/docker_container.py b/ci/ray_ci/docker_container.py index 73a6f3d2879c..8a83e671e4b7 100644 --- a/ci/ray_ci/docker_container.py +++ b/ci/ray_ci/docker_container.py @@ -47,6 +47,7 @@ def __init__( architecture: str = DEFAULT_ARCHITECTURE, canonical_tag: str = None, upload: bool = False, + use_base_extra: bool = False, ) -> None: assert "RAYCI_CHECKOUT_DIR" in os.environ, "RAYCI_CHECKOUT_DIR not set" diff --git a/ci/ray_ci/ray_docker_container.py b/ci/ray_ci/ray_docker_container.py index 5bc41a1c5cdb..799310e009bb 100644 --- a/ci/ray_ci/ray_docker_container.py +++ b/ci/ray_ci/ray_docker_container.py @@ -14,16 +14,17 @@ class RayDockerContainer(DockerContainer): Container for building and publishing ray docker images """ - def run(self) -> None: + def run(self, use_base_extra: bool = False) -> None: """ Build and publish ray docker images """ assert "RAYCI_BUILD_ID" in os.environ, "RAYCI_BUILD_ID not set" rayci_build_id = os.environ["RAYCI_BUILD_ID"] + base_name = "base" if not use_base_extra else "base-extra" if self.architecture == DEFAULT_ARCHITECTURE: - suffix = "base" + suffix = base_name else: - suffix = f"base-{self.architecture}" + suffix = f"{base_name}-{self.architecture}" base_image = ( f"{_DOCKER_ECR_REPO}:{rayci_build_id}" diff --git a/docker/base-extra/Dockerfile b/docker/base-extra/Dockerfile new file mode 100644 index 000000000000..cf286db30a69 --- /dev/null +++ b/docker/base-extra/Dockerfile @@ -0,0 +1,189 @@ +# syntax=docker/dockerfile:1.3-labs + +ARG BASE_IMAGE="rayproject/ray:latest" + +FROM "$BASE_IMAGE" + +ENV TERM=xterm + +ARG SSH_PORT=5020 + +RUN </dev/stderr + exit 1 +fi + +# Create boto config; makes gsutil happy. +echo "[GoogleCompute]" > "${HOME}/.boto" +echo "service_account = default" >> "${HOME}/.boto" +chmod 600 "${HOME}/.boto" + +if [[ "$ARCH" == "x86_64" ]]; then + sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub + sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub +else + sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/arm64/7fa2af80.pub + # Nvidia does not have machine-learning repo for arm64 +fi + +echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ + | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list +wget -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg \ + | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + +# Add gdb since ray dashboard uses `memray attach`, which requires gdb. +sudo apt-get update -y +sudo apt-get install -y google-cloud-sdk supervisor vim zsh nfs-common zip unzip build-essential ssh curl gdb +sudo apt-get autoclean + +# Install azcopy +AZCOPY_VERSION="10.30.0" +AZCOPY_TMP="$(mktemp -d)" +( + cd "${AZCOPY_TMP}" + if [[ "$ARCH" == "x86_64" ]]; then + curl -sSfL "https://github.com/Azure/azure-storage-azcopy/releases/download/v${AZCOPY_VERSION}/azcopy_linux_amd64_${AZCOPY_VERSION}.tar.gz" \ + -o- | tar -xz "azcopy_linux_amd64_${AZCOPY_VERSION}/azcopy" + sudo mv "azcopy_linux_amd64_${AZCOPY_VERSION}/azcopy" /usr/local/bin/azcopy + else + curl -sSfL "https://github.com/Azure/azure-storage-azcopy/releases/download/v${AZCOPY_VERSION}/azcopy_linux_arm64_${AZCOPY_VERSION}.tar.gz" \ + -o- | tar -xz "azcopy_linux_arm64_${AZCOPY_VERSION}/azcopy" + sudo mv "azcopy_linux_arm64_${AZCOPY_VERSION}/azcopy" /usr/local/bin/azcopy + fi +) +rm -rf "${AZCOPY_TMP}" + +# Install dynolog, only on x86_64 machines. +if [[ "$ARCH" == "x86_64" ]]; then + DYNOLOG_TMP="$(mktemp -d)" + ( + cd "${DYNOLOG_TMP}" + curl -sSL https://github.com/facebookincubator/dynolog/releases/download/v0.3.2/dynolog_0.3.2-0-amd64.deb -o dynolog_0.3.2-0-amd64.deb + sudo dpkg -i dynolog_0.3.2-0-amd64.deb + ) + rm -rf "${DYNOLOG_TMP}" +fi + +# Python dependencies to install. To specify a version, please make the change +# in OSS ray repository, but not here. +PYTHON_REQUIREMENTS=( + azure-identity + jupyterlab + ipywidgets + grpcio + grpcio-tools + + # Pinning jupyter_server_terminals==0.4.4 , the higher version will break the + # webterminal when using an older version of terminado. + jupyter_server_terminals==0.4.4 + + # [backend] is for installing anyscale CLI for use in the anyscale cloud. + "anyscale[backend]" +) + + +PYTHON_VERSION="$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + +uv pip install --system --no-cache-dir --index-strategy unsafe-best-match \ + -c /home/ray/requirements_compiled.txt \ + "${PYTHON_REQUIREMENTS[@]}" + +# Install awscli v2 +AWSCLI_TMP="$(mktemp -d)" +( + cd "${AWSCLI_TMP}" + curl -sfL "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip" + unzip -q awscliv2.zip + sudo ./aws/install +) +rm -rf "${AWSCLI_TMP}" + +# Cleanup unused packages and caches. +$HOME/anaconda3/bin/conda clean -y -all + +# Work around for https://bugs.launchpad.net/ubuntu/+source/openssh/+bug/45234 +sudo mkdir -p /var/run/sshd +# Configure ssh port +echo Port $SSH_PORT | sudo tee -a /etc/ssh/sshd_config + +if [[ ! -d /usr/local/cuda ]]; then + EFA_VERSION="1.42.0" + GDRCOPY_VERSION="" + AWS_OFI_NCCL_VERSION="" +elif [[ -d "/usr/local/cuda-11" ]]; then + EFA_VERSION="1.28.0" + GDRCOPY_VERSION="2.4" + AWS_OFI_NCCL_VERSION="1.7.3-aws" +elif [[ -d "/usr/local/cuda-12" ]]; then + EFA_VERSION="1.42.0" + GDRCOPY_VERSION="2.5" + AWS_OFI_NCCL_VERSION="1.15.0" +else + echo "Unsupported CUDA major version" + exit 1 +fi + +# Install EFA +wget -q "https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_VERSION}.tar.gz" -O "/tmp/aws-efa-installer-${EFA_VERSION}.tar.gz" +wget -q "https://efa-installer.amazonaws.com/aws-efa-installer.key" -O /tmp/aws-efa-installer.key && gpg --import /tmp/aws-efa-installer.key +gpg --fingerprint Date: Wed, 13 Aug 2025 17:06:31 -0700 Subject: [PATCH 089/634] [ci] raydepsets: implementing build arg sets (1/2) (#55408) - converting build arg sets into a dictionary instead of a list - updating naming convention for depsets with build_arg_sets ( suffix: _${BUILD_ARG_SET} for depset name in the config) - adding unit tests --------- Signed-off-by: elliot-barn Signed-off-by: Elliot Barnwell Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- ci/raydepsets/tests/test_cli.py | 80 ++++++++++++++----- .../tests/test_data/test.depsets.yaml | 26 +++--- ci/raydepsets/tests/test_workspace.py | 18 +---- ci/raydepsets/workspace.py | 48 +++++------ 4 files changed, 95 insertions(+), 77 deletions(-) diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index b623a7db3cfa..0fea81effa7e 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -189,15 +189,15 @@ def test_subset(self): constraints=["requirement_constraints_test.txt"], requirements=["requirements_test.txt", "requirements_test_subset.txt"], append_flags=["--no-annotate", "--no-header"], - name="general_depset", + name="general_depset__py311_cpu", output="requirements_compiled_general.txt", ) # Subset general_depset with requirements_test.txt (should lock emoji & pyperclip) manager.subset( - source_depset="general_depset", + source_depset="general_depset__py311_cpu", requirements=["requirements_test.txt"], append_flags=["--no-annotate", "--no-header"], - name="subset_general_depset", + name="subset_general_depset__py311_cpu", output="requirements_compiled_subset_general.txt", ) output_file = Path(tmpdir) / "requirements_compiled_subset_general.txt" @@ -220,16 +220,16 @@ def test_subset_does_not_exist(self): constraints=["requirement_constraints_test.txt"], requirements=["requirements_test.txt", "requirements_test_subset.txt"], append_flags=["--no-annotate", "--no-header"], - name="general_depset", + name="general_depset__py311_cpu", output="requirements_compiled_general.txt", ) with self.assertRaises(RuntimeError): manager.subset( - source_depset="general_depset", + source_depset="general_depset__py311_cpu", requirements=["requirements_compiled_test.txt"], append_flags=["--no-annotate", "--no-header"], - name="subset_general_depset", + name="subset_general_depset__py311_cpu", output="requirements_compiled_subset_general.txt", ) @@ -238,7 +238,7 @@ def test_check_if_subset_exists(self): copy_data_to_tmpdir(tmpdir) manager = _create_test_manager(tmpdir) source_depset = Depset( - name="general_depset", + name="general_depset__py311_cpu", operation="compile", requirements=["requirements_1.txt", "requirements_2.txt"], constraints=["requirement_constraints_1.txt"], @@ -334,20 +334,48 @@ def test_build_graph(self): assert len(manager.build_graph.nodes()) == 6 assert len(manager.build_graph.edges()) == 3 # assert that the compile depsets are first - assert manager.build_graph.nodes["general_depset"]["operation"] == "compile" + assert ( + manager.build_graph.nodes["general_depset__py311_cpu"]["operation"] + == "compile" + ) assert ( manager.build_graph.nodes["subset_general_depset"]["operation"] == "subset" ) assert ( - manager.build_graph.nodes["expand_general_depset"]["operation"] + manager.build_graph.nodes["expand_general_depset__py311_cpu"][ + "operation" + ] == "expand" ) sorted_nodes = list(topological_sort(manager.build_graph)) # assert that the root nodes are the compile depsets assert "ray_base_test_depset" in sorted_nodes[:3] - assert "general_depset" in sorted_nodes[:3] - assert "build_args_test_depset_py311" in sorted_nodes[:3] + assert "general_depset__py311_cpu" in sorted_nodes[:3] + assert "build_args_test_depset__py311_cpu" in sorted_nodes[:3] + + def test_build_graph_predecessors(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = _create_test_manager(tmpdir) + assert manager.build_graph is not None + assert ( + manager.build_graph.nodes["general_depset__py311_cpu"]["operation"] + == "compile" + ) + assert ( + manager.build_graph.nodes["expanded_depset__py311_cpu"]["operation"] + == "compile" + ) + assert ( + manager.build_graph.nodes["expand_general_depset__py311_cpu"][ + "operation" + ] + == "expand" + ) + assert set( + manager.build_graph.predecessors("expand_general_depset__py311_cpu") + ) == {"general_depset__py311_cpu", "expanded_depset__py311_cpu"} def test_build_graph_bad_operation(self): with tempfile.TemporaryDirectory() as tmpdir: @@ -390,22 +418,22 @@ def test_expand(self): constraints=["requirement_constraints_test.txt"], requirements=["requirements_test.txt"], append_flags=["--no-annotate", "--no-header"], - name="general_depset", + name="general_depset__py311_cpu", output="requirements_compiled_general.txt", ) manager.compile( constraints=[], requirements=["requirements_expanded.txt"], append_flags=["--no-annotate", "--no-header"], - name="expanded_depset", + name="expanded_depset__py311_cpu", output="requirements_compiled_expanded.txt", ) manager.expand( - depsets=["general_depset", "expanded_depset"], + depsets=["general_depset__py311_cpu", "expanded_depset__py311_cpu"], constraints=["requirement_constraints_expand.txt"], append_flags=["--no-annotate", "--no-header"], requirements=[], - name="expand_general_depset", + name="expand_general_depset__py311_cpu", output="requirements_compiled_expand_general.txt", ) output_file = Path(tmpdir) / "requirements_compiled_expand_general.txt" @@ -434,15 +462,15 @@ def test_expand_with_requirements(self): constraints=["requirement_constraints_test.txt"], requirements=["requirements_test.txt"], append_flags=["--no-annotate", "--no-header"], - name="general_depset", + name="general_depset__py311_cpu", output="requirements_compiled_general.txt", ) manager.expand( - depsets=["general_depset"], + depsets=["general_depset__py311_cpu"], requirements=["requirements_expanded.txt"], constraints=["requirement_constraints_expand.txt"], append_flags=["--no-annotate", "--no-header"], - name="expand_general_depset", + name="expand_general_depset__py311_cpu", output="requirements_compiled_expand_general.txt", ) output_file = Path(tmpdir) / "requirements_compiled_expand_general.txt" @@ -458,9 +486,8 @@ def test_get_depset_with_build_arg_set(self): config_path="test.depsets.yaml", workspace_dir=tmpdir, ) - depset = manager.get_depset("build_args_test_depset_py311") - assert depset.name == "build_args_test_depset_py311" - assert depset.build_arg_set_name == "py311_cpu" + depset = manager.get_depset("build_args_test_depset__py311_cpu") + assert depset.name == "build_args_test_depset__py311_cpu" def test_get_depset_without_build_arg_set(self): with tempfile.TemporaryDirectory() as tmpdir: @@ -471,7 +498,16 @@ def test_get_depset_without_build_arg_set(self): ) depset = manager.get_depset("ray_base_test_depset") assert depset.name == "ray_base_test_depset" - assert depset.build_arg_set_name is None + + def test_get_depset_with_build_arg_set_and_no_build_arg_set_provided(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = DependencySetManager( + config_path="test.depsets.yaml", + workspace_dir=tmpdir, + ) + with self.assertRaises(KeyError): + manager.get_depset("build_args_test_depset_py311") if __name__ == "__main__": diff --git a/ci/raydepsets/tests/test_data/test.depsets.yaml b/ci/raydepsets/tests/test_data/test.depsets.yaml index c5385b541887..2508025e2524 100644 --- a/ci/raydepsets/tests/test_data/test.depsets.yaml +++ b/ci/raydepsets/tests/test_data/test.depsets.yaml @@ -1,10 +1,8 @@ build_arg_sets: - - name: py311_cpu - build_args: + py311_cpu: CUDA_VERSION: cpu PYTHON_VERSION: py311 - - name: py311_cuda128 - build_args: + py311_cuda128: CUDA_VERSION: 128 PYTHON_VERSION: py311 @@ -16,12 +14,14 @@ depsets: constraints: - requirement_constraints_test.txt output: requirements_compiled.txt - - name: general_depset + - name: general_depset__${PYTHON_VERSION}_${CUDA_VERSION} operation: compile requirements: - requirements_test.txt output: requirements_compiled_general.txt - - name: build_args_test_depset_${PYTHON_VERSION} + build_arg_sets: + - py311_cpu + - name: build_args_test_depset__${PYTHON_VERSION}_${CUDA_VERSION} operation: compile requirements: - requirements_test.txt @@ -30,20 +30,24 @@ depsets: - py311_cpu - name: subset_general_depset operation: subset - source_depset: general_depset + source_depset: general_depset__py311_cpu requirements: - requirement_constraints_subset.txt output: requirements_compiled_subset_general.txt - - name: expanded_depset + - name: expanded_depset__${PYTHON_VERSION}_${CUDA_VERSION} operation: compile requirements: - requirements_expanded.txt output: requirements_compiled_expanded.txt - - name: expand_general_depset + build_arg_sets: + - py311_cpu + - name: expand_general_depset__${PYTHON_VERSION}_${CUDA_VERSION} operation: expand depsets: - - general_depset - - expanded_depset + - general_depset__${PYTHON_VERSION}_${CUDA_VERSION} + - expanded_depset__${PYTHON_VERSION}_${CUDA_VERSION} constraints: - requirement_constraints_expand.txt output: requirements_compiled_expand_general.txt + build_arg_sets: + - py311_cpu diff --git a/ci/raydepsets/tests/test_workspace.py b/ci/raydepsets/tests/test_workspace.py index 2f2429e44a64..430ab1247b90 100644 --- a/ci/raydepsets/tests/test_workspace.py +++ b/ci/raydepsets/tests/test_workspace.py @@ -5,7 +5,7 @@ import pytest from ci.raydepsets.tests.utils import copy_data_to_tmpdir -from ci.raydepsets.workspace import BuildArgSet, Workspace, _substitute_build_args +from ci.raydepsets.workspace import Workspace, _substitute_build_args, BuildArgSet def test_workspace_init(): @@ -19,30 +19,18 @@ def test_parse_build_arg_sets(): copy_data_to_tmpdir(tmpdir) workspace = Workspace(dir=tmpdir) config = workspace.load_config(path=Path(tmpdir) / "test.depsets.yaml") - assert config.build_arg_sets[0].name == "py311_cpu" - assert config.build_arg_sets[0].build_args == { + assert config.build_arg_sets["py311_cpu"].build_args == { "CUDA_VERSION": "cpu", "PYTHON_VERSION": "py311", } - assert config.build_arg_sets[1].name == "py311_cuda128" - assert config.build_arg_sets[1].build_args == { + assert config.build_arg_sets["py311_cuda128"].build_args == { "CUDA_VERSION": 128, "PYTHON_VERSION": "py311", } -def test_from_dict_build_arg_set_matrix(): - with tempfile.TemporaryDirectory() as tmpdir: - copy_data_to_tmpdir(tmpdir) - workspace = Workspace(dir=tmpdir) - config = workspace.load_config(path=Path(tmpdir) / "test.depsets.yaml") - config.build_arg_sets[0].build_args["PYTHON_VERSION"] = "py312" - config.build_arg_sets[0].build_args["CUDA_VERSION"] = "cu128" - - def test_substitute_build_args(): build_arg_set = BuildArgSet( - name="py311_cpu", build_args={ "PYTHON_VERSION": "py311", "CUDA_VERSION": "cu128", diff --git a/ci/raydepsets/workspace.py b/ci/raydepsets/workspace.py index afc5fc3618b0..68d0f9f9bcf6 100644 --- a/ci/raydepsets/workspace.py +++ b/ci/raydepsets/workspace.py @@ -8,7 +8,6 @@ @dataclass class BuildArgSet: - name: str build_args: Dict[str, str] @@ -23,7 +22,6 @@ class Depset: append_flags: List[str] source_depset: Optional[str] = None depsets: Optional[List[str]] = None - build_arg_set_name: Optional[str] = None def _substitute_build_args(obj: Any, build_arg_set: BuildArgSet): @@ -40,7 +38,7 @@ def _substitute_build_args(obj: Any, build_arg_set: BuildArgSet): return obj -def _dict_to_depset(depset: dict, build_arg_set_name: Optional[str] = None) -> Depset: +def _dict_to_depset(depset: dict) -> Depset: return Depset( name=depset.get("name"), requirements=depset.get("requirements", []), @@ -49,7 +47,6 @@ def _dict_to_depset(depset: dict, build_arg_set_name: Optional[str] = None) -> D output=depset.get("output"), source_depset=depset.get("source_depset"), depsets=depset.get("depsets", []), - build_arg_set_name=build_arg_set_name, override_flags=depset.get("override_flags", []), append_flags=depset.get("append_flags", []), ) @@ -58,42 +55,35 @@ def _dict_to_depset(depset: dict, build_arg_set_name: Optional[str] = None) -> D @dataclass class Config: depsets: List[Depset] = field(default_factory=list) - build_arg_sets: List[BuildArgSet] = field(default_factory=list) + build_arg_sets: Dict[str, BuildArgSet] = field(default_factory=dict) - @staticmethod - def from_dict(data: dict) -> "Config": - build_arg_sets = Config.parse_build_arg_sets(data.get("build_arg_sets", [])) + @classmethod + def from_dict(cls, data: dict) -> "Config": + build_arg_sets = cls.parse_build_arg_sets(data.get("build_arg_sets", {})) raw_depsets = data.get("depsets", []) depsets = [] for depset in raw_depsets: - build_arg_set_matrix = depset.get("build_arg_sets", []) - if build_arg_set_matrix: - for build_arg_set_name in build_arg_set_matrix: - build_arg_set = next( - ( - build_arg_set - for build_arg_set in build_arg_sets - if build_arg_set.name == build_arg_set_name - ), - None, - ) + build_arg_set_keys = depset.get("build_arg_sets", []) + if build_arg_set_keys: + # Expand the depset for each build arg set + for build_arg_set_key in build_arg_set_keys: + build_arg_set = build_arg_sets[build_arg_set_key] if build_arg_set is None: - raise KeyError(f"Build arg set {build_arg_set_name} not found") + raise KeyError(f"Build arg set {build_arg_set_key} not found") depset_yaml = _substitute_build_args(depset, build_arg_set) - depsets.append(_dict_to_depset(depset_yaml, build_arg_set_name)) + depsets.append(_dict_to_depset(depset_yaml)) else: - depsets.append(_dict_to_depset(depset=depset)) + depsets.append(_dict_to_depset(depset)) return Config(depsets=depsets, build_arg_sets=build_arg_sets) @staticmethod - def parse_build_arg_sets(build_arg_sets: List[dict]) -> List[BuildArgSet]: - return [ - BuildArgSet( - name=build_arg_set.get("name", None), - build_args=build_arg_set.get("build_args", []), + def parse_build_arg_sets(build_arg_sets: Dict[str, dict]) -> Dict[str, BuildArgSet]: + return { + key: BuildArgSet( + build_args=build_arg_set, ) - for build_arg_set in build_arg_sets - ] + for key, build_arg_set in build_arg_sets.items() + } class Workspace: From 9838ad64d43dbd25b77acfd834500cd96f793e28 Mon Sep 17 00:00:00 2001 From: yi wang <48236141+my-vegetable-has-exploded@users.noreply.github.com> Date: Thu, 14 Aug 2025 08:32:54 +0800 Subject: [PATCH 090/634] [DOC][Tune] fix: remove extra space in tune documentation (#55125) Signed-off-by: my-vegetable-has-exploded Co-authored-by: matthewdeng --- doc/source/tune/key-concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tune/key-concepts.rst b/doc/source/tune/key-concepts.rst index 18d02aba6318..1bd43ae963cb 100644 --- a/doc/source/tune/key-concepts.rst +++ b/doc/source/tune/key-concepts.rst @@ -33,7 +33,7 @@ and the :ref:`Class API `. Both are valid ways of defining a `trainable`, but the Function API is generally recommended and is used throughout the rest of this guide. -Consider an example of optimizing a simple objective function like ``a * (x ** 2) + b `` in which ``a`` and ``b`` are the +Consider an example of optimizing a simple objective function like ``a * (x ** 2) + b`` in which ``a`` and ``b`` are the hyperparameters we want to tune to `minimize` the objective. Since the objective also has a variable ``x``, we need to test for different values of ``x``. Given concrete choices for ``a``, ``b`` and ``x`` we can evaluate the objective function and get a `score` to minimize. From 54ae92386d2b4600e1a9327b4f83c4c48742a412 Mon Sep 17 00:00:00 2001 From: Timothy Seah Date: Wed, 13 Aug 2025 17:40:01 -0700 Subject: [PATCH 091/634] [train] Change DEFAULT variables from strings to bools (#55581) All of these constants are used as the default value of [`env_bool`](https://github.com/ray-project/ray/blob/master/python/ray/_private/ray_constants.py#L41), which returns a bool. Technically this is a no-op since "1" evaluates to True anyway, but this is misleading because "0" actually also evaluates to True. Signed-off-by: Timothy Seah Co-authored-by: Timothy Seah --- python/ray/train/v2/_internal/constants.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/ray/train/v2/_internal/constants.py b/python/ray/train/v2/_internal/constants.py index 2eedeeff3593..65a5eef0b2fc 100644 --- a/python/ray/train/v2/_internal/constants.py +++ b/python/ray/train/v2/_internal/constants.py @@ -46,7 +46,7 @@ # Environment variable to enable the print function patching. ENABLE_PRINT_PATCH_ENV_VAR = "RAY_TRAIN_ENABLE_PRINT_PATCH" -DEFAULT_ENABLE_PRINT_PATCH = "1" +DEFAULT_ENABLE_PRINT_PATCH = True # V2 feature flag. V2_ENABLED_ENV_VAR = "RAY_TRAIN_V2_ENABLED" @@ -56,8 +56,8 @@ "RAY_TRAIN_ENABLE_CONTROLLER_STRUCTURED_LOGGING" ) ENABLE_WORKER_STRUCTURED_LOGGING_ENV_VAR = "RAY_TRAIN_ENABLE_WORKER_STRUCTURED_LOGGING" -DEFAULT_ENABLE_CONTROLLER_LOGGING = "1" -DEFAULT_ENABLE_WORKER_LOGGING = "1" +DEFAULT_ENABLE_CONTROLLER_LOGGING = True +DEFAULT_ENABLE_WORKER_LOGGING = True # Environment variables to configure reconciliation interval for Train state actor. # This determines how many seconds the state actor will wait between @@ -65,7 +65,7 @@ ENABLE_STATE_ACTOR_RECONCILIATION_ENV_VAR = ( "RAY_TRAIN_ENABLE_STATE_ACTOR_RECONCILIATION" ) -DEFAULT_ENABLE_STATE_ACTOR_RECONCILIATION = "1" +DEFAULT_ENABLE_STATE_ACTOR_RECONCILIATION = True STATE_ACTOR_RECONCILIATION_INTERVAL_S_ENV_VAR = ( "RAY_TRAIN_STATE_ACTOR_RECONCILIATION_INTERVAL_S" ) @@ -105,7 +105,7 @@ # Whether or not to run the controller as an actor. RUN_CONTROLLER_AS_ACTOR_ENV_VAR = "RAY_TRAIN_RUN_CONTROLLER_AS_ACTOR" -DEFAULT_RUN_CONTROLLER_AS_ACTOR = "1" +DEFAULT_RUN_CONTROLLER_AS_ACTOR = True def is_v2_enabled() -> bool: From ceaa4fb6f5db3189f77a1ed0f2c407de47ce4792 Mon Sep 17 00:00:00 2001 From: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Date: Wed, 13 Aug 2025 18:22:54 -0700 Subject: [PATCH 092/634] [Serve.llm] Use DEFAULT_MAX_ONGOING_REQUESTS for DPServer (#55583) Signed-off-by: Rui Qiao --- .../llm/_internal/serve/deployments/data_parallel/dp_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py index 0c3cce80e090..200a54c2dfad 100644 --- a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py +++ b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py @@ -2,6 +2,7 @@ from typing import Optional from ray import serve +from ray.llm._internal.serve.configs.constants import DEFAULT_MAX_ONGOING_REQUESTS from ray.llm._internal.serve.configs.server_models import LLMConfig from ray.llm._internal.serve.deployments.data_parallel.dp_rank_assigner import ( DPRankAssigner, @@ -76,6 +77,7 @@ def build_dp_deployment( deployment_options = { "name": name, "num_replicas": dp_size, + "max_ongoing_requests": DEFAULT_MAX_ONGOING_REQUESTS, } return DPServer.as_deployment(deployment_options).bind( llm_config=llm_config, dp_rank_assigner=dp_rank_assigner From 1699dc367f71ac05db8486ac70758090c37403a7 Mon Sep 17 00:00:00 2001 From: Neil Girdhar Date: Wed, 13 Aug 2025 21:33:45 -0400 Subject: [PATCH 093/634] Suppress type error (#50994) Signed-off-by: Neil Girdhar Co-authored-by: matthewdeng --- python/ray/train/constants.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/ray/train/constants.py b/python/ray/train/constants.py index d76965a10338..459c33a7a8f4 100644 --- a/python/ray/train/constants.py +++ b/python/ray/train/constants.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Any import ray from ray._private.ray_constants import env_bool @@ -43,7 +44,9 @@ def _get_ray_train_session_dir() -> str: TUNE_CHECKPOINT_ID = "_current_checkpoint_id" # Deprecated configs can use this value to detect if the user has set it. -_DEPRECATED_VALUE = "DEPRECATED" +# This has type Any to allow it to be assigned to any annotated parameter +# without causing type errors. +_DEPRECATED_VALUE: Any = "DEPRECATED" # ================================================== From f677f564cc56c07e7c93d29c33e2f7314ef34fa1 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Wed, 13 Aug 2025 19:42:02 -0700 Subject: [PATCH 094/634] [core] Improve gcs publish perf + clean up publisher in general (#55560) This PR is focused on two things removing a lot of unnecessary copies when publishing from the GCS + when subscribing to the GCS from + cleaning up publisher related code, e.g. publish functions took callbacks that were always nullptr, always returned Status::OK, etc. There's no actual functional changes in this PR. Copy killing that matters: https://github.com/ray-project/ray/blob/4e5f03e7a1d06b9da8f3a9329400d426055f8ea4/src/ray/gcs/gcs_server/pubsub_handler.cc#L49-L59 Every GCS publish will result in an extra copy here because the `pubsub_reply` we create is heap allocated while the actual reply is arena allocated, so the swap will result in a copy of everything every time we publish to every subscriber. Also, there were multiple extra copies of messages inside gcs_pub_sub.cc when the PythonGcsPublisher publishes and when the PythonGcsSubscriber gets messages. --------- Signed-off-by: dayshah --- python/ray/_raylet.pyx | 19 +- python/ray/includes/common.pxd | 2 +- python/ray/includes/gcs_client.pxi | 5 +- src/mock/ray/gcs/gcs_client/accessor.h | 6 +- src/ray/core_worker/core_worker.cc | 6 +- .../core_worker/test/reference_count_test.cc | 8 +- src/ray/gcs/callback.h | 1 + src/ray/gcs/gcs_client/accessor.cc | 14 +- src/ray/gcs/gcs_client/accessor.h | 6 +- .../gcs/gcs_client/test/gcs_client_test.cc | 14 - src/ray/gcs/gcs_server/gcs_actor_manager.cc | 43 +-- .../gcs_autoscaler_state_manager.cc | 5 +- src/ray/gcs/gcs_server/gcs_job_manager.cc | 23 +- src/ray/gcs/gcs_server/gcs_node_manager.cc | 66 ++--- src/ray/gcs/gcs_server/gcs_node_manager.h | 2 +- src/ray/gcs/gcs_server/gcs_server.cc | 6 +- src/ray/gcs/gcs_server/gcs_worker_manager.cc | 29 +- src/ray/gcs/gcs_server/pubsub_handler.cc | 23 +- src/ray/gcs/gcs_server/pubsub_handler.h | 1 - src/ray/gcs/pb_util.h | 9 +- src/ray/gcs/pb_utils.cc | 23 +- src/ray/gcs/pubsub/gcs_pub_sub.cc | 99 +++---- src/ray/gcs/pubsub/gcs_pub_sub.h | 59 ++--- src/ray/pubsub/README.md | 2 +- src/ray/pubsub/publisher.cc | 85 +++--- src/ray/pubsub/publisher.h | 86 +++--- src/ray/pubsub/test/integration_test.cc | 3 +- src/ray/pubsub/test/publisher_test.cc | 247 ++++++++++++------ src/ray/raylet/node_manager.cc | 12 +- src/ray/raylet/worker_pool.cc | 4 +- 30 files changed, 440 insertions(+), 468 deletions(-) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index db0abd516ed7..d375820f7449 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -2911,22 +2911,25 @@ cdef class GcsLogSubscriber(_GcsSubscriber): with nogil: check_status(self.inner.get().PollLogs(&key_id, timeout_ms, &log_batch)) - c_log_lines = PythonGetLogBatchLines(log_batch) - - log_lines = [] - for c_log_line in c_log_lines: - log_lines.append(c_log_line.decode()) - - return { + result = { "ip": log_batch.ip().decode(), "pid": log_batch.pid().decode(), "job": log_batch.job_id().decode(), "is_err": log_batch.is_error(), - "lines": log_lines, "actor_name": log_batch.actor_name().decode(), "task_name": log_batch.task_name().decode(), } + with nogil: + c_log_lines = PythonGetLogBatchLines(move(log_batch)) + + log_lines = [] + for c_log_line in c_log_lines: + log_lines.append(c_log_line.decode()) + + result["lines"] = log_lines + return result + # This class should only be used for tests cdef class _TestOnly_GcsActorSubscriber(_GcsSubscriber): diff --git a/python/ray/includes/common.pxd b/python/ray/includes/common.pxd index f9a6314dea1d..ab24ec1622ce 100644 --- a/python/ray/includes/common.pxd +++ b/python/ray/includes/common.pxd @@ -668,7 +668,7 @@ cdef extern from "ray/gcs/pubsub/gcs_pub_sub.h" nogil: CRayStatus Close() cdef extern from "ray/gcs/pubsub/gcs_pub_sub.h" namespace "ray::gcs" nogil: - c_vector[c_string] PythonGetLogBatchLines(const CLogBatch& log_batch) + c_vector[c_string] PythonGetLogBatchLines(CLogBatch log_batch) cdef extern from "ray/gcs/gcs_client/gcs_client.h" namespace "ray::gcs" nogil: unordered_map[c_string, c_string] PythonGetNodeLabels( diff --git a/python/ray/includes/gcs_client.pxi b/python/ray/includes/gcs_client.pxi index 63fd99cf8c3c..177bf48fbba3 100644 --- a/python/ray/includes/gcs_client.pxi +++ b/python/ray/includes/gcs_client.pxi @@ -628,9 +628,8 @@ cdef class InnerGcsClient: error_info.set_timestamp(time.time()) with nogil: - check_status_timeout_as_rpc_error( - self.inner.get().Publisher().PublishError( - move(c_key_id), move(error_info), timeout_ms)) + self.inner.get().Publisher().PublishError( + move(c_key_id), move(error_info), timeout_ms) def publish_logs(self, log_json: dict, timeout = None): cdef: diff --git a/src/mock/ray/gcs/gcs_client/accessor.h b/src/mock/ray/gcs/gcs_client/accessor.h index 47d920125293..344729b2065b 100644 --- a/src/mock/ray/gcs/gcs_client/accessor.h +++ b/src/mock/ray/gcs/gcs_client/accessor.h @@ -192,11 +192,7 @@ namespace gcs { class MockErrorInfoAccessor : public ErrorInfoAccessor { public: - MOCK_METHOD(void, - AsyncReportJobError, - (const std::shared_ptr &data_ptr, - const StatusCallback &callback), - (override)); + MOCK_METHOD(void, AsyncReportJobError, (rpc::ErrorTableData data), (override)); }; } // namespace gcs diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 22f3f7fd3513..f6c664909723 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -3700,8 +3700,10 @@ void CoreWorker::HandlePubsubLongPolling(rpc::PubsubLongPollingRequest request, rpc::SendReplyCallback send_reply_callback) { const auto subscriber_id = NodeID::FromBinary(request.subscriber_id()); RAY_LOG(DEBUG).WithField(subscriber_id) << "Got a long polling request from a node"; - object_info_publisher_->ConnectToSubscriber( - request, reply, std::move(send_reply_callback)); + object_info_publisher_->ConnectToSubscriber(request, + reply->mutable_publisher_id(), + reply->mutable_pub_messages(), + std::move(send_reply_callback)); } void CoreWorker::HandlePubsubCommandBatch(rpc::PubsubCommandBatchRequest request, diff --git a/src/ray/core_worker/test/reference_count_test.cc b/src/ray/core_worker/test/reference_count_test.cc index 206a86263fa1..7f5f235d3c14 100644 --- a/src/ray/core_worker/test/reference_count_test.cc +++ b/src/ray/core_worker/test/reference_count_test.cc @@ -277,9 +277,11 @@ class MockDistributedPublisher : public pubsub::PublisherInterface { return true; } - void ConnectToSubscriber(const rpc::PubsubLongPollingRequest &request, - rpc::PubsubLongPollingReply *reply, - rpc::SendReplyCallback send_reply_callback) override {} + void ConnectToSubscriber( + const rpc::PubsubLongPollingRequest &request, + std::string *publisher_id, + google::protobuf::RepeatedPtrField *pub_messages, + rpc::SendReplyCallback send_reply_callback) override {} pubsub::pub_internal::SubscriptionIndex *directory_; SubscriptionCallbackMap *subscription_callback_map_; diff --git a/src/ray/gcs/callback.h b/src/ray/gcs/callback.h index e4ac07a57407..5d00d80805d4 100644 --- a/src/ray/gcs/callback.h +++ b/src/ray/gcs/callback.h @@ -17,6 +17,7 @@ #include #include +#include "absl/container/flat_hash_map.h" #include "ray/common/status.h" namespace ray { diff --git a/src/ray/gcs/gcs_client/accessor.cc b/src/ray/gcs/gcs_client/accessor.cc index 72440b02d387..da8142a4d4fb 100644 --- a/src/ray/gcs/gcs_client/accessor.cc +++ b/src/ray/gcs/gcs_client/accessor.cc @@ -873,19 +873,13 @@ void TaskInfoAccessor::AsyncGetTaskEvents( ErrorInfoAccessor::ErrorInfoAccessor(GcsClient *client_impl) : client_impl_(client_impl) {} -void ErrorInfoAccessor::AsyncReportJobError( - const std::shared_ptr &data_ptr, - const StatusCallback &callback) { - auto job_id = JobID::FromBinary(data_ptr->job_id()); +void ErrorInfoAccessor::AsyncReportJobError(rpc::ErrorTableData data) { + auto job_id = JobID::FromBinary(data.job_id()); RAY_LOG(DEBUG) << "Publishing job error, job id = " << job_id; rpc::ReportJobErrorRequest request; - request.mutable_job_error()->CopyFrom(*data_ptr); + *request.mutable_job_error() = std::move(data); client_impl_->GetGcsRpcClient().ReportJobError( - request, - [job_id, callback](const Status &status, rpc::ReportJobErrorReply &&reply) { - if (callback) { - callback(status); - } + request, [job_id](const Status &status, rpc::ReportJobErrorReply &&reply) { RAY_LOG(DEBUG) << "Finished publishing job error, job id = " << job_id; }); } diff --git a/src/ray/gcs/gcs_client/accessor.h b/src/ray/gcs/gcs_client/accessor.h index cd229ad63ce3..0545fb2268da 100644 --- a/src/ray/gcs/gcs_client/accessor.h +++ b/src/ray/gcs/gcs_client/accessor.h @@ -540,10 +540,8 @@ class ErrorInfoAccessor { /// duplicate messages currently cause failures (the GCS doesn't allow it). A /// natural way to do this is to have finer-grained time stamps. /// - /// \param data_ptr The error message that will be reported to GCS. - /// \param callback Callback that will be called when report is complete. - virtual void AsyncReportJobError(const std::shared_ptr &data_ptr, - const StatusCallback &callback); + /// \param data The error message that will be reported to GCS. + virtual void AsyncReportJobError(rpc::ErrorTableData data); private: GcsClient *client_impl_; diff --git a/src/ray/gcs/gcs_client/test/gcs_client_test.cc b/src/ray/gcs/gcs_client/test/gcs_client_test.cc index 7735f1264927..0739777b954f 100644 --- a/src/ray/gcs/gcs_client/test/gcs_client_test.cc +++ b/src/ray/gcs/gcs_client/test/gcs_client_test.cc @@ -371,13 +371,6 @@ class GcsClientTest : public ::testing::TestWithParam { return resources; } - bool ReportJobError(const std::shared_ptr &error_table_data) { - std::promise promise; - gcs_client_->Errors().AsyncReportJobError( - error_table_data, [&promise](Status status) { promise.set_value(status.ok()); }); - return WaitReady(promise.get_future(), timeout_ms_); - } - bool SubscribeToWorkerFailures( const gcs::ItemCallback &subscribe) { std::promise promise; @@ -654,13 +647,6 @@ TEST_P(GcsClientTest, TestWorkerInfo) { WaitForExpectedCount(worker_failure_count, 2); } -TEST_P(GcsClientTest, TestErrorInfo) { - // Report a job error to GCS. - JobID job_id = JobID::FromInt(1); - auto error_table_data = Mocker::GenErrorTableData(job_id); - ASSERT_TRUE(ReportJobError(error_table_data)); -} - TEST_P(GcsClientTest, TestJobTableResubscribe) { // TODO(mwtian): Support resubscribing with GCS pubsub. GTEST_SKIP(); diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index 8683fed03eb5..bc84131b6c95 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -798,12 +798,11 @@ Status GcsActorManager::RegisterActor(const ray::rpc::RegisterActorRequest &requ "explicitly connect to this namespace with ray.init(namespace=\"" << actor->GetRayNamespace() << "\", ...)"; - auto error_data_ptr = gcs::CreateErrorTableData( + auto error_data = CreateErrorTableData( "detached_actor_anonymous_namespace", stream.str(), absl::Now(), job_id); - RAY_LOG(WARNING) << error_data_ptr->SerializeAsString(); - RAY_CHECK_OK( - gcs_publisher_->PublishError(job_id.Hex(), *error_data_ptr, nullptr)); + RAY_LOG(WARNING) << error_data.SerializeAsString(); + gcs_publisher_->PublishError(job_id.Hex(), std::move(error_data)); } actors_in_namespace.emplace(actor->GetName(), actor->GetActorID()); } else { @@ -866,8 +865,8 @@ Status GcsActorManager::RegisterActor(const ray::rpc::RegisterActorRequest &requ return; } - RAY_CHECK_OK(gcs_publisher_->PublishActor( - actor->GetActorID(), actor->GetActorTableData(), nullptr)); + gcs_publisher_->PublishActor(actor->GetActorID(), + actor->GetActorTableData()); // Invoke all callbacks for all registration requests of this actor // (duplicated requests are included) and remove all of them from // actor_to_register_callbacks_. @@ -948,7 +947,7 @@ Status GcsActorManager::CreateActor(const ray::rpc::CreateActorRequest &request, current_sys_time_ms()); // Pub this state for dashboard showing. - RAY_CHECK_OK(gcs_publisher_->PublishActor(actor_id, actor_table_data, nullptr)); + gcs_publisher_->PublishActor(actor_id, actor_table_data); actor->WriteActorExportEvent(); RemoveUnresolvedActor(actor); @@ -1168,8 +1167,8 @@ void GcsActorManager::DestroyActor(const ActorID &actor_id, if (done_callback) { done_callback(); } - RAY_CHECK_OK(gcs_publisher_->PublishActor( - actor_id, GenActorDataOnlyWithStates(*actor_table_data), nullptr)); + gcs_publisher_->PublishActor(actor_id, + GenActorDataOnlyWithStates(*actor_table_data)); if (!is_restartable) { RAY_CHECK_OK(gcs_table_storage_->ActorTaskSpecTable().Delete( actor_id, {[](auto) {}, io_context_})); @@ -1404,8 +1403,8 @@ void GcsActorManager::SetPreemptedAndPublish(const NodeID &node_id) { actor_id, actor_table_data, {[this, actor_id, actor_table_data](Status status) { - RAY_CHECK_OK(gcs_publisher_->PublishActor( - actor_id, GenActorDataOnlyWithStates(actor_table_data), nullptr)); + gcs_publisher_->PublishActor(actor_id, + GenActorDataOnlyWithStates(actor_table_data)); }, io_context_})); } @@ -1483,8 +1482,8 @@ void GcsActorManager::RestartActor(const ActorID &actor_id, if (done_callback) { done_callback(); } - RAY_CHECK_OK(gcs_publisher_->PublishActor( - actor_id, GenActorDataOnlyWithStates(*mutable_actor_table_data), nullptr)); + gcs_publisher_->PublishActor( + actor_id, GenActorDataOnlyWithStates(*mutable_actor_table_data)); actor->WriteActorExportEvent(); }, io_context_})); @@ -1512,8 +1511,8 @@ void GcsActorManager::RestartActor(const ActorID &actor_id, if (done_callback) { done_callback(); } - RAY_CHECK_OK(gcs_publisher_->PublishActor( - actor_id, GenActorDataOnlyWithStates(*mutable_actor_table_data), nullptr)); + gcs_publisher_->PublishActor( + actor_id, GenActorDataOnlyWithStates(*mutable_actor_table_data)); RAY_CHECK_OK(gcs_table_storage_->ActorTaskSpecTable().Delete( actor_id, {[](auto) {}, io_context_})); actor->WriteActorExportEvent(); @@ -1618,14 +1617,18 @@ void GcsActorManager::OnActorCreationSuccess(const std::shared_ptr &ac RAY_CHECK(!node_id.IsNil()); RAY_CHECK(created_actors_[node_id].emplace(worker_id, actor_id).second); - auto actor_table_data = *mutable_actor_table_data; + auto actor_data_only_with_states = + GenActorDataOnlyWithStates(*mutable_actor_table_data); // The backend storage is reliable in the future, so the status must be ok. RAY_CHECK_OK(gcs_table_storage_->ActorTable().Put( actor_id, - actor_table_data, - {[this, actor_id, actor_table_data, actor, reply](Status status) { - RAY_CHECK_OK(gcs_publisher_->PublishActor( - actor_id, GenActorDataOnlyWithStates(actor_table_data), nullptr)); + *mutable_actor_table_data, + {[this, + actor_id, + actor_data_only_with_states = std::move(actor_data_only_with_states), + actor, + reply](Status status) mutable { + gcs_publisher_->PublishActor(actor_id, std::move(actor_data_only_with_states)); actor->WriteActorExportEvent(); // Invoke all callbacks for all registration requests of this actor (duplicated // requests are included) and remove all of them from diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc index e3873fe82ff9..2c234c92d6ac 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc @@ -93,10 +93,9 @@ void GcsAutoscalerStateManager::HandleReportAutoscalingState( if (gcs_publisher_ != nullptr) { std::string error_type = "infeasible_resource_requests"; - auto error_data_ptr = gcs::CreateErrorTableData( + auto error_data = CreateErrorTableData( error_type, error_message, absl::FromUnixMillis(current_time_ms())); - RAY_CHECK_OK( - gcs_publisher_->PublishError(session_name_, *error_data_ptr, nullptr)); + gcs_publisher_->PublishError(session_name_, std::move(error_data)); } } }; diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.cc b/src/ray/gcs/gcs_server/gcs_job_manager.cc index 8211d1c77ba2..3b42b2119db2 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_job_manager.cc @@ -21,6 +21,7 @@ #include #include +#include "absl/strings/match.h" #include "ray/gcs/pb_util.h" #include "ray/stats/metric.h" @@ -47,7 +48,7 @@ void GcsJobManager::WriteDriverJobExportEvent(rpc::JobTableData job_data) const if (!export_event_write_enabled_) { return; } - if (job_data.config().ray_namespace().find(kRayInternalNamespacePrefix) == 0) { + if (absl::StartsWith(job_data.config().ray_namespace(), kRayInternalNamespacePrefix)) { // Namespace of this job starts with _ray_internal_ so // don't write export event. return; @@ -102,15 +103,13 @@ void GcsJobManager::HandleAddJob(rpc::AddJobRequest request, job_table_data = mutable_job_table_data, reply, send_reply_callback = - std::move(send_reply_callback)](const Status &status) { - RAY_CHECK(thread_checker_.IsOnSameThread()); - + std::move(send_reply_callback)](const Status &status) mutable { + WriteDriverJobExportEvent(job_table_data); if (!status.ok()) { RAY_LOG(ERROR).WithField(job_id).WithField("driver_pid", job_table_data.driver_pid()) << "Failed to register job."; } else { - RAY_CHECK_OK(gcs_publisher_.PublishJob(job_id, job_table_data, /*done=*/nullptr)); if (job_table_data.config().has_runtime_env_info()) { runtime_env_manager_.AddURIReference(job_id.Hex(), job_table_data.config().runtime_env_info()); @@ -123,16 +122,14 @@ void GcsJobManager::HandleAddJob(rpc::AddJobRequest request, // Intentionally not checking return value, since the function could be invoked for // multiple times and requires idempotency (i.e. due to retry). running_job_start_times_.insert({job_id, job_table_data.start_time()}); + gcs_publisher_.PublishJob(job_id, std::move(job_table_data)); } - WriteDriverJobExportEvent(job_table_data); + GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); }; - Status status = gcs_table_storage_.JobTable().Put( - job_id, mutable_job_table_data, {on_done, io_context_}); - if (!status.ok()) { - on_done(status); - } + RAY_UNUSED(gcs_table_storage_.JobTable().Put( + job_id, mutable_job_table_data, {std::move(on_done), io_context_})); } void GcsJobManager::MarkJobAsFinished(rpc::JobTableData job_table_data, @@ -151,7 +148,7 @@ void GcsJobManager::MarkJobAsFinished(rpc::JobTableData job_table_data, if (!status.ok()) { RAY_LOG(ERROR).WithField(job_id) << "Failed to mark job as finished."; } else { - RAY_CHECK_OK(gcs_publisher_.PublishJob(job_id, job_table_data, nullptr)); + gcs_publisher_.PublishJob(job_id, job_table_data); runtime_env_manager_.RemoveURIReference(job_id.Hex()); ClearJobInfos(job_table_data); RAY_LOG(DEBUG).WithField(job_id) << "Marked job as finished."; @@ -456,7 +453,7 @@ void GcsJobManager::HandleReportJobError(rpc::ReportJobErrorRequest request, rpc::ReportJobErrorReply *reply, rpc::SendReplyCallback send_reply_callback) { auto job_id = JobID::FromBinary(request.job_error().job_id()); - RAY_CHECK_OK(gcs_publisher_.PublishError(job_id.Hex(), request.job_error(), nullptr)); + gcs_publisher_.PublishError(job_id.Hex(), std::move(*request.mutable_job_error())); GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); } diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.cc b/src/ray/gcs/gcs_server/gcs_node_manager.cc index 112e45a594d4..15347e782001 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_node_manager.cc @@ -90,15 +90,15 @@ void GcsNodeManager::HandleRegisterNode(rpc::RegisterNodeRequest request, .WithField("node_address", node_info.node_manager_address()) << "Registering new node."; - auto on_done = - [this, node_id, node_info, reply, send_reply_callback](const Status &status) { - RAY_CHECK_OK(status) << "Failed to register node '" << node_id << "'."; - RAY_LOG(DEBUG).WithField(node_id) << "Finished registering node."; - RAY_CHECK_OK(gcs_publisher_->PublishNodeInfo(node_id, node_info, nullptr)); - AddNode(std::make_shared(node_info)); - WriteNodeExportEvent(node_info); - GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); - }; + auto on_done = [this, node_id, node_info_copy = node_info, reply, send_reply_callback]( + const Status &status) mutable { + RAY_CHECK_OK(status) << "Failed to register node '" << node_id << "'."; + RAY_LOG(DEBUG).WithField(node_id) << "Finished registering node."; + AddNode(std::make_shared(node_info_copy)); + WriteNodeExportEvent(node_info_copy); + gcs_publisher_->PublishNodeInfo(node_id, std::move(node_info_copy)); + GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); + }; if (node_info.is_head_node()) { // mark all old head nodes as dead if exists: // 1. should never happen when HA is not used @@ -114,18 +114,17 @@ void GcsNodeManager::HandleRegisterNode(rpc::RegisterNodeRequest request, RAY_CHECK_LE(head_nodes.size(), 1UL); if (head_nodes.size() == 1) { OnNodeFailure(head_nodes[0], - [this, node_id, node_info, on_done](const Status &status) { - RAY_CHECK_OK(status); + [this, node_id, node_info, on_done = std::move(on_done)]() { RAY_CHECK_OK(gcs_table_storage_->NodeTable().Put( node_id, node_info, {on_done, io_context_})); }); } else { RAY_CHECK_OK(gcs_table_storage_->NodeTable().Put( - node_id, node_info, {on_done, io_context_})); + node_id, node_info, {std::move(on_done), io_context_})); } } else { - RAY_CHECK_OK( - gcs_table_storage_->NodeTable().Put(node_id, node_info, {on_done, io_context_})); + RAY_CHECK_OK(gcs_table_storage_->NodeTable().Put( + node_id, node_info, {std::move(on_done), io_context_})); } ++counts_[CountType::REGISTER_NODE_REQUEST]; } @@ -166,7 +165,7 @@ void GcsNodeManager::HandleUnregisterNode(rpc::UnregisterNodeRequest request, node_info_delta->set_end_time_ms(node->end_time_ms()); auto on_put_done = [this, node_id, node_info_delta, node](const Status &status) { - RAY_CHECK_OK(gcs_publisher_->PublishNodeInfo(node_id, *node_info_delta, nullptr)); + gcs_publisher_->PublishNodeInfo(node_id, *node_info_delta); WriteNodeExportEvent(*node); }; RAY_CHECK_OK( @@ -430,9 +429,9 @@ std::shared_ptr GcsNodeManager::RemoveNode( .WithField("ip", removed_node->node_manager_address()) << error_message.str(); RAY_LOG(WARNING) << error_message.str(); - auto error_data_ptr = gcs::CreateErrorTableData( + auto error_data = CreateErrorTableData( type, error_message.str(), absl::FromUnixMillis(current_time_ms())); - RAY_CHECK_OK(gcs_publisher_->PublishError(node_id.Hex(), *error_data_ptr, nullptr)); + gcs_publisher_->PublishError(node_id.Hex(), std::move(error_data)); } // Notify all listeners. @@ -443,8 +442,8 @@ std::shared_ptr GcsNodeManager::RemoveNode( return removed_node; } -void GcsNodeManager::OnNodeFailure(const NodeID &node_id, - const StatusCallback &node_table_updated_callback) { +void GcsNodeManager::OnNodeFailure( + const NodeID &node_id, const std::function &node_table_updated_callback) { auto maybe_node = GetAliveNode(node_id); if (maybe_node.has_value()) { rpc::NodeDeathInfo death_info = InferDeathInfo(node_id); @@ -453,24 +452,27 @@ void GcsNodeManager::OnNodeFailure(const NodeID &node_id, node->set_end_time_ms(current_sys_time_ms()); AddDeadNodeToCache(node); - auto node_info_delta = std::make_shared(); - node_info_delta->set_node_id(node->node_id()); - node_info_delta->set_state(node->state()); - node_info_delta->set_end_time_ms(node->end_time_ms()); - node_info_delta->mutable_death_info()->CopyFrom(node->death_info()); - - auto on_done = [this, node_id, node_table_updated_callback, node_info_delta, node]( - const Status &status) { + rpc::GcsNodeInfo node_info_delta; + node_info_delta.set_node_id(node->node_id()); + node_info_delta.set_state(node->state()); + node_info_delta.set_end_time_ms(node->end_time_ms()); + node_info_delta.mutable_death_info()->CopyFrom(node->death_info()); + + auto on_done = [this, + node_id, + node_table_updated_callback, + node_info_delta = std::move(node_info_delta), + node](const Status &status) mutable { WriteNodeExportEvent(*node); if (node_table_updated_callback != nullptr) { - node_table_updated_callback(Status::OK()); + node_table_updated_callback(); } - RAY_CHECK_OK(gcs_publisher_->PublishNodeInfo(node_id, *node_info_delta, nullptr)); + gcs_publisher_->PublishNodeInfo(node_id, std::move(node_info_delta)); }; - RAY_CHECK_OK( - gcs_table_storage_->NodeTable().Put(node_id, *node, {on_done, io_context_})); + RAY_CHECK_OK(gcs_table_storage_->NodeTable().Put( + node_id, *node, {std::move(on_done), io_context_})); } else if (node_table_updated_callback != nullptr) { - node_table_updated_callback(Status::OK()); + node_table_updated_callback(); } } diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.h b/src/ray/gcs/gcs_server/gcs_node_manager.h index eb3107fcc4ba..ff196f60b263 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/ray/gcs/gcs_server/gcs_node_manager.h @@ -96,7 +96,7 @@ class GcsNodeManager : public rpc::NodeInfoHandler { /// \param node_table_updated_callback The status callback function after /// faled node info is updated to gcs node table. void OnNodeFailure(const NodeID &node_id, - const StatusCallback &node_table_updated_callback); + const std::function &node_table_updated_callback); /// Add an alive node. /// diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index f5fff4004c96..d8d4b502f2a7 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -137,11 +137,7 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, RAY_LOG(FATAL) << "Unexpected storage type: " << storage_type_; } - // Init GCS publisher instance. - std::unique_ptr inner_publisher; - // Init grpc based pubsub on GCS. - // TODO(yic): Move this into GcsPublisher. - inner_publisher = std::make_unique( + auto inner_publisher = std::make_unique( /*channels=*/ std::vector{ rpc::ChannelType::GCS_ACTOR_CHANNEL, diff --git a/src/ray/gcs/gcs_server/gcs_worker_manager.cc b/src/ray/gcs/gcs_server/gcs_worker_manager.cc index 3e6962595874..b45e47cf080c 100644 --- a/src/ray/gcs/gcs_server/gcs_worker_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_worker_manager.cc @@ -40,7 +40,7 @@ void GcsWorkerManager::HandleReportWorkerFailure( GetWorkerInfo( worker_id, {[this, reply, send_reply_callback, worker_id, request = std::move(request)]( - const std::optional &result) { + std::optional result) { const auto &worker_address = request.worker_failure().worker_address(); const auto node_id = NodeID::FromBinary(worker_address.node_id()); std::string message = @@ -63,10 +63,10 @@ void GcsWorkerManager::HandleReportWorkerFailure( "are lots of this logs, that might indicate there are " "unexpected failures in the cluster."; } - auto worker_failure_data = std::make_shared(); - if (result) { - worker_failure_data->CopyFrom(*result); - } + auto worker_failure_data = + result.has_value() + ? std::make_shared(std::move(*result)) + : std::make_shared(); worker_failure_data->MergeFrom(request.worker_failure()); worker_failure_data->set_is_alive(false); @@ -75,15 +75,16 @@ void GcsWorkerManager::HandleReportWorkerFailure( } auto on_done = [this, - worker_address, - worker_id, node_id, + worker_id, worker_failure_data, reply, - send_reply_callback](const Status &status) { + send_reply_callback, + worker_ip_address = + worker_address.ip_address()](const Status &status) { if (!status.ok()) { RAY_LOG(ERROR).WithField(worker_id).WithField(node_id).WithField( - "worker_address", worker_address.ip_address()) + "worker_address", worker_ip_address) << "Failed to report worker failure"; } else { if (!IsIntentionalWorkerFailure(worker_failure_data->exit_type())) { @@ -95,8 +96,7 @@ void GcsWorkerManager::HandleReportWorkerFailure( worker_failure.set_worker_id( worker_failure_data->worker_address().worker_id()); worker_failure.set_node_id(worker_failure_data->worker_address().node_id()); - RAY_CHECK_OK( - gcs_publisher_.PublishWorkerFailure(worker_id, worker_failure, nullptr)); + gcs_publisher_.PublishWorkerFailure(worker_id, std::move(worker_failure)); } GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); }; @@ -105,11 +105,8 @@ void GcsWorkerManager::HandleReportWorkerFailure( // receives the worker registration information first and then the worker failure // message, so we delete the get operation. Related issues: // https://github.com/ray-project/ray/pull/11599 - Status status = gcs_table_storage_.WorkerTable().Put( - worker_id, *worker_failure_data, {on_done, io_context_}); - if (!status.ok()) { - on_done(status); - } + RAY_UNUSED(gcs_table_storage_.WorkerTable().Put( + worker_id, *worker_failure_data, {std::move(on_done), io_context_})); if (request.worker_failure().exit_type() == rpc::WorkerExitType::SYSTEM_ERROR || request.worker_failure().exit_type() == diff --git a/src/ray/gcs/gcs_server/pubsub_handler.cc b/src/ray/gcs/gcs_server/pubsub_handler.cc index 1bc889c912ef..865acac8f9e7 100644 --- a/src/ray/gcs/gcs_server/pubsub_handler.cc +++ b/src/ray/gcs/gcs_server/pubsub_handler.cc @@ -14,7 +14,6 @@ #include "ray/gcs/gcs_server/pubsub_handler.h" -#include #include #include @@ -43,23 +42,13 @@ void InternalPubSubHandler::HandleGcsSubscriberPoll( rpc::GcsSubscriberPollReply *reply, rpc::SendReplyCallback send_reply_callback) { rpc::PubsubLongPollingRequest pubsub_req; - pubsub_req.set_subscriber_id(request.subscriber_id()); - pubsub_req.set_publisher_id(request.publisher_id()); + pubsub_req.set_subscriber_id(std::move(*request.mutable_subscriber_id())); + pubsub_req.set_publisher_id(std::move(*request.mutable_publisher_id())); pubsub_req.set_max_processed_sequence_id(request.max_processed_sequence_id()); - auto pubsub_reply = std::make_shared(); - auto pubsub_reply_ptr = pubsub_reply.get(); - gcs_publisher_.GetPublisher().ConnectToSubscriber( - pubsub_req, - pubsub_reply_ptr, - [reply, - reply_cb = std::move(send_reply_callback), - pubsub_reply = std::move(pubsub_reply)](ray::Status status, - std::function success_cb, - std::function failure_cb) { - reply->mutable_pub_messages()->Swap(pubsub_reply->mutable_pub_messages()); - reply->set_publisher_id(std::move(*pubsub_reply->mutable_publisher_id())); - reply_cb(std::move(status), std::move(success_cb), std::move(failure_cb)); - }); + gcs_publisher_.GetPublisher().ConnectToSubscriber(pubsub_req, + reply->mutable_publisher_id(), + reply->mutable_pub_messages(), + std::move(send_reply_callback)); } // Similar for HandleGcsSubscriberPoll() above, needs to use diff --git a/src/ray/gcs/gcs_server/pubsub_handler.h b/src/ray/gcs/gcs_server/pubsub_handler.h index fc5d92dd9abf..34554c74bc39 100644 --- a/src/ray/gcs/gcs_server/pubsub_handler.h +++ b/src/ray/gcs/gcs_server/pubsub_handler.h @@ -20,7 +20,6 @@ #include "absl/container/flat_hash_set.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" #include "ray/rpc/gcs/gcs_rpc_server.h" -#include "src/ray/protobuf/gcs_service.grpc.pb.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/pb_util.h b/src/ray/gcs/pb_util.h index 2bb0c6ace6d4..54b7315e504f 100644 --- a/src/ray/gcs/pb_util.h +++ b/src/ray/gcs/pb_util.h @@ -64,11 +64,10 @@ inline std::shared_ptr CreateJobTableData( } /// Helper function to produce error table data. -std::shared_ptr CreateErrorTableData( - const std::string &error_type, - const std::string &error_msg, - absl::Time timestamp, - const JobID &job_id = JobID::Nil()); +rpc::ErrorTableData CreateErrorTableData(const std::string &error_type, + const std::string &error_msg, + absl::Time timestamp, + const JobID &job_id = JobID::Nil()); /// Helper function to produce worker failure data. inline std::shared_ptr CreateWorkerFailureData( diff --git a/src/ray/gcs/pb_utils.cc b/src/ray/gcs/pb_utils.cc index 1c510b9b5902..c75e78a12167 100644 --- a/src/ray/gcs/pb_utils.cc +++ b/src/ray/gcs/pb_utils.cc @@ -23,27 +23,26 @@ namespace ray::gcs { -std::shared_ptr CreateErrorTableData( - const std::string &error_type, - const std::string &error_msg, - absl::Time timestamp, - const JobID &job_id) { +rpc::ErrorTableData CreateErrorTableData(const std::string &error_type, + const std::string &error_msg, + absl::Time timestamp, + const JobID &job_id) { uint32_t max_error_msg_size_bytes = RayConfig::instance().max_error_msg_size_bytes(); - auto error_info_ptr = std::make_shared(); - error_info_ptr->set_type(error_type); + rpc::ErrorTableData error_info; + error_info.set_type(error_type); if (error_msg.length() > max_error_msg_size_bytes) { std::string formatted_error_message = absl::StrFormat( "The message size exceeds %d bytes. Find the full log from the log files. Here " "is abstract: %s", max_error_msg_size_bytes, std::string_view{error_msg}.substr(0, max_error_msg_size_bytes)); - error_info_ptr->set_error_message(std::move(formatted_error_message)); + error_info.set_error_message(std::move(formatted_error_message)); } else { - error_info_ptr->set_error_message(error_msg); + error_info.set_error_message(error_msg); } - error_info_ptr->set_timestamp(absl::ToUnixMillis(timestamp)); - error_info_ptr->set_job_id(job_id.Binary()); - return error_info_ptr; + error_info.set_timestamp(absl::ToUnixMillis(timestamp)); + error_info.set_job_id(job_id.Binary()); + return error_info; } } // namespace ray::gcs diff --git a/src/ray/gcs/pubsub/gcs_pub_sub.cc b/src/ray/gcs/pubsub/gcs_pub_sub.cc index ea722b03f5e1..ea2884d29306 100644 --- a/src/ray/gcs/pubsub/gcs_pub_sub.cc +++ b/src/ray/gcs/pubsub/gcs_pub_sub.cc @@ -24,74 +24,45 @@ namespace ray { namespace gcs { -Status GcsPublisher::PublishActor(const ActorID &id, - rpc::ActorTableData message, - const StatusCallback &done) { +void GcsPublisher::PublishActor(const ActorID &id, rpc::ActorTableData message) { rpc::PubMessage msg; msg.set_channel_type(rpc::ChannelType::GCS_ACTOR_CHANNEL); msg.set_key_id(id.Binary()); *msg.mutable_actor_message() = std::move(message); publisher_->Publish(std::move(msg)); - if (done != nullptr) { - done(Status::OK()); - } - return Status::OK(); } -Status GcsPublisher::PublishJob(const JobID &id, - const rpc::JobTableData &message, - const StatusCallback &done) { +void GcsPublisher::PublishJob(const JobID &id, rpc::JobTableData message) { rpc::PubMessage msg; msg.set_channel_type(rpc::ChannelType::GCS_JOB_CHANNEL); msg.set_key_id(id.Binary()); - *msg.mutable_job_message() = message; + *msg.mutable_job_message() = std::move(message); publisher_->Publish(std::move(msg)); - if (done != nullptr) { - done(Status::OK()); - } - return Status::OK(); } -Status GcsPublisher::PublishNodeInfo(const NodeID &id, - const rpc::GcsNodeInfo &message, - const StatusCallback &done) { +void GcsPublisher::PublishNodeInfo(const NodeID &id, rpc::GcsNodeInfo message) { rpc::PubMessage msg; msg.set_channel_type(rpc::ChannelType::GCS_NODE_INFO_CHANNEL); msg.set_key_id(id.Binary()); - *msg.mutable_node_info_message() = message; + *msg.mutable_node_info_message() = std::move(message); publisher_->Publish(std::move(msg)); - if (done != nullptr) { - done(Status::OK()); - } - return Status::OK(); } -Status GcsPublisher::PublishWorkerFailure(const WorkerID &id, - const rpc::WorkerDeltaData &message, - const StatusCallback &done) { +void GcsPublisher::PublishWorkerFailure(const WorkerID &id, + rpc::WorkerDeltaData message) { rpc::PubMessage msg; msg.set_channel_type(rpc::ChannelType::GCS_WORKER_DELTA_CHANNEL); msg.set_key_id(id.Binary()); - *msg.mutable_worker_delta_message() = message; + *msg.mutable_worker_delta_message() = std::move(message); publisher_->Publish(std::move(msg)); - if (done != nullptr) { - done(Status::OK()); - } - return Status::OK(); } -Status GcsPublisher::PublishError(const std::string &id, - const rpc::ErrorTableData &message, - const StatusCallback &done) { +void GcsPublisher::PublishError(std::string id, rpc::ErrorTableData message) { rpc::PubMessage msg; msg.set_channel_type(rpc::ChannelType::RAY_ERROR_INFO_CHANNEL); - msg.set_key_id(id); - *msg.mutable_error_info_message() = message; + msg.set_key_id(std::move(id)); + *msg.mutable_error_info_message() = std::move(message); publisher_->Publish(std::move(msg)); - if (done != nullptr) { - done(Status::OK()); - } - return Status::OK(); } std::string GcsPublisher::DebugString() const { return publisher_->DebugString(); } @@ -113,7 +84,7 @@ Status GcsSubscriber::SubscribeAllJobs( std::make_unique(), rpc::ChannelType::GCS_JOB_CHANNEL, gcs_address_, - [done](Status status) { + [done](const Status &status) { if (done != nullptr) { done(status); } @@ -145,7 +116,7 @@ Status GcsSubscriber::SubscribeActor( rpc::ChannelType::GCS_ACTOR_CHANNEL, gcs_address_, id.Binary(), - [done](Status status) { + [done](const Status &status) { if (done != nullptr) { done(status); } @@ -181,7 +152,7 @@ void GcsSubscriber::SubscribeAllNodeInfo(const ItemCallback &s std::make_unique(), rpc::ChannelType::GCS_NODE_INFO_CHANNEL, gcs_address_, - [done](Status status) { + [done](const Status &status) { if (done != nullptr) { done(status); } @@ -206,7 +177,7 @@ Status GcsSubscriber::SubscribeAllWorkerFailures( rpc::ChannelType::GCS_WORKER_DELTA_CHANNEL, gcs_address_, /*subscribe_done_callback=*/ - [done](Status status) { + [done](const Status &status) { if (done != nullptr) { done(status); } @@ -216,24 +187,22 @@ Status GcsSubscriber::SubscribeAllWorkerFailures( return Status::OK(); } -std::vector PythonGetLogBatchLines(const rpc::LogBatch &log_batch) { - return std::vector(log_batch.lines().begin(), log_batch.lines().end()); +std::vector PythonGetLogBatchLines(rpc::LogBatch log_batch) { + return std::vector( + std::make_move_iterator(log_batch.mutable_lines()->begin()), + std::make_move_iterator(log_batch.mutable_lines()->end())); } PythonGcsSubscriber::PythonGcsSubscriber(const std::string &gcs_address, int gcs_port, rpc::ChannelType channel_type, - const std::string &subscriber_id, - const std::string &worker_id) - : channel_type_(channel_type), - subscriber_id_(subscriber_id), - publisher_id_(""), - worker_id_(worker_id), - max_processed_sequence_id_(0), - closed_(false) { - channel_ = rpc::GcsRpcClient::CreateGcsChannel(gcs_address, gcs_port); - pubsub_stub_ = rpc::InternalPubSubGcsService::NewStub(channel_); -} + std::string subscriber_id, + std::string worker_id) + : channel_(rpc::GcsRpcClient::CreateGcsChannel(gcs_address, gcs_port)), + pubsub_stub_(rpc::InternalPubSubGcsService::NewStub(channel_)), + channel_type_(channel_type), + subscriber_id_(std::move(subscriber_id)), + worker_id_(std::move(worker_id)) {} Status PythonGcsSubscriber::Subscribe() { absl::MutexLock lock(&mu_); @@ -309,7 +278,7 @@ Status PythonGcsSubscriber::DoPoll(int64_t timeout_ms, rpc::PubMessage *message) max_processed_sequence_id_ = 0; } last_batch_size_ = reply.pub_messages().size(); - for (auto &cur_pub_msg : reply.pub_messages()) { + for (auto &cur_pub_msg : *reply.mutable_pub_messages()) { if (cur_pub_msg.sequence_id() <= max_processed_sequence_id_) { RAY_LOG(WARNING) << "Ignoring out of order message " << cur_pub_msg.sequence_id(); continue; @@ -324,7 +293,7 @@ Status PythonGcsSubscriber::DoPoll(int64_t timeout_ms, rpc::PubMessage *message) } } - *message = queue_.front(); + *message = std::move(queue_.front()); queue_.pop_front(); return Status::OK(); @@ -335,8 +304,8 @@ Status PythonGcsSubscriber::PollError(std::string *key_id, rpc::ErrorTableData *data) { rpc::PubMessage message; RAY_RETURN_NOT_OK(DoPoll(timeout_ms, &message)); - *key_id = message.key_id(); - *data = message.error_info_message(); + *key_id = std::move(*message.mutable_key_id()); + *data = std::move(*message.mutable_error_info_message()); return Status::OK(); } @@ -345,8 +314,8 @@ Status PythonGcsSubscriber::PollLogs(std::string *key_id, rpc::LogBatch *data) { rpc::PubMessage message; RAY_RETURN_NOT_OK(DoPoll(timeout_ms, &message)); - *key_id = message.key_id(); - *data = message.log_batch_message(); + *key_id = std::move(*message.mutable_key_id()); + *data = std::move(*message.mutable_log_batch_message()); return Status::OK(); } @@ -355,8 +324,8 @@ Status PythonGcsSubscriber::PollActor(std::string *key_id, rpc::ActorTableData *data) { rpc::PubMessage message; RAY_RETURN_NOT_OK(DoPoll(timeout_ms, &message)); - *key_id = message.key_id(); - *data = message.actor_message(); + *key_id = std::move(*message.mutable_key_id()); + *data = std::move(*message.mutable_actor_message()); return Status::OK(); } diff --git a/src/ray/gcs/pubsub/gcs_pub_sub.h b/src/ray/gcs/pubsub/gcs_pub_sub.h index c9abb5112aa3..89abb70397f9 100644 --- a/src/ray/gcs/pubsub/gcs_pub_sub.h +++ b/src/ray/gcs/pubsub/gcs_pub_sub.h @@ -16,21 +16,16 @@ #include #include -#include #include -#include #include #include -#include "absl/container/flat_hash_map.h" #include "absl/synchronization/mutex.h" -#include "ray/common/ray_config.h" #include "ray/gcs/callback.h" #include "ray/pubsub/publisher.h" #include "ray/pubsub/subscriber.h" #include "src/ray/protobuf/gcs.pb.h" #include "src/ray/protobuf/gcs_service.grpc.pb.h" -#include "src/ray/protobuf/gcs_service.pb.h" namespace ray { namespace gcs { @@ -48,8 +43,6 @@ class GcsPublisher { RAY_CHECK(publisher_); } - virtual ~GcsPublisher() = default; - /// Returns the underlying pubsub::Publisher. Caller does not take ownership. pubsub::Publisher &GetPublisher() const { return *publisher_; } @@ -63,32 +56,16 @@ class GcsPublisher { /// TODO: Verify GCS pubsub satisfies the streaming semantics. /// TODO: Implement optimization for channels where only latest data per ID is useful. - Status PublishActor(const ActorID &id, - rpc::ActorTableData message, - const StatusCallback &done); - - // TODO(dayshah): Look at possibility of moving all of these rpc messages + void PublishActor(const ActorID &id, rpc::ActorTableData message); - Status PublishJob(const JobID &id, - const rpc::JobTableData &message, - const StatusCallback &done); + void PublishJob(const JobID &id, rpc::JobTableData message); - virtual Status PublishNodeInfo(const NodeID &id, - const rpc::GcsNodeInfo &message, - const StatusCallback &done); + void PublishNodeInfo(const NodeID &id, rpc::GcsNodeInfo message); /// Actually rpc::WorkerDeltaData is not a delta message. - Status PublishWorkerFailure(const WorkerID &id, - const rpc::WorkerDeltaData &message, - const StatusCallback &done); - - virtual Status PublishError(const std::string &id, - const rpc::ErrorTableData &message, - const StatusCallback &done); + void PublishWorkerFailure(const WorkerID &id, rpc::WorkerDeltaData message); - /// TODO: remove once it is converted to GRPC-based push broadcasting. - Status PublishResourceBatch(const rpc::ResourceUsageBatchData &message, - const StatusCallback &done); + void PublishError(std::string id, rpc::ErrorTableData message); /// Prints debugging info for the publisher. std::string DebugString() const; @@ -104,9 +81,8 @@ class GcsSubscriber { public: /// Initializes GcsSubscriber with GCS based GcsSubscribers. // TODO(mwtian): Support restarted GCS publisher, at the same or a different address. - GcsSubscriber(const rpc::Address &gcs_address, - std::unique_ptr subscriber) - : gcs_address_(gcs_address), subscriber_(std::move(subscriber)) {} + GcsSubscriber(rpc::Address gcs_address, std::unique_ptr subscriber) + : gcs_address_(std::move(gcs_address)), subscriber_(std::move(subscriber)) {} /// Subscribe*() member functions below would be incrementally converted to use the GCS /// based subscriber, if available. @@ -141,11 +117,11 @@ class GcsSubscriber { // This client is only supposed to be used from Cython / Python class RAY_EXPORT PythonGcsSubscriber { public: - explicit PythonGcsSubscriber(const std::string &gcs_address, - int gcs_port, - rpc::ChannelType channel_type, - const std::string &subscriber_id, - const std::string &worker_id); + PythonGcsSubscriber(const std::string &gcs_address, + int gcs_port, + rpc::ChannelType channel_type, + std::string subscriber_id, + std::string worker_id); /// Register a subscription for the subscriber's channel type. /// @@ -173,22 +149,23 @@ class RAY_EXPORT PythonGcsSubscriber { mutable absl::Mutex mu_; - std::unique_ptr pubsub_stub_; std::shared_ptr channel_; + std::unique_ptr pubsub_stub_; + const rpc::ChannelType channel_type_; const std::string subscriber_id_; std::string publisher_id_; const std::string worker_id_; - int64_t max_processed_sequence_id_ ABSL_GUARDED_BY(mu_); - int64_t last_batch_size_ ABSL_GUARDED_BY(mu_); + int64_t max_processed_sequence_id_ ABSL_GUARDED_BY(mu_) = 0; + int64_t last_batch_size_ ABSL_GUARDED_BY(mu_) = 0; std::deque queue_ ABSL_GUARDED_BY(mu_); - bool closed_ ABSL_GUARDED_BY(mu_); + bool closed_ ABSL_GUARDED_BY(mu_) = false; std::shared_ptr current_polling_context_ ABSL_GUARDED_BY(mu_); }; /// Get the .lines() attribute of a LogBatch as a std::vector /// (this is needed so it can be wrapped in Cython) -std::vector PythonGetLogBatchLines(const rpc::LogBatch &log_batch); +std::vector PythonGetLogBatchLines(rpc::LogBatch log_batch); } // namespace gcs } // namespace ray diff --git a/src/ray/pubsub/README.md b/src/ray/pubsub/README.md index 4e8c6b477287..d57793b7e901 100644 --- a/src/ray/pubsub/README.md +++ b/src/ray/pubsub/README.md @@ -83,7 +83,7 @@ it doesn't use Ray's GCS for that. Subscriber detects the publisher failures from the long polling request. A single long polling request is initiated from the subscriber, and it sends them -again and again whenever replied as long as there are subscribing entreis. If +again and again whenever replied as long as there are subscribing entries. If the publisher fails, the long polling request is also failed, so that the subscriber can detect the failures of publishers. All metadata is cleaned up in this case. diff --git a/src/ray/pubsub/publisher.cc b/src/ray/pubsub/publisher.cc index ec9de99534a8..059466242f7e 100644 --- a/src/ray/pubsub/publisher.cc +++ b/src/ray/pubsub/publisher.cc @@ -27,7 +27,7 @@ namespace pubsub { namespace pub_internal { -bool EntityState::Publish(std::shared_ptr msg, size_t msg_size) { +bool EntityState::Publish(const std::shared_ptr &msg, size_t msg_size) { if (subscribers_.empty()) { return false; } @@ -48,7 +48,8 @@ bool EntityState::Publish(std::shared_ptr msg, size_t msg_size) // to implement inflight message tracking across subscribers with non-atomic // ref-counting or with a LRU-like data structure tracking the range of buffered // messages for each subscriber. - auto front_msg = pending_messages_.front().lock(); + auto &[front_msg_weak, front_msg_size] = pending_messages_.front(); + auto front_msg = front_msg_weak.lock(); if (front_msg == nullptr) { // The message has no other reference. // This means that it has been published to all subscribers. @@ -78,14 +79,12 @@ bool EntityState::Publish(std::shared_ptr msg, size_t msg_size) // The first message in the queue has been published to all subscribers, or // it has been dropped due to memory cap. Subtract it from memory // accounting. + total_size_ -= front_msg_size; pending_messages_.pop(); - total_size_ -= message_sizes_.front(); - message_sizes_.pop(); } - pending_messages_.push(msg); + pending_messages_.emplace(msg, msg_size); total_size_ += msg_size; - message_sizes_.push(msg_size); for (auto &[id, subscriber] : subscribers_) { subscriber->QueueMessage(msg); @@ -122,7 +121,7 @@ int64_t SubscriptionIndex::GetNumBufferedBytes() const { return num_bytes_buffered; } -bool SubscriptionIndex::Publish(std::shared_ptr pub_message, +bool SubscriptionIndex::Publish(const std::shared_ptr &pub_message, size_t msg_size) { const bool publish_to_all = subscribers_to_all_->Publish(pub_message, msg_size); bool publish_to_entity = false; @@ -257,6 +256,7 @@ std::unique_ptr SubscriptionIndex::CreateEntityState( case rpc::ChannelType::RAY_ERROR_INFO_CHANNEL: case rpc::ChannelType::RAY_LOG_CHANNEL: case rpc::ChannelType::RAY_NODE_RESOURCE_USAGE_CHANNEL: + // Not critical if some messages are dropped. return std::make_unique( RayConfig::instance().max_grpc_message_size(), RayConfig::instance().publisher_entity_buffer_max_bytes()); @@ -268,6 +268,7 @@ std::unique_ptr SubscriptionIndex::CreateEntityState( case rpc::ChannelType::GCS_JOB_CHANNEL: case rpc::ChannelType::GCS_NODE_INFO_CHANNEL: case rpc::ChannelType::GCS_WORKER_DELTA_CHANNEL: + // Critical if messages are dropped. return std::make_unique(RayConfig::instance().max_grpc_message_size(), /*max_buffered_bytes=*/-1); @@ -277,12 +278,13 @@ std::unique_ptr SubscriptionIndex::CreateEntityState( } } -void SubscriberState::ConnectToSubscriber(const rpc::PubsubLongPollingRequest &request, - rpc::PubsubLongPollingReply *reply, - rpc::SendReplyCallback send_reply_callback) { +void SubscriberState::ConnectToSubscriber( + const rpc::PubsubLongPollingRequest &request, + std::string *publisher_id, + google::protobuf::RepeatedPtrField *pub_messages, + rpc::SendReplyCallback send_reply_callback) { int64_t max_processed_sequence_id = request.max_processed_sequence_id(); - if (request.publisher_id().empty() || - publisher_id_ != PublisherID::FromBinary(request.publisher_id())) { + if (request.publisher_id().empty() || publisher_id_binary_ != request.publisher_id()) { // in case the publisher_id mismatches, we should ignore the // max_processed_sequence_id. max_processed_sequence_id = 0; @@ -300,38 +302,33 @@ void SubscriberState::ConnectToSubscriber(const rpc::PubsubLongPollingRequest &r PublishIfPossible(/*force_noop=*/true); } RAY_CHECK(!long_polling_connection_); - RAY_CHECK(reply != nullptr); - RAY_CHECK(send_reply_callback != nullptr); - long_polling_connection_ = - std::make_unique(reply, std::move(send_reply_callback)); + long_polling_connection_ = std::make_unique( + publisher_id, pub_messages, std::move(send_reply_callback)); last_connection_update_time_ms_ = get_time_ms_(); - PublishIfPossible(); + PublishIfPossible(/*force_noop=*/false); } -void SubscriberState::QueueMessage(const std::shared_ptr &pub_message, - bool try_publish) { +void SubscriberState::QueueMessage(const std::shared_ptr &pub_message) { RAY_LOG(DEBUG) << "enqueue: " << pub_message->sequence_id(); mailbox_.push_back(pub_message); - if (try_publish) { - PublishIfPossible(); - } + PublishIfPossible(/*force_noop=*/false); } -bool SubscriberState::PublishIfPossible(bool force_noop) { +void SubscriberState::PublishIfPossible(bool force_noop) { if (!long_polling_connection_) { - return false; + return; } if (!force_noop && mailbox_.empty()) { - return false; + return; } // No message should have been added to the reply. - RAY_CHECK(long_polling_connection_->reply->pub_messages().empty()); - *long_polling_connection_->reply->mutable_publisher_id() = publisher_id_.Binary(); + RAY_CHECK(long_polling_connection_->pub_messages->empty()); + *long_polling_connection_->publisher_id = publisher_id_binary_; int64_t num_total_bytes = 0; if (!force_noop) { for (auto it = mailbox_.begin(); it != mailbox_.end(); it++) { - if (long_polling_connection_->reply->pub_messages().size() >= publish_batch_size_) { + if (long_polling_connection_->pub_messages->size() >= publish_batch_size_) { break; } @@ -350,20 +347,16 @@ bool SubscriberState::PublishIfPossible(bool force_noop) { // Avoid sending empty message to the subscriber. The message might have been // cleared because the subscribed entity's buffer was full. if (msg.inner_message_case() != rpc::PubMessage::INNER_MESSAGE_NOT_SET) { - *long_polling_connection_->reply->add_pub_messages() = msg; + *long_polling_connection_->pub_messages->Add() = msg; } } } - - RAY_LOG(DEBUG) << "sending reply back" - << long_polling_connection_->reply->DebugString(); long_polling_connection_->send_reply_callback(Status::OK(), nullptr, nullptr); // Clean up & update metadata. long_polling_connection_.reset(); // Clean up & update metadata. last_connection_update_time_ms_ = get_time_ms_(); - return true; } bool SubscriberState::CheckNoLeaks() const { @@ -381,10 +374,11 @@ bool SubscriberState::IsActive() const { } // namespace pub_internal -void Publisher::ConnectToSubscriber(const rpc::PubsubLongPollingRequest &request, - rpc::PubsubLongPollingReply *reply, - rpc::SendReplyCallback send_reply_callback) { - RAY_CHECK(reply != nullptr); +void Publisher::ConnectToSubscriber( + const rpc::PubsubLongPollingRequest &request, + std::string *publisher_id, + google::protobuf::RepeatedPtrField *pub_messages, + rpc::SendReplyCallback send_reply_callback) { RAY_CHECK(send_reply_callback != nullptr); const auto subscriber_id = SubscriberID::FromBinary(request.subscriber_id()); @@ -406,7 +400,8 @@ void Publisher::ConnectToSubscriber(const rpc::PubsubLongPollingRequest &request auto &subscriber = it->second; // May flush the current long poll with an empty message, if a poll request exists. - subscriber->ConnectToSubscriber(request, reply, std::move(send_reply_callback)); + subscriber->ConnectToSubscriber( + request, publisher_id, pub_messages, std::move(send_reply_callback)); } bool Publisher::RegisterSubscription(const rpc::ChannelType channel_type, @@ -466,9 +461,9 @@ bool Publisher::UnregisterSubscription(const rpc::ChannelType channel_type, return subscription_index_it->second.EraseEntry(key_id.value_or(""), subscriber_id); } -bool Publisher::UnregisterSubscriber(const SubscriberID &subscriber_id) { +void Publisher::UnregisterSubscriber(const SubscriberID &subscriber_id) { absl::MutexLock lock(&mutex_); - return UnregisterSubscriberInternal(subscriber_id); + UnregisterSubscriberInternal(subscriber_id); } void Publisher::UnregisterAll() { @@ -484,24 +479,20 @@ void Publisher::UnregisterAll() { } } -int Publisher::UnregisterSubscriberInternal(const SubscriberID &subscriber_id) { +void Publisher::UnregisterSubscriberInternal(const SubscriberID &subscriber_id) { RAY_LOG(DEBUG) << "Unregistering subscriber " << subscriber_id.Hex(); - int erased = 0; for (auto &index : subscription_index_map_) { - if (index.second.EraseSubscriber(subscriber_id)) { - erased += 1; - } + index.second.EraseSubscriber(subscriber_id); } auto it = subscribers_.find(subscriber_id); if (it == subscribers_.end()) { - return erased; + return; } auto &subscriber = it->second; // Flush the long polling connection because otherwise the reply could be leaked. subscriber->PublishIfPossible(/*force_noop=*/true); subscribers_.erase(it); - return erased; } void Publisher::CheckDeadSubscribers() { diff --git a/src/ray/pubsub/publisher.h b/src/ray/pubsub/publisher.h index b61027e11928..e3a2e92e9e59 100644 --- a/src/ray/pubsub/publisher.h +++ b/src/ray/pubsub/publisher.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include @@ -31,7 +30,6 @@ #include "ray/common/asio/periodical_runner.h" #include "ray/common/id.h" #include "ray/rpc/server_call.h" -#include "src/ray/protobuf/common.pb.h" #include "src/ray/protobuf/pubsub.pb.h" namespace ray { @@ -55,7 +53,7 @@ class EntityState { /// Publishes the message to subscribers of the entity. /// Returns true if there are subscribers, returns false otherwise. - bool Publish(std::shared_ptr pub_message, size_t msg_size); + bool Publish(const std::shared_ptr &pub_message, size_t msg_size); /// Manages the set of subscribers of this entity. bool AddSubscriber(SubscriberState *subscriber); @@ -74,18 +72,20 @@ class EntityState { private: // Tracks inflight messages. The messages have shared ownership by // individual subscribers, and get deleted after no subscriber has - // the message in buffer. - std::queue> pending_messages_; - // Size of each inflight message. - std::queue message_sizes_; + // the message in buffer. Also stores the size of the message so that we can keep track + // of total_size_. + std::queue, size_t>> pending_messages_; + // Protobuf messages fail to serialize if 2GB or larger. Cap published // message batches to this size to ensure that we can publish each message // batch. Individual messages larger than this limit will also be dropped. // TODO(swang): Pubsub clients should also ensure that they don't try to // publish messages larger than this. const size_t max_message_size_bytes_; + // Set to -1 to disable buffering. const int64_t max_buffered_bytes_; + // Total size of inflight messages. size_t total_size_ = 0; }; @@ -95,15 +95,11 @@ class EntityState { class SubscriptionIndex { public: explicit SubscriptionIndex(rpc::ChannelType channel_type); - ~SubscriptionIndex() = default; - - SubscriptionIndex(SubscriptionIndex &&) noexcept = default; - SubscriptionIndex &operator=(SubscriptionIndex &&) noexcept = default; /// Publishes the message to relevant subscribers. /// Returns true if there are subscribers listening on the entity key of the message, /// returns false otherwise. - bool Publish(std::shared_ptr pub_message, size_t msg_size); + bool Publish(const std::shared_ptr &pub_message, size_t msg_size); /// Adds a new subscriber and the key it subscribes to. /// When `key_id` is empty, the subscriber subscribes to all keys. @@ -153,11 +149,15 @@ class SubscriptionIndex { }; struct LongPollConnection { - LongPollConnection(rpc::PubsubLongPollingReply *reply, + LongPollConnection(std::string *publisher_id, + google::protobuf::RepeatedPtrField *pub_messages, rpc::SendReplyCallback send_reply_callback) - : reply(reply), send_reply_callback(send_reply_callback) {} + : publisher_id(publisher_id), + pub_messages(pub_messages), + send_reply_callback(std::move(send_reply_callback)) {} - rpc::PubsubLongPollingReply *reply; + std::string *publisher_id; + google::protobuf::RepeatedPtrField *pub_messages; rpc::SendReplyCallback send_reply_callback; }; @@ -174,38 +174,34 @@ class SubscriberState { connection_timeout_ms_(connection_timeout_ms), publish_batch_size_(publish_batch_size), last_connection_update_time_ms_(get_time_ms_()), - publisher_id_(publisher_id) {} + publisher_id_binary_(publisher_id.Binary()) {} ~SubscriberState() { // Force a push to close the long-polling. // Otherwise, there will be a connection leak. - PublishIfPossible(true); + PublishIfPossible(/*force_noop=*/true); } + SubscriberState(const SubscriberState &) = delete; + SubscriberState &operator=(const SubscriberState &) = delete; + /// Connect to the subscriber. Currently, it means we cache the long polling request to - /// memory. Once the bidirectional gRPC streaming is enabled, we should replace it. - /// - /// \param reply pubsub long polling reply. - /// \param send_reply_callback A callback to reply to the long polling subscriber. - void ConnectToSubscriber(const rpc::PubsubLongPollingRequest &request, - rpc::PubsubLongPollingReply *reply, - rpc::SendReplyCallback send_reply_callback); + /// memory. + void ConnectToSubscriber( + const rpc::PubsubLongPollingRequest &request, + std::string *publisher_id, + google::protobuf::RepeatedPtrField *pub_messages, + rpc::SendReplyCallback send_reply_callback); /// Queue the pubsub message to publish to the subscriber. - /// - /// \param pub_message A message to publish. - /// \param try_publish If true, try publishing the object id if there is a connection. - /// Currently only set to false in tests. - void QueueMessage(const std::shared_ptr &pub_message, - bool try_publish = true); + void QueueMessage(const std::shared_ptr &pub_message); /// Publish all queued messages if possible. /// /// \param force_noop If true, reply to the subscriber with an empty message, regardless /// of whethere there is any queued message. This is for cases where the current poll /// might have been cancelled, or the subscriber might be dead. - /// \return True if it publishes. False otherwise. - bool PublishIfPossible(bool force_noop = false); + void PublishIfPossible(bool force_noop); /// Testing only. Return true if there's no metadata remained in the private attribute. bool CheckNoLeaks() const; @@ -235,7 +231,7 @@ class SubscriberState { const int64_t publish_batch_size_; /// The last time long polling was connected in milliseconds. double last_connection_update_time_ms_; - PublisherID publisher_id_; + std::string publisher_id_binary_; }; } // namespace pub_internal @@ -251,9 +247,11 @@ class PublisherInterface { /// TODO(sang): Currently, we need to pass the callback for connection because we are /// using long polling internally. This should be changed once the bidirectional grpc /// streaming is supported. - virtual void ConnectToSubscriber(const rpc::PubsubLongPollingRequest &request, - rpc::PubsubLongPollingReply *reply, - rpc::SendReplyCallback send_reply_callback) = 0; + virtual void ConnectToSubscriber( + const rpc::PubsubLongPollingRequest &request, + std::string *publisher_id, + google::protobuf::RepeatedPtrField *pub_messages, + rpc::SendReplyCallback send_reply_callback) = 0; /// Register the subscription. /// @@ -339,11 +337,11 @@ class Publisher : public PublisherInterface { "Publisher.CheckDeadSubscribers"); } - ~Publisher() override = default; - - void ConnectToSubscriber(const rpc::PubsubLongPollingRequest &request, - rpc::PubsubLongPollingReply *reply, - rpc::SendReplyCallback send_reply_callback) override; + void ConnectToSubscriber( + const rpc::PubsubLongPollingRequest &request, + std::string *publisher_id, + google::protobuf::RepeatedPtrField *pub_messages, + rpc::SendReplyCallback send_reply_callback) override; bool RegisterSubscription(const rpc::ChannelType channel_type, const SubscriberID &subscriber_id, @@ -362,8 +360,7 @@ class Publisher : public PublisherInterface { /// to it anymore. /// /// \param subscriber_id The node id of the subscriber to unsubscribe. - /// \return True if erased. False otherwise. - bool UnregisterSubscriber(const SubscriberID &subscriber_id); + void UnregisterSubscriber(const SubscriberID &subscriber_id); /// Flushes all inflight pollings and unregisters all subscribers. void UnregisterAll(); @@ -418,7 +415,7 @@ class Publisher : public PublisherInterface { /// Private fields /// - int UnregisterSubscriberInternal(const SubscriberID &subscriber_id) + void UnregisterSubscriberInternal(const SubscriberID &subscriber_id) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Periodic runner to invoke CheckDeadSubscribers. @@ -467,7 +464,6 @@ class Publisher : public PublisherInterface { int64_t next_sequence_id_ ABSL_GUARDED_BY(mutex_) = 0; /// A unique identifier identifies the publisher_id. - /// TODO(scv119) add docs about the semantics. const PublisherID publisher_id_; }; diff --git a/src/ray/pubsub/test/integration_test.cc b/src/ray/pubsub/test/integration_test.cc index 6884b46b64a2..35a7f338b0b5 100644 --- a/src/ray/pubsub/test/integration_test.cc +++ b/src/ray/pubsub/test/integration_test.cc @@ -47,7 +47,8 @@ class SubscriberServiceImpl final : public rpc::SubscriberService::CallbackServi rpc::PubsubLongPollingReply *reply) override { auto *reactor = context->DefaultReactor(); publisher_->ConnectToSubscriber(*request, - reply, + reply->mutable_publisher_id(), + reply->mutable_pub_messages(), [reactor](ray::Status status, std::function success_cb, std::function failure_cb) { diff --git a/src/ray/pubsub/test/publisher_test.cc b/src/ray/pubsub/test/publisher_test.cc index 7c09e0598012..9165bd52d7fa 100644 --- a/src/ray/pubsub/test/publisher_test.cc +++ b/src/ray/pubsub/test/publisher_test.cc @@ -115,8 +115,11 @@ class PublisherTest : public ::testing::Test { rpc::SendReplyCallback callback = [pubsub_reply](Status status, std::function success, std::function failure) {}; - subscriber->ConnectToSubscriber(request, pubsub_reply.get(), callback); - subscriber->PublishIfPossible(); + subscriber->ConnectToSubscriber(request, + pubsub_reply->mutable_publisher_id(), + pubsub_reply->mutable_pub_messages(), + callback); + subscriber->PublishIfPossible(/*force_noop=*/false); return pubsub_reply; } @@ -358,79 +361,93 @@ TEST_F(PublisherTest, TestSubscriber) { 10, kDefaultPublisherId); // If there's no connection, it will return false. - ASSERT_FALSE(subscriber->PublishIfPossible()); + subscriber->PublishIfPossible(/*force_noop=*/false); // Try connecting. - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); // Reconnection should still succeed. - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); // No result should have been returned. ASSERT_TRUE(object_ids_published.empty()); - // Since there's no objects pending to be published, it should return false. - ASSERT_FALSE(subscriber->PublishIfPossible()); + subscriber->PublishIfPossible(/*force_noop=*/false); + ASSERT_TRUE(object_ids_published.empty()); - absl::flat_hash_set published_objects; + absl::flat_hash_set expected_published_objects; // Make sure publishing one object works as expected. auto oid = ObjectID::FromRandom(); subscriber->QueueMessage( - std::make_shared(GeneratePubMessage(oid, GetNextSequenceId())), - /*try_publish=*/false); - published_objects.emplace(oid); - ASSERT_TRUE(subscriber->PublishIfPossible()); + std::make_shared(GeneratePubMessage(oid, GetNextSequenceId()))); + expected_published_objects.emplace(oid); + subscriber->PublishIfPossible(/*force_noop=*/false); ASSERT_TRUE(object_ids_published.contains(oid)); // No object is pending to be published, and there's no connection. - ASSERT_FALSE(subscriber->PublishIfPossible()); + subscriber->PublishIfPossible(/*force_noop=*/false); // Add 3 oids and see if it works properly. for (int i = 0; i < 3; i++) { oid = ObjectID::FromRandom(); subscriber->QueueMessage( - std::make_shared(GeneratePubMessage(oid, GetNextSequenceId())), - /*try_publish=*/false); - published_objects.emplace(oid); + std::make_shared(GeneratePubMessage(oid, GetNextSequenceId()))); + expected_published_objects.emplace(oid); } // Since there's no connection, objects won't be published. - ASSERT_FALSE(subscriber->PublishIfPossible()); - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); - for (auto cur_oid : published_objects) { - ASSERT_TRUE(object_ids_published.contains(cur_oid)); - } + subscriber->PublishIfPossible(/*force_noop=*/false); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); + ASSERT_EQ(expected_published_objects, object_ids_published); // Queue is not cleaned up if max_processed_sequence_id hasn't // been set properly. request_.set_max_processed_sequence_id(1); - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); ASSERT_FALSE(subscriber->CheckNoLeaks()); // If we set wrong publisher_id, the queue won't be cleaned up. request_.set_publisher_id(NodeID::FromRandom().Binary()); request_.set_max_processed_sequence_id(sequence_id_); - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); ASSERT_FALSE(subscriber->CheckNoLeaks()); // By sending back max_processed_sequence_id, the subscriber's sending queue // is cleaned up. request_.set_max_processed_sequence_id(sequence_id_); request_.set_publisher_id(kDefaultPublisherId.Binary()); - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); ASSERT_TRUE(subscriber->CheckNoLeaks()); } TEST_F(PublisherTest, TestSubscriberBatchSize) { absl::flat_hash_set object_ids_published; - int64_t max_processed_seuquence_id = 0; - send_reply_callback = - [this, &object_ids_published, &max_processed_seuquence_id]( - Status status, std::function success, std::function failure) { - for (int i = 0; i < reply.pub_messages_size(); i++) { - const auto &msg = reply.pub_messages(i); - const auto oid = - ObjectID::FromBinary(msg.worker_object_eviction_message().object_id()); - object_ids_published.emplace(oid); - max_processed_seuquence_id = - std::max(msg.sequence_id(), max_processed_seuquence_id); - } - reply = rpc::PubsubLongPollingReply(); - }; + int64_t max_processed_sequence_id = 0; + send_reply_callback = [this, &object_ids_published, &max_processed_sequence_id]( + Status status, + std::function success, + std::function failure) { + for (int i = 0; i < reply.pub_messages_size(); i++) { + const auto &msg = reply.pub_messages(i); + const auto oid = + ObjectID::FromBinary(msg.worker_object_eviction_message().object_id()); + object_ids_published.emplace(oid); + max_processed_sequence_id = std::max(msg.sequence_id(), max_processed_sequence_id); + } + reply = rpc::PubsubLongPollingReply(); + }; auto max_publish_size = 5; auto subscriber = std::make_shared( @@ -439,21 +456,19 @@ TEST_F(PublisherTest, TestSubscriberBatchSize) { subscriber_timeout_ms_, max_publish_size, kDefaultPublisherId); - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); - absl::flat_hash_set published_objects; std::vector oids; for (int i = 0; i < 10; i++) { auto oid = ObjectID::FromRandom(); oids.push_back(oid); subscriber->QueueMessage( - std::make_shared(GeneratePubMessage(oid, GetNextSequenceId())), - /*try_publish=*/false); - published_objects.emplace(oid); + std::make_shared(GeneratePubMessage(oid, GetNextSequenceId()))); } - // Make sure only up to batch size is published. - ASSERT_TRUE(subscriber->PublishIfPossible()); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); for (int i = 0; i < max_publish_size; i++) { ASSERT_TRUE(object_ids_published.contains(oids[i])); @@ -463,9 +478,12 @@ TEST_F(PublisherTest, TestSubscriberBatchSize) { } // Remaining messages are published upon polling. - ASSERT_EQ(max_processed_seuquence_id, max_publish_size); - request_.set_max_processed_sequence_id(max_processed_seuquence_id); - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); + ASSERT_EQ(max_processed_sequence_id, max_publish_size); + request_.set_max_processed_sequence_id(max_processed_sequence_id); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); for (int i = 0; i < 10; i++) { ASSERT_TRUE(object_ids_published.contains(oids[i])); } @@ -488,7 +506,10 @@ TEST_F(PublisherTest, TestSubscriberActiveTimeout) { 10, kDefaultPublisherId); - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); // Connection is not timed out yet. ASSERT_TRUE(subscriber->IsActive()); @@ -510,7 +531,10 @@ TEST_F(PublisherTest, TestSubscriberActiveTimeout) { // New connection is established. reply = rpc::PubsubLongPollingReply(); - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); ASSERT_TRUE(subscriber->IsActive()); ASSERT_TRUE(subscriber->ConnectionExists()); @@ -539,7 +563,10 @@ TEST_F(PublisherTest, TestSubscriberActiveTimeout) { // Notify that message 1 is safe to be GCed. request_.set_max_processed_sequence_id(1); reply = rpc::PubsubLongPollingReply(); - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); ASSERT_TRUE(subscriber->CheckNoLeaks()); } @@ -561,8 +588,11 @@ TEST_F(PublisherTest, TestSubscriberDisconnected) { kDefaultPublisherId); // Suppose the new connection is removed. - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); - subscriber->PublishIfPossible(/*force*/ true); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); + subscriber->PublishIfPossible(/*force_noop=*/true); ASSERT_EQ(reply_cnt, 1); ASSERT_TRUE(subscriber->IsActive()); ASSERT_FALSE(subscriber->ConnectionExists()); @@ -579,8 +609,11 @@ TEST_F(PublisherTest, TestSubscriberDisconnected) { ASSERT_FALSE(subscriber->ConnectionExists()); // New connection is coming in. - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); - subscriber->PublishIfPossible(/*force*/ true); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); + subscriber->PublishIfPossible(/*force_noop=*/true); ASSERT_EQ(reply_cnt, 2); // Some time has passed, but it is not timed out yet. @@ -590,8 +623,11 @@ TEST_F(PublisherTest, TestSubscriberDisconnected) { // Another connection is made, so it shouldn't timeout until the next timeout is // reached. - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); - subscriber->PublishIfPossible(/*force*/ true); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); + subscriber->PublishIfPossible(/*force_noop=*/true); ASSERT_EQ(reply_cnt, 3); current_time_ += subscriber_timeout_ms_ / 2; ASSERT_TRUE(subscriber->IsActive()); @@ -623,15 +659,21 @@ TEST_F(PublisherTest, TestSubscriberTimeoutComplicated) { kDefaultPublisherId); // Suppose the new connection is removed. - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); - subscriber->PublishIfPossible(/*force*/ true); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); + subscriber->PublishIfPossible(/*force_noop=*/true); ASSERT_EQ(reply_cnt, 1); ASSERT_TRUE(subscriber->IsActive()); ASSERT_FALSE(subscriber->ConnectionExists()); // Some time has passed, and the connection is removed. current_time_ += subscriber_timeout_ms_ - 1; - subscriber->ConnectToSubscriber(request_, &reply, send_reply_callback); + subscriber->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); current_time_ += 2; // Timeout shouldn't happen because the connection has been refreshed. ASSERT_TRUE(subscriber->IsActive()); @@ -640,7 +682,7 @@ TEST_F(PublisherTest, TestSubscriberTimeoutComplicated) { // Right before the timeout, connection is removed. In this case, timeout shouldn't also // happen. current_time_ += subscriber_timeout_ms_ - 1; - subscriber->PublishIfPossible(/*force*/ true); + subscriber->PublishIfPossible(/*force_noop=*/true); current_time_ += 2; ASSERT_TRUE(subscriber->IsActive()); ASSERT_FALSE(subscriber->ConnectionExists()); @@ -670,7 +712,10 @@ TEST_F(PublisherTest, TestBasicSingleSubscriber) { const auto oid = ObjectID::FromRandom(); - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); publisher_->RegisterSubscription( rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); publisher_->Publish(GeneratePubMessage(oid, 0)); @@ -698,7 +743,10 @@ TEST_F(PublisherTest, TestNoConnectionWhenRegistered) { publisher_->Publish(GeneratePubMessage(oid)); // Nothing has been published because there's no connection. ASSERT_EQ(batched_ids.size(), 0); - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); // When the connection is coming, it should be published. ASSERT_EQ(batched_ids[0], oid); } @@ -729,7 +777,10 @@ TEST_F(PublisherTest, TestMultiObjectsFromSingleNode) { ASSERT_EQ(batched_ids.size(), 0); // Now connection is initiated, and all oids are published. - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); for (int i = 0; i < num_oids; i++) { const auto oid_test = oids[i]; const auto published_oid = batched_ids[i]; @@ -770,7 +821,10 @@ TEST_F(PublisherTest, TestMultiObjectsFromMultiNodes) { // Check all of nodes are publishing objects properly. for (int i = 0; i < num_nodes; i++) { - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); const auto oid_test = oids[i]; const auto published_oid = batched_ids[i]; ASSERT_EQ(oid_test, published_oid); @@ -780,6 +834,7 @@ TEST_F(PublisherTest, TestMultiObjectsFromMultiNodes) { TEST_F(PublisherTest, TestMultiSubscribers) { absl::flat_hash_set batched_ids; int reply_invoked = 0; + reply = rpc::PubsubLongPollingReply(); send_reply_callback = [this, &batched_ids, &reply_invoked]( Status status, std::function success, std::function failure) { @@ -789,7 +844,7 @@ TEST_F(PublisherTest, TestMultiSubscribers) { ObjectID::FromBinary(msg.worker_object_eviction_message().object_id()); batched_ids.emplace(oid); } - reply = rpc::PubsubLongPollingReply(); + reply.Clear(); reply_invoked += 1; }; @@ -809,7 +864,10 @@ TEST_F(PublisherTest, TestMultiSubscribers) { // Check all of nodes are publishing objects properly. for (int i = 0; i < num_nodes; i++) { - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); } publisher_->Publish(GeneratePubMessage(oid)); ASSERT_EQ(batched_ids.size(), 1); @@ -847,7 +905,10 @@ TEST_F(PublisherTest, TestBatch) { // Now connection is initiated, and all oids are published. request_.set_max_processed_sequence_id(max_processed_sequence_id); - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); for (int i = 0; i < num_oids; i++) { const auto oid_test = oids[i]; const auto published_oid = batched_ids[i]; @@ -865,7 +926,10 @@ TEST_F(PublisherTest, TestBatch) { publisher_->Publish(GeneratePubMessage(oid)); } request_.set_max_processed_sequence_id(max_processed_sequence_id); - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); ASSERT_EQ(num_oids, oids.size()); ASSERT_EQ(num_oids, batched_ids.size()); for (int i = 0; i < num_oids; i++) { @@ -884,7 +948,10 @@ TEST_F(PublisherTest, TestNodeFailureWhenConnectionExisted) { }; const auto oid = ObjectID::FromRandom(); - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); // This information should be cleaned up as the subscriber is dead. publisher_->RegisterSubscription( rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); @@ -900,8 +967,7 @@ TEST_F(PublisherTest, TestNodeFailureWhenConnectionExisted) { publisher_->CheckDeadSubscribers(); // Connection should be replied (removed) when the subscriber is unregistered. - int erased = publisher_->UnregisterSubscriber(subscriber_id_); - ASSERT_EQ(erased, 0); + publisher_->UnregisterSubscriber(subscriber_id_); ASSERT_TRUE(publisher_->CheckNoLeaks()); // New subscriber is registsered for some reason. Since there's no new long polling @@ -911,8 +977,7 @@ TEST_F(PublisherTest, TestNodeFailureWhenConnectionExisted) { rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); current_time_ += subscriber_timeout_ms_; publisher_->CheckDeadSubscribers(); - erased = publisher_->UnregisterSubscriber(subscriber_id_); - ASSERT_EQ(erased, 0); + publisher_->UnregisterSubscriber(subscriber_id_); ASSERT_TRUE(publisher_->CheckNoLeaks()); } @@ -935,7 +1000,10 @@ TEST_F(PublisherTest, TestNodeFailureWhenConnectionDoesntExist) { ASSERT_EQ(long_polling_connection_replied, false); // Connect should be removed eventually to avoid having a memory leak. - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); ASSERT_EQ(long_polling_connection_replied, true); // Nothing happens at first. publisher_->CheckDeadSubscribers(); @@ -970,7 +1038,10 @@ TEST_F(PublisherTest, TestUnregisterSubscription) { }; const auto oid = ObjectID::FromRandom(); - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); publisher_->RegisterSubscription( rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); ASSERT_EQ(long_polling_connection_replied, false); @@ -1011,28 +1082,31 @@ TEST_F(PublisherTest, TestUnregisterSubscriber) { // Test basic. const auto oid = ObjectID::FromRandom(); - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); publisher_->RegisterSubscription( rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); ASSERT_EQ(long_polling_connection_replied, false); - int erased = publisher_->UnregisterSubscriber(subscriber_id_); - ASSERT_TRUE(erased); + publisher_->UnregisterSubscriber(subscriber_id_); // Make sure the long polling request is replied to avoid memory leak. ASSERT_EQ(long_polling_connection_replied, true); // Test when registration wasn't done. long_polling_connection_replied = false; - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); - erased = publisher_->UnregisterSubscriber(subscriber_id_); - ASSERT_FALSE(erased); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); + publisher_->UnregisterSubscriber(subscriber_id_); ASSERT_EQ(long_polling_connection_replied, true); // Test when connect wasn't done. long_polling_connection_replied = false; publisher_->RegisterSubscription( rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); - erased = publisher_->UnregisterSubscriber(subscriber_id_); - ASSERT_TRUE(erased); + publisher_->UnregisterSubscriber(subscriber_id_); ASSERT_EQ(long_polling_connection_replied, false); ASSERT_TRUE(publisher_->CheckNoLeaks()); } @@ -1082,7 +1156,10 @@ TEST_F(PublisherTest, TestPublishFailure) { const auto oid = ObjectID::FromRandom(); - publisher_->ConnectToSubscriber(request_, &reply, send_reply_callback); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); publisher_->RegisterSubscription( rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); publisher_->PublishFailure(rpc::ChannelType::WORKER_OBJECT_EVICTION, oid.Binary()); diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 7dd636db78e2..d947dd8c845c 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -1397,9 +1397,9 @@ void NodeManager::DisconnectClient(const std::shared_ptr &clie .WithField("node_id", self_node_id_.Hex()) .WithField("job_id", worker->GetAssignedJobId().Hex()) << error_message_str; - auto error_data_ptr = gcs::CreateErrorTableData( + auto error_data = gcs::CreateErrorTableData( type, error_message_str, absl::FromUnixMillis(current_time_ms()), job_id); - gcs_client_.Errors().AsyncReportJobError(error_data_ptr, nullptr); + gcs_client_.Errors().AsyncReportJobError(std::move(error_data)); } } @@ -1593,9 +1593,9 @@ void NodeManager::ProcessPushErrorRequestMessage(const uint8_t *message_data) { // TODO(hjiang): Figure out what's the unit for `PushErrorRequest`. double timestamp = message->timestamp(); JobID job_id = from_flatbuf(*message->job_id()); - auto error_data_ptr = gcs::CreateErrorTableData( + auto error_data = gcs::CreateErrorTableData( type, error_message, absl::FromUnixMillis(timestamp), job_id); - gcs_client_.Errors().AsyncReportJobError(error_data_ptr, nullptr); + gcs_client_.Errors().AsyncReportJobError(std::move(error_data)); } void NodeManager::HandleGetResourceLoad(rpc::GetResourceLoadRequest request, @@ -2027,9 +2027,9 @@ void NodeManager::MarkObjectsAsFailed( << " object may hang forever."; std::string error_message = stream.str(); RAY_LOG(ERROR) << error_message; - auto error_data_ptr = gcs::CreateErrorTableData( + auto error_data = gcs::CreateErrorTableData( "task", error_message, absl::FromUnixMillis(current_time_ms()), job_id); - gcs_client_.Errors().AsyncReportJobError(error_data_ptr, nullptr); + gcs_client_.Errors().AsyncReportJobError(std::move(error_data)); } } } diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index 6162847962f3..0b4f61a18576 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -1692,9 +1692,9 @@ void WorkerPool::WarnAboutSize() { std::string warning_message_str = warning_message.str(); RAY_LOG(WARNING) << warning_message_str; - auto error_data_ptr = gcs::CreateErrorTableData( + auto error_data = gcs::CreateErrorTableData( "worker_pool_large", warning_message_str, get_time_()); - gcs_client_.Errors().AsyncReportJobError(error_data_ptr, nullptr); + gcs_client_.Errors().AsyncReportJobError(std::move(error_data)); } } } From fd681ee6e3a74f08918eec34ea7a5d2f9b502f39 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Wed, 13 Aug 2025 20:36:49 -0700 Subject: [PATCH 095/634] [ci] raydepsets: implementing build arg sets (2/2) (#55423) 1/2 here: https://github.com/ray-project/ray/pull/55408 - implementing get depset by name and optional build arg set - adding unit tests --------- Signed-off-by: elliot-barn Signed-off-by: Elliot Barnwell Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- ci/raydepsets/cli.py | 47 +++++++++++++++++++-------- ci/raydepsets/tests/test_cli.py | 17 ++++++---- ci/raydepsets/tests/test_workspace.py | 21 ++++++++++++ 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index a6e046c2d2a2..c127f1218ae5 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -31,23 +31,43 @@ def cli(): @cli.command() @click.argument("config_path", default="ci/raydepsets/ray.depsets.yaml") -@click.option("--workspace-dir", default=None) -@click.option("--name", default=None) -@click.option("--uv-cache-dir", default=None) +@click.option( + "--workspace-dir", + default=None, + help="The path to the workspace directory. If not specified, $BUILD_WORKSPACE_DIRECTORY will be used.", +) +@click.option( + "--name", + default=None, + help="The name of the dependency set to load. If not specified, all dependency sets will be loaded.", +) +@click.option( + "--build-arg-set", + default=None, + help="The name of the build arg set to use. If not specified, a depset matching the name with no build arg set will be loaded.", +) +@click.option( + "--uv-cache-dir", default=None, help="The directory to cache uv dependencies" +) def load( config_path: str, workspace_dir: Optional[str], name: Optional[str], + build_arg_set: Optional[str], uv_cache_dir: Optional[str], ): - """Load a dependency sets from a config file.""" + """ + Load dependency sets from a config file. + Args: + config_path: The path to the config file. If not specified, ci/raydepsets/ray.depsets.yaml will be used. + """ manager = DependencySetManager( config_path=config_path, workspace_dir=workspace_dir, uv_cache_dir=uv_cache_dir, ) if name: - manager.execute_single(manager.get_depset(name)) + manager.execute_single(_get_depset(manager.config.depsets, name)) else: manager.execute() @@ -91,12 +111,6 @@ def execute(self): depset = self.build_graph.nodes[node]["depset"] self.execute_single(depset) - def get_depset(self, name: str) -> Depset: - for depset in self.config.depsets: - if depset.name == name: - return depset - raise KeyError(f"Dependency set {name} not found") - def exec_uv_cmd(self, cmd: str, args: List[str]) -> str: cmd = [self._uv_binary, "pip", cmd, *args] click.echo(f"Executing command: {cmd}") @@ -173,7 +187,7 @@ def subset( override_flags: Optional[List[str]] = None, ): """Subset a dependency set.""" - source_depset = self.get_depset(source_depset) + source_depset = _get_depset(self.config.depsets, source_depset) self.check_subset_exists(source_depset, requirements) self.compile( constraints=[source_depset.output], @@ -198,7 +212,7 @@ def expand( # handle both depsets and requirements depset_req_list = [] for depset_name in depsets: - depset = self.get_depset(depset_name) + depset = _get_depset(self.config.depsets, depset_name) depset_req_list.extend(depset.requirements) if requirements: depset_req_list.extend(requirements) @@ -222,6 +236,13 @@ def check_subset_exists(self, source_depset: Depset, requirements: List[str]): ) +def _get_depset(depsets: List[Depset], name: str) -> Depset: + for depset in depsets: + if depset.name == name: + return depset + raise KeyError(f"Dependency set {name} not found") + + def _flatten_flags(flags: List[str]) -> List[str]: """ Flatten a list of flags into a list of strings. diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 0fea81effa7e..37083e62f348 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -14,9 +14,9 @@ from ci.raydepsets.cli import ( DEFAULT_UV_FLAGS, DependencySetManager, - Depset, _append_uv_flags, _flatten_flags, + _get_depset, _override_uv_flags, _uv_binary, load, @@ -28,6 +28,9 @@ save_file_as, save_packages_to_file, ) +from ci.raydepsets.workspace import ( + Depset, +) _REPO_NAME = "io_ray" _runfiles = runfiles.Create() @@ -74,12 +77,12 @@ def test_dependency_set_manager_init(self): ] assert manager.config.depsets[0].output == "requirements_compiled.txt" - def test_dependency_set_manager_get_depset(self): + def test_get_depset(self): with tempfile.TemporaryDirectory() as tmpdir: copy_data_to_tmpdir(tmpdir) manager = _create_test_manager(tmpdir) with self.assertRaises(KeyError): - manager.get_depset("fake_depset") + _get_depset(manager.config.depsets, "fake_depset") def test_uv_binary_exists(self): assert _uv_binary() is not None @@ -486,7 +489,9 @@ def test_get_depset_with_build_arg_set(self): config_path="test.depsets.yaml", workspace_dir=tmpdir, ) - depset = manager.get_depset("build_args_test_depset__py311_cpu") + depset = _get_depset( + manager.config.depsets, "build_args_test_depset__py311_cpu" + ) assert depset.name == "build_args_test_depset__py311_cpu" def test_get_depset_without_build_arg_set(self): @@ -496,7 +501,7 @@ def test_get_depset_without_build_arg_set(self): config_path="test.depsets.yaml", workspace_dir=tmpdir, ) - depset = manager.get_depset("ray_base_test_depset") + depset = _get_depset(manager.config.depsets, "ray_base_test_depset") assert depset.name == "ray_base_test_depset" def test_get_depset_with_build_arg_set_and_no_build_arg_set_provided(self): @@ -507,7 +512,7 @@ def test_get_depset_with_build_arg_set_and_no_build_arg_set_provided(self): workspace_dir=tmpdir, ) with self.assertRaises(KeyError): - manager.get_depset("build_args_test_depset_py311") + _get_depset(manager.config.depsets, "build_args_test_depset_py311") if __name__ == "__main__": diff --git a/ci/raydepsets/tests/test_workspace.py b/ci/raydepsets/tests/test_workspace.py index 430ab1247b90..318fc07c52c4 100644 --- a/ci/raydepsets/tests/test_workspace.py +++ b/ci/raydepsets/tests/test_workspace.py @@ -47,5 +47,26 @@ def test_substitute_build_args(): assert substituted_depset["name"] == "test_depset_py311_cu128" +def test_invalid_build_arg_set(): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + with open(Path(tmpdir) / "test.depsets.yaml", "w") as f: + f.write( + """ +depsets: + - name: invalid_build_arg_set + operation: compile + requirements: + - requirements_test.txt + output: requirements_compiled_invalid_build_arg_set.txt + build_arg_sets: + - invalid_build_arg_set + """ + ) + with pytest.raises(KeyError): + workspace = Workspace(dir=tmpdir) + workspace.load_config(path=Path(tmpdir) / "test.depsets.yaml") + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) From 15887001ded1eca621f6890952c5c2a90d4e58a8 Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Wed, 13 Aug 2025 20:56:08 -0700 Subject: [PATCH 096/634] [core] Store local_raylet_rpc_client in raylet_client_pool (#55490) Signed-off-by: joshlee --- src/ray/core_worker/core_worker_process.cc | 13 +++---- .../task_submission/normal_task_submitter.cc | 35 ++++--------------- .../task_submission/normal_task_submitter.h | 16 +++------ .../test/normal_task_submitter_test.cc | 35 +++++++++++++------ 4 files changed, 42 insertions(+), 57 deletions(-) diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index 6ea8b140350e..e8218c180a9e 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -495,12 +495,13 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( return address_opt; }; - auto lease_policy = RayConfig::instance().locality_aware_leasing_enabled() - ? std::unique_ptr( - std::make_unique( - *reference_counter, node_addr_factory, rpc_address)) - : std::unique_ptr( - std::make_unique(rpc_address)); + auto lease_policy = + RayConfig::instance().locality_aware_leasing_enabled() + ? std::unique_ptr( + std::make_unique( + *reference_counter, node_addr_factory, raylet_address)) + : std::unique_ptr( + std::make_unique(raylet_address)); auto normal_task_submitter = std::make_unique( rpc_address, diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 6ee82372d73c..16bb82b9e69a 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -205,7 +205,8 @@ void NormalTaskSubmitter::CancelWorkerLeaseIfNeeded(const SchedulingKey &schedul for (auto &pending_lease_request : scheduling_key_entry.pending_lease_requests) { // There is an in-flight lease request. Cancel it. - auto raylet_client = GetOrConnectRayletClient(&pending_lease_request.second); + auto raylet_client = + raylet_client_pool_->GetOrConnectByAddress(pending_lease_request.second); auto &task_id = pending_lease_request.first; RAY_LOG(DEBUG) << "Canceling lease request " << task_id; raylet_client->CancelWorkerLease( @@ -228,29 +229,6 @@ void NormalTaskSubmitter::CancelWorkerLeaseIfNeeded(const SchedulingKey &schedul } } -std::shared_ptr NormalTaskSubmitter::GetOrConnectRayletClient( - const rpc::Address *raylet_address) { - std::shared_ptr raylet_client; - RAY_CHECK(raylet_address != nullptr); - if (NodeID::FromBinary(raylet_address->node_id()) != local_node_id_) { - // A remote raylet was specified. Connect to the raylet if needed. - NodeID node_id = NodeID::FromBinary(raylet_address->node_id()); - auto it = remote_raylet_clients_.find(node_id); - if (it == remote_raylet_clients_.end()) { - RAY_LOG(INFO) << "Connecting to raylet " << node_id; - it = remote_raylet_clients_ - .emplace(node_id, - raylet_client_pool_->GetOrConnectByAddress(*raylet_address)) - .first; - } - raylet_client = it->second; - } else { - raylet_client = local_raylet_client_; - } - - return raylet_client; -} - void NormalTaskSubmitter::ReportWorkerBacklog() { absl::MutexLock lock(&mu_); ReportWorkerBacklogInternal(); @@ -279,8 +257,7 @@ void NormalTaskSubmitter::ReportWorkerBacklogInternal() { backlog_report.set_backlog_size(backlog.second.second); backlog_reports.emplace_back(backlog_report); } - local_raylet_client_->ReportWorkerBacklog( - WorkerID::FromBinary(rpc_address_.worker_id()), backlog_reports); + local_raylet_client_->ReportWorkerBacklog(worker_id_, backlog_reports); } void NormalTaskSubmitter::ReportWorkerBacklogIfNeeded( @@ -341,7 +318,7 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli raylet_address = &best_node_address; } - auto raylet_client = GetOrConnectRayletClient(raylet_address); + auto raylet_client = raylet_client_pool_->GetOrConnectByAddress(*raylet_address); const TaskID task_id = resource_spec.TaskId(); const std::string task_name = resource_spec.GetName(); RAY_LOG(DEBUG) << "Requesting lease from raylet " @@ -366,7 +343,7 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli absl::MutexLock lock(&mu_); auto &scheduling_key_entry = scheduling_key_entries_[scheduling_key]; - auto raylet_client = GetOrConnectRayletClient(&raylet_address); + auto raylet_client = raylet_client_pool_->GetOrConnectByAddress(raylet_address); scheduling_key_entry.pending_lease_requests.erase(task_id); if (status.ok()) { @@ -457,7 +434,7 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli RequestNewWorkerIfNeeded(scheduling_key, &reply.retry_at_raylet_address()); } - } else if (raylet_client != local_raylet_client_) { + } else if (NodeID::FromBinary(raylet_address.node_id()) != local_node_id_) { // A lease request to a remote raylet failed. Retry locally if the lease is // still needed. // TODO(swang): Fail after some number of retries? diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.h b/src/ray/core_worker/task_submission/normal_task_submitter.h index f72628319465..1ff97333e5d7 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.h +++ b/src/ray/core_worker/task_submission/normal_task_submitter.h @@ -107,6 +107,7 @@ class NormalTaskSubmitter { task_manager_(task_manager), lease_timeout_ms_(lease_timeout_ms), local_node_id_(local_node_id), + worker_id_(WorkerID::FromBinary(rpc_address_.worker_id())), worker_type_(worker_type), core_worker_client_pool_(std::move(core_worker_client_pool)), job_id_(job_id), @@ -171,12 +172,6 @@ class NormalTaskSubmitter { const google::protobuf::RepeatedPtrField &assigned_resources) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - /// Get an existing lease client or connect a new one. If a raylet_address is - /// provided, this connects to a remote raylet. Else, this connects to the - /// local raylet. - std::shared_ptr GetOrConnectRayletClient( - const rpc::Address *raylet_address) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - /// Report worker backlog information to the local raylet void ReportWorkerBacklogInternal() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); @@ -246,13 +241,9 @@ class NormalTaskSubmitter { /// Address of our RPC server. rpc::Address rpc_address_; - // Client that can be used to lease and return workers from the local raylet. + /// Client that can be used to lease and return workers from the local raylet. std::shared_ptr local_raylet_client_; - /// Cache of gRPC clients to remote raylets. - absl::flat_hash_map> - remote_raylet_clients_ ABSL_GUARDED_BY(mu_); - /// Raylet client pool for producing new clients to request leases from remote nodes. std::shared_ptr raylet_client_pool_; @@ -274,6 +265,9 @@ class NormalTaskSubmitter { /// if a remote raylet tells us to spill the task back to the local raylet. const NodeID local_node_id_; + /// The local worker ID. + const WorkerID worker_id_; + /// The type of this core worker process. const WorkerType worker_type_; diff --git a/src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc index 43821ecd5f94..fc58a3bdfb4e 100644 --- a/src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc @@ -476,7 +476,8 @@ TaskSpecification WithRandomTaskId(const TaskSpecification &task_spec) { class NormalTaskSubmitterTest : public testing::Test { public: NormalTaskSubmitterTest() - : raylet_client_pool(std::make_shared( + : local_node_id(NodeID::FromRandom()), + raylet_client_pool(std::make_shared( [](const rpc::Address &) { return std::make_shared(); })), raylet_client(std::make_shared()), worker_client(std::make_shared()), @@ -486,7 +487,10 @@ class NormalTaskSubmitterTest : public testing::Test { task_manager(std::make_unique()), actor_creator(std::make_shared()), lease_policy(std::make_unique()), - lease_policy_ptr(lease_policy.get()) {} + lease_policy_ptr(lease_policy.get()) { + address.set_node_id(local_node_id.Binary()); + lease_policy_ptr->SetNodeID(local_node_id); + } NormalTaskSubmitter CreateNormalTaskSubmitter( std::shared_ptr rate_limiter, @@ -494,16 +498,24 @@ class NormalTaskSubmitterTest : public testing::Test { std::function(const rpc::Address &)> raylet_client_factory = nullptr, std::shared_ptr custom_memory_store = nullptr, - int64_t lease_timeout_ms = kLongTimeout, - NodeID local_node_id = NodeID::Nil()) { + int64_t lease_timeout_ms = kLongTimeout) { if (custom_memory_store != nullptr) { store = custom_memory_store; } if (raylet_client_factory == nullptr) { raylet_client_pool = std::make_shared( - [](const rpc::Address &) { return std::make_shared(); }); + [this](const rpc::Address &) { return this->raylet_client; }); } else { - raylet_client_pool = std::make_shared(raylet_client_factory); + raylet_client_pool = std::make_shared( + [this, raylet_client_factory]( + const rpc::Address &addr) -> std::shared_ptr { + NodeID addr_node_id = NodeID::FromBinary(addr.node_id()); + if (addr_node_id == local_node_id) { + return this->raylet_client; + } else { + return raylet_client_factory(addr); + } + }); } return NormalTaskSubmitter( address, @@ -523,6 +535,7 @@ class NormalTaskSubmitterTest : public testing::Test { boost::asio::steady_timer(io_context)); } + NodeID local_node_id; rpc::Address address; std::shared_ptr raylet_client_pool; std::shared_ptr raylet_client; @@ -1375,16 +1388,13 @@ TEST_F(NormalTaskSubmitterTest, TestSpillbackRoundTrip) { remote_raylet_clients[addr.port()] = client; return client; }; - auto local_node_id = NodeID::FromRandom(); - lease_policy_ptr->SetNodeID(local_node_id); auto store = DefaultCoreWorkerMemoryStoreWithThread::CreateShared(); auto submitter = CreateNormalTaskSubmitter(std::make_shared(1), WorkerType::WORKER, raylet_client_factory, store, - kLongTimeout, - local_node_id); + kLongTimeout); TaskSpecification task = BuildEmptyTaskSpec(); ASSERT_TRUE(submitter.SubmitTask(task).ok()); @@ -1444,18 +1454,21 @@ void TestSchedulingKey(const std::shared_ptr store, const TaskSpecification &different) { rpc::Address address; auto raylet_client = std::make_shared(); + auto raylet_client_pool = std::make_shared( + [&](const rpc::Address &addr) { return raylet_client; }); auto worker_client = std::make_shared(); auto client_pool = std::make_shared( [&](const rpc::Address &addr) { return worker_client; }); auto task_manager = std::make_unique(); auto actor_creator = std::make_shared(); auto lease_policy = std::make_unique(); + lease_policy->SetNodeID(NodeID::FromRandom()); instrumented_io_context io_context; NormalTaskSubmitter submitter( address, raylet_client, client_pool, - nullptr, + raylet_client_pool, std::move(lease_policy), store, *task_manager, From 4dd73213096635cf78a1a69db84f244bb05ec50f Mon Sep 17 00:00:00 2001 From: lkchen Date: Wed, 13 Aug 2025 21:39:54 -0700 Subject: [PATCH 097/634] [data.llm] Add FAQ to doc, explain STRICT_PACK strategy used in data.llm (#55505) Signed-off-by: Linkun --- doc/source/data/working-with-llms.rst | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/doc/source/data/working-with-llms.rst b/doc/source/data/working-with-llms.rst index 8b12c4add575..d9b052b03e83 100644 --- a/doc/source/data/working-with-llms.rst +++ b/doc/source/data/working-with-llms.rst @@ -343,14 +343,34 @@ Data for the following features and attributes is collected to improve Ray Data If you would like to opt-out from usage data collection, you can follow :ref:`Ray usage stats ` to turn it off. -.. _production_guide: +.. _faqs: -Production guide +Frequently Asked Questions (FAQs) -------------------------------------------------- +.. TODO(#55491): Rewrite this section once the restriction is lifted. +.. _cross_node_parallelism: + +How to configure LLM stage to parallelize across multiple nodes? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +At the moment, Ray Data LLM doesn't support cross-node parallelism (either +tensor parallelism or pipeline parallelism). + +The processing pipeline is designed to run on a single node. The number of +GPUs is calculated as the product of the tensor parallel size and the pipeline +parallel size, and apply +[`STRICT_PACK` strategy](https://docs.ray.io/en/latest/ray-core/scheduling/placement-group.html#pgroup-strategy) +to ensure that each replica of the LLM stage is executed on a single node. + +Nevertheless, you can still horizontally scale the LLM stage to multiple nodes +as long as each replica (TP * PP) fits into a single node. The number of +replicas is configured by the `concurrency` argument in +:class:`vLLMEngineProcessorConfig `. + .. _model_cache: -Caching model weight to remote object storage +How to cache model weight to remote object storage ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ While deploying Ray Data LLM to large scale clusters, model loading may be rate From 02340e1f402b8ebde104e92c9941b149e5555acb Mon Sep 17 00:00:00 2001 From: harshit-anyscale Date: Thu, 14 Aug 2025 10:21:53 +0530 Subject: [PATCH 098/634] add support for async inference (#54824) This PR aims to provide basic support for asynchronous inference in the ray serve. RFC can be found at: https://github.com/ray-project/ray/issues/54652 The PR doesn't contains all the implementation pieces as having all the code changes in a single PR would be very difficult to review. Missing pieces are - implementation of failed and unprocessed task queue for the celery task processor - add more detailed and thorough tests for the same. These missing pieces will be taken care of in the subsequent PRs. --------- Signed-off-by: harshit --- doc/source/serve/api/index.md | 3 + python/ray/serve/schema.py | 63 ++- python/ray/serve/task_consumer.py | 206 ++++++++ python/ray/serve/task_processor.py | 462 ++++++++++++++++++ python/ray/serve/tests/BUILD | 1 + python/ray/serve/tests/test_task_processor.py | 206 ++++++++ .../serve/tests/unit/test_task_consumer.py | 284 +++++++++++ python/requirements.txt | 1 + python/requirements_compiled.txt | 31 +- .../requirements_compiled_ray_py311_cpu.txt | 74 +++ .../requirements_compiled_ray_py311_cu121.txt | 74 +++ .../requirements_compiled_ray_py311_cu128.txt | 74 +++ ...quirements_compiled_ray_test_py311_cpu.txt | 63 +++ ...irements_compiled_ray_test_py311_cu121.txt | 63 +++ ...irements_compiled_ray_test_py311_cu128.txt | 63 +++ ...requirements_compiled_rayllm_py311_cpu.txt | 74 +++ ...quirements_compiled_rayllm_py311_cu121.txt | 74 +++ ...quirements_compiled_rayllm_py311_cu128.txt | 74 +++ ...rements_compiled_rayllm_test_py311_cpu.txt | 63 +++ ...ments_compiled_rayllm_test_py311_cu121.txt | 63 +++ ...ments_compiled_rayllm_test_py311_cu128.txt | 63 +++ python/setup.py | 11 + 22 files changed, 2088 insertions(+), 2 deletions(-) create mode 100644 python/ray/serve/task_consumer.py create mode 100644 python/ray/serve/task_processor.py create mode 100644 python/ray/serve/tests/test_task_processor.py create mode 100644 python/ray/serve/tests/unit/test_task_consumer.py diff --git a/doc/source/serve/api/index.md b/doc/source/serve/api/index.md index 52d8c502eb3f..75e60e8577d6 100644 --- a/doc/source/serve/api/index.md +++ b/doc/source/serve/api/index.md @@ -384,6 +384,9 @@ Content-Type: application/json schema.ServeApplicationSchema schema.DeploymentSchema schema.RayActorOptionsSchema + schema.CeleryAdapterConfig + schema.TaskProcessorConfig + schema.TaskResult ``` (serve-rest-api-response-schema)= diff --git a/python/ray/serve/schema.py b/python/ray/serve/schema.py index 607097fee8a7..5ba0d8e0a344 100644 --- a/python/ray/serve/schema.py +++ b/python/ray/serve/schema.py @@ -2,7 +2,7 @@ from collections import Counter from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional, Set, Union +from typing import Any, Callable, Dict, List, Optional, Set, Union from zlib import crc32 from ray._common.pydantic_compat import ( @@ -1202,3 +1202,64 @@ def _get_user_facing_json_serializable_dict( ) return values + + +@PublicAPI(stability="alpha") +class CeleryAdapterConfig(BaseModel): + """ + Celery adapter config. You can use it to configure the Celery task processor for your Serve application. + """ + + broker_url: str = Field(..., description="The URL of the broker to use for Celery.") + backend_url: str = Field( + ..., description="The URL of the backend to use for Celery." + ) + broker_transport_options: Optional[Dict[str, Any]] = Field( + default=None, description="The broker transport options to use for Celery." + ) + worker_concurrency: Optional[int] = Field( + default=10, + description="The number of concurrent worker threads for the task processor.", + ) + + +@PublicAPI(stability="alpha") +class TaskProcessorConfig(BaseModel): + """ + Task processor config. You can use it to configure the task processor for your Serve application. + """ + + queue_name: str = Field( + ..., description="The name of the queue to use for task processing." + ) + adapter: Union[str, Callable] = Field( + default="ray.serve.task_processor.CeleryTaskProcessorAdapter", + description="The adapter to use for task processing. By default, Celery is used.", + ) + adapter_config: Any = Field(..., description="The adapter config.") + max_retries: Optional[int] = Field( + default=3, + description="The maximum number of times to retry a task before marking it as failed.", + ) + failed_task_queue_name: Optional[str] = Field( + default=None, + description="The name of the failed task queue. This is used to move failed tasks to a dead-letter queue after max retries.", + ) + unprocessable_task_queue_name: Optional[str] = Field( + default=None, + description="The name of the unprocessable task queue. This is used to move unprocessable tasks(like tasks with serialization issue, or missing handler) to a dead-letter queue.", + ) + + +@PublicAPI(stability="alpha") +class TaskResult(BaseModel): + """ + Task result Model. + """ + + id: str = Field(..., description="The ID of the task.") + status: str = Field(..., description="The status of the task.") + created_at: Optional[float] = Field( + default=None, description="The timestamp of the task creation." + ) + result: Any = Field(..., description="The result of the task.") diff --git a/python/ray/serve/task_consumer.py b/python/ray/serve/task_consumer.py new file mode 100644 index 000000000000..8210bc69ab31 --- /dev/null +++ b/python/ray/serve/task_consumer.py @@ -0,0 +1,206 @@ +import inspect +import logging +from functools import wraps +from typing import Callable, Optional + +from ray._common.utils import import_attr +from ray.serve._private.constants import SERVE_LOGGER_NAME +from ray.serve.schema import TaskProcessorConfig +from ray.serve.task_processor import TaskProcessorAdapter +from ray.util.annotations import PublicAPI + +logger = logging.getLogger(SERVE_LOGGER_NAME) + + +@PublicAPI(stability="alpha") +def instantiate_adapter_from_config( + task_processor_config: TaskProcessorConfig, +) -> TaskProcessorAdapter: + """ + Create a TaskProcessorAdapter instance from the provided configuration and call .initialize(). This function supports two ways to specify an adapter: + + 1. String path: A fully qualified module path to an adapter class + Example: "ray.serve.task_processor.CeleryTaskProcessorAdapter" + + 2. Class reference: A direct reference to an adapter class + Example: CeleryTaskProcessorAdapter + + Args: + task_processor_config: Configuration object containing adapter specification. + + Returns: + An initialized TaskProcessorAdapter instance ready for use. + + Raises: + ValueError: If the adapter string path is malformed or cannot be imported. + TypeError: If the adapter is not a string or callable class. + + Example: + .. code-block:: python + + config = TaskProcessorConfig( + adapter="my.module.CustomAdapter", + adapter_config={"param": "value"}, + queue_name="my_queue" + ) + adapter = instantiate_adapter_from_config(config) + """ + + adapter = task_processor_config.adapter + + # Handle string-based adapter specification (module path) + if isinstance(adapter, str): + adapter_class = import_attr(adapter) + + elif callable(adapter): + adapter_class = adapter + + else: + raise TypeError( + f"Adapter must be either a string path or a callable class, got {type(adapter).__name__}: {adapter}" + ) + + try: + adapter_instance = adapter_class(config=task_processor_config) + except Exception as e: + raise RuntimeError(f"Failed to instantiate {adapter_class.__name__}: {e}") + + if not isinstance(adapter_instance, TaskProcessorAdapter): + raise TypeError( + f"{adapter_class.__name__} must inherit from TaskProcessorAdapter, got {type(adapter_instance).__name__}" + ) + + try: + adapter_instance.initialize(config=task_processor_config) + except Exception as e: + raise RuntimeError(f"Failed to initialize {adapter_class.__name__}: {e}") + + return adapter_instance + + +@PublicAPI(stability="alpha") +def task_consumer(*, task_processor_config: TaskProcessorConfig): + """ + Decorator to mark a class as a TaskConsumer. + + Args: + task_processor_config: Configuration for the task processor (required) + + Note: + This decorator must be used with parentheses: + @task_consumer(task_processor_config=config) + + Returns: + A wrapper class that inherits from the target class and implements the task consumer functionality. + + Example: + .. code-block:: python + + from ray import serve + from ray.serve.task_consumer import task_consumer, task_handler + + @serve.deployment + @task_consumer(task_processor_config=config) + class MyTaskConsumer: + + @task_handler(name="my_task") + def my_task(self, *args, **kwargs): + pass + + """ + + def decorator(target_cls): + class TaskConsumerWrapper(target_cls): + _adapter: TaskProcessorAdapter + + def __init__(self, *args, **kwargs): + target_cls.__init__(self, *args, **kwargs) + + self._adapter = instantiate_adapter_from_config(task_processor_config) + + for name, method in inspect.getmembers( + target_cls, predicate=inspect.isfunction + ): + if getattr(method, "_is_task_handler", False): + task_name = getattr(method, "_task_name", name) + + # Create a callable that properly binds the method to this instance + bound_method = getattr(self, name) + + self._adapter.register_task_handle(bound_method, task_name) + + try: + self._adapter.start_consumer() + logger.info("task consumer started successfully") + except Exception as e: + logger.error(f"Failed to start task consumer: {e}") + raise + + def __del__(self): + self._adapter.stop_consumer() + self._adapter.shutdown() + + if hasattr(target_cls, "__del__"): + target_cls.__del__(self) + + return TaskConsumerWrapper + + return decorator + + +@PublicAPI(stability="alpha") +def task_handler( + _func: Optional[Callable] = None, *, name: Optional[str] = None +) -> Callable: + """ + Decorator to mark a method as a task handler. + Optionally specify a task name. Default is the method name. + + Arguments: + _func: The function to decorate. + name: The name of the task. + + Returns: + A wrapper function that is marked as a task handler. + + Example: + .. code-block:: python + + from ray import serve + from ray.serve.task_consumer import task_consumer, task_handler + + @serve.deployment + @task_consumer(task_processor_config=config) + class MyTaskConsumer: + + @task_handler(name="my_task") + def my_task(self, *args, **kwargs): + pass + + """ + + # Validate name parameter if provided + if name is not None and (not isinstance(name, str) or not name.strip()): + raise ValueError(f"Task name must be a non-empty string, got {name}") + + def decorator(f): + # async functions are not supported yet in celery `threads` worker pool + if not inspect.iscoroutinefunction(f): + + @wraps(f) + def wrapper(*args, **kwargs): + return f(*args, **kwargs) + + wrapper._is_task_handler = True # type: ignore + wrapper._task_name = name or f.__name__ # type: ignore + return wrapper + + else: + raise NotImplementedError("Async task handlers are not supported yet") + + if _func is not None: + # Used without arguments: @task_handler + return decorator(_func) + else: + # Used with arguments: @task_handler(name="...") + return decorator diff --git a/python/ray/serve/task_processor.py b/python/ray/serve/task_processor.py new file mode 100644 index 000000000000..ed7435b75981 --- /dev/null +++ b/python/ray/serve/task_processor.py @@ -0,0 +1,462 @@ +import logging +import threading +import time +from abc import ABC, abstractmethod +from enum import Enum, auto +from typing import Any, Callable, Dict, List, Optional, Set + +from celery import Celery + +from ray.serve._private.constants import SERVE_LOGGER_NAME +from ray.serve.schema import ( + CeleryAdapterConfig, + TaskProcessorConfig, + TaskResult, +) +from ray.util.annotations import PublicAPI + +logger = logging.getLogger(SERVE_LOGGER_NAME) + + +@PublicAPI(stability="alpha") +class AsyncCapability(Enum): + """ + Enum defining different async capabilities a TaskProcessor can support. + + Each capability represents an async operation that an adapter may or may not + support. Use TaskProcessorAdapter.supports_async_capability() to check if + a specific capability is available before using the corresponding async method. + """ + + ENQUEUE_TASK = auto() # Ability to enqueue tasks asynchronously + GET_TASK_STATUS = auto() # Ability to retrieve task status asynchronously + CANCEL_TASK = auto() # Ability to cancel tasks asynchronously + GET_METRICS = auto() # Ability to retrieve metrics asynchronously + HEALTH_CHECK = auto() # Ability to perform health checks asynchronously + + +@PublicAPI(stability="alpha") +class TaskProcessorAdapter(ABC): + """ + Abstract base class for task processing adapters. + + Subclasses can support different combinations of sync and async operations. + Use supports_async_capability() to check if a specific async operation is supported. + """ + + def __init__(self): + """ + Initialize the TaskProcessorAdapter. + + Sets up an empty set of async capabilities. Subclasses should add their + supported async capabilities to self._async_capabilities in their __init__ + method. + """ + self._async_capabilities: Set[AsyncCapability] = set() + + @property + def async_capabilities(self) -> Set[AsyncCapability]: + """ + Get the set of async capabilities supported by this adapter. + + Returns: + Set[AsyncCapability]: A copy of the set containing all async capabilities + supported by this adapter. Modifying the returned set will not affect + the adapter's capabilities. + """ + return self._async_capabilities.copy() + + def supports_async_capability(self, capability: AsyncCapability) -> bool: + """ + Check if this adapter supports a specific async capability. + + Args: + capability: The AsyncCapability enum value to check for. + + Returns: + bool: True if the capability is supported, False otherwise. + """ + return capability in self._async_capabilities + + def supports_any_async(self) -> bool: + """ + Check if this adapter supports any async operations. + + Returns: + bool: True if at least one async capability is supported, False if this is a sync-only adapter. + """ + return len(self._async_capabilities) > 0 + + @abstractmethod + def initialize(self, config: TaskProcessorConfig): + """ + Initialize the task processor with the given configuration. + + Args: + config: TaskProcessorConfig containing adapter-specific configuration, queue names, retry settings, and other options. + """ + pass + + @abstractmethod + def register_task_handle(self, func: Callable, name: Optional[str] = None): + """ + Register a function as a task handler. + + Args: + func: The function to register as a task handler. + name: Custom name for the task. + """ + pass + + @abstractmethod + def enqueue_task_sync( + self, + task_name: str, + args: Optional[Any] = None, + kwargs: Optional[Any] = None, + **options, + ) -> TaskResult: + """ + Enqueue a task for execution synchronously. + + Args: + task_name: Name of the registered task to execute. + args: Positional arguments to pass to the task function. + kwargs: Keyword arguments to pass to the task function. + **options: Additional adapter-specific options for task execution. + + Returns: + TaskResult: Object containing task ID, status, and other metadata. + """ + pass + + @abstractmethod + def get_task_status_sync(self, task_id: str) -> TaskResult: + """ + Retrieve the current status of a task synchronously. + + Args: + task_id: Unique identifier of the task to query. + + Returns: + TaskResult: Object containing current task status, result, and other metadata. + """ + pass + + @abstractmethod + def start_consumer(self, **kwargs): + """ + Start the task consumer/worker process. + """ + pass + + @abstractmethod + def stop_consumer(self, timeout: float = 10.0): + """ + Stop the task consumer gracefully. + + Args: + timeout: Maximum time in seconds to wait for the consumer to stop. + """ + pass + + @abstractmethod + def shutdown(self): + """ + Shutdown the task processor and clean up resources. + """ + pass + + @abstractmethod + def cancel_task_sync(self, task_id: str) -> bool: + """ + Cancel a task synchronously. + + Args: + task_id: Unique identifier of the task to cancel. + + Returns: + bool: True if cancellation was requested successfully. + """ + pass + + @abstractmethod + def get_metrics_sync(self) -> Dict[str, Any]: + """ + Get metrics synchronously. + + Returns: + Dict[str, Any]: Adapter-specific metrics data. + """ + pass + + @abstractmethod + def health_check_sync(self) -> List[Dict]: + """ + Perform health check synchronously. + + Returns: + List[Dict]: Health status information for workers/components. + """ + pass + + async def enqueue_task_async( + self, + task_name: str, + args: Optional[Any] = None, + kwargs: Optional[Any] = None, + **options, + ) -> TaskResult: + """ + Enqueue a task asynchronously. + + Args: + task_name: Name of the registered task to execute. + args: Positional arguments to pass to the task function. + kwargs: Keyword arguments to pass to the task function. + **options: Additional adapter-specific options for task execution. + + Returns: + TaskResult: Object containing task ID, status, and other metadata. + + Raises: + NotImplementedError: If async enqueue is not supported by this adapter. + """ + if not self.supports_async_capability(AsyncCapability.ENQUEUE_TASK): + raise NotImplementedError( + f"{self.__class__.__name__} does not support async task enqueueing. " + f"Use enqueue_task_sync() instead or check supports_async_capability() first." + ) + + raise NotImplementedError("Subclass must implement enqueue_task_async function") + + async def get_task_status_async(self, task_id: str) -> TaskResult: + """ + Get task status asynchronously. + + Args: + task_id: Unique identifier of the task to query. + + Returns: + TaskResult: Object containing current task status, result, and other metadata. + + Raises: + NotImplementedError: If async status retrieval is not supported by this adapter. + """ + if not self.supports_async_capability(AsyncCapability.GET_TASK_STATUS): + raise NotImplementedError( + f"{self.__class__.__name__} does not support async task status retrieval. " + f"Use get_task_status_sync() instead or check supports_async_capability() first." + ) + + raise NotImplementedError( + "Subclass must implement get_task_status_async function" + ) + + async def cancel_task_async(self, task_id: str) -> bool: + """ + Cancel a task. + + Args: + task_id: Unique identifier of the task to cancel. + + Returns: + bool: True if cancellation was requested successfully. + + Raises: + NotImplementedError: If async task cancellation is not supported by this adapter. + """ + if not self.supports_async_capability(AsyncCapability.CANCEL_TASK): + raise NotImplementedError( + f"{self.__class__.__name__} does not support async task cancellation. " + f"Check supports_async_capability() first." + ) + + raise NotImplementedError("Subclass must implement cancel_task_async function") + + async def get_metrics_async(self) -> Dict[str, Any]: + """ + Get metrics asynchronously. + + Returns: + Dict[str, Any]: Adapter-specific metrics data. + + Raises: + NotImplementedError: If async metrics retrieval is not supported by this adapter. + """ + if not self.supports_async_capability(AsyncCapability.GET_METRICS): + raise NotImplementedError( + f"{self.__class__.__name__} does not support async metrics retrieval. " + f"Check supports_async_capability() first." + ) + + raise NotImplementedError("Subclass must implement get_metrics_async function") + + async def health_check_async(self) -> List[Dict]: + """ + Perform health check asynchronously. + + Returns: + List[Dict]: Health status information for workers/components. + + Raises: + NotImplementedError: If async health check is not supported by this adapter. + """ + if not self.supports_async_capability(AsyncCapability.HEALTH_CHECK): + raise NotImplementedError( + f"{self.__class__.__name__} does not support async health check. " + f"Check supports_async_capability() first." + ) + + raise NotImplementedError("Subclass must implement health_check_async function") + + +@PublicAPI(stability="alpha") +class CeleryTaskProcessorAdapter(TaskProcessorAdapter): + """ + Celery-based task processor adapter. + This adapter does NOT support any async operations. + All operations must be performed synchronously. + """ + + _app: Celery + _config: TaskProcessorConfig + _worker_thread: Optional[threading.Thread] = None + + def __init__(self, config: TaskProcessorConfig): + super().__init__() + + if not isinstance(config.adapter_config, CeleryAdapterConfig): + raise TypeError( + "TaskProcessorConfig.adapter_config must be an instance of CeleryAdapterConfig" + ) + + self._config = config + + # Celery adapter does not support any async capabilities + # self._async_capabilities is already an empty set from parent class + + def initialize(self, config: TaskProcessorConfig): + self._app = Celery( + config.queue_name, + backend=config.adapter_config.backend_url, + broker=config.adapter_config.broker_url, + ) + + self._app.conf.update( + loglevel="info", + worker_pool="threads", + worker_concurrency=config.adapter_config.worker_concurrency, + max_retries=config.max_retries, + task_default_queue=config.queue_name, + # Store task results so they can be retrieved after completion + task_ignore_result=False, + # Acknowledge tasks only after completion (not when received) for better reliability + task_acks_late=True, + # Reject and requeue tasks when worker is lost to prevent data loss + reject_on_worker_lost=True, + ) + + if config.adapter_config.broker_transport_options is not None: + self._app.conf.update( + broker_transport_options=config.adapter_config.broker_transport_options, + ) + + ### TODO(harshit|SERVE-987): add the failed_task_queue_name and unprocessable_task_queue_name business logic here + + def register_task_handle(self, func, name=None): + task_options = { + "autoretry_for": (Exception,), + "retry_kwargs": {"max_retries": self._config.max_retries}, + "retry_backoff": True, + "retry_backoff_max": 60, # Max backoff of 60 seconds + "retry_jitter": False, # Disable jitter for predictable testing + } + + if name: + self._app.task(name=name, **task_options)(func) + else: + self._app.task(**task_options)(func) + + def enqueue_task_sync( + self, task_name, args=None, kwargs=None, **options + ) -> TaskResult: + task_response = self._app.send_task( + task_name, + args=args, + kwargs=kwargs, + queue=self._config.queue_name, + **options, + ) + + return TaskResult( + id=task_response.id, + status=task_response.status, + created_at=time.time(), + result=task_response.result, + ) + + def get_task_status_sync(self, task_id) -> TaskResult: + task_details = self._app.AsyncResult(task_id) + return TaskResult( + id=task_details.id, + result=task_details.result, + status=task_details.status, + ) + + def start_consumer(self, **kwargs): + """Starts the Celery worker thread.""" + if self._worker_thread is not None and self._worker_thread.is_alive(): + logger.info("Celery worker thread is already running.") + return + + self._worker_thread = threading.Thread( + target=self._app.worker_main, + args=(("worker", f"--hostname={self._app.main}"),), + ) + self._worker_thread.start() + + logger.info(f"Celery worker thread started with hostname: {self._app.main}") + + def stop_consumer(self, timeout: float = 10.0): + """Signals the Celery worker to shut down and waits for it to terminate.""" + if self._worker_thread is None or not self._worker_thread.is_alive(): + logger.info("Celery worker thread is not running.") + return + + logger.info("Sending shutdown signal to Celery worker...") + + # Use the worker's hostname for targeted shutdown + self._app.control.broadcast( + "shutdown", destination=[f"celery@{self._app.main}"] + ) + self._worker_thread.join(timeout=timeout) + + if self._worker_thread.is_alive(): + logger.warning(f"Worker thread did not terminate after {timeout} seconds.") + else: + logger.info("Celery worker thread has stopped.") + + self._worker_thread = None + + def shutdown(self): + self._app.control.shutdown() + + def cancel_task_sync(self, task_id) -> bool: + return self._app.AsyncResult(task_id).cancel() + + def get_metrics_sync(self) -> Dict[str, Any]: + """ + Returns the metrics of the Celery worker synchronously. + More details can be found here: https://docs.celeryq.dev/en/stable/reference/celery.app.control.html#celery.app.control.Inspect.stats + """ + return self._app.control.inspect().stats() + + def health_check_sync(self) -> List[Dict]: + """ + Checks the health of the Celery worker synchronously. + Returns a list of dictionaries, each containing the worker name and a dictionary with the health status. + Example: [{'celery@192.168.1.100': {'ok': 'pong'}}] + More details can be found here: https://docs.celeryq.dev/en/stable/reference/celery.app.control.html#celery.app.control.Control.ping + """ + return self._app.control.ping() diff --git a/python/ray/serve/tests/BUILD b/python/ray/serve/tests/BUILD index 3db7c5d6a0e8..997cc2de7a2b 100644 --- a/python/ray/serve/tests/BUILD +++ b/python/ray/serve/tests/BUILD @@ -47,6 +47,7 @@ py_test_module_list( "test_persistence.py", "test_proxy_actor_wrapper.py", "test_replica_request_context.py", + "test_task_processor.py", "test_util.py", "test_websockets.py", ], diff --git a/python/ray/serve/tests/test_task_processor.py b/python/ray/serve/tests/test_task_processor.py new file mode 100644 index 000000000000..171c67df8477 --- /dev/null +++ b/python/ray/serve/tests/test_task_processor.py @@ -0,0 +1,206 @@ +import sys +import tempfile +from pathlib import Path + +import pytest + +import ray +from ray import serve +from ray._common.test_utils import wait_for_condition +from ray.serve.schema import CeleryAdapterConfig, TaskProcessorConfig +from ray.serve.task_consumer import ( + instantiate_adapter_from_config, + task_consumer, + task_handler, +) + + +@ray.remote +def send_request_to_queue(processor_config: TaskProcessorConfig, data): + adapter_instance = instantiate_adapter_from_config( + task_processor_config=processor_config + ) + result = adapter_instance.enqueue_task_sync("process_request", args=[data]) + assert result.id is not None + return result.id + + +@pytest.fixture(scope="function") +def temp_queue_directory(): + """Creates a temporary directory with 'queue', 'results', and 'control' subdirectories for task consumer tests.""" + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir_path = Path(tmpdir) + + data_folder_queue = tmpdir_path / "queue" + data_folder_queue.mkdir() + + results_path = tmpdir_path / "results" + results_path.mkdir() + + control_path = tmpdir_path / "control" + control_path.mkdir() + + yield { + "queue_path": data_folder_queue, + "results_path": results_path, + "control_path": control_path, + } + + +@pytest.fixture(scope="function") +def transport_options(temp_queue_directory): + """Create standard transport options for filesystem broker.""" + + queue_path = temp_queue_directory["queue_path"] + control_path = temp_queue_directory["control_path"] + + return { + # Incoming message queue - where new task messages are written when sent to broker + "data_folder_in": queue_path, + # Outgoing message storage - where task results and responses are written after completion + "data_folder_out": queue_path, + # Processed message archive - where messages are moved after successful processing + "data_folder_processed": queue_path, + # Control message storage - where Celery management and control commands are stored + "control_folder": control_path, + } + + +@pytest.fixture(scope="function") +def create_processor_config(temp_queue_directory, transport_options): + """Create a TaskProcessorConfig with common defaults.""" + + def _create(**kwargs): + results_path = temp_queue_directory["results_path"] + + config_params = { + "queue_name": "my_default_app_queue", + "adapter_config": CeleryAdapterConfig( + broker_url="filesystem://", + backend_url=f"file://{results_path}", + broker_transport_options=transport_options, + ), + } + config_params.update(kwargs) + + return TaskProcessorConfig(**config_params) + + return _create + + +class TestTaskConsumerWithRayServe: + """Test task consumer integration with Ray Serve.""" + + def test_task_consumer_as_serve_deployment( + self, temp_queue_directory, serve_instance, create_processor_config + ): + """Test that task consumers can be used as Ray Serve deployments.""" + processor_config = create_processor_config() + + @serve.deployment + @task_consumer(task_processor_config=processor_config) + class ServeTaskConsumer: + def __init__(self): + self.data_received = None + self.task_received = False + + @task_handler(name="process_request") + def process_request(self, data): + self.task_received = True + self.data_received = data + + def assert_task_received(self): + assert self.task_received is True + assert self.data_received is not None + assert self.data_received == "test_data_1" + + # Deploy the consumer as a Serve deployment + handle = serve.run(ServeTaskConsumer.bind()) + send_request_to_queue.remote(processor_config, "test_data_1") + + def assert_result(): + try: + # `assert_task_received` will throw AssertionError if the task was not received or data is not as expected + handle.assert_task_received.remote().result() + return True + except Exception: + return False + + wait_for_condition(assert_result) + + def test_task_consumer_as_serve_deployment_with_failed_task( + self, temp_queue_directory, serve_instance, create_processor_config + ): + """Test that task consumers can be used as Ray Serve deployments.""" + processor_config = create_processor_config( + failed_task_queue_name="my_failed_task_queue" + ) + + @serve.deployment + @task_consumer(task_processor_config=processor_config) + class ServeTaskConsumer: + def __init__(self): + self.num_calls = 0 + + @task_handler(name="process_request") + def process_request(self, data): + self.num_calls += 1 + raise ValueError("Task failed as expected") + + def get_num_calls(self): + return self.num_calls + + handle = serve.run(ServeTaskConsumer.bind()) + task_id_ref = send_request_to_queue.remote(processor_config, "test_data_1") + task_id = ray.get(task_id_ref) + + adapter_instance = instantiate_adapter_from_config( + task_processor_config=processor_config + ) + + def assert_result(): + result = adapter_instance.get_task_status_sync(task_id) + + if ( + result.status == "FAILURE" + and result.result is not None + and isinstance(result.result, ValueError) + and str(result.result) == "Task failed as expected" + and handle.get_num_calls.remote().result() + == 1 + processor_config.max_retries + ): + return True + else: + return False + + wait_for_condition(assert_result, timeout=10) + + def test_task_consumer_as_serve_deployment_with_async_task_handler( + self, temp_queue_directory, serve_instance, create_processor_config + ): + """Test that task consumers properly raise NotImplementedError for async task handlers.""" + processor_config = create_processor_config() + + # Test that async task handlers raise NotImplementedError during decoration + with pytest.raises( + NotImplementedError, + match="Async task handlers are not supported yet", + ): + + @serve.deployment + @task_consumer(task_processor_config=processor_config) + class ServeTaskConsumer: + def __init__(self): + self.data_received = None + self.task_received = False + + # This async task handler should raise NotImplementedError during decoration + @task_handler(name="process_request") + async def process_request(self, data): + self.task_received = True + self.data_received = data + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", "-s", __file__])) diff --git a/python/ray/serve/tests/unit/test_task_consumer.py b/python/ray/serve/tests/unit/test_task_consumer.py new file mode 100644 index 000000000000..4fe3c547ed01 --- /dev/null +++ b/python/ray/serve/tests/unit/test_task_consumer.py @@ -0,0 +1,284 @@ +import sys +import uuid +from typing import Any, Dict, List +from unittest.mock import MagicMock, call + +import pytest + +from ray.serve.schema import CeleryAdapterConfig, TaskProcessorConfig, TaskResult +from ray.serve.task_consumer import task_consumer, task_handler +from ray.serve.task_processor import TaskProcessorAdapter + + +class MockTaskProcessorAdapter(TaskProcessorAdapter): + """Mock adapter for testing task processor functionality.""" + + _start_consumer_received: bool = False + _stop_consumer_received: bool = False + _shutdown_received: bool = False + + def __init__(self, config: TaskProcessorConfig): + self._config = config + self.register_task_handle_mock = MagicMock() + + def initialize(self, config: TaskProcessorConfig): + pass + + def register_task_handle(self, func, name=None): + self.register_task_handle_mock(func, name=name) + + def enqueue_task_sync( + self, task_name, args=None, kwargs=None, **options + ) -> TaskResult: + pass + + def get_task_status_sync(self, task_id) -> TaskResult: + pass + + def start_consumer(self, **kwargs): + self._start_consumer_received = True + + def stop_consumer(self, timeout: float = 10.0): + self._stop_consumer_received = True + + def shutdown(self): + self._shutdown_received = True + + def cancel_task_sync(self, task_id) -> bool: + pass + + def get_metrics_sync(self) -> Dict[str, Any]: + pass + + def health_check_sync(self) -> List[Dict]: + pass + + +@pytest.fixture +def config(): + """Provides a mock TaskProcessorConfig.""" + queue_name = f"test_queue_{uuid.uuid4().hex}" + return TaskProcessorConfig( + queue_name=queue_name, + adapter_config=CeleryAdapterConfig( + broker_url="fake://", + backend_url="fake://", + ), + adapter=MockTaskProcessorAdapter, + ) + + +class TestTaskHandlerDecorator: + """Test the task_handler decorator.""" + + def _create_and_test_handler(self, decorator_args=None, expected_name=None): + """Helper to create and test a task handler.""" + mock = MagicMock() + + if decorator_args is None: + + @task_handler + def test_handler(): + mock() + + else: + + @task_handler(**decorator_args) + def test_handler(): + mock() + + test_handler() + + assert mock.call_count == 1 + assert test_handler._task_name == expected_name + + def test_task_handler_decorator_with_name(self): + self._create_and_test_handler( + decorator_args={"name": "my_task"}, expected_name="my_task" + ) + + def test_task_handler_decorator_without_name(self): + self._create_and_test_handler(expected_name="test_handler") + + @pytest.mark.parametrize("invalid_name", ["", " ", 123]) + def test_task_handler_decorator_invalid_name(self, invalid_name): + """Test various invalid task names.""" + with pytest.raises( + ValueError, + match=f"Task name must be a non-empty string, got {invalid_name}", + ): + + @task_handler(name=invalid_name) + def my_task_handler(): + pass + + def test_task_handler_on_callable_object_without_name_attr(self): + """Test that AttributeError is raised for callables with no __name__.""" + + class MyCallable: + """A simple callable class without a __name__ attribute on instances.""" + + def __call__(self): + pass + + with pytest.raises(AttributeError): + task_handler(MyCallable()) + + +class TestTaskConsumerDecorator: + """Test the task_consumer decorator.""" + + def _verify_and_cleanup(self, instance, expected_calls=None): + """Verify consumer and cleanup instance.""" + adapter = instance._adapter + assert adapter._start_consumer_received + + if expected_calls is not None: + if expected_calls: + calls = [call(method, name=name) for method, name in expected_calls] + adapter.register_task_handle_mock.assert_has_calls( + calls, any_order=False + ) + assert adapter.register_task_handle_mock.call_count == len( + expected_calls + ) + else: + adapter.register_task_handle_mock.assert_not_called() + + del instance + + def _run_consumer_test( + self, config, consumer_class_factory, expected_calls_factory=None + ): + """Run a consumer test with factory functions.""" + consumer_class = consumer_class_factory(config) + instance = consumer_class() + + expected_calls = ( + expected_calls_factory(instance) if expected_calls_factory else None + ) + + self._verify_and_cleanup(instance, expected_calls) + + def test_task_consumer_basic(self, config): + """Test basic functionality of the task_consumer decorator.""" + + def make_consumer(cfg): + @task_consumer(task_processor_config=cfg) + class MyConsumer: + @task_handler + def my_task(self): + pass + + return MyConsumer + + self._run_consumer_test( + config, make_consumer, lambda inst: [(inst.my_task, "my_task")] + ) + + def test_task_consumer_multiple_handlers(self, config): + """Test with multiple task handlers.""" + + def make_consumer(cfg): + @task_consumer(task_processor_config=cfg) + class MyConsumer: + @task_handler + def task1(self): + pass + + @task_handler + def task2(self): + pass + + return MyConsumer + + self._run_consumer_test( + config, + make_consumer, + lambda inst: [(inst.task1, "task1"), (inst.task2, "task2")], + ) + + def test_task_consumer_custom_names(self, config): + """Test task handlers with and without custom names.""" + + def make_consumer(cfg): + @task_consumer(task_processor_config=cfg) + class MyConsumer: + @task_handler(name="custom_task") + def task1(self): + pass + + @task_handler + def task2(self): + pass + + return MyConsumer + + self._run_consumer_test( + config, + make_consumer, + lambda inst: [(inst.task1, "custom_task"), (inst.task2, "task2")], + ) + + def test_task_consumer_init_args(self, config): + """Test that __init__ arguments are passed correctly.""" + + @task_consumer(task_processor_config=config) + class MyConsumer: + def __init__(self, value): + self.value = value + + instance = MyConsumer(value=42) + assert instance.value == 42 + self._verify_and_cleanup(instance) + + def test_task_consumer_no_handlers(self, config): + """Test with a class that has no task handlers.""" + + def make_consumer(cfg): + @task_consumer(task_processor_config=cfg) + class MyConsumer: + def some_method(self): + pass + + return MyConsumer + + self._run_consumer_test(config, make_consumer, lambda inst: []) + + def test_task_consumer_inheritance(self, config): + """Test that inherited task handlers are registered.""" + + def make_consumer(cfg): + class BaseConsumer: + @task_handler + def base_task(self): + pass + + @task_consumer(task_processor_config=cfg) + class DerivedConsumer(BaseConsumer): + @task_handler + def derived_task(self): + pass + + return DerivedConsumer + + self._run_consumer_test( + config, + make_consumer, + lambda inst: [ + (inst.base_task, "base_task"), + (inst.derived_task, "derived_task"), + ], + ) + + def test_task_consumer_no_args_decorator(self): + """Test using @task_consumer without arguments raises TypeError.""" + with pytest.raises(TypeError): + + @task_consumer + class MyConsumer: + pass + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", "-s", __file__])) diff --git a/python/requirements.txt b/python/requirements.txt index eb70dadc5f4e..2fbf538897d0 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -61,3 +61,4 @@ py-spy>=0.2.0; python_version < '3.12' py-spy>=0.4.0; python_version >= '3.12' memray; sys_platform != "win32" # memray is not supported on Windows pyOpenSSL +celery diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt index d4cc152a26c0..343a3f4a396b 100644 --- a/python/requirements_compiled.txt +++ b/python/requirements_compiled.txt @@ -79,6 +79,8 @@ alembic==1.12.1 # optuna altair==5.1.2 # via gradio +amqp==5.3.1 + # via kombu annotated-types==0.6.0 # via pydantic antlr4-python3-runtime==4.11.1 @@ -203,6 +205,8 @@ beautifulsoup4==4.11.1 # via # -r python/requirements/test-requirements.txt # nbconvert +billiard==4.2.1 + # via celery black==22.10.0 # via -r python/requirements/lint-requirements.txt bleach==6.1.0 @@ -249,6 +253,8 @@ cachetools==5.5.2 # google-auth # mlflow-skinny # pyiceberg +celery==5.5.3 + # via -r python/requirements.txt certifi==2025.1.31 # via # -r python/requirements/cloud-requirements.txt @@ -285,7 +291,11 @@ click==8.1.7 # -r python/requirements/cloud-requirements.txt # aim # black + # celery + # click-didyoumean # click-option-group + # click-plugins + # click-repl # dask # distributed # flask @@ -298,8 +308,14 @@ click==8.1.7 # typer # uvicorn # wandb +click-didyoumean==0.3.1 + # via celery click-option-group==0.5.6 # via semgrep +click-plugins==1.1.1.2 + # via celery +click-repl==0.3.0 + # via celery clickhouse-connect==0.8.10 # via -r python/requirements/ml/data-test-requirements.txt cloudpickle==2.2.0 ; python_version < "3.12" @@ -963,6 +979,8 @@ kiwisolver==1.4.5 # via matplotlib knack==0.11.0 # via azure-cli-core +kombu==5.5.4 + # via celery kubernetes==24.2.0 # via -r python/requirements/test-requirements.txt labmaze==1.0.6 @@ -1363,6 +1381,7 @@ packaging==23.0 # jupyterlab-server # jupytext # knack + # kombu # lazy-loader # lightning-utilities # matplotlib @@ -1484,7 +1503,9 @@ prometheus-client==0.19.0 promise==2.3 # via tensorflow-datasets prompt-toolkit==3.0.41 - # via ipython + # via + # click-repl + # ipython propcache==0.3.0 # via # aiohttp @@ -1735,6 +1756,7 @@ python-dateutil==2.8.2 # aim # arrow # botocore + # celery # freezegun # google-cloud-bigquery # graphene @@ -2381,6 +2403,8 @@ typing-extensions==4.12.2 # tensorflow # torch # typer +tzdata==2025.2 + # via kombu tzlocal==5.3 # via -r python/requirements/cloud-requirements.txt ujson==5.10.0 @@ -2435,6 +2459,11 @@ uvloop==0.21.0 # vmc-draas-client-bindings # vsphere-automation-sdk # via vsphere-automation-sdk +vine==5.1.0 + # via + # amqp + # celery + # kombu virtualenv==20.29.1 # via # -r python/requirements.txt diff --git a/python/requirements_compiled_ray_py311_cpu.txt b/python/requirements_compiled_ray_py311_cpu.txt index 5cac2c637eaf..3f38abd46278 100644 --- a/python/requirements_compiled_ray_py311_cpu.txt +++ b/python/requirements_compiled_ray_py311_cpu.txt @@ -113,6 +113,12 @@ aiosignal==1.3.1 \ # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # aiohttp +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -134,12 +140,24 @@ attrs==25.1.0 \ # aiohttp # jsonschema # referencing +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # celery cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # google-auth +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -302,8 +320,30 @@ click==8.1.7 \ # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -742,6 +782,12 @@ jsonschema-specifications==2024.10.1 \ # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # jsonschema +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # celery lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 @@ -1164,6 +1210,7 @@ packaging==23.0 \ # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # -r python/requirements.txt + # kombu # lazy-loader # scikit-image # tensorboardx @@ -1285,6 +1332,12 @@ prometheus-client==0.19.0 \ # -c python/requirements_compiled_ray_test_py311_cpu.txt # -r python/requirements.txt # opentelemetry-exporter-prometheus +prompt-toolkit==3.0.41 \ + --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ + --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ --hash=sha256:03acd9ff19021bd0567582ac88f821b66883e158274183b9e5586f678984f8fe \ @@ -1619,6 +1672,7 @@ python-dateutil==2.8.2 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via # -c python/requirements_compiled_ray_test_py311_cpu.txt + # celery # pandas pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ @@ -1937,6 +1991,12 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 @@ -1949,6 +2009,14 @@ uvicorn==0.22.0 \ # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # -r python/requirements.txt +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 @@ -1981,6 +2049,12 @@ watchfiles==0.19.0 \ # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # -r python/requirements.txt +wcwidth==0.2.13 \ + --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ + --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # prompt-toolkit yarl==1.18.3 \ --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ --hash=sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193 \ diff --git a/python/requirements_compiled_ray_py311_cu121.txt b/python/requirements_compiled_ray_py311_cu121.txt index f50394202b62..0b2fee2cae5c 100644 --- a/python/requirements_compiled_ray_py311_cu121.txt +++ b/python/requirements_compiled_ray_py311_cu121.txt @@ -113,6 +113,12 @@ aiosignal==1.3.1 \ # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # aiohttp +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -134,12 +140,24 @@ attrs==25.1.0 \ # aiohttp # jsonschema # referencing +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # celery cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # google-auth +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -302,8 +320,30 @@ click==8.1.7 \ # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -742,6 +782,12 @@ jsonschema-specifications==2024.10.1 \ # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # jsonschema +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # celery lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 @@ -1164,6 +1210,7 @@ packaging==23.0 \ # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # -r python/requirements.txt + # kombu # lazy-loader # scikit-image # tensorboardx @@ -1285,6 +1332,12 @@ prometheus-client==0.19.0 \ # -c python/requirements_compiled_ray_test_py311_cu121.txt # -r python/requirements.txt # opentelemetry-exporter-prometheus +prompt-toolkit==3.0.41 \ + --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ + --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ --hash=sha256:03acd9ff19021bd0567582ac88f821b66883e158274183b9e5586f678984f8fe \ @@ -1619,6 +1672,7 @@ python-dateutil==2.8.2 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via # -c python/requirements_compiled_ray_test_py311_cu121.txt + # celery # pandas pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ @@ -1937,6 +1991,12 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 @@ -1949,6 +2009,14 @@ uvicorn==0.22.0 \ # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # -r python/requirements.txt +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 @@ -1981,6 +2049,12 @@ watchfiles==0.19.0 \ # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # -r python/requirements.txt +wcwidth==0.2.13 \ + --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ + --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # prompt-toolkit yarl==1.18.3 \ --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ --hash=sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193 \ diff --git a/python/requirements_compiled_ray_py311_cu128.txt b/python/requirements_compiled_ray_py311_cu128.txt index 28c3d535db38..206ecd7e66d8 100644 --- a/python/requirements_compiled_ray_py311_cu128.txt +++ b/python/requirements_compiled_ray_py311_cu128.txt @@ -113,6 +113,12 @@ aiosignal==1.3.1 \ # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # aiohttp +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -134,12 +140,24 @@ attrs==25.1.0 \ # aiohttp # jsonschema # referencing +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # celery cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # google-auth +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -302,8 +320,30 @@ click==8.1.7 \ # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -742,6 +782,12 @@ jsonschema-specifications==2024.10.1 \ # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # jsonschema +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # celery lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 @@ -1128,6 +1174,7 @@ packaging==23.0 \ # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # -r python/requirements.txt + # kombu # lazy-loader # scikit-image # tensorboardx @@ -1249,6 +1296,12 @@ prometheus-client==0.19.0 \ # -c python/requirements_compiled_ray_test_py311_cu128.txt # -r python/requirements.txt # opentelemetry-exporter-prometheus +prompt-toolkit==3.0.41 \ + --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ + --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ --hash=sha256:03acd9ff19021bd0567582ac88f821b66883e158274183b9e5586f678984f8fe \ @@ -1583,6 +1636,7 @@ python-dateutil==2.8.2 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via # -c python/requirements_compiled_ray_test_py311_cu128.txt + # celery # pandas pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ @@ -1900,6 +1954,12 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 @@ -1912,6 +1972,14 @@ uvicorn==0.22.0 \ # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # -r python/requirements.txt +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 @@ -1944,6 +2012,12 @@ watchfiles==0.19.0 \ # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # -r python/requirements.txt +wcwidth==0.2.13 \ + --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ + --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # prompt-toolkit yarl==1.18.3 \ --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ --hash=sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193 \ diff --git a/python/requirements_compiled_ray_test_py311_cpu.txt b/python/requirements_compiled_ray_test_py311_cpu.txt index 989c6654e83d..81ea73f51d7f 100644 --- a/python/requirements_compiled_ray_test_py311_cpu.txt +++ b/python/requirements_compiled_ray_test_py311_cpu.txt @@ -127,6 +127,12 @@ aiosqlite==0.19.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # ypy-websocket +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -238,6 +244,12 @@ beautifulsoup4==4.11.1 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # nbconvert +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery bleach==6.1.0 \ --hash=sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe \ --hash=sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6 @@ -265,6 +277,12 @@ cachetools==5.5.2 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # google-auth +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -430,8 +448,30 @@ click==8.1.7 \ # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -1299,6 +1339,12 @@ jupyterlab-widgets==3.0.11 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # ipywidgets +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 @@ -1879,6 +1925,7 @@ packaging==23.0 \ # jupyter-server # jupyterlab # jupyterlab-server + # kombu # lazy-loader # nbconvert # pytest @@ -2053,6 +2100,7 @@ prompt-toolkit==3.0.41 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via # -c /tmp/ray-deps/requirements_compiled.txt + # click-repl # ipython propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -2503,6 +2551,7 @@ python-dateutil==2.8.2 \ # -r python/requirements/cloud-requirements.txt # arrow # botocore + # celery # jupyter-client # pandas python-json-logger==2.0.7 \ @@ -3105,6 +3154,12 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # kombu tzlocal==5.3 \ --hash=sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2 \ --hash=sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c @@ -3131,6 +3186,14 @@ uvicorn==0.22.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 diff --git a/python/requirements_compiled_ray_test_py311_cu121.txt b/python/requirements_compiled_ray_test_py311_cu121.txt index f9fb02769003..87cdc9200725 100644 --- a/python/requirements_compiled_ray_test_py311_cu121.txt +++ b/python/requirements_compiled_ray_test_py311_cu121.txt @@ -127,6 +127,12 @@ aiosqlite==0.19.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # ypy-websocket +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -238,6 +244,12 @@ beautifulsoup4==4.11.1 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # nbconvert +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery bleach==6.1.0 \ --hash=sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe \ --hash=sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6 @@ -265,6 +277,12 @@ cachetools==5.5.2 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # google-auth +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -430,8 +448,30 @@ click==8.1.7 \ # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -1299,6 +1339,12 @@ jupyterlab-widgets==3.0.11 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # ipywidgets +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 @@ -1879,6 +1925,7 @@ packaging==23.0 \ # jupyter-server # jupyterlab # jupyterlab-server + # kombu # lazy-loader # nbconvert # pytest @@ -2053,6 +2100,7 @@ prompt-toolkit==3.0.41 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via # -c /tmp/ray-deps/requirements_compiled.txt + # click-repl # ipython propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -2503,6 +2551,7 @@ python-dateutil==2.8.2 \ # -r python/requirements/cloud-requirements.txt # arrow # botocore + # celery # jupyter-client # pandas python-json-logger==2.0.7 \ @@ -3105,6 +3154,12 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # kombu tzlocal==5.3 \ --hash=sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2 \ --hash=sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c @@ -3131,6 +3186,14 @@ uvicorn==0.22.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 diff --git a/python/requirements_compiled_ray_test_py311_cu128.txt b/python/requirements_compiled_ray_test_py311_cu128.txt index 21697b75b978..44f0ef21f41c 100644 --- a/python/requirements_compiled_ray_test_py311_cu128.txt +++ b/python/requirements_compiled_ray_test_py311_cu128.txt @@ -127,6 +127,12 @@ aiosqlite==0.19.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # ypy-websocket +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -238,6 +244,12 @@ beautifulsoup4==4.11.1 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # nbconvert +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery bleach==6.1.0 \ --hash=sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe \ --hash=sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6 @@ -265,6 +277,12 @@ cachetools==5.5.2 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # google-auth +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -430,8 +448,30 @@ click==8.1.7 \ # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -1299,6 +1339,12 @@ jupyterlab-widgets==3.0.11 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # ipywidgets +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # celery lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 @@ -1879,6 +1925,7 @@ packaging==23.0 \ # jupyter-server # jupyterlab # jupyterlab-server + # kombu # lazy-loader # nbconvert # pytest @@ -2053,6 +2100,7 @@ prompt-toolkit==3.0.41 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via # -c /tmp/ray-deps/requirements_compiled.txt + # click-repl # ipython propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -2503,6 +2551,7 @@ python-dateutil==2.8.2 \ # -r python/requirements/cloud-requirements.txt # arrow # botocore + # celery # jupyter-client # pandas python-json-logger==2.0.7 \ @@ -3105,6 +3154,12 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # kombu tzlocal==5.3 \ --hash=sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2 \ --hash=sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c @@ -3131,6 +3186,14 @@ uvicorn==0.22.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 diff --git a/python/requirements_compiled_rayllm_py311_cpu.txt b/python/requirements_compiled_rayllm_py311_cpu.txt index f98d7fb01c4c..3d4262a623c4 100644 --- a/python/requirements_compiled_rayllm_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_py311_cpu.txt @@ -114,6 +114,12 @@ aiosignal==1.3.1 \ # via # -c python/requirements_compiled_rayllm_test_py311_cpu.txt # aiohttp +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -143,6 +149,12 @@ attrs==25.1.0 \ # aiohttp # jsonschema # referencing +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # celery blake3==1.0.4 \ --hash=sha256:00605aa59923205c6a4f21131840840eb2d9a754c59b163357d890566755b97a \ --hash=sha256:08f46c2f1c5f369f07409e3e4ff248bcb22617cd741f2224873d85982dd6034e \ @@ -287,6 +299,12 @@ cbor2==5.6.5 \ # via # -c python/requirements_compiled_rayllm_test_py311_cpu.txt # vllm +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -452,9 +470,31 @@ click==8.1.7 \ # via # -c python/requirements_compiled_rayllm_test_py311_cpu.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # ray # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -1172,6 +1212,12 @@ jsonschema-specifications==2024.10.1 \ # via # -c python/requirements_compiled_rayllm_test_py311_cpu.txt # jsonschema +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ --hash=sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80 @@ -1834,6 +1880,7 @@ packaging==23.0 \ # -c python/requirements_compiled_rayllm_test_py311_cpu.txt # -r python/requirements.txt # huggingface-hub + # kombu # lazy-loader # lm-format-enforcer # ray @@ -1975,6 +2022,12 @@ prometheus-fastapi-instrumentator==7.0.2 \ # via # -c python/requirements_compiled_rayllm_test_py311_cpu.txt # vllm +prompt-toolkit==3.0.41 \ + --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ + --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 + # via + # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ --hash=sha256:03acd9ff19021bd0567582ac88f821b66883e158274183b9e5586f678984f8fe \ @@ -2509,6 +2562,7 @@ python-dateutil==2.8.2 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # celery # pandas python-dotenv==1.0.1 \ --hash=sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca \ @@ -3334,6 +3388,12 @@ typing-extensions==4.12.2 \ # torch # typer # vllm +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 @@ -3389,6 +3449,14 @@ uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'c # via # -c python/requirements_compiled_rayllm_test_py311_cpu.txt # uvicorn +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 @@ -3429,6 +3497,12 @@ watchfiles==0.19.0 \ # -r python/requirements.txt # uvicorn # vllm +wcwidth==0.2.13 \ + --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ + --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 + # via + # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # prompt-toolkit websockets==15.0 \ --hash=sha256:0e389efe46ccb25a1f93d08c7a74e8123a2517f7b7458f043bd7529d1a63ffeb \ --hash=sha256:0f2205cdb444a42a7919690238fb5979a05439b9dbb73dd47c863d39640d85ab \ diff --git a/python/requirements_compiled_rayllm_py311_cu121.txt b/python/requirements_compiled_rayllm_py311_cu121.txt index b28613cf00e4..3e6dad764e0e 100644 --- a/python/requirements_compiled_rayllm_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_py311_cu121.txt @@ -114,6 +114,12 @@ aiosignal==1.3.1 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu121.txt # aiohttp +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -143,6 +149,12 @@ attrs==25.1.0 \ # aiohttp # jsonschema # referencing +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # celery blake3==1.0.4 \ --hash=sha256:00605aa59923205c6a4f21131840840eb2d9a754c59b163357d890566755b97a \ --hash=sha256:08f46c2f1c5f369f07409e3e4ff248bcb22617cd741f2224873d85982dd6034e \ @@ -287,6 +299,12 @@ cbor2==5.6.5 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu121.txt # vllm +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -452,9 +470,31 @@ click==8.1.7 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu121.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # ray # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -1172,6 +1212,12 @@ jsonschema-specifications==2024.10.1 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu121.txt # jsonschema +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ --hash=sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80 @@ -1950,6 +1996,7 @@ packaging==23.0 \ # -c python/requirements_compiled_rayllm_test_py311_cu121.txt # -r python/requirements.txt # huggingface-hub + # kombu # lazy-loader # lm-format-enforcer # ray @@ -2091,6 +2138,12 @@ prometheus-fastapi-instrumentator==7.0.2 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu121.txt # vllm +prompt-toolkit==3.0.41 \ + --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ + --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ --hash=sha256:03acd9ff19021bd0567582ac88f821b66883e158274183b9e5586f678984f8fe \ @@ -2625,6 +2678,7 @@ python-dateutil==2.8.2 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # celery # pandas python-dotenv==1.0.1 \ --hash=sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca \ @@ -3469,6 +3523,12 @@ typing-extensions==4.12.2 \ # torch # typer # vllm +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 @@ -3524,6 +3584,14 @@ uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'c # via # -c python/requirements_compiled_rayllm_test_py311_cu121.txt # uvicorn +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 @@ -3564,6 +3632,12 @@ watchfiles==0.19.0 \ # -r python/requirements.txt # uvicorn # vllm +wcwidth==0.2.13 \ + --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ + --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # prompt-toolkit websockets==15.0 \ --hash=sha256:0e389efe46ccb25a1f93d08c7a74e8123a2517f7b7458f043bd7529d1a63ffeb \ --hash=sha256:0f2205cdb444a42a7919690238fb5979a05439b9dbb73dd47c863d39640d85ab \ diff --git a/python/requirements_compiled_rayllm_py311_cu128.txt b/python/requirements_compiled_rayllm_py311_cu128.txt index a944365fb566..05be84969dbb 100644 --- a/python/requirements_compiled_rayllm_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_py311_cu128.txt @@ -114,6 +114,12 @@ aiosignal==1.3.1 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu128.txt # aiohttp +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -143,6 +149,12 @@ attrs==25.1.0 \ # aiohttp # jsonschema # referencing +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # celery blake3==1.0.5 \ --hash=sha256:03638a6dc8546365c3576fdb293fb2c53b898ac80525b5742d9cf00b4f44dea5 \ --hash=sha256:043a226cebfedff7b51ab9c87d4476c06d2cd10776855eaa9c619f2272b3c32e \ @@ -287,6 +299,12 @@ cbor2==5.6.5 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu128.txt # vllm +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -452,9 +470,31 @@ click==8.1.7 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu128.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # ray # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -1173,6 +1213,12 @@ jsonschema-specifications==2024.10.1 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu128.txt # jsonschema +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ --hash=sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80 @@ -1874,6 +1920,7 @@ packaging==23.0 \ # -c python/requirements_compiled_rayllm_test_py311_cu128.txt # -r python/requirements.txt # huggingface-hub + # kombu # lazy-loader # lm-format-enforcer # ray @@ -2015,6 +2062,12 @@ prometheus-fastapi-instrumentator==7.1.0 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu128.txt # vllm +prompt-toolkit==3.0.41 \ + --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ + --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ --hash=sha256:03acd9ff19021bd0567582ac88f821b66883e158274183b9e5586f678984f8fe \ @@ -2549,6 +2602,7 @@ python-dateutil==2.8.2 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # celery # pandas python-dotenv==1.1.0 \ --hash=sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5 \ @@ -3362,6 +3416,12 @@ typing-extensions==4.12.2 \ # torch # typer # vllm +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 @@ -3417,6 +3477,14 @@ uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'c # via # -c python/requirements_compiled_rayllm_test_py311_cu128.txt # uvicorn +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 @@ -3457,6 +3525,12 @@ watchfiles==0.19.0 \ # -r python/requirements.txt # uvicorn # vllm +wcwidth==0.2.13 \ + --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ + --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 + # via + # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # prompt-toolkit websockets==15.0.1 \ --hash=sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2 \ --hash=sha256:0a34631031a8f05657e8e90903e656959234f3a04552259458aac0b0f9ae6fd9 \ diff --git a/python/requirements_compiled_rayllm_test_py311_cpu.txt b/python/requirements_compiled_rayllm_test_py311_cpu.txt index e96f530c2310..f3933c7fedad 100644 --- a/python/requirements_compiled_rayllm_test_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_test_py311_cpu.txt @@ -133,6 +133,12 @@ alabaster==0.7.16 \ --hash=sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65 \ --hash=sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92 # via sphinx +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -255,6 +261,12 @@ beautifulsoup4==4.11.1 \ # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # nbconvert +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # celery blake3==1.0.4 \ --hash=sha256:00605aa59923205c6a4f21131840840eb2d9a754c59b163357d890566755b97a \ --hash=sha256:08f46c2f1c5f369f07409e3e4ff248bcb22617cd741f2224873d85982dd6034e \ @@ -416,6 +428,12 @@ cbor2==5.6.5 \ --hash=sha256:fde21ac1cf29336a31615a2c469a9cb03cf0add3ae480672d4d38cda467d07fc \ --hash=sha256:fe11c2eb518c882cfbeed456e7a552e544893c17db66fe5d3230dbeaca6b615c # via vllm +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -584,9 +602,31 @@ click==8.1.7 \ # -c python/requirements_compiled_ray_test_py311_cpu.txt # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # ray # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -1712,6 +1752,12 @@ jupytext==1.16.7 \ --hash=sha256:912f9d9af7bd3f15470105e5c5dddf1669b2d8c17f0c55772687fc5a4a73fe69 \ --hash=sha256:fc4e97f0890e22062c4ef10313c7ca960b07b3767246a1fef7585888cc2afe5d # via -r python/requirements/llm/llm-test-requirements.txt +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ --hash=sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80 @@ -2554,6 +2600,7 @@ packaging==23.0 \ # jupyterlab # jupyterlab-server # jupytext + # kombu # lazy-loader # lm-format-enforcer # nbconvert @@ -2746,6 +2793,7 @@ prompt-toolkit==3.0.41 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via # -c python/requirements_compiled_ray_test_py311_cpu.txt + # click-repl # ipython propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -3373,6 +3421,7 @@ python-dateutil==2.8.2 \ # -r python/requirements/cloud-requirements.txt # arrow # botocore + # celery # jupyter-client # pandas python-dotenv==1.0.1 \ @@ -4392,6 +4441,12 @@ typing-extensions==4.12.2 \ # torch # typer # vllm +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # kombu tzlocal==5.3 \ --hash=sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2 \ --hash=sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c @@ -4459,6 +4514,14 @@ uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'c --hash=sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816 \ --hash=sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2 # via uvicorn +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled_ray_test_py311_cpu.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 diff --git a/python/requirements_compiled_rayllm_test_py311_cu121.txt b/python/requirements_compiled_rayllm_test_py311_cu121.txt index b7987154666c..164a9b50f775 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu121.txt @@ -133,6 +133,12 @@ alabaster==0.7.16 \ --hash=sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65 \ --hash=sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92 # via sphinx +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -255,6 +261,12 @@ beautifulsoup4==4.11.1 \ # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # nbconvert +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # celery blake3==1.0.4 \ --hash=sha256:00605aa59923205c6a4f21131840840eb2d9a754c59b163357d890566755b97a \ --hash=sha256:08f46c2f1c5f369f07409e3e4ff248bcb22617cd741f2224873d85982dd6034e \ @@ -416,6 +428,12 @@ cbor2==5.6.5 \ --hash=sha256:fde21ac1cf29336a31615a2c469a9cb03cf0add3ae480672d4d38cda467d07fc \ --hash=sha256:fe11c2eb518c882cfbeed456e7a552e544893c17db66fe5d3230dbeaca6b615c # via vllm +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -584,9 +602,31 @@ click==8.1.7 \ # -c python/requirements_compiled_ray_test_py311_cu121.txt # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # ray # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -1712,6 +1752,12 @@ jupytext==1.16.7 \ --hash=sha256:912f9d9af7bd3f15470105e5c5dddf1669b2d8c17f0c55772687fc5a4a73fe69 \ --hash=sha256:fc4e97f0890e22062c4ef10313c7ca960b07b3767246a1fef7585888cc2afe5d # via -r python/requirements/llm/llm-test-requirements.txt +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ --hash=sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80 @@ -2645,6 +2691,7 @@ packaging==23.0 \ # jupyterlab # jupyterlab-server # jupytext + # kombu # lazy-loader # lm-format-enforcer # nbconvert @@ -2837,6 +2884,7 @@ prompt-toolkit==3.0.41 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via # -c python/requirements_compiled_ray_test_py311_cu121.txt + # click-repl # ipython propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -3464,6 +3512,7 @@ python-dateutil==2.8.2 \ # -r python/requirements/cloud-requirements.txt # arrow # botocore + # celery # jupyter-client # pandas python-dotenv==1.0.1 \ @@ -4503,6 +4552,12 @@ typing-extensions==4.12.2 \ # torch # typer # vllm +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # kombu tzlocal==5.3 \ --hash=sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2 \ --hash=sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c @@ -4570,6 +4625,14 @@ uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'c --hash=sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816 \ --hash=sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2 # via uvicorn +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled_ray_test_py311_cu121.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 diff --git a/python/requirements_compiled_rayllm_test_py311_cu128.txt b/python/requirements_compiled_rayllm_test_py311_cu128.txt index b50bdfd3c41b..e94cd04041cb 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu128.txt @@ -133,6 +133,12 @@ alabaster==0.7.16 \ --hash=sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65 \ --hash=sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92 # via sphinx +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d @@ -255,6 +261,12 @@ beautifulsoup4==4.11.1 \ # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # nbconvert +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # celery blake3==1.0.5 \ --hash=sha256:03638a6dc8546365c3576fdb293fb2c53b898ac80525b5742d9cf00b4f44dea5 \ --hash=sha256:043a226cebfedff7b51ab9c87d4476c06d2cd10776855eaa9c619f2272b3c32e \ @@ -416,6 +428,12 @@ cbor2==5.6.5 \ --hash=sha256:fde21ac1cf29336a31615a2c469a9cb03cf0add3ae480672d4d38cda467d07fc \ --hash=sha256:fe11c2eb518c882cfbeed456e7a552e544893c17db66fe5d3230dbeaca6b615c # via vllm +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe @@ -584,9 +602,31 @@ click==8.1.7 \ # -c python/requirements_compiled_ray_test_py311_cu128.txt # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # ray # typer # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 @@ -1712,6 +1752,12 @@ jupytext==1.17.2 \ --hash=sha256:4f85dc43bb6a24b75491c5c434001ad5ef563932f68f15dd3e1c8ce12a4a426b \ --hash=sha256:772d92898ac1f2ded69106f897b34af48ce4a85c985fa043a378ff5a65455f02 # via -r python/requirements/llm/llm-test-requirements.txt +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ --hash=sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80 @@ -2568,6 +2614,7 @@ packaging==23.0 \ # jupyterlab # jupyterlab-server # jupytext + # kombu # lazy-loader # lm-format-enforcer # nbconvert @@ -2760,6 +2807,7 @@ prompt-toolkit==3.0.41 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via # -c python/requirements_compiled_ray_test_py311_cu128.txt + # click-repl # ipython propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -3387,6 +3435,7 @@ python-dateutil==2.8.2 \ # -r python/requirements/cloud-requirements.txt # arrow # botocore + # celery # jupyter-client # pandas python-dotenv==1.1.0 \ @@ -4395,6 +4444,12 @@ typing-extensions==4.12.2 \ # torch # typer # vllm +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # kombu tzlocal==5.3 \ --hash=sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2 \ --hash=sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c @@ -4462,6 +4517,14 @@ uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'c --hash=sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816 \ --hash=sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2 # via uvicorn +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled_ray_test_py311_cu128.txt + # amqp + # celery + # kombu virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 diff --git a/python/setup.py b/python/setup.py index 71bd4a54a810..c0eb418d688c 100644 --- a/python/setup.py +++ b/python/setup.py @@ -299,6 +299,17 @@ def get_packages(self): ) ) + # This is required for supporting the asynchronous inference, allowing the ray serve applications to + # allow asynchronously execute their code, via the use of celery task processor. + setup_spec.extras["serve-async-inference"] = list( + set( + setup_spec.extras["serve"] + + [ + "celery", + ] + ) + ) + if RAY_EXTRA_CPP: setup_spec.extras["cpp"] = ["ray-cpp==" + setup_spec.version] From e4410d09cd0de2a7b2e6e507c12b92d2741cd6ea Mon Sep 17 00:00:00 2001 From: Nikhil G Date: Wed, 13 Aug 2025 22:52:11 -0700 Subject: [PATCH 099/634] [serve.llm] fix: improve error handling for invalid model_id (#55589) Signed-off-by: Nikhil Ghosh --- .../_internal/serve/configs/server_models.py | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/python/ray/llm/_internal/serve/configs/server_models.py b/python/ray/llm/_internal/serve/configs/server_models.py index d7f3b27a1944..f4d42174ef4e 100644 --- a/python/ray/llm/_internal/serve/configs/server_models.py +++ b/python/ray/llm/_internal/serve/configs/server_models.py @@ -223,8 +223,16 @@ def _infer_supports_vision(self, model_id_or_path: str) -> None: attribute based on whether the config has `vision_config`. All LVM models has `vision_config` setup. """ - hf_config = transformers.PretrainedConfig.from_pretrained(model_id_or_path) - self._supports_vision = hasattr(hf_config, "vision_config") + try: + hf_config = transformers.PretrainedConfig.from_pretrained(model_id_or_path) + self._supports_vision = hasattr(hf_config, "vision_config") + except Exception as e: + raise ValueError( + f"Failed to load Hugging Face config for model_id='{model_id_or_path}'.\ + Ensure `model_id` is a valid Hugging Face repo or a local path that \ + contains a valid `config.json` file. " + f"Original error: {repr(e)}" + ) from e def _set_model_architecture( self, @@ -236,9 +244,23 @@ def _set_model_architecture( attribute based on whether the config has `architectures`. """ if model_id_or_path: - hf_config = transformers.PretrainedConfig.from_pretrained(model_id_or_path) - if hasattr(hf_config, "architectures") and hf_config.architectures: - self._model_architecture = hf_config.architectures[0] + try: + hf_config = transformers.PretrainedConfig.from_pretrained( + model_id_or_path + ) + if ( + hf_config + and hasattr(hf_config, "architectures") + and hf_config.architectures + ): + self._model_architecture = hf_config.architectures[0] + except Exception as e: + raise ValueError( + f"Failed to load Hugging Face config for model_id='{model_id_or_path}'.\ + Ensure `model_id` is a valid Hugging Face repo or a local path that \ + contains a valid `config.json` file. " + f"Original error: {repr(e)}" + ) from e if model_architecture: self._model_architecture = model_architecture From 4b6dba34d50d647a7929b1e9079954511a69c759 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Thu, 14 Aug 2025 00:59:20 -0700 Subject: [PATCH 100/634] [ci] fix incorrect ml-baseextra depends_on (#55596) to depends on the right wanda job Signed-off-by: Lonnie Liu --- .buildkite/_forge.rayci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml index 667d64c36cc8..3681e39d9b43 100644 --- a/.buildkite/_forge.rayci.yml +++ b/.buildkite/_forge.rayci.yml @@ -150,7 +150,7 @@ steps: PYTHON_VERSION: "{{matrix.python}}" CUDA_VERSION: "{{matrix.cuda}}" IMAGE_TYPE: "ray-ml" - depends_on: raycudabaseextra + depends_on: ray-mlcudabase - name: ray-mlcpubase label: "wanda: ray-ml.py{{matrix}}.cpu.base" @@ -176,4 +176,4 @@ steps: env: PYTHON_VERSION: "{{matrix}}" IMAGE_TYPE: "ray-ml" - depends_on: raycpubaseextra + depends_on: ray-mlcpubase From 6afaeda7dc7eb700076ae98b5b356568a293cde2 Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Thu, 14 Aug 2025 17:08:01 +0200 Subject: [PATCH 101/634] [RLlib] Add docs for Implicit Q-Learning. (#55422) --- doc/source/rllib/rllib-algorithms.rst | 32 +++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/doc/source/rllib/rllib-algorithms.rst b/doc/source/rllib/rllib-algorithms.rst index 41c109d5962b..1fdd788f4c7b 100644 --- a/doc/source/rllib/rllib-algorithms.rst +++ b/doc/source/rllib/rllib-algorithms.rst @@ -39,6 +39,10 @@ as well as multi-GPU training on multi-node (GPU) clusters when using the `Anysc +-----------------------------------------------------------------------------+------------------------------+------------------------------------+--------------------------------+ | :ref:`BC (Behavior Cloning) ` | |single_agent| | |multi_gpu| |multi_node_multi_gpu| | |cont_actions| |discr_actions| | +-----------------------------------------------------------------------------+------------------------------+------------------------------------+--------------------------------+ +| :ref:`CQL (Conservative Q-Learning) ` | |single_agent| | |multi_gpu| |multi_node_multi_gpu| | |cont_actions| | ++-----------------------------------------------------------------------------+------------------------------+------------------------------------+--------------------------------+ +| :ref:`IQL (Implicit Q-Learning) ` | |single_agent| | |multi_gpu| |multi_node_multi_gpu| | |cont_actions| | ++-----------------------------------------------------------------------------+------------------------------+------------------------------------+--------------------------------+ | :ref:`MARWIL (Monotonic Advantage Re-Weighted Imitation Learning) ` | |single_agent| | |multi_gpu| |multi_node_multi_gpu| | |cont_actions| |discr_actions| | +-----------------------------------------------------------------------------+------------------------------+------------------------------------+--------------------------------+ | **Algorithm Extensions and -Plugins** | @@ -183,7 +187,7 @@ Asynchronous Proximal Policy Optimization (APPO) In a training iteration, APPO requests samples from all EnvRunners asynchronously and the collected episode samples are returned to the main algorithm process as Ray references rather than actual objects available on the local process. APPO then passes these episode references to the Learners for asynchronous updates of the model. - RLlib doesn't always synch back the weights to the EnvRunners right after a new model version is available. + RLlib doesn't always sync back the weights to the EnvRunners right after a new model version is available. To account for the EnvRunners being off-policy, APPO uses a procedure called v-trace, `described in the IMPALA paper `__. APPO scales out on both axes, supporting multiple EnvRunners for sample collection and multiple GPU- or CPU-based Learners @@ -363,6 +367,30 @@ Conservative Q-Learning (CQL) :members: training +.. _iql: + +Implicit Q-Learning (IQL) +------------------------- +`[paper] `__ +`[implementation] `__ + + **IQL architecture:** IQL (Implicit Q-Learning) is an offline RL algorithm that never needs to evaluate actions outside of + the dataset, but still enables the learned policy to improve substantially over the best behavior in the data through + generalization. Instead of standard TD-error minimization, it introduces a value function trained through expectile regression, + which yields a conservative estimate of returns. This allows policy improvement through advantage-weighted behavior cloning, + ensuring safer generalization without explicit exploration. + + The `IQLLearner` replaces the usual TD-based value loss with an expectile regression loss, and trains the policy to imitate + high-advantage actions—enabling substantial performance gains over the behavior policy using only in-dataset actions. + +**Tuned examples:** +`Pendulum-v1 `__ + +**IQL-specific configs** and :ref:`generic algorithm settings `): + +.. autoclass:: ray.rllib.algorithms.iql.iql.IQLConfig + :members: training + .. _marwil: Monotonic Advantage Re-Weighted Imitation Learning (MARWIL) @@ -376,7 +404,7 @@ Monotonic Advantage Re-Weighted Imitation Learning (MARWIL) **MARWIL architecture:** MARWIL is a hybrid imitation learning and policy gradient algorithm suitable for training on batched historical data. When the ``beta`` hyperparameter is set to zero, the MARWIL objective reduces to plain - imitation learning (see `BC`_). MARWIL uses Ray.Data to tap into its parallel data + imitation learning (see `BC`_). MARWIL uses Ray. Data to tap into its parallel data processing capabilities. In one training iteration, MARWIL reads episodes in parallel from offline files, for example `parquet `__, by the n DataWorkers. Connector pipelines preprocess these episodes into train batches and send these as data iterators directly to the n Learners for updating the model. From 7518fd8be262c5f1bdc8246e0a3c5cc7db5d1bd6 Mon Sep 17 00:00:00 2001 From: Jun-Hao Wan Date: Fri, 15 Aug 2025 00:09:47 +0800 Subject: [PATCH 102/634] [Doc][KubeRay] Add InteractiveMode description for `ray-job-quick-start.md` (#55570) Signed-off-by: win5923 Signed-off-by: Kai-Hsun Chen Co-authored-by: Kai-Hsun Chen --- .../kubernetes/getting-started/rayjob-quick-start.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/source/cluster/kubernetes/getting-started/rayjob-quick-start.md b/doc/source/cluster/kubernetes/getting-started/rayjob-quick-start.md index b7102c0852b4..f70a700aa333 100644 --- a/doc/source/cluster/kubernetes/getting-started/rayjob-quick-start.md +++ b/doc/source/cluster/kubernetes/getting-started/rayjob-quick-start.md @@ -49,8 +49,11 @@ To understand the following content better, you should understand the difference * `metadata` (Optional): See {ref}`Ray Jobs CLI API Reference ` for more details about the `--metadata-json` option. * `entrypointNumCpus` / `entrypointNumGpus` / `entrypointResources` (Optional): See {ref}`Ray Jobs CLI API Reference ` for more details. * `backoffLimit` (Optional, added in version 1.2.0): Specifies the number of retries before marking this RayJob failed. Each retry creates a new RayCluster. The default value is 0. -* Submission configuration - * `submissionMode` (Optional): `submissionMode` specifies how RayJob submits the Ray job to the RayCluster. In "K8sJobMode", the KubeRay operator creates a submitter Kubernetes Job to submit the Ray job. In "HTTPMode", the KubeRay operator sends a request to the RayCluster to create a Ray job. The default value is "K8sJobMode". +* Submission configuration + * `submissionMode` (Optional): Specifies how RayJob submits the Ray job to the RayCluster. There are three possible values, with the default being `K8sJobMode`. + * `K8sJobMode`: The KubeRay operator creates a submitter Kubernetes Job to submit the Ray job. + * `HTTPMode`: The KubeRay operator sends a request to the RayCluster to create a Ray job. + * `InteractiveMode`: The KubeRay operator waits for the user to submit a job to the RayCluster. This mode is currently in alpha and the [KubeRay kubectl plugin](kubectl-plugin) relies on it. * `submitterPodTemplate` (Optional): Defines the Pod template for the submitter Kubernetes Job. This field is only effective when `submissionMode` is "K8sJobMode". * `RAY_DASHBOARD_ADDRESS` - The KubeRay operator injects this environment variable to the submitter Pod. The value is `$HEAD_SERVICE:$DASHBOARD_PORT`. * `RAY_JOB_SUBMISSION_ID` - The KubeRay operator injects this environment variable to the submitter Pod. The value is the `RayJob.Status.JobId` of the RayJob. @@ -201,4 +204,4 @@ kind delete cluster * [RayJob Batch Inference Example](kuberay-batch-inference-example) * [Priority Scheduling with RayJob and Kueue](kuberay-kueue-priority-scheduling-example) -* [Gang Scheduling with RayJob and Kueue](kuberay-kueue-gang-scheduling-example) \ No newline at end of file +* [Gang Scheduling with RayJob and Kueue](kuberay-kueue-gang-scheduling-example) From ea27578265182b3b721b0b6b5a9f2d6a49e6e61b Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Thu, 14 Aug 2025 10:01:25 -0700 Subject: [PATCH 103/634] [ci] remove unused `use_base_extra` (#55604) added incorrectly in a past change Signed-off-by: Lonnie Liu --- ci/ray_ci/docker_container.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/ray_ci/docker_container.py b/ci/ray_ci/docker_container.py index 8a83e671e4b7..73a6f3d2879c 100644 --- a/ci/ray_ci/docker_container.py +++ b/ci/ray_ci/docker_container.py @@ -47,7 +47,6 @@ def __init__( architecture: str = DEFAULT_ARCHITECTURE, canonical_tag: str = None, upload: bool = False, - use_base_extra: bool = False, ) -> None: assert "RAYCI_CHECKOUT_DIR" in os.environ, "RAYCI_CHECKOUT_DIR not set" From f0b0aadd65b3a842ed42ef870ac3067ea42f30af Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Thu, 14 Aug 2025 10:01:39 -0700 Subject: [PATCH 104/634] [image] add base-extra for aarch64 images (#55586) for easier use on ray cluster hosters like anyscale. Signed-off-by: Lonnie Liu --- .buildkite/_forge.rayci.yml | 7 +++ .buildkite/linux_aarch64.rayci.yml | 71 +++++++++++++++++----- ci/docker/ray.cpu.base-extra.wanda.yaml | 8 +-- ci/docker/ray.cpu.base.aarch64.wanda.yaml | 10 --- ci/docker/ray.cpu.base.wanda.yaml | 4 +- ci/docker/ray.cuda.base-extra.wanda.yaml | 8 +-- ci/docker/ray.cuda.base.aarch64.wanda.yaml | 10 --- ci/docker/ray.cuda.base.wanda.yaml | 4 +- 8 files changed, 75 insertions(+), 47 deletions(-) delete mode 100644 ci/docker/ray.cpu.base.aarch64.wanda.yaml delete mode 100644 ci/docker/ray.cuda.base.aarch64.wanda.yaml diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml index 3681e39d9b43..e74f90edf048 100644 --- a/.buildkite/_forge.rayci.yml +++ b/.buildkite/_forge.rayci.yml @@ -19,6 +19,7 @@ steps: - "3.12" env: PYTHON_VERSION: "{{matrix}}" + ARCH_SUFFIX: "" - name: raycpubaseextra label: "wanda: ray.py{{matrix}}.cpu.base-extra" @@ -31,6 +32,7 @@ steps: env: PYTHON_VERSION: "{{matrix}}" IMAGE_TYPE: "ray" + ARCH_SUFFIX: "" depends_on: raycpubase - name: raycudabase @@ -58,6 +60,7 @@ steps: env: PYTHON_VERSION: "{{matrix.python}}" CUDA_VERSION: "{{matrix.cuda}}" + ARCH_SUFFIX: "" - name: raycudabaseextra label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" @@ -82,6 +85,7 @@ steps: PYTHON_VERSION: "{{matrix.python}}" CUDA_VERSION: "{{matrix.cuda}}" IMAGE_TYPE: "ray" + ARCH_SUFFIX: "" depends_on: raycudabase - name: ray-llmbase @@ -114,6 +118,7 @@ steps: PYTHON_VERSION: "{{matrix.python}}" CUDA_VERSION: "{{matrix.cuda}}" IMAGE_TYPE: "ray-llm" + ARCH_SUFFIX: "" depends_on: ray-llmbase - name: ray-mlcudabase @@ -150,6 +155,7 @@ steps: PYTHON_VERSION: "{{matrix.python}}" CUDA_VERSION: "{{matrix.cuda}}" IMAGE_TYPE: "ray-ml" + ARCH_SUFFIX: "" depends_on: ray-mlcudabase - name: ray-mlcpubase @@ -176,4 +182,5 @@ steps: env: PYTHON_VERSION: "{{matrix}}" IMAGE_TYPE: "ray-ml" + ARCH_SUFFIX: "" depends_on: ray-mlcpubase diff --git a/.buildkite/linux_aarch64.rayci.yml b/.buildkite/linux_aarch64.rayci.yml index 0ba034ec1064..91bab540ab12 100644 --- a/.buildkite/linux_aarch64.rayci.yml +++ b/.buildkite/linux_aarch64.rayci.yml @@ -13,13 +13,43 @@ steps: wanda: ci/docker/manylinux.aarch64.wanda.yaml instance_type: builder-arm64 + - name: raycpubase-aarch64 + label: "wanda: ray.py{{matrix}}.cpu.base (aarch64)" + tags: + - python_dependencies + - docker + wanda: ci/docker/ray.cpu.base.wanda.yaml + matrix: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + instance_type: builder-arm64 + env: + PYTHON_VERSION: "{{matrix}}" + ARCH_SUFFIX: "-aarch64" + + - name: raycpubaseextra-aarch64 + label: "wanda: ray.py{{matrix}}.cpu.base-extra (aarch64)" + wanda: ci/docker/ray.cpu.base-extra.wanda.yaml + matrix: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + instance_type: builder-arm64 + env: + PYTHON_VERSION: "{{matrix}}" + IMAGE_TYPE: "ray" + ARCH_SUFFIX: "-aarch64" + depends_on: raycpubase-aarch64 + - name: raycudabase-aarch64 label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base (aarch64)" tags: - python_dependencies - docker - - core_cpp - wanda: ci/docker/ray.cuda.base.aarch64.wanda.yaml + wanda: ci/docker/ray.cuda.base.wanda.yaml matrix: setup: python: @@ -40,22 +70,34 @@ steps: env: PYTHON_VERSION: "{{matrix.python}}" CUDA_VERSION: "{{matrix.cuda}}" + ARCH_SUFFIX: "-aarch64" - - name: raycpubase-aarch64 - label: "wanda: ray.py{{matrix}}.cpu.base (aarch64)" - tags: - - python_dependencies - - docker - - core_cpp - wanda: ci/docker/ray.cpu.base.aarch64.wanda.yaml + - name: raycudabaseextra-aarch64 + label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra (aarch64)" + wanda: ci/docker/ray.cuda.base-extra.wanda.yaml matrix: - - "3.9" - - "3.10" - - "3.11" - - "3.12" + setup: + python: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + cuda: + - "11.7.1-cudnn8" + - "11.8.0-cudnn8" + - "12.1.1-cudnn8" + - "12.3.2-cudnn9" + - "12.4.1-cudnn" + - "12.5.1-cudnn" + - "12.6.3-cudnn" + - "12.8.1-cudnn" instance_type: builder-arm64 env: - PYTHON_VERSION: "{{matrix}}" + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + IMAGE_TYPE: "ray" + ARCH_SUFFIX: "-aarch64" + depends_on: raycudabase-aarch64 - label: ":tapioca: build: wheel {{matrix}} (aarch64)" tags: @@ -81,7 +123,6 @@ steps: tags: - python_dependencies - docker - - core_cpp - oss instance_type: medium-arm64 commands: diff --git a/ci/docker/ray.cpu.base-extra.wanda.yaml b/ci/docker/ray.cpu.base-extra.wanda.yaml index 0792fba51ec9..2a73f4bc7192 100644 --- a/ci/docker/ray.cpu.base-extra.wanda.yaml +++ b/ci/docker/ray.cpu.base-extra.wanda.yaml @@ -1,7 +1,7 @@ -name: "$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra" -froms: ["cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base"] +name: "$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra$ARCH_SUFFIX" +froms: ["cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX"] dockerfile: docker/base-extra/Dockerfile build_args: - - BASE_IMAGE=cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base + - BASE_IMAGE=cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX tags: - - cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra + - cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra$ARCH_SUFFIX diff --git a/ci/docker/ray.cpu.base.aarch64.wanda.yaml b/ci/docker/ray.cpu.base.aarch64.wanda.yaml deleted file mode 100644 index 70d81359ad31..000000000000 --- a/ci/docker/ray.cpu.base.aarch64.wanda.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: "ray-py$PYTHON_VERSION-cpu-base-aarch64" -froms: ["ubuntu:22.04"] -dockerfile: docker/base-deps/Dockerfile -srcs: - - python/requirements_compiled.txt -build_args: - - PYTHON_VERSION - - BASE_IMAGE=ubuntu:22.04 -tags: - - cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base-aarch64 diff --git a/ci/docker/ray.cpu.base.wanda.yaml b/ci/docker/ray.cpu.base.wanda.yaml index 895605ed8f71..ecb8f1c3f1e9 100644 --- a/ci/docker/ray.cpu.base.wanda.yaml +++ b/ci/docker/ray.cpu.base.wanda.yaml @@ -1,4 +1,4 @@ -name: "ray-py$PYTHON_VERSION-cpu-base" +name: "ray-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX" froms: ["ubuntu:22.04"] dockerfile: docker/base-deps/Dockerfile srcs: @@ -7,4 +7,4 @@ build_args: - PYTHON_VERSION - BASE_IMAGE=ubuntu:22.04 tags: - - cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base + - cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX diff --git a/ci/docker/ray.cuda.base-extra.wanda.yaml b/ci/docker/ray.cuda.base-extra.wanda.yaml index ff1fab0a919f..10584d5d984f 100644 --- a/ci/docker/ray.cuda.base-extra.wanda.yaml +++ b/ci/docker/ray.cuda.base-extra.wanda.yaml @@ -1,7 +1,7 @@ -name: "$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra" -froms: ["cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base"] +name: "$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra$ARCH_SUFFIX" +froms: ["cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX"] dockerfile: docker/base-extra/Dockerfile build_args: - - BASE_IMAGE=cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base + - BASE_IMAGE=cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX tags: - - cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra + - cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra$ARCH_SUFFIX diff --git a/ci/docker/ray.cuda.base.aarch64.wanda.yaml b/ci/docker/ray.cuda.base.aarch64.wanda.yaml deleted file mode 100644 index 325525355b44..000000000000 --- a/ci/docker/ray.cuda.base.aarch64.wanda.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: "ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base-aarch64" -froms: ["nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04"] -dockerfile: docker/base-deps/Dockerfile -srcs: - - python/requirements_compiled.txt -build_args: - - PYTHON_VERSION - - BASE_IMAGE=nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04 -tags: - - cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base-aarch64 diff --git a/ci/docker/ray.cuda.base.wanda.yaml b/ci/docker/ray.cuda.base.wanda.yaml index 0bcd7611c921..44b47fc0dde2 100644 --- a/ci/docker/ray.cuda.base.wanda.yaml +++ b/ci/docker/ray.cuda.base.wanda.yaml @@ -1,4 +1,4 @@ -name: "ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base" +name: "ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX" froms: ["nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04"] dockerfile: docker/base-deps/Dockerfile srcs: @@ -7,4 +7,4 @@ build_args: - PYTHON_VERSION - BASE_IMAGE=nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04 tags: - - cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base + - cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX From af41960a49e85863709ef36fb4968f0021d730b3 Mon Sep 17 00:00:00 2001 From: Stephanie Wang Date: Thu, 14 Aug 2025 10:02:16 -0700 Subject: [PATCH 105/634] [core][gpu-object] Add a user-facing call to wait for tensor to be freed (#55076) This adds a call `ray.experimental.wait_tensor_freed` that allows user code to check when a tensor that it put into Ray's GPU object store has been freed. Unlike the normal Ray object store, the GPU object store is just a Python data structure on the actor, which allows us to avoid copying. This means that the actor can keep a reference to an object in its store. The API call allows the actor to check when the object has been freed from the store, so that it can safely write to the tensor again. Closes #52341. --------- Signed-off-by: Stephanie wang Signed-off-by: Stephanie Wang Co-authored-by: Kai-Hsun Chen --- python/ray/_private/serialization.py | 10 +- python/ray/_private/worker.py | 9 +- python/ray/experimental/__init__.py | 3 +- .../gpu_object_manager/__init__.py | 7 +- .../gpu_object_manager/gpu_object_manager.py | 28 ++++- .../gpu_object_manager/gpu_object_store.py | 68 +++++++---- python/ray/tests/test_gpu_objects_gloo.py | 115 +++++++++++++++++- 7 files changed, 198 insertions(+), 42 deletions(-) diff --git a/python/ray/_private/serialization.py b/python/ray/_private/serialization.py index bc2ac1af8d63..077aa90c3f57 100644 --- a/python/ray/_private/serialization.py +++ b/python/ray/_private/serialization.py @@ -7,8 +7,6 @@ if TYPE_CHECKING: import torch - from ray.experimental.gpu_object_manager.gpu_object_store import GPUObject - import google.protobuf.message import ray._private.utils @@ -506,7 +504,7 @@ def deserialize_objects( self, serialized_ray_objects: List[SerializedRayObject], object_refs, - gpu_objects: Dict[str, "GPUObject"], + gpu_objects: Dict[str, List["torch.Tensor"]], ): assert len(serialized_ray_objects) == len(object_refs) # initialize the thread-local field @@ -524,11 +522,7 @@ def deserialize_objects( if object_ref is not None: object_id = object_ref.hex() if object_id in gpu_objects: - gpu_object = gpu_objects[object_id] - object_tensors = gpu_object.data - gpu_object.num_readers -= 1 - if gpu_object.num_readers == 0: - gpu_objects.pop(object_id) + object_tensors = gpu_objects[object_id] obj = self._deserialize_object( data, metadata, diff --git a/python/ray/_private/worker.py b/python/ray/_private/worker.py index dacaa9e2ff7f..d81789c60f89 100644 --- a/python/ray/_private/worker.py +++ b/python/ray/_private/worker.py @@ -38,6 +38,9 @@ ) from urllib.parse import urlparse +if TYPE_CHECKING: + import torch + import colorama import ray @@ -101,9 +104,6 @@ from ray.widgets import Template from ray.widgets.util import repr_with_fallback -if TYPE_CHECKING: - from ray.experimental.gpu_object_manager import GPUObject - SCRIPT_MODE = 0 WORKER_MODE = 1 LOCAL_MODE = 2 @@ -871,7 +871,7 @@ def raise_errors(self, serialized_objects, object_refs): _unhandled_error_handler(e) def deserialize_objects(self, serialized_objects, object_refs): - gpu_objects: Dict[str, GPUObject] = {} + gpu_objects: Dict[str, List["torch.Tensor"]] = {} for obj_ref, (_, _, tensor_transport) in zip(object_refs, serialized_objects): # If using a non-object store transport, then tensors will be sent # out-of-band. Get them before deserializing the object store data. @@ -886,7 +886,6 @@ def deserialize_objects(self, serialized_objects, object_refs): gpu_objects[object_id] = self.gpu_object_manager.get_gpu_object( object_id ) - gpu_objects[object_id].num_readers += 1 # Function actor manager or the import thread may call pickle.loads # at the same time which can lead to failed imports diff --git a/python/ray/experimental/__init__.py b/python/ray/experimental/__init__.py index 57e565dd2d44..c6c5403a4ea0 100644 --- a/python/ray/experimental/__init__.py +++ b/python/ray/experimental/__init__.py @@ -1,10 +1,11 @@ from ray.experimental.dynamic_resources import set_resource from ray.experimental.locations import get_local_object_locations, get_object_locations -from ray.experimental.gpu_object_manager import GPUObjectManager +from ray.experimental.gpu_object_manager import GPUObjectManager, wait_tensor_freed __all__ = [ "get_object_locations", "get_local_object_locations", "set_resource", "GPUObjectManager", + "wait_tensor_freed", ] diff --git a/python/ray/experimental/gpu_object_manager/__init__.py b/python/ray/experimental/gpu_object_manager/__init__.py index 9cc5b4d9c981..13be59395445 100644 --- a/python/ray/experimental/gpu_object_manager/__init__.py +++ b/python/ray/experimental/gpu_object_manager/__init__.py @@ -1,3 +1,6 @@ -from ray.experimental.gpu_object_manager.gpu_object_manager import GPUObjectManager +from ray.experimental.gpu_object_manager.gpu_object_manager import ( + GPUObjectManager, + wait_tensor_freed, +) -__all__ = ["GPUObjectManager"] +__all__ = ["GPUObjectManager", "wait_tensor_freed"] diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py index 06a2b040e49e..c45ff36572aa 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py @@ -11,7 +11,6 @@ import torch from ray.experimental.gpu_object_manager.gpu_object_store import ( GPUObjectStore, - GPUObject, ) # GPUObjectMeta is a named tuple containing the source actor, tensor transport @@ -28,6 +27,31 @@ class GPUObjectMeta(NamedTuple): tensor_meta: List[Tuple["torch.Size", "torch.dtype"]] +# TODO(swang): Uncomment and add an API docs page and example usage. +# @PublicAPI(stability="alpha") +def wait_tensor_freed(tensor: "torch.Tensor", timeout: Optional[float] = None): + """ + Wait for the tensor to be freed from this actor's GPU object store. + + This function is useful for cases where an actor keeps a reference to a + tensor after returning the tensor from a task annotated with + `@ray.method(tensor_transport=...)`. Tensors that are returned by these + tasks may be sent to other actors while the corresponding `ray.ObjectRef` is + still in scope. If the actor modifies the tensor while it is still in the + actor's GPU object store, then Ray may end up sending invalid data to other + tasks. Call this function to ensure that the `ray.ObjectRef` has gone out of + scope and therefore the tensor is safe to write to again. + + Args: + tensor: The tensor to wait to be freed. + timeout: The timeout in seconds. Set to None to wait indefinitely. Note + that this function could then hang if the `ray.ObjectRef` that + refers to this tensor never goes out of scope. + """ + gpu_object_manager = ray.worker.global_worker.gpu_object_manager + gpu_object_manager.gpu_object_store.wait_tensor_freed(tensor, timeout) + + class GPUObjectManager: def __init__(self): # A dictionary that maps from owned object's ID to GPUObjectMeta. @@ -268,7 +292,7 @@ def trigger_out_of_band_tensor_transfer( communicator.name, dst_actor, obj_id, src_rank, tensor_meta ) - def get_gpu_object(self, object_id: str) -> "GPUObject": + def get_gpu_object(self, object_id: str) -> List["torch.Tensor"]: """ Get the GPU object for a given object ID. """ diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_store.py b/python/ray/experimental/gpu_object_manager/gpu_object_store.py index b16e056685c7..9a15f0da7cc4 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_store.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_store.py @@ -1,6 +1,7 @@ from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Set import threading +from collections import defaultdict import ray.util.collective as collective from ray._private.custom_types import TensorTransportEnum @@ -45,7 +46,7 @@ def __ray_send__(self, communicator_name: str, obj_id: str, dst_rank: int): assert gpu_object_store.has_object( obj_id ), f"obj_id={obj_id} not found in GPU object store" - tensors = gpu_object_store.get_object(obj_id).data + tensors = gpu_object_store.get_object(obj_id) backend = collective.get_group_handle(communicator_name).backend() device = COLLECTIVE_BACKEND_TO_TORCH_DEVICE[backend] @@ -92,7 +93,7 @@ def __ray_get_tensor_meta__(self, obj_id: str): # it could take arbitrarily long and we don't want to trigger a spurious # timeout. gpu_object = gpu_object_store.wait_and_get_object(obj_id) - return [(t.shape, t.dtype) for t in gpu_object.data] + return [(t.shape, t.dtype) for t in gpu_object] def __ray_fetch_gpu_object__(self, obj_id: str): @@ -104,21 +105,15 @@ def __ray_fetch_gpu_object__(self, obj_id: str): obj_id ), f"obj_id={obj_id} not found in GPU object store" gpu_object = gpu_object_store.get_object(obj_id) - return gpu_object.data + return gpu_object @dataclass -class GPUObject: +class _GPUObject: # A list of tensors representing the GPU object. data: List["torch.Tensor"] # Whether the GPU object is the primary copy. is_primary: bool - # The number of reads allowed to the GPU object before it will be GCed from this actor. - # This is used to implement garbage collection for receiver actors, - # handling cases where the same GPU object reference is passed to the - # same actor task multiple times. For sender actors, we still rely on - # the object store's reference counting mechanism. - num_readers: int = 0 class GPUObjectStore: @@ -137,19 +132,27 @@ def __init__(self): # A dictionary that maps from an object ID to a list of tensors. # # Note: Currently, `gpu_object_store` is only supported for Ray Actors. - self._gpu_object_store: Dict[str, GPUObject] = {} + self._gpu_object_store: Dict[str, _GPUObject] = {} + # Mapping from tensor to the IDs of objects that contain it. + self._tensor_to_object_ids: Dict["torch.Tensor", Set[str]] = defaultdict(set) # Synchronization for GPU object store. self._lock = threading.RLock() # Signal when an object becomes present in the object store. self._object_present_cv = threading.Condition(self._lock) + # Signal when an object is freed from the object store. + self._object_freed_cv = threading.Condition(self._lock) def has_object(self, obj_id: str) -> bool: with self._lock: return obj_id in self._gpu_object_store - def get_object(self, obj_id: str) -> Optional[GPUObject]: + def has_tensor(self, tensor: "torch.Tensor") -> bool: with self._lock: - return self._gpu_object_store[obj_id] + return tensor in self._tensor_to_object_ids + + def get_object(self, obj_id: str) -> Optional[List["torch.Tensor"]]: + with self._lock: + return self._gpu_object_store[obj_id].data def add_object( self, @@ -166,7 +169,9 @@ def add_object( is_primary: Whether the GPU object is the primary copy. """ with self._object_present_cv: - self._gpu_object_store[obj_id] = GPUObject( + for tensor in gpu_object: + self._tensor_to_object_ids[tensor].add(obj_id) + self._gpu_object_store[obj_id] = _GPUObject( gpu_object, is_primary, ) @@ -181,7 +186,7 @@ def is_primary_copy(self, obj_id: str) -> bool: def wait_and_get_object( self, obj_id: str, timeout: Optional[float] = None - ) -> GPUObject: + ) -> List["torch.Tensor"]: """Atomically waits for the GPU object to be present in the GPU object store, then gets it. If the object is not present after the optional timeout, raise a TimeoutError. @@ -200,7 +205,7 @@ def wait_and_get_object( def wait_and_pop_object( self, obj_id: str, timeout: Optional[float] = None - ) -> GPUObject: + ) -> List["torch.Tensor"]: """Atomically waits for the GPU object to be present in the GPU object store, then pops it. If the object is not present after the optional timeout, raise a TimeoutError. @@ -230,20 +235,39 @@ def _wait_object(self, obj_id: str, timeout: Optional[float] = None) -> None: indefinitely. """ with self._object_present_cv: - present = self._object_present_cv.wait_for( + if not self._object_present_cv.wait_for( lambda: obj_id in self._gpu_object_store, timeout=timeout - ) - if not present: + ): raise TimeoutError( f"ObjectRef({obj_id}) not found in GPU object store after {timeout}s, transfer may have failed. Please report this issue on GitHub: https://github.com/ray-project/ray/issues/new/choose" ) - def pop_object(self, obj_id: str) -> GPUObject: + def pop_object(self, obj_id: str) -> List["torch.Tensor"]: with self._lock: assert ( obj_id in self._gpu_object_store ), f"obj_id={obj_id} not found in GPU object store" - return self._gpu_object_store.pop(obj_id) + gpu_object = self._gpu_object_store.pop(obj_id) + for tensor in gpu_object.data: + self._tensor_to_object_ids[tensor].remove(obj_id) + if len(self._tensor_to_object_ids[tensor]) == 0: + self._tensor_to_object_ids.pop(tensor) + self._object_freed_cv.notify_all() + return gpu_object.data + + def wait_tensor_freed( + self, tensor: "torch.Tensor", timeout: Optional[float] = None + ) -> None: + """ + Wait for the object to be freed from the GPU object store. + """ + with self._object_freed_cv: + if not self._object_freed_cv.wait_for( + lambda: tensor not in self._tensor_to_object_ids, timeout=timeout + ): + raise TimeoutError( + f"Tensor {tensor} not freed from GPU object store after {timeout}s. The tensor will not be freed until all ObjectRefs containing the tensor have gone out of scope." + ) def get_num_objects(self) -> int: """ diff --git a/python/ray/tests/test_gpu_objects_gloo.py b/python/ray/tests/test_gpu_objects_gloo.py index ead9c93461a1..9a771eb2f052 100644 --- a/python/ray/tests/test_gpu_objects_gloo.py +++ b/python/ray/tests/test_gpu_objects_gloo.py @@ -2,7 +2,9 @@ import random import torch import pytest +import threading import ray +import time from ray.experimental.collective import create_collective_group from ray._private.custom_types import TensorTransportEnum from ray._common.test_utils import wait_for_condition @@ -36,8 +38,7 @@ def get_out_of_band_tensors(self, obj_id: str, timeout=None): ) if timeout is None: timeout = 0 - gpu_object = gpu_object_store.wait_and_get_object(obj_id, timeout) - return gpu_object.data + return gpu_object_store.wait_and_get_object(obj_id, timeout) def get_num_gpu_objects(self): gpu_object_manager = ray._private.worker.global_worker.gpu_object_manager @@ -542,5 +543,115 @@ def test_app_error_fetch_to_driver(ray_start_regular): assert torch.equal(ray.get(ref), small_tensor) +def test_write_after_save(ray_start_regular): + """Check that an actor can safely write to a tensor after saving it to its + local state by calling `ray.experimental.wait_tensor_freed`.""" + + @ray.remote(enable_tensor_transport=True) + class GPUTestActor: + @ray.method(tensor_transport="gloo") + def save(self, data: torch.Tensor): + # Save the tensor to the actor's local state. + self.data = data + return data + + def receive(self, data: torch.Tensor): + return data + + def increment_saved(self): + ray.experimental.wait_tensor_freed(self.data) + # Write to the saved tensor. + self.data += 1 + return self.data + + world_size = 2 + actors = [GPUTestActor.remote() for _ in range(world_size)] + create_collective_group(actors, backend="torch_gloo") + + medium_tensor = torch.randn((500, 500)) + sender, receiver = actors + ref = sender.save.remote(medium_tensor) + # Sender writes to the GPU object while Ray sends the object to a receiver + # task in the background. + tensor1 = sender.increment_saved.remote() + tensor2 = receiver.receive.remote(ref) + + # The sender task should not have returned yet because the ObjectRef is + # still in scope. + with pytest.raises(ray.exceptions.GetTimeoutError): + ray.get(tensor1, timeout=1) + + del ref + # Check that Ray completed the transfer of the original tensor before the + # sender writes to it. + assert torch.allclose(ray.get(tensor1), medium_tensor + 1) + assert torch.allclose(ray.get(tensor2), medium_tensor) + + +def test_wait_tensor_freed(ray_start_regular): + """Unit test for ray.experimental.wait_tensor_freed. Check that the call + returns when the tensor has been freed from the GPU object store.""" + gpu_object_store = ray.worker.global_worker.gpu_object_manager.gpu_object_store + obj_id = "random_id" + tensor = torch.randn((1,)) + gpu_object_store.add_object(obj_id, [tensor], is_primary=True) + + assert gpu_object_store.has_object(obj_id) + with pytest.raises(TimeoutError): + ray.experimental.wait_tensor_freed(tensor, timeout=1) + assert gpu_object_store.has_object(obj_id) + + # Simulate garbage collection in a background thread. + def gc(): + time.sleep(0.1) + gpu_object_store.pop_object(obj_id) + + gc_thread = threading.Thread(target=gc) + gc_thread.start() + # Now the wait_tensor_freed call should be able to return. + ray.experimental.wait_tensor_freed(tensor) + gc_thread.join() + assert not gpu_object_store.has_object(obj_id) + + +def test_wait_tensor_freed_double_tensor(ray_start_regular): + """Unit test for ray.experimental.wait_tensor_freed when multiple objects + contain the same tensor.""" + gpu_object_store = ray.worker.global_worker.gpu_object_manager.gpu_object_store + obj_id1 = "random_id1" + obj_id2 = "random_id2" + tensor = torch.randn((1,)) + gpu_object_store.add_object(obj_id1, [tensor], is_primary=True) + gpu_object_store.add_object(obj_id2, [tensor], is_primary=True) + + assert gpu_object_store.has_object(obj_id1) + assert gpu_object_store.has_object(obj_id2) + with pytest.raises(TimeoutError): + ray.experimental.wait_tensor_freed(tensor, timeout=1) + assert gpu_object_store.has_object(obj_id1) + assert gpu_object_store.has_object(obj_id2) + + # Simulate garbage collection in a background thread. + def gc(obj_id): + time.sleep(0.1) + gpu_object_store.pop_object(obj_id) + + # Free one object. Tensor should still be stored. + gc_thread = threading.Thread(target=gc, args=(obj_id1,)) + gc_thread.start() + with pytest.raises(TimeoutError): + ray.experimental.wait_tensor_freed(tensor, timeout=1) + gc_thread.join() + assert not gpu_object_store.has_object(obj_id1) + + # Free the other object. Now the wait_tensor_freed call should be able to + # return. + gc_thread = threading.Thread(target=gc, args=(obj_id2,)) + gc_thread.start() + ray.experimental.wait_tensor_freed(tensor) + gc_thread.join() + assert not gpu_object_store.has_object(obj_id2) + + if __name__ == "__main__": sys.exit(pytest.main(["-sv", __file__])) From c4d990cafe01ce4f6caec38e814217310fcc0a1c Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:02:48 -0700 Subject: [PATCH 106/634] [ci] add rayci build id tags for release test images (#55605) in addition to current tags. first step to migrate to use rayci build id tags to stop release test jobs from cross-talking to each other Signed-off-by: Lonnie Liu --- ci/ray_ci/docker_container.py | 15 ++++--- ci/ray_ci/test_anyscale_docker_container.py | 47 +++++++++++++-------- ci/ray_ci/test_base.py | 2 +- ci/ray_ci/test_ray_docker_container.py | 20 +++++++-- 4 files changed, 57 insertions(+), 27 deletions(-) diff --git a/ci/ray_ci/docker_container.py b/ci/ray_ci/docker_container.py index 73a6f3d2879c..9544cef62167 100644 --- a/ci/ray_ci/docker_container.py +++ b/ci/ray_ci/docker_container.py @@ -82,24 +82,29 @@ def _get_image_version_tags(self, external: bool) -> List[str]: external: If True, return the external image tags. If False, return the internal image tags. """ - branch = os.environ.get("BUILDKITE_BRANCH") + branch = os.environ.get("BUILDKITE_BRANCH", "") sha_tag = os.environ["BUILDKITE_COMMIT"][:6] + rayci_build_id = os.environ["RAYCI_BUILD_ID"] pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") formatted_date = datetime.now().strftime("%y%m%d") if branch == "master": if external and os.environ.get("RAYCI_SCHEDULE") == "nightly": return [f"nightly.{formatted_date}.{sha_tag}", "nightly"] - return [sha_tag] + return [sha_tag, rayci_build_id] if branch and branch.startswith("releases/"): release_name = branch[len("releases/") :] - return [f"{release_name}.{sha_tag}"] + release_tag = f"{release_name}.{sha_tag}" + if external: + # Avoid saving build ID ones when saving it on public registries. + return [release_tag] + return [release_tag, rayci_build_id] if pr != "false": - return [f"pr-{pr}.{sha_tag}"] + return [f"pr-{pr}.{sha_tag}", rayci_build_id] - return [sha_tag] + return [sha_tag, rayci_build_id] def _get_canonical_tag(self) -> str: # The canonical tag is the first tag in the list of tags. The list of tag is diff --git a/ci/ray_ci/test_anyscale_docker_container.py b/ci/ray_ci/test_anyscale_docker_container.py index 777a98447d6f..52b275ddafb8 100644 --- a/ci/ray_ci/test_anyscale_docker_container.py +++ b/ci/ray_ci/test_anyscale_docker_container.py @@ -32,26 +32,37 @@ def _mock_run_script(input: List[str]) -> None: aws_ecr = _DOCKER_ECR_REPO.split("/")[0] aws_prj = f"{aws_ecr}/anyscale/ray-ml" gcp_prj = f"{_DOCKER_GCP_REGISTRY}/anyscale/ray-ml" - assert cmd == [ - "./ci/build/build-anyscale-docker.sh " - f"rayproject/ray-ml:123456-{pv}-cu121 " - f"{aws_prj}:123456-{pv}-cu121 requirements_ml_byod_{v}.txt {aws_ecr}", - "./release/gcloud_docker_login.sh release/aws2gce_iam.json", - "export PATH=$(pwd)/google-cloud-sdk/bin:$PATH", - f"docker tag {aws_prj}:123456-{pv}-cu121 {aws_prj}:123456-{pv}-cu121", - f"docker push {aws_prj}:123456-{pv}-cu121", - f"docker tag {aws_prj}:123456-{pv}-cu121 {gcp_prj}:123456-{pv}-cu121", - f"docker push {gcp_prj}:123456-{pv}-cu121", - f"docker tag {aws_prj}:123456-{pv}-cu121 {aws_prj}:123456-{pv}-gpu", - f"docker push {aws_prj}:123456-{pv}-gpu", - f"docker tag {aws_prj}:123456-{pv}-cu121 {gcp_prj}:123456-{pv}-gpu", - f"docker push {gcp_prj}:123456-{pv}-gpu", - f"docker tag {aws_prj}:123456-{pv}-cu121 {aws_prj}:123456-{pv}", - f"docker push {aws_prj}:123456-{pv}", - f"docker tag {aws_prj}:123456-{pv}-cu121 {gcp_prj}:123456-{pv}", - f"docker push {gcp_prj}:123456-{pv}", + + tags_want = [ + f"123456-{pv}-cu121", + f"123456-{pv}-gpu", + f"123456-{pv}", + f"a1b2c3d4-{pv}-cu121", + f"a1b2c3d4-{pv}-gpu", + f"a1b2c3d4-{pv}", ] + push_cmds_want = [] + for tag in tags_want: + push_cmds_want += [ + f"docker tag {aws_prj}:123456-{pv}-cu121 {aws_prj}:{tag}", + f"docker push {aws_prj}:{tag}", + f"docker tag {aws_prj}:123456-{pv}-cu121 {gcp_prj}:{tag}", + f"docker push {gcp_prj}:{tag}", + ] + + assert ( + cmd + == [ + "./ci/build/build-anyscale-docker.sh " + f"rayproject/ray-ml:123456-{pv}-cu121 " + f"{aws_prj}:123456-{pv}-cu121 requirements_ml_byod_{v}.txt {aws_ecr}", + "./release/gcloud_docker_login.sh release/aws2gce_iam.json", + "export PATH=$(pwd)/google-cloud-sdk/bin:$PATH", + ] + + push_cmds_want + ) + def test_requirements_file(self) -> None: container = AnyscaleDockerContainer("3.11", "cu12.1.1-cudnn8", "ray-ml") assert container._get_requirement_file() == "requirements_ml_byod_3.11.txt" diff --git a/ci/ray_ci/test_base.py b/ci/ray_ci/test_base.py index d0cbc4c253bc..ec237d64e418 100644 --- a/ci/ray_ci/test_base.py +++ b/ci/ray_ci/test_base.py @@ -14,7 +14,7 @@ def setUp(self) -> None: os.environ, { "RAYCI_CHECKOUT_DIR": "/ray", - "RAYCI_BUILD_ID": "123", + "RAYCI_BUILD_ID": "a1b2c3d4", "RAYCI_WORK_REPO": "rayproject/citemp", "BUILDKITE_COMMIT": "123456", "BUILDKITE_BRANCH": "master", diff --git a/ci/ray_ci/test_ray_docker_container.py b/ci/ray_ci/test_ray_docker_container.py index ce2bc3021e58..e60528207c9b 100644 --- a/ci/ray_ci/test_ray_docker_container.py +++ b/ci/ray_ci/test_ray_docker_container.py @@ -29,7 +29,7 @@ def _mock_run_script(input: List[str]) -> None: side_effect=_mock_run_script, ): sha = "123456" - ray_ci_build_id = "123" + ray_ci_build_id = "a1b2c3d4" cuda = "cu12.4.1-cudnn" # Run with default python version and ray image @@ -99,7 +99,7 @@ def _mock_run_script(input: List[str]) -> None: ): formatted_date = datetime.now().strftime("%y%m%d") sha = "123456" - ray_ci_build_id = "123" + ray_ci_build_id = "a1b2c3d4" # Run with default python version and ray image self.cmds = [] @@ -195,7 +195,7 @@ def _mock_run_script(input: List[str]) -> None: os.environ, {"RAYCI_SCHEDULE": "daytime"} ): sha = "123456" - ray_ci_build_id = "123" + ray_ci_build_id = "a1b2c3d4" cuda = "cu11.8.0-cudnn8" # Run with default python version and ray image @@ -280,6 +280,7 @@ def test_get_image_tags(self) -> None: # bulk logic of _get_image_tags is tested in its callers (get_image_name and # get_canonical_tag), so we only test the basic cases here sha = "123456" + rayci_build_id = "a1b2c3d4" v = DEFAULT_PYTHON_VERSION pv = self.get_python_version(v) container = RayDockerContainer(v, "cpu", "ray") @@ -290,6 +291,10 @@ def test_get_image_tags(self) -> None: f"{sha}-cpu", f"{sha}-{pv}", f"{sha}", + f"{rayci_build_id}-{pv}-cpu", + f"{rayci_build_id}-cpu", + f"{rayci_build_id}-{pv}", + f"{rayci_build_id}", ] with mock.patch.dict(os.environ, {"RAYCI_SCHEDULE": "nightly"}): assert container._get_image_tags(external=True) == [ @@ -305,6 +310,7 @@ def test_get_image_tags(self) -> None: def test_get_image_name(self) -> None: sha = "123456" + rayci_build_id = "a1b2c3d4" v = DEFAULT_PYTHON_VERSION pv = self.get_python_version(v) formatted_date = datetime.now().strftime("%y%m%d") @@ -315,6 +321,10 @@ def test_get_image_name(self) -> None: f"rayproject/ray:{sha}-cpu", f"rayproject/ray:{sha}-{pv}", f"rayproject/ray:{sha}", + f"rayproject/ray:{rayci_build_id}-{pv}-cpu", + f"rayproject/ray:{rayci_build_id}-cpu", + f"rayproject/ray:{rayci_build_id}-{pv}", + f"rayproject/ray:{rayci_build_id}", ] with mock.patch.dict(os.environ, {"RAYCI_SCHEDULE": "nightly"}): @@ -335,6 +345,7 @@ def test_get_image_name(self) -> None: with mock.patch.dict(os.environ, {"RAYCI_SCHEDULE": "daytime"}): assert container._get_image_names() == [ f"rayproject/ray-llm:{sha}-{pv}-cu124", + f"rayproject/ray-llm:{rayci_build_id}-{pv}-cu124", ] with mock.patch.dict(os.environ, {"RAYCI_SCHEDULE": "nightly"}): @@ -351,6 +362,9 @@ def test_get_image_name(self) -> None: f"rayproject/ray-ml:{sha}-{pv}-cu121", f"rayproject/ray-ml:{sha}-{pv}-gpu", f"rayproject/ray-ml:{sha}-{pv}", + f"rayproject/ray-ml:{rayci_build_id}-{pv}-cu121", + f"rayproject/ray-ml:{rayci_build_id}-{pv}-gpu", + f"rayproject/ray-ml:{rayci_build_id}-{pv}", ] with mock.patch.dict(os.environ, {"RAYCI_SCHEDULE": "nightly"}): From 6a9938a73ff6d39ee72dcb68667a52b0ba658e8b Mon Sep 17 00:00:00 2001 From: Mengjin Yan Date: Thu, 14 Aug 2025 11:05:39 -0700 Subject: [PATCH 107/634] [Core] Add Logic to Check Label Selector in PG Scheduling (#55599) Signed-off-by: Mengjin Yan --- python/ray/tests/test_bundle_label_selector.py | 18 ++++++++++-------- src/ray/raylet/scheduling/policy/scorer.cc | 6 ++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/python/ray/tests/test_bundle_label_selector.py b/python/ray/tests/test_bundle_label_selector.py index 890fa2845dcc..8a029dbf03c8 100644 --- a/python/ray/tests/test_bundle_label_selector.py +++ b/python/ray/tests/test_bundle_label_selector.py @@ -11,22 +11,26 @@ def test_bundle_label_selector_with_repeated_labels(ray_start_cluster): cluster = ray_start_cluster - num_nodes = 2 - for _ in range(num_nodes): - cluster.add_node(num_cpus=4, labels={"ray.io/accelerator-type": "A100"}) + cluster.add_node(num_cpus=4, labels={"ray.io/accelerator-type": "A100"}) + node = cluster.add_node(num_cpus=4, labels={"ray.io/accelerator-type": "TPU"}) ray.init(address=cluster.address) bundles = [{"CPU": 1}, {"CPU": 1}] - label_selector = [{"ray.io/accelerator-type": "A100"}] * 2 + label_selector = [{"ray.io/accelerator-type": "TPU"}] * 2 placement_group = ray.util.placement_group( name="repeated_labels_pg", - strategy="PACK", bundles=bundles, bundle_label_selector=label_selector, ) ray.get(placement_group.ready()) + bundles_to_node_id = ray.util.placement_group_table()[placement_group.id.hex()][ + "bundles_to_node_id" + ] + assert bundles_to_node_id[0] == node.node_id + assert bundles_to_node_id[1] == node.node_id + placement_group_assert_no_leak([placement_group]) @@ -42,7 +46,6 @@ def test_unschedulable_bundle_label_selector(ray_start_cluster): placement_group = ray.util.placement_group( name="unschedulable_labels_pg", - strategy="STRICT_PACK", bundles=bundles, bundle_label_selector=label_selector, ) @@ -53,7 +56,7 @@ def test_unschedulable_bundle_label_selector(ray_start_cluster): state = ray.util.placement_group_table()[placement_group.id.hex()]["stats"][ "scheduling_state" ] - assert state == "INFEASIBLE" + assert state == "NO_RESOURCES" def test_bundle_label_selectors_match_bundle_resources(ray_start_cluster): @@ -89,7 +92,6 @@ def test_bundle_label_selectors_match_bundle_resources(ray_start_cluster): pg = ray.util.placement_group( name="label_selectors_match_resources", - strategy="SPREAD", bundles=bundles, bundle_label_selector=bundle_label_selectors, ) diff --git a/src/ray/raylet/scheduling/policy/scorer.cc b/src/ray/raylet/scheduling/policy/scorer.cc index b8c67f3d920d..6bb07b8007ac 100644 --- a/src/ray/raylet/scheduling/policy/scorer.cc +++ b/src/ray/raylet/scheduling/policy/scorer.cc @@ -21,6 +21,12 @@ namespace raylet_scheduling_policy { double LeastResourceScorer::Score(const ResourceRequest &required_resources, const NodeResources &node_resources) { + // Check if the node has required labels before scoring on the resources. + const auto &label_selector = required_resources.GetLabelSelector(); + if (!node_resources.HasRequiredLabels(label_selector)) { + return -1.; + } + // In GCS-based actor scheduling, the `NodeResources` are only acquired or released by // actor scheduling, instead of being updated by resource reports from raylets. So we // have to subtract normal task resources (if exist) from the current available From 6d7234b1b54ebc8d77ed9a127ce02b9ff4f9854c Mon Sep 17 00:00:00 2001 From: coqian Date: Thu, 14 Aug 2025 11:06:05 -0700 Subject: [PATCH 108/634] [Data] Update the export API to refresh the dataset and operator states (#55355) ## Why are these changes needed? This PR is a revert of [#55333](https://github.com/ray-project/ray/pull/55333) and resolves conflict by [#55163](https://github.com/ray-project/ray/pull/55163) Original description: Some frequently used metadata fields are missing in the export API schema: - For both dataset and operator: state, execution start and end time These fields are important for us to observe the lifecycle of the datasets and operators, and can be used to improve the accuracy of reported metrics, such as throughput, which relies on the duration. Summary of change: - Add state, execution start and end time at the export API schema - Add a new state enum `PENDING` for dataset and operator, to represent the state when they are not running yet. - Refresh the metadata when ever the state of dataset/operator gets updated. And the event will always contains the latest snapshot of all the metadata. ## Related issue number ## Checks - [X] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [X] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [X] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: cong.qian --- .../data/_internal/execution/dataset_state.py | 22 ++++ .../_internal/execution/streaming_executor.py | 7 +- .../ray/data/_internal/metadata_exporter.py | 24 +++- python/ray/data/_internal/stats.py | 114 +++++++++++++----- python/ray/data/tests/test_state_export.py | 105 ++++++++++++++++ .../protobuf/export_dataset_metadata.proto | 36 +++++- 6 files changed, 276 insertions(+), 32 deletions(-) create mode 100644 python/ray/data/_internal/execution/dataset_state.py diff --git a/python/ray/data/_internal/execution/dataset_state.py b/python/ray/data/_internal/execution/dataset_state.py new file mode 100644 index 000000000000..702963234baf --- /dev/null +++ b/python/ray/data/_internal/execution/dataset_state.py @@ -0,0 +1,22 @@ +import enum + + +class DatasetState(enum.IntEnum): + """Enum representing the possible states of a dataset during execution.""" + + UNKNOWN = 0 + RUNNING = 1 + FINISHED = 2 + FAILED = 3 + PENDING = 4 + + def __str__(self): + return self.name + + @classmethod + def from_string(cls, text): + """Get enum by name.""" + try: + return cls[text] # This uses the name to lookup the enum + except KeyError: + return cls.UNKNOWN diff --git a/python/ray/data/_internal/execution/streaming_executor.py b/python/ray/data/_internal/execution/streaming_executor.py index 6806c69bdd9d..4e2dfdd9aa6a 100644 --- a/python/ray/data/_internal/execution/streaming_executor.py +++ b/python/ray/data/_internal/execution/streaming_executor.py @@ -9,6 +9,7 @@ BackpressurePolicy, get_backpressure_policies, ) +from ray.data._internal.execution.dataset_state import DatasetState from ray.data._internal.execution.execution_callback import get_execution_callbacks from ray.data._internal.execution.interfaces import ( ExecutionResources, @@ -37,7 +38,7 @@ ) from ray.data._internal.metadata_exporter import Topology as TopologyMetadata from ray.data._internal.progress_bar import ProgressBar -from ray.data._internal.stats import DatasetState, DatasetStats, StatsManager, Timer +from ray.data._internal.stats import DatasetStats, StatsManager, Timer from ray.data.context import OK_PREFIX, WARN_PREFIX, DataContext from ray.util.metrics import Gauge @@ -548,7 +549,9 @@ def _get_state_dict(self, state): "progress": last_state.num_completed_tasks, "total": last_op.num_outputs_total(), "total_rows": last_op.num_output_rows_total(), - "end_time": time.time() if state != DatasetState.RUNNING.name else None, + "end_time": time.time() + if state in (DatasetState.FINISHED.name, DatasetState.FAILED.name) + else None, "operators": { f"{self._get_operator_id(op, i)}": { "name": op.name, diff --git a/python/ray/data/_internal/metadata_exporter.py b/python/ray/data/_internal/metadata_exporter.py index 5ff8242cfc5c..17150c5b006c 100644 --- a/python/ray/data/_internal/metadata_exporter.py +++ b/python/ray/data/_internal/metadata_exporter.py @@ -12,6 +12,7 @@ check_export_api_enabled, get_export_event_logger, ) +from ray.data._internal.execution.dataset_state import DatasetState from ray.data.context import DataContext if TYPE_CHECKING: @@ -59,11 +60,17 @@ class Operator: sub_stages: List of sub-stages contained within this operator. args: User-specified arguments associated with the operator, which may include configuration settings, options, or other relevant data for the operator. + execution_start_time: The timestamp when the operator execution begins. + execution_end_time: The timestamp when the operator execution ends. + state: The state of the operator. """ name: str id: str uuid: str + execution_start_time: Optional[float] + execution_end_time: Optional[float] + state: str input_dependencies: List[str] = field(default_factory=list) sub_stages: List[SubStage] = field(default_factory=list) args: Dict[str, Any] = field(default_factory=dict) @@ -107,6 +114,9 @@ def create_topology_metadata( op_to_id[dep] for dep in op.input_dependencies if dep in op_to_id ], args=sanitize_for_struct(op._get_logical_args()), + execution_start_time=None, + execution_end_time=None, + state=DatasetState.PENDING.name, ) # Add sub-stages if they exist @@ -130,8 +140,11 @@ class DatasetMetadata: job_id: The ID of the job running this dataset. topology: The structure of the dataset's operator DAG. dataset_id: The unique ID of the dataset. - start_time: The timestamp when the dataset execution started. + start_time: The timestamp when the dataset is registered. data_context: The DataContext attached to the dataset. + execution_start_time: The timestamp when the dataset execution starts. + execution_end_time: The timestamp when the dataset execution ends. + state: The state of the dataset. """ job_id: str @@ -139,6 +152,9 @@ class DatasetMetadata: dataset_id: str start_time: float data_context: DataContext + execution_start_time: Optional[float] + execution_end_time: Optional[float] + state: str def _add_ellipsis(s: str, truncate_length: int) -> str: @@ -206,6 +222,9 @@ def dataset_metadata_to_proto(dataset_metadata: DatasetMetadata) -> Any: id=op.id, uuid=op.uuid, args=args, + execution_start_time=op.execution_start_time, + execution_end_time=op.execution_end_time, + state=ProtoOperator.OperatorState.Value(op.state), ) # Add input dependencies @@ -231,6 +250,9 @@ def dataset_metadata_to_proto(dataset_metadata: DatasetMetadata) -> Any: job_id=dataset_metadata.job_id, start_time=dataset_metadata.start_time, data_context=data_context, + execution_start_time=dataset_metadata.execution_start_time, + execution_end_time=dataset_metadata.execution_end_time, + state=ProtoDatasetMetadata.DatasetState.Value(dataset_metadata.state), ) proto_dataset_metadata.topology.CopyFrom(proto_topology) diff --git a/python/ray/data/_internal/stats.py b/python/ray/data/_internal/stats.py index 869aa49b84ca..d00b45c89b8a 100644 --- a/python/ray/data/_internal/stats.py +++ b/python/ray/data/_internal/stats.py @@ -1,5 +1,5 @@ import collections -import enum +import copy import logging import threading import time @@ -14,6 +14,7 @@ import ray from ray.actor import ActorHandle from ray.data._internal.block_list import BlockList +from ray.data._internal.execution.dataset_state import DatasetState from ray.data._internal.execution.interfaces.op_runtime_metrics import ( NODE_UNKNOWN, MetricsGroup, @@ -21,7 +22,11 @@ NodeMetrics, OpRuntimeMetrics, ) -from ray.data._internal.metadata_exporter import Topology, get_dataset_metadata_exporter +from ray.data._internal.metadata_exporter import ( + DatasetMetadata, + Topology, + get_dataset_metadata_exporter, +) from ray.data._internal.util import capfirst from ray.data.block import BlockStats from ray.data.context import DataContext @@ -170,6 +175,7 @@ def __init__(self, max_stats=1000): # Initialize the metadata exporter self._metadata_exporter = get_dataset_metadata_exporter() + self.dataset_metadatas: Dict[str, DatasetMetadata] = {} # Ray Data dashboard metrics # Everything is a gauge because we need to reset all of @@ -477,7 +483,7 @@ def register_dataset( start_time = time.time() self.datasets[dataset_tag] = { "job_id": job_id, - "state": DatasetState.RUNNING.name, + "state": DatasetState.PENDING.name, "progress": 0, "total": 0, "total_rows": 0, @@ -485,7 +491,7 @@ def register_dataset( "end_time": None, "operators": { operator: { - "state": DatasetState.RUNNING.name, + "state": DatasetState.PENDING.name, "progress": 0, "total": 0, "queued_blocks": 0, @@ -494,16 +500,19 @@ def register_dataset( }, } if self._metadata_exporter is not None: - from ray.data._internal.metadata_exporter import DatasetMetadata - - dataset_metadata = DatasetMetadata( + self.dataset_metadatas[dataset_tag] = DatasetMetadata( job_id=job_id, topology=topology, dataset_id=dataset_tag, start_time=start_time, data_context=data_context, + execution_start_time=None, + execution_end_time=None, + state=DatasetState.PENDING.name, + ) + self._metadata_exporter.export_dataset_metadata( + self.dataset_metadatas[dataset_tag] ) - self._metadata_exporter.export_dataset_metadata(dataset_metadata) def update_dataset(self, dataset_tag: str, state: Dict[str, Any]): self.datasets[dataset_tag].update(state) @@ -527,8 +536,10 @@ def update_dataset(self, dataset_tag: str, state: Dict[str, Any]): state_string = state.get("state", DatasetState.UNKNOWN.name) state_enum = DatasetState.from_string(state_string) self.data_dataset_state.set(state_enum.value, dataset_tags) + self.update_dataset_metadata_state(dataset_tag, state_string) # Update operator-level metrics + operator_states: Dict[str, str] = {} for operator, op_state in state.get("operators", {}).items(): operator_tags = { "dataset": dataset_tag, @@ -548,12 +559,79 @@ def update_dataset(self, dataset_tag: str, state: Dict[str, Any]): state_string = op_state.get("state", DatasetState.UNKNOWN.name) state_enum = DatasetState.from_string(state_string) self.data_operator_state.set(state_enum.value, operator_tags) + operator_states[operator] = state_string + + self.update_dataset_metadata_operator_states(dataset_tag, operator_states) def get_datasets(self, job_id: Optional[str] = None): if not job_id: return self.datasets return {k: v for k, v in self.datasets.items() if v["job_id"] == job_id} + def update_dataset_metadata_state(self, dataset_id: str, new_state: str): + if dataset_id not in self.dataset_metadatas: + return + update_time = time.time() + dataset_metadata = self.dataset_metadatas[dataset_id] + if dataset_metadata.state == new_state: + return + updated_dataset_metadata = copy.deepcopy(dataset_metadata) + updated_dataset_metadata.state = new_state + if new_state == DatasetState.RUNNING.name: + updated_dataset_metadata.execution_start_time = update_time + elif new_state in (DatasetState.FINISHED.name, DatasetState.FAILED.name): + updated_dataset_metadata.execution_end_time = update_time + # Update metadata of running operators + for operator in updated_dataset_metadata.topology.operators: + if operator.state == DatasetState.RUNNING.name: + operator.state = new_state + operator.execution_end_time = update_time + + self.dataset_metadatas[dataset_id] = updated_dataset_metadata + self._metadata_exporter.export_dataset_metadata(updated_dataset_metadata) + + def update_dataset_metadata_operator_states( + self, dataset_id: str, operator_states: Dict[str, str] + ): + if dataset_id not in self.dataset_metadatas: + return + + dataset_metadata = self.dataset_metadatas[dataset_id] + update_needed = False + for operator in dataset_metadata.topology.operators: + if ( + operator.id in operator_states + and operator.state != operator_states[operator.id] + ): + update_needed = True + break + + if not update_needed: + return + + updated_dataset_metadata = copy.deepcopy(dataset_metadata) + update_time = time.time() + for operator in updated_dataset_metadata.topology.operators: + if operator.id in operator_states: + new_state = operator_states[operator.id] + if operator.state == new_state: + continue + operator.state = new_state + if new_state == DatasetState.RUNNING.name: + operator.execution_start_time = update_time + elif new_state in ( + DatasetState.FINISHED.name, + DatasetState.FAILED.name, + ): + operator.execution_end_time = update_time + # Handle outlier case for InputDataBuffer, which is marked as finished immediately and does not have a RUNNING state. + # Set the execution time the same as its end time + if not operator.execution_start_time: + operator.execution_start_time = update_time + + self.dataset_metadatas[dataset_id] = updated_dataset_metadata + self._metadata_exporter.export_dataset_metadata(updated_dataset_metadata) + def _create_tags( self, dataset_tag: str, @@ -823,26 +901,6 @@ def get_dataset_id_from_stats_actor(self) -> str: StatsManager = _StatsManager() -class DatasetState(enum.IntEnum): - """Enum representing the possible states of a dataset during execution.""" - - UNKNOWN = 0 - RUNNING = 1 - FINISHED = 2 - FAILED = 3 - - def __str__(self): - return self.name - - @classmethod - def from_string(cls, text): - """Get enum by name.""" - try: - return cls[text] # This uses the name to lookup the enum - except KeyError: - return cls.UNKNOWN - - class DatasetStats: """Holds the execution times for a given Dataset. diff --git a/python/ray/data/tests/test_state_export.py b/python/ray/data/tests/test_state_export.py index cfa815b32294..570927446d55 100644 --- a/python/ray/data/tests/test_state_export.py +++ b/python/ray/data/tests/test_state_export.py @@ -7,6 +7,7 @@ import ray from ray.data import DataContext +from ray.data._internal.execution.dataset_state import DatasetState from ray.data._internal.logical.interfaces import LogicalOperator from ray.data._internal.metadata_exporter import ( UNKNOWN, @@ -147,6 +148,9 @@ def dummy_dataset_topology(): uuid="uuid_0", input_dependencies=[], sub_stages=[], + execution_start_time=1.0, + execution_end_time=1.0, + state="FINISHED", args=sanitize_for_struct(dummy_operator._get_args()), ), Operator( @@ -155,6 +159,9 @@ def dummy_dataset_topology(): uuid="uuid_1", input_dependencies=["Input_0"], sub_stages=[], + execution_start_time=0.0, + execution_end_time=0.0, + state="RUNNING", args=sanitize_for_struct(dummy_operator._get_args()), ), ], @@ -244,6 +251,9 @@ def dummy_dataset_topology_expected_output(): }, "input_dependencies": [], "sub_stages": [], + "execution_start_time": 1.0, + "execution_end_time": 1.0, + "state": "FINISHED", }, { "name": "ReadRange->Map()->Filter()", @@ -323,6 +333,9 @@ def dummy_dataset_topology_expected_output(): }, }, "sub_stages": [], + "execution_start_time": 0.0, + "execution_end_time": 0.0, + "state": "RUNNING", }, ] } @@ -447,6 +460,9 @@ def test_export_multiple_datasets( uuid="second_uuid_0", input_dependencies=[], sub_stages=[], + execution_start_time=1.0, + execution_end_time=1.0, + state="FINISHED", ), Operator( name="ReadRange->Map()", @@ -454,6 +470,9 @@ def test_export_multiple_datasets( uuid="second_uuid_1", input_dependencies=["Input_0"], sub_stages=[], + execution_start_time=2.0, + execution_end_time=0.0, + state="RUNNING", ), ], ) @@ -613,6 +632,92 @@ def test_sanitize_for_struct(input_obj, expected_output, truncate_length): assert result == expected_output, f"Expected {expected_output}, got {result}" +def test_update_dataset_metadata_state( + ray_start_cluster_with_export_api_write, dummy_dataset_topology +): + """Test dataset state update at the export API""" + stats_actor = _get_or_create_stats_actor() + # Register dataset + ray.get( + stats_actor.register_dataset.remote( + job_id=STUB_JOB_ID, + dataset_tag=STUB_DATASET_ID, + operator_tags=["Input_0", "ReadRange->Map()->Filter()_1"], + topology=dummy_dataset_topology, + data_context=DataContext.get_current(), + ) + ) + # Check that export files were created as expected + data = _get_exported_data() + assert len(data) == 1 + assert data[0]["event_data"]["state"] == DatasetState.PENDING.name + + # Test update state to RUNNING + ray.get( + stats_actor.update_dataset_metadata_state.remote( + dataset_id=STUB_DATASET_ID, new_state=DatasetState.RUNNING.name + ) + ) + data = _get_exported_data() + assert len(data) == 2 + assert data[1]["event_data"]["state"] == DatasetState.RUNNING.name + assert data[1]["event_data"]["execution_start_time"] > 0 + + # Test update to FINISHED + ray.get( + stats_actor.update_dataset_metadata_state.remote( + dataset_id=STUB_DATASET_ID, new_state=DatasetState.FINISHED.name + ) + ) + data = _get_exported_data() + assert len(data) == 3 + assert data[2]["event_data"]["state"] == DatasetState.FINISHED.name + assert data[2]["event_data"]["execution_end_time"] > 0 + assert ( + data[2]["event_data"]["topology"]["operators"][1]["state"] + == DatasetState.FINISHED.name + ) + assert data[2]["event_data"]["topology"]["operators"][1]["execution_end_time"] > 0 + + +def test_update_dataset_metadata_operator_states( + ray_start_cluster_with_export_api_write, dummy_dataset_topology +): + stats_actor = _get_or_create_stats_actor() + # Register dataset + ray.get( + stats_actor.register_dataset.remote( + dataset_tag=STUB_DATASET_ID, + operator_tags=["Input_0", "ReadRange->Map()->Filter()_1"], + topology=dummy_dataset_topology, + job_id=STUB_JOB_ID, + data_context=DataContext.get_current(), + ) + ) + data = _get_exported_data() + assert len(data) == 1 + assert ( + data[0]["event_data"]["topology"]["operators"][1]["state"] + == DatasetState.RUNNING.name + ) + + # Test update to FINISHED + operator_id = "ReadRange->Map()->Filter()_1" + ray.get( + stats_actor.update_dataset_metadata_operator_states.remote( + dataset_id=STUB_DATASET_ID, + operator_states={operator_id: DatasetState.FINISHED.name}, + ) + ) + data = _get_exported_data() + assert len(data) == 2 + assert ( + data[1]["event_data"]["topology"]["operators"][1]["state"] + == DatasetState.FINISHED.name + ) + assert data[1]["event_data"]["topology"]["operators"][1]["execution_end_time"] > 0 + + if __name__ == "__main__": import sys diff --git a/src/ray/protobuf/export_dataset_metadata.proto b/src/ray/protobuf/export_dataset_metadata.proto index ce78d1458b8d..a508b5c3d186 100644 --- a/src/ray/protobuf/export_dataset_metadata.proto +++ b/src/ray/protobuf/export_dataset_metadata.proto @@ -30,6 +30,14 @@ message SubStage { // Represents a data processing operator in the DAG message Operator { + enum OperatorState { + UNKNOWN = 0; + RUNNING = 1; + FINISHED = 2; + FAILED = 3; + PENDING = 4; + } + // Name of the operator string name = 1; @@ -53,6 +61,15 @@ message Operator { // can be found in `_get_logical_args`, and is used to help understand how a // user's arguments lead to a dataset's state execution google.protobuf.Struct args = 6; + + // The timestamp when execution starts (in seconds since epoch) + double execution_start_time = 7; + + // The timestamp when execution ends (in seconds since epoch) + double execution_end_time = 8; + + // The state of the operator + OperatorState state = 9; } // Represents the complete structure of the operator DAG @@ -63,6 +80,14 @@ message Topology { // Top-level message containing full metadata about a Ray Data execution message ExportDatasetMetadata { + enum DatasetState { + UNKNOWN = 0; + RUNNING = 1; + FINISHED = 2; + FAILED = 3; + PENDING = 4; + } + // The operator DAG structure Topology topology = 1; @@ -72,9 +97,18 @@ message ExportDatasetMetadata { // The Ray Job ID string job_id = 3; - // The timestamp when execution started (in seconds since epoch) + // The timestamp when dataset is registered (in seconds since epoch) double start_time = 4; // The data context attached to the dataset. google.protobuf.Struct data_context = 5; + + // The timestamp when execution starts (in seconds since epoch) + double execution_start_time = 6; + + // The timestamp when execution ends (in seconds since epoch) + double execution_end_time = 7; + + // The state of the dataset + DatasetState state = 8; } From 400ea7716c50afe006ab69a5398fa5d3c2e08373 Mon Sep 17 00:00:00 2001 From: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:46:59 -0700 Subject: [PATCH 109/634] [serve.llm][docs] Documentation for prefix cache-aware router (#55218) Signed-off-by: Seiji Eicher Signed-off-by: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Co-authored-by: angelinalg <122562471+angelinalg@users.noreply.github.com> --- .../prefix_aware_example.py | 90 ++++++++++ doc/source/serve/llm/index.md | 3 +- .../serve/llm/prefix-aware-request-router.md | 157 ++++++++++++++++++ .../prefix_aware/prefix_aware_router.py | 12 ++ 4 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 doc/source/llm/doc_code/serve/prefix_aware_router/prefix_aware_example.py create mode 100644 doc/source/serve/llm/prefix-aware-request-router.md diff --git a/doc/source/llm/doc_code/serve/prefix_aware_router/prefix_aware_example.py b/doc/source/llm/doc_code/serve/prefix_aware_router/prefix_aware_example.py new file mode 100644 index 000000000000..bb6bf734fac2 --- /dev/null +++ b/doc/source/llm/doc_code/serve/prefix_aware_router/prefix_aware_example.py @@ -0,0 +1,90 @@ +""" +This file serves as a documentation example and CI test. + +Structure: +1. Monkeypatch setup: Ensures serve.run is non-blocking and removes accelerator requirements for CI testing. +2. Docs example (between __prefix_aware_example_start/end__): Embedded in Sphinx docs via literalinclude. +3. Test validation (deployment status polling + cleanup) +""" + +import time +from ray import serve +from ray.serve.schema import ApplicationStatus +from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME +from ray.serve import llm + +_original_serve_run = serve.run +_original_build_openai_app = llm.build_openai_app + + +def _non_blocking_serve_run(app, **kwargs): + """Forces blocking=False for testing""" + kwargs["blocking"] = False + return _original_serve_run(app, **kwargs) + + +def _testing_build_openai_app(llm_serving_args): + """Removes accelerator requirements for testing""" + for config in llm_serving_args["llm_configs"]: + config.accelerator_type = None + + return _original_build_openai_app(llm_serving_args) + + +serve.run = _non_blocking_serve_run +llm.build_openai_app = _testing_build_openai_app + +# __prefix_aware_example_start__ +from ray import serve +from ray.serve.llm import LLMConfig, build_openai_app +from ray.serve.llm.request_router import PrefixCacheAffinityRouter + +llm_config = LLMConfig( + model_loading_config={ + "model_id": "qwen-0.5b", + "model_source": "Qwen/Qwen2.5-0.5B-Instruct", + }, + deployment_config={ + "autoscaling_config": { + "min_replicas": 4, + "max_replicas": 4, + }, + "request_router_config": { + "request_router_class": PrefixCacheAffinityRouter, + "request_router_kwargs": { + "imbalanced_threshold": 5, # More aggressive load balancing + "match_rate_threshold": 0.15, # Require 15% match rate + "do_eviction": True, # Enable memory management + "eviction_threshold_chars": 500_000, + "eviction_target_chars": 400_000, + "eviction_interval_secs": 30, + }, + }, + }, + runtime_env={"env_vars": {"VLLM_USE_V1": "1"}}, +) + +app = build_openai_app({"llm_configs": [llm_config]}) +serve.run(app, blocking=True) +# __prefix_aware_example_end__ + +status = ApplicationStatus.NOT_STARTED +timeout_seconds = 180 +start_time = time.time() + +while ( + status != ApplicationStatus.RUNNING and time.time() - start_time < timeout_seconds +): + status = serve.status().applications[SERVE_DEFAULT_APP_NAME].status + + if status in [ApplicationStatus.DEPLOY_FAILED, ApplicationStatus.UNHEALTHY]: + raise AssertionError(f"Deployment failed with status: {status}") + + time.sleep(1) + +if status != ApplicationStatus.RUNNING: + raise AssertionError( + f"Deployment failed to reach RUNNING status within {timeout_seconds}s. Current status: {status}" + ) + +serve.shutdown() diff --git a/doc/source/serve/llm/index.md b/doc/source/serve/llm/index.md index 358704b9d101..0cf2a27aa989 100644 --- a/doc/source/serve/llm/index.md +++ b/doc/source/serve/llm/index.md @@ -56,6 +56,5 @@ The LLMConfig class specifies model details such as: Quickstart Prefill/Decode Disaggregation +Cache-aware request routing ``` - - diff --git a/doc/source/serve/llm/prefix-aware-request-router.md b/doc/source/serve/llm/prefix-aware-request-router.md new file mode 100644 index 000000000000..d8a73e599fa7 --- /dev/null +++ b/doc/source/serve/llm/prefix-aware-request-router.md @@ -0,0 +1,157 @@ +(prefix-aware-request-router-guide)= +# `PrefixCacheAffinityRouter` for LLM inference optimization + +:::{warning} +This API is in alpha and may change before becoming stable. +::: + +LLM inference can benefit significantly from cache locality optimization. When one replica processes multiple prompts that share a prefix, the engine can reuse previously computed KV-cache entries, reducing computation overhead and improving response times. This technique is known as [Automatic Prefix Caching (APC)](https://docs.vllm.ai/en/stable/features/automatic_prefix_caching.html) in vLLM. The `PrefixCacheAffinityRouter` is designed specifically for this use case. + +This guide covers: +- Understanding the prefix cache-aware routing algorithm +- Building the components of a prefix-aware router +- Configuration parameters and their impact + +(prefix-aware-algorithm)= +## How Ray Serve LLM prefix cache-aware routing works + +The `PrefixCacheAffinityRouter` implements a multi-tier routing strategy that balances cache locality with load distribution: + +### 1. Load balance check +First, it evaluates whether the current load is balanced across replicas by comparing queue lengths. If the difference between the highest and lowest queue lengths is below the `imbalanced_threshold`, it proceeds with prefix cache-aware routing. + +### 2. Prefix matching strategy +When load is balanced, the router uses a prefix tree to find replicas that have previously processed similar input text: + +- **High Match Rate (≥10%)**: Routes to replicas with the highest prefix match rate for better cache hit rates +- **Low Match Rate (<10%)**: Falls back to replicas with the lowest prefix cache utilization to increase utilization +- **No Prefix Data**: Uses the default Power of Two Choices selection + +### 3. Imbalanced load fallback +When load is imbalanced (queue length difference exceeds threshold), the router prioritizes load balancing over cache locality and falls back to the standard Power of Two Choices algorithm. + +### Prefix tree management +The router maintains a distributed prefix tree actor that: +- Tracks input text prefixes processed by each replica +- Supports automatic eviction of old entries to manage memory usage +- Persists across router instances using Ray's detached actor pattern + +(building-prefix-aware-components)= +## Building prefix-aware router components + +This section breaks down the key components of `PrefixCacheAffinityRouter` and shows how they work together. For a more basic example, see {ref}`custom-request-router-guide`. + +### Base RequestRouter foundation + +Like all custom routers in Ray Serve, the `PrefixCacheAffinityRouter` extends the base [`RequestRouter`](../api/doc/ray.serve.request_router.RequestRouter.rst) class. The two core methods that define router behavior are: + +- **`choose_replicas()`**: The main routing logic that selects which replicas should handle a request +- **`on_request_routed()`**: A callback that updates router state after a request is successfully routed + +For a detailed explanation of these methods and their parameters, see the {ref}`simple-uniform-request-router` example in the custom request router guide. + +### 1. Load balance detection component + +The first component evaluates whether the current load is balanced across replicas: + +```{literalinclude} ../../../../python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py +:start-after: __begin_load_balance_component__ +:end-before: __end_load_balance_component__ +:language: python +:caption: prefix_aware_router.py +``` + +This component prioritizes load balancing over cache locality when replicas become too imbalanced. + + +### 2. Prefix tree management component + +The prefix tree component is implemented as a detached Ray actor that manages prefix tracking across the Serve application. The actual tree structure uses a multi-tenant prefix tree (approximate radix tree). + +This distributed architecture allows the prefix information to persist across router restarts and be shared among multiple router instances. + +### 3. Prefix matching logic component + +The core prefix matching component implements the routing decision logic in the `_prefix_match_best_replicas` method. When load is balanced, it performs prefix matching to find the best replica: + +```{literalinclude} ../../../../python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py +:start-after: __begin_prefix_match_component__ +:end-before: __end_prefix_match_component__ +:language: python +:caption: prefix_aware_router.py +``` + +This logic implements the three-tier strategy: +1. **High match rate**: Routes to replicas with the highest prefix match when `match_rate >= match_rate_threshold` +2. **Low match rate**: Falls back to replicas with smallest KV-cache usage when match rate is below threshold +3. **No match**: Fall back to default Power of Two Choices selection when `_prefix_match_best_replicas` returns to `choose_replicas`. + +### 4. Integration with Power of Two choices + +The prefix-aware router extends the proven Power of Two Choices algorithm, falling back to it when prefix-based routing would degenerate. `PrefixCacheAffinityRouter` integrates this component in the `choose_replicas` method: + +```{literalinclude} ../../../../python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py +:start-after: __begin_pow2_router_base__ +:end-before: __end_pow2_router_base__ +:language: python +:caption: prefix_aware_router.py +``` + + +### 5. State management and callbacks + +The router uses the `on_request_routed()` callback to update the prefix tree with routing decisions: + +```{literalinclude} ../../../../python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py +:start-after: __begin_on_request_routed__ +:end-before: __end_on_request_routed__ +:language: python +:caption: prefix_aware_router.py +``` + +When a replica dies, the router uses the `on_replica_actor_died` callback to remove the replica's entries from the shared prefix tree: +```{literalinclude} ../../../../python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py +:start-after: __begin_on_replica_actor_died__ +:end-before: __end_on_replica_actor_died__ +:language: python +:caption: prefix_aware_router.py +``` + +(mixin-components)= +## Mixin components + +The `PrefixCacheAffinityRouter` inherits from two mixins. For more details about these and other available mixins, see {ref}`utility-mixin`. The router uses these mixins to optimize the list of candidate replicas against which it calculates prefix cache hit rate. + +The [`LocalityMixin`](../api/doc/ray.serve.request_router.LocalityMixin.rst) provides locality-aware routing to optimize network latency by preferring replicas on the same node. The [`MultiplexMixin`](../api/doc/ray.serve.request_router.MultiplexMixin.rst) enables model multiplexing support by tracking which models are loaded on each replica and routing requests to replicas that already have the requested model in memory. + +## Configuration parameters + +The `PrefixCacheAffinityRouter` provides several configuration parameters to tune its behavior: + +### Core routing parameters + +- **`imbalanced_threshold`** (default: 10): Queue length difference threshold for considering load balanced. Lower values prioritize load balancing over cache locality. + +- **`match_rate_threshold`** (default: 0.1): Minimum prefix match rate (0.0-1.0) required to use prefix cache-aware routing. Higher values require stronger prefix matches before routing for cache locality. + +### Memory management parameters + +- **`do_eviction`** (default: False): Enable automatic eviction of old prefix tree entries to approximate the LLM engine's eviction policy. + +- **`eviction_threshold_chars`** (default: 400,000): Maximum number of characters in the prefix tree before the LLM engine triggers an eviction. + +- **`eviction_target_chars`** (default: 360,000): Target number of characters to reduce the prefix tree to during eviction. + +- **`eviction_interval_secs`** (default: 10): Interval in seconds between eviction checks for when eviction is enabled. + +(deploy-llm-with-prefix-aware-router)= +## Deploying LLM applications with Prefix Cache-Aware Routing + +Deploy an LLM application using the prefix cache-aware request router as follows: + +```{literalinclude} ../../llm/doc_code/serve/prefix_aware_router/prefix_aware_example.py +:start-after: __prefix_aware_example_start__ +:end-before: __prefix_aware_example_end__ +:language: python +:caption: prefix_aware_example.py +``` diff --git a/python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py b/python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py index 3fb897753ad4..a3904e70398d 100644 --- a/python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py +++ b/python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py @@ -167,6 +167,7 @@ async def _prefix_match_best_replicas( ): input_text = self._extract_text_from_request(pending_request) if input_text is not None: + # Start Sphinx tag: __begin_load_balance_component__ # Check for imbalanced load. highest_queue_len = 0 lowest_queue_len = float("inf") @@ -195,6 +196,8 @@ async def _prefix_match_best_replicas( is_imbalanced = ( highest_queue_len - lowest_queue_len > self._imbalanced_threshold ) + # End Sphinx tag: __end_load_balance_component__ + # Start Sphinx tag: __begin_prefix_match_component__ if not is_imbalanced: # Convert candidate replica IDs to strings for prefix matching. candidate_replica_ids_strings = [ @@ -221,6 +224,7 @@ async def _prefix_match_best_replicas( and len(matched_tenant_id_strings) > 0 ): chosen_replica_id_strings = matched_tenant_id_strings + # End Sphinx tag: __end_prefix_match_component__ return [ [ self._replicas[ReplicaID.from_full_id_str(chosen_id_string)] @@ -228,11 +232,14 @@ async def _prefix_match_best_replicas( ] ] + # Start Sphinx tag: __begin_on_replica_actor_died__ def on_replica_actor_died(self, replica_id: ReplicaID): """Drop replica from replica set so it's not considered for future requests.""" super().on_replica_actor_died(replica_id) ray.get(self._tree_actor.remove_tenants.remote([replica_id.to_full_id_str()])) + # End Sphinx tag: __end_on_replica_actor_died__ + def update_replicas(self, replicas: List[RunningReplica]): """Update the set of available replicas to be considered for routing. @@ -283,6 +290,7 @@ async def choose_replicas( model ID are available after that timeout, it will fall back to the regular procedure. """ + # Start Sphinx tag: __begin_pow2_router_base__ # Get fallback replicas from PowerOfTwoChoicesRequestRouter fallback_replicas = await PowerOfTwoChoicesRequestRouter.choose_replicas( self, @@ -291,6 +299,7 @@ async def choose_replicas( ) if pending_request is None or not fallback_replicas: return fallback_replicas + # End Sphinx tag: __end_pow2_router_base__ if ( pending_request is not None @@ -324,6 +333,7 @@ async def choose_replicas( return fallback_replicas + # Start Sphinx tag: __begin_on_request_routed__ def on_request_routed( self, pending_request: PendingRequest, @@ -349,3 +359,5 @@ def on_request_routed( input_text, replica_id.to_full_id_str(), time.time() ) ) + + # End Sphinx tag: __end_on_request_routed__ From 37158a22a44edb10d499b53d1f38f00315234a14 Mon Sep 17 00:00:00 2001 From: harshit-anyscale Date: Fri, 15 Aug 2025 00:21:29 +0530 Subject: [PATCH 110/634] skip test task processor for windows (#55616) - skipping test task processor for windows to unblock Signed-off-by: harshit --- python/ray/serve/tests/test_task_processor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ray/serve/tests/test_task_processor.py b/python/ray/serve/tests/test_task_processor.py index 171c67df8477..d31192d2d0b5 100644 --- a/python/ray/serve/tests/test_task_processor.py +++ b/python/ray/serve/tests/test_task_processor.py @@ -89,6 +89,7 @@ def _create(**kwargs): return _create +@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.") class TestTaskConsumerWithRayServe: """Test task consumer integration with Ray Serve.""" From 49d336cb332da4cdfff894e95ea6f0189f1b05ff Mon Sep 17 00:00:00 2001 From: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:53:36 -0700 Subject: [PATCH 111/634] [Serve.llm] Improve PrefixCacheAffinityRouter text normalization compat (#55588) Signed-off-by: Seiji Eicher --- .../prefix_aware/prefix_aware_router.py | 46 ++++++++++-- .../test_prefix_aware_request_router.py | 75 +++++++++++++++++++ 2 files changed, 115 insertions(+), 6 deletions(-) diff --git a/python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py b/python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py index a3904e70398d..6d3027b0bef5 100644 --- a/python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py +++ b/python/ray/llm/_internal/serve/request_router/prefix_aware/prefix_aware_router.py @@ -2,6 +2,7 @@ import logging import time from typing import ( + Any, List, Optional, ) @@ -139,14 +140,47 @@ def _extract_text_from_request(self, pending_request: PendingRequest) -> str: "No request with message or prompt attribute found in pending_request.args" ) - # Convert list of messages to concatenated string + return self._normalize_prompt_to_string(prompt) + + def _coerce_to_text(self, value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, list): + return "".join(self._coerce_to_text(item) for item in value) + if isinstance(value, dict): + text_value = value.get("text") + if isinstance(text_value, str): + return text_value + if "content" in value: + return self._coerce_to_text(value["content"]) + + return "" + + def _normalize_prompt_to_string(self, prompt: Any) -> str: + """Normalize prompt/messages a single string of characters. + This is not exhaustive (e.g. thinking parts, multimodal are not supported). + TODO(seiji): find a more maintainable way to normalize the prompt/messages. + + Supported: + - string → return as-is + - list of strings → concat + - list of message dicts with 'content' as string → concat + - list of message dicts with 'content' as list of dicts → concat the 'text' fields from those parts + """ + if isinstance(prompt, str): + return prompt + if isinstance(prompt, list): - concatenated_messages = "".join( - msg.get("content", "") for msg in prompt if "content" in msg + return "".join( + self._coerce_to_text( + message.get("content") if isinstance(message, dict) else message + ) + for message in prompt ) - return concatenated_messages - else: - return prompt + + return "" async def _prefix_match_best_replicas( self, diff --git a/python/ray/llm/tests/serve/cpu/deployments/test_prefix_aware_request_router.py b/python/ray/llm/tests/serve/cpu/deployments/test_prefix_aware_request_router.py index a3ac2ced6185..5d28ef464b06 100644 --- a/python/ray/llm/tests/serve/cpu/deployments/test_prefix_aware_request_router.py +++ b/python/ray/llm/tests/serve/cpu/deployments/test_prefix_aware_request_router.py @@ -284,6 +284,74 @@ async def test_eviction_task_creation(self, prefix_request_router): ray.get(prefix_request_router._tree_actor.stop_eviction_loop.remote()) await asyncio.sleep(0.1) + +class TestPromptNormalization: + """Tests for input normalization in the prefix-aware router.""" + + def test_normalize_prompt_string(self, prefix_request_router): + req = fake_pending_request(prompt="Hello world") + normalized = prefix_request_router._extract_text_from_request(req) + assert normalized == "Hello world" + + def test_normalize_messages_list_of_strings(self, prefix_request_router): + req = fake_pending_request(messages=["Hello", " ", "world"]) + normalized = prefix_request_router._extract_text_from_request(req) + assert normalized == "Hello world" + + def test_normalize_messages_dict_content_string(self, prefix_request_router): + req = fake_pending_request( + messages=[ + {"content": "Hello"}, + {"content": " world"}, + ] + ) + normalized = prefix_request_router._extract_text_from_request(req) + assert normalized == "Hello world" + + def test_normalize_messages_dict_content_list_of_dicts_text( + self, prefix_request_router + ): + req = fake_pending_request( + messages=[ + { + "content": [ + {"type": "text", "text": "Hello"}, + {"type": "text", "text": " world"}, + ] + } + ] + ) + normalized = prefix_request_router._extract_text_from_request(req) + assert normalized == "Hello world" + + def test_normalize_messages_dict_content_list_of_strings( + self, prefix_request_router + ): + req = fake_pending_request(messages=[{"content": ["Hello", " ", "world"]}]) + normalized = prefix_request_router._extract_text_from_request(req) + assert normalized == "Hello world" + + def test_normalize_unsupported_returns_empty(self, prefix_request_router): + # For now, unsupported multimodal parts should be ignored, resulting in empty string + req = fake_pending_request( + messages=[ + { + "content": [ + { + "type": "image_url", + "image_url": {"url": "http://example.com"}, + }, + ] + } + ] + ) + normalized = prefix_request_router._extract_text_from_request(req) + assert normalized == "" + + def test_extract_raises_when_no_prompt_or_messages(self, prefix_request_router): + with pytest.raises(ValueError): + _ = prefix_request_router._extract_text_from_request(fake_pending_request()) + @pytest.mark.asyncio @pytest.mark.parametrize( "prefix_request_router", @@ -331,3 +399,10 @@ async def test_eviction_threshold_behavior(self, prefix_request_router): ray.get(prefix_request_router._tree_actor.stop_eviction_loop.remote()) await asyncio.sleep(0.1) + + +if __name__ == "__main__": + import sys + + exit_code = pytest.main(["-vs", __file__]) + sys.exit(exit_code) From 61bc2e8139e21429d487b0824391c26dcd596cc3 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:56:37 -0700 Subject: [PATCH 112/634] [ci] read gce credentials file from global config when building anyscale images (#55580) rather than using the hard-coded filename Signed-off-by: Lonnie Liu Signed-off-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> --- ci/ray_ci/anyscale_docker_container.py | 4 +++- ci/ray_ci/test_anyscale_docker_container.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/ray_ci/anyscale_docker_container.py b/ci/ray_ci/anyscale_docker_container.py index 7144d065a1e1..ccf847590ceb 100644 --- a/ci/ray_ci/anyscale_docker_container.py +++ b/ci/ray_ci/anyscale_docker_container.py @@ -1,5 +1,6 @@ from ci.ray_ci.container import _DOCKER_ECR_REPO, _DOCKER_GCP_REGISTRY from ci.ray_ci.docker_container import DockerContainer +from ray_release.configs.global_config import get_global_config class AnyscaleDockerContainer(DockerContainer): @@ -18,12 +19,13 @@ def run(self) -> None: anyscale_image = f"{aws_registry}/anyscale/{self.image_type}:{tag}" requirement = self._get_requirement_file() + gce_credentials = get_global_config()["aws2gce_credentials"] cmds = [ # build docker image "./ci/build/build-anyscale-docker.sh " + f"{ray_image} {anyscale_image} {requirement} {aws_registry}", # gcloud login - "./release/gcloud_docker_login.sh release/aws2gce_iam.json", + f"./release/gcloud_docker_login.sh {gce_credentials}", "export PATH=$(pwd)/google-cloud-sdk/bin:$PATH", ] # TODO(can): remove the alias when release test infra uses only the canonical diff --git a/ci/ray_ci/test_anyscale_docker_container.py b/ci/ray_ci/test_anyscale_docker_container.py index 52b275ddafb8..192fcfa2c051 100644 --- a/ci/ray_ci/test_anyscale_docker_container.py +++ b/ci/ray_ci/test_anyscale_docker_container.py @@ -8,6 +8,7 @@ from ci.ray_ci.anyscale_docker_container import AnyscaleDockerContainer from ci.ray_ci.container import _DOCKER_ECR_REPO, _DOCKER_GCP_REGISTRY from ci.ray_ci.test_base import RayCITestBase +from ray_release.configs.global_config import get_global_config class TestAnyscaleDockerContainer(RayCITestBase): @@ -32,6 +33,7 @@ def _mock_run_script(input: List[str]) -> None: aws_ecr = _DOCKER_ECR_REPO.split("/")[0] aws_prj = f"{aws_ecr}/anyscale/ray-ml" gcp_prj = f"{_DOCKER_GCP_REGISTRY}/anyscale/ray-ml" + gce_credentials = get_global_config()["aws2gce_credentials"] tags_want = [ f"123456-{pv}-cu121", @@ -57,7 +59,7 @@ def _mock_run_script(input: List[str]) -> None: "./ci/build/build-anyscale-docker.sh " f"rayproject/ray-ml:123456-{pv}-cu121 " f"{aws_prj}:123456-{pv}-cu121 requirements_ml_byod_{v}.txt {aws_ecr}", - "./release/gcloud_docker_login.sh release/aws2gce_iam.json", + f"./release/gcloud_docker_login.sh {gce_credentials}", "export PATH=$(pwd)/google-cloud-sdk/bin:$PATH", ] + push_cmds_want From fc4ace25a81cf68b71e21c00f1be2532d5c6c148 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 14 Aug 2025 13:59:45 -0700 Subject: [PATCH 113/634] [release] Script to build custom BYOD image (#55577) Add `custom_byod_build` as a python binary that the Buildkite jobs can call to build & push custom BYOD images --------- Signed-off-by: kevin --- release/BUILD.bazel | 27 ++++++++ .../ray_release/scripts/custom_byod_build.py | 14 +++++ .../tests/test_custom_byod_build.py | 63 +++++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 release/ray_release/scripts/custom_byod_build.py create mode 100644 release/ray_release/tests/test_custom_byod_build.py diff --git a/release/BUILD.bazel b/release/BUILD.bazel index a7f8320bd23a..a726911feac2 100644 --- a/release/BUILD.bazel +++ b/release/BUILD.bazel @@ -438,6 +438,24 @@ py_test( ], ) +py_test( + name = "test_custom_byod_build", + size = "small", + srcs = ["ray_release/tests/test_custom_byod_build.py"], + data = [ + "ray_release/configs/oss_config.yaml", + ], + exec_compatible_with = ["//:hermetic_python"], + tags = [ + "release_unit", + "team:ci", + ], + deps = [ + ":ray_release", + bk_require("pytest"), + ], +) + py_test( name = "test_cluster_manager", size = "small", @@ -688,3 +706,12 @@ py_binary( ":ray_release", ], ) + +py_binary( + name = "custom_byod_build", + srcs = ["ray_release/scripts/custom_byod_build.py"], + exec_compatible_with = ["//:hermetic_python"], + deps = [ + ":ray_release", + ], +) diff --git a/release/ray_release/scripts/custom_byod_build.py b/release/ray_release/scripts/custom_byod_build.py new file mode 100644 index 000000000000..c773b46ff5ca --- /dev/null +++ b/release/ray_release/scripts/custom_byod_build.py @@ -0,0 +1,14 @@ +import click +from ray_release.byod.build import build_anyscale_custom_byod_image + + +@click.command() +@click.option("--image-name", type=str, required=True) +@click.option("--base-image", type=str, required=True) +@click.option("--post-build-script", type=str, required=True) +def main(image_name: str, base_image: str, post_build_script: str): + build_anyscale_custom_byod_image(image_name, base_image, post_build_script) + + +if __name__ == "__main__": + main() diff --git a/release/ray_release/tests/test_custom_byod_build.py b/release/ray_release/tests/test_custom_byod_build.py new file mode 100644 index 000000000000..7e29a352096d --- /dev/null +++ b/release/ray_release/tests/test_custom_byod_build.py @@ -0,0 +1,63 @@ +import sys +import pytest +from unittest.mock import patch + +from click.testing import CliRunner +from ray_release.scripts.custom_byod_build import main + + +@patch("ray_release.scripts.custom_byod_build.build_anyscale_custom_byod_image") +def test_custom_byod_build(mock_build_anyscale_custom_byod_image): + mock_build_anyscale_custom_byod_image.return_value = None + runner = CliRunner() + result = runner.invoke( + main, + [ + "--image-name", + "test-image", + "--base-image", + "test-base-image", + "--post-build-script", + "test_post_build_script.sh", + ], + ) + assert result.exit_code == 0 + + +@patch("ray_release.scripts.custom_byod_build.build_anyscale_custom_byod_image") +def test_custom_byod_build_missing_arg(mock_build_anyscale_custom_byod_image): + mock_build_anyscale_custom_byod_image.return_value = None + runner = CliRunner() + result = runner.invoke( + main, + [ + "--base-image", + "test-base-image", + "--post-build-script", + "test_post_build_script.sh", + ], + ) + assert result.exit_code == 2 + assert "Error: Missing option '--image-name'" in result.output + + result = runner.invoke( + main, + [ + "--image-name", + "test-image", + "--post-build-script", + "test_post_build_script.sh", + ], + ) + assert result.exit_code == 2 + assert "Error: Missing option '--base-image'" in result.output + + result = runner.invoke( + main, ["--image-name", "test-image", "--base-image", "test-base-image"] + ) + assert result.exit_code == 2 + assert "Error: Missing option '--post-build-script'" in result.output + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", __file__])) From 1c55991ce455632e1ab9839cb4c25f3e4ddc379c Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Thu, 14 Aug 2025 14:10:44 -0700 Subject: [PATCH 114/634] [core][otel] change+simplify the feature flag for open telemetry (#55592) Change and simplify the feature flag to enable open telemetry. This will enable us to enable open telemetry for the next Ray release version, without worrying about messing up previous Ray release versions. Test: - CI Signed-off-by: Cuong Nguyen --- python/ray/_private/ray_constants.py | 17 ++------ .../modules/reporter/reporter_agent.py | 7 ++-- .../modules/reporter/tests/test_reporter.py | 6 +-- python/ray/tests/BUILD | 3 +- python/ray/tests/test_metric_cardinality.py | 13 ++----- python/ray/tests/test_metrics_agent.py | 16 ++------ python/ray/util/metrics.py | 2 +- release/release_tests.yaml | 39 +++++++------------ src/ray/common/ray_config_def.h | 13 ++----- src/ray/stats/metric.cc | 2 +- src/ray/stats/metric.h | 2 +- src/ray/stats/stats.h | 16 ++++++-- src/ray/stats/tests/BUILD.bazel | 2 +- 13 files changed, 49 insertions(+), 89 deletions(-) diff --git a/python/ray/_private/ray_constants.py b/python/ray/_private/ray_constants.py index bad2a4585938..01b194e62696 100644 --- a/python/ray/_private/ray_constants.py +++ b/python/ray/_private/ray_constants.py @@ -578,20 +578,9 @@ def gcs_actor_scheduling_enabled(): # Defaults to False to use pynvml to collect usage. RAY_METRIC_ENABLE_GPU_NVSMI = env_bool("RAY_metric_enable_gpu_nvsmi", False) -# Whether enable OpenTelemetry as the metrics collection backend on the driver -# component. This flag is only used during the migration of the metric collection -# backend from OpenCensus to OpenTelemetry. It will be removed in the future. -RAY_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY_ON_AGENT = env_bool( - "RAY_experimental_enable_open_telemetry_on_agent", False -) - -# Whether enable OpenTelemetry as the metrics collection backend on the core -# components (core workers, gcs server, raylet, etc.). This flag is only used during -# the migration of the metric collection backend from OpenCensus to OpenTelemetry. -# It will be removed in the future. -RAY_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY_ON_CORE = env_bool( - "RAY_experimental_enable_open_telemetry_on_core", False -) +# Whether enable OpenTelemetry as the metrics collection backend. The default is +# using OpenCensus. +RAY_ENABLE_OPEN_TELEMETRY = env_bool("RAY_enable_open_telemetry", False) # How long to wait for a fetch to complete during ray.get before timing out and raising an exception to the user. # diff --git a/python/ray/dashboard/modules/reporter/reporter_agent.py b/python/ray/dashboard/modules/reporter/reporter_agent.py index 835a5b1be6a8..16355729011f 100644 --- a/python/ray/dashboard/modules/reporter/reporter_agent.py +++ b/python/ray/dashboard/modules/reporter/reporter_agent.py @@ -41,8 +41,7 @@ from ray._private.metrics_agent import Gauge, MetricsAgent, Record from ray._private.ray_constants import ( DEBUG_AUTOSCALING_STATUS, - RAY_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY_ON_AGENT, - RAY_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY_ON_CORE, + RAY_ENABLE_OPEN_TELEMETRY, env_integer, ) from ray._private.telemetry.open_telemetry_metric_recorder import ( @@ -1734,7 +1733,7 @@ def _compose_stats_payload( records = self._to_records(stats, cluster_stats) - if RAY_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY_ON_AGENT: + if RAY_ENABLE_OPEN_TELEMETRY: self._open_telemetry_metric_recorder.record_and_export( records, global_tags={ @@ -1758,7 +1757,7 @@ def _compose_stats_payload( async def run(self, server): if server: reporter_pb2_grpc.add_ReporterServiceServicer_to_server(self, server) - if RAY_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY_ON_CORE: + if RAY_ENABLE_OPEN_TELEMETRY: metrics_service_pb2_grpc.add_MetricsServiceServicer_to_server( self, server ) diff --git a/python/ray/dashboard/modules/reporter/tests/test_reporter.py b/python/ray/dashboard/modules/reporter/tests/test_reporter.py index 1c4ee8a0f8d0..9390275b0297 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_reporter.py +++ b/python/ray/dashboard/modules/reporter/tests/test_reporter.py @@ -200,11 +200,11 @@ def enable_open_telemetry(request): Fixture to enable OpenTelemetry for the test. """ if request.param: - os.environ["RAY_experimental_enable_open_telemetry_on_agent"] = "1" + os.environ["RAY_enable_open_telemetry"] = "1" else: - os.environ["RAY_experimental_enable_open_telemetry_on_agent"] = "0" + os.environ["RAY_enable_open_telemetry"] = "0" yield - os.environ.pop("RAY_experimental_enable_open_telemetry_on_agent", None) + os.environ.pop("RAY_enable_open_telemetry", None) @pytest.mark.skipif(prometheus_client is None, reason="prometheus_client not installed") diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD index 46dccaa12c0e..f4e604144df9 100644 --- a/python/ray/tests/BUILD +++ b/python/ray/tests/BUILD @@ -83,8 +83,7 @@ py_test_module_list( py_test_module_list( size = "medium", env = { - "RAY_experimental_enable_open_telemetry_on_agent": "1", - "RAY_experimental_enable_open_telemetry_on_core": "1", + "RAY_enable_open_telemetry": "1", }, files = [ "test_metric_cardinality.py", diff --git a/python/ray/tests/test_metric_cardinality.py b/python/ray/tests/test_metric_cardinality.py index 426daaaa2181..c7b429cdef9d 100644 --- a/python/ray/tests/test_metric_cardinality.py +++ b/python/ray/tests/test_metric_cardinality.py @@ -39,14 +39,7 @@ def _setup_cluster_for_test(request, ray_start_cluster): _system_config={ "enable_metrics_collection": True, "metric_cardinality_level": core_metric_cardinality_level, - "experimental_enable_open_telemetry_on_agent": os.getenv( - "RAY_experimental_enable_open_telemetry_on_agent" - ) - == "1", - "experimental_enable_open_telemetry_on_core": os.getenv( - "RAY_experimental_enable_open_telemetry_on_core" - ) - == "1", + "enable_open_telemetry": os.getenv("RAY_enable_open_telemetry") == "1", } ) cluster.wait_for_nodes() @@ -152,8 +145,8 @@ def test_cardinality_recommended_and_legacy_levels( # implementation doesn't support low cardinality. @pytest.mark.skipif(prometheus_client is None, reason="Prometheus not installed") @pytest.mark.skipif( - os.getenv("RAY_experimental_enable_open_telemetry_on_agent") != "1", - reason="OpenTelemetry is not enabled on agent", + os.getenv("RAY_enable_open_telemetry") != "1", + reason="OpenTelemetry is not enabled", ) @pytest.mark.parametrize( "_setup_cluster_for_test,cardinality_level", diff --git a/python/ray/tests/test_metrics_agent.py b/python/ray/tests/test_metrics_agent.py index a167b701012b..4ce83269cc69 100644 --- a/python/ray/tests/test_metrics_agent.py +++ b/python/ray/tests/test_metrics_agent.py @@ -196,14 +196,7 @@ def _setup_cluster_for_test(request, ray_start_cluster): "event_stats_print_interval_ms": 500, "event_stats": True, "enable_metrics_collection": enable_metrics_collection, - "experimental_enable_open_telemetry_on_agent": os.getenv( - "RAY_experimental_enable_open_telemetry_on_agent" - ) - == "1", - "experimental_enable_open_telemetry_on_core": os.getenv( - "RAY_experimental_enable_open_telemetry_on_core" - ) - == "1", + "enable_open_telemetry": os.getenv("RAY_enable_open_telemetry") == "1", } ) # Add worker nodes. @@ -277,8 +270,7 @@ async def ping(self): @pytest.mark.skipif(prometheus_client is None, reason="Prometheus not installed") @pytest.mark.skipif( - os.environ.get("RAY_experimental_enable_open_telemetry_on_core") == "1" - and sys.platform == "darwin", + os.environ.get("RAY_enable_open_telemetry") == "1" and sys.platform == "darwin", reason="OpenTelemetry is not working on macOS yet.", ) @pytest.mark.parametrize("_setup_cluster_for_test", [True], indirect=True) @@ -319,7 +311,7 @@ def test_cases(): "test_driver_counter_total", "test_gauge", ] - if os.environ.get("RAY_experimental_enable_open_telemetry_on_core") != "1" + if os.environ.get("RAY_enable_open_telemetry") != "1" else [ "test_counter_total", "test_driver_counter_total", @@ -770,7 +762,7 @@ def wrap_test_case_for_retry(): @pytest.mark.skipif(sys.platform == "win32", reason="Not working in Windows.") @pytest.mark.skipif( - os.environ.get("RAY_experimental_enable_open_telemetry_on_core") == "1", + os.environ.get("RAY_enable_open_telemetry") == "1", reason="OpenTelemetry backend does not support Counter exported as gauge.", ) def test_counter_exported_as_gauge(shutdown_only): diff --git a/python/ray/util/metrics.py b/python/ray/util/metrics.py index ec1988044b65..614dd34d41fa 100644 --- a/python/ray/util/metrics.py +++ b/python/ray/util/metrics.py @@ -191,7 +191,7 @@ def __init__( if self._discard_metric: self._metric = None else: - if os.environ.get("RAY_experimental_enable_open_telemetry_on_core") == "1": + if os.environ.get("RAY_enable_open_telemetry") == "1": """ For the new opentelemetry implementation, we'll correctly use Counter rather than Sum. diff --git a/release/release_tests.yaml b/release/release_tests.yaml index a854d73267af..df48adcc55d7 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -2858,8 +2858,7 @@ cluster: byod: runtime_env: - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: stress_tests/placement_group_tests_compute.yaml run: @@ -2933,8 +2932,7 @@ cluster: byod: runtime_env: - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: tpl_64.yaml run: @@ -3215,8 +3213,7 @@ cluster: byod: runtime_env: - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: stress_tests/stress_tests_compute.yaml run: @@ -3258,8 +3255,7 @@ cluster: byod: runtime_env: - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: stress_tests/stress_tests_compute.yaml run: @@ -3393,8 +3389,7 @@ cluster: byod: runtime_env: - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: stress_tests/stress_tests_single_node_oom_compute.yaml run: @@ -3558,8 +3553,7 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: single_node.yaml run: @@ -3587,8 +3581,7 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: object_store.yaml run: @@ -3617,8 +3610,7 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: object_store/small_objects.yaml run: @@ -3642,8 +3634,7 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: object_store/large_objects.yaml run: @@ -3667,8 +3658,7 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: distributed.yaml run: @@ -3717,8 +3707,7 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: distributed.yaml run: @@ -3747,8 +3736,7 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: distributed.yaml run: @@ -3798,8 +3786,7 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_experimental_enable_open_telemetry_on_agent=1 - - RAY_experimental_enable_open_telemetry_on_core=1 + - RAY_enable_open_telemetry=1 cluster_compute: many_nodes.yaml run: diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index 5aa0e3ef306e..313446827d30 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -520,16 +520,9 @@ RAY_CONFIG(bool, enable_metrics_collection, true) /// RAY_METRIC_CARDINALITY_LEVEL in ray_constants.py RAY_CONFIG(std::string, metric_cardinality_level, "legacy") -/// Whether enable OpenTelemetry as the metrics collection backend on the driver -/// component. This flag is only used during the migration of the metric collection -/// backend from OpenCensus to OpenTelemetry. It will be removed in the future. -RAY_CONFIG(bool, experimental_enable_open_telemetry_on_agent, false) - -/// Whether enable OpenTelemetry as the metrics collection backend on the core -/// components (core workers, gcs server, raylet, etc.). This flag is only used during -/// the migration of the metric collection backend from OpenCensus to OpenTelemetry. -/// It will be removed in the future. -RAY_CONFIG(bool, experimental_enable_open_telemetry_on_core, false) +/// Whether enable OpenTelemetry as the metrics collection backend. The default is +/// using OpenCensus. +RAY_CONFIG(bool, enable_open_telemetry, false) /// Comma separated list of components we enable grpc metrics collection for. /// Only effective if `enable_metrics_collection` is also true. Will have some performance diff --git a/src/ray/stats/metric.cc b/src/ray/stats/metric.cc index e5e620fbb5ae..c2dcd38ddc26 100644 --- a/src/ray/stats/metric.cc +++ b/src/ray/stats/metric.cc @@ -113,7 +113,7 @@ void Metric::Record(double value, TagsType tags) { return; } - if (::RayConfig::instance().experimental_enable_open_telemetry_on_core()) { + if (::RayConfig::instance().enable_open_telemetry()) { // Register the metric if it hasn't been registered yet; otherwise, this is a no-op. // We defer metric registration until the first time it's recorded, rather than during // construction, to avoid issues with static initialization order. Specifically, our diff --git a/src/ray/stats/metric.h b/src/ray/stats/metric.h index daacf41e1c10..10322d4e2c85 100644 --- a/src/ray/stats/metric.h +++ b/src/ray/stats/metric.h @@ -265,7 +265,7 @@ void RegisterView(const std::string &name, const std::string &description, const std::vector &tag_keys, const std::vector &buckets) { - if (!::RayConfig::instance().experimental_enable_open_telemetry_on_core()) { + if (!::RayConfig::instance().enable_open_telemetry()) { // OpenTelemetry is not enabled, register the view as an OpenCensus view. using I = StatsTypeMap; auto view_descriptor = opencensus::stats::ViewDescriptor() diff --git a/src/ray/stats/stats.h b/src/ray/stats/stats.h index 3bdadbe51f23..74c1f25e8a0e 100644 --- a/src/ray/stats/stats.h +++ b/src/ray/stats/stats.h @@ -46,6 +46,12 @@ using OpenTelemetryMetricRecorder = ray::telemetry::OpenTelemetryMetricRecorder; static std::shared_ptr metrics_io_service_pool; static absl::Mutex stats_mutex; +// Returns true if OpenCensus should be enabled. +static inline bool should_enable_open_census() { + return !RayConfig::instance().enable_open_telemetry() || + !RayConfig::instance().enable_grpc_metrics_collection_for().empty(); +} + /// Initialize stats for a process. /// NOTE: /// - stats::Init should be called only once per PROCESS. Redundant calls will be just @@ -84,14 +90,15 @@ static inline void Init( absl::Milliseconds(std::max(RayConfig::instance().metrics_report_interval_ms() / 2, static_cast(500)))); // Register the metric recorder. - if (RayConfig::instance().experimental_enable_open_telemetry_on_core()) { + if (RayConfig::instance().enable_open_telemetry()) { OpenTelemetryMetricRecorder::GetInstance().RegisterGrpcExporter( BuildAddress("127.0.0.1", metrics_agent_port), std::chrono::milliseconds( absl::ToInt64Milliseconds(StatsConfig::instance().GetReportInterval())), std::chrono::milliseconds( absl::ToInt64Milliseconds(StatsConfig::instance().GetHarvestInterval()))); - } else { + } + if (should_enable_open_census()) { metrics_io_service_pool = std::make_shared(1); metrics_io_service_pool->Run(); instrumented_io_context *metrics_io_service = metrics_io_service_pool->Get(); @@ -123,9 +130,10 @@ static inline void Shutdown() { // Return if stats had never been initialized. return; } - if (RayConfig::instance().experimental_enable_open_telemetry_on_core()) { + if (RayConfig::instance().enable_open_telemetry()) { OpenTelemetryMetricRecorder::GetInstance().Shutdown(); - } else { + } + if (should_enable_open_census()) { metrics_io_service_pool->Stop(); opencensus::stats::DeltaProducer::Get()->Shutdown(); opencensus::stats::StatsExporter::Shutdown(); diff --git a/src/ray/stats/tests/BUILD.bazel b/src/ray/stats/tests/BUILD.bazel index e1d3f7d66f0a..90bfa18328a8 100644 --- a/src/ray/stats/tests/BUILD.bazel +++ b/src/ray/stats/tests/BUILD.bazel @@ -5,7 +5,7 @@ ray_cc_test( size = "small", srcs = ["metric_with_open_telemetry_test.cc"], env = { - "RAY_experimental_enable_open_telemetry_on_core": "1", + "RAY_enable_open_telemetry": "1", }, tags = ["team:core"], deps = [ From 21bc4528339420623c2f2a1958c7fb68b5dd8a8c Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Thu, 14 Aug 2025 14:42:57 -0700 Subject: [PATCH 115/634] [core] Fix ubsan for publisher_test (#55621) Signed-off-by: dayshah --- src/ray/pubsub/test/publisher_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ray/pubsub/test/publisher_test.cc b/src/ray/pubsub/test/publisher_test.cc index 9165bd52d7fa..4d225ba75d28 100644 --- a/src/ray/pubsub/test/publisher_test.cc +++ b/src/ray/pubsub/test/publisher_test.cc @@ -342,6 +342,7 @@ TEST_F(PublisherTest, TestSubscriptionIndexIdempotency) { TEST_F(PublisherTest, TestSubscriber) { absl::flat_hash_set object_ids_published; + reply = rpc::PubsubLongPollingReply(); send_reply_callback = [this, &object_ids_published](Status status, std::function success, std::function failure) { @@ -351,7 +352,7 @@ TEST_F(PublisherTest, TestSubscriber) { ObjectID::FromBinary(msg.worker_object_eviction_message().object_id()); object_ids_published.emplace(oid); } - reply = rpc::PubsubLongPollingReply(); + reply.Clear(); }; auto subscriber = std::make_shared( From 078d055ad2520b433db28ddc5e48a45bdc0d64a2 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Thu, 14 Aug 2025 16:44:08 -0700 Subject: [PATCH 116/634] [ci] raydepsets changing load to build (1/4) (#55627) updating cli command from load to build Signed-off-by: elliot-barn --- ci/raydepsets/cli.py | 4 ++-- ci/raydepsets/tests/test_cli.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index c127f1218ae5..79872c2b3387 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -49,7 +49,7 @@ def cli(): @click.option( "--uv-cache-dir", default=None, help="The directory to cache uv dependencies" ) -def load( +def build( config_path: str, workspace_dir: Optional[str], name: Optional[str], @@ -57,7 +57,7 @@ def load( uv_cache_dir: Optional[str], ): """ - Load dependency sets from a config file. + Build dependency sets from a config file. Args: config_path: The path to the config file. If not specified, ci/raydepsets/ray.depsets.yaml will be used. """ diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 37083e62f348..a00ddfd79e05 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -19,7 +19,7 @@ _get_depset, _override_uv_flags, _uv_binary, - load, + build, ) from ci.raydepsets.tests.utils import ( append_to_file, @@ -52,7 +52,7 @@ def _create_test_manager( class TestCli(unittest.TestCase): def test_cli_load_fail_no_config(self): result = CliRunner().invoke( - load, + build, [ "fake_path/test.depsets.yaml", "--workspace-dir", @@ -157,7 +157,7 @@ def test_compile_by_depset_name(self): uv_cache_dir = Path(tmpdir) / "uv_cache" result = CliRunner().invoke( - load, + build, [ "test.depsets.yaml", "--workspace-dir", From f8ee5c9629f99c88af1e919a8ba2191a0c07f607 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:44:58 -0700 Subject: [PATCH 117/634] [ci] pipe through `RAYCI_DISABLE_JAVA` for manylinux base image building (#55606) so that when we do not need java, we can skip installing JDK in the image. Signed-off-by: Lonnie Liu --- ci/build/build-manylinux-forge.sh | 2 +- ci/docker/manylinux.Dockerfile | 2 ++ ci/docker/manylinux.aarch64.wanda.yaml | 1 + ci/docker/manylinux.wanda.yaml | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/build/build-manylinux-forge.sh b/ci/build/build-manylinux-forge.sh index 29a52e3651a6..8c98069343f7 100755 --- a/ci/build/build-manylinux-forge.sh +++ b/ci/build/build-manylinux-forge.sh @@ -22,7 +22,7 @@ if [[ "${HOSTTYPE-}" == "x86_64" ]]; then fi # Install ray java dependencies. -if [[ "${RAY_INSTALL_JAVA}" == "1" ]]; then +if [[ "${RAYCI_DISABLE_JAVA:-false}" != "true" && "${RAY_INSTALL_JAVA:-1}" == "1" ]]; then sudo yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel maven java -version JAVA_BIN="$(readlink -f "$(command -v java)")" diff --git a/ci/docker/manylinux.Dockerfile b/ci/docker/manylinux.Dockerfile index d090e53b5e1f..c9c3111b3077 100644 --- a/ci/docker/manylinux.Dockerfile +++ b/ci/docker/manylinux.Dockerfile @@ -4,8 +4,10 @@ ARG HOSTTYPE FROM quay.io/pypa/manylinux2014_${HOSTTYPE}:2024-07-02-9ac04ee ARG BUILDKITE_BAZEL_CACHE_URL +ARG RAYCI_DISABLE_JAVA=false ENV BUILD_JAR=1 +ENV RAYCI_DISABLE_JAVA=$RAYCI_DISABLE_JAVA ENV RAY_INSTALL_JAVA=1 ENV BUILDKITE_BAZEL_CACHE_URL=$BUILDKITE_BAZEL_CACHE_URL diff --git a/ci/docker/manylinux.aarch64.wanda.yaml b/ci/docker/manylinux.aarch64.wanda.yaml index 5b72e6df5bd3..fb5827b560dc 100644 --- a/ci/docker/manylinux.aarch64.wanda.yaml +++ b/ci/docker/manylinux.aarch64.wanda.yaml @@ -5,5 +5,6 @@ srcs: - ci/build/build-manylinux-forge.sh build_args: - BUILDKITE_BAZEL_CACHE_URL + - RAYCI_DISABLE_JAVA - HOSTTYPE=aarch64 dockerfile: ci/docker/manylinux.Dockerfile diff --git a/ci/docker/manylinux.wanda.yaml b/ci/docker/manylinux.wanda.yaml index 3e01ed3a2cd6..5b72115f3cc7 100644 --- a/ci/docker/manylinux.wanda.yaml +++ b/ci/docker/manylinux.wanda.yaml @@ -5,5 +5,6 @@ srcs: - ci/build/build-manylinux-forge.sh build_args: - BUILDKITE_BAZEL_CACHE_URL + - RAYCI_DISABLE_JAVA - HOSTTYPE=x86_64 dockerfile: ci/docker/manylinux.Dockerfile From 69f421884419c8c39a363eeb6b459bd77b6f0017 Mon Sep 17 00:00:00 2001 From: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Date: Thu, 14 Aug 2025 17:35:01 -0700 Subject: [PATCH 118/634] [Serve] Add route_prefix field to DeploymentVersion (#55407) This PR adds `route_prefix` to `DeploymentVersion` class to allow robust light weight config update with `route_prefix`. --------- Signed-off-by: doyoung --- python/ray/serve/_private/deployment_state.py | 4 +++- python/ray/serve/_private/replica.py | 15 +++++++++---- python/ray/serve/_private/version.py | 14 +++++++++++-- python/ray/serve/tests/test_deploy_app_2.py | 14 +++++++------ .../serve/tests/test_deployment_version.py | 21 +++++++++++++++++++ 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/python/ray/serve/_private/deployment_state.py b/python/ray/serve/_private/deployment_state.py index 3f5cd5f2717d..6bf969f9538f 100644 --- a/python/ray/serve/_private/deployment_state.py +++ b/python/ray/serve/_private/deployment_state.py @@ -131,6 +131,7 @@ def create( placement_group_bundles=info.replica_config.placement_group_bundles, placement_group_strategy=info.replica_config.placement_group_strategy, max_replicas_per_node=info.replica_config.max_replicas_per_node, + route_prefix=info.route_prefix, ) return cls(info, target_num_replicas, version, deleting) @@ -597,7 +598,7 @@ def reconfigure(self, version: DeploymentVersion) -> bool: deployment_config.user_config ) self._ready_obj_ref = self._actor_handle.reconfigure.remote( - deployment_config + deployment_config, version.route_prefix ) self._version = version @@ -1745,6 +1746,7 @@ def deploy(self, deployment_info: DeploymentInfo) -> bool: != deployment_info.deployment_config or curr_deployment_info.replica_config.ray_actor_options != deployment_info.replica_config.ray_actor_options + or curr_deployment_info.route_prefix != deployment_info.route_prefix or deployment_info.version is None or curr_deployment_info.version != deployment_info.version ) diff --git a/python/ray/serve/_private/replica.py b/python/ray/serve/_private/replica.py index d1831ea1c7fe..298c1bbfe804 100644 --- a/python/ray/serve/_private/replica.py +++ b/python/ray/serve/_private/replica.py @@ -751,7 +751,9 @@ async def initialize(self, deployment_config: DeploymentConfig): except Exception: raise RuntimeError(traceback.format_exc()) from None - async def reconfigure(self, deployment_config: DeploymentConfig): + async def reconfigure( + self, deployment_config: DeploymentConfig, route_prefix: Optional[str] = None + ): try: user_config_changed = ( deployment_config.user_config != self._deployment_config.user_config @@ -762,7 +764,7 @@ async def reconfigure(self, deployment_config: DeploymentConfig): ) self._deployment_config = deployment_config self._version = DeploymentVersion.from_deployment_version( - self._version, deployment_config + self._version, deployment_config, route_prefix ) self._metrics_manager.set_autoscaling_config( @@ -784,6 +786,9 @@ async def reconfigure(self, deployment_config: DeploymentConfig): self._set_internal_replica_context( servable_object=self._user_callable_wrapper.user_callable ) + + self._route_prefix = route_prefix + except Exception: raise RuntimeError(traceback.format_exc()) from None @@ -1041,8 +1046,10 @@ async def check_health(self): async def record_routing_stats(self) -> Dict[str, Any]: return await self._replica_impl.record_routing_stats() - async def reconfigure(self, deployment_config) -> ReplicaMetadata: - await self._replica_impl.reconfigure(deployment_config) + async def reconfigure( + self, deployment_config, route_prefix: Optional[str] = None + ) -> ReplicaMetadata: + await self._replica_impl.reconfigure(deployment_config, route_prefix) return self._replica_impl.get_metadata() def _preprocess_request_args( diff --git a/python/ray/serve/_private/version.py b/python/ray/serve/_private/version.py index 9242dfc928e9..1c064a9a9dc7 100644 --- a/python/ray/serve/_private/version.py +++ b/python/ray/serve/_private/version.py @@ -22,6 +22,7 @@ def __init__( placement_group_bundles: Optional[List[Dict[str, float]]] = None, placement_group_strategy: Optional[str] = None, max_replicas_per_node: Optional[int] = None, + route_prefix: Optional[str] = None, ): if code_version is not None and not isinstance(code_version, str): raise TypeError(f"code_version must be str, got {type(code_version)}.") @@ -37,12 +38,16 @@ def __init__( self.placement_group_bundles = placement_group_bundles self.placement_group_strategy = placement_group_strategy self.max_replicas_per_node = max_replicas_per_node + self.route_prefix = route_prefix self.compute_hashes() @classmethod - def from_deployment_version(cls, deployment_version, deployment_config): + def from_deployment_version( + cls, deployment_version, deployment_config, route_prefix: Optional[str] = None + ): version_copy = deepcopy(deployment_version) version_copy.deployment_config = deployment_config + version_copy.route_prefix = route_prefix version_copy.compute_hashes() return version_copy @@ -95,11 +100,15 @@ def compute_hashes(self): combined_placement_group_options ) self.placement_group_options_hash = crc32(serialized_placement_group_options) + # Include app-level route prefix in the version hashes so changing + # it triggers an in-place reconfigure of running replicas. + serialized_route_prefix = _serialize(self.route_prefix) # If this changes, DeploymentReplica.reconfigure() will call reconfigure on the # actual replica actor self.reconfigure_actor_hash = crc32( - self._get_serialized_options( + serialized_route_prefix + + self._get_serialized_options( [DeploymentOptionUpdateType.NeedsActorReconfigure] ) ) @@ -111,6 +120,7 @@ def compute_hashes(self): + serialized_ray_actor_options + serialized_placement_group_options + str(self.max_replicas_per_node).encode("utf-8") + + serialized_route_prefix + self._get_serialized_options( [ DeploymentOptionUpdateType.NeedsReconfigure, diff --git a/python/ray/serve/tests/test_deploy_app_2.py b/python/ray/serve/tests/test_deploy_app_2.py index 98eff882586b..fe073c07eb6b 100644 --- a/python/ray/serve/tests/test_deploy_app_2.py +++ b/python/ray/serve/tests/test_deploy_app_2.py @@ -20,6 +20,7 @@ ) from ray.serve._private.test_utils import ( check_num_replicas_eq, + get_application_url, ) from ray.serve.schema import ( ApplicationStatus, @@ -558,23 +559,24 @@ def test_change_route_prefix(serve_instance): "import_path": "ray.serve.tests.test_config_files.pid.node", } client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]})) - wait_for_condition(check_running) - pid1 = httpx.get("http://localhost:8000/old").json()[0] - + url = get_application_url() + pid1 = httpx.get(url).json()[0] # Redeploy application with route prefix /new. app_config["route_prefix"] = "/new" client.deploy_apps(ServeDeploySchema(**{"applications": [app_config]})) - + wait_for_condition(check_running) # Check that the old route is gone and the response from the new route # has the same PID (replica wasn't restarted). def check_switched(): # Old route should be gone - resp = httpx.get("http://localhost:8000/old") + url = get_application_url(exclude_route_prefix=True) + resp = httpx.get(f"{url}/old") assert "Path '/old' not found." in resp.text # Response from new route should be same PID - pid2 = httpx.get("http://localhost:8000/new").json()[0] + url = get_application_url() + pid2 = httpx.get(url).json()[0] assert pid2 == pid1 return True diff --git a/python/ray/serve/tests/test_deployment_version.py b/python/ray/serve/tests/test_deployment_version.py index ce37a9100e74..b1202aee3134 100644 --- a/python/ray/serve/tests/test_deployment_version.py +++ b/python/ray/serve/tests/test_deployment_version.py @@ -17,6 +17,27 @@ def get_version(): assert len(set(ray.get([get_version.remote() for _ in range(100)]))) == 1 +def test_route_prefix_changes_trigger_reconfigure_hash(): + """Test that route prefix changes trigger a reconfigure hash change.""" + cfg = DeploymentConfig() + v1 = DeploymentVersion( + code_version="same version", + deployment_config=cfg, + ray_actor_options={}, + route_prefix="/a", + ) + v2 = DeploymentVersion( + code_version="same version", + deployment_config=cfg, + ray_actor_options={}, + route_prefix="/b", + ) + assert v1.reconfigure_actor_hash != v2.reconfigure_actor_hash + # Should not require a full actor restart if nothing else changed + assert not v1.requires_actor_restart(v2) + assert v1.requires_actor_reconfigure(v2) + + if __name__ == "__main__": import sys From c7c7e7c8fb99bd1081fe4949ccdff2614e6ce8ca Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Thu, 14 Aug 2025 17:45:05 -0700 Subject: [PATCH 119/634] [ci] upgrading uv binary and updating test (2/4) (#55626) - upgrading uv from 0.7.20 -> 0.8.10 to gain parity with uv used compile llm lock files job - updating unit test Signed-off-by: elliot-barn --- WORKSPACE | 4 ++-- ci/raydepsets/tests/test_cli.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 73158be8e560..e56c6a109b05 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -125,8 +125,8 @@ filegroup( visibility = ["//visibility:public"], ) """, - sha256 = "10f204426ff188925d22a53c1d0310d190a8d4d24513712e1b8e2ca9873f0666", - urls = ["https://github.com/astral-sh/uv/releases/download/0.7.20/uv-x86_64-unknown-linux-gnu.tar.gz"], + sha256 = "2c4392591fe9469d006452ef22f32712f35087d87fb1764ec03e23544eb8770d", + urls = ["https://github.com/astral-sh/uv/releases/download/0.8.10/uv-x86_64-unknown-linux-gnu.tar.gz"], ) http_archive( diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index a00ddfd79e05..6401b8d766be 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -94,7 +94,7 @@ def test_uv_version(self): stderr=subprocess.PIPE, ) assert result.returncode == 0 - assert "uv 0.7.20" in result.stdout.decode("utf-8") + assert "uv 0.8.10" in result.stdout.decode("utf-8") assert result.stderr.decode("utf-8") == "" def test_compile(self): From c62889c8d2c72e4e3466f31995c43d2f0189b10e Mon Sep 17 00:00:00 2001 From: goutamvenkat-anyscale Date: Thu, 14 Aug 2025 18:53:49 -0700 Subject: [PATCH 120/634] [Train] - Bump up test size for test_data_integration (#55633) Signed-off-by: Goutam V --- python/ray/train/v2/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/train/v2/BUILD b/python/ray/train/v2/BUILD index 8b0128a3a15d..de2284e3a96e 100644 --- a/python/ray/train/v2/BUILD +++ b/python/ray/train/v2/BUILD @@ -71,7 +71,7 @@ py_test( py_test( name = "test_data_integration", - size = "small", + size = "medium", srcs = ["tests/test_data_integration.py"], env = {"RAY_TRAIN_V2_ENABLED": "1"}, tags = [ From 486935db5ede79b419623f29e2593c76a0df57c9 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Thu, 14 Aug 2025 19:25:41 -0700 Subject: [PATCH 121/634] [core] add test rules for container tests (#55622) The `core: container` test is pretty flaky on premerge and block PRs from time to time. This PR add a test rule to only run this test on a change that touches `python/ray/runtime_env`. Test: - CI Signed-off-by: Cuong Nguyen --- .buildkite/core.rayci.yml | 12 ++++++------ ci/pipeline/test_rules.txt | 4 ++++ python/ray/tests/BUILD | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.buildkite/core.rayci.yml b/.buildkite/core.rayci.yml index 09542fe0a8f3..e5e1afca4e0d 100644 --- a/.buildkite/core.rayci.yml +++ b/.buildkite/core.rayci.yml @@ -60,7 +60,7 @@ steps: - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/_common/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core --install-mask all-ray-libraries --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3 - --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,multi_gpu,spark_on_ray,ray_client,compiled_graphs,dask + --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,runtime_env_container,manual,multi_gpu,spark_on_ray,ray_client,compiled_graphs,dask --install-mask all-ray-libraries - label: ":ray: core: cgraph python tests" @@ -86,7 +86,7 @@ steps: --install-mask all-ray-libraries --workers 4 --worker-id "{{matrix.worker_id}}" --parallelism-per-worker 3 --python-version {{matrix.python}} - --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,multi_gpu,spark_on_ray,ray_client,dask + --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,runtime_env_container,manual,multi_gpu,spark_on_ray,ray_client,dask depends_on: corebuild-multipy matrix: setup: @@ -115,7 +115,7 @@ steps: --install-mask all-ray-libraries --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3 --test-env=TEST_EXTERNAL_REDIS=1 - --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,multi_gpu,spark_on_ray,ray_client,dask + --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,runtime_env_container,manual,multi_gpu,spark_on_ray,ray_client,dask - label: ":ray: core: memory pressure tests" tags: @@ -433,10 +433,10 @@ steps: - raycpubase - corebuild - - label: ":ray: core: container tests" + - label: ":ray: core: runtime env container tests" tags: - - python - docker + - runtime_env_container - oss instance_type: medium commands: @@ -447,7 +447,7 @@ steps: # Disable test DB, these tests will never succeed if run in the flaky step. - RAYCI_DISABLE_TEST_DB=1 bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core --install-mask all-ray-libraries - --only-tags container + --only-tags runtime_env_container depends_on: - manylinux - forge diff --git a/ci/pipeline/test_rules.txt b/ci/pipeline/test_rules.txt index e13a83b25e12..bc102a99c74e 100644 --- a/ci/pipeline/test_rules.txt +++ b/ci/pipeline/test_rules.txt @@ -99,6 +99,10 @@ python/ray/util/spark/ @ python spark_on_ray ; +python/ray/runtime_env/ +@ python runtime_env_container +; + python/ @ ml tune train data # Python changes might impact cross language stack in Java. diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD index f4e604144df9..7716ecc22d9a 100644 --- a/python/ray/tests/BUILD +++ b/python/ray/tests/BUILD @@ -1089,8 +1089,8 @@ py_test( size = "large", srcs = ["test_runtime_env_container.py"], tags = [ - "container", "exclusive", + "runtime_env_container", "team:core", ], deps = [ From fe54c9554106b1e4b89c52833b6251143b0092e5 Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Thu, 14 Aug 2025 22:02:24 -0700 Subject: [PATCH 122/634] [ci] Add hook to clean the Ray address file before the test run starts (#54715) Co-authored-by: Jiajun Yao --- python/ray/tests/conftest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/ray/tests/conftest.py b/python/ray/tests/conftest.py index a3738328233c..7909600440d9 100644 --- a/python/ray/tests/conftest.py +++ b/python/ray/tests/conftest.py @@ -1468,6 +1468,15 @@ def random_ascii_file(request): yield fp +# Clean up Ray address file before the test run starts, since sometimes bazel test times out +# and kill the test process, without cleaning up the Ray address file. +def pytest_sessionstart(session): + """Called after the Session object has been created and before performing collection and entering the run test loop.""" + + # Delete the cluster address file just in case. + ray._common.utils.reset_ray_address() + + """ pytest httpserver related test fixtures """ From d95ef0c74138e5a529b5f4b0134177d5aa9bdee0 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Fri, 15 Aug 2025 00:00:49 -0700 Subject: [PATCH 123/634] [ci] release test: use rayci to perform test init (#55629) so that rayci buildid can be populated Signed-off-by: Lonnie Liu --- .buildkite/release/config.yml | 5 +++++ .buildkite/release/test-init.sh | 31 +++++++++++++++++++++++++++++++ .buildkite/release/test.rayci.yml | 11 +++++++++++ 3 files changed, 47 insertions(+) create mode 100644 .buildkite/release/test-init.sh create mode 100644 .buildkite/release/test.rayci.yml diff --git a/.buildkite/release/config.yml b/.buildkite/release/config.yml index 6dffd5492011..7917e6356526 100644 --- a/.buildkite/release/config.yml +++ b/.buildkite/release/config.yml @@ -15,3 +15,8 @@ env: RAYCI_SKIP_UPLOAD: "true" hook_env_keys: - RAYCI_CHECKOUT_DIR +build_env_keys: + - AUTOMATIC + - RELEASE_FREQUENCY +docker_plugin: + allow_mount_buildkite_agent: true diff --git a/.buildkite/release/test-init.sh b/.buildkite/release/test-init.sh new file mode 100644 index 000000000000..85282c9002c7 --- /dev/null +++ b/.buildkite/release/test-init.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -euo pipefail + +if [[ ${BUILDKITE_COMMIT} == "HEAD" ]]; then + BUILDKITE_COMMIT="$(git rev-parse HEAD)" + export BUILDKITE_COMMIT +fi + +aws ecr get-login-password --region us-west-2 | \ + docker login --username AWS --password-stdin 029272617770.dkr.ecr.us-west-2.amazonaws.com + +bash release/gcloud_docker_login.sh release/aws2gce_iam.json +export PATH="${PWD}/google-cloud-sdk/bin:$PATH" + +if [[ "${AUTOMATIC:-0}" == "1" && "${BUILDKITE_BRANCH}" == "master" ]]; then + export REPORT_TO_RAY_TEST_DB=1 +fi + +RUN_FLAGS=() + +if [[ "${AUTOMATIC:-0}" == "0" || "${BUILDKITE_BRANCH}" == "releases/"* ]]; then + RUN_FLAGS+=(--run-jailed-tests) +fi +if [[ "${BUILDKITE_BRANCH}" != "releases/"* ]]; then + RUN_FLAGS+=(--run-unstable-tests) +fi + +echo "---- Build test steps" +bazelisk run //release:build_pipeline -- "${RUN_FLAGS[@]}" \ + | buildkite-agent pipeline upload diff --git a/.buildkite/release/test.rayci.yml b/.buildkite/release/test.rayci.yml new file mode 100644 index 000000000000..3bd5b4aff514 --- /dev/null +++ b/.buildkite/release/test.rayci.yml @@ -0,0 +1,11 @@ +group: test init +tags: + - oss +steps: + - label: "test init" + key: test-init + instance_type: release-medium + commands: + - /bin/bash .buildkite/release/test-init.sh + mount_buildkite_agent: true + depends_on: forge From de1494e57497b6c57037edf83044ee507fb80159 Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Fri, 15 Aug 2025 09:34:30 -0700 Subject: [PATCH 124/634] [serve] Refactor the router and handle (#55635) ## Why are these changes needed? Refactor Serve deployment handle and router. --------- Signed-off-by: akyang-anyscale --- python/ray/serve/_private/common.py | 8 + python/ray/serve/_private/handle_options.py | 2 + .../ray/serve/_private/local_testing_mode.py | 7 + python/ray/serve/_private/replica_result.py | 39 +++- .../serve/_private/request_router/common.py | 3 + .../request_router/replica_wrapper.py | 56 ++--- .../_private/request_router/request_router.py | 6 + python/ray/serve/_private/router.py | 215 +++++++++++------- python/ray/serve/handle.py | 13 ++ .../serve/tests/test_actor_replica_wrapper.py | 23 +- .../serve/tests/test_autoscaling_policy.py | 16 +- .../tests/unit/test_local_testing_mode.py | 25 +- .../tests/unit/test_pow_2_request_router.py | 4 +- python/ray/serve/tests/unit/test_router.py | 37 +-- 14 files changed, 285 insertions(+), 169 deletions(-) diff --git a/python/ray/serve/_private/common.py b/python/ray/serve/_private/common.py index 5746e26a56ae..682b186a6d7f 100644 --- a/python/ray/serve/_private/common.py +++ b/python/ray/serve/_private/common.py @@ -746,3 +746,11 @@ class CreatePlacementGroupRequest: target_node_id: str name: str runtime_env: Optional[str] = None + + +# This error is used to raise when a by-value DeploymentResponse is converted to an +# ObjectRef. +OBJ_REF_NOT_SUPPORTED_ERROR = RuntimeError( + "Converting by-value DeploymentResponses to ObjectRefs is not supported. " + "Use handle.options(_by_reference=True) to enable it." +) diff --git a/python/ray/serve/_private/handle_options.py b/python/ray/serve/_private/handle_options.py index ce2c624fce22..86ac4bc78ad2 100644 --- a/python/ray/serve/_private/handle_options.py +++ b/python/ray/serve/_private/handle_options.py @@ -62,6 +62,8 @@ def copy_and_update(self, **kwargs) -> "DynamicHandleOptionsBase": @dataclass(frozen=True) class DynamicHandleOptions(DynamicHandleOptionsBase): + _by_reference: bool = True + def copy_and_update(self, **kwargs) -> "DynamicHandleOptions": new_kwargs = {} diff --git a/python/ray/serve/_private/local_testing_mode.py b/python/ray/serve/_private/local_testing_mode.py index e09ad3ff2097..c43eb4667a75 100644 --- a/python/ray/serve/_private/local_testing_mode.py +++ b/python/ray/serve/_private/local_testing_mode.py @@ -104,6 +104,9 @@ class LocalReplicaResult(ReplicaResult): "Converting DeploymentResponses to ObjectRefs is not supported " "in local testing mode." ) + REJECTION_NOT_SUPPORTED_ERROR = RuntimeError( + "Request rejection is not supported in local testing mode." + ) def __init__( self, @@ -153,6 +156,10 @@ async def async_wrapper(self, *args, **kwargs): else: return wrapper + @_process_response + async def get_rejection_response(self): + raise self.REJECTION_NOT_SUPPORTED_ERROR + @_process_response def get(self, timeout_s: Optional[float]): assert ( diff --git a/python/ray/serve/_private/replica_result.py b/python/ray/serve/_private/replica_result.py index 780b4a44bd45..9deaf00ef774 100644 --- a/python/ray/serve/_private/replica_result.py +++ b/python/ray/serve/_private/replica_result.py @@ -1,5 +1,7 @@ import asyncio import inspect +import logging +import pickle import threading import time from abc import ABC, abstractmethod @@ -7,12 +9,20 @@ from typing import Callable, Coroutine, Optional, Union import ray -from ray.serve._private.common import RequestMetadata +from ray.exceptions import TaskCancelledError +from ray.serve._private.common import ReplicaQueueLengthInfo, RequestMetadata +from ray.serve._private.constants import SERVE_LOGGER_NAME from ray.serve._private.utils import calculate_remaining_timeout, generate_request_id from ray.serve.exceptions import RequestCancelledError +logger = logging.getLogger(SERVE_LOGGER_NAME) + class ReplicaResult(ABC): + @abstractmethod + async def get_rejection_response(self) -> Optional[ReplicaQueueLengthInfo]: + raise NotImplementedError + @abstractmethod def get(self, timeout_s: Optional[float]): raise NotImplementedError @@ -57,6 +67,8 @@ def __init__( self, obj_ref_or_gen: Union[ray.ObjectRef, ray.ObjectRefGenerator], metadata: RequestMetadata, + *, + with_rejection: bool = False, ): self._obj_ref: Optional[ray.ObjectRef] = None self._obj_ref_gen: Optional[ray.ObjectRefGenerator] = None @@ -64,6 +76,8 @@ def __init__( self._request_id: str = metadata.request_id self._object_ref_or_gen_sync_lock = threading.Lock() self._lazy_object_ref_or_gen_asyncio_lock = None + self._with_rejection = with_rejection + self._rejection_response = None if isinstance(obj_ref_or_gen, ray.ObjectRefGenerator): self._obj_ref_gen = obj_ref_or_gen @@ -116,6 +130,29 @@ async def async_wrapper(self, *args, **kwargs): else: return wrapper + @_process_response + async def get_rejection_response(self) -> Optional[ReplicaQueueLengthInfo]: + """Get the queue length info from the replica to handle rejection.""" + assert ( + self._with_rejection and self._obj_ref_gen is not None + ), "get_rejection_response() can only be called when request rejection is enabled." + + try: + if self._rejection_response is None: + response = await (await self._obj_ref_gen.__anext__()) + self._rejection_response = pickle.loads(response) + + return self._rejection_response + except asyncio.CancelledError as e: + # HTTP client disconnected or request was explicitly canceled. + logger.info( + "Cancelling request that has already been assigned to a replica." + ) + self.cancel() + raise e from None + except TaskCancelledError: + raise asyncio.CancelledError() + @_process_response def get(self, timeout_s: Optional[float]): assert ( diff --git a/python/ray/serve/_private/request_router/common.py b/python/ray/serve/_private/request_router/common.py index a58659d11ad2..4653daca2185 100644 --- a/python/ray/serve/_private/request_router/common.py +++ b/python/ray/serve/_private/request_router/common.py @@ -49,6 +49,9 @@ class PendingRequest: ) """Context for request routing, used to track routing attempts and backoff.""" + resolved: bool = False + """Whether the arguments have been resolved.""" + def reset_future(self): """Reset the `asyncio.Future`, must be called if this request is re-used.""" self.future = asyncio.Future() diff --git a/python/ray/serve/_private/request_router/replica_wrapper.py b/python/ray/serve/_private/request_router/replica_wrapper.py index af4fcbe05c7f..d42266c312cf 100644 --- a/python/ray/serve/_private/request_router/replica_wrapper.py +++ b/python/ray/serve/_private/request_router/replica_wrapper.py @@ -1,27 +1,20 @@ import asyncio -import logging import pickle from abc import ABC, abstractmethod -from typing import Any, Dict, Optional, Set, Tuple, Union +from typing import Any, Dict, Optional, Set import ray -from ray import ObjectRef, ObjectRefGenerator from ray.actor import ActorHandle -from ray.exceptions import TaskCancelledError from ray.serve._private.common import ( ReplicaID, - ReplicaQueueLengthInfo, RunningReplicaInfo, ) -from ray.serve._private.constants import SERVE_LOGGER_NAME from ray.serve._private.replica_result import ActorReplicaResult, ReplicaResult from ray.serve._private.request_router.common import PendingRequest from ray.serve._private.utils import JavaActorHandleProxy from ray.serve.generated.serve_pb2 import RequestMetadata as RequestMetadataProto from ray.util.annotations import PublicAPI -logger = logging.getLogger(SERVE_LOGGER_NAME) - class ReplicaWrapper(ABC): """This is used to abstract away details of the transport layer @@ -36,7 +29,7 @@ def send_request_java(self, pr: PendingRequest) -> ReplicaResult: @abstractmethod def send_request_python( self, pr: PendingRequest, *, with_rejection: bool - ) -> Tuple[ReplicaResult, Optional[ReplicaQueueLengthInfo]]: + ) -> ReplicaResult: """Send request to Python replica. If sending request with rejection, the replica will yield a @@ -77,9 +70,9 @@ def send_request_java(self, pr: PendingRequest) -> ActorReplicaResult: pr.metadata, ) - def _send_request_python( + def send_request_python( self, pr: PendingRequest, *, with_rejection: bool - ) -> Union[ObjectRef, ObjectRefGenerator]: + ) -> ActorReplicaResult: """Send the request to a Python replica.""" if with_rejection: # Call a separate handler that may reject the request. @@ -95,29 +88,10 @@ def _send_request_python( else: method = self._actor_handle.handle_request - return method.remote(pickle.dumps(pr.metadata), *pr.args, **pr.kwargs) - - async def send_request_python( - self, pr: PendingRequest, with_rejection: bool - ) -> Tuple[ActorReplicaResult, Optional[ReplicaQueueLengthInfo]]: - obj_ref_gen = self._send_request_python(pr, with_rejection=with_rejection) - - if not with_rejection: - return ActorReplicaResult(obj_ref_gen, pr.metadata), None - - try: - first_ref = await obj_ref_gen.__anext__() - queue_len_info: ReplicaQueueLengthInfo = pickle.loads(await first_ref) - return ActorReplicaResult(obj_ref_gen, pr.metadata), queue_len_info - except asyncio.CancelledError as e: - # HTTP client disconnected or request was explicitly canceled. - logger.info( - "Cancelling request that has already been assigned to a replica." - ) - ray.cancel(obj_ref_gen) - raise e from None - except TaskCancelledError: - raise asyncio.CancelledError() + obj_ref_gen = method.remote(pickle.dumps(pr.metadata), *pr.args, **pr.kwargs) + return ActorReplicaResult( + obj_ref_gen, pr.metadata, with_rejection=with_rejection + ) @PublicAPI(stability="alpha") @@ -196,17 +170,13 @@ async def get_queue_len(self, *, deadline_s: float) -> int: ray.cancel(obj_ref) raise - async def send_request( + def try_send_request( self, pr: PendingRequest, with_rejection: bool - ) -> Tuple[Optional[ReplicaResult], Optional[ReplicaQueueLengthInfo]]: - """Send request to this replica.""" + ) -> ReplicaResult: + """Try to send the request to this replica. It may be rejected.""" wrapper = self._get_replica_wrapper(pr) if self._replica_info.is_cross_language: assert not with_rejection, "Request rejection not supported for Java." - return wrapper.send_request_java(pr), None - - result, queue_len_info = await wrapper.send_request_python(pr, with_rejection) - if queue_len_info and not queue_len_info.accepted: - return None, queue_len_info + return wrapper.send_request_java(pr) - return result, queue_len_info + return wrapper.send_request_python(pr, with_rejection=with_rejection) diff --git a/python/ray/serve/_private/request_router/request_router.py b/python/ray/serve/_private/request_router/request_router.py index 11153f58c59a..2d95689d5b41 100644 --- a/python/ray/serve/_private/request_router/request_router.py +++ b/python/ray/serve/_private/request_router/request_router.py @@ -624,6 +624,12 @@ def on_new_queue_len_info( replica_id, queue_len_info.num_ongoing_requests ) + def on_send_request(self, replica_id: ReplicaID): + """Increment queue length cache when a request is sent to a replica.""" + if self._use_replica_queue_len_cache: + num_ongoing_requests = self._replica_queue_len_cache.get(replica_id) or 0 + self._replica_queue_len_cache.update(replica_id, num_ongoing_requests + 1) + def update_replicas(self, replicas: List[RunningReplica]): """Update the set of available replicas to be considered for routing. diff --git a/python/ray/serve/_private/router.py b/python/ray/serve/_private/router.py index ad4f67a117b7..82f90674184b 100644 --- a/python/ray/serve/_private/router.py +++ b/python/ray/serve/_private/router.py @@ -18,7 +18,6 @@ Dict, List, Optional, - Tuple, Union, ) @@ -174,9 +173,21 @@ def wrap_request_assignment(self, request_meta: RequestMetadata): logger.warning(e.message) raise e + self.inc_num_total_requests(request_meta.route) + yield + + @contextmanager + def wrap_queued_request(self, is_retry: bool, num_curr_replicas: int): + """Increment queued requests gauge and maybe push autoscaling metrics to controller.""" try: - self.inc_num_total_requests(request_meta.route) self.inc_num_queued_requests() + # Optimization: if there are currently zero replicas for a deployment, + # push handle metric to controller to allow for fast cold start time. + # Only do this on the first attempt to route the request. + if not is_retry and self.should_send_scaled_to_zero_optimized_push( + curr_num_replicas=num_curr_replicas + ): + self.push_autoscaling_metrics_to_controller() yield finally: @@ -571,25 +582,26 @@ def update_deployment_config(self, deployment_config: DeploymentConfig): async def _resolve_request_arguments( self, - request_metadata: RequestMetadata, - request_args: Tuple[Any], - request_kwargs: Dict[str, Any], - ) -> Tuple[Tuple[Any], Dict[str, Any]]: + pr: PendingRequest, + ) -> None: """Asynchronously resolve and replace top-level request args and kwargs.""" - new_args = list(request_args) - new_kwargs = request_kwargs.copy() + if pr.resolved: + return + + new_args = list(pr.args) + new_kwargs = pr.kwargs.copy() # Map from index -> task for resolving positional arg resolve_arg_tasks = {} - for i, obj in enumerate(request_args): - task = await self._resolve_request_arg_func(obj, request_metadata) + for i, obj in enumerate(pr.args): + task = await self._resolve_request_arg_func(obj, pr.metadata) if task is not None: resolve_arg_tasks[i] = task # Map from key -> task for resolving key-word arg resolve_kwarg_tasks = {} - for k, obj in request_kwargs.items(): - task = await self._resolve_request_arg_func(obj, request_metadata) + for k, obj in pr.kwargs.items(): + task = await self._resolve_request_arg_func(obj, pr.metadata) if task is not None: resolve_kwarg_tasks[k] = task @@ -606,8 +618,9 @@ async def _resolve_request_arguments( for key, task in resolve_kwarg_tasks.items(): new_kwargs[key] = task.result() - # Return new args and new kwargs - return new_args, new_kwargs + pr.args = new_args + pr.kwargs = new_kwargs + pr.resolved = True def _process_finished_request( self, @@ -639,9 +652,99 @@ def _process_finished_request( f"Request failed because {replica_id} is temporarily unavailable." ) + async def _route_and_send_request_once( + self, + pr: PendingRequest, + response_id: str, + is_retry: bool, + ) -> Optional[ReplicaResult]: + result: Optional[ReplicaResult] = None + replica: Optional[RunningReplica] = None + try: + num_curr_replicas = len(self.request_router.curr_replicas) + with self._metrics_manager.wrap_queued_request(is_retry, num_curr_replicas): + # If the pending request is uninitialized, we do so by resolving the + # request arguments. This should only be done once per request, and + # should happen after incrementing `num_queued_requests`, so that Serve + # can upscale the downstream deployment while arguments are resolving. + if not pr.resolved: + await self._resolve_request_arguments(pr) + + replica = await self.request_router._choose_replica_for_request( + pr, is_retry=is_retry + ) + + # If the queue len cache is disabled or we're sending a request to Java, + # then directly send the query and hand the response back. The replica will + # never reject requests in this code path. + with_rejection = ( + self._enable_strict_max_ongoing_requests + and not replica.is_cross_language + ) + result = replica.try_send_request(pr, with_rejection=with_rejection) + + # Proactively update the queue length cache. + self.request_router.on_send_request(replica.replica_id) + + # Keep track of requests that have been sent out to replicas + if RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE: + _request_context = ray.serve.context._get_serve_request_context() + request_id: str = _request_context.request_id + self._metrics_manager.inc_num_running_requests_for_replica( + replica.replica_id + ) + callback = partial( + self._process_finished_request, + replica.replica_id, + request_id, + response_id, + ) + result.add_done_callback(callback) + + if not with_rejection: + return result + + queue_info = await result.get_rejection_response() + self.request_router.on_new_queue_len_info(replica.replica_id, queue_info) + if queue_info.accepted: + self.request_router.on_request_routed(pr, replica.replica_id, result) + return result + + except asyncio.CancelledError: + # NOTE(edoakes): this is not strictly necessary because there are + # currently no `await` statements between getting the ref and returning, + # but I'm adding it defensively. + if result is not None: + result.cancel() + + raise + except ActorDiedError: + # Replica has died but controller hasn't notified the router yet. + # Don't consider this replica for requests in the future, and retry + # routing request. + if replica is not None: + self.request_router.on_replica_actor_died(replica.replica_id) + logger.warning( + f"{replica.replica_id} will not be considered for future " + "requests because it has died." + ) + except ActorUnavailableError: + # There are network issues, or replica has died but GCS is down so + # ActorUnavailableError will be raised until GCS recovers. For the + # time being, invalidate the cache entry so that we don't try to + # send requests to this replica without actively probing, and retry + # routing request. + if replica is not None: + self.request_router.on_replica_actor_unavailable(replica.replica_id) + logger.warning(f"{replica.replica_id} is temporarily unavailable.") + + return None + async def route_and_send_request( - self, pr: PendingRequest - ) -> Tuple[ReplicaResult, ReplicaID]: + self, + pr: PendingRequest, + response_id: str, + ) -> ReplicaResult: """Choose a replica for the request and send it. This will block indefinitely if no replicas are available to handle the @@ -650,54 +753,21 @@ async def route_and_send_request( # Wait for the router to be initialized before sending the request. await self._request_router_initialized.wait() - r = await self.request_router._choose_replica_for_request(pr) - - # If the queue len cache is disabled or we're sending a request to Java, - # then directly send the query and hand the response back. The replica will - # never reject requests in this code path. - if not self._enable_strict_max_ongoing_requests or r.is_cross_language: - result, _ = await r.send_request(pr, with_rejection=False) - return result, r.replica_id - + is_retry = False while True: - result = None - try: - result, queue_info = await r.send_request(pr, with_rejection=True) - self.request_router.on_new_queue_len_info(r.replica_id, queue_info) - if queue_info.accepted: - self.request_router.on_request_routed(pr, r.replica_id, result) - return result, r.replica_id - except asyncio.CancelledError: - # NOTE(edoakes): this is not strictly necessary because there are - # currently no `await` statements between getting the ref and returning, - # but I'm adding it defensively. - if result is not None: - result.cancel() - - raise - except ActorDiedError: - # Replica has died but controller hasn't notified the router yet. - # Don't consider this replica for requests in the future, and retry - # routing request. - self.request_router.on_replica_actor_died(r.replica_id) - logger.warning( - f"{r.replica_id} will not be considered for future " - "requests because it has died." - ) - except ActorUnavailableError: - # There are network issues, or replica has died but GCS is down so - # ActorUnavailableError will be raised until GCS recovers. For the - # time being, invalidate the cache entry so that we don't try to - # send requests to this replica without actively probing, and retry - # routing request. - self.request_router.on_replica_actor_unavailable(r.replica_id) - logger.warning(f"{r.replica_id} is temporarily unavailable.") + result = await self._route_and_send_request_once( + pr, + response_id, + is_retry, + ) + if result is not None: + return result # If the replica rejects the request, retry the routing process. The # request will be placed on the front of the queue to avoid tail latencies. # TODO(edoakes): this retry procedure is not perfect because it'll reset the # process of choosing candidates replicas (i.e., for locality-awareness). - r = await self.request_router._choose_replica_for_request(pr, is_retry=True) + is_retry = True async def assign_request( self, @@ -725,41 +795,16 @@ async def assign_request( await self._request_router_initialized.wait() with self._metrics_manager.wrap_request_assignment(request_meta): - # Optimization: if there are currently zero replicas for a deployment, - # push handle metric to controller to allow for fast cold start time. - if self._metrics_manager.should_send_scaled_to_zero_optimized_push( - curr_num_replicas=len(self.request_router.curr_replicas) - ): - self._metrics_manager.push_autoscaling_metrics_to_controller() - replica_result = None try: - request_args, request_kwargs = await self._resolve_request_arguments( - request_meta, request_args, request_kwargs - ) - replica_result, replica_id = await self.route_and_send_request( + replica_result = await self.route_and_send_request( PendingRequest( args=list(request_args), kwargs=request_kwargs, metadata=request_meta, ), + response_id, ) - - # Keep track of requests that have been sent out to replicas - if RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE: - _request_context = ray.serve.context._get_serve_request_context() - request_id: str = _request_context.request_id - self._metrics_manager.inc_num_running_requests_for_replica( - replica_id - ) - callback = partial( - self._process_finished_request, - replica_id, - request_id, - response_id, - ) - replica_result.add_done_callback(callback) - return replica_result except asyncio.CancelledError: # NOTE(edoakes): this is not strictly necessary because diff --git a/python/ray/serve/handle.py b/python/ray/serve/handle.py index f95e593655e9..5356a18db2de 100644 --- a/python/ray/serve/handle.py +++ b/python/ray/serve/handle.py @@ -9,6 +9,7 @@ from ray import serve from ray._raylet import ObjectRefGenerator from ray.serve._private.common import ( + OBJ_REF_NOT_SUPPORTED_ERROR, DeploymentHandleSource, DeploymentID, RequestMetadata, @@ -499,6 +500,9 @@ async def _to_object_ref(self) -> ray.ObjectRef: ServeUsageTag.DEPLOYMENT_HANDLE_TO_OBJECT_REF_API_USED.record("1") + if not self._request_metadata._by_reference: + raise OBJ_REF_NOT_SUPPORTED_ERROR + replica_result = await self._fetch_future_result_async() return await replica_result.to_object_ref_async() @@ -523,6 +527,9 @@ def _to_object_ref_sync( ServeUsageTag.DEPLOYMENT_HANDLE_TO_OBJECT_REF_API_USED.record("1") + if not self._request_metadata._by_reference: + raise OBJ_REF_NOT_SUPPORTED_ERROR + if not _allow_running_in_asyncio_loop and is_running_in_asyncio_loop(): raise RuntimeError( "Sync methods should not be called from within an `asyncio` event " @@ -640,6 +647,9 @@ async def _to_object_ref_gen(self) -> ObjectRefGenerator: ServeUsageTag.DEPLOYMENT_HANDLE_TO_OBJECT_REF_API_USED.record("1") + if not self._request_metadata._by_reference: + raise OBJ_REF_NOT_SUPPORTED_ERROR + replica_result = await self._fetch_future_result_async() return replica_result.to_object_ref_gen() @@ -661,6 +671,9 @@ def _to_object_ref_gen_sync( ServeUsageTag.DEPLOYMENT_HANDLE_TO_OBJECT_REF_API_USED.record("1") + if not self._request_metadata._by_reference: + raise OBJ_REF_NOT_SUPPORTED_ERROR + if not _allow_running_in_asyncio_loop and is_running_in_asyncio_loop(): raise RuntimeError( "Sync methods should not be called from within an `asyncio` event " diff --git a/python/ray/serve/tests/test_actor_replica_wrapper.py b/python/ray/serve/tests/test_actor_replica_wrapper.py index f0bd3041e275..19b6957cf35a 100644 --- a/python/ray/serve/tests/test_actor_replica_wrapper.py +++ b/python/ray/serve/tests/test_actor_replica_wrapper.py @@ -69,6 +69,8 @@ async def handle_request_with_rejection( async with send_signal_on_cancellation(cancelled_signal_actor): await executing_signal_actor.send.remote() + return + # Special case: if "raise_task_cancelled_error" is in kwargs, raise TaskCancelledError # This simulates the scenario where the underlying Ray task gets cancelled if kwargs.pop("raise_task_cancelled_error", False): @@ -116,7 +118,7 @@ async def test_send_request_without_rejection(setup_fake_replica, is_streaming: is_streaming=is_streaming, ), ) - replica_result, _ = await replica.send_request(pr, with_rejection=False) + replica_result = replica.try_send_request(pr, with_rejection=False) if is_streaming: assert isinstance(replica_result.to_object_ref_gen(), ObjectRefGenerator) for i in range(5): @@ -150,11 +152,12 @@ async def test_send_request_with_rejection( is_streaming=is_streaming, ), ) - replica_result, info = await replica.send_request(pr, with_rejection=True) + replica_result = replica.try_send_request(pr, with_rejection=True) + info = await replica_result.get_rejection_response() assert info.accepted == accepted assert info.num_ongoing_requests == 10 if not accepted: - assert replica_result is None + pass elif is_streaming: assert isinstance(replica_result.to_object_ref_gen(), ObjectRefGenerator) for i in range(5): @@ -190,21 +193,22 @@ async def test_send_request_with_rejection_cancellation(setup_fake_replica): # Send request should hang because the downstream actor method call blocks # before sending the system message. - send_request_task = get_or_create_event_loop().create_task( - replica.send_request(pr, with_rejection=True) + replica_result = replica.try_send_request(pr, with_rejection=True) + request_task = get_or_create_event_loop().create_task( + replica_result.get_rejection_response() ) # Check that the downstream actor method call has started. await executing_signal_actor.wait.remote() - _, pending = await asyncio.wait([send_request_task], timeout=0.001) + _, pending = await asyncio.wait([request_task], timeout=0.001) assert len(pending) == 1 # Cancel the task. This should cause the downstream actor method call to # be cancelled (verified via signal actor). - send_request_task.cancel() + request_task.cancel() with pytest.raises(asyncio.CancelledError): - await send_request_task + await request_task await cancelled_signal_actor.wait.remote() @@ -237,8 +241,9 @@ async def test_send_request_with_rejection_task_cancelled_error(setup_fake_repli ) # The TaskCancelledError should be caught and converted to asyncio.CancelledError + replica_result = replica.try_send_request(pr, with_rejection=True) with pytest.raises(asyncio.CancelledError): - await replica.send_request(pr, with_rejection=True) + await replica_result.get_rejection_response() if __name__ == "__main__": diff --git a/python/ray/serve/tests/test_autoscaling_policy.py b/python/ray/serve/tests/test_autoscaling_policy.py index 32b66f6c305b..b3bbe85b06e8 100644 --- a/python/ray/serve/tests/test_autoscaling_policy.py +++ b/python/ray/serve/tests/test_autoscaling_policy.py @@ -411,8 +411,8 @@ def test_e2e_scale_up_down_basic(min_replicas, serve_instance_with_signal): max_ongoing_requests=1000, ) class A: - def __call__(self): - ray.get(signal.wait.remote()) + async def __call__(self): + await signal.wait.remote() handle = serve.run(A.bind()) wait_for_condition( @@ -598,8 +598,8 @@ class A: def __init__(self): logging.getLogger("ray.serve").setLevel(logging.ERROR) - def __call__(self): - ray.get(signal.wait.remote()) + async def __call__(self): + await signal.wait.remote() handle = serve.run(A.bind()) wait_for_condition( @@ -662,8 +662,8 @@ def test_e2e_intermediate_downscaling(serve_instance_with_signal): max_ongoing_requests=1000, ) class A: - def __call__(self): - ray.get(signal.wait.remote()) + async def __call__(self): + await signal.wait.remote() handle = serve.run(A.bind()) wait_for_condition( @@ -1047,9 +1047,9 @@ def test_e2e_preserve_prev_replicas_rest_api(serve_instance_with_signal): import os @serve.deployment -def g(): +async def g(): signal = ray.get_actor("signal123") - ray.get(signal.wait.remote()) + await signal.wait.remote() return os.getpid() diff --git a/python/ray/serve/tests/unit/test_local_testing_mode.py b/python/ray/serve/tests/unit/test_local_testing_mode.py index bc5eae880864..d0ac157693fb 100644 --- a/python/ray/serve/tests/unit/test_local_testing_mode.py +++ b/python/ray/serve/tests/unit/test_local_testing_mode.py @@ -1,5 +1,6 @@ import logging import os +import re import sys import pytest @@ -57,6 +58,18 @@ def __init__(self, h: DeploymentHandle, should_raise: bool): def test_to_object_ref_error_message(): + def _get_error_match(by_reference: bool) -> str: + if by_reference: + return ( + "Converting DeploymentResponses to ObjectRefs " + "is not supported in local testing mode." + ) + else: + return re.escape( + "Converting by-value DeploymentResponses to ObjectRefs is not supported. " + "Use handle.options(_by_reference=True) to enable it." + ) + @serve.deployment class Inner: pass @@ -67,22 +80,18 @@ def __init__(self, h: DeploymentHandle): self._h = h async def __call__(self): + match = _get_error_match(self._h.handle_options._by_reference) with pytest.raises( RuntimeError, - match=( - "Converting DeploymentResponses to ObjectRefs " - "is not supported in local testing mode." - ), + match=match, ): await self._h.remote()._to_object_ref() h = serve.run(Outer.bind(Inner.bind()), _local_testing_mode=True) + match = _get_error_match(h.handle_options._by_reference) with pytest.raises( RuntimeError, - match=( - "Converting DeploymentResponses to ObjectRefs " - "is not supported in local testing mode." - ), + match=match, ): h.remote()._to_object_ref_sync() diff --git a/python/ray/serve/tests/unit/test_pow_2_request_router.py b/python/ray/serve/tests/unit/test_pow_2_request_router.py index cef2801833f8..54ac8d612202 100644 --- a/python/ray/serve/tests/unit/test_pow_2_request_router.py +++ b/python/ray/serve/tests/unit/test_pow_2_request_router.py @@ -119,7 +119,9 @@ async def get_queue_len(self, *, deadline_s: float) -> int: self.get_queue_len_was_cancelled = True raise - def send_request(self, pr: PendingRequest) -> ReplicaResult: + def try_send_request( + self, pr: PendingRequest, with_rejection: bool + ) -> ReplicaResult: raise NotImplementedError() def send_request_with_rejection(self, pr: PendingRequest) -> ReplicaResult: diff --git a/python/ray/serve/tests/unit/test_router.py b/python/ray/serve/tests/unit/test_router.py index 9017efd18c73..6e8949b1e11b 100644 --- a/python/ray/serve/tests/unit/test_router.py +++ b/python/ray/serve/tests/unit/test_router.py @@ -43,11 +43,20 @@ class FakeReplicaResult(ReplicaResult): - def __init__(self, replica_id, is_generator_object: bool): + def __init__( + self, + replica_id, + is_generator_object: bool, + queue_len_info: Optional[ReplicaQueueLengthInfo] = None, + ): self._replica_id = replica_id self._is_generator_object = is_generator_object + self._queue_len_info = queue_len_info self.cancelled = False + async def get_rejection_response(self): + return self._queue_len_info + def get(self, timeout_s: Optional[float]): raise NotImplementedError @@ -101,9 +110,9 @@ def is_cross_language(self) -> bool: def get_queue_len(self, *, deadline_s: float) -> int: raise NotImplementedError - async def send_request( + def try_send_request( self, pr: PendingRequest, with_rejection: bool - ) -> Tuple[Optional[FakeReplicaResult], Optional[ReplicaQueueLengthInfo]]: + ) -> FakeReplicaResult: if with_rejection: if self._error: raise self._error @@ -115,21 +124,16 @@ async def send_request( self._queue_len_info is not None ), "Must set queue_len_info to use `send_request_with_rejection`." - return ( - FakeReplicaResult(self._replica_id, is_generator_object=True), - self._queue_len_info, + return FakeReplicaResult( + self._replica_id, + is_generator_object=True, + queue_len_info=self._queue_len_info, ) else: if pr.metadata.is_streaming: - return ( - FakeReplicaResult(self._replica_id, is_generator_object=True), - None, - ) + return FakeReplicaResult(self._replica_id, is_generator_object=True) else: - return ( - FakeReplicaResult(self._replica_id, is_generator_object=False), - None, - ) + return FakeReplicaResult(self._replica_id, is_generator_object=False) class FakeRequestRouter(RequestRouter): @@ -180,6 +184,11 @@ def on_new_queue_len_info( replica_id, queue_len_info.num_ongoing_requests ) + def on_send_request(self, replica_id: ReplicaID): + if self._use_queue_len_cache: + num_ongoing_requests = self._replica_queue_len_cache.get(replica_id) or 0 + self._replica_queue_len_cache.update(replica_id, num_ongoing_requests + 1) + def on_replica_actor_unavailable(self, replica_id: ReplicaID): self._replica_queue_len_cache.invalidate_key(replica_id) From c7a7d41b4bbd7509b0cb7cc112fd5ac9af5e55af Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Fri, 15 Aug 2025 18:42:41 +0200 Subject: [PATCH 125/634] [serve] Add a function with a Warning to migrate constants that use `or` expression. (#55464) ## Why are these changes needed? In the `serve` package some of the constants which are initialized from environment variables are silently replaced empty values as `0` with their default values even if a user set them to `0` explicitly. In addition, they are also can be set to negative values which is likely not expected. The list of the constants: ``` PROXY_HEALTH_CHECK_TIMEOUT_S PROXY_HEALTH_CHECK_PERIOD_S PROXY_READY_CHECK_TIMEOUT_S PROXY_MIN_DRAINING_PERIOD_S -- RAY_SERVE_KV_TIMEOUT_S ``` It happens because of the `or value` structure. This PR introduces: - temporary function `get_env_float_non_zero_with_warning` with `FutureWarning`. The function is showing a warning in the following format in case of unexpected value: ``` FutureWarning: Got unexpected value `0.0` for `RAY_SERVE_PROXY_HEALTH_CHECK_TIMEOUT_S` environment variable! Starting from version `2.50.0`, the environment variable will require a positive value. Setting `RAY_SERVE_PROXY_HEALTH_CHECK_TIMEOUT_S` to `10.0`. PROXY_HEALTH_CHECK_TIMEOUT_S = get_env_float_non_zero_with_warning( -- or FutureWarning: Got unexpected value `-1.0` for `RAY_SERVE_PROXY_HEALTH_CHECK_TIMEOUT_S` environment variable! Starting from version `2.50.0`, the environment variable will require a positive value. Setting `RAY_SERVE_PROXY_HEALTH_CHECK_TIMEOUT_S` to `-1.0`. PROXY_HEALTH_CHECK_TIMEOUT_S = get_env_float_non_zero_with_warning( -- or FutureWarning: Got unexpected value `0.0` for `RAY_SERVE_KV_TIMEOUT_S` environment variable! Starting from version `2.50.0`, the environment variable will require a positive value. Setting `RAY_SERVE_KV_TIMEOUT_S` to `None`. RAY_SERVE_KV_TIMEOUT_S = get_env_float_non_zero_with_warning( ``` If the input value is positive, no warning will be emit. - `None` default value support for env variables (introduced for the `RAY_SERVE_KV_TIMEOUT_S`) - `todo` comment for removing the function: `todo: replace this function with 'get_env_float_positive' for the '2.50.0' release.` ## Related issue number Closes #55454 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: axreldable --- python/ray/serve/_private/constants.py | 21 +-- python/ray/serve/_private/constants_utils.py | 77 ++++++++--- .../serve/tests/unit/test_constants_utils.py | 126 +++++++++++++++--- 3 files changed, 182 insertions(+), 42 deletions(-) diff --git a/python/ray/serve/_private/constants.py b/python/ray/serve/_private/constants.py index e6aaac1f62ad..dd46f0c6c20c 100644 --- a/python/ray/serve/_private/constants.py +++ b/python/ray/serve/_private/constants.py @@ -4,6 +4,7 @@ get_env_bool, get_env_float, get_env_float_non_negative, + get_env_float_non_zero_with_warning, get_env_int, get_env_int_positive, get_env_str, @@ -131,15 +132,15 @@ DEFAULT_TARGET_ONGOING_REQUESTS = 2 # HTTP Proxy health check configs -PROXY_HEALTH_CHECK_TIMEOUT_S = ( - get_env_float("RAY_SERVE_PROXY_HEALTH_CHECK_TIMEOUT_S", 10.0) or 10.0 +PROXY_HEALTH_CHECK_TIMEOUT_S = get_env_float_non_zero_with_warning( + "RAY_SERVE_PROXY_HEALTH_CHECK_TIMEOUT_S", 10.0 ) -PROXY_HEALTH_CHECK_PERIOD_S = ( - get_env_float("RAY_SERVE_PROXY_HEALTH_CHECK_PERIOD_S", 10.0) or 10.0 +PROXY_HEALTH_CHECK_PERIOD_S = get_env_float_non_zero_with_warning( + "RAY_SERVE_PROXY_HEALTH_CHECK_PERIOD_S", 10.0 ) -PROXY_READY_CHECK_TIMEOUT_S = ( - get_env_float("RAY_SERVE_PROXY_READY_CHECK_TIMEOUT_S", 5.0) or 5.0 +PROXY_READY_CHECK_TIMEOUT_S = get_env_float_non_zero_with_warning( + "RAY_SERVE_PROXY_READY_CHECK_TIMEOUT_S", 5.0 ) # Number of times in a row that a HTTP proxy must fail the health check before @@ -147,8 +148,8 @@ PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD = 3 # The minimum drain period for a HTTP proxy. -PROXY_MIN_DRAINING_PERIOD_S = ( - get_env_float("RAY_SERVE_PROXY_MIN_DRAINING_PERIOD_S", 30.0) or 30.0 +PROXY_MIN_DRAINING_PERIOD_S = get_env_float_non_zero_with_warning( + "RAY_SERVE_PROXY_MIN_DRAINING_PERIOD_S", 30.0 ) # The time in seconds that the http proxy state waits before # rechecking whether the proxy actor is drained or not. @@ -166,7 +167,9 @@ CLIENT_CHECK_CREATION_POLLING_INTERVAL_S = 0.1 # Timeout for GCS internal KV service -RAY_SERVE_KV_TIMEOUT_S = get_env_float("RAY_SERVE_KV_TIMEOUT_S", 0.0) or None +RAY_SERVE_KV_TIMEOUT_S = get_env_float_non_zero_with_warning( + "RAY_SERVE_KV_TIMEOUT_S", None +) # Timeout for GCS RPC request RAY_GCS_RPC_TIMEOUT_S = 3.0 diff --git a/python/ray/serve/_private/constants_utils.py b/python/ray/serve/_private/constants_utils.py index d06790701930..0ac97ab3eabd 100644 --- a/python/ray/serve/_private/constants_utils.py +++ b/python/ray/serve/_private/constants_utils.py @@ -1,4 +1,5 @@ import os +import warnings from typing import Callable, List, Optional, Type, TypeVar @@ -49,27 +50,38 @@ def parse_latency_buckets(bucket_str: str, default_buckets: List[float]) -> List def _get_env_value( name: str, - default: T, + default: Optional[T], value_type: Type[T], - value_requirement: Optional[Callable[[T], bool]] = None, - error_message: str = None, -) -> T: + validation_func: Optional[Callable[[T], bool]] = None, + expected_value_description: str = None, +) -> Optional[T]: """Get environment variable with type conversion and validation. + This function retrieves an environment variable, converts it to the specified type, + and optionally validates the converted value. + Args: name: The name of the environment variable. default: Default value to use if the environment variable is not set. - value_type: Type to convert the environment variable value to. - value_requirement: Optional function to validate the converted value. - error_message: Error message description for validation failures. + If None, the function will return None without validation. + value_type: Type to convert the environment variable value to (e.g., int, float, str). + validation_func: Optional function that takes the converted value and returns + a boolean indicating whether the value is valid. + expected_value_description: Description of the expected value characteristics + (e.g., "positive", "non negative") used in error messages. Returns: - The converted and validated environment variable value. + The environment variable value converted to the specified type and validated, + or the default value if the environment variable is not set. Raises: - ValueError: If type conversion fails or validation fails. + ValueError: If the environment variable value cannot be converted to the specified + type, or if it fails the optional validation check. """ raw = os.environ.get(name, default) + if raw is None: + return None + try: value = value_type(raw) except ValueError as e: @@ -77,15 +89,16 @@ def _get_env_value( f"Environment variable `{name}` value `{raw}` cannot be converted to `{value_type.__name__}`!" ) from e - if value_requirement and not value_requirement(value): + if validation_func and not validation_func(value): raise ValueError( - f"Got unexpected value `{value}` for `{name}` environment variable! Expected {error_message} `{value_type.__name__}`." + f"Got unexpected value `{value}` for `{name}` environment variable! " + f"Expected {expected_value_description} `{value_type.__name__}`." ) return value -def get_env_int(name: str, default: int) -> int: +def get_env_int(name: str, default: Optional[int]) -> Optional[int]: """Get environment variable as an integer. Args: @@ -101,7 +114,7 @@ def get_env_int(name: str, default: int) -> int: return _get_env_value(name, default, int) -def get_env_int_positive(name: str, default: int) -> int: +def get_env_int_positive(name: str, default: Optional[int]) -> Optional[int]: """Get environment variable as a positive integer. Args: @@ -117,7 +130,7 @@ def get_env_int_positive(name: str, default: int) -> int: return _get_env_value(name, default, int, lambda x: x > 0, "positive") -def get_env_int_non_negative(name: str, default: int) -> int: +def get_env_int_non_negative(name: str, default: Optional[int]) -> Optional[int]: """Get environment variable as a non-negative integer. Args: @@ -133,7 +146,7 @@ def get_env_int_non_negative(name: str, default: int) -> int: return _get_env_value(name, default, int, lambda x: x >= 0, "non negative") -def get_env_float(name: str, default: float) -> float: +def get_env_float(name: str, default: Optional[float]) -> Optional[float]: """Get environment variable as a float. Args: @@ -149,7 +162,7 @@ def get_env_float(name: str, default: float) -> float: return _get_env_value(name, default, float) -def get_env_float_positive(name: str, default: float) -> float: +def get_env_float_positive(name: str, default: Optional[float]) -> Optional[float]: """Get environment variable as a positive float. Args: @@ -165,7 +178,7 @@ def get_env_float_positive(name: str, default: float) -> float: return _get_env_value(name, default, float, lambda x: x > 0, "positive") -def get_env_float_non_negative(name: str, default: float) -> float: +def get_env_float_non_negative(name: str, default: Optional[float]) -> Optional[float]: """Get environment variable as a non-negative float. Args: @@ -207,3 +220,33 @@ def get_env_bool(name: str, default: Optional[str]) -> bool: True if the environment variable value is "1", False otherwise. """ return os.environ.get(name, default) == "1" + + +def get_env_float_non_zero_with_warning( + name: str, default: Optional[float] +) -> Optional[float]: + """Introduced for backward compatibility for constants: + + PROXY_HEALTH_CHECK_TIMEOUT_S + PROXY_HEALTH_CHECK_PERIOD_S + PROXY_READY_CHECK_TIMEOUT_S + PROXY_MIN_DRAINING_PERIOD_S + RAY_SERVE_KV_TIMEOUT_S + + todo: replace this function with 'get_env_float_positive' for the '2.50.0' release. + """ + removal_version = "2.50.0" + + env_value = get_env_float(name, default) + backward_compatible_result = env_value or default + + if env_value is not None and env_value <= 0: + # warning message if unexpected value + warnings.warn( + f"Got unexpected value `{env_value}` for `{name}` environment variable! " + f"Starting from version `{removal_version}`, the environment variable will require a positive value. " + f"Setting `{name}` to `{backward_compatible_result}`. ", + FutureWarning, + stacklevel=2, + ) + return backward_compatible_result diff --git a/python/ray/serve/tests/unit/test_constants_utils.py b/python/ray/serve/tests/unit/test_constants_utils.py index 3051d29b3c6b..96bcbcbd05e6 100644 --- a/python/ray/serve/tests/unit/test_constants_utils.py +++ b/python/ray/serve/tests/unit/test_constants_utils.py @@ -2,11 +2,13 @@ from unittest.mock import patch import pytest +from testfixtures import mock from ray.serve._private.constants_utils import ( get_env_bool, get_env_float, get_env_float_non_negative, + get_env_float_non_zero_with_warning, get_env_float_positive, get_env_int, get_env_int_non_negative, @@ -95,13 +97,13 @@ def mock_environ(): class TestEnvValueFunctions: def test_get_env_int(self, mock_environ): - assert 0 == get_env_int("TEST_VAR", 0) + assert get_env_int("TEST_VAR", 0) == 0 mock_environ["TEST_VAR"] = "42" - assert 42 == get_env_int("TEST_VAR", 0) + assert get_env_int("TEST_VAR", 0) == 42 mock_environ["TEST_VAR"] = "-1" - assert -1 == get_env_int("TEST_VAR", 0) + assert get_env_int("TEST_VAR", 0) == -1 mock_environ["TEST_VAR"] = "0.1" with pytest.raises(ValueError, match=".*`0.1` cannot be converted to `int`!*"): @@ -112,34 +114,37 @@ def test_get_env_int(self, mock_environ): get_env_int_positive("TEST_VAR", 5) def test_get_env_int_positive(self, mock_environ): - assert 1 == get_env_int_positive("TEST_VAR", 1) + assert get_env_int_positive("TEST_VAR", 1) == 1 mock_environ["TEST_VAR"] = "42" - assert 42 == get_env_int_positive("TEST_VAR", 0) + assert get_env_int_positive("TEST_VAR", 1) == 42 mock_environ["TEST_VAR"] = "-1" with pytest.raises(ValueError, match=".*Expected positive `int`.*"): get_env_int_positive("TEST_VAR", 5) def test_get_env_int_non_negative(self, mock_environ): - assert 0 == get_env_int_non_negative("TEST_VAR", 0) - assert 1 == get_env_int_non_negative("TEST_VAR", 1) + assert get_env_int_non_negative("TEST_VAR", 0) == 0 + assert get_env_int_non_negative("TEST_VAR", 1) == 1 mock_environ["TEST_VAR"] = "42" - assert 42 == get_env_int_non_negative("TEST_VAR", 0) + assert get_env_int_non_negative("TEST_VAR", 0) == 42 mock_environ["TEST_VAR"] = "-1" with pytest.raises(ValueError, match=".*Expected non negative `int`.*"): get_env_int_non_negative("TEST_VAR", 5) + with pytest.raises(ValueError, match=".*Expected non negative `int`.*"): + get_env_int_non_negative("TEST_VAR_FROM_DEFAULT", -1) + def test_get_env_float(self, mock_environ): - assert 0.0 == get_env_float("TEST_VAR", 0.0) + assert get_env_float("TEST_VAR", 0.0) == 0.0 mock_environ["TEST_VAR"] = "3.14" - assert 3.14 == get_env_float("TEST_VAR", 0.0) + assert get_env_float("TEST_VAR", 0.0) == 3.14 mock_environ["TEST_VAR"] = "-2.5" - assert -2.5 == get_env_float("TEST_VAR", 0.0) + assert get_env_float("TEST_VAR", 0.0) == -2.5 mock_environ["TEST_VAR"] = "abc" with pytest.raises( @@ -148,21 +153,28 @@ def test_get_env_float(self, mock_environ): get_env_float("TEST_VAR", 0.0) def test_get_env_float_positive(self, mock_environ): - assert 1.5 == get_env_float_positive("TEST_VAR", 1.5) + assert get_env_float_positive("TEST_VAR", 1.5) == 1.5 + assert get_env_float_positive("TEST_VAR", None) is None mock_environ["TEST_VAR"] = "42.5" - assert 42.5 == get_env_float_positive("TEST_VAR", 0.0) + assert get_env_float_positive("TEST_VAR", 1.0) == 42.5 mock_environ["TEST_VAR"] = "-1.2" with pytest.raises(ValueError, match=".*Expected positive `float`.*"): get_env_float_positive("TEST_VAR", 5.0) + with pytest.raises(ValueError, match=".*Expected positive `float`.*"): + get_env_float_positive("TEST_VAR_FROM_DEFAULT", 0.0) + + with pytest.raises(ValueError, match=".*Expected positive `float`.*"): + get_env_float_positive("TEST_VAR_FROM_DEFAULT", -1) + def test_get_env_float_non_negative(self, mock_environ): - assert 0.0 == get_env_float_non_negative("TEST_VAR", 0.0) - assert 1.5 == get_env_float_non_negative("TEST_VAR", 1.5) + assert get_env_float_non_negative("TEST_VAR", 0.0) == 0.0 + assert get_env_float_non_negative("TEST_VAR", 1.5) == 1.5 mock_environ["TEST_VAR"] = "42.5" - assert 42.5 == get_env_float_non_negative("TEST_VAR", 0.0) + assert get_env_float_non_negative("TEST_VAR", 0.0) == 42.5 mock_environ["TEST_VAR"] = "-1.2" with pytest.raises(ValueError, match=".*Expected non negative `float`.*"): @@ -191,6 +203,88 @@ def test_get_env_bool(self, mock_environ): assert get_env_bool("NONEXISTENT_VAR", "0") is False +class TestDeprecationFunctions: + def test_current_behavior(self, mock_environ): + mock_environ["OLD_VAR_NEG"] = "-1" + assert get_env_float("OLD_VAR_NEG", 10.0) or 10.0 == -1.0 + assert (get_env_float("OLD_VAR_NEG", 0.0) or None) == -1.0 + + mock_environ["OLD_VAR_ZERO"] = "0" + assert get_env_float("OLD_VAR_ZERO", 10.0) or 10.0 == 10.0 + + assert get_env_float("NOT_SET", 10.0) or 10.0 == 10.0 + + assert (get_env_float("NOT_SET", 0.0) or None) is None + + @mock.patch("ray.__version__", "2.49.0") # Version before 2.50.0 + def test_with_positive_value_before_250(self, mock_environ): + env_name = "TEST_POSITIVE_FLOAT" + mock_environ[env_name] = "5.5" + + result = get_env_float_non_zero_with_warning(env_name, 10.0) + + assert result == 5.5 + + @mock.patch("ray.__version__", "2.49.0") # Version before 2.50.0 + def test_with_non_positive_value_before_250(self, mock_environ): + env_name = "TEST_NON_POSITIVE_FLOAT" + mock_environ[env_name] = "-2.5" + + with pytest.warns(FutureWarning) as record: + result = get_env_float_non_zero_with_warning(env_name, 10.0) + + assert result == -2.5 + assert len(record) == 1 + assert "will require a positive value" in str(record[0].message) + + @mock.patch("ray.__version__", "2.49.0") # Version before 2.50.0 + def test_with_zero_value_before_250(self, mock_environ): + env_name = "TEST_ZERO_FLOAT" + mock_environ[env_name] = "0.0" + + with pytest.warns(FutureWarning) as record: + result = get_env_float_non_zero_with_warning(env_name, 10.0) + + assert result == 10.0 + assert len(record) == 1 + assert "will require a positive value" in str(record[0].message) + + @mock.patch("ray.__version__", "2.49.0") # Version before 2.50.0 + def test_with_no_env_value_before_250(self): + env_name = "TEST_MISSING_FLOAT" + # Don't set environment variable + + result = get_env_float_non_zero_with_warning(env_name, 1.0) + + assert result == 1.0 + + @mock.patch("ray.__version__", "2.50.0") # Version at 2.50.0 + def test_remain_the_same_behavior_at_2_50(self, mock_environ): + env_name = "TEST_FLOAT" + mock_environ[env_name] = "2.0" + + assert get_env_float_non_zero_with_warning(env_name, 1.0) == 2.0 + + mock_environ["TEST_VAR"] = "-1.2" + assert get_env_float_non_zero_with_warning("TEST_VAR", 5.0) == -1.2 + + mock_environ["TEST_VAR"] = "0.0" + assert get_env_float_non_zero_with_warning("TEST_VAR", 5.0) == 5.0 + + @mock.patch("ray.__version__", "2.51.0") # Version after 2.50.0 + def test_remain_the_same_behavior_after_2_50(self, mock_environ): + env_name = "TEST_FLOAT" + mock_environ[env_name] = "2.0" + + assert get_env_float_non_zero_with_warning(env_name, 1.0) == 2.0 + + mock_environ["TEST_VAR"] = "-1.2" + assert get_env_float_non_zero_with_warning("TEST_VAR", 5.0) == -1.2 + + mock_environ["TEST_VAR"] = "0.0" + assert get_env_float_non_zero_with_warning("TEST_VAR", 5.0) == 5.0 + + if __name__ == "__main__": import sys From c5a16768c71c354738fc4bef552bd4a58c6b3089 Mon Sep 17 00:00:00 2001 From: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Date: Fri, 15 Aug 2025 09:43:09 -0700 Subject: [PATCH 126/634] [Serve] Update test_http_routes to use get_application_url (#55623) Updates one of the serve tests, test_http_routes, so it can start using get_application_url instead of hardcoded urls. --------- Signed-off-by: doyoung Signed-off-by: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> --- python/ray/serve/_private/test_utils.py | 5 +- python/ray/serve/tests/test_http_routes.py | 77 ++++++++++++---------- 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/python/ray/serve/_private/test_utils.py b/python/ray/serve/_private/test_utils.py index 1d95c41a4893..d742b1c8a2b0 100644 --- a/python/ray/serve/_private/test_utils.py +++ b/python/ray/serve/_private/test_utils.py @@ -741,7 +741,9 @@ def get_application_urls( app_name in serve_details["applications"] ), f"App {app_name} not found in serve details. Use this method only when the app is known to be running." route_prefix = serve_details["applications"][app_name]["route_prefix"] - if exclude_route_prefix: + # route_prefix is set to None when route_prefix value is specifically set to None + # in the config used to deploy the app. + if exclude_route_prefix or route_prefix is None: route_prefix = "" if isinstance(protocol, str): protocol = RequestProtocol(protocol) @@ -753,7 +755,6 @@ def get_application_urls( for target_group in target_groups if target_group.protocol == protocol ] - if len(target_groups) == 0: raise ValueError( f"No target group found for app {app_name} with protocol {protocol} and route prefix {route_prefix}" diff --git a/python/ray/serve/tests/test_http_routes.py b/python/ray/serve/tests/test_http_routes.py index 8d2cbe4410ae..4fafc1360e73 100644 --- a/python/ray/serve/tests/test_http_routes.py +++ b/python/ray/serve/tests/test_http_routes.py @@ -8,6 +8,7 @@ import ray from ray import serve from ray.serve._private.constants import SERVE_DEFAULT_APP_NAME +from ray.serve._private.test_utils import get_application_url def test_path_validation(serve_instance): @@ -67,15 +68,22 @@ def __call__(self, *args): serve.run(D2.bind(), name="app2", route_prefix="/hello/world") routes = httpx.get("http://localhost:8000/-/routes").json() - assert len(routes) == 2, routes - assert httpx.get("http://localhost:8000/D1").text == "D1" - assert httpx.get("http://localhost:8000/D1").status_code == 200 - assert httpx.get("http://localhost:8000/hello/world").text == "D2" - assert httpx.get("http://localhost:8000/hello/world").status_code == 200 + app1_url = get_application_url(app_name="app1") + app2_url = get_application_url(app_name="app2") + + assert httpx.get(app1_url).text == "D1" + assert httpx.get(app1_url).status_code == 200 + assert httpx.get(app2_url).text == "D2" + assert httpx.get(app2_url).status_code == 200 assert httpx.get("http://localhost:8000/not_exist").status_code == 404 - assert httpx.get("http://localhost:8000/").status_code == 404 + + app1_url = get_application_url(app_name="app1", exclude_route_prefix=True) + app2_url = get_application_url(app_name="app2", exclude_route_prefix=True) + + assert httpx.get(f"{app1_url}/").status_code == 404 + assert httpx.get(f"{app2_url}/").status_code == 404 def test_deployment_without_route(serve_instance): @@ -85,8 +93,8 @@ def __call__(self, *args): return "1" serve.run(D.bind(), route_prefix=None) - routes = httpx.get("http://localhost:8000/-/routes").json() - assert len(routes) == 0 + routes = httpx.get("http://localhost:8000/-/routes") + assert len(routes.json()) == 0 # make sure the deployment is not exposed under the default route r = httpx.get("http://localhost:8000/") @@ -99,16 +107,17 @@ class D1: pass serve.run(D1.bind()) - - routes = httpx.get("http://localhost:8000/-/routes").json() + url = get_application_url(exclude_route_prefix=True) + routes = httpx.get(f"{url}/-/routes").json() assert len(routes) == 1 assert "/" in routes, routes assert routes["/"] == SERVE_DEFAULT_APP_NAME def test_path_prefixing_1(serve_instance): - def check_req(subpath, text=None, status=None): - r = httpx.get(f"http://localhost:8000{subpath}") + def check_req(subpath, app_name, text=None, status=None): + url = get_application_url(app_name=app_name, exclude_route_prefix=True) + r = httpx.get(f"{url}{subpath}") if text is not None: assert r.text == text, f"{r.text} != {text}" if status is not None: @@ -122,10 +131,10 @@ def __call__(self, *args): return "1" serve.run(D1.bind(), route_prefix="/hello", name="app1") - check_req("/", status=404) - check_req("/hello", text="1") - check_req("/hello/", text="1") - check_req("/hello/a", text="1") + check_req("/", "app1", status=404) + check_req("/hello", "app1", text="1") + check_req("/hello/", "app1", text="1") + check_req("/hello/a", "app1", text="1") @serve.deployment class D2: @@ -133,10 +142,10 @@ def __call__(self, *args): return "2" serve.run(D2.bind(), route_prefix="/", name="app2") - check_req("/hello/", text="1") - check_req("/hello/a", text="1") - check_req("/", text="2") - check_req("/a", text="2") + check_req("/hello/", "app1", text="1") + check_req("/hello/a", "app1", text="1") + check_req("/", "app2", text="2") + check_req("/a", "app2", text="2") @serve.deployment class D3: @@ -144,9 +153,9 @@ def __call__(self, *args): return "3" serve.run(D3.bind(), route_prefix="/hello/world", name="app3") - check_req("/hello/", text="1") - check_req("/", text="2") - check_req("/hello/world/", text="3") + check_req("/hello/", "app1", text="1") + check_req("/", "app2", text="2") + check_req("/hello/world/", "app3", text="3") app = FastAPI() @@ -162,11 +171,11 @@ def subpath(self, p: str): return p serve.run(D4.bind(), route_prefix="/hello/world/again", name="app4") - check_req("/hello/") == "1" - check_req("/") == "2" - check_req("/hello/world/") == "3" - check_req("/hello/world/again/") == "4" - check_req("/hello/world/again/hi") == '"hi"' + check_req("/hello/", "app1") == "1" + check_req("/", "app2") == "2" + check_req("/hello/world/", "app3") == "3" + check_req("/hello/world/again/", "app4") == "4" + check_req("/hello/world/again/hi", "app4") == '"hi"' @pytest.mark.parametrize("base_path", ["", "subpath"]) @@ -201,14 +210,13 @@ def redirect_twice(self, request: Request): if route_prefix != "/": route_prefix += "/" - r = httpx.get(f"http://localhost:8000{route_prefix}redirect", follow_redirects=True) + url = get_application_url(exclude_route_prefix=True) + r = httpx.get(f"{url}{route_prefix}redirect", follow_redirects=True) assert r.status_code == 200 assert len(r.history) == 1 assert r.json() == "hello from /" - r = httpx.get( - f"http://localhost:8000{route_prefix}redirect2", follow_redirects=True - ) + r = httpx.get(f"{url}{route_prefix}redirect2", follow_redirects=True) assert r.status_code == 200 assert len(r.history) == 2 assert r.json() == "hello from /" @@ -220,7 +228,9 @@ def f(): _ = 1 / 0 serve.run(f.bind()) - r = httpx.get("http://localhost:8000/f") + url = get_application_url(exclude_route_prefix=True) + # Error is raised when the request reaches the deployed replica. + r = httpx.get(f"{url}/f") assert r.status_code == 500 assert r.text == "Internal Server Error" @@ -234,6 +244,7 @@ def h(): time.sleep(100) # Don't return here to leave time for actor exit. serve.run(h.bind()) + # Error is raised before the request reaches the deployed replica as the replica does not exist. r = httpx.get("http://localhost:8000/h") assert r.status_code == 500 From 20c84e6193d22d29f25cc36e76ea455417349562 Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Fri, 15 Aug 2025 09:56:09 -0700 Subject: [PATCH 127/634] [serve] Add model composition serve benchmarks (#55549) ## Why are these changes needed? Model composition is a common paradigm we should also track performance for. --------- Signed-off-by: akyang-anyscale --- .../ray/serve/_private/benchmarks/common.py | 37 +++++ .../serve_tests/workloads/microbenchmarks.py | 145 ++++++++++-------- 2 files changed, 122 insertions(+), 60 deletions(-) diff --git a/python/ray/serve/_private/benchmarks/common.py b/python/ray/serve/_private/benchmarks/common.py index f5daad3d493f..1c51801662f7 100644 --- a/python/ray/serve/_private/benchmarks/common.py +++ b/python/ray/serve/_private/benchmarks/common.py @@ -170,6 +170,43 @@ def __call__(self, *args, **kwargs): return b"" +@serve.deployment +class ModelComp: + def __init__(self, child): + logging.getLogger("ray.serve").setLevel(logging.WARNING) + self._child = child + + async def __call__(self, *args, **kwargs): + return await self._child.remote() + + +@serve.deployment +class GrpcDeployment: + def __init__(self): + logging.getLogger("ray.serve").setLevel(logging.WARNING) + + async def grpc_call(self, user_message): + return serve_pb2.ModelOutput(output=9) + + async def call_with_string(self, user_message): + return serve_pb2.ModelOutput(output=9) + + +@serve.deployment +class GrpcModelComp: + def __init__(self, child): + logging.getLogger("ray.serve").setLevel(logging.WARNING) + self._child = child + + async def grpc_call(self, user_message): + await self._child.remote() + return serve_pb2.ModelOutput(output=9) + + async def call_with_string(self, user_message): + await self._child.remote() + return serve_pb2.ModelOutput(output=9) + + @serve.deployment class Streamer: def __init__(self, tokens_per_request: int, inter_token_delay_ms: int = 10): diff --git a/release/serve_tests/workloads/microbenchmarks.py b/release/serve_tests/workloads/microbenchmarks.py index 4cf10ab9e615..983cef1d5f3a 100644 --- a/release/serve_tests/workloads/microbenchmarks.py +++ b/release/serve_tests/workloads/microbenchmarks.py @@ -26,6 +26,9 @@ do_single_http_batch, generate_payload, Noop, + ModelComp, + GrpcDeployment, + GrpcModelComp, IntermediateRouter, run_latency_benchmark, run_throughput_benchmark, @@ -60,18 +63,6 @@ STREAMING_NUM_TRIALS = 10 -@serve.deployment -class GrpcDeployment: - def __init__(self): - logging.getLogger("ray.serve").setLevel(logging.WARNING) - - async def grpc_call(self, user_message): - return serve_pb2.ModelOutput(output=9) - - async def call_with_string(self, user_message): - return serve_pb2.ModelOutput(output=9) - - def convert_throughput_to_perf_metrics( name: str, mean: float, @@ -161,21 +152,32 @@ async def _main( for max_ongoing_requests, concurrency in zip( throughput_max_ongoing_requests, concurrencies ): - serve.run( - Noop.options(max_ongoing_requests=max_ongoing_requests).bind() - ) - url = get_application_url(use_localhost=True) - mean, std, _ = await run_throughput_benchmark( - fn=partial(do_single_http_batch, batch_size=concurrency, url=url), - multiplier=concurrency, - num_trials=NUM_TRIALS, - trial_runtime=TRIAL_RUNTIME_S, - ) - test_name = get_throughput_test_name("http", max_ongoing_requests) - perf_metrics.extend( - convert_throughput_to_perf_metrics(test_name, mean, std) - ) - serve.shutdown() + workloads = { + "http": Noop.options( + max_ongoing_requests=max_ongoing_requests + ).bind(), + "http_model_comp": ModelComp.options( + max_ongoing_requests=max_ongoing_requests + ).bind( + Noop.options(max_ongoing_requests=max_ongoing_requests).bind() + ), + } + for name, app in workloads.items(): + serve.run(app) + url = get_application_url(use_localhost=True) + mean, std, _ = await run_throughput_benchmark( + fn=partial( + do_single_http_batch, batch_size=concurrency, url=url + ), + multiplier=concurrency, + num_trials=NUM_TRIALS, + trial_runtime=TRIAL_RUNTIME_S, + ) + test_name = get_throughput_test_name(name, max_ongoing_requests) + perf_metrics.extend( + convert_throughput_to_perf_metrics(test_name, mean, std) + ) + serve.shutdown() if run_streaming: # Direct streaming between replica @@ -285,28 +287,35 @@ async def _main( for max_ongoing_requests, concurrency in zip( throughput_max_ongoing_requests, concurrencies ): - serve.start(grpc_options=serve_grpc_options) - serve.run( - GrpcDeployment.options( + workloads = { + "grpc": GrpcDeployment.options( max_ongoing_requests=max_ongoing_requests - ).bind() - ) - target = get_application_url( - protocol=RequestProtocol.GRPC, use_localhost=True - ) - mean, std, _ = await run_throughput_benchmark( - fn=partial( - do_single_grpc_batch, batch_size=concurrency, target=target + ).bind(), + "grpc_model_comp": GrpcModelComp.options( + max_ongoing_requests=max_ongoing_requests + ).bind( + Noop.options(max_ongoing_requests=max_ongoing_requests).bind() ), - multiplier=concurrency, - num_trials=NUM_TRIALS, - trial_runtime=TRIAL_RUNTIME_S, - ) - test_name = get_throughput_test_name("grpc", max_ongoing_requests) - perf_metrics.extend( - convert_throughput_to_perf_metrics(test_name, mean, std) - ) - serve.shutdown() + } + for name, app in workloads.items(): + serve.start(grpc_options=serve_grpc_options) + serve.run(app) + target = get_application_url( + protocol=RequestProtocol.GRPC, use_localhost=True + ) + mean, std, _ = await run_throughput_benchmark( + fn=partial( + do_single_grpc_batch, batch_size=concurrency, target=target + ), + multiplier=concurrency, + num_trials=NUM_TRIALS, + trial_runtime=TRIAL_RUNTIME_S, + ) + test_name = get_throughput_test_name(name, max_ongoing_requests) + perf_metrics.extend( + convert_throughput_to_perf_metrics(test_name, mean, std) + ) + serve.shutdown() # Handle if run_handle: @@ -328,21 +337,37 @@ async def _main( for max_ongoing_requests, concurrency in zip( throughput_max_ongoing_requests, concurrencies ): - h: DeploymentHandle = serve.run( - Benchmarker.options(max_ongoing_requests=max_ongoing_requests).bind( + workloads = { + "handle": Benchmarker.options( + max_ongoing_requests=max_ongoing_requests + ).bind( Noop.options(max_ongoing_requests=max_ongoing_requests).bind() + ), + "handle_model_comp": Benchmarker.options( + max_ongoing_requests=max_ongoing_requests + ).bind( + ModelComp.options( + max_ongoing_requests=max_ongoing_requests + ).bind( + Noop.options( + max_ongoing_requests=max_ongoing_requests + ).bind() + ) + ), + } + for name, app in workloads.items(): + h: DeploymentHandle = serve.run(app) + + mean, std, _ = await h.run_throughput_benchmark.remote( + batch_size=concurrency, + num_trials=NUM_TRIALS, + trial_runtime=TRIAL_RUNTIME_S, ) - ) - mean, std, _ = await h.run_throughput_benchmark.remote( - batch_size=concurrency, - num_trials=NUM_TRIALS, - trial_runtime=TRIAL_RUNTIME_S, - ) - test_name = get_throughput_test_name("handle", max_ongoing_requests) - perf_metrics.extend( - convert_throughput_to_perf_metrics(test_name, mean, std) - ) - serve.shutdown() + test_name = get_throughput_test_name(name, max_ongoing_requests) + perf_metrics.extend( + convert_throughput_to_perf_metrics(test_name, mean, std) + ) + serve.shutdown() if run_streaming: h: DeploymentHandle = serve.run( From a39bc679bace4dfaa334c88572effbc5b952a59f Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:14:33 -0700 Subject: [PATCH 128/634] [serve] pin the version of wrk used in serve ci base (#55650) and clone with depth=1 Signed-off-by: Lonnie Liu --- ci/docker/serve.build.Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/docker/serve.build.Dockerfile b/ci/docker/serve.build.Dockerfile index 1b38777f0f25..8753d5a2bd37 100644 --- a/ci/docker/serve.build.Dockerfile +++ b/ci/docker/serve.build.Dockerfile @@ -27,7 +27,11 @@ if [[ "${PYTHON-}" != "3.12" ]]; then tensorflow tensorflow-probability torch torchvision \ transformers aioboto3 fi -git clone https://github.com/wg/wrk.git /tmp/wrk && pushd /tmp/wrk && make -j && sudo cp wrk /usr/local/bin && popd + +git clone --branch=4.2.0 --depth=1 https://github.com/wg/wrk.git /tmp/wrk +make -C /tmp/wrk -j +sudo cp /tmp/wrk/wrk /usr/local/bin/wrk +rm -rf /tmp/wrk # Install custom Pydantic version if requested. if [[ -n "${PYDANTIC_VERSION-}" ]]; then From 0b77c72a0133d407fb58a9114764e652a37e963c Mon Sep 17 00:00:00 2001 From: Justin Yu Date: Fri, 15 Aug 2025 10:48:54 -0700 Subject: [PATCH 129/634] [data] Wrap batch index in a `BatchMetadata` class (#55643) Wrap batch metadata in a dataclass that we can extend in the future. Signed-off-by: Justin Yu --- .../_internal/block_batching/interfaces.py | 22 +++++++++++++------ .../_internal/block_batching/iter_batches.py | 4 ++-- .../ray/data/_internal/block_batching/util.py | 20 +++++++++-------- .../tests/block_batching/test_iter_batches.py | 16 +++++++++----- .../data/tests/block_batching/test_util.py | 22 ++++++++++++------- 5 files changed, 52 insertions(+), 32 deletions(-) diff --git a/python/ray/data/_internal/block_batching/interfaces.py b/python/ray/data/_internal/block_batching/interfaces.py index 452b6d850b93..4f0bed6b3dd4 100644 --- a/python/ray/data/_internal/block_batching/interfaces.py +++ b/python/ray/data/_internal/block_batching/interfaces.py @@ -7,30 +7,38 @@ @dataclass -class Batch: - """A batch of data with a corresponding index. +class BatchMetadata: + """Metadata associated with a batch. Attributes: batch_idx: The global index of this batch so that downstream operations can maintain ordering. - data: The batch of data. """ batch_idx: int + + +@dataclass +class Batch: + """A batch of data. + + Attributes: + metadata: Metadata associated with this batch. + data: The batch of data. + """ + + metadata: BatchMetadata data: DataBatch class CollatedBatch(Batch): - """A batch of collated data with a corresponding index. + """A batch of collated data. Attributes: - batch_idx: The global index of this batch so that downstream operations can - maintain ordering. data: The batch of data which is the output of a user provided collate_fn Therefore, the type of this data can be Any. """ - batch_idx: int data: Any diff --git a/python/ray/data/_internal/block_batching/iter_batches.py b/python/ray/data/_internal/block_batching/iter_batches.py index cfad92a7acd7..f807ae2078dc 100644 --- a/python/ray/data/_internal/block_batching/iter_batches.py +++ b/python/ray/data/_internal/block_batching/iter_batches.py @@ -382,8 +382,8 @@ def restore_original_order(batch_iter: Iterator[Batch]) -> Iterator[Batch]: next_index_required = 0 buffer: Dict[int, Batch] = {} for batch in batch_iter: - assert batch.batch_idx not in buffer - buffer[batch.batch_idx] = batch + assert batch.metadata.batch_idx not in buffer + buffer[batch.metadata.batch_idx] = batch while next_index_required in buffer: yield buffer.pop(next_index_required) next_index_required += 1 diff --git a/python/ray/data/_internal/block_batching/util.py b/python/ray/data/_internal/block_batching/util.py index 4cea60abca80..a1678b569062 100644 --- a/python/ray/data/_internal/block_batching/util.py +++ b/python/ray/data/_internal/block_batching/util.py @@ -1,3 +1,4 @@ +import dataclasses import logging import threading from contextlib import nullcontext @@ -8,6 +9,7 @@ from ray.data._internal.batcher import Batcher, ShufflingBatcher from ray.data._internal.block_batching.interfaces import ( Batch, + BatchMetadata, BlockPrefetcher, CollatedBatch, ) @@ -120,7 +122,7 @@ def get_iter_next_batch_s_timer(): while batcher.has_batch(): with get_iter_next_batch_s_timer(): batch = batcher.next_batch() - yield Batch(global_counter, batch) + yield Batch(metadata=BatchMetadata(batch_idx=global_counter), data=batch) global_counter += 1 # Signal to the batcher that there are no more blocks to add. @@ -130,38 +132,38 @@ def get_iter_next_batch_s_timer(): while batcher.has_batch(): with get_iter_next_batch_s_timer(): batch = batcher.next_batch() - yield Batch(global_counter, batch) + yield Batch(metadata=BatchMetadata(batch_idx=global_counter), data=batch) global_counter += 1 # Get any remaining data. if not drop_last and batcher.has_any(): with get_iter_next_batch_s_timer(): batch = batcher.next_batch() - yield Batch(global_counter, batch) + yield Batch(metadata=BatchMetadata(batch_idx=global_counter), data=batch) global_counter += 1 def format_batches( - block_iter: Iterator[Batch], + batch_iter: Iterator[Batch], batch_format: Optional[str], stats: Optional[DatasetStats] = None, ) -> Iterator[Batch]: """Given an iterator of blocks, returns an iterator of formatted batches. Args: - block_iter: An iterator over blocks. + batch_iter: An iterator over batches. batch_format: The batch format to use. stats: An optional stats object to record formatting times. Returns: An iterator over batch index and the formatted batch. """ - for batch in block_iter: + for batch in batch_iter: with stats.iter_format_batch_s.timer() if stats else nullcontext(): formatted_batch = BlockAccessor.for_block(batch.data).to_batch_format( batch_format ) - yield Batch(batch.batch_idx, formatted_batch) + yield dataclasses.replace(batch, data=formatted_batch) def collate( @@ -180,7 +182,7 @@ def collate( for batch in batch_iter: with stats.iter_collate_batch_s.timer() if stats else nullcontext(): collated_batch = collate_fn(batch.data) - yield CollatedBatch(batch.batch_idx, collated_batch) + yield CollatedBatch(metadata=batch.metadata, data=collated_batch) def finalize_batches( @@ -204,7 +206,7 @@ def finalize_batches( for batch in batch_iter: with stats.iter_finalize_batch_s.timer() if stats else nullcontext(): finalized_batch = finalize_fn(batch.data) - yield CollatedBatch(batch.batch_idx, finalized_batch) + yield dataclasses.replace(batch, data=finalized_batch) def extract_data_from_batch(batch_iter: Iterator[Batch]) -> Iterator[Any]: diff --git a/python/ray/data/tests/block_batching/test_iter_batches.py b/python/ray/data/tests/block_batching/test_iter_batches.py index 36f4b8a5005c..7ee6812fab9a 100644 --- a/python/ray/data/tests/block_batching/test_iter_batches.py +++ b/python/ray/data/tests/block_batching/test_iter_batches.py @@ -8,7 +8,11 @@ import pytest import ray -from ray.data._internal.block_batching.interfaces import Batch, BlockPrefetcher +from ray.data._internal.block_batching.interfaces import ( + Batch, + BatchMetadata, + BlockPrefetcher, +) from ray.data._internal.block_batching.iter_batches import ( BatchIterator, prefetch_batches_locally, @@ -95,14 +99,14 @@ def prefetch_blocks(self, block_refs: List[ObjectRef[Block]]): def test_restore_from_original_order(): base_iterator = [ - Batch(1, None), - Batch(0, None), - Batch(3, None), - Batch(2, None), + Batch(BatchMetadata(batch_idx=1), None), + Batch(BatchMetadata(batch_idx=0), None), + Batch(BatchMetadata(batch_idx=3), None), + Batch(BatchMetadata(batch_idx=2), None), ] ordered = list(restore_original_order(iter(base_iterator))) - idx = [batch.batch_idx for batch in ordered] + idx = [batch.metadata.batch_idx for batch in ordered] assert idx == [0, 1, 2, 3] diff --git a/python/ray/data/tests/block_batching/test_util.py b/python/ray/data/tests/block_batching/test_util.py index 098ed64a4004..f8be82e43281 100644 --- a/python/ray/data/tests/block_batching/test_util.py +++ b/python/ray/data/tests/block_batching/test_util.py @@ -10,7 +10,7 @@ import pytest import ray -from ray.data._internal.block_batching.interfaces import Batch +from ray.data._internal.block_batching.interfaces import Batch, BatchMetadata from ray.data._internal.block_batching.util import ( _calculate_ref_hits, blocks_to_batches, @@ -64,13 +64,17 @@ def test_blocks_to_batches(block_size, drop_last): assert leftover_batches == 1 assert full_batches == (dataset_size // batch_size) - assert [batch.batch_idx for batch in batch_iter] == list(range(len(batch_iter))) + assert [batch.metadata.batch_idx for batch in batch_iter] == list( + range(len(batch_iter)) + ) @pytest.mark.parametrize("batch_format", ["pandas", "numpy", "pyarrow"]) def test_format_batches(batch_format): block_iter = block_generator(num_rows=2, num_blocks=2) - batch_iter = (Batch(i, block) for i, block in enumerate(block_iter)) + batch_iter = ( + Batch(BatchMetadata(batch_idx=i), block) for i, block in enumerate(block_iter) + ) batch_iter = list(format_batches(batch_iter, batch_format=batch_format)) for batch in batch_iter: @@ -82,7 +86,9 @@ def test_format_batches(batch_format): assert isinstance(batch.data, dict) assert isinstance(batch.data["foo"], np.ndarray) - assert [batch.batch_idx for batch in batch_iter] == list(range(len(batch_iter))) + assert [batch.metadata.batch_idx for batch in batch_iter] == list( + range(len(batch_iter)) + ) def test_collate(): @@ -90,13 +96,13 @@ def collate_fn(batch): return pa.table({"bar": [1] * 2}) batches = [ - Batch(i, data) + Batch(BatchMetadata(batch_idx=i), data) for i, data in enumerate(block_generator(num_rows=2, num_blocks=2)) ] batch_iter = collate(batches, collate_fn=collate_fn) for i, batch in enumerate(batch_iter): - assert batch.batch_idx == i + assert batch.metadata.batch_idx == i assert batch.data == pa.table({"bar": [1] * 2}) @@ -105,13 +111,13 @@ def finalize_fn(batch): return pa.table({"bar": [1] * 2}) batches = [ - Batch(i, data) + Batch(BatchMetadata(batch_idx=i), data) for i, data in enumerate(block_generator(num_rows=2, num_blocks=2)) ] batch_iter = finalize_batches(batches, finalize_fn=finalize_fn) for i, batch in enumerate(batch_iter): - assert batch.batch_idx == i + assert batch.metadata.batch_idx == i assert batch.data == pa.table({"bar": [1] * 2}) From 8d6d9fa4c63e7d1e7ecd7f14347c1a565efe4d95 Mon Sep 17 00:00:00 2001 From: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:56:53 -0700 Subject: [PATCH 130/634] [serve.llm] Correct Pyright lints for Ray Serve LLM examples (#55284) Signed-off-by: Seiji Eicher --- .../serve/builders/application_builders.py | 7 +- .../_internal/serve/configs/server_models.py | 45 +++++++++++-- .../tests/serve/cpu/configs/test_models.py | 64 ++++++++++++++++++- python/ray/serve/llm/__init__.py | 10 +-- python/ray/serve/tests/test_api.py | 51 ++++++++++++++- 5 files changed, 163 insertions(+), 14 deletions(-) diff --git a/python/ray/llm/_internal/serve/builders/application_builders.py b/python/ray/llm/_internal/serve/builders/application_builders.py index 201974b36514..45c81da72aec 100644 --- a/python/ray/llm/_internal/serve/builders/application_builders.py +++ b/python/ray/llm/_internal/serve/builders/application_builders.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Sequence +from typing import Any, Dict, List, Optional, Sequence, overload from ray.llm._internal.serve.configs.server_models import ( LLMConfig, @@ -52,6 +52,11 @@ def _get_llm_deployments( return llm_deployments +@overload +def build_openai_app(llm_serving_args: Dict[str, Any]) -> Application: + ... + + def build_openai_app(llm_serving_args: LLMServingArgs) -> Application: rayllm_args = LLMServingArgs.model_validate(llm_serving_args).parse_args() diff --git a/python/ray/llm/_internal/serve/configs/server_models.py b/python/ray/llm/_internal/serve/configs/server_models.py index f4d42174ef4e..5be9ff21a1f9 100644 --- a/python/ray/llm/_internal/serve/configs/server_models.py +++ b/python/ray/llm/_internal/serve/configs/server_models.py @@ -139,15 +139,15 @@ class ModelLoadingConfig(BaseModelExtended): class LLMConfig(BaseModelExtended): runtime_env: Optional[Dict[str, Any]] = Field( - None, + default=None, description=( "The runtime_env to use for the model deployment replica " "and the engine workers." ), ) - model_loading_config: ModelLoadingConfig = Field( - description="The settings for how to download and expose the model." + model_loading_config: Union[Dict[str, Any], ModelLoadingConfig] = Field( + description="The settings for how to download and expose the model. Validated against ModelLoadingConfig." ) llm_engine: str = Field( @@ -177,8 +177,9 @@ class LLMConfig(BaseModelExtended): description=f"The type of accelerator runs the model on. Only the following values are supported: {str([t.value for t in GPUType])}", ) - lora_config: Optional[LoraConfig] = Field( - default=None, description="Settings for LoRA adapter." + lora_config: Optional[Union[Dict[str, Any], LoraConfig]] = Field( + default=None, + description="Settings for LoRA adapter. Validated against LoraConfig.", ) deployment_config: Dict[str, Any] = Field( @@ -190,7 +191,7 @@ class LLMConfig(BaseModelExtended): `autoscaling_config`, `max_queued_requests`, `user_config`, `health_check_period_s`, `health_check_timeout_s`, `graceful_shutdown_wait_loop_s`, `graceful_shutdown_timeout_s`, - `logging_config`. + `logging_config`, `request_router_config`. For more details, see the `Ray Serve Documentation `_. """, ) @@ -209,7 +210,7 @@ class LLMConfig(BaseModelExtended): ) log_engine_metrics: Optional[bool] = Field( - False, + default=False, description="Enable additional engine metrics via Ray Prometheus port. Only compatible with V1 vLLM engine. NOTE: once v1 is fully rolled out, we will remove this flag and turn it on by default.", ) @@ -332,6 +333,36 @@ def validate_deployment_config(cls, value: Dict[str, Any]) -> Dict[str, Any]: return value + @field_validator("model_loading_config") + def validate_model_loading_config( + cls, value: Union[Dict[str, Any], ModelLoadingConfig] + ) -> ModelLoadingConfig: + """Validates the model loading config dictionary.""" + if isinstance(value, ModelLoadingConfig): + return value + + try: + model_loading_config = ModelLoadingConfig(**value) + except Exception as e: + raise ValueError(f"Invalid model_loading_config: {value}") from e + + return model_loading_config + + @field_validator("lora_config") + def validate_lora_config( + cls, value: Optional[Union[Dict[str, Any], LoraConfig]] + ) -> Optional[LoraConfig]: + """Validates the lora config dictionary.""" + if value is None or isinstance(value, LoraConfig): + return value + + try: + lora_config = LoraConfig(**value) + except Exception as e: + raise ValueError(f"Invalid lora_config: {value}") from e + + return lora_config + @model_validator(mode="after") def _check_log_stats_with_metrics(self): # Require disable_log_stats is not set to True when log_engine_metrics is enabled. diff --git a/python/ray/llm/tests/serve/cpu/configs/test_models.py b/python/ray/llm/tests/serve/cpu/configs/test_models.py index 057e27b8c65a..6f5bfab6d1b6 100644 --- a/python/ray/llm/tests/serve/cpu/configs/test_models.py +++ b/python/ray/llm/tests/serve/cpu/configs/test_models.py @@ -4,7 +4,11 @@ import pydantic import pytest -from ray.llm._internal.serve.configs.server_models import LLMConfig, ModelLoadingConfig +from ray.llm._internal.serve.configs.server_models import ( + LLMConfig, + LoraConfig, + ModelLoadingConfig, +) CONFIG_DIRS_PATH = str(Path(__file__).parent / "configs") @@ -302,5 +306,63 @@ def test_log_engine_metrics_disable_log_stats_validation(self): ) +class TestFieldValidators: + """Test the field validators for dict validation.""" + + def test_model_loading_config_dict_validation(self): + """Test that model_loading_config accepts and validates dict input.""" + config_dict = {"model_id": "microsoft/DialoGPT-medium"} + + llm_config = LLMConfig(model_loading_config=config_dict, llm_engine="vLLM") + + assert isinstance(llm_config.model_loading_config, ModelLoadingConfig) + assert llm_config.model_loading_config.model_id == "microsoft/DialoGPT-medium" + + def test_model_loading_config_validation_error(self): + """Test that invalid dict raises proper validation error.""" + with pytest.raises(pydantic.ValidationError) as exc_info: + LLMConfig( + model_loading_config={"invalid_field": "value"}, llm_engine="vLLM" + ) + + assert "Invalid model_loading_config" in str(exc_info.value) + + def test_lora_config_dict_validation(self): + """Test that lora_config accepts and validates dict input.""" + llm_config = LLMConfig( + model_loading_config={"model_id": "test"}, + lora_config=None, + llm_engine="vLLM", + ) + + assert llm_config.lora_config is None + + lora_dict = { + "dynamic_lora_loading_path": "s3://bucket/lora", + "max_num_adapters_per_replica": 8, + } + + llm_config2 = LLMConfig( + model_loading_config={"model_id": "test"}, + lora_config=lora_dict, + llm_engine="vLLM", + ) + + assert isinstance(llm_config2.lora_config, LoraConfig) + assert llm_config2.lora_config.max_num_adapters_per_replica == 8 + assert llm_config2.lora_config.dynamic_lora_loading_path == "s3://bucket/lora" + + def test_lora_config_validation_error(self): + """Test that invalid lora config dict raises proper validation error.""" + with pytest.raises(pydantic.ValidationError) as exc_info: + LLMConfig( + model_loading_config={"model_id": "test"}, + lora_config={"max_num_adapters_per_replica": "invalid_string"}, + llm_engine="vLLM", + ) + + assert "Invalid lora_config" in str(exc_info.value) + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/serve/llm/__init__.py b/python/ray/serve/llm/__init__.py index 6bee92952088..3b04709a0bae 100644 --- a/python/ray/serve/llm/__init__.py +++ b/python/ray/serve/llm/__init__.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional, Union from ray.llm._internal.serve.configs.server_models import ( CloudMirrorConfig as _CloudMirrorConfig, @@ -160,7 +160,9 @@ async def query_model(model_handle): @PublicAPI(stability="alpha") -def build_openai_app(llm_serving_args: "LLMServingArgs") -> "Application": +def build_openai_app( + llm_serving_args: Union["LLMServingArgs", Dict[str, Any]] +) -> "Application": """Helper to build an OpenAI compatible app with the llm deployment setup from the given llm serving args. This is the main entry point for users to create a Serve application serving LLMs. @@ -252,8 +254,8 @@ def build_openai_app(llm_serving_args: "LLMServingArgs") -> "Application": Args: - llm_serving_args: The list of llm configs or the paths to the llm config to - build the app. + llm_serving_args: Either a dict with "llm_configs" key containing a list of + LLMConfig objects, or an LLMServingArgs object. Returns: The configured Ray Serve Application router. diff --git a/python/ray/serve/tests/test_api.py b/python/ray/serve/tests/test_api.py index 75b76b9fb7a2..b3d5419a79d3 100644 --- a/python/ray/serve/tests/test_api.py +++ b/python/ray/serve/tests/test_api.py @@ -1,7 +1,7 @@ import asyncio import os import sys -from typing import Dict, List, Optional +from typing import Dict, List, Optional, overload import httpx import pytest @@ -1136,6 +1136,55 @@ def test_custom_request_router_kwargs(serve_instance): assert handle.remote().result() == "Hello, world!" +def test_overloaded_app_builder_signatures(): + """Test that call_user_app_builder_with_args_if_necessary validates the base + function signature with a pydantic basemodel, rather than the overload that + accepts a dict (for the sake of lint permissiveness). + """ + + class Config(BaseModel): + name: str + value: int = 42 + + @serve.deployment + class MockDeployment: + def __call__(self): + return "mock" + + mock_app = MockDeployment.bind() + + # Overloaded function where the implementation has a pydantic annotation + @overload + def overloaded_builder(args: dict) -> Application: + ... + + def overloaded_builder(args: Config) -> Application: + """Implementation with pydantic BaseModel annotation.""" + + assert isinstance(args, Config), f"Expected Config but got {type(args)}" + return mock_app + + # Test 1: Valid input should work and convert to Config model + result = call_user_app_builder_with_args_if_necessary( + overloaded_builder, {"name": "test", "value": 123} + ) + assert isinstance(result, Application) + + # Test 2: Invalid dict input should raise validation error + # Missing required field 'name' + with pytest.raises(ValidationError): + call_user_app_builder_with_args_if_necessary( + overloaded_builder, {"value": 123} # Missing required 'name' field + ) + + # Test 3: Wrong type should also raise validation error + with pytest.raises(ValidationError): + call_user_app_builder_with_args_if_necessary( + overloaded_builder, + {"name": "test", "value": "not_an_int"}, # 'value' should be int + ) + + if __name__ == "__main__": import sys From 616b9a19b42305ba5602e4f3bcab81c1e19cf3a0 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 15 Aug 2025 13:05:36 -0500 Subject: [PATCH 131/634] [core] Clean up `RayletIpcClientInterface` (#55651) Splits out `raylet_ipc_client_interface.h` into its own target. Sub-interfaces that use the client should only depend on this interface, not the full `raylet_ipc_client` target. This improves incremental builds. For example, now if `raylet_ipc_client.{h,cc}` changes (including any of its transitive dependencies), the core worker `store_provider` targets no longer need to be recompiled. They'll only be recompiled if `raylet_ipc_client_interface.h` changes, which should be much less frequent. I've also moved the `FakeRayletIpcClient` into the source tree. --------- Signed-off-by: Edward Oakes --- BUILD.bazel | 4 +- src/ray/core_worker/BUILD.bazel | 4 +- src/ray/core_worker/core_worker.cc | 2 +- src/ray/core_worker/core_worker.h | 6 +- .../memory_store/memory_store.cc | 4 +- .../memory_store/memory_store.h | 6 +- .../store_provider/plasma_store_provider.cc | 9 +- .../store_provider/plasma_store_provider.h | 6 +- src/ray/core_worker/test/BUILD.bazel | 1 + src/ray/core_worker/test/core_worker_test.cc | 2 +- src/ray/ipc/BUILD.bazel | 42 +++- .../ipc/fake_raylet_ipc_client.h} | 2 +- src/ray/ipc/raylet_ipc_client.h | 157 +-------------- src/ray/ipc/raylet_ipc_client_interface.h | 190 ++++++++++++++++++ 14 files changed, 250 insertions(+), 185 deletions(-) rename src/{fakes/ray/ipc/raylet_ipc_client.h => ray/ipc/fake_raylet_ipc_client.h} (98%) create mode 100644 src/ray/ipc/raylet_ipc_client_interface.h diff --git a/BUILD.bazel b/BUILD.bazel index fb6f2b58e163..804a9591b91d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -118,7 +118,9 @@ ray_cc_library( ray_cc_library( name = "ray_fakes", - hdrs = glob(["src/fakes/**/*.h"]), + hdrs = glob( + ["src/fakes/**/*.h"], + ), deps = [ "//src/ray/common:asio", "//src/ray/raylet_client:raylet_client_lib", diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index a47df3870715..8d41c345f4c2 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -212,7 +212,7 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:ray_config", "//src/ray/common:status", - "//src/ray/ipc:raylet_ipc_client", + "//src/ray/ipc:raylet_ipc_client_interface", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/synchronization", @@ -346,7 +346,7 @@ ray_cc_library( "//src/ray/common:ray_config", "//src/ray/common:status", "//src/ray/common:task_common", - "//src/ray/ipc:raylet_ipc_client", + "//src/ray/ipc:raylet_ipc_client_interface", "//src/ray/object_manager/plasma:plasma_client", "//src/ray/protobuf:common_cc_proto", "@com_google_absl//absl/container:flat_hash_map", diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index f6c664909723..53288a3c45d9 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -273,7 +273,7 @@ CoreWorker::CoreWorker( std::unique_ptr core_worker_server, rpc::Address rpc_address, std::shared_ptr gcs_client, - std::shared_ptr raylet_ipc_client, + std::shared_ptr raylet_ipc_client, std::shared_ptr local_raylet_rpc_client, boost::thread &io_thread, std::shared_ptr reference_counter, diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index 839b8a96f177..2299894a3956 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -48,7 +48,7 @@ #include "ray/core_worker/task_execution/task_receiver.h" #include "ray/core_worker/task_submission/normal_task_submitter.h" #include "ray/gcs/gcs_client/gcs_client.h" -#include "ray/ipc/raylet_ipc_client.h" +#include "ray/ipc/raylet_ipc_client_interface.h" #include "ray/pubsub/publisher.h" #include "ray/pubsub/subscriber.h" #include "ray/raylet_client/raylet_client.h" @@ -180,7 +180,7 @@ class CoreWorker { std::unique_ptr core_worker_server, rpc::Address rpc_address, std::shared_ptr gcs_client, - std::shared_ptr raylet_ipc_client, + std::shared_ptr raylet_ipc_client, std::shared_ptr local_raylet_rpc_client, boost::thread &io_thread, std::shared_ptr reference_counter, @@ -1740,7 +1740,7 @@ class CoreWorker { std::shared_ptr gcs_client_; // Client to the local Raylet that goes over a local socket. - std::shared_ptr raylet_ipc_client_; + std::shared_ptr raylet_ipc_client_; // Client to the local Raylet that goes over a gRPC connection. std::shared_ptr local_raylet_rpc_client_; diff --git a/src/ray/core_worker/store_provider/memory_store/memory_store.cc b/src/ray/core_worker/store_provider/memory_store/memory_store.cc index e6b58485728f..3898e048354e 100644 --- a/src/ray/core_worker/store_provider/memory_store/memory_store.cc +++ b/src/ray/core_worker/store_provider/memory_store/memory_store.cc @@ -21,7 +21,7 @@ #include #include "ray/common/ray_config.h" -#include "ray/ipc/raylet_ipc_client.h" +#include "ray/ipc/raylet_ipc_client_interface.h" namespace ray { namespace core { @@ -136,7 +136,7 @@ std::shared_ptr GetRequest::Get(const ObjectID &object_id) const { CoreWorkerMemoryStore::CoreWorkerMemoryStore( instrumented_io_context &io_context, ReferenceCounter *counter, - std::shared_ptr raylet_ipc_client, + std::shared_ptr raylet_ipc_client, std::function check_signals, std::function unhandled_exception_handler, std::function( diff --git a/src/ray/core_worker/store_provider/memory_store/memory_store.h b/src/ray/core_worker/store_provider/memory_store/memory_store.h index 509938d6dd61..17d1d4df1555 100644 --- a/src/ray/core_worker/store_provider/memory_store/memory_store.h +++ b/src/ray/core_worker/store_provider/memory_store/memory_store.h @@ -27,7 +27,7 @@ #include "ray/common/status.h" #include "ray/core_worker/context.h" #include "ray/core_worker/reference_count.h" -#include "ray/ipc/raylet_ipc_client.h" +#include "ray/ipc/raylet_ipc_client_interface.h" namespace ray { namespace core { @@ -54,7 +54,7 @@ class CoreWorkerMemoryStore { explicit CoreWorkerMemoryStore( instrumented_io_context &io_context, ReferenceCounter *counter = nullptr, - std::shared_ptr raylet_ipc_client = nullptr, + std::shared_ptr raylet_ipc_client = nullptr, std::function check_signals = nullptr, std::function unhandled_exception_handler = nullptr, std::function(const RayObject &object, @@ -210,7 +210,7 @@ class CoreWorkerMemoryStore { ReferenceCounter *ref_counter_; // If set, this will be used to notify worker blocked / unblocked on get calls. - std::shared_ptr raylet_ipc_client_; + std::shared_ptr raylet_ipc_client_; /// Protects the data structures below. mutable absl::Mutex mu_; diff --git a/src/ray/core_worker/store_provider/plasma_store_provider.cc b/src/ray/core_worker/store_provider/plasma_store_provider.cc index caaa2b30ef34..7dd0589e6e77 100644 --- a/src/ray/core_worker/store_provider/plasma_store_provider.cc +++ b/src/ray/core_worker/store_provider/plasma_store_provider.cc @@ -23,7 +23,7 @@ #include "ray/common/ray_config.h" #include "ray/common/status.h" #include "ray/common/status_or.h" -#include "ray/ipc/raylet_ipc_client.h" +#include "ray/ipc/raylet_ipc_client_interface.h" #include "src/ray/protobuf/common.pb.h" namespace ray { @@ -62,7 +62,7 @@ BufferTracker::UsedObjects() const { CoreWorkerPlasmaStoreProvider::CoreWorkerPlasmaStoreProvider( const std::string &store_socket, - const std::shared_ptr raylet_ipc_client, + const std::shared_ptr raylet_ipc_client, ReferenceCounter &reference_counter, std::function check_signals, bool warmup, @@ -255,8 +255,9 @@ Status CoreWorkerPlasmaStoreProvider::GetExperimentalMutableObject( return store_client_->GetExperimentalMutableObject(object_id, mutable_object); } -Status UnblockIfNeeded(const std::shared_ptr &raylet_client, - const WorkerContext &ctx) { +Status UnblockIfNeeded( + const std::shared_ptr &raylet_client, + const WorkerContext &ctx) { if (ctx.CurrentTaskIsDirectCall()) { // NOTE: for direct call actors, we still need to issue an unblock IPC to release // get subscriptions, even if the worker isn't blocked. diff --git a/src/ray/core_worker/store_provider/plasma_store_provider.h b/src/ray/core_worker/store_provider/plasma_store_provider.h index 867fb739e9e6..c0997f5c0222 100644 --- a/src/ray/core_worker/store_provider/plasma_store_provider.h +++ b/src/ray/core_worker/store_provider/plasma_store_provider.h @@ -28,7 +28,7 @@ #include "ray/core_worker/common.h" #include "ray/core_worker/context.h" #include "ray/core_worker/reference_count.h" -#include "ray/ipc/raylet_ipc_client.h" +#include "ray/ipc/raylet_ipc_client_interface.h" #include "ray/object_manager/plasma/client.h" #include "src/ray/protobuf/common.pb.h" @@ -96,7 +96,7 @@ class CoreWorkerPlasmaStoreProvider { public: CoreWorkerPlasmaStoreProvider( const std::string &store_socket, - const std::shared_ptr raylet_ipc_client, + const std::shared_ptr raylet_ipc_client, ReferenceCounter &reference_counter, std::function check_signals, bool warmup, @@ -235,7 +235,7 @@ class CoreWorkerPlasmaStoreProvider { /// \return status Status WarmupStore(); - const std::shared_ptr raylet_ipc_client_; + const std::shared_ptr raylet_ipc_client_; std::shared_ptr store_client_; /// Used to look up a plasma object's owner. ReferenceCounter &reference_counter_; diff --git a/src/ray/core_worker/test/BUILD.bazel b/src/ray/core_worker/test/BUILD.bazel index f94145237eb9..d9fef40c7b6d 100644 --- a/src/ray/core_worker/test/BUILD.bazel +++ b/src/ray/core_worker/test/BUILD.bazel @@ -230,6 +230,7 @@ ray_cc_test( "//src/ray/core_worker:core_worker_lib", "//src/ray/core_worker:memory_store", "//src/ray/core_worker:reference_count", + "//src/ray/ipc:fake_raylet_ipc_client", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/core_worker/test/core_worker_test.cc b/src/ray/core_worker/test/core_worker_test.cc index bf45ffa279ef..0fa7170c1838 100644 --- a/src/ray/core_worker/test/core_worker_test.cc +++ b/src/ray/core_worker/test/core_worker_test.cc @@ -25,7 +25,6 @@ #include #include "fakes/ray/common/asio/fake_periodical_runner.h" -#include "fakes/ray/ipc/raylet_ipc_client.h" #include "fakes/ray/pubsub/publisher.h" #include "fakes/ray/pubsub/subscriber.h" #include "fakes/ray/rpc/raylet/raylet_client.h" @@ -40,6 +39,7 @@ #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/core_worker/task_submission/actor_task_submitter.h" #include "ray/core_worker/task_submission/normal_task_submitter.h" +#include "ray/ipc/fake_raylet_ipc_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" namespace ray { diff --git a/src/ray/ipc/BUILD.bazel b/src/ray/ipc/BUILD.bazel index 08d01f774235..501206ad3693 100644 --- a/src/ray/ipc/BUILD.bazel +++ b/src/ray/ipc/BUILD.bazel @@ -1,19 +1,24 @@ load("//bazel:ray.bzl", "ray_cc_library") ray_cc_library( - name = "client_connection", - srcs = [ - "client_connection.cc", - ], - hdrs = [ - "client_connection.h", - ], + name = "raylet_ipc_client_interface", + hdrs = ["raylet_ipc_client_interface.h"], deps = [ - "//src/ray/common:asio", - "//src/ray/common:event_stats", + "//src/ray/common:buffer", "//src/ray/common:id", "//src/ray/common:status", "//src/ray/flatbuffers:node_manager_generated", + "//src/ray/protobuf:common_cc_proto", + "//src/ray/util:process", + "@com_google_absl//absl/container:flat_hash_set", + ], +) + +ray_cc_library( + name = "fake_raylet_ipc_client", + hdrs = ["fake_raylet_ipc_client.h"], + deps = [ + "//src/ray/ipc:raylet_ipc_client_interface", ], ) @@ -21,6 +26,7 @@ ray_cc_library( name = "raylet_ipc_client", srcs = ["raylet_ipc_client.cc"], hdrs = ["raylet_ipc_client.h"], + visibility = ["//src/ray/core_worker:__pkg__"], deps = [ ":client_connection", "//src/ray/common:asio", @@ -28,9 +34,27 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:status", "//src/ray/flatbuffers:node_manager_generated", + "//src/ray/ipc:raylet_ipc_client_interface", "//src/ray/protobuf:common_cc_proto", "//src/ray/util:logging", "//src/ray/util:process", "@com_google_absl//absl/container:flat_hash_set", ], ) + +ray_cc_library( + name = "client_connection", + srcs = [ + "client_connection.cc", + ], + hdrs = [ + "client_connection.h", + ], + deps = [ + "//src/ray/common:asio", + "//src/ray/common:event_stats", + "//src/ray/common:id", + "//src/ray/common:status", + "//src/ray/flatbuffers:node_manager_generated", + ], +) diff --git a/src/fakes/ray/ipc/raylet_ipc_client.h b/src/ray/ipc/fake_raylet_ipc_client.h similarity index 98% rename from src/fakes/ray/ipc/raylet_ipc_client.h rename to src/ray/ipc/fake_raylet_ipc_client.h index dc7b0ca92bc0..1b8d32c8d227 100644 --- a/src/fakes/ray/ipc/raylet_ipc_client.h +++ b/src/ray/ipc/fake_raylet_ipc_client.h @@ -14,7 +14,7 @@ #pragma once -#include "ray/ipc/raylet_ipc_client.h" +#include "ray/ipc/raylet_ipc_client_interface.h" namespace ray { namespace ipc { diff --git a/src/ray/ipc/raylet_ipc_client.h b/src/ray/ipc/raylet_ipc_client.h index 15ee56670513..b7faec8e8ca6 100644 --- a/src/ray/ipc/raylet_ipc_client.h +++ b/src/ray/ipc/raylet_ipc_client.h @@ -28,167 +28,15 @@ #include "ray/common/status_or.h" #include "ray/flatbuffers/node_manager_generated.h" #include "ray/ipc/client_connection.h" +#include "ray/ipc/raylet_ipc_client_interface.h" #include "ray/util/process.h" #include "src/ray/protobuf/common.pb.h" using MessageType = ray::protocol::MessageType; namespace ray { - -class RayletIpcClientInterface { - public: - virtual ~RayletIpcClientInterface() = default; - - /// Register this client (worker) with the local Raylet. - /// - /// \param worker_id The worker_id of the connecting worker. - /// \param worker_type The worker type of the connecting worker. - /// \param job_id The job ID that the connecting worker is associated with. - /// \param runtime_env_hash The runtime_env hash of the connecting worker. - /// \param language The language of the connecting worker. - /// \param ip_address The ip_address of the connecting worker. - /// \param serialized_job_config The serialized job config of the connecting worker. - /// \param startup_token The token that was passed to this worker at startup. - /// \param[out] node_id The node ID for the local Raylet. - /// \param[out] assigned_port The assigned port for the worker to listen on. If zero, - /// the worker should pick a port randomly. - virtual ray::Status RegisterClient(const WorkerID &worker_id, - rpc::WorkerType worker_type, - const JobID &job_id, - int runtime_env_hash, - const rpc::Language &language, - const std::string &ip_address, - const std::string &serialized_job_config, - const StartupToken &startup_token, - NodeID *node_id, - int *assigned_port) = 0; - - /// Notify the raylet that this client is disconnecting gracefully. This - /// is used by actors to exit gracefully so that the raylet doesn't - /// propagate an error message to the driver. - /// - /// It's a blocking call. - /// - /// \param disconnect_type The reason why this worker process is disconnected. - /// \param disconnect_detail The detailed reason for a given exit. - /// \return ray::Status. - virtual ray::Status Disconnect( - const rpc::WorkerExitType &exit_type, - const std::string &exit_detail, - const std::shared_ptr &creation_task_exception_pb_bytes) = 0; - - /// Tell the raylet which port this worker's gRPC server is listening on. - /// - /// \param port The port. - /// \return ray::Status. - virtual Status AnnounceWorkerPortForWorker(int port) = 0; - - /// Tell the raylet this driver and its job is ready to run, with port and entrypoint. - /// - /// \param port The port. - /// \param entrypoint The entrypoint of the driver's job. - /// \return ray::Status. - virtual Status AnnounceWorkerPortForDriver(int port, const std::string &entrypoint) = 0; - - /// Tell the raylet that the client has finished executing a task. - /// - /// \return ray::Status. - virtual ray::Status ActorCreationTaskDone() = 0; - - /// Ask the Raylet to pull a set of objects to the local node. - /// - /// This request is asynchronous. - /// - /// \param object_ids The IDs of the objects to pull. - /// \param owner_addresses The owner addresses of the objects. - /// \return ray::Status. - virtual ray::Status AsyncGetObjects( - const std::vector &object_ids, - const std::vector &owner_addresses) = 0; - - /// Wait for the given objects until timeout expires or num_return objects are - /// found. - /// - /// \param object_ids The objects to wait for. - /// \param owner_addresses The addresses of the workers that own the objects. - /// \param num_returns The number of objects to wait for. - /// \param timeout_milliseconds Duration, in milliseconds, to wait before returning. - /// \param result A pair with the first element containing the object ids that were - /// found, and the second element the objects that were not found. - /// \return ray::StatusOr containing error status or the set of object ids that were - /// found. - virtual ray::StatusOr> Wait( - const std::vector &object_ids, - const std::vector &owner_addresses, - int num_returns, - int64_t timeout_milliseconds) = 0; - - /// Tell the Raylet to cancel the get request from this worker. - /// - /// \return ray::Status. - virtual ray::Status CancelGetRequest() = 0; - - /// Notify the raylet that this client is blocked. This is only used for direct task - /// calls. Note that ordering of this with respect to Unblock calls is important. - /// - /// \return ray::Status. - virtual ray::Status NotifyDirectCallTaskBlocked() = 0; - - /// Notify the raylet that this client is unblocked. This is only used for direct task - /// calls. Note that ordering of this with respect to Block calls is important. - /// - /// \return ray::Status. - virtual ray::Status NotifyDirectCallTaskUnblocked() = 0; - - /// Wait for the given objects asynchronously. - /// - /// The core worker will be notified over gRPC when the wait completes. - /// - /// \param references The objects to wait for. - /// \param tag Value that will be sent to the core worker via gRPC on completion. - /// \return ray::Status. - virtual ray::Status WaitForActorCallArgs( - const std::vector &references, int64_t tag) = 0; - - /// Push an error to the relevant driver. - /// - /// \param The ID of the job_id that the error is for. - /// \param The type of the error. - /// \param The error message. - /// \param The timestamp of the error. - /// \return ray::Status. - virtual ray::Status PushError(const ray::JobID &job_id, - const std::string &type, - const std::string &error_message, - double timestamp) = 0; - - /// Free a list of objects from object stores. - /// - /// \param object_ids A list of ObjectsIDs to be deleted. - /// \param local_only Whether keep this request with local object store - /// or send it to all the object stores. - /// \return ray::Status. - virtual ray::Status FreeObjects(const std::vector &object_ids, - bool local_only) = 0; - - /// Subscribe this worker to a notification when the provided object is ready in the - /// local object store. - /// - /// The worker will be notified over gRPC when the object is ready. - /// - /// \param object_id The ID of the object to subscribe to. - /// \param owner_address The address of the owner of the object. - virtual void SubscribePlasmaReady(const ObjectID &object_id, - const rpc::Address &owner_address) = 0; -}; - namespace ipc { -/// Interface for interacting with the local Raylet over a socket. -/// -/// Message ordering on the socket is guaranteed. -/// -/// If the socket is broken and the local Raylet is detected to be dead, calling any -/// method on the client will quick exit the process. + class RayletIpcClient : public RayletIpcClientInterface { public: /// Connect to the Raylet over a local socket. @@ -275,5 +123,4 @@ class RayletIpcClient : public RayletIpcClientInterface { }; } // namespace ipc - } // namespace ray diff --git a/src/ray/ipc/raylet_ipc_client_interface.h b/src/ray/ipc/raylet_ipc_client_interface.h new file mode 100644 index 000000000000..4d92c5c5023a --- /dev/null +++ b/src/ray/ipc/raylet_ipc_client_interface.h @@ -0,0 +1,190 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "absl/container/flat_hash_set.h" +#include "ray/common/buffer.h" +#include "ray/common/id.h" +#include "ray/common/status.h" +#include "ray/common/status_or.h" +#include "ray/flatbuffers/node_manager_generated.h" +#include "ray/util/process.h" +#include "src/ray/protobuf/common.pb.h" + +using MessageType = ray::protocol::MessageType; + +namespace ray { +namespace ipc { + +/// Interface for interacting with the local Raylet over a socket. +/// +/// Message ordering is guaranteed. +/// +/// If the local Raylet is detected to be dead, calling any +/// method on the client will un-gracefully exit the process. +class RayletIpcClientInterface { + public: + virtual ~RayletIpcClientInterface() = default; + + /// Register this client (worker) with the local Raylet. + /// + /// \param worker_id The worker_id of the connecting worker. + /// \param worker_type The worker type of the connecting worker. + /// \param job_id The job ID that the connecting worker is associated with. + /// \param runtime_env_hash The runtime_env hash of the connecting worker. + /// \param language The language of the connecting worker. + /// \param ip_address The ip_address of the connecting worker. + /// \param serialized_job_config The serialized job config of the connecting worker. + /// \param startup_token The token that was passed to this worker at startup. + /// \param[out] node_id The node ID for the local Raylet. + /// \param[out] assigned_port The assigned port for the worker to listen on. If zero, + /// the worker should pick a port randomly. + virtual ray::Status RegisterClient(const WorkerID &worker_id, + rpc::WorkerType worker_type, + const JobID &job_id, + int runtime_env_hash, + const rpc::Language &language, + const std::string &ip_address, + const std::string &serialized_job_config, + const StartupToken &startup_token, + NodeID *node_id, + int *assigned_port) = 0; + + /// Notify the raylet that this client is disconnecting gracefully. This + /// is used by actors to exit gracefully so that the raylet doesn't + /// propagate an error message to the driver. + /// + /// It's a blocking call. + /// + /// \param disconnect_type The reason why this worker process is disconnected. + /// \param disconnect_detail The detailed reason for a given exit. + /// \return ray::Status. + virtual ray::Status Disconnect( + const rpc::WorkerExitType &exit_type, + const std::string &exit_detail, + const std::shared_ptr &creation_task_exception_pb_bytes) = 0; + + /// Tell the raylet which port this worker's gRPC server is listening on. + /// + /// \param port The port. + /// \return ray::Status. + virtual Status AnnounceWorkerPortForWorker(int port) = 0; + + /// Tell the raylet this driver and its job is ready to run, with port and entrypoint. + /// + /// \param port The port. + /// \param entrypoint The entrypoint of the driver's job. + /// \return ray::Status. + virtual Status AnnounceWorkerPortForDriver(int port, const std::string &entrypoint) = 0; + + /// Tell the raylet that the client has finished executing a task. + /// + /// \return ray::Status. + virtual ray::Status ActorCreationTaskDone() = 0; + + /// Ask the Raylet to pull a set of objects to the local node. + /// + /// This request is asynchronous. + /// + /// \param object_ids The IDs of the objects to pull. + /// \param owner_addresses The owner addresses of the objects. + /// \return ray::Status. + virtual ray::Status AsyncGetObjects( + const std::vector &object_ids, + const std::vector &owner_addresses) = 0; + + /// Wait for the given objects until timeout expires or num_return objects are + /// found. + /// + /// \param object_ids The objects to wait for. + /// \param owner_addresses The addresses of the workers that own the objects. + /// \param num_returns The number of objects to wait for. + /// \param timeout_milliseconds Duration, in milliseconds, to wait before returning. + /// \param result A pair with the first element containing the object ids that were + /// found, and the second element the objects that were not found. + /// \return ray::StatusOr containing error status or the set of object ids that were + /// found. + virtual ray::StatusOr> Wait( + const std::vector &object_ids, + const std::vector &owner_addresses, + int num_returns, + int64_t timeout_milliseconds) = 0; + + /// Tell the Raylet to cancel the get request from this worker. + /// + /// \return ray::Status. + virtual ray::Status CancelGetRequest() = 0; + + /// Notify the raylet that this client is blocked. This is only used for direct task + /// calls. Note that ordering of this with respect to Unblock calls is important. + /// + /// \return ray::Status. + virtual ray::Status NotifyDirectCallTaskBlocked() = 0; + + /// Notify the raylet that this client is unblocked. This is only used for direct task + /// calls. Note that ordering of this with respect to Block calls is important. + /// + /// \return ray::Status. + virtual ray::Status NotifyDirectCallTaskUnblocked() = 0; + + /// Wait for the given objects asynchronously. + /// + /// The core worker will be notified over gRPC when the wait completes. + /// + /// \param references The objects to wait for. + /// \param tag Value that will be sent to the core worker via gRPC on completion. + /// \return ray::Status. + virtual ray::Status WaitForActorCallArgs( + const std::vector &references, int64_t tag) = 0; + + /// Push an error to the relevant driver. + /// + /// \param job_id The ID of the job_id that the error is for. + /// \param type The type of the error. + /// \param error_message The error message. + /// \param timestamp The timestamp of the error. + /// \return ray::Status. + virtual ray::Status PushError(const ray::JobID &job_id, + const std::string &type, + const std::string &error_message, + double timestamp) = 0; + + /// Free a list of objects from object stores. + /// + /// \param object_ids A list of ObjectsIDs to be deleted. + /// \param local_only Whether keep this request with local object store + /// or send it to all the object stores. + /// \return ray::Status. + virtual ray::Status FreeObjects(const std::vector &object_ids, + bool local_only) = 0; + + /// Subscribe this worker to a notification when the provided object is ready in the + /// local object store. + /// + /// The worker will be notified over gRPC when the object is ready. + /// + /// \param object_id The ID of the object to subscribe to. + /// \param owner_address The address of the owner of the object. + virtual void SubscribePlasmaReady(const ObjectID &object_id, + const rpc::Address &owner_address) = 0; +}; + +} // namespace ipc +} // namespace ray From 2cdb27e49d3c4935fe90236f9affa15b5696a42f Mon Sep 17 00:00:00 2001 From: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Date: Fri, 15 Aug 2025 11:07:19 -0700 Subject: [PATCH 132/634] [Serve] Update route prefix assignment for ReplicaBase.reconfigure() (#55657) Update assigning value that was slipped from #55407 --------- Signed-off-by: doyoung --- python/ray/serve/_private/replica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/serve/_private/replica.py b/python/ray/serve/_private/replica.py index 298c1bbfe804..5ce22606f4b9 100644 --- a/python/ray/serve/_private/replica.py +++ b/python/ray/serve/_private/replica.py @@ -787,7 +787,7 @@ async def reconfigure( servable_object=self._user_callable_wrapper.user_callable ) - self._route_prefix = route_prefix + self._route_prefix = self._version.route_prefix except Exception: raise RuntimeError(traceback.format_exc()) from None From 44e0aea628f1f221345aeddaafce3b82d91cf9fa Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Fri, 15 Aug 2025 20:44:34 +0200 Subject: [PATCH 133/634] [RLlib] Fix `ImportError` in Atari examples. (#54967) ## Why are these changes needed? Running Atari with RLlib results in error described in #53836 . This related to the version of `gymnasium` installed when calling `ray[rllib]` and then later installing `gymnasium[atari,accept-rom-license]`. Using `gymnasium=1.1.1` resolves this error. ## Related issue number Closes #53836 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: simonsays1980 --- python/requirements.txt | 2 +- python/requirements_compiled.txt | 2 +- python/requirements_compiled_ray_py311_cpu.txt | 6 +++--- python/requirements_compiled_ray_py311_cu121.txt | 6 +++--- python/requirements_compiled_ray_py311_cu128.txt | 6 +++--- python/requirements_compiled_ray_test_py311_cpu.txt | 6 +++--- python/requirements_compiled_ray_test_py311_cu121.txt | 6 +++--- python/requirements_compiled_ray_test_py311_cu128.txt | 6 +++--- python/requirements_compiled_rayllm_py311_cpu.txt | 6 +++--- python/requirements_compiled_rayllm_py311_cu121.txt | 6 +++--- python/requirements_compiled_rayllm_py311_cu128.txt | 6 +++--- python/requirements_compiled_rayllm_test_py311_cpu.txt | 6 +++--- python/requirements_compiled_rayllm_test_py311_cu121.txt | 6 +++--- python/requirements_compiled_rayllm_test_py311_cu128.txt | 6 +++--- python/setup.py | 2 +- 15 files changed, 39 insertions(+), 39 deletions(-) diff --git a/python/requirements.txt b/python/requirements.txt index 2fbf538897d0..ac13f72f2d67 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -41,7 +41,7 @@ opentelemetry-api opentelemetry-exporter-prometheus opentelemetry-proto fastapi -gymnasium==1.0.0 +gymnasium==1.1.1 virtualenv!=20.21.1,>=20.0.24 opencensus aiohttp_cors diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt index 343a3f4a396b..da646a8d36f4 100644 --- a/python/requirements_compiled.txt +++ b/python/requirements_compiled.txt @@ -711,7 +711,7 @@ gsutil==5.27 # via -r python/requirements/docker/ray-docker-requirements.txt gunicorn==20.1.0 # via mlflow -gymnasium==1.0.0 +gymnasium==1.1.1 # via # -r python/requirements.txt # minigrid diff --git a/python/requirements_compiled_ray_py311_cpu.txt b/python/requirements_compiled_ray_py311_cpu.txt index 3f38abd46278..5148759ee51f 100644 --- a/python/requirements_compiled_ray_py311_cpu.txt +++ b/python/requirements_compiled_ray_py311_cpu.txt @@ -732,9 +732,9 @@ grpcio==1.66.2 \ # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # -r python/requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # -r python/requirements.txt diff --git a/python/requirements_compiled_ray_py311_cu121.txt b/python/requirements_compiled_ray_py311_cu121.txt index 0b2fee2cae5c..7d1d3712bf1d 100644 --- a/python/requirements_compiled_ray_py311_cu121.txt +++ b/python/requirements_compiled_ray_py311_cu121.txt @@ -732,9 +732,9 @@ grpcio==1.66.2 \ # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # -r python/requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # -r python/requirements.txt diff --git a/python/requirements_compiled_ray_py311_cu128.txt b/python/requirements_compiled_ray_py311_cu128.txt index 206ecd7e66d8..aa408bf03239 100644 --- a/python/requirements_compiled_ray_py311_cu128.txt +++ b/python/requirements_compiled_ray_py311_cu128.txt @@ -732,9 +732,9 @@ grpcio==1.66.2 \ # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # -r python/requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # -r python/requirements.txt diff --git a/python/requirements_compiled_ray_test_py311_cpu.txt b/python/requirements_compiled_ray_test_py311_cpu.txt index 81ea73f51d7f..490288181590 100644 --- a/python/requirements_compiled_ray_test_py311_cpu.txt +++ b/python/requirements_compiled_ray_test_py311_cpu.txt @@ -1100,9 +1100,9 @@ grpcio-tools==1.62.3 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt diff --git a/python/requirements_compiled_ray_test_py311_cu121.txt b/python/requirements_compiled_ray_test_py311_cu121.txt index 87cdc9200725..56fc67650c57 100644 --- a/python/requirements_compiled_ray_test_py311_cu121.txt +++ b/python/requirements_compiled_ray_test_py311_cu121.txt @@ -1100,9 +1100,9 @@ grpcio-tools==1.62.3 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt diff --git a/python/requirements_compiled_ray_test_py311_cu128.txt b/python/requirements_compiled_ray_test_py311_cu128.txt index 44f0ef21f41c..c103a02fea12 100644 --- a/python/requirements_compiled_ray_test_py311_cu128.txt +++ b/python/requirements_compiled_ray_test_py311_cu128.txt @@ -1100,9 +1100,9 @@ grpcio-tools==1.62.3 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt diff --git a/python/requirements_compiled_rayllm_py311_cpu.txt b/python/requirements_compiled_rayllm_py311_cpu.txt index 3d4262a623c4..0b8df56dcfc6 100644 --- a/python/requirements_compiled_rayllm_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_py311_cpu.txt @@ -953,9 +953,9 @@ grpcio==1.66.2 \ # via # -c python/requirements_compiled_rayllm_test_py311_cpu.txt # -r python/requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c python/requirements_compiled_rayllm_test_py311_cpu.txt # -r python/requirements.txt diff --git a/python/requirements_compiled_rayllm_py311_cu121.txt b/python/requirements_compiled_rayllm_py311_cu121.txt index 3e6dad764e0e..0e2ca1879291 100644 --- a/python/requirements_compiled_rayllm_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_py311_cu121.txt @@ -953,9 +953,9 @@ grpcio==1.66.2 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu121.txt # -r python/requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c python/requirements_compiled_rayllm_test_py311_cu121.txt # -r python/requirements.txt diff --git a/python/requirements_compiled_rayllm_py311_cu128.txt b/python/requirements_compiled_rayllm_py311_cu128.txt index 05be84969dbb..236ed1153440 100644 --- a/python/requirements_compiled_rayllm_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_py311_cu128.txt @@ -953,9 +953,9 @@ grpcio==1.66.2 \ # via # -c python/requirements_compiled_rayllm_test_py311_cu128.txt # -r python/requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c python/requirements_compiled_rayllm_test_py311_cu128.txt # -r python/requirements.txt diff --git a/python/requirements_compiled_rayllm_test_py311_cpu.txt b/python/requirements_compiled_rayllm_test_py311_cpu.txt index f3933c7fedad..5c2a97b2d58f 100644 --- a/python/requirements_compiled_rayllm_test_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_test_py311_cpu.txt @@ -1310,9 +1310,9 @@ grpcio-tools==1.62.3 \ # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # -r python/requirements/cloud-requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c python/requirements_compiled_ray_test_py311_cpu.txt # -r python/requirements.txt diff --git a/python/requirements_compiled_rayllm_test_py311_cu121.txt b/python/requirements_compiled_rayllm_test_py311_cu121.txt index 164a9b50f775..d80ae177859f 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu121.txt @@ -1310,9 +1310,9 @@ grpcio-tools==1.62.3 \ # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # -r python/requirements/cloud-requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c python/requirements_compiled_ray_test_py311_cu121.txt # -r python/requirements.txt diff --git a/python/requirements_compiled_rayllm_test_py311_cu128.txt b/python/requirements_compiled_rayllm_test_py311_cu128.txt index e94cd04041cb..a589ed8ebe0a 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu128.txt @@ -1309,9 +1309,9 @@ grpcio-tools==1.62.3 \ # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # -r python/requirements/cloud-requirements.txt -gymnasium==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c python/requirements_compiled_ray_test_py311_cu128.txt # -r python/requirements.txt diff --git a/python/setup.py b/python/setup.py index c0eb418d688c..82a08fda81f3 100644 --- a/python/setup.py +++ b/python/setup.py @@ -315,7 +315,7 @@ def get_packages(self): setup_spec.extras["rllib"] = setup_spec.extras["tune"] + [ "dm_tree", - "gymnasium==1.0.0", + "gymnasium==1.1.1", "lz4", "ormsgpack==1.7.0", "pyyaml", From b819ed4add79492dcdc58d7df277bbd1d438f11b Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Fri, 15 Aug 2025 12:07:22 -0700 Subject: [PATCH 134/634] [core] Fix objects_valid with except from BaseException (#55602) We would encounter a ray check failure on `objects_valid` whenever we get a function throws an exception that extends from `BaseException` instead of `Exception`. Fixing that by just excepting `BaseException` instead of `Exception` when we are vulnerable to exceptions thrown from user Python code. We still have to special case `SystemExit` and `KeyboardInterrupt` because we can consider those as critical errors ourselves and treat them as worker shutdown or task cancellation signals respectively. Closes https://github.com/ray-project/ray/issues/43411 Signed-off-by: dayshah --- python/ray/_raylet.pyx | 11 +++-- python/ray/tests/test_basic.py | 10 +++++ python/ray/tests/test_failure.py | 44 ++++++++++++++++---- python/ray/tests/test_streaming_generator.py | 42 +++++++++++++++++++ 4 files changed, 96 insertions(+), 11 deletions(-) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index d375820f7449..a88e81009fa1 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -1029,7 +1029,7 @@ def serialize_retry_exception_allowlist(retry_exception_allowlist, function_desc cdef c_bool determine_if_retryable( c_bool should_retry_exceptions, - Exception e, + e: BaseException, const c_string serialized_retry_exception_allowlist, FunctionDescriptor function_descriptor, ): @@ -2012,7 +2012,9 @@ cdef void execute_task( task_exception = False except AsyncioActorExit as e: exit_current_actor_if_asyncio() - except Exception as e: + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as e: is_retryable_error[0] = determine_if_retryable( should_retry_exceptions, e, @@ -2121,8 +2123,9 @@ cdef void execute_task( None, # ref_generator_id c_tensor_transport ) - - except Exception as e: + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as e: num_errors_stored = store_task_errors( worker, e, task_exception, actor, actor_id, function_name, task_type, title, caller_address, returns, application_error, c_tensor_transport) diff --git a/python/ray/tests/test_basic.py b/python/ray/tests/test_basic.py index 68d1773adff0..a43312247687 100644 --- a/python/ray/tests/test_basic.py +++ b/python/ray/tests/test_basic.py @@ -1192,6 +1192,16 @@ def f(): assert False +def test_base_exception_raised(ray_start_shared_local_modes): + @ray.remote + def f(): + raise BaseException("rip") + return 1 + + with pytest.raises(BaseException): + ray.get(f.remote()) + + def test_import_ray_does_not_import_grpc(): # First unload grpc and ray if "grpc" in sys.modules: diff --git a/python/ray/tests/test_failure.py b/python/ray/tests/test_failure.py index d2d2f650138a..ef551d68ea8d 100644 --- a/python/ray/tests/test_failure.py +++ b/python/ray/tests/test_failure.py @@ -380,25 +380,55 @@ def foo(): assert isinstance(ex, RayTaskError) -def test_baseexception_task(ray_start_regular): +def test_baseexception_task(ray_start_regular_shared): + class MyBaseException(BaseException): + pass + @ray.remote def task(): - raise BaseException("abc") + raise MyBaseException("abc") - with pytest.raises(ray.exceptions.WorkerCrashedError): + with pytest.raises(MyBaseException): ray.get(task.remote()) -def test_baseexception_actor(ray_start_regular): +def test_baseexception_actor_task(ray_start_regular_shared): + class MyBaseException(BaseException): + pass + @ray.remote class Actor: def f(self): - raise BaseException("abc") + raise MyBaseException("abc") - with pytest.raises(ActorDiedError): - a = Actor.remote() + async def async_f(self): + raise MyBaseException("abc") + + a = Actor.remote() + with pytest.raises(MyBaseException): ray.get(a.f.remote()) + a = Actor.remote() + with pytest.raises(MyBaseException): + ray.get(a.async_f.remote()) + + +def test_baseexception_actor_creation(ray_start_regular_shared): + class MyBaseException(BaseException): + pass + + @ray.remote + class Actor: + def __init__(self): + raise MyBaseException("abc") + + a = Actor.remote() + try: + ray.get(a.__ray_ready__.remote()) + raise Exception("abc") + except ActorDiedError as e: + assert "MyBaseException" in str(e) + @pytest.mark.skip("This test does not work yet.") @pytest.mark.parametrize("ray_start_object_store_memory", [10**6], indirect=True) diff --git a/python/ray/tests/test_streaming_generator.py b/python/ray/tests/test_streaming_generator.py index 2e166e190ba5..2e0136d91161 100644 --- a/python/ray/tests/test_streaming_generator.py +++ b/python/ray/tests/test_streaming_generator.py @@ -576,6 +576,48 @@ async def async_f(self): ray.get(next(g)) +def test_baseexception_streaming_generator(shutdown_only): + ray.init() + + class MyBaseException(BaseException): + pass + + @ray.remote + def raise_at_beginning(): + raise MyBaseException("rip") + yield 1 + + raise_at_beginning_ref = raise_at_beginning.remote() + with pytest.raises(MyBaseException): + ray.get(next(raise_at_beginning_ref)) + + @ray.remote + def raise_at_middle(): + for i in range(1, 10): + if i == 5: + raise MyBaseException("rip") + yield i + + raise_at_middle_ref = raise_at_middle.remote() + for i in range(1, 5): + assert i == ray.get(next(raise_at_middle_ref)) + with pytest.raises(MyBaseException): + ray.get(next(raise_at_middle_ref)) + + @ray.remote(_generator_backpressure_num_objects=1) + def raise_after_backpressure(): + for i in range(1, 10): + if i == 5: + raise MyBaseException("rip") + yield i + + raise_after_backpressure_ref = raise_after_backpressure.remote() + for i in range(1, 5): + assert i == ray.get(next(raise_after_backpressure_ref)) + with pytest.raises(MyBaseException): + ray.get(next(raise_after_backpressure_ref)) + + if __name__ == "__main__": sys.exit(pytest.main(["-sv", __file__])) From 10af9d897bbdaae4202580ba14dea1d6efcb525b Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Fri, 15 Aug 2025 12:37:21 -0700 Subject: [PATCH 135/634] [ci] raydepsets: generating llm lock files (4/4) (#55500) - generating llm lock files with raydepsets --------- Signed-off-by: elliot-barn --- ci/compile_llm_requirements.sh | 70 ++----------------- ci/raydepsets/cli.py | 19 ++--- ci/raydepsets/ray.depsets.yaml | 27 ------- ci/raydepsets/rayllm.depsets.yaml | 68 ++++++++++++++++++ ci/raydepsets/tests/test_cli.py | 27 +++++-- ci/test_compile_llm_requirements.sh | 5 -- .../requirements_compiled_ray_py311_cpu.txt | 2 +- .../requirements_compiled_ray_py311_cu121.txt | 2 +- .../requirements_compiled_ray_py311_cu128.txt | 2 +- ...quirements_compiled_ray_test_py311_cpu.txt | 2 +- ...irements_compiled_ray_test_py311_cu121.txt | 2 +- ...irements_compiled_ray_test_py311_cu128.txt | 2 +- ...requirements_compiled_rayllm_py311_cpu.txt | 2 +- ...quirements_compiled_rayllm_py311_cu121.txt | 2 +- ...quirements_compiled_rayllm_py311_cu128.txt | 2 +- ...rements_compiled_rayllm_test_py311_cpu.txt | 2 +- ...ments_compiled_rayllm_test_py311_cu121.txt | 2 +- ...ments_compiled_rayllm_test_py311_cu128.txt | 2 +- 18 files changed, 114 insertions(+), 126 deletions(-) delete mode 100644 ci/raydepsets/ray.depsets.yaml create mode 100644 ci/raydepsets/rayllm.depsets.yaml diff --git a/ci/compile_llm_requirements.sh b/ci/compile_llm_requirements.sh index 5932d1e005b9..f3ae705fe228 100755 --- a/ci/compile_llm_requirements.sh +++ b/ci/compile_llm_requirements.sh @@ -9,71 +9,13 @@ if [[ "${PYTHON_CODE}" != "py311" ]]; then exit 1 fi -for CUDA_CODE in cpu cu121 cu128; do - PYTHON_CUDA_CODE="${PYTHON_CODE}_${CUDA_CODE}" +mkdir -p /tmp/ray-deps - echo "--- Compile dependencies for ${PYTHON_CODE}_${CUDA_CODE}" +# Remove the GPU constraints +cp python/requirements_compiled.txt /tmp/ray-deps/requirements_compiled.txt +sed -i '/^--extra-index-url /d' /tmp/ray-deps/requirements_compiled.txt +sed -i '/^--find-links /d' /tmp/ray-deps/requirements_compiled.txt - UV_PIP_COMPILE=( - uv pip compile --generate-hashes --strip-extras - --unsafe-package ray - # setuptools should not be pinned. - --unsafe-package setuptools - --index-url "https://pypi.org/simple" - --extra-index-url "https://download.pytorch.org/whl/${CUDA_CODE}" - --index-strategy unsafe-best-match - --no-strip-markers - --emit-index-url - --emit-find-links - ) - - mkdir -p /tmp/ray-deps - - # Remove the GPU constraints - cp python/requirements_compiled.txt /tmp/ray-deps/requirements_compiled.txt - sed -i '/^--extra-index-url /d' /tmp/ray-deps/requirements_compiled.txt - sed -i '/^--find-links /d' /tmp/ray-deps/requirements_compiled.txt - - # First, extract base test dependencies from the current compiled mono repo one. - # This also expands to the indirect dependencies for this Python version & platform. - # - # Needs to use the exact torch version. - echo "--- Compile ray base test dependencies" - "${UV_PIP_COMPILE[@]}" \ - -c "/tmp/ray-deps/requirements_compiled.txt" \ - "python/requirements.txt" \ - "python/requirements/cloud-requirements.txt" \ - "python/requirements/base-test-requirements.txt" \ - -o "python/requirements_compiled_ray_test_${PYTHON_CUDA_CODE}.txt" - - # Second, expand it into LLM test dependencies - echo "--- Compile LLM test dependencies" - "${UV_PIP_COMPILE[@]}" \ - -c "python/requirements_compiled_ray_test_${PYTHON_CUDA_CODE}.txt" \ - "python/requirements.txt" \ - "python/requirements/cloud-requirements.txt" \ - "python/requirements/base-test-requirements.txt" \ - "python/requirements/llm/llm-requirements.txt" \ - "python/requirements/llm/llm-test-requirements.txt" \ - -o "python/requirements_compiled_rayllm_test_${PYTHON_CUDA_CODE}.txt" - - # Third, extract the ray base dependencies from ray base test dependencies. - # TODO(aslonnie): This should be used for installing ray in the container images. - echo "--- Compile ray base dependencies" - "${UV_PIP_COMPILE[@]}" \ - -c "python/requirements_compiled_ray_test_${PYTHON_CUDA_CODE}.txt" \ - "python/requirements.txt" \ - -o "python/requirements_compiled_ray_${PYTHON_CUDA_CODE}.txt" - - # Finally, extract the LLM dependencies from the LLM test dependencies, - # which is also an expansion of the ray base dependencies. - # TODO(aslonnie): This should be used for installing ray[llm] in the container images. - echo "--- Compile LLM dependencies" - "${UV_PIP_COMPILE[@]}" \ - -c "python/requirements_compiled_rayllm_test_${PYTHON_CUDA_CODE}.txt" \ - "python/requirements.txt" \ - "python/requirements/llm/llm-requirements.txt" \ - -o "python/requirements_compiled_rayllm_${PYTHON_CUDA_CODE}.txt" -done +bazel run //ci/raydepsets:raydepsets -- build ci/raydepsets/rayllm.depsets.yaml echo "--- Done" diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index 79872c2b3387..867e33c21060 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -12,14 +12,14 @@ DEFAULT_UV_FLAGS = """ --generate-hashes --strip-extras - --no-strip-markers - --emit-index-url - --emit-find-links --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match + --no-strip-markers + --emit-index-url + --emit-find-links --quiet """.split() @@ -166,15 +166,15 @@ def compile( if override_flags: args = _override_uv_flags(override_flags, args) if append_flags: - args = _append_uv_flags(append_flags, args) + args.extend(_flatten_flags(append_flags)) if constraints: for constraint in constraints: - args.extend(["-c", self.get_path(constraint)]) + args.extend(["-c", constraint]) if requirements: for requirement in requirements: - args.extend([self.get_path(requirement)]) + args.extend([requirement]) if output: - args.extend(["-o", self.get_path(output)]) + args.extend(["-o", output]) self.exec_uv_cmd("compile", args) def subset( @@ -271,11 +271,6 @@ def _override_uv_flags(flags: List[str], args: List[str]) -> List[str]: return new_args + _flatten_flags(flags) -def _append_uv_flags(flags: List[str], args: List[str]) -> List[str]: - args.extend(flags) - return args - - def _uv_binary(): r = runfiles.Create() system = platform.system() diff --git a/ci/raydepsets/ray.depsets.yaml b/ci/raydepsets/ray.depsets.yaml deleted file mode 100644 index a6b5c0a33a12..000000000000 --- a/ci/raydepsets/ray.depsets.yaml +++ /dev/null @@ -1,27 +0,0 @@ -depsets: - - name: subset_general_depset - operation: subset - source_depset: general_depset - requirements: - - python/requirements/cloud-requirements.txt - output: ci/raydepsets/test/requirements_compiled_subset_general_py311_cpu.txt - - name: ray_base_test_depset - requirements: - - python/requirements.txt - - python/requirements/cloud-requirements.txt - - python/requirements/base-test-requirements.txt - constraints: - - python/requirements_compiled_ray_test_py311_cpu.txt - output: ci/raydepsets/test/requirements_compiled_ray_test_py311_cpu.txt - operation: compile - - name: general_depset - operation: compile - requirements: - - python/requirements.txt - output: python/test/requirements_compiled_general_py311_cpu.txt - - name: subset_general_depset - operation: subset - source_depset: general_depset - requirements: - - ci/raydepsets/cloud-requirements.txt - output: python/test/requirements_compiled_subset_general_py311_cpu.txt diff --git a/ci/raydepsets/rayllm.depsets.yaml b/ci/raydepsets/rayllm.depsets.yaml new file mode 100644 index 000000000000..e2a7a030b5e1 --- /dev/null +++ b/ci/raydepsets/rayllm.depsets.yaml @@ -0,0 +1,68 @@ +build_arg_sets: + cpu: + PYTHON_VERSION: py311 + CUDA_CODE: cpu + cu121: + PYTHON_VERSION: py311 + CUDA_CODE: cu121 + cu128: + PYTHON_VERSION: py311 + CUDA_CODE: cu128 + + +.common_settings: &common_settings + override_flags: + - --extra-index-url https://download.pytorch.org/whl/${CUDA_CODE} + append_flags: + - --python-version=3.11 + build_arg_sets: + - cpu + - cu121 + - cu128 + +depsets: +# First, extract base test dependencies from the current compiled mono repo one. +# This also expands to the indirect dependencies for this Python version & platform. + - name: ray_base_test_depset_${PYTHON_VERSION}_${CUDA_CODE} + <<: *common_settings + requirements: + - python/requirements.txt + - python/requirements/cloud-requirements.txt + - python/requirements/base-test-requirements.txt + constraints: + - /tmp/ray-deps/requirements_compiled.txt + output: python/requirements_compiled_ray_test_${PYTHON_VERSION}_${CUDA_CODE}.txt + operation: compile + +# Second, expand it into LLM test dependencies. + - name: compiled_ray_llm_test_depset_${PYTHON_VERSION}_${CUDA_CODE} + <<: *common_settings + operation: expand + requirements: + - python/requirements.txt + - python/requirements/cloud-requirements.txt + - python/requirements/base-test-requirements.txt + - python/requirements/llm/llm-requirements.txt + - python/requirements/llm/llm-test-requirements.txt + constraints: + - python/requirements_compiled_ray_test_${PYTHON_VERSION}_${CUDA_CODE}.txt + output: python/requirements_compiled_rayllm_test_${PYTHON_VERSION}_${CUDA_CODE}.txt + +# Third, subset the base test dependencies into Ray dependencies. + - name: compiled_ray_depset_${PYTHON_VERSION}_${CUDA_CODE} + <<: *common_settings + operation: subset + source_depset: ray_base_test_depset_${PYTHON_VERSION}_${CUDA_CODE} + requirements: + - python/requirements.txt + output: python/requirements_compiled_ray_${PYTHON_VERSION}_${CUDA_CODE}.txt + +# Fourth, subset the LLM test dependencies into RayLLM dependencies. + - name: compiled_ray_llm_depset_${PYTHON_VERSION}_${CUDA_CODE} + <<: *common_settings + operation: subset + source_depset: compiled_ray_llm_test_depset_${PYTHON_VERSION}_${CUDA_CODE} + requirements: + - python/requirements.txt + - python/requirements/llm/llm-requirements.txt + output: python/requirements_compiled_rayllm_${PYTHON_VERSION}_${CUDA_CODE}.txt diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 6401b8d766be..421a12b3c132 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -14,7 +14,6 @@ from ci.raydepsets.cli import ( DEFAULT_UV_FLAGS, DependencySetManager, - _append_uv_flags, _flatten_flags, _get_depset, _override_uv_flags, @@ -151,6 +150,27 @@ def test_compile_update_package(self): output_text_valid = output_file_valid.read_text() assert output_text == output_text_valid + def test_compile_with_append_and_override_flags(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = _create_test_manager(tmpdir) + manager.compile( + constraints=["requirement_constraints_test.txt"], + requirements=["requirements_test.txt"], + append_flags=["--no-annotate", "--python-version 3.10"], + override_flags=["--extra-index-url https://dummyurl.com"], + name="ray_base_test_depset", + output="requirements_compiled.txt", + ) + output_file = Path(tmpdir) / "requirements_compiled.txt" + output_text = output_file.read_text() + assert "--python-version 3.10" in output_text + assert "--extra-index-url https://dummyurl.com" in output_text + assert ( + "--extra-index-url https://download.pytorch.org/whl/cu128" + not in output_text + ) + def test_compile_by_depset_name(self): with tempfile.TemporaryDirectory() as tmpdir: copy_data_to_tmpdir(tmpdir) @@ -276,11 +296,6 @@ def test_get_path(self): == f"{tmpdir}/requirements_test.txt" ) - def test_append_uv_flags(self): - assert _append_uv_flags( - ["--no-annotate", "--no-header"], DEFAULT_UV_FLAGS.copy() - ) == DEFAULT_UV_FLAGS.copy() + ["--no-annotate", "--no-header"] - def test_override_uv_flag_single_flag(self): expected_flags = DEFAULT_UV_FLAGS.copy() expected_flags.remove("--extra-index-url") diff --git a/ci/test_compile_llm_requirements.sh b/ci/test_compile_llm_requirements.sh index 6351add67743..80a4a8d0a044 100755 --- a/ci/test_compile_llm_requirements.sh +++ b/ci/test_compile_llm_requirements.sh @@ -2,11 +2,6 @@ set -e -# Install uv and set up Python -pip install uv -uv python install 3.11 -uv python pin 3.11 - # Create a temporary directory for backup files and setup cleanup trap TEMP_DIR=$(mktemp -d) cleanup() { diff --git a/python/requirements_compiled_ray_py311_cpu.txt b/python/requirements_compiled_ray_py311_cpu.txt index 5148759ee51f..d8911ad845de 100644 --- a/python/requirements_compiled_ray_py311_cpu.txt +++ b/python/requirements_compiled_ray_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/requirements_compiled_ray_py311_cu121.txt b/python/requirements_compiled_ray_py311_cu121.txt index 7d1d3712bf1d..7bf7aa57a1a7 100644 --- a/python/requirements_compiled_ray_py311_cu121.txt +++ b/python/requirements_compiled_ray_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/requirements_compiled_ray_py311_cu128.txt b/python/requirements_compiled_ray_py311_cu128.txt index aa408bf03239..c126614250e6 100644 --- a/python/requirements_compiled_ray_py311_cu128.txt +++ b/python/requirements_compiled_ray_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/python/requirements_compiled_ray_test_py311_cpu.txt b/python/requirements_compiled_ray_test_py311_cpu.txt index 490288181590..d7a2552b6dc4 100644 --- a/python/requirements_compiled_ray_test_py311_cpu.txt +++ b/python/requirements_compiled_ray_test_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/requirements_compiled_ray_test_py311_cu121.txt b/python/requirements_compiled_ray_test_py311_cu121.txt index 56fc67650c57..ec4ae33bcdb5 100644 --- a/python/requirements_compiled_ray_test_py311_cu121.txt +++ b/python/requirements_compiled_ray_test_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/requirements_compiled_ray_test_py311_cu128.txt b/python/requirements_compiled_ray_test_py311_cu128.txt index c103a02fea12..f27f420acd03 100644 --- a/python/requirements_compiled_ray_test_py311_cu128.txt +++ b/python/requirements_compiled_ray_test_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/python/requirements_compiled_rayllm_py311_cpu.txt b/python/requirements_compiled_rayllm_py311_cpu.txt index 0b8df56dcfc6..21c5a9872eff 100644 --- a/python/requirements_compiled_rayllm_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_rayllm_test_py311_cpu.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 -c python/requirements_compiled_rayllm_test_py311_cpu.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/requirements_compiled_rayllm_py311_cu121.txt b/python/requirements_compiled_rayllm_py311_cu121.txt index 0e2ca1879291..530878b8b211 100644 --- a/python/requirements_compiled_rayllm_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_rayllm_test_py311_cu121.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 -c python/requirements_compiled_rayllm_test_py311_cu121.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/requirements_compiled_rayllm_py311_cu128.txt b/python/requirements_compiled_rayllm_py311_cu128.txt index 236ed1153440..e7ec34152746 100644 --- a/python/requirements_compiled_rayllm_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_rayllm_test_py311_cu128.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 -c python/requirements_compiled_rayllm_test_py311_cu128.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/python/requirements_compiled_rayllm_test_py311_cpu.txt b/python/requirements_compiled_rayllm_test_py311_cpu.txt index 5c2a97b2d58f..0fd5d38ace03 100644 --- a/python/requirements_compiled_rayllm_test_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_test_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/requirements_compiled_rayllm_test_py311_cu121.txt b/python/requirements_compiled_rayllm_test_py311_cu121.txt index d80ae177859f..c89112408569 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/requirements_compiled_rayllm_test_py311_cu128.txt b/python/requirements_compiled_rayllm_test_py311_cu128.txt index a589ed8ebe0a..9bcf8ad47312 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 From d6dce722f0ff25a55a3b3a4749bd32821bcccbec Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 15 Aug 2025 14:54:28 -0500 Subject: [PATCH 136/634] [serve] Fix easy `ray._private` dependency (#55659) Signed-off-by: Edward Oakes --- python/ray/runtime_context.py | 1 + python/ray/serve/tests/test_persistence.py | 7 ++++--- python/ray/serve/tests/test_serve_ha.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/python/ray/runtime_context.py b/python/ray/runtime_context.py index 567b7ac2c7e7..5257c9688a80 100644 --- a/python/ray/runtime_context.py +++ b/python/ray/runtime_context.py @@ -501,6 +501,7 @@ def current_actor(self): @property def gcs_address(self): """Get the GCS address of the ray cluster. + Returns: The GCS address of the cluster. """ diff --git a/python/ray/serve/tests/test_persistence.py b/python/ray/serve/tests/test_persistence.py index f2a79348791b..0c924ccb817d 100644 --- a/python/ray/serve/tests/test_persistence.py +++ b/python/ray/serve/tests/test_persistence.py @@ -4,7 +4,8 @@ def test_new_driver(serve_instance): - script = """ + run_string_as_driver( + """ import ray ray.init(address="{}", namespace="default_test_namespace") @@ -16,9 +17,9 @@ def driver(): serve.run(driver.bind(), name="app") """.format( - ray._private.worker._global_node.address + ray.get_runtime_context().gcs_address, + ) ) - run_string_as_driver(script) handle = serve.get_app_handle("app") assert handle.remote().result() == "OK!" diff --git a/python/ray/serve/tests/test_serve_ha.py b/python/ray/serve/tests/test_serve_ha.py index a15cff0a3a47..6af3593b50a4 100644 --- a/python/ray/serve/tests/test_serve_ha.py +++ b/python/ray/serve/tests/test_serve_ha.py @@ -109,7 +109,7 @@ def check_for_head_node_come_back_up(): import ray import requests from ray.serve.schema import ServeInstanceDetails -from ray._private.resource_and_label_spec import HEAD_NODE_RESOURCE_NAME +from ray._common.constants import HEAD_NODE_RESOURCE_NAME ray.init(address="auto") head_node_id = ray.get_runtime_context().get_node_id() serve_details = ServeInstanceDetails( From fc967a55018cf85d7f73381985273f429d14cb81 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 15 Aug 2025 13:30:44 -0700 Subject: [PATCH 137/634] [Core] Simplify get_event_aggregator_grpc_stub to not depend on webui_url (#55640) Signed-off-by: Jiajun Yao --- .../aggregator/tests/test_aggregator_agent.py | 37 +++++++++---------- python/ray/tests/test_metrics_agent.py | 2 +- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py index f7f1735110ba..2b974f87618c 100644 --- a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py @@ -14,10 +14,8 @@ from ray._raylet import GcsClient import ray.dashboard.consts as dashboard_consts from ray._private.test_utils import ( - wait_until_server_available, find_free_port, ) -from ray._common.network_utils import parse_address, build_address from ray.core.generated.events_event_aggregator_service_pb2_grpc import ( EventAggregatorServiceStub, @@ -69,23 +67,24 @@ def fake_timestamp(): ) -def get_event_aggregator_grpc_stub(webui_url, gcs_address, head_node_id): +def get_event_aggregator_grpc_stub(gcs_address, head_node_id): """ An helper function to get the gRPC stub for the event aggregator agent. Should only be used in tests. """ - ip, _ = parse_address(webui_url) - agent_address = build_address(ip, ray_constants.DEFAULT_DASHBOARD_AGENT_LISTEN_PORT) - assert wait_until_server_available(agent_address) gcs_address = gcs_address gcs_client = GcsClient(address=gcs_address) - agent_addr = gcs_client.internal_kv_get( - f"{dashboard_consts.DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX}{head_node_id}".encode(), - namespace=ray_constants.KV_NAMESPACE_DASHBOARD, - timeout=dashboard_consts.GCS_RPC_TIMEOUT_SECONDS, - ) - ip, http_port, grpc_port = json.loads(agent_addr) + + def get_addr(): + return gcs_client.internal_kv_get( + f"{dashboard_consts.DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX}{head_node_id}".encode(), + namespace=ray_constants.KV_NAMESPACE_DASHBOARD, + timeout=dashboard_consts.GCS_RPC_TIMEOUT_SECONDS, + ) + + wait_for_condition(lambda: get_addr() is not None) + ip, _, grpc_port = json.loads(get_addr()) options = ray_constants.GLOBAL_GRPC_OPTIONS channel = init_grpc_channel(f"{ip}:{grpc_port}", options=options) return EventAggregatorServiceStub(channel) @@ -120,7 +119,7 @@ def test_aggregator_agent_receive_publish_events_normally( ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( - cluster.webui_url, cluster.gcs_address, cluster.head_node.node_id + cluster.gcs_address, cluster.head_node.node_id ) httpserver.expect_request("/", method="POST").respond_with_data("", status=200) @@ -175,7 +174,7 @@ def test_aggregator_agent_receive_event_full( ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( - cluster.webui_url, cluster.gcs_address, cluster.head_node.node_id + cluster.gcs_address, cluster.head_node.node_id ) httpserver.expect_request("/", method="POST").respond_with_data("", status=200) @@ -222,7 +221,7 @@ def test_aggregator_agent_receive_multiple_events( ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( - cluster.webui_url, cluster.gcs_address, cluster.head_node.node_id + cluster.gcs_address, cluster.head_node.node_id ) httpserver.expect_request("/", method="POST").respond_with_data("", status=200) @@ -279,7 +278,7 @@ def test_aggregator_agent_receive_multiple_events_failures( ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( - cluster.webui_url, cluster.gcs_address, cluster.head_node.node_id + cluster.gcs_address, cluster.head_node.node_id ) httpserver.expect_request("/", method="POST").respond_with_data("", status=200) request = AddEventsRequest( @@ -326,7 +325,7 @@ def test_aggregator_agent_receive_empty_events( ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( - cluster.webui_url, cluster.gcs_address, cluster.head_node.node_id + cluster.gcs_address, cluster.head_node.node_id ) httpserver.expect_request("/", method="POST").respond_with_data("", status=200) request = AddEventsRequest( @@ -347,7 +346,7 @@ def test_aggregator_agent_profile_events_not_exposed( """Test that profile events are not sent when not in exposable event types.""" cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( - cluster.webui_url, cluster.gcs_address, cluster.head_node.node_id + cluster.gcs_address, cluster.head_node.node_id ) httpserver.expect_request("/", method="POST").respond_with_data("", status=200) @@ -401,7 +400,7 @@ def test_aggregator_agent_receive_profile_events( ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( - cluster.webui_url, cluster.gcs_address, cluster.head_node.node_id + cluster.gcs_address, cluster.head_node.node_id ) httpserver.expect_request("/", method="POST").respond_with_data("", status=200) diff --git a/python/ray/tests/test_metrics_agent.py b/python/ray/tests/test_metrics_agent.py index 4ce83269cc69..9f5fbc059431 100644 --- a/python/ray/tests/test_metrics_agent.py +++ b/python/ray/tests/test_metrics_agent.py @@ -489,7 +489,7 @@ def test_metrics_export_event_aggregator_agent( ): cluster = ray_start_cluster_head_with_env_vars stub = get_event_aggregator_grpc_stub( - cluster.webui_url, cluster.gcs_address, cluster.head_node.node_id + cluster.gcs_address, cluster.head_node.node_id ) httpserver.expect_request("/", method="POST").respond_with_data("", status=200) From 5fbeff61f889af7eddb7ca7b55ec6a6c8939bc2b Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 15 Aug 2025 16:48:53 -0500 Subject: [PATCH 138/634] [core] Unify test directory layout on `.../tests/` (#55652) We currently have multiple different patterns for test files: - `*_test.cc` in the same file as the implementation. - `test/*_test.cc` (with `BUILD.bazel` in the test dir or sometimes in the parent dir). - `tests/*_test.cc` (with `BUILD.bazel` in the test dir or sometimes in the parent dir). Unifying on: - `tests/*_test.cc` - `tests/BUILD.bazel` for test targets --------- Signed-off-by: Edward Oakes --- src/ray/common/BUILD.bazel | 13 +-- .../common/cgroup/{test => tests}/BUILD.bazel | 10 +- .../{test => tests}/cgroup_test_utils.cc | 4 +- .../{test => tests}/cgroup_test_utils.h | 0 .../{test => tests}/cgroup_v2_setup_test.cc | 6 +- .../cgroup_v2_utils_privileged_test.cc | 2 +- .../cgroup_v2_utils_unprivileged_test.cc | 2 +- .../{test => tests}/fake_cgroup_setup_test.cc | 2 +- .../cgroup2/{test => tests}/BUILD.bazel | 2 +- .../{test => tests}/cgroup_test_utils.cc | 2 +- .../{test => tests}/cgroup_test_utils.h | 0 .../sysfs_cgroup_driver_test.cc | 2 +- src/ray/common/{test => tests}/BUILD.bazel | 11 ++ .../common/{test => tests}/asio_defer_test.cc | 0 .../bundle_location_index_test.cc | 0 .../{test => tests}/event_stats_test.cc | 0 .../common/{test => tests}/grpc_util_test.cc | 0 src/ray/common/{test => tests}/id_test.cc | 0 .../{test => tests}/label_selector_test.cc | 0 .../{test => tests}/memory_monitor_test.cc | 0 .../common/{test => tests}/postable_test.cc | 0 .../common/{test => tests}/ray_config_test.cc | 0 .../common/{test => tests}/ray_syncer_test.cc | 0 .../resource_instance_set_test.cc | 0 .../{test => tests}/resource_request_test.cc | 0 .../{test => tests}/resource_set_test.cc | 0 .../{test => tests}/scheduling_ids_test.cc | 0 .../{ => tests}/source_location_test.cc | 2 +- .../common/{test => tests}/status_or_test.cc | 2 +- src/ray/common/{test => tests}/status_test.cc | 0 .../syncer_service_e2e_test.cc | 0 .../common/{test => tests}/task_spec_test.cc | 0 src/ray/common/{test => tests}/testing.h | 0 .../{test => tests}/BUILD.bazel | 0 .../concurrency_group_manager_test.cc | 0 .../{test => tests}/fiber_state_test.cc | 0 .../{test => tests}/scheduling_queue_test.cc | 0 .../{test => tests}/task_receiver_test.cc | 0 .../{test => tests}/thread_pool_test.cc | 0 .../{test => tests}/BUILD.bazel | 0 .../actor_task_submitter_test.cc | 0 .../dependency_resolver_test.cc | 0 .../direct_actor_transport_test.cc | 0 .../normal_task_submitter_test.cc | 0 .../out_of_order_actor_submit_queue_test.cc | 0 .../core_worker/{test => tests}/BUILD.bazel | 0 .../{test => tests}/actor_creator_test.cc | 0 .../{test => tests}/actor_manager_test.cc | 0 .../core_worker_resubmit_queue_test.cc | 0 .../{test => tests}/core_worker_test.cc | 0 .../{test => tests}/generator_waiter_test.cc | 0 .../{test => tests}/lease_policy_test.cc | 0 .../{test => tests}/memory_store_test.cc | 0 .../mutable_object_provider_test.cc | 0 .../object_recovery_manager_test.cc | 0 .../{test => tests}/reference_count_test.cc | 0 .../task_event_buffer_export_event_test.cc | 0 .../{test => tests}/task_event_buffer_test.cc | 0 .../{test => tests}/task_manager_test.cc | 0 .../gcs_client/{test => tests}/BUILD.bazel | 8 +- .../{test => tests}/accessor_test.cc | 0 .../gcs_client_reconnection_test.cc | 2 +- .../{test => tests}/gcs_client_test.cc | 2 +- .../global_state_accessor_test.cc | 2 +- .../gcs_server/{test => tests}/BUILD.bazel | 42 +++---- .../gcs_actor_manager_export_event_test.cc | 4 +- .../gcs_job_manager_export_event_test.cc | 4 +- .../gcs_node_manager_export_event_test.cc | 4 +- .../{test => tests}/gcs_actor_manager_test.cc | 4 +- .../gcs_actor_scheduler_mock_test.cc | 0 .../gcs_actor_scheduler_test.cc | 4 +- .../gcs_autoscaler_state_manager_test.cc | 4 +- .../gcs_function_manager_test.cc | 0 .../gcs_health_check_manager_test.cc | 0 .../{test => tests}/gcs_job_manager_test.cc | 4 +- .../{test => tests}/gcs_kv_manager_test.cc | 0 .../{test => tests}/gcs_node_manager_test.cc | 4 +- .../gcs_placement_group_mgr_mock_test.cc | 2 +- .../gcs_placement_group_mgr_test.cc | 4 +- .../gcs_placement_group_scheduler_test.cc | 4 +- .../gcs_resource_manager_test.cc | 2 +- .../{test => tests}/gcs_server_rpc_test.cc | 2 +- .../{test => tests}/gcs_server_test_util.h | 0 .../gcs_table_storage_test_base.h | 2 +- .../{test => tests}/gcs_task_manager_test.cc | 2 +- .../gcs_worker_manager_test.cc | 4 +- .../in_memory_gcs_table_storage_test.cc | 2 +- .../redis_gcs_table_storage_test.cc | 2 +- .../usage_stats_client_test.cc | 0 .../store_client/{test => tests}/BUILD.bazel | 0 .../in_memory_store_client_test.cc | 2 +- .../observable_store_client_test.cc | 2 +- .../redis_store_client_test.cc | 2 +- .../{test => tests}/store_client_test_base.h | 0 src/ray/gcs/{test => tests}/BUILD.bazel | 0 .../{test => tests}/callback_reply_test.cc | 0 src/ray/gcs/{test => tests}/gcs_test_util.h | 0 .../redis_async_context_test.cc | 0 src/ray/ipc/{test => tests}/BUILD.bazel | 0 .../{test => tests}/client_connection_test.cc | 0 .../plasma/{test => tests}/BUILD.bazel | 0 .../{test => tests}/eviction_policy_test.cc | 0 .../fallback_allocator_test.cc | 0 .../{test => tests}/mutable_object_test.cc | 0 .../{test => tests}/obj_lifecycle_mgr_test.cc | 0 .../{test => tests}/object_store_test.cc | 0 .../{test => tests}/stats_collector_test.cc | 0 .../{test => tests}/BUILD.bazel | 0 .../create_request_queue_test.cc | 0 .../{test => tests}/get_request_queue_test.cc | 0 .../object_buffer_pool_test.cc | 0 .../ownership_object_directory_test.cc | 0 .../{test => tests}/pull_manager_test.cc | 0 .../{test => tests}/push_manager_test.cc | 0 .../{test => tests}/spilled_object_test.cc | 0 src/ray/pubsub/{test => tests}/BUILD.bazel | 0 .../{test => tests}/integration_test.cc | 0 .../pubsub/{test => tests}/publisher_test.cc | 0 .../pubsub/{test => tests}/subscriber_test.cc | 0 src/ray/raylet/scheduling/BUILD.bazel | 109 +----------------- .../scheduling/policy/tests/BUILD.bazel | 30 +++++ .../hybrid_scheduling_policy_test.cc | 0 .../{ => tests}/scheduling_policy_test.cc | 0 src/ray/raylet/scheduling/tests/BUILD.bazel | 79 +++++++++++++ .../cluster_resource_manager_test.cc | 0 .../cluster_resource_scheduler_2_test.cc | 0 .../cluster_resource_scheduler_test.cc | 0 .../{ => tests}/cluster_task_manager_test.cc | 2 +- .../local_resource_manager_test.cc | 0 src/ray/raylet/{test => tests}/BUILD.bazel | 2 +- .../dependency_manager_test.cc | 0 .../local_object_manager_test.cc | 2 +- .../local_task_manager_test.cc | 2 +- .../{test => tests}/node_manager_test.cc | 2 +- .../placement_group_resource_manager_test.cc | 2 +- .../runtime_env_agent_client_test.cc | 0 src/ray/raylet/{test => tests}/util.h | 0 .../{test => tests}/wait_manager_test.cc | 0 ...rker_killing_policy_group_by_owner_test.cc | 2 +- ...rker_killing_policy_retriable_fifo_test.cc | 2 +- .../worker_killing_policy_test.cc | 2 +- .../{test => tests}/worker_pool_test.cc | 0 .../node_manager/{test => tests}/BUILD.bazel | 0 .../raylet_client_pool_test.cc | 0 src/ray/rpc/{test => tests}/BUILD.bazel | 0 .../core_worker_client_pool_test.cc | 0 .../{test => tests}/grpc_bench/BUILD.bazel | 0 .../rpc/{test => tests}/grpc_bench/Dockerfile | 0 src/ray/rpc/{test => tests}/grpc_bench/README | 0 .../{test => tests}/grpc_bench/grpc_bench.cc | 0 .../grpc_bench/helloworld.proto | 0 .../grpc_server_client_test.cc | 0 src/ray/rpc/{test => tests}/rpc_chaos_test.cc | 0 src/ray/stats/BUILD.bazel | 33 +----- src/ray/stats/tests/BUILD.bazel | 31 +++++ .../{ => tests}/metric_exporter_grpc_test.cc | 0 src/ray/stats/{ => tests}/stats_test.cc | 0 src/ray/util/internal/tests/BUILD.bazel | 2 +- .../tests/stream_redirection_handle_test.cc | 2 +- src/ray/util/tests/BUILD.bazel | 10 +- src/ray/util/tests/pipe_logger_test.cc | 2 +- src/ray/util/tests/process_cleanup_test.cc | 2 +- .../util/tests/scoped_dup2_wrapper_test.cc | 2 +- .../util/tests/spdlog_newliner_sink_test.cc | 2 +- .../tests/stream_redirection_exit_test.cc | 2 +- 165 files changed, 253 insertions(+), 251 deletions(-) rename src/ray/common/cgroup/{test => tests}/BUILD.bazel (89%) rename src/ray/common/cgroup/{test => tests}/cgroup_test_utils.cc (93%) rename src/ray/common/cgroup/{test => tests}/cgroup_test_utils.h (100%) rename src/ray/common/cgroup/{test => tests}/cgroup_v2_setup_test.cc (96%) rename src/ray/common/cgroup/{test => tests}/cgroup_v2_utils_privileged_test.cc (97%) rename src/ray/common/cgroup/{test => tests}/cgroup_v2_utils_unprivileged_test.cc (97%) rename src/ray/common/cgroup/{test => tests}/fake_cgroup_setup_test.cc (98%) rename src/ray/common/cgroup2/{test => tests}/BUILD.bazel (95%) rename src/ray/common/cgroup2/{test => tests}/cgroup_test_utils.cc (98%) rename src/ray/common/cgroup2/{test => tests}/cgroup_test_utils.h (100%) rename src/ray/common/cgroup2/{test => tests}/sysfs_cgroup_driver_test.cc (98%) rename src/ray/common/{test => tests}/BUILD.bazel (95%) rename src/ray/common/{test => tests}/asio_defer_test.cc (100%) rename src/ray/common/{test => tests}/bundle_location_index_test.cc (100%) rename src/ray/common/{test => tests}/event_stats_test.cc (100%) rename src/ray/common/{test => tests}/grpc_util_test.cc (100%) rename src/ray/common/{test => tests}/id_test.cc (100%) rename src/ray/common/{test => tests}/label_selector_test.cc (100%) rename src/ray/common/{test => tests}/memory_monitor_test.cc (100%) rename src/ray/common/{test => tests}/postable_test.cc (100%) rename src/ray/common/{test => tests}/ray_config_test.cc (100%) rename src/ray/common/{test => tests}/ray_syncer_test.cc (100%) rename src/ray/common/{test => tests}/resource_instance_set_test.cc (100%) rename src/ray/common/{test => tests}/resource_request_test.cc (100%) rename src/ray/common/{test => tests}/resource_set_test.cc (100%) rename src/ray/common/{test => tests}/scheduling_ids_test.cc (100%) rename src/ray/common/{ => tests}/source_location_test.cc (93%) rename src/ray/common/{test => tests}/status_or_test.cc (99%) rename src/ray/common/{test => tests}/status_test.cc (100%) rename src/ray/common/{test => tests}/syncer_service_e2e_test.cc (100%) rename src/ray/common/{test => tests}/task_spec_test.cc (100%) rename src/ray/common/{test => tests}/testing.h (100%) rename src/ray/core_worker/task_execution/{test => tests}/BUILD.bazel (100%) rename src/ray/core_worker/task_execution/{test => tests}/concurrency_group_manager_test.cc (100%) rename src/ray/core_worker/task_execution/{test => tests}/fiber_state_test.cc (100%) rename src/ray/core_worker/task_execution/{test => tests}/scheduling_queue_test.cc (100%) rename src/ray/core_worker/task_execution/{test => tests}/task_receiver_test.cc (100%) rename src/ray/core_worker/task_execution/{test => tests}/thread_pool_test.cc (100%) rename src/ray/core_worker/task_submission/{test => tests}/BUILD.bazel (100%) rename src/ray/core_worker/task_submission/{test => tests}/actor_task_submitter_test.cc (100%) rename src/ray/core_worker/task_submission/{test => tests}/dependency_resolver_test.cc (100%) rename src/ray/core_worker/task_submission/{test => tests}/direct_actor_transport_test.cc (100%) rename src/ray/core_worker/task_submission/{test => tests}/normal_task_submitter_test.cc (100%) rename src/ray/core_worker/task_submission/{test => tests}/out_of_order_actor_submit_queue_test.cc (100%) rename src/ray/core_worker/{test => tests}/BUILD.bazel (100%) rename src/ray/core_worker/{test => tests}/actor_creator_test.cc (100%) rename src/ray/core_worker/{test => tests}/actor_manager_test.cc (100%) rename src/ray/core_worker/{test => tests}/core_worker_resubmit_queue_test.cc (100%) rename src/ray/core_worker/{test => tests}/core_worker_test.cc (100%) rename src/ray/core_worker/{test => tests}/generator_waiter_test.cc (100%) rename src/ray/core_worker/{test => tests}/lease_policy_test.cc (100%) rename src/ray/core_worker/{test => tests}/memory_store_test.cc (100%) rename src/ray/core_worker/{test => tests}/mutable_object_provider_test.cc (100%) rename src/ray/core_worker/{test => tests}/object_recovery_manager_test.cc (100%) rename src/ray/core_worker/{test => tests}/reference_count_test.cc (100%) rename src/ray/core_worker/{test => tests}/task_event_buffer_export_event_test.cc (100%) rename src/ray/core_worker/{test => tests}/task_event_buffer_test.cc (100%) rename src/ray/core_worker/{test => tests}/task_manager_test.cc (100%) rename src/ray/gcs/gcs_client/{test => tests}/BUILD.bazel (91%) rename src/ray/gcs/gcs_client/{test => tests}/accessor_test.cc (100%) rename src/ray/gcs/gcs_client/{test => tests}/gcs_client_reconnection_test.cc (99%) rename src/ray/gcs/gcs_client/{test => tests}/gcs_client_test.cc (99%) rename src/ray/gcs/gcs_client/{test => tests}/global_state_accessor_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/BUILD.bazel (88%) rename src/ray/gcs/gcs_server/{test => tests}/export_api/gcs_actor_manager_export_event_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/export_api/gcs_job_manager_export_event_test.cc (98%) rename src/ray/gcs/gcs_server/{test => tests}/export_api/gcs_node_manager_export_event_test.cc (98%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_actor_manager_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_actor_scheduler_mock_test.cc (100%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_actor_scheduler_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_autoscaler_state_manager_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_function_manager_test.cc (100%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_health_check_manager_test.cc (100%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_job_manager_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_kv_manager_test.cc (100%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_node_manager_test.cc (98%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_placement_group_mgr_mock_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_placement_group_mgr_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_placement_group_scheduler_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_resource_manager_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_server_rpc_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_server_test_util.h (100%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_table_storage_test_base.h (99%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_task_manager_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/gcs_worker_manager_test.cc (99%) rename src/ray/gcs/gcs_server/{test => tests}/in_memory_gcs_table_storage_test.cc (94%) rename src/ray/gcs/gcs_server/{test => tests}/redis_gcs_table_storage_test.cc (96%) rename src/ray/gcs/gcs_server/{test => tests}/usage_stats_client_test.cc (100%) rename src/ray/gcs/store_client/{test => tests}/BUILD.bazel (100%) rename src/ray/gcs/store_client/{test => tests}/in_memory_store_client_test.cc (95%) rename src/ray/gcs/store_client/{test => tests}/observable_store_client_test.cc (95%) rename src/ray/gcs/store_client/{test => tests}/redis_store_client_test.cc (99%) rename src/ray/gcs/store_client/{test => tests}/store_client_test_base.h (100%) rename src/ray/gcs/{test => tests}/BUILD.bazel (100%) rename src/ray/gcs/{test => tests}/callback_reply_test.cc (100%) rename src/ray/gcs/{test => tests}/gcs_test_util.h (100%) rename src/ray/gcs/{test => tests}/redis_async_context_test.cc (100%) rename src/ray/ipc/{test => tests}/BUILD.bazel (100%) rename src/ray/ipc/{test => tests}/client_connection_test.cc (100%) rename src/ray/object_manager/plasma/{test => tests}/BUILD.bazel (100%) rename src/ray/object_manager/plasma/{test => tests}/eviction_policy_test.cc (100%) rename src/ray/object_manager/plasma/{test => tests}/fallback_allocator_test.cc (100%) rename src/ray/object_manager/plasma/{test => tests}/mutable_object_test.cc (100%) rename src/ray/object_manager/plasma/{test => tests}/obj_lifecycle_mgr_test.cc (100%) rename src/ray/object_manager/plasma/{test => tests}/object_store_test.cc (100%) rename src/ray/object_manager/plasma/{test => tests}/stats_collector_test.cc (100%) rename src/ray/object_manager/{test => tests}/BUILD.bazel (100%) rename src/ray/object_manager/{test => tests}/create_request_queue_test.cc (100%) rename src/ray/object_manager/{test => tests}/get_request_queue_test.cc (100%) rename src/ray/object_manager/{test => tests}/object_buffer_pool_test.cc (100%) rename src/ray/object_manager/{test => tests}/ownership_object_directory_test.cc (100%) rename src/ray/object_manager/{test => tests}/pull_manager_test.cc (100%) rename src/ray/object_manager/{test => tests}/push_manager_test.cc (100%) rename src/ray/object_manager/{test => tests}/spilled_object_test.cc (100%) rename src/ray/pubsub/{test => tests}/BUILD.bazel (100%) rename src/ray/pubsub/{test => tests}/integration_test.cc (100%) rename src/ray/pubsub/{test => tests}/publisher_test.cc (100%) rename src/ray/pubsub/{test => tests}/subscriber_test.cc (100%) create mode 100644 src/ray/raylet/scheduling/policy/tests/BUILD.bazel rename src/ray/raylet/scheduling/policy/{ => tests}/hybrid_scheduling_policy_test.cc (100%) rename src/ray/raylet/scheduling/policy/{ => tests}/scheduling_policy_test.cc (100%) create mode 100644 src/ray/raylet/scheduling/tests/BUILD.bazel rename src/ray/raylet/scheduling/{ => tests}/cluster_resource_manager_test.cc (100%) rename src/ray/raylet/scheduling/{ => tests}/cluster_resource_scheduler_2_test.cc (100%) rename src/ray/raylet/scheduling/{ => tests}/cluster_resource_scheduler_test.cc (100%) rename src/ray/raylet/scheduling/{ => tests}/cluster_task_manager_test.cc (99%) rename src/ray/raylet/scheduling/{ => tests}/local_resource_manager_test.cc (100%) rename src/ray/raylet/{test => tests}/BUILD.bazel (99%) rename src/ray/raylet/{test => tests}/dependency_manager_test.cc (100%) rename src/ray/raylet/{test => tests}/local_object_manager_test.cc (99%) rename src/ray/raylet/{test => tests}/local_task_manager_test.cc (99%) rename src/ray/raylet/{test => tests}/node_manager_test.cc (99%) rename src/ray/raylet/{test => tests}/placement_group_resource_manager_test.cc (99%) rename src/ray/raylet/{test => tests}/runtime_env_agent_client_test.cc (100%) rename src/ray/raylet/{test => tests}/util.h (100%) rename src/ray/raylet/{test => tests}/wait_manager_test.cc (100%) rename src/ray/raylet/{test => tests}/worker_killing_policy_group_by_owner_test.cc (99%) rename src/ray/raylet/{test => tests}/worker_killing_policy_retriable_fifo_test.cc (99%) rename src/ray/raylet/{test => tests}/worker_killing_policy_test.cc (99%) rename src/ray/raylet/{test => tests}/worker_pool_test.cc (100%) rename src/ray/rpc/node_manager/{test => tests}/BUILD.bazel (100%) rename src/ray/rpc/node_manager/{test => tests}/raylet_client_pool_test.cc (100%) rename src/ray/rpc/{test => tests}/BUILD.bazel (100%) rename src/ray/rpc/{test => tests}/core_worker_client_pool_test.cc (100%) rename src/ray/rpc/{test => tests}/grpc_bench/BUILD.bazel (100%) rename src/ray/rpc/{test => tests}/grpc_bench/Dockerfile (100%) rename src/ray/rpc/{test => tests}/grpc_bench/README (100%) rename src/ray/rpc/{test => tests}/grpc_bench/grpc_bench.cc (100%) rename src/ray/rpc/{test => tests}/grpc_bench/helloworld.proto (100%) rename src/ray/rpc/{test => tests}/grpc_server_client_test.cc (100%) rename src/ray/rpc/{test => tests}/rpc_chaos_test.cc (100%) rename src/ray/stats/{ => tests}/metric_exporter_grpc_test.cc (100%) rename src/ray/stats/{ => tests}/stats_test.cc (100%) diff --git a/src/ray/common/BUILD.bazel b/src/ray/common/BUILD.bazel index aaa5dc7f6109..7e95589b69ad 100644 --- a/src/ray/common/BUILD.bazel +++ b/src/ray/common/BUILD.bazel @@ -1,4 +1,4 @@ -load("//bazel:ray.bzl", "ray_cc_library", "ray_cc_test") +load("//bazel:ray.bzl", "ray_cc_library") ray_cc_library( name = "compat", @@ -312,14 +312,3 @@ ray_cc_library( srcs = ["source_location.cc"], hdrs = ["source_location.h"], ) - -ray_cc_test( - name = "source_location_test", - size = "small", - srcs = ["source_location_test.cc"], - tags = ["team:core"], - deps = [ - ":source_location", - "@com_google_googletest//:gtest_main", - ], -) diff --git a/src/ray/common/cgroup/test/BUILD.bazel b/src/ray/common/cgroup/tests/BUILD.bazel similarity index 89% rename from src/ray/common/cgroup/test/BUILD.bazel rename to src/ray/common/cgroup/tests/BUILD.bazel index e0777ca2bec1..5ad80e0fe6aa 100644 --- a/src/ray/common/cgroup/test/BUILD.bazel +++ b/src/ray/common/cgroup/tests/BUILD.bazel @@ -12,7 +12,7 @@ ray_cc_test( ], deps = [ "//src/ray/common/cgroup:cgroup_setup", - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "@com_google_googletest//:gtest_main", ], ) @@ -28,7 +28,7 @@ ray_cc_test( ], deps = [ "//src/ray/common/cgroup:cgroup_setup", - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "@com_google_googletest//:gtest_main", ], ) @@ -42,7 +42,7 @@ ray_cc_test( ], deps = [ "//src/ray/common/cgroup:fake_cgroup_setup", - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "@com_google_googletest//:gtest_main", ], ) @@ -59,7 +59,7 @@ ray_cc_test( ":cgroup_test_utils", "//src/ray/common/cgroup:cgroup_setup", "//src/ray/common/cgroup:cgroup_utils", - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "@com_google_googletest//:gtest_main", ], ) @@ -70,7 +70,7 @@ ray_cc_library( srcs = ["cgroup_test_utils.cc"], hdrs = ["cgroup_test_utils.h"], deps = [ - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "//src/ray/util:compat", "//src/ray/util:container_util", "//src/ray/util:filesystem", diff --git a/src/ray/common/cgroup/test/cgroup_test_utils.cc b/src/ray/common/cgroup/tests/cgroup_test_utils.cc similarity index 93% rename from src/ray/common/cgroup/test/cgroup_test_utils.cc rename to src/ray/common/cgroup/tests/cgroup_test_utils.cc index 3303c4270b75..bdc373fd69d6 100644 --- a/src/ray/common/cgroup/test/cgroup_test_utils.cc +++ b/src/ray/common/cgroup/tests/cgroup_test_utils.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/common/cgroup/test/cgroup_test_utils.h" +#include "ray/common/cgroup/tests/cgroup_test_utils.h" #include @@ -21,7 +21,7 @@ #include "absl/strings/str_split.h" #include "absl/strings/strip.h" -#include "ray/common/test/testing.h" +#include "ray/common/tests/testing.h" #include "ray/util/container_util.h" #include "ray/util/filesystem.h" diff --git a/src/ray/common/cgroup/test/cgroup_test_utils.h b/src/ray/common/cgroup/tests/cgroup_test_utils.h similarity index 100% rename from src/ray/common/cgroup/test/cgroup_test_utils.h rename to src/ray/common/cgroup/tests/cgroup_test_utils.h diff --git a/src/ray/common/cgroup/test/cgroup_v2_setup_test.cc b/src/ray/common/cgroup/tests/cgroup_v2_setup_test.cc similarity index 96% rename from src/ray/common/cgroup/test/cgroup_v2_setup_test.cc rename to src/ray/common/cgroup/tests/cgroup_v2_setup_test.cc index 3feff7298d8a..1cc57433c9f3 100644 --- a/src/ray/common/cgroup/test/cgroup_v2_setup_test.cc +++ b/src/ray/common/cgroup/tests/cgroup_v2_setup_test.cc @@ -20,7 +20,7 @@ // https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/8/html/managing_monitoring_and_updating_the_kernel/using-cgroups-v2-to-control-distribution-of-cpu-time-for-applications_managing-monitoring-and-updating-the-kernel#mounting-cgroups-v2_using-cgroups-v2-to-control-distribution-of-cpu-time-for-applications // // Execution command: -// sudo bazel-bin/src/ray/common/cgroup/test/cgroup_v2_setup_test +// sudo bazel-bin/src/ray/common/cgroup/tests/cgroup_v2_setup_test #include #include @@ -35,8 +35,8 @@ #include "ray/common/cgroup/cgroup_setup.h" #include "ray/common/cgroup/cgroup_utils.h" -#include "ray/common/cgroup/test/cgroup_test_utils.h" -#include "ray/common/test/testing.h" +#include "ray/common/cgroup/tests/cgroup_test_utils.h" +#include "ray/common/tests/testing.h" namespace ray { diff --git a/src/ray/common/cgroup/test/cgroup_v2_utils_privileged_test.cc b/src/ray/common/cgroup/tests/cgroup_v2_utils_privileged_test.cc similarity index 97% rename from src/ray/common/cgroup/test/cgroup_v2_utils_privileged_test.cc rename to src/ray/common/cgroup/tests/cgroup_v2_utils_privileged_test.cc index ec1c12e4f8d5..14b0bf9182e7 100644 --- a/src/ray/common/cgroup/test/cgroup_v2_utils_privileged_test.cc +++ b/src/ray/common/cgroup/tests/cgroup_v2_utils_privileged_test.cc @@ -15,7 +15,7 @@ #include #include "ray/common/cgroup/cgroup_setup.h" -#include "ray/common/test/testing.h" +#include "ray/common/tests/testing.h" namespace ray::internal { diff --git a/src/ray/common/cgroup/test/cgroup_v2_utils_unprivileged_test.cc b/src/ray/common/cgroup/tests/cgroup_v2_utils_unprivileged_test.cc similarity index 97% rename from src/ray/common/cgroup/test/cgroup_v2_utils_unprivileged_test.cc rename to src/ray/common/cgroup/tests/cgroup_v2_utils_unprivileged_test.cc index 723f38bc4dfc..38626b4ca313 100644 --- a/src/ray/common/cgroup/test/cgroup_v2_utils_unprivileged_test.cc +++ b/src/ray/common/cgroup/tests/cgroup_v2_utils_unprivileged_test.cc @@ -20,7 +20,7 @@ #include #include "ray/common/cgroup/cgroup_setup.h" -#include "ray/common/test/testing.h" +#include "ray/common/tests/testing.h" namespace ray::internal { diff --git a/src/ray/common/cgroup/test/fake_cgroup_setup_test.cc b/src/ray/common/cgroup/tests/fake_cgroup_setup_test.cc similarity index 98% rename from src/ray/common/cgroup/test/fake_cgroup_setup_test.cc rename to src/ray/common/cgroup/tests/fake_cgroup_setup_test.cc index 59c15dabb9ab..fd29f13391ec 100644 --- a/src/ray/common/cgroup/test/fake_cgroup_setup_test.cc +++ b/src/ray/common/cgroup/tests/fake_cgroup_setup_test.cc @@ -19,7 +19,7 @@ #include #include -#include "ray/common/test/testing.h" +#include "ray/common/tests/testing.h" namespace ray { diff --git a/src/ray/common/cgroup2/test/BUILD.bazel b/src/ray/common/cgroup2/tests/BUILD.bazel similarity index 95% rename from src/ray/common/cgroup2/test/BUILD.bazel rename to src/ray/common/cgroup2/tests/BUILD.bazel index e829d9c9e080..c31181a7ec2d 100644 --- a/src/ray/common/cgroup2/test/BUILD.bazel +++ b/src/ray/common/cgroup2/tests/BUILD.bazel @@ -26,7 +26,7 @@ ray_cc_test( "//src/ray/common:status", "//src/ray/common:status_or", "//src/ray/common/cgroup2:sysfs_cgroup_driver", - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "@com_google_absl//absl/strings:str_format", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/common/cgroup2/test/cgroup_test_utils.cc b/src/ray/common/cgroup2/tests/cgroup_test_utils.cc similarity index 98% rename from src/ray/common/cgroup2/test/cgroup_test_utils.cc rename to src/ray/common/cgroup2/tests/cgroup_test_utils.cc index e61ad82e633c..c31109dddb54 100644 --- a/src/ray/common/cgroup2/test/cgroup_test_utils.cc +++ b/src/ray/common/cgroup2/tests/cgroup_test_utils.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/common/cgroup2/test/cgroup_test_utils.h" +#include "ray/common/cgroup2/tests/cgroup_test_utils.h" #include #include diff --git a/src/ray/common/cgroup2/test/cgroup_test_utils.h b/src/ray/common/cgroup2/tests/cgroup_test_utils.h similarity index 100% rename from src/ray/common/cgroup2/test/cgroup_test_utils.h rename to src/ray/common/cgroup2/tests/cgroup_test_utils.h diff --git a/src/ray/common/cgroup2/test/sysfs_cgroup_driver_test.cc b/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc similarity index 98% rename from src/ray/common/cgroup2/test/sysfs_cgroup_driver_test.cc rename to src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc index 275a122e808f..a349be577616 100644 --- a/src/ray/common/cgroup2/test/sysfs_cgroup_driver_test.cc +++ b/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc @@ -20,7 +20,7 @@ #include #include "gtest/gtest.h" -#include "ray/common/cgroup2/test/cgroup_test_utils.h" +#include "ray/common/cgroup2/tests/cgroup_test_utils.h" #include "ray/common/status.h" #include "ray/common/status_or.h" diff --git a/src/ray/common/test/BUILD.bazel b/src/ray/common/tests/BUILD.bazel similarity index 95% rename from src/ray/common/test/BUILD.bazel rename to src/ray/common/tests/BUILD.bazel index ca25a451e3ea..cbb4211981b5 100644 --- a/src/ray/common/test/BUILD.bazel +++ b/src/ray/common/tests/BUILD.bazel @@ -243,3 +243,14 @@ ray_cc_test( "@com_google_googletest//:gtest_main", ], ) + +ray_cc_test( + name = "source_location_test", + size = "small", + srcs = ["source_location_test.cc"], + tags = ["team:core"], + deps = [ + "//src/ray/common:source_location", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/common/test/asio_defer_test.cc b/src/ray/common/tests/asio_defer_test.cc similarity index 100% rename from src/ray/common/test/asio_defer_test.cc rename to src/ray/common/tests/asio_defer_test.cc diff --git a/src/ray/common/test/bundle_location_index_test.cc b/src/ray/common/tests/bundle_location_index_test.cc similarity index 100% rename from src/ray/common/test/bundle_location_index_test.cc rename to src/ray/common/tests/bundle_location_index_test.cc diff --git a/src/ray/common/test/event_stats_test.cc b/src/ray/common/tests/event_stats_test.cc similarity index 100% rename from src/ray/common/test/event_stats_test.cc rename to src/ray/common/tests/event_stats_test.cc diff --git a/src/ray/common/test/grpc_util_test.cc b/src/ray/common/tests/grpc_util_test.cc similarity index 100% rename from src/ray/common/test/grpc_util_test.cc rename to src/ray/common/tests/grpc_util_test.cc diff --git a/src/ray/common/test/id_test.cc b/src/ray/common/tests/id_test.cc similarity index 100% rename from src/ray/common/test/id_test.cc rename to src/ray/common/tests/id_test.cc diff --git a/src/ray/common/test/label_selector_test.cc b/src/ray/common/tests/label_selector_test.cc similarity index 100% rename from src/ray/common/test/label_selector_test.cc rename to src/ray/common/tests/label_selector_test.cc diff --git a/src/ray/common/test/memory_monitor_test.cc b/src/ray/common/tests/memory_monitor_test.cc similarity index 100% rename from src/ray/common/test/memory_monitor_test.cc rename to src/ray/common/tests/memory_monitor_test.cc diff --git a/src/ray/common/test/postable_test.cc b/src/ray/common/tests/postable_test.cc similarity index 100% rename from src/ray/common/test/postable_test.cc rename to src/ray/common/tests/postable_test.cc diff --git a/src/ray/common/test/ray_config_test.cc b/src/ray/common/tests/ray_config_test.cc similarity index 100% rename from src/ray/common/test/ray_config_test.cc rename to src/ray/common/tests/ray_config_test.cc diff --git a/src/ray/common/test/ray_syncer_test.cc b/src/ray/common/tests/ray_syncer_test.cc similarity index 100% rename from src/ray/common/test/ray_syncer_test.cc rename to src/ray/common/tests/ray_syncer_test.cc diff --git a/src/ray/common/test/resource_instance_set_test.cc b/src/ray/common/tests/resource_instance_set_test.cc similarity index 100% rename from src/ray/common/test/resource_instance_set_test.cc rename to src/ray/common/tests/resource_instance_set_test.cc diff --git a/src/ray/common/test/resource_request_test.cc b/src/ray/common/tests/resource_request_test.cc similarity index 100% rename from src/ray/common/test/resource_request_test.cc rename to src/ray/common/tests/resource_request_test.cc diff --git a/src/ray/common/test/resource_set_test.cc b/src/ray/common/tests/resource_set_test.cc similarity index 100% rename from src/ray/common/test/resource_set_test.cc rename to src/ray/common/tests/resource_set_test.cc diff --git a/src/ray/common/test/scheduling_ids_test.cc b/src/ray/common/tests/scheduling_ids_test.cc similarity index 100% rename from src/ray/common/test/scheduling_ids_test.cc rename to src/ray/common/tests/scheduling_ids_test.cc diff --git a/src/ray/common/source_location_test.cc b/src/ray/common/tests/source_location_test.cc similarity index 93% rename from src/ray/common/source_location_test.cc rename to src/ray/common/tests/source_location_test.cc index fe5c5b3078ec..74d938cee0d6 100644 --- a/src/ray/common/source_location_test.cc +++ b/src/ray/common/tests/source_location_test.cc @@ -35,7 +35,7 @@ TEST(SourceLocationTest, StringifyTest) { auto loc = RAY_LOC(); std::stringstream ss{}; ss << loc; - EXPECT_EQ(ss.str(), "src/ray/common/source_location_test.cc:35"); + EXPECT_EQ(ss.str(), "src/ray/common/tests/source_location_test.cc:35"); } } diff --git a/src/ray/common/test/status_or_test.cc b/src/ray/common/tests/status_or_test.cc similarity index 99% rename from src/ray/common/test/status_or_test.cc rename to src/ray/common/tests/status_or_test.cc index 4a5f41a4542e..5c20ab4e387a 100644 --- a/src/ray/common/test/status_or_test.cc +++ b/src/ray/common/tests/status_or_test.cc @@ -19,7 +19,7 @@ #include #include -#include "ray/common/test/testing.h" +#include "ray/common/tests/testing.h" namespace ray { diff --git a/src/ray/common/test/status_test.cc b/src/ray/common/tests/status_test.cc similarity index 100% rename from src/ray/common/test/status_test.cc rename to src/ray/common/tests/status_test.cc diff --git a/src/ray/common/test/syncer_service_e2e_test.cc b/src/ray/common/tests/syncer_service_e2e_test.cc similarity index 100% rename from src/ray/common/test/syncer_service_e2e_test.cc rename to src/ray/common/tests/syncer_service_e2e_test.cc diff --git a/src/ray/common/test/task_spec_test.cc b/src/ray/common/tests/task_spec_test.cc similarity index 100% rename from src/ray/common/test/task_spec_test.cc rename to src/ray/common/tests/task_spec_test.cc diff --git a/src/ray/common/test/testing.h b/src/ray/common/tests/testing.h similarity index 100% rename from src/ray/common/test/testing.h rename to src/ray/common/tests/testing.h diff --git a/src/ray/core_worker/task_execution/test/BUILD.bazel b/src/ray/core_worker/task_execution/tests/BUILD.bazel similarity index 100% rename from src/ray/core_worker/task_execution/test/BUILD.bazel rename to src/ray/core_worker/task_execution/tests/BUILD.bazel diff --git a/src/ray/core_worker/task_execution/test/concurrency_group_manager_test.cc b/src/ray/core_worker/task_execution/tests/concurrency_group_manager_test.cc similarity index 100% rename from src/ray/core_worker/task_execution/test/concurrency_group_manager_test.cc rename to src/ray/core_worker/task_execution/tests/concurrency_group_manager_test.cc diff --git a/src/ray/core_worker/task_execution/test/fiber_state_test.cc b/src/ray/core_worker/task_execution/tests/fiber_state_test.cc similarity index 100% rename from src/ray/core_worker/task_execution/test/fiber_state_test.cc rename to src/ray/core_worker/task_execution/tests/fiber_state_test.cc diff --git a/src/ray/core_worker/task_execution/test/scheduling_queue_test.cc b/src/ray/core_worker/task_execution/tests/scheduling_queue_test.cc similarity index 100% rename from src/ray/core_worker/task_execution/test/scheduling_queue_test.cc rename to src/ray/core_worker/task_execution/tests/scheduling_queue_test.cc diff --git a/src/ray/core_worker/task_execution/test/task_receiver_test.cc b/src/ray/core_worker/task_execution/tests/task_receiver_test.cc similarity index 100% rename from src/ray/core_worker/task_execution/test/task_receiver_test.cc rename to src/ray/core_worker/task_execution/tests/task_receiver_test.cc diff --git a/src/ray/core_worker/task_execution/test/thread_pool_test.cc b/src/ray/core_worker/task_execution/tests/thread_pool_test.cc similarity index 100% rename from src/ray/core_worker/task_execution/test/thread_pool_test.cc rename to src/ray/core_worker/task_execution/tests/thread_pool_test.cc diff --git a/src/ray/core_worker/task_submission/test/BUILD.bazel b/src/ray/core_worker/task_submission/tests/BUILD.bazel similarity index 100% rename from src/ray/core_worker/task_submission/test/BUILD.bazel rename to src/ray/core_worker/task_submission/tests/BUILD.bazel diff --git a/src/ray/core_worker/task_submission/test/actor_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc similarity index 100% rename from src/ray/core_worker/task_submission/test/actor_task_submitter_test.cc rename to src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc diff --git a/src/ray/core_worker/task_submission/test/dependency_resolver_test.cc b/src/ray/core_worker/task_submission/tests/dependency_resolver_test.cc similarity index 100% rename from src/ray/core_worker/task_submission/test/dependency_resolver_test.cc rename to src/ray/core_worker/task_submission/tests/dependency_resolver_test.cc diff --git a/src/ray/core_worker/task_submission/test/direct_actor_transport_test.cc b/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc similarity index 100% rename from src/ray/core_worker/task_submission/test/direct_actor_transport_test.cc rename to src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc diff --git a/src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc similarity index 100% rename from src/ray/core_worker/task_submission/test/normal_task_submitter_test.cc rename to src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc diff --git a/src/ray/core_worker/task_submission/test/out_of_order_actor_submit_queue_test.cc b/src/ray/core_worker/task_submission/tests/out_of_order_actor_submit_queue_test.cc similarity index 100% rename from src/ray/core_worker/task_submission/test/out_of_order_actor_submit_queue_test.cc rename to src/ray/core_worker/task_submission/tests/out_of_order_actor_submit_queue_test.cc diff --git a/src/ray/core_worker/test/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel similarity index 100% rename from src/ray/core_worker/test/BUILD.bazel rename to src/ray/core_worker/tests/BUILD.bazel diff --git a/src/ray/core_worker/test/actor_creator_test.cc b/src/ray/core_worker/tests/actor_creator_test.cc similarity index 100% rename from src/ray/core_worker/test/actor_creator_test.cc rename to src/ray/core_worker/tests/actor_creator_test.cc diff --git a/src/ray/core_worker/test/actor_manager_test.cc b/src/ray/core_worker/tests/actor_manager_test.cc similarity index 100% rename from src/ray/core_worker/test/actor_manager_test.cc rename to src/ray/core_worker/tests/actor_manager_test.cc diff --git a/src/ray/core_worker/test/core_worker_resubmit_queue_test.cc b/src/ray/core_worker/tests/core_worker_resubmit_queue_test.cc similarity index 100% rename from src/ray/core_worker/test/core_worker_resubmit_queue_test.cc rename to src/ray/core_worker/tests/core_worker_resubmit_queue_test.cc diff --git a/src/ray/core_worker/test/core_worker_test.cc b/src/ray/core_worker/tests/core_worker_test.cc similarity index 100% rename from src/ray/core_worker/test/core_worker_test.cc rename to src/ray/core_worker/tests/core_worker_test.cc diff --git a/src/ray/core_worker/test/generator_waiter_test.cc b/src/ray/core_worker/tests/generator_waiter_test.cc similarity index 100% rename from src/ray/core_worker/test/generator_waiter_test.cc rename to src/ray/core_worker/tests/generator_waiter_test.cc diff --git a/src/ray/core_worker/test/lease_policy_test.cc b/src/ray/core_worker/tests/lease_policy_test.cc similarity index 100% rename from src/ray/core_worker/test/lease_policy_test.cc rename to src/ray/core_worker/tests/lease_policy_test.cc diff --git a/src/ray/core_worker/test/memory_store_test.cc b/src/ray/core_worker/tests/memory_store_test.cc similarity index 100% rename from src/ray/core_worker/test/memory_store_test.cc rename to src/ray/core_worker/tests/memory_store_test.cc diff --git a/src/ray/core_worker/test/mutable_object_provider_test.cc b/src/ray/core_worker/tests/mutable_object_provider_test.cc similarity index 100% rename from src/ray/core_worker/test/mutable_object_provider_test.cc rename to src/ray/core_worker/tests/mutable_object_provider_test.cc diff --git a/src/ray/core_worker/test/object_recovery_manager_test.cc b/src/ray/core_worker/tests/object_recovery_manager_test.cc similarity index 100% rename from src/ray/core_worker/test/object_recovery_manager_test.cc rename to src/ray/core_worker/tests/object_recovery_manager_test.cc diff --git a/src/ray/core_worker/test/reference_count_test.cc b/src/ray/core_worker/tests/reference_count_test.cc similarity index 100% rename from src/ray/core_worker/test/reference_count_test.cc rename to src/ray/core_worker/tests/reference_count_test.cc diff --git a/src/ray/core_worker/test/task_event_buffer_export_event_test.cc b/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc similarity index 100% rename from src/ray/core_worker/test/task_event_buffer_export_event_test.cc rename to src/ray/core_worker/tests/task_event_buffer_export_event_test.cc diff --git a/src/ray/core_worker/test/task_event_buffer_test.cc b/src/ray/core_worker/tests/task_event_buffer_test.cc similarity index 100% rename from src/ray/core_worker/test/task_event_buffer_test.cc rename to src/ray/core_worker/tests/task_event_buffer_test.cc diff --git a/src/ray/core_worker/test/task_manager_test.cc b/src/ray/core_worker/tests/task_manager_test.cc similarity index 100% rename from src/ray/core_worker/test/task_manager_test.cc rename to src/ray/core_worker/tests/task_manager_test.cc diff --git a/src/ray/gcs/gcs_client/test/BUILD.bazel b/src/ray/gcs/gcs_client/tests/BUILD.bazel similarity index 91% rename from src/ray/gcs/gcs_client/test/BUILD.bazel rename to src/ray/gcs/gcs_client/tests/BUILD.bazel index 842853c628cf..cc428b88a513 100644 --- a/src/ray/gcs/gcs_client/test/BUILD.bazel +++ b/src/ray/gcs/gcs_client/tests/BUILD.bazel @@ -9,7 +9,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/ray/gcs/gcs_client:gcs_client_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -33,7 +33,7 @@ ray_cc_test( "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_client:global_state_accessor_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -60,7 +60,7 @@ ray_cc_test( deps = [ "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:network_util", "@com_google_googletest//:gtest_main", ], @@ -86,7 +86,7 @@ ray_cc_test( deps = [ "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:network_util", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/gcs/gcs_client/test/accessor_test.cc b/src/ray/gcs/gcs_client/tests/accessor_test.cc similarity index 100% rename from src/ray/gcs/gcs_client/test/accessor_test.cc rename to src/ray/gcs/gcs_client/tests/accessor_test.cc diff --git a/src/ray/gcs/gcs_client/test/gcs_client_reconnection_test.cc b/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc similarity index 99% rename from src/ray/gcs/gcs_client/test/gcs_client_reconnection_test.cc rename to src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc index fd828cb27aa4..409d16bbadaf 100644 --- a/src/ray/gcs/gcs_client/test/gcs_client_reconnection_test.cc +++ b/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc @@ -25,7 +25,7 @@ #include "ray/gcs/gcs_client/accessor.h" #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/rpc/gcs/gcs_rpc_client.h" #include "ray/util/network_util.h" #include "ray/util/path_utils.h" diff --git a/src/ray/gcs/gcs_client/test/gcs_client_test.cc b/src/ray/gcs/gcs_client/tests/gcs_client_test.cc similarity index 99% rename from src/ray/gcs/gcs_client/test/gcs_client_test.cc rename to src/ray/gcs/gcs_client/tests/gcs_client_test.cc index 0739777b954f..6df7b90bd2ab 100644 --- a/src/ray/gcs/gcs_client/test/gcs_client_test.cc +++ b/src/ray/gcs/gcs_client/tests/gcs_client_test.cc @@ -24,7 +24,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/gcs/gcs_client/accessor.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/rpc/gcs/gcs_rpc_client.h" #include "ray/util/network_util.h" #include "ray/util/path_utils.h" diff --git a/src/ray/gcs/gcs_client/test/global_state_accessor_test.cc b/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc similarity index 99% rename from src/ray/gcs/gcs_client/test/global_state_accessor_test.cc rename to src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc index c5d186e41ec7..f761cb5c90b9 100644 --- a/src/ray/gcs/gcs_client/test/global_state_accessor_test.cc +++ b/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc @@ -21,7 +21,7 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/rpc/gcs/gcs_rpc_client.h" #include "ray/util/path_utils.h" diff --git a/src/ray/gcs/gcs_server/test/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel similarity index 88% rename from src/ray/gcs/gcs_server/test/BUILD.bazel rename to src/ray/gcs/gcs_server/tests/BUILD.bazel index aa110ed7ae24..b0af67f3acd5 100644 --- a/src/ray/gcs/gcs_server/test/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -21,7 +21,7 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -47,7 +47,7 @@ ray_cc_test( ], deps = [ "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest", ], ) @@ -69,7 +69,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest", ], ) @@ -114,7 +114,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -130,7 +130,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -146,7 +146,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -165,7 +165,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -184,7 +184,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -201,7 +201,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -234,7 +234,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -250,7 +250,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:process", "@com_google_googletest//:gtest_main", ], @@ -284,8 +284,8 @@ ray_cc_test( deps = [ ":gcs_table_storage_test_lib", "//src/ray/gcs/gcs_server:gcs_table_storage", - "//src/ray/gcs/store_client/test:store_client_test_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/store_client/tests:store_client_test_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest", ], ) @@ -299,8 +299,8 @@ ray_cc_test( ":gcs_table_storage_test_lib", "//src/ray/common:test_util", "//src/ray/gcs/gcs_server:gcs_table_storage", - "//src/ray/gcs/store_client/test:store_client_test_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/store_client/tests:store_client_test_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -316,7 +316,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -331,7 +331,7 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -347,7 +347,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -364,7 +364,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -381,7 +381,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -398,7 +398,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/gcs/gcs_server/test/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/export_api/gcs_actor_manager_export_event_test.cc rename to src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc index e5942c4da032..d3940e6c5021 100644 --- a/src/ray/gcs/gcs_server/test/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc @@ -23,8 +23,8 @@ // clang-format off #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/test/gcs_server_test_util.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" diff --git a/src/ray/gcs/gcs_server/test/export_api/gcs_job_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc similarity index 98% rename from src/ray/gcs/gcs_server/test/export_api/gcs_job_manager_export_event_test.cc rename to src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc index ffcbeb7d3676..049a6df3a4e3 100644 --- a/src/ray/gcs/gcs_server/test/export_api/gcs_job_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc @@ -20,9 +20,9 @@ // clang-format off #include "gtest/gtest.h" -#include "ray/gcs/gcs_server/test/gcs_server_test_util.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/pubsub/publisher.h" diff --git a/src/ray/gcs/gcs_server/test/export_api/gcs_node_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc similarity index 98% rename from src/ray/gcs/gcs_server/test/export_api/gcs_node_manager_export_event_test.cc rename to src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc index 137a1e271c62..52da5b41a4b9 100644 --- a/src/ray/gcs/gcs_server/test/export_api/gcs_node_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc @@ -20,8 +20,8 @@ #include #include -#include "ray/gcs/gcs_server/test/gcs_server_test_util.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/util/event.h" #include "ray/util/string_utils.h" diff --git a/src/ray/gcs/gcs_server/test/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_actor_manager_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc index ba3e54770746..303085699800 100644 --- a/src/ray/gcs/gcs_server/test/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc @@ -21,8 +21,8 @@ // clang-format off #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/test/gcs_server_test_util.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" diff --git a/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc similarity index 100% rename from src/ray/gcs/gcs_server/test/gcs_actor_scheduler_mock_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc diff --git a/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc index 128d5df7124b..b11e3f8ed624 100644 --- a/src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc @@ -23,8 +23,8 @@ // clang-format off #include "ray/common/asio/asio_util.h" #include "ray/gcs/gcs_server/gcs_actor_scheduler.h" -#include "ray/gcs/gcs_server/test/gcs_server_test_util.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "mock/ray/pubsub/publisher.h" // clang-format on diff --git a/src/ray/gcs/gcs_server/test/gcs_autoscaler_state_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_autoscaler_state_manager_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc index a706feba521b..2b95dae2729c 100644 --- a/src/ray/gcs/gcs_server/test/gcs_autoscaler_state_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc @@ -24,8 +24,8 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/test/gcs_server_test_util.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/gcs/gcs_server/store_client_kv.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" #include "mock/ray/gcs/gcs_server/gcs_placement_group_mgr.h" diff --git a/src/ray/gcs/gcs_server/test/gcs_function_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc similarity index 100% rename from src/ray/gcs/gcs_server/test/gcs_function_manager_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc diff --git a/src/ray/gcs/gcs_server/test/gcs_health_check_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc similarity index 100% rename from src/ray/gcs/gcs_server/test/gcs_health_check_manager_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc diff --git a/src/ray/gcs/gcs_server/test/gcs_job_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_job_manager_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc index eab0a5333a08..f21a5f4a926c 100644 --- a/src/ray/gcs/gcs_server/test/gcs_job_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc @@ -19,9 +19,9 @@ // clang-format off #include "gtest/gtest.h" -#include "ray/gcs/gcs_server/test/gcs_server_test_util.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/pubsub/publisher.h" diff --git a/src/ray/gcs/gcs_server/test/gcs_kv_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc similarity index 100% rename from src/ray/gcs/gcs_server/test/gcs_kv_manager_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc diff --git a/src/ray/gcs/gcs_server/test/gcs_node_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc similarity index 98% rename from src/ray/gcs/gcs_server/test/gcs_node_manager_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc index 69bb1aee77d3..ced40b4863e1 100644 --- a/src/ray/gcs/gcs_server/test/gcs_node_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc @@ -18,8 +18,8 @@ // clang-format off #include "gtest/gtest.h" -#include "ray/gcs/gcs_server/test/gcs_server_test_util.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/rpc/node_manager/node_manager_client.h" #include "ray/rpc/node_manager/raylet_client_pool.h" #include "mock/ray/pubsub/publisher.h" diff --git a/src/ray/gcs/gcs_server/test/gcs_placement_group_mgr_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_mock_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_placement_group_mgr_mock_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_mock_test.cc index c02924582618..643d4f454112 100644 --- a/src/ray/gcs/gcs_server/test/gcs_placement_group_mgr_mock_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_mock_test.cc @@ -25,7 +25,7 @@ #include "mock/ray/gcs/gcs_server/gcs_resource_manager.h" #include "mock/ray/gcs/store_client/store_client.h" #include "ray/util/counter_map.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" // clang-format on using namespace ::testing; // NOLINT diff --git a/src/ray/gcs/gcs_server/test/gcs_placement_group_mgr_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_placement_group_mgr_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc index 660c2c320363..853684b7fd85 100644 --- a/src/ray/gcs/gcs_server/test/gcs_placement_group_mgr_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc @@ -21,8 +21,8 @@ // clang-format off #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/test/gcs_server_test_util.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" #include "ray/util/counter_map.h" #include "mock/ray/pubsub/publisher.h" diff --git a/src/ray/gcs/gcs_server/test/gcs_placement_group_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_placement_group_scheduler_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc index 0e1d30a5a357..26d1a9ea6226 100644 --- a/src/ray/gcs/gcs_server/test/gcs_placement_group_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc @@ -21,8 +21,8 @@ // clang-format off #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/test/gcs_server_test_util.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/util/counter_map.h" #include "mock/ray/pubsub/publisher.h" diff --git a/src/ray/gcs/gcs_server/test/gcs_resource_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_resource_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_resource_manager_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_resource_manager_test.cc index 23591b264573..8a417a77363f 100644 --- a/src/ray/gcs/gcs_server/test/gcs_resource_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_resource_manager_test.cc @@ -21,7 +21,7 @@ #include "gtest/gtest.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/test/gcs_server_rpc_test.cc b/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_server_rpc_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc index 7929c693f933..81798c28be7d 100644 --- a/src/ray/gcs/gcs_server/test/gcs_server_rpc_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc @@ -20,7 +20,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/ray_config.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/rpc/gcs/gcs_rpc_client.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/test/gcs_server_test_util.h b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h similarity index 100% rename from src/ray/gcs/gcs_server/test/gcs_server_test_util.h rename to src/ray/gcs/gcs_server/tests/gcs_server_test_util.h diff --git a/src/ray/gcs/gcs_server/test/gcs_table_storage_test_base.h b/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_table_storage_test_base.h rename to src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h index 5252b6a99eec..8e628085e38f 100644 --- a/src/ray/gcs/gcs_server/test/gcs_table_storage_test_base.h +++ b/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h @@ -21,7 +21,7 @@ #include "ray/common/id.h" #include "ray/common/test_util.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/test/gcs_task_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_task_manager_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc index 8349ea04eb22..efb4c30b9a49 100644 --- a/src/ray/gcs/gcs_server/test/gcs_task_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc @@ -27,7 +27,7 @@ #include "ray/common/id.h" #include "ray/common/status.h" #include "ray/gcs/pb_util.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/test/gcs_worker_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/test/gcs_worker_manager_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc index 5607919bfb8e..d1e5f47b90a5 100644 --- a/src/ray/gcs/gcs_server/test/gcs_worker_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc @@ -23,8 +23,8 @@ // clang-format off #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/test/gcs_server_test_util.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "mock/ray/pubsub/publisher.h" #include "src/ray/protobuf/gcs.pb.h" #include "src/ray/protobuf/common.pb.h" diff --git a/src/ray/gcs/gcs_server/test/in_memory_gcs_table_storage_test.cc b/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc similarity index 94% rename from src/ray/gcs/gcs_server/test/in_memory_gcs_table_storage_test.cc rename to src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc index 9142d119b9bb..5bd694e52d7f 100644 --- a/src/ray/gcs/gcs_server/test/in_memory_gcs_table_storage_test.cc +++ b/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc @@ -18,7 +18,7 @@ #include "ray/common/test_util.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/gcs_server/test/gcs_table_storage_test_base.h" +#include "ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h" #include "ray/gcs/store_client/in_memory_store_client.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/test/redis_gcs_table_storage_test.cc b/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc similarity index 96% rename from src/ray/gcs/gcs_server/test/redis_gcs_table_storage_test.cc rename to src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc index 568db9638f11..591149ec9ffa 100644 --- a/src/ray/gcs/gcs_server/test/redis_gcs_table_storage_test.cc +++ b/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc @@ -17,7 +17,7 @@ #include "gtest/gtest.h" #include "ray/common/test_util.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/gcs_server/test/gcs_table_storage_test_base.h" +#include "ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h" #include "ray/gcs/store_client/redis_store_client.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/test/usage_stats_client_test.cc b/src/ray/gcs/gcs_server/tests/usage_stats_client_test.cc similarity index 100% rename from src/ray/gcs/gcs_server/test/usage_stats_client_test.cc rename to src/ray/gcs/gcs_server/tests/usage_stats_client_test.cc diff --git a/src/ray/gcs/store_client/test/BUILD.bazel b/src/ray/gcs/store_client/tests/BUILD.bazel similarity index 100% rename from src/ray/gcs/store_client/test/BUILD.bazel rename to src/ray/gcs/store_client/tests/BUILD.bazel diff --git a/src/ray/gcs/store_client/test/in_memory_store_client_test.cc b/src/ray/gcs/store_client/tests/in_memory_store_client_test.cc similarity index 95% rename from src/ray/gcs/store_client/test/in_memory_store_client_test.cc rename to src/ray/gcs/store_client/tests/in_memory_store_client_test.cc index c5f8e81ec284..0fa5a47116f6 100644 --- a/src/ray/gcs/store_client/test/in_memory_store_client_test.cc +++ b/src/ray/gcs/store_client/tests/in_memory_store_client_test.cc @@ -16,7 +16,7 @@ #include -#include "ray/gcs/store_client/test/store_client_test_base.h" +#include "ray/gcs/store_client/tests/store_client_test_base.h" namespace ray { diff --git a/src/ray/gcs/store_client/test/observable_store_client_test.cc b/src/ray/gcs/store_client/tests/observable_store_client_test.cc similarity index 95% rename from src/ray/gcs/store_client/test/observable_store_client_test.cc rename to src/ray/gcs/store_client/tests/observable_store_client_test.cc index 17a1f751aeb9..39f084b522af 100644 --- a/src/ray/gcs/store_client/test/observable_store_client_test.cc +++ b/src/ray/gcs/store_client/tests/observable_store_client_test.cc @@ -17,7 +17,7 @@ #include #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/store_client/test/store_client_test_base.h" +#include "ray/gcs/store_client/tests/store_client_test_base.h" namespace ray { diff --git a/src/ray/gcs/store_client/test/redis_store_client_test.cc b/src/ray/gcs/store_client/tests/redis_store_client_test.cc similarity index 99% rename from src/ray/gcs/store_client/test/redis_store_client_test.cc rename to src/ray/gcs/store_client/tests/redis_store_client_test.cc index b473b3167b91..dcf53b4f6f26 100644 --- a/src/ray/gcs/store_client/test/redis_store_client_test.cc +++ b/src/ray/gcs/store_client/tests/redis_store_client_test.cc @@ -24,7 +24,7 @@ #include "ray/common/test_util.h" #include "ray/gcs/redis_client.h" -#include "ray/gcs/store_client/test/store_client_test_base.h" +#include "ray/gcs/store_client/tests/store_client_test_base.h" #include "ray/util/path_utils.h" using namespace std::chrono_literals; // NOLINT diff --git a/src/ray/gcs/store_client/test/store_client_test_base.h b/src/ray/gcs/store_client/tests/store_client_test_base.h similarity index 100% rename from src/ray/gcs/store_client/test/store_client_test_base.h rename to src/ray/gcs/store_client/tests/store_client_test_base.h diff --git a/src/ray/gcs/test/BUILD.bazel b/src/ray/gcs/tests/BUILD.bazel similarity index 100% rename from src/ray/gcs/test/BUILD.bazel rename to src/ray/gcs/tests/BUILD.bazel diff --git a/src/ray/gcs/test/callback_reply_test.cc b/src/ray/gcs/tests/callback_reply_test.cc similarity index 100% rename from src/ray/gcs/test/callback_reply_test.cc rename to src/ray/gcs/tests/callback_reply_test.cc diff --git a/src/ray/gcs/test/gcs_test_util.h b/src/ray/gcs/tests/gcs_test_util.h similarity index 100% rename from src/ray/gcs/test/gcs_test_util.h rename to src/ray/gcs/tests/gcs_test_util.h diff --git a/src/ray/gcs/test/redis_async_context_test.cc b/src/ray/gcs/tests/redis_async_context_test.cc similarity index 100% rename from src/ray/gcs/test/redis_async_context_test.cc rename to src/ray/gcs/tests/redis_async_context_test.cc diff --git a/src/ray/ipc/test/BUILD.bazel b/src/ray/ipc/tests/BUILD.bazel similarity index 100% rename from src/ray/ipc/test/BUILD.bazel rename to src/ray/ipc/tests/BUILD.bazel diff --git a/src/ray/ipc/test/client_connection_test.cc b/src/ray/ipc/tests/client_connection_test.cc similarity index 100% rename from src/ray/ipc/test/client_connection_test.cc rename to src/ray/ipc/tests/client_connection_test.cc diff --git a/src/ray/object_manager/plasma/test/BUILD.bazel b/src/ray/object_manager/plasma/tests/BUILD.bazel similarity index 100% rename from src/ray/object_manager/plasma/test/BUILD.bazel rename to src/ray/object_manager/plasma/tests/BUILD.bazel diff --git a/src/ray/object_manager/plasma/test/eviction_policy_test.cc b/src/ray/object_manager/plasma/tests/eviction_policy_test.cc similarity index 100% rename from src/ray/object_manager/plasma/test/eviction_policy_test.cc rename to src/ray/object_manager/plasma/tests/eviction_policy_test.cc diff --git a/src/ray/object_manager/plasma/test/fallback_allocator_test.cc b/src/ray/object_manager/plasma/tests/fallback_allocator_test.cc similarity index 100% rename from src/ray/object_manager/plasma/test/fallback_allocator_test.cc rename to src/ray/object_manager/plasma/tests/fallback_allocator_test.cc diff --git a/src/ray/object_manager/plasma/test/mutable_object_test.cc b/src/ray/object_manager/plasma/tests/mutable_object_test.cc similarity index 100% rename from src/ray/object_manager/plasma/test/mutable_object_test.cc rename to src/ray/object_manager/plasma/tests/mutable_object_test.cc diff --git a/src/ray/object_manager/plasma/test/obj_lifecycle_mgr_test.cc b/src/ray/object_manager/plasma/tests/obj_lifecycle_mgr_test.cc similarity index 100% rename from src/ray/object_manager/plasma/test/obj_lifecycle_mgr_test.cc rename to src/ray/object_manager/plasma/tests/obj_lifecycle_mgr_test.cc diff --git a/src/ray/object_manager/plasma/test/object_store_test.cc b/src/ray/object_manager/plasma/tests/object_store_test.cc similarity index 100% rename from src/ray/object_manager/plasma/test/object_store_test.cc rename to src/ray/object_manager/plasma/tests/object_store_test.cc diff --git a/src/ray/object_manager/plasma/test/stats_collector_test.cc b/src/ray/object_manager/plasma/tests/stats_collector_test.cc similarity index 100% rename from src/ray/object_manager/plasma/test/stats_collector_test.cc rename to src/ray/object_manager/plasma/tests/stats_collector_test.cc diff --git a/src/ray/object_manager/test/BUILD.bazel b/src/ray/object_manager/tests/BUILD.bazel similarity index 100% rename from src/ray/object_manager/test/BUILD.bazel rename to src/ray/object_manager/tests/BUILD.bazel diff --git a/src/ray/object_manager/test/create_request_queue_test.cc b/src/ray/object_manager/tests/create_request_queue_test.cc similarity index 100% rename from src/ray/object_manager/test/create_request_queue_test.cc rename to src/ray/object_manager/tests/create_request_queue_test.cc diff --git a/src/ray/object_manager/test/get_request_queue_test.cc b/src/ray/object_manager/tests/get_request_queue_test.cc similarity index 100% rename from src/ray/object_manager/test/get_request_queue_test.cc rename to src/ray/object_manager/tests/get_request_queue_test.cc diff --git a/src/ray/object_manager/test/object_buffer_pool_test.cc b/src/ray/object_manager/tests/object_buffer_pool_test.cc similarity index 100% rename from src/ray/object_manager/test/object_buffer_pool_test.cc rename to src/ray/object_manager/tests/object_buffer_pool_test.cc diff --git a/src/ray/object_manager/test/ownership_object_directory_test.cc b/src/ray/object_manager/tests/ownership_object_directory_test.cc similarity index 100% rename from src/ray/object_manager/test/ownership_object_directory_test.cc rename to src/ray/object_manager/tests/ownership_object_directory_test.cc diff --git a/src/ray/object_manager/test/pull_manager_test.cc b/src/ray/object_manager/tests/pull_manager_test.cc similarity index 100% rename from src/ray/object_manager/test/pull_manager_test.cc rename to src/ray/object_manager/tests/pull_manager_test.cc diff --git a/src/ray/object_manager/test/push_manager_test.cc b/src/ray/object_manager/tests/push_manager_test.cc similarity index 100% rename from src/ray/object_manager/test/push_manager_test.cc rename to src/ray/object_manager/tests/push_manager_test.cc diff --git a/src/ray/object_manager/test/spilled_object_test.cc b/src/ray/object_manager/tests/spilled_object_test.cc similarity index 100% rename from src/ray/object_manager/test/spilled_object_test.cc rename to src/ray/object_manager/tests/spilled_object_test.cc diff --git a/src/ray/pubsub/test/BUILD.bazel b/src/ray/pubsub/tests/BUILD.bazel similarity index 100% rename from src/ray/pubsub/test/BUILD.bazel rename to src/ray/pubsub/tests/BUILD.bazel diff --git a/src/ray/pubsub/test/integration_test.cc b/src/ray/pubsub/tests/integration_test.cc similarity index 100% rename from src/ray/pubsub/test/integration_test.cc rename to src/ray/pubsub/tests/integration_test.cc diff --git a/src/ray/pubsub/test/publisher_test.cc b/src/ray/pubsub/tests/publisher_test.cc similarity index 100% rename from src/ray/pubsub/test/publisher_test.cc rename to src/ray/pubsub/tests/publisher_test.cc diff --git a/src/ray/pubsub/test/subscriber_test.cc b/src/ray/pubsub/tests/subscriber_test.cc similarity index 100% rename from src/ray/pubsub/test/subscriber_test.cc rename to src/ray/pubsub/tests/subscriber_test.cc diff --git a/src/ray/raylet/scheduling/BUILD.bazel b/src/ray/raylet/scheduling/BUILD.bazel index 8d7c28b6dd77..eef39d32e4e5 100644 --- a/src/ray/raylet/scheduling/BUILD.bazel +++ b/src/ray/raylet/scheduling/BUILD.bazel @@ -1,4 +1,4 @@ -load("//bazel:ray.bzl", "ray_cc_library", "ray_cc_test") +load("//bazel:ray.bzl", "ray_cc_library") ray_cc_library( name = "scheduler", @@ -269,110 +269,3 @@ ray_cc_library( "//src/ray/common:task_common", ], ) - -ray_cc_test( - name = "cluster_resource_scheduler_test", - size = "small", - srcs = [ - "cluster_resource_scheduler_test.cc", - ], - tags = ["team:core"], - deps = [ - ":cluster_resource_scheduler", - "//:ray_mock", - "//src/ray/common:ray_config", - "//src/ray/common:task_common", - "//src/ray/common:test_util", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "cluster_resource_scheduler_2_test", - size = "small", - srcs = [ - "cluster_resource_scheduler_2_test.cc", - ], - tags = ["team:core"], - deps = [ - ":cluster_resource_scheduler", - ":scheduling_context", - ":scheduling_options", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "local_resource_manager_test", - size = "small", - srcs = [ - "local_resource_manager_test.cc", - ], - tags = ["team:core"], - deps = [ - ":local_resource_manager", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "scheduling_policy_test", - size = "small", - srcs = [ - "policy/scheduling_policy_test.cc", - ], - tags = ["team:core"], - deps = [ - ":composite_scheduling_policy", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "hybrid_scheduling_policy_test", - size = "small", - srcs = [ - "policy/hybrid_scheduling_policy_test.cc", - ], - tags = ["team:core"], - deps = [ - ":composite_scheduling_policy", - ":hybrid_scheduling_policy", - "@com_google_absl//absl/random:mock_distributions", - "@com_google_absl//absl/random:mocking_bit_gen", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "cluster_task_manager_test", - size = "small", - srcs = [ - "cluster_task_manager_test.cc", - ], - tags = ["team:core"], - deps = [ - ":cluster_resource_scheduler", - ":cluster_task_manager", - "//:ray_mock", - "//src/ray/common:id", - "//src/ray/common:task_common", - "//src/ray/common:test_util", - "//src/ray/raylet:local_task_manager", - "//src/ray/raylet/test:util", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "cluster_resource_manager_test", - size = "small", - srcs = [ - "cluster_resource_manager_test.cc", - ], - tags = ["team:core"], - deps = [ - ":cluster_resource_manager", - "@com_google_googletest//:gtest_main", - ], -) diff --git a/src/ray/raylet/scheduling/policy/tests/BUILD.bazel b/src/ray/raylet/scheduling/policy/tests/BUILD.bazel new file mode 100644 index 000000000000..a9ee6d460cd0 --- /dev/null +++ b/src/ray/raylet/scheduling/policy/tests/BUILD.bazel @@ -0,0 +1,30 @@ +load("//bazel:ray.bzl", "ray_cc_test") + +ray_cc_test( + name = "scheduling_policy_test", + size = "small", + srcs = [ + "scheduling_policy_test.cc", + ], + tags = ["team:core"], + deps = [ + "//src/ray/raylet/scheduling:composite_scheduling_policy", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "hybrid_scheduling_policy_test", + size = "small", + srcs = [ + "hybrid_scheduling_policy_test.cc", + ], + tags = ["team:core"], + deps = [ + "//src/ray/raylet/scheduling:composite_scheduling_policy", + "//src/ray/raylet/scheduling:hybrid_scheduling_policy", + "@com_google_absl//absl/random:mock_distributions", + "@com_google_absl//absl/random:mocking_bit_gen", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/raylet/scheduling/policy/hybrid_scheduling_policy_test.cc b/src/ray/raylet/scheduling/policy/tests/hybrid_scheduling_policy_test.cc similarity index 100% rename from src/ray/raylet/scheduling/policy/hybrid_scheduling_policy_test.cc rename to src/ray/raylet/scheduling/policy/tests/hybrid_scheduling_policy_test.cc diff --git a/src/ray/raylet/scheduling/policy/scheduling_policy_test.cc b/src/ray/raylet/scheduling/policy/tests/scheduling_policy_test.cc similarity index 100% rename from src/ray/raylet/scheduling/policy/scheduling_policy_test.cc rename to src/ray/raylet/scheduling/policy/tests/scheduling_policy_test.cc diff --git a/src/ray/raylet/scheduling/tests/BUILD.bazel b/src/ray/raylet/scheduling/tests/BUILD.bazel new file mode 100644 index 000000000000..1e3d0cdae67f --- /dev/null +++ b/src/ray/raylet/scheduling/tests/BUILD.bazel @@ -0,0 +1,79 @@ +load("//bazel:ray.bzl", "ray_cc_test") + +ray_cc_test( + name = "cluster_resource_scheduler_test", + size = "small", + srcs = [ + "cluster_resource_scheduler_test.cc", + ], + tags = ["team:core"], + deps = [ + "//:ray_mock", + "//src/ray/common:ray_config", + "//src/ray/common:task_common", + "//src/ray/common:test_util", + "//src/ray/raylet/scheduling:cluster_resource_scheduler", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "cluster_resource_scheduler_2_test", + size = "small", + srcs = [ + "cluster_resource_scheduler_2_test.cc", + ], + tags = ["team:core"], + deps = [ + "//src/ray/raylet/scheduling:cluster_resource_scheduler", + "//src/ray/raylet/scheduling:scheduling_context", + "//src/ray/raylet/scheduling:scheduling_options", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "local_resource_manager_test", + size = "small", + srcs = [ + "local_resource_manager_test.cc", + ], + tags = ["team:core"], + deps = [ + "//src/ray/raylet/scheduling:local_resource_manager", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "cluster_task_manager_test", + size = "small", + srcs = [ + "cluster_task_manager_test.cc", + ], + tags = ["team:core"], + deps = [ + "//:ray_mock", + "//src/ray/common:id", + "//src/ray/common:task_common", + "//src/ray/common:test_util", + "//src/ray/raylet:local_task_manager", + "//src/ray/raylet/scheduling:cluster_resource_scheduler", + "//src/ray/raylet/scheduling:cluster_task_manager", + "//src/ray/raylet/tests:util", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "cluster_resource_manager_test", + size = "small", + srcs = [ + "cluster_resource_manager_test.cc", + ], + tags = ["team:core"], + deps = [ + "//src/ray/raylet/scheduling:cluster_resource_manager", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/raylet/scheduling/cluster_resource_manager_test.cc b/src/ray/raylet/scheduling/tests/cluster_resource_manager_test.cc similarity index 100% rename from src/ray/raylet/scheduling/cluster_resource_manager_test.cc rename to src/ray/raylet/scheduling/tests/cluster_resource_manager_test.cc diff --git a/src/ray/raylet/scheduling/cluster_resource_scheduler_2_test.cc b/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_2_test.cc similarity index 100% rename from src/ray/raylet/scheduling/cluster_resource_scheduler_2_test.cc rename to src/ray/raylet/scheduling/tests/cluster_resource_scheduler_2_test.cc diff --git a/src/ray/raylet/scheduling/cluster_resource_scheduler_test.cc b/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc similarity index 100% rename from src/ray/raylet/scheduling/cluster_resource_scheduler_test.cc rename to src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc diff --git a/src/ray/raylet/scheduling/cluster_task_manager_test.cc b/src/ray/raylet/scheduling/tests/cluster_task_manager_test.cc similarity index 99% rename from src/ray/raylet/scheduling/cluster_task_manager_test.cc rename to src/ray/raylet/scheduling/tests/cluster_task_manager_test.cc index 88fa3a320dc1..9ce9ccaca635 100644 --- a/src/ray/raylet/scheduling/cluster_task_manager_test.cc +++ b/src/ray/raylet/scheduling/tests/cluster_task_manager_test.cc @@ -34,7 +34,7 @@ #include "ray/common/test_util.h" #include "ray/raylet/local_task_manager.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" -#include "ray/raylet/test/util.h" +#include "ray/raylet/tests/util.h" #include "mock/ray/gcs/gcs_client/gcs_client.h" // clang-format on diff --git a/src/ray/raylet/scheduling/local_resource_manager_test.cc b/src/ray/raylet/scheduling/tests/local_resource_manager_test.cc similarity index 100% rename from src/ray/raylet/scheduling/local_resource_manager_test.cc rename to src/ray/raylet/scheduling/tests/local_resource_manager_test.cc diff --git a/src/ray/raylet/test/BUILD.bazel b/src/ray/raylet/tests/BUILD.bazel similarity index 99% rename from src/ray/raylet/test/BUILD.bazel rename to src/ray/raylet/tests/BUILD.bazel index abd8e45e5e4f..71350f2d30ad 100644 --- a/src/ray/raylet/test/BUILD.bazel +++ b/src/ray/raylet/tests/BUILD.bazel @@ -70,7 +70,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:id", "//src/ray/common:task_common", - "//src/ray/gcs/test:gcs_test_util_lib", + "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/raylet:placement_group_resource_manager", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/raylet/test/dependency_manager_test.cc b/src/ray/raylet/tests/dependency_manager_test.cc similarity index 100% rename from src/ray/raylet/test/dependency_manager_test.cc rename to src/ray/raylet/tests/dependency_manager_test.cc diff --git a/src/ray/raylet/test/local_object_manager_test.cc b/src/ray/raylet/tests/local_object_manager_test.cc similarity index 99% rename from src/ray/raylet/test/local_object_manager_test.cc rename to src/ray/raylet/tests/local_object_manager_test.cc index d98fb7dc834d..3688518efa90 100644 --- a/src/ray/raylet/test/local_object_manager_test.cc +++ b/src/ray/raylet/tests/local_object_manager_test.cc @@ -31,7 +31,7 @@ #include "ray/gcs/gcs_client/accessor.h" #include "ray/object_manager/ownership_object_directory.h" #include "ray/pubsub/subscriber.h" -#include "ray/raylet/test/util.h" +#include "ray/raylet/tests/util.h" #include "ray/raylet/worker_pool.h" #include "ray/rpc/grpc_client.h" #include "ray/rpc/worker/core_worker_client.h" diff --git a/src/ray/raylet/test/local_task_manager_test.cc b/src/ray/raylet/tests/local_task_manager_test.cc similarity index 99% rename from src/ray/raylet/test/local_task_manager_test.cc rename to src/ray/raylet/tests/local_task_manager_test.cc index fe505a303340..0c3dcac90442 100644 --- a/src/ray/raylet/test/local_task_manager_test.cc +++ b/src/ray/raylet/tests/local_task_manager_test.cc @@ -32,7 +32,7 @@ #include "ray/common/task/task_util.h" #include "ray/common/test_util.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" -#include "ray/raylet/test/util.h" +#include "ray/raylet/tests/util.h" namespace ray::raylet { diff --git a/src/ray/raylet/test/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc similarity index 99% rename from src/ray/raylet/test/node_manager_test.cc rename to src/ray/raylet/tests/node_manager_test.cc index 3ef9df384f98..55e4d75a1468 100644 --- a/src/ray/raylet/test/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -36,7 +36,7 @@ #include "ray/object_manager/plasma/client.h" #include "ray/raylet/local_object_manager_interface.h" #include "ray/raylet/scheduling/cluster_task_manager.h" -#include "ray/raylet/test/util.h" +#include "ray/raylet/tests/util.h" namespace ray::raylet { using ::testing::_; diff --git a/src/ray/raylet/test/placement_group_resource_manager_test.cc b/src/ray/raylet/tests/placement_group_resource_manager_test.cc similarity index 99% rename from src/ray/raylet/test/placement_group_resource_manager_test.cc rename to src/ray/raylet/tests/placement_group_resource_manager_test.cc index 0e1530f74c63..9247c75ffb7c 100644 --- a/src/ray/raylet/test/placement_group_resource_manager_test.cc +++ b/src/ray/raylet/tests/placement_group_resource_manager_test.cc @@ -24,7 +24,7 @@ #include "ray/common/bundle_spec.h" #include "ray/common/id.h" #include "ray/common/scheduling/resource_set.h" -#include "ray/gcs/test/gcs_test_util.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "mock/ray/gcs/gcs_client/gcs_client.h" // clang-format on diff --git a/src/ray/raylet/test/runtime_env_agent_client_test.cc b/src/ray/raylet/tests/runtime_env_agent_client_test.cc similarity index 100% rename from src/ray/raylet/test/runtime_env_agent_client_test.cc rename to src/ray/raylet/tests/runtime_env_agent_client_test.cc diff --git a/src/ray/raylet/test/util.h b/src/ray/raylet/tests/util.h similarity index 100% rename from src/ray/raylet/test/util.h rename to src/ray/raylet/tests/util.h diff --git a/src/ray/raylet/test/wait_manager_test.cc b/src/ray/raylet/tests/wait_manager_test.cc similarity index 100% rename from src/ray/raylet/test/wait_manager_test.cc rename to src/ray/raylet/tests/wait_manager_test.cc diff --git a/src/ray/raylet/test/worker_killing_policy_group_by_owner_test.cc b/src/ray/raylet/tests/worker_killing_policy_group_by_owner_test.cc similarity index 99% rename from src/ray/raylet/test/worker_killing_policy_group_by_owner_test.cc rename to src/ray/raylet/tests/worker_killing_policy_group_by_owner_test.cc index 7328ae41e2ef..965712213786 100644 --- a/src/ray/raylet/test/worker_killing_policy_group_by_owner_test.cc +++ b/src/ray/raylet/tests/worker_killing_policy_group_by_owner_test.cc @@ -21,7 +21,7 @@ #include "gtest/gtest.h" #include "ray/common/task/task_spec.h" -#include "ray/raylet/test/util.h" +#include "ray/raylet/tests/util.h" #include "ray/raylet/worker_killing_policy.h" namespace ray { diff --git a/src/ray/raylet/test/worker_killing_policy_retriable_fifo_test.cc b/src/ray/raylet/tests/worker_killing_policy_retriable_fifo_test.cc similarity index 99% rename from src/ray/raylet/test/worker_killing_policy_retriable_fifo_test.cc rename to src/ray/raylet/tests/worker_killing_policy_retriable_fifo_test.cc index 0c512233fc7b..27d07b7e5417 100644 --- a/src/ray/raylet/test/worker_killing_policy_retriable_fifo_test.cc +++ b/src/ray/raylet/tests/worker_killing_policy_retriable_fifo_test.cc @@ -19,7 +19,7 @@ #include "gtest/gtest.h" #include "ray/common/task/task_spec.h" -#include "ray/raylet/test/util.h" +#include "ray/raylet/tests/util.h" #include "ray/raylet/worker_killing_policy.h" namespace ray { diff --git a/src/ray/raylet/test/worker_killing_policy_test.cc b/src/ray/raylet/tests/worker_killing_policy_test.cc similarity index 99% rename from src/ray/raylet/test/worker_killing_policy_test.cc rename to src/ray/raylet/tests/worker_killing_policy_test.cc index c9c0ef5ed572..1026b616c09d 100644 --- a/src/ray/raylet/test/worker_killing_policy_test.cc +++ b/src/ray/raylet/tests/worker_killing_policy_test.cc @@ -19,7 +19,7 @@ #include "gtest/gtest.h" #include "ray/common/task/task_spec.h" -#include "ray/raylet/test/util.h" +#include "ray/raylet/tests/util.h" namespace ray { diff --git a/src/ray/raylet/test/worker_pool_test.cc b/src/ray/raylet/tests/worker_pool_test.cc similarity index 100% rename from src/ray/raylet/test/worker_pool_test.cc rename to src/ray/raylet/tests/worker_pool_test.cc diff --git a/src/ray/rpc/node_manager/test/BUILD.bazel b/src/ray/rpc/node_manager/tests/BUILD.bazel similarity index 100% rename from src/ray/rpc/node_manager/test/BUILD.bazel rename to src/ray/rpc/node_manager/tests/BUILD.bazel diff --git a/src/ray/rpc/node_manager/test/raylet_client_pool_test.cc b/src/ray/rpc/node_manager/tests/raylet_client_pool_test.cc similarity index 100% rename from src/ray/rpc/node_manager/test/raylet_client_pool_test.cc rename to src/ray/rpc/node_manager/tests/raylet_client_pool_test.cc diff --git a/src/ray/rpc/test/BUILD.bazel b/src/ray/rpc/tests/BUILD.bazel similarity index 100% rename from src/ray/rpc/test/BUILD.bazel rename to src/ray/rpc/tests/BUILD.bazel diff --git a/src/ray/rpc/test/core_worker_client_pool_test.cc b/src/ray/rpc/tests/core_worker_client_pool_test.cc similarity index 100% rename from src/ray/rpc/test/core_worker_client_pool_test.cc rename to src/ray/rpc/tests/core_worker_client_pool_test.cc diff --git a/src/ray/rpc/test/grpc_bench/BUILD.bazel b/src/ray/rpc/tests/grpc_bench/BUILD.bazel similarity index 100% rename from src/ray/rpc/test/grpc_bench/BUILD.bazel rename to src/ray/rpc/tests/grpc_bench/BUILD.bazel diff --git a/src/ray/rpc/test/grpc_bench/Dockerfile b/src/ray/rpc/tests/grpc_bench/Dockerfile similarity index 100% rename from src/ray/rpc/test/grpc_bench/Dockerfile rename to src/ray/rpc/tests/grpc_bench/Dockerfile diff --git a/src/ray/rpc/test/grpc_bench/README b/src/ray/rpc/tests/grpc_bench/README similarity index 100% rename from src/ray/rpc/test/grpc_bench/README rename to src/ray/rpc/tests/grpc_bench/README diff --git a/src/ray/rpc/test/grpc_bench/grpc_bench.cc b/src/ray/rpc/tests/grpc_bench/grpc_bench.cc similarity index 100% rename from src/ray/rpc/test/grpc_bench/grpc_bench.cc rename to src/ray/rpc/tests/grpc_bench/grpc_bench.cc diff --git a/src/ray/rpc/test/grpc_bench/helloworld.proto b/src/ray/rpc/tests/grpc_bench/helloworld.proto similarity index 100% rename from src/ray/rpc/test/grpc_bench/helloworld.proto rename to src/ray/rpc/tests/grpc_bench/helloworld.proto diff --git a/src/ray/rpc/test/grpc_server_client_test.cc b/src/ray/rpc/tests/grpc_server_client_test.cc similarity index 100% rename from src/ray/rpc/test/grpc_server_client_test.cc rename to src/ray/rpc/tests/grpc_server_client_test.cc diff --git a/src/ray/rpc/test/rpc_chaos_test.cc b/src/ray/rpc/tests/rpc_chaos_test.cc similarity index 100% rename from src/ray/rpc/test/rpc_chaos_test.cc rename to src/ray/rpc/tests/rpc_chaos_test.cc diff --git a/src/ray/stats/BUILD.bazel b/src/ray/stats/BUILD.bazel index 4a088305a22b..05496b27ae4f 100644 --- a/src/ray/stats/BUILD.bazel +++ b/src/ray/stats/BUILD.bazel @@ -1,4 +1,4 @@ -load("//bazel:ray.bzl", "ray_cc_library", "ray_cc_test") +load("//bazel:ray.bzl", "ray_cc_library") ray_cc_library( name = "stats_metric", @@ -55,34 +55,3 @@ ray_cc_library( "@com_github_grpc_grpc//:grpc_opencensus_plugin", ], ) - -ray_cc_test( - name = "stats_test", - size = "small", - srcs = ["stats_test.cc"], - tags = [ - "no_tsan", - "stats", - "team:core", - ], - deps = [ - ":stats_lib", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "metric_exporter_grpc_test", - size = "small", - srcs = [ - "metric_exporter_grpc_test.cc", - ], - tags = [ - "stats", - "team:core", - ], - deps = [ - ":stats_lib", - "@com_google_googletest//:gtest_main", - ], -) diff --git a/src/ray/stats/tests/BUILD.bazel b/src/ray/stats/tests/BUILD.bazel index 90bfa18328a8..ac78d6de4d00 100644 --- a/src/ray/stats/tests/BUILD.bazel +++ b/src/ray/stats/tests/BUILD.bazel @@ -13,3 +13,34 @@ ray_cc_test( "@com_google_googletest//:gtest_main", ], ) + +ray_cc_test( + name = "stats_test", + size = "small", + srcs = ["stats_test.cc"], + tags = [ + "no_tsan", + "stats", + "team:core", + ], + deps = [ + "//src/ray/stats:stats_lib", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "metric_exporter_grpc_test", + size = "small", + srcs = [ + "metric_exporter_grpc_test.cc", + ], + tags = [ + "stats", + "team:core", + ], + deps = [ + "//src/ray/stats:stats_lib", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/stats/metric_exporter_grpc_test.cc b/src/ray/stats/tests/metric_exporter_grpc_test.cc similarity index 100% rename from src/ray/stats/metric_exporter_grpc_test.cc rename to src/ray/stats/tests/metric_exporter_grpc_test.cc diff --git a/src/ray/stats/stats_test.cc b/src/ray/stats/tests/stats_test.cc similarity index 100% rename from src/ray/stats/stats_test.cc rename to src/ray/stats/tests/stats_test.cc diff --git a/src/ray/util/internal/tests/BUILD.bazel b/src/ray/util/internal/tests/BUILD.bazel index bff2005719a1..59c88140efc6 100644 --- a/src/ray/util/internal/tests/BUILD.bazel +++ b/src/ray/util/internal/tests/BUILD.bazel @@ -12,7 +12,7 @@ ray_cc_test( "no_tsan", ], deps = [ - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "//src/ray/util", "//src/ray/util/internal:stream_redirection_handle", "@com_google_googletest//:gtest_main", diff --git a/src/ray/util/internal/tests/stream_redirection_handle_test.cc b/src/ray/util/internal/tests/stream_redirection_handle_test.cc index a4ac9e87aed4..fd963af8d1c1 100644 --- a/src/ray/util/internal/tests/stream_redirection_handle_test.cc +++ b/src/ray/util/internal/tests/stream_redirection_handle_test.cc @@ -22,7 +22,7 @@ #include #include -#include "ray/common/test/testing.h" +#include "ray/common/tests/testing.h" #include "ray/util/filesystem.h" #include "ray/util/util.h" diff --git a/src/ray/util/tests/BUILD.bazel b/src/ray/util/tests/BUILD.bazel index 443b97190955..f9d38609354c 100644 --- a/src/ray/util/tests/BUILD.bazel +++ b/src/ray/util/tests/BUILD.bazel @@ -254,7 +254,7 @@ ray_cc_test( srcs = ["pipe_logger_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "//src/ray/util", "//src/ray/util:pipe_logger", "//src/ray/util:scoped_env_setter", @@ -276,7 +276,7 @@ ray_cc_test( "no_tsan", ], deps = [ - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "//src/ray/util", "//src/ray/util:stream_redirection", "@com_google_googletest//:gtest_main", @@ -312,7 +312,7 @@ ray_cc_test( srcs = ["spdlog_newliner_sink_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "//src/ray/util:filesystem", "//src/ray/util:spdlog_fd_sink", "//src/ray/util:spdlog_newliner_sink", @@ -349,7 +349,7 @@ ray_cc_test( srcs = ["process_cleanup_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "//src/ray/util", "//src/ray/util:filesystem", "//src/ray/util:process_cleaner", @@ -363,7 +363,7 @@ ray_cc_test( srcs = ["scoped_dup2_wrapper_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/common/test:testing", + "//src/ray/common/tests:testing", "//src/ray/util:compat", "//src/ray/util:filesystem", "//src/ray/util:scoped_dup2_wrapper", diff --git a/src/ray/util/tests/pipe_logger_test.cc b/src/ray/util/tests/pipe_logger_test.cc index afcf07a1433b..d6749e6ee545 100644 --- a/src/ray/util/tests/pipe_logger_test.cc +++ b/src/ray/util/tests/pipe_logger_test.cc @@ -23,7 +23,7 @@ #include #include -#include "ray/common/test/testing.h" +#include "ray/common/tests/testing.h" #include "ray/util/filesystem.h" #include "ray/util/scoped_env_setter.h" #include "ray/util/temporary_directory.h" diff --git a/src/ray/util/tests/process_cleanup_test.cc b/src/ray/util/tests/process_cleanup_test.cc index a34ddc49cf75..6e9f07652f20 100644 --- a/src/ray/util/tests/process_cleanup_test.cc +++ b/src/ray/util/tests/process_cleanup_test.cc @@ -23,7 +23,7 @@ #include #include -#include "ray/common/test/testing.h" +#include "ray/common/tests/testing.h" #include "ray/util/filesystem.h" #include "ray/util/process_cleaner.h" #include "ray/util/util.h" diff --git a/src/ray/util/tests/scoped_dup2_wrapper_test.cc b/src/ray/util/tests/scoped_dup2_wrapper_test.cc index 822635850558..7db58f66e217 100644 --- a/src/ray/util/tests/scoped_dup2_wrapper_test.cc +++ b/src/ray/util/tests/scoped_dup2_wrapper_test.cc @@ -21,7 +21,7 @@ #include #include -#include "ray/common/test/testing.h" +#include "ray/common/tests/testing.h" #include "ray/util/compat.h" #include "ray/util/filesystem.h" #include "ray/util/temporary_directory.h" diff --git a/src/ray/util/tests/spdlog_newliner_sink_test.cc b/src/ray/util/tests/spdlog_newliner_sink_test.cc index c5439e2b4bb4..84ed7438b21a 100644 --- a/src/ray/util/tests/spdlog_newliner_sink_test.cc +++ b/src/ray/util/tests/spdlog_newliner_sink_test.cc @@ -21,7 +21,7 @@ #include #include -#include "ray/common/test/testing.h" +#include "ray/common/tests/testing.h" #include "ray/util/compat.h" #include "ray/util/filesystem.h" #include "ray/util/spdlog_fd_sink.h" diff --git a/src/ray/util/tests/stream_redirection_exit_test.cc b/src/ray/util/tests/stream_redirection_exit_test.cc index cd1ef17b2462..1644f06462aa 100644 --- a/src/ray/util/tests/stream_redirection_exit_test.cc +++ b/src/ray/util/tests/stream_redirection_exit_test.cc @@ -21,7 +21,7 @@ #include #include -#include "ray/common/test/testing.h" +#include "ray/common/tests/testing.h" #include "ray/util/filesystem.h" #include "ray/util/stream_redirection.h" #include "ray/util/util.h" From 9fdea0314ef90cedc341285398bb51d79475b6fd Mon Sep 17 00:00:00 2001 From: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Date: Fri, 15 Aug 2025 15:16:27 -0700 Subject: [PATCH 139/634] [Serve.llm] Support multi-node data parallel with set_dp_master_info() (#55653) Signed-off-by: Rui Qiao Signed-off-by: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Co-authored-by: kourosh hakhamaneshi <31483498+kouroshHakha@users.noreply.github.com> --- .../data_parallel/dp_rank_assigner.py | 12 +++++++++ .../deployments/data_parallel/dp_server.py | 25 ++++++++++++++++++- .../serve/deployments/llm/vllm/vllm_models.py | 20 ++++++++++++++- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py index b9b23065609e..f48496796e4d 100644 --- a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py +++ b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py @@ -16,6 +16,9 @@ def __init__(self, dp_size: int): self.dp_size = dp_size self.lock = asyncio.Lock() self.next_rank = 0 + self.dp_address = None + self.dp_rpc_port = None + self.master_info_event = asyncio.Event() async def register(self, replica_ctx: "serve.context.ReplicaContext"): async with self.lock: @@ -28,3 +31,12 @@ async def register(self, replica_ctx: "serve.context.ReplicaContext"): rank = self.next_rank self.next_rank += 1 return rank + + async def set_dp_master_info(self, dp_address: str, dp_rpc_port: int): + self.dp_address = dp_address + self.dp_rpc_port = dp_rpc_port + self.master_info_event.set() + + async def get_dp_master_info(self): + await self.master_info_event.wait() + return self.dp_address, self.dp_rpc_port diff --git a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py index 200a54c2dfad..fc396847363a 100644 --- a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py +++ b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py @@ -1,7 +1,9 @@ import logging +import time from typing import Optional from ray import serve +from ray.experimental.collective.util import get_address_and_port from ray.llm._internal.serve.configs.constants import DEFAULT_MAX_ONGOING_REQUESTS from ray.llm._internal.serve.configs.server_models import LLMConfig from ray.llm._internal.serve.deployments.data_parallel.dp_rank_assigner import ( @@ -29,10 +31,31 @@ async def __init__(self, llm_config: LLMConfig, dp_rank_assigner: DeploymentHand replica_ctx = serve.get_replica_context() self.dp_rank = await self.dp_rank_assigner.register.remote(replica_ctx) - logger.info(f"DP rank: {self.dp_rank}") + logger.info(f"DP rank {self.dp_rank} has registered") + + if self.dp_rank == 0: + self.dp_address, self.dp_rpc_port = get_address_and_port() + await self.dp_rank_assigner.set_dp_master_info.remote( + self.dp_address, self.dp_rpc_port + ) + logger.info( + f"DP rank {self.dp_rank} has set DP master info: {self.dp_address}, {self.dp_rpc_port}" + ) + else: + timestamp = time.time() + ( + self.dp_address, + self.dp_rpc_port, + ) = await self.dp_rank_assigner.get_dp_master_info.remote() + logger.info( + f"DP rank {self.dp_rank} got DP master info: {self.dp_address}, {self.dp_rpc_port}, " + f"after waiting for {time.time() - timestamp:.3f} seconds" + ) # override the engine_kwargs to assign the DP rank. llm_config.engine_kwargs["data_parallel_rank"] = self.dp_rank + llm_config.engine_kwargs["data_parallel_address"] = self.dp_address + llm_config.engine_kwargs["data_parallel_rpc_port"] = self.dp_rpc_port await super().__init__(llm_config) diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py index 12b7635243ad..4ca753eb9b12 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py @@ -6,6 +6,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.openai.cli_args import FrontendArgs +import ray from ray.llm._internal.common.base_pydantic import BaseModelExtended from ray.llm._internal.common.utils.cloud_utils import CloudMirrorConfig from ray.llm._internal.common.utils.import_utils import try_import @@ -194,12 +195,20 @@ def placement_strategy(self) -> str: @property def placement_bundles(self) -> List[Dict[str, float]]: + dp_rank = self.engine_kwargs.get("data_parallel_rank", None) + if self.resources_per_bundle: bundle = self.resources_per_bundle else: bundle = {"GPU": 1} if self.accelerator_type: bundle[self.ray_accelerator_type()] = 0.001 + if dp_rank is not None: + # For data parallel, we put the placement group on the same node + # as the driver. This is needed to pass ray_utils._verify_bundles() + # validation in vLLM. + node_ip = ray.util.get_node_ip_address() + bundle["node:" + node_ip] = 0.001 bundles = [bundle for _ in range(self.num_devices)] return bundles @@ -238,6 +247,7 @@ def get_or_create_pg(self) -> PlacementGroup: If we are already in a placement group, return the existing placement group. Else, create a new placement group based on the scaling config. """ + dp_rank = self.engine_kwargs.get("data_parallel_rank", None) pg = get_current_placement_group() if pg: logger.debug( @@ -245,6 +255,10 @@ def get_or_create_pg(self) -> PlacementGroup: pg.id, placement_group_table(pg), ) + if dp_rank is not None: + raise NotImplementedError( + "Data parallel is not supported with VLLMEngine already in a placement group" + ) else: if not ALLOW_NEW_PLACEMENT_GROUPS_IN_DEPLOYMENT: raise RuntimeError( @@ -252,8 +266,12 @@ def get_or_create_pg(self) -> PlacementGroup: "Change RAYLLM_ALLOW_NEW_PLACEMENT_GROUPS_IN_DEPLOYMENT " "if this is not intended." ) + name = "" if dp_rank is None else f"dp_{dp_rank}" + pg = placement_group( - self.placement_bundles, strategy=self.placement_strategy + bundles=self.placement_bundles, + strategy=self.placement_strategy, + name=name, ) logger.info(f"Using new placement group {pg}. {placement_group_table(pg)}") From 4c6993ee347e3a4d1ff9a26fb3daddd9bf50783c Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Fri, 15 Aug 2025 16:51:13 -0700 Subject: [PATCH 140/634] [Data] Decouple actor and node autoscaling (#55673) ## Why are these changes needed? Actor pool autoscaling and node autoscaling are currently tied together in a single `Autoscaler` base class, even though they work mostly independently. This coupling makes testing harder (you have to mock unused dependencies), complicates the interface, and forces you to touch unrelated code when extending one type of autoscaling. This PR splits `Autoscaler` into `ActorAutoscaler` and `ClusterAutoscaler` to simplify testing, reduce complexity, and make future extensions easier. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Balaji Veeramani --- .../_internal/actor_autoscaler/__init__.py | 30 ++++ .../autoscaling_actor_pool.py | 0 .../actor_autoscaler/base_actor_autoscaler.py | 31 ++++ .../default_actor_autoscaler.py} | 157 ++++++------------ .../_internal/cluster_autoscaler/__init__.py | 19 +++ .../base_cluster_autoscaler.py} | 6 +- .../default_cluster_autoscaler.py | 107 ++++++++++++ .../execution/autoscaler/__init__.py | 33 ---- .../_internal/execution/autoscaler/util.py | 48 ------ .../execution/interfaces/physical_operator.py | 2 +- .../operators/actor_pool_map_operator.py | 8 +- .../_internal/execution/streaming_executor.py | 20 ++- .../tests/test_actor_pool_map_operator.py | 4 +- python/ray/data/tests/test_autoscaler.py | 13 +- .../test_executor_resource_management.py | 4 +- python/ray/data/tests/test_operators.py | 4 +- 16 files changed, 274 insertions(+), 212 deletions(-) create mode 100644 python/ray/data/_internal/actor_autoscaler/__init__.py rename python/ray/data/_internal/{execution/autoscaler => actor_autoscaler}/autoscaling_actor_pool.py (100%) create mode 100644 python/ray/data/_internal/actor_autoscaler/base_actor_autoscaler.py rename python/ray/data/_internal/{execution/autoscaler/default_autoscaler.py => actor_autoscaler/default_actor_autoscaler.py} (59%) create mode 100644 python/ray/data/_internal/cluster_autoscaler/__init__.py rename python/ray/data/_internal/{execution/autoscaler/autoscaler.py => cluster_autoscaler/base_cluster_autoscaler.py} (88%) create mode 100644 python/ray/data/_internal/cluster_autoscaler/default_cluster_autoscaler.py delete mode 100644 python/ray/data/_internal/execution/autoscaler/__init__.py delete mode 100644 python/ray/data/_internal/execution/autoscaler/util.py diff --git a/python/ray/data/_internal/actor_autoscaler/__init__.py b/python/ray/data/_internal/actor_autoscaler/__init__.py new file mode 100644 index 000000000000..6d29cbc9e78c --- /dev/null +++ b/python/ray/data/_internal/actor_autoscaler/__init__.py @@ -0,0 +1,30 @@ +from typing import TYPE_CHECKING + +from .autoscaling_actor_pool import ActorPoolScalingRequest, AutoscalingActorPool +from .base_actor_autoscaler import ActorAutoscaler +from .default_actor_autoscaler import DefaultActorAutoscaler + +if TYPE_CHECKING: + from ray.data._internal.execution.resource_manager import ResourceManager + from ray.data._internal.execution.streaming_executor_state import Topology + from ray.data.context import AutoscalingConfig + + +def create_actor_autoscaler( + topology: "Topology", + resource_manager: "ResourceManager", + config: "AutoscalingConfig", +) -> ActorAutoscaler: + return DefaultActorAutoscaler( + topology, + resource_manager, + config=config, + ) + + +__all__ = [ + "ActorAutoscaler", + "ActorPoolScalingRequest", + "AutoscalingActorPool", + "create_actor_autoscaler", +] diff --git a/python/ray/data/_internal/execution/autoscaler/autoscaling_actor_pool.py b/python/ray/data/_internal/actor_autoscaler/autoscaling_actor_pool.py similarity index 100% rename from python/ray/data/_internal/execution/autoscaler/autoscaling_actor_pool.py rename to python/ray/data/_internal/actor_autoscaler/autoscaling_actor_pool.py diff --git a/python/ray/data/_internal/actor_autoscaler/base_actor_autoscaler.py b/python/ray/data/_internal/actor_autoscaler/base_actor_autoscaler.py new file mode 100644 index 000000000000..aebdb89bb431 --- /dev/null +++ b/python/ray/data/_internal/actor_autoscaler/base_actor_autoscaler.py @@ -0,0 +1,31 @@ +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +from ray.util.annotations import DeveloperAPI + +if TYPE_CHECKING: + from ray.data._internal.execution.resource_manager import ResourceManager + from ray.data._internal.execution.streaming_executor_state import Topology + + +@DeveloperAPI +class ActorAutoscaler(ABC): + """Abstract interface for Ray Data actor autoscaler.""" + + def __init__( + self, + topology: "Topology", + resource_manager: "ResourceManager", + ): + self._topology = topology + self._resource_manager = resource_manager + + @abstractmethod + def try_trigger_scaling(self): + """Try trigger autoscaling. + + This method will be called each time when StreamingExecutor makes + a scheduling decision. A subclass should override this method to + handle the autoscaling of `AutoscalingActorPool`s. + """ + ... diff --git a/python/ray/data/_internal/execution/autoscaler/default_autoscaler.py b/python/ray/data/_internal/actor_autoscaler/default_actor_autoscaler.py similarity index 59% rename from python/ray/data/_internal/execution/autoscaler/default_autoscaler.py rename to python/ray/data/_internal/actor_autoscaler/default_actor_autoscaler.py index 86ba8ec5e771..50cc1662fcdb 100644 --- a/python/ray/data/_internal/execution/autoscaler/default_autoscaler.py +++ b/python/ray/data/_internal/actor_autoscaler/default_actor_autoscaler.py @@ -1,15 +1,9 @@ import logging import math -import time -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING, Optional -import ray -from .autoscaler import Autoscaler from .autoscaling_actor_pool import ActorPoolScalingRequest, AutoscalingActorPool -from .util import get_max_scale_up -from ray.data._internal.execution.autoscaling_requester import ( - get_or_create_autoscaling_requester_actor, -) +from .base_actor_autoscaler import ActorAutoscaler from ray.data._internal.execution.interfaces.execution_options import ExecutionResources from ray.data.context import WARN_PREFIX, AutoscalingConfig @@ -18,24 +12,18 @@ from ray.data._internal.execution.resource_manager import ResourceManager from ray.data._internal.execution.streaming_executor_state import OpState, Topology - logger = logging.getLogger(__name__) -class DefaultAutoscaler(Autoscaler): - - # Min number of seconds between two autoscaling requests. - MIN_GAP_BETWEEN_AUTOSCALING_REQUESTS = 20 - +class DefaultActorAutoscaler(ActorAutoscaler): def __init__( self, topology: "Topology", resource_manager: "ResourceManager", *, - execution_id: str, config: AutoscalingConfig, ): - super().__init__(topology, resource_manager, execution_id) + super().__init__(topology, resource_manager) self._actor_pool_scaling_up_threshold = ( config.actor_pool_util_upscaling_threshold @@ -46,12 +34,14 @@ def __init__( self._validate_autoscaling_config() - # Last time when a request was sent to Ray's autoscaler. - self._last_request_time = 0 - def try_trigger_scaling(self): - self._try_scale_up_cluster() - self._try_scale_up_or_down_actor_pool() + for op, state in self._topology.items(): + actor_pools = op.get_autoscaling_actor_pools() + for actor_pool in actor_pools: + # Trigger auto-scaling + actor_pool.scale( + self._derive_target_scaling_config(actor_pool, op, state) + ) def _derive_target_scaling_config( self, @@ -99,7 +89,7 @@ def _derive_target_scaling_config( reason="operator exceeding resource quota" ) budget = self._resource_manager.get_budget(op) - if get_max_scale_up(actor_pool, budget) == 0: + if _get_max_scale_up(actor_pool, budget) == 0: return ActorPoolScalingRequest.no_op(reason="exceeded resource limits") return ActorPoolScalingRequest.upscale( @@ -129,86 +119,6 @@ def _derive_target_scaling_config( ) ) - def _try_scale_up_or_down_actor_pool(self): - for op, state in self._topology.items(): - actor_pools = op.get_autoscaling_actor_pools() - for actor_pool in actor_pools: - # Trigger auto-scaling - actor_pool.scale( - self._derive_target_scaling_config(actor_pool, op, state) - ) - - def _try_scale_up_cluster(self): - """Try to scale up the cluster to accomodate the provided in-progress workload. - - This makes a resource request to Ray's autoscaler consisting of the current, - aggregate usage of all operators in the DAG + the incremental usage of all - operators that are ready for dispatch (i.e. that have inputs queued). If the - autoscaler were to grant this resource request, it would allow us to dispatch - one task for every ready operator. - - Note that this resource request does not take the global resource limits or the - liveness policy into account; it only tries to make the existing resource usage - + one more task per ready operator feasible in the cluster. - """ - # Limit the frequency of autoscaling requests. - now = time.time() - if now - self._last_request_time < self.MIN_GAP_BETWEEN_AUTOSCALING_REQUESTS: - return - - # Scale up the cluster, if no ops are allowed to run, but there are still data - # in the input queues. - no_runnable_op = all( - not op_state._scheduling_status.runnable - for _, op_state in self._topology.items() - ) - any_has_input = any( - op_state._pending_dispatch_input_bundles_count() > 0 - for _, op_state in self._topology.items() - ) - if not (no_runnable_op and any_has_input): - return - - self._last_request_time = now - - # Get resource usage for all ops + additional resources needed to launch one - # more task for each ready op. - resource_request = [] - - def to_bundle(resource: ExecutionResources) -> Dict: - req = {} - if resource.cpu: - req["CPU"] = math.ceil(resource.cpu) - if resource.gpu: - req["GPU"] = math.ceil(resource.gpu) - return req - - for op, state in self._topology.items(): - per_task_resource = op.incremental_resource_usage() - task_bundle = to_bundle(per_task_resource) - resource_request.extend([task_bundle] * op.num_active_tasks()) - # Only include incremental resource usage for ops that are ready for - # dispatch. - if state._pending_dispatch_input_bundles_count() > 0: - # TODO(Clark): Scale up more aggressively by adding incremental resource - # usage for more than one bundle in the queue for this op? - resource_request.append(task_bundle) - - self._send_resource_request(resource_request) - - def _send_resource_request(self, resource_request): - # Make autoscaler resource request. - actor = get_or_create_autoscaling_requester_actor() - actor.request_resources.remote(resource_request, self._execution_id) - - def on_executor_shutdown(self): - # Make request for zero resources to autoscaler for this execution. - actor = get_or_create_autoscaling_requester_actor() - actor.request_resources.remote({}, self._execution_id) - - def get_total_resources(self) -> ExecutionResources: - return ExecutionResources.from_resource_dict(ray.cluster_resources()) - def _validate_autoscaling_config(self): for op, state in self._topology.items(): for actor_pool in op.get_autoscaling_actor_pools(): @@ -229,3 +139,46 @@ def _validate_actor_pool_autoscaling_config( f"actor pool is configured to avoid buffering (its " f"`max_tasks_in_flight_per_actor` == `max_concurrency`)" ) + + +def _get_max_scale_up( + actor_pool: AutoscalingActorPool, + budget: Optional[ExecutionResources], +) -> Optional[int]: + """Get the maximum number of actors that can be scaled up. + + Args: + actor_pool: The actor pool to scale up. + budget: The budget to scale up. + + Returns: + The maximum number of actors that can be scaled up, or `None` if you can + scale up infinitely. + """ + if budget is None: + return None + + assert budget.cpu >= 0 and budget.gpu >= 0 + + num_cpus_per_actor = actor_pool.per_actor_resource_usage().cpu + num_gpus_per_actor = actor_pool.per_actor_resource_usage().gpu + assert num_cpus_per_actor >= 0 and num_gpus_per_actor >= 0 + + max_cpu_scale_up: float = float("inf") + if num_cpus_per_actor > 0 and not math.isinf(budget.cpu): + max_cpu_scale_up = budget.cpu // num_cpus_per_actor + + max_gpu_scale_up: float = float("inf") + if num_gpus_per_actor > 0 and not math.isinf(budget.gpu): + max_gpu_scale_up = budget.gpu // num_gpus_per_actor + + max_scale_up = min(max_cpu_scale_up, max_gpu_scale_up) + if math.isinf(max_scale_up): + return None + else: + assert not math.isnan(max_scale_up), ( + budget, + num_cpus_per_actor, + num_gpus_per_actor, + ) + return int(max_scale_up) diff --git a/python/ray/data/_internal/cluster_autoscaler/__init__.py b/python/ray/data/_internal/cluster_autoscaler/__init__.py new file mode 100644 index 000000000000..01e15270a0c4 --- /dev/null +++ b/python/ray/data/_internal/cluster_autoscaler/__init__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from .base_cluster_autoscaler import ClusterAutoscaler +from .default_cluster_autoscaler import DefaultClusterAutoscaler + +if TYPE_CHECKING: + from ray.data._internal.execution.resource_manager import ResourceManager + from ray.data._internal.execution.streaming_executor_state import Topology + + +def create_cluster_autoscaler( + topology: "Topology", resource_manager: "ResourceManager", *, execution_id: str +) -> ClusterAutoscaler: + return DefaultClusterAutoscaler( + topology, resource_manager, execution_id=execution_id + ) + + +__all__ = ["ClusterAutoscaler"] diff --git a/python/ray/data/_internal/execution/autoscaler/autoscaler.py b/python/ray/data/_internal/cluster_autoscaler/base_cluster_autoscaler.py similarity index 88% rename from python/ray/data/_internal/execution/autoscaler/autoscaler.py rename to python/ray/data/_internal/cluster_autoscaler/base_cluster_autoscaler.py index ea49631472c7..cdca9187a70f 100644 --- a/python/ray/data/_internal/execution/autoscaler/autoscaler.py +++ b/python/ray/data/_internal/cluster_autoscaler/base_cluster_autoscaler.py @@ -12,8 +12,8 @@ @DeveloperAPI -class Autoscaler(ABC): - """Abstract interface for Ray Data autoscaler.""" +class ClusterAutoscaler(ABC): + """Abstract interface for Ray Data cluster autoscaler.""" def __init__( self, @@ -31,7 +31,7 @@ def try_trigger_scaling(self): This method will be called each time when StreamingExecutor makes a scheduling decision. A subclass should override this method to - handle the autoscaling of both the cluster and `AutoscalingActorPool`s. + handle the autoscaling of the cluster. """ ... diff --git a/python/ray/data/_internal/cluster_autoscaler/default_cluster_autoscaler.py b/python/ray/data/_internal/cluster_autoscaler/default_cluster_autoscaler.py new file mode 100644 index 000000000000..0ab1d3ea56ce --- /dev/null +++ b/python/ray/data/_internal/cluster_autoscaler/default_cluster_autoscaler.py @@ -0,0 +1,107 @@ +import logging +import math +import time +from typing import TYPE_CHECKING, Dict + +import ray +from .base_cluster_autoscaler import ClusterAutoscaler +from ray.data._internal.execution.autoscaling_requester import ( + get_or_create_autoscaling_requester_actor, +) +from ray.data._internal.execution.interfaces import ExecutionResources + +if TYPE_CHECKING: + from ray.data._internal.execution.resource_manager import ResourceManager + from ray.data._internal.execution.streaming_executor_state import Topology + + +logger = logging.getLogger(__name__) + + +class DefaultClusterAutoscaler(ClusterAutoscaler): + + # Min number of seconds between two autoscaling requests. + MIN_GAP_BETWEEN_AUTOSCALING_REQUESTS = 20 + + def __init__( + self, + topology: "Topology", + resource_manager: "ResourceManager", + *, + execution_id: str, + ): + super().__init__(topology, resource_manager, execution_id) + + # Last time when a request was sent to Ray's autoscaler. + self._last_request_time = 0 + + def try_trigger_scaling(self): + """Try to scale up the cluster to accomodate the provided in-progress workload. + + This makes a resource request to Ray's autoscaler consisting of the current, + aggregate usage of all operators in the DAG + the incremental usage of all + operators that are ready for dispatch (i.e. that have inputs queued). If the + autoscaler were to grant this resource request, it would allow us to dispatch + one task for every ready operator. + + Note that this resource request does not take the global resource limits or the + liveness policy into account; it only tries to make the existing resource usage + + one more task per ready operator feasible in the cluster. + """ + # Limit the frequency of autoscaling requests. + now = time.time() + if now - self._last_request_time < self.MIN_GAP_BETWEEN_AUTOSCALING_REQUESTS: + return + + # Scale up the cluster, if no ops are allowed to run, but there are still data + # in the input queues. + no_runnable_op = all( + not op_state._scheduling_status.runnable + for _, op_state in self._topology.items() + ) + any_has_input = any( + op_state._pending_dispatch_input_bundles_count() > 0 + for _, op_state in self._topology.items() + ) + if not (no_runnable_op and any_has_input): + return + + self._last_request_time = now + + # Get resource usage for all ops + additional resources needed to launch one + # more task for each ready op. + resource_request = [] + + def to_bundle(resource: ExecutionResources) -> Dict: + req = {} + if resource.cpu: + req["CPU"] = math.ceil(resource.cpu) + if resource.gpu: + req["GPU"] = math.ceil(resource.gpu) + return req + + for op, state in self._topology.items(): + per_task_resource = op.incremental_resource_usage() + task_bundle = to_bundle(per_task_resource) + resource_request.extend([task_bundle] * op.num_active_tasks()) + # Only include incremental resource usage for ops that are ready for + # dispatch. + if state._pending_dispatch_input_bundles_count() > 0: + # TODO(Clark): Scale up more aggressively by adding incremental resource + # usage for more than one bundle in the queue for this op? + resource_request.append(task_bundle) + + self._send_resource_request(resource_request) + + def _send_resource_request(self, resource_request): + # Make autoscaler resource request. + actor = get_or_create_autoscaling_requester_actor() + actor.request_resources.remote(resource_request, self._execution_id) + + def on_executor_shutdown(self): + # Make request for zero resources to autoscaler for this execution. + actor = get_or_create_autoscaling_requester_actor() + actor.request_resources.remote({}, self._execution_id) + + def get_total_resources(self) -> ExecutionResources: + return ExecutionResources.from_resource_dict(ray.cluster_resources()) diff --git a/python/ray/data/_internal/execution/autoscaler/__init__.py b/python/ray/data/_internal/execution/autoscaler/__init__.py deleted file mode 100644 index 5a566026d591..000000000000 --- a/python/ray/data/_internal/execution/autoscaler/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -from typing import TYPE_CHECKING - -from .autoscaler import Autoscaler -from .autoscaling_actor_pool import AutoscalingActorPool -from .default_autoscaler import DefaultAutoscaler - -if TYPE_CHECKING: - from ..resource_manager import ResourceManager - from ..streaming_executor_state import Topology - from ray.data.context import AutoscalingConfig - - -def create_autoscaler( - topology: "Topology", - resource_manager: "ResourceManager", - config: "AutoscalingConfig", - *, - execution_id: str -) -> Autoscaler: - return DefaultAutoscaler( - topology, - resource_manager, - config=config, - execution_id=execution_id, - ) - - -__all__ = [ - "Autoscaler", - "DefaultAutoscaler", - "create_autoscaler", - "AutoscalingActorPool", -] diff --git a/python/ray/data/_internal/execution/autoscaler/util.py b/python/ray/data/_internal/execution/autoscaler/util.py deleted file mode 100644 index 550e2a0066e5..000000000000 --- a/python/ray/data/_internal/execution/autoscaler/util.py +++ /dev/null @@ -1,48 +0,0 @@ -import math -from typing import Optional - -from .autoscaling_actor_pool import AutoscalingActorPool -from ray.data._internal.execution.interfaces import ExecutionResources - - -def get_max_scale_up( - actor_pool: AutoscalingActorPool, - budget: Optional[ExecutionResources], -) -> Optional[int]: - """Get the maximum number of actors that can be scaled up. - - Args: - actor_pool: The actor pool to scale up. - budget: The budget to scale up. - - Returns: - The maximum number of actors that can be scaled up, or `None` if you can - scale up infinitely. - """ - if budget is None: - return None - - assert budget.cpu >= 0 and budget.gpu >= 0 - - num_cpus_per_actor = actor_pool.per_actor_resource_usage().cpu - num_gpus_per_actor = actor_pool.per_actor_resource_usage().gpu - assert num_cpus_per_actor >= 0 and num_gpus_per_actor >= 0 - - max_cpu_scale_up: float = float("inf") - if num_cpus_per_actor > 0 and not math.isinf(budget.cpu): - max_cpu_scale_up = budget.cpu // num_cpus_per_actor - - max_gpu_scale_up: float = float("inf") - if num_gpus_per_actor > 0 and not math.isinf(budget.gpu): - max_gpu_scale_up = budget.gpu // num_gpus_per_actor - - max_scale_up = min(max_cpu_scale_up, max_gpu_scale_up) - if math.isinf(max_scale_up): - return None - else: - assert not math.isnan(max_scale_up), ( - budget, - num_cpus_per_actor, - num_gpus_per_actor, - ) - return int(max_scale_up) diff --git a/python/ray/data/_internal/execution/interfaces/physical_operator.py b/python/ray/data/_internal/execution/interfaces/physical_operator.py index 7de458c239b1..12f7242aa3c8 100644 --- a/python/ray/data/_internal/execution/interfaces/physical_operator.py +++ b/python/ray/data/_internal/execution/interfaces/physical_operator.py @@ -18,7 +18,7 @@ import ray from .ref_bundle import RefBundle from ray._raylet import ObjectRefGenerator -from ray.data._internal.execution.autoscaler.autoscaling_actor_pool import ( +from ray.data._internal.actor_autoscaler.autoscaling_actor_pool import ( AutoscalingActorPool, ) from ray.data._internal.execution.interfaces.execution_options import ( diff --git a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py index 9cc21cfb512c..c79350dcedfe 100644 --- a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py @@ -10,11 +10,13 @@ import ray from ray.actor import ActorHandle from ray.core.generated import gcs_pb2 -from ray.data._internal.compute import ActorPoolStrategy -from ray.data._internal.execution.autoscaler import AutoscalingActorPool -from ray.data._internal.execution.autoscaler.default_autoscaler import ( +from ray.data._internal.actor_autoscaler import ( + AutoscalingActorPool, +) +from ray.data._internal.actor_autoscaler.autoscaling_actor_pool import ( ActorPoolScalingRequest, ) +from ray.data._internal.compute import ActorPoolStrategy from ray.data._internal.execution.bundle_queue import create_bundle_queue from ray.data._internal.execution.bundle_queue.bundle_queue import BundleQueue from ray.data._internal.execution.interfaces import ( diff --git a/python/ray/data/_internal/execution/streaming_executor.py b/python/ray/data/_internal/execution/streaming_executor.py index 4e2dfdd9aa6a..7d95c7e4fc72 100644 --- a/python/ray/data/_internal/execution/streaming_executor.py +++ b/python/ray/data/_internal/execution/streaming_executor.py @@ -4,7 +4,10 @@ import time from typing import Dict, List, Optional, Tuple -from ray.data._internal.execution.autoscaler import create_autoscaler +from ray.data._internal.actor_autoscaler import ( + create_actor_autoscaler, +) +from ray.data._internal.cluster_autoscaler import create_cluster_autoscaler from ray.data._internal.execution.backpressure_policy import ( BackpressurePolicy, get_backpressure_policies, @@ -175,18 +178,22 @@ def execute( self._resource_manager = ResourceManager( self._topology, self._options, - lambda: self._autoscaler.get_total_resources(), + lambda: self._cluster_autoscaler.get_total_resources(), self._data_context, ) self._backpressure_policies = get_backpressure_policies( self._data_context, self._topology, self._resource_manager ) - self._autoscaler = create_autoscaler( + self._cluster_autoscaler = create_cluster_autoscaler( self._topology, self._resource_manager, - config=self._data_context.autoscaling_config, execution_id=self._dataset_id, ) + self._actor_autoscaler = create_actor_autoscaler( + self._topology, + self._resource_manager, + config=self._data_context.autoscaling_config, + ) self._has_op_completed = dict.fromkeys(self._topology, False) @@ -294,7 +301,7 @@ def shutdown(self, force: bool, exception: Optional[Exception] = None): for callback in get_execution_callbacks(self._data_context): callback.after_execution_fails(self, exception) - self._autoscaler.on_executor_shutdown() + self._cluster_autoscaler.on_executor_shutdown() dur = time.perf_counter() - start @@ -462,7 +469,8 @@ def _scheduling_loop_step(self, topology: Topology) -> bool: self._refresh_progress_bars(topology) # Trigger autoscaling - self._autoscaler.try_trigger_scaling() + self._cluster_autoscaler.try_trigger_scaling() + self._actor_autoscaler.try_trigger_scaling() update_operator_states(topology) self._refresh_progress_bars(topology) diff --git a/python/ray/data/tests/test_actor_pool_map_operator.py b/python/ray/data/tests/test_actor_pool_map_operator.py index e2627c29a187..f12f8da8df40 100644 --- a/python/ray/data/tests/test_actor_pool_map_operator.py +++ b/python/ray/data/tests/test_actor_pool_map_operator.py @@ -14,9 +14,7 @@ import ray from ray._common.test_utils import wait_for_condition from ray.actor import ActorHandle -from ray.data._internal.execution.autoscaler.default_autoscaler import ( - ActorPoolScalingRequest, -) +from ray.data._internal.actor_autoscaler import ActorPoolScalingRequest from ray.data._internal.execution.bundle_queue import FIFOBundleQueue from ray.data._internal.execution.interfaces import ExecutionResources from ray.data._internal.execution.interfaces.physical_operator import _ActorPoolInfo diff --git a/python/ray/data/tests/test_autoscaler.py b/python/ray/data/tests/test_autoscaler.py index f2d0d2369ead..a41c7d3bebff 100644 --- a/python/ray/data/tests/test_autoscaler.py +++ b/python/ray/data/tests/test_autoscaler.py @@ -8,10 +8,11 @@ import ray from ray.data import ExecutionResources -from ray.data._internal.execution.autoscaler.default_autoscaler import ( +from ray.data._internal.actor_autoscaler import ( ActorPoolScalingRequest, - DefaultAutoscaler, + DefaultActorAutoscaler, ) +from ray.data._internal.cluster_autoscaler import DefaultClusterAutoscaler from ray.data._internal.execution.operators.actor_pool_map_operator import _ActorPool from ray.data._internal.execution.operators.base_physical_operator import ( InternalQueueOperatorMixin, @@ -30,10 +31,9 @@ def test_actor_pool_scaling(): resource_manager = MagicMock( spec=ResourceManager, get_budget=MagicMock(return_value=None) ) - autoscaler = DefaultAutoscaler( + autoscaler = DefaultActorAutoscaler( topology=MagicMock(), resource_manager=resource_manager, - execution_id="execution_id", config=AutoscalingConfig( actor_pool_util_upscaling_threshold=1.0, actor_pool_util_downscaling_threshold=0.5, @@ -240,15 +240,14 @@ def test_cluster_scaling(): op2: op_state2, } - autoscaler = DefaultAutoscaler( + autoscaler = DefaultClusterAutoscaler( topology=topology, resource_manager=MagicMock(), execution_id="execution_id", - config=AutoscalingConfig(), ) autoscaler._send_resource_request = MagicMock() - autoscaler._try_scale_up_cluster() + autoscaler.try_trigger_scaling() autoscaler._send_resource_request.assert_called_once_with( [{"CPU": 1}, {"CPU": 2}, {"CPU": 2}] diff --git a/python/ray/data/tests/test_executor_resource_management.py b/python/ray/data/tests/test_executor_resource_management.py index d863842ec412..73e5a23326fb 100644 --- a/python/ray/data/tests/test_executor_resource_management.py +++ b/python/ray/data/tests/test_executor_resource_management.py @@ -1,10 +1,8 @@ import pytest import ray +from ray.data._internal.actor_autoscaler import ActorPoolScalingRequest from ray.data._internal.compute import ActorPoolStrategy, TaskPoolStrategy -from ray.data._internal.execution.autoscaler.default_autoscaler import ( - ActorPoolScalingRequest, -) from ray.data._internal.execution.interfaces import ExecutionOptions, ExecutionResources from ray.data._internal.execution.operators.input_data_buffer import InputDataBuffer from ray.data._internal.execution.operators.limit_operator import LimitOperator diff --git a/python/ray/data/tests/test_operators.py b/python/ray/data/tests/test_operators.py index e361b237fc37..7f46d75d6dc8 100644 --- a/python/ray/data/tests/test_operators.py +++ b/python/ray/data/tests/test_operators.py @@ -11,10 +11,8 @@ import ray from ray._common.test_utils import wait_for_condition +from ray.data._internal.actor_autoscaler import ActorPoolScalingRequest from ray.data._internal.compute import ActorPoolStrategy, TaskPoolStrategy -from ray.data._internal.execution.autoscaler.default_autoscaler import ( - ActorPoolScalingRequest, -) from ray.data._internal.execution.interfaces import ( ExecutionOptions, PhysicalOperator, From 628df247832fa0e51274a6d53ae750eb9b54a794 Mon Sep 17 00:00:00 2001 From: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Date: Fri, 15 Aug 2025 17:12:20 -0700 Subject: [PATCH 141/634] [serve.llm] Handle push telemetry race conditions (#55558) Signed-off-by: Rui Qiao --- .../deployments/data_parallel/dp_server.py | 6 ---- .../serve/deployments/llm/llm_server.py | 4 +-- .../observability/usage_telemetry/usage.py | 34 ++++++++++++++++++- .../usage_telemetry/test_usage.py | 30 ++++++++++++++++ 4 files changed, 64 insertions(+), 10 deletions(-) diff --git a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py index fc396847363a..c59d2761ed84 100644 --- a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py +++ b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py @@ -59,12 +59,6 @@ async def __init__(self, llm_config: LLMConfig, dp_rank_assigner: DeploymentHand await super().__init__(llm_config) - def _push_telemetry_report(self): - # Only push telemetry report for the first DP replica. - if self.dp_rank == 0: - # TODO(rui): refine the telemetry report for DP deployment. - super()._push_telemetry_report() - @classmethod def as_deployment(cls, deployment_options: dict) -> serve.Deployment: return serve.deployment(cls).options(**deployment_options) diff --git a/python/ray/llm/_internal/serve/deployments/llm/llm_server.py b/python/ray/llm/_internal/serve/deployments/llm/llm_server.py index da20c9b805a8..551329753526 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/llm_server.py +++ b/python/ray/llm/_internal/serve/deployments/llm/llm_server.py @@ -204,7 +204,6 @@ async def start(self): if self._engine_cls is not None: self.engine = self._engine_cls(self._llm_config) await asyncio.wait_for(self._start_engine(), timeout=ENGINE_START_TIMEOUT_S) - self._push_telemetry_report() def _init_multiplex_loader( self, model_downloader_cls: Optional[Type[LoraModelLoader]] = None @@ -252,8 +251,7 @@ async def _start_engine(self): await self.engine.start() - def _push_telemetry_report(self): - """Push telemetry reports for the model in the current deployment.""" + # Push telemetry reports for the model in the current deployment. push_telemetry_report_for_all_models(all_models=[self._llm_config]) def _get_batch_interval_ms(self, stream: bool = True) -> int: diff --git a/python/ray/llm/_internal/serve/observability/usage_telemetry/usage.py b/python/ray/llm/_internal/serve/observability/usage_telemetry/usage.py index 061e34460d66..b011db4810cd 100644 --- a/python/ray/llm/_internal/serve/observability/usage_telemetry/usage.py +++ b/python/ray/llm/_internal/serve/observability/usage_telemetry/usage.py @@ -1,3 +1,5 @@ +import random +import time from enum import Enum from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence @@ -203,9 +205,39 @@ def _get_or_create_telemetry_agent() -> TelemetryAgent: return telemetry_agent +def _retry_get_telemetry_agent( + max_retries: int = 5, base_delay: float = 0.1 +) -> TelemetryAgent: + max_retries = 5 + base_delay = 0.1 + + telemetry_agent = None + for attempt in range(max_retries): + try: + telemetry_agent = _get_or_create_telemetry_agent() + return telemetry_agent + except ValueError as e: + # Due to race conditions among multiple replicas, we may get: + # ValueError: Actor with name 'llm_serve_telemetry' already + # exists in the namespace llm_serve_telemetry + logger.info( + "Attempt %s/%s to get telemetry agent failed", attempt + 1, max_retries + ) + if attempt == max_retries - 1: + raise e + + # Exponential backoff with jitter + exponential_delay = base_delay * (2**attempt) + jitter = random.uniform(0, 0.5) + delay = exponential_delay + jitter + # Max total wait time is ~3.5 seconds for 5 attempts. + time.sleep(delay) + + def _push_telemetry_report(model: Optional[TelemetryModel] = None) -> None: """Push telemetry report for a model.""" - telemetry_agent = _get_or_create_telemetry_agent() + telemetry_agent = _retry_get_telemetry_agent() + assert telemetry_agent is not None ray.get(telemetry_agent.record.remote(model)) diff --git a/python/ray/llm/tests/serve/cpu/observability/usage_telemetry/test_usage.py b/python/ray/llm/tests/serve/cpu/observability/usage_telemetry/test_usage.py index 3b561d8c8312..022314082619 100644 --- a/python/ray/llm/tests/serve/cpu/observability/usage_telemetry/test_usage.py +++ b/python/ray/llm/tests/serve/cpu/observability/usage_telemetry/test_usage.py @@ -13,6 +13,7 @@ from ray.llm._internal.serve.observability.usage_telemetry.usage import ( HardwareUsage, _get_or_create_telemetry_agent, + _retry_get_telemetry_agent, push_telemetry_report_for_all_models, ) @@ -136,6 +137,35 @@ def fake_get_gpu_type(*args, **kwargs): } +@ray.remote(num_cpus=0) +class Replica: + def wait_for_init(self): + """ + When this method returns, the actor initialization is guaranteed + to be complete. + + This is used for synchronization between multiple replicas, + increasing the chance for get_telemetry_agent() to be called + at the same time. + """ + pass + + def get_telemetry_agent(self): + return _retry_get_telemetry_agent() + + +def test_telemetry_race_condition(): + replicas = [Replica.remote() for _ in range(30)] + init_refs = [replica.wait_for_init.remote() for replica in replicas] + ray.get(init_refs) + + get_refs = [replica.get_telemetry_agent.remote() for replica in replicas] + telemetry_agents = ray.get(get_refs) + for telemetry_agent in telemetry_agents: + assert telemetry_agent is not None + assert len(set(telemetry_agents)) == 1 + + def test_infer_gpu_from_hardware(): # Test with a valid GPU type def fake_get_gpu_type(*args, **kwargs): From 418f56258e2085a3f370696930a04ae83e7e0103 Mon Sep 17 00:00:00 2001 From: kourosh hakhamaneshi <31483498+kouroshHakha@users.noreply.github.com> Date: Sat, 16 Aug 2025 03:19:20 +0200 Subject: [PATCH 142/634] [serve.llm] Add reset_prefix_cache remote method to llm server (#55658) Signed-off-by: Kourosh Hakhamaneshi --- .../serve/deployments/llm/llm_engine.py | 4 ++++ .../serve/deployments/llm/llm_server.py | 17 ++++++++++++++ .../serve/deployments/llm/vllm/vllm_engine.py | 4 ++++ .../cpu/deployments/llm/test_llm_server.py | 23 +++++++++++++++++++ .../llm/tests/serve/mocks/mock_vllm_engine.py | 5 ++++ 5 files changed, 53 insertions(+) diff --git a/python/ray/llm/_internal/serve/deployments/llm/llm_engine.py b/python/ray/llm/_internal/serve/deployments/llm/llm_engine.py index ea20ab3d18d3..89bf1a2f5cc9 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/llm_engine.py +++ b/python/ray/llm/_internal/serve/deployments/llm/llm_engine.py @@ -36,6 +36,10 @@ async def resolve_lora(self, lora_model: DiskMultiplexConfig): """Mounts the LoRA model on the engine, given the local disk path.""" pass + @abc.abstractmethod + async def reset_prefix_cache(self) -> None: + """Reset the prefix cache of the underlying engine""" + @abc.abstractmethod async def chat( self, request: "ChatCompletionRequest" diff --git a/python/ray/llm/_internal/serve/deployments/llm/llm_server.py b/python/ray/llm/_internal/serve/deployments/llm/llm_server.py index 551329753526..f7f4d274e611 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/llm_server.py +++ b/python/ray/llm/_internal/serve/deployments/llm/llm_server.py @@ -104,6 +104,10 @@ async def check_health(self) -> None: """ ... + @abstractmethod + async def reset_prefix_cache(self) -> None: + """Reset the prefix cache of the underlying engine""" + # TODO (Kourosh): This does not belong here. async def llm_config(self) -> Optional[LLMConfig]: return None @@ -393,6 +397,19 @@ async def check_health(self) -> None: logger.error("Engine health check failed in LLMServer.check_health: %s", e) raise e + async def reset_prefix_cache(self) -> None: + """Reset the prefix cache of the underlying engine""" + if self.engine is None: + return + try: + await self.engine.reset_prefix_cache() + except Exception as e: + logger.error( + "Engine reset prefix cache failed in LLMServer.reset_prefix_cache: %s", + e, + ) + raise e + async def llm_config(self) -> Optional[LLMConfig]: return self._llm_config diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py index a6654ca89c4e..28ad58b09bb5 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py @@ -450,3 +450,7 @@ async def check_health(self) -> None: except BaseException as e: logger.error("Healthcheck failed. The replica will be restarted") raise e from None + + async def reset_prefix_cache(self) -> None: + assert self._engine_client is not None, "engine_client is not initialized" + await self._engine_client.reset_prefix_cache() diff --git a/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py b/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py index 65dcd74cf38e..6b7627980490 100644 --- a/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py +++ b/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py @@ -175,6 +175,29 @@ async def check_health(self): # Check that the health check method was called assert server.engine.check_health_called + @pytest.mark.asyncio + async def test_reset_prefix_cache(self, mock_llm_config): + """Test reset prefix cache functionality.""" + + # Mock the engine's reset_prefix_cache method + class LocalMockEngine(MockVLLMEngine): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.reset_prefix_cache_called = False + + async def reset_prefix_cache(self): + self.reset_prefix_cache_called = True + + # Create a server with a mocked engine + server = LLMServer.sync_init(mock_llm_config, engine_cls=LocalMockEngine) + await server.start() + + # Perform the health check, no exceptions should be raised + await server.reset_prefix_cache() + + # Check that the reset prefix cache method was called + assert server.engine.reset_prefix_cache_called + @pytest.mark.asyncio async def test_llm_config_property(self, mock_llm_config): """Test the llm_config property.""" diff --git a/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py b/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py index 5db89b1f3c0c..5fd900d1c6fb 100644 --- a/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py +++ b/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py @@ -51,6 +51,11 @@ async def check_health(self) -> None: if not self.started: raise RuntimeError("Engine not started") + async def reset_prefix_cache(self) -> None: + """Reset the prefix cache of the mock engine.""" + if not self.started: + raise RuntimeError("Engine not started") + async def chat( self, request: ChatCompletionRequest ) -> AsyncGenerator[Union[str, ChatCompletionResponse, ErrorResponse], None]: From a44df1655f3031860f3afd4cc81fc0dc6ab5d6f0 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Fri, 15 Aug 2025 23:38:03 -0700 Subject: [PATCH 143/634] [ci] release test: fix to use small for test init (#55677) otherwise the permission is incorrect Signed-off-by: Lonnie Liu --- .buildkite/release/test.rayci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/release/test.rayci.yml b/.buildkite/release/test.rayci.yml index 3bd5b4aff514..4faa7cf05fca 100644 --- a/.buildkite/release/test.rayci.yml +++ b/.buildkite/release/test.rayci.yml @@ -4,7 +4,7 @@ tags: steps: - label: "test init" key: test-init - instance_type: release-medium + instance_type: release commands: - /bin/bash .buildkite/release/test-init.sh mount_buildkite_agent: true From 9ae08276c6c466557281dca28477e9ad1d374687 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Sat, 16 Aug 2025 11:16:44 -0700 Subject: [PATCH 144/634] [core] Update base exception group tests (#55684) Signed-off-by: dayshah --- python/ray/tests/test_exceptiongroup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/tests/test_exceptiongroup.py b/python/ray/tests/test_exceptiongroup.py index 88012d507355..26a4d33cbf8b 100644 --- a/python/ray/tests/test_exceptiongroup.py +++ b/python/ray/tests/test_exceptiongroup.py @@ -21,7 +21,7 @@ def test_baseexceptiongroup_task(ray_start_regular): def task(): raise baseexceptiongroup - with pytest.raises(ray.exceptions.WorkerCrashedError): + with pytest.raises(ray.exceptions.RayTaskError): # noqa: F821 ray.get(task.remote()) @@ -35,7 +35,7 @@ class Actor: def f(self): raise baseexceptiongroup - with pytest.raises(ray.exceptions.ActorDiedError): + with pytest.raises(ray.exceptions.RayTaskError): # noqa: F821 a = Actor.remote() ray.get(a.f.remote()) From 03b07db82ab52c5886edd94885fa12d7c30b7b39 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Sat, 16 Aug 2025 19:41:24 -0700 Subject: [PATCH 145/634] [core] Fix test_failure on windows (#55687) ## Why are these changes needed? Mixing ray_start_regular and ray_start_regular_shared in the same file can lead to unexpected behavior where cluster state can unexpectedly carry over into setup for another test. Here on windows *test_put_error1, test_put_error2,* and *test_version_mismatch are* skipped so *test_export_large_objects* runs directly after *test_baseexception_actor_creation* causing issues during its setup. In a follow up will just create another test file for all basexception related tests so they can use a shared cluster. Signed-off-by: dayshah --- python/ray/tests/test_failure.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/ray/tests/test_failure.py b/python/ray/tests/test_failure.py index ef551d68ea8d..06de8f5dd70b 100644 --- a/python/ray/tests/test_failure.py +++ b/python/ray/tests/test_failure.py @@ -380,7 +380,7 @@ def foo(): assert isinstance(ex, RayTaskError) -def test_baseexception_task(ray_start_regular_shared): +def test_baseexception_task(ray_start_regular): class MyBaseException(BaseException): pass @@ -392,7 +392,7 @@ def task(): ray.get(task.remote()) -def test_baseexception_actor_task(ray_start_regular_shared): +def test_baseexception_actor_task(ray_start_regular): class MyBaseException(BaseException): pass @@ -413,7 +413,7 @@ async def async_f(self): ray.get(a.async_f.remote()) -def test_baseexception_actor_creation(ray_start_regular_shared): +def test_baseexception_actor_creation(ray_start_regular): class MyBaseException(BaseException): pass From 6561061f79b31be4f7cecb20e34bdc92e374ef16 Mon Sep 17 00:00:00 2001 From: Jason Li <57246540+JasonLi1909@users.noreply.github.com> Date: Sun, 17 Aug 2025 11:56:41 -0700 Subject: [PATCH 146/634] Fixing Circular Import in ray.train.v2.lightning.lightning_utils (#55668) Importing `RayTrainReportCallback` from `ray.train.lightning._lightning_utils` in `ray.train.v2.lightning.lightning_utils` causes a circular import in the case that `ray.train.v2.lightning.lightning_utils` is loaded before `ray.train.lightning`. This PR removes the `ray.train.v2.lightning` module and migrates the changes upstream to the original `RayTrainReportCallback` class. --------- Signed-off-by: JasonLi1909 --- python/ray/train/lightning/__init__.py | 6 -- .../ray/train/lightning/_lightning_utils.py | 23 +++++--- python/ray/train/v2/lightning/__init__.py | 0 .../ray/train/v2/lightning/lightning_utils.py | 58 ------------------- 4 files changed, 14 insertions(+), 73 deletions(-) delete mode 100644 python/ray/train/v2/lightning/__init__.py delete mode 100644 python/ray/train/v2/lightning/lightning_utils.py diff --git a/python/ray/train/lightning/__init__.py b/python/ray/train/lightning/__init__.py index c8e413a10308..8be5886a805c 100644 --- a/python/ray/train/lightning/__init__.py +++ b/python/ray/train/lightning/__init__.py @@ -19,12 +19,6 @@ RayTrainReportCallback, prepare_trainer, ) -from ray.train.v2._internal.constants import is_v2_enabled - -if is_v2_enabled(): - from ray.train.v2.lightning.lightning_utils import ( # noqa: F811 - RayTrainReportCallback, - ) __all__ = [ "prepare_trainer", diff --git a/python/ray/train/lightning/_lightning_utils.py b/python/ray/train/lightning/_lightning_utils.py index 2157287af516..2da924a3357c 100644 --- a/python/ray/train/lightning/_lightning_utils.py +++ b/python/ray/train/lightning/_lightning_utils.py @@ -9,7 +9,7 @@ from packaging.version import Version import ray -from ray import train +import ray.train from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag from ray.train import Checkpoint from ray.util import PublicAPI @@ -182,16 +182,16 @@ def __init__(self, *args, **kwargs): record_extra_usage_tag(TagKey.TRAIN_LIGHTNING_RAYLIGHTNINGENVIRONMENT, "1") def world_size(self) -> int: - return train.get_context().get_world_size() + return ray.train.get_context().get_world_size() def global_rank(self) -> int: - return train.get_context().get_world_rank() + return ray.train.get_context().get_world_rank() def local_rank(self) -> int: - return train.get_context().get_local_rank() + return ray.train.get_context().get_local_rank() def node_rank(self) -> int: - return train.get_context().get_node_rank() + return ray.train.get_context().get_node_rank() def set_world_size(self, size: int) -> None: # Disable it since `world_size()` directly returns data from Train context. @@ -259,9 +259,14 @@ class RayTrainReportCallback(pl.callbacks.Callback): def __init__(self) -> None: super().__init__() - self.trial_name = train.get_context().get_trial_name() - self.local_rank = train.get_context().get_local_rank() - self.tmpdir_prefix = Path(tempfile.gettempdir(), self.trial_name).as_posix() + job_id = ray.get_runtime_context().get_job_id() + experiment_name = ray.train.get_context().get_experiment_name() + self.local_rank = ray.train.get_context().get_local_rank() + + self.tmpdir_prefix = Path( + tempfile.gettempdir(), + f"lightning_checkpoints-job_id={job_id}-name={experiment_name}", + ).as_posix() if os.path.isdir(self.tmpdir_prefix) and self.local_rank == 0: shutil.rmtree(self.tmpdir_prefix) @@ -286,7 +291,7 @@ def on_train_epoch_end(self, trainer, pl_module) -> None: # Report to train session checkpoint = Checkpoint.from_directory(tmpdir) - train.report(metrics=metrics, checkpoint=checkpoint) + ray.train.report(metrics=metrics, checkpoint=checkpoint) # Add a barrier to ensure all workers finished reporting here trainer.strategy.barrier() diff --git a/python/ray/train/v2/lightning/__init__.py b/python/ray/train/v2/lightning/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/python/ray/train/v2/lightning/lightning_utils.py b/python/ray/train/v2/lightning/lightning_utils.py deleted file mode 100644 index 4d417248c73d..000000000000 --- a/python/ray/train/v2/lightning/lightning_utils.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import shutil -import tempfile -from pathlib import Path - -import ray.train -from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag -from ray.train.lightning._lightning_utils import ( - RayTrainReportCallback as RayTrainReportCallbackV1, - import_lightning, -) -from ray.util import PublicAPI - -pl = import_lightning() - - -@PublicAPI(stability="beta") -class RayTrainReportCallback(RayTrainReportCallbackV1): - """A simple callback that reports checkpoints to Ray on train epoch end. - - This callback is a subclass of `lightning.pytorch.callbacks.Callback - `_. - - It fetches the latest `trainer.callback_metrics` and reports together with - the checkpoint on each training epoch end. - - Checkpoints will be saved in the following structure: - - checkpoint_{timestamp}/ Ray Train's checkpoint folder - └─ checkpoint.ckpt Lightning's checkpoint format - - For customized reporting and checkpointing logic, implement your own - `lightning.pytorch.callbacks.Callback` following this user - guide: :ref:`Saving and Loading Checkpoints `. - """ - - def __init__(self) -> None: - # TODO: Upstream this change into ray.train.lightning. - # The difference in this version is removing the trial directory usage. - job_id = ray.get_runtime_context().get_job_id() - experiment_name = ray.train.get_context().get_experiment_name() - self.local_rank = ray.train.get_context().get_local_rank() - - # Create a root temporary directory for storing local checkpoints - # before persisting to storage. - # Lightning's checkpointing implementation requires that this directory - # is a common path across all workers. - # Construct the path prefix with the job id and experiment name, - # which are shared across workers for a Ray Train run. - # This path should not be shared across different Ray Train runs. - self.tmpdir_prefix = Path( - tempfile.gettempdir(), - f"lightning_checkpoints-job_id={job_id}-name={experiment_name}", - ).as_posix() - if os.path.isdir(self.tmpdir_prefix) and self.local_rank == 0: - shutil.rmtree(self.tmpdir_prefix) - - record_extra_usage_tag(TagKey.TRAIN_LIGHTNING_RAYTRAINREPORTCALLBACK, "1") From 7321aeed2957a5a71ccb34c2212cd8f4c63a9fab Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Sun, 17 Aug 2025 18:34:27 -0500 Subject: [PATCH 147/634] [core] Remove unnecessary publisher dependency from raylet (#55678) Signed-off-by: Edward Oakes --- src/ray/raylet/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index c06b8e9f3a4e..dba01e0a56da 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -230,7 +230,6 @@ ray_cc_library( "//src/ray/object_manager", "//src/ray/object_manager:ownership_object_directory", "//src/ray/object_manager/plasma:plasma_client", - "//src/ray/pubsub:publisher", "//src/ray/pubsub:subscriber", "//src/ray/raylet/scheduling:scheduler", "//src/ray/rpc:core_worker_client", From dde4dbad440ada233d5b3e13a990cf25c20ec60e Mon Sep 17 00:00:00 2001 From: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Date: Sun, 17 Aug 2025 21:33:48 -0700 Subject: [PATCH 148/634] [Serve.llm] Fix DPServer allocation to CPU node (#55688) Signed-off-by: Rui Qiao --- .../serve/builders/application_builders.py | 2 +- .../_internal/serve/configs/server_models.py | 54 ++++++++++++++++--- .../deployments/data_parallel/dp_server.py | 45 ++++++---------- .../serve/deployments/llm/vllm/vllm_models.py | 12 ----- .../prefill_decode_disagg.py | 14 +++-- .../cpu/builders/test_application_builders.py | 2 +- 6 files changed, 70 insertions(+), 59 deletions(-) diff --git a/python/ray/llm/_internal/serve/builders/application_builders.py b/python/ray/llm/_internal/serve/builders/application_builders.py index 45c81da72aec..769477044e1d 100644 --- a/python/ray/llm/_internal/serve/builders/application_builders.py +++ b/python/ray/llm/_internal/serve/builders/application_builders.py @@ -22,7 +22,7 @@ def build_llm_deployment( name_prefix: Optional[str] = None, deployment_kwargs: Optional[dict] = None, ) -> Application: - name_prefix = name_prefix or "LLMDeployment:" + name_prefix = name_prefix or "LLMServer:" deployment_kwargs = deployment_kwargs or {} deployment_options = llm_config.get_serve_options( diff --git a/python/ray/llm/_internal/serve/configs/server_models.py b/python/ray/llm/_internal/serve/configs/server_models.py index 5be9ff21a1f9..910fce823c0c 100644 --- a/python/ray/llm/_internal/serve/configs/server_models.py +++ b/python/ray/llm/_internal/serve/configs/server_models.py @@ -29,6 +29,7 @@ ) from ray.llm._internal.common.utils.import_utils import try_import from ray.llm._internal.serve.configs.constants import ( + DEFAULT_MAX_ONGOING_REQUESTS, DEFAULT_MULTIPLEX_DOWNLOAD_TIMEOUT_S, DEFAULT_MULTIPLEX_DOWNLOAD_TRIES, ENABLE_WORKER_PROCESS_SETUP_HOOK, @@ -411,6 +412,18 @@ def get_engine_config(self) -> EngineConfigType: return self._engine_config + def update_engine_kwargs(self, **kwargs: Any) -> None: + """Update the engine_kwargs and the engine_config engine_kwargs. + + This is typically called during engine starts, when certain engine_kwargs + (e.g., data_parallel_rank) become available. + """ + self.engine_kwargs.update(kwargs) + # engine_config may be created before engine starts, this makes sure + # the engine_config is updated with the latest engine_kwargs. + if self._engine_config: + self._engine_config.engine_kwargs.update(kwargs) + def _set_deployment_placement_options(self) -> Dict[str, Any]: deployment_config = self.deployment_config engine_config = self.get_engine_config() @@ -487,7 +500,7 @@ def get_serve_options( The dictionary to use in .options() when creating the deployment. """ - deployment_config = self._set_deployment_placement_options() + deployment_options = self._set_deployment_placement_options() default_runtime_env = ray.get_runtime_context().runtime_env if ENABLE_WORKER_PROCESS_SETUP_HOOK: @@ -495,28 +508,53 @@ def get_serve_options( "worker_process_setup_hook" ] = "ray.llm._internal.serve._worker_process_setup_hook" - ray_actor_options = deployment_config.get("ray_actor_options", {}) + ray_actor_options = deployment_options.get("ray_actor_options", {}) ray_actor_options["runtime_env"] = { **default_runtime_env, # Existing runtime_env should take precedence over the default. **ray_actor_options.get("runtime_env", {}), **(self.runtime_env if self.runtime_env else {}), } - deployment_config["ray_actor_options"] = ray_actor_options + deployment_options["ray_actor_options"] = ray_actor_options # Set the name of the deployment config to map to the model ID. - if "name" not in deployment_config: - deployment_config["name"] = self._get_deployment_name() + if "name" not in deployment_options: + deployment_options["name"] = self._get_deployment_name() if name_prefix: - deployment_config["name"] = name_prefix + deployment_config["name"] + deployment_options["name"] = name_prefix + deployment_options["name"] + + # Configure DP deployment options. + # TODO(rui): move the following to DPServer, e.g., + # deployment_options = DPServer.get_deployment_options(llm_config) + dp_size = self.engine_kwargs.get("data_parallel_size", None) + if dp_size is not None: + if "num_replicas" in deployment_options: + raise ValueError( + "num_replicas should not be specified for DP deployment, " + "use engine_kwargs.data_parallel_size instead." + ) + if "autoscaling_config" in deployment_options: + raise ValueError( + "autoscaling_config is not supported for DP deployment, " + "use engine_kwargs.data_parallel_size to set a fixed number " + "of replicas instead." + ) + deployment_options["num_replicas"] = dp_size + deployment_options["max_ongoing_requests"] = DEFAULT_MAX_ONGOING_REQUESTS + if deployment_options["placement_group_strategy"] != "STRICT_PACK": + logger.warning( + f"DP deployment with placement_strategy={deployment_options['placement_group_strategy']} " + "is not supported. Using STRICT_PACK instead." + ) + deployment_options["placement_group_strategy"] = "STRICT_PACK" - return deployment_config + return deployment_options def setup_engine_backend(self): self._setup_kv_connector_backend() def _setup_kv_connector_backend(self): - """Private method to setup kv connector dependning on the local deployment state""" + """Private method to setup kv connector depending on the local deployment state""" # 1. validate that the backend is one of the backends supported (Nixl or LMCache) kv_transfer_config = self.engine_kwargs.get("kv_transfer_config") if not kv_transfer_config: diff --git a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py index c59d2761ed84..7fb1cc2f49a1 100644 --- a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py +++ b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py @@ -4,7 +4,6 @@ from ray import serve from ray.experimental.collective.util import get_address_and_port -from ray.llm._internal.serve.configs.constants import DEFAULT_MAX_ONGOING_REQUESTS from ray.llm._internal.serve.configs.server_models import LLMConfig from ray.llm._internal.serve.deployments.data_parallel.dp_rank_assigner import ( DPRankAssigner, @@ -31,7 +30,7 @@ async def __init__(self, llm_config: LLMConfig, dp_rank_assigner: DeploymentHand replica_ctx = serve.get_replica_context() self.dp_rank = await self.dp_rank_assigner.register.remote(replica_ctx) - logger.info(f"DP rank {self.dp_rank} has registered") + logger.info(f"DP rank {self.dp_rank} registered with rank assigner") if self.dp_rank == 0: self.dp_address, self.dp_rpc_port = get_address_and_port() @@ -39,7 +38,9 @@ async def __init__(self, llm_config: LLMConfig, dp_rank_assigner: DeploymentHand self.dp_address, self.dp_rpc_port ) logger.info( - f"DP rank {self.dp_rank} has set DP master info: {self.dp_address}, {self.dp_rpc_port}" + f"DP rank {self.dp_rank} has set DP master info: " + f"data_parallel_address={self.dp_address}, " + f"data_parallel_rpc_port={self.dp_rpc_port}" ) else: timestamp = time.time() @@ -48,14 +49,18 @@ async def __init__(self, llm_config: LLMConfig, dp_rank_assigner: DeploymentHand self.dp_rpc_port, ) = await self.dp_rank_assigner.get_dp_master_info.remote() logger.info( - f"DP rank {self.dp_rank} got DP master info: {self.dp_address}, {self.dp_rpc_port}, " - f"after waiting for {time.time() - timestamp:.3f} seconds" + f"DP rank {self.dp_rank} got DP master info: " + f"data_parallel_address={self.dp_address}, " + f"data_parallel_rpc_port={self.dp_rpc_port}, " + f"waited {time.time() - timestamp:.3f} seconds" ) - # override the engine_kwargs to assign the DP rank. - llm_config.engine_kwargs["data_parallel_rank"] = self.dp_rank - llm_config.engine_kwargs["data_parallel_address"] = self.dp_address - llm_config.engine_kwargs["data_parallel_rpc_port"] = self.dp_rpc_port + # Update the engine_kwargs to assign the DP information + llm_config.update_engine_kwargs( + data_parallel_rank=self.dp_rank, + data_parallel_address=self.dp_address, + data_parallel_rpc_port=self.dp_rpc_port, + ) await super().__init__(llm_config) @@ -75,27 +80,9 @@ def build_dp_deployment( raise ValueError( "data_parallel_size should be greater than 1 for DP deployment." ) + + deployment_options = llm_config.get_serve_options(name_prefix=name_prefix) dp_rank_assigner = DPRankAssigner.bind(dp_size=dp_size) - name_prefix = name_prefix or "DPLLMDeployment:" - name = name_prefix + llm_config._get_deployment_name() - if "num_replicas" in llm_config.deployment_config: - raise ValueError( - "num_replicas should not be specified for DP deployment, " - "use engine_kwargs.data_parallel_size instead." - ) - if "autoscaling_config" in llm_config.deployment_config: - raise ValueError( - "autoscaling_config is not supported for DP deployment, " - "use engine_kwargs.data_parallel_size to set a fixed number " - "of replicas instead." - ) - # TODO(rui): support data_parallel_backend=ray and unify - # deployment_options handling with LLMDeployment. - deployment_options = { - "name": name, - "num_replicas": dp_size, - "max_ongoing_requests": DEFAULT_MAX_ONGOING_REQUESTS, - } return DPServer.as_deployment(deployment_options).bind( llm_config=llm_config, dp_rank_assigner=dp_rank_assigner ) diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py index 4ca753eb9b12..081b0b2b6807 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py @@ -6,7 +6,6 @@ from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.openai.cli_args import FrontendArgs -import ray from ray.llm._internal.common.base_pydantic import BaseModelExtended from ray.llm._internal.common.utils.cloud_utils import CloudMirrorConfig from ray.llm._internal.common.utils.import_utils import try_import @@ -195,7 +194,6 @@ def placement_strategy(self) -> str: @property def placement_bundles(self) -> List[Dict[str, float]]: - dp_rank = self.engine_kwargs.get("data_parallel_rank", None) if self.resources_per_bundle: bundle = self.resources_per_bundle @@ -203,12 +201,6 @@ def placement_bundles(self) -> List[Dict[str, float]]: bundle = {"GPU": 1} if self.accelerator_type: bundle[self.ray_accelerator_type()] = 0.001 - if dp_rank is not None: - # For data parallel, we put the placement group on the same node - # as the driver. This is needed to pass ray_utils._verify_bundles() - # validation in vLLM. - node_ip = ray.util.get_node_ip_address() - bundle["node:" + node_ip] = 0.001 bundles = [bundle for _ in range(self.num_devices)] return bundles @@ -255,10 +247,6 @@ def get_or_create_pg(self) -> PlacementGroup: pg.id, placement_group_table(pg), ) - if dp_rank is not None: - raise NotImplementedError( - "Data parallel is not supported with VLLMEngine already in a placement group" - ) else: if not ALLOW_NEW_PLACEMENT_GROUPS_IN_DEPLOYMENT: raise RuntimeError( diff --git a/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py b/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py index d825a30a2161..0c0ff2cff450 100644 --- a/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py +++ b/python/ray/llm/_internal/serve/deployments/prefill_decode_disagg/prefill_decode_disagg.py @@ -184,14 +184,12 @@ def build_pd_openai_app(pd_serving_args: dict) -> Application: for config in [pd_config.prefill_config, pd_config.decode_config]: if "kv_transfer_config" not in config.engine_kwargs: - config.engine_kwargs.update( - { - "kv_transfer_config": dict( - kv_connector="NixlConnector", - kv_role="kv_both", - engine_id=str(uuid.uuid4()), - ) - } + config.update_engine_kwargs( + kv_transfer_config=dict( + kv_connector="NixlConnector", + kv_role="kv_both", + engine_id=str(uuid.uuid4()), + ) ) prefill_deployment = build_llm_deployment( diff --git a/python/ray/llm/tests/serve/cpu/builders/test_application_builders.py b/python/ray/llm/tests/serve/cpu/builders/test_application_builders.py index a64dbd803c47..0bb3bbd48da4 100644 --- a/python/ray/llm/tests/serve/cpu/builders/test_application_builders.py +++ b/python/ray/llm/tests/serve/cpu/builders/test_application_builders.py @@ -196,7 +196,7 @@ def test_build_llm_deployment( app = build_llm_deployment(llm_config_with_mock_engine) assert isinstance(app, serve.Application) handle = serve.run(app) - assert handle.deployment_name.startswith("LLMDeployment") + assert handle.deployment_name.startswith("LLMServer") def test_build_llm_deployment_with_name_prefix( self, From b830b8d3ee64f7c661d4bfa5fb0e7be99ff871a5 Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Mon, 18 Aug 2025 12:30:24 +0200 Subject: [PATCH 149/634] [RLlib - Offline] Fix some bugs in the docs for IQL and CQL (#55614) --- doc/source/rllib/rllib-algorithms.rst | 4 ++-- rllib/algorithms/iql/iql.py | 28 ++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/doc/source/rllib/rllib-algorithms.rst b/doc/source/rllib/rllib-algorithms.rst index 1fdd788f4c7b..dd8e84713c03 100644 --- a/doc/source/rllib/rllib-algorithms.rst +++ b/doc/source/rllib/rllib-algorithms.rst @@ -361,7 +361,7 @@ Conservative Q-Learning (CQL) **Tuned examples:** `Pendulum-v1 `__ -**CQL-specific configs** and :ref:`generic algorithm settings `): +**CQL-specific configs** (see also :ref:`generic algorithm settings `): .. autoclass:: ray.rllib.algorithms.cql.cql.CQLConfig :members: training @@ -386,7 +386,7 @@ Implicit Q-Learning (IQL) **Tuned examples:** `Pendulum-v1 `__ -**IQL-specific configs** and :ref:`generic algorithm settings `): +**IQL-specific configs** (see also :ref:`generic algorithm settings `): .. autoclass:: ray.rllib.algorithms.iql.iql.IQLConfig :members: training diff --git a/rllib/algorithms/iql/iql.py b/rllib/algorithms/iql/iql.py index c7175f6d1a7f..893555002708 100644 --- a/rllib/algorithms/iql/iql.py +++ b/rllib/algorithms/iql/iql.py @@ -22,7 +22,7 @@ class IQLConfig(MARWILConfig): from ray.rllib.algorithms.iql import IQLConfig # Run this from the ray directory root. - config = IQLConfig().training(lr=0.00001, gamma=0.99) + config = IQLConfig().training(actor_lr=0.00001, gamma=0.99) config = config.offline_data( input_="./rllib/tests/data/pendulum/pendulum-v1_enormous") @@ -99,6 +99,32 @@ def training( tau: Optional[float] = NotProvided, **kwargs, ) -> "IQLConfig": + """Sets the training related configuration. + + Args: + beta: The temperature to scaling advantages in exponential terms. + Must be >> 0.0. The higher this parameter the less greedy + (exploitative) the policy becomes. It also means that the policy + is fitting less to the best actions in the dataset. + twin_q: If a twin-Q architecture should be used (advisable). + expectile: The expectile to use in expectile regression for the value + function. For high expectiles the value function tries to match + the upper tail of the Q-value distribution. + actor_lr: The learning rate for the actor network. Actor learning rates + greater than critic learning rates work well in experiments. + critic_lr: The learning rate for the Q-network. Critic learning rates + greater than value function learning rates work well in experiments. + value_lr: The learning rate for the value function network. + target_network_update_freq: The number of timesteps in between the target + Q-network is fixed. Note, too high values here could harm convergence. + The target network is updated via Polyak-averaging. + tau: The update parameter for Polyak-averaging of the target Q-network. + The higher this value the faster the weights move towards the actual + Q-network. + + Return: + This updated `AlgorithmConfig` object. + """ super().training(**kwargs) if twin_q is not NotProvided: From 1e5094fd5cbfef1de738243b84436b94a7499304 Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Mon, 18 Aug 2025 15:13:05 +0200 Subject: [PATCH 150/634] [RLlib - Offline RL] Fix bug in `return_iterator` in multi-learner settings. (#55693) --- rllib/algorithms/algorithm_config.py | 6 +++--- rllib/algorithms/cql/cql.py | 11 ++++++++--- rllib/algorithms/marwil/marwil.py | 8 +++++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/rllib/algorithms/algorithm_config.py b/rllib/algorithms/algorithm_config.py index 2c6120b86452..19e8801319af 100644 --- a/rllib/algorithms/algorithm_config.py +++ b/rllib/algorithms/algorithm_config.py @@ -2989,7 +2989,7 @@ def evaluation( if offline_evaluation_type is not NotProvided: self.offline_evaluation_type = offline_evaluation_type if offline_eval_runner_class is not NotProvided: - self.offline_eval_runner_cls = offline_eval_runner_class + self.offline_eval_runner_class = offline_eval_runner_class if offline_loss_for_module_fn is not NotProvided: self.offline_loss_for_module_fn = offline_loss_for_module_fn if offline_eval_batch_size_per_runner is not NotProvided: @@ -5328,8 +5328,8 @@ def _validate_offline_settings(self): from ray.rllib.offline.offline_evaluation_runner import OfflineEvaluationRunner - if self.prelearner_class and not issubclass( - self.prelearner_class, OfflineEvaluationRunner + if self.offline_eval_runner_class and not issubclass( + self.offline_eval_runner_class, OfflineEvaluationRunner ): self._value_error( "Unknown `offline_eval_runner_class`. OfflineEvaluationRunner class needs to inherit " diff --git a/rllib/algorithms/cql/cql.py b/rllib/algorithms/cql/cql.py index de972a119a90..e2f3ac2eff44 100644 --- a/rllib/algorithms/cql/cql.py +++ b/rllib/algorithms/cql/cql.py @@ -302,15 +302,20 @@ def training_step(self) -> None: # Sampling from offline data. with self.metrics.log_time((TIMERS, OFFLINE_SAMPLING_TIMER)): + # If we should use an iterator in the learner(s). Note, in case of + # multiple learners we must always return a list of iterators. + return_iterator = return_iterator = ( + self.config.num_learners > 0 + or self.config.dataset_num_iters_per_learner != 1 + ) + # Return an iterator in case we are using remote learners. batch_or_iterator = self.offline_data.sample( num_samples=self.config.train_batch_size_per_learner, num_shards=self.config.num_learners, # Return an iterator, if a `Learner` should update # multiple times per RLlib iteration. - return_iterator=self.config.dataset_num_iters_per_learner > 1 - if self.config.dataset_num_iters_per_learner - else True, + return_iterator=return_iterator, ) # Updating the policy. diff --git a/rllib/algorithms/marwil/marwil.py b/rllib/algorithms/marwil/marwil.py index b0a06ae6d2d8..4ebf1d9333a4 100644 --- a/rllib/algorithms/marwil/marwil.py +++ b/rllib/algorithms/marwil/marwil.py @@ -457,11 +457,13 @@ class (multi-/single-learner setup) and evaluation on # the user that sth. is not right, although it is as # we do not step the env. with self.metrics.log_time((TIMERS, OFFLINE_SAMPLING_TIMER)): + # If we should use an iterator in the learner(s). Note, in case of + # multiple learners we must always return a list of iterators. return_iterator = ( - self.config.dataset_num_iters_per_learner > 1 - if self.config.dataset_num_iters_per_learner - else True + self.config.num_learners > 0 + or self.config.dataset_num_iters_per_learner != 1 ) + # Sampling from offline data. batch_or_iterator = self.offline_data.sample( num_samples=self.config.train_batch_size_per_learner, From e0d8e6f46a8734e16f28831941d937d5961c1d12 Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Mon, 18 Aug 2025 15:26:58 +0200 Subject: [PATCH 151/634] [RLlib] - Fix `TensorType` (#55694) --- rllib/utils/typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rllib/utils/typing.py b/rllib/utils/typing.py index 81116fbcacaf..3f7f559b6e2c 100644 --- a/rllib/utils/typing.py +++ b/rllib/utils/typing.py @@ -39,8 +39,8 @@ jnp = jax.numpy # Represents a generic tensor type. -# This could be an np.ndarray, tf.Tensor, or a torch.Tensor. -TensorType = Union[np.array, "jnp.ndarray", "tf.Tensor", "torch.Tensor"] +# This could be an np.ndarray, jnp.ndarray, tf.Tensor, or a torch.Tensor. +TensorType = Union[np.ndarray, "jnp.ndarray", "tf.Tensor", "torch.Tensor"] # Either a plain tensor, or a dict or tuple of tensors (or StructTensors). TensorStructType = Union[TensorType, dict, tuple] From faf06e09e55558fb36c72e91a5cf8a7e3da8b8c6 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Mon, 18 Aug 2025 07:33:47 -0700 Subject: [PATCH 152/634] [core] Follow-up to address comments of BaseException PR #55602 (#55690) ## Why are these changes needed? Address comments from #55602 - Moving the base exception and exception group tests into their own file so they can use a shared fixture - Adding comment for SystemExit and KeyboardInterrupt behavior - Adding tests to test behavior if user code raises SystemExit or KeyboardInterrupt --------- Signed-off-by: dayshah --- python/ray/_raylet.pyx | 2 + python/ray/tests/BUILD | 2 +- ...group.py => test_baseexceptionandgroup.py} | 133 ++++++++++++++++-- python/ray/tests/test_failure.py | 50 ------- python/ray/tests/test_streaming_generator.py | 42 ------ 5 files changed, 127 insertions(+), 102 deletions(-) rename python/ray/tests/{test_exceptiongroup.py => test_baseexceptionandgroup.py} (60%) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index a88e81009fa1..ee02c709fafa 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -2013,6 +2013,7 @@ cdef void execute_task( except AsyncioActorExit as e: exit_current_actor_if_asyncio() except (KeyboardInterrupt, SystemExit): + # Special casing these two because Ray can raise them raise except BaseException as e: is_retryable_error[0] = determine_if_retryable( @@ -2124,6 +2125,7 @@ cdef void execute_task( c_tensor_transport ) except (KeyboardInterrupt, SystemExit): + # Special casing these two because Ray can raise them raise except BaseException as e: num_errors_stored = store_task_errors( diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD index 7716ecc22d9a..5f1198c5bd99 100644 --- a/python/ray/tests/BUILD +++ b/python/ray/tests/BUILD @@ -510,6 +510,7 @@ py_test_module_list( "test_async_compat.py", "test_asyncio_cluster.py", "test_autoscaling_policy.py", + "test_baseexceptionandgroup.py", "test_bounded_unix_sockets.py", "test_component_failures.py", "test_concurrency_group.py", @@ -519,7 +520,6 @@ py_test_module_list( "test_distributed_sort.py", "test_environ.py", "test_error_ray_not_initialized.py", - "test_exceptiongroup.py", "test_gcs_pubsub.py", "test_get_or_create_actor.py", "test_grpc_client_credentials.py", diff --git a/python/ray/tests/test_exceptiongroup.py b/python/ray/tests/test_baseexceptionandgroup.py similarity index 60% rename from python/ray/tests/test_exceptiongroup.py rename to python/ray/tests/test_baseexceptionandgroup.py index 26a4d33cbf8b..a363bdf84eaf 100644 --- a/python/ray/tests/test_exceptiongroup.py +++ b/python/ray/tests/test_baseexceptionandgroup.py @@ -1,18 +1,129 @@ +import pytest import sys from textwrap import dedent -import pytest - import ray -from ray.exceptions import RayTaskError +from ray.exceptions import ( + RayTaskError, + ActorDiedError, + TaskCancelledError, + WorkerCrashedError, +) + + +def test_baseexception_task(ray_start_regular_shared): + class MyBaseException(BaseException): + pass + + @ray.remote + def task(): + raise MyBaseException("abc") + + with pytest.raises(MyBaseException): + ray.get(task.remote()) + + +def test_baseexception_actor_task(ray_start_regular_shared): + class MyBaseException(BaseException): + pass + + @ray.remote + class Actor: + def f(self): + raise MyBaseException("abc") + + async def async_f(self): + raise MyBaseException("abc") + + a = Actor.remote() + with pytest.raises(MyBaseException): + ray.get(a.f.remote()) + + with pytest.raises(MyBaseException): + ray.get(a.async_f.remote()) + + +def test_baseexception_actor_creation(ray_start_regular_shared): + class MyBaseException(BaseException): + pass + + @ray.remote + class Actor: + def __init__(self): + raise MyBaseException("abc") + + with pytest.raises(ActorDiedError) as e: + a = Actor.remote() + ray.get(a.__ray_ready__.remote()) + assert "MyBaseException" in str(e.value) + + +def test_baseexception_streaming_generator(ray_start_regular_shared): + class MyBaseException(BaseException): + pass + + @ray.remote + def raise_at_beginning(): + raise MyBaseException("rip") + yield 1 + + raise_at_beginning_ref = raise_at_beginning.remote() + with pytest.raises(MyBaseException): + ray.get(next(raise_at_beginning_ref)) + + @ray.remote + def raise_at_middle(): + for i in range(1, 10): + if i == 5: + raise MyBaseException("rip") + yield i + + raise_at_middle_ref = raise_at_middle.remote() + for i in range(1, 5): + assert i == ray.get(next(raise_at_middle_ref)) + with pytest.raises(MyBaseException): + ray.get(next(raise_at_middle_ref)) + + @ray.remote(_generator_backpressure_num_objects=1) + def raise_after_backpressure(): + for i in range(1, 10): + if i == 5: + raise MyBaseException("rip") + yield i + + raise_after_backpressure_ref = raise_after_backpressure.remote() + for i in range(1, 5): + assert i == ray.get(next(raise_after_backpressure_ref)) + with pytest.raises(MyBaseException): + ray.get(next(raise_after_backpressure_ref)) + + +def test_raise_system_exit(ray_start_regular_shared): + @ray.remote + def task(): + raise SystemExit("abc") + + with pytest.raises(WorkerCrashedError): + ray.get(task.remote()) + + +def test_raise_keyboard_interrupt(ray_start_regular_shared): + @ray.remote + def task(): + raise KeyboardInterrupt("abc") + + with pytest.raises(TaskCancelledError): + ray.get(task.remote()) + -pytestmark = pytest.mark.skipif( +skip_if_python_less_than_3_11 = pytest.mark.skipif( sys.version_info < (3, 11), reason="ExceptionGroup is only available in Python 3.11+", ) -def test_baseexceptiongroup_task(ray_start_regular): +@skip_if_python_less_than_3_11 +def test_baseexceptiongroup_task(ray_start_regular_shared): baseexceptiongroup = BaseExceptionGroup( # noqa: F821 "test baseexceptiongroup", [BaseException("abc")] ) @@ -25,7 +136,8 @@ def task(): ray.get(task.remote()) -def test_baseexceptiongroup_actor(ray_start_regular): +@skip_if_python_less_than_3_11 +def test_baseexceptiongroup_actor(ray_start_regular_shared): baseexceptiongroup = BaseExceptionGroup( # noqa: F821 "test baseexceptiongroup", [BaseException("abc")] ) @@ -40,7 +152,8 @@ def f(self): ray.get(a.f.remote()) -def test_except_exceptiongroup(ray_start_regular): +@skip_if_python_less_than_3_11 +def test_except_exceptiongroup(ray_start_regular_shared): exceptiongroup = ExceptionGroup( # noqa: F821 "test exceptiongroup", [ValueError(), TypeError()] ) @@ -74,7 +187,8 @@ def f(self): assert isinstance(ex.exceptions[1], TypeError) -def test_except_star_exception(ray_start_regular): +@skip_if_python_less_than_3_11 +def test_except_star_exception(ray_start_regular_shared): @ray.remote def task(): raise ValueError @@ -126,7 +240,8 @@ def f(self): exec(python_code) -def test_except_star_exceptiongroup(ray_start_regular): +@skip_if_python_less_than_3_11 +def test_except_star_exceptiongroup(ray_start_regular_shared): exceptiongroup = ExceptionGroup( # noqa: F821 "test exceptiongroup", [ValueError(), TypeError()] ) diff --git a/python/ray/tests/test_failure.py b/python/ray/tests/test_failure.py index 06de8f5dd70b..aa77f870e193 100644 --- a/python/ray/tests/test_failure.py +++ b/python/ray/tests/test_failure.py @@ -380,56 +380,6 @@ def foo(): assert isinstance(ex, RayTaskError) -def test_baseexception_task(ray_start_regular): - class MyBaseException(BaseException): - pass - - @ray.remote - def task(): - raise MyBaseException("abc") - - with pytest.raises(MyBaseException): - ray.get(task.remote()) - - -def test_baseexception_actor_task(ray_start_regular): - class MyBaseException(BaseException): - pass - - @ray.remote - class Actor: - def f(self): - raise MyBaseException("abc") - - async def async_f(self): - raise MyBaseException("abc") - - a = Actor.remote() - with pytest.raises(MyBaseException): - ray.get(a.f.remote()) - - a = Actor.remote() - with pytest.raises(MyBaseException): - ray.get(a.async_f.remote()) - - -def test_baseexception_actor_creation(ray_start_regular): - class MyBaseException(BaseException): - pass - - @ray.remote - class Actor: - def __init__(self): - raise MyBaseException("abc") - - a = Actor.remote() - try: - ray.get(a.__ray_ready__.remote()) - raise Exception("abc") - except ActorDiedError as e: - assert "MyBaseException" in str(e) - - @pytest.mark.skip("This test does not work yet.") @pytest.mark.parametrize("ray_start_object_store_memory", [10**6], indirect=True) def test_put_error1(ray_start_object_store_memory, error_pubsub): diff --git a/python/ray/tests/test_streaming_generator.py b/python/ray/tests/test_streaming_generator.py index 2e0136d91161..2e166e190ba5 100644 --- a/python/ray/tests/test_streaming_generator.py +++ b/python/ray/tests/test_streaming_generator.py @@ -576,48 +576,6 @@ async def async_f(self): ray.get(next(g)) -def test_baseexception_streaming_generator(shutdown_only): - ray.init() - - class MyBaseException(BaseException): - pass - - @ray.remote - def raise_at_beginning(): - raise MyBaseException("rip") - yield 1 - - raise_at_beginning_ref = raise_at_beginning.remote() - with pytest.raises(MyBaseException): - ray.get(next(raise_at_beginning_ref)) - - @ray.remote - def raise_at_middle(): - for i in range(1, 10): - if i == 5: - raise MyBaseException("rip") - yield i - - raise_at_middle_ref = raise_at_middle.remote() - for i in range(1, 5): - assert i == ray.get(next(raise_at_middle_ref)) - with pytest.raises(MyBaseException): - ray.get(next(raise_at_middle_ref)) - - @ray.remote(_generator_backpressure_num_objects=1) - def raise_after_backpressure(): - for i in range(1, 10): - if i == 5: - raise MyBaseException("rip") - yield i - - raise_after_backpressure_ref = raise_after_backpressure.remote() - for i in range(1, 5): - assert i == ray.get(next(raise_after_backpressure_ref)) - with pytest.raises(MyBaseException): - ray.get(next(raise_after_backpressure_ref)) - - if __name__ == "__main__": sys.exit(pytest.main(["-sv", __file__])) From be423b042d0370456a8abead58fd6502eeb6c6d4 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Mon, 18 Aug 2025 09:37:47 -0700 Subject: [PATCH 153/634] [ci] allowing spaces in append args field on depsets (3/4) (#55625) - Allowing for spaces in append args (splitting append arg flags before appending) - adding a couple unit tests --------- Signed-off-by: elliot-barn Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> --- ci/raydepsets/tests/test_cli.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 421a12b3c132..052a34e0de40 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -296,6 +296,36 @@ def test_get_path(self): == f"{tmpdir}/requirements_test.txt" ) + def test_append_uv_flags_exist_in_output(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = _create_test_manager(tmpdir) + manager.compile( + constraints=[], + requirements=["requirements_test.txt"], + name="general_depset", + output="requirements_compiled_general.txt", + append_flags=["--python-version=3.10"], + ) + output_file = Path(tmpdir) / "requirements_compiled_general.txt" + output_text = output_file.read_text() + assert "--python-version=3.10" in output_text + + def test_append_uv_flags_with_space_in_flag(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = _create_test_manager(tmpdir) + manager.compile( + constraints=[], + requirements=["requirements_test.txt"], + name="general_depset", + output="requirements_compiled_general.txt", + append_flags=["--python-version 3.10"], + ) + output_file = Path(tmpdir) / "requirements_compiled_general.txt" + output_text = output_file.read_text() + assert "--python-version 3.10" in output_text + def test_override_uv_flag_single_flag(self): expected_flags = DEFAULT_UV_FLAGS.copy() expected_flags.remove("--extra-index-url") From ae0d4fc04f7d56e77c080a24bf998a67a3e88631 Mon Sep 17 00:00:00 2001 From: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Date: Mon, 18 Aug 2025 09:58:37 -0700 Subject: [PATCH 154/634] [Serve] Update test_deploy_2.py with get_application_url (#55665) We remove the hardcoded url within the test to use `get_application_url()` --------- Signed-off-by: doyoung --- python/ray/serve/tests/test_deploy_2.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/python/ray/serve/tests/test_deploy_2.py b/python/ray/serve/tests/test_deploy_2.py index bf809d8d3ceb..62abad6082e0 100644 --- a/python/ray/serve/tests/test_deploy_2.py +++ b/python/ray/serve/tests/test_deploy_2.py @@ -266,13 +266,20 @@ def __call__(self): url = get_application_url("HTTP", app_name="app1") assert httpx.get(f"{url}").text == "hello alice" - proxy_url = "http://localhost:8000/-/routes" - routes = httpx.get(proxy_url).json() + url_without_route_prefix = get_application_url( + "HTTP", app_name="app1", exclude_route_prefix=True + ) + routes_url = f"{url_without_route_prefix}/-/routes" + routes = httpx.get(routes_url).json() assert routes["/app1"] == "app1" url = get_application_url("HTTP", app_name="app2") assert httpx.get(f"{url}").text == "hello bob" - routes = httpx.get(proxy_url).json() + url_without_route_prefix = get_application_url( + "HTTP", app_name="app2", exclude_route_prefix=True + ) + routes_url = f"{url_without_route_prefix}/-/routes" + routes = httpx.get(routes_url).json() assert routes["/app2"] == "app2" app1_status = serve.status().applications["app1"] From 796858a91c9a98b7785fdf012096b4a3e5f22cca Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Mon, 18 Aug 2025 19:10:31 +0200 Subject: [PATCH 155/634] [RLlib] Set default to 'log_gradients=False' to stabilize tests (#55695) ## Why are these changes needed? Right now `log_gradients` is by default `True` and this appears to destabilize tests (see #47717). This PR switches the default to `False`. ## Related issue number Closes #47717 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: simonsays1980 --- rllib/algorithms/algorithm_config.py | 4 ++-- .../examples/debugging/deterministic_sampling_and_training.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rllib/algorithms/algorithm_config.py b/rllib/algorithms/algorithm_config.py index 19e8801319af..98fe844d7a4a 100644 --- a/rllib/algorithms/algorithm_config.py +++ b/rllib/algorithms/algorithm_config.py @@ -565,7 +565,7 @@ def __init__(self, algo_class: Optional[type] = None): self.min_time_s_per_iteration = None self.min_train_timesteps_per_iteration = 0 self.min_sample_timesteps_per_iteration = 0 - self.log_gradients = True + self.log_gradients = False # `self.checkpointing()` self.export_native_model_files = False @@ -3653,7 +3653,7 @@ def reporting( executed. Set to 0 or None for no minimum timesteps. log_gradients: Log gradients to results. If this is `True` the global norm of the gradients dictionariy for each optimizer is logged to results. - The default is `True`. + The default is `False`. Returns: This updated AlgorithmConfig object. diff --git a/rllib/examples/debugging/deterministic_sampling_and_training.py b/rllib/examples/debugging/deterministic_sampling_and_training.py index 11319c4da112..219b5db07168 100644 --- a/rllib/examples/debugging/deterministic_sampling_and_training.py +++ b/rllib/examples/debugging/deterministic_sampling_and_training.py @@ -93,6 +93,8 @@ .environment("env" if args.num_agents > 0 else "CartPole-v1") # Make sure every environment gets a fixed seed. .debugging(seed=args.seed) + # Log gradients and check them in the test. + .reporting(log_gradients=True) ) # Add a simple multi-agent setup. From a81c7c9fbd5ed70e8aaae5cc2f1bc3e284ec8723 Mon Sep 17 00:00:00 2001 From: Timothy Seah Date: Mon, 18 Aug 2025 10:38:44 -0700 Subject: [PATCH 156/634] [train][tune] Train Controller is always actor + fix tune integration to enable this (#55556) In the past, we used `RUN_CONTROLLER_AS_ACTOR_ENV_VAR` to toggle whether to run the controller as a separate actor (we want this in most cases) or on the current actor (we wanted this in Tune so we can propagate `ray.train.report` from Train to Tune using the `TuneReportCallback`). However, in order to implement `get_all_reported_checkpoints` (https://github.com/ray-project/ray/pull/54555), we need to pass the Train Controller actor to all the Train Worker actors. This method wouldn't work when using Train from Tune because the Train Controller actor handle would be the Tune Trainable actor handle which does not have the async `get_all_reported_checkpoints` method. This PR gets rid of `RUN_CONTROLLER_AS_ACTOR_ENV_VAR` once and for all by making all communication between Train and Tune happen through a lightweight `ray.util.Queue` actor instead of forcing Train and Tune to happen on the same process. --------- Signed-off-by: Timothy Seah Co-authored-by: Timothy Seah --- .../train/doc_code/train_tune_interop.py | 3 +- python/ray/train/_internal/session.py | 44 +++++++++++++----- python/ray/train/tests/test_session.py | 42 +++++++++++++++++ python/ray/train/v2/_internal/constants.py | 4 -- .../ray/train/v2/api/data_parallel_trainer.py | 45 ++++++++----------- python/ray/tune/integration/ray_train.py | 12 ++++- .../ray/tune/trainable/function_trainable.py | 12 ----- 7 files changed, 105 insertions(+), 57 deletions(-) diff --git a/doc/source/train/doc_code/train_tune_interop.py b/doc/source/train/doc_code/train_tune_interop.py index 167528f33988..0b630cbe459d 100644 --- a/doc/source/train/doc_code/train_tune_interop.py +++ b/doc/source/train/doc_code/train_tune_interop.py @@ -65,7 +65,8 @@ def train_driver_fn(config: dict): # Launch a single Train run. -train_driver_fn({"num_workers": 4, "train_loop_config": {"lr": 1e-3}}) +# Note that you can only create a TuneReportCallback in a Ray Tune session. +# train_driver_fn({"num_workers": 4, "train_loop_config": {"lr": 1e-3}}) # Launch a sweep of hyperparameters with Ray Tune. diff --git a/python/ray/train/_internal/session.py b/python/ray/train/_internal/session.py index dec5b062ef4f..1bef98b15025 100644 --- a/python/ray/train/_internal/session.py +++ b/python/ray/train/_internal/session.py @@ -36,6 +36,7 @@ ) from ray.train.error import SessionMisuseError from ray.train.utils import _log_deprecation_warning +from ray.util import queue as ray_queue from ray.util.annotations import DeveloperAPI, PublicAPI from ray.util.debug import log_once from ray.util.placement_group import _valid_resource_shape @@ -205,6 +206,9 @@ def reset( # Queue for sending results across threads. self.result_queue = queue.Queue(1) + # Queue for sending results from training actor to main thread. + self._inter_actor_queue: Optional[ray_queue.Queue[Dict]] = None + # Queue for raising exceptions from runner thread to main thread. # The error queue has a max size of one to prevent stacking error and force # error reporting to block until finished. @@ -282,24 +286,14 @@ def get_next(self) -> Optional[_TrainingResult]: result = None # While training is still ongoing, attempt to get the result. while result is None and self.training_thread.is_alive(): - try: - result = self.result_queue.get( - block=True, timeout=_RESULT_FETCH_TIMEOUT - ) - except queue.Empty: - pass + result = self._get_result_from_queues(block=True) # If no result was found, then the runner must no longer be alive. if result is None: # Try one last time to fetch results in case results were # reported in between the time of the last check and the # termination of the thread runner. - try: - result = self.result_queue.get( - block=False, timeout=_RESULT_FETCH_TIMEOUT - ) - except queue.Empty: - pass + result = self._get_result_from_queues(block=False) # check if error occurred inside the thread runner. if result is None: @@ -325,6 +319,32 @@ def get_next(self) -> Optional[_TrainingResult]: # Return None if there are no more results to fetch. return result + def _get_or_create_inter_actor_queue(self): + """Get or create the inter-actor queue.""" + if self._inter_actor_queue is None: + self._inter_actor_queue = ray_queue.Queue(1, actor_options={"num_cpus": 0}) + return self._inter_actor_queue + + def _get_result_from_queues(self, block: bool) -> Optional[_TrainingResult]: + """Get result from result queue. Pass result from training actor result queue if needed.""" + result = None + if self._inter_actor_queue is not None: + try: + inter_actor_item = self._inter_actor_queue.get( + block=block, timeout=_RESULT_FETCH_TIMEOUT + ) + if inter_actor_item: + # Must release continue_lock to allow report to work. + self.continue_lock.release() + self.report(inter_actor_item) + except ray_queue.Empty: + pass + try: + result = self.result_queue.get(block=block, timeout=_RESULT_FETCH_TIMEOUT) + except queue.Empty: + pass + return result + def _auto_fill_metrics(self, result: dict) -> dict: """Add autofilled metrics and update attributes.""" current_time = time.time() diff --git a/python/ray/train/tests/test_session.py b/python/ray/train/tests/test_session.py index c70e4ecbbd8e..071aeeaa18af 100644 --- a/python/ray/train/tests/test_session.py +++ b/python/ray/train/tests/test_session.py @@ -17,6 +17,7 @@ ) from ray.train._internal.accelerator import Accelerator from ray.train._internal.session import ( + _TrainingResult, get_accelerator, get_session, init_session, @@ -170,6 +171,47 @@ def test_report_after_finish(session): shutdown_session() +@pytest.mark.parametrize( + "block,put_result_queue,put_actor_queue", + [ + (False, False, False), + (False, False, True), + (False, True, False), + (True, False, False), + (True, False, True), + (True, True, False), + ], +) +def test_get_result_from_queues(session, block, put_result_queue, put_actor_queue): + """Verify that we get the expected _TrainingResult from each result queue. + + `block` describes whether we wait for a result or return after a timeout. + This argument should have no impact on this unit test. + `put_result_queue` and `put_actor_queue` are mutually exclusive and describe + which queue has results to process. The returned _TrainingResult should be + from the expected queue. + """ + result_queue_training_result = _TrainingResult( + checkpoint=None, + metrics={"result_queue_metric_key": "result_queue_metric_value"}, + ) + if put_result_queue: + session.result_queue.put(result_queue_training_result, block=True) + inter_actor_result = {"inter_actor_metric_key": "inter_actor_metric_value"} + if put_actor_queue: + session._get_or_create_inter_actor_queue().put(inter_actor_result, block=True) + result = session._get_result_from_queues(block=block) + if put_result_queue: + assert result == result_queue_training_result + elif put_actor_queue: + assert ( + result.metrics["inter_actor_metric_key"] + == inter_actor_result["inter_actor_metric_key"] + ) + else: + assert result is None + + def test_no_start(session): with pytest.raises(RuntimeError): session.get_next() diff --git a/python/ray/train/v2/_internal/constants.py b/python/ray/train/v2/_internal/constants.py index 65a5eef0b2fc..c71e3e48b468 100644 --- a/python/ray/train/v2/_internal/constants.py +++ b/python/ray/train/v2/_internal/constants.py @@ -103,10 +103,6 @@ # The environment variable to enable the Ray Train Metrics. METRICS_ENABLED_ENV_VAR = "RAY_TRAIN_METRICS_ENABLED" -# Whether or not to run the controller as an actor. -RUN_CONTROLLER_AS_ACTOR_ENV_VAR = "RAY_TRAIN_RUN_CONTROLLER_AS_ACTOR" -DEFAULT_RUN_CONTROLLER_AS_ACTOR = True - def is_v2_enabled() -> bool: return env_bool(V2_ENABLED_ENV_VAR, False) diff --git a/python/ray/train/v2/api/data_parallel_trainer.py b/python/ray/train/v2/api/data_parallel_trainer.py index 369d8762e87d..6eef7c9bb247 100644 --- a/python/ray/train/v2/api/data_parallel_trainer.py +++ b/python/ray/train/v2/api/data_parallel_trainer.py @@ -1,7 +1,7 @@ -import asyncio import logging import signal import sys +import threading from typing import Any, Callable, Dict, List, Optional, Union import ray @@ -39,9 +39,7 @@ from ray.train.v2._internal.callbacks.state_manager import StateManagerCallback from ray.train.v2._internal.callbacks.user_callback import UserCallbackHandler from ray.train.v2._internal.constants import ( - DEFAULT_RUN_CONTROLLER_AS_ACTOR, METRICS_ENABLED_ENV_VAR, - RUN_CONTROLLER_AS_ACTOR_ENV_VAR, get_env_vars_to_propagate, ) from ray.train.v2._internal.execution.callback import RayTrainCallback @@ -197,31 +195,26 @@ def _create_default_callbacks(self) -> List[RayTrainCallback]: return callbacks def _initialize_and_run_controller(self, **controller_init_kwargs) -> Result: - run_controller_as_actor = env_bool( - RUN_CONTROLLER_AS_ACTOR_ENV_VAR, DEFAULT_RUN_CONTROLLER_AS_ACTOR - ) - if run_controller_as_actor: - # Attach the controller to the node running the driver script. - controller_actor_cls = ray.remote( - num_cpus=0, - scheduling_strategy=NodeAffinitySchedulingStrategy( - node_id=ray.get_runtime_context().get_node_id(), soft=False - ), - # TODO: Extract env variables that affect controller behavior - # and pass them as explicit args - runtime_env={"env_vars": get_env_vars_to_propagate()}, - )(TrainController) - - controller = controller_actor_cls.remote(**controller_init_kwargs) - + # Attach the controller to the node running the driver script. + controller_actor_cls = ray.remote( + num_cpus=0, + scheduling_strategy=NodeAffinitySchedulingStrategy( + node_id=ray.get_runtime_context().get_node_id(), soft=False + ), + # TODO: Extract env variables that affect controller behavior + # and pass them as explicit args + runtime_env={"env_vars": get_env_vars_to_propagate()}, + )(TrainController) + + controller = controller_actor_cls.remote(**controller_init_kwargs) + + # If this is not the main thread - as is the case when running in Tune - + # registering the SIGINT handler raises an exception. + if threading.current_thread() is threading.main_thread(): self._register_sigint_handler(controller) - ray.get(controller.run.remote()) - return ray.get(controller.get_result.remote()) - else: - controller = TrainController(**controller_init_kwargs) - asyncio.run(controller.run()) - return controller.get_result() + ray.get(controller.run.remote()) + return ray.get(controller.get_result.remote()) def _register_sigint_handler(self, controller: ActorHandle[TrainController]): """Register SIGINT handler so user Ctrl C gracefully aborts run.""" diff --git a/python/ray/tune/integration/ray_train.py b/python/ray/tune/integration/ray_train.py index f8dfc60d3abf..89302a65869a 100644 --- a/python/ray/tune/integration/ray_train.py +++ b/python/ray/tune/integration/ray_train.py @@ -1,9 +1,10 @@ from typing import Any, Dict, List, Optional -import ray.tune from ray.train import Checkpoint as RayTrainCheckpoint +from ray.train._internal.session import get_session from ray.train.v2._internal.execution.context import TrainRunContext from ray.train.v2.api.callback import UserCallback +from ray.tune.trainable.trainable_fn_utils import _in_tune_session from ray.util.annotations import DeveloperAPI CHECKPOINT_PATH_KEY = "checkpoint_path" @@ -13,6 +14,13 @@ class TuneReportCallback(UserCallback): """Propagate metrics and checkpoint paths from Ray Train workers to Ray Tune.""" + def __init__(self): + if not _in_tune_session(): + raise RuntimeError("TuneReportCallback must be used in a Tune session.") + self._training_actor_item_queue = ( + get_session()._get_or_create_inter_actor_queue() + ) + def after_report( self, run_context: TrainRunContext, @@ -29,4 +37,4 @@ def after_report( if checkpoint: metrics[CHECKPOINT_PATH_KEY] = checkpoint.path - ray.tune.report(metrics=metrics) + self._training_actor_item_queue.put(metrics) diff --git a/python/ray/tune/trainable/function_trainable.py b/python/ray/tune/trainable/function_trainable.py index 9dc9ff02cbfd..e7110275d2c0 100644 --- a/python/ray/tune/trainable/function_trainable.py +++ b/python/ray/tune/trainable/function_trainable.py @@ -16,7 +16,6 @@ init_session, shutdown_session, ) -from ray.train.v2._internal.constants import RUN_CONTROLLER_AS_ACTOR_ENV_VAR from ray.tune.execution.placement_groups import PlacementGroupFactory from ray.tune.result import DEFAULT_METRIC, RESULT_DUPLICATE, SHOULD_CHECKPOINT from ray.tune.trainable.trainable import Trainable @@ -65,17 +64,6 @@ def setup(self, config): ) self._last_training_result: Optional[_TrainingResult] = None - # NOTE: This environment variable is used to disable the - # spawning a new actor for Ray Train drivers being launched - # within Tune functions. - # There are 2 reasons for this: - # 1. Ray Tune already spawns an actor, so we can run the Ray Train - # driver directly in the same actor. - # 2. This allows `ray.tune.report` to be called within Ray Train driver - # callbacks, since it needs to be called on the same process as the - # Tune FunctionTrainable actor. - os.environ[RUN_CONTROLLER_AS_ACTOR_ENV_VAR] = "0" - def _trainable_func(self, config: Dict[str, Any]): """Subclasses can override this to set the trainable func.""" From 6c90c0de34f5b3f618db076c4f3197f78aefc8bf Mon Sep 17 00:00:00 2001 From: yi wang <48236141+my-vegetable-has-exploded@users.noreply.github.com> Date: Tue, 19 Aug 2025 02:00:27 +0800 Subject: [PATCH 157/634] [Data] explain API for dataset (#55482) ## Why are these changes needed? Introduce explain() for dataset, which output logical plan and physical plan. ## Related issue number part of #55052 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [x] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: my-vegetable-has-exploded Signed-off-by: Richard Liaw Co-authored-by: Richard Liaw --- python/ray/data/_internal/plan.py | 81 +++++++++++++++-------- python/ray/data/dataset.py | 26 ++++++++ python/ray/data/tests/test_consumption.py | 44 ++++++++++++ 3 files changed, 122 insertions(+), 29 deletions(-) diff --git a/python/ray/data/_internal/plan.py b/python/ray/data/_internal/plan.py index 148d144d6d10..a471949d89e3 100644 --- a/python/ray/data/_internal/plan.py +++ b/python/ray/data/_internal/plan.py @@ -11,6 +11,7 @@ from ray.data._internal.logical.interfaces import SourceOperator from ray.data._internal.logical.interfaces.logical_operator import LogicalOperator from ray.data._internal.logical.interfaces.logical_plan import LogicalPlan +from ray.data._internal.logical.interfaces.operator import Operator from ray.data._internal.logical.operators.read_operator import Read from ray.data._internal.stats import DatasetStats from ray.data._internal.util import unify_ref_bundles_schema @@ -111,6 +112,54 @@ def __repr__(self) -> str: f")" ) + def explain(self) -> str: + """Return a string representation of the logical and physical plan.""" + from ray.data._internal.logical.optimizers import get_execution_plan + + logical_plan = self._logical_plan + logical_plan_str, _ = self.generate_plan_string(logical_plan.dag) + logical_plan_str = "-------- Logical Plan --------\n" + logical_plan_str + + physical_plan = get_execution_plan(self._logical_plan) + physical_plan_str, _ = self.generate_plan_string( + physical_plan.dag, show_op_repr=True + ) + physical_plan_str = "-------- Physical Plan --------\n" + physical_plan_str + + return logical_plan_str + physical_plan_str + + @staticmethod + def generate_plan_string( + op: Operator, + curr_str: str = "", + depth: int = 0, + including_source: bool = True, + show_op_repr: bool = False, + ): + """Traverse (DFS) the Plan DAG and + return a string representation of the operators.""" + if not including_source and isinstance(op, SourceOperator): + return curr_str, depth + + curr_max_depth = depth + + # For logical plan, only show the operator name like "Aggregate". + # But for physical plan, show the operator class name as well like "AllToAllOperator[Aggregate]". + op_str = repr(op) if show_op_repr else op.name + + if depth == 0: + curr_str += f"{op_str}\n" + else: + trailing_space = " " * ((depth - 1) * 3) + curr_str += f"{trailing_space}+- {op_str}\n" + + for input in op.input_dependencies: + curr_str, input_max_depth = ExecutionPlan.generate_plan_string( + input, curr_str, depth + 1, including_source, show_op_repr + ) + curr_max_depth = max(curr_max_depth, input_max_depth) + return curr_str, curr_max_depth + def get_plan_as_string(self, dataset_cls: Type["Dataset"]) -> str: """Create a cosmetic string representation of this execution plan. @@ -128,35 +177,9 @@ def get_plan_as_string(self, dataset_cls: Type["Dataset"]) -> str: plan_str = "" plan_max_depth = 0 if not self.has_computed_output(): - - def generate_logical_plan_string( - op: LogicalOperator, - curr_str: str = "", - depth: int = 0, - ): - """Traverse (DFS) the LogicalPlan DAG and - return a string representation of the operators.""" - if isinstance(op, SourceOperator): - return curr_str, depth - - curr_max_depth = depth - op_name = op.name - if depth == 0: - curr_str += f"{op_name}\n" - else: - trailing_space = " " * ((depth - 1) * 3) - curr_str += f"{trailing_space}+- {op_name}\n" - - for input in op.input_dependencies: - curr_str, input_max_depth = generate_logical_plan_string( - input, curr_str, depth + 1 - ) - curr_max_depth = max(curr_max_depth, input_max_depth) - return curr_str, curr_max_depth - - # generate_logical_plan_string(self._logical_plan.dag) - plan_str, plan_max_depth = generate_logical_plan_string( - self._logical_plan.dag + # using dataset as source here, so don't generate source operator in generate_plan_string + plan_str, plan_max_depth = self.generate_plan_string( + self._logical_plan.dag, including_source=False ) if self._snapshot_bundle is not None: diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 8766434c425a..2d619381c66f 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -5965,6 +5965,32 @@ def stats(self) -> str: return self._write_ds.stats() return self._get_stats_summary().to_string() + @PublicAPI(api_group=IM_API_GROUP, stability="alpha") + def explain(self): + """Show the logical plan and physical plan of the dataset. + + Examples: + + .. testcode:: + + import ray + from ray.data import Dataset + ds: Dataset = ray.data.range(10, override_num_blocks=10) + ds = ds.map(lambda x: x + 1) + ds.explain() + + .. testoutput:: + + -------- Logical Plan -------- + Map() + +- ReadRange + -------- Physical Plan -------- + TaskPoolMapOperator[ReadRange->Map()] + +- InputDataBuffer[Input] + + """ + print(self._plan.explain()) + def _get_stats_summary(self) -> DatasetStatsSummary: return self._plan.stats().to_summary() diff --git a/python/ray/data/tests/test_consumption.py b/python/ray/data/tests/test_consumption.py index 4176bce5aa7b..c5b4fd3f5b65 100644 --- a/python/ray/data/tests/test_consumption.py +++ b/python/ray/data/tests/test_consumption.py @@ -535,6 +535,50 @@ def my_dummy_fn(x): ) +def test_dataset_explain(ray_start_regular_shared, capsys): + ds = ray.data.range(10, override_num_blocks=10) + ds = ds.map(lambda x: x) + + ds.explain() + captured = capsys.readouterr() + assert captured.out.rstrip() == ( + "-------- Logical Plan --------\n" + "Map()\n" + "+- ReadRange\n" + "-------- Physical Plan --------\n" + "TaskPoolMapOperator[ReadRange->Map()]\n" + "+- InputDataBuffer[Input]" + ) + + ds = ds.filter(lambda x: x["id"] > 0) + ds.explain() + captured = capsys.readouterr() + assert captured.out.rstrip() == ( + "-------- Logical Plan --------\n" + "Filter()\n" + "+- Map()\n" + " +- ReadRange\n" + "-------- Physical Plan --------\n" + "TaskPoolMapOperator[ReadRange->Map()->Filter()]\n" + "+- InputDataBuffer[Input]" + ) + ds = ds.random_shuffle().map(lambda x: x) + ds.explain() + captured = capsys.readouterr() + assert captured.out.rstrip() == ( + "-------- Logical Plan --------\n" + "Map()\n" + "+- RandomShuffle\n" + " +- Filter()\n" + " +- Map()\n" + " +- ReadRange\n" + "-------- Physical Plan --------\n" + "TaskPoolMapOperator[Map()]\n" + "+- AllToAllOperator[ReadRange->Map()->Filter()->RandomShuffle]\n" + " +- InputDataBuffer[Input]" + ) + + @pytest.mark.parametrize("lazy", [False, True]) def test_limit(ray_start_regular_shared, lazy): ds = ray.data.range(100, override_num_blocks=20) From b95fc3e0757a89dea38f243c9a29f3768f82b98f Mon Sep 17 00:00:00 2001 From: Sampan S Nayak Date: Mon, 18 Aug 2025 23:47:39 +0530 Subject: [PATCH 158/634] [core] Add logic to convert TaskProfileEvent to RayEvent before sending to event aggregator (#55138) As part of oneEvent effort, all individual task event objects (such as task definition event, task execution event, etc) are being consolidated under one type: RayEvent. This pr adds the translation logic to convert the `TaskProfileEvent` ->` rpc::events::RayEvent object` + tests to verify that the translation and subsequent section of the `TaskEventBufferImpl` correctly deal with the constructed RayEvent. Signed-off-by: sampan Signed-off-by: Sampan S Nayak Co-authored-by: sampan Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Mengjin Yan --- src/ray/core_worker/profile_event.cc | 3 +- src/ray/core_worker/task_event_buffer.cc | 55 ++++- src/ray/core_worker/task_event_buffer.h | 19 +- .../tests/scheduling_queue_test.cc | 2 + .../tests/task_receiver_test.cc | 2 + .../tests/task_event_buffer_test.cc | 202 +++++++++++++++++- .../core_worker/tests/task_manager_test.cc | 2 + 7 files changed, 272 insertions(+), 13 deletions(-) diff --git a/src/ray/core_worker/profile_event.cc b/src/ray/core_worker/profile_event.cc index a6c8348dd8b7..6da5ec40c0c0 100644 --- a/src/ray/core_worker/profile_event.cc +++ b/src/ray/core_worker/profile_event.cc @@ -48,7 +48,8 @@ ProfileEvent::ProfileEvent(TaskEventBuffer &task_event_buffer, worker_context.GetWorkerID().Binary(), node_ip_address, event_name, - absl::GetCurrentTimeNanos()); + absl::GetCurrentTimeNanos(), + task_event_buffer_.GetSessionName()); } ProfileEvent::~ProfileEvent() { diff --git a/src/ray/core_worker/task_event_buffer.cc b/src/ray/core_worker/task_event_buffer.cc index bfc85faad1f7..e3ead75ec489 100644 --- a/src/ray/core_worker/task_event_buffer.cc +++ b/src/ray/core_worker/task_event_buffer.cc @@ -55,13 +55,15 @@ TaskProfileEvent::TaskProfileEvent(TaskID task_id, std::string component_id, std::string node_ip_address, std::string event_name, - int64_t start_time) + int64_t start_time, + std::string session_name) : TaskEvent(task_id, job_id, attempt_number), component_type_(std::move(component_type)), component_id_(std::move(component_id)), node_ip_address_(std::move(node_ip_address)), event_name_(std::move(event_name)), - start_time_(start_time) {} + start_time_(start_time), + session_name_(session_name) {} void TaskStatusEvent::ToRpcTaskEvents(rpc::TaskEvents *rpc_task_events) { // Base fields @@ -342,9 +344,42 @@ void TaskProfileEvent::ToRpcTaskExportEvents( event_entry->set_extra_data(std::move(extra_data_)); } +void TaskProfileEvent::PopulateRpcRayEventBaseFields( + rpc::events::RayEvent &ray_event, google::protobuf::Timestamp timestamp) { + ray_event.set_event_id(UniqueID::FromRandom().Binary()); + ray_event.set_source_type(rpc::events::RayEvent::CORE_WORKER); + ray_event.mutable_timestamp()->CopyFrom(timestamp); + ray_event.set_severity(rpc::events::RayEvent::INFO); + ray_event.set_event_type(rpc::events::RayEvent::TASK_PROFILE_EVENT); + ray_event.set_session_name(session_name_); +} + void TaskProfileEvent::ToRpcRayEvents(RayEventsPair &ray_events_pair) { - // TODO(myan): #54515 need to further figure out how to migrate the task profile event - // to the new ray event format. + auto &[task_profile_Event, null_event] = ray_events_pair; + // Second element of the RayEventsPair will always be empty for TaskProfileEvent + null_event = std::nullopt; + + // Using profile start time as the event generation timestamp + google::protobuf::Timestamp timestamp = AbslTimeNanosToProtoTimestamp(start_time_); + + // Populate Ray event base fields + auto &ray_event = task_profile_Event.emplace(); + PopulateRpcRayEventBaseFields(ray_event, timestamp); + + // Populate the task profile event + auto *task_profile_events = ray_event.mutable_task_profile_events(); + task_profile_events->set_task_id(task_id_.Binary()); + task_profile_events->set_job_id(job_id_.Binary()); + task_profile_events->set_attempt_number(attempt_number_); + auto profile_events = task_profile_events->mutable_profile_events(); + profile_events->set_component_type(component_type_); + profile_events->set_component_id(component_id_); + profile_events->set_node_ip_address(node_ip_address_); + auto event_entry = profile_events->add_events(); + event_entry->set_event_name(event_name_); + event_entry->set_start_time(start_time_); + event_entry->set_end_time(end_time_); + event_entry->set_extra_data(std::move(extra_data_)); } bool TaskEventBufferImpl::RecordTaskStatusEventIfNeeded( @@ -590,14 +625,16 @@ TaskEventBufferImpl::CreateRayEventsDataToSend( auto data = std::make_unique(); // Move the ray events. for (auto &[task_attempt, ray_events_pair] : agg_task_events) { - auto &[task_definition_event_rpc, task_execution_event_rpc] = ray_events_pair; - if (task_definition_event_rpc) { + // For TaskStatusEvent: first = task definition event, second = task execution event + // For TaskProfileEvent: first = task profile event, second = nullopt (empty) + auto &[first_event, second_event] = ray_events_pair; + if (first_event) { auto events = data->add_events(); - *events = std::move(task_definition_event_rpc.value()); + *events = std::move(first_event.value()); } - if (task_execution_event_rpc) { + if (second_event) { auto events = data->add_events(); - *events = std::move(task_execution_event_rpc.value()); + *events = std::move(second_event.value()); } } diff --git a/src/ray/core_worker/task_event_buffer.h b/src/ray/core_worker/task_event_buffer.h index 5d68e1dc50db..eaf6511b3b33 100644 --- a/src/ray/core_worker/task_event_buffer.h +++ b/src/ray/core_worker/task_event_buffer.h @@ -44,6 +44,8 @@ using TaskAttempt = std::pair; /// A pair of rpc::events::RayEvent. /// When converting the TaskStatusEvent, the pair will be populated with the /// rpc::events::TaskDefinitionEvent and rpc::events::TaskExecutionEvent respectively. +/// When converting the TaskProfileEvent, only the first element of the pair will be +/// populated with rpc::events::TaskProfileEvents using RayEventsPair = std::pair, std::optional>; @@ -213,13 +215,16 @@ class TaskProfileEvent : public TaskEvent { std::string component_id, std::string node_ip_address, std::string event_name, - int64_t start_time); + int64_t start_time, + std::string session_name); void ToRpcTaskEvents(rpc::TaskEvents *rpc_task_events) override; void ToRpcTaskExportEvents( std::shared_ptr rpc_task_export_event_data) override; + /// Note: The extra data will be moved when this is called and will no longer be usable. + /// Second element of the RayEventsPair will always be empty for TaskProfileEvent. void ToRpcRayEvents(RayEventsPair &ray_events) override; bool IsProfileEvent() const override { return true; } @@ -229,6 +234,9 @@ class TaskProfileEvent : public TaskEvent { void SetExtraData(const std::string &extra_data) { extra_data_ = extra_data; } private: + // Helper functions to populate the base fields of rpc::events::RayEvent + void PopulateRpcRayEventBaseFields(rpc::events::RayEvent &ray_event, + google::protobuf::Timestamp timestamp); /// The below fields mirror rpc::ProfileEvent std::string component_type_; std::string component_id_; @@ -237,6 +245,8 @@ class TaskProfileEvent : public TaskEvent { int64_t start_time_{}; int64_t end_time_{}; std::string extra_data_; + /// The current Ray session name. + std::string session_name_; }; /// @brief An enum class defining counters to be used in TaskEventBufferImpl. @@ -351,6 +361,9 @@ class TaskEventBuffer { /// Return a string that describes the task event buffer stats. virtual std::string DebugString() = 0; + + /// Return the current Ray session name. + virtual std::string GetSessionName() const = 0; }; /// Implementation of TaskEventBuffer. @@ -397,6 +410,8 @@ class TaskEventBufferImpl : public TaskEventBuffer { std::string DebugString() override; + std::string GetSessionName() const override { return session_name_; } + private: /// Add a task status event to be reported. /// @@ -591,6 +606,8 @@ class TaskEventBufferImpl : public TaskEventBuffer { FRIEND_TEST(TaskEventBufferTestLimitProfileEvents, TestBufferSizeLimitProfileEvents); FRIEND_TEST(TaskEventBufferTestLimitProfileEvents, TestLimitProfileEventsPerTask); FRIEND_TEST(TaskEventTestWriteExport, TestWriteTaskExportEvents); + FRIEND_TEST(TaskEventBufferTest, TestCreateRayEventsDataWithProfileEvents); + FRIEND_TEST(TaskEventBufferTest, TestMixedStatusAndProfileEventsToRayEvents); }; } // namespace worker diff --git a/src/ray/core_worker/task_execution/tests/scheduling_queue_test.cc b/src/ray/core_worker/task_execution/tests/scheduling_queue_test.cc index 32752abc4ee3..4eff53179edf 100644 --- a/src/ray/core_worker/task_execution/tests/scheduling_queue_test.cc +++ b/src/ray/core_worker/task_execution/tests/scheduling_queue_test.cc @@ -83,6 +83,8 @@ class MockTaskEventBuffer : public worker::TaskEventBuffer { return true; } + std::string GetSessionName() const override { return "test-session-name"; } + std::vector> task_events; }; diff --git a/src/ray/core_worker/task_execution/tests/task_receiver_test.cc b/src/ray/core_worker/task_execution/tests/task_receiver_test.cc index ee1d2bfa7d8b..647be3564063 100644 --- a/src/ray/core_worker/task_execution/tests/task_receiver_test.cc +++ b/src/ray/core_worker/task_execution/tests/task_receiver_test.cc @@ -120,6 +120,8 @@ class MockTaskEventBuffer : public worker::TaskEventBuffer { } std::string DebugString() override { return ""; } + + std::string GetSessionName() const override { return "test-session-name"; } }; class TaskReceiverTest : public ::testing::Test { diff --git a/src/ray/core_worker/tests/task_event_buffer_test.cc b/src/ray/core_worker/tests/task_event_buffer_test.cc index b57467f5e3d2..7616e6dac332 100644 --- a/src/ray/core_worker/tests/task_event_buffer_test.cc +++ b/src/ray/core_worker/tests/task_event_buffer_test.cc @@ -180,8 +180,15 @@ class TaskEventBufferTest : public ::testing::Test { } std::unique_ptr GenProfileTaskEvent(TaskID task_id, int32_t attempt_num) { - return std::make_unique( - task_id, JobID::FromInt(0), attempt_num, "", "", "", "test_event", 1); + return std::make_unique(task_id, + JobID::FromInt(0), + attempt_num, + "", + "", + "", + "test_event", + 1, + "test_session_name"); } static void CompareTaskEventData(const rpc::TaskEventData &actual_data, @@ -917,6 +924,197 @@ TEST_F(TaskEventBufferTest, TestGracefulDestruction) { delete task_event_buffer_.release(); } +TEST_F(TaskEventBufferTest, TestTaskProfileEventToRpcRayEvents) { + auto task_id = RandomTaskId(); + auto job_id = JobID::FromInt(123); + int32_t attempt_number = 1; + std::string component_type = "core_worker"; + std::string component_id = "worker_123"; + std::string node_ip = "192.168.1.1"; + std::string event_name = "test_profile_event"; + int64_t start_time = 1000; + + auto profile_event = std::make_unique(task_id, + job_id, + attempt_number, + component_type, + component_id, + node_ip, + event_name, + start_time, + "test_session_name"); + + // Set end time and extra data to test full population + profile_event->SetEndTime(2000); + profile_event->SetExtraData("test_extra_data"); + + RayEventsPair ray_events_pair; + profile_event->ToRpcRayEvents(ray_events_pair); + + auto &[first_event, second_event] = ray_events_pair; + + // Verify that the second event is nullopt (empty) + EXPECT_FALSE(second_event.has_value()) + << "TaskProfileEvent should set second element of RayEventsPair to nullopt"; + + // Verify that the first event contains the profile event + ASSERT_TRUE(first_event.has_value()) + << "TaskProfileEvent should populate first element of RayEventsPair"; + + const auto &ray_event = first_event.value(); + + // Verify base fields + EXPECT_EQ(ray_event.source_type(), rpc::events::RayEvent::CORE_WORKER); + EXPECT_EQ(ray_event.event_type(), rpc::events::RayEvent::TASK_PROFILE_EVENT); + EXPECT_EQ(ray_event.severity(), rpc::events::RayEvent::INFO); + EXPECT_FALSE(ray_event.event_id().empty()); + EXPECT_EQ(ray_event.session_name(), "test_session_name"); + + // Verify task profile events are populated + ASSERT_TRUE(ray_event.has_task_profile_events()); + const auto &task_profile_events = ray_event.task_profile_events(); + + EXPECT_EQ(task_profile_events.task_id(), task_id.Binary()); + EXPECT_EQ(task_profile_events.job_id(), job_id.Binary()); + EXPECT_EQ(task_profile_events.attempt_number(), attempt_number); + + // Verify profile event + ASSERT_TRUE(task_profile_events.has_profile_events()); + const auto &profile_events = task_profile_events.profile_events(); + + EXPECT_EQ(profile_events.component_type(), component_type); + EXPECT_EQ(profile_events.component_id(), component_id); + EXPECT_EQ(profile_events.node_ip_address(), node_ip); + + // Verify event entry + ASSERT_EQ(profile_events.events_size(), 1); + const auto &event_entry = profile_events.events(0); + + EXPECT_EQ(event_entry.event_name(), event_name); + EXPECT_EQ(event_entry.start_time(), start_time); + EXPECT_EQ(event_entry.end_time(), 2000); + EXPECT_EQ(event_entry.extra_data(), "test_extra_data"); +} + +TEST_F(TaskEventBufferTest, TestCreateRayEventsDataWithProfileEvents) { + // Test that CreateRayEventsDataToSend correctly handles profile events + // by only including the first element of RayEventsPair + + auto task_id = RandomTaskId(); + auto job_id = JobID::FromInt(456); + int32_t attempt_number = 2; + + // Create a profile event + auto profile_event = std::make_unique(task_id, + job_id, + attempt_number, + "core_worker", + "worker_456", + "192.168.1.2", + "profile_test", + 5000, + "test_session_name"); + profile_event->SetEndTime(6000); + + absl::flat_hash_map agg_ray_events; + TaskAttempt task_attempt = std::make_pair(task_id, attempt_number); + + // Populate the ray events pair + RayEventsPair ray_events_pair; + profile_event->ToRpcRayEvents(ray_events_pair); + agg_ray_events[task_attempt] = std::move(ray_events_pair); + + // Create the data using the real implementation + absl::flat_hash_set dropped_task_attempts; + auto ray_events_data = task_event_buffer_->CreateRayEventsDataToSend( + std::move(agg_ray_events), dropped_task_attempts); + + // Verify that exactly one event was added (only the profile event, not the nullopt + // second) + ASSERT_EQ(ray_events_data->events_size(), 1); + + const auto &event = ray_events_data->events(0); + EXPECT_EQ(event.event_type(), rpc::events::RayEvent::TASK_PROFILE_EVENT); + EXPECT_EQ(event.session_name(), "test_session_name"); + EXPECT_TRUE(event.has_task_profile_events()); + + const auto &task_profile_events = event.task_profile_events(); + EXPECT_EQ(task_profile_events.task_id(), task_id.Binary()); + EXPECT_EQ(task_profile_events.job_id(), job_id.Binary()); + EXPECT_EQ(task_profile_events.attempt_number(), attempt_number); +} + +TEST_F(TaskEventBufferTest, TestMixedStatusAndProfileEventsToRayEvents) { + // Test that a mix of status events and profile events are correctly handled + auto task_id1 = RandomTaskId(); + auto task_id2 = RandomTaskId(); + auto job_id = JobID::FromInt(789); + + // Create a status event (should populate both elements of RayEventsPair) + auto status_event = GenStatusTaskEvent(task_id1, 1, 1000); + + // Create a profile event (should populate only first element) + auto profile_event = std::make_unique(task_id2, + job_id, + 1, + "core_worker", + "worker_789", + "192.168.1.3", + "mixed_test", + 7000, + "test_session_name"); + + // Create aggregated events + absl::flat_hash_map agg_ray_events; + + // Add status event + RayEventsPair status_ray_events_pair; + status_event->ToRpcRayEvents(status_ray_events_pair); + agg_ray_events[std::make_pair(task_id1, 1)] = std::move(status_ray_events_pair); + + // Add profile event + RayEventsPair profile_ray_events_pair; + profile_event->ToRpcRayEvents(profile_ray_events_pair); + agg_ray_events[std::make_pair(task_id2, 1)] = std::move(profile_ray_events_pair); + + // Create the data + absl::flat_hash_set dropped_task_attempts; + auto ray_events_data = task_event_buffer_->CreateRayEventsDataToSend( + std::move(agg_ray_events), dropped_task_attempts); + + // Should have 2 events: 1 from status event (execution only since no task_spec) + 1 + // from profile event + ASSERT_EQ(ray_events_data->events_size(), 2); + + // Count event types + int task_definition_events = 0; + int task_execution_events = 0; + int task_profile_events = 0; + + for (const auto &event : ray_events_data->events()) { + switch (event.event_type()) { + case rpc::events::RayEvent::TASK_DEFINITION_EVENT: + task_definition_events++; + break; + case rpc::events::RayEvent::TASK_EXECUTION_EVENT: + task_execution_events++; + break; + case rpc::events::RayEvent::TASK_PROFILE_EVENT: + task_profile_events++; + break; + default: + FAIL() << "Unexpected event type: " << event.event_type(); + } + } + + EXPECT_EQ(task_definition_events, 0) + << "Should have 0 task definition events since GenStatusTaskEvent has no task_spec"; + EXPECT_EQ(task_execution_events, 1) + << "Should have 1 task execution event from status event"; + EXPECT_EQ(task_profile_events, 1) + << "Should have 1 task profile event from profile event"; +} + INSTANTIATE_TEST_SUITE_P(TaskEventBufferTest, TaskEventBufferTestDifferentDestination, ::testing::Values(DifferentDestination{true, true}, diff --git a/src/ray/core_worker/tests/task_manager_test.cc b/src/ray/core_worker/tests/task_manager_test.cc index 3830d9821474..826f4d3511c4 100644 --- a/src/ray/core_worker/tests/task_manager_test.cc +++ b/src/ray/core_worker/tests/task_manager_test.cc @@ -132,6 +132,8 @@ class MockTaskEventBuffer : public worker::TaskEventBuffer { bool include_task_info, std::optional state_update), (override)); + + MOCK_METHOD(std::string, GetSessionName, (), (const, override)); }; class TaskManagerTest : public ::testing::Test { From 81856dfad0ab26dffc5d9209ae297f8acd16ce9a Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 18 Aug 2025 11:28:28 -0700 Subject: [PATCH 159/634] [wheel] when `RAY_DISABLE_EXTRA_CPP=1`, do not build cpp stuff (#55697) this gives us a way to safely skip the ray-cpp building parts when building ray wheel. Signed-off-by: Lonnie Liu --- python/setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index 82a08fda81f3..8918efef130b 100644 --- a/python/setup.py +++ b/python/setup.py @@ -29,6 +29,7 @@ ROOT_DIR = os.path.dirname(__file__) BUILD_JAVA = os.getenv("RAY_INSTALL_JAVA") == "1" +BUILD_CPP = os.getenv("RAY_DISABLE_EXTRA_CPP") != "1" SKIP_BAZEL_BUILD = os.getenv("SKIP_BAZEL_BUILD") == "1" BAZEL_ARGS = os.getenv("BAZEL_ARGS") BAZEL_LIMIT_CPUS = os.getenv("BAZEL_LIMIT_CPUS") @@ -130,7 +131,7 @@ def get_packages(self): ) RAY_EXTRA_CPP = True # Disable extra cpp for the development versions. - if "dev" in setup_spec.version or os.getenv("RAY_DISABLE_EXTRA_CPP") == "1": + if "dev" in setup_spec.version or not BUILD_CPP: RAY_EXTRA_CPP = False # Ideally, we could include these files by putting them in a @@ -710,7 +711,7 @@ def pip_run(build_ext): if SKIP_BAZEL_BUILD: build(False, False, False) else: - build(True, BUILD_JAVA, True) + build(True, BUILD_JAVA, BUILD_CPP) if setup_spec.type == SetupType.RAY: setup_spec.files_to_include += ray_files From 5afa2abcb65980e2ab558076b39e9a44bd2e3566 Mon Sep 17 00:00:00 2001 From: Potato Date: Tue, 19 Aug 2025 02:46:17 +0800 Subject: [PATCH 160/634] [Data]Fix sort_benchmark url not found error (#55692) The url is invalid as we changed the name for `sort.py` in https://github.com/ray-project/ray/pull/49017 --------- Signed-off-by: Potato --- doc/source/data/shuffling-data.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/data/shuffling-data.rst b/doc/source/data/shuffling-data.rst index a146370aeb09..11f720ec8f7e 100644 --- a/doc/source/data/shuffling-data.rst +++ b/doc/source/data/shuffling-data.rst @@ -189,8 +189,8 @@ To try out push-based shuffle, set the environment variable ``RAY_DATA_PUSH_BASE .. code-block:: bash - $ wget https://raw.githubusercontent.com/ray-project/ray/master/release/nightly_tests/dataset/sort.py - $ RAY_DATA_PUSH_BASED_SHUFFLE=1 python sort.py --num-partitions=10 --partition-size=1e7 + $ wget https://raw.githubusercontent.com/ray-project/ray/master/release/nightly_tests/dataset/sort_benchmark.py + $ RAY_DATA_PUSH_BASED_SHUFFLE=1 python sort_benchmark.py --num-partitions=10 --partition-size=1e7 # Dataset size: 10 partitions, 0.01GB partition size, 0.1GB total # [dataset]: Run `pip install tqdm` to enable progress reporting. From ec4056ea67e4226fea2f11abaf4e16bf5a3aba14 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 18 Aug 2025 13:53:47 -0500 Subject: [PATCH 161/634] [ci] Add ability for users to include `.user.bazelrc` file (#55698) I wanted a way to turn on `--incompatible_strict_action_env` by default without having an untracked change in my `.bazelrc` constantly and without needing to pass the `--config` flag all the time. This PR allows users to define a `.user.bazelrc` file for such changes. For example, to turn on `--incompatible_strict_action_env` by default, I've added this file: ``` build --config=strict test --config=strict ``` Signed-off-by: Edward Oakes --- .bazelrc | 8 +++++++- .gitignore | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 8c2be7ea8586..5e4727932249 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,7 +4,10 @@ build --enable_platform_specific_config build:linux --workspace_status_command="bash ./bazel/workspace_status.sh" # Provides users an option to turn on strict action env. -# TODO(aslonnie): make this default; fix the python tests.. +# TODO(aslonnie): make this default, fix the python tests. +# NOTE(edoakes): enable this by default locally by adding a .user.bazelrc file with: +# build --config=strict +# test --config=strict build:strict --incompatible_strict_action_env # To distinguish different incompatible environments. @@ -220,6 +223,9 @@ build:cgroup --sandbox_writable_path=/sys/fs/cgroup --config=llvm # ci/env/install-llvm-dependencies.sh try-import %workspace%/.llvm-local.bazelrc +# Allow users to define custom options. +try-import %workspace%/.user.bazelrc + # Even with sandbox mode bazel prioritizes system headers over the ones in the sandbox. # It picks up the system headers when someone has protobuf installed via Homebrew. # Work around for https://github.com/bazelbuild/bazel/issues/8053 diff --git a/.gitignore b/.gitignore index ae8dd2240350..e33fde76315d 100644 --- a/.gitignore +++ b/.gitignore @@ -126,6 +126,7 @@ scripts/nodes.txt .idea/**/tasks.xml .idea/dictionaries .llvm-local.bazelrc +.user.bazelrc .aider* # Sensitive or high-churn files: From 6326b2c539d4337019dc5107b569e272fc8a8fcf Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Tue, 19 Aug 2025 00:34:29 +0530 Subject: [PATCH 162/634] [core] Call `__ray_shutdown__` method during actor graceful shutdown (#54584) ## Why are these changes needed? This PR introduces a new `__ray_shutdown__ ` method mechanism for Ray actors to perform deterministic resource cleanup before actor termination. This addresses issue #53169 by providing a reliable alternative to `__del__` methods for critical cleanup operations. The new `__ray_shutdown__ ` method can be explicitly overriden and provides: - Deterministic execution: Called explicitly by Ray during actor shutdown. - Reliable timing: Executes at the exact right moment before process termination. - Optionality: Actors without the method continue to work normally. Main changes: 1. `core_worker.cc` - Add cleanup call in Shutdown() 2. `_raylet.pyx` - Add callback registration 3. `worker.py` - Register callback when actor is created ## Related issue number Closes #53169 --------- Signed-off-by: Sagar Sumit --- python/ray/_raylet.pyx | 34 +++++ python/ray/includes/libcoreworker.pxd | 1 + python/ray/tests/test_actor_failures.py | 153 ++++++++++++++++++++++ src/ray/core_worker/core_worker.cc | 7 + src/ray/core_worker/core_worker.h | 3 + src/ray/core_worker/core_worker_options.h | 3 + 6 files changed, 201 insertions(+) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index ee02c709fafa..fa06590fd21e 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -2200,6 +2200,7 @@ cdef execute_task_with_cancellation_handler( actor_id = core_worker.get_actor_id() actor = actor_class.__new__(actor_class) worker.actors[actor_id] = actor + # Record the actor class via :actor_name: magic token in the log. # # (Phase 1): this covers code run before __init__ finishes. @@ -2744,6 +2745,38 @@ cdef void terminate_asyncio_thread() nogil: core_worker = ray._private.worker.global_worker.core_worker core_worker.stop_and_join_asyncio_threads_if_exist() + +cdef void call_actor_shutdown() noexcept nogil: + """C++ wrapper function that calls the Python actor shutdown callback.""" + with gil: + _call_actor_shutdown() + + +def _call_actor_shutdown(): + """Internal function that calls actor's __ray_shutdown__ method.""" + try: + worker = ray._private.worker.global_worker + + if not worker.actors: + return + + actor_id, actor_instance = next(iter(worker.actors.items())) + if actor_instance is not None: + # Only call __ray_shutdown__ if the method exists and is callable + # This preserves backward compatibility: actors without __ray_shutdown__ + # use Python's normal exit flow (including atexit handlers) + if hasattr(actor_instance, '__ray_shutdown__') and callable(getattr(actor_instance, '__ray_shutdown__')): + try: + actor_instance.__ray_shutdown__() + except Exception: + logger.exception("Error during actor __ray_shutdown__ method") + # Always clean up the actor instance + worker.actors.pop(actor_id, None) + except Exception: + # Catch any system-level exceptions to prevent propagation to C++ + logger.exception("System error during actor shutdown callback") + + cdef class StreamRedirector: @staticmethod def redirect_stdout(const c_string &file_path, uint64_t rotation_max_size, uint64_t rotation_max_file_count, c_bool tee_to_stdout, c_bool tee_to_stderr): @@ -3030,6 +3063,7 @@ cdef class CoreWorker: options.is_local_mode = local_mode options.kill_main = kill_main_task options.terminate_asyncio_thread = terminate_asyncio_thread + options.actor_shutdown_callback = call_actor_shutdown options.serialized_job_config = serialized_job_config options.metrics_agent_port = metrics_agent_port options.runtime_env_hash = runtime_env_hash diff --git a/python/ray/includes/libcoreworker.pxd b/python/ray/includes/libcoreworker.pxd index 88f7252a3a8a..5402a7e2f48d 100644 --- a/python/ray/includes/libcoreworker.pxd +++ b/python/ray/includes/libcoreworker.pxd @@ -422,6 +422,7 @@ cdef extern from "ray/core_worker/core_worker.h" nogil: const c_vector[c_string]&) nogil) run_on_util_worker_handler (void(const CRayObject&) nogil) unhandled_exception_handler (c_bool(const CTaskID &c_task_id) nogil) cancel_async_actor_task + (void() noexcept nogil) actor_shutdown_callback (void(c_string *stack_out) nogil) get_lang_stack c_bool is_local_mode int num_workers diff --git a/python/ray/tests/test_actor_failures.py b/python/ray/tests/test_actor_failures.py index 5043784d3b34..6397b1b48b9c 100644 --- a/python/ray/tests/test_actor_failures.py +++ b/python/ray/tests/test_actor_failures.py @@ -4,7 +4,9 @@ import os import signal import sys +import tempfile import time +from typing import Callable, Generator import pytest import numpy as np @@ -29,6 +31,32 @@ def ray_init_with_task_retry_delay(): ray.shutdown() +@pytest.fixture +def tempfile_factory() -> Generator[Callable[[], str], None, None]: + """Yields a factory function to generate tempfiles that will be deleted after the test run.""" + files = [] + + def create_temp_file(): + temp_file = tempfile.NamedTemporaryFile(delete=False) + temp_file.close() + files.append(temp_file.name) + return temp_file.name + + yield create_temp_file + + # Cleanup all created files + for file_path in files: + try: + os.unlink(file_path) + except Exception: + pass + + +def check_file_exists_and_not_empty(file_path): + """Helper to check if file exists and has content.""" + return os.path.exists(file_path) and os.path.getsize(file_path) > 0 + + @pytest.mark.parametrize( "ray_start_regular", [ @@ -1248,5 +1276,130 @@ def get_pid(self): assert ray.get(refs) == [3, 4, 5] +def test_actor_user_shutdown_method(ray_start_regular_shared, tempfile_factory): + """Test that __ray_shutdown__ method is called during actor termination.""" + shutdown_file = tempfile_factory() + + @ray.remote + class UserShutdownActor: + def __init__(self): + pass + + def __ray_shutdown__(self): + with open(shutdown_file, "w") as f: + f.write("ray_shutdown_called") + f.flush() + + def get_ready(self): + return "ready" + + actor = UserShutdownActor.remote() + ray.get(actor.get_ready.remote()) + actor.__ray_terminate__.remote() + + wait_for_condition(lambda: check_file_exists_and_not_empty(shutdown_file)) + + with open(shutdown_file, "r") as f: + assert f.read() == "ray_shutdown_called" + + +def test_actor_ray_shutdown_handles_exceptions( + ray_start_regular_shared, tempfile_factory +): + """Test that Ray handles unhandled exceptions in __ray_shutdown__ gracefully.""" + shutdown_file = tempfile_factory() + + @ray.remote + class ExceptionActor: + def __ray_shutdown__(self): + # Write to file before raising exception + with open(shutdown_file, "w") as f: + f.write("cleanup_started") + f.flush() + + # Let exception propagate to Ray's machinery + raise ValueError("Unhandled exception in __ray_shutdown__") + + def get_ready(self): + return "ready" + + actor = ExceptionActor.remote() + ray.get(actor.get_ready.remote()) + actor.__ray_terminate__.remote() + + # Verify that despite the exception: + # 1. File was written (cleanup started) + # 2. Actor shuts down properly (no system crash) + wait_for_condition(lambda: check_file_exists_and_not_empty(shutdown_file)) + + with open(shutdown_file, "r") as f: + assert f.read() == "cleanup_started" + + +def test_actor_atexit_handler_dont_conflict_with_ray_shutdown( + ray_start_regular_shared, tempfile_factory +): + """Test that atexit handler methods don't conflict with __ray_shutdown__ and both run.""" + shutdown_file = tempfile_factory() + atexit_file = tempfile_factory() + + @ray.remote + class CleanupActor: + def __init__(self): + atexit.register(self.cleanup) + + def __ray_shutdown__(self): + with open(shutdown_file, "w") as f: + f.write("ray_shutdown_called") + f.flush() + + def cleanup(self): + with open(atexit_file, "w") as f: + f.write("atexit_cleanup_called") + f.flush() + + def get_ready(self): + return "ready" + + actor = CleanupActor.remote() + ray.get(actor.get_ready.remote()) + actor.__ray_terminate__.remote() + + wait_for_condition(lambda: check_file_exists_and_not_empty(shutdown_file)) + + with open(shutdown_file, "r") as f: + assert f.read() == "ray_shutdown_called" + wait_for_condition(lambda: check_file_exists_and_not_empty(atexit_file)) + with open(atexit_file, "r") as f: + assert f.read() == "atexit_cleanup_called" + + +def test_actor_ray_shutdown_dont_interfere_with_kill( + ray_start_regular_shared, tempfile_factory +): + """Test __ray_shutdown__ is not called when actor is killed with ray.kill().""" + shutdown_file = tempfile_factory() + + @ray.remote + class KillableActor: + def __ray_shutdown__(self): + with open(shutdown_file, "w") as f: + f.write("shutdown_called_kill") + f.flush() + + def get_ready(self): + return "ready" + + def sleep_forever(self): + time.sleep(3600) + + actor = KillableActor.remote() + ray.get(actor.get_ready.remote()) + _ = actor.sleep_forever.remote() + ray.kill(actor) + + wait_for_condition(lambda: not check_file_exists_and_not_empty(shutdown_file)) + + if __name__ == "__main__": sys.exit(pytest.main(["-sv", __file__])) diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 53288a3c45d9..83ac7000178b 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -334,6 +334,7 @@ CoreWorker::CoreWorker( max_direct_call_object_size_(RayConfig::instance().max_direct_call_object_size()), task_event_buffer_(std::move(task_event_buffer)), pid_(pid), + actor_shutdown_callback_(std::move(options_.actor_shutdown_callback)), runtime_env_json_serialization_cache_(kDefaultSerializationCacheCap) { // Initialize task receivers. if (options_.worker_type == WorkerType::WORKER || options_.is_local_mode) { @@ -499,6 +500,12 @@ void CoreWorker::Shutdown() { RAY_LOG(INFO) << "Shutdown was called more than once, ignoring."; return; } + + // For actors, perform cleanup before shutdown proceeds. + if (!GetWorkerContext().GetCurrentActorID().IsNil() && actor_shutdown_callback_) { + RAY_LOG(INFO) << "Calling actor shutdown callback before shutdown"; + actor_shutdown_callback_(); + } RAY_LOG(INFO) << "Shutting down."; if (options_.worker_type == WorkerType::WORKER) { diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index 2299894a3956..3380aa0e489e 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -1900,6 +1900,9 @@ class CoreWorker { /// Worker's PID uint32_t pid_; + /// Callback to cleanup actor instance before shutdown + std::function actor_shutdown_callback_; + // Guards generator_ids_pending_deletion_. absl::Mutex generator_ids_pending_deletion_mutex_; diff --git a/src/ray/core_worker/core_worker_options.h b/src/ray/core_worker/core_worker_options.h index 4eee61e4d75a..5e139e9758e6 100644 --- a/src/ray/core_worker/core_worker_options.h +++ b/src/ray/core_worker/core_worker_options.h @@ -96,6 +96,7 @@ struct CoreWorkerOptions { get_lang_stack(nullptr), kill_main(nullptr), cancel_async_actor_task(nullptr), + actor_shutdown_callback(nullptr), is_local_mode(false), terminate_asyncio_thread(nullptr), serialized_job_config(""), @@ -174,6 +175,8 @@ struct CoreWorkerOptions { // Should return a boolean indicating if the task was successfully cancelled or not. // If not, the client will retry. std::function cancel_async_actor_task; + /// Callback to shutdown actor instance before shutdown. + std::function actor_shutdown_callback; /// Is local mode being used. bool is_local_mode; /// The function to destroy asyncio event and loops. From 01b9e5b1a6b913041b299d7cd262254cfc99503a Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 18 Aug 2025 12:33:07 -0700 Subject: [PATCH 163/634] [ci] release test: use rayci build id for image tags (#55619) rather than using commit based tags. this avoids runs across different runs on the same commit to crosstalk to each other. --------- Signed-off-by: Lonnie Liu --- release/ray_release/scripts/build_pipeline.py | 9 +++- release/ray_release/test.py | 22 ++------ release/ray_release/tests/test_byod_build.py | 26 ++++++---- release/ray_release/tests/test_test.py | 50 ++++++++++++------- 4 files changed, 59 insertions(+), 48 deletions(-) diff --git a/release/ray_release/scripts/build_pipeline.py b/release/ray_release/scripts/build_pipeline.py index 9b6ff71adc1f..5c6cc5b5998f 100644 --- a/release/ray_release/scripts/build_pipeline.py +++ b/release/ray_release/scripts/build_pipeline.py @@ -21,7 +21,6 @@ from ray_release.configs.global_config import init_global_config from ray_release.exception import ReleaseTestCLIError, ReleaseTestConfigError from ray_release.logger import logger -from ray_release.wheels import get_buildkite_repo_branch PIPELINE_ARTIFACT_PATH = "/tmp/pipeline_artifacts" @@ -150,10 +149,16 @@ def main( if no_concurrency_limit: logger.warning("Concurrency is not limited for this run!") - _, buildkite_branch = get_buildkite_repo_branch() if os.environ.get("REPORT_TO_RAY_TEST_DB", False): env["REPORT_TO_RAY_TEST_DB"] = "1" + # Pipe through RAYCI_BUILD_ID from the forge step. + # TODO(khluu): convert the steps to rayci steps and stop passing through + # RAYCI_BUILD_ID. + build_id = os.environ.get("RAYCI_BUILD_ID") + if build_id: + env["RAYCI_BUILD_ID"] = build_id + steps = get_step_for_test_group( grouped_tests, minimum_run_per_test=run_per_test, diff --git a/release/ray_release/test.py b/release/ray_release/test.py index 03ee427338f5..c912a7b00bdb 100644 --- a/release/ray_release/test.py +++ b/release/ray_release/test.py @@ -541,23 +541,11 @@ def get_byod_base_image_tag(self) -> str: # TODO(can): this is a temporary backdoor that should be removed # once civ2 is fully rolled out. return byod_image_tag - commit = os.environ.get( - "COMMIT_TO_TEST", - os.environ["BUILDKITE_COMMIT"], - ) - branch = os.environ.get( - "BRANCH_TO_TEST", - os.environ["BUILDKITE_BRANCH"], - ) - pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") - ray_version = commit[:6] - if pr != "false": - ray_version = f"pr-{pr}.{ray_version}" - elif branch.startswith("releases/"): - release_name = branch[len("releases/") :] - ray_version = f"{release_name}.{ray_version}" - python_version = f"py{self.get_python_version().replace('.', '')}" - return f"{ray_version}-{python_version}-{self.get_tag_suffix()}" + build_id = os.environ.get("RAYCI_BUILD_ID", "") + if not build_id: + raise ValueError("RAYCI_BUILD_ID is not set") + python_version = "py" + self.get_python_version().replace(".", "") + return f"{build_id}-{python_version}-{self.get_tag_suffix()}" def get_byod_image_tag(self) -> str: """ diff --git a/release/ray_release/tests/test_byod_build.py b/release/ray_release/tests/test_byod_build.py index 920b2c826e36..8548a309632f 100644 --- a/release/ray_release/tests/test_byod_build.py +++ b/release/ray_release/tests/test_byod_build.py @@ -59,7 +59,10 @@ def _mock_check_call( with patch("ray_release.byod.build._image_exist", return_value=False), patch.dict( "os.environ", - {"BUILDKITE_COMMIT": "abc123", "BUILDKITE_BRANCH": "master"}, + { + "BUILDKITE_COMMIT": "abc123", + "RAYCI_BUILD_ID": "a1b2c3d4", + }, ), patch("subprocess.check_call", side_effect=_mock_check_call,), patch( "subprocess.check_output", return_value=b"abc123", @@ -74,8 +77,8 @@ def _mock_check_call( test.get_byod_post_build_script(), ) assert "docker build --build-arg BASE_IMAGE=029272617770.dkr.ecr.us-west-2." - "amazonaws.com/anyscale/ray:abc123-py37 -t 029272617770.dkr.ecr.us-west-2." - "amazonaws.com/anyscale/ray:abc123-py37-c3fc5fc6d84cea4d7ab885c6cdc966542e" + "amazonaws.com/anyscale/ray:a1b2c3d4-py37 -t 029272617770.dkr.ecr.us-west-2." + "amazonaws.com/anyscale/ray:a1b2c3d4-py37-c3fc5fc6d84cea4d7ab885c6cdc966542e" "f59e4c679b8c970f2f77b956bfd8fb" in " ".join(cmds[0]) @@ -92,7 +95,10 @@ def _mock_image_exist(image: str) -> bool: "ray_release.byod.build._download_dataplane_build_file", return_value=None ), patch( "os.environ", - {"BUILDKITE_COMMIT": "abc123", "BUILDKITE_BRANCH": "master"}, + { + "BUILDKITE_COMMIT": "abc123", + "RAYCI_BUILD_ID": "a1b2c3d4", + }, ), patch( "subprocess.check_call", return_value=None ), patch( @@ -127,12 +133,12 @@ def _mock_image_exist(image: str) -> bool: aws_cr = global_config["byod_aws_cr"] gcp_cr = global_config["byod_gcp_cr"] assert images == [ - f"{aws_cr}/anyscale/ray:abc123-py39-cpu", - f"{aws_cr}/anyscale/ray-ml:abc123-py39-gpu", - f"{aws_cr}/anyscale/ray:abc123-py39-cu121", - f"{aws_cr}/anyscale/ray:abc123-py39-cu116", - f"{aws_cr}/anyscale/ray:abc123-py311-cu118", - f"{gcp_cr}/anyscale/ray:abc123-py39-cpu", + f"{aws_cr}/anyscale/ray:a1b2c3d4-py39-cpu", + f"{aws_cr}/anyscale/ray-ml:a1b2c3d4-py39-gpu", + f"{aws_cr}/anyscale/ray:a1b2c3d4-py39-cu121", + f"{aws_cr}/anyscale/ray:a1b2c3d4-py39-cu116", + f"{aws_cr}/anyscale/ray:a1b2c3d4-py311-cu118", + f"{gcp_cr}/anyscale/ray:a1b2c3d4-py39-cpu", ] diff --git a/release/ray_release/tests/test_test.py b/release/ray_release/tests/test_test.py index e4899de2ebb7..513eba2a9878 100644 --- a/release/ray_release/tests/test_test.py +++ b/release/ray_release/tests/test_test.py @@ -90,8 +90,9 @@ def test_get_python_version(): def test_get_ray_image(): - os.environ["BUILDKITE_BRANCH"] = "master" - os.environ["BUILDKITE_COMMIT"] = "1234567890" + os.environ["RAYCI_BUILD_ID"] = "a1b2c3d4" + + # These images are NOT saved on Docker Hub, but on private ECR. assert ( _stub_test( { @@ -99,7 +100,7 @@ def test_get_ray_image(): "cluster": {"byod": {}}, } ).get_ray_image() - == "rayproject/ray:123456-py39-cpu" + == "rayproject/ray:a1b2c3d4-py39-cpu" ) assert ( _stub_test( @@ -112,7 +113,7 @@ def test_get_ray_image(): }, } ).get_ray_image() - == "rayproject/ray-ml:123456-py39-gpu" + == "rayproject/ray-ml:a1b2c3d4-py39-gpu" ) assert ( _stub_test( @@ -125,23 +126,34 @@ def test_get_ray_image(): }, } ).get_ray_image() - == "rayproject/ray-llm:123456-py311-cu124" - ) - os.environ["BUILDKITE_BRANCH"] = "releases/1.0.0" - assert ( - _stub_test({"cluster": {"byod": {}}}).get_ray_image() - == "rayproject/ray:1.0.0.123456-py39-cpu" + == "rayproject/ray-llm:a1b2c3d4-py311-cu124" ) - with mock.patch.dict(os.environ, {"BUILDKITE_PULL_REQUEST": "123"}): + + # When RAY_IMAGE_TAG is set, we use the RAYCI_BUILD_ID. + with mock.patch.dict(os.environ, {"RAY_IMAGE_TAG": "my_tag"}): assert ( _stub_test({"cluster": {"byod": {}}}).get_ray_image() - == "rayproject/ray:pr-123.123456-py39-cpu" + == "rayproject/ray:my_tag" ) - with mock.patch.dict(os.environ, {"RAY_IMAGE_TAG": "my_tag"}): + + with mock.patch.dict(os.environ, {"BUILDKITE_BRANCH": "releases/1.0.0"}): + # Even on release branches, we also use the RAYCI_BUILD_ID. assert ( _stub_test({"cluster": {"byod": {}}}).get_ray_image() - == "rayproject/ray:my_tag" + == "rayproject/ray:a1b2c3d4-py39-cpu" ) + with mock.patch.dict(os.environ, {"BUILDKITE_PULL_REQUEST": "123"}): + assert ( + _stub_test({"cluster": {"byod": {}}}).get_ray_image() + == "rayproject/ray:a1b2c3d4-py39-cpu" + ) + + # Unless RAY_IMAGE_TAG is set, we use the RAYCI_BUILD_ID. + with mock.patch.dict(os.environ, {"RAY_IMAGE_TAG": "my_tag"}): + assert ( + _stub_test({"cluster": {"byod": {}}}).get_ray_image() + == "rayproject/ray:my_tag" + ) def test_get_byod_runtime_env(): @@ -161,11 +173,10 @@ def test_get_byod_runtime_env(): def test_get_anyscale_byod_image(): - os.environ["BUILDKITE_BRANCH"] = "master" - os.environ["BUILDKITE_COMMIT"] = "1234567890" + os.environ["RAYCI_BUILD_ID"] = "a1b2c3d4" assert ( _stub_test({"python": "3.7", "cluster": {"byod": {}}}).get_anyscale_byod_image() - == f"{get_global_config()['byod_ecr']}/{DATAPLANE_ECR_REPO}:123456-py37-cpu" + == f"{get_global_config()['byod_ecr']}/{DATAPLANE_ECR_REPO}:a1b2c3d4-py37-cpu" ) assert _stub_test( { @@ -177,7 +188,8 @@ def test_get_anyscale_byod_image(): }, } ).get_anyscale_byod_image() == ( - f"{get_global_config()['byod_ecr']}/" f"{DATAPLANE_ECR_ML_REPO}:123456-py38-gpu" + f"{get_global_config()['byod_ecr']}/" + f"{DATAPLANE_ECR_ML_REPO}:a1b2c3d4-py38-gpu" ) assert _stub_test( { @@ -191,7 +203,7 @@ def test_get_anyscale_byod_image(): } ).get_anyscale_byod_image() == ( f"{get_global_config()['byod_ecr']}" - f"/{DATAPLANE_ECR_ML_REPO}:123456-py38-gpu-" + f"/{DATAPLANE_ECR_ML_REPO}:a1b2c3d4-py38-gpu-" "ab7ed2b7a7e8d3f855a7925b0d296b0f9c75fac91882aba47854d92d27e13e53" ) From 9128c40da7a8166bb7a9ca7025b01d8a7a5e38db Mon Sep 17 00:00:00 2001 From: Sven Mika Date: Mon, 18 Aug 2025 22:40:55 +0200 Subject: [PATCH 164/634] [RLlib] Fix MetricsLogger/Stats throughput bugs. (#55696) --- rllib/env/__init__.py | 2 - rllib/env/policy_server_input.py | 270 ------------------ .../ppo/multi_agent_cartpole_ppo.py | 2 +- rllib/utils/metrics/metrics_logger.py | 1 - rllib/utils/metrics/stats.py | 114 +++++--- .../metrics/tests/test_metrics_logger.py | 78 ++++- 6 files changed, 145 insertions(+), 322 deletions(-) delete mode 100644 rllib/env/policy_server_input.py diff --git a/rllib/env/__init__.py b/rllib/env/__init__.py index 2e48374d784c..ca9de7949565 100644 --- a/rllib/env/__init__.py +++ b/rllib/env/__init__.py @@ -4,7 +4,6 @@ from ray.rllib.env.external_multi_agent_env import ExternalMultiAgentEnv from ray.rllib.env.multi_agent_env import MultiAgentEnv from ray.rllib.env.policy_client import PolicyClient -from ray.rllib.env.policy_server_input import PolicyServerInput from ray.rllib.env.remote_base_env import RemoteBaseEnv from ray.rllib.env.vector_env import VectorEnv @@ -31,7 +30,6 @@ "PettingZooEnv", "ParallelPettingZooEnv", "PolicyClient", - "PolicyServerInput", "RemoteBaseEnv", "Unity3DEnv", "VectorEnv", diff --git a/rllib/env/policy_server_input.py b/rllib/env/policy_server_input.py deleted file mode 100644 index 48598a51d9eb..000000000000 --- a/rllib/env/policy_server_input.py +++ /dev/null @@ -1,270 +0,0 @@ -from collections import deque -from http.server import HTTPServer, SimpleHTTPRequestHandler -import logging -import queue -from socketserver import ThreadingMixIn -import threading -import time -import traceback - -from typing import List -import ray.cloudpickle as pickle -from ray.rllib.env.policy_client import ( - _create_embedded_rollout_worker, - Commands, -) -from ray.rllib.offline.input_reader import InputReader -from ray.rllib.offline.io_context import IOContext -from ray.rllib.policy.sample_batch import SampleBatch -from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.evaluation.metrics import RolloutMetrics -from ray.rllib.evaluation.sampler import SamplerInput -from ray.rllib.utils.typing import SampleBatchType -from ray._common.network_utils import build_address - -logger = logging.getLogger(__name__) - - -@OldAPIStack -class PolicyServerInput(ThreadingMixIn, HTTPServer, InputReader): - def __init__( - self, - ioctx: IOContext, - address: str, - port: int, - idle_timeout: float = 3.0, - max_sample_queue_size: int = 20, - ): - self.rollout_worker = ioctx.worker - # Protect ourselves from having a bottleneck on the server (learning) side. - # Once the queue (deque) is full, we throw away 50% (oldest - # samples first) of the samples, warn, and continue. - self.samples_queue = deque(maxlen=max_sample_queue_size) - self.metrics_queue = queue.Queue() - self.idle_timeout = idle_timeout - - # Forwards client-reported metrics directly into the local rollout - # worker. - if self.rollout_worker.sampler is not None: - # This is a bit of a hack since it is patching the get_metrics - # function of the sampler. - - def get_metrics(): - completed = [] - while True: - try: - completed.append(self.metrics_queue.get_nowait()) - except queue.Empty: - break - - return completed - - self.rollout_worker.sampler.get_metrics = get_metrics - else: - # If there is no sampler, act like if there would be one to collect - # metrics from - class MetricsDummySampler(SamplerInput): - """This sampler only maintains a queue to get metrics from.""" - - def __init__(self, metrics_queue): - """Initializes a MetricsDummySampler instance. - - Args: - metrics_queue: A queue of metrics - """ - self.metrics_queue = metrics_queue - - def get_data(self) -> SampleBatchType: - raise NotImplementedError - - def get_extra_batches(self) -> List[SampleBatchType]: - raise NotImplementedError - - def get_metrics(self) -> List[RolloutMetrics]: - """Returns metrics computed on a policy client rollout worker.""" - completed = [] - while True: - try: - completed.append(self.metrics_queue.get_nowait()) - except queue.Empty: - break - return completed - - self.rollout_worker.sampler = MetricsDummySampler(self.metrics_queue) - - # Create a request handler that receives commands from the clients - # and sends data and metrics into the queues. - handler = _make_handler( - self.rollout_worker, self.samples_queue, self.metrics_queue - ) - try: - import time - - time.sleep(1) - HTTPServer.__init__(self, (address, port), handler) - except OSError: - print(f"Creating a PolicyServer on {build_address(address, port)} failed!") - import time - - time.sleep(1) - raise - - logger.info( - "Starting connector server at " - f"{build_address(self.server_name, self.server_port)}" - ) - - # Start the serving thread, listening on socket and handling commands. - serving_thread = threading.Thread(name="server", target=self.serve_forever) - serving_thread.daemon = True - serving_thread.start() - - # Start a dummy thread that puts empty SampleBatches on the queue, just - # in case we don't receive anything from clients (or there aren't - # any). The latter would block sample collection entirely otherwise, - # even if other workers' PolicyServerInput receive incoming data from - # actual clients. - heart_beat_thread = threading.Thread( - name="heart-beat", target=self._put_empty_sample_batch_every_n_sec - ) - heart_beat_thread.daemon = True - heart_beat_thread.start() - - @override(InputReader) - def next(self): - # Blocking wait until there is something in the deque. - while len(self.samples_queue) == 0: - time.sleep(0.1) - # Utilize last items first in order to remain as closely as possible - # to operating on-policy. - return self.samples_queue.pop() - - def _put_empty_sample_batch_every_n_sec(self): - # Places an empty SampleBatch every `idle_timeout` seconds onto the - # `samples_queue`. This avoids hanging of all RolloutWorkers parallel - # to this one in case this PolicyServerInput does not have incoming - # data (e.g. no client connected) and the driver algorithm uses parallel - # synchronous sampling (e.g. PPO). - while True: - time.sleep(self.idle_timeout) - self.samples_queue.append(SampleBatch()) - - -def _make_handler(rollout_worker, samples_queue, metrics_queue): - # Only used in remote inference mode. We must create a new rollout worker - # then since the original worker doesn't have the env properly wrapped in - # an ExternalEnv interface. - child_rollout_worker = None - inference_thread = None - lock = threading.Lock() - - def setup_child_rollout_worker(): - nonlocal lock - - with lock: - nonlocal child_rollout_worker - nonlocal inference_thread - - if child_rollout_worker is None: - ( - child_rollout_worker, - inference_thread, - ) = _create_embedded_rollout_worker( - rollout_worker.creation_args(), report_data - ) - child_rollout_worker.set_weights(rollout_worker.get_weights()) - - def report_data(data): - nonlocal child_rollout_worker - - batch = data["samples"] - batch.decompress_if_needed() - samples_queue.append(batch) - # Deque is full -> purge 50% (oldest samples) - if len(samples_queue) == samples_queue.maxlen: - logger.warning( - "PolicyServerInput queue is full! Purging half of the samples (oldest)." - ) - for _ in range(samples_queue.maxlen // 2): - samples_queue.popleft() - for rollout_metric in data["metrics"]: - metrics_queue.put(rollout_metric) - - if child_rollout_worker is not None: - child_rollout_worker.set_weights( - rollout_worker.get_weights(), rollout_worker.get_global_vars() - ) - - class Handler(SimpleHTTPRequestHandler): - def __init__(self, *a, **kw): - super().__init__(*a, **kw) - - def do_POST(self): - content_len = int(self.headers.get("Content-Length"), 0) - raw_body = self.rfile.read(content_len) - parsed_input = pickle.loads(raw_body) - try: - response = self.execute_command(parsed_input) - self.send_response(200) - self.end_headers() - self.wfile.write(pickle.dumps(response)) - except Exception: - self.send_error(500, traceback.format_exc()) - - def execute_command(self, args): - command = args["command"] - response = {} - - # Local inference commands: - if command == Commands.GET_WORKER_ARGS: - logger.info("Sending worker creation args to client.") - response["worker_args"] = rollout_worker.creation_args() - elif command == Commands.GET_WEIGHTS: - logger.info("Sending worker weights to client.") - response["weights"] = rollout_worker.get_weights() - response["global_vars"] = rollout_worker.get_global_vars() - elif command == Commands.REPORT_SAMPLES: - logger.info( - "Got sample batch of size {} from client.".format( - args["samples"].count - ) - ) - report_data(args) - - # Remote inference commands: - elif command == Commands.START_EPISODE: - setup_child_rollout_worker() - assert inference_thread.is_alive() - response["episode_id"] = child_rollout_worker.env.start_episode( - args["episode_id"], args["training_enabled"] - ) - elif command == Commands.GET_ACTION: - assert inference_thread.is_alive() - response["action"] = child_rollout_worker.env.get_action( - args["episode_id"], args["observation"] - ) - elif command == Commands.LOG_ACTION: - assert inference_thread.is_alive() - child_rollout_worker.env.log_action( - args["episode_id"], args["observation"], args["action"] - ) - elif command == Commands.LOG_RETURNS: - assert inference_thread.is_alive() - if args["done"]: - child_rollout_worker.env.log_returns( - args["episode_id"], args["reward"], args["info"], args["done"] - ) - else: - child_rollout_worker.env.log_returns( - args["episode_id"], args["reward"], args["info"] - ) - elif command == Commands.END_EPISODE: - assert inference_thread.is_alive() - child_rollout_worker.env.end_episode( - args["episode_id"], args["observation"] - ) - else: - raise ValueError("Unknown command: {}".format(command)) - return response - - return Handler diff --git a/rllib/tuned_examples/ppo/multi_agent_cartpole_ppo.py b/rllib/tuned_examples/ppo/multi_agent_cartpole_ppo.py index ea88cca34180..72f020f3664d 100644 --- a/rllib/tuned_examples/ppo/multi_agent_cartpole_ppo.py +++ b/rllib/tuned_examples/ppo/multi_agent_cartpole_ppo.py @@ -44,7 +44,7 @@ ) stop = { - NUM_ENV_STEPS_SAMPLED_LIFETIME: 300000, + NUM_ENV_STEPS_SAMPLED_LIFETIME: 400000, # Divide by num_agents to get actual return per agent. f"{ENV_RUNNER_RESULTS}/{EPISODE_RETURN_MEAN}": 450.0 * (args.num_agents or 1), } diff --git a/rllib/utils/metrics/metrics_logger.py b/rllib/utils/metrics/metrics_logger.py index 627ec5b9d579..28432ea7cca3 100644 --- a/rllib/utils/metrics/metrics_logger.py +++ b/rllib/utils/metrics/metrics_logger.py @@ -761,7 +761,6 @@ def aggregate( key: Optional top-level key under which to log all keys/key sequences found in the n `stats_dicts`. """ - assert isinstance(stats_dicts, list), "stats_dicts must be a list" all_keys = set() def traverse_and_add_paths(d, path=()): diff --git a/rllib/utils/metrics/stats.py b/rllib/utils/metrics/stats.py index a764639effdd..d4f77aeee342 100644 --- a/rllib/utils/metrics/stats.py +++ b/rllib/utils/metrics/stats.py @@ -243,7 +243,8 @@ def __init__( self._throughput_stats = None if throughput is not False: self._throughput_stats = Stats( - # We have to check for bool here because in Python, bool is a subclass of int + # We have to check for bool here because in Python, bool is a subclass + # of int. init_values=[throughput] if ( isinstance(throughput, (int, float)) @@ -258,9 +259,9 @@ def __init__( throughput_ema_coeff=None, ) if init_values is not None: - self._last_push_time = time.perf_counter() + self._last_throughput_measure_time = time.perf_counter() else: - self._last_push_time = ( + self._last_throughput_measure_time = ( -1 ) # Track last push time for throughput calculation @@ -302,14 +303,7 @@ def push(self, value: Any) -> None: self.check_value(value) # If throughput tracking is enabled, calculate it based on time between pushes if self.has_throughput: - current_time = time.perf_counter() - if self._last_push_time >= 0: - time_diff = current_time - self._last_push_time - if time_diff > 0: # Avoid division by zero - current_throughput = value / time_diff - self._throughput_stats.push(current_throughput) - self._last_push_time = current_time - + self._recompute_throughput(value) # Handle different reduction methods if self._window is not None: # For windowed operations, append to values and trim if needed @@ -447,11 +441,7 @@ class for details on the reduction logic applied to the values list, based on # `clear_on_reduce` -> Clear the values list. if self._clear_on_reduce: self._set_values([]) - # If we clear on reduce, following reduce calls should not return the - # old values. - self._has_new_values = True else: - self._has_new_values = False self._set_values(reduced_internal_values_list) else: reduced_internal_values_list = None @@ -510,10 +500,6 @@ def merge_on_time_axis(self, other: "Stats") -> None: """ self.values.extend(other.values) - # Adopt `other`'s current throughput estimate (it's the newer one). - if self.has_throughput: - self._throughput_stats.merge_on_time_axis(other._throughput_stats) - # Mark that we have new values since we modified the values list self._has_new_values = True @@ -587,9 +573,8 @@ def merge_in_parallel(self, *others: "Stats") -> None: continue tmp_values.append(stats.values[-i]) - # Now reduce across `tmp_values` based on the reduce-settings of this Stats. - # TODO (sven) : explain why all this - + # Now reduce across `tmp_values` based on the reduce-settings of this + # Stats. if self._reduce_per_index_on_aggregate: n_values = 1 else: @@ -603,10 +588,10 @@ def merge_in_parallel(self, *others: "Stats") -> None: # We add [sum(tmp_values) / n_values] * n_values to the new values # list instead of tmp_values, because every incoming element should # have the same weight. - reduced_value = ( - self._reduced_values(values=tmp_values)[0][0] / n_values - ) - new_values.extend([reduced_value] * n_values) + added_sum = self._reduced_values(values=tmp_values)[0][0] + new_values.extend([added_sum / n_values] * n_values) + if self.has_throughput: + self._recompute_throughput(added_sum) else: new_values.extend( self._reduced_values(values=tmp_values)[0] * n_values @@ -619,16 +604,37 @@ def merge_in_parallel(self, *others: "Stats") -> None: self._set_values(list(reversed(new_values))) - # Adopt `other`'s current throughput estimate (it's the newer one). - if self.has_throughput: - other_throughput_stats = [ - other._throughput_stats for other in others if other.has_throughput - ] - self._throughput_stats.merge_in_parallel(*other_throughput_stats) - # Mark that we have new values since we modified the values list self._has_new_values = True + def _clear_throughput(self) -> None: + """Clears the throughput Stats, if applicable and `self` has throughput. + + Also resets `self._last_throughput_measure_time` to -1 such that the Stats + object has to create a new timestamp first, before measuring any new throughput + values. + """ + if self.has_throughput: + self._throughput_stats._set_values([]) + self._last_throughput_measure_time = -1 + + def _recompute_throughput(self, value) -> None: + """Recomputes the current throughput value of this Stats instance.""" + # Make sure this Stats object does measure throughput. + assert self.has_throughput + # Take the current time stamp. + current_time = time.perf_counter() + # Check, whether we have a previous timestamp (non -1). + if self._last_throughput_measure_time >= 0: + # Compute the time delta. + time_diff = current_time - self._last_throughput_measure_time + # Avoid divisions by zero. + if time_diff > 0: + # Push new throughput value into our throughput stats object. + self._throughput_stats.push(value / time_diff) + # Update the time stamp of the most recent throughput computation (this one). + self._last_throughput_measure_time = current_time + @staticmethod def _numpy_if_necessary(values): # Torch tensor handling. Convert to CPU/numpy first. @@ -978,35 +984,48 @@ def merge_stats(base_stats: Optional[Stats], incoming_stats: List[Stats]) -> Sta new_root_stats = True else: new_root_stats = False + # Nothing to be merged + if len(incoming_stats) == 0: + return base_stats if new_root_stats: # We need to deepcopy here first because stats from incoming_stats may be altered in the future base_stats = copy.deepcopy(incoming_stats[0]) + base_stats._clear_throughput() + # Note that we may take a mean of means here, which is not the same as a + # mean of all values. In the future, we could implement a weighted mean + # of means here by introducing a new Stats object that counts samples + # for each mean Stats object. + if len(incoming_stats) > 1: + base_stats.merge_in_parallel(*incoming_stats[1:]) elif len(incoming_stats) > 0: # Special case: `base_stats` is a lifetime sum (reduce=sum, # clear_on_reduce=False) -> We subtract the previous value (from 2 # `reduce()` calls ago) from all to-be-merged stats, so we don't count # twice the older sum from before. + + # Also, for the merged, new throughput value, we need to find out what the + # actual value-delta is between before the last reduce and the current one. + + added_sum = 0.0 # Used in `base_stats._recompute_throughput` if applicable. if ( base_stats._reduce_method == "sum" and base_stats._inf_window and base_stats._clear_on_reduce is False ): for stat in incoming_stats: - reduce_by = stat.get_reduce_history()[-2][0] - base_stats.values[-1] -= reduce_by - else: - # Nothing to be merged - return base_stats + # Subtract "lifetime counts" from the Stat's values to not count + # older "lifetime counts" more than once. + _hist = stat.get_reduce_history() + prev_reduction, new_reduction = _hist[-2][0], _hist[-1][0] + # This may not be populated yet -> use 0 then. + if np.isnan(prev_reduction): + prev_reduction = 0 + base_stats.values[-1] -= prev_reduction + # Keep track of how many counts we actually gained (for throughput + # recomputation). + added_sum += new_reduction - prev_reduction - if new_root_stats: - # Note that we may take a mean of means here, which is not the same as a - # mean of all values. In the future, we could implement a weighted mean - # of means here by introducing a new Stats object that counts samples - # for each mean Stats object. - if len(incoming_stats) > 1: - base_stats.merge_in_parallel(*incoming_stats[1:]) - elif len(incoming_stats) > 0: if len(incoming_stats) > 1: # There are more than one incoming parallel others -> Merge all of # them in parallel (equal importance). @@ -1020,5 +1039,8 @@ def merge_stats(base_stats: Optional[Stats], incoming_stats: List[Stats]) -> Sta base_stats._set_values(incoming_stats[0].values.copy()) else: base_stats.merge_on_time_axis(incoming_stats[0]) + # Keep track of throughput through the sum of added counts. + if base_stats.has_throughput: + base_stats._recompute_throughput(added_sum) return base_stats diff --git a/rllib/utils/metrics/tests/test_metrics_logger.py b/rllib/utils/metrics/tests/test_metrics_logger.py index dbd9e0096b04..485af1e86dfc 100644 --- a/rllib/utils/metrics/tests/test_metrics_logger.py +++ b/rllib/utils/metrics/tests/test_metrics_logger.py @@ -3,6 +3,7 @@ import numpy as np import torch +import ray from ray.rllib.utils.metrics.metrics_logger import MetricsLogger from ray.rllib.utils.test_utils import check @@ -235,8 +236,8 @@ def test_throughput_tracking(logger): check(logger.peek("count"), num_iters * 2 + 1) approx_throughput = (num_iters * 2 + 1) / (end_time - start_time) check( - logger.peek("count", throughput=True), approx_throughput, rtol=0.1 - ) # 10% tolerance in throughput + logger.peek("count", throughput=True), approx_throughput, rtol=0.15 + ) # 15% tolerance in throughput # Test _get_throughputs() method without key (returns all throughputs) throughputs = logger.peek(throughput=True) @@ -274,6 +275,79 @@ def test_throughput_tracking(logger): check("count_throughput" in all_throughputs["nested"], True) +def test_throughput_aggregation(): + """Test aggregation of throughput metrics from different (remote) sources.""" + + @ray.remote + class EnvRunner: + def __init__(self): + self.metrics = MetricsLogger() + + def increase(self, count=1): + self.metrics.log_value( + "counter", + count, + reduce="sum", + clear_on_reduce=False, # lifetime counter + with_throughput=True, + ) + + def get_metrics(self): + return self.metrics.reduce() + + env_runners = [EnvRunner.remote() for _ in range(3)] + + # Main logger. + main_metrics = MetricsLogger() + + env_runners[0].increase.remote(count=0) + env_runners[1].increase.remote(count=0) + _ = [ray.get(act.get_metrics.remote()) for act in env_runners] + + # Add 1 count for actor0 and 5 counts for actor1 to the lifetime counters + # in each of the 5 iterations. + # 5 iterations -> expect final count of 5 * 6 = 30 + for _ in range(5): + time.sleep(0.1) + env_runners[0].increase.remote(count=1) + env_runners[1].increase.remote(count=5) + + # Pull metrics from both actors. + results = [ray.get(act.get_metrics.remote()) for act in env_runners] + main_metrics.aggregate(results) + # The first aggregate (before the key even exists in `main_metrics`, throughput + # should be NaN. + check(main_metrics.peek("counter"), 30) + # After first aggregation, throughput should be NaN, b/c the Stats did not exist + # within the `MetricsLogger`. + assert np.isnan(main_metrics.stats["counter"].throughput) + + # Add 1 count for actor0 and 2 counts for actor1 to the lifetime counters + # in each of the 5 iterations. + # 5 iterations each 1 sec -> expect throughput of 3/0.2sec = 5/sec. + for _ in range(5): + time.sleep(0.2) + env_runners[0].increase.remote(count=1) + env_runners[1].increase.remote(count=2) + results = [ray.get(act.get_metrics.remote()) for act in env_runners] + main_metrics.aggregate(results) + + check(main_metrics.peek("counter"), 30 + 15) + tp = main_metrics.stats["counter"].throughput + check(tp, 15, atol=2) + + time.sleep(1.0) + env_runners[2].increase.remote(count=50) + results = ray.get(env_runners[2].get_metrics.remote()) + main_metrics.aggregate([results]) + + check(main_metrics.peek("counter"), 30 + 15 + 50) + tp = main_metrics.stats["counter"].throughput + # Expect throughput - due to the EMA - to be only slightly higher than + # the original value of 15. + check(tp, 16, atol=2) + + def test_reset_and_delete(logger): """Test reset and delete functionality.""" # Log some values From 1cb4c2c212e5a153e74d86f1e0d2e48942a19502 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Mon, 18 Aug 2025 15:12:37 -0700 Subject: [PATCH 165/634] [core] rename ray/telemetry to ray/observability (#55703) As title. According to @edoakes, ray telemetry has a different meaning in the ray eco-system. Observability directory will consists for metrics, events and log related infra. Test: - CI Signed-off-by: Cuong Nguyen --- src/ray/{telemetry => observability}/BUILD.bazel | 0 .../open_telemetry_metric_recorder.cc | 8 ++++---- .../open_telemetry_metric_recorder.h | 4 ++-- src/ray/{telemetry => observability}/tests/BUILD.bazel | 2 +- .../tests/open_telemetry_metric_recorder_test.cc | 6 +++--- src/ray/stats/BUILD.bazel | 2 +- src/ray/stats/metric.h | 4 ++-- src/ray/stats/stats.h | 4 ++-- src/ray/stats/tests/metric_with_open_telemetry_test.cc | 8 ++++---- 9 files changed, 19 insertions(+), 19 deletions(-) rename src/ray/{telemetry => observability}/BUILD.bazel (100%) rename src/ray/{telemetry => observability}/open_telemetry_metric_recorder.cc (98%) rename src/ray/{telemetry => observability}/open_telemetry_metric_recorder.h (99%) rename src/ray/{telemetry => observability}/tests/BUILD.bazel (80%) rename src/ray/{telemetry => observability}/tests/open_telemetry_metric_recorder_test.cc (96%) diff --git a/src/ray/telemetry/BUILD.bazel b/src/ray/observability/BUILD.bazel similarity index 100% rename from src/ray/telemetry/BUILD.bazel rename to src/ray/observability/BUILD.bazel diff --git a/src/ray/telemetry/open_telemetry_metric_recorder.cc b/src/ray/observability/open_telemetry_metric_recorder.cc similarity index 98% rename from src/ray/telemetry/open_telemetry_metric_recorder.cc rename to src/ray/observability/open_telemetry_metric_recorder.cc index 8db1f8e44984..8c851ed84f05 100644 --- a/src/ray/telemetry/open_telemetry_metric_recorder.cc +++ b/src/ray/observability/open_telemetry_metric_recorder.cc @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/telemetry/open_telemetry_metric_recorder.h" +#include "ray/observability/open_telemetry_metric_recorder.h" #include #include @@ -33,7 +33,7 @@ // Anonymous namespace that contains the private callback functions for the // OpenTelemetry metrics. namespace { -using ray::telemetry::OpenTelemetryMetricRecorder; +using ray::observability::OpenTelemetryMetricRecorder; static void _DoubleGaugeCallback(opentelemetry::metrics::ObserverResult observer, void *state) { @@ -50,7 +50,7 @@ static void _DoubleGaugeCallback(opentelemetry::metrics::ObserverResult observer } // anonymous namespace namespace ray { -namespace telemetry { +namespace observability { OpenTelemetryMetricRecorder &OpenTelemetryMetricRecorder::GetInstance() { // Note: This creates a singleton instance of the OpenTelemetryMetricRecorder. The @@ -275,5 +275,5 @@ void OpenTelemetryMetricRecorder::SetSynchronousMetricValue( } } -} // namespace telemetry +} // namespace observability } // namespace ray diff --git a/src/ray/telemetry/open_telemetry_metric_recorder.h b/src/ray/observability/open_telemetry_metric_recorder.h similarity index 99% rename from src/ray/telemetry/open_telemetry_metric_recorder.h rename to src/ray/observability/open_telemetry_metric_recorder.h index 5401da24a994..f21181f1739a 100644 --- a/src/ray/telemetry/open_telemetry_metric_recorder.h +++ b/src/ray/observability/open_telemetry_metric_recorder.h @@ -31,7 +31,7 @@ #include "absl/container/flat_hash_map.h" namespace ray { -namespace telemetry { +namespace observability { // OpenTelemetryMetricRecorder is a singleton class that initializes the OpenTelemetry // grpc exporter and creates a Meter for recording metrics. It is responsible for @@ -159,5 +159,5 @@ class OpenTelemetryMetricRecorder { friend class OpenTelemetryMetricRecorderTest; friend class OpenTelemetryMetricRecorderTest_TestGaugeMetric_Test; }; -} // namespace telemetry +} // namespace observability } // namespace ray diff --git a/src/ray/telemetry/tests/BUILD.bazel b/src/ray/observability/tests/BUILD.bazel similarity index 80% rename from src/ray/telemetry/tests/BUILD.bazel rename to src/ray/observability/tests/BUILD.bazel index 78af015a1fd0..e150e80aab35 100644 --- a/src/ray/telemetry/tests/BUILD.bazel +++ b/src/ray/observability/tests/BUILD.bazel @@ -6,7 +6,7 @@ ray_cc_test( srcs = ["open_telemetry_metric_recorder_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/telemetry:open_telemetry_metric_recorder", + "//src/ray/observability:open_telemetry_metric_recorder", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/telemetry/tests/open_telemetry_metric_recorder_test.cc b/src/ray/observability/tests/open_telemetry_metric_recorder_test.cc similarity index 96% rename from src/ray/telemetry/tests/open_telemetry_metric_recorder_test.cc rename to src/ray/observability/tests/open_telemetry_metric_recorder_test.cc index 0c38f24a1532..c4cca1a6b896 100644 --- a/src/ray/telemetry/tests/open_telemetry_metric_recorder_test.cc +++ b/src/ray/observability/tests/open_telemetry_metric_recorder_test.cc @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/telemetry/open_telemetry_metric_recorder.h" +#include "ray/observability/open_telemetry_metric_recorder.h" #include "gtest/gtest.h" namespace ray { -namespace telemetry { +namespace observability { class OpenTelemetryMetricRecorderTest : public ::testing::Test { public: @@ -84,5 +84,5 @@ TEST_F(OpenTelemetryMetricRecorderTest, TestHistogramMetric) { ASSERT_TRUE(recorder_.IsMetricRegistered("test_histogram")); } -} // namespace telemetry +} // namespace observability } // namespace ray diff --git a/src/ray/stats/BUILD.bazel b/src/ray/stats/BUILD.bazel index 05496b27ae4f..3187c1043ca6 100644 --- a/src/ray/stats/BUILD.bazel +++ b/src/ray/stats/BUILD.bazel @@ -14,7 +14,7 @@ ray_cc_library( ], deps = [ "//src/ray/common:ray_config", - "//src/ray/telemetry:open_telemetry_metric_recorder", + "//src/ray/observability:open_telemetry_metric_recorder", "//src/ray/util", "//src/ray/util:logging", "//src/ray/util:size_literals", diff --git a/src/ray/stats/metric.h b/src/ray/stats/metric.h index 10322d4e2c85..f53056492dcc 100644 --- a/src/ray/stats/metric.h +++ b/src/ray/stats/metric.h @@ -29,7 +29,7 @@ #include "opencensus/stats/stats_exporter.h" #include "opencensus/tags/tag_key.h" #include "ray/common/ray_config.h" -#include "ray/telemetry/open_telemetry_metric_recorder.h" +#include "ray/observability/open_telemetry_metric_recorder.h" #include "ray/util/logging.h" namespace ray { @@ -39,7 +39,7 @@ namespace stats { /// Include tag_defs.h to define tag items #include "ray/stats/tag_defs.h" -using OpenTelemetryMetricRecorder = ray::telemetry::OpenTelemetryMetricRecorder; +using OpenTelemetryMetricRecorder = ray::observability::OpenTelemetryMetricRecorder; /// StatsConfig per process. /// Note that this is not thread-safe. Don't modify its internal values diff --git a/src/ray/stats/stats.h b/src/ray/stats/stats.h index 74c1f25e8a0e..af952e5b3ccc 100644 --- a/src/ray/stats/stats.h +++ b/src/ray/stats/stats.h @@ -28,9 +28,9 @@ #include "ray/common/asio/io_service_pool.h" #include "ray/common/id.h" #include "ray/common/ray_config.h" +#include "ray/observability/open_telemetry_metric_recorder.h" #include "ray/stats/metric.h" #include "ray/stats/metric_exporter.h" -#include "ray/telemetry/open_telemetry_metric_recorder.h" #include "ray/util/logging.h" #include "ray/util/network_util.h" @@ -40,7 +40,7 @@ namespace stats { #include -using OpenTelemetryMetricRecorder = ray::telemetry::OpenTelemetryMetricRecorder; +using OpenTelemetryMetricRecorder = ray::observability::OpenTelemetryMetricRecorder; // TODO(sang) Put all states and logic into a singleton class Stats. static std::shared_ptr metrics_io_service_pool; diff --git a/src/ray/stats/tests/metric_with_open_telemetry_test.cc b/src/ray/stats/tests/metric_with_open_telemetry_test.cc index d6aeb3cbb471..4c337d47fb76 100644 --- a/src/ray/stats/tests/metric_with_open_telemetry_test.cc +++ b/src/ray/stats/tests/metric_with_open_telemetry_test.cc @@ -14,14 +14,14 @@ #include "gtest/gtest.h" #include "ray/common/ray_config.h" +#include "ray/observability/open_telemetry_metric_recorder.h" #include "ray/stats/metric.h" -#include "ray/telemetry/open_telemetry_metric_recorder.h" namespace ray { -namespace telemetry { +namespace observability { using namespace std::literals; -using OpenTelemetryMetricRecorder = ray::telemetry::OpenTelemetryMetricRecorder; +using OpenTelemetryMetricRecorder = ray::observability::OpenTelemetryMetricRecorder; using StatsConfig = ray::stats::StatsConfig; using TagsMap = absl::flat_hash_map; @@ -233,5 +233,5 @@ INSTANTIATE_TEST_SUITE_P( {"Tag2", "Value5"}}, /*expected_value=*/82.0})); -} // namespace telemetry +} // namespace observability } // namespace ray From 3ea021227eaeb0404c42cf09015bc685eb097cfb Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 18 Aug 2025 17:28:48 -0500 Subject: [PATCH 166/634] [core] Separate targets for pubsub interfaces (#55681) Move publisher & subscriber interfaces into their own header files & build targets. Update relevant callsites to use them. Unfortunately, `reference_count_test` reaches into internal implementation details of the publisher and this dependency was a little tricky to break, so not touching it here. --------- Signed-off-by: Edward Oakes --- BUILD.bazel | 12 +- src/fakes/ray/pubsub/publisher.h | 8 +- src/mock/ray/pubsub/BUILD.bazel | 17 +++ src/mock/ray/pubsub/publisher.h | 17 ++- src/mock/ray/pubsub/subscriber.h | 14 -- src/mock/ray/rpc/worker/core_worker_client.h | 5 +- src/ray/core_worker/BUILD.bazel | 4 +- src/ray/core_worker/reference_count.h | 4 +- src/ray/core_worker/tests/BUILD.bazel | 10 +- .../core_worker/tests/reference_count_test.cc | 17 ++- src/ray/gcs/gcs_client/gcs_client.cc | 2 + src/ray/gcs/gcs_server/tests/BUILD.bazel | 15 +- .../gcs_actor_manager_export_event_test.cc | 2 +- .../gcs_job_manager_export_event_test.cc | 1 - .../tests/gcs_actor_manager_test.cc | 2 +- .../tests/gcs_actor_scheduler_mock_test.cc | 1 - .../gcs_autoscaler_state_manager_test.cc | 1 - src/ray/gcs/pubsub/BUILD.bazel | 4 +- src/ray/gcs/pubsub/gcs_pub_sub.h | 13 +- src/ray/object_manager/BUILD.bazel | 2 +- src/ray/object_manager/tests/BUILD.bazel | 2 +- src/ray/pubsub/BUILD.bazel | 25 +++ src/ray/pubsub/publisher.cc | 31 ++-- src/ray/pubsub/publisher.h | 106 +++---------- src/ray/pubsub/publisher_interface.h | 92 ++++++++++++ src/ray/pubsub/subscriber.cc | 32 ++-- src/ray/pubsub/subscriber.h | 141 ++--------------- src/ray/pubsub/subscriber_interface.h | 142 ++++++++++++++++++ src/ray/pubsub/tests/integration_test.cc | 2 + src/ray/pubsub/tests/publisher_test.cc | 8 +- src/ray/pubsub/tests/subscriber_test.cc | 4 +- src/ray/raylet/BUILD.bazel | 2 +- src/ray/raylet/local_object_manager.h | 2 +- src/ray/raylet/tests/BUILD.bazel | 1 + src/ray/rpc/worker/core_worker_client.h | 4 + 35 files changed, 440 insertions(+), 305 deletions(-) create mode 100644 src/mock/ray/pubsub/BUILD.bazel create mode 100644 src/ray/pubsub/publisher_interface.h create mode 100644 src/ray/pubsub/subscriber_interface.h diff --git a/BUILD.bazel b/BUILD.bazel index 804a9591b91d..1c77a53d4e73 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -110,14 +110,24 @@ refresh_compile_commands( ray_cc_library( name = "ray_mock", + # NOTE(edoakes): we are moving towards fine-grained mock and fake targets. + # Do not include new files in this target, instead make a BUILD.bazel file + # in the subdirectory and exclude it here. hdrs = glob( ["src/mock/**/*.h"], - exclude = ["src/mock/ray/common/ray_syncer/ray_syncer.h"], + exclude = [ + "src/mock/ray/common/pubsub/publisher.h", + "src/mock/ray/common/pubsub/subscriber.h", + "src/mock/ray/common/ray_syncer/ray_syncer.h", + ], ), ) ray_cc_library( name = "ray_fakes", + # NOTE(edoakes): we are moving towards fine-grained mock and fake targets. + # Do not include new files in this target, instead make a BUILD.bazel file + # in the subdirectory and exclude it here. hdrs = glob( ["src/fakes/**/*.h"], ), diff --git a/src/fakes/ray/pubsub/publisher.h b/src/fakes/ray/pubsub/publisher.h index b8daaf958f31..d6a5c7a6dae0 100644 --- a/src/fakes/ray/pubsub/publisher.h +++ b/src/fakes/ray/pubsub/publisher.h @@ -22,7 +22,7 @@ namespace pubsub { class FakePublisher : public Publisher { public: bool RegisterSubscription(const rpc::ChannelType channel_type, - const SubscriberID &subscriber_id, + const UniqueID &subscriber_id, const std::optional &key_id) override { return true; } @@ -33,10 +33,14 @@ class FakePublisher : public Publisher { const std::string &key_id) override {} bool UnregisterSubscription(const rpc::ChannelType channel_type, - const SubscriberID &subscriber_id, + const UniqueID &subscriber_id, const std::optional &key_id) override { return true; } + + void UnregisterSubscriber(const UniqueID &subscriber_id) override { return; } + + std::string DebugString() const override { return "FakePublisher"; } }; } // namespace pubsub diff --git a/src/mock/ray/pubsub/BUILD.bazel b/src/mock/ray/pubsub/BUILD.bazel new file mode 100644 index 000000000000..1a40ca0ac02c --- /dev/null +++ b/src/mock/ray/pubsub/BUILD.bazel @@ -0,0 +1,17 @@ +load("//bazel:ray.bzl", "ray_cc_library") + +ray_cc_library( + name = "mock_publisher", + hdrs = ["publisher.h"], + deps = [ + "//src/ray/pubsub:publisher_interface", + ], +) + +ray_cc_library( + name = "mock_subscriber", + hdrs = ["subscriber.h"], + deps = [ + "//src/ray/pubsub:subscriber_interface", + ], +) diff --git a/src/mock/ray/pubsub/publisher.h b/src/mock/ray/pubsub/publisher.h index 899f34fd140b..dfad1d21c2ce 100644 --- a/src/mock/ray/pubsub/publisher.h +++ b/src/mock/ray/pubsub/publisher.h @@ -12,15 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "ray/pubsub/publisher_interface.h" + namespace ray { namespace pubsub { -class MockPublisher : public Publisher { +class MockPublisher : public PublisherInterface { public: + MOCK_METHOD(void, + ConnectToSubscriber, + (const rpc::PubsubLongPollingRequest &request, + std::string *publisher_id, + google::protobuf::RepeatedPtrField *pub_messages, + rpc::SendReplyCallback send_reply_callback), + (override)); MOCK_METHOD(bool, RegisterSubscription, (const rpc::ChannelType channel_type, - const SubscriberID &subscriber_id, + const UniqueID &subscriber_id, const std::optional &key_id), (override)); MOCK_METHOD(void, Publish, (rpc::PubMessage pub_message), (override)); @@ -31,9 +40,11 @@ class MockPublisher : public Publisher { MOCK_METHOD(bool, UnregisterSubscription, (const rpc::ChannelType channel_type, - const SubscriberID &subscriber_id, + const UniqueID &subscriber_id, const std::optional &key_id), (override)); + MOCK_METHOD(void, UnregisterSubscriber, (const UniqueID &subscriber_id), (override)); + MOCK_METHOD(std::string, DebugString, (), (const, override)); }; } // namespace pubsub diff --git a/src/mock/ray/pubsub/subscriber.h b/src/mock/ray/pubsub/subscriber.h index 2aa671795ee9..cf21d3a4e022 100644 --- a/src/mock/ray/pubsub/subscriber.h +++ b/src/mock/ray/pubsub/subscriber.h @@ -20,20 +20,6 @@ namespace ray { namespace pubsub { -class MockSubscriberClientInterface : public SubscriberClientInterface { - public: - MOCK_METHOD(void, - PubsubLongPolling, - (const rpc::PubsubLongPollingRequest &request, - const rpc::ClientCallback &callback), - (override)); - MOCK_METHOD(void, - PubsubCommandBatch, - (const rpc::PubsubCommandBatchRequest &request, - const rpc::ClientCallback &callback), - (override)); -}; - class MockSubscriber : public SubscriberInterface { public: MOCK_METHOD(bool, diff --git a/src/mock/ray/rpc/worker/core_worker_client.h b/src/mock/ray/rpc/worker/core_worker_client.h index 3e7e4d734c4c..019d4b42842c 100644 --- a/src/mock/ray/rpc/worker/core_worker_client.h +++ b/src/mock/ray/rpc/worker/core_worker_client.h @@ -15,13 +15,13 @@ #pragma once #include "gmock/gmock.h" +#include "ray/pubsub/subscriber_interface.h" #include "ray/rpc/worker/core_worker_client.h" namespace ray { namespace rpc { -class MockCoreWorkerClientInterface : public ray::pubsub::MockSubscriberClientInterface, - public CoreWorkerClientInterface { +class MockCoreWorkerClientInterface : public CoreWorkerClientInterface { public: MOCK_METHOD(void, PushActorTask, @@ -129,6 +129,7 @@ class MockCoreWorkerClientInterface : public ray::pubsub::MockSubscriberClientIn (const AssignObjectOwnerRequest &request, const ClientCallback &callback), (override)); + MOCK_METHOD(std::string, DebugString, (), (const, override)); }; class MockCoreWorkerClientConfigurableRunningTasks diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 8d41c345f4c2..f4410f8a9695 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -153,8 +153,8 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/core_worker:lease_policy", "//src/ray/protobuf:common_cc_proto", - "//src/ray/pubsub:publisher", - "//src/ray/pubsub:subscriber", + "//src/ray/pubsub:publisher_interface", + "//src/ray/pubsub:subscriber_interface", "//src/ray/rpc:core_worker_client", "//src/ray/rpc:grpc_server", "//src/ray/util:logging", diff --git a/src/ray/core_worker/reference_count.h b/src/ray/core_worker/reference_count.h index c58cbe5af91a..4214ded8d24c 100644 --- a/src/ray/core_worker/reference_count.h +++ b/src/ray/core_worker/reference_count.h @@ -28,8 +28,8 @@ #include "absl/synchronization/mutex.h" #include "ray/common/id.h" #include "ray/core_worker/lease_policy.h" -#include "ray/pubsub/publisher.h" -#include "ray/pubsub/subscriber.h" +#include "ray/pubsub/publisher_interface.h" +#include "ray/pubsub/subscriber_interface.h" #include "ray/rpc/grpc_server.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index d9fef40c7b6d..8f195aeb3944 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -34,12 +34,14 @@ ray_cc_test( srcs = ["reference_count_test.cc"], tags = ["team:core"], deps = [ - "//:ray_mock", + "//src/mock/ray/pubsub:mock_publisher", + "//src/mock/ray/pubsub:mock_subscriber", "//src/ray/common:asio", "//src/ray/common:ray_object", "//src/ray/core_worker:memory_store", "//src/ray/pubsub:publisher", - "//src/ray/pubsub:subscriber", + "//src/ray/pubsub:publisher_interface", + "//src/ray/pubsub:subscriber_interface", "@com_google_absl//absl/functional:bind_front", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", @@ -54,6 +56,8 @@ ray_cc_test( deps = [ "//:ray_fakes", "//:ray_mock", + "//src/mock/ray/pubsub:mock_publisher", + "//src/mock/ray/pubsub:mock_subscriber", "//src/ray/common:task_common", "//src/ray/common:test_util", "//src/ray/core_worker:memory_store", @@ -72,6 +76,8 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", + "//src/mock/ray/pubsub:mock_publisher", + "//src/mock/ray/pubsub:mock_subscriber", "//src/ray/common:task_common", "//src/ray/common:test_util", "//src/ray/core_worker:memory_store", diff --git a/src/ray/core_worker/tests/reference_count_test.cc b/src/ray/core_worker/tests/reference_count_test.cc index 7f5f235d3c14..4ca580c67ddc 100644 --- a/src/ray/core_worker/tests/reference_count_test.cc +++ b/src/ray/core_worker/tests/reference_count_test.cc @@ -29,7 +29,8 @@ #include "ray/common/ray_object.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/pubsub/publisher.h" -#include "ray/pubsub/subscriber.h" +#include "ray/pubsub/publisher_interface.h" +#include "ray/pubsub/subscriber_interface.h" namespace ray { namespace core { @@ -129,7 +130,7 @@ class MockDistributedSubscriber : public pubsub::SubscriberInterface { MockDistributedSubscriber(pubsub::pub_internal::SubscriptionIndex *dict, SubscriptionCallbackMap *sub_callback_map, SubscriptionFailureCallbackMap *sub_failure_callback_map, - pubsub::SubscriberID subscriber_id, + UniqueID subscriber_id, PublisherFactoryFn client_factory) : directory_(dict), subscription_callback_map_(sub_callback_map), @@ -225,7 +226,7 @@ class MockDistributedSubscriber : public pubsub::SubscriberInterface { pubsub::pub_internal::SubscriptionIndex *directory_; SubscriptionCallbackMap *subscription_callback_map_; SubscriptionFailureCallbackMap *subscription_failure_callback_map_; - pubsub::SubscriberID subscriber_id_; + UniqueID subscriber_id_; std::unique_ptr subscriber_; PublisherFactoryFn client_factory_; }; @@ -243,7 +244,7 @@ class MockDistributedPublisher : public pubsub::PublisherInterface { ~MockDistributedPublisher() = default; bool RegisterSubscription(const rpc::ChannelType channel_type, - const pubsub::SubscriberID &subscriber_id, + const UniqueID &subscriber_id, const std::optional &key_id_binary) override { RAY_CHECK(false) << "No need to implement it for testing."; return false; @@ -272,17 +273,21 @@ class MockDistributedPublisher : public pubsub::PublisherInterface { } bool UnregisterSubscription(const rpc::ChannelType channel_type, - const pubsub::SubscriberID &subscriber_id, + const UniqueID &subscriber_id, const std::optional &key_id_binary) override { return true; } + void UnregisterSubscriber(const UniqueID &subscriber_id) override { return; } + void ConnectToSubscriber( const rpc::PubsubLongPollingRequest &request, std::string *publisher_id, google::protobuf::RepeatedPtrField *pub_messages, rpc::SendReplyCallback send_reply_callback) override {} + std::string DebugString() const override { return ""; } + pubsub::pub_internal::SubscriptionIndex *directory_; SubscriptionCallbackMap *subscription_callback_map_; SubscriptionFailureCallbackMap *subscription_failure_callback_map_; @@ -343,6 +348,8 @@ class MockWorkerClient : public MockCoreWorkerClientInterface { num_requests_++; } + std::string DebugString() const override { return ""; } + bool FlushBorrowerCallbacks() { // Flush all the borrower callbacks. This means that after this function is invoked, // all of ref_counts will be tracked. diff --git a/src/ray/gcs/gcs_client/gcs_client.cc b/src/ray/gcs/gcs_client/gcs_client.cc index a26fe77180ca..6f9623ea5d4e 100644 --- a/src/ray/gcs/gcs_client/gcs_client.cc +++ b/src/ray/gcs/gcs_client/gcs_client.cc @@ -48,6 +48,8 @@ class GcsSubscriberClient final : public pubsub::SubscriberClientInterface { const rpc::PubsubCommandBatchRequest &request, const rpc::ClientCallback &callback) final; + std::string DebugString() const final { return ""; } + private: const std::shared_ptr rpc_client_; }; diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index b0af67f3acd5..67d3aa490fde 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -112,7 +112,7 @@ ray_cc_test( tags = ["team:core"], deps = [ ":gcs_server_test_util", - "//:ray_mock", + "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", @@ -129,6 +129,8 @@ ray_cc_test( deps = [ ":gcs_server_test_util", "//:ray_mock", + "//src/mock/ray/pubsub:mock_publisher", + "//src/mock/ray/pubsub:mock_subscriber", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", @@ -164,6 +166,7 @@ ray_cc_test( deps = [ ":gcs_server_test_util", "//:ray_mock", + "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", @@ -182,7 +185,7 @@ ray_cc_test( ], deps = [ ":gcs_server_test_util", - "//:ray_mock", + "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", @@ -198,7 +201,7 @@ ray_cc_test( tags = ["team:core"], deps = [ ":gcs_server_test_util", - "//:ray_mock", + "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", @@ -235,6 +238,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", + "//src/ray/pubsub:publisher", "@com_google_googletest//:gtest_main", ], ) @@ -248,7 +252,7 @@ ray_cc_test( tags = ["team:core"], deps = [ ":gcs_server_test_util", - "//:ray_mock", + "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:process", @@ -363,6 +367,7 @@ ray_cc_test( deps = [ ":gcs_server_test_util", "//:ray_mock", + "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", @@ -396,7 +401,7 @@ ray_cc_test( ], deps = [ ":gcs_server_test_util", - "//:ray_mock", + "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc index d3940e6c5021..a7bc18a0216d 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc @@ -28,7 +28,7 @@ #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" -#include "mock/ray/pubsub/publisher.h" +#include "ray/pubsub/publisher.h" #include "ray/util/event.h" // clang-format on diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc index 049a6df3a4e3..33b80d640bf4 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc @@ -26,7 +26,6 @@ #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/pubsub/publisher.h" -#include "mock/ray/pubsub/subscriber.h" #include "mock/ray/rpc/worker/core_worker_client.h" // clang-format on diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc index 303085699800..8754b5a404dc 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc @@ -24,9 +24,9 @@ #include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/pubsub/publisher.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" -#include "mock/ray/pubsub/publisher.h" // clang-format on namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc index 2671f360ec09..af01302aef3b 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc @@ -23,7 +23,6 @@ #include "mock/ray/gcs/store_client/store_client.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "mock/ray/raylet_client/raylet_client.h" -#include "mock/ray/pubsub/subscriber.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/test_util.h" // clang-format on diff --git a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc index 2b95dae2729c..5f75f531f38b 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc @@ -32,7 +32,6 @@ #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "mock/ray/gcs/gcs_server/gcs_actor_manager.h" #include "mock/ray/gcs/store_client/store_client.h" -#include "mock/ray/pubsub/subscriber.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/gcs/gcs_server/gcs_autoscaler_state_manager.h" diff --git a/src/ray/gcs/pubsub/BUILD.bazel b/src/ray/gcs/pubsub/BUILD.bazel index e5a50a771846..362186603fe2 100644 --- a/src/ray/gcs/pubsub/BUILD.bazel +++ b/src/ray/gcs/pubsub/BUILD.bazel @@ -8,8 +8,8 @@ ray_cc_library( "//src/ray/common:ray_config", "//src/ray/gcs:gcs_callback", "//src/ray/gcs:gcs_redis_client", - "//src/ray/pubsub:publisher", - "//src/ray/pubsub:subscriber", + "//src/ray/pubsub:publisher_interface", + "//src/ray/pubsub:subscriber_interface", "//src/ray/rpc:gcs_client", ], ) diff --git a/src/ray/gcs/pubsub/gcs_pub_sub.h b/src/ray/gcs/pubsub/gcs_pub_sub.h index 89abb70397f9..f5b8ddead4f3 100644 --- a/src/ray/gcs/pubsub/gcs_pub_sub.h +++ b/src/ray/gcs/pubsub/gcs_pub_sub.h @@ -22,8 +22,8 @@ #include "absl/synchronization/mutex.h" #include "ray/gcs/callback.h" -#include "ray/pubsub/publisher.h" -#include "ray/pubsub/subscriber.h" +#include "ray/pubsub/publisher_interface.h" +#include "ray/pubsub/subscriber_interface.h" #include "src/ray/protobuf/gcs.pb.h" #include "src/ray/protobuf/gcs_service.grpc.pb.h" @@ -38,13 +38,13 @@ class GcsPublisher { /// Initializes GcsPublisher with GCS based publishers. /// Publish*() member functions below would be incrementally converted to use the GCS /// based publisher, if available. - explicit GcsPublisher(std::unique_ptr publisher) + explicit GcsPublisher(std::unique_ptr publisher) : publisher_(std::move(publisher)) { RAY_CHECK(publisher_); } /// Returns the underlying pubsub::Publisher. Caller does not take ownership. - pubsub::Publisher &GetPublisher() const { return *publisher_; } + pubsub::PublisherInterface &GetPublisher() const { return *publisher_; } /// Each publishing method below publishes to a different "channel". /// ID is the entity which the message is associated with, e.g. ActorID for Actor data. @@ -71,7 +71,7 @@ class GcsPublisher { std::string DebugString() const; private: - const std::unique_ptr publisher_; + const std::unique_ptr publisher_; }; /// \class GcsSubscriber @@ -81,7 +81,8 @@ class GcsSubscriber { public: /// Initializes GcsSubscriber with GCS based GcsSubscribers. // TODO(mwtian): Support restarted GCS publisher, at the same or a different address. - GcsSubscriber(rpc::Address gcs_address, std::unique_ptr subscriber) + GcsSubscriber(rpc::Address gcs_address, + std::unique_ptr subscriber) : gcs_address_(std::move(gcs_address)), subscriber_(std::move(subscriber)) {} /// Subscribe*() member functions below would be incrementally converted to use the GCS diff --git a/src/ray/object_manager/BUILD.bazel b/src/ray/object_manager/BUILD.bazel index 3013567accad..55b27775f600 100644 --- a/src/ray/object_manager/BUILD.bazel +++ b/src/ray/object_manager/BUILD.bazel @@ -67,7 +67,7 @@ ray_cc_library( "//src/ray/common:asio", "//src/ray/common:id", "//src/ray/gcs/gcs_client:gcs_client_lib", - "//src/ray/pubsub:subscriber", + "//src/ray/pubsub:subscriber_interface", "//src/ray/rpc:core_worker_client", "@com_google_absl//absl/container:flat_hash_map", ], diff --git a/src/ray/object_manager/tests/BUILD.bazel b/src/ray/object_manager/tests/BUILD.bazel index 4a7192248f8b..9d56d86614e6 100644 --- a/src/ray/object_manager/tests/BUILD.bazel +++ b/src/ray/object_manager/tests/BUILD.bazel @@ -36,7 +36,7 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - "//:ray_mock", + "//src/mock/ray/pubsub:mock_subscriber", "//src/ray/object_manager:ownership_object_directory", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/pubsub/BUILD.bazel b/src/ray/pubsub/BUILD.bazel index ba280ed8eb04..b2c5d89e0efb 100644 --- a/src/ray/pubsub/BUILD.bazel +++ b/src/ray/pubsub/BUILD.bazel @@ -1,5 +1,16 @@ load("//bazel:ray.bzl", "ray_cc_library") +ray_cc_library( + name = "publisher_interface", + hdrs = ["publisher_interface.h"], + deps = [ + "//src/ray/protobuf:pubsub_cc_grpc", + # NOTE(edoakes): we only seem to need `SendReplyCallback` from server_call.h. + # We should move that definition to its own target. + "//src/ray/rpc:server_call", + ], +) + ray_cc_library( name = "publisher", srcs = ["publisher.cc"], @@ -8,6 +19,7 @@ ray_cc_library( "//src/ray/protobuf:pubsub_cc_grpc", # NOTE(edoakes): we only seem to need `SendReplyCallback` from server_call.h. # We should move that definition to its own target. + "//src/ray/pubsub:publisher_interface", "//src/ray/rpc:server_call", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -15,6 +27,18 @@ ray_cc_library( ], ) +ray_cc_library( + name = "subscriber_interface", + hdrs = ["subscriber_interface.h"], + deps = [ + "//src/ray/protobuf:common_cc_proto", + "//src/ray/protobuf:pubsub_cc_grpc", + # NOTE(edoakes): we only seem to need `ClientCallback` from client_call.h. + # We should move that definition to its own target. + "//src/ray/rpc:client_call", + ], +) + ray_cc_library( name = "subscriber", srcs = ["subscriber.cc"], @@ -24,6 +48,7 @@ ray_cc_library( # NOTE(edoakes): we only seem to need `ClientCallback` from client_call.h. # We should move that definition to its own target. "//src/ray/rpc:client_call", + "//src/ray/pubsub:subscriber_interface", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/synchronization", diff --git a/src/ray/pubsub/publisher.cc b/src/ray/pubsub/publisher.cc index 059466242f7e..86eab003fa26 100644 --- a/src/ray/pubsub/publisher.cc +++ b/src/ray/pubsub/publisher.cc @@ -96,12 +96,11 @@ bool EntityState::AddSubscriber(SubscriberState *subscriber) { return subscribers_.emplace(subscriber->id(), subscriber).second; } -bool EntityState::RemoveSubscriber(const SubscriberID &id) { - return subscribers_.erase(id) > 0; +bool EntityState::RemoveSubscriber(const UniqueID &subscriber_id) { + return subscribers_.erase(subscriber_id) > 0; } -const absl::flat_hash_map &EntityState::Subscribers() - const { +const absl::flat_hash_map &EntityState::Subscribers() const { return subscribers_; } @@ -150,9 +149,9 @@ bool SubscriptionIndex::AddEntry(const std::string &key_id, SubscriberState *sub return key_added; } -std::vector SubscriptionIndex::GetSubscriberIdsByKeyId( +std::vector SubscriptionIndex::GetSubscriberIdsByKeyId( const std::string &key_id) const { - std::vector subscribers; + std::vector subscribers; if (!subscribers_to_all_->Subscribers().empty()) { for (const auto &[sub_id, sub] : subscribers_to_all_->Subscribers()) { subscribers.push_back(sub_id); @@ -167,7 +166,7 @@ std::vector SubscriptionIndex::GetSubscriberIdsByKeyId( return subscribers; } -bool SubscriptionIndex::EraseSubscriber(const SubscriberID &subscriber_id) { +bool SubscriptionIndex::EraseSubscriber(const UniqueID &subscriber_id) { // Erase subscriber of all keys. if (subscribers_to_all_->RemoveSubscriber(subscriber_id)) { return true; @@ -197,7 +196,7 @@ bool SubscriptionIndex::EraseSubscriber(const SubscriberID &subscriber_id) { } bool SubscriptionIndex::EraseEntry(const std::string &key_id, - const SubscriberID &subscriber_id) { + const UniqueID &subscriber_id) { // Erase the subscriber of all keys. if (key_id.empty()) { return subscribers_to_all_->RemoveSubscriber(subscriber_id); @@ -239,7 +238,7 @@ bool SubscriptionIndex::HasKeyId(const std::string &key_id) const { return entities_.contains(key_id); } -bool SubscriptionIndex::HasSubscriber(const SubscriberID &subscriber_id) const { +bool SubscriptionIndex::HasSubscriber(const UniqueID &subscriber_id) const { if (subscribers_to_all_->Subscribers().contains(subscriber_id)) { return true; } @@ -381,7 +380,7 @@ void Publisher::ConnectToSubscriber( rpc::SendReplyCallback send_reply_callback) { RAY_CHECK(send_reply_callback != nullptr); - const auto subscriber_id = SubscriberID::FromBinary(request.subscriber_id()); + const auto subscriber_id = UniqueID::FromBinary(request.subscriber_id()); RAY_LOG(DEBUG) << "Long polling connection initiated by " << subscriber_id.Hex() << ", publisher_id " << publisher_id_.Hex(); absl::MutexLock lock(&mutex_); @@ -405,7 +404,7 @@ void Publisher::ConnectToSubscriber( } bool Publisher::RegisterSubscription(const rpc::ChannelType channel_type, - const SubscriberID &subscriber_id, + const UniqueID &subscriber_id, const std::optional &key_id) { absl::MutexLock lock(&mutex_); auto it = subscribers_.find(subscriber_id); @@ -453,7 +452,7 @@ void Publisher::PublishFailure(const rpc::ChannelType channel_type, } bool Publisher::UnregisterSubscription(const rpc::ChannelType channel_type, - const SubscriberID &subscriber_id, + const UniqueID &subscriber_id, const std::optional &key_id) { absl::MutexLock lock(&mutex_); auto subscription_index_it = subscription_index_map_.find(channel_type); @@ -461,7 +460,7 @@ bool Publisher::UnregisterSubscription(const rpc::ChannelType channel_type, return subscription_index_it->second.EraseEntry(key_id.value_or(""), subscriber_id); } -void Publisher::UnregisterSubscriber(const SubscriberID &subscriber_id) { +void Publisher::UnregisterSubscriber(const UniqueID &subscriber_id) { absl::MutexLock lock(&mutex_); UnregisterSubscriberInternal(subscriber_id); } @@ -470,7 +469,7 @@ void Publisher::UnregisterAll() { absl::MutexLock lock(&mutex_); // Save the subscriber IDs to be removed, because UnregisterSubscriberInternal() // erases from subscribers_. - std::vector ids; + std::vector ids; for (const auto &[id, subscriber] : subscribers_) { ids.push_back(id); } @@ -479,7 +478,7 @@ void Publisher::UnregisterAll() { } } -void Publisher::UnregisterSubscriberInternal(const SubscriberID &subscriber_id) { +void Publisher::UnregisterSubscriberInternal(const UniqueID &subscriber_id) { RAY_LOG(DEBUG) << "Unregistering subscriber " << subscriber_id.Hex(); for (auto &index : subscription_index_map_) { index.second.EraseSubscriber(subscriber_id); @@ -497,7 +496,7 @@ void Publisher::UnregisterSubscriberInternal(const SubscriberID &subscriber_id) void Publisher::CheckDeadSubscribers() { absl::MutexLock lock(&mutex_); - std::vector dead_subscribers; + std::vector dead_subscribers; for (const auto &it : subscribers_) { const auto &subscriber = it.second; diff --git a/src/ray/pubsub/publisher.h b/src/ray/pubsub/publisher.h index e3a2e92e9e59..009fa3e1edfa 100644 --- a/src/ray/pubsub/publisher.h +++ b/src/ray/pubsub/publisher.h @@ -29,6 +29,7 @@ #include "absl/synchronization/mutex.h" #include "ray/common/asio/periodical_runner.h" #include "ray/common/id.h" +#include "ray/pubsub/publisher_interface.h" #include "ray/rpc/server_call.h" #include "src/ray/protobuf/pubsub.pb.h" @@ -36,9 +37,6 @@ namespace ray { namespace pubsub { -using SubscriberID = UniqueID; -using PublisherID = UniqueID; - namespace pub_internal { class SubscriberState; @@ -57,17 +55,17 @@ class EntityState { /// Manages the set of subscribers of this entity. bool AddSubscriber(SubscriberState *subscriber); - bool RemoveSubscriber(const SubscriberID &id); + bool RemoveSubscriber(const UniqueID &subscriber_id); /// Gets the current set of subscribers, keyed by subscriber IDs. - const absl::flat_hash_map &Subscribers() const; + const absl::flat_hash_map &Subscribers() const; size_t GetNumBufferedBytes() const { return total_size_; } protected: // Subscribers of this entity. // The underlying SubscriberState is owned by Publisher. - absl::flat_hash_map subscribers_; + absl::flat_hash_map subscribers_; private: // Tracks inflight messages. The messages have shared ownership by @@ -108,11 +106,11 @@ class SubscriptionIndex { /// Erases the subscriber from this index. /// Returns whether the subscriber exists before the call. - bool EraseSubscriber(const SubscriberID &subscriber_id); + bool EraseSubscriber(const UniqueID &subscriber_id); /// Erases the subscriber from the particular key. /// When `key_id` is empty, the subscriber subscribes to all keys. - bool EraseEntry(const std::string &key_id, const SubscriberID &subscriber_id); + bool EraseEntry(const std::string &key_id, const UniqueID &subscriber_id); /// Test only. /// Returns true if the entity id exists in the index. @@ -122,11 +120,11 @@ class SubscriptionIndex { /// Test only. /// Returns true if the subscriber id exists in the index, including both per-entity /// and all-entity subscribers. - bool HasSubscriber(const SubscriberID &subscriber_id) const; + bool HasSubscriber(const UniqueID &subscriber_id) const; /// Returns a vector of subscriber ids that are subscribing to the given object ids. /// Test only. - std::vector GetSubscriberIdsByKeyId(const std::string &key_id) const; + std::vector GetSubscriberIdsByKeyId(const std::string &key_id) const; int64_t GetNumBufferedBytes() const; @@ -144,8 +142,7 @@ class SubscriptionIndex { absl::flat_hash_map> entities_; // Mapping from subscriber IDs -> subscribed key ids. // Reverse index of key_id_to_subscribers_. - absl::flat_hash_map> - subscribers_to_key_id_; + absl::flat_hash_map> subscribers_to_key_id_; }; struct LongPollConnection { @@ -164,11 +161,11 @@ struct LongPollConnection { /// Keeps the state of each connected subscriber. class SubscriberState { public: - SubscriberState(SubscriberID subscriber_id, + SubscriberState(UniqueID subscriber_id, std::function get_time_ms, uint64_t connection_timeout_ms, int64_t publish_batch_size, - PublisherID publisher_id) + UniqueID publisher_id) : subscriber_id_(subscriber_id), get_time_ms_(std::move(get_time_ms)), connection_timeout_ms_(connection_timeout_ms), @@ -214,11 +211,11 @@ class SubscriberState { bool IsActive() const; /// Returns the ID of this subscriber. - const SubscriberID &id() const { return subscriber_id_; } + const UniqueID &id() const { return subscriber_id_; } private: /// Subscriber ID, for logging and debugging. - const SubscriberID subscriber_id_; + const UniqueID subscriber_id_; /// Inflight long polling reply callback, for replying to the subscriber. std::unique_ptr long_polling_connection_; /// Queued messages to publish. @@ -236,60 +233,6 @@ class SubscriberState { } // namespace pub_internal -/// Publisher interface. Note that message ids are passed as a string to avoid templated -/// definition which doesn't go well with virtual methods. -class PublisherInterface { - public: - virtual ~PublisherInterface() = default; - - /// Handle a long poll request from `subscriber_id`. - /// - /// TODO(sang): Currently, we need to pass the callback for connection because we are - /// using long polling internally. This should be changed once the bidirectional grpc - /// streaming is supported. - virtual void ConnectToSubscriber( - const rpc::PubsubLongPollingRequest &request, - std::string *publisher_id, - google::protobuf::RepeatedPtrField *pub_messages, - rpc::SendReplyCallback send_reply_callback) = 0; - - /// Register the subscription. - /// - /// \param channel_type The type of the channel. - /// \param subscriber_id The node id of the subscriber. - /// \param key_id The key_id that the subscriber is subscribing to. std::nullopt if - /// subscribing to all. - /// \return True if registration is new. False otherwise. - virtual bool RegisterSubscription(const rpc::ChannelType channel_type, - const SubscriberID &subscriber_id, - const std::optional &key_id) = 0; - - /// Publish the given object id to subscribers. - /// - /// \param pub_message The message to publish. - /// Required to contain channel_type and key_id fields. - virtual void Publish(rpc::PubMessage pub_message) = 0; - - /// Publish to the subscriber that the given key id is not available anymore. - /// It will invoke the failure callback on the subscriber side. - /// - /// \param channel_type The type of the channel. - /// \param key_id The message id to publish. - virtual void PublishFailure(const rpc::ChannelType channel_type, - const std::string &key_id) = 0; - - /// Unregister subscription. It means the given object id won't be published to the - /// subscriber anymore. - /// - /// \param channel_type The type of the channel. - /// \param subscriber_id The node id of the subscriber. - /// \param key_id The key_id of the subscriber. std::nullopt if subscribing to all. - /// \return True if erased. False otherwise. - virtual bool UnregisterSubscription(const rpc::ChannelType channel_type, - const SubscriberID &subscriber_id, - const std::optional &key_id) = 0; -}; - /// Protocol detail /// /// - Subscriber always send a long polling connection as long as there are subscribed @@ -321,7 +264,7 @@ class Publisher : public PublisherInterface { std::function get_time_ms, const uint64_t subscriber_timeout_ms, int64_t publish_batch_size, - PublisherID publisher_id = NodeID::FromRandom()) + UniqueID publisher_id = NodeID::FromRandom()) : periodical_runner_(&periodical_runner), get_time_ms_(std::move(get_time_ms)), subscriber_timeout_ms_(subscriber_timeout_ms), @@ -344,7 +287,7 @@ class Publisher : public PublisherInterface { rpc::SendReplyCallback send_reply_callback) override; bool RegisterSubscription(const rpc::ChannelType channel_type, - const SubscriberID &subscriber_id, + const UniqueID &subscriber_id, const std::optional &key_id) override; void Publish(rpc::PubMessage pub_message) override; @@ -353,14 +296,12 @@ class Publisher : public PublisherInterface { const std::string &key_id) override; bool UnregisterSubscription(const rpc::ChannelType channel_type, - const SubscriberID &subscriber_id, + const UniqueID &subscriber_id, const std::optional &key_id) override; - /// Remove the subscriber. Once the subscriber is removed, messages won't be published - /// to it anymore. - /// - /// \param subscriber_id The node id of the subscriber to unsubscribe. - void UnregisterSubscriber(const SubscriberID &subscriber_id); + void UnregisterSubscriber(const UniqueID &subscriber_id) override; + + std::string DebugString() const override; /// Flushes all inflight pollings and unregisters all subscribers. void UnregisterAll(); @@ -384,8 +325,6 @@ class Publisher : public PublisherInterface { /// having a timer per subscriber. void CheckDeadSubscribers(); - std::string DebugString() const; - private: /// /// Testing fields @@ -415,7 +354,7 @@ class Publisher : public PublisherInterface { /// Private fields /// - void UnregisterSubscriberInternal(const SubscriberID &subscriber_id) + void UnregisterSubscriberInternal(const UniqueID &subscriber_id) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Periodic runner to invoke CheckDeadSubscribers. @@ -434,7 +373,7 @@ class Publisher : public PublisherInterface { mutable absl::Mutex mutex_; /// Mapping of node id -> subscribers. - absl::flat_hash_map> + absl::flat_hash_map> subscribers_ ABSL_GUARDED_BY(mutex_); /// Index that stores the mapping of messages <-> subscribers. @@ -463,8 +402,7 @@ class Publisher : public PublisherInterface { /// of a channel. int64_t next_sequence_id_ ABSL_GUARDED_BY(mutex_) = 0; - /// A unique identifier identifies the publisher_id. - const PublisherID publisher_id_; + const UniqueID publisher_id_; }; } // namespace pubsub diff --git a/src/ray/pubsub/publisher_interface.h b/src/ray/pubsub/publisher_interface.h new file mode 100644 index 000000000000..2ff6bafe7ecc --- /dev/null +++ b/src/ray/pubsub/publisher_interface.h @@ -0,0 +1,92 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "ray/common/id.h" +#include "ray/rpc/server_call.h" +#include "src/ray/protobuf/pubsub.pb.h" + +namespace ray { +namespace pubsub { + +/// Publisher interface. Note that message ids are passed as a string to avoid templated +/// definition which doesn't go well with virtual methods. +class PublisherInterface { + public: + virtual ~PublisherInterface() = default; + + /// Handle a long poll request from `subscriber_id`. + /// + /// TODO(sang): Currently, we need to pass the callback for connection because we are + /// using long polling internally. This should be changed once the bidirectional grpc + /// streaming is supported. + virtual void ConnectToSubscriber( + const rpc::PubsubLongPollingRequest &request, + std::string *publisher_id, + google::protobuf::RepeatedPtrField *pub_messages, + rpc::SendReplyCallback send_reply_callback) = 0; + + /// Register the subscription. + /// + /// \param channel_type The type of the channel. + /// \param subscriber_id The ID of the subscriber. + /// \param key_id The key_id that the subscriber is subscribing to. std::nullopt if + /// subscribing to all. + /// \return True if registration is new. False otherwise. + virtual bool RegisterSubscription(const rpc::ChannelType channel_type, + const UniqueID &subscriber_id, + const std::optional &key_id) = 0; + + /// Publish the given object id to subscribers. + /// + /// \param pub_message The message to publish. + /// Required to contain channel_type and key_id fields. + virtual void Publish(rpc::PubMessage pub_message) = 0; + + /// Publish to the subscriber that the given key id is not available anymore. + /// It will invoke the failure callback on the subscriber side. + /// + /// \param channel_type The type of the channel. + /// \param key_id The message id to publish. + virtual void PublishFailure(const rpc::ChannelType channel_type, + const std::string &key_id) = 0; + + /// Unregister subscription. It means the given object id won't be published to the + /// subscriber anymore. + /// + /// \param channel_type The type of the channel. + /// \param subscriber_id The ID of the subscriber. + /// \param key_id The key_id of the subscriber. std::nullopt if subscribing to all. + /// \return True if erased. False otherwise. + virtual bool UnregisterSubscription(const rpc::ChannelType channel_type, + const UniqueID &subscriber_id, + const std::optional &key_id) = 0; + + /// Unregister subscriber. No messages on any channels will be published to it anymore. + /// + /// \param subscriber_id The ID of the subscriber. + virtual void UnregisterSubscriber(const UniqueID &subscriber_id) = 0; + + virtual std::string DebugString() const = 0; +}; + +} // namespace pubsub +} // namespace ray diff --git a/src/ray/pubsub/subscriber.cc b/src/ray/pubsub/subscriber.cc index ed167d78f097..89f9aca913c2 100644 --- a/src/ray/pubsub/subscriber.cc +++ b/src/ray/pubsub/subscriber.cc @@ -23,7 +23,7 @@ namespace ray { namespace pubsub { namespace { -const PublisherID kDefaultPublisherID{}; +const UniqueID kDefaultUniqueID{}; } /////////////////////////////////////////////////////////////////////////////// @@ -36,7 +36,7 @@ bool SubscriberChannel::Subscribe( SubscriptionItemCallback subscription_callback, SubscriptionFailureCallback subscription_failure_callback) { cum_subscribe_requests_++; - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); if (key_id) { return subscription_map_[publisher_id] @@ -59,7 +59,7 @@ bool SubscriberChannel::Subscribe( bool SubscriberChannel::Unsubscribe(const rpc::Address &publisher_address, const std::optional &key_id) { cum_unsubscribe_requests_++; - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); // Find subscription info. auto subscription_it = subscription_map_.find(publisher_id); @@ -94,7 +94,7 @@ bool SubscriberChannel::Unsubscribe(const rpc::Address &publisher_address, bool SubscriberChannel::IsSubscribed(const rpc::Address &publisher_address, const std::string &key_id) const { - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); auto subscription_it = subscription_map_.find(publisher_id); if (subscription_it == subscription_map_.end()) { return false; @@ -122,7 +122,7 @@ bool SubscriberChannel::CheckNoLeaks() const { void SubscriberChannel::HandlePublishedMessage(const rpc::Address &publisher_address, rpc::PubMessage &&pub_message) const { - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); auto subscription_it = subscription_map_.find(publisher_id); // If there's no more subscription, do nothing. if (subscription_it == subscription_map_.end()) { @@ -154,7 +154,7 @@ void SubscriberChannel::HandlePublishedMessage(const rpc::Address &publisher_add void SubscriberChannel::HandlePublisherFailure(const rpc::Address &publisher_address, const Status &status) { - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); const auto &subscription_it = subscription_map_.find(publisher_id); // If there's no more subscription, do nothing. if (subscription_it == subscription_map_.end()) { @@ -183,7 +183,7 @@ void SubscriberChannel::HandlePublisherFailure(const rpc::Address &publisher_add void SubscriberChannel::HandlePublisherFailure(const rpc::Address &publisher_address, const std::string &key_id) { - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); const auto &subscription_it = subscription_map_.find(publisher_id); // If there's no more subscription, do nothing. if (subscription_it == subscription_map_.end()) { @@ -279,7 +279,7 @@ bool Subscriber::Unsubscribe(const rpc::ChannelType channel_type, command->cmd.mutable_unsubscribe_message(); absl::MutexLock lock(&mutex_); - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); commands_[publisher_id].emplace(std::move(command)); SendCommandBatchIfPossible(publisher_address); @@ -294,7 +294,7 @@ bool Subscriber::UnsubscribeChannel(const rpc::ChannelType channel_type, command->cmd.mutable_unsubscribe_message(); absl::MutexLock lock(&mutex_); - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); commands_[publisher_id].emplace(std::move(command)); SendCommandBatchIfPossible(publisher_address); @@ -330,7 +330,7 @@ bool Subscriber::SubscribeInternal( command->cmd.mutable_subscribe_message()->Swap(sub_message.get()); } command->done_cb = std::move(subscribe_done_callback); - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); absl::MutexLock lock(&mutex_); commands_[publisher_id].emplace(std::move(command)); @@ -345,7 +345,7 @@ bool Subscriber::SubscribeInternal( void Subscriber::MakeLongPollingConnectionIfNotConnected( const rpc::Address &publisher_address) { - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); auto publishers_connected_it = publishers_connected_.find(publisher_id); if (publishers_connected_it == publishers_connected_.end()) { publishers_connected_.emplace(publisher_id); @@ -354,7 +354,7 @@ void Subscriber::MakeLongPollingConnectionIfNotConnected( } void Subscriber::MakeLongPollingPubsubConnection(const rpc::Address &publisher_address) { - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); RAY_LOG(DEBUG) << "Make a long polling request to " << publisher_id; auto subscriber_client = get_client_(publisher_address); rpc::PubsubLongPollingRequest long_polling_request; @@ -373,7 +373,7 @@ void Subscriber::MakeLongPollingPubsubConnection(const rpc::Address &publisher_a void Subscriber::HandleLongPollingResponse(const rpc::Address &publisher_address, const Status &status, rpc::PubsubLongPollingReply &&reply) { - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); RAY_LOG(DEBUG) << "Long polling request has been replied from " << publisher_id; RAY_CHECK(publishers_connected_.count(publisher_id)); @@ -390,9 +390,9 @@ void Subscriber::HandleLongPollingResponse(const rpc::Address &publisher_address commands_.erase(publisher_id); } else { RAY_CHECK(!reply.publisher_id().empty()) << "publisher_id is empty."; - auto reply_publisher_id = PublisherID::FromBinary(reply.publisher_id()); + auto reply_publisher_id = UniqueID::FromBinary(reply.publisher_id()); if (reply_publisher_id != processed_sequences_[publisher_id].first) { - if (processed_sequences_[publisher_id].first != kDefaultPublisherID) { + if (processed_sequences_[publisher_id].first != kDefaultUniqueID) { RAY_LOG(INFO) << "Received publisher_id " << reply_publisher_id.Hex() << " is different from last seen publisher_id " << processed_sequences_[publisher_id].first @@ -447,7 +447,7 @@ void Subscriber::HandleLongPollingResponse(const rpc::Address &publisher_address } void Subscriber::SendCommandBatchIfPossible(const rpc::Address &publisher_address) { - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); auto command_batch_sent_it = command_batch_sent_.find(publisher_id); // If there's no in flight command batch request to the publisher, diff --git a/src/ray/pubsub/subscriber.h b/src/ray/pubsub/subscriber.h index b76068c49775..0bdcdcb2674e 100644 --- a/src/ray/pubsub/subscriber.h +++ b/src/ray/pubsub/subscriber.h @@ -27,6 +27,7 @@ #include "absl/container/flat_hash_set.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" +#include "ray/pubsub/subscriber_interface.h" #include "ray/rpc/client_call.h" #include "src/ray/protobuf/common.pb.h" #include "src/ray/protobuf/pubsub.pb.h" @@ -35,17 +36,6 @@ namespace ray { namespace pubsub { -using SubscriberID = UniqueID; -using PublisherID = UniqueID; -using SubscribeDoneCallback = std::function; -using SubscriptionItemCallback = std::function; -using SubscriptionFailureCallback = - std::function; - -/////////////////////////////////////////////////////////////////////////////// -/// SubscriberChannel Abstraction -/////////////////////////////////////////////////////////////////////////////// - /// Subscription info stores metadata that is needed for subscriptions. struct SubscriptionInfo { SubscriptionInfo(SubscriptionItemCallback i_cb, SubscriptionFailureCallback f_cb) @@ -134,7 +124,7 @@ class SubscriberChannel { const std::string &key_id); /// Return true if the subscription exists for a given publisher id. - bool SubscriptionExists(const PublisherID &publisher_id) { + bool SubscriptionExists(const UniqueID &publisher_id) { return subscription_map_.contains(publisher_id); } @@ -156,7 +146,7 @@ class SubscriberChannel { /// subscribed. std::optional GetSubscriptionItemCallback( const rpc::Address &publisher_address, const std::string &key_id) const { - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); auto subscription_it = subscription_map_.find(publisher_id); if (subscription_it == subscription_map_.end()) { return absl::nullopt; @@ -175,7 +165,7 @@ class SubscriberChannel { /// subscribed. std::optional GetFailureCallback( const rpc::Address &publisher_address, const std::string &key_id) const { - const auto publisher_id = PublisherID::FromBinary(publisher_address.worker_id()); + const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); auto subscription_it = subscription_map_.find(publisher_id); if (subscription_it == subscription_map_.end()) { return absl::nullopt; @@ -193,7 +183,7 @@ class SubscriberChannel { const rpc::ChannelType channel_type_; /// Mapping of the publisher ID -> subscription info for the publisher. - absl::flat_hash_map subscription_map_; + absl::flat_hash_map subscription_map_; /// An event loop to execute RPC callbacks. This should be equivalent to the client /// pool's io service. @@ -208,113 +198,6 @@ class SubscriberChannel { mutable uint64_t cum_processed_messages_ = 0; }; -/////////////////////////////////////////////////////////////////////////////// -/// Subscriber Abstraction -/////////////////////////////////////////////////////////////////////////////// - -/// Interface for the pubsub client. -class SubscriberInterface { - public: - /// There are two modes of subscriptions. Each channel can only be subscribed in one - /// mode, i.e. - /// - Calling Subscribe() to subscribe to one or more entities in a channel - /// - Calling SubscribeChannel() once to subscribe to all entities in a channel - /// It is an error to call both Subscribe() and SubscribeChannel() on the same channel - /// type. This restriction can be relaxed later, if there is a use case. - - /// Subscribe to entity key_id in channel channel_type. - /// NOTE(sang): All the callbacks could be executed in a different thread from a caller. - /// For example, Subscriber executes callbacks on a passed io_service. - /// - /// \param sub_message The subscription message. - /// \param channel_type The channel to subscribe to. - /// \param publisher_address Address of the publisher to subscribe the object. - /// \param key_id The entity id to subscribe from the publisher. - /// \param subscription_callback A callback that is invoked whenever the given entity - /// information is received by the subscriber. - /// \param subscription_failure_callback A callback that is invoked whenever the - /// connection to publisher is broken (e.g. the publisher fails). - /// \return True if inserted, false if the key already exists and this becomes a no-op. - [[nodiscard]] virtual bool Subscribe( - std::unique_ptr sub_message, - rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - const std::string &key_id, - SubscribeDoneCallback subscribe_done_callback, - SubscriptionItemCallback subscription_callback, - SubscriptionFailureCallback subscription_failure_callback) = 0; - - /// Subscribe to all entities in channel channel_type. - /// - /// \param sub_message The subscription message. - /// \param channel_type The channel to subscribe to. - /// \param publisher_address Address of the publisher to subscribe the object. - /// \param subscription_callback A callback that is invoked whenever an entity - /// information is received by the subscriber. - /// \param subscription_failure_callback A callback that is invoked whenever the - /// connection to publisher is broken (e.g. the publisher fails). - /// \return True if inserted, false if the channel is already subscribed and this - /// becomes a no-op. - [[nodiscard]] virtual bool SubscribeChannel( - std::unique_ptr sub_message, - rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - SubscribeDoneCallback subscribe_done_callback, - SubscriptionItemCallback subscription_callback, - SubscriptionFailureCallback subscription_failure_callback) = 0; - - /// Unsubscribe the entity if the entity has been subscribed with Subscribe(). - /// NOTE: Calling this method inside subscription_failure_callback is not allowed. - /// - /// \param channel_type The channel to unsubscribe from. - /// \param publisher_address The publisher address that it will unsubscribe from. - /// \param key_id The entity id to unsubscribe. - /// \return Returns whether the entity key_id has been subscribed before. - virtual bool Unsubscribe(const rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - const std::string &key_id) = 0; - - /// Unsubscribe from the channel_type. Must be paired with SubscribeChannel(). - /// NOTE: Calling this method inside subscription_failure_callback is not allowed. - /// - /// \param channel_type The channel to unsubscribe from. - /// \param publisher_address The publisher address that it will unsubscribe from. - /// \return Returns whether the entity key_id has been subscribed before. - virtual bool UnsubscribeChannel(const rpc::ChannelType channel_type, - const rpc::Address &publisher_address) = 0; - - /// Test only. - /// Checks if the entity key_id is being subscribed to specifically. - /// Does not consider if SubscribeChannel() has been called on the channel. - /// - /// \param publisher_address The publisher address to check. - /// \param key_id The entity id to check. - [[nodiscard]] virtual bool IsSubscribed(const rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - const std::string &key_id) const = 0; - - /// Return the statistics string for the subscriber. - virtual std::string DebugString() const = 0; - - virtual ~SubscriberInterface() {} -}; - -/// The grpc client that the subscriber needs. -class SubscriberClientInterface { - public: - /// Send a long polling request to a core worker for pubsub operations. - virtual void PubsubLongPolling( - const rpc::PubsubLongPollingRequest &request, - const rpc::ClientCallback &callback) = 0; - - /// Send a pubsub command batch request to a core worker for pubsub operations. - virtual void PubsubCommandBatch( - const rpc::PubsubCommandBatchRequest &request, - const rpc::ClientCallback &callback) = 0; - - virtual ~SubscriberClientInterface() = default; -}; - /// The pubsub client implementation. The class is thread-safe. /// /// Protocol details: @@ -332,7 +215,7 @@ class SubscriberClientInterface { class Subscriber : public SubscriberInterface { public: Subscriber( - const SubscriberID subscriber_id, + const UniqueID subscriber_id, const std::vector &channels, const int64_t max_command_batch_size, std::function(const rpc::Address &)> @@ -454,7 +337,7 @@ class Subscriber : public SubscriberInterface { ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); /// Return true if the given publisher id has subscription to any of channel. - bool SubscriptionExists(const PublisherID &publisher_id) + bool SubscriptionExists(const UniqueID &publisher_id) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) { return std::any_of(channels_.begin(), channels_.end(), [publisher_id](const auto &p) { return p.second->SubscriptionExists(publisher_id); @@ -462,7 +345,7 @@ class Subscriber : public SubscriberInterface { } /// Self node's identifying information. - const SubscriberID subscriber_id_; + const UniqueID subscriber_id_; /// The command batch size for the subscriber. const int64_t max_command_batch_size_; @@ -483,14 +366,14 @@ class Subscriber : public SubscriberInterface { SubscribeDoneCallback done_cb; }; using CommandQueue = std::queue>; - absl::flat_hash_map commands_ ABSL_GUARDED_BY(mutex_); + absl::flat_hash_map commands_ ABSL_GUARDED_BY(mutex_); /// A set to cache the connected publisher ids. "Connected" means the long polling /// request is in flight. - absl::flat_hash_set publishers_connected_ ABSL_GUARDED_BY(mutex_); + absl::flat_hash_set publishers_connected_ ABSL_GUARDED_BY(mutex_); /// A set to keep track of in-flight command batch requests - absl::flat_hash_set command_batch_sent_ ABSL_GUARDED_BY(mutex_); + absl::flat_hash_set command_batch_sent_ ABSL_GUARDED_BY(mutex_); /// Mapping of channel type to channels. absl::flat_hash_map> channels_ @@ -498,7 +381,7 @@ class Subscriber : public SubscriberInterface { /// Keeps track of last processed by publisher. /// Note the publisher_id only change if gcs failover. - absl::flat_hash_map> processed_sequences_ + absl::flat_hash_map> processed_sequences_ ABSL_GUARDED_BY(mutex_); }; diff --git a/src/ray/pubsub/subscriber_interface.h b/src/ray/pubsub/subscriber_interface.h new file mode 100644 index 000000000000..42d8fbfeac8c --- /dev/null +++ b/src/ray/pubsub/subscriber_interface.h @@ -0,0 +1,142 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "ray/common/id.h" +#include "ray/rpc/client_call.h" +#include "src/ray/protobuf/common.pb.h" +#include "src/ray/protobuf/pubsub.pb.h" + +namespace ray { + +namespace pubsub { + +using SubscribeDoneCallback = std::function; +using SubscriptionItemCallback = std::function; +using SubscriptionFailureCallback = + std::function; + +/// Interface for a subscriber to one or more pubsub channels from a publisher. +class SubscriberInterface { + public: + /// There are two modes of subscriptions. Each channel can only be subscribed in one + /// mode, i.e. + /// - Calling Subscribe() to subscribe to one or more entities in a channel + /// - Calling SubscribeChannel() once to subscribe to all entities in a channel + /// It is an error to call both Subscribe() and SubscribeChannel() on the same channel + /// type. This restriction can be relaxed later, if there is a use case. + + /// Subscribe to entity key_id in channel channel_type. + /// NOTE(sang): All the callbacks could be executed in a different thread from a caller. + /// For example, Subscriber executes callbacks on a passed io_service. + /// + /// \param sub_message The subscription message. + /// \param channel_type The channel to subscribe to. + /// \param publisher_address Address of the publisher to subscribe the object. + /// \param key_id The entity id to subscribe from the publisher. + /// \param subscription_callback A callback that is invoked whenever the given entity + /// information is received by the subscriber. + /// \param subscription_failure_callback A callback that is invoked whenever the + /// connection to publisher is broken (e.g. the publisher fails). + /// \return True if inserted, false if the key already exists and this becomes a no-op. + virtual bool Subscribe(std::unique_ptr sub_message, + rpc::ChannelType channel_type, + const rpc::Address &publisher_address, + const std::string &key_id, + SubscribeDoneCallback subscribe_done_callback, + SubscriptionItemCallback subscription_callback, + SubscriptionFailureCallback subscription_failure_callback) = 0; + + /// Subscribe to all entities in channel channel_type. + /// + /// \param sub_message The subscription message. + /// \param channel_type The channel to subscribe to. + /// \param publisher_address Address of the publisher to subscribe the object. + /// \param subscription_callback A callback that is invoked whenever an entity + /// information is received by the subscriber. + /// \param subscription_failure_callback A callback that is invoked whenever the + /// connection to publisher is broken (e.g. the publisher fails). + /// \return True if inserted, false if the channel is already subscribed and this + /// becomes a no-op. + virtual bool SubscribeChannel( + std::unique_ptr sub_message, + rpc::ChannelType channel_type, + const rpc::Address &publisher_address, + SubscribeDoneCallback subscribe_done_callback, + SubscriptionItemCallback subscription_callback, + SubscriptionFailureCallback subscription_failure_callback) = 0; + + /// Unsubscribe the entity if the entity has been subscribed with Subscribe(). + /// NOTE: Calling this method inside subscription_failure_callback is not allowed. + /// + /// \param channel_type The channel to unsubscribe from. + /// \param publisher_address The publisher address that it will unsubscribe from. + /// \param key_id The entity id to unsubscribe. + /// \return Returns whether the entity key_id has been subscribed before. + virtual bool Unsubscribe(const rpc::ChannelType channel_type, + const rpc::Address &publisher_address, + const std::string &key_id) = 0; + + /// Unsubscribe from the channel_type. Must be paired with SubscribeChannel(). + /// NOTE: Calling this method inside subscription_failure_callback is not allowed. + /// + /// \param channel_type The channel to unsubscribe from. + /// \param publisher_address The publisher address that it will unsubscribe from. + /// \return Returns whether the entity key_id has been subscribed before. + virtual bool UnsubscribeChannel(const rpc::ChannelType channel_type, + const rpc::Address &publisher_address) = 0; + + /// Test only. + /// Checks if the entity key_id is being subscribed to specifically. + /// Does not consider if SubscribeChannel() has been called on the channel. + /// + /// \param publisher_address The publisher address to check. + /// \param key_id The entity id to check. + virtual bool IsSubscribed(const rpc::ChannelType channel_type, + const rpc::Address &publisher_address, + const std::string &key_id) const = 0; + + virtual std::string DebugString() const = 0; + + virtual ~SubscriberInterface() {} +}; + +/// Interface for the client used by a subscriber. +class SubscriberClientInterface { + public: + /// Send a long polling request to a publisher. + virtual void PubsubLongPolling( + const rpc::PubsubLongPollingRequest &request, + const rpc::ClientCallback &callback) = 0; + + /// Send a pubsub command batch to a publisher. + virtual void PubsubCommandBatch( + const rpc::PubsubCommandBatchRequest &request, + const rpc::ClientCallback &callback) = 0; + + virtual std::string DebugString() const = 0; + + virtual ~SubscriberClientInterface() = default; +}; + +} // namespace pubsub + +} // namespace ray diff --git a/src/ray/pubsub/tests/integration_test.cc b/src/ray/pubsub/tests/integration_test.cc index 35a7f338b0b5..5dc62b32cb6b 100644 --- a/src/ray/pubsub/tests/integration_test.cc +++ b/src/ray/pubsub/tests/integration_test.cc @@ -132,6 +132,8 @@ class CallbackSubscriberClient final : public pubsub::SubscriberClientInterface }); } + std::string DebugString() const { return ""; } + private: std::unique_ptr stub_; }; diff --git a/src/ray/pubsub/tests/publisher_test.cc b/src/ray/pubsub/tests/publisher_test.cc index 4d225ba75d28..4ce919eb94e8 100644 --- a/src/ray/pubsub/tests/publisher_test.cc +++ b/src/ray/pubsub/tests/publisher_test.cc @@ -88,8 +88,8 @@ class PublisherTest : public ::testing::Test { return pub_message; } - bool HasSubscriber(const std::vector &subscribers, - const SubscriberID &subscriber) { + bool HasSubscriber(const std::vector &subscribers, + const UniqueID &subscriber) { return std::find(subscribers.begin(), subscribers.end(), subscriber) != subscribers.end(); } @@ -131,7 +131,7 @@ class PublisherTest : public ::testing::Test { absl::flat_hash_map> subscribers_map_; const uint64_t subscriber_timeout_ms_ = 30000; double current_time_; - const SubscriberID subscriber_id_ = SubscriberID::FromRandom(); + const UniqueID subscriber_id_ = UniqueID::FromRandom(); rpc::PubsubLongPollingRequest request_; std::vector> subscribers_; int64_t sequence_id_ = 0; @@ -283,7 +283,7 @@ TEST_F(PublisherTest, TestSubscriptionIndexEraseSubscriber) { SubscriptionIndex subscription_index(rpc::ChannelType::RAY_ERROR_INFO_CHANNEL); auto oid = ObjectID::FromRandom(); auto &subscribers = subscribers_map_[oid]; - std::vector subscriber_ids; + std::vector subscriber_ids; // Add entries. for (int i = 0; i < 6; i++) { diff --git a/src/ray/pubsub/tests/subscriber_test.cc b/src/ray/pubsub/tests/subscriber_test.cc index 6d0b55ae8438..e926cbaf48e7 100644 --- a/src/ray/pubsub/tests/subscriber_test.cc +++ b/src/ray/pubsub/tests/subscriber_test.cc @@ -121,6 +121,8 @@ class MockWorkerClient : public pubsub::SubscriberClientInterface { int GetNumberOfInFlightLongPollingRequests() { return long_polling_callbacks.size(); } + std::string DebugString() const override { return ""; } + ~MockWorkerClient(){}; std::deque> long_polling_callbacks; @@ -128,7 +130,7 @@ class MockWorkerClient : public pubsub::SubscriberClientInterface { std::queue requests_; int64_t sequence_id_ = 0; int64_t max_processed_sequence_id_ = 0; - std::string publisher_id_ = pubsub::PublisherID::FromRandom().Binary(); + std::string publisher_id_ = UniqueID::FromRandom().Binary(); }; namespace pubsub { diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index dba01e0a56da..0dad9cb2d129 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -164,7 +164,7 @@ ray_cc_library( "//src/ray/object_manager:object_directory", "//src/ray/object_manager:object_manager_common", "//src/ray/protobuf:node_manager_cc_proto", - "//src/ray/pubsub:subscriber", + "//src/ray/pubsub:subscriber_interface", "//src/ray/rpc:core_worker_client", ], ) diff --git a/src/ray/raylet/local_object_manager.h b/src/ray/raylet/local_object_manager.h index e9cef263b973..80bf496eda1d 100644 --- a/src/ray/raylet/local_object_manager.h +++ b/src/ray/raylet/local_object_manager.h @@ -26,7 +26,7 @@ #include "ray/gcs/gcs_client/accessor.h" #include "ray/object_manager/common.h" #include "ray/object_manager/object_directory.h" -#include "ray/pubsub/subscriber.h" +#include "ray/pubsub/subscriber_interface.h" #include "ray/raylet/local_object_manager_interface.h" #include "ray/raylet/worker_pool.h" #include "ray/rpc/worker/core_worker_client_pool.h" diff --git a/src/ray/raylet/tests/BUILD.bazel b/src/ray/raylet/tests/BUILD.bazel index 71350f2d30ad..eb558ec5c368 100644 --- a/src/ray/raylet/tests/BUILD.bazel +++ b/src/ray/raylet/tests/BUILD.bazel @@ -179,6 +179,7 @@ ray_cc_test( ":util", "//:ray_fakes", "//:ray_mock", + "//src/mock/ray/pubsub:mock_subscriber", "//src/ray/common:ray_object", "//src/ray/object_manager/plasma:plasma_client", "//src/ray/raylet:local_object_manager_interface", diff --git a/src/ray/rpc/worker/core_worker_client.h b/src/ray/rpc/worker/core_worker_client.h index cab4480b0c98..1ad30728cf62 100644 --- a/src/ray/rpc/worker/core_worker_client.h +++ b/src/ray/rpc/worker/core_worker_client.h @@ -192,6 +192,8 @@ class CoreWorkerClientInterface : public pubsub::SubscriberClientInterface { virtual void FreeActorObject(const FreeActorObjectRequest &request, const ClientCallback &callback) {} + virtual std::string DebugString() const { return ""; } + virtual ~CoreWorkerClientInterface() = default; }; @@ -369,6 +371,8 @@ class CoreWorkerClient : public std::enable_shared_from_this, CoreWorkerService, NumPendingTasks, *request, callback, grpc_client_, timeout_ms); } + std::string DebugString() const override { return ""; } + /// Send as many pending tasks as possible. This method is thread-safe. /// /// The client will guarantee no more than kMaxBytesInFlight bytes of RPCs are being From fd3f23593de38fec41c8321da7c169b08eb768cc Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 18 Aug 2025 17:32:25 -0500 Subject: [PATCH 167/634] [core] Remove unnecessary dependency of raylet->gcs (#55710) The raylet binary was depending on all of the `gcs/` directory for absolutely no reason :( --------- Signed-off-by: Edward Oakes --- src/ray/raylet/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index 0dad9cb2d129..6c3669449f46 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -112,6 +112,7 @@ ray_cc_library( "//src/ray/common:runtime_env", "//src/ray/common:status", "//src/ray/common:task_common", + "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/ipc:client_connection", "//src/ray/util:network_util", @@ -225,7 +226,6 @@ ray_cc_library( "//src/ray/common:memory_monitor", "//src/ray/core_worker:experimental_mutable_object_provider", "//src/ray/flatbuffers:node_manager_generated", - "//src/ray/gcs", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/object_manager", "//src/ray/object_manager:ownership_object_directory", From e9160b72338c4d682af2eb0249f442bd1ff4992d Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Mon, 18 Aug 2025 15:39:46 -0700 Subject: [PATCH 168/634] [core] Not overriding accelerator id env vars when num_accelerators is 0 or not set (#54928) --- python/ray/_private/accelerators/__init__.py | 6 ++- .../ray/_private/accelerators/accelerator.py | 13 +++++ python/ray/_private/utils.py | 39 ++++++++++++-- python/ray/_private/worker.py | 14 +++++ python/ray/_raylet.pyx | 24 +++++---- python/ray/tests/test_basic.py | 53 +++++++++++++++++++ 6 files changed, 133 insertions(+), 16 deletions(-) diff --git a/python/ray/_private/accelerators/__init__.py b/python/ray/_private/accelerators/__init__.py index 003074ad71fb..4cb14fef7956 100644 --- a/python/ray/_private/accelerators/__init__.py +++ b/python/ray/_private/accelerators/__init__.py @@ -1,6 +1,9 @@ from typing import Optional, Set -from ray._private.accelerators.accelerator import AcceleratorManager +from ray._private.accelerators.accelerator import ( + RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO_ENV_VAR, + AcceleratorManager, +) from ray._private.accelerators.amd_gpu import AMDGPUAcceleratorManager from ray._private.accelerators.hpu import HPUAcceleratorManager from ray._private.accelerators.intel_gpu import IntelGPUAcceleratorManager @@ -77,4 +80,5 @@ def get_accelerator_manager_for_resource( "get_all_accelerator_managers", "get_all_accelerator_resource_names", "get_accelerator_manager_for_resource", + "RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO_ENV_VAR", ] diff --git a/python/ray/_private/accelerators/accelerator.py b/python/ray/_private/accelerators/accelerator.py index a2cd98565fd2..4b5332cb8a07 100644 --- a/python/ray/_private/accelerators/accelerator.py +++ b/python/ray/_private/accelerators/accelerator.py @@ -1,6 +1,19 @@ from abc import ABC, abstractmethod from typing import Dict, List, Optional, Tuple +# https://github.com/ray-project/ray/issues/54868 +# In the future, ray will avoid overriding the accelerator ids environment variables +# when the number of accelerators is zero. +# For example, when this environment variable is set, if a user sets `num_gpus=0` +# in the `ray.init()` call, the environment variable `CUDA_VISIBLE_DEVICES` will +# not be set to an empty string. +# +# This environment variable is used to disable this behavior temporarily. +# And to avoid breaking changes, this environment variable is set to True by default +# to follow the previous behavior. +# +RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO_ENV_VAR = "RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO" + class AcceleratorManager(ABC): """This class contains all the functions needed for supporting diff --git a/python/ray/_private/utils.py b/python/ray/_private/utils.py index 84ab89094544..9204f46b6463 100644 --- a/python/ray/_private/utils.py +++ b/python/ray/_private/utils.py @@ -266,18 +266,46 @@ def set_omp_num_threads_if_unset() -> bool: return True -def set_visible_accelerator_ids() -> None: +def set_visible_accelerator_ids() -> Mapping[str, Optional[str]]: """Set (CUDA_VISIBLE_DEVICES, ONEAPI_DEVICE_SELECTOR, HIP_VISIBLE_DEVICES, NEURON_RT_VISIBLE_CORES, TPU_VISIBLE_CHIPS , HABANA_VISIBLE_MODULES ,...) - environment variables based on the accelerator runtime. + environment variables based on the accelerator runtime. Return the original + environment variables. """ + from ray._private.ray_constants import env_bool + + original_visible_accelerator_env_vars = {} + override_on_zero = env_bool( + ray._private.accelerators.RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO_ENV_VAR, + True, + ) for resource_name, accelerator_ids in ( ray.get_runtime_context().get_accelerator_ids().items() ): + # If no accelerator ids are set, skip overriding the environment variable. + if not override_on_zero and len(accelerator_ids) == 0: + continue + env_var = ray._private.accelerators.get_accelerator_manager_for_resource( + resource_name + ).get_visible_accelerator_ids_env_var() + original_visible_accelerator_env_vars[env_var] = os.environ.get(env_var, None) ray._private.accelerators.get_accelerator_manager_for_resource( resource_name ).set_current_process_visible_accelerator_ids(accelerator_ids) + return original_visible_accelerator_env_vars + + +def reset_visible_accelerator_env_vars( + original_visible_accelerator_env_vars: Mapping[str, Optional[str]] +) -> None: + """Reset the visible accelerator env vars to the original values.""" + for env_var, env_value in original_visible_accelerator_env_vars.items(): + if env_value is None: + os.environ.pop(env_var, None) + else: + os.environ[env_var] = env_value + class Unbuffered(object): """There's no "built-in" solution to programatically disabling buffering of @@ -342,9 +370,10 @@ def _get_docker_cpus( # See: https://bugs.openjdk.java.net/browse/JDK-8146115 if os.path.exists(cpu_quota_file_name) and os.path.exists(cpu_period_file_name): try: - with open(cpu_quota_file_name, "r") as quota_file, open( - cpu_period_file_name, "r" - ) as period_file: + with ( + open(cpu_quota_file_name, "r") as quota_file, + open(cpu_period_file_name, "r") as period_file, + ): cpu_quota = float(quota_file.read()) / float(period_file.read()) except Exception: logger.exception("Unexpected error calculating docker cpu quota.") diff --git a/python/ray/_private/worker.py b/python/ray/_private/worker.py index d81789c60f89..05e8b44f9220 100644 --- a/python/ray/_private/worker.py +++ b/python/ray/_private/worker.py @@ -1979,6 +1979,20 @@ def sigterm_handler(signum, frame): for hook in _post_init_hooks: hook() + # Check and show accelerator override warning during driver initialization + from ray._private.ray_constants import env_bool + + override_on_zero = env_bool( + ray._private.accelerators.RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO_ENV_VAR, + True, + ) + if override_on_zero and log_once("ray_accel_env_var_override_on_zero"): + logger.warning( + "Tip: In future versions of Ray, Ray will no longer override accelerator " + "visible devices env var if num_gpus=0 or num_gpus=None (default). To enable " + "this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0" + ) + node_id = global_worker.core_worker.get_current_node_id() global_node_address_info = _global_node.address_info.copy() global_node_address_info["webui_url"] = _remove_protocol_from_url(dashboard_url) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index fa06590fd21e..821b994cfdb7 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -2179,16 +2179,15 @@ cdef execute_task_with_cancellation_handler( title = f"ray::{task_name}" # Automatically restrict the GPUs (CUDA), neuron_core, TPU accelerator - # runtime_ids to restrict availability to this task. + # runtime_ids, OMP_NUM_THREADS to restrict availability to this task. # Once actor is created, users can change the visible accelerator ids within # an actor task and we don't want to reset it. if (task_type != TASK_TYPE_ACTOR_TASK): - ray._private.utils.set_visible_accelerator_ids() - - # Automatically configure OMP_NUM_THREADS to the assigned CPU number. - # It will be unset after the task execution if it was overwridden here. - # No-op if already set. - omp_num_threads_overriden = ray._private.utils.set_omp_num_threads_if_unset() + original_visible_accelerator_env_vars = ray._private.utils.set_visible_accelerator_ids() + omp_num_threads_overriden = ray._private.utils.set_omp_num_threads_if_unset() + else: + original_visible_accelerator_env_vars = None + omp_num_threads_overriden = False # Initialize the actor if this is an actor creation task. We do this here # before setting the current task ID so that we can get the execution info, @@ -2289,9 +2288,14 @@ cdef execute_task_with_cancellation_handler( with current_task_id_lock: current_task_id = None - if omp_num_threads_overriden: - # Reset the OMP_NUM_THREADS environ if it was set. - os.environ.pop("OMP_NUM_THREADS", None) + if (task_type == TASK_TYPE_NORMAL_TASK): + if original_visible_accelerator_env_vars: + # Reset the visible accelerator env vars for normal tasks, since they may be reused. + ray._private.utils.reset_visible_accelerator_env_vars(original_visible_accelerator_env_vars) + if omp_num_threads_overriden: + # Reset the OMP_NUM_THREADS environ if it was set. + os.environ.pop("OMP_NUM_THREADS", None) + if execution_info.max_calls != 0: # Reset the state of the worker for the next task to execute. diff --git a/python/ray/tests/test_basic.py b/python/ray/tests/test_basic.py index a43312247687..6300b1e7aa1a 100644 --- a/python/ray/tests/test_basic.py +++ b/python/ray/tests/test_basic.py @@ -657,6 +657,59 @@ def check(): ) +# https://github.com/ray-project/ray/issues/54868 +def test_not_override_accelerator_ids_when_num_accelerators_is_zero(): + not_override_check_script = """ +import ray +ray.init() + + +@ray.remote(num_gpus=0) +def check(): + import os + assert "CUDA_VISIBLE_DEVICES" not in os.environ + +@ray.remote(num_gpus=0) +class Actor: + def check(self): + import os + assert "CUDA_VISIBLE_DEVICES" not in os.environ + +print("task check", ray.get(check.remote())) +print("actor check", ray.get(Actor.options(num_gpus=0).remote().check.remote())) +""" + + run_string_as_driver( + not_override_check_script, + dict( + os.environ, + **{"RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO": "0"}, + ), + ) + + override_check_script = """ +import ray +ray.init() + + +@ray.remote(num_gpus=0) +def check(): + import os + assert os.environ.get("CUDA_VISIBLE_DEVICES") == "" + +@ray.remote(num_gpus=0) +class Actor: + def check(self): + import os + assert os.environ.get("CUDA_VISIBLE_DEVICES") == "" + +print("task check", ray.get(check.remote())) +print("actor check", ray.get(Actor.options(num_gpus=0).remote().check.remote())) +""" + + run_string_as_driver(override_check_script) + + def test_put_get(shutdown_only): ray.init(num_cpus=0) From 7424ffbdc7b5df15141c66c23c1adafa36cd431b Mon Sep 17 00:00:00 2001 From: vincenthhan <46981434+BestVIncent@users.noreply.github.com> Date: Tue, 19 Aug 2025 07:18:28 +0800 Subject: [PATCH 169/634] [llm] support custom s3 endpoint when downloading models from remote (#55458) Signed-off-by: vincenthhan Co-authored-by: vincenthhan --- python/ray/llm/_internal/common/utils/cloud_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/ray/llm/_internal/common/utils/cloud_utils.py b/python/ray/llm/_internal/common/utils/cloud_utils.py index f654e044cd6e..209b3a30ec15 100644 --- a/python/ray/llm/_internal/common/utils/cloud_utils.py +++ b/python/ray/llm/_internal/common/utils/cloud_utils.py @@ -148,7 +148,13 @@ def get_fs_and_path(object_uri: str) -> Tuple[pa_fs.FileSystem, str]: object_uri = f"{scheme}://{parts[1]}" if object_uri.startswith("s3://"): - fs = pa_fs.S3FileSystem(anonymous=anonymous) + endpoint = os.getenv("AWS_ENDPOINT_URL_S3", None) + virtual_hosted_style = os.getenv("AWS_S3_ADDRESSING_STYLE", None) + fs = pa_fs.S3FileSystem( + anonymous=anonymous, + endpoint_override=endpoint, + force_virtual_addressing=(virtual_hosted_style == "virtual"), + ) path = object_uri[5:] # Remove "s3://" elif object_uri.startswith("gs://"): fs = pa_fs.GcsFileSystem(anonymous=anonymous) From 6993ba79da529a44fb23b1717acac3d83aa5dcef Mon Sep 17 00:00:00 2001 From: Jeffrey Wang Date: Mon, 18 Aug 2025 16:19:27 -0700 Subject: [PATCH 170/634] [data.llm] Adjust LLM engine timing logic (#55595) Signed-off-by: jeffreyjeffreywang Co-authored-by: jeffreyjeffreywang --- .../batch/stages/sglang_engine_stage.py | 20 ++++++++++--------- .../batch/stages/vllm_engine_stage.py | 20 ++++++++++--------- .../gpu/stages/test_sglang_engine_stage.py | 4 +++- .../gpu/stages/test_vllm_engine_stage.py | 13 ++++++++---- 4 files changed, 34 insertions(+), 23 deletions(-) diff --git a/python/ray/llm/_internal/batch/stages/sglang_engine_stage.py b/python/ray/llm/_internal/batch/stages/sglang_engine_stage.py index a49ef7c18bce..f140b666b15d 100644 --- a/python/ray/llm/_internal/batch/stages/sglang_engine_stage.py +++ b/python/ray/llm/_internal/batch/stages/sglang_engine_stage.py @@ -177,22 +177,25 @@ async def _prepare_llm_request(self, row: Dict[str, Any]) -> SGLangEngineRequest async def generate_async( self, row: Dict[str, Any] - ) -> Tuple[SGLangEngineRequest, Dict[str, Any]]: + ) -> Tuple[SGLangEngineRequest, Dict[str, Any], float]: """Process a single request. Args: request: The request. Returns: - A tuple of index in batch, request output and bypassed custom fields. + A tuple of index in batch, request output and bypassed custom fields, and time taken. """ request = await self._prepare_llm_request(row) + t = time.perf_counter() async with self.semaphore: output = await self._generate_async(request) + time_taken = time.perf_counter() - t + output_data = SGLangOutputData.from_sglang_engine_output(output) - return request, output_data.model_dump() + return request, output_data.model_dump(), time_taken async def _generate_async(self, request: SGLangEngineRequest) -> Any: """Process a single request. @@ -321,29 +324,28 @@ async def udf(self, batch: List[Dict[str, Any]]) -> AsyncIterator[Dict[str, Any] The response of the SGLang engine. """ batch_uuid = uuid.uuid4() - t = time.perf_counter() + batch_start_time = time.perf_counter() tasks = [asyncio.create_task(self.llm.generate_async(row)) for row in batch] - time_taken = -1.0 for resp in asyncio.as_completed(tasks): - request, output = await resp - time_taken = time.perf_counter() - t + request, output, time_taken_llm = await resp yield { **output, "request_id": request.request_id, self.IDX_IN_BATCH_COLUMN: request.idx_in_batch, "batch_uuid": batch_uuid.hex, - "time_taken_llm": time_taken, + "time_taken_llm": time_taken_llm, "params": str(request.params), } + batch_time_taken = time.perf_counter() - batch_start_time logger.info( "[SGLang] Elapsed time for batch %s with size %d: %s", batch_uuid.hex, len(batch), - time_taken, + batch_time_taken, ) def __del__(self): diff --git a/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py b/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py index 481da600da7b..5395e448621e 100644 --- a/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py +++ b/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py @@ -302,22 +302,25 @@ async def _prepare_llm_request(self, row: Dict[str, Any]) -> vLLMEngineRequest: async def generate_async( self, row: Dict[str, Any] - ) -> Tuple[vLLMEngineRequest, Dict[str, Any]]: + ) -> Tuple[vLLMEngineRequest, Dict[str, Any], float]: """Process a single request. Args: request: The request. Returns: - A tuple of index in batch, request output and bypassed custom fields. + A tuple of index in batch, request output and bypassed custom fields, and time taken. """ request = await self._prepare_llm_request(row) + t = time.perf_counter() async with self.semaphore: output = await self._generate_async(request) + time_taken = time.perf_counter() - t + output_data = vLLMOutputData.from_vllm_engine_output(output) - return request, output_data.model_dump() + return request, output_data.model_dump(), time_taken async def generate_async_v0(self, request: vLLMEngineRequest) -> Any: """Process a single request. @@ -539,31 +542,30 @@ async def udf(self, batch: List[Dict[str, Any]]) -> AsyncIterator[Dict[str, Any] The response of the vLLM engine. """ batch_uuid = uuid.uuid4() - t = time.perf_counter() + batch_start_time = time.perf_counter() tasks = [asyncio.create_task(self.llm.generate_async(row)) for row in batch] - time_taken = -1.0 for resp in asyncio.as_completed(tasks): - request, output = await resp - time_taken = time.perf_counter() - t + request, output, time_taken_llm = await resp yield { **output, "request_id": request.request_id, self.IDX_IN_BATCH_COLUMN: request.idx_in_batch, "batch_uuid": batch_uuid.hex, - "time_taken_llm": time_taken, + "time_taken_llm": time_taken_llm, "params": str(request.params), } + batch_time_taken = time.perf_counter() - batch_start_time # TODO: Add metrics to the UDf wrapper so that we don't need # timer in UDFs anymore. logger.info( "[vLLM] Elapsed time for batch %s with size %d: %s", batch_uuid.hex, len(batch), - time_taken, + batch_time_taken, ) # Log engine stats after each batch is done conditioned on the flag diff --git a/python/ray/llm/tests/batch/gpu/stages/test_sglang_engine_stage.py b/python/ray/llm/tests/batch/gpu/stages/test_sglang_engine_stage.py index 9f0d3a453f0f..f23dcd98c40c 100644 --- a/python/ray/llm/tests/batch/gpu/stages/test_sglang_engine_stage.py +++ b/python/ray/llm/tests/batch/gpu/stages/test_sglang_engine_stage.py @@ -42,6 +42,7 @@ async def mock_generate(row): "generated_text": f"Response to: {row['prompt']}", "num_generated_tokens": 3, }, + 0.1, # time_taken_llm ) mock_instance.generate_async.side_effect = mock_generate @@ -226,9 +227,10 @@ async def test_sglang_wrapper( assert mock_generate_async.call_count == batch_size # Verify the outputs match expected values - for i, (request, output) in enumerate(results): + for i, (request, output, time_taken_llm) in enumerate(results): assert output["prompt"] == f"Test {i}" assert output["num_generated_tokens"] == i + 5 # max_new_tokens we set + assert time_taken_llm > 0 @pytest.mark.asyncio diff --git a/python/ray/llm/tests/batch/gpu/stages/test_vllm_engine_stage.py b/python/ray/llm/tests/batch/gpu/stages/test_vllm_engine_stage.py index 8e165fadbb7e..d05805f13eea 100644 --- a/python/ray/llm/tests/batch/gpu/stages/test_vllm_engine_stage.py +++ b/python/ray/llm/tests/batch/gpu/stages/test_vllm_engine_stage.py @@ -46,6 +46,7 @@ async def mock_generate(row): "num_generated_tokens": 3, "time_per_token": 0.1, }, + 0.1, # time_taken_llm ) mock_instance.generate_async.side_effect = mock_generate @@ -298,10 +299,11 @@ async def test_vllm_wrapper_generate(model_llama_3_2_216M): tasks = [asyncio.create_task(wrapper.generate_async(row)) for row in batch] for resp in asyncio.as_completed(tasks): - request, output = await resp + request, output, time_taken_llm = await resp params = request.params max_tokens = params.max_tokens assert max_tokens == output["num_generated_tokens"] + assert time_taken_llm > 0 # Clean up GPU memory wrapper.shutdown() @@ -332,8 +334,9 @@ async def test_vllm_wrapper_embed(model_opt_125m): tasks = [asyncio.create_task(wrapper.generate_async(row)) for row in batch] for resp in asyncio.as_completed(tasks): - _, output = await resp + _, output, time_taken_llm = await resp assert output["embeddings"].shape == (768,) + assert time_taken_llm > 0 # Clean up GPU memory wrapper.shutdown() @@ -380,10 +383,11 @@ async def test_vllm_wrapper_lora(model_llama_3_2_216M, model_llama_3_2_216M_lora tasks = [asyncio.create_task(wrapper.generate_async(row)) for row in batch] for resp in asyncio.as_completed(tasks): - request, output = await resp + request, output, time_taken_llm = await resp params = request.params max_tokens = params.max_tokens assert max_tokens == output["num_generated_tokens"] + assert time_taken_llm > 0 # Clean up GPU memory wrapper.shutdown() @@ -430,12 +434,13 @@ class AnswerModel(BaseModel): tasks = [asyncio.create_task(wrapper.generate_async(row)) for row in batch] for resp in asyncio.as_completed(tasks): - _, output = await resp + _, output, time_taken_llm = await resp json_obj = json.loads(output["generated_text"]) assert "answer" in json_obj assert isinstance(json_obj["answer"], int) assert "explain" in json_obj assert isinstance(json_obj["explain"], str) + assert time_taken_llm > 0 # Clean up GPU memory wrapper.shutdown() From 30c8122962dcb1285fd4324313770a53693ce863 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 18 Aug 2025 16:46:06 -0700 Subject: [PATCH 171/634] [image] refactor apt package installation (#55701) avoid reinstalling packages that are already installed in the base image also rename the saved requirements file to `extra-test-requirements.txt` Signed-off-by: Lonnie Liu --- docker/base-extra/Dockerfile | 17 ++++++++++++++++- release/ray_release/byod/byod.Dockerfile | 21 +++++++++------------ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/docker/base-extra/Dockerfile b/docker/base-extra/Dockerfile index cf286db30a69..02b851b2f594 100644 --- a/docker/base-extra/Dockerfile +++ b/docker/base-extra/Dockerfile @@ -41,8 +41,23 @@ wget -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg \ | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - # Add gdb since ray dashboard uses `memray attach`, which requires gdb. + +APT_PKGS=( + google-cloud-sdk + supervisor + vim + zsh + nfs-common + zip + unzip + build-essential + ssh + curl + gdb +) + sudo apt-get update -y -sudo apt-get install -y google-cloud-sdk supervisor vim zsh nfs-common zip unzip build-essential ssh curl gdb +sudo apt-get install -y "${APT_PKGS[@]}" sudo apt-get autoclean # Install azcopy diff --git a/release/ray_release/byod/byod.Dockerfile b/release/ray_release/byod/byod.Dockerfile index 91a5117575ed..ce8c2a8080ca 100644 --- a/release/ray_release/byod/byod.Dockerfile +++ b/release/ray_release/byod/byod.Dockerfile @@ -6,38 +6,35 @@ FROM "$BASE_IMAGE" ARG PIP_REQUIREMENTS +COPY "$PIP_REQUIREMENTS" extra-test-requirements.txt + RUN < Date: Mon, 18 Aug 2025 21:36:12 -0700 Subject: [PATCH 172/634] [Serve.llm] Support colocating local DP ranks in DPRankAssigner (#55720) Signed-off-by: Rui Qiao Signed-off-by: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> --- .../data_parallel/dp_rank_assigner.py | 105 ++++++++++++++++-- .../deployments/data_parallel/dp_server.py | 15 ++- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py index f48496796e4d..ac65ba259216 100644 --- a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py +++ b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_rank_assigner.py @@ -1,7 +1,11 @@ import asyncio +import logging +from typing import Dict, List, Optional from ray import serve +logger = logging.getLogger(__name__) + @serve.deployment(num_replicas=1) class DPRankAssigner: @@ -12,15 +16,70 @@ class DPRankAssigner: deployment. """ - def __init__(self, dp_size: int): - self.dp_size = dp_size - self.lock = asyncio.Lock() - self.next_rank = 0 - self.dp_address = None - self.dp_rpc_port = None - self.master_info_event = asyncio.Event() + def __init__(self, dp_size: int, dp_size_per_node: Optional[int] = None): + self.dp_size: int = dp_size + self.dp_size_per_node: Optional[int] = dp_size_per_node + self.lock: asyncio.Lock = asyncio.Lock() + self.dp_address: Optional[str] = None + self.dp_rpc_port: Optional[int] = None + self.master_info_event: asyncio.Event = asyncio.Event() + + # Fields for _register_random_placement(): + # Next rank to assign + self.next_rank: Optional[int] = None + + # Fields for _register_node_pack_placement(): + # Number of nodes to assign to + self.num_nodes: Optional[int] = None + # Map from node id to available ranks + self.node_to_avail_ranks: Dict[str, List[int]] = {} + + if dp_size_per_node is None: + self.next_rank = 0 + logger.info( + f"Using random placement rank assigner for DP size {self.dp_size}" + ) + else: + if self.dp_size_per_node <= 0: + raise ValueError( + f"dp_size_per_node {self.dp_size_per_node} must be greater than 0" + ) + if self.dp_size % self.dp_size_per_node != 0: + raise ValueError( + f"dp_size {self.dp_size} must be divisible by dp_size_per_node {self.dp_size_per_node}" + ) + self.num_nodes = self.dp_size // self.dp_size_per_node + logger.info( + f"Using node pack placement rank assigner for DP size {self.dp_size}" + f"with dp_size_per_node {self.dp_size_per_node}" + ) + + async def register( + self, replica_ctx: "serve.context.ReplicaContext", node_id: Optional[str] = None + ): + """ + Register a replica and assign a rank to it. - async def register(self, replica_ctx: "serve.context.ReplicaContext"): + Args: + replica_ctx: The replica context. + node_id: The node id of the replica. + + Returns: + The rank of the replica. + """ + if self.dp_size_per_node is None: + return await self._register_random_placement() + else: + if node_id is None: + raise ValueError("node_id is required for node pack placement") + return await self._register_node_pack_placement(node_id) + + async def _register_random_placement(self): + """ + Assign a rank based on random placement. + + The ranks are assigned in a random order, regardless of its node id. + """ async with self.lock: if self.next_rank >= self.dp_size: raise ValueError( @@ -32,6 +91,36 @@ async def register(self, replica_ctx: "serve.context.ReplicaContext"): self.next_rank += 1 return rank + async def _register_node_pack_placement(self, node_id: str): + """ + Assign a rank based on node pack placement. + + This should be used for DeepEP which assumes that the ranks ranging from + [dp_rank_per_node * node_rank, dp_rank_per_node * (node_rank + 1) - 1] are + assigned to the same node. + + For example, if dp_size_per_node is 8, and there are 16 ranks in total, then + the ranks [0, 7] should be assigned to one node, and ranks [8, 15] should be + assigned to another node. + """ + async with self.lock: + if not self.node_to_avail_ranks: + self.node_to_avail_ranks[node_id] = list( + range(1, self.dp_size_per_node) + ) + return 0 + elif node_id not in self.node_to_avail_ranks: + node_rank = len(self.node_to_avail_ranks) + assert node_rank < self.num_nodes + rank = node_rank * self.dp_size_per_node + self.node_to_avail_ranks[node_id] = list( + range(rank + 1, rank + self.dp_size_per_node) + ) + return rank + else: + rank = self.node_to_avail_ranks[node_id].pop(0) + return rank + async def set_dp_master_info(self, dp_address: str, dp_rpc_port: int): self.dp_address = dp_address self.dp_rpc_port = dp_rpc_port diff --git a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py index 7fb1cc2f49a1..8e2bc445f3fd 100644 --- a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py +++ b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py @@ -9,6 +9,7 @@ DPRankAssigner, ) from ray.llm._internal.serve.deployments.llm.llm_server import LLMServer +from ray.runtime_context import get_runtime_context from ray.serve.deployment import Application from ray.serve.handle import DeploymentHandle @@ -29,7 +30,9 @@ async def __init__(self, llm_config: LLMConfig, dp_rank_assigner: DeploymentHand self.dp_rank_assigner = dp_rank_assigner replica_ctx = serve.get_replica_context() - self.dp_rank = await self.dp_rank_assigner.register.remote(replica_ctx) + node_id = get_runtime_context().get_node_id() + self.dp_rank = await self.dp_rank_assigner.register.remote(replica_ctx, node_id) + logger.info(f"DP rank {self.dp_rank} registered with rank assigner") if self.dp_rank == 0: @@ -81,8 +84,16 @@ def build_dp_deployment( "data_parallel_size should be greater than 1 for DP deployment." ) + # TODO(rui): figure out a better way to pass in dp_size_per_node. + # NOTE: we cannot use engine_kwargs.data_parallel_size_local to specify + # the number of ranks per node because that has special semantics in vLLM. + dp_size_per_node = llm_config.experimental_configs.get("dp_size_per_node", None) + deployment_options = llm_config.get_serve_options(name_prefix=name_prefix) - dp_rank_assigner = DPRankAssigner.bind(dp_size=dp_size) + dp_rank_assigner = DPRankAssigner.bind( + dp_size=dp_size, dp_size_per_node=dp_size_per_node + ) + return DPServer.as_deployment(deployment_options).bind( llm_config=llm_config, dp_rank_assigner=dp_rank_assigner ) From 69bc6c1e8394ef5846bf3d3d36a7fd384441c5a1 Mon Sep 17 00:00:00 2001 From: Ibrahim Rabbani Date: Mon, 18 Aug 2025 21:38:58 -0700 Subject: [PATCH 173/634] [core] ray.put returns an ObjectRef without an owner_address. (#55636) Signed-off-by: irabbani Signed-off-by: Ibrahim Rabbani Co-authored-by: Edward Oakes --- python/ray/_private/worker.py | 15 ++++++--------- python/ray/_raylet.pyx | 25 ++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/python/ray/_private/worker.py b/python/ray/_private/worker.py index 05e8b44f9220..8e31acf2fc78 100644 --- a/python/ray/_private/worker.py +++ b/python/ray/_private/worker.py @@ -852,15 +852,12 @@ def put_object( # reference will be created. If another reference is created and # removed before this one, it will corrupt the state in the # reference counter. - return ray.ObjectRef( - self.core_worker.put_serialized_object_and_increment_local_ref( - serialized_value, - pin_object=pin_object, - owner_address=owner_address, - _is_experimental_channel=_is_experimental_channel, - ), - # The initial local reference is already acquired internally. - skip_adding_local_ref=True, + return self.core_worker.put_object( + serialized_value, + pin_object=pin_object, + owner_address=owner_address, + inline_small_object=True, + _is_experimental_channel=_is_experimental_channel, ) def raise_errors(self, serialized_objects, object_refs): diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index 821b994cfdb7..ad030028441d 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -1002,7 +1002,6 @@ cdef prepare_args_internal( ))) incremented_put_arg_ids.push_back(put_id) - cdef raise_if_dependency_failed(arg): """This method is used to improve the readability of backtrace. @@ -3429,6 +3428,30 @@ cdef class CoreWorker: CCoreWorkerProcess.GetCoreWorker() .ExperimentalRegisterMutableObjectReader(c_object_id)) + def put_object( + self, + serialized_object, + *, + c_bool pin_object, + owner_address, + c_bool inline_small_object, + c_bool _is_experimental_channel, + ): + """Create an object reference with the current worker as the owner. + """ + created_object = self.put_serialized_object_and_increment_local_ref( + serialized_object, pin_object, owner_address, inline_small_object, _is_experimental_channel) + if owner_address is None: + owner_address = CCoreWorkerProcess.GetCoreWorker().GetRpcAddress().SerializeAsString() + + # skip_adding_local_ref is True because it's already added through the call to + # put_serialized_object_and_increment_local_ref. + return ObjectRef( + created_object, + owner_address, + skip_adding_local_ref=True + ) + def put_serialized_object_and_increment_local_ref( self, serialized_object, From be33b6fb411b21d2bb2cadfc8755a66c195d2272 Mon Sep 17 00:00:00 2001 From: avigyabb <98926738+avigyabb@users.noreply.github.com> Date: Mon, 18 Aug 2025 21:41:43 -0700 Subject: [PATCH 174/634] [Core] Bind runtime env agent and dashboard agent http server to specified ip instead of 0.0.0.0 (#55431) Signed-off-by: avigyabb Signed-off-by: avibasnet31 Co-authored-by: avibasnet31 Co-authored-by: Jiajun Yao --- cpp/test_submit_cpp_job.py | 4 +--- python/ray/_private/runtime_env/agent/main.py | 5 +---- python/ray/dashboard/http_server_agent.py | 2 +- .../modules/job/tests/test_job_agent.py | 20 +++++++++---------- .../modules/reporter/tests/test_healthz.py | 4 ++-- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/cpp/test_submit_cpp_job.py b/cpp/test_submit_cpp_job.py index 5ab6e753bf8f..695079a50c6a 100644 --- a/cpp/test_submit_cpp_job.py +++ b/cpp/test_submit_cpp_job.py @@ -21,9 +21,7 @@ def headers(): @pytest.fixture(scope="module") def job_sdk_client(headers): - with _ray_start( - include_dashboard=True, num_cpus=1, _node_ip_address="0.0.0.0" - ) as ctx: + with _ray_start(include_dashboard=True, num_cpus=1) as ctx: address = ctx.address_info["webui_url"] assert wait_until_server_available(address) yield JobSubmissionClient(format_web_url(address), headers=headers) diff --git a/python/ray/_private/runtime_env/agent/main.py b/python/ray/_private/runtime_env/agent/main.py index f9beaa6167c9..e65de4d63bd4 100644 --- a/python/ray/_private/runtime_env/agent/main.py +++ b/python/ray/_private/runtime_env/agent/main.py @@ -218,13 +218,10 @@ def parent_dead_callback(msg): check_raylet_task = create_check_raylet_task( args.log_dir, gcs_client, parent_dead_callback, loop ) - runtime_env_agent_ip = ( - "127.0.0.1" if args.node_ip_address == "127.0.0.1" else "0.0.0.0" - ) try: web.run_app( app, - host=runtime_env_agent_ip, + host=args.node_ip_address, port=args.runtime_env_agent_port, loop=loop, ) diff --git a/python/ray/dashboard/http_server_agent.py b/python/ray/dashboard/http_server_agent.py index 846df9c565b9..b71ba6c5a38b 100644 --- a/python/ray/dashboard/http_server_agent.py +++ b/python/ray/dashboard/http_server_agent.py @@ -44,7 +44,7 @@ async def _start_site_with_retry( try: site = aiohttp.web.TCPSite( self.runner, - "127.0.0.1" if self.ip == "127.0.0.1" else "0.0.0.0", + self.ip, self.listen_port, ) await site.start() diff --git a/python/ray/dashboard/modules/job/tests/test_job_agent.py b/python/ray/dashboard/modules/job/tests/test_job_agent.py index 7922c1cdff97..73229cc261a1 100644 --- a/python/ray/dashboard/modules/job/tests/test_job_agent.py +++ b/python/ray/dashboard/modules/job/tests/test_job_agent.py @@ -25,7 +25,7 @@ run_string_as_driver_nonblocking, wait_until_server_available, ) -from ray._common.network_utils import parse_address, build_address +from ray._common.network_utils import build_address from ray.dashboard.modules.job.common import ( JOB_ACTOR_NAME_TEMPLATE, SUPERVISOR_ACTOR_RAY_NAMESPACE, @@ -77,8 +77,8 @@ def __init__(self, *args, **kwargs): @pytest_asyncio.fixture async def job_sdk_client(make_sure_dashboard_http_port_unused): with _ray_start(include_dashboard=True, num_cpus=1) as ctx: - ip, _ = parse_address(ctx.address_info["webui_url"]) - agent_address = build_address(ip, DEFAULT_DASHBOARD_AGENT_LISTEN_PORT) + node_ip = ctx.address_info["node_ip_address"] + agent_address = build_address(node_ip, DEFAULT_DASHBOARD_AGENT_LISTEN_PORT) assert wait_until_server_available(agent_address) head_address = ctx.address_info["webui_url"] assert wait_until_server_available(head_address) @@ -469,8 +469,8 @@ async def test_job_log_in_multiple_node( dashboard_agent_listen_port=DEFAULT_DASHBOARD_AGENT_LISTEN_PORT + 2 ) - ip, _ = parse_address(cluster.webui_url) - agent_address = build_address(ip, DEFAULT_DASHBOARD_AGENT_LISTEN_PORT) + node_ip = cluster.head_node.node_ip_address + agent_address = build_address(node_ip, DEFAULT_DASHBOARD_AGENT_LISTEN_PORT) assert wait_until_server_available(agent_address) client = JobAgentSubmissionClient(format_web_url(agent_address)) @@ -595,18 +595,18 @@ async def test_non_default_dashboard_agent_http_port(tmp_path): """ import subprocess - cmd = ( - "ray start --head " f"--dashboard-agent-listen-port {get_current_unused_port()}" - ) + dashboard_agent_port = get_current_unused_port() + cmd = "ray start --head " f"--dashboard-agent-listen-port {dashboard_agent_port}" subprocess.check_output(cmd, shell=True) try: # We will need to wait for the ray to be started in the subprocess. address_info = ray.init("auto", ignore_reinit_error=True).address_info - ip, _ = parse_address(address_info["webui_url"]) + node_ip = address_info["node_ip_address"] + dashboard_agent_listen_port = address_info["dashboard_agent_listen_port"] - agent_address = build_address(ip, dashboard_agent_listen_port) + agent_address = build_address(node_ip, dashboard_agent_listen_port) print("agent address = ", agent_address) agent_client = JobAgentSubmissionClient(format_web_url(agent_address)) diff --git a/python/ray/dashboard/modules/reporter/tests/test_healthz.py b/python/ray/dashboard/modules/reporter/tests/test_healthz.py index 9087a5581674..fd613e0d59e1 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_healthz.py +++ b/python/ray/dashboard/modules/reporter/tests/test_healthz.py @@ -25,7 +25,7 @@ def test_healthz_head(monkeypatch, ray_start_cluster): def test_healthz_agent_1(monkeypatch, ray_start_cluster): agent_port = find_free_port() h = ray_start_cluster.add_node(dashboard_agent_listen_port=agent_port) - uri = f"http://localhost:{agent_port}/api/local_raylet_healthz" + uri = f"http://{h.node_ip_address}:{agent_port}/api/local_raylet_healthz" wait_for_condition(lambda: requests.get(uri).status_code == 200) @@ -43,7 +43,7 @@ def test_healthz_agent_2(monkeypatch, ray_start_cluster): agent_port = find_free_port() h = ray_start_cluster.add_node(dashboard_agent_listen_port=agent_port) - uri = f"http://localhost:{agent_port}/api/local_raylet_healthz" + uri = f"http://{h.node_ip_address}:{agent_port}/api/local_raylet_healthz" wait_for_condition(lambda: requests.get(uri).status_code == 200) From f797480b014262ffdf7b33a431fcbc34c0d95b2f Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Tue, 19 Aug 2025 00:10:43 -0700 Subject: [PATCH 175/634] [core] Correct bytes in flight when objects <5mb (#54349) Signed-off-by: dayshah --- python/ray/includes/ray_config.pxd | 2 - python/ray/includes/ray_config.pxi | 4 - src/ray/common/ray_config_def.h | 4 +- src/ray/object_manager/chunk_object_reader.cc | 6 +- src/ray/object_manager/chunk_object_reader.h | 5 + src/ray/object_manager/object_manager.cc | 23 +- src/ray/object_manager/object_manager.h | 2 +- src/ray/object_manager/push_manager.cc | 34 +- src/ray/object_manager/push_manager.h | 80 ++--- .../object_manager/tests/push_manager_test.cc | 295 ++++++++++-------- src/ray/raylet/main.cc | 2 + 11 files changed, 249 insertions(+), 208 deletions(-) diff --git a/python/ray/includes/ray_config.pxd b/python/ray/includes/ray_config.pxd index 7189c2b5bd14..9459cbbef77b 100644 --- a/python/ray/includes/ray_config.pxd +++ b/python/ray/includes/ray_config.pxd @@ -37,8 +37,6 @@ cdef extern from "ray/common/ray_config.h" nogil: int object_manager_push_timeout_ms() const - uint64_t object_manager_default_chunk_size() const - uint32_t maximum_gcs_deletion_batch_size() const int64_t max_direct_call_object_size() const diff --git a/python/ray/includes/ray_config.pxi b/python/ray/includes/ray_config.pxi index d83273b4800f..d1506678bae4 100644 --- a/python/ray/includes/ray_config.pxi +++ b/python/ray/includes/ray_config.pxi @@ -61,10 +61,6 @@ cdef class Config: def object_manager_push_timeout_ms(): return RayConfig.instance().object_manager_push_timeout_ms() - @staticmethod - def object_manager_default_chunk_size(): - return RayConfig.instance().object_manager_default_chunk_size() - @staticmethod def maximum_gcs_deletion_batch_size(): return RayConfig.instance().maximum_gcs_deletion_batch_size() diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index 313446827d30..827cff80da73 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -334,9 +334,7 @@ RAY_CONFIG(uint64_t, object_manager_default_chunk_size, 5 * 1024 * 1024) /// The maximum number of outbound bytes to allow to be outstanding. This avoids /// excessive memory usage during object broadcast to many receivers. -RAY_CONFIG(uint64_t, - object_manager_max_bytes_in_flight, - ((uint64_t)2) * 1024 * 1024 * 1024) +RAY_CONFIG(int64_t, object_manager_max_bytes_in_flight, (int64_t)2 * 1024 * 1024 * 1024) /// Maximum number of ids in one batch to send to GCS to delete keys. RAY_CONFIG(uint32_t, maximum_gcs_deletion_batch_size, 1000) diff --git a/src/ray/object_manager/chunk_object_reader.cc b/src/ray/object_manager/chunk_object_reader.cc index 2038033751c5..950a546e1470 100644 --- a/src/ray/object_manager/chunk_object_reader.cc +++ b/src/ray/object_manager/chunk_object_reader.cc @@ -50,7 +50,7 @@ std::optional ChunkObjectReader::GetChunk(uint64_t chunk_index) con auto offset = cur_chunk_offset; auto data_size = std::min(object_->GetDataSize() - cur_chunk_offset, cur_chunk_size); if (!object_->ReadFromDataSection(offset, data_size, result)) { - return std::optional(); + return std::nullopt; } } @@ -61,9 +61,9 @@ std::optional ChunkObjectReader::GetChunk(uint64_t chunk_index) con auto size = std::min(cur_chunk_offset + cur_chunk_size - object_->GetDataSize(), cur_chunk_size); if (!object_->ReadFromMetadataSection(offset, size, result)) { - return std::optional(); + return std::nullopt; } } - return std::optional(std::move(result)); + return result; } }; // namespace ray diff --git a/src/ray/object_manager/chunk_object_reader.h b/src/ray/object_manager/chunk_object_reader.h index 097d2c84e863..b70df5c1a9f5 100644 --- a/src/ray/object_manager/chunk_object_reader.h +++ b/src/ray/object_manager/chunk_object_reader.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include @@ -41,6 +42,10 @@ class ChunkObjectReader { const IObjectReader &GetObject() const { return *object_; } + uint64_t ChunkSize() const { + return std::min(chunk_size_, object_->GetDataSize() + object_->GetMetadataSize()); + } + private: const std::shared_ptr object_; const uint64_t chunk_size_; diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index 90d8cbe5d93b..62fa2d51b2d0 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -117,13 +117,10 @@ ObjectManager::ObjectManager( restore_spilled_object_(restore_spilled_object), get_spilled_object_url_(std::move(get_spilled_object_url)), pull_retry_timer_(*main_service_, - boost::posix_time::milliseconds(config.timer_freq_ms)) { + boost::posix_time::milliseconds(config.timer_freq_ms)), + push_manager_(std::make_unique(config_.max_bytes_in_flight)) { RAY_CHECK_GT(config_.rpc_service_threads_number, 0); - push_manager_.reset(new PushManager(/* max_chunks_in_flight= */ std::max( - static_cast(1L), - static_cast(config_.max_bytes_in_flight / config_.object_chunk_size)))); - pull_retry_timer_.async_wait([this](const boost::system::error_code &e) { Tick(e); }); auto object_is_local = [this](const ObjectID &object_id) { @@ -493,8 +490,13 @@ void ObjectManager::PushObjectInternal(const ObjectID &object_id, << ", total data size: " << chunk_reader->GetObject().GetObjectSize(); auto push_id = UniqueID::FromRandom(); + uint64_t push_max_chunk_size = chunk_reader->ChunkSize(); push_manager_->StartPush( - node_id, object_id, chunk_reader->GetNumChunks(), [=](int64_t chunk_id) { + node_id, + object_id, + chunk_reader->GetNumChunks(), + push_max_chunk_size, + [=](int64_t chunk_id) { rpc_service_.post( [=]() { // Post to the multithreaded RPC event loop so that data is copied @@ -505,11 +507,14 @@ void ObjectManager::PushObjectInternal(const ObjectID &object_id, node_id, chunk_id, rpc_client, - [=](const Status &status) { + [this, push_max_chunk_size](const Status &status) { // Post back to the main event loop because the // PushManager is not thread-safe. - main_service_->post([this]() { push_manager_->OnChunkComplete(); }, - "ObjectManager.Push"); + this->main_service_->post( + [this, push_max_chunk_size]() { + this->push_manager_->OnChunkComplete(push_max_chunk_size); + }, + "ObjectManager.Push"); }, chunk_reader, from_disk); diff --git a/src/ray/object_manager/object_manager.h b/src/ray/object_manager/object_manager.h index 593f2d2b1455..4a1b3f4535fa 100644 --- a/src/ray/object_manager/object_manager.h +++ b/src/ray/object_manager/object_manager.h @@ -53,7 +53,7 @@ struct ObjectManagerConfig { /// Object chunk size, in bytes uint64_t object_chunk_size; /// Max object push bytes in flight. - uint64_t max_bytes_in_flight; + int64_t max_bytes_in_flight; /// The store socket name. std::string store_socket_name; /// The time in milliseconds to wait until a Push request diff --git a/src/ray/object_manager/push_manager.cc b/src/ray/object_manager/push_manager.cc index 1ce1e0258dc8..25701fe32c5c 100644 --- a/src/ray/object_manager/push_manager.cc +++ b/src/ray/object_manager/push_manager.cc @@ -24,6 +24,7 @@ namespace ray { void PushManager::StartPush(const NodeID &dest_id, const ObjectID &obj_id, int64_t num_chunks, + int64_t max_chunk_size, std::function send_chunk_fn) { auto push_id = std::make_pair(dest_id, obj_id); RAY_CHECK(num_chunks > 0); @@ -37,18 +38,19 @@ void PushManager::StartPush(const NodeID &dest_id, dest_id, obj_id, num_chunks, + max_chunk_size, std::move(send_chunk_fn)); } else { RAY_LOG(DEBUG) << "Duplicate push request " << push_id.first << ", " << push_id.second << ", resending all the chunks."; - RAY_CHECK_NE(it->second->num_chunks_to_send, 0); + RAY_CHECK_NE(it->second->num_chunks_to_send_, 0); chunks_remaining_ += it->second->ResendAllChunks(std::move(send_chunk_fn)); } ScheduleRemainingPushes(); } -void PushManager::OnChunkComplete() { - chunks_in_flight_ -= 1; +void PushManager::OnChunkComplete(int64_t push_max_chunk_size) { + bytes_in_flight_ -= push_max_chunk_size; chunks_remaining_ -= 1; ScheduleRemainingPushes(); } @@ -62,23 +64,23 @@ void PushManager::ScheduleRemainingPushes() { // Loop over all active pushes for approximate round-robin prioritization. bool keep_looping = true; - while (chunks_in_flight_ < max_chunks_in_flight_ && keep_looping) { + while (bytes_in_flight_ < max_bytes_in_flight_ && keep_looping) { // Loop over each active push and try to send another chunk. - // If we could push out a chunk and haven't reached the chunks_in_flight_ limit, + // If we could push out a chunk and haven't reached the max_bytes_in_flight_ limit, // we'll loop again to try to send more chunks. keep_looping = false; auto iter = push_requests_with_chunks_to_send_.begin(); while (iter != push_requests_with_chunks_to_send_.end() && - chunks_in_flight_ < max_chunks_in_flight_) { + bytes_in_flight_ < max_bytes_in_flight_) { auto &push_state = *iter; push_state.SendOneChunk(); - chunks_in_flight_ += 1; - if (push_state.num_chunks_to_send == 0) { - auto push_state_map_iter = push_state_map_.find(push_state.node_id); + bytes_in_flight_ += push_state.max_chunk_size_; + if (push_state.num_chunks_to_send_ == 0) { + auto push_state_map_iter = push_state_map_.find(push_state.node_id_); RAY_CHECK(push_state_map_iter != push_state_map_.end()); auto &dest_map = push_state_map_iter->second; - auto dest_map_iter = dest_map.find(push_state.object_id); + auto dest_map_iter = dest_map.find(push_state.object_id_); RAY_CHECK(dest_map_iter != dest_map.end()); iter = push_requests_with_chunks_to_send_.erase(dest_map_iter->second); @@ -107,18 +109,16 @@ void PushManager::HandleNodeRemoved(const NodeID &node_id) { void PushManager::RecordMetrics() const { ray::stats::STATS_push_manager_num_pushes_remaining.Record( - NumPushRequestsWithChunksToSend()); - ray::stats::STATS_push_manager_chunks.Record(NumChunksInFlight(), "InFlight"); - ray::stats::STATS_push_manager_chunks.Record(NumChunksRemaining(), "Remaining"); + push_requests_with_chunks_to_send_.size()); + ray::stats::STATS_push_manager_chunks.Record(chunks_remaining_, "Remaining"); } std::string PushManager::DebugString() const { std::stringstream result; result << "PushManager:"; - result << "\n- num pushes remaining: " << NumPushRequestsWithChunksToSend(); - result << "\n- num chunks in flight: " << NumChunksInFlight(); - result << "\n- num chunks remaining: " << NumChunksRemaining(); - result << "\n- max chunks allowed: " << max_chunks_in_flight_; + result << "\n- num pushes remaining: " << push_requests_with_chunks_to_send_.size(); + result << "\n- num chunks remaining: " << chunks_remaining_; + result << "\n- max bytes allowed: " << max_bytes_in_flight_; return result.str(); } diff --git a/src/ray/object_manager/push_manager.h b/src/ray/object_manager/push_manager.h index 79195d2327e3..b61903e9123f 100644 --- a/src/ray/object_manager/push_manager.h +++ b/src/ray/object_manager/push_manager.h @@ -28,12 +28,10 @@ class PushManager { public: /// Create a push manager. /// - /// \param max_chunks_in_flight Max number of chunks allowed to be in flight + /// \param max_bytes_in_flight Max number of bytes allowed to be in flight /// from this PushManager (this raylet). - explicit PushManager(int64_t max_chunks_in_flight) - : max_chunks_in_flight_(max_chunks_in_flight) { - RAY_CHECK_GT(max_chunks_in_flight_, 0); - }; + explicit PushManager(int64_t max_bytes_in_flight) + : max_bytes_in_flight_(max_bytes_in_flight){}; /// Start pushing an object subject to max chunks in flight limit. /// @@ -42,91 +40,97 @@ class PushManager { /// \param dest_id The node to send to. /// \param obj_id The object to send. /// \param num_chunks The total number of chunks to send. + /// \param max_chunk_size See comment for max_chunk_size_ in PushState. /// \param send_chunk_fn This function will be called with args 0...{num_chunks-1}. /// The caller promises to call PushManager::OnChunkComplete() /// once a call to send_chunk_fn finishes. void StartPush(const NodeID &dest_id, const ObjectID &obj_id, int64_t num_chunks, + int64_t max_chunk_size, std::function send_chunk_fn); /// Called every time a chunk completes to trigger additional sends. /// TODO(ekl) maybe we should cancel the entire push on error. - void OnChunkComplete(); + void OnChunkComplete(int64_t push_max_chunk_size); /// Cancel all pushes that have not yet been sent to the removed node. void HandleNodeRemoved(const NodeID &node_id); - /// Return the number of chunks currently in flight. For metrics and testing. - int64_t NumChunksInFlight() const { return chunks_in_flight_; }; + void RecordMetrics() const; - /// Return the number of chunks remaining. For metrics and testing. - int64_t NumChunksRemaining() const { return chunks_remaining_; } + int64_t BytesInFlight() const { return bytes_in_flight_; } - /// Return the number of push requests with remaining chunks. For metrics and testing. - int64_t NumPushRequestsWithChunksToSend() const { - return push_requests_with_chunks_to_send_.size(); - }; + int64_t ChunksRemaining() const { return chunks_remaining_; } - /// Record the internal metrics. - void RecordMetrics() const; + int64_t PushesInFlight() const { return push_state_map_.size(); } + + int64_t PushRequestsRemaining() const { + return push_requests_with_chunks_to_send_.size(); + } std::string DebugString() const; private: FRIEND_TEST(TestPushManager, TestPushState); - FRIEND_TEST(TestPushManager, TestNodeRemoved); /// Tracks the state of an active object push to another node. struct PushState { - NodeID node_id; - ObjectID object_id; + NodeID node_id_; + ObjectID object_id_; /// total number of chunks of this object. - int64_t num_chunks; + int64_t num_chunks_; + /// the max size of a chunk for this object in bytes, used to count bytes_in_flight_ + /// and assure it stays under max_bytes_in_flight_. This means we can overcount for + /// the last chunk but we're accepting that to keep the code simpler. + int64_t max_chunk_size_; /// The function to send chunks with. - std::function chunk_send_fn; + std::function chunk_send_fn_; + /// The index of the next chunk to send. - int64_t next_chunk_id = 0; + int64_t next_chunk_id_ = 0; /// The number of chunks remaining to send. - int64_t num_chunks_to_send; + int64_t num_chunks_to_send_; PushState(NodeID node_id, ObjectID object_id, int64_t num_chunks, + int64_t max_chunk_size, std::function chunk_send_fn) - : node_id(node_id), - object_id(object_id), - num_chunks(num_chunks), - chunk_send_fn(std::move(chunk_send_fn)), - num_chunks_to_send(num_chunks) {} + : node_id_(node_id), + object_id_(object_id), + num_chunks_(num_chunks), + max_chunk_size_(max_chunk_size), + chunk_send_fn_(std::move(chunk_send_fn)), + num_chunks_to_send_(num_chunks) {} /// Resend all chunks and returns how many more chunks will be sent. int64_t ResendAllChunks(std::function send_fn) { - chunk_send_fn = std::move(send_fn); - int64_t additional_chunks_to_send = num_chunks - num_chunks_to_send; - num_chunks_to_send = num_chunks; + chunk_send_fn_ = std::move(send_fn); + int64_t additional_chunks_to_send = num_chunks_ - num_chunks_to_send_; + num_chunks_to_send_ = num_chunks_; return additional_chunks_to_send; } /// Send one chunk. Return true if a new chunk is sent, false if no more chunk to /// send. void SendOneChunk() { - num_chunks_to_send--; + num_chunks_to_send_--; // Send the next chunk for this push. - chunk_send_fn(next_chunk_id); - next_chunk_id = (next_chunk_id + 1) % num_chunks; + chunk_send_fn_(next_chunk_id_); + next_chunk_id_ = (next_chunk_id_ + 1) % num_chunks_; } }; /// Called on completion events to trigger additional pushes. void ScheduleRemainingPushes(); - /// Max number of chunks in flight allowed. - const int64_t max_chunks_in_flight_; + /// Max number of bytes in flight allowed. + const int64_t max_bytes_in_flight_; - /// Running count of chunks in flight, used to limit progress of in_flight_pushes_. - int64_t chunks_in_flight_ = 0; + /// Running count of bytes in flight + int64_t bytes_in_flight_ = 0; /// Remaining count of chunks to push to other nodes. int64_t chunks_remaining_ = 0; diff --git a/src/ray/object_manager/tests/push_manager_test.cc b/src/ray/object_manager/tests/push_manager_test.cc index d3f5188a9e3b..084d9078184b 100644 --- a/src/ray/object_manager/tests/push_manager_test.cc +++ b/src/ray/object_manager/tests/push_manager_test.cc @@ -27,78 +27,86 @@ TEST(TestPushManager, TestSingleTransfer) { results.resize(10); auto node_id = NodeID::FromRandom(); auto obj_id = ObjectID::FromRandom(); - PushManager pm(5); - pm.StartPush(node_id, obj_id, 10, [&](int64_t chunk_id) { results[chunk_id] = 1; }); - ASSERT_EQ(pm.NumChunksInFlight(), 5); - ASSERT_EQ(pm.NumChunksRemaining(), 10); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); + PushManager pm(25); + int64_t push_max_chunk_size = 5; + pm.StartPush(node_id, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { + results[chunk_id] = 1; + }); + ASSERT_EQ(pm.BytesInFlight(), 25); + ASSERT_EQ(pm.ChunksRemaining(), 10); + ASSERT_EQ(pm.PushRequestsRemaining(), 1); for (int i = 0; i < 10; i++) { - pm.OnChunkComplete(); + pm.OnChunkComplete(push_max_chunk_size); } - ASSERT_EQ(pm.NumChunksInFlight(), 0); - ASSERT_EQ(pm.NumChunksRemaining(), 0); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); + ASSERT_EQ(pm.BytesInFlight(), 0); + ASSERT_EQ(pm.ChunksRemaining(), 0); + ASSERT_EQ(pm.PushRequestsRemaining(), 0); for (int i = 0; i < 10; i++) { ASSERT_EQ(results[i], 1); } } TEST(TestPushManager, TestPushState) { + int64_t push_max_chunk_size = 5; // normal sending. { std::vector sent_chunks; PushManager::PushState state{ - NodeID::FromRandom(), ObjectID::FromRandom(), 2, [&](int64_t chunk_id) { - sent_chunks.push_back(chunk_id); - }}; - ASSERT_EQ(state.num_chunks, 2); - ASSERT_EQ(state.next_chunk_id, 0); - ASSERT_EQ(state.num_chunks_to_send, 2); + NodeID::FromRandom(), + ObjectID::FromRandom(), + 2, + push_max_chunk_size, + [&](int64_t chunk_id) { sent_chunks.push_back(chunk_id); }}; + ASSERT_EQ(state.num_chunks_, 2); + ASSERT_EQ(state.next_chunk_id_, 0); + ASSERT_EQ(state.num_chunks_to_send_, 2); state.SendOneChunk(); - ASSERT_EQ(state.num_chunks, 2); - ASSERT_EQ(state.next_chunk_id, 1); - ASSERT_EQ(state.num_chunks_to_send, 1); + ASSERT_EQ(state.num_chunks_, 2); + ASSERT_EQ(state.next_chunk_id_, 1); + ASSERT_EQ(state.num_chunks_to_send_, 1); ASSERT_EQ(sent_chunks, (std::vector{0})); state.SendOneChunk(); - ASSERT_EQ(state.num_chunks, 2); - ASSERT_EQ(state.next_chunk_id, 0); - ASSERT_EQ(state.num_chunks_to_send, 0); + ASSERT_EQ(state.num_chunks_, 2); + ASSERT_EQ(state.next_chunk_id_, 0); + ASSERT_EQ(state.num_chunks_to_send_, 0); ASSERT_EQ(sent_chunks, (std::vector{0, 1})); - ASSERT_EQ(state.num_chunks_to_send, 0); + ASSERT_EQ(state.num_chunks_to_send_, 0); } // resend all chunks. { std::vector sent_chunks; PushManager::PushState state{ - NodeID::FromRandom(), ObjectID::FromRandom(), 3, [&](int64_t chunk_id) { - sent_chunks.push_back(chunk_id); - }}; + NodeID::FromRandom(), + ObjectID::FromRandom(), + 3, + push_max_chunk_size, + [&](int64_t chunk_id) { sent_chunks.push_back(chunk_id); }}; state.SendOneChunk(); - ASSERT_EQ(state.num_chunks, 3); - ASSERT_EQ(state.next_chunk_id, 1); - ASSERT_EQ(state.num_chunks_to_send, 2); + ASSERT_EQ(state.num_chunks_, 3); + ASSERT_EQ(state.next_chunk_id_, 1); + ASSERT_EQ(state.num_chunks_to_send_, 2); ASSERT_EQ(sent_chunks, (std::vector{0})); // resend chunks when 1 chunk is in flight. ASSERT_EQ(1, state.ResendAllChunks([&](int64_t chunk_id) { sent_chunks.push_back(chunk_id); })); - ASSERT_EQ(state.num_chunks, 3); - ASSERT_EQ(state.next_chunk_id, 1); - ASSERT_EQ(state.num_chunks_to_send, 3); + ASSERT_EQ(state.num_chunks_, 3); + ASSERT_EQ(state.next_chunk_id_, 1); + ASSERT_EQ(state.num_chunks_to_send_, 3); for (auto i = 0; i < 3; i++) { state.SendOneChunk(); - ASSERT_EQ(state.num_chunks, 3); - ASSERT_EQ(state.next_chunk_id, (2 + i) % 3); - ASSERT_EQ(state.num_chunks_to_send, 3 - i - 1); + ASSERT_EQ(state.num_chunks_, 3); + ASSERT_EQ(state.next_chunk_id_, (2 + i) % 3); + ASSERT_EQ(state.num_chunks_to_send_, 3 - i - 1); } ASSERT_EQ(sent_chunks, (std::vector{0, 1, 2, 0})); - ASSERT_EQ(state.num_chunks_to_send, 0); + ASSERT_EQ(state.num_chunks_to_send_, 0); } } @@ -107,37 +115,42 @@ TEST(TestPushManager, TestRetryDuplicates) { results.resize(10); auto node_id = NodeID::FromRandom(); auto obj_id = ObjectID::FromRandom(); - PushManager pm(5); + PushManager pm(25); // First push request. - pm.StartPush(node_id, obj_id, 10, [&](int64_t chunk_id) { results[chunk_id] = 1; }); - ASSERT_EQ(pm.NumChunksInFlight(), 5); - ASSERT_EQ(pm.NumChunksRemaining(), 10); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); + int64_t push_max_chunk_size = 5; + pm.StartPush(node_id, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { + results[chunk_id] = 1; + }); + ASSERT_EQ(pm.BytesInFlight(), 25); + ASSERT_EQ(pm.ChunksRemaining(), 10); + ASSERT_EQ(pm.PushRequestsRemaining(), 1); // Second push request will resent the full chunks. - pm.StartPush(node_id, obj_id, 10, [&](int64_t chunk_id) { results[chunk_id] = 2; }); - ASSERT_EQ(pm.NumChunksInFlight(), 5); - ASSERT_EQ(pm.NumChunksRemaining(), 15); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); + pm.StartPush(node_id, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { + results[chunk_id] = 2; + }); + ASSERT_EQ(pm.BytesInFlight(), 25); + ASSERT_EQ(pm.ChunksRemaining(), 15); + ASSERT_EQ(pm.PushRequestsRemaining(), 1); // first 5 chunks will be sent by first push request. for (int i = 0; i < 5; i++) { - pm.OnChunkComplete(); + pm.OnChunkComplete(push_max_chunk_size); } for (int i = 0; i < 5; i++) { ASSERT_EQ(results[i], 1); } - ASSERT_EQ(pm.NumChunksInFlight(), 5); - ASSERT_EQ(pm.NumChunksRemaining(), 10); + ASSERT_EQ(pm.BytesInFlight(), 25); + ASSERT_EQ(pm.ChunksRemaining(), 10); // we will resend all chunks by second push request. for (int i = 0; i < 10; i++) { - pm.OnChunkComplete(); + pm.OnChunkComplete(push_max_chunk_size); } for (int i = 0; i < 10; i++) { ASSERT_EQ(results[i], 2); } - ASSERT_EQ(pm.NumChunksInFlight(), 0); - ASSERT_EQ(pm.NumChunksRemaining(), 0); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); + ASSERT_EQ(pm.BytesInFlight(), 0); + ASSERT_EQ(pm.ChunksRemaining(), 0); + ASSERT_EQ(pm.PushRequestsRemaining(), 0); } TEST(TestPushManager, TestResendWholeObject) { @@ -145,34 +158,39 @@ TEST(TestPushManager, TestResendWholeObject) { results.resize(10); auto node_id = NodeID::FromRandom(); auto obj_id = ObjectID::FromRandom(); - PushManager pm(5); - pm.StartPush(node_id, obj_id, 10, [&](int64_t chunk_id) { results[chunk_id] = 1; }); - ASSERT_EQ(pm.NumChunksInFlight(), 5); - ASSERT_EQ(pm.NumChunksRemaining(), 10); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); + PushManager pm(25); + int64_t push_max_chunk_size = 5; + pm.StartPush(node_id, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { + results[chunk_id] = 1; + }); + ASSERT_EQ(pm.BytesInFlight(), 25); + ASSERT_EQ(pm.ChunksRemaining(), 10); + ASSERT_EQ(pm.PushRequestsRemaining(), 1); for (int i = 0; i < 5; i++) { - pm.OnChunkComplete(); + pm.OnChunkComplete(push_max_chunk_size); } // All chunks have been sent out - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); - ASSERT_EQ(pm.NumChunksRemaining(), 5); + ASSERT_EQ(pm.PushRequestsRemaining(), 0); + ASSERT_EQ(pm.ChunksRemaining(), 5); // resend this object, and it needs to be added to the traversal list. - pm.StartPush(node_id, obj_id, 10, [&](int64_t chunk_id) { results[chunk_id] = 2; }); - ASSERT_EQ(pm.NumChunksInFlight(), 5); - ASSERT_EQ(pm.NumChunksRemaining(), 15); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); + pm.StartPush(node_id, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { + results[chunk_id] = 2; + }); + ASSERT_EQ(pm.BytesInFlight(), 25); + ASSERT_EQ(pm.ChunksRemaining(), 15); + ASSERT_EQ(pm.PushRequestsRemaining(), 1); // we will resend all chunks by second push request. for (int i = 0; i < 15; i++) { - pm.OnChunkComplete(); + pm.OnChunkComplete(push_max_chunk_size); } for (int i = 0; i < 10; i++) { ASSERT_EQ(results[i], 2); } - ASSERT_EQ(pm.NumChunksInFlight(), 0); - ASSERT_EQ(pm.NumChunksRemaining(), 0); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); + ASSERT_EQ(pm.BytesInFlight(), 0); + ASSERT_EQ(pm.ChunksRemaining(), 0); + ASSERT_EQ(pm.PushRequestsRemaining(), 0); } TEST(TestPushManager, TestMultipleTransfers) { @@ -185,30 +203,31 @@ TEST(TestPushManager, TestMultipleTransfers) { auto obj_id = ObjectID::FromRandom(); int num_active1 = 0; int num_active2 = 0; - PushManager pm(5); - pm.StartPush(node1, obj_id, 10, [&](int64_t chunk_id) { + PushManager pm(25); + int64_t push_max_chunk_size = 5; + pm.StartPush(node1, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { results1[chunk_id] = 1; num_active1++; }); - pm.StartPush(node2, obj_id, 10, [&](int64_t chunk_id) { + pm.StartPush(node2, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { results2[chunk_id] = 2; num_active2++; }); - ASSERT_EQ(pm.NumChunksInFlight(), 5); - ASSERT_EQ(pm.NumChunksRemaining(), 20); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 2); + ASSERT_EQ(pm.BytesInFlight(), 25); + ASSERT_EQ(pm.ChunksRemaining(), 20); + ASSERT_EQ(pm.PushRequestsRemaining(), 2); for (int i = 0; i < 20; i++) { if (num_active1 > 0) { - pm.OnChunkComplete(); + pm.OnChunkComplete(push_max_chunk_size); num_active1--; } else if (num_active2 > 0) { - pm.OnChunkComplete(); + pm.OnChunkComplete(push_max_chunk_size); num_active2--; } } - ASSERT_EQ(pm.NumChunksInFlight(), 0); - ASSERT_EQ(pm.NumChunksRemaining(), 0); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); + ASSERT_EQ(pm.BytesInFlight(), 0); + ASSERT_EQ(pm.ChunksRemaining(), 0); + ASSERT_EQ(pm.PushRequestsRemaining(), 0); for (int i = 0; i < 10; i++) { ASSERT_EQ(results1[i], 1); } @@ -222,42 +241,55 @@ TEST(TestPushManager, TestPushMultipleObject) { auto obj_id_1 = ObjectID::FromRandom(); auto obj_id_2 = ObjectID::FromRandom(); auto obj_id_3 = ObjectID::FromRandom(); - PushManager pm(3); + PushManager pm(15); absl::flat_hash_map> result; - pm.StartPush(node_id, obj_id_1, 4, [&, obj_id = obj_id_1](int64_t chunk_id) { - ASSERT_FALSE(result[obj_id].contains(chunk_id)); - result[obj_id].insert(chunk_id); - }); - pm.StartPush(node_id, obj_id_2, 1, [&, obj_id = obj_id_2](int64_t chunk_id) { - ASSERT_FALSE(result[obj_id].contains(chunk_id)); - result[obj_id].insert(chunk_id); - }); - pm.StartPush(node_id, obj_id_3, 2, [&, obj_id = obj_id_3](int64_t chunk_id) { - ASSERT_FALSE(result[obj_id].contains(chunk_id)); - result[obj_id].insert(chunk_id); - }); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 3); - ASSERT_EQ(pm.NumChunksInFlight(), 3); - ASSERT_EQ(pm.NumChunksRemaining(), 7); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 3); - - pm.OnChunkComplete(); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 2); - pm.OnChunkComplete(); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); - pm.OnChunkComplete(); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); - pm.OnChunkComplete(); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); - - pm.OnChunkComplete(); - pm.OnChunkComplete(); - pm.OnChunkComplete(); - - ASSERT_EQ(pm.NumChunksInFlight(), 0); - ASSERT_EQ(pm.NumChunksRemaining(), 0); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); + int64_t push_max_chunk_size = 5; + pm.StartPush(node_id, + obj_id_1, + 4, + push_max_chunk_size, + [&, obj_id = obj_id_1](int64_t chunk_id) { + ASSERT_FALSE(result[obj_id].contains(chunk_id)); + result[obj_id].insert(chunk_id); + }); + pm.StartPush(node_id, + obj_id_2, + 1, + push_max_chunk_size, + [&, obj_id = obj_id_2](int64_t chunk_id) { + ASSERT_FALSE(result[obj_id].contains(chunk_id)); + result[obj_id].insert(chunk_id); + }); + pm.StartPush(node_id, + obj_id_3, + 2, + push_max_chunk_size, + [&, obj_id = obj_id_3](int64_t chunk_id) { + ASSERT_FALSE(result[obj_id].contains(chunk_id)); + result[obj_id].insert(chunk_id); + }); + ASSERT_EQ(pm.PushRequestsRemaining(), 3); + ASSERT_EQ(pm.BytesInFlight(), 15); + ASSERT_EQ(pm.ChunksRemaining(), 7); + ASSERT_EQ(pm.PushRequestsRemaining(), 3); + + pm.OnChunkComplete(push_max_chunk_size); + ASSERT_EQ(pm.PushRequestsRemaining(), 2); + pm.OnChunkComplete(push_max_chunk_size); + ASSERT_EQ(pm.PushRequestsRemaining(), 1); + pm.OnChunkComplete(push_max_chunk_size); + ASSERT_EQ(pm.PushRequestsRemaining(), 1); + pm.OnChunkComplete(push_max_chunk_size); + ASSERT_EQ(pm.PushRequestsRemaining(), 0); + + pm.OnChunkComplete(push_max_chunk_size); + pm.OnChunkComplete(push_max_chunk_size); + pm.OnChunkComplete(push_max_chunk_size); + + ASSERT_EQ(pm.BytesInFlight(), 0); + ASSERT_EQ(pm.ChunksRemaining(), 0); + ASSERT_EQ(pm.PushRequestsRemaining(), 0); ASSERT_EQ(result[obj_id_1].size(), 4); ASSERT_EQ(result[obj_id_2].size(), 1); @@ -265,48 +297,49 @@ TEST(TestPushManager, TestPushMultipleObject) { } TEST(TestPushManager, TestNodeRemoved) { - PushManager pm(3); + PushManager pm(15); // Start pushing two objects to node 1. auto node_id_1 = NodeID::FromRandom(); auto obj_id_1 = ObjectID::FromRandom(); auto obj_id_2 = ObjectID::FromRandom(); - pm.StartPush(node_id_1, obj_id_1, 4, [](int64_t) {}); - pm.StartPush(node_id_1, obj_id_2, 2, [](int64_t) {}); + int64_t push_max_chunk_size = 5; + pm.StartPush(node_id_1, obj_id_1, 4, push_max_chunk_size, [](int64_t) {}); + pm.StartPush(node_id_1, obj_id_2, 2, push_max_chunk_size, [](int64_t) {}); // Start pushing one object to node 2. auto node_id_2 = NodeID::FromRandom(); auto obj_id_3 = ObjectID::FromRandom(); - pm.StartPush(node_id_2, obj_id_3, 3, [](int64_t) {}); + pm.StartPush(node_id_2, obj_id_3, 3, push_max_chunk_size, [](int64_t) {}); // 3 chunks in flight for 3 objects to two nodes. - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 3); - ASSERT_EQ(pm.NumChunksInFlight(), 3); - ASSERT_EQ(pm.push_state_map_.size(), 2); - ASSERT_EQ(pm.push_requests_with_chunks_to_send_.size(), 3); + ASSERT_EQ(pm.PushRequestsRemaining(), 3); + ASSERT_EQ(pm.BytesInFlight(), 15); + ASSERT_EQ(pm.PushesInFlight(), 2); + ASSERT_EQ(pm.PushRequestsRemaining(), 3); // Remove Node 1. This should cause its associated push requests to be cleaned up. pm.HandleNodeRemoved(node_id_1); - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); - ASSERT_EQ(pm.NumChunksInFlight(), 3); - ASSERT_EQ(pm.push_state_map_.size(), 1); - ASSERT_EQ(pm.push_requests_with_chunks_to_send_.size(), 1); + ASSERT_EQ(pm.PushRequestsRemaining(), 1); + ASSERT_EQ(pm.BytesInFlight(), 15); + ASSERT_EQ(pm.PushesInFlight(), 1); + ASSERT_EQ(pm.PushRequestsRemaining(), 1); // All 3 in flight chunks finish. // All pushes should be done with chunks to node 2 in flight. for (int i = 0; i < 3; i++) { - pm.OnChunkComplete(); + pm.OnChunkComplete(push_max_chunk_size); } - ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); - ASSERT_EQ(pm.NumChunksInFlight(), 3); - ASSERT_EQ(pm.push_state_map_.size(), 0); - ASSERT_EQ(pm.push_requests_with_chunks_to_send_.size(), 0); + ASSERT_EQ(pm.PushRequestsRemaining(), 0); + ASSERT_EQ(pm.BytesInFlight(), 15); + ASSERT_EQ(pm.PushesInFlight(), 0); + ASSERT_EQ(pm.PushRequestsRemaining(), 0); // The in flight chunks complete. for (int i = 0; i < 3; i++) { - pm.OnChunkComplete(); + pm.OnChunkComplete(push_max_chunk_size); } - ASSERT_EQ(pm.NumChunksInFlight(), 0); + ASSERT_EQ(pm.BytesInFlight(), 0); } } // namespace ray diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 7193230c0729..c9f73d3cdf95 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -473,6 +473,8 @@ int main(int argc, char *argv[]) { object_manager_config.object_store_memory = object_store_memory; object_manager_config.max_bytes_in_flight = RayConfig::instance().object_manager_max_bytes_in_flight(); + RAY_CHECK_GT(object_manager_config.max_bytes_in_flight, 0) + << "object_manager_max_bytes_in_flight must be greater than 0"; object_manager_config.plasma_directory = plasma_directory; object_manager_config.fallback_directory = fallback_directory; object_manager_config.huge_pages = huge_pages; From c4482d2fc6d7956104c5b0208a7cc14120737652 Mon Sep 17 00:00:00 2001 From: Ibrahim Rabbani Date: Tue, 19 Aug 2025 07:47:57 -0700 Subject: [PATCH 176/634] [core] Remove job submission code for using JobAgent on a random worker node. (#55718) When a Job is submitted through the SDK/JobClient, the request goes to the dashboard's JobHead. The JobHead submits a request to a JobAgent which has a JobManager. The JobManager creates a JobSupervisor actor which manages the lifecycle of the job. In #47147, the `RAY_JOB_AGENT_USE_HEAD_NODE_ONLY` feature flag to force head node's JobAgent to be used for job submission. The flag was intended to be a temporary kill switch if head_node only scheduling had issues. Now that #47147 has been merged for over a year, I'm cleaning up the flag in this PR and making it the default (and only behavior). --------- Signed-off-by: irabbani --- python/ray/dashboard/modules/job/job_head.py | 113 +--------- .../modules/job/tests/test_http_job_server.py | 206 +----------------- .../dashboard/modules/job/tests/test_sdk.py | 130 ++--------- 3 files changed, 25 insertions(+), 424 deletions(-) diff --git a/python/ray/dashboard/modules/job/job_head.py b/python/ray/dashboard/modules/job/job_head.py index 333075b10679..06bc1cfe37d1 100644 --- a/python/ray/dashboard/modules/job/job_head.py +++ b/python/ray/dashboard/modules/job/job_head.py @@ -7,19 +7,17 @@ import time import traceback from datetime import datetime -from random import choice -from typing import AsyncIterator, Dict, List, Optional, Tuple +from typing import AsyncIterator, Dict, Optional, Tuple import aiohttp.web from aiohttp.client import ClientResponse from aiohttp.web import Request, Response, StreamResponse import ray -import ray.dashboard.consts as dashboard_consts from ray import NodeID from ray._common.utils import get_or_create_event_loop, load_class from ray._common.pydantic_compat import BaseModel, Extra, Field, validator -from ray._private.ray_constants import KV_NAMESPACE_DASHBOARD, env_bool +from ray._private.ray_constants import KV_NAMESPACE_DASHBOARD from ray._private.runtime_env.packaging import ( package_exists, pin_runtime_env_uri, @@ -57,12 +55,6 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -# Feature flag controlling whether critical Ray Job control operations are performed -# exclusively by the Job Agent running on the Head node (or randomly sampled Worker one) -# -# NOTE: This flag serves as a temporary kill-switch and should be eventually cleaned up -RAY_JOB_AGENT_USE_HEAD_NODE_ONLY = env_bool("RAY_JOB_AGENT_USE_HEAD_NODE_ONLY", True) - class RayActivityStatus(str, enum.Enum): ACTIVE = "ACTIVE" @@ -249,86 +241,7 @@ async def get_target_agent( Raises: TimeoutError: If the operation times out. """ - if RAY_JOB_AGENT_USE_HEAD_NODE_ONLY: - return await self._get_head_node_agent(timeout_s) - - return await self._pick_random_agent(timeout_s) - - async def _pick_random_agent( - self, timeout_s: float - ) -> Optional[JobAgentSubmissionClient]: - """ - Try to disperse as much as possible to select one of - the `CANDIDATE_AGENT_NUMBER` agents to solve requests. - the agents will not pop from `self._agents` unless - it's dead. Saved in `self._agents` is the agent that was - used before. - Strategy: - 1. if the number of `self._agents` has reached - `CANDIDATE_AGENT_NUMBER`, randomly select one agent from - `self._agents`. - 2. if not, randomly select one agent from all available agents, - it is possible that the selected one already exists in - `self._agents`. - - If there's no agent available at all, or there's exception, it will retry every - `TRY_TO_GET_AGENT_INFO_INTERVAL_SECONDS` seconds indefinitely. - - Args: - timeout_s: The timeout for the operation. - - Returns: - A `JobAgentSubmissionClient` for interacting with jobs via an agent process. - - Raises: - TimeoutError: If the operation times out. - """ - start_time_s = time.time() - last_exception = None - while time.time() < start_time_s + timeout_s: - try: - return await self._pick_random_agent_once() - except Exception as e: - last_exception = e - logger.exception( - f"Failed to pick a random agent, retrying in {TRY_TO_GET_AGENT_INFO_INTERVAL_SECONDS} seconds..." - ) - await asyncio.sleep(TRY_TO_GET_AGENT_INFO_INTERVAL_SECONDS) - raise TimeoutError( - f"Failed to pick a random agent within {timeout_s} seconds. The last exception is {last_exception}" - ) - - async def _pick_random_agent_once(self) -> JobAgentSubmissionClient: - """ - Query the internal kv for all agent infos, and pick agents randomly. May raise - exception if there's no agent available at all or there's network error. - """ - # NOTE: Following call will block until there's at least 1 agent info - # being populated from GCS - agent_node_ids = await self._fetch_all_agent_node_ids() - - # delete dead agents. - for dead_node in set(self._agents) - set(agent_node_ids): - client = self._agents.pop(dead_node) - await client.close() - - if len(self._agents) >= dashboard_consts.CANDIDATE_AGENT_NUMBER: - node_id = choice(list(self._agents)) - return self._agents[node_id] - else: - # Randomly select one from among all agents, it is possible that - # the selected one already exists in `self._agents` - node_id = choice(list(agent_node_ids)) - - if node_id not in self._agents: - # Fetch agent info from InternalKV, and create a new - # JobAgentSubmissionClient. May raise if the node_id is removed in - # InternalKV after the _fetch_all_agent_node_ids, though unlikely. - ip, http_port, _ = await self._fetch_agent_info(node_id) - agent_http_address = f"http://{build_address(ip, http_port)}" - self._agents[node_id] = JobAgentSubmissionClient(agent_http_address) - - return self._agents[node_id] + return await self._get_head_node_agent(timeout_s) async def _get_head_node_agent_once(self) -> JobAgentSubmissionClient: head_node_id_hex = await get_head_node_id(self.gcs_client) @@ -374,26 +287,6 @@ async def _get_head_node_agent(self, timeout_s: float) -> JobAgentSubmissionClie f"Failed to get head node agent within {timeout_s} seconds. The last exception is {exception}" ) - async def _fetch_all_agent_node_ids(self) -> List[NodeID]: - """ - Fetches all NodeIDs with agent infos in the cluster. - - May raise exception if there's no agent available at all or there's network error. - Returns: List[NodeID] - """ - keys = await self.gcs_client.async_internal_kv_keys( - f"{DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX}".encode(), - namespace=KV_NAMESPACE_DASHBOARD, - timeout=GCS_RPC_TIMEOUT_SECONDS, - ) - if not keys: - # No agent keys found, retry - raise Exception("No agents found in InternalKV.") - return [ - NodeID.from_hex(key[len(DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX) :].decode()) - for key in keys - ] - async def _fetch_agent_info(self, target_node_id: NodeID) -> Tuple[str, int, int]: """ Fetches agent info by the Node ID. May raise exception if there's network error or the diff --git a/python/ray/dashboard/modules/job/tests/test_http_job_server.py b/python/ray/dashboard/modules/job/tests/test_http_job_server.py index 07151a341038..b66df1178bfa 100644 --- a/python/ray/dashboard/modules/job/tests/test_http_job_server.py +++ b/python/ray/dashboard/modules/job/tests/test_http_job_server.py @@ -1,4 +1,3 @@ -import asyncio import json import logging import os @@ -8,7 +7,7 @@ import tempfile import time from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Optional from unittest.mock import patch import pytest @@ -17,7 +16,6 @@ import yaml import ray -from ray import NodeID from ray._private.runtime_env.packaging import ( create_package, download_and_unpack_package, @@ -26,16 +24,10 @@ from ray._private.test_utils import ( chdir, format_web_url, - ray_constants, wait_until_server_available, ) -from ray.dashboard.consts import ( - DASHBOARD_AGENT_ADDR_IP_PREFIX, - DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX, -) from ray.dashboard.modules.dashboard_sdk import ClusterInfo, parse_cluster_info from ray.dashboard.modules.job.common import uri_to_http_components -from ray.dashboard.modules.job.job_head import JobHead from ray.dashboard.modules.job.pydantic_models import JobDetails from ray.dashboard.modules.job.tests.test_cli_integration import set_env_var from ray.dashboard.modules.version import CURRENT_VERSION @@ -746,202 +738,6 @@ def test_jobs_env_hook(job_sdk_client: JobSubmissionClient): assert f.read().strip() == "Ray rocks!" -@pytest.mark.asyncio -async def test_job_head_pick_random_job_agent(monkeypatch): - with set_env_var("CANDIDATE_AGENT_NUMBER", "2"): - import importlib - - importlib.reload(ray.dashboard.consts) - - # Fake GCS client - class _FakeGcsClient: - def __init__(self): - self._kv: Dict[bytes, bytes] = {} - - @staticmethod - def ensure_bytes(key: Union[bytes, str]) -> bytes: - return key.encode() if isinstance(key, str) else key - - async def async_internal_kv_put( - self, key: Union[bytes, str], value: bytes, **kwargs - ): - key = self.ensure_bytes(key) - self._kv[key] = value - - async def async_internal_kv_get(self, key: Union[bytes, str], **kwargs): - key = self.ensure_bytes(key) - return self._kv.get(key, None) - - async def async_internal_kv_multi_get( - self, keys: List[Union[bytes, str]], **kwargs - ): - return {key: self.internal_kv_get(key) for key in keys} - - async def async_internal_kv_del(self, key: Union[bytes, str], **kwargs): - key = self.ensure_bytes(key) - self._kv.pop(key) - - async def async_internal_kv_keys(self, prefix: Union[bytes, str], **kwargs): - prefix = self.ensure_bytes(prefix) - return [key for key in self._kv.keys() if key.startswith(prefix)] - - class MockJobHead(JobHead): - def __init__(self): - self._agents = dict() - self._gcs_client = _FakeGcsClient() - - @property - def gcs_client(self): - # Overrides JobHead.gcs_client - return self._gcs_client - - job_head = MockJobHead() - job_head._gcs_client = _FakeGcsClient() - - async def add_agent(agent): - node_id = agent[0] - node_ip = agent[1]["ipAddress"] - http_port = agent[1]["httpPort"] - grpc_port = agent[1]["grpcPort"] - - await job_head._gcs_client.async_internal_kv_put( - f"{DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX}{node_id.hex()}".encode(), - json.dumps([node_ip, http_port, grpc_port]).encode(), - namespace=ray_constants.KV_NAMESPACE_DASHBOARD, - ) - await job_head._gcs_client.async_internal_kv_put( - f"{DASHBOARD_AGENT_ADDR_IP_PREFIX}{node_ip}".encode(), - json.dumps([node_id.hex(), http_port, grpc_port]).encode(), - namespace=ray_constants.KV_NAMESPACE_DASHBOARD, - ) - - async def del_agent(agent): - node_id = agent[0] - node_ip = agent[1]["ipAddress"] - await job_head._gcs_client.async_internal_kv_del( - f"{DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX}{node_id.hex()}".encode(), - namespace=ray_constants.KV_NAMESPACE_DASHBOARD, - ) - await job_head._gcs_client.async_internal_kv_del( - f"{DASHBOARD_AGENT_ADDR_IP_PREFIX}{node_ip}".encode(), - namespace=ray_constants.KV_NAMESPACE_DASHBOARD, - ) - - head_node_id = NodeID.from_random() - await job_head._gcs_client.async_internal_kv_put( - ray_constants.KV_HEAD_NODE_ID_KEY, - head_node_id.hex().encode(), - namespace=ray_constants.KV_NAMESPACE_JOB, - ) - - agent_1 = ( - head_node_id, - dict( - ipAddress="1.1.1.1", - httpPort=1, - grpcPort=1, - httpAddress="1.1.1.1:1", - ), - ) - agent_2 = ( - NodeID.from_random(), - dict( - ipAddress="2.2.2.2", - httpPort=2, - grpcPort=2, - httpAddress="2.2.2.2:2", - ), - ) - agent_3 = ( - NodeID.from_random(), - dict( - ipAddress="3.3.3.3", - httpPort=3, - grpcPort=3, - httpAddress="3.3.3.3:3", - ), - ) - - # Disable Head-node routing for the Ray job critical ops (enabling - # random agent sampling) - monkeypatch.setattr( - f"{JobHead.__module__}.RAY_JOB_AGENT_USE_HEAD_NODE_ONLY", False - ) - - # Check only 1 agent present, only agent being returned - await add_agent(agent_1) - job_agent_client = await job_head.get_target_agent() - assert job_agent_client._agent_address == "http://1.1.1.1:1" - - # Remove only agent, no agents present, should time out - await del_agent(agent_1) - with pytest.raises(asyncio.TimeoutError): - await asyncio.wait_for(job_head.get_target_agent(), timeout=3) - - # Enable Head-node routing for the Ray job critical ops (disabling - # random agent sampling) - monkeypatch.setattr( - f"{JobHead.__module__}.RAY_JOB_AGENT_USE_HEAD_NODE_ONLY", True - ) - - # Add 3 agents - await add_agent(agent_1) - await add_agent(agent_2) - await add_agent(agent_3) - - # Make sure returned agent is a head-node - # NOTE: We run 3 tims to make sure we're not hitting branch probabilistically - for _ in range(3): - job_agent_client = await job_head.get_target_agent() - assert job_agent_client._agent_address == "http://1.1.1.1:1" - - # Disable Head-node routing for the Ray job critical ops (enabling - # random agent sampling) - monkeypatch.setattr( - f"{JobHead.__module__}.RAY_JOB_AGENT_USE_HEAD_NODE_ONLY", False - ) - - # Theoretically, the probability of failure is 1/3^100 - addresses_1 = set() - for address in range(100): - job_agent_client = await job_head.get_target_agent() - addresses_1.add(job_agent_client._agent_address) - assert len(addresses_1) == 2 - addresses_2 = set() - for address in range(100): - job_agent_client = await job_head.get_target_agent() - addresses_2.add(job_agent_client._agent_address) - assert len(addresses_2) == 2 and addresses_1 == addresses_2 - - for agent in [agent_1, agent_2, agent_3]: - if f"http://{agent[1]['httpAddress']}" in addresses_2: - break - await del_agent(agent) - - # Theoretically, the probability of failure is 1/2^100 - addresses_3 = set() - for address in range(100): - job_agent_client = await job_head.get_target_agent() - addresses_3.add(job_agent_client._agent_address) - assert len(addresses_3) == 2 - assert addresses_2 - addresses_3 == {f"http://{agent[1]['httpAddress']}"} - addresses_4 = set() - for address in range(100): - job_agent_client = await job_head.get_target_agent() - addresses_4.add(job_agent_client._agent_address) - assert addresses_4 == addresses_3 - - for agent in [agent_1, agent_2, agent_3]: - if f"http://{agent[1]['httpAddress']}" in addresses_4: - break - await del_agent(agent) - address = None - for _ in range(3): - job_agent_client = await job_head.get_target_agent() - assert address is None or address == job_agent_client._agent_address - address = job_agent_client._agent_address - - @pytest.mark.asyncio async def test_get_upload_package(ray_start_context, tmp_path): assert wait_until_server_available(ray_start_context["webui_url"]) diff --git a/python/ray/dashboard/modules/job/tests/test_sdk.py b/python/ray/dashboard/modules/job/tests/test_sdk.py index 41507e3530de..cc7ec6bf0f90 100644 --- a/python/ray/dashboard/modules/job/tests/test_sdk.py +++ b/python/ray/dashboard/modules/job/tests/test_sdk.py @@ -12,7 +12,6 @@ from ray._common.test_utils import wait_for_condition import ray.experimental.internal_kv as kv from ray._private.ray_constants import ( - DEFAULT_DASHBOARD_AGENT_LISTEN_PORT, KV_NAMESPACE_DASHBOARD, ) from ray._private.test_utils import ( @@ -37,8 +36,6 @@ from ray.tests.conftest import _ray_start from ray.util.state import list_nodes -import psutil - def _check_job_succeeded(client: JobSubmissionClient, job_id: str) -> bool: status = client.get_job_status(job_id) @@ -166,13 +163,6 @@ def test_temporary_uri_reference(monkeypatch, expiration_s): print("Internal KV was GC'ed at time ", time.time() - start) -@pytest.fixture -def mock_candidate_number(): - os.environ["CANDIDATE_AGENT_NUMBER"] = "2" - yield - os.environ.pop("CANDIDATE_AGENT_NUMBER", None) - - def get_register_agents_number(gcs_client): keys = gcs_client.internal_kv_keys( prefix=DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX, @@ -188,124 +178,46 @@ def get_register_agents_number(gcs_client): { "include_dashboard": True, "env_vars": { - "CANDIDATE_AGENT_NUMBER": "2", RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR: "1", "RAY_health_check_initial_delay_ms": "0", "RAY_health_check_period_ms": "1000", - "RAY_JOB_AGENT_USE_HEAD_NODE_ONLY": "0", }, } ], indirect=True, ) -def test_job_head_choose_job_agent_E2E(ray_start_cluster_head_with_env_vars): +def test_head_node_job_agent_always_used(ray_start_cluster_head_with_env_vars): + """Makes sure that job submission always uses the head node's job agent. + + 1. Create a cluster with a worker node and a head node. + 2. Submit 10 jobs. + 3. Make sure they all execute on the head node's job agent. + """ cluster = ray_start_cluster_head_with_env_vars assert wait_until_server_available(cluster.webui_url) is True webui_url = cluster.webui_url webui_url = format_web_url(webui_url) client = JobSubmissionClient(webui_url) - gcs_client = GcsClient(address=cluster.gcs_address) - def submit_job_and_wait_finish(): - submission_id = client.submit_job(entrypoint="echo hello") - - wait_for_condition( - _check_job_succeeded, client=client, job_id=submission_id, timeout=30 - ) + cluster_nodes = cluster.list_all_nodes() + assert len(cluster_nodes) == 1 and cluster_nodes[0].is_head + head_node_id = cluster_nodes[0].node_id - head_http_port = DEFAULT_DASHBOARD_AGENT_LISTEN_PORT - worker_1_http_port = 52366 - cluster.add_node(dashboard_agent_listen_port=worker_1_http_port) - wait_for_condition(lambda: get_register_agents_number(gcs_client) == 2, timeout=20) - assert len(cluster.worker_nodes) == 1 - node_try_to_kill = list(cluster.worker_nodes)[0] - - def make_sure_worker_node_run_job(port): - actors = ray.state.actors() - - def _kill_all_driver(): - for _, actor_info in actors.items(): - if actor_info["State"] != "ALIVE": - continue - if actor_info["Name"].startswith("_ray_internal_job_actor"): - proc = psutil.Process(actor_info["Pid"]) - try: - proc.kill() - except Exception: - pass + # add a worker node. + cluster.add_node() - try: - for _, actor_info in actors.items(): - if actor_info["State"] != "ALIVE": - continue - if actor_info["Name"].startswith("_ray_internal_job_actor"): - proc = psutil.Process(actor_info["Pid"]) - parent_proc = proc.parent() - if f"--listen-port={port}" in " ".join(parent_proc.cmdline()): - _kill_all_driver() - return True - except Exception as ex: - print("Got exception:", ex) - raise - client.submit_job(entrypoint="sleep 3600") - return False - - # Make `list(cluster.worker_nodes)[0]` and head node called at least once - wait_for_condition( - lambda: make_sure_worker_node_run_job(worker_1_http_port), timeout=60 - ) - wait_for_condition( - lambda: make_sure_worker_node_run_job(head_http_port), timeout=60 - ) + job_ids = [client.submit_job(entrypoint="echo hello")] - worker_2_http_port = 52367 - cluster.add_node(dashboard_agent_listen_port=worker_2_http_port) - wait_for_condition(lambda: get_register_agents_number(gcs_client) == 3, timeout=20) + for job_id in job_ids: + wait_for_condition( + _check_job_succeeded, client=client, job_id=job_id, timeout=30 + ) - # The third `JobAgent` will not be called here. - submit_job_and_wait_finish() - submit_job_and_wait_finish() - submit_job_and_wait_finish() - - def get_all_new_supervisor_actor_info(old_supervisor_actor_ids): - all_actors = ray.state.state.actor_table(None) - res = dict() - for actor_id, actor_info in all_actors.items(): - if actor_id in old_supervisor_actor_ids: - continue - if not actor_info["Name"].startswith("_ray_internal_job_actor"): - continue - res[actor_id] = actor_info - return res - - old_supervisor_actor_ids = set() - new_supervisor_actor = get_all_new_supervisor_actor_info(old_supervisor_actor_ids) - new_owner_port = set() - for actor_id, actor_info in new_supervisor_actor.items(): - old_supervisor_actor_ids.add(actor_id) - new_owner_port.add(actor_info["OwnerAddress"]["Port"]) - - assert len(new_owner_port) == 2 - old_owner_port = new_owner_port - - node_try_to_kill.kill_raylet() - - # make sure the head updates the info of the dead node. - wait_for_condition(lambda: get_register_agents_number(gcs_client) == 2, timeout=20) - - # Make sure the third JobAgent will be called here. - wait_for_condition( - lambda: make_sure_worker_node_run_job(worker_2_http_port), timeout=60 - ) + actors = ray.state.actors() - new_supervisor_actor = get_all_new_supervisor_actor_info(old_supervisor_actor_ids) - new_owner_port = set() - for actor_id, actor_info in new_supervisor_actor.items(): - old_supervisor_actor_ids.add(actor_id) - new_owner_port.add(actor_info["OwnerAddress"]["Port"]) - assert len(new_owner_port) == 2 - assert len(old_owner_port - new_owner_port) == 1 - assert len(new_owner_port - old_owner_port) == 1 + for _, actor_info in actors.items(): + if actor_info["Name"].startswith("_ray_internal_job_actor"): + assert actor_info["Address"]["NodeID"] == head_node_id @pytest.mark.parametrize( From f53e38b119ab19c27db6f32d76710a3dd8c6e9c1 Mon Sep 17 00:00:00 2001 From: tannerdwood <71387269+tannerdwood@users.noreply.github.com> Date: Tue, 19 Aug 2025 08:44:44 -0700 Subject: [PATCH 177/634] [Core] Update DLAMI Information in aws.md (#55702) Signed-off-by: Tanner Wood Co-authored-by: Tanner Wood --- .../vms/user-guides/launching-clusters/aws.md | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/source/cluster/vms/user-guides/launching-clusters/aws.md b/doc/source/cluster/vms/user-guides/launching-clusters/aws.md index 3f7a3f7aba08..20d8f3298e23 100644 --- a/doc/source/cluster/vms/user-guides/launching-clusters/aws.md +++ b/doc/source/cluster/vms/user-guides/launching-clusters/aws.md @@ -154,7 +154,7 @@ CloudWatch integration with Ray requires an AMI (or Docker image) with the Unifi AMIs with the Unified CloudWatch Agent pre-installed are provided by the Amazon Ray Team, and are currently available in the us-east-1, us-east-2, us-west-1, and us-west-2 regions. Please direct any questions, comments, or issues to the `Amazon Ray Team `_. -The table below lists AMIs with the Unified CloudWatch Agent pre-installed in each region, and you can also find AMIs at `amazon-ray README `_. +The table below lists AMIs with the Unified CloudWatch Agent pre-installed in each region, and you can also find AMIs at `DLAMI Release Notes `_. Each DLAMI (Deep Learning AMI) is pre-installed with the Unified CloudWatch Agent, and its corresponding release notes include AWS CLI commands to query the latest AMI ID. .. list-table:: All available unified CloudWatch agent images @@ -162,22 +162,22 @@ The table below lists AMIs with the Unified CloudWatch Agent pre-installed in ea - AMI ID - Region - Unified CloudWatch Agent Version - * - AWS Deep Learning AMI (Ubuntu 18.04, 64-bit) - - ami-069f2811478f86c20 + * - AWS Deep Learning AMI (Ubuntu 24.04, 64-bit) + - ami-087feac195f30e722 - us-east-1 - - v1.247348.0b251302 - * - AWS Deep Learning AMI (Ubuntu 18.04, 64-bit) - - ami-058cc0932940c2b8b + - v1.300057.1b1167 + * - AWS Deep Learning AMI (Ubuntu 24.04, 64-bit) + - ami-0ed6c422a7c93278a - us-east-2 - - v1.247348.0b251302 - * - AWS Deep Learning AMI (Ubuntu 18.04, 64-bit) - - ami-044f95c9ef12883ef + - v1.300057.1b1167 + * - AWS Deep Learning AMI (Ubuntu 24.04, 64-bit) + - ami-0c5ddf2c101267018 - us-west-1 - - v1.247348.0b251302 - * - AWS Deep Learning AMI (Ubuntu 18.04, 64-bit) - - ami-0d88d9cbe28fac870 + - v1.300057.1b1167 + * - AWS Deep Learning AMI (Ubuntu 24.04, 64-bit) + - ami-0cfd95c6c87d00570 - us-west-2 - - v1.247348.0b251302 + - v1.300057.1b1167 .. note:: @@ -213,12 +213,12 @@ Getting started ray.head.default: node_config: InstanceType: c5a.large - ImageId: ami-0d88d9cbe28fac870 # Unified CloudWatch agent pre-installed AMI, us-west-2 + ImageId: ami-0cfd95c6c87d00570 # Unified CloudWatch agent pre-installed AMI, us-west-2 resources: {} ray.worker.default: node_config: InstanceType: c5a.large - ImageId: ami-0d88d9cbe28fac870 # Unified CloudWatch agent pre-installed AMI, us-west-2 + ImageId: ami-0cfd95c6c87d00570 # Unified CloudWatch agent pre-installed AMI, us-west-2 IamInstanceProfile: Name: ray-autoscaler-cloudwatch-v1 resources: {} @@ -275,11 +275,11 @@ The following CLI command returns the latest available Unified CloudWatch Agent ray.head.default: node_config: InstanceType: c5a.large - ImageId: ami-0d88d9cbe28fac870 + ImageId: ami-0cfd95c6c87d00570 ray.worker.default: node_config: InstanceType: c5a.large - ImageId: ami-0d88d9cbe28fac870 + ImageId: ami-0cfd95c6c87d00570 To build your own AMI with the Unified CloudWatch Agent installed: From a86bb60df41987bfee65b227fcce69a7eee44b9e Mon Sep 17 00:00:00 2001 From: Justin Yu Date: Tue, 19 Aug 2025 08:58:10 -0700 Subject: [PATCH 178/634] [core] Fix actor import error message for async actors (#55722) ## Summary When the Ray actor class fails to import upon actor creation, we create a TemporaryActor in its place to emit an error message. However, for async actors, the TemporaryActor creation fails to initialize due having no async methods. This PR adds a dummy async method to handle this case. ## Example error ```python Traceback (most recent call last): File "", line 1, in File "", line 35, in File "/Users/justin/Developer/ray/python/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper return fn(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^ File "/Users/justin/Developer/ray/python/ray/_private/client_mode_hook.py", line 104, in wrapper return func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ File "/Users/justin/Developer/ray/python/ray/_private/worker.py", line 2896, in get values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/justin/Developer/ray/python/ray/_private/worker.py", line 970, in get_objects raise value ray.exceptions.ActorDiedError: The actor died because of an error raised in its creation task, ray::Foo.__init__() (pid=42078, ip=127.0.0.1, actor_id=7000b00899a3a8b1d05bbdc601000000, repr=<__main__.FunctionActorManager._create_fake_actor_class..TemporaryActor object at 0x10732dc10>) ray.exceptions.ActorDiedError: The actor died unexpectedly before finishing this task. class_name: TemporaryActor actor_id: 7000b00899a3a8b1d05bbdc601000000 Failed to create actor. You set the async flag, but the actor does not have any coroutine functions. (TemporaryActor pid=42078) The original cause of the RayTaskError () isn't serializable: cannot pickle 'google._upb._message.Descriptor' object. Overwriting the cause to a RayError. ``` --------- Signed-off-by: Justin Yu --- python/ray/_private/function_manager.py | 6 +++++- python/ray/tests/test_output.py | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/python/ray/_private/function_manager.py b/python/ray/_private/function_manager.py index 854a50249d0a..76d98d536d62 100644 --- a/python/ray/_private/function_manager.py +++ b/python/ray/_private/function_manager.py @@ -600,7 +600,11 @@ def _create_fake_actor_class( self, actor_class_name, actor_method_names, traceback_str ): class TemporaryActor: - pass + async def __dummy_method(self): + """Dummy method for this fake actor class to work for async actors. + Without this method, this temporary actor class fails to initialize + if the original actor class was async.""" + pass def temporary_actor_method(*args, **kwargs): raise RuntimeError( diff --git a/python/ray/tests/test_output.py b/python/ray/tests/test_output.py index 1ab0d893e889..a753040931bf 100644 --- a/python/ray/tests/test_output.py +++ b/python/ray/tests/test_output.py @@ -351,8 +351,9 @@ def _check_events(): @pytest.mark.skipif(sys.platform == "win32", reason="Failing on Windows.") -def test_fail_importing_actor(): - script = """ +@pytest.mark.parametrize("async_actor", [True, False]) +def test_fail_importing_actor(async_actor): + script = f""" import os import sys import tempfile @@ -380,7 +381,7 @@ class Foo: def __init__(self): self.x = module.temporary_python_file() - def ready(self): + {"async " if async_actor else ""}def ready(self): pass finally: os.unlink(f.name) From 7834d12c586a2373794935a5863dcc84145b8ade Mon Sep 17 00:00:00 2001 From: avigyabb <98926738+avigyabb@users.noreply.github.com> Date: Tue, 19 Aug 2025 10:36:27 -0700 Subject: [PATCH 179/634] GPU ObjectRef w/in data structures validation before serialization (#55027) Right now we only allow passing GPU ObjectRefs as direct arguments. Passing the ObjectRefs inside of a Python data structure like a list and then calling ray.get on the other side will error. We should throw an exception for now until this is supported. Added validation for list objects in worker.py that there are no GPU object ref elements. Closes #54997 --------- Signed-off-by: avigyabb Signed-off-by: avigyabb <98926738+avigyabb@users.noreply.github.com> Co-authored-by: Stephanie Wang --- python/ray/_private/serialization.py | 11 +++++++++ python/ray/tests/test_gpu_objects_gloo.py | 30 +++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/python/ray/_private/serialization.py b/python/ray/_private/serialization.py index 077aa90c3f57..81b502882e28 100644 --- a/python/ray/_private/serialization.py +++ b/python/ray/_private/serialization.py @@ -161,6 +161,17 @@ def compiled_dag_ref_reducer(obj): def object_ref_reducer(obj): worker = ray._private.worker.global_worker worker.check_connected() + + # Check if this is a GPU ObjectRef being serialized inside a collection + if ( + self.is_in_band_serialization() + and worker.gpu_object_manager.is_managed_object(obj.hex()) + ): + raise ValueError( + "Passing GPU ObjectRefs inside data structures is not yet supported. " + "Pass GPU ObjectRefs directly as task arguments instead. For example, use `foo.remote(ref)` instead of `foo.remote([ref])`." + ) + self.add_contained_object_ref( obj, allow_out_of_band_serialization=( diff --git a/python/ray/tests/test_gpu_objects_gloo.py b/python/ray/tests/test_gpu_objects_gloo.py index 9a771eb2f052..ff0c233a43bb 100644 --- a/python/ray/tests/test_gpu_objects_gloo.py +++ b/python/ray/tests/test_gpu_objects_gloo.py @@ -494,6 +494,36 @@ def test_tensor_extracted_from_tensordict_in_gpu_object_store(ray_start_regular) assert torch.equal(ret_val_src[1], td["reward"]) +def test_gpu_object_ref_in_list_throws_exception(ray_start_regular): + """Test that passing GPU ObjectRefs inside lists as task arguments raises an error.""" + + print("loc2") + actor = GPUTestActor.remote() + create_collective_group([actor], backend="torch_gloo") + + tensor = torch.randn((1,)) + + # Test: GPU ref passed directly to task should work + gpu_ref = actor.echo.remote(tensor) + result = actor.double.remote(gpu_ref) + assert ray.get(result) == pytest.approx(tensor * 2) + + # Test: GPU ref inside a list should fail during task submission + with pytest.raises( + ValueError, + match="Passing GPU ObjectRefs inside data structures is not yet supported", + ): + actor.double.remote([gpu_ref]) + + # Test: Mixed list with GPU ref and normal data should also fail + normal_ref = ray.put("normal_data") + with pytest.raises( + ValueError, + match="Passing GPU ObjectRefs inside data structures is not yet supported", + ): + actor.double.remote([gpu_ref, normal_ref]) + + def test_app_error_inter_actor(ray_start_regular): world_size = 2 actors = [GPUTestActor.remote() for _ in range(world_size)] From 769a32e1ad7ba023d8fd9b194e8a4d6d73ac9273 Mon Sep 17 00:00:00 2001 From: Ricardo Decal Date: Tue, 19 Aug 2025 11:47:46 -0700 Subject: [PATCH 180/634] [Benchmark] Upstreaming text embeddings benchmarks (#54912) ## Why are these changes needed? Upstreams the text embeddings benchmarks to fill out dashboards ## Related issue number NA ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Ricardo Decal --- .../dataset/autoscaling_gpu_g6e_2xl_aws.yaml | 18 ++ .../dataset/fixed_size_gpu_g6e_2xl_aws.yaml | 18 ++ .../dataset/text_embeddings_benchmark.py | 196 ++++++++++++++++++ .../byod/requirements_llm_byod_3.11.txt | 2 + release/release_tests.yaml | 49 +++++ 5 files changed, 283 insertions(+) create mode 100644 release/nightly_tests/dataset/autoscaling_gpu_g6e_2xl_aws.yaml create mode 100644 release/nightly_tests/dataset/fixed_size_gpu_g6e_2xl_aws.yaml create mode 100644 release/nightly_tests/dataset/text_embeddings_benchmark.py diff --git a/release/nightly_tests/dataset/autoscaling_gpu_g6e_2xl_aws.yaml b/release/nightly_tests/dataset/autoscaling_gpu_g6e_2xl_aws.yaml new file mode 100644 index 000000000000..d8ca5be2d561 --- /dev/null +++ b/release/nightly_tests/dataset/autoscaling_gpu_g6e_2xl_aws.yaml @@ -0,0 +1,18 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +head_node_type: + name: head_node + instance_type: m5.2xlarge + +worker_node_types: + - name: 1xL40S_8CPU_64GB + instance_type: g6e.2xlarge + max_workers: 15 + min_workers: 0 + use_spot: false + - name: 16CPU_64GB + instance_type: m5.4xlarge + max_workers: 20 + min_workers: 0 + use_spot: false diff --git a/release/nightly_tests/dataset/fixed_size_gpu_g6e_2xl_aws.yaml b/release/nightly_tests/dataset/fixed_size_gpu_g6e_2xl_aws.yaml new file mode 100644 index 000000000000..bcb5da42d911 --- /dev/null +++ b/release/nightly_tests/dataset/fixed_size_gpu_g6e_2xl_aws.yaml @@ -0,0 +1,18 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +head_node_type: + name: head_node + instance_type: m5.2xlarge + +worker_node_types: + - name: 1xL40S_8CPU_64GB + instance_type: g6e.2xlarge + max_workers: 15 + min_workers: 15 + use_spot: false + - name: 16CPU_64GB + instance_type: m5.4xlarge + max_workers: 20 + min_workers: 20 + use_spot: false diff --git a/release/nightly_tests/dataset/text_embeddings_benchmark.py b/release/nightly_tests/dataset/text_embeddings_benchmark.py new file mode 100644 index 000000000000..e2fbccff429d --- /dev/null +++ b/release/nightly_tests/dataset/text_embeddings_benchmark.py @@ -0,0 +1,196 @@ +""" +Benchmark a text embeddings job +""" + +import argparse +import uuid +import time +from typing import Dict, List +from numpy import ndarray + +import ray +import torch +from sentence_transformers import SentenceTransformer +from langchain_text_splitters import ( + RecursiveCharacterTextSplitter, + CharacterTextSplitter, +) + +from benchmark import Benchmark, BenchmarkMetric + +# Subset of the data so that benchmark completes in ~20 minutes. +DEFAULT_SOURCE_DIRECTORY_S3 = "s3://air-example-data/common-pile-mirror/arxiv_papers/arxiv_papers-train-00001-of-00042.parquet" +# Add a random prefix to avoid conflicts between different runs. +WRITE_PATH = f"s3://ray-data-write-benchmark/{uuid.uuid4().hex}/" + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Text Embeddings Batch Inference Benchmark" + ) + parser.add_argument( + "--source-directory", + type=str, + default=DEFAULT_SOURCE_DIRECTORY_S3, + help="S3 URI of source documents", + ) + parser.add_argument( + "--chunk-concurrency", + type=int, + default=20, + help="Concurrency for Chunker stage", + ) + parser.add_argument( + "--chunk-cpus", type=int, default=None, help="Number of CPUs per Chunker" + ) + parser.add_argument( + "--chunk-method", + choices=["fixed", "recursive"], + default="recursive", + help="Chunking method", + ) + parser.add_argument( + "--chunk-size", type=int, default=1200, help="Chunk size for text splitting" + ) + parser.add_argument( + "--chunk-overlap", + type=int, + default=100, + help="Number of overlapping boundary characters between text chunks.", + ) + parser.add_argument( + "--embed-batch-size", + type=int, + default=256, + help="Batch size for embedding inference", + ) + parser.add_argument( + "--embed-concurrency", + type=int, + default=15, + help="Number of Embedder replicas", + ) + parser.add_argument( + "--num-gpus", type=int, default=1, help="Number of GPUs per Embedder" + ) + parser.add_argument( + "--model-name", + type=str, + default="Salesforce/SFR-Embedding-Code-400M_R", + help="Embedding model name", + ) + parser.add_argument( + "--smoke-test", + action="store_true", + help="Runs a smoke test with a small subset of the data", + ) + parser.add_argument( + "--chaos-test", + action="store_true", + default=False, + help="Enable chaos testing to simulate node failures", + ) + return parser.parse_args() + + +class Chunker: + def __init__(self, method: str, chunk_size: int, chunk_overlap: int): + if method == "fixed": + self.splitter = CharacterTextSplitter( + chunk_size=chunk_size, chunk_overlap=chunk_overlap + ) + else: + self.splitter = RecursiveCharacterTextSplitter( + chunk_size=chunk_size, chunk_overlap=chunk_overlap + ) + + def __call__(self, page: Dict) -> List[Dict]: + return [ + { + "text": text, + "source": page["source"], + "chunk_id": f"{page['id']}_{str(uuid.uuid4())}", + "doc_id": page["id"], + } + for text in self.splitter.split_text(page["text"]) + ] + + +class Embedder: + def __init__(self, model_name: str): + self.model = SentenceTransformer( + model_name, + device="cuda" if torch.cuda.is_available() else "cpu", + trust_remote_code=True, + ) + + def __call__(self, batch: Dict[str, ndarray]) -> Dict[str, ndarray]: + batch["embeddings"] = self.model.encode( + batch["text"], convert_to_numpy=True, batch_size=len(batch["text"]) + ) + return batch + + +def main(args): + start_time = time.time() + ds = ray.data.read_parquet( + args.source_directory, + include_paths=True, + ) + metadata_fetch_end = time.time() + metadata_fetching_s = metadata_fetch_end - start_time + if args.smoke_test: + ds = ds.limit(100) + + ds = ds.flat_map( + Chunker( + method=args.chunk_method, + chunk_size=args.chunk_size, + chunk_overlap=args.chunk_overlap, + ), + concurrency=args.chunk_concurrency, + num_cpus=args.chunk_cpus, + ) + ds = ds.map_batches( + Embedder, + fn_constructor_kwargs={"model_name": args.model_name}, + batch_size=args.embed_batch_size, + concurrency=args.embed_concurrency, + num_gpus=args.num_gpus, + ) + ds.write_parquet(WRITE_PATH, num_rows_per_file=5_000) + end_time = time.time() + runtime_s = end_time - start_time + num_rows = ray.data.read_parquet(WRITE_PATH).count() + throughput_rows_s = num_rows / runtime_s + + # Compute metrics for time and throughput without metadata fetch + runtime_s_wo_metadata_fetch = end_time - metadata_fetch_end + throughput_rows_s_wo_metadata_fetch = num_rows / runtime_s_wo_metadata_fetch + + # Report chaos testing node failures + if args.chaos_test: + dead_nodes = [node["NodeID"] for node in ray.nodes() if not node["Alive"]] + assert dead_nodes, "No dead nodes during chaos test" + print(f"Total chaos killed: {dead_nodes}") + + return { + BenchmarkMetric.RUNTIME: runtime_s, + BenchmarkMetric.NUM_ROWS: num_rows, + BenchmarkMetric.THROUGHPUT: throughput_rows_s, + "source_directory": args.source_directory, + "model_name": args.model_name, + "chunk_method": args.chunk_method, + "metadata_fetching_s": metadata_fetching_s, + "runtime_s_wo_metadata_fetch": runtime_s_wo_metadata_fetch, + "throughput_rows_s_wo_metadata_fetch": throughput_rows_s_wo_metadata_fetch, + "chaos_test": args.chaos_test, + } + + +if __name__ == "__main__": + args = parse_args() + print(f"Writing to {WRITE_PATH}") + benchmark = Benchmark() + benchmark.run_fn("text-embeddings-benchmark", main, args) + benchmark.write_result() diff --git a/release/ray_release/byod/requirements_llm_byod_3.11.txt b/release/ray_release/byod/requirements_llm_byod_3.11.txt index 2bf87ef15bed..e2558df8f389 100644 --- a/release/ray_release/byod/requirements_llm_byod_3.11.txt +++ b/release/ray_release/byod/requirements_llm_byod_3.11.txt @@ -3,3 +3,5 @@ pytest-timeout==2.1.0 locust==2.33.0 orjson==3.10.15 backoff==2.2.1 +langchain_text_splitters==0.3.9 +sentence-transformers==5.1.0 diff --git a/release/release_tests.yaml b/release/release_tests.yaml index df48adcc55d7..d62589e6f1b9 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -4604,6 +4604,55 @@ pytest -sv test_batch_sglang.py +- name: text_embeddings_benchmark_{{scaling}} + frequency: nightly + python: "3.11" # necessary for the llm-cu128 image + working_dir: nightly_tests + team: data + + cluster: + byod: + type: llm-cu128 + cluster_compute: dataset/{{scaling}}_gpu_g6e_2xl_aws.yaml + + matrix: + setup: + scaling: [fixed_size, autoscaling] + + run: + timeout: 3600 + script: > + python dataset/text_embeddings_benchmark.py --embed-concurrency 15 + +# Note: release tests do not support specifying both 'matrix' and 'variations' +# in a test definition, so split off preemptible tests here. +- name: text_embeddings_benchmark_{{scaling}}_preemptible + frequency: nightly + python: "3.11" + working_dir: nightly_tests + team: data + + cluster: + byod: + type: llm-cu128 + cluster_compute: dataset/{{scaling}}_gpu_g6e_2xl_aws.yaml + + matrix: + setup: + scaling: [fixed_size, autoscaling] + + run: + timeout: 3600 + # Notes: + # - Not using true spot instances. We simulate spot preemption using TerminateEC2InstanceWithGracePeriod to soft-kill the workers. This is so that we can + # control the kill schedule. + # - Batch size is always fixed, so kill schedule is deterministic. + prepare: > + python setup_chaos.py --chaos TerminateEC2InstanceWithGracePeriod + --batch-size-to-kill 5 --max-to-kill 15 --kill-delay 30 --kill-interval 100 + script: > + python dataset/text_embeddings_benchmark.py --chaos-test --embed-concurrency 15 + ####################### # Ray examples tests ####################### From 2647c9bd2b2cef0728a0b4f96e2fc83e4bb94c02 Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Tue, 19 Aug 2025 12:05:54 -0700 Subject: [PATCH 181/634] [data] data metrics grouping (#55495) https://github.com/user-attachments/assets/8caa7448-35b1-4945-9e41-82fd9efca4f3 ## Why are these changes needed? - ray data dashboard is ugly - grouping them into pending inputs, inputs, outputs, overview, pending outputs, scheduling loop, resource usage/budget, and iteration - also changed the metric from external outqueue of op1 to external inqueue of op2 (so that i can combine the internal + external inqueues easily) ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- .../dashboards/data_dashboard_panels.py | 1640 +++++++++-------- .../interfaces/op_runtime_metrics.py | 8 +- .../execution/streaming_executor_state.py | 5 +- python/ray/data/tests/test_stats.py | 27 +- 4 files changed, 941 insertions(+), 739 deletions(-) diff --git a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py index b763d30ca950..f73736e3c56c 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py @@ -4,6 +4,7 @@ DashboardConfig, Panel, Target, + Row, ) # When adding a new panels for an OpRuntimeMetric, follow this format: @@ -15,7 +16,7 @@ # targets=[ # Target( # expr=f"sum(ray_data_{metric.name}" -# + "{{{global_filters}}}) by (dataset, operator)", +# + "{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)", # legend=legend, # ) # ], @@ -24,738 +25,931 @@ # ) -DATA_GRAFANA_PANELS = [ - # Ray Data Metrics (Overview) - Panel( - id=1, - title="Bytes Spilled", - description="Amount spilled by dataset operators. DataContext.enable_get_object_locations_for_metrics must be set to True to report this metric", - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_spilled_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Bytes Spilled: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=3, - title="Bytes Freed", - description="Amount freed by dataset operators.", - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_freed_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Bytes Freed: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=4, - title="Object Store Memory", - description="Amount of memory store used by dataset operators.", - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_current_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Current Usage: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=5, - title="CPUs (logical slots)", - description="Logical CPUs allocated to dataset operators.", - unit="cores", - targets=[ - Target( - expr='sum(ray_data_cpu_usage_cores{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="CPU Usage: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=6, - title="GPUs (logical slots)", - description="Logical GPUs allocated to dataset operators.", - unit="cores", - targets=[ - Target( - expr='sum(ray_data_gpu_usage_cores{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="GPU Usage: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=7, - title="Bytes Output / Second", - description="Bytes output per second by dataset operators.", - unit="Bps", - targets=[ - Target( - expr='sum(rate(ray_data_output_bytes{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Bytes Output / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=55, - title="Max Bytes to Read", - description="Maximum bytes to read from streaming generator buffer.", - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_max_bytes_to_read{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Max Bytes to Read: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=11, - title="Throughput (Rows Output / Second)", - description="Total rows output per second by dataset operators.", - unit="rows/sec", - targets=[ - Target( - expr='sum(rate(ray_data_output_rows{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Rows Output / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - # Ray Data Metrics (Inputs) - Panel( - id=17, - title="Input Blocks Received by Operator / Second", - description="Number of input blocks received by operator per second.", - unit="blocks/sec", - targets=[ - Target( - expr='sum(rate(ray_data_num_inputs_received{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Blocks Received / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=18, - title="Input Bytes Received by Operator / Second", - description="Byte size of input blocks received by operator per second.", - unit="Bps", - targets=[ - Target( - expr='sum(rate(ray_data_bytes_inputs_received{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Bytes Received / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=19, - title="Input Blocks Processed by Tasks / Second", - description=( - "Number of input blocks that operator's tasks have finished processing per second." - ), - unit="blocks/sec", - targets=[ - Target( - expr='sum(rate(ray_data_num_task_inputs_processed{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Blocks Processed / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=20, - title="Input Bytes Processed by Tasks / Second", - description=( - "Byte size of input blocks that operator's tasks have finished processing per second." - ), - unit="Bps", - targets=[ - Target( - expr='sum(rate(ray_data_bytes_task_inputs_processed{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Bytes Processed / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=21, - title="Input Bytes Submitted to Tasks / Second", - description="Byte size of input blocks passed to submitted tasks per second.", - unit="Bps", - targets=[ - Target( - expr='sum(rate(ray_data_bytes_inputs_of_submitted_tasks{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Bytes Submitted / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=22, - title="Blocks Generated by Tasks / Second", - description="Number of output blocks generated by tasks per second.", - unit="blocks/sec", - targets=[ - Target( - expr='sum(rate(ray_data_num_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Blocks Generated / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=23, - title="Bytes Generated by Tasks / Second", - description="Byte size of output blocks generated by tasks per second.", - unit="Bps", - targets=[ - Target( - expr='sum(rate(ray_data_bytes_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Bytes Generated / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=49, - title="Average Bytes Generated / Output Block", - description="Average byte size of output blocks generated by tasks.", - unit="bytes", - targets=[ - Target( - expr='increase(ray_data_bytes_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[5m])', - legend="Average Bytes Generated / Output Block: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=50, - title="Average Number of Output Blocks / Task", - description="Average number of output blocks generated by tasks.", - unit="blocks", - targets=[ - Target( - expr='increase(ray_data_num_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', - legend="Average Number of Output Blocks / Task: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=24, - title="Rows Generated by Tasks / Second", - description="Number of rows in generated output blocks from finished tasks per second.", - unit="rows/sec", - targets=[ - Target( - expr='sum(rate(ray_data_rows_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Rows Generated / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=25, - title="Output Blocks Taken by Downstream Operators / Second", - description="Number of output blocks taken by downstream operators per second.", - unit="blocks/sec", - targets=[ - Target( - expr='sum(rate(ray_data_num_outputs_taken{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Blocks Taken / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=26, - title="Output Bytes Taken by Downstream Operators / Second", - description=( - "Byte size of output blocks taken by downstream operators per second." - ), - unit="Bps", - targets=[ - Target( - expr='sum(rate(ray_data_bytes_outputs_taken{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', - legend="Bytes Taken / Second: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=43, - title="Output Bytes from Finished Tasks / Second (by Node)", - description=( - "Byte size of output blocks from finished tasks per second, grouped by node." +# Ray Data Metrics (Overview) +BYTES_SPILLED_PANEL = Panel( + id=1, + title="Bytes Spilled", + description="Amount spilled by dataset operators. DataContext.enable_get_object_locations_for_metrics must be set to True to report this metric", + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_spilled_bytes{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Bytes Spilled: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +BYTES_FREED_PANEL = Panel( + id=3, + title="Bytes Freed", + description="Amount freed by dataset operators.", + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_freed_bytes{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Bytes Freed: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +OBJECT_STORE_MEMORY_PANEL = Panel( + id=4, + title="Object Store Memory", + description="Amount of memory store used by dataset operators.", + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_current_bytes{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Current Usage: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +CPU_USAGE_PANEL = Panel( + id=5, + title="Logical Slots Being Used (CPU)", + description="Logical CPUs currently being used by dataset operators.", + unit="cores", + targets=[ + Target( + expr='sum(ray_data_cpu_usage_cores{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="CPU Usage: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +GPU_USAGE_PANEL = Panel( + id=6, + title="Logical Slots Being Used (GPU)", + description="Logical GPUs currently being used by dataset operators.", + unit="cores", + targets=[ + Target( + expr='sum(ray_data_gpu_usage_cores{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="GPU Usage: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +BYTES_OUTPUT_PER_SECOND_PANEL = Panel( + id=7, + title="Bytes Output / Second", + description="Bytes output per second by dataset operators.", + unit="Bps", + targets=[ + Target( + expr='sum(rate(ray_data_output_bytes{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Bytes Output / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +ROWS_OUTPUT_PER_SECOND_PANEL = Panel( + id=11, + title="Rows Output / Second", + description="Total rows output per second by dataset operators.", + unit="rows/sec", + targets=[ + Target( + expr='sum(rate(ray_data_output_rows{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Rows Output / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +# Ray Data Metrics (Inputs) +INPUT_BLOCKS_RECEIVED_PANEL = Panel( + id=17, + title="Input Blocks Received by Operator / Second", + description="Number of input blocks received by operator per second.", + unit="blocks/sec", + targets=[ + Target( + expr='sum(rate(ray_data_num_inputs_received{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Blocks Received / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +INPUT_BYTES_RECEIVED_PANEL = Panel( + id=18, + title="Input Bytes Received by Operator / Second", + description="Byte size of input blocks received by operator per second.", + unit="Bps", + targets=[ + Target( + expr='sum(rate(ray_data_bytes_inputs_received{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Bytes Received / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +INPUT_BLOCKS_PROCESSED_PANEL = Panel( + id=19, + title="Input Blocks Processed by Tasks / Second", + description=( + "Number of input blocks that operator's tasks have finished processing per second." + ), + unit="blocks/sec", + targets=[ + Target( + expr='sum(rate(ray_data_num_task_inputs_processed{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Blocks Processed / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +INPUT_BYTES_PROCESSED_PANEL = Panel( + id=20, + title="Input Bytes Processed by Tasks / Second", + description=( + "Byte size of input blocks that operator's tasks have finished processing per second." + ), + unit="Bps", + targets=[ + Target( + expr='sum(rate(ray_data_bytes_task_inputs_processed{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Bytes Processed / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +INPUT_BYTES_SUBMITTED_PANEL = Panel( + id=21, + title="Input Bytes Submitted to Tasks / Second", + description="Byte size of input blocks passed to submitted tasks per second.", + unit="Bps", + targets=[ + Target( + expr='sum(rate(ray_data_bytes_inputs_of_submitted_tasks{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Bytes Submitted / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +# Ray Data Metrics (Outputs) +BLOCKS_GENERATED_PANEL = Panel( + id=22, + title="Blocks Generated by Tasks / Second", + description="Number of output blocks generated by tasks per second.", + unit="blocks/sec", + targets=[ + Target( + expr='sum(rate(ray_data_num_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Blocks Generated / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +BYTES_GENERATED_PANEL = Panel( + id=23, + title="Bytes Generated by Tasks / Second", + description="Byte size of output blocks generated by tasks per second.", + unit="Bps", + targets=[ + Target( + expr='sum(rate(ray_data_bytes_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Bytes Generated / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +ROWS_GENERATED_PANEL = Panel( + id=24, + title="Rows Generated by Tasks / Second", + description="Number of rows in generated output blocks from finished tasks per second.", + unit="rows/sec", + targets=[ + Target( + expr='sum(rate(ray_data_rows_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Rows Generated / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +OUTPUT_BLOCKS_TAKEN_PANEL = Panel( + id=25, + title="Output Blocks Taken by Downstream Operators / Second", + description="Number of output blocks taken by downstream operators per second.", + unit="blocks/sec", + targets=[ + Target( + expr='sum(rate(ray_data_num_outputs_taken{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Blocks Taken / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +OUTPUT_BYTES_TAKEN_PANEL = Panel( + id=26, + title="Output Bytes Taken by Downstream Operators / Second", + description=( + "Byte size of output blocks taken by downstream operators per second." + ), + unit="Bps", + targets=[ + Target( + expr='sum(rate(ray_data_bytes_outputs_taken{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + legend="Bytes Taken / Second: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +AVERAGE_BYTES_PER_BLOCK_PANEL = Panel( + id=49, + title="Average Bytes Generated / Output Block", + description="Average byte size of output blocks generated by tasks.", + unit="bytes", + targets=[ + Target( + expr='increase(ray_data_bytes_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[5m]) / increase(ray_data_num_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[5m])', + legend="Average Bytes Generated / Output Block: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +AVERAGE_BLOCKS_PER_TASK_PANEL = Panel( + id=50, + title="Average Number of Output Blocks / Task", + description="Average number of output blocks generated by tasks.", + unit="blocks", + targets=[ + Target( + expr='increase(ray_data_num_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters} operator=~"$Operator"}}[5m])', + legend="Average Number of Output Blocks / Task: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +OUTPUT_BYTES_BY_NODE_PANEL = Panel( + id=43, + title="Output Bytes from Finished Tasks / Second (by Node)", + description=( + "Byte size of output blocks from finished tasks per second, grouped by node." + ), + unit="Bps", + targets=[ + Target( + expr='sum(rate(ray_data_bytes_outputs_of_finished_tasks_per_node{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, node_ip)', + legend="Bytes output / Second: {{dataset}}, {{node_ip}}", + ) + ], + fill=0, + stack=False, +) + +BLOCKS_BY_NODE_PANEL = Panel( + id=48, + title="Blocks from Finished Tasks / Second (by Node)", + description=( + "Number of output blocks from finished tasks per second, grouped by node." + ), + unit="blocks/s", + targets=[ + Target( + expr='sum(rate(ray_data_blocks_outputs_of_finished_tasks_per_node{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, node_ip)', + legend="Blocks output / Second: {{dataset}}, {{node_ip}}", + ) + ], + fill=0, + stack=False, +) + +# Ray Data Metrics (Tasks) +SUBMITTED_TASKS_PANEL = Panel( + id=29, + title="Submitted Tasks", + description="Number of submitted tasks.", + unit="tasks", + targets=[ + Target( + expr='sum(ray_data_num_tasks_submitted{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Submitted Tasks: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +RUNNING_TASKS_PANEL = Panel( + id=30, + title="Running Tasks", + description="Number of running tasks.", + unit="tasks", + targets=[ + Target( + expr='sum(ray_data_num_tasks_running{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Running Tasks: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +TASKS_WITH_OUTPUT_PANEL = Panel( + id=31, + title="Tasks with output blocks", + description="Number of tasks that already have output.", + unit="tasks", + targets=[ + Target( + expr='sum(ray_data_num_tasks_have_outputs{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Tasks with output blocks: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +FINISHED_TASKS_PANEL = Panel( + id=32, + title="Finished Tasks", + description="Number of finished tasks.", + unit="tasks", + targets=[ + Target( + expr='sum(ray_data_num_tasks_finished{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Finished Tasks: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +FAILED_TASKS_PANEL = Panel( + id=33, + title="Failed Tasks", + description="Number of failed tasks.", + unit="tasks", + targets=[ + Target( + expr='sum(ray_data_num_tasks_failed{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Failed Tasks: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +TASK_THROUGHPUT_BY_NODE_PANEL = Panel( + id=46, + title="Task Throughput (by Node)", + description="Number of finished tasks per second, grouped by node.", + unit="tasks/s", + targets=[ + Target( + expr='sum(rate(ray_data_num_tasks_finished_per_node{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, node_ip)', + legend="Finished Tasks: {{dataset}}, {{node_ip}}", + ) + ], + fill=0, + stack=False, +) + +BLOCK_GENERATION_TIME_PANEL = Panel( + id=8, + title="Block Generation Time", + description="Time spent generating blocks in tasks.", + unit="seconds", + targets=[ + Target( + expr='sum(ray_data_block_generation_time{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Block Generation Time: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +TASK_SUBMISSION_BACKPRESSURE_PANEL = Panel( + id=37, + title="Task Submission Backpressure Time", + description="Time spent in task submission backpressure.", + unit="seconds", + targets=[ + Target( + expr='sum(ray_data_task_submission_backpressure_time{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Backpressure Time: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=True, +) + +# Task Completion Time Percentiles +TASK_COMPLETION_TIME_PANEL = Panel( + id=38, + title="Task Completion Time", + description="Time spent running tasks to completion w/ backpressure.", + unit="seconds", + targets=[ + Target( + expr='increase(ray_data_task_completion_time{{{global_filters} operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters} operator=~"$Operator"}}[5m])', + legend="Task Completion Time: {{dataset}}, {{operator}}", ), - unit="Bps", - targets=[ - Target( - expr="sum(rate(ray_data_bytes_outputs_of_finished_tasks_per_node{{{global_filters}}}[1m])) by (dataset, node_ip)", - legend="Bytes output / Second: {{dataset}}, {{node_ip}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=48, - title="Blocks from Finished Tasks / Second (by Node)", - description=( - "Number of output blocks from finished tasks per second, grouped by node." + ], + fill=0, + stack=False, +) + +TASK_OUTPUT_BACKPRESSURE_TIME_PANEL = Panel( + id=39, + title="Task Output Backpressure Time", + description="Time spent in output backpressure.", + unit="seconds", + targets=[ + Target( + expr='increase(ray_data_task_output_backpressure_time{{{global_filters} operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters} operator=~"$Operator"}}[5m])', + legend="Task Output Backpressure Time: {{dataset}}, {{operator}}", ), - unit="blocks/s", - targets=[ - Target( - expr="sum(rate(ray_data_blocks_outputs_of_finished_tasks_per_node{{{global_filters}}}[1m])) by (dataset, node_ip)", - legend="Blocks output / Second: {{dataset}}, {{node_ip}}", - ) - ], - fill=0, - stack=False, - ), - # Ray Data Metrics (Tasks) - Panel( - id=29, - title="Submitted Tasks", - description="Number of submitted tasks.", - unit="tasks", - targets=[ - Target( - expr='sum(ray_data_num_tasks_submitted{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Submitted Tasks: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=30, - title="Running Tasks", - description="Number of running tasks.", - unit="tasks", - targets=[ - Target( - expr='sum(ray_data_num_tasks_running{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Running Tasks: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=31, - title="Tasks with output blocks", - description="Number of tasks that already have output.", - unit="tasks", - targets=[ - Target( - expr='sum(ray_data_num_tasks_have_outputs{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Tasks with output blocks: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=32, - title="Finished Tasks", - description="Number of finished tasks.", - unit="tasks", - targets=[ - Target( - expr='sum(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Finished Tasks: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=46, - title="Task Throughput (by Node)", - description="Number of finished tasks per second, grouped by node.", - unit="tasks/s", - targets=[ - Target( - expr="sum(rate(ray_data_num_tasks_finished_per_node{{{global_filters}}}[1m])) by (dataset, node_ip)", - legend="Finished Tasks: {{dataset}}, {{node_ip}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=33, - title="Failed Tasks", - description="Number of failed tasks.", - unit="tasks", - targets=[ - Target( - expr='sum(ray_data_num_tasks_failed{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Failed Tasks: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=8, - title="Block Generation Time", - description="Time spent generating blocks in tasks.", - unit="seconds", - targets=[ - Target( - expr='sum(ray_data_block_generation_time{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Block Generation Time: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=37, - title="Task Submission Backpressure Time", - description="Time spent in task submission backpressure.", - unit="seconds", - targets=[ - Target( - expr='sum(ray_data_task_submission_backpressure_time{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Backpressure Time: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=True, - ), - Panel( - id=38, - title="Task Completion Time", - description="Time spent running tasks to completion w/ backpressure.", - unit="seconds", - targets=[ - Target( - expr='increase(ray_data_task_completion_time{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', - legend="Task Completion Time: {{dataset}}, {{operator}}", - ), - ], - fill=0, - stack=False, - ), - Panel( - id=39, - title="Task Output Backpressure Time", - description="Time spent in output backpressure.", - unit="seconds", - targets=[ - Target( - expr='increase(ray_data_task_output_backpressure_time{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', - legend="Task Output Backpressure Time: {{dataset}}, {{operator}}", - ), - ], - fill=0, - stack=False, - ), - Panel( - id=40, - title="Task Completion Time Without Backpressure", - description="Time spent running tasks to completion w/o backpressure.", - unit="seconds", - targets=[ - Target( - expr='increase(ray_data_task_completion_time_without_backpressure{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', - legend="Task Completion Time w/o Backpressure: {{dataset}}, {{operator}}", - ), - ], - fill=0, - stack=False, - ), - # Ray Data Metrics (Object Store Memory) - Panel( - id=13, - title="Operator Internal Inqueue Size (Blocks)", - description="Number of blocks in operator's internal input queue", - unit="blocks", - targets=[ - Target( - expr='sum(ray_data_obj_store_mem_internal_inqueue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Number of Blocks: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=14, - title="Operator Internal Inqueue Size (Bytes)", - description="Byte size of input blocks in the operator's internal input queue.", - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_obj_store_mem_internal_inqueue{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Bytes Size: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=True, - ), - Panel( - id=15, - title="Operator Internal Outqueue Size (Blocks)", - description="Number of blocks in operator's internal output queue", - unit="blocks", - targets=[ - Target( - expr='sum(ray_data_obj_store_mem_internal_outqueue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Number of Blocks: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=16, - title="Operator Internal Outqueue Size (Bytes)", - description=( - "Byte size of output blocks in the operator's internal output queue." + ], + fill=0, + stack=False, +) + +TASK_COMPLETION_TIME_WITHOUT_BACKPRESSURE_PANEL = Panel( + id=40, + title="Task Completion Time Without Backpressure", + description="Time spent running tasks to completion w/o backpressure.", + unit="seconds", + targets=[ + Target( + expr='increase(ray_data_task_completion_time_without_backpressure{{{global_filters} operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters} operator=~"$Operator"}}[5m])', + legend="Task Completion Time w/o Backpressure: {{dataset}}, {{operator}}", ), - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_obj_store_mem_internal_outqueue{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Bytes Size: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=True, - ), - Panel( - id=2, - title="Operator External OutQueue Size (Blocks)", - description="Number of blocks in operator's external output queue", - unit="blocks", - targets=[ - Target( - expr='sum(ray_data_num_output_queue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Number of Blocks: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=27, - title="Operator External OutQueue Size (bytes)", - description="Byte size of blocks in operator's external output queue", - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_num_output_queue_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Number of Bytes: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=34, - title="Size of Blocks used in Pending Tasks (Bytes)", - description="Byte size of input blocks used by pending tasks.", - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_obj_store_mem_pending_task_inputs{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Bytes Size: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=True, - ), - Panel( - id=35, - title="Freed Memory in Object Store (Bytes)", - description="Byte size of freed memory in object store.", - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_obj_store_mem_freed{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Bytes Size: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=True, - ), - Panel( - id=36, - title="Spilled Memory in Object Store (Bytes)", - description="Byte size of spilled memory in object store.", - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_obj_store_mem_spilled{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Bytes Size: {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=True, - ), - # Ray Data Metrics (Iteration) - Panel( - id=12, - title="Iteration Initialization Time", - description="Seconds spent in iterator initialization code", - unit="seconds", - targets=[ - Target( - expr="sum(ray_data_iter_initialize_seconds{{{global_filters}}}) by (dataset)", - legend="Seconds: {{dataset}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=9, - title="Iteration Blocked Time", - description="Seconds user thread is blocked by iter_batches()", - unit="seconds", - targets=[ - Target( - expr="sum(ray_data_iter_total_blocked_seconds{{{global_filters}}}) by (dataset)", - legend="Seconds: {{dataset}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=10, - title="Iteration User Time", - description="Seconds spent in user code", - unit="seconds", - targets=[ - Target( - expr="sum(ray_data_iter_user_seconds{{{global_filters}}}) by (dataset)", - legend="Seconds: {{dataset}}", - ) - ], - fill=0, - stack=False, - ), - # Ray Data Metrics (Miscellaneous) - Panel( - id=47, - title="Scheduling Loop Duration", - description=("Duration of the scheduling loop in seconds."), - unit="seconds", - targets=[ - Target( - expr="sum(ray_data_sched_loop_duration_s{{{global_filters}}}) by (dataset)", - legend="Scheduling Loop Duration: {{dataset}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=51, - title="Budget (CPU)", - description=("Budget (CPU) for the operator."), - unit="cpu", - targets=[ - Target( - expr='sum(ray_data_cpu_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Budget (CPU): {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=52, - title="Budget (GPU)", - description=("Budget (GPU) for the operator."), - unit="gpu", - targets=[ - Target( - expr='sum(ray_data_gpu_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Budget (GPU): {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=53, - title="Budget (Memory)", - description=("Budget (Memory) for the operator."), - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_memory_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Budget (Memory): {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, - ), - Panel( - id=54, - title="Budget (Object Store Memory)", - description=("Budget (Object Store Memory) for the operator."), - unit="bytes", - targets=[ - Target( - expr='sum(ray_data_object_store_memory_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', - legend="Budget (Object Store Memory): {{dataset}}, {{operator}}", - ) - ], - fill=0, - stack=False, + ], + fill=0, + stack=False, +) + +# Ray Data Metrics (Object Store Memory) +INTERNAL_INQUEUE_BLOCKS_PANEL = Panel( + id=13, + title="Operator Internal Inqueue Size (Blocks)", + description="Number of blocks in operator's internal input queue", + unit="blocks", + targets=[ + Target( + expr='sum(ray_data_obj_store_mem_internal_inqueue_blocks{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Number of Blocks: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +INTERNAL_INQUEUE_BYTES_PANEL = Panel( + id=14, + title="Operator Internal Inqueue Size (Bytes)", + description="Byte size of input blocks in the operator's internal input queue.", + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_obj_store_mem_internal_inqueue{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Bytes Size: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=True, +) + +INTERNAL_OUTQUEUE_BLOCKS_PANEL = Panel( + id=15, + title="Operator Internal Outqueue Size (Blocks)", + description="Number of blocks in operator's internal output queue", + unit="blocks", + targets=[ + Target( + expr='sum(ray_data_obj_store_mem_internal_outqueue_blocks{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Number of Blocks: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +INTERNAL_OUTQUEUE_BYTES_PANEL = Panel( + id=16, + title="Operator Internal Outqueue Size (Bytes)", + description=("Byte size of output blocks in the operator's internal output queue."), + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_obj_store_mem_internal_outqueue{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Bytes Size: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=True, +) + +EXTERNAL_INQUEUE_BLOCKS_PANEL = Panel( + id=2, + title="Operator External OutQueue Size (Blocks)", + description="Number of blocks in operator's external output queue", + unit="blocks", + targets=[ + Target( + expr='sum(ray_data_num_external_inqueue_blocks{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Number of Blocks: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +EXTERNAL_INQUEUE_BYTES_PANEL = Panel( + id=27, + title="Operator External OutQueue Size (bytes)", + description="Byte size of blocks in operator's external output queue", + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_num_external_inqueue_blocks{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Number of Bytes: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +# Combined Inqueue and Outqueue Blocks Panel +COMBINED_INQUEUE_OUTQUEUE_BLOCKS_PANEL = Panel( + id=56, + title="Operator Combined Internal + External Inqueue Size (Blocks)", + description="Total number of blocks in operator's internal + external input queue.", + unit="blocks", + targets=[ + Target( + expr='sum(ray_data_obj_store_mem_internal_inqueue_blocks{{{global_filters} operator=~"$Operator"}} + ray_data_num_external_inqueue_blocks{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Combined Blocks: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +PENDING_TASK_INPUTS_PANEL = Panel( + id=34, + title="Size of Blocks used in Pending Tasks (Bytes)", + description="Byte size of input blocks used by pending tasks.", + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_obj_store_mem_pending_task_inputs{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Bytes Size: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=True, +) + +FREED_MEMORY_PANEL = Panel( + id=35, + title="Freed Memory in Object Store (Bytes)", + description="Byte size of freed memory in object store.", + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_obj_store_mem_freed{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Bytes Size: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=True, +) + +SPILLED_MEMORY_PANEL = Panel( + id=36, + title="Spilled Memory in Object Store (Bytes)", + description="Byte size of spilled memory in object store.", + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_obj_store_mem_spilled{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Bytes Size: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=True, +) + +# Ray Data Metrics (Iteration) +ITERATION_INITIALIZATION_PANEL = Panel( + id=12, + title="Iteration Initialization Time", + description="Seconds spent in iterator initialization code", + unit="seconds", + targets=[ + Target( + expr='sum(ray_data_iter_initialize_seconds{{{global_filters} operator=~"$Operator"}}) by (dataset)', + legend="Seconds: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +ITERATION_BLOCKED_PANEL = Panel( + id=9, + title="Iteration Blocked Time", + description="Seconds user thread is blocked by iter_batches()", + unit="seconds", + targets=[ + Target( + expr='sum(ray_data_iter_total_blocked_seconds{{{global_filters} operator=~"$Operator"}}) by (dataset)', + legend="Seconds: {{dataset}}", + ) + ], + fill=0, + stack=False, +) + +ITERATION_USER_PANEL = Panel( + id=10, + title="Iteration User Time", + description="Seconds spent in user code", + unit="seconds", + targets=[ + Target( + expr='sum(ray_data_iter_user_seconds{{{global_filters} operator=~"$Operator"}}) by (dataset)', + legend="Seconds: {{dataset}}", + ) + ], + fill=0, + stack=False, +) + +# Ray Data Metrics (Miscellaneous) +SCHEDULING_LOOP_DURATION_PANEL = Panel( + id=47, + title="Scheduling Loop Duration", + description=("Duration of the scheduling loop in seconds."), + unit="seconds", + targets=[ + Target( + expr='sum(ray_data_sched_loop_duration_s{{{global_filters} operator=~"$Operator"}}) by (dataset)', + legend="Scheduling Loop Duration: {{dataset}}", + ) + ], + fill=0, + stack=False, +) + +MAX_BYTES_TO_READ_PANEL = Panel( + id=55, + title="Max Bytes to Read", + description="Maximum bytes to read from streaming generator buffer.", + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_max_bytes_to_read{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Max Bytes to Read: {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +# Budget Panels +CPU_BUDGET_PANEL = Panel( + id=51, + title="Budget (CPU)", + description=("Budget (CPU) for the operator."), + unit="cpu", + targets=[ + Target( + expr='sum(ray_data_cpu_budget{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Budget (CPU): {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +GPU_BUDGET_PANEL = Panel( + id=52, + title="Budget (GPU)", + description=("Budget (GPU) for the operator."), + unit="gpu", + targets=[ + Target( + expr='sum(ray_data_gpu_budget{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Budget (GPU): {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +MEMORY_BUDGET_PANEL = Panel( + id=53, + title="Budget (Memory)", + description=("Budget (Memory) for the operator."), + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_memory_budget{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Budget (Memory): {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +OBJECT_STORE_MEMORY_BUDGET_PANEL = Panel( + id=54, + title="Budget (Object Store Memory)", + description=("Budget (Object Store Memory) for the operator."), + unit="bytes", + targets=[ + Target( + expr='sum(ray_data_object_store_memory_budget{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + legend="Budget (Object Store Memory): {{dataset}}, {{operator}}", + ) + ], + fill=0, + stack=False, +) + +DATA_GRAFANA_ROWS = [ + # Overview Row + Row( + title="Overview", + id=99, + panels=[ + BYTES_GENERATED_PANEL, + BLOCKS_GENERATED_PANEL, + ROWS_GENERATED_PANEL, + OBJECT_STORE_MEMORY_PANEL, + RUNNING_TASKS_PANEL, + COMBINED_INQUEUE_OUTQUEUE_BLOCKS_PANEL, + ], + collapsed=False, + ), + # Pending Inputs Row + Row( + title="Pending Inputs", + id=100, + panels=[ + INTERNAL_INQUEUE_BLOCKS_PANEL, + INTERNAL_INQUEUE_BYTES_PANEL, + EXTERNAL_INQUEUE_BLOCKS_PANEL, + EXTERNAL_INQUEUE_BYTES_PANEL, + PENDING_TASK_INPUTS_PANEL, + ], + collapsed=True, + ), + # Inputs Row + Row( + title="Inputs", + id=101, + panels=[ + INPUT_BLOCKS_RECEIVED_PANEL, + INPUT_BYTES_RECEIVED_PANEL, + INPUT_BLOCKS_PROCESSED_PANEL, + INPUT_BYTES_PROCESSED_PANEL, + INPUT_BYTES_SUBMITTED_PANEL, + ], + collapsed=True, + ), + # Pending Outputs Row + Row( + title="Pending Outputs", + id=102, + panels=[ + INTERNAL_OUTQUEUE_BLOCKS_PANEL, + INTERNAL_OUTQUEUE_BYTES_PANEL, + MAX_BYTES_TO_READ_PANEL, + ], + collapsed=True, + ), + # Outputs Row + Row( + title="Outputs", + id=103, + panels=[ + OUTPUT_BLOCKS_TAKEN_PANEL, + OUTPUT_BYTES_TAKEN_PANEL, + OUTPUT_BYTES_BY_NODE_PANEL, + BLOCKS_BY_NODE_PANEL, + BYTES_OUTPUT_PER_SECOND_PANEL, + ROWS_OUTPUT_PER_SECOND_PANEL, + AVERAGE_BYTES_PER_BLOCK_PANEL, + AVERAGE_BLOCKS_PER_TASK_PANEL, + BLOCK_GENERATION_TIME_PANEL, + ], + collapsed=True, + ), + # Tasks + Row( + title="Tasks", + id=104, + panels=[ + TASK_COMPLETION_TIME_PANEL, + TASK_COMPLETION_TIME_WITHOUT_BACKPRESSURE_PANEL, + TASK_OUTPUT_BACKPRESSURE_TIME_PANEL, + TASK_SUBMISSION_BACKPRESSURE_PANEL, + TASK_THROUGHPUT_BY_NODE_PANEL, + TASKS_WITH_OUTPUT_PANEL, + SUBMITTED_TASKS_PANEL, + FINISHED_TASKS_PANEL, + FAILED_TASKS_PANEL, + ], + collapsed=True, + ), + # Resource Budget / Usage Row + Row( + title="Resource Budget / Usage", + id=105, + panels=[ + CPU_USAGE_PANEL, + GPU_USAGE_PANEL, + CPU_BUDGET_PANEL, + GPU_BUDGET_PANEL, + MEMORY_BUDGET_PANEL, + OBJECT_STORE_MEMORY_BUDGET_PANEL, + FREED_MEMORY_PANEL, + SPILLED_MEMORY_PANEL, + BYTES_SPILLED_PANEL, + BYTES_FREED_PANEL, + ], + collapsed=True, + ), + # Scheduling Loop Row + Row( + title="Scheduling Loop", + id=106, + panels=[ + SCHEDULING_LOOP_DURATION_PANEL, + ], + collapsed=True, + ), + # Iteration Row + Row( + title="Iteration", + id=107, + panels=[ + ITERATION_INITIALIZATION_PANEL, + ITERATION_BLOCKED_PANEL, + ITERATION_USER_PANEL, + ], + collapsed=True, ), ] -ids = [] -for panel in DATA_GRAFANA_PANELS: - ids.append(panel.id) -assert len(ids) == len( - set(ids) -), f"Duplicated id found. Use unique id for each panel. {ids}" +# Get all panel IDs from both top-level panels and panels within rows +all_panel_ids = [] +for row in DATA_GRAFANA_ROWS: + all_panel_ids.append(row.id) + all_panel_ids.extend(panel.id for panel in row.panels) + +assert len(all_panel_ids) == len( + set(all_panel_ids) +), f"Duplicated id found. Use unique id for each panel. {all_panel_ids}" data_dashboard_config = DashboardConfig( name="DATA", default_uid="rayDataDashboard", - panels=DATA_GRAFANA_PANELS, + rows=DATA_GRAFANA_ROWS, standard_global_filters=[ 'dataset=~"$DatasetID"', 'SessionName=~"$SessionName"', diff --git a/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py b/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py index 47436d5f329b..a7066a979f15 100644 --- a/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py +++ b/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py @@ -324,14 +324,14 @@ class OpRuntimeMetrics(metaclass=OpRuntimesMetricsMeta): description=("Number of rows generated by finished tasks."), metrics_group=MetricsGroup.OUTPUTS, ) - num_output_queue_blocks: int = metric_field( + num_external_inqueue_blocks: int = metric_field( default=0, - description="Number of blocks in the output queue", + description="Number of blocks in the external inqueue", metrics_group=MetricsGroup.OUTPUTS, ) - num_output_queue_bytes: int = metric_field( + num_external_inqueue_bytes: int = metric_field( default=0, - description="Byte size of blocks in the output queue", + description="Byte size of blocks in the external inqueue", metrics_group=MetricsGroup.OUTPUTS, ) diff --git a/python/ray/data/_internal/execution/streaming_executor_state.py b/python/ray/data/_internal/execution/streaming_executor_state.py index 717ad190cc5a..586e6817e81f 100644 --- a/python/ray/data/_internal/execution/streaming_executor_state.py +++ b/python/ray/data/_internal/execution/streaming_executor_state.py @@ -303,8 +303,9 @@ def add_output(self, ref: RefBundle) -> None: self.op.metrics.num_alive_actors = actor_info.running self.op.metrics.num_restarting_actors = actor_info.restarting self.op.metrics.num_pending_actors = actor_info.pending - self.op.metrics.num_output_queue_blocks = self.output_queue.num_blocks - self.op.metrics.num_output_queue_bytes = self.output_queue.memory_usage + for next_op in self.op.output_dependencies: + next_op.metrics.num_external_inqueue_blocks = self.output_queue.num_blocks + next_op.metrics.num_external_inqueue_bytes = self.output_queue.memory_usage def refresh_progress_bar(self, resource_manager: ResourceManager) -> None: """Update the console with the latest operator progress.""" diff --git a/python/ray/data/tests/test_stats.py b/python/ray/data/tests/test_stats.py index 0c93e23bcbd0..0a3a32e9d63e 100644 --- a/python/ray/data/tests/test_stats.py +++ b/python/ray/data/tests/test_stats.py @@ -99,8 +99,8 @@ def gen_expected_metrics( "'num_outputs_of_finished_tasks': N", "'bytes_outputs_of_finished_tasks': N", "'rows_outputs_of_finished_tasks': N", - "'num_output_queue_blocks': N", - "'num_output_queue_bytes': N", + "'num_external_inqueue_blocks': N", + "'num_external_inqueue_bytes': N", "'num_tasks_submitted': N", "'num_tasks_running': Z", "'num_tasks_have_outputs': N", @@ -160,8 +160,8 @@ def gen_expected_metrics( "'num_outputs_of_finished_tasks': Z", "'bytes_outputs_of_finished_tasks': Z", "'rows_outputs_of_finished_tasks': Z", - "'num_output_queue_blocks': N", - "'num_output_queue_bytes': N", + "'num_external_inqueue_blocks': N", + "'num_external_inqueue_bytes': N", "'num_tasks_submitted': Z", "'num_tasks_running': Z", "'num_tasks_have_outputs': Z", @@ -675,8 +675,8 @@ def test_dataset__repr__(ray_start_regular_shared, restore_data_context): " num_outputs_of_finished_tasks: N,\n" " bytes_outputs_of_finished_tasks: N,\n" " rows_outputs_of_finished_tasks: N,\n" - " num_output_queue_blocks: N,\n" - " num_output_queue_bytes: N,\n" + " num_external_inqueue_blocks: N,\n" + " num_external_inqueue_bytes: N,\n" " num_tasks_submitted: N,\n" " num_tasks_running: Z,\n" " num_tasks_have_outputs: N,\n" @@ -805,8 +805,8 @@ def check_stats(): " num_outputs_of_finished_tasks: N,\n" " bytes_outputs_of_finished_tasks: N,\n" " rows_outputs_of_finished_tasks: N,\n" - " num_output_queue_blocks: N,\n" - " num_output_queue_bytes: N,\n" + " num_external_inqueue_blocks: N,\n" + " num_external_inqueue_bytes: N,\n" " num_tasks_submitted: N,\n" " num_tasks_running: Z,\n" " num_tasks_have_outputs: N,\n" @@ -890,8 +890,8 @@ def check_stats(): " num_outputs_of_finished_tasks: N,\n" " bytes_outputs_of_finished_tasks: N,\n" " rows_outputs_of_finished_tasks: N,\n" - " num_output_queue_blocks: N,\n" - " num_output_queue_bytes: N,\n" + " num_external_inqueue_blocks: N,\n" + " num_external_inqueue_bytes: N,\n" " num_tasks_submitted: N,\n" " num_tasks_running: Z,\n" " num_tasks_have_outputs: N,\n" @@ -1852,6 +1852,13 @@ def test_op_metrics_logging(): "Operator InputDataBuffer[Input] completed. Operator Metrics:\n" + gen_expected_metrics(is_map=False) ) # .replace("'obj_store_mem_used': N", "'obj_store_mem_used': Z") + # InputDataBuffer has no inqueue, manually set to 0 + input_str = input_str.replace( + "'num_external_inqueue_blocks': N", "'num_external_inqueue_blocks': Z" + ) + input_str = input_str.replace( + "'num_external_inqueue_bytes': N", "'num_external_inqueue_bytes': Z" + ) map_str = ( "Operator TaskPoolMapOperator[ReadRange->MapBatches()] completed. " "Operator Metrics:\n" From 46bded1ba8a0edad8198b0607ccd7c7c73514c95 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 19 Aug 2025 15:20:06 -0500 Subject: [PATCH 182/634] [core] Combine `RedisClient` into `RedisStoreClient` (#55655) - `RedisClient` was a thin and meaningless abstraction over `RedisContext`, so now the `RedisStoreClient` uses the `RedisContext` directly. - The only other user of `RedisClient` was the "periodic health checker." I've moved this into `RedisStoreClient` as well. - Also removed some unnecessary child classes: `InMemoryGcsTableStorage`, `RedisGcsTableStorage`. Before merging I plan to fold the `RayConfig` options used in `RedisStoreClient` into the `RedisClientOptions`. --------- Signed-off-by: Edward Oakes --- BUILD.bazel | 2 +- python/ray/includes/global_state_accessor.pxd | 15 +- src/ray/common/ray_config_def.h | 5 +- src/ray/gcs/BUILD.bazel | 34 ----- src/ray/gcs/gcs_client/BUILD.bazel | 2 +- src/ray/gcs/gcs_server/BUILD.bazel | 26 ++-- .../gcs/gcs_server/gcs_placement_group_mgr.h | 1 + .../gcs_server/gcs_redis_failure_detector.cc | 60 -------- .../gcs_server/gcs_redis_failure_detector.h | 69 --------- src/ray/gcs/gcs_server/gcs_server.cc | 49 +++---- src/ray/gcs/gcs_server/gcs_server.h | 14 +- src/ray/gcs/gcs_server/gcs_table_storage.cc | 1 + src/ray/gcs/gcs_server/gcs_table_storage.h | 29 +--- src/ray/gcs/gcs_server/tests/BUILD.bazel | 8 +- .../gcs_actor_manager_export_event_test.cc | 4 +- .../gcs_node_manager_export_event_test.cc | 4 +- .../tests/gcs_actor_manager_test.cc | 4 +- .../tests/gcs_actor_scheduler_test.cc | 3 +- .../gcs_server/tests/gcs_kv_manager_test.cc | 10 +- .../tests/gcs_placement_group_mgr_test.cc | 3 +- .../gcs_placement_group_scheduler_test.cc | 3 +- .../gcs_server/tests/gcs_server_test_util.h | 1 + .../tests/gcs_worker_manager_test.cc | 3 +- .../tests/in_memory_gcs_table_storage_test.cc | 3 +- .../tests/redis_gcs_table_storage_test.cc | 14 +- src/ray/gcs/pubsub/BUILD.bazel | 1 - src/ray/gcs/redis_client.cc | 58 -------- src/ray/gcs/redis_client.h | 85 ------------ src/ray/gcs/store_client/BUILD.bazel | 35 +++-- .../{ => store_client}/redis_async_context.cc | 2 +- .../{ => store_client}/redis_async_context.h | 0 .../gcs/{ => store_client}/redis_context.cc | 2 +- .../gcs/{ => store_client}/redis_context.h | 2 +- .../gcs/store_client/redis_store_client.cc | 131 ++++++++++++------ src/ray/gcs/store_client/redis_store_client.h | 61 ++++++-- src/ray/gcs/store_client/tests/BUILD.bazel | 44 +++++- .../tests/redis_async_context_test.cc | 4 +- .../tests/redis_callback_reply_test.cc} | 2 +- .../tests/redis_store_client_test.cc | 12 +- src/ray/gcs/tests/BUILD.bazel | 34 +---- 40 files changed, 298 insertions(+), 542 deletions(-) delete mode 100644 src/ray/gcs/gcs_server/gcs_redis_failure_detector.cc delete mode 100644 src/ray/gcs/gcs_server/gcs_redis_failure_detector.h delete mode 100644 src/ray/gcs/redis_client.cc delete mode 100644 src/ray/gcs/redis_client.h rename src/ray/gcs/{ => store_client}/redis_async_context.cc (99%) rename src/ray/gcs/{ => store_client}/redis_async_context.h (100%) rename src/ray/gcs/{ => store_client}/redis_context.cc (99%) rename src/ray/gcs/{ => store_client}/redis_context.h (99%) rename src/ray/gcs/{ => store_client}/tests/redis_async_context_test.cc (96%) rename src/ray/gcs/{tests/callback_reply_test.cc => store_client/tests/redis_callback_reply_test.cc} (98%) diff --git a/BUILD.bazel b/BUILD.bazel index 1c77a53d4e73..55adc25a0bc2 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -249,10 +249,10 @@ pyx_library( "//:src/ray/ray_exported_symbols.lds", "//:src/ray/ray_version_script.lds", "//src/ray/core_worker:core_worker_lib", - "//src/ray/gcs:gcs_redis_client", "//src/ray/gcs/gcs_client:gcs_python_callbacks", "//src/ray/gcs/gcs_client:global_state_accessor_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/store_client:redis_store_client", "//src/ray/protobuf:serialization_cc_proto", "//src/ray/thirdparty/setproctitle", "//src/ray/util", diff --git a/python/ray/includes/global_state_accessor.pxd b/python/ray/includes/global_state_accessor.pxd index f6733151e800..1d6e5cd1c8c9 100644 --- a/python/ray/includes/global_state_accessor.pxd +++ b/python/ray/includes/global_state_accessor.pxd @@ -71,7 +71,6 @@ cdef extern from * namespace "ray::gcs" nogil: """ #include #include "ray/gcs/gcs_server/store_client_kv.h" - #include "ray/gcs/redis_client.h" #include "ray/gcs/store_client/redis_store_client.h" namespace ray { namespace gcs { @@ -94,23 +93,19 @@ cdef extern from * namespace "ray::gcs" nogil: /*log_rotation_max_size=*/1ULL << 29, /*log_rotation_file_num=*/10); - RedisClientOptions options(host, port, username, password, use_ssl); - std::string config_list; RAY_CHECK(absl::Base64Unescape(config, &config_list)); RayConfig::instance().initialize(config_list); instrumented_io_context io_service{/*enable_lag_probe=*/false, /*running_on_single_thread=*/true}; + // Set heartbeat_interval_ms to 0 to disable health checking for this temporary client. + RedisClientOptions options{host, port, username, password, use_ssl, /*heartbeat_interval_ms=*/0}; - auto redis_client = std::make_shared(options); - auto status = redis_client->Connect(io_service); - RAY_CHECK_OK(status) << "Failed to connect to redis."; - - auto cli = std::make_unique( - std::make_unique(std::move(redis_client))); + auto client = std::make_unique( + std::make_unique(io_service, options)); bool ret_val = false; - cli->Get("session", key, {[&](std::optional result) { + client->Get("session", key, {[&](std::optional result) { if (result.has_value()) { *data = result.value(); ret_val = true; diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index 827cff80da73..74a932c136a5 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -358,8 +358,9 @@ RAY_CONFIG(uint32_t, RAY_CONFIG(int64_t, gcs_service_connect_retries, 50) /// Waiting time for each gcs service connection. RAY_CONFIG(int64_t, internal_gcs_service_connect_wait_milliseconds, 100) -/// The interval at which the gcs server will check if redis has gone down. -/// When this happens, gcs server will kill itself. +/// The interval at which the gcs server will health check the connection to the +/// external Redis server. If a health check fails, the GCS will crash itself. +/// Set to zero to disable health checking. RAY_CONFIG(uint64_t, gcs_redis_heartbeat_interval_milliseconds, 100) /// Duration to wait between retries for leasing worker in gcs server. RAY_CONFIG(uint32_t, gcs_lease_worker_retry_interval_ms, 200) diff --git a/src/ray/gcs/BUILD.bazel b/src/ray/gcs/BUILD.bazel index 8c2d190bd421..8ffa4b6639e9 100644 --- a/src/ray/gcs/BUILD.bazel +++ b/src/ray/gcs/BUILD.bazel @@ -1,29 +1,5 @@ load("//bazel:ray.bzl", "ray_cc_library") -ray_cc_library( - name = "gcs_redis_client", - srcs = [ - "redis_async_context.cc", - "redis_client.cc", - "redis_context.cc", - ], - hdrs = [ - "redis_async_context.h", - "redis_client.h", - "redis_context.h", - ], - deps = [ - "//:hiredis", - "//src/ray/common:asio", - "//src/ray/common:ray_config", - "//src/ray/common:status", - "//src/ray/stats:stats_lib", - "//src/ray/util:exponential_backoff", - "//src/ray/util:network_util", - "@boost//:asio", - ], -) - ray_cc_library( name = "gcs_pb_util", srcs = ["pb_utils.cc"], @@ -45,13 +21,3 @@ ray_cc_library( "//src/ray/common:status", ], ) - -ray_cc_library( - name = "gcs", - deps = [ - ":gcs_callback", - ":gcs_pb_util", - ":gcs_redis_client", - "//src/ray/rpc:node_manager_client", - ], -) diff --git a/src/ray/gcs/gcs_client/BUILD.bazel b/src/ray/gcs/gcs_client/BUILD.bazel index 3d78c1f66247..dc15ab29dfb5 100644 --- a/src/ray/gcs/gcs_client/BUILD.bazel +++ b/src/ray/gcs/gcs_client/BUILD.bazel @@ -15,7 +15,7 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/pubsub:gcs_pub_sub_lib", - "//src/ray/gcs/store_client:gcs_redis_store_client", + "//src/ray/gcs/store_client:redis_store_client", "//src/ray/protobuf:usage_cc_proto", "//src/ray/pubsub:subscriber", "//src/ray/rpc:gcs_client", diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 179b54c35b3d..e698c17d75c2 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -19,10 +19,10 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:status", "//src/ray/gcs:gcs_callback", - "//src/ray/gcs/store_client:gcs_in_memory_store_client", - "//src/ray/gcs/store_client:gcs_observable_store_client", - "//src/ray/gcs/store_client:gcs_redis_store_client", + "//src/ray/gcs/store_client", "//src/ray/protobuf:gcs_cc_proto", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", ], ) @@ -80,7 +80,7 @@ ray_cc_library( hdrs = ["store_client_kv.h"], deps = [ ":gcs_kv_manager", - "//src/ray/gcs/store_client:gcs_store_client", + "//src/ray/gcs/store_client", ], ) @@ -109,17 +109,6 @@ ray_cc_library( ], ) -ray_cc_library( - name = "gcs_redis_failure_detector", - srcs = ["gcs_redis_failure_detector.cc"], - hdrs = ["gcs_redis_failure_detector.h"], - deps = [ - "//src/ray/common:asio", - "//src/ray/common:ray_config", - "//src/ray/gcs:gcs_redis_client", - ], -) - ray_cc_library( name = "gcs_worker_manager", srcs = ["gcs_worker_manager.cc"], @@ -233,7 +222,6 @@ ray_cc_library( ":gcs_job_manager", ":gcs_kv_manager", ":gcs_pubsub_handler", - ":gcs_redis_failure_detector", ":gcs_runtime_env_handler", ":gcs_server_io_context_policy", ":gcs_state_util", @@ -243,7 +231,10 @@ ray_cc_library( ":gcs_usage_stats_client", ":gcs_worker_manager", "//src/ray/gcs/pubsub:gcs_pub_sub_lib", - "//src/ray/gcs/store_client:gcs_observable_store_client", + "//src/ray/gcs/store_client", + "//src/ray/gcs/store_client:in_memory_store_client", + "//src/ray/gcs/store_client:observable_store_client", + "//src/ray/gcs/store_client:redis_store_client", "//src/ray/protobuf:autoscaler_cc_grpc", "//src/ray/protobuf:gcs_service_cc_grpc", "//src/ray/pubsub:publisher", @@ -253,6 +244,7 @@ ray_cc_library( "//src/ray/rpc:gcs_server", "//src/ray/rpc:node_manager_client", "//src/ray/util:counter_map", + "//src/ray/util:exponential_backoff", "//src/ray/util:network_util", "//src/ray/util:thread_checker", "//src/ray/util:throttler", diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h index 36d6e1b8f0b8..fd046856fc17 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h @@ -35,6 +35,7 @@ #include "ray/gcs/pubsub/gcs_pub_sub.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/util/counter_map.h" +#include "ray/util/exponential_backoff.h" #include "src/ray/protobuf/gcs_service.pb.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/gcs_redis_failure_detector.cc b/src/ray/gcs/gcs_server/gcs_redis_failure_detector.cc deleted file mode 100644 index 79ba225b4202..000000000000 --- a/src/ray/gcs/gcs_server/gcs_redis_failure_detector.cc +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ray/gcs/gcs_server/gcs_redis_failure_detector.h" - -#include -#include - -#include "ray/common/ray_config.h" -#include "ray/gcs/redis_client.h" - -namespace ray { -namespace gcs { - -GcsRedisFailureDetector::GcsRedisFailureDetector( - instrumented_io_context &io_service, - std::shared_ptr redis_client, - std::function callback) - : io_service_(io_service), - redis_client_(std::move(redis_client)), - callback_(std::move(callback)) {} - -void GcsRedisFailureDetector::Start() { - RAY_LOG(INFO) << "Starting redis failure detector."; - periodical_runner_ = PeriodicalRunner::Create(io_service_); - periodical_runner_->RunFnPeriodically( - [this] { DetectRedis(); }, - RayConfig::instance().gcs_redis_heartbeat_interval_milliseconds(), - "GcsRedisFailureDetector.deadline_timer.detect_redis_failure"); -} - -void GcsRedisFailureDetector::Stop() { - RAY_LOG(INFO) << "Stopping redis failure detector."; - periodical_runner_.reset(); -} - -void GcsRedisFailureDetector::DetectRedis() { - auto redis_callback = [this](const std::shared_ptr &reply) { - if (reply->IsNil()) { - RAY_LOG(ERROR) << "Redis is inactive."; - this->io_service_.dispatch(this->callback_, "GcsRedisFailureDetector.DetectRedis"); - } - }; - auto *cxt = redis_client_->GetPrimaryContext(); - cxt->RunArgvAsync({"PING"}, redis_callback); -} - -} // namespace gcs -} // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_redis_failure_detector.h b/src/ray/gcs/gcs_server/gcs_redis_failure_detector.h deleted file mode 100644 index 1928226a33b7..000000000000 --- a/src/ray/gcs/gcs_server/gcs_redis_failure_detector.h +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/asio/periodical_runner.h" - -namespace ray { -namespace gcs { - -// Forward declaration. -class RedisClient; - -/// GcsRedisFailureDetector is responsible for monitoring redis and binding GCS server and -/// redis life cycle together. GCS client subscribes to redis messages and it cannot sense -/// whether the redis is inactive unless we go to ping redis voluntarily. But there are -/// many GCS clients, if they all Ping redis, the redis load will be high. So we ping -/// redis on GCS server and GCS client can sense whether redis is normal through RPC -/// connection with GCS server. -class GcsRedisFailureDetector { - public: - /// Create a GcsRedisFailureDetector. - /// - /// \param io_service The event loop to run the monitor on. - /// \param redis_context The redis context is used to ping redis. - /// \param callback Callback that will be called when redis is detected as not alive. - explicit GcsRedisFailureDetector(instrumented_io_context &io_service, - std::shared_ptr redis_client, - std::function callback); - - /// Start detecting redis. - void Start(); - - /// Stop detecting redis. - void Stop(); - - protected: - /// Check that if redis is inactive. - void DetectRedis(); - - private: - instrumented_io_context &io_service_; - - std::shared_ptr redis_client_; - - /// The runner to run function periodically. - std::shared_ptr periodical_runner_; - - /// A function is called when redis is detected to be unavailable. - std::function callback_; -}; - -} // namespace gcs -} // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index d8d4b502f2a7..44afef0fb65a 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -30,6 +30,10 @@ #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/gcs_worker_manager.h" #include "ray/gcs/gcs_server/store_client_kv.h" +#include "ray/gcs/store_client/in_memory_store_client.h" +#include "ray/gcs/store_client/observable_store_client.h" +#include "ray/gcs/store_client/redis_store_client.h" +#include "ray/gcs/store_client/store_client.h" #include "ray/pubsub/publisher.h" #include "ray/util/network_util.h" #include "ray/util/util.h" @@ -118,25 +122,22 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, // GcsInternalKVManager, to avoid congestion on the latter. RAY_LOG(INFO) << "GCS storage type is " << storage_type_; auto &io_context = io_context_provider_.GetDefaultIOContext(); + std::unique_ptr store_client; switch (storage_type_) { case StorageType::IN_MEMORY: - gcs_table_storage_ = std::make_unique(); + store_client = + std::make_unique(std::make_unique()); break; case StorageType::REDIS_PERSIST: { - auto redis_client = CreateRedisClient(io_context); - gcs_table_storage_ = std::make_unique(redis_client); - // Init redis failure detector. - gcs_redis_failure_detector_ = - std::make_unique(io_context, redis_client, []() { - RAY_LOG(FATAL) << "Redis connection failed. Shutdown GCS."; - }); - gcs_redis_failure_detector_->Start(); + store_client = CreateRedisStoreClient(io_context); break; } default: RAY_LOG(FATAL) << "Unexpected storage type: " << storage_type_; } + gcs_table_storage_ = std::make_unique(std::move(store_client)); + auto inner_publisher = std::make_unique( /*channels=*/ std::vector{ @@ -159,14 +160,6 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, GcsServer::~GcsServer() { Stop(); } -RedisClientOptions GcsServer::GetRedisClientOptions() const { - return RedisClientOptions(config_.redis_address, - config_.redis_port, - config_.redis_username, - config_.redis_password, - config_.enable_redis_ssl); -} - void GcsServer::Start() { // Load gcs tables data asynchronously. auto gcs_init_data = std::make_shared(*gcs_table_storage_); @@ -322,9 +315,6 @@ void GcsServer::Stop() { kv_manager_.reset(); is_stopped_ = true; - if (gcs_redis_failure_detector_) { - gcs_redis_failure_detector_->Stop(); - } RAY_LOG(INFO) << "GCS server stopped."; } @@ -595,8 +585,8 @@ void GcsServer::InitKVManager() { auto &io_context = io_context_provider_.GetIOContext(); switch (storage_type_) { case (StorageType::REDIS_PERSIST): - instance = std::make_unique( - std::make_unique(CreateRedisClient(io_context))); + instance = + std::make_unique(CreateRedisStoreClient(io_context)); break; case (StorageType::IN_MEMORY): instance = std::make_unique( @@ -873,12 +863,17 @@ std::string GcsServer::GetDebugState() const { return stream.str(); } -std::shared_ptr GcsServer::CreateRedisClient( +std::unique_ptr GcsServer::CreateRedisStoreClient( instrumented_io_context &io_service) { - auto redis_client = std::make_shared(GetRedisClientOptions()); - auto status = redis_client->Connect(io_service); - RAY_CHECK_OK(status) << "Failed to init redis gcs client"; - return redis_client; + return std::make_unique( + io_service, + RedisClientOptions{ + config_.redis_address, + config_.redis_port, + config_.redis_username, + config_.redis_password, + config_.enable_redis_ssl, + RayConfig::instance().gcs_redis_heartbeat_interval_milliseconds()}); } void GcsServer::PrintAsioStats() { diff --git a/src/ray/gcs/gcs_server/gcs_server.h b/src/ray/gcs/gcs_server/gcs_server.h index 755a466b172f..491300996a83 100644 --- a/src/ray/gcs/gcs_server/gcs_server.h +++ b/src/ray/gcs/gcs_server/gcs_server.h @@ -26,7 +26,6 @@ #include "ray/gcs/gcs_server/gcs_health_check_manager.h" #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" -#include "ray/gcs/gcs_server/gcs_redis_failure_detector.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/gcs_server_io_context_policy.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" @@ -35,7 +34,9 @@ #include "ray/gcs/gcs_server/runtime_env_handler.h" #include "ray/gcs/gcs_server/usage_stats_client.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" -#include "ray/gcs/redis_client.h" +#include "ray/gcs/store_client/in_memory_store_client.h" +#include "ray/gcs/store_client/observable_store_client.h" +#include "ray/gcs/store_client/redis_store_client.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/scheduling/cluster_task_manager.h" #include "ray/rpc/client_call.h" @@ -129,9 +130,6 @@ class GcsServer { } protected: - /// Generate the redis client options - RedisClientOptions GetRedisClientOptions() const; - void DoStart(const GcsInitData &gcs_init_data); /// Initialize gcs node manager. @@ -213,8 +211,8 @@ class GcsServer { /// Print the asio event loop stats for debugging. void PrintAsioStats(); - /// Get or connect to a redis server - std::shared_ptr CreateRedisClient(instrumented_io_context &io_service); + std::unique_ptr CreateRedisStoreClient( + instrumented_io_context &io_service); void TryGlobalGC(); @@ -253,8 +251,6 @@ class GcsServer { std::unique_ptr gcs_node_manager_; /// The health check manager. std::shared_ptr gcs_healthcheck_manager_; - /// The gcs redis failure detector. - std::unique_ptr gcs_redis_failure_detector_; /// The gcs placement group manager. std::unique_ptr gcs_placement_group_manager_; /// The gcs actor manager. diff --git a/src/ray/gcs/gcs_server/gcs_table_storage.cc b/src/ray/gcs/gcs_server/gcs_table_storage.cc index df636bee6a8a..49c0b7ca87df 100644 --- a/src/ray/gcs/gcs_server/gcs_table_storage.cc +++ b/src/ray/gcs/gcs_server/gcs_table_storage.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/container/flat_hash_map.h" #include "ray/common/asio/postable.h" #include "ray/common/id.h" #include "ray/common/status.h" diff --git a/src/ray/gcs/gcs_server/gcs_table_storage.h b/src/ray/gcs/gcs_server/gcs_table_storage.h index af0992cbf4e1..92baeb8f38b4 100644 --- a/src/ray/gcs/gcs_server/gcs_table_storage.h +++ b/src/ray/gcs/gcs_server/gcs_table_storage.h @@ -19,9 +19,9 @@ #include #include -#include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/store_client/observable_store_client.h" -#include "ray/gcs/store_client/redis_store_client.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "ray/gcs/store_client/store_client.h" #include "src/ray/protobuf/gcs.pb.h" namespace ray { @@ -206,10 +206,6 @@ class GcsWorkerTable : public GcsTable { } }; -/// \class GcsTableStorage -/// -/// This class is not meant to be used directly. All gcs table storage classes should -/// derive from this class and override class member variables. class GcsTableStorage { public: explicit GcsTableStorage(std::shared_ptr store_client) @@ -269,24 +265,5 @@ class GcsTableStorage { std::unique_ptr worker_table_; }; -/// \class RedisGcsTableStorage -/// RedisGcsTableStorage is an implementation of `GcsTableStorage` -/// that uses redis as storage. -class RedisGcsTableStorage : public GcsTableStorage { - public: - explicit RedisGcsTableStorage(std::shared_ptr redis_client) - : GcsTableStorage(std::make_shared(std::move(redis_client))) {} -}; - -/// \class InMemoryGcsTableStorage -/// InMemoryGcsTableStorage is an implementation of `GcsTableStorage` -/// that uses memory as storage. -class InMemoryGcsTableStorage : public GcsTableStorage { - public: - explicit InMemoryGcsTableStorage() - : GcsTableStorage(std::make_shared( - std::make_unique())) {} -}; - } // namespace gcs } // namespace ray diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index 67d3aa490fde..92254188d8d8 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -82,6 +82,7 @@ ray_cc_library( deps = [ "//:ray_fakes", "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs/store_client:in_memory_store_client", ], ) @@ -204,6 +205,7 @@ ray_cc_test( "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], @@ -266,7 +268,7 @@ ray_cc_library( "gcs_table_storage_test_base.h", ], deps = [ - "//src/ray/gcs/store_client:gcs_redis_store_client", + "//src/ray/gcs/store_client:redis_store_client", ], ) @@ -303,6 +305,7 @@ ray_cc_test( ":gcs_table_storage_test_lib", "//src/ray/common:test_util", "//src/ray/gcs/gcs_server:gcs_table_storage", + "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/store_client/tests:store_client_test_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", @@ -369,6 +372,7 @@ ray_cc_test( "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], @@ -386,6 +390,7 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], @@ -403,6 +408,7 @@ ray_cc_test( ":gcs_server_test_util", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc index a7bc18a0216d..0b5c6494ea3a 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc @@ -26,6 +26,7 @@ #include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/store_client/in_memory_store_client.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/pubsub/publisher.h" @@ -153,7 +154,8 @@ class GcsActorManagerTest : public ::testing::Test { /*batch_size=*/100); gcs_publisher_ = std::make_unique(std::move(publisher)); - gcs_table_storage_ = std::make_unique(); + gcs_table_storage_ = + std::make_unique(std::make_unique()); kv_ = std::make_unique(); function_manager_ = std::make_unique(*kv_, io_service_); auto actor_scheduler = std::make_unique(); diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc index 52da5b41a4b9..a310f656b7af 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc @@ -21,6 +21,7 @@ #include #include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/util/event.h" #include "ray/util/string_utils.h" @@ -50,7 +51,8 @@ class GcsNodeManagerExportAPITest : public ::testing::Test { [this](const rpc::Address &) { return raylet_client_; }); gcs_publisher_ = std::make_unique( std::make_unique()); - gcs_table_storage_ = std::make_unique(); + gcs_table_storage_ = std::make_unique( + std::make_unique()); RayConfig::instance().initialize( R"( diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc index 8754b5a404dc..80f423f8ffb0 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc @@ -24,6 +24,7 @@ #include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/pubsub/publisher.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" @@ -128,7 +129,8 @@ class GcsActorManagerTest : public ::testing::Test { gcs_publisher_ = std::make_unique(std::move(publisher)); store_client_ = std::make_shared(); - gcs_table_storage_ = std::make_unique(); + gcs_table_storage_ = + std::make_unique(std::make_unique()); kv_ = std::make_unique(); function_manager_ = std::make_unique(*kv_, io_service_); auto scheduler = std::make_unique(); diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc index b11e3f8ed624..72ed153bc9b8 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc @@ -44,7 +44,8 @@ class GcsActorSchedulerTest : public ::testing::Test { gcs_publisher_ = std::make_shared( std::make_unique()); store_client_ = std::make_shared(); - gcs_table_storage_ = std::make_shared(); + gcs_table_storage_ = + std::make_unique(std::make_unique()); gcs_node_manager_ = std::make_shared(gcs_publisher_.get(), gcs_table_storage_.get(), io_context_->GetIoService(), diff --git a/src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc index 228d446e4947..9792fd9e0898 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc @@ -36,13 +36,11 @@ class GcsKVManagerTest : public ::testing::TestWithParam { io_service.get_executor()); io_service.run(); }); - ray::gcs::RedisClientOptions redis_client_options( - "127.0.0.1", ray::TEST_REDIS_SERVER_PORTS.front(), "", "", false); + ray::gcs::RedisClientOptions options{"127.0.0.1", + ray::TEST_REDIS_SERVER_PORTS.front()}; if (GetParam() == "redis") { - auto client = std::make_shared(redis_client_options); - RAY_CHECK_OK(client->Connect(io_service)); kv_instance = std::make_unique( - std::make_unique(client)); + std::make_unique(io_service, options)); } else if (GetParam() == "memory") { kv_instance = std::make_unique( std::make_unique()); @@ -52,11 +50,9 @@ class GcsKVManagerTest : public ::testing::TestWithParam { void TearDown() override { io_service.stop(); thread_io_service->join(); - redis_client.reset(); kv_instance.reset(); } - std::unique_ptr redis_client; std::unique_ptr thread_io_service; instrumented_io_context io_service; std::unique_ptr kv_instance; diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc index 853684b7fd85..ca7355faf94e 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc @@ -85,7 +85,8 @@ class GcsPlacementGroupManagerTest : public ::testing::Test { cluster_resource_manager_(io_service_) { gcs_publisher_ = std::make_shared(std::make_unique()); - gcs_table_storage_ = std::make_unique(); + gcs_table_storage_ = + std::make_unique(std::make_unique()); gcs_node_manager_ = std::make_shared(); gcs_resource_manager_ = std::make_shared( io_service_, cluster_resource_manager_, *gcs_node_manager_, NodeID::FromRandom()); diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc index 26d1a9ea6226..cfeef60a5fa6 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc @@ -46,7 +46,8 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { for (int index = 0; index < 3; ++index) { raylet_clients_.push_back(std::make_shared()); } - gcs_table_storage_ = std::make_shared(); + gcs_table_storage_ = std::make_unique( + std::make_unique()); gcs_publisher_ = std::make_shared( std::make_unique()); auto local_node_id = NodeID::FromRandom(); diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h index 93ad48a844d2..c350f54a3d9d 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h +++ b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h @@ -34,6 +34,7 @@ #include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" #include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" +#include "ray/gcs/store_client/in_memory_store_client.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc index d1e5f47b90a5..0d34847e9ced 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc @@ -39,7 +39,8 @@ class GcsWorkerManagerTest : public Test { GcsWorkerManagerTest() { gcs_publisher_ = std::make_shared(std::make_unique()); - gcs_table_storage_ = std::make_shared(); + gcs_table_storage_ = + std::make_unique(std::make_unique()); } void SetUp() override { diff --git a/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc b/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc index 5bd694e52d7f..40c68497f628 100644 --- a/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc +++ b/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc @@ -26,7 +26,8 @@ namespace ray { class InMemoryGcsTableStorageTest : public gcs::GcsTableStorageTestBase { public: void SetUp() override { - gcs_table_storage_ = std::make_shared(); + gcs_table_storage_ = std::make_shared( + std::make_unique()); } }; diff --git a/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc b/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc index 591149ec9ffa..9261ca5042d6 100644 --- a/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc +++ b/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc @@ -29,17 +29,13 @@ class RedisGcsTableStorageTest : public gcs::GcsTableStorageTestBase { static void TearDownTestCase() { TestSetupUtil::ShutDownRedisServers(); } void SetUp() override { - gcs::RedisClientOptions options("127.0.0.1", TEST_REDIS_SERVER_PORTS.front(), "", ""); - redis_client_ = std::make_shared(options); - RAY_CHECK_OK(redis_client_->Connect(*io_service_pool_->Get())); - - gcs_table_storage_ = std::make_shared(redis_client_); + auto &io_service = *io_service_pool_->Get(); + gcs::RedisClientOptions options{"127.0.0.1", TEST_REDIS_SERVER_PORTS.front()}; + gcs_table_storage_ = std::make_shared( + std::make_unique(io_service, options)); } - void TearDown() override { redis_client_->Disconnect(); } - - protected: - std::shared_ptr redis_client_; + void TearDown() override {} }; TEST_F(RedisGcsTableStorageTest, TestGcsTableApi) { TestGcsTableApi(); } diff --git a/src/ray/gcs/pubsub/BUILD.bazel b/src/ray/gcs/pubsub/BUILD.bazel index 362186603fe2..5c61f477186a 100644 --- a/src/ray/gcs/pubsub/BUILD.bazel +++ b/src/ray/gcs/pubsub/BUILD.bazel @@ -7,7 +7,6 @@ ray_cc_library( deps = [ "//src/ray/common:ray_config", "//src/ray/gcs:gcs_callback", - "//src/ray/gcs:gcs_redis_client", "//src/ray/pubsub:publisher_interface", "//src/ray/pubsub:subscriber_interface", "//src/ray/rpc:gcs_client", diff --git a/src/ray/gcs/redis_client.cc b/src/ray/gcs/redis_client.cc deleted file mode 100644 index 4f547ac9cb1c..000000000000 --- a/src/ray/gcs/redis_client.cc +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ray/gcs/redis_client.h" - -#include - -#include "ray/common/ray_config.h" -#include "ray/gcs/redis_context.h" - -extern "C" { -#include "hiredis/hiredis.h" -} - -namespace ray { -namespace gcs { -RedisClient::RedisClient(const RedisClientOptions &options) : options_(options) {} - -Status RedisClient::Connect(instrumented_io_context &io_service) { - RAY_CHECK(!is_connected_); - - if (options_.server_ip_.empty()) { - RAY_LOG(ERROR) << "Failed to connect, redis server address is empty."; - return Status::Invalid("Redis server address is invalid!"); - } - - primary_context_ = std::make_unique(io_service); - - RAY_CHECK_OK(primary_context_->Connect(options_.server_ip_, - options_.server_port_, - /*username=*/options_.username_, - /*password=*/options_.password_, - /*enable_ssl=*/options_.enable_ssl_)); - - is_connected_ = true; - RAY_LOG(DEBUG) << "RedisClient connected."; - - return Status::OK(); -} - -void RedisClient::Disconnect() { - RAY_CHECK(is_connected_); - is_connected_ = false; - RAY_LOG(DEBUG) << "RedisClient disconnected."; -} -} // namespace gcs -} // namespace ray diff --git a/src/ray/gcs/redis_client.h b/src/ray/gcs/redis_client.h deleted file mode 100644 index d3cfcd655128..000000000000 --- a/src/ray/gcs/redis_client.h +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include - -#include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/status.h" -#include "ray/gcs/redis_context.h" -#include "ray/util/logging.h" - -namespace ray { -namespace gcs { -class RedisClientOptions { - public: - RedisClientOptions(const std::string &ip, - int port, - const std::string &username, - const std::string &password, - bool enable_ssl = false) - : server_ip_(ip), - server_port_(port), - username_(username), - password_(password), - enable_ssl_(enable_ssl) {} - - // Redis server address - std::string server_ip_; - int server_port_; - - // Username of Redis. - std::string username_; - - // Password of Redis. - std::string password_; - - // Whether to use tls/ssl for redis connection - bool enable_ssl_ = false; -}; - -/// \class RedisClient -/// This class is used to send commands to Redis. -class RedisClient { - public: - explicit RedisClient(const RedisClientOptions &options); - - /// Connect to Redis. Non-thread safe. - /// Call this function before calling other functions. - /// - /// \param io_service The event loop for this client. - /// This io_service must be single-threaded. Because `RedisAsioClient` is - /// non-thread safe. - /// \return Status - Status Connect(instrumented_io_context &io_service); - - /// Disconnect with Redis. Non-thread safe. - void Disconnect(); - - RedisContext *GetPrimaryContext() { return primary_context_.get(); } - - protected: - RedisClientOptions options_; - - /// Whether this client is connected to redis. - bool is_connected_{false}; - - // The following context writes everything to the primary shard - std::unique_ptr primary_context_; -}; -} // namespace gcs -} // namespace ray diff --git a/src/ray/gcs/store_client/BUILD.bazel b/src/ray/gcs/store_client/BUILD.bazel index 0be49eb13690..11bd87964695 100644 --- a/src/ray/gcs/store_client/BUILD.bazel +++ b/src/ray/gcs/store_client/BUILD.bazel @@ -1,7 +1,7 @@ load("//bazel:ray.bzl", "ray_cc_library") ray_cc_library( - name = "gcs_store_client", + name = "store_client", hdrs = ["store_client.h"], deps = [ "//src/ray/common:asio", @@ -12,25 +12,40 @@ ray_cc_library( ) ray_cc_library( - name = "gcs_redis_store_client", - srcs = ["redis_store_client.cc"], - hdrs = ["redis_store_client.h"], + name = "redis_store_client", + srcs = [ + "redis_async_context.cc", + "redis_context.cc", + "redis_store_client.cc", + ], + hdrs = [ + "redis_async_context.h", + "redis_context.h", + "redis_store_client.h", + ], deps = [ - ":gcs_store_client", + ":store_client", + "//:hiredis", + "//src/ray/common:asio", + "//src/ray/common:ray_config", + "//src/ray/common:status", "//src/ray/gcs:gcs_callback", - "//src/ray/gcs:gcs_redis_client", + "//src/ray/stats:stats_lib", "//src/ray/util:container_util", + "//src/ray/util:exponential_backoff", + "//src/ray/util:network_util", + "@boost//:asio", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/synchronization", ], ) ray_cc_library( - name = "gcs_in_memory_store_client", + name = "in_memory_store_client", srcs = ["in_memory_store_client.cc"], hdrs = ["in_memory_store_client.h"], deps = [ - ":gcs_store_client", + ":store_client", "//src/ray/common:asio", "//src/ray/gcs:gcs_callback", "//src/ray/util:concurrent_flat_map", @@ -39,11 +54,11 @@ ray_cc_library( ) ray_cc_library( - name = "gcs_observable_store_client", + name = "observable_store_client", srcs = ["observable_store_client.cc"], hdrs = ["observable_store_client.h"], deps = [ - ":gcs_store_client", + ":store_client", "//src/ray/gcs:gcs_callback", "//src/ray/util", ], diff --git a/src/ray/gcs/redis_async_context.cc b/src/ray/gcs/store_client/redis_async_context.cc similarity index 99% rename from src/ray/gcs/redis_async_context.cc rename to src/ray/gcs/store_client/redis_async_context.cc index 10df58cf5365..d022009b9a10 100644 --- a/src/ray/gcs/redis_async_context.cc +++ b/src/ray/gcs/store_client/redis_async_context.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/redis_async_context.h" +#include "ray/gcs/store_client/redis_async_context.h" #include #include diff --git a/src/ray/gcs/redis_async_context.h b/src/ray/gcs/store_client/redis_async_context.h similarity index 100% rename from src/ray/gcs/redis_async_context.h rename to src/ray/gcs/store_client/redis_async_context.h diff --git a/src/ray/gcs/redis_context.cc b/src/ray/gcs/store_client/redis_context.cc similarity index 99% rename from src/ray/gcs/redis_context.cc rename to src/ray/gcs/store_client/redis_context.cc index 3ee8c564e978..541c5edf9cc6 100644 --- a/src/ray/gcs/redis_context.cc +++ b/src/ray/gcs/store_client/redis_context.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/redis_context.h" +#include "ray/gcs/store_client/redis_context.h" #include #include diff --git a/src/ray/gcs/redis_context.h b/src/ray/gcs/store_client/redis_context.h similarity index 99% rename from src/ray/gcs/redis_context.h rename to src/ray/gcs/store_client/redis_context.h index cc3ec46f3c43..d66f3a04bf12 100644 --- a/src/ray/gcs/redis_context.h +++ b/src/ray/gcs/store_client/redis_context.h @@ -23,7 +23,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/status.h" -#include "ray/gcs/redis_async_context.h" +#include "ray/gcs/store_client/redis_async_context.h" #include "ray/stats/metric.h" #include "ray/stats/tag_defs.h" #include "ray/util/exponential_backoff.h" diff --git a/src/ray/gcs/store_client/redis_store_client.cc b/src/ray/gcs/store_client/redis_store_client.cc index 8863fb4f111f..7fe2065478e9 100644 --- a/src/ray/gcs/store_client/redis_store_client.cc +++ b/src/ray/gcs/store_client/redis_store_client.cc @@ -26,7 +26,7 @@ #include "absl/cleanup/cleanup.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" -#include "ray/gcs/redis_context.h" +#include "ray/common/ray_config.h" #include "ray/util/container_util.h" #include "ray/util/logging.h" @@ -118,14 +118,45 @@ void RedisStoreClient::MGetValues( } } -RedisStoreClient::RedisStoreClient(std::shared_ptr redis_client) - : external_storage_namespace_(::RayConfig::instance().external_storage_namespace()), - redis_client_(std::move(redis_client)) { +std::shared_ptr ConnectRedisContext(instrumented_io_context &io_service, + const RedisClientOptions &options) { + RAY_CHECK(!options.ip.empty()) << "Redis IP address cannot be empty."; + auto context = std::make_shared(io_service); + RAY_CHECK_OK(context->Connect(options.ip, + options.port, + /*username=*/options.username, + /*password=*/options.password, + /*enable_ssl=*/options.enable_ssl)) + << "Failed to connect to Redis."; + return context; +} + +RedisStoreClient::RedisStoreClient(instrumented_io_context &io_service, + const RedisClientOptions &options) + : io_service_(io_service), + options_(options), + external_storage_namespace_(::RayConfig::instance().external_storage_namespace()), + primary_context_(ConnectRedisContext(io_service, options)) { RAY_CHECK(!absl::StrContains(external_storage_namespace_, kClusterSeparator)) << "Storage namespace (" << external_storage_namespace_ << ") shouldn't contain " << kClusterSeparator << "."; + + // Health check Redis periodically and crash if it becomes unavailable. + periodic_health_check_runner_ = PeriodicalRunner::Create(io_service_); + periodic_health_check_runner_->RunFnPeriodically( + [this] { + AsyncCheckHealth({[](const Status &status) { + RAY_CHECK_OK(status) + << "Redis connection failed unexpectedly."; + }, + io_service_}); + }, + options.heartbeat_interval_ms, + "RedisStoreClient.redis_health_check"); } +RedisStoreClient::~RedisStoreClient() { periodic_health_check_runner_.reset(); } + Status RedisStoreClient::AsyncPut(const std::string &table_name, const std::string &key, std::string data, @@ -171,7 +202,7 @@ Status RedisStoreClient::AsyncGet( Status RedisStoreClient::AsyncGetAll( const std::string &table_name, Postable)> callback) { - RedisScanner::ScanKeysAndValues(redis_client_, + RedisScanner::ScanKeysAndValues(primary_context_, RedisKey{external_storage_namespace_, table_name}, RedisMatchPattern::Any(), std::move(callback)); @@ -275,7 +306,7 @@ void RedisStoreClient::SendRedisCmdWithKeys(std::vector keys, auto num_ready_keys = std::make_shared(0); std::function send_redis = [this, num_ready_keys = num_ready_keys, - concurrency_keys, // Copied! + concurrency_keys, command = std::move(command), redis_callback = std::move(redis_callback)]() mutable { @@ -289,23 +320,23 @@ void RedisStoreClient::SendRedisCmdWithKeys(std::vector keys, } } // Send the actual request - auto *cxt = redis_client_->GetPrimaryContext(); - cxt->RunArgvAsync(command.ToRedisArgs(), - [this, - concurrency_keys, // Copied! - redis_callback = std::move(redis_callback)](auto reply) { - std::vector> requests; - { - absl::MutexLock lock(&mu_); - requests = TakeRequestsFromSendingQueue(concurrency_keys); - } - for (auto &request : requests) { - request(); - } - if (redis_callback) { - redis_callback(reply); - } - }); + primary_context_->RunArgvAsync( + command.ToRedisArgs(), + [this, + concurrency_keys, // Copied! + redis_callback = std::move(redis_callback)](auto reply) { + std::vector> requests; + { + absl::MutexLock lock(&mu_); + requests = TakeRequestsFromSendingQueue(concurrency_keys); + } + for (auto &request : requests) { + request(); + } + if (redis_callback) { + redis_callback(reply); + } + }); }; { @@ -353,25 +384,25 @@ Status RedisStoreClient::DeleteByKeys(const std::string &table, RedisStoreClient::RedisScanner::RedisScanner( PrivateCtorTag ctor_tag, - std::shared_ptr redis_client, + std::shared_ptr primary_context, RedisKey redis_key, RedisMatchPattern match_pattern, Postable)> callback) : redis_key_(std::move(redis_key)), match_pattern_(std::move(match_pattern)), - redis_client_(std::move(redis_client)), + primary_context_(std::move(primary_context)), callback_(std::move(callback)) { cursor_ = 0; pending_request_count_ = 0; } void RedisStoreClient::RedisScanner::ScanKeysAndValues( - std::shared_ptr redis_client, + std::shared_ptr primary_context, RedisKey redis_key, RedisMatchPattern match_pattern, Postable)> callback) { auto scanner = std::make_shared(PrivateCtorTag(), - std::move(redis_client), + std::move(primary_context), std::move(redis_key), std::move(match_pattern), std::move(callback)); @@ -402,8 +433,7 @@ void RedisStoreClient::RedisScanner::Scan() { } command.args.push_back("COUNT"); command.args.push_back(std::to_string(batch_count)); - auto *primary_context = redis_client_->GetPrimaryContext(); - primary_context->RunArgvAsync( + primary_context_->RunArgvAsync( command.ToRedisArgs(), // self_ref to keep the scanner alive until the callback is called, even if it // releases its self_ref in Scan(). @@ -451,14 +481,13 @@ Status RedisStoreClient::AsyncGetNextJobID(Postable callback) { RedisCommand command = { "INCRBY", RedisKey{external_storage_namespace_, "JobCounter"}, {"1"}}; - auto *cxt = redis_client_->GetPrimaryContext(); - - cxt->RunArgvAsync(command.ToRedisArgs(), - [callback = std::move(callback)]( - const std::shared_ptr &reply) mutable { - auto job_id = static_cast(reply->ReadAsInteger()); - std::move(callback).Post("GcsStore.GetNextJobID", job_id); - }); + primary_context_->RunArgvAsync( + command.ToRedisArgs(), + [callback = + std::move(callback)](const std::shared_ptr &reply) mutable { + auto job_id = static_cast(reply->ReadAsInteger()); + std::move(callback).Post("GcsStore.GetNextJobID", job_id); + }); return Status::OK(); } @@ -467,7 +496,7 @@ Status RedisStoreClient::AsyncGetKeys(const std::string &table_name, const std::string &prefix, Postable)> callback) { RedisScanner::ScanKeysAndValues( - redis_client_, + primary_context_, RedisKey{external_storage_namespace_, table_name}, RedisMatchPattern::Prefix(prefix), std::move(callback).TransformArg( @@ -497,6 +526,21 @@ Status RedisStoreClient::AsyncExists(const std::string &table_name, return Status::OK(); } +void RedisStoreClient::AsyncCheckHealth(Postable callback) { + auto redis_callback = [callback = std::move(callback)]( + const std::shared_ptr &reply) mutable { + Status status = Status::OK(); + if (reply->IsNil()) { + status = Status::IOError("Unexpected connection error."); + } else if (reply->IsError()) { + status = reply->ReadAsStatus(); + } + std::move(callback).Dispatch("RedisStoreClient.AsyncCheckHealth", status); + }; + + primary_context_->RunArgvAsync({"PING"}, redis_callback); +} + // Returns True if at least 1 key is deleted, False otherwise. bool RedisDelKeyPrefixSync(const std::string &host, int32_t port, @@ -504,11 +548,10 @@ bool RedisDelKeyPrefixSync(const std::string &host, const std::string &password, bool use_ssl, const std::string &external_storage_namespace) { - RedisClientOptions options(host, port, username, password, use_ssl); - auto cli = std::make_unique(options); - instrumented_io_context io_service{/*enable_lag_probe=*/false, /*running_on_single_thread=*/true}; + RedisClientOptions options{host, port, username, password, use_ssl}; + std::shared_ptr context = ConnectRedisContext(io_service, options); auto thread = std::make_unique([&]() { boost::asio::executor_work_guard work( @@ -521,10 +564,6 @@ bool RedisDelKeyPrefixSync(const std::string &host, thread->join(); }); - auto status = cli->Connect(io_service); - RAY_CHECK_OK(status) << "Failed to connect to redis"; - - auto *context = cli->GetPrimaryContext(); // Delete all such keys by using empty table name. RedisKey redis_key{external_storage_namespace, /*table_name=*/""}; std::vector cmd{"KEYS", @@ -540,7 +579,7 @@ bool RedisDelKeyPrefixSync(const std::string &host, << external_storage_namespace; return true; } - auto delete_one_sync = [context](const std::string &key) { + auto delete_one_sync = [&context](const std::string &key) { auto del_cmd = std::vector{"DEL", key}; std::promise> promise; context->RunArgvAsync(del_cmd, diff --git a/src/ray/gcs/store_client/redis_store_client.h b/src/ray/gcs/store_client/redis_store_client.h index 39d73d16bfeb..241488fe7702 100644 --- a/src/ray/gcs/store_client/redis_store_client.h +++ b/src/ray/gcs/store_client/redis_store_client.h @@ -24,10 +24,11 @@ #include "absl/container/flat_hash_set.h" #include "absl/synchronization/mutex.h" +#include "ray/common/asio/asio_util.h" +#include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/asio/periodical_runner.h" #include "ray/common/asio/postable.h" -#include "ray/common/ray_config.h" -#include "ray/gcs/redis_client.h" -#include "ray/gcs/redis_context.h" +#include "ray/gcs/store_client/redis_context.h" #include "ray/gcs/store_client/store_client.h" #include "src/ray/protobuf/gcs.pb.h" @@ -90,7 +91,30 @@ inline std::ostream &operator<<(std::ostream &os, const RedisConcurrencyKey &key return os; } +struct RedisClientOptions { + // Redis server address. + std::string ip; + int port; + + // Redis username and password. + std::string username = ""; + std::string password = ""; + + // Whether to use TLS/SSL for the connection. + bool enable_ssl = false; + + // The interval between health checks to Redis. + // If a health check fails, the client will crash the process. + // Set to 0 to disable health checking. + uint64_t heartbeat_interval_ms = 1000; +}; + // StoreClient using Redis as persistence backend. +// +// The StoreClient does not currently handle any failures (transient or otherwise) of +// the Redis server. A periodic health check runs in the background and it will crash +// the process if the Redis server cannot be reached. +// // Note in redis term a "key" points to a hash table and a "field" is a key, a "value" // is just a value. We double quote "key" and "field" to avoid confusion. // @@ -110,7 +134,13 @@ inline std::ostream &operator<<(std::ostream &os, const RedisConcurrencyKey &key // [1] https://github.com/ray-project/ray/pull/35123#issuecomment-1546549046 class RedisStoreClient : public StoreClient { public: - explicit RedisStoreClient(std::shared_ptr redis_client); + /// Connect to Redis. Not thread safe. + /// + /// \param io_service The event loop for this client. Must be single threaded. + /// \param options The options for connecting to Redis. + explicit RedisStoreClient(instrumented_io_context &io_service, + const RedisClientOptions &options); + ~RedisStoreClient(); Status AsyncPut(const std::string &table_name, const std::string &key, @@ -150,6 +180,11 @@ class RedisStoreClient : public StoreClient { Postable callback) override; private: + // Check if Redis is available. + // + // \param callback The callback that will be called with a Status. OK means healthy. + void AsyncCheckHealth(Postable callback); + /// \class RedisScanner /// /// This class is used to HSCAN data from a Redis table. @@ -167,13 +202,13 @@ class RedisStoreClient : public StoreClient { // Don't call this. Use ScanKeysAndValues instead. explicit RedisScanner( PrivateCtorTag tag, - std::shared_ptr redis_client, + std::shared_ptr primary_context, RedisKey redis_key, RedisMatchPattern match_pattern, Postable)> callback); static void ScanKeysAndValues( - std::shared_ptr redis_client, + std::shared_ptr primary_context, RedisKey redis_key, RedisMatchPattern match_pattern, Postable)> callback); @@ -185,6 +220,7 @@ class RedisStoreClient : public StoreClient { void Scan(); void OnScanCallback(const std::shared_ptr &reply); + /// The table name that the scanner will scan. RedisKey redis_key_; @@ -204,7 +240,7 @@ class RedisStoreClient : public StoreClient { /// The pending shard scan count. std::atomic pending_request_count_{0}; - std::shared_ptr redis_client_; + std::shared_ptr primary_context_; Postable)> callback_; @@ -260,8 +296,17 @@ class RedisStoreClient : public StoreClient { const std::vector &keys, Postable)> callback); + instrumented_io_context &io_service_; + + RedisClientOptions options_; + std::string external_storage_namespace_; - std::shared_ptr redis_client_; + + // The following context writes everything to the primary shard. + std::shared_ptr primary_context_; + + std::shared_ptr periodic_health_check_runner_; + absl::Mutex mu_; // The pending redis requests queue for each key. diff --git a/src/ray/gcs/store_client/tests/BUILD.bazel b/src/ray/gcs/store_client/tests/BUILD.bazel index 6ceedbe51516..571e081c1a6c 100644 --- a/src/ray/gcs/store_client/tests/BUILD.bazel +++ b/src/ray/gcs/store_client/tests/BUILD.bazel @@ -5,7 +5,7 @@ ray_cc_library( hdrs = ["store_client_test_base.h"], deps = [ "//src/ray/common:test_util", - "//src/ray/gcs/store_client:gcs_redis_store_client", + "//src/ray/gcs/store_client", ], ) @@ -24,7 +24,7 @@ ray_cc_test( tags = ["team:core"], deps = [ ":store_client_test_lib", - "//src/ray/gcs/store_client:gcs_redis_store_client", + "//src/ray/gcs/store_client:redis_store_client", "@boost//:optional", "@com_google_googletest//:gtest_main", ], @@ -52,7 +52,7 @@ ray_cc_test( ], deps = [ ":store_client_test_lib", - "//src/ray/gcs/store_client:gcs_redis_store_client", + "//src/ray/gcs/store_client:redis_store_client", "@boost//:optional", "@com_google_googletest//:gtest_main", ], @@ -65,7 +65,7 @@ ray_cc_test( tags = ["team:core"], deps = [ ":store_client_test_lib", - "//src/ray/gcs/store_client:gcs_in_memory_store_client", + "//src/ray/gcs/store_client:in_memory_store_client", "@com_google_googletest//:gtest_main", ], ) @@ -77,8 +77,40 @@ ray_cc_test( tags = ["team:core"], deps = [ ":store_client_test_lib", - "//src/ray/gcs/store_client:gcs_in_memory_store_client", - "//src/ray/gcs/store_client:gcs_observable_store_client", + "//src/ray/gcs/store_client:in_memory_store_client", + "//src/ray/gcs/store_client:observable_store_client", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "redis_callback_reply_test", + size = "small", + srcs = ["redis_callback_reply_test.cc"], + tags = ["team:core"], + deps = [ + "//src/ray/gcs/store_client:redis_store_client", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "redis_async_context_test", + size = "small", + srcs = ["redis_async_context_test.cc"], + args = [ + "$(location //:redis-server)", + "$(location //:redis-cli)", + ], + data = [ + "//:redis-cli", + "//:redis-server", + ], + tags = ["team:core"], + deps = [ + "//src/ray/common:test_util", + "//src/ray/gcs/store_client:redis_store_client", + "//src/ray/util", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/gcs/tests/redis_async_context_test.cc b/src/ray/gcs/store_client/tests/redis_async_context_test.cc similarity index 96% rename from src/ray/gcs/tests/redis_async_context_test.cc rename to src/ray/gcs/store_client/tests/redis_async_context_test.cc index 6309e867995a..94ebe207ee39 100644 --- a/src/ray/gcs/tests/redis_async_context_test.cc +++ b/src/ray/gcs/store_client/tests/redis_async_context_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/redis_async_context.h" +#include "ray/gcs/store_client/redis_async_context.h" #include #include @@ -21,7 +21,7 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/test_util.h" -#include "ray/gcs/redis_context.h" +#include "ray/gcs/store_client/redis_context.h" #include "ray/util/logging.h" #include "ray/util/path_utils.h" diff --git a/src/ray/gcs/tests/callback_reply_test.cc b/src/ray/gcs/store_client/tests/redis_callback_reply_test.cc similarity index 98% rename from src/ray/gcs/tests/callback_reply_test.cc rename to src/ray/gcs/store_client/tests/redis_callback_reply_test.cc index c6221e42d2ec..ad8e5b3ee48c 100644 --- a/src/ray/gcs/tests/callback_reply_test.cc +++ b/src/ray/gcs/store_client/tests/redis_callback_reply_test.cc @@ -16,7 +16,7 @@ #include #include "gtest/gtest.h" -#include "ray/gcs/redis_context.h" +#include "ray/gcs/store_client/redis_context.h" extern "C" { #include "hiredis/hiredis.h" diff --git a/src/ray/gcs/store_client/tests/redis_store_client_test.cc b/src/ray/gcs/store_client/tests/redis_store_client_test.cc index dcf53b4f6f26..e094f6c31cbc 100644 --- a/src/ray/gcs/store_client/tests/redis_store_client_test.cc +++ b/src/ray/gcs/store_client/tests/redis_store_client_test.cc @@ -23,7 +23,6 @@ #include #include "ray/common/test_util.h" -#include "ray/gcs/redis_client.h" #include "ray/gcs/store_client/tests/store_client_test_base.h" #include "ray/util/path_utils.h" @@ -73,17 +72,14 @@ class RedisStoreClientTest : public StoreClientTestBase { } void InitStoreClient() override { - RedisClientOptions options("127.0.0.1", TEST_REDIS_SERVER_PORTS.front(), "", ""); - redis_client_ = std::make_shared(options); - RAY_CHECK_OK(redis_client_->Connect(*io_service_pool_->Get())); - - store_client_ = std::make_shared(redis_client_); + auto &io_context = *io_service_pool_->Get(); + RedisClientOptions options{"127.0.0.1", TEST_REDIS_SERVER_PORTS.front()}; + store_client_ = std::make_shared(io_context, options); } - void DisconnectStoreClient() override { redis_client_->Disconnect(); } + void DisconnectStoreClient() override { store_client_.reset(); } protected: - std::shared_ptr redis_client_; std::unique_ptr t_; std::atomic stopped_ = false; }; diff --git a/src/ray/gcs/tests/BUILD.bazel b/src/ray/gcs/tests/BUILD.bazel index c89258455258..8485c8c5a794 100644 --- a/src/ray/gcs/tests/BUILD.bazel +++ b/src/ray/gcs/tests/BUILD.bazel @@ -1,4 +1,4 @@ -load("//bazel:ray.bzl", "ray_cc_library", "ray_cc_test") +load("//bazel:ray.bzl", "ray_cc_library") ray_cc_library( name = "gcs_test_util_lib", @@ -12,35 +12,3 @@ ray_cc_library( "//src/ray/protobuf:gcs_service_cc_grpc", ], ) - -ray_cc_test( - name = "callback_reply_test", - size = "small", - srcs = ["callback_reply_test.cc"], - tags = ["team:core"], - deps = [ - "//src/ray/gcs:gcs_redis_client", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "redis_async_context_test", - size = "small", - srcs = ["redis_async_context_test.cc"], - args = [ - "$(location //:redis-server)", - "$(location //:redis-cli)", - ], - data = [ - "//:redis-cli", - "//:redis-server", - ], - tags = ["team:core"], - deps = [ - "//src/ray/common:test_util", - "//src/ray/gcs:gcs_redis_client", - "//src/ray/util", - "@com_google_googletest//:gtest_main", - ], -) From 4afc68fb9dcdbba540b3bbe01a65abe84d7d702e Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 19 Aug 2025 15:30:14 -0500 Subject: [PATCH 183/634] [core] Remove `_TestOnly_GcsActorSubscriber` (#55738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This class was only used in a single test, that appeared to be testing its own logic 🤔 Remove to avoid proliferation. --------- Signed-off-by: Edward Oakes --- python/ray/_raylet.pyx | 41 -------- .../modules/node/tests/test_actor.py | 93 ------------------- python/ray/includes/common.pxd | 3 - src/ray/gcs/pubsub/gcs_pub_sub.cc | 10 -- src/ray/gcs/pubsub/gcs_pub_sub.h | 3 - 5 files changed, 150 deletions(-) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index ad030028441d..83798c1da252 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -2972,47 +2972,6 @@ cdef class GcsLogSubscriber(_GcsSubscriber): return result -# This class should only be used for tests -cdef class _TestOnly_GcsActorSubscriber(_GcsSubscriber): - """Subscriber to actor updates. Thread safe. - - Usage example: - subscriber = GcsActorSubscriber() - # Subscribe to the actor channel. - subscriber.subscribe() - ... - while running: - actor_data = subscriber.poll() - ...... - # Unsubscribe from the channel. - subscriber.close() - """ - - def __init__(self, address, worker_id=None): - self._construct(address, GCS_ACTOR_CHANNEL, worker_id) - - def poll(self, timeout=None): - """Polls for new actor messages. - - Returns: - A byte string of function key. - None if polling times out or subscriber closed. - """ - cdef: - CActorTableData actor_data - c_string key_id - int64_t timeout_ms = round(1000 * timeout) if timeout else -1 - - with nogil: - check_status(self.inner.get().PollActor( - &key_id, timeout_ms, &actor_data)) - - info = ActorTableData.FromString( - actor_data.SerializeAsString()) - - return [(key_id, info)] - - cdef class CoreWorker: def __cinit__(self, worker_type, store_socket, raylet_socket, diff --git a/python/ray/dashboard/modules/node/tests/test_actor.py b/python/ray/dashboard/modules/node/tests/test_actor.py index e4afeb0b818d..e374a28a2c0a 100644 --- a/python/ray/dashboard/modules/node/tests/test_actor.py +++ b/python/ray/dashboard/modules/node/tests/test_actor.py @@ -7,7 +7,6 @@ import requests import ray -import ray.dashboard.utils as dashboard_utils from ray._private.test_utils import format_web_url, wait_until_server_available from ray.dashboard.modules.node import actor_consts from ray.dashboard.tests.conftest import * # noqa @@ -239,98 +238,6 @@ def get_actor_id(self): raise Exception(f"Timed out while testing, {ex_stack}") -def test_actor_pubsub(disable_aiohttp_cache, ray_start_with_dashboard): - timeout = 5 - assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) - address_info = ray_start_with_dashboard - - sub = ray._raylet._TestOnly_GcsActorSubscriber(address=address_info["gcs_address"]) - sub.subscribe() - - @ray.remote - class DummyActor: - def __init__(self): - pass - - # Create a dummy actor. - a = DummyActor.remote() - - def handle_pub_messages(msgs, timeout, expect_num): - start_time = time.time() - while time.time() - start_time < timeout and len(msgs) < expect_num: - published = sub.poll(timeout=timeout) - for _, actor_data in published: - if actor_data is None: - continue - msgs.append(actor_data) - - msgs = [] - handle_pub_messages(msgs, timeout, 3) - # Assert we received published actor messages with state - # DEPENDENCIES_UNREADY, PENDING_CREATION and ALIVE. - assert len(msgs) == 3, msgs - - # Kill actor. - ray.kill(a) - handle_pub_messages(msgs, timeout, 4) - - # Assert we received published actor messages with state DEAD. - assert len(msgs) == 4 - - def actor_table_data_to_dict(message): - return dashboard_utils.message_to_dict( - message, - { - "actorId", - "parentId", - "jobId", - "workerId", - "nodeId", - "callerId", - "taskId", - "parentTaskId", - "sourceActorId", - "placementGroupId", - }, - always_print_fields_with_no_presence=False, - ) - - non_state_keys = ("actorId", "jobId") - - for msg in msgs: - actor_data_dict = actor_table_data_to_dict(msg) - # DEPENDENCIES_UNREADY is 0, which would not be kept in dict. We - # need check its original value. - if msg.state == 0: - assert len(actor_data_dict) > 5 - for k in non_state_keys: - assert k in actor_data_dict - # For status that is not DEPENDENCIES_UNREADY, only states fields will - # be published. - elif actor_data_dict["state"] in ("ALIVE", "DEAD"): - assert actor_data_dict.keys() >= { - "state", - "address", - "timestamp", - "pid", - "rayNamespace", - } - elif actor_data_dict["state"] == "PENDING_CREATION": - assert actor_data_dict.keys() == { - "state", - "address", - "actorId", - "jobId", - "ownerAddress", - "className", - "serializedRuntimeEnv", - "rayNamespace", - "functionDescriptor", - } - else: - raise Exception("Unknown state: {}".format(actor_data_dict["state"])) - - def test_nil_node(enable_test_module, disable_aiohttp_cache, ray_start_with_dashboard): assert wait_until_server_available(ray_start_with_dashboard["webui_url"]) is True webui_url = ray_start_with_dashboard["webui_url"] diff --git a/python/ray/includes/common.pxd b/python/ray/includes/common.pxd index ab24ec1622ce..82d14d92e63a 100644 --- a/python/ray/includes/common.pxd +++ b/python/ray/includes/common.pxd @@ -662,9 +662,6 @@ cdef extern from "ray/gcs/pubsub/gcs_pub_sub.h" nogil: CRayStatus PollLogs( c_string* key_id, int64_t timeout_ms, CLogBatch* data) - CRayStatus PollActor( - c_string* key_id, int64_t timeout_ms, CActorTableData* data) - CRayStatus Close() cdef extern from "ray/gcs/pubsub/gcs_pub_sub.h" namespace "ray::gcs" nogil: diff --git a/src/ray/gcs/pubsub/gcs_pub_sub.cc b/src/ray/gcs/pubsub/gcs_pub_sub.cc index ea2884d29306..354775796205 100644 --- a/src/ray/gcs/pubsub/gcs_pub_sub.cc +++ b/src/ray/gcs/pubsub/gcs_pub_sub.cc @@ -319,16 +319,6 @@ Status PythonGcsSubscriber::PollLogs(std::string *key_id, return Status::OK(); } -Status PythonGcsSubscriber::PollActor(std::string *key_id, - int64_t timeout_ms, - rpc::ActorTableData *data) { - rpc::PubMessage message; - RAY_RETURN_NOT_OK(DoPoll(timeout_ms, &message)); - *key_id = std::move(*message.mutable_key_id()); - *data = std::move(*message.mutable_actor_message()); - return Status::OK(); -} - Status PythonGcsSubscriber::Close() { std::shared_ptr current_polling_context; { diff --git a/src/ray/gcs/pubsub/gcs_pub_sub.h b/src/ray/gcs/pubsub/gcs_pub_sub.h index f5b8ddead4f3..abc40f733bbc 100644 --- a/src/ray/gcs/pubsub/gcs_pub_sub.h +++ b/src/ray/gcs/pubsub/gcs_pub_sub.h @@ -137,9 +137,6 @@ class RAY_EXPORT PythonGcsSubscriber { /// Polls for new log messages. Status PollLogs(std::string *key_id, int64_t timeout_ms, rpc::LogBatch *data); - /// Polls for actor messages. - Status PollActor(std::string *key_id, int64_t timeout_ms, rpc::ActorTableData *data); - /// Closes the subscriber and its active subscription. Status Close(); From 53387138a1320f31e64c38290d5053aaaa13f591 Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Tue, 19 Aug 2025 13:57:58 -0700 Subject: [PATCH 184/634] [core][GPU Objects] Support nixl as tensor transport backend (#54459) Support nixl as tensor transport backend when using gpu objects. The code is tested manually. Related tests may be put in next pr since integrating nixl in ci needs some work to do. --------- Signed-off-by: Stephanie Wang Signed-off-by: kaihsun Co-authored-by: Stephanie Wang Co-authored-by: kaihsun --- python/ray/_private/custom_types.py | 1 + .../ray/experimental/collective/__init__.py | 3 + .../collective/collective_tensor_transport.py | 149 ++++++++++++++++ .../collective/nixl_tensor_transport.py | 118 +++++++++++++ .../collective/tensor_transport_manager.py | 160 ++++++++++++++++++ python/ray/experimental/collective/util.py | 36 ++++ .../gpu_object_manager/gpu_object_manager.py | 138 +++++---------- .../gpu_object_manager/gpu_object_store.py | 70 ++++---- python/ray/tests/BUILD | 1 + python/ray/tests/test_gpu_objects_nixl.py | 46 +++++ python/ray/util/collective/collective.py | 50 ++++-- .../collective_group/nixl_backend.py | 96 +++++++++++ python/ray/util/collective/types.py | 69 ++++++++ .../requirements/ml/dl-gpu-requirements.txt | 1 + src/ray/protobuf/common.proto | 2 + 15 files changed, 799 insertions(+), 141 deletions(-) create mode 100644 python/ray/experimental/collective/collective_tensor_transport.py create mode 100644 python/ray/experimental/collective/nixl_tensor_transport.py create mode 100644 python/ray/experimental/collective/tensor_transport_manager.py create mode 100644 python/ray/tests/test_gpu_objects_nixl.py create mode 100644 python/ray/util/collective/collective_group/nixl_backend.py diff --git a/python/ray/_private/custom_types.py b/python/ray/_private/custom_types.py index 2237972c3b6b..9b20312ec3c1 100644 --- a/python/ray/_private/custom_types.py +++ b/python/ray/_private/custom_types.py @@ -125,6 +125,7 @@ class TensorTransportEnum(Enum): OBJECT_STORE = TensorTransport.Value("OBJECT_STORE") NCCL = TensorTransport.Value("NCCL") GLOO = TensorTransport.Value("GLOO") + NIXL = TensorTransport.Value("NIXL") @classmethod def from_str(cls, name: str) -> "TensorTransportEnum": diff --git a/python/ray/experimental/collective/__init__.py b/python/ray/experimental/collective/__init__.py index de2314acf6ab..8a526e691684 100644 --- a/python/ray/experimental/collective/__init__.py +++ b/python/ray/experimental/collective/__init__.py @@ -9,6 +9,8 @@ destroy_collective_group, destroy_all_collective_groups, ) +from ray.experimental.collective.util import get_tensor_transport_manager + __all__ = [ "allgather", @@ -18,4 +20,5 @@ "create_collective_group", "destroy_collective_group", "destroy_all_collective_groups", + "get_tensor_transport_manager", ] diff --git a/python/ray/experimental/collective/collective_tensor_transport.py b/python/ray/experimental/collective/collective_tensor_transport.py new file mode 100644 index 000000000000..d4dece6f1808 --- /dev/null +++ b/python/ray/experimental/collective/collective_tensor_transport.py @@ -0,0 +1,149 @@ +from typing import Optional, List, TYPE_CHECKING + +import ray +from ray.experimental.collective.tensor_transport_manager import ( + TensorTransportManager, + TensorTransportEnum, +) + +from ray.util.collective.types import ( + CollectiveTransportMetadata, + CollectiveCommunicatorMetadata, +) + +if TYPE_CHECKING: + import torch + + +class CollectiveTensorTransport(TensorTransportManager): + @staticmethod + def is_one_sided() -> bool: + return False + + @staticmethod + def get_tensor_transport_metadata( + src_actor: "ray.actor.ActorHandle", + obj_id: str, + tensor_transport: TensorTransportEnum, + ) -> CollectiveTransportMetadata: + def __ray_get_tensor_transport_metadata__( + self: "ray.actor.ActorHandle", + obj_id: str, + tensor_transport: TensorTransportEnum, + ) -> CollectiveTransportMetadata: + + from ray._private.worker import global_worker + from ray.util.collective.types import CollectiveTransportMetadata + + gpu_object_store = global_worker.gpu_object_manager.gpu_object_store + # NOTE: We do not specify a timeout here because the user task that returns + # it could take arbitrarily long and we don't want to trigger a spurious + # timeout. + gpu_object = gpu_object_store.wait_and_get_object(obj_id) + return CollectiveTransportMetadata( + tensor_meta=[(t.shape, t.dtype) for t in gpu_object], + ) + + # Submit a Ray actor task to the source actor to get the tensor metadata. + # The metadata is a list of tuples, where each tuple contains the shape and dtype + # of a tensor in the GPU object store. This function returns an ObjectRef that + # points to the tensor metadata. + # NOTE(swang): We put this task on the background thread to avoid tasks + # executing on the main thread blocking this task. + + return src_actor.__ray_call__.options(concurrency_group="_ray_system").remote( + __ray_get_tensor_transport_metadata__, obj_id, tensor_transport + ) + + @staticmethod + def get_communicator_metadata( + src_actor: "ray.actor.ActorHandle", + dst_actor: "ray.actor.ActorHandle", + backend: Optional[str] = None, + ) -> CollectiveCommunicatorMetadata: + + from ray.experimental.collective import get_collective_groups + + communicators = get_collective_groups( + [src_actor, dst_actor], + backend=backend, + ) + # TODO(kevin85421): Support multiple communicators. + if len(communicators) == 0: + raise ValueError( + f"No communicators found for actors {src_actor} and {dst_actor}. " + "Create a communicator with " + "`ray.experimental.collective.create_collective_group` " + "before calling actor tasks." + ) + elif len(communicators) > 1: + raise ValueError( + f"There are {len(communicators)} possible communicators that contain actors {src_actor} and {dst_actor}. " + "Currently, GPU objects only support one communicator. Please make sure only " + "one communicator exists." + ) + communicator = communicators[0] + src_rank = communicator.get_rank(src_actor) + if src_rank == -1: + raise ValueError( + f"Sender actor {src_actor} not found in communicator. " + "Please make sure the sender and receiver are in the same communicator." + ) + dst_rank = communicator.get_rank(dst_actor) + if dst_rank == -1: + raise ValueError( + f"Receiver actor {dst_actor} not found in communicator. " + "Please make sure the sender and receiver are in the same communicator." + ) + + communicator_metadata = CollectiveCommunicatorMetadata( + communicator_name=communicator.name, + src_rank=src_rank, + dst_rank=dst_rank, + ) + return communicator_metadata + + @staticmethod + def recv_multiple_tensors( + tensors, + tensor_transport_metadata: CollectiveTransportMetadata, + communicator_metadata: CollectiveCommunicatorMetadata, + ): + from ray.util.collective import types + from ray.util.collective.collective import recv + + assert isinstance( + tensor_transport_metadata, types.CollectiveTransportMetadata + ), "metadata must be a CollectiveTransportMetadata object for non-NIXL transport" + assert isinstance( + communicator_metadata, types.CollectiveCommunicatorMetadata + ), "metadata must be a CollectiveCommunicatorMetadata object for non-NIXL transport" + + for tensor in tensors: + recv( + tensor, + communicator_metadata.src_rank, + communicator_metadata.communicator_name, + ) + + @staticmethod + def send_multiple_tensors( + tensors: List["torch.Tensor"], + tensor_transport_metadata: CollectiveTransportMetadata, + communicator_metadata: CollectiveCommunicatorMetadata, + device: "torch.device", + ): + import ray.util.collective as collective + + for tensor in tensors: + if tensor.device.type != device.type: + # TODO(swang): Right now there is no way to catch this error + # and the receiving Ray task will hang. + raise ValueError( + f"tensor device {tensor.device} does not match device {device}" + ) + collective.send( + tensor, + communicator_metadata.dst_rank, + communicator_metadata.communicator_name, + ) diff --git a/python/ray/experimental/collective/nixl_tensor_transport.py b/python/ray/experimental/collective/nixl_tensor_transport.py new file mode 100644 index 000000000000..5723e7b1d686 --- /dev/null +++ b/python/ray/experimental/collective/nixl_tensor_transport.py @@ -0,0 +1,118 @@ +from typing import Optional, List, TYPE_CHECKING + +import ray +from ray.experimental.collective.tensor_transport_manager import ( + TensorTransportManager, + TensorTransportEnum, +) +from ray.util.collective.types import ( + NIXL_GROUP_NAME, + NixlTransportMetadata, + NixlCommunicatorMetadata, +) + +if TYPE_CHECKING: + import torch + + +class NixlTensorTransport(TensorTransportManager): + @staticmethod + def is_one_sided() -> bool: + return True + + @staticmethod + def get_tensor_transport_metadata( + src_actor: "ray.actor.ActorHandle", + obj_id: str, + tensor_transport: TensorTransportEnum, + ) -> NixlTransportMetadata: + from ray.util.collective.collective_group.nixl_backend import NixlBackend + + def __ray_get_tensor_transport_metadata__( + self: "ray.actor.ActorHandle", + obj_id: str, + tensor_transport: TensorTransportEnum, + ) -> NixlTransportMetadata: + + from ray._private.worker import global_worker + from ray.util.collective.types import NixlTransportMetadata + + gpu_object_store = global_worker.gpu_object_manager.gpu_object_store + # NOTE: We do not specify a timeout here because the user task that returns + # it could take arbitrarily long and we don't want to trigger a spurious + # timeout. + gpu_object = gpu_object_store.wait_and_get_object(obj_id) + from ray.util.collective.collective import get_group_handle + + nixl_backend: NixlBackend = get_group_handle(NIXL_GROUP_NAME) + if gpu_object: + serialized_descs, agent_meta = nixl_backend.get_nixl_metadata( + gpu_object + ) + else: + serialized_descs, agent_meta = None, None + return NixlTransportMetadata( + tensor_meta=[(t.shape, t.dtype) for t in gpu_object], + nixl_serialized_descs=serialized_descs, + nixl_agent_meta=agent_meta, + ) + + # Submit a Ray actor task to the source actor to get the tensor metadata. + # The metadata is a list of tuples, where each tuple contains the shape and dtype + # of a tensor in the GPU object store. This function returns an ObjectRef that + # points to the tensor metadata. + # NOTE(swang): We put this task on the background thread to avoid tasks + # executing on the main thread blocking this task. + + return src_actor.__ray_call__.options(concurrency_group="_ray_system").remote( + __ray_get_tensor_transport_metadata__, obj_id, tensor_transport + ) + + @staticmethod + def get_communicator_metadata( + src_actor: "ray.actor.ActorHandle", + dst_actor: "ray.actor.ActorHandle", + backend: Optional[str] = None, + ) -> NixlCommunicatorMetadata: + + communicator_metadata = NixlCommunicatorMetadata( + communicator_name=NIXL_GROUP_NAME, + ) + + return communicator_metadata + + @staticmethod + def recv_multiple_tensors( + tensors, + tensor_transport_metadata: NixlTransportMetadata, + communicator_metadata: NixlCommunicatorMetadata, + ): + from ray.util.collective.collective import get_group_handle + from ray.util.collective import types + + if tensors: + g = get_group_handle(communicator_metadata.communicator_name) + + assert isinstance( + tensor_transport_metadata, types.NixlTransportMetadata + ), "metadata must be a NixlTransportMetadata object for NIXL transport" + assert isinstance( + communicator_metadata, types.NixlCommunicatorMetadata + ), "metadata must be a NixlCommunicatorMetadata object for NIXL transport" + + g.recv( + tensors, + tensor_transport_metadata.nixl_serialized_descs, + tensor_transport_metadata.nixl_agent_meta, + ) + + @staticmethod + def send_multiple_tensors( + tensors: List["torch.Tensor"], + tensor_transport_metadata: NixlTransportMetadata, + communicator_metadata: NixlCommunicatorMetadata, + device: "torch.device", + ): + raise NotImplementedError( + "NIXL transport does not support send_multiple_tensors, since it is a one-sided transport." + ) diff --git a/python/ray/experimental/collective/tensor_transport_manager.py b/python/ray/experimental/collective/tensor_transport_manager.py new file mode 100644 index 000000000000..302b6998b699 --- /dev/null +++ b/python/ray/experimental/collective/tensor_transport_manager.py @@ -0,0 +1,160 @@ +from abc import ABC, abstractmethod +from typing import List, Optional, TYPE_CHECKING +from ray.util.collective.types import TensorTransportMetadata, CommunicatorMetadata +from ray._private.custom_types import TensorTransportEnum + +import ray + +if TYPE_CHECKING: + import torch + + +class TensorTransportManager(ABC): + @staticmethod + @abstractmethod + def is_one_sided() -> bool: + """Whether the backend is one-sided. + + Returns: + bool: True if the backend is one-sided, False otherwise. + """ + + @staticmethod + @abstractmethod + def get_tensor_transport_metadata( + src_actor: "ray.actor.ActorHandle", + obj_id: str, + tensor_transport: TensorTransportEnum, + ) -> TensorTransportMetadata: + """ + Get the tensor transport metadata for the GPU object. + This function retrieves metadata about tensors stored in the GPU object store, + including their shapes, dtypes, and any transport-specific metadata, e.g., NIXL descriptors. + + Args: + src_actor: The actor that runs this function. + obj_id: The ID of the GPU object to get metadata for + tensor_transport: The tensor transport protocol to use for the GPU object. + + Returns: + TensorTransportMetadata: A named tuple containing the tensor metadata. + """ + + @staticmethod + @abstractmethod + def get_communicator_metadata( + src_actor: "ray.actor.ActorHandle", + dst_actor: "ray.actor.ActorHandle", + backend: Optional[str] = None, + ) -> CommunicatorMetadata: + """ + Get the communicator metadata (e.g. communicator name, src/dst rank) for the send/recv operation. + This function is called before sending the GPU object. + + Args: + src_actor: The actor that runs this function. + dst_actor: The actor that runs this function. + backend: The backend to use for the collective operation. + + Returns: + CommunicatorMetadata: The communicator metadata. + """ + + @staticmethod + def send_object( + src_actor: "ray.actor.ActorHandle", + obj_id: str, + tensor_transport_metadata_ref: TensorTransportMetadata, + communicator_metadata_ref: CommunicatorMetadata, + ): + """ + Send the GPU object to the destination actor. + + Args: + src_actor: The actor that runs this function. + obj_id: The ID of the GPU object to send. + tensor_transport_metadata_ref: The ObjectRef of tensor transport metadata for the GPU object. + communicator_metadata_ref: The ObjectRef of communicator metadata for the send/recv operation. + """ + from ray.experimental.gpu_object_manager.gpu_object_store import __ray_send__ + + # Send tensors stored in the `src_actor`'s GPU object store to the + # destination rank `dst_rank`. + # NOTE(swang): We put this task on the background thread to avoid tasks + # executing on the main thread blocking the data transfer. + src_actor.__ray_call__.options(concurrency_group="_ray_system").remote( + __ray_send__, + obj_id, + tensor_transport_metadata_ref, + communicator_metadata_ref, + ) + + @staticmethod + def recv_object( + dst_actor: "ray.actor.ActorHandle", + obj_id: str, + tensor_transport_metadata_ref: TensorTransportMetadata, + communicator_metadata_ref: CommunicatorMetadata, + ): + """ + Receive the GPU object from the source actor. + This function receives tensors from the source rank and stores them in the + `dst_actor`'s GPU object store. + + Args: + dst_actor: The actor that runs this function. + obj_id: The ID of the GPU object to receive. + tensor_transport_metadata_ref: The ObjectRef of tensor transport metadata for the GPU object. + communicator_metadata_ref: The ObjectRef of communicator metadata for the send/recv operation. + """ + from ray.experimental.gpu_object_manager.gpu_object_store import __ray_recv__ + + # Receive tensors from the source rank and store them in the + # `dst_actor`'s GPU object store. + # + # NOTE(swang): We put this task on the background thread to avoid tasks + # executing on the main thread blocking the data transfer. Technically, + # this is only needed for the sender task, but we put the receiver task + # on the same background thread to ensure that all communication + # operations are executed in a global order. + dst_actor.__ray_call__.options(concurrency_group="_ray_system").remote( + __ray_recv__, + obj_id, + tensor_transport_metadata_ref, + communicator_metadata_ref, + ) + + @staticmethod + @abstractmethod + def recv_multiple_tensors( + tensors: List["torch.Tensor"], + tensor_transport_metadata: TensorTransportMetadata, + communicator_metadata: CommunicatorMetadata, + ): + """ + Receive multiple tensors from the source actor. + + Args: + tensors: The pre-allocated tensor space to receive the tensors. + tensor_transport_metadata: The tensor transport metadata for the GPU object. + communicator_metadata: The communicator metadata for the send/recv operation. + + """ + + @staticmethod + @abstractmethod + def send_multiple_tensors( + tensors: List["torch.Tensor"], + tensor_transport_metadata: TensorTransportMetadata, + communicator_metadata: CommunicatorMetadata, + device: "torch.device", + ): + """ + Send multiple tensors to the destination actor. + + Args: + tensors: The tensors to send. + tensor_transport_metadata: The tensor transport metadata for the GPU object. + communicator_metadata: The communicator metadata for the send/recv operation. + device: The device to send the tensors to. + """ diff --git a/python/ray/experimental/collective/util.py b/python/ray/experimental/collective/util.py index ea518a002458..fc6ef64229fb 100644 --- a/python/ray/experimental/collective/util.py +++ b/python/ray/experimental/collective/util.py @@ -4,6 +4,42 @@ import ray +from ray.util.collective.types import Backend +from ray.experimental.collective.tensor_transport_manager import TensorTransportManager +from ray.experimental.collective.nixl_tensor_transport import NixlTensorTransport +from ray.experimental.collective.collective_tensor_transport import ( + CollectiveTensorTransport, +) + +# Singleton instances for tensor transport managers +_nixl_tensor_transport_manager = None +_collective_tensor_transport_manager = None + + +def get_tensor_transport_manager( + tensor_transport: Backend, +) -> "TensorTransportManager": + """Get the tensor transport manager for the given tensor transport protocol. + + Args: + tensor_transport: The tensor transport protocol to use for the GPU object. + + Returns: + TensorTransportManager: The tensor transport manager for the given tensor transport protocol. + """ + if tensor_transport == Backend.NIXL: + global _nixl_tensor_transport_manager + if _nixl_tensor_transport_manager is None: + _nixl_tensor_transport_manager = NixlTensorTransport() + return _nixl_tensor_transport_manager + elif tensor_transport == Backend.TORCH_GLOO or tensor_transport == Backend.NCCL: + global _collective_tensor_transport_manager + if _collective_tensor_transport_manager is None: + _collective_tensor_transport_manager = CollectiveTensorTransport() + return _collective_tensor_transport_manager + else: + raise ValueError(f"Unsupported tensor transport protocol: {tensor_transport}") + def find_free_port() -> int: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py index c45ff36572aa..7d7f031269e6 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, List import threading import ray @@ -6,12 +6,12 @@ from ray._raylet import ObjectRef from ray._private import ray_constants - if TYPE_CHECKING: - import torch from ray.experimental.gpu_object_manager.gpu_object_store import ( GPUObjectStore, ) + from ray.util.collective.types import TensorTransportMetadata + import torch # GPUObjectMeta is a named tuple containing the source actor, tensor transport # backend, and tensor metadata. @@ -24,7 +24,7 @@ class GPUObjectMeta(NamedTuple): # Must be a valid backend name as defined in # `ray.util.collective.types.Backend`. tensor_transport_backend: str - tensor_meta: List[Tuple["torch.Size", "torch.dtype"]] + tensor_transport_meta: "TensorTransportMetadata" # TODO(swang): Uncomment and add an API docs page and example usage. @@ -77,23 +77,6 @@ def gpu_object_store(self) -> "ray.experimental.GPUObjectStore": self._gpu_object_store = GPUObjectStore() return self._gpu_object_store - def _get_tensor_meta( - self, src_actor: "ray.actor.ActorHandle", obj_id: str - ) -> ObjectRef: - from ray.experimental.gpu_object_manager.gpu_object_store import ( - __ray_get_tensor_meta__, - ) - - # Submit a Ray actor task to the source actor to get the tensor metadata. - # The metadata is a list of tuples, where each tuple contains the shape and dtype - # of a tensor in the GPU object store. This function returns an ObjectRef that - # points to the tensor metadata. - # NOTE(swang): We put this task on the background thread to avoid tasks - # executing on the main thread blocking this task. - return src_actor.__ray_call__.options(concurrency_group="_ray_system").remote( - __ray_get_tensor_meta__, obj_id - ) - def is_managed_object(self, obj_id: str) -> bool: """ Check if the GPU object is managed by this process. @@ -125,61 +108,28 @@ def add_gpu_object_ref( from ray.experimental.gpu_object_manager.gpu_object_store import ( _tensor_transport_to_collective_backend, ) + from ray.experimental.collective import get_tensor_transport_manager tensor_transport_backend = _tensor_transport_to_collective_backend( tensor_transport ) obj_id = obj_ref.hex() - tensor_meta = self._get_tensor_meta(src_actor, obj_id) + tensor_transport_manager = get_tensor_transport_manager( + tensor_transport_backend + ) + tensor_meta = tensor_transport_manager.get_tensor_transport_metadata( + src_actor, obj_id, tensor_transport + ) self.managed_gpu_object_metadata[obj_id] = GPUObjectMeta( src_actor=src_actor, tensor_transport_backend=tensor_transport_backend, - tensor_meta=tensor_meta, + tensor_transport_meta=tensor_meta, ) def _get_gpu_object_metadata(self, obj_ref: ObjectRef) -> GPUObjectMeta: obj_id = obj_ref.hex() return self.managed_gpu_object_metadata[obj_id] - def _send_object( - self, - communicator_name: str, - src_actor: "ray.actor.ActorHandle", - obj_id: str, - dst_rank: int, - ): - from ray.experimental.gpu_object_manager.gpu_object_store import __ray_send__ - - # Send tensors stored in the `src_actor`'s GPU object store to the - # destination rank `dst_rank`. - # NOTE(swang): We put this task on the background thread to avoid tasks - # executing on the main thread blocking the data transfer. - src_actor.__ray_call__.options(concurrency_group="_ray_system").remote( - __ray_send__, communicator_name, obj_id, dst_rank - ) - - def _recv_object( - self, - communicator_name: str, - dst_actor: "ray.actor.ActorHandle", - obj_id: str, - src_rank: int, - tensor_meta: List[Tuple["torch.Size", "torch.dtype"]], - ): - from ray.experimental.gpu_object_manager.gpu_object_store import __ray_recv__ - - # Receive tensors from the source rank and store them in the - # `dst_actor`'s GPU object store. - # - # NOTE(swang): We put this task on the background thread to avoid tasks - # executing on the main thread blocking the data transfer. Technically, - # this is only needed for the sender task, but we put the receiver task - # on the same background thread to ensure that all communication - # operations are executed in a global order. - dst_actor.__ray_call__.options(concurrency_group="_ray_system").remote( - __ray_recv__, communicator_name, obj_id, src_rank, tensor_meta - ) - def fetch_object(self, obj_id: str): """ Fetches the GPU object from the source actor's GPU object store via the object store @@ -231,6 +181,7 @@ def trigger_out_of_band_tensor_transfer( dst_actor: The target actor to receive tensors task_args: List of arguments for the target actor task that may contain ObjectRefs. """ + gpu_object_refs = set() for arg in task_args: # If an ObjectRef is managed, it means the actual value is a list of tensors stored @@ -240,56 +191,45 @@ def trigger_out_of_band_tensor_transfer( continue if self.is_managed_object(arg.hex()): gpu_object_refs.add(arg) + if gpu_object_refs: + from ray.experimental.collective import get_tensor_transport_manager # Count the number of readers for each GPU object. for obj_ref in gpu_object_refs: # Import get_collective_groups here to avoid dependency on # collective libraries for default Ray installation. - from ray.experimental.collective import get_collective_groups gpu_object_meta = self._get_gpu_object_metadata(obj_ref) src_actor = gpu_object_meta.src_actor - tensor_meta = gpu_object_meta.tensor_meta - communicators = get_collective_groups( - [src_actor, dst_actor], backend=gpu_object_meta.tensor_transport_backend - ) - # TODO(kevin85421): Support multiple communicators. - if len(communicators) == 0: - raise ValueError( - f"No communicators found for actors {src_actor} and {dst_actor}. " - "Create a communicator with " - "`ray.experimental.collective.create_collective_group` " - "before calling actor tasks." - ) - elif len(communicators) > 1: - raise ValueError( - f"There are {len(communicators)} possible communicators that contain actors {src_actor} and {dst_actor}. " - "Currently, GPU objects only support one communicator. Please make sure only " - "one communicator exists." - ) - communicator = communicators[0] - src_rank = communicator.get_rank(src_actor) - if src_rank == -1: - raise ValueError( - f"Sender actor {src_actor} not found in communicator. " - "Please make sure the sender and receiver are in the same communicator." - ) - dst_rank = communicator.get_rank(dst_actor) - if dst_rank == -1: - raise ValueError( - f"Receiver actor {dst_actor} not found in communicator. " - "Please make sure the sender and receiver are in the same communicator." - ) - if src_rank == dst_rank: - # If the source and destination ranks are the same, the tensors can + tensor_transport_meta = gpu_object_meta.tensor_transport_meta + if src_actor._actor_id == dst_actor._actor_id: + # If the source and destination actors are the same, the tensors can # be transferred intra-process, so we skip the out-of-band tensor # transfer. continue + obj_id = obj_ref.hex() - self._send_object(communicator.name, src_actor, obj_id, dst_rank) - self._recv_object( - communicator.name, dst_actor, obj_id, src_rank, tensor_meta + tensor_transport_manager = get_tensor_transport_manager( + gpu_object_meta.tensor_transport_backend + ) + communicator_meta = tensor_transport_manager.get_communicator_metadata( + src_actor, + dst_actor, + gpu_object_meta.tensor_transport_backend, + ) + if not tensor_transport_manager.is_one_sided(): + tensor_transport_manager.send_object( + src_actor, + obj_id, + tensor_transport_meta, + communicator_meta, + ) + tensor_transport_manager.recv_object( + dst_actor, + obj_id, + tensor_transport_meta, + communicator_meta, ) def get_gpu_object(self, object_id: str) -> List["torch.Tensor"]: diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_store.py b/python/ray/experimental/gpu_object_manager/gpu_object_store.py index 9a15f0da7cc4..6b39a5856554 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_store.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_store.py @@ -1,12 +1,15 @@ from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple, Set +from typing import Dict, List, Optional, Set import threading from collections import defaultdict import ray.util.collective as collective from ray._private.custom_types import TensorTransportEnum -from ray.util.collective.types import Backend - +from ray.util.collective.types import ( + Backend, + CommunicatorMetadata, + TensorTransportMetadata, +) try: import torch @@ -19,11 +22,15 @@ TENSOR_TRANSPORT_TO_COLLECTIVE_BACKEND = { TensorTransportEnum.NCCL: Backend.NCCL, TensorTransportEnum.GLOO: Backend.TORCH_GLOO, + TensorTransportEnum.NIXL: Backend.NIXL, } COLLECTIVE_BACKEND_TO_TORCH_DEVICE = { Backend.NCCL: torch.device("cuda"), Backend.TORCH_GLOO: torch.device("cpu"), + # TODO(Qiaolin-Yu): NIXL could also transfer tensors from CPU to CPU. + # More details in https://github.com/ray-project/ray/issues/55587. + Backend.NIXL: torch.device("cuda"), } @@ -38,7 +45,12 @@ def _tensor_transport_to_collective_backend( ) -def __ray_send__(self, communicator_name: str, obj_id: str, dst_rank: int): +def __ray_send__( + self, + obj_id: str, + tensor_transport_meta: TensorTransportMetadata, + communicator_meta: CommunicatorMetadata, +): """Helper function that runs on the src actor to send tensors to the dst actor.""" from ray._private.worker import global_worker @@ -46,54 +58,54 @@ def __ray_send__(self, communicator_name: str, obj_id: str, dst_rank: int): assert gpu_object_store.has_object( obj_id ), f"obj_id={obj_id} not found in GPU object store" + tensors = gpu_object_store.get_object(obj_id) - backend = collective.get_group_handle(communicator_name).backend() + backend = collective.get_group_handle(communicator_meta.communicator_name).backend() device = COLLECTIVE_BACKEND_TO_TORCH_DEVICE[backend] - for tensor in tensors: - if tensor.device.type != device.type: - # TODO(swang): Right now there is no way to catch this error - # and the receiving Ray task will hang. - raise ValueError( - f"tensor device {tensor.device} does not match device {device}" - ) - collective.send(tensor, dst_rank, group_name=communicator_name) + from ray.experimental.collective import get_tensor_transport_manager + + tensor_transport_manager = get_tensor_transport_manager(backend) + tensor_transport_manager.send_multiple_tensors( + tensors, + tensor_transport_meta, + communicator_meta, + device=device, + ) def __ray_recv__( self, - communicator_name: str, obj_id: str, - src_rank: int, - tensor_meta: List[Tuple["torch.Size", "torch.dtype"]], + tensor_transport_meta: TensorTransportMetadata, + communicator_meta: CommunicatorMetadata, ): """Helper function that runs on the dst actor to receive tensors from the src actor.""" from ray._private.worker import global_worker - backend = collective.get_group_handle(communicator_name).backend() + from ray.experimental.collective import get_tensor_transport_manager + + backend = collective.get_group_handle(communicator_meta.communicator_name).backend() + device = COLLECTIVE_BACKEND_TO_TORCH_DEVICE[backend] + tensor_meta = tensor_transport_meta.tensor_meta gpu_object_store = global_worker.gpu_object_manager.gpu_object_store tensors = [] for meta in tensor_meta: shape, dtype = meta tensor = torch.zeros(shape, dtype=dtype, device=device) - collective.recv(tensor, src_rank, group_name=communicator_name) tensors.append(tensor) - gpu_object_store.add_object(obj_id, tensors) - -def __ray_get_tensor_meta__(self, obj_id: str): - """Helper function that runs on the src actor to get the tensor metadata.""" - from ray._private.worker import global_worker + tensor_transport_manager = get_tensor_transport_manager(backend) + tensor_transport_manager.recv_multiple_tensors( + tensors, + tensor_transport_meta, + communicator_meta, + ) - gpu_object_store = global_worker.gpu_object_manager.gpu_object_store - # NOTE: We do not specify a timeout here because the user task that returns - # it could take arbitrarily long and we don't want to trigger a spurious - # timeout. - gpu_object = gpu_object_store.wait_and_get_object(obj_id) - return [(t.shape, t.dtype) for t in gpu_object] + gpu_object_store.add_object(obj_id, tensors) def __ray_fetch_gpu_object__(self, obj_id: str): diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD index 5f1198c5bd99..e6c86d9adac5 100644 --- a/python/ray/tests/BUILD +++ b/python/ray/tests/BUILD @@ -630,6 +630,7 @@ py_test_module_list( env = {"RAY_PYTEST_USE_GPU": "1"}, files = [ "test_gpu_objects_nccl.py", + "test_gpu_objects_nixl.py", ], tags = [ "exclusive", diff --git a/python/ray/tests/test_gpu_objects_nixl.py b/python/ray/tests/test_gpu_objects_nixl.py new file mode 100644 index 000000000000..e46a24358d37 --- /dev/null +++ b/python/ray/tests/test_gpu_objects_nixl.py @@ -0,0 +1,46 @@ +import sys +import torch +import pytest +import ray + + +@ray.remote(num_gpus=1, num_cpus=0, enable_tensor_transport=True) +class GPUTestActor: + @ray.method(tensor_transport="nixl") + def echo(self, data): + return data.to("cuda") + + def sum(self, data): + return data.sum().item() + + +@pytest.mark.parametrize("ray_start_regular", [{"num_gpus": 2}], indirect=True) +def test_p2p(ray_start_regular): + world_size = 2 + actors = [GPUTestActor.remote() for _ in range(world_size)] + + src_actor, dst_actor = actors[0], actors[1] + + # Create test tensor + tensor = torch.tensor([1, 2, 3]) + ref = src_actor.echo.remote(tensor) + + # Trigger tensor transfer from src to dst actor + result = dst_actor.sum.remote(ref) + assert tensor.sum().item() == ray.get(result) + + +@pytest.mark.parametrize("ray_start_regular", [{"num_gpus": 1}], indirect=True) +def test_intra_gpu_tensor_transfer(ray_start_regular): + actor = GPUTestActor.remote() + + tensor = torch.tensor([1, 2, 3]) + + # Intra-actor communication for pure GPU tensors + ref = actor.echo.remote(tensor) + result = actor.sum.remote(ref) + assert tensor.sum().item() == ray.get(result) + + +if __name__ == "__main__": + sys.exit(pytest.main(["-sv", __file__])) diff --git a/python/ray/util/collective/collective.py b/python/ray/util/collective/collective.py index 9265d8b06b97..1d92b838d7f6 100644 --- a/python/ray/util/collective/collective.py +++ b/python/ray/util/collective/collective.py @@ -1,4 +1,5 @@ """APIs exposed under the namespace ray.util.collective.""" + import logging import os from typing import List @@ -36,6 +37,13 @@ except ImportError: _TORCH_DISTRIBUTED_AVAILABLE = False +try: + from ray.util.collective.collective_group.nixl_backend import NixlBackend + + _NIXL_AVAILABLE = True +except ImportError: + _NIXL_AVAILABLE = False + def nccl_available(): global _LOG_NCCL_WARNING @@ -57,6 +65,10 @@ def torch_distributed_available(): return _TORCH_DISTRIBUTED_AVAILABLE +def nixl_available(): + return _NIXL_AVAILABLE + + class GroupManager(object): """Use this class to manage the collective groups we created so far. @@ -98,6 +110,10 @@ def create_collective_group( "Creating torch.distributed GLOO group: '{}'...".format(group_name) ) g = TorchGLOOGroup(world_size, rank, group_name) + elif backend == types.Backend.NIXL: + _check_backend_availability(backend) + logger.debug("Creating NIXL Backend: '{}'...".format(group_name)) + g = NixlBackend() else: raise RuntimeError(f"Unexpected backend: {backend}") @@ -719,19 +735,24 @@ def get_group_handle(group_name: str = "default"): if not is_group_initialized(group_name): # try loading from remote info store try: - # if the information is stored in an Info object, - # get and create the group. - name = "info_" + group_name - mgr = ray.get_actor(name=name) - ids, world_size, rank, backend, gloo_timeout = ray.get( - mgr.get_info.remote() - ) - worker = ray._private.worker.global_worker - id_ = worker.core_worker.get_actor_id() - r = rank[ids.index(id_)] - _group_mgr.create_collective_group( - backend, world_size, r, group_name, gloo_timeout - ) + if group_name == types.NIXL_GROUP_NAME: + _group_mgr.create_collective_group( + types.Backend.NIXL, None, None, group_name, None + ) + else: + # if the information is stored in an Info object, + # get and create the group. + name = "info_" + group_name + mgr = ray.get_actor(name=name) + ids, world_size, rank, backend, gloo_timeout = ray.get( + mgr.get_info.remote() + ) + worker = ray._private.worker.global_worker + id_ = worker.core_worker.get_actor_id() + r = rank[ids.index(id_)] + _group_mgr.create_collective_group( + backend, world_size, r, group_name, gloo_timeout + ) except ValueError as exc: # check if this group is initialized using options() if ( @@ -781,6 +802,9 @@ def _check_backend_availability(backend: types.Backend): elif backend == types.Backend.TORCH_GLOO: if not torch_distributed_available(): raise RuntimeError("torch.distributed is not available.") + elif backend == types.Backend.NIXL: + if not nixl_available(): + raise RuntimeError("NIXL is not available.") def _check_inside_actor(): diff --git a/python/ray/util/collective/collective_group/nixl_backend.py b/python/ray/util/collective/collective_group/nixl_backend.py new file mode 100644 index 000000000000..4861a4818301 --- /dev/null +++ b/python/ray/util/collective/collective_group/nixl_backend.py @@ -0,0 +1,96 @@ +from nixl._api import nixl_agent, nixl_agent_config +import ray +from ray.util.collective.types import Backend +from typing import TYPE_CHECKING, List, Tuple +import time + +if TYPE_CHECKING: + import torch + + +class NixlBackend: + """Backend implementation for NIXL tensor transport. + + This class provides functionality for transferring tensors using NIXL. It handles + initialization of the NIXL agent, receiving tensors, and managing NIXL metadata. + """ + + def __init__(self): + """Initialize the NIXL backend. + + Creates a NIXL agent with UCX backend. + """ + agent_config = nixl_agent_config(backends=["UCX"]) + ctx = ray.get_runtime_context() + actor_id = ctx.get_actor_id() + self._nixl_agent = nixl_agent(actor_id, agent_config) + + @classmethod + def backend(cls): + """Get the backend type. + + Returns: + Backend.NIXL: The backend type enum value for NIXL. + """ + return Backend.NIXL + + def recv( + self, + tensors: List["torch.Tensor"], + nixl_serialized_descs: bytes, + remote_nixl_agent_meta: bytes, + ): + """Receive tensors from a remote NIXL agent. + + Args: + tensors: List of tensors to receive into. + nixl_serialized_descs: Serialized NIXL descriptors for the remote tensors. + remote_nixl_agent_meta: Metadata about the remote NIXL agent. + + Raises: + RuntimeError: If the NIXL transfer enters an error state. + """ + nixl_agent = self._nixl_agent + remote_descs = nixl_agent.deserialize_descs(nixl_serialized_descs) + local_descs = nixl_agent.register_memory(tensors) + remote_name = nixl_agent.add_remote_agent(remote_nixl_agent_meta) + + xfer_handle = nixl_agent.initialize_xfer( + "READ", local_descs.trim(), remote_descs, remote_name + ) + + state = nixl_agent.transfer(xfer_handle) + if state == "ERR": + raise RuntimeError("NIXL transfer got to Error state.") + # Since current nixl does not provide a better way, we need to check the state of + # the transfer continuously. + while True: + state = nixl_agent.check_xfer_state(xfer_handle) + if state == "ERR": + raise RuntimeError("NIXL transfer got to Error state.") + if state == "PROC": + time.sleep(0.001) # Avoid busy waiting + elif state == "DONE": + break + + nixl_agent.release_xfer_handle(xfer_handle) + nixl_agent.deregister_memory(local_descs) + + def get_nixl_metadata(self, tensors: List["torch.Tensor"]) -> Tuple[bytes, bytes]: + """Get NIXL metadata for a set of tensors. + + Args: + tensors: List of tensors to get metadata for. + + Returns: + tuple: A tuple containing: + - Serialized NIXL descriptors for the tensors + - Metadata about this NIXL agent + """ + nixl_agent = self._nixl_agent + reg_descs = nixl_agent.register_memory(tensors) + xfer_descs = reg_descs.trim() + return ( + nixl_agent.get_serialized_descs(xfer_descs), + nixl_agent.get_agent_metadata(), + ) diff --git a/python/ray/util/collective/types.py b/python/ray/util/collective/types.py index 7cb4babaa2bd..06a05ae71549 100644 --- a/python/ray/util/collective/types.py +++ b/python/ray/util/collective/types.py @@ -1,12 +1,19 @@ """Types conversion between different backends.""" + from enum import Enum from dataclasses import dataclass from datetime import timedelta +from typing import List, Tuple, TYPE_CHECKING, Optional + +from numpy import int32 _NUMPY_AVAILABLE = True _TORCH_AVAILABLE = True _CUPY_AVAILABLE = True +if TYPE_CHECKING: + import torch + try: import torch as th # noqa: F401 except ImportError: @@ -34,6 +41,7 @@ class Backend(object): GLOO = "gloo" # Use gloo through torch.distributed. TORCH_GLOO = "torch_gloo" + NIXL = "nixl" UNRECOGNIZED = "unrecognized" def __new__(cls, name: str): @@ -47,6 +55,64 @@ def __new__(cls, name: str): return backend +@dataclass +class TensorTransportMetadata: + """Metadata for tensors stored in the GPU object store. + + Args: + tensor_meta: A list of tuples, each containing the shape and dtype of a tensor. + """ + + tensor_meta: List[Tuple["torch.Size", "torch.dtype"]] + + +@dataclass +class NixlTransportMetadata(TensorTransportMetadata): + """Metadata for tensors stored in the GPU object store for NIXL transport. + + Args: + nixl_serialized_descs: Serialized tensor descriptors for NIXL transport. + nixl_agent_meta: The additional metadata of the remote NIXL agent. + """ + + nixl_serialized_descs: Optional[bytes] = None + nixl_agent_meta: Optional[bytes] = None + + +@dataclass +class CollectiveTransportMetadata(TensorTransportMetadata): + """Metadata for tensors stored in the GPU object store for collective transport.""" + + +@dataclass +class CommunicatorMetadata: + """Metadata for the communicator. + + Args: + communicator_name: The name of the communicator. + """ + + communicator_name: str = "" + + +@dataclass +class CollectiveCommunicatorMetadata(CommunicatorMetadata): + """Metadata for the collective communicator (e.g. NCCL, GLOO). + + Args: + src_rank: The rank of the source actor. + dst_rank: The rank of the destination actor. + """ + + src_rank: Optional[int32] = None + dst_rank: Optional[int32] = None + + +@dataclass +class NixlCommunicatorMetadata(CommunicatorMetadata): + """Metadata for the NIXL communicator.""" + + class ReduceOp(Enum): SUM = 0 PRODUCT = 1 @@ -56,6 +122,9 @@ class ReduceOp(Enum): unset_timeout_ms = timedelta(milliseconds=-1) +# This is used to identify the collective group for NIXL. +NIXL_GROUP_NAME = "ray_internal_nixl_group" + @dataclass class AllReduceOptions: diff --git a/python/requirements/ml/dl-gpu-requirements.txt b/python/requirements/ml/dl-gpu-requirements.txt index 59d6b0fb10df..ab46a6df8157 100644 --- a/python/requirements/ml/dl-gpu-requirements.txt +++ b/python/requirements/ml/dl-gpu-requirements.txt @@ -16,3 +16,4 @@ torch-cluster==1.6.3+pt23cu121 torch-spline-conv==1.2.2+pt23cu121 cupy-cuda12x==13.1.0; sys_platform != 'darwin' +nixl==0.4.0; sys_platform != 'darwin' diff --git a/src/ray/protobuf/common.proto b/src/ray/protobuf/common.proto index 9b3f04f2e59a..0fb1ade03022 100644 --- a/src/ray/protobuf/common.proto +++ b/src/ray/protobuf/common.proto @@ -707,6 +707,8 @@ enum TensorTransport { NCCL = 1; // Use GLOO for tensor transport. GLOO = 2; + // Use NIXL for tensor transport. + NIXL = 3; } // Argument in the task. From ec20f9a16ef07cc82a74bac72b47abdc27fd6e5c Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Tue, 19 Aug 2025 14:12:10 -0700 Subject: [PATCH 185/634] [docs/data] Add Autoscaling Config for Context docs (#55712) ## Why are these changes needed? Missing documentation for autoscaling configuration. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Richard Liaw --- doc/source/data/api/data_context.rst | 5 +++- python/ray/data/context.py | 35 ++++++++++++++-------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/doc/source/data/api/data_context.rst b/doc/source/data/api/data_context.rst index d86c640b20e2..eac742df73c8 100644 --- a/doc/source/data/api/data_context.rst +++ b/doc/source/data/api/data_context.rst @@ -3,7 +3,7 @@ Global configuration ==================== -.. currentmodule:: ray.data +.. currentmodule:: ray.data.context .. autoclass:: DataContext @@ -12,3 +12,6 @@ Global configuration :toctree: doc/ DataContext.get_current + + +.. autoclass:: AutoscalingConfig diff --git a/python/ray/data/context.py b/python/ray/data/context.py index 4058adf1ba2f..8c8b2c9023a5 100644 --- a/python/ray/data/context.py +++ b/python/ray/data/context.py @@ -230,23 +230,24 @@ class ShuffleStrategy(str, enum.Enum): @DeveloperAPI @dataclass class AutoscalingConfig: - # Actor Pool utilization threshold for upscaling. Once Actor Pool - # exceeds this utilization threshold it will start adding new actors. - # - # NOTE: Actor Pool utilization is defined as ratio of - # - # - Number of submitted tasks to - # - Max number of tasks the current set of Actors in the pool could run - # (defined as Ray Actor's `max_concurrency` * `pool.num_running_actors`) - # - # This utilization value could exceed 100%, when the number of submitted tasks - # exceed available concurrency-slots to run them in the current set of actors. - # - # This is possible when `max_tasks_in_flight_per_actor` (defaults to 2 x - # of `max_concurrency`) > Actor's `max_concurrency` and allows to overlap - # task execution with the fetching of the blocks for the next task providing - # for ability to negotiate a trade-off between autoscaling speed and resource - # efficiency (ie making tasks wait instead of immediately triggering execution) + """Configuration for autoscaling of Ray Data. + + Args: + actor_pool_util_upscaling_threshold: Actor Pool utilization threshold for upscaling. + Once Actor Pool exceeds this utilization threshold it will start adding new actors. + Actor Pool utilization is defined as ratio of number of submitted tasks to the + number of available concurrency-slots to run them in the current set of actors. + This utilization value could exceed 100%, when the number of submitted tasks + exceed available concurrency-slots to run them in the current set of actors. + This is possible when `max_tasks_in_flight_per_actor` + (defaults to 2 x of `max_concurrency`) > Actor's `max_concurrency` + and allows to overlap task execution with the fetching of the blocks + for the next task providing for ability to negotiate a trade-off + between autoscaling speed and resource efficiency (i.e., + making tasks wait instead of immediately triggering execution). + actor_pool_util_downscaling_threshold: Actor Pool utilization threshold for downscaling. + """ + actor_pool_util_upscaling_threshold: float = ( DEFAULT_ACTOR_POOL_UTIL_UPSCALING_THRESHOLD ) From e633a4d16125d20b19d1b220c2167af7616f8e53 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Tue, 19 Aug 2025 15:21:46 -0700 Subject: [PATCH 186/634] [core][1eventx/01] job event: add schema for driver job event (#55032) This is the first in a series of PRs to support JobEvent in the oneevent framework. The full effort will include adding the JobEvent schema, introducing a generic interface for exporting different types of events to the Event Aggregator, and implementing the necessary integration logic. ---- In this initial PR, we add the DriverJobEvent schema. To be consistent with the design of task events, we also split this into - DriverDefinitionJobEvent: the static information about this job, e.g. node id, script, etc. - DriverExecutionJobEvent: the runtime, dynamic information about this job, e.g. state Test: - CI Signed-off-by: Cuong Nguyen --- .../modules/aggregator/aggregator_agent.py | 6 +- .../aggregator/tests/test_aggregator_agent.py | 139 ++++++++++++++++++ src/ray/protobuf/BUILD.bazel | 32 ++++ src/ray/protobuf/events_base_event.proto | 6 + .../events_driver_job_definition_event.proto | 38 +++++ .../events_driver_job_execution_event.proto | 43 ++++++ 6 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 src/ray/protobuf/events_driver_job_definition_event.proto create mode 100644 src/ray/protobuf/events_driver_job_execution_event.proto diff --git a/python/ray/dashboard/modules/aggregator/aggregator_agent.py b/python/ray/dashboard/modules/aggregator/aggregator_agent.py index d970623633a9..967016c39ce9 100644 --- a/python/ray/dashboard/modules/aggregator/aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/aggregator_agent.py @@ -77,7 +77,11 @@ # Valid values: TASK_DEFINITION_EVENT, TASK_EXECUTION_EVENT, ACTOR_TASK_DEFINITION_EVENT, ACTOR_TASK_EXECUTION_EVENT # The list of all supported event types can be found in src/ray/protobuf/events_base_event.proto (EventType enum) # By default TASK_PROFILE_EVENT is not exposed to external services -DEFAULT_EXPOSABLE_EVENT_TYPES = "TASK_DEFINITION_EVENT,TASK_EXECUTION_EVENT,ACTOR_TASK_DEFINITION_EVENT,ACTOR_TASK_EXECUTION_EVENT" +DEFAULT_EXPOSABLE_EVENT_TYPES = ( + "TASK_DEFINITION_EVENT,TASK_EXECUTION_EVENT," + "ACTOR_TASK_DEFINITION_EVENT,ACTOR_TASK_EXECUTION_EVENT," + "DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_EXECUTION_EVENT" +) EXPOSABLE_EVENT_TYPES = os.environ.get( f"{env_var_prefix}_EXPOSABLE_EVENT_TYPES", DEFAULT_EXPOSABLE_EVENT_TYPES ) diff --git a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py index 2b974f87618c..e658849969bc 100644 --- a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py @@ -28,6 +28,17 @@ from ray.core.generated.events_base_event_pb2 import RayEvent from ray.core.generated.profile_events_pb2 import ProfileEvents, ProfileEventEntry from ray.core.generated.events_task_profile_events_pb2 import TaskProfileEvents +from ray.core.generated.events_driver_job_definition_event_pb2 import ( + DriverJobDefinitionEvent, +) +from ray.core.generated.events_driver_job_execution_event_pb2 import ( + DriverJobExecutionEvent, +) +from ray.core.generated.runtime_env_common_pb2 import ( + RuntimeEnvInfo, + RuntimeEnvUris, + RuntimeEnvConfig, +) from ray.dashboard.modules.aggregator.aggregator_agent import AggregatorAgent @@ -486,5 +497,133 @@ def _verify_profile_event_json(req_json, expected_timestamp): assert event_entry["extraData"] == '{"cpu_usage": 0.8}' +@_with_aggregator_port +def test_aggregator_agent_receive_driver_job_definition_event( + ray_start_cluster_head_with_env_vars, httpserver +): + cluster = ray_start_cluster_head_with_env_vars + stub = get_event_aggregator_grpc_stub( + cluster.gcs_address, cluster.head_node.node_id + ) + httpserver.expect_request("/", method="POST").respond_with_data("", status=200) + test_time = 1751302230130457542 + seconds, nanos = divmod(test_time, 10**9) + timestamp = Timestamp(seconds=seconds, nanos=nanos) + request = AddEventsRequest( + events_data=RayEventsData( + events=[ + RayEvent( + event_id=b"1", + source_type=RayEvent.SourceType.CORE_WORKER, + event_type=RayEvent.EventType.DRIVER_JOB_DEFINITION_EVENT, + timestamp=timestamp, + severity=RayEvent.Severity.INFO, + message="driver job event", + driver_job_definition_event=DriverJobDefinitionEvent( + job_id=b"1", + config=DriverJobDefinitionEvent.Config( + runtime_env_info=RuntimeEnvInfo( + serialized_runtime_env="{}", + uris=RuntimeEnvUris( + working_dir_uri="file:///tmp/ray/runtime_env", + py_modules_uris=[], + ), + runtime_env_config=RuntimeEnvConfig( + setup_timeout_seconds=10, + eager_install=True, + log_files=[], + ), + ), + metadata={}, + ), + ), + ), + ], + task_events_metadata=TaskEventsMetadata( + dropped_task_attempts=[], + ), + ) + ) + stub.AddEvents(request) + wait_for_condition(lambda: len(httpserver.log) == 1) + req, _ = httpserver.log[0] + req_json = json.loads(req.data) + assert req_json[0]["message"] == "driver job event" + assert ( + req_json[0]["driverJobDefinitionEvent"]["config"]["runtimeEnvInfo"][ + "serializedRuntimeEnv" + ] + == "{}" + ) + assert ( + req_json[0]["driverJobDefinitionEvent"]["config"]["runtimeEnvInfo"]["uris"][ + "workingDirUri" + ] + == "file:///tmp/ray/runtime_env" + ) + assert ( + req_json[0]["driverJobDefinitionEvent"]["config"]["runtimeEnvInfo"][ + "runtimeEnvConfig" + ]["setupTimeoutSeconds"] + == 10.0 + ) + + +@_with_aggregator_port +def test_aggregator_agent_receive_driver_job_execution_event( + ray_start_cluster_head_with_env_vars, httpserver +): + cluster = ray_start_cluster_head_with_env_vars + stub = get_event_aggregator_grpc_stub( + cluster.gcs_address, cluster.head_node.node_id + ) + httpserver.expect_request("/", method="POST").respond_with_data("", status=200) + test_time = 1751302230130457542 + seconds, nanos = divmod(test_time, 10**9) + timestamp = Timestamp(seconds=seconds, nanos=nanos) + request = AddEventsRequest( + events_data=RayEventsData( + events=[ + RayEvent( + event_id=b"1", + source_type=RayEvent.SourceType.CORE_WORKER, + event_type=RayEvent.EventType.DRIVER_JOB_EXECUTION_EVENT, + timestamp=timestamp, + severity=RayEvent.Severity.INFO, + message="driver job execution event", + driver_job_execution_event=DriverJobExecutionEvent( + job_id=b"1", + states=[ + DriverJobExecutionEvent.StateTimestamp( + state=DriverJobExecutionEvent.State.CREATED, + timestamp=Timestamp(seconds=1234567890), + ), + DriverJobExecutionEvent.StateTimestamp( + state=DriverJobExecutionEvent.State.FAILURE, + timestamp=Timestamp(seconds=1234567890), + ), + ], + ), + ), + ], + task_events_metadata=TaskEventsMetadata( + dropped_task_attempts=[], + ), + ) + ) + stub.AddEvents(request) + wait_for_condition(lambda: len(httpserver.log) == 1) + req, _ = httpserver.log[0] + req_json = json.loads(req.data) + assert req_json[0]["message"] == "driver job execution event" + assert ( + req_json[0]["driverJobExecutionEvent"]["jobId"] + == base64.b64encode(b"1").decode() + ) + assert len(req_json[0]["driverJobExecutionEvent"]["states"]) == 2 + assert req_json[0]["driverJobExecutionEvent"]["states"][0]["state"] == "CREATED" + assert req_json[0]["driverJobExecutionEvent"]["states"][1]["state"] == "FAILURE" + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/src/ray/protobuf/BUILD.bazel b/src/ray/protobuf/BUILD.bazel index a4b1f3c61ff4..6d896d64245d 100644 --- a/src/ray/protobuf/BUILD.bazel +++ b/src/ray/protobuf/BUILD.bazel @@ -487,16 +487,48 @@ proto_library( ], ) +proto_library( + name = "events_driver_job_definition_event_proto", + srcs = ["events_driver_job_definition_event.proto"], + deps = [ + ":common_proto", + ":runtime_env_common_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + +proto_library( + name = "events_driver_job_execution_event_proto", + srcs = ["events_driver_job_execution_event.proto"], + deps = [ + ":common_proto", + ":runtime_env_common_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + cc_proto_library( name = "events_task_profile_events_cc_proto", deps = [":events_task_profile_events_proto"], ) +cc_proto_library( + name = "events_driver_job_definition_event_cc_proto", + deps = [":events_driver_job_definition_event_proto"], +) + +cc_proto_library( + name = "events_driver_job_execution_event_cc_proto", + deps = [":events_driver_job_execution_event_proto"], +) + proto_library( name = "events_base_event_proto", srcs = ["events_base_event.proto"], deps = [ ":events_actor_task_definition_event_proto", + ":events_driver_job_definition_event_proto", + ":events_driver_job_execution_event_proto", ":events_task_definition_event_proto", ":events_task_execution_event_proto", ":events_task_profile_events_proto", diff --git a/src/ray/protobuf/events_base_event.proto b/src/ray/protobuf/events_base_event.proto index a68a9e02c112..3b5c9e913944 100644 --- a/src/ray/protobuf/events_base_event.proto +++ b/src/ray/protobuf/events_base_event.proto @@ -21,6 +21,8 @@ import "src/ray/protobuf/events_actor_task_definition_event.proto"; import "src/ray/protobuf/events_task_definition_event.proto"; import "src/ray/protobuf/events_task_execution_event.proto"; import "src/ray/protobuf/events_task_profile_events.proto"; +import "src/ray/protobuf/events_driver_job_definition_event.proto"; +import "src/ray/protobuf/events_driver_job_execution_event.proto"; // This is the base message for all ray events. message RayEvent { @@ -46,6 +48,8 @@ message RayEvent { TASK_EXECUTION_EVENT = 2; ACTOR_TASK_DEFINITION_EVENT = 3; TASK_PROFILE_EVENT = 4; + DRIVER_JOB_DEFINITION_EVENT = 5; + DRIVER_JOB_EXECUTION_EVENT = 6; } // The severities of events that can be generated. @@ -85,4 +89,6 @@ message RayEvent { TaskExecutionEvent task_execution_event = 9; ActorTaskDefinitionEvent actor_task_definition_event = 10; TaskProfileEvents task_profile_events = 11; + DriverJobDefinitionEvent driver_job_definition_event = 12; + DriverJobExecutionEvent driver_job_execution_event = 13; } diff --git a/src/ray/protobuf/events_driver_job_definition_event.proto b/src/ray/protobuf/events_driver_job_definition_event.proto new file mode 100644 index 000000000000..498982ee220d --- /dev/null +++ b/src/ray/protobuf/events_driver_job_definition_event.proto @@ -0,0 +1,38 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; +import "src/ray/protobuf/runtime_env_common.proto"; + +package ray.rpc.events; + +// Message containing the definition information of a driver job. +// The message is expected to be emitted once per job creation. +// +// For runtime information associated with this event, see DriverJobExecutionEvent. +message DriverJobDefinitionEvent { + message Config { + RuntimeEnvInfo runtime_env_info = 1; + map metadata = 2; + } + + bytes job_id = 1; + int64 driver_pid = 3; + bytes driver_node_id = 4; + string entrypoint = 5; + Config config = 6; +} diff --git a/src/ray/protobuf/events_driver_job_execution_event.proto b/src/ray/protobuf/events_driver_job_execution_event.proto new file mode 100644 index 000000000000..73153b7b05e2 --- /dev/null +++ b/src/ray/protobuf/events_driver_job_execution_event.proto @@ -0,0 +1,43 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; +import "src/ray/protobuf/runtime_env_common.proto"; +import "src/ray/protobuf/common.proto"; + +package ray.rpc.events; + +// Message containing the execution information of a driver job. It can be used to +// capture the full state transition history. +// +// For static information associated with this event, see DriverJobDefinitionEvent. +message DriverJobExecutionEvent { + enum State { + UNSPECIFIED = 0; + CREATED = 1; + FAILURE = 2; + SUCCESS = 3; + } + + message StateTimestamp { + State state = 1; + google.protobuf.Timestamp timestamp = 2; + } + + bytes job_id = 1; + repeated StateTimestamp states = 2; +} From 3decf16b434b99848751a6646efe6842b23ac7fd Mon Sep 17 00:00:00 2001 From: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Date: Tue, 19 Aug 2025 15:51:13 -0700 Subject: [PATCH 187/634] [Serve] Updates test_deploy_2.py to hardcode route_prefix for test_change_route_prefix (#55700) This test specifies the route_prefix value for one of the urls for better readability. --------- Signed-off-by: doyoung --- python/ray/serve/tests/test_deploy_app_2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/serve/tests/test_deploy_app_2.py b/python/ray/serve/tests/test_deploy_app_2.py index fe073c07eb6b..5d04dc45d60d 100644 --- a/python/ray/serve/tests/test_deploy_app_2.py +++ b/python/ray/serve/tests/test_deploy_app_2.py @@ -575,8 +575,8 @@ def check_switched(): assert "Path '/old' not found." in resp.text # Response from new route should be same PID - url = get_application_url() - pid2 = httpx.get(url).json()[0] + url = get_application_url(exclude_route_prefix=True) + pid2 = httpx.get(f"{url}/new").json()[0] assert pid2 == pid1 return True From f2ca998a3f19a8d808623361fcfd0198b18a1bf2 Mon Sep 17 00:00:00 2001 From: lkchen Date: Tue, 19 Aug 2025 17:37:14 -0700 Subject: [PATCH 188/634] [ci] clean up `.github/dependabot.yml`: remove entry for "compact requirements" (#55744) fixes yaml grammar error. --------- Signed-off-by: Linkun --- .github/dependabot.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 25358b043673..5f43c1d9f812 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,16 +21,6 @@ updates: open-pull-requests-limit: 5 reviewers: - "ray-project/ray-tune" - # compat requirements should not be updated - - package-ecosystem: "pip" - directory: "/python/requirements/compat" - commit-message: - prefix: "[air/do-not-merge]" - include: "scope" - ignore: * - open-pull-requests-limit: 0 - reviewers: - - "ray-project/ray-tune" # Data Requirements. - package-ecosystem: "pip" directory: "/python/requirements/data_processing" From 1df700eaa445c391f7297b9eafd59a99234d448d Mon Sep 17 00:00:00 2001 From: Justin Yu Date: Tue, 19 Aug 2025 21:48:17 -0700 Subject: [PATCH 189/634] [train] Revert "Make ray.train.get_dataset_shard lazily configure the dataset sharding (#55230)" (#55760) ## Summary Reverts #55230 because it introduced a performance regression. Ray dataset objects can contain a lot of metadata (e.g., for `read_images`, if the dataset has a lot of files, the metadata contains all of the filepaths) and the dataset object size can get into the GBs. These objects take a while to serialize and ship over to remote actors. #55230 introduced a `DatasetManager` actor that would send over the dataset iterator objects one by one to each train worker. The actor method return values would be serialized one by one, which meant that the train workers would need to wait one by one to receive their data iterator object. ## Repro script ```python import ray import ray.data from ray.data import DataContext from ray.train.v2._internal.callbacks.datasets import DatasetManager from ray.data.datasource.partitioning import Partitioning train_dir = "s3://anyscale-imagenet/ILSVRC/Data/CLS-LOC/train" train_partitioning = Partitioning( "dir", base_dir=train_dir, field_names=["class"] ) train_ds = ray.data.read_images( train_dir, mode="RGB", include_paths=False, partitioning=train_partitioning, ) num_workers = 16 datasets = {"train": train_ds, "val": train_ds} dataset_manager = ray.remote(DatasetManager).remote( datasets=datasets, data_config=ray.train.DataConfig(), data_context=DataContext.get_current(), world_size=num_workers, worker_node_ids=None, ) def get_size_bytes(obj): import ray.cloudpickle as ray_pickle size_bytes = len(ray_pickle.dumps(obj)) return size_bytes import time start = time.perf_counter() ray.get([consumer.remote(it) for it in iters]) end = time.perf_counter() print("elapsed: ", end - start) @ray.remote(num_gpus=1) def consumer(dm, rank): from ray.train.v2._internal.callbacks.datasets import DatasetShardMetadata dataset_info = DatasetShardMetadata("train", rank) import time start = time.perf_counter() ds_shard = ray.get(dm.get_dataset_shard.remote(dataset_info)) end = time.perf_counter() size_mb = get_size_bytes(ds_shard) / (1024*1024) print(f"[{rank=}] TIME TO GET THE DATASET SHARD (SIZE={size_mb}MB):", end - start) tasks = ray.get([consumer.remote(dataset_manager, rank) for rank in range(num_workers)]) ``` ```python (consumer pid=119449, ip=10.0.159.6) [rank=0] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 31.03426499699981 (consumer pid=111901, ip=10.0.152.248) [rank=5] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 42.119795100000374 (consumer pid=130910, ip=10.0.188.145) [rank=12] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 53.78782823600068 (consumer pid=111903, ip=10.0.152.248) [rank=7] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 66.02914485200017 (consumer pid=111897, ip=10.0.152.248) [rank=3] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 77.95694264599933 (consumer pid=111899, ip=10.0.152.248) [rank=6] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 89.89265315800003 (consumer pid=119443, ip=10.0.159.6) [rank=1] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 102.69119121899985 (consumer pid=119452, ip=10.0.159.6) [rank=4] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 114.67034755900022 (consumer pid=130914, ip=10.0.188.145) [rank=14] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 125.7356306720003 (consumer pid=130913, ip=10.0.188.145) [rank=13] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 137.7492523689998 (consumer pid=119450, ip=10.0.159.6) [rank=2] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 151.02738245799992 (consumer pid=130915, ip=10.0.188.145) [rank=15] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 162.16050169899972 (consumer pid=115521, ip=10.0.159.125) [rank=8] TIME TO GET THE DATASET SHARD (SIZE=782.4418258666992MB): 174.8550526050003 ``` --------- Signed-off-by: Justin Yu --- .../train/v2/_internal/callbacks/datasets.py | 183 ++------------- .../train/v2/_internal/execution/context.py | 40 +--- .../v2/_internal/execution/train_fn_utils.py | 8 +- .../execution/worker_group/worker.py | 8 +- .../train/v2/tests/test_data_integration.py | 218 +----------------- 5 files changed, 36 insertions(+), 421 deletions(-) diff --git a/python/ray/train/v2/_internal/callbacks/datasets.py b/python/ray/train/v2/_internal/callbacks/datasets.py index 3b31dcb96644..a51b633d457a 100644 --- a/python/ray/train/v2/_internal/callbacks/datasets.py +++ b/python/ray/train/v2/_internal/callbacks/datasets.py @@ -1,162 +1,20 @@ -import asyncio import copy -from dataclasses import dataclass from typing import Any, Callable, Dict, List, Union import ray.train -from ray.data import DataIterator, Dataset, NodeIdStr +from ray.data import Dataset from ray.data.context import DataContext from ray.train.v2._internal.execution.callback import WorkerGroupCallback from ray.train.v2._internal.execution.worker_group.worker_group import ( Worker, WorkerGroup, ) -from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy # A type representing either a ray.data.Dataset or a function that returns a # ray.data.Dataset and accepts no arguments. GenDataset = Union[Dataset, Callable[[], Dataset]] -@dataclass -class DatasetShardMetadata: - """Metadata about a dataset shard used for lookup and configuration.""" - - dataset_name: str - world_rank: int - - -class DatasetManager: - """Manages the dataset shards for datasets configured in the trainer.""" - - def __init__( - self, - datasets: Dict[str, GenDataset], - data_config: ray.train.DataConfig, - data_context: DataContext, - world_size: int, - worker_node_ids: List[NodeIdStr], - ): - self._datasets = {k: v() if callable(v) else v for k, v in datasets.items()} - self._data_config = data_config - self._datasets_to_split = ( - set(self._datasets.keys()) - if data_config._datasets_to_split == "all" - else set(data_config._datasets_to_split) - ) - self._world_size = world_size - self._worker_node_ids = worker_node_ids - - # Maps dataset name to a list of cached `DataIterator`s corresponding to - # Train worker ranks. - self._dataset_iterators: Dict[str, List[DataIterator]] = {} - - # A condition variable to synchronize the calls to the async `get_dataset_shard` method. - self._condition = asyncio.Condition() - - DataContext._set_current(data_context) - - def _create_dataset_iterators( - self, dataset_info: DatasetShardMetadata, base_dataset: Dataset - ) -> List[DataIterator]: - dataset_name = dataset_info.dataset_name - - iterators_per_rank = self._data_config.configure( - datasets={dataset_name: base_dataset}, - world_size=self._world_size, - worker_handles=None, - worker_node_ids=self._worker_node_ids, - ) - assert len(iterators_per_rank) == self._world_size - # TODO: Update DataConfig to return a List[DataIterator] directly - # for configuring a single dataset. - # Convert the List[Dict[str, DataIterator]] to a List[DataIterator], - # since we only configured one dataset. - return [iterators_per_rank[i][dataset_name] for i in range(self._world_size)] - - def _get_unsharded_dataset_iterator( - self, dataset_info: DatasetShardMetadata - ) -> DataIterator: - """Returns the dataset iterator for a dataset that is excluded - from `DataConfig.datasets_to_split`. - - Note that this method is NOT a barrier across workers and can be called - by any subset of workers and will return immediately. - """ - dataset_name = dataset_info.dataset_name - world_rank = dataset_info.world_rank - - if dataset_name not in self._dataset_iterators: - self._dataset_iterators[dataset_name] = self._create_dataset_iterators( - dataset_info, self._datasets[dataset_name] - ) - - return self._dataset_iterators[dataset_name][world_rank] - - async def _get_sharded_dataset_iterator( - self, dataset_info: DatasetShardMetadata - ) -> DataIterator: - """Returns the dataset iterator for a dataset that is included - in `DataConfig.datasets_to_split`. - - Note that this method is a barrier across workers, - and all workers must call this method before training. - """ - dataset_name = dataset_info.dataset_name - world_rank = dataset_info.world_rank - - async with self._condition: - if dataset_name in self._dataset_iterators: - # If the dataset iterators have already been created, return the - # existing one. - iterator = self._dataset_iterators[dataset_name][world_rank] - elif world_rank == 0: - # In this case, the dataset iterators have not been created yet. - # The dataset only needs to be configured once globally for all workers. - # Do it only when the rank 0 worker calls this method. - iterators = self._create_dataset_iterators( - dataset_info, self._datasets[dataset_name] - ) - iterator = iterators[world_rank] - - # Cache the dataset iterators for future use. - self._dataset_iterators[dataset_name] = iterators - self._condition.notify_all() - else: - # Wait for the dataset iterators to be created by the rank 0 worker. - await self._condition.wait_for( - lambda: dataset_name in self._dataset_iterators - ) - iterator = self._dataset_iterators[dataset_name][world_rank] - return iterator - - async def get_dataset_shard( - self, - dataset_info: DatasetShardMetadata, - ) -> DataIterator: - """Create and return the dataset shard iterator for a Ray Train worker's - call to `ray.train.get_dataset_shard`. - - This method is a barrier that should be called by all Ray Train workers at once. - If the dataset iterators have already been created, return the existing ones. - Otherwise, create the dataset iterators and cache them. - - Here's an example of how this method is used with 4 workers: - - Rank 2 calls get_dataset_shard, waits on the condition variable. - Rank 1 calls get_dataset_shard, waits on the condition variable. - Rank 0 calls get_dataset_shard, creates the dataset iterators, caches them, - and notifies all workers hanging on the condition variable. - Rank 3 calls get_dataset_shard, returns the cached iterator. - """ - dataset_name = dataset_info.dataset_name - - if dataset_name in self._datasets_to_split: - return await self._get_sharded_dataset_iterator(dataset_info) - else: - return self._get_unsharded_dataset_iterator(dataset_info) - - class DatasetsSetupCallback(WorkerGroupCallback): """The callback to setup Ray Datasets for the worker group.""" @@ -173,7 +31,7 @@ def __init__( # Capture the current DataContext to propagate it to # the Train workers later. # The propagation works in the following way: - # 1. This callback is created when user creates the Trainer. + # 1. This callback is created when user create the Trainer. # 2. Then this callback will be passed to the Controller actor. # 3. Lastly, when the worker group is initialized, the Controller # will call the `after_worker_group_start` callback to propagate @@ -187,39 +45,26 @@ def get_train_total_resources( these resources logically from its available pool.""" return scaling_config.total_resources - # -------------------------- - # WorkerGroupCallback - # -------------------------- - def before_init_train_context(self, workers: List[Worker]) -> Dict[str, List[Any]]: - if not self._datasets: - return {"dataset_manager": [None] * len(workers)} - - world_size = len(workers) - worker_node_ids = [worker.metadata.node_id for worker in workers] + # Configure dataset shards + datasets = {k: v() if callable(v) else v for k, v in self._datasets.items()} + node_ids = [worker.metadata.node_id for worker in workers] + # Notify the DataConfig about the total resources reserved for training. total_train_resources = self.get_train_total_resources(self._scaling_config) self._data_config.set_train_total_resources( total_train_resources.get("CPU", 0), total_train_resources.get("GPU", 0) ) - dataset_manager = ( - ray.remote(DatasetManager) - .options( - num_cpus=0, - scheduling_strategy=NodeAffinitySchedulingStrategy( - ray.get_runtime_context().get_node_id(), soft=False - ), - ) - .remote( - datasets=self._datasets, - data_config=self._data_config, - data_context=self._data_context, - world_size=world_size, - worker_node_ids=worker_node_ids, - ) + dataset_shards = self._data_config.configure( + datasets, + world_size=len(workers), + worker_handles=None, + worker_node_ids=node_ids, ) - return {"dataset_manager": [dataset_manager] * len(workers)} + assert len(dataset_shards) == len(workers) + + return {"dataset_shards": dataset_shards} def after_worker_group_start(self, worker_group: WorkerGroup): # Propagate DataContext diff --git a/python/ray/train/v2/_internal/execution/context.py b/python/ray/train/v2/_internal/execution/context.py index ecb3ae3f9cd8..fd2f0df8e23e 100644 --- a/python/ray/train/v2/_internal/execution/context.py +++ b/python/ray/train/v2/_internal/execution/context.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional import ray -from ray.actor import ActorHandle from ray.data import DataIterator, Dataset from ray.train import BackendConfig, Checkpoint, DataConfig from ray.train._internal import session @@ -18,10 +17,6 @@ from ray.train.v2.api.config import RunConfig, ScalingConfig if TYPE_CHECKING: - from ray.train.v2._internal.callbacks.datasets import ( - DatasetManager, - DatasetShardMetadata, - ) from ray.train.v2._internal.execution.callback import TrainContextCallback from ray.train.v2._internal.execution.worker_group.thread_runner import ThreadRunner @@ -97,11 +92,9 @@ class TrainContext: distributed_context: DistributedContext execution_context: ExecutionContext storage_context: StorageContext + dataset_shards: Dict[str, DataIterator] checkpoint: Optional[Checkpoint] = None - dataset_manager: Optional[ActorHandle["DatasetManager"]] = None - _cached_dataset_shards: Dict[str, DataIterator] = field(default_factory=dict) - @_copy_doc(session.get_experiment_name) def get_experiment_name(self) -> str: return self.train_run_context.run_config.name @@ -140,7 +133,7 @@ def get_synchronization_actor(self): def get_checkpoint(self): return self.checkpoint - def get_dataset_shard(self, dataset_info: "DatasetShardMetadata") -> DataIterator: + def get_dataset_shard(self, dataset_name: str) -> DataIterator: """Returns the :class:`ray.data.DataIterator` shard for this worker. Call :meth:`~ray.data.DataIterator.iter_torch_batches` or @@ -148,34 +141,19 @@ def get_dataset_shard(self, dataset_info: "DatasetShardMetadata") -> DataIterato appropriate framework-specific data type. Args: - dataset_info: The shard metadata, including the dataset name and worker rank. - + dataset_name: Name of the dataset shard. Returns: The ``DataIterator`` shard with the given name for this worker. - Raises: KeyError: If the dataset shard with the given name is not found. """ - dataset_name = dataset_info.dataset_name - error = KeyError( - f"Dataset shard for '{dataset_name}' not found. " - "Please ensure that the dataset is passed through the Trainer `datasets` " - "argument." - ) - - if self.dataset_manager is None: - raise error - - if dataset_info.dataset_name in self._cached_dataset_shards: - return self._cached_dataset_shards[dataset_info.dataset_name] - try: - shard = ray.get(self.dataset_manager.get_dataset_shard.remote(dataset_info)) - except KeyError as e: - raise error from e - - self._cached_dataset_shards[dataset_info.dataset_name] = shard - return shard + return self.dataset_shards[dataset_name] + except KeyError: + raise KeyError( + f"Dataset {dataset_name} not found. Available datasets: " + f"{list(self.dataset_shards.keys())}." + ) def get_context_callbacks(self) -> List["TrainContextCallback"]: return self.execution_context.train_context_callbacks diff --git a/python/ray/train/v2/_internal/execution/train_fn_utils.py b/python/ray/train/v2/_internal/execution/train_fn_utils.py index b6b784de163e..28bf683fda2d 100644 --- a/python/ray/train/v2/_internal/execution/train_fn_utils.py +++ b/python/ray/train/v2/_internal/execution/train_fn_utils.py @@ -57,13 +57,7 @@ def get_dataset_shard(self, dataset_name: str) -> DataIterator: Returns: The DataIterator shard for this worker. """ - from ray.train.v2._internal.callbacks.datasets import DatasetShardMetadata - - dataset_info = DatasetShardMetadata( - dataset_name=dataset_name, - world_rank=get_internal_train_context().get_world_rank(), - ) - return get_internal_train_context().get_dataset_shard(dataset_info) + return get_internal_train_context().get_dataset_shard(dataset_name) def get_context(self) -> ExternalTrainContext: return ExternalTrainContext() diff --git a/python/ray/train/v2/_internal/execution/worker_group/worker.py b/python/ray/train/v2/_internal/execution/worker_group/worker.py index b3fec846128d..667ab318296b 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/worker.py +++ b/python/ray/train/v2/_internal/execution/worker_group/worker.py @@ -10,6 +10,7 @@ import ray._private.ray_constants as ray_constants from .thread_runner import ThreadRunner from ray.actor import ActorHandle +from ray.data.iterator import DataIterator from ray.train import Checkpoint from ray.train.v2._internal.constants import ( DEFAULT_ENABLE_WORKER_LOGGING, @@ -19,6 +20,7 @@ TrainContextCallback, WorkerCallback, ) +from ray.train.v2._internal.execution.checkpoint.sync_actor import SynchronizationActor from ray.train.v2._internal.execution.context import ( DistributedContext, ExecutionContext, @@ -187,10 +189,10 @@ def init_train_context( self, train_run_context: TrainRunContext, distributed_context: DistributedContext, - synchronization_actor: ActorHandle, + synchronization_actor: SynchronizationActor, storage_context: StorageContext, worker_callbacks: List[Union[WorkerCallback, TrainContextCallback]], - dataset_manager: Optional[ActorHandle] = None, + dataset_shards: Dict[str, DataIterator] = None, checkpoint: Optional[Checkpoint] = None, ): self._callbacks = [c for c in worker_callbacks if isinstance(c, WorkerCallback)] @@ -209,8 +211,8 @@ def init_train_context( train_context_callbacks=context_callbacks_to_propagate, ), storage_context=storage_context, + dataset_shards=dataset_shards or {}, checkpoint=checkpoint, - dataset_manager=dataset_manager, ) # Configure the train and root logger for the worker processes. if ray_constants.env_bool( diff --git a/python/ray/train/v2/tests/test_data_integration.py b/python/ray/train/v2/tests/test_data_integration.py index ff5d5acf9b0f..fe8159d5190f 100644 --- a/python/ray/train/v2/tests/test_data_integration.py +++ b/python/ray/train/v2/tests/test_data_integration.py @@ -1,4 +1,3 @@ -import asyncio from unittest.mock import MagicMock import pytest @@ -8,11 +7,7 @@ from ray.data import DataContext, ExecutionResources from ray.data._internal.iterator.stream_split_iterator import StreamSplitDataIterator from ray.data.tests.conftest import restore_data_context # noqa: F401 -from ray.train.v2._internal.callbacks.datasets import ( - DatasetManager, - DatasetShardMetadata, - DatasetsSetupCallback, -) +from ray.train.v2._internal.callbacks import DatasetsSetupCallback from ray.train.v2._internal.execution.context import TrainRunContext from ray.train.v2._internal.execution.worker_group.worker_group import ( WorkerGroupContext, @@ -92,31 +87,13 @@ def test_dataset_setup_callback(ray_start_4_cpus): data_config=data_config, scaling_config=scaling_config, ) - dataset_manager_for_each_worker = callback.before_init_train_context( - worker_group.get_workers() - )["dataset_manager"] - assert len(dataset_manager_for_each_worker) == NUM_WORKERS - - # We should send the same dataset manager to all workers. - dataset_manager = dataset_manager_for_each_worker[0] - assert all( - manager == dataset_manager for manager in dataset_manager_for_each_worker - ) - - def get_rank_0_shard(dataset_name: str): - for i in range(1, NUM_WORKERS): - dataset_manager.get_dataset_shard.remote( - DatasetShardMetadata(dataset_name=dataset_name, world_rank=i) - ) - - return ray.get( - dataset_manager.get_dataset_shard.remote( - DatasetShardMetadata(dataset_name=dataset_name, world_rank=0) - ) - ) + dataset_shards = callback.before_init_train_context(worker_group.get_workers())[ + "dataset_shards" + ] + assert len(dataset_shards) == NUM_WORKERS - processed_train_ds = get_rank_0_shard("train") - processed_valid_ds = get_rank_0_shard("valid") + processed_train_ds = dataset_shards[0]["train"] + processed_valid_ds = dataset_shards[0]["valid"] assert isinstance(processed_train_ds, StreamSplitDataIterator) assert not isinstance(processed_valid_ds, StreamSplitDataIterator) @@ -132,187 +109,6 @@ def get_rank_0_shard(dataset_name: str): ) -async def get_dataset_shard_for_worker( - dataset_manager: DatasetManager, - dataset_name: str, - num_workers: int, - worker_rank: int, -): - return await asyncio.create_task( - dataset_manager.get_dataset_shard( - DatasetShardMetadata(dataset_name=dataset_name, world_rank=worker_rank) - ) - ) - - -async def get_dataset_shard_for_all_workers( - dataset_manager: DatasetManager, - dataset_name: str, - num_workers: int, -): - return await asyncio.gather( - *[ - get_dataset_shard_for_worker(dataset_manager, dataset_name, num_workers, i) - for i in range(num_workers) - ] - ) - - -@pytest.mark.asyncio -async def test_get_multiple_datasets_serially(ray_start_4_cpus): - """Tests DatasetManager.get_dataset_shard for multiple datasets, - called serially by each worker. This is the typical case. - - Workers 0, 1: - ray.train.get_dataset_shard("sharded_1") - ray.train.get_dataset_shard("sharded_2") - ray.train.get_dataset_shard("unsharded") - """ - - NUM_ROWS = 100 - NUM_TRAIN_WORKERS = 2 - - sharded_ds_1 = ray.data.range(NUM_ROWS) - sharded_ds_2 = ray.data.range(NUM_ROWS) - unsharded_ds = ray.data.range(NUM_ROWS) - - dataset_manager = DatasetManager( - datasets={ - "sharded_1": sharded_ds_1, - "sharded_2": sharded_ds_2, - "unsharded": unsharded_ds, - }, - data_config=ray.train.DataConfig(datasets_to_split=["sharded_1", "sharded_2"]), - data_context=DataContext.get_current(), - world_size=NUM_TRAIN_WORKERS, - worker_node_ids=None, - ) - - shards = await get_dataset_shard_for_all_workers( - dataset_manager, "sharded_1", NUM_TRAIN_WORKERS - ) - assert all(isinstance(shard, StreamSplitDataIterator) for shard in shards) - assert [shard._base_dataset.name for shard in shards] == [ - "sharded_1" - ] * NUM_TRAIN_WORKERS - - shards = await get_dataset_shard_for_all_workers( - dataset_manager, "sharded_2", NUM_TRAIN_WORKERS - ) - assert all(isinstance(shard, StreamSplitDataIterator) for shard in shards) - assert [shard._base_dataset.name for shard in shards] == [ - "sharded_2" - ] * NUM_TRAIN_WORKERS - - shards = await get_dataset_shard_for_all_workers( - dataset_manager, "unsharded", NUM_TRAIN_WORKERS - ) - assert not any(isinstance(shard, StreamSplitDataIterator) for shard in shards) - assert [shard._base_dataset.name for shard in shards] == [ - "unsharded" - ] * NUM_TRAIN_WORKERS - - -@pytest.mark.asyncio -async def test_get_multiple_datasets_interleaved(ray_start_4_cpus): - """Tests DatasetManager.get_dataset_shard for multiple datasets, - called in an interleaved order by workers. - - Worker 0: - ray.train.get_dataset_shard("train") - ray.train.get_dataset_shard("valid") - - Worker 1: - ray.train.get_dataset_shard("valid") - ray.train.get_dataset_shard("train") - """ - - NUM_ROWS = 100 - NUM_TRAIN_WORKERS = 2 - - train_ds = ray.data.range(NUM_ROWS) - valid_ds = ray.data.range(NUM_ROWS) - - dataset_manager = DatasetManager( - datasets={"train": train_ds, "valid": valid_ds}, - data_config=ray.train.DataConfig(datasets_to_split="all"), - data_context=DataContext.get_current(), - world_size=NUM_TRAIN_WORKERS, - worker_node_ids=None, - ) - - tasks = [ - get_dataset_shard_for_worker(dataset_manager, "train", NUM_TRAIN_WORKERS, 0), - get_dataset_shard_for_worker(dataset_manager, "valid", NUM_TRAIN_WORKERS, 1), - get_dataset_shard_for_worker(dataset_manager, "train", NUM_TRAIN_WORKERS, 1), - get_dataset_shard_for_worker(dataset_manager, "valid", NUM_TRAIN_WORKERS, 0), - ] - iterators = await asyncio.gather(*tasks) - assert all(isinstance(iterator, StreamSplitDataIterator) for iterator in iterators) - assert [iterator._base_dataset.name for iterator in iterators] == [ - "train", - "valid", - "train", - "valid", - ] - - -@pytest.mark.asyncio -async def test_get_multiple_datasets_rank_specific(ray_start_4_cpus): - """Tests rank-specific DatasetManager.get_dataset_shard calls. - - # Epoch 1 - ray.train.get_dataset_shard("train") - - # Validation, which only happens on worker 0. - if world_rank == 0: - ray.train.get_dataset_shard("valid") - - # Epoch 2 - ray.train.get_dataset_shard("train") - """ - - NUM_ROWS = 100 - NUM_TRAIN_WORKERS = 2 - - train_ds = ray.data.range(NUM_ROWS) - valid_ds = ray.data.range(NUM_ROWS) - - dataset_manager = DatasetManager( - datasets={"train": train_ds, "valid": valid_ds}, - data_config=ray.train.DataConfig(datasets_to_split=["train"]), - data_context=DataContext.get_current(), - world_size=NUM_TRAIN_WORKERS, - worker_node_ids=None, - ) - - # ray.train.get_dataset_shard("train") - iterators = await get_dataset_shard_for_all_workers( - dataset_manager, "train", NUM_TRAIN_WORKERS - ) - assert all(isinstance(iterator, StreamSplitDataIterator) for iterator in iterators) - assert [iterator._base_dataset.name for iterator in iterators] == [ - "train" - ] * NUM_TRAIN_WORKERS - - # if world_rank == 0: - # ray.train.get_dataset_shard("valid") - iterator = await get_dataset_shard_for_worker( - dataset_manager, "valid", NUM_TRAIN_WORKERS, 0 - ) - assert not isinstance(iterator, StreamSplitDataIterator) - assert iterator._base_dataset.name == "valid" - - # ray.train.get_dataset_shard("train") - iterators = await get_dataset_shard_for_all_workers( - dataset_manager, "train", NUM_TRAIN_WORKERS - ) - assert all(isinstance(iterator, StreamSplitDataIterator) for iterator in iterators) - assert [iterator._base_dataset.name for iterator in iterators] == [ - "train" - ] * NUM_TRAIN_WORKERS - - if __name__ == "__main__": import sys From 359d241d9a741a294fb08194360fed8f2349f2b3 Mon Sep 17 00:00:00 2001 From: goutamvenkat-anyscale Date: Tue, 19 Aug 2025 21:55:11 -0700 Subject: [PATCH 190/634] [Data] Make _hash_partition a separate function (#55759) --- .../_internal/arrow_ops/transform_pyarrow.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py index 71a85c91d168..b97cf4d641c6 100644 --- a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py +++ b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py @@ -67,6 +67,21 @@ def _create_empty_table(schema: "pyarrow.Schema"): return pa.table(arrays, schema=schema) +def _hash_partition( + table: "pyarrow.Table", + num_partitions: int, +) -> np.ndarray: + + partitions = np.zeros((table.num_rows,), dtype=np.int64) + for i in range(table.num_rows): + _tuple = tuple(c[i] for c in table.columns) + partitions[i] = hash(_tuple) % num_partitions + + # Convert to ndarray to compute hash partition indices + # more efficiently + return partitions + + def hash_partition( table: "pyarrow.Table", *, @@ -90,15 +105,7 @@ def hash_partition( return {0: table} projected_table = table.select(hash_cols) - - partitions = np.zeros((projected_table.num_rows,)) - for i in range(projected_table.num_rows): - _tuple = tuple(c[i] for c in projected_table.columns) - partitions[i] = hash(_tuple) % num_partitions - - # Convert to ndarray to compute hash partition indices - # more efficiently - partitions_array = np.asarray(partitions) + partitions_array = _hash_partition(projected_table, num_partitions=num_partitions) # For every partition compile list of indices of rows falling # under that partition indices = [np.where(partitions_array == p)[0] for p in range(num_partitions)] From 5c3131182720819976947b551c9f62d76c95d1ec Mon Sep 17 00:00:00 2001 From: lkchen Date: Tue, 19 Aug 2025 23:59:41 -0700 Subject: [PATCH 191/634] [depset] make `compile_llm_requirements.sh` runnable on macos (#55664) apple silicon macbooks only though. --------- Signed-off-by: Linkun --- BUILD.bazel | 25 +++++++++++++++++++ WORKSPACE | 15 ++++++++++- ci/compile_llm_requirements.sh | 4 +-- ci/raydepsets/BUILD.bazel | 2 +- ci/raydepsets/cli.py | 13 ++++++---- ci/raydepsets/rayllm.depsets.yaml | 1 + .../requirements_compiled_ray_py311_cpu.txt | 2 +- .../requirements_compiled_ray_py311_cu121.txt | 2 +- .../requirements_compiled_ray_py311_cu128.txt | 2 +- ...quirements_compiled_ray_test_py311_cpu.txt | 2 +- ...irements_compiled_ray_test_py311_cu121.txt | 2 +- ...irements_compiled_ray_test_py311_cu128.txt | 2 +- ...requirements_compiled_rayllm_py311_cpu.txt | 2 +- ...quirements_compiled_rayllm_py311_cu121.txt | 2 +- ...quirements_compiled_rayllm_py311_cu128.txt | 2 +- ...rements_compiled_rayllm_test_py311_cpu.txt | 2 +- ...ments_compiled_rayllm_test_py311_cu121.txt | 2 +- ...ments_compiled_rayllm_test_py311_cu128.txt | 2 +- 18 files changed, 63 insertions(+), 21 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 55adc25a0bc2..96d476d599ca 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -87,6 +87,31 @@ config_setting( flag_values = {":jemalloc_flag": "true"}, ) +config_setting( + name = "linux_x86_64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) + +config_setting( + name = "darwin_aarch64", + constraint_values = [ + "@platforms//os:osx", + "@platforms//cpu:aarch64", + ], +) + +alias( + name = "uv_file", + actual = select({ + "//:linux_x86_64": "@uv_x86_64-linux//:file", + "//:darwin_aarch64": "@uv_aarch64-darwin//:file", + "//conditions:default": "@uv_x86_64-linux//:file", + }), +) + # bazel run :refresh_compile_commands for compile_commands generation for clangd # https://github.com/hedronvision/bazel-compile-commands-extractor?tab=readme-ov-file#vscode - directions for clangd config refresh_compile_commands( diff --git a/WORKSPACE b/WORKSPACE index e56c6a109b05..086d69f189ca 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -117,7 +117,7 @@ filegroup( ) http_archive( - name = "uv_x86_64", + name = "uv_x86_64-linux", build_file_content = """ filegroup( name = "file", @@ -129,6 +129,19 @@ filegroup( urls = ["https://github.com/astral-sh/uv/releases/download/0.8.10/uv-x86_64-unknown-linux-gnu.tar.gz"], ) +http_archive( + name = "uv_aarch64-darwin", + build_file_content = """ +filegroup( + name = "file", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) +""", + sha256 = "5200278ae00b5c0822a7db7a99376b2167e8e9391b29c3de22f9e4fdebc9c0e8", + urls = ["https://github.com/astral-sh/uv/releases/download/0.8.10/uv-aarch64-apple-darwin.tar.gz"], +) + http_archive( name = "com_github_storypku_bazel_iwyu", sha256 = "aa78c331a2cb139f73f7d74eeb4d5ab29794af82023ef5d6d5194f76b7d37449", diff --git a/ci/compile_llm_requirements.sh b/ci/compile_llm_requirements.sh index f3ae705fe228..a9b5f9d06660 100755 --- a/ci/compile_llm_requirements.sh +++ b/ci/compile_llm_requirements.sh @@ -13,8 +13,8 @@ mkdir -p /tmp/ray-deps # Remove the GPU constraints cp python/requirements_compiled.txt /tmp/ray-deps/requirements_compiled.txt -sed -i '/^--extra-index-url /d' /tmp/ray-deps/requirements_compiled.txt -sed -i '/^--find-links /d' /tmp/ray-deps/requirements_compiled.txt +sed -e '/^--extra-index-url /d' -e '/^--find-links /d' /tmp/ray-deps/requirements_compiled.txt > /tmp/ray-deps/requirements_compiled.txt.tmp +mv /tmp/ray-deps/requirements_compiled.txt.tmp /tmp/ray-deps/requirements_compiled.txt bazel run //ci/raydepsets:raydepsets -- build ci/raydepsets/rayllm.depsets.yaml diff --git a/ci/raydepsets/BUILD.bazel b/ci/raydepsets/BUILD.bazel index eb4d56c06a20..b253519ec9dd 100644 --- a/ci/raydepsets/BUILD.bazel +++ b/ci/raydepsets/BUILD.bazel @@ -14,7 +14,7 @@ py_library( srcs = [ "cli.py", ], - data = ["@uv_x86_64//:file"], + data = ["//:uv_file"], deps = [ ":workspace", ci_require("bazel-runfiles"), diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index 867e33c21060..4d1037387a07 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -274,8 +274,11 @@ def _override_uv_flags(flags: List[str], args: List[str]) -> List[str]: def _uv_binary(): r = runfiles.Create() system = platform.system() - if system != "Linux" or platform.processor() != "x86_64": - raise RuntimeError( - f"Unsupported platform/processor: {system}/{platform.processor()}" - ) - return r.Rlocation("uv_x86_64/uv-x86_64-unknown-linux-gnu/uv") + processor = platform.processor() + + if system == "Linux" and processor == "x86_64": + return r.Rlocation("uv_x86_64-linux/uv-x86_64-unknown-linux-gnu/uv") + elif system == "Darwin" and (processor == "arm" or processor == "aarch64"): + return r.Rlocation("uv_aarch64-darwin/uv-aarch64-apple-darwin/uv") + else: + raise RuntimeError(f"Unsupported platform/processor: {system}/{processor}") diff --git a/ci/raydepsets/rayllm.depsets.yaml b/ci/raydepsets/rayllm.depsets.yaml index e2a7a030b5e1..e50b8fc4837d 100644 --- a/ci/raydepsets/rayllm.depsets.yaml +++ b/ci/raydepsets/rayllm.depsets.yaml @@ -15,6 +15,7 @@ build_arg_sets: - --extra-index-url https://download.pytorch.org/whl/${CUDA_CODE} append_flags: - --python-version=3.11 + - --python-platform=linux build_arg_sets: - cpu - cu121 diff --git a/python/requirements_compiled_ray_py311_cpu.txt b/python/requirements_compiled_ray_py311_cpu.txt index d8911ad845de..ad9a94b18e3b 100644 --- a/python/requirements_compiled_ray_py311_cpu.txt +++ b/python/requirements_compiled_ray_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/requirements_compiled_ray_py311_cu121.txt b/python/requirements_compiled_ray_py311_cu121.txt index 7bf7aa57a1a7..40b0814f0d07 100644 --- a/python/requirements_compiled_ray_py311_cu121.txt +++ b/python/requirements_compiled_ray_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/requirements_compiled_ray_py311_cu128.txt b/python/requirements_compiled_ray_py311_cu128.txt index c126614250e6..e03e8073bde2 100644 --- a/python/requirements_compiled_ray_py311_cu128.txt +++ b/python/requirements_compiled_ray_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/python/requirements_compiled_ray_test_py311_cpu.txt b/python/requirements_compiled_ray_test_py311_cpu.txt index d7a2552b6dc4..6d33de854539 100644 --- a/python/requirements_compiled_ray_test_py311_cpu.txt +++ b/python/requirements_compiled_ray_test_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/requirements_compiled_ray_test_py311_cu121.txt b/python/requirements_compiled_ray_test_py311_cu121.txt index ec4ae33bcdb5..8c0d7d9b187a 100644 --- a/python/requirements_compiled_ray_test_py311_cu121.txt +++ b/python/requirements_compiled_ray_test_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/requirements_compiled_ray_test_py311_cu128.txt b/python/requirements_compiled_ray_test_py311_cu128.txt index f27f420acd03..b61e16287344 100644 --- a/python/requirements_compiled_ray_test_py311_cu128.txt +++ b/python/requirements_compiled_ray_test_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/python/requirements_compiled_rayllm_py311_cpu.txt b/python/requirements_compiled_rayllm_py311_cpu.txt index 21c5a9872eff..3c7bccfc3a3e 100644 --- a/python/requirements_compiled_rayllm_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 -c python/requirements_compiled_rayllm_test_py311_cpu.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/requirements_compiled_rayllm_test_py311_cpu.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/requirements_compiled_rayllm_py311_cu121.txt b/python/requirements_compiled_rayllm_py311_cu121.txt index 530878b8b211..5a778ecca825 100644 --- a/python/requirements_compiled_rayllm_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 -c python/requirements_compiled_rayllm_test_py311_cu121.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_rayllm_test_py311_cu121.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/requirements_compiled_rayllm_py311_cu128.txt b/python/requirements_compiled_rayllm_py311_cu128.txt index e7ec34152746..a3b95b0166a9 100644 --- a/python/requirements_compiled_rayllm_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 -c python/requirements_compiled_rayllm_test_py311_cu128.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_rayllm_test_py311_cu128.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/python/requirements_compiled_rayllm_test_py311_cpu.txt b/python/requirements_compiled_rayllm_test_py311_cpu.txt index 0fd5d38ace03..b69b89a5ea19 100644 --- a/python/requirements_compiled_rayllm_test_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_test_py311_cpu.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cpu.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/requirements_compiled_rayllm_test_py311_cu121.txt b/python/requirements_compiled_rayllm_test_py311_cu121.txt index c89112408569..ebf1056ff7c6 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu121.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu121.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/requirements_compiled_rayllm_test_py311_cu128.txt b/python/requirements_compiled_rayllm_test_py311_cu128.txt index 9bcf8ad47312..913cbd464409 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu128.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu128.txt --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 From 27a57a649f280aa8b9d48f3747db85d96a780121 Mon Sep 17 00:00:00 2001 From: Kamil Kaczmarek Date: Wed, 20 Aug 2025 02:30:05 -0700 Subject: [PATCH 192/634] [RLlib; docs] Fix formatting of class references. (#55764) --- doc/source/rllib/multi-agent-envs.rst | 15 +++++++-------- .../rllib/new-api-stack-migration-guide.rst | 4 ++-- doc/source/rllib/rllib-examples.rst | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/doc/source/rllib/multi-agent-envs.rst b/doc/source/rllib/multi-agent-envs.rst index 52312b98fefc..9c4a48e10a55 100644 --- a/doc/source/rllib/multi-agent-envs.rst +++ b/doc/source/rllib/multi-agent-envs.rst @@ -34,20 +34,19 @@ RLlib's MultiAgentEnv API .. hint:: - This paragraph describes RLlib's own :py:class`~ray.rllib.env.multi_agent_env.MultiAgentEnv` API, which is the + This paragraph describes RLlib's own :py:class:`~ray.rllib.env.multi_agent_env.MultiAgentEnv` API, which is the recommended way of defining your own multi-agent environment logic. However, if you are already using a third-party multi-agent API, RLlib offers wrappers for :ref:`Farama's PettingZoo API ` as well as :ref:`DeepMind's OpenSpiel API `. -The :py:class`~ray.rllib.env.multi_agent_env.MultiAgentEnv` API of RLlib closely follows the +The :py:class:`~ray.rllib.env.multi_agent_env.MultiAgentEnv` API of RLlib closely follows the conventions and APIs of `Farama's gymnasium (single-agent) `__ envs and even subclasses from `gymnasium.Env`, however, instead of publishing individual observations, rewards, and termination/truncation flags -from `reset()` and `step()`, a custom :py:class`~ray.rllib.env.multi_agent_env.MultiAgentEnv` implementation -outputs dictionaries, one for observations, one for rewards, etc..in which agent IDs map -In each such multi-agent dictionary, agent IDs map to the respective individual agent's observation/reward/etc.. +from `reset()` and `step()`, a custom :py:class:`~ray.rllib.env.multi_agent_env.MultiAgentEnv` implementation +outputs separate dictionaries for observations, rewards, etc., where each dictionary maps agent IDs to the corresponding values for each agent. -Here is a first draft of an example :py:class`~ray.rllib.env.multi_agent_env.MultiAgentEnv` implementation: +Here is a first draft of an example :py:class:`~ray.rllib.env.multi_agent_env.MultiAgentEnv` implementation: .. code-block:: @@ -72,7 +71,7 @@ Here is a first draft of an example :py:class`~ray.rllib.env.multi_agent_env.Mul Agent Definitions ~~~~~~~~~~~~~~~~~ -The number of agents in your environment and their IDs are entirely controlled by your :py:class`~ray.rllib.env.multi_agent_env.MultiAgentEnv` +The number of agents in your environment and their IDs are entirely controlled by your :py:class:`~ray.rllib.env.multi_agent_env.MultiAgentEnv` code. Your env decides, which agents start after an episode reset, which agents enter the episode at a later point, which agents terminate the episode early, and which agents stay in the episode until the entire episode ends. @@ -371,7 +370,7 @@ you can use grouping in conjunction with the policy mapping API described in pri Third Party Multi-Agent Env APIs -------------------------------- -Besides RLlib's own :py:class`~ray.rllib.env.multi_agent_env.MultiAgentEnv` API, you can also use +Besides RLlib's own :py:class:`~ray.rllib.env.multi_agent_env.MultiAgentEnv` API, you can also use various third-party APIs and libraries to implement custom multi-agent envs. diff --git a/doc/source/rllib/new-api-stack-migration-guide.rst b/doc/source/rllib/new-api-stack-migration-guide.rst index 9eb426dcca93..9ab92d53be0c 100644 --- a/doc/source/rllib/new-api-stack-migration-guide.rst +++ b/doc/source/rllib/new-api-stack-migration-guide.rst @@ -330,7 +330,7 @@ Custom callbacks ---------------- If you're using custom callbacks on the old API stack, you're subclassing the ``DefaultCallbacks`` class, -which the Ray team renamed to :py:class`~ray.rllib.callbacks.callbacks.RLlibCallback`. +which the Ray team renamed to :py:class:`~ray.rllib.callbacks.callbacks.RLlibCallback`. You can continue this approach with the new API stack and pass your custom subclass to your config like the following: .. testcode:: @@ -340,7 +340,7 @@ You can continue this approach with the new API stack and pass your custom subcl However, if you're overriding those methods that triggered on the :py:class:`~ray.rllib.env.env_runner.EnvRunner` side, for example, ``on_episode_start/stop/step/etc...``, you may have to translate some call arguments. -The following is a one-to-one translation guide for these types of :py:class`~ray.rllib.callbacks.callbacks.RLlibCallback` +The following is a one-to-one translation guide for these types of :py:class:`~ray.rllib.callbacks.callbacks.RLlibCallback` methods: .. testcode:: diff --git a/doc/source/rllib/rllib-examples.rst b/doc/source/rllib/rllib-examples.rst index 16d49f4917d0..f5842747d60c 100644 --- a/doc/source/rllib/rllib-examples.rst +++ b/doc/source/rllib/rllib-examples.rst @@ -198,7 +198,7 @@ Environments - `Async gym vectorization, parallelizing sub-environments `__: Shows how the `gym_env_vectorize_mode` config setting can significantly speed up your - :py:class`~ray.rllib.env.env_runner.EnvRunner` actors, if your RL environment is slow and you are + :py:class:`~ray.rllib.env.env_runner.EnvRunner` actors, if your RL environment is slow and you're using `num_envs_per_env_runner > 1`. The reason for the performance gain is that each sub-environment runs in its own process. - `Custom env rendering method `__: From decdb4bfc5497d2eb5a77f58f3d6e66626906fe3 Mon Sep 17 00:00:00 2001 From: Hassam Ullah Sheikh Date: Wed, 20 Aug 2025 02:48:11 -0700 Subject: [PATCH 193/634] [RLlib; docs] Fixing typo in the RLlib documentation. (#55752) --- doc/source/rllib/key-concepts.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/rllib/key-concepts.rst b/doc/source/rllib/key-concepts.rst index 7d71b1fc8353..18bbb82f91fa 100644 --- a/doc/source/rllib/key-concepts.rst +++ b/doc/source/rllib/key-concepts.rst @@ -17,12 +17,12 @@ key concepts and general architecture of RLlib. **RLlib overview:** The central component of RLlib is the :py:class:`~ray.rllib.algorithms.algorithm.Algorithm` class, acting as a runtime for executing your RL experiments. Your gateway into using an :ref:`Algorithm ` is the - :py:class:`~ray.rllib.algorithms.algorithm_config.AlgorithmConfig` (cyan) class, allowing + :py:class:`~ray.rllib.algorithms.algorithm_config.AlgorithmConfig` (cyan) class, allowing you to manage available configuration settings, for example learning rate or model architecture. Most :py:class:`~ray.rllib.algorithms.algorithm.Algorithm` objects have - :py:class:`~ray.rllib.env.env_runner.EnvRunner` actors (blue) to collect training samples + :py:class:`~ray.rllib.env.env_runner.EnvRunner` actors (blue) to collect training samples from the :ref:`RL environment ` and - :py:class:`~ray.rllib.core.learner.learner.Learner` actors (yellow) + :py:class:`~ray.rllib.core.learner.learner.Learner` actors (yellow) to compute gradients and update your :ref:`models `. The algorithm synchronizes model weights after an update. From f9bea9d1bfaea57589db40876da28c0d4c54fb37 Mon Sep 17 00:00:00 2001 From: MatthewCWeston <61944935+MatthewCWeston@users.noreply.github.com> Date: Wed, 20 Aug 2025 10:14:51 -0500 Subject: [PATCH 194/634] [RLlib] Fixes Implementation of Shared Encoder (#54571) ## Why are these changes needed? The examples section features an implementation of a shared encoder, but the code currently present crashes when run due to a few bugs: - The custom MultiRLModule's modified forward method does not run, as it does not override _forward_exploration, forward_train, and _forward_inference. It might be beneficial to refactor the base MultiRLModule class to facilitate easier extensibility; this PR overrides VPGMultiRLModuleWithSharedEncoder with the best solution I could identify. - The testcode instantiates a PPOConfig, but the modules provided are designed for VPG. I've changed the config line accordingly. - One of the imports in the testcode specified an older location for MultiRLModuleSpec, which I patched. - Additionally, I've implemented a custom VPG learner with a shared optimizer based on Sven's notes about stabilizing training below the testcode. This is used in the new example script. I have included a working example script, multi_agent/shared_encoder_cartpole.py, which demonstrates the shared encoder's functionality. As pettingzoo_shared_value_function.py is still in development, it provides a working example of a shared module under the new API. ## Related issue number Fixes a crash in the example code that persists after the fixes made in Commit [5c64312](https://github.com/ray-project/ray/commit/5c64312736e99f816dd57bf8990a7d7480a8e071). Relates to discussion on the official forum [here](https://discuss.ray.io/t/best-practices-for-implementing-a-shared-critic/22673/3). ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [x] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [x] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Matthew Co-authored-by: Kamil Kaczmarek Co-authored-by: Artur Niederfahrenhorst Co-authored-by: Artur Niederfahrenhorst --- rllib/BUILD | 15 ++ .../vpg_torch_learner_shared_optimizer.py | 32 ++++ .../multi_agent/shared_encoder_cartpole.py | 164 ++++++++++++++++++ .../classes/vpg_using_shared_encoder_rlm.py | 143 +++++++++++---- 4 files changed, 323 insertions(+), 31 deletions(-) create mode 100644 rllib/examples/learners/classes/vpg_torch_learner_shared_optimizer.py create mode 100644 rllib/examples/multi_agent/shared_encoder_cartpole.py diff --git a/rllib/BUILD b/rllib/BUILD index 6c061cedb07f..b35b343ef212 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -5013,6 +5013,21 @@ py_test( ], ) +py_test( + name = "examples/multi_agent/shared_encoder_cartpole", + size = "medium", + srcs = ["examples/multi_agent/shared_encoder_cartpole.py"], + args = [ + "--stop-iter=10", + ], + main = "examples/multi_agent/shared_encoder_cartpole.py", + tags = [ + "examples", + "exclusive", + "team:rllib", + ], +) + # @OldAPIStack py_test( name = "examples/multi_agent/self_play_with_open_spiel_connect_4_ppo_tf_old_api_stack", diff --git a/rllib/examples/learners/classes/vpg_torch_learner_shared_optimizer.py b/rllib/examples/learners/classes/vpg_torch_learner_shared_optimizer.py new file mode 100644 index 000000000000..4594cf1f2f28 --- /dev/null +++ b/rllib/examples/learners/classes/vpg_torch_learner_shared_optimizer.py @@ -0,0 +1,32 @@ +from ray.rllib.examples.learners.classes.vpg_torch_learner import VPGTorchLearner +from ray.rllib.core.learner.torch.torch_learner import TorchLearner +from ray.rllib.utils.annotations import override +from ray.rllib.utils.framework import try_import_torch + +torch, _ = try_import_torch() + + +class VPGTorchLearnerSharedOptimizer(VPGTorchLearner): + """ + In order for a shared module to learn properly, a special, multi-agent Learner + has been set up. There is only one optimizer (used to train all submodules, e.g. + a shared encoder and n policy nets), in order to not destabilize learning. The + latter may happen if more than one optimizer would try to alternatingly optimize + the same shared submodule. + """ + + @override(TorchLearner) + def configure_optimizers(self) -> None: + # Get and aggregate parameters for every module + param_list = [] + for m in self.module.values(): + if self.rl_module_is_compatible(m): + param_list.extend(m.parameters()) + + self.register_optimizer( + optimizer_name="shared_optimizer", + optimizer=torch.optim.Adam(params=param_list), + params=param_list, + # For the policy learning rate, we use the "main" lr in the AlgorithmConfig. + lr_or_lr_schedule=self.config.lr, + ) diff --git a/rllib/examples/multi_agent/shared_encoder_cartpole.py b/rllib/examples/multi_agent/shared_encoder_cartpole.py new file mode 100644 index 000000000000..caea04adef8c --- /dev/null +++ b/rllib/examples/multi_agent/shared_encoder_cartpole.py @@ -0,0 +1,164 @@ +"""A runnable example involving the use of a shared encoder module. + +How to run this script +---------------------- +`python [script file name].py --num-agents=2` + +Control the number of agents and policies (RLModules) via --num-agents. +--encoder-emb-dim sets the encoder output dimension, and --no-shared-encoder +runs the experiment with independent encoders. + +For debugging, use the following additional command line options +`--no-tune --num-env-runners=0` +which should allow you to set breakpoints anywhere in the RLlib code and +have the execution stop there for inspection and debugging. + +For logging to your WandB account, use: +`--wandb-key=[your WandB API key] --wandb-project=[some project name] +--wandb-run-name=[optional: WandB run name (within the defined project)]` + + +Results to expect +----------------- +Under the shared encoder architecture, the target reward of 700 will typically be reached well before 100,000 iterations. A trial concludes as below: + ++---------------------+------------+-----------------+--------+------------------+-------+-------------------+-------------+-------------+ +| Trial name | status | loc | iter | total time (s) | ts | combined return | return p1 | return p0 | +|---------------------+------------+-----------------+--------+------------------+-------+-------------------+-------------+-------------| +| VPG_env_ab318_00000 | TERMINATED | 127.0.0.1:37375 | 33 | 44.2689 | 74197 | 611.35 | 191.71 | 419.64 | ++---------------------+------------+-----------------+--------+------------------+-------+-------------------+-------------+-------------+ + +Without a shared encoder, a lower reward is typically achieved after training for the full 100,000 timesteps: + ++---------------------+------------+-----------------+--------+------------------+--------+-------------------+-------------+-------------+ +| Trial name | status | loc | iter | total time (s) | ts | combined return | return p0 | return p1 | +|---------------------+------------+-----------------+--------+------------------+--------+-------------------+-------------+-------------| +| VPG_env_2e79e_00000 | TERMINATED | 127.0.0.1:39076 | 37 | 52.127 | 103894 | 526.66 | 85.78 | 440.88 | ++---------------------+------------+-----------------+--------+------------------+--------+-------------------+-------------+-------------+ + + +""" + +import gymnasium as gym +from ray.rllib.core.rl_module.rl_module import RLModuleSpec +from ray.rllib.core.rl_module.multi_rl_module import MultiRLModuleSpec + +from ray.rllib.examples.algorithms.classes.vpg import VPGConfig +from ray.rllib.examples.learners.classes.vpg_torch_learner_shared_optimizer import ( + VPGTorchLearnerSharedOptimizer, +) +from ray.rllib.examples.envs.classes.multi_agent import MultiAgentCartPole +from ray.rllib.examples.rl_modules.classes.vpg_using_shared_encoder_rlm import ( + SHARED_ENCODER_ID, + SharedEncoder, + VPGPolicyAfterSharedEncoder, + VPGMultiRLModuleWithSharedEncoder, + VPGPolicyNoSharedEncoder, +) +from ray.rllib.utils.test_utils import ( + add_rllib_example_script_args, + run_rllib_example_script_experiment, +) +from ray.tune.registry import register_env + +parser = add_rllib_example_script_args( + default_iters=200, + default_timesteps=100000, + default_reward=600.0, +) +parser.set_defaults( + algo="VPG", + num_agents=2, +) +parser.add_argument("--encoder-emb-dim", type=int, default=64) +parser.add_argument("--no-shared-encoder", action="store_true") + +if __name__ == "__main__": + args = parser.parse_args() + assert args.algo == "VPG", "The shared encoder example is meant for VPG agents." + assert args.num_agents == 2, "This example makes use of two agents." + + single_agent_env = gym.make( + "CartPole-v1" + ) # To allow instantiation of shared encoder + + EMBEDDING_DIM = args.encoder_emb_dim # encoder output dim + + if args.no_shared_encoder: + print("Running experiment without shared encoder") + specs = MultiRLModuleSpec( + rl_module_specs={ + # Large policy net. + "p0": RLModuleSpec( + module_class=VPGPolicyNoSharedEncoder, + model_config={ + "embedding_dim": EMBEDDING_DIM, + "hidden_dim": 64, + }, + ), + # Small policy net. + "p1": RLModuleSpec( + module_class=VPGPolicyNoSharedEncoder, + model_config={ + "embedding_dim": EMBEDDING_DIM, + "hidden_dim": 64, + }, + ), + } + ) + else: + specs = MultiRLModuleSpec( + multi_rl_module_class=VPGMultiRLModuleWithSharedEncoder, + rl_module_specs={ + # Shared encoder. + SHARED_ENCODER_ID: RLModuleSpec( + module_class=SharedEncoder, + model_config={"embedding_dim": EMBEDDING_DIM}, + observation_space=single_agent_env.observation_space, + action_space=single_agent_env.action_space, + ), + # Large policy net. + "p0": RLModuleSpec( + module_class=VPGPolicyAfterSharedEncoder, + model_config={ + "embedding_dim": EMBEDDING_DIM, + "hidden_dim": 64, + }, + ), + # Small policy net. + "p1": RLModuleSpec( + module_class=VPGPolicyAfterSharedEncoder, + model_config={ + "embedding_dim": EMBEDDING_DIM, + "hidden_dim": 64, + }, + ), + }, + ) + + # Register our environment with tune. + register_env( + "env", + lambda _: MultiAgentCartPole(config={"num_agents": args.num_agents}), + ) + + base_config = ( + VPGConfig() + .environment("env" if args.num_agents > 0 else "CartPole-v1") + .training( + learner_class=VPGTorchLearnerSharedOptimizer + if not args.no_shared_encoder + else None, + train_batch_size=2048, + lr=1e-2, + ) + .multi_agent( + policies={"p0", "p1"}, + policy_mapping_fn=lambda agent_id, episode, **kw: f"p{agent_id}", + ) + .rl_module( + rl_module_spec=specs, + ) + ) + + run_rllib_example_script_experiment(base_config, args) diff --git a/rllib/examples/rl_modules/classes/vpg_using_shared_encoder_rlm.py b/rllib/examples/rl_modules/classes/vpg_using_shared_encoder_rlm.py index 05a350de5ff0..2cd2809a6206 100644 --- a/rllib/examples/rl_modules/classes/vpg_using_shared_encoder_rlm.py +++ b/rllib/examples/rl_modules/classes/vpg_using_shared_encoder_rlm.py @@ -3,7 +3,15 @@ from ray.rllib.core import Columns from ray.rllib.core.rl_module.multi_rl_module import MultiRLModule from ray.rllib.core.rl_module.torch.torch_rl_module import TorchRLModule +from ray.rllib.core.models.base import ENCODER_OUT +from ray.rllib.utils.annotations import override +from ray.rllib.utils.typing import ModuleID +from typing import ( + Any, + Dict, + Union, +) SHARED_ENCODER_ID = "shared_encoder" @@ -34,8 +42,7 @@ def setup(self): ) def _forward(self, batch, **kwargs): - # Embeddings can be found in the batch under the "encoder_embeddings" key. - embeddings = batch["encoder_embeddings"] + embeddings = batch[ENCODER_OUT] # Get the output of the encoder logits = self._pi_head(embeddings) return {Columns.ACTION_DIST_INPUTS: logits} @@ -48,23 +55,35 @@ class VPGMultiRLModuleWithSharedEncoder(MultiRLModule): """VPG (vanilla pol. gradient)-style MultiRLModule handling a shared encoder. # __sphinx_doc_mrlm_end__ - This MultiRLModule needs to be configured appropriately as follows: + This MultiRLModule needs to be configured appropriately as below. .. testcode:: # __sphinx_doc_how_to_run_begin__ import gymnasium as gym - from ray.rllib.algorithms.ppo import PPOConfig - from ray.rllib.core import MultiRLModuleSpec, RLModuleSpec + from ray.rllib.core.rl_module.rl_module import RLModuleSpec + from ray.rllib.core.rl_module.multi_rl_module import MultiRLModuleSpec + + from ray.rllib.examples.algorithms.classes.vpg import VPGConfig + from ray.rllib.examples.learners.classes.vpg_torch_learner_shared_optimizer import VPGTorchLearnerSharedOptimizer from ray.rllib.examples.envs.classes.multi_agent import MultiAgentCartPole + from ray.rllib.examples.rl_modules.classes.vpg_using_shared_encoder_rlm import ( + SHARED_ENCODER_ID, + SharedEncoder, + VPGPolicyAfterSharedEncoder, + VPGMultiRLModuleWithSharedEncoder, + ) single_agent_env = gym.make("CartPole-v1") EMBEDDING_DIM = 64 # encoder output dim config = ( - PPOConfig() + VPGConfig() .environment(MultiAgentCartPole, env_config={"num_agents": 2}) + .training( + learner_class=VPGTorchLearnerSharedOptimizer, + ) .multi_agent( # Declare the two policies trained. policies={"p0", "p1"}, @@ -74,6 +93,7 @@ class VPGMultiRLModuleWithSharedEncoder(MultiRLModule): ) .rl_module( rl_module_spec=MultiRLModuleSpec( + multi_rl_module_class=VPGMultiRLModuleWithSharedEncoder, rl_module_specs={ # Shared encoder. SHARED_ENCODER_ID: RLModuleSpec( @@ -102,47 +122,52 @@ class VPGMultiRLModuleWithSharedEncoder(MultiRLModule): ), ) ) - algo = config.build() - print(algo.get_module("p0")) + algo = config.build_algo() + print(algo.train()) # __sphinx_doc_how_to_run_end__ - - Also note that in order to learn properly, a special, multi-agent Learner - accounting for the shared encoder must be setup. This Learner should have only - one optimizer (used to train all submodules: encoder and the n policy nets) in - order to not destabilize learning. The latter would happen if more than one - optimizer would try to alternatingly optimize the same shared encoder submodule. # __sphinx_doc_mrlm_2_begin__ """ def setup(self): # Call the super's setup(). super().setup() - # Assert, we have the shared encoder submodule. - assert ( - SHARED_ENCODER_ID in self._rl_modules - and isinstance(self._rl_modules[SHARED_ENCODER_ID], SharedEncoder) - and len(self._rl_modules) > 1 - ) + assert SHARED_ENCODER_ID in self._rl_modules and len(self._rl_modules) > 1 # Assign the encoder to a convenience attribute. self.encoder = self._rl_modules[SHARED_ENCODER_ID] - def _forward(self, batch, **kwargs): + def _forward(self, batch, forward_type, **kwargs): # Collect our policies' outputs in this dict. - outputs = {} - + fwd_out = {} # Loop through the policy nets (through the given batch's keys). for policy_id, policy_batch in batch.items(): - rl_module = self._rl_modules[policy_id] + # Feed this policy's observation into the shared encoder + encoder_output = self.encoder._forward(batch[policy_id]) + policy_batch[ENCODER_OUT] = encoder_output[ENCODER_OUT] + # Get the desired module + m = getattr(self._rl_modules[policy_id], forward_type) + # Pass the policy's embeddings through the policy net. + fwd_out[policy_id] = m(batch[policy_id], **kwargs) + return fwd_out - # Pass policy's observations through shared encoder to get the features for - # this policy. - policy_batch["encoder_embeddings"] = self.encoder._forward(batch[policy_id]) + # These methods could probably stand to be adjusted in MultiRLModule using something like this, so that subclasses that tweak _forward don't need to rewrite all of them. The prior implementation errored out because of this issue. + @override(MultiRLModule) + def _forward_inference( + self, batch: Dict[str, Any], **kwargs + ) -> Union[Dict[str, Any], Dict[ModuleID, Dict[str, Any]]]: + return self._forward(batch, "_forward_inference", **kwargs) - # Pass the policy's embeddings through the policy net. - outputs[policy_id] = rl_module._forward(batch[policy_id], **kwargs) + @override(MultiRLModule) + def _forward_exploration( + self, batch: Dict[str, Any], **kwargs + ) -> Union[Dict[str, Any], Dict[ModuleID, Dict[str, Any]]]: + return self._forward(batch, "_forward_exploration", **kwargs) - return outputs + @override(MultiRLModule) + def _forward_train( + self, batch: Dict[str, Any], **kwargs + ) -> Union[Dict[str, Any], Dict[ModuleID, Dict[str, Any]]]: + return self._forward(batch, "_forward_train", **kwargs) # __sphinx_doc_mrlm_2_end__ @@ -165,7 +190,63 @@ def setup(self): def _forward(self, batch, **kwargs): # Pass observations through the net and return outputs. - return {"encoder_embeddings": self._net(batch[Columns.OBS])} + return {ENCODER_OUT: self._net(batch[Columns.OBS])} # __sphinx_doc_encoder_end__ + + +# __sphinx_doc_ns_encoder_begin__ +class VPGIndividualEncoder(torch.nn.Module): + def __init__(self, observation_space, embedding_dim): + """ + An individual version of SharedEncoder, supporting direct comparison between + the two architectures. + """ + super().__init__() + + input_dim = observation_space.shape[0] + + # A very simple encoder network. + self._net = torch.nn.Sequential( + torch.nn.Linear(input_dim, embedding_dim), + ) + + def forward(self, batch, **kwargs): + # Pass observations through the net and return outputs. + return {ENCODER_OUT: self._net(batch[Columns.OBS])} + + +# __sphinx_doc_ns_encoder_end__ + + +# __sphinx_doc_ns_policy_begin__ +class VPGPolicyNoSharedEncoder(TorchRLModule): + """ + A VPG (vanilla pol. gradient)-style RLModule that doesn't use a shared encoder. + Facilitates experiments comparing shared and individual encoder architectures. + """ + + def setup(self): + super().setup() + + # Incoming feature dim from the encoder. + embedding_dim = self.model_config["embedding_dim"] + hidden_dim = self.model_config["hidden_dim"] + + self._pi_head = torch.nn.Sequential( + torch.nn.Linear(embedding_dim, hidden_dim), + torch.nn.ReLU(), + torch.nn.Linear(hidden_dim, self.action_space.n), + ) + self.encoder = VPGIndividualEncoder(self.observation_space, embedding_dim) + + def _forward(self, batch, **kwargs): + if ENCODER_OUT not in batch: + batch = self.encoder(batch) + embeddings = batch[ENCODER_OUT] + logits = self._pi_head(embeddings) + return {Columns.ACTION_DIST_INPUTS: logits} + + +# __sphinx_doc_ns_policy_end__ From a36512b8e039c5016b18406c322177f4ffc7614f Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Wed, 20 Aug 2025 08:35:06 -0700 Subject: [PATCH 195/634] [Core] Remove dead code (#55770) Signed-off-by: Jiajun Yao --- python/ray/_private/ray_constants.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/ray/_private/ray_constants.py b/python/ray/_private/ray_constants.py index 01b194e62696..33de100852ba 100644 --- a/python/ray/_private/ray_constants.py +++ b/python/ray/_private/ray_constants.py @@ -107,11 +107,6 @@ def env_set_by_user(key): OBJECT_STORE_MINIMUM_MEMORY_BYTES = 75 * 1024 * 1024 # Each ObjectRef currently uses about 3KB of caller memory. CALLER_MEMORY_USAGE_PER_OBJECT_REF = 3000 -# Match max_direct_call_object_size in -# src/ray/common/ray_config_def.h. -# TODO(swang): Ideally this should be pulled directly from the -# config in case the user overrides it. -DEFAULT_MAX_DIRECT_CALL_OBJECT_SIZE = 100 * 1024 # Above this number of bytes, raise an error by default unless the user sets # RAY_ALLOW_SLOW_STORAGE=1. This avoids swapping with large object stores. REQUIRE_SHM_SIZE_THRESHOLD = 10**10 From 7fbe70710e38507f89301a57b5bb82e1268cd5fd Mon Sep 17 00:00:00 2001 From: Mengjin Yan Date: Wed, 20 Aug 2025 09:37:32 -0700 Subject: [PATCH 196/634] [Core] Expose Default Fields in the Task Json Message (#55765) This PR fixed 2 issues we found in the task event pipeline with additional tests added: 1. When converting the task events to json, add the option to include the fields with default values 2. A typo in generating the add event response when the http endpoint is not configured. --------- Signed-off-by: Mengjin Yan --- python/ray/_private/protobuf_compat.py | 9 +- .../modules/aggregator/aggregator_agent.py | 10 +- .../aggregator/tests/test_aggregator_agent.py | 302 ++++++++++++++++-- 3 files changed, 286 insertions(+), 35 deletions(-) diff --git a/python/ray/_private/protobuf_compat.py b/python/ray/_private/protobuf_compat.py index 66971d8812d9..01256a5a82c3 100644 --- a/python/ray/_private/protobuf_compat.py +++ b/python/ray/_private/protobuf_compat.py @@ -1,6 +1,6 @@ import inspect -from google.protobuf.json_format import MessageToDict +from google.protobuf.json_format import MessageToDict, MessageToJson """ This module provides a compatibility layer for different versions of the protobuf @@ -21,7 +21,7 @@ def _protobuf_has_old_arg_name(): def rename_always_print_fields_with_no_presence(kwargs): """ - Protobuf version 5.26.0rc2 renamed argument for `MessageToDict`: + Protobuf version 5.26.0rc2 renamed argument for `MessageToDict` and `MessageToJson`: `including_default_value_fields` -> `always_print_fields_with_no_presence`. See https://github.com/protocolbuffers/protobuf/commit/06e7caba58ede0220b110b89d08f329e5f8a7537#diff-8de817c14d6a087981503c9aea38730b1b3e98f4e306db5ff9d525c7c304f234L129 # noqa: E501 @@ -45,3 +45,8 @@ def rename_always_print_fields_with_no_presence(kwargs): def message_to_dict(*args, **kwargs): kwargs = rename_always_print_fields_with_no_presence(kwargs) return MessageToDict(*args, **kwargs) + + +def message_to_json(*args, **kwargs): + kwargs = rename_always_print_fields_with_no_presence(kwargs) + return MessageToJson(*args, **kwargs) diff --git a/python/ray/dashboard/modules/aggregator/aggregator_agent.py b/python/ray/dashboard/modules/aggregator/aggregator_agent.py index 967016c39ce9..228685b90fbb 100644 --- a/python/ray/dashboard/modules/aggregator/aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/aggregator_agent.py @@ -10,8 +10,7 @@ from urllib3.util import Retry from requests import Session from requests.adapters import HTTPAdapter - -from google.protobuf.json_format import MessageToJson +from ray._private.protobuf_compat import message_to_json try: import prometheus_client @@ -211,7 +210,7 @@ def _receive_events(self, request): Receives events from the request, adds them to the event buffer, """ if not self._event_processing_enabled: - return events_event_aggregator_service_pb2.AddEventReply() + return events_event_aggregator_service_pb2.AddEventsReply() # TODO(myan) #54515: Considering adding a mechanism to also send out the events # metadata (e.g. dropped task attempts) to help with event processing at the @@ -271,7 +270,10 @@ def _send_events_to_external_service(self, event_batch) -> None: # Convert protobuf objects to JSON dictionaries for HTTP POST filtered_event_batch_json = [ - json.loads(MessageToJson(event)) for event in filtered_event_batch + json.loads( + message_to_json(event, always_print_fields_with_no_presence=True) + ) + for event in filtered_event_batch ] try: diff --git a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py index e658849969bc..1853029f006f 100644 --- a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py @@ -26,6 +26,12 @@ TaskEventsMetadata, ) from ray.core.generated.events_base_event_pb2 import RayEvent +from ray.core.generated.events_task_definition_event_pb2 import ( + TaskDefinitionEvent, +) +from ray.core.generated.events_task_execution_event_pb2 import ( + TaskExecutionEvent, +) from ray.core.generated.profile_events_pb2 import ProfileEvents, ProfileEventEntry from ray.core.generated.events_task_profile_events_pb2 import TaskProfileEvents from ray.core.generated.events_driver_job_definition_event_pb2 import ( @@ -39,6 +45,15 @@ RuntimeEnvUris, RuntimeEnvConfig, ) +from ray.core.generated.common_pb2 import ( + TaskType, + Language, + FunctionDescriptor, + PythonFunctionDescriptor, + TaskStatus, + ErrorType, + RayErrorInfo, +) from ray.dashboard.modules.aggregator.aggregator_agent import AggregatorAgent @@ -124,6 +139,47 @@ def test_aggregator_agent_http_target_not_enabled( assert agent._event_processing_enabled == expected_event_processing_enabled +@pytest.mark.parametrize( + "ray_start_cluster_head_with_env_vars", + [ + { + "env_vars": { + "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": "", + }, + }, + ], + indirect=True, +) +def test_aggregator_agent_event_processing_disabled( + ray_start_cluster_head_with_env_vars, httpserver, fake_timestamp +): + cluster = ray_start_cluster_head_with_env_vars + stub = get_event_aggregator_grpc_stub( + cluster.gcs_address, cluster.head_node.node_id + ) + + httpserver.expect_request("/", method="POST").respond_with_data("", status=200) + + request = AddEventsRequest( + events_data=RayEventsData( + events=[ + RayEvent( + event_id=b"1", + source_type=RayEvent.SourceType.CORE_WORKER, + event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, + timestamp=fake_timestamp[0], + severity=RayEvent.Severity.INFO, + message="hello", + ), + ], + task_events_metadata=TaskEventsMetadata( + dropped_task_attempts=[], + ), + ) + ) + stub.AddEvents(request) + + @_with_aggregator_port def test_aggregator_agent_receive_publish_events_normally( ray_start_cluster_head_with_env_vars, httpserver, fake_timestamp @@ -394,45 +450,174 @@ def test_aggregator_agent_profile_events_not_exposed( assert req_json[0]["eventType"] == "TASK_DEFINITION_EVENT" -@pytest.mark.parametrize( - "ray_start_cluster_head_with_env_vars", - [ - { - "env_vars": { - "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": _EVENT_AGGREGATOR_AGENT_TARGET_ADDR, - "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "TASK_DEFINITION_EVENT,TASK_EXECUTION_EVENT,ACTOR_TASK_DEFINITION_EVENT,ACTOR_TASK_EXECUTION_EVENT,TASK_PROFILE_EVENT", +def _create_task_definition_event_proto(timestamp): + return RayEvent( + event_id=b"1", + source_type=RayEvent.SourceType.CORE_WORKER, + event_type=RayEvent.EventType.TASK_DEFINITION_EVENT, + timestamp=timestamp, + severity=RayEvent.Severity.INFO, + session_name="test_session", + task_definition_event=TaskDefinitionEvent( + task_id=b"1", + task_attempt=1, + task_type=TaskType.NORMAL_TASK, + language=Language.PYTHON, + task_func=FunctionDescriptor( + python_function_descriptor=PythonFunctionDescriptor( + module_name="test_module", + class_name="test_class", + function_name="test_function", + function_hash="test_hash", + ), + ), + task_name="test_task", + required_resources={ + "CPU": 1.0, + "GPU": 0.0, }, - }, - ], - indirect=True, -) -def test_aggregator_agent_receive_profile_events( - ray_start_cluster_head_with_env_vars, httpserver, fake_timestamp -): - cluster = ray_start_cluster_head_with_env_vars - stub = get_event_aggregator_grpc_stub( - cluster.gcs_address, cluster.head_node.node_id + runtime_env_info=RuntimeEnvInfo( + serialized_runtime_env="{}", + ), + job_id=b"1", + parent_task_id=b"1", + placement_group_id=b"1", + ref_ids={ + "key1": b"value1", + "key2": b"value2", + }, + ), ) - httpserver.expect_request("/", method="POST").respond_with_data("", status=200) - request = AddEventsRequest( - events_data=RayEventsData( - events=[_create_profile_event_request(fake_timestamp[0])], - task_events_metadata=TaskEventsMetadata( - dropped_task_attempts=[], +def _verify_task_definition_event_json(req_json, expected_timestamp): + assert len(req_json) == 1 + + # Verify the base event fields + assert req_json[0]["eventId"] == base64.b64encode(b"1").decode() + assert req_json[0]["sourceType"] == "CORE_WORKER" + assert req_json[0]["eventType"] == "TASK_DEFINITION_EVENT" + assert req_json[0]["timestamp"] == expected_timestamp + assert req_json[0]["severity"] == "INFO" + assert ( + req_json[0]["message"] == "" + ) # Make sure the default value is included when it is not set + assert req_json[0]["sessionName"] == "test_session" + + # Verify the task definition event specific fields + assert ( + req_json[0]["taskDefinitionEvent"]["taskId"] == base64.b64encode(b"1").decode() + ) + assert req_json[0]["taskDefinitionEvent"]["taskAttempt"] == 1 + assert req_json[0]["taskDefinitionEvent"]["taskType"] == "NORMAL_TASK" + assert req_json[0]["taskDefinitionEvent"]["language"] == "PYTHON" + assert ( + req_json[0]["taskDefinitionEvent"]["taskFunc"]["pythonFunctionDescriptor"][ + "moduleName" + ] + == "test_module" + ) + assert ( + req_json[0]["taskDefinitionEvent"]["taskFunc"]["pythonFunctionDescriptor"][ + "className" + ] + == "test_class" + ) + assert ( + req_json[0]["taskDefinitionEvent"]["taskFunc"]["pythonFunctionDescriptor"][ + "functionName" + ] + == "test_function" + ) + assert ( + req_json[0]["taskDefinitionEvent"]["taskFunc"]["pythonFunctionDescriptor"][ + "functionHash" + ] + == "test_hash" + ) + assert req_json[0]["taskDefinitionEvent"]["taskName"] == "test_task" + assert req_json[0]["taskDefinitionEvent"]["requiredResources"] == { + "CPU": 1.0, + "GPU": 0.0, + } + assert ( + req_json[0]["taskDefinitionEvent"]["runtimeEnvInfo"]["serializedRuntimeEnv"] + == "{}" + ) + assert ( + req_json[0]["taskDefinitionEvent"]["jobId"] == base64.b64encode(b"1").decode() + ) + assert ( + req_json[0]["taskDefinitionEvent"]["parentTaskId"] + == base64.b64encode(b"1").decode() + ) + assert ( + req_json[0]["taskDefinitionEvent"]["placementGroupId"] + == base64.b64encode(b"1").decode() + ) + assert req_json[0]["taskDefinitionEvent"]["refIds"] == { + "key1": base64.b64encode(b"value1").decode(), + "key2": base64.b64encode(b"value2").decode(), + } + + +def _create_task_execution_event_proto(timestamp): + return RayEvent( + event_id=b"1", + source_type=RayEvent.SourceType.CORE_WORKER, + event_type=RayEvent.EventType.TASK_EXECUTION_EVENT, + timestamp=timestamp, + severity=RayEvent.Severity.INFO, + session_name="test_session", + task_execution_event=TaskExecutionEvent( + task_id=b"1", + task_attempt=1, + task_state={ + TaskStatus.RUNNING: timestamp, + }, + ray_error_info=RayErrorInfo( + error_type=ErrorType.TASK_EXECUTION_EXCEPTION, ), - ) + node_id=b"1", + worker_id=b"1", + worker_pid=1, + ), ) - stub.AddEvents(request) - wait_for_condition(lambda: len(httpserver.log) == 1) +def _verify_task_execution_event_json(req_json, expected_timestamp): + assert len(req_json) == 1 - req, _ = httpserver.log[0] - req_json = json.loads(req.data) + # Verify the base event fields + assert req_json[0]["eventId"] == base64.b64encode(b"1").decode() + assert req_json[0]["sourceType"] == "CORE_WORKER" + assert req_json[0]["eventType"] == "TASK_EXECUTION_EVENT" + assert req_json[0]["timestamp"] == expected_timestamp + assert req_json[0]["severity"] == "INFO" + assert ( + req_json[0]["message"] == "" + ) # Make sure the default value is included when it is not set + assert req_json[0]["sessionName"] == "test_session" - _verify_profile_event_json(req_json, fake_timestamp[1]) + # Verify the task execution event specific fields + assert ( + req_json[0]["taskExecutionEvent"]["taskId"] == base64.b64encode(b"1").decode() + ) + assert req_json[0]["taskExecutionEvent"]["taskAttempt"] == 1 + assert req_json[0]["taskExecutionEvent"]["taskState"] == { + "8": expected_timestamp, + } + assert ( + req_json[0]["taskExecutionEvent"]["rayErrorInfo"]["errorType"] + == "TASK_EXECUTION_EXCEPTION" + ) + assert ( + req_json[0]["taskExecutionEvent"]["nodeId"] == base64.b64encode(b"1").decode() + ) + assert ( + req_json[0]["taskExecutionEvent"]["workerId"] == base64.b64encode(b"1").decode() + ) + assert req_json[0]["taskExecutionEvent"]["workerPid"] == 1 def _create_profile_event_request(timestamp): @@ -497,6 +682,65 @@ def _verify_profile_event_json(req_json, expected_timestamp): assert event_entry["extraData"] == '{"cpu_usage": 0.8}' +# tuple: (create_event, verify) +EVENT_TYPES_TO_TEST = [ + pytest.param( + _create_task_definition_event_proto, + _verify_task_definition_event_json, + id="task_definition_event", + ), + pytest.param( + _create_task_execution_event_proto, + _verify_task_execution_event_json, + id="task_execution_event", + ), + pytest.param( + _create_profile_event_request, _verify_profile_event_json, id="profile_event" + ), +] + + +@pytest.mark.parametrize("create_event, verify_event", EVENT_TYPES_TO_TEST) +@pytest.mark.parametrize( + "ray_start_cluster_head_with_env_vars", + [ + { + "env_vars": { + "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": _EVENT_AGGREGATOR_AGENT_TARGET_ADDR, + "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "TASK_DEFINITION_EVENT,TASK_EXECUTION_EVENT,ACTOR_TASK_DEFINITION_EVENT,ACTOR_TASK_EXECUTION_EVENT,TASK_PROFILE_EVENT", + }, + }, + ], + indirect=True, +) +def test_aggregator_agent_receive_events( + create_event, + verify_event, + ray_start_cluster_head_with_env_vars, + httpserver, + fake_timestamp, +): + cluster = ray_start_cluster_head_with_env_vars + stub = get_event_aggregator_grpc_stub( + cluster.gcs_address, cluster.head_node.node_id + ) + httpserver.expect_request("/", method="POST").respond_with_data("", status=200) + request = AddEventsRequest( + events_data=RayEventsData( + events=[create_event(fake_timestamp[0])], + task_events_metadata=TaskEventsMetadata( + dropped_task_attempts=[], + ), + ) + ) + + stub.AddEvents(request) + wait_for_condition(lambda: len(httpserver.log) == 1) + req, _ = httpserver.log[0] + req_json = json.loads(req.data) + verify_event(req_json, fake_timestamp[1]) + + @_with_aggregator_port def test_aggregator_agent_receive_driver_job_definition_event( ray_start_cluster_head_with_env_vars, httpserver From a98def4040f2b37375c6ac82f3afec6a54a5b888 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Wed, 20 Aug 2025 11:49:10 -0500 Subject: [PATCH 197/634] [core] Remove `core_worker_lib` dependency from `normal_task_submitter_test` (#55783) I must've missed this one earlier. --------- Signed-off-by: Edward Oakes --- src/ray/core_worker/task_submission/tests/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ray/core_worker/task_submission/tests/BUILD.bazel b/src/ray/core_worker/task_submission/tests/BUILD.bazel index 036e4a297df1..707a4d310589 100644 --- a/src/ray/core_worker/task_submission/tests/BUILD.bazel +++ b/src/ray/core_worker/task_submission/tests/BUILD.bazel @@ -67,8 +67,8 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:task_common", "//src/ray/common:test_util", - "//src/ray/core_worker:core_worker_lib", "//src/ray/core_worker:memory_store", + "//src/ray/core_worker/task_submission:normal_task_submitter", "//src/ray/raylet_client:raylet_client_lib", "//src/ray/rpc:core_worker_client", "@com_google_googletest//:gtest", From f92f9c14eef9e8a8cdbd02b138b840e0c6710449 Mon Sep 17 00:00:00 2001 From: Rueian Date: Wed, 20 Aug 2025 09:54:57 -0700 Subject: [PATCH 198/634] [core][autoscaler] let the event logger print the cluster resources with the sum of total resources on each node. (#55746) ## Why are these changes needed? Currently, the autoscaler event logger will print the number of current cluster resources by multiplying the resource_quantity in the node config and the current node count. This doesn't work if nodes have different sizes than their configs. This PR changes to print out the sum of total_resources in all current live nodes. There are no behavior changes in the INFO level logs for now. For DEBEG level logs, instead of printing `Current cluster shape`, now we print `Current cluster resources`. ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Rueian --- python/ray/autoscaler/v2/event_logger.py | 48 ++++++++++++------- python/ray/autoscaler/v2/scheduler.py | 23 ++++++++- .../autoscaler/v2/tests/test_event_logger.py | 19 +------- 3 files changed, 53 insertions(+), 37 deletions(-) diff --git a/python/ray/autoscaler/v2/event_logger.py b/python/ray/autoscaler/v2/event_logger.py index 316378867e27..b4db3d2b798b 100644 --- a/python/ray/autoscaler/v2/event_logger.py +++ b/python/ray/autoscaler/v2/event_logger.py @@ -3,8 +3,6 @@ from typing import Dict, List, Optional from ray._private.event.event_logger import EventLoggerAdapter -from ray.autoscaler.v2.instance_manager.config import NodeTypeConfig -from ray.autoscaler.v2.schema import NodeType from ray.autoscaler.v2.utils import ResourceRequestUtil from ray.core.generated.autoscaler_pb2 import ( ClusterResourceConstraint, @@ -31,8 +29,7 @@ def __init__(self, logger: EventLoggerAdapter): def log_cluster_scheduling_update( self, - node_type_configs: Dict[NodeType, NodeTypeConfig], - cluster_shape: Dict[NodeType, int], + cluster_resources: Dict[str, float], launch_requests: Optional[List[LaunchRequest]] = None, terminate_requests: Optional[List[TerminationRequest]] = None, infeasible_requests: Optional[List[ResourceRequest]] = None, @@ -42,7 +39,29 @@ def log_cluster_scheduling_update( ] = None, ) -> None: """ - Log any update of the cluster scheduling state. + Log updates to the autoscaler scheduling state. + + Emits: + - info logs for node launches and terminations (counts grouped by node type). + - an info log summarizing the cluster size after a resize (CPUs/GPUs/TPUs). + - warnings describing infeasible single resource requests, infeasible gang + (placement group) requests, and infeasible cluster resource constraints. + + Args: + cluster_resources: Mapping of resource name to total resources for the + current cluster state. + launch_requests: Node launch requests issued in this scheduling step. + terminate_requests: Node termination requests issued in this scheduling + step. + infeasible_requests: Resource requests that could not be satisfied by + any available node type. + infeasible_gang_requests: Gang/placement group requests that could not + be scheduled. + infeasible_cluster_resource_constraints: Cluster-level resource + constraints that could not be satisfied. + + Returns: + None """ # Log any launch events. @@ -78,23 +97,16 @@ def log_cluster_scheduling_update( # Cluster shape changes. if launch_requests or terminate_requests: - total_resources = defaultdict(float) - - for node_type, count in cluster_shape.items(): - node_config = node_type_configs[node_type] - for resource_name, resource_quantity in node_config.resources.items(): - total_resources[resource_name] += resource_quantity * count - - num_cpus = total_resources.get("CPU", 0) + num_cpus = cluster_resources.get("CPU", 0) log_str = f"Resized to {int(num_cpus)} CPUs" - if "GPU" in total_resources: - log_str += f", {int(total_resources['GPU'])} GPUs" - if "TPU" in total_resources: - log_str += f", {int(total_resources['TPU'])} TPUs" + if "GPU" in cluster_resources: + log_str += f", {int(cluster_resources['GPU'])} GPUs" + if "TPU" in cluster_resources: + log_str += f", {int(cluster_resources['TPU'])} TPUs" self._logger.info(f"{log_str}.") - self._logger.debug(f"Current cluster shape: {dict(cluster_shape)}.") + self._logger.debug(f"Current cluster resources: {dict(cluster_resources)}.") # Log any infeasible requests. if infeasible_requests: diff --git a/python/ray/autoscaler/v2/scheduler.py b/python/ray/autoscaler/v2/scheduler.py index 641baa2c81a9..9fafab0ca7a4 100644 --- a/python/ray/autoscaler/v2/scheduler.py +++ b/python/ray/autoscaler/v2/scheduler.py @@ -825,6 +825,26 @@ def get_cluster_shape(self) -> Dict[NodeType, int]: cluster_shape[node.node_type] += 1 return cluster_shape + def get_cluster_resources(self) -> Dict[str, float]: + """ + Aggregate total cluster resources. + + Sums each node's `total_resources` across the current context, + excluding nodes marked `TO_TERMINATE`. + + Returns: + A dict mapping resource names to their summed resources. + """ + cluster_resources = defaultdict(float) + for node in self._nodes: + if node.status == SchedulingNodeStatus.TO_TERMINATE: + # Skip the nodes that are to be terminated. + continue + + for key, value in node.total_resources.items(): + cluster_resources[key] += value + return cluster_resources + def get_idle_timeout_s(self) -> Optional[float]: return self._idle_timeout_s @@ -949,8 +969,7 @@ def schedule(self, request: SchedulingRequest) -> SchedulingReply: infeasible_requests=infeasible_requests, infeasible_gang_requests=infeasible_gang_requests, infeasible_cluster_resource_constraints=infeasible_constraints, - cluster_shape=ctx.get_cluster_shape(), - node_type_configs=ctx.get_node_type_configs(), + cluster_resources=ctx.get_cluster_resources(), ) except Exception: logger.exception("Failed to emit event logs.") diff --git a/python/ray/autoscaler/v2/tests/test_event_logger.py b/python/ray/autoscaler/v2/tests/test_event_logger.py index da127b1a2be0..1f7a339aa903 100644 --- a/python/ray/autoscaler/v2/tests/test_event_logger.py +++ b/python/ray/autoscaler/v2/tests/test_event_logger.py @@ -5,7 +5,6 @@ import pytest from ray.autoscaler.v2.event_logger import AutoscalerEventLogger -from ray.autoscaler.v2.instance_manager.config import NodeTypeConfig from ray.autoscaler.v2.tests.util import MockEventLogger from ray.autoscaler.v2.utils import ResourceRequestUtil from ray.core.generated.autoscaler_pb2 import ( @@ -83,21 +82,7 @@ def test_log_scheduling_updates(): ) ) ], - cluster_shape={"type-1": 1, "type-2": 2}, - node_type_configs={ - "type-1": NodeTypeConfig( - name="type-1", - max_worker_nodes=10, - min_worker_nodes=1, - resources={"CPU": 1, "GPU": 1}, - ), - "type-2": NodeTypeConfig( - name="type-2", - max_worker_nodes=10, - min_worker_nodes=1, - resources={"CPU": 2, "GPU": 2, "TPU": 1}, - ), - }, + cluster_resources={"CPU": 5, "GPU": 5, "TPU": 2}, ) assert mock_logger.get_logs("info") == [ @@ -117,7 +102,7 @@ def test_log_scheduling_updates(): assert mock_logger.get_logs("error") == [] assert mock_logger.get_logs("debug") == [ - "Current cluster shape: {'type-1': 1, 'type-2': 2}." + "Current cluster resources: {'CPU': 5, 'GPU': 5, 'TPU': 2}." ] From 98e1bc41254645c4602d81f65586d3c4c2f9b61b Mon Sep 17 00:00:00 2001 From: Anmol Singh Date: Wed, 20 Aug 2025 22:30:21 +0530 Subject: [PATCH 199/634] Default dashboard usability improvements (#55620) ## Why are these changes needed? Right now the Ray Default Grafana dashboard has usability issues that are relatively easy to address with some basic restructuring + improved naming. Doc that explains the current issues, proposed changes + has a bunch of screenshot previews: https://docs.google.com/document/d/1aZ4wXmiA2uhpiY4GBu4QPLjgAJunwnerld7UVxP5QMU/ This PR introduces all the required code changes for the proposed restructuring + panel renaming. ## New layout screenshots image image image image image image image image ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: anmol Co-authored-by: anmol --- .../dashboards/default_dashboard_panels.py | 463 ++++++++++-------- python/ray/tests/test_metrics_head.py | 15 +- 2 files changed, 270 insertions(+), 208 deletions(-) diff --git a/python/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py index 00cfc8194ac8..72d43ccbd8b5 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/default_dashboard_panels.py @@ -3,6 +3,7 @@ from ray.dashboard.modules.metrics.dashboards.common import ( DashboardConfig, Panel, + Row, Target, ) @@ -30,11 +31,86 @@ def max_plus_pending(max_resource, pending_resource): # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # IMPORTANT: Please keep this in sync with Metrics.tsx and ray-metrics.rst # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -DEFAULT_GRAFANA_PANELS = [ +OVERVIEW_AND_HEALTH_PANELS = [ + Panel( + id=24, + title="Node Count", + description='Note: not impacted by "Instance" variable.\n\nA total number of active failed, and pending nodes from the cluster. \n\nACTIVE: A node is alive and available.\n\nFAILED: A node is dead and not available. The node is considered dead when the raylet process on the node is terminated. The node will get into the failed state if it cannot be provided (e.g., there\'s no available node from the cloud provider) or failed to setup (e.g., setup_commands have errors). \n\nPending: A node is being started by the Ray cluster launcher. The node is unavailable now because it is being provisioned and initialized.', + unit="nodes", + targets=[ + Target( + expr="sum(autoscaler_active_nodes{{{global_filters}}}) by (NodeType)", + legend="Active Nodes: {{NodeType}}", + ), + Target( + expr="sum(autoscaler_recently_failed_nodes{{{global_filters}}}) by (NodeType)", + legend="Failed Nodes: {{NodeType}}", + ), + Target( + expr="sum(autoscaler_pending_nodes{{{global_filters}}}) by (NodeType)", + legend="Pending Nodes: {{NodeType}}", + ), + ], + ), + Panel( + id=41, + title="Cluster Utilization", + description="Aggregated utilization of all physical resources (CPU, GPU, memory, disk, or etc.) across the cluster.", + unit="%", + targets=[ + # CPU + Target( + expr='avg(ray_node_cpu_utilization{{instance=~"$Instance",{global_filters}}})', + legend="CPU (physical)", + ), + # GPU + Target( + expr='sum(ray_node_gpus_utilization{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_gpus_available{{instance=~"$Instance",{global_filters}}}) or vector(0))', + legend="GPU (physical)", + ), + # Memory + Target( + expr='sum(ray_node_mem_used{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_mem_total{{instance=~"$Instance",{global_filters}}})) * 100', + legend="Memory (RAM)", + ), + # GRAM + Target( + expr='sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 100', + legend="GRAM", + ), + # Object Store + Target( + expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) / on() sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}}) * 100', + legend="Object Store Memory", + ), + # Disk + Target( + expr='sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_disk_free{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}})) * 100', + legend="Disk", + ), + ], + fill=0, + stack=False, + ), + Panel( + id=44, + title="Ray OOM Kills (Tasks and Actors)", + description="The number of tasks and actors killed by the Ray Out of Memory killer due to high memory pressure. Metrics are broken down by IP and the name. https://docs.ray.io/en/master/ray-core/scheduling/ray-oom-prevention.html.", + unit="failures", + targets=[ + Target( + expr='sum(ray_memory_manager_worker_eviction_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (Name, instance, RayNodeType)', + legend="OOM Killed: {{Name}}, {{instance}} ({{RayNodeType}})", + ), + ], + ), +] + +RAY_TASKS_ACTORS_PLACEMENT_GROUPS_PANELS = [ Panel( id=26, - title="Scheduler Task State", - description="Current number of tasks in a particular state.\n\nState: the task state, as described by rpc::TaskState proto in common.proto. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.", + title="All Tasks by State", + description="Current count of tasks, grouped by scheduler state (e.g., pending, running, finished).\n\nState: the task state, as described by rpc::TaskStatus proto in common.proto. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.", unit="tasks", targets=[ Target( @@ -51,8 +127,8 @@ def max_plus_pending(max_resource, pending_resource): ), Panel( id=35, - title="Requested Live Tasks by Name", - description="Current number of (live) tasks with a particular name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.", + title="Active Tasks by Name", + description="Current count of active tasks (i.e. pending or running; not finished), grouped by task name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.", unit="tasks", targets=[ Target( @@ -70,7 +146,7 @@ def max_plus_pending(max_resource, pending_resource): Panel( id=38, title="Running Tasks by Name", - description="Current number of (running) tasks with a particular name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.", + description="Current count of tasks that are currently executing, grouped by task name. Task resubmissions due to failures or object reconstruction are shown with (retry) in the label.", unit="tasks", targets=[ Target( @@ -87,8 +163,8 @@ def max_plus_pending(max_resource, pending_resource): ), Panel( id=33, - title="Scheduler Actor State", - description='Note: not impacted by "Instance" variable.\n\nCurrent number of actors in a particular state.\n\nState: the actor state, as described by rpc::ActorTableData proto in gcs.proto.', + title="All Actors by State", + description='Note: not impacted by "Instance" variable.\n\nCurrent count of actors, grouped by lifecycle state (e.g., alive, restarting, dead/terminated).\n\nState: the actor state, as described by rpc::ActorTableData proto in gcs.proto.', unit="actors", targets=[ Target( @@ -99,8 +175,8 @@ def max_plus_pending(max_resource, pending_resource): ), Panel( id=42, - title="Live Actor State", - description="Current number of alive actors in a particular state.\n\nState: IDLE, RUNNING_TASK, RUNNING_IN_RAY_GET, RUNNING_IN_RAY_WAIT", + title="Active Actors by State", + description="Current count of alive actors (i.e. not dead/terminated), grouped by state.\n\nState: the actor state, as described by rpc::ActorTableData proto in gcs.proto.", unit="actors", targets=[ Target( @@ -111,8 +187,8 @@ def max_plus_pending(max_resource, pending_resource): ), Panel( id=36, - title="Live Actors by Name", - description="Current number of alive actors with a particular name.", + title="Active Actors by Name", + description="Current count of alive actors, grouped by actor name.", unit="actors", targets=[ Target( @@ -121,9 +197,24 @@ def max_plus_pending(max_resource, pending_resource): ) ], ), + Panel( + id=40, + title="All Placement Groups by State", + description='Note: not impacted by "Instance" variable.\n\nCurrent count of placement groups, grouped by state.\n\nState: the placement group state, as described by the rpc::PlacementGroupTableData proto in gcs.proto.', + unit="placement groups", + targets=[ + Target( + expr="sum(ray_placement_groups{{{global_filters}}}) by (State)", + legend="{{State}}", + ) + ], + ), +] + +RAY_RESOURCES_PANELS = [ Panel( id=27, - title="Scheduler CPUs (logical slots)", + title="Logical CPUs used", description="Logical CPU usage of Ray. The dotted line indicates the total number of CPUs. The logical CPU is allocated by `num_cpus` arguments from tasks and actors. PENDING means the number of CPUs that will be available when new nodes are up after the autoscaler scales up.\n\nNOTE: Ray's logical CPU is different from physical CPU usage. Ray's logical CPU is allocated by `num_cpus` arguments.", unit="cores", targets=[ @@ -143,25 +234,9 @@ def max_plus_pending(max_resource, pending_resource): ), ], ), - Panel( - id=29, - title="Object Store Memory", - description="Object store memory usage by location. The dotted line indicates the object store memory capacity.\n\nLocation: where the memory was allocated, which is MMAP_SHM or MMAP_DISK to indicate memory-mapped page, SPILLED to indicate spillage to disk, and WORKER_HEAP for objects small enough to be inlined in worker memory. Refer to metric_defs.cc for more information.", - unit="bytes", - targets=[ - Target( - expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (Location)', - legend="{{Location}}", - ), - Target( - expr='sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}})', - legend="MAX", - ), - ], - ), Panel( id=28, - title="Scheduler GPUs (logical slots)", + title="Logical GPUs used", description="Logical GPU usage of Ray. The dotted line indicates the total number of GPUs. The logical GPU is allocated by `num_gpus` arguments from tasks and actors. PENDING means the number of GPUs that will be available when new nodes are up after the autoscaler scales up.", unit="GPUs", targets=[ @@ -182,78 +257,121 @@ def max_plus_pending(max_resource, pending_resource): ], ), Panel( - id=40, - title="Scheduler Placement Groups", - description='Note: not impacted by "Instance" variable.\n\nCurrent number of placement groups in a particular state.\n\nState: the placement group state, as described by the rpc::PlacementGroupTable proto in gcs.proto.', - unit="placement groups", + id=29, + title="Object Store Memory", + description="Object store memory usage by location. The dotted line indicates the object store memory capacity.\n\nLocation: where the memory was allocated, which is MMAP_SHM or MMAP_DISK to indicate memory-mapped page, SPILLED to indicate spillage to disk, and WORKER_HEAP for objects small enough to be inlined in worker memory. Refer to metric_defs.cc for more information.", + unit="bytes", targets=[ Target( - expr="sum(ray_placement_groups{{{global_filters}}}) by (State)", - legend="{{State}}", - ) + expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) by (Location)', + legend="{{Location}}", + ), + Target( + expr='sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}})', + legend="MAX", + ), ], ), +] + +NODE_HARDWARE_UTILIZATION_BY_RAY_COMPONENT_PANELS = [ Panel( - id=2, - title="Node CPU (hardware utilization)", - description="", + id=37, + title="Node CPU by Component", + description="The physical (hardware) CPU usage across the cluster, broken down by component. This reports the summed CPU usage per Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.", unit="cores", targets=[ Target( - expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType",{global_filters}}} / 100) by (instance, RayNodeType)', - legend="CPU Usage: {{instance}} ({{RayNodeType}})", + # ray_component_cpu_percentage returns a percentage that can be > 100. It means that it uses more than 1 CPU. + expr='sum(ray_component_cpu_percentage{{instance=~"$Instance",{global_filters}}}) by (Component) / 100', + legend="{{Component}}", ), Target( - expr='sum(ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', + expr='sum(ray_node_cpu_count{{instance=~"$Instance",{global_filters}}})', legend="MAX", ), ], ), Panel( - id=8, - title="Node GPU (hardware utilization)", - description="Node's physical (hardware) GPU usage. The dotted line means the total number of hardware GPUs from the cluster. ", - unit="GPUs", + id=34, + title="Node Memory by Component", + description="The physical (hardware) memory usage across the cluster, broken down by component. This reports the summed RSS-SHM per Ray component, which corresponds to an approximate memory usage per proc. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.", + unit="bytes", targets=[ Target( - expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} / 100) by (instance, RayNodeType, GpuIndex, GpuDeviceName)', - legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}", + expr='(sum(ray_component_rss_mb{{instance=~"$Instance",{global_filters}}} * 1024 * 1024) by (Component)) - (sum(ray_component_mem_shared_bytes{{instance=~"$Instance",{global_filters}}}) by (Component))', + legend="{{Component}}", ), Target( - expr='sum(ray_node_gpus_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', + expr='sum(ray_node_mem_shared_bytes{{instance=~"$Instance",{global_filters}}})', + legend="shared_memory", + ), + Target( + expr='sum(ray_node_mem_total{{instance=~"$Instance",{global_filters}}})', legend="MAX", ), ], ), Panel( - id=6, - title="Node Disk", - description="Node's physical (hardware) disk usage. The dotted line means the total amount of disk space from the cluster.\n\nNOTE: When Ray is deployed within a container, this shows the disk usage from the host machine. ", + id=45, + title="Node GPU by Component", + description="The physical (hardware) GPU usage across the cluster, broken down by component. This reports the summed GPU usage per Ray component.", + unit="GPUs", + targets=[ + Target( + expr="sum(ray_component_gpu_percentage{{{global_filters}}} / 100) by (Component)", + legend="{{Component}}", + ), + ], + ), + Panel( + id=46, + title="Node GPU Memory by Component", + description="The physical (hardware) GPU memory usage across the cluster, broken down by component. This reports the summed GPU memory usage per Ray component.", unit="bytes", targets=[ Target( - expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', - legend="Disk Used: {{instance}} ({{RayNodeType}})", + expr="sum(ray_component_gpu_memory_mb{{{global_filters}}} * 1024 * 1024) by (Component)", + legend="{{Component}}", ), Target( - expr='sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', + expr='(sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 1024 * 1024', legend="MAX", ), ], ), +] + +NODE_HARDWARE_UTILIZATION_PANELS = [ Panel( - id=32, - title="Node Disk IO Speed", - description="Disk IO per node.", - unit="Bps", + id=2, + title="Node CPU utilization", + description="", + unit="cores", targets=[ Target( - expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', - legend="Write: {{instance}} ({{RayNodeType}})", + expr='sum(ray_node_cpu_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType",{global_filters}}} / 100) by (instance, RayNodeType)', + legend="CPU Usage: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', - legend="Read: {{instance}} ({{RayNodeType}})", + expr='sum(ray_node_cpu_count{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', + legend="MAX", + ), + ], + ), + Panel( + id=8, + title="Node GPU utilization", + description="Node's physical (hardware) GPU usage. The dotted line means the total number of hardware GPUs from the cluster. ", + unit="GPUs", + targets=[ + Target( + expr='sum(ray_node_gpus_utilization{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} / 100) by (instance, RayNodeType, GpuIndex, GpuDeviceName)', + legend="GPU Usage: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}", + ), + Target( + expr='sum(ray_node_gpus_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', + legend="MAX", ), ], ), @@ -275,7 +393,7 @@ def max_plus_pending(max_resource, pending_resource): ), Panel( id=48, - title="Node Memory Percentage (heap + object store)", + title="Node Memory % (heap + object store)", description="The percentage of physical (hardware) memory usage for each node.", unit="%", targets=[ @@ -288,73 +406,75 @@ def max_plus_pending(max_resource, pending_resource): stack=False, ), Panel( - id=44, - title="Node Out of Memory Failures by Name", - description="The number of tasks and actors killed by the Ray Out of Memory killer due to high memory pressure. Metrics are broken down by IP and the name. https://docs.ray.io/en/master/ray-core/scheduling/ray-oom-prevention.html.", - unit="failures", + id=18, + title="Node GPU Memory (GRAM)", + description="The physical (hardware) GPU memory usage for each node. The dotted line means the total amount of GPU memory from the cluster.", + unit="bytes", targets=[ Target( - expr='sum(ray_memory_manager_worker_eviction_total{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (Name, instance, RayNodeType)', - legend="OOM Killed: {{Name}}, {{instance}} ({{RayNodeType}})", + expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * 1024 * 1024) by (instance, RayNodeType, GpuIndex, GpuDeviceName)', + legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}", + ), + Target( + expr='(sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})) * 1024 * 1024', + legend="MAX", ), ], ), Panel( - id=34, - title="Node Memory by Component", - description="The physical (hardware) memory usage across the cluster, broken down by component. This reports the summed RSS-SHM per Ray component, which corresponds to an approximate memory usage per proc. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.", + id=6, + title="Node Disk", + description="Node's physical (hardware) disk usage. The dotted line means the total amount of disk space from the cluster.\n\nNOTE: When Ray is deployed within a container, this shows the disk usage from the host machine. ", unit="bytes", targets=[ Target( - expr='(sum(ray_component_rss_mb{{instance=~"$Instance",{global_filters}}} * 1e6) by (Component)) - (sum(ray_component_mem_shared_bytes{{instance=~"$Instance",{global_filters}}}) by (Component))', - legend="{{Component}}", - ), - Target( - expr='sum(ray_node_mem_shared_bytes{{instance=~"$Instance",{global_filters}}})', - legend="shared_memory", + expr='sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Disk Used: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_mem_total{{instance=~"$Instance",{global_filters}}})', + expr='sum(ray_node_disk_free{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})', legend="MAX", ), ], ), Panel( - id=37, - title="Node CPU by Component", - description="The physical (hardware) CPU usage across the cluster, broken down by component. This reports the summed CPU usage per Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the process (that contains method names) names of running tasks/actors.", - unit="cores", + id=32, + title="Node Disk IO Speed", + description="Disk IO per node.", + unit="Bps", targets=[ Target( - # ray_component_cpu_percentage returns a percentage that can be > 100. It means that it uses more than 1 CPU. - expr='sum(ray_component_cpu_percentage{{instance=~"$Instance",{global_filters}}}) by (Component) / 100', - legend="{{Component}}", + expr='sum(ray_node_disk_io_write_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Write: {{instance}} ({{RayNodeType}})", ), Target( - expr='sum(ray_node_cpu_count{{instance=~"$Instance",{global_filters}}})', - legend="MAX", + expr='sum(ray_node_disk_io_read_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Read: {{instance}} ({{RayNodeType}})", ), ], ), Panel( - id=18, - title="Node GPU Memory (GRAM)", - description="The physical (hardware) GPU memory usage for each node. The dotted line means the total amount of GPU memory from the cluster.", - unit="bytes", + id=20, + title="Node Network", + description="Network speed per node", + unit="Bps", targets=[ Target( - expr='sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}} * 1024 * 1024) by (instance, RayNodeType, GpuIndex, GpuDeviceName)', - legend="Used GRAM: {{instance}} ({{RayNodeType}}), gpu.{{GpuIndex}}, {{GpuDeviceName}}", + expr='sum(ray_node_network_receive_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Recv: {{instance}} ({{RayNodeType}})", ), Target( - expr='(sum(ray_node_gram_available{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}})) * 1024 * 1024', - legend="MAX", + expr='sum(ray_node_network_send_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', + legend="Send: {{instance}} ({{RayNodeType}})", ), ], ), +] + +NODE_TPU_UTILIZATION_PANELS = [ Panel( id=50, - title="Node TPU Tensorcore Utilization (Percentage)", + title="Node TPU Tensorcore Utilization %", description="Percentage of tensorcore utilization for the TPUs on this node. Computed by dividing the number of tensorcore operations by the maximum supported number of operations during the sample period.", unit="%", targets=[ @@ -366,7 +486,7 @@ def max_plus_pending(max_resource, pending_resource): ), Panel( id=51, - title="Node TPU High Bandwidth Memory Utilization (Percentage)", + title="Node TPU High Bandwidth Memory Utilization %", description="Percentage of bandwidth memory utilization for the TPUs on this node. Computed by dividing the memory bandwidth used by the maximum supported memory bandwidth limit during the sample period.", unit="%", targets=[ @@ -378,7 +498,7 @@ def max_plus_pending(max_resource, pending_resource): ), Panel( id=52, - title="Node TPU Duty Cycle (Percentage)", + title="Node TPU Duty Cycle %", description="Percentage of time over the sample period during which the TPU is actively processing.", unit="%", targets=[ @@ -404,116 +524,51 @@ def max_plus_pending(max_resource, pending_resource): ), ], ), - Panel( - id=20, - title="Node Network", - description="Network speed per node", - unit="Bps", - targets=[ - Target( - expr='sum(ray_node_network_receive_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', - legend="Recv: {{instance}} ({{RayNodeType}})", - ), - Target( - expr='sum(ray_node_network_send_speed{{instance=~"$Instance", RayNodeType=~"$RayNodeType", {global_filters}}}) by (instance, RayNodeType)', - legend="Send: {{instance}} ({{RayNodeType}})", - ), - ], +] + +DEFAULT_GRAFANA_ROWS = [ + Row( + title="Overview and Health", + id=1001, + panels=OVERVIEW_AND_HEALTH_PANELS, + collapsed=False, ), - Panel( - id=24, - title="Node Count", - description='Note: not impacted by "Instance" variable.\n\nA total number of active failed, and pending nodes from the cluster. \n\nACTIVE: A node is alive and available.\n\nFAILED: A node is dead and not available. The node is considered dead when the raylet process on the node is terminated. The node will get into the failed state if it cannot be provided (e.g., there\'s no available node from the cloud provider) or failed to setup (e.g., setup_commands have errors). \n\nPending: A node is being started by the Ray cluster launcher. The node is unavailable now because it is being provisioned and initialized.', - unit="nodes", - targets=[ - Target( - expr="sum(autoscaler_active_nodes{{{global_filters}}}) by (NodeType)", - legend="Active Nodes: {{NodeType}}", - ), - Target( - expr="sum(autoscaler_recently_failed_nodes{{{global_filters}}}) by (NodeType)", - legend="Failed Nodes: {{NodeType}}", - ), - Target( - expr="sum(autoscaler_pending_nodes{{{global_filters}}}) by (NodeType)", - legend="Pending Nodes: {{NodeType}}", - ), - ], + Row( + title="Ray Tasks, Actors and Placement Groups", + id=1002, + panels=RAY_TASKS_ACTORS_PLACEMENT_GROUPS_PANELS, + collapsed=False, ), - Panel( - id=41, - title="Cluster Utilization", - description="Aggregated utilization of all physical resources (CPU, GPU, memory, disk, or etc.) across the cluster.", - unit="%", - targets=[ - # CPU - Target( - expr='avg(ray_node_cpu_utilization{{instance=~"$Instance",{global_filters}}})', - legend="CPU (physical)", - ), - # GPU - Target( - expr='sum(ray_node_gpus_utilization{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_gpus_available{{instance=~"$Instance",{global_filters}}}) or vector(0))', - legend="GPU (physical)", - ), - # Memory - Target( - expr='sum(ray_node_mem_used{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_mem_total{{instance=~"$Instance",{global_filters}}})) * 100', - legend="Memory (RAM)", - ), - # GRAM - Target( - expr='sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}})) * 100', - legend="GRAM", - ), - # Object Store - Target( - expr='sum(ray_object_store_memory{{instance=~"$Instance",{global_filters}}}) / on() sum(ray_resources{{Name="object_store_memory",instance=~"$Instance",{global_filters}}}) * 100', - legend="Object Store Memory", - ), - # Disk - Target( - expr='sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}}) / on() (sum(ray_node_disk_free{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_disk_usage{{instance=~"$Instance",{global_filters}}})) * 100', - legend="Disk", - ), - ], - fill=0, - stack=False, + Row( + title="Ray Resources", + id=1003, + panels=RAY_RESOURCES_PANELS, + collapsed=False, ), - Panel( - id=45, - title="Node GPU by Component", - description="The physical (hardware) GPU usage across the cluster, broken down by component. This reports the summed GPU usage per Ray component.", - unit="GPUs", - targets=[ - Target( - expr="sum(ray_component_gpu_percentage{{{global_filters}}} / 100) by (Component)", - legend="{{Component}}", - ), - ], + Row( + title="Hardware Utilization by Ray Component", + id=1004, + panels=NODE_HARDWARE_UTILIZATION_BY_RAY_COMPONENT_PANELS, + collapsed=False, ), - Panel( - id=46, - title="Node GPU Memory by Component", - description="The physical (hardware) GPU memory usage across the cluster, broken down by component. This reports the summed GPU memory usage per Ray component.", - unit="bytes", - targets=[ - Target( - expr="sum(ray_component_gpu_memory_mb{{{global_filters}}}) by (Component)", - legend="{{Component}}", - ), - Target( - expr='(sum(ray_node_gram_available{{instance=~"$Instance",{global_filters}}}) + sum(ray_node_gram_used{{instance=~"$Instance",{global_filters}}}))*1024*1024', - legend="MAX", - ), - ], + Row( + title="Hardware Utilization by Node", + id=1005, + panels=NODE_HARDWARE_UTILIZATION_PANELS, + collapsed=False, + ), + Row( + title="TPU Utilization by Node", + id=1006, + panels=NODE_TPU_UTILIZATION_PANELS, + collapsed=True, ), ] - ids = [] -for panel in DEFAULT_GRAFANA_PANELS: - ids.append(panel.id) +for row in DEFAULT_GRAFANA_ROWS: + ids.append(row.id) + ids.extend(panel.id for panel in row.panels) assert len(ids) == len( set(ids) ), f"Duplicated id found. Use unique id for each panel. {ids}" @@ -521,7 +576,7 @@ def max_plus_pending(max_resource, pending_resource): default_dashboard_config = DashboardConfig( name="DEFAULT", default_uid="rayDefaultDashboard", - panels=DEFAULT_GRAFANA_PANELS, + rows=DEFAULT_GRAFANA_ROWS, standard_global_filters=[ 'SessionName=~"$SessionName"', 'ray_io_cluster=~"$Cluster"', diff --git a/python/ray/tests/test_metrics_head.py b/python/ray/tests/test_metrics_head.py index 6fdcb3306404..0a7f6133ab99 100644 --- a/python/ray/tests/test_metrics_head.py +++ b/python/ray/tests/test_metrics_head.py @@ -7,7 +7,7 @@ import tempfile from ray.dashboard.modules.metrics.dashboards.default_dashboard_panels import ( - DEFAULT_GRAFANA_PANELS, + DEFAULT_GRAFANA_ROWS, ) from ray.dashboard.modules.metrics.dashboards.serve_dashboard_panels import ( SERVE_GRAFANA_PANELS, @@ -132,6 +132,9 @@ def test_metrics_folder_with_dashboard_override( contents = json.loads(f.read()) assert contents["uid"] == uid for panel in contents["panels"]: + if panel["type"] == "row": + # Row panels don't have targets + continue for target in panel["targets"]: # Check for standard_global_filters assert 'SessionName=~"$SessionName"' in target["expr"] @@ -154,6 +157,9 @@ def test_metrics_folder_with_dashboard_override( found_max = False found_max_pending = False for panel in contents["panels"]: + if panel["type"] == "row": + # Row panels don't have series overrides + continue for override in panel.get("seriesOverrides", []): if override.get("alias") == "MAX": assert override["fill"] == 0 @@ -210,9 +216,10 @@ def test_metrics_folder_when_dashboard_disabled(): def test_default_dashboard_utilizes_global_filters(): - for panel in DEFAULT_GRAFANA_PANELS: - for target in panel.targets: - assert "{global_filters}" in target.expr + for row in DEFAULT_GRAFANA_ROWS: + for panel in row.panels: + for target in panel.targets: + assert "{global_filters}" in target.expr def test_serve_dashboard_utilizes_global_filters(): From ffa18133f434c46fc2dde3a949e041cfb1e83e5a Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Wed, 20 Aug 2025 11:18:57 -0700 Subject: [PATCH 200/634] [Data] Make `ray.get` patch less brittle in `test_sort` (#55768) ## Why are these changes needed? `test_push_based_shuffle_reduce_stage_scheduling` patches `ray.get` with a function that takes a single `refs` argument. This is problematic because if Ray Data calls `ray.get` with any other arguments (e.g., `timeout`), then Python errors and complains that argument is unexpected. > TypeError: ray_get_override() got an unexpected keyword argument 'timeout' To address this issue, this PR allows the patched `ray.get` to accept additional arguments. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Balaji Veeramani --- python/ray/data/tests/test_sort.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/data/tests/test_sort.py b/python/ray/data/tests/test_sort.py index c97207e01f4b..7467b86c9f02 100644 --- a/python/ray/data/tests/test_sort.py +++ b/python/ray/data/tests/test_sort.py @@ -558,9 +558,9 @@ def options(**task_options): def patch_ray_get(callback): original_ray_get = ray.get - def ray_get_override(object_refs): + def ray_get_override(object_refs, *args, **kwargs): callback(object_refs) - return original_ray_get(object_refs) + return original_ray_get(object_refs, *args, **kwargs) ray.get = ray_get_override return original_ray_get From 67f6b46a95d3d9acdb0dd90aa1ef2e5a33ad74a4 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Wed, 20 Aug 2025 11:33:33 -0700 Subject: [PATCH 201/634] [core] Kill Retrying to get node with node ID log (#55785) Signed-off-by: dayshah --- src/ray/gcs/gcs_client/global_state_accessor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ray/gcs/gcs_client/global_state_accessor.cc b/src/ray/gcs/gcs_client/global_state_accessor.cc index f52e24f05967..64d55d5876c1 100644 --- a/src/ray/gcs/gcs_client/global_state_accessor.cc +++ b/src/ray/gcs/gcs_client/global_state_accessor.cc @@ -429,7 +429,7 @@ ray::Status GlobalStateAccessor::GetNode(const std::string &node_id_hex_str, ". The node registration may not be complete yet before the timeout." + " Try increase the RAY_raylet_start_wait_time_s config."); } - RAY_LOG(INFO) << "Retrying to get node with node ID " << node_id_hex_str; + RAY_LOG(DEBUG) << "Retrying to get node with node ID " << node_id_hex_str; // Some of the information may not be in GCS yet, so wait a little bit. std::this_thread::sleep_for(std::chrono::seconds(1)); } From 2807c0e4c6542434b98b8026c9974aa68b210dd7 Mon Sep 17 00:00:00 2001 From: Tianyi Date: Thu, 21 Aug 2025 02:56:31 +0800 Subject: [PATCH 202/634] [core] enable -Wshadow for all c++ targets (#53194) Signed-off-by: tianyi-ge --- bazel/ray.bzl | 1 + bazel/ray_deps_setup.bzl | 6 + cpp/include/ray/api/actor_creator.h | 4 +- cpp/include/ray/api/actor_task_caller.h | 4 +- cpp/include/ray/api/ray_runtime.h | 22 +- cpp/include/ray/api/task_caller.h | 4 +- cpp/src/ray/runtime/abstract_ray_runtime.cc | 12 +- .../runtime/task/local_mode_task_submitter.cc | 2 +- .../ray/runtime/task/native_task_submitter.cc | 20 +- cpp/src/ray/test/cluster/cluster_mode_test.cc | 16 +- cpp/src/ray/test/cluster/counter.cc | 6 +- src/ray/common/BUILD.bazel | 1 + src/ray/common/memory_monitor.cc | 6 +- src/ray/common/ray_syncer/ray_syncer.cc | 24 +- .../scheduling/resource_instance_set.cc | 4 +- src/ray/common/scheduling/scheduling_ids.h | 10 +- src/ray/common/status.h | 9 +- src/ray/common/task/task_spec.h | 26 +-- src/ray/common/task/task_util.h | 18 +- src/ray/core_worker/common.h | 40 ++-- src/ray/core_worker/core_worker.cc | 33 +-- src/ray/core_worker/core_worker_process.cc | 6 +- .../experimental_mutable_object_provider.cc | 4 +- .../java/io_ray_runtime_RayNativeRuntime.cc | 120 +++++----- .../io_ray_runtime_gcs_GlobalStateAccessor.cc | 16 +- ...io_ray_runtime_object_NativeObjectStore.cc | 40 ++-- ...io_ray_runtime_task_NativeTaskSubmitter.cc | 58 ++--- .../core_worker/object_recovery_manager.cc | 4 +- src/ray/core_worker/reference_count.cc | 138 ++++++------ src/ray/core_worker/reference_count.h | 36 +-- .../task_execution/actor_scheduling_queue.cc | 20 +- .../concurrency_group_manager.cc | 6 +- .../out_of_order_actor_scheduling_queue.cc | 13 +- .../task_execution/task_receiver.cc | 48 ++-- src/ray/core_worker/task_manager.cc | 148 ++++++------ src/ray/core_worker/task_manager.h | 56 ++--- .../task_submission/actor_task_submitter.cc | 210 +++++++++--------- .../task_submission/actor_task_submitter.h | 70 +++--- .../task_submission/dependency_resolver.cc | 4 +- .../task_submission/dependency_resolver.h | 4 +- .../task_submission/normal_task_submitter.cc | 25 ++- .../out_of_order_actor_submit_queue.cc | 3 +- .../out_of_order_actor_submit_queue.h | 3 +- .../sequential_actor_submit_queue.cc | 3 +- .../sequential_actor_submit_queue.h | 5 +- .../tests/actor_task_submitter_test.cc | 4 +- .../out_of_order_actor_submit_queue_test.cc | 2 +- .../tests/task_event_buffer_test.cc | 16 +- src/ray/gcs/gcs_client/accessor.cc | 37 +-- src/ray/gcs/gcs_server/gcs_actor_manager.cc | 43 ++-- src/ray/gcs/gcs_server/gcs_actor_manager.h | 6 +- .../gcs_server/gcs_health_check_manager.cc | 17 +- src/ray/gcs/gcs_server/gcs_job_manager.cc | 18 +- .../gcs/gcs_server/gcs_placement_group_mgr.cc | 33 +-- src/ray/gcs/gcs_server/gcs_server.cc | 4 +- src/ray/gcs/gcs_server/gcs_table_storage.cc | 8 +- .../tests/gcs_actor_manager_test.cc | 9 +- .../gcs_server/tests/gcs_server_test_util.h | 4 +- .../gcs/store_client/redis_store_client.cc | 14 +- src/ray/gcs/store_client/redis_store_client.h | 4 +- src/ray/ipc/client_connection.cc | 4 +- src/ray/object_manager/object_buffer_pool.cc | 47 ++-- src/ray/object_manager/object_buffer_pool.h | 38 ++-- src/ray/object_manager/object_manager.cc | 3 +- .../ownership_object_directory.cc | 12 +- src/ray/object_manager/plasma/common.h | 90 ++++---- .../plasma/create_request_queue.cc | 39 ++-- .../plasma/create_request_queue.h | 24 +- .../plasma/get_request_queue.cc | 40 ++-- .../object_manager/plasma/get_request_queue.h | 10 +- .../plasma/obj_lifecycle_mgr.cc | 26 +-- src/ray/object_manager/plasma/object_store.cc | 18 +- src/ray/object_manager/plasma/plasma.cc | 2 +- .../object_manager/plasma/plasma_allocator.cc | 12 +- .../object_manager/plasma/stats_collector.cc | 8 +- src/ray/object_manager/plasma/store.cc | 29 +-- .../plasma/tests/eviction_policy_test.cc | 16 +- .../plasma/tests/fallback_allocator_test.cc | 8 +- .../plasma/tests/obj_lifecycle_mgr_test.cc | 12 +- .../plasma/tests/object_store_test.cc | 44 ++-- .../plasma/tests/stats_collector_test.cc | 39 ++-- src/ray/object_manager/pull_manager.cc | 16 +- src/ray/object_manager/pull_manager.h | 26 +-- .../tests/get_request_queue_test.cc | 14 +- src/ray/pubsub/publisher.cc | 10 +- src/ray/pubsub/publisher.h | 12 +- src/ray/pubsub/subscriber.cc | 6 +- src/ray/raylet/dependency_manager.cc | 24 +- src/ray/raylet/dependency_manager.h | 38 ++-- src/ray/raylet/local_object_manager.cc | 28 +-- src/ray/raylet/local_object_manager.h | 16 +- src/ray/raylet/local_task_manager.cc | 117 +++++----- src/ray/raylet/main.cc | 16 +- src/ray/raylet/node_manager.cc | 40 ++-- src/ray/raylet/raylet.cc | 4 +- src/ray/raylet/runtime_env_agent_client.cc | 7 +- .../raylet/scheduling/cluster_task_manager.cc | 38 ++-- src/ray/raylet/scheduling/internal.h | 26 +-- .../affinity_with_bundle_scheduling_policy.cc | 6 +- .../policy/bundle_scheduling_policy.cc | 22 +- .../policy/composite_scheduling_policy.cc | 8 +- .../policy/composite_scheduling_policy.h | 4 +- .../policy/hybrid_scheduling_policy.cc | 38 ++-- .../policy/node_affinity_scheduling_policy.cc | 12 +- .../policy/node_label_scheduling_policy.cc | 4 +- .../policy/random_scheduling_policy.cc | 6 +- .../scheduling/policy/scheduling_options.h | 62 +++--- .../policy/spread_scheduling_policy.cc | 10 +- .../scheduling/scheduler_resource_reporter.cc | 2 +- src/ray/raylet/wait_manager.cc | 26 +-- src/ray/raylet/wait_manager.h | 18 +- .../worker_killing_policy_group_by_owner.h | 4 +- src/ray/raylet/worker_pool.cc | 100 ++++----- src/ray/raylet/worker_pool.h | 49 ++-- src/ray/rpc/worker/core_worker_client.cc | 2 +- src/ray/rpc/worker/core_worker_client.h | 10 +- src/ray/rpc/worker/core_worker_client_pool.cc | 10 +- src/ray/rpc/worker/core_worker_client_pool.h | 12 +- thirdparty/patches/abseil-cpp-shadow.patch | 12 + thirdparty/patches/msgpack-shadow.patch | 12 + 120 files changed, 1485 insertions(+), 1430 deletions(-) create mode 100644 thirdparty/patches/abseil-cpp-shadow.patch create mode 100644 thirdparty/patches/msgpack-shadow.patch diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 9ef8b4a9c07b..2c273128fd74 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -16,6 +16,7 @@ COPTS_WITHOUT_LOG = select({ "-Wconversion-null", "-Wno-misleading-indentation", "-Wimplicit-fallthrough", + "-Wshadow", ], }) + select({ "//:clang-cl": [ diff --git a/bazel/ray_deps_setup.bzl b/bazel/ray_deps_setup.bzl index 5ece022cdb29..13a6a96c162e 100644 --- a/bazel/ray_deps_setup.bzl +++ b/bazel/ray_deps_setup.bzl @@ -255,6 +255,10 @@ def ray_deps_setup(): urls = [ "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz", ], + patches = [ + # TODO (israbbani): #55430 Separate the compiler flags and remove this patch + "@io_ray//thirdparty/patches:abseil-cpp-shadow.patch", + ], ) # OpenCensus depends on jupp0r/prometheus-cpp @@ -357,6 +361,8 @@ def ray_deps_setup(): sha256 = "83c37c9ad926bbee68d564d9f53c6cbb057c1f755c264043ddd87d89e36d15bb", patches = [ "@io_ray//thirdparty/patches:msgpack-windows-iovec.patch", + # TODO (israbbani): #55430 Separate the compiler flags and remove this patch + "@io_ray//thirdparty/patches:msgpack-shadow.patch", ], ) diff --git a/cpp/include/ray/api/actor_creator.h b/cpp/include/ray/api/actor_creator.h index 0c59b007355c..973068e36766 100644 --- a/cpp/include/ray/api/actor_creator.h +++ b/cpp/include/ray/api/actor_creator.h @@ -92,14 +92,14 @@ ActorHandle, is_x_lang_v> ActorCreator::Remote(Args &&...a if constexpr (is_x_lang_v) { using ArgsTuple = std::tuple; - Arguments::WrapArgs(remote_function_holder_.lang_type, + Arguments::WrapArgs(remote_function_holder_.lang_type_, &args_, std::make_index_sequence{}, std::forward(args)...); } else { StaticCheck(); using ArgsTuple = RemoveReference_t>; - Arguments::WrapArgs(remote_function_holder_.lang_type, + Arguments::WrapArgs(remote_function_holder_.lang_type_, &args_, std::make_index_sequence{}, std::forward(args)...); diff --git a/cpp/include/ray/api/actor_task_caller.h b/cpp/include/ray/api/actor_task_caller.h index 9824234357d8..d0c22fabeb01 100644 --- a/cpp/include/ray/api/actor_task_caller.h +++ b/cpp/include/ray/api/actor_task_caller.h @@ -69,14 +69,14 @@ ObjectRef> ActorTaskCaller::Remote( if constexpr (is_x_lang_v) { using ArgsTuple = std::tuple; - Arguments::WrapArgs(remote_function_holder_.lang_type, + Arguments::WrapArgs(remote_function_holder_.lang_type_, &args_, std::make_index_sequence{}, std::forward(args)...); } else { StaticCheck(); using ArgsTuple = RemoveReference_t>>; - Arguments::WrapArgs(remote_function_holder_.lang_type, + Arguments::WrapArgs(remote_function_holder_.lang_type_, &args_, std::make_index_sequence{}, std::forward(args)...); diff --git a/cpp/include/ray/api/ray_runtime.h b/cpp/include/ray/api/ray_runtime.h index 8a8bf35e83ce..a56c95f148d7 100644 --- a/cpp/include/ray/api/ray_runtime.h +++ b/cpp/include/ray/api/ray_runtime.h @@ -32,24 +32,24 @@ struct RemoteFunctionHolder { RemoteFunctionHolder(const std::string &module_name, const std::string &function_name, const std::string &class_name = "", - LangType lang_type = LangType::CPP) { - this->module_name = module_name; - this->function_name = function_name; - this->class_name = class_name; - this->lang_type = lang_type; - } + LangType lang_type = LangType::CPP) + : module_name_(module_name), + function_name_(function_name), + class_name_(class_name), + lang_type_(lang_type) {} + RemoteFunctionHolder(std::string func_name) { if (func_name.empty()) { throw RayException( "Function not found. Please use RAY_REMOTE to register this function."); } - function_name = std::move(func_name); + function_name_ = std::move(func_name); } - std::string module_name; - std::string function_name; - std::string class_name; - LangType lang_type = LangType::CPP; + std::string module_name_; + std::string function_name_; + std::string class_name_; + LangType lang_type_ = LangType::CPP; }; class RayRuntime { diff --git a/cpp/include/ray/api/task_caller.h b/cpp/include/ray/api/task_caller.h index ca61c49c594f..c3b24f6dbe8c 100644 --- a/cpp/include/ray/api/task_caller.h +++ b/cpp/include/ray/api/task_caller.h @@ -83,14 +83,14 @@ ObjectRef> TaskCaller::Remote( if constexpr (is_x_lang_v) { using ArgsTuple = std::tuple; - Arguments::WrapArgs(remote_function_holder_.lang_type, + Arguments::WrapArgs(remote_function_holder_.lang_type_, &args_, std::make_index_sequence{}, std::forward(args)...); } else { StaticCheck(); using ArgsTuple = RemoveReference_t>; - Arguments::WrapArgs(remote_function_holder_.lang_type, + Arguments::WrapArgs(remote_function_holder_.lang_type_, &args_, std::make_index_sequence{}, std::forward(args)...); diff --git a/cpp/src/ray/runtime/abstract_ray_runtime.cc b/cpp/src/ray/runtime/abstract_ray_runtime.cc index 50b3f9f9073d..0f6ec2e24b4a 100644 --- a/cpp/src/ray/runtime/abstract_ray_runtime.cc +++ b/cpp/src/ray/runtime/abstract_ray_runtime.cc @@ -172,7 +172,7 @@ InvocationSpec BuildInvocationSpec1(TaskType task_type, invocation_spec.remote_function_holder = remote_function_holder; invocation_spec.actor_id = actor; invocation_spec.args = - TransformArgs(args, remote_function_holder.lang_type != LangType::CPP); + TransformArgs(args, remote_function_holder.lang_type_ != LangType::CPP); return invocation_spec; } @@ -199,23 +199,23 @@ std::string AbstractRayRuntime::CallActor( std::vector &args, const CallOptions &call_options) { InvocationSpec invocation_spec{}; - if (remote_function_holder.lang_type == LangType::PYTHON) { + if (remote_function_holder.lang_type_ == LangType::PYTHON) { const auto native_actor_handle = CoreWorkerProcess::GetCoreWorker().GetActorHandle( ray::ActorID::FromBinary(actor)); auto function_descriptor = native_actor_handle->ActorCreationTaskFunctionDescriptor(); auto typed_descriptor = function_descriptor->As(); RemoteFunctionHolder func_holder = remote_function_holder; - func_holder.module_name = typed_descriptor->ModuleName(); - func_holder.class_name = typed_descriptor->ClassName(); + func_holder.module_name_ = typed_descriptor->ModuleName(); + func_holder.class_name_ = typed_descriptor->ClassName(); invocation_spec = BuildInvocationSpec1( TaskType::ACTOR_TASK, func_holder, args, ActorID::FromBinary(actor)); - } else if (remote_function_holder.lang_type == LangType::JAVA) { + } else if (remote_function_holder.lang_type_ == LangType::JAVA) { const auto native_actor_handle = CoreWorkerProcess::GetCoreWorker().GetActorHandle( ray::ActorID::FromBinary(actor)); auto function_descriptor = native_actor_handle->ActorCreationTaskFunctionDescriptor(); auto typed_descriptor = function_descriptor->As(); RemoteFunctionHolder func_holder = remote_function_holder; - func_holder.class_name = typed_descriptor->ClassName(); + func_holder.class_name_ = typed_descriptor->ClassName(); invocation_spec = BuildInvocationSpec1( TaskType::ACTOR_TASK, func_holder, args, ActorID::FromBinary(actor)); } else { diff --git a/cpp/src/ray/runtime/task/local_mode_task_submitter.cc b/cpp/src/ray/runtime/task/local_mode_task_submitter.cc index 90cba57d573b..6c91f2516b19 100644 --- a/cpp/src/ray/runtime/task/local_mode_task_submitter.cc +++ b/cpp/src/ray/runtime/task/local_mode_task_submitter.cc @@ -37,7 +37,7 @@ ObjectID LocalModeTaskSubmitter::Submit(InvocationSpec &invocation, /// Maybe some information of TaskSpecification are not reasonable or invalid. /// We will enhance this after implement the cluster mode. auto functionDescriptor = FunctionDescriptorBuilder::BuildCpp( - invocation.remote_function_holder.function_name); + invocation.remote_function_holder.function_name_); rpc::Address address; std::unordered_map required_resources; std::unordered_map required_placement_resources; diff --git a/cpp/src/ray/runtime/task/native_task_submitter.cc b/cpp/src/ray/runtime/task/native_task_submitter.cc index 8983eb857ae4..b2e035cc415e 100644 --- a/cpp/src/ray/runtime/task/native_task_submitter.cc +++ b/cpp/src/ray/runtime/task/native_task_submitter.cc @@ -26,23 +26,23 @@ using ray::core::CoreWorkerProcess; using ray::core::TaskOptions; RayFunction BuildRayFunction(InvocationSpec &invocation) { - if (invocation.remote_function_holder.lang_type == LangType::CPP) { + if (invocation.remote_function_holder.lang_type_ == LangType::CPP) { auto function_descriptor = FunctionDescriptorBuilder::BuildCpp( - invocation.remote_function_holder.function_name, + invocation.remote_function_holder.function_name_, "", - invocation.remote_function_holder.class_name); + invocation.remote_function_holder.class_name_); return RayFunction(ray::Language::CPP, function_descriptor); - } else if (invocation.remote_function_holder.lang_type == LangType::PYTHON) { + } else if (invocation.remote_function_holder.lang_type_ == LangType::PYTHON) { auto function_descriptor = FunctionDescriptorBuilder::BuildPython( - invocation.remote_function_holder.module_name, - invocation.remote_function_holder.class_name, - invocation.remote_function_holder.function_name, + invocation.remote_function_holder.module_name_, + invocation.remote_function_holder.class_name_, + invocation.remote_function_holder.function_name_, ""); return RayFunction(ray::Language::PYTHON, function_descriptor); - } else if (invocation.remote_function_holder.lang_type == LangType::JAVA) { + } else if (invocation.remote_function_holder.lang_type_ == LangType::JAVA) { auto function_descriptor = FunctionDescriptorBuilder::BuildJava( - invocation.remote_function_holder.class_name, - invocation.remote_function_holder.function_name, + invocation.remote_function_holder.class_name_, + invocation.remote_function_holder.function_name_, ""); return RayFunction(ray::Language::JAVA, function_descriptor); } else { diff --git a/cpp/src/ray/test/cluster/cluster_mode_test.cc b/cpp/src/ray/test/cluster/cluster_mode_test.cc index 2e7d2ff9a31a..3bbd0809c393 100644 --- a/cpp/src/ray/test/cluster/cluster_mode_test.cc +++ b/cpp/src/ray/test/cluster/cluster_mode_test.cc @@ -586,20 +586,20 @@ TEST(RayClusterModeTest, GetNamespaceApiTest) { class Pip { public: - std::vector packages; - bool pip_check = false; + std::vector packages_; + bool pip_check_ = false; Pip() = default; Pip(const std::vector &packages, bool pip_check) - : packages(packages), pip_check(pip_check) {} + : packages_(packages), pip_check_(pip_check) {} }; void to_json(nlohmann::json &j, const Pip &pip) { - j = nlohmann::json{{"packages", pip.packages}, {"pip_check", pip.pip_check}}; + j = nlohmann::json{{"packages", pip.packages_}, {"pip_check", pip.pip_check_}}; }; void from_json(const nlohmann::json &j, Pip &pip) { - j.at("packages").get_to(pip.packages); - j.at("pip_check").get_to(pip.pip_check); + j.at("packages").get_to(pip.packages_); + j.at("pip_check").get_to(pip.pip_check_); }; TEST(RayClusterModeTest, RuntimeEnvApiTest) { @@ -618,8 +618,8 @@ TEST(RayClusterModeTest, RuntimeEnvApiTest) { // Deserialize auto runtime_env_2 = ray::RuntimeEnv::Deserialize(serialized_runtime_env); auto pip2 = runtime_env_2.Get("pip"); - EXPECT_EQ(pip2.packages, pip.packages); - EXPECT_EQ(pip2.pip_check, pip.pip_check); + EXPECT_EQ(pip2.packages_, pip.packages_); + EXPECT_EQ(pip2.pip_check_, pip.pip_check_); auto working_dir2 = runtime_env_2.Get("working_dir"); EXPECT_EQ(working_dir2, working_dir); diff --git a/cpp/src/ray/test/cluster/counter.cc b/cpp/src/ray/test/cluster/counter.cc index eb77917189d9..7c994d3e0d99 100644 --- a/cpp/src/ray/test/cluster/counter.cc +++ b/cpp/src/ray/test/cluster/counter.cc @@ -84,10 +84,10 @@ bool Counter::CheckRestartInActorCreationTask() { return is_restared; } bool Counter::CheckRestartInActorTask() { return ray::WasCurrentActorRestarted(); } ray::ActorHandle Counter::CreateChildActor(std::string actor_name) { - auto child_actor = + auto new_child_actor = ray::Actor(RAY_FUNC(Counter::FactoryCreate)).SetName(actor_name).Remote(); - child_actor.Task(&Counter::GetCount).Remote().Get(); - return child_actor; + new_child_actor.Task(&Counter::GetCount).Remote().Get(); + return new_child_actor; } std::string Counter::GetNamespaceInActor() { return ray::GetNamespace(); } diff --git a/src/ray/common/BUILD.bazel b/src/ray/common/BUILD.bazel index 7e95589b69ad..992728a11bc7 100644 --- a/src/ray/common/BUILD.bazel +++ b/src/ray/common/BUILD.bazel @@ -282,6 +282,7 @@ ray_cc_library( srcs = ["status.cc"], hdrs = ["status.h"], deps = [ + ":macros", ":source_location", "//src/ray/util:macros", "//src/ray/util:visibility", diff --git a/src/ray/common/memory_monitor.cc b/src/ray/common/memory_monitor.cc index 98a33da0cd84..2a07d57b3e52 100644 --- a/src/ray/common/memory_monitor.cc +++ b/src/ray/common/memory_monitor.cc @@ -51,10 +51,10 @@ MemoryMonitor::MemoryMonitor(instrumented_io_context &io_service, << " system memory), total system memory bytes: " << total_memory_bytes; runner_->RunFnPeriodically( [this] { - auto [used_memory_bytes, total_memory_bytes] = GetMemoryBytes(); + auto [used_mem_bytes, total_mem_bytes] = GetMemoryBytes(); MemorySnapshot system_memory; - system_memory.used_bytes = used_memory_bytes; - system_memory.total_bytes = total_memory_bytes; + system_memory.used_bytes = used_mem_bytes; + system_memory.total_bytes = total_mem_bytes; bool is_usage_above_threshold = IsUsageAboveThreshold(system_memory, computed_threshold_bytes_); diff --git a/src/ray/common/ray_syncer/ray_syncer.cc b/src/ray/common/ray_syncer/ray_syncer.cc index 8fadae0f9957..7991fdcd2c92 100644 --- a/src/ray/common/ray_syncer/ray_syncer.cc +++ b/src/ray/common/ray_syncer/ray_syncer.cc @@ -86,11 +86,11 @@ void RaySyncer::Connect(const std::string &node_id, /* message_processor */ [this](auto msg) { BroadcastMessage(std::move(msg)); }, /* cleanup_cb */ - [this, channel](RaySyncerBidiReactor *reactor, bool restart) { - const std::string &node_id = reactor->GetRemoteNodeID(); - auto iter = sync_reactors_.find(node_id); + [this, channel](RaySyncerBidiReactor *bidi_reactor, bool restart) { + const std::string &remote_node_id = bidi_reactor->GetRemoteNodeID(); + auto iter = sync_reactors_.find(remote_node_id); if (iter != sync_reactors_.end()) { - if (iter->second != reactor) { + if (iter->second != bidi_reactor) { // The client is already reconnected. return; } @@ -99,14 +99,14 @@ void RaySyncer::Connect(const std::string &node_id, if (restart) { execute_after( io_context_, - [this, node_id, channel]() { - RAY_LOG(INFO).WithField(NodeID::FromBinary(node_id)) + [this, remote_node_id, channel]() { + RAY_LOG(INFO).WithField(NodeID::FromBinary(remote_node_id)) << "Connection is broken. Reconnect to node."; - Connect(node_id, channel); + Connect(remote_node_id, channel); }, /* delay_microseconds = */ std::chrono::milliseconds(2000)); } else { - node_state_->RemoveNode(node_id); + node_state_->RemoveNode(remote_node_id); } }, /* stub */ std::move(stub)); @@ -124,7 +124,7 @@ void RaySyncer::Connect(RaySyncerBidiReactor *reactor) { boost::asio::dispatch( io_context_.get_executor(), std::packaged_task([this, reactor]() { - auto [_, is_new] = sync_reactors_.emplace(reactor->GetRemoteNodeID(), reactor); + auto is_new = sync_reactors_.emplace(reactor->GetRemoteNodeID(), reactor).second; RAY_CHECK(is_new) << NodeID::FromBinary(reactor->GetRemoteNodeID()) << " has already registered."; // Send the view for new connections. @@ -223,13 +223,13 @@ ServerBidiReactor *RaySyncerService::StartSync(grpc::CallbackServerContext *cont syncer_.GetLocalNodeID(), /*message_processor=*/[this](auto msg) mutable { syncer_.BroadcastMessage(msg); }, /*cleanup_cb=*/ - [this](RaySyncerBidiReactor *reactor, bool reconnect) mutable { + [this](RaySyncerBidiReactor *bidi_reactor, bool reconnect) mutable { // No need to reconnect for server side. RAY_CHECK(!reconnect); - const auto &node_id = reactor->GetRemoteNodeID(); + const auto &node_id = bidi_reactor->GetRemoteNodeID(); auto iter = syncer_.sync_reactors_.find(node_id); if (iter != syncer_.sync_reactors_.end()) { - if (iter->second != reactor) { + if (iter->second != bidi_reactor) { // There is a new connection to the node, no need to clean up. // This can happen when there is transient network error and the client // reconnects. The sequence of events are: diff --git a/src/ray/common/scheduling/resource_instance_set.cc b/src/ray/common/scheduling/resource_instance_set.cc index 4765dd7c5c5e..ed07a13fe24b 100644 --- a/src/ray/common/scheduling/resource_instance_set.cc +++ b/src/ray/common/scheduling/resource_instance_set.cc @@ -202,8 +202,8 @@ NodeResourceInstanceSet::TryAllocate(const ResourceSet &resource_demands) { allocations[resource_id] = std::move(*allocation); } else { // Allocation failed. Restore partially allocated resources. - for (const auto &[resource_id, allocation] : allocations) { - Free(resource_id, allocation); + for (const auto &[id, allocated] : allocations) { + Free(id, allocated); } return std::nullopt; } diff --git a/src/ray/common/scheduling/scheduling_ids.h b/src/ray/common/scheduling/scheduling_ids.h index 1e8ead6ba118..e3b067bc53b8 100644 --- a/src/ray/common/scheduling/scheduling_ids.h +++ b/src/ray/common/scheduling/scheduling_ids.h @@ -143,15 +143,15 @@ inline std::ostream &operator<<( /// the singleton map with PredefinedResources. template <> inline StringIdMap &BaseSchedulingID::GetMap() { - static std::unique_ptr map{[]() { - std::unique_ptr map(new StringIdMap()); - map->InsertOrDie(kCPU_ResourceLabel, CPU) + static std::unique_ptr singleton_map{[]() { + std::unique_ptr map_ptr(new StringIdMap()); + map_ptr->InsertOrDie(kCPU_ResourceLabel, CPU) .InsertOrDie(kGPU_ResourceLabel, GPU) .InsertOrDie(kObjectStoreMemory_ResourceLabel, OBJECT_STORE_MEM) .InsertOrDie(kMemory_ResourceLabel, MEM); - return map; + return map_ptr; }()}; - return *map; + return *singleton_map; } namespace scheduling { diff --git a/src/ray/common/status.h b/src/ray/common/status.h index 3b9ba8a6cc43..1230b449c072 100644 --- a/src/ray/common/status.h +++ b/src/ray/common/status.h @@ -33,6 +33,7 @@ #include #include "absl/strings/str_cat.h" +#include "ray/common/macros.h" #include "ray/common/source_location.h" #include "ray/util/macros.h" #include "ray/util/visibility.h" @@ -52,10 +53,10 @@ class error_code; // If the status is not OK, CHECK-fail immediately, appending the status to the // logged message. The message can be appended with <<. -#define RAY_CHECK_OK(s) \ - if (const ::ray::Status &_status_ = (s); true) \ - RAY_CHECK_WITH_DISPLAY(_status_.ok(), #s) \ - << "Status not OK: " << _status_.ToString() << " " +#define RAY_CHECK_OK(s) \ + if (const ::ray::Status & RAY_UNIQUE_VARIABLE(_s) = (s); true) \ + RAY_CHECK_WITH_DISPLAY(RAY_UNIQUE_VARIABLE(_s).ok(), #s) \ + << "Status not OK: " << RAY_UNIQUE_VARIABLE(_s).ToString() << " " namespace ray { diff --git a/src/ray/common/task/task_spec.h b/src/ray/common/task/task_spec.h index a076309d4b07..d23fd5bff286 100644 --- a/src/ray/common/task/task_spec.h +++ b/src/ray/common/task/task_spec.h @@ -81,12 +81,12 @@ struct SchedulingClassDescriptor { LabelSelector ls, FunctionDescriptor fd, int64_t d, - rpc::SchedulingStrategy scheduling_strategy) + rpc::SchedulingStrategy sched_strategy) : resource_set(std::move(rs)), label_selector(std::move(ls)), function_descriptor(std::move(fd)), depth(d), - scheduling_strategy(std::move(scheduling_strategy)) {} + scheduling_strategy(std::move(sched_strategy)) {} ResourceSet resource_set; LabelSelector label_selector; FunctionDescriptor function_descriptor; @@ -155,17 +155,17 @@ namespace std { template <> struct hash { size_t operator()(const ray::rpc::LabelOperator &label_operator) const { - size_t hash = std::hash()(label_operator.label_operator_case()); + size_t hash_value = std::hash()(label_operator.label_operator_case()); if (label_operator.has_label_in()) { for (const auto &value : label_operator.label_in().values()) { - hash ^= std::hash()(value); + hash_value ^= std::hash()(value); } } else if (label_operator.has_label_not_in()) { for (const auto &value : label_operator.label_not_in().values()) { - hash ^= std::hash()(value); + hash_value ^= std::hash()(value); } } - return hash; + return hash_value; } }; @@ -239,25 +239,25 @@ namespace ray { /// a executing thread pool. struct ConcurrencyGroup { // Name of this group. - std::string name; + std::string name_; // Max concurrency of this group. - uint32_t max_concurrency; + uint32_t max_concurrency_; // Function descriptors of the actor methods in this group. - std::vector function_descriptors; + std::vector function_descriptors_; ConcurrencyGroup() = default; ConcurrencyGroup(const std::string &name, uint32_t max_concurrency, const std::vector &fds) - : name(name), max_concurrency(max_concurrency), function_descriptors(fds) {} + : name_(name), max_concurrency_(max_concurrency), function_descriptors_(fds) {} - std::string GetName() const { return name; } + std::string GetName() const { return name_; } - uint32_t GetMaxConcurrency() const { return max_concurrency; } + uint32_t GetMaxConcurrency() const { return max_concurrency_; } std::vector GetFunctionDescriptors() const { - return function_descriptors; + return function_descriptors_; } }; diff --git a/src/ray/common/task/task_util.h b/src/ray/common/task/task_util.h index 1665f9e96298..5779cd2ca3f2 100644 --- a/src/ray/common/task/task_util.h +++ b/src/ray/common/task/task_util.h @@ -30,17 +30,17 @@ namespace ray { /// Stores the task failure reason. struct TaskFailureEntry { /// The task failure details. - rpc::RayErrorInfo ray_error_info; + rpc::RayErrorInfo ray_error_info_; /// The creation time of this entry. - std::chrono::steady_clock::time_point creation_time; + std::chrono::steady_clock::time_point creation_time_; /// Whether this task should be retried. - bool should_retry; + bool should_retry_; TaskFailureEntry(const rpc::RayErrorInfo &ray_error_info, bool should_retry) - : ray_error_info(ray_error_info), - creation_time(std::chrono::steady_clock::now()), - should_retry(should_retry) {} + : ray_error_info_(ray_error_info), + creation_time_(std::chrono::steady_clock::now()), + should_retry_(should_retry) {} }; /// Argument of a task. @@ -271,10 +271,10 @@ class TaskSpecBuilder { actor_creation_spec->set_serialized_actor_handle(serialized_actor_handle); for (const auto &concurrency_group : concurrency_groups) { rpc::ConcurrencyGroup *group = actor_creation_spec->add_concurrency_groups(); - group->set_name(concurrency_group.name); - group->set_max_concurrency(concurrency_group.max_concurrency); + group->set_name(concurrency_group.name_); + group->set_max_concurrency(concurrency_group.max_concurrency_); // Fill into function descriptor. - for (auto &item : concurrency_group.function_descriptors) { + for (auto &item : concurrency_group.function_descriptors_) { rpc::FunctionDescriptor *fd = group->add_function_descriptors(); *fd = item->GetMessage(); } diff --git a/src/ray/core_worker/common.h b/src/ray/core_worker/common.h index de2a41013a7a..2cbdbf5bdafa 100644 --- a/src/ray/core_worker/common.h +++ b/src/ray/core_worker/common.h @@ -223,35 +223,35 @@ struct PlacementGroupCreationOptions { NodeID soft_target_node_id = NodeID::Nil(), std::vector> bundle_label_selector = {}) - : name(std::move(name)), - strategy(strategy), - bundles(std::move(bundles)), - is_detached(is_detached_p), - max_cpu_fraction_per_node(max_cpu_fraction_per_node), - soft_target_node_id(soft_target_node_id), - bundle_label_selector(std::move(bundle_label_selector)) { - RAY_CHECK(soft_target_node_id.IsNil() || strategy == PlacementStrategy::STRICT_PACK) + : name_(std::move(name)), + strategy_(strategy), + bundles_(std::move(bundles)), + is_detached_(is_detached_p), + max_cpu_fraction_per_node_(max_cpu_fraction_per_node), + soft_target_node_id_(soft_target_node_id), + bundle_label_selector_(std::move(bundle_label_selector)) { + RAY_CHECK(soft_target_node_id_.IsNil() || strategy_ == PlacementStrategy::STRICT_PACK) << "soft_target_node_id only works with STRICT_PACK now"; } /// The name of the placement group. - const std::string name; + const std::string name_; /// The strategy to place the bundle in Placement Group. - const PlacementStrategy strategy = rpc::PACK; + const PlacementStrategy strategy_ = rpc::PACK; /// The resource bundles in this placement group. - const std::vector> bundles; + const std::vector> bundles_; /// Whether to keep the placement group persistent after its creator dead. - const bool is_detached = false; + const bool is_detached_ = false; /// The maximum fraction of CPU cores this placement group can take up on each node. - const double max_cpu_fraction_per_node; + const double max_cpu_fraction_per_node_; /// ID of the target node where bundles should be placed /// iff the target node has enough available resources and alive. /// Otherwise, the bundles can be placed elsewhere. /// Nil means there is no target node. /// This only applies to STRICT_PACK pg. - const NodeID soft_target_node_id; + const NodeID soft_target_node_id_; /// The label selectors to apply per-bundle in this placement group. - const std::vector> bundle_label_selector; + const std::vector> bundle_label_selector_; }; class ObjectLocation { @@ -311,13 +311,13 @@ namespace std { template <> struct hash { size_t operator()(const ray::rpc::LineageReconstructionTask &task) const { - size_t hash = std::hash()(task.name()); - hash ^= std::hash()(task.status()); + size_t hash_value = std::hash()(task.name()); + hash_value ^= std::hash()(task.status()); for (const auto &label : task.labels()) { - hash ^= std::hash()(label.first); - hash ^= std::hash()(label.second); + hash_value ^= std::hash()(label.first); + hash_value ^= std::hash()(label.second); } - return hash; + return hash_value; } }; } // namespace std diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 83ac7000178b..eef949f69a09 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -1338,7 +1338,7 @@ Status CoreWorker::ExperimentalRegisterMutableObjectReaderRemote( conn->RegisterMutableObjectReader( req, [&promise, num_replied, num_requests, addr]( - const Status &status, const rpc::RegisterMutableObjectReaderReply &reply) { + const Status &status, const rpc::RegisterMutableObjectReaderReply &) { RAY_CHECK_OK(status); *num_replied += 1; if (*num_replied == num_requests) { @@ -1853,8 +1853,8 @@ json CoreWorker::OverrideRuntimeEnv(const json &child, std::shared_ptr CoreWorker::OverrideTaskOrActorRuntimeEnvInfo( const std::string &serialized_runtime_env_info) const { - auto factory = [this](const std::string &serialized_runtime_env_info) { - return OverrideTaskOrActorRuntimeEnvInfoImpl(serialized_runtime_env_info); + auto factory = [this](const std::string &runtime_env_info_str) { + return OverrideTaskOrActorRuntimeEnvInfoImpl(runtime_env_info_str); }; return runtime_env_json_serialization_cache_.GetOrCreate(serialized_runtime_env_info, std::move(factory)); @@ -2303,7 +2303,7 @@ Status CoreWorker::CreateActor(const RayFunction &function, Status CoreWorker::CreatePlacementGroup( const PlacementGroupCreationOptions &placement_group_creation_options, PlacementGroupID *return_placement_group_id) { - const auto &bundles = placement_group_creation_options.bundles; + const auto &bundles = placement_group_creation_options.bundles_; for (const auto &bundle : bundles) { for (const auto &resource : bundle) { if (resource.first == kBundle_ResourceLabel) { @@ -2318,16 +2318,16 @@ Status CoreWorker::CreatePlacementGroup( PlacementGroupSpecBuilder builder; builder.SetPlacementGroupSpec( placement_group_id, - placement_group_creation_options.name, - placement_group_creation_options.bundles, - placement_group_creation_options.strategy, - placement_group_creation_options.is_detached, - placement_group_creation_options.max_cpu_fraction_per_node, - placement_group_creation_options.soft_target_node_id, + placement_group_creation_options.name_, + placement_group_creation_options.bundles_, + placement_group_creation_options.strategy_, + placement_group_creation_options.is_detached_, + placement_group_creation_options.max_cpu_fraction_per_node_, + placement_group_creation_options.soft_target_node_id_, worker_context_->GetCurrentJobID(), worker_context_->GetCurrentActorID(), worker_context_->CurrentActorDetached(), - placement_group_creation_options.bundle_label_selector); + placement_group_creation_options.bundle_label_selector_); PlacementGroupSpecification placement_group_spec = builder.Build(); *return_placement_group_id = placement_group_id; RAY_LOG(INFO).WithField(placement_group_id) @@ -3132,12 +3132,12 @@ bool CoreWorker::PinExistingReturnObject(const ObjectID &return_id, owner_address, {return_id}, generator_id, - [return_id, pinned_return_object](const Status &status, + [return_id, pinned_return_object](const Status &pin_object_status, const rpc::PinObjectIDsReply &reply) { // RPC to the local raylet should never fail. - if (!status.ok()) { + if (!pin_object_status.ok()) { RAY_LOG(ERROR) << "Request to local raylet to pin object failed: " - << status.ToString(); + << pin_object_status.ToString(); return; } if (!reply.successes(0)) { @@ -3586,8 +3586,9 @@ void CoreWorker::HandleWaitForActorRefDeleted( // Send a response to trigger cleaning up the actor state once the handle is // no longer in scope. - auto respond = [send_reply_callback](const ActorID &actor_id) { - RAY_LOG(DEBUG).WithField(actor_id) << "Replying to HandleWaitForActorRefDeleted"; + auto respond = [send_reply_callback](const ActorID &respond_actor_id) { + RAY_LOG(DEBUG).WithField(respond_actor_id) + << "Replying to HandleWaitForActorRefDeleted"; send_reply_callback(Status::OK(), nullptr, nullptr); }; diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index e8218c180a9e..db60abd62293 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -382,9 +382,9 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( // from the middle of user operations. core_worker->io_service_.post( [this, obj]() { - auto core_worker = GetCoreWorker(); - if (core_worker->options_.unhandled_exception_handler != nullptr) { - core_worker->options_.unhandled_exception_handler(obj); + auto this_core_worker = GetCoreWorker(); + if (this_core_worker->options_.unhandled_exception_handler != nullptr) { + this_core_worker->options_.unhandled_exception_handler(obj); } }, "CoreWorker.HandleException"); diff --git a/src/ray/core_worker/experimental_mutable_object_provider.cc b/src/ray/core_worker/experimental_mutable_object_provider.cc index ff5d2addac85..b97b19347278 100644 --- a/src/ray/core_worker/experimental_mutable_object_provider.cc +++ b/src/ray/core_worker/experimental_mutable_object_provider.cc @@ -242,9 +242,9 @@ void MutableObjectProvider::PollWriterClosure( object->GetData()->Data(), object->GetMetadata()->Data(), [this, &io_context, writer_object_id, remote_readers, num_replied]( - const Status &status, const rpc::PushMutableObjectReply &reply) { + const Status &push_object_status, const rpc::PushMutableObjectReply &reply) { *num_replied += 1; - if (!status.ok()) { + if (!push_object_status.ok()) { RAY_LOG(ERROR) << "Failed to transfer object to a remote node for an object id " << writer_object_id << ". It can cause hang."; diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc b/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc index 739bc6e5f63e..7584536ead71 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc @@ -69,18 +69,18 @@ jobject ToJavaArgs(JNIEnv *env, jobject args_array_list = NativeVectorToJavaList>( env, args, - [check_results, &i](JNIEnv *env, + [check_results, &i](JNIEnv *inner_env, const std::shared_ptr &native_object) { if (*(check_results + (i++))) { // If the type of this argument is ByteBuffer, we create a // DirectByteBuffer here To avoid data copy. // TODO(kfstorm): Check native_object->GetMetadata() == "RAW" - jobject obj = env->NewDirectByteBuffer(native_object->GetData()->Data(), - native_object->GetData()->Size()); + jobject obj = inner_env->NewDirectByteBuffer( + native_object->GetData()->Data(), native_object->GetData()->Size()); RAY_CHECK(obj); return obj; } - return NativeRayObjectToJavaNativeRayObject(env, native_object); + return NativeRayObjectToJavaNativeRayObject(inner_env, native_object); }); env->ReleaseBooleanArrayElements(java_check_results, check_results, JNI_ABORT); return args_array_list; @@ -152,7 +152,7 @@ Java_io_ray_runtime_RayNativeRuntime_nativeInitialize(JNIEnv *env, // errors for Java. *is_retryable_error = false; - JNIEnv *env = GetJNIEnv(); + JNIEnv *inner_env = GetJNIEnv(); RAY_CHECK(java_task_executor); // convert RayFunction @@ -168,53 +168,56 @@ Java_io_ray_runtime_RayNativeRuntime_nativeInitialize(JNIEnv *env, } if (!ray_function_array_list) { ray_function_array_list = - NativeRayFunctionDescriptorToJavaStringList(env, function_descriptor); + NativeRayFunctionDescriptorToJavaStringList(inner_env, function_descriptor); fd_vector.emplace_back(function_descriptor, ray_function_array_list); } // convert args // TODO(kfstorm): Avoid copying binary data from Java to C++ jbooleanArray java_check_results = static_cast( - env->CallObjectMethod(java_task_executor, - java_task_executor_parse_function_arguments, - ray_function_array_list)); - RAY_CHECK_JAVA_EXCEPTION(env); - jobject args_array_list = ToJavaArgs(env, java_check_results, args); + inner_env->CallObjectMethod(java_task_executor, + java_task_executor_parse_function_arguments, + ray_function_array_list)); + RAY_CHECK_JAVA_EXCEPTION(inner_env); + jobject args_array_list = ToJavaArgs(inner_env, java_check_results, args); // invoke Java method - jobject java_return_objects = env->CallObjectMethod(java_task_executor, - java_task_executor_execute, - ray_function_array_list, - args_array_list); + jobject java_return_objects = + inner_env->CallObjectMethod(java_task_executor, + java_task_executor_execute, + ray_function_array_list, + args_array_list); // Check whether the exception is `IntentionalSystemExit`. - jthrowable throwable = env->ExceptionOccurred(); + jthrowable throwable = inner_env->ExceptionOccurred(); if (throwable) { Status status_to_return = Status::OK(); - if (env->IsInstanceOf(throwable, - java_ray_intentional_system_exit_exception_class)) { + if (inner_env->IsInstanceOf(throwable, + java_ray_intentional_system_exit_exception_class)) { status_to_return = Status::IntentionalSystemExit(""); - } else if (env->IsInstanceOf(throwable, java_ray_actor_exception_class)) { - creation_task_exception_pb = SerializeActorCreationException(env, throwable); + } else if (inner_env->IsInstanceOf(throwable, java_ray_actor_exception_class)) { + creation_task_exception_pb = + SerializeActorCreationException(inner_env, throwable); status_to_return = Status::CreationTaskError(""); } else { RAY_LOG(ERROR) << "Unknown java exception was thrown while executing tasks."; } *application_error = status_to_return.ToString(); - env->ExceptionClear(); + inner_env->ExceptionClear(); return status_to_return; } - RAY_CHECK_JAVA_EXCEPTION(env); + RAY_CHECK_JAVA_EXCEPTION(inner_env); int64_t task_output_inlined_bytes = 0; // Process return objects. if (!returns->empty()) { std::vector> return_objects; JavaListToNativeVector>( - env, + inner_env, java_return_objects, &return_objects, - [](JNIEnv *env, jobject java_native_ray_object) { - return JavaNativeRayObjectToNativeRayObject(env, java_native_ray_object); + [](JNIEnv *object_env, jobject java_native_ray_object) { + return JavaNativeRayObjectToNativeRayObject(object_env, + java_native_ray_object); }); for (size_t i = 0; i < return_objects.size(); i++) { auto &result_id = (*returns)[i].first; @@ -251,9 +254,9 @@ Java_io_ray_runtime_RayNativeRuntime_nativeInitialize(JNIEnv *env, } } - env->DeleteLocalRef(java_check_results); - env->DeleteLocalRef(java_return_objects); - env->DeleteLocalRef(args_array_list); + inner_env->DeleteLocalRef(java_check_results); + inner_env->DeleteLocalRef(java_return_objects); + inner_env->DeleteLocalRef(args_array_list); return Status::OK(); }; @@ -273,9 +276,9 @@ Java_io_ray_runtime_RayNativeRuntime_nativeInitialize(JNIEnv *env, absl::MutexLock lock(&mutex); int64_t start = current_time_ms(); if (last_gc_time_ms + 1000 < start) { - JNIEnv *env = GetJNIEnv(); + JNIEnv *inner_env = GetJNIEnv(); RAY_LOG(DEBUG) << "Calling System.gc() ..."; - env->CallStaticObjectMethod(java_system_class, java_system_gc); + inner_env->CallStaticObjectMethod(java_system_class, java_system_gc); last_gc_time_ms = current_time_ms(); RAY_LOG(DEBUG) << "GC finished in " << static_cast(last_gc_time_ms - start) / 1000 @@ -315,34 +318,36 @@ Java_io_ray_runtime_RayNativeRuntime_nativeInitialize(JNIEnv *env, return std::make_shared( object.GetData(), object.GetMetadata(), object.GetNestedRefs(), true); } - JNIEnv *env = GetJNIEnv(); - auto java_byte_array = NativeBufferToJavaByteArray(env, object.GetData()); - auto raw_object_id_byte_array = NativeStringToJavaByteArray(env, object_id.Binary()); + JNIEnv *inner_env = GetJNIEnv(); + auto java_byte_array = NativeBufferToJavaByteArray(inner_env, object.GetData()); + auto raw_object_id_byte_array = + NativeStringToJavaByteArray(inner_env, object_id.Binary()); RAY_LOG(DEBUG) << "Allocating Java byte array for object " << object_id; - env->CallStaticVoidMethod(java_object_ref_impl_class, - java_object_ref_impl_class_on_memory_store_object_allocated, - raw_object_id_byte_array, - java_byte_array); - auto java_weak_ref = CreateJavaWeakRef(env, java_byte_array); + inner_env->CallStaticVoidMethod( + java_object_ref_impl_class, + java_object_ref_impl_class_on_memory_store_object_allocated, + raw_object_id_byte_array, + java_byte_array); + auto java_weak_ref = CreateJavaWeakRef(inner_env, java_byte_array); // This shared_ptr will be captured by the data_factory. So when the data_factory // is destructed, we deference the java_weak_ref. std::shared_ptr java_weak_ref_ptr{ reinterpret_cast(java_weak_ref), [](auto p) { - JNIEnv *env = GetJNIEnv(); - env->DeleteLocalRef(reinterpret_cast(p)); + JNIEnv *deleter_env = GetJNIEnv(); + deleter_env->DeleteLocalRef(reinterpret_cast(p)); }}; // Remove this local reference because this byte array is fate-sharing with the // ObjectRefImpl in Java frontend. - env->DeleteLocalRef(java_byte_array); - env->DeleteLocalRef(raw_object_id_byte_array); + inner_env->DeleteLocalRef(java_byte_array); + inner_env->DeleteLocalRef(raw_object_id_byte_array); auto data_factory = [java_weak_ref_ptr, object_id]() -> std::shared_ptr { - JNIEnv *env = GetJNIEnv(); - jbyteArray java_byte_array = (jbyteArray)env->CallObjectMethod( + JNIEnv *data_env = GetJNIEnv(); + jbyteArray _java_byte_array = (jbyteArray)data_env->CallObjectMethod( reinterpret_cast(java_weak_ref_ptr.get()), java_weak_reference_get); - RAY_CHECK_JAVA_EXCEPTION(env); - RAY_CHECK(java_byte_array != nullptr) + RAY_CHECK_JAVA_EXCEPTION(data_env); + RAY_CHECK(_java_byte_array != nullptr) << "The java byte array is null of object " << object_id; - return std::make_shared(env, java_byte_array); + return std::make_shared(data_env, _java_byte_array); }; std::shared_ptr metadata_buffer = object.GetMetadata(); return std::make_shared(metadata_buffer, @@ -407,22 +412,23 @@ JNIEXPORT void JNICALL Java_io_ray_runtime_RayNativeRuntime_nativeKillActor( JNIEXPORT jobject JNICALL Java_io_ray_runtime_RayNativeRuntime_nativeGetResourceIds(JNIEnv *env, jclass) { - auto key_converter = [](JNIEnv *env, const std::string &str) -> jstring { - return env->NewStringUTF(str.c_str()); + auto key_converter = [](JNIEnv *inner_env, const std::string &str) -> jstring { + return inner_env->NewStringUTF(str.c_str()); }; auto value_converter = - [](JNIEnv *env, const std::vector> &value) -> jobject { - auto elem_converter = [](JNIEnv *env, + [](JNIEnv *inner_env, + const std::vector> &value) -> jobject { + auto elem_converter = [](JNIEnv *object_env, const std::pair &elem) -> jobject { - jobject java_item = env->NewObject(java_resource_value_class, - java_resource_value_init, - (jlong)elem.first, - (jdouble)elem.second); - RAY_CHECK_JAVA_EXCEPTION(env); + jobject java_item = object_env->NewObject(java_resource_value_class, + java_resource_value_init, + (jlong)elem.first, + (jdouble)elem.second); + RAY_CHECK_JAVA_EXCEPTION(object_env); return java_item; }; return NativeVectorToJavaList>( - env, value, std::move(elem_converter)); + inner_env, value, std::move(elem_converter)); }; ResourceMappingType resource_mapping = CoreWorkerProcess::GetCoreWorker().GetResourceIDs(); diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_gcs_GlobalStateAccessor.cc b/src/ray/core_worker/lib/java/io_ray_runtime_gcs_GlobalStateAccessor.cc index e99c24530581..258263f176b1 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_gcs_GlobalStateAccessor.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_gcs_GlobalStateAccessor.cc @@ -64,8 +64,8 @@ JNIEXPORT jobject JNICALL Java_io_ray_runtime_gcs_GlobalStateAccessor_nativeGetA auto *gcs_accessor = reinterpret_cast(gcs_accessor_ptr); auto job_info_list = gcs_accessor->GetAllJobInfo(); return NativeVectorToJavaList( - env, job_info_list, [](JNIEnv *env, const std::string &str) { - return NativeStringToJavaByteArray(env, str); + env, job_info_list, [](JNIEnv *inner_env, const std::string &str) { + return NativeStringToJavaByteArray(inner_env, str); }); } @@ -85,8 +85,8 @@ Java_io_ray_runtime_gcs_GlobalStateAccessor_nativeGetAllNodeInfo(JNIEnv *env, auto *gcs_accessor = reinterpret_cast(gcs_accessor_ptr); auto node_info_list = gcs_accessor->GetAllNodeInfo(); return NativeVectorToJavaList( - env, node_info_list, [](JNIEnv *env, const std::string &str) { - return NativeStringToJavaByteArray(env, str); + env, node_info_list, [](JNIEnv *inner_env, const std::string &str) { + return NativeStringToJavaByteArray(inner_env, str); }); } @@ -110,8 +110,8 @@ Java_io_ray_runtime_gcs_GlobalStateAccessor_nativeGetAllActorInfo( auto actor_info_list = gcs_accessor->GetAllActorInfo(std::nullopt, job_id, actor_state_name); return NativeVectorToJavaList( - env, actor_info_list, [](JNIEnv *env, const std::string &str) { - return NativeStringToJavaByteArray(env, str); + env, actor_info_list, [](JNIEnv *inner_env, const std::string &str) { + return NativeStringToJavaByteArray(inner_env, str); }); } @@ -161,8 +161,8 @@ Java_io_ray_runtime_gcs_GlobalStateAccessor_nativeGetAllPlacementGroupInfo( auto *gcs_accessor = reinterpret_cast(gcs_accessor_ptr); auto placement_group_info_list = gcs_accessor->GetAllPlacementGroupInfo(); return NativeVectorToJavaList( - env, placement_group_info_list, [](JNIEnv *env, const std::string &str) { - return NativeStringToJavaByteArray(env, str); + env, placement_group_info_list, [](JNIEnv *inner_env, const std::string &str) { + return NativeStringToJavaByteArray(inner_env, str); }); } diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_object_NativeObjectStore.cc b/src/ray/core_worker/lib/java/io_ray_runtime_object_NativeObjectStore.cc index e83a3773f184..2c0f5548ec0a 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_object_NativeObjectStore.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_object_NativeObjectStore.cc @@ -128,9 +128,10 @@ Java_io_ray_runtime_object_NativeObjectStore_nativePut___3BLio_ray_runtime_objec JNIEXPORT jobject JNICALL Java_io_ray_runtime_object_NativeObjectStore_nativeGet( JNIEnv *env, jclass, jobject ids, jlong timeoutMs) { std::vector object_ids; - JavaListToNativeVector(env, ids, &object_ids, [](JNIEnv *env, jobject id) { - return JavaByteArrayToId(env, static_cast(id)); - }); + JavaListToNativeVector( + env, ids, &object_ids, [](JNIEnv *inner_env, jobject id) { + return JavaByteArrayToId(inner_env, static_cast(id)); + }); std::vector> results; auto status = CoreWorkerProcess::GetCoreWorker().Get( object_ids, static_cast(timeoutMs), results); @@ -148,8 +149,8 @@ Java_io_ray_runtime_object_NativeObjectStore_nativeWait(JNIEnv *env, jboolean fetch_local) { std::vector object_ids; JavaListToNativeVector( - env, objectIds, &object_ids, [](JNIEnv *env, jobject id) { - return JavaByteArrayToId(env, static_cast(id)); + env, objectIds, &object_ids, [](JNIEnv *inner_env, jobject id) { + return JavaByteArrayToId(inner_env, static_cast(id)); }); std::vector results; auto status = CoreWorkerProcess::GetCoreWorker().Wait(object_ids, @@ -158,20 +159,21 @@ Java_io_ray_runtime_object_NativeObjectStore_nativeWait(JNIEnv *env, &results, static_cast(fetch_local)); THROW_EXCEPTION_AND_RETURN_IF_NOT_OK(env, status, nullptr); - return NativeVectorToJavaList(env, results, [](JNIEnv *env, const bool &item) { - jobject java_item = - env->NewObject(java_boolean_class, java_boolean_init, (jboolean)item); - RAY_CHECK_JAVA_EXCEPTION(env); - return java_item; - }); + return NativeVectorToJavaList( + env, results, [](JNIEnv *inner_env, const bool &item) { + jobject java_item = + inner_env->NewObject(java_boolean_class, java_boolean_init, (jboolean)item); + RAY_CHECK_JAVA_EXCEPTION(inner_env); + return java_item; + }); } JNIEXPORT void JNICALL Java_io_ray_runtime_object_NativeObjectStore_nativeDelete( JNIEnv *env, jclass, jobject objectIds, jboolean localOnly) { std::vector object_ids; JavaListToNativeVector( - env, objectIds, &object_ids, [](JNIEnv *env, jobject id) { - return JavaByteArrayToId(env, static_cast(id)); + env, objectIds, &object_ids, [](JNIEnv *inner_env, jobject id) { + return JavaByteArrayToId(inner_env, static_cast(id)); }); auto status = CoreWorkerProcess::GetCoreWorker().Delete(object_ids, static_cast(localOnly)); @@ -207,15 +209,15 @@ Java_io_ray_runtime_object_NativeObjectStore_nativeGetAllReferenceCounts(JNIEnv return NativeMapToJavaMap>( env, reference_counts, - [](JNIEnv *env, const ObjectID &key) { - return IdToJavaByteArray(env, key); + [](JNIEnv *inner_env, const ObjectID &key) { + return IdToJavaByteArray(inner_env, key); }, - [](JNIEnv *env, const std::pair &value) { - jlongArray array = env->NewLongArray(2); - jlong *elements = env->GetLongArrayElements(array, nullptr); + [](JNIEnv *inner_env, const std::pair &value) { + jlongArray array = inner_env->NewLongArray(2); + jlong *elements = inner_env->GetLongArrayElements(array, nullptr); elements[0] = static_cast(value.first); elements[1] = static_cast(value.second); - env->ReleaseLongArrayElements(array, elements, 0); + inner_env->ReleaseLongArrayElements(array, elements, 0); return array; }); } diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc index 531b19c3d58e..fc565d26e8c0 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc @@ -66,10 +66,10 @@ inline const RayFunction &ToRayFunction(JNIEnv *env, return fd_vector.back().second; } -inline std::vector> ToTaskArgs(JNIEnv *env, jobject args) { +inline std::vector> ToTaskArgs(JNIEnv *inner_env, jobject args) { std::vector> task_args; JavaListToNativeVector>( - env, args, &task_args, [](JNIEnv *env, jobject arg) { + inner_env, args, &task_args, [](JNIEnv *env, jobject arg) { auto java_id = env->GetObjectField(arg, java_function_arg_id); if (java_id) { auto java_id_bytes = static_cast( @@ -99,12 +99,12 @@ inline std::unordered_map ToResources(JNIEnv *env, return JavaMapToNativeMap( env, java_resources, - [](JNIEnv *env, jobject java_key) { - return JavaStringToNativeString(env, (jstring)java_key); + [](JNIEnv *inner_env, jobject java_key) { + return JavaStringToNativeString(inner_env, (jstring)java_key); }, - [](JNIEnv *env, jobject java_value) { - double value = env->CallDoubleMethod(java_value, java_double_double_value); - RAY_CHECK_JAVA_EXCEPTION(env); + [](JNIEnv *inner_env, jobject java_value) { + double value = inner_env->CallDoubleMethod(java_value, java_double_double_value); + RAY_CHECK_JAVA_EXCEPTION(inner_env); return value; }); } @@ -232,34 +232,35 @@ inline ActorCreationOptions ToActorCreationOptions(JNIEnv *env, env, java_concurrency_groups_field, &concurrency_groups, - [](JNIEnv *env, jobject java_concurrency_group_impl) { + [](JNIEnv *inner_env, jobject java_concurrency_group_impl) { RAY_CHECK(java_concurrency_group_impl != nullptr); - jobject java_func_descriptors = - env->CallObjectMethod(java_concurrency_group_impl, - java_concurrency_group_impl_get_function_descriptors); - RAY_CHECK_JAVA_EXCEPTION(env); + jobject java_func_descriptors = inner_env->CallObjectMethod( + java_concurrency_group_impl, + java_concurrency_group_impl_get_function_descriptors); + RAY_CHECK_JAVA_EXCEPTION(inner_env); std::vector native_func_descriptors; JavaListToNativeVector( - env, + inner_env, java_func_descriptors, &native_func_descriptors, - [](JNIEnv *env, jobject java_func_descriptor) { + [](JNIEnv *converter_env, jobject java_func_descriptor) { RAY_CHECK(java_func_descriptor != nullptr); - const jint hashcode = GetHashCodeOfJavaObject(env, java_func_descriptor); + const jint hashcode = + GetHashCodeOfJavaObject(converter_env, java_func_descriptor); ray::FunctionDescriptor native_func = - ToRayFunction(env, java_func_descriptor, hashcode) + ToRayFunction(converter_env, java_func_descriptor, hashcode) .GetFunctionDescriptor(); return native_func; }); // Put func_descriptors into this task group. const std::string concurrency_group_name = JavaStringToNativeString( - env, - (jstring)env->GetObjectField(java_concurrency_group_impl, - java_concurrency_group_impl_name)); - const uint32_t max_concurrency = env->GetIntField( + inner_env, + (jstring)inner_env->GetObjectField(java_concurrency_group_impl, + java_concurrency_group_impl_name)); + const uint32_t _max_concurrency = inner_env->GetIntField( java_concurrency_group_impl, java_concurrency_group_impl_max_concurrency); return ray::ConcurrencyGroup{ - concurrency_group_name, max_concurrency, native_func_descriptors}; + concurrency_group_name, _max_concurrency, native_func_descriptors}; }); auto java_serialized_runtime_env = (jstring)env->GetObjectField( actorCreationOptions, java_actor_creation_options_serialized_runtime_env); @@ -340,16 +341,17 @@ inline PlacementGroupCreationOptions ToPlacementGroupCreationOptions( placementGroupCreationOptions, java_placement_group_creation_options_bundles); std::vector> bundles; JavaListToNativeVector>( - env, java_bundles, &bundles, [](JNIEnv *env, jobject java_bundle) { + env, java_bundles, &bundles, [](JNIEnv *inner_env, jobject java_bundle) { return JavaMapToNativeMap( - env, + inner_env, java_bundle, - [](JNIEnv *env, jobject java_key) { - return JavaStringToNativeString(env, (jstring)java_key); + [](JNIEnv *key_env, jobject java_key) { + return JavaStringToNativeString(key_env, (jstring)java_key); }, - [](JNIEnv *env, jobject java_value) { - double value = env->CallDoubleMethod(java_value, java_double_double_value); - RAY_CHECK_JAVA_EXCEPTION(env); + [](JNIEnv *value_env, jobject java_value) { + double value = + value_env->CallDoubleMethod(java_value, java_double_double_value); + RAY_CHECK_JAVA_EXCEPTION(value_env); return value; }); }); diff --git a/src/ray/core_worker/object_recovery_manager.cc b/src/ray/core_worker/object_recovery_manager.cc index 238dd3634268..1fd5669589c9 100644 --- a/src/ray/core_worker/object_recovery_manager.cc +++ b/src/ray/core_worker/object_recovery_manager.cc @@ -71,8 +71,8 @@ bool ObjectRecoveryManager::RecoverObject(const ObjectID &object_id) { // gcs_client. object_lookup_( object_id, - [this](const ObjectID &object_id, std::vector locations) { - PinOrReconstructObject(object_id, std::move(locations)); + [this](const ObjectID &object_id_to_lookup, std::vector locations) { + PinOrReconstructObject(object_id_to_lookup, std::move(locations)); }); } else if (requires_recovery) { RAY_LOG(DEBUG).WithField(object_id) << "Recovery already started for object"; diff --git a/src/ray/core_worker/reference_count.cc b/src/ray/core_worker/reference_count.cc index 6f15095e3e24..6e328900a907 100644 --- a/src/ray/core_worker/reference_count.cc +++ b/src/ray/core_worker/reference_count.cc @@ -38,7 +38,7 @@ bool ReferenceCounter::OwnedByUs(const ObjectID &object_id) const { absl::MutexLock lock(&mutex_); auto it = object_id_refs_.find(object_id); if (it != object_id_refs_.end()) { - return it->second.owned_by_us; + return it->second.owned_by_us_; } return false; } @@ -102,12 +102,12 @@ bool ReferenceCounter::AddBorrowedObjectInternal(const ObjectID &object_id, } RAY_LOG(DEBUG) << "Adding borrowed object " << object_id; - it->second.owner_address = owner_address; + it->second.owner_address_ = owner_address; it->second.foreign_owner_already_monitoring |= foreign_owner_already_monitoring; if (!outer_id.IsNil()) { auto outer_it = object_id_refs_.find(outer_id); - if (outer_it != object_id_refs_.end() && !outer_it->second.owned_by_us) { + if (outer_it != object_id_refs_.end() && !outer_it->second.owned_by_us_) { RAY_LOG(DEBUG) << "Setting borrowed inner ID " << object_id << " contained_in_borrowed: " << outer_id; RAY_CHECK_NE(object_id, outer_id); @@ -143,18 +143,18 @@ void ReferenceCounter::AddObjectRefStats( auto ref_proto = stats->add_object_refs(); ref_proto->set_object_id(ref.first.Binary()); - ref_proto->set_call_site(ref.second.call_site); - ref_proto->set_object_size(ref.second.object_size); + ref_proto->set_call_site(ref.second.call_site_); + ref_proto->set_object_size(ref.second.object_size_); ref_proto->set_local_ref_count(ref.second.local_ref_count); ref_proto->set_submitted_task_ref_count(ref.second.submitted_task_ref_count); auto it = pinned_objects.find(ref.first); if (it != pinned_objects.end()) { ref_proto->set_pinned_in_memory(true); // If some info isn't available, fallback to getting it from the pinned info. - if (ref.second.object_size <= 0) { + if (ref.second.object_size_ <= 0) { ref_proto->set_object_size(it->second.first); } - if (ref.second.call_site.empty()) { + if (ref.second.call_site_.empty()) { ref_proto->set_call_site(it->second.second); } } @@ -162,7 +162,7 @@ void ReferenceCounter::AddObjectRefStats( ref_proto->add_contained_in_owned(obj_id.Binary()); } - if (ref.second.owned_by_us && !ref.second.pending_creation) { + if (ref.second.owned_by_us_ && !ref.second.pending_creation_) { // For finished tasks only, we set the status here instead of in the // TaskManager in case the task spec has already been GCed. ref_proto->set_task_status(rpc::TaskStatus::FINISHED); @@ -225,15 +225,15 @@ void ReferenceCounter::AddDynamicReturn(const ObjectID &object_id, } RAY_LOG(DEBUG) << "Adding dynamic return " << object_id << " contained in generator object " << generator_id; - RAY_CHECK(outer_it->second.owned_by_us); - RAY_CHECK(outer_it->second.owner_address.has_value()); - rpc::Address owner_address(outer_it->second.owner_address.value()); + RAY_CHECK(outer_it->second.owned_by_us_); + RAY_CHECK(outer_it->second.owner_address_.has_value()); + rpc::Address owner_address(outer_it->second.owner_address_.value()); RAY_UNUSED(AddOwnedObjectInternal(object_id, {}, owner_address, - outer_it->second.call_site, + outer_it->second.call_site_, /*object_size=*/-1, - outer_it->second.is_reconstructable, + outer_it->second.is_reconstructable_, /*add_local_ref=*/false, std::optional())); AddNestedObjectIdsInternal(generator_id, {object_id}, owner_address); @@ -258,17 +258,17 @@ void ReferenceCounter::OwnDynamicStreamingTaskReturnRef(const ObjectID &object_i } RAY_LOG(DEBUG) << "Adding dynamic return " << object_id << " contained in generator object " << generator_id; - RAY_CHECK(outer_it->second.owned_by_us); - RAY_CHECK(outer_it->second.owner_address.has_value()); - rpc::Address owner_address(outer_it->second.owner_address.value()); + RAY_CHECK(outer_it->second.owned_by_us_); + RAY_CHECK(outer_it->second.owner_address_.has_value()); + rpc::Address owner_address(outer_it->second.owner_address_.value()); // We add a local reference here. The ref removal will be handled // by the ObjectRefStream. RAY_UNUSED(AddOwnedObjectInternal(object_id, {}, owner_address, - outer_it->second.call_site, + outer_it->second.call_site_, /*object_size=*/-1, - outer_it->second.is_reconstructable, + outer_it->second.is_reconstructable_, /*add_local_ref=*/true, std::optional())); } @@ -370,7 +370,7 @@ void ReferenceCounter::UpdateObjectSize(const ObjectID &object_id, int64_t objec absl::MutexLock lock(&mutex_); auto it = object_id_refs_.find(object_id); if (it != object_id_refs_.end()) { - it->second.object_size = object_size; + it->second.object_size_ = object_size; PushToLocationSubscribers(it); } } @@ -529,16 +529,16 @@ void ReferenceCounter::UpdateFinishedTaskReferences( int64_t ReferenceCounter::ReleaseLineageReferences(ReferenceTable::iterator ref) { int64_t lineage_bytes_evicted = 0; std::vector argument_ids; - if (on_lineage_released_ && ref->second.owned_by_us) { + if (on_lineage_released_ && ref->second.owned_by_us_) { RAY_LOG(DEBUG) << "Releasing lineage for object " << ref->first; lineage_bytes_evicted += on_lineage_released_(ref->first, &argument_ids); // The object is still in scope by the application and it was // reconstructable with lineage. Mark that its lineage has been evicted so // we can return the right error during reconstruction. if (!ref->second.OutOfScope(lineage_pinning_enabled_) && - ref->second.is_reconstructable) { + ref->second.is_reconstructable_) { ref->second.lineage_evicted = true; - ref->second.is_reconstructable = false; + ref->second.is_reconstructable_ = false; } } @@ -609,8 +609,8 @@ bool ReferenceCounter::GetOwnerInternal(const ObjectID &object_id, return false; } - if (it->second.owner_address) { - *owner_address = *it->second.owner_address; + if (it->second.owner_address_) { + *owner_address = *it->second.owner_address_; return true; } else { return false; @@ -667,7 +667,7 @@ void ReferenceCounter::FreePlasmaObjects(const std::vector &object_ids // The object is still in scope. It will be removed from this set // once its Reference has been deleted. freed_objects_.insert(object_id); - if (!it->second.owned_by_us) { + if (!it->second.owned_by_us_) { RAY_LOG(WARNING) << "Tried to free an object " << object_id << " that we did not create. The object value may not be released."; @@ -697,7 +697,7 @@ void ReferenceCounter::DeleteReferenceInternal(ReferenceTable::iterator it, auto inner_it = object_id_refs_.find(inner_id); if (inner_it != object_id_refs_.end()) { RAY_LOG(DEBUG) << "Try to delete inner object " << inner_id; - if (it->second.owned_by_us) { + if (it->second.owned_by_us_) { // If this object ID was nested in an owned object, make sure that // the outer object counted towards the ref count for the inner // object. @@ -745,7 +745,7 @@ void ReferenceCounter::EraseReference(ReferenceTable::iterator it) { reconstructable_owned_objects_index_.erase(index_it); } freed_objects_.erase(it->first); - if (it->second.owned_by_us) { + if (it->second.owned_by_us_) { if (ObjectID::IsActorID(it->first)) { num_actors_owned_by_us_--; } else { @@ -787,7 +787,7 @@ void ReferenceCounter::OnObjectOutOfScopeOrFreed(ReferenceTable::iterator it) { } void ReferenceCounter::UnsetObjectPrimaryCopy(ReferenceTable::iterator it) { - it->second.pinned_at_node_id.reset(); + it->second.pinned_at_node_id_.reset(); if (it->second.spilled && !it->second.spilled_node_id.IsNil()) { it->second.spilled = false; it->second.spilled_url = ""; @@ -831,7 +831,7 @@ void ReferenceCounter::ResetObjectsOnRemovedNode(const NodeID &node_id) { absl::MutexLock lock(&mutex_); for (auto it = object_id_refs_.begin(); it != object_id_refs_.end(); it++) { const auto &object_id = it->first; - if (it->second.pinned_at_node_id.value_or(NodeID::Nil()) == node_id || + if (it->second.pinned_at_node_id_.value_or(NodeID::Nil()) == node_id || it->second.spilled_node_id == node_id) { UnsetObjectPrimaryCopy(it); if (!it->second.OutOfScope(lineage_pinning_enabled_)) { @@ -861,17 +861,17 @@ void ReferenceCounter::UpdateObjectPinnedAtRaylet(const ObjectID &object_id, // The object is still in scope. Track the raylet location until the object // has gone out of scope or the raylet fails, whichever happens first. - if (it->second.pinned_at_node_id.has_value()) { + if (it->second.pinned_at_node_id_.has_value()) { RAY_LOG(INFO).WithField(object_id) << "Updating primary location for object to node " << node_id - << ", but it already has a primary location " << *it->second.pinned_at_node_id + << ", but it already has a primary location " << *it->second.pinned_at_node_id_ << ". This should only happen during reconstruction"; } // Only the owner tracks the location. - RAY_CHECK(it->second.owned_by_us); + RAY_CHECK(it->second.owned_by_us_); if (!it->second.OutOfScope(lineage_pinning_enabled_)) { if (!is_node_dead_(node_id)) { - it->second.pinned_at_node_id = node_id; + it->second.pinned_at_node_id_ = node_id; } else { UnsetObjectPrimaryCopy(it); objects_to_recover_.push_back(object_id); @@ -887,10 +887,10 @@ bool ReferenceCounter::IsPlasmaObjectPinnedOrSpilled(const ObjectID &object_id, absl::MutexLock lock(&mutex_); auto it = object_id_refs_.find(object_id); if (it != object_id_refs_.end()) { - if (it->second.owned_by_us) { + if (it->second.owned_by_us_) { *owned_by_us = true; *spilled = it->second.spilled; - *pinned_at = it->second.pinned_at_node_id.value_or(NodeID::Nil()); + *pinned_at = it->second.pinned_at_node_id_.value_or(NodeID::Nil()); } return true; } @@ -996,7 +996,7 @@ bool ReferenceCounter::GetAndClearLocalBorrowersInternal( // because it is possible to receive a reference to an object that we already // own, e.g., if we execute a task that has an object ID in its arguments // that we created in an earlier task. - if (ref.owned_by_us) { + if (ref.owned_by_us_) { // Return true because we have the ref, but there is no need to return it // since we own the object. return true; @@ -1081,16 +1081,16 @@ void ReferenceCounter::MergeRemoteBorrowers(const ObjectID &object_id, // local table. for (const auto &contained_in_borrowed_id : borrower_it->second.nested().contained_in_borrowed_ids) { - RAY_CHECK(borrower_ref.owner_address); + RAY_CHECK(borrower_ref.owner_address_); AddBorrowedObjectInternal(object_id, contained_in_borrowed_id, - *borrower_ref.owner_address, + *borrower_ref.owner_address_, /*foreign_owner_already_monitoring=*/false); } // If we own this ID, then wait for all new borrowers to reach a ref count // of 0 before GCing the object value. - if (it->second.owned_by_us) { + if (it->second.owned_by_us_) { for (const auto &addr : new_borrowers) { WaitForRefRemoved(it, addr); } @@ -1139,10 +1139,10 @@ void ReferenceCounter::WaitForRefRemoved(const ReferenceTable::iterator &ref_it, auto sub_message = std::make_unique(); auto *request = sub_message->mutable_worker_ref_removed_message(); // Only the owner should send requests to borrowers. - RAY_CHECK(ref_it->second.owned_by_us); + RAY_CHECK(ref_it->second.owned_by_us_); request->mutable_reference()->set_object_id(object_id.Binary()); request->mutable_reference()->mutable_owner_address()->CopyFrom( - *ref_it->second.owner_address); + *ref_it->second.owner_address_); request->set_contained_in_id(contained_in_id.Binary()); request->set_intended_worker_id(addr.worker_id()); request->set_subscriber_worker_id(rpc_address_.worker_id()); @@ -1167,10 +1167,12 @@ void ReferenceCounter::WaitForRefRemoved(const ReferenceTable::iterator &ref_it, const Status &) { // When the request is failed, there's no new borrowers ref published from this // borrower. - const auto object_id = ObjectID::FromBinary(object_id_binary); - RAY_LOG(DEBUG).WithField(object_id).WithField(WorkerID::FromBinary(addr.worker_id())) + const auto failed_borrower_object_id = ObjectID::FromBinary(object_id_binary); + RAY_LOG(DEBUG) + .WithField(failed_borrower_object_id) + .WithField(WorkerID::FromBinary(addr.worker_id())) << "WaitForRefRemoved failed for object, dest worker"; - CleanupBorrowersOnRefRemoved({}, object_id, addr); + CleanupBorrowersOnRefRemoved({}, failed_borrower_object_id, addr); }; RAY_CHECK( @@ -1199,7 +1201,7 @@ void ReferenceCounter::AddNestedObjectIdsInternal(const ObjectID &object_id, // We own object_id. This is a `ray.put()` case OR returning an object ID // from a task and the task's caller executed in the same process as us. if (it != object_id_refs_.end()) { - RAY_CHECK(it->second.owned_by_us); + RAY_CHECK(it->second.owned_by_us_); // The outer object is still in scope. Mark the inner ones as being // contained in the outer object ID so we do not GC the inner objects // until the outer object goes out of scope. @@ -1232,7 +1234,7 @@ void ReferenceCounter::AddNestedObjectIdsInternal(const ObjectID &object_id, inner_it = object_id_refs_.emplace(inner_id, Reference()).first; } // Add the task's caller as a borrower. - if (inner_it->second.owned_by_us) { + if (inner_it->second.owned_by_us_) { auto inserted = inner_it->second.mutable_borrow()->borrowers.insert(owner_address).second; if (inserted) { @@ -1389,8 +1391,8 @@ void ReferenceCounter::UpdateObjectPendingCreationInternal(const ObjectID &objec auto it = object_id_refs_.find(object_id); bool push = false; if (it != object_id_refs_.end()) { - push = (it->second.pending_creation != pending_creation); - it->second.pending_creation = pending_creation; + push = (it->second.pending_creation_ != pending_creation); + it->second.pending_creation_ = pending_creation; } if (push) { PushToLocationSubscribers(it); @@ -1462,11 +1464,11 @@ std::optional ReferenceCounter::GetLocalityData( } // The size of this object. - const auto object_size = it->second.object_size; + const auto object_size = it->second.object_size_; if (object_size < 0) { // We don't know the object size so we can't returned valid locality data. RAY_LOG(DEBUG).WithField(object_id) - << "Reference [" << it->second.call_site + << "Reference [" << it->second.call_site_ << "] for object has an unknown object size, locality data not available"; return absl::nullopt; } @@ -1479,8 +1481,8 @@ std::optional ReferenceCounter::GetLocalityData( auto node_ids = it->second.locations; // Add location of the primary copy since the object must be there: either in memory or // spilled. - if (it->second.pinned_at_node_id.has_value()) { - node_ids.emplace(it->second.pinned_at_node_id.value()); + if (it->second.pinned_at_node_id_.has_value()) { + node_ids.emplace(it->second.pinned_at_node_id_.value()); } // We should only reach here if we have valid locality data to return. @@ -1500,13 +1502,13 @@ bool ReferenceCounter::ReportLocalityData(const ObjectID &object_id, << " The object has probably already been freed."; return false; } - RAY_CHECK(!it->second.owned_by_us) + RAY_CHECK(!it->second.owned_by_us_) << "ReportLocalityData should only be used for borrowed references."; for (const auto &location : locations) { it->second.locations.emplace(location); } if (object_size > 0) { - it->second.object_size = object_size; + it->second.object_size_ = object_size; } return true; } @@ -1517,7 +1519,7 @@ void ReferenceCounter::AddBorrowerAddress(const ObjectID &object_id, auto it = object_id_refs_.find(object_id); RAY_CHECK(it != object_id_refs_.end()); - RAY_CHECK(it->second.owned_by_us) + RAY_CHECK(it->second.owned_by_us_) << "AddBorrowerAddress should only be used for owner references."; RAY_CHECK(borrower_address.worker_id() != rpc_address_.worker_id()) @@ -1542,7 +1544,7 @@ bool ReferenceCounter::IsObjectReconstructable(const ObjectID &object_id, return false; } *lineage_evicted = it->second.lineage_evicted; - return it->second.is_reconstructable; + return it->second.is_reconstructable_; } void ReferenceCounter::UpdateObjectPendingCreation(const ObjectID &object_id, @@ -1557,23 +1559,23 @@ bool ReferenceCounter::IsObjectPendingCreation(const ObjectID &object_id) const if (it == object_id_refs_.end()) { return false; } - return it->second.pending_creation; + return it->second.pending_creation_; } void ReferenceCounter::PushToLocationSubscribers(ReferenceTable::iterator it) { const auto &object_id = it->first; const auto &locations = it->second.locations; - auto object_size = it->second.object_size; + auto object_size = it->second.object_size_; const auto &spilled_url = it->second.spilled_url; const auto &spilled_node_id = it->second.spilled_node_id; - const auto &optional_primary_node_id = it->second.pinned_at_node_id; + const auto &optional_primary_node_id = it->second.pinned_at_node_id_; const auto &primary_node_id = optional_primary_node_id.value_or(NodeID::Nil()); RAY_LOG(DEBUG).WithField(object_id) << "Published message for object, " << locations.size() << " locations, spilled url: [" << spilled_url << "], spilled node ID: " << spilled_node_id << ", and object size: " << object_size << ", and primary node ID: " << primary_node_id << ", pending creation? " - << it->second.pending_creation; + << it->second.pending_creation_; rpc::PubMessage pub_message; pub_message.set_key_id(object_id.Binary()); pub_message.set_channel_type(rpc::ChannelType::WORKER_OBJECT_LOCATIONS_CHANNEL); @@ -1604,15 +1606,15 @@ void ReferenceCounter::FillObjectInformationInternal( for (const auto &node_id : it->second.locations) { object_info->add_node_ids(node_id.Binary()); } - int64_t object_size = it->second.object_size; + int64_t object_size = it->second.object_size_; if (object_size > 0) { - object_info->set_object_size(it->second.object_size); + object_info->set_object_size(it->second.object_size_); } object_info->set_spilled_url(it->second.spilled_url); object_info->set_spilled_node_id(it->second.spilled_node_id.Binary()); - auto primary_node_id = it->second.pinned_at_node_id.value_or(NodeID::Nil()); + auto primary_node_id = it->second.pinned_at_node_id_.value_or(NodeID::Nil()); object_info->set_primary_node_id(primary_node_id.Binary()); - object_info->set_pending_creation(it->second.pending_creation); + object_info->set_pending_creation(it->second.pending_creation_); object_info->set_did_spill(it->second.did_spill); } @@ -1670,7 +1672,7 @@ std::string ReferenceCounter::Reference::DebugString() const { ReferenceCounter::Reference ReferenceCounter::Reference::FromProto( const rpc::ObjectReferenceCount &ref_count) { Reference ref; - ref.owner_address = ref_count.reference().owner_address(); + ref.owner_address_ = ref_count.reference().owner_address(); ref.local_ref_count = ref_count.has_local_ref() ? 1 : 0; for (const auto &borrower : ref_count.borrowers()) { @@ -1692,8 +1694,8 @@ ReferenceCounter::Reference ReferenceCounter::Reference::FromProto( void ReferenceCounter::Reference::ToProto(rpc::ObjectReferenceCount *ref, bool deduct_local_ref) const { - if (owner_address) { - ref->mutable_reference()->mutable_owner_address()->CopyFrom(*owner_address); + if (owner_address_) { + ref->mutable_reference()->mutable_owner_address()->CopyFrom(*owner_address_); } ref->set_has_local_ref(RefCount() > (deduct_local_ref ? 1 : 0)); for (const auto &borrower : borrow().borrowers) { @@ -1719,7 +1721,7 @@ std::optional ReferenceCounter::GetTensorTransport( if (it == object_id_refs_.end()) { return absl::nullopt; } - return it->second.tensor_transport; + return it->second.tensor_transport_; } } // namespace core diff --git a/src/ray/core_worker/reference_count.h b/src/ray/core_worker/reference_count.h index 4214ded8d24c..d168f749e42f 100644 --- a/src/ray/core_worker/reference_count.h +++ b/src/ray/core_worker/reference_count.h @@ -646,7 +646,7 @@ class ReferenceCounter : public ReferenceCounterInterface, /// Constructor for a reference whose origin is unknown. Reference() = default; Reference(std::string call_site, int64_t object_size) - : call_site(std::move(call_site)), object_size(object_size) {} + : call_site_(std::move(call_site)), object_size_(object_size) {} /// Constructor for a reference that we created. Reference(rpc::Address owner_address, std::string call_site, @@ -654,14 +654,14 @@ class ReferenceCounter : public ReferenceCounterInterface, bool is_reconstructable, std::optional pinned_at_node_id, rpc::TensorTransport tensor_transport) - : call_site(std::move(call_site)), - object_size(object_size), - owner_address(std::move(owner_address)), - pinned_at_node_id(std::move(pinned_at_node_id)), - tensor_transport(tensor_transport), - owned_by_us(true), - is_reconstructable(is_reconstructable), - pending_creation(!pinned_at_node_id.has_value()) {} + : call_site_(std::move(call_site)), + object_size_(object_size), + owner_address_(std::move(owner_address)), + pinned_at_node_id_(std::move(pinned_at_node_id)), + tensor_transport_(tensor_transport), + owned_by_us_(true), + is_reconstructable_(is_reconstructable), + pending_creation_(!pinned_at_node_id_.has_value()) {} /// Constructor from a protobuf. This is assumed to be a message from /// another process, so the object defaults to not being owned by us. @@ -694,7 +694,7 @@ class ReferenceCounter : public ReferenceCounterInterface, bool was_stored_in_objects = !borrow().stored_in_objects.empty(); bool has_lineage_references = false; - if (lineage_pinning_enabled && owned_by_us && !is_reconstructable) { + if (lineage_pinning_enabled && owned_by_us_ && !is_reconstructable_) { has_lineage_references = lineage_ref_count > 0; } @@ -756,9 +756,9 @@ class ReferenceCounter : public ReferenceCounterInterface, std::string DebugString() const; /// Description of the call site where the reference was created. - std::string call_site = ""; + std::string call_site_ = ""; /// Object size if known, otherwise -1; - int64_t object_size = -1; + int64_t object_size_ = -1; /// If this object is owned by us and stored in plasma, this contains all /// object locations. absl::flat_hash_set locations; @@ -766,25 +766,25 @@ class ReferenceCounter : public ReferenceCounterInterface, /// owner, then this is added during creation of the Reference. If this is /// process is a borrower, the borrower must add the owner's address before /// using the ObjectID. - std::optional owner_address; + std::optional owner_address_; /// If this object is owned by us and stored in plasma, and reference /// counting is enabled, then some raylet must be pinning the object value. /// This is the address of that raylet. - std::optional pinned_at_node_id; + std::optional pinned_at_node_id_; /// TODO(kevin85421): Make tensor_transport a required field for all constructors. /// /// The transport used for the object. - rpc::TensorTransport tensor_transport = rpc::TensorTransport::OBJECT_STORE; + rpc::TensorTransport tensor_transport_ = rpc::TensorTransport::OBJECT_STORE; /// Whether we own the object. If we own the object, then we are /// responsible for tracking the state of the task that creates the object /// (see task_manager.h). - bool owned_by_us = false; + bool owned_by_us_ = false; // Whether this object can be reconstructed via lineage. If false, then the // object's value will be pinned as long as it is referenced by any other // object's lineage. This should be set to false if the object was created // by ray.put(), a task that cannot be retried, or its lineage was evicted. - bool is_reconstructable = false; + bool is_reconstructable_ = false; /// Whether the lineage of this object was evicted due to memory pressure. bool lineage_evicted = false; /// The number of tasks that depend on this object that may be retried in @@ -843,7 +843,7 @@ class ReferenceCounter : public ReferenceCounterInterface, bool has_nested_refs_to_report = false; /// Whether the task that creates this object is scheduled/executing. - bool pending_creation = false; + bool pending_creation_ = false; /// Whether or not this object was spilled. bool did_spill = false; diff --git a/src/ray/core_worker/task_execution/actor_scheduling_queue.cc b/src/ray/core_worker/task_execution/actor_scheduling_queue.cc index ebd201a4481b..9eee6f987a8a 100644 --- a/src/ray/core_worker/task_execution/actor_scheduling_queue.cc +++ b/src/ray/core_worker/task_execution/actor_scheduling_queue.cc @@ -104,29 +104,29 @@ void ActorSchedulingQueue::Add( rpc::TaskStatus::PENDING_ACTOR_TASK_ARGS_FETCH, /* include_task_info */ false)); waiter_.Wait(dependencies, [this, seq_no, is_retry, retry_request]() mutable { - InboundRequest *inbound_request = nullptr; + InboundRequest *inbound_req = nullptr; if (is_retry) { // retry_request is guaranteed to be a valid pointer for retries because it // won't be erased from the retry list until its dependencies are fetched and // ExecuteRequest happens. - inbound_request = retry_request; + inbound_req = retry_request; } else if (auto it = pending_actor_tasks_.find(seq_no); it != pending_actor_tasks_.end()) { // For non-retry tasks, we need to check if the task is still in the map because // it can be erased due to being canceled via a higher `client_processed_up_to_`. - inbound_request = &it->second; + inbound_req = &it->second; } - if (inbound_request != nullptr) { - const auto &task_spec = inbound_request->TaskSpec(); + if (inbound_req != nullptr) { + const auto &inbound_req_task_spec = inbound_req->TaskSpec(); RAY_UNUSED(task_event_buffer_.RecordTaskStatusEventIfNeeded( - task_spec.TaskId(), - task_spec.JobId(), - task_spec.AttemptNumber(), - task_spec, + inbound_req_task_spec.TaskId(), + inbound_req_task_spec.JobId(), + inbound_req_task_spec.AttemptNumber(), + inbound_req_task_spec, rpc::TaskStatus::PENDING_ACTOR_TASK_ORDERING_OR_CONCURRENCY, /* include_task_info */ false)); - inbound_request->MarkDependenciesResolved(); + inbound_req->MarkDependenciesResolved(); ScheduleRequests(); } }); diff --git a/src/ray/core_worker/task_execution/concurrency_group_manager.cc b/src/ray/core_worker/task_execution/concurrency_group_manager.cc index b1eaf375637a..12f0dfe63232 100644 --- a/src/ray/core_worker/task_execution/concurrency_group_manager.cc +++ b/src/ray/core_worker/task_execution/concurrency_group_manager.cc @@ -33,11 +33,11 @@ ConcurrencyGroupManager::ConcurrencyGroupManager( std::function()> initialize_thread_callback) : initialize_thread_callback_(std::move(initialize_thread_callback)) { for (auto &group : concurrency_groups) { - const auto name = group.name; - const auto max_concurrency = group.max_concurrency; + const auto name = group.name_; + const auto max_concurrency = group.max_concurrency_; auto executor = std::make_shared(max_concurrency, initialize_thread_callback_); - auto &fds = group.function_descriptors; + auto &fds = group.function_descriptors_; for (auto fd : fds) { functions_to_executor_index_[fd->ToString()] = executor; } diff --git a/src/ray/core_worker/task_execution/out_of_order_actor_scheduling_queue.cc b/src/ray/core_worker/task_execution/out_of_order_actor_scheduling_queue.cc index 7c6740b1898c..f68843ad6829 100644 --- a/src/ray/core_worker/task_execution/out_of_order_actor_scheduling_queue.cc +++ b/src/ray/core_worker/task_execution/out_of_order_actor_scheduling_queue.cc @@ -45,7 +45,8 @@ OutOfOrderActorSchedulingQueue::OutOfOrderActorSchedulingQueue( ss << "Setting actor as asyncio with max_concurrency=" << fiber_max_concurrency << ", and defined concurrency groups are:" << std::endl; for (const auto &concurrency_group : concurrency_groups) { - ss << "\t" << concurrency_group.name << " : " << concurrency_group.max_concurrency; + ss << "\t" << concurrency_group.name_ << " : " + << concurrency_group.max_concurrency_; } RAY_LOG(INFO) << ss.str(); } @@ -188,12 +189,12 @@ void OutOfOrderActorSchedulingQueue::RunRequest(InboundRequest request) { waiter_.Wait(dependencies, [this, request = std::move(request)]() mutable { RAY_CHECK_EQ(std::this_thread::get_id(), main_thread_id_); - const TaskSpecification &task_spec = request.TaskSpec(); + const TaskSpecification &task = request.TaskSpec(); RAY_UNUSED(task_event_buffer_.RecordTaskStatusEventIfNeeded( - task_spec.TaskId(), - task_spec.JobId(), - task_spec.AttemptNumber(), - task_spec, + task.TaskId(), + task.JobId(), + task.AttemptNumber(), + task, rpc::TaskStatus::PENDING_ACTOR_TASK_ORDERING_OR_CONCURRENCY, /* include_task_info */ false)); diff --git a/src/ray/core_worker/task_execution/task_receiver.cc b/src/ray/core_worker/task_execution/task_receiver.cc index 18236a0c72a4..759ce1ee2e28 100644 --- a/src/ray/core_worker/task_execution/task_receiver.cc +++ b/src/ray/core_worker/task_execution/task_receiver.cc @@ -51,9 +51,10 @@ void TaskReceiver::HandleTask(rpc::PushTaskRequest request, } auto accept_callback = [this, reply, resource_ids = std::move(resource_ids)]( - const TaskSpecification &task_spec, - const rpc::SendReplyCallback &send_reply_callback) mutable { - auto num_returns = task_spec.NumReturns(); + const TaskSpecification &accepted_task_spec, + const rpc::SendReplyCallback + &accepted_send_reply_callback) mutable { + auto num_returns = accepted_task_spec.NumReturns(); RAY_CHECK(num_returns >= 0); std::vector>> return_objects; @@ -61,7 +62,7 @@ void TaskReceiver::HandleTask(rpc::PushTaskRequest request, std::vector> streaming_generator_returns; bool is_retryable_error = false; std::string application_error; - auto status = task_handler_(task_spec, + auto status = task_handler_(accepted_task_spec, std::move(resource_ids), &return_objects, &dynamic_return_objects, @@ -108,8 +109,9 @@ void TaskReceiver::HandleTask(rpc::PushTaskRequest request, } if (objects_valid) { - if (task_spec.ReturnsDynamic()) { - size_t num_dynamic_returns_expected = task_spec.DynamicReturnIds().size(); + if (accepted_task_spec.ReturnsDynamic()) { + size_t num_dynamic_returns_expected = + accepted_task_spec.DynamicReturnIds().size(); if (num_dynamic_returns_expected > 0) { RAY_CHECK(dynamic_return_objects.size() == num_dynamic_returns_expected) << "Expected " << num_dynamic_returns_expected @@ -132,15 +134,15 @@ void TaskReceiver::HandleTask(rpc::PushTaskRequest request, return_object.first, return_object.second, return_object_proto); } - if (task_spec.IsActorCreationTask()) { - concurrency_groups_ = task_spec.ConcurrencyGroups(); + if (accepted_task_spec.IsActorCreationTask()) { + concurrency_groups_ = accepted_task_spec.ConcurrencyGroups(); if (is_asyncio_) { fiber_state_manager_ = std::make_shared>( concurrency_groups_, fiber_max_concurrency_, initialize_thread_callback_); } else { // If the actor is an asyncio actor, then this concurrency group manager // for BoundedExecutor will never be used, so we don't need to initialize it. - const int default_max_concurrency = task_spec.MaxActorConcurrency(); + const int default_max_concurrency = accepted_task_spec.MaxActorConcurrency(); pool_manager_ = std::make_shared>( concurrency_groups_, default_max_concurrency, initialize_thread_callback_); } @@ -151,16 +153,17 @@ void TaskReceiver::HandleTask(rpc::PushTaskRequest request, RAY_CHECK_OK(actor_creation_task_done_()); if (status.IsCreationTaskError()) { RAY_LOG(WARNING) << "Actor creation task finished with errors, task_id: " - << task_spec.TaskId() - << ", actor_id: " << task_spec.ActorCreationId() + << accepted_task_spec.TaskId() + << ", actor_id: " << accepted_task_spec.ActorCreationId() << ", status: " << status; } else { // Set the actor repr name if it's customized by the actor. if (!actor_repr_name_.empty()) { reply->set_actor_repr_name(actor_repr_name_); } - RAY_LOG(INFO) << "Actor creation task finished, task_id: " << task_spec.TaskId() - << ", actor_id: " << task_spec.ActorCreationId() + RAY_LOG(INFO) << "Actor creation task finished, task_id: " + << accepted_task_spec.TaskId() + << ", actor_id: " << accepted_task_spec.ActorCreationId() << ", actor_repr_name: " << actor_repr_name_; } } @@ -172,28 +175,29 @@ void TaskReceiver::HandleTask(rpc::PushTaskRequest request, reply->set_worker_exiting(true); if (objects_valid) { // This happens when max_calls is hit. We still need to return the objects. - send_reply_callback(Status::OK(), nullptr, nullptr); + accepted_send_reply_callback(Status::OK(), nullptr, nullptr); } else { - send_reply_callback(status, nullptr, nullptr); + accepted_send_reply_callback(status, nullptr, nullptr); } } else { RAY_CHECK_OK(status); RAY_CHECK(objects_valid); - send_reply_callback(Status::OK(), nullptr, nullptr); + accepted_send_reply_callback(Status::OK(), nullptr, nullptr); } }; - auto cancel_callback = [reply](const TaskSpecification &task_spec, - const Status &status, - const rpc::SendReplyCallback &send_reply_callback) { - if (task_spec.IsActorTask()) { + auto cancel_callback = [reply]( + const TaskSpecification &canceled_task_spec, + const Status &status, + const rpc::SendReplyCallback &canceled_send_reply_callback) { + if (canceled_task_spec.IsActorTask()) { // We consider cancellation of actor tasks to be a push task RPC failure. - send_reply_callback(status, nullptr, nullptr); + canceled_send_reply_callback(status, nullptr, nullptr); } else { // We consider cancellation of normal tasks to be an in-band cancellation of a // successful RPC. reply->set_was_cancelled_before_running(true); - send_reply_callback(status, nullptr, nullptr); + canceled_send_reply_callback(status, nullptr, nullptr); } }; diff --git a/src/ray/core_worker/task_manager.cc b/src/ray/core_worker/task_manager.cc index 0992f15ff2f9..d70717bc49f7 100644 --- a/src/ray/core_worker/task_manager.cc +++ b/src/ray/core_worker/task_manager.cc @@ -273,17 +273,17 @@ std::vector TaskManager::AddPendingTask( return_ids.push_back(return_id); rpc::ObjectReference ref; - auto object_id = spec.ReturnId(i); - ref.set_object_id(object_id.Binary()); + auto return_object_id = spec.ReturnId(i); + ref.set_object_id(return_object_id.Binary()); ref.mutable_owner_address()->CopyFrom(caller_address); ref.set_call_site(call_site); // Register the callback to free the GPU object when it is out of scope. - auto tensor_transport = reference_counter_.GetTensorTransport(object_id); + auto tensor_transport = reference_counter_.GetTensorTransport(return_object_id); if (tensor_transport.value_or(rpc::TensorTransport::OBJECT_STORE) != rpc::TensorTransport::OBJECT_STORE) { reference_counter_.AddObjectOutOfScopeOrFreedCallback( - object_id, [this](const ObjectID &object_id) { + return_object_id, [this](const ObjectID &object_id) { auto actor_id = ObjectID::ToActorID(object_id); auto rpc_client = get_actor_rpc_client_callback_(actor_id); auto request = rpc::FreeActorObjectRequest(); @@ -351,13 +351,13 @@ std::optional TaskManager::ResubmitTask( return rpc::ErrorType::OBJECT_UNRECONSTRUCTABLE_MAX_ATTEMPTS_EXCEEDED; } auto &task_entry = it->second; - if (task_entry.is_canceled) { + if (task_entry.is_canceled_) { return rpc::ErrorType::TASK_CANCELLED; } - if (task_entry.spec.IsStreamingGenerator() && + if (task_entry.spec_.IsStreamingGenerator() && task_entry.GetStatus() == rpc::TaskStatus::SUBMITTED_TO_WORKER) { - if (task_entry.num_retries_left == 0) { + if (task_entry.num_retries_left_ == 0) { // If the last attempt is in progress. return rpc::ErrorType::OBJECT_UNRECONSTRUCTABLE_MAX_ATTEMPTS_EXCEEDED; } @@ -374,7 +374,7 @@ std::optional TaskManager::ResubmitTask( SetupTaskEntryForResubmit(task_entry); } - spec = task_entry.spec; + spec = task_entry.spec_; } if (should_queue_generator_resubmit) { @@ -405,19 +405,19 @@ void TaskManager::SetupTaskEntryForResubmit(TaskEntry &task_entry) { rpc::TaskStatus::PENDING_ARGS_AVAIL, /* state_update */ std::nullopt, /* include_task_info */ true, - task_entry.spec.AttemptNumber() + 1); + task_entry.spec_.AttemptNumber() + 1); num_pending_tasks_++; // The task is pending again, so it's no longer counted as lineage. If // the task finishes and we still need the spec, we'll add the task back // to the footprint sum. - total_lineage_footprint_bytes_ -= task_entry.lineage_footprint_bytes; - task_entry.lineage_footprint_bytes = 0; + total_lineage_footprint_bytes_ -= task_entry.lineage_footprint_bytes_; + task_entry.lineage_footprint_bytes_ = 0; - if (task_entry.num_retries_left > 0) { - task_entry.num_retries_left--; + if (task_entry.num_retries_left_ > 0) { + task_entry.num_retries_left_--; } else { - RAY_CHECK(task_entry.num_retries_left == -1); + RAY_CHECK(task_entry.num_retries_left_ == -1); } } @@ -472,7 +472,7 @@ void TaskManager::MarkGeneratorFailedAndResubmit(const TaskID &task_id) { worker::TaskStatusEvent::TaskStateUpdate(error_info)); SetupTaskEntryForResubmit(task_entry); - spec = task_entry.spec; + spec = task_entry.spec_; } // Note: Don't need to call UpdateReferencesForResubmit because CompletePendingTask or @@ -619,7 +619,7 @@ Status TaskManager::TryReadObjectRefStream(const ObjectID &generator_id, absl::MutexLock lock(&mu_); auto it = submissible_tasks_.find(generator_id.TaskId()); if (it != submissible_tasks_.end()) { - backpressure_threshold = it->second.spec.GeneratorBackpressureNumObjects(); + backpressure_threshold = it->second.spec_.GeneratorBackpressureNumObjects(); } } @@ -770,8 +770,8 @@ bool TaskManager::HandleReportGeneratorItemReturns( absl::MutexLock lock(&mu_); auto it = submissible_tasks_.find(task_id); if (it != submissible_tasks_.end()) { - backpressure_threshold = it->second.spec.GeneratorBackpressureNumObjects(); - if (it->second.spec.AttemptNumber() > attempt_number) { + backpressure_threshold = it->second.spec_.GeneratorBackpressureNumObjects(); + if (it->second.spec_.AttemptNumber() > attempt_number) { // Generator task reports can arrive at any time. If the first attempt // fails, we may receive a report from the first executor after the // second attempt has started. In this case, we should ignore the first @@ -929,7 +929,7 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, auto it = submissible_tasks_.find(task_id); RAY_CHECK(it != submissible_tasks_.end()) << "Tried to complete task that was not pending " << task_id; - spec = it->second.spec; + spec = it->second.spec_; // Record any dynamically returned objects. We need to store these with the // task spec so that the worker will recreate them if the task gets @@ -942,7 +942,7 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, spec.AddDynamicReturnId(dynamic_return_id); } for (const auto &dynamic_return_id : dynamic_returns_in_plasma) { - it->second.reconstructable_return_ids.insert(dynamic_return_id); + it->second.reconstructable_return_ids_.insert(dynamic_return_id); } if (spec.IsStreamingGenerator()) { @@ -962,7 +962,7 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, // cause a memory leak of the task metadata, because we will // never receive a callback from the ReferenceCounter to erase // the task. - it->second.reconstructable_return_ids.insert( + it->second.reconstructable_return_ids_.insert( ObjectID::FromBinary(return_id_info.object_id())); } } @@ -975,14 +975,14 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, for (const auto &direct_return_id : direct_return_ids) { RAY_LOG(DEBUG) << "Task " << it->first << " returned direct object " << direct_return_id << ", now has " - << it->second.reconstructable_return_ids.size() + << it->second.reconstructable_return_ids_.size() << " plasma returns in scope"; - it->second.reconstructable_return_ids.erase(direct_return_id); + it->second.reconstructable_return_ids_.erase(direct_return_id); } RAY_LOG(DEBUG) << "Task " << it->first << " now has " - << it->second.reconstructable_return_ids.size() + << it->second.reconstructable_return_ids_.size() << " plasma returns in scope"; - it->second.num_successful_executions++; + it->second.num_successful_executions_++; if (is_application_error) { SetTaskStatus( @@ -998,13 +998,13 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, // A finished task can only be re-executed if it has some number of // retries left and returned at least one object that is still in use and // stored in plasma. - bool task_retryable = it->second.num_retries_left != 0 && - !it->second.reconstructable_return_ids.empty(); + bool task_retryable = it->second.num_retries_left_ != 0 && + !it->second.reconstructable_return_ids_.empty(); if (task_retryable) { // Pin the task spec if it may be retried again. release_lineage = false; - it->second.lineage_footprint_bytes = it->second.spec.GetMessage().ByteSizeLong(); - total_lineage_footprint_bytes_ += it->second.lineage_footprint_bytes; + it->second.lineage_footprint_bytes_ = it->second.spec_.GetMessage().ByteSizeLong(); + total_lineage_footprint_bytes_ += it->second.lineage_footprint_bytes_; if (total_lineage_footprint_bytes_ > max_lineage_bytes_) { RAY_LOG(INFO) << "Total lineage size is " << total_lineage_footprint_bytes_ / 1e6 << "MB, which exceeds the limit of " << max_lineage_bytes_ / 1e6 @@ -1074,13 +1074,13 @@ bool TaskManager::RetryTaskIfPossible(const TaskID &task_id, auto &task_entry = it->second; RAY_CHECK(task_entry.IsPending()) << "Tried to retry task that was not pending " << task_id; - spec = task_entry.spec; - num_retries_left = task_entry.num_retries_left; - num_oom_retries_left = task_entry.num_oom_retries_left; + spec = task_entry.spec_; + num_retries_left = task_entry.num_retries_left_; + num_oom_retries_left = task_entry.num_oom_retries_left_; if (task_failed_due_to_oom) { if (num_oom_retries_left > 0) { will_retry = true; - task_entry.num_oom_retries_left--; + task_entry.num_oom_retries_left_--; } else if (num_oom_retries_left == -1) { will_retry = true; } else { @@ -1095,13 +1095,13 @@ bool TaskManager::RetryTaskIfPossible(const TaskID &task_id, node_info->death_info().reason() == rpc::NodeDeathInfo::AUTOSCALER_DRAIN_PREEMPTED; } - if (num_retries_left > 0 || (is_preempted && task_entry.spec.IsRetriable())) { + if (num_retries_left > 0 || (is_preempted && task_entry.spec_.IsRetriable())) { will_retry = true; if (is_preempted) { RAY_LOG(INFO) << "Task " << task_id << " failed due to node preemption on node " << task_entry.GetNodeId() << ", not counting against retries"; } else { - task_entry.num_retries_left--; + task_entry.num_retries_left_--; } } else if (num_retries_left == -1) { will_retry = true; @@ -1110,8 +1110,8 @@ bool TaskManager::RetryTaskIfPossible(const TaskID &task_id, } } // Keep `num_retries_left` and `num_oom_retries_left` up to date - num_retries_left = task_entry.num_retries_left; - num_oom_retries_left = task_entry.num_oom_retries_left; + num_retries_left = task_entry.num_retries_left_; + num_oom_retries_left = task_entry.num_oom_retries_left_; if (will_retry) { // Record the old attempt status as FAILED. @@ -1125,7 +1125,7 @@ bool TaskManager::RetryTaskIfPossible(const TaskID &task_id, rpc::TaskStatus::PENDING_ARGS_AVAIL, /* state_update */ std::nullopt, /* include_task_info */ true, - task_entry.spec.AttemptNumber() + 1); + task_entry.spec_.AttemptNumber() + 1); } } @@ -1185,8 +1185,8 @@ void TaskManager::FailPendingTask(const TaskID &task_id, } RAY_CHECK(it->second.IsPending()) << "Tried to fail task that was not pending " << task_id; - spec = it->second.spec; - if (it->second.is_canceled && error_type != rpc::ErrorType::TASK_CANCELLED) { + spec = it->second.spec_; + if (it->second.is_canceled_ && error_type != rpc::ErrorType::TASK_CANCELLED) { // If the task is marked as cancelled before reaching FailPendingTask (which is // essentially the final state of the task lifecycle), that failure reason takes // precedence. @@ -1352,36 +1352,36 @@ int64_t TaskManager::RemoveLineageReference(const ObjectID &object_id, } RAY_LOG(DEBUG) << "Plasma object " << object_id << " out of scope"; - for (const auto &plasma_id : it->second.reconstructable_return_ids) { + for (const auto &plasma_id : it->second.reconstructable_return_ids_) { RAY_LOG(DEBUG) << "Task " << task_id << " has " << plasma_id << " in scope"; } - it->second.reconstructable_return_ids.erase(object_id); + it->second.reconstructable_return_ids_.erase(object_id); RAY_LOG(DEBUG) << "Task " << task_id << " now has " - << it->second.reconstructable_return_ids.size() + << it->second.reconstructable_return_ids_.size() << " plasma returns in scope"; - if (it->second.reconstructable_return_ids.empty() && !it->second.IsPending()) { + if (it->second.reconstructable_return_ids_.empty() && !it->second.IsPending()) { // If the task can no longer be retried, decrement the lineage ref count // for each of the task's args. - for (size_t i = 0; i < it->second.spec.NumArgs(); i++) { - if (it->second.spec.ArgByRef(i)) { - released_objects->push_back(it->second.spec.ArgObjectId(i)); + for (size_t i = 0; i < it->second.spec_.NumArgs(); i++) { + if (it->second.spec_.ArgByRef(i)) { + released_objects->push_back(it->second.spec_.ArgObjectId(i)); } else { - const auto &inlined_refs = it->second.spec.ArgInlinedRefs(i); + const auto &inlined_refs = it->second.spec_.ArgInlinedRefs(i); for (const auto &inlined_ref : inlined_refs) { released_objects->push_back(ObjectID::FromBinary(inlined_ref.object_id())); } } } - if (it->second.spec.IsActorTask()) { + if (it->second.spec_.IsActorTask()) { // We need to decrement the actor lineage ref count here // since it's incremented during TaskManager::AddPendingTask. - const auto actor_creation_return_id = it->second.spec.ActorCreationDummyObjectId(); + const auto actor_creation_return_id = it->second.spec_.ActorCreationDummyObjectId(); released_objects->push_back(actor_creation_return_id); } - total_lineage_footprint_bytes_ -= it->second.lineage_footprint_bytes; + total_lineage_footprint_bytes_ -= it->second.lineage_footprint_bytes_; // The task has finished and none of the return IDs are in scope anymore, // so it is safe to remove the task spec. submissible_tasks_.erase(it); @@ -1404,10 +1404,10 @@ void TaskManager::MarkTaskNoRetryInternal(const TaskID &task_id, bool canceled) absl::MutexLock lock(&mu_); auto it = submissible_tasks_.find(task_id); if (it != submissible_tasks_.end()) { - it->second.num_retries_left = 0; - it->second.num_oom_retries_left = 0; + it->second.num_retries_left_ = 0; + it->second.num_oom_retries_left_ = 0; if (canceled) { - it->second.is_canceled = true; + it->second.is_canceled_ = true; } } } @@ -1432,9 +1432,9 @@ absl::flat_hash_set TaskManager::GetTaskReturnObjectsToStoreInPlasma( // from submissible_tasks_. Do nothing in this case. return {}; } - first_execution = it->second.num_successful_executions == 0; + first_execution = it->second.num_successful_executions_ == 0; if (!first_execution) { - store_in_plasma_ids = it->second.reconstructable_return_ids; + store_in_plasma_ids = it->second.reconstructable_return_ids_; } if (first_execution_out != nullptr) { *first_execution_out = first_execution; @@ -1502,7 +1502,7 @@ std::optional TaskManager::GetTaskSpec(const TaskID &task_id) if (it == submissible_tasks_.end()) { return std::optional(); } - return it->second.spec; + return it->second.spec_; } std::vector TaskManager::GetPendingChildrenTasks( @@ -1510,7 +1510,7 @@ std::vector TaskManager::GetPendingChildrenTasks( std::vector ret_vec; absl::MutexLock lock(&mu_); for (const auto &it : submissible_tasks_) { - if (it.second.IsPending() && (it.second.spec.ParentTaskId() == parent_task_id)) { + if (it.second.IsPending() && (it.second.spec_.ParentTaskId() == parent_task_id)) { ret_vec.push_back(it.first); } } @@ -1528,7 +1528,7 @@ void TaskManager::AddTaskStatusInfo(rpc::CoreWorkerStats *stats) const { continue; } ref->set_task_status(it->second.GetStatus()); - ref->set_attempt_number(it->second.spec.AttemptNumber()); + ref->set_attempt_number(it->second.spec_.AttemptNumber()); } } @@ -1568,19 +1568,19 @@ void TaskManager::SetTaskStatus( std::optional state_update, bool include_task_info, std::optional attempt_number) { - RAY_LOG(DEBUG).WithField(task_entry.spec.TaskId()) + RAY_LOG(DEBUG).WithField(task_entry.spec_.TaskId()) << "Setting task status from " << rpc::TaskStatus_Name(task_entry.GetStatus()) << " to " << rpc::TaskStatus_Name(status); task_entry.SetStatus(status); const int32_t attempt_number_to_record = - attempt_number.value_or(task_entry.spec.AttemptNumber()); + attempt_number.value_or(task_entry.spec_.AttemptNumber()); const auto state_update_to_record = state_update.value_or(worker::TaskStatusEvent::TaskStateUpdate()); - RAY_UNUSED(task_event_buffer_.RecordTaskStatusEventIfNeeded(task_entry.spec.TaskId(), - task_entry.spec.JobId(), + RAY_UNUSED(task_event_buffer_.RecordTaskStatusEventIfNeeded(task_entry.spec_.TaskId(), + task_entry.spec_.JobId(), attempt_number_to_record, - task_entry.spec, + task_entry.spec_, status, include_task_info, state_update_to_record)); @@ -1597,19 +1597,19 @@ TaskManager::GetOngoingLineageReconstructionTasks( continue; } - if (task_entry.num_successful_executions == 0) { + if (task_entry.num_successful_executions_ == 0) { // Not lineage reconstruction task continue; } rpc::LineageReconstructionTask task; - task.set_name(task_entry.spec.GetName()); + task.set_name(task_entry.spec_.GetName()); task.set_status(task_entry.GetStatus()); - if (task_entry.spec.IsNormalTask()) { - task.mutable_labels()->insert(task_entry.spec.GetMessage().labels().begin(), - task_entry.spec.GetMessage().labels().end()); - } else if (task_entry.spec.IsActorTask()) { - auto actor_handle = actor_manager.GetActorHandle(task_entry.spec.ActorId()); + if (task_entry.spec_.IsNormalTask()) { + task.mutable_labels()->insert(task_entry.spec_.GetMessage().labels().begin(), + task_entry.spec_.GetMessage().labels().end()); + } else if (task_entry.spec_.IsActorTask()) { + auto actor_handle = actor_manager.GetActorHandle(task_entry.spec_.ActorId()); RAY_CHECK(actor_handle) << "Actor task must be submitted via actor handle"; const auto &labels = actor_handle->GetLabels(); task.mutable_labels()->insert(labels.begin(), labels.end()); @@ -1638,7 +1638,7 @@ void TaskManager::FillTaskInfo(rpc::GetCoreWorkerStatsReply *reply, const auto &task_entry = task_it.second; auto entry = reply->add_owned_task_info_entries(); - const auto &task_spec = task_entry.spec; + const auto &task_spec = task_entry.spec_; const auto &task_state = task_entry.GetStatus(); const auto &node_id = task_entry.GetNodeId(); rpc::TaskType type; @@ -1683,10 +1683,10 @@ ObjectID TaskManager::TaskGeneratorId(const TaskID &task_id) const { if (it == submissible_tasks_.end()) { return ObjectID::Nil(); } - if (!it->second.spec.ReturnsDynamic()) { + if (!it->second.spec_.ReturnsDynamic()) { return ObjectID::Nil(); } - return it->second.spec.ReturnId(0); + return it->second.spec_.ReturnId(0); } std::vector ExtractPlasmaDependencies(const TaskSpecification &spec) { diff --git a/src/ray/core_worker/task_manager.h b/src/ray/core_worker/task_manager.h index 25f5c90717c9..e8025bc0f7e5 100644 --- a/src/ray/core_worker/task_manager.h +++ b/src/ray/core_worker/task_manager.h @@ -501,41 +501,41 @@ class TaskManager : public TaskManagerInterface { private: struct TaskEntry { - TaskEntry(TaskSpecification spec_arg, - int num_retries_left_arg, + TaskEntry(TaskSpecification spec, + int num_retries_left, size_t num_returns, TaskStatusCounter &counter, int64_t num_oom_retries_left) - : spec(std::move(spec_arg)), - num_retries_left(num_retries_left_arg), - counter(&counter), - num_oom_retries_left(num_oom_retries_left), - is_canceled(false) { - reconstructable_return_ids.reserve(num_returns); + : spec_(std::move(spec)), + num_retries_left_(num_retries_left), + counter_(&counter), + num_oom_retries_left_(num_oom_retries_left), + is_canceled_(false) { + reconstructable_return_ids_.reserve(num_returns); for (size_t i = 0; i < num_returns; i++) { - reconstructable_return_ids.insert(spec.ReturnId(i)); + reconstructable_return_ids_.insert(spec_.ReturnId(i)); } - status = - std::make_tuple(spec.GetName(), rpc::TaskStatus::PENDING_ARGS_AVAIL, false); - counter.Increment(status); + status_ = + std::make_tuple(spec_.GetName(), rpc::TaskStatus::PENDING_ARGS_AVAIL, false); + counter_->Increment(status_); } void SetStatus(rpc::TaskStatus new_status) { - auto new_tuple = std::make_tuple(spec.GetName(), new_status, is_retry_); + auto new_tuple = std::make_tuple(spec_.GetName(), new_status, is_retry_); if (IsPending()) { - counter->Swap(status, new_tuple); + counter_->Swap(status_, new_tuple); } else { // FINISHED and FAILED are monotonically increasing. // TODO(jjyao): We should use Counter instead of Gauge // for FINISHED and FAILED tasks. - counter->Increment(new_tuple); + counter_->Increment(new_tuple); } - status = std::move(new_tuple); + status_ = std::move(new_tuple); } void MarkRetry() { is_retry_ = true; } - rpc::TaskStatus GetStatus() const { return std::get<1>(status); } + rpc::TaskStatus GetStatus() const { return std::get<1>(status_); } // Get the NodeID where the task is executed. NodeID GetNodeId() const { return node_id_; } @@ -555,25 +555,25 @@ class TaskManager : public TaskManagerInterface { /// - The task is still pending execution. This means that the task may /// fail and so it may be retried in the future. /// - The task finished execution, but it has num_retries_left > 0 and - /// reconstructable_return_ids is not empty. This means that the task may + /// reconstructable_return_ids_ is not empty. This means that the task may /// be retried in the future to recreate its return objects. /// TODO(swang): The TaskSpec protobuf must be copied into the /// PushTaskRequest protobuf when sent to a worker so that we can retry it if /// the worker fails. We could avoid this by either not caching the full /// TaskSpec for tasks that cannot be retried (e.g., actor tasks), or by /// storing a shared_ptr to a PushTaskRequest protobuf for all tasks. - TaskSpecification spec; + TaskSpecification spec_; // Number of times this task may be resubmitted. If this reaches 0, then // the task entry may be erased. - int32_t num_retries_left; + int32_t num_retries_left_; // Reference to the task stats tracker. - TaskStatusCounter *counter; + TaskStatusCounter *counter_; // Number of times this task may be resubmitted if the task failed // due to out of memory failure. - int32_t num_oom_retries_left; + int32_t num_oom_retries_left_; // Whether the task has been marked for cancellation. // Canceled tasks will never be retried. - bool is_canceled; + bool is_canceled_; // Objects returned by this task that are reconstructable. This is set // objects may be reconstructed by resubmitting the task. Once the task // finishes its first execution, then the objects that the task returned by @@ -584,18 +584,18 @@ class TaskManager : public TaskManagerInterface { // 2) There are no tasks that depend on the object. This includes both // pending tasks and tasks that finished execution but that may be // retried in the future. - absl::flat_hash_set reconstructable_return_ids; + absl::flat_hash_set reconstructable_return_ids_; // The size of this (serialized) task spec in bytes, if the task spec is // not pending, i.e. it is being pinned because it's in another object's // lineage. We cache this because the task spec protobuf can mutate // out-of-band. - int64_t lineage_footprint_bytes = 0; + int64_t lineage_footprint_bytes_ = 0; // Number of times this task successfully completed execution so far. - int num_successful_executions = 0; + int num_successful_executions_ = 0; private: // The task's current execution and metric status (name, status, is_retry). - std::tuple status; + std::tuple status_; // The node id where task is executed. NodeID node_id_; // Whether this is a task retry due to task failure. @@ -656,7 +656,7 @@ class TaskManager : public TaskManagerInterface { /// Shutdown if all tasks are finished and shutdown is scheduled. void ShutdownIfNeeded() ABSL_LOCKS_EXCLUDED(mu_); - /// Updates the task entry state (e.g. status, is_retry, lineage_footprint_bytes, + /// Updates the task entry state (e.g. status, is_retry, lineage_footprint_bytes_, /// num_retries_left) + related global task manager state. void SetupTaskEntryForResubmit(TaskEntry &task_entry) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.cc b/src/ray/core_worker/task_submission/actor_task_submitter.cc index 506f07feb3b3..33ece8caf203 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.cc +++ b/src/ray/core_worker/task_submission/actor_task_submitter.cc @@ -35,8 +35,8 @@ void ActorTaskSubmitter::NotifyGCSWhenActorOutOfScope( { absl::MutexLock lock(&mu_); if (auto iter = client_queues_.find(actor_id); iter != client_queues_.end()) { - if (iter->second.state != rpc::ActorTableData::DEAD) { - iter->second.pending_out_of_scope_death = true; + if (iter->second.state_ != rpc::ActorTableData::DEAD) { + iter->second.pending_out_of_scope_death_ = true; } } } @@ -74,8 +74,7 @@ void ActorTaskSubmitter::AddActorQueueIfNotExists(const ActorID &actor_id, << "Set actor max pending calls to " << max_pending_calls; inserted = client_queues_ .emplace(actor_id, - ClientQueue(actor_id, - allow_out_of_order_execution, + ClientQueue(allow_out_of_order_execution, max_pending_calls, fail_if_actor_unreachable, owned)) @@ -91,9 +90,7 @@ void ActorTaskSubmitter::AddActorQueueIfNotExists(const ActorID &actor_id, Status ActorTaskSubmitter::SubmitActorCreationTask(TaskSpecification task_spec) { RAY_CHECK(task_spec.IsActorCreationTask()); - const auto actor_id = task_spec.ActorCreationId(); - const auto task_id = task_spec.TaskId(); - RAY_LOG(DEBUG).WithField(actor_id).WithField(task_id) + RAY_LOG(DEBUG).WithField(task_spec.ActorCreationId()).WithField(task_spec.TaskId()) << "Submitting actor creation task"; resolver_.ResolveDependencies(task_spec, [this, task_spec](Status status) mutable { // NOTE: task_spec here is capture copied (from a stack variable) and also @@ -118,16 +115,17 @@ Status ActorTaskSubmitter::SubmitActorCreationTask(TaskSpecification task_spec) RAY_LOG(DEBUG).WithField(actor_id).WithField(task_id) << "Creating actor via GCS"; actor_creator_.AsyncCreateActor( task_spec, - [this, actor_id, task_id](Status status, const rpc::CreateActorReply &reply) { - if (status.ok() || status.IsCreationTaskError()) { + [this, actor_id, task_id](Status create_actor_status, + const rpc::CreateActorReply &reply) { + if (create_actor_status.ok() || create_actor_status.IsCreationTaskError()) { rpc::PushTaskReply push_task_reply; push_task_reply.mutable_borrowed_refs()->CopyFrom(reply.borrowed_refs()); - if (status.IsCreationTaskError()) { + if (create_actor_status.IsCreationTaskError()) { RAY_LOG(INFO).WithField(actor_id).WithField(task_id) << "Actor creation failed and we will not be retrying the " "creation task"; // Update the task execution error to be CreationTaskError. - push_task_reply.set_task_execution_error(status.ToString()); + push_task_reply.set_task_execution_error(create_actor_status.ToString()); } else { RAY_LOG(DEBUG).WithField(actor_id).WithField(task_id) << "Created actor"; } @@ -137,11 +135,11 @@ Status ActorTaskSubmitter::SubmitActorCreationTask(TaskSpecification task_spec) task_id, push_task_reply, reply.actor_address(), - /*is_application_error=*/status.IsCreationTaskError()); + /*is_application_error=*/create_actor_status.IsCreationTaskError()); } else { // Either fails the rpc call or actor scheduling cancelled. rpc::RayErrorInfo ray_error_info; - if (status.IsSchedulingCancelled()) { + if (create_actor_status.IsSchedulingCancelled()) { RAY_LOG(DEBUG).WithField(actor_id).WithField(task_id) << "Actor creation cancelled"; task_manager_.MarkTaskNoRetry(task_id); @@ -150,7 +148,7 @@ Status ActorTaskSubmitter::SubmitActorCreationTask(TaskSpecification task_spec) } } else { RAY_LOG(INFO).WithField(actor_id).WithField(task_id) - << "Failed to create actor with status: " << status; + << "Failed to create actor with status: " << create_actor_status; } // Actor creation task retry happens in GCS // and transient rpc errors are retried in gcs client @@ -158,7 +156,7 @@ Status ActorTaskSubmitter::SubmitActorCreationTask(TaskSpecification task_spec) RAY_UNUSED(task_manager_.FailPendingTask( task_id, rpc::ErrorType::ACTOR_CREATION_FAILED, - &status, + &create_actor_status, ray_error_info.has_actor_died_error() ? &ray_error_info : nullptr)); } }); @@ -179,29 +177,27 @@ Status ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { absl::MutexLock lock(&mu_); auto queue = client_queues_.find(actor_id); RAY_CHECK(queue != client_queues_.end()); - if (queue->second.state == rpc::ActorTableData::DEAD && - queue->second.is_restartable && queue->second.owned) { + if (queue->second.state_ == rpc::ActorTableData::DEAD && + queue->second.is_restartable_ && queue->second.owned_) { RestartActorForLineageReconstruction(actor_id); } - if (queue->second.state != rpc::ActorTableData::DEAD) { + if (queue->second.state_ != rpc::ActorTableData::DEAD) { // We must fix the send order prior to resolving dependencies, which may // complete out of order. This ensures that we will not deadlock due to // backpressure. The receiving actor will execute the tasks according to // this sequence number. send_pos = task_spec.SequenceNumber(); - queue->second.actor_submit_queue->Emplace(send_pos, task_spec); - queue->second.cur_pending_calls++; + queue->second.actor_submit_queue_->Emplace(send_pos, task_spec); + queue->second.cur_pending_calls_++; task_queued = true; } } if (task_queued) { io_service_.post( - [task_spec, send_pos, this]() mutable { + [task_spec, task_id, actor_id, send_pos, this]() mutable { // We must release the lock before resolving the task dependencies since // the callback may get called in the same call stack. - auto actor_id = task_spec.ActorId(); - auto task_id = task_spec.TaskId(); resolver_.ResolveDependencies( task_spec, [this, send_pos, actor_id, task_id](Status status) { task_manager_.MarkDependenciesResolved(task_id); @@ -210,7 +206,7 @@ Status ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { absl::MutexLock lock(&mu_); auto queue = client_queues_.find(actor_id); RAY_CHECK(queue != client_queues_.end()); - auto &actor_submit_queue = queue->second.actor_submit_queue; + auto &actor_submit_queue = queue->second.actor_submit_queue_; // Only dispatch tasks if the submitted task is still queued. The task // may have been dequeued if the actor has since failed. if (actor_submit_queue->Contains(send_pos)) { @@ -239,7 +235,7 @@ Status ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { { absl::MutexLock lock(&mu_); const auto queue_it = client_queues_.find(task_spec.ActorId()); - const auto &death_cause = queue_it->second.death_cause; + const auto &death_cause = queue_it->second.death_cause_; error_info = gcs::GetErrorInfoFromActorDeathCause(death_cause); error_type = error_info.error_type(); } @@ -264,9 +260,9 @@ Status ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { } void ActorTaskSubmitter::DisconnectRpcClient(ClientQueue &queue) { - queue.rpc_client = nullptr; - core_worker_client_pool_.Disconnect(WorkerID::FromBinary(queue.worker_id)); - queue.worker_id.clear(); + queue.rpc_client_ = nullptr; + core_worker_client_pool_.Disconnect(WorkerID::FromBinary(queue.worker_id_)); + queue.worker_id_.clear(); } void ActorTaskSubmitter::FailInflightTasksOnRestart( @@ -295,7 +291,7 @@ void ActorTaskSubmitter::ConnectActor(const ActorID &actor_id, auto queue = client_queues_.find(actor_id); RAY_CHECK(queue != client_queues_.end()); - if (num_restarts < queue->second.num_restarts) { + if (num_restarts < queue->second.num_restarts_) { // This message is about an old version of the actor and the actor has // already restarted since then. Skip the connection. RAY_LOG(INFO).WithField(actor_id) @@ -303,32 +299,32 @@ void ActorTaskSubmitter::ConnectActor(const ActorID &actor_id, return; } - if (queue->second.rpc_client && - queue->second.rpc_client->Addr().ip_address() == address.ip_address() && - queue->second.rpc_client->Addr().port() == address.port()) { + if (queue->second.rpc_client_ && + queue->second.rpc_client_->Addr().ip_address() == address.ip_address() && + queue->second.rpc_client_->Addr().port() == address.port()) { RAY_LOG(DEBUG).WithField(actor_id) << "Skip actor that has already been connected"; return; } - if (queue->second.state == rpc::ActorTableData::DEAD) { + if (queue->second.state_ == rpc::ActorTableData::DEAD) { // This message is about an old version of the actor and the actor has // already died since then. Skip the connection. return; } - queue->second.num_restarts = num_restarts; - if (queue->second.rpc_client) { + queue->second.num_restarts_ = num_restarts; + if (queue->second.rpc_client_) { // Clear the client to the old version of the actor. DisconnectRpcClient(queue->second); - inflight_task_callbacks = std::move(queue->second.inflight_task_callbacks); - queue->second.inflight_task_callbacks.clear(); + inflight_task_callbacks = std::move(queue->second.inflight_task_callbacks_); + queue->second.inflight_task_callbacks_.clear(); } - queue->second.state = rpc::ActorTableData::ALIVE; + queue->second.state_ = rpc::ActorTableData::ALIVE; // Update the mapping so new RPCs go out with the right intended worker id. - queue->second.worker_id = address.worker_id(); + queue->second.worker_id_ = address.worker_id(); // Create a new connection to the actor. - queue->second.rpc_client = core_worker_client_pool_.GetOrConnect(address); + queue->second.rpc_client_ = core_worker_client_pool_.GetOrConnect(address); SendPendingTasks(actor_id); } @@ -341,17 +337,17 @@ void ActorTaskSubmitter::RestartActorForLineageReconstruction(const ActorID &act RAY_LOG(INFO).WithField(actor_id) << "Reconstructing actor"; auto queue = client_queues_.find(actor_id); RAY_CHECK(queue != client_queues_.end()); - RAY_CHECK(queue->second.owned) << "Only owner can restart the dead actor"; - RAY_CHECK(queue->second.is_restartable) << "This actor is no longer restartable"; - queue->second.state = rpc::ActorTableData::RESTARTING; - queue->second.num_restarts_due_to_lineage_reconstructions += 1; + RAY_CHECK(queue->second.owned_) << "Only owner can restart the dead actor"; + RAY_CHECK(queue->second.is_restartable_) << "This actor is no longer restartable"; + queue->second.state_ = rpc::ActorTableData::RESTARTING; + queue->second.num_restarts_due_to_lineage_reconstructions_ += 1; actor_creator_.AsyncRestartActorForLineageReconstruction( actor_id, - queue->second.num_restarts_due_to_lineage_reconstructions, + queue->second.num_restarts_due_to_lineage_reconstructions_, [this, actor_id, num_restarts_due_to_lineage_reconstructions = - queue->second.num_restarts_due_to_lineage_reconstructions](Status status) { + queue->second.num_restarts_due_to_lineage_reconstructions_](Status status) { if (!status.ok()) { RAY_LOG(ERROR).WithField(actor_id) << "Failed to reconstruct actor. Error message: " << status.ToString(); @@ -382,7 +378,7 @@ void ActorTaskSubmitter::DisconnectActor(const ActorID &actor_id, if (!dead) { RAY_CHECK_GT(num_restarts, 0); } - if (num_restarts <= queue->second.num_restarts && !dead) { + if (num_restarts <= queue->second.num_restarts_ && !dead) { // This message is about an old version of the actor that has already been // restarted successfully. Skip the message handling. RAY_LOG(INFO).WithField(actor_id) @@ -394,20 +390,20 @@ void ActorTaskSubmitter::DisconnectActor(const ActorID &actor_id, // permanently dead or the new client will be inserted once the actor is // restarted. DisconnectRpcClient(queue->second); - inflight_task_callbacks = std::move(queue->second.inflight_task_callbacks); - queue->second.inflight_task_callbacks.clear(); + inflight_task_callbacks = std::move(queue->second.inflight_task_callbacks_); + queue->second.inflight_task_callbacks_.clear(); if (dead) { - queue->second.state = rpc::ActorTableData::DEAD; - queue->second.death_cause = death_cause; - queue->second.pending_out_of_scope_death = false; - queue->second.is_restartable = is_restartable; + queue->second.state_ = rpc::ActorTableData::DEAD; + queue->second.death_cause_ = death_cause; + queue->second.pending_out_of_scope_death_ = false; + queue->second.is_restartable_ = is_restartable; - if (queue->second.is_restartable && queue->second.owned) { + if (queue->second.is_restartable_ && queue->second.owned_) { // Actor is out of scope so there should be no inflight actor tasks. - RAY_CHECK(queue->second.wait_for_death_info_tasks.empty()); + RAY_CHECK(queue->second.wait_for_death_info_tasks_.empty()); RAY_CHECK(inflight_task_callbacks.empty()); - if (!queue->second.actor_submit_queue->Empty()) { + if (!queue->second.actor_submit_queue_->Empty()) { // There are pending lineage reconstruction tasks. RestartActorForLineageReconstruction(actor_id); } @@ -416,18 +412,18 @@ void ActorTaskSubmitter::DisconnectActor(const ActorID &actor_id, RAY_LOG(INFO).WithField(actor_id) << "Failing pending tasks for actor because the actor is already dead."; - task_ids_to_fail = queue->second.actor_submit_queue->ClearAllTasks(); + task_ids_to_fail = queue->second.actor_submit_queue_->ClearAllTasks(); // We need to execute this outside of the lock to prevent deadlock. - wait_for_death_info_tasks = std::move(queue->second.wait_for_death_info_tasks); + wait_for_death_info_tasks = std::move(queue->second.wait_for_death_info_tasks_); // Reset the queue - queue->second.wait_for_death_info_tasks = + queue->second.wait_for_death_info_tasks_ = std::deque>(); } - } else if (queue->second.state != rpc::ActorTableData::DEAD) { + } else if (queue->second.state_ != rpc::ActorTableData::DEAD) { // Only update the actor's state if it is not permanently dead. The actor // will eventually get restarted or marked as permanently dead. - queue->second.state = rpc::ActorTableData::RESTARTING; - queue->second.num_restarts = num_restarts; + queue->second.state_ = rpc::ActorTableData::RESTARTING; + queue->second.num_restarts_ = num_restarts; } } @@ -461,7 +457,7 @@ void ActorTaskSubmitter::DisconnectActor(const ActorID &actor_id, << wait_for_death_info_tasks.size(); for (auto &task : wait_for_death_info_tasks) { GetTaskManagerWithoutMu().FailPendingTask( - task->task_spec.TaskId(), error_type, &task->status, &error_info); + task->task_spec_.TaskId(), error_type, &task->status_, &error_info); } } } @@ -471,8 +467,8 @@ void ActorTaskSubmitter::DisconnectActor(const ActorID &actor_id, void ActorTaskSubmitter::FailTaskWithError(const PendingTaskWaitingForDeathInfo &task) { rpc::RayErrorInfo error_info; - if (!task.actor_preempted) { - error_info = task.timeout_error_info; + if (!task.actor_preempted_) { + error_info = task.timeout_error_info_; } else { // Special error for preempted actor. The task "timed out" because the actor may // not have sent a notification to the gcs; regardless we already know it's @@ -480,7 +476,7 @@ void ActorTaskSubmitter::FailTaskWithError(const PendingTaskWaitingForDeathInfo auto actor_death_cause = error_info.mutable_actor_died_error(); auto actor_died_error_context = actor_death_cause->mutable_actor_died_error_context(); actor_died_error_context->set_reason(rpc::ActorDiedErrorContext::NODE_DIED); - actor_died_error_context->set_actor_id(task.task_spec.ActorId().Binary()); + actor_died_error_context->set_actor_id(task.task_spec_.ActorId().Binary()); auto node_death_info = actor_died_error_context->mutable_node_death_info(); node_death_info->set_reason(rpc::NodeDeathInfo::AUTOSCALER_DRAIN_PREEMPTED); node_death_info->set_reason_message( @@ -489,7 +485,7 @@ void ActorTaskSubmitter::FailTaskWithError(const PendingTaskWaitingForDeathInfo error_info.set_error_message("Actor died by preemption."); } GetTaskManagerWithoutMu().FailPendingTask( - task.task_spec.TaskId(), error_info.error_type(), &task.status, &error_info); + task.task_spec_.TaskId(), error_info.error_type(), &task.status_, &error_info); } void ActorTaskSubmitter::CheckTimeoutTasks() { @@ -502,12 +498,12 @@ void ActorTaskSubmitter::CheckTimeoutTasks() { { absl::MutexLock lock(&mu_); for (auto &[actor_id, client_queue] : client_queues_) { - auto &deque = client_queue.wait_for_death_info_tasks; + auto &deque = client_queue.wait_for_death_info_tasks_; auto deque_itr = deque.begin(); - while (deque_itr != deque.end() && (*deque_itr)->deadline_ms < now) { + while (deque_itr != deque.end() && (*deque_itr)->deadline_ms_ < now) { // Populate the info of whether the actor is preempted. If so we hard fail the // task. - (*deque_itr)->actor_preempted = client_queue.preempted; + (*deque_itr)->actor_preempted_ = client_queue.preempted_; timeout_tasks.push_back(*deque_itr); deque_itr = deque.erase(deque_itr); } @@ -523,17 +519,17 @@ void ActorTaskSubmitter::SendPendingTasks(const ActorID &actor_id) { auto it = client_queues_.find(actor_id); RAY_CHECK(it != client_queues_.end()); auto &client_queue = it->second; - auto &actor_submit_queue = client_queue.actor_submit_queue; - if (client_queue.pending_out_of_scope_death) { + auto &actor_submit_queue = client_queue.actor_submit_queue_; + if (client_queue.pending_out_of_scope_death_) { // Wait until the actor is dead and then decide // whether we should fail pending tasks or restart the actor. // If the actor is restarted, ConnectActor will be called // and pending tasks will be sent at that time. return; } - if (!client_queue.rpc_client) { - if (client_queue.state == rpc::ActorTableData::RESTARTING && - client_queue.fail_if_actor_unreachable) { + if (!client_queue.rpc_client_) { + if (client_queue.state_ == rpc::ActorTableData::RESTARTING && + client_queue.fail_if_actor_unreachable_) { // When `fail_if_actor_unreachable` is true, tasks submitted while the actor is in // `RESTARTING` state fail immediately. while (true) { @@ -561,7 +557,7 @@ void ActorTaskSubmitter::SendPendingTasks(const ActorID &actor_id) { if (!task.has_value()) { break; } - RAY_CHECK(!client_queue.worker_id.empty()); + RAY_CHECK(!client_queue.worker_id_.empty()); PushActorTask(client_queue, /*task_spec=*/task->first, /*skip_queue=*/task->second); } } @@ -577,12 +573,12 @@ void ActorTaskSubmitter::PushActorTask(ClientQueue &queue, // access the task. request->mutable_task_spec()->CopyFrom(task_spec.GetMessage()); - request->set_intended_worker_id(queue.worker_id); + request->set_intended_worker_id(queue.worker_id_); request->set_sequence_number(task_spec.SequenceNumber()); const auto actor_id = task_spec.ActorId(); - const auto num_queued = queue.inflight_task_callbacks.size(); + const auto num_queued = queue.inflight_task_callbacks_.size(); RAY_LOG(DEBUG).WithField(task_id).WithField(actor_id) << "Pushing task to actor, actor id " << actor_id << " seq no " << request->sequence_number() << " num queued " << num_queued; @@ -592,38 +588,38 @@ void ActorTaskSubmitter::PushActorTask(ClientQueue &queue, next_queueing_warn_threshold_ *= 2; } - rpc::Address addr(queue.rpc_client->Addr()); + rpc::Address addr(queue.rpc_client_->Addr()); rpc::ClientCallback reply_callback = [this, addr, task_spec](const Status &status, const rpc::PushTaskReply &reply) { HandlePushTaskReply(status, reply, addr, task_spec); }; const TaskAttempt task_attempt = std::make_pair(task_id, task_spec.AttemptNumber()); - queue.inflight_task_callbacks.emplace(task_attempt, std::move(reply_callback)); + queue.inflight_task_callbacks_.emplace(task_attempt, std::move(reply_callback)); rpc::ClientCallback wrapped_callback = [this, task_attempt, actor_id](const Status &status, rpc::PushTaskReply &&reply) { - rpc::ClientCallback reply_callback; + rpc::ClientCallback push_task_reply_callback; { absl::MutexLock lock(&mu_); auto it = client_queues_.find(actor_id); RAY_CHECK(it != client_queues_.end()); - auto &queue = it->second; - auto callback_it = queue.inflight_task_callbacks.find(task_attempt); - if (callback_it == queue.inflight_task_callbacks.end()) { + auto &client_queue = it->second; + auto callback_it = client_queue.inflight_task_callbacks_.find(task_attempt); + if (callback_it == client_queue.inflight_task_callbacks_.end()) { RAY_LOG(DEBUG).WithField(task_attempt.first) << "The task has already been marked as failed. Ignore the reply."; return; } - reply_callback = std::move(callback_it->second); - queue.inflight_task_callbacks.erase(callback_it); + push_task_reply_callback = std::move(callback_it->second); + client_queue.inflight_task_callbacks_.erase(callback_it); } - reply_callback(status, std::move(reply)); + push_task_reply_callback(status, std::move(reply)); }; task_manager_.MarkTaskWaitingForExecution(task_id, NodeID::FromBinary(addr.node_id()), WorkerID::FromBinary(addr.worker_id())); - queue.rpc_client->PushActorTask( + queue.rpc_client_->PushActorTask( std::move(request), skip_queue, std::move(wrapped_callback)); } @@ -644,7 +640,7 @@ void ActorTaskSubmitter::HandlePushTaskReply(const Status &status, auto queue_pair = client_queues_.find(actor_id); RAY_CHECK(queue_pair != client_queues_.end()); auto &queue = queue_pair->second; - queue.cur_pending_calls--; + queue.cur_pending_calls_--; } } if (resubmit_generator) { @@ -697,9 +693,9 @@ void ActorTaskSubmitter::HandlePushTaskReply(const Status &status, // - If we did not get the death reason: raise ACTOR_UNAVAILABLE with the status. // - If we did not get the death reason, but *the actor is preempted*: raise // ACTOR_DIED. See `CheckTimeoutTasks`. - is_actor_dead = queue.state == rpc::ActorTableData::DEAD; + is_actor_dead = queue.state_ == rpc::ActorTableData::DEAD; if (is_actor_dead) { - const auto &death_cause = queue.death_cause; + const auto &death_cause = queue.death_cause_; error_info = gcs::GetErrorInfoFromActorDeathCause(death_cause); fail_immediately = error_info.has_actor_died_error() && error_info.actor_died_error().has_oom_context() && @@ -745,14 +741,14 @@ void ActorTaskSubmitter::HandlePushTaskReply(const Status &status, auto queue_pair = client_queues_.find(actor_id); RAY_CHECK(queue_pair != client_queues_.end()); auto &queue = queue_pair->second; - queue.wait_for_death_info_tasks.push_back( + queue.wait_for_death_info_tasks_.push_back( std::make_shared( death_info_grace_period_ms, task_spec, status, error_info)); RAY_LOG(INFO).WithField(task_spec.TaskId()) << "PushActorTask failed because of network error, this task " "will be stashed away and waiting for Death info from GCS" ", wait_queue_size=" - << queue.wait_for_death_info_tasks.size(); + << queue.wait_for_death_info_tasks_.size(); } else { // TODO(vitsai): if we don't need death info, just fail the request. { @@ -770,7 +766,7 @@ void ActorTaskSubmitter::HandlePushTaskReply(const Status &status, auto queue_pair = client_queues_.find(actor_id); RAY_CHECK(queue_pair != client_queues_.end()); auto &queue = queue_pair->second; - queue.cur_pending_calls--; + queue.cur_pending_calls_--; } } @@ -782,7 +778,7 @@ std::optional ActorTaskSubmitter::GetLocalActor if (iter == client_queues_.end()) { return std::nullopt; } else { - return iter->second.state; + return iter->second.state_; } } @@ -790,7 +786,7 @@ bool ActorTaskSubmitter::IsActorAlive(const ActorID &actor_id) const { absl::MutexLock lock(&mu_); auto iter = client_queues_.find(actor_id); - return (iter != client_queues_.end() && iter->second.rpc_client); + return (iter != client_queues_.end() && iter->second.rpc_client_); } std::optional ActorTaskSubmitter::GetActorAddress( @@ -802,27 +798,27 @@ std::optional ActorTaskSubmitter::GetActorAddress( return std::nullopt; } - const auto &rpc_client = iter->second.rpc_client; + const auto &rpc_client = iter->second.rpc_client_; if (rpc_client == nullptr) { return std::nullopt; } - return iter->second.rpc_client->Addr(); + return iter->second.rpc_client_->Addr(); } bool ActorTaskSubmitter::PendingTasksFull(const ActorID &actor_id) const { absl::MutexLock lock(&mu_); auto it = client_queues_.find(actor_id); RAY_CHECK(it != client_queues_.end()); - return it->second.max_pending_calls > 0 && - it->second.cur_pending_calls >= it->second.max_pending_calls; + return it->second.max_pending_calls_ > 0 && + it->second.cur_pending_calls_ >= it->second.max_pending_calls_; } size_t ActorTaskSubmitter::NumPendingTasks(const ActorID &actor_id) const { absl::MutexLock lock(&mu_); auto it = client_queues_.find(actor_id); RAY_CHECK(it != client_queues_.end()); - return it->second.cur_pending_calls; + return it->second.cur_pending_calls_; } bool ActorTaskSubmitter::CheckActorExists(const ActorID &actor_id) const { @@ -886,17 +882,17 @@ Status ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursiv auto queue = client_queues_.find(actor_id); RAY_CHECK(queue != client_queues_.end()); - if (queue->second.state == rpc::ActorTableData::DEAD) { + if (queue->second.state_ == rpc::ActorTableData::DEAD) { // No need to decrement cur_pending_calls because it doesn't matter. RAY_LOG(DEBUG).WithField(task_id) << "Task's actor is already dead. Ignoring the cancel request."; return Status::OK(); } - task_queued = queue->second.actor_submit_queue->Contains(send_pos); + task_queued = queue->second.actor_submit_queue_->Contains(send_pos); if (task_queued) { auto dep_resolved = - queue->second.actor_submit_queue->DependenciesResolved(send_pos); + queue->second.actor_submit_queue_->DependenciesResolved(send_pos); if (!dep_resolved) { RAY_LOG(DEBUG).WithField(task_id) << "Task has been resolving dependencies. Cancel to resolve dependencies"; @@ -904,7 +900,7 @@ Status ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursiv } RAY_LOG(DEBUG).WithField(task_id) << "Task was queued. Mark a task is canceled from a queue."; - queue->second.actor_submit_queue->MarkTaskCanceled(send_pos); + queue->second.actor_submit_queue_->MarkTaskCanceled(send_pos); } } @@ -936,12 +932,12 @@ Status ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursiv RAY_LOG(DEBUG).WithField(task_id) << "Task was sent to an actor. Send a cancel RPC."; auto queue = client_queues_.find(actor_id); RAY_CHECK(queue != client_queues_.end()); - if (!queue->second.rpc_client) { + if (!queue->second.rpc_client_) { RetryCancelTask(task_spec, recursive, 1000); return Status::OK(); } - const auto &client = queue->second.rpc_client; + const auto &client = queue->second.rpc_client_; auto request = rpc::CancelTaskRequest(); request.set_intended_task_id(task_spec.TaskIdBinary()); request.set_force_kill(force_kill); diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.h b/src/ray/core_worker/task_submission/actor_task_submitter.h index 1d699bd3c8ca..160ad7487a6a 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.h +++ b/src/ray/core_worker/task_submission/actor_task_submitter.h @@ -91,7 +91,7 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { void SetPreempted(const ActorID &actor_id) { absl::MutexLock lock(&mu_); if (auto iter = client_queues_.find(actor_id); iter != client_queues_.end()) { - iter->second.preempted = true; + iter->second.preempted_ = true; } } @@ -252,20 +252,20 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { private: struct PendingTaskWaitingForDeathInfo { - int64_t deadline_ms; - TaskSpecification task_spec; - ray::Status status; - rpc::RayErrorInfo timeout_error_info; - bool actor_preempted = false; + int64_t deadline_ms_; + TaskSpecification task_spec_; + ray::Status status_; + rpc::RayErrorInfo timeout_error_info_; + bool actor_preempted_ = false; PendingTaskWaitingForDeathInfo(int64_t deadline_ms, TaskSpecification task_spec, ray::Status status, rpc::RayErrorInfo timeout_error_info) - : deadline_ms(deadline_ms), - task_spec(std::move(task_spec)), - status(std::move(status)), - timeout_error_info(std::move(timeout_error_info)) {} + : deadline_ms_(deadline_ms), + task_spec_(std::move(task_spec)), + status_(std::move(status)), + timeout_error_info_(std::move(timeout_error_info)) {} }; /// A helper function to get task manager without holding mu_ /// We should use this function when access @@ -277,50 +277,49 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { } struct ClientQueue { - ClientQueue(ActorID actor_id, - bool allow_out_of_order_execution, + ClientQueue(bool allow_out_of_order_execution, int32_t max_pending_calls, bool fail_if_actor_unreachable, bool owned) - : max_pending_calls(max_pending_calls), - fail_if_actor_unreachable(fail_if_actor_unreachable), - owned(owned) { + : max_pending_calls_(max_pending_calls), + fail_if_actor_unreachable_(fail_if_actor_unreachable), + owned_(owned) { if (allow_out_of_order_execution) { - actor_submit_queue = std::make_unique(actor_id); + actor_submit_queue_ = std::make_unique(); } else { - actor_submit_queue = std::make_unique(actor_id); + actor_submit_queue_ = std::make_unique(); } } /// The current state of the actor. If this is ALIVE, then we should have /// an RPC client to the actor. If this is DEAD, then all tasks in the /// queue will be marked failed and all other ClientQueue state is ignored. - rpc::ActorTableData::ActorState state = rpc::ActorTableData::DEPENDENCIES_UNREADY; + rpc::ActorTableData::ActorState state_ = rpc::ActorTableData::DEPENDENCIES_UNREADY; /// The reason why this actor is dead. /// If the context is not set, it means the actor is not dead. - rpc::ActorDeathCause death_cause; + rpc::ActorDeathCause death_cause_; /// How many times this actor has been restarted before. Starts at -1 to /// indicate that the actor is not yet created. This is used to drop stale /// messages from the GCS. - int64_t num_restarts = -1; + int64_t num_restarts_ = -1; /// How many times this actor has been lineage reconstructured. /// This is used to drop stale messages. - int64_t num_restarts_due_to_lineage_reconstructions = 0; + int64_t num_restarts_due_to_lineage_reconstructions_ = 0; /// Whether this actor exits by spot preemption. - bool preempted = false; + bool preempted_ = false; /// The RPC client. We use shared_ptr to enable shared_from_this for /// pending client callbacks. - std::shared_ptr rpc_client = nullptr; + std::shared_ptr rpc_client_ = nullptr; /// The intended worker ID of the actor. - std::string worker_id = ""; + std::string worker_id_ = ""; /// The actor is out of scope but the death info is not published /// to this worker yet. - bool pending_out_of_scope_death = false; + bool pending_out_of_scope_death_ = false; /// If the actor is dead, whether it can be restarted. - bool is_restartable = false; + bool is_restartable_ = false; /// The queue that orders actor requests. - std::unique_ptr actor_submit_queue; + std::unique_ptr actor_submit_queue_; /// Tasks that can't be sent because 1) the callee actor is dead. 2) network error. /// For 1) the task will wait for the DEAD state notification, then mark task as @@ -336,34 +335,35 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { /// `timeout_error_info`. One special case is when the actor is preempted, where /// the actor may not be dead *just yet* but we want to treat it as dead. In this /// case we hard code an error info. - std::deque> wait_for_death_info_tasks; + std::deque> + wait_for_death_info_tasks_; /// Stores all callbacks of inflight tasks. An actor task is inflight /// if the PushTask RPC is sent but the reply is not received yet. absl::flat_hash_map> - inflight_task_callbacks; + inflight_task_callbacks_; /// The max number limit of task capacity used for back pressure. /// If the number of tasks in requests >= max_pending_calls, it can't continue to /// push task to ClientQueue. - const int32_t max_pending_calls; + const int32_t max_pending_calls_; /// The current task number in this client queue. - int32_t cur_pending_calls = 0; + int32_t cur_pending_calls_ = 0; /// Whether to fail newly submitted tasks immediately when the actor is unreachable. - bool fail_if_actor_unreachable = true; + bool fail_if_actor_unreachable_ = true; /// Whether the current process is owner of the actor. - bool owned; + bool owned_; /// Returns debug string for class. /// /// \return string. std::string DebugString() const { std::ostringstream stream; - stream << "max_pending_calls=" << max_pending_calls - << " cur_pending_calls=" << cur_pending_calls; + stream << "max_pending_calls=" << max_pending_calls_ + << " cur_pending_calls=" << cur_pending_calls_; return stream.str(); } }; diff --git a/src/ray/core_worker/task_submission/dependency_resolver.cc b/src/ray/core_worker/task_submission/dependency_resolver.cc index a9600f535465..e11b0c95645e 100644 --- a/src/ray/core_worker/task_submission/dependency_resolver.cc +++ b/src/ray/core_worker/task_submission/dependency_resolver.cc @@ -165,7 +165,7 @@ void LocalDependencyResolver::ResolveDependencies( contained_ids); } if (resolved_task_state) { - resolved_task_state->on_dependencies_resolved(resolved_task_state->status); + resolved_task_state->on_dependencies_resolved_(resolved_task_state->status); } }); } @@ -195,7 +195,7 @@ void LocalDependencyResolver::ResolveDependencies( } if (resolved_task_state) { - resolved_task_state->on_dependencies_resolved(resolved_task_state->status); + resolved_task_state->on_dependencies_resolved_(resolved_task_state->status); } }); } diff --git a/src/ray/core_worker/task_submission/dependency_resolver.h b/src/ray/core_worker/task_submission/dependency_resolver.h index 2d9624144017..aa625ba9a266 100644 --- a/src/ray/core_worker/task_submission/dependency_resolver.h +++ b/src/ray/core_worker/task_submission/dependency_resolver.h @@ -79,7 +79,7 @@ class LocalDependencyResolver { : task(std::move(t)), actor_dependencies_remaining(actor_ids.size()), status(Status::OK()), - on_dependencies_resolved(std::move(on_dependencies_resolved)) { + on_dependencies_resolved_(std::move(on_dependencies_resolved)) { local_dependencies.reserve(deps.size()); for (const auto &dep : deps) { local_dependencies.emplace(dep, /*ray_object=*/nullptr); @@ -97,7 +97,7 @@ class LocalDependencyResolver { size_t obj_dependencies_remaining; /// Dependency resolution status. Status status; - std::function on_dependencies_resolved; + std::function on_dependencies_resolved_; }; /// The in-memory store. diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 16bb82b9e69a..ad7619849889 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -342,9 +342,10 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli { absl::MutexLock lock(&mu_); - auto &scheduling_key_entry = scheduling_key_entries_[scheduling_key]; - auto raylet_client = raylet_client_pool_->GetOrConnectByAddress(raylet_address); - scheduling_key_entry.pending_lease_requests.erase(task_id); + auto &sched_entry = scheduling_key_entries_[scheduling_key]; + auto raylet_lease_client = + raylet_client_pool_->GetOrConnectByAddress(raylet_address); + sched_entry.pending_lease_requests.erase(task_id); if (status.ok()) { if (reply.canceled()) { @@ -387,9 +388,9 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli ", task_name=", task_name)); - tasks_to_fail = std::move(scheduling_key_entry.task_queue); - scheduling_key_entry.task_queue.clear(); - if (scheduling_key_entry.CanDelete()) { + tasks_to_fail = std::move(sched_entry.task_queue); + sched_entry.task_queue.clear(); + if (sched_entry.CanDelete()) { scheduling_key_entries_.erase(scheduling_key); } } else { @@ -412,11 +413,11 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli << WorkerID::FromBinary(reply.worker_address().worker_id()); AddWorkerLeaseClient(reply.worker_address(), - std::move(raylet_client), + std::move(raylet_lease_client), reply.resource_mapping(), scheduling_key, task_id); - RAY_CHECK(scheduling_key_entry.active_workers.size() >= 1); + RAY_CHECK(sched_entry.active_workers.size() >= 1); OnWorkerIdle(reply.worker_address(), scheduling_key, /*was_error=*/false, @@ -472,9 +473,9 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli << "because the raylet is " "unavailable (crashed)."; error_info.set_error_message(ss.str()); - tasks_to_fail = std::move(scheduling_key_entry.task_queue); - scheduling_key_entry.task_queue.clear(); - if (scheduling_key_entry.CanDelete()) { + tasks_to_fail = std::move(sched_entry.task_queue); + sched_entry.task_queue.clear(); + if (sched_entry.CanDelete()) { scheduling_key_entries_.erase(scheduling_key); } } else { @@ -574,7 +575,7 @@ void NormalTaskSubmitter::PushNormalTask( addr, get_task_failure_cause_reply_status, get_task_failure_cause_reply); - absl::MutexLock lock(&mu_); + absl::MutexLock task_submission_state_lock(&mu_); if (!will_retry) { // Task submission and task cancellation are the only two other code // paths that clean up the cancelled_tasks_ map. If the task is not diff --git a/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.cc b/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.cc index 75afa6274b71..61541d513624 100644 --- a/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.cc +++ b/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.cc @@ -20,8 +20,7 @@ namespace ray { namespace core { -OutofOrderActorSubmitQueue::OutofOrderActorSubmitQueue(ActorID actor_id) - : kActorId(actor_id) {} +OutofOrderActorSubmitQueue::OutofOrderActorSubmitQueue() {} void OutofOrderActorSubmitQueue::Emplace(uint64_t position, const TaskSpecification &spec) { diff --git a/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.h b/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.h index 26284014a300..3af1acba54d4 100644 --- a/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.h +++ b/src/ray/core_worker/task_submission/out_of_order_actor_submit_queue.h @@ -34,7 +34,7 @@ namespace core { */ class OutofOrderActorSubmitQueue : public IActorSubmitQueue { public: - explicit OutofOrderActorSubmitQueue(ActorID actor_id); + OutofOrderActorSubmitQueue(); /// Add a task into the queue. void Emplace(uint64_t position, const TaskSpecification &spec) override; /// If a task exists. @@ -60,7 +60,6 @@ class OutofOrderActorSubmitQueue : public IActorSubmitQueue { bool Empty() override; private: - ActorID kActorId; absl::btree_map> pending_queue_; absl::btree_map> sending_queue_; }; diff --git a/src/ray/core_worker/task_submission/sequential_actor_submit_queue.cc b/src/ray/core_worker/task_submission/sequential_actor_submit_queue.cc index dc81606fd79f..773df5c22f6b 100644 --- a/src/ray/core_worker/task_submission/sequential_actor_submit_queue.cc +++ b/src/ray/core_worker/task_submission/sequential_actor_submit_queue.cc @@ -19,8 +19,7 @@ namespace ray { namespace core { -SequentialActorSubmitQueue::SequentialActorSubmitQueue(ActorID actor_id) - : actor_id(actor_id) {} +SequentialActorSubmitQueue::SequentialActorSubmitQueue() {} void SequentialActorSubmitQueue::Emplace(uint64_t sequence_no, const TaskSpecification &spec) { diff --git a/src/ray/core_worker/task_submission/sequential_actor_submit_queue.h b/src/ray/core_worker/task_submission/sequential_actor_submit_queue.h index a971bdd9125a..f54c7f9a75be 100644 --- a/src/ray/core_worker/task_submission/sequential_actor_submit_queue.h +++ b/src/ray/core_worker/task_submission/sequential_actor_submit_queue.h @@ -31,7 +31,7 @@ namespace core { */ class SequentialActorSubmitQueue : public IActorSubmitQueue { public: - explicit SequentialActorSubmitQueue(ActorID actor_id); + SequentialActorSubmitQueue(); /// Add a task into the queue. void Emplace(uint64_t sequence_no, const TaskSpecification &task_spec) override; /// If a task exists. @@ -57,9 +57,6 @@ class SequentialActorSubmitQueue : public IActorSubmitQueue { bool Empty() override; private: - /// The ID of the actor. - ActorID actor_id; - /// The actor's pending requests, ordered by the sequence number in the request. /// The bool indicates whether the dependencies for that task have been resolved yet. /// A task will be sent after its dependencies are resolved. diff --git a/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc index 6aad5b7fc53a..09874520b038 100644 --- a/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc @@ -816,8 +816,8 @@ TEST_P(ActorTaskSubmitterTest, TestPendingTasks) { ASSERT_TRUE(submitter_.PendingTasksFull(actor_id)); // All the replies comes, the queue shouble be empty. - for (auto &task : tasks) { - ASSERT_TRUE(worker_client_->ReplyPushTask(task.GetTaskAttempt(), Status::OK())); + for (auto &task_spec : tasks) { + ASSERT_TRUE(worker_client_->ReplyPushTask(task_spec.GetTaskAttempt(), Status::OK())); } ASSERT_FALSE(submitter_.PendingTasksFull(actor_id)); } diff --git a/src/ray/core_worker/task_submission/tests/out_of_order_actor_submit_queue_test.cc b/src/ray/core_worker/task_submission/tests/out_of_order_actor_submit_queue_test.cc index eabbab200fa0..bbaefd7b780f 100644 --- a/src/ray/core_worker/task_submission/tests/out_of_order_actor_submit_queue_test.cc +++ b/src/ray/core_worker/task_submission/tests/out_of_order_actor_submit_queue_test.cc @@ -35,7 +35,7 @@ TaskSpecification BuildTaskSpec(uint64_t seq) { } // namespace TEST(OutofOrderActorSubmitQueueTest, PassThroughTest) { - OutofOrderActorSubmitQueue queue(ActorID{}); + OutofOrderActorSubmitQueue queue; // insert request 0 1 2 3 4 std::vector task_ids; for (uint64_t i = 0; i < 5; i++) { diff --git a/src/ray/core_worker/tests/task_event_buffer_test.cc b/src/ray/core_worker/tests/task_event_buffer_test.cc index 7616e6dac332..d117bed1560a 100644 --- a/src/ray/core_worker/tests/task_event_buffer_test.cc +++ b/src/ray/core_worker/tests/task_event_buffer_test.cc @@ -437,12 +437,12 @@ TEST_P(TaskEventBufferTestDifferentDestination, TestFlushEvents) { task_event->ToRpcRayEvents(ray_events_pair); auto [task_definition_event, task_execution_event] = ray_events_pair; if (task_definition_event) { - auto event = expected_ray_events_data.add_events(); - *event = std::move(task_definition_event.value()); + auto new_event = expected_ray_events_data.add_events(); + *new_event = std::move(task_definition_event.value()); } if (task_execution_event) { - auto event = expected_ray_events_data.add_events(); - *event = std::move(task_execution_event.value()); + auto new_event = expected_ray_events_data.add_events(); + *new_event = std::move(task_execution_event.value()); } } @@ -752,12 +752,12 @@ TEST_P(TaskEventBufferTestLimitBufferDifferentDestination, event->ToRpcRayEvents(ray_events_pair); auto [task_definition_event, task_execution_event] = ray_events_pair; if (task_definition_event) { - auto event = expected_ray_events_data.add_events(); - *event = std::move(task_definition_event.value()); + auto new_event = expected_ray_events_data.add_events(); + *new_event = std::move(task_definition_event.value()); } if (task_execution_event) { - auto event = expected_ray_events_data.add_events(); - *event = std::move(task_execution_event.value()); + auto new_event = expected_ray_events_data.add_events(); + *new_event = std::move(task_execution_event.value()); } } diff --git a/src/ray/gcs/gcs_client/accessor.cc b/src/ray/gcs/gcs_client/accessor.cc index da8142a4d4fb..1656f86eabdf 100644 --- a/src/ray/gcs/gcs_client/accessor.cc +++ b/src/ray/gcs/gcs_client/accessor.cc @@ -71,14 +71,15 @@ Status JobInfoAccessor::AsyncSubscribeAll( const SubscribeCallback &subscribe, const StatusCallback &done) { RAY_CHECK(subscribe != nullptr); - fetch_all_data_operation_ = [this, subscribe](const StatusCallback &done) { - auto callback = [subscribe, done](const Status &status, - std::vector &&job_info_list) { + fetch_all_data_operation_ = [this, subscribe](const StatusCallback &done_callback) { + auto callback = [subscribe, done_callback]( + const Status &status, + std::vector &&job_info_list) { for (auto &job_info : job_info_list) { subscribe(JobID::FromBinary(job_info.job_id()), std::move(job_info)); } - if (done) { - done(status); + if (done_callback) { + done_callback(status); } }; AsyncGetAll(/*job_or_submission_id=*/std::nullopt, @@ -87,8 +88,8 @@ Status JobInfoAccessor::AsyncSubscribeAll( callback, /*timeout_ms=*/-1); }; - subscribe_operation_ = [this, subscribe](const StatusCallback &done) { - return client_impl_->GetGcsSubscriber().SubscribeAllJobs(subscribe, done); + subscribe_operation_ = [this, subscribe](const StatusCallback &done_callback) { + return client_impl_->GetGcsSubscriber().SubscribeAllJobs(subscribe, done_callback); }; return subscribe_operation_( [this, done](const Status &status) { fetch_all_data_operation_(done); }); @@ -438,9 +439,9 @@ void ActorInfoAccessor::AsyncResubscribe() { // server first, then fetch data from the GCS server. absl::MutexLock lock(&mutex_); for (auto &[actor_id, resubscribe_op] : resubscribe_operations_) { - RAY_CHECK_OK(resubscribe_op([this, actor_id = actor_id](const Status &status) { - absl::MutexLock lock(&mutex_); - auto fetch_data_operation = fetch_data_operations_[actor_id]; + RAY_CHECK_OK(resubscribe_op([this, id = actor_id](const Status &status) { + absl::MutexLock callback_lock(&mutex_); + auto fetch_data_operation = fetch_data_operations_[id]; // `fetch_data_operation` is called in the callback function of subscribe. // Before that, if the user calls `AsyncUnsubscribe` function, the corresponding // fetch function will be deleted, so we need to check if it's null. @@ -633,15 +634,15 @@ void NodeInfoAccessor::AsyncSubscribeToNodeChange( node_change_callback_ = std::move(subscribe); RAY_CHECK(node_change_callback_ != nullptr); - fetch_node_data_operation_ = [this](const StatusCallback &done) { + fetch_node_data_operation_ = [this](const StatusCallback &done_callback) { AsyncGetAll( - [this, done](const Status &status, - std::vector &&node_info_list) { + [this, done_callback](const Status &status, + std::vector &&node_info_list) { for (auto &node_info : node_info_list) { HandleNotification(std::move(node_info)); } - if (done) { - done(status); + if (done_callback) { + done_callback(status); } }, /*timeout_ms=*/-1); @@ -890,8 +891,9 @@ WorkerInfoAccessor::WorkerInfoAccessor(GcsClient *client_impl) Status WorkerInfoAccessor::AsyncSubscribeToWorkerFailures( const ItemCallback &subscribe, const StatusCallback &done) { RAY_CHECK(subscribe != nullptr); - subscribe_operation_ = [this, subscribe](const StatusCallback &done) { - return client_impl_->GetGcsSubscriber().SubscribeAllWorkerFailures(subscribe, done); + subscribe_operation_ = [this, subscribe](const StatusCallback &done_callback) { + return client_impl_->GetGcsSubscriber().SubscribeAllWorkerFailures(subscribe, + done_callback); }; return subscribe_operation_(done); } @@ -1429,7 +1431,6 @@ void AutoscalerStateAccessor::AsyncGetClusterStatus( int64_t timeout_ms, const OptionalItemCallback &callback) { rpc::autoscaler::GetClusterStatusRequest request; - rpc::autoscaler::GetClusterStatusRequest reply; client_impl_->GetGcsRpcClient().GetClusterStatus( request, [callback](const Status &status, rpc::autoscaler::GetClusterStatusReply &&reply) { diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index bc84131b6c95..a895bbb13ad0 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -404,15 +404,15 @@ void GcsActorManager::HandleRegisterActor(rpc::RegisterActorRequest request, RAY_LOG(INFO).WithField(actor_id.JobId()).WithField(actor_id) << "Registering actor"; Status status = RegisterActor( - request, [reply, send_reply_callback, actor_id](const Status &status) { - if (status.ok()) { + request, [reply, send_reply_callback, actor_id](const Status ®ister_status) { + if (register_status.ok()) { RAY_LOG(INFO).WithField(actor_id.JobId()).WithField(actor_id) << "Registered actor"; } else { RAY_LOG(WARNING).WithField(actor_id.JobId()).WithField(actor_id) - << "Failed to register actor: " << status.ToString(); + << "Failed to register actor: " << register_status.ToString(); } - GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); + GCS_RPC_SEND_REPLY(send_reply_callback, reply, register_status); }); if (!status.ok()) { RAY_LOG(WARNING).WithField(actor_id.JobId()).WithField(actor_id) @@ -498,12 +498,15 @@ void GcsActorManager::HandleRestartActorForLineageReconstruction( // should overwrite the actor state to DEAD to avoid race condition. return; } - auto iter = actor_to_restart_for_lineage_reconstruction_callbacks_.find( - actor->GetActorID()); - RAY_CHECK(iter != actor_to_restart_for_lineage_reconstruction_callbacks_.end() && - !iter->second.empty()); - auto callbacks = std::move(iter->second); - actor_to_restart_for_lineage_reconstruction_callbacks_.erase(iter); + auto restart_callback_iter = + actor_to_restart_for_lineage_reconstruction_callbacks_.find( + actor->GetActorID()); + RAY_CHECK(restart_callback_iter != + actor_to_restart_for_lineage_reconstruction_callbacks_.end() && + !restart_callback_iter->second.empty()); + auto callbacks = std::move(restart_callback_iter->second); + actor_to_restart_for_lineage_reconstruction_callbacks_.erase( + restart_callback_iter); for (auto &callback : callbacks) { callback(actor); } @@ -840,11 +843,11 @@ Status GcsActorManager::RegisterActor(const ray::rpc::RegisterActorRequest &requ RAY_CHECK_OK(gcs_table_storage_->ActorTable().Put( actor->GetActorID(), *actor->GetMutableActorTableData(), - {[this, actor](Status status) { + {[this, actor](Status put_status) { RAY_CHECK(thread_checker_.IsOnSameThread()); // The backend storage is supposed to be reliable, so the status must be // ok. - RAY_CHECK_OK(status); + RAY_CHECK_OK(put_status); actor->WriteActorExportEvent(); auto registered_actor_it = registered_actors_.find(actor->GetActorID()); auto callback_iter = @@ -1033,12 +1036,12 @@ void GcsActorManager::PollOwnerForActorRefDeleted( auto client = worker_client_pool_.GetOrConnect(actor->GetOwnerAddress()); it = workers.emplace(owner_id, Owner(std::move(client))).first; } - it->second.children_actor_ids.insert(actor_id); + it->second.children_actor_ids_.insert(actor_id); rpc::WaitForActorRefDeletedRequest wait_request; wait_request.set_intended_worker_id(owner_id.Binary()); wait_request.set_actor_id(actor_id.Binary()); - it->second.client->WaitForActorRefDeleted( + it->second.client_->WaitForActorRefDeleted( wait_request, [this, owner_node_id, owner_id, actor_id]( Status status, const rpc::WaitForActorRefDeletedReply &reply) { @@ -1253,7 +1256,7 @@ void GcsActorManager::OnWorkerDead(const ray::NodeID &node_id, auto owner = it->second.find(worker_id); // Make a copy of the children actor IDs since we will delete from the // list. - const auto children_ids = owner->second.children_actor_ids; + const auto children_ids = owner->second.children_actor_ids_; for (const auto &child_id : children_ids) { DestroyActor(child_id, GenOwnerDiedCause(GetActor(child_id), @@ -1327,7 +1330,7 @@ void GcsActorManager::OnNodeDead(std::shared_ptr node, absl::flat_hash_map children_ids; // Make a copy of all the actor IDs owned by workers on the dead node. for (const auto &owner : it->second) { - for (const auto &child_id : owner.second.children_actor_ids) { + for (const auto &child_id : owner.second.children_actor_ids_) { children_ids.emplace(owner.first, child_id); } } @@ -1768,8 +1771,8 @@ void GcsActorManager::RemoveActorFromOwner(const std::shared_ptr &acto auto worker_it = node.find(owner_id); RAY_CHECK(worker_it != node.end()); auto &owner = worker_it->second; - RAY_CHECK(owner.children_actor_ids.erase(actor_id)); - if (owner.children_actor_ids.empty()) { + RAY_CHECK(owner.children_actor_ids_.erase(actor_id)); + if (owner.children_actor_ids_.empty()) { node.erase(worker_it); if (node.empty()) { owners_.erase(owner_node_id); @@ -1899,8 +1902,8 @@ bool GcsActorManager::RemovePendingActor(std::shared_ptr actor) { const auto &actor_id = actor->GetActorID(); auto pending_it = std::find_if(pending_actors_.begin(), pending_actors_.end(), - [actor_id](const std::shared_ptr &actor) { - return actor->GetActorID() == actor_id; + [actor_id](const std::shared_ptr &this_actor) { + return this_actor->GetActorID() == actor_id; }); // The actor was pending scheduling. Remove it from the queue. diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.h b/src/ray/gcs/gcs_server/gcs_actor_manager.h index 263683ece7ab..69a06df1f6cc 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.h +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.h @@ -525,11 +525,11 @@ class GcsActorManager : public rpc::ActorInfoHandler { /// A data structure representing an actor's owner. struct Owner { explicit Owner(std::shared_ptr client) - : client(std::move(client)) {} + : client_(std::move(client)) {} /// A client that can be used to contact the owner. - std::shared_ptr client; + std::shared_ptr client_; /// The IDs of actors owned by this worker. - absl::flat_hash_set children_actor_ids; + absl::flat_hash_set children_actor_ids_; }; /// Poll an actor's owner so that we will receive a notification when the diff --git a/src/ray/gcs/gcs_server/gcs_health_check_manager.cc b/src/ray/gcs/gcs_server/gcs_health_check_manager.cc index 239a18c2aa4c..109a313d5394 100644 --- a/src/ray/gcs/gcs_server/gcs_health_check_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_health_check_manager.cc @@ -173,8 +173,8 @@ void GcsHealthCheckManager::HealthCheckContext::StartHealthCheck() { response_ptr, [this, start = now, context = std::move(context), response = std::move(response)]( ::grpc::Status status) { - auto manager = manager_.lock(); - if (manager == nullptr) { + auto gcs_health_check_manager = manager_.lock(); + if (gcs_health_check_manager == nullptr) { delete this; return; } @@ -183,14 +183,14 @@ void GcsHealthCheckManager::HealthCheckContext::StartHealthCheck() { STATS_health_check_rpc_latency_ms.Record( absl::ToInt64Milliseconds(absl::Now() - start)); - manager->io_service_.post( + gcs_health_check_manager->io_service_.post( [this, status, response = std::move(response)]() { if (stopped_) { delete this; return; } - auto manager = manager_.lock(); - if (manager == nullptr) { + auto mgr = manager_.lock(); + if (mgr == nullptr) { delete this; return; } @@ -201,7 +201,7 @@ void GcsHealthCheckManager::HealthCheckContext::StartHealthCheck() { if (status.ok() && response->status() == HealthCheckResponse::SERVING) { // Health check passed. - health_check_remaining_ = manager->failure_threshold_; + health_check_remaining_ = mgr->failure_threshold_; } else { --health_check_remaining_; RAY_LOG(WARNING) @@ -213,15 +213,14 @@ void GcsHealthCheckManager::HealthCheckContext::StartHealthCheck() { } if (health_check_remaining_ == 0) { - manager->FailNode(node_id_); + mgr->FailNode(node_id_); delete this; } else { // Do another health check. // // TODO(hjiang): Able to reduce a few health check based on know resource // usage communication between GCS and raylet. - timer_.expires_from_now( - boost::posix_time::milliseconds(manager->period_ms_)); + timer_.expires_from_now(boost::posix_time::milliseconds(mgr->period_ms_)); timer_.async_wait([this](auto) { StartHealthCheck(); }); } }, diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.cc b/src/ray/gcs/gcs_server/gcs_job_manager.cc index 3b42b2119db2..a807da1d5ee6 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_job_manager.cc @@ -190,11 +190,11 @@ void GcsJobManager::HandleMarkJobFinished(rpc::MarkJobFinishedRequest request, Status status = gcs_table_storage_.JobTable().Get( job_id, - {[this, job_id, send_reply](Status status, + {[this, job_id, send_reply](Status get_status, std::optional result) { RAY_CHECK(thread_checker_.IsOnSameThread()); - if (status.ok() && result) { + if (get_status.ok() && result) { MarkJobAsFinished(*result, send_reply); return; } @@ -202,11 +202,11 @@ void GcsJobManager::HandleMarkJobFinished(rpc::MarkJobFinishedRequest request, if (!result.has_value()) { RAY_LOG(ERROR).WithField(job_id) << "Tried to mark job as finished, but no job table entry was found."; - } else if (!status.ok()) { + } else if (!get_status.ok()) { RAY_LOG(ERROR).WithField(job_id) - << "Failed to mark job as finished: " << status; + << "Failed to mark job as finished: " << get_status; } - send_reply(status); + send_reply(get_status); }, io_context_}); if (!status.ok()) { @@ -414,8 +414,8 @@ void GcsJobManager::HandleGetAllJobInfo(rpc::GetAllJobInfoRequest request, send_reply_callback, job_data_key_to_indices, num_finished_tasks, - try_send_reply](const auto &result) { - for (const auto &data : result) { + try_send_reply](const auto &job_info_result) { + for (const auto &data : job_info_result) { const std::string &job_data_key = data.first; // The JobInfo stored by the Ray Job API. const std::string &job_info_json = data.second; @@ -430,8 +430,8 @@ void GcsJobManager::HandleGetAllJobInfo(rpc::GetAllJobInfoRequest request, << job_info_json << " Error: " << status.message(); } // Add the JobInfo to the correct indices in the reply. - for (int i : job_data_key_to_indices.at(job_data_key)) { - reply->mutable_job_info_list(i)->mutable_job_info()->CopyFrom( + for (int j : job_data_key_to_indices.at(job_data_key)) { + reply->mutable_job_info_list(j)->mutable_job_info()->CopyFrom( jobs_api_info); } } diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc index 9e9694801586..71b200e2cd7c 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc @@ -270,11 +270,12 @@ void GcsPlacementGroupManager::RegisterPlacementGroup( // The backend storage is supposed to be reliable, so the status must be ok. RAY_CHECK_OK(status); if (registered_placement_groups_.contains(placement_group_id)) { - auto iter = placement_group_to_register_callbacks_.find(placement_group_id); - auto callbacks = std::move(iter->second); - placement_group_to_register_callbacks_.erase(iter); - for (const auto &callback : callbacks) { - callback(status); + auto register_callback_iter = + placement_group_to_register_callbacks_.find(placement_group_id); + auto callbacks = std::move(register_callback_iter->second); + placement_group_to_register_callbacks_.erase(register_callback_iter); + for (const auto ®ister_callback : callbacks) { + register_callback(status); } SchedulePendingPlacementGroups(); } else { @@ -442,14 +443,14 @@ void GcsPlacementGroupManager::SchedulePendingPlacementGroups() { gcs_placement_group_scheduler_->ScheduleUnplacedBundles(SchedulePgRequest{ /*placement_group=*/placement_group, /*failure_callback=*/ - [this, backoff](std::shared_ptr placement_group, + [this, backoff](std::shared_ptr failure_placement_group, bool is_feasible) { OnPlacementGroupCreationFailed( - std::move(placement_group), backoff, is_feasible); + std::move(failure_placement_group), backoff, is_feasible); }, /*success_callback=*/ - [this](std::shared_ptr placement_group) { - OnPlacementGroupCreationSuccess(placement_group); + [this](std::shared_ptr success_placement_group) { + OnPlacementGroupCreationSuccess(success_placement_group); }}); is_new_placement_group_scheduled = true; } @@ -550,8 +551,9 @@ void GcsPlacementGroupManager::RemovePlacementGroup( auto pending_it = std::find_if( infeasible_placement_groups_.begin(), infeasible_placement_groups_.end(), - [placement_group_id](const std::shared_ptr &placement_group) { - return placement_group->GetPlacementGroupID() == placement_group_id; + [placement_group_id]( + const std::shared_ptr &this_placement_group) { + return this_placement_group->GetPlacementGroupID() == placement_group_id; }); if (pending_it != infeasible_placement_groups_.end()) { // The placement group is infeasible now, remove it from the queue. @@ -1020,13 +1022,14 @@ void GcsPlacementGroupManager::Initialize(const GcsInitData &gcs_init_data) { prepared_pgs.emplace_back(SchedulePgRequest{ placement_group, /*failure_callback=*/ - [this](std::shared_ptr placement_group, bool is_feasible) { + [this](std::shared_ptr failure_placement_group, + bool is_feasible) { OnPlacementGroupCreationFailed( - std::move(placement_group), CreateDefaultBackoff(), is_feasible); + std::move(failure_placement_group), CreateDefaultBackoff(), is_feasible); }, /*success_callback=*/ - [this](std::shared_ptr placement_group) { - OnPlacementGroupCreationSuccess(placement_group); + [this](std::shared_ptr success_placement_group) { + OnPlacementGroupCreationSuccess(success_placement_group); }, }); } diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 44afef0fb65a..65402804ed86 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -187,7 +187,7 @@ void GcsServer::GetOrGenerateClusterId( {[this, continuation = std::move(continuation)]( std::optional provided_cluster_id) mutable { if (!provided_cluster_id.has_value()) { - instrumented_io_context &io_context = continuation.io_context(); + instrumented_io_context &io_ctx = continuation.io_context(); ClusterID cluster_id = ClusterID::FromRandom(); RAY_LOG(INFO).WithField(cluster_id) << "Generated new cluster ID."; kv_manager_->GetInstance().Put( @@ -202,7 +202,7 @@ void GcsServer::GetOrGenerateClusterId( .Dispatch("GcsServer.GetOrGenerateClusterId.continuation", cluster_id); }, - io_context}); + io_ctx}); } else { ClusterID cluster_id = ClusterID::FromBinary(provided_cluster_id.value()); RAY_LOG(INFO).WithField(cluster_id) diff --git a/src/ray/gcs/gcs_server/gcs_table_storage.cc b/src/ray/gcs/gcs_server/gcs_table_storage.cc index 49c0b7ca87df..c3bd5d226ae7 100644 --- a/src/ray/gcs/gcs_server/gcs_table_storage.cc +++ b/src/ray/gcs/gcs_server/gcs_table_storage.cc @@ -52,16 +52,16 @@ Status GcsTable::Get(const Key &key, Postable)> callback) { // We can't use TransformArg here because we need to return 2 arguments. return store_client_->AsyncGet( - table_name_, key.Binary(), std::move(callback).Rebind([](auto callback) { - return [callback = std::move(callback)](Status status, - std::optional result) { + table_name_, key.Binary(), std::move(callback).Rebind([](auto async_get_callback) { + return [cb = std::move(async_get_callback)](Status status, + std::optional result) { std::optional value; if (result) { Data data; data.ParseFromString(*result); value = std::move(data); } - callback(status, std::move(value)); + cb(status, std::move(value)); }; })); } diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc index 80f423f8ffb0..e639ac58fc4c 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc @@ -1164,12 +1164,13 @@ TEST_F(GcsActorManagerTest, TestGetAllActorInfoFilters) { create_actor_request.mutable_task_spec()->CopyFrom( registered_actor->GetCreationTaskSpecification().GetMessage()); std::vector> finished_actors; - Status status = gcs_actor_manager_->CreateActor( + Status create_status = gcs_actor_manager_->CreateActor( create_actor_request, [&finished_actors](const std::shared_ptr &actor, const rpc::PushTaskReply &reply, const Status &status) { finished_actors.emplace_back(actor); }); + ASSERT_TRUE(create_status.ok()); auto actor = mock_actor_scheduler_->actors.back(); mock_actor_scheduler_->actors.pop_back(); @@ -1187,9 +1188,9 @@ TEST_F(GcsActorManagerTest, TestGetAllActorInfoFilters) { auto request1 = Mocker::GenRegisterActorRequest(job_id_other, /*max_restarts=*/0, /*detached=*/false); - Status status = - gcs_actor_manager_->RegisterActor(request1, [](const Status &status) {}); - ASSERT_TRUE(status.ok()); + Status register_status = + gcs_actor_manager_->RegisterActor(request1, [](const Status &) {}); + ASSERT_TRUE(register_status.ok()); io_service_.run_one(); } diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h index c350f54a3d9d..43c2bb1a163d 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h +++ b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h @@ -132,7 +132,7 @@ struct GcsServerMocker { } bool GrantWorkerLease() { - return GrantWorkerLease("", 0, WorkerID::FromRandom(), node_id, NodeID::Nil()); + return GrantWorkerLease("", 0, WorkerID::FromRandom(), node_id_, NodeID::Nil()); } bool GrantWorkerLease(const std::string &address, @@ -299,7 +299,7 @@ struct GcsServerMocker { int num_leases_canceled = 0; int num_release_unused_workers = 0; int num_get_task_failure_causes = 0; - NodeID node_id = NodeID::FromRandom(); + NodeID node_id_ = NodeID::FromRandom(); std::list> drain_raylet_callbacks = {}; std::list> callbacks = {}; std::list> cancel_callbacks = {}; diff --git a/src/ray/gcs/store_client/redis_store_client.cc b/src/ray/gcs/store_client/redis_store_client.cc index 7fe2065478e9..2be5d86ee10f 100644 --- a/src/ray/gcs/store_client/redis_store_client.cc +++ b/src/ray/gcs/store_client/redis_store_client.cc @@ -427,9 +427,9 @@ void RedisStoreClient::RedisScanner::Scan() { // Scan by prefix from Redis. RedisCommand command = {"HSCAN", redis_key_, {std::to_string(cursor_.value())}}; - if (match_pattern_.escaped != "*") { + if (match_pattern_.escaped_ != "*") { command.args.push_back("MATCH"); - command.args.push_back(match_pattern_.escaped); + command.args.push_back(match_pattern_.escaped_); } command.args.push_back("COUNT"); command.args.push_back(std::to_string(batch_count)); @@ -567,7 +567,7 @@ bool RedisDelKeyPrefixSync(const std::string &host, // Delete all such keys by using empty table name. RedisKey redis_key{external_storage_namespace, /*table_name=*/""}; std::vector cmd{"KEYS", - RedisMatchPattern::Prefix(redis_key.ToString()).escaped}; + RedisMatchPattern::Prefix(redis_key.ToString()).escaped_}; std::promise> promise; context->RunArgvAsync(cmd, [&promise](const std::shared_ptr &reply) { promise.set_value(reply); @@ -581,12 +581,12 @@ bool RedisDelKeyPrefixSync(const std::string &host, } auto delete_one_sync = [&context](const std::string &key) { auto del_cmd = std::vector{"DEL", key}; - std::promise> promise; + std::promise> prom; context->RunArgvAsync(del_cmd, - [&promise](const std::shared_ptr &reply) { - promise.set_value(reply); + [&prom](const std::shared_ptr &callback_reply) { + prom.set_value(callback_reply); }); - auto del_reply = promise.get_future().get(); + auto del_reply = prom.get_future().get(); return del_reply->ReadAsInteger() > 0; }; size_t num_deleted = 0; diff --git a/src/ray/gcs/store_client/redis_store_client.h b/src/ray/gcs/store_client/redis_store_client.h index 241488fe7702..ec7b5535b3b5 100644 --- a/src/ray/gcs/store_client/redis_store_client.h +++ b/src/ray/gcs/store_client/redis_store_client.h @@ -49,10 +49,10 @@ struct RedisMatchPattern { static const RedisMatchPattern kAny("*"); return kAny; } - const std::string escaped; + const std::string escaped_; private: - explicit RedisMatchPattern(std::string escaped) : escaped(std::move(escaped)) {} + explicit RedisMatchPattern(std::string escaped) : escaped_(std::move(escaped)) {} }; struct RedisCommand { diff --git a/src/ray/ipc/client_connection.cc b/src/ray/ipc/client_connection.cc index 87be9cb3e0af..31b3df2c8b87 100644 --- a/src/ray/ipc/client_connection.cc +++ b/src/ray/ipc/client_connection.cc @@ -249,8 +249,8 @@ void ServerConnection::DoAsyncWrites() { } // Helper function to call all handlers with the input status. - auto call_handlers = [this](const ray::Status &status, int num_messages) { - for (int i = 0; i < num_messages; i++) { + auto call_handlers = [this](const ray::Status &status, int num_msgs) { + for (int i = 0; i < num_msgs; i++) { auto write_buffer = std::move(async_write_queue_.front()); write_buffer->handler(status); async_write_queue_.pop_front(); diff --git a/src/ray/object_manager/object_buffer_pool.cc b/src/ray/object_manager/object_buffer_pool.cc index e5f55b2266bf..00dcbe2b8564 100644 --- a/src/ray/object_manager/object_buffer_pool.cc +++ b/src/ray/object_manager/object_buffer_pool.cc @@ -107,14 +107,14 @@ ray::Status ObjectBufferPool::CreateChunk(const ObjectID &object_id, RAY_RETURN_NOT_OK(EnsureBufferExists( object_id, owner_address, data_size, metadata_size, chunk_index)); auto &state = create_buffer_state_.at(object_id); - if (chunk_index >= state.chunk_state.size()) { + if (chunk_index >= state.chunk_state_.size()) { return ray::Status::IOError("Object size mismatch"); } - if (state.chunk_state[chunk_index] != CreateChunkState::AVAILABLE) { + if (state.chunk_state_[chunk_index] != CreateChunkState::AVAILABLE) { // There can be only one reference to this chunk at any given time. return ray::Status::IOError("Chunk already received by a different thread."); } - state.chunk_state[chunk_index] = CreateChunkState::REFERENCED; + state.chunk_state_[chunk_index] = CreateChunkState::REFERENCED; return ray::Status::OK(); } @@ -128,35 +128,36 @@ void ObjectBufferPool::WriteChunk(const ObjectID &object_id, absl::MutexLock lock(&pool_mutex_); auto it = create_buffer_state_.find(object_id); if (it == create_buffer_state_.end() || - chunk_index >= it->second.chunk_state.size() || - it->second.chunk_state.at(chunk_index) != CreateChunkState::REFERENCED) { + chunk_index >= it->second.chunk_state_.size() || + it->second.chunk_state_.at(chunk_index) != CreateChunkState::REFERENCED) { RAY_LOG(DEBUG) << "Object " << object_id << " aborted before chunk " << chunk_index << " could be sealed"; return; } - if (it->second.data_size != data_size || it->second.metadata_size != metadata_size) { + if (it->second.data_size_ != data_size || + it->second.metadata_size_ != metadata_size) { RAY_LOG(DEBUG) << "Object " << object_id << " size mismatch, rejecting chunk"; return; } - RAY_CHECK(it->second.chunk_info.size() > chunk_index); + RAY_CHECK(it->second.chunk_info_.size() > chunk_index); - chunk_info = it->second.chunk_info.at(chunk_index); - RAY_CHECK(data.size() == chunk_info->buffer_length) + chunk_info = it->second.chunk_info_.at(chunk_index); + RAY_CHECK(data.size() == chunk_info->buffer_length_) << "size mismatch! data size: " << data.size() - << " chunk size: " << chunk_info->buffer_length; + << " chunk size: " << chunk_info->buffer_length_; // Update the state from REFERENCED To SEALED before releasing the lock to ensure // that no other thread sees a REFERENCED state. - it->second.chunk_state.at(chunk_index) = CreateChunkState::SEALED; + it->second.chunk_state_.at(chunk_index) = CreateChunkState::SEALED; // Increment the number of inflight copies to ensure Abort // does not release the buffer. - it->second.num_inflight_copies++; + it->second.num_inflight_copies_++; } RAY_CHECK(chunk_info.has_value()) << "chunk_info is not set"; // The num_inflight_copies is used to ensure that another thread cannot call Release // on the object_id, which makes the unguarded copy call safe. - std::memcpy(chunk_info->data, data.data(), chunk_info->buffer_length); + std::memcpy(chunk_info->data_, data.data(), chunk_info->buffer_length_); { // Ensure the process of object_id Seal and Release is mutex guarded. @@ -165,9 +166,9 @@ void ObjectBufferPool::WriteChunk(const ObjectID &object_id, // Abort cannot be called during inflight copy operations. RAY_CHECK(it != create_buffer_state_.end()); // Decrement the number of inflight copies to ensure Abort can release the buffer. - it->second.num_inflight_copies--; - it->second.num_seals_remaining--; - if (it->second.num_seals_remaining == 0) { + it->second.num_inflight_copies_--; + it->second.num_seals_remaining_--; + if (it->second.num_seals_remaining_ == 0) { RAY_CHECK_OK(store_client_->Seal(object_id)); RAY_CHECK_OK(store_client_->Release(object_id)); create_buffer_state_.erase(it); @@ -186,7 +187,7 @@ void ObjectBufferPool::AbortCreateInternal(const ObjectID &object_id) { auto no_copy_inflight = [this, object_id]() { pool_mutex_.AssertReaderHeld(); auto it = create_buffer_state_.find(object_id); - return it == create_buffer_state_.end() || it->second.num_inflight_copies == 0; + return it == create_buffer_state_.end() || it->second.num_inflight_copies_ == 0; }; pool_mutex_.Await(absl::Condition(&no_copy_inflight)); @@ -230,8 +231,8 @@ ray::Status ObjectBufferPool::EnsureBufferExists(const ObjectID &object_id, // Buffer for object_id already exists and the size matches ours. { auto it = create_buffer_state_.find(object_id); - if (it != create_buffer_state_.end() && it->second.data_size == data_size && - it->second.metadata_size == metadata_size) { + if (it != create_buffer_state_.end() && it->second.data_size_ == data_size && + it->second.metadata_size_ == metadata_size) { return ray::Status::OK(); } } @@ -258,10 +259,10 @@ ray::Status ObjectBufferPool::EnsureBufferExists(const ObjectID &object_id, { auto it = create_buffer_state_.find(object_id); if (it != create_buffer_state_.end()) { - RAY_CHECK(it->second.data_size != data_size || - it->second.metadata_size != metadata_size); + RAY_CHECK(it->second.data_size_ != data_size || + it->second.metadata_size_ != metadata_size); RAY_LOG(WARNING) << "Object " << object_id << " size (" << data_size - << ") differs from the original (" << it->second.data_size + << ") differs from the original (" << it->second.data_size_ << "). This is likely due to re-execution of a task with a " "nondeterministic output. Recreating object with size " << data_size << "."; @@ -317,7 +318,7 @@ ray::Status ObjectBufferPool::EnsureBufferExists(const ObjectID &object_id, std::forward_as_tuple(metadata_size, data_size, BuildChunks(object_id, mutable_data, data_size, data))); - RAY_CHECK(inserted.first->second.chunk_info.size() == num_chunks); + RAY_CHECK(inserted.first->second.chunk_info_.size() == num_chunks); RAY_LOG(DEBUG) << "Created object " << object_id << " in plasma store, number of chunks: " << num_chunks << ", chunk index: " << chunk_index; diff --git a/src/ray/object_manager/object_buffer_pool.h b/src/ray/object_manager/object_buffer_pool.h index 6951e7ff01fd..1a5345520120 100644 --- a/src/ray/object_manager/object_buffer_pool.h +++ b/src/ray/object_manager/object_buffer_pool.h @@ -39,18 +39,18 @@ class ObjectBufferPool { uint8_t *data, uint64_t buffer_length, std::shared_ptr buffer_ref) - : chunk_index(chunk_index), - data(data), - buffer_length(buffer_length), - buffer_ref(buffer_ref){}; + : chunk_index_(chunk_index), + data_(data), + buffer_length_(buffer_length), + buffer_ref_(buffer_ref){}; /// The index of this object chunk within the object, starting with 0. - uint64_t chunk_index; + uint64_t chunk_index_; /// A pointer to the start position of this object chunk. - uint8_t *data; + uint8_t *data_; /// The size of this object chunk. - uint64_t buffer_length; + uint64_t buffer_length_; /// A shared reference to the underlying buffer, keeping it alive. - std::shared_ptr buffer_ref; + std::shared_ptr buffer_ref_; }; /// Constructor. @@ -174,25 +174,25 @@ class ObjectBufferPool { CreateBufferState(uint64_t metadata_size, uint64_t data_size, std::vector chunk_info) - : metadata_size(metadata_size), - data_size(data_size), - chunk_info(chunk_info), - chunk_state(chunk_info.size(), CreateChunkState::AVAILABLE), - num_seals_remaining(chunk_info.size()) {} + : metadata_size_(metadata_size), + data_size_(data_size), + chunk_info_(chunk_info), + chunk_state_(chunk_info.size(), CreateChunkState::AVAILABLE), + num_seals_remaining_(chunk_info.size()) {} /// Total size of the object metadata. - uint64_t metadata_size; + uint64_t metadata_size_; /// Total size of the object data. - uint64_t data_size; + uint64_t data_size_; /// A vector maintaining information about the chunks which comprise /// an object. - std::vector chunk_info; + std::vector chunk_info_; /// The state of each chunk, which is used to enforce strict state /// transitions of each chunk. - std::vector chunk_state; + std::vector chunk_state_; /// The number of chunks left to seal before the buffer is sealed. - uint64_t num_seals_remaining; + uint64_t num_seals_remaining_; /// The number of inflight copy operations. - uint64_t num_inflight_copies = 0; + uint64_t num_inflight_copies_ = 0; }; /// Returned when GetChunk or CreateChunk fails. diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index 62fa2d51b2d0..53b18b32d11c 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -833,7 +833,8 @@ void ObjectManager::Tick(const boost::system::error_code &e) { auto interval = boost::posix_time::milliseconds(config_.timer_freq_ms); pull_retry_timer_.expires_from_now(interval); - pull_retry_timer_.async_wait([this](const boost::system::error_code &e) { Tick(e); }); + pull_retry_timer_.async_wait( + [this](const boost::system::error_code &err) { Tick(err); }); } } // namespace ray diff --git a/src/ray/object_manager/ownership_object_directory.cc b/src/ray/object_manager/ownership_object_directory.cc index 0abe8b4edd8e..a7046463e700 100644 --- a/src/ray/object_manager/ownership_object_directory.cc +++ b/src/ray/object_manager/ownership_object_directory.cc @@ -345,25 +345,25 @@ ray::Status OwnershipBasedObjectDirectory::SubscribeObjectLocations( auto failure_callback = [this, owner_address](const std::string &object_id_binary, const Status &status) { - const auto object_id = ObjectID::FromBinary(object_id_binary); + const auto obj_id = ObjectID::FromBinary(object_id_binary); rpc::WorkerObjectLocationsPubMessage location_info; if (!status.ok()) { - RAY_LOG(INFO).WithField(object_id) + RAY_LOG(INFO).WithField(obj_id) << "Failed to get the location: " << status.ToString(); - mark_as_failed_(object_id, rpc::ErrorType::OWNER_DIED); + mark_as_failed_(obj_id, rpc::ErrorType::OWNER_DIED); } else { // Owner is still alive but published a failure because the ref was // deleted. - RAY_LOG(INFO).WithField(object_id) + RAY_LOG(INFO).WithField(obj_id) << "Failed to get the location for object, already released by distributed " "reference counting protocol"; - mark_as_failed_(object_id, rpc::ErrorType::OBJECT_DELETED); + mark_as_failed_(obj_id, rpc::ErrorType::OBJECT_DELETED); } // Location lookup can fail if the owner is reachable but no longer has a // record of this ObjectRef, most likely due to an issue with the // distributed reference counting protocol. ObjectLocationSubscriptionCallback(location_info, - object_id, + obj_id, /*location_lookup_failed*/ true); }; diff --git a/src/ray/object_manager/plasma/common.h b/src/ray/object_manager/plasma/common.h index a9f2128a01e2..3796905b5d72 100644 --- a/src/ray/object_manager/plasma/common.h +++ b/src/ray/object_manager/plasma/common.h @@ -58,19 +58,19 @@ inline constexpr std::string_view kCorruptedRequestErrorMessage = // Represents a chunk of allocated memory. struct Allocation { /// Pointer to the allocated memory. - void *address; + void *address_; /// Num bytes of the allocated memory. - int64_t size; + int64_t size_; /// The file descriptor of the memory mapped file where the memory allocated. - MEMFD_TYPE fd; + MEMFD_TYPE fd_; /// The offset in bytes in the memory mapped file of the allocated memory. - ptrdiff_t offset; + ptrdiff_t offset_; /// Device number of the allocated memory. - int device_num; + int device_num_; /// the total size of this mapped memory. - int64_t mmap_size; + int64_t mmap_size_; /// if it was fallback allocated. - bool fallback_allocated; + bool fallback_allocated_; // only allow moves. RAY_DISALLOW_COPY_AND_ASSIGN(Allocation); @@ -86,23 +86,23 @@ struct Allocation { int device_num, int64_t mmap_size, bool fallback_allocated) - : address(address), - size(size), - fd(std::move(fd)), - offset(offset), - device_num(device_num), - mmap_size(mmap_size), - fallback_allocated(fallback_allocated) {} + : address_(address), + size_(size), + fd_(std::move(fd)), + offset_(offset), + device_num_(device_num), + mmap_size_(mmap_size), + fallback_allocated_(fallback_allocated) {} // Test only Allocation() - : address(nullptr), - size(0), - fd(), - offset(0), - device_num(0), - mmap_size(0), - fallback_allocated(false) {} + : address_(nullptr), + size_(0), + fd_(), + offset_(0), + device_num_(0), + mmap_size_(0), + fallback_allocated_(false) {} friend class PlasmaAllocator; friend class DummyAllocator; @@ -120,21 +120,21 @@ class LocalObject { RAY_DISALLOW_COPY_AND_ASSIGN(LocalObject); - int64_t GetObjectSize() const { return object_info.GetObjectSize(); } + int64_t GetObjectSize() const { return object_info_.GetObjectSize(); } - bool Sealed() const { return state == ObjectState::PLASMA_SEALED; } + bool Sealed() const { return state_ == ObjectState::PLASMA_SEALED; } - int32_t GetRefCount() const { return ref_count; } + int32_t GetRefCount() const { return ref_count_; } - const ray::ObjectInfo &GetObjectInfo() const { return object_info; } + const ray::ObjectInfo &GetObjectInfo() const { return object_info_; } - const Allocation &GetAllocation() const { return allocation; } + const Allocation &GetAllocation() const { return allocation_; } - const plasma::flatbuf::ObjectSource &GetSource() const { return source; } + const plasma::flatbuf::ObjectSource &GetSource() const { return source_; } ray::PlasmaObjectHeader *GetPlasmaObjectHeader() const { - RAY_CHECK(object_info.is_mutable) << "Object is not mutable"; - auto header_ptr = static_cast(allocation.address); + RAY_CHECK(object_info_.is_mutable) << "Object is not mutable"; + auto header_ptr = static_cast(allocation_.address_); return reinterpret_cast(header_ptr); } @@ -143,11 +143,11 @@ class LocalObject { if (check_sealed) { RAY_DCHECK(Sealed()); } - object->store_fd = GetAllocation().fd; - object->header_offset = GetAllocation().offset; - object->data_offset = GetAllocation().offset; - object->metadata_offset = GetAllocation().offset + GetObjectInfo().data_size; - if (object_info.is_mutable) { + object->store_fd = GetAllocation().fd_; + object->header_offset = GetAllocation().offset_; + object->data_offset = GetAllocation().offset_; + object->metadata_offset = GetAllocation().offset_ + GetObjectInfo().data_size; + if (object_info_.is_mutable) { object->data_offset += sizeof(ray::PlasmaObjectHeader); object->metadata_offset += sizeof(ray::PlasmaObjectHeader); }; @@ -157,10 +157,10 @@ class LocalObject { // sizes locally depending on what data is written to the channel, but the // plasma store keeps the original data and metadata size. object->allocated_size = object->data_size + object->metadata_size; - object->device_num = GetAllocation().device_num; - object->mmap_size = GetAllocation().mmap_size; - object->fallback_allocated = GetAllocation().fallback_allocated; - object->is_experimental_mutable_object = object_info.is_mutable; + object->device_num = GetAllocation().device_num_; + object->mmap_size = GetAllocation().mmap_size_; + object->fallback_allocated = GetAllocation().fallback_allocated_; + object->is_experimental_mutable_object = object_info_.is_mutable; } private: @@ -174,19 +174,19 @@ class LocalObject { friend struct GetRequestQueueTest; /// Allocation Info; - Allocation allocation; + Allocation allocation_; /// Ray object info; - ray::ObjectInfo object_info; + ray::ObjectInfo object_info_; /// Number of clients currently using this object. /// TODO: ref_count probably shouldn't belong to LocalObject. - mutable int32_t ref_count; + mutable int32_t ref_count_; /// Unix epoch of when this object was created. - int64_t create_time; + int64_t create_time_; /// How long creation of this object took. - int64_t construct_duration; + int64_t construct_duration_; /// The state of the object, e.g., whether it is open or sealed. - ObjectState state; + ObjectState state_; /// The source of the object. Used for debugging purposes. - plasma::flatbuf::ObjectSource source; + plasma::flatbuf::ObjectSource source_; }; } // namespace plasma diff --git a/src/ray/object_manager/plasma/create_request_queue.cc b/src/ray/object_manager/plasma/create_request_queue.cc index 1ed969efa69d..3c9b9b55451c 100644 --- a/src/ray/object_manager/plasma/create_request_queue.cc +++ b/src/ray/object_manager/plasma/create_request_queue.cc @@ -55,8 +55,8 @@ bool CreateRequestQueue::GetRequestResult(uint64_t req_id, return false; } - *result = it->second->result; - *error = it->second->error; + *result = it->second->result_; + *error = it->second->error_; fulfilled_requests_.erase(it); return true; } @@ -75,8 +75,8 @@ std::pair CreateRequestQueue::TryRequestImmediately( Status CreateRequestQueue::ProcessRequest(bool fallback_allocator, std::unique_ptr &request) { - request->error = request->create_callback(fallback_allocator, &request->result); - if (request->error == PlasmaError::OutOfMemory) { + request->error_ = request->create_callback_(fallback_allocator, &request->result_); + if (request->error_ == PlasmaError::OutOfMemory) { return Status::ObjectStoreFull(""); } else { return Status::OK(); @@ -92,10 +92,11 @@ Status CreateRequestQueue::ProcessRequests() { // if allocation failed due to OOM, and fs_monitor_ indicates the local disk is full, // we should failed the request with out of disk error - if ((*request_it)->error == PlasmaError::OutOfMemory && fs_monitor_.OverCapacity()) { - (*request_it)->error = PlasmaError::OutOfDisk; - RAY_LOG(INFO) << "Out-of-disk: Failed to create object " << (*request_it)->object_id - << " of size " << (*request_it)->object_size / 1024 / 1024 << "MB\n"; + if ((*request_it)->error_ == PlasmaError::OutOfMemory && fs_monitor_.OverCapacity()) { + (*request_it)->error_ = PlasmaError::OutOfDisk; + RAY_LOG(INFO) << "Out-of-disk: Failed to create object " + << (*request_it)->object_id_ << " of size " + << (*request_it)->object_size_ / 1024 / 1024 << "MB\n"; FinishRequest(request_it); return Status::OutOfDisk("System running out of disk."); } @@ -132,15 +133,15 @@ Status CreateRequestQueue::ProcessRequests() { if (!status.ok()) { // This only happens when an allocation is bigger than available disk space. // We should throw OutOfDisk Error here. - (*request_it)->error = PlasmaError::OutOfDisk; + (*request_it)->error_ = PlasmaError::OutOfDisk; std::string dump = ""; if (dump_debug_info_callback_ && !logged_oom) { dump = dump_debug_info_callback_(); logged_oom = true; } RAY_LOG(INFO) << "Out-of-disk: Failed to create object " - << (*request_it)->object_id << " of size " - << (*request_it)->object_size / 1024 / 1024 << "MB\n" + << (*request_it)->object_id_ << " of size " + << (*request_it)->object_size_ / 1024 / 1024 << "MB\n" << dump; } FinishRequest(request_it); @@ -154,22 +155,22 @@ void CreateRequestQueue::FinishRequest( std::list>::iterator request_it) { // Fulfill the request. auto &request = *request_it; - auto it = fulfilled_requests_.find(request->request_id); + auto it = fulfilled_requests_.find(request->request_id_); RAY_CHECK(it != fulfilled_requests_.end()); RAY_CHECK(it->second == nullptr); it->second = std::move(request); - RAY_CHECK(num_bytes_pending_ >= it->second->object_size); - num_bytes_pending_ -= it->second->object_size; + RAY_CHECK(num_bytes_pending_ >= it->second->object_size_); + num_bytes_pending_ -= it->second->object_size_; queue_.erase(request_it); } void CreateRequestQueue::RemoveDisconnectedClientRequests( const std::shared_ptr &client) { for (auto it = queue_.begin(); it != queue_.end();) { - if ((*it)->client == client) { - fulfilled_requests_.erase((*it)->request_id); - RAY_CHECK(num_bytes_pending_ >= (*it)->object_size); - num_bytes_pending_ -= (*it)->object_size; + if ((*it)->client_ == client) { + fulfilled_requests_.erase((*it)->request_id_); + RAY_CHECK(num_bytes_pending_ >= (*it)->object_size_); + num_bytes_pending_ -= (*it)->object_size_; it = queue_.erase(it); } else { it++; @@ -177,7 +178,7 @@ void CreateRequestQueue::RemoveDisconnectedClientRequests( } for (auto it = fulfilled_requests_.begin(); it != fulfilled_requests_.end();) { - if (it->second && it->second->client == client) { + if (it->second && it->second->client_ == client) { fulfilled_requests_.erase(it++); } else { it++; diff --git a/src/ray/object_manager/plasma/create_request_queue.h b/src/ray/object_manager/plasma/create_request_queue.h index 607443e67ba7..80ca6092d54a 100644 --- a/src/ray/object_manager/plasma/create_request_queue.h +++ b/src/ray/object_manager/plasma/create_request_queue.h @@ -126,32 +126,32 @@ class CreateRequestQueue { const std::shared_ptr &client, CreateObjectCallback create_callback, size_t object_size) - : object_id(object_id), - request_id(request_id), - client(client), - create_callback(create_callback), - object_size(object_size) {} + : object_id_(object_id), + request_id_(request_id), + client_(client), + create_callback_(create_callback), + object_size_(object_size) {} // The ObjectID to create. - const ObjectID object_id; + const ObjectID object_id_; // A request ID that can be returned to the caller to get the result once // ready. - const uint64_t request_id; + const uint64_t request_id_; // A pointer to the client, used as a key to delete requests that were made // by a client that is now disconnected. - const std::shared_ptr client; + const std::shared_ptr client_; // A callback to attempt to create the object. - const CreateObjectCallback create_callback; + const CreateObjectCallback create_callback_; - const size_t object_size; + const size_t object_size_; // The results of the creation call. These should be sent back to the // client once ready. - PlasmaError error = PlasmaError::OK; - PlasmaObject result = {}; + PlasmaError error_ = PlasmaError::OK; + PlasmaObject result_ = {}; }; /// Process a single request. Sets the request's error result to the error diff --git a/src/ray/object_manager/plasma/get_request_queue.cc b/src/ray/object_manager/plasma/get_request_queue.cc index 286440f3778b..097c71061f68 100644 --- a/src/ray/object_manager/plasma/get_request_queue.cc +++ b/src/ray/object_manager/plasma/get_request_queue.cc @@ -23,11 +23,11 @@ GetRequest::GetRequest(instrumented_io_context &io_context, const std::shared_ptr &client, const std::vector &object_ids, int64_t num_unique_objects_to_wait_for) - : client(client), - object_ids(object_ids.begin(), object_ids.end()), - objects(object_ids.size()), - num_unique_objects_to_wait_for(num_unique_objects_to_wait_for), - num_unique_objects_satisfied(0), + : client_(client), + object_ids_(object_ids.begin(), object_ids.end()), + objects_(object_ids.size()), + num_unique_objects_to_wait_for_(num_unique_objects_to_wait_for), + num_unique_objects_satisfied_(0), timer_(io_context) {} void GetRequest::AsyncWait( @@ -64,20 +64,20 @@ void GetRequestQueue::AddRequest(const std::shared_ptr &client, auto entry = object_lifecycle_mgr_.GetObject(object_id); if (entry != nullptr && entry->Sealed()) { // Update the get request to take into account the present object. - auto *plasma_object = &get_request->objects[object_id]; + auto *plasma_object = &get_request->objects_[object_id]; entry->ToPlasmaObject(plasma_object, /* checksealed */ true); - get_request->num_unique_objects_satisfied += 1; + get_request->num_unique_objects_satisfied_ += 1; std::optional fallback_allocated_fd = std::nullopt; - if (entry->GetAllocation().fallback_allocated) { - fallback_allocated_fd = entry->GetAllocation().fd; + if (entry->GetAllocation().fallback_allocated_) { + fallback_allocated_fd = entry->GetAllocation().fd_; } object_satisfied_callback_(object_id, fallback_allocated_fd, get_request); } else { // Add a placeholder plasma object to the get request to indicate that the // object is not present. This will be parsed by the client. We set the // data size to -1 to indicate that the object is not present. - get_request->objects[object_id].data_size = -1; + get_request->objects_[object_id].data_size = -1; // Add the get request to the relevant data structures. object_get_requests_[object_id].push_back(get_request); } @@ -85,8 +85,8 @@ void GetRequestQueue::AddRequest(const std::shared_ptr &client, // If all of the objects are present already or if the timeout is 0, return to // the client. - if (get_request->num_unique_objects_satisfied == - get_request->num_unique_objects_to_wait_for || + if (get_request->num_unique_objects_satisfied_ == + get_request->num_unique_objects_to_wait_for_ || timeout_ms == 0) { OnGetRequestCompleted(get_request); } else if (timeout_ms != -1) { @@ -108,7 +108,7 @@ void GetRequestQueue::RemoveGetRequestsForClient( absl::flat_hash_set> get_requests_to_remove; for (auto const &pair : object_get_requests_) { for (const auto &get_request : pair.second) { - if (get_request->client == client) { + if (get_request->client_ == client) { get_requests_to_remove.insert(get_request); } } @@ -133,7 +133,7 @@ void GetRequestQueue::RemoveGetRequest(const std::shared_ptr &get_re // Remove the get request from each of the relevant object_get_requests hash // tables if it is present there. It should only be present there if the get // request timed out or if it was issued by a client that has disconnected. - for (const auto &object_id : get_request->object_ids) { + for (const auto &object_id : get_request->object_ids_) { auto object_request_iter = object_get_requests_.find(object_id); if (object_request_iter != object_get_requests_.end()) { auto &get_requests = object_request_iter->second; @@ -170,18 +170,18 @@ void GetRequestQueue::MarkObjectSealed(const ObjectID &object_id) { auto get_request = get_requests[index]; auto entry = object_lifecycle_mgr_.GetObject(object_id); RAY_CHECK(entry != nullptr); - auto *plasma_object = &get_request->objects[object_id]; + auto *plasma_object = &get_request->objects_[object_id]; entry->ToPlasmaObject(plasma_object, /* check sealed */ true); - get_request->num_unique_objects_satisfied += 1; + get_request->num_unique_objects_satisfied_ += 1; std::optional fallback_allocated_fd = std::nullopt; - if (entry->GetAllocation().fallback_allocated) { - fallback_allocated_fd = entry->GetAllocation().fd; + if (entry->GetAllocation().fallback_allocated_) { + fallback_allocated_fd = entry->GetAllocation().fd_; } object_satisfied_callback_(object_id, fallback_allocated_fd, get_request); // If this get request is done, reply to the client. - if (get_request->num_unique_objects_satisfied == - get_request->num_unique_objects_to_wait_for) { + if (get_request->num_unique_objects_satisfied_ == + get_request->num_unique_objects_to_wait_for_) { OnGetRequestCompleted(get_request); } else { // The call to ReturnFromGet will remove the current element in the diff --git a/src/ray/object_manager/plasma/get_request_queue.h b/src/ray/object_manager/plasma/get_request_queue.h index 96387999d2d8..c9585f279a62 100644 --- a/src/ray/object_manager/plasma/get_request_queue.h +++ b/src/ray/object_manager/plasma/get_request_queue.h @@ -37,17 +37,17 @@ struct GetRequest { const std::vector &object_ids, int64_t num_unique_objects_to_wait_for); /// The client that called get. - std::shared_ptr client; + std::shared_ptr client_; /// The object IDs involved in this request. This is used in the reply. - std::vector object_ids; + std::vector object_ids_; /// The object information for the objects in this request. This is used in /// the reply. - absl::flat_hash_map objects; + absl::flat_hash_map objects_; /// The minimum number of objects to wait for in this request. - const int64_t num_unique_objects_to_wait_for; + const int64_t num_unique_objects_to_wait_for_; /// The number of object requests in this wait request that are already /// satisfied. - int64_t num_unique_objects_satisfied; + int64_t num_unique_objects_satisfied_; void AsyncWait(int64_t timeout_ms, std::function on_timeout); diff --git a/src/ray/object_manager/plasma/obj_lifecycle_mgr.cc b/src/ray/object_manager/plasma/obj_lifecycle_mgr.cc index c49eec1e473d..66b426365265 100644 --- a/src/ray/object_manager/plasma/obj_lifecycle_mgr.cc +++ b/src/ray/object_manager/plasma/obj_lifecycle_mgr.cc @@ -76,12 +76,12 @@ flatbuf::PlasmaError ObjectLifecycleManager::AbortObject(const ObjectID &object_ RAY_LOG(ERROR) << "To abort an object it must be in the object table."; return PlasmaError::ObjectNonexistent; } - if (entry->state == ObjectState::PLASMA_SEALED) { + if (entry->state_ == ObjectState::PLASMA_SEALED) { RAY_LOG(ERROR) << "To abort an object it must not have been sealed."; return PlasmaError::ObjectSealed; } - bool abort_while_using = entry->ref_count > 0; + bool abort_while_using = entry->ref_count_ > 0; DeleteObjectInternal(object_id); if (abort_while_using) { @@ -98,7 +98,7 @@ PlasmaError ObjectLifecycleManager::DeleteObject(const ObjectID &object_id) { } // TODO(scv119): should we delete unsealed with ref_count 0? - if (entry->state != ObjectState::PLASMA_SEALED) { + if (entry->state_ != ObjectState::PLASMA_SEALED) { // To delete an object it must have been sealed, // otherwise there might be memeory corruption. // Put it into deletion cache, it will be deleted later. @@ -106,7 +106,7 @@ PlasmaError ObjectLifecycleManager::DeleteObject(const ObjectID &object_id) { return PlasmaError::ObjectNotSealed; } - if (entry->ref_count != 0) { + if (entry->ref_count_ != 0) { // To delete an object, there must be no clients currently using it. // Put it into deletion cache, it will be deleted later. earger_deletion_objects_.emplace(object_id); @@ -133,12 +133,12 @@ bool ObjectLifecycleManager::AddReference(const ObjectID &object_id) { } // If there are no other clients using this object, notify the eviction policy // that the object is being used. - if (entry->ref_count == 0) { + if (entry->ref_count_ == 0) { // Tell the eviction policy that this object is being used. eviction_policy_->BeginObjectAccess(object_id); } // Increase reference count. - entry->ref_count++; + entry->ref_count_++; stats_collector_->OnObjectRefIncreased(*entry); RAY_LOG(DEBUG) << "Object " << object_id << " reference has incremented" << ", num bytes in use is now " << GetNumBytesInUse(); @@ -147,17 +147,17 @@ bool ObjectLifecycleManager::AddReference(const ObjectID &object_id) { bool ObjectLifecycleManager::RemoveReference(const ObjectID &object_id) { auto entry = object_store_->GetObject(object_id); - if (!entry || entry->ref_count == 0) { + if (!entry || entry->ref_count_ == 0) { RAY_LOG(ERROR) << object_id << " doesn't exist, or its ref count is already 0, remove reference failed."; return false; } - entry->ref_count--; + entry->ref_count_--; stats_collector_->OnObjectRefDecreased(*entry); - if (entry->ref_count > 0) { + if (entry->ref_count_ > 0) { return true; } @@ -231,9 +231,9 @@ void ObjectLifecycleManager::EvictObjects(const std::vector &object_id // error. Maybe we should also support deleting objects that have been // created but not sealed. RAY_CHECK(entry != nullptr) << "To evict an object it must be in the object table."; - RAY_CHECK(entry->state == ObjectState::PLASMA_SEALED) + RAY_CHECK(entry->state_ == ObjectState::PLASMA_SEALED) << "To evict an object it must have been sealed."; - RAY_CHECK(entry->ref_count == 0) + RAY_CHECK(entry->ref_count_ == 0) << "To evict an object, there must be no clients currently using it."; DeleteObjectInternal(object_id); @@ -244,7 +244,7 @@ void ObjectLifecycleManager::DeleteObjectInternal(const ObjectID &object_id) { auto entry = object_store_->GetObject(object_id); RAY_CHECK(entry != nullptr); - bool aborted = entry->state == ObjectState::PLASMA_CREATED; + bool aborted = entry->state_ == ObjectState::PLASMA_CREATED; stats_collector_->OnObjectDeleting(*entry); earger_deletion_objects_.erase(object_id); @@ -263,7 +263,7 @@ int64_t ObjectLifecycleManager::GetNumBytesInUse() const { bool ObjectLifecycleManager::IsObjectSealed(const ObjectID &object_id) const { auto entry = GetObject(object_id); - return entry && entry->state == ObjectState::PLASMA_SEALED; + return entry && entry->state_ == ObjectState::PLASMA_SEALED; } int64_t ObjectLifecycleManager::GetNumObjectsCreatedTotal() const { diff --git a/src/ray/object_manager/plasma/object_store.cc b/src/ray/object_manager/plasma/object_store.cc index a2f952ffc6e4..ee7639c48bc1 100644 --- a/src/ray/object_manager/plasma/object_store.cc +++ b/src/ray/object_manager/plasma/object_store.cc @@ -43,11 +43,11 @@ const LocalObject *ObjectStore::CreateObject(const ray::ObjectInfo &object_info, auto ptr = std::make_unique(std::move(allocation.value())); auto entry = object_table_.emplace(object_info.object_id, std::move(ptr)).first->second.get(); - entry->object_info = object_info; - entry->state = ObjectState::PLASMA_CREATED; - entry->create_time = std::time(nullptr); - entry->construct_duration = -1; - entry->source = source; + entry->object_info_ = object_info; + entry->state_ = ObjectState::PLASMA_CREATED; + entry->create_time_ = std::time(nullptr); + entry->construct_duration_ = -1; + entry->source_ = source; #if defined(__APPLE__) || defined(__linux__) if (object_info.is_mutable) { @@ -70,11 +70,11 @@ const LocalObject *ObjectStore::GetObject(const ObjectID &object_id) const { const LocalObject *ObjectStore::SealObject(const ObjectID &object_id) { auto entry = GetMutableObject(object_id); - if (entry == nullptr || entry->state == ObjectState::PLASMA_SEALED) { + if (entry == nullptr || entry->state_ == ObjectState::PLASMA_SEALED) { return nullptr; } - entry->state = ObjectState::PLASMA_SEALED; - entry->construct_duration = std::time(nullptr) - entry->create_time; + entry->state_ = ObjectState::PLASMA_SEALED; + entry->construct_duration_ = std::time(nullptr) - entry->create_time_; return entry; } @@ -83,7 +83,7 @@ bool ObjectStore::DeleteObject(const ObjectID &object_id) { if (entry == nullptr) { return false; } - allocator_.Free(std::move(entry->allocation)); + allocator_.Free(std::move(entry->allocation_)); object_table_.erase(object_id); return true; } diff --git a/src/ray/object_manager/plasma/plasma.cc b/src/ray/object_manager/plasma/plasma.cc index 84182fc5e0f3..04bf03779d94 100644 --- a/src/ray/object_manager/plasma/plasma.cc +++ b/src/ray/object_manager/plasma/plasma.cc @@ -24,5 +24,5 @@ namespace plasma { LocalObject::LocalObject(Allocation allocation) - : allocation(std::move(allocation)), ref_count(0) {} + : allocation_(std::move(allocation)), ref_count_(0) {} } // namespace plasma diff --git a/src/ray/object_manager/plasma/plasma_allocator.cc b/src/ray/object_manager/plasma/plasma_allocator.cc index 840b3c4599d2..6127fd146fb1 100644 --- a/src/ray/object_manager/plasma/plasma_allocator.cc +++ b/src/ray/object_manager/plasma/plasma_allocator.cc @@ -120,12 +120,12 @@ std::optional PlasmaAllocator::FallbackAllocate(size_t bytes) { } void PlasmaAllocator::Free(Allocation allocation) { - RAY_CHECK(allocation.address != nullptr) << "Cannot free the nullptr"; - RAY_LOG(DEBUG) << "deallocating " << allocation.size << " at " << allocation.address; - dlfree(allocation.address); - allocated_ -= allocation.size; - if (internal::IsOutsideInitialAllocation(allocation.address)) { - fallback_allocated_ -= allocation.size; + RAY_CHECK(allocation.address_ != nullptr) << "Cannot free the nullptr"; + RAY_LOG(DEBUG) << "deallocating " << allocation.size_ << " at " << allocation.address_; + dlfree(allocation.address_); + allocated_ -= allocation.size_; + if (internal::IsOutsideInitialAllocation(allocation.address_)) { + fallback_allocated_ -= allocation.size_; } } diff --git a/src/ray/object_manager/plasma/stats_collector.cc b/src/ray/object_manager/plasma/stats_collector.cc index aa3b95a617b1..dd9584b85727 100644 --- a/src/ray/object_manager/plasma/stats_collector.cc +++ b/src/ray/object_manager/plasma/stats_collector.cc @@ -27,7 +27,7 @@ void ObjectStatsCollector::OnObjectCreated(const LocalObject &obj) { const auto &kAllocation = obj.GetAllocation(); bytes_by_loc_seal_.Increment( - {/*fallback_allocated*/ kAllocation.fallback_allocated, /*sealed*/ false}, + {/*fallback_allocated*/ kAllocation.fallback_allocated_, /*sealed*/ false}, kObjectSize); num_objects_created_total_ += 1; @@ -65,8 +65,8 @@ void ObjectStatsCollector::OnObjectSealed(const LocalObject &obj) { const auto kObjectSize = obj.GetObjectInfo().GetObjectSize(); const auto &kAllocation = obj.GetAllocation(); - bytes_by_loc_seal_.Swap({kAllocation.fallback_allocated, /* sealed */ false}, - {kAllocation.fallback_allocated, /* sealed */ true}, + bytes_by_loc_seal_.Swap({kAllocation.fallback_allocated_, /* sealed */ false}, + {kAllocation.fallback_allocated_, /* sealed */ true}, kObjectSize); num_objects_unsealed_--; @@ -91,7 +91,7 @@ void ObjectStatsCollector::OnObjectDeleting(const LocalObject &obj) { const auto kSource = obj.GetSource(); const auto &kAllocation = obj.GetAllocation(); - bytes_by_loc_seal_.Decrement({kAllocation.fallback_allocated, obj.Sealed()}, + bytes_by_loc_seal_.Decrement({kAllocation.fallback_allocated_, obj.Sealed()}, kObjectSize); if (kSource == plasma::flatbuf::ObjectSource::CreatedByWorker) { diff --git a/src/ray/object_manager/plasma/store.cc b/src/ray/object_manager/plasma/store.cc index 976189682ae5..d08fc2f8e427 100644 --- a/src/ray/object_manager/plasma/store.cc +++ b/src/ray/object_manager/plasma/store.cc @@ -106,7 +106,8 @@ PlasmaStore::PlasmaStore(instrumented_io_context &main_service, std::optional fallback_allocated_fd, const auto &request) ABSL_NO_THREAD_SAFETY_ANALYSIS { mutex_.AssertHeld(); - this->AddToClientObjectIds(object_id, fallback_allocated_fd, request->client); + this->AddToClientObjectIds( + object_id, fallback_allocated_fd, request->client_); }, [this](const auto &request) { this->ReturnFromGet(request); }) { ray::SetCloseOnExec(acceptor_); @@ -185,8 +186,8 @@ PlasmaError PlasmaStore::CreateObject(const ray::ObjectInfo &object_info, entry->ToPlasmaObject(result, /* check sealed */ false); // Record that this client is using this object. std::optional fallback_allocated_fd = std::nullopt; - if (entry->GetAllocation().fallback_allocated) { - fallback_allocated_fd = entry->GetAllocation().fd; + if (entry->GetAllocation().fallback_allocated_) { + fallback_allocated_fd = entry->GetAllocation().fd_; } AddToClientObjectIds(object_info.object_id, fallback_allocated_fd, client); return PlasmaError::OK; @@ -203,8 +204,8 @@ void PlasmaStore::ReturnFromGet(const std::shared_ptr &get_request) absl::flat_hash_set fds_to_send; std::vector store_fds; std::vector mmap_sizes; - for (const auto &object_id : get_request->object_ids) { - const PlasmaObject &object = get_request->objects[object_id]; + for (const auto &object_id : get_request->object_ids_) { + const PlasmaObject &object = get_request->objects_[object_id]; MEMFD_TYPE fd = object.store_fd; if (object.data_size != -1 && fds_to_send.count(fd) == 0 && fd.first != INVALID_FD) { fds_to_send.insert(fd); @@ -213,10 +214,10 @@ void PlasmaStore::ReturnFromGet(const std::shared_ptr &get_request) } } // Send the get reply to the client. - Status s = SendGetReply(std::dynamic_pointer_cast(get_request->client), - &get_request->object_ids[0], - get_request->objects, - get_request->object_ids.size(), + Status s = SendGetReply(std::dynamic_pointer_cast(get_request->client_), + &get_request->object_ids_[0], + get_request->objects_, + get_request->object_ids_.size(), store_fds, mmap_sizes); // If we successfully sent the get reply message to the client, then also send @@ -224,14 +225,14 @@ void PlasmaStore::ReturnFromGet(const std::shared_ptr &get_request) if (s.ok()) { // Send all of the file descriptors for the present objects. for (MEMFD_TYPE store_fd : store_fds) { - Status send_fd_status = get_request->client->SendFd(store_fd); + Status send_fd_status = get_request->client_->SendFd(store_fd); if (!send_fd_status.ok()) { RAY_LOG(ERROR) << "Failed to send mmap results to client on fd " - << get_request->client; + << get_request->client_; } } } else { - RAY_LOG(ERROR) << "Failed to send Get reply to client on fd " << get_request->client; + RAY_LOG(ERROR) << "Failed to send Get reply to client on fd " << get_request->client_; } } @@ -311,8 +312,8 @@ void PlasmaStore::ConnectClient(const boost::system::error_code &error) { return ProcessClientMessage(std::move(client), message_type, message); }, /*connection_error_handler=*/ - [this](std::shared_ptr client, const boost::system::error_code &error) - -> void { return HandleClientConnectionError(std::move(client), error); }, + [this](std::shared_ptr client, const boost::system::error_code &err) + -> void { return HandleClientConnectionError(std::move(client), err); }, std::move(socket_)); // Start receiving messages. diff --git a/src/ray/object_manager/plasma/tests/eviction_policy_test.cc b/src/ray/object_manager/plasma/tests/eviction_policy_test.cc index a6367a997285..dacb8d1f020f 100644 --- a/src/ray/object_manager/plasma/tests/eviction_policy_test.cc +++ b/src/ray/object_manager/plasma/tests/eviction_policy_test.cc @@ -115,17 +115,17 @@ TEST(EvictionPolicyTest, Test) { ObjectID key4 = ObjectID::FromRandom(); LocalObject object1{Allocation()}; - object1.object_info.data_size = 10; - object1.object_info.metadata_size = 0; + object1.object_info_.data_size = 10; + object1.object_info_.metadata_size = 0; LocalObject object2{Allocation()}; - object2.object_info.data_size = 20; - object2.object_info.metadata_size = 0; + object2.object_info_.data_size = 20; + object2.object_info_.metadata_size = 0; LocalObject object3{Allocation()}; - object3.object_info.data_size = 30; - object3.object_info.metadata_size = 0; + object3.object_info_.data_size = 30; + object3.object_info_.metadata_size = 0; LocalObject object4{Allocation()}; - object4.object_info.data_size = 40; - object4.object_info.metadata_size = 0; + object4.object_info_.data_size = 40; + object4.object_info_.metadata_size = 0; auto init_object_store = [&](EvictionPolicy &policy) { EXPECT_CALL(store, GetObject(_)) diff --git a/src/ray/object_manager/plasma/tests/fallback_allocator_test.cc b/src/ray/object_manager/plasma/tests/fallback_allocator_test.cc index 3ba76efbaeca..b947dfb6729b 100644 --- a/src/ray/object_manager/plasma/tests/fallback_allocator_test.cc +++ b/src/ray/object_manager/plasma/tests/fallback_allocator_test.cc @@ -48,11 +48,11 @@ TEST(FallbackPlasmaAllocatorTest, FallbackPassThroughTest) { { auto allocation_1 = allocator.Allocate(object_size); EXPECT_TRUE(allocation_1.has_value()); - EXPECT_FALSE(allocation_1->fallback_allocated); + EXPECT_FALSE(allocation_1->fallback_allocated_); auto allocation_2 = allocator.Allocate(object_size); EXPECT_TRUE(allocation_2.has_value()); - EXPECT_FALSE(allocation_2->fallback_allocated); + EXPECT_FALSE(allocation_2->fallback_allocated_); EXPECT_EQ(2 * object_size, allocator.Allocated()); @@ -75,7 +75,7 @@ TEST(FallbackPlasmaAllocatorTest, FallbackPassThroughTest) { auto allocation = allocator.Allocate(kMB); expect_allocated += kMB; EXPECT_TRUE(allocation.has_value()); - EXPECT_FALSE(allocation->fallback_allocated); + EXPECT_FALSE(allocation->fallback_allocated_); EXPECT_EQ(expect_allocated, allocator.Allocated()); EXPECT_EQ(0, allocator.FallbackAllocated()); allocations.push_back(std::move(allocation.value())); @@ -97,7 +97,7 @@ TEST(FallbackPlasmaAllocatorTest, FallbackPassThroughTest) { expect_allocated += kMB; expect_fallback_allocated += kMB; EXPECT_TRUE(allocation.has_value()); - EXPECT_TRUE(allocation->fallback_allocated); + EXPECT_TRUE(allocation->fallback_allocated_); EXPECT_EQ(expect_allocated, allocator.Allocated()); EXPECT_EQ(expect_fallback_allocated, allocator.FallbackAllocated()); fallback_allocations.push_back(std::move(allocation.value())); diff --git a/src/ray/object_manager/plasma/tests/obj_lifecycle_mgr_test.cc b/src/ray/object_manager/plasma/tests/obj_lifecycle_mgr_test.cc index 580f400a554d..e83dca206f60 100644 --- a/src/ray/object_manager/plasma/tests/obj_lifecycle_mgr_test.cc +++ b/src/ray/object_manager/plasma/tests/obj_lifecycle_mgr_test.cc @@ -76,12 +76,12 @@ struct ObjectLifecycleManagerTest : public Test { std::move(eviction_policy), delete_object_cb, std::move(stats_collector))); - sealed_object_.state = ObjectState::PLASMA_SEALED; - not_sealed_object_.state = ObjectState::PLASMA_CREATED; - one_ref_object_.state = ObjectState::PLASMA_SEALED; - one_ref_object_.ref_count = 1; - two_ref_object_.state = ObjectState::PLASMA_SEALED; - two_ref_object_.ref_count = 2; + sealed_object_.state_ = ObjectState::PLASMA_SEALED; + not_sealed_object_.state_ = ObjectState::PLASMA_CREATED; + one_ref_object_.state_ = ObjectState::PLASMA_SEALED; + one_ref_object_.ref_count_ = 1; + two_ref_object_.state_ = ObjectState::PLASMA_SEALED; + two_ref_object_.ref_count_ = 2; } MockEvictionPolicy *eviction_policy_; diff --git a/src/ray/object_manager/plasma/tests/object_store_test.cc b/src/ray/object_manager/plasma/tests/object_store_test.cc index 0d5704c9a484..67c4f3e44039 100644 --- a/src/ray/object_manager/plasma/tests/object_store_test.cc +++ b/src/ray/object_manager/plasma/tests/object_store_test.cc @@ -42,22 +42,22 @@ T Random(T max = std::numeric_limits::max()) { Allocation CreateAllocation(Allocation alloc, int64_t size, bool fallback_allocated = false) { - alloc.size = size; - alloc.offset = Random(); - alloc.mmap_size = Random(); - alloc.fallback_allocated = fallback_allocated; + alloc.size_ = size; + alloc.offset_ = Random(); + alloc.mmap_size_ = Random(); + alloc.fallback_allocated_ = fallback_allocated; return alloc; } const std::string Serialize(const Allocation &allocation) { return absl::StrFormat("%p/%d/%d/%d/%d/%d/%d", - allocation.address, - allocation.size, - allocation.fd.first, - allocation.fd.second, - allocation.offset, - allocation.device_num, - allocation.mmap_size); + allocation.address_, + allocation.size_, + allocation.fd_.first, + allocation.fd_.second, + allocation.offset_, + allocation.device_num_, + allocation.mmap_size_); } ObjectInfo CreateObjectInfo(ObjectID object_id, int64_t object_size) { @@ -106,11 +106,11 @@ TEST(ObjectStoreTest, PassThroughTest) { })); auto entry = store.CreateObject(info, {}, /*fallback_allocate*/ false); EXPECT_NE(entry, nullptr); - EXPECT_EQ(entry->ref_count, 0); - EXPECT_EQ(entry->state, ObjectState::PLASMA_CREATED); - EXPECT_EQ(alloc_str, Serialize(entry->allocation)); - EXPECT_EQ(info, entry->object_info); - EXPECT_FALSE(entry->allocation.fallback_allocated); + EXPECT_EQ(entry->ref_count_, 0); + EXPECT_EQ(entry->state_, ObjectState::PLASMA_CREATED); + EXPECT_EQ(alloc_str, Serialize(entry->allocation_)); + EXPECT_EQ(info, entry->object_info_); + EXPECT_FALSE(entry->allocation_.fallback_allocated_); // verify get auto entry1 = store.GetObject(kId1); @@ -123,7 +123,7 @@ TEST(ObjectStoreTest, PassThroughTest) { // seal object auto entry3 = store.SealObject(kId1); EXPECT_EQ(entry3, entry); - EXPECT_EQ(entry3->state, ObjectState::PLASMA_SEALED); + EXPECT_EQ(entry3->state_, ObjectState::PLASMA_SEALED); // seal non existing EXPECT_EQ(nullptr, store.SealObject(kId2)); @@ -168,11 +168,11 @@ TEST(ObjectStoreTest, PassThroughTest) { auto entry = store.CreateObject(info, {}, /*fallback_allocate*/ true); EXPECT_NE(entry, nullptr); - EXPECT_EQ(entry->ref_count, 0); - EXPECT_EQ(entry->state, ObjectState::PLASMA_CREATED); - EXPECT_EQ(alloc_str, Serialize(entry->allocation)); - EXPECT_EQ(info, entry->object_info); - EXPECT_TRUE(entry->allocation.fallback_allocated); + EXPECT_EQ(entry->ref_count_, 0); + EXPECT_EQ(entry->state_, ObjectState::PLASMA_CREATED); + EXPECT_EQ(alloc_str, Serialize(entry->allocation_)); + EXPECT_EQ(info, entry->object_info_); + EXPECT_TRUE(entry->allocation_.fallback_allocated_); // delete unsealed EXPECT_CALL(allocator, Free(_)).Times(1).WillOnce(Invoke([&](auto &&allocation) { diff --git a/src/ray/object_manager/plasma/tests/stats_collector_test.cc b/src/ray/object_manager/plasma/tests/stats_collector_test.cc index 4831e95a6f80..9c25e7152ffe 100644 --- a/src/ray/object_manager/plasma/tests/stats_collector_test.cc +++ b/src/ray/object_manager/plasma/tests/stats_collector_test.cc @@ -39,7 +39,7 @@ class DummyAllocator : public IAllocator { std::optional Allocate(size_t bytes) override { allocated_ += bytes; auto allocation = Allocation(); - allocation.size = bytes; + allocation.size_ = bytes; return std::move(allocation); } @@ -47,7 +47,7 @@ class DummyAllocator : public IAllocator { return absl::nullopt; } - void Free(Allocation allocation) override { allocated_ -= allocation.size; } + void Free(Allocation allocation) override { allocated_ -= allocation.size_; } int64_t GetFootprintLimit() const override { return std::numeric_limits::max(); @@ -99,39 +99,40 @@ struct ObjectStatsCollectorTest : public Test { for (const auto &obj_entry : object_store_->object_table_) { const auto &obj = obj_entry.second; - if (obj->ref_count > 0) { + if (obj->ref_count_ > 0) { num_objects_in_use++; - num_bytes_in_use += obj->object_info.GetObjectSize(); + num_bytes_in_use += obj->object_info_.GetObjectSize(); } - if (obj->state == ObjectState::PLASMA_CREATED) { + if (obj->state_ == ObjectState::PLASMA_CREATED) { num_objects_unsealed++; - num_bytes_unsealed += obj->object_info.GetObjectSize(); + num_bytes_unsealed += obj->object_info_.GetObjectSize(); } else { - if (obj->ref_count == 1 && - obj->source == plasma::flatbuf::ObjectSource::CreatedByWorker) { + if (obj->ref_count_ == 1 && + obj->source_ == plasma::flatbuf::ObjectSource::CreatedByWorker) { num_objects_spillable++; - num_bytes_spillable += obj->object_info.GetObjectSize(); + num_bytes_spillable += obj->object_info_.GetObjectSize(); } - if (obj->ref_count == 0) { + if (obj->ref_count_ == 0) { num_objects_evictable++; - num_bytes_evictable += obj->object_info.GetObjectSize(); + num_bytes_evictable += obj->object_info_.GetObjectSize(); } } - if (obj->source == plasma::flatbuf::ObjectSource::CreatedByWorker) { + if (obj->source_ == plasma::flatbuf::ObjectSource::CreatedByWorker) { num_objects_created_by_worker++; - num_bytes_created_by_worker += obj->object_info.GetObjectSize(); - } else if (obj->source == plasma::flatbuf::ObjectSource::RestoredFromStorage) { + num_bytes_created_by_worker += obj->object_info_.GetObjectSize(); + } else if (obj->source_ == plasma::flatbuf::ObjectSource::RestoredFromStorage) { num_objects_restored++; - num_bytes_restored += obj->object_info.GetObjectSize(); - } else if (obj->source == plasma::flatbuf::ObjectSource::ReceivedFromRemoteRaylet) { + num_bytes_restored += obj->object_info_.GetObjectSize(); + } else if (obj->source_ == + plasma::flatbuf::ObjectSource::ReceivedFromRemoteRaylet) { num_objects_received++; - num_bytes_received += obj->object_info.GetObjectSize(); - } else if (obj->source == plasma::flatbuf::ObjectSource::ErrorStoredByRaylet) { + num_bytes_received += obj->object_info_.GetObjectSize(); + } else if (obj->source_ == plasma::flatbuf::ObjectSource::ErrorStoredByRaylet) { num_objects_errored++; - num_bytes_errored += obj->object_info.GetObjectSize(); + num_bytes_errored += obj->object_info_.GetObjectSize(); } } diff --git a/src/ray/object_manager/pull_manager.cc b/src/ray/object_manager/pull_manager.cc index f91bff777782..ca616c3b0737 100644 --- a/src/ray/object_manager/pull_manager.cc +++ b/src/ray/object_manager/pull_manager.cc @@ -70,7 +70,7 @@ uint64_t PullManager::Pull(const std::vector &object_ref_b BundlePullRequest bundle_pull_request(ObjectRefsToIds(deduplicated), task_key); const uint64_t req_id = next_req_id_++; RAY_LOG(DEBUG) << "Start pull request " << req_id - << ". Bundle size: " << bundle_pull_request.objects.size(); + << ". Bundle size: " << bundle_pull_request.objects_.size(); for (const auto &ref : deduplicated) { const auto obj_id = ObjectRefToId(ref); @@ -127,7 +127,7 @@ bool PullManager::ActivateNextBundlePullRequest(BundlePullRequestQueue &bundles, // First calculate the bytes we need. int64_t bytes_to_pull = 0; - for (const auto &obj_id : next_request.objects) { + for (const auto &obj_id : next_request.objects_) { const bool needs_pull = active_object_pull_requests_.count(obj_id) == 0; if (needs_pull) { // This is the first bundle request in the queue to require this object. @@ -158,7 +158,7 @@ bool PullManager::ActivateNextBundlePullRequest(BundlePullRequestQueue &bundles, << " num bytes being pulled: " << num_bytes_being_pulled_ << " num bytes available: " << num_bytes_available_; num_bytes_being_pulled_ += bytes_to_pull; - for (const auto &obj_id : next_request.objects) { + for (const auto &obj_id : next_request.objects_) { const bool needs_pull = active_object_pull_requests_.count(obj_id) == 0; active_object_pull_requests_[obj_id].insert(next_request_id); if (needs_pull) { @@ -184,7 +184,7 @@ void PullManager::DeactivateBundlePullRequest( uint64_t request_id, std::unordered_set *objects_to_cancel) { const auto &request = map_find_or_die(bundles.requests, request_id); - for (const auto &obj_id : request.objects) { + for (const auto &obj_id : request.objects_) { absl::MutexLock lock(&active_objects_mu_); auto it = active_object_pull_requests_.find(obj_id); if (it == active_object_pull_requests_.end() || !it->second.erase(request_id)) { @@ -337,7 +337,7 @@ std::vector PullManager::CancelPull(uint64_t request_id) { // Erase this pull request. std::vector object_ids_to_cancel_subscription; - for (const auto &obj_id : bundle_it->second.objects) { + for (const auto &obj_id : bundle_it->second.objects_) { auto it = object_pull_requests_.find(obj_id); if (it != object_pull_requests_.end()) { RAY_LOG(DEBUG) << "Removing an object pull request of id: " << obj_id; @@ -680,12 +680,12 @@ std::string PullManager::BundleInfo(const BundlePullRequestQueue &bundles) const } const auto &bundle = it->second; std::stringstream result; - result << bundle.objects.size() << " objects"; + result << bundle.objects_.size() << " objects"; if (!bundle.IsPullable()) { result << " (inactive, waiting for object sizes or locations)"; } else { size_t num_bytes_needed = 0; - for (const auto &obj_id : bundle.objects) { + for (const auto &obj_id : bundle.objects_) { num_bytes_needed += map_find_or_die(object_pull_requests_, obj_id).object_size; } result << ", " << num_bytes_needed << " bytes"; @@ -713,7 +713,7 @@ int64_t PullManager::NextRequestBundleSize(const BundlePullRequestQueue &bundles // Calculate the bytes we need. int64_t bytes_needed_calculated = 0; - for (const auto &obj_id : next_request.objects) { + for (const auto &obj_id : next_request.objects_) { bool needs_pull = active_object_pull_requests_.count(obj_id) == 0; if (needs_pull) { // This is the first bundle request in the queue to require this object. diff --git a/src/ray/object_manager/pull_manager.h b/src/ray/object_manager/pull_manager.h index 7cd7598fcb27..c81d1ed13ee8 100644 --- a/src/ray/object_manager/pull_manager.h +++ b/src/ray/object_manager/pull_manager.h @@ -225,25 +225,25 @@ class PullManager { struct BundlePullRequest { BundlePullRequest(std::vector requested_objects, const TaskMetricsKey &task_key) - : objects(std::move(requested_objects)), task_key(task_key) {} + : objects_(std::move(requested_objects)), task_key_(task_key) {} // All the objects that this bundle is trying to pull. - const std::vector objects; + const std::vector objects_; // All the objects that are pullable. - absl::flat_hash_set pullable_objects; + absl::flat_hash_set pullable_objects_; // The name of the task, if a task arg request, otherwise the empty string. - const TaskMetricsKey task_key; + const TaskMetricsKey task_key_; void MarkObjectAsPullable(const ObjectID &object) { - pullable_objects.emplace(object); + pullable_objects_.emplace(object); } void MarkObjectAsUnpullable(const ObjectID &object) { - pullable_objects.erase(object); + pullable_objects_.erase(object); } // A bundle is pullable if we know the sizes of all objects // and none of them is pending creation due to object reconstruction. - bool IsPullable() const { return pullable_objects.size() == objects.size(); } + bool IsPullable() const { return pullable_objects_.size() == objects_.size(); } }; /// A helper structure for tracking all the bundle pull requests for a particular bundle @@ -286,7 +286,7 @@ class PullManager { requests.emplace(request_id, request); if (request.IsPullable()) { inactive_requests.emplace(request_id); - inactive_by_name.Increment(request.task_key); + inactive_by_name.Increment(request.task_key_); RAY_CHECK_EQ(inactive_requests.size(), inactive_by_name.Total()); } } @@ -294,7 +294,7 @@ class PullManager { void ActivateBundlePullRequest(uint64_t request_id) { RAY_CHECK_EQ(inactive_requests.erase(request_id), 1u); active_requests.emplace(request_id); - auto task_key = map_find_or_die(requests, request_id).task_key; + auto task_key = map_find_or_die(requests, request_id).task_key_; inactive_by_name.Decrement(task_key); RAY_CHECK_EQ(inactive_requests.size(), inactive_by_name.Total()); } @@ -302,7 +302,7 @@ class PullManager { void DeactivateBundlePullRequest(uint64_t request_id) { RAY_CHECK_EQ(active_requests.erase(request_id), 1u); inactive_requests.emplace(request_id); - auto task_key = map_find_or_die(requests, request_id).task_key; + auto task_key = map_find_or_die(requests, request_id).task_key_; inactive_by_name.Increment(task_key); RAY_CHECK_EQ(inactive_requests.size(), inactive_by_name.Total()); } @@ -311,7 +311,7 @@ class PullManager { RAY_CHECK(map_find_or_die(requests, request_id).IsPullable()); RAY_CHECK_EQ(active_requests.count(request_id), 0u); inactive_requests.emplace(request_id); - auto task_key = map_find_or_die(requests, request_id).task_key; + auto task_key = map_find_or_die(requests, request_id).task_key_; inactive_by_name.Increment(task_key); RAY_CHECK_EQ(inactive_requests.size(), inactive_by_name.Total()); } @@ -324,14 +324,14 @@ class PullManager { auto it = inactive_requests.find(request_id); if (it != inactive_requests.end()) { inactive_requests.erase(it); - auto task_key = map_find_or_die(requests, request_id).task_key; + auto task_key = map_find_or_die(requests, request_id).task_key_; inactive_by_name.Decrement(task_key); RAY_CHECK_EQ(inactive_requests.size(), inactive_by_name.Total()); } } void RemoveBundlePullRequest(uint64_t request_id) { - auto task_key = map_find_or_die(requests, request_id).task_key; + auto task_key = map_find_or_die(requests, request_id).task_key_; requests.erase(request_id); if (active_requests.find(request_id) != active_requests.end()) { active_requests.erase(request_id); diff --git a/src/ray/object_manager/tests/get_request_queue_test.cc b/src/ray/object_manager/tests/get_request_queue_test.cc index e3e8c40ba615..6a3ec655801e 100644 --- a/src/ray/object_manager/tests/get_request_queue_test.cc +++ b/src/ray/object_manager/tests/get_request_queue_test.cc @@ -64,22 +64,22 @@ struct GetRequestQueueTest : public Test { Test::SetUp(); object_id1 = ObjectID::FromRandom(); object_id2 = ObjectID::FromRandom(); - object1.object_info.data_size = 10; - object1.object_info.metadata_size = 0; - object2.object_info.data_size = 10; - object2.object_info.metadata_size = 0; + object1.object_info_.data_size = 10; + object1.object_info_.metadata_size = 0; + object2.object_info_.data_size = 10; + object2.object_info_.metadata_size = 0; } void TearDown() override { io_context_.stop(); } protected: - void MarkObject(LocalObject &object, ObjectState state) { object.state = state; } + void MarkObject(LocalObject &object, ObjectState state) { object.state_ = state; } void MarkObjectFallbackAllocated(LocalObject &object, bool fallback_allocated, MEMFD_TYPE fd) { - object.allocation.fallback_allocated = fallback_allocated; - object.allocation.fd = fd; + object.allocation_.fallback_allocated_ = fallback_allocated; + object.allocation_.fd_ = fd; } bool IsGetRequestExist(GetRequestQueue &queue, const ObjectID &object_id) { diff --git a/src/ray/pubsub/publisher.cc b/src/ray/pubsub/publisher.cc index 86eab003fa26..2f4cdd05f176 100644 --- a/src/ray/pubsub/publisher.cc +++ b/src/ray/pubsub/publisher.cc @@ -322,12 +322,12 @@ void SubscriberState::PublishIfPossible(bool force_noop) { } // No message should have been added to the reply. - RAY_CHECK(long_polling_connection_->pub_messages->empty()); - *long_polling_connection_->publisher_id = publisher_id_binary_; + RAY_CHECK(long_polling_connection_->pub_messages_->empty()); + *long_polling_connection_->publisher_id_ = publisher_id_binary_; int64_t num_total_bytes = 0; if (!force_noop) { for (auto it = mailbox_.begin(); it != mailbox_.end(); it++) { - if (long_polling_connection_->pub_messages->size() >= publish_batch_size_) { + if (long_polling_connection_->pub_messages_->size() >= publish_batch_size_) { break; } @@ -346,11 +346,11 @@ void SubscriberState::PublishIfPossible(bool force_noop) { // Avoid sending empty message to the subscriber. The message might have been // cleared because the subscribed entity's buffer was full. if (msg.inner_message_case() != rpc::PubMessage::INNER_MESSAGE_NOT_SET) { - *long_polling_connection_->pub_messages->Add() = msg; + *long_polling_connection_->pub_messages_->Add() = msg; } } } - long_polling_connection_->send_reply_callback(Status::OK(), nullptr, nullptr); + long_polling_connection_->send_reply_callback_(Status::OK(), nullptr, nullptr); // Clean up & update metadata. long_polling_connection_.reset(); diff --git a/src/ray/pubsub/publisher.h b/src/ray/pubsub/publisher.h index 009fa3e1edfa..81165e1a8c5e 100644 --- a/src/ray/pubsub/publisher.h +++ b/src/ray/pubsub/publisher.h @@ -149,13 +149,13 @@ struct LongPollConnection { LongPollConnection(std::string *publisher_id, google::protobuf::RepeatedPtrField *pub_messages, rpc::SendReplyCallback send_reply_callback) - : publisher_id(publisher_id), - pub_messages(pub_messages), - send_reply_callback(std::move(send_reply_callback)) {} + : publisher_id_(publisher_id), + pub_messages_(pub_messages), + send_reply_callback_(std::move(send_reply_callback)) {} - std::string *publisher_id; - google::protobuf::RepeatedPtrField *pub_messages; - rpc::SendReplyCallback send_reply_callback; + std::string *publisher_id_; + google::protobuf::RepeatedPtrField *pub_messages_; + rpc::SendReplyCallback send_reply_callback_; }; /// Keeps the state of each connected subscriber. diff --git a/src/ray/pubsub/subscriber.cc b/src/ray/pubsub/subscriber.cc index 89f9aca913c2..c23b6ecb47fc 100644 --- a/src/ray/pubsub/subscriber.cc +++ b/src/ray/pubsub/subscriber.cc @@ -488,9 +488,9 @@ void Subscriber::SendCommandBatchIfPossible(const rpc::Address &publisher_addres Status status, const rpc::PubsubCommandBatchReply &reply) { { absl::MutexLock lock(&mutex_); - auto command_batch_sent_it = command_batch_sent_.find(publisher_id); - RAY_CHECK(command_batch_sent_it != command_batch_sent_.end()); - command_batch_sent_.erase(command_batch_sent_it); + auto command_batch_sent_iter = command_batch_sent_.find(publisher_id); + RAY_CHECK(command_batch_sent_iter != command_batch_sent_.end()); + command_batch_sent_.erase(command_batch_sent_iter); } for (const auto &done : done_cb) { if (done) { diff --git a/src/ray/raylet/dependency_manager.cc b/src/ray/raylet/dependency_manager.cc index 27d283762109..359f399f5a95 100644 --- a/src/ray/raylet/dependency_manager.cc +++ b/src/ray/raylet/dependency_manager.cc @@ -197,20 +197,20 @@ bool DependencyManager::RequestTaskDependencies( it->second.dependent_tasks.insert(task_id); } - for (const auto &obj_id : task_entry->dependencies) { + for (const auto &obj_id : task_entry->dependencies_) { if (local_objects_.count(obj_id)) { task_entry->DecrementMissingDependencies(); } } if (!required_objects.empty()) { - task_entry->pull_request_id = + task_entry->pull_request_id_ = object_manager_.Pull(required_objects, BundlePriority::TASK_ARGS, task_key); RAY_LOG(DEBUG) << "Started pull for dependencies of task " << task_id - << " request: " << task_entry->pull_request_id; + << " request: " << task_entry->pull_request_id_; } - return task_entry->num_missing_dependencies == 0; + return task_entry->num_missing_dependencies_ == 0; } void DependencyManager::RemoveTaskDependencies(const TaskID &task_id) { @@ -219,13 +219,13 @@ void DependencyManager::RemoveTaskDependencies(const TaskID &task_id) { RAY_CHECK(task_entry != queued_task_requests_.end()) << "Can't remove dependencies of tasks that are not queued."; - if (task_entry->second->pull_request_id > 0) { + if (task_entry->second->pull_request_id_ > 0) { RAY_LOG(DEBUG) << "Canceling pull for dependencies of task " << task_id - << " request: " << task_entry->second->pull_request_id; - object_manager_.CancelPull(task_entry->second->pull_request_id); + << " request: " << task_entry->second->pull_request_id_; + object_manager_.CancelPull(task_entry->second->pull_request_id_); } - for (const auto &obj_id : task_entry->second->dependencies) { + for (const auto &obj_id : task_entry->second->dependencies_) { auto it = required_objects_.find(obj_id); RAY_CHECK(it != required_objects_.end()); it->second.dependent_tasks.erase(task_id); @@ -251,7 +251,7 @@ std::vector DependencyManager::HandleObjectMissing( // If the dependent task had all of its arguments ready, it was ready to // run but must be switched to waiting since one of its arguments is now // missing. - if (task_entry->num_missing_dependencies == 0) { + if (task_entry->num_missing_dependencies_ == 0) { waiting_task_ids.push_back(dependent_task_id); // During normal execution we should be able to include the check // RAY_CHECK(pending_tasks_.count(dependent_task_id) == 1); @@ -283,7 +283,7 @@ std::vector DependencyManager::HandleObjectLocal(const ray::ObjectID &ob task_entry->DecrementMissingDependencies(); // If the dependent task now has all of its arguments ready, it's ready // to run. - if (task_entry->num_missing_dependencies == 0) { + if (task_entry->num_missing_dependencies_ == 0) { ready_task_ids.push_back(dependent_task_id); } } @@ -316,9 +316,9 @@ std::vector DependencyManager::HandleObjectLocal(const ray::ObjectID &ob bool DependencyManager::TaskDependenciesBlocked(const TaskID &task_id) const { auto it = queued_task_requests_.find(task_id); RAY_CHECK(it != queued_task_requests_.end()); - RAY_CHECK(it->second->pull_request_id != 0); + RAY_CHECK(it->second->pull_request_id_ != 0); return !object_manager_.PullRequestActiveOrWaitingForMetadata( - it->second->pull_request_id); + it->second->pull_request_id_); } std::string DependencyManager::DebugString() const { diff --git a/src/ray/raylet/dependency_manager.h b/src/ray/raylet/dependency_manager.h index 6788f399e266..01ac2ec5ff53 100644 --- a/src/ray/raylet/dependency_manager.h +++ b/src/ray/raylet/dependency_manager.h @@ -230,47 +230,47 @@ class DependencyManager : public TaskDependencyManagerInterface { TaskDependencies(const absl::flat_hash_set &deps, CounterMap> &counter_map, const TaskMetricsKey &task_key) - : dependencies(std::move(deps)), - num_missing_dependencies(dependencies.size()), - waiting_task_counter_map(counter_map), - task_key(task_key) { - if (num_missing_dependencies > 0) { - waiting_task_counter_map.Increment(task_key); + : dependencies_(std::move(deps)), + num_missing_dependencies_(dependencies_.size()), + waiting_task_counter_map_(counter_map), + task_key_(task_key) { + if (num_missing_dependencies_ > 0) { + waiting_task_counter_map_.Increment(task_key_); } } /// The objects that the task depends on. These are the arguments to the /// task. These must all be simultaneously local before the task is ready /// to execute. Objects are removed from this set once /// UnsubscribeGetDependencies is called. - absl::flat_hash_set dependencies; + absl::flat_hash_set dependencies_; /// The number of object arguments that are not available locally. This /// must be zero before the task is ready to execute. - size_t num_missing_dependencies; + size_t num_missing_dependencies_; /// Used to identify the pull request for the dependencies to the object /// manager. - uint64_t pull_request_id = 0; + uint64_t pull_request_id_ = 0; /// Reference to the counter map for metrics tracking. - CounterMap> &waiting_task_counter_map; + CounterMap> &waiting_task_counter_map_; /// The task name / is_retry tuple used for metrics tracking. - const TaskMetricsKey task_key; + const TaskMetricsKey task_key_; void IncrementMissingDependencies() { - if (num_missing_dependencies == 0) { - waiting_task_counter_map.Increment(task_key); + if (num_missing_dependencies_ == 0) { + waiting_task_counter_map_.Increment(task_key_); } - num_missing_dependencies++; + num_missing_dependencies_++; } void DecrementMissingDependencies() { - num_missing_dependencies--; - if (num_missing_dependencies == 0) { - waiting_task_counter_map.Decrement(task_key); + num_missing_dependencies_--; + if (num_missing_dependencies_ == 0) { + waiting_task_counter_map_.Decrement(task_key_); } } ~TaskDependencies() { - if (num_missing_dependencies > 0) { - waiting_task_counter_map.Decrement(task_key); + if (num_missing_dependencies_ > 0) { + waiting_task_counter_map_.Decrement(task_key_); } } }; diff --git a/src/ray/raylet/local_object_manager.cc b/src/ray/raylet/local_object_manager.cc index 02c280e8c9cb..37862bb69d05 100644 --- a/src/ray/raylet/local_object_manager.cc +++ b/src/ray/raylet/local_object_manager.cc @@ -51,7 +51,7 @@ void LocalObjectManager::PinObjectsAndWaitForFree( pinned_objects_.emplace(object_id, std::move(object)); } else { auto original_worker_id = - WorkerID::FromBinary(inserted.first->second.owner_address.worker_id()); + WorkerID::FromBinary(inserted.first->second.owner_address_.worker_id()); auto new_worker_id = WorkerID::FromBinary(owner_address.worker_id()); if (original_worker_id != new_worker_id) { // TODO(swang): Handle this case. We should use the new owner address @@ -82,17 +82,17 @@ void LocalObjectManager::PinObjectsAndWaitForFree( auto subscription_callback = [this, owner_address](const rpc::PubMessage &msg) { RAY_CHECK(msg.has_worker_object_eviction_message()); const auto &object_eviction_msg = msg.worker_object_eviction_message(); - const auto object_id = ObjectID::FromBinary(object_eviction_msg.object_id()); - ReleaseFreedObject(object_id); + const auto obj_id = ObjectID::FromBinary(object_eviction_msg.object_id()); + ReleaseFreedObject(obj_id); core_worker_subscriber_->Unsubscribe( - rpc::ChannelType::WORKER_OBJECT_EVICTION, owner_address, object_id.Binary()); + rpc::ChannelType::WORKER_OBJECT_EVICTION, owner_address, obj_id.Binary()); }; // Callback that is invoked when the owner of the object id is dead. auto owner_dead_callback = [this, owner_address](const std::string &object_id_binary, const Status &) { - const auto object_id = ObjectID::FromBinary(object_id_binary); - ReleaseFreedObject(object_id); + const auto obj_id = ObjectID::FromBinary(object_id_binary); + ReleaseFreedObject(obj_id); }; auto sub_message = std::make_unique(); @@ -111,14 +111,14 @@ void LocalObjectManager::PinObjectsAndWaitForFree( void LocalObjectManager::ReleaseFreedObject(const ObjectID &object_id) { // Only free the object if it is not already freed. auto it = local_objects_.find(object_id); - if (it == local_objects_.end() || it->second.is_freed) { + if (it == local_objects_.end() || it->second.is_freed_) { return; } // Mark the object as freed. NOTE(swang): We have to mark this instead of // deleting the entry immediately in case the object is currently being // spilled. In that case, we should process the free event once the object // spill is complete. - it->second.is_freed = true; + it->second.is_freed_ = true; RAY_LOG(DEBUG) << "Unpinning object " << object_id; // The object should be in one of these states: pinned, spilling, or spilled. @@ -326,13 +326,13 @@ void LocalObjectManager::SpillObjectsInternal( RAY_CHECK(it != objects_pending_spill_.end()); auto freed_it = local_objects_.find(object_id); // If the object hasn't already been freed, spill it. - if (freed_it == local_objects_.end() || freed_it->second.is_freed) { + if (freed_it == local_objects_.end() || freed_it->second.is_freed_) { num_bytes_pending_spill_ -= it->second->GetSize(); objects_pending_spill_.erase(it); } else { auto ref = request.add_object_refs_to_spill(); ref->set_object_id(object_id.Binary()); - ref->mutable_owner_address()->CopyFrom(freed_it->second.owner_address); + ref->mutable_owner_address()->CopyFrom(freed_it->second.owner_address_); RAY_LOG(DEBUG) << "Sending spill request for object " << object_id; requested_objects_to_spill.push_back(object_id); } @@ -423,19 +423,19 @@ void LocalObjectManager::OnObjectSpilled(const std::vector &object_ids // Asynchronously Update the spilled URL. auto freed_it = local_objects_.find(object_id); - if (freed_it == local_objects_.end() || freed_it->second.is_freed) { + if (freed_it == local_objects_.end() || freed_it->second.is_freed_) { RAY_LOG(DEBUG) << "Spilled object already freed, skipping send of spilled URL to " "object directory for object " << object_id; continue; } - const auto &worker_addr = freed_it->second.owner_address; + const auto &worker_addr = freed_it->second.owner_address_; object_directory_->ReportObjectSpilled( object_id, self_node_id_, worker_addr, object_url, - freed_it->second.generator_id.value_or(ObjectID::Nil()), + freed_it->second.generator_id_.value_or(ObjectID::Nil()), is_external_storage_type_fs_); } } @@ -555,7 +555,7 @@ void LocalObjectManager::ProcessSpilledObjectsDeleteQueue(uint32_t max_batch_siz // Update current spilled objects metrics RAY_CHECK(local_objects_.contains(object_id)) << "local objects should contain the spilled object: " << object_id; - spilled_bytes_current_ -= local_objects_.at(object_id).object_size; + spilled_bytes_current_ -= local_objects_.at(object_id).object_size_; } else { // If the object was not spilled, it gets pinned again. Unpin here to // prevent a memory leak. diff --git a/src/ray/raylet/local_object_manager.h b/src/ray/raylet/local_object_manager.h index 80bf496eda1d..d26851bbb364 100644 --- a/src/ray/raylet/local_object_manager.h +++ b/src/ray/raylet/local_object_manager.h @@ -184,14 +184,14 @@ class LocalObjectManager : public LocalObjectManagerInterface { LocalObjectInfo(const rpc::Address &owner_address, const ObjectID &generator_id, size_t object_size) - : owner_address(owner_address), - generator_id(generator_id.IsNil() ? std::nullopt - : std::optional(generator_id)), - object_size(object_size) {} - rpc::Address owner_address; - bool is_freed = false; - std::optional generator_id; - size_t object_size; + : owner_address_(owner_address), + generator_id_(generator_id.IsNil() ? std::nullopt + : std::optional(generator_id)), + object_size_(object_size) {} + rpc::Address owner_address_; + bool is_freed_ = false; + std::optional generator_id_; + size_t object_size_; }; FRIEND_TEST(LocalObjectManagerTest, TestTryToSpillObjectsZero); diff --git a/src/ray/raylet/local_task_manager.cc b/src/ray/raylet/local_task_manager.cc index d37571f3d2f2..cf1956fa91dc 100644 --- a/src/ray/raylet/local_task_manager.cc +++ b/src/ray/raylet/local_task_manager.cc @@ -68,11 +68,11 @@ void LocalTaskManager::QueueAndScheduleTask(std::shared_ptr work // locally. RAY_CHECK(cluster_resource_scheduler_.GetClusterResourceManager().HasFeasibleResources( self_scheduling_node_id_, - ResourceMapToResourceRequest(work->task.GetTaskSpecification() + ResourceMapToResourceRequest(work->task_.GetTaskSpecification() .GetRequiredPlacementResources() .GetResourceMap(), /*requires_object_store_memory=*/false))) - << work->task.GetTaskSpecification().DebugString() << " " + << work->task_.GetTaskSpecification().DebugString() << " " << cluster_resource_scheduler_.GetClusterResourceManager() .GetNodeResources(self_scheduling_node_id_) .DebugString(); @@ -81,7 +81,7 @@ void LocalTaskManager::QueueAndScheduleTask(std::shared_ptr work } void LocalTaskManager::WaitForTaskArgsRequests(std::shared_ptr work) { - const auto &task = work->task; + const auto &task = work->task_; const auto &task_id = task.GetTaskSpecification().TaskId(); const auto &scheduling_key = task.GetTaskSpecification().GetSchedulingClass(); auto object_ids = task.GetTaskSpecification().GetDependencies(); @@ -185,7 +185,7 @@ void LocalTaskManager::DispatchScheduledTasksToWorkers() { // have the same CPU resource requirements. RAY_CHECK(!cur_dispatch_queue.empty()); const auto &work = cur_dispatch_queue.front(); - const auto &task_spec = work->task.GetTaskSpecification(); + const auto &task_spec = work->task_.GetTaskSpecification(); auto cpu_request_ = task_spec.GetRequiredResources().Get(scheduling::ResourceID::CPU()).Double(); if (cpu_request_ > 0) { @@ -249,7 +249,7 @@ void LocalTaskManager::DispatchScheduledTasksToWorkers() { bool is_infeasible = false; for (auto work_it = dispatch_queue.begin(); work_it != dispatch_queue.end();) { auto &work = *work_it; - const auto &task = work->task; + const auto &task = work->task_; const auto &spec = task.GetTaskSpecification(); TaskID task_id = spec.TaskId(); if (work->GetState() == internal::WorkStatus::WAITING_FOR_WORKER) { @@ -357,7 +357,7 @@ void LocalTaskManager::DispatchScheduledTasksToWorkers() { sched_cls_info.running_tasks.insert(spec.TaskId()); // The local node has the available resources to run the task, so we should run // it. - work->allocated_instances = allocated_instances; + work->allocated_instances_ = allocated_instances; work->SetStateWaitingForWorker(); bool is_detached_actor = spec.IsDetachedActor(); auto &owner_address = spec.CallerAddress(); @@ -400,7 +400,7 @@ void LocalTaskManager::DispatchScheduledTasksToWorkers() { info_by_sched_cls_.erase(scheduling_class); } if (is_infeasible) { - const auto &front_task = dispatch_queue.front()->task.GetTaskSpecification(); + const auto &front_task = dispatch_queue.front()->task_.GetTaskSpecification(); RAY_LOG(ERROR) << "A task got scheduled to a node even though it was infeasible. " "Please report an issue on GitHub.\nTask: " << front_task.DebugString(); @@ -437,7 +437,7 @@ void LocalTaskManager::SpillWaitingTasks() { auto it = waiting_task_queue_.end(); while (it != waiting_task_queue_.begin()) { it--; - const auto &task = (*it)->task; + const auto &task = (*it)->task_; const auto &spec = task.GetTaskSpecification(); const auto &task_id = spec.TaskId(); @@ -496,7 +496,7 @@ void LocalTaskManager::SpillWaitingTasks() { bool LocalTaskManager::TrySpillback(const std::shared_ptr &work, bool &is_infeasible) { - const auto &spec = work->task.GetTaskSpecification(); + const auto &spec = work->task_.GetTaskSpecification(); auto scheduling_node_id = cluster_resource_scheduler_.GetBestSchedulableNode( spec, // We should prefer to stay local if possible @@ -530,10 +530,10 @@ bool LocalTaskManager::PoppedWorkerHandler( bool is_detached_actor, const rpc::Address &owner_address, const std::string &runtime_env_setup_error_message) { - const auto &reply = work->reply; - const auto &callback = work->callback; + const auto &reply = work->reply_; + const auto &callback = work->callback_; const bool canceled = work->GetState() == internal::WorkStatus::CANCELLED; - const auto &task = work->task; + const auto &task = work->task_; bool dispatched = false; if (!canceled) { @@ -556,31 +556,32 @@ bool LocalTaskManager::PoppedWorkerHandler( // scheduler_resource_reporter.cc. Maybe we can use `boost::any_range` to only expose // a view of the Work ptrs, but I got dependency issues // (can't include boost/range/any_range.hpp). - auto erase_from_dispatch_queue_fn = [this](const std::shared_ptr &work, - const SchedulingClass &scheduling_class) { - auto shapes_it = tasks_to_dispatch_.find(scheduling_class); - RAY_CHECK(shapes_it != tasks_to_dispatch_.end()); - auto &dispatch_queue = shapes_it->second; - bool erased = false; - for (auto work_it = dispatch_queue.begin(); work_it != dispatch_queue.end(); - work_it++) { - if (*work_it == work) { - dispatch_queue.erase(work_it); - erased = true; - break; - } - } - if (dispatch_queue.empty()) { - tasks_to_dispatch_.erase(shapes_it); - } - RAY_CHECK(erased); + auto erase_from_dispatch_queue_fn = + [this](const std::shared_ptr &work_to_erase, + const SchedulingClass &_scheduling_class) { + auto shapes_it = tasks_to_dispatch_.find(_scheduling_class); + RAY_CHECK(shapes_it != tasks_to_dispatch_.end()); + auto &dispatch_queue = shapes_it->second; + bool erased = false; + for (auto work_it = dispatch_queue.begin(); work_it != dispatch_queue.end(); + work_it++) { + if (*work_it == work_to_erase) { + dispatch_queue.erase(work_it); + erased = true; + break; + } + } + if (dispatch_queue.empty()) { + tasks_to_dispatch_.erase(shapes_it); + } + RAY_CHECK(erased); - const auto &task = work->task; - if (!task.GetDependencies().empty()) { - task_dependency_manager_.RemoveTaskDependencies( - task.GetTaskSpecification().TaskId()); - } - }; + const auto &_task = work_to_erase->task_; + if (!_task.GetDependencies().empty()) { + task_dependency_manager_.RemoveTaskDependencies( + _task.GetTaskSpecification().TaskId()); + } + }; if (canceled) { // Task has been canceled. @@ -595,8 +596,8 @@ bool LocalTaskManager::PoppedWorkerHandler( dispatched = false; // We've already acquired resources so we need to release them. cluster_resource_scheduler_.GetLocalResourceManager().ReleaseWorkerResources( - work->allocated_instances); - work->allocated_instances = nullptr; + work->allocated_instances_); + work->allocated_instances_ = nullptr; // Release pinned task args. ReleaseTaskArgs(task_id); RemoveFromRunningTasksIfExists(task); @@ -612,8 +613,8 @@ bool LocalTaskManager::PoppedWorkerHandler( // eventually. The task will be removed from dispatch queue in // `CancelTask`. CancelTasks( - [task_id](const auto &work) { - return task_id == work->task.GetTaskSpecification().TaskId(); + [task_id](const auto &w) { + return task_id == w->task_.GetTaskSpecification().TaskId(); }, rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_RUNTIME_ENV_SETUP_FAILED, /*scheduling_failure_message*/ runtime_env_setup_error_message); @@ -643,7 +644,7 @@ bool LocalTaskManager::PoppedWorkerHandler( RAY_LOG(DEBUG) << "Dispatching task " << task_id << " to worker " << worker->WorkerId(); - Dispatch(worker, leased_workers_, work->allocated_instances, task, reply, callback); + Dispatch(worker, leased_workers_, work->allocated_instances_, task, reply, callback); erase_from_dispatch_queue_fn(work, scheduling_class); dispatched = true; } @@ -653,16 +654,16 @@ bool LocalTaskManager::PoppedWorkerHandler( void LocalTaskManager::Spillback(const NodeID &spillback_to, const std::shared_ptr &work) { - auto send_reply_callback = work->callback; + auto send_reply_callback = work->callback_; - if (work->grant_or_reject) { - work->reply->set_rejected(true); + if (work->grant_or_reject_) { + work->reply_->set_rejected(true); send_reply_callback(); return; } num_task_spilled_++; - const auto &task = work->task; + const auto &task = work->task_; const auto &task_spec = task.GetTaskSpecification(); RAY_LOG(DEBUG) << "Spilling task " << task_spec.TaskId() << " to node " << spillback_to; @@ -677,7 +678,7 @@ void LocalTaskManager::Spillback(const NodeID &spillback_to, RAY_CHECK(node_info_ptr) << "Spilling back to a node manager, but no GCS info found for node " << spillback_to; - auto reply = work->reply; + auto reply = work->reply_; reply->mutable_retry_at_raylet_address()->set_ip_address( node_info_ptr->node_manager_address()); reply->mutable_retry_at_raylet_address()->set_port(node_info_ptr->node_manager_port()); @@ -695,7 +696,7 @@ void LocalTaskManager::TasksUnblocked(const std::vector &ready_ids) { auto it = waiting_tasks_index_.find(task_id); if (it != waiting_tasks_index_.end()) { auto work = *it->second; - const auto &task = work->task; + const auto &task = work->task_; const auto &scheduling_key = task.GetTaskSpecification().GetSchedulingClass(); RAY_LOG(DEBUG) << "Args ready, task can be dispatched " << task.GetTaskSpecification().TaskId(); @@ -838,8 +839,8 @@ namespace { void ReplyCancelled(const std::shared_ptr &work, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) { - auto reply = work->reply; - auto callback = work->callback; + auto reply = work->reply_; + auto callback = work->callback_; reply->set_canceled(true); reply->set_failure_type(failure_type); reply->set_scheduling_failure_message(scheduling_failure_message); @@ -867,11 +868,11 @@ bool LocalTaskManager::CancelTasks( waiting_task_queue_, [&](const std::shared_ptr &work) { if (predicate(work)) { ReplyCancelled(work, failure_type, scheduling_failure_message); - if (!work->task.GetTaskSpecification().GetDependencies().empty()) { + if (!work->task_.GetTaskSpecification().GetDependencies().empty()) { task_dependency_manager_.RemoveTaskDependencies( - work->task.GetTaskSpecification().TaskId()); + work->task_.GetTaskSpecification().TaskId()); } - waiting_tasks_index_.erase(work->task.GetTaskSpecification().TaskId()); + waiting_tasks_index_.erase(work->task_.GetTaskSpecification().TaskId()); tasks_cancelled = true; return true; } else { @@ -886,21 +887,21 @@ void LocalTaskManager::CancelTaskToDispatch( const std::shared_ptr &work, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) { - const TaskID task_id = work->task.GetTaskSpecification().TaskId(); + const TaskID task_id = work->task_.GetTaskSpecification().TaskId(); RAY_LOG(DEBUG) << "Canceling task " << task_id << " from dispatch queue."; ReplyCancelled(work, failure_type, scheduling_failure_message); if (work->GetState() == internal::WorkStatus::WAITING_FOR_WORKER) { // We've already acquired resources so we need to release them. cluster_resource_scheduler_.GetLocalResourceManager().ReleaseWorkerResources( - work->allocated_instances); + work->allocated_instances_); // Release pinned task args. ReleaseTaskArgs(task_id); } - if (!work->task.GetTaskSpecification().GetDependencies().empty()) { + if (!work->task_.GetTaskSpecification().GetDependencies().empty()) { task_dependency_manager_.RemoveTaskDependencies( - work->task.GetTaskSpecification().TaskId()); + work->task_.GetTaskSpecification().TaskId()); } - RemoveFromRunningTasksIfExists(work->task); + RemoveFromRunningTasksIfExists(work->task_); work->SetStateCancelled(); } @@ -914,7 +915,7 @@ const RayTask *LocalTaskManager::AnyPendingTasksForResourceAcquisition( auto &work_queue = shapes_it.second; for (const auto &work_it : work_queue) { const auto &work = *work_it; - const auto &task = work_it->task; + const auto &task = work_it->task_; // If the work is not in the waiting state, it will be scheduled soon or won't be // scheduled. Consider as non-pending. diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index c9f73d3cdf95..7e5f55839d95 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -673,8 +673,8 @@ int main(int argc, char *argv[]) { ray::scheduling::NodeID(raylet_node_id.Binary()), node_manager_config.resource_config.GetResourceMap(), /*is_node_available_fn*/ - [&](ray::scheduling::NodeID node_id) { - return gcs_client->Nodes().Get(NodeID::FromBinary(node_id.Binary())) != nullptr; + [&](ray::scheduling::NodeID id) { + return gcs_client->Nodes().Get(NodeID::FromBinary(id.Binary())) != nullptr; }, /*get_used_object_store_memory*/ [&]() { @@ -697,8 +697,8 @@ int main(int argc, char *argv[]) { /*labels*/ node_manager_config.labels); - auto get_node_info_func = [&](const NodeID &node_id) { - return gcs_client->Nodes().Get(node_id); + auto get_node_info_func = [&](const NodeID &id) { + return gcs_client->Nodes().Get(id); }; auto announce_infeasible_task = [](const ray::RayTask &task) { /// Publish the infeasible task error to GCS so that drivers can subscribe to it @@ -767,11 +767,11 @@ int main(int argc, char *argv[]) { announce_infeasible_task, *local_task_manager); - auto raylet_client_factory = [&](const NodeID &node_id) { - const ray::rpc::GcsNodeInfo *node_info = gcs_client->Nodes().Get(node_id); - RAY_CHECK(node_info) << "No GCS info for node " << node_id; + auto raylet_client_factory = [&](const NodeID &id) { + const ray::rpc::GcsNodeInfo *node_info = gcs_client->Nodes().Get(id); + RAY_CHECK(node_info) << "No GCS info for node " << id; auto addr = ray::rpc::RayletClientPool::GenerateRayletAddress( - node_id, node_info->node_manager_address(), node_info->node_manager_port()); + id, node_info->node_manager_address(), node_info->node_manager_port()); return raylet_client_pool->GetOrConnectByAddress(std::move(addr)); }; diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index d947dd8c845c..c87731497c26 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -515,7 +515,8 @@ void NodeManager::HandleReleaseUnusedBundles(rpc::ReleaseUnusedBundlesRequest re // so that pg bundle can be returned. local_task_manager_.CancelTasks( [&](const std::shared_ptr &work) { - const auto bundle_id = work->task.GetTaskSpecification().PlacementGroupBundleId(); + const auto bundle_id = + work->task_.GetTaskSpecification().PlacementGroupBundleId(); return !bundle_id.first.IsNil() && (0 == in_use_bundles.count(bundle_id)) && (work->GetState() == internal::WorkStatus::WAITING_FOR_WORKER); }, @@ -602,10 +603,10 @@ void NodeManager::HandleGetTaskFailureCause(rpc::GetTaskFailureCauseRequest requ auto it = task_failure_reasons_.find(task_id); if (it != task_failure_reasons_.end()) { RAY_LOG(DEBUG) << "task " << task_id << " has failure reason " - << ray::gcs::RayErrorInfoToString(it->second.ray_error_info) - << ", fail immediately: " << !it->second.should_retry; - reply->mutable_failure_cause()->CopyFrom(it->second.ray_error_info); - reply->set_fail_task_immediately(!it->second.should_retry); + << ray::gcs::RayErrorInfoToString(it->second.ray_error_info_) + << ", fail immediately: " << !it->second.should_retry_; + reply->mutable_failure_cause()->CopyFrom(it->second.ray_error_info_); + reply->set_fail_task_immediately(!it->second.should_retry_); } else { RAY_LOG(INFO) << "didn't find failure cause for task " << task_id; } @@ -1125,14 +1126,14 @@ Status NodeManager::ProcessRegisterClientRequestMessageImpl( static_cast(protocol::MessageType::RegisterClientReply), fbb.GetSize(), fbb.GetBufferPointer(), - [this, client](const ray::Status &status) { - if (!status.ok()) { + [this, client](const ray::Status &write_msg_status) { + if (!write_msg_status.ok()) { DisconnectClient(client, /*graceful=*/false, rpc::WorkerExitType::SYSTEM_ERROR, "Worker is failed because the raylet couldn't reply the " "registration request: " + - status.ToString()); + write_msg_status.ToString()); } }); }; @@ -1246,13 +1247,13 @@ void NodeManager::SendPortAnnouncementResponse( static_cast(protocol::MessageType::AnnounceWorkerPortReply), fbb.GetSize(), fbb.GetBufferPointer(), - [this, client](const ray::Status &status) { - if (!status.ok()) { - DisconnectClient( - client, - /*graceful=*/false, - rpc::WorkerExitType::SYSTEM_ERROR, - "Failed to send AnnounceWorkerPortReply to client: " + status.ToString()); + [this, client](const ray::Status &write_msg_status) { + if (!write_msg_status.ok()) { + DisconnectClient(client, + /*graceful=*/false, + rpc::WorkerExitType::SYSTEM_ERROR, + "Failed to send AnnounceWorkerPortReply to client: " + + write_msg_status.ToString()); } }); } @@ -1805,7 +1806,8 @@ void NodeManager::HandleCancelResourceReserve( // In the case of placement group removal, we should cancel all the lease requests. local_task_manager_.CancelTasks( [&](const std::shared_ptr &work) { - const auto bundle_id = work->task.GetTaskSpecification().PlacementGroupBundleId(); + const auto bundle_id = + work->task_.GetTaskSpecification().PlacementGroupBundleId(); return bundle_id.first == bundle_spec.PlacementGroupId(); }, rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_PLACEMENT_GROUP_REMOVED, @@ -2573,8 +2575,8 @@ void NodeManager::HandleFormatGlobalMemoryInfo( auto store_reply = [replies, reply, num_nodes, send_reply_callback, include_memory_info]( - rpc::GetNodeStatsReply &&local_reply) { - replies->push_back(std::move(local_reply)); + rpc::GetNodeStatsReply &&get_node_status_local_reply) { + replies->push_back(std::move(get_node_status_local_reply)); if (replies->size() >= num_nodes) { if (include_memory_info) { reply->set_memory_summary(FormatMemoryInfo(*replies)); @@ -2914,7 +2916,7 @@ void NodeManager::GCTaskFailureReason() { for (const auto &entry : task_failure_reasons_) { auto duration = static_cast( std::chrono::duration_cast( - std::chrono::steady_clock::now() - entry.second.creation_time) + std::chrono::steady_clock::now() - entry.second.creation_time_) .count()); if (duration > RayConfig::instance().task_failure_entry_ttl_ms()) { RAY_LOG(INFO).WithField(entry.first) diff --git a/src/ray/raylet/raylet.cc b/src/ray/raylet/raylet.cc index 22978ad459a5..0079aa73b7a4 100644 --- a/src/ray/raylet/raylet.cc +++ b/src/ray/raylet/raylet.cc @@ -158,8 +158,8 @@ void Raylet::HandleAccept(const boost::system::error_code &error) { if (!error) { ConnectionErrorHandler error_handler = [this]( std::shared_ptr client, - const boost::system::error_code &error) { - node_manager_.HandleClientConnectionError(client, error); + const boost::system::error_code &err) { + node_manager_.HandleClientConnectionError(client, err); }; MessageHandler message_handler = [this](std::shared_ptr client, diff --git a/src/ray/raylet/runtime_env_agent_client.cc b/src/ray/raylet/runtime_env_agent_client.cc index 2c9b04740a31..28d090f57965 100644 --- a/src/ray/raylet/runtime_env_agent_client.cc +++ b/src/ray/raylet/runtime_env_agent_client.cc @@ -230,9 +230,10 @@ class SessionPool { void enqueue(std::shared_ptr session) { if (running_sessions_.size() < max_concurrency_) { running_sessions_.insert(session); - session->run(/*finished_callback=*/[this](std::shared_ptr session) { - this->remove_session_from_running(session); - }); + session->run( + /*finished_callback=*/[this](std::shared_ptr session_to_remove) { + this->remove_session_from_running(session_to_remove); + }); } else { pending_sessions_.emplace(std::move(session)); } diff --git a/src/ray/raylet/scheduling/cluster_task_manager.cc b/src/ray/raylet/scheduling/cluster_task_manager.cc index 18ddbcf5d36d..2eaf1edb2f55 100644 --- a/src/ray/raylet/scheduling/cluster_task_manager.cc +++ b/src/ray/raylet/scheduling/cluster_task_manager.cc @@ -77,8 +77,8 @@ namespace { void ReplyCancelled(const internal::Work &work, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) { - auto reply = work.reply; - auto callback = work.callback; + auto reply = work.reply_; + auto callback = work.callback_; reply->set_canceled(true); reply->set_failure_type(failure_type); reply->set_scheduling_failure_message(scheduling_failure_message); @@ -96,7 +96,7 @@ bool ClusterTaskManager::CancelTasks( tasks_to_schedule_, [&](const std::shared_ptr &work) { if (predicate(work)) { RAY_LOG(DEBUG) << "Canceling task " - << work->task.GetTaskSpecification().TaskId() + << work->task_.GetTaskSpecification().TaskId() << " from schedule queue."; ReplyCancelled(*work, failure_type, scheduling_failure_message); tasks_cancelled = true; @@ -110,7 +110,7 @@ bool ClusterTaskManager::CancelTasks( infeasible_tasks_, [&](const std::shared_ptr &work) { if (predicate(work)) { RAY_LOG(DEBUG) << "Canceling task " - << work->task.GetTaskSpecification().TaskId() + << work->task_.GetTaskSpecification().TaskId() << " from infeasible queue."; ReplyCancelled(*work, failure_type, scheduling_failure_message); tasks_cancelled = true; @@ -159,7 +159,7 @@ bool ClusterTaskManager::IsWorkWithResourceShape( const std::shared_ptr &work, const std::vector &target_resource_shapes) { SchedulingClass scheduling_class = - work->task.GetTaskSpecification().GetSchedulingClass(); + work->task_.GetTaskSpecification().GetSchedulingClass(); ResourceSet resource_set = TaskSpecification::GetSchedulingClassDescriptor(scheduling_class).resource_set; for (const auto &target_resource_shape : target_resource_shapes) { @@ -177,8 +177,8 @@ bool ClusterTaskManager::CancelAllTasksOwnedBy( // Only tasks and regular actors are canceled because their lifetime is // the same as the owner. auto predicate = [node_id](const std::shared_ptr &work) { - return !work->task.GetTaskSpecification().IsDetachedActor() && - work->task.GetTaskSpecification().CallerNodeId() == node_id; + return !work->task_.GetTaskSpecification().IsDetachedActor() && + work->task_.GetTaskSpecification().CallerNodeId() == node_id; }; return CancelTasks(predicate, failure_type, scheduling_failure_message); @@ -191,8 +191,8 @@ bool ClusterTaskManager::CancelAllTasksOwnedBy( // Only tasks and regular actors are canceled because their lifetime is // the same as the owner. auto predicate = [worker_id](const std::shared_ptr &work) { - return !work->task.GetTaskSpecification().IsDetachedActor() && - work->task.GetTaskSpecification().CallerWorkerId() == worker_id; + return !work->task_.GetTaskSpecification().IsDetachedActor() && + work->task_.GetTaskSpecification().CallerWorkerId() == worker_id; }; return CancelTasks(predicate, failure_type, scheduling_failure_message); @@ -213,7 +213,7 @@ void ClusterTaskManager::ScheduleAndDispatchTasks() { // there are not enough available resources blocks other // tasks from being scheduled. const std::shared_ptr &work = *work_it; - RayTask task = work->task; + RayTask task = work->task_; RAY_LOG(DEBUG) << "Scheduling pending task " << task.GetTaskSpecification().TaskId(); auto scheduling_node_id = cluster_resource_scheduler_.GetBestSchedulableNode( @@ -269,7 +269,7 @@ void ClusterTaskManager::ScheduleAndDispatchTasks() { // Only announce the first item as infeasible. auto &cur_work_queue = shapes_it->second; const auto &work = cur_work_queue[0]; - const RayTask task = work->task; + const RayTask task = work->task_; if (announce_infeasible_task_) { announce_infeasible_task_(task); } @@ -306,7 +306,7 @@ void ClusterTaskManager::TryScheduleInfeasibleTask() { // We only need to check the first item because every task has the same shape. // If the first entry is infeasible, that means everything else is the same. const auto work = work_queue[0]; - RayTask task = work->task; + RayTask task = work->task_; RAY_LOG(DEBUG) << "Check if the infeasible task is schedulable in any node. task_id:" << task.GetTaskSpecification().TaskId(); bool is_infeasible; @@ -339,7 +339,7 @@ bool ClusterTaskManager::CancelTask( rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) { auto predicate = [task_id](const std::shared_ptr &work) { - return work->task.GetTaskSpecification().TaskId() == task_id; + return work->task_.GetTaskSpecification().TaskId() == task_id; }; return CancelTasks(predicate, failure_type, scheduling_failure_message); @@ -373,7 +373,7 @@ const RayTask *ClusterTaskManager::AnyPendingTasksForResourceAcquisition( auto &work_queue = shapes_it.second; for (const auto &work_it : work_queue) { const auto &work = *work_it; - const auto &task = work_it->task; + const auto &task = work_it->task_; // If the work is not in the waiting state, it will be scheduled soon or won't be // scheduled. Consider as non-pending. @@ -426,17 +426,17 @@ void ClusterTaskManager::ScheduleOnNode(const NodeID &spillback_to, return; } - auto send_reply_callback = work->callback; + auto send_reply_callback = work->callback_; - if (work->grant_or_reject) { - work->reply->set_rejected(true); + if (work->grant_or_reject_) { + work->reply_->set_rejected(true); send_reply_callback(); return; } internal_stats_.TaskSpilled(); - const auto &task = work->task; + const auto &task = work->task_; const auto &task_spec = task.GetTaskSpecification(); RAY_LOG(DEBUG) << "Spilling task " << task_spec.TaskId() << " to node " << spillback_to; @@ -451,7 +451,7 @@ void ClusterTaskManager::ScheduleOnNode(const NodeID &spillback_to, RAY_CHECK(node_info_ptr) << "Spilling back to a node manager, but no GCS info found for node " << spillback_to; - auto reply = work->reply; + auto reply = work->reply_; reply->mutable_retry_at_raylet_address()->set_ip_address( node_info_ptr->node_manager_address()); reply->mutable_retry_at_raylet_address()->set_port(node_info_ptr->node_manager_port()); diff --git a/src/ray/raylet/scheduling/internal.h b/src/ray/raylet/scheduling/internal.h index 66630199bafc..8e98eb342ff2 100644 --- a/src/ray/raylet/scheduling/internal.h +++ b/src/ray/raylet/scheduling/internal.h @@ -55,24 +55,24 @@ enum class UnscheduledWorkCause { /// dispatch/spillback and the callback to trigger it. class Work { public: - RayTask task; - bool grant_or_reject; - bool is_selected_based_on_locality; - rpc::RequestWorkerLeaseReply *reply; - std::function callback; - std::shared_ptr allocated_instances; + RayTask task_; + bool grant_or_reject_; + bool is_selected_based_on_locality_; + rpc::RequestWorkerLeaseReply *reply_; + std::function callback_; + std::shared_ptr allocated_instances_; Work(RayTask task, bool grant_or_reject, bool is_selected_based_on_locality, rpc::RequestWorkerLeaseReply *reply, std::function callback, WorkStatus status = WorkStatus::WAITING) - : task(std::move(task)), - grant_or_reject(grant_or_reject), - is_selected_based_on_locality(is_selected_based_on_locality), - reply(reply), - callback(std::move(callback)), - allocated_instances(nullptr), + : task_(std::move(task)), + grant_or_reject_(grant_or_reject), + is_selected_based_on_locality_(is_selected_based_on_locality), + reply_(reply), + callback_(std::move(callback)), + allocated_instances_(nullptr), status_(status){}; Work(const Work &Work) = delete; Work &operator=(const Work &work) = delete; @@ -95,7 +95,7 @@ class Work { UnscheduledWorkCause GetUnscheduledCause() const { return unscheduled_work_cause_; } bool PrioritizeLocalNode() const { - return grant_or_reject || is_selected_based_on_locality; + return grant_or_reject_ || is_selected_based_on_locality_; } private: diff --git a/src/ray/raylet/scheduling/policy/affinity_with_bundle_scheduling_policy.cc b/src/ray/raylet/scheduling/policy/affinity_with_bundle_scheduling_policy.cc index bfe46d314abc..7d27b2892552 100644 --- a/src/ray/raylet/scheduling/policy/affinity_with_bundle_scheduling_policy.cc +++ b/src/ray/raylet/scheduling/policy/affinity_with_bundle_scheduling_policy.cc @@ -43,11 +43,11 @@ bool AffinityWithBundleSchedulingPolicy::IsNodeFeasibleAndAvailable( scheduling::NodeID AffinityWithBundleSchedulingPolicy::Schedule( const ResourceRequest &resource_request, SchedulingOptions options) { - RAY_CHECK(options.scheduling_type == SchedulingType::AFFINITY_WITH_BUNDLE); + RAY_CHECK(options.scheduling_type_ == SchedulingType::AFFINITY_WITH_BUNDLE); auto bundle_scheduling_context = dynamic_cast( - options.scheduling_context.get()); + options.scheduling_context_.get()); const BundleID &bundle_id = bundle_scheduling_context->GetAffinityBundleID(); if (bundle_id.second != -1) { const auto &node_id_opt = bundle_location_index_.GetBundleLocation(bundle_id); @@ -63,7 +63,7 @@ scheduling::NodeID AffinityWithBundleSchedulingPolicy::Schedule( const auto &bundle_locations_opt = bundle_location_index_.GetBundleLocations(pg_id); if (bundle_locations_opt) { // Find a target with gpu nodes avoided (if required). - if (options.avoid_gpu_nodes) { + if (options.avoid_gpu_nodes_) { for (const auto &iter : *(bundle_locations_opt.value())) { auto target_node_id = scheduling::NodeID(iter.second.first.Binary()); if (IsNodeFeasibleAndAvailable( diff --git a/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.cc b/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.cc index a758dba70b7a..2eac56977c51 100644 --- a/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.cc +++ b/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.cc @@ -216,7 +216,7 @@ std::pair BundleSchedulingPolicy::GetBestNode( if (AllocationWillExceedMaxCpuFraction( node_resources, required_resources, - options.max_cpu_fraction_per_node, + options.max_cpu_fraction_per_node_, available_cpus_before_bundle_scheduling.at(node_id))) { continue; } @@ -240,7 +240,7 @@ SchedulingResult BundlePackSchedulingPolicy::Schedule( SchedulingOptions options) { RAY_CHECK(!resource_request_list.empty()); - auto candidate_nodes = SelectCandidateNodes(options.scheduling_context.get()); + auto candidate_nodes = SelectCandidateNodes(options.scheduling_context_.get()); if (candidate_nodes.empty()) { RAY_LOG(DEBUG) << "The candidate nodes is empty, return directly."; return SchedulingResult::Infeasible(); @@ -290,7 +290,7 @@ SchedulingResult BundlePackSchedulingPolicy::Schedule( // exceed max cpu fraction. node_resources, *iter->second, - options.max_cpu_fraction_per_node, + options.max_cpu_fraction_per_node_, available_cpus_before_bundle_scheduling.at(best_node.first))) { // Then allocate it. RAY_CHECK(cluster_resource_manager_.SubtractNodeAvailableResources( @@ -329,7 +329,7 @@ SchedulingResult BundleSpreadSchedulingPolicy::Schedule( SchedulingOptions options) { RAY_CHECK(!resource_request_list.empty()); - auto candidate_nodes = SelectCandidateNodes(options.scheduling_context.get()); + auto candidate_nodes = SelectCandidateNodes(options.scheduling_context_.get()); if (candidate_nodes.empty()) { RAY_LOG(DEBUG) << "The candidate nodes is empty, return directly."; return SchedulingResult::Infeasible(); @@ -399,7 +399,7 @@ SchedulingResult BundleStrictPackSchedulingPolicy::Schedule( SchedulingOptions options) { RAY_CHECK(!resource_request_list.empty()); - auto candidate_nodes = SelectCandidateNodes(options.scheduling_context.get()); + auto candidate_nodes = SelectCandidateNodes(options.scheduling_context_.get()); if (candidate_nodes.empty()) { RAY_LOG(DEBUG) << "The candidate nodes is empty, return directly."; return SchedulingResult::Infeasible(); @@ -431,7 +431,7 @@ SchedulingResult BundleStrictPackSchedulingPolicy::Schedule( // exceed max cpu fraction. node_resources, aggregated_resource_request, - options.max_cpu_fraction_per_node, + options.max_cpu_fraction_per_node_, available_cpus_before_bundle_scheduling.at(entry.first))); return allocatable; }); @@ -444,13 +444,13 @@ SchedulingResult BundleStrictPackSchedulingPolicy::Schedule( std::pair best_node(scheduling::NodeID::Nil(), nullptr); - if (!options.bundle_strict_pack_soft_target_node_id.IsNil()) { - if (candidate_nodes.contains(options.bundle_strict_pack_soft_target_node_id)) { + if (!options.bundle_strict_pack_soft_target_node_id_.IsNil()) { + if (candidate_nodes.contains(options.bundle_strict_pack_soft_target_node_id_)) { best_node = GetBestNode( aggregated_resource_request, absl::flat_hash_map{ - {options.bundle_strict_pack_soft_target_node_id, - candidate_nodes[options.bundle_strict_pack_soft_target_node_id]}}, + {options.bundle_strict_pack_soft_target_node_id_, + candidate_nodes[options.bundle_strict_pack_soft_target_node_id_]}}, options, available_cpus_before_bundle_scheduling); } @@ -485,7 +485,7 @@ SchedulingResult BundleStrictSpreadSchedulingPolicy::Schedule( RAY_CHECK(!resource_request_list.empty()); // Filter candidate nodes. - auto candidate_nodes = SelectCandidateNodes(options.scheduling_context.get()); + auto candidate_nodes = SelectCandidateNodes(options.scheduling_context_.get()); if (candidate_nodes.empty()) { RAY_LOG(DEBUG) << "The candidate nodes is empty, return directly."; return SchedulingResult::Infeasible(); diff --git a/src/ray/raylet/scheduling/policy/composite_scheduling_policy.cc b/src/ray/raylet/scheduling/policy/composite_scheduling_policy.cc index 1cb8a5677445..5afb5763cc5d 100644 --- a/src/ray/raylet/scheduling/policy/composite_scheduling_policy.cc +++ b/src/ray/raylet/scheduling/policy/composite_scheduling_policy.cc @@ -22,7 +22,7 @@ namespace raylet_scheduling_policy { scheduling::NodeID CompositeSchedulingPolicy::Schedule( const ResourceRequest &resource_request, SchedulingOptions options) { - switch (options.scheduling_type) { + switch (options.scheduling_type_) { case SchedulingType::SPREAD: return spread_policy_.Schedule(resource_request, options); case SchedulingType::RANDOM: @@ -38,7 +38,7 @@ scheduling::NodeID CompositeSchedulingPolicy::Schedule( default: RAY_LOG(FATAL) << "Unsupported scheduling type: " << static_cast::type>( - options.scheduling_type); + options.scheduling_type_); } UNREACHABLE; } @@ -46,7 +46,7 @@ scheduling::NodeID CompositeSchedulingPolicy::Schedule( SchedulingResult CompositeBundleSchedulingPolicy::Schedule( const std::vector &resource_request_list, SchedulingOptions options) { - switch (options.scheduling_type) { + switch (options.scheduling_type_) { case SchedulingType::BUNDLE_PACK: return bundle_pack_policy_.Schedule(resource_request_list, options); case SchedulingType::BUNDLE_SPREAD: @@ -58,7 +58,7 @@ SchedulingResult CompositeBundleSchedulingPolicy::Schedule( default: RAY_LOG(FATAL) << "Unsupported scheduling type: " << static_cast::type>( - options.scheduling_type); + options.scheduling_type_); } UNREACHABLE; } diff --git a/src/ray/raylet/scheduling/policy/composite_scheduling_policy.h b/src/ray/raylet/scheduling/policy/composite_scheduling_policy.h index d5cf66ae8be3..185a29521619 100644 --- a/src/ray/raylet/scheduling/policy/composite_scheduling_policy.h +++ b/src/ray/raylet/scheduling/policy/composite_scheduling_policy.h @@ -29,7 +29,7 @@ namespace ray { namespace raylet_scheduling_policy { /// A composite scheduling policy that routes the request to the underlining -/// scheduling_policy according to the scheduling_type. +/// scheduling_policy according to the scheduling_type_. class CompositeSchedulingPolicy : public ISchedulingPolicy { public: CompositeSchedulingPolicy(scheduling::NodeID local_node_id, @@ -64,7 +64,7 @@ class CompositeSchedulingPolicy : public ISchedulingPolicy { }; /// A composite scheduling policy that routes the request to the underlining -/// bundle_scheduling_policy according to the scheduling_type. +/// bundle_scheduling_policy according to the scheduling_type_. class CompositeBundleSchedulingPolicy : public IBundleSchedulingPolicy { public: explicit CompositeBundleSchedulingPolicy( diff --git a/src/ray/raylet/scheduling/policy/hybrid_scheduling_policy.cc b/src/ray/raylet/scheduling/policy/hybrid_scheduling_policy.cc index 6bf60d2a2d8a..1f82f2d6f153 100644 --- a/src/ray/raylet/scheduling/policy/hybrid_scheduling_policy.cc +++ b/src/ray/raylet/scheduling/policy/hybrid_scheduling_policy.cc @@ -182,28 +182,28 @@ scheduling::NodeID HybridSchedulingPolicy::ScheduleImpl( scheduling::NodeID HybridSchedulingPolicy::Schedule( const ResourceRequest &resource_request, SchedulingOptions options) { - RAY_CHECK(options.scheduling_type == SchedulingType::HYBRID) + RAY_CHECK(options.scheduling_type_ == SchedulingType::HYBRID) << "HybridPolicy policy requires type = HYBRID"; - if (!options.avoid_gpu_nodes || resource_request.Has(ResourceID::GPU())) { + if (!options.avoid_gpu_nodes_ || resource_request.Has(ResourceID::GPU())) { return ScheduleImpl(resource_request, - options.spread_threshold, - options.avoid_local_node, - options.require_node_available, + options.spread_threshold_, + options.avoid_local_node_, + options.require_node_available_, NodeFilter::kAny, - options.preferred_node_id, - options.schedule_top_k_absolute, - options.scheduler_top_k_fraction); + options.preferred_node_id_, + options.schedule_top_k_absolute_, + options.scheduler_top_k_fraction_); } // Try schedule on non-GPU nodes. auto best_node_id = ScheduleImpl(resource_request, - options.spread_threshold, - options.avoid_local_node, + options.spread_threshold_, + options.avoid_local_node_, /*require_node_available*/ true, NodeFilter::kNonGpu, - options.preferred_node_id, - options.schedule_top_k_absolute, - options.scheduler_top_k_fraction); + options.preferred_node_id_, + options.schedule_top_k_absolute_, + options.scheduler_top_k_fraction_); if (!best_node_id.IsNil()) { return best_node_id; } @@ -211,13 +211,13 @@ scheduling::NodeID HybridSchedulingPolicy::Schedule( // If we cannot find any available node from non-gpu nodes, fallback to the original // scheduling return ScheduleImpl(resource_request, - options.spread_threshold, - options.avoid_local_node, - options.require_node_available, + options.spread_threshold_, + options.avoid_local_node_, + options.require_node_available_, NodeFilter::kAny, - options.preferred_node_id, - options.schedule_top_k_absolute, - options.scheduler_top_k_fraction); + options.preferred_node_id_, + options.schedule_top_k_absolute_, + options.scheduler_top_k_fraction_); } } // namespace raylet_scheduling_policy diff --git a/src/ray/raylet/scheduling/policy/node_affinity_scheduling_policy.cc b/src/ray/raylet/scheduling/policy/node_affinity_scheduling_policy.cc index 737aa33a80f8..13e4dea53ed5 100644 --- a/src/ray/raylet/scheduling/policy/node_affinity_scheduling_policy.cc +++ b/src/ray/raylet/scheduling/policy/node_affinity_scheduling_policy.cc @@ -19,24 +19,24 @@ namespace raylet_scheduling_policy { scheduling::NodeID NodeAffinitySchedulingPolicy::Schedule( const ResourceRequest &resource_request, SchedulingOptions options) { - RAY_CHECK(options.scheduling_type == SchedulingType::NODE_AFFINITY); + RAY_CHECK(options.scheduling_type_ == SchedulingType::NODE_AFFINITY); - scheduling::NodeID target_node_id = scheduling::NodeID(options.node_affinity_node_id); + scheduling::NodeID target_node_id = scheduling::NodeID(options.node_affinity_node_id_); if (nodes_.contains(target_node_id) && is_node_alive_(target_node_id) && nodes_.at(target_node_id).GetLocalView().IsFeasible(resource_request)) { - if (!options.node_affinity_spill_on_unavailable && - !options.node_affinity_fail_on_unavailable) { + if (!options.node_affinity_spill_on_unavailable_ && + !options.node_affinity_fail_on_unavailable_) { return target_node_id; } else if (nodes_.at(target_node_id).GetLocalView().IsAvailable(resource_request)) { return target_node_id; } } - if (!options.node_affinity_soft) { + if (!options.node_affinity_soft_) { return scheduling::NodeID::Nil(); } - options.scheduling_type = SchedulingType::HYBRID; + options.scheduling_type_ = SchedulingType::HYBRID; return hybrid_policy_.Schedule(resource_request, options); } diff --git a/src/ray/raylet/scheduling/policy/node_label_scheduling_policy.cc b/src/ray/raylet/scheduling/policy/node_label_scheduling_policy.cc index c5393b464198..2bbd935a96dd 100644 --- a/src/ray/raylet/scheduling/policy/node_label_scheduling_policy.cc +++ b/src/ray/raylet/scheduling/policy/node_label_scheduling_policy.cc @@ -21,9 +21,9 @@ namespace raylet_scheduling_policy { scheduling::NodeID NodeLabelSchedulingPolicy::Schedule( const ResourceRequest &resource_request, SchedulingOptions options) { - RAY_CHECK(options.scheduling_type == SchedulingType::NODE_LABEL); + RAY_CHECK(options.scheduling_type_ == SchedulingType::NODE_LABEL); auto context = - dynamic_cast(options.scheduling_context.get()); + dynamic_cast(options.scheduling_context_.get()); const auto &scheduling_strategy = context->GetSchedulingStrategy(); RAY_CHECK(scheduling_strategy.has_node_label_scheduling_strategy()); const auto &node_label_scheduling_strategy = diff --git a/src/ray/raylet/scheduling/policy/random_scheduling_policy.cc b/src/ray/raylet/scheduling/policy/random_scheduling_policy.cc index 423aad73ca9c..f48a0c9c5bf1 100644 --- a/src/ray/raylet/scheduling/policy/random_scheduling_policy.cc +++ b/src/ray/raylet/scheduling/policy/random_scheduling_policy.cc @@ -22,15 +22,15 @@ namespace raylet_scheduling_policy { scheduling::NodeID RandomSchedulingPolicy::Schedule( const ResourceRequest &resource_request, SchedulingOptions options) { - RAY_CHECK(options.scheduling_type == SchedulingType::RANDOM) + RAY_CHECK(options.scheduling_type_ == SchedulingType::RANDOM) << "HybridPolicy policy requires type = RANDOM"; scheduling::NodeID best_node = scheduling::NodeID::Nil(); if (nodes_.empty()) { return best_node; } - RAY_CHECK(options.spread_threshold == 0 && !options.avoid_local_node && - options.require_node_available && !options.avoid_gpu_nodes) + RAY_CHECK(options.spread_threshold_ == 0 && !options.avoid_local_node_ && + options.require_node_available_ && !options.avoid_gpu_nodes_) << "Random policy requires spread_threshold = 0, " << "avoid_local_node = false, " << "require_node_available = true, " diff --git a/src/ray/raylet/scheduling/policy/scheduling_options.h b/src/ray/raylet/scheduling/policy/scheduling_options.h index 6a44cec601e4..fe1d1f2bb8ad 100644 --- a/src/ray/raylet/scheduling/policy/scheduling_options.h +++ b/src/ray/raylet/scheduling/policy/scheduling_options.h @@ -87,11 +87,11 @@ struct SchedulingOptions { } SchedulingOptions scheduling_options = Hybrid(avoid_local_node, require_node_available); - scheduling_options.scheduling_type = SchedulingType::NODE_AFFINITY; - scheduling_options.node_affinity_node_id = node_id; - scheduling_options.node_affinity_soft = soft; - scheduling_options.node_affinity_spill_on_unavailable = spill_on_unavailable; - scheduling_options.node_affinity_fail_on_unavailable = fail_on_unavailable; + scheduling_options.scheduling_type_ = SchedulingType::NODE_AFFINITY; + scheduling_options.node_affinity_node_id_ = node_id; + scheduling_options.node_affinity_soft_ = soft; + scheduling_options.node_affinity_spill_on_unavailable_ = spill_on_unavailable; + scheduling_options.node_affinity_fail_on_unavailable_ = fail_on_unavailable; return scheduling_options; } @@ -157,7 +157,7 @@ struct SchedulingOptions { /*require_node_available*/ true, /*avoid_gpu_nodes*/ false, /*max_cpu_fraction_per_node*/ max_cpu_fraction_per_node); - scheduling_options.bundle_strict_pack_soft_target_node_id = soft_target_node_id; + scheduling_options.bundle_strict_pack_soft_target_node_id_ = soft_target_node_id; return scheduling_options; } @@ -174,32 +174,32 @@ struct SchedulingOptions { /*scheduling_context*/ std::move(scheduling_context)); } - SchedulingType scheduling_type; - float spread_threshold; - bool avoid_local_node; - bool require_node_available; - bool avoid_gpu_nodes; + SchedulingType scheduling_type_; + float spread_threshold_; + bool avoid_local_node_; + bool require_node_available_; + bool avoid_gpu_nodes_; // Maximum reservable CPU fraction per node. It is applied across multiple // bundles, individually. E.g., when you have 2 bundles {CPU: 4} from 2 different // scheduilng request, and there's one node with {CPU: 8}, only 1 bundle from 1 request // can be scheduled on this node. This is only used for bundle scheduling policies // (bundle pack, spread). - double max_cpu_fraction_per_node; + double max_cpu_fraction_per_node_; // ID of the target node where bundles should be placed // iff the target node has enough available resources. // Otherwise, the bundles can be placed elsewhere. // This is only used by PG STRICT_PACK scheduling. - scheduling::NodeID bundle_strict_pack_soft_target_node_id = scheduling::NodeID::Nil(); - std::shared_ptr scheduling_context; - std::string node_affinity_node_id; - bool node_affinity_soft = false; - bool node_affinity_spill_on_unavailable = false; - bool node_affinity_fail_on_unavailable = false; + scheduling::NodeID bundle_strict_pack_soft_target_node_id_ = scheduling::NodeID::Nil(); + std::shared_ptr scheduling_context_; + std::string node_affinity_node_id_; + bool node_affinity_soft_ = false; + bool node_affinity_spill_on_unavailable_ = false; + bool node_affinity_fail_on_unavailable_ = false; // The node where the task is preferred to be placed. By default, this node id // is empty, which means no preferred node. - std::string preferred_node_id; - int32_t schedule_top_k_absolute; - float scheduler_top_k_fraction; + std::string preferred_node_id_; + int32_t schedule_top_k_absolute_; + float scheduler_top_k_fraction_; private: SchedulingOptions( @@ -213,16 +213,16 @@ struct SchedulingOptions { const std::string &preferred_node_id = std::string(), int32_t schedule_top_k_absolute = RayConfig::instance().scheduler_top_k_absolute(), float scheduler_top_k_fraction = RayConfig::instance().scheduler_top_k_fraction()) - : scheduling_type(type), - spread_threshold(spread_threshold), - avoid_local_node(avoid_local_node), - require_node_available(require_node_available), - avoid_gpu_nodes(avoid_gpu_nodes), - max_cpu_fraction_per_node(max_cpu_fraction_per_node), - scheduling_context(std::move(scheduling_context)), - preferred_node_id(preferred_node_id), - schedule_top_k_absolute(schedule_top_k_absolute), - scheduler_top_k_fraction(scheduler_top_k_fraction) {} + : scheduling_type_(type), + spread_threshold_(spread_threshold), + avoid_local_node_(avoid_local_node), + require_node_available_(require_node_available), + avoid_gpu_nodes_(avoid_gpu_nodes), + max_cpu_fraction_per_node_(max_cpu_fraction_per_node), + scheduling_context_(std::move(scheduling_context)), + preferred_node_id_(preferred_node_id), + schedule_top_k_absolute_(schedule_top_k_absolute), + scheduler_top_k_fraction_(scheduler_top_k_fraction) {} friend class ::ray::raylet::SchedulingPolicyTest; friend class HybridSchedulingPolicyTest; diff --git a/src/ray/raylet/scheduling/policy/spread_scheduling_policy.cc b/src/ray/raylet/scheduling/policy/spread_scheduling_policy.cc index 076d1845dcb6..1d53494ddbcc 100644 --- a/src/ray/raylet/scheduling/policy/spread_scheduling_policy.cc +++ b/src/ray/raylet/scheduling/policy/spread_scheduling_policy.cc @@ -24,8 +24,8 @@ namespace raylet_scheduling_policy { scheduling::NodeID SpreadSchedulingPolicy::Schedule( const ResourceRequest &resource_request, SchedulingOptions options) { - RAY_CHECK(options.spread_threshold == 0 && - options.scheduling_type == SchedulingType::SPREAD) + RAY_CHECK(options.spread_threshold_ == 0 && + options.scheduling_type_ == SchedulingType::SPREAD) << "SpreadPolicy policy requires spread_threshold = 0 and type = SPREAD"; std::vector round; round.reserve(nodes_.size()); @@ -37,13 +37,13 @@ scheduling::NodeID SpreadSchedulingPolicy::Schedule( // Spread among available nodes first. // If there is no available nodes, we spread among feasible nodes. for (bool available_nodes_only : - (options.require_node_available ? std::vector{true} - : std::vector{true, false})) { + (options.require_node_available_ ? std::vector{true} + : std::vector{true, false})) { size_t round_index = spread_scheduling_next_index_; for (size_t i = 0; i < round.size(); ++i, ++round_index) { const auto &node_id = round[round_index % round.size()]; const auto &node = map_find_or_die(nodes_, node_id); - if (node_id == local_node_id_ && options.avoid_local_node) { + if (node_id == local_node_id_ && options.avoid_local_node_) { continue; } if (!is_node_alive_(node_id) || !node.GetLocalView().IsFeasible(resource_request)) { diff --git a/src/ray/raylet/scheduling/scheduler_resource_reporter.cc b/src/ray/raylet/scheduling/scheduler_resource_reporter.cc index 0523851724a0..597e7ddf5277 100644 --- a/src/ray/raylet/scheduling/scheduler_resource_reporter.cc +++ b/src/ray/raylet/scheduling/scheduler_resource_reporter.cc @@ -139,7 +139,7 @@ void SchedulerResourceReporter::FillResourceUsage(rpc::ResourcesData &data) cons auto cnt = pair.second.size(); // We should only report dispatching tasks that do not have resources allocated. for (const auto &task : pair.second) { - if (task->allocated_instances) { + if (task->allocated_instances_) { cnt--; } } diff --git a/src/ray/raylet/wait_manager.cc b/src/ray/raylet/wait_manager.cc index 8745848f2f59..ae26e0cf9e56 100644 --- a/src/ray/raylet/wait_manager.cc +++ b/src/ray/raylet/wait_manager.cc @@ -40,19 +40,19 @@ void WaitManager::Wait(const std::vector &object_ids, auto &wait_request = wait_requests_.at(wait_id); for (const auto &object_id : object_ids) { if (is_object_local_(object_id)) { - wait_request.ready.emplace(object_id); + wait_request.ready_.emplace(object_id); } } - for (const auto &object_id : wait_request.object_ids) { + for (const auto &object_id : wait_request.object_ids_) { object_to_wait_requests_[object_id].emplace(wait_id); } - if (wait_request.ready.size() >= wait_request.num_required_objects || - wait_request.timeout_ms == 0) { + if (wait_request.ready_.size() >= wait_request.num_required_objects_ || + wait_request.timeout_ms_ == 0) { // Requirements already satisfied. WaitComplete(wait_id); - } else if (wait_request.timeout_ms != -1) { + } else if (wait_request.timeout_ms_ != -1) { // If a timeout was provided, then set a timer. If there are no // enough locally available objects by the time the timer expires, // then we will return from the Wait. @@ -65,14 +65,14 @@ void WaitManager::Wait(const std::vector &object_ids, } WaitComplete(wait_id); }, - wait_request.timeout_ms); + wait_request.timeout_ms_); } } void WaitManager::WaitComplete(uint64_t wait_id) { auto &wait_request = map_find_or_die(wait_requests_, wait_id); - for (const auto &object_id : wait_request.object_ids) { + for (const auto &object_id : wait_request.object_ids_) { auto &requests = object_to_wait_requests_.at(object_id); requests.erase(wait_id); if (requests.empty()) { @@ -83,15 +83,15 @@ void WaitManager::WaitComplete(uint64_t wait_id) { // Order objects according to input order. std::vector ready; std::vector remaining; - for (const auto &object_id : wait_request.object_ids) { - if (ready.size() < wait_request.num_required_objects && - wait_request.ready.count(object_id) > 0) { + for (const auto &object_id : wait_request.object_ids_) { + if (ready.size() < wait_request.num_required_objects_ && + wait_request.ready_.count(object_id) > 0) { ready.push_back(object_id); } else { remaining.push_back(object_id); } } - wait_request.callback(ready, remaining); + wait_request.callback_(ready, remaining); wait_requests_.erase(wait_id); RAY_LOG(DEBUG) << "Wait request " << wait_id << " finished: ready " << ready.size() << " remaining " << remaining.size(); @@ -105,8 +105,8 @@ void WaitManager::HandleObjectLocal(const ray::ObjectID &object_id) { std::vector complete_waits; for (const auto &wait_id : object_to_wait_requests_.at(object_id)) { auto &wait_request = map_find_or_die(wait_requests_, wait_id); - wait_request.ready.emplace(object_id); - if (wait_request.ready.size() >= wait_request.num_required_objects) { + wait_request.ready_.emplace(object_id); + if (wait_request.ready_.size() >= wait_request.num_required_objects_) { complete_waits.emplace_back(wait_id); } } diff --git a/src/ray/raylet/wait_manager.h b/src/ray/raylet/wait_manager.h index 5b9f3cad0d45..de66735165c3 100644 --- a/src/ray/raylet/wait_manager.h +++ b/src/ray/raylet/wait_manager.h @@ -66,20 +66,20 @@ class WaitManager { const WaitCallback &callback, const std::vector &object_ids, uint64_t num_required_objects) - : timeout_ms(timeout_ms), - callback(callback), - object_ids(object_ids), - num_required_objects(num_required_objects) {} + : timeout_ms_(timeout_ms), + callback_(callback), + object_ids_(object_ids), + num_required_objects_(num_required_objects) {} /// The period of time to wait before invoking the callback. - const int64_t timeout_ms; + const int64_t timeout_ms_; /// The callback invoked when Wait is complete. - WaitCallback callback; + WaitCallback callback_; /// Ordered input object_ids. - const std::vector object_ids; + const std::vector object_ids_; /// The number of required objects. - const uint64_t num_required_objects; + const uint64_t num_required_objects_; /// The objects that have been locally available. - std::unordered_set ready; + std::unordered_set ready_; }; /// Completion handler for Wait. diff --git a/src/ray/raylet/worker_killing_policy_group_by_owner.h b/src/ray/raylet/worker_killing_policy_group_by_owner.h index c5f3e95b5282..2d57b6227662 100644 --- a/src/ray/raylet/worker_killing_policy_group_by_owner.h +++ b/src/ray/raylet/worker_killing_policy_group_by_owner.h @@ -35,8 +35,8 @@ namespace raylet { /// Key groups on its owner id. For non-retriable task the owner id is itself, /// Since non-retriable task forms its own group. struct GroupKey { - explicit GroupKey(const TaskID &owner_id) : owner_id(owner_id) {} - const TaskID &owner_id; + explicit GroupKey(const TaskID &owner_id) : owner_id_(owner_id) {} + const TaskID &owner_id_; }; struct Group { diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index 0b4f61a18576..76335cf95924 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -117,7 +117,7 @@ WorkerPool::WorkerPool(instrumented_io_context &io_service, gcs_client_(gcs_client), native_library_path_(std::move(native_library_path)), starting_worker_timeout_callback_(std::move(starting_worker_timeout_callback)), - ray_debugger_external(ray_debugger_external), + ray_debugger_external_(ray_debugger_external), first_job_registered_python_worker_count_(0), first_job_driver_wait_num_python_workers_( std::min(num_prestarted_python_workers, maximum_startup_concurrency_)), @@ -389,7 +389,7 @@ WorkerPool::BuildProcessCommandArgs(const Language &language, worker_command_args.push_back("--language=" + Language_Name(language)); } - if (ray_debugger_external) { + if (ray_debugger_external_) { worker_command_args.push_back("--ray-debugger-external"); } @@ -627,14 +627,14 @@ void WorkerPool::MonitorPopWorkerRequestForRegistration( // Capture timer in lambda to copy it once, so that it can avoid destructing timer. timer->async_wait([timer, pop_worker_request = std::move(pop_worker_request), this]( const boost::system::error_code e) mutable { - auto &state = GetStateForLanguage(pop_worker_request->language); + auto &state = GetStateForLanguage(pop_worker_request->language_); auto &requests = state.pending_registration_requests; auto it = std::find(requests.begin(), requests.end(), pop_worker_request); if (it != requests.end()) { // Pop and fail the task... requests.erase(it); PopWorkerStatus status = PopWorkerStatus::WorkerPendingRegistration; - PopWorkerCallbackAsync(pop_worker_request->callback, nullptr, status); + PopWorkerCallbackAsync(pop_worker_request->callback_, nullptr, status); } }); } @@ -1061,9 +1061,8 @@ void WorkerPool::PushWorker(const std::shared_ptr &worker) { auto it = std::find_if( state.pending_registration_requests.begin(), state.pending_registration_requests.end(), - [this, &worker](const std::shared_ptr &pop_worker_request) { - return WorkerFitsForTask(*worker, *pop_worker_request) == - WorkerUnfitForTaskReason::NONE; + [this, &worker](const std::shared_ptr &request) { + return WorkerFitsForTask(*worker, *request) == WorkerUnfitForTaskReason::NONE; }); if (it != state.pending_registration_requests.end()) { pop_worker_request = *it; @@ -1074,9 +1073,8 @@ void WorkerPool::PushWorker(const std::shared_ptr &worker) { auto it = std::find_if( state.pending_start_requests.begin(), state.pending_start_requests.end(), - [this, &worker](const std::shared_ptr &pop_worker_request) { - return WorkerFitsForTask(*worker, *pop_worker_request) == - WorkerUnfitForTaskReason::NONE; + [this, &worker](const std::shared_ptr &request) { + return WorkerFitsForTask(*worker, *request) == WorkerUnfitForTaskReason::NONE; }); if (it != state.pending_start_requests.end()) { pop_worker_request = *it; @@ -1085,7 +1083,7 @@ void WorkerPool::PushWorker(const std::shared_ptr &worker) { } if (pop_worker_request) { - bool used = pop_worker_request->callback(worker, PopWorkerStatus::OK, ""); + bool used = pop_worker_request->callback_(worker, PopWorkerStatus::OK, ""); if (!used) { // Retry PushWorker. Maybe it can be used by other tasks. // Can we have tail call optimization for this? :) @@ -1207,9 +1205,9 @@ void WorkerPool::KillIdleWorker(const IdleWorkerEntry &entry) { } rpc_client->Exit( request, [this, entry](const ray::Status &status, const rpc::ExitReply &r) { - const auto &idle_worker = entry.worker; + const auto &worker = entry.worker; - RAY_CHECK(pending_exit_idle_workers_.erase(idle_worker->WorkerId())); + RAY_CHECK(pending_exit_idle_workers_.erase(worker->WorkerId())); if (!status.ok()) { RAY_LOG(ERROR) << "Failed to send exit request: " << status.ToString(); } @@ -1217,19 +1215,19 @@ void WorkerPool::KillIdleWorker(const IdleWorkerEntry &entry) { // In case of failed to send request, we remove it from pool as well // TODO(iycheng): We should handle the grpc failure in better way. if (!status.ok() || r.success()) { - RAY_LOG(DEBUG) << "Removed worker " << idle_worker->WorkerId(); - auto &worker_state = GetStateForLanguage(idle_worker->GetLanguage()); + RAY_LOG(DEBUG) << "Removed worker " << worker->WorkerId(); + auto &worker_state = GetStateForLanguage(worker->GetLanguage()); // If we could kill the worker properly, we remove them from the idle // pool. - RemoveWorker(worker_state.idle, idle_worker); + RemoveWorker(worker_state.idle, worker); // We always mark the worker as dead. // If the worker is not idle at this moment, we'd want to mark it as dead // so it won't be reused later. - if (!idle_worker->IsDead()) { - idle_worker->MarkDead(); + if (!worker->IsDead()) { + worker->MarkDead(); } } else { - RAY_LOG(DEBUG) << "Failed to remove worker " << idle_worker->WorkerId(); + RAY_LOG(DEBUG) << "Failed to remove worker " << worker->WorkerId(); // We re-insert the idle worker to the back of the queue if it fails to // kill the worker (e.g., when the worker owns the object). Without this, // if the first N workers own objects, it can't kill idle workers that are @@ -1248,10 +1246,10 @@ WorkerUnfitForTaskReason WorkerPool::WorkerFitsForTask( if (pending_exit_idle_workers_.contains(worker.WorkerId())) { return WorkerUnfitForTaskReason::OTHERS; } - if (worker.GetLanguage() != pop_worker_request.language) { + if (worker.GetLanguage() != pop_worker_request.language_) { return WorkerUnfitForTaskReason::OTHERS; } - if (worker.GetWorkerType() != pop_worker_request.worker_type) { + if (worker.GetWorkerType() != pop_worker_request.worker_type_) { return WorkerUnfitForTaskReason::OTHERS; } @@ -1260,27 +1258,27 @@ WorkerUnfitForTaskReason WorkerPool::WorkerFitsForTask( // NOTE(edoakes): the job ID for a worker with no detached actor ID must still match, // which is checked below. The pop_worker_request for a task rooted in a detached // actor will have the job ID of the job that created the detached actor. - if (!pop_worker_request.root_detached_actor_id.IsNil() && + if (!pop_worker_request.root_detached_actor_id_.IsNil() && !worker.GetRootDetachedActorId().IsNil() && - pop_worker_request.root_detached_actor_id != worker.GetRootDetachedActorId()) { + pop_worker_request.root_detached_actor_id_ != worker.GetRootDetachedActorId()) { return WorkerUnfitForTaskReason::ROOT_MISMATCH; } // Only consider workers that haven't been assigned to a job yet or have been assigned // to the requested job. const auto worker_job_id = worker.GetAssignedJobId(); - if (!worker_job_id.IsNil() && pop_worker_request.job_id != worker_job_id) { + if (!worker_job_id.IsNil() && pop_worker_request.job_id_ != worker_job_id) { return WorkerUnfitForTaskReason::ROOT_MISMATCH; } // If the request asks for a is_gpu, and the worker is assigned a different is_gpu, // then skip it. - if (!OptionalsMatchOrEitherEmpty(pop_worker_request.is_gpu, worker.GetIsGpu())) { + if (!OptionalsMatchOrEitherEmpty(pop_worker_request.is_gpu_, worker.GetIsGpu())) { return WorkerUnfitForTaskReason::OTHERS; } // If the request asks for a is_actor_worker, and the worker is assigned a different // is_actor_worker, then skip it. - if (!OptionalsMatchOrEitherEmpty(pop_worker_request.is_actor_worker, + if (!OptionalsMatchOrEitherEmpty(pop_worker_request.is_actor_worker_, worker.GetIsActorWorker())) { return WorkerUnfitForTaskReason::OTHERS; } @@ -1288,12 +1286,12 @@ WorkerUnfitForTaskReason WorkerPool::WorkerFitsForTask( // Even if the task doesn't have a runtime_env specified, we cannot schedule it to a // worker with a runtime_env because the task is expected to run in the base // environment. - if (worker.GetRuntimeEnvHash() != pop_worker_request.runtime_env_hash) { + if (worker.GetRuntimeEnvHash() != pop_worker_request.runtime_env_hash_) { return WorkerUnfitForTaskReason::RUNTIME_ENV_MISMATCH; } // Skip if the dynamic_options doesn't match. if (LookupWorkerDynamicOptions(worker.GetStartupToken()) != - pop_worker_request.dynamic_options) { + pop_worker_request.dynamic_options_) { return WorkerUnfitForTaskReason::DYNAMIC_OPTIONS_MISMATCH; } return WorkerUnfitForTaskReason::NONE; @@ -1302,48 +1300,48 @@ WorkerUnfitForTaskReason WorkerPool::WorkerFitsForTask( void WorkerPool::StartNewWorker( const std::shared_ptr &pop_worker_request) { auto start_worker_process_fn = [this]( - std::shared_ptr pop_worker_request, + std::shared_ptr request, const std::string &serialized_runtime_env_context) { - auto &state = GetStateForLanguage(pop_worker_request->language); + auto &state = GetStateForLanguage(request->language_); const std::string &serialized_runtime_env = - pop_worker_request->runtime_env_info.serialized_runtime_env(); + request->runtime_env_info_.serialized_runtime_env(); PopWorkerStatus status = PopWorkerStatus::OK; auto [proc, startup_token] = - StartWorkerProcess(pop_worker_request->language, - pop_worker_request->worker_type, - pop_worker_request->job_id, + StartWorkerProcess(request->language_, + request->worker_type_, + request->job_id_, &status, - pop_worker_request->dynamic_options, - pop_worker_request->runtime_env_hash, + request->dynamic_options_, + request->runtime_env_hash_, serialized_runtime_env_context, - pop_worker_request->runtime_env_info, - pop_worker_request->worker_startup_keep_alive_duration); + request->runtime_env_info_, + request->worker_startup_keep_alive_duration_); if (status == PopWorkerStatus::OK) { RAY_CHECK(proc.IsValid()); WarnAboutSize(); - state.pending_registration_requests.emplace_back(pop_worker_request); - MonitorPopWorkerRequestForRegistration(pop_worker_request); + state.pending_registration_requests.emplace_back(request); + MonitorPopWorkerRequestForRegistration(request); } else if (status == PopWorkerStatus::TooManyStartingWorkerProcesses) { // TODO(jjyao) As an optimization, we don't need to delete the runtime env // but reuse it the next time we retry the request. DeleteRuntimeEnvIfPossible(serialized_runtime_env); - state.pending_start_requests.emplace_back(std::move(pop_worker_request)); + state.pending_start_requests.emplace_back(std::move(request)); } else { DeleteRuntimeEnvIfPossible(serialized_runtime_env); - PopWorkerCallbackAsync(std::move(pop_worker_request->callback), nullptr, status); + PopWorkerCallbackAsync(std::move(request->callback_), nullptr, status); } }; const std::string &serialized_runtime_env = - pop_worker_request->runtime_env_info.serialized_runtime_env(); + pop_worker_request->runtime_env_info_.serialized_runtime_env(); if (!IsRuntimeEnvEmpty(serialized_runtime_env)) { // create runtime env. GetOrCreateRuntimeEnv( serialized_runtime_env, - pop_worker_request->runtime_env_info.runtime_env_config(), - pop_worker_request->job_id, + pop_worker_request->runtime_env_info_.runtime_env_config(), + pop_worker_request->job_id_, [this, start_worker_process_fn, pop_worker_request]( bool successful, const std::string &serialized_runtime_env_context, @@ -1352,7 +1350,7 @@ void WorkerPool::StartNewWorker( start_worker_process_fn(pop_worker_request, serialized_runtime_env_context); } else { process_failed_runtime_env_setup_failed_++; - pop_worker_request->callback( + pop_worker_request->callback_( nullptr, PopWorkerStatus::RuntimeEnvCreationFailed, /*runtime_env_setup_error_message*/ setup_error_message); @@ -1428,7 +1426,7 @@ std::shared_ptr WorkerPool::FindAndPopIdleWorker( } return false; }; - auto &state = GetStateForLanguage(pop_worker_request.language); + auto &state = GetStateForLanguage(pop_worker_request.language_); auto worker_it = std::find_if(idle_of_all_languages_.rbegin(), idle_of_all_languages_.rend(), worker_fits_for_task_fn); @@ -1448,8 +1446,8 @@ std::shared_ptr WorkerPool::FindAndPopIdleWorker( // Assigned workers should always match the request's job_id // *except* if the task originates from a detached actor. RAY_CHECK(worker->GetAssignedJobId().IsNil() || - worker->GetAssignedJobId() == pop_worker_request.job_id || - !pop_worker_request.root_detached_actor_id.IsNil()); + worker->GetAssignedJobId() == pop_worker_request.job_id_ || + !pop_worker_request.root_detached_actor_id_.IsNil()); return worker; } @@ -1462,9 +1460,9 @@ void WorkerPool::PopWorker(std::shared_ptr pop_worker_request) return; } RAY_CHECK(worker->GetAssignedJobId().IsNil() || - worker->GetAssignedJobId() == pop_worker_request->job_id); + worker->GetAssignedJobId() == pop_worker_request->job_id_); ray_metric_num_workers_started_from_cache_.Record(1); - PopWorkerCallbackAsync(pop_worker_request->callback, worker, PopWorkerStatus::OK); + PopWorkerCallbackAsync(pop_worker_request->callback_, worker, PopWorkerStatus::OK); } void WorkerPool::PrestartWorkers(const TaskSpecification &task_spec, diff --git a/src/ray/raylet/worker_pool.h b/src/ray/raylet/worker_pool.h index 2f8b02e3b588..8da7d3046472 100644 --- a/src/ray/raylet/worker_pool.h +++ b/src/ray/raylet/worker_pool.h @@ -86,18 +86,18 @@ using PopWorkerCallback = const std::string &runtime_env_setup_error_message)>; struct PopWorkerRequest { - const rpc::Language language; - const rpc::WorkerType worker_type; - const JobID job_id; // can be Nil - const ActorID root_detached_actor_id; // can be Nil - const std::optional is_gpu; - const std::optional is_actor_worker; - const rpc::RuntimeEnvInfo runtime_env_info; - const int runtime_env_hash; - const std::vector dynamic_options; - std::optional worker_startup_keep_alive_duration; - - PopWorkerCallback callback; + const rpc::Language language_; + const rpc::WorkerType worker_type_; + const JobID job_id_; // can be Nil + const ActorID root_detached_actor_id_; // can be Nil + const std::optional is_gpu_; + const std::optional is_actor_worker_; + const rpc::RuntimeEnvInfo runtime_env_info_; + const int runtime_env_hash_; + const std::vector dynamic_options_; + std::optional worker_startup_keep_alive_duration_; + + PopWorkerCallback callback_; PopWorkerRequest(rpc::Language lang, rpc::WorkerType worker_type, @@ -110,18 +110,17 @@ struct PopWorkerRequest { std::vector options, std::optional worker_startup_keep_alive_duration, PopWorkerCallback callback) - : language(lang), - worker_type(worker_type), - job_id(job), - root_detached_actor_id(root_actor_id), - is_gpu(gpu), - is_actor_worker(actor_worker), - runtime_env_info(std::move(runtime_env_info)), - // this-> is needed to disambiguate the member variable from the ctor arg. - runtime_env_hash(runtime_env_hash), - dynamic_options(std::move(options)), - worker_startup_keep_alive_duration(worker_startup_keep_alive_duration), - callback(std::move(callback)) {} + : language_(lang), + worker_type_(worker_type), + job_id_(job), + root_detached_actor_id_(root_actor_id), + is_gpu_(gpu), + is_actor_worker_(actor_worker), + runtime_env_info_(std::move(runtime_env_info)), + runtime_env_hash_(runtime_env_hash), + dynamic_options_(std::move(options)), + worker_startup_keep_alive_duration_(worker_startup_keep_alive_duration), + callback_(std::move(callback)) {} }; /// \class IOWorkerPoolInterface @@ -872,7 +871,7 @@ class WorkerPool : public WorkerPoolInterface { /// The callback that will be triggered once it times out to start a worker. std::function starting_worker_timeout_callback_; /// If 1, expose Ray debuggers started by the workers externally (to this node). - int ray_debugger_external; + int ray_debugger_external_; /// If the first job has already been registered. bool first_job_registered_ = false; diff --git a/src/ray/rpc/worker/core_worker_client.cc b/src/ray/rpc/worker/core_worker_client.cc index a182c020de7d..6863511dfe05 100644 --- a/src/ray/rpc/worker/core_worker_client.cc +++ b/src/ray/rpc/worker/core_worker_client.cc @@ -110,7 +110,7 @@ void CoreWorkerClient::SendRequests() { [this, this_ptr, seq_no, task_size, callback = std::move(pair.second)]( Status status, rpc::PushTaskReply &&reply) { { - absl::MutexLock lock(&mutex_); + absl::MutexLock lk(&mutex_); if (seq_no > max_finished_seq_no_) { max_finished_seq_no_ = seq_no; } diff --git a/src/ray/rpc/worker/core_worker_client.h b/src/ray/rpc/worker/core_worker_client.h index 1ad30728cf62..341f07e9786a 100644 --- a/src/ray/rpc/worker/core_worker_client.h +++ b/src/ray/rpc/worker/core_worker_client.h @@ -37,11 +37,11 @@ namespace std { template <> struct hash { size_t operator()(const ray::rpc::Address &addr) const { - size_t hash = std::hash()(addr.port()); - hash ^= std::hash()(addr.ip_address()); - hash ^= std::hash()(addr.worker_id()); - hash ^= std::hash()(addr.node_id()); - return hash; + size_t hash_value = std::hash()(addr.port()); + hash_value ^= std::hash()(addr.ip_address()); + hash_value ^= std::hash()(addr.worker_id()); + hash_value ^= std::hash()(addr.node_id()); + return hash_value; } }; } // namespace std diff --git a/src/ray/rpc/worker/core_worker_client_pool.cc b/src/ray/rpc/worker/core_worker_client_pool.cc index 96f46cc3cbe8..c66b860daaaf 100644 --- a/src/ray/rpc/worker/core_worker_client_pool.cc +++ b/src/ray/rpc/worker/core_worker_client_pool.cc @@ -138,15 +138,15 @@ std::shared_ptr CoreWorkerClientPool::GetOrConnect( RAY_LOG(DEBUG) << "Connected to worker " << worker_id << " with address " << BuildAddress(addr_proto.ip_address(), addr_proto.port()); - return entry.core_worker_client; + return entry.core_worker_client_; } void CoreWorkerClientPool::RemoveIdleClients() { while (!client_list_.empty()) { - auto worker_id = client_list_.back().worker_id; - auto node_id = client_list_.back().node_id; + auto worker_id = client_list_.back().worker_id_; + auto node_id = client_list_.back().node_id_; // The last client in the list is the least recent accessed client. - if (client_list_.back().core_worker_client->IsIdleAfterRPCs()) { + if (client_list_.back().core_worker_client_->IsIdleAfterRPCs()) { worker_client_map_.erase(worker_id); EraseFromNodeClientMap(node_id, worker_id); client_list_.pop_back(); @@ -169,7 +169,7 @@ void CoreWorkerClientPool::Disconnect(ray::WorkerID id) { if (it == worker_client_map_.end()) { return; } - EraseFromNodeClientMap(it->second->node_id, /*worker_id=*/id); + EraseFromNodeClientMap(it->second->node_id_, /*worker_id=*/id); client_list_.erase(it->second); worker_client_map_.erase(it); } diff --git a/src/ray/rpc/worker/core_worker_client_pool.h b/src/ray/rpc/worker/core_worker_client_pool.h index f26ae8d81938..370ac45cf6fd 100644 --- a/src/ray/rpc/worker/core_worker_client_pool.h +++ b/src/ray/rpc/worker/core_worker_client_pool.h @@ -90,13 +90,13 @@ class CoreWorkerClientPool { CoreWorkerClientEntry(WorkerID worker_id, NodeID node_id, std::shared_ptr core_worker_client) - : worker_id(std::move(worker_id)), - node_id(std::move(node_id)), - core_worker_client(std::move(core_worker_client)) {} + : worker_id_(std::move(worker_id)), + node_id_(std::move(node_id)), + core_worker_client_(std::move(core_worker_client)) {} - WorkerID worker_id; - NodeID node_id; - std::shared_ptr core_worker_client; + WorkerID worker_id_; + NodeID node_id_; + std::shared_ptr core_worker_client_; }; /// A list of open connections from the most recent accessed to the least recent diff --git a/thirdparty/patches/abseil-cpp-shadow.patch b/thirdparty/patches/abseil-cpp-shadow.patch new file mode 100644 index 000000000000..b014e8a6dd23 --- /dev/null +++ b/thirdparty/patches/abseil-cpp-shadow.patch @@ -0,0 +1,12 @@ +diff --git absl/container/internal/btree.h absl/container/internal/btree.h +--- absl/container/internal/btree.h ++++ absl/container/internal/btree.h +@@ -223,7 +223,7 @@ struct key_compare_adapter { + + public: + using Base::Base; +- checked_compare(Compare comp) : Base(std::move(comp)) {} // NOLINT ++ checked_compare(Compare _comp) : Base(std::move(_comp)) {} // NOLINT + + // Allow converting to Compare for use in key_comp()/value_comp(). + explicit operator Compare() const { return comp(); } diff --git a/thirdparty/patches/msgpack-shadow.patch b/thirdparty/patches/msgpack-shadow.patch new file mode 100644 index 000000000000..581a1e7fde3a --- /dev/null +++ b/thirdparty/patches/msgpack-shadow.patch @@ -0,0 +1,12 @@ +diff --git include/msgpack/v1/adaptor/fixint.hpp include/msgpack/v1/adaptor/fixint.hpp +--- include/msgpack/v1/adaptor/fixint.hpp ++++ include/msgpack/v1/adaptor/fixint.hpp +@@ -24,7 +24,7 @@ template + struct fix_int { + typedef T value_type; + fix_int() : value(0) { } +- fix_int(T value) : value(value) { } ++ fix_int(T _value) : value(_value) { } + + operator T() const { return value; } + From d63f464491a079e7bf90155d09fface72c71d189 Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Thu, 21 Aug 2025 02:27:49 +0530 Subject: [PATCH 203/634] [core] Introduce `ShutdownCoordinator` and unified core worker shutdown entry points (#54244) ## Why are these changes needed? This PR is the first installment of the core worker shutdown refactor. It replaces the previous tangle of atomics, flags, and scattered helper functions with a single, test-driven shutdown coordinator and a thin executor layer. All existing shutdown entry-points (`Shutdown()`, `Exit()`, `ForceExit()`, RPC `HandleExit`, signal-generated exits, etc.) are now routed through one code path that is: 1. Thread-safe & idempotent 2. Reason-aware 3. Unit-tested ## High-level design All shutdown entry points (`Shutdown()`, `Exit()`, `ForceExit()`, RPC `HandleExit`, etc.) call `ShutdownCoordinator::RequestShutdown(...)`. The coordinator: - Tracks `ShutdownState` and `ShutdownReason` under a single mutex. - Latches a forced shutdown at entry (so only one force executes). - Delegates actual work to an injectable `ShutdownExecutorInterface` implementation (`CoreWorkerShutdownExecutor`) to keep shutdown orchestration separate from teardown steps. ### Key Components or Changes - `ShutdownCoordinator` (src/ray/core_worker/shutdown_coordinator.{h,cc}) - Switched from atomics to a single `std::mutex` for clarity and correctness. - Added early force latch (`force_started_`) and a single-execution guard (`force_executed_`). - Improved error messages (construction-time `RAY_CHECK`) and actionable fatal for unknown worker types. - `CoreWorkerShutdownExecutor` (src/ray/core_worker/core_worker_shutdown_executor.{h,cc}) - Documented semantics of graceful vs force vs worker/handle exit. - Guarded IO-thread join to avoid self-join deadlocks. - Best-effort idempotent disconnect semantics; force path kills child processes, disconnects, and terminates. - `CoreWorker` integration (src/ray/core_worker/core_worker.{h,cc}) - Added `shutdown_coordinator_` and routed all exits through it. - Added `IsIdle()` helper. - Legacy atomics removed - Tests (src/ray/core_worker/tests/shutdown_coordinator_test.cc) - Added a tiny `FakeShutdownExecutor` and `NoOpShutdownExecutor` to help with the tests. - Added concurrency tests. - Added state transition tests. ## Behavioral notes - Idempotency: Subsequent shutdown requests no-op once shutdown has started. - Concurrency: - (graceful, forced): force latches and preempts; executes once. - (graceful, graceful): only the first succeeds. - (forced, forced): only the first executes; subsequent force requests return false (do not change reason). - (forced, graceful): graceful returns false; force already latched. ### Performance Notes Shutdown paths are control-path and take a mutex for clarity; hot paths checks are unaffected. ### Future Work 1. Unify signal handlers to call RequestShutdown. 2. Extract more cleanup logic into the executor with dependency-injected mocks. 3. Add metrics and tracing once the coordinator is fully wired. 4. Migrate actor-specific shutdown semantics and delete remaining dead code (`ForceExit`, `KillChildProcs`) once safe. --------- Signed-off-by: Sagar Sumit --- src/ray/core_worker/BUILD.bazel | 4 + src/ray/core_worker/core_worker.cc | 270 ++++-------- src/ray/core_worker/core_worker.h | 28 +- .../core_worker_shutdown_executor.cc | 309 +++++++++++++ .../core_worker_shutdown_executor.h | 104 +++++ src/ray/core_worker/shutdown_coordinator.cc | 293 +++++++++++++ src/ray/core_worker/shutdown_coordinator.h | 294 +++++++++++++ src/ray/core_worker/tests/BUILD.bazel | 12 + .../tests/shutdown_coordinator_test.cc | 406 ++++++++++++++++++ 9 files changed, 1535 insertions(+), 185 deletions(-) create mode 100644 src/ray/core_worker/core_worker_shutdown_executor.cc create mode 100644 src/ray/core_worker/core_worker_shutdown_executor.h create mode 100644 src/ray/core_worker/shutdown_coordinator.cc create mode 100644 src/ray/core_worker/shutdown_coordinator.h create mode 100644 src/ray/core_worker/tests/shutdown_coordinator_test.cc diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index f4410f8a9695..625744d2d526 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -5,11 +5,15 @@ ray_cc_library( srcs = [ "core_worker.cc", "core_worker_process.cc", + "core_worker_shutdown_executor.cc", + "shutdown_coordinator.cc", ], hdrs = [ "core_worker.h", "core_worker_process.h", "core_worker_rpc_proxy.h", + "core_worker_shutdown_executor.h", + "shutdown_coordinator.h", ], deps = [ ":actor_handle", diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index eef949f69a09..c816dd2a09e6 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -23,6 +23,9 @@ #include #include +#include "ray/core_worker/core_worker_shutdown_executor.h" +#include "ray/core_worker/shutdown_coordinator.h" + #ifndef _WIN32 #include #endif @@ -123,6 +126,28 @@ std::optional TryGetLocalObjectLocation( return CreateObjectLocation(object_info); } +/// Converts rpc::WorkerExitType to ShutdownReason +/// \param exit_type The worker exit type to convert +/// \param is_force_exit If true, INTENDED_USER_EXIT maps to kForcedExit; otherwise +/// kGracefulExit +ShutdownReason ConvertExitTypeToShutdownReason(rpc::WorkerExitType exit_type, + bool is_force_exit = false) { + switch (exit_type) { + case rpc::WorkerExitType::INTENDED_SYSTEM_EXIT: + return ShutdownReason::kIntentionalShutdown; + case rpc::WorkerExitType::INTENDED_USER_EXIT: + return is_force_exit ? ShutdownReason::kForcedExit : ShutdownReason::kGracefulExit; + case rpc::WorkerExitType::USER_ERROR: + return ShutdownReason::kUserError; + case rpc::WorkerExitType::SYSTEM_ERROR: + return ShutdownReason::kUnexpectedError; + case rpc::WorkerExitType::NODE_OUT_OF_MEMORY: + return ShutdownReason::kOutOfMemory; + default: + return ShutdownReason::kUnexpectedError; + } +} + } // namespace JobID GetProcessJobID(const CoreWorkerOptions &options) { @@ -489,61 +514,23 @@ CoreWorker::CoreWorker( // NOTE: This also marks the worker as available in Raylet. We do this at the very end // in case there is a problem during construction. ConnectToRayletInternal(); + + // Initialize shutdown coordinator last - after all services are ready + // Create concrete shutdown executor that implements real shutdown operations + auto shutdown_executor = std::make_unique(this); + shutdown_coordinator_ = std::make_unique( + std::move(shutdown_executor), options_.worker_type); + + RAY_LOG(DEBUG) << "Initialized unified shutdown coordinator with concrete executor for " + "worker type: " + << WorkerTypeString(options_.worker_type); } // NOLINT(readability/fn_size) CoreWorker::~CoreWorker() { RAY_LOG(INFO) << "Core worker is destructed"; } void CoreWorker::Shutdown() { - // Ensure that the shutdown logic runs at most once. - bool expected = false; - if (!is_shutdown_.compare_exchange_strong(expected, /*desired=*/true)) { - RAY_LOG(INFO) << "Shutdown was called more than once, ignoring."; - return; - } - - // For actors, perform cleanup before shutdown proceeds. - if (!GetWorkerContext().GetCurrentActorID().IsNil() && actor_shutdown_callback_) { - RAY_LOG(INFO) << "Calling actor shutdown callback before shutdown"; - actor_shutdown_callback_(); - } - RAY_LOG(INFO) << "Shutting down."; - - if (options_.worker_type == WorkerType::WORKER) { - // Running in a main thread. - // Asyncio coroutines could still run after CoreWorker is removed because it is - // running in a different thread. This can cause segfault because coroutines try to - // access CoreWorker methods that are already garbage collected. We should complete - // all coroutines before shutting down in order to prevent this. - if (worker_context_->CurrentActorIsAsync()) { - options_.terminate_asyncio_thread(); - } - task_execution_service_.stop(); - } - - task_event_buffer_->FlushEvents(/*forced=*/true); - task_event_buffer_->Stop(); - - io_service_.stop(); - RAY_LOG(INFO) << "Waiting for joining a core worker io thread. If it hangs here, there " - "might be deadlock or a high load in the core worker io service."; - if (io_thread_.joinable()) { - io_thread_.join(); - } - - // Shutdown gRPC server - core_worker_server_->Shutdown(); - - // Now that gcs_client is not used within io service, we can reset the pointer and clean - // it up. - if (gcs_client_) { - RAY_LOG(INFO) << "Disconnecting a GCS client."; - // TODO(hjiang): Move the Disconnect() logic - // to GcsClient destructor. - gcs_client_->Disconnect(); - gcs_client_.reset(); - } - - RAY_LOG(INFO) << "Core worker ready to be deallocated."; + shutdown_coordinator_->RequestShutdown( + /*force_shutdown=*/false, ShutdownReason::kGracefulExit, "ray.shutdown() called"); } void CoreWorker::ConnectToRayletInternal() { @@ -650,118 +637,29 @@ void CoreWorker::Exit( const rpc::WorkerExitType exit_type, const std::string &detail, const std::shared_ptr &creation_task_exception_pb_bytes) { - // Ensure that the exit logic runs at most once. - bool expected = false; - if (!is_exited_.compare_exchange_strong(expected, /*desired=*/true)) { - RAY_LOG(INFO) << "Exit was called multipled times, ignoring."; - return; - } + // Preserve actor creation failure details by marking a distinct shutdown reason + // when initialization raised an exception. An exception payload is provided. + ShutdownReason reason = creation_task_exception_pb_bytes != nullptr + ? ShutdownReason::kActorCreationFailed + : ConvertExitTypeToShutdownReason(exit_type); - RAY_LOG(INFO) << "Exit signal received, this process will exit after all outstanding " - "tasks have finished" - << ", exit_type=" << rpc::WorkerExitType_Name(exit_type) - << ", detail=" << detail; - { - absl::MutexLock lock(&mutex_); - RAY_CHECK_NE(detail, ""); - exiting_detail_ = std::optional{detail}; - } - - // Callback to shutdown. - auto shutdown = [this, exit_type, detail, creation_task_exception_pb_bytes]() { - // To avoid problems, make sure shutdown is always called from the same - // event loop each time. - task_execution_service_.post( - [this, exit_type, detail, creation_task_exception_pb_bytes]() { - rpc::DrainServerCallExecutor(); - KillChildProcs(); - // Disconnect should be put close to Shutdown - // https://github.com/ray-project/ray/pull/34883 - // TODO(iycheng): Improve the Process.h and make it able to monitor - // process liveness - Disconnect(exit_type, detail, creation_task_exception_pb_bytes); - Shutdown(); - }, - "CoreWorker.Shutdown"); - }; - // Callback to drain objects once all pending tasks have been drained. - auto drain_references_callback = [this, shutdown]() { - // Post to the event loop to avoid a deadlock between the TaskManager and - // the ReferenceCounter. The deadlock can occur because this callback may - // get called by the TaskManager while the ReferenceCounter's lock is held, - // but the callback itself must acquire the ReferenceCounter's lock to - // drain the object references. - task_execution_service_.post( - [this, shutdown]() { - RAY_LOG(INFO) << "Wait for currently executing tasks in the underlying thread " - "pools to finish."; - // Wait for currently executing tasks in the underlying thread pools to - // finish. Note that if tasks have been posted to the thread pools but not - // started yet, they will not be executed. - task_receiver_->Stop(); - - // Release resources only after tasks have stopped executing. - auto status = raylet_ipc_client_->NotifyDirectCallTaskBlocked(); - if (!status.ok()) { - RAY_LOG(WARNING) - << "Failed to notify Raylet. The raylet may have already shut down or " - << "the connection was lost."; - } - - bool not_actor_task = false; - { - absl::MutexLock lock(&mutex_); - not_actor_task = actor_id_.IsNil(); - } - if (not_actor_task) { - // Normal tasks should not hold any object references in the heap after - // executing, but they could in the case that one was stored as a glob - // variable (anti-pattern, but possible). We decrement the reference count - // for all local references to account for this. After this call, the only - // references left to drain should be those that are in use by remote - // workers. If these workers hold their references forever, the call to - // drain the reference counter will hang forever and this process will not - // exit until it is forcibly removed (e.g., via SIGKILL). - // - // NOTE(edoakes): this is only safe to do _after_ we have drained executing - // tasks in the task_receiver_, otherwise there might still be user code - // running that relies on the state of the reference counter. - // See: https://github.com/ray-project/ray/pull/53002. - RAY_LOG(INFO) - << "Releasing local references, then draining reference counter."; - reference_counter_->ReleaseAllLocalReferences(); - reference_counter_->DrainAndShutdown(shutdown); - } else { - // If we are an actor, then we may be holding object references in the - // heap. Then, we should not wait to drain the object references before - // shutdown since this could hang. - RAY_LOG(INFO) - << "Not draining reference counter since this is an actor worker."; - shutdown(); - } - }, - "CoreWorker.DrainAndShutdown"); - }; - - task_manager_->DrainAndShutdown(drain_references_callback); + shutdown_coordinator_->RequestShutdown(/*force_shutdown=*/false, + reason, + detail, + ShutdownCoordinator::kInfiniteTimeout, + creation_task_exception_pb_bytes); } void CoreWorker::ForceExit(const rpc::WorkerExitType exit_type, const std::string &detail) { - RAY_LOG(WARNING) << "Force exit the process. " - << " Details: " << detail; + RAY_LOG(DEBUG) << "ForceExit called: exit_type=" << static_cast(exit_type) + << ", detail=" << detail; - KillChildProcs(); - // Disconnect should be put close to Exit - // https://github.com/ray-project/ray/pull/34883 - // TODO(iycheng): Improve the Process.h and make it able to monitor - // process liveness - Disconnect(exit_type, detail); + ShutdownReason reason = ConvertExitTypeToShutdownReason(exit_type, true); + shutdown_coordinator_->RequestShutdown( + /*force_shutdown=*/true, reason, detail, std::chrono::milliseconds{0}, nullptr); - // NOTE(hchen): Use `QuickExit()` to force-exit this process without doing cleanup. - // `exit()` will destruct static objects in an incorrect order, which will lead to - // core dumps. - QuickExit(); + RAY_LOG(DEBUG) << "ForceExit: shutdown request completed"; } const WorkerID &CoreWorker::GetWorkerID() const { return worker_context_->GetWorkerID(); } @@ -2756,7 +2654,7 @@ void CoreWorker::RunTaskExecutionLoop() { "CoreWorker.CheckSignal"); } task_execution_service_.run(); - RAY_CHECK(is_shutdown_) + RAY_CHECK(shutdown_coordinator_ && shutdown_coordinator_->IsShuttingDown()) << "Task execution loop was terminated without calling shutdown API."; } @@ -4091,11 +3989,14 @@ void CoreWorker::HandleKillActor(rpc::KillActorRequest request, if (request.force_kill()) { RAY_LOG(INFO) << "Force kill actor request has received. exiting immediately... " << kill_actor_reason; + RAY_LOG(DEBUG) << "HandleKillActor: About to call ForceExit"; // If we don't need to restart this actor, we notify raylet before force killing it. ForceExit( rpc::WorkerExitType::INTENDED_SYSTEM_EXIT, absl::StrCat("Worker exits because the actor is killed. ", kill_actor_reason)); + RAY_LOG(DEBUG) << "HandleKillActor: ForceExit completed"; } else { + RAY_LOG(DEBUG) << "HandleKillActor: About to call Exit"; Exit(rpc::WorkerExitType::INTENDED_SYSTEM_EXIT, absl::StrCat("Worker exits because the actor is killed. ", kill_actor_reason)); } @@ -4293,16 +4194,12 @@ void CoreWorker::HandleDeleteSpilledObjects(rpc::DeleteSpilledObjectsRequest req void CoreWorker::HandleExit(rpc::ExitRequest request, rpc::ExitReply *reply, rpc::SendReplyCallback send_reply_callback) { - const size_t num_objects_with_references = reference_counter_->Size(); - const size_t num_pending_tasks = task_manager_->NumPendingTasks(); - const int64_t pins_in_flight = local_raylet_rpc_client_->GetPinsInFlight(); - // We consider the worker to be idle if it doesn't have object references and it doesn't - // have any object pinning RPCs in flight and it doesn't have pending tasks. - bool is_idle = (num_objects_with_references == 0) && (pins_in_flight == 0) && - (num_pending_tasks == 0); + bool is_idle = IsIdle(); bool force_exit = request.force_exit(); RAY_LOG(DEBUG) << "Exiting: is_idle: " << is_idle << " force_exit: " << force_exit; if (!is_idle) { + const size_t num_pending_tasks = task_manager_->NumPendingTasks(); + const int64_t pins_in_flight = local_raylet_rpc_client_->GetPinsInFlight(); RAY_LOG_EVERY_MS(INFO, 60000) << "Worker is not idle: reference counter: " << reference_counter_->DebugString() << " # pins in flight: " << pins_in_flight @@ -4319,21 +4216,29 @@ void CoreWorker::HandleExit(rpc::ExitRequest request, send_reply_callback( Status::OK(), [this, will_exit, force_exit]() { - // If the worker is idle, we exit. + if (!will_exit) { + return; + } + + ShutdownReason reason; + std::string detail; + if (force_exit) { - ForceExit(rpc::WorkerExitType::INTENDED_SYSTEM_EXIT, - "Worker force exits because its job has finished"); - } else if (will_exit) { - Exit(rpc::WorkerExitType::INTENDED_SYSTEM_EXIT, - "Worker exits because it was idle (it doesn't have objects it owns while " - "no task or actor has been scheduled) for a long time."); + reason = ShutdownReason::kForcedExit; + detail = "Worker force exited because its job has finished"; + } else { + reason = ShutdownReason::kIdleTimeout; + detail = "Worker exited because it was idle for a long time"; } + + shutdown_coordinator_->RequestShutdown(force_exit, reason, detail); }, - // We need to kill it regardless if the RPC failed. + // Fallback on RPC failure - still attempt shutdown [this]() { - Exit(rpc::WorkerExitType::INTENDED_SYSTEM_EXIT, - "Worker exits because it was idle (it doesn't have objects it owns while " - "no task or actor has been scheduled) for a long time."); + shutdown_coordinator_->RequestShutdown( + /*force_shutdown=*/false, + ShutdownReason::kIdleTimeout, + "Worker exited due to RPC failure during idle exit"); }); } @@ -4501,9 +4406,20 @@ rpc::JobConfig CoreWorker::GetJobConfig() const { return worker_context_->GetCurrentJobConfig(); } -bool CoreWorker::IsExiting() const { - absl::MutexLock lock(&mutex_); - return exiting_detail_.has_value(); +bool CoreWorker::IsExiting() const { return shutdown_coordinator_->ShouldEarlyExit(); } + +bool CoreWorker::IsIdle(size_t num_objects_with_references, + int64_t pins_in_flight, + size_t num_pending_tasks) const { + return (num_objects_with_references == 0) && (pins_in_flight == 0) && + (num_pending_tasks == 0); +} + +bool CoreWorker::IsIdle() const { + const size_t num_objects_with_references = reference_counter_->Size(); + const size_t num_pending_tasks = task_manager_->NumPendingTasks(); + const int64_t pins_in_flight = local_raylet_rpc_client_->GetPinsInFlight(); + return IsIdle(num_objects_with_references, pins_in_flight, num_pending_tasks); } Status CoreWorker::WaitForActorRegistered(const std::vector &ids) { diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index 3380aa0e489e..2099d4b740ff 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -42,6 +42,7 @@ #include "ray/core_worker/object_recovery_manager.h" #include "ray/core_worker/profile_event.h" #include "ray/core_worker/reference_count.h" +#include "ray/core_worker/shutdown_coordinator.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/core_worker/store_provider/plasma_store_provider.h" #include "ray/core_worker/task_event_buffer.h" @@ -1673,6 +1674,17 @@ class CoreWorker { const int64_t timeout_ms, std::vector> &results); + /// Helper to compute idleness from precomputed counters. + /// + /// We consider the worker to be idle if it doesn't have object references and it + /// doesn't have any object pinning RPCs in flight and it doesn't have pending tasks. + bool IsIdle(size_t num_objects_with_references, + int64_t pins_in_flight, + size_t num_pending_tasks) const; + + /// Convenience overload that fetches counters and evaluates idleness. + bool IsIdle() const; + /// Get the caller ID used to submit tasks from this worker to an actor. /// /// \return The caller ID. For non-actor tasks, this is the current task ID. @@ -1872,14 +1884,10 @@ class CoreWorker { /// If this value is set, it means the exit process has begun. std::optional exiting_detail_ ABSL_GUARDED_BY(mutex_); - /// TODO(kevin85421): the shutdown logic contained in `Disconnect`, `Exit`, and - /// `Shutdown` should be unified to avoid mistakes due to complex dependent semantics. - /// See https://github.com/ray-project/ray/issues/51642. - - /// Used to ensure that the `CoreWorker::Exit` method is called at most once. - std::atomic is_exited_ = false; - /// Used to ensure that the `CoreWorker::Shutdown` method is called at most once. - std::atomic is_shutdown_ = false; + /// Unified shutdown coordinator that manages all shutdown operations. + /// Implements a thread-safe, single state machine that coordinates + /// all shutdown entry points. + std::unique_ptr shutdown_coordinator_; int64_t max_direct_call_object_size_; @@ -1926,6 +1934,10 @@ class CoreWorker { /// Used to ensure we only subscribe to node changes once. std::once_flag subscribe_to_node_changes_flag_; + // Grant CoreWorkerShutdownExecutor access to CoreWorker internals for orchestrating + // the shutdown procedure without exposing additional public APIs. + friend class CoreWorkerShutdownExecutor; + /// Used to block in certain spots if the GCS node cache is needed. std::mutex gcs_client_node_cache_populated_mutex_; std::condition_variable gcs_client_node_cache_populated_cv_; diff --git a/src/ray/core_worker/core_worker_shutdown_executor.cc b/src/ray/core_worker/core_worker_shutdown_executor.cc new file mode 100644 index 000000000000..03cfc937d81a --- /dev/null +++ b/src/ray/core_worker/core_worker_shutdown_executor.cc @@ -0,0 +1,309 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/core_worker/core_worker_shutdown_executor.h" + +#include +#include +#include +#include + +#include "ray/core_worker/core_worker.h" + +namespace ray { + +namespace core { + +CoreWorkerShutdownExecutor::CoreWorkerShutdownExecutor(CoreWorker *core_worker) + : core_worker_(core_worker) {} + +void CoreWorkerShutdownExecutor::ExecuteGracefulShutdown( + std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) { + RAY_LOG(INFO) << "Executing graceful shutdown: " << exit_type << " - " << detail + << " (timeout: " << timeout_ms.count() << "ms)"; + + // For actors, perform cleanup before shutdown proceeds. + if (!core_worker_->worker_context_->GetCurrentActorID().IsNil() && + core_worker_->actor_shutdown_callback_) { + RAY_LOG(INFO) << "Calling actor shutdown callback before shutdown"; + core_worker_->actor_shutdown_callback_(); + } + + if (core_worker_->options_.worker_type == WorkerType::WORKER) { + // Running in a main thread. + // Asyncio coroutines could still run after CoreWorker is removed because it is + // running in a different thread. This can cause segfault because coroutines try to + // access CoreWorker methods that are already garbage collected. We should complete + // all coroutines before shutting down in order to prevent this. + if (core_worker_->worker_context_->CurrentActorIsAsync()) { + core_worker_->options_.terminate_asyncio_thread(); + } + core_worker_->task_execution_service_.stop(); + } + + core_worker_->task_event_buffer_->FlushEvents(/*forced=*/true); + core_worker_->task_event_buffer_->Stop(); + + core_worker_->io_service_.stop(); + RAY_LOG(INFO) << "Waiting for joining a core worker io thread. If it hangs here, there " + "might be deadlock or a high load in the core worker io service."; + if (core_worker_->io_thread_.joinable()) { + // Check if we're already running in the IO thread to avoid self-join deadlock + if (core_worker_->io_thread_.get_id() != boost::this_thread::get_id()) { + core_worker_->io_thread_.join(); + } else { + RAY_LOG(INFO) + << "Skipping IO thread join since we're already running in the IO thread"; + } + } + + // Shutdown gRPC server + core_worker_->core_worker_server_->Shutdown(); + + // Now that gcs_client is not used within io service, we can reset the pointer and clean + // it up. + if (core_worker_->gcs_client_) { + RAY_LOG(INFO) << "Disconnecting a GCS client."; + // TODO(55607): Move the Disconnect() logic to GcsClient destructor. + // https://github.com/ray-project/ray/issues/55607 + core_worker_->gcs_client_->Disconnect(); + core_worker_->gcs_client_.reset(); + } + + RAY_LOG(INFO) << "Core worker ready to be deallocated."; +} + +void CoreWorkerShutdownExecutor::ExecuteForceShutdown(std::string_view exit_type, + std::string_view detail) { + KillChildProcessesImmediately(); + DisconnectServices(exit_type, detail, nullptr); + QuickExit(); +} + +void CoreWorkerShutdownExecutor::ExecuteWorkerExit(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) { + ExecuteExit(exit_type, detail, timeout_ms, nullptr); +} + +void CoreWorkerShutdownExecutor::ExecuteExit( + std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms, + const std::shared_ptr &creation_task_exception_pb_bytes) { + RAY_LOG(INFO) << "Executing worker exit: " << exit_type << " - " << detail + << " (timeout: " << timeout_ms.count() << "ms)"; + + { + absl::MutexLock lock(&core_worker_->mutex_); + RAY_CHECK_NE(detail, ""); + core_worker_->exiting_detail_ = std::optional{detail}; + } + + auto shutdown_callback = [this, + exit_type = std::string(exit_type), + detail = std::string(detail), + creation_task_exception_pb_bytes]() { + // To avoid problems, make sure shutdown is always called from the same + // event loop each time. + core_worker_->task_execution_service_.post( + [this, exit_type, detail, creation_task_exception_pb_bytes]() { + rpc::DrainServerCallExecutor(); + KillChildProcessesImmediately(); + // Disconnect should be put close to Shutdown + // https://github.com/ray-project/ray/pull/34883 + DisconnectServices(exit_type, detail, creation_task_exception_pb_bytes); + ExecuteGracefulShutdown( + exit_type, "Post-exit graceful shutdown", std::chrono::milliseconds{30000}); + }, + "CoreWorker.Shutdown"); + }; + + auto drain_references_callback = [this, shutdown_callback]() { + // Post to the event loop to avoid a deadlock between the TaskManager and + // the ReferenceCounter. The deadlock can occur because this callback may + // get called by the TaskManager while the ReferenceCounter's lock is held, + // but the callback itself must acquire the ReferenceCounter's lock to + // drain the object references. + core_worker_->task_execution_service_.post( + [this, shutdown_callback]() { + RAY_LOG(INFO) << "Wait for currently executing tasks in the underlying thread " + "pools to finish."; + // Wait for currently executing tasks in the underlying thread pools to + // finish. Note that if tasks have been posted to the thread pools but not + // started yet, they will not be executed. + core_worker_->task_receiver_->Stop(); + + // Release resources only after tasks have stopped executing. + auto status = core_worker_->raylet_ipc_client_->NotifyDirectCallTaskBlocked(); + if (!status.ok()) { + RAY_LOG(WARNING) + << "Failed to notify Raylet. The raylet may have already shut down or " + << "the connection was lost."; + } + + bool not_actor_task = false; + { + absl::MutexLock lock(&core_worker_->mutex_); + not_actor_task = core_worker_->actor_id_.IsNil(); + } + if (not_actor_task) { + // Normal tasks should not hold any object references in the heap after + // executing, but they could in the case that one was stored as a glob + // variable (anti-pattern, but possible). We decrement the reference count + // for all local references to account for this. After this call, the only + // references left to drain should be those that are in use by remote + // workers. If these workers hold their references forever, the call to + // drain the reference counter will hang forever and this process will not + // exit until it is forcibly removed (e.g., via SIGKILL). + // + // NOTE(edoakes): this is only safe to do _after_ we have drained executing + // tasks in the task_receiver_, otherwise there might still be user code + // running that relies on the state of the reference counter. + // See: https://github.com/ray-project/ray/pull/53002. + RAY_LOG(INFO) + << "Releasing local references, then draining reference counter."; + core_worker_->reference_counter_->ReleaseAllLocalReferences(); + core_worker_->reference_counter_->DrainAndShutdown(shutdown_callback); + } else { + // If we are an actor, then we may be holding object references in the + // heap. Then, we should not wait to drain the object references before + // shutdown since this could hang. + RAY_LOG(INFO) + << "Not draining reference counter since this is an actor worker."; + shutdown_callback(); + } + }, + "CoreWorker.DrainAndShutdown"); + }; + + core_worker_->task_manager_->DrainAndShutdown(drain_references_callback); +} + +void CoreWorkerShutdownExecutor::ExecuteHandleExit(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) { + RAY_LOG(INFO) << "Executing handle exit: " << exit_type << " - " << detail + << " (timeout: " << timeout_ms.count() << "ms)"; + + if (ShouldWorkerIdleExit()) { + auto actual_timeout = timeout_ms; + if (actual_timeout.count() == -1) { + actual_timeout = std::chrono::milliseconds{10000}; // 10s default + } + + ExecuteWorkerExit(exit_type, detail, actual_timeout); + } else { + RAY_LOG(INFO) << "Worker not idle, ignoring exit request: " << detail; + } +} + +void CoreWorkerShutdownExecutor::KillChildProcessesImmediately() { + if (!RayConfig::instance().kill_child_processes_on_worker_exit()) { + RAY_LOG(DEBUG) + << "kill_child_processes_on_worker_exit is not true, skipping KillChildProcs"; + return; + } + + RAY_LOG(DEBUG) << "kill_child_processes_on_worker_exit true, KillChildProcs"; + auto maybe_child_procs = GetAllProcsWithPpid(GetPID()); + + // Enumerating child procs is not supported on this platform. + if (!maybe_child_procs) { + RAY_LOG(DEBUG) << "Killing leaked procs not supported on this platform."; + return; + } + + const auto &child_procs = *maybe_child_procs; + const auto child_procs_str = absl::StrJoin(child_procs, ","); + RAY_LOG(INFO) << "Try killing all child processes of this worker as it exits. " + << "Child process pids: " << child_procs_str; + + for (const auto &child_pid : child_procs) { + auto maybe_error_code = KillProc(child_pid); + RAY_CHECK(maybe_error_code) + << "Expected this path to only be called when KillProc is supported."; + auto error_code = *maybe_error_code; + + RAY_LOG(INFO) << "Kill result for child pid " << child_pid << ": " + << error_code.message() << ", bool " << static_cast(error_code); + if (error_code) { + RAY_LOG(WARNING) << "Unable to kill potentially leaked process " << child_pid + << ": " << error_code.message(); + } + } +} + +bool CoreWorkerShutdownExecutor::ShouldWorkerIdleExit() const { + return core_worker_->IsIdle(); +} + +void CoreWorkerShutdownExecutor::DisconnectServices( + std::string_view exit_type, + std::string_view detail, + const std::shared_ptr &creation_task_exception_pb_bytes) { + core_worker_->RecordMetrics(); + + if (core_worker_->options_.worker_type == WorkerType::DRIVER && + core_worker_->task_event_buffer_->Enabled() && + !RayConfig::instance().task_events_skip_driver_for_test()) { + auto task_event = std::make_unique( + core_worker_->worker_context_->GetCurrentTaskID(), + core_worker_->worker_context_->GetCurrentJobID(), + /* attempt_number */ 0, + rpc::TaskStatus::FINISHED, + /* timestamp */ absl::GetCurrentTimeNanos(), + /*is_actor_task_event=*/ + core_worker_->worker_context_->GetCurrentActorID().IsNil(), + core_worker_->options_.session_name); + core_worker_->task_event_buffer_->AddTaskEvent(std::move(task_event)); + } + + opencensus::stats::StatsExporter::ExportNow(); + if (core_worker_->connected_) { + RAY_LOG(INFO) << "Sending disconnect message to the local raylet."; + core_worker_->connected_ = false; + if (core_worker_->raylet_ipc_client_) { + rpc::WorkerExitType worker_exit_type = rpc::WorkerExitType::INTENDED_USER_EXIT; + if (exit_type == "INTENDED_SYSTEM_EXIT") { + worker_exit_type = rpc::WorkerExitType::INTENDED_SYSTEM_EXIT; + } else if (exit_type == "USER_ERROR") { + worker_exit_type = rpc::WorkerExitType::USER_ERROR; + } else if (exit_type == "SYSTEM_ERROR") { + worker_exit_type = rpc::WorkerExitType::SYSTEM_ERROR; + } else if (exit_type == "NODE_OUT_OF_MEMORY") { + worker_exit_type = rpc::WorkerExitType::NODE_OUT_OF_MEMORY; + } + + Status status = core_worker_->raylet_ipc_client_->Disconnect( + worker_exit_type, std::string(detail), creation_task_exception_pb_bytes); + if (status.ok()) { + RAY_LOG(INFO) << "Disconnected from the local raylet."; + } else { + RAY_LOG(WARNING) << "Failed to disconnect from the local raylet: " << status; + } + } + } +} + +void CoreWorkerShutdownExecutor::QuickExit() { + RAY_LOG(WARNING) << "Quick exit - terminating process immediately"; + RAY_LOG(WARNING) << "Quick exit - calling std::quick_exit(1)"; + std::quick_exit(1); + RAY_LOG(WARNING) << "Quick exit - this line should never be reached"; +} +} // namespace core +} // namespace ray diff --git a/src/ray/core_worker/core_worker_shutdown_executor.h b/src/ray/core_worker/core_worker_shutdown_executor.h new file mode 100644 index 000000000000..9617f0c1564f --- /dev/null +++ b/src/ray/core_worker/core_worker_shutdown_executor.h @@ -0,0 +1,104 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "ray/core_worker/shutdown_coordinator.h" + +namespace ray { + +namespace core { + +class CoreWorker; + +/// Concrete implementation of `ShutdownExecutorInterface` that executes actual +/// shutdown operations for `CoreWorker`. +/// +/// Semantics overview: +/// - Graceful shutdown (ExecuteGracefulShutdown): stop accepting new work, drain ongoing +/// work, flush task +/// events, stop services (task execution service, gRPC server, IO service), +/// disconnect from the GCS/raylet, and join the IO thread if safe. This path +/// attempts best-effort cleanup to preserve observability and avoid resource +/// leaks. It may take up to `timeout_ms` for certain steps. +/// - Force shutdown (ExecuteForceShutdown): immediately kill child processes, disconnect +/// services, and +/// terminate the process without draining or cleanup. This path is used to +/// break out of hung or long-running shutdowns and should be considered +/// preemptive; it sacrifices cleanup for determinism. +/// - Worker exit (ExecuteWorkerExit): worker-type-specific graceful +/// shutdown that handles task draining and optional actor creation failure +/// payloads, then proceeds with the graceful sequence. +/// - Handle exit (ExecuteHandleExit): conditional exit that first checks worker +/// idleness and only proceeds when idle; otherwise it is ignored. +class CoreWorkerShutdownExecutor : public ShutdownExecutorInterface { + public: + /// Constructor with CoreWorker reference for accessing internals + /// \param core_worker Reference to the CoreWorker instance + explicit CoreWorkerShutdownExecutor(CoreWorker *core_worker); + + ~CoreWorkerShutdownExecutor() override = default; + + /// Execute graceful shutdown sequence. + /// Stops task execution, flushes task events, stops IO/gRPC services, joins IO + /// thread when not self, and disconnects from GCS. Best-effort cleanup. + void ExecuteGracefulShutdown(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) override; + + /// Execute force shutdown sequence. + /// Kills child processes, disconnects services, and terminates the process. + /// Skips draining/cleanup for fast, deterministic termination. + void ExecuteForceShutdown(std::string_view exit_type, std::string_view detail) override; + + /// Execute worker exit sequence with task draining. + /// Drains tasks/references as applicable for worker mode, then performs + /// graceful shutdown. + void ExecuteWorkerExit(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) override; + + void ExecuteExit(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms, + const std::shared_ptr + &creation_task_exception_pb_bytes) override; + + /// Execute handle exit sequence with idle checking. + /// Only performs worker exit if the worker is currently idle; otherwise, it + /// logs and returns without action. + void ExecuteHandleExit(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) override; + + void KillChildProcessesImmediately() override; + + bool ShouldWorkerIdleExit() const override; + + private: + /// Reference to CoreWorker for accessing shutdown operations + CoreWorker *core_worker_; + + void DisconnectServices( + std::string_view exit_type, + std::string_view detail, + const std::shared_ptr &creation_task_exception_pb_bytes); + void QuickExit(); +}; +} // namespace core +} // namespace ray diff --git a/src/ray/core_worker/shutdown_coordinator.cc b/src/ray/core_worker/shutdown_coordinator.cc new file mode 100644 index 000000000000..75e13a2fbef9 --- /dev/null +++ b/src/ray/core_worker/shutdown_coordinator.cc @@ -0,0 +1,293 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/core_worker/shutdown_coordinator.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "ray/common/buffer.h" // LocalMemoryBuffer +#include "ray/core_worker/common.h" // for WorkerType alias +namespace ray { + +namespace core { + +ShutdownCoordinator::ShutdownCoordinator( + std::unique_ptr executor, WorkerType worker_type) + : executor_(std::move(executor)), worker_type_(worker_type) { + RAY_CHECK(executor_) + << "ShutdownCoordinator requires a non-null ShutdownExecutorInterface. " + << "This indicates a construction-time bug. " + << "Pass a concrete executor (e.g., CoreWorkerShutdownExecutor) " + << "when creating the coordinator."; +} + +bool ShutdownCoordinator::RequestShutdown( + bool force_shutdown, + ShutdownReason reason, + std::string_view detail, + std::chrono::milliseconds timeout_ms, + const std::shared_ptr &creation_task_exception_pb_bytes) { + bool should_execute = false; + bool execute_force = force_shutdown; + { + std::lock_guard lock(mu_); + if (state_ == ShutdownState::kShutdown) { + return false; + } + // If a force request arrives, latch it immediately to guarantee single execution. + if (force_shutdown) { + if (force_started_) { + return false; + } + force_started_ = true; + reason_ = reason; + shutdown_detail_ = std::string(detail); + if (state_ == ShutdownState::kRunning) { + state_ = ShutdownState::kShuttingDown; + } + should_execute = true; + } else { + if (state_ != ShutdownState::kRunning) { + return false; + } + state_ = ShutdownState::kShuttingDown; + reason_ = reason; + shutdown_detail_ = std::string(detail); + should_execute = true; + } + } + + if (!should_execute) { + return false; + } + + ExecuteShutdownSequence( + execute_force, detail, timeout_ms, creation_task_exception_pb_bytes); + return true; +} + +bool ShutdownCoordinator::TryInitiateShutdown(ShutdownReason reason) { + // Legacy compatibility - delegate to graceful shutdown by default + return RequestShutdown(false, reason, "", kInfiniteTimeout, nullptr); +} + +bool ShutdownCoordinator::TryTransitionToDisconnecting() { + std::lock_guard lock(mu_); + if (state_ != ShutdownState::kShuttingDown) { + return false; + } + state_ = ShutdownState::kDisconnecting; + return true; +} + +bool ShutdownCoordinator::TryTransitionToShutdown() { + std::lock_guard lock(mu_); + if (state_ != ShutdownState::kShuttingDown && state_ != ShutdownState::kDisconnecting) { + return false; + } + state_ = ShutdownState::kShutdown; + return true; +} + +ShutdownState ShutdownCoordinator::GetState() const { + std::lock_guard lock(mu_); + return state_; +} + +ShutdownReason ShutdownCoordinator::GetReason() const { + std::lock_guard lock(mu_); + return reason_; +} + +bool ShutdownCoordinator::ShouldEarlyExit() const { + std::lock_guard lock(mu_); + return state_ != ShutdownState::kRunning; +} + +bool ShutdownCoordinator::IsRunning() const { + return GetState() == ShutdownState::kRunning; +} + +bool ShutdownCoordinator::IsShuttingDown() const { + return GetState() != ShutdownState::kRunning; +} + +bool ShutdownCoordinator::IsShutdown() const { + return GetState() == ShutdownState::kShutdown; +} + +std::string ShutdownCoordinator::GetStateString() const { + switch (GetState()) { + case ShutdownState::kRunning: + return "Running"; + case ShutdownState::kShuttingDown: + return "ShuttingDown"; + case ShutdownState::kDisconnecting: + return "Disconnecting"; + case ShutdownState::kShutdown: + return "Shutdown"; + default: + return "Unknown"; + } +} + +// Methods that execute shutdown logic + +void ShutdownCoordinator::ExecuteShutdownSequence( + bool force_shutdown, + std::string_view detail, + std::chrono::milliseconds timeout_ms, + const std::shared_ptr &creation_task_exception_pb_bytes) { + switch (worker_type_) { + case WorkerType::DRIVER: + ExecuteDriverShutdown(force_shutdown, detail, timeout_ms); + break; + case WorkerType::WORKER: + case WorkerType::SPILL_WORKER: + case WorkerType::RESTORE_WORKER: + ExecuteWorkerShutdown( + force_shutdown, detail, timeout_ms, creation_task_exception_pb_bytes); + break; + default: + RAY_LOG(FATAL) << "Unknown worker type: " << static_cast(worker_type_) + << ". This should be unreachable. Please file a bug at " + << "https://github.com/ray-project/ray/issues."; + break; + } +} + +void ShutdownCoordinator::ExecuteGracefulShutdown(std::string_view detail, + std::chrono::milliseconds timeout_ms) { + TryTransitionToDisconnecting(); + executor_->ExecuteGracefulShutdown(GetExitTypeString(), detail, timeout_ms); + TryTransitionToShutdown(); +} + +void ShutdownCoordinator::ExecuteForceShutdown(std::string_view detail) { + // Force shutdown bypasses normal state transitions and terminates immediately + // This ensures that force shutdowns can interrupt hanging graceful shutdowns + { + std::lock_guard lock(mu_); + if (force_executed_) { + return; + } + force_executed_ = true; + } + executor_->ExecuteForceShutdown(GetExitTypeString(), detail); + + // Only update state if we're not already in final state + // (force shutdown should have terminated the process by now) + TryTransitionToShutdown(); +} + +void ShutdownCoordinator::ExecuteDriverShutdown(bool force_shutdown, + std::string_view detail, + std::chrono::milliseconds timeout_ms) { + if (force_shutdown) { + ExecuteForceShutdown(detail); + } else { + ExecuteGracefulShutdown(detail, timeout_ms); + } +} + +void ShutdownCoordinator::ExecuteWorkerShutdown( + bool force_shutdown, + std::string_view detail, + std::chrono::milliseconds timeout_ms, + const std::shared_ptr &creation_task_exception_pb_bytes) { + if (force_shutdown) { + ExecuteForceShutdown(detail); + return; + } + + ShutdownReason reason = GetReason(); + + if (reason == ShutdownReason::kActorCreationFailed) { + TryTransitionToDisconnecting(); + executor_->ExecuteExit( + GetExitTypeString(), detail, timeout_ms, creation_task_exception_pb_bytes); + } else if (reason == ShutdownReason::kUserError || + reason == ShutdownReason::kGracefulExit || + reason == ShutdownReason::kIntentionalShutdown || + reason == ShutdownReason::kUnexpectedError || + reason == ShutdownReason::kOutOfMemory || + reason == ShutdownReason::kActorKilled) { + TryTransitionToDisconnecting(); + executor_->ExecuteWorkerExit(GetExitTypeString(), detail, timeout_ms); + } else if (reason == ShutdownReason::kIdleTimeout || + reason == ShutdownReason::kJobFinished) { + TryTransitionToDisconnecting(); + executor_->ExecuteHandleExit(GetExitTypeString(), detail, timeout_ms); + } else { + ExecuteGracefulShutdown(detail, timeout_ms); + } +} + +std::string ShutdownCoordinator::GetExitTypeString() const { + switch (GetReason()) { + case ShutdownReason::kIdleTimeout: + case ShutdownReason::kIntentionalShutdown: + return "INTENDED_SYSTEM_EXIT"; + case ShutdownReason::kUserError: + return "USER_ERROR"; + case ShutdownReason::kActorCreationFailed: + return "USER_ERROR"; + case ShutdownReason::kUnexpectedError: + return "SYSTEM_ERROR"; + case ShutdownReason::kOutOfMemory: + return "NODE_OUT_OF_MEMORY"; + case ShutdownReason::kForcedExit: + case ShutdownReason::kGracefulExit: + default: + return "INTENDED_USER_EXIT"; + } +} + +std::string ShutdownCoordinator::GetReasonString() const { + switch (GetReason()) { + case ShutdownReason::kNone: + return "None"; + case ShutdownReason::kIntentionalShutdown: + return "IntentionalShutdown"; + case ShutdownReason::kUnexpectedError: + return "UnexpectedError"; + case ShutdownReason::kIdleTimeout: + return "IdleTimeout"; + case ShutdownReason::kGracefulExit: + return "GracefulExit"; + case ShutdownReason::kForcedExit: + return "ForcedExit"; + case ShutdownReason::kUserError: + return "UserError"; + case ShutdownReason::kOutOfMemory: + return "OutOfMemory"; + case ShutdownReason::kJobFinished: + return "JobFinished"; + case ShutdownReason::kActorKilled: + return "ActorKilled"; + case ShutdownReason::kActorCreationFailed: + return "ActorCreationFailed"; + default: + return "Unknown"; + } +} + +} // namespace core +} // namespace ray diff --git a/src/ray/core_worker/shutdown_coordinator.h b/src/ray/core_worker/shutdown_coordinator.h new file mode 100644 index 000000000000..b2a68ec4af39 --- /dev/null +++ b/src/ray/core_worker/shutdown_coordinator.h @@ -0,0 +1,294 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +// Bring in WorkerType alias and common types +#include "ray/core_worker/common.h" + +namespace ray { +class LocalMemoryBuffer; +} // namespace ray + +namespace ray { + +namespace core { + +/// Interface for executing shutdown operations that the coordinator invokes. +class ShutdownExecutorInterface { + public: + virtual ~ShutdownExecutorInterface() = default; + + virtual void ExecuteGracefulShutdown(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) = 0; + + virtual void ExecuteForceShutdown(std::string_view exit_type, + std::string_view detail) = 0; + + virtual void ExecuteWorkerExit(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) = 0; + + virtual void ExecuteExit(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms, + const std::shared_ptr<::ray::LocalMemoryBuffer> + &creation_task_exception_pb_bytes) = 0; + + virtual void ExecuteHandleExit(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) = 0; + + // Best-effort cleanup of child processes spawned by this worker process to + // avoid leaked subprocesses holding expensive resources (e.g., CUDA contexts). + // + // - Intended to be called during shutdown (including force paths). + // - Only targets direct children of the current process; crash paths can still leak + // (subreaper not yet used). + // - No-ops when disabled by configuration + // (RayConfig::kill_child_processes_on_worker_exit()). + // - Platform-dependent: process enumeration may be unavailable on some OSes. + virtual void KillChildProcessesImmediately() = 0; + + virtual bool ShouldWorkerIdleExit() const = 0; +}; + +/// Reasons for worker shutdown. Used for observability and debugging. +enum class ShutdownReason : std::uint8_t { + kNone = 0, + kIntentionalShutdown = 1, + kUnexpectedError = 2, + kIdleTimeout = 3, + kGracefulExit = 4, + kForcedExit = 5, + kUserError = 6, + kOutOfMemory = 7, + kJobFinished = 8, + kActorKilled = 9, + kActorCreationFailed = 10 +}; + +/// Shutdown state representing the current lifecycle phase of worker shutdown. +/// The state machine supports two paths with only forward transitions: +/// +/// Normal shutdown: kRunning -> kShuttingDown -> kDisconnecting -> kShutdown +/// Force shutdown: kRunning -> kShuttingDown -> kShutdown (bypasses kDisconnecting) +/// +/// State semantics: +/// - kRunning: Normal operation, accepting new work +/// - kShuttingDown: Shutdown initiated, draining existing work, no new work accepted +/// - kDisconnecting: Disconnecting from services (raylet, GCS), cleanup phase +/// - kShutdown: Final state, all cleanup complete, ready for process termination +enum class ShutdownState : std::uint8_t { + kRunning = 0, + kShuttingDown = 1, + kDisconnecting = 2, + kShutdown = 3 +}; + +/// Thread-safe coordinator for managing worker shutdown state and transitions. +/// +/// Uses a single mutex to serialize state transitions and to capture the shutdown +/// reason exactly once. We favor simple, readable synchronization because shutdown is +/// control-path, not throughput-critical. +/// +/// Key features: +/// - Atomic state transitions with integrated reason tracking +/// - Idempotent shutdown operations +/// - Performance optimized for hot-path checks +/// - Thread-safe from any thread context +/// +/// Usage: +/// auto coordinator = std::make_unique(); +/// +/// // Try to initiate shutdown (only the first caller succeeds) +/// if (coordinator->TryInitiateShutdown(ShutdownReason::kGracefulExit)) { +/// // This thread should execute shutdown sequence +/// } +/// +/// // Fast check for early exit in performance-critical paths +/// if (coordinator->ShouldEarlyExit()) { +/// return Status::Invalid("Worker is shutting down"); +/// } +class ShutdownCoordinator { + public: + static constexpr std::chrono::milliseconds kInfiniteTimeout{-1}; + /// Constructor + /// + /// \param executor Shutdown executor implementation + /// \param worker_type Type of worker for shutdown behavior customization + explicit ShutdownCoordinator(std::unique_ptr executor, + WorkerType worker_type = WorkerType::WORKER); + + ~ShutdownCoordinator() = default; + + // Non-copyable and non-movable for safety + ShutdownCoordinator(const ShutdownCoordinator &) = delete; + ShutdownCoordinator &operator=(const ShutdownCoordinator &) = delete; + ShutdownCoordinator(ShutdownCoordinator &&) = delete; + ShutdownCoordinator &operator=(ShutdownCoordinator &&) = delete; + + /// Request shutdown with configurable timeout and fallback behavior. + /// + /// Single entry-point that captures the first shutdown reason, chooses the + /// worker-type-specific path, and optionally falls back to force. Additional + /// graceful requests are ignored; a concurrent force may override the reason + /// and proceed. + /// + /// \param force_shutdown If true, force immediate shutdown; if false, graceful shutdown + /// \param reason The reason for shutdown initiation + /// \param detail Optional detailed explanation + /// \param timeout_ms Timeout for graceful shutdown (-1 = no timeout) + /// \return true if this call initiated shutdown, false if already shutting down + bool RequestShutdown(bool force_shutdown, + ShutdownReason reason, + std::string_view detail = "", + std::chrono::milliseconds timeout_ms = kInfiniteTimeout, + const std::shared_ptr<::ray::LocalMemoryBuffer> + &creation_task_exception_pb_bytes = nullptr); + + /// Legacy method for compatibility - delegates to RequestShutdown + /// TODO (codope): This is public for now to ease incremental migration and testing. + /// Consider removing or making private once all call sites are wired to + /// RequestShutdown directly. + /// \param reason The reason for shutdown initiation + /// \return true if this call initiated shutdown, false if already shutting down + bool TryInitiateShutdown(ShutdownReason reason); + + /// Attempt to transition to disconnecting state. + /// + /// Begins the disconnection/cleanup phase (e.g., GCS/raylet disconnect). Only + /// valid from kShuttingDown. + /// + /// \return true if transition succeeded, false if invalid state + /// TODO (codope): Public-for-now to support targeted tests; make private when tests + /// drive behavior exclusively via RequestShutdown. + /// TODO (codope): Once private, we can consider removing the internal mutex acquisition + /// here and in TryTransitionToShutdown(), since RequestShutdown serializes the + /// execution path and only a single thread invokes transitions. + bool TryTransitionToDisconnecting(); + + /// Attempt to transition to final shutdown state. + /// + /// Finalizes shutdown. Allowed from kDisconnecting (normal) or kShuttingDown + /// (force path). + /// + /// \return true if transition succeeded, false if invalid state + /// TODO (codope): Public-for-now to support targeted tests; make private when tests + /// drive behavior exclusively via RequestShutdown. + /// TODO (codope): Once private, we can consider removing the internal mutex acquisition + /// here and in TryTransitionToDisconnecting(), since RequestShutdown serializes the + /// execution path and only a single thread invokes transitions. + bool TryTransitionToShutdown(); + + /// Get the current shutdown state (mutex-protected, fast path safe). + /// + /// \return Current shutdown state + ShutdownState GetState() const; + + /// Get the shutdown reason. + /// + /// The reason is set when shutdown is first initiated and remains + /// constant throughout the shutdown process. + /// + /// \return Shutdown reason (kNone if not shutting down) + ShutdownReason GetReason() const; + + /// Check if worker should early-exit from operations. + /// + /// Recommended hot-path check; returns true for any non-running state. + /// + /// \return true if operations should be aborted, false if normal operation + bool ShouldEarlyExit() const; + + /// Check if worker is in running state. + /// + /// \return true if in kRunning state, false otherwise + bool IsRunning() const; + + /// Check if shutdown has been initiated. + /// + /// \return true if in any shutdown state, false if still running + bool IsShuttingDown() const; + + /// Check if worker has completed shutdown. + /// + /// \return true if in kShutdown state, false otherwise + bool IsShutdown() const; + + /// Get string representation of current state. + /// + /// \return Human-readable state description + std::string GetStateString() const; + + /// Get string representation of exit type based on shutdown reason. + std::string GetExitTypeString() const; + + /// Get string representation of shutdown reason. + /// + /// \return Human-readable reason description + std::string GetReasonString() const; + + private: + /// Execute shutdown sequence based on worker type and mode + void ExecuteShutdownSequence( + bool force_shutdown, + std::string_view detail, + std::chrono::milliseconds timeout_ms, + const std::shared_ptr<::ray::LocalMemoryBuffer> &creation_task_exception_pb_bytes); + + /// Executes graceful path; transitions to Disconnecting/Shutdown + void ExecuteGracefulShutdown(std::string_view detail, + std::chrono::milliseconds timeout_ms); + + /// Executes force path; guarded to run at most once + void ExecuteForceShutdown(std::string_view detail); + + void ExecuteDriverShutdown(bool force_shutdown, + std::string_view detail, + std::chrono::milliseconds timeout_ms); + /// Worker-type specific shutdown behavior + /// - Honors kActorCreationFailed with serialized exception payloads + /// - Uses worker-idle checks for idle exits + /// - Drains tasks/references before disconnect + void ExecuteWorkerShutdown( + bool force_shutdown, + std::string_view detail, + std::chrono::milliseconds timeout_ms, + const std::shared_ptr<::ray::LocalMemoryBuffer> &creation_task_exception_pb_bytes); + + // Executor and configuration + std::unique_ptr executor_; + WorkerType worker_type_; + + // Mutex-guarded shutdown state + mutable std::mutex mu_; + ShutdownState state_ = ShutdownState::kRunning; + ShutdownReason reason_ = ShutdownReason::kNone; + bool force_executed_ = false; + bool force_started_ = false; + + /// Shutdown detail for observability (set once during shutdown initiation) + std::string shutdown_detail_; +}; +} // namespace core +} // namespace ray diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index 8f195aeb3944..e201590fdd06 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -11,6 +11,18 @@ ray_cc_test( ], ) +ray_cc_test( + name = "shutdown_coordinator_test", + size = "medium", + srcs = ["shutdown_coordinator_test.cc"], + tags = ["team:core"], + deps = [ + "//src/ray/core_worker:core_worker_lib", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + ray_cc_test( name = "memory_store_test", size = "small", diff --git a/src/ray/core_worker/tests/shutdown_coordinator_test.cc b/src/ray/core_worker/tests/shutdown_coordinator_test.cc new file mode 100644 index 000000000000..6f0250ad3b8f --- /dev/null +++ b/src/ray/core_worker/tests/shutdown_coordinator_test.cc @@ -0,0 +1,406 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/core_worker/shutdown_coordinator.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "ray/common/buffer.h" + +namespace ray { +namespace core { + +// Simple fake executor for tests without gmock. +class FakeShutdownExecutor : public ShutdownExecutorInterface { + public: + std::atomic graceful_calls{0}; + std::atomic force_calls{0}; + std::atomic worker_exit_calls{0}; + std::atomic handle_exit_calls{0}; + std::atomic idle_exit_allowed{false}; + + std::string last_exit_type; + std::string last_detail; + + void ExecuteGracefulShutdown(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) override { + graceful_calls++; + last_exit_type = std::string(exit_type); + last_detail = std::string(detail); + } + void ExecuteForceShutdown(std::string_view exit_type, + std::string_view detail) override { + force_calls++; + last_exit_type = std::string(exit_type); + last_detail = std::string(detail); + } + void ExecuteWorkerExit(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) override { + worker_exit_calls++; + last_exit_type = std::string(exit_type); + last_detail = std::string(detail); + } + void ExecuteExit(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms, + const std::shared_ptr<::ray::LocalMemoryBuffer> + &creation_task_exception_pb_bytes) override { + worker_exit_calls++; + last_exit_type = std::string(exit_type); + last_detail = std::string(detail); + } + void ExecuteHandleExit(std::string_view exit_type, + std::string_view detail, + std::chrono::milliseconds timeout_ms) override { + handle_exit_calls++; + last_exit_type = std::string(exit_type); + last_detail = std::string(detail); + } + void KillChildProcessesImmediately() override {} + bool ShouldWorkerIdleExit() const override { return idle_exit_allowed.load(); } +}; + +// No-op executor used in disabled/manual-transition tests. +class NoOpShutdownExecutor : public ShutdownExecutorInterface { + public: + void ExecuteGracefulShutdown(std::string_view, + std::string_view, + std::chrono::milliseconds) override {} + void ExecuteForceShutdown(std::string_view, std::string_view) override {} + void ExecuteWorkerExit(std::string_view, + std::string_view, + std::chrono::milliseconds) override {} + void ExecuteExit(std::string_view, + std::string_view, + std::chrono::milliseconds, + const std::shared_ptr<::ray::LocalMemoryBuffer> &) override {} + void ExecuteHandleExit(std::string_view, + std::string_view, + std::chrono::milliseconds) override {} + void KillChildProcessesImmediately() override {} + bool ShouldWorkerIdleExit() const override { return false; } +}; + +class ShutdownCoordinatorTest : public ::testing::Test { + protected: + // Helper to create coordinator with specific worker type + std::unique_ptr CreateCoordinator( + WorkerType worker_type = WorkerType::WORKER) { + auto fake = std::make_unique(); + return std::make_unique(std::move(fake), worker_type); + } +}; + +TEST_F(ShutdownCoordinatorTest, InitialStateWithNoTransitions_IsRunning) { + auto coordinator = CreateCoordinator(); + + EXPECT_EQ(coordinator->GetState(), ShutdownState::kRunning); + EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kNone); + EXPECT_TRUE(coordinator->IsRunning()); + EXPECT_FALSE(coordinator->IsShuttingDown()); + EXPECT_FALSE(coordinator->IsShutdown()); + EXPECT_FALSE(coordinator->ShouldEarlyExit()); +} + +TEST_F(ShutdownCoordinatorTest, RequestShutdown_IdempotentBehavior) { + auto coordinator = CreateCoordinator(); + + // First graceful request should succeed + EXPECT_TRUE(coordinator->RequestShutdown( + false, ShutdownReason::kGracefulExit, "test_graceful")); + const auto state = coordinator->GetState(); + EXPECT_TRUE(state == ShutdownState::kDisconnecting || + state == ShutdownState::kShutdown); + EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kGracefulExit); + + // A second graceful request should be ignored + EXPECT_FALSE( + coordinator->RequestShutdown(false, ShutdownReason::kUserError, "test_graceful2")); + EXPECT_EQ(coordinator->GetReason(), + ShutdownReason::kGracefulExit); // Reason is unchanged + + // A force-kill request should succeed and override the graceful one + EXPECT_TRUE( + coordinator->RequestShutdown(true, ShutdownReason::kForcedExit, "test_force")); + EXPECT_EQ(coordinator->GetState(), ShutdownState::kShutdown); + EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kForcedExit); // Reason is updated +} + +TEST_F(ShutdownCoordinatorTest, + TryInitiateShutdown_DelegatesToGraceful_OnlyFirstSucceeds) { + auto coordinator = CreateCoordinator(); + + EXPECT_TRUE(coordinator->TryInitiateShutdown(ShutdownReason::kUserError)); + const auto state = coordinator->GetState(); + EXPECT_TRUE(state == ShutdownState::kShuttingDown || + state == ShutdownState::kDisconnecting); + EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kUserError); + + // Second call should fail + EXPECT_FALSE(coordinator->TryInitiateShutdown(ShutdownReason::kForcedExit)); + EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kUserError); // unchanged +} + +TEST_F(ShutdownCoordinatorTest, + RequestShutdown_Graceful_SetsDisconnecting_ThenTryTransitionToShutdown_Succeeds) { + auto coordinator = std::make_unique( + std::make_unique(), WorkerType::WORKER); + + // Running -> ShuttingDown -> Disconnecting + EXPECT_TRUE( + coordinator->RequestShutdown(false /*graceful*/, ShutdownReason::kGracefulExit)); + + // worker path enters Disconnecting and requires explicit final step. + EXPECT_EQ(coordinator->GetState(), ShutdownState::kDisconnecting); + EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kGracefulExit); + + // Disconnecting -> Shutdown + EXPECT_TRUE(coordinator->TryTransitionToShutdown()); + EXPECT_EQ(coordinator->GetState(), ShutdownState::kShutdown); + + // Further transitions are no-ops. + EXPECT_FALSE(coordinator->TryTransitionToDisconnecting()); + EXPECT_FALSE(coordinator->TryTransitionToShutdown()); +} + +TEST_F(ShutdownCoordinatorTest, InvalidTransitions_FromRunning_Fail) { + auto coordinator = CreateCoordinator(); + + // Cannot transition to disconnecting from running + EXPECT_FALSE(coordinator->TryTransitionToDisconnecting()); + EXPECT_EQ(coordinator->GetState(), ShutdownState::kRunning); + + // Cannot transition to shutdown from running + EXPECT_FALSE(coordinator->TryTransitionToShutdown()); + EXPECT_EQ(coordinator->GetState(), ShutdownState::kRunning); +} + +TEST_F(ShutdownCoordinatorTest, ForceShutdown_TransitionsDirectlyToShutdown) { + auto coordinator = CreateCoordinator(); + + // Running -> Shutdown (completes immediately with mocked dependencies) + EXPECT_TRUE(coordinator->RequestShutdown(true, // force + ShutdownReason::kForcedExit)); + + // Already in shutdown state, manual transition should fail + EXPECT_FALSE(coordinator->TryTransitionToShutdown()); + EXPECT_EQ(coordinator->GetState(), ShutdownState::kShutdown); +} + +TEST_F(ShutdownCoordinatorTest, + RequestShutdown_Graceful_OnlyOneInitiatorUnderConcurrency) { + auto coordinator = CreateCoordinator(); + + constexpr int num_threads = 10; + std::atomic success_count{0}; + std::vector threads; + + // Launch multiple threads trying to initiate shutdown + for (int i = 0; i < num_threads; ++i) { + threads.emplace_back([&coordinator, &success_count, i]() { + if (coordinator->RequestShutdown(false, // graceful + ShutdownReason::kGracefulExit, + "thread_" + std::to_string(i))) { + success_count.fetch_add(1); + } + }); + } + + // Wait for all threads + for (auto &thread : threads) { + thread.join(); + } + + // Only one thread should have succeeded + EXPECT_EQ(success_count.load(), 1); + const auto state = coordinator->GetState(); + EXPECT_TRUE(state == ShutdownState::kShuttingDown || + state == ShutdownState::kDisconnecting); + EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kGracefulExit); +} + +TEST_F(ShutdownCoordinatorTest, Driver_GracefulReasonRecorded) { + auto coordinator = CreateCoordinator(WorkerType::DRIVER); + + EXPECT_TRUE(coordinator->RequestShutdown(false, // graceful + ShutdownReason::kGracefulExit)); + + EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kGracefulExit); +} + +TEST_F(ShutdownCoordinatorTest, Driver_ForceReasonRecorded) { + auto coordinator = CreateCoordinator(WorkerType::DRIVER); + + EXPECT_TRUE(coordinator->RequestShutdown(true, // force + ShutdownReason::kForcedExit)); + + EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kForcedExit); +} + +TEST_F(ShutdownCoordinatorTest, Worker_GracefulInitiates) { + auto coordinator = CreateCoordinator(WorkerType::WORKER); + + EXPECT_TRUE(coordinator->RequestShutdown(false, // graceful + ShutdownReason::kGracefulExit)); +} + +TEST_F(ShutdownCoordinatorTest, Worker_ExecuteWorkerExit_OnUserError) { + auto coordinator = CreateCoordinator(WorkerType::WORKER); + + EXPECT_TRUE(coordinator->RequestShutdown(false, // graceful + ShutdownReason::kUserError)); +} + +TEST_F(ShutdownCoordinatorTest, Worker_HandleExit_OnIdleTimeout) { + auto coordinator = CreateCoordinator(WorkerType::WORKER); + + EXPECT_TRUE(coordinator->RequestShutdown(false, // graceful + ShutdownReason::kIdleTimeout)); +} + +TEST_F(ShutdownCoordinatorTest, ShouldEarlyExit_Performance_IsFast) { + auto coordinator = CreateCoordinator(); + auto start = std::chrono::high_resolution_clock::now(); + constexpr int iterations = 1000000; + volatile bool result = false; + + for (int i = 0; i < iterations; ++i) { + result = coordinator->ShouldEarlyExit(); + } + + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start); + + // Should be very fast (less than 100ns per call on modern hardware) + double ns_per_call = static_cast(duration.count()) / iterations; + EXPECT_LT(ns_per_call, 100.0) + << "ShouldEarlyExit too slow: " << ns_per_call << "ns per call"; + + // Prevent unused variable warning + (void)result; +} + +TEST_F(ShutdownCoordinatorTest, StringRepresentations_StateAndReason_AreReadable) { + auto coordinator = CreateCoordinator(); + + EXPECT_EQ(coordinator->GetStateString(), "Running"); + EXPECT_EQ(coordinator->GetReasonString(), "None"); + + coordinator->RequestShutdown(false, ShutdownReason::kGracefulExit); // graceful + + EXPECT_EQ(coordinator->GetStateString(), "Disconnecting"); + EXPECT_EQ(coordinator->GetReasonString(), "GracefulExit"); + + coordinator->TryTransitionToShutdown(); + EXPECT_EQ(coordinator->GetStateString(), "Shutdown"); +} + +TEST_F(ShutdownCoordinatorTest, ExitTypeStringMapping_UserError_IsUSER_ERROR) { + auto coordinator = CreateCoordinator(); + coordinator->RequestShutdown(false, ShutdownReason::kUserError); + EXPECT_EQ(coordinator->GetExitTypeString(), "USER_ERROR"); +} + +TEST_F(ShutdownCoordinatorTest, ExitTypeStringMapping_OOM_IsNODE_OUT_OF_MEMORY) { + auto coordinator = CreateCoordinator(); + coordinator->RequestShutdown(false, ShutdownReason::kOutOfMemory); + EXPECT_EQ(coordinator->GetExitTypeString(), "NODE_OUT_OF_MEMORY"); +} + +TEST_F(ShutdownCoordinatorTest, + ExitTypeStringMapping_IdleTimeout_IsINTENDED_SYSTEM_EXIT) { + auto coordinator = CreateCoordinator(); + coordinator->RequestShutdown(false, ShutdownReason::kIdleTimeout); + EXPECT_EQ(coordinator->GetExitTypeString(), "INTENDED_SYSTEM_EXIT"); +} + +TEST_F(ShutdownCoordinatorTest, ShouldEarlyExit_MemoryOrdering_ConcurrentVisibility) { + auto coordinator = CreateCoordinator(); + + std::atomic thread1_saw_shutdown{false}; + std::atomic thread2_saw_shutdown{false}; + + std::thread thread1([&coordinator, &thread1_saw_shutdown]() { + coordinator->RequestShutdown(false, ShutdownReason::kGracefulExit); // graceful + thread1_saw_shutdown.store(true); + }); + + std::thread thread2([&coordinator, &thread2_saw_shutdown]() { + while (!coordinator->ShouldEarlyExit()) { + std::this_thread::yield(); + } + thread2_saw_shutdown.store(true); + }); + + thread1.join(); + thread2.join(); + + // Both threads should have seen the shutdown state + EXPECT_TRUE(thread1_saw_shutdown.load()); + EXPECT_TRUE(thread2_saw_shutdown.load()); + EXPECT_TRUE(coordinator->ShouldEarlyExit()); +} + +TEST_F(ShutdownCoordinatorTest, Concurrent_GracefulVsForce_ForceExecutesOnce) { + auto fake = std::make_unique(); + auto *fake_ptr = fake.get(); + auto coordinator = + std::make_unique(std::move(fake), WorkerType::WORKER); + + std::thread t1([&] { + coordinator->RequestShutdown(false, ShutdownReason::kGracefulExit, "graceful"); + }); + std::thread t2( + [&] { coordinator->RequestShutdown(true, ShutdownReason::kForcedExit, "force"); }); + t1.join(); + t2.join(); + + EXPECT_EQ(coordinator->GetState(), ShutdownState::kShutdown); + EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kForcedExit); + EXPECT_EQ(fake_ptr->force_calls.load(), 1); + EXPECT_LE(fake_ptr->graceful_calls.load(), 1); +} + +TEST_F(ShutdownCoordinatorTest, Concurrent_DoubleForce_ForceExecutesOnce) { + auto fake = std::make_unique(); + auto *fake_ptr = fake.get(); + auto coordinator = + std::make_unique(std::move(fake), WorkerType::WORKER); + + std::thread t1( + [&] { coordinator->RequestShutdown(true, ShutdownReason::kForcedExit, "force1"); }); + std::thread t2( + [&] { coordinator->RequestShutdown(true, ShutdownReason::kForcedExit, "force2"); }); + t1.join(); + t2.join(); + + EXPECT_EQ(coordinator->GetState(), ShutdownState::kShutdown); + EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kForcedExit); + EXPECT_EQ(fake_ptr->force_calls.load(), 1); + EXPECT_EQ(fake_ptr->graceful_calls.load(), 0); + EXPECT_EQ(fake_ptr->last_detail, "force1"); +} + +} // namespace core +} // namespace ray From d678b14f58afff06335263874bec96aba9804d9f Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:08:03 -0700 Subject: [PATCH 204/634] [core] directory for public proto (#55761) Currently, all of our protobuf files are organized in a flat directory under `src/ray/protobuf`. This PR introduces a public subdirectory and moves some existing files into it. Future PRs will add a private subdirectory and relocate files accordingly. This structure provides a clear indication of the intended exposure level of each protobuf file. This change also affects how we compile these files. After compilation, they remain in a flat directory (`ray/core/generated`). We flatten all sub-imports using the existing sed mechanism. Test: - CI Signed-off-by: Cuong Nguyen --- BUILD.bazel | 1 + src/ray/protobuf/BUILD.bazel | 44 +++++-------------- src/ray/protobuf/events_base_event.proto | 4 +- src/ray/protobuf/public/BUILD.bazel | 34 ++++++++++++++ .../events_driver_job_definition_event.proto | 0 .../events_driver_job_execution_event.proto | 0 6 files changed, 47 insertions(+), 36 deletions(-) create mode 100644 src/ray/protobuf/public/BUILD.bazel rename src/ray/protobuf/{ => public}/events_driver_job_definition_event.proto (100%) rename src/ray/protobuf/{ => public}/events_driver_job_execution_event.proto (100%) diff --git a/BUILD.bazel b/BUILD.bazel index 96d476d599ca..5473ddce9e41 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -457,6 +457,7 @@ genrule( # NOTE(hchen): Protobuf doesn't allow specifying Python package name. So we use this `sed` # command to change the import path in the generated file. + sed -i -E 's/from src.ray.protobuf.public/from ./' "$${files[@]}" sed -i -E 's/from src.ray.protobuf/from ./' "$${files[@]}" # Help the generated serve files to have the correct module serve_files=($$(ls "$$tmpdir"/ray/serve/generated/*_pb2*.py)) diff --git a/src/ray/protobuf/BUILD.bazel b/src/ray/protobuf/BUILD.bazel index 6d896d64245d..01cc6a39f45e 100644 --- a/src/ray/protobuf/BUILD.bazel +++ b/src/ray/protobuf/BUILD.bazel @@ -8,7 +8,10 @@ package(default_visibility = ["//visibility:public"]) proto_library( name = "common_proto", srcs = ["common.proto"], - visibility = ["//java:__subpackages__"], + visibility = [ + ":__subpackages__", + "//java:__subpackages__", + ], deps = [ ":runtime_env_common_proto", ], @@ -54,7 +57,10 @@ cc_proto_library( proto_library( name = "runtime_env_common_proto", srcs = ["runtime_env_common.proto"], - visibility = ["//java:__subpackages__"], + visibility = [ + ":__subpackages__", + "//java:__subpackages__", + ], ) proto_library( @@ -487,51 +493,21 @@ proto_library( ], ) -proto_library( - name = "events_driver_job_definition_event_proto", - srcs = ["events_driver_job_definition_event.proto"], - deps = [ - ":common_proto", - ":runtime_env_common_proto", - "@com_google_protobuf//:timestamp_proto", - ], -) - -proto_library( - name = "events_driver_job_execution_event_proto", - srcs = ["events_driver_job_execution_event.proto"], - deps = [ - ":common_proto", - ":runtime_env_common_proto", - "@com_google_protobuf//:timestamp_proto", - ], -) - cc_proto_library( name = "events_task_profile_events_cc_proto", deps = [":events_task_profile_events_proto"], ) -cc_proto_library( - name = "events_driver_job_definition_event_cc_proto", - deps = [":events_driver_job_definition_event_proto"], -) - -cc_proto_library( - name = "events_driver_job_execution_event_cc_proto", - deps = [":events_driver_job_execution_event_proto"], -) - proto_library( name = "events_base_event_proto", srcs = ["events_base_event.proto"], deps = [ ":events_actor_task_definition_event_proto", - ":events_driver_job_definition_event_proto", - ":events_driver_job_execution_event_proto", ":events_task_definition_event_proto", ":events_task_execution_event_proto", ":events_task_profile_events_proto", + "//src/ray/protobuf/public:events_driver_job_definition_event_proto", + "//src/ray/protobuf/public:events_driver_job_execution_event_proto", "@com_google_protobuf//:timestamp_proto", ], ) diff --git a/src/ray/protobuf/events_base_event.proto b/src/ray/protobuf/events_base_event.proto index 3b5c9e913944..fab95867c473 100644 --- a/src/ray/protobuf/events_base_event.proto +++ b/src/ray/protobuf/events_base_event.proto @@ -21,8 +21,8 @@ import "src/ray/protobuf/events_actor_task_definition_event.proto"; import "src/ray/protobuf/events_task_definition_event.proto"; import "src/ray/protobuf/events_task_execution_event.proto"; import "src/ray/protobuf/events_task_profile_events.proto"; -import "src/ray/protobuf/events_driver_job_definition_event.proto"; -import "src/ray/protobuf/events_driver_job_execution_event.proto"; +import "src/ray/protobuf/public/events_driver_job_definition_event.proto"; +import "src/ray/protobuf/public/events_driver_job_execution_event.proto"; // This is the base message for all ray events. message RayEvent { diff --git a/src/ray/protobuf/public/BUILD.bazel b/src/ray/protobuf/public/BUILD.bazel new file mode 100644 index 000000000000..991c911f16f1 --- /dev/null +++ b/src/ray/protobuf/public/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_cc//cc:defs.bzl", "cc_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "events_driver_job_definition_event_proto", + srcs = ["events_driver_job_definition_event.proto"], + deps = [ + "//src/ray/protobuf:common_proto", + "//src/ray/protobuf:runtime_env_common_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + +cc_proto_library( + name = "events_driver_job_definition_event_cc_proto", + deps = [":events_driver_job_definition_event_proto"], +) + +proto_library( + name = "events_driver_job_execution_event_proto", + srcs = ["events_driver_job_execution_event.proto"], + deps = [ + "//src/ray/protobuf:common_proto", + "//src/ray/protobuf:runtime_env_common_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + +cc_proto_library( + name = "events_driver_job_execution_event_cc_proto", + deps = [":events_driver_job_execution_event_proto"], +) diff --git a/src/ray/protobuf/events_driver_job_definition_event.proto b/src/ray/protobuf/public/events_driver_job_definition_event.proto similarity index 100% rename from src/ray/protobuf/events_driver_job_definition_event.proto rename to src/ray/protobuf/public/events_driver_job_definition_event.proto diff --git a/src/ray/protobuf/events_driver_job_execution_event.proto b/src/ray/protobuf/public/events_driver_job_execution_event.proto similarity index 100% rename from src/ray/protobuf/events_driver_job_execution_event.proto rename to src/ray/protobuf/public/events_driver_job_execution_event.proto From 42058689e9f23f1d1823cdcfcd8800751f50c618 Mon Sep 17 00:00:00 2001 From: lkchen Date: Wed, 20 Aug 2025 14:10:36 -0700 Subject: [PATCH 205/634] [LLM] Bump to nixl==0.4.1 (#55671) Signed-off-by: Linkun --- docker/ray-llm/Dockerfile | 4 ++-- python/requirements/llm/llm-requirements.txt | 2 +- python/requirements_compiled_rayllm_py311_cpu.txt | 14 +++++++++----- .../requirements_compiled_rayllm_py311_cu121.txt | 14 +++++++++----- .../requirements_compiled_rayllm_py311_cu128.txt | 14 +++++++++----- ...requirements_compiled_rayllm_test_py311_cpu.txt | 14 +++++++++----- ...quirements_compiled_rayllm_test_py311_cu121.txt | 14 +++++++++----- ...quirements_compiled_rayllm_test_py311_cu128.txt | 14 +++++++++----- 8 files changed, 57 insertions(+), 33 deletions(-) diff --git a/docker/ray-llm/Dockerfile b/docker/ray-llm/Dockerfile index a1e4595724de..d7038f92b883 100644 --- a/docker/ray-llm/Dockerfile +++ b/docker/ray-llm/Dockerfile @@ -76,7 +76,7 @@ sudo apt-get install -y kmod pkg-config librdmacm-dev cmake --no-questions ) -UCX_VERSION="1.18.1" +UCX_VERSION="1.19.0" ( echo "Installing UCX ${UCX_VERSION}" cd "${TEMP_DIR}" @@ -109,7 +109,7 @@ UCX_VERSION="1.18.1" ) # Keep in sync with llm-requirements.txt -NIXL_VERSION="0.3.1" +NIXL_VERSION="0.4.1" ( echo "Installing NIXL ${NIXL_VERSION}" # NIXL needs meson pybind11 ninja, but should have been included in requirements_*.txt diff --git a/python/requirements/llm/llm-requirements.txt b/python/requirements/llm/llm-requirements.txt index 9f2815ec0951..d8eb58a1fd7d 100644 --- a/python/requirements/llm/llm-requirements.txt +++ b/python/requirements/llm/llm-requirements.txt @@ -16,4 +16,4 @@ hf_transfer transformers<4.54.0 # Due to https://github.com/vllm-project/vllm-ascend/issues/2046 # nixl version Needs to be in sync with the one in ray-llm/Dockerfile -nixl==0.3.1 +nixl==0.4.1 diff --git a/python/requirements_compiled_rayllm_py311_cpu.txt b/python/requirements_compiled_rayllm_py311_cpu.txt index 3c7bccfc3a3e..9ba0d45b6817 100644 --- a/python/requirements_compiled_rayllm_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_py311_cpu.txt @@ -1672,11 +1672,15 @@ ninja==1.11.1.3 \ # -r python/requirements/llm/llm-requirements.txt # vllm # xgrammar -nixl==0.3.1 \ - --hash=sha256:20428ad2668062a79045fae83cc5cba1f4019d4a2c7053cc8549c3a1533f8a75 \ - --hash=sha256:70b8932b50ccf1a13ac8fa2e10a4b78290baae9f963bfecfa67684104331a94b \ - --hash=sha256:8c144839484b3076f0b34ad8ceaeaff05c23399cf57ca85f2a94b44e1475a39b \ - --hash=sha256:ff59996ad05a7e4ba6c8beba0f1d8ac2f9e53df696a15af0d3340028e2f16081 +nixl==0.4.1 \ + --hash=sha256:10c7b4a44f89c3fbff3e20cb84973be95f8df36ee336fb108275ed1839fec1f1 \ + --hash=sha256:510cc9e824ad53cac71ce55ff41160f2a9e1507ceb52eb871b775fe1e42beb87 \ + --hash=sha256:8a3d83b28c16b795bdc281f1489b9d247f6e6088ad96ca96406072a36d6354b7 \ + --hash=sha256:9381fd3986d227c7ccb2607c03bbea559ec80f951e2ea47c1fbf381e4cd97164 \ + --hash=sha256:9ab7e580e9962ebdcda8c17f8548858d3fdb648621367d8e717ca317b534b778 \ + --hash=sha256:db144821de7912cb2502052b3070a1ac276b8b019470e6efdfce9c237ffe130d \ + --hash=sha256:e33102b85b3f95a8c95e59b59b29aabd03d47b5bce619de506b9bb83739cf60d \ + --hash=sha256:f16092dd445542e82e3db3553f6c7697ec5a2e837f19d416401283ae245826f9 # via # -c python/requirements_compiled_rayllm_test_py311_cpu.txt # -r python/requirements/llm/llm-requirements.txt diff --git a/python/requirements_compiled_rayllm_py311_cu121.txt b/python/requirements_compiled_rayllm_py311_cu121.txt index 5a778ecca825..4ad206d3186f 100644 --- a/python/requirements_compiled_rayllm_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_py311_cu121.txt @@ -1672,11 +1672,15 @@ ninja==1.11.1.3 \ # -r python/requirements/llm/llm-requirements.txt # vllm # xgrammar -nixl==0.3.1 \ - --hash=sha256:20428ad2668062a79045fae83cc5cba1f4019d4a2c7053cc8549c3a1533f8a75 \ - --hash=sha256:70b8932b50ccf1a13ac8fa2e10a4b78290baae9f963bfecfa67684104331a94b \ - --hash=sha256:8c144839484b3076f0b34ad8ceaeaff05c23399cf57ca85f2a94b44e1475a39b \ - --hash=sha256:ff59996ad05a7e4ba6c8beba0f1d8ac2f9e53df696a15af0d3340028e2f16081 +nixl==0.4.1 \ + --hash=sha256:10c7b4a44f89c3fbff3e20cb84973be95f8df36ee336fb108275ed1839fec1f1 \ + --hash=sha256:510cc9e824ad53cac71ce55ff41160f2a9e1507ceb52eb871b775fe1e42beb87 \ + --hash=sha256:8a3d83b28c16b795bdc281f1489b9d247f6e6088ad96ca96406072a36d6354b7 \ + --hash=sha256:9381fd3986d227c7ccb2607c03bbea559ec80f951e2ea47c1fbf381e4cd97164 \ + --hash=sha256:9ab7e580e9962ebdcda8c17f8548858d3fdb648621367d8e717ca317b534b778 \ + --hash=sha256:db144821de7912cb2502052b3070a1ac276b8b019470e6efdfce9c237ffe130d \ + --hash=sha256:e33102b85b3f95a8c95e59b59b29aabd03d47b5bce619de506b9bb83739cf60d \ + --hash=sha256:f16092dd445542e82e3db3553f6c7697ec5a2e837f19d416401283ae245826f9 # via # -c python/requirements_compiled_rayllm_test_py311_cu121.txt # -r python/requirements/llm/llm-requirements.txt diff --git a/python/requirements_compiled_rayllm_py311_cu128.txt b/python/requirements_compiled_rayllm_py311_cu128.txt index a3b95b0166a9..47338e202f4b 100644 --- a/python/requirements_compiled_rayllm_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_py311_cu128.txt @@ -1636,11 +1636,15 @@ ninja==1.11.1.4 \ # -r python/requirements/llm/llm-requirements.txt # vllm # xgrammar -nixl==0.3.1 \ - --hash=sha256:20428ad2668062a79045fae83cc5cba1f4019d4a2c7053cc8549c3a1533f8a75 \ - --hash=sha256:70b8932b50ccf1a13ac8fa2e10a4b78290baae9f963bfecfa67684104331a94b \ - --hash=sha256:8c144839484b3076f0b34ad8ceaeaff05c23399cf57ca85f2a94b44e1475a39b \ - --hash=sha256:ff59996ad05a7e4ba6c8beba0f1d8ac2f9e53df696a15af0d3340028e2f16081 +nixl==0.4.1 \ + --hash=sha256:10c7b4a44f89c3fbff3e20cb84973be95f8df36ee336fb108275ed1839fec1f1 \ + --hash=sha256:510cc9e824ad53cac71ce55ff41160f2a9e1507ceb52eb871b775fe1e42beb87 \ + --hash=sha256:8a3d83b28c16b795bdc281f1489b9d247f6e6088ad96ca96406072a36d6354b7 \ + --hash=sha256:9381fd3986d227c7ccb2607c03bbea559ec80f951e2ea47c1fbf381e4cd97164 \ + --hash=sha256:9ab7e580e9962ebdcda8c17f8548858d3fdb648621367d8e717ca317b534b778 \ + --hash=sha256:db144821de7912cb2502052b3070a1ac276b8b019470e6efdfce9c237ffe130d \ + --hash=sha256:e33102b85b3f95a8c95e59b59b29aabd03d47b5bce619de506b9bb83739cf60d \ + --hash=sha256:f16092dd445542e82e3db3553f6c7697ec5a2e837f19d416401283ae245826f9 # via # -c python/requirements_compiled_rayllm_test_py311_cu128.txt # -r python/requirements/llm/llm-requirements.txt diff --git a/python/requirements_compiled_rayllm_test_py311_cpu.txt b/python/requirements_compiled_rayllm_test_py311_cpu.txt index b69b89a5ea19..bcacf5033722 100644 --- a/python/requirements_compiled_rayllm_test_py311_cpu.txt +++ b/python/requirements_compiled_rayllm_test_py311_cpu.txt @@ -2373,11 +2373,15 @@ ninja==1.11.1.3 \ # -r python/requirements/llm/llm-requirements.txt # vllm # xgrammar -nixl==0.3.1 \ - --hash=sha256:20428ad2668062a79045fae83cc5cba1f4019d4a2c7053cc8549c3a1533f8a75 \ - --hash=sha256:70b8932b50ccf1a13ac8fa2e10a4b78290baae9f963bfecfa67684104331a94b \ - --hash=sha256:8c144839484b3076f0b34ad8ceaeaff05c23399cf57ca85f2a94b44e1475a39b \ - --hash=sha256:ff59996ad05a7e4ba6c8beba0f1d8ac2f9e53df696a15af0d3340028e2f16081 +nixl==0.4.1 \ + --hash=sha256:10c7b4a44f89c3fbff3e20cb84973be95f8df36ee336fb108275ed1839fec1f1 \ + --hash=sha256:510cc9e824ad53cac71ce55ff41160f2a9e1507ceb52eb871b775fe1e42beb87 \ + --hash=sha256:8a3d83b28c16b795bdc281f1489b9d247f6e6088ad96ca96406072a36d6354b7 \ + --hash=sha256:9381fd3986d227c7ccb2607c03bbea559ec80f951e2ea47c1fbf381e4cd97164 \ + --hash=sha256:9ab7e580e9962ebdcda8c17f8548858d3fdb648621367d8e717ca317b534b778 \ + --hash=sha256:db144821de7912cb2502052b3070a1ac276b8b019470e6efdfce9c237ffe130d \ + --hash=sha256:e33102b85b3f95a8c95e59b59b29aabd03d47b5bce619de506b9bb83739cf60d \ + --hash=sha256:f16092dd445542e82e3db3553f6c7697ec5a2e837f19d416401283ae245826f9 # via -r python/requirements/llm/llm-requirements.txt notebook==6.5.7 \ --hash=sha256:04eb9011dfac634fbd4442adaf0a8c27cd26beef831fe1d19faf930c327768e4 \ diff --git a/python/requirements_compiled_rayllm_test_py311_cu121.txt b/python/requirements_compiled_rayllm_test_py311_cu121.txt index ebf1056ff7c6..70d213fe4e0c 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu121.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu121.txt @@ -2373,11 +2373,15 @@ ninja==1.11.1.3 \ # -r python/requirements/llm/llm-requirements.txt # vllm # xgrammar -nixl==0.3.1 \ - --hash=sha256:20428ad2668062a79045fae83cc5cba1f4019d4a2c7053cc8549c3a1533f8a75 \ - --hash=sha256:70b8932b50ccf1a13ac8fa2e10a4b78290baae9f963bfecfa67684104331a94b \ - --hash=sha256:8c144839484b3076f0b34ad8ceaeaff05c23399cf57ca85f2a94b44e1475a39b \ - --hash=sha256:ff59996ad05a7e4ba6c8beba0f1d8ac2f9e53df696a15af0d3340028e2f16081 +nixl==0.4.1 \ + --hash=sha256:10c7b4a44f89c3fbff3e20cb84973be95f8df36ee336fb108275ed1839fec1f1 \ + --hash=sha256:510cc9e824ad53cac71ce55ff41160f2a9e1507ceb52eb871b775fe1e42beb87 \ + --hash=sha256:8a3d83b28c16b795bdc281f1489b9d247f6e6088ad96ca96406072a36d6354b7 \ + --hash=sha256:9381fd3986d227c7ccb2607c03bbea559ec80f951e2ea47c1fbf381e4cd97164 \ + --hash=sha256:9ab7e580e9962ebdcda8c17f8548858d3fdb648621367d8e717ca317b534b778 \ + --hash=sha256:db144821de7912cb2502052b3070a1ac276b8b019470e6efdfce9c237ffe130d \ + --hash=sha256:e33102b85b3f95a8c95e59b59b29aabd03d47b5bce619de506b9bb83739cf60d \ + --hash=sha256:f16092dd445542e82e3db3553f6c7697ec5a2e837f19d416401283ae245826f9 # via -r python/requirements/llm/llm-requirements.txt notebook==6.5.7 \ --hash=sha256:04eb9011dfac634fbd4442adaf0a8c27cd26beef831fe1d19faf930c327768e4 \ diff --git a/python/requirements_compiled_rayllm_test_py311_cu128.txt b/python/requirements_compiled_rayllm_test_py311_cu128.txt index 913cbd464409..c8ee9734ba19 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu128.txt +++ b/python/requirements_compiled_rayllm_test_py311_cu128.txt @@ -2336,11 +2336,15 @@ ninja==1.11.1.4 \ # -r python/requirements/llm/llm-requirements.txt # vllm # xgrammar -nixl==0.3.1 \ - --hash=sha256:20428ad2668062a79045fae83cc5cba1f4019d4a2c7053cc8549c3a1533f8a75 \ - --hash=sha256:70b8932b50ccf1a13ac8fa2e10a4b78290baae9f963bfecfa67684104331a94b \ - --hash=sha256:8c144839484b3076f0b34ad8ceaeaff05c23399cf57ca85f2a94b44e1475a39b \ - --hash=sha256:ff59996ad05a7e4ba6c8beba0f1d8ac2f9e53df696a15af0d3340028e2f16081 +nixl==0.4.1 \ + --hash=sha256:10c7b4a44f89c3fbff3e20cb84973be95f8df36ee336fb108275ed1839fec1f1 \ + --hash=sha256:510cc9e824ad53cac71ce55ff41160f2a9e1507ceb52eb871b775fe1e42beb87 \ + --hash=sha256:8a3d83b28c16b795bdc281f1489b9d247f6e6088ad96ca96406072a36d6354b7 \ + --hash=sha256:9381fd3986d227c7ccb2607c03bbea559ec80f951e2ea47c1fbf381e4cd97164 \ + --hash=sha256:9ab7e580e9962ebdcda8c17f8548858d3fdb648621367d8e717ca317b534b778 \ + --hash=sha256:db144821de7912cb2502052b3070a1ac276b8b019470e6efdfce9c237ffe130d \ + --hash=sha256:e33102b85b3f95a8c95e59b59b29aabd03d47b5bce619de506b9bb83739cf60d \ + --hash=sha256:f16092dd445542e82e3db3553f6c7697ec5a2e837f19d416401283ae245826f9 # via -r python/requirements/llm/llm-requirements.txt notebook==6.5.7 \ --hash=sha256:04eb9011dfac634fbd4442adaf0a8c27cd26beef831fe1d19faf930c327768e4 \ From 25d7644d8c110e39b711c40f4999cd39d1c425f6 Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Wed, 20 Aug 2025 14:41:19 -0700 Subject: [PATCH 206/634] Change the ray_accel_env_var_override_on_zero warning to future_warning (#55791) Before this pr: ``` python /ray-workspace/temp.py 2025-08-18 18:26:21,621 INFO worker.py:1943 -- Started a local Ray instance. View the dashboard at http://127.0.0.1:8265 2025-08-18 18:26:53,873 WARNING worker.py:1991 -- Tip: In future versions of Ray, Ray will no longer override accelerator visible devices env var if num_gpus=0 or num_gpus=None (default). To enable this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0 ``` After this pr: ``` 2025-08-20 19:08:41,923 INFO worker.py:1939 -- Started a local Ray instance. View the dashboard at http://127.0.0.1:8265 /sgl-workspace/ray/python/ray/_private/worker.py:1987: FutureWarning: Tip: In future versions of Ray, Ray will no longer override accelerator visible devices env var if num_gpus=0 or num_gpus=None (default). To enable this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0 warnings.warn( ``` #54928 --- python/ray/_private/worker.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/ray/_private/worker.py b/python/ray/_private/worker.py index 8e31acf2fc78..ebb15ee817e1 100644 --- a/python/ray/_private/worker.py +++ b/python/ray/_private/worker.py @@ -1984,10 +1984,11 @@ def sigterm_handler(signum, frame): True, ) if override_on_zero and log_once("ray_accel_env_var_override_on_zero"): - logger.warning( + warnings.warn( "Tip: In future versions of Ray, Ray will no longer override accelerator " "visible devices env var if num_gpus=0 or num_gpus=None (default). To enable " - "this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0" + "this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0", + FutureWarning, ) node_id = global_worker.core_worker.get_current_node_id() From 4efc09af349d2e39b6b862daa3da903caa63eacc Mon Sep 17 00:00:00 2001 From: avigyabb <98926738+avigyabb@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:50:37 -0700 Subject: [PATCH 207/634] [Core] Bind dashboard agent grpc to specified ip instead of 0.0.0.0 (#55732) Signed-off-by: avigyabb --- python/ray/_private/services.py | 2 +- python/ray/dashboard/agent.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/python/ray/_private/services.py b/python/ray/_private/services.py index 3abe99bf23b4..94040866676d 100644 --- a/python/ray/_private/services.py +++ b/python/ray/_private/services.py @@ -1792,7 +1792,7 @@ def start_raylet( os.path.join(RAY_PATH, "dashboard", "agent.py"), f"--node-ip-address={node_ip_address}", f"--metrics-export-port={metrics_export_port}", - f"--dashboard-agent-port={metrics_agent_port}", + f"--grpc-port={metrics_agent_port}", f"--listen-port={dashboard_agent_listen_port}", "--node-manager-port=RAY_NODE_MANAGER_PORT_PLACEHOLDER", f"--object-store-name={plasma_store_name}", diff --git a/python/ray/dashboard/agent.py b/python/ray/dashboard/agent.py index 1c46afee5372..976a9ee7e1a3 100644 --- a/python/ray/dashboard/agent.py +++ b/python/ray/dashboard/agent.py @@ -24,7 +24,7 @@ class DashboardAgent: def __init__( self, node_ip_address, - dashboard_agent_port, + grpc_port, gcs_address, cluster_id_hex, minimal, @@ -54,7 +54,7 @@ def __init__( self.temp_dir = temp_dir self.session_dir = session_dir self.log_dir = log_dir - self.dashboard_agent_port = dashboard_agent_port + self.grpc_port = grpc_port self.metrics_export_port = metrics_export_port self.node_manager_port = node_manager_port self.events_export_addr = events_export_addr @@ -111,11 +111,10 @@ def _init_non_minimal(self): ), ) # noqa ) - grpc_ip = "127.0.0.1" if self.ip == "127.0.0.1" else "0.0.0.0" try: - self.grpc_port = add_port_to_grpc_server( - self.server, build_address(grpc_ip, self.dashboard_agent_port) - ) + add_port_to_grpc_server(self.server, build_address(self.ip, self.grpc_port)) + if self.ip != "127.0.0.1" and self.ip != "localhost": + add_port_to_grpc_server(self.server, f"127.0.0.1:{self.grpc_port}") except Exception: # TODO(SongGuyang): Catch the exception here because there is # port conflict issue which brought from static port. We should @@ -129,7 +128,7 @@ def _init_non_minimal(self): else: logger.info( "Dashboard agent grpc address: %s", - build_address(grpc_ip, self.grpc_port), + build_address(self.ip, self.grpc_port), ) # If the agent is not minimal it should start the http server @@ -263,7 +262,7 @@ async def wait_forever(): help="The port to expose metrics through Prometheus.", ) parser.add_argument( - "--dashboard-agent-port", + "--grpc-port", required=True, type=int, help="The port on which the dashboard agent will receive GRPCs.", @@ -424,7 +423,7 @@ async def wait_forever(): agent = DashboardAgent( args.node_ip_address, - args.dashboard_agent_port, + args.grpc_port, args.gcs_address, args.cluster_id_hex, args.minimal, From 9671f678b613d569d4fd4fe7351657f0556d4b19 Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Wed, 20 Aug 2025 15:23:23 -0700 Subject: [PATCH 208/634] [Data] Fix broken code snippets in user guides (#55519) In https://github.com/ray-project/ray/pull/51334, we discovered we weren't actually testing code snippets in our user guides. As a result, there are several broken code snippets in our guides. This PR fixes some of those code snippets, and re-enables testing on the user guides. --------- Signed-off-by: Balaji Veeramani --- bazel/python.bzl | 18 ++++++++++ doc/BUILD.bazel | 15 +++------ doc/source/data/data-internals.rst | 9 ++++- doc/source/data/inspecting-data.rst | 6 ++-- doc/source/data/loading-data.rst | 34 ++++++++++++------- doc/source/data/performance-tips.rst | 18 ++++++---- doc/source/data/saving-data.rst | 42 ++++++++++++++---------- doc/source/data/working-with-images.rst | 2 +- doc/source/data/working-with-pytorch.rst | 4 +-- 9 files changed, 97 insertions(+), 51 deletions(-) diff --git a/bazel/python.bzl b/bazel/python.bzl index 4ebe4cffdcdc..3c15ae6f527d 100644 --- a/bazel/python.bzl +++ b/bazel/python.bzl @@ -17,6 +17,24 @@ def _convert_target_to_import_path(t): # 3) Replace '/' with '.' to form an import path. return t.replace("/", ".") +def doctest_each(files, gpu = False, deps=[], srcs=[], data=[], args=[], size="medium", tags=[], pytest_plugin_file="//bazel:default_doctest_pytest_plugin.py", **kwargs): + # Unlike the `doctest` macro, `doctest_each` runs `pytest` on each file separately. + # This is useful to run tests in parallel and more clearly report the test results. + for file in files: + doctest( + files = [file], + gpu = gpu, + name = paths.split_extension(file)[0], + deps = deps, + srcs = srcs, + data = data, + args = args, + size = size, + tags = tags, + pytest_plugin_file = pytest_plugin_file, + **kwargs + ) + def doctest(files, gpu = False, name="doctest", deps=[], srcs=[], data=[], args=[], size="medium", tags=[], pytest_plugin_file="//bazel:default_doctest_pytest_plugin.py", **kwargs): # NOTE: If you run `pytest` on `__init__.py`, it tries to test all files in that # package. We don't want that, so we exclude it from the list of input files. diff --git a/doc/BUILD.bazel b/doc/BUILD.bazel index 2d0ac0634687..617dd3bac2a3 100644 --- a/doc/BUILD.bazel +++ b/doc/BUILD.bazel @@ -1,6 +1,6 @@ load("@py_deps_buildkite//:requirements.bzl", ci_require = "requirement") load("@rules_python//python:defs.bzl", "py_test") -load("//bazel:python.bzl", "doctest", "py_test_run_all_notebooks", "py_test_run_all_subdirectory") +load("//bazel:python.bzl", "doctest", "doctest_each", "py_test_run_all_notebooks", "py_test_run_all_subdirectory") exports_files(["test_myst_doc.py"]) @@ -480,8 +480,7 @@ doctest( tags = ["team:core"], ) -doctest( - name = "doctest[data]", +doctest_each( files = glob( include = [ "source/data/**/*.md", @@ -492,15 +491,9 @@ doctest( "source/data/batch_inference.rst", "source/data/transforming-data.rst", # These tests are currently failing. - "source/data/loading-data.rst", - "source/data/data-internals.rst", - "source/data/inspecting-data.rst", - "source/data/loading-data.rst", - "source/data/performance-tips.rst", - "source/data/saving-data.rst", - "source/data/working-with-images.rst", "source/data/working-with-llms.rst", - "source/data/working-with-pytorch.rst", + # These don't contain code snippets. + "source/data/api/**/*.rst", ], ), pytest_plugin_file = "//python/ray/data:tests/doctest_pytest_plugin.py", diff --git a/doc/source/data/data-internals.rst b/doc/source/data/data-internals.rst index d3d3917f429d..d4a449939b14 100644 --- a/doc/source/data/data-internals.rst +++ b/doc/source/data/data-internals.rst @@ -179,12 +179,19 @@ To add custom optimization rules, implement a class that extends ``Rule`` and co import ray from ray.data._internal.logical.interfaces import Rule + from ray.data._internal.logical.optimizers import get_logical_ruleset class CustomRule(Rule): def apply(self, plan): ... - ray.data._internal.logical.optimizers.DEFAULT_LOGICAL_RULES.append(CustomRule) + logical_ruleset = get_logical_ruleset() + logical_ruleset.add(CustomRule) + +.. testcode:: + :hide: + + logical_ruleset.remove(CustomRule) Types of physical operators ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/data/inspecting-data.rst b/doc/source/data/inspecting-data.rst index 986b0d82b6e1..0936204fc655 100644 --- a/doc/source/data/inspecting-data.rst +++ b/doc/source/data/inspecting-data.rst @@ -123,12 +123,11 @@ of the returned batch, set ``batch_format``. print(batch) .. testoutput:: - :options: +NORMALIZE_WHITESPACE + :options: +MOCK sepal length (cm) sepal width (cm) ... petal width (cm) target 0 5.1 3.5 ... 0.2 0 1 4.9 3.0 ... 0.2 0 - For more information on working with batches, see :ref:`Transforming batches ` and @@ -143,7 +142,10 @@ Ray Data calculates statistics during execution for each operator, such as wall To view stats about your :class:`Datasets `, call :meth:`Dataset.stats() ` on an executed dataset. The stats are also persisted under `/tmp/ray/session_*/logs/ray-data/ray-data.log`. For more on how to read this output, see :ref:`Monitoring Your Workload with the Ray Data Dashboard `. +.. This snippet below is skipped because of https://github.com/ray-project/ray/issues/54101. + .. testcode:: + :skipif: True import ray import datasets diff --git a/doc/source/data/loading-data.rst b/doc/source/data/loading-data.rst index ce80c8f21e9a..3abe4e31fb8e 100644 --- a/doc/source/data/loading-data.rst +++ b/doc/source/data/loading-data.rst @@ -486,13 +486,16 @@ Ray Data interoperates with distributed data processing frameworks like `Daft `__. + .. testcode:: + :skipif: True import daft import ray - ray.init() - df = daft.from_pydict({"int_col": [i for i in range(10000)], "str_col": [str(i) for i in range(10000)]}) ds = ray.data.from_daft(df) @@ -512,7 +515,12 @@ Ray Data interoperates with distributed data processing frameworks like `Daft >> import ray - >>> from pyiceberg.expressions import EqualTo - >>> ds = ray.data.read_iceberg( - ... table_identifier="db_name.table_name", - ... row_filter=EqualTo("column_name", "literal_value"), - ... catalog_kwargs={"name": "default", "type": "glue"} - ... ) + import ray + from pyiceberg.expressions import EqualTo + ds = ray.data.read_iceberg( + table_identifier="db_name.table_name", + row_filter=EqualTo("column_name", "literal_value"), + catalog_kwargs={"name": "default", "type": "glue"} + ) + ds.show(3) .. testoutput:: + :options: +MOCK {'col1': 0, 'col2': '0'} {'col1': 1, 'col2': '1'} @@ -622,6 +630,7 @@ Ray Data interoperates with distributed data processing frameworks like `Daft `_ objects aren't supported. + .. This snippet below is skipped because of https://github.com/ray-project/ray/issues/54837. + .. testcode:: + :skipif: True import ray.data from datasets import load_dataset diff --git a/doc/source/data/performance-tips.rst b/doc/source/data/performance-tips.rst index 9657d0235f35..1155663b22f4 100644 --- a/doc/source/data/performance-tips.rst +++ b/doc/source/data/performance-tips.rst @@ -51,7 +51,7 @@ For example, the following code batches multiple files into the same read task t ray.init(num_cpus=2) # Repeat the iris.csv file 16 times. - ds = ray.data.read_csv(["example://iris.csv"] * 16) + ds = ray.data.read_csv(["s3://anonymous@ray-example-data/iris.csv"] * 16) print(ds.materialize()) .. testoutput:: @@ -81,7 +81,7 @@ Notice how the number of output blocks is equal to ``override_num_blocks`` in th ray.init(num_cpus=2) # Repeat the iris.csv file 16 times. - ds = ray.data.read_csv(["example://iris.csv"] * 16, override_num_blocks=16) + ds = ray.data.read_csv(["s3://anonymous@ray-example-data/iris.csv"] * 16, override_num_blocks=16) print(ds.materialize()) .. testoutput:: @@ -143,7 +143,7 @@ For example, the following code executes :func:`~ray.data.read_csv` with only on # Pretend there are two CPUs. ray.init(num_cpus=2) - ds = ray.data.read_csv("example://iris.csv").map(lambda row: row) + ds = ray.data.read_csv("s3://anonymous@ray-example-data/iris.csv").map(lambda row: row) print(ds.materialize().stats()) .. testoutput:: @@ -171,7 +171,7 @@ For example, this code sets the number of files equal to ``override_num_blocks`` # Pretend there are two CPUs. ray.init(num_cpus=2) - ds = ray.data.read_csv("example://iris.csv", override_num_blocks=1).map(lambda row: row) + ds = ray.data.read_csv("s3://anonymous@ray-example-data/iris.csv", override_num_blocks=1).map(lambda row: row) print(ds.materialize().stats()) .. testoutput:: @@ -205,15 +205,21 @@ calling :func:`~ray.data.Dataset.select_columns`, since column selection is push .. testcode:: import ray + # Read just two of the five columns of the Iris dataset. - ray.data.read_parquet( + ds = ray.data.read_parquet( "s3://anonymous@ray-example-data/iris.parquet", columns=["sepal.length", "variety"], ) + + print(ds.schema()) .. testoutput:: - Dataset(num_rows=150, schema={sepal.length: double, variety: string}) + Column Type + ------ ---- + sepal.length double + variety string .. _data_memory: diff --git a/doc/source/data/saving-data.rst b/doc/source/data/saving-data.rst index d37215541584..7347f0bc4c5b 100644 --- a/doc/source/data/saving-data.rst +++ b/doc/source/data/saving-data.rst @@ -228,7 +228,7 @@ number of files & their sizes (since every block could potentially carry the row print_directory_tree("/tmp/sales_partitioned") .. testoutput:: - :options: +NORMALIZE_WHITESPACE + :options: +MOCK sales_partitioned/ city=NYC/ @@ -301,24 +301,10 @@ Ray Data interoperates with distributed data processing frameworks like `Daft `__, call - :meth:`Dataset.to_dask() `. - - .. testcode:: - - import ray - - ds = ray.data.read_csv("s3://anonymous@ray-example-data/iris.csv") - - df = ds.to_dask() - - df + print(df) .. testoutput:: + :options: +MOCK ╭───────────────────┬──────────────────┬───────────────────┬──────────────────┬────────╮ │ sepal length (cm) ┆ sepal width (cm) ┆ petal length (cm) ┆ petal width (cm) ┆ target │ @@ -345,6 +331,25 @@ Ray Data interoperates with distributed data processing frameworks like `Daft `__, call + :meth:`Dataset.to_dask() `. + + .. + We skip the code snippet below because `to_dask` doesn't work with PyArrow + 14 and later. For more information, see https://github.com/ray-project/ray/issues/54837 + + .. testcode:: + :skipif: True + + import ray + + ds = ray.data.read_csv("s3://anonymous@ray-example-data/iris.csv") + + df = ds.to_dask() + .. tab-item:: Spark To convert a :class:`~ray.data.dataset.Dataset` to a `Spark DataFrame @@ -352,6 +357,7 @@ Ray Data interoperates with distributed data processing frameworks like `Daft `. .. testcode:: + :skipif: True import ray import raydp @@ -367,6 +373,7 @@ Ray Data interoperates with distributed data processing frameworks like `Daft `. .. testcode:: + :skipif: True import ray diff --git a/doc/source/data/working-with-images.rst b/doc/source/data/working-with-images.rst index 1f53fbcdf568..3152d1a90c07 100644 --- a/doc/source/data/working-with-images.rst +++ b/doc/source/data/working-with-images.rst @@ -147,7 +147,7 @@ To view the full list of supported file formats, see the Column Type ------ ---- - image numpy.ndarray(shape=(32, 32, 3), dtype=uint8) + img struct label int64 diff --git a/doc/source/data/working-with-pytorch.rst b/doc/source/data/working-with-pytorch.rst index a101333379b3..19e9d59d53bc 100644 --- a/doc/source/data/working-with-pytorch.rst +++ b/doc/source/data/working-with-pytorch.rst @@ -229,8 +229,8 @@ You can use built-in Torch transforms from ``torchvision``, ``torchtext``, and ` Column Type ------ ---- - text - tokenized_text + text string + tokenized_text list .. _batch_inference_pytorch: From 161ec2920dd390ff1e97c71295436fead145268b Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Wed, 20 Aug 2025 18:01:59 -0700 Subject: [PATCH 209/634] [ci] parameterizing config path for compiling deps (#55757) parameterizing config path for raydepsets --------- Signed-off-by: elliot-barn Signed-off-by: Elliot Barnwell Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- ci/compile_llm_requirements.sh | 4 +++- ci/test_compile_llm_requirements.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/compile_llm_requirements.sh b/ci/compile_llm_requirements.sh index a9b5f9d06660..2b3c0de1cb04 100755 --- a/ci/compile_llm_requirements.sh +++ b/ci/compile_llm_requirements.sh @@ -2,6 +2,8 @@ set -euo pipefail +CONFIG_PATH="${1:-ci/raydepsets/rayllm.depsets.yaml}" + PYTHON_CODE="$(python -c "import sys; v=sys.version_info; print(f'py{v.major}{v.minor}')")" if [[ "${PYTHON_CODE}" != "py311" ]]; then echo "--- Python version is not 3.11" @@ -16,6 +18,6 @@ cp python/requirements_compiled.txt /tmp/ray-deps/requirements_compiled.txt sed -e '/^--extra-index-url /d' -e '/^--find-links /d' /tmp/ray-deps/requirements_compiled.txt > /tmp/ray-deps/requirements_compiled.txt.tmp mv /tmp/ray-deps/requirements_compiled.txt.tmp /tmp/ray-deps/requirements_compiled.txt -bazel run //ci/raydepsets:raydepsets -- build ci/raydepsets/rayllm.depsets.yaml +bazel run //ci/raydepsets:raydepsets -- build "${CONFIG_PATH}" echo "--- Done" diff --git a/ci/test_compile_llm_requirements.sh b/ci/test_compile_llm_requirements.sh index 80a4a8d0a044..d9e08caf3c77 100755 --- a/ci/test_compile_llm_requirements.sh +++ b/ci/test_compile_llm_requirements.sh @@ -22,7 +22,7 @@ for LOCK_TYPE in "${LOCK_TYPES[@]}"; do done done -./ci/compile_llm_requirements.sh +./ci/compile_llm_requirements.sh ci/raydepsets/rayllm.depsets.yaml # Copy files to artifact mount on Buildkite for LOCK_TYPE in "${LOCK_TYPES[@]}"; do From 881cfab6f1ee4928813e2e060aebfed981d29d13 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Wed, 20 Aug 2025 18:47:41 -0700 Subject: [PATCH 210/634] [core] Add pubsub documentation, cleanup subscriber + tests (#55735) Signed-off-by: dayshah --- src/fakes/ray/pubsub/subscriber.h | 33 ++--- .../ray/gcs/gcs_server/gcs_actor_manager.h | 5 + .../ray/gcs/gcs_server/gcs_node_manager.h | 4 + src/mock/ray/pubsub/BUILD.bazel | 8 -- src/mock/ray/pubsub/publisher.h | 3 + src/mock/ray/pubsub/subscriber.h | 70 ----------- src/ray/core_worker/reference_count.cc | 15 ++- src/ray/core_worker/tests/BUILD.bazel | 5 +- .../tests/object_recovery_manager_test.cc | 6 +- .../core_worker/tests/reference_count_test.cc | 50 +++----- .../core_worker/tests/task_manager_test.cc | 6 +- src/ray/gcs/gcs_client/gcs_client.cc | 4 - src/ray/gcs/gcs_server/tests/BUILD.bazel | 1 - .../gcs_job_manager_export_event_test.cc | 14 +-- .../tests/gcs_actor_scheduler_mock_test.cc | 14 +-- .../gcs_autoscaler_state_manager_test.cc | 26 ++-- .../gcs_server/tests/gcs_job_manager_test.cc | 12 +- src/ray/gcs/pubsub/gcs_pub_sub.cc | 30 ++--- .../ownership_object_directory.cc | 4 +- src/ray/object_manager/tests/BUILD.bazel | 2 +- .../tests/ownership_object_directory_test.cc | 7 +- src/ray/protobuf/pubsub.proto | 27 ++--- src/ray/pubsub/README.md | 103 ++++++++++++---- src/ray/pubsub/subscriber.cc | 113 +++++------------- src/ray/pubsub/subscriber.h | 59 +++------ src/ray/pubsub/subscriber_interface.h | 59 +++------ src/ray/pubsub/tests/BUILD.bazel | 2 +- ...ion_test.cc => pubsub_integration_test.cc} | 7 +- src/ray/pubsub/tests/subscriber_test.cc | 65 +++++----- src/ray/raylet/.gitkeep | 0 src/ray/raylet/local_object_manager.cc | 14 +-- src/ray/raylet/tests/BUILD.bazel | 1 - .../raylet/tests/local_object_manager_test.cc | 83 +++++++------ src/ray/raylet/tests/node_manager_test.cc | 7 +- 34 files changed, 349 insertions(+), 510 deletions(-) delete mode 100644 src/mock/ray/pubsub/subscriber.h rename src/ray/pubsub/tests/{integration_test.cc => pubsub_integration_test.cc} (98%) delete mode 100644 src/ray/raylet/.gitkeep diff --git a/src/fakes/ray/pubsub/subscriber.h b/src/fakes/ray/pubsub/subscriber.h index 5abd3d33ba2d..b0afd5dd03fc 100644 --- a/src/fakes/ray/pubsub/subscriber.h +++ b/src/fakes/ray/pubsub/subscriber.h @@ -14,7 +14,7 @@ #pragma once -#include "ray/pubsub/subscriber.h" +#include "ray/pubsub/subscriber_interface.h" namespace ray { namespace pubsub { @@ -32,39 +32,22 @@ class FakeSubscriberClient : public SubscriberClientInterface { class FakeSubscriber : public SubscriberInterface { public: - bool Subscribe( + void Subscribe( std::unique_ptr sub_message, - const rpc::ChannelType channel_type, + rpc::ChannelType channel_type, const rpc::Address &owner_address, - const std::string &key_id, + const std::optional &key_id, pubsub::SubscribeDoneCallback subscribe_done_callback, pubsub::SubscriptionItemCallback subscription_callback, - pubsub::SubscriptionFailureCallback subscription_failure_callback) override { - return true; - } - - bool SubscribeChannel( - std::unique_ptr sub_message, - const rpc::ChannelType channel_type, - const rpc::Address &owner_address, - pubsub::SubscribeDoneCallback subscribe_done_callback, - pubsub::SubscriptionItemCallback subscription_callback, - pubsub::SubscriptionFailureCallback subscription_failure_callback) override { - return true; - } + pubsub::SubscriptionFailureCallback subscription_failure_callback) override {} - bool Unsubscribe(const rpc::ChannelType channel_type, + bool Unsubscribe(rpc::ChannelType channel_type, const rpc::Address &publisher_address, - const std::string &key_id) override { - return true; - } - - bool UnsubscribeChannel(const rpc::ChannelType channel_type, - const rpc::Address &publisher_address) override { + const std::optional &key_id) override { return true; } - bool IsSubscribed(const rpc::ChannelType channel_type, + bool IsSubscribed(rpc::ChannelType channel_type, const rpc::Address &publisher_address, const std::string &key_id) const override { return false; diff --git a/src/mock/ray/gcs/gcs_server/gcs_actor_manager.h b/src/mock/ray/gcs/gcs_server/gcs_actor_manager.h index b960f753ac3d..7799fd7a9c01 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_actor_manager.h +++ b/src/mock/ray/gcs/gcs_server/gcs_actor_manager.h @@ -12,6 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#pragma once + +#include "gmock/gmock.h" +#include "ray/gcs/gcs_server/gcs_actor_manager.h" + namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_node_manager.h b/src/mock/ray/gcs/gcs_server/gcs_node_manager.h index 5d1851b867a8..919324cb0996 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/mock/ray/gcs/gcs_server/gcs_node_manager.h @@ -11,7 +11,11 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + +#pragma once + #include "gmock/gmock.h" +#include "ray/gcs/gcs_server/gcs_node_manager.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/pubsub/BUILD.bazel b/src/mock/ray/pubsub/BUILD.bazel index 1a40ca0ac02c..23bfce50a7f4 100644 --- a/src/mock/ray/pubsub/BUILD.bazel +++ b/src/mock/ray/pubsub/BUILD.bazel @@ -7,11 +7,3 @@ ray_cc_library( "//src/ray/pubsub:publisher_interface", ], ) - -ray_cc_library( - name = "mock_subscriber", - hdrs = ["subscriber.h"], - deps = [ - "//src/ray/pubsub:subscriber_interface", - ], -) diff --git a/src/mock/ray/pubsub/publisher.h b/src/mock/ray/pubsub/publisher.h index dfad1d21c2ce..1e384427db95 100644 --- a/src/mock/ray/pubsub/publisher.h +++ b/src/mock/ray/pubsub/publisher.h @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#pragma once + +#include "gmock/gmock.h" #include "ray/pubsub/publisher_interface.h" namespace ray { diff --git a/src/mock/ray/pubsub/subscriber.h b/src/mock/ray/pubsub/subscriber.h deleted file mode 100644 index cf21d3a4e022..000000000000 --- a/src/mock/ray/pubsub/subscriber.h +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2021 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include "gmock/gmock.h" -#include "ray/pubsub/subscriber.h" - -namespace ray { -namespace pubsub { - -class MockSubscriber : public SubscriberInterface { - public: - MOCK_METHOD(bool, - Subscribe, - (std::unique_ptr sub_message, - const rpc::ChannelType channel_type, - const rpc::Address &owner_address, - const std::string &key_id, - pubsub::SubscribeDoneCallback subscribe_done_callback, - pubsub::SubscriptionItemCallback subscription_callback, - pubsub::SubscriptionFailureCallback subscription_failure_callback), - (override)); - - MOCK_METHOD(bool, - SubscribeChannel, - (std::unique_ptr sub_message, - const rpc::ChannelType channel_type, - const rpc::Address &owner_address, - pubsub::SubscribeDoneCallback subscribe_done_callback, - pubsub::SubscriptionItemCallback subscription_callback, - pubsub::SubscriptionFailureCallback subscription_failure_callback), - (override)); - - MOCK_METHOD(bool, - Unsubscribe, - (const rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - const std::string &key_id), - (override)); - - MOCK_METHOD(bool, - UnsubscribeChannel, - (const rpc::ChannelType channel_type, - const rpc::Address &publisher_address), - (override)); - - MOCK_METHOD(bool, - IsSubscribed, - (const rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - const std::string &key_id), - (const, override)); - - MOCK_METHOD(std::string, DebugString, (), (const, override)); -}; - -} // namespace pubsub -} // namespace ray diff --git a/src/ray/core_worker/reference_count.cc b/src/ray/core_worker/reference_count.cc index 6e328900a907..11c237904d2b 100644 --- a/src/ray/core_worker/reference_count.cc +++ b/src/ray/core_worker/reference_count.cc @@ -1175,14 +1175,13 @@ void ReferenceCounter::WaitForRefRemoved(const ReferenceTable::iterator &ref_it, CleanupBorrowersOnRefRemoved({}, failed_borrower_object_id, addr); }; - RAY_CHECK( - object_info_subscriber_->Subscribe(std::move(sub_message), - rpc::ChannelType::WORKER_REF_REMOVED_CHANNEL, - addr, - object_id.Binary(), - /*subscribe_done_callback=*/nullptr, - message_published_callback, - publisher_failed_callback)); + object_info_subscriber_->Subscribe(std::move(sub_message), + rpc::ChannelType::WORKER_REF_REMOVED_CHANNEL, + addr, + object_id.Binary(), + /*subscribe_done_callback=*/nullptr, + message_published_callback, + publisher_failed_callback); } void ReferenceCounter::AddNestedObjectIds(const ObjectID &object_id, diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index e201590fdd06..abf81304b176 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -46,8 +46,8 @@ ray_cc_test( srcs = ["reference_count_test.cc"], tags = ["team:core"], deps = [ + "//:ray_fakes", "//src/mock/ray/pubsub:mock_publisher", - "//src/mock/ray/pubsub:mock_subscriber", "//src/ray/common:asio", "//src/ray/common:ray_object", "//src/ray/core_worker:memory_store", @@ -69,7 +69,6 @@ ray_cc_test( "//:ray_fakes", "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", - "//src/mock/ray/pubsub:mock_subscriber", "//src/ray/common:task_common", "//src/ray/common:test_util", "//src/ray/core_worker:memory_store", @@ -87,9 +86,9 @@ ray_cc_test( srcs = ["task_manager_test.cc"], tags = ["team:core"], deps = [ + "//:ray_fakes", "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", - "//src/mock/ray/pubsub:mock_subscriber", "//src/ray/common:task_common", "//src/ray/common:test_util", "//src/ray/core_worker:memory_store", diff --git a/src/ray/core_worker/tests/object_recovery_manager_test.cc b/src/ray/core_worker/tests/object_recovery_manager_test.cc index 55eeaebf706d..99024c599bde 100644 --- a/src/ray/core_worker/tests/object_recovery_manager_test.cc +++ b/src/ray/core_worker/tests/object_recovery_manager_test.cc @@ -20,12 +20,12 @@ #include #include +#include "fakes/ray/pubsub/subscriber.h" #include "fakes/ray/rpc/raylet/raylet_client.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include "mock/ray/core_worker/task_manager_interface.h" #include "mock/ray/pubsub/publisher.h" -#include "mock/ray/pubsub/subscriber.h" #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" #include "ray/common/test_util.h" @@ -124,7 +124,7 @@ class ObjectRecoveryManagerTestBase : public ::testing::Test { : local_node_id_(NodeID::FromRandom()), io_context_("TestOnly.ObjectRecoveryManagerTestBase"), publisher_(std::make_shared()), - subscriber_(std::make_shared()), + subscriber_(std::make_shared()), object_directory_(std::make_shared()), memory_store_( std::make_shared(io_context_.GetIoService())), @@ -177,7 +177,7 @@ class ObjectRecoveryManagerTestBase : public ::testing::Test { // Used by memory_store_. InstrumentedIOContextWithThread io_context_; std::shared_ptr publisher_; - std::shared_ptr subscriber_; + std::shared_ptr subscriber_; std::shared_ptr object_directory_; std::shared_ptr memory_store_; std::shared_ptr raylet_client_pool_; diff --git a/src/ray/core_worker/tests/reference_count_test.cc b/src/ray/core_worker/tests/reference_count_test.cc index 4ca580c67ddc..44f753bbe693 100644 --- a/src/ray/core_worker/tests/reference_count_test.cc +++ b/src/ray/core_worker/tests/reference_count_test.cc @@ -20,10 +20,10 @@ #include #include "absl/functional/bind_front.h" +#include "fakes/ray/pubsub/subscriber.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include "mock/ray/pubsub/publisher.h" -#include "mock/ray/pubsub/subscriber.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/asio/periodical_runner.h" #include "ray/common/ray_object.h" @@ -44,7 +44,7 @@ class ReferenceCountTest : public ::testing::Test { virtual void SetUp() { rpc::Address addr; publisher_ = std::make_shared(); - subscriber_ = std::make_shared(); + subscriber_ = std::make_shared(); rc = std::make_unique( addr, publisher_.get(), subscriber_.get(), [](const NodeID &node_id) { return false; @@ -61,7 +61,7 @@ class ReferenceCountTest : public ::testing::Test { void AssertNoLeaks() { ASSERT_EQ(rc->NumObjectIDsInScope(), 0); } std::shared_ptr publisher_; - std::shared_ptr subscriber_; + std::shared_ptr subscriber_; }; class ReferenceCountLineageEnabledTest : public ::testing::Test { @@ -70,7 +70,7 @@ class ReferenceCountLineageEnabledTest : public ::testing::Test { virtual void SetUp() { rpc::Address addr; publisher_ = std::make_shared(); - subscriber_ = std::make_shared(); + subscriber_ = std::make_shared(); rc = std::make_unique( addr, publisher_.get(), @@ -86,7 +86,7 @@ class ReferenceCountLineageEnabledTest : public ::testing::Test { } std::shared_ptr publisher_; - std::shared_ptr subscriber_; + std::shared_ptr subscriber_; }; /// The 2 classes below are implemented to support distributed mock test using @@ -146,11 +146,11 @@ class MockDistributedSubscriber : public pubsub::SubscriberInterface { ~MockDistributedSubscriber() = default; - bool Subscribe( - const std::unique_ptr sub_message, - const rpc::ChannelType channel_type, + void Subscribe( + std::unique_ptr sub_message, + rpc::ChannelType channel_type, const rpc::Address &publisher_address, - const std::string &key_id_binary, + const std::optional &key_id_binary, pubsub::SubscribeDoneCallback subscribe_done_callback, pubsub::SubscriptionItemCallback subscription_callback, pubsub::SubscriptionFailureCallback subscription_failure_callback) override { @@ -166,9 +166,9 @@ class MockDistributedSubscriber : public pubsub::SubscriberInterface { } // Due to the test env, there are times that the same message id from the same // subscriber is subscribed twice. We should just no-op in this case. - if (!(directory_->HasKeyId(key_id_binary) && + if (!(directory_->HasKeyId(*key_id_binary) && directory_->HasSubscriber(subscriber_id_))) { - directory_->AddEntry(key_id_binary, subscriber_.get()); + directory_->AddEntry(*key_id_binary, subscriber_.get()); } const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); const auto id = GenerateID(publisher_id, subscriber_id_); @@ -184,34 +184,18 @@ class MockDistributedSubscriber : public pubsub::SubscriberInterface { .first; } - const auto oid = ObjectID::FromBinary(key_id_binary); + const auto oid = ObjectID::FromBinary(*key_id_binary); callback_it->second.emplace(oid, subscription_callback); - return failure_callback_it->second.emplace(oid, subscription_failure_callback).second; + failure_callback_it->second.emplace(oid, subscription_failure_callback); } - bool SubscribeChannel( - const std::unique_ptr sub_message, - const rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - pubsub::SubscribeDoneCallback subscribe_done_callback, - pubsub::SubscriptionItemCallback subscription_callback, - pubsub::SubscriptionFailureCallback subscription_failure_callback) override { - RAY_LOG(FATAL) << "Unimplemented!"; - return false; - } - - bool Unsubscribe(const rpc::ChannelType channel_type, + bool Unsubscribe(rpc::ChannelType channel_type, const rpc::Address &publisher_address, - const std::string &key_id_binary) override { - return true; - } - - bool UnsubscribeChannel(const rpc::ChannelType channel_type, - const rpc::Address &publisher_address) override { + const std::optional &key_id_binary) override { return true; } - bool IsSubscribed(const rpc::ChannelType channel_type, + bool IsSubscribed(rpc::ChannelType channel_type, const rpc::Address &publisher_address, const std::string &key_id_binary) const override { return directory_->HasKeyId(key_id_binary) && @@ -837,7 +821,7 @@ TEST(MemoryStoreIntegrationTest, TestSimple) { RayObject buffer(std::make_shared(data, sizeof(data)), nullptr, {}); auto publisher = std::make_shared(); - auto subscriber = std::make_shared(); + auto subscriber = std::make_shared(); auto rc = std::make_shared( rpc::Address(), publisher.get(), diff --git a/src/ray/core_worker/tests/task_manager_test.cc b/src/ray/core_worker/tests/task_manager_test.cc index 826f4d3511c4..81eee6f6ca8b 100644 --- a/src/ray/core_worker/tests/task_manager_test.cc +++ b/src/ray/core_worker/tests/task_manager_test.cc @@ -20,11 +20,11 @@ #include #include +#include "fakes/ray/pubsub/subscriber.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include "mock/ray/gcs/gcs_client/gcs_client.h" #include "mock/ray/pubsub/publisher.h" -#include "mock/ray/pubsub/subscriber.h" #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" #include "ray/common/test_util.h" @@ -143,7 +143,7 @@ class TaskManagerTest : public ::testing::Test { : lineage_pinning_enabled_(lineage_pinning_enabled), addr_(GetRandomWorkerAddr()), publisher_(std::make_shared()), - subscriber_(std::make_shared()), + subscriber_(std::make_shared()), task_event_buffer_mock_(std::make_unique()), mock_gcs_client_(std::make_shared()), reference_counter_(std::make_shared( @@ -215,7 +215,7 @@ class TaskManagerTest : public ::testing::Test { bool did_queue_generator_resubmit_ = false; rpc::Address addr_; std::shared_ptr publisher_; - std::shared_ptr subscriber_; + std::shared_ptr subscriber_; std::unique_ptr task_event_buffer_mock_; std::shared_ptr mock_gcs_client_; std::shared_ptr reference_counter_; diff --git a/src/ray/gcs/gcs_client/gcs_client.cc b/src/ray/gcs/gcs_client/gcs_client.cc index 6f9623ea5d4e..b5d3b1f2bf4f 100644 --- a/src/ray/gcs/gcs_client/gcs_client.cc +++ b/src/ray/gcs/gcs_client/gcs_client.cc @@ -38,8 +38,6 @@ class GcsSubscriberClient final : public pubsub::SubscriberClientInterface { explicit GcsSubscriberClient(const std::shared_ptr &rpc_client) : rpc_client_(rpc_client) {} - ~GcsSubscriberClient() final = default; - void PubsubLongPolling( const rpc::PubsubLongPollingRequest &request, const rpc::ClientCallback &callback) final; @@ -48,8 +46,6 @@ class GcsSubscriberClient final : public pubsub::SubscriberClientInterface { const rpc::PubsubCommandBatchRequest &request, const rpc::ClientCallback &callback) final; - std::string DebugString() const final { return ""; } - private: const std::shared_ptr rpc_client_; }; diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index 92254188d8d8..60b092b2778a 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -131,7 +131,6 @@ ray_cc_test( ":gcs_server_test_util", "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", - "//src/mock/ray/pubsub:mock_subscriber", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc index 33b80d640bf4..c4cd529da5df 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc @@ -16,19 +16,15 @@ #include #include -#include "ray/gcs/gcs_server/gcs_job_manager.h" - -// clang-format off #include "gtest/gtest.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" -#include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/tests/gcs_test_util.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/pubsub/publisher.h" #include "mock/ray/rpc/worker/core_worker_client.h" - -// clang-format on +#include "ray/gcs/gcs_server/gcs_job_manager.h" +#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/store_client/in_memory_store_client.h" +#include "ray/gcs/tests/gcs_test_util.h" using json = nlohmann::json; diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc index af01302aef3b..c0333a646c0b 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc @@ -15,23 +15,22 @@ #include #include #include -// clang-format off -#include "gtest/gtest.h" + #include "gmock/gmock.h" -#include "ray/gcs/gcs_server/gcs_actor_manager.h" -#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" -#include "mock/ray/gcs/store_client/store_client.h" +#include "gtest/gtest.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" +#include "mock/ray/gcs/store_client/store_client.h" #include "mock/ray/raylet_client/raylet_client.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/test_util.h" -// clang-format on +#include "ray/gcs/gcs_server/gcs_actor_manager.h" +#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" using namespace ::testing; // NOLINT namespace ray { -using raylet::NoopLocalTaskManager; namespace gcs { + struct MockCallback { MOCK_METHOD(void, Call, ((std::shared_ptr))); void operator()(std::shared_ptr a) { return Call(a); } @@ -176,5 +175,6 @@ TEST_F(GcsActorSchedulerMockTest, KillWorkerLeak2) { actor_scheduler->CancelOnWorker(node_id, worker_id); push_normal_task_cb(Status::OK(), rpc::PushTaskReply()); } + } // namespace gcs } // namespace ray diff --git a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc index 5f75f531f38b..5f3f587c17b6 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc @@ -12,30 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -// clang-format off -#include -#include -#include +#include "ray/gcs/gcs_server/gcs_autoscaler_state_manager.h" + #include +#include #include +#include #include -#include +#include +#include #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "mock/ray/gcs/gcs_server/gcs_actor_manager.h" +#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" +#include "mock/ray/gcs/gcs_server/gcs_placement_group_mgr.h" +#include "mock/ray/gcs/store_client/store_client.h" +#include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/gcs/gcs_server/store_client_kv.h" #include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/tests/gcs_test_util.h" -#include "ray/gcs/gcs_server/store_client_kv.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" -#include "mock/ray/gcs/gcs_server/gcs_placement_group_mgr.h" -#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" -#include "mock/ray/gcs/gcs_server/gcs_actor_manager.h" -#include "mock/ray/gcs/store_client/store_client.h" -#include "mock/ray/rpc/worker/core_worker_client.h" - -#include "ray/gcs/gcs_server/gcs_autoscaler_state_manager.h" -// clang-format on namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc index f21a5f4a926c..b41e4d727e8f 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc @@ -17,18 +17,14 @@ #include #include -// clang-format off #include "gtest/gtest.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" -#include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/tests/gcs_test_util.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/pubsub/publisher.h" -#include "mock/ray/pubsub/subscriber.h" #include "mock/ray/rpc/worker/core_worker_client.h" - -// clang-format on +#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/store_client/in_memory_store_client.h" +#include "ray/gcs/tests/gcs_test_util.h" namespace ray { diff --git a/src/ray/gcs/pubsub/gcs_pub_sub.cc b/src/ray/gcs/pubsub/gcs_pub_sub.cc index 354775796205..976779ec6c94 100644 --- a/src/ray/gcs/pubsub/gcs_pub_sub.cc +++ b/src/ray/gcs/pubsub/gcs_pub_sub.cc @@ -79,18 +79,18 @@ Status GcsSubscriber::SubscribeAllJobs( auto subscription_failure_callback = [](const std::string &, const Status &status) { RAY_LOG(WARNING) << "Subscription to Job channel failed: " << status.ToString(); }; - // Ignore if the subscription already exists, because the resubscription is intentional. - RAY_UNUSED(subscriber_->SubscribeChannel( + subscriber_->Subscribe( std::make_unique(), rpc::ChannelType::GCS_JOB_CHANNEL, gcs_address_, + /*key_id=*/std::nullopt, [done](const Status &status) { if (done != nullptr) { done(status); } }, std::move(subscribe_item_callback), - std::move(subscription_failure_callback))); + std::move(subscription_failure_callback)); return Status::OK(); } @@ -110,19 +110,18 @@ Status GcsSubscriber::SubscribeActor( RAY_LOG(WARNING) << "Subscription to Actor " << id.Hex() << " failed: " << status.ToString(); }; - // Ignore if the subscription already exists, because the resubscription is intentional. - RAY_UNUSED(subscriber_->Subscribe( + subscriber_->Subscribe( std::make_unique(), rpc::ChannelType::GCS_ACTOR_CHANNEL, gcs_address_, - id.Binary(), + /*key_id=*/id.Binary(), [done](const Status &status) { if (done != nullptr) { done(status); } }, std::move(subscription_callback), - std::move(subscription_failure_callback))); + std::move(subscription_failure_callback)); return Status::OK(); } @@ -147,18 +146,18 @@ void GcsSubscriber::SubscribeAllNodeInfo(const ItemCallback &s auto subscription_failure_callback = [](const std::string &, const Status &status) { RAY_LOG(WARNING) << "Subscription to NodeInfo channel failed: " << status.ToString(); }; - // Ignore if the subscription already exists, because the resubscription is intentional. - RAY_UNUSED(subscriber_->SubscribeChannel( + subscriber_->Subscribe( std::make_unique(), rpc::ChannelType::GCS_NODE_INFO_CHANNEL, gcs_address_, + /*key_id=*/std::nullopt, [done](const Status &status) { if (done != nullptr) { done(status); } }, std::move(subscribe_item_callback), - std::move(subscription_failure_callback))); + std::move(subscription_failure_callback)); } Status GcsSubscriber::SubscribeAllWorkerFailures( @@ -172,10 +171,11 @@ Status GcsSubscriber::SubscribeAllWorkerFailures( << status.ToString(); }; // Ignore if the subscription already exists, because the resubscription is intentional. - RAY_UNUSED(subscriber_->SubscribeChannel( + subscriber_->Subscribe( std::make_unique(), rpc::ChannelType::GCS_WORKER_DELTA_CHANNEL, gcs_address_, + /*key_id=*/std::nullopt, /*subscribe_done_callback=*/ [done](const Status &status) { if (done != nullptr) { @@ -183,7 +183,7 @@ Status GcsSubscriber::SubscribeAllWorkerFailures( } }, std::move(subscribe_item_callback), - std::move(subscription_failure_callback))); + std::move(subscription_failure_callback)); return Status::OK(); } @@ -216,9 +216,9 @@ Status PythonGcsSubscriber::Subscribe() { rpc::GcsSubscriberCommandBatchRequest request; request.set_subscriber_id(subscriber_id_); request.set_sender_id(worker_id_); - auto *cmd = request.add_commands(); - cmd->set_channel_type(channel_type_); - cmd->mutable_subscribe_message(); + auto *command = request.add_commands(); + command->set_channel_type(channel_type_); + command->mutable_subscribe_message(); rpc::GcsSubscriberCommandBatchReply reply; grpc::Status status = diff --git a/src/ray/object_manager/ownership_object_directory.cc b/src/ray/object_manager/ownership_object_directory.cc index a7046463e700..3e3d958f5a17 100644 --- a/src/ray/object_manager/ownership_object_directory.cc +++ b/src/ray/object_manager/ownership_object_directory.cc @@ -370,14 +370,14 @@ ray::Status OwnershipBasedObjectDirectory::SubscribeObjectLocations( auto sub_message = std::make_unique(); sub_message->mutable_worker_object_locations_message()->Swap(request.get()); - RAY_CHECK(object_location_subscriber_->Subscribe( + object_location_subscriber_->Subscribe( std::move(sub_message), rpc::ChannelType::WORKER_OBJECT_LOCATIONS_CHANNEL, owner_address, object_id.Binary(), /*subscribe_done_callback=*/nullptr, /*Success callback=*/msg_published_callback, - /*Failure callback=*/failure_callback)); + /*Failure callback=*/failure_callback); auto location_state = LocationListenerState(); location_state.owner_address = owner_address; diff --git a/src/ray/object_manager/tests/BUILD.bazel b/src/ray/object_manager/tests/BUILD.bazel index 9d56d86614e6..c59944be6ddc 100644 --- a/src/ray/object_manager/tests/BUILD.bazel +++ b/src/ray/object_manager/tests/BUILD.bazel @@ -36,7 +36,7 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - "//src/mock/ray/pubsub:mock_subscriber", + "//:ray_fakes", "//src/ray/object_manager:ownership_object_directory", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/object_manager/tests/ownership_object_directory_test.cc b/src/ray/object_manager/tests/ownership_object_directory_test.cc index 9fc8964fdfb7..4156ed167954 100644 --- a/src/ray/object_manager/tests/ownership_object_directory_test.cc +++ b/src/ray/object_manager/tests/ownership_object_directory_test.cc @@ -21,9 +21,9 @@ #include #include +#include "fakes/ray/pubsub/subscriber.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "mock/ray/pubsub/subscriber.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/status.h" #include "ray/gcs/gcs_client/accessor.h" @@ -126,7 +126,7 @@ class OwnershipBasedObjectDirectoryTest : public ::testing::Test { /*fetch_cluster_id_if_nil=*/false), gcs_client_mock_( new MockGcsClient(options_, std::make_unique())), - subscriber_(std::make_shared()), + subscriber_(std::make_shared()), owner_client(std::make_shared()), client_pool([&](const rpc::Address &addr) { return owner_client; }) { RayConfig::instance().initialize(R"({"max_object_report_batch_size": 20})"); @@ -199,7 +199,7 @@ class OwnershipBasedObjectDirectoryTest : public ::testing::Test { instrumented_io_context io_service_; gcs::GcsClientOptions options_; std::shared_ptr gcs_client_mock_; - std::shared_ptr subscriber_; + std::shared_ptr subscriber_; std::shared_ptr owner_client; rpc::CoreWorkerClientPool client_pool; std::unique_ptr obod_; @@ -486,7 +486,6 @@ TEST_F(OwnershipBasedObjectDirectoryTest, TestNotifyOnUpdate) { UniqueID callback_id = UniqueID::FromRandom(); ObjectID obj_id = ObjectID::FromRandom(); int num_callbacks = 0; - EXPECT_CALL(*subscriber_, Subscribe(_, _, _, _, _, _, _)).WillOnce(Return(true)); ASSERT_TRUE( obod_ ->SubscribeObjectLocations(callback_id, diff --git a/src/ray/protobuf/pubsub.proto b/src/ray/protobuf/pubsub.proto index 89d3ab55a76e..eba420d2097e 100644 --- a/src/ray/protobuf/pubsub.proto +++ b/src/ray/protobuf/pubsub.proto @@ -24,7 +24,6 @@ import "src/ray/protobuf/logging.proto"; /// For example, for pubsub channels that are used by core workers, /// they have the prefix WORKER_. enum ChannelType { - reserved 9; /// A channel for object eviction. WORKER_OBJECT_EVICTION = 0; /// A channel for ref removed. @@ -44,7 +43,7 @@ enum ChannelType { /// A channel for logs from various Ray components. RAY_LOG_CHANNEL = 8; /// A channel for reporting node resource usage stats. - RAY_NODE_RESOURCE_USAGE_CHANNEL = 10; + RAY_NODE_RESOURCE_USAGE_CHANNEL = 9; } /// @@ -52,7 +51,6 @@ enum ChannelType { /// message PubMessage { - reserved 10, 14; /// Channel type for this publish message. ChannelType channel_type = 1; /// The key id (e.g., object id) in bytes. @@ -62,19 +60,17 @@ message PubMessage { WorkerObjectEvictionMessage worker_object_eviction_message = 3; WorkerRefRemovedMessage worker_ref_removed_message = 4; WorkerObjectLocationsPubMessage worker_object_locations_message = 5; + FailureMessage failure_message = 6; ActorTableData actor_message = 7; JobTableData job_message = 8; GcsNodeInfo node_info_message = 9; - WorkerDeltaData worker_delta_message = 11; - ErrorTableData error_info_message = 12; - LogBatch log_batch_message = 13; - NodeResourceUsage node_resource_usage_message = 15; - - // The message that indicates the given key id is not available anymore. - FailureMessage failure_message = 6; + WorkerDeltaData worker_delta_message = 10; + ErrorTableData error_info_message = 11; + LogBatch log_batch_message = 12; + NodeResourceUsage node_resource_usage_message = 13; } /// A monotonically increasing sequence_id generated by the publisher. - int64 sequence_id = 16; + int64 sequence_id = 14; } message WorkerObjectEvictionMessage { @@ -117,8 +113,7 @@ message WorkerObjectLocationsPubMessage { } /// Indicating the subscriber needs to handle failure callback. -message FailureMessage { -} +message FailureMessage {} /// /// Subscribe @@ -141,8 +136,7 @@ message Command { } } -message UnsubscribeMessage { -} +message UnsubscribeMessage {} /// Each of subscribe command needs to include request body because in Ray's pubsub /// module, it doesn't subscribe the same data structure (like for Redis, @@ -225,8 +219,7 @@ message PubsubCommandBatchRequest { repeated Command commands = 2; } -message PubsubCommandBatchReply { -} +message PubsubCommandBatchReply {} service SubscriberService { /// The long polling request sent to the publisher for pubsub operations. diff --git a/src/ray/pubsub/README.md b/src/ray/pubsub/README.md index d57793b7e901..75d32cead1f3 100644 --- a/src/ray/pubsub/README.md +++ b/src/ray/pubsub/README.md @@ -1,7 +1,7 @@ # Pubsub module -The doc is written on June 9th 2021. The implementation can be changed in any -time, and the documentation could be out of date. +This doc has last been updated on Aug 19, 2025. This doc should be updated +as the implementation changes. ## Motivation @@ -31,6 +31,9 @@ situation. - Publisher: A process that publishes messages to subscribers. - Subscriber: A process that subscribes channels from publishers. - Channel: Equivalent to topic in Kafka. +- Key/Entity: A specific item you care about in the channel. E.g. in + the actor channel, you only care about a specific actor id so that's + the key you subscribe to. Not all channels have keys you can subscribe by. - Command: Equivalent to Redis pubsub's command. E.g., Subscribe / Unsubscribe. ## Features @@ -45,36 +48,90 @@ situation. subscribers. - Subscriber failure detection. The subscriber failure is tracked by publishers. -- The module is general and can be used in arbitrary two core ray components. +- The module is general and can be used in two arbitrary Ray components. ## Limitation -- If messages are published before it is subscribed from the publisher, they - are lost. -- It doesn't handle the fault tolerance by design because raylet -> core_worker - (the most common use case) doesn't require it. The fault tolerance needs to - be implemented in the higher layer. +If messages are published before a subscription, they're lost. ## Implementation -The pubsub module doesn't have a broker like traditional pubsub systems because -there's no use case. In the pubsub module, all publishers are also brokers. The -performance, especially a throughput is not a requirement when developed, and -the module is not designed for high throughput. +In this pubsub implementation, publishers directly send messages to subscribers. +There are no intermediary brokers. The performance, especially throughput +wasn't a requirement when developed, and therefore the module isn't designed +for high throughput. ### Basic mechanism -Between the publisher and subscriber, there's only 1 long-polling connection. -The long polling connection is initiated from the subscriber when there are -subscribing entries from the publisher. Whenever publisher publishes messages, -they are batched to the reply of the long polling request in FIFO order. +#### PubsubCommandBatch +A command is an operation from a subscriber to publisher. Subscribe and +Unsubscribe are the only commands. Commands are served by `PubsubCommandBatch`, +which batches them in the FIFO order. We limit to it one in-flight `PubsubCommandBatchRequest` +at a time to prevent out of order subscribes / unsubscribes. Because of this, +we have to queue up commands and therefore have to batch commands when sending them. + +#### PubsubLongPolling +Between the publisher and subscriber, there's only 1 long-polling connection +(only one in-flight request), no matter how many separate channels / keys the +subscriber is subscribed to. The subscriber will always have an in-flight +`PubsubLongPollingRequest` as long as it's subscribed to something. Whenever a +publisher publishes messages to that subscriber, they're batched to the reply +of the long polling request in FIFO order. + +### Pubsub Code Flow +Breakdown of the pubsub flow from the subscriber and publisher +Note that this section ignores fault tolerance. + +#### Subscriber Actions + +1. **On a Subscribe call** + - Sends a `PubsubCommandBatchRequest` with its own `subscriber_id` and a `SubMessage` + Command containing `channel_type` and optionally `key_id` + - Sends a `PubsubLongPollingRequest` with its own `subscriber_id` + +2. **Subscribe done** + - Receives `PubsubCommandBatchReply` and runs a callback if provided on subscribe + - Sends new commands to publisher if they've been queued up, e.g. another subscribe to + something else or an unsubscribe to something + - Only allows one in-flight `PubsubCommandBatchRequest` to ensure command ordering + +3. **Message Processing** + - Receives reply to `PubsubLongPollingRequest` and processes published messages + - Sends another `PubsubLongPollingRequest` if subscription still exists + +4. **Unsubscribe** + - Sends `PubsubCommandBatchRequest` with `UnsubscribeMessage` when unsubscribing + +#### Publisher Actions + +1. **Subscribe Handling** + - Receives `PubsubCommandBatchRequest` and creates a `SubscriberState` for the + subscriber if it doesn't exist + - Registers subscription for the given channel + key by setting up a relation between + an `EntityState` and a `SubscriberState` + - Note that the publisher maintains a `SubscriptionIndex` for each channel, and each + `SubscriptionIndex` holds `EntityState` objects for each key. Each `EntityState` + holds `SubscriberState` pointers to send / queue up messages to send. There's a + special `EntityState` in every `SubscriptionIndex` for "subscribing to all" + +2. **Initial Long Polling Request** + - Receives `PubsubLongPollingRequest` and creates `SubscriberState` if it doesn't exist. + Note that the `SubscriberState` might not exist because the initial `PubsubLongPollingRequest` + could arrive before the associated `PubsubCommandBatchRequest`. + - Creates a `LongPollConnection` in the `SubscriberState` to store the reply + reply callback + - Attempts to publish by replying to the request if mailbox already contains messages + - If mailbox is empty, waits until next relevant publish to reply and send the publish + +3. **Subsequent Long Polling** + - Receives a subsequent `PubsubLongPollingRequest` from the subscriber and checks mailbox + - Publishes messages if mailbox isn't empty, or waits for relevant publish to reply + +4. **Unsubscribe** + - Receives unsubscribe command and unregisters `SubscriberState` from the appropriate + `EntityState` + - Erases the `EntityState` if it no longer contains any `SubscriberState` pointers + - Periodically cleans up "Dead" `SubscriberState`'s -### Commands - -A command is an operation from a subscriber to publisher. For example, -Subscribe or Unsubscribe could be a command. Commands are served by a separate -RPC, which also batches them in the FIFO order. Subscriber keeps sending -commands until they are not queued. There's no backpressure mechanism here. ### Fault detection @@ -93,4 +150,4 @@ as there are subscribing entries from them. If subscribers are failed, they are not sending any more long polling requests. Publishers refreshes the long polling request every 30 seconds to check if the subscriber is still alive. If the subscriber doesn't initiate a long polling request for more than certain -threshold, subscriber is condiered to be failed and all metadata is cleaned up. +threshold, the subscriber is considered failed and all metadata is cleaned up. diff --git a/src/ray/pubsub/subscriber.cc b/src/ray/pubsub/subscriber.cc index c23b6ecb47fc..792cf9fd403a 100644 --- a/src/ray/pubsub/subscriber.cc +++ b/src/ray/pubsub/subscriber.cc @@ -30,7 +30,7 @@ const UniqueID kDefaultUniqueID{}; /// SubscriberChannel /////////////////////////////////////////////////////////////////////////////// -bool SubscriberChannel::Subscribe( +void SubscriberChannel::Subscribe( const rpc::Address &publisher_address, const std::optional &key_id, SubscriptionItemCallback subscription_callback, @@ -39,21 +39,18 @@ bool SubscriberChannel::Subscribe( const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); if (key_id) { - return subscription_map_[publisher_id] - .per_entity_subscription - .try_emplace(*key_id, - SubscriptionInfo(std::move(subscription_callback), - std::move(subscription_failure_callback))) - .second; + subscription_map_[publisher_id].per_entity_subscription.try_emplace( + *key_id, + SubscriptionInfo(std::move(subscription_callback), + std::move(subscription_failure_callback))); + return; } auto &all_entities_subscription = subscription_map_[publisher_id].all_entities_subscription; - if (all_entities_subscription != nullptr) { - return false; + if (all_entities_subscription == nullptr) { + all_entities_subscription = std::make_unique( + std::move(subscription_callback), std::move(subscription_failure_callback)); } - all_entities_subscription = std::make_unique( - std::move(subscription_callback), std::move(subscription_failure_callback)); - return true; } bool SubscriberChannel::Unsubscribe(const rpc::Address &publisher_address, @@ -232,50 +229,15 @@ std::string SubscriberChannel::DebugString() const { /// Subscriber /////////////////////////////////////////////////////////////////////////////// -Subscriber::~Subscriber() { - // TODO(mwtian): flush Subscriber and ensure there is no leak during destruction. - // TODO(ryw): Remove this subscriber from the service by GcsUnregisterSubscriber. -} - -bool Subscriber::Subscribe(std::unique_ptr sub_message, - const rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - const std::string &key_id, - SubscribeDoneCallback subscribe_done_callback, - SubscriptionItemCallback subscription_callback, - SubscriptionFailureCallback subscription_failure_callback) { - return SubscribeInternal(std::move(sub_message), - channel_type, - publisher_address, - key_id, - std::move(subscribe_done_callback), - std::move(subscription_callback), - std::move(subscription_failure_callback)); -} - -bool Subscriber::SubscribeChannel( - std::unique_ptr sub_message, - const rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - SubscribeDoneCallback subscribe_done_callback, - SubscriptionItemCallback subscription_callback, - SubscriptionFailureCallback subscription_failure_callback) { - return SubscribeInternal(std::move(sub_message), - channel_type, - publisher_address, - std::nullopt, - std::move(subscribe_done_callback), - std::move(subscription_callback), - std::move(subscription_failure_callback)); -} - -bool Subscriber::Unsubscribe(const rpc::ChannelType channel_type, +bool Subscriber::Unsubscribe(rpc::ChannelType channel_type, const rpc::Address &publisher_address, - const std::string &key_id) { + const std::optional &key_id) { // Batch the unsubscribe command. auto command = std::make_unique(); command->cmd.set_channel_type(channel_type); - command->cmd.set_key_id(key_id); + if (key_id.has_value()) { + command->cmd.set_key_id(*key_id); + } command->cmd.mutable_unsubscribe_message(); absl::MutexLock lock(&mutex_); @@ -286,22 +248,7 @@ bool Subscriber::Unsubscribe(const rpc::ChannelType channel_type, return Channel(channel_type)->Unsubscribe(publisher_address, key_id); } -bool Subscriber::UnsubscribeChannel(const rpc::ChannelType channel_type, - const rpc::Address &publisher_address) { - // Batch the unsubscribe command. - auto command = std::make_unique(); - command->cmd.set_channel_type(channel_type); - command->cmd.mutable_unsubscribe_message(); - - absl::MutexLock lock(&mutex_); - const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); - commands_[publisher_id].emplace(std::move(command)); - SendCommandBatchIfPossible(publisher_address); - - return Channel(channel_type)->Unsubscribe(publisher_address, std::nullopt); -} - -bool Subscriber::IsSubscribed(const rpc::ChannelType channel_type, +bool Subscriber::IsSubscribed(rpc::ChannelType channel_type, const rpc::Address &publisher_address, const std::string &key_id) const { absl::MutexLock lock(&mutex_); @@ -312,18 +259,17 @@ bool Subscriber::IsSubscribed(const rpc::ChannelType channel_type, return channel->IsSubscribed(publisher_address, key_id); } -bool Subscriber::SubscribeInternal( - std::unique_ptr sub_message, - const rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - const std::optional &key_id, - SubscribeDoneCallback subscribe_done_callback, - SubscriptionItemCallback subscription_callback, - SubscriptionFailureCallback subscription_failure_callback) { +void Subscriber::Subscribe(std::unique_ptr sub_message, + rpc::ChannelType channel_type, + const rpc::Address &publisher_address, + const std::optional &key_id, + SubscribeDoneCallback subscribe_done_callback, + SubscriptionItemCallback subscription_callback, + SubscriptionFailureCallback subscription_failure_callback) { // Batch a subscribe command. auto command = std::make_unique(); command->cmd.set_channel_type(channel_type); - if (key_id) { + if (key_id.has_value()) { command->cmd.set_key_id(*key_id); } if (sub_message != nullptr) { @@ -336,7 +282,7 @@ bool Subscriber::SubscribeInternal( commands_[publisher_id].emplace(std::move(command)); SendCommandBatchIfPossible(publisher_address); MakeLongPollingConnectionIfNotConnected(publisher_address); - return Channel(channel_type) + this->Channel(channel_type) ->Subscribe(publisher_address, key_id, std::move(subscription_callback), @@ -364,7 +310,8 @@ void Subscriber::MakeLongPollingPubsubConnection(const rpc::Address &publisher_a long_polling_request.set_max_processed_sequence_id(processed_state.second); subscriber_client->PubsubLongPolling( long_polling_request, - [this, publisher_address](Status status, rpc::PubsubLongPollingReply &&reply) { + [this, publisher_address](const Status &status, + rpc::PubsubLongPollingReply &&reply) { absl::MutexLock lock(&mutex_); HandleLongPollingResponse(publisher_address, status, std::move(reply)); }); @@ -375,7 +322,6 @@ void Subscriber::HandleLongPollingResponse(const rpc::Address &publisher_address rpc::PubsubLongPollingReply &&reply) { const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); RAY_LOG(DEBUG) << "Long polling request has been replied from " << publisher_id; - RAY_CHECK(publishers_connected_.count(publisher_id)); if (!status.ok()) { // If status is not okay, we treat that the publisher is dead. @@ -398,8 +344,7 @@ void Subscriber::HandleLongPollingResponse(const rpc::Address &publisher_address << processed_sequences_[publisher_id].first << ", this can only happen when gcs failsover."; } - // reset publisher_id and processed_sequence - // if the publisher_id changes. + // reset publisher_id and processed_sequence if the publisher_id changes. processed_sequences_[publisher_id].first = reply_publisher_id; processed_sequences_[publisher_id].second = 0; } @@ -501,8 +446,8 @@ void Subscriber::SendCommandBatchIfPossible(const rpc::Address &publisher_addres // This means the publisher has failed. // The publisher dead detection & command clean up will be done // from the long polling request. - RAY_LOG(DEBUG) << "The command batch request to " << publisher_id - << " has failed"; + RAY_LOG(WARNING) << "The command batch request to " << publisher_id + << " has failed"; } { absl::MutexLock lock(&mutex_); diff --git a/src/ray/pubsub/subscriber.h b/src/ray/pubsub/subscriber.h index 0bdcdcb2674e..584c5a849452 100644 --- a/src/ray/pubsub/subscriber.h +++ b/src/ray/pubsub/subscriber.h @@ -70,11 +70,11 @@ class SubscriberChannel { /// /// \param publisher_address Address of the publisher to subscribe the object. /// \param message id The message id to subscribe from the publisher. - /// \param subscription_callback A callback that is invoked whenever the given object - /// information is published. - /// \param subscription_failure_callback A callback that is - /// invoked whenever the publisher is dead (or failed). - bool Subscribe(const rpc::Address &publisher_address, + /// \param subscription_item_callback A callback that is invoked whenever the given + /// object information is published. + /// \param subscription_failure_callback A callback that is invoked whenever the + /// publisher is dead (or failed). + void Subscribe(const rpc::Address &publisher_address, const std::optional &key_id, SubscriptionItemCallback subscription_item_callback, SubscriptionFailureCallback subscription_failure_callback); @@ -129,7 +129,7 @@ class SubscriberChannel { } /// Return the channel type of this subscribe channel. - const rpc::ChannelType GetChannelType() const { return channel_type_; } + rpc::ChannelType GetChannelType() const { return channel_type_; } /// Return the statistics of the specific channel. std::string DebugString() const; @@ -149,14 +149,14 @@ class SubscriberChannel { const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); auto subscription_it = subscription_map_.find(publisher_id); if (subscription_it == subscription_map_.end()) { - return absl::nullopt; + return std::nullopt; } if (subscription_it->second.all_entities_subscription != nullptr) { return subscription_it->second.all_entities_subscription->item_cb; } auto callback_it = subscription_it->second.per_entity_subscription.find(key_id); if (callback_it == subscription_it->second.per_entity_subscription.end()) { - return absl::nullopt; + return std::nullopt; } return callback_it->second.item_cb; } @@ -168,14 +168,14 @@ class SubscriberChannel { const auto publisher_id = UniqueID::FromBinary(publisher_address.worker_id()); auto subscription_it = subscription_map_.find(publisher_id); if (subscription_it == subscription_map_.end()) { - return absl::nullopt; + return std::nullopt; } if (subscription_it->second.all_entities_subscription != nullptr) { return subscription_it->second.all_entities_subscription->failure_cb; } auto callback_it = subscription_it->second.per_entity_subscription.find(key_id); if (callback_it == subscription_it->second.per_entity_subscription.end()) { - return absl::nullopt; + return std::nullopt; } return callback_it->second.failure_cb; } @@ -223,39 +223,26 @@ class Subscriber : public SubscriberInterface { instrumented_io_context *callback_service) : subscriber_id_(subscriber_id), max_command_batch_size_(max_command_batch_size), - get_client_(get_client) { + get_client_(std::move(get_client)) { for (auto type : channels) { channels_.emplace(type, std::make_unique(type, callback_service)); } } - ~Subscriber(); - - bool Subscribe(std::unique_ptr sub_message, - const rpc::ChannelType channel_type, + void Subscribe(std::unique_ptr sub_message, + rpc::ChannelType channel_type, const rpc::Address &publisher_address, - const std::string &key_id, + const std::optional &key_id, SubscribeDoneCallback subscribe_done_callback, SubscriptionItemCallback subscription_callback, SubscriptionFailureCallback subscription_failure_callback) override; - bool SubscribeChannel( - std::unique_ptr sub_message, - rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - SubscribeDoneCallback subscribe_done_callback, - SubscriptionItemCallback subscription_callback, - SubscriptionFailureCallback subscription_failure_callback) override; - - bool Unsubscribe(const rpc::ChannelType channel_type, + bool Unsubscribe(rpc::ChannelType channel_type, const rpc::Address &publisher_address, - const std::string &key_id) override; - - bool UnsubscribeChannel(const rpc::ChannelType channel_type, - const rpc::Address &publisher_address) override; + const std::optional &key_id) override; - bool IsSubscribed(const rpc::ChannelType channel_type, + bool IsSubscribed(rpc::ChannelType channel_type, const rpc::Address &publisher_address, const std::string &key_id) const override; @@ -290,18 +277,6 @@ class Subscriber : public SubscriberInterface { // Testing only. Check if there are leaks. bool CheckNoLeaks() const ABSL_LOCKS_EXCLUDED(mutex_); - /// - /// Private fields - /// - - bool SubscribeInternal(std::unique_ptr sub_message, - const rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - const std::optional &key_id, - SubscribeDoneCallback subscribe_done_callback, - SubscriptionItemCallback subscription_callback, - SubscriptionFailureCallback subscription_failure_callback); - /// Create a long polling connection to the publisher for receiving the published /// messages. /// NOTE(sang): Note that the subscriber needs to "ensure" that the long polling diff --git a/src/ray/pubsub/subscriber_interface.h b/src/ray/pubsub/subscriber_interface.h index 42d8fbfeac8c..86db9130410e 100644 --- a/src/ray/pubsub/subscriber_interface.h +++ b/src/ray/pubsub/subscriber_interface.h @@ -39,10 +39,10 @@ class SubscriberInterface { public: /// There are two modes of subscriptions. Each channel can only be subscribed in one /// mode, i.e. - /// - Calling Subscribe() to subscribe to one or more entities in a channel - /// - Calling SubscribeChannel() once to subscribe to all entities in a channel - /// It is an error to call both Subscribe() and SubscribeChannel() on the same channel - /// type. This restriction can be relaxed later, if there is a use case. + /// - Calling Subscribe() to subscribe to one or more entities in a channel. + /// - Calling Subscribe() once to subscribe to all entities in a channel. + /// NOTE: It is an error to call both Subscribe() to all entities and then only + /// subscribe to one entity on the same channel type. /// Subscribe to entity key_id in channel channel_type. /// NOTE(sang): All the callbacks could be executed in a different thread from a caller. @@ -51,72 +51,45 @@ class SubscriberInterface { /// \param sub_message The subscription message. /// \param channel_type The channel to subscribe to. /// \param publisher_address Address of the publisher to subscribe the object. - /// \param key_id The entity id to subscribe from the publisher. + /// \param key_id The entity id to subscribe from the publisher. Subscribes to all + /// entities if nullopt. /// \param subscription_callback A callback that is invoked whenever the given entity /// information is received by the subscriber. /// \param subscription_failure_callback A callback that is invoked whenever the /// connection to publisher is broken (e.g. the publisher fails). - /// \return True if inserted, false if the key already exists and this becomes a no-op. - virtual bool Subscribe(std::unique_ptr sub_message, + virtual void Subscribe(std::unique_ptr sub_message, rpc::ChannelType channel_type, const rpc::Address &publisher_address, - const std::string &key_id, + const std::optional &key_id, SubscribeDoneCallback subscribe_done_callback, SubscriptionItemCallback subscription_callback, SubscriptionFailureCallback subscription_failure_callback) = 0; - /// Subscribe to all entities in channel channel_type. - /// - /// \param sub_message The subscription message. - /// \param channel_type The channel to subscribe to. - /// \param publisher_address Address of the publisher to subscribe the object. - /// \param subscription_callback A callback that is invoked whenever an entity - /// information is received by the subscriber. - /// \param subscription_failure_callback A callback that is invoked whenever the - /// connection to publisher is broken (e.g. the publisher fails). - /// \return True if inserted, false if the channel is already subscribed and this - /// becomes a no-op. - virtual bool SubscribeChannel( - std::unique_ptr sub_message, - rpc::ChannelType channel_type, - const rpc::Address &publisher_address, - SubscribeDoneCallback subscribe_done_callback, - SubscriptionItemCallback subscription_callback, - SubscriptionFailureCallback subscription_failure_callback) = 0; - /// Unsubscribe the entity if the entity has been subscribed with Subscribe(). /// NOTE: Calling this method inside subscription_failure_callback is not allowed. /// /// \param channel_type The channel to unsubscribe from. /// \param publisher_address The publisher address that it will unsubscribe from. - /// \param key_id The entity id to unsubscribe. + /// \param key_id The entity id to unsubscribe. Unsubscribes from all entities if + /// nullopt. /// \return Returns whether the entity key_id has been subscribed before. - virtual bool Unsubscribe(const rpc::ChannelType channel_type, + virtual bool Unsubscribe(rpc::ChannelType channel_type, const rpc::Address &publisher_address, - const std::string &key_id) = 0; - - /// Unsubscribe from the channel_type. Must be paired with SubscribeChannel(). - /// NOTE: Calling this method inside subscription_failure_callback is not allowed. - /// - /// \param channel_type The channel to unsubscribe from. - /// \param publisher_address The publisher address that it will unsubscribe from. - /// \return Returns whether the entity key_id has been subscribed before. - virtual bool UnsubscribeChannel(const rpc::ChannelType channel_type, - const rpc::Address &publisher_address) = 0; + const std::optional &key_id) = 0; /// Test only. /// Checks if the entity key_id is being subscribed to specifically. - /// Does not consider if SubscribeChannel() has been called on the channel. + /// Does not consider if the subscriber is subscribed to all entities in a channel. /// /// \param publisher_address The publisher address to check. /// \param key_id The entity id to check. - virtual bool IsSubscribed(const rpc::ChannelType channel_type, + virtual bool IsSubscribed(rpc::ChannelType channel_type, const rpc::Address &publisher_address, const std::string &key_id) const = 0; virtual std::string DebugString() const = 0; - virtual ~SubscriberInterface() {} + virtual ~SubscriberInterface() = default; }; /// Interface for the client used by a subscriber. @@ -132,8 +105,6 @@ class SubscriberClientInterface { const rpc::PubsubCommandBatchRequest &request, const rpc::ClientCallback &callback) = 0; - virtual std::string DebugString() const = 0; - virtual ~SubscriberClientInterface() = default; }; diff --git a/src/ray/pubsub/tests/BUILD.bazel b/src/ray/pubsub/tests/BUILD.bazel index 0f19f2b7f356..f1879f8008e6 100644 --- a/src/ray/pubsub/tests/BUILD.bazel +++ b/src/ray/pubsub/tests/BUILD.bazel @@ -27,7 +27,7 @@ ray_cc_test( ray_cc_test( name = "pubsub_integration_test", size = "small", - srcs = ["integration_test.cc"], + srcs = ["pubsub_integration_test.cc"], tags = ["team:core"], deps = [ "//src/ray/protobuf:pubsub_cc_grpc", diff --git a/src/ray/pubsub/tests/integration_test.cc b/src/ray/pubsub/tests/pubsub_integration_test.cc similarity index 98% rename from src/ray/pubsub/tests/integration_test.cc rename to src/ray/pubsub/tests/pubsub_integration_test.cc index 5dc62b32cb6b..626242d3befc 100644 --- a/src/ray/pubsub/tests/integration_test.cc +++ b/src/ray/pubsub/tests/pubsub_integration_test.cc @@ -239,10 +239,11 @@ TEST_F(IntegrationTest, SubscribersToOneIDAndAllIDs) { std::vector actors_2; auto subscriber_2 = CreateSubscriber(); - subscriber_2->SubscribeChannel( + subscriber_2->Subscribe( std::make_unique(), rpc::ChannelType::GCS_ACTOR_CHANNEL, address_proto_, + /*key_id=*/std::nullopt, /*subscribe_done_callback=*/ [&counter](Status status) { RAY_CHECK_OK(status); @@ -293,7 +294,9 @@ TEST_F(IntegrationTest, SubscribersToOneIDAndAllIDs) { subscriber_1->Unsubscribe( rpc::ChannelType::GCS_ACTOR_CHANNEL, address_proto_, subscribed_actor); - subscriber_2->UnsubscribeChannel(rpc::ChannelType::GCS_ACTOR_CHANNEL, address_proto_); + subscriber_2->Unsubscribe(rpc::ChannelType::GCS_ACTOR_CHANNEL, + address_proto_, + /*key_id=*/std::nullopt); // Waiting here is necessary to avoid invalid memory access during shutdown. // TODO(mwtian): cancel inflight polls during subscriber shutdown, and remove the diff --git a/src/ray/pubsub/tests/subscriber_test.cc b/src/ray/pubsub/tests/subscriber_test.cc index e926cbaf48e7..c2b212c13ce9 100644 --- a/src/ray/pubsub/tests/subscriber_test.cc +++ b/src/ray/pubsub/tests/subscriber_test.cc @@ -121,8 +121,6 @@ class MockWorkerClient : public pubsub::SubscriberClientInterface { int GetNumberOfInFlightLongPollingRequests() { return long_polling_callbacks.size(); } - std::string DebugString() const override { return ""; } - ~MockWorkerClient(){}; std::deque> long_polling_callbacks; @@ -274,12 +272,13 @@ TEST_F(SubscriberTest, TestIgnoreOutofOrderMessage) { const auto owner_addr = GenerateOwnerAddress(); const auto object_id = ObjectID::FromRandom(); const auto object_id1 = ObjectID::FromRandom(); - subscriber_->SubscribeChannel(std::make_unique(), - channel, - owner_addr, - /*subscribe_done_callback=*/nullptr, - subscription_callback, - failure_callback); + subscriber_->Subscribe(std::make_unique(), + channel, + owner_addr, + /*key_id=*/std::nullopt, + /*subscribe_done_callback=*/nullptr, + subscription_callback, + failure_callback); ASSERT_TRUE(owner_client->ReplyCommandBatch()); std::vector objects_batched; @@ -320,12 +319,13 @@ TEST_F(SubscriberTest, TestPublisherFailsOver) { const auto owner_addr = GenerateOwnerAddress(); const auto object_id = ObjectID::FromRandom(); const auto object_id1 = ObjectID::FromRandom(); - subscriber_->SubscribeChannel(std::make_unique(), - channel, - owner_addr, - /*subscribe_done_callback=*/nullptr, - subscription_callback, - failure_callback); + subscriber_->Subscribe(std::make_unique(), + channel, + owner_addr, + /*key_id=*/std::nullopt, + /*subscribe_done_callback=*/nullptr, + subscription_callback, + failure_callback); ASSERT_TRUE(owner_client->ReplyCommandBatch()); std::vector objects_batched; @@ -458,9 +458,9 @@ TEST_F(SubscriberTest, TestCallbackNotInvokedForNonSubscribedObject) { ASSERT_EQ(object_subscribed_[object_id], 0); } -TEST_F(SubscriberTest, TestSubscribeChannelEntities) { +TEST_F(SubscriberTest, TestSubscribeAllEntities) { /// - /// Make sure SubscribeChannel() can receive all entities from a channel. + /// Make sure Subscribe() can receive all entities from a channel. /// auto subscription_callback = [this](const rpc::PubMessage &msg) { @@ -469,12 +469,13 @@ TEST_F(SubscriberTest, TestSubscribeChannelEntities) { auto failure_callback = EMPTY_FAILURE_CALLBACK; const auto owner_addr = GenerateOwnerAddress(); - subscriber_->SubscribeChannel(std::make_unique(), - channel, - owner_addr, - /*subscribe_done_callback=*/nullptr, - subscription_callback, - failure_callback); + subscriber_->Subscribe(std::make_unique(), + channel, + owner_addr, + /*key_id=*/std::nullopt, + /*subscribe_done_callback=*/nullptr, + subscription_callback, + failure_callback); ASSERT_TRUE(owner_client->ReplyCommandBatch()); ASSERT_EQ(owner_client->GetNumberOfInFlightLongPollingRequests(), 1); @@ -503,7 +504,7 @@ TEST_F(SubscriberTest, TestSubscribeChannelEntities) { } // Unsubscribe from the channel. - ASSERT_TRUE(subscriber_->UnsubscribeChannel(channel, owner_addr)); + ASSERT_TRUE(subscriber_->Unsubscribe(channel, owner_addr, /*key_id=*/std::nullopt)); } TEST_F(SubscriberTest, TestIgnoreBatchAfterUnsubscription) { @@ -551,14 +552,15 @@ TEST_F(SubscriberTest, TestIgnoreBatchAfterUnsubscribeFromAll) { auto failure_callback = EMPTY_FAILURE_CALLBACK; const auto owner_addr = GenerateOwnerAddress(); - subscriber_->SubscribeChannel(std::make_unique(), - channel, - owner_addr, - /*subscribe_done_callback=*/nullptr, - subscription_callback, - failure_callback); + subscriber_->Subscribe(std::make_unique(), + channel, + owner_addr, + /*key_id=*/std::nullopt, + /*subscribe_done_callback=*/nullptr, + subscription_callback, + failure_callback); ASSERT_TRUE(owner_client->ReplyCommandBatch()); - ASSERT_TRUE(subscriber_->UnsubscribeChannel(channel, owner_addr)); + ASSERT_TRUE(subscriber_->Unsubscribe(channel, owner_addr, /*key_id=*/std::nullopt)); ASSERT_TRUE(owner_client->ReplyCommandBatch()); const auto object_id = ObjectID::FromRandom(); @@ -981,9 +983,6 @@ TEST_F(SubscriberTest, TestIsSubscribed) { ASSERT_FALSE(subscriber_->IsSubscribed(channel, owner_addr, object_id.Binary())); } -// TODO(sang): Need to add a network failure test once we support network failure -// properly. - } // namespace pubsub } // namespace ray diff --git a/src/ray/raylet/.gitkeep b/src/ray/raylet/.gitkeep deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/src/ray/raylet/local_object_manager.cc b/src/ray/raylet/local_object_manager.cc index 37862bb69d05..6530bd08653b 100644 --- a/src/ray/raylet/local_object_manager.cc +++ b/src/ray/raylet/local_object_manager.cc @@ -98,13 +98,13 @@ void LocalObjectManager::PinObjectsAndWaitForFree( auto sub_message = std::make_unique(); sub_message->mutable_worker_object_eviction_message()->Swap(wait_request.get()); - RAY_CHECK(core_worker_subscriber_->Subscribe(std::move(sub_message), - rpc::ChannelType::WORKER_OBJECT_EVICTION, - owner_address, - object_id.Binary(), - /*subscribe_done_callback=*/nullptr, - subscription_callback, - owner_dead_callback)); + core_worker_subscriber_->Subscribe(std::move(sub_message), + rpc::ChannelType::WORKER_OBJECT_EVICTION, + owner_address, + object_id.Binary(), + /*subscribe_done_callback=*/nullptr, + subscription_callback, + owner_dead_callback); } } diff --git a/src/ray/raylet/tests/BUILD.bazel b/src/ray/raylet/tests/BUILD.bazel index eb558ec5c368..71350f2d30ad 100644 --- a/src/ray/raylet/tests/BUILD.bazel +++ b/src/ray/raylet/tests/BUILD.bazel @@ -179,7 +179,6 @@ ray_cc_test( ":util", "//:ray_fakes", "//:ray_mock", - "//src/mock/ray/pubsub:mock_subscriber", "//src/ray/common:ray_object", "//src/ray/object_manager/plasma:plasma_client", "//src/ray/raylet:local_object_manager_interface", diff --git a/src/ray/raylet/tests/local_object_manager_test.cc b/src/ray/raylet/tests/local_object_manager_test.cc index 3688518efa90..2a366322e74a 100644 --- a/src/ray/raylet/tests/local_object_manager_test.cc +++ b/src/ray/raylet/tests/local_object_manager_test.cc @@ -47,18 +47,17 @@ using ::testing::_; class MockSubscriber : public pubsub::SubscriberInterface { public: - bool Subscribe( + void Subscribe( const std::unique_ptr sub_message, - const rpc::ChannelType channel_type, + rpc::ChannelType channel_type, const rpc::Address &owner_address, - const std::string &key_id_binary, + const std::optional &key_id_binary, pubsub::SubscribeDoneCallback subscribe_done_callback, pubsub::SubscriptionItemCallback subscription_callback, pubsub::SubscriptionFailureCallback subscription_failure_callback) override { auto worker_id = WorkerID::FromBinary(owner_address.worker_id()); callbacks[worker_id].push_back( - std::make_pair(ObjectID::FromBinary(key_id_binary), subscription_callback)); - return true; + std::make_pair(ObjectID::FromBinary(*key_id_binary), subscription_callback)); } bool PublishObjectEviction(WorkerID worker_id = WorkerID::Nil()) { @@ -87,25 +86,13 @@ class MockSubscriber : public pubsub::SubscriberInterface { return true; } - MOCK_METHOD6(SubscribeChannel, - bool(std::unique_ptr sub_message, - const rpc::ChannelType channel_type, - const rpc::Address &owner_address, - pubsub::SubscribeDoneCallback subscribe_done_callback, - pubsub::SubscriptionItemCallback subscription_callback, - pubsub::SubscriptionFailureCallback subscription_failure_callback)); - MOCK_METHOD3(Unsubscribe, - bool(const rpc::ChannelType channel_type, + bool(rpc::ChannelType channel_type, const rpc::Address &publisher_address, - const std::string &key_id_binary)); - - MOCK_METHOD2(UnsubscribeChannel, - bool(const rpc::ChannelType channel_type, - const rpc::Address &publisher_address)); + const std::optional &key_id_binary)); MOCK_CONST_METHOD3(IsSubscribed, - bool(const rpc::ChannelType channel_type, + bool(rpc::ChannelType channel_type, const rpc::Address &publisher_address, const std::string &key_id_binary)); @@ -459,7 +446,9 @@ TEST_F(LocalObjectManagerTest, TestPin) { for (size_t i = 0; i < free_objects_batch_size; i++) { ASSERT_TRUE(freed.empty()); - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); } std::unordered_set expected(object_ids.begin(), object_ids.end()); @@ -945,7 +934,9 @@ TEST_F(LocalObjectManagerTest, TestDeleteNoSpilledObjects) { for (size_t i = 0; i < free_objects_batch_size; i++) { ASSERT_TRUE(freed.empty()); - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); } @@ -995,7 +986,9 @@ TEST_F(LocalObjectManagerTest, TestDeleteSpilledObjects) { // All objects are out of scope now. for (size_t i = 0; i < free_objects_batch_size; i++) { - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); } @@ -1054,7 +1047,9 @@ TEST_F(LocalObjectManagerTest, TestDeleteURLRefCount) { // Everything is evicted except the last object. In this case, ref count is still > 0. for (size_t i = 0; i < free_objects_batch_size - 1; i++) { - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); } manager.ProcessSpilledObjectsDeleteQueue(/* max_batch_size */ 30); @@ -1068,7 +1063,10 @@ TEST_F(LocalObjectManagerTest, TestDeleteURLRefCount) { // The last reference is deleted. EXPECT_CALL(*subscriber_, - Unsubscribe(_, _, object_ids[free_objects_batch_size - 1].Binary())); + Unsubscribe(_, + _, + std::make_optional( + object_ids[free_objects_batch_size - 1].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); manager.ProcessSpilledObjectsDeleteQueue(/* max_batch_size */ 30); deleted_urls_size = worker_pool.io_worker_client->ReplyDeleteSpilledObjects(); @@ -1136,7 +1134,9 @@ TEST_F(LocalObjectManagerTest, TestDeleteSpillingObjectsBlocking) { // Every object has gone out of scope. for (size_t i = 0; i < spilled_urls_size; i++) { - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); } // Now, deletion queue would process only the first spill set. Everything else won't be @@ -1204,7 +1204,9 @@ TEST_F(LocalObjectManagerTest, TestDeleteMaxObjects) { // Every reference has gone out of scope. for (size_t i = 0; i < free_objects_batch_size; i++) { - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); } @@ -1256,7 +1258,8 @@ TEST_F(LocalObjectManagerTest, TestDeleteURLRefCountRaceCondition) { ASSERT_EQ(GetCurrentSpilledCount(), object_ids_to_spill.size()); ASSERT_EQ(GetCurrentSpilledBytes(), object_size * object_ids_to_spill.size()); - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[0].Binary())); + EXPECT_CALL(*subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[0].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); // Delete operation is called. In this case, the file with the url should not be // deleted. @@ -1270,7 +1273,9 @@ TEST_F(LocalObjectManagerTest, TestDeleteURLRefCountRaceCondition) { // Everything else is now deleted. for (size_t i = 1; i < free_objects_batch_size; i++) { - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); } manager.ProcessSpilledObjectsDeleteQueue(/* max_batch_size */ 30); @@ -1337,7 +1342,9 @@ TEST_F(LocalObjectManagerTest, TestDuplicatePin) { auto owner_id1 = WorkerID::FromBinary(owner_address.worker_id()); for (size_t i = 0; i < free_objects_batch_size; i++) { ASSERT_TRUE(freed.empty()); - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction(owner_id1)); } std::unordered_set expected(object_ids.begin(), object_ids.end()); @@ -1378,7 +1385,9 @@ TEST_F(LocalObjectManagerTest, TestDuplicatePinAndSpill) { auto owner_id1 = WorkerID::FromBinary(owner_address.worker_id()); for (size_t i = 0; i < free_objects_batch_size; i++) { ASSERT_TRUE(freed.empty()); - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction(owner_id1)); } std::unordered_set expected(object_ids.begin(), object_ids.end()); @@ -1604,7 +1613,9 @@ TEST_F(LocalObjectManagerTest, TestPinBytes) { // Delete all (spilled) objects. for (size_t i = 0; i < free_objects_batch_size; i++) { - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); } manager.ProcessSpilledObjectsDeleteQueue(/* max_batch_size */ 30); @@ -1666,7 +1677,9 @@ TEST_F(LocalObjectManagerTest, TestConcurrentSpillAndDelete1) { // Delete all objects while they're being spilled. for (size_t i = 0; i < free_objects_batch_size; i++) { - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); } @@ -1737,7 +1750,9 @@ TEST_F(LocalObjectManagerTest, TestConcurrentSpillAndDelete2) { // Delete all objects while allocating an IO worker. for (size_t i = 0; i < free_objects_batch_size; i++) { - EXPECT_CALL(*subscriber_, Unsubscribe(_, _, object_ids[i].Binary())); + EXPECT_CALL( + *subscriber_, + Unsubscribe(_, _, std::make_optional(object_ids[i].Binary()))); ASSERT_TRUE(subscriber_->PublishObjectEviction()); } diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index 55e4d75a1468..56bd7d90f873 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -21,6 +21,7 @@ #include #include +#include "fakes/ray/pubsub/subscriber.h" #include "fakes/ray/rpc/raylet/raylet_client.h" #include "gmock/gmock.h" #include "mock/ray/core_worker/experimental_mutable_object_provider.h" @@ -28,7 +29,6 @@ #include "mock/ray/object_manager/object_directory.h" #include "mock/ray/object_manager/object_manager.h" #include "mock/ray/object_manager/plasma/client.h" -#include "mock/ray/pubsub/subscriber.h" #include "mock/ray/raylet/local_task_manager.h" #include "mock/ray/raylet/worker_pool.h" #include "mock/ray/rpc/worker/core_worker_client.h" @@ -391,7 +391,7 @@ class NodeManagerTest : public ::testing::Test { node_manager_config.maximum_startup_concurrency = 1; node_manager_config.store_socket_name = "test_store_socket"; - core_worker_subscriber_ = std::make_unique(); + core_worker_subscriber_ = std::make_unique(); mock_object_directory_ = std::make_unique(); mock_object_manager_ = std::make_unique(); @@ -409,7 +409,6 @@ class NodeManagerTest : public ::testing::Test { EXPECT_CALL(*mock_gcs_client_, DebugString()).WillRepeatedly(Return("")); EXPECT_CALL(*mock_object_manager_, DebugString()).WillRepeatedly(Return("")); EXPECT_CALL(*mock_object_directory_, DebugString()).WillRepeatedly(Return("")); - EXPECT_CALL(*core_worker_subscriber_, DebugString()).WillRepeatedly(Return("")); raylet_node_id_ = NodeID::FromRandom(); @@ -508,7 +507,7 @@ class NodeManagerTest : public ::testing::Test { rpc::RayletClientPool raylet_client_pool_; NodeID raylet_node_id_; - std::unique_ptr core_worker_subscriber_; + std::unique_ptr core_worker_subscriber_; std::unique_ptr cluster_resource_scheduler_; std::unique_ptr local_task_manager_; std::unique_ptr cluster_task_manager_; From 9fb7d772619b2781e9a44bec468feae771b3bf25 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Wed, 20 Aug 2025 21:04:39 -0500 Subject: [PATCH 211/634] [core] Remove global `util` target (#55782) Split the `util.h` target into appropriate sub-targets, fixed all targets that depend on it to depend on the sub-targets instead. --------- Signed-off-by: Edward Oakes --- BUILD.bazel | 2 +- cpp/BUILD.bazel | 1 - cpp/src/ray/config_internal.cc | 3 +- cpp/src/ray/util/process_helper.cc | 1 - python/ray/includes/global_state_accessor.pxd | 1 + src/ray/common/BUILD.bazel | 17 +- src/ray/common/asio/asio_util.h | 1 - src/ray/common/cgroup/BUILD.bazel | 3 +- src/ray/common/cgroup/cgroup_setup.cc | 1 - src/ray/common/id.cc | 1 - src/ray/common/id.h | 1 - src/ray/common/memory_monitor.cc | 1 - src/ray/common/scheduling/scheduling_ids.h | 1 - src/ray/common/status.h | 1 + src/ray/common/test_util.cc | 2 +- src/ray/common/test_util.h | 1 - src/ray/common/tests/BUILD.bazel | 1 + src/ray/common/tests/ray_syncer_test.cc | 1 + src/ray/core_worker/BUILD.bazel | 6 +- src/ray/core_worker/common.cc | 2 + src/ray/core_worker/core_worker.cc | 1 - src/ray/core_worker/core_worker_process.cc | 1 - .../experimental_mutable_object_manager.cc | 1 + .../core_worker/object_recovery_manager.cc | 2 - .../store_provider/plasma_store_provider.cc | 1 + src/ray/core_worker/task_manager.cc | 2 +- src/ray/core_worker/tests/BUILD.bazel | 1 + .../core_worker/tests/actor_creator_test.cc | 1 + src/ray/gcs/BUILD.bazel | 1 + src/ray/gcs/gcs_client/BUILD.bazel | 1 + .../gcs/gcs_client/global_state_accessor.cc | 1 + src/ray/gcs/gcs_client/tests/BUILD.bazel | 5 + .../tests/gcs_client_reconnection_test.cc | 2 +- .../gcs/gcs_client/tests/gcs_client_test.cc | 2 +- .../tests/global_state_accessor_test.cc | 1 + src/ray/gcs/gcs_server/BUILD.bazel | 2 + src/ray/gcs/gcs_server/gcs_actor_manager.cc | 1 + src/ray/gcs/gcs_server/gcs_actor_scheduler.cc | 1 + .../gcs_autoscaler_state_manager.cc | 1 + src/ray/gcs/gcs_server/gcs_job_manager.cc | 1 + src/ray/gcs/gcs_server/gcs_node_manager.cc | 1 + .../gcs/gcs_server/gcs_placement_group_mgr.h | 1 + src/ray/gcs/gcs_server/gcs_server.cc | 1 - src/ray/gcs/gcs_server/gcs_server_main.cc | 2 +- src/ray/gcs/pb_util.h | 1 + src/ray/gcs/store_client/BUILD.bazel | 1 - src/ray/gcs/store_client/redis_context.cc | 1 - src/ray/gcs/store_client/tests/BUILD.bazel | 8 +- .../tests/redis_async_context_test.cc | 1 + .../tests/redis_store_client_test.cc | 2 + src/ray/ipc/BUILD.bazel | 3 + src/ray/ipc/client_connection.cc | 3 +- src/ray/ipc/raylet_ipc_client.cc | 16 +- src/ray/ipc/tests/BUILD.bazel | 1 + src/ray/ipc/tests/client_connection_test.cc | 1 + src/ray/object_manager/BUILD.bazel | 1 + src/ray/object_manager/common.cc | 2 + src/ray/object_manager/plasma/BUILD.bazel | 4 +- src/ray/object_manager/plasma/connection.cc | 1 + .../plasma/create_request_queue.cc | 1 - src/ray/object_manager/plasma/store.cc | 4 +- .../object_manager/plasma/tests/BUILD.bazel | 1 + .../plasma/tests/fallback_allocator_test.cc | 3 +- .../plasma/tests/mutable_object_test.cc | 1 + src/ray/raylet/BUILD.bazel | 10 +- src/ray/raylet/agent_manager.cc | 1 - src/ray/raylet/agent_manager.h | 1 + src/ray/raylet/local_object_manager.cc | 1 - src/ray/raylet/local_object_manager.h | 1 + src/ray/raylet/main.cc | 4 +- src/ray/raylet/node_manager.cc | 5 +- .../raylet/placement_group_resource_manager.h | 1 - src/ray/raylet/raylet.cc | 2 +- src/ray/raylet/runtime_env_agent_client.cc | 2 + src/ray/raylet/tests/BUILD.bazel | 1 + src/ray/raylet/tests/worker_pool_test.cc | 1 + src/ray/raylet/worker_pool.cc | 2 +- .../rpc/node_manager/raylet_client_pool.cc | 2 - src/ray/stats/BUILD.bazel | 1 - src/ray/stats/metric_exporter.h | 1 - src/ray/util/BUILD.bazel | 45 ++-- src/ray/util/event.cc | 2 +- src/ray/util/internal/tests/BUILD.bazel | 4 +- .../tests/stream_redirection_handle_test.cc | 9 +- src/ray/util/network_util.cc | 134 ++++++++++- src/ray/util/network_util.h | 36 +++ src/ray/util/pipe_logger.cc | 2 + src/ray/util/pipe_logger.h | 2 +- src/ray/util/process.cc | 5 + src/ray/util/process.h | 4 + src/ray/util/raii.h | 43 ++++ src/ray/util/spdlog_fd_sink.h | 2 +- src/ray/util/spdlog_newliner_sink.h | 1 - src/ray/util/stream_redirection.cc | 6 +- src/ray/util/string_utils.cc | 13 + src/ray/util/string_utils.h | 3 + src/ray/util/temporary_directory.cc | 4 +- src/ray/util/tests/BUILD.bazel | 34 +-- src/ray/util/tests/logging_test.cc | 2 +- src/ray/util/tests/network_util_test.cc | 39 +++ src/ray/util/tests/pipe_logger_test.cc | 25 +- src/ray/util/tests/process_cleanup_test.cc | 5 +- .../tests/{util_test.cc => process_test.cc} | 51 +--- src/ray/util/tests/signal_test.cc | 4 +- .../util/tests/spdlog_newliner_sink_test.cc | 14 +- .../tests/stream_redirection_exit_test.cc | 6 +- src/ray/util/{timestamp_utils.h => time.cc} | 17 +- src/ray/util/time.h | 51 ++++ src/ray/util/util.cc | 227 ------------------ src/ray/util/util.h | 189 --------------- 110 files changed, 546 insertions(+), 607 deletions(-) create mode 100644 src/ray/util/raii.h rename src/ray/util/tests/{util_test.cc => process_test.cc} (52%) rename src/ray/util/{timestamp_utils.h => time.cc} (59%) create mode 100644 src/ray/util/time.h delete mode 100644 src/ray/util/util.cc delete mode 100644 src/ray/util/util.h diff --git a/BUILD.bazel b/BUILD.bazel index 5473ddce9e41..65c06e57345c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -280,8 +280,8 @@ pyx_library( "//src/ray/gcs/store_client:redis_store_client", "//src/ray/protobuf:serialization_cc_proto", "//src/ray/thirdparty/setproctitle", - "//src/ray/util", "//src/ray/util:memory", + "//src/ray/util:raii", "//src/ray/util:stream_redirection", "//src/ray/util:stream_redirection_options", ], diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index 3164cf7cacf5..745316e11a32 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -67,7 +67,6 @@ cc_library( "//src/ray/common:task_common", "//src/ray/core_worker:core_worker_lib", "//src/ray/gcs/gcs_client:global_state_accessor_lib", - "//src/ray/util", "//src/ray/util:cmd_line_utils", "//src/ray/util:network_util", "//src/ray/util:process", diff --git a/cpp/src/ray/config_internal.cc b/cpp/src/ray/config_internal.cc index 01ee6001502c..b5ae0b2d227b 100644 --- a/cpp/src/ray/config_internal.cc +++ b/cpp/src/ray/config_internal.cc @@ -22,6 +22,7 @@ #include "absl/flags/parse.h" #include "absl/strings/str_split.h" #include "nlohmann/json.hpp" +#include "ray/common/id.h" #include "ray/util/network_util.h" ABSL_FLAG(std::string, ray_address, "", "The address of the Ray cluster to connect to."); @@ -235,7 +236,7 @@ void ConfigInternal::Init(RayConfig &config, int argc, char **argv) { ray_namespace = FLAGS_ray_job_namespace.CurrentValue(); } if (ray_namespace.empty()) { - ray_namespace = GenerateUUIDV4(); + ray_namespace = UniqueID::FromRandom().Hex(); } } diff --git a/cpp/src/ray/util/process_helper.cc b/cpp/src/ray/util/process_helper.cc index 63c75ffaf24f..fbddae45f7b9 100644 --- a/cpp/src/ray/util/process_helper.cc +++ b/cpp/src/ray/util/process_helper.cc @@ -21,7 +21,6 @@ #include "ray/util/cmd_line_utils.h" #include "ray/util/network_util.h" #include "ray/util/process.h" -#include "ray/util/util.h" #include "src/ray/protobuf/gcs.pb.h" namespace ray { diff --git a/python/ray/includes/global_state_accessor.pxd b/python/ray/includes/global_state_accessor.pxd index 1d6e5cd1c8c9..7d115b469ce9 100644 --- a/python/ray/includes/global_state_accessor.pxd +++ b/python/ray/includes/global_state_accessor.pxd @@ -72,6 +72,7 @@ cdef extern from * namespace "ray::gcs" nogil: #include #include "ray/gcs/gcs_server/store_client_kv.h" #include "ray/gcs/store_client/redis_store_client.h" + #include "ray/util/raii.h" namespace ray { namespace gcs { diff --git a/src/ray/common/BUILD.bazel b/src/ray/common/BUILD.bazel index 992728a11bc7..5eea6f6c8bea 100644 --- a/src/ray/common/BUILD.bazel +++ b/src/ray/common/BUILD.bazel @@ -19,10 +19,11 @@ ray_cc_library( ":id", ":ray_object", "//src/ray/protobuf:common_cc_proto", - "//src/ray/util", "//src/ray/util:cmd_line_utils", "//src/ray/util:network_util", "//src/ray/util:path_utils", + "//src/ray/util:process", + "//src/ray/util:time", "@boost//:optional", "@com_google_googletest//:gtest", ], @@ -61,7 +62,6 @@ ray_cc_library( deps = [ ":ray_config", ":status", - "//src/ray/util", "//src/ray/util:logging", "//src/ray/util:type_traits", "@com_github_grpc_grpc//:grpc++", @@ -80,7 +80,7 @@ ray_cc_library( deps = [ ":asio", ":ray_config", - "//src/ray/util", + "//src/ray/util:process", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_googletest//:gtest_prod", @@ -94,7 +94,6 @@ ray_cc_library( hdrs = ["file_system_monitor.h"], deps = [ ":asio", - "//src/ray/util", "//src/ray/util:event", "@com_google_googletest//:gtest_prod", ], @@ -133,7 +132,7 @@ ray_cc_library( ":status", "//src/ray/protobuf:common_cc_proto", "//src/ray/protobuf:gcs_cc_proto", - "//src/ray/util", + "//src/ray/thirdparty:sha256", "//src/ray/util:random", "@com_github_google_flatbuffers//:flatbuffers", "@msgpack", @@ -180,7 +179,6 @@ ray_cc_library( ":ray_object", ":runtime_env", "//src/ray/flatbuffers:node_manager_generated", - "//src/ray/util", "//src/ray/util:container_util", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -209,7 +207,6 @@ ray_cc_library( deps = [ ":event_stats", ":ray_config", - "//src/ray/util", "//src/ray/util:array", "//src/ray/util:function_traits", "@boost//:asio", @@ -229,7 +226,6 @@ ray_cc_library( deps = [ ":ray_config", "//src/ray/stats:stats_metric", - "//src/ray/util", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/synchronization", ], @@ -244,8 +240,8 @@ ray_cc_library( "ray_internal_flag_def.h", ], deps = [ - "//src/ray/util", - "@com_google_absl//absl/algorithm", + "//src/ray/util:logging", + "@boost//:algorithm", "@com_google_absl//absl/strings", "@nlohmann_json", ], @@ -284,6 +280,7 @@ ray_cc_library( deps = [ ":macros", ":source_location", + "//src/ray/util:logging", "//src/ray/util:macros", "//src/ray/util:visibility", "@boost//:system", diff --git a/src/ray/common/asio/asio_util.h b/src/ray/common/asio/asio_util.h index 38564c8ac0e3..793729fb5d89 100644 --- a/src/ray/common/asio/asio_util.h +++ b/src/ray/common/asio/asio_util.h @@ -25,7 +25,6 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/util/array.h" #include "ray/util/thread_utils.h" -#include "ray/util/util.h" template std::shared_ptr execute_after( diff --git a/src/ray/common/cgroup/BUILD.bazel b/src/ray/common/cgroup/BUILD.bazel index a67086c14e75..1487689d8985 100644 --- a/src/ray/common/cgroup/BUILD.bazel +++ b/src/ray/common/cgroup/BUILD.bazel @@ -14,8 +14,9 @@ ray_cc_library( ":cgroup_utils", ":constants", "//src/ray/common:macros", - "//src/ray/util", + "//src/ray/util:filesystem", "//src/ray/util:invoke_once_token", + "//src/ray/util:logging", "//src/ray/util:path_utils", "@com_google_absl//absl/strings:str_format", ], diff --git a/src/ray/common/cgroup/cgroup_setup.cc b/src/ray/common/cgroup/cgroup_setup.cc index 3087e172d457..5a3903768b7f 100644 --- a/src/ray/common/cgroup/cgroup_setup.cc +++ b/src/ray/common/cgroup/cgroup_setup.cc @@ -85,7 +85,6 @@ Status CheckCgroupV2MountedRW(const std::string &directory) { #include "ray/util/invoke_once_token.h" #include "ray/util/logging.h" #include "ray/util/path_utils.h" -#include "ray/util/util.h" namespace ray { diff --git a/src/ray/common/id.cc b/src/ray/common/id.cc index 91041d75d70f..53b033dfb7f5 100644 --- a/src/ray/common/id.cc +++ b/src/ray/common/id.cc @@ -25,7 +25,6 @@ #include "ray/common/constants.h" #include "ray/common/status.h" #include "ray/util/macros.h" -#include "ray/util/util.h" extern "C" { #include "ray/thirdparty/sha256.h" diff --git a/src/ray/common/id.h b/src/ray/common/id.h index 6296c717253c..736c116a341a 100644 --- a/src/ray/common/id.h +++ b/src/ray/common/id.h @@ -27,7 +27,6 @@ #include "ray/common/constants.h" #include "ray/util/logging.h" #include "ray/util/random.h" -#include "ray/util/util.h" #include "ray/util/visibility.h" namespace ray { diff --git a/src/ray/common/memory_monitor.cc b/src/ray/common/memory_monitor.cc index 2a07d57b3e52..1c60943402a9 100644 --- a/src/ray/common/memory_monitor.cc +++ b/src/ray/common/memory_monitor.cc @@ -23,7 +23,6 @@ #include "ray/common/ray_config.h" #include "ray/util/logging.h" #include "ray/util/process.h" -#include "ray/util/util.h" namespace ray { diff --git a/src/ray/common/scheduling/scheduling_ids.h b/src/ray/common/scheduling/scheduling_ids.h index e3b067bc53b8..e054bc5c9c60 100644 --- a/src/ray/common/scheduling/scheduling_ids.h +++ b/src/ray/common/scheduling/scheduling_ids.h @@ -25,7 +25,6 @@ #include "ray/common/constants.h" #include "ray/common/ray_config.h" #include "ray/util/logging.h" -#include "ray/util/util.h" namespace ray { diff --git a/src/ray/common/status.h b/src/ray/common/status.h index 1230b449c072..58c83f3abb8f 100644 --- a/src/ray/common/status.h +++ b/src/ray/common/status.h @@ -35,6 +35,7 @@ #include "absl/strings/str_cat.h" #include "ray/common/macros.h" #include "ray/common/source_location.h" +#include "ray/util/logging.h" #include "ray/util/macros.h" #include "ray/util/visibility.h" diff --git a/src/ray/common/test_util.cc b/src/ray/common/test_util.cc index 14307488c87e..0fed9ae8dc5e 100644 --- a/src/ray/common/test_util.cc +++ b/src/ray/common/test_util.cc @@ -28,7 +28,7 @@ #include "ray/util/network_util.h" #include "ray/util/path_utils.h" #include "ray/util/process.h" -#include "ray/util/util.h" +#include "ray/util/time.h" namespace ray { diff --git a/src/ray/common/test_util.h b/src/ray/common/test_util.h index 8be836b49e01..cdb9ae4dfc88 100644 --- a/src/ray/common/test_util.h +++ b/src/ray/common/test_util.h @@ -21,7 +21,6 @@ #include "gtest/gtest.h" #include "ray/common/asio/asio_util.h" #include "ray/common/id.h" -#include "ray/util/util.h" #include "src/ray/protobuf/common.pb.h" namespace ray { diff --git a/src/ray/common/tests/BUILD.bazel b/src/ray/common/tests/BUILD.bazel index cbb4211981b5..76b56ffbb30d 100644 --- a/src/ray/common/tests/BUILD.bazel +++ b/src/ray/common/tests/BUILD.bazel @@ -54,6 +54,7 @@ ray_cc_test( "//src/ray/rpc:grpc_server", "//src/ray/util:network_util", "//src/ray/util:path_utils", + "//src/ray/util:raii", "@com_github_grpc_grpc//:grpc++", "@com_google_googletest//:gtest", ], diff --git a/src/ray/common/tests/ray_syncer_test.cc b/src/ray/common/tests/ray_syncer_test.cc index 6982980efedf..cb2b81579eb4 100644 --- a/src/ray/common/tests/ray_syncer_test.cc +++ b/src/ray/common/tests/ray_syncer_test.cc @@ -40,6 +40,7 @@ #include "ray/rpc/grpc_server.h" #include "ray/util/network_util.h" #include "ray/util/path_utils.h" +#include "ray/util/raii.h" using ray::NodeID; using ::testing::_; diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 625744d2d526..388e6b3a9d8b 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -46,7 +46,6 @@ ray_cc_library( "//src/ray/rpc:core_worker_client", "//src/ray/rpc:core_worker_server", "//src/ray/stats:stats_lib", - "//src/ray/util", "//src/ray/util:container_util", "//src/ray/util:env", "//src/ray/util:event", @@ -86,6 +85,7 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:ray_object", "//src/ray/common:task_common", + "//src/ray/util:process", ], ) @@ -251,9 +251,9 @@ ray_cc_library( "//src/ray/protobuf:common_cc_proto", "//src/ray/protobuf:core_worker_cc_proto", "//src/ray/stats:stats_metric", - "//src/ray/util", "//src/ray/util:counter_map", "//src/ray/util:exponential_backoff", + "//src/ray/util:time", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", @@ -272,6 +272,7 @@ ray_cc_library( "//src/ray/common:task_common", "//src/ray/object_manager:object_manager_common", "//src/ray/object_manager/plasma:plasma_client", + "//src/ray/util:time", "@com_google_absl//absl/container:node_hash_map", "@com_google_absl//absl/strings", "@com_google_googletest//:gtest_prod", @@ -353,6 +354,7 @@ ray_cc_library( "//src/ray/ipc:raylet_ipc_client_interface", "//src/ray/object_manager/plasma:plasma_client", "//src/ray/protobuf:common_cc_proto", + "//src/ray/util:time", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", ], diff --git a/src/ray/core_worker/common.cc b/src/ray/core_worker/common.cc index e82ed8d8fbed..a28a54053051 100644 --- a/src/ray/core_worker/common.cc +++ b/src/ray/core_worker/common.cc @@ -19,6 +19,8 @@ #include #include +#include "ray/util/process.h" + namespace ray { namespace core { diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index c816dd2a09e6..8efe6a7878bf 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -47,7 +47,6 @@ #include "ray/util/container_util.h" #include "ray/util/event.h" #include "ray/util/subreaper.h" -#include "ray/util/util.h" using json = nlohmann::json; using MessageType = ray::protocol::MessageType; diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index db60abd62293..59fe6ad14f0f 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -45,7 +45,6 @@ #include "ray/util/stream_redirection.h" #include "ray/util/stream_redirection_options.h" #include "ray/util/subreaper.h" -#include "ray/util/util.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/experimental_mutable_object_manager.cc b/src/ray/core_worker/experimental_mutable_object_manager.cc index 9b083e4830e7..ca551d97eb59 100644 --- a/src/ray/core_worker/experimental_mutable_object_manager.cc +++ b/src/ray/core_worker/experimental_mutable_object_manager.cc @@ -24,6 +24,7 @@ #include "absl/strings/str_format.h" #include "ray/common/ray_config.h" #include "ray/object_manager/common.h" +#include "ray/util/time.h" namespace ray { namespace experimental { diff --git a/src/ray/core_worker/object_recovery_manager.cc b/src/ray/core_worker/object_recovery_manager.cc index 1fd5669589c9..893fb4fabd6d 100644 --- a/src/ray/core_worker/object_recovery_manager.cc +++ b/src/ray/core_worker/object_recovery_manager.cc @@ -18,8 +18,6 @@ #include #include -#include "ray/util/util.h" - namespace ray { namespace core { diff --git a/src/ray/core_worker/store_provider/plasma_store_provider.cc b/src/ray/core_worker/store_provider/plasma_store_provider.cc index 7dd0589e6e77..98da568c13b5 100644 --- a/src/ray/core_worker/store_provider/plasma_store_provider.cc +++ b/src/ray/core_worker/store_provider/plasma_store_provider.cc @@ -24,6 +24,7 @@ #include "ray/common/status.h" #include "ray/common/status_or.h" #include "ray/ipc/raylet_ipc_client_interface.h" +#include "ray/util/time.h" #include "src/ray/protobuf/common.pb.h" namespace ray { diff --git a/src/ray/core_worker/task_manager.cc b/src/ray/core_worker/task_manager.cc index d70717bc49f7..70693b8f2a19 100644 --- a/src/ray/core_worker/task_manager.cc +++ b/src/ray/core_worker/task_manager.cc @@ -27,7 +27,7 @@ #include "ray/core_worker/actor_manager.h" #include "ray/gcs/pb_util.h" #include "ray/util/exponential_backoff.h" -#include "ray/util/util.h" +#include "ray/util/time.h" #include "src/ray/protobuf/common.pb.h" namespace ray { diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index abf81304b176..96b6769cc948 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -155,6 +155,7 @@ ray_cc_test( "//src/ray/core_worker:actor_creator", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/util:path_utils", + "//src/ray/util:raii", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/core_worker/tests/actor_creator_test.cc b/src/ray/core_worker/tests/actor_creator_test.cc index 9e50e896d151..4bb50d9a6004 100644 --- a/src/ray/core_worker/tests/actor_creator_test.cc +++ b/src/ray/core_worker/tests/actor_creator_test.cc @@ -20,6 +20,7 @@ #include "ray/core_worker/actor_creator.h" #include "ray/common/test_util.h" #include "ray/util/path_utils.h" +#include "ray/util/raii.h" #include "mock/ray/gcs/gcs_client/gcs_client.h" // clang-format on diff --git a/src/ray/gcs/BUILD.bazel b/src/ray/gcs/BUILD.bazel index 8ffa4b6639e9..eaaae3219fdd 100644 --- a/src/ray/gcs/BUILD.bazel +++ b/src/ray/gcs/BUILD.bazel @@ -11,6 +11,7 @@ ray_cc_library( "//src/ray/common:task_common", "//src/ray/protobuf:autoscaler_cc_proto", "//src/ray/protobuf:export_task_event_cc_proto", + "//src/ray/util:time", ], ) diff --git a/src/ray/gcs/gcs_client/BUILD.bazel b/src/ray/gcs/gcs_client/BUILD.bazel index dc15ab29dfb5..52870712ee48 100644 --- a/src/ray/gcs/gcs_client/BUILD.bazel +++ b/src/ray/gcs/gcs_client/BUILD.bazel @@ -31,6 +31,7 @@ ray_cc_library( hdrs = ["global_state_accessor.h"], deps = [ ":gcs_client_lib", + "//src/ray/util:time", ], ) diff --git a/src/ray/gcs/gcs_client/global_state_accessor.cc b/src/ray/gcs/gcs_client/global_state_accessor.cc index 64d55d5876c1..89b3582150c1 100644 --- a/src/ray/gcs/gcs_client/global_state_accessor.cc +++ b/src/ray/gcs/gcs_client/global_state_accessor.cc @@ -23,6 +23,7 @@ #include #include "ray/common/asio/instrumented_io_context.h" +#include "ray/util/time.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_client/tests/BUILD.bazel b/src/ray/gcs/gcs_client/tests/BUILD.bazel index cc428b88a513..57054315512a 100644 --- a/src/ray/gcs/gcs_client/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_client/tests/BUILD.bazel @@ -34,6 +34,8 @@ ray_cc_test( "//src/ray/gcs/gcs_client:global_state_accessor_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", + "//src/ray/util:path_utils", + "//src/ray/util:raii", "@com_google_googletest//:gtest_main", ], ) @@ -62,6 +64,7 @@ ray_cc_test( "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:network_util", + "//src/ray/util:raii", "@com_google_googletest//:gtest_main", ], ) @@ -88,6 +91,8 @@ ray_cc_test( "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:network_util", + "//src/ray/util:path_utils", + "//src/ray/util:raii", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc b/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc index 409d16bbadaf..1669031fab25 100644 --- a/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc +++ b/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc @@ -29,7 +29,7 @@ #include "ray/rpc/gcs/gcs_rpc_client.h" #include "ray/util/network_util.h" #include "ray/util/path_utils.h" -#include "ray/util/util.h" +#include "ray/util/raii.h" using namespace std::chrono_literals; // NOLINT using namespace ray; // NOLINT diff --git a/src/ray/gcs/gcs_client/tests/gcs_client_test.cc b/src/ray/gcs/gcs_client/tests/gcs_client_test.cc index 6df7b90bd2ab..7b68e37d1516 100644 --- a/src/ray/gcs/gcs_client/tests/gcs_client_test.cc +++ b/src/ray/gcs/gcs_client/tests/gcs_client_test.cc @@ -28,7 +28,7 @@ #include "ray/rpc/gcs/gcs_rpc_client.h" #include "ray/util/network_util.h" #include "ray/util/path_utils.h" -#include "ray/util/util.h" +#include "ray/util/raii.h" using namespace std::chrono_literals; // NOLINT diff --git a/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc b/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc index f761cb5c90b9..3613d177f4dd 100644 --- a/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc +++ b/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc @@ -24,6 +24,7 @@ #include "ray/gcs/tests/gcs_test_util.h" #include "ray/rpc/gcs/gcs_rpc_client.h" #include "ray/util/path_utils.h" +#include "ray/util/raii.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index e698c17d75c2..9e482b9736bc 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -248,6 +248,7 @@ ray_cc_library( "//src/ray/util:network_util", "//src/ray/util:thread_checker", "//src/ray/util:throttler", + "//src/ray/util:time", "//src/ray/util:type_traits", "@boost//:bimap", "@com_google_absl//absl/container:btree", @@ -263,6 +264,7 @@ ray_cc_binary( deps = [ ":gcs_server_lib", "//src/ray/stats:stats_lib", + "//src/ray/util:raii", "//src/ray/util:stream_redirection", "//src/ray/util:stream_redirection_options", "@com_github_gflags_gflags//:gflags", diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index a895bbb13ad0..626631f91dc1 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -25,6 +25,7 @@ #include "ray/common/ray_config.h" #include "ray/gcs/pb_util.h" #include "ray/stats/metric_defs.h" +#include "ray/util/time.h" namespace { /// The error message constructed from below methods is user-facing, so please avoid diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc index 35b9c03dd146..746de6923558 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc @@ -23,6 +23,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/ray_config.h" #include "ray/gcs/gcs_server/gcs_actor_manager.h" +#include "ray/util/time.h" #include "src/ray/protobuf/node_manager.pb.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc index 2c234c92d6ac..8a84b8a00c63 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc @@ -25,6 +25,7 @@ #include "ray/gcs/gcs_server/state_util.h" #include "ray/gcs/pb_util.h" #include "ray/util/string_utils.h" +#include "ray/util/time.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.cc b/src/ray/gcs/gcs_server/gcs_job_manager.cc index a807da1d5ee6..e71c69bccedc 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_job_manager.cc @@ -24,6 +24,7 @@ #include "absl/strings/match.h" #include "ray/gcs/pb_util.h" #include "ray/stats/metric.h" +#include "ray/util/time.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.cc b/src/ray/gcs/gcs_server/gcs_node_manager.cc index 15347e782001..bd14e119811a 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_node_manager.cc @@ -25,6 +25,7 @@ #include "ray/gcs/pb_util.h" #include "ray/util/event.h" #include "ray/util/logging.h" +#include "ray/util/time.h" #include "src/ray/protobuf/gcs.pb.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h index fd046856fc17..d5956a93609a 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h @@ -36,6 +36,7 @@ #include "ray/rpc/worker/core_worker_client.h" #include "ray/util/counter_map.h" #include "ray/util/exponential_backoff.h" +#include "ray/util/time.h" #include "src/ray/protobuf/gcs_service.pb.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 65402804ed86..90c61aa24da7 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -36,7 +36,6 @@ #include "ray/gcs/store_client/store_client.h" #include "ray/pubsub/publisher.h" #include "ray/util/network_util.h" -#include "ray/util/util.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/gcs_server_main.cc b/src/ray/gcs/gcs_server/gcs_server_main.cc index 74c371cb4201..53f95d4f402a 100644 --- a/src/ray/gcs/gcs_server/gcs_server_main.cc +++ b/src/ray/gcs/gcs_server/gcs_server_main.cc @@ -24,9 +24,9 @@ #include "ray/gcs/store_client/redis_store_client.h" #include "ray/stats/stats.h" #include "ray/util/event.h" +#include "ray/util/raii.h" #include "ray/util/stream_redirection.h" #include "ray/util/stream_redirection_options.h" -#include "ray/util/util.h" #include "src/ray/protobuf/gcs_service.pb.h" DEFINE_string(redis_address, "", "The ip address of redis."); diff --git a/src/ray/gcs/pb_util.h b/src/ray/gcs/pb_util.h index 54b7315e504f..52f222ba443a 100644 --- a/src/ray/gcs/pb_util.h +++ b/src/ray/gcs/pb_util.h @@ -23,6 +23,7 @@ #include "ray/common/id.h" #include "ray/common/ray_config.h" #include "ray/common/task/task_spec.h" +#include "ray/util/time.h" #include "src/ray/protobuf/autoscaler.pb.h" #include "src/ray/protobuf/export_task_event.pb.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/store_client/BUILD.bazel b/src/ray/gcs/store_client/BUILD.bazel index 11bd87964695..4a2af91411b8 100644 --- a/src/ray/gcs/store_client/BUILD.bazel +++ b/src/ray/gcs/store_client/BUILD.bazel @@ -60,6 +60,5 @@ ray_cc_library( deps = [ ":store_client", "//src/ray/gcs:gcs_callback", - "//src/ray/util", ], ) diff --git a/src/ray/gcs/store_client/redis_context.cc b/src/ray/gcs/store_client/redis_context.cc index 541c5edf9cc6..7c40cab5a5ba 100644 --- a/src/ray/gcs/store_client/redis_context.cc +++ b/src/ray/gcs/store_client/redis_context.cc @@ -23,7 +23,6 @@ #include "ray/common/asio/asio_util.h" #include "ray/stats/metric_defs.h" #include "ray/util/network_util.h" -#include "ray/util/util.h" extern "C" { #include "hiredis/async.h" diff --git a/src/ray/gcs/store_client/tests/BUILD.bazel b/src/ray/gcs/store_client/tests/BUILD.bazel index 571e081c1a6c..5e48c8ef39be 100644 --- a/src/ray/gcs/store_client/tests/BUILD.bazel +++ b/src/ray/gcs/store_client/tests/BUILD.bazel @@ -25,6 +25,9 @@ ray_cc_test( deps = [ ":store_client_test_lib", "//src/ray/gcs/store_client:redis_store_client", + "//src/ray/util:network_util", + "//src/ray/util:path_utils", + "//src/ray/util:raii", "@boost//:optional", "@com_google_googletest//:gtest_main", ], @@ -53,6 +56,9 @@ ray_cc_test( deps = [ ":store_client_test_lib", "//src/ray/gcs/store_client:redis_store_client", + "//src/ray/util:network_util", + "//src/ray/util:path_utils", + "//src/ray/util:raii", "@boost//:optional", "@com_google_googletest//:gtest_main", ], @@ -110,7 +116,7 @@ ray_cc_test( deps = [ "//src/ray/common:test_util", "//src/ray/gcs/store_client:redis_store_client", - "//src/ray/util", + "//src/ray/util:raii", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/gcs/store_client/tests/redis_async_context_test.cc b/src/ray/gcs/store_client/tests/redis_async_context_test.cc index 94ebe207ee39..0bb967c55c84 100644 --- a/src/ray/gcs/store_client/tests/redis_async_context_test.cc +++ b/src/ray/gcs/store_client/tests/redis_async_context_test.cc @@ -24,6 +24,7 @@ #include "ray/gcs/store_client/redis_context.h" #include "ray/util/logging.h" #include "ray/util/path_utils.h" +#include "ray/util/raii.h" extern "C" { #include "hiredis/async.h" diff --git a/src/ray/gcs/store_client/tests/redis_store_client_test.cc b/src/ray/gcs/store_client/tests/redis_store_client_test.cc index e094f6c31cbc..0e3f0ea2a4c0 100644 --- a/src/ray/gcs/store_client/tests/redis_store_client_test.cc +++ b/src/ray/gcs/store_client/tests/redis_store_client_test.cc @@ -24,7 +24,9 @@ #include "ray/common/test_util.h" #include "ray/gcs/store_client/tests/store_client_test_base.h" +#include "ray/util/network_util.h" #include "ray/util/path_utils.h" +#include "ray/util/raii.h" using namespace std::chrono_literals; // NOLINT namespace ray { diff --git a/src/ray/ipc/BUILD.bazel b/src/ray/ipc/BUILD.bazel index 501206ad3693..38585313420b 100644 --- a/src/ray/ipc/BUILD.bazel +++ b/src/ray/ipc/BUILD.bazel @@ -56,5 +56,8 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:status", "//src/ray/flatbuffers:node_manager_generated", + "//src/ray/util:network_util", + "//src/ray/util:process", + "//src/ray/util:time", ], ) diff --git a/src/ray/ipc/client_connection.cc b/src/ray/ipc/client_connection.cc index 31b3df2c8b87..24de00f14c3f 100644 --- a/src/ray/ipc/client_connection.cc +++ b/src/ray/ipc/client_connection.cc @@ -30,8 +30,9 @@ #include "ray/common/event_stats.h" #include "ray/common/ray_config.h" +#include "ray/util/network_util.h" #include "ray/util/process.h" -#include "ray/util/util.h" +#include "ray/util/time.h" #if defined(_WIN32) #include diff --git a/src/ray/ipc/raylet_ipc_client.cc b/src/ray/ipc/raylet_ipc_client.cc index dc693d281978..0cb7b229af82 100644 --- a/src/ray/ipc/raylet_ipc_client.cc +++ b/src/ray/ipc/raylet_ipc_client.cc @@ -26,6 +26,7 @@ #include "ray/flatbuffers/node_manager_generated.h" #include "ray/ipc/client_connection.h" #include "ray/util/logging.h" +#include "ray/util/process.h" namespace { @@ -280,7 +281,20 @@ void RayletIpcClient::SubscribePlasmaReady(const ObjectID &object_id, } void ShutdownIfLocalRayletDisconnected(const Status &status) { - if (!status.ok() && IsRayletFailed(RayConfig::instance().RAYLET_PID())) { + // Check if the Raylet process is still alive. + // If we know the Raylet PID, check using that. + // Else, assume the Raylet is our parent process. + bool raylet_alive = true; + auto raylet_pid = RayConfig::instance().RAYLET_PID(); + if (!raylet_pid.empty()) { + if (!IsProcessAlive(static_cast(std::stoi(raylet_pid)))) { + raylet_alive = false; + } + } else if (!IsParentProcessAlive()) { + raylet_alive = false; + } + + if (!status.ok() && !raylet_alive) { RAY_LOG(WARNING) << "Exiting because the Raylet IPC connection failed and the local " "Raylet is dead. Status: " << status; diff --git a/src/ray/ipc/tests/BUILD.bazel b/src/ray/ipc/tests/BUILD.bazel index c6a7c9b76078..9a39013a51f9 100644 --- a/src/ray/ipc/tests/BUILD.bazel +++ b/src/ray/ipc/tests/BUILD.bazel @@ -9,6 +9,7 @@ ray_cc_test( "//src/ray/common:asio", "//src/ray/common:id", "//src/ray/ipc:client_connection", + "//src/ray/util:network_util", "@boost//:asio", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/ipc/tests/client_connection_test.cc b/src/ray/ipc/tests/client_connection_test.cc index a287518daf73..3839ccf564d9 100644 --- a/src/ray/ipc/tests/client_connection_test.cc +++ b/src/ray/ipc/tests/client_connection_test.cc @@ -24,6 +24,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/util/network_util.h" namespace ray { namespace raylet { diff --git a/src/ray/object_manager/BUILD.bazel b/src/ray/object_manager/BUILD.bazel index 55b27775f600..5ffc9b9081ed 100644 --- a/src/ray/object_manager/BUILD.bazel +++ b/src/ray/object_manager/BUILD.bazel @@ -110,6 +110,7 @@ ray_cc_library( "//src/ray/common:ray_config", "//src/ray/common:status", "@com_google_absl//absl/strings", + "@com_google_absl//absl/time", ], ) diff --git a/src/ray/object_manager/common.cc b/src/ray/object_manager/common.cc index 349ffbd8881f..d7f205a96668 100644 --- a/src/ray/object_manager/common.cc +++ b/src/ray/object_manager/common.cc @@ -17,6 +17,8 @@ #include #include "absl/strings/str_cat.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" #include "ray/common/ray_config.h" namespace ray { diff --git a/src/ray/object_manager/plasma/BUILD.bazel b/src/ray/object_manager/plasma/BUILD.bazel index 29aea9b47cd6..e3c41627e1ec 100644 --- a/src/ray/object_manager/plasma/BUILD.bazel +++ b/src/ray/object_manager/plasma/BUILD.bazel @@ -44,7 +44,6 @@ ray_cc_library( "//src/ray/common:status_or", "//src/ray/object_manager:object_manager_common", "//src/ray/protobuf:common_cc_proto", - "//src/ray/util", "//src/ray/util:compat", "//src/ray/util:visibility", "@com_google_absl//absl/container:flat_hash_map", @@ -108,7 +107,7 @@ ray_cc_library( "//src/ray/ipc:client_connection", "//src/ray/object_manager:object_manager_common", "//src/ray/stats:stats_metric", - "//src/ray/util", + "//src/ray/util:network_util", "@boost//:bind", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_set", @@ -286,6 +285,7 @@ ray_cc_library( "//src/ray/protobuf:common_cc_proto", "//src/ray/util:compat", "//src/ray/util:logging", + "//src/ray/util:process", "@com_github_google_flatbuffers//:flatbuffers", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", diff --git a/src/ray/object_manager/plasma/connection.cc b/src/ray/object_manager/plasma/connection.cc index b8c7549af91b..6ba8a102488c 100644 --- a/src/ray/object_manager/plasma/connection.cc +++ b/src/ray/object_manager/plasma/connection.cc @@ -25,6 +25,7 @@ #include "ray/object_manager/plasma/plasma_generated.h" #include "ray/object_manager/plasma/protocol.h" #include "ray/util/logging.h" +#include "ray/util/process.h" namespace plasma { diff --git a/src/ray/object_manager/plasma/create_request_queue.cc b/src/ray/object_manager/plasma/create_request_queue.cc index 3c9b9b55451c..9a9c0966a9c2 100644 --- a/src/ray/object_manager/plasma/create_request_queue.cc +++ b/src/ray/object_manager/plasma/create_request_queue.cc @@ -22,7 +22,6 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/object_manager/plasma/common.h" -#include "ray/util/util.h" namespace plasma { diff --git a/src/ray/object_manager/plasma/store.cc b/src/ray/object_manager/plasma/store.cc index d08fc2f8e427..790dbdf5b33a 100644 --- a/src/ray/object_manager/plasma/store.cc +++ b/src/ray/object_manager/plasma/store.cc @@ -50,7 +50,7 @@ #include "ray/object_manager/plasma/plasma_allocator.h" #include "ray/object_manager/plasma/protocol.h" #include "ray/stats/metric_defs.h" -#include "ray/util/util.h" +#include "ray/util/network_util.h" namespace ph = boost::placeholders; namespace fb = plasma::flatbuf; @@ -78,7 +78,7 @@ PlasmaStore::PlasmaStore(instrumented_io_context &main_service, ray::DeleteObjectCallback delete_object_callback) : io_context_(main_service), socket_name_(socket_name), - acceptor_(main_service, ParseUrlEndpoint(socket_name)), + acceptor_(main_service, ray::ParseUrlEndpoint(socket_name)), socket_(main_service), allocator_(allocator), fs_monitor_(fs_monitor), diff --git a/src/ray/object_manager/plasma/tests/BUILD.bazel b/src/ray/object_manager/plasma/tests/BUILD.bazel index 423cfac1a128..3b53f4011d06 100644 --- a/src/ray/object_manager/plasma/tests/BUILD.bazel +++ b/src/ray/object_manager/plasma/tests/BUILD.bazel @@ -5,6 +5,7 @@ ray_cc_test( srcs = ["fallback_allocator_test.cc"], tags = ["team:core"], deps = [ + "//src/ray/common:id", "//src/ray/object_manager/plasma:plasma_allocator", "@com_google_absl//absl/strings:str_format", "@com_google_googletest//:gtest_main", diff --git a/src/ray/object_manager/plasma/tests/fallback_allocator_test.cc b/src/ray/object_manager/plasma/tests/fallback_allocator_test.cc index b947dfb6729b..addfabfee759 100644 --- a/src/ray/object_manager/plasma/tests/fallback_allocator_test.cc +++ b/src/ray/object_manager/plasma/tests/fallback_allocator_test.cc @@ -27,7 +27,8 @@ namespace plasma { namespace { const int64_t kMB = 1024 * 1024; std::string CreateTestDir() { - path directory = std::filesystem::temp_directory_path() / GenerateUUIDV4(); + path directory = + std::filesystem::temp_directory_path() / ray::UniqueID::FromRandom().Hex(); create_directories(directory); return directory.string(); } diff --git a/src/ray/object_manager/plasma/tests/mutable_object_test.cc b/src/ray/object_manager/plasma/tests/mutable_object_test.cc index 327595f9214f..7fda37e2c6a7 100644 --- a/src/ray/object_manager/plasma/tests/mutable_object_test.cc +++ b/src/ray/object_manager/plasma/tests/mutable_object_test.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include #include diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index 6c3669449f46..e2601e02ba74 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -9,7 +9,6 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:ray_config", "//src/ray/protobuf:gcs_cc_proto", - "//src/ray/util", "//src/ray/util:event", "//src/ray/util:logging", "//src/ray/util:process", @@ -63,7 +62,6 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:task_common", "//src/ray/raylet/scheduling:cluster_resource_scheduler", - "//src/ray/util", "//src/ray/util:container_util", "@com_google_absl//absl/container:flat_hash_map", ], @@ -116,6 +114,7 @@ ray_cc_library( "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/ipc:client_connection", "//src/ray/util:network_util", + "//src/ray/util:time", "@boost//:system", "@com_google_absl//absl/strings", ], @@ -134,6 +133,8 @@ ray_cc_library( "//src/ray/protobuf:gcs_cc_proto", "//src/ray/protobuf:runtime_env_agent_cc_proto", "//src/ray/util:logging", + "//src/ray/util:process", + "//src/ray/util:time", "@boost//:beast", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/strings:str_format", @@ -167,6 +168,7 @@ ray_cc_library( "//src/ray/protobuf:node_manager_cc_proto", "//src/ray/pubsub:subscriber_interface", "//src/ray/rpc:core_worker_client", + "//src/ray/util:time", ], ) @@ -240,6 +242,7 @@ ray_cc_library( "//src/ray/util:container_util", "//src/ray/util:network_util", "//src/ray/util:throttler", + "//src/ray/util:time", "@boost//:system", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", @@ -261,6 +264,7 @@ ray_cc_library( "//src/ray/common:asio", "//src/ray/object_manager", "//src/ray/util:network_util", + "//src/ray/util:time", "@boost//:asio", ], ) @@ -285,8 +289,10 @@ ray_cc_binary( "//src/ray/util:cmd_line_utils", "//src/ray/util:event", "//src/ray/util:process", + "//src/ray/util:raii", "//src/ray/util:stream_redirection", "//src/ray/util:stream_redirection_options", + "//src/ray/util:time", "@com_github_gflags_gflags//:gflags", "@nlohmann_json", ], diff --git a/src/ray/raylet/agent_manager.cc b/src/ray/raylet/agent_manager.cc index ebf55761812a..26e142d824a1 100644 --- a/src/ray/raylet/agent_manager.cc +++ b/src/ray/raylet/agent_manager.cc @@ -23,7 +23,6 @@ #include "ray/util/logging.h" #include "ray/util/process.h" #include "ray/util/thread_utils.h" -#include "ray/util/util.h" namespace ray { namespace raylet { diff --git a/src/ray/raylet/agent_manager.h b/src/ray/raylet/agent_manager.h index 30fc60f024a2..a220bc515471 100644 --- a/src/ray/raylet/agent_manager.h +++ b/src/ray/raylet/agent_manager.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include diff --git a/src/ray/raylet/local_object_manager.cc b/src/ray/raylet/local_object_manager.cc index 6530bd08653b..a7100a828c96 100644 --- a/src/ray/raylet/local_object_manager.cc +++ b/src/ray/raylet/local_object_manager.cc @@ -22,7 +22,6 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/stats/metric_defs.h" -#include "ray/util/util.h" namespace ray { diff --git a/src/ray/raylet/local_object_manager.h b/src/ray/raylet/local_object_manager.h index d26851bbb364..b04604eaa5b8 100644 --- a/src/ray/raylet/local_object_manager.h +++ b/src/ray/raylet/local_object_manager.h @@ -30,6 +30,7 @@ #include "ray/raylet/local_object_manager_interface.h" #include "ray/raylet/worker_pool.h" #include "ray/rpc/worker/core_worker_client_pool.h" +#include "ray/util/time.h" namespace ray { diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 7e5f55839d95..04cfe875968c 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -38,9 +38,11 @@ #include "ray/util/cmd_line_utils.h" #include "ray/util/event.h" #include "ray/util/process.h" +#include "ray/util/raii.h" #include "ray/util/stream_redirection.h" #include "ray/util/stream_redirection_options.h" #include "ray/util/subreaper.h" +#include "ray/util/time.h" #include "scheduling/cluster_task_manager.h" using json = nlohmann::json; @@ -852,7 +854,7 @@ int main(int argc, char *argv[]) { drain_request->reason() == ray::rpc::autoscaler::DrainNodeReason::DRAIN_NODE_REASON_PREEMPTION && drain_request->deadline_timestamp_ms() != 0 && - drain_request->deadline_timestamp_ms() < current_sys_time_ms()) { + drain_request->deadline_timestamp_ms() < ray::current_sys_time_ms()) { node_death_info.set_reason(ray::rpc::NodeDeathInfo::AUTOSCALER_DRAIN_PREEMPTED); node_death_info.set_reason_message(drain_request->reason_message()); } else { diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index c87731497c26..684eb7c93daf 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -51,7 +51,8 @@ #include "ray/util/cmd_line_utils.h" #include "ray/util/event.h" #include "ray/util/network_util.h" -#include "ray/util/util.h" +#include "ray/util/string_utils.h" +#include "ray/util/time.h" namespace { @@ -343,7 +344,7 @@ void NodeManager::RegisterGcs() { [this] { std::stringstream debug_msg; debug_msg << DebugString() << "\n\n"; - RAY_LOG(INFO) << AppendToEachLine(debug_msg.str(), "[state-dump] "); + RAY_LOG(INFO) << PrependToEachLine(debug_msg.str(), "[state-dump] "); ReportWorkerOOMKillStats(); }, event_stats_print_interval_ms, diff --git a/src/ray/raylet/placement_group_resource_manager.h b/src/ray/raylet/placement_group_resource_manager.h index 4439bf17c392..76dc72e5a244 100644 --- a/src/ray/raylet/placement_group_resource_manager.h +++ b/src/ray/raylet/placement_group_resource_manager.h @@ -24,7 +24,6 @@ #include "ray/common/placement_group.h" #include "ray/common/scheduling/resource_set.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" -#include "ray/util/util.h" namespace ray { diff --git a/src/ray/raylet/raylet.cc b/src/ray/raylet/raylet.cc index 0079aa73b7a4..27a1c8c1a04b 100644 --- a/src/ray/raylet/raylet.cc +++ b/src/ray/raylet/raylet.cc @@ -29,7 +29,7 @@ #include "ray/object_manager/object_manager.h" #include "ray/object_manager/ownership_object_directory.h" #include "ray/util/network_util.h" -#include "ray/util/util.h" +#include "ray/util/time.h" namespace { diff --git a/src/ray/raylet/runtime_env_agent_client.cc b/src/ray/raylet/runtime_env_agent_client.cc index 28d090f57965..4934f3da36cb 100644 --- a/src/ray/raylet/runtime_env_agent_client.cc +++ b/src/ray/raylet/runtime_env_agent_client.cc @@ -29,6 +29,8 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/status.h" #include "ray/util/logging.h" +#include "ray/util/process.h" +#include "ray/util/time.h" #include "src/ray/protobuf/runtime_env_agent.pb.h" namespace beast = boost::beast; // from diff --git a/src/ray/raylet/tests/BUILD.bazel b/src/ray/raylet/tests/BUILD.bazel index 71350f2d30ad..d85e2e164fea 100644 --- a/src/ray/raylet/tests/BUILD.bazel +++ b/src/ray/raylet/tests/BUILD.bazel @@ -33,6 +33,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/raylet:worker_pool", "//src/ray/util:path_utils", + "//src/ray/util:raii", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/raylet/tests/worker_pool_test.cc b/src/ray/raylet/tests/worker_pool_test.cc index af7df9dcd114..490619ebfa4b 100644 --- a/src/ray/raylet/tests/worker_pool_test.cc +++ b/src/ray/raylet/tests/worker_pool_test.cc @@ -34,6 +34,7 @@ #include "ray/raylet/runtime_env_agent_client.h" #include "ray/util/path_utils.h" #include "ray/util/process.h" +#include "ray/util/raii.h" #include "src/ray/protobuf/runtime_env_agent.pb.h" using json = nlohmann::json; diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index 76335cf95924..182070e8777b 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -36,7 +36,7 @@ #include "ray/stats/metric_defs.h" #include "ray/util/logging.h" #include "ray/util/network_util.h" -#include "ray/util/util.h" +#include "ray/util/time.h" DEFINE_stats(worker_register_time_ms, "end to end latency of register a worker process.", diff --git a/src/ray/rpc/node_manager/raylet_client_pool.cc b/src/ray/rpc/node_manager/raylet_client_pool.cc index 013a69055eae..8c85a8457bb9 100644 --- a/src/ray/rpc/node_manager/raylet_client_pool.cc +++ b/src/ray/rpc/node_manager/raylet_client_pool.cc @@ -18,8 +18,6 @@ #include #include -#include "ray/util/util.h" - namespace ray { namespace rpc { diff --git a/src/ray/stats/BUILD.bazel b/src/ray/stats/BUILD.bazel index 3187c1043ca6..cdabcc11f1d7 100644 --- a/src/ray/stats/BUILD.bazel +++ b/src/ray/stats/BUILD.bazel @@ -15,7 +15,6 @@ ray_cc_library( deps = [ "//src/ray/common:ray_config", "//src/ray/observability:open_telemetry_metric_recorder", - "//src/ray/util", "//src/ray/util:logging", "//src/ray/util:size_literals", "@com_github_jupp0r_prometheus_cpp//pull", diff --git a/src/ray/stats/metric_exporter.h b/src/ray/stats/metric_exporter.h index a2f00914a620..b05444cf6370 100644 --- a/src/ray/stats/metric_exporter.h +++ b/src/ray/stats/metric_exporter.h @@ -24,7 +24,6 @@ #include "ray/rpc/metrics_agent_client.h" #include "ray/stats/metric.h" #include "ray/util/logging.h" -#include "ray/util/util.h" namespace ray { namespace stats { diff --git a/src/ray/util/BUILD.bazel b/src/ray/util/BUILD.bazel index 938a46b0b8d1..749c95438b70 100644 --- a/src/ray/util/BUILD.bazel +++ b/src/ray/util/BUILD.bazel @@ -112,6 +112,8 @@ ray_cc_library( deps = [ ":logging", ":mutex_protected", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", ], ) @@ -123,7 +125,7 @@ ray_cc_library( ":logging", ":random", ":string_utils", - ":timestamp_utils", + ":time", "//src/ray/protobuf:event_cc_proto", "//src/ray/protobuf:export_event_cc_proto", "@boost//:asio", @@ -145,8 +147,9 @@ ray_cc_library( ) ray_cc_library( - name = "timestamp_utils", - hdrs = ["timestamp_utils.h"], + name = "time", + srcs = ["time.cc"], + hdrs = ["time.h"], ) ray_cc_library( @@ -219,26 +222,19 @@ ray_cc_library( srcs = ["network_util.cc"], hdrs = ["network_util.h"], deps = [ + ":filesystem", + ":string_utils", "@boost//:asio", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ], ) -# TODO(hjiang): Split URL related functions into a separate util target. ray_cc_library( - name = "util", - srcs = ["util.cc"], - hdrs = ["util.h"], - deps = [ - ":filesystem", - ":logging", - ":macros", - ":process", - ":string_utils", - "//src/ray/thirdparty:sha256", - "@boost//:asio", - "@com_google_absl//absl/container:flat_hash_map", - ], + name = "raii", + hdrs = ["raii.h"], + deps = [], ) ray_cc_library( @@ -261,8 +257,8 @@ ray_cc_library( name = "shared_lru", hdrs = ["shared_lru.h"], deps = [ + ":logging", ":map_utils", - ":util", "@com_google_absl//absl/container:flat_hash_map", ], ) @@ -288,16 +284,19 @@ ray_cc_library( hdrs = ["pipe_logger.h"], deps = [ ":compat", + ":logging", ":spdlog_fd_sink", ":spdlog_newliner_sink", ":stream_redirection_options", ":thread_utils", - ":util", "//src/ray/common:ray_config", + "//src/ray/common:status", "@boost//:iostreams", "@com_github_spdlog//:spdlog", "@com_google_absl//absl/container:inlined_vector", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/synchronization", ], ) @@ -309,9 +308,8 @@ ray_cc_library( ":pipe_logger", ":scoped_dup2_wrapper", ":stream_redirection_options", - ":util", "//src/ray/util/internal:stream_redirection_handle", - "@com_google_absl//absl/container:inlined_vector", + "@com_google_absl//absl/container:flat_hash_map", ], ) @@ -320,7 +318,7 @@ ray_cc_library( hdrs = ["spdlog_fd_sink.h"], deps = [ ":compat", - ":util", + "//src/ray/common:status", "@com_github_spdlog//:spdlog", ], ) @@ -330,7 +328,6 @@ ray_cc_library( hdrs = ["spdlog_newliner_sink.h"], deps = [ ":compat", - ":util", "@com_github_spdlog//:spdlog", ], ) @@ -340,7 +337,7 @@ ray_cc_library( srcs = ["temporary_directory.cc"], hdrs = ["temporary_directory.h"], deps = [ - ":util", + "//src/ray/common:id", "@com_google_absl//absl/strings:str_format", ], ) diff --git a/src/ray/util/event.cc b/src/ray/util/event.cc index e704c8684738..bcb4422c6502 100644 --- a/src/ray/util/event.cc +++ b/src/ray/util/event.cc @@ -25,7 +25,7 @@ #include "ray/util/random.h" #include "ray/util/string_utils.h" -#include "ray/util/timestamp_utils.h" +#include "ray/util/time.h" using json = nlohmann::json; diff --git a/src/ray/util/internal/tests/BUILD.bazel b/src/ray/util/internal/tests/BUILD.bazel index 59c88140efc6..a3de8e2dddf9 100644 --- a/src/ray/util/internal/tests/BUILD.bazel +++ b/src/ray/util/internal/tests/BUILD.bazel @@ -12,9 +12,11 @@ ray_cc_test( "no_tsan", ], deps = [ + "//src/ray/common:id", "//src/ray/common/tests:testing", - "//src/ray/util", + "//src/ray/util:filesystem", "//src/ray/util/internal:stream_redirection_handle", + "@com_google_absl//absl/strings:str_format", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/util/internal/tests/stream_redirection_handle_test.cc b/src/ray/util/internal/tests/stream_redirection_handle_test.cc index fd963af8d1c1..d47ce0d5296e 100644 --- a/src/ray/util/internal/tests/stream_redirection_handle_test.cc +++ b/src/ray/util/internal/tests/stream_redirection_handle_test.cc @@ -22,14 +22,17 @@ #include #include +#include "absl/strings/str_format.h" +#include "ray/common/id.h" #include "ray/common/tests/testing.h" #include "ray/util/filesystem.h" -#include "ray/util/util.h" namespace ray::internal { namespace { +inline std::string RandomID() { return UniqueID::FromRandom().Hex(); } + // Output logging files to cleanup at process termination. std::vector log_files; void CleanupOutputLogFiles() { @@ -54,7 +57,7 @@ TEST(LoggingUtilTest, WriteContentWithNewliner) { constexpr std::string_view kLogLine1 = "hello\n"; constexpr std::string_view kLogLine2 = "world\n"; - const std::string test_file_path = absl::StrFormat("%s.err", GenerateUUIDV4()); + const std::string test_file_path = absl::StrFormat("%s.err", RandomID()); const std::string log_file_path1 = test_file_path; const std::string log_file_path2 = absl::StrFormat("%s.1", test_file_path); log_files.emplace_back(log_file_path1); @@ -100,7 +103,7 @@ TEST(LoggingUtilTest, WriteContentWithFlush) { constexpr std::string_view kLogLine1 = "hello"; constexpr std::string_view kLogLine2 = "world"; - const std::string test_file_path = absl::StrFormat("%s.err", GenerateUUIDV4()); + const std::string test_file_path = absl::StrFormat("%s.err", RandomID()); const std::string log_file_path1 = test_file_path; const std::string log_file_path2 = absl::StrFormat("%s.1", test_file_path); log_files.emplace_back(log_file_path1); diff --git a/src/ray/util/network_util.cc b/src/ray/util/network_util.cc index 1a3c8a9c65af..2364a42265c4 100644 --- a/src/ray/util/network_util.cc +++ b/src/ray/util/network_util.cc @@ -16,11 +16,22 @@ #include #include +#include +#ifndef _WIN32 +#include +#endif +#include +#include #include #include +#include +#include "absl/strings/match.h" #include "absl/strings/str_format.h" +#include "ray/util/filesystem.h" +#include "ray/util/string_utils.h" +using boost::asio::io_context; using boost::asio::ip::tcp; namespace ray { @@ -62,7 +73,7 @@ std::optional> ParseAddress(const std::string &addres } bool CheckPortFree(int port) { - boost::asio::io_context io_service; + io_context io_service; tcp::socket socket(io_service); socket.open(tcp::v4()); boost::system::error_code ec; @@ -71,4 +82,125 @@ bool CheckPortFree(int port) { return !ec.failed(); } +std::string EndpointToUrl( + const boost::asio::generic::basic_endpoint &ep, + bool include_scheme) { + std::string result, scheme; + switch (ep.protocol().family()) { + case AF_INET: { + scheme = "tcp://"; + tcp::endpoint e(tcp::v4(), 0); + RAY_CHECK_EQ(e.size(), ep.size()); + const sockaddr *src = ep.data(); + sockaddr *dst = e.data(); + *reinterpret_cast(dst) = *reinterpret_cast(src); + std::ostringstream ss; + ss << e; + result = ss.str(); + break; + } + case AF_INET6: { + scheme = "tcp://"; + tcp::endpoint e(tcp::v6(), 0); + RAY_CHECK_EQ(e.size(), ep.size()); + const sockaddr *src = ep.data(); + sockaddr *dst = e.data(); + *reinterpret_cast(dst) = *reinterpret_cast(src); + std::ostringstream ss; + ss << e; + result = ss.str(); + break; + } +#if defined(BOOST_ASIO_HAS_LOCAL_SOCKETS) && !defined(_WIN32) + case AF_UNIX: + scheme = "unix://"; + result.append(reinterpret_cast(ep.data())->sun_path, + ep.size() - offsetof(sockaddr_un, sun_path)); + break; +#endif + default: + RAY_LOG(FATAL) << "unsupported protocol family: " << ep.protocol().family(); + break; + } + if (include_scheme) { + result.insert(0, scheme); + } + return result; +} + +boost::asio::generic::basic_endpoint +ParseUrlEndpoint(const std::string &endpoint, int default_port) { + // Syntax reference: https://en.wikipedia.org/wiki/URL#Syntax + // Note that we're a bit more flexible, to allow parsing "127.0.0.1" as a URL. + boost::asio::generic::stream_protocol::endpoint result; + std::string address = endpoint, scheme; + if (absl::StartsWith(address, "unix://")) { + scheme = "unix://"; + address.erase(0, scheme.size()); + } else if (!address.empty() && ray::IsDirSep(address[0])) { + scheme = "unix://"; + } else if (absl::StartsWith(address, "tcp://")) { + scheme = "tcp://"; + address.erase(0, scheme.size()); + } else { + scheme = "tcp://"; + } + if (scheme == "unix://") { +#if defined(BOOST_ASIO_HAS_LOCAL_SOCKETS) && !defined(_WIN32) + size_t maxlen = sizeof(sockaddr_un().sun_path) / sizeof(*sockaddr_un().sun_path) - 1; + RAY_CHECK(address.size() <= maxlen) + << "AF_UNIX path length cannot exceed " << maxlen << " bytes: " << address; + result = boost::asio::local::stream_protocol::endpoint(address); +#else + RAY_LOG(FATAL) << "UNIX-domain socket endpoints are not supported: " << endpoint; +#endif + } else if (scheme == "tcp://") { + std::string::const_iterator i = address.begin(); + std::string host = ScanToken(i, "[%*[^][/]]"); + host = host.empty() ? ScanToken(i, "%*[^/:]") : host.substr(1, host.size() - 2); + std::string port_str = ScanToken(i, ":%*d"); + int port = port_str.empty() ? default_port : std::stoi(port_str.substr(1)); + result = tcp::endpoint(boost::asio::ip::make_address(host), port); + } else { + RAY_LOG(FATAL) << "Unable to parse socket endpoint: " << endpoint; + } + return result; +} + +std::shared_ptr> ParseURL(std::string url) { + auto result = std::make_shared>(); + std::string delimiter = "?"; + size_t pos = 0; + pos = url.find(delimiter); + if (pos == std::string::npos) { + return result; + } + + const std::string base_url = url.substr(0, pos); + result->emplace("url", base_url); + url.erase(0, pos + delimiter.length()); + const std::string query_delimeter = "&"; + + auto parse_key_value_with_equal_delimter = + [](std::string_view key_value) -> std::pair { + // Parse the query key value pair. + const std::string key_value_delimter = "="; + size_t key_value_pos = key_value.find(key_value_delimter); + std::string_view key = key_value.substr(0, key_value_pos); + return std::make_pair(key, key_value.substr(key.size() + 1)); + }; + + while ((pos = url.find(query_delimeter)) != std::string::npos) { + std::string_view token = std::string_view{url}.substr(0, pos); + auto key_value_pair = parse_key_value_with_equal_delimter(token); + result->emplace(std::string(key_value_pair.first), + std::string(key_value_pair.second)); + url.erase(0, pos + delimiter.length()); + } + std::string_view token = std::string_view{url}.substr(0, pos); + auto key_value_pair = parse_key_value_with_equal_delimter(token); + result->emplace(std::string(key_value_pair.first), std::string(key_value_pair.second)); + return result; +} + } // namespace ray diff --git a/src/ray/util/network_util.h b/src/ray/util/network_util.h index 5ca3b48bde45..c10a062b6089 100644 --- a/src/ray/util/network_util.h +++ b/src/ray/util/network_util.h @@ -15,9 +15,21 @@ #pragma once #include +#include #include #include +#include "absl/container/flat_hash_map.h" + +// Boost forward-declarations (to avoid forcing slow header inclusions) +namespace boost::asio::generic { + +template +class basic_endpoint; +class stream_protocol; + +} // namespace boost::asio::generic + namespace ray { /// Build a network address string from host and port. @@ -43,4 +55,28 @@ std::optional> ParseAddress(const std::string &addres /// \return true if the port is available, false otherwise. bool CheckPortFree(int port); +/// Converts the given endpoint (such as TCP or UNIX domain socket address) to a string. +/// \param include_scheme Whether to include the scheme prefix (such as tcp://). +/// This is recommended to avoid later ambiguity when parsing. +std::string EndpointToUrl( + const boost::asio::generic::basic_endpoint &ep, + bool include_scheme = true); + +/// Parses the endpoint socket address of a URL. +/// If a scheme:// prefix is absent, the address family is guessed automatically. +/// For TCP/IP, the endpoint comprises the IP address and port number in the URL. +/// For UNIX domain sockets, the endpoint comprises the socket path. +boost::asio::generic::basic_endpoint +ParseUrlEndpoint(const std::string &endpoint, int default_port = 0); + +/// Parse the url and return a pair of base_url and query string map. +/// EX) http://abc?num_objects=9&offset=8388878 +/// will be returned as +/// { +/// url: http://abc, +/// num_objects: 9, +/// offset: 8388878 +/// } +std::shared_ptr> ParseURL(std::string url); + } // namespace ray diff --git a/src/ray/util/pipe_logger.cc b/src/ray/util/pipe_logger.cc index d16a6589d666..a440dffada3f 100644 --- a/src/ray/util/pipe_logger.cc +++ b/src/ray/util/pipe_logger.cc @@ -30,7 +30,9 @@ #include #include "absl/container/inlined_vector.h" +#include "absl/strings/str_format.h" #include "absl/strings/str_split.h" +#include "absl/synchronization/mutex.h" #include "ray/common/ray_config.h" #include "ray/util/spdlog_fd_sink.h" #include "ray/util/spdlog_newliner_sink.h" diff --git a/src/ray/util/pipe_logger.h b/src/ray/util/pipe_logger.h index 538b925fbdaf..9715403528a1 100644 --- a/src/ray/util/pipe_logger.h +++ b/src/ray/util/pipe_logger.h @@ -25,9 +25,9 @@ #include #include +#include "ray/common/status.h" #include "ray/util/compat.h" #include "ray/util/stream_redirection_options.h" -#include "ray/util/util.h" #include "spdlog/logger.h" namespace ray { diff --git a/src/ray/util/process.cc b/src/ray/util/process.cc index 3412b2d5f902..b1bb78ee77cd 100644 --- a/src/ray/util/process.cc +++ b/src/ray/util/process.cc @@ -767,6 +767,11 @@ std::optional> GetAllProcsWithPpid(pid_t parent_pid) { #endif } +void QuickExit() { + ray::RayLog::ShutDownRayLog(); + _Exit(1); +} + } // namespace ray namespace std { diff --git a/src/ray/util/process.h b/src/ray/util/process.h index b0773811e50a..e222a6f7bfcb 100644 --- a/src/ray/util/process.h +++ b/src/ray/util/process.h @@ -31,6 +31,7 @@ #include #include "ray/util/compat.h" +#include "ray/util/logging.h" #ifndef PID_MAX_LIMIT // This is defined by Linux to be the maximum allowable number of processes @@ -156,6 +157,9 @@ std::optional KillProc(pid_t pid); // Currently only supported on Linux. Returns nullopt on other platforms. std::optional> GetAllProcsWithPpid(pid_t parent_pid); +/// Terminate the process without cleaning up the resources. +void QuickExit(); + } // namespace ray // We only define operators required by the standard library (==, hash): diff --git a/src/ray/util/raii.h b/src/ray/util/raii.h new file mode 100644 index 000000000000..c9baf921e5cd --- /dev/null +++ b/src/ray/util/raii.h @@ -0,0 +1,43 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +class InitShutdownRAII { + public: + /// Type of the Shutdown function. + using ShutdownFunc = void (*)(); + + /// Create an instance of InitShutdownRAII which will call shutdown + /// function when it is out of scope. + /// + /// \param init_func The init function. + /// \param shutdown_func The shutdown function. + /// \param args The arguments for the init function. + template + InitShutdownRAII(InitFunc init_func, ShutdownFunc shutdown_func, Args &&...args) + : shutdown_(shutdown_func) { + init_func(args...); + } + + /// Destructor of InitShutdownRAII which will call the shutdown function. + ~InitShutdownRAII() { + if (shutdown_ != nullptr) { + shutdown_(); + } + } + + private: + ShutdownFunc shutdown_; +}; diff --git a/src/ray/util/spdlog_fd_sink.h b/src/ray/util/spdlog_fd_sink.h index c307b85e2b4d..9dd0249512e7 100644 --- a/src/ray/util/spdlog_fd_sink.h +++ b/src/ray/util/spdlog_fd_sink.h @@ -16,8 +16,8 @@ #include +#include "ray/common/status.h" #include "ray/util/compat.h" -#include "ray/util/util.h" namespace ray { diff --git a/src/ray/util/spdlog_newliner_sink.h b/src/ray/util/spdlog_newliner_sink.h index 11aa5234bc26..9a8570743d10 100644 --- a/src/ray/util/spdlog_newliner_sink.h +++ b/src/ray/util/spdlog_newliner_sink.h @@ -24,7 +24,6 @@ #include "absl/strings/str_split.h" #include "ray/util/compat.h" -#include "ray/util/util.h" namespace ray { diff --git a/src/ray/util/stream_redirection.cc b/src/ray/util/stream_redirection.cc index b806196079b4..04d3b5c54623 100644 --- a/src/ray/util/stream_redirection.cc +++ b/src/ray/util/stream_redirection.cc @@ -23,18 +23,14 @@ #include #include -#include "absl/container/inlined_vector.h" +#include "absl/container/flat_hash_map.h" #include "ray/util/compat.h" #include "ray/util/internal/stream_redirection_handle.h" -#include "ray/util/util.h" namespace ray { namespace { -// TODO(hjiang): Revisit later, should be able to save some heap allocation with -// absl::InlinedVector. -// // Maps from original stream file fd (i.e. stdout/stderr) to its stream redirector. absl::flat_hash_map redirection_file_handles; diff --git a/src/ray/util/string_utils.cc b/src/ray/util/string_utils.cc index ffd03b7065fc..b042cb058075 100644 --- a/src/ray/util/string_utils.cc +++ b/src/ray/util/string_utils.cc @@ -40,4 +40,17 @@ std::string ScanToken(std::string::const_iterator &c_str, std::string format) { } return result; } + +std::string PrependToEachLine(const std::string &str, const std::string &prefix) { + std::stringstream ss; + ss << prefix; + for (char c : str) { + ss << c; + if (c == '\n') { + ss << prefix; + } + } + return ss.str(); +} + } // namespace ray diff --git a/src/ray/util/string_utils.h b/src/ray/util/string_utils.h index db1b9279a71a..557176bdb6da 100644 --- a/src/ray/util/string_utils.h +++ b/src/ray/util/string_utils.h @@ -98,4 +98,7 @@ StatusOr StringToInt(const std::string &input) noexcept { return StatusOr(value); } +// Prepend the prefix to each line of str. +std::string PrependToEachLine(const std::string &str, const std::string &prefix); + } // namespace ray diff --git a/src/ray/util/temporary_directory.cc b/src/ray/util/temporary_directory.cc index a6803e8ea365..1b3cc40780e0 100644 --- a/src/ray/util/temporary_directory.cc +++ b/src/ray/util/temporary_directory.cc @@ -17,7 +17,7 @@ #include #include -#include "ray/util/util.h" +#include "ray/common/id.h" namespace ray { @@ -25,7 +25,7 @@ ScopedTemporaryDirectory::ScopedTemporaryDirectory(const std::string &dir) { temporary_directory_ = dir.empty() ? std::filesystem::temp_directory_path() : std::filesystem::path{dir}; // Manually generate a directory name by appending UUID. - temporary_directory_ = temporary_directory_ / GenerateUUIDV4(); + temporary_directory_ = temporary_directory_ / UniqueID::FromRandom().Hex(); RAY_CHECK(std::filesystem::create_directory(temporary_directory_)); } ScopedTemporaryDirectory::~ScopedTemporaryDirectory() { diff --git a/src/ray/util/tests/BUILD.bazel b/src/ray/util/tests/BUILD.bazel index f9d38609354c..20c5edd2f103 100644 --- a/src/ray/util/tests/BUILD.bazel +++ b/src/ray/util/tests/BUILD.bazel @@ -5,7 +5,6 @@ ray_cc_test( srcs = ["array_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/util", "//src/ray/util:array", "@com_google_googletest//:gtest_main", ], @@ -16,7 +15,6 @@ ray_cc_test( srcs = ["function_traits_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/util", "//src/ray/util:function_traits", "@com_google_googletest//:gtest_main", ], @@ -40,7 +38,6 @@ ray_cc_test( linkstatic = True, tags = ["team:core"], deps = [ - "//src/ray/util", "//src/ray/util:container_util", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -54,7 +51,6 @@ ray_cc_test( srcs = ["counter_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/util", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], @@ -72,7 +68,6 @@ ray_cc_test( deps = [ "//src/ray/common:ray_config", "//src/ray/protobuf:gcs_cc_proto", - "//src/ray/util", "//src/ray/util:event", "//src/ray/util:path_utils", "@boost//:range", @@ -86,7 +81,6 @@ ray_cc_test( srcs = ["exponential_backoff_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/util", "//src/ray/util:exponential_backoff", "@com_google_googletest//:gtest_main", ], @@ -120,9 +114,10 @@ ray_cc_test( ], deps = [ "//src/ray/common:status", - "//src/ray/util", "//src/ray/util:env", + "//src/ray/util:filesystem", "//src/ray/util:path_utils", + "//src/ray/util:time", "@boost//:asio", "@com_google_absl//absl/strings:str_format", "@com_google_googletest//:gtest_main", @@ -147,7 +142,7 @@ ray_cc_test( srcs = ["sequencer_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/util", + "//src/ray/util:logging", "//src/ray/util:sequencer", "@com_google_googletest//:gtest_main", ], @@ -159,9 +154,9 @@ ray_cc_test( srcs = ["signal_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/util", "//src/ray/util:logging", "//src/ray/util:path_utils", + "//src/ray/util:raii", "@com_google_googletest//:gtest_main", ], ) @@ -172,7 +167,6 @@ ray_cc_test( srcs = ["throttler_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/util", "//src/ray/util:throttler", "@com_google_absl//absl/time", "@com_google_googletest//:gtest_main", @@ -180,13 +174,12 @@ ray_cc_test( ) ray_cc_test( - name = "util_test", + name = "process_test", size = "small", - srcs = ["util_test.cc"], + srcs = ["process_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/util", - "@boost//:asio", + "//src/ray/util:process", "@boost//:process", "@com_google_googletest//:gtest_main", ], @@ -254,11 +247,13 @@ ray_cc_test( srcs = ["pipe_logger_test.cc"], tags = ["team:core"], deps = [ + "//src/ray/common:id", "//src/ray/common/tests:testing", - "//src/ray/util", + "//src/ray/util:filesystem", "//src/ray/util:pipe_logger", "//src/ray/util:scoped_env_setter", "//src/ray/util:temporary_directory", + "@com_google_absl//absl/strings:str_format", "@com_google_googletest//:gtest_main", ], ) @@ -276,9 +271,11 @@ ray_cc_test( "no_tsan", ], deps = [ + "//src/ray/common:id", "//src/ray/common/tests:testing", - "//src/ray/util", + "//src/ray/util:filesystem", "//src/ray/util:stream_redirection", + "@com_google_absl//absl/strings:str_format", "@com_google_googletest//:gtest_main", ], ) @@ -312,11 +309,13 @@ ray_cc_test( srcs = ["spdlog_newliner_sink_test.cc"], tags = ["team:core"], deps = [ + "//src/ray/common:id", "//src/ray/common/tests:testing", "//src/ray/util:filesystem", "//src/ray/util:spdlog_fd_sink", "//src/ray/util:spdlog_newliner_sink", "//src/ray/util:temporary_directory", + "@com_google_absl//absl/strings:str_format", "@com_google_googletest//:gtest_main", ], ) @@ -349,10 +348,11 @@ ray_cc_test( srcs = ["process_cleanup_test.cc"], tags = ["team:core"], deps = [ + "//src/ray/common:id", "//src/ray/common/tests:testing", - "//src/ray/util", "//src/ray/util:filesystem", "//src/ray/util:process_cleaner", + "@com_google_absl//absl/strings:str_format", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/util/tests/logging_test.cc b/src/ray/util/tests/logging_test.cc index bd320bd98f7f..ab82497afccf 100644 --- a/src/ray/util/tests/logging_test.cc +++ b/src/ray/util/tests/logging_test.cc @@ -30,7 +30,7 @@ #include "ray/util/env.h" #include "ray/util/filesystem.h" #include "ray/util/path_utils.h" -#include "ray/util/util.h" +#include "ray/util/time.h" using namespace testing; // NOLINT using json = nlohmann::json; diff --git a/src/ray/util/tests/network_util_test.cc b/src/ray/util/tests/network_util_test.cc index f1033a574fb4..f8d18fee804f 100644 --- a/src/ray/util/tests/network_util_test.cc +++ b/src/ray/util/tests/network_util_test.cc @@ -14,6 +14,11 @@ #include "ray/util/network_util.h" +#include +#include +#include +#include + #include "gtest/gtest.h" namespace ray { @@ -74,4 +79,38 @@ TEST(NetworkUtilTest, TestParseAddress) { ASSERT_FALSE(result.has_value()); } +TEST(NetworkUtilTest, UrlIpTcpParseTest) { + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("tcp://[::1]:1/", 0), false), "[::1]:1"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("tcp://[::1]/", 0), false), "[::1]:0"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("tcp://[::1]:1", 0), false), "[::1]:1"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("tcp://[::1]", 0), false), "[::1]:0"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("tcp://127.0.0.1:1/", 0), false), + "127.0.0.1:1"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("tcp://127.0.0.1/", 0), false), "127.0.0.1:0"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("tcp://127.0.0.1:1", 0), false), + "127.0.0.1:1"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("tcp://127.0.0.1", 0), false), "127.0.0.1:0"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("[::1]:1/", 0), false), "[::1]:1"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("[::1]/", 0), false), "[::1]:0"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("[::1]:1", 0), false), "[::1]:1"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("[::1]", 0), false), "[::1]:0"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("127.0.0.1:1/", 0), false), "127.0.0.1:1"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("127.0.0.1/", 0), false), "127.0.0.1:0"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("127.0.0.1:1", 0), false), "127.0.0.1:1"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("127.0.0.1", 0), false), "127.0.0.1:0"); +#ifndef _WIN32 + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("unix:///tmp/sock"), false), "/tmp/sock"); + ASSERT_EQ(EndpointToUrl(ParseUrlEndpoint("/tmp/sock"), false), "/tmp/sock"); +#endif +} + +TEST(NetworkUtilTest, ParseURLTest) { + const std::string url = "http://abc?num_objects=9&offset=8388878&size=8388878"; + auto parsed_url = *ParseURL(url); + ASSERT_EQ(parsed_url["url"], "http://abc"); + ASSERT_EQ(parsed_url["num_objects"], "9"); + ASSERT_EQ(parsed_url["offset"], "8388878"); + ASSERT_EQ(parsed_url["size"], "8388878"); +} + } // namespace ray diff --git a/src/ray/util/tests/pipe_logger_test.cc b/src/ray/util/tests/pipe_logger_test.cc index d6749e6ee545..40b80894f035 100644 --- a/src/ray/util/tests/pipe_logger_test.cc +++ b/src/ray/util/tests/pipe_logger_test.cc @@ -23,16 +23,19 @@ #include #include +#include "absl/strings/str_format.h" +#include "ray/common/id.h" #include "ray/common/tests/testing.h" #include "ray/util/filesystem.h" #include "ray/util/scoped_env_setter.h" #include "ray/util/temporary_directory.h" -#include "ray/util/util.h" namespace ray { namespace { +inline std::string RandomID() { return UniqueID::FromRandom().Hex(); } + constexpr std::string_view kLogLine1 = "hello\n"; constexpr std::string_view kLogLine2 = "world\n"; @@ -43,7 +46,7 @@ TEST_P(PipeLoggerTest, RedirectionTest) { ScopedEnvSetter scoped_env_setter{"RAY_pipe_logger_read_buf_size", pipe_buffer_size.data()}; ScopedTemporaryDirectory scoped_directory; - const auto test_file_path = scoped_directory.GetDirectory() / GenerateUUIDV4(); + const auto test_file_path = scoped_directory.GetDirectory() / RandomID(); // Take the default option, which doesn't have rotation enabled. StreamRedirectionOption stream_redirection_opt{}; @@ -65,7 +68,7 @@ TEST_P(PipeLoggerTest, RedirectionWithTee) { ScopedEnvSetter scoped_env_setter{"RAY_pipe_logger_read_buf_size", pipe_buffer_size.data()}; ScopedTemporaryDirectory scoped_directory; - const auto test_file_path = scoped_directory.GetDirectory() / GenerateUUIDV4(); + const auto test_file_path = scoped_directory.GetDirectory() / RandomID(); StreamRedirectionOption stream_redirection_opt{}; stream_redirection_opt.file_path = test_file_path.string(); @@ -94,7 +97,7 @@ TEST_P(PipeLoggerTest, RotatedRedirectionWithTee) { ScopedEnvSetter scoped_env_setter{"RAY_pipe_logger_read_buf_size", pipe_buffer_size.data()}; ScopedTemporaryDirectory scoped_directory; - const auto uuid = GenerateUUIDV4(); + const auto uuid = RandomID(); const auto test_file_path = scoped_directory.GetDirectory() / uuid; const auto log_file_path1 = test_file_path; const auto log_file_path2 = @@ -139,7 +142,7 @@ TEST_P(PipeLoggerTest, CompatibilityTest) { { constexpr std::string_view kContent = "hello"; ScopedTemporaryDirectory scoped_directory; - const auto test_file_path = scoped_directory.GetDirectory() / GenerateUUIDV4(); + const auto test_file_path = scoped_directory.GetDirectory() / RandomID(); StreamRedirectionOption logging_option{}; logging_option.file_path = test_file_path.string(); @@ -165,7 +168,7 @@ TEST_P(PipeLoggerTest, CompatibilityTest) { { constexpr std::string_view kContent = "hello\n"; ScopedTemporaryDirectory scoped_directory; - const auto test_file_path = scoped_directory.GetDirectory() / GenerateUUIDV4(); + const auto test_file_path = scoped_directory.GetDirectory() / RandomID(); StreamRedirectionOption logging_option{}; logging_option.file_path = test_file_path.string(); @@ -190,7 +193,7 @@ TEST_P(PipeLoggerTest, CompatibilityTest) { { constexpr std::string_view kContent = "hello\nworld"; ScopedTemporaryDirectory scoped_directory; - const auto test_file_path = scoped_directory.GetDirectory() / GenerateUUIDV4(); + const auto test_file_path = scoped_directory.GetDirectory() / RandomID(); StreamRedirectionOption logging_option{}; logging_option.file_path = test_file_path.string(); @@ -216,7 +219,7 @@ TEST_P(PipeLoggerTest, CompatibilityTest) { { constexpr std::string_view kContent = "hello\nworld\n"; ScopedTemporaryDirectory scoped_directory; - const auto test_file_path = scoped_directory.GetDirectory() / GenerateUUIDV4(); + const auto test_file_path = scoped_directory.GetDirectory() / RandomID(); StreamRedirectionOption logging_option{}; logging_option.file_path = test_file_path.string(); @@ -241,7 +244,7 @@ TEST_P(PipeLoggerTest, CompatibilityTest) { { constexpr std::string_view kContent = "helloworld\n\n\n"; ScopedTemporaryDirectory scoped_directory; - const auto test_file_path = scoped_directory.GetDirectory() / GenerateUUIDV4(); + const auto test_file_path = scoped_directory.GetDirectory() / RandomID(); StreamRedirectionOption logging_option{}; logging_option.file_path = test_file_path.string(); @@ -266,7 +269,7 @@ TEST_P(PipeLoggerTest, CompatibilityTest) { { constexpr std::string_view kContent = "hello\n\n\nworld"; ScopedTemporaryDirectory scoped_directory; - const auto test_file_path = scoped_directory.GetDirectory() / GenerateUUIDV4(); + const auto test_file_path = scoped_directory.GetDirectory() / RandomID(); StreamRedirectionOption logging_option{}; logging_option.file_path = test_file_path.string(); @@ -292,7 +295,7 @@ TEST_P(PipeLoggerTest, CompatibilityTest) { { constexpr std::string_view kContent = "hello\n\nworld\n\n"; ScopedTemporaryDirectory scoped_directory; - const auto test_file_path = scoped_directory.GetDirectory() / GenerateUUIDV4(); + const auto test_file_path = scoped_directory.GetDirectory() / RandomID(); StreamRedirectionOption logging_option{}; logging_option.file_path = test_file_path.string(); diff --git a/src/ray/util/tests/process_cleanup_test.cc b/src/ray/util/tests/process_cleanup_test.cc index 6e9f07652f20..bf3c1fdb2370 100644 --- a/src/ray/util/tests/process_cleanup_test.cc +++ b/src/ray/util/tests/process_cleanup_test.cc @@ -23,10 +23,11 @@ #include #include +#include "absl/strings/str_format.h" +#include "ray/common/id.h" #include "ray/common/tests/testing.h" #include "ray/util/filesystem.h" #include "ray/util/process_cleaner.h" -#include "ray/util/util.h" namespace ray { @@ -34,7 +35,7 @@ namespace { TEST(ProcessCleanerTest, BasicTest) { const std::string kTestFname = - absl::StrFormat("/tmp/process_cleanup_%s", GenerateUUIDV4()); + absl::StrFormat("/tmp/process_cleanup_%s", UniqueID::FromRandom().Hex()); auto test_func = [fname = kTestFname]() { std::fstream f{fname, std::ios::app | std::ios::out}; f << "helloworld"; diff --git a/src/ray/util/tests/util_test.cc b/src/ray/util/tests/process_test.cc similarity index 52% rename from src/ray/util/tests/util_test.cc rename to src/ray/util/tests/process_test.cc index b32e0cc3dfa9..f58e25679748 100644 --- a/src/ray/util/tests/util_test.cc +++ b/src/ray/util/tests/process_test.cc @@ -12,62 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/util/util.h" +#include "ray/util/process.h" + +#include +#include -#include #include #include #include -#include #include #include -#include "gmock/gmock.h" -#include "gtest/gtest.h" #include "ray/util/logging.h" -#include "ray/util/process.h" - -using namespace std::chrono_literals; // NOLINT namespace ray { -template -static std::string to_str(const T &obj, bool include_scheme) { - return EndpointToUrl(obj, include_scheme); -} - -TEST(UtilTest, UrlIpTcpParseTest) { - ASSERT_EQ(to_str(ParseUrlEndpoint("tcp://[::1]:1/", 0), false), "[::1]:1"); - ASSERT_EQ(to_str(ParseUrlEndpoint("tcp://[::1]/", 0), false), "[::1]:0"); - ASSERT_EQ(to_str(ParseUrlEndpoint("tcp://[::1]:1", 0), false), "[::1]:1"); - ASSERT_EQ(to_str(ParseUrlEndpoint("tcp://[::1]", 0), false), "[::1]:0"); - ASSERT_EQ(to_str(ParseUrlEndpoint("tcp://127.0.0.1:1/", 0), false), "127.0.0.1:1"); - ASSERT_EQ(to_str(ParseUrlEndpoint("tcp://127.0.0.1/", 0), false), "127.0.0.1:0"); - ASSERT_EQ(to_str(ParseUrlEndpoint("tcp://127.0.0.1:1", 0), false), "127.0.0.1:1"); - ASSERT_EQ(to_str(ParseUrlEndpoint("tcp://127.0.0.1", 0), false), "127.0.0.1:0"); - ASSERT_EQ(to_str(ParseUrlEndpoint("[::1]:1/", 0), false), "[::1]:1"); - ASSERT_EQ(to_str(ParseUrlEndpoint("[::1]/", 0), false), "[::1]:0"); - ASSERT_EQ(to_str(ParseUrlEndpoint("[::1]:1", 0), false), "[::1]:1"); - ASSERT_EQ(to_str(ParseUrlEndpoint("[::1]", 0), false), "[::1]:0"); - ASSERT_EQ(to_str(ParseUrlEndpoint("127.0.0.1:1/", 0), false), "127.0.0.1:1"); - ASSERT_EQ(to_str(ParseUrlEndpoint("127.0.0.1/", 0), false), "127.0.0.1:0"); - ASSERT_EQ(to_str(ParseUrlEndpoint("127.0.0.1:1", 0), false), "127.0.0.1:1"); - ASSERT_EQ(to_str(ParseUrlEndpoint("127.0.0.1", 0), false), "127.0.0.1:0"); -#ifndef _WIN32 - ASSERT_EQ(to_str(ParseUrlEndpoint("unix:///tmp/sock"), false), "/tmp/sock"); - ASSERT_EQ(to_str(ParseUrlEndpoint("/tmp/sock"), false), "/tmp/sock"); -#endif -} - -TEST(UtilTest, ParseURLTest) { - const std::string url = "http://abc?num_objects=9&offset=8388878&size=8388878"; - auto parsed_url = *ParseURL(url); - ASSERT_EQ(parsed_url["url"], "http://abc"); - ASSERT_EQ(parsed_url["num_objects"], "9"); - ASSERT_EQ(parsed_url["offset"], "8388878"); - ASSERT_EQ(parsed_url["size"], "8388878"); -} - TEST(UtilTest, IsProcessAlive) { namespace bp = boost::process; bp::child c("bash"); @@ -75,7 +34,7 @@ TEST(UtilTest, IsProcessAlive) { c.join(); for (int i = 0; i < 5; ++i) { if (IsProcessAlive(pid)) { - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(std::chrono::seconds(1)); } else { break; } diff --git a/src/ray/util/tests/signal_test.cc b/src/ray/util/tests/signal_test.cc index 23324fa07991..9e37b41fbab2 100644 --- a/src/ray/util/tests/signal_test.cc +++ b/src/ray/util/tests/signal_test.cc @@ -14,14 +14,16 @@ #include +#include #include #include #include +#include #include "gtest/gtest.h" #include "ray/util/logging.h" #include "ray/util/path_utils.h" -#include "ray/util/util.h" +#include "ray/util/raii.h" // This test just print some call stack information. namespace ray { diff --git a/src/ray/util/tests/spdlog_newliner_sink_test.cc b/src/ray/util/tests/spdlog_newliner_sink_test.cc index 84ed7438b21a..565c90d57cfd 100644 --- a/src/ray/util/tests/spdlog_newliner_sink_test.cc +++ b/src/ray/util/tests/spdlog_newliner_sink_test.cc @@ -21,6 +21,8 @@ #include #include +#include "absl/strings/str_format.h" +#include "ray/common/id.h" #include "ray/common/tests/testing.h" #include "ray/util/compat.h" #include "ray/util/filesystem.h" @@ -32,6 +34,8 @@ namespace ray { namespace { +inline std::string RandomID() { return ray::UniqueID::FromRandom().Hex(); } + std::shared_ptr CreateLogger() { auto fd_formatter = std::make_unique( "%v", spdlog::pattern_time_type::local, std::string("")); @@ -182,7 +186,7 @@ TEST(NewlinerSinkWithFileinkTest, AppendAndFlushTest) { // Case-1: string with newliner at the end. { - const auto filepath = (dir.GetDirectory() / GenerateUUIDV4()).string(); + const auto filepath = (dir.GetDirectory() / RandomID()).string(); auto logger = CreateLogger(filepath); constexpr std::string_view kContent = "hello\n"; @@ -199,7 +203,7 @@ TEST(NewlinerSinkWithFileinkTest, AppendAndFlushTest) { // Case-2: string with no newliner at the end. { - const auto filepath = (dir.GetDirectory() / GenerateUUIDV4()).string(); + const auto filepath = (dir.GetDirectory() / RandomID()).string(); auto logger = CreateLogger(filepath); constexpr std::string_view kContent = "hello"; @@ -218,7 +222,7 @@ TEST(NewlinerSinkWithFileinkTest, AppendAndFlushTest) { // Case-3: newliner in the middle, with trailing newliner. { - const auto filepath = (dir.GetDirectory() / GenerateUUIDV4()).string(); + const auto filepath = (dir.GetDirectory() / RandomID()).string(); auto logger = CreateLogger(filepath); constexpr std::string_view kContent = "hello\nworld\n"; @@ -235,7 +239,7 @@ TEST(NewlinerSinkWithFileinkTest, AppendAndFlushTest) { // // Case-4: newliner in the middle, without trailing newliner. { - const auto filepath = (dir.GetDirectory() / GenerateUUIDV4()).string(); + const auto filepath = (dir.GetDirectory() / RandomID()).string(); auto logger = CreateLogger(filepath); constexpr std::string_view kContent = "hello\nworld"; @@ -254,7 +258,7 @@ TEST(NewlinerSinkWithFileinkTest, AppendAndFlushTest) { // // Case-5: multiple writes. { - const auto filepath = (dir.GetDirectory() / GenerateUUIDV4()).string(); + const auto filepath = (dir.GetDirectory() / RandomID()).string(); auto logger = CreateLogger(filepath); constexpr std::string_view kContent1 = "hello\nworld"; constexpr std::string_view kContent2 = "hello\nworld\n"; diff --git a/src/ray/util/tests/stream_redirection_exit_test.cc b/src/ray/util/tests/stream_redirection_exit_test.cc index 1644f06462aa..f753d2a1a71e 100644 --- a/src/ray/util/tests/stream_redirection_exit_test.cc +++ b/src/ray/util/tests/stream_redirection_exit_test.cc @@ -21,10 +21,11 @@ #include #include +#include "absl/strings/str_format.h" +#include "ray/common/id.h" #include "ray/common/tests/testing.h" #include "ray/util/filesystem.h" #include "ray/util/stream_redirection.h" -#include "ray/util/util.h" namespace ray { @@ -34,7 +35,8 @@ constexpr std::string_view kLogLine2 = "world"; } // namespace TEST(LoggingUtilTest, RedirectStderr) { - const std::string test_file_path = absl::StrFormat("%s.err", GenerateUUIDV4()); + const std::string test_file_path = + absl::StrFormat("%s.err", UniqueID::FromRandom().Hex()); // Works via `dup`, so have to execute before we redirect via `dup2` and close stderr. testing::internal::CaptureStderr(); diff --git a/src/ray/util/timestamp_utils.h b/src/ray/util/time.cc similarity index 59% rename from src/ray/util/timestamp_utils.h rename to src/ray/util/time.cc index 69d034cb9ceb..c305ca04e42a 100644 --- a/src/ray/util/timestamp_utils.h +++ b/src/ray/util/time.cc @@ -12,16 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#pragma once - -#include +#include "ray/util/time.h" namespace ray { -inline int64_t current_sys_time_s() { - std::chrono::seconds s_since_epoch = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()); - return s_since_epoch.count(); +std::optional ToTimeoutPoint(int64_t timeout_ms) { + std::optional timeout_point; + if (timeout_ms == -1) { + return timeout_point; + } + auto now = std::chrono::steady_clock::now(); + auto timeout_duration = std::chrono::milliseconds(timeout_ms); + timeout_point.emplace(now + timeout_duration); + return timeout_point; } } // namespace ray diff --git a/src/ray/util/time.h b/src/ray/util/time.h new file mode 100644 index 000000000000..473d2cb69356 --- /dev/null +++ b/src/ray/util/time.h @@ -0,0 +1,51 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +namespace ray { + +/// Return the number of milliseconds since the steady clock epoch. NOTE: The +/// returned timestamp may be used for accurately measuring intervals but has +/// no relation to wall clock time. It must not be used for synchronization +/// across multiple nodes. +inline int64_t current_time_ms() { + std::chrono::milliseconds ms_since_epoch = + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()); + return ms_since_epoch.count(); +} + +inline int64_t current_sys_time_ms() { + std::chrono::milliseconds ms_since_epoch = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()); + return ms_since_epoch.count(); +} + +inline int64_t current_sys_time_s() { + std::chrono::seconds s_since_epoch = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()); + return s_since_epoch.count(); +} + +/// Converts a timeout in milliseconds to a timeout point. +/// \param timeout_ms The timeout in milliseconds. +/// \return The timeout point, or std::nullopt if timeout_ms is -1. +std::optional ToTimeoutPoint(int64_t timeout_ms); + +} // namespace ray diff --git a/src/ray/util/util.cc b/src/ray/util/util.cc deleted file mode 100644 index c01f16b74952..000000000000 --- a/src/ray/util/util.cc +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2020 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ray/util/util.h" - -#include -#include -#ifndef _WIN32 -#include -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#ifndef _WIN32 -#include -#endif -#include - -#include "absl/strings/match.h" -#include "ray/util/filesystem.h" -#include "ray/util/logging.h" -#include "ray/util/process.h" -#include "ray/util/string_utils.h" - -std::string EndpointToUrl( - const boost::asio::generic::basic_endpoint &ep, - bool include_scheme) { - std::string result, scheme; - switch (ep.protocol().family()) { - case AF_INET: { - scheme = "tcp://"; - boost::asio::ip::tcp::endpoint e(boost::asio::ip::tcp::v4(), 0); - RAY_CHECK_EQ(e.size(), ep.size()); - const sockaddr *src = ep.data(); - sockaddr *dst = e.data(); - *reinterpret_cast(dst) = *reinterpret_cast(src); - std::ostringstream ss; - ss << e; - result = ss.str(); - break; - } - case AF_INET6: { - scheme = "tcp://"; - boost::asio::ip::tcp::endpoint e(boost::asio::ip::tcp::v6(), 0); - RAY_CHECK_EQ(e.size(), ep.size()); - const sockaddr *src = ep.data(); - sockaddr *dst = e.data(); - *reinterpret_cast(dst) = *reinterpret_cast(src); - std::ostringstream ss; - ss << e; - result = ss.str(); - break; - } -#if defined(BOOST_ASIO_HAS_LOCAL_SOCKETS) && !defined(_WIN32) - case AF_UNIX: - scheme = "unix://"; - result.append(reinterpret_cast(ep.data())->sun_path, - ep.size() - offsetof(sockaddr_un, sun_path)); - break; -#endif - default: - RAY_LOG(FATAL) << "unsupported protocol family: " << ep.protocol().family(); - break; - } - if (include_scheme) { - result.insert(0, scheme); - } - return result; -} - -boost::asio::generic::basic_endpoint -ParseUrlEndpoint(const std::string &endpoint, int default_port) { - // Syntax reference: https://en.wikipedia.org/wiki/URL#Syntax - // Note that we're a bit more flexible, to allow parsing "127.0.0.1" as a URL. - boost::asio::generic::stream_protocol::endpoint result; - std::string address = endpoint, scheme; - if (absl::StartsWith(address, "unix://")) { - scheme = "unix://"; - address.erase(0, scheme.size()); - } else if (!address.empty() && ray::IsDirSep(address[0])) { - scheme = "unix://"; - } else if (absl::StartsWith(address, "tcp://")) { - scheme = "tcp://"; - address.erase(0, scheme.size()); - } else { - scheme = "tcp://"; - } - if (scheme == "unix://") { -#if defined(BOOST_ASIO_HAS_LOCAL_SOCKETS) && !defined(_WIN32) - size_t maxlen = sizeof(sockaddr_un().sun_path) / sizeof(*sockaddr_un().sun_path) - 1; - RAY_CHECK(address.size() <= maxlen) - << "AF_UNIX path length cannot exceed " << maxlen << " bytes: " << address; - result = boost::asio::local::stream_protocol::endpoint(address); -#else - RAY_LOG(FATAL) << "UNIX-domain socket endpoints are not supported: " << endpoint; -#endif - } else if (scheme == "tcp://") { - std::string::const_iterator i = address.begin(); - std::string host = ::ray::ScanToken(i, "[%*[^][/]]"); - host = - host.empty() ? ::ray::ScanToken(i, "%*[^/:]") : host.substr(1, host.size() - 2); - std::string port_str = ::ray::ScanToken(i, ":%*d"); - int port = port_str.empty() ? default_port : std::stoi(port_str.substr(1)); - result = boost::asio::ip::tcp::endpoint(boost::asio::ip::make_address(host), port); - } else { - RAY_LOG(FATAL) << "Unable to parse socket endpoint: " << endpoint; - } - return result; -} - -std::shared_ptr> ParseURL(std::string url) { - auto result = std::make_shared>(); - std::string delimiter = "?"; - size_t pos = 0; - pos = url.find(delimiter); - if (pos == std::string::npos) { - return result; - } - - const std::string base_url = url.substr(0, pos); - result->emplace("url", base_url); - url.erase(0, pos + delimiter.length()); - const std::string query_delimeter = "&"; - - auto parse_key_value_with_equal_delimter = - [](std::string_view key_value) -> std::pair { - // Parse the query key value pair. - const std::string key_value_delimter = "="; - size_t key_value_pos = key_value.find(key_value_delimter); - std::string_view key = key_value.substr(0, key_value_pos); - return std::make_pair(key, key_value.substr(key.size() + 1)); - }; - - while ((pos = url.find(query_delimeter)) != std::string::npos) { - std::string_view token = std::string_view{url}.substr(0, pos); - auto key_value_pair = parse_key_value_with_equal_delimter(token); - result->emplace(std::string(key_value_pair.first), - std::string(key_value_pair.second)); - url.erase(0, pos + delimiter.length()); - } - std::string_view token = std::string_view{url}.substr(0, pos); - auto key_value_pair = parse_key_value_with_equal_delimter(token); - result->emplace(std::string(key_value_pair.first), std::string(key_value_pair.second)); - return result; -} - -std::string GenerateUUIDV4() { - thread_local std::random_device rd; - thread_local std::mt19937 gen(rd()); - std::uniform_int_distribution<> dis(0, 15); - std::uniform_int_distribution<> dis2(8, 11); - - std::stringstream ss; - int i; - ss << std::hex; - for (i = 0; i < 8; i++) { - ss << dis(gen); - } - ss << "-"; - for (i = 0; i < 4; i++) { - ss << dis(gen); - } - ss << "-4"; - for (i = 0; i < 3; i++) { - ss << dis(gen); - } - ss << "-"; - ss << dis2(gen); - for (i = 0; i < 3; i++) { - ss << dis(gen); - } - ss << "-"; - for (i = 0; i < 12; i++) { - ss << dis(gen); - }; - return ss.str(); -} - -namespace ray { - -bool IsRayletFailed(const std::string &raylet_pid) { - auto should_shutdown = false; - if (!raylet_pid.empty()) { - auto pid = static_cast(std::stoi(raylet_pid)); - if (!IsProcessAlive(pid)) { - should_shutdown = true; - } - } else if (!IsParentProcessAlive()) { - should_shutdown = true; - } - return should_shutdown; -} - -void QuickExit() { - ray::RayLog::ShutDownRayLog(); - _Exit(1); -} - -std::optional ToTimeoutPoint(int64_t timeout_ms) { - std::optional timeout_point; - if (timeout_ms == -1) { - return timeout_point; - } - auto now = std::chrono::steady_clock::now(); - auto timeout_duration = std::chrono::milliseconds(timeout_ms); - timeout_point.emplace(now + timeout_duration); - return timeout_point; -} - -} // namespace ray diff --git a/src/ray/util/util.h b/src/ray/util/util.h deleted file mode 100644 index 8f0baecb5b1b..000000000000 --- a/src/ray/util/util.h +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#ifdef __APPLE__ -#include -#endif - -#ifdef __linux__ -#include -#endif - -#ifdef _WIN32 -#ifndef _WINDOWS_ -#ifndef WIN32_LEAN_AND_MEAN // Sorry for the inconvenience. Please include any related - // headers you need manually. - // (https://stackoverflow.com/a/8294669) -#define WIN32_LEAN_AND_MEAN // Prevent inclusion of WinSock2.h -#endif -#include // Force inclusion of WinGDI here to resolve name conflict -#endif -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "absl/container/flat_hash_map.h" -#include "ray/util/logging.h" -#include "ray/util/macros.h" - -#ifdef _WIN32 -#include // to ensure getpid() on Windows -#endif - -// Boost forward-declarations (to avoid forcing slow header inclusions) -namespace boost::asio::generic { - -template -class basic_endpoint; -class stream_protocol; - -} // namespace boost::asio::generic - -// Append append_str to the beginning of each line of str. -inline std::string AppendToEachLine(const std::string &str, - const std::string &append_str) { - std::stringstream ss; - ss << append_str; - for (char c : str) { - ss << c; - if (c == '\n') { - ss << append_str; - } - } - return ss.str(); -} - -inline int64_t current_sys_time_s() { - std::chrono::seconds s_since_epoch = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()); - return s_since_epoch.count(); -} - -/// Return the number of milliseconds since the steady clock epoch. NOTE: The -/// returned timestamp may be used for accurately measuring intervals but has -/// no relation to wall clock time. It must not be used for synchronization -/// across multiple nodes. -/// -/// TODO(rkn): This function appears in multiple places. It should be -/// deduplicated. -/// -/// \return The number of milliseconds since the steady clock epoch. -inline int64_t current_time_ms() { - std::chrono::milliseconds ms_since_epoch = - std::chrono::duration_cast( - std::chrono::steady_clock::now().time_since_epoch()); - return ms_since_epoch.count(); -} - -inline int64_t current_sys_time_ms() { - std::chrono::milliseconds ms_since_epoch = - std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()); - return ms_since_epoch.count(); -} - -inline int64_t current_sys_time_us() { - std::chrono::microseconds mu_since_epoch = - std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()); - return mu_since_epoch.count(); -} - -std::string GenerateUUIDV4(); - -/// Converts the given endpoint (such as TCP or UNIX domain socket address) to a string. -/// \param include_scheme Whether to include the scheme prefix (such as tcp://). -/// This is recommended to avoid later ambiguity when parsing. -std::string EndpointToUrl( - const boost::asio::generic::basic_endpoint &ep, - bool include_scheme = true); - -/// Parses the endpoint socket address of a URL. -/// If a scheme:// prefix is absent, the address family is guessed automatically. -/// For TCP/IP, the endpoint comprises the IP address and port number in the URL. -/// For UNIX domain sockets, the endpoint comprises the socket path. -boost::asio::generic::basic_endpoint -ParseUrlEndpoint(const std::string &endpoint, int default_port = 0); - -/// Parse the url and return a pair of base_url and query string map. -/// EX) http://abc?num_objects=9&offset=8388878 -/// will be returned as -/// { -/// url: http://abc, -/// num_objects: 9, -/// offset: 8388878 -/// } -std::shared_ptr> ParseURL(std::string url); - -class InitShutdownRAII { - public: - /// Type of the Shutdown function. - using ShutdownFunc = void (*)(); - - /// Create an instance of InitShutdownRAII which will call shutdown - /// function when it is out of scope. - /// - /// \param init_func The init function. - /// \param shutdown_func The shutdown function. - /// \param args The arguments for the init function. - template - InitShutdownRAII(InitFunc init_func, ShutdownFunc shutdown_func, Args &&...args) - : shutdown_(shutdown_func) { - init_func(args...); - } - - /// Destructor of InitShutdownRAII which will call the shutdown function. - ~InitShutdownRAII() { - if (shutdown_ != nullptr) { - shutdown_(); - } - } - - private: - ShutdownFunc shutdown_; -}; - -struct EnumClassHash { - template - std::size_t operator()(T t) const { - return static_cast(t); - } -}; - -namespace ray { - -/// Return true if the raylet is failed. This util function is only meant to be used by -/// core worker modules. -bool IsRayletFailed(const std::string &raylet_pid); - -/// Teriminate the process without cleaning up the resources. -void QuickExit(); - -/// Converts a timeout in milliseconds to a timeout point. -/// \param[in] timeout_ms The timeout in milliseconds. -/// \return The timeout point, or std::nullopt if timeout_ms is -1. -std::optional ToTimeoutPoint(int64_t timeout_ms); - -} // namespace ray From 95c297ab6086259d2307bdf77f6b3fcca3674bc8 Mon Sep 17 00:00:00 2001 From: matthewdeng Date: Wed, 20 Aug 2025 21:25:19 -0700 Subject: [PATCH 212/634] [train] fix test_jax_trainer imports (#55799) Signed-off-by: Matthew Deng --- python/ray/train/v2/tests/test_jax_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/train/v2/tests/test_jax_trainer.py b/python/ray/train/v2/tests/test_jax_trainer.py index a6449577181b..cc77f03b1ae6 100644 --- a/python/ray/train/v2/tests/test_jax_trainer.py +++ b/python/ray/train/v2/tests/test_jax_trainer.py @@ -2,8 +2,8 @@ import ray from ray.tests.conftest import _ray_start_cluster +from ray.train import RunConfig, ScalingConfig from ray.train.v2._internal.constants import HEALTH_CHECK_INTERVAL_S_ENV_VAR -from ray.train.v2.api.config import RunConfig, ScalingConfig from ray.train.v2.jax import JaxTrainer From 5dea97a65a8a1a536aa8eca69fa0c4f0f720002e Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Thu, 21 Aug 2025 00:02:57 -0700 Subject: [PATCH 213/634] [Core] Remove dead code (#55805) Signed-off-by: Jiajun Yao --- .../gcs_server/gcs_redis_failure_detector.h | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 src/mock/ray/gcs/gcs_server/gcs_redis_failure_detector.h diff --git a/src/mock/ray/gcs/gcs_server/gcs_redis_failure_detector.h b/src/mock/ray/gcs/gcs_server/gcs_redis_failure_detector.h deleted file mode 100644 index d3b5948df8e7..000000000000 --- a/src/mock/ray/gcs/gcs_server/gcs_redis_failure_detector.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace ray { -namespace gcs { - -class MockGcsRedisFailureDetector : public GcsRedisFailureDetector { - public: -}; - -} // namespace gcs -} // namespace ray From d92a3234654c3a102292548cc15be3b64ef5da0f Mon Sep 17 00:00:00 2001 From: Artur Niederfahrenhorst Date: Thu, 21 Aug 2025 18:13:31 +0200 Subject: [PATCH 214/634] [RLlib] Add StepFailedRecreateEnv exception (#55146) ## Why are these changes needed? There exist environments that are inherently unstable (for example because they rely on a system that is not under the control of the user). This PR introduces a special exception that a user-defined environment can raise in case the external environment fails and requires a reset. RLlib catches the exception and recreates the environment that raised said exception without logging an error. --- .../rllib/package_ref/env/env_runner.rst | 7 +++ rllib/algorithms/algorithm_config.py | 6 +- rllib/env/env_errors.py | 18 ++++++ rllib/env/env_runner.py | 14 +++-- .../env/tests/test_single_agent_env_runner.py | 56 ++++++++++++++++++- 5 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 rllib/env/env_errors.py diff --git a/doc/source/rllib/package_ref/env/env_runner.rst b/doc/source/rllib/package_ref/env/env_runner.rst index b1f7fb8401ad..a47bd2256e25 100644 --- a/doc/source/rllib/package_ref/env/env_runner.rst +++ b/doc/source/rllib/package_ref/env/env_runner.rst @@ -45,6 +45,13 @@ Cleanup EnvRunner.stop +rllib.env.env_errors.StepFailedRecreateEnvError +------------------------------------------------ + +.. currentmodule:: ray.rllib.env.env_errors + +.. autoclass:: StepFailedRecreateEnvError + Single-agent and multi-agent EnvRunners --------------------------------------- diff --git a/rllib/algorithms/algorithm_config.py b/rllib/algorithms/algorithm_config.py index 98fe844d7a4a..018554b8540d 100644 --- a/rllib/algorithms/algorithm_config.py +++ b/rllib/algorithms/algorithm_config.py @@ -3794,9 +3794,11 @@ def fault_tolerance( True). restart_failed_sub_environments: If True and any sub-environment (within a vectorized env) throws any error during env stepping, the - Sampler tries to restart the faulty sub-environment. This is done + EnvRunner tries to restart the faulty sub-environment. This is done without disturbing the other (still intact) sub-environment and without - the EnvRunner crashing. + the EnvRunner crashing. You can raise + `ray.rllib.env.env_runner.StepFailedRecreateEnvError` from your + environment's `step` method to not log the error. num_consecutive_env_runner_failures_tolerance: The number of consecutive times an EnvRunner failure (also for evaluation) is tolerated before finally crashing the Algorithm. Only useful if either diff --git a/rllib/env/env_errors.py b/rllib/env/env_errors.py new file mode 100644 index 000000000000..cb52892db1da --- /dev/null +++ b/rllib/env/env_errors.py @@ -0,0 +1,18 @@ +"""Error classes for RLlib environment operations.""" + +from ray.util.annotations import PublicAPI + + +@PublicAPI(stability="alpha") +class StepFailedRecreateEnvError(Exception): + """An exception that signals that the environment step failed and the environment needs to be reset. + + This exception may be raised by the environment's `step` method. + It is then caught by the `EnvRunner` and the environment is reset. + This can be useful if your environment is unstable, regularely crashing in a certain way. + For example, if you connect to an external simulator that you have little control over. + You can detect such crashes in your step method and throw this error to not log the error. + Use this with caution, as it may lead to infinite loops of resetting the environment. + """ + + pass diff --git a/rllib/env/env_runner.py b/rllib/env/env_runner.py index 7d3a7a9d488e..6da4fcaf68e0 100644 --- a/rllib/env/env_runner.py +++ b/rllib/env/env_runner.py @@ -7,6 +7,7 @@ import ray from ray.rllib.core import COMPONENT_RL_MODULE +from ray.rllib.env.env_errors import StepFailedRecreateEnvError from ray.rllib.utils.actor_manager import FaultAwareApply from ray.rllib.utils.debug import update_global_seed_if_necessary from ray.rllib.utils.framework import try_import_tf @@ -25,6 +26,7 @@ ENV_RESET_FAILURE = "env_reset_failure" ENV_STEP_FAILURE = "env_step_failure" +NUM_ENV_STEP_FAILURES_LIFETIME = "num_env_step_failures" # TODO (sven): As soon as RolloutWorker is no longer supported, make this base class @@ -232,11 +234,11 @@ def _try_env_step(self, actions): results = self.env.step(actions) return results except Exception as e: + self.metrics.log_value(NUM_ENV_STEP_FAILURES_LIFETIME, 1, reduce="sum") + if self.config.restart_failed_sub_environments: - logger.exception( - "Stepping the env resulted in an error! The original error " - f"is: {e.args[0]}" - ) + if not isinstance(e, StepFailedRecreateEnvError): + logger.exception("Stepping the env resulted in an error!") # Recreate the env. self.make_env() # And return that the stepping failed. The caller will then handle @@ -244,6 +246,10 @@ def _try_env_step(self, actions): # data and repeating the step attempt). return ENV_STEP_FAILURE else: + if isinstance(e, StepFailedRecreateEnvError): + raise ValueError( + "Environment raised StepFailedRecreateEnvError but config.restart_failed_sub_environments is False." + ) from e raise e def _convert_to_tensor(self, struct) -> TensorType: diff --git a/rllib/env/tests/test_single_agent_env_runner.py b/rllib/env/tests/test_single_agent_env_runner.py index 4d5f8808aa84..0aac37bb3f83 100644 --- a/rllib/env/tests/test_single_agent_env_runner.py +++ b/rllib/env/tests/test_single_agent_env_runner.py @@ -1,12 +1,14 @@ from functools import partial -import unittest +from unittest.mock import patch +import unittest import gymnasium as gym import ray from ray import tune from ray.rllib.algorithms.algorithm_config import AlgorithmConfig from ray.rllib.env.single_agent_env_runner import SingleAgentEnvRunner +from ray.rllib.env.env_runner import StepFailedRecreateEnvError from ray.rllib.env.utils import _gym_env_creator from ray.rllib.examples.envs.classes.simple_corridor import SimpleCorridor from ray.rllib.utils.test_utils import check @@ -75,7 +77,7 @@ def test_sample(self): self.assertTrue(sum_ in [128, 129]) def test_async_vector_env(self): - """Tests, whether SingleAgentGymEnvRunner can run with vector envs.""" + """Tests, whether SingleAgentEnvRunner can run with vector envs.""" for env in ["CartPole-v1", SimpleCorridor, "tune-registered"]: config = ( @@ -101,7 +103,7 @@ def test_async_vector_env(self): env_runner.stop() def test_distributed_env_runner(self): - """Tests, whether SingleAgentGymEnvRunner can be distributed.""" + """Tests, whether SingleAgentEnvRunner can be distributed.""" remote_class = ray.remote(num_cpus=1, num_gpus=0)(SingleAgentEnvRunner) @@ -142,6 +144,54 @@ def test_distributed_env_runner(self): ], ) + @patch("ray.rllib.env.env_runner.logger") + def test_step_failed_reset_required(self, mock_logger): + """Tests, whether SingleAgentEnvRunner can handle StepFailedResetRequired.""" + # Define an env that raises StepFailedResetRequired + + class ErrorRaisingEnv(gym.Env): + def __init__(self, config=None): + # As per gymnasium standard, provide observation and action spaces in your + # constructor. + self.observation_space = gym.spaces.Discrete(2) + self.action_space = gym.spaces.Discrete(2) + self.exception_type = config["exception_type"] + + def reset(self, *, seed=None, options=None): + return self.observation_space.sample(), {} + + def step(self, action): + raise self.exception_type() + + config = ( + AlgorithmConfig() + .environment( + ErrorRaisingEnv, + env_config={"exception_type": StepFailedRecreateEnvError}, + ) + .env_runners(num_envs_per_env_runner=1, rollout_fragment_length=10) + .fault_tolerance(restart_failed_sub_environments=True) + ) + env_runner = SingleAgentEnvRunner(config=config) + + # Check that we don't log the error on the first step (because we don't raise StepFailedResetRequired) + # We need two steps because the first one naturally raises ResetNeeded because we try to step before the env is reset. + env_runner._try_env_reset() + env_runner._try_env_step(actions=[None]) + + assert mock_logger.exception.call_count == 0 + + config.environment(ErrorRaisingEnv, env_config={"exception_type": ValueError}) + + env_runner = SingleAgentEnvRunner(config=config) + + # Check that we don't log the error on the first step (because we don't raise StepFailedResetRequired) + # We need two steps because the first one naturally raises ResetNeeded because we try to step before the env is reset. + env_runner._try_env_reset() + env_runner._try_env_step(actions=[None]) + + assert mock_logger.exception.call_count == 1 + if __name__ == "__main__": import pytest From dded833ee72e4c4bab64fda31a0d9b8150960d91 Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Thu, 21 Aug 2025 23:02:25 +0530 Subject: [PATCH 215/634] [core] Fix RAY_CHECK failure during shutdown due to plasma store race condition (#55367) ## Why are these changes needed? Workers crash with a fatal `RAY_CHECK` failure when the plasma store connection is broken during shutdown, causing the following error: ``` RAY_CHECK failed: PutInLocalPlasmaStore(object, object_id, true) Status not OK: IOError: Broken pipe ``` Stacktrace: ``` core_worker.cc:720 C Check failed: PutInLocalPlasmaStore(object, object_id, true) Status not OK: IOError: Broken pipe *** StackTrace Information *** /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(+0x141789a) [0x7924dd2c689a] ray::operator<<() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray6RayLogD1Ev+0x479) [0x7924dd2c9319] ray::RayLog::~RayLog() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(+0x95cc8a) [0x7924dc80bc8a] ray::core::CoreWorker::CoreWorker()::{lambda()#13}::operator()() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core11TaskManager27MarkTaskReturnObjectsFailedERKNS_17TaskSpecificationENS_3rpc9ErrorTypeEPKNS5_12RayErrorInfoERKN4absl12lts_2023080213flat_hash_setINS_8ObjectIDENSB_13hash_internal4HashISD_EESt8equal_toISD_ESaISD_EEE+0x679) [0x7924dc868f29] ray::core::TaskManager::MarkTaskReturnObjectsFailed() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core11TaskManager15FailPendingTaskERKNS_6TaskIDENS_3rpc9ErrorTypeEPKNS_6StatusEPKNS5_12RayErrorInfoE+0x416) [0x7924dc86f186] ray::core::TaskManager::FailPendingTask() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(+0x9a90e6) [0x7924dc8580e6] ray::core::NormalTaskSubmitter::RequestNewWorkerIfNeeded()::{lambda()#1}::operator()() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray3rpc14ClientCallImplINS0_23RequestWorkerLeaseReplyEE15OnReplyReceivedEv+0x68) [0x7924dc94aa48] ray::rpc::ClientCallImpl<>::OnReplyReceived() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(_ZNSt17_Function_handlerIFvvEZN3ray3rpc17ClientCallManager29PollEventsFromCompletionQueueEiEUlvE_E9_M_invokeERKSt9_Any_data+0x15) [0x7924dc79e285] std::_Function_handler<>::_M_invoke() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(+0xd9b4c8) [0x7924dcc4a4c8] EventTracker::RecordExecution() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(+0xd4648e) [0x7924dcbf548e] std::_Function_handler<>::_M_invoke() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(+0xd46906) [0x7924dcbf5906] boost::asio::detail::completion_handler<>::do_complete() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(+0x13f417b) [0x7924dd2a317b] boost::asio::detail::scheduler::do_run_one() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(+0x13f5af9) [0x7924dd2a4af9] boost::asio::detail::scheduler::run() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(+0x13f6202) [0x7924dd2a5202] boost::asio::io_context::run() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(_ZN3ray4core10CoreWorker12RunIOServiceEv+0x91) [0x7924dc793a61] ray::core::CoreWorker::RunIOService() /home/ray/anaconda3/lib/python3.11/site-packages/ray/_raylet.so(+0xcba0b0) [0x7924dcb690b0] thread_proxy /lib/x86_64-linux-gnu/libc.so.6(+0x94ac3) [0x7924dde71ac3] /lib/x86_64-linux-gnu/libc.so.6(+0x126850) [0x7924ddf03850] ``` Stack trace flow: 1. Task lease request fails -> `NormalTaskSubmitter::RequestNewWorkerIfNeeded()` callback. 2. Triggers `TaskManager::FailPendingTask()` -> `MarkTaskReturnObjectsFailed()`. 3. System attempts to store error objects in plasma via `put_in_local_plasma_callback_`. 4. Plasma connection is broken (raylet/plasma store already shut down). 5. `RAY_CHECK_OK()` in the callback causes fatal crash instead of graceful handling. Root Cause: This is a shutdown ordering race condition: 1. Raylet shuts down first: The raylet stops its IO context ([main_service_.stop()](https://github.com/ray-project/ray/blob/77c5475195e56a26891d88460973198391d20edf/src/ray/object_manager/plasma/store_runner.cc#L146)) which closes plasma store connections. 2. Worker still processes callbacks: Core worker continues processing pending callbacks on separate threads. 3. Broken connection: When the callback tries to store error objects in plasma, the connection is already closed. 4. Fatal crash: The `RAY_CHECK_OK()` treats this as an unexpected error and crashes the process. Fix: 1. Shutdown-aware plasma operations - Add `CoreWorker::IsShuttingDown()` method to check shutdown state. - Skip plasma operations entirely when shutdown is in progress. - Prevents attempting operations on already-closed connections. 2. Targeted error handling for connection failures - Replace blanket `RAY_CHECK_OK()` with specific error type checking. - Handle connection errors (Broken pipe, Connection reset, Bad file descriptor) as warnings during shutdown scenarios. - Maintain `RAY_CHECK_OK()` for other error types to catch real issues. --------- Signed-off-by: Sagar Sumit --- src/ray/core_worker/core_worker.h | 13 + src/ray/core_worker/core_worker_process.cc | 10 +- src/ray/core_worker/task_manager.cc | 122 ++++++--- src/ray/core_worker/task_manager.h | 16 +- .../core_worker/tests/task_manager_test.cc | 254 ++++++++++++++++++ 5 files changed, 376 insertions(+), 39 deletions(-) diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index 2099d4b740ff..cffda26e8c85 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -1494,6 +1494,19 @@ class CoreWorker { std::string *application_error); /// Put an object in the local plasma store. + /// + /// Return status semantics: + /// - Status::OK(): The object was created (or already existed) and bookkeeping was + /// updated. Note: an internal ObjectExists from the plasma provider is treated + /// as OK and does not surface here. + /// - Status::ObjectStoreFull(): The local plasma store is out of memory (or out of + /// disk when spilling). The error message contains context and a short memory + /// report. + /// - Status::IOError(): IPC/connection failures while talking to the plasma store + /// (e.g., broken pipe/connection reset during shutdown, store not reachable). + /// + /// Call sites that run during shutdown may choose to tolerate IOError specifically, + /// but should treat all other statuses as real failures. Status PutInLocalPlasmaStore(const RayObject &object, const ObjectID &object_id, bool pin_object); diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index 59fe6ad14f0f..783a184ca09f 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -423,8 +423,14 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( /*put_in_local_plasma_callback=*/ [this](const RayObject &object, const ObjectID &object_id) { auto core_worker = GetCoreWorker(); - RAY_CHECK_OK( - core_worker->PutInLocalPlasmaStore(object, object_id, /*pin_object=*/true)); + auto put_status = + core_worker->PutInLocalPlasmaStore(object, object_id, /*pin_object=*/true); + if (!put_status.ok()) { + RAY_LOG(WARNING).WithField(object_id) + << "Failed to put object in plasma store: " << put_status; + return put_status; + } + return Status::OK(); }, /* retry_task_callback= */ [this](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) { diff --git a/src/ray/core_worker/task_manager.cc b/src/ray/core_worker/task_manager.cc index 70693b8f2a19..75f84cd21ad4 100644 --- a/src/ray/core_worker/task_manager.cc +++ b/src/ray/core_worker/task_manager.cc @@ -533,10 +533,10 @@ size_t TaskManager::NumPendingTasks() const { return num_pending_tasks_; } -bool TaskManager::HandleTaskReturn(const ObjectID &object_id, - const rpc::ReturnObject &return_object, - const NodeID &worker_node_id, - bool store_in_plasma) { +StatusOr TaskManager::HandleTaskReturn(const ObjectID &object_id, + const rpc::ReturnObject &return_object, + const NodeID &worker_node_id, + bool store_in_plasma) { bool direct_return = false; reference_counter_.UpdateObjectSize(object_id, return_object.size()); RAY_LOG(DEBUG) << "Task return object " << object_id << " has size " @@ -579,7 +579,15 @@ bool TaskManager::HandleTaskReturn(const ObjectID &object_id, /*copy_data=*/false, tensor_transport.value_or(rpc::TensorTransport::OBJECT_STORE)); if (store_in_plasma) { - put_in_local_plasma_callback_(object, object_id); + Status s = put_in_local_plasma_callback_(object, object_id); + int retry_count = 0; + while (!s.ok() && s.IsTransientObjectStoreFull() && retry_count < 3) { + retry_count++; + s = put_in_local_plasma_callback_(object, object_id); + } + if (!s.ok()) { + return s; + } } else { in_memory_store_.Put(object, object_id); direct_return = true; @@ -813,10 +821,15 @@ bool TaskManager::HandleReportGeneratorItemReturns( } // When an object is reported, the object is ready to be fetched. reference_counter_.UpdateObjectPendingCreation(object_id, false); - HandleTaskReturn(object_id, - return_object, - NodeID::FromBinary(request.worker_addr().node_id()), - /*store_in_plasma=*/store_in_plasma_ids.contains(object_id)); + StatusOr put_res = + HandleTaskReturn(object_id, + return_object, + NodeID::FromBinary(request.worker_addr().node_id()), + /*store_in_plasma=*/store_in_plasma_ids.contains(object_id)); + if (!put_res.ok()) { + RAY_LOG(WARNING).WithField(object_id) + << "Failed to handle streaming dynamic return: " << put_res.status(); + } } // Handle backpressure if needed. @@ -900,23 +913,54 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, reference_counter_.AddDynamicReturn(object_id, generator_id); dynamic_return_ids.push_back(object_id); } - if (!HandleTaskReturn(object_id, - return_object, - NodeID::FromBinary(worker_addr.node_id()), - store_in_plasma_ids.contains(object_id))) { - if (first_execution) { - dynamic_returns_in_plasma.push_back(object_id); + StatusOr direct_or = + HandleTaskReturn(object_id, + return_object, + NodeID::FromBinary(worker_addr.node_id()), + store_in_plasma_ids.contains(object_id)); + if (!direct_or.ok()) { + RAY_LOG(WARNING).WithField(object_id) + << "Failed to handle dynamic task return: " << direct_or.status(); + Status st = direct_or.status(); + rpc::ErrorType err_type = rpc::ErrorType::WORKER_DIED; + if (st.IsObjectStoreFull() || st.IsTransientObjectStoreFull()) { + err_type = rpc::ErrorType::OUT_OF_MEMORY; } + rpc::RayErrorInfo err_info; + err_info.set_error_message(st.ToString()); + FailOrRetryPendingTask(task_id, + err_type, + &st, + /*ray_error_info=*/&err_info, + /*mark_task_object_failed=*/true, + /*fail_immediately=*/true); + return; + } else if (!direct_or.value() && first_execution) { + dynamic_returns_in_plasma.push_back(object_id); } } } for (const auto &return_object : reply.return_objects()) { const auto object_id = ObjectID::FromBinary(return_object.object_id()); - if (HandleTaskReturn(object_id, - return_object, - NodeID::FromBinary(worker_addr.node_id()), - store_in_plasma_ids.contains(object_id))) { + StatusOr direct_or = HandleTaskReturn(object_id, + return_object, + NodeID::FromBinary(worker_addr.node_id()), + store_in_plasma_ids.contains(object_id)); + if (!direct_or.ok()) { + RAY_LOG(WARNING).WithField(object_id) + << "Failed to handle task return: " << direct_or.status(); + // If storing return in plasma failed, treat as system failure for this attempt. + // Do not proceed with normal completion. Mark task failed immediately. + Status st = direct_or.status(); + FailOrRetryPendingTask(task_id, + rpc::ErrorType::WORKER_DIED, + &st, + /*ray_error_info=*/nullptr, + /*mark_task_object_failed=*/true, + /*fail_immediately=*/true); + return; + } else if (direct_or.value()) { direct_return_ids.push_back(object_id); } } @@ -1040,10 +1084,16 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, const auto generator_return_id = spec.StreamingGeneratorReturnId(i); RAY_CHECK_EQ(reply.return_objects_size(), 1); const auto &return_object = reply.return_objects(0); - HandleTaskReturn(generator_return_id, - return_object, - NodeID::FromBinary(worker_addr.node_id()), - store_in_plasma_ids.contains(generator_return_id)); + StatusOr res = + HandleTaskReturn(generator_return_id, + return_object, + NodeID::FromBinary(worker_addr.node_id()), + store_in_plasma_ids.contains(generator_return_id)); + if (!res.ok()) { + RAY_LOG(WARNING).WithField(generator_return_id) + << "Failed to handle generator return during app error propagation: " + << res.status(); + } } } } @@ -1454,18 +1504,26 @@ void TaskManager::MarkTaskReturnObjectsFailed( int64_t num_returns = spec.NumReturns(); for (int i = 0; i < num_returns; i++) { const auto object_id = ObjectID::FromIndex(task_id, /*index=*/i + 1); + // Always place an error marker in local memory to unblock waiters quickly. + in_memory_store_.Put(error, object_id); + // Best-effort plasma put if the object was meant to be in plasma. if (store_in_plasma_ids.contains(object_id)) { - put_in_local_plasma_callback_(error, object_id); - } else { - in_memory_store_.Put(error, object_id); + Status s = put_in_local_plasma_callback_(error, object_id); + if (!s.ok()) { + RAY_LOG(WARNING).WithField(object_id) + << "Failed to put error object in plasma: " << s; + } } } if (spec.ReturnsDynamic()) { for (const auto &dynamic_return_id : spec.DynamicReturnIds()) { + in_memory_store_.Put(error, dynamic_return_id); if (store_in_plasma_ids.contains(dynamic_return_id)) { - put_in_local_plasma_callback_(error, dynamic_return_id); - } else { - in_memory_store_.Put(error, dynamic_return_id); + Status s = put_in_local_plasma_callback_(error, dynamic_return_id); + if (!s.ok()) { + RAY_LOG(WARNING).WithField(dynamic_return_id) + << "Failed to put error object in plasma: " << s; + } } } } @@ -1488,7 +1546,11 @@ void TaskManager::MarkTaskReturnObjectsFailed( for (size_t i = 0; i < num_streaming_generator_returns; i++) { const auto generator_return_id = spec.StreamingGeneratorReturnId(i); if (store_in_plasma_ids.contains(generator_return_id)) { - put_in_local_plasma_callback_(error, generator_return_id); + Status s = put_in_local_plasma_callback_(error, generator_return_id); + if (!s.ok()) { + RAY_LOG(WARNING).WithField(generator_return_id) + << "Failed to put error object in plasma: " << s; + } } else { in_memory_store_.Put(error, generator_return_id); } diff --git a/src/ray/core_worker/task_manager.h b/src/ray/core_worker/task_manager.h index e8025bc0f7e5..4b1c32e1e873 100644 --- a/src/ray/core_worker/task_manager.h +++ b/src/ray/core_worker/task_manager.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -25,6 +26,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/synchronization/mutex.h" #include "ray/common/id.h" +#include "ray/common/status.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/core_worker/task_event_buffer.h" #include "ray/core_worker/task_manager_interface.h" @@ -42,7 +44,7 @@ class ActorManager; using TaskStatusCounter = CounterMap>; using PutInLocalPlasmaCallback = - std::function; + std::function; using RetryTaskCallback = std::function; using ReconstructObjectCallback = std::function; @@ -608,12 +610,12 @@ class TaskManager : public TaskManagerInterface { ABSL_LOCKS_EXCLUDED(mu_); /// Update nested ref count info and store the in-memory value for a task's - /// return object. Returns true if the task's return object was returned - /// directly by value. - bool HandleTaskReturn(const ObjectID &object_id, - const rpc::ReturnObject &return_object, - const NodeID &worker_node_id, - bool store_in_plasma) ABSL_LOCKS_EXCLUDED(mu_); + /// return object. On success, sets direct_return_out to true if the object's value + /// was returned directly by value (not stored in plasma). + StatusOr HandleTaskReturn(const ObjectID &object_id, + const rpc::ReturnObject &return_object, + const NodeID &worker_node_id, + bool store_in_plasma) ABSL_LOCKS_EXCLUDED(mu_); /// Remove a lineage reference to this object ID. This should be called /// whenever a task that depended on this object ID can no longer be retried. diff --git a/src/ray/core_worker/tests/task_manager_test.cc b/src/ray/core_worker/tests/task_manager_test.cc index 81eee6f6ca8b..b8e4c5722253 100644 --- a/src/ray/core_worker/tests/task_manager_test.cc +++ b/src/ray/core_worker/tests/task_manager_test.cc @@ -160,6 +160,7 @@ class TaskManagerTest : public ::testing::Test { *reference_counter_, [this](const RayObject &object, const ObjectID &object_id) { stored_in_plasma.insert(object_id); + return Status::OK(); }, [this](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) { num_retries_++; @@ -1360,6 +1361,191 @@ TEST_F(TaskManagerLineageTest, TestResubmittedDynamicReturnsTaskFails) { ASSERT_EQ(stored_in_plasma.size(), 3); } +// High-level tests around plasma put failures and retries using a real memory store +TEST_F(TaskManagerTest, PlasmaPut_ObjectStoreFull_FailsTaskAndWritesError) { + auto local_ref_counter = std::make_shared( + addr_, + publisher_.get(), + subscriber_.get(), + /*is_node_dead=*/[this](const NodeID &) { return node_died_; }, + lineage_pinning_enabled_); + auto local_store = std::make_shared(io_context_.GetIoService(), + local_ref_counter.get()); + + TaskManager failing_mgr( + *local_store, + *local_ref_counter, + /*put_in_local_plasma_callback=*/ + [](const RayObject &, const ObjectID &) { + return Status::ObjectStoreFull("simulated"); + }, + [this](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) { + num_retries_++; + last_delay_ms_ = delay_ms; + last_object_recovery_ = object_recovery; + return Status::OK(); + }, + [this](const TaskSpecification &spec) { + return this->did_queue_generator_resubmit_; + }, + [](const JobID &, const std::string &, const std::string &, double) { + return Status::OK(); + }, + /*max_lineage_bytes*/ 1024 * 1024, + *task_event_buffer_mock_.get(), + [](const ActorID &) -> std::shared_ptr { + return nullptr; + }, + mock_gcs_client_); + + rpc::Address caller_address; + auto spec = CreateTaskHelper(1, {}); + failing_mgr.AddPendingTask(caller_address, spec, ""); + failing_mgr.MarkDependenciesResolved(spec.TaskId()); + failing_mgr.MarkTaskWaitingForExecution( + spec.TaskId(), NodeID::FromRandom(), WorkerID::FromRandom()); + + rpc::PushTaskReply reply; + auto return_object = reply.add_return_objects(); + auto return_id = spec.ReturnId(0); + return_object->set_object_id(return_id.Binary()); + return_object->set_in_plasma(true); + failing_mgr.CompletePendingTask( + spec.TaskId(), reply, rpc::Address(), /*app_err=*/false); + + ASSERT_FALSE(failing_mgr.IsTaskPending(spec.TaskId())); + std::vector> results; + WorkerContext ctx(WorkerType::WORKER, WorkerID::FromRandom(), JobID::FromInt(0)); + RAY_CHECK_OK(local_store->Get({return_id}, 1, 0, ctx, false, &results)); + ASSERT_EQ(results.size(), 1); + ASSERT_TRUE(results[0]->IsException()); +} + +TEST_F(TaskManagerTest, PlasmaPut_TransientFull_RetriesThenSucceeds) { + std::shared_ptr> attempts = std::make_shared>(0); + auto local_ref_counter = std::make_shared( + addr_, + publisher_.get(), + subscriber_.get(), + /*is_node_dead=*/[this](const NodeID &) { return node_died_; }, + lineage_pinning_enabled_); + auto local_store = std::make_shared(io_context_.GetIoService(), + local_ref_counter.get()); + TaskManager retry_mgr( + *local_store, + *local_ref_counter, + /*put_in_local_plasma_callback=*/ + [attempts](const RayObject &, const ObjectID &) { + int n = ++(*attempts); + if (n < 3) { + return Status::TransientObjectStoreFull("retry"); + } + return Status::OK(); + }, + [this](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) { + num_retries_++; + last_delay_ms_ = delay_ms; + last_object_recovery_ = object_recovery; + return Status::OK(); + }, + [this](const TaskSpecification &spec) { + return this->did_queue_generator_resubmit_; + }, + [](const JobID &, const std::string &, const std::string &, double) { + return Status::OK(); + }, + /*max_lineage_bytes*/ 1024 * 1024, + *task_event_buffer_mock_.get(), + [](const ActorID &) -> std::shared_ptr { + return nullptr; + }, + mock_gcs_client_); + + rpc::Address caller_address; + auto spec = CreateTaskHelper(1, {}); + retry_mgr.AddPendingTask(caller_address, spec, ""); + retry_mgr.MarkDependenciesResolved(spec.TaskId()); + retry_mgr.MarkTaskWaitingForExecution( + spec.TaskId(), NodeID::FromRandom(), WorkerID::FromRandom()); + + rpc::PushTaskReply reply; + auto return_object = reply.add_return_objects(); + auto return_id = spec.ReturnId(0); + return_object->set_object_id(return_id.Binary()); + return_object->set_in_plasma(true); + retry_mgr.CompletePendingTask(spec.TaskId(), reply, rpc::Address(), /*app_err=*/false); + + std::vector> results; + WorkerContext ctx(WorkerType::WORKER, WorkerID::FromRandom(), JobID::FromInt(0)); + RAY_CHECK_OK(local_store->Get({return_id}, 1, 0, ctx, false, &results)); + ASSERT_EQ(results.size(), 1); + ASSERT_TRUE(results[0]->IsInPlasmaError()); +} + +TEST_F(TaskManagerTest, DynamicReturn_PlasmaPutFailure_FailsTaskImmediately) { + bool first_fail_done = false; + auto local_ref_counter = std::make_shared( + addr_, + publisher_.get(), + subscriber_.get(), + /*is_node_dead=*/[this](const NodeID &) { return node_died_; }, + lineage_pinning_enabled_); + auto local_store = std::make_shared(io_context_.GetIoService(), + local_ref_counter.get()); + TaskManager dyn_mgr( + *local_store, + *local_ref_counter, + /*put_in_local_plasma_callback=*/ + [&first_fail_done](const RayObject &, const ObjectID &) { + if (!first_fail_done) { + first_fail_done = true; + return Status::IOError("broken pipe"); + } + return Status::OK(); + }, + [this](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) { + num_retries_++; + last_delay_ms_ = delay_ms; + last_object_recovery_ = object_recovery; + return Status::OK(); + }, + [this](const TaskSpecification &spec) { + return this->did_queue_generator_resubmit_; + }, + [](const JobID &, const std::string &, const std::string &, double) { + return Status::OK(); + }, + /*max_lineage_bytes*/ 1024 * 1024, + *task_event_buffer_mock_.get(), + [](const ActorID &) -> std::shared_ptr { + return nullptr; + }, + mock_gcs_client_); + + auto spec = CreateTaskHelper(1, {}, /*dynamic_returns=*/true); + dyn_mgr.AddPendingTask(addr_, spec, "", /*num_retries=*/0); + dyn_mgr.MarkDependenciesResolved(spec.TaskId()); + dyn_mgr.MarkTaskWaitingForExecution( + spec.TaskId(), NodeID::FromRandom(), WorkerID::FromRandom()); + + rpc::PushTaskReply reply; + auto generator_id = spec.ReturnId(0); + auto gen_obj = reply.add_return_objects(); + gen_obj->set_object_id(generator_id.Binary()); + auto data = GenerateRandomBuffer(); + gen_obj->set_data(data->Data(), data->Size()); + for (int i = 0; i < 2; i++) { + auto dyn_id = ObjectID::FromIndex(spec.TaskId(), i + 2); + auto dyn_obj = reply.add_dynamic_return_objects(); + dyn_obj->set_object_id(dyn_id.Binary()); + dyn_obj->set_data(data->Data(), data->Size()); + dyn_obj->set_in_plasma(true); + } + + dyn_mgr.CompletePendingTask(spec.TaskId(), reply, rpc::Address(), /*app_err=*/false); + ASSERT_FALSE(dyn_mgr.IsTaskPending(spec.TaskId())); +} + TEST_F(TaskManagerTest, TestObjectRefStreamCreateDelete) { /** * Test create and deletion of stream works. @@ -2706,6 +2892,74 @@ TEST_F(TaskManagerTest, TestTaskRetriedOnNodePreemption) { // Cleanup manager_.FailPendingTask(spec.TaskId(), rpc::ErrorType::WORKER_DIED); } + +class PlasmaShutdownRaceTest : public ::testing::Test { + public: + PlasmaShutdownRaceTest() : is_shutting_down_(false) {} + + Status SimulatePlasmaCallback(const ObjectID &object_id, bool simulate_failure) { + if (is_shutting_down_) { + skipped_operations_.insert(object_id); + return Status::OK(); + } + + if (simulate_failure) { + auto status = Status::IOError("Broken pipe"); + if (status.IsIOError() && is_shutting_down_) { + tolerated_operations_.insert(object_id); + return Status::OK(); + } else { + failed_operations_.insert(object_id); + return status; + } + } + + successful_operations_.insert(object_id); + return Status::OK(); + } + + void SetShuttingDown(bool shutting_down) { is_shutting_down_ = shutting_down; } + + protected: + bool is_shutting_down_; + std::unordered_set skipped_operations_; + std::unordered_set tolerated_operations_; + std::unordered_set successful_operations_; + std::unordered_set failed_operations_; +}; + +// Test plasma callback behavior during shutdown to prevent RAY_CHECK crashes +TEST_F(PlasmaShutdownRaceTest, PlasmaCallbackHandlesShutdownRaceCondition) { + auto object_id = ObjectID::FromRandom(); + + SetShuttingDown(false); + ASSERT_TRUE(SimulatePlasmaCallback(object_id, false).ok()); + ASSERT_EQ(successful_operations_.count(object_id), 1); + + auto object_id2 = ObjectID::FromRandom(); + auto status = SimulatePlasmaCallback(object_id2, true); + ASSERT_FALSE(status.ok()); + ASSERT_TRUE(status.IsIOError()); + ASSERT_EQ(failed_operations_.count(object_id2), 1); + + auto object_id3 = ObjectID::FromRandom(); + SetShuttingDown(true); + ASSERT_TRUE(SimulatePlasmaCallback(object_id3, false).ok()); + ASSERT_EQ(skipped_operations_.count(object_id3), 1); + + auto object_id4 = ObjectID::FromRandom(); + SetShuttingDown(false); + auto status4 = Status::IOError("Broken pipe"); + SetShuttingDown(true); + + if (status4.IsIOError() && is_shutting_down_) { + tolerated_operations_.insert(object_id4); + } else { + failed_operations_.insert(object_id4); + } + ASSERT_EQ(tolerated_operations_.count(object_id4), 1); +} + } // namespace core } // namespace ray From 0a24e7b0977db9968ac3664ceee3935a3ff75c13 Mon Sep 17 00:00:00 2001 From: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Date: Thu, 21 Aug 2025 11:05:14 -0700 Subject: [PATCH 216/634] [Serve] Resolve linux test_logging.py flakiness (#55751) `test_logging_disable_stdout` was flaky as the [log messages checked from the log file were not all existing when the check was done](https://buildkite.com/ray-project/postmerge/builds/12333#0198bece-690d-4517-9757-f1b5c13a597e/177-1416). So we allow more time for all the log messages to be written in the log file with reattempts using `wait_for_condition()`. --------- Signed-off-by: doyoung --- python/ray/serve/tests/test_logging.py | 71 ++++++++++++++------------ 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/python/ray/serve/tests/test_logging.py b/python/ray/serve/tests/test_logging.py index 01a551613a25..df6377a6acad 100644 --- a/python/ray/serve/tests/test_logging.py +++ b/python/ray/serve/tests/test_logging.py @@ -1172,39 +1172,44 @@ def disable_stdout(): httpx.get(url, timeout=None) # Check if each of the logs exist in Serve's log files. - from_serve_logger_check = False - from_print_check = False - from_error_check = False - direct_from_stdout = False - direct_from_stderr = False - multiline_log = False - for log_file in os.listdir(logs_dir): - if log_file.startswith("replica_default_disable_stdout"): - with open(logs_dir / log_file) as f: - for line in f: - structured_log = json.loads(line) - message = structured_log["message"] - exc_text = structured_log.get("exc_text", "") - if "from_serve_logger" in message: - from_serve_logger_check = True - elif "from_print" in message: - from_print_check = True - - # Error was logged from replica directly. - elif "from_error" in exc_text: - from_error_check = True - elif "direct_from_stdout" in message: - direct_from_stdout = True - elif "direct_from_stderr" in message: - direct_from_stderr = True - elif "this\nis\nmultiline\nlog\n" in message: - multiline_log = True - assert from_serve_logger_check - assert from_print_check - assert from_error_check - assert direct_from_stdout - assert direct_from_stderr - assert multiline_log + def _all_expected_logs_exist(): + from_serve_logger_check = False + from_print_check = False + from_error_check = False + direct_from_stdout = False + direct_from_stderr = False + multiline_log = False + + for log_file in os.listdir(logs_dir): + if log_file.startswith("replica_default_disable_stdout"): + with open(logs_dir / log_file) as f: + for line in f: + structured_log = json.loads(line) + message = structured_log["message"] + exc_text = structured_log.get("exc_text", "") + + if "from_serve_logger" in message: + from_serve_logger_check = True + elif "from_print" in message: + from_print_check = True + elif "from_error" in exc_text: + from_error_check = True + elif "direct_from_stdout" in message: + direct_from_stdout = True + elif "direct_from_stderr" in message: + direct_from_stderr = True + elif "this\nis\nmultiline\nlog\n" in message: + multiline_log = True + + assert from_serve_logger_check + assert from_print_check + assert from_error_check + assert direct_from_stdout + assert direct_from_stderr + assert multiline_log + return True + + wait_for_condition(_all_expected_logs_exist) @pytest.mark.skipif(sys.platform == "win32", reason="Fail to look for temp dir.") From 212c97b042b875d9bf04baa89431387342f8b7c5 Mon Sep 17 00:00:00 2001 From: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Date: Thu, 21 Aug 2025 11:05:39 -0700 Subject: [PATCH 217/634] [Serve] Move test from test failure to proxy (#55743) For better fit, moving `test_http_proxy_failure` from `test_failure.py` to `test_proxy.py` --------- Signed-off-by: doyoung --- python/ray/serve/_private/test_utils.py | 21 ++++++ .../serve/tests/test_controller_recovery.py | 15 ++-- python/ray/serve/tests/test_failure.py | 74 +++---------------- python/ray/serve/tests/test_proxy.py | 39 ++++++++++ 4 files changed, 78 insertions(+), 71 deletions(-) diff --git a/python/ray/serve/_private/test_utils.py b/python/ray/serve/_private/test_utils.py index d742b1c8a2b0..c3c9142ab475 100644 --- a/python/ray/serve/_private/test_utils.py +++ b/python/ray/serve/_private/test_utils.py @@ -9,6 +9,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union import grpc +import httpx import requests from starlette.requests import Request @@ -16,6 +17,7 @@ import ray.util.state as state_api from ray import serve from ray._common.network_utils import build_address +from ray._common.test_utils import wait_for_condition from ray.actor import ActorHandle from ray.serve._private.client import ServeControllerClient from ray.serve._private.common import ( @@ -813,3 +815,22 @@ def get_application_url( def check_running(app_name: str = SERVE_DEFAULT_APP_NAME): assert serve.status().applications[app_name].status == ApplicationStatus.RUNNING return True + + +def request_with_retries(timeout=30, app_name=SERVE_DEFAULT_APP_NAME): + result_holder = {"resp": None} + + def _attempt() -> bool: + try: + url = get_application_url("HTTP", app_name=app_name) + result_holder["resp"] = httpx.get(url, timeout=timeout) + return True + except (httpx.RequestError, IndexError): + return False + + try: + wait_for_condition(_attempt, timeout=timeout) + return result_holder["resp"] + except RuntimeError as e: + # Preserve previous API by raising TimeoutError on expiry + raise TimeoutError from e diff --git a/python/ray/serve/tests/test_controller_recovery.py b/python/ray/serve/tests/test_controller_recovery.py index b5cbf0af1637..493931872ea9 100644 --- a/python/ray/serve/tests/test_controller_recovery.py +++ b/python/ray/serve/tests/test_controller_recovery.py @@ -19,9 +19,12 @@ SERVE_NAMESPACE, SERVE_PROXY_NAME, ) -from ray.serve._private.test_utils import check_replica_counts, get_application_url +from ray.serve._private.test_utils import ( + check_replica_counts, + get_application_url, + request_with_retries, +) from ray.serve.schema import LoggingConfig, ServeDeploySchema -from ray.serve.tests.test_failure import request_with_retries from ray.util.state import list_actors @@ -51,9 +54,7 @@ def __call__(self, *args): serve.run(TransientConstructorFailureDeployment.bind(), name="app") for _ in range(10): - response = request_with_retries( - "/recover_start_from_replica_actor_names/", timeout=30, app_name="app" - ) + response = request_with_retries(timeout=30, app_name="app") assert response.text == "hii" # Assert 2 replicas are running in deployment deployment after partially # successful deploy() call with transient error @@ -96,9 +97,7 @@ def __call__(self, *args): lambda: get_application_url("HTTP", "app", use_localhost=True) is not None ) for _ in range(10): - response = request_with_retries( - "/recover_start_from_replica_actor_names/", timeout=30, app_name="app" - ) + response = request_with_retries(timeout=30, app_name="app") assert response.text == "hii" # Ensure recovered replica names are the same diff --git a/python/ray/serve/tests/test_failure.py b/python/ray/serve/tests/test_failure.py index 2df70aef6a1f..c2e1bc548495 100644 --- a/python/ray/serve/tests/test_failure.py +++ b/python/ray/serve/tests/test_failure.py @@ -16,26 +16,12 @@ from ray.serve._private.test_utils import ( Counter, check_num_replicas_eq, - get_application_url, get_deployment_details, + request_with_retries, tlog, ) -def request_with_retries(endpoint, timeout=30, app_name=SERVE_DEFAULT_APP_NAME): - start = time.time() - while True: - try: - return httpx.get( - get_application_url("HTTP", app_name=app_name) + endpoint, - timeout=timeout, - ) - except (httpx.RequestError, IndexError): - if time.time() - start > timeout: - raise TimeoutError - time.sleep(0.1) - - @pytest.mark.skip(reason="Consistently failing.") def test_controller_failure(serve_instance): @serve.deployment(name="controller_failure") @@ -44,16 +30,16 @@ def function(_): serve.run(function.bind()) - assert request_with_retries("/controller_failure/", timeout=1).text == "hello1" + assert request_with_retries(timeout=1).text == "hello1" for _ in range(10): - response = request_with_retries("/controller_failure/", timeout=30) + response = request_with_retries(timeout=30) assert response.text == "hello1" ray.kill(serve.context._global_client._controller, no_restart=False) for _ in range(10): - response = request_with_retries("/controller_failure/", timeout=30) + response = request_with_retries(timeout=30) assert response.text == "hello1" def function2(_): @@ -64,7 +50,7 @@ def function2(_): serve.run(function.options(func_or_class=function2).bind()) def check_controller_failure(): - response = request_with_retries("/controller_failure/", timeout=30) + response = request_with_retries(timeout=30) return response.text == "hello2" wait_for_condition(check_controller_failure) @@ -78,50 +64,12 @@ def function3(_): ray.kill(serve.context._global_client._controller, no_restart=False) for _ in range(10): - response = request_with_retries("/controller_failure/", timeout=30) + response = request_with_retries(timeout=30) assert response.text == "hello2" - response = request_with_retries("/controller_failure_2/", timeout=30) + response = request_with_retries(timeout=30) assert response.text == "hello3" -def _kill_http_proxies(): - http_proxies = ray.get( - serve.context._global_client._controller.get_proxies.remote() - ) - for http_proxy in http_proxies.values(): - ray.kill(http_proxy, no_restart=False) - - -def test_http_proxy_failure(serve_instance): - @serve.deployment(name="proxy_failure") - def function(_): - return "hello1" - - serve.run(function.bind()) - - assert request_with_retries("/proxy_failure/", timeout=1.0).text == "hello1" - - for _ in range(10): - response = request_with_retries("/proxy_failure/", timeout=30) - assert response.text == "hello1" - - _kill_http_proxies() - - def function2(_): - return "hello2" - - serve.run(function.options(func_or_class=function2).bind()) - - def check_new(): - for _ in range(10): - response = request_with_retries("/proxy_failure/", timeout=30) - if response.text != "hello2": - return False - return True - - wait_for_condition(check_new) - - def _get_worker_handles(deployment_name: str, app_name: str = SERVE_DEFAULT_APP_NAME): id = DeploymentID(name=deployment_name, app_name=app_name) controller = serve.context._global_client._controller @@ -141,7 +89,7 @@ def __call__(self, *args): serve.run(Worker1.bind()) # Get the PID of the worker. - old_pid = request_with_retries("/worker_failure/", timeout=1).text + old_pid = request_with_retries(timeout=1).text # Kill the worker. handles = _get_worker_handles("worker_failure") @@ -151,7 +99,7 @@ def __call__(self, *args): # Wait until the worker is killed and a one is started. start = time.time() while time.time() - start < 30: - response = request_with_retries("/worker_failure/", timeout=30) + response = request_with_retries(timeout=30) if response.text != old_pid: break else: @@ -192,7 +140,7 @@ def __call__(self, *args): start = time.time() while time.time() - start < 30: time.sleep(0.1) - response = request_with_retries("/replica_failure/", timeout=1).text + response = request_with_retries(timeout=1).text assert response in ["1", "2"] responses.add(response) if len(responses) > 1: @@ -211,7 +159,7 @@ def __call__(self, *args): try: # The timeout needs to be small here because the request to # the restarting worker will hang. - request_with_retries("/replica_failure/", timeout=0.1) + request_with_retries(timeout=0.1) break except TimeoutError: time.sleep(0.1) diff --git a/python/ray/serve/tests/test_proxy.py b/python/ray/serve/tests/test_proxy.py index 6393194fc53b..fb7e38c1a743 100644 --- a/python/ray/serve/tests/test_proxy.py +++ b/python/ray/serve/tests/test_proxy.py @@ -16,6 +16,7 @@ from ray.serve._private.test_utils import ( ping_grpc_healthz, ping_grpc_list_applications, + request_with_retries, ) from ray.serve.config import gRPCOptions from ray.serve.generated import serve_pb2 @@ -224,5 +225,43 @@ def check_replicas_on_worker_nodes(): ping_grpc_healthz(worker_node_channel, test_draining=True) +def _kill_http_proxies(): + http_proxies = ray.get( + serve.context._global_client._controller.get_proxies.remote() + ) + for http_proxy in http_proxies.values(): + ray.kill(http_proxy, no_restart=False) + + +def test_http_proxy_failure(serve_instance): + @serve.deployment(name="proxy_failure") + def function(_): + return "hello1" + + serve.run(function.bind()) + + assert request_with_retries(timeout=1.0).text == "hello1" + + for _ in range(10): + response = request_with_retries(timeout=30) + assert response.text == "hello1" + + _kill_http_proxies() + + def function2(_): + return "hello2" + + serve.run(function.options(func_or_class=function2).bind()) + + def check_new(): + for _ in range(10): + response = request_with_retries(timeout=30) + if response.text != "hello2": + return False + return True + + wait_for_condition(check_new) + + if __name__ == "__main__": sys.exit(pytest.main(["-v", "-s", __file__])) From 8590c61fba674c15d7f18c915956f17e6893ab49 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Thu, 21 Aug 2025 14:43:41 -0700 Subject: [PATCH 218/634] [core] improve the robustness of metric reporting (#55152) **Context** This PR is related to the Ray metrics infrastructure. In Ray, each node runs a centralized process called the DashboardAgent, which acts as a gRPC server. Other processes on the same node send their per-process metrics to this server. The DashboardAgent then aggregates these metrics and exports them to Prometheus. The DashboardAgent is spawned by the raylet. **Problem** Currently, the `GCS` server also depends on the DashboardAgent to export its internal metrics. However, the DashboardAgent is often spawned significantly later than the GCS. This leads to failed RPC requests from GCS to the DashboardAgent, resulting in dropped metrics and noisy error logs in GCS. The same issue in theory also applies to `raylet` and `core_worker`, even though the initialization gap is not observed during my testing, likely because the gap is small. **Solution** To address this, this PR introduces an `InitExporter` function inside the `MetricsAgentClient` that repeatedly retries the connection to the DashboardAgent until it succeeds. Only after a successful connection is established will GCS/raylet/core-worker begin sending metrics. To check if the gRPC server in DashboardAgent is ready, the implementation creates a new `HealthCheck` endpoint on the server side of the `MetricsAgent`. I fixed this only for the OpenTelemetry-backed infrastructure. Addressing it for OpenCensus has complication (it doesn't lazily store metrics before init) so I don't bother spend more efforts for something that will be soon deprecated. Test: - CI - `test_metrics_agent.py` is a comprehensive test that provides confidence everything is working properly. Run this test locally and check that errors are no longer observed inside gcs log. Signed-off-by: Cuong Nguyen --- .../modules/reporter/reporter_agent.py | 11 +++ src/ray/core_worker/BUILD.bazel | 1 + src/ray/core_worker/core_worker_process.cc | 9 ++ src/ray/core_worker/core_worker_process.h | 4 + src/ray/gcs/gcs_server/BUILD.bazel | 1 + src/ray/gcs/gcs_server/gcs_server.cc | 11 +++ src/ray/gcs/gcs_server/gcs_server.h | 4 + src/ray/gcs/gcs_server/gcs_server_main.cc | 1 + src/ray/protobuf/reporter.proto | 6 ++ src/ray/raylet/BUILD.bazel | 1 + src/ray/raylet/main.cc | 8 ++ src/ray/rpc/BUILD.bazel | 1 + src/ray/rpc/metrics_agent_client.cc | 69 ++++++++++++++ src/ray/rpc/metrics_agent_client.h | 55 +++++++++-- src/ray/rpc/tests/BUILD.bazel | 13 +++ .../rpc/tests/metrics_agent_client_test.cc | 94 +++++++++++++++++++ src/ray/stats/metric_exporter.cc | 23 +++-- src/ray/stats/metric_exporter.h | 1 + src/ray/stats/stats.h | 29 ++++-- .../stats/tests/metric_exporter_grpc_test.cc | 6 ++ 20 files changed, 325 insertions(+), 23 deletions(-) create mode 100644 src/ray/rpc/metrics_agent_client.cc create mode 100644 src/ray/rpc/tests/metrics_agent_client_test.cc diff --git a/python/ray/dashboard/modules/reporter/reporter_agent.py b/python/ray/dashboard/modules/reporter/reporter_agent.py index 16355729011f..ea842e8eacc7 100644 --- a/python/ray/dashboard/modules/reporter/reporter_agent.py +++ b/python/ray/dashboard/modules/reporter/reporter_agent.py @@ -534,6 +534,17 @@ async def MemoryProfiling(self, request, context): output=output, success=success, warning=warning ) + async def HealthCheck( + self, + _request: reporter_pb2.HealthCheckRequest, + _context: ServicerContext, + ) -> reporter_pb2.HealthCheckReply: + """This is a health check endpoint for the reporter agent. + + It is used to check if the reporter agent is ready to receive requests. + """ + return reporter_pb2.HealthCheckReply() + async def ReportOCMetrics(self, request, context): # Do nothing if metrics collection is disabled. if self._metrics_collection_disabled: diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 388e6b3a9d8b..428ab30eace4 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -45,6 +45,7 @@ ray_cc_library( "//src/ray/raylet_client:raylet_client_lib", "//src/ray/rpc:core_worker_client", "//src/ray/rpc:core_worker_server", + "//src/ray/rpc:metrics_agent_client", "//src/ray/stats:stats_lib", "//src/ray/util:container_util", "//src/ray/util:env", diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index 783a184ca09f..c870835c47bd 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -786,6 +786,15 @@ CoreWorkerProcessImpl::CoreWorkerProcessImpl(const CoreWorkerOptions &options) auto worker = CreateCoreWorker(options_, worker_id_); auto write_locked = core_worker_.LockForWrite(); write_locked.Get() = worker; + // Initialize metrics agent client. + metrics_agent_client_ = std::make_unique( + "127.0.0.1", + options_.metrics_agent_port, + io_service_, + *write_locked.Get()->client_call_manager_); + metrics_agent_client_->WaitForServerReady([this](const Status &server_status) { + stats::InitOpenTelemetryExporter(options_.metrics_agent_port, server_status); + }); } } diff --git a/src/ray/core_worker/core_worker_process.h b/src/ray/core_worker/core_worker_process.h index 5a6eb569b0ab..f516315e3aa2 100644 --- a/src/ray/core_worker/core_worker_process.h +++ b/src/ray/core_worker/core_worker_process.h @@ -19,6 +19,7 @@ #include #include "ray/core_worker/core_worker_options.h" +#include "ray/rpc/metrics_agent_client.h" #include "ray/util/mutex_protected.h" namespace ray { @@ -177,6 +178,9 @@ class CoreWorkerProcessImpl { /// The proxy service handler that routes the RPC calls to the core worker. std::unique_ptr service_handler_; + + /// The client to export metrics to the metrics agent. + std::unique_ptr metrics_agent_client_; }; } // namespace core } // namespace ray diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 9e482b9736bc..f81fbdcb7479 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -242,6 +242,7 @@ ray_cc_library( "//src/ray/raylet_client:raylet_client_lib", "//src/ray/rpc:core_worker_client", "//src/ray/rpc:gcs_server", + "//src/ray/rpc:metrics_agent_client", "//src/ray/rpc:node_manager_client", "//src/ray/util:counter_map", "//src/ray/util:exponential_backoff", diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 90c61aa24da7..95d0a52496e3 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -35,6 +35,7 @@ #include "ray/gcs/store_client/redis_store_client.h" #include "ray/gcs/store_client/store_client.h" #include "ray/pubsub/publisher.h" +#include "ray/stats/stats.h" #include "ray/util/network_util.h" namespace ray { @@ -155,6 +156,11 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, /*publisher_id=*/NodeID::FromRandom()); gcs_publisher_ = std::make_unique(std::move(inner_publisher)); + metrics_agent_client_ = std::make_unique( + "127.0.0.1", + config_.metrics_agent_port, + io_context_provider_.GetDefaultIOContext(), + client_call_manager_); } GcsServer::~GcsServer() { Stop(); } @@ -268,6 +274,11 @@ void GcsServer::DoStart(const GcsInitData &gcs_init_data) { // Init usage stats client. InitUsageStatsClient(); + // Init OpenTelemetry exporter. + metrics_agent_client_->WaitForServerReady([this](const Status &server_status) { + stats::InitOpenTelemetryExporter(config_.metrics_agent_port, server_status); + }); + // Start RPC server when all tables have finished loading initial // data. rpc_server_.Run(); diff --git a/src/ray/gcs/gcs_server/gcs_server.h b/src/ray/gcs/gcs_server/gcs_server.h index 491300996a83..85a02d75f8b2 100644 --- a/src/ray/gcs/gcs_server/gcs_server.h +++ b/src/ray/gcs/gcs_server/gcs_server.h @@ -41,6 +41,7 @@ #include "ray/raylet/scheduling/cluster_task_manager.h" #include "ray/rpc/client_call.h" #include "ray/rpc/gcs/gcs_rpc_server.h" +#include "ray/rpc/metrics_agent_client.h" #include "ray/rpc/node_manager/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/throttler.h" @@ -55,6 +56,7 @@ struct GcsServerConfig { std::string grpc_server_name = "GcsServer"; uint16_t grpc_server_port = 0; uint16_t grpc_server_thread_num = 1; + uint16_t metrics_agent_port = 0; std::string redis_username; std::string redis_password; std::string redis_address; @@ -294,6 +296,8 @@ class GcsServer { int task_pending_schedule_detected_ = 0; /// Throttler for global gc std::unique_ptr global_gc_throttler_; + /// Client to call a metrics agent gRPC server. + std::unique_ptr metrics_agent_client_; }; } // namespace gcs diff --git a/src/ray/gcs/gcs_server/gcs_server_main.cc b/src/ray/gcs/gcs_server/gcs_server_main.cc index 53f95d4f402a..7d7fa32f635e 100644 --- a/src/ray/gcs/gcs_server/gcs_server_main.cc +++ b/src/ray/gcs/gcs_server/gcs_server_main.cc @@ -155,6 +155,7 @@ int main(int argc, char *argv[]) { gcs_server_config.redis_username = redis_username; gcs_server_config.retry_redis = retry_redis; gcs_server_config.node_ip_address = node_ip_address; + gcs_server_config.metrics_agent_port = metrics_agent_port; gcs_server_config.log_dir = log_dir; gcs_server_config.raylet_config_list = config_list; gcs_server_config.session_name = session_name; diff --git a/src/ray/protobuf/reporter.proto b/src/ray/protobuf/reporter.proto index 552b5a95a9c4..3397c3da1c60 100644 --- a/src/ray/protobuf/reporter.proto +++ b/src/ray/protobuf/reporter.proto @@ -83,6 +83,10 @@ message ReportOCMetricsRequest { message ReportOCMetricsReply {} +message HealthCheckRequest {} + +message HealthCheckReply {} + // Service for communicating with the reporter agent module on a remote node. service ReporterService { // Report OpenCensus metrics to the local metrics agent. @@ -91,6 +95,8 @@ service ReporterService { rpc CpuProfiling(CpuProfilingRequest) returns (CpuProfilingReply); rpc GpuProfiling(GpuProfilingRequest) returns (GpuProfilingReply); rpc MemoryProfiling(MemoryProfilingRequest) returns (MemoryProfilingReply); + // Health check to validate whether the service is running + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckReply); } message StreamLogRequest { diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index e2601e02ba74..f5305944d209 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -285,6 +285,7 @@ ray_cc_binary( "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/object_manager:ownership_object_directory", "//src/ray/raylet/scheduling:cluster_task_manager", + "//src/ray/rpc:metrics_agent_client", "//src/ray/stats:stats_lib", "//src/ray/util:cmd_line_utils", "//src/ray/util:event", diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 04cfe875968c..3af97a4c55d5 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -285,6 +285,8 @@ int main(int argc, char *argv[]) { /// A manager to resolve objects needed by queued tasks and workers that /// called `ray.get` or `ray.wait`. std::unique_ptr dependency_manager; + /// The client to export metrics to the metrics agent. + std::unique_ptr metrics_agent_client; /// Map of workers leased out to clients. absl::flat_hash_map> leased_workers; @@ -827,6 +829,12 @@ int main(int argc, char *argv[]) { {ray::stats::NodeAddressKey, node_ip_address}, {ray::stats::SessionNameKey, session_name}}; ray::stats::Init(global_tags, metrics_agent_port, WorkerID::Nil()); + metrics_agent_client = std::make_unique( + "127.0.0.1", metrics_agent_port, main_service, *client_call_manager); + metrics_agent_client->WaitForServerReady( + [metrics_agent_port](const ray::Status &server_status) { + ray::stats::InitOpenTelemetryExporter(metrics_agent_port, server_status); + }); // Initialize event framework. This should be done after the node manager is // initialized. diff --git a/src/ray/rpc/BUILD.bazel b/src/ray/rpc/BUILD.bazel index 776a24e80abd..41e9e92f0121 100644 --- a/src/ray/rpc/BUILD.bazel +++ b/src/ray/rpc/BUILD.bazel @@ -51,6 +51,7 @@ ray_cc_library( ray_cc_library( name = "metrics_agent_client", + srcs = ["metrics_agent_client.cc"], hdrs = ["metrics_agent_client.h"], visibility = ["//visibility:public"], deps = [ diff --git a/src/ray/rpc/metrics_agent_client.cc b/src/ray/rpc/metrics_agent_client.cc new file mode 100644 index 000000000000..29dc579af14c --- /dev/null +++ b/src/ray/rpc/metrics_agent_client.cc @@ -0,0 +1,69 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/rpc/metrics_agent_client.h" + +#include +#include + +#include "ray/util/logging.h" + +namespace ray { +namespace rpc { + +void MetricsAgentClientImpl::WaitForServerReady( + std::function init_exporter_fn) { + WaitForServerReadyWithRetry( + init_exporter_fn, 0, kMetricAgentInitMaxRetries, kMetricAgentInitRetryDelayMs); +} + +void MetricsAgentClientImpl::WaitForServerReadyWithRetry( + std::function init_exporter_fn, + int retry_count, + int max_retry, + int retry_interval_ms) { + if (exporter_initialized_) { + return; + } + + RAY_LOG(INFO) << "Initializing exporter ..."; + HealthCheck(rpc::HealthCheckRequest(), + [this, init_exporter_fn](auto &status, auto &&reply) { + if (status.ok() && !exporter_initialized_) { + init_exporter_fn(status); + exporter_initialized_ = true; + RAY_LOG(INFO) << "Exporter initialized."; + } + }); + if (retry_count >= max_retry) { + init_exporter_fn(Status::RpcError("The metrics agent server is not ready.", 14)); + return; + } + retry_count++; + retry_timer_->expires_after(std::chrono::milliseconds(retry_interval_ms)); + retry_timer_->async_wait( + [this, init_exporter_fn, retry_count, max_retry, retry_interval_ms]( + const boost::system::error_code &error) { + if (!error) { + WaitForServerReadyWithRetry( + init_exporter_fn, retry_count, max_retry, retry_interval_ms); + } else { + RAY_LOG(ERROR) << "Failed to initialize exporter. Data will not be exported to " + "the metrics agent."; + } + }); +} + +} // namespace rpc +} // namespace ray diff --git a/src/ray/rpc/metrics_agent_client.h b/src/ray/rpc/metrics_agent_client.h index 9af5cf290c80..c5a6085d23a1 100644 --- a/src/ray/rpc/metrics_agent_client.h +++ b/src/ray/rpc/metrics_agent_client.h @@ -30,6 +30,11 @@ namespace ray { namespace rpc { +/// The maximum number of retries to wait for the server to be ready. +/// This setting allows for 30 seconds of retries. +constexpr int kMetricAgentInitMaxRetries = 30; +constexpr int kMetricAgentInitRetryDelayMs = 1000; + /// Client used for communicating with a remote node manager server. class MetricsAgentClient { public: @@ -40,6 +45,20 @@ class MetricsAgentClient { /// \param[in] request The request message. /// \param[in] callback The callback function that handles reply. VOID_RPC_CLIENT_VIRTUAL_METHOD_DECL(ReporterService, ReportOCMetrics) + + /// Send a health check request to the metrics agent. + /// + /// \param[in] request The request message. + /// \param[in] callback The callback function that handles reply. + VOID_RPC_CLIENT_VIRTUAL_METHOD_DECL(ReporterService, HealthCheck) + + /// Initialize an exporter (e.g. metrics, events exporter). + /// + /// This function ensures that the server is ready to receive metrics before + /// initializing the exporter. If the server is not ready, it will retry for + /// a number of times. + virtual void WaitForServerReady( + std::function init_exporter_fn) = 0; }; class MetricsAgentClientImpl : public MetricsAgentClient { @@ -48,15 +67,17 @@ class MetricsAgentClientImpl : public MetricsAgentClient { /// /// \param[in] address Address of the metrics agent server. /// \param[in] port Port of the metrics agent server. + /// \param[in] io_service The `instrumented_io_context` used for managing requests. /// \param[in] client_call_manager The `ClientCallManager` used for managing requests. MetricsAgentClientImpl(const std::string &address, const int port, - instrumented_io_context &io_service) - : client_call_manager_(io_service, /*record_stats=*/true) { + instrumented_io_context &io_service, + rpc::ClientCallManager &client_call_manager) { RAY_LOG(DEBUG) << "Initiate the metrics client of address:" << BuildAddress(address, port); - grpc_client_ = std::make_unique>( - address, port, client_call_manager_); + grpc_client_ = + std::make_unique>(address, port, client_call_manager); + retry_timer_ = std::make_unique(io_service); }; VOID_RPC_CLIENT_METHOD(ReporterService, @@ -65,11 +86,33 @@ class MetricsAgentClientImpl : public MetricsAgentClient { /*method_timeout_ms*/ -1, override) + VOID_RPC_CLIENT_METHOD(ReporterService, + HealthCheck, + grpc_client_, + /*method_timeout_ms*/ -1, + override) + + /// Wait for the server to be ready. Invokes the callback with the final readiness + /// status of the server. + void WaitForServerReady(std::function init_exporter_fn) override; + private: - /// Call Manager for gRPC client. - rpc::ClientCallManager client_call_manager_; /// The RPC client. std::unique_ptr> grpc_client_; + /// Timer for retrying to initialize the OpenTelemetry exporter. + std::unique_ptr retry_timer_; + /// Whether the exporter is initialized. + bool exporter_initialized_ = false; + /// Wait for the server to be ready with a retry count. Invokes the callback + /// with the status of the server. This is a helper function for WaitForServerReady. + void WaitForServerReadyWithRetry(std::function init_exporter_fn, + int retry_count, + int max_retry, + int retry_interval_ms); + + friend class MetricsAgentClientTest; + FRIEND_TEST(MetricsAgentClientTest, WaitForServerReadyWithRetrySuccess); + FRIEND_TEST(MetricsAgentClientTest, WaitForServerReadyWithRetryFailure); }; } // namespace rpc diff --git a/src/ray/rpc/tests/BUILD.bazel b/src/ray/rpc/tests/BUILD.bazel index 8bd523f17c97..d2803b88c651 100644 --- a/src/ray/rpc/tests/BUILD.bazel +++ b/src/ray/rpc/tests/BUILD.bazel @@ -41,3 +41,16 @@ ray_cc_test( "@com_google_googletest//:gtest_main", ], ) + +ray_cc_test( + name = "metrics_agent_client_test", + size = "small", + srcs = [ + "metrics_agent_client_test.cc", + ], + tags = ["team:core"], + deps = [ + "//src/ray/rpc:metrics_agent_client", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/rpc/tests/metrics_agent_client_test.cc b/src/ray/rpc/tests/metrics_agent_client_test.cc new file mode 100644 index 000000000000..114252116794 --- /dev/null +++ b/src/ray/rpc/tests/metrics_agent_client_test.cc @@ -0,0 +1,94 @@ +// Copyright 2024 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/rpc/metrics_agent_client.h" + +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace ray { +namespace rpc { + +constexpr int kCountToReturnOk = 3; +constexpr int kRetryIntervalMs = 100; + +class TestableMetricsAgentClientImpl : public MetricsAgentClientImpl { + public: + TestableMetricsAgentClientImpl(const std::string &address, + const int port, + instrumented_io_context &io_service, + rpc::ClientCallManager &client_call_manager, + int count_to_return_ok) + : MetricsAgentClientImpl(address, port, io_service, client_call_manager), + count_to_return_ok_(count_to_return_ok) {} + + // HealthCheck is a macro+template method that supposes to invoke the callback upon + // the completion of an RPC call. We override it to invoke the callback directly + // without the RPC call. Ideally we would create a GrpcClientMock that overrides + // the RPC call. However, currently the RPC call is a template method, which cannot + // be overridden. + void HealthCheck(const HealthCheckRequest &request, + const ClientCallback &callback) override { + health_check_count_++; + if (health_check_count_ <= count_to_return_ok_) { + callback(Status::RpcError("Failed to connect to the metrics agent server.", 14), + HealthCheckReply()); + } else { + callback(Status::OK(), HealthCheckReply()); + } + } + + private: + int count_to_return_ok_; + int health_check_count_ = 1; +}; + +class MetricsAgentClientTest : public ::testing::Test { + protected: + void SetUp() override { + client_call_manager_ = std::make_unique(io_service_, true); + client_ = std::make_unique( + "127.0.0.1", 8000, io_service_, *client_call_manager_, kCountToReturnOk); + } + + instrumented_io_context io_service_; + std::unique_ptr client_; + std::unique_ptr client_call_manager_; +}; + +TEST_F(MetricsAgentClientTest, WaitForServerReadyWithRetrySuccess) { + client_->WaitForServerReadyWithRetry( + [](const Status &server_status) { ASSERT_TRUE(server_status.ok()); }, + 0, + kCountToReturnOk, + kRetryIntervalMs); + io_service_.run_for(std::chrono::milliseconds(kCountToReturnOk * kRetryIntervalMs)); + ASSERT_TRUE(client_->exporter_initialized_); +} + +TEST_F(MetricsAgentClientTest, WaitForServerReadyWithRetryFailure) { + client_->WaitForServerReadyWithRetry( + [](const Status &server_status) { ASSERT_FALSE(server_status.ok()); }, + 0, + kCountToReturnOk - 2, + kRetryIntervalMs); + io_service_.run_for(std::chrono::milliseconds(kCountToReturnOk * kRetryIntervalMs)); + ASSERT_FALSE(client_->exporter_initialized_); +} + +} // namespace rpc +} // namespace ray diff --git a/src/ray/stats/metric_exporter.cc b/src/ray/stats/metric_exporter.cc index 5a3762ab45d0..76bc63abe4d8 100644 --- a/src/ray/stats/metric_exporter.cc +++ b/src/ray/stats/metric_exporter.cc @@ -30,26 +30,31 @@ OpenCensusProtoExporter::OpenCensusProtoExporter(const int port, const WorkerID &worker_id, size_t report_batch_size, size_t max_grpc_payload_size) - : OpenCensusProtoExporter( - std::make_shared(address, port, io_service), - worker_id, - report_batch_size, - max_grpc_payload_size) {} + : client_call_manager_( + std::make_unique(io_service, /*record_stats=*/true)), + worker_id_(worker_id), + report_batch_size_(report_batch_size), + // To make sure we're not overflowing Agent's set gRPC max message size, we will be + // tracking target payload binary size and make sure it stays w/in 95% of the + // threshold + proto_payload_size_threshold_bytes_((size_t)(max_grpc_payload_size * .95f)) { + absl::MutexLock l(&mu_); + client_ = std::make_shared( + address, port, io_service, *client_call_manager_); +} OpenCensusProtoExporter::OpenCensusProtoExporter( std::shared_ptr agent_client, const WorkerID &worker_id, size_t report_batch_size, size_t max_grpc_payload_size) + : worker_id_(worker_id), report_batch_size_(report_batch_size), - // To make sure we're not overflowing Agent's set gRPC max message size, we will be - // tracking target payload binary size and make sure it stays w/in 95% of the - // threshold proto_payload_size_threshold_bytes_((size_t)(max_grpc_payload_size * .95f)) { absl::MutexLock l(&mu_); client_ = std::move(agent_client); -}; +} /// Hack. We want to add GlobalTags to all our metrics, but gRPC OpenCencus plugin is not /// configurable at all so we don't have chance to add our own tags. We use this hack to diff --git a/src/ray/stats/metric_exporter.h b/src/ray/stats/metric_exporter.h index b05444cf6370..c1ea7197823c 100644 --- a/src/ray/stats/metric_exporter.h +++ b/src/ray/stats/metric_exporter.h @@ -105,6 +105,7 @@ class OpenCensusProtoExporter final : public opencensus::stats::StatsExporter::H /// Lock to protect the client mutable absl::Mutex mu_; /// Client to call a metrics agent gRPC server. + std::unique_ptr client_call_manager_; std::shared_ptr client_ ABSL_GUARDED_BY(&mu_); /// The worker ID of the current component. WorkerID worker_id_; diff --git a/src/ray/stats/stats.h b/src/ray/stats/stats.h index af952e5b3ccc..9347b70bfad2 100644 --- a/src/ray/stats/stats.h +++ b/src/ray/stats/stats.h @@ -90,14 +90,6 @@ static inline void Init( absl::Milliseconds(std::max(RayConfig::instance().metrics_report_interval_ms() / 2, static_cast(500)))); // Register the metric recorder. - if (RayConfig::instance().enable_open_telemetry()) { - OpenTelemetryMetricRecorder::GetInstance().RegisterGrpcExporter( - BuildAddress("127.0.0.1", metrics_agent_port), - std::chrono::milliseconds( - absl::ToInt64Milliseconds(StatsConfig::instance().GetReportInterval())), - std::chrono::milliseconds( - absl::ToInt64Milliseconds(StatsConfig::instance().GetHarvestInterval()))); - } if (should_enable_open_census()) { metrics_io_service_pool = std::make_shared(1); metrics_io_service_pool->Run(); @@ -122,6 +114,27 @@ static inline void Init( StatsConfig::instance().SetIsInitialized(true); } +static inline void InitOpenTelemetryExporter(const int metrics_agent_port, + const Status &metrics_agent_server_status) { + if (!RayConfig::instance().enable_open_telemetry()) { + return; + } + if (!metrics_agent_server_status.ok()) { + RAY_LOG(ERROR) << "Failed to initialize OpenTelemetry exporter. Data will not be " + "exported to the " + << "metrics agent. Server status: " << metrics_agent_server_status; + return; + } + OpenTelemetryMetricRecorder::GetInstance().RegisterGrpcExporter( + /*endpoint=*/std::string("127.0.0.1:") + std::to_string(metrics_agent_port), + /*interval=*/ + std::chrono::milliseconds( + absl::ToInt64Milliseconds(StatsConfig::instance().GetReportInterval())), + /*timeout=*/ + std::chrono::milliseconds( + absl::ToInt64Milliseconds(StatsConfig::instance().GetHarvestInterval()))); +} + /// Shutdown the initialized stats library. /// This cleans up various threads and metadata for stats library. static inline void Shutdown() { diff --git a/src/ray/stats/tests/metric_exporter_grpc_test.cc b/src/ray/stats/tests/metric_exporter_grpc_test.cc index a823a863ebd3..a7553c88f792 100644 --- a/src/ray/stats/tests/metric_exporter_grpc_test.cc +++ b/src/ray/stats/tests/metric_exporter_grpc_test.cc @@ -57,6 +57,12 @@ class MockMetricsAgentClient : public rpc::MetricsAgentClient { callback(Status::OK(), {}); } + void HealthCheck(const rpc::HealthCheckRequest &request, + const rpc::ClientCallback &callback) override {} + + void WaitForServerReady(std::function init_exporter_fn) override { + } + const std::vector &CollectedReportOCMetricsRequests() const { return reportOCMetricsRequests_; From 77ccb7aacaebb209368a59ec1b97fdf0ed978246 Mon Sep 17 00:00:00 2001 From: Alima Azamat <92766804+alimaazamat@users.noreply.github.com> Date: Thu, 21 Aug 2025 14:47:44 -0700 Subject: [PATCH 219/634] [Core][Azure] If azure cluster launcher keypair doesnt exist create one automatically + doc typo fix (#54596) Signed-off-by: alimaazamat --- .../user-guides/launching-clusters/azure.md | 72 +++++++----- .../ray/autoscaler/_private/_azure/config.py | 111 ++++++++++++++++-- python/ray/autoscaler/_private/gcp/config.py | 57 ++------- python/ray/autoscaler/_private/util.py | 44 +++++++ python/ray/autoscaler/azure/defaults.yaml | 10 +- python/ray/autoscaler/azure/example-full.yaml | 11 +- .../autoscaler/azure/example-gpu-docker.yaml | 10 +- .../ray/autoscaler/azure/example-minimal.yaml | 13 +- 8 files changed, 217 insertions(+), 111 deletions(-) diff --git a/doc/source/cluster/vms/user-guides/launching-clusters/azure.md b/doc/source/cluster/vms/user-guides/launching-clusters/azure.md index 7fb6b4402ef2..638b66dfb740 100644 --- a/doc/source/cluster/vms/user-guides/launching-clusters/azure.md +++ b/doc/source/cluster/vms/user-guides/launching-clusters/azure.md @@ -60,38 +60,46 @@ Download the reference example locally: wget https://raw.githubusercontent.com/ray-project/ray/master/python/ray/autoscaler/azure/example-full.yaml ``` -To connect to the provisioned head node VM, you need to ensure that you properly configure the `auth.ssh_private_key`, `auth.ssh_public_key`, and `file_mounts` configuration values to point to file paths on your local environment that have a valid key pair. By default the configuration assumes `$HOME/.ssh/id_rsa` and `$HOME/.ssh/id_rsa.pub`. If you have a different set of key pair files you want to use (for example a `ed25519` pair), update the `example-full.yaml` configurations to use them. - -For example a custom-configured `example-full.yaml` file might look like the following if you're using a `ed25519` key pair: - -```sh -$ git diff example-full.yaml -diff --git a/python/ray/autoscaler/azure/example-full.yaml b/python/ray/autoscaler/azure/example-full.yaml -index b25f1b07f1..c65fb77219 100644 ---- a/python/ray/autoscaler/azure/example-full.yaml -+++ b/python/ray/autoscaler/azure/example-full.yaml -@@ -61,9 +61,9 @@ auth: - ssh_user: ubuntu - # You must specify paths to matching private and public key pair files. - # Use `ssh-keygen -t rsa -b 4096` to generate a new ssh key pair. -- ssh_private_key: ~/.ssh/id_rsa -+ ssh_private_key: ~/.ssh/id_ed25519 - # Changes to this should match what is specified in file_mounts. -- ssh_public_key: ~/.ssh/id_rsa.pub -+ ssh_public_key: ~/.ssh/id_ed25519.pub - - # You can make more specific customization to node configurations can be made using the ARM template azure-vm-template.json file. - # See this documentation here: https://docs.microsoft.com/en-us/azure/templates/microsoft.compute/2019-03-01/virtualmachines -@@ -128,7 +128,7 @@ head_node_type: ray.head.default - file_mounts: { - # "/path1/on/remote/machine": "/path1/on/local/machine", - # "/path2/on/remote/machine": "/path2/on/local/machine", -- "~/.ssh/id_rsa.pub": "~/.ssh/id_rsa.pub"} -+ "~/.ssh/id_ed25519.pub": "~/.ssh/id_ed25519.pub"} - - # Files or directories to copy from the head node to the worker nodes. The format is a - # list of paths. Ray copies the same path on the head node to the worker node. - ``` + + +##### Automatic SSH Key Generation + +To connect to the provisioned head node VM, Ray has automatic SSH Key Generation if none are specified in the config. This is the simplest approach and requires no manual key management. + +The default configuration in `example-full.yaml` uses automatic key generation: + +```yaml +auth: + ssh_user: ubuntu + # SSH keys are auto-generated if not specified + # Uncomment and specify custom paths if you want to use existing keys: + # ssh_private_key: /path/to/your/key.pem + # ssh_public_key: /path/to/your/key.pub +``` + +##### (Optional) Manual SSH Key Configuration + +If you prefer to use your own existing SSH keys, uncomment and specify both of the key paths in the `auth` section. + +For example, to use an existing `ed25519` key pair: + +```yaml +auth: + ssh_user: ubuntu + ssh_private_key: ~/.ssh/id_ed25519 + ssh_public_key: ~/.ssh/id_ed25519.pub +``` + +Or for RSA keys: + +```yaml +auth: + ssh_user: ubuntu + ssh_private_key: ~/.ssh/id_rsa + ssh_public_key: ~/.ssh/id_rsa.pub +``` + +Both methods inject the public key directly into the VM's `~/.ssh/authorized_keys` via Azure ARM templates. #### Launch the Ray cluster on Azure diff --git a/python/ray/autoscaler/_private/_azure/config.py b/python/ray/autoscaler/_private/_azure/config.py index 5320b1698ec3..1c2b1a91a90f 100644 --- a/python/ray/autoscaler/_private/_azure/config.py +++ b/python/ray/autoscaler/_private/_azure/config.py @@ -1,5 +1,6 @@ import json import logging +import os import random from hashlib import sha256 from pathlib import Path @@ -10,6 +11,12 @@ from azure.mgmt.resource import ResourceManagementClient from azure.mgmt.resource.resources.models import DeploymentMode +from ray.autoscaler._private.util import ( + generate_rsa_key_pair, + generate_ssh_key_name, + generate_ssh_key_paths, +) + UNIQUE_ID_LEN = 4 logger = logging.getLogger(__name__) @@ -218,22 +225,104 @@ def _configure_resource_group(config): def _configure_key_pair(config): + """ + Configure SSH keypair. Use user specified custom paths, otherwise, + generate Ray-specific keypair in this format: "ray-autoscaler_azure_{region}_{resource_group}_{ssh_user}_{index}" + """ ssh_user = config["auth"]["ssh_user"] public_key = None - # search if the keys exist - for key_type in ["ssh_private_key", "ssh_public_key"]: - try: - key_path = Path(config["auth"][key_type]).expanduser() - except KeyError: - raise Exception("Config must define {}".format(key_type)) - except TypeError: - raise Exception("Invalid config value for {}".format(key_type)) - assert key_path.is_file(), "Could not find ssh key: {}".format(key_path) + # Check if user specified custom SSH key paths + user_specified_private_key = "ssh_private_key" in config["auth"] + user_specified_public_key = "ssh_public_key" in config["auth"] + + # Validate that the user either specfied both keys or none, but not just one + if user_specified_private_key != user_specified_public_key: + if user_specified_private_key: + missing_key, specified_key = "ssh_public_key", "ssh_private_key" + else: + missing_key, specified_key = "ssh_private_key", "ssh_public_key" + raise ValueError( + f"{specified_key} is specified but {missing_key} is missing. " + "Both SSH key paths must be specified together, or omit both from " + "your config to use auto-generated keys." + ) + + if user_specified_private_key and user_specified_public_key: + # User specified custom paths + private_key_path = Path(config["auth"]["ssh_private_key"]).expanduser() + public_key_path = Path(config["auth"]["ssh_public_key"]).expanduser() + + # Validate that user-specified keys exist + missing_keys = [] + if not private_key_path.is_file(): + missing_keys.append(f"ssh_private_key: {private_key_path}") + if not public_key_path.is_file(): + missing_keys.append(f"ssh_public_key: {public_key_path}") + + if missing_keys: + raise ValueError( + "SSH key files from config do not exist: {}. " + "Please create the keys or remove the custom paths from your config " + "to use auto-generated keys.".format(", ".join(missing_keys)) + ) + logger.info( + "Using specified SSH keys from config: {} and {}".format( + private_key_path, public_key_path + ) + ) - if key_type == "ssh_public_key": - with open(key_path, "r") as f: + with open(public_key_path, "r") as f: + public_key = f.read() + else: + # Generate Ray-specific keys + region = config["provider"]["location"] + resource_group = config["provider"]["resource_group"] + + # Generate single deterministic key name for this configuration + key_name = generate_ssh_key_name( + "azure", None, region, resource_group, ssh_user + ) + public_key_path, private_key_path = generate_ssh_key_paths(key_name) + + # Check if this key pair already exists + if os.path.exists(private_key_path) and os.path.exists(public_key_path): + logger.info( + "Using existing Ray-specific SSH keys: {} and {}".format( + private_key_path, public_key_path + ) + ) + with open(public_key_path, "r") as f: public_key = f.read() + else: + # Create a key pair since it doesn't exist locally + logger.info( + "Generating new Ray-specific SSH key pair at {} and {}".format( + private_key_path, public_key_path + ) + ) + os.makedirs(os.path.dirname(private_key_path), exist_ok=True) + public_key, private_key = generate_rsa_key_pair() + with open( + private_key_path, + "w", + opener=lambda path, flags: os.open(path, flags, 0o600), + ) as f: + f.write(private_key) + with open(public_key_path, "w") as f: + f.write(public_key) + + assert os.path.exists( + private_key_path + ), "Private key file {} not found for user {}".format( + private_key_path, ssh_user + ) + + config["auth"]["ssh_private_key"] = private_key_path + config["auth"]["ssh_public_key"] = public_key_path + if "file_mounts" not in config: + config["file_mounts"] = {} + config["file_mounts"]["~/.ssh/id_rsa.pub"] = public_key_path for node_type in config["available_node_types"].values(): azure_arm_parameters = node_type["node_config"].setdefault( diff --git a/python/ray/autoscaler/_private/gcp/config.py b/python/ray/autoscaler/_private/gcp/config.py index b48a7e984762..e527e21ab556 100644 --- a/python/ray/autoscaler/_private/gcp/config.py +++ b/python/ray/autoscaler/_private/gcp/config.py @@ -9,9 +9,6 @@ import google_auth_httplib2 import googleapiclient import httplib2 -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa from google.oauth2 import service_account from google.oauth2.credentials import Credentials as OAuthCredentials from googleapiclient import discovery, errors @@ -19,7 +16,12 @@ from ray._private.accelerators import TPUAcceleratorManager from ray._private.accelerators import tpu from ray.autoscaler._private.gcp.node import MAX_POLLS, POLL_INTERVAL, GCPNodeType -from ray.autoscaler._private.util import check_legacy_fields +from ray.autoscaler._private.util import ( + check_legacy_fields, + generate_rsa_key_pair, + generate_ssh_key_name, + generate_ssh_key_paths, +) logger = logging.getLogger(__name__) @@ -244,43 +246,6 @@ def wait_for_compute_global_operation(project_name, operation, compute): return result -def key_pair_name(i, region, project_id, ssh_user): - """Returns the ith default gcp_key_pair_name.""" - key_name = "{}_gcp_{}_{}_{}_{}".format(RAY, region, project_id, ssh_user, i) - return key_name - - -def key_pair_paths(key_name): - """Returns public and private key paths for a given key_name.""" - public_key_path = os.path.expanduser("~/.ssh/{}.pub".format(key_name)) - private_key_path = os.path.expanduser("~/.ssh/{}.pem".format(key_name)) - return public_key_path, private_key_path - - -def generate_rsa_key_pair(): - """Create public and private ssh-keys.""" - - key = rsa.generate_private_key( - backend=default_backend(), public_exponent=65537, key_size=2048 - ) - - public_key = ( - key.public_key() - .public_bytes( - serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH - ) - .decode("utf-8") - ) - - pem = key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - ).decode("utf-8") - - return public_key, pem - - def _has_tpus_in_node_configs(config: dict) -> bool: """Check if any nodes in config are TPUs.""" node_configs = [ @@ -555,10 +520,14 @@ def _configure_key_pair(config, compute): # Try a few times to get or create a good key pair. key_found = False for i in range(10): - key_name = key_pair_name( - i, config["provider"]["region"], config["provider"]["project_id"], ssh_user + key_name = generate_ssh_key_name( + "gcp", + i, + config["provider"]["region"], + config["provider"]["project_id"], + ssh_user, ) - public_key_path, private_key_path = key_pair_paths(key_name) + public_key_path, private_key_path = generate_ssh_key_paths(key_name) for ssh_key in ssh_keys: key_parts = ssh_key.split(" ") diff --git a/python/ray/autoscaler/_private/util.py b/python/ray/autoscaler/_private/util.py index 7da96de2b14b..17d463304957 100644 --- a/python/ray/autoscaler/_private/util.py +++ b/python/ray/autoscaler/_private/util.py @@ -990,3 +990,47 @@ def format_no_node_type_string(node_type: dict): output_lines.append(output_line) return "\n ".join(output_lines) + + +def generate_rsa_key_pair(): + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import rsa + + key = rsa.generate_private_key( + backend=default_backend(), public_exponent=65537, key_size=2048 + ) + + public_key = ( + key.public_key() + .public_bytes( + serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH + ) + .decode("utf-8") + ) + + pem = key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ).decode("utf-8") + + return public_key, pem + + +def generate_ssh_key_paths(key_name): + public_key_path = os.path.expanduser("~/.ssh/{}.pub".format(key_name)) + private_key_path = os.path.expanduser("~/.ssh/{}.pem".format(key_name)) + return public_key_path, private_key_path + + +def generate_ssh_key_name(provider, i, region, identifier, ssh_user): + RAY_PREFIX = "ray-autoscaler" + if i is not None: + return "{}_{}_{}_{}_{}_{}".format( + RAY_PREFIX, provider, region, identifier, ssh_user, i + ) + else: + return "{}_{}_{}_{}_{}".format( + RAY_PREFIX, provider, region, identifier, ssh_user + ) diff --git a/python/ray/autoscaler/azure/defaults.yaml b/python/ray/autoscaler/azure/defaults.yaml index 592a0f02e681..1c0e32655a3a 100644 --- a/python/ray/autoscaler/azure/defaults.yaml +++ b/python/ray/autoscaler/azure/defaults.yaml @@ -36,11 +36,10 @@ provider: # How Ray will authenticate with newly launched nodes. auth: ssh_user: ubuntu - # you must specify paths to matching private and public key pair files - # use `ssh-keygen -t rsa -b 4096` to generate a new ssh key pair - ssh_private_key: ~/.ssh/id_rsa - # changes to this should match what is specified in file_mounts - ssh_public_key: ~/.ssh/id_rsa.pub + # SSH keys will be auto-generated with Ray-specific names if not specified + # Uncomment and specify custom paths if you want to use different existing keys: + # ssh_private_key: /path/to/your/key.pem + # ssh_public_key: /path/to/your/key.pub # More specific customization to node configurations can be made using the ARM template azure-vm-template.json file # See documentation here: https://docs.microsoft.com/en-us/azure/templates/microsoft.compute/2019-03-01/virtualmachines @@ -92,7 +91,6 @@ head_node_type: ray.head.default file_mounts: { # "/path1/on/remote/machine": "/path1/on/local/machine", # "/path2/on/remote/machine": "/path2/on/local/machine", - "~/.ssh/id_rsa.pub": "~/.ssh/id_rsa.pub" } # Files or directories to copy from the head node to the worker nodes. The format is a diff --git a/python/ray/autoscaler/azure/example-full.yaml b/python/ray/autoscaler/azure/example-full.yaml index 6bb911268f05..fbcf05f82b17 100644 --- a/python/ray/autoscaler/azure/example-full.yaml +++ b/python/ray/autoscaler/azure/example-full.yaml @@ -59,11 +59,10 @@ provider: # How Ray will authenticate with newly launched nodes. auth: ssh_user: ubuntu - # You must specify paths to matching private and public key pair files. - # Use `ssh-keygen -t rsa -b 4096` to generate a new ssh key pair. - ssh_private_key: ~/.ssh/id_rsa - # Changes to this should match what is specified in file_mounts. - ssh_public_key: ~/.ssh/id_rsa.pub + # SSH keys will be auto-generated with Ray-specific names if not specified + # Uncomment and specify custom paths if you want to use different existing keys: + # ssh_private_key: /path/to/your/key.pem + # ssh_public_key: /path/to/your/key.pub # You can make more specific customization to node configurations can be made using the ARM template azure-vm-template.json file. # See this documentation here: https://docs.microsoft.com/en-us/azure/templates/microsoft.compute/2019-03-01/virtualmachines @@ -127,7 +126,7 @@ head_node_type: ray.head.default file_mounts: { # "/path1/on/remote/machine": "/path1/on/local/machine", # "/path2/on/remote/machine": "/path2/on/local/machine", - "~/.ssh/id_rsa.pub": "~/.ssh/id_rsa.pub"} +} # Files or directories to copy from the head node to the worker nodes. The format is a # list of paths. Ray copies the same path on the head node to the worker node. diff --git a/python/ray/autoscaler/azure/example-gpu-docker.yaml b/python/ray/autoscaler/azure/example-gpu-docker.yaml index 3ebc763e7d26..6f322324b98f 100644 --- a/python/ray/autoscaler/azure/example-gpu-docker.yaml +++ b/python/ray/autoscaler/azure/example-gpu-docker.yaml @@ -43,11 +43,10 @@ provider: # How Ray will authenticate with newly launched nodes. auth: ssh_user: ubuntu - # you must specify paths to matching private and public key pair files - # use `ssh-keygen -t rsa -b 4096` to generate a new ssh key pair - ssh_private_key: ~/.ssh/id_rsa - # changes to this should match what is specified in file_mounts - ssh_public_key: ~/.ssh/id_rsa.pub + # SSH keys will be auto-generated with Ray-specific names if not specified + # Uncomment and specify custom paths if you want to use different existing keys: + # ssh_private_key: /path/to/your/key.pem + # ssh_public_key: /path/to/your/key.pub # Tell the autoscaler the allowed node types and the resources they provide. # The key is the name of the node type, which is just for debugging purposes. @@ -98,7 +97,6 @@ head_node_type: ray.head.gpu file_mounts: { # "/path1/on/remote/machine": "/path1/on/local/machine", # "/path2/on/remote/machine": "/path2/on/local/machine", - "~/.ssh/id_rsa.pub": "~/.ssh/id_rsa.pub" } # List of commands that will be run before `setup_commands`. If docker is diff --git a/python/ray/autoscaler/azure/example-minimal.yaml b/python/ray/autoscaler/azure/example-minimal.yaml index 768eef14a325..de51922482df 100644 --- a/python/ray/autoscaler/azure/example-minimal.yaml +++ b/python/ray/autoscaler/azure/example-minimal.yaml @@ -14,13 +14,14 @@ provider: # How Ray will authenticate with newly launched nodes. auth: ssh_user: ubuntu - # you must specify paths to matching private and public key pair files - # use `ssh-keygen -t rsa -b 4096` to generate a new ssh key pair - ssh_private_key: ~/.ssh/id_rsa - # changes to this should match what is specified in file_mounts - ssh_public_key: ~/.ssh/id_rsa.pub + # SSH keys will be auto-generated with Ray-specific names if not specified + # Uncomment and specify custom paths if you want to use different existing keys: + # ssh_private_key: /path/to/your/key.pem + # ssh_public_key: /path/to/your/key.pub # Files or directories to copy to the head and worker nodes. The format is a # dictionary from REMOTE_PATH: LOCAL_PATH, e.g. file_mounts: { - "~/.ssh/id_rsa.pub": "~/.ssh/id_rsa.pub"} +# "/path1/on/remote/machine": "/path1/on/local/machine", +# "/path2/on/remote/machine": "/path2/on/local/machine", +} From b3c94c12291cc4d424e7e3203d770136070a5093 Mon Sep 17 00:00:00 2001 From: Rueian Date: Thu, 21 Aug 2025 14:49:57 -0700 Subject: [PATCH 220/634] [core][autoscaler][IPPR] add HandleResizeLocalResourceInstances (#54807) ## Why are these changes needed? This is a part of Autoscaler <> Kubernetes In-place Pod Resizing integration. After the Autoscaler finds out that an upscale resize has succeeded at the Kubernetes side, it needs to ask the Raylet on the Pod to resize its logical resources with the new `ResizeLocalResourceInstances` API. ```proto message ResizeLocalResourceInstancesRequest { // Map of resource names to their desired total quantities // For example: {"CPU": 4, "memory": 1000000} map resources = 1; } message ResizeLocalResourceInstancesReply { // Current total resources after the resize operation map total_resources = 1; } rpc ResizeLocalResourceInstances(ResizeLocalResourceInstancesRequest) returns (ResizeLocalResourceInstancesReply) ``` The new API accepts a desired resources map and will return an updated `total_resources` map if it resizes its logical resources successfully. Note that the returned `total_resources` map may differ from the desired resources map in the case of downsizing. The API will reject a desired resources map which contains `unit instance resources`, such as GPU, with `INVALID_ARGUMENT`. Such resources can't be simply resized by the delta of desired amount between old amount and are not the goal of In-place Pod Resizing currently. The API will send an on-demand `RESOURCE_VIEW` message with its latest logical resources to GCS after the resize. GCS will then broadcast the update to all other Raylets. ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Rueian Signed-off-by: Rueian --- src/ray/protobuf/node_manager.proto | 21 +++ src/ray/raylet/node_manager.cc | 98 ++++++++++ src/ray/raylet/node_manager.h | 6 + src/ray/raylet/tests/node_manager_test.cc | 176 ++++++++++++++++++ .../rpc/node_manager/node_manager_server.h | 6 + 5 files changed, 307 insertions(+) diff --git a/src/ray/protobuf/node_manager.proto b/src/ray/protobuf/node_manager.proto index 4a6da212b2dd..16ddfd364243 100644 --- a/src/ray/protobuf/node_manager.proto +++ b/src/ray/protobuf/node_manager.proto @@ -132,6 +132,17 @@ message CancelResourceReserveRequest { message CancelResourceReserveReply {} +message ResizeLocalResourceInstancesRequest { + // Map of resource names to their desired total quantities + // For example: {"CPU": 4, "memory": 1000000} + map resources = 1; +} + +message ResizeLocalResourceInstancesReply { + // Current total resources after the resize operation + map total_resources = 1; +} + // Release a worker back to its raylet. message ReturnWorkerRequest { // Port of the leased worker that we are now returning. @@ -449,6 +460,16 @@ service NodeManagerService { // Failure: Has retry behavior, could be improved to just use retriable grpc client. rpc CancelResourceReserve(CancelResourceReserveRequest) returns (CancelResourceReserveReply); + // Adjust the total number of local resource instances on the raylet to match the + // specified values. + // Success: Returns the updated total resources for the node. If downsizing would make + // available resources negative, the raylet clamps the reduction so that available + // becomes zero. + // Failure: Returns INVALID_ARGUMENT if the request attempts to resize a unit instance + // resource (e.g., GPU), as these cannot be resized by this API. In the cases of + // network errors, the caller should retry the request. + rpc ResizeLocalResourceInstances(ResizeLocalResourceInstancesRequest) + returns (ResizeLocalResourceInstancesReply); // Cancel a pending lease request. This only returns success if the // lease request was not yet granted. // Failure: TODO: This needs to handle network failure diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 684eb7c93daf..51c5d204e481 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -33,6 +33,7 @@ #include "ray/common/buffer.h" #include "ray/common/common_protocol.h" #include "ray/common/constants.h" +#include "ray/common/grpc_util.h" #include "ray/common/memory_monitor.h" #include "ray/common/scheduling/scheduling_ids.h" #include "ray/common/status.h" @@ -1846,6 +1847,103 @@ void NodeManager::HandleCancelResourceReserve( send_reply_callback(Status::OK(), nullptr, nullptr); } +void NodeManager::HandleResizeLocalResourceInstances( + rpc::ResizeLocalResourceInstancesRequest request, + rpc::ResizeLocalResourceInstancesReply *reply, + rpc::SendReplyCallback send_reply_callback) { + const auto &target_resource_map = request.resources(); + + // Check if any resource is a unit instance resource + // Unit instance resources (e.g., GPU) cannot be resized with this API + for (const auto &[resource_name, target_value] : target_resource_map) { + if (ResourceID(resource_name).IsUnitInstanceResource()) { + std::string error_msg = absl::StrFormat( + "Cannot resize unit instance resource '%s'. Unit instance resources " + "(e.g., GPU) cannot be resized dynamically.", + resource_name); + send_reply_callback(Status::InvalidArgument(error_msg), nullptr, nullptr); + return; + } + } + + // Get current local resources and convert to resource maps + const auto ¤t_resources = + cluster_resource_scheduler_.GetLocalResourceManager().GetLocalResources(); + const auto ¤t_total_map = + current_resources.GetTotalResourceInstances().ToNodeResourceSet().GetResourceMap(); + const auto ¤t_available_map = current_resources.GetAvailableResourceInstances() + .ToNodeResourceSet() + .GetResourceMap(); + + // Calculate delta resource map (target - current) and clamp to avoid + // making available resources negative + absl::flat_hash_map delta_resource_map; + for (const auto &[resource_name, target_value] : target_resource_map) { + double current_total = 0.0; + double current_available = 0.0; + + if (auto total_it = current_total_map.find(resource_name); + total_it != current_total_map.end()) { + current_total = total_it->second; + } + + if (auto available_it = current_available_map.find(resource_name); + available_it != current_available_map.end()) { + current_available = available_it->second; + } + + double delta_value = target_value - current_total; + + // Clamp so current_available never goes below 0. + // For example, if delta_value is -4 but the current_available is 2, + // then clamp delta_value to -2. + if (delta_value < -current_available) { + delta_value = -current_available; + } + + if (delta_value != 0.0) { + delta_resource_map[resource_name] = delta_value; + } + } + + // Convert the delta resource map to NodeResourceInstanceSet and apply + if (!delta_resource_map.empty()) { + NodeResourceSet delta_resources(delta_resource_map); + NodeResourceInstanceSet delta_instances(delta_resources); + + // Apply deltas for each resource + for (const auto &resource_id : delta_resources.ExplicitResourceIds()) { + const auto &instances = delta_instances.Get(resource_id); + cluster_resource_scheduler_.GetLocalResourceManager().AddLocalResourceInstances( + resource_id, instances); + } + } + + // Get updated resource state and populate reply + const auto &updated_resources = + cluster_resource_scheduler_.GetLocalResourceManager().GetLocalResources(); + const auto &updated_total_map = + updated_resources.GetTotalResourceInstances().ToNodeResourceSet().GetResourceMap(); + const auto &updated_available_map = updated_resources.GetAvailableResourceInstances() + .ToNodeResourceSet() + .GetResourceMap(); + + if (!delta_resource_map.empty()) { + // Log the updated resources + RAY_LOG(INFO) << "Successfully resized local resources. Current total resources: " + << debug_string(updated_total_map); + RAY_LOG(INFO) << "Available resources: " << debug_string(updated_available_map); + // Trigger scheduling to account for the new resources + cluster_task_manager_.ScheduleAndDispatchTasks(); + } + + // Populate the reply with the current resource state + auto *total_resources = reply->mutable_total_resources(); + total_resources->insert(updated_total_map.begin(), updated_total_map.end()); + + send_reply_callback(Status::OK(), nullptr, nullptr); +} + void NodeManager::HandleReturnWorker(rpc::ReturnWorkerRequest request, rpc::ReturnWorkerReply *reply, rpc::SendReplyCallback send_reply_callback) { diff --git a/src/ray/raylet/node_manager.h b/src/ray/raylet/node_manager.h index df53758583c3..2727f0d3d4a0 100644 --- a/src/ray/raylet/node_manager.h +++ b/src/ray/raylet/node_manager.h @@ -279,6 +279,12 @@ class NodeManager : public rpc::NodeManagerServiceHandler, rpc::PinObjectIDsReply *reply, rpc::SendReplyCallback send_reply_callback) override; + /// Handle a `ResizeLocalResourceInstances` request. + void HandleResizeLocalResourceInstances( + rpc::ResizeLocalResourceInstancesRequest request, + rpc::ResizeLocalResourceInstancesReply *reply, + rpc::SendReplyCallback send_reply_callback) override; + private: FRIEND_TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog); diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index 56bd7d90f873..7041b4152ee0 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -33,6 +33,7 @@ #include "mock/ray/raylet/worker_pool.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/buffer.h" +#include "ray/common/scheduling/cluster_resource_data.h" #include "ray/object_manager/plasma/client.h" #include "ray/raylet/local_object_manager_interface.h" #include "ray/raylet/scheduling/cluster_task_manager.h" @@ -746,6 +747,181 @@ TEST_F(NodeManagerTest, TestPinningAnObjectPendingDeletionFails) { EXPECT_FALSE(failed_pin_reply.successes(0)); } +TEST_F(NodeManagerTest, TestResizeLocalResourceInstancesSuccessful) { + // Test 1: Up scaling (increasing resource capacity) + rpc::ResizeLocalResourceInstancesRequest request; + rpc::ResizeLocalResourceInstancesReply reply; + + (*request.mutable_resources())["CPU"] = 8.0; + (*request.mutable_resources())["memory"] = 16000000.0; + + bool callback_called = false; + + node_manager_->HandleResizeLocalResourceInstances( + request, + &reply, + [&callback_called]( + Status s, std::function success, std::function failure) { + callback_called = true; + EXPECT_TRUE(s.ok()); + }); + EXPECT_TRUE(callback_called); + + // Check that reply contains the updated resources + EXPECT_EQ(reply.total_resources().at("CPU"), 8.0); + EXPECT_EQ(reply.total_resources().at("memory"), 16000000.0); + + // Test 2: Down scaling (decreasing resources) + (*request.mutable_resources())["CPU"] = 4.0; + (*request.mutable_resources())["memory"] = 8000000.0; + + reply.Clear(); + callback_called = false; + node_manager_->HandleResizeLocalResourceInstances( + request, + &reply, + [&callback_called]( + Status s, std::function success, std::function failure) { + callback_called = true; + EXPECT_TRUE(s.ok()); + }); + EXPECT_TRUE(callback_called); + + // Check that reply contains the updated (reduced) resources + EXPECT_EQ(reply.total_resources().at("CPU"), 4.0); + EXPECT_EQ(reply.total_resources().at("memory"), 8000000.0); + + // Test 3: No changes (same values) + reply.Clear(); + callback_called = false; + node_manager_->HandleResizeLocalResourceInstances( + request, + &reply, + [&callback_called]( + Status s, std::function success, std::function failure) { + callback_called = true; + EXPECT_TRUE(s.ok()); + }); + EXPECT_TRUE(callback_called); + + // Should still succeed and return current state + EXPECT_EQ(reply.total_resources().at("CPU"), 4.0); + EXPECT_EQ(reply.total_resources().at("memory"), 8000000.0); + + // Test 4: Now update only CPU, leaving memory unchanged + request.mutable_resources()->clear(); + (*request.mutable_resources())["CPU"] = 8.0; // Double the CPU + + reply.Clear(); + callback_called = false; + node_manager_->HandleResizeLocalResourceInstances( + request, + &reply, + [&callback_called]( + Status s, std::function success, std::function failure) { + callback_called = true; + EXPECT_TRUE(s.ok()); + }); + EXPECT_TRUE(callback_called); + + // Check that CPU was updated, and memory was unchanged + EXPECT_EQ(reply.total_resources().at("CPU"), 8.0); + EXPECT_EQ(reply.total_resources().at("memory"), 8000000.0); +} + +TEST_F(NodeManagerTest, TestResizeLocalResourceInstancesInvalidArgument) { + // Test trying to resize unit instance resources (GPU, etc.) + rpc::ResizeLocalResourceInstancesRequest request; + rpc::ResizeLocalResourceInstancesReply reply; + + (*request.mutable_resources())["GPU"] = 4.0; // GPU is a unit instance resource + + bool callback_called = false; + + node_manager_->HandleResizeLocalResourceInstances( + request, + &reply, + [&callback_called]( + Status s, std::function success, std::function failure) { + callback_called = true; + EXPECT_FALSE(s.ok()); + EXPECT_TRUE(s.IsInvalidArgument()); + // Check the error message contains expected details + std::string error_msg = s.message(); + EXPECT_TRUE(error_msg.find("Cannot resize unit instance resource 'GPU'") != + std::string::npos); + EXPECT_TRUE(error_msg.find("Unit instance resources") != std::string::npos); + EXPECT_TRUE(error_msg.find("cannot be resized dynamically") != std::string::npos); + }); + + // The callback should have been called with an InvalidArgument status + EXPECT_TRUE(callback_called); +} + +TEST_F(NodeManagerTest, TestResizeLocalResourceInstancesClamps) { + // Test 1: Best effort downsizing + rpc::ResizeLocalResourceInstancesRequest request; + rpc::ResizeLocalResourceInstancesReply reply; + + // Initialize resources to a known state + (*request.mutable_resources())["CPU"] = 8.0; + (*request.mutable_resources())["memory"] = 16000000.0; + + bool callback_called = false; + node_manager_->HandleResizeLocalResourceInstances( + request, + &reply, + [&callback_called]( + Status s, std::function success, std::function failure) { + callback_called = true; + EXPECT_TRUE(s.ok()); + }); + EXPECT_TRUE(callback_called); + + // Simulate resource usage by allocating task resources through the local resource + // manager: Use 6 out of 8 CPUs and 2 are free. + const absl::flat_hash_map task_resources = {{"CPU", 6.0}}; + std::shared_ptr task_allocation = + std::make_shared(); + bool allocation_success = + cluster_resource_scheduler_->GetLocalResourceManager().AllocateLocalTaskResources( + task_resources, task_allocation); + EXPECT_TRUE(allocation_success); + + // Now request to downsize CPU to 4. Should clamp to 6. + callback_called = false; + (*request.mutable_resources())["CPU"] = 4.0; + reply.Clear(); + node_manager_->HandleResizeLocalResourceInstances( + request, + &reply, + [&callback_called]( + Status s, std::function success, std::function failure) { + callback_called = true; + EXPECT_TRUE(s.ok()); + }); + EXPECT_TRUE(callback_called); + // Total CPU should be clamped to 6 because there are only 2 CPUs available. + // It should resize from 8 to 6 instead of resizing to 4. + EXPECT_EQ(reply.total_resources().at("CPU"), 6.0); + + // Test 2: Extreme request (e.g., 0). Should clamp to current usage. + callback_called = false; + (*request.mutable_resources())["CPU"] = 0.0; + reply.Clear(); + node_manager_->HandleResizeLocalResourceInstances( + request, + &reply, + [&callback_called]( + Status s, std::function success, std::function failure) { + callback_called = true; + EXPECT_TRUE(s.ok()); + }); + EXPECT_TRUE(callback_called); + // With 6 used, total should remain 6 + EXPECT_EQ(reply.total_resources().at("CPU"), 6.0); +} + } // namespace ray::raylet int main(int argc, char **argv) { diff --git a/src/ray/rpc/node_manager/node_manager_server.h b/src/ray/rpc/node_manager/node_manager_server.h index f7e1cc37f171..7e0a726832e4 100644 --- a/src/ray/rpc/node_manager/node_manager_server.h +++ b/src/ray/rpc/node_manager/node_manager_server.h @@ -48,6 +48,7 @@ namespace rpc { RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(PrepareBundleResources) \ RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(CommitBundleResources) \ RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(CancelResourceReserve) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ResizeLocalResourceInstances) \ RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ReleaseUnusedBundles) \ RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GetSystemConfig) \ RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(IsLocalWorkerDead) \ @@ -136,6 +137,11 @@ class NodeManagerServiceHandler { rpc::CancelResourceReserveReply *reply, rpc::SendReplyCallback send_reply_callback) = 0; + virtual void HandleResizeLocalResourceInstances( + rpc::ResizeLocalResourceInstancesRequest request, + rpc::ResizeLocalResourceInstancesReply *reply, + rpc::SendReplyCallback send_reply_callback) = 0; + virtual void HandlePinObjectIDs(PinObjectIDsRequest request, PinObjectIDsReply *reply, SendReplyCallback send_reply_callback) = 0; From 9cd0366a741d55b2d64a136f14edd5b5e5880335 Mon Sep 17 00:00:00 2001 From: Jason Li <57246540+JasonLi1909@users.noreply.github.com> Date: Thu, 21 Aug 2025 15:02:36 -0700 Subject: [PATCH 221/634] Make Ray Train Dashboard Panel Ids Static (#55559) Currently, the ids of the Ray Train Dashboard panels use a PanelId.next() function that sets the id of a new panel to be the last id incremented by 1. This can make it difficult to add panels anywhere aside from the bottom of the dashboard. This PR introduces static panel ids to the Ray Train Dashboard. --------- Signed-off-by: JasonLi1909 --- .../dashboards/train_dashboard_panels.py | 43 +++++++------------ 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py index e003c27c82a0..f6f88729f7ed 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py @@ -7,22 +7,9 @@ ) -class PanelId: - """ - A class to generate unique panel IDs. - """ - - id = 0 - - @staticmethod - def next(): - PanelId.id += 1 - return PanelId.id - - # Ray Train Metrics (Controller) CONTROLLER_STATE_PANEL = Panel( - id=PanelId.next(), + id=1, title="Controller State", description="Current state of the train controller.", unit="", @@ -35,7 +22,7 @@ def next(): ) CONTROLLER_OPERATION_TIME_PANEL = Panel( - id=PanelId.next(), + id=2, title="Controller Operation Time", description="Time taken by the controller for worker group operations.", unit="seconds", @@ -55,7 +42,7 @@ def next(): # Ray Train Metrics (Worker) WORKER_CHECKPOINT_REPORT_TIME_PANEL = Panel( - id=PanelId.next(), + id=3, title="Checkpoint Report Time", description="Time taken to report a checkpoint to storage.", unit="seconds", @@ -71,7 +58,7 @@ def next(): # Core System Resources CPU_UTILIZATION_PANEL = Panel( - id=PanelId.next(), + id=4, title="CPU Usage", description="CPU core utilization across all workers.", unit="cores", @@ -88,7 +75,7 @@ def next(): ) MEMORY_UTILIZATION_PANEL = Panel( - id=PanelId.next(), + id=5, title="Total Memory Usage", description="Total physical memory used vs total available memory.", unit="bytes", @@ -105,7 +92,7 @@ def next(): ) MEMORY_DETAILED_PANEL = Panel( - id=PanelId.next(), + id=6, title="Memory Allocation Details", description="Memory allocation details including available and shared memory.", unit="bytes", @@ -124,7 +111,7 @@ def next(): # GPU Resources # TODO: Add GPU Device/Index as a filter. GPU_UTILIZATION_PANEL = Panel( - id=PanelId.next(), + id=7, title="GPU Usage", description="GPU utilization across all workers.", unit="GPUs", @@ -141,7 +128,7 @@ def next(): ) GPU_MEMORY_UTILIZATION_PANEL = Panel( - id=PanelId.next(), + id=8, title="GPU Memory Usage", description="GPU memory usage across all workers.", unit="bytes", @@ -159,7 +146,7 @@ def next(): # Storage Resources DISK_UTILIZATION_PANEL = Panel( - id=PanelId.next(), + id=9, title="Disk Space Usage", description="Disk space usage across all workers.", unit="bytes", @@ -176,7 +163,7 @@ def next(): ) DISK_THROUGHPUT_PANEL = Panel( - id=PanelId.next(), + id=10, title="Disk Throughput", description="Current disk read/write throughput.", unit="Bps", @@ -193,7 +180,7 @@ def next(): ) DISK_OPERATIONS_PANEL = Panel( - id=PanelId.next(), + id=11, title="Disk Operations", description="Current disk read/write operations per second.", unit="ops/s", @@ -211,7 +198,7 @@ def next(): # Network Resources NETWORK_THROUGHPUT_PANEL = Panel( - id=PanelId.next(), + id=12, title="Network Throughput", description="Current network send/receive throughput.", unit="Bps", @@ -228,7 +215,7 @@ def next(): ) NETWORK_TOTAL_PANEL = Panel( - id=PanelId.next(), + id=13, title="Network Total Traffic", description="Total network traffic sent/received.", unit="bytes", @@ -250,7 +237,7 @@ def next(): # Train Metrics Row Row( title="Train Metrics", - id=PanelId.next(), + id=14, panels=[ # Ray Train Metrics (Controller) CONTROLLER_STATE_PANEL, @@ -263,7 +250,7 @@ def next(): # System Resources Row Row( title="Resource Utilization", - id=PanelId.next(), + id=15, panels=[ CPU_UTILIZATION_PANEL, MEMORY_UTILIZATION_PANEL, From d81133a2d2f674468025e461a05ac3a4d066dd7f Mon Sep 17 00:00:00 2001 From: Abrar Sheikh Date: Thu, 21 Aug 2025 15:35:34 -0700 Subject: [PATCH 222/634] introduce deployment rank manager (#55729) Part 1 of https://github.com/ray-project/ray/pull/54938 This PR introduces `DeploymentRankManager` class that will manage ranks for all replicas within a deployment. In next PR I will use this object to integrate with DeploymentState --------- Signed-off-by: abrar --- python/ray/serve/_private/constants.py | 4 + python/ray/serve/_private/deployment_state.py | 263 ++++++++++++++ .../unit/test_deployment_rank_manager.py | 343 ++++++++++++++++++ 3 files changed, 610 insertions(+) create mode 100644 python/ray/serve/tests/unit/test_deployment_rank_manager.py diff --git a/python/ray/serve/_private/constants.py b/python/ray/serve/_private/constants.py index dd46f0c6c20c..0f682244c709 100644 --- a/python/ray/serve/_private/constants.py +++ b/python/ray/serve/_private/constants.py @@ -462,5 +462,9 @@ "RAY_SERVE_REQUEST_PATH_LOG_BUFFER_SIZE", 1 ) +# Feature flag to fail the deployment if the rank is not set. +# TODO (abrar): Remove this flag after the feature is stable. +RAY_SERVE_FAIL_ON_RANK_ERROR = get_env_bool("RAY_SERVE_FAIL_ON_RANK_ERROR", "0") + # The message to return when the replica is healthy. HEALTHY_MESSAGE = "success" diff --git a/python/ray/serve/_private/deployment_state.py b/python/ray/serve/_private/deployment_state.py index 6bf969f9538f..8a33bc5eedc9 100644 --- a/python/ray/serve/_private/deployment_state.py +++ b/python/ray/serve/_private/deployment_state.py @@ -38,6 +38,7 @@ MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT, MAX_PER_REPLICA_RETRY_COUNT, RAY_SERVE_ENABLE_TASK_EVENTS, + RAY_SERVE_FAIL_ON_RANK_ERROR, RAY_SERVE_FORCE_STOP_UNHEALTHY_REPLICAS, RAY_SERVE_USE_COMPACT_SCHEDULING_STRATEGY, REPLICA_HEALTH_CHECK_UNHEALTHY_THRESHOLD, @@ -1389,6 +1390,268 @@ def __repr__(self): return repr(self._replicas) +class DeploymentRankManager: + """Manages replica ranks for a deployment. + This class handles rank assignment, release, consistency checking, and reassignment. + It maintains the rank system invariants and provides a clean interface for rank operations. + """ + + def __init__(self, _fail_on_error: Optional[bool] = None): + # Maps replica_id to assigned rank + self._replica_ranks: Dict[str, int] = {} + # Set of available ranks (initially empty, grows as target replicas change) + self._released_ranks: Set[int] = set() + # Next rank to assign (increments as new replicas are created) + self._next_rank: int = 0 + # Whether to fail on rank errors (for testing control) + self._fail_on_error = ( + _fail_on_error + if _fail_on_error is not None + else RAY_SERVE_FAIL_ON_RANK_ERROR + ) + + def assign_rank(self, replica_id: str) -> int: + """Assign a rank to a new replica. + Args: + replica_id: The unique ID of the replica + Returns: + The assigned rank + Raises: + RuntimeError: If the replica already has a rank assigned + """ + if replica_id in self._replica_ranks: + raise RuntimeError( + f"Replica {replica_id} already has a rank assigned: {self._replica_ranks[replica_id]}" + ) + + # First try to reuse an available rank + if self._released_ranks: + rank = min(self._released_ranks) + self._released_ranks.remove(rank) + else: + # Otherwise use the next available rank + rank = self._next_rank + self._next_rank += 1 + + self._replica_ranks[replica_id] = rank + return rank + + def release_rank(self, replica_id: str) -> None: + """Release a rank when a replica is stopped. + Args: + replica_id: The unique ID of the replica whose rank should be released + """ + if replica_id not in self._replica_ranks: + raise RuntimeError(f"Replica {replica_id} has no rank assigned") + + rank = self._replica_ranks.pop(replica_id) + self._released_ranks.add(rank) + + def recover_rank(self, replica_id: str, rank: int) -> None: + """Recover a rank from a live replica during controller restart. + Args: + replica_id: The unique ID of the replica + rank: The rank to recover + Raises: + RuntimeError: If the replica already has a rank or the rank is invalid + ValueError: If the rank is invalid (negative) + """ + if replica_id in self._replica_ranks: + raise RuntimeError(f"Replica {replica_id} already has a rank assigned") + + self._replica_ranks[replica_id] = rank + + # Update available ranks tracking + if rank in self._released_ranks: + self._released_ranks.remove(rank) + + # Update next_rank to ensure we don't assign duplicates + if rank >= self._next_rank: + self._next_rank = rank + 1 + + def get_replica_rank(self, replica_id: str) -> Optional[int]: + """Get the rank assigned to a replica. + Args: + replica_id: The unique ID of the replica + Returns: + The assigned rank, or None if no rank is assigned + """ + if replica_id not in self._replica_ranks: + raise RuntimeError(f"Replica {replica_id} has no rank assigned") + return self._replica_ranks.get(replica_id) + + def get_replica_ranks_mapping(self) -> Dict[str, int]: + """Get a copy of the current replica ranks mapping. + Returns: + A copy of the replica_id to rank mapping + """ + return self._replica_ranks.copy() + + def check_rank_consistency_and_reassign_minimally( + self, + active_replicas: List["DeploymentReplica"], + ) -> List["DeploymentReplica"]: + """Verify rank system invariants and reassign ranks when needed. + This method ensures: + 1. All active replicas have ranks + 2. No duplicate ranks exist + 3. Ranks are contiguous when at target replica count + Args: + active_replicas: List of currently active replicas + Returns: + List of replicas that need to be reconfigured with new ranks + Raises: + RuntimeError: If rank system invariants are violated + """ + if not active_replicas: + return [] + + active_replica_ids = { + replica.replica_id.unique_id for replica in active_replicas + } + replica_ids_needs_reconfiguration = set() + + # Check for stale ranks - this should never happen + stale_replica_ids = set(self._replica_ranks.keys()) - active_replica_ids + if stale_replica_ids: + logger.error( + f"Found stale ranks for replicas: {stale_replica_ids}. " + "This should never happen. Please report this as a bug." + ) + if self._fail_on_error: + raise RuntimeError("Controller rank system is in an invalid state.") + # TODO (abrar): handle this case by removing the stale ranks, but remove this when + # RAY_SERVE_FAIL_ON_RANK_ERROR is set to 1 in the future + for replica_id in stale_replica_ids: + self.release_rank(replica_id) + replica_ids_needs_reconfiguration.add(replica_id) + + # Verify system invariants - all active replicas must have ranks + unranked_replica_ids = active_replica_ids - set(self._replica_ranks.keys()) + if unranked_replica_ids: + logger.error( + f"Found active replicas without ranks: {unranked_replica_ids}. " + "This should never happen. Please report this as a bug." + ) + if self._fail_on_error: + raise RuntimeError("Controller rank system is in an invalid state.") + # TODO (abrar): handle this case by assigning new ranks to the unranked replicas + # but remove this when RAY_SERVE_FAIL_ON_RANK_ERROR is set to 1 in the future + for replica_id in unranked_replica_ids: + self.assign_rank(replica_id) + replica_ids_needs_reconfiguration.add(replica_id) + + # Check for duplicate ranks - this should never happen + rank_counts = {} + for replica_id, rank in self._replica_ranks.copy().items(): + if replica_id in active_replica_ids: # Only check active replicas + rank_counts[rank] = rank_counts.get(rank, 0) + 1 + if rank_counts[rank] > 1: + logger.error( + f"Found duplicate rank {rank} assigned to multiple replicas. " + "This should never happen. Please report this as a bug." + ) + if self._fail_on_error: + raise RuntimeError( + "Controller rank system is in an invalid state." + ) + # TODO (abrar): handle this case by releasing the rank of the replica with the duplicate rank + # and assigning a new rank to the replica with the duplicate rank + # but remove this when RAY_SERVE_FAIL_ON_RANK_ERROR is set to 1 in the future + self._replica_ranks.pop(replica_id) + self.assign_rank(replica_id) + replica_ids_needs_reconfiguration.add(replica_id) + + # Check if we need to reassign ranks for contiguity + # Only force contiguity when at target replica count (e.g., after autoscaling down) + current_ranks = sorted(self._replica_ranks.values()) + expected_ranks = list(range(len(active_replicas))) + + replicas_needing_reconfiguration = [] + + if current_ranks != expected_ranks: + logger.info( + f"Deployment at target replica count but ranks are not contiguous. " + f"Current: {current_ranks}, Expected: {expected_ranks}. " + "Performing minimal reassignment." + ) + replicas_needing_reconfiguration.extend( + self._perform_minimal_rank_reassignment(active_replicas) + ) + + # TODO (abrar): remove this when RAY_SERVE_FAIL_ON_RANK_ERROR is set to 1 in the future + for replica in active_replicas: + if replica.replica_id.unique_id in replica_ids_needs_reconfiguration: + replicas_needing_reconfiguration.append(replica) + + return replicas_needing_reconfiguration + + def _perform_minimal_rank_reassignment( + self, active_replicas: List["DeploymentReplica"] + ) -> List["DeploymentReplica"]: + """Perform minimal rank reassignment to achieve contiguity. + This method reassigns ranks while minimizing the number of replicas that need + to be reconfigured. It prioritizes keeping existing ranks when possible. + Args: + active_replicas: List of currently active replicas + Returns: + List of replicas that need to be reconfigured with new ranks + """ + target_ranks_set = set(range(len(active_replicas))) + + # Find which replicas need new ranks + replicas_needing_ranks = [] + replicas_keeping_ranks = [] + + for replica in active_replicas: + replica_id = replica.replica_id.unique_id + current_rank = self.get_replica_rank(replica_id) + + if current_rank in target_ranks_set: + # This replica can keep its rank + target_ranks_set.remove(current_rank) # O(1) operation + replicas_keeping_ranks.append(replica) + else: + # This replica needs a new rank + replicas_needing_ranks.append(replica) + + # Convert remaining target ranks to sorted list for deterministic assignment + available_ranks = sorted(target_ranks_set) + + # Assign new ranks to replicas that need them + for i, replica in enumerate(replicas_needing_ranks): + replica_id = replica.replica_id.unique_id + new_rank = available_ranks[i] # O(1) operation + + # Store the old rank before updating + old_rank = self._replica_ranks[replica_id] + + logger.info( + f"Reassigning replica {replica_id}: rank {old_rank} -> {new_rank}" + ) + + # Update the rank mapping + self._replica_ranks[replica_id] = new_rank + # Remove the newly assigned rank from available ranks + self._released_ranks.discard(new_rank) + # Add the old rank back to available ranks for reuse + self._released_ranks.add(old_rank) + + # Log the reassignment summary + logger.info( + f"Minimal reassignment complete: {len(replicas_keeping_ranks)} replicas kept ranks, " + f"{len(replicas_needing_ranks)} replicas reassigned" + ) + + return replicas_needing_ranks + + def clear(self) -> None: + """Clear all rank data. Used for testing and reset.""" + self._replica_ranks.clear() + self._released_ranks.clear() + self._next_rank = 0 + + class DeploymentState: """Manages the target state and replicas for a single deployment.""" diff --git a/python/ray/serve/tests/unit/test_deployment_rank_manager.py b/python/ray/serve/tests/unit/test_deployment_rank_manager.py new file mode 100644 index 000000000000..211ce31b9471 --- /dev/null +++ b/python/ray/serve/tests/unit/test_deployment_rank_manager.py @@ -0,0 +1,343 @@ +import pytest + +from ray.serve._private.common import DeploymentID, ReplicaID +from ray.serve._private.deployment_state import DeploymentRankManager + + +@pytest.fixture +def rank_manager(): + """Fixture providing a fresh DeploymentRankManager instance for each test.""" + return DeploymentRankManager() + + +class MockDeploymentReplica: + """Mock replica for testing without heavy dependencies.""" + + def __init__( + self, + replica_id: str, + deployment_name: str = "test_deployment", + app_name: str = "test_app", + ): + self.replica_id = ReplicaID( + unique_id=replica_id, + deployment_id=DeploymentID(name=deployment_name, app_name=app_name), + ) + + def __str__(self): + return f"MockDeploymentReplica(replica_id={self.replica_id})" + + +class TestDeploymentRankManager: + """Test cases for DeploymentRankManager.""" + + def test_init(self, rank_manager): + """Test initialization creates empty state.""" + assert rank_manager._replica_ranks == {} + assert rank_manager._released_ranks == set() + assert rank_manager._next_rank == 0 + + def test_assign_rank_first_replica(self, rank_manager): + """Test assigning rank to first replica.""" + rank = rank_manager.assign_rank("replica_1") + assert rank == 0 + assert rank_manager._replica_ranks["replica_1"] == 0 + assert rank_manager._next_rank == 1 + assert rank_manager._released_ranks == set() + + def test_assign_rank_multiple_replicas(self, rank_manager): + """Test assigning ranks to multiple replicas.""" + rank1 = rank_manager.assign_rank("replica_1") + rank2 = rank_manager.assign_rank("replica_2") + rank3 = rank_manager.assign_rank("replica_3") + + assert rank1 == 0 + assert rank2 == 1 + assert rank3 == 2 + assert rank_manager._next_rank == 3 + assert len(rank_manager._replica_ranks) == 3 + + def test_assign_rank_reuses_released_ranks(self, rank_manager): + """Test that released ranks are reused before assigning new ones.""" + # Assign ranks to 3 replicas + rank_manager.assign_rank("replica_1") + rank_manager.assign_rank("replica_2") + rank_manager.assign_rank("replica_3") + + # Release middle rank + rank_manager.release_rank("replica_2") + assert 1 in rank_manager._released_ranks + + # New replica should get the released rank + rank = rank_manager.assign_rank("replica_4") + assert rank == 1 + assert 1 not in rank_manager._released_ranks + + def test_assign_rank_duplicate_fails(self): + """Test assigning rank to replica that already has one fails when flag is enabled.""" + rank_manager = DeploymentRankManager() + rank_manager.assign_rank("replica_1") + + with pytest.raises(RuntimeError, match="already has a rank assigned"): + rank_manager.assign_rank("replica_1") + + def test_release_rank(self, rank_manager): + """Test releasing a rank makes it available for reuse.""" + rank_manager.assign_rank("replica_1") + rank_manager.assign_rank("replica_2") + + rank_manager.release_rank("replica_1") + + assert "replica_1" not in rank_manager._replica_ranks + assert 0 in rank_manager._released_ranks + assert "replica_2" in rank_manager._replica_ranks + + def test_release_rank_nonexistent_replica(self): + """Test releasing rank for non-existent replica is safe.""" + rank_manager = DeploymentRankManager() + with pytest.raises(RuntimeError, match="has no rank assigned"): + rank_manager.release_rank("nonexistent") + + def test_recover_rank_basic(self, rank_manager): + """Test basic rank recovery.""" + rank_manager.recover_rank("replica_1", 5) + + assert rank_manager._replica_ranks["replica_1"] == 5 + assert rank_manager._next_rank == 6 + + def test_recover_rank_updates_next_rank(self, rank_manager): + """Test that recovering a high rank updates next_rank appropriately.""" + rank_manager.assign_rank("replica_1") # Gets rank 0 + rank_manager.recover_rank("replica_2", 10) + + assert rank_manager._next_rank == 11 + + # New replica should get rank 11 + rank = rank_manager.assign_rank("replica_3") + assert rank == 11 + + def test_recover_rank_removes_from_available(self, rank_manager): + """Test that recovering a rank removes it from available ranks.""" + rank_manager.assign_rank("replica_1") + rank_manager.assign_rank("replica_2") + rank_manager.release_rank("replica_1") # Rank 0 becomes available + + assert 0 in rank_manager._released_ranks + + # Recover rank 0 + rank_manager.recover_rank("replica_3", 0) + + assert 0 not in rank_manager._released_ranks + assert rank_manager._replica_ranks["replica_3"] == 0 + + def test_recover_rank_duplicate_fails(self): + """Test recovering rank for replica that already has one fails when flag is enabled.""" + rank_manager = DeploymentRankManager() + rank_manager.assign_rank("replica_1") + + with pytest.raises(RuntimeError, match="already has a rank assigned"): + rank_manager.recover_rank("replica_1", 5) + + def test_get_replica_rank_existing(self, rank_manager): + """Test getting rank for existing replica.""" + rank_manager.assign_rank("replica_1") + rank = rank_manager.get_replica_rank("replica_1") + assert rank == 0 + + def test_get_replica_rank_nonexistent_fails(self): + """Test getting rank for non-existent replica fails when flag is enabled.""" + rank_manager = DeploymentRankManager() + with pytest.raises(RuntimeError, match="has no rank assigned"): + rank_manager.get_replica_rank("nonexistent") + + def test_get_replica_ranks_mapping(self, rank_manager): + """Test getting copy of replica ranks mapping.""" + rank_manager.assign_rank("replica_1") + rank_manager.assign_rank("replica_2") + + mapping = rank_manager.get_replica_ranks_mapping() + expected = {"replica_1": 0, "replica_2": 1} + + assert mapping == expected + + # Verify it's a copy + mapping["replica_3"] = 2 + assert "replica_3" not in rank_manager._replica_ranks + + def test_clear(self, rank_manager): + """Test clearing all rank data.""" + rank_manager.assign_rank("replica_1") + rank_manager.assign_rank("replica_2") + rank_manager.release_rank("replica_1") + + rank_manager.clear() + + assert rank_manager._replica_ranks == {} + assert rank_manager._released_ranks == set() + assert rank_manager._next_rank == 0 + + def test_check_rank_consistency_empty_replicas(self, rank_manager): + """Test consistency check with no active replicas.""" + result = rank_manager.check_rank_consistency_and_reassign_minimally([]) + assert result == [] + + def test_check_rank_consistency_contiguous_ranks(self, rank_manager): + """Test consistency check with contiguous ranks (no reassignment needed).""" + # Set up contiguous ranks + replica1 = MockDeploymentReplica("replica_1") + replica2 = MockDeploymentReplica("replica_2") + replica3 = MockDeploymentReplica("replica_3") + + rank_manager.assign_rank("replica_1") # rank 0 + rank_manager.assign_rank("replica_2") # rank 1 + rank_manager.assign_rank("replica_3") # rank 2 + + result = rank_manager.check_rank_consistency_and_reassign_minimally( + [replica1, replica2, replica3] + ) + + assert result == [] + + def test_check_rank_consistency_non_contiguous_ranks(self, rank_manager): + """Test consistency check with non-contiguous ranks (reassignment needed).""" + # Set up non-contiguous ranks (simulate a replica being removed) + replica1 = MockDeploymentReplica("replica_1") + replica2 = MockDeploymentReplica("replica_2") + replica3 = MockDeploymentReplica("replica_3") + + # Manually set up non-contiguous ranks + rank_manager._replica_ranks = { + "replica_1": 0, + "replica_2": 2, # Gap at rank 1 + "replica_3": 3, + } + + result = rank_manager.check_rank_consistency_and_reassign_minimally( + [replica1, replica2, replica3] + ) + + # Should reassign some replicas to make ranks contiguous + assert len(result) > 0 + + # After reassignment, ranks should be contiguous + final_ranks = sorted(rank_manager._replica_ranks.values()) + expected_ranks = [0, 1, 2] + assert final_ranks == expected_ranks + + def test_minimal_reassignment_keeps_existing_when_possible(self, rank_manager): + """Test that minimal reassignment keeps existing ranks when possible.""" + replica1 = MockDeploymentReplica("replica_1") + replica2 = MockDeploymentReplica("replica_2") + replica3 = MockDeploymentReplica("replica_3") + replica4 = MockDeploymentReplica("replica_4") + + # Set up ranks: 0, 2, 5, 7 (non-contiguous) + rank_manager._replica_ranks = { + "replica_1": 0, # Should keep this + "replica_2": 2, # Should keep this + "replica_3": 5, # Should be reassigned to 1 + "replica_4": 7, # Should be reassigned to 3 + } + + result = rank_manager.check_rank_consistency_and_reassign_minimally( + [replica1, replica2, replica3, replica4] + ) + + # Verify minimal reassignment + assert len(result) == 2 # Only 2 replicas should be reassigned + reassigned_ids = {r.replica_id.unique_id for r in result} + assert reassigned_ids == {"replica_3", "replica_4"} + + # Verify final ranks are contiguous + final_ranks = sorted(rank_manager._replica_ranks.values()) + assert final_ranks == [0, 1, 2, 3] + + # Verify that replica_1 and replica_2 kept their original ranks + assert rank_manager._replica_ranks["replica_1"] == 0 + assert rank_manager._replica_ranks["replica_2"] == 2 + + def test_check_rank_consistency_unranked_replicas_fails_when_flag_enabled(self): + """Test consistency check fails when active replicas have no ranks and flag is enabled.""" + rank_manager = DeploymentRankManager(_fail_on_error=True) + replica1 = MockDeploymentReplica("replica_1") + + with pytest.raises( + RuntimeError, match="Controller rank system is in an invalid state" + ): + rank_manager.check_rank_consistency_and_reassign_minimally([replica1]) + + def test_check_rank_consistency_unranked_replicas_logs_when_flag_disabled(self): + """Test consistency check only logs when active replicas have no ranks and flag is disabled.""" + rank_manager = DeploymentRankManager(_fail_on_error=False) + replica1 = MockDeploymentReplica("replica_1") + + # When flag is disabled, it logs error but still tries to proceed with reassignment + # However, the reassignment will fail when trying to access ranks that don't exist + result = rank_manager.check_rank_consistency_and_reassign_minimally([replica1]) + assert result == [replica1] + + def test_check_rank_consistency_stale_ranks_fails_when_flag_enabled(self): + """Test consistency check fails when there are stale ranks and flag is enabled.""" + rank_manager = DeploymentRankManager(_fail_on_error=True) + replica1 = MockDeploymentReplica("replica_1") + + # Set up stale rank (replica not in active list) + rank_manager.assign_rank("replica_1") + rank_manager.assign_rank("stale_replica") + + with pytest.raises( + RuntimeError, match="Controller rank system is in an invalid state" + ): + rank_manager.check_rank_consistency_and_reassign_minimally([replica1]) + + def test_check_rank_consistency_stale_ranks_logs_when_flag_disabled(self): + """Test consistency check only logs when there are stale ranks and flag is disabled.""" + rank_manager = DeploymentRankManager(_fail_on_error=False) + replica1 = MockDeploymentReplica("replica_1") + + # Set up stale rank (replica not in active list) + rank_manager.assign_rank("replica_1") + rank_manager.assign_rank("stale_replica") + + # When flag is disabled, it logs error but continues with reassignment + # Since only replica_1 is active and has rank 0, no reassignment needed + result = rank_manager.check_rank_consistency_and_reassign_minimally([replica1]) + assert result == [] + + def test_check_rank_consistency_duplicate_ranks_fails_when_flag_enabled(self): + """Test consistency check fails when there are duplicate ranks and flag is enabled.""" + rank_manager = DeploymentRankManager(_fail_on_error=True) + replica1 = MockDeploymentReplica("replica_1") + replica2 = MockDeploymentReplica("replica_2") + + # Manually create duplicate ranks (this should never happen in normal operation) + rank_manager._replica_ranks = {"replica_1": 0, "replica_2": 0} # Duplicate! + + with pytest.raises( + RuntimeError, match="Controller rank system is in an invalid state" + ): + rank_manager.check_rank_consistency_and_reassign_minimally( + [replica1, replica2] + ) + + def test_check_rank_consistency_duplicate_ranks_logs_when_flag_disabled(self): + """Test consistency check only logs when there are duplicate ranks and flag is disabled.""" + rank_manager = DeploymentRankManager(_fail_on_error=False) + replica1 = MockDeploymentReplica("replica_1") + replica2 = MockDeploymentReplica("replica_2") + + # Manually create duplicate ranks (this should never happen in normal operation) + rank_manager._replica_ranks = {"replica_1": 0, "replica_2": 0} # Duplicate! + rank_manager._next_rank = 1 + + # When flag is disabled, it logs error but still performs reassignment to fix the issue + result = rank_manager.check_rank_consistency_and_reassign_minimally( + [replica1, replica2] + ) + assert result == [replica2] or result == [replica1] + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", "-s", __file__])) From 028f4b9637efc836ab3db1014f16a7034dad3072 Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Thu, 21 Aug 2025 15:39:17 -0700 Subject: [PATCH 223/634] [serve] add throughput opt env var for serve (#55804) ## Why are these changes needed? consolidates throughput opt env var into a single one ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: akyang-anyscale --- python/ray/serve/_private/constants.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/ray/serve/_private/constants.py b/python/ray/serve/_private/constants.py index 0f682244c709..168fcb718ecf 100644 --- a/python/ray/serve/_private/constants.py +++ b/python/ray/serve/_private/constants.py @@ -468,3 +468,12 @@ # The message to return when the replica is healthy. HEALTHY_MESSAGE = "success" + +# If throughput optimized Ray Serve is enabled, set the following constants. +# This should be at the end. +RAY_SERVE_THROUGHPUT_OPTIMIZED = get_env_bool("RAY_SERVE_THROUGHPUT_OPTIMIZED", "0") +if RAY_SERVE_THROUGHPUT_OPTIMIZED: + RAY_SERVE_RUN_USER_CODE_IN_SEPARATE_THREAD = False + RAY_SERVE_REQUEST_PATH_LOG_BUFFER_SIZE = 1000 + RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP = False + RAY_SERVE_LOG_TO_STDERR = False From 413f3591dea88323d7d49d7809803c4c30e8d0d6 Mon Sep 17 00:00:00 2001 From: kourosh hakhamaneshi <31483498+kouroshHakha@users.noreply.github.com> Date: Fri, 22 Aug 2025 02:29:46 +0200 Subject: [PATCH 224/634] [serve.llm] Fixed DP DSV3 issues (#55802) Signed-off-by: Kourosh Hakhamaneshi --- .../_internal/serve/configs/server_models.py | 44 ++++++++++++++++--- .../deployments/data_parallel/dp_server.py | 5 ++- .../llm/vllm/kv_transfer_backends/base.py | 20 +++++++-- .../kv_transfer_backends/nixl_connector.py | 31 ++++++++++--- .../serve/deployments/llm/vllm/vllm_models.py | 3 +- .../tests/serve/cpu/configs/test_models.py | 19 ++++---- 6 files changed, 94 insertions(+), 28 deletions(-) diff --git a/python/ray/llm/_internal/serve/configs/server_models.py b/python/ray/llm/_internal/serve/configs/server_models.py index 910fce823c0c..3be3e246929a 100644 --- a/python/ray/llm/_internal/serve/configs/server_models.py +++ b/python/ray/llm/_internal/serve/configs/server_models.py @@ -424,6 +424,38 @@ def update_engine_kwargs(self, **kwargs: Any) -> None: if self._engine_config: self._engine_config.engine_kwargs.update(kwargs) + def _merge_replica_actor_and_child_actor_bundles( + self, + child_actor_bundles: List[Dict[str, float]], + replica_actor_bundle: Dict[str, float], + ) -> List[Dict[str, float]]: + """Sum up the bundles from replica actor bundles with the first bundle from child actor bundles. + + This is because the replica actor will use the first bundle in the list, and we want to collocate the replica actor with the child actor. + So we need to group them together. + + So for example: + child_actor_bundles = [{"GPU": 1, "CPU": 1}, {"GPU": 1, "CPU": 1}] + replica_actor_bundle = {"GPU": 0, "CPU": 1, "memory": 100} + return [{"GPU": 1, "CPU": 2, "memory": 100}, {"GPU": 1, "CPU": 1}] + """ + + if not child_actor_bundles: + return [replica_actor_bundle] + + if not replica_actor_bundle: + return child_actor_bundles + + first_bundle = child_actor_bundles[0] + bundle_key_set = set(first_bundle.keys()) | set(replica_actor_bundle.keys()) + + for key in bundle_key_set: + first_bundle[key] = replica_actor_bundle.get(key, 0) + first_bundle.get( + key, 0 + ) + + return [first_bundle] + child_actor_bundles[1:] + def _set_deployment_placement_options(self) -> Dict[str, Any]: deployment_config = self.deployment_config engine_config = self.get_engine_config() @@ -449,15 +481,17 @@ def _set_deployment_placement_options(self) -> Dict[str, Any]: ) try: - bundles = engine_config.placement_bundles + child_actor_bundles = engine_config.placement_bundles except ValueError: # May happen if all bundles are empty. - bundles = [] + child_actor_bundles = [] - bundles = [replica_actor_resources] + bundles + pg_bundles = self._merge_replica_actor_and_child_actor_bundles( + child_actor_bundles, replica_actor_resources + ) deployment_config.update( { - "placement_group_bundles": bundles, + "placement_group_bundles": pg_bundles, "placement_group_strategy": engine_config.placement_strategy, } ) @@ -569,7 +603,7 @@ def _setup_kv_connector_backend(self): raise ValueError(f"Unsupported connector type: {kv_connector}") # 2. Setup the backend - kv_connector_backend = kv_connector_backend_class(kv_transfer_config) + kv_connector_backend = kv_connector_backend_class(self) kv_connector_backend.setup() diff --git a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py index 8e2bc445f3fd..11c8d8e25e98 100644 --- a/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py +++ b/python/ray/llm/_internal/serve/deployments/data_parallel/dp_server.py @@ -76,6 +76,7 @@ def build_dp_deployment( llm_config: LLMConfig, *, name_prefix: Optional[str] = None, + options_override: Optional[dict] = None, ) -> Application: """Build a data parallel LLM deployment.""" dp_size = llm_config.engine_kwargs.get("data_parallel_size", 1) @@ -89,10 +90,12 @@ def build_dp_deployment( # the number of ranks per node because that has special semantics in vLLM. dp_size_per_node = llm_config.experimental_configs.get("dp_size_per_node", None) - deployment_options = llm_config.get_serve_options(name_prefix=name_prefix) dp_rank_assigner = DPRankAssigner.bind( dp_size=dp_size, dp_size_per_node=dp_size_per_node ) + deployment_options = llm_config.get_serve_options(name_prefix=name_prefix) + if options_override: + deployment_options.update(options_override) return DPServer.as_deployment(deployment_options).bind( llm_config=llm_config, dp_rank_assigner=dp_rank_assigner diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/base.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/base.py index 420f7f44a06f..2999a147c887 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/base.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/base.py @@ -1,17 +1,29 @@ import abc import random import string -from typing import Any, Dict +from typing import TYPE_CHECKING, Any, Dict + +if TYPE_CHECKING: + from ray.llm._internal.serve.configs.server_models import LLMConfig class BaseConnectorBackend(abc.ABC): - def __init__(self, kv_transfer_config: Dict[str, Any]): + def __init__(self, llm_config: "LLMConfig"): """Base class for connector backends. Args: - kv_transfer_config: Configuration for the KV transfer. + llm_config: The llm configuration for this engine """ - self.kv_transfer_config = kv_transfer_config + self.llm_config = llm_config + + @property + def kv_transfer_config(self) -> Dict[str, Any]: + engine_kwargs = self.llm_config.engine_kwargs + kv_transfer_config = engine_kwargs.get("kv_transfer_config") + assert ( + kv_transfer_config is not None + ), "In Connector backend, kv_transfer_config is not set" + return kv_transfer_config def _get_unique_suffix(self, len: int = 6) -> str: """Generates unique alphanumeric suffix. diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/nixl_connector.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/nixl_connector.py index 5cdbb37f744e..76036a5f3117 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/nixl_connector.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/kv_transfer_backends/nixl_connector.py @@ -6,6 +6,28 @@ class NixlConnectorBackend(BaseConnectorBackend): + def _set_side_channel_port(self): + from vllm import envs as vllm_envs, utils as vllm_utils + + if not vllm_envs.is_set("VLLM_NIXL_SIDE_CHANNEL_PORT"): + base_port: int = int( + self.llm_config.experimental_configs.get( + "NIXL_SIDE_CHANNEL_PORT_BASE", vllm_utils.get_open_port() + ) + ) + # If dp_rank is set, we should use the + # base port + dp_rank as the side channel port + # due to a potential ray condition for getting the free ports. + dp_rank = self.llm_config.engine_kwargs.get("data_parallel_rank", 0) + port = base_port + dp_rank + os.environ["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(port) + + def _set_side_channel_host(self): + from vllm import envs as vllm_envs, utils as vllm_utils + + if not vllm_envs.is_set("VLLM_NIXL_SIDE_CHANNEL_HOST"): + os.environ["VLLM_NIXL_SIDE_CHANNEL_HOST"] = vllm_utils.get_ip() + def setup(self) -> None: """Initialize the NIXL connector backend. @@ -20,7 +42,7 @@ def setup(self) -> None: ValueError: If the current vLLM version doesn't support the required NIXL environment variables. """ - from vllm import envs as vllm_envs, utils as vllm_utils + from vllm import envs as vllm_envs if ( "VLLM_NIXL_SIDE_CHANNEL_PORT" not in vllm_envs.environment_variables @@ -32,11 +54,8 @@ def setup(self) -> None: "that you are using an older version of vLLM." ) - if not vllm_envs.is_set("VLLM_NIXL_SIDE_CHANNEL_PORT"): - port: int = vllm_utils.get_open_port() - os.environ["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(port) - if not vllm_envs.is_set("VLLM_NIXL_SIDE_CHANNEL_HOST"): - os.environ["VLLM_NIXL_SIDE_CHANNEL_HOST"] = vllm_utils.get_ip() + self._set_side_channel_port() + self._set_side_channel_host() # We need to overwrite the engine_id to make it unique across replicas. engine_id = self.kv_transfer_config.get("engine_id", self._get_unique_suffix()) diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py index 081b0b2b6807..36a5444fc564 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py @@ -1,3 +1,4 @@ +import copy import dataclasses import os from typing import Any, Dict, List, Optional @@ -201,7 +202,7 @@ def placement_bundles(self) -> List[Dict[str, float]]: bundle = {"GPU": 1} if self.accelerator_type: bundle[self.ray_accelerator_type()] = 0.001 - bundles = [bundle for _ in range(self.num_devices)] + bundles = [copy.deepcopy(bundle) for _ in range(self.num_devices)] return bundles diff --git a/python/ray/llm/tests/serve/cpu/configs/test_models.py b/python/ray/llm/tests/serve/cpu/configs/test_models.py index 6f5bfab6d1b6..b88ca0524029 100644 --- a/python/ray/llm/tests/serve/cpu/configs/test_models.py +++ b/python/ray/llm/tests/serve/cpu/configs/test_models.py @@ -178,8 +178,7 @@ def test_get_serve_options_with_accelerator_type(self): "max_replicas": 10, } assert serve_options["placement_group_bundles"] == [ - {"CPU": 1, "GPU": 0}, - {"GPU": 1, "accelerator_type:A100-40G": 0.001}, + {"CPU": 1, "GPU": 1, "accelerator_type:A100-40G": 0.001}, ] assert serve_options["placement_group_strategy"] == "STRICT_PACK" assert serve_options["name"] == "Test:test_model" @@ -214,10 +213,7 @@ def test_get_serve_options_without_accelerator_type(self): "initial_replicas": 1, "max_replicas": 10, } - assert serve_options["placement_group_bundles"] == [ - {"CPU": 1, "GPU": 0}, - {"GPU": 1}, - ] + assert serve_options["placement_group_bundles"] == [{"CPU": 1, "GPU": 1}] assert serve_options["placement_group_strategy"] == "STRICT_PACK" assert serve_options["name"] == "Test:test_model" @@ -239,8 +235,9 @@ def test_resources_per_bundle(self): model_loading_config=dict(model_id="test_model"), engine_kwargs=dict(tensor_parallel_size=3, pipeline_parallel_size=2), ).get_serve_options(name_prefix="Test:") - assert serve_options["placement_group_bundles"] == [{"CPU": 1, "GPU": 0}] + [ - {"GPU": 1} for _ in range(6) + + assert serve_options["placement_group_bundles"] == [{"CPU": 1, "GPU": 1}] + [ + {"GPU": 1} for _ in range(5) ] # Test the custom resource bundle @@ -249,9 +246,9 @@ def test_resources_per_bundle(self): engine_kwargs=dict(tensor_parallel_size=3, pipeline_parallel_size=2), resources_per_bundle={"XPU": 1}, ).get_serve_options(name_prefix="Test:") - assert serve_options["placement_group_bundles"] == [{"CPU": 1, "GPU": 0}] + [ - {"XPU": 1} for _ in range(6) - ] + assert serve_options["placement_group_bundles"] == [ + {"CPU": 1, "GPU": 0, "XPU": 1} + ] + [{"XPU": 1} for _ in range(5)] def test_engine_config_cached(self): """Test that the engine config is cached and not recreated when calling From df97d693ae108fabbeefe447afb457c0f2d75686 Mon Sep 17 00:00:00 2001 From: avigyabb <98926738+avigyabb@users.noreply.github.com> Date: Fri, 22 Aug 2025 07:47:20 -0700 Subject: [PATCH 225/634] [Core] Bind grpc servers to specified ip instead of 0.0.0.0 (#55484) Signed-off-by: avigyabb Signed-off-by: avigyabb <98926738+avigyabb@users.noreply.github.com> Co-authored-by: Jiajun Yao --- python/ray/tests/test_autoscaler.py | 2 +- python/ray/tests/test_multi_node_3.py | 2 +- src/ray/core_worker/core_worker_process.cc | 6 ++---- src/ray/core_worker/tests/core_worker_test.cc | 4 ++-- src/ray/gcs/gcs_server/gcs_server.cc | 2 +- .../gcs_server/tests/gcs_health_check_manager_test.cc | 2 +- src/ray/object_manager/object_manager.cc | 2 +- src/ray/raylet/node_manager.cc | 5 ++--- src/ray/raylet/tests/node_manager_test.cc | 2 ++ src/ray/rpc/grpc_server.cc | 3 +-- src/ray/rpc/grpc_server.h | 11 +++++------ src/ray/rpc/tests/grpc_bench/grpc_bench.cc | 2 +- src/ray/rpc/tests/grpc_server_client_test.cc | 2 +- 13 files changed, 21 insertions(+), 24 deletions(-) diff --git a/python/ray/tests/test_autoscaler.py b/python/ray/tests/test_autoscaler.py index a728071528c2..506327bd6980 100644 --- a/python/ray/tests/test_autoscaler.py +++ b/python/ray/tests/test_autoscaler.py @@ -3517,7 +3517,7 @@ def __init__(self, *args, **kwargs): _internal_kv_initialized=Mock(return_value=False), ): monitor = Monitor( - address="localhost:12345", + address=f"{ray.util.get_node_ip_address()}:12345", autoscaling_config="", log_dir=self.tmpdir, ) diff --git a/python/ray/tests/test_multi_node_3.py b/python/ray/tests/test_multi_node_3.py index f741baa7bef0..8c89115e2bda 100644 --- a/python/ray/tests/test_multi_node_3.py +++ b/python/ray/tests/test_multi_node_3.py @@ -241,7 +241,7 @@ def f(): indirect=True, ) def test_using_hostnames(call_ray_start): - ray.init(_node_ip_address="localhost", address="localhost:6379") + ray.init(address=call_ray_start) @ray.remote def f(): diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index c870835c47bd..fa3f1ef415e1 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -247,10 +247,8 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( std::move(raylet_address), *client_call_manager, /*raylet_unavailable_timeout_callback=*/[] {}); - auto core_worker_server = - std::make_unique(WorkerTypeString(options.worker_type), - assigned_port, - options.node_ip_address == "127.0.0.1"); + auto core_worker_server = std::make_unique( + WorkerTypeString(options.worker_type), assigned_port, options.node_ip_address); // Start RPC server after all the task receivers are properly initialized and we have // our assigned port from the raylet. core_worker_server->RegisterService( diff --git a/src/ray/core_worker/tests/core_worker_test.cc b/src/ray/core_worker/tests/core_worker_test.cc index 0fa7170c1838..f3f538e4191e 100644 --- a/src/ray/core_worker/tests/core_worker_test.cc +++ b/src/ray/core_worker/tests/core_worker_test.cc @@ -104,8 +104,8 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { auto service_handler = std::make_unique(); auto worker_context = std::make_unique( WorkerType::WORKER, WorkerID::FromRandom(), JobID::FromInt(1)); - auto core_worker_server = - std::make_unique(WorkerTypeString(options.worker_type), 0, true); + auto core_worker_server = std::make_unique( + WorkerTypeString(options.worker_type), 0, "127.0.0.1"); core_worker_server->RegisterService( std::make_unique(io_service_, *service_handler), false /* token_auth */); diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 95d0a52496e3..f31db35d8696 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -61,7 +61,7 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, storage_type_(GetStorageType()), rpc_server_(config.grpc_server_name, config.grpc_server_port, - config.node_ip_address == "127.0.0.1", + config.node_ip_address, ClusterID::Nil(), config.grpc_server_thread_num, /*keepalive_time_ms=*/RayConfig::instance().grpc_keepalive_time_ms()), diff --git a/src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc index 8ca66a12522f..ed376e4224b0 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc @@ -88,7 +88,7 @@ class GcsHealthCheckManagerTest : public ::testing::Test { auto node_id = NodeID::FromRandom(); auto port = GetFreePort(); RAY_LOG(INFO) << "Get port " << port; - auto server = std::make_shared(node_id.Hex(), port, true); + auto server = std::make_shared(node_id.Hex(), port, "127.0.0.1"); auto channel = grpc::CreateChannel(BuildAddress("localhost", port), grpc::InsecureChannelCredentials()); diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index 53b18b32d11c..6abe66bd2ea6 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -107,7 +107,7 @@ ObjectManager::ObjectManager( rpc_work_(rpc_service_.get_executor()), object_manager_server_("ObjectManager", config_.object_manager_port, - config_.object_manager_address == "127.0.0.1", + config_.object_manager_address, ClusterID::Nil(), config_.rpc_service_threads_number), client_call_manager_(main_service, diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 51c5d204e481..3880f3fe3215 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -160,9 +160,8 @@ NodeManager::NodeManager( RAY_UNUSED(execute_after( io_service_, fn, std::chrono::milliseconds(delay_ms))); }), - node_manager_server_("NodeManager", - config.node_manager_port, - config.node_manager_address == "127.0.0.1"), + node_manager_server_( + "NodeManager", config.node_manager_port, config.node_manager_address), local_object_manager_(local_object_manager), leased_workers_(leased_workers), high_plasma_storage_usage_(RayConfig::instance().high_plasma_storage_usage()), diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index 7041b4152ee0..36b263b87207 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -389,6 +389,8 @@ class NodeManagerTest : public ::testing::Test { })"); NodeManagerConfig node_manager_config{}; + node_manager_config.node_manager_address = "127.0.0.1"; + node_manager_config.node_manager_port = 0; node_manager_config.maximum_startup_concurrency = 1; node_manager_config.store_socket_name = "test_store_socket"; diff --git a/src/ray/rpc/grpc_server.cc b/src/ray/rpc/grpc_server.cc index 047ff734e94b..c644dc85280e 100644 --- a/src/ray/rpc/grpc_server.cc +++ b/src/ray/rpc/grpc_server.cc @@ -62,8 +62,7 @@ void GrpcServer::Shutdown() { void GrpcServer::Run() { uint32_t specified_port = port_; - std::string server_address = - BuildAddress((listen_to_localhost_only_ ? "127.0.0.1" : "0.0.0.0"), port_); + std::string server_address = BuildAddress(ip_address_, port_); grpc::ServerBuilder builder; // Disable the SO_REUSEPORT option. We don't need it in ray. If the option is enabled // (default behavior in grpc), we may see multiple workers listen on the same port and diff --git a/src/ray/rpc/grpc_server.h b/src/ray/rpc/grpc_server.h index 686c4a68b2a4..56566569c359 100644 --- a/src/ray/rpc/grpc_server.h +++ b/src/ray/rpc/grpc_server.h @@ -95,14 +95,14 @@ class GrpcServer { /// GrpcServer(std::string name, const uint32_t port, - bool listen_to_localhost_only, + std::string ip_address, const ClusterID &cluster_id = ClusterID::Nil(), int num_threads = 1, int64_t keepalive_time_ms = 7200000 /*2 hours, grpc default*/) : name_(std::move(name)), port_(port), - listen_to_localhost_only_(listen_to_localhost_only), - cluster_id_(ClusterID::Nil()), + ip_address_(std::move(ip_address)), + cluster_id_(cluster_id), is_shutdown_(true), num_threads_(num_threads), keepalive_time_ms_(keepalive_time_ms) { @@ -161,9 +161,8 @@ class GrpcServer { const std::string name_; /// Port of this server. int port_; - /// Listen to localhost (127.0.0.1) only if it's true, otherwise listen to all network - /// interfaces (0.0.0.0) - const bool listen_to_localhost_only_; + /// IP address of this server. + const std::string ip_address_; /// Token representing ID of this cluster. ClusterID cluster_id_; /// Indicates whether this server is in shutdown state. diff --git a/src/ray/rpc/tests/grpc_bench/grpc_bench.cc b/src/ray/rpc/tests/grpc_bench/grpc_bench.cc index 86b8e7ef4e27..f1268c93d764 100644 --- a/src/ray/rpc/tests/grpc_bench/grpc_bench.cc +++ b/src/ray/rpc/tests/grpc_bench/grpc_bench.cc @@ -71,7 +71,7 @@ int main() { const auto env = std::getenv("GRPC_SERVER_CPUS"); const auto parallelism = env ? std::atoi(env) : std::thread::hardware_concurrency(); - GrpcServer server("grpc_bench", 50051, false, ClusterID::Nil(), parallelism); + GrpcServer server("grpc_bench", 50051, "0.0.0.0", ClusterID::Nil(), parallelism); instrumented_io_context main_service; std::thread t([&main_service] { boost::asio::executor_work_guard work( diff --git a/src/ray/rpc/tests/grpc_server_client_test.cc b/src/ray/rpc/tests/grpc_server_client_test.cc index b020b7b45b8b..c4bbd1e42033 100644 --- a/src/ray/rpc/tests/grpc_server_client_test.cc +++ b/src/ray/rpc/tests/grpc_server_client_test.cc @@ -111,7 +111,7 @@ class TestGrpcServerClientFixture : public ::testing::Test { handler_io_service_work_(handler_io_service_.get_executor()); handler_io_service_.run(); }); - grpc_server_.reset(new GrpcServer("test", 0, true)); + grpc_server_.reset(new GrpcServer("test", 0, "127.0.0.1")); grpc_server_->RegisterService( std::make_unique(handler_io_service_, test_service_handler_), false); From 9af216b6d02becaa68c076a670f5771729c2606c Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 22 Aug 2025 11:44:56 -0500 Subject: [PATCH 226/634] [core] Move Redis health check runner back to GCS server main (#55812) TSAN tests are failing after my refactor PR: https://github.com/ray-project/ray/pull/55655 This is because the `PeriodicalRunner` does not wait for any spawned async tasks to complete in its destructor. In the current setup, after the `PeriodicalRunner` is destroyed, the `RedisStoreClient` is immediately destroyed. However, there may still be tasks on the IO service that didn't get cancelled and will try to access the `RedisStoreClient`, leading to invalid memory accesses. Simplest fix is to move the `PeriodicalRunner` back to `gcs_server.cc` and capture a shared pointer to the client in the callback. I also considered fixing the `PeriodicalRunner` destructor, but that code is hard to navigate right now. --------- Signed-off-by: Edward Oakes --- python/ray/includes/global_state_accessor.pxd | 4 +- src/ray/gcs/gcs_server/gcs_server.cc | 53 +++++++++++-------- src/ray/gcs/gcs_server/gcs_server.h | 3 +- .../gcs/store_client/redis_store_client.cc | 15 ------ src/ray/gcs/store_client/redis_store_client.h | 11 +--- 5 files changed, 35 insertions(+), 51 deletions(-) diff --git a/python/ray/includes/global_state_accessor.pxd b/python/ray/includes/global_state_accessor.pxd index 7d115b469ce9..38a6703cf25f 100644 --- a/python/ray/includes/global_state_accessor.pxd +++ b/python/ray/includes/global_state_accessor.pxd @@ -99,9 +99,7 @@ cdef extern from * namespace "ray::gcs" nogil: RayConfig::instance().initialize(config_list); instrumented_io_context io_service{/*enable_lag_probe=*/false, /*running_on_single_thread=*/true}; - // Set heartbeat_interval_ms to 0 to disable health checking for this temporary client. - RedisClientOptions options{host, port, username, password, use_ssl, /*heartbeat_interval_ms=*/0}; - + RedisClientOptions options{host, port, username, password, use_ssl}; auto client = std::make_unique( std::make_unique(io_service, options)); diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index f31db35d8696..503414c9685a 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -122,14 +122,29 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, // GcsInternalKVManager, to avoid congestion on the latter. RAY_LOG(INFO) << "GCS storage type is " << storage_type_; auto &io_context = io_context_provider_.GetDefaultIOContext(); - std::unique_ptr store_client; + std::shared_ptr store_client; switch (storage_type_) { case StorageType::IN_MEMORY: store_client = - std::make_unique(std::make_unique()); + std::make_shared(std::make_unique()); break; case StorageType::REDIS_PERSIST: { - store_client = CreateRedisStoreClient(io_context); + auto redis_store_client = + std::make_shared(io_context, GetRedisClientOptions()); + // Health check Redis periodically and crash if it becomes unavailable. + // NOTE: periodical_runner_ must run on the same IO context as the Redis client. + periodical_runner_->RunFnPeriodically( + [redis_store_client, &io_context] { + redis_store_client->AsyncCheckHealth( + {[](const Status &status) { + RAY_CHECK_OK(status) << "Redis connection failed unexpectedly."; + }, + io_context}); + }, + RayConfig::instance().gcs_redis_heartbeat_interval_milliseconds(), + "GCSServer.redis_health_check"); + + store_client = redis_store_client; break; } default: @@ -590,24 +605,25 @@ void GcsServer::InitUsageStatsClient() { } void GcsServer::InitKVManager() { - // TODO(yic): Use a factory with configs - std::unique_ptr instance; auto &io_context = io_context_provider_.GetIOContext(); + std::unique_ptr store_client; switch (storage_type_) { case (StorageType::REDIS_PERSIST): - instance = - std::make_unique(CreateRedisStoreClient(io_context)); + store_client = + std::make_unique(io_context, GetRedisClientOptions()); break; case (StorageType::IN_MEMORY): - instance = std::make_unique( - std::make_unique(std::make_unique())); + store_client = + std::make_unique(std::make_unique()); break; default: RAY_LOG(FATAL) << "Unexpected storage type! " << storage_type_; } kv_manager_ = std::make_unique( - std::move(instance), config_.raylet_config_list, io_context); + std::make_unique(std::move(store_client)), + config_.raylet_config_list, + io_context); kv_manager_->GetInstance().Put( "", @@ -873,17 +889,12 @@ std::string GcsServer::GetDebugState() const { return stream.str(); } -std::unique_ptr GcsServer::CreateRedisStoreClient( - instrumented_io_context &io_service) { - return std::make_unique( - io_service, - RedisClientOptions{ - config_.redis_address, - config_.redis_port, - config_.redis_username, - config_.redis_password, - config_.enable_redis_ssl, - RayConfig::instance().gcs_redis_heartbeat_interval_milliseconds()}); +RedisClientOptions GcsServer::GetRedisClientOptions() { + return RedisClientOptions{config_.redis_address, + config_.redis_port, + config_.redis_username, + config_.redis_password, + config_.enable_redis_ssl}; } void GcsServer::PrintAsioStats() { diff --git a/src/ray/gcs/gcs_server/gcs_server.h b/src/ray/gcs/gcs_server/gcs_server.h index 85a02d75f8b2..25b3bdea7d9a 100644 --- a/src/ray/gcs/gcs_server/gcs_server.h +++ b/src/ray/gcs/gcs_server/gcs_server.h @@ -213,8 +213,7 @@ class GcsServer { /// Print the asio event loop stats for debugging. void PrintAsioStats(); - std::unique_ptr CreateRedisStoreClient( - instrumented_io_context &io_service); + RedisClientOptions GetRedisClientOptions(); void TryGlobalGC(); diff --git a/src/ray/gcs/store_client/redis_store_client.cc b/src/ray/gcs/store_client/redis_store_client.cc index 2be5d86ee10f..f564481ea2cf 100644 --- a/src/ray/gcs/store_client/redis_store_client.cc +++ b/src/ray/gcs/store_client/redis_store_client.cc @@ -140,23 +140,8 @@ RedisStoreClient::RedisStoreClient(instrumented_io_context &io_service, RAY_CHECK(!absl::StrContains(external_storage_namespace_, kClusterSeparator)) << "Storage namespace (" << external_storage_namespace_ << ") shouldn't contain " << kClusterSeparator << "."; - - // Health check Redis periodically and crash if it becomes unavailable. - periodic_health_check_runner_ = PeriodicalRunner::Create(io_service_); - periodic_health_check_runner_->RunFnPeriodically( - [this] { - AsyncCheckHealth({[](const Status &status) { - RAY_CHECK_OK(status) - << "Redis connection failed unexpectedly."; - }, - io_service_}); - }, - options.heartbeat_interval_ms, - "RedisStoreClient.redis_health_check"); } -RedisStoreClient::~RedisStoreClient() { periodic_health_check_runner_.reset(); } - Status RedisStoreClient::AsyncPut(const std::string &table_name, const std::string &key, std::string data, diff --git a/src/ray/gcs/store_client/redis_store_client.h b/src/ray/gcs/store_client/redis_store_client.h index ec7b5535b3b5..6456ee606d86 100644 --- a/src/ray/gcs/store_client/redis_store_client.h +++ b/src/ray/gcs/store_client/redis_store_client.h @@ -26,7 +26,6 @@ #include "absl/synchronization/mutex.h" #include "ray/common/asio/asio_util.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/asio/periodical_runner.h" #include "ray/common/asio/postable.h" #include "ray/gcs/store_client/redis_context.h" #include "ray/gcs/store_client/store_client.h" @@ -102,11 +101,6 @@ struct RedisClientOptions { // Whether to use TLS/SSL for the connection. bool enable_ssl = false; - - // The interval between health checks to Redis. - // If a health check fails, the client will crash the process. - // Set to 0 to disable health checking. - uint64_t heartbeat_interval_ms = 1000; }; // StoreClient using Redis as persistence backend. @@ -140,7 +134,6 @@ class RedisStoreClient : public StoreClient { /// \param options The options for connecting to Redis. explicit RedisStoreClient(instrumented_io_context &io_service, const RedisClientOptions &options); - ~RedisStoreClient(); Status AsyncPut(const std::string &table_name, const std::string &key, @@ -179,12 +172,12 @@ class RedisStoreClient : public StoreClient { const std::string &key, Postable callback) override; - private: // Check if Redis is available. // // \param callback The callback that will be called with a Status. OK means healthy. void AsyncCheckHealth(Postable callback); + private: /// \class RedisScanner /// /// This class is used to HSCAN data from a Redis table. @@ -305,8 +298,6 @@ class RedisStoreClient : public StoreClient { // The following context writes everything to the primary shard. std::shared_ptr primary_context_; - std::shared_ptr periodic_health_check_runner_; - absl::Mutex mu_; // The pending redis requests queue for each key. From d8b3b743bcdcaa8776b496a84f3b3760425f4da8 Mon Sep 17 00:00:00 2001 From: "Owen Lin (You-Cheng Lin)" <106612301+owenowenisme@users.noreply.github.com> Date: Sat, 23 Aug 2025 01:32:21 +0800 Subject: [PATCH 227/634] [Core][Raylet] Remove redundant std::move (#55839) Signed-off-by: You-Cheng Lin (Owen) --- src/ray/raylet/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 3af97a4c55d5..4ce139301327 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -776,7 +776,7 @@ int main(int argc, char *argv[]) { RAY_CHECK(node_info) << "No GCS info for node " << id; auto addr = ray::rpc::RayletClientPool::GenerateRayletAddress( id, node_info->node_manager_address(), node_info->node_manager_port()); - return raylet_client_pool->GetOrConnectByAddress(std::move(addr)); + return raylet_client_pool->GetOrConnectByAddress(addr); }; plasma_client = std::make_unique(); From 781c73fe44531cbb948c9ce349fff76961b6ec04 Mon Sep 17 00:00:00 2001 From: Potato Date: Sat, 23 Aug 2025 01:52:50 +0800 Subject: [PATCH 228/634] [DOC][Train] Fix typo for Instantiating in ray train doc (#55826) Signed-off-by: Potato --- doc/source/train/user-guides/data-loading-preprocessing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/train/user-guides/data-loading-preprocessing.rst b/doc/source/train/user-guides/data-loading-preprocessing.rst index 5db4669578ad..9e95dd1445e9 100644 --- a/doc/source/train/user-guides/data-loading-preprocessing.rst +++ b/doc/source/train/user-guides/data-loading-preprocessing.rst @@ -322,7 +322,7 @@ For more details, see the following sections for each framework: .. tip:: When using Torch or Hugging Face Datasets directly without Ray Data, make sure to instantiate your Dataset *inside* the ``train_loop_per_worker``. - Instatiating the Dataset outside of the ``train_loop_per_worker`` and passing it in via global scope + Instantiating the Dataset outside of the ``train_loop_per_worker`` and passing it in via global scope can cause errors due to the Dataset not being serializable. .. note:: From b0b77f6b0a9be7394cf5e304fedce63024f7707f Mon Sep 17 00:00:00 2001 From: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Date: Fri, 22 Aug 2025 12:33:45 -0700 Subject: [PATCH 229/634] [Train] move collective implementations to train_fn_utils (#55689) This PR moves the implementations of collectives to `TrainFnUtils`. This would unblock the local mode that is introduced in https://github.com/ray-project/ray/pull/55487 --------- Signed-off-by: xgui Signed-off-by: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python/ray/train/collective/collectives.py | 44 +-------------- .../v2/_internal/execution/collective_impl.py | 56 +++++++++++++++++++ .../v2/_internal/execution/train_fn_utils.py | 19 +++++++ python/ray/train/v2/tests/test_collective.py | 10 ++-- python/ray/train/v2/tests/test_persistence.py | 17 +----- 5 files changed, 85 insertions(+), 61 deletions(-) create mode 100644 python/ray/train/v2/_internal/execution/collective_impl.py diff --git a/python/ray/train/collective/collectives.py b/python/ray/train/collective/collectives.py index 3b06fc369e32..8c3dc0e43916 100644 --- a/python/ray/train/collective/collectives.py +++ b/python/ray/train/collective/collectives.py @@ -1,18 +1,9 @@ import logging from typing import Optional, TypeVar -import ray -import ray.cloudpickle as pickle -from ray.train.v2._internal.execution.context import ( - get_train_context as get_internal_train_context, -) from ray.train.v2._internal.execution.train_fn_utils import get_train_fn_utils from ray.util.annotations import PublicAPI -# For reference, {1:1} is 19 bytes, {"1":"1"} is 21 bytes, -# and {"12345": "12345"} is 25 bytes. -_MAX_BROADCAST_SIZE_BYTES = 1000 - T = TypeVar("T", bound=Optional[object]) @@ -59,29 +50,7 @@ def train_func(): pickle.PicklingError: If the data is not pickleable. TypeError: If the data is not pickleable. """ - # Validate data. - if data is not None: - data_bytes = len(pickle.dumps(data)) - if data_bytes > _MAX_BROADCAST_SIZE_BYTES: - logger.warning( - f"Data size {data_bytes} bytes exceeds the maximum broadcast " - f"size of {_MAX_BROADCAST_SIZE_BYTES} bytes" - ) - - # Send data to all workers. - # TODO (xgui): We should not expose get_synchronization_actor() from internal_context here. - # Maybe create one public barrier API inside `TrainFnUtils` - sync_actor = get_internal_train_context().get_synchronization_actor() - train_context = get_train_fn_utils().get_context() - - return ray.get( - sync_actor.broadcast_from_rank_zero.remote( - world_rank=train_context.get_world_rank(), - world_size=train_context.get_world_size(), - data=data, - caller_method_name="ray.train.collective.broadcast_from_rank_zero", - ) - ) + return get_train_fn_utils().broadcast_from_rank_zero(data) @PublicAPI(stability="alpha") @@ -109,13 +78,4 @@ def train_func(): trainer = TorchTrainer(train_func) trainer.fit() """ - train_context = get_train_fn_utils().get_context() - sync_actor = get_internal_train_context().get_synchronization_actor() - return ray.get( - sync_actor.broadcast_from_rank_zero.remote( - world_rank=train_context.get_world_rank(), - world_size=train_context.get_world_size(), - data=None, - caller_method_name="ray.train.collective.barrier", - ) - ) + return get_train_fn_utils().barrier() diff --git a/python/ray/train/v2/_internal/execution/collective_impl.py b/python/ray/train/v2/_internal/execution/collective_impl.py new file mode 100644 index 000000000000..0d91567046aa --- /dev/null +++ b/python/ray/train/v2/_internal/execution/collective_impl.py @@ -0,0 +1,56 @@ +import logging +from typing import Any + +import ray +import ray.cloudpickle as pickle +from ray.train.v2._internal.execution.context import get_train_context + +# For reference, {1:1} is 19 bytes, {"1":"1"} is 21 bytes, +# and {"12345": "12345"} is 25 bytes. +_MAX_BROADCAST_SIZE_BYTES = 1000 + + +logger = logging.getLogger(__name__) + + +def barrier() -> None: + """ + Create a barrier across all training workers. + """ + train_context = get_train_context() + sync_actor = train_context.get_synchronization_actor() + return ray.get( + sync_actor.broadcast_from_rank_zero.remote( + world_rank=train_context.get_world_rank(), + world_size=train_context.get_world_size(), + data=None, + caller_method_name="ray.train.collective.barrier", + ) + ) + + +def broadcast_from_rank_zero(data: Any) -> Any: + """Broadcast data from the rank 0 worker to all other workers. + + This method is used by the public API function :func:`ray.train.collective.broadcast_from_rank_zero`. + Users should typically call ``ray.train.collective.broadcast_from_rank_zero()`` instead of calling this method directly. + """ + # Validate data. + if data is not None: + data_bytes = len(pickle.dumps(data)) + if data_bytes > _MAX_BROADCAST_SIZE_BYTES: + logger.warning( + f"Data size {data_bytes} bytes exceeds the maximum broadcast " + f"size of {_MAX_BROADCAST_SIZE_BYTES} bytes" + ) + + train_context = get_train_context() + sync_actor = train_context.get_synchronization_actor() + return ray.get( + sync_actor.broadcast_from_rank_zero.remote( + world_rank=train_context.get_world_rank(), + world_size=train_context.get_world_size(), + data=data, + caller_method_name="ray.train.collective.broadcast_from_rank_zero", + ) + ) diff --git a/python/ray/train/v2/_internal/execution/train_fn_utils.py b/python/ray/train/v2/_internal/execution/train_fn_utils.py index 28bf683fda2d..c960b32b5f45 100644 --- a/python/ray/train/v2/_internal/execution/train_fn_utils.py +++ b/python/ray/train/v2/_internal/execution/train_fn_utils.py @@ -3,6 +3,7 @@ from ray.data import DataIterator from ray.train import Checkpoint +from ray.train.v2._internal.execution import collective_impl from ray.train.v2._internal.execution.context import ( get_train_context as get_internal_train_context, ) @@ -62,6 +63,24 @@ def get_dataset_shard(self, dataset_name: str) -> DataIterator: def get_context(self) -> ExternalTrainContext: return ExternalTrainContext() + def barrier(self) -> None: + """Create a barrier across all workers. + + All workers must call this method before the training function can continue. + + This method is used by the public API function :func:`ray.train.collective.barrier`. + Users should typically call ``ray.train.collective.barrier()`` instead of calling this method directly. + """ + return collective_impl.barrier() + + def broadcast_from_rank_zero(self, data: Any) -> Any: + """Broadcast data from the rank 0 worker to all other workers. + + This method is used by the public API function :func:`ray.train.collective.broadcast_from_rank_zero`. + Users should typically call ``ray.train.collective.broadcast_from_rank_zero()`` instead of calling this method directly. + """ + return collective_impl.broadcast_from_rank_zero(data) + _train_fn_utils: Optional[TrainFnUtils] = None _train_fn_utils_lock = threading.Lock() diff --git a/python/ray/train/v2/tests/test_collective.py b/python/ray/train/v2/tests/test_collective.py index d8196ea420a5..046eedf2f979 100644 --- a/python/ray/train/v2/tests/test_collective.py +++ b/python/ray/train/v2/tests/test_collective.py @@ -4,7 +4,7 @@ import ray import ray.train.collective -from ray.train.collective import collectives +from ray.train.v2._internal.execution import collective_impl from ray.train.v2.api.data_parallel_trainer import DataParallelTrainer @@ -49,12 +49,14 @@ def train_fn(): def test_broadcast_from_rank_zero_data_too_big(ray_start_4_cpus): def train_fn(): - collectives.logger = mock.create_autospec(collectives.logger, instance=True) - collectives._MAX_BROADCAST_SIZE_BYTES = 0 + collective_impl.logger = mock.create_autospec( + collective_impl.logger, instance=True + ) + collective_impl._MAX_BROADCAST_SIZE_BYTES = 0 rank = ray.train.get_context().get_world_rank() value = ray.train.collective.broadcast_from_rank_zero({"key": rank}) assert value == {"key": 0} - collectives.logger.warning.assert_called_once() + collective_impl.logger.warning.assert_called_once() trainer = DataParallelTrainer( train_fn, diff --git a/python/ray/train/v2/tests/test_persistence.py b/python/ray/train/v2/tests/test_persistence.py index 84468f1ea57a..b1891bbc6e89 100644 --- a/python/ray/train/v2/tests/test_persistence.py +++ b/python/ray/train/v2/tests/test_persistence.py @@ -14,6 +14,7 @@ import ray import ray.train +import ray.train.collective from ray._common.test_utils import simulate_s3_bucket from ray.air._internal.uri_utils import URI from ray.train import ( @@ -24,11 +25,7 @@ ScalingConfig, ) from ray.train.v2._internal.constants import HEALTH_CHECK_INTERVAL_S_ENV_VAR -from ray.train.v2._internal.execution.context import ( - get_train_context as get_internal_train_context, -) from ray.train.v2._internal.execution.storage import _download_from_fs_path -from ray.train.v2._internal.execution.train_fn_utils import get_train_fn_utils from ray.train.v2.api.data_parallel_trainer import DataParallelTrainer @@ -215,17 +212,7 @@ def train_fn(config): # which will cause the test assertions to fail. # This should be fixed by forcing a queue flush on all workers before # executing the failure decisions. - sync_actor = get_internal_train_context().get_synchronization_actor() - train_context = get_train_fn_utils().get_context() - - ray.get( - sync_actor.broadcast_from_rank_zero.remote( - world_rank=train_context.get_world_rank(), - world_size=train_context.get_world_size(), - data="barrier", - caller_method_name="caller_method_name", - ) - ) + ray.train.collective.barrier() if i in config.get("fail_iters", []): raise RuntimeError(f"Failing on iter={i}!!") From 6ea4489f2adfd58be4485a7c5580c66a7409d2fb Mon Sep 17 00:00:00 2001 From: Goku Mohandas Date: Fri, 22 Aug 2025 13:46:49 -0700 Subject: [PATCH 230/634] added uv support (#55821) --- .../.anyscaleignore | 18 + .../e2e-multimodal-ai-workloads/.gitignore | 103 + .../e2e-multimodal-ai-workloads/README.ipynb | 32 +- .../e2e-multimodal-ai-workloads/README.md | 31 +- .../e2e-multimodal-ai-workloads/ci/build.sh | 2 +- .../configs/generate_embeddings.yaml | 71 + .../configs/service.yaml | 69 + .../configs/train_model.yaml | 71 + .../e2e-multimodal-ai-workloads/containerfile | 4 +- .../doggos/{ => doggos}/__init__.py | 0 .../doggos/{ => doggos}/data.py | 9 +- .../doggos/{ => doggos}/embed.py | 9 +- .../doggos/{ => doggos}/infer.py | 0 .../doggos/{ => doggos}/model.py | 0 .../doggos/{ => doggos}/serve.py | 41 +- .../doggos/{ => doggos}/train.py | 2 +- .../doggos/{ => doggos}/utils.py | 0 .../doggos/pyproject.toml | 13 + .../notebooks/01-Batch-Inference.ipynb | 405 +- .../notebooks/02-Distributed-Training.ipynb | 3543 ++++++++++++++--- .../notebooks/03-Online-Serving.ipynb | 206 +- .../pyproject.toml | 22 + .../requirements.txt | 6 +- .../e2e-multimodal-ai-workloads/uv.lock | 3506 ++++++++++++++++ 24 files changed, 7383 insertions(+), 780 deletions(-) create mode 100644 doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/.anyscaleignore create mode 100644 doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/.gitignore create mode 100644 doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/generate_embeddings.yaml create mode 100644 doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/service.yaml create mode 100644 doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/train_model.yaml rename doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/{ => doggos}/__init__.py (100%) rename doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/{ => doggos}/data.py (83%) rename doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/{ => doggos}/embed.py (94%) rename doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/{ => doggos}/infer.py (100%) rename doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/{ => doggos}/model.py (100%) rename doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/{ => doggos}/serve.py (83%) rename doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/{ => doggos}/train.py (99%) rename doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/{ => doggos}/utils.py (100%) create mode 100644 doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/pyproject.toml create mode 100644 doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/pyproject.toml create mode 100644 doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/uv.lock diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/.anyscaleignore b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/.anyscaleignore new file mode 100644 index 000000000000..30c59efc983b --- /dev/null +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/.anyscaleignore @@ -0,0 +1,18 @@ +# This file is used to exclude files from Anyscale Workspaces snapshots. +# Use this to prevent large or unnecessary files from being included in your snapshots, +# which helps reduce snapshot size and creation time. See documentation for more details: +# https://docs.anyscale.com/platform/workspaces/workspaces-files/#excluding-files-with-anyscaleignore +# +# Syntax examples: +# *.txt # Ignore files with a .txt extension at the same level as `.anyscaleignore`. +# **/*.txt # Ignore files with a .txt extension in ANY directory. +# folder/ # Ignore all files under "folder/". The slash at the end is optional. +# folder/*.txt # Ignore files with a .txt extension under "folder/". +# path/to/filename.py # Ignore a specific file by providing its relative path. +# file_[1,2].txt # Ignore file_1.txt and file_2.txt. + +# Exclude Python virtual environments (.venv/) from snapshots. Virtual environments contain +# all installed Python dependencies, which can be multiple gigabytes in size. These directories +# are typically recreatable from requirements files and don't need to be included in snapshots. +# The ** pattern ensures all .venv directories are excluded regardless of location in your project. +**/.venv/ diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/.gitignore b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/.gitignore new file mode 100644 index 000000000000..a9fffb1a10c6 --- /dev/null +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/.gitignore @@ -0,0 +1,103 @@ +# VSCode +.vscode/ +.idea + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Flask: +instance/ +.webassets-cache + +# Scrapy: +.scrapy + +# Sphinx +docs/_build/ + +# PyBuilder +target/ + +# IPython +.ipynb_checkpoints +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# PEP 582 +__pypackages__/ + +# Celery +celerybeat-schedule +celerybeat.pid + +# Environment +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mkdocs +site/ + +# Airflow +airflow/airflow.db + +# MacOS +.DS_Store + +# Clean up +.trash/ \ No newline at end of file diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/README.ipynb b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/README.ipynb index 2559cc045ccd..dbf5eeb84d69 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/README.ipynb +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/README.ipynb @@ -18,7 +18,7 @@ "\n", "\n", "💻 Run this entire tutorial on [Anyscale](https://www.anyscale.com/) for free:\n", - "**https://console.anyscale.com/template-preview/image-search-and-classification**\n", + "**https://console.anyscale.com/template-preview/image-search-and-classification** or access the repository [here](https://github.com/ray-project/ray/tree/master/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads).\n", "\n", "This tutorial focuses on the fundamental challenges of multimodal AI workloads at scale:\n", "\n", @@ -42,6 +42,7 @@ "- **Development tools**: Spin up a remote session from your local IDE (Cursor, VS Code, etc.) and start coding, using the same tools you love but with the power of Anyscale's compute.\n", "- **Dependencies**: Install dependencies using familiar tools like pip or uv. Anyscale propagates all dependencies to the cluster's worker nodes.\n", "- **Compute**: Leverage any reserved instance capacity, spot instance from any compute provider of your choice by deploying Anyscale into your account. Alternatively, you can use the Anyscale cloud for a full serverless experience.\n", + " - Under the hood, a cluster spins up and is efficiently managed by Anyscale.\n", "- **Debugging**: Leverage a [distributed debugger](https://docs.anyscale.com/platform/workspaces/workspaces-debugging/#distributed-debugger) to get the same VS Code-like debugging experience.\n", "\n", "Learn more about Anyscale Workspaces in the [official documentation](https://docs.anyscale.com/platform/workspaces/).\n", @@ -50,11 +51,40 @@ " \n", "\n", "\n", + "### Additional dependencies\n", + "\n", + "You can choose to manage the additional dependencies through `uv` or `pip`. \n", + "\n", + "#### uv\n", + "\n", + "```bash\n", + "# UV setup instructions\n", + "uv init . # this creates pyproject.toml, uv lockfile, etc.\n", + "ray_wheel_url=http://localhost:9478/ray/$(pip freeze | grep -oP '^ray @ file:///home/ray/\\.whl/\\K.*')\n", + "uv add \"$ray_wheel_url[data, train, tune, serve]\" # to use anyscale's performant ray runtime\n", + "uv add $(grep -v '^\\s*#' requirements.txt)\n", + "uv add --editable ./doggos\n", + "```\n", + "\n", + "#### Pip\n", + "\n", + "```bash\n", + "# Pip setup instructions\n", + "pip install -q -r /home/ray/default/requirements.txt\n", + "pip install -e ./doggos\n", + "```\n", + "\n", "**Note**: Run the entire tutorial for free on [Anyscale](https://console.anyscale.com/)—all dependencies come pre-installed, and compute autoscales automatically. To run it elsewhere, install the dependencies from the [`containerfile`](https://github.com/anyscale/multimodal-ai/tree/main/containerfile) and provision the appropriate GPU resources.\n", "\n", "## Production\n", "Seamlessly integrate with your existing CI/CD pipelines by leveraging the Anyscale [CLI](https://docs.anyscale.com/reference/quickstart-cli) or [SDK](https://docs.anyscale.com/reference/quickstart-sdk) to deploy [highly available services](https://docs.anyscale.com/platform/services) and run [reliable batch jobs](https://docs.anyscale.com/platform/jobs). Developing in an environment nearly identical to production—a multi-node cluster—drastically accelerates the dev-to-prod transition. This tutorial also introduces proprietary RayTurbo features that optimize workloads for performance, fault tolerance, scale, and observability.\n", "\n", + "```bash\n", + "anyscale job submit -f /home/ray/default/configs/generate_embeddings.yaml\n", + "anyscale job submit -f /home/ray/default/configs/train_model.yaml\n", + "anyscale service deploy -f /home/ray/default/configs/service.yaml\n", + "```\n", + "\n", "## No infrastructure headaches\n", "Abstract away infrastructure from your ML/AI developers so they can focus on their core ML development. You can additionally better manage compute resources and costs with [enterprise governance and observability](https://www.anyscale.com/blog/enterprise-governance-observability) and [admin capabilities](https://docs.anyscale.com/administration/overview) so you can set [resource quotas](https://docs.anyscale.com/reference/resource-quotas/), set [priorities for different workloads](https://docs.anyscale.com/administration/cloud-deployment/global-resource-scheduler) and gain [observability of your utilization across your entire compute fleet](https://docs.anyscale.com/administration/resource-management/telescope-dashboard).\n", "Users running on a Kubernetes cloud (EKS, GKE, etc.) can still access the proprietary RayTurbo optimizations demonstrated in this tutorial by deploying the [Anyscale Kubernetes Operator](https://docs.anyscale.com/administration/cloud-deployment/kubernetes/)." diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/README.md b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/README.md index 60b2e693cafa..ad77132b8cf7 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/README.md +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/README.md @@ -14,7 +14,7 @@ notebooks/03-Online-Serving 💻 Run this entire tutorial on [Anyscale](https://www.anyscale.com/) for free: -**https://console.anyscale.com/template-preview/image-search-and-classification** +**https://console.anyscale.com/template-preview/image-search-and-classification** or access the repository [here](https://github.com/ray-project/ray/tree/master/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads). This tutorial focuses on the fundamental challenges of multimodal AI workloads at scale: @@ -47,11 +47,40 @@ Learn more about Anyscale Workspaces in the [official documentation](https://doc +### Additional dependencies + +You can choose to manage the additional dependencies through `uv` or `pip`. + +#### uv + +```bash +# UV setup instructions +uv init . # this creates pyproject.toml, uv lockfile, etc. +ray_wheel_url=http://localhost:9478/ray/$(pip freeze | grep -oP '^ray @ file:///home/ray/\.whl/\K.*') +uv add "$ray_wheel_url[data, train, tune, serve]" # to use anyscale's performant ray runtime +uv add $(grep -v '^\s*#' requirements.txt) +uv add --editable ./doggos +``` + +#### Pip + +```bash +# Pip setup instructions +pip install -q -r /home/ray/default/requirements.txt +pip install -e ./doggos +``` + **Note**: Run the entire tutorial for free on [Anyscale](https://console.anyscale.com/)—all dependencies come pre-installed, and compute autoscales automatically. To run it elsewhere, install the dependencies from the [`containerfile`](https://github.com/anyscale/multimodal-ai/tree/main/containerfile) and provision the appropriate GPU resources. ## Production Seamlessly integrate with your existing CI/CD pipelines by leveraging the Anyscale [CLI](https://docs.anyscale.com/reference/quickstart-cli) or [SDK](https://docs.anyscale.com/reference/quickstart-sdk) to deploy [highly available services](https://docs.anyscale.com/platform/services) and run [reliable batch jobs](https://docs.anyscale.com/platform/jobs). Developing in an environment nearly identical to production—a multi-node cluster—drastically accelerates the dev-to-prod transition. This tutorial also introduces proprietary RayTurbo features that optimize workloads for performance, fault tolerance, scale, and observability. +```bash +anyscale job submit -f /home/ray/default/configs/generate_embeddings.yaml +anyscale job submit -f /home/ray/default/configs/train_model.yaml +anyscale service deploy -f /home/ray/default/configs/service.yaml +``` + ## No infrastructure headaches Abstract away infrastructure from your ML/AI developers so they can focus on their core ML development. You can additionally better manage compute resources and costs with [enterprise governance and observability](https://www.anyscale.com/blog/enterprise-governance-observability) and [admin capabilities](https://docs.anyscale.com/administration/overview) so you can set [resource quotas](https://docs.anyscale.com/reference/resource-quotas/), set [priorities for different workloads](https://docs.anyscale.com/administration/cloud-deployment/global-resource-scheduler) and gain [observability of your utilization across your entire compute fleet](https://docs.anyscale.com/administration/resource-management/telescope-dashboard). diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/ci/build.sh b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/ci/build.sh index bda0ef917f47..05ff13248bd7 100755 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/ci/build.sh +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/ci/build.sh @@ -5,7 +5,7 @@ set -exo pipefail # Install Python dependencies pip3 install --no-cache-dir \ "matplotlib==3.10.0" \ - "torch==2.7.0" \ + "torch==2.7.1" \ "transformers==4.52.3" \ "scikit-learn==1.6.0" \ "mlflow==2.19.0" \ diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/generate_embeddings.yaml b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/generate_embeddings.yaml new file mode 100644 index 000000000000..fcb9020fe206 --- /dev/null +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/generate_embeddings.yaml @@ -0,0 +1,71 @@ +# View the docs https://docs.anyscale.com/reference/job-api#jobconfig. + +name: image-batch-embeddings + +# When empty, use the default image. This can be an Anyscale-provided base image +# like anyscale/ray:2.43.0-slim-py312-cu125, a user-provided base image (provided +# that it meets certain specs), or you can build new images using the Anyscale +# image builder at https://console.anyscale-staging.com/v2/container-images. +image_uri: anyscale/ray:2.48.0-slim-py312-cu128 +# containerfile: /home/ray/default/containerfile + +# When empty, Anyscale will auto-select the instance types. You can also specify +# minimum and maximum resources. +compute_config: +# head_node: +# instance_type: m5.2xlarge +# worker_nodes: +# - instance_type: m5.16xlarge +# min_nodes: 0 +# max_nodes: 100 +# - instance_type: m7a.24xlarge +# min_nodes: 0 +# max_nodes: 100 +# market_type: PREFER_SPOT # Defaults to ON_DEMAND +# - instance_type: g4dn.2xlarge +# min_nodes: 0 +# max_nodes: 100 +# market_type: PREFER_SPOT # Defaults to ON_DEMAND +# min_resources: +# CPU: 100 +# GPU: 1 +# max_resources: +# CPU: 5000 +# GPU: 100 + +# Path to a local directory or a remote URI to a .zip file (S3, GS, HTTP) that +# will be the working directory for the job. The files in the directory will be +# automatically uploaded to the job environment in Anyscale. +working_dir: /home/ray/default +excludes: # (Optional) List of files to exclude from being packaged up for the job. + - .git + - .env + - .venv + - '**/*.egg-info/**' + - '**/.DS_Store/**' + - '**/__pycache__/**' + +requirements: # (Optional) List of requirements files to install. Can also be a path to a requirements.txt. + - ipywidgets==8.1.3 + - matplotlib==3.10.0 + - mlflow==2.19.0 + - torch==2.7.1 + - transformers==4.52.3 + - scikit-learn==1.6.0 +env_vars: # (Optional) Dictionary of environment variables to set in the job. + # MY_ENV_VAR: my_value + # ANOTHER_ENV_VAR: another_value +py_modules: # (Optional) A list of local directories or remote URIs that will be added to the Python path. + - /home/ray/default/doggos + # - /path/to/my_module + # - s3://my_bucket/my_module + +# When empty, this uses the default Anyscale Cloud in your organization. +cloud: + +# The script to run in your job. You can also do "uv run main.py" if you have a +# pyproject.toml file in your working_dir. +entrypoint: python doggos/doggos/embed.py # uv run doggos/doggos/embed.py # remove the requirements and py_modules + +# If there is an error, do not retry. +max_retries: 0 \ No newline at end of file diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/service.yaml b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/service.yaml new file mode 100644 index 000000000000..c3b1d2abd4cb --- /dev/null +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/service.yaml @@ -0,0 +1,69 @@ +# View the docs https://docs.anyscale.com/reference/service-api#serviceconfig. + +name: doggos-app + +# When empty, use the default image. This can be an Anyscale-provided base image +# like anyscale/ray:2.43.0-slim-py312-cu125, a user-provided base image (provided +# that it meets certain specs), or you can build new images using the Anyscale +# image builder at https://console.anyscale-staging.com/v2/container-images. +image_uri: anyscale/ray:2.48.0-slim-py312-cu128 +# containerfile: /home/ray/default/containerfile + +# When empty, Anyscale will auto-select the instance types. You can also specify +# minimum and maximum resources. +compute_config: +# head_node: +# instance_type: m5.2xlarge +# worker_nodes: +# - instance_type: m5.16xlarge +# min_nodes: 0 +# max_nodes: 100 +# - instance_type: m7a.24xlarge +# min_nodes: 0 +# max_nodes: 100 +# market_type: PREFER_SPOT # Defaults to ON_DEMAND +# - instance_type: g4dn.2xlarge +# min_nodes: 0 +# max_nodes: 100 +# market_type: PREFER_SPOT # Defaults to ON_DEMAND +# min_resources: +# CPU: 100 +# GPU: 1 +# max_resources: +# CPU: 5000 +# GPU: 100 + +# Path to a local directory or a remote URI to a .zip file (S3, GS, HTTP) that +# will be the working directory for the job. The files in the directory will be +# automatically uploaded to the job environment in Anyscale. +working_dir: /home/ray/default +excludes: # (Optional) List of files to exclude from being packaged up for the job. + - .git + - .env + - .venv + - '**/*.egg-info/**' + - '**/.DS_Store/**' + - '**/__pycache__/**' + +requirements: # (Optional) List of requirements files to install. Can also be a path to a requirements.txt. + - ipywidgets==8.1.3 + - matplotlib==3.10.0 + - mlflow==2.19.0 + - torch==2.7.1 + - transformers==4.52.3 + - scikit-learn==1.6.0 +env_vars: # (Optional) Dictionary of environment variables to set in the job. + # MY_ENV_VAR: my_value + # ANOTHER_ENV_VAR: another_value +py_modules: # (Optional) A list of local directories or remote URIs that will be added to the Python path. + - /home/ray/default/doggos + # - /path/to/my_module + # - s3://my_bucket/my_module +py_executable: python # uv run # remove the requirements and py_modules + +# When empty, this uses the default Anyscale Cloud in your organization. +cloud: + +# Speciy the Ray Serve app to deploy. +applications: +- import_path: doggos.serve:app \ No newline at end of file diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/train_model.yaml b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/train_model.yaml new file mode 100644 index 000000000000..28305413a642 --- /dev/null +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/configs/train_model.yaml @@ -0,0 +1,71 @@ +# View the docs https://docs.anyscale.com/reference/job-api#jobconfig. + +name: train-image-model + +# When empty, use the default image. This can be an Anyscale-provided base image +# like anyscale/ray:2.43.0-slim-py312-cu125, a user-provided base image (provided +# that it meets certain specs), or you can build new images using the Anyscale +# image builder at https://console.anyscale-staging.com/v2/container-images. +image_uri: anyscale/ray:2.48.0-slim-py312-cu128 +# containerfile: /home/ray/default/containerfile + +# When empty, Anyscale will auto-select the instance types. You can also specify +# minimum and maximum resources. +compute_config: +# head_node: +# instance_type: m5.2xlarge +# worker_nodes: +# - instance_type: m5.16xlarge +# min_nodes: 0 +# max_nodes: 100 +# - instance_type: m7a.24xlarge +# min_nodes: 0 +# max_nodes: 100 +# market_type: PREFER_SPOT # Defaults to ON_DEMAND +# - instance_type: g4dn.2xlarge +# min_nodes: 0 +# max_nodes: 100 +# market_type: PREFER_SPOT # Defaults to ON_DEMAND +# min_resources: +# CPU: 100 +# GPU: 1 +# max_resources: +# CPU: 5000 +# GPU: 100 + +# Path to a local directory or a remote URI to a .zip file (S3, GS, HTTP) that +# will be the working directory for the job. The files in the directory will be +# automatically uploaded to the job environment in Anyscale. +working_dir: /home/ray/default +excludes: # (Optional) List of files to exclude from being packaged up for the job. + - .git + - .env + - .venv + - '**/*.egg-info/**' + - '**/.DS_Store/**' + - '**/__pycache__/**' + +requirements: # (Optional) List of requirements files to install. Can also be a path to a requirements.txt. + - ipywidgets==8.1.3 + - matplotlib==3.10.0 + - mlflow==2.19.0 + - torch==2.7.1 + - transformers==4.52.3 + - scikit-learn==1.6.0 +env_vars: # (Optional) Dictionary of environment variables to set in the job. + # MY_ENV_VAR: my_value + # ANOTHER_ENV_VAR: another_value +py_modules: # (Optional) A list of local directories or remote URIs that will be added to the Python path. + - /home/ray/default/doggos + # - /path/to/my_module + # - s3://my_bucket/my_module + +# When empty, this uses the default Anyscale Cloud in your organization. +cloud: + +# The script to run in your job. You can also do "uv run main.py" if you have a +# pyproject.toml file in your working_dir. +entrypoint: python doggos/doggos/train.py # uv run doggos/doggos/train.py # remove the requirements and py_modules + +# If there is an error, do not retry. +max_retries: 0 \ No newline at end of file diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/containerfile b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/containerfile index 43b34335edda..4a370bacac53 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/containerfile +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/containerfile @@ -1,6 +1,6 @@ # Start with an Anyscale base image. # Use the drop-down above to browse through all available images. -FROM anyscale/ray:2.47.0-slim-py312-cu128 +FROM anyscale/ray:2.48.0-slim-py312-cu128 # Add your pip dependencies here. Disable cache for a smaller image to optimize build and cluster startup time. # RUN pip install --no-cache-dir --upgrade @@ -13,5 +13,5 @@ FROM anyscale/ray:2.47.0-slim-py312-cu128 # Add other build commands here. # RUN echo "Testing Ray import..." && python -c "import ray" RUN python3 -m pip install --no-cache-dir \ - "matplotlib==3.10.0" "torch==2.7.0" "transformers==4.52.3" \ + "matplotlib==3.10.0" "torch==2.7.1" "transformers==4.52.3" \ "scikit-learn==1.6.0" "mlflow==2.19.0" "ipywidgets==8.1.3" diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/__init__.py b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/__init__.py similarity index 100% rename from doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/__init__.py rename to doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/__init__.py diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/data.py b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/data.py similarity index 83% rename from doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/data.py rename to doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/data.py index 81fd2cbe3b3a..fd672fccb9e8 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/data.py +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/data.py @@ -29,12 +29,15 @@ def transform(self, ds): ) ds = ds.map_batches( EmbedImages, - fn_constructor_kwargs={"model_id": "openai/clip-vit-base-patch32"}, - fn_kwargs={"device": "cuda"}, + fn_constructor_kwargs={ + "model_id": "openai/clip-vit-base-patch32", + "device": "cuda", + }, # class kwargs + fn_kwargs={}, concurrency=4, batch_size=64, num_gpus=1, - accelerator_type="L4", + accelerator_type="T4", ) ds = ds.drop_columns(["image"]) return ds diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/embed.py b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/embed.py similarity index 94% rename from doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/embed.py rename to doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/embed.py index 6979139177c2..88a3680dbb4d 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/embed.py +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/embed.py @@ -99,12 +99,15 @@ def display_top_matches(url, matches): # Batch embedding generation embeddings_ds = ds.map_batches( EmbedImages, - fn_constructor_kwargs={"model_id": "openai/clip-vit-base-patch32"}, - fn_kwargs={"device": "cuda"}, + fn_constructor_kwargs={ + "model_id": "openai/clip-vit-base-patch32", + "device": "cuda", + }, # class kwargs + fn_kwargs={}, concurrency=4, batch_size=64, num_gpus=1, - accelerator_type="L4", + accelerator_type="T4", ) embeddings_ds = embeddings_ds.drop_columns(["image"]) # remove image column diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/infer.py b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/infer.py similarity index 100% rename from doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/infer.py rename to doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/infer.py diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/model.py b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/model.py similarity index 100% rename from doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/model.py rename to doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/model.py diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/serve.py b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/serve.py similarity index 83% rename from doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/serve.py rename to doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/serve.py index 52b85f02defe..350d2a0cc789 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/serve.py +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/serve.py @@ -13,20 +13,12 @@ from ray import serve -# Define app -api = FastAPI( - title="doggos", - description="classify your dog", - version="0.1", -) - @serve.deployment( num_replicas="1", ray_actor_options={ - "num_cpus": 4, "num_gpus": 1, - "accelerator_type": "L4", + "accelerator_type": "T4", }, ) class ClassPredictor: @@ -49,13 +41,21 @@ def get_probabilities(self, url): ) with torch.inference_mode(): embedding = self.model.get_image_features(**inputs).cpu().numpy() - probabilities = self.predictor.predict_probabilities( + outputs = self.predictor.predict_probabilities( collate_fn({"embedding": embedding}) ) - return probabilities + return {"probabilities": outputs["probabilities"][0]} -@serve.deployment(num_replicas="1", ray_actor_options={"num_cpus": 2}) +# Define app +api = FastAPI( + title="doggos", + description="classify your dog", + version="0.1", +) + + +@serve.deployment @serve.ingress(api) class Doggos: def __init__(self, classifier): @@ -65,27 +65,28 @@ def __init__(self, classifier): async def predict(self, request: Request): data = await request.json() probabilities = await self.classifier.get_probabilities.remote(url=data["url"]) - return { - "probabilities": probabilities, - } + return probabilities -# Model registry +# Model registry. model_registry = "/mnt/user_storage/mlflow/doggos" experiment_name = "doggos" mlflow.set_tracking_uri(f"file:{model_registry}") -# Best run +# Get best_run's artifact_dir. sorted_runs = mlflow.search_runs( - experiment_names=[experiment_name], - order_by=["metrics.val_loss ASC"], + experiment_names=[experiment_name], order_by=["metrics.val_loss ASC"] ) best_run = sorted_runs.iloc[0] artifacts_dir = urlparse(best_run.artifact_uri).path # Define app app = Doggos.bind( - classifier=ClassPredictor.bind(artifacts_dir=artifacts_dir), + classifier=ClassPredictor.bind( + model_id="openai/clip-vit-base-patch32", + artifacts_dir=artifacts_dir, + device="cuda", + ) ) if __name__ == "__main__": diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/train.py b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/train.py similarity index 99% rename from doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/train.py rename to doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/train.py index 8d060d480c6f..328ce36c91cd 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/train.py +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/train.py @@ -148,7 +148,7 @@ def train_loop_per_worker(config): num_workers=num_workers, use_gpu=True, resources_per_worker={"CPU": 8, "GPU": 2}, - accelerator_type="L4", + accelerator_type="T4", ) # Datasets diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/utils.py b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/utils.py similarity index 100% rename from doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/utils.py rename to doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/utils.py diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/pyproject.toml b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/pyproject.toml new file mode 100644 index 000000000000..5ec5cff96a4b --- /dev/null +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "doggos" +version = "0.1.0" +requires-python = ">=3.12" +description = "doggos multimodal ai package" + +[tool.setuptools.packages.find] +where = ["."] +include = ["doggos*"] \ No newline at end of file diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/01-Batch-Inference.ipynb b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/01-Batch-Inference.ipynb index 858f43fa7b64..f50bf23508fd 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/01-Batch-Inference.ipynb +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/01-Batch-Inference.ipynb @@ -30,14 +30,35 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[92mSuccessfully registered `matplotlib, torch` and 4 other packages to be installed on all cluster nodes.\u001b[0m\n", - "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_eys8cskj5aivghbf773dp2vmcd?workspace-tab=dependencies\u001b[0m\n" + "\u001b[92mSuccessfully registered `ipywidgets, matplotlib` and 4 other packages to be installed on all cluster nodes.\u001b[0m\n", + "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_1dp3fa7w5hu3i83ldsi7lqvp9t?workspace-tab=dependencies\u001b[0m\n", + "\u001b[92mSuccessfully registered `doggos` package to be installed on all cluster nodes.\u001b[0m\n", + "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_1dp3fa7w5hu3i83ldsi7lqvp9t?workspace-tab=dependencies\u001b[0m\n" ] } ], "source": [ "%%bash\n", - "pip install -q \"matplotlib==3.10.0\" \"torch==2.7.1\" \"transformers==4.52.3\" \"scikit-learn==1.6.0\" \"mlflow==2.19.0\" \"ipywidgets==8.1.3\"" + "pip install -q -r /home/ray/default/requirements.txt\n", + "pip install -q -e /home/ray/default/doggos\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note**: A kernel restart may be required for all dependencies to become available. \n", + "\n", + "If using **uv**, then:\n", + "1. Turn off the runtime dependencies (`Dependencies` tab up top > Toggle off `Pip packages`). And no need to run the `pip install` commands above.\n", + "2. Change the python kernel of this notebook to use the `venv` (Click on `base (Python x.yy.zz)` on top right cordern of notebook > `Select another Kernel` > `Python Environments...` > `Create Python Environment` > `Venv` > `Use Existing`) and done! Now all the notebook's cells will use the virtual env.\n", + "3. Change the py executable to use `uv run` instead of `python` by adding this line after importing ray.\n", + "```python\n", + "import os\n", + "os.environ.pop(\"RAY_RUNTIME_ENV_HOOK\", None)\n", + "import ray\n", + "ray.init(runtime_env={\"py_executable\": \"uv run\", \"working_dir\": \"/home/ray/default\"})\n", + "```" ] }, { @@ -47,7 +68,7 @@ "outputs": [], "source": [ "%load_ext autoreload\n", - "%autoreload all" + "%autoreload all\n" ] }, { @@ -59,7 +80,27 @@ "import os\n", "import ray\n", "import sys\n", - "sys.path.append(os.path.abspath(\"..\"))" + "sys.path.append(os.path.abspath(\"../doggos/\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# If using UV\n", + "# os.environ.pop(\"RAY_RUNTIME_ENV_HOOK\", None)\n", + "# ray.init(runtime_env={\"py_executable\": \"uv run\", \"working_dir\": \"/home/ray/default\"})\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from doggos import utils\n" ] }, { @@ -80,20 +121,23 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:04:39,249\tINFO worker.py:1723 -- Connecting to existing Ray cluster at address: 10.0.52.172:6379...\n", - "2025-06-23 14:04:39,260\tINFO worker.py:1908 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttps://session-gcwehd9xxjzkv5lxv8lgcdgx2n.i.anyscaleuserdata.com \u001b[39m\u001b[22m\n", - "2025-06-23 14:04:39,266\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_a644723e367c78760222a7f2fcce949b2fe72f7b.zip' (1.92MiB) to Ray cluster...\n", - "2025-06-23 14:04:39,275\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_a644723e367c78760222a7f2fcce949b2fe72f7b.zip'.\n", - "2025-06-23 14:04:39,581\tINFO dataset.py:3048 -- Tip: Use `take_batch()` instead of `take() / show()` to return records in pandas or numpy batch format.\n", - "2025-06-23 14:04:39,583\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_11_0\n", - "2025-06-23 14:04:39,594\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_11_0. Full logs are in /tmp/ray/session_2025-06-23_13-49-50_102769_2149/logs/ray-data\n", - "2025-06-23 14:04:39,595\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_11_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> LimitOperator[limit=1]\n" + "2025-08-22 00:14:08,238\tINFO worker.py:1747 -- Connecting to existing Ray cluster at address: 10.0.52.10:6379...\n", + "2025-08-22 00:14:08,250\tINFO worker.py:1918 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttps://session-466hy7cqu1gzrp8zk8l4byz7l7.i.anyscaleuserdata.com \u001b[39m\u001b[22m\n", + "2025-08-22 00:14:08,255\tINFO packaging.py:588 -- Creating a file package for local module '/home/ray/default/doggos/doggos'.\n", + "2025-08-22 00:14:08,258\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_0193267f6c9951ce.zip' (0.02MiB) to Ray cluster...\n", + "2025-08-22 00:14:08,259\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_0193267f6c9951ce.zip'.\n", + "2025-08-22 00:14:08,262\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_6d26725922931a7a9e87fca928dfafe4f4e5e54b.zip' (1.18MiB) to Ray cluster...\n", + "2025-08-22 00:14:08,268\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_6d26725922931a7a9e87fca928dfafe4f4e5e54b.zip'.\n", + "2025-08-22 00:14:08,550\tINFO dataset.py:3057 -- Tip: Use `take_batch()` instead of `take() / show()` to return records in pandas or numpy batch format.\n", + "2025-08-22 00:14:08,552\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_59_0\n", + "2025-08-22 00:14:08,641\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_59_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", + "2025-08-22 00:14:08,642\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_59_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> LimitOperator[limit=1]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9dbfa59a93134b189b928b743d442130", + "model_id": "b6862146286847ef9294638c1aa3d311", "version_major": 2, "version_minor": 0 }, @@ -107,7 +151,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d44b4fa98a6343d9a31b3dba01234981", + "model_id": "86a135d7d9cd45bc8ad9e5f0e5c477ad", "version_major": 2, "version_minor": 0 }, @@ -121,7 +165,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3c0885d0896e4fac8b5a9bcb0f3833f1", + "model_id": "4bfc88e39a7b450c945855a2d3f908e4", "version_major": 2, "version_minor": 0 }, @@ -135,7 +179,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a05586f2d15948bba416a1200dcd8fa6", + "model_id": "1071a49d524e424498985dc424a029a1", "version_major": 2, "version_minor": 0 }, @@ -150,62 +194,63 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:05:56,467\tINFO streaming_executor.py:227 -- ✔️ Dataset dataset_11_0 execution finished in 76.87 seconds\n" + "2025-08-22 00:14:08,686\tWARNING resource_manager.py:130 -- ⚠️ Ray's object store is configured to use only 28.2% of available memory (67.8GB out of 240.5GB total). For optimal Ray Data performance, we recommend setting the object store to at least 50% of available memory. You can do this by setting the 'object_store_memory' parameter when calling ray.init() or by setting the RAY_DEFAULT_OBJECT_STORE_MEMORY_PROPORTION environment variable.\n", + "2025-08-22 00:15:25,802\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_59_0 execution finished in 77.16 seconds\n" ] }, { "data": { "text/plain": [ - "[{'image': array([[[ 32, 52, 77],\n", - " [ 27, 47, 72],\n", - " [ 28, 43, 72],\n", + "[{'image': array([[[123, 118, 78],\n", + " [125, 120, 80],\n", + " [128, 120, 83],\n", " ...,\n", - " [235, 235, 233],\n", - " [236, 236, 234],\n", - " [236, 236, 234]],\n", + " [162, 128, 83],\n", + " [162, 128, 83],\n", + " [161, 127, 82]],\n", " \n", - " [[ 34, 51, 77],\n", - " [ 30, 47, 73],\n", - " [ 30, 45, 74],\n", + " [[123, 118, 78],\n", + " [125, 120, 80],\n", + " [127, 119, 82],\n", " ...,\n", - " [233, 233, 231],\n", - " [233, 233, 231],\n", - " [233, 233, 231]],\n", + " [162, 128, 83],\n", + " [162, 128, 83],\n", + " [161, 127, 82]],\n", " \n", - " [[ 35, 50, 79],\n", - " [ 32, 47, 76],\n", - " [ 33, 48, 79],\n", + " [[123, 118, 78],\n", + " [125, 120, 80],\n", + " [127, 119, 82],\n", " ...,\n", - " [237, 237, 237],\n", - " [237, 237, 237],\n", - " [237, 237, 237]],\n", + " [161, 128, 83],\n", + " [161, 128, 83],\n", + " [160, 127, 82]],\n", " \n", " ...,\n", " \n", - " [[ 55, 80, 76],\n", - " [ 65, 90, 86],\n", - " [ 56, 78, 75],\n", + " [[235, 234, 239],\n", + " [233, 232, 237],\n", + " [221, 220, 225],\n", " ...,\n", - " [142, 168, 133],\n", - " [157, 184, 149],\n", - " [140, 170, 132]],\n", + " [158, 95, 54],\n", + " [150, 85, 53],\n", + " [151, 88, 57]],\n", " \n", - " [[ 52, 72, 70],\n", - " [ 77, 97, 95],\n", - " [ 78, 97, 95],\n", + " [[219, 220, 222],\n", + " [227, 228, 230],\n", + " [222, 223, 225],\n", " ...,\n", - " [125, 151, 112],\n", - " [141, 169, 128],\n", - " [180, 211, 167]],\n", + " [153, 91, 54],\n", + " [146, 83, 52],\n", + " [149, 88, 59]],\n", " \n", - " [[ 92, 108, 107],\n", - " [123, 139, 138],\n", - " [135, 149, 149],\n", + " [[213, 217, 216],\n", + " [217, 221, 220],\n", + " [213, 214, 216],\n", " ...,\n", - " [125, 152, 109],\n", - " [ 87, 116, 68],\n", - " [127, 159, 109]]], dtype=uint8),\n", - " 'path': 'doggos-dataset/train/saint_bernard/saint_bernard_7024.jpg'}]" + " [153, 91, 54],\n", + " [144, 83, 54],\n", + " [149, 88, 60]]], dtype=uint8),\n", + " 'path': 'doggos-dataset/train/border_collie/border_collie_1055.jpg'}]" ] }, "execution_count": null, @@ -220,7 +265,7 @@ " include_paths=True, \n", " shuffle=\"files\",\n", ")\n", - "ds.take(1)" + "ds.take(1)\n" ] }, { @@ -265,7 +310,7 @@ "source": [ "def add_class(row):\n", " row[\"class\"] = row[\"path\"].rsplit(\"/\", 3)[-2]\n", - " return row" + " return row\n" ] }, { @@ -278,7 +323,7 @@ "ds = ds.map(add_class,\n", " num_cpus=1,\n", " num_gpus=0,\n", - " concurrency=4)" + " concurrency=4)\n" ] }, { @@ -328,7 +373,7 @@ "import numpy as np\n", "from PIL import Image\n", "import torch\n", - "from transformers import CLIPModel, CLIPProcessor" + "from transformers import CLIPModel, CLIPProcessor\n" ] }, { @@ -354,7 +399,7 @@ " with torch.inference_mode():\n", " batch[\"embedding\"] = self.model.get_image_features(**inputs).cpu().numpy()\n", "\n", - " return batch" + " return batch\n" ] }, { @@ -393,9 +438,9 @@ " concurrency=4,\n", " batch_size=64,\n", " num_gpus=1,\n", - " accelerator_type=\"L4\",\n", + " accelerator_type=\"T4\",\n", ")\n", - "embeddings_ds = embeddings_ds.drop_columns([\"image\"]) # remove image column" + "embeddings_ds = embeddings_ds.drop_columns([\"image\"]) # remove image column\n" ] }, { @@ -441,7 +486,7 @@ "metadata": {}, "outputs": [], "source": [ - "import shutil" + "import shutil\n" ] }, { @@ -453,15 +498,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:06:01,973\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_16_0\n", - "2025-06-23 14:06:02,000\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_16_0. Full logs are in /tmp/ray/session_2025-06-23_13-49-50_102769_2149/logs/ray-data\n", - "2025-06-23 14:06:02,002\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_16_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" + "2025-08-22 00:15:55,241\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_64_0\n", + "2025-08-22 00:15:55,265\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_64_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", + "2025-08-22 00:15:55,267\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_64_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5933c94751554584b0efe1af1c11b265", + "model_id": "6d183707412548d5acd113c34ed06a4c", "version_major": 2, "version_minor": 0 }, @@ -472,17 +517,10 @@ "metadata": {}, "output_type": "display_data" }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-06-23 14:06:02,029\tINFO actor_pool_map_operator.py:633 -- Scaling up actor pool by 4 (reason=scaling to min size, running=0, restarting=0, pending=0)\n" - ] - }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "09923297706a4c6ca9bdc8c217fef9dd", + "model_id": "658e7e6b56fd4ddca63a85f903dc598c", "version_major": 2, "version_minor": 0 }, @@ -496,7 +534,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "61862104a5dd47b8be8364d4a9f91677", + "model_id": "ea845b3839f341fe96882d806ad16146", "version_major": 2, "version_minor": 0 }, @@ -510,7 +548,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2f089288b9214fde9a8c93dad13fc7ab", + "model_id": "83e3d887e66844b7a414f95163268c4f", "version_major": 2, "version_minor": 0 }, @@ -524,7 +562,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f22efa3031dd4b86903f20c17d00946f", + "model_id": "dca912f45aab4457b4188f74fb21ab63", "version_major": 2, "version_minor": 0 }, @@ -538,7 +576,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "915c9ece141c44519b93546cc8ab7724", + "model_id": "1b0ab6aaffeb45aea51b8a3c7b75540a", "version_major": 2, "version_minor": 0 }, @@ -549,25 +587,34 @@ "metadata": {}, "output_type": "display_data" }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(autoscaler +2m12s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n", + "\u001b[36m(autoscaler +2m17s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", + "\u001b[36m(autoscaler +2m17s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", + "\u001b[36m(autoscaler +2m57s)\u001b[0m [autoscaler] Cluster upscaled to {104 CPU, 8 GPU}.\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=2910, ip=10.0.69.70)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "2025-06-23 14:06:22,379\tINFO actor_pool_map_operator.py:661 -- Scaled down actor pool by 1 (reason=None; running=3, restarting=0, pending=0)\n", - "2025-06-23 14:06:22,744\tINFO streaming_executor.py:227 -- ✔️ Dataset dataset_16_0 execution finished in 20.74 seconds\n", - "2025-06-23 14:06:22,842\tINFO dataset.py:4603 -- Data sink Parquet finished. 2880 rows and 5.8MB data written.\n" + "\u001b[36m(_MapWorker pid=3333, ip=10.0.27.32)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "\u001b[36m(MapBatches(drop_columns)->Write pid=116142)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\n", + "\u001b[36m(_MapWorker pid=3332, ip=10.0.27.32)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)\u001b[0m\n", + "\u001b[36m(MapBatches(drop_columns)->Write pid=34034, ip=10.0.171.239)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\u001b[32m [repeated 32x across cluster]\u001b[0m\n", + "2025-08-22 00:18:30,236\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_64_0 execution finished in 154.97 seconds\n", + "2025-08-22 00:18:30,323\tINFO dataset.py:4621 -- Data sink Parquet finished. 2880 rows and 5.8MB data written.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[36m(autoscaler +5m51s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n", - "\u001b[36m(autoscaler +5m51s)\u001b[0m [autoscaler] Downscaling node i-018706717a4455b75 (node IP: 10.0.65.200) due to node idle termination.\n", - "\u001b[36m(autoscaler +5m51s)\u001b[0m [autoscaler] Downscaling node i-0e3238b7f703616e7 (node IP: 10.0.127.236) due to node idle termination.\n", - "\u001b[36m(autoscaler +5m51s)\u001b[0m [autoscaler] Downscaling node i-0fcefb76d19edf42b (node IP: 10.0.49.153) due to node idle termination.\n", - "\u001b[36m(autoscaler +5m56s)\u001b[0m [autoscaler] Cluster resized to {8 CPU, 2 GPU}.\n" + "\u001b[36m(autoscaler +6m52s)\u001b[0m [autoscaler] Downscaling node i-0b5c2c9a5a27cfba2 (node IP: 10.0.27.32) due to node idle termination.\n", + "\u001b[36m(autoscaler +6m52s)\u001b[0m [autoscaler] Cluster resized to {56 CPU, 4 GPU}.\n" ] } ], @@ -576,7 +623,7 @@ "embeddings_path = os.path.join(\"/mnt/cluster_storage\", \"doggos/embeddings\")\n", "if os.path.exists(embeddings_path): \n", " shutil.rmtree(embeddings_path) # clean up\n", - "embeddings_ds.write_parquet(embeddings_path)" + "embeddings_ds.write_parquet(embeddings_path)\n" ] }, { @@ -662,19 +709,28 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Output\n", + "(anyscale +0.9s) Submitting job with config JobConfig(name='image-batch-embeddings', image_uri='anyscale/ray:2.48.0-slim-py312-cu128', compute_config=None, env_vars=None, py_modules=['/home/ray/default/doggos'], py_executable=None, cloud=None, project=None, ray_version=None, job_queue_config=None).\n", + "(anyscale +3.0s) Uploading local dir '/home/ray/default' to cloud storage.\n", + "(anyscale +4.2s) Uploading local dir '/home/ray/default/doggos' to cloud storage.\n", + "(anyscale +5.2s) Job 'image-batch-embeddings' submitted, ID: 'prodjob_cmhr6w7l9fb42be6xjsz1rnxsl'.\n", + "(anyscale +5.2s) View the job in the UI: https://console.anyscale.com/jobs/prodjob_cmhr6w7l9fb42be6xjsz1rnxsl\n", + "(anyscale +5.2s) Use `--wait` to wait for the job to run and stream logs.\n" + ] + } + ], "source": [ - "```bash\n", - "# Production batch job.\n", - "anyscale job submit --name=generate-doggos-embeddings \\\n", - " --containerfile=\"/home/ray/default/containerfile\" \\\n", - " --compute-config=\"/home/ray/default/configs/aws.yaml\" \\\n", - " --working-dir=\"/home/ray/default\" \\\n", - " --exclude=\"\" \\\n", - " --max-retries=0 \\\n", - " -- python doggos/embed.py\n", - "```" + "%%bash\n", + "# Production batch embedding generation job\n", + "anyscale job submit -f /home/ray/default/configs/generate_embeddings.yaml\n" ] }, { @@ -708,7 +764,7 @@ "from PIL import Image\n", "import numpy as np\n", "import requests\n", - "from doggos.embed import get_top_matches, display_top_matches" + "from doggos.embed import get_top_matches, display_top_matches\n" ] }, { @@ -719,7 +775,7 @@ "source": [ "def url_to_array(url):\n", " return np.array(Image.open(\n", - " BytesIO(requests.get(url).content)).convert(\"RGB\"))" + " BytesIO(requests.get(url).content)).convert(\"RGB\"))\n" ] }, { @@ -727,6 +783,13 @@ "execution_count": null, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n" + ] + }, { "data": { "text/plain": [ @@ -744,7 +807,7 @@ "image = url_to_array(url=url)\n", "embedding_generator = EmbedImages(model_id=\"openai/clip-vit-base-patch32\", device=\"cpu\")\n", "embedding = embedding_generator({\"image\": [image]})[\"embedding\"][0]\n", - "np.shape(embedding)" + "np.shape(embedding)\n" ] }, { @@ -756,15 +819,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:13:37,591\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_20_0\n", - "2025-06-23 14:13:37,597\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_20_0. Full logs are in /tmp/ray/session_2025-06-23_13-49-50_102769_2149/logs/ray-data\n", - "2025-06-23 14:13:37,598\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_20_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles]\n" + "2025-08-22 00:23:04,494\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_66_0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-08-22 00:23:04,500\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_66_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", + "2025-08-22 00:23:04,501\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_66_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "95d21fb5261949a69e7b9b52e9c93605", + "model_id": "7b04bc8e4d444e82950af699e0891b1e", "version_major": 2, "version_minor": 0 }, @@ -778,7 +847,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ef0d324d477c42e587bbc19eed93697d", + "model_id": "5c7fffbb6cf44a29904a90f20015ab9b", "version_major": 2, "version_minor": 0 }, @@ -792,7 +861,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e896cad5fe264723a3cc3cd4f6a64ebf", + "model_id": "57a3c6fc76d448b49941e9459d88b051", "version_major": 2, "version_minor": 0 }, @@ -807,12 +876,12 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:13:38,825\tINFO streaming_executor.py:227 -- ✔️ Dataset dataset_20_0 execution finished in 1.23 seconds\n" + "2025-08-22 00:23:05,178\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_66_0 execution finished in 0.68 seconds\n" ] }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABdEAAAGHCAYAAABfzRvzAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsvXmgZUV1Nb521bnvdTeT2KCiGMBmUFHUD000yiDOIuKHQlBJQwTUnxIHNIJDggQEjQMYFRU1KgIh4oga0E9posaoETUkJioOEKNGZB56ePdU7d8fe++qOuee+4YGAbtrwe17X506darqnHffrlWrVhEzMyoqKioqKioqKioqKioqKioqKioqKioqJuDu6gpUVFRUVFRUVFRUVFRUVFRUVFRUVFRU3F1RSfSKioqKioqKioqKioqKioqKioqKioqKKagkekVFRUVFRUVFRUVFRUVFRUVFRUVFRcUUVBK9oqKioqKioqKioqKioqKioqKioqKiYgoqiV5RUVFRUVFRUVFRUVFRUVFRUVFRUVExBZVEr6ioqKioqKioqKioqKioqKioqKioqJiCSqJXVFRUVFRUVFRUVFRUVFRUVFRUVFRUTEEl0SsqKioqKioqKioqKioqKioqKioqKiqmoJLoFRUVFRUVFRUVFRUVFRUVFRUVFRUVFVNQSfSKioqKioqKuyWOOuoobLnllre7nKuuugpEhLe97W0L5n3jG98IIrrd1/x9wV3R3pe85CV40pOedKdec6m45JJLsOWWW+K3v/3tXV2VioqKioqKioqKioq7ASqJXlFRUVFRUVFxF+Oss87CRz7ykbu6Gr9z/PznP8cHP/hBvO51r5s49qEPfQgPetCDsGzZMuy2225417vetehyr7zyShx++OHYcccdsWLFCjzwgQ/EX//1X2Pt2rWdfF/60pdw9NFH4yEPeQi899h5550Hy3vqU5+KXXfdFaeffvqS2ldRUVFRUVFR8fsCE1Nce+21d1kdLrvsMhARLrvssrusDncEdt55Zxx11FHp56F2HXXUUVNjz9uDKlC581BJ9IqKioqKiooKxRve8AasW7fuTr/uXUWi39ntfec734lddtkFj3/84zvp73//+3HMMcdgzz33xLve9S485jGPwcte9jK85S1vWbDMX/ziF/jDP/xDfPOb38Rxxx2HM888E495zGNw0kkn4bnPfW4n7/nnn4/zzz8f22yzDe573/vOW+6LXvQivP/978ctt9yy9IZWVFRUVFRUVFRU/I5RBSp3LiqJXlFRUVFRUXG3wm233XaXXbtpGixbtmzePDFGrF+//k6q0cajbVvMzc0NHrM+Xkx7l4J+YF1iPB7jvPPOw2GHHdZJX7duHV7/+tfjwAMPxCc+8Qkce+yxOOecc/D85z8fp5xyCm644YZ5r/mxj30MN954I77whS/gxBNPxAtf+EJ8+MMfxurVq3HRRRd1zj/ttNNw880345//+Z/xsIc9bN5yn/3sZ2PDhg248MILF9HyioqKioqKioqKiun4wAc+gB/96Ed3aJlVoHLnopLoFRUVFRUVFYvCmjVrQET49Kc/PXHs/PPPBxHhX/7lX1LapZdein322QdbbLEF7nGPe+Dggw/Gf/3Xf3XOs2Wk//mf/4nnPe952HbbbfG4xz1uah2+//3vY/vtt8f++++PW2+9FQDwne98B095ylOw3XbbYfny5dhll13wghe8YPD8s88+G6tWrcLs7Cwe9ahH4V//9V8H61OCiHDcccfhvPPOw5577onZ2VlccsklAIBf/vKXeMELXoB73/vemJ2dxZ577om/+7u/m6cXJ7HzzjvjBz/4Af7pn/4JRAQiwv7775+O33jjjXjFK16B+9///pidncWuu+6Kt7zlLYgxpjyl7/uZZ56Z2vif//mf8/bxNE/0c889F3vvvTeWL1+Oe97znjj88MPxi1/8opNn//33x0Me8hBcfvnl2HfffbFixYpBFYzh61//Oq699lo88YlP7KSvWbMG1113HV7ykpd00l/60pfitttuwxe+8IV5++/mm28GANz73vfupO+www5wzmFmZial3fe+98VoNJq3PMO97nUv7LXXXvjsZz+7qPwVFRUVFRUVFRVdMPOduurxrhTjLITRaITZ2dk7rLwqULnzUUn0ioqKioqKikVh//33x/3vf3+cd955E8fOO+88rFq1Co95zGMAAF/+8pfxlKc8Bddccw3e+MY34vjjj8c3vvENPPaxj8VVV101cf6hhx6KtWvX4rTTTsOxxx47eP1//dd/xQEHHIBHPOIRuPjii7HlllvimmuuwZOf/GRcddVVOPHEE/Gud70Lz3/+8/HNb35z4vzzzz8fb33rW/GiF70Ip556Kq666ioccsghGI/HC7b90ksvxStf+Ur8yZ/8Cd75zndi5513xm9+8xs8+tGPxpe//GUcd9xxeOc734ldd90VRx99NM4888wFyzSceeaZ2HHHHfHABz4QH/vYx/Cxj30Mr3/96wGIsnu//fbDueeei9WrV+Nv//Zv8djHPhavfe1rcfzxx0+U9eEPfxjvete78MIXvhBvf/vbcc973jMdW0wfA8Cb3vQmrF69Grvtthve8Y534BWveAW+8pWvYN9998WNN97YyXvdddfhaU97Gh7+8IfjzDPPnFDBlPjGN74BIsIjHvGITvr3vvc9AMAjH/nITvree+8N51w6Pg024XD00Ufj+9//Pn7xi1/gH/7hH/De974XL3vZy7DFFlvMe/582HvvvfGNb3xjo8+vqKioqKioqLi749prr8Vhhx2GrbfeGitXrsTLX/7yiVWXbdvilFNOSUKNnXfeGa973euwYcOGTr6dd94Zz3jGM/DFL34Rj3zkI7F8+XK8//3vBwD8z//8D571rGdhiy22wL3udS+88pWvnDjf8K1vfQtPfepTsc0222DFihXYb7/98M///M+dPEsV4/Sxfv16vPGNb8Tuu++OZcuWYYcddsAhhxyCn/70pynPbbfdhle96lVJzLLHHnvgbW97G5h50dcxDHmixxhx5plnYs8998SyZctw73vfGy960YsWJLqBKlC5S8AVFRUVFRUVFYvEa1/7Wp6dneUbb7wxpV1zzTXcNA2fdNJJKe3hD3843+te9+Lrrrsupf3bv/0bO+d49erVKe2kk05iAPzc5z534lpHHnkkb7HFFszM/PWvf5233nprPvDAA3n9+vUpz6c//WkGwP/6r/86tc4///nPGQCvXLmSr7/++pT+2c9+lgHw5z73uYn6lADAzjn+wQ9+0Ek/+uijeYcdduBrr722k3744YfzNttsw2vXrp1apz723HNP3m+//SbSTznlFN5iiy34xz/+cSf9xBNPZO89//d//3enjVtvvTVfc801nbzz9XG/vVdddRV77/lNb3pTJ9+///u/c9M0nfT99tuPAfD73ve+RbXxiCOO4JUrV06kv/SlL2Xv/eA522+/PR9++OELln3KKafw8uXLGUB6vf71r5/3nAMPPJB32mmnefOcdtppDIB/85vfLFiHioqKioqKiorfJ1gc+NCHPpQPOuggfve7381HHHEEA+A//dM/7eQ98sgjGQA/5znP4fe85z28evVqBsDPetazOvl22mkn3nXXXXnbbbflE088kd/3vvfxmjVreO3atbz77rvzsmXL+DWveQ2feeaZvPfee/Nee+3FAHjNmjWpjK985Ss8MzPDj3nMY/jtb387n3HGGbzXXnvxzMwMf+tb35qo/4Mf/GA++OCD+ayzzuL3vOc9i2p727b8hCc8gQHw4Ycfzu9+97v59NNP5wMOOIA/85nPMDNzjJEPOOAAJiI+5phj+N3vfjcfdNBBDIBf8YpXTLT7yCOPTD+vWbNmol1HHnnkROx5zDHHcNM0fOyxx/L73vc+PuGEE3iLLbbgRz3qUTw3NzdvG0499VQmIr7pppsm0ofi1w0bNrBzjo8//vh5y7344osZAD/zmc/k733ve/zf//3ffMEFF/DWW2890e4Si4mtjznmGN5uu+3mzXN3RnOnsfUVFRUVFRUVv/dYvXo1Tj/9dHziE5/A0UcfDQD4h3/4B7RtiyOOOAIA8Otf/xrf//738ZrXvKajhN5rr73wpCc9Cf/4j/84Ue6LX/ziqddcs2YNDjroIDz5yU/GBRdc0FE/3OMe9wAAfP7zn8fDHvaweZUQf/Inf4Jtt902/bzPPvsAAH72s58t2O799tsPD37wg9PPzIxPfvKTOOyww8DMuPbaa9OxpzzlKbjgggvw3e9+F4997GMXLHs+XHjhhdhnn32w7bbbdq7xxCc+EW9+85vx1a9+Fc9//vNT+rOf/Wxsv/32g2XN18eGT33qU4gx4rDDDutc7z73uQ922203rFmzpmPZMjs7iz/7sz9bVFuuu+66Tv8b1q1b17mnJZYtW7aoJcA777wz9t13Xzz72c/GypUr8YUvfAGnnXYa7nOf++C4445bVP2GYPW99tprca973Wujy6moqKioqKiouLtil112Sergl770pdh6661x1lln4dWvfjX22msv/Nu//Rs++tGP4phjjsEHPvABAMBLXvIS3Ote98Lb3vY2rFmzprMa8Sc/+QkuueQSPOUpT0lp73znO/HjH/8YH//4x3HooYcCAI499tgJCxBmxotf/GI8/vGPx8UXX5xsB1/0ohdhzz33xBve8AZ86Utf6pzzsIc9DOeff/6S2nzOOefgK1/5Ct7xjnfgla98ZUo/8cQTk8r8oosuwqWXXopTTz01rRJ96UtfikMPPRTvfOc7cdxxx2HVqlVLum6Jr3/96/jgBz+I8847D8973vNS+uMf/3g89alPxYUXXthJ7+OHP/wh7nnPe2LrrbfupP/617+G934idp2ZmcHKlSvxq1/9at56PfWpT8Upp5yC0047DRdddFFKf/3rX49TTz11KU2cwAMe8ABce+21uOaaa34vY+tq51JRUVFRUVGxaDzwgQ/Eox71qI6ly3nnnYdHP/rR2HXXXQEAV199NQBgjz32mDj/QQ96EK699toJv8Jddtll8Hrr16/HgQceiEc84hH4+Mc/PkG27rfffnj2s5+Nk08+Gdtttx0OPvhgfPjDHx5cGvoHf/AHnZ+NIF3Mcsl+/X7729/ixhtvxNlnn43tt9++8zJS+Zprrlmw3IVw5ZVX4pJLLpm4hi3b7F9jWj8udKy8HjNjt912m7jmf/3Xf01c7373u99UAnwIPLD0dfny5VM3QF2/fj2WL18+b5kXXHABXvjCF+KDH/wgjj32WBxyyCH40Ic+hCOPPBInnHACrrvuukXXb1p9h3zjKyoqKioqKio2Bbz0pS/t/Pznf/7nAJCEL/betxJ81ateBQAT9iC77LJLh0C3MnbYYQc85znPSWkrVqzAC1/4wk6+73//+7jyyivxvOc9D9dddx2uvfbaNHZ4whOegK9+9audfYGAxQlF+vjkJz+J7bbbLrW1hMV9//iP/wjvPV72spd1jr/qVa8CM+Piiy9e8nVLXHjhhdhmm23wpCc9KbXz2muvxd57740tt9wSa9asmff8O0OgcvbZZ+OTn/wkXvCCF+C0007Du9/97sU1bgpKgcrvI6oSvaKioqKiomJJWL16NV7+8pfjf/7nf7BhwwZ885vfvN0B1TSidHZ2Fk9/+tPx2c9+Fpdccgme8YxndI4TET7xiU/gm9/8Jj73uc/hi1/8Il7wghfg7W9/O775zW9iyy23THm994PXGCJ2F6qfBe9HHHEEjjzyyMFz9tprrwXLXQgxRjzpSU/Ca17zmsHju++++7z1XOyx8npEhIsvvniwv8r+XGyZhpUrVw5OWOywww4IIUwoUubm5nDdddfhvve977zlnnXWWXjEIx6BHXfcsZP+zGc+Ex/5yEfwve99b8IrcrGw+m633XYbdX5FRUVFRUVFxd0du+22W+fnVatWwTmX9jG6+uqr4ZxLghnDfe5zH9zjHvdIAhrDkHDj6quvxq677johTOiLbq688koAmBpfA8BNN93UIY8XIxTp46c//Sn22GMPNM10WvTqq6/Gfe97X2y11Vad9Ac96EHp+O3BlVdeiZtuummqInsxgpzfpUDlxz/+cYqvDznkEMQYccIJJ+C5z30uVq5cuWDd5qvv76tApZLoFRUVFRUVFUvC4YcfjuOPPx5///d/j3Xr1mE0GuFP/uRP0vGddtoJAPCjH/1o4twf/vCH2G677Ra92SMR4bzzzsPBBx+MQw89FBdffHHaSLLEox/9aDz60Y/Gm970Jpx//vl4/vOfjwsuuADHHHPMxjVyAWy//fbYaqutEELYaIK2xLRActWqVbj11lvvkGssBqtWrQIzY5dddpkg6G8vHvjAB+K8887DTTfdhG222SalP/zhDwcAfOc738HTn/70lP6d73wHMcZ0fBp+85vfDKpwbMPYtm03us4///nPsd122021yKmoqKioqKio2NQwLS5dLPG5FJFFHyZUeetb3zo1Brw9oo67E2KMuNe97tVZ4VtiofizClTufFQ7l4qKioqKioolYbvttsPTnvY0nHvuuTjvvPPw1Kc+tRMI7bDDDnj4wx+Oj370o7jxxhtT+n/8x3/gS1/6UocoXQxmZmbwqU99Co961KNw0EEH4dvf/nY6dsMNN0woMCzgHrJ0uaPgvcezn/1sfPKTn8R//Md/TBz/7W9/u6Tytthii05fGQ477DD8y7/8C774xS9OHLvxxhtvF0E8hEMOOQTee5x88skT/crMt8sa5TGPeQyYGZdffnkn/YADDsA973lPvPe97+2kv/e978WKFStw4IEHprRrr70WP/zhD7F27dqUtvvuu+N73/sefvzjH3fO//u//3s4527XioDLL78cj3nMYzb6/IqKioqKioqKuztM/W34yU9+ghgjdt55ZwAikIkxTuT7zW9+gxtvvDEJaObDTjvthJ/+9KcT8WVfdGMe41tvvTWe+MQnDr7m2wNpsVi1ahV+9KMfJdHFtDr/6le/wi233NJJ/+EPf5iO3946XHfddXjsYx872M6+X3wfD3zgA3HDDTfgpptu6qSXApUSSxGohBAm0qtApZLoFRUVFRUVFRuB1atX44orrsCPf/zjtKFoibe+9a247rrr8JjHPAZve9vbcMopp+CAAw7ANttsgze+8Y1Lvt7y5cvx+c9/HnvssQee9rSnJeL6ox/9KPbYYw+ccMIJOPvss/H2t78dhxxyCLbeeuslk/VLxZvf/GbssMMO+KM/+iO84hWvwNlnn403v/nNOOywwwb94OfD3nvvjSuuuAKnnnoqLrjgAlx66aUAgL/4i7/A//k//wfPeMYzcOyxx+J973sf3v72t+Ooo47CjjvuOEi83x6sWrUKp556Ks4//3w87nGPw1vf+la8733vwwknnIA99tgDH/7whze67Mc97nFYuXIlvvzlL3fSly9fjlNOOQWf//znceihh+KDH/wgjjzySJx77rl4/etf39mc9t3vfjce9KAHdSZS/uIv/gIhBOyzzz445ZRTcNZZZ+HpT386PvOZz+AFL3hBR21jfXzqqafiJz/5CW666ab08+c+97lOva655hpcccUVOPjggze6zRUVFRUVFRUVd3e85z3v6fz8rne9CwDwtKc9DQBSTH3mmWd28r3jHe8AgI7gYRqe/vSn41e/+hU+8YlPpLS1a9fi7LPP7uTbe++9sWrVKrztbW/DrbfeOlHOUoUq0/DsZz8b11577aAlpRH9T3/60xFCmMhzxhlngIhS/2wsDjvsMIQQcMopp0wca9t2wTi/ClTufFQ7l4qKioqKiool46CDDsK2226LGCOe+cxnThx/4hOfiEsuuQQnnXQS/uqv/gqj0Qj77bcf3vKWt2yUbyEgipQvfvGL2HffffGkJz0JX/va17Dffvvh29/+Ni644AL85je/wTbbbIM//MM/xHnnnbfR11ks7n3ve+Pb3/42/vqv/xqf+tSncNZZZ2HlypXYc8898Za3vGVJZf3VX/0Vrr76avzN3/wNbrnlFuy333444IADsGLFCvzTP/0TTjvtNFx44YU455xzsPXWW2P33XfHySef3LFFuaNw4oknYvfdd8cZZ5yBk08+GQBw//vfH09+8pMH7/ViMTMzg+c///m48MILcdppp3WOveQlL8FoNMLb3/52XHTRRbj//e+PM844Ay9/+csXLHfffffFN77xDbzxjW/EWWedheuuuw677LIL3vSmN014yX/3u9/FX/7lX3bS7OcjjzwSBx10UEr/1Kc+hdnZWRx22GEb2+SKioqKioqKirs9fv7zn+OZz3wmnvrUp+Jf/uVfcO655+J5z3teUkI/7GEPw5FHHomzzz4bN954Y4q/P/rRj+JZz3oWHv/4xy94jWOPPRbvfve7sXr1alx++eXYYYcd8LGPfQwrVqzo5HPO4YMf/CCe9rSnYc8998Sf/dmf4X73ux9++ctfYs2aNdh6660nhA8bg9WrV+Occ87B8ccfj29/+9vYZ599cNttt+HLX/4yXvKSl+Dggw/GQQcdhMc//vF4/etfj6uuugoPe9jD8KUvfQmf/exn8YpXvCKp5jcW++23H170ohfh9NNPx/e//308+clPxmg0wpVXXokLL7wQ73znOzsbsfZRClQOOOCAlG4ClZe+9KU49NBD8ZSnPAVf+9rXcO655+JNb3rThEDl5JNPxpo1a5Jl5l/8xV/g4osvxj777IPjjjsOK1euxOc//3lcfPHFOOaYYyYEKhdddBEAdAQqgDw3ZWxtApX+Rra/V+CKioqKioqKiiViPB7z9ttvzy94wQvu6qpU/B7hpz/9KY9GI/7yl798V1dlQTz84Q/nV7ziFXd1NSoqKioqKioqfic46aSTGAD/53/+Jz/nOc/hrbbairfddls+7rjjeN26dZ284/GYTz75ZN5ll114NBrx/e9/f37ta1/L69ev7+Tbaaed+MADDxy83tVXX83PfOYzecWKFbzddtvxy1/+cr7kkksYAK9Zs6aT93vf+x4fcsghvHLlSp6dneWddtqJDzvsMP7KV74yUf/f/va3G9X+tWvX8utf//rUpvvc5z78nOc8h3/605+mPLfccgu/8pWv5Pve9748Go14t91247e+9a0cY5xo95FHHpl+XrNmzUS7jjzySN5pp50m6nH22Wfz3nvvzcuXL+etttqKH/rQh/JrXvMa/tWvfrVgG172spfxrrvuOnjs7LPP5j322INnZmZ41apVfMYZZ0zU2/qw3//f+ta3+GlPexrf5z734dFoxLvvvju/6U1v4vF43Mn34Q9/mAEMvsr+YGZ+73vfyytWrOCbb755wXbdXUHMA1u5VlRUVFRUVFTMg0984hM49NBDcdlll2G//fa7q6tT8XuE/+//+//wk5/8BP/v//2/u7oqU3HJJZfgOc95Dn72s591NmSqqKioqKioqKiouLvgZz/7GR74wAfi4osvxhOe8IS7ujrz4hGPeAT2339/nHHGGXd1VTYalUSvqKioqKioWDS+9a1v4YorrsApp5yC7bbbDt/97nfv6irdrfHb3/52cGMew8zMTGdJZUVFRUVFRUVFRUVFxWJRBSp3HiqJXlFRUVFRUbFoHHXUUTj33HPx8Ic/HB/5yEfwkIc85K6u0t0aO++8M66++uqpx/fbbz9cdtlld16FKioqKioqKioqKjZRzM3N4frrr583zzbbbIPly5ffSTWq2JRQSfSKioqKioqKit8R/vmf/xnr1q2benzbbbfF3nvvfSfWqKKioqKioqKiomLTxGWXXbbgRqcf/vCHcdRRR905FarYpFBJ9IqKioqKioqKioqKioqKioqKiorfa9xwww24/PLL582z5557YocddriTalSxKaGS6BUVFRUVFRUVFRUVFRUVFRUVFRUVFRVT4O7qClRUVFRUVFRUVFRUVFRUVFRUVFRUVFTcXdHc1RWoqKioqKio+N3gsUecChBARHDk9LMDAQARHBEovRoArvg5vwDAOZc+E0j+HcgnRROcc+nztPehtGnn9c9Z6NjQtYbKHI/HWLduHbz3WLFiBbz3YDB6RQ2WKX0BYGBNX78uKfO0goeutdhyASxlYeGSFiFSbp6dx8wgos7PQ+UPpW90Xh6+Xplv6L3MG2OcN0//+MQLDOaYfo4xTuS1MpgDOM7pZ0sDmCNizGnf+odThnq9oqKioqKioqKiouJuhkqiV1RUVFRUbKJwzuu7myCRJ18eRoyXRPYgqc7AfCR6/3pWVv99iNheiERfzPtCecp8MzMzaNsWMUa0bYumabQO04nmThk8zItPI7vhbh+JPl+dFoslEe7I7TOiuGz/NGJ76DpLSZ9I4+HrLUSiL3S9/sTAEDHeJdG7/dDP0zkXXsnyMh8ByGkVFRUVFRUVmw/eddYhKY5iBkKICCEghhwbxBgRIzCOjBi7cYzFX957eC9xPnMEiOG9pFssDQCzs7OIUa4RQkDbtgiBEUJIE/3eOymvyeOAHPtFxNiCWYQ4znk458FM8G6EmZkZNE0D7xs4jODdLBxJnjS2QAOCA0DaHhEuBG4RYwCjlWtgLIITFRxInRkcI9owhhx0ADsQOcQYMB7PYW68ASGMQU7GEctnt4FzDbz3aLwHOQcCgQjwTYNR02D58uVYvnwLkFsBggPHMQgRHFtsWLcWt916K66/7gasX7sec+vnENoIbgijFTNYscUyrFjm0TSA8wxwQIzy4hgRYwAhwkHiPqIGM7PLMLtsCyxfsSW8n0HjZ+GaERwRIq/Fhg03Yv2GtZibm0OMMfU1R4e2ZYQWCJERYouW12NuPIe2HcN7h+XLl8O7GRB5EEYARgB7MALaeCvWb9iAdhwRAmM816ZnKoQWIYzRhrE8CxFgDogcwIgwuVAMOiqKHiDp+0BR42JOz2VXTFLE9JHlfJb3UnjiiOCbBjOjGYxmRqAAcATIz2BmdgswNxiP9fcjjhHadVgx22Cn+90X99r2XlgxsyXIA4yIMQesXbcO195wPX71v9fgxptuwVwb0bYREYSgYz0gInKUZ0YHW+QA7wneO3zmgu/O+ztcSfSKioqKiopNFCWZPY1El0CbRKFOrnNeP28m0btllcH6NGLdjvXPmY9EHyKGpynZ57tW+d6vIyBE+ng8TgGfc6K0n4aJspegGL8jSPQh8vV3RaIDwl/31efzkegLpW9U3nlI9P77NKX6tOuVJHhJjNsxI9Yjxw6JPvTqlJcGjHFqnoqKioqKiorNB+Q8vPNwjmRlWmwROIpowTmAWeI/AgicSL4yfnDOdWL1EDmFl7byjagbGxpxba8y7nHOgVyOw72XWFzy9gUE+o5MtKfrUgRzCyaLcSIAp4SsA+AAIiVVGcwtgADmMRgtmFspk+R45KCxV7SrJKLa2tO28rMdBwCiCKKgZUlfmECmbTcghDkwAkCMZhSVRG8BDuAQsG79WqzfsA5tO0YMATFyUb6Q5G0bASJ4yD1klokK1v51YHDS5LSYm1sPBsOPPEbcglxEDBt0IuVWzI1vw7hdhza0ABgED4cRmB1YFS3ee5B3IPZwfgZzc1Iv7730N9sYRiYg2jDGOMikCSArkWPUiQkGYgRCAGIgrX8EKOZOI4Dg4Bud/NB7yJAJjBBbhCBku3Oy4jmt6tQ6p3uATKTbMy2Pu9zTyAFAA+dJDniCc4DzHs3Iox0HhDYCTPANoWkcnNffA+cAYkQmnViidLeYgcik1ZFnnRkgfaaEVJfmxgjEuLDjeSXRKyoqKioqNlGIEp3gnBHGJcGt1i6JTM6f7yglupVREq/TSHB7n0auW6Bfps2nWp+WPnT92dlZAEhKHd94DUK76BPIRDSVah8ktTUgXSyWVPYSUN6Tja2EBKGZdO4O1CbT+0qV+fL20yWBJvLO996/ng06FzpH1D/dMuw8B5cmFBYk0SODnS+umwd8Vl4l0SsqKioqKjYvhBiFIiUvJJ9zgBJ/zEI1RiUcfeNBSsqGEDrllHGsdw6+EcZWYg1WRbpHCIwY7WVEYRYIWOhlYwNAiEegjG+kbnaMmeF9X8AiVnaBGYAHnAexEucwcYp8ZijBSa22tgXQCpEeS6FBlJzEcA6Fgjqkz5JPy1fWNMSxKPMLMhhK7I/bOTmfx/AeCBzQ+BEQAzgGhPEY69bfhrVr10reGDRWdHCNKJZBAQxSghbgGDTOC6l/ohLJ0kWi+kYLrFt/M+LsLOBawJGuhl0H5vUIPKeTCQCxA3sCSMhiUZk7RIg6nSnC+Ryvkt1LEuV35IgYWrRtQDsOADuEALQtox3nVQghWr/q/SQPRwyQk4kG1nuuzwxHRkREm1YL2HM0KRJJ40rP6XmyPDbmk+dQVjuMx4BvZoTQ5hYhjuF8g6ZppP+JwIHgvUyUOCdlOO/BxPAsvzP27Mv1HYh08oiLZ74zPsnP0GJi80qiV1RUVFRUbKIoSfJpSvT8mu6HPpk2LX3+NPsMDFvMLPQ+pCYfyruUNKvLzMxMWkIpxKdPpGoiUXuKe2hPLJrUnkKiTz0/TgZy0/L+zgjZYnDVn8jok9BlHfsEep8g73+ej1i3AdJiCfSyPn2SvK8479fd7nXZjvyKiDzc/0TUJciJwcGrKksGrgB3CPRKoldUVFRUVGxeaFsGkaiBiQCwg6MG5FlsJ6IQxGBbAccTcYopjzvWGSHCwlSLX7JlCyciPibVds7rnIN3XlXIolbO11TluCqLOZIQyHqtMkaSc1oAHpEdHOXJgTx+IL2GThYku5hWSE4uLD+Qydk2BJgqW+oMxGiiIA+HmJTHc3MbMBpFIfojJ+W1KbRlQkLI25k4h9mZGXgCOESs37Aea9fegvXr16MdM2LUiQrvARdBqraPkaD8MUI7BkN9alRoJJMlUCGT9k8IwIaANqzHhvFtIAdV6Y8BjBGC9AGRA8iDMQa5Bo3zADzAQGAhrtuxKO1tvGJq7xgjYjDSWOwqQ2sTMUA7DpibGxf3S21ZYP2NZNOSVgxEyGRMyIR7TJMsUKW+PafW18giLpK+tnsAIOVhnWhoW1Gkow1omhFcM4MQ5hA5wlGjdjItQJwmCwhmPeRAjuEBeNeIdY/+Z9cD5zEBkVcOvdF6sT5H3fHKNFQSvaKioqKiYhPFEOFdfp72WkiJbqsq5yPdh2xZhq7fT+/WfeH3aWnTzumfZ8F/0zRpcMGqRu7n658P7V1HCy/9S5hi5zIRtDEvPi+WRqL3Se158xbe8P1zpqnI+3mH6tYnz4dI707ZA0p0y79UhXp/QFoeG1Ki54wOboqdixHoSdkFD3bSe+Wxfj0qKioqKioqNh+0Sbkb4MjBO7E4yWpx2z8FaNvxRLxSxqJdIh1gn1ehmkLYzs/Kc1kdZ2IW80N3zmXylO1caH10JR8yYSrvAVCiUmw0WGw7uIXZRJIq0HO9nU4QiFpbVNvqid7ZiJ2U+FTFd2y1HA84pyp3TtePsQHQgpnVosMU7F6IeRNIOBG0hNBi/fqA9eO1GI0aeAJiGzG3fg7r121AaFmtThwIqmp2ct8kOlZP++TXrvEoSxtjNAtAcScBAcQk7W7Xow0AeYLzACGAeQ6mhHbkZPKAPIgcvFOyOwLMEQ0RGA7ezwjhzl4J7ih9ANLJiZDU4uKDL33Ttm0ivO3FYLRqp0Mk6yFkMsWU6EAI6o9uEzz6XNo9S4S1Cl9iVP1Q5HSsHzvbmNFsgja0ETEwGnJwIHBo87iToxDps0KUyyWVLCcCks2P7DUg5xFiYIQYOkImefY9iEaZ/EdIkzbzoZLoFRUVFRUVmyickz/z08ju0nbFSO9FbULqAAuQhgjxaWn2uZ+vn26Y7/gQabsQiT6tHAvgmBlzc3MIZuvivS4hHPZ4115YNCGtF5ySTJP5BpTog3nvIvQJ6ttz7nzkuCR01eX2XpLS5blDZQwF7mV+IP8eDBHlzKKasmtaGf3VCqwjjggHRIIjp0oalwL1/KqoqKioqKjYfGCxBxAYaGMrFhmmHi6E4pn47hLnIYRObJpjG1XecgSoa60hJzs4Akj2I1VVOxXjAUrkL5KqGWA2RbnZqggxm4l/AnMjIS7HibGAS+pgI9WlH0xpLwR6AHNAiKzkp1PSWMhqotJ6I6Yyyv7h6ABEDaED5ub6dRFiNvl3swhneG4scVsbEcdRaXlGiBExBHgSwp4c9MVJYW6KdmYhf2UPH077Gpmi3hGpUnwOoChjqaTwjyBYXGqEtBDiRB6gVoniqJMvwIxvQPAAHEILzOmz4hwlhTbDYRzENzyEVj3kx7pCIfvIM4LQ6GbGj+SMo5veMmJ0YgWk9kBhyoa35OQOk8vPbGgBUH4W+2ISiaMpTZy0COB2jIYJcD7liW0rHvQMqK9OerE+i6ENOo7TlQDkwJB2CNeu94f0XlK2MZJfjO5KjSFUEr2ioqKiomITRV8NvqDX+ZRjVlaXgB7OZ8c3Vom+EGm+GGJ8senlz0SEpmnEm5BFkT4ajToK4mESHcWAYPo1gLTCcxAT+Rlpk6fFYCnE+pJU6x0t+mQ5ffJ7KdcbUpH3z0nnTiHRl/rev6ahsyS6l7dz3kC9h/KLbMcBPg+Ch8qrqKioqKio2HzAymCLYjroxvaiEJfVkDkvKZmdVNRF3GGbi1pZKT9HmEZ4Qu2rxKYjn/2kHaVXZCPlsw97CKTEMCWik8glIl/iJ0pEqCNbuZlFOSYyEGJXQyQmJd9NMR2EzE3EedQNL6UPcj+omh1lXIX80vJJiff80g0tWQhXCTkZkaIcYwYHsUIRH/modiYEI8mBCFAAKakOs2nh0IvpdKNYmN2MTYBI+eTM+iUiku6jaXJ1baso0RlMbWoEQfrXkZzkaAYcHVrSsmIh9GCxnpGFDkryxxZtO0bbBmQhh3WWquuJk3I/xogQZcWAPAuMtuXiHg2skoh5wkOeFWmWTLbIygXLK/df+svI7IYcmMXzHaMWrinj7AjYCgdqQB2//ez5345b2Vi1DWiD/Ny2IZHmMgGC9FymvqCFCXSgkugVFRUVFRWbLJxzSSlNzpa79Ylyl2bgF02uG3XcI+cN08oYOmZp5bH+56Fj/ev106flnZbGzEl53oY8WDESvZxE6J44zIsPke7zEdITZRPgBkqej3hdLJF+R5LoiylvIRJ9KM8Qid5PL88ZKms+Er1/fqnk6pdVEuw8UE5Zh0SYxyiWLsXAtyynVMJXVFRUVFRUbB4IIRPU5oHOql5mJaRh5CsYVChjSxK9LwbIccnkir4kiCGnZLoSt96jDOEz2ZtjlRhE4U3K75IYfcDsN8pzYpTjQlK6InY2EjyiVOIzN934j0ntY7IPt5G15EJnAZ/YudjKwGwBI0Q6p2uQqquNTJc6m9AHCIjazwwOAAezP8l2LAxG5ACKLSgGhOhEnW5K9IJ4zmbxonQnJWk5AuxCqgNg5wBgBwenVjPWR5l4Zw4AxjCLHKgi3hEQxScGTdOkDWNjDAhhjHE7hza0aKMIhISElhezrVhg6ygAHuWoJk9OsKrYWf3VASKfjtuEhNxz62fLB5CzVQtSrveqHk/lx+5zElsEliff6RgMABAZkeQ+28ROevaYtd0BaegQGeOxTFSFGOEYcN4BxdhYijYSHYtCJdErKioqKio2UVDhQUjoEdcg9dwzshkDBPt0Er2/EWnnureTRJ+PXC/T+3n6dZhMLwNpabD3TgPOqAFoBGOS4OxPFHT7eSBtoF+mM+jDavahds2H3wkpOw8/LkEysnBmyQWTnqsRuJWFPDjMg0QaTB8iwqe9l3n6G2vNh7wZaN5Y1Ejx8njns3MwOVk5EVMuza6oqKioqKjYvLB+/XpV9QpJS+rtHNPml0CKVbmFbXY4tLm9oR/XlD+PRqPuOdS3cTGiVtTLEhOXAgIjO7N4wDlSMtI2IjWyXohdIVMZzsWk9C0Jd4F5sJc+6HKdEKAkuhChkQPIhcIIhgC2jUa756bPSuxqj8i5pIp4SswvEmVNAEIEm11JCKqUJlXIi/2M4yB1cjHZLtp17Vr2VqqeQQAje53nK4vnvKjuJS723rznA5yTvvMuqk2nSxvPMrWqCgeC+p9DbWecl3K8J4xGIqryugkp0RxCaHWSIoKc1kltbqRN9hwa4V3G0g6yGsAmR8oxnEvni/KcQaSbl+qkRtRNdVknQcAyjWHjihiB2AaxY2mKyZjAIOdSm1knXYgiAovX+7gdo21b9YCPokZvAwKrCt7lON57vR8dEn3h+LyS6BUVFRUVFZsoyGXlRfdFKdgt34F8bF7rlwlFuwThyfZEgxT77CiratLAwDkNWhNrCpTEMymtXAbcRZqW2CmjVMNMkuicYiNKdWEQR3EUdBKQzzQOzF4HEi2cIzSNR95opk/WTyQV6DLLVLwmyuinTSlj/thukUz2EvhbNqJ8SkFMnCYAbAAzdD1O1eMUmMvYgfI1ys18mHNwDRLlSYdwZ30OCibfLpJ+lnexYaG04ZAMMLRAMouWyZeoiOwgyaQUdz3Y+0r1cvPQiAak6hgbBJEut4WeU1FRUVFRUbH5YMOGsZKGQhALmZ6V0wYiwEGUy967gqgELJoUwrFrR2eEtaWbQKQksEUt3CCrf4147xL53Rfr9fLKPLE54Uz0E4EdgSgTxNIWQrbsKNXiAWYdwwWBn69D4pHOEeCg8buSoYk0z7F/UiAXwWsZa4llDasbjLbA7FFU3cwaT6a+UGI7qcdBQBISKYmsYwq9IgBKEw3OO7l/JJuOOs9gnXxg9T7hAASO4hhPLHVyYmkTIqtFjt2VBkwEjq2GvQTApRUFKYbVh2g0GkFsVcTeZdQ2CCHC/NxDFA95dgxHAMiB9blL40R7Hh0BXBD56TKUniupt66qiNLWRKbrquhIpaAkTzo4RzI+1ItLOazmOAwKYpTDOsnCUcZqImwRFXpoAzZsWI+5uQ0Yt2PdYFee2TZEnUQCzDLI/PFp2tLiAVQSvaKioqKiYhNFlzCftF/p+5xPe9nxIZ/zoc1JO9cyUr1HarvedbuVRiLqNbFDgNu7S+XZoMK41S5hD4jeJfdJ2UuMfCn5PPIe48CIIQC+EbW6EbG9AGuC6J8GtgHRJO7WqmReqHkl3T/N+iVF/sXAiQF2BdedB0J2UePCo9XBlEP9iwgXrj6O3TIsJmawyKNMnhShBHp+DkuVeFJbMTpKp6G8QPY9H1Klu4JcjxFwjjvnVFRUVFRUVGweaNwKUceO5xBiUKLRlOOsKnEPIocYWo2tZQNJCRsY3luaIEbZkJM0Dg9BNor03qvnNiEqaeqcRwRjHAO8xkqhDekcjoSgViqiVJbzkhhCCXSvGnqNrkFE8I7QkKx0BTkhRJ0XujzqWIJM7ezARIjsEDgihLxhqQWAkRgBsiGmWRySku+JqGfpl1RPBgKMSFbyVccFAaJmD2lVoUxkMMmWpxwYiEAMHtC6ODiEAHhH4OgRAyGQx1gV36CYPNbZscaoESAPP/JoRrLhKlNUEREnUt4U5CCxKYnKgBOANkaMRo0KjBgNCA05wLUwyx+wKfkdvB+BHBDGAW2IQsq7BiECcA6Rx2ijbHrvRg3sEXIRaQWuJ0A2d1US3ZNsOepk3yhA+tZiWZt4EaGSxtiR4Z0DeSBC6hKJEFjU5QDDeZ1kMEtRG15RFI965+HcCM57AF5Jc/GsdyHCwcuETRyDo1chV0AI6zAOazE3XoeIMchH0CjK8GEsT05S1kM83okcnFeC35UzEPP8Di/qN72ioqKioqLi9w5GcE973Z6NRofSBvMXavGhfPZZP6BUuJeYVJZjkJwnykr0bt6u9/t8GI1GiMxo21a9rZuCsO9i0QQ4ZS3PRpdxF8AI5cXmXQwxnMss7xmQ2fGCCFdVjqO0lkCGbJ16mUqoPIZeXhtKyQWjDrTK1QvZtiUrp4Z81Pt57Xepn6dPtNt5Zu1SUVFRUVFRsXlByGKZoI9KHDM4CV8MRATnTS2evcIFQhhL3CVkbwTEjoRZ1cXi95xXyOnKPzAiAxyEOAeymt3ezRM8BiOhs6Lb3u3cHHNDCWCx+QiBO21J7UgjA6iPdfYeNzKeTf4QzUovqi+7Kqo70XRAWlI4BcwqmCEunQNhMSal1Yx2bwDAmzIHttIytEgqdiLAMYOpq8Y3JbUjUaELOatjFt3AVRZHmmd41LqVG9FDPdiz6AkUwAFw8EKsw0sfoQXDoQ3iOR50M1Qi81iXdjsnq2qdD/DwIMcgp5uo8hgAwyOCKSu1AzvdEDZibpzFJJ5Jvf1TD8Pp8+Z0IkUmQaSOkfMEEBCT4MWeeVGgk1gBMYNJfPeRvPidrkYQCZUQ8ATiCII8h8QthG1vAWrlOfGAZ4Aalikn1tUDOgGjcz1630yws/CYrJLoFRUVFRUVmyi8F5WKkXzzEeoLKdKBrurcUJZbko8pHeioffvX6KfDDRPtZVq6Bibz9j2y5V00BzRQxhDIOYRoy0rj/BuLzlPOYN5p6XdTIn3Sw3J+2DM3VE75Lj+44XSYQsf0TSYenz9vSVYPlVsS3P3ntSS9pxHnZZlDZfWXVJckfP9z3Vi0oqKioqJi80OeYC/ihN7moZbP9+KT8nh/wh5KnsvGipz2/GlbIJHTSgYjEgKHZKNi0WkIbSozqh2HXFMsVLq2MejEO05tGiMRCqdxMA+LegBCG7skPpT8BThNGrAqIKLajpjHuvVhJ6yciA+R8soulZkjHRoL5NiuiBGN8FcrPiTS3/JkQUcpUnLOw3mfNnJN6R6IaAFyEJsVSsQ+ERUbY3b71u6LjwzfjHQNgNYliqd+1BUEMUj9YqQ0aRM5wux3TAXuXAOiiDaotQnJqs20ooGBGAhtK0S7PGu2FtjGXlLXTKI7+NSnjMY36i3PADlV2wOIqi5yAJjg4HSVckibhEYQvHNZgKIqfSJKN9JugUboOiHFIMfwXsYPIxCiF9sc1g1Wmb1e3ERfqcoLopLoFRUVFRUVmyimqc43xs5liESej4RP1yv0Ih0CvLdBUkl4w85My/uKa+o/6VhxjsVUFkOX6U5VLTnfdAKTwBg1XpZtxgCOLZyfGRQnLIUH/X0j0IHJAcrGl2ObfBbl5VsyGbQyxKJFs+nYRwJl6ubjpFLnVFRSMfXyiQrKjlBKKwel8ynQ+6R5ScZ77ztEejmxUw527dmvdi4VFRUVFRWbJ8p4OXK2HSnj4xhDih+cc+nVV6wTkW7EKQpwKYIgHLXYfUgMArStWHVEJdzLVXNlDJPV4QzmrDovifOyDgDEqiWdO7mJe3/MEDobc06S83ZO7pPsRU4ohRi28hC6p3smcbvijW595X1SfGN1KGPDyBGkBHBaTUlQBXXeOynHu0pGu0wWU0MAMYhVoe4doGR8iKwbukJJXnknInhP+pkRItAwQOQhkxSk/uMu3f92rN7y8ACN4BzJZqW6yWhM9yYCZJuXQjYg9ZQmKlp9pkCMJspEQquMtWzKiTQJ4IiUSDfLHk4WQI6cWhs6jcLlnoUQRRQugnX101cqnGWSQJ7JkCYX0u3UiQxYmt0X3SjV+Qg/iiJ8d/K7ENuIttV4nAHHAJF4vDMDxOqZvwAqiV5RUVFRUbGJok+WL1aJvhibFwvo5yPhgUyiDyk+JtXoPEiMl/m77/30tPJyMm+h6OiXOwmGd4RR4zE3F8QbvREfySHcfg58aWrvOxPWn7e/HNsUaXLA0leG53TNbcoTdd8kFHkpp6NIz8+CqV9yPhkaaFlEqgaatHOxegx5pQOF53lxnqllyt+TEuXxaulSUVFRUVGx+cEsU1I8QB7OCTFZriIlpkR0D03OW1lG8uYYyAhWhqltOwpzU/qmn4EsW0CvbIBZyXFJ7FiipHiGxYIjEODgUKqk2et4wEQwMjAo6oWCNDahAVSp7cWWw6crZiJViV3t1Uxup3YUcZjGgYZ+nEZcjBuoR6Qyi9o7RlBkxGgrCkX5bHXrjIMcdeJnU23bJptZ8OPAoU33OavyBaUK3dokfaT1j9of7FUoHwuneobzTsQm7MAjBsijbcdo2xZtGCOEFkCE90AzIrjG581BQ0jqpcA+EdshsG5YJC+7nc4RfJOV6ASHAMAHQnTi+27PWrbQEYW4KdTF3iYClAUnzIzQtmjgwI1N9EQw8rOY9lsyJTqx7EHkhERnD0RV2ZeTO/bRkaxZXoy+pZLoFRUVFRUVmyjmU5/PpyJfLLlun/vvfaI8bQY0zzn5XHTSFnO9ofdumg4haJK0HCKvS2Wx9x4hBIzHY8zMzAySrHdXAvzORp8wLtG/Lzlrt/+GyjASvW/RMmSd0h8UzZcX0BWlnO/jkPJ8Ia90oKsq75Pr5YCo/D2ar78qKioqKioqNj1YXGC2K0SEpvHJCqNpmhxHQEj1tm07ccagclt/NLV6TjeyWhXPIegmjEamM6LWRby3S7GJK5TXmCB4JZ/ZxJhwhsGu9yqENyluJgKTKIC7q/i61jV2ni8IfrteYFOfZ1KaC6VzZ5kjT6rLU+yoqvLUnvJ+FVMGQqJHtU2RczxxEpwkGxetM7OQw6TFi62KEL3MDPsv6qQKx4igG8yS9g1HRqSonuZyH4XbjnpfI2LUiQO2/tIJhEiI3BbqalF1M7eI3CKEMWJshTz3BOcZjRf+mhkIhQBpxE5X6IoFUCSLnYsJAScTCl4nSEzUFCAKeuc4eaOLat3ESUKoy2rNUlhE+RmNslkq84zdTnm2WfrA7m9Ucp1cBLkITx6RGEziIy/iGO1blmkVjlCCP7dlPlQSvaKioqKiYhOFBSELKdEt73zHh1Tn5TWmpZd+6AsT44DTzXpgG/0AkE1gtCxLTwGRRK72buqOTMZbLboK6HxsqN/kGuQIjXcScMaA0M7J7vSOEHQ5rBsg5ufD7xttWgbHC+ddPDEseYfuyeT17LlIyh3LQ/lsU92UiqMiVclyHWwVqYHd1OdgsbBBXqkU6ivYh4j4ioqKioqKis0LeTUbgVlIc984VWGLgKNpGoQQQJxJcdvIs08Cl2V636C72k9jVFZCVy1DQjCCVby0xTc7gpzEtWLVIS9G3lh0mkWLIUDY21JNb+pqUtLUe2cngokTsV+u8rMxQrm/kktxXt78kyzWK4QNMUawkriiUudMjPdI0tKWpUwT/3CJEzkKESubgjoVSJuYAqo4d/Ce0DSjHO8BCIggBjwIbWTZCFPV29HIcxZrEZnscGLNwkDTOLHoMYW9Z51s8UL6JtW+tpMDnGsKy0BWK0SdKHAMooDIY51oiAACnAecN8uYFuS8PgcMD/NKl+NpjoVY8qRVoNou5DGbU2GUc4RAQvlHTuuC1WLGyGxR+nesGVlshGIgHS+g8zuQFjNACfYoli9tK6r+EGQCKT2fpoB3UreYNr4lLYPTxMNCqCR6RUVFRUXFJor5yPPyODBMolseO15+Ls8xTCPRabF5iVUxjl768PW4twkp7IpDavaB/plGZBKsD8T4gzliPB5jPB7LYMc5RAodcnTTxSTRfUeVa0qZxVxPp0nSOTagywNIObVbHqUBUl7im9NNxRILdZOpkkq/8/4LQIcgt3PL80sFvA0iyzJKkr2ioqKioqJi80Db2uadZj+BpNRuGp/ig6ZphOhjnoifJ4hsOCEvmZIS2+IkUU0HIIiVSOSg1iSmTGclKT0cyQrM0WiULD2EBLe62qaepVCgiI9U1S2EKsM8vSVepk4aQzZCFSV1Dt6ahtRnXMltEuFM5HIfGukTI+SZW3DU+N05EJwQyC4TsHaexYKdMYaqoGOIskFnkF6Vtkh5BJkcIO/gPVQxnSdEMhlbWoXEJOZxHhCv8wAxAo/JVlBeXvvG6zl5A1VZRSBWI0aNSyzJMAI4xggXI7xvdHwnxD6cEyI7iDe+b4V0N9sU5hYxOgQiUDOSsZhODDQjmXwJkdRn3IhvnZRJtZT75fUek94zIgAR8J4xsomezliSkLzNdW4lcEztNmI7Ru6O42xCiGN+hagkekA7jggtEGGe83mcGUMQL36WNkaOsM2WbOPUhVBJ9IqKioqKik0UG2vdspgNR+3YQu9Gok+kD+aNMF58obyAqDOGFPGTQbIFX73AyNTtmEhORLz3Ho1tGBkjYgiy87zP/uhVVbxxEK/Lxead9I23gUlnSW45yTKYnlcqMJMW4qaWUQ6GSoK874M+jWwf+txfhl1RUVFRUVGxeSD//Rficzwew0XCaNSAqElqW+dMldy1hcve2IVHOtmqTALYgWE+57aBqBHqYu0hJPqkPYvER0ocp5gIuvGkxUfdWIxV4WvKbKlel4ws7WVK0UtgDMZFTm1ghHwX8tOr6pgcgRIxagS5h20zwxFoo4NL7Tdim6fyo6KEVqsbJdFJjNwLYQ0JAa5WK84RmkZJf8qkrzXPVk4CnNTozsXUV2KtEhG5u8KgFCzFmG1w0iqAos/K64YQQGQq9kY2+HQOTGJt4kBwkeAbQlDLcedkhCQWOqa0V/scLx7o0eskj/aBTFwQYogAR7HUYR1nEcAwmyKZeGCKYnnTxKRQ17us90+JdL05jmWagNmBQMk2JrRtuhfSv1EmhUKLSIwQhUAfz40xN9ciBoDJA2TjNQbHVjdfzRNAsodq+WAsLHCpJHpFRUVFRcUmiiES3TCNHF804a7LHyVoMrUGgJSefzYlOnrvw0S6BaAlQW6fbamgtg+WL13WVuuhtH0pr18i1bdMs/cgga4jAjUNoP6VsW3BRGiaRoLaJdyPWNbx9wbcCy7vqLyYEqcO944pqoby2DHqKNG5l54HNul8AohNX1RsNCrrczNBHmM6Ply36Yr1aSS6+D7WjUUrKioqKio2R5iNSQgBDYvNSZ8sj0pQljYqQ7GDxc9RsosBXk/RG9XuQn7O5LIR29mWzmKVclWm1MeI1bINUGuOfpw9LRZKx1XZzlx6omdVvhG5QiQzSDhttVGU65l9h4kyrO0mppG+cul6Np7orjiM4MBJycy6oSlYPOnzeEXsVyiaGEPK987Kjqo0l/xRiXmQkOjMgG8AswFnFpvBqH3YHx+V/VCmcYx6n41Ep/QcOed1VYLmBwHk0kRI0ziE4CE08AjOq6WLkxjZe5/IcsDGck4mMpR0bxpVtrfirR5CK+0mIb4BuR4p+e1ZxmKeoWXZcyAKf1O1cxR/eGIHYkJUv3OwjJ+ccyDrE1X+xxgRKIBVYT4et5gbt5jb0IoS3Ut9ZFVC2c8Wt2ssTlBl+uJGaJVEr6ioqKio2ERB5IqlkBpIAUIeTiXGCwLdlNo5GpI3XapZqswT4azlA+ioyo1sTz8Xafnd/KmnKcy7P5d6GDJ1cUHuk9XZ8g3wuxPkeortsmcjEaGZmcF43ToEZlCMGOmGozKgWJwtR6rrYnhmxhTK9s7GgFqcMYVQHlaWT1NdD+aFGbf0y0CatEhEtxXCEP9LZg3ipS5ZXW5qE1PuaHl6tZiubcoh6KZKGtg7C7Ypld99SW/0X+SyBynISHjdCopoEVqXioqKioqKik0JbdsmQlQghGvbBhDNwVTO43GLxlEnZgfQOzfD4pFMgguB61IMg3Q9ZtlYtE/Ml6rn8me5bkyENoCOBUvRlF6dJlcRWro2DNlyJpOaoj53+rOWz0HaEq0/OPlfywrRkZKxQaxflEwWqxVR59umm1aHUtgQA2cCXUcOaUUtlfchizdks0tC4wmFO0+KC2NkMIKeIx7nEeJPnvpXVdicNhTNpL3lyT7nUSdfpE+c+atHVoV6RIwtmHWyQ2Ne8tYXDKBB0xCahoRQJ+ljAtDOiSe63ZO8MgGpL1kV7jMzABARQrZMFJ9zD++8bFRKANiBYisrAzyKZ1FtdyKL+h9AYMD5BiCHyITYAsGJ5VAbgxL5Lm2AmyaXHCOw2rmMAzZs2IC5loHowCS2LqPGgagRixsEhNAWnvm2SmFxQqBKoldUVFRUVGyqSIGPS0FQCgr15Upy3ILFFGT2SO9ErDM8MgmalOEl8W3kOLIaIR0r0rrp3LmmvVswP1GGpds586jep3ZRP2AaYLqJZKd5571sAhQjQowgp9scLYoVX2xoVmS+e7Dok7ij6jbUIczz9mc6UuSxp8be40Qlcw4b+Ng7IEtsbY9TI7oZSGkx6s9gRDtWvKKR9b30NJ4iUlJeVGYOqqpZfE9VVFRUVFRUbAJo21jEpaLEDi0jhhbjuQByBO88vA/wXib1zQMaJHYXgIRLgWNSZluMIeS4WYRA83bTZIJfrC28F4tC2UwUkMjYiT+6E/l3jKETiw+R48wsXtNFnOScRObeNYCjFDMJWS0Es6jJoyqRxaaFY7ZxISWJSQn0QDGNVyIY8E6IXxJVNzvpI+agMZgSpEbcl/If1s1VGUJwBwICgaKq10nsRGQfJE7OIJH1JM3PIJBXYlcVEhxFqS2KcvMh1xqwjZU8iBzgIshHeI1BCdnnPDBLvfTeMEQWziC0umlrjFH6W61QKC1JiJjBHGDjQFXzc+PQNl4338zPU7OiQeSItg1JIa+tgSNG0+j9CAGjkQNHB44NuJFVneQIjSM0Xjen1bg6JoU8iudS7oVsbMugkMd1MJ99AmIjz0tLQGyBZkTwjRDtgRmBA7iNGLct2tCiDbKHVYQHWJ6rVhX8M43a1MSAiIjGA01jm6LapMnC0Xkl0Stw1FFH4bLLLsNVV111V1eloqKiouIOhOupJroENjrLL7uq9C7xnM9XJQIwlaweIq6H0svlqNPO6actlHfo+HwE+jQMW78QZmdnsX79ehAR2rbF8uXL0bbjastxB2KawmopcMiK9dLrfPh6MohKJHw5aQMbBOoSZ+Y8cUPUUXDNV2fnXCevWbnUjUUrKioqKio2L4Q2qHhFYs1s0aHKYyI4r5s7ulY2ayxiWu99jnFBYCfbO0Y2mxbz5lZbkVBsdmmiF0diy+EZjR+pH3tWowOFXzoZCd+NecoN2K0dMHW5rSkk+xmFSttEBqxe3lH8q5m0HVllQAEgJc15rMr6JP4REtaEO2ZxAojPtdTfGp03d5cGSUdEBkIEYnDgwODg4QIV9jAEdhb3iQiinNAQ5t0hguHhpR56XUrKcmu3KMiFQLcNVxnkdEzmSWdM9N5GQpgzMt7awQCPwNSAOaJVGxfbiNbI8rzs2DZyjYnEdo2Dcx5NANrW9vRRex+IJzxI7FWsf2UbKJloCSEgtKLcJ4pwTiwPxTiF4R3Be0KjSwgiAzNuBE6bjuY+iZERnN4vT9q3uvktGI5U3MIAO4cYPLZcMYtly0ZwXuTzDNlYlDkm1f9o1MCBEB0hspN9XHXyRsRgEY4ifEOYGUm3OxIRmaO859U0VBJ9CfjBD36A008/HWvWrMG1116LlStX4oADDsDrXvc6PPjBD76rq1dRUVFRUdFBf3lmn8zuvyy93NRmKK+H6Uy6BPm08wzzkezyjom06XnnPz4tbx/Tjg2lz8zMYDweA0DylVyo/IqlYdrS3yVBFUr9zUL73pw2ALMl0P289jkNHFW+ZceGNhS1vGUZJWlubaue6BUVFRUVFZsfshJdSEU24jDmdCEnCYwxvKcUxzvnEBvAu0ykI3JSLptFi8UoEuPEFH8LYSs+116ZQO9H8IWFB6Ce28UGpv3YzMrrx07CZ5sndh43WNv68V2MhSpdmfZEdBtBHkjt+KIQ207sbsxeBXCJFGcuxxPFJAVn+xUq+52BEIU45gBZJhidiIXIdipFUkbHqCpv5eE5xLRSQC5qdjqiqiZmtQ8hTY9w3oRMMlECNouSptiAVW1fKG9Cmmx6YovA+We5h9mHvFgnqW1keN0k1TmPvGms+JubBZA8Q1EbJs+NzXyYdY/U16mli5DQrKsprL3OQTc11TFllHtmKwLyM8BgcvBgBIjqnxmIpIaRzIDXsSYTaERwPMKKFcswM/LwFOUOM4viP46FHHeM2ZkGkQgRHpF01MoyN+McI4QGsWE4L3nJAc47+OIezIdKoi8Sn/rUp/Dc5z4X97znPXH00Udjl112wVVXXYUPfehD+MQnPoF/+Id/wMEHH3xXV3Oj8IEPfKCqoSoqKio2QTgNaEFIG8XIz9njj1zxWYNOl4KcYZV68j5HzicB4ZAKPPtkL05xnpUPlj7NzmVaGWWwv7EE+jQ1ekmkj8djNI2f9xoVdx7KgVwmx/PkTJcstzRTG+U8OS/SMlxTTkHTda+o9NkM1U0Fj3R+lzTPA9tsJ1NRUVFRUVGxeaBtVaddhL2JEFUiNu28TgExOiUmXdrbJboI2/ARAJhDb6PQPNnvnFfPcKee0oTIrZCKzsH7BswQj+g2aAyc1bimrC69uYFpRHr3mGFiU1HmRCpHzsQztAtMlU+Ug7g8LrHNK3OsF0K2JMmEOSdFPqtCn6zDIer6wLK5awyi/CYWFXqp6mEN6JgZLrIo1gEQRzlGjKZpRO1MBWkfWZzYI0FMVqJOBDhV2stkBiHvN5VEIKKv1lUHDNmwU4lzkFq45HvcWdWgkwUycQFEyMSB+cF3hU4WO0e5op4XQlSP+hZ5LOhgfv3OBYxGXsZ+Hkl5zzHCEcRz3Ts4IrXpsXkOvcksm4iymWoSxO6HhUyXPbw41c9BLH48NZidITiMwWEDmDwiAaEdI8YNALfwjjE7gijfyYF8A+dGepGIGMcIwSOyEO4zMx7kbJNYD+cWpsgrib4I/PSnP8Wf/umf4gEPeAC++tWvYvvtt0/HXv7yl2OfffbBEUccgSuuuAK77LLLnVq3tWvXYsWKFberjNFodAfVpqKioqLi7oS0sSdEOa6Utlqx6AsAymNk+csYUpdPpnCnDJT1fDs+8d4lyDNZj05aLnM+gr1IyxdfkFRfKpE+RKLbAGBmZiZtCjUejzEzM4OsnBkm/TcGJfl7d8Mdohb/HaCsF6fniYtjKWd5Fkq10vzNkuCfCHBsy6VVlWRluvzRCPqh5+nu2H8VFRUVFRUVv1uIDUbXzkX0jK6IH4CoG2lSymObbIrYxfssFuAYlZA2yxUpR9TrYgEzGjVwXq4bNG7xvoH3jXhgxxyfmwe6EbWmXjbMFwfKeaVAIYt0jEs30lyU6Pkz2FbpUX6pMt05n9okZZQiiWzXImXFpKpO9WBScj6rq2M0KxASAl99vcswMU8A2AaenKxaOAIIYmmCSKlcmyNhOPH7jhDlBYTwdyk+tPto9ZJEu69SIyHQxcaGQY3s49O2eWPMybjWlOjSZsdGqlMxEdKf2ECaeBAiPaT7BRC8IzjdEEtWMoiNC0hXZQYRkVg+rxuLslmHchQ/dw4ApF4xREQj8m0GBaJaJwK8czqWJXCQ1RkOARznEIKHdyMQGG07hxDmAB7DocWyWY9IOtnikJ77GOX3j52Q60SMUdOIoIxkksjRwkr0uo50EXjrW9+KtWvX4uyzz+4Q6ACw3Xbb4f3vfz9uvfVWvPWtb03pRx11FHbeeeeJst74xjcOfuGce+652HvvvbF8+XLc8573xOGHH45f/OIXnTz7778/HvKQh+Dyyy/HvvvuixUrVuB1r3sdjjzySGy33XZJGVfiyU9+MvbYY49529ev61VXXQUiwtve9ja85z3vwQMe8ACsWLECT37yk/GLX/wCzIxTTjkFO+64I5YvX46DDz4Y119/fafMz372szjwwANx3/veF7Ozs1i1ahVOOeWUzvJ3g11j+fLl+MM//EN87Wtfw/7774/999+/k2/Dhg046aSTsOuuu2J2dhb3v//98ZrXvAYbNmyYt30VFRUVmyu892iaJr2XL9lEyNQpPilM+q8yn3OmQKCJfF21+rBNTJ9MnMw3P9ndec1zbctfvk8ra9q1+ijbOjMzA1b1yng8TgrjhcpdChbqz98nLKXO09pc3udp5adjsufS4Mt5GSDZOxWrJ0o1z3z1KJdVl78j5cv5yXxD51ZUVFRUVFRsPshq6cmXEc0xsvhOq+I4BnmFNsompEZ2BiCqajgUm0z24ZzrxM02LhiNRmgaj6bJanWn3tUhtGjbsZKpXdu60rpuUnRSqLGVhKW0+Slp3bPCnXly35rumKAbK4ltDee+iVyk6ec2FP1k1wCYXSaLA4lKnMv6m9CIeimZmjZVO0clgWNE27b6Eo/yEKLY1EQhzdtW7hUYCCGibeUVizrbxILUT8neSKmPAKcTCVl4JBMkTu9fJuat+mI10713NnaZm5vDeCybcI7nxmjHY1Wgx6JPI8bjVvMFbV+2DIIq7O0lsTYXL1GqO47JM122RJWfvQMaJwR9oy/nhIhvvEPjSVTtDeSzZzgXwHGMGOcQwxzaMEaIY0R9EQUsW9ZgdsZh1ADeRTiM4anFyMvmosuWeWyxfBZbbbEMy5aNsGymweyMx0zjMfLVzuUOwec+9znsvPPO2GeffQaP77vvvth5553xuc99DmedddaSy3/Tm96Ev/zLv8Rhhx2GY445Br/97W/xrne9C/vuuy++973v4R73uEfKe9111+FpT3saDj/8cBxxxBG4973vjS222ALnnHMOvvjFL+IZz3hGyvu///u/uPTSS3HSSSctuU4AcN5552Fubg5//ud/juuvvx5/8zd/g8MOOwwHHHAALrvsMpxwwgn4yU9+gne961149atfjb/7u79L537kIx/BlltuieOPPx5bbrklLr30UvzVX/0Vbr755s5kw3vf+14cd9xx2GefffDKV74SV111FZ71rGdh2223xY477pjyxRjxzGc+E1//+tfxwhe+EA960IPw7//+7zjjjDPw4x//GJ/5zGc2qo0VFRUVmzIa75Jie5jEzu/lZ6exazfdbFYsuDTSOCt5++V0z+u+C/rKdVNgTyrPy8/Tjsv5XYVPqfSZVsYQho5b2UaUymY+4mPpZdedjVahT88/3L7bg+mbbA63eTH5FrreYsuerx6m8l+U2p8mzxmoWcqbrzH5OeVVkYwsn9VBFGd1er8CznU3Hy09QauNXkVFRUVFxeYHExUaoZ2tN3LsmojqkP3Im6YpJuLF2zrblpgdR+hM1kt8omIZjV+ZGU5tOaQM9Yt2HsyhILYpeWHbRpjyeTIGa9sWzJxEN5kEl2tYOd24jxCDkfHqa05qcVKQ6KJiJoznQuqjPBYpfOR1LxxR5dvmqJzaJNvOoyCrudBw55W2KW7TDVzJETj9FxFU9e8gavQIRowOaAPIOZBazbBUAAzx7jYrHSACFOFaII4YMzyCZyWclfQOodwUtXwHwCH1td2LbOND6dlxzgmRDxYFvm5CyiyEv/StV1FQRGgjxqFN95LIIYQ5hBDhyIMgExXeO4xGI6xbt1YnWiS/kOJeV0sEhMgg5/OzBuhqAJIVE8wgll1VvXMywaFtJTLxi4wbiQA0ohInB0RuMW7XI2IMx+KqHmJEiEFEMiB40s1mk2e+rHRgYjjHaDzBNwTvbPymz8gilOiVRF8AN910E371q18t6He+11574aKLLsItt9yCrbbaatHlX3311TjppJNw6qmn4nWve11KP+SQQ/CIRzwCZ511Vif9f//3f/G+970PL3rRi1JajBE77rgjzj333A6J/vd///eIMeKII45YdH1K/PKXv8SVV16JbbbZBoB84Z9++ulYt24dvvOd78gOwAB++9vf4rzzzsN73/tezM7OAgDOP/98LF++PJX14he/GC9+8Ytx1lln4dRTT8Xs7Czm5ubwl3/5l3jUox6FSy+9NJW311574aijjuqQ6Oeffz6+/OUv45/+6Z/wuMc9LqU/5CEPwYtf/GJ84xvfwB//8R9vVDsrKioqNlWYt/l8Sm37XKaXG4QOK7uHieq+ircsf77P3TS3+PMG2jzt2vMR70NlzAf7ezU3N4cQAtq2xezsbFJ63HFq8emq69tVKk2qfubLuxTcEaT7fPWYnxTX42TEdkm6pxoiq3jEJ5OTpygm8ptiicjJkljKijEZKKCTN78YrAOpckADdP3RKyoqKioqKjYfyCS82Y+oeMWXemdT9mbi2VaCeu/SJpLZwiRbuZSWJiXKeENiH7XgiGKrEVp5mSo8KaMjTYSiRtoaIT8kakibeBIhhNCJO9N7tHisjNNJ+4bU8qT07S6FMlmsY0KGrgKe0ucYKTuXJA0PmTZieDDRafBQnlxvqH0MiGRzTbKxjGy0mTYUdYToACH+ATSUFPREotIO2iemPLc2JcW6epcTKX3PKlByQON8wblHhNiKvYr2TSRbqWD96FIfxlCURR6mu/euASGCyKd7YGp1U9SPxyomIgI8gRop1OVuRuTkfi5e5ewA80q3GLr4HImhoTxYVetQP/zAQbzrA8u8CwDmFmPdcBVE8J4gFvWUJolg9jtgMInnPDGpy46S58XkzXyo60gXwC233AIACxLjdtzyLxaf+tSnEGPEYYcdhmuvvTa97nOf+2C33XbDmjVrOvlnZ2fxZ3/2Z5005xye//znJxLfcN555+GP//iPN9qn/dBDD00EOgD80R/9EQDgiCOOSASCpc/NzeGXv/xlSisJ9FtuuQXXXnst9tlnH6xduxY//OEPAQDf+c53cN111+HYY4/tlPf85z8f2267bacuF154IR70oAfhgQ98YKefDjjgAACY6KeKioqKimECvW8vUVq1TPvcT/M9q4rFWFZMq8s0gv/2vIbKuCPKtXKcc8kSB4Aq0rPi+I64zu/ydUf1xV39mnqf04qJ8sUwNUtStQBpMyGi6RZF047PZ2eUrY+6li9D9a2oqKioqKjYPGCbOJJjNCPCsmUzWL58BrPLRpiZbTCaaTAz22BmdoSZmRFmZhqMRl5ib2+WirIZZFoJV5DqUZXSmVCWtDa0KV6NEQgtMB5HjOeCWHWMNT0AMRjRLj7tfbK8r4KWdmmeZJ0ihGssLUJCYU8TjWymRLACtgloTLYhQtiGpI6Xa2WRg22CWZK7Vn6MZr/Cnesmi5R5tAyJ7EeemGAlcqXDjXB1qR1c2PFYfe1aVk8jsTlSqlsbAubmxtiwYQ4bNsxhPG57G3xa7zhRhROle5xEIUVj5L4EtO0YY7VtEQ/1PNGSNxENxbmkYz1RkIvtzyj51IcQUz3LyZZk/5Ksh/SetBFtiJIeGdH6w/pKX9E80pkQQAiqWA8MBAbayAgcETiijQFzocU4tJgL8nlDGzAOAQEMJtLNSZFWD8iLAWLYSmuGWCC1UcqNOlvCiyDRqxJ9ASyWHL/llltARNhuu+2WVP6VV14JZsZuu+02eLy/6ef97nc/3cSsi9WrV+Mtb3kLPv3pT2P16tX40Y9+hMsvvxzve9/7llSfEn/wB3/Q+dkI9fvf//6D6TfccENK+8EPfoA3vOENuPTSS3HzzTd38t90000ARIUPALvuumvneNM0E37yV155Jf7rv/5rwpPecM011yymSRUVFRWbFby3pZTlK6s3ht6FACx/BmxpXc5L9r8GI1Tk6b8DtrFPeczQrwNilyAtMZQ+LdRZShnD50+kpAEDkIl0C2LH4zFGo1FS5iwd/XOmtmwjyu6VULRj/jrcMde7I1FatEz1FHcAdCBDhPQ+FYRiqae9I0todFBIgAbZ3ZPtGiwrRdPyYHKuUKnbpmAu1b8q0SsqKioqKjYvkGN4EEajBrOzMxiNRrB9dmxTTtlsMsI3WYhi8UMScESLR4WALGO70uIjjQOCEO1D+cQ/PCoBDDCLSlmuJWbepZjAzjVSvlSXBzaLFqkbu0ySlzFR6g/lo4mVUCfboV0IWfvsdIPKckPRvk1Mh6BOaTFdl4qYVrqGpijNe/UkYF7G3cxgOA+y2JTUTKnfU3ipSvkQIhhjoI0AxjpOcyr2IFkJABOMeK1u6MTC5X3sizSYI2II8L4rOskrGKyhWSQkeSLamFcdW7/GGNI9t7GfWQs5FpJfbIAAcAQHwDm1D1IrFjmPlUBHmnhgiP1KhGycygxQjKJKpwhiwCHAs6jZ2TkQh3RrImVSvo1BFPC2aS1najwJagAQy8a2YqMjCnzmyT0c+6gk+gLYZpttcN/73hdXXHHFvPmuuOIK7LjjjongnjY472+saQ/gxRdfnL4QS2y55Zadn0uFd4kHP/jB2HvvvXHuuedi9erVOPfcczEzM4PDDjts3nrPh6H6zJduv8Q33ngj9ttvP2y99db467/+a6xatQrLli3Dd7/7XZxwwgkb5QMaY8RDH/pQvOMd7xg83if2KyoqKipkV3OYMrZUvpKGe+U7DSvGLa37DkD9AykXiNLuXIITgBMpX55r6KdRUtR0Qt3yZyrPHEa3nt20+dK7teoHywxHlL0TidB4D44eY1V3WHnDft0pVJy4Hg9dLkX1/fpN8xcvC6EF8g55eE8u/90YAn3aEKN/PSG2h8ofqke3nKFzJ8lyV6TZXbMLc3fQpHmYKb3LI61BPjlddixPIff/s0ER+ry7Bu1OgnQq7F+SCqqioqKioqJis8HMbAPnCKPRDGZmmkSuuiTCMIGAqc0jkq+5ktiyeaXmItuDJccUpjA23/O2bRE5IgSXiGOzqeNiQ0uoEhwMEBqxjnEB5GJahQp0BQ2ZUDVSO6Y69dEVcRih29UyuLQxk6iHQ1Dv8RhgfuPdMQpy8JeU8Fy8StLegr+CfCeALB7j/JbamCK8IoPW3wJ4iwOtnoD0LXRVABB1BUGOX5lJN4MFGC1AQX3vpdyoZeeJA7nhUX29yTmxGITGmErA2/mWDma993af8oRJGq9AvcmbBo4cWg5CXJOJQfLqBtkLitPz4l0jbY9R+1Y3b2Uh8UOQDUTJMRwTnOP8DOo4h0mV6kaka1zuSq+XwCAXMQKhcXJuiKJjF4sjD4q20kLq17J+tj2MSDzdHROcTqJIXC91DjHKBrALoJLoi8BBBx2E97///fj617/e8eM2fO1rX8NVV12F448/PqVtu+22uPHGGyfymvrasGrVKjAzdtllF+y+++63q56rV6/G8ccfj1//+tc4//zzceCBB07YotwZuOyyy3DdddfhU5/6FPbdd9+U/vOf/7yTb6eddgIA/OQnP8HjH//4lN62La666irstddeKW3VqlX4t3/7NzzhCU9YUD1YUVFRUSGwANbUIPJ50kqCiIC0kU+XNB9MU287Kkj4FJSmoMjKJdA0xfAAaCDotjZkDJHU8+edzD+9DAmnhiCe2OmzJ4Ab3WRIFDm2xHWgVgAinJn8DR2egJ/CSg8R8Tx5nIaLZSORF6xDDpIXA8YQOT+M6flo4lhftZ3HSt1GUDnL0kuTcRLr1A8XHLoMoPqqdRsQ2Wc4Smk2oIqJRI9KoFPxWesN8blk4nRu+V9FRUVFRUXF5oPRSMjo0YjgvKhsAcADGnRktUgIEkXI5owOznkAYqvh3EhJUsBRo2pjIAsoCMwOIdjEvcM4WKxGAEIiU4lIiVSLDwsLO0/wPosyup7rTsUhEiyZ8l2unVXjWVzSHVsIsQuQV7KWi+iMKfumEzS+MgEPiTOHszGNAyDkspDWaq/SiR85lZGFOZTt59nBsYf0BIG4WE0oTRW3eooACeGr3Q0OWhfrnxjRBiGbCUogE0DMwFisfMgRnNeaEuBcgxgb2CRKTIEogKQaV9LfOe150napx3m6D4TIAQyC8w1ADhEOxNDJF7GgSTY02vdEgPNAQw0YEeO5MbJlTFRRim56y1n5bhZDUloERye+5fostW2AdwRqPJhIJy30OSCoL3kExzECAzGo8IRM5MKgCFG7+waeRpAVovL7wiGCPODICRFvCn4WEYzZzYCc/qZln3QHks1jnd27hWPz6om+CLz61a/GihUr8KIXvQjXXXdd59j111+PF7/4xdh6661x3HHHpfRVq1bhpptu6ijYf/3rX+PTn/505/xDDjkE3nucfPLJE4NBZp643nx47nOfCyLCy1/+cvzsZz/b6A1Fby9MqV62Z25uDmeddVYn3yMf+UisXLkSH/jAB9IOwYB4uZfWMABw2GGH4Ze//CU+8IEPTFxv3bp1uO222+7IJlRUVFRsEiiXXpafs7e5T68hj/O+Kj0FS+mlecnBEXXKS6+7iUf4tPYM9pv903vRQJpvfLJea9tWlT/T2rzUvnASlG/si5aYPl85i63vEtq3lOd4aeVOvrr1G1IyLfI50pc9IrbCwzn5HUi/P+mcsuzJ61RUVFRUVFRsPhjNOIxmHJqRQ9MQRiOH0chjNOMxGhGaEeAbIS1nZ0dYpl7pvpE4wigWRw0AjxgcwB6OslIdUCuLyAgt0I4ZMRDaltCOGa15oRd+6OYlnhxMirhFCHElIqHxPxy8E/9sgnmBiwVJjIQYoB7rsgFl9mPviiJkw1SC94DzDOfZWNUU0xFsEiETtuRUmsBiwSHkuVjPTEXardLI9AhEAqKDi0LIEjsQ+7S0sEOkk/lr21Wk762OUHW6+IMDIYqfdxuANrBsxBlajENEMNU0MZgcIntE9gjRIUSHGEnOaxnjltEGfW/Lewq1fHHa1yye9jGq+ppAzkNIcw+Gl3oFQowODA9yM3B+BHJNUqc7J7bSo5kRnG5kK8+D2Oo0DWFm5DBqCKMRMDMizM46zMw6fZ7lntr4kHVlRGgDYivKcHtm8vLNiBhbxDCHtp1DO55DCC3aNiCMg7QnMDhAV0uQTKRwTIp7IkpCLsmvvvSREKNHCA5t6zAeE9qxQxg3iK0X///oQZHgFkGRVyX6IrDrrrvinHPOwXOf+1w89KEPxdFHH41ddtkFV111FT70oQ/hhhtuwAUXXNDZwPPwww/HCSecgP/7f/8vXvayl2Ht2rV473vfi9133x3f/e53U75Vq1bh1FNPxWtf+1pcddVVeNaznoWtttoKP//5z/HpT38aL3zhC/HqV796UfXcfvvt8dSnPhUXXngh7nGPe+DAAw+8w/tiMfjjP/5jbLvttjjyyCPxspe9DESEj33sYxOTBDMzM3jjG9+IP//zP8cBBxyAww47DFdddRU+8pGPYNWqVZ3B5Z/+6Z/i4x//OF784hdjzZo1eOxjH4sQAn74wx/i4x//OL74xS/ikY985J3d1IqKioq7NcpNPe1zf7NPey83UinT+2mAKSBUDZEIRSUFC3U6QdS5QwFtJjuH0vuYot7GkKJZgrzbn3eayr2b7kh8LYNu2BRCi9Go0WsNtVuuu3gsLu/vjpCd3veD9Vhk3tJmZfH1WEw69d7Law7b2yz2eJnHOdfZTNbSzfccADhm79Fpr4qKioqKiorNB7Z/jrxyTA6giA8ivAe8F7ou+6V3X2LFonGaigRMTSt0r1hsEJlC2VbJCdls+8uw66/4KwUGqgJXr3Ore9/KxRDV1kP2JBVCPXl8M+CcbBLaF7hwsmEBHAm3bbEVUVax5/qJh3VeNSmkce7HoRWLlCxcksy7JMntPqREXTWo6nImHdewbaZq8SwhcFZplxt4sqaHKBMjjKA2lw7kRB0exVCk059dX/tyVaa8RAzFabNZs2rM/ZQJ5X6ZMTJcZIj2tVjlwC1C0Hrqs+GdQ6Rcj6YRC6LGSRxPzlTwgJMOQmghm3xGqL2NPbPSN03TTNTL2mbPf9RnW4T48nxHYjgaywSOH8GRs0e/Y2sDyD4BIbDuHaCrI2zzU5J9w7xZB8GDyOtEyMKoJPoi8exnPxvf/e53cfrpp+ODH/wgrrnmGsQYsWzZMlx++eV48IMf3Mm/cuVKfPrTn8bxxx+P17zmNdhll11w+umn48orr+yQ6ABw4oknYvfdd8cZZ5yBk08+GYB4fD/5yU/GM5/5zCXVc/Xq1fj85z+Pww47DLOzs7ev0RuJlStX4vOf/zxe9apX4Q1veAO23XZbHHHEEXjCE56ApzzlKZ28xx13HJgZb3/72/HqV78aD3vYw3DRRRfhZS97GZYtW5byOefwmc98BmeccQbOOeccfPrTn8aKFSvwgAc8AC9/+ctvtxVORUVFxaaIkiwfUsB2guTClsWIXl0k2E0jCW7dQBkZXKTLeRN1s0ivhyEScxrXuFiiO9VgWvrtKNcRwI7QNF6DPiHSm7yutriWBJqLJ6WXtmCwX787jKRdLD+vNj+LzDz1vg5WYerkyvD1hsp2jgbTbVCVBldTjvc/l+S5DQAsrdz4tO+XP23gWVFRUVFRUbFpYzQaqZpaSHTAyE2LM2zFqKzulxiCEUOLEKKqjaOovU1RnSxgHNLOLdEI4L4FCwDiRGyXL4tR0mdkkpjZxhRCOJpXe7LbIChJrNcLSPvBiDe7iSecKprz+CQR/QZ1lrGYSvpocl89IfgtListCGUE0xcDDTk/yIc8yZBCdEIi0TlGIZLJJiYko9jV2Bgpk8SRW/H2jqQq+QCQ+durHSRFmL2I7bmUJw2k3jZ5YnFj7gdWMYeHj06V+bYxqEubsFo/mHVJKiuY9Y30n3ix2wRLbxIgiqe6bI4q3v1N4zHycg+TXahNguhmsDHIyoByP0SLfUvBSe+OyrxG1JURUU1zoqw8cE6sjFpq5SFxMY1HY9BVApwnmEKUewTd8JRZf1ccwFGsZeQhciD4NAmxEIirDGajcc455+Coo47CEUccgXPOOeeurg4A4LOf/Sye9axn4atf/Sr22Wefu7o6G4UYI7bffnsccsghg/YtFRUVFRWLw1+d/U8AFkmiD6T3YeneZRJ9Wv6FyptmZzFMok8PVRZbxnzpS8k7rX4hBIzHY8QYMRqNMDs7OzGxIAH9UsIuWSK6sXWe1m9LCf2WmnexW4ffEUrsaecvZv/y8txS5TP/ezctD2i6KqE0SJmSt/z59JccsMjWVlRUVFRUVPy+49WvF1FhGbOZ6lZU2A5N08A3ssdODBHjscSYbRsQWiHSYyxjKeoQgMzIGyxGJBWwxZQuEaCuowa3emX7R/FEN6IcQMpvsW8Z34QgftYyIWAKeejGj1Z2fpmquVSii2IYvfYBjDAwrlDiNsaC+xYiuTzX8vdV4gBAUYREjh08ezh2yRedKcL+YwoIjhHIrGBK5beVHQoiXeJAUfEHnbiIaEYOzgFN48TyxJPY4pBPdbWX1bWst7H8dn9kNQPBeftZSG4hqRnMbbLhAZyQ0ONWY2WCc43u52RKeV15ELX+MSqJLu1tmkZJ9Chq7qaR/aJYiOzYRoznIuY2BMytFzsWsNONRoX8HzVyTVu9zICu5g0Yx4AN+qzDbHIiwZMTi5mmgXf6O+KdeK0Tow0tQpRzQpQJjBCQrHFkA1dOkwxiwQjM+AajmQajpklpJ/31p+f9Ha5K9NuB1atX49e//jVOPPFE7LjjjjjttNPu6irhAx/4AB7wgAcMboB6d8T69esniIZzzjkH119/Pfbff/+7rmIVFRUVmwly4GyBWfZANAVDX4kODQqz+qIsL6syUrFDtiaabgrxznsPnMqZvNYwppCzppROovsp11u0KQkAjuKC6BzYEVoGOLTg4OFUjS5KCkzdNPXuj6HeGLoh85w/4Lqy+DmNYdU60bT0YSV6uWy0PLdUoqcrFkqs/C4VLy1dLK+9m8rGBnXTyxpWvVdUVFRUVFRs+iitNQAoYQ4wiYJXFOBiHde2Ib1CYPW7NiVxKlHjbyFPHXmAGAFBryd5UFqYFBP/RJT2tsv102Mur6hr2zaRoUbuyjVtDavFYVnZzS6PK/L4ooy5yp6hFCOm2IxspWNX9MOM5P2d83YnKEqV/eB9KJT8RVeanl+OUxZAxLR5qVaVsp2fTSowM0JBPgMRIFFy+6YUV6idZrmYgLLVzTCJrs9K1HKtH2zOgsTvPImsIUpydCx0nCrkWXzIo27wSrq5qHN6NyOCTpbYBIjlA1jsPU2NzoCDk41BG0bTEoKXFQwxkpjvMJJqHmRxt4wFneNOG6SOTjzf1bqe1NtdYnHrDwcGI8SQ9lmMXNqzyKRMCAGRs4UNkzzfI6f+95SvuxAqiX47ccIJJ+CEE064q6uBCy64AFdccQW+8IUv4J3vfOeS1HZ3Jb75zW/ila98JQ499FCsXLkS3/3ud/GhD30ID3nIQ3DooYfe1dWrqKio+L3HkALd0k1NAmg4S0V+52wlY8qf3y2IQ+d9+HoDBGrn3L4qfqARU1w7NuZv3bTrTl5uaWV778AsYZWpc9Jmk0UfxylB/B2hkr89508bXAxdav6VAUP5MXH/ltICGUguIZ2mlZ+f23xufyC3sC+6YWhQVpLkVk7pn76U8isqKioqKio2LRjhGmOplHZKEEqc0LaMGIP8HMXbuW1lQ1Ah0gOc84Pq8HwhAJTVyy4JOQi+cWiayf2R7L1jPecgTiQoxw4O43GrsY5PhHmIUTYU5dBpczsOaBoP34jiXNTAWYjQVV+LnUfbxnTM+SyAEGsO6IaamUC2zU8BIUpLlb3VnYiS4tmubWOUMhrM6ndOJHTggOhIzFzYVM3WpbFzT1N9tC/MwsU5QjsOIIpoHeuGqh5R/c1tEqO0cek/O3Z/hFS3ttnKVSGMjUz2XsjtxnswU9rktQ2h2Cg2IoRMkFvMKm2TzpUJFkoqdOe7ginJG8EpzYEoav9H2CrcnNcmALK4RUh5houAbzycG4HZIYwZgQMcSRuc85gZeX3GYnpG7V63bdCLZDI8PV9yN3ViSMZnRLZHgK4MWISTZiXRNxE897nPxZZbbomjjz4aL3nJS+7q6iwaO++8M+5///vjb//2b3H99dfjnve8J1avXo03v/nNmJmZuaurV1FRUfF7jTIoXcgf3Tvq+JwnlUdRlrzLP8x9pQdQkpFZpT7sAV6S9xi4bj/vEHG78ST6dAIdgHoTLq5sUZBIgGkb5ZgSJRHpYjSIODSbMG877jyydSnK6Kn3A8OadSOt7+i6TUs3b8lp6CvCp1LuE/mGj5ef+xNVqU6FV3r5qqioqKioqNh8YCpmi0dNTR1ZfM4llRCZEDkghogQWrTjgLY1r2hREJsi3Ow8XBFyRwYcCM3IJWsOaOm2oWlX1d1dJZfegym8s60LkditOIJuDoT0YojqmZlVbVw2vitakE0sARR2h8zUEZyUggRrXCLt1Tfb1OjQFbPs0PHdXsx4IZPf2t6iDvaJI4MpW81ICClH04avqvjPqnb1O48AO86lKfkfIyfi1hTf89VZNpM1gQojkvUjwzlOPuKhlRUCzoslihHvIQSENqTnyMh381GXcmwMFwFCfsaKl3fUm6goRVYixnKe0fBIrVny/bRrdT3vNS42ol3HkM6xkvPdeN2p2EuuD7AjeG+TOfpMmu+/A4gZUNW9eKFL3RlWFw/ne8/rFFQSfRPB7+tAbOedd8ZFF110V1ejoqKiYpOEKRqmEeclue4d0PXrVlUAZfV5DugobUBTwvLKO+dgbICkLAnIHLBMD16GrFCW+rdv6HoTZQJgOCwm4AZkoCOUexTPRzhZ/hgiCPJqfCPBbmTEgb74XZHod5ziechkfLjvhi6ZrVBuD6bd68XXQ+oCmJULERfLRidtVvoE+lB6P21oY9EJm5fi54qKioqKiorNB1F9m6HKc0AU1G2rylwmmEo3tAEhRoRW/J5tVZvF9xZrzMzMwHnb8FLT1SKkafyEf7orCFDLD2Ai3slWL4WIpogJGYCyxmkzR44W60gGJlPBlySoKYVb5aqzxZ4Qy5KnFNxYGab2Np/3qNfMJKsQqFRsXNpvV7dtVBDTU5a+FueEiHR9G7cwR50EKSxXnLUxwsZQBEIzagBEkIs6mQH43kb0srGs69Q7WbswI4ZyRbCd4zrtF1tBhmtcaluyhtGNQo3El/iU4H0zITbi9ExFVfvLhZ1v4JyHqM4J4ABOYz+kjXNJ1/fmjWGNtM5jMrmnMdkg2hoAAsFRA3jAsSncjbhniI3P0LhKZ6ZA0s/kRHkegVZXIZDzcA0DrgWcrsBepO1mJdErKioqKio2UQyR5cOKdEDlI7awTgsoFePFO3V9NMrge+K1gDHK0DLSaXnuCPSv06dgk5Zikdd05GSSgPLSUedcWkpp1yISZY2fqpIeqkn5rrk2cuJg/msZuTxVSz6Qd76LLqWGC2MxSvDFVMAGgrn+pP9P+pXPd63FWrK4gmQ3cl2WKseO+qaioqKioqJi80DbthorAoBTwjOqtNgp4R3Vx1mVwyEmi5TuKjcqNv+EKG/1mIQYahcSs785NF3yGAEeJ8o2lCtaSyuRMiZKKudoxGa52o67RCe7pEg3At1UzFKOJqFQyhcinzJWJSI4kj2JzCZGCNzs7V3We3r8JtclxKRMHgIX9S4nGsrNSgsNeo4ByTZqpUR2S105k9CYHEvZCleziRSbl2xzYxMOjIAYG033cC7fyxi7QhjrMzghtrNYiuDIw5FL5L4Q1QGMCOdkAiUygwMhegcK9lzI/UrXgAqrEJOoytphEzLdiRrZlFWsVrRNaZJC72HMKz7T749uAiuPTwvnApyLiCHoM8PJssUTwcPDh6jtARoHOBd0jCZ2RIsZ/lUSvaKioqKiYhNFSZInFQkiyHzvjDwnArEHMWRDI9tFHqa0prSszspjGBPJMIrdAo8y/iBMZ1wLHrPgjHN5KbAuBgUp0F6Qnp9yxXQdKoL3iVppvfP1hrMRvJOd4aFqIHJOd7knxBAkmwa73slGNt0FohrYd6rCvfc+pqUvjjAfSisDWmidOwd7V+EB85Y0cJuoVVayoHPe4kjk7vmLSJ9234rnFdrntopAsnP3vagjFYNAsokQZeM7PWH9RsVH6t1aPaEK0SsqKioqKjYvtG2A9xbr2mS7KGHF51uUxkaaiu1GUAJS40iOqkZXwpcB12SiWzZH1CCDhUQEjPyVtBg5eW8nsccA4Sp2GATvsvrdjtnPtvlpjAyCT0R6DnqmEdcMKHEq8bCRp65DBJfXNdGHEdASb1E631TYQ2OEfnnMDGITt2Q7EaAXoXJxPuc2ZXs+2+hS2gHK8adzMskxGjVoGoLzBCCCKAId65RszWNK9KZp0n1J74XIxprDzMXkTASRKMSZGeN2LJvMgpTIt/g0gtlpu3sTKMWqgPw86YapQWY5xmQTFwHwPo2eoBvbWpwde2F6f/KhvBddxbyMUxMhr/YtpGNR5wGCh4MX3Xoo7nka04g/utwDAM4hcpMU70SMxgOgMUIMgPbhQqgkekVFRUVFxSYKIob3LntzewnAmqZR9YEtryMAI5BrIEsrIxwxfEOyHI5l4yDAiEuAKGpgUxCLbKoByDuMvR+qXVogmMhNIxtFdTLNUqWI7OBQKh9yuweUxEM/TyExCdmbEbmWRflFu8X8RQJMBigyPAhMDlEtbzhEIdcbAnfUF9Z+HQBQXs46WGnkXhus96JI2e496l+KlXjmNAAQi5pOZw1cp3P+RMUG8ltwuxh0ZlnKa/Jger7G9Pp0Wx7TwKkcQNlYKR8DOBb+jVacWbhoi+w9cugMFnKdWe/14iYRKioqKioqKjYNcJQYkYUlhs2sOwcEjpCtD1tEBAQOCKwqdCakUwAhtr2H80IkS+isaeQ1lia0Y1W5m0iFADCBQ7YfKa1D+iAHISApAuzStW1j0xiiqucJgHhMh5ht80SFne0yjN+PyUJFNxNNBHgW0hDZCkKJ1QAjWYs4Xl/pfAAx5o6aZstXWsUgauybo7R0frQXO0SoWjr9F8EksV5EBHNQvQ4JUUviS+69qNBJVw7oMKdQZQdwZFWAE7xHoaiXBsZo46Mcm0qe0vM8IIRihTERODqQz6S/+Jk7xBBTX1pgHGIQmZJjsUMvxnYpntb2tRQByAa3gK2+lFW3kSMiSZ+0MQAcchmc72G5ua7E2KQhsgpUOCJwK2Q5efGjZ9sI1YPACAjaJxGBIa9W420nTyTIVg57wEWwC2Ae61hYN/DlqHtaLTyYWjSJ/sBzbu5oiOQ9z37YwK+7gCHnL+e0sjdqOUO08NLYzlCpuEjSFekgVFZ85JvsOGuVqFdeLoAK1Vl5sXIOKw/002+tPaB6yHEezicigC2rDaCo13bpt/5sWTlmtF5ttNyol/aQuucBoWrziNFtda98KprTuaZ+iaf8mfywL7BEG+gAWygMexYG7q223Qk/gGJjaKSK9wbzVCi7OlVkSv2bCQ5K504+Q6kVsJ4n6pabl89oP7n+2eUzPzxc746DufP8AEP9LM9KJ42Q/rhNMielypO79dHC3/+05QM1q6io2Lwh5LcEbAyCgyOH0OpGNh5JrRLCHOCjLAl1ko/sO4kkMLbv5hyI23fc8Lt9Xoz1RT7XzpH0vuIl5+9/9y7YFUtET42TFDGUvt6JWMQmMXb/aEN2sZegN2qQi/zHYLAueXBj7VrIm/H2gbtFK1Esf4EKnXlnlDIddt5i62tehoMzBXcg+r7lQB5Q5fTJ83Lfp+EUrIP69i/lKxegAwoatorpq70qKioqKioqNn04NwMiFShwjn2NKGYOAFqJwYltwC9xqMaijoyQVgU7hMBMXJwz9TAkDzuVBBuJqeUU5HmMseO1LrA6kJ6uwhtyIhiIsqlliEIym9I9KCsuBL/4Z5Ou1pRqZX/2GBihNTK33AiUVa1NcOVeNp24DLDNS6lY6RnZA3FYUVzGhKwVdkCO7yCu70RONnZl6ToTSQiJri+tD5Pct4io8Z/Vi+E8oWl0XKBp5IqNUnW1rW8IvvHw3otHOkHV7TFtCBqjkO0y+QGdVNFzfL7nMQalLT0II73nUfsQMtZrjecyQVK2PpFrWn8Vnu5OLV9c9pEXT3XbUJUQIyEwZKrBMZhCseEtdWPl4mdmJ9diZ04sidQHAkZ+BGJC20Y4RLF3cQ528wJHhMCIgTAOeUwDpy7+7NSuhgFy0oeI+kxrW3hxY8tFk+j9RdNpTKUPGukGCNOGWd7K6Sx7pfz8k5nOp2FkGsRZWvlrwEYicz5ONtorx4Wcf80m6NWSqE0j4m7mTNh2Wq4PZTHILIjmgjpAyVTn3EYw50FZZ0KirGPRVwRGJGlzudDdCGhb4FDw3nJfSjK3qGv/nbhsb76CiuO6pDtbf/f1abm28kVmGwJo2S7f3yJ7uqaN2Ik59Ync29xn3X7qEzbdHtNLFtfIuRJBUxD++ZlQwr0Y8Haef+sT6C9aMWnTedi47LSy1kX/ondfphDo8uvC+oVqZaTfpIqKiooJjMdzaLxT0g+6RBIYj1vI3yIJajdsWI8Y1mNmdoRZWgbnRjKdGvMmK0TqE2cK9E6gW34PT/88DSWBPnTONN9r8wPs5h2+hlucTBvSL1MiKObO3xH5W6h/NdgGQRicCBAfbFXgoEuoDn22Ac5Q+4YCvCX5xtOU9hVlpUnzJZS7VEp8yRMhE+f3g/Hh40PPUz+9TJvPMzNvjARk70uL9/LzIb82Tga06RmhzjNelegVFRUVFRWbF8wuwjamBNCLBzLJ68gB3mLK7AdttoEm7gCAyAwEiWEJ0ZjcFNM455I9DAchw62ctOFkMcGfyXXldIgS6Q8wQmjFs71V8tKU5brxpqnVvTcP61JUo9RoCGi1DLNEsbqCAIqi2o5gNI1LcbF0V9T2AQGhiMezTU3fB70vfsiNNR6oiP3ynEMiGydZGqQ4uaNu19tibXbOwTcNnAOYQ+K5bLKAIWMs73yyzQGLf77VtW2D9HlkbXcAs/SJ2L/Y5qsA4FBaEJLGpGRcJIkinpl0haX0vXMOjJhsdjodIV0As69xLsexMYqHvwiuZOzoyaeOoBgAVZ1zLJ6VtCo6P28hBIQIhEg6OaOxdgTgGzTOyzMWZZNVI/Pb0GJurpU+08kEjgAFeX6iz1wakWzeGpnBrGPiNCZZODbfSDuXzFznPQKUGqXOj0XWYoCoWQhA0Dy+OC3l79HRuTiZLXIFWZ4e4M4vw+QgsF89kBKRnM/PhKbOy+gvTFrqYAQr5c/dkqFEdncSIDHeVFDamcEvWmftLUlSaaw8+uZTm/u8q+nXnxOBnpfI974rQLA9g3MrJsaN3KumNsAWeJflpbpT/vKn3vHU7qLbOhy4Hjfy3NmTDvRaSd06FdMUE6r+RGSg+xzal2q6RP5yZEBn4yZH+bZcKP/Q6whAfzl6z4X1CZcpnCaFhpCfx2577BnBwPGKiooKAJhbfwtmGpmpj4ERNZi57dbbQEQYjUbwrsHNt9yAcbgFW221DKB7YNnM1iDMgskD7CFKEICpBShqoAos9OWzELE7TMAX39PME3mLswtVSr/MyQBoKV+TQz6KVp8+wTq0kqysi3kYxhjBATk4Lsochv0dHTo+0L6l/h2Y+LtStMnI5CKOWSw9PnUCYignL/yM3B5MKs5pMH0JJRafNAi1iRV77JzGb2y57Jrd68pA6nY2sKKioqKiouL3Cm0bAFAiWG1jSVMXmyoYUCKau/FySXqX6Z4D2APG8JhdY4wMjmIHYq8k5+Pshy4bkE6ukusIBiEMNjMhtJw2LI0lMVpsANkl+qlXbibuRWkN7RcV6URorCQMWC7POB2X7EBIGa1cdlZSW//Y+yCJDqhin5TD4SyU4JILzOUZhwTIxEiMfXEGUp1M8d5jL5H4LN0sVMqQc7t2N5z6yzjJfh9n/wWnCnGxPRH/fVkBQFFib+ccmpHT/vXqwd8ik1STcTMg9zZwEL9yn21C82RMzkvkZOIDALcAyDafjRPlZ7pN26HtTkQ5ItpI8I7hXYNm1KDxHsHsY9TGJgQgiPMRuOiXSIRIQCT7nSKQeGkgcrcei8ESSPR808vHLaUQ0moTIJPrXZKzS7TqWENL4CLXZAvsJwdkjxyaLA8FaSvEb1bxSqLVNT/0adMzbYC3JlG5bRnZZN6UQd/ASIh1KS/0edTGZvJB2pBWN+ivvz23Vi+yAXpm/lOrS5UaWztSAcUN6fUjFV3MOhmSNpgbaNGEigvdohPFTXkATeX9sYEmu4lyhrgHm3RJ6vri+dMrWaHpeKkmFxLe/Hhzayb2CXBWhyESxiqTJzcGMcGTa716/dn/HekQ6/Zj5/7aSb3rQJ4JcQbofgFVVFRUlLj11v+Fc1vDe4+2zRvN3Hzz9Zjb0MJ7j5mZWdxyy424+bZf4x732BLbrbwftt4amBltjcYTGu8R09ctgzjAsYP8tRz+7pyu5O3m7ZPo853SLy8v9Jk8aejai/6etL9XC9TByP40NJiYEGC1xvGIUYl0jqDoUszTD+D7/tnyd3Ry0DGtLYtt49CmoP1yOsqd4t/5y50+oTCRdxHPyMaW0V+qO58SvT+Ymm/ixgaZZb4yf/+zbVzE+rAWuwCIYqzOgFdUVFRUVGxWaMfqm01OVeUSC3TJaFaXCerEmIDEvyEEcIzKmTg4JeFdQZSLV7US7sEIa10VV9THlNJlXF5u9GjXtA8x2vWhn0WJnvJzjvEz32SEMGAEtU0C5IiUU33FeqSspUv1KMcMSQSp9TOSubzORGw2EPPZZpW2crdUVXLqAO5wRiWJ3p0wUC9zpxY7xWQByKW6ElGx2rcBYH7cXGyqSql93TrbJqR5AobIi98+zI6GIBuYUloNkDkxL33KDhw9QggYj0nsU5SAT+McIlWmo2hvjqVlpa0Tj3ImtG2A+exbW33TAEFjYpfbYP3A3O1D48aYKd0LcjImHc2MMDMaoXEEig5tBGJohXAPHjGIcz3bOIw8ODYQR6MIFwlRInH51yYuXPeZnQ9LINFtlkQGdA7C5DNDb76wgwX111H0ovfZ3h2UcCfSRa+TZC2j8I/WGxZN+ZMKV1K5d275oRxrm++1g9mg9GqtmTMh3P2F0cc5XbE/MWC/80bkg3RRr5LzDigmo0rCVssxAr9cNpJ+4bRug6RDwdIW6QSdHCjrznlA54u6lAtVqDinRG+LMeV6pUdsc7T+EL2j2e/XuUOQqwI8lcBFCSV5PjDQZZsnsY3ekO5hp0+4qFH5fJSkdcrXRa4tDyVKuUMEDmSiIi0j6TxrQ7mLe2tkudWPSZ+RSqBXVFRMx/XXXY223RoEQggBo9EsmlGDDetvxS233ApRo89gw4b1uPGGX4B5K3BsEaPDVisitthyO8CPUvDInP/WU++7G5gkLw1d8nmSmOy+d49NI0qL2HISA+rtyb89lj6YOJieJ+aLIK/XjqGfSSN9jnkDy1KlDkwjb7t/63JDpmGgM4b+Rkznzyfr0A9C5jsH6MQQ85dfDMo2En31/EKYRnzPR6CXqq88oCkHaEOKdv09sGDPnlOSnyNHxNgusqcqKioqKioqNh04VdjK5otcbOAoSm4gBv0ZABUbtlnMEkMm251jwAMcSos5kk0alZQOSqIDppoWHwLvPZqmSeXa57SCsiCZwaQEaxB/7mCEelahE3pEKDJhauQ2c0zXMyW+Iwb34n+TqeQ4rIwdC+5H7WOycnt4/NGP9fI7clzeYXk4XyP/NFl2EdOXY5lsrdK9d3lCQXkzUJqcYA4girIRaePShqtGbNskAel9NNV+CLEYV1jMKvkCxaQIt/yiHPcgyIrj8Vg2JvXs0AagnIAo75VZCTlHGCWLSu1LJcLtGbLnNoDgwYCuuJBNRqXO4jRf9Cnyc2/30rzWm6YRz3inFj8xKuPnIdY2NhGlnvx2f9LqDE79bHImZvHzN2t18ovj1hZNovtiQGlUt2OSX/zU6DQ+KH4JujRzR6XUIyu5V2FXlFNukupZN740n59ESlpZmW1Owx0yIjnPwOSvmKISRV7q/vp0SPtEZMPcmPpjeSrK0Zc+AEz6oDnqXL8Y0ukDln9x+0Svzbp1py0mP9qP+hXUqx8X/aN3le3BmSRHrN1WVqrCUB6arE/xSPTIDOocH/op1WeiSuXKA/uyUPKcOT2Pdr86fHqfjF/gFybdX/un8+DnL8z5zi/nWidufPFM5eZnQsoy2+VSvgW8bSsqKjZf/O8v/ws33bACRMDMaAbb3OMemJ2ZxYa1t2DdbTfBOYdWgx3Em7Bh3RjXzs3BscdoO4+ttthCgnCiFFgkVcBGTOBNqrVz+jQ1+oLBzOB34NA5wyrpyeJtFdoAEZ8UCotTK5D9vYeuTHOEqMtrTT0yZBFT1qXflqX2+sQKLECXuC79/i2ESSnEPHmntnmYEF9M3mmf50sry57vWesPxCxMG5yjYKQYa6gObTvG3Nzc1GtVVFRUVFRUbJoQ0k82jAQT5toWDFHuEiip0QFCMzPqktmF+KJUAsfIcEoWkjLLAZlOEksXITaNTPU9b/XRaARAVObQPCJeNfsN8c2WzRtN8c6pbONqyvg2Ri7I5F5My5yEsc5Ln8QIJWkdnM+e4ka2W/wNQCcJzEPdJ7F4GqIMxmcDNi5Kz4QYQRyUeHRwunlqyRcl4t+p/U4iZvMERVbSS9tNVS6K/YjRTIMYWfu51b4K4NimtsvER0DTeIDzCgLAqKdMaFu9Sh9zqafXmre66aiQ3t67pBwHO8QA3Zh0Odowho8OIXjEGLRdUgoRwTemfhd+eFC0RHnVplncOHKIsU3HpY6xuB9Gnst/MTLMwsEYU+esnkbGy6agzNIuc9WIUVZ6RI7gwGgDST8WRtRGtEsfN9pGu4cLj2QWT6KDECDqc3v+TY3Mvd+HrJ7NiTYcpeJTyR3ao1lWmcj8ust0RtmudE5JOhuRilzPXA+tharoQfaLYwMo/YJi6ijKrZ6lMMsmBDzyzIuld86xtEQms3j0ZFkSumQwcltSe6yuxRKTYvarvN7AozzxYyJhLbMNBNO1KH8J6TkeBIfupAmK/uimSYHD9ekS/+X983pGhxhPN6loQmcwK/9Qysadi2bCo/y5PJg9tkxZJndH7hmX/WAzlp1y5iFSCLlvkevZ6QP9eWLwTgNL0PUelH1fUVFRMQ3X/u9/YzTTYNnsLLbaeivM+jmEmRHW3nYL1t16E7wjjEYNnPdosB5z61ps2HATRmiw5fLlGG+zFUJYD+YGrhlhNDtKwTEwTD4uhpD8XWPo0m7Kl+ZkXrNRG4JFJJPT8JPIy0tto20wI4QI5/OmrHdFPy2FvLZ4YJElY7GZN6bd/XOmqcinEejDKxqG78GQL6iUkweu872LDV8swiwLnj1aEFiXu1ZUVFRUVFRsPijV2xIrqrqcYockdy4T6maPAQDeN/BeNp2U40J08pTY1BS6HKGbQQIEh9IOJOftxlU5TiH12I6IISRSV4Q1DKBrB2NCkzItq8TzhIBzDoEZjhhQhbOkZ9EKazxtthtSGCUSVggbaZMeFP/vgfHIUMwI7T3Xie+RSN0ORKmp98epNR9UfZ1JYSm/nHwQUpyIMDc3BlRtbe8cxZ7HN0DjGb4xsUZmF+V+zYAbApj0Ocj7LIWgG4Ky+Hw7UluYhnSCIshGtUkIpBMT7EBkm40SQnRoW0IIlJ6tdC9dvrfe5b4o+9q5ru2h002AxKLIVmPa5qVl1wpH6x2BRwCCrk6I9sQEMFow1A7JARx0IgMBoAjnZDKJOcI7gvME7wGQWR+V/LSo2OV3wJ6jPHabD4sm0WX7A4bj/FkaWwxMjQ+cgsStF+QpIfW9SPXJNpLMx11SjxWNshuV1NrCVJpLa4+S7lVMH0RSupS1bcoilwSlqZrRPR2A+QShZFjz8LFPgqfJOft60/4rv+wYOeNAv6ViKfc9W330Go5sk9D8hWfnR20zWQ9MPCDD1y5V0LlUnuiX8v72l7l3+mjiRJrIyHrdQdVgJ2ve0MtpWmmlA+oSJ5OzksWGt8U9sy/tVFI6ZBYxC4DyPbD8BJ2EAoovae4Wlu5P8WXUmYDprq4AMNmfFRUVFYr73Ws73HTjjdhw6y1o0KKJc5hdNsLc3Hq0a29G9ARuCU0zgosOIcxhvG4D1t+2ArfetBVuWr4McMvRjhvMzG6Jrbe5J7bYchv53qGshCnf5/t8e7CUcgbJUmD4b8pQ3r46oAR3Yuhc+AL1ERu8/PcmeUHeJSQ6sIi/ZAWWQvYuJe/QYG/4/CH1EDBJaFvaUL6lYCi/XCdb89l79zwLAbk4lvM4R/DeZRVaRUVFRUVFxWaDECLaVkhNwGLCzK3EyKq2dQhtRIghEZdGpHc9xi2WLP0RHEyyGQu7FSNEs8VFxpAgQI8oC6REaD5DuSIH2xxVSNRMuIoS2yXRo7Qzx2pGnIJMwe5hbgGSL1vcMFj3RBQCP3nHs6m0J1e1Tovlyncjj9haZGJSU0Z3ab0OdyWkNCeLnBhDivGtv7K9SndD2O5KAmHwYgS4kSv5xqlNi9qN6CQDdPIi27lIOW0bhaTWiRXvZfNPmYDQjVsJMImxKcRlZYTxZA4U+pMdWZ1tzwHA8L4BqIzNc0dl/33WdunmsTEghryZ7NBYxDcERHXvCBEgUZXDtRAiPSTOzHl5xkNsAYwBGoOohfcAOQffkH4OIFlSXdxJl1cwgEHIbV4IiybRHRedCC72vs0kc/+S/aFRf74n/+LpNfQsp8eikpd50OuUUCyUypTckjpkJUF59cTs959+qOoZ+V/qHE71pOI8NrKeKBHYpYI5nd9/HmxQ1btGx1W94GwTkVpe1/qqIHEJsA2E06CYSKx2yseAkY+nNnVmYrp5HTLhDphbESVCd6B5KO9np8upfF77xIoR1KrcKhvdeWLKyQzutKnzjJUdk2pU1ME6hfI1h8C9D+lZXmgMnuoz+bSDbNImk+cT3ULIX2LFNcs/YWWVK4FeUVExH/Z+6CPw7//+7/if//lvrI+3oeGIONdgw4Z1CON1ok6ITnzQeQSOQBivxfp1N+CGG36hQckKxLgM29zjfli+bGvQCg/yas82EKTeHQj0ofxukEKfQrgW1m8lcsCN9N28ULhVli9LLzktIbS0O/+7fPr1htXoiy918RQ6yaZZ/b+XS1RnlwORyYHfwliKaj0fy3+rTZmeFUM5gjNBQU6VEW8MLWJol9TOioqKioqKit9/5M04s03L5Go6dU6gkGISI9CjxpBlHOm9gyhxRaHslIxmZgQE6K59iWwWojVMiCJLMYKdL/+Zgl33rvNK+MJUyqboBYxULgWaokKOIBVAlpMBmechdFe7GhEdEZQoT7wjZ5JWElxRViZ9+v065Iseooh4nZZjIt2Y2j5ZRhY75v0EpR+yHzn3rpNWHMTu2CCq8bk5dkSn9ykReKx9Q4k0N9V9tnEhtdgRxbttLkvE8E0xLiKgbaGTHh6u4wFuvNx86LbH+DSrh3mTh7RaQWd6QkBsgz63RqCXYzZKzwGDQS7Cse4jqXtFOscgCmCMRYgC2SCVAVBowTQGYw6gFr4ZwXmG81EndUSJztEDZMQ5ADiEVlXsPgIuW4bPh8VvLFpurAn55YnFz5nE7XCnMvOgQwcbwDJsx1gjyYtyCkzYeNpmm0U5BCHDE2lLSEfzs6KPdiLccxu6yIxpIpm1fh66IKNoY0mqlqquju97kTcNtooeSrk6xGiR1PliKYhuznmp+JDuRVlkWXb6GixIDv3HFNsEs+rJX1aeemWmnN3JCYJuAltkpvLh6FQpsxCl/U63pSjaL09T/iIp+qJgo9OlObeN8lKA1G9LIhEInWe1j27zygqxqg+1HWDdNLXIq8fK50R+N6isstwTaxyj82U45N1bUVFR0a5nzK0do90QQDFiPa3F3HpGG+YAbuFpBvAOxKQ7swe0G+awvrkVN7sR5loCeAVGM/eQTUYBANT5m/+7JtAzJssbukL3bxVNSc+pHWp98G9QTutPaPM8uftXt+Dae68bMpmPo0ZH/fgCAPp/ixYNSvV0NBFNycBgomjO4oPhRiwIxnDwudRWLNYTfdozJislbGBVfp52vf75QPl3vDiC1E8DhP0Emd85S6MY8yUFdZbhVlRUVFRUVGweMHuP8ueszBYbjbZt4bwTaw5xxCg2zxSyHECOJSmK57VTxS1TileIXCdmiSzKZGbu+Grbtbt5zac6W2Kklfwa4Djn4VwDU48TESIHWMwkUZBsimkcTuI4nFNeTojdopeQleCyiWRW4MdEHpdyBSPx5f9JwnwwPUbhKxkgRCmPpXEpArQYvackNyV227awcYKp9UuRbUmgd218Mv8mMWIL5qi8ocdo5GACarvPzgn7GQpFt5Qbk+VOCAzvgRCA8Vi88MUeCGhb6ysPRxGECNuHk2ETO+VkSvnkUrq/bdt2bHPEo97uD6dnPIYIhBaxVY/1BUhqaatwX40jOBZfEkcAOYb5oGsv6vPZAhTgvFpQq4WLtMPBmOXIpNbnk4IrjozQhkUNvRZNonfcrfuELdsMgT0A2rl6hv5aap+r3w5ycZ0e61+1GKSaUtkXx+U62dKjJHatNA/zt1byVcsCG9FeDHMKstg80Y3wlXJIqVz9hSjGZmW9rXlmF2PlyGW7AzOb1euDoZMVOgvVGdtnyyd5yNg+5y+PSRIh/zJ0Wk2Fz3ZnCkQe1EHioZh9A/T6KG5BzjIBuY+cnpD8zEy0vviRi442xVf+JaOyTgOD4VKRPlgn5FnEMhsNlNcvO3fvxOwBAOhXcfGcDg3I0VO9FQS6rdBIX+MlgQ5UAr2iomIq/uVfvodbbr0R5BqwC1g3Xgeem0NkXerWElwzCwoec+sj1m0IGI+B9ub1iLweERvgfCPL4ByJcp2iKCGoa58xn4L49hLq5QqdTrno/i1LEUARrgAWPnHHpoXtbyX7XBjkjwoTg2nSasN5PZezx7msiBv+K570K2lSXvZRmXEO49gizI1BDIxGIxAcTJjNzizKQvrrRJ2Se3/OtNFmgyYb7FCKHaxPzMqNBv9yEEjXnZXlWy8s5i8NTcof5NwYJwroD0zSjzQZ2ALTPe01ktK4DrqzkHkkIv0RL+OEdO9gsdlk2Zw6rkvq50GTnGyDuayGKgn7qI+U3Ie2DWhDC1DEsuUzU56bioqKioqKik0XakDMmbCzuNIpScYAOAZQsusQAtRUzmILJwSQ+IebD7jwN4nIVfJa4iSzXHEAI5HBIRRELyI4CtOVLW91k0b1VU+tSCGfiBsdRZDZfnAExQjbLLS7WSOlGIqc7Hco4wiJ47JqGxo/OzA7IUDhRO2shDqBBjhZB6moy6IRzjJS1uCNNdCLTHBMACd9N4hz/zLleF4U+eYrLv2IGNTGBUqMJoYSDKdtEpJZJkJC8sUHOFnCEEWxm07kIMM3I4ABpx723stL7xQQlHMsJjpCjIgcQcwI0cMF8Tf3vuCPSAhjRy2cy6R/npworKvBqW2SJh7iMQQgZots6ORAQaBa5fTZYd2A1uqgfZ34Md2Elhw8Ad7rignbiZPleQkU0viP1bddLI8YTePgvQhYWWNwRkRiOznIs2GTNmA0I3n2gk5EhCl7IpVYghI9D7qsnWnwSsWAy8hezudYZ8qXRHl8kuZNPVpIrckGfWyK6JLslMGMjXVKEtche093bD1g5Gv2d5JLKdmpbXD5Molitu+K0gqm/MVNDz2XA8TS/iY3O43PtB5C2JLedP1S6BD9AxtKUlYod78/yie3vPDQsDlRCkVacV1C735MDuCt79KD0atkeWVmTqsAHPr17hL9aQZviDzR58s2nwVgm/h2BsZJfU9DPSRkiS2hsT5OPEtxH9PZVNAPNkOCiRuQmmP3hvRLkQbydiY27Pkga0/xwBSTKZOTJRUVFRVdXHPNNQAFLFsuSywjy5K1GFsgOpDzaJoGbYjYsGGMDRtahCi7n7dtxPoNG7Bs2XI47+BnPHwDkI8ddQUwnfgsjy/+2OK/1fLfr4IMLQnP8lOH5M+By8R3vFQK/e/1fLp8sadg2QKUiRoBEsjb3wk57pwTkjwtUzVls3zfE3KQ6YpS++3uJzjNbUtk87o9+3PGVroMtgb++pZ9Wab1QjbtjyFafYLe1/J4YpDTj586bRt6Xnjy753cgjLOma8W5d/h9MOUxy0f66rOU1U6k/kpiipV6Jo+sXmVKnOYGaOZ0dDFKyoqKioqKjZReD+DpvFgbtQjXG1TOIhlhbO4MsJRA4AmuB5RNQeYD7r3HrY5ZFd5nb2nzUqFmUGqVo8FcS3nQFS+nfiGlTgGsk+5EaoiMHHEgJKbIOPEzHJFVOQEU9onzwM4b6puSD2ikC8xSCRHyLY0UUnoRMCnTsl8noFcI3VQb/KohHjU8YsR6GACbJNUCJlK2laOEZEiIgUwouxt5yy2JhAayCaeTt6jrSwQEttI99yvep3oVC2ux5xOQqiHt02AuMbLJIpakUTEJM4URTrBNR5e+68NcwgcETkoxS7t5dggtEBwBGq8ikQcYmS07RjOq2CHkSyGTNXObGIQuTdESNdmynVlzqLewOLhj8igGOEYYDjh3JToz/Fy19ZF7qlL8Xa0NrCMY0BARMSYWVhAfWZMOCMkfI7J08oJp0IhVhtN5+Cc2Bk1jUeMBAqENpAq6ufHEpToEAIvqc76JCqQZDudZzix6d3BWWKS5RwHJMV1B/obTJ3zy6UVmZTMQ1dST28uc2ZyEvkm2DlWhimdSv8daTdSut6WtHVDNhjpXqQcvJfDanvYpB65BjDimmXzsbKfxB++HOwp0Q/Mq5ROw1/KA2FLFQU6JvYyLfvalmYPEtCJ0C36uRgRU++Cnf5J5U9am1hvya1ndE5LxI3cF1fcl7J7SqseLgooq2Qb2OYtDXrEC/fa0Gk77Le5+PLOLUqTIqRTC5T7vOy/zpNTqOWovE754JaVK34c1v5VVFRs7lixfBajWYeZWcK4vRXcAn7UYNmyGTSjBsuWLQc5IIxbuMZjuWswbgPaKDPy5ByWrViOrbbeEitWzIIoYBzWwVGTdoVfSGW+GAL9d+EJ3i+bjRhnAPZXqxPI9P9CUfdvGmxJqf31tmMx/wHq/L2wiIGKv4USyLmG4JjBbUhKF9OsyAo3ndx1i+8XR26wP4f8wtP8b7e1/Q4oQ44J9OZRctrQZL2bptUfLnuxrR74E63p8z9z3RUU8+frv087NpQ+dE1bog3ICoSKioqKioqKzQfezwCIcG6kolgLQKOS6GLLAgAcSskhp3zMAey8cA4O8GrnYlwKA6ow9nAupviVo1yrjS2E1LaSCysSfcnKxoLcBNIxQ8deloBsG2LkuFhv2Map2ZLEZccDEouSGNWawzb6tDqzWKDEEFMdpH+UxUsKaauPnV9ySkiEfurLBCOCta46h2H6c2ZGpKhq8ogAhiOPiJDrljbKjAACIgcYjW6rFUv7FpBOlug98N6pitrpZAol0QWxso1jU/brhqLs0vXbVjarDYFTHGobvgKuE3s2JCS+ENoEV9qwqEJelPUxaYrknhc2lKmfKfV9UvirEIi0v/rPTOr1In62MokIzufnLaZ6McZR7pPUneT3BS7ztM5WCTjdS8DDJliE82TYpqTOidKdyOm9EL17Q0KoL4TFK9GBPLgzdpG7R+Xf7pKKiSqkgZ2Sh/pUmzLI68BuiLSlXjH2S1dWxL5mnI0GC8LRiszDWr1pBWPavYYu7UhFddvdd7IcHjwWpGmR1vmZY1FXuU5TlqRkMVN2NC+vycVERB7YD48KTWWfiNfMRxc1y8z30KAalD3TJy/QI38TMSyfhWeQi2biNw9Ki05Jv7T9ewJtM5UcNvIzkZvAuW3G8cMmSLKPkqnQS/V5uqbxIqVqzy6c2lf0A5f10MmXwuqo7AtKFyr6KLXD7jJ1+4C40x/mrz7/REpFRcXmCuJWlrq1QpQDAFg2x162bBmapkGIjGY0AqMB8wgRG7DutrVgahEDy+aiLqING7B2/c0YhzEIM1i+fEssX75clNU89D2OlHZnEul9Ajm/DxDnBdLXug5AqPdXXrjubjyREXo/29LD/M0vgbzavzgP50Vd4Vh2/2HIMkYHB+PdafCP8PR2D9qSFMFrDmKHy1zKPaCBgDjHZYvDNFvEQZX7kh6PnjqgV3aXEB8ufIhALweR0451qkyUNk8qyzUPUufqFHhFRUVFRcXmBPM1FwKyfHlRJHsl0onAzkMsTIwIjwBFOBolktc4BWd2LkSJrI2RQWiTFQyrglf8yqUOAGAS70xUayTFatuinEUitsEptgab57TEXqwKwpIEd0aQqne6dy4R4VKOWqMgGu+ZRA62aWVkURIzAIpmJ0OpHiUBy1FjbCPBIytpbzwO5/qauh5ObFAK3kesHdUikLKpizqeKIktyvMYxa5PPMYjxE6kOzEBoHfPZcNQ70hJXYtLORHdRLKRrA8OoXWaT/qMI6FtGeNxi/FcQFBbGctjr9SHUSYKZEWkXDOr8+W+hWB+/QzvTf0fi3rZhrYdxhLErNRYVwDNxusVYxxTo9tz1GU489iNzDtb+yW3QZ4L77x6puuEhJML+0QAWt/LfSGIFQ0oAqQTOcVmtzLhs/CAYwl2LlIRcReaJFfLn9OApD8IIvTSpNOcnmTELhdZ8+AmM5uEPMi1FCM6XXG8W7niXAhx3s2XS+ts/KgVYTA8UXLuKJtBOSOSJyd0kJWId2Or869scVbqt0TkoiBGbfCbiPLi2vaAcPkIGylrDeTUPiNnqdOn5YLvVHAnhdKrrPFE9pSZtL6pRiyzI7afsH5lpZONWC6KSC1Jn5WUJlDZ2NwHdqZ+ERQTkZ2mlctqqDhn6NclTS70J43IapcrmmdAzQcdgP4xy9ZARV+XPq0FWZ5eZadSzldaDqX+WRK5UFFRsblgZiSBcjtuMZ6bwzhswCw8ZmdXgMh3J2UZmBsHjNsIcg1CYNx6221wo+WYXXEz3HW/wU233gznZjEz2gr3vMf2cG47LF++vHPNaWTifLijlehDRDpPXIP1u7/cJp2R/tDnL96c3z7xIuqcGHuLbiQxcoR3Dt57UcREIIaggaJHo38wIg3VeYFLTpFV91Ug3PkLnUGLnJFl6CZX/fNTexeJwaxDBPj0dg130XAdppPeNG+++Qj0hcq3SaZy8DQajTAajerGohUVFRUVFZsZZGNIcU1g9fk2S5P/n703D7TlKMu9f1XVw1prT2dMchISEgKiokwBxA8DCWOYuUQQAjIGgswyKKOBi5CLIINEVPBexgRkUsAYQCBehTBIhCuIQoAQyHjO2WePa+ihqr4/aujutfcZEpAh7DdZZ+3VQ3V1dXX3W8/71PMqJVAqSLqAFSoyoA3Gg9NRRDfiVNYaUIGV7Ri2fqIjSvoMgdYDmrjZkEDHtwFLGuREbKN/7TTIg367zwzoZT6kVCglWkxi7+8IG/MQCdHGn1Q8B2EFUmWE4rDWj03awLMD1a2ZZjM3+Fn0ETtuXTO7E1x7SgNCejnnkDTUesWXtjZwG4/zvniDuwR0v6WTHijcWED72hgsOoJ7UrbHDcLr1yuvZy+QIiRXDdI5jjlujAekpWOIK+l050PeI6MFWoOuNVVV0+ipgxDG9zMRr01grSuF60ce8HZa+IGF7s7PgdbEOiJEy5+1LmgAUU5FGNe3lJ9p4L79+UaMTqG1dglHTSufULstjZtda0WQcvF9IWiW67CfAOtmBgTsLJxrqmST9FS4DuZAfj/DIuJ2PlAhGznMIF10KLthci6IjZrkInTALqCIgDhlmBbIKrqb+bQKmxyLOI1lWpIlDNGURz4buLcBItsgZ7hpRQtkn8aL2xrn3RGZm86gjcYIS5qkmHhOfoCE13RqI+NMQc2+nYQN92gDM7cCJc3xNxlDtpnjsUnCjdJgskBInhr2k/EmD9ekiw00CxqAu2n7TlChfe2n/5i6trHcjV0D2XrYidYeIZAQmzI+sTqQcSu40q2E2HByTWMFILwJaLUAhdB+cfdWIx98nB1PKrSR9fr3UjTBkpCgdlP2fgxWhEBHAMg3HqjJK9wOYoWX0yb12rIt27It0xqERlhIZILWgsmkwjBGG8h6OVIlWJwuYqUNxkrStI8Riqo2LC6tUGnJ6tqELO+R9wYMejtQImdubp5er9ea2rfR/rsA9OAwxUBza0AwzYw3YdYbeKfeT73UFUY7J1BKQaIUUiiUypyjHZ14gPZMryYgKoJ3639b/94zntXTMEHwRARXlhSWNFHoyqJxTqu0TZnBdzlMEvto7WQ9neVt8Nxu8l6b2rbNWgkm2n5AaPLo7IdtfH2PDIdv3JnpulgRwfxp13Lapus5vW6zc2v/PpgdTspluoyD9eFpMD1Ymqbkeb7FRN+yLduyLduyLfsFsyRJPDDudb6N8/hk1MYWniHr/FZjBFYal8DSKxh0/CMPNuu6YdQ6a/tpHhglsJ/9FpF1aFv+Ttg+rG+Dww2Iaq1BSkmSKFQi47ZtADxIqISkpg6Qlf7bETK00RgddNM9Uxin9249gz7O6hNNvaddL2ucEvi0ukFoI0QbKG2WW+0o5QIck9kfSgqXqzBgM8IDSYK2f2ojCOeul2+vsJFnOSslIxgd2lkp5cDbkPDVNuMOp2WuG0DbgBESIwPeZWNyUj+hNY5XwnqtieBzkjid8bp2y5NEOi17GQIfgd3tkDrXTuE4PqACHlh3ddR11Wl/KWL6Tt9mNo4PZUvbPixrdNe74zhaWCD+nIIkkNENm15KgTbNWEtJfFBAoIQkUQlpmpCkCikE1tRoXfmAjL+XfBBKtmYCHEFe0RsAootmsBO7pfWDUdGCBlsDLUVLVqULq8a/uwfpYqUtfDiCkQGQbO0S6zfNZA4DUBkkMHzhjYxLC5C1bey8ufmM1UgKElEgbY3SilqkVFZhZYYUvkP4m6UV/InlhcQFQYPUqfPY2EHCWLTdLkY4jfjQfFME/hbQ25icapcATsdrF849Pnzi2XZA8ghMEID7FvgwfdwI5Pq/N2RQdccLOkQN/iBaSHlTxPRgPPY52zwsm2vZOrYHLYIi+YbGapXdbszO848pEL7dWaeq3Fxo0bR7fIC6AuNsh1jD6XJbWvIiQC9TV7UlBRNlCWK0pTnWlm3Zlm3ZZpYmCilThLLURjlWejmiLg11ZjHjCkSFSjImhUGb1Dm6MqGXDcj7syS9PrWGqtaozLokN0B4V05/NrMbC7Afah/3+LMbjhuctGnQVOBY4LWuKSYT1ldXGK6vUpUFiVTkvYyZ/oBBf0DWmyFJcqRSSCEjkdzoEJC2Xb8hHCuSB9zUTDfQsQih/YDJDQpCSmwllUschI3MjFpKEp9I6kgB9GCHAobbbRf9rc6+3f3jtrbrFwghXMKkeM7t8sOUyg1H3vR4ho0z28K7NQwENiEebX5OU2UfbLvNpVe6vw8n13Iw+ZbN9g0DhbBMa02SHFk+gS3bsi3bsi3bsi27aZlSAmPAeCDQIlu+bABTZAOmtoBEh3k0PpW1Hijy7GVrrQMWI+gQsA3Hdo7+TgPeNT9te13btwcIPosHII3FCifxYY3AaAcUO1fY42JYB2xCBNClJ50I6WRJAISufUJML1ljHY6CkBgR5DaUx2I8Mx0H0DeyMK7ONrAxO5LCDXje+IIQ9NGldJReYQXCNG1sw7XwGF7bTxYiANQOpNYtIotjdVus1xBXSqGUjO0TgOQk8WzpqNXQ1F9rTWXdeMEY42RajI4wFTSyQI45HfTunRxNYPMrKUE6drvF+noIam1RSL+/bvmsia+nxy/DeVqD1da3uzu/sqoRHnCXnnVucQRk93fD+JaeYRqDBwKqqkII6QjLugWm+77rZIecBrrRBm0bzfwAwBvrxh4uP6IkjcEahZKKNEkbn9tqjFFUdUFdGzcDGBfUCfeKEAKVHN43P2IQvQGvg6711KRdITYb28SuFDeLf7Q0n9vrWsBle2yRxMWiYTF3AMx2csUWcOt/x7JEIxvT3rdJUNqwfLXWSFPSs0Pm7Co9OcEaTS37LNptFFYhpHLTbVrgbwPmevA/gK8RxI1jxIj7T4O5sr0fYDyL3t3AbbDALwzRwtbyDqdc+AdDGwj2qxqGc3txawQaAPVQaY8cuJ+2fZSm0ePzqwHOpWiiprGo1kHF1ArhAQrH4m5eALEWHVC+W107VW7DImyOLzoBj+kRd+vCdDGC5vzjscOLgtgm4RpOn2YA36fPXyA616BddyGcMm9nn1inLQB9y7Zsyw5utjZoYaC2VFajazCVQBuDFaVjXEiBSgxpNs+gN0CqHG0lWTZDf2aBtDdgXFSkvR47d+5i+86d5GqW+dmd9Hq9Juja0kY/EvuRAPTgBHtwsm2baVDrumZcjphMxqyvr7N8YJHlpQOsr65gTU2/l9PPM/q9nNm5eWZmt9Prz9Hr9cnznCRR3rFKUJ5B07yAIAwuAnPcae81x3cDIYPEZYLPk8wzRDRJ4pP+WOcM1tr7DlJsjgYfzI6QkQ0QNB2720GWJbG+Yd9m0GDjYYQ/RylVBInDNptrom+s25EFgYMrbadfxIfZ7eAbb8Yu32ybTYvttMvmZUyD7qE/BgA9gOjOyd9iom/Zlm3Zlm3Zlv0iWWD/AjE1nkNuTGT9OljPtPwZ52MFn0zrNlHQbgCPIfhZgpAQ0ngoZTPyyzRT29WzmWVqWn6tQbikpVHCxAEh1kivSd3CpUSL/e4BTiKW2MyxF6JGWOmIG8J6AoulreEgnQaJB1INgQkvlSdRivZMRqfp3QIfaTAs97fxLG1jtTuMFV5yXnh2t3bXBIvRjhtvPR5lbfCPg1/s0B9jfA5Dr60tcEA7qgmUKCVQiYzs8KiWYERkeQvPqDY6SLB0/cnAxnZxAOOB6sSVJxuJbI2JAZYWIIoQyinrxOCMhTgTQHpf1sT6WYtX5zAY7YMoTlbclxfIrv5vGfDXQOht9M/DTAiXCJQ4bnDJTLt5nEKgQseks9ZLumivky+QClLlk6yG+6CDMQuv7S+RKo33g9Y11pqo/+7KxvXhw9gRg+iJCB3HTbGQMULQ2EGGIq2vNpgeIMAW+7bVyQO5KQCpDWBo43por3OXHdGUTQSSQ8d2QHSThIC4TrQieuFstK2QeszAjpjRB+jJZYQyVOIo1pnHeKTUihY43aai2wBy+zr5Z4oPsLlj++niLnpi43MySMQ0skyy1WLNuYdzjsBzbMEW0B3bowFwhS+gDfb6MGFzGdrIfzyRFm89liea87bNs7sDUoeatNu4DSK3VkQcXoYHrG3t6LnmtqmTCCckuu0DAmFNp1/a9qYxcNJasWknbpXZAdVFjLQK//emwPnUiYvWPz72TADe7fS+AkTTCVrtFNpk+t7Zsi3bsi1rbHV51TkdGLSw1FZjBEihqEtcNm9pqU2NTGEm75H1ZqgqibGKsrKMihG1gbl0gFI90nTA3MwCc7NzEQycBren7b+LiS6VcjqRHkwPn+CA1XXNeDxmNBqysnaA5eUDLO7fz9rKMqauyVLFzh0LDHo9JIa6HLO2VDAcTkjzFfK8x8zsLDODAWmWkqUZSinS1DEbpFKI+NYKDBID1kQnTOuK8WREVVVgLblKMGXqdNHTPMq+SKmco2wsRkAim2mRP4pt3sYWNlluNzioBmsMVV1jjHHBcNxU5DQwO9yO7noI2Qn0H9ps9Gw2W2fjX9D1JY/ADvFi3JxdvnlbHCnb/FDLwndVVdR13WLsyC0Qfcu2bMu2bMu27BfMjK09kxaM9trlHvaMkhxSRJC4LWnSALcNcOFVKZpElDHBaHD1AiAqOJT8InTJAs23k8cIZAmJAGkjgSLO5EdgrWzAU1dC/Fe02JbWiJj3x1rlAbLAtHaAkqQ9+1WgpJPBkZ5x79pHEAj2wfUTvo1d/Yn7twMIoU1AUMvageitjzUWrRXY2oHbEoQvs4UmRb/fScl4YNdrhCOdjIhjRSdR0iVIh4TfDkG3QW3eA9auveq6oq41da0x2q/321hjOnVRSjvf0l83YxyWKGQjheOulTuuMaLFOvdkICEb3DFgTaI1PtDG1cVYn1jUge5SKqQKRGB3fjJo5cumP7SB9SZpqauPk5pp93GD8TIu+PvCWLeNrh0734Sci0KQSnfdTEsz3aUyDZrzLpCgVEoYs+k6gPeBgOWkcw5nRwyit24NNkcdG1i8fV+GTKvTe4XxiqSZwBAhct/5ZdhONEdw612ZSoRL3AKWCZ0j/O2B1lBuKCskXfDnJIXAChsvmLYaY0qMrqj1hKJYx8gl0lxSy3mELJHK0vQGXwchI0Ie/201SNAmN1OgsBD4CFjTUEEmxrWFjU2tRHNFXEK0FkAb8OwOWN1As9PjxRCZCQEbQWv/1oUSgq6UT6ymiOcr/QXsbiO6OzQN0bo5m3q2Lnf8DvruDfstAOet8+6A562DbDIwDmpVzap2pQVRd2d613aQwT/gNxt/iw1/tH5G4F8059aB/Zs+K1tlNJHkbpRiaj7Ilm3Zlm1Zx+qydtwC5TQLhUyoTI3RkKQpaZaiMolQirzXI80z0jSjNlCONfVkhEoHZP0ZZgbbnFa6do5OXdcdVu3BmL3/LQB62HcKPJfSTXusqorx2LHOFxcXWV5eYv/itZTFCGs1M70eO446il07t7Nz2zx5moLVFJMxZVkyLGBcV6wPC9aHq07vUSUMBjMkiaLXyxkMBvR6A7I0R6ncD0zaCTc1LhlTRZrBzOyA+ZkB2wYz1KVmUhm0EayPC2pt3Mw2nL6hNm4skSvPHrkBbXNky53PM20hOVVRFKytrbG6usJkPGE8HlGWlXe2JbOzM+zYsZNt2xYYDGbiLITAxt9Q7qaM72n5te66rkPk/jzY1pud88FI/BtZ5Ju3W3tmRbtvH0orfTMpF2vdvTKZTKjrmizLyLLssAPZLduyLduyLduyLbvpmTGVJysE3WoB1vmvTp/ZSWMYo70fIRoQ3TiWs2MLuySdgb0cmMqOCWwj9hVkUxq8qmEKt/l5QAtM9CtFIDAqT0QNLPAAkjfMYZcE04HsDsdqM90dWCrwALFwIG6QJDHGHSOC7zFZXcBbJCERJrGGPgDhSdYBR7TCIlAR28L7poEhLYVjzItQitCO0a0BbSOInihHPtKmRtra4YNWRwa4A7o9c15qjJFowixDi0WDMFjr624kSOGvr/BSNB5HNDWWOoLwdStRqAPMRdTqdte6NWvUn5tbbSLDO1xUp6suEdqg/TWTMsw0MLST0QafOARqnGRPkCN09dJaE9j3UqhIClHSgfJKgFQeSPfJPYUhBkSCj5wkSQzEGGNQyiXRrY3Feqa9NrqpizUxf2toF4MFY6m19u2hovqF9bCZtBLlZXEM1pclXVJWE3A5Ffv+wcYPbbtBiUXbULBb2Ab/uqyhNuZH63dbcqV9O9sQkUDEhABd0N1GPNqdZpgA0gbMO5g27WkFDWhM80zwi2MSTiuorGZcDrHlkMyU2GpMWS2R2xFCToAeNrdYKqq6QIqELHHhvqhzLloHa7DvuM5OL/cDxJYcj2O426buwrZlU9pXVmz4aoDdZvDZMPJbG4eje9y4/fDpXD//I0SO3E7+XGOAYhM5kvbYdBpknpom0QaJQ/sEhloT+Oh+T7XgptZZ7tsxnudURCf8GQIKAbWP/adVZdmqS4CyN+OTde4Z0Tzk3Dq7YePAaIfuvRLPu3MtgpktHH3LtmzLNrXjTzgeKRNUoqh1xep4xOpondJo0rxPb5CTZQorBFneJ1EZSZIzl/bIe5KikhgSdh91DMfs2cPM/BxZkpImadT3C+Cx1rWX95iuxcEZvTcKRAwvKeuev1K653RdVQyLIZPxmLW1NZaXllhePsDy0jLD4RpC1mzbNsuunTvZtWMbc3OzzAz65In0zAkLcwNqbRiVllFlKYuSyWRCURSU5QitC7CglCLPc/r9Pv3+HL18jjRNSFPlEthIN0jSpsLamtm5GW52s2M5bvd2BomgKCz7D4xZWlmnMpZKF2grnIPpnWMjDCRyU1A6NoVv300Dv4do44P5EypNKIqC1bVlfnDl97niu99leXmJ9bU1JuMxKklI05Tt23bwq7/268CJ9Ho9sizDaO2ZTlGA7JAW87Rstm4aqI7ncthi/f4H71sbGOMHA9vpgudhNLYZaB58Oj9x152XdzQCGF8Uru/kWU6WZn75kQVItmzLtmzLtmzLtuymYdK7ScY6EqLTRzcRnDYEUFxHsNABzSayzlsoVlzWkWixhkbaxDnNQgoUQUIE2mhDA4Q35JQmn4vHKFr+vjF1BPet9TRRIWK5QjR4RfCXjAe7Q7JMB0Q3Mh1hWwfM0/W/cLrrvgUbXCq0jz8n23IYPbxHkCfxCDtCKMdP9tKMiVKOTBvawpMWHctcIZRCWIXCODlM2yTzDFreUgq0rjC2dsC0BKTxfztwOU2VZ87rZgyl3BF17ZnxHiUWwsS2aIAo14YELe/geQqFEBInYePXxfZv/MymHzlAXEoNIrDjQ1LSbl8ItGxjDHVd+2snOqQZgWrpvrsEn6HMpq+1Ayq2QyQJQHokyloR+3ucfdE6p8B8d+Rlty7opgsLSkg381MIapWQSO2CN8IRpV3C26C/L8BKQmLVDoB9CDtiEL0BzFuAY7ggcejQbBIure6wi5ttuoCjaJVvicGyWI5t/RYoEUrayASOYGaYxuLLjOuFB3xbdYSgP26p6pKiGDI3vpYFhmSiRpoVhB6R5BlKDSiNBaGRWBJfXyECkdy1hbGtY07BpeGcTIgQBXRUuGiK7ewX0PYGXI3Td6bGh+1LHoIQTSU8oz1kFAYnj4SIAHpr68ZkOzQSgP3Ajvdt16zuAudTdTvY6DcC6G1JEyE2K2bDjg7rnu7oGysRHrSB3Q/NQ7p9JNv6R8QOZKfq34UfYhbieLCpqy3av6a3bd1Rwvf1Tr/vXut2ACEGjQ7aQFu2ZVv2i27HHHcsWA1GU5RjjKiZmBRjE0SeIdKMUk+Y7fepK0M2N4MxCbuOOY60v41hYRgXFXnPsdF379hNnqVoW5IkCpkarKiwSIQSWFu7RKbC5RVx0wjFxufiIWxDcsuQzd64AYeSKr4jNIKiLCknI8br6ywd2M/q0n7WVpcZrqxQlxN6Wcqx2wbsPGY723duY2FhgTRJMEZjqoL1SYUxmkQ6mRaVJPTyhDyXmEGKrnuOdW8cK6SqSoqixJZDJnpCPRkxydZIAziqEsCidUVVTUgzyUw/J0sUWWJdUNxUFJM1qrp07AmZuETlFgQGTYXRmrKWKClJlEt6ZLXTpPQkl9abyHs/U+/ZNjOpYyGxEwZHXZAYoai0pNSC9bV1rvvBFXzjX/8ZTE1ZF0irSaTAGMnS1X12bJvjmGOOpdaCXKRI7/waAc1QJFzxjbPFBO56bnBVBYgp5y5I3m3aXzZxdsM7P+zS+FKtvtXaVtj2kuDvtPax4Rzci7epSsvD9UmbhA0zK/294fUWhYFcpeQiJRPO/a7qMDV4y7Zsy7Zsy7Zsy34RLOBRbtzf0qoWjSxB8DMcyEyUdAnfDkh2igQWhw8JaaOP4vYPmtY+aG/cSgeQa1+XzXAQx5YOzGDnd8qoAGGsRusq4iQxISg+n088R+n134P2dQPIBg6Dk6AhAqi27cdGgkjbJ2yA2PZ4wmFCAVaTGG28P0kE0J0utpORsQinwS6c3IohIswEAD0kBhVIhEyQVgPeH5du5qgQJp4PCKRVCAkqcUxsoRwgL6UgSQLxyCKVIFESoVzFE6lJVI2uNXVtUVKTyJqqMujaUinTkjoxkaHdMOwluq4Q0kTJGIRB+FxMoQ1CXzHWJ0M1rRYUFmLi1uA7O1Z4rZ0euuseEil8+VIglXZyKco2MjXCDzU8EdpK47FHMGikEAjpZh7Yjha6mylrrG2SzbavswhMf+FyL1rhZDD9FF4jwCioKo0EKumY8dYkIA0a0wHRQ2JT64M5RwKgww1iojc3RHd5A0NKRGfSsaWrES3ilg1Q2Lkp/I0I4XEQNL8FQTnF3UvdyFbEYMPxOpGvEC3xje5rJmgeGsJvJ3CRQWNqEj2hz/UM1JA0H6PNACFngBwhJAqD1SU26cd2UT5yWMdEZ43mEr7uFmK2XkkTdcNHU5TxET3wjPwI+bqPbc6padnQ2uE50xpWywa0tYgQuAJLqy1aAEdrDCla5bhr0VzrAOBOa4CH5AJNIVO/N/071A4P6IsNW9C6XsQSLJuPqTuoNe0QhutisfPRDXD4I7b6VCdQ0ynZxr3jFZ4ONE3ttwH0joGCoCXW2mbDMd1bwSWXNYTYnxDhZXhkN/yWbdmW/WKZVAK0xYoaQQ04p1fIFCszhMqwRqM1ZGmPqqjJBgOy3gyq12fH/AwHllfIsz4zMwOyNKGXpyATkA7gtSa8SyVaG8fo9s5q5Cm0n1ERsOw+M4MJFRLleCfZeG6vcIl4BJayLCgnJaPxhPW1NVaXDrC+uszq0iKT0RoJhkGesrBzJ0fv3sH8wjy9uT4yVUghKcuS0XjEaDhiOBxisWRZRr/XJ8szUpWSed3zREmyJEWpXkcupixLyrJkXNSMRyuMR9InTUqRUjCZjFlZOUBVT1havJ6qGFIefwxz/YTFA6v84Or9FHUC6SwyG6CSHgJBIl3A2wicA4lAW+OTLDWDh0YOzDbNegQWfIrwPopvZQva+lTjFjAV9XiNVGoSW6NsjTKWooKqKJmM1l1QQYepxSp6gdO+4mbxdbv55addQFO3g89m2PTExdRKX4EWv6HxA2zj47ZXxBmALaDfDahCmaJhPQUkPvRtP4fUWA0YhK3AlG7gQA9p3cyJg8vZbNmWbdmWbdmWbdlN0QTK4VYysLIDeB5A5uAbKIyRmMj47lISrE8+iQCRNHhWoxTQwrsiG9tprBvhpEYaaRP3XwBqBQmIhMDKNlZT1hVCWLSpqUyBNhWBNewkabyv7kHcwDaeZh03JyB9IlXR+mxsrYBvBx30uEZ0JfUi/obFodNhH9eyQnpZZOvxOGkQSKRInW8nDEiHBRp0VIYIhBTj2zzUS0jrwWBPTJEWIdIYPEAGAN3NYBVSeqjJt4UVCM+utyFhp3bfWEuSONdSC4sStpFzEZogzBKurZuVa0FokBopDEJZNybzUxW0qbFoD+bj6+fHFlI6Nr5wyUgD7mqNwzadzrvyUjkO6FbSkCpBmliSxJAkkjSVLZ11i7XunHSoh8fWXD5UjcClbnUscS8ZYw01LuGrCcB6a9Ti+pQrxODI0a4h3XqtDVrW1MJSCEdcypIEkUgXkLIaE4F042dVGKwXxD+SWaJHnljUXaFN17UHRzKGgAIs2gIbBR7wa5XV2j5KlwBtHe8ot0KQeemycENRMjxEwsBIhFJbzPOwbAqklAiEhJkko0xzkBmJNeRyHWFrEtVzOqcJaBEE+43L12sNMSqlS5SdoLBokSBEjhGJAxrwYvimRJoaaUskAo2kFj0gY3racnvAFwNu02Muf05BDSYCwKFd/LBWhQJNA75GJDwMBkPhob1x91eQfAmrwnFC0GJTO9hyiA+58GCGcJ0PD6AfjI228SCupLZy+DRA3W7SOIiOX5apRfGlFBa2Ae6mrs0+8TIIpvYLL7QGPEdMsdo77SeaQE+HEd8+yJZt2ZZtWdfW11YQpkYJ47KQGwtWIpOMvDfH7Nw8VhegS6rSsLy6j3ymJJvbwY6sz/a5GYzWKKmY6ecuKYupkCIDmyKsY09b/y5PVOKfUQ5Udd/BMYH4vDxY8sngRKv4CG/2sxZTuSSdyysrrK2ucGBxH8vLB1hfXUFg6WcpO3fOsDA3w/b5ORbm55gZ9FBKYRCMxhNGoxHj8TjKtEjppv0Vk5LxaOLYNkLQy3N6vR5pmkYd6yRJSJKEXq9HnufUdc1AW2rt5Domk5KiGFMXhrIoQDh/Ze/e6ziwtJfLvmRIbEVVWWpyduw+jqOOO4mZbBBn4RnwgKxF+ERO2gkvNok8fTAeBCJOX9vYppvp1Iv2S8mKMNzwA5CgxSnisWpdg9eENNa6xD8ycYlHrUs53/EfjtDag7vDb3sjXnLWbtomm5UtNumPNlC5GqQcx6Lyg93IrPL+jBUtn8AipEUpRTEu0PWILKvZtWOOXTvnkFIyHguqekvOZcu2bMu2bMu27BfJAkju/C2D1hBwrsAyB+dfSOV8lCib4pNWOvA54BUeM2vpfMftOzIpHjyOIIXHYWzAn6wH2/HSLQ3rt/k0oL8JetXGgcvOVRVYpfy+xpdnok54EyCAkNtPxPrLjt+6EfORnWXTeZLaQPwUMhbbTqP9NirKd0QqqvBa9MIJpFvr2MlmQz3EBr+0nRsonocAPDM8Mtp9GzZYUKCkGqQQGOHaDOvY0QEDctIoIRBiMSIkTvXyND5ZqZBe41zWnkxrEUHrG+ezJ4nwyWfddZGqwaSsDX3CBw6kRVmfnym0gyd2polFKrBCYVEI6UhWpuV+WyvQoo7BGYR1eHHEcyXWa7EbY6lqQ+3HVUYbWt0FEdtdxuCIA+oNFu3Hm06zXmvQUqG1xFFWFNImGBt02U3sS3XtAkoBbz4SYO3ImejTA5EW40Z0N4x/OlF331EistgGzcFuwvptQMjAtPLHaYHtMsjEiEaPUggbWVqhFkLg5V+aMok3V2MSYjwnTxKEkJSVobQFeTpAkSBI0CiMVRiZIGXqpln7qeplVWGrNbbZJfqJoLQ9hswykQOUzBG4yJao15lhyIKcILBM6LHKdiok2irfMT2fK7CnWudkWwu6AHCI5IXHYGCZN5fLZVMm/Oi0SSynzXRvPTTiX37ge4OGtC3gOC7w/4bqtE63+3crsNChnscEsmJqVSPb0tY7tTTBADt1gzQAd2jHg4MBNoDZfr/2DRcDOnHd1HFCoKcFgKtYgVaSWdxDxxrrH4QuENTRYvdTG24ocLFlW7ZlvzhWlRMkBimdlysQSJUwM7ON7btvxsL27RSTMaO1ZZQqsbLCKhgNV5mdjFhb2c/C3AJYQy+XKGWoyhJjaqTq0+v1UDLpvKesDs837ywaLz8S31gHNxdsji6vSyxj3LTR8fo6y8sH2L9/PwcWFxmur1DW6whr6A967Ni+jZ3bdzA/N8Og36OX50ghqMoJq2sF6+tjRiMHnEPjbEspST3rPE3TCB4rzzqfTCaMx2NCEpyghx6BdSUQSpBmAwYzfeqqpqpryrKkKHKXlLOaUFUF43KCLSeUlcbKirS3ymBtlSwb+GujUFKiUomxGaUxUSvSCIsKLSgFwqpOax4p0Oxdty4ZwS+zWKSSKJWQpU73vpporKmQpkZJUEIhlSJPE8fQD4mCsB5k3pgh5GB1kwcLphyhHbI3hQGjDYD35m9LN0tvExA9lBHB+Oa9LXy54dt7pj7WEygkBl2W1MUYzIS5GckJJ2znmGN2YTWsrU9YWhrd+JPfsi3bsi3bsi3bsp87a5KPS59LqAGTgzZ18M6UDBIc/iOFZ3M3s+UD3qA20ZkOzO9pzXEnAaL8DEo5NbvfJd9sGORdMCrURcmEoOFujMcpbKOrbq3w/n9z/FCvBtfr1rmtm93+duu7v9sM9kZ+JsjCdFnrHc1thAOFEoUUFuOcOTcD0ftxQbe942n6hK5NO3avaacFBU7OsnVNgsSO1tox9ZVsNOg9KSMklZXSxO0BB3rjEnkK6XBLJ1Ht9MillG4b6QB3RA3oeLGczI6TqkmUJEgHSRWY3K7+SapA+ICD0e56KRsxpygfIyxKGJI05McSKJm6WbMEzNGxzKUxDvQXxuUnlA6Q983uy3QdzGowddBg7840UFJhrYw4pIsnONY9MkjX+FybUiKUBuUY9toK6jrFmJQwHm2umWlcfNfQh7Uj10QnnEArdiJaQK2lFXKgA4oDrco0UGoDZDdRjWZQ1ySqDJ09jGGaB0arA4eOSgB4bWe/cBwR6yViFASgNoZJWaDrCaoe05OGNMlRapYkzZFIVCIwErSW1AZEIkmVi/wU2jCpCrJyjZz9zEtDKecoraSih4pQp0HZkoFeZqBWyZWllNuobIoQCRMcs8/R3hukPw7fArggmoXhdzxHv62M4HOQASHeIB1Qu/tFW+LEz8zBBTTClWku2QYLd0Or0FjfTiTJds5D2ub4nc3DRd/keE1PagPY3QdrTHIRamSJ01Bob95q4GaqdrMuhCWE2KxvEo/RgOGhfqKzvul/TRwj9k/RBIpCtLSuaqSUJGloM3+ePmrsZ/v4pBRbtmVbtmVdq+sChaWWFq0NZVFSl4JB2mfX7uPYsfsoJuMxK/l+1peuQ9shWW+AFIa6WGdx75i1lQPs3L6DYpSilaAYT9A6pd/fRiIsMsvcVDwTdBZBKgdIO+ZJSDR5sBeHjf8Gp9PUNWVRMBkPmYxHjNdXWVk6wPLSIsP1NXRdATU75wfMzc+wY/t2ZmdnSdOUNHHv5eHYMc6HwzWKcYnVwiVNnVsgz3OSJIkDhDRNGAxmmJ2d8Ukya8piQlmWXrqlYDIp0FpTVSVlWaCUIstSkiwjSVNUokiUIlEpuUmo8wQzk6HNLFVVMZmMqIsxQleMxiVl5ZgrZTFhZWmRNBuS5T3yXk6aJuA1BUPbCWtj8lYpRENCCObf1dMtu1EyRMR3c8uFig6slJIkkWR5RpImmFKAkCghSZTEiAQyF0SQKjiixpchD3qZNw44Wu/Qg/SJDfXeZMmhgPQ4mLL2EMcSGypiIRBxpg6iWuW1HQjAyhhkD+Oe8XDMeH2NyXgFtKCcrGDrARaFrsaYanKI2m/Zlm3Zlm3Zlm3ZTdsCmB7wzilJE+/rBQb3Bl7rFC4mZQPyTufECTMNHYAunORHBIWJ5FfaGIavY0OQcBhEwwRuO0ki1kO0cA1oQOz2x2FOZsO2m+/j2qABw0XnE5jF0ic/tcLhfNPHjHrauOCBOy3tAHIsQhuqqnaSe8LJlmgce9/YkPBTbii3uR6hTi5PThMMgEa2J1xL6/NJNdiTA4slxhN5aBFkXdu640ufTFRJRZKknlEukdLJx+Bz8lirwZioVQ9+vcejpHDSn01Qw0YnWNIkFhWyuT4BsFdCk6UJSaKacYV1fbnZ3rG8k8SDVlhXtk+kq+tARLFImZAmKcY62Rmr6ijv6S6q9PdDAOp9sES62RrCSkQA6yVOPkcaDE4WRiCxNpkK0NjOublZqAcdMES7AUx016zx39DQbb2RNntXdPFU0Vre+ooIq8SBvh2wO+zbBij9WEe1buzmZg1F2lhWG7QM4HCIklmjEVSkwkBVkE9WSKo1MmoSMSQR2unIqgEqSbC2BEDZHEUPFwNywv5FVVIWa6TFGkKNQRUIlYDQvh7+AagNdVUzqTV9WSKSESjFbDIAk1LXiiE5Ku37m4/YEAF4bVBs0WpPi2r6fAv4Jd540/rl8Rq1r3NreWCv++7llk+NVtvAcbAAurvCRbMw1CGUHwILtnlUt4vvJhLrrp0GzGN0xTbQdZxS0+2Wm5xAq8y2+GlrfZtRHmdHhCpJ0WmH1l3SAtlbswJCOX4KlfDBknBO8YVkNKauEUoiVAqyCYqAz6xcaxfIElso+pZt2ZZtNGOcHp7EOTSpUqRKkmc9ZmcWUOkMeqwZF9CfmaOsxggMdTFmaf91aCvYu28vt7rlrTDFUSRKMpmMUbLHwrYRiRxT1znG1NTaUJQVoEizPjOz86RpTqIyx0wI715az0ZrfZ4HsMZQ6ZrRZMJ4NGJ5eYmVpUXWV1YoRkPqagwYZvs9dhy9k5mZHrOzOVmWkmY5CIE2mvGoYDyZMJ4UFGXpWB4IsiRj+/btbNu2jTzPCVNGw6AjyzLPRJckErAuQWqWJaSpY6CXZUntWeZ1XVJVJXIy9o6rJEkSsiwnSROyNANS9+n3MbOzGF1hdEFVWiZFzWRSM1pbYTIao9KcLM8diJ5lyETRn5l3Dr8Q8R3nAuJJfF/FYHl7KmbLusm03ZJmjfUx2TDYMJEdkyTKnUOSooREWYXyILpNM+eQ25BQKkwflaCnHIWDmdjknRz6xGabb7Zx2we90eboIBuO6Qcu4RDOvxJA2vgvfv9Q50YWxrqEVbVgbXmdfXuvIk9r+kmNnqwjUKyv1xSTrXf3lm3Zlm3Zlm3ZL5JJz+COwK5wBEpHRjEtIF3gsuQ5iTgCgBxy3olN/DvRAoo9Btqwed1YQEgB1niNbd0ChUH4OY+2xYiPQHVEcNzywEj3wsguN16UtoOgly6EcEkpbSNHE4iBUja+eKhnCATEYmxQq/CoVAvfC+cd5GQa5rbz1ALzPZRjLUhpPSDvAxcenMdaqGrKwvv3XifcCOuvAkRJFW8hULFZICDmQ7QNGB3JlN5f7DCr/RZCuhkKyus8WjMVRBGgpEIIR9xJktTrkDtipZSOqmlsjdFersUDoIGdL3DYq5KO4S2V9Ou1rx+OSW7aDH63p1JBR9/4YI0gJE9FuKSlwgclLAnW6sh2F8LLwgg3/kBqL/9iSJRF5iCVpqo1JjDhbRgGyRigcP0FtK0wtkZYg8URrITQJNKQJo6Iaq3TPMckCJn4GQ0uwOCrEnqRH1f9GDXRDzYVty3HEmHOViMHmZ8IbApaQ5XwABAeFLcNyOh3kAFc9GhrV/OoC6rKVnlhWeggEbSNgzbXSXI7YVaM6IlVjNhHaRbBpE7Yv1pHyRrMEBgh5A5qPUOh5qjULGmSkSlFZTRlNUFWE4TVaCupNQ4YEDVGV+gkixGjmoRRWTOgRqSGJBmT21WMnaBNxhrbsEKRpT1kCwzvsLZsq8GFZ3L7CxHOrwGaW+DtNHOq9at9jZtHl3tg6HA9RTMg31BA+8+o0RrG+c1At/OQaBUxXeo0DBDLbzpEWNCg/Z16iNbfhzhgC+BuAJ52f3Q9NfQnERGgTZKO+r4maeoo4gk3dY5BoAig+2QhFqdlgMGYilSWJNYlrRVagRQYLMpaEl0jdYmVCil6bNmWbdmWTduk1Mz2M/eS0M4x0rp2rGprKYsaqTIMkuuu34swY8aTFbbv3MnevdeSJAnleMwPv/8t9GSF2dkZxuMRQiqqaoWqXCBJEzflz8LS0ipp1mcws4096gSsnSUbbAPjnCvjGcvO+YO6rrDayZ+sr6+xvLbK6to6qyvLLC3uZzIZI0xNL0tYmBuwbX6eHdvmmJ2dJUsThICqqqgmFVVdMZoUjMcTxyIx7t0jpSJRCUo6PfOQFFQIQVVVHUaLUoper0eWSnp5Sq+XY60lz3Pm5+eoa6dB6KYZ1jG5aFVV1LpE1yWFqakrhU6cY+ucWwfCiyRByB5aW3qloSprylJTVJpSV4zHBaOx93GkIskWyTOnv56kCXnWI81zVJqTJKl/9TW+UZsR0x6EdNhAxmvUC3zyTDeN2LWDjHi9UpKZ2T6yHiFMTSYT8jRDIymlc9jxAyAEbnqqCn7G4ZHt4HdsNiX2yHOfBE+/GSQ2TKQwBbotKSi62/piQqJ3J/nifAIZWET+PzfS8rMEbfOOD2wR6wdSbkDotB1NWVOOCtaWV1kzI9aX9vHviSDPB2TpPP3+duDeR3auW7ZlW7ZlW7ZlW/Zzb1r7JIgWGla3jYBhlIk7GO5Cgwl1JE1wYgZhnWMfi+AmRfwssLaD/+jcRZcPaDrRp9P6DmUEQVzH4nbs+QDsy8hkF7jlxjYA8zRrO2o0ROkX29luM+wx+LVtf88B8bJTZ3dQh/0FNzEkZg1IlJIJkCCES5ZpjUFYl5Sy1m52rTEaKwzGXwcrgjwl8fib1TfK4Hj/uEmI6rEl7zsnifKzYgVSuE8jUx18cYWVFmECa8YHB6xBSUduSRLlZqn6a+80zg3a+PPVDdkD67XbJQiPTQqRBKaIl4YRsT9GMkmnnaUjzlgLwkSCOR5QF1jPIPdgOdLpuwvjggYBubbgEsBaf64etxUGIXXU3A/jBccm744dLDnGapwQtnYfUZMqHCivwnEt1iiwiQ9KGIzVfkwUztditObHmli0Ew9q9emGKOyB64bz1AG5JY32tYibh32gy9S1cadI2A8AcQtk7wC/wsfAwvoQiaOdmDTcNq6W2tYkZsKcXGMhWYL8Glb1XiZlnwqJECPqckyiIEsGaGmxNkHLHJnO0sv7JEkCtSYVgK3IRO10SsUAbRN/hgaMQfhpz7UFpRKE1ihKcmGcjIzoUentpHKAEcRp3KEPRlJ3OPlw3gFAn4oStVHlRsO8DZSHdmrDzTaCxPYgDPYNIHncszVwbi1316Zbrc2G2FPQeHediMJBra1aILl/DwW9WNH8E99RcesQgAh19A+3ZvNmsC39WclW/441bQPorfJCu4pWg4W9hWdcxn3CuVmBqd3DGj1BUZOamh4FPSWwVjPRCdokKFvRZ0LPDJHCUKg5anUDlJm2bMu27BfGFlfXEGIOMokuJxSlYyVgaqpiTNafYzwaUpYFRVGgy3VGkzG1LlBJQjEZk6cKzIT9e69i73UaXVckmWI4vI6VlTlUklBpx0QfjWrSbJadO49lfm6eJEkxtiJNcrBBz05T6xKjKybFiNFoyOrqKktLB1heWmJ9fZ2imIC1zM8M2LH9KLYtuASh/TwjSxTGaC+xUlJMCiaTCXWtXQZ5a0FIUpWQpAm93oAsTZmfm2FmZoa1tTWuuuqHrK2tU5ZFdJy0rknTjB07trPnmF3sOeYoZmZm3HseWlqLgjBNtv0pyoKyLKgqpxlf1xrtE44KUZOoBKEkVmhSmThd9SSjlxtqY6m0oaoqRqVj0JdFQVmMGHtdyjTN6PUG5P0Bea/PoD9DkmYOnBdhYBIGQ4LAxpHT0xTjOyk4sc27MpQVnPssy7C9HonVKKtJlaI0brZfTG4qrBuAiOBFH7nJAE63TByMib4JMB8GEm4/v533Ixtt8i7IztS38GwNa61nDfkZZdYnbQr7C99mgRjQAt6llA5f90wmYzXCVGSJJVMWqQ3lpGBU1KSJZLRaMBhYeunsDWqvLduyLduyLduyLfv5tihlYvEAtQfwjO24KEI4f7at2TyNmASmt9PHDiBncywHiG6ogSspEgvC3w2Y7XALD3BE1nTbD2utZwoTtA3LOxwvsr5bgH5oC2jA9M1kaNp4XwPFBAyrYbR39KyFCZgzgbXcAO0KIRO3LxLpdbTDeQR2vQPRwQjv9Qmfq040IPO01nt7nIBnw8fggSDio1IKtFZobZymuAfR3SxPT8QwpsVYD41pMdZnjhQGlThJykCCceQWl58oSCw6nX0vQyjwQZBwvmEGQJBsFrGsNoG3HVRpgh6OXBJ8YhPjBCI65YE1bqxxwLoH3xtcjnj9LK7NhDFeEz5IhbpNta67fcKfS9MjDU73XZMmrh2EDGOTINrvZnxYY/xMWpA+yawxTq9+uv9tZkcOorcBw/aKdgt4036gEfp5A5574fqpG7fBXQPgG/S32+BvADxtCOq4beJK2/rdJCQVLTA/XvBYZUNZjkGukPeuQ8kfMuFqJmaAkttQWYa1Cwi1nZo+Fk2SaFIh0FL6RFQGbWqoxiSmJPWJU0st0dZgZIWgxlJTGwO6pG/GzNgxmazArFOVQ4TsUbGdws4ifMIu4aNA0mfDtCEARdOjQzqvQJyOgz93wp3LFNrExPZrIfMx9OH2s74Nw1WVEZRuX4xu1E1YMdU5GosPTJo+sWGbzXZs95ewRUDMRfc1IlpVFK1/NxQnNx6tHdRpEpbGoForkOCuRdw+AvFNeSKW11QqRGalvwdE0E0VAcPw02eqEVm9zECsk2HoK01fgrZ91nROZSBXmvlkzEDuBSlZT27GiGyz1tuyLduyX3CrSVgdFZhaIEzFZFxQ6QQhLOPxkHwwxJiSfj9lMugxrFaZGfRZXVlmMNOnl6VIqbHViNGkdnrgVYlKLJNJytpKD5WlIBVCJGTZPMbkKCm9RErfsTx8UlOwlOWY9fU1VteWWVlZYnl5kZXVFcbjEaaqmen12bZzgR3btzE3O0u/3yNLXJ11XbM+nFAUE8qyYjwqfN6IhNn+gNTrkyfKSbBkWc/t75dba1lZWeG6665n7969WGtjslApJYPBgLqune/gpz5qXbvknnQZEPH94B3KXt4jSxPqOkPrmqKoKMua0lQUhWaiawwalCWRkjRJyVRKkij6WcJASQx95q1hUtXUuqIcjdFaU1c1ui4YDyuK8ZAkzVjPeuS9HnlvQJpmZHkvJlZyzu/mrHQpHLwubPCRfOAXsJgWo9olC1JSoEwzoy+8x4yt/cyp1vs0jFiOyBqplM7Sg7HQNyvWj3aCMw2Nbykg+nxt368VCvcuhh+c2ibRURiYNcduD15qN0DEBWyE1X66tAFrXDImXVGVBaZeQ4kJaWKohQHPbtKmQvQqsq3495Zt2ZZt2U3eTjzxRE477TTe+c53AvBP//RPnH766VxyySWcdtppP7bjfPnLX+a3fuu3uPzyy7n5zW/+Yyv3x2lVVXGLW9yCF7/4xTz96U//aVfnp2NW0kYOuiB68I08OJy4ZIrWA35um6lZfFZgjQDVXT4NVLet7TdZA8bqVn0CuNp1vDqyKMZG/5GWjnsDljcBgXCOXUcugPtN4tPp2Yld3XO/MAKLouOTC9mqQ/jPBsJD9xwEEuHzNQWWvdvU420t7XUbAgCCDiA+zazXWnfa2THKmxmRbfwzSaXXp08IzG4AowM215RtjKZt1jptdgMoIzCmxtgKSQLCCesbP+NU1zW1Ll2mTgKA77x5pQTWSjfGMaphoGNBN5I1YTYD2KinH84jYJNB1jEcI+jSh2O59vXUZhuEcdxHIn3yVelmMxhBoryP7UlK1pcfSEENgG4RJC0g3eJkcwxCGkeqkV4y1DYzcIX06iRBqgbpObZ+W7nxfpm2zbDMzTcUDTgYWbdtDDt8C0gQJDjJigQn1aJC5w8REP9buPsdJZxuq5IW5aMwHY10ghZS0E9vjt+A5e6EEn9iAaz0azHWUJsKbSq0LhG6AKuZVBWTYhljRqQypa9KMlZQEmoxz9hsp7ALWLETK2adbpCogRqtS3Q1ZMaO6VMi6hJjLBOtGNsZqmQWmWQoKTC6JK1X2CWW2Jbsp6f20U+WMbamqCXjSlCSomUKrSnV7tNEqJRwt72K10PEAX98+GwA0IVXq3LXpR2wiMpWcZcA1bcedq0bIV679oNwqq81QHK3s4eb7bBdUzR1awPlTTisOWxza/tPa4EI5yhC9M229mrXFt83bSwvHC78lq2ymnWhTMdWl7Ec4iBc4MfmQZOsddOEeABGoyjoiQlzcsgCS8yzn769jp65jhmxyNHqGvbI77LD/oAZey1KX4Uqv0te7yWv9h2uRX+u7BWveAVCCPbv3/9Tq8M//dM/IYTgn/7pn35qdfhx2IknnsgTnvCE+Huz83rCE57AiSee+GM/9tOf/nTuc5/7/NjL/XHaJz7xCWZnZ9m376Z1DwWbmd/BpLIMJzVWJMhEgdUUkxGj9WVWVhYZj1YYDZeZne0zHK6zvHyAuioQVpMlgiwRYCuUNGSZQgmNrsdMxqusrS4yWl9GVxOEB5yVUORZH2xConpuemRdMRyusvf6a/n+Fd/lO9/5L/7rP7/Bd7/zX1x77VVMxusM+jnH7TmKW550PLc44Vj2HLWThbkBmRJU5YS11VUWFxfZt38/S8srDMdj0jxhbmGOo47ZxZ6bHc1xxx/DsccexdFHbWf3ru3s3DHLwnyP2ZmULEuibvnMYIYdO3ZwzDHHsGvXLo466iiOO+44jj32OHbu3EmSJpTlhLX1VdbWVxkO11hfX2U0HrI+XGN1bYXV1WVWVpdZXV1juDZiPBwxGRdOS7GswFqUVGRJjyzJUTIFK6lqw3A8YW1tjbX1VcajEZPxiHIywZqaREn6/Yy5mT67d23j6J3bOWrXdnZum2OQp6QSbF0xWltm5cB+lg/sY2lxH4v793HgwAFWV1eZTCZorTuskQb89+/94ISG95UQ0VEV3lm21lBXFXVZUJcldVVS69oxOLxj24rru7Kjj9f9bLA4YNk4GDmi/f127fODQ4DwBL+m+QbPRvEzI8LH6sovK9B1QV0VVOWEqlinLJYpi2WKyRLF5ADF5ADj8SKT8X4m40Umk/2MhnsZDa9nMtkPcky/J+j3UvI0IVESBRSTISvL19+Iu/qmYU94whOYnf3Rmfjf//73EULw+te//rDbBt/iF8V+Gue79d7/xbRLL72UV7ziFSwvL/+0q/ILbS996Ut59KMfvQFA/8///E/OOOMMZmdn2bFjB7/7u797xP1/Mplw3nnn8au/+qsMBgOOO+44HvGIR/Af//Efne2uvfZaXvSiF3H66aczNzd30DFUmqY873nP49WvfjWTyS9mcu0g22IMnvlqI/NYa4PRjhFb19rPajRobdG6reUdQO7wHTStPRuXBnxsYyHNx+1jjYiMX61NBPSjX2bawDix3mH7NvBvrXSk+U18uzZzu/nbdJYH0DSwgdvbN4B2109sW1NnG/W0Q/0CMI1vK4efOYm+iE95NnIMaNAQQQOS3jDEm/oeCXO5wdAERhOvsbuuDfu5HTRwCTo9PiWdBItUkKSCLJOkmSRJHSAufELRoBtvjcvbWNcVVT2hqseU1YSyGlOWY4qyoCjHFKUjJhXFxC0rxu5TjqmqCXVdYozTHTemRmuHo2pdUdcuP5Rbph2gHyQjCW3uryMB0ZUO7EeClZggDaQSpEpQaeq+ZUqiskh0UkqRpilplpJmictXlaVkmSJPW58kIU9TEpWiVIqSGdJ/hEhApiAzhMwQKkfKHoYUY1MMOZYMYw9PTr0BHJgmuhMB0hYbOG7lgdJGp6ZZ3jCKHYupw24PHSYWGFOxAo1USyirhc96gN0BywEEbersLpcFJlVNWQyRpqAnDH1RMSMmSAzDyqDtGrXWpPTJEijF0EVBhEv4YEQfIRMyJZHKIOwIXU/I9TrWrlOKGp0OqEVFIiQGhRAZWZK7ttBjZs1+tqtrWMivRtar1FahxXYKfRTreidjuR1kGkFbEx8OzaDQ2nCebUDZ+iiWB3Rb2zvwvMU0DwzyFoU7tlkXse6041Q1NuDQYpMfYmrDNti9qVsvppe7BW0poPa+7e8g29M5x4OOHdqRwo3nFOI92BBAaoPfLVmZDsjemibTOW4zTTwmtfXXUEovCVAX9PQqcwzJbQGmxgqN0AXWTpBop3lVX0Vdj9HJNoQssWYdxffpJYePmG3Zlv0k7YorruCv//qv+eQnP7lh3f/+3/+b17/+9VxxxRUcf/zxPPvZz+ZZz3rWEZV7+eWX8/KXv5zPfe5zHDhwgBNOOIGzzjqLF7zgBQwGAwBGoxHveMc7+OhHP8rXv/511tfXueUtb8lTn/pUnvrUp6JUk8zvjDPO4Ja3vCXnnXceb3jDG348J/8zZEcfewLX1hpTrIBwyTNzqxmPVtm792rE0iLzC9upq4LajMl6OaOlIXUxpsgTepkiTRRWVwgpyRKJnOmzvl4wGY9JEoUVAqkysBVpDgJJVRr27V3C6j5KCqBkefkAi4v7WV5doioLhLD0+hm7di2wsG2e2dlZZrKcTICtDWVVMB46WZNJUVHVtWOOpymz83Pkec7c3AyDfp/+oEeinLagEgI8EOzeXQ4YVWqWQhukUMzOzpFlOXNzc565LcjznMFgQJalSFlHzfMsc86U01F3WoJGu0zv1low0g8ytGNy2zC9Uzi2kXHzwJRMMFiqusZoQ1WVFNYyEutkaYpKU5IsRWYZIlVIIegriTCWPFX08ox+P6eqDVWlGY0n1HXNZLiGthYrUqRKyfOMfr9PnvccM9+3S5KmUT/R+Ki2CMzxwCzxf6tERfkaXdcIXYOusEpRC0NtEsdYD+1ME/TeYGITYNv692p0tDurnHYi07vYjQF77wy2BzM2vmSJrKIAm4dk3NigDQnW1lhdxYCADT6ldOdvrPEDS4M1FdYUWKsx1ripvkaD8NI2OLZLVRWUxZhqPESXJTDBmgllMaEqSrTWDEewtr54uFt4y24C9ta3vpXBYNAJaN8U7af53gf41Kc+xd/8zd/wpS99if/8z//k+OOP5/vf//6G8m7q7/2fhl166aW88pWv5AlPeALbtm37aVfnF9K+9rWv8elPf5pLL720s/yqq67i7ne/OwsLC7zmNa9hfX2d17/+9Xz961/ny1/+cvRxDmaPecxj+NjHPsZTnvIU7njHO3LNNdfw53/+5/zmb/4mX//61yNg/61vfYvXvva13OpWt+LXf/3X+cIXvnDQMp/4xCfyohe9iAsvvJAnPelJP/rJ/5xZSCDqgGhNXTfAcTuJJIA2EAGHiL2F2ZIOmAzSMMZYL93RHOtg+uLYZn3DHG/5TMZG+Q1XAxHr3gDHLTAJGdmSbfC7XY9pskQbeNrA4o5tFWReiLkWm2MKjz22ZXAccKtt7dnl3WBD0B13Uicttn0H7LcNnim8BxkJoQ0gP62JHlQv2m3cIHANguV8SxD+3EA4hYsWyxuc/xlni7ZwUAReWsZ47MmAFSSJAJSbJKo1QvtEntLE8wt9S1dVJKJYKzFWITQIqZE6kHkd8UhJ5TTUowaGax/jAwuW5hpJKbBaIK1stNWtRcosNorsBDWMZ+VLn3BXgHESyAiLtAnGB3oCITvgykJ4tQzbZevH/hHHCFN9xqPDUjgpRon1szHwvvyPVc7FRig2dgHZGs/YZvAkWn90wE6/Qoq2TrmI+0Rmb2vf8G+4ZGGs14D4IQHURhmTJoew+1R1iRktkpeLKFGR5ZJMWYQdU5s1RvWIxKbMZilZlmPlHCt1RS00ibIYalCKFMcWS61FyQKdjBgnNSu1cAlFlcTKDJnmKOVlXXSNqof0WUOJ67H1D7A2JU3mqExObbczUbuo0gXyvEcifaDB4jMChxucdp9o2jbcnCJs1FrTYonF5gmM6PYz1bYU7Tc8azsH29zCulbEbopHTvs0NgLyrcBAa/k0YC5bKLrsbDoNjHcfYhuKjgUfBICOVW814BRA7nNfu4eYXyniFBCvmaUhTPNGKTfg1iXC1giVIowlqYb0GZHLMcoOXbI9n1lY2DVkkqBrMMUq49EVpPnxmGwbUs0iVUIvqTY/hy3bsiOwt7/97UcYRT9ye/Ob38xJJ53E6aef3ln+V3/1VzztaU/jzDPP5HnPex7/8i//wrOf/WxGoxF/+Id/eMgyf/jDH3KXu9yFhYUFnvnMZ7Jjxw6+8IUvcO6553LZZZfx0Y9+FIDvfe97POtZz+Je97oXz3ve85ifn+eTn/wkT3/60/niF7/Iu971rk6555xzDi94wQt45Stfydzc3I+1HX7advTuYyiHayxeN2Y0mZAnEuoCXWtW9hpEPkNdlWzfuRM9EaRJjpQpiIqyMgxHBUoJismYwUwfY6QPLDqgXGvHgJhYA30Y9HKEkayuLLO8uM6BfftRCoStGE9GVFVJkiTMLcyysG2B2dkBc7MzZHnmHO+6ZjwpqCYFw/GIsizcoEAp8jyn188d2D4zQ5ZlzPR7TnJEuXTNUsrmVeQ8vJZTPKKqCiwl/ZmM/kzOYGYOU7uBi5BuWqSuIM0liUpIZEKiUsBSmdo/z3EsEuOHLiI4lD6I7cFXKSXj0YT19ZFjpguXuDTpJWT9HmqmjzWWqiypyppJNcGOCoRSSM9YLlNJqpQDwz0bI+kp8lyR96CqaiZFQVnWFEWNqQvGpaAeD5lkGXnWJ+/1kDIwNhywrlKJVBIhXUDJWKckqKTAVjWJcE5lolKwhkQ6h98KgbHC6b6XE5QpSGyJtJnTJrcaa9z7yAZGlPFTO2kxjwwIXJtHzXEvhdL1YPw0UWNckKI1JbbtdIQBYCjfT9IlvPxDYKOua89QCmC7AGqMqRxw75lOIEiTxE2ZNY4V5tg0ldvWlxdY92H2pEuCZJB1hSwLTDGhmoyxdUldTiiKUVMHa6n1oZyqLftx28te9jJe9KIX/cSP+9a3vpVdu3b9xEH0n/T5/jTf+wAXXnghf/M3f8Md73hHjj322EOWe1N+72/ZL6a94x3v4IQTTuCud71rZ/lrXvMahsMhl112GSeccAIAd7nLXbjPfe7DO9/5Tp761KcetMyrr76aj3zkI7zgBS/gda97XVx+6qmncs973pOPfOQj/P7v/z4Ap5xyCouLi+zYsYMPfehDPOIRjzhoudu2beO+970v73znO38hQfQAeAfmudYao22L5d0Ckb3f057l58DcBkPrltuA0FHruos+x3UHB7Y9oaGB5SJw2nycb9XA6y1glKZ+TT6aNiYUQG3bxtFDzZheGEHqCCYHYDkApMLLnoQ6OT+9W4dwMjYexnoZGuG0WiLxIvj0sT1EGCNb/39IgErrfAM5w7ZaI9SVTn0jmTUmqw9a7i09ctxYy51XAPVtRMuChI1LLJoifZLQdn2cT9rkB2rPAnDXVXq5x3A+BtHyb0H7IIVx+TpjvMED8aGfCjf+ksonprVe+SIkzTWWJErBWLQn0AjfL4yUqLBM+LyMAj9GEAgT5IzaihDev0cj0J1eGC30FZ841kcfmqBIS4ra4u41rd09dDg7YhBdNVczdoiYn9E2IGMcsrTBUgKT2EcoRBh4NqcZ5HbactVhcEqMF4SHRwP0SgLb3G8X6MO+blEhxxhkVWJHB6hXL6eya/Tn55GDnCRZx1b7mJRjenKCyHuk2QxCzTGjUsbaIERNP02w0mV9taQoNIp1tFjHohkagdIDNAqkxerM1cUa0DVWaybFOsL8AMtV5Mk8Sg4wYgaR5Ag5YKbXJ0lShJAuQuWB9MCEtjHVZXfgFaZou81C6MGvk60HY/s5tsFiw99AEw12P3Vztbvy9KOzDZB3NpoqdxrLdqfh+kTo4+3ksQepYvfndACB0M7N8ljH9hSM1klGyaBYMWLflPFBbzHCoGwBtibVGQJDWi7TFyOkUVRGIqjJlEtAW1Wr6GoFhaBiSCbXXTeSGZPxGvWkwtq9GKMR6TZ6KWT9rYH4DTVrLZPJhH6//xM53nA4ZGZm5idyrBtqaZr+WMurqooLLriApz3taZ3l4/GYl770pTzwgQ/kQx/6EABPecpTMMbwqle9iqc+9als3779oOW+5z3vYXl5mc997nPc5ja3AeCpT30qxhje/e53s7S0xPbt2znmmGP4+te/HrcBN2B+0pOexDve8Q5e/vKXc8tb3jKuO/PMM3nWs57FBz/4wZucM79jx05MOaYYrrG094eYyr1FRF1iGYJQjNbXyPIBs/2cLO2TZ33KsqaqLDqTaGMZDgvq2pD3MpJEMRyVjEcVWa6wvRRJTioHKJGhK83q5ABlqZEqJUsVWSKYGfTZsWs727YtMBgM6PV6cVZANSmZTCYUk4KyqBywXFcoJej3B8zNzTIY9OgPep5lnZNIhfTOYfBvrTXRwTStKahgkaJGCu0yvtuaqrbY8ZCqqlEeNBVGk/RyrM1IVcpgMEOe5w7gz/17OLJwbPRJTGCfW0tVVRTFhNW1Nfbt28u+vfsYjYdI6QIBM7OzLCxsY35+gSzPyPKcRDn2RV3X1LUHbQmgr6HSE4QQDkhPGrZHL5PkaQ9rexQTQ11bN/W3rtFlzaiYMF6XSJWSZj3SLCfLUrI8IckUMukh0gyU8zuE1WBqpK2xdU2apKRJQoKlFhYrE5RISIykLkaMVhcZry6QCENZGYqQt8Yarw9pvLSeAGPiVM9wYYRsOfdx2q3xb1r3XtNGY7XGRi3y4EM6z9J4vX03/nGBE3SFxCBkTKFOnEHQmnrr3u8aP/84DlCFEJja+1vWIjzALh1FLJ6DkNLp/1uvH6l13F5rQ4IgVYr52RlSqZgZ9DHGX9fNHbGbvP0034VJksREwQczYwxlWdLr9X5CtbpxVtfuXtqMRRra+EjO94bYaDTqML/b9tN+74MDC9/+9reTpikPetCD+MY3vnHQcm/K7/2ftL3iFa/gla98JQAnnXRSXH7FFVdw4oknUtc15513Hu985zu56qqr2LNnD2eddRbnnnuue78Cj3/847nooou49tprN/ik973vfbnyyiv51re+dUT1ecITnsCHPvQh/v3f/53f+73f43Of+xwLCws87WlP4+Uvf3mH5TocDvmjP/ojPvCBD7B3715OPPFEnvKUp/D85z+/s92RnAO498yrX/1q/vIv/5IDBw7wG7/xG5x//vlH3JZf+tKXOPfcc/nCF75AVVXc+c535jWveQ13u9vdDrvv3/3d33HPe95zA+v4wx/+MA960IMigA5w73vfm1/6pV/iAx/4wCFB9LW1NQCOPvrozvI9e/YAdMZQNzQYdZ/73IfnPve5HDhwgB07dtygfX/ezYHnPqBvLEZLz0R32uZuYpxnNougN+1msCsVpPm0C+DLRiLZybM0gHcDzLSSL3qAw82cDLl+ZLPSCqyxKC8lTMAzjIkSKXVlqStLl84YEk3ahgHuFmNooyd+axtw5BbUHDB1749Jv8IFDiwmoJABmomBhcbXD0dSoofFyUziSRjWaoSyWKPRdYVSGRKFNYaUhMqUrlyp0N7HMxjCxADvAhLJqB5tDAC1EZ726uvbeLJuW+d62pinKAQ8HPjtvFGtvR/tWaMGGUkkDSPfOlJp9JYdmxyr/CzJRufdWFCkfmaBibNIQ8LOyhik9CQkD8SGPiYk6NqCsjQnYmKdA1HJBXVcf5CeHCOsdXiYV24RlK4Bsa5tfB8RQdbFurKt9frkSiAw6LrAmAohDIkMsx0adrljwZvYh5vAhu0g6lK4vJAm6j4LrPEsfAHaWmpj0O375hB2xJrooSEdCO4+QZc7ZpMVTq87EV4HHa/dLdw2iWz+DlpMwg+4IjsdWsuCFnrQcnLa6YmARFhUAPFp63TTfIcOay1lNSHRI2YpsNUyw9WrKYYHsGadRIzJlWIuH9DPMoQwGF1QVC4Bm2NV1UghyZUkVQlGlBhbUZVDqmpEWSwi66tJzF5MtQh24jqxdbpAtampdc24SDkw7rN/tMpqtcLElk6yRUjfoSzCitgObS34wLj3fbH5eCA3tLXy24gWqNtcSPzDaOoDdKVwWiA8G7dtfjcyOxCicF3gPFw7vAaXFBYVI4ndg3SOuXF1a10TSogyKpv33KYntM5hA6gezr9Vgc52vo2Fr6PTPxdxv3BuYT/l+6cUoIShZ9eZMyvk9RIDvc6CGLJDHGCnuJajxPVsF/vJ6hXKqqCyOZBRluusDa9hceV6lg9cw/LiFayvL7opX9UKZfltxuOvURTfB33tQc7/59v279/PIx/5SObn59m5cyfPec5zNujn1XXNq171Kk4++WTyPOfEE0/kJS95CUVRdLY78cQTedCDHsQnP/lJ7nSnO9Hv9/mrv/orwE11fNjDHsbMzAxHHXUUv//7v79h/2Bf+tKXOOOMM1hYcEDgPe5xDz7/+c93tgk6pN/85jc566yz2L59O7/1W791xOc9mUx4xStewS/90i/R6/XYs2cPD3/4w/nud78btxkOhzz/+c/n+OOPJ89zbn3rW/P617++FcA6cttME90Yw5ve9CZuc5vb0Ov1OProoznnnHNYWlo6bHmf+9zn2L9/P/e+9707yy+55BIWFxc3JBJ6xjOewXA45KKLLjpkuaurq8DmzrxLYunAhF27dnUA9GD/43/8D8BpQ7btqKOO4ra3vW2H0XZTsSRP2LZrBws7toNULK2ugkgQKgUUxgiybEBVG9ZHY1bW1xhOCupaU2mLtpDlA+YWdtLrz5MkfcbjirW1CXUtkeQIkWFtQl3b6LgppRgM+mzfNs9Ru3Zy7J49nHDCCdzsZsexY8cO8jzHGMN4PGZ5eZnFxUWWlpZYXVujKAqEhJmZATt27OSoo3axffs2ZmZnHXjuQaH2VNLIognBVyFb0wMlUiqkUiiVIKygmBSsra6ytLTMgQNLrKysMh6OqCoHTEkhyfMeg74D0bM8o9dvAOheP2Mw02Mwk9Pv5wz6Ob1MIdCMhqscWNzLlVd8l+9f8R2u/P53+eEPruCqH36fH/7gCq78/ne56off55qrf8D+fddTTIbUdYEQhkE/Y/vCHDt3LrBz+wIzgzmyrIe1kqKoGQ0nDNcLRusVo2HJaKipSoHRCYN+j/n5Adu2zTC/0GcwSEhSg6XAUqD1OsVkmbXV/SwtXsf+669j3/XXsH/vdSzt3+v01Rf3M1xbZrS2ymQ4BGNI0hSZZAiZIjzTRQDD9VX2XnsN1139Q/Zffw0H9l7N4vVXs3LgOq+Vv8Ro7QDjtWWGK4usr+xjuLrIaPUAw7UDjNYPMFzbzyh81hcZry8yCp+1/QxX9zFa2894fYnh6jLD1RVGa6uM19eYDNcoxkNMWWCqAlsV2LrA1iVGl06z0Ws1On3zkqoq0LrE2hrr9R1dgCX4GDZ+jNEYrT17pcUpCkwwHEFDa01VVdR17QIHUxSWXp6zY8d29uzZw/HHH8+JJ96cW9ziFpx00knc4ha3+G+9/2+IXXLJJQgh+Nu//dsN6y688EKEEJ3p+Z/97Gc59dRTmZmZYdu2bTz0oQ/d8Gy9oe/Cr33ta+zevZvTTjuN9fV1AL7yla9wv/vdj127dtHv9znppJMOCnq+7W1vi77Ane98Z/71X/910/q0TQjBM5/5TC644AJuc5vbkOc5n/jEJwDHwnzSk57E0UcfTZ7n3OY2t+H//J//c4hW3Ggnnngi//Ef/8H//b//N/ad0047La5fXl7muc99bnyX3/KWt+S1r31tZ3ZYW/f9TW96UzzHb37zm4ds44Npor/3ve/llFNOod/vs2PHDh71qEfxwx/+sLPNaaedxq/92q9x2WWXcfe7353BYMBLXvKSg57nT/u9D3DsscceMSngpvze/0nbwx/+cB796EcD8MY3vpH3vOc9vOc972H37t0AnH322fzRH/0Rd7zjHXnjG9/IPe5xD8477zwe9ahHxTJ+93d/l8XFxQ1SQNdddx2f/exneexjH3uD6qS15owzzuDoo4/mT/7kTzjllFM499xzOffcc+M21loe8pCH8MY3vpEzzjiDN7zhDdz61rfmhS98Ic973vM65R3JOQD80R/9ES9/+cu53e1ux+te9zpucYtbcN/73pfhcHjYOn/2s5/l7ne/O6urq5x77rm85jWvYXl5mXve8558+ctfPuS+V199NT/4wQ+44x3vuGH53r17udOd7rRhn7vc5S589atfPWS5J598Mje72c340z/9Uz7+8Y9z1VVX8eUvf5mnPe1pnHTSSRvO/4bYKaecgrV2g/zML4IFRm6Q13CfANJaF2RvAYUbC2gwGNH2X2wAET1aEn8HkLktX7JJobYFfLTBkk1Y6w2bvvk4jW+DCTrfutF4b7TcWx+I50oL3IzHjNIaXoM8lu114z3pxGg79fF1043GeVP31sdXJAKi04lUW80U948ga7eNhBDIAJa6qzvVyKLz3W4/HdrJuHOsvcSP+7i/2zI6jVxOU04A+B1Ybv3MBosN7dXRlw99sOknjSZ+q68Y0TDs43nK1r6thtrQYu19HF4XMV8hUD4gpBKJUiEY5PHm0Iwi6OBrrNVOe91UaFs5Uo51Gu21rql0RX0En0oXnb+rusDgiEvG9znz45RzaXywKWizxUJvmMwCx+oJw9kAMnrIO7J1w4A3LHPbBlmM5mhufwUNcBnKlU2N1CbAqMVpWCZ6QiZGSFFQC0UpK6QcIUXqxOeTnWitSChAQFkNqbkOLWoSuY1e2iNPQciKGgl6iBVDar3EaLTGaLROIhXbexkTW1IYkHKBShgnuF9NsMUaph6SCoWSPVCGRCm0nMfKGZRIqKzGYEhQKDq3ebyv3XPXtZftXJsmcMAm16vTbq0rJuK3AA/Ah9+2fV3blz0+4MJDoluH9uFb8wjcdW/+3Aikd2pMJ3oZgizW9xW1Wd2marJZ+e2fMnLa/Bm1gwShjBAWJQDoAVBvTjb8HdowlGUt6LqmVw+ZN4sgJZlM6MsJGWtYM3SJ5KymNBmV7lEyQ2UXMHoVPRqyVg4RpmAy2YtknZm8x9xcSsoKtV1nZA+g1Moh2uHn1x75yEdy4oknct555/HFL36RP/uzP2NpaYl3v/vdcZuzzz6bd73rXfz2b/82z3/+8/nSl77Eeeedx3/+539uAAO+9a1v8ehHP5pzzjmHpzzlKdz61rdmPB5zr3vdix/84Ac8+9nP5thjj+U973kPn/3sZzfU57Of/Sz3v//9ozMupeQd73gH97znPfmXf/kX7nKXu3S2f8QjHsGtbnUrXvOa1xwxuK215kEPehCf+cxneNSjHsVznvMc1tbW+Md//Ee+8Y1vcPLJJ0fH/5JLLuHJT34yt7/97fnkJz/JC1/4Qq6++mre+MY33ojW7to555zDO9/5Tp74xCfy7Gc/myuuuILzzz+fr371q3z+858/5ED10ksvRQjBHe5wh87y4KxPO/OnnHIKUkq++tWvHnKgdNppp/Ha176WJz/5ybzyla9k586dXHrppfzFX/wFz372sw/LbrzuuusAB7JP2ymnnMLf/d3fHXL/n0dL0oTB7Cy7jjqKlQN7WV1bYVJZkl6fJJ8jm93GjmNuhsgy1lf3uSeqlFipkGlG3pvBWBjMLjjZlSz1iSsdE2F+2zbm5uZJ0x4qyTDGkqYpCws7mJ2bZ3Z2jl6WoSSRdT4ajSjLkrquqaoqApAAiUqYmx0wM5hhZmaGXi+n18tIUydlIrzUWQCGlJResiwESxsm8bQWo1ASbZzWdXgDZElGPxuQJopBr8fMoE+vl9Pvz9DP+6QqiQyYRLlpldqzUIQQGKupypJiMmJ5yQUDrrv+OpYOLLG0tMRwNGQyGbtXttGM65LxeMhwfZXrrr2aPM9ZmN/OvA/KzQxmGQwGpGmKUAmp6iGSnDTJqFTlE0zVVIHxLC2JApVYTGZJEqdfKPLEJR9KoKxx0i0C53DWhrrSaA21hdo4iRYjBApDIgyTtQPsX9xLVZVR8sUl8FRgBZKa5aX9XHXl91hbW2FmZh5tLbNzM2zfNsvc/Dy9LHNsHz9osWFaadQ1FKjE+ZDCD/5Ey+sJzH5hwzu18Q3bjriMA8noMWLD9MY4SNVeKkZPuQJBr7EZIIX+E5hCna1bALoQIgKd4butPeoYXm6qbZ73cEwhS9TVb025/lmw0047jeOPP54LLrggBhyDXXDBBZx88sn85m/+JgCf/vSnuf/9788tbnELXvGKVzAej3nLW97C3e52N/7t3/5tQ1D2SN6F//qv/8r97pkbtxIAAQAASURBVHc/7nSnO/HRj36Ufr/P3r17ue9978vu3bt50YtexLZt2/j+97/PRz7ykQ37X3jhhaytrXHOOecghOBP/uRPePjDH873vve9wwKrn/3sZ/nABz7AM5/5THbt2sWJJ57I9ddfz13vetcIsu/evZuLL76YJz/5yayurvLc5z73iNr1TW96E8961rOYnZ3lpS99KdAAwqPRiHvc4x5cffXVnHPOOZxwwglceumlvPjFL+baa6/lTW96U6esd7zjHUwmE5761KeS53mHvXmk/sarX/1qXv7yl/PIRz6Ss88+m3379vGWt7yFu9/97nz1q1/t6FkvLi5y//vfn0c96lE89rGP3QBkt+1n9b1/KLupvvd/0nbb296WO97xjrzvfe/jYQ97WOf+/3//7//xrne9i7PPPpu3v/3tgEs+e9RRR/H617+eSy65hNNPP5173vOe3OxmN+O9730vD3rQg+L+73vf+zDG3GAQfTKZcMYZZ/Bnf/Zn8ZgPfvCDee1rX8uzn/1sdu3axcc+9jE++9nP8sd//Mfx3nzGM57BIx7xCN785jfzzGc+k5NPPvmIz2Hfvn38yZ/8CQ984AP5+Mc/Hp/vL33pS3nNa15zyPpaa3na057G6aefzsUXXxz3Peecc7jNbW7Dy172Mj71qU8ddP//+q//ArozAcAl+4SGOd62PXv2cODAAYqi6LDp25amKR/+8Ic566yzeMhDHhKXn3LKKVx66aU/kv59CCJ/85vf7FzzXwRrJxJtf1sTgHMLVkbioCPZSp9oEkdKlC3gcQqK2wh42yn/mO4O8bvF8LUN4dpOlTktCxKOEYuaOjbQ8ami7+Mo10TkyWMs1jo9bL8xYda/aLHbA0blqjCtid1IlASZv4Z0E0ifDsM67Oj8CMbvcQZjG3xt4ZzT7+T2bEhrrctVZNwMU7eiuS5BngZC4lCBEi74EKRSnH9rAZ+k1ufycWMGTWCAtzXvGzA86KS3pVKa+hkD0lXO168po9GOdyC+le3mahBGGy+l6HxAIKVCycSRdLwYo/HYr2s3L79ojGf6i4jcCeskbRoirIgdV+uWBI+vjxu+Bdkd4YMOblyla0tVBVmlw/vmRw6iy82iYdZ3kAAy0jSkFS3xfw+mB/CxtcxFJnw7B5YPrgzROZLwsjH+V2j4WIcGI3Ui962bpRrSr5bI9SpGjCnViFTsRdmRS+Bo5kEolOpj9JDaWBKVYCxIOaCX76af9+nnoOsKzBLz+RKCEZoxthxRyRH9fJ6sN2ZOzTCpBUNRM8ZQ1hPsaB/J+EoW1FXM9odIMYNMS6eZLgTSCnJpsMJFWRKpCMzs+PDC3ezCBkDdIgPQLXDt075moT1C+7T2i9cqbBkfvqIFd3ch9wi0d47Qvk5dXXoRAiLTdRKtcuMfB3lAhQdnC+j2ty+Gg02laMn6TDdGy2QM2MQeSgO8t/o2Tf8KYH5Y3wDoMa2HT3BLE+wwFqENqlphNlkhUxVCGzAFxhZoM/b1SUjZTU0fKxPqukKpPmk+x+rahEk1jynXSKQiK/sYW4C0yN7JVPZWm7ffz7mddNJJkSX0jGc8g/n5ed761rfyghe8gNve9rZH7NgG+853vsMnPvEJ7ne/+8Vlb37zm/n2t7/NBz7wgajf95SnPIXb3e52nbrcGAf3dre7HRdeeOENOud3v/vdfOYzn+ENb3hD1BkEeNGLXhRfwkfq+N9Y+9znPsdf//Vfc8EFF3DWWWfF5aeffjpnnHEGH/zgBzvLp+2//uu/2LFjB/Pz853l1157LUopjjrqqM7yLMvYuXMn11xzzSHrdcYZZ/CqV72K17zmNXzsYx+Ly1/60pfyx3/8x4fctyxL3vSmN3HSSSdx5zvfecP6W9ziFuzfv5+9e/duqN/Ps2ksIlHM79jOrmOOY3VtjZXlVWZ6KTu272L7Uccxu+MoaivBFFhbkff7XH/d9SAE/dl5l3jFGNJ8ll7eI01nUbLvAMIkQSWpZ2vn7Nq5i9m5Oebntzm5lsRpaldlyXB9jcmkoKoqrHUslcAkXJifp9/v0897DHp9+v0eeZ4jlexMV51+VyTe0WgDm5slM8LvKqxACEWW5iTzfRYWtqH8DLM0USTSzairq5LhcIQxGpVIf2yDNk6qpaoL6srpvK+trrCytMSBxUUOLDnwfDQa+YSlkjSRjs0tfMpvIdCmYjgcM1wXrK0uI69RpElKr99nMBgw6PdJspyZ2R0M+gMGgxnSNCdNE7KshzHWByA0k8kEYwx5D5JEIJVCSOnYFB6wTpQLqCjlBhbVpKYqNUWlGZdePqeqGQ/XsbpiMlxmfW0NsIgkpdZQGxzLRgmUUZRFwfXXXs3+vdeR5z2USjh6z25yeRyzPYFIZxDWTf11M0Sdky/DTMQwfVQ0TkGYzWaMjfVXgLASPwvWh75NzEmiPPsnBLMt1iVa9bIwTkfRJYINU1RDv4g67D4Qo5RLqCql7ADj04PBUEbYrr08gOtBFz/0yahNKcLg82cLRBdC8NjHPpY3vOENrKyssLCwAMC+ffv41Kc+Fd81AC984QujNnUAch/2sIdxhzvcgXPPPXdD3onDvQs///nP84AHPIBTTz2VD3/4wxHQufTSS1laWuJTn/pUB4Td7Hn/gx/8gMsvvzxKe9z61rfmoQ99KJ/85CcPC9B861vf4utf/zq/+qu/GpedffbZaK35+te/zs6dOwF42tOexqMf/Whe8YpXcM455xyRHNzDHvYwXvayl7Fr164NQOAb3vAGvvvd7/LVr36VW93K+XHnnHMOxx57LK973evibLNgV111Fd/5znciw7dtR+JvXHnllZx77rn88R//cYdV/vCHP5w73OEOvPWtb+0sv+666/jLv/xLzjnnnMOe58/ie/9wdlN97/8s2T/8wz8AbGB2P//5z+f1r389F110EaeffjpSSh7zmMfwZ3/2Z6ytrUVpkAsuuID/7//7/zaAw0diz3zmM+PfIRh20UUX8elPf5pHPepR/MM//ANKKZ797GdvqNuHPvQhLr74Yp75zGce8Tl8+tOfpixLnvWsZ3We7c997nMPC6J/7Wtf4/LLL+dlL3sZi4vdhNP3ute9eM973uPepZsEd4G4z7Q00njsxpibgeRBsmo8Hh8URA9l3v72t+cRj3gEd73rXfnOd77DeeedxyMe8Qj+8R//8UZLX4W67t+//0bt//NsgRkcGNJd2qQXQZZejcADp9KDmM3vNijZxXqnwe42cNn2SzZaG0C0PrGo2FBm2++ZXhdA9Pby8Pc0ESGAQV1ViQCImjhLsMHcZYNLHQLcjuCr2QwDb4P9rd+bbBXB9k5Q4hB+m2iwMqfnLYmZHaeuCTRJU0PSzBAUce3j4wsIQqJLa914CB/gcGmHLNoE7NOVpY37GGPculawI2CoXbC7SyQxwrVfkG0xxgtKi+44qwHlm6CL86fD362+JOJcYX+uMh5fCBGBdDczwYAIMjrhOF6ZxBrv83t6rVeEiBixP6WYojD2s3i0pp6+ssZWnoXueOgH0b3u2A1ILAr2oDdcdzvCzUwAFmkiG61lEWikWYcQkXEMDZheG4M2GoElSxKk37kz/rCANQjjkjZq6yMr1YisWiEt91FX1yL0NcylBXOJIBfrKDsglRnWGkqhMMxTkmLkHEIcjRbzCNUDM0SJfSTmGizXkSiDyDKSWcjQKDUiyx34naYzFJMKU40pixo9XqMeX4Mc/IB+UjEY9DDCYhmCGpObCmNHYC0jK4HMXeRWQKoBqP3p2qb9LOFvDzHb5ndkbIs4L6BdjL9OLVCbVjkBKbYtffm4maUtAWPinu3ZAs2yuCT2hW5Fwg0XHqru6CFC2EyKCfttcCNCOeEmt+1lroz2McOveAsfZDAbxvedAFDYtqV3FQJC4WEftdKBWiRUYg7sBDO5jrq+lqoaY0WGNhUkfdLeHrIkIVE15XhM2s+x2dGUkwrBCJVmpGIWlSxAuods7iSUKhHZHsb6pun8P+MZz+j8ftaznsVb3/pW/uEf/oHb3va2R+zYBjvppJM6ADo4B3/Pnj389m//dlw2GAx46lOfyh/8wR/EZTfGwZ3WBj0S+/CHP8yuXbt41rOetWFd6HdH6vjfWPvgBz/IwsIC97nPfTrO7SmnnMLs7CyXXHLJIUH0xcXFTTVOx+Pxpvqt4Jz54Owfyk488UTufve7c+aZZ7Jz504uuugiXvOa13DMMccc8pyf+cxn8s1vfpOLLrpoU43YtjN/UxpMF3VNlih6g1m2797FzaqTGf3XtylqQ28ww8K2HaBSJuMCpVLKqgYh2bFrF7rWGCvI8pyqrEnSPjLJSBLBzp05LuGNQaUJO3fsYNv2BWZnZ0nT1IOLmvFwQlVWlEVBWZYYY0hTl7x70O+TZplLGJrnpFlGmqTkae7BTohBX+Mn202BlZXR8Xf4bjuCway1KJlgtEt0mSYpWd5nx47tSClIlCKBqJNotKYsJghhUYmkqgomkyFDP/NsbX2V0XiNleUV1laWGI9GFJOJ1xiU9HuZf+dKkkSRppmrG03iUWOdRndV1WChLDWTyYiVpUUHwCcJeW/AYOA01Gdn55idmSPNcuI0X89EEQKKccUEx7hX0iUOdXqQhirX9HpOPzlJEvq9nF4GA2OYqTTjonSgfD+lLEaMlCaXMJOnTMZDxuMR68MhdV0jBKQyIevlTMYTdFXTm5thMBgwN8hRokZQgymxRiNxeXWC5Jl7T1qXyNU7K21FNoFESicl1BmMWK/naJs+4ULXBhO0O0WIXVfRewjv5MDwiX6TDewkx4QJFsDzRrOydUw/eOwC8d2BYdPnWjMAjPFTbc3PLIgO8LjHPY7zzjuPD33oQzz5yU8G4G/+5m+o6zoCwNdeey1f+9rX+IM/+IMOE/q2t70t97nPfeJ7uW2HehdecsklPPjBD+a+970v73//+zvviMBy/Pu//3tud7vbHZJR/ju/8zud986pp54KuETTh7N73OMeHQDdWsuHP/xhHvnIR2Kt7bwH73e/+/H+97+ff/u3fzsineJD2Qc/+EFOPfVUtm/f3jnGve99b/7X//pf/PM//zOPecxj4vIzzzxzUwAdjszf+MhHPoIxhkc+8pGd4x1zzDHc6la34pJLLumA6Hme88QnPvGIzuVn8b1/OLupvvd/luzKK69EStnJQwOuz23bto0rr7wyLnvc4x7Ha1/7Wv72b/+Wxz3ucXzrW9/isssu4y//8i9v8HGllBvksn7pl34JcPJIoW7HHnvsBi3vX/mVX4nrb8g5hO8QEAu2e/fuQ+r+A1x++eWA04Y/mK2srBy2nGmcJgT6NpOoDPKYhwoGrqyscOqpp/LCF76Q5z//+XH5ne50J0477TTe8Y538Hu/93uHrNPh6vqz9h78SViQPbGmQWWEACscSSCyzgUIJbxmdZtQ0szAa/zfBvCFxl+ZnmG3eTBGtD54lnMXZNuM3b65bQTQoQGMOzP9WqBzOHY4L5ARRHfum/R61gEYbx9/k7r4RW18qQP+chgeum2jT4e29nmKMDtACWSjo+B8U0/aqOug0Y73Ed21dd8+WCICVcR6ND0ETmT0IwOQbr1/7LTFjddFD7Mc/IzO1n3WkDvM1P3n/HJhA9DsiB/Cg+rgdNMbAkpIZBqkZRqAerrZrLEYAe7SO32R0HYhOCGEdGQd09KwCOdnDWinOW99nxCebNvKD0q4asjWebXxwNjm1t9zThPeKosK53IEgudHDqJHKNMf2BIB1AAtxpMNIKl1N3SYjBIx3rhd69u3iIswOAS0DdpaW5OYAjCIOsEmmYOHA/sIF9WzdUFSL6PsGEioaktSDbGjRariWsrR95lVI7K8z2xP0pcluRiTqwnWLiGRWDWDVbvQLDCpZzA1JOUQle5F1t9HmOtB7AerEGIHiczIkoJhcT1CzaPSbUBJQoIZauzEMB7tZXl9Hwu9dRBjlFKkCn9TrDDDIrnNUHXfTdVGIJVjdkWWVqs9mra0/vrER1a3bZt4XVwbpsLEeya8xNrtHa87dFbGomzsDIEpLlubtsvqguXdO6qZfm1bVW2BJL6f4a68S1JAYJq3yp2KDLT7mgQvAdOWAwrVFzSJcafKpGkjQejHfplw9WpHC5UIyTmaFxzaIITGqoTKzDA2IxI9w3BlibJcJJ+5JVbkjEpFT+2in20nVRlJpkEcTT2ZUNbfZTz+HlU1ccltRYZKZ0nzOfo9i1WzTJJDO1U/rzbtiJ588slIKTsO8JE657BxmmMo45a3vOUGJ+7Wt7515/eNcXBvDHPmu9/9Lre+9a0PmQzsSB3/G2uXX345KysrBx1U7t2797BlbOZc9ft9yrLcdPsjSfL6/ve/n6c+9al8+9vf5mY3uxngGHTGGP7wD/+QRz/60ZEx2LbXve51vP3tb+dVr3oVD3jAAw5Z35uaM19WFbqqGPRSjjpmD9YKJkXF3v1LLC+v0ZtdRiQTllZWGY2XkErRy/tYA3naZ252gbquydOEoqiQKPI0Y2bGyY/Mzc0xOztLlmURGK3rksl4TFGMqaoKXWskkrzXI+/36ff79HouQWiSJKRpSpqmkQEsrMH6KVTO+RXR2RXtZ7UVSOnuk80kNoJ8THAUpX8vJklCL89AQl0VpGmGNoZKlyQS8FM/i0lBMVmnriuWVw5w4MA+DiztZ21thfXhKnVdNlMyvdMsE0mYIqlkilQJiUpRifLMCYEVDtwODKQk99Ms/SDFaDf9UpuS4XDC+voSi4vXkOc9er0+ed4jz3qkaY6Srv2yvI8SGU6z3DgnF9yUTlORFTm6ttQ5ZJklETVJokhUQpIo+n0XuJ9MJuh6lnr7AnUxoZiMGY1HrK+vs7KyynB9jbKY+PYZO7mgXo89xxzDrp076c/26PVTlARdFe78pE8SJIlTL4PPl6jcqV12GFn+8nrnPC7zSa20doETKX0iJu0SrgolwfoBgICYNEs4zcU2w8cdz+cB1XinuhnYtdlWbf3Jdl/bbIAY9nd1dH5NmqUoqdC0memOGNSeCv2zYL/8y7/Mne98Zy644IIIol9wwQXc9a53je/Z8H6ZfkeCewd98pOf3JA89GDvwslkwgMf+EBOOeUUPvCBD2x4793jHvfgzDPP5JWvfCVvfOMbOe2003jYwx7GWWedtYE52U6aBw1AeiR5PKbrt2/fPpaXl3nb297G2972tk33OZL34OHs8ssv59///d8PCoxPH+NQPsWR+BuXX3451toNvlWw6SDFcccdd1AAfDP7WXrv35D63tTe+z+LdiRt/Ku/+quccsopvPe97+Vxj3sc733ve8myjEc+8pE/gRoe3v47+0l4F7zuda/j9re//abbzM7OHnT/cA9MP++CjEuQdWnbtddeG3PUHMw+/OEPc/3113ekXMA9m+fn5/n85z9/o0H0UNfNJBZv6hYA9OaZ6XP/qQBKikgMUIoIngcgvQGcgzyLZwSLBrsI69vf4Xha6y4RdRpMsQ70tDIAzps/39sz76K8i/WlTW0/TWzxpx1JhwEsd+cnXR5AKUlTRV1W3rdzOY6Mz0fTnvU3VTMfBGg/47uM/I5aQnwXxMoi5UbycGTaB2Q+7uSvZcSPPBjeQseCP9puK611i7m9kYzRYG3Od3bXX4JwMyxdn2gjcDTAemDhWxAiiES7rwh0h0N44LDVsxDIOLYJ2xtjPI7YsMPDXtaC0bYp0RcWiCyNPkW37xoNQiQxhxVWorzsocXjvNa6BKfCRBBc+dOWyqJk076uvfD9RXTavAkANU0WiDWO6GPYoGZxELvBmujhW8aL3cVXg/xFWNEGctvgrusT1v9uZZu1zcDZxxacjqausOUqOQVCgNYDTDKLSvpxerK2NboakVZryGoVWxswEkzJpJgg6hprCmaSitkBzPQsWS5IsllI5sAMEHoJYxNqk2NFjjAThBlRlGskZpHU7kOKAxTVKlWtkHaOUpeMJ6tMJntZn+wlH5zARA8ZG40sD9CvDFV9gJpliuoApV4GuwNEBlIgbIUSy+ga0iphxlasC4UVEiGdBuym12Tqd6Mo1Wrj9tb+0mjrOpYSTQduLFyr8JAId5njYTfPh/YsgM5RYilCtPpD6+EcOnMnIuY7hrTNTIUwgyhECcOMlm4S0RChm26dsMytUHHbZpZEV5alKTNgNLHZBQ2AjnA3cHzYu9pFff/4sPDa8soipItwkaRMxgo7qVkbVUg7gzAZWf84MjGPlkdTqe2YJMMoQ8UYWy+TpH3m53YyWt9PWYyozIDSzjKpeyS1Rco+mzTATdIOPlvgyM7/SKZeH8xujIP7oxzvp2nGGI466iguuOCCTdcfbMAfbOfOnZsCF3v27EFrvWHqdFmWLC4ucuyxxx6y3Le+9a3c4Q53iAPpYA95yEN45zvfyVe/+tUNSc3e+c538od/+Ic87WlP42Uve9lBy76pOvNWGw9SQpbkLCxsZ/fRBZNSs7o+YvnAfnqz86wtL1KZEiEMKlFkqWOFSyHoZTlSCPIsZ252lrm5eWZnF+gNBmRZhpQSXVfUVcl4PKIsC6rSMZzSJGF2MKDfH9Dv98nz3Em/+P2Ucg6xkz4RXX8gfPsBg4yOTuOGTQ8QIICrG1kzzoWzGFNTVROETDCmRnswU+sSK8DUJaPhiMl4xKSYMB6PWFlZYmV1idFonaoq0aZywfAkASkRyrEnnMOZIIVEyQSpEpRyIKrw6xHT9QsgOuDZI1rXGKPB1mjtE5OP1xiP1/35SLI0I0kzsiwnz3pkag4pHNM8yRy4LpUCYbGmwlqJrqDKatJEkKWKJAnBC4WQiplBH8gxeuCuqdetH08mbFtbZ7i+xng0RNcVq8tLjCcjlBCoNCXJcifrkghUkpEo5ZNsOlkVbXEBEryWvVRUtQezA1sKkEq5wWNgixuXoN1q7RIJReDZRsaOtSbOVjXWIpQbAMRBgWc14XuOFAIhPZtHQGD0t0G19sAwLAsDozbADsT+C1BVFUVRUJaFG/AogcwamRhwLKR2X/5Zssc97nE85znP4aqrrqIoCr74xS9y/vnn/0hlHuxdmOc5D3jAA/joRz/KJz7xiQ2yK0IIPvShD/HFL36Rj3/843zyk5/kSU96En/6p3/KF7/4xc47NwTNpu1wM2g3q1+4ro997GMPGji/7W1ve9hyD2fGGO5zn/t0Zr21LTBnD1bPI13XPp4QgosvvnjT9vpRfJifpff+kdpN9b3/07CDPctufvObY4zh8ssvj0QPgOuvv57l5WVufvObd7Z/3OMex/Oe9zyuvfZaLrzwQh74wAceln29mRlj+N73vte5h7797W8DRM32m9/85nz605/uyMdAoy8e6nak5xC+L7/88g4Lft++fYcN5gUJxvn5+RvVn3/5l38ZgCuuuKKz/LjjjmP37t185Stf2bDPl7/85YOOZ4Jdf/31ADF4HcxaJ8kX8tncGAt1bbfpL4zZBg8JTN42Mc/5Nw40ViqAso0US4PfNOB4ZCbLjQzj6eB/AJ6nwfVQN8eKNl4r23uooltWe7826z2i+ZtYG3ey1rYQnyDJ5/x+YyxJllFXJXk2w/zsPONJwaQoGY7GpGkSj62UQmvTwirbSNjGWanNb9v820ZV/b/OL4baVBjdSsoZkpRiol8XWNtSNESM6fOlTZ5JEg+gN7MZA4gsPH7livCAtGyv9+cmXJ4ilSQELfFwLjGRqmlA8jZo3tSvRS/1gJYIEsVCRcmgEMQJMzhtvMZdeqojAoV29ufiJcFlC5cLZCMp3f5hLOjAfull8gXWWIRIwEoP0vt2lkFGxiCt9dKV0ufNCtdfRFDdBQ6aesegk23qBCFAZfnxaqJPleXaYdoxbW5m0f2nBXy2JV7aZdsohdFQ8q0Hcw3CaoQp6dkVMiZYMWBoC2q7jcoOUEq5gWhdUJUF9aRAV2MSW5OICsyQ2lT0s4x+osjkOlIWkOymTHZi7Qy6zhlPxiAqRKaxtsDUJTklGUM3vbiG0hYUtWBUCOpqgjWC4WjMaCLRxqLW16iEptdL6KdLzOU52/QadaYZ9GsSUaPNCF1WCLUfIeZItGVSaYxOScWARE8wqUHJbttPx0Ys1l+GdtDCt6uPFAUpGCEsRa0ZlgUCSS9JSROFFaJVaMMsR8Qj+DVt0X66BwtRR6bB8+4GMQYlRLzGbYA9giRA4stvonjt/tK1DnjfXiJCaonWFq12Eu2dW8uEDAz2dvkWRLO/EC1gP9QrHsa1YyoFRgmE0Nh6hKnXqKtFrE2wIiGVil5vBsECRb6dsr+LJMkx5QTZ76OEJRXHkqQ9VLKXlQNXIHtHY5MdlAwY1QolM6y98U7Mz7JdfvnlHXbVd77zHYwxHQf4hjjnm9nNb35zvvGNb2wI7HzrW9/qbPejOrhHaieffDJf+tKXqKrqoFPXj9Tx/1Hq8OlPf5q73e1uNyoQ8Mu//MtccMEFHV1dIDrrX/nKVzqM8K985SsYY47Imd9sMFVVFcAGZ/6jH/0oZ599Ng9/+MP58z//80OWfcUVV7Br167DBgh+3qyX95zshXXO1mBmll27djMeT1he/h7XX3cVO3fvRgmDSlOMcMk900ShEkkiJHmes3PnDnbv3MHMzAxpmmFFQm1d208mE4rJKALpQrjkooN+j9nBDHneI/OyLQ3Y6N5mzsnT6BjAFaRK0WTA8DIeBKeMhrWBJcy5a4PpbaCzwzqxhiSV5HlKniUgJCqBup4wXF1lbWWJqppQTkasriwzGoYEqBVVXaLriiR1zO1Mum+lFFYprPTBAKmQyun6BX2/JmmOpCNSZv1U0vDt5ixirEGa2tOkS+/8eafYM1e00VijKYsRxWTIGoqEFfBsjiR1gYo0y1BJQpKljsme98nyFKUgSyVpmpGkfjZAkpFlGcqfl0wUKYKehRljmFsoKCYTJpMRVVlQTPYwXFthNBoihWS9KKmwZLkiKzVZlpElqZOWiddHYq1wyVmNAavdlzV+kJA4ooR1yX1cIlftpzwbPzstepEEb8MxZFps/pZjLAQI4/pcGKA0gRiBkPhjdac9h/4UgO/p6dDBjDHUdR0HOJH9rlRMiBv2jwwzv900MPGzYI961KN43vOex/ve9z7G4zFpmvI7v/M7cX14v0y/I8G9g3bt2nXEyR6FEFxwwQU89KEP5RGPeAQXX3wxp5122obt7nrXu3LXu96VV7/61Vx44YU85jGP4f3vfz9nn332jTvJw9ju3buZm5tDa/1jed8fDGA8+eSTWV9f/2/1KaaPZ63lpJNO2gDQ/6j2s/LevyF2U33v/zQs3PPLy8ud5Q94wAN4yUtewpve9Cb+6q/+Ki5/wxveAMADH/jAzvaPfvSjef7zn89znvMcvve97/G6173uRtfp/PPPj4lFrbWcf/75pGnKve51r1i3t73tbZx//vm8+MUvjvu98Y1vRAjB/e9//xt0Dve+971J05S3vOUt3Pe+9433/XSC4M3slFNO4eSTT+b1r389Z5111oaA1r59+w7ZT4877jiOP/74TcHyM888k3e961388Ic/jDkWPvOZz/Dtb3+7k3upqiq++93vsrCwEBns4Tnx/ve/n1e84hVx24997GMMh8MNiYRviF122WUIIWLC6l8km2ZOt3OwBDAvfpLAQvcz6gKysQFEd8nuMY2/0h7bTs+ea5jRDYhqPdjYsI8daSMkedysjI6UScu3mrbNQH3nj4Vk8yCkxEnZSIy2ZGmPNO1TVRVSpuzYPkeaDVlfXyfQR+s6tB2x3iE/o4OYAnrWxaZs959DWjhHKSUySdC1xvpkplrXNDNlp0+als/pAgbW+6fWCpJEdQFt0Q6StIB4JAIVy3KzTiVSKAc2E+RiiAB6A6Rblx+r5fu2/drONRG2dYwAyrZBv6ZfWGtQKuAUYVwTT9vXwUNnQrjhWjgX0ZKrETIuk4FAbJU7mUSSmRota2pRu5m61iKMRSl/MGOQRiOkdWOwEAgQEmsC+C8wrayn1ng5Sdv0GyHaY8bDdokboonegKmxsf0tEoZ/08BpYJU1v5t9Qgt39gmgqUc2w2DZWOsYPcWIRKwxlxcIWZJhWauhlAIje7i0UwbqCqtHCL1CXaxg0AxSQ55LpNmBpqaq10m0gXqIUBMmdcpoNGF1rabSawiVkqqKPO0j1Db6vV1Yu4ximXE5YmW4wtL6AuNiHTBO61WnaGMxxRiZjJmbkcz3+mRMKBm7BGWZRUlDVa9jhECZWaRcR+sMoYE6x6oSi89Wq0JEy7W0ECKCDVhQm4Dnbau0odIVSjjd0EkxhHKIkBlGLEDaQ7Yw9BiNE3gEuduRQrSUCIJ0r12nHgFcbi1tg+ytR//mlY8geqNY1QG9bffvAGp3j+b36zwkuodoY98bAhFAO0MyAfz3y6zwCUrDgwd3rbSuUKYgEyWJHmH0EkIfQJprqMorwRYUFZhyLwlHQ//mVPk8Mp9BCKdnK0oBlQI1g0wMvT5MkuuwpkRkA8i2o5WfGl6ub96GP+f253/+59z3vveNv9/ylrcA3GDH9lD2gAc8gE996lN86EMfiolFR6PRhincP6qDe6R25plnctFFF3H++ed3nFtoHJ4jdfxvrD3ykY/krW99a0zm1ba6rllfX49atZvZb/7mb2Kt5bLLLuOe97xnXH7Pe96THTt28Bd/8RedwfRf/MVfMBgMOtdr//797N+/nxNOOIHBYAA4Z/5Tn/oU3/72tzsAwPve9z6klB1m4D//8z/zqEc9irvf/e5ccMEFB03GFOyyyy67STryiUxxLxfnUCUqYdvCPHV1FPsX93PNtdewuixYWNgBaYZSGUmiEFiyNGXXrh0cc8xRzM3OkKepY0VXEya1ZVzWTCZjyqLw2teCmX6PNHMAej/veXA2dcz2IE+G9wV8XFJKGaVO4joaR0YgGu074ZzQxgFXrb83fkemgxBIq6HUSAzG1BRlhbWaSTFm/77rWFzch64K6mLMZDymqiuwFqmkk53pO/Z8kiQxGCClA9GN8lMRReISewqFJEFIhQwZ5/0LLA44hG18IOuccYsjDUiveZjYzIHsxqDRaFsjpUEJjUZD0Ca0oHWJ0QWmMjCR4HXRhUxIkszJwPTcbACXoDR1QHeexNkBWeoSuiZpivDnKpUCpcgHA/J+n5l6DqM1k/GI+W3bKCYTqrL0iTMNpa6ZrBcIUZGlCZlvrySRZGlKlqaOdRI00r2jba3F4AExAVY6Brp79xoXULey8442/h8j2owjf/1bLKtgjq3UnYFntHGfKRmgwBIKFoDxg01RDgPWUE6SpGS5kzlyA1/RGiw1Pt3Pmu3atYv73//+vPe972UymXDGGWd0mLp79uzh9re/Pe9617t48YtfHN8F3/jGN/jUpz61IXnm4SzLMj7ykY9wv/vdjwc/+MF85jOf4S53uQvgmMLbtm3rtFMAXTfT+P1xmVKKM888kwsvvJBvfOMb/Nqv/Vpn/Q1938/MzGwAF8G9a1/xilfwyU9+ckO+luXlZWZnZw8p7XZD7eEPfzgvfvGLeeUrX8l73/veTrtaazlw4MCNlkb5WXjv31C7qb73fxp2yimnAC7h66Me9SjSNOXBD34wt7vd7Xj84x/P2972NpaXl7nHPe7Bl7/8Zd71rnfxsIc9jNNPP71Tzu7du2MC+23bth2RH7+Z9Xo9PvGJT/D4xz+e3/iN3+Diiy/moosu4iUveUm8dx/84Adz+umn89KXvpTvf//73O52t+NTn/oUH/3oR3nuc58byTNHeg67d+/mBS94Aeeddx4PetCDeMADHsBXv/pVLr744sPOdpBS8td//dfc//735za3uQ1PfOITOe6447j66qu55JJLmJ+f5+Mf//ghy3joQx/K3/7t324gBb3kJS/hgx/8IKeffjrPec5zWF9f53Wvex2//uu/3sl5cPXVV/Mrv/IrPP7xj+ed73xnbKPb3OY2/M//+T+58sorY2LR888/nz179kTZr2Ah2e9//Md/APCe97yHz33ucwAbZoP+4z/+I3e7291u9DPn59mCL9EGuiMr2eteO3auQCUOnJAtcC8AnEKE3C2BjCgxwnQIAAeXjWsAdJfMsWGzN/J6YX380//eCKhHooDooH3dI27w0z1BU2h/LgHsFqgkIc/6LC+vce0119Dr95iZGbBtx3bSNKWu6yiJMh00cIz4Ruy4DaJHLMviggPdU2udo0HXxsUQhEBrjZSSoigc2TNVbjaGrsny7Ijk+QKBYvPfzsdt5IIDkN7yaz3zO+qUC48NQLyWDSgtO/sSx1Wu38R2iFifQNAEJGK9DGgbmP5NslGH05pY5+Yiu8BL6ENBStEiPTm15YNbf96t8ZqUCco6Nqu0CmM0lapJak1pK8dGxyVTDeC3lNapPihX93YOpIAng3T1NS7A0p69EdrH7bP5rMZpO2LPLAlsnxDU8ct9/uCmHfxfgjZY2VLB6SCUgXnW4KEiAK+tiyGFBJVCkoNJEXZMzoRMJaATlk0fbIbWBWm1TGIXqexexvUKo2KZlJKZZIZetpNiMseorNFmiVqPmBWLZPI7pHoHulhnPFlDi5uTyu3YMqO2A5LBdgZkZLKGGsqiZmV1iav2TqjMjJ/qDkpIyrKi15PsnBXsnh2S50OsrVCk1FWFoQBrsDZFiqOAYzFGIcwIJQVKJdRSYuOAPMia2BgtU+5OmQKufTgj4N8IjDWUVYmuRghRQT1hplpEmSG1nKeQCZVMyNJkSjImFGxb11V4DLlBrafGka19Nw4Mpf+YTlLRAEi3t5wenNLectN1NjyA26B5+6GxWZChpdsvNyDn3cBPs1rE39Jv5vqqaAUMQBuNnqwjy73kapWBrDD1Acp6L8X4Oxi9n9rMUtlZ6log7IhMacdaFxaRKIQWoGqMGjAuJcbmGJmT5ttI8wFp1iPvz6BUSiI11D97A/Efh11xxRU85CEP4YwzzuALX/gC733veznrrLO43e1uBxy5Y3soe8pTnsL555/P4x73OC677DL27NnDe97znjiAC/bjcHCPxB73uMfx7ne/m+c973l8+ctf5tRTT2U4HPLpT3+apz/96Tz0oQ89Ysf/xto97nEPzjnnHM477zy+9rWvcd/73pc0Tbn88sv54Ac/yJvf/OZOItZp+63f+i127tzJpz/96c5gut/v86pXvYpnPOMZPOIRj+B+97sf//Iv/8J73/teXv3qV3eS1J1//vm88pWv5JJLLonMxBe+8IVcfPHFnHrqqTzzmc9k586d/P3f/z0XX3wxZ599dpwWfuWVV/KQhzwEIQS//du/zQc/+MFO/W5729t2Bt579+7l3//93zcksr1JmE90KTBYbUC5JJqzMwP2HLObyWTIysoKkyzl/2fvz4Mly676fvSzp3NOZt6pqrqrq7vVrR4koQFZknlMAUYgT/Az/B5+8Z7DNn6YMGE7zM9+RDiMQ/jnnzAWJsA8zHNACAgPAoexIZgceoAFtp9NEOAfYAssLIwQUs9d1TXcukNmnmkP74+1z8mTt263WoORgNoRVXkz8+QZ9tln77W+67u+a8fNscpitUHpxGI2Y3exwCgFMdA0HXW9om4amj5J7Q6tqSrHvNqjKgvm1YyyLFjM5sI8HzWmRetRZeNamw3bYkxRHGbarAsozGSYrilTZoa8WjaGzzZ4PjWqYwwolWibNbePbnHr1g3WdQ0KDg9vcXpyRFMvKYwihh5IOGdG4NQYO6Zt2qzfrpQRyRJj6ZXUaNE5vVIMzJxiqWQFlCVcbdZbxWBZEYlyjxB2h0oRFUCHgX2dsCZlUFm0wfu+I+bUyhA8zgkzJkaF76OwuZMi+Z6+b6mbFebUYKww1OWfoyoKqqqiKkvm8wXlbIYrSrQ2uFyIVCmVmffCwrHWslOW7HJA9IGuk8KxvW+p65pmvabvO9ZtYLXuJKvBGJwxlGXJbFZROI02HqUTyirIzhBkx2LQJ9SSUadTTlfNQYU4ceCG2iZabVjeKU6M90nK7DAeYWNATx3OgSE+HT/DuDqr/Tn8ba3YbINGJyAFZa0lpjBmI8SYtsD5T0cQHWQdGub4d77znXd8/x3f8R182Zd9GZ//+Z/P137t11LXNd/93d/N/v7+Flvx5bbZbMZP/dRP8ba3vY0v+7Iv4+d//uf5zM/8TH7wB3+Qd73rXfzpP/2nefzxxzk9PeWf/JN/wt7e3ovWt/hktW/7tm/jP/7H/8jnfu7n8pf/8l/m9a9/PYeHh7zvfe/j3//7f8/h4eHL3tdnfdZn8b3f+718y7d8C6961au4fPkyb3vb2/iGb/gG3vOe9/DlX/7lfM3XfA2f9VmfxWq14jd+4zf4sR/7MZ588slPqtTI448/zrd8y7fwjd/4jTz55JN85Vd+Jbu7uzzxxBP85E/+JH/lr/wV/tbf+lsf174/1es+wPvf/37e8573AJK5eHx8PIJ6b3rTm/iKr/iKcdvf1+v+p6B99md/Nu985zv5vu/7Pt773vcSY+SJJ55gsVjwT//pP+Wxxx7jB37gB/jJn/xJrly5wjd+4zfyTd/0Tefu66u/+qv5qZ/6Kf7Mn/kzL6nX/VLNGMN73/te/tpf+2t8wzd8A7u7u3zTN30T73jHO8ZttNa85z3v4R3veAc/8iM/wrvf/W4eeeQRvuM7vmOriCbwsq/hW77lW6iqiu/7vu8b54+f+7mfe1nBgC/+4i/mP//n/8w73/lOvud7voflcsmVK1f43M/9XP7qX/2rH/X3f+kv/SW+53u+h1/8xV/kC7/wC8fPH3roIX7+53+ev/k3/yZvf/vbKYqCP/Wn/hTf+Z3f+VH7tygKfuEXfoF3vvOd/PRP/zT/+l//a3Z3d/nKr/xKvvVbv/WO+en/+D/+j633//yf//Px7ymIfnx8zM/93M/xrne966Ne1+/HJiD6QILYSLoMyMvGNiYTQabg9LbNK39vgO6zNvA0Q24qv7Jlf2TmMgyAvBqLYJIz8c8C52dB9Ok5vRQTfetVBQHQdQJls48g11KVM649/wJXr75AinB8dEpSkQu3b/Pwww8TQqRpaqwxhBhHDHFUEVAp24YThDLbfEJCHnCzFyNzCjM6spFd0VqztztHG0NRWJqm4eT0eLQZ72wTkH64SSpr3GuTAxfDeakN8H8G15raq4xfpZxFOYDoaQyqsPXzDXK7uV3bUpdb6G7ajIPB50pJSC2DvMvwL/iEGoI2ZujzjYb6oC+ulCKiskTRdj8NZCnJQLA44zDYjKkZYgj0psdrj9YFhF58hSh67FpF2a8OBC/EJ4XKBIRzbO1JgCDGTYaoMOHjHaTxF2svG0QfmcBD/54B02EDgsvfG7AVdeZRyo7jFJAdAfThoR4jBwmrhQnvQ8JHjbIFhh6oKdQcF1tCWKNpUeGEUP8Ox8cf4mR5xNFJYL/aZW+2wJo52MC6bqi7ObMyYtQpJj1PCLdJ3uLcq3HuUZRZQOgxZkbba47WLbb06L5lXQcOjy03DxM+dBzsJhYzifqk5EnJs7uYs1PW2MLT9KeE1OFjgfcNTvW44iLWvJKo7sGHktYf0vsV2uyjVClMNiMPkrCd9fhAqTSZILbRX8aZBzYRPV+j+yPKcIJJN0ipoSgUIZzShwprNWZUDR9ushoCdENMakvaJPurW8fOMP6wl/HB29QlnmLVmzQV2N7XdFxtQPYpc/3F2oQ1PwL+adM1kxNQDAzHfOZDEYGzB1BDodZh+zT+To+DffO4xZhIfY31K2x3REo38LomhTWgSe5eAqcot4s2e3REtLuHoixZaelrYwxKW3yKtBT0ZpfgC2zp2Luyg64WuJ17UG6OtiWKDqM/vYqTfbLaj/zIj/COd7yDt7/97Vhr+et//a/fkdr5sRrnZ9t8Puc//If/wN/4G3+D7/7u72Y+n/NVX/VVfNmXfRlf+qVfurXtJ2rgvpxmjOFnfuZnxrT1H//xH+fSpUt84Rd+IW984xuBj83w/3jb933f9/FZn/VZfP/3fz9/5+/8Hay1PPLII/yFv/AX+IIv+IKX/G1RFHzVV30VP/qjP3oHk/3rvu7rcM7xnd/5nbznPe/hoYce4ru+67v4+q//+o96Tl/0RV/EL/3SL/H3/t7f413vehe3bt3i0Ucf5R/8g3+wpSn7xBNPcHx8DHCug/xN3/RNWyD6T/zET1CW5adN8apPZotqqGEiAT4fA0ZriqriypUr9L4jJU8MHSp5rAVjJKCrraX3gVXTsFwtiaHDGs1iPufeywfYYiZMY1fgrMVYw858IfIdIyi+cQZgY3QP+n4D41zaYMhsPlcZRE+DVAeytg1rRIwerXJaXv6NAKyRkNnFfdfRtg3r1W2uv3CNp595huevXqXrO4zR+OAJvsMAzliStlIE0og0i9ZG1mTtxOg1jqg1KAPaoo3DagF4lR6kOzITBDNGYwcpEm3MZBHPBmaKm/cZGFcanMvXEzM4m3LGnUoYZUX2TGUDNzSAl9TNIhHSoNst6a7ee0Lo8L6mrqWfjTGj1E5RlOzvH7B3cMBiZxeUoe1aJGVUZQa+lkKmrsjscoOrHK6qmKdECoG+a6SwbNPSti1tW+N70VZf12vMqqGqasqywDqFtRpXWFwu/pldG7SsuBmsHsYHG7bUIL8yZBpoLX8z0VU/Y3+GELKe5WbN3KSVSj/G/P3AVNkK0I8O6fS3nHmvRqa/sQoVs5ZjLmY6OKiD5uinY/uKr/gKLly4QIzxjoJyILIF733ve0dQyjnHW9/6Vr7927/94yqoDSKV9rM/+7N80Rd9EX/8j/9xfuEXfmEMjP/wD/8wL7zwAvv7+3zO53wOP/RDP/RxH+fltvvuu49f+ZVf4e///b/PT/zET/Cud72LS5cu8YY3vIFv//Zv/5j29Y53vIOnnnqKf/gP/yGnp6e89a1v5W1vexvz+Zyf//mf51u/9Vv50R/9Uf7Fv/gX7O3t8ZrXvIZv/uZv3pJF+WS1t7/97bzmNa/hu77ru/jmb/5mQEC2P/En/sS59/rltk/1ug/wvve97w4Qb3j/F//iX9wC0X8/r/ufqvZ3/+7fPbf2jLWWd7zjHVsA9ku1oZjtx5rVcrY99thj/OzP/uxLbrOzs8M/+kf/aMxgfbH2cq9Ba33udk8++eTW+y/+4i++A4AEybT58R//8Zc8xou1t7zlLbztbW/j+7//+7dAdIA3vOENH7UvHnnkkXPP6cKFCy+rj4Bzf39ee/e7382lS5f483/+z7+s7X+/NSFaCGA3FoDcwrSnLGTOgi55my20hClgeBawfpGzGPczwqlpsH82Nruw3s+e3/Z93mKks73t9O0d56U2QO1mS4W1BbdvH/Hss89R1x2FLem6Bm0TV59/gZ3FHrt7uxjd4n3IAQIyeLjZ/1DzZvsC1ObfWbB6OBcFxERMgRDFn3C2YH9vH2scbddijWN3p8D7nrpZc8YYnIC1QpPZZNlODi1nmeUaZSxsw/qKIdt0q5+j4HJRyW+1GgIf2dZMaSSapJhyZaJ4x9jYHh8ZMI9xtE+Hr4WwEwhhg60JzGY2RCg96dchMJDPSwp/ClAuyJv4SIPPNyJySvZlMMK5V2C0kxpTSmOUE8JRzM8OWVFfDWMpjKD+Zr9nZFqGLhyCRHm8jJkXI6j/0k2llznb/dmfqUlquKlp8rBPTkhtbvumm9geoGoj8TLKvYxgJpuBNZ57ousDq9Upan3IQTjk3qJmtwxoq1imHRp9gU5XpHZJd/Q0p9f/T24dfoibN4559nbg0oULvO6xR7hy4SEsBaFdEftn2N+5zc7eKa7U1O2co+X9tOp19PoiSVcUsSf1HWhFWTVcml3DxKc5OnqeZ2603D5JJAJO1xQ2SHqHgUsXE698cMbFvR20CbT9Eev1Ct9XxOApS8ts8Ti2eAOt36XrIt4fiWNVPEZvX81peR+9naNwoC3KFMLIV8hwSdKxZ6edSYAJ7yMnq2Ps8jqVP8b5Y1K8JTfGHtAUD9HNH6Ca7VHYSepCGgNcmSMniHlM28e6YzDBHeczna7O/nhzj6eg+0Yz/yxT69zhPA70lxjGatCFlz1otTnX8RzuYDcOC8f0+Gn87ebYw3nLLnwIqH5F0R1hlldx4QhLgzUFIUX6fklf3yAGhbH3YJXl3vvup1hcYmkvUhc7VFVBCNA3LfXxbXyzomsaZlZT6A5XzUh2TjQlzhU4Wmx3m//tj37Oi/fB3Xa3/S63j3zkI7z2ta/l3/7bfzvqT366tre85S188Rd/Md/1Xd/1qT6VT3r737/ju2Wyi4EYAin2aAVVWdB3HddfuMZzzz7NjZuHlNUeF++5zGyxAGWxRcFisWB3b4ed+YzdnTk784qd+YzCVQxSKjprXqsMEJ9j6nPeHP2ixv0IRKs7XtVodyggEkKP1oqQi/70fU9d19R1w+nyhLZpqes169WKw1vPc3pym9PTU9quxWgBgZWSoKikzsq1BG0zQC4guTY2S7XYfN0ZQB/kWvRUo3IbmN26ZoShvtU36iybJ43/dPIZVB9YQmnj8EzYQKSEyoGQmIMIKYq+uuiKe4KXtNMQ5DsfQmboW5SRa9g/uMj9DzzIAw++gtlsnuX0ekIvYG+McZS0cc7lVytBF60xKqGJBO9zmquMu7YVhvp6vaZpmmycC+hdlI7ZzFEWBqtF59BohY6JEEVqRaWEMZt+G4p6DsygacGl81KMh/sy/R0w/m7ox/OYVeNdOcvcumNsKrz3YgsqRVmKlJIA6OIcxaiJYRIwIfL/+vpPTuDzk9m89zzwwAN8xVd8Bf/sn/2zT/Xp3G2/R9rddf9u+2S0L//yL+d//I//we/8zu98FBDw/PY1X/M1/NiP/VjWTf6D1X75l3+ZP/JH/ggf+tCHPuH6SP+zWt/3PP7447z97W/n677u6z7Vp/MpaX/pr/wvI/YwsHa3CcEbFMXkAvEjXjKC34w/Gv6MMW0AzJG5PLUfmbwOx56ws8csPktVVZgsw6gUo+0UQpDMw77fYqXLyQ0gcdqc6xSUHDGgDfBvstqhtU4k9zAc7F3kN97/31meLFnXDYUtpC5RIdIfxhpe+9rXcuHiBW7cuIH3nqSSMPzJ0h5sH2tAIMuiYrFYoLXFWodSCpvkvH3oqOuarmtIKRJious8VTknoTi8dcRqtabtWnb3F1y+fA8HF/e4evVZUHHMss0alFu2qdJK5Hr1BDtSm/5LSROjEt3vECd9u7ExRerZjPhVWc6YVTtixyOEDe97Ep62q2maNTF6UoripxnNlF2+FXgZiM9a7HOj9XjvxJeQaE/KWB1KZKW1FkKONtKHWg+yiBnjVUl8LAyVc7jCCQHH2ZGU4myFtQXWlBSuojBWjhsDTbdmVS+pmzWruqZpW2JSaGMJPhJ8Rwy9hCpUJmkZsC73tRZ/ccD19ACYT7Fssq58lHCDVop/+Pd/5iWf4Y+psKiI9OfnOz/Mahqx4UygavgsbYBysvMnZCNJTJ6CrSM4r4aoADmColDaElMJOqBoxCkJHVovMdoT+hWp7/Cth9agUsX+TFKebx4dsVddZKcqcPMDVHRgr9P011g3p8S0Q1UdELoeH2vqek3vW4wP9OmUrjlkEVdUrsKYV7I388CaPizpWqibJb63LOaORVHg+8Cy7jHMMPoAZyKFCfR9T1FcQZsrhOQIydL4E2KaobWj0AVGLVFe0XqLVxXe7YM+kBSGPHDH+IsaIohqHAIAfUgs6yXd6S3U8gZ9WknEM5YSjTK79HYHawvckCZEjhKpDds9x4UYojgjey4zw+Pk3k1u+dZn6cwHWwTxQRM2TcfBJsJ1Nm64zcAfHN1hZ3lMTvY37iTHu8ZxOHkdz2n4PI/NTGJj0G9P4/nkSWb4vdoEjEyOQqoQRK8+ypjtY0I0cucYXVIVM2blPlo70DN6SpJ2KGWJSY/AgTAdS3RpCMagHERj6ZMhRYvRjhQ74vkhhrvtbvuUtccee4yv/dqv5du+7ds+rZ3p9773vXzoQx/6qAyd36tNG4dKUYLgCUhiZCSE7XXPPfdCTPQ+sa57urZhb38fbR0hJYy1FGXJhQsXOdjbwRmNNSJxMkzIotedj5eZtXeyG7aByZE5fAcrRW0VYnwxB3rUC0ygkiJ6z3J1yo0bN7h27RqHh4ccHx9R1zV9L8VBQ9/ktD89Fjk1WirbT4FYrUVCLimL0aIHrrVDYQTgV0YK3ygjwW0UI7tCnb32O848Oy+TNSypvKbkz9TZtS3mNWmTeqry12O1mhSlhkoKWWM8sEkT3gDpNut/kxLee3ovoHpMEELKAYgaYzT7+/soo+k7T9/60Wnquo71ej321RagbrQwypVGW0flSpRWLHYZHa+6rmmahr739F2XnQ4IMUEKJBIxgMpgs9EaY2wG3rdBa2DLiZtKrQxSPMO2U/D8bBslYCb/pvufHmO6z+n5hBDGf4Mjl82J0Q5RiBMk5xJ4yaHyKWz/5t/8G27cuMFXf/VXf6pP5W77PdTurvt32yfSfviHf5j3v//9/PRP/zT/+B//4zvW0uPjY+q6fsl9XLly5X/mKX7at8/93M+l67pP9Wm8ZHPO8fTTT3+qT+NT2gSonNoYg60wABEbYGL8PLdsjUxA9I1dcyZ5joEc/WKvso1iIGgopUaZ3E1Gf7a3X8Rg2bKrBqBwPNc0ApTnMqBHkqOwqEkaawtOTk5ZLldZqkThfSCGSGoj5awi9IHr165TliVlWdL3PdZZ+r7HFY6UK1oOxAp5nSJK29cyEFPGTESEjKKA3Z0dmtrz5JPPsFquASXSvSGQYsAVhvl8RtPUG9sQURef2qMakSkcpSvVFOuSG7cJQGyTOmIUYs7ASh9JI3Hj26E2dXlSisTACManlERjPKYJlCf1Ejd9lAk5SYDwmPEu8u7jkDHBEChRY7BCgi8q12lSm7GFvI/52nxI2bZXJIIEUJSQZlQUaZgQIlFFTAa/RU1dYbShKkuUNvQ+kJJCWT1UU80vAZUiMSmRgsy/hUggoiLoXIgVGAlgZB95KDYatrIjzm8vG0R3KgpgqrPMx9CpE2duC9ecguojyj98t5HmSJDBzU1UQy4KdB4Q2lpSNaOJPdF3JBOJGmLqScHju2Pq/jb1sqY+uUVdJ6rZfeykJaY5xSdP6o/o+kNStSBREdQO0QdU00PUmHIXV0DhGlbLSN8UJAxt27L2x7juGnOjqfb2qdyCnTm04RAVF0QW+HiDncLzwKU9Lu5atPLouGC3uoey6OjS/8Cn5/DeklSFUjNC9Hi/JqaePlpIHmOXlKbFpA6XToj6Ck14LXUoUbkCrjjHQZzNJMMDkzVYBVUnhUDqVpjVddTqOsoZKCzalISUaChJumJmi1GWZDMZT2/o5u309sKg3rr57szmk7b9y6FIKpNFQU1+txlCm2llxMLT5put3SMaTPmv8Yjjww4EQKfN53eC8zkte/hgBNrT1rG2klBUEnBqlHTJaVnBywTnFgJ49K3ckxQxSqJsVTUDNaONmtAlvAoEk7A54pi0Q9kZKTLKI+jZXFJifMTHYWEyeL1d5PJu+/RrXdd9VA3V/f19ZrPZ79IZ/c9v3/u93/upPoWP2r70S7/09zdbScqho7VCO1krhtzRlGBWzbl48RKruub69VvUjRTV3LtQQYikGCispSwczmVAPoH3Qeb+CZg41Vsc2nlppcPri4Hlagi850l/MMCnE//gRCgExL99eMhTTz3Jc889y61bt6ibNV3XEqMXgDolnBW2hLFO2DVG9PesczCkEBoByBMClksGmEEP4DmapAY2eQbPlSYlPdo6o8DXwBhne407+5d8P8njGwqJI/qBJJ0B9DQC50Pxpo3HlIH9JNqNSmfDOa/UOkZ09JiBiR2CbG8sJAREj4kQIut1zenpir29FlcU4ozESPQeUsIaA87Rti0+JfquozPCjhGDVArYFllz3VonPWU0s9mC2WwhLHXviSHgQ0/fr8UGUoGUevquI7SdBDSsE4fknADNS4Hq00DNWT3Q6e+G+34WRB/acNyNMxPvGPcgQYK+70dtzDim74nWpTiIYtdoLbbbUPzo06X98i//Mu9///t55zvfyVve8hbe+ta3fqpP6dO63bhxY9TxP68VRbGl+f0Hod1d9++2j7f9uT/359jZ2eFrv/Zrz2Uof/3Xfz0/+IM/+JL7eLlyInfb3fapbFpvZyqOuMSYwShI2ZQpvtl2mxAwtYnOgujTdtYWOs9mGljPnLFxBuLKncdL5+5jOImzxzprv6X8W60NIUR0trtvHt4QyRQlJBeSSArGGIghobTh6OiIaj7j/geusFqtNtcTp/01tf02APad7Yz9l8QWd64ghMhTTz3F0dEJfdcTo0IbRR87fGhY7Fbcc+8Bfd8K63nsL5FUEW3xfM1aQUxCehnu1+R+b/pZTWzOoY83WalSSFML4BziGPRImUQjBVclAzWmkK9vWsR0jKBs+mXA05TYq6J/znieMRfkHMD/wVG7I4iSBvs3jv0g+42EFNEx138aAghKoZQHZUh4FB6rcyBi8GkUo63ukP13XjI9rTEoLCkHDWJKOft6GLFpvBekQes9j0cjftzwnMnf6UWfoWl7+Ux0Ggz5JNUmFWCywdmgzvjxBmW/87uYdSmVBpPVSkZye9ZC1VrhoqbNWjteOVql8D34/ha+foGT0xXPXe/ol7eZ6YB2Bjfbx6eA7htMCjTrY8LiMlqlXGwrMTdzZjNNOT8gaMe6i+wsdqiqHeq643R9i6ZZ07uWG0c1hb6IoSKpGeXsCoVyYI9RdsFBseTiwYz5DPq4pDQXKN0+zjRoc4kmPIsxM2IwKNVBsKiQ0Mzo+54+HBPTMSl4UniCFJ+nmL8Zbfbxfp/elCRlCf0a45eUqUYpQ61KujDHuDlGaYIPtN0p1i9xyoPy6BQwyguzrtgFNxOGohZtWcFU0pm7M9y2QR99I+ezfUcnmkRp+5uhjUC47GiEubfmjjQdQgk9OYpE9Sb7HpHvND7IQymAaZBn8+hOL2eEZBinjkmQYKjheuYMN6DOMJ7VJIiEgDTeB5LvKRWockFKitDX6KRJIUBy2DzW+5DQRSJETxcjPm0qb2udsM4SY4VVGvwaR8AZJYw/aro+kKImakf6OAvv3G2/e+2XfumX+JIv+ZKX3Obd7343X/M1X/O7c0J32x+IFqPU1FBaSXah1qQYGHS3QTGbLbhy+Qpt23P95i2Oj29TLebMZnNSDKyWS45LR4oBqzVGIfIdE+B5YOeeyy6ftLNg51mDevv9Zh961KyeBmTFqFqtVjz95FP85gc+wOHhTUIMoKQAjtVi7GqlcUbneiMGa0u0dShjRYplImmitEFjMnitGHXNM6Au55RBc2Xkbz1Iom3IANM/2XzEGd/pnDaGZCcG/2atGx2oNCyDMdv7AqZLwCGvfCnrqOuEShIEMSmiU8IF0XpMEUKM9N5TNzUnJ6c899xzAFy4cBFnLSnIvbJGmPnOGmZVhRjmjA6KMNoDPoJvO2g6ULK2OVdk1lCBNlb050NPqRzeG5pmSb1aE0OL73v6ukFrLYHF5LbTYs+MrfPA8iHt+Oy4OesYnucQbt2NSdDnvO+nxxqON2TuiUk0OKZiMwzbGKNeEoD9VLTv/d7v5V/+y3/Jm9/8Zn7gB37gU306n/btsz/7s3nqqade9Pu3vvWt/Kf/9J9+907obrvbfg+3jwaA/+2//bdflk76D/zAD9ydv+62T+u2bXcoMu83226DkSjPQwgboPpsZtwddkzGbc4e5zwAPY04irpjf/qMjSQa2ecD0Nt2eRrxk/OPuU1oGK4/Zt1xY4Wgcfv2EcGHTBoRAMlZizEG73vKqkIpzeHt21y4eAFXOHrvsdbR992kqOQExH9JbHRAgUB0uS3OGWJMPPXkk9y8eQut7bi/lJAsTac4OTnlwsU9IQbpiY9DQsWNryVs7SEzeDvIMAQeUtSMBFM1AOtxEkzJ/RcjUSW8EdlEraRY6QCyD9KFUhsq2+e5xqQawPItXySOGJ3EARJpBNGHcbcJsmitRlxv80/6W7IHBvRtUn9IR0yIBB0APRJOBtJJCAGSpsdjg8pgvMenIBifFvmYiMhv2ihEVm00xijweTwnL3rxQ/AiD1vB9YUIK0ErRcz+sYxbnTHCl9deNohexiNaKrza2UTPthzEjdO3QSMnOxhuGEPShryPIRCTR4eELUq01tLdSsmNBkJKhOBR3RrVr+gJLPso0aCupa9vsjp+npvXDnn+Vs3l/Svs7SywrIj9mqZumB3s4LRit4ByZlG6RMeKvfISe/MeY2HdRZzuWbgC31tu+TVrevp+RdRw2mmunzZcXHQsFo7dgwV9LElHLX1Q2FIxm1mcC8Q+0YdTfJgRU4P3NUFLEEJrULGhYJfCLOiCBQwxNfTdMV71aGXpQgn9MVX5DFW6TN+XBDXHd2vK9hYLdQrao5gT7H0k5QjG4GMPocWqgHYOs3sBHdYYo4imxBc7mHJBslYKWSogpRztEbBiM7lvUh7OTj1TwDsxAOjbt30cBueA5NONhzExPs9qCh1M9zONmE12qDgztoa9SimHwYHVaQAmpheSxq1T3ufmnCdAfR7DGjVIXTEg60OgLZGkQ5MlxEDfe+g6TLdEm0LYjMkRgwQ3ggmoYgdjC7oYsSkKw11rSUNxBqsLUtfhQo9LLTZ6bFyTUiRqh63mpBHAuds+Xdub3vQm/t2/+3cvuc0b3vCG36Wzudv+oLTBeJI5PmYmecRqKR5KDBhj2d3d4+LFCzRdy2pds1qeUBQFCUXXdXRdhwKKzEYXQNNstPDO6FJPwc6BtbD92VkDfbrubBv+07bFekmQQmC9vMH1F17g6PbtLKUhK4nWkmpobZYcsQ496Jxb0TxHGVLWOcdYMBalclGbNNg6sgAkNNrYHEAdzndYUbbXzXPiB8MdASWskGFbWWemTsX0x2ZjP407Nai0kXYRuzcxFmgajHMSUggojqFuna1dA2ACJsbswASM8SSladuWGzduZEfCsLezS1kULBZi/w2yKMN9G+6HgMgJn6DvPU3b0LYtfdcRU9ouYuoKrFHozJDpe9FurNdrfN/Qdy3dusY5Ac+JibIsz2WSD+Nvw8LZMMfP00efjtNhO+/9HdufHa8v1YbfTSVuNn0zbkVKMJvN2N3dpazcp13a/V3w6WNrP/RDP/SS8hIXLlz4XTybu+1u+/3dXv/61/P617/+U30ad9vd9gk3kebYEBDG4qLAWRBtIDoO8nzj32TsQ29A720w+MUB9MmZnLGrNttsbHM9klrOBrrOZaJPvpsec8jiOx9Y12SsmdPTZWaWa2IQAoPvPd5n+14Jax0gdT3Xrr3A/fdfoffL0f6LMaLNmXN4CQhda0aAGNTIjL927QVu3LiJyrax2NNCyIkx0vee5XJJShE9iLuPPcuWnSznNfTTRjZYxkEG+VOc9P9wPhphUG/Im8MYGMBypTUxbvtMg/zKUEBTxwEYl210moD4edylJNtoLcVl7zR9J8CboLR3fDfl047BgSTFQJUChHuESpqQIkYb0TNP4qvEJCz6wW+NwZMIoITBH0KgsJrCWkLIvocX0rUxGqLJgYrAkFEwsI7S+G9zPwY8z5gsRanNOdd9Z3vZIPpevMmteInOiHZ3Ik2YydJB42N73hjNiP/AllIoYoj03ZoqnjIzCagIqiAoC7oAK4yvGCJd35HaNalb0Xct675mtVriuyNUMpyurQxuD6fLE2b0JG6jVU9Z7FK4Et8GLJp5MaOY7xEpWBSeedVi1QlF0bOYRXp9m66rUfVt2uKEU9NDscPO4kHW0ZLqFUVxwt7OPbg0w5pj2nATHxWo3Vw91lC3h9TrW2jdUVS3KeYJbTyFUmi9T9QFlVKElDita6JfYcpAYRUpVVTVIxTFDjNzgjFHkHZZhkjwNX2zYhVvYdwpmAsYtUNgD5TChw5ij1URrAJVkLQhKIi6wLtddLmgKiusFVZ2FwN915CSACrGlEgBNCmyJoFNYYdv7vEGYJ8+UmcnkM1kvhkaerOLcbsRBp5kOQwT0JDKMTy0Z+b/O4I2gwwAauDyDeD4tmzQOGbV5vut/euU2efZqVZZ+VZtJkC5toQ2UChLUguiirQdpNCRfIsuCpSdU+iEjZB8Q9IlzlWkakGwc3RQBO9JyYh8i5GIWx8jiYALNYXqcEAKKyKWVu1gnWj63m2f3u3ChQv8sT/2xz7Vp3G3/QFro3E0GsFiOMUYiaHPc6IYJvdcukjbt8QYWa2WGOu4ePEeKRLZi0HjnMMOwKWxd4DnU0B8G1TnRQF02Db27TkSF0NRmzScezbG6r5hVa+5dfs2ne8pyxKlwFopPGStxdqBMS8g+iBjorQhKitBSGVISueipiLYpcew7gYo1yMbWfaZIBuGG6bJVHZEQOeAGOVajDq1AWzFccjMI6Wykefz9mLoDfb3Fsv5jpaLf6fN+3Ht1Tqz0+MIsqcUxRbTYpxrrXDG0McoGVUpsV6vWS2X7O/uUhQOkw3UqBQh+GwUb1gxKQ6pphsWkvzTxNDjvR810a21WK2wCorCkmJHU9fcuHGDvluTYsRlmsx6vUbPJQtryHoY2lkwfFo4NISw5dgpJfsY/h62OSvX8lIM9SnIft74nf4t90thc+F2Hzzz2YyDC/ssFnOcc3jvz7mXd9vvlfYFX/AFn+pTuNvutrvtbrvbfs81lW2xAVjdSLdIS1tA3xQ03QasE0SIasAmGL/bOtoZe2ZgDb/46W3bNAOge2fG6FlW+Z3Ey4+W7UciS5IoQoisVmv6PjDYsUYbTCHAZt9LHZ2u69EaSlNx69YhFy9doixnLDMBSOw/kRe8M3Bw3vXmgIRSWGPpk+fq89d47rmreD8lWoiNjIKoIl3X0jRCLtGDrb252K16O8P1h5AxJzXBm5I69/6CBEgiZPt9sHdjtncTMQh7fLwQcpZB2mjtp6jIkuVin2tyZsEUyZUmJnGaAP1n7VzNIOsikqFnSbZqJJZOySkpRXpgIJ9qrUey0uC+xBxECFm6OhEJyWeiryehIcWRea8YWPmM56q1Hpnow3M1via5bimSms+FnGmqFUpJrayXg6K/bOQtKUXSGmsSGo84fLmrBo2LAehn8lieATtVBt81iOOYItYv2Ym1xDP0nJXao2cPnWYy+POuYwzU9ZrV6gVOl7doOuloayKdd+wsLmBVg1WGQq9R8YSd3YtUew9QOkOlGlSKOAtFWZJ0QUyapl9hdA+pQ5mOmWqoTKLaj5TBMZvvctoFbFHho8EAsyJS2CW+PaZrb1HXJ6wxrNYz7HwGyeBDQ/BLLDBzFVXaQSVwag4aokrAKYW5ycysUPaYebGgshUBT1IFRkdUOsGka1RxToheihnEwGnTofuaYraLLSIKT9srQlNjuhZiJ8znoIgZOPDaEU2JtSXOWLRW+BCpmxXUNyjwJF3QmQVJV1g3w9liBMCnAHoeGPnepiHIw4ZxvomSTjeFoSCpGgdwjsOO254duvLgblJQpmNq+H4qrTKUektJ+jnpASdXeeKajsnpq9oKBJikJvJkw2SSXxlKyaVccVn2H3VB0jsE6zDFHBMvSCE+wPk1NMekcp9UzDBFQs2lr0PjUSphFTgFSQW0DphUE+Ka1B3jsMyMogm3sMyIcRdiT4gfVR/gbrvb7rY/gK0sS4gBsbKBwRgnEnwPMZKSxznDbFZx8WCfru84vH1KvV7RzOZjIUSlNKUrqFyRydPboPmUcbFdfBEGhsM2SyKeY5xB7O8EFqdGv9YaawwxRU6WJzx/7XlOlqcksaowzmYtbpv/5XMxDqWkIKjSGrSwzmWS11kAUKOSZtCSHyq4DKtOCKKrp5Sk3ooR6zPjYcMQmrJuNvIdhpg8iV6MuzTUw4nEkIF0suZfLtRpjYYsJ5O2rauzPZRf1ZlttDBskP2PhnwU7UkSwmLPTJHCSUaaFEzKgHSMBO/p0kayZHAMpvcnxogPolMYQqTve3zej9V6k8IaI33XEgBVWIpijrMFKM+Q1qnyPvu+l77LGXIhhJGdPvTxMH42abHbLCzgXCmYbeP+jBNwxlE87/OzLK+zwSPy+KibDu87YvRo7alrhdKeMpY0TfMi9/Nuu9vutj/I7fE/YpiVBa6EK/clFrMZB9UbWN14hPf/2oc4PHkaU8LOJc3sIGJnEVP1vOozLnLhYJenn3iB9bJhd2Hpu5prz/T89m94lNd0bUChMNbSdn6sdxaImREIs0Kzu6hA9+giEnWgms25556L1HXNrVsnaKPY3StYrRsOLu5yumxoV4H9hWN3zxBUi5sn7r3vEt1J5JWP3Eux2+FpOb7dsD+/hI6avrU888w1IodEH9ktHuHJDx0TQ81iv2XnUsFsvsfV5zxJNTz08A57u/ukWBC6Bt/3HK5O2b1UUs4KmqPIh3/9BT7z8S/iM9/wFn7lv/x7YrpJ3/S85pHPo+BBmqVjb0/xJ7/0NZTl8+h0m9Lu4PSB2AEmAkbqpliRcVBDXZSUkIwyWX+G9XC6Dk3XJ2CzlllD0lpe89pjMqtVCraDjmprXYkx4r3HWitZ5ZP9Apt9AEltg3bjfpkyne8M/A5+9nlM4+m/KZA6Zm0Zleu13QmcDr8LIWCtvWObgYGttR6DylvrNOGO/Y3XjCKFuHWN42+1wsdAzH04PZeza/h5tszw9wjKTdZ4H0Epzav/8F/hE2kJlbGwwW4bbMjtfj8LsKZMfNi+RwObenN9ZzM8h2s+W2T9LKgeY8Tos/f7znE+7cPhd1tX9xJA/mSzXBNTYa2jrmtOTk7w3uP7gIpCLnG2ABRFUeAnMiBd22Os4foL13n4kYcndtiL2crn33sxg+U5aqPn6PYJzzzzHCFEuq7fGnfOWbz3KDv0Z2K1WrG7t3uHX3Oe3wNkm3+7L4fn5A7wdGs8DASQuOm/NEje5M2HryakI60GKZoNbjtkQUwQvrzLJLrvYzKuGvc3vZcpbfC04byUGhE4KUY6HmOQf9mQXaakF8brk3PuSWjlgURMPSH2G0KSkaAAKaGVwlpNQmOxIx6IFv8hZvA+ppjlXEDFTR8NuLUxmTw1+IifTBC9V3MR96fPzpjDKLsFLG666UwbBlP+f8BBtVGgDF0q6fslu6ZBG0/CsmKRdXs0yigKo+hTZHl6zPGtFzg8fJaTdYcr5xzsl8zmFQcXLjO72ODrU3EQzQUWi4vYmWXmZixMSVQ9Ph5jmaOtpfeJtl6S4iGahHNQWYPynsI6Lu4EVOo4rdfgKlbdAuUczhp88PSxJVHTNiuuLSPGK15x6YDFXDFzCzCRHXuRe3YfYGdnThfXKGPxyeGJtPoQ0nXS4pB1P8O4C5iigBTx4TaJSN/X6DjDqQsUqcBESMmjtEPrfayqcEbh/QrftJjTG9go6RGaDmJHUgVRlaRiB21nKKXxUdIf6mZFrI9YNC9QcELUjl6VdPqAVXmZcn5AZYsNGJ7/V8MDMTx4mzgKU4mU6etEVJwNKJ4B7hFs2byi4pmAzKBrNSD1myPokfGemELKwzyqUBNWecoAkHxnRqb6ZJLP+9nsV7CWDSAEKsjCZ7RCm3xcrUi2BFVg0g74HhUDOrQQayjnqOoCrlrgVINWkaQCvc5FY4fibf2KWVoDPXU8xacG4z2YHjiV8w4NqWsBd/apu9vutrvtbkMsBmROsZoUEl3X0TY1bV1neZdAVRVYu2BvbwcfA3XTUTctp6cnuKKga1v6tqPvPC5H6ZPaGOTnsVM2oCVMjcUXAyJVDmKacc49z8mTQssCsHbcPjnimeefoe4bXFmgncUVFbYosNZhjUVbg0ZnxrnOjPMMniud1y2VV5EcZU2KlPQY/B1VHmMa62+oGIkxEHyH9w19245sFdg4M2LwgzWWRCDEluADSmliSvg+6x1GAV9n1Qy9WIgymHa5n2EIIAxL4HZT25HqIVStsk4i22zraTEglSKD81wkhdYW30uApVnXrJYrtMrOS76+2WzGbDY7x1GONG2L9wHvCxIpFy/fGAhjwaMQqIqCvb09ikKhVKCqKmJoSUGKn3rvSTFilN5iok/Hw8AAmo6tjbP50gykYZyeZaMPhv55vz1PxzwlYSoN7PfN7xJNs2a1OqXva5Yry81DQ12v6dqWoij5yv/rV7/o+d1td9vd9gezlTuGy/fO2FkUWBLLQ4Ordvh/fOWf5S/82TlPPP0BHn31g/yDf/i/E/Upe7slqjDceP42t28c0dWRqqjo+4YQAouF5nM+/2E++N8OWZ6sMcbSBY8tlMgkWJHFCiliFFSVwRSSTm+srEEnhzXzouH+By+xu3As10u01rQKmmWDigoVHJcuXKLrDtnbL1FF4pknDrFJoXTPY6+9REoGZxe0tebksOZ3fvsqB/sH7F/aIRVrVqdLHn/Vozz71POc3L7J0bqmKHucPeBVr3oE5xK/9r6nOT5qeMPrLlEUmtu3e26dBGazNfXxikv3XuK55z+CTYZ2veLCpR0Wl/Z4/PFXU5/ucM9nPMrq9Cq3bx9x6VKP01ECxMmLzi5JwHOVMngjAWmUBNJTDCiVaJqGoigy0L5Z9/t+A7xNg7iiV+zRRt4P6xkgQLnZQDNTMLSqqnGNORswHgBoZYzUhMltWMP6vs9r3aaux7BODuDzkN037G+4lmHbLdB8EkRXShF8GP3+YZ/TbbckNib7jjFm2tum34bjxFx8EOK5YDBAChsZtSHIPvZzRvhelHZwJng+7dMp+WHaj6NdoYzITXxCTTNkv4+gYxokOGK2lTbA4gZEHe7LNlAs/XNOzULOBCUmQYVh/8PnW/1zRx9tgPTpv7MBh/HX59jvZ4upjgdS5CxBw+npitPTFSkpqrLCGUdRlAxs9UTAKk1KMj4i4HtP3TScHJ9QlrNRJu9jUboVMotoqh8e3ubDv/MEXdvLPQkCxGqjsdZgjcWHIOz5wpJIW0A7MLLSUw6MDEqLm/E9bCsEl4Exvw1U6zGAMoyNoWjo0N/T30zv1WQvGb87S0Ddfj+F1Db3cZP9O+x/Y2cbBkkixjGZ3w/U2DQJEKkkvliWc4gkQooQAypqWWOGYG6KUruLnpSkMKpC2OcpBTbymXnOUUJSckqKzw44pFaWECM+9CTvcz2wDdZ3dn47649+tPbyNdFVj9FLRPqjoFYLUAuUtuKHjp0+edgmN2R6s+S8NEYp0Io6wjo4nE8UtkK5Oc5alFaShpwkpSCGjtPTW9y49iFOD5+kXkfcvMSlC+zMHuKBixe5NLOQGup6iY8NRhdgoXRrdgqPseDTKetmBxUKDBrvE/V6iQod82KHRmlKpUn9CX27JPoTUmjxQRH8mgs7l6mqGSGdcPPoOa4fXqdedTg15/Co4Z4dz0P3PsjD972KnbKi1Hsc7F1mb3efVX3Kujti2d3g1vIpdBeIOlK6gHYVyjoCAd9BiCtC6lC6JIZn0e4hbJxTUuZorsVZTeEUOi2x/QobTnDxKqEvwBzkh7hHKUssStTsgHK2i9KOGCNN2xDWR7j1DVR3E69ugzFoSqxL9MWF8Z7pMcg1AM458rm5qZN7vBkLeutzxoV2HB55kRuBlvGztPWZmjz0m7G10YXd4OmTYmwDmJ/YOq6c10baZVPCI21dimwzpJ1I9E5lQGAIuKYI0WfAPUcl0VLML/okzMO+RnUnENZoWxK0BAiMghKPUYEm5ge8j9B3lOE6s3QVQo9VC9YpkrqODk+XAtEWKF2glBkBko+3/df/+k8m92ESLEkJlaQPYjL0YZcQ7yGxj7JGElLC+Qv28PpibL7N3GAkO0X1KLNE2xOMq7FqF+f2KOwOqAqtKrQtZNFHEfK8k6LUVmjrhpPj45HJIBMtDEyRRJBFNwa5rjGXAGCTRid69FnKSBvKsmJvb59ZNRMGpNlc33RhiSnhU+T09JSbL9zkqSef4flnr3HjhZs88ZGnePbZJ+i6JUdHR/Rdy8HeLlVV4qxFq8TBYs5yteT2yQmn6zVN29OFIIvJAOQlchRZzs9ay2y+Sxc0ZTGjquZo7YhBYbSj6z1lWTFf7GCNxRiNsRpUR4wdfV8T2xbfd8QUCMHTNDVtWwtrJgVMNtIGw30w5suyQmuLswWuLEEZfIhEpEhhoS33Xr6fL/yiL2G+u8dTTz/Nkx/5CCe3bxJCx3J5IgY1SlL1snSU1oZqvsfFi5e4fN89XLlymfvuu5crV66wv79HNStxbttZ2YzZOF7HIB0xGL9aawrrtgxh7z1934/G8tQxmBrtQ7R8ABCttWfAQxnHAg72hODpuobVesXx8TFNV28MiXFuCqTU03NMz3VC7FAxoSLoqFHJoqLBIPrVf/fv/bMXf4BfovW+xWglQKmyxNhzcnKbG9dfoF6tMFqKPu7t76KUZ2dvh4ODPdZ1x81btwVEdadUthCmxWKHqnBoM7ANNu0sS2n4DDIuPTFYzrKatvZDHIHizfdT9pDUSTk5OebZ557l2o3r+BixRSXscluCcaCsgNAYkjKkrIEuj/4G2B0ckiHQqtQwR+nNsqI2SZqGbJsoSB5i7OnqE1bLU3Gc5cKFYaQYdRyLogCV6Lqarm2xzpGSEgM9ihyLsxVVlsyRQqjCBtF6M37Ot+2GIlHSgyKQNmFpM00zTZmyNHTyIAYZ0TZRGoOzlrZZc3x8LGtfChwcHFBYS1lV7O3uMp/Pt+6x7FkRYiT4SIhZlma4b1plozsDzl6Y52VVEkKDdZayLAm+ynNSGp9j8lgYQOqBjf5iDLPhuqdjc3jdmrcnTPazvz/rZE/3u8USYzNOpzUCIOGDx/uOuj7l9tEtel/jnKLrO9q2wZhPLAD+R//qoxJc0Eb6qu8wxsrzoy3GOVm/o6d0FmsVvm+x1lG6PamLgAA7hSsxxsoY9g1ds6bte4yrcOWMqAxKW7q+p2lXFGWBMZqqKtFa0fUtfd/hlcgjLdwMZwxdU9N0a9CB+bxkZhxt31G3LT54Wt+LNabAKkXhrJARkhQyLrTG1y1d24KBqqqYz+ZoIPhA9OLMOj0nkWh9g3IKT6JuPX1IVNrhkiYZhSos2sl1dE3W7HdWgEPrWLhCioERQStCHzHaERR0KVCUBRZN6j3B9wTTCyCGoa5bYgRnC1KCuq0J0QvrlAw4KE3T9aQUKTJwWRQFi8WCpmlYrVbjmlSWJfP5fAQr+r7HGMPBwcXxXrVtw3K5HAGOGCNWKebzOV3X0ff9GIACWNctMSmc02glUlTzckEMkoFTOGGLLpdLVqs1Mch5zGZzysqBFlBL6k+YvG8r6y0bAMoZRQySQdz1HdpKQWqHJfby3HW+o4s9rnIo5SZ6sYq26+j6XtbrPmC1RSFgy2KxAMQuiSmhTIE2IoWVkOwg76WPjRN5yt1yxk5RsTo95Wh5CloTtUGbKj/LER+8kFmI+b3YRkVRoND0bU9Kivl8QVmWdF1P17egAz54uj4QIljtcFp0fH/p3R/5uJ7tV7/2In2/ou86THyQz3/zn+LS/DM4vtHyC7/4b/mV9/0CVx68yKXd+/jw0ze58cJ1ogrMFor5rmMx3yW6SOEMxlTs7jr6pucLvujNfPADz/H0s0/hrKI0hhANdd2TiDinmJWGwka0adE6oowQ3/b2LTdv3uLw1gl7FwxG99RdQoWItZbKFayaJfjEvKxYnRxRzR37sxmhTzTrlugVTRMgLvitDxzx7JPXuPfSDqfHNTE13Ht5wcF9F/nIB6/y3PO3ObgwY3fu8KFHozi8cczhrUPaVcvuXJ65arHHW97yehYHjt6vuPb0DZ764E0OZo4nn/gtUD0PveIBimLO1eu3eNMb/hBa7ZA4Zm/XEONxBnU82gSsc1irELY5oKDvetq2Yz5fYK0d145pVtTUTp5+B1nXOduQymxqiwzbDVJjG0CKrf0O+5iuZcPxhvcpxjyXbvtg2+si43fb17FZ184LIE/X2G0AL2MCk8+nQewBtHVOaoBMiwlqrVGosV+mAfKBJRvzPny2F7bBL7YY6NbaMVtOGZ1tjm1A/OyaPb22syDosN3ZzzsfJNP/E2iKIcNRiW86NTVyoc1NP27bHGeDKMN5vqzjnrkeOHv9+fupXZfS1rgZ7t0URD9zkHH76TNwnr2vUHKvlM4s9FNijJTFDJ1BZO8DhSsoioIYfc56DEgtH03XRZq64fbRMVeuXN4635fXUg7wGE6OTzk8vM3JyZKURLpwCHZpI4E076WWj+iQpyzpqO5gtwzPM/klTboqhAF9zv3PRiN98K3E9RG7eTjPFDOZJyYGHfeUyLKQm/sn/oDOOFfG5CYuqOx/Q6qZ+lnD3xtwfnPig182AOVyDBgY6Nu+32YcqOzDnGdTD301XGeG4EkpSlAwefFVjRYGepTgbhoY5XJS4/wRc3+CQg02gpL9qXOemXEuUlPy1ycRRL/gnkTrROSA0/4Cnhl+ImuxOdZwo0bBjinimU9W7rhSwsyinNP1Ea9qNAWkhI6eQM8Qx0nRk3xHfXqTUF+H9SHUHSRDYxvW8x3Ke16JK/dRqkep2/TdMYojyuKEojjBFSWoS9w4DHRaYwuYFzOIiXZt0KlDxTUeuH16zPHt52j7BlsUaFOgjEGXFcpoTlcrlvVtnr56xLWbNWFtuO/CPp/9qj/Em179Gh6+7z4evO8RnEZSoVPCupLd+T5JvYKQIjePn+Gp67+C73q0q+iUxtOKwWoSUR3QxZaEwqqENh2OyFxFehNIWKzxggf4mhhqTLhOZZYEfYU2aaKpiJQku4NZXMLN9nBFQR8jve9QscGmDroVfVNjnYFUgJ3Jb4yjsprCiKzOyNIbRvnWE7AZyFptMOvN2Nie1KafD6C8UttRYzUsM/n7zRdngzL5QRijnxtQRCMPz2YcDt9tjj0C/nmiiwFSVCSdBIAaj6XGoIHRiWjAp0gKkYDCKJMNeTWmsKQYSc0K60/RJqFSwIZTaFYkOlQB+BrnE8Z3FChKA6q7Th9uoEIN3IcKc9adpQ8aO1sQqUDZXDDvE1vMQxoCFduLQEoJlVkNCZ0npun9uXPhfrEJ8uW0wShMyhPVWsAvFIqI0hFLICYBt1LK6YlRGCFpNEiHiXu8azKZpiHqHlFRgCOVs11Gba+Pdn4gx4/bumUJAVp98jRNx61bt1mtauq65YVrL3DjxnV63xGipBS1Xcu6cZRlQdM2+L6jPj3FFZa27UTrzRp0Au9FNmJTrM4QwlBXIlE4x8VLF+m6nr5v6doaZys632KMResO3y9RFIAhJTA20bZLVstTlI9Ya0iIkeq9Z6zs7XuS0aMsRtu2Y1/EmLC2kPNKTsJNKRKipL4VpcUqw333XuHyA6+gmu1yYf8izz79FE899WHarpMaADGitLB1jVL03tOHY6SACJSly6B9mY0YNYKcdy5020xnNVlUX2pBfDHw7cUckakhvzEYuOO8Xvy4m2Ch0ZqkLDoldJI0ahU0KhlU0pi0nbb5sba2rSmdQyvRgev6luPjI27evM7y9BSrtTDUk2d3Z8bcz7Cu4OLFA2KEGzdus1wumZUVbdMQvUcjmoFj9uDkOs9jLAzP5dl+mzLYRwMmG3lTdtHwm+G9MYau6zg9PeWpZ55i3azR1oghbizaOowpcjHnXJhyKByawc3BWhWJOUZbZry/SVL75IPEWFsDOT+lRaNQK9AqEENLuz6ladpR0080HiWoZK0lVhVaQ9OuOT05xVgr60yU4FhCs5jvsL9YYBUYJc62Fq2wLSP5bJsup8M5jx9NmOYMUHoa1PDTKOcy3a8xBmsdTVNzcnzMrBDgqqoqysy6mzLvxn7LBiwlW58ptclKGBziodhRIrFcrgk+oJXGOYtKCZ8lYZq2pWu7MeB1lgl13vM2BQOG85iCD2cd0unY3fRbOnfbISV92P8wNvu+n6TcmzwOROrHh56mWdF0S7ROzGYl916+JAy+T6AZPcgLCEBZsHlebFFiyxJSwHuNUjl9Nq+z0USpi5NZXahJvQFE6qFQCmUN1awkKoUPCeVDtnM2Nl0IXgB07wk6oVWipyV0itB3xCCszb7vsVEKYGUSEjpptDXi6KSE1pakosgoGE3ohX2pjUEZxoBmDAGfazWkEMFaSdnFM9T0sVYyZqxWkq9nFMrqHEvTWBxeQ1AalaBAY5UmqIj3vay/QeGcIWqZs5quwwCFccwWc6Ie2KGamMgFrgTgJgY0IjsQcsDfh57oBUgKasNy9N6fq/ffd52MsbyWlUWBMQoIxNhnRmSiKEqqqpLCZymOgSbnHLPZTM69aTYmfAbzrDFUiwUGi++kVkbXd9RNTdu2aJ3XfiXMR+syOy5kzVEtQSHvPT6FXAgs0GYGaczzbcprgDaG0Cc0GmtdllswGONommZksg7PUUoJVUp/D3PTIH8Bw4wWxTZLQ1GyDEopRdt3pABOaazWLJuaPgSsNWhj8FEyXnrfEYLHFZayEFDe93JtY79rTdsIO26YD4rCobTBRAPa432icBVO6y3w4WNtiUBZWOo1FOYBYv0KHv+ML+DGtad5/NHX4YoZn/t5b+bX/tt/5tozh1y+8BhPPfskx0drVvPELd1CdHT+hP2LmsKVPPDgPh/4zQ+xXge0SxibMFYRkyJpy3rdoVWicImqTFgbiVqYfljN5YPLdE1HiJ43vvEh/uuv/jY6dizmM8qy5PRkzasfu4RzPSfLU2aLEpvX9eWqJjaergksT1ue+J0bhMahgbau2dkzhE7xzEc8hV5y9fkjFruaLrTsmRlKw3weODo64uKFyzz04B6HRzepFh2L/cQTT3+Q9Gzk0r2Gyw/sYPWc+nDFY/e/lpPbt7HG0HeR+y7fz/MvXOXoeE19dIPQljz4ipZ7LohNEFPAB/CNBzWt+2KpZkUO0MTx87Isx+f17FoxtV9GzeQ8BwNba0/btkLQSAqtNutajPIsD0E0Capvy6kN+x7Y18O+x6BaBsu3fXBZ98X+Ox+cfbGg8rDmydwfspzLBuyf2nSbGjZSk2QIBG7W541td3bNHuepM2v7xv/K8m8TG3HoGzkuGfjcfDZsOwXzz/oAw/GnxxquBZB6OvETW7sHe+jMp3LcjSHHYLudF9Q/7z69WHux3073PWhIyzmcveewsSOn13BO/6mX7w8NiExKidVqSdMIiatrexRmrE2klcFZx4ULexirN3ZMEAnCrvWcnpxy8eIFnHWZxPHy8Qhj5Nm8evUqp6eneB/Q2lIUMxaLWbbdpR5j8IIdaaVkvRnG1Zl+HXEgtvs4jXjEAIgPPmTKigdDfyVIm4yD4bebcTGA6Ik4wFlkAH3E4DITXU30S+QLRvxN5WOrwb+WY6Rxc8VWX2bWvOiUZ/8lsuHlqHP8Z73xRbZ8GLW5tqBCLpA77F8ypclBpjheMKQgkjNCTBc7RuUsHpG9FjtgIB0OxEghUsbxFPTmgifX/0kG0Rfm1+n1m1j6HVq9Q8IJYKhUJtgNTtjQ43IiZ4tFbk5KIigUlj7NUDriQ8QmwLdYVWwmTBTWyEUaW+biWQFnNWW1h7UVoVfUXcGl8iKknr5tCeqUmYnMy4ZylvCxY9WtODopOVrdwqcl87JkVpRYs4u1GkVLwZLrL1zlxgu3aKKiOliAqyiqgl0Cerkm+YZrh7d54VrD6bJnv3C8+vIDfOUf+ZPcc+EiH/jAr/BL//G9HL3wBL5dstjd5eFHHuKVj/8hrtz/GvYv3s8Dl17N/vwyLl3g6slvouNtVv4GJjpUVHjvMGYXpUqMvQ+NoTBrsD0q9kSiROlUQddDXdfgD7FKoZwFJeB5sjPUbJei2sEVhTzkPqAImNQTYgs6kkxJosQUe/jygFjdSzXbpbCSbZAmYPOAQWiAKftNQVJJkpSG+6cGVVnGh5WzgARjLGuAPAUcHR/uzUGnyrAbSDzleNgwYmDQWh/2unlmzy6UG6MDIIvKyUQY8/HT5lcpA/paSSVgUpeDGIrIHJIYXVK0LZHwGAXWlmL8pIjuDiEtiakm+IAxe8yiow8dVdC4uELFRhaB2BNVoMXRJpEkcHYHtCWgKaw4ip9Ik4lyiNzJB5tJPwMwKCJxDIDlLt4ybLb79WM9pwHolpQqrZLkYgWTgyEa4iCwo7d+JwILcTQK5dh3FjtJKQqwhYzR8ToGgOXcc942YELW3xvZy3nhjDEQgfWq4erz11mvWu65dC+3Dm7y3z/wG6xWp4TUo63FlQV1XTOfV5TOieZa22B7S4hy3cYUEjTIqT5ilEr6VFk6go+UZQUkdnYqYE7bdiyXK4xRWOMwdmBuJcrSjEZv265o61NUCqicBmqsMArLsiDGQO876uUpdb2mruutNLyB6WGtFKSMMdD2LSEkUgYDutRw+/A2q9WKWTVnPt+lrE4oqwWz+R51U+P9asO0iFIYRBZIz3p9ytGxYTavMMYwn8/Z2+tomw5Io2b0cG+HANx0FE6ZJOrMQnmeUXp20Z9+ft6iOgXmhjE3PdbmN+c/C4OjIdJbkQG2NVFYjUQlmSAf87O0aaenK0JVMisdhTWQEsH39F1H17VEbYhBsghQmhAVOsJsNmd/P3JyuqRet/R9T900rOuanZ0dlBnmQkSLPE+2AgINz5Z8qEejUOeMHvleazc6i+L85MCHkjlbDK8MsiudQWcxVtd1zelqyY0b14FEURY4V1AWM8qywmgB0E0ufopSJOPGdUwPC9Lwfx4/m08UDEVp1NaWjEVx0rAWSEX3tus4Pjmm7WphWU8AzaIoCYsdtIHlasnh4SEghqMYvxqtC8KFwMHBRXzwmFigs9GoYON4bzlYuZ9zBfpRMW1rXG/eJlTeXA22KJKCKfMi2hC8xyiFsgXaBrres1yvqJuGajbD+0BdNxgt7N1NkCRlrXklBcozy3Yw4mMSebbeC7CcorA2Ywh0TSMAWkqAlYwfLd+v6hbfdWitabtedO9zYViUBOH0xriQ60ubeXoDjg/3I249+0MwQRyZ4VqgKIrR2YlZuiclUH2f959ZqymiNGOq/QDypTw+fMh6jsljDMzmJbP5XBjc6hMrCi6sc3HuiyxhFLMDUVYVrqxIwaOVIvg2g8+RFKBXPaDyPKrPPMMRXZSURqOsIaqh2FNP69fEOMjaOBlfBCn4pYXd7HsPnUdHRQyeiMdqS1u3dKHJsKcAwzaz4UkBnRSFsSSlsIWhKh3dukaFKEzJDMSHGPC+J3gvc4jWJC3At7ZZM1MlbJaEtCh0AnQipo7oFUYpytKQrMbqinrdYFSico62D7QkfAyYogQnLNI+eHwbmFUV5byiqirafk3oOnm2tMa4XMDWKFLocEZTFBVN17GuI9ooZkUpUwyDXRLou1YCxUrhY5QixjFS57E/m82oqorZbEaKAR9bQvQonSgKR1XNqMo5fe+JoZVUaa0w2lJUJSEE+uCpmNH7gPcdvQdjnIxzlbCFIUVY1WtiilRZskmC+TIXpT6RdISQ8EnusdI6swb9GIzvfStzX1ngnMU6uTajHcFIWNLqQp4/jQD6zpFSHJ+fEVQLiXImKf299/iYg/0IMSH0TQ7oSuG5vu9ROYOni0Lq0p2m7XvWXUOMAe9zvk4SeUt5Tj0mKQnEKIMrrLD9QDIQrMOHRFTi4IvchyJEkcuoqoLeSyZCYTRDNtDH01S74PR0hfEXaNMu9dKxXgUeffzVPPjKh3lT3VJW8OpXvZ4/+iV/ivsfuMLNW7f4lz/0r/gvv/bLfP4X/F/4r7/2q7zw3JL6RJH0Gt9d5/R0ybqJXLhnRow9vQ+EXrS6hxTPvd05s3kgxJqoE40X8zuoiC0sKQQefPiAEB/gA//tKeaV4dKli9xzacFyWYNpuPKKAmsj+3v7XHtuyc6ipNot0UoCzOtVoj5eonFoHTDWo1nw2MOfyUd+5xq97+lSw2xu8N4AHacnK/oucaMOLKslrjLcPLzN4t6WBx6eU9dr2vaYG4eHuKrk8kMHHH74aR579DFuHx2TouPpp5/hsUcv8Oij97NwD6H7F9jdOYV0jNICpvW+wxayBmmjMVrWDaUQGz1ptCruyGKc2pzToOxZEHeUGZ2Av+P3qDt8qyHDVubmOwPAW6/ngM0jCJw2cnMbjGUA5IU8MLTz7OOpzTuVTpl+P5U+mV7j4DcMAbDB7pOAPtuBsfHY59u/AwivkwRwh+DjwGgXe9JkkG/bN50GO6bgPbAFmA+vU7b1cC7e94TzGNgfQxOrZaNrzeS+aAVJSeA7pc08cvZ+D37rxJw9t53tg6m2+vD95n02BNJ5+5j4w9nvHuzh6SGmQOTZYNLW/czjvCxLVqsVN67fZL1aC6Dq4yZhVElWpjGKo+Pj0S+c7tN7T9M01OuGg4N9YhsmtvJHb94Hjm4fcfPmLUKQPhpIXBLkEp9Ja412iq4PxJgkuzspuq7PfScB3RQSIduSAwlmY0sOsj2C54YwYC1M/CXDUIMvxiHDfgDOtwNaaQCWx+cl41/DwUd/LG2/z4D15h4Nc4hgYwom9+ss1iPrqtjYwzOpSYackanH8x8DaiqCCiQFQYATIRRoA0HWXq2F9KDJAfmMxaUYCCSIWUkgCcteaTX6dTlUT4o5wBI3sjmDHynKDRNS74AzasHsyNjdy3G7X7b1vgxvoO5fwVJdYJ12UbrKaThksDJPwGQwVG1AztHvHBGrDFwCYPC95COECCn0FDZCrAkGPIqoHJVzLHYvsL93LyfFDsk4FrZkPjtgXi24OJsz04GubXGFZW/vPvAVhSoxscZR0DQ965WmbgyHx8fUnefi3r2knT1CgsImiD22V6xWmpOVJRW7+LVDF4ZVe4LVu1hTULcdN09qrNJcKAru2St58+texd7Ogt/49V/iB9/9PRzePqTQkYsHMx64cpEQVxydHvPs00/x+Ge8kQcefh2L+T28+uHPo/+dU26uPD4lYtD4tEDpgpAMSi3Q5p7sCCwxaCyR3keUOoBiRhs8vVd0a49hRbkT0YsF0c3BznDVDrNqBlrRdD1dtya2a1zspIilmxGNQ7mK3jhCdQ92dkDpynyfs6wJm3s5OutnwOnhrZ6miTC99zAA3dNHcyrJMkw2SiHpt8NW4/MvD76ZYAkDVzDvARhY45tjJNgUCh1/ur3BqEvlA+TgjcoVgFViZCGjFSn0OF8zSyfEpKkbD9ZJRFNLKqvxHaVWVKZEhZbY3Sb4I0hrCrNC9RGnPEbPaJVC9Q06HUok0QiIEVJLdIaUHEVV4Yo5khVssuzRxw+ygThykDIjcsquFQcFxGEgCsNpKKJ6xyrLZuE8+/fZRfzcpqQyc/QepT1JGVAOox0pemHFS2FmUIGNmjF5oj4LWg6gk/wvgEgQcMwMoLTixU5tMEKHNPmQIlEJMNN7KQ6oJwB8DLBcrun7yHJZE8uUiyGfcLI8oQ89pIizhmo+o+06fAh0XQshEFAoYym0xViLAUqUyLAYYRv2nSclqBYO50qKoiT6wGwmxYLLzPRo2w6njSxkOqLp0VhSisLKMxarNMvTGkhYV40LaIwS3S/LkpSEvTYYnCEE2ralbVsxWOo1KE0fJOJUlBVaG+q2RynN6ckxrrAcHBywXC0pZ3N2dvfovYDuTVwD4qiEXA8AerTWrFYrDg8PqaoZ63XNatVQluUoqXNe2tV5Rj/kRT17w2c1lF8sfessm+hcPT+GOU1WPzG4t8f8nU/KhsUp+1WgDEYpNBGDALTEbAJ+AiD60c1jmrLk4oU9Cu3QSRHajtj1QkzQCoxF25K2hz4oiAFjFWVRMSurbOQI+Ht0esKFSxfZLefooeCTnrD9p38PT95goGQQ3QxaoFnCByXBKPlbGAPn3T8xwrJkQYqsVkvaugYE+HfWCiCXMqijtcz3BsRVDJsbMYLKueqFSsRxJUqjlNd02+GcYoiomAMxIRKTJiRL3UVOljV1s5Q9pZBZlZa+CxRuhjYa76GfaCzmBGyZx60FZ/FKilHrPmBTNkiHjJl0dkTJeSt95xgeLkBtvYOUcqA7idFPEjssxoC2KveMxhSR0DXUbcvpak01m2NtQQwQumORCUqSbqmUgEvJSIEo6+wWqNA0NUPKbsgs4qZpiT7SNDXHx8eE4BFya0HS4FNL08HytEETUcowW+yy2EkYHwlJ5LnkmkaPUILwY+Az3PE3TB0SRUSNQQqtZSwq3TO4I4P0U4wBpYUdE1MPBHEKsoxUTFKwdUhTJ0j6b9c3oDMLRwt71bpCJDQ+gVaWs8yoi1hbAOC9AuS9tS5LbAXx2oaxkPvDe4/WRkgkemBA6qzXKw5MH3r8IG2SCzyRhgJxAqwP0hnOFRtAMniMKiidE6eSRNN5sSOUAKbOSIEurZTIySSYuQqcpqxKrNWYKF2ndBKmhBI2syXPBSHXzrEy/q0zoMV2MUaKCOsEhEgk0XsZA6UrJJsKjUHA1hQ9bb3Gq4gyClcUlNUMMLR9Q0oBY6CaFRSzQiQggjC/pACZxxiNUw5jFGVhIUhGQmEtzCoSEmSKKRGV2BJ91xODxxhFURhAWHcAPiVslk2ZzURWru1qfOxQCqqqxNthfR7OZ3A2pb/arhtRDuMMUSMZUErWnqZpUAiY4YzNYLBItqWoaJoGYyU4NMhZqZQIbUsTI4v5HOscxCQsMCWzZ8jp2EYZzFicUeNcKWxxHVEYkoqgwTidizdL2nUMga7t8L1Ha6nLEKJIQSWEwa40OatNgnhdL0FinbN4klIYawgk2q6m8x1WG5yZ/FZFtBWA31iNT54UIpW1GGdJMWG0xhpNUVpCSvjoKYzCWE3oJYhoBilSyMGAj//5Pn6uxHc73LP/av6ff/Z/42DnIVQyXL15lflsQTmrWNdHYCyuWBC842D3Qb70j/7fefMbP58Pffg3uHV1yV51Hyl5XNmxWi5xVeLBewt29irqRtE0keVpwrmKdg1ORRbzCuNWFFbjU6TvEilqbp+ccOniBa4+dcTVG8+ye9HiZobT04a6fh6IzHcrZoUFk6i7jvtmO8RwlOuAKK5du05M8MgjF/jA+67jLCgVmFWaqphRNyf83/7M2/iX/+pHaDovjFPfs38hYRycHkNfB+rac+nygje/8XFunDzDyelt+i4XSrdgnKeYwcFlx83jZ2hbjVUzYuj5wH9/H953/OE3vpnHHjogpZNRNqRpaorSkZIXuagY8RnU0jnzB4AUUMqM64pMqxPAZgIwTzOWsmk3timpY2gvVo9jAMherCXuzKgazmtKYpzub9BYlkC+2vpu+vuz+5raZgMzdZAymwL3w3YDSH5nYXLpq67rtveZvxv6Y/r52N9xO+PMGDOu8b7vUHnOml7DWRme6XVNfYG2bcfg9EZDXM7fWLN9Ez+OFoNoaGsiOZwq9kiIko2QJaUGHENYvtl2mUjLooRwGAbt7Sg+hZzzRuZvMPfk3uSM+QzoToV3RRIooZzOPrKQtwbsJsQekUf1gNge6AHTyaznEczJ/5SCbJ+mIFiP0YakBoA00dc9vgvEXq5hd7GHVpmcEKWWUNN1WOdYt7VIjM1nnC6XYzZCDEnWUQ3GKEISgtZU9m/AH8U2F5/WWUvTeK4+e43QJ7q2o3AOpRLrtcirhRhIKm3Gk4oYXJZoMXSdJ0SPNkmyzqMGrNhURCFFRUXuLmLIuvchjcVxk9JEM0iwpBHgvqPuVBJSjdkKUIhW+HCPGYNoMe9vkD3K3qoajjFkbMuOU0oyF4xz2CTIMozVCZgfxyKcgz84+L9T31j+GaWzBHgeJ0Fl1QexKQbf2Mde6nUh9jl5vKrh/IZnWIFWFpEszOSWBN7na89jK4ZADB4y6UV2PPHtlR6zggdC0gTcfNH2skH0m6dX6Nyc1lUkXWAHJFJA+xHEzO6YOHtD3w3fD5N42uhA+eDpfI+JEUwhRpTWKAI6dpig8QZK5ZkZxX2LHe57xavpF5a+6ymqXS7uHbBTVRR6jaFB6wW2rCh3HVY7/CoQ/ArjV6S2hygsm7ZZ0s0TTd/TBTg9PkHPako8zRpWq4Azidh7+q4m0GBdoPEdWsGimmFCy7prmTlNVWpQkdu3r9F3NcTEYm/GweUDZge7uNmcdVtT1Ue8cO15QPHgQ69jd7HPQ5f/EOtnTll3nlJXxDhj2SSimWPMjMZ3mNCgk5cIdYxYHIqS4GdkVUtCdBTlHtbNiK5ElSVBGcGCU4IA0fcYv8Z1RxRJjNhYzegweF2Rijm22KMsZ1irGbII9GRAKflg8pCcXdQ3LF8BvdPWZ3qzFwb2sDxWaXxoRE82oY1FR5kQlDz58rskO1L5jdrQBSf73no7gq3jVunO38uzldCpEyaVK9CFw2Td2Oh7UugJKaFDjaXJETMw8ZQiJWYkUrJ0Xot8i/HY1JG6G4TVh2i6m2ijmS8uYOzuWCxBJ0fo1vSpQekZIZxQuiWF3WNuncjsaAemQqUAIaJ89jA/gRbSJvqf/xjfD9IECQHRdV6EJlnHW+08AH14/5JAupJJVQr3SQVmlQImbQI4Mrfk4n9JmN8pO4qSXhvZBOhgmLiHNXyY0AeW4stpMcasrS0sragFTOm6bix6p40RhzA4lss161VL30WeP3yBZ599jrZrSClmpqKnayNt07CzmAHQNQ06JcpSUZUl1ji0E/DDhyCa4XlhLVyi66T4SVXNhBnVR1SVsxK0LNoxBCDisvRA9KIVGvIiYpQR59zaDOiYfI59ZoxEghd2lbV2ZJdNmZ0h1KzXaynClB3kJhfLm2X919/+7d/kbX/yT1LNCsqqYPdgjxDuw/uethF5mOBb+r4Rx3mycPd9OxaI6dqe9aphNispym1DfdPOH19Th2YKoA3fvfiQVB91mxf73fTfi2wk32uNMjLPGpUw2ghrElnUh+S3j7cdH59QO0vhNLuLmRS3icJaEMaslWrkWv75IE6Ci8Jy3dnZISU1piGPepNKUxRiQihA5wkhJHXH9WdsfBKMyOb2aOxlf0QBKo4ahENfTtsA/F29epXnn39exqc22XmTf2RmVUhRzn0AD9UZx3C8FdtAx0v191kmjRikCmMcs9mCvYMD5n2FUgkrEztaW4yy7O3tZ+aKyMrElAuEKUlVjVFTVCWSESBZL9aZkVU1dUpfbC49f7yd89nQv+Oqm4FzndlIKaGMQgdLD6xXNbdv38ZZhzWO6CK171meHFGvTkTTX8ncnXK9BmNzQAVhc7dtZpvHRNe19H1gvarpuk6Chtpk7eWZSLoA1hiKwklAL0ph+wFwb5p6DOwNc8Eg2TXE5AfwYlrYLI03eWNvCFlFwPOh4JswnwxakQHSlq5rxYlMAp4rLaD64C2qHAwS4Cehk0EKxTdYo7GmgJy6TzSk+ImB6FobvJf5umlayrIUgNpKBpc4CkbWhAxUB6VzmqsagQ+5Tp2BjoQzUlCv8x19F4kqUbgCk8DYntiLDJgrbHbehA0tTmlHVVhmbsai2CGGSNPWtH0tAGxRYo3B6pwdpcFay7yqsqyVEk1p4zJYqjDWEfESNMlSSihFUZYCQMeITwntHNoZfGjp/aYIHCgMUl8k5ixFbQU0SiHSdiKrFqMnJJGkUZUEgpQRw9Bag9YKZ63otiNAqg8SBuu9p+1ayrKg61u87yH0+KbFeIs2VshC2RhxzuZgPlKcXouUkTWasnA4K9Is1oh0UukKCutGQLftxU6qKotSkb4bpNgk2GCy7rJSirbLa22MklGowDpH4QqstgI4kUg5eFDOK3wS3d++D/RBauBEFaWYXxRCQutFXz/plBnKoivee09RFBhnxB81Cp8iOmmiEXvBx1aAB+1zH4vNEpPo3KsEoe+JXoDUVb0SIkOUegsxJhaLQuyvLuXimBbVC4nFxyQBYbfRSfU5480Wjvl8Rt9JtqdWImsi4BNZbkqKnzlrUUbj+w5tHa6QmhV92xNCjy0KQLLyJHCrJcgYMxHn42yve8UXsjN/GJ0u8cRv3+RzPv8Bun7Jsl0y25lx9caTLFe3uO/yBV7xilegUfzqr/w6H/nwMxSF5U1vfBPHx7d43ev/EMv1kv/vz/4QbTimKiLa9SRVE+lQxnLPvffw9BNHNI1k+hwdnvLKx3Zws4Im9HQnntgZVrVn7te88tUly/Y2xu1wz+UdrrzuEd73qx9Aobl5q6ZoNPde2eX+K5d4/vkVpH3e/KaH+fDTv0FRCGlOGbhwccb6uCb5hE0LjA7Y6gb7lw/5sv/1TfybH38f82rB/r7lM9+4Rx9PUGnG73xwxYMPPMDOfk3njzg6qVGxIHjFfDajdBmwVho7T7h5RXujZzGb0TU1O7N9vPc8+cTv8MgDf5gYEkFF+thTFfMMaHrEW43ja4xkwEbmXGOKce48q3F+HpN7aw3nDBs66zEI4HWn77RhSMOw1pxtim0bYAq+ST0GAdWGrQdm7HkkkfM030fZmMm1TuXNpuc2rM0hhJzVtSHiTO3DlG3GLstWGWMygWcAEKfErk1/GmPEu1MbuZapbrpzTtaKM4GNs2zzs9eqc/bvEBAYwP/pv0Q4t/8/lqYy0KlSyjaDSPiJDatISWOyjrME+RnHzGjSqUmui8q+bhxGwtAGn+fOz4Y+HQBHxv7dtr+nY0/IdoPNFcZ9SXwn2/Z3BGzSBkyf2Jvi1ye6uuX0ZInvpLCuspau7Qghs7vVIPFlSDoJKznEHPgangkh6DRNI+u0s8Teb/WFjEF1R6AnJTi6fcLx8XK0p5RW+Tn3DHCBkkUwBzEkMz6GRPDQtiIHZguNTimzplXONsug80S6ReaU3BeDk8RGFmULW02QiGOAdjKKNvdoeqVqmygybYLbTYMK21rmMMwZ21kmqHxi45w4nFe+vaM/N+3bzXOntcp+4CZANkhZxiDSfirfa1QmFUVNDJ4QIhDG79UEUI/4fD4RbRxai72kk0KHSOwl0y9kDEQNACYbsD+ETTbFkCXwch7vlw2ir9gnqRkoi9Uak3VpyXT/gee0cUrzzRrh8umDON5KtILCOkKvSSrgjMOFHps6jF8SUDRqjuojtjvFusjswh525zGC74hRjO3CGtFRX94mpMS8PGBhLSQHs0ss15a9nQVdWLEOLZVucFpkYnxoufHCIavrv023o3GxZ7084ejokD3lKXxJ61ckatacUF66D1s4dFySuhUxdNy+fcxq1WBtSTWruO/Bit21YX93xv49+ySlqLuahTWsm1NMNef46Jjof5NHHn0tlxZ7PHrvGwk3ZqxDhYkFTew57TpSWBP9TTS30NGgOSAki08GYwtiCvRdQseGxUKYg24+I1WaXif6FFChIXbiPJr2BNMeUqSVuBUJWizRVALOOtE9tW6o8pzR6uyVjhjlAHqkYRIYH4utMYDaxCU3837a7GPcdgBw88hJUZiZSQwW5Vv5nTUkKeObjY3hAWdrn2ebUpuvxyrJ4+cqM9RlRUohBxpij6YjUeGtEUKj79C+hdDhwhJSQ6NnBMCkhpKamfOQDDYVxChSDbQ3oHkSE25ieU4WrOjwbSCqhE+atgu062Pm1QxTOJw1ONNhCiMFZu2MTs2IbkZInpjvXzinuOfH0jZRRnKwQjor5sU1qmHllH86b/9iBPgXAxBfCkQX+Z44zhiM90VLqo82KGWFGTWA+okRMJGCo5vjb46pRsPyrBE16GVtBs3WMiR9EFMGzAM6JDwtbSdM7L7vMVkOpShn9GvN6cmS3Z19ZtUezz3zDKfLUwEFtUYbQ9d3pMz8ODruRp3NGCJt12GKEqM0XROkyK82IoughAEbQsrMQYe1jhAS89mCvg9cunQvi8Wc+XzOBz/4W2ijWS9PcnZQpO+zfEvfC7iHGnVHpV+2F9y+76VwWjaEByNVWCCDMadAGQlwKEmLJglzLqXIb37gv/Hss09x/0OvZGd3wWxecXpsxShRItOyXke6rs4p4SmDWNB1HavVKTeuX6dwFUVRsbNbEUJ1/phSd/yx5bQopQcL88znd47Vs8Ggjx1I5yV+l1kbOjPlMmBldMIo8bt1khR3rQ2Rj//57rqO4HvJdjjjMIjG/IY5XpSlSKYkAQ0FLJRCbkVRkoDlcsnpaslip8LoKms/p5EBpYwd7eWt52qc6yeZKmlg9viccrft2JxlCSklKZvHx8f81m/9Fh/5yBOkDF7IMzKVhjEwMo1kPjnrJA7OqtYZ6BkdACRYeE6/nwWwB1A2pYRzlvlsTiitEPyHZwQl+vbGgoqURUXIc1dKAZUlcPo+4myFtQWFq6jK2YZR/xJjddpHL7dl/w1Spj+M6+F2Zo7S4oz6GKTIqNIUrmBvd4foO1b1CVevPsOtW9fwvs3gACM4n5Kwwn1Oue69z2aDSNygHbcPj0gpsL9/wO7uHgmPbsR5lf4yxHmJweWAXsfp8pj12oBKmzTZFEUWKm4s0g1jZgDTJ05dHgVn2YNDf/scRJT+ilnKK2QmTZIMH5MwJubxJX1ljc3zpQQiYxTGurMGlBSq8l2Pii2kT7CeSRjOS54fn9meKSX6rodcjDHm5Vtbg7IgmQMy9suywBhH23Z5XclFaRWi9WxLjNaU1QITIzFavKlBiZ5u1/VjgUmtFYUzFMayW+1wYfcSbdNx61ZP1yE2fjWTecN7ISeohEmamauwQNM2JB/xnccHL/KVSpzV1ndZwkWCZ4Oz1feBGGA2n9GHiPdIkUefMBqi0XSIHE9MSTJWtCUkRdd76nVL23dooCrKzLqP6JSZh0BpLS73Vx+jMNZjpOuFidr3UkBUG4UPPW3XoDJo7UOPIeWMLUNhNEYZnNZoZ6X2Tg4+DczWWVUCmr4oiFEC4s5Y2q7DakvpKkKMY3ZPr7zYzdZiCslKiLnQsczpsob3vsc4g1YWbRxFUQkAE3phWCphkxtrcK7AmLiZR1OibtcjuKQLBTrgVU/SJc44ihTxMRKSx1SFpFvrROi9yPxEQ6E1IeZ6PNriCkOfckaDl6yJGMSbrQoH1onsSPTi3E/ABFdYoi6x+dm2vaPte0IOMtgIKkSGR18P+qkRVIyoKBkkRSGyMzEKyz3FgO8Cqe+xRUkfIzpGilmFLQqU0qTO57klS1ikkLMGJajyYtlrL6fNd3Z5/LHX4JtdlusV63qNqwIPPLTPwYEDp/npf/FeHn34UZYnn0nvE7/12x/EuoorDzzIxUsX+BN//M9w8/CQug2QJJuJoISYFLwUoVe7/Pb/uEboI5cuFjzy4AUuXCooFxFdQRk1637N1Rduc3IcSN7y6OMz6mZNNYNL9y144YUXJDPDJ3ZnFabUXLu2ZLlsubh3Dw89coWnnz5if/YYN164wSteOWN3cRHbX+HZpz9MWdakZDg99VycK46Or/Lrv/YEXZv4gs9/E3/ij70Kz/M8ffWDFNUOr3nDZ/IzP/XzvHJh6Po1NipUtBSF5eLBjKDWhOhpQ+Dw1opS9wSvODm9yvr4eYwqeeWj+zzy4KPUJx9mfo+BaFDWULdrIZXgKQqLUmHM0EkokelUWTog9gREO1nlTEeRvkyQgcYBlB0zfbU6I5uyycSLg7+bs/QG+wElWbooCbrHKGxMa6ys33nd3QTEt22mYdyTJTmdc8OBxZeSLbbG33k28SCjMgDLG5tD5+sfMAO5FqWNBOl8GO0zsQ4mTPM8J8/nc5bLZQbTGQut6lyMfAgwjIEHNBgxZCKyD22HjBdpg/yvyDgk+l76wlq5/o09oMfXvvdStDI/0z6045ycUsJak5/xT4y8Ntwnqdw2zLG5Bk7+ZuObq3H+SqSxcGJCbNVEzihPZKnZvGWaZjRsS1QMUhuSfR/G/Y+Z9xOsbgBMt1A9tT1GBiB7+FvrNG49AEBKCTlDjiO/t8Zhkh5r3sQY8b1ImxVFIdlCoR+PF7NNN9QImBJbrLEjuaIwRZbo2tauH8BeCQilfC8Tp6enNE0DyFo5PDdjTCtl7Cqlse9SimLf9lGkpJqWslpIP+WCo+IjCKFHarcNQPbwzGZrdPCLBkSajS2eMuEmO9oTm/UMtjXuY2J/j88k42824LqafLXt8w59NWJkY0cM4/P8jJc4irMPeuRqlHfRuWDiGLjJB7mznoLJzz0MUoogJJ8YAlaM2KyQkDIhCayyY5BE5SChzDs5MBWH+fFOX2oA0EfA/mX4/C8bRE/VJWxRoq1E/QeMKzGwQKfIar6Bgw76mX2pfLaJhMtSMEolFv0a16yxyaP7DhtXFAZcKtnpe8LpbWbplMXuDK1LKf7hI74PhJyeafqauIrgV5j9GcYGIpo5FaQVM53Yd4kj1ty7o7i029P3t3n29m9ycuMZOK1Q3tPVK6rCo7tjusx6QCXWvsbGJYu9gtmswLlAijWnp4Z/94u/yKsf+Szue+jVPPL4w6yWJzIQsnOvEbmC+vSY3/q1/8jFi5exrqS0iYsHByx8w/27D/N8rVHRsUuPj8d0YQ39Ic4EYqjofKCPitN6nUHFjqQcmJJydpmirLBOoVSDCh58QmtJ69VEfHtESsc43dIHh08VfUTYv0WBK6oxJREy4DwC3JsJdPvBG95vFvxhbLDFEM8TwxB6SWxmgGGjJLrhpMxE8ZHgW2b9MZaemOa0cYaykmqa7fozA/acMTwcZZgMkyywwySkFKQY8W2P7mtUX+NiQ0oFUUEKYiDjW5RvSd2aFNYEnQhWE3VCpZaYVgR1TNIa0hxiSeqXpPYmfXtI6G8T0/PEULOONY2aCwtIl/io8V2PM49RVDK+NC3O9JiyI/aVFE8qS1qcpAkDd2jUfIzNq4ne5dhPEoXtUyAhGvpKNRgajPKAY/p0nzfhnGVtTlP4ZI4YbkZeYLBERA9VaWErSWSxyBrHWR4gQYxGzitH6lOO5CYQR2G863mxTzKRKi1FSmOSiTcmLWniDJqHemMcKEVAUtMJieiFedV1Nev1SU5P1ZSuoqlbbl6vef7Z63Rt5Mq993PflXt5+umCixcvcrI8wUcpOte3PhvEskAUzo2V69uuZWdw0MjskKhywQxPSpJ+H0PC9wnnCi5cvJeidLz1S76I+y5f5v4r9/PbH/wg9brmve/9t6yyxpzJrIGYxAnQ2lBZKXCaSDnN0tJ1huAjqPVW8chBb1DJzRznCGsL1MACVsJY7nyPNpaj24e871d/mf/1FQ8xLwr2Fzssq2MuXbxADD11vaTvLK3W2diRYEHMgFVd19y4eR1jDAcX9mjbfZkTsrbiwGwehllMwuJWmBFI0yoDBpqPqmO4zTLeOCMvGQAa577JNoPtOIAP8gTk94NDEFHRolOB0h6tYk6xs5L6mAQg0R8/hp4dHQmItF2HCv3GOByuDXFKrLO4Qu5l27a0dUvdtGOATTS/T1g3a1bLB7n/8iWqqsI5x3w+l+fb93ecg9oyvLbPTWQyJEi1Mfa2+3twYADatuXq1as888wzrNYrsBpt7ah9LqCfsD6nALSwvc+k706MqSFot3EOmNbmnFzHdtNaCVvaFRRFKYX4IigVIYTR+JMxJGwf7yNt04teJ6IRqJUmBDDW03eevo9Ym6Q4ebqzP84bjx9tnG4uBEYZmyHgMTHKh1TQYf42xhFNT9c0rJYrmrpmd2cu4Kc1JB04XR5R10sBvtNUWk0OqHO6ZgwB7wP33XeZ+x94EB8FdH3+6vP0vuF0GUF5yqLEOSdOVZLUYW1l/u59K3My0z6ZGtt50srXtNUvo5MxuadpMNA1JMkAYZh7GaTlFM4qpDizsFmU0RiTRodaKYXVWmQ7rCNamcd8b4ipkzGhIqooJKCSNDF+YiB6iuB7SY8vC5FZHJ1RNCrLy8UsdSH9NIFaJvOT955WtfncwuCiY4wT219XWKWoZgo9c1J4s+/y/KtH3WurHckHgu+BSJFllgpX4IwSkkeI9DHirMNpk1PYoSxKQoxSTDb0AswocVT73tP2rWR/KU3UuaDXGBC1aFvQ9w1Ki66+7zoiClM4otb0OQjc+Y7gIxZF7DxdDPgktQDmhQWjianPWTFa6tooRWUtKiZS19GnQB8iXa/ovTj5VVWiVA5maIVzBUEb+r4jKLJEICQhseUiWjo/M2CN5Hx6xAFPUYLdkukW8X2fKS0S1NBa5SB3mx1tAUGMdmgjgHzf9+Oa2vcdkYirCogiWaAQ7fUMz9D1UhMlQX7+Eq6Q+hK9bzFR5tWqKnHR0nUdfepoQs3MzYWpXxR0UUAG4woJSviAD4EUOpTVUvgrJcm8sgZ8Tx+zrE0ElYQFPq8qTDWj7roRNBEdfMkE0F5SxI1SmR0sxR6tEaCg0Aqrc8q4svgEwQcavxYZMCVF6gqtKIyhC4PmvickSVLvvccUBYnMTrVW7HOl0VbT+5ZIIHrEL0lCOHDOfdzP9q+87xf4z7/yq3zB530Zezv38x3/7/8PBxctN29/GB9FauWe/QdZr9b8ix/8V1y6dB+L/V2eeOr9vPLRV/Ar/+XXMXrG537e51A8b4Wp3c+IdSSViWAhdJoXrp3SrxMPPFjxqscPKApNoKWNCdVpTmuRSd3fr+jXNctDz/MmUVWBbtHQnhT81m/d4r57FhSV4WTpuXjPPl3bUp+2vNA8z81bVynNnEsX/jCvfOBBjg9/m2tPP8mta57Xvu4BGn8VYsFyGXDO8/zV5wmx4dKliuPVb3HtesfFewvmuxXXblzl6Paz7FxoCclQWcuOFRbi5fsuoQuPTxGU49b1jnpdceWB1/LBJ36TdvkcGsUrHrzAfVce497LjpIjYpwJYETAlQaRx7KknGnijNQVSAm62JGIo70mZLOQbUsltZwUORsUAarH9YfN+B3siWGtUmC1JllNH0TOclMXLDO7Y8Y6tDyzPtcZkfmPDL5vtMWHwsJd14l9n4T1rHKmoWTqZM3ybH9Nszana+dUG3wAsmEbDBzW15GUxRBUH6QaJOtuuG4B6TbFTxeLnS3pFNmd3TqPodsGtvAArKuc2dTnzJUBgLdGirIKASBnpcQo2vYjUC377Hw/yqDEJMoIt27d4sKFC7iiHM8pJS+ZRp9A00KzkPp2I0gu68DYRyrjJZOhMhROjnlb8WVTlr9I6IHkmv2MMfgwIDl5rAxSiimRC1VvshaUEpkq2Zfc3RHkH7fRk3uVba0hsy6P2eGeDa9DZrGchspgtyJ4L2uREiJCDD4Hh41kbCk3FtBGZcmvGHJmY+63wW/Px3MuF6ZmWhtgUwB3mo2+Wq05XS7p+lw/ZiurRPppDGolMZrtEPRKkRgS62XNyckpi91KAPQgmSshE7gHwt/wrKhce0/UQ/SIwWzs2qHvPtpIGnyYbf9kCBYMftVIZmbYdGDCb4hV0+yQlIMFMh9kizFpyEoBAwpMxghj3DzLw/d31CUbfgPZ1h5ksBh/J8TGJFlv2X8dgXWdiZ0pZRJWEjVJJHAkfmJAZMY1fRdIyFxnraXrRCd9IFqdldwatOnlWj6JIPpsvicP2eBsqgmwyvDYpwlLOY03SyL6SZgAMdL3HhV6dAyo6HH1ktnqOnvr55mpgFaGIrTYJMy5mHxOb0/YQmGNpNsk5UiFyvp3nhRC9gA7QufxRy3eKJIKaJNwUXOgZ6ADzAs6G9DmiNO2ZcetuBlbDm+v0N6zUxqq0lJY6FVi2Xp8DMwqYX6m2AEBZzzOJVIwPH/zFh96+iP8sc97G29+y//CU0/9nzTNkq5tCX0kBHHsXFGgUuDk6BqzcsZvvu//xyOPPMzqcMV9D30GsdjnMO3iaem6NYXWVNW9FFrRdpEbt5ccL09YdZGqsDiTcOU+2BnG7GGMpMQqf4IOEdVEFBbrFIUOdOmEPt4gJI9KF4npMt7MSeU+upiP2kgpqvHh2L7PwyOgMhw+eTAZBP7zBMEmRWKYyKd7yrEtUpJIvU5JHJQYiLGFvhV9VF+j/ApXKDqvAEc0AoKfVTIZMCoGEETJMj6ex+Bgx5QZTqBMPvcQoW+gXWK7JSp2aAJJGaIpSLEjNSfE5hgVGtAG66TQWmEaHD1JBVarY0inhDTHmAsUgKhSl/SxxDcd0d9g6ddofQll76Fw4PtnUbEl+D10nGWGzC1CX6NMhY0PoFVJyMVQIxkUDNuR1o+1SbmGxFCMNfcSgUBQUVJ8VdbVTj3gJ1tt5u6zQM95TNINwzczwFVOB1MaKEjBElBSyEdpUBatHEYXGLUB0UkpB2WF6RVJshDprGs1wgQCBCiVMAOL3URUzKnd5NSgCaCvsu5hylHTEDwhtrS+ywX2TlivjiFGdncuCENSl9xz4R5Oj2o+fO0Jnn3yKa4+/wyHh7dYL1ei6ZnZJVabzGaQ8e+9p5xV7B8coIzFukKc5mx6Gm1RyuKcVIRPKVHN5lTVgocffoRXv/bVvOENr+OxRx/m/vvvY2ex4DPf8GrWy5abt27zi7/0S3KfgzC9HAUxefpe5k1xPI0wJVOPaJI7lDFY5SRwmreJ2fgIKRGIOfVyRlkImFqWM4w2EENm4Fh+/b/+Fx575HEuHNxLv24hJnb3dklEblz3rJYnomXohfkcUszSAsLorOs1TbvG+5bZrMz+xmDIbRb5lFKW+xEnZWBwa2VzHyZCHrtnx+bZz86O54/WXmrbIWOGoXSoGowQDcmgos2sjJzjLG/kuRtn3Y+vxRgxWsCFvu/RcVNYKqVEytIsSomeoPdiZASf6LJMwuBsKaVZrxtu3LjJ6vSU46P7RvmNK1eusLOzwA7FQY0ZJQpiHPQfcxJgPr4PiT5rTY8pvkAMUjx26jgNRvDJyQnPPPMM165dI4aIq0oKV+KKEuOcMCetA5UzWNTAJ9+ef7buz2jXKBh0x7dYVtv3eJuJPtRNkGKbwqRqUESsFkcvJVBJSxHJpGialuPjk8yqHAI7GpRhD2FtDQaoZMvceb4vNt5e7pgVe3aod6LG2VLlYEMaHSphOBlboK089+u2oW4bDvZ2SLFlf3+X3YMdIi2pF/alHvapBokUw3q9ZlZVtNFz3z0XuefCAXXnUdzL4a3rOKPwXcPJUU9ZllRVRUqJrl7TtQ3Rqsxm2cg5DY7Y5trFUbJWtDzP2jAyxtTo5EnXainsawbwQGyFkFPDrZ2uWwptpJim1hFJeBA9yiHFOUZh6llnUCZhtBShCiGO4EmKCt9H/Ce4dpug0UGRfMBVmsKWUlArAUFhosimBDxKGwEHYyBEBSpiCHTe43I/tb5DJZ8L2srYxTjhytV5bQyRvd0DmqZmHZbECMbKfO2sQ5tE09fUzRq7PGZntkM5K8UeLAyp1AQv123NAHYHSAajHaVL1G1HCjJf+AS9D3Q+4YMiZnvRxIgmUuYgWtAKZS2FWrAzq+j7jhN1QlQJV4pMUoorQi8Fwhof0WhJH9caUxZAoE0dKqix1LOKWccYoRPEiADECCvT05JMlh4gYJSwO7XRlLMd1ExLPYD/P23/+WxZmqX3Yb/XbXPMdekzy3RVdXX39BjMDMZyAJIKSYwAKYoIEKQohf4J6i/gH6AI6YsipAjKMPQVpERRDIkUoAHIAMahx7TvnuqqysrMSnvtMdu8Th/Wu8+5WV09XagO7YqszHvuMfvs/Zq1nvWs59GaWJJuZYx4B9RWjF5TIKYsIJcWj5HgezGdtpZIYogeHWwBWSxWiQzbprtiu+0wWsxSR98RgHY2E9mZEBi8F9+VfsDUlqoyjENgGHpiGEucoTBaFBNVKQrFFBhH8TdprXShWGtJwYtmLKaYmA6EpBhHSwiyZ1iVCQTpSlNSABi8R2WRfIop4mMkBki9RxtF9JF+06FQOGPQtSUphSl40ATOaWMEvEiJYRxQVtbuGCIKaKoKVZsC3HuykU5AOzHXUqYPHqcMtTXYqiKTCGNP9IEYIAYgKYy1hCQ6tyZFXMowRrKPOKuonCO5hn4YICtMcuRS9MB98Rjis8eNm3doZ4YPH/0Jp8/g13/t75LUFc9Ov0VSZwyhwlX3ef+r73Pz+C1++OMf8eMff5cXZ0/4P/5f/rf8/b//P+PV81f8V//1P+KXf+V9ttuB9dZT1xatLN02MPSJcfD8xm+/ycEB2CowRgNK5BS2V2v60TNfKt76So0KlmcPAy+ejty623ByOOPJJx39OnOpA0dLRfaZ8xevSKOn6waahWF+bDF24NmrH/Pyecev/+Y93nv3q7y8NXJ58ZJsK+o2YmPH/NDw/NVjfudf+wbOtnTdOU+ff8hmaFgcz/DjCAm+9v5Xubo8J4eOu3dnsi+YLTduHNG0t3n5YuDZ+pwXT0759/+t3+akusdf/+gveePNGXfv3+TVecfYP+Tr791CZ8XDn3zMN7/xDcbOi7nxkLFNhVaaEDO5H9DGYU0tckkTEInaxQ7X0c6pc/P6HvUamLQbGrmkNMUMD2FUZ/Y51MSCVqp4yeR9hiNsy1xQkglQmrqpAiklmqaRz85T51/Cx30srLQu65ZiYoleP98JeL6ex302x7t+vAa+q32HF0zF7nQNwBIvhqnodD3mg2nP17vXXQdKQcD9EPY6686Jl1MsZtQTacQ5tyNupGvg3E7nvLDsnXPEIe7O6eDgQLqOS8cuiK/Gdtt96bn9eYekoK93HE7A9RRLTfGiQoqbuhTIU/GekGLBFAdN4Omka512Mc+O5KhB2cmraM+SNkaJH4nZ5/dqAuPLGDSFdCD3mIIiTPI8Uh5Q5GsxqYAwO7wgSVfG5Pc1kSBSmkzaVZEXdeJBUlUir5IgIwCr3BeBMo0VksasWZBiRClXurz3BQoZN9KNoYuagR9Hzk5P6fuemEQWLoZQPIb2Mey+sCSkP1CoTCFtpSI52kkH1S623EHmcg/YrRAFlJ/m0R5k3xG880Qyhgko392/13yjXr8/n5fH7s26S7xf1iddCnw5IoX0spTtnlvy6te00VHlGsCkrV+i412hooSP5Z4WEoeacty8Wx9fP0/KnCzvteuMlw4amdO5rJNSuMjT9Zq8DFGkJFJICclpd4TugiloDVq7It1UCoCpKBrkCUT/2Wvb9eMLg+jO6J0UTvl+5VvLFJW2gXLTUyamXALDSIqe6D02jBjf4/ottd/SMlLnyGy8oh5e4uJW0P8cqSuHLYYwqGJwpTXOSIv2BMxnYFZXqKbebSopRXofGWPEe6lIRBLKWhrtuN22HNsDhjyQrOJVuuJ8NufiYMmL08tiCABgGH1ikxLJ1LTLihvHc5bLCms6vN/KIm0UxknK+NHDv6b6N/4t3nzra4T4nGfPPiGFK5SP5BhQJhPHnjiMBN+zxnP7+IRnjwM5Wl7+yf+bX/+D/xEGg0+a0UYGFM4eQM7EcEXfX7LZXhCyRTmNre7iZvdR7Qm2aiUxD4GsBrxf0XdbwhA4mB1ApQj5HMJDjNZ4fUKoltjmFq45wDiRllB6AtCv3excFt3ri3BZmLKa0vA9ELXTplb74TL9owixXAOH9q7FpESMA2pcYWKHHgOEAUwm2ZqMQVmHkqGwG4/X30srsBlaoEGYPD7DVsl5WQWqLJZewaTJJuaLPSptMaHD0KFCRzZWEs9xC8MaHTtyjii3INsakzM6bqlqi9YNmDfJw5pNd0VWI4321E6jzAmd70j+LjaD1i3KvUm9+IbIdvg5Ln5EpS8IQ0eML1FmQ6O/grMbfOyJVhYpaWOXdkx+wbayRCCr6+1aZZFTZZLvw7VrP09/rt3Yn3NMgdX1hf9nH2q/QGuLmJvYawtbYV1NmnEp7oLO3SKcp/G4D/zE6Mm8Fqhdb7ndtRvuArWypgwDm/6Ki9Up225FGAcqU9H3A6pquVhfYJlx/8497t96wHq14sc/+gH/zX/zX9N3PT5La7rWGlNVVJWjbeqiDezph5G6bskps9mIWacxFuM0JAHklIK2nRFCxDnHfL7g5OSE3/3d3+HXfvWbLJY1TS2Bg0be/2/9rV/jW3/+52y3212rPxTW4dAXsD7Rti3BB5F6KQWVODFpc96x/SZtw5QSYbr/5edbN2/xW7/1OxweHhLGgf/un/33vDo959GjT/jP//N/xH/4H/7POTo+IBG5uLokBAFLp+DYToYtUbTap/MMIdB1HY8fP+bo6JCvfOUrLBazYor3s5herwNfuyLJzxiT07///3KU6GkC8DL781GfmTtl9O2SKmkZ/fIw+p6hUK5p2I93MezJNEVvOyOsCe/FqwSlsFVVCquin97OZqSUGX3g9PxC1lrrCDFyfHQkUg7WYoymbVuaIkWghHZZykLyWcZqsrJFe7BotGvQtFwH0ac1I4TA1dUV2+1WWCt2Yp2LLI1IuExBkURkO5f1DDHH11hWu05e8mtyLmkCkT5nPHwWwJZzk89OSXSpt9s15EQs0hMxJipbQ5aujfWm5/LykmHoUaUmqJTF2gpXtYw+FNZowNX6M58lY2EyN7r+u3+lQxsZfXmHTkGWPVWC+7RLMJNSIq/ghEna991Ov7RuG+aLOU3TstKGIfYYpi4jYaCHnCEoyBHvB+paZD9iClhrGMdhJw/R9z0heGHulvVSOlTSfp+e9n6lX7sf5RN392kqjuxCfPX6mrB7TELv8p7T9RQGvjD79myVLBURJtaWAATSAeNHMXCytrTbk0s8I4UpYUFlUpjGpC4M4C9/aEQWZPQBP3iRSzHSyaKdo3G1fMdKgVF0QyesrxDQVph2ISZyEtZ3IjP4ochcFNBEy+8m5p7Kipl1OFNhdQVWYStDRsZsGCLDKHuJ6bdS3KoFwIgqsfE9lauYVTN5bj/g+5HK1vgkJX3jHLqyDCHQ9z0+JXJSGFUjHRBainRZzLVSDAzBU4fAopmznB0x+p7Rj4xplKQwaWHIe1+6fCjAm9xn4zTOVWRiAVUsztZYWzEOgxB2lEiBhDJ2skLkcaLcxxAzYNjHKYq6nlFVLVkpfBRpLSlMCdPUR8/gR6ytZD3TlqwC3vdUTrrxbGXJWRGSgMAWpGNUicll7wNN5fDDgB8G5k1LTBKjW2PRrsLjwYoZahgGwjjKnwQK6aYxWlNqkHJNpqQ2JWIIGG1F19+KGWtOkRwiCTEWjskTcsJgyFkkjJzVdL34IfT9UL5HLDmjmL+No8fZwNgNjH0xGLQOo3xZWwuLVWgtUjTxfs8M9UXzNIFzFRrR+e+ix8eRrB02KyFbFcApIwxjoy11VRHCyLbvAY1VDjBkLbmvcZqsBTwTgAZUEjai04bWNRgqlHagDNvQS/fjLzC9//xffp+Qz3jn3Tv89m/+fcZN5vGnT+j6K+rZmr4T/5u/+ssf8Etf/23IFV//xtdZ/dVzXrz8Cf/p/+l/zb/3P/2PuPfgmO9890+5vDglqUiIlhBrrJuxXl1w5+6Sdp5RdmTIPcktGMfEdjtwdTnS9QNtW7Fdr6jdIc4lxlGzuUysLxVPHnYQNa9ebFFjy/HRARcXG27eOuH2TUXIA6YKHN9cYHXF2fmaly9P+fCDZ9T2Ft/8pV9iSHM2w2OaZaadJepZy3IJ89mci4vAxYuR4BXr1Zq2qTg/veD84gW/9mvvsb46JRdT+gdv3sNHT1Pd5ObyFh9+589ZNFd88JN/gTGJN99uhchmoTlc0ywqUgveBC66C0IccVnAFSGDgCSMiqQUShmJWRIyAEwp0RaSwQRIKz0Bkz/DaPO1Dm05JrBKl66J6/IGuzg1F+A+C/vYWL1jY08a20mawXb7uYD5IyhFGEeRSdHS8RJjLAChRGUx7gfsdUNOMewr+wGyzwoIxU5a6dpLpbt1AuMz17qF9n+m30/FhpT28dlExJjyg8+C93swXZWuPYvW5pr0pFy3qfjwurHkvigwFTgmaRDvPWdnp9y6ffu1z52KENM9cbbi4OALQ2ife0x56iTjk2EH1u40pCdUszwPpiKngIPG2uKfEyUOkbZ5uV+7/GKKDfOOKCHZfJZoyYLOeh+fpIwxCoHbJhBXCAC7GFGrwl6fOgxkwJXSC1NYuZOgmWL5JMzgyWMpFe5rYm/ePhl1N01Dukb6mQoY5CzqAMUwcioSaGVLN1tFihGd2fl5TSGugMxaitNKOhXWmzXn5+fCbjYaP47S8aUlfnU7uTX5bikJKWlv1ClzKcUiV5aloywaWRNQ03WZOgD1NWxC7cBbityLdJyXu3YtrFewGwOK1415yzf77ODaXfv9PSi/ztPP+7kgE3U/LqebqHYgtLxWZFimQs31s9u/ZyoEVZGwKfGwmghBn82398WiPd6odl3w8pm5YBEis5RDFIw5I9JYSgpmKoWdmbdwKXWRyJJ7Ya0r48Xu15IYd+D8dHZqp3n/Nx9feAUwWolI/i7fKqBnqQbE4MnBo6KH4MlxRIcRG0ZsGKhDxzJtqeMWNawxYUNjFLUzGDJJy3RPBrSSJMAWhFSqbOWGMS0ARdg/F1DUSGArA6zCpUQ/evqSjEIil0qrspmIYwgSsNu6Zvb2nFv1IU8Xrzi/OGMcNxhjGFNiVBXzowPu3m84OTTkPODHimEYwRiM0wx9YjGrODt/xtnFS5xW1JVl0R7QX41cnj1j3A40bcNy0VI3FaqtcMXReLx8SYjw4QePuXOz5cHX/3VidcKoB6yt8ErRj2vOVqe8OP+EmMCYFqsr6tlt1OIeqZoTVUZHTyhtj37MjJvAttsQfUXT1lR1TWVuoWyF1kcYd4heHOKqFjVN9kzRfCvHa/n5HhyfsNVdVZNpwpXFW+2WjPKSaQHIZdEr75ghh0yOAeKA9h0mDqgY8UlMEI2RjQvniFrjnGHX8QSYabsobs8HPnEYAkfRY0gMxrKpKwalIHjUOBBzYHSWEekwUKEj+QvieE7OA6g1Nmv0CCk1DGMghwG0FHVMPSehSeMWQ5LHbUOtWnLWdOuRVxcvGVzmYHnM6B0hHqP1G1h7BBrq2RtUzT1hhhmF9QPkgZAuyP4SywGRN7D6LWK6Sc5utzWaous/VWK/7JFV0YBkHxDuWRBlMb1+/68thPL0LwbgXGc3/PyTkk2dsuntWhymFqgsvLCcJ5MpAVq4prsn57YPIlTZNGFvFHM9uJ2es3u5UgVEz0Wf+4KLy6cMY48zFo1is+5Y+YzKLaE7ZTk/EFdxNO++8x6//7u/zz/9p/+E08tTlFElQNFFc80XTfAtMURpJbO26Phl0HmnP78/14hzFXfv3uWtN9/h+OSEWdvQto66suJKXiqKxmTeffct7t+/z0cffYRSFd6LpnuMIi2j1KThKRucdZbQ96V1jl2F1pbK9ASYTwGotYZZO2O+OOBwueTr77/P7//+73P79k3+J//Ov8t/9p/9X/mjP/kznj9/xjgOvPv++8zmM/IT2G7XKCWt15vNFZUV5vleqkHAPK01V1dXzOczXr58yeHhIbN5y5E7vAZ6fWb4XAPHdw7ln5O0/E2M3s8bp38T++ZnHa8Dd68zYT+bSO0fkzkYP+/L/SscE4MAVOkkuM6oKDIpZQ3PhQE/hUjayj6stREwUVva+Zy6aXFGcXi05GC5FO1CJ1qElxcdTz99QlPX3Ll9i+V8Tts21HXD8uCAZtZImzRi/BZiEmAiy7kaLQaAE4NlGvfeezabDZeXl2w2GwAxfFO6tErKrpOymogOr6UP15lTrwGv02SfAkdk/hulP3trdsdPMSjiBAgJ40KMDgPOineBUpmmbqmqWuTWqgoM+DBI8lcMXq1rmC8OODo8oGnr0o6qy5/XjUWvA8J/UxHoc8eqAhCSwnStStWGnfbiNeB5ki8wweC9Z73pubhcUVnNwcGMo6Mb3Ll9j+12w3a9lhggX19/5bDW7taNzWYtWvC6IsbIrVu3qCsBK+O1wuZkdpmzQ+tciiXXdBYLqWL6YtNYV5jdd0CVyGO6brvHJqaRMK+cE9ZhTLnIyEWEOaKv7SsFRNNTcpRLXqdJqTDFYmIcJ1aVyG9Y53BVQwyRmDxSlHLXzv3LHWPyKKtxusKnzHYYqWthC1d1i3GOmCKzqkU7U6TLZByJTmfGjyM+i9yLdkaIMN6Tiy5oznJtlJIixziMGAzHJ8elU0U6q4ZxFGmPGOU9oszbEGVfyyrgYyDkgNO2zIGRdbcih8xCzRkIYBW1a8kK1t0WPwyiNY7BYkhatN1d5bBaE/3IEDtp9S37RbiWrFNadENIjKOXblglfiByz23phBEAdhg6JuktYzRRKTZFikTFQEgBchKiT5JCsDVmFxYp2Mka+dHjbKRtG1njTCkKxSSdcWm6xsVU2laI54nFNLWszQpsLXJ2xEzygRAiNknxz1hb1m2EcQ60TQsohq6nUoa6bmialqoeudpcMWy34rEyeoy2ApoAhEgwYkpKTlhlcMagtKYyhpzB+xE1mZOHKHIxqiS0JkIKYtSaAo2bMfYDl+eX+E6KM2RI48jEDIt+JPiRYAT4mDWtrKkpS7I8evriBSOZeSLkkVzWEmMMffD0Q49WlqZpIRejwhjLqp4R9qaA6DkGIRtoXQz7ylgJIj/kqpqkwStFUBFVaQJRwPeYhDiliyYPGmMrDmctyljWmy0heAx7UbkvcxwsT+jDmn7Ycnb2in6laJo5AClrnGv41r/8Du+/fcDLT/+Yg5M5Ta35xjd+iWxeYY3l//Zf/Z9p60PZk/RAyh5X18QMzz9doVTNvXvHKOvBCnCwGc5YXY0Qa2KyHB00IsO3OgcC1gWCj1jt8IPsf9Fr2npOzo5PPn7FjaOG7cXAvD3g4uqKem756CdXHB4FTm4e40Pg6MRx/94S7CUny0P0ds3NWwv8OHD26pznj0b8eMa8ucuN41tkdUEOl9TWMZtFTHXJ2fqvSSkyq45YLg7wsafbDrRW8/3vPOHi5cD6KvHtv/o2X333AQfzm5y92nDz9h2ePfshj8Ka07Nn/MFv/y4+BXrfU1Uz6XQoBr7aGqLOeDxJy4ahClDGlMtqQ0YAG5QSEE9RYmwhAEzyg7EUcq6bhU+/e934espH1G7vzCmRfEQb0c72hVWb4jV5CfYAsazPewZ4jIXlXga9MnLO4v+hd926E2N7+nytFXrC1ZQq4HwshTohOk2Fdyka76UilFLofL1bbN9ZKEUHRY5i+qfLfq60QulJKzntzEmvFxagSG2Ue+C9rMXOST7s/Uhd1zum+cRAn+6HLlv7OI47Ioi1irv3bnN5tWI2X+y8PqZOyUkLXkDrX0xG1SgpyMecUJoiAzgVJYqsi379e4LAKrradypoJYXkkBSpFGxSltxxOnIuXa0FGM65dP2T5bNLYUeKlbnEoxp9jash9BTJo7QBi3T3pOI3oZQiKkhxYjC/7kEm30POb/L+mMb25mpFzuKT03W9yJ56L/uS3QOelLmhsowzo7V0/GjJUdu2ZrvZsJi3HB4cSOe3taVbITCOoWjG7w1jX716Rd9LdyV6KowXKcgsZGBTWdkPtcGHVIpDAhBP+t1kXZQqoGlaUurJRrz0fPRMMYUq8mH70K8AzTmTg9y7PZtdlY6KvRTfvrNgv3bIRVH7TsdihjwVY9LO0HQPfqukRIZ4t0Xtq2AxFKxGT52Wr2vry5gqZ6/V/uekpjSqAOi5mHZmRIp2mq9lV84TOK9LZizdf9JRoco1Frw5p8nLLoqSRC6vm4qXapoPpQs5IRJPBZNMWUxipdtjT6SU3BZCmLCgck++QN79hZE3Pw74KIFHThFiQAWP9gPGjzjfUY0b2jTgksemHh23mDSiU8AmT+s0TiuyTkQjLBCrJqMKXSavpO9S9SxtSxOYhbT5q1hc0JXaFXNzLmLwZt/K39Y1VV0RUy4BWYLS/ueNGDKqbLA6U80dB2/O+NqdB2w3V2z7jrPtJaf9mgsM1XHL0WFiNhvo+o4urtF1g04zZnpBSgMhZU5PX/LBBz/ka2/fpakrnG0IQ2C76ujWPV3XE33H0dJiSQylmtVrAfpjd8nq9AX19ilvLjTbynAZai4jbENgEwe0dajoaSqFqY9IzU2YHaJcTQoeFSMhQj8k/GjwcYY2jn6MJOVROuHsIdoeovUBmKq0gZZ2i2vzYH9chyM++zi73004xFQTk5/3TMCpSgWqtMIV4DYrrJbn+uRFviVHonbEppVF3SWss2AsqWi2S2FSEnad8+4zjPfMu0vupUtusKZSij4vuEhHbJUmjBvwF4w60nGAig1DzCi/xY6X+HhFzAmyRmHROaDJqLomW0sYe2FP4si2Ek1+UxGUReVEq2XMGlth62PGBKuxIYTMxeqUxlia9m2axRHWzclUaNugs8M5g47nwDOgIqaKkBucnqN0hc4RFXpI4AjohFy7X+QwWZiGec/mlhuoUUkWMiU7VxkjJVNUuUji/HSicP2x68yB1wAfPUm65P2I2Y290smSovxN0bqiGLwgIEWIvoAVpTVxB59N5wGgERdxecak5TYBiddbC6VanslqimhK3paTaMDGEdFwl0U8RLg8uyKMWxpbEzw8fvSUJ4+f8MnDj3j+/Amr9Vpa8Iy8/3bbUVcVfQGsqqpC10Y0Qr20OCttqKxDW4dztbBiECDAe8+zZ884OrrB3Xv3cNbuKutGKzGyVVJovHFywptvvsmLFy/YbFa7qrwEFgrvBTib2ByUQEkY0CXIcZZZ3ZBSottudwDWFOzHkqhuNhv+0T/6Rzx+9Jjf//3f55vf/CX+k//kP+Hhoyd0w8id2/fovafra2azGdY6Dg8PWa0uqKoKUiwrTWIs7ZOVq8jAMG65uDzj1ekr3nrrrZ3Uwv7eXa8c78eYBITm2nj4/KT254Hjnw++vvaM10DOHWD3medNAN60puYCXqry730LJtMs+IVgtonxLuM67fZYay1VrlA64opJrTHi4WGSRZuiGZgzFH3+BBjr0CZhNMxmS+7ee8CsbQhhxA89F8FzeXHJk9UlyY9sD5Z4P3JwuOT+gwcck7B1Tc5K/D1iLIZxMm+NUjgjrKqpWDOxi05PT3n58iXbaQzuGOiWqe9USUaLRljp+yKCmLZeN44UdpR4JUzXWPreQDw5Ph+U/mxLsVKi43h0dISxGn/7Bqp8nlISINeupm1nKK3xwXN0coQYM1rR+i7FPW1qmqYu8y/t1qrXi3yvg+ef/ff143MLRapg5lC6yQoLZRdg73VP5ZpI26e2DqU8223HMz9KcKoVs7bmxs27bLYd/XrD5vJ8V/O83ukzfY8YI1dXVwxjgOLX0lTCmLbGisGuMdSuKomOEQNKnYoM1t4M9/PndpECu8ZKnx7fgex7fF3+tgZbNMXxkZQDzogECKU9Vf5nsHYqNERCGkUWKyd6L0zYYZz2RURK0Bnqdo61js26A61xVU1Vtfvi8Jc8ki5SJWi8j/icMGRQCaulWy0Ej7bscqTKWZwTCZMcE8kH/DiSc8CqGq0MPklLc/Ce4CVpsaYijp4YIlerS6zThBQwToEWbexh6AGL0Q5lBGgc/CjrQ+hRxlJracsWRnZA6YypNVEHusIkNloz9AMpBxonCbdJGZMNGENMoNBoIwCHcTWtskVbNtMNPVJgL+O/SCYZ49DKkXIo3QFZyBhJDOOmp+6ApBhI2pCdw1hDVDCOHU6Jbm0MA8pa0XIPkXEYUMbgqpq6rUUGIoUSpxbiT6FYSaI6STeVgmCGyTg8Kagqh60bxhCoqoZaO7r1hhSkcJeDmAsaI4xElKZuGxEo8pE4enCFkKQy1kkbs4AJWs4nSfemkRCPkIXlmLyHMr2SD1L4JNN7LySnrAl+LLmXJLu6gjgO9L4rJuaabTeyudqgkmicKwTIESkfQx+ExGGVZjZvMVrvJPAqbam0SLpkEmlioJHFC6NyoCU3DCnhdPH3UBaLZkwZH0bpqkAAXw24UhgkZ0JOdN2AURmrLK1rWDQLUoTRaEYd6UMPSXRXs48k7fGDJwWF1hVV08j4yMh4jwFrLM58eU30P/g7f5t//IefcH62pZlZbhwd8OTpC9q2Zdtv0Nny27/9O7x555t8+vicftzyJ3/8bT599deo6pL5QmOrgWxeMTt0NMsGY1sWJ/Dy1RXnZ5433jzBlgSiG+Bq0zOEAa00m3VHGB23H9ziycNzUtTEvMVVGqLh1q0jUsj80je+xhv332Q+18yaGQ8/+IgPf/TXRB+4ON8Qc+D0omcYIpvthhBeUpmGu/dmvP3OAWPY8MmPRly75ZNHj/jlX/o17tx8gxgS//0//S4//v4HLI4Uv/rrh9y+Z7h5q+bu7Xsks2XwK4yxbIYN8+UR600gesuf/tm3+fF3I/1Q81u/9w7DNvD8+YbxIHB59ZI//tMfsDicQQ4M2y1/8id/BEFxfPMGecxsei9FXT8S84BpK7I1bP2auI7cODjBR4dBAJsQRoytQCm891hjdrrLU7wM7GIFpUxhUsoeex0cts7tZJ+mYxzHstcpjJOin0+JZBRnp6ccLQ9o9eRltu9gVCUGiTGy2WyYzxf7mHliu++A8j3geT0ezjmLl0ExzZXz3etn5yxESuvcjskuucUehKeArNfj531sMuEAaVcEnXIuybvCDtSfrpN8xj4uCsHvgNkQPOv1muVyKUXcsJet6boO5xxVVXF6+pK6ETnCCTyLKXF5eclsPt8RF3Z7bbquB2+/EMj2Nx0SukzXcNIVn2QvXv/bXGtpuU4byEj3PiphdEZNutu5SJhMT9T7wo5WkJUiKS2kQ7V/Nwo4O7HQZdiqXd6v0pQJJFSOpWgjXj9FlFB+m0S7PeVc8tG8M9o0xaNiyjmMRtbL4qXhfTFBVgqKbAulMOT9SApSUJ1yeG0sTVNLoTpFhmGU58XIvJ1hlMgsEhOzupHxMo5UyyWnr85YXVyy3WzwXsi81hjp+ELkcrQS6aYUJYd3VuT4BEyf7pF0tBGlGD5rJV8eR5HXEdPdKR+Rwu3EKhcBhkiOiqg0kYJ75onRPzG0Sw5wbQy9TtC6duQp5JjA5r1MIeRC+JikbKc3za+/AXIzsxEMSPTxr2Mr5fxzfu2c0qTYkiMSQCSUKvGK2RNXSgZSwHa1J0OVc5rIhD+VvhQ/FEpBQrrOJFRHGemUU2IcLrCg/Hc9H8tZl2uUyricuqSRQiWvf6efdXxhEF29ekzle+rQYWJAx5HKb5mljip7dBhQvsMQBcQxqoDfEHUiG4XRecdok0Q0Fr3QvEsUtba7LyGa1dJeMZntAEX3adJxNTvAL2VIpXpiLBgtiVdUSZJpNVVkEr0fSd1AHEe0UrTWMXM1agEcHjD6kSGOrIPnMkR8laBas+4eC4Mjd1jrWNYzxm1F3SfGvmcbPD/5+CO++uYtqmrOYpk5WK55VZ2xjiN1sgyXI1sfOTo0LGrFZnXJth85mNe8/eAWMwXh5U84ZMV77Rv8pHN0NFhnaecGqCF2aH2IOfgGzG+Cm5WEMmONJLujqnZtxCRFzoG+W2FsQFcVKQZghDQIa0JXaCtQ5lRDLuvntc2iDHqmrgABPyfi2b51aA/IT69R0wRJ0vqscy6gYnn/nHbtNaoAUlEbPAajMkMGmw0BYQagBXwPEbIPuDjgiBBGZuOao/SMNj5kiM/xeYlr3+Fm5RmTZj2cs/IvGLUiqvtktSAPHpWko0JA44qkHL1qUbaC+kgMs1LCby5Fr7hqiVmBmxN1ps+R6EfRYHYncHRE3YomvSKT+p46OSqOqeYLdDPH1AeEweNchTUzbKrQo8EQhTVqHE4rGtWDasg+ofRABTJHQoWODrj3RafzTx2yqMhCObXRTKC2zLMSkOVc2v/3u/PngYs/BRx+DqC4+7fa/W8PgmXIOZByKHrknqx8cVa2YgiaEjGO+DCKbloKgNm/VUHj95XO6+e4P6frQeMOkNlhLmrXLSGJrqapW4ZxFHPQ0VMZRQiKxeKIGzdOWF2tcTHhmoaqbTm5eYt20fLq7AWbbiUmhMaUNexaa6JxAhb7kZhyYQEUo84y/bQW5vfE8Hv29BltM+Pl81f077zNcl7JzdQyJ9GK5XLO/fv3efz4MT/84Sld14k2tgHIeD9eY8tI6/QUPEqRQNohQ9HFfl1/WK5jGD3rq5WA/ibwJ3/yJ9y+fZu7d++x3vTcvn27mAIamlQLY24IjEWPte82BN8z9h1jEvOYcRxpmoaYpvbMXJzTcwmcrrXSXQOK91Xp18egjIOfBsqvP/dvAtH/JpB9z9yd1rq9ht7f/J77oSa5hHhnKLUvJqTpd1/ymLo2QpRWyRyCGPNpTVUJiG6Lsa1zFc5WpJgxJpTPnRIHdknhOHhyDrTtiuOSlAjAIEzltm3Yrq8IwbNaXfH8+afUjaPrN5ycnODqFrRhDIkQMyEVsEspYSp6D3liTsWii9/x6tUrXrx4wTAMu0TIVBXGVTjrBCAzBWTVFgpjg4nZk/fFMqWE87BbcyZwNYuci84/m4n+2QLhxGqp65qU58RoZS9DfpdiKsbGujDDFFXt5NoqKWJNCWMs19G5ipxNaR39fFbEZ8/jX2lckKU9PQngWqLN0qL7+jNlFMh3cVVF30W2257z8yvapqGyB1RuxuHhTY5OztmsLgXYtDJHdyylLPGCD5Ft15PQVLVFqT1TTAAGiQenJGvqRskqY+3UGUF53nU2yx4kF+PUfVfS9Wn4U/uQAnRiDAmlCmNHWxTSMSHNTEV3hwQmIl4cxXw7efpu4Gq1ou9GyBrnatqmxrjIfNbSolHZELOhH0YGP1KNam8g9iUPnwPONmjlqG1DTML09mOPcgaVhYGVVII+sbq6JOVE3TTMFge07Zw0BrZ5wxg8YQxo5wpQKcduzQ+R2jicEZOvzXaNcUb8MbLHh5HRjyIJYjS2Et+PMXopYFWGpmnQtma73eD9yBhGlJPVbkg9lgZhESfGoQNEdz6FyMT01EjbcwgiKDYWUHfqeAwl7lUI4abKUjBQytK2s5Lc9ugdoCXyAwKmgNZ+JwmQs3SMubqicQ4/dsQx7zxNnDNk4yRmjZHgPXH0VNYxW86IQfwScpLuBZUzOitypLDlAzHka+C9mGAZY8gxslgumc0XIvGWFU4ZtKsxjWUIiRwyta1oq4YQA+2ypnYOQqLfbiletoQUJLZAEnopnIrha7fuRGYwRFoF0SnCGCBKh6fFEWJgzJkhBlJJ8Ptu3BmDaW2IKeNTLMBMxDrN6Lf0216+t1JYhDvhnKWqK8iK5EXKZ76coZUqBZ0yLYvHhnOWnCNKWyKBvkhhKQzei6RMZS0mi160ARpbkcfAuOkwTvaFFGQPa6oapw19N8o81wpdSdHeaktTCeklqkjjDGMQjy5VJEdRARVLwQGRGuxHkeUbhoGYIqYAR1/2+L//P/8zxjHy/vvv8P/9w/8Xt2/d4/adQ87OTnn+4gqnB24dvmDs/pKPP3rMv/8P/yG/8Xtv87/7P/xvMI3G2MzyUHNyMmM2d1S1QRvIVebk7hK/PaVtKnLKdJuRVTeSmbNobrDZbvDbS44ONRfnlzhn2Qw9s4VGpYaDuzfoNgNXl2vWFxf85EefEFXk4MixqDT1XNNUNcd3Zjx+dU5lVNn3O+aV5nhxg8PDGZt+xfmrFU+eBPqw4a2v1Hzvez+mUoc4p/mDf+1v8Zu/uuSDhz+g7075/rdPaWeJB28tSSYxP56jXEcYrzg8epOHH7/ixz94wVffecB/+B/9D2mqE5TZYnJD7hUf/OAvadpjum7N4ycvuH/nLXLwRB9JwJ/8yz/nxvFN3nr7HdbjKOahVrENVzx68ogUE8NVx6987Ze5vbiPSjXkwo7Uibpu+P53v8uv/sqvSBG1xFnXWdASQw3MZvNdXDnFuDFGfCdz7HqBeGJhS4wA2llCHHny/BkX5+cslksxI/ZezDSt2YGyMUoeVdcV2lq22462bVEgZqJTDhITtatlXykdLRNbV/xZAjH6HRFhAupzzlJIiF5yGmTtNHYPnscxlLX19YL6pE+tFFirdx3FAizudZeB3TW8ziLWJnN5ecFisSCOw+56bbaXVLXdMYUBpqL7ZDZ5cuNE9hetd6xnWX+l8DUB9p81U53Mgn9xEH0qopT4TqXCYCz+QKqQgyhA6iSrQQlF1JRxJKLKqAL+FfgMfR1EV5S46TpbOZOKdvkEsKpiyChdf5NkSy5nkCkQOTtGep7kO6QbIiVIGvEADHoXT5ESuUiEtW0rUo+NANrbzUAsmvaTLvpemkTY85L3Sg7jTIXRpuzvhoPlEQcHS6raYa3h8HDObN4wDD1aCzmjaRqGYShFfrmHl5eX8o1Kd5SQ8/TOcDWlhMWgjXTI5skvRBuiEQkQUiJPktNTMTsUWT9jidoLSUAGTOH6lHGmhZw0sbWjllg/qkwIEw5S7kma8m61u/E7supn8tepE+F6B4B85m7k7B6fCDM/E0Qvn5kNGCZCz7V4u5zLDiGc5oSSOGciPsp9RPZnfR03mjpc8g5zmOZE2pl8Thr002dnjJqwZJEz1IWEi9K7rqAJ58nkghlNeXbBunZfvEjFaVvG394T4ucdXxhEf+v5X2BCR00Q1mvOkIKA5RRNyEq00KU9oxgwZNApFjmWwhHVSqq3SjRJQ9HKMcoUcxtDVrowRRIqi+v6Tp8rpF3rpS1tpNPGFAvjXGdAm71zc9EomdqStdM4Zxl9hfdhZ5qngJBEZ04Hi7WJA63BZpKe06k5F/Y263xGsKMsFNoTHOR8QDs75vH5itUQaas57SxzdOMW776vWNSP+OjHH3Dv1jG1cjz76CVtPXL3bstsaTEqcPum4qBWVM6i+hV31DMu3Ns8Go8gO2obsItAVR2g3NuMs3eguUHSjjh2MGxEPzxP7BaHomPoz9BqS10nnLJoVcuAjR5rPEFJxLpjGLOvee3rX2XSsctY9wpG1wH0aws96jMJq1a71toYJ5AkSMsICasVogcVsCkQY8YmMajzMbE1FjA4LWMipkz2CboVs3RJk9dk37H057j8CZv8lJivsOZt6tBR+XNyguiv6MKKrbEMSUyYNAEdp+vWkpQhaUu2LbpZQn1AU9ekEEQ+JQcwDpshKIVKI77riEahq2NJKipDsxTmT44BN/a0h4e4sKbCo40jZ4OuWowGa8DogFaHGAayh6q9z3Jxi4WrUf5SWo1DxugIuSLHI4xbftGp/LlHKhXPVEwPCu+cCRCU/wScJV7fmadq4H7x/rzjOhNj+vmzx1Qh3TMUAuRRujzigNWusNFDAfTyDhgU85EkG9w1UHU3hndg+r4aO23Y09rxmpHONOqV2hvlKtDaYYwwpZRL1GZBv5UgYhg8McOb77wDCe49eMC777/Ht7/95/z1j3+ArSsO3IEw8PoOP45Yo5nP51hrGcZEt9mINmhVo42FIi1A8lLBN3uwuK5rRj9ycX7Jy+evOH15zsFyRtuaPYM2C+P88PAQay1d1xFCYLVagUpUlXQ+5Bx3LWMTG89Vhio4YhR9uC5PmtX7+zgBh8ZIsBpj5MaNW9y7d49/8k/+kJcvT/kH/+AfkHLk7NU5TdsyXy6Zz1oODg/p+45h6Dh99YLZbIbKmb7v8H4sJqqBUDS8nbN4P3B1eSkBfhagvaqK2Wy521Jh/zxQ/PMq2q8D3X8T4P3Z1/ys3+2eo/Uu2diHLj/7ELZkCaSvrbUxTXp6X/4Q7cFSjIj74rWc3z7Kvh507c9L5pRIEElr7dXVFevNFV23IabIwWJOKDrK3WbFdr3hYLmQ4reCpqnZbK94/PhjHj16SDd6CR61IWWRFUlMAZHClcLsNC8nvcq+6OpOiYyagFZrRO6hgK+UNaRgPdMyJcGRmloj806HkOl5SHCVSlCnPueuTcnu9Z/TDujvWa1XhDBgjSIVk7sYIoI1KWL0IimXU4lxorDOROEB62pu3LhDXVdU2aHzVNDaf+ZnWfDT739We/FPjVcFKCMNlFoSo+l6qcmN+9pnSvgmZst1WV+HraLbevouoI4sbXPAfDawmB8xmy+4uDhnav7JOeOsk7GkRX5n8B5lPCF2VC6BErbdFBhPLbLTz5IFsjMZm77XTieT6/NSkZE48nqSwWtznV2hVWlN0qJdnyOgReM+Z12SKvEEELPiQGYAJUwsckLnTNj2bHrPar0h+kxVtxwCNQrjE1frjpwHht4zFkBBqXAtWfxyRwwjKUSRbtE1Poxi1jsOjNagbSUJxCi+PF2/JXjPOAz4kKlvONGDdqJDPRQd/1zuW0Y0tZuqRmdFCmKGmoDtdoutDNZpsk/4OMgaUpLejOxflXMY50ApQoJaS1LTD7IPGq3QVoOOZBUIKTP2A+SEj6LbXVf1zjsh4EkKUvDge1IMKFdhnCHESNf3GOeoJxAqi+4lxrJcLNHacHUlAHhVSUG7baU7SsBb0elXml07e8yR2lbopFF1xWLWoki4SqQxlBKAfOh6fPBEH0ghlCZl0STWxmGMpWkarJXY1xfTTufkdyIdJ0QOay0mC5DdWMd6tUElj0Mzq6QrcYjCUGdxwHqzxlqDNYahG0ljwipZ05xz+Bjphx7XOOpWutfEcDqyXXdoFagB3w8MfsAoC1XGzSrqtmUcBmHmq4wfA2H0KBSuroT9lTXDmEgYbF1TWc127Mkp4YwT4o4WU1VbaayTz3eVRRuLcRKjdN2WMXqJtZ2D4LHSWMSynWNnitXacLldM0TPGEUmqHENtbNUaFQopJwhMG4GlDPMZjO6dQ8psbixwCTDsBkIocc5gzmY4+qaAHQxEHxiyKGANhqTNJmI04ZaW1zl8FqkLEKC1Ua8OoZxlKKj0tJt8yWP2/cN2zV4v2F54Di7/BjsnMXSYqtjNuueb/3Vf0caFTka/vf/6SOWRw3adojsikHbSDWLNAuNrRJV7USrd4D3fumY7B2np1d8/Mk52WYUFcfzyHZYc+/eIcuDin4YaBqoagvWc+vmMS+eDDz88IKjY+l+bA4M9TIxXw7cu3Wb1rRs1htmh5r7iyOiUVydDWLiqzRnL1c8eXzKN3/1kGfPOjZrhW01xydH6Lyg30DfP+d7P3zO0eJ9vvLefeazN3ny+BNI5zx9dMmryxV2prnzluXoJHN20XF1qVkuD3nv3W/S94kXT19Q2QM++eS7GBV5+OGPiH3k1//WV/nKmzPS0PPGvQfMZkuMrdGu5uzykh8+/ICTm8eE7YrhvCc1GVVlcu85OFrgasvoB1LMVHXFq9Pn3Lh1h/6qp+sHfIxYJWPg408+4s6du2Tg08cf4irH8cEdxiBys5JzTMCNFOh8MQAVCSNhV0vXncz7pDKPPn1M0Il1f8Wqu6I9uEldzYg54IP4BVS1Q2Xo+pGmblBEHj95yDvvvicyklE6EZu6Rmkr+YerCqAUODs75ej4WCS8Cq4zSYJJyCoAeLdZU9UN2hpCDKxWV5yc3CB60VzvQ7i2lwtIvzcHF+86raQovWN5X8vVpi4dVaqCiYC2itPTF9SVA8QYNCYhXCidydmjlNuZlU4yIrIlRGGu5n0nrWJSRijc1RQlplRFTmdKHjPiE/MLyrlofT3GSqgkWt8gHkLTOUmQEvfxqJok6Qo8aUDlLEC6vNvOJm3qLJ+IPahrJq+q5BjEEmer6UXlYwT4nB4UYlZC78wh5U8GiQ10AdKn/bDodk95RUrpGtvcY4zE713fsVqvGYscXEqJurK7IsqU02utqZxjVrdUrmE2m7FcHnDr1h2ODg9p2hpjhWEtylwB4zR184Cz01M+efSI+XzOYrHEGFvyiZHLy0uqpkGNgWHwEo9oXfZ/YVIbK3myLgoIRmuUigVzKNcrQ06Z1dUVR0eLUrgosoOFiGhLd94ORC9FomSkuyNECMHK2CgSLEolciqElOKtk5MUT3ZI3bXc83qH/XXyl/w9Fb/y7rW7icb+oX26IVIq+zBVFSzkWncNr2M/uZBlEpLT7vAYQbBJaT+nBSSezvl6dwo7+RZRnipju7D5NRKjT/IvQplPZY4W4l+SjpGU407yNwu1nkmXXr7PHtiXx/bX9ecdXxhEv5FOZUDs9CiBbIuTaYHbsrQgTHCGRpGVKno0lAElC5LSIhOhctonOVMlAtENBEsMnhQndvu0gBtUcUpPKpKz3plKQCJMi6IxWCPgywQeSIVLbrzRmrpy1M7tFquYEsoHUk4oranl5ETDMBtm1YyFOyK6N8lGzNdWdsuLcEVXN5h6TtUu+fjpc772Rittyk1D286p6or7d28ybDe8PL/Ed57T7RpDz/Gh5ehgTh4iKa0ZO485uEdlFSY7bDIsdY1VmeRm0Nwhz97A1jeIrmUYevruinD5AhVW5GxwusKogEtrWrthXiWaucXO5iR7QFJLgjsg2xkoaanZrdcyE/YD4FrRptRtXqtwMj1/eu1ukZcn6VL4USmjDfgs2pQm9qjkUcagqpqspFVJGzDeY4IwT6xyoDUjGqMsNk/6UIocRppxxSK+pOUVOV5RhytSumAkkFVDZWpyiGyj6EFf+TWXcYuv5sJAUaCTgPfKuKIXXwsY4yqUrVBGYazGGqls5uCLfmrE5QRJKvfZNkTXousWE4teJYo49tjCqnKqQfmeHCVhVNoRcxC9zZip3SE5a3AnVO1d6vYIZ6EenhGH5/TjGUaDssdUi5ac5l90Kn/uMc3jfK06t2fw7YH0nX6Smu7tBDj/fAByWtxfW+x/+llMrUtSHffEPBBjL6ZgyRBLEB4ThDAyhoFUdFvVZ6qkE3B+/fP35/M6Y3mqgE9mctd3E6PNTrGLqMge+q1nM664uhjpNprDQ8um61mGSAqRs/Nzzi/PcbXj9p3b9MOa9aqTAlKIBdDLuwqo1gFQaGMFpBg9Sg3UjTwm6kKyWVjrCoPccXl1wQ++/yPu3bvDg/u3aRsxDCyTmJQCm82GbrsVzerLC/q+o64dOWlSnFq7hEUQpzZk62jqhhwjXeyE7ZcSuYCp169bXVmqWU0MidXqkhs3Tvi93/kdrLX8sz/8p/zW7/0uz56/4O6DB6XVvfgvFDb+weEhT58+pht6Rl8M7lIsjI/pc+S7rFYrac1Tirqui7ahLgWG/QK2v7dS2NtpnV0fA2VcTGvWzxvHn9dRsf/dtTjk2vPUNCB3n5t3f6a3yUzBaGHlJDH1zARifn0p/lc9tL7mJ7D71jL+tdEim1aCau89KEMM0p5JqdrHEFFWiuIpZrptx9nZOZerS1brFU1dQ4rM6orFXAz0VpuO+XzO0WxJ01Tk3BD8wNV6w2rdQSlSUvTrlLEl2DRUyuwKD5NT+uiLbEaJI1TRBKW0872u7SZtwaoIfiulSSqjkyRr0gI7xS4K4VlP16a0Ff6MIGoqvk33eZKKCDFwtbrk2bNPWa8vZY8sSVgIsRD6NTF50JHCGZLXK1XmHszmC5bzBdYomtqRlTCntClMaHVNkqQMvL0E0LWI/trxuSz2nfKRqG+qItk1JfTT66bujpQUkYQ1TvQfUyaFIIaBw8hyecLR8Q1WqzNevFjQPX9RzBmF5FBFuXZ1Y4uRo0IZ0R5GGSlEJDGFki+mycqQJz0JpWTP0WZ/8kqx0yq/lljuI9DX/THkaXuGyjSPI0EKyeWuqKwljVUGMUEs7bY5IAmVkm6gMBaT1YHNZhQJPY/o02ZNSJrURZzTuMpgtGU2n3N43BTShsbqX4yJnoJn6DqsasBYkWcJkeBHul7hamkbzjGQw0QVSyLRGFbUuuJgvmAchEWosSXhSMRUWqm1xihNjjK3ckxEIiF5srYkFCF5QpSWetLA2CMt5CkzWEvTzoSp5RPBezbrNdtuI3NUGyZNzOQhD6XwYgx9FmZ94xoySjpXciJmKVwl7zE5U1mFRfamcdjiYo1uaoiRMIyElLGNYTZbMssaP0ZhSuKBRIgjwyCMYtFytSidUCrhyOgwooJBhUAcPd5VBYwQ40xjDM5Y+s1WgPRxpC9dOkZpQjGvqmtb5pAi20RTm10xP8ZcOm8GjNY4pQj9wEAhJY2B4IX15qOseyonXK05mC2JxRjcjyN+GEWtO5fuUq2ZzxdkY1AObM2uvd5WDlMHjK1ICvwwMg49ztWMKDZIB57PcVf8y1EYZlNBs6pqslL0PqF0AQFKZ7JzBkNFjApZ8jWmMtjaoKICI/JenR/YDr10H1cVfYz4XLoKcpEeCp6DwyPaqsJHz8v1Jd04QFbMbUOtDC6LFnkeAiYkal0xJshRk4Oicg21bcsqYQm+IwZPVUkxNqhM6LaEIPrKlZI5ZJTEYjYLy1q7inUIDMNIGkb6fsBHTywGiVnlIgvw5Y4Hb9zgyaNzum6EZHnw4Kucn10SogbnqRaZtNmQFMQx8+TFIw4Gxd0Hc+qlw1ZFfmCRMXNHQnO6Hslak1LHNm3p1oqzF4HlbMmDt2boWjqVnz3b8OmrSw77I5aLlqOTQ7K6JDgv+dRzz6/+8ns8uHuEa3tW/hlR9cyaA3Z7qvas8paDWy3KZI4PF1y8ymwuRqwaqA38+PsXnF+JPObdhSNjUfaI7//oJ9y+A4bI7ECxHZ9xftnTNJrbN9/h5q0VHz36EFNrtpsVa2f4eP0pq1eO3/jmr/Ot/+4h3/v2P8ekA77xlTcI7hE3vtbz/r/hqaqICT+hygvm/YLcXbHe9lSzJRfjlr4KNLcbTnnCvHEoA0klxqse1opbd95i86qhzxvu33GMacuQtzx+8Yh2foKZLXny8ow37h7R94lHrz4mtgntKv7i0T/l3Xff46C5ydnlisVsTuXsTip3dXXB8mgpWh/ZElKk69csFjPW28RifoOhv5K4q9J8evaIITzn5aUwN28c3Mc6Q9aJRGQ7dmLenQOz2jAOV4S4wuctbuG4OlsTt+fcv3uHpmlZX0Zc1QIBbTNj7Egssa4lZ0diQOteYpZcdKDzyMvT5zx44w1cXfG9H36P9955jxSi5OohiT9CO5f1tXTbRT+SUyCnAes0GPFmePXqFTdu3sI4iyHjfWDbebbdwPHJDFN5fNjy7PlzLBZn58RkAOmovbw45+TkJhPOvdmuqZxItWmjySoSs0drS/QRp4zEMz7w4vSUwxs3sCpRG5G8zAhIXVlTZGUSwVb8xXe+yy/97S+/d5uiqS9bsjCdVdF6llhsknQR6VtRid0Tu1AimZKRHN4U9q8qXZT7+A0Bz6/lvHqK/4u3YOHc72LMtCORlMdQcj5SXkDpjL6WmwgJDyIKnUBbhS2603v/MekcrJ3Be3nxOPScn56z2Wx2+6Dk7tJFNsV1KSXqumY2a5nXMw4Pjrh16zYHBwdUrhG2cYr0m4Fh6MTgOnqWh0v86RnzxYL/+D/+X3H37n3+5I/+mI8+/hgyXF5eyb4ZE4+fveTFq1Px0bgWNwoGoFFG9LUnQF8kb0rZosRLKRq2m46cMvPlXCRi0sCkKS5EtJKjM1FPpbMupYxPGe9jkYGjxAQZ2W73xq+paM7v8ldKfFvu+3Rjpn9PRsc5I1gK071U5Z5PN3I3tOSfBcDOseAyRZpm+tRJOeJ6Xrs7dmNjktSTdXSXvygpFu0IGzntgHR53V5+9Fplh32XRvnsNAm2yPukgi3ElBkGL3F9yQcnv4EJQJ8wrj0xJ8NUwNrLEPzM4wuD6G1Ti/mXEIZ2+ruTVpYYQSmMskXnp3x4aWUzRu/arFGTBrFBT89XoteUUiKS0NpiygQMWViYO2kXW0CV0cvnxrBjJxkjAXBKgZgiKmqc1QJ2FhMoVaqM5CymL7vKnMIhmq/OicnEdE65bGYBhaYmScmJhVUcLRM3TMc6ZGhalgdLrK1B1VQu4FyDdQ03b9/B+oHHL0+ptWG5nHPme0KMzBcLrJvh3ALVHNJ+7X/A/J1fhnHLG9lxT7UkHOv+HZ6fv+CDs8gpNUc64xi4GC/wl485ff4Rl6sVtw9O+OrdN1g2CmYtKt6gqizBZEZrGfWCrT6iswekao4qLa15SsTLYJ3mwecOpZ/GZa6B6Hn3FooCogOF6ExOSirJwxXKdySj0foY08zJtiZHz6hGqC1Ga9rKkbXBVy2YksAn0Dlhs2euE7qXdnByBeoIpSH4vuh2dUR9yUjgYvSswoYhj1T2FpXRkhznnkgmKU3SDq1TMUORDdeaGbbIApEdQwiyaKZMZS1Z1aAOMU2LqWeYypFDJmaIXlg1dhwwaaT0NpMwZCt6vt53RB+BlqwMVs9Z1HA4P+BgtsDh6VVFDJ4x9lTuTTA3sXpOzL+YsWjIXqqN5SZNlT7BYZWoBCvZOHNKYiJVDJWugxGfBWs+D3TM1xb317oUECNRlaXtPedI1iMpd4S0RpEgV6QsrKWQ5bzHOBC8x9kaO0k1cP3zdttB+RwBXoS5aYqBZhbpmKQxpkFrV1rQgJzF8CWBzaBjJHae/nxguxlYbxLdAK7xfPr0JX6IVE7zwY9/wPNnTxj6NeenpxgSy9mcoe/JNpbCaRYtciNAtjYOUwxTlOx26JzErwABQOTeJMgjm3WPIrLarnn06BHnF++zPLhbtPYkyc0xk/zI6uqC9dUl/XZFJhF8wOiIMS11AQTk+kDwI5qMoYCatmKInZjGFebyZIo03UtjNYvlkqZpODt9wQ/+vOP46IRsDN/+q+/wm7/7ezx48x36MbPqzmnnmnZRsfpwjbaO5dExF6sropJ29lgq9NJGKnr0OWe2XccwDEXahWvMkrIBTiCaRrRusyTzuQBzkjiXsaFKSfULMEo+j7X+usxVKli5oM+qgHATi1UC4muBxgQUokqlPaLwoCDnMj6moOJnALpf5JgMVZWClCPO7mVwtLYYmzGmQmGoTIVKos1PlIKR1aUzK0uBa1KaTmTRN02Jg+WStq5RIdNisc2CMUb6GElaM1/MybFnvfYiLxERiQMtmJ7cAil6a63pS4FEKylshyiO7EkJAKOslbYdbdA4DBUGJ+yESU9wKqCgRIYKiuZtAawUoh8JKGX2iUne1Q5KAL//xXS3pTAhwaxWmawhpJHtsOb07DlXV+ei8xi8jLgsbYvyljIeU94D8SDBcUwZlQJtpVnOLMtFxWbUoKxwI7QqDBhdvllJUJDHc0olByrdQ1N8LAOBqWagp/NIElyS92yl68B3noS9AaUsxiTpCtRgdM3Z2ZrLdaS90DRzQ91W3H7jDTbDwNPTFc+fPRN5OZUJWbGYz1C6kpVFGzIVumqxzsm1UpqoxWQ1o0lFIiJnZG20BpxhDMLknswbKUUAym0/Pj6hrRvOzs6k/dc5SdC0ZvSeqqqpm5rZbI4xms22IwOuEqNPraV9vqpE3shaV1qhDc4Ia886iWVRsvZJx4neablrrYkhi9ayFbb3pO3pihYzRY/xFzl0ioS+Y6AiGhlXMQsL0fcdPiZqV5GLtwhIB6fSGp8Sq/UVKidiCFRVJXtD9GLSliPkLGaoIeB7T/IRVaQ4LIaErM2xtPbv2v793my3H6QQ0TQzrFUE77m6uqQfOuZzMdxM3qNUJuZIipnFYlHWBIeqRMs8K412CpMi0aeS5MdSX4mk6Nlp6GaZ3zkMDJs1PiTaZLBWir3WOVCBsZgAD+PAYGNhYEqyJelKxBnJd/zQ0fcj3XZLjIl2NpNcqHgaGCNkHlX2rG4YSCiaukZkYQLWVEwt80opjLagpQhntXTmkkUfXKfIuB3IQ6JyDjWKhEgisRnX+AjZaZJKuLZCZxj7XghDIYo+vEqEoEk5oqyhqizKgtKJIXjG4FHWiGG0qwSAHwM5ZDDCOO/7K9ZmS8qZGEacVdJOryWvU1rtwJzgM1WtqasW8oDSHq2lY6Oe1Wgna1PV1FjnUDGjbKIfBrptz+hHFosFyUZCKoQmBdrIvOmHntV6zXwx5/DggKuhI+aMTlqY6LYijB7vExSJl6PDIzbBQxLjunnbyrWJURi4VhdJwETIotkbsxfTVqWIRU6qqRt8P+KDeAeZypFS4Gq1JkbxRQgxSmGlqsTDy3752PzkZkPXzTh9OfCr3/xlfve3/03Ilm9961v8f/7pf0PnV2gFo89YbXFVJqvA/KChOZR4vR8zV6vAar1iNpvhPQQyr043dNuezTkM28zRkWZ5sMC2ipcXK2LyLBY1Tz+54DmOzWXgK++fkGPioycXfPNr3+Q7//K7PHv2E77+K3eIBo5u3MRaTz9uCXFgcWJwzVKKC96TvGd+6DC6IYXMi+cdh4cz2kpRNYrFAoYh8vGnD/He8+TxhpNDy+PHp7z57k2crdlcbPnOX32IszPuPfgaq+6KcZtYn63IrWZ5OOcf/5M/5eWjQJUdt2+1XDx/yvKu5s7xbYbwhJAy67OB52eJX3twG6ct2WpSFTi8sWRZK1Sj6bYbwtYz4OnigApw5+Qu637FBz/5gPfefcAxh7x80dPM3uPscsPLZytiHpnTsR2XWFfjVeDF5XOOTm5y2W1Yj0E8kFTHZkyEXPPJxx/x9ttvcb45R7WG3oPWLTFFhtDhZg0/fvghb73p2Fx8iraKk9s3+N4nL0H3PDt9RFPfoho3mKDp+zUnNw64OHvFwcEBp1crfIa6Clx0V5yvL7h56y79IP4OxijOLl/Szm6Ay3z65FNu3jrAzh0vLl5x8+SB+CXEgAqy52plWG0ucLWnSxf0HPLBTz7kst+wiZ4wBIie7WZN5yMv/vqMO7fucu/uffw4UFWGx5885K033+TV2TlHR0fkDOu+46Yz/OThR8wXLYmRH/3wIe++8z4fPnlCVY2cX7yEpGmbOcrC+nLD3Tv3GYbAdhy50zZSBA/SCRVTJBAxTuHzSMoevBIJtJAw1pLGyBgiP/rgx9y//waNq7lx4ybBjwzeo4tiwdVmzen2Edqsf6G9e5LO0wrBHZLkeTufMcVOFlmkbuXfe+IP6GxIOWEoRIwQyUrATgG7VWHsFvmMCTQsfyZ9c/Elm+QjMzqVwjuTDPNErki7PCYXwH0CbKVokfdysAWszSqRjcgpGiuyaMvZgtn8gKv1lnF8yvqqEzKNknMV1neFq8xObhQUbd1ycLBkuZjhrGEceoZBZHy897TzGVkZFgdL7t+/z9/+7d/i9p07aK35yle+wmaz4fT8iovzFTFF/Oh58623uTi7oO9H4jhytVqJQgJSmNLKkCPUrkETJXdWxdhVzP1ENhrwUaFVw7w9otYGrUey2hvRWmOY5IW00rucgix5l4kZg8FrwVujybtuMTEdLhrzOqOywlqNgSJhVuJ+raXLNwYMYoAsZB0pqGQ94SJyn2JOpDjluFNnpyRCefI0BCE+azEJ1pjydyH17HLfqQUily7gvHu7CQ9UBT/NO/DeTNwnITDtmO+Cy+QcUUh3gdKTzFNEawSnKl21kxB1jEWSJSd8GCdKUCHyKsgixzgVAHZKBKXgo8hFxvfnx+ZfeHcXoEKXNvCS/JU2dbO7TnlX4ZoAhjjpJmm1TzSmC57kBmpjSqVNE/1YbqsE49ZRkrx9wjmBPSlnMTvdseGLg7TOpGSIQRYFQ4aykEyvE7PCqaKkdgZIEjzLgjo5+kImhMgQIkPwZB9QMRegXxYJN3PMAds0zA8OOTk8xhhIYcBZR/Ce7CPzquHGwQHb9Zr1ZuSTZxvao5b58phZ3WLcjIMHX2N+coDVI8pkZrEjOQ3tMYuDY27ffJObr8643Cbu1J6m+5hTf8F37CUjiaAb7rQ17x9r7h4uqIOBtGSImVMPKx/pchbJGFOTtJhIJZXJep+ITov0/rj+wwQG7Z9//dfXfzblnxNIMUk6TR+Qhw6TOgwenU8wumLUFm9r0JqZszRWAKjRGgZEH9NoRaMiBzrhEEme9RactlTOElmg1JLECp9G6ipRqYxpDbaDXtdUbYOximHsUKoj2yNGXXM5DqS8wSqHygFtHDp5nKqwRkGIjAibSwyKHKgaXTVoV0nLktEYDZvNQIpjceIum0sBhLSrhIWj5SqFrNHJy8KhwDZzXN3QVALe1lbRugBaY+cH+HzAGDI+/WKbeU6T5NEEDhb9rFxU0HSShbuALmmqTn8Ge3wNFP8cRuRPP77jse8HyWfAylzmq1DUJClUKeFzJkS/0/vTyspGPbU9sN/YX8P21bVPvVb1nrRPrQ5kLQxokXpROzdoAfwE6LfGsd1cYfQc5zRVXfPuO+9SWcPZ6QvOzs8Yh4GLs3OiH6isxWdpJ6zrmtd1lDNd3+0M6oR9C34ccdZIyxiT/IZc9BAkcHj16iWXl3/KydGSjz9+yO3bJ8xm1e47aiPr9/Pnz4pGnKxZ1oru6iRFYSbWRQEHJ2Z+CIEQgjw3C4sw5YSzDm01k+HParWi73uqqqKpanqz4vT0FdkYqnbOX37rz9AK/rW/86/T1o7zy3Oauubw8IDHjx+TkkjUtG1Ld3W1a8Gcgo8JsJ7A80knu64PdvtKvrbp7YdZLu+DBBHXquPTfvI3yRDt3uXz2LyvFY9e//1rBpY7XwH4qWo9AubvaulpqsbvQfRfxHxQadk+xYg3SaEw7S4CZOkwc64qZoO2BCoRfN4FLFlNbYCi/amRP1YpnNFU1lI7R2UtWgmL1GgBXKuqItY1m00neuvaiVEXmhCyGEYFKVprLe3RKAFoEpMTe94B2xiDshZlHVlZorJFW3oq7BWjzusg+HSvBEEuxsFGdNh3bOD9xjVJBL1e9GP3++stkVLwUuSY8WPEdwNRSxISSqIhgZ4UI3NphU1TgAlUdUsKXoL1qmHWzLDaSs14pwOqiaV4q40pHbVTwAkUhn26xk6Zzv16m+QkwafKeeQsraMZ6dALoeihasMU/Vote4FW0/8si+WM7WbN85cvqFrHjZMTnKu4dfsut27f48WLU87OL3GVofVJ7k9WOAc6StnI1p6QZQyMIaAQtq5zRvY5a3Guom4bDg6XYt4XI9aKLMY0zyaN/IODA9566y2cMZy+Ot2B6EqpXUxnjHnt9T7I+DZGZAa1kbgwFbNbxaRnL/eYFGX8ZGGu52IQHVOSfBRF8IF+GBGfDtlLQllrUwzy2uBFuuMXOGoNfd/RRWhmQgKIAKYh+wHve7Lxu/UeLMogMohhJIxrNibIPmQV0XuGYU0Y+sK+3bPaQml71xpUJd8zF5Mvox2RjLGWkFYMQ0dOwq4utQPqyjGOiRA7+rEjxhEQeQKtFTpHfJRm9kn705pKDKGSIQGurtFZ5D3QmjFGKquonGIIPTkrrGhBEb0YgW/GDpUstvdccUlIsUg6Sst29JmuizRtBj0S4kjdCmifUiBkQ04Kpxw4S1Qan0B72auNyaRsaJsK5WqaRdES1prBZ1nDcpAioZF1LoxJtFNVRKlITh5FlrpFVNh6BhjS6Al5AKMLG84xpsSQIutuix41ZjQc5ANS8vT9Ru5tziLz07agRMYldJFm1jLTDSZKNx8RrHXUlUFlRd910tavHKPPaJ12UlXi7zBiWycSXHnSQxjxY4erBCDQucLqhhgsRlVoPRKjJ2iwtqLSlnY2g5QZw8CQIr4sX5V1VNYRyBgrOsi2qlBWOmJ6PxL6Fb1KGOM4nB8wc5GUDHXdEkhsuiui9yxnLe2s5W5zQgJOz88YBpG7GcMakxV1pTB2VkzwAlFlYh6nRkti9ORRmIiNqQgq0YUtW5VIJpBcILuRmAZhI8aEUQ0GMTudjBi/zPHpsxdcrHoShr/67rf43vd/zMnJbb761a+Rc8JqyzCMVK6BZEk6YJ2i9z2tm7PdDox9zaw94MWnK56FLc55sk20S0dlaua1ZehHyAOb/pJl7cjJU1Wa7WqkahRxNDx7uuLFiwuU1lTW8ZhP+F/8L//H/NGf/hlD3PLw4Qb3yPP1by6ZLZY0tsZWk8lqwlUzTBsxGNq2Ytxe8c1v3ucv/+Ixv/yr79D7DUpt+OsfPeb5c+hWiqrKHM3nfPevPmXbrcgpcPfmbX77N3+H7//gx/zRv/gub7/7NnfuvMWzlz8ixsA6rFn3AW0aZrbl6KDioHGc3NXEdY9p51Su4umjC/rTkXBU0ZuRtAj0uePerbe4WG14+INHLKo5TUw8X73iol9jlebrf+d9Pv7gCSx6Xg4fs/7wKW1znwfLG3z/u9+hnmeG+IqtWdL3X+ODDz6A+pTu2Yp33vs62VqSrnj07ENqXRG2mbEPbOKaRy8+4vjGkscvH/PqwhOj4uatG/z5t/+Yu/dust4OVAdLzs+e8JV3HnA2vOSyP8PYyPNnD7m4Cnz9vcDx4S0+ffGEj572hDhyeHjEd779PX7v934Pxg1PL55ztL2Ff5E4O3/FnRsnjL7nyYtPaJbn3LrxBt//ybd5K9zj4vKUmzfukq9OqYylbR3bbcfZ2Yo7t2+TTODbP/pT3nznDo/OPuQ7H3wI6oDw8Qfcu33Ai2cfk3zHeh14792v8/LqKfXSYLWm6yJXwyln2wWqsTw5e8bNWzdYxRV/8t0/5uWrF6z7FVH1/Oov/20+eP4D3n5wl7/+4Ec8uH+Pp49foLdXHN895vz8iplf8/L0jMWtYx6+eMq8mnHn4IRhHFHWsNquyS7y6YtHZBV5887Xca6lG7Z03Ya2rXBzRXe+4pOnHzKMiffyV7l58yZX3YYYI8vlkk3a8PHj76B0/PmT+G84TCEkZTWxzos83zXN6P0f2Oe009/7TkgJA4XtNRVPIRdChDB+JyYyyLqu1USH2b/3JCs5ETJ2RwHmRfov/VRMuX/a5IdX0MEUUaZIZlA697Ws6y9fnnJxtWHoA6OXuEgIPxKPW2fE98iI5nXbtiwWCxaLOVUpYMbkSVly3NliRj+MvPvee/y9v/dv8/a772BttfPP8n7k5ctzvv/9HzOMA5sClh8sRQ73cDEj+UOIgfVW2OTGWGZNRV3X1FajUsSpQGUtIxuG0OOj4JZJG1o0s/aAzWqgOprT2ophHHc+J2an5qGkK2IKjpCctmDyGC3mo2JuLlK1MSaiTsSkSryuqKwRIN1IjC5H8Q1MmmBNkYIpBsK7cbMnGMYoJrMTwZAsXbKfPZRCtN+VwRTCUqEAsa+nFOxFSY4xSWxqPRVtpLifd2zwaaxMwLAiK10wfOkNlUK6XCfDJL9i0NoJSS2rQrSevBQiIUT8JGG6+wLs8SVCyfsUk8a/AOjCcRUfrZ9/fGEQPaYJwFbkPOkcRZgS11JZmFhcO22ZgmDlnIoAP8JgzRPLqbQksNc8SlGCNqwCrdHWimFPThC86FErJc7sKpTzS2RCaUtXGKMlUI3y+2ngaiNMuphl8IomaKm+KI3SSdLwXfVPqNPGWLTN1LkWY9MQCCXpzxkMhtoKa6bSCp0SwQ/03QWJQFU5zi4vMIO0ab46XfH02YbZvOHG4bEgXcZgmxkHd+8yrJ+zOX9OTpqYLPOb98iDwR3eQyvN/ZMj7sw69PopNjzlqNnwxv3IV49u8d1Vw2Gd+c235ix1h1p3jOtXbLsrWu6xXV+Slses9YqzqHimDOdZ0+VMMiKbsqMbquscyNJmtHvgGoC+e0T4ya8DlTI8EsVYLFNaPCuiq0XXdrhitt3ijCdWh6QxCrND16AyOmypNURfoTUkHTAZ5kbTpgEbOi67My5Wj2icQtmq6P8ZjFqilaExNQ7D3CmavGRwxQAjbzBphXaW5Cpp+fRrkj/FuhlGHxBHUJVD9UhinzMuDYx+S9SGrDOubiRAAgGbk5hgmRwwOUDOaFtJFVABwZNKO7RSFmwli+C4xuSRymRQYqQ65g4dP6UbH5PyK9o6Ubue1RhY9x1ZV190Kn/uIU7FU4dGKfcpaTlKJHQpsqii25b11Dzz+cffpBv9sw+1A71kkTO74pxUCRM5CchCTgV48cLa3b3D6yz0n/k57FnwovWsCVGc32OKmKLLJ9dGgLGURMoiJ433kXGI5Kz45JNHuHrBye27jH3HkBOb9ZrKWfRizrhtGJSw+0IQt3rRETe7cxAtONHSnK6fbDzsmMTTZjT3BVQcAAEAAElEQVRtxJOxTrcdWPXn/OEf/mPu3LnBW2++wZtv3aNyZTMsLudXl1cFPNJiLlXapEYfRGcNJdq11qJNZhgGYQsXINsYReUqIDGOoxiOprFM8H1L1GRcGqylcg6tHWMYSSnwF3+WOH/xgn/vP/iHaKXYbra0dctyvuDFs6dsNxu00qIZawzDMMj1LyZHkzb25eXlLhiIcTLhme7sT99v8WAQKQ+z08zfj9MJSP9X4YR+Vp7os5rV199b9k05u89i8bumyel/6jOPx58OVP9Vjj1LoAQXWdZWcYOXzxHNR1MkqwCtMEnW6xRKRwNJgEMd0UYYMzoGdPLYnKg0VKWbS6tMYzWVVjTO0lYNTsO2C2y6WNiQtXgwmETCE/IousUpo0zEBAFdY8qEWIyPlCrsR4uxFcbVKNeiXA227F1KwKupS4Zc9vgs+qTZSFyRYyxekRJAyjUH2cWkdXVaf4Di/L4H2WW8FOZFyhjlcKbF6ApFJbIzttoV7MlGwMQcyabo+BXWqVKK0AWUMjT1AbVbYE1LVbWk7ZUUtAIkPXUJQakKXasQiu79rrR9bQ2eWOXXTYZSTMJOLtfFGJEycNbR992uu5AsBTufEtZqMAI8GqN3rakxJrabjnkz0s4aDg4O+I3f+A3eePCA7XZLVVmW8znL+YLFfMZsNqOua1xVcXB0jKur3bk6W1HX7R7w1hZrHbZyUjQt57wzCCvr09QqC7JOWGOp64YYIq5yTEz8SXd2KmCIOZkwaUPpSsp+0qO1JR6VQnMIiRhGyD0hjNIyGvdmgsMwErxIIfXDyHq95vT8lRhgprKOai26wDkz9j3rzYb/4B/8O196foNlGAfQI+gOZS22qSQO1lnMmv2AMaokZUGYSmgpSKRA122x1tCYBq0qtoX11ja1FA1KEa+qLBpN0qJ/7qPsZ03bYJxjGKVt2SpNUoYhenKI0n0yDPSbNVpDSD1Wa+p2JuZTIUsqQS5DWuSbXBY/GdFf70iIxrw1iqquycngckYTpTsmStFR9sZCotEapcVnKKWR9boTsN9ZFos5Rlly3pve5eKZYJQYlykl973vA7PW0DQtTTuBEhXD0GNMLnq8CWMsroRjCsmd/HbAWjB2KkRCyuLlYou8QC4dZ6MfGTqRCKmqSnKREDAB2noGKMYAISaGoaMfepYHSzhYYLWmqSqMUWK0XklhNKTIGLysZDmTYpJCUMo7uYJdga7Io+kSX0wFvKqSLyX9uMJE9zHhgyeriCFj0eTkGYeAVomYwm6NyTmQIvgxMl8cMW9FM3t9tSZpYRpWlXRMiGGsXHvnHNY6UJpsLMl4fE5sxp621ihrcEokcjCabdcTcuDgaMHbDx5wuFgQlZHGKpPYbjtiiFydnVMpKXBlo7C2wnskVslR7lWK+NDhw8i8aaVAbBXGGYY4EroMBow1jEPGWOna06oqkn0BrdyXntlPHm0LW87Qtoahv+RiteWf//EPqdvA9tRz48YxwSvWV2Jm6lPCx8jT5yuILQezE15+uuXs5cjmakNVG27ed4wm0TSOo5szMo6u2+BjT8LhKoW1irfeOuThuGITonBYsmLsM0EFfnD1CV/5ygF/79/9Fb7/w494+EnPj793ytnLFYtDy903Wt585wQfFXXVoK0lpMDpq0tOn5wSNomD2cD9Ny2vTh/SzBeYqJgvWsKjDmMMbQNXl55f/uY3ePbsE87PLrh8HvjJD8/4g3/jV7nYPCLkRzx9nmgayxB6qjqzWMxhU3FyVDGGF9x544RsrtisV7issJXll7/2Li8/WYHKIt9jEjffvsPTi2c8e3rKo4+ecntxg9/+W9/EHjT0Tz9hc7Xmez/+Hnfu3KO6NWO1fc5Gn7NNp3z4vT9C3zBQG7qrC86eJbbLyI235yTTMA+eH378x8zmx2QbeLV6yLKZ8/iTF9w4uYdqLN//6Hv8wR/8Lk+vHvPpqytSMjxdfYx3K152G/ox8Wffv+D4aIF65Xny6ceoKqCdIZrMJy8/YO03fOXNr/Li5TNevXzGb/zmr/MXP/wjVv2Wf/GX/4zlYY1ZWH7w4Q/4lW/8Kq8unjGfW8anK3700ffYJs+Nkwds48CPP7lgtbrgyavn3Ln1NtF7jA48/PgJb731VW7cu4VXA5fDK9YfP+fRp684Pfdoc0BPx7/484+5d7dm1hpWl5474Tav1qcchbmw07cr+mHL2YenzBZLRj9wOjzmJw8/IObAq7OXZJMIKqEfGtKYeX7x1zidOL3SXPVXzBZzurwl14m//NFfcrA45tn5K4Z+5OvvvE9SkbOrV7w4e4VpNM1hw7OLxywOZ7h5ZDOc8uff+zNOz1/yta+9Cyrh1RlPnp1y9/7bfHrxAQ9f/gBnHev1mve++h6X3QWfvPyQpv3F8m5tJuAalM7SlVPqkgI9XQfQJ5Yb7IVQBYyMKRQ8TZjgKaud/vTUIQsUKQt276mV7O0pT0SfCRDdE21SAe+FkyJV188D0CUN3Mecejr3LIbeEzickL0mBOi2W87Oz9hut9RVLXt8IbdKLJd3JK6mqWiahtlsLgSuiaRUYsDJR+zNN+/w9/7tf5v33/+adBIVeVRTiEIvXjyXODc7YoyM48Dp6SiyZkbRzmYslyODDwz9KKSH0i4bQiCHINcoy/4ouRRoY6lcRdO2vDo9ZdZabt86xJlEiJK3SMws+SAIuTgrvb9+ei9Ha7VI6sSksVrA9TBJCYUghA5EKs1ZU+LKPfaRFGSrcdYIlgI71Q/Ji2TcxSidV4LtaYlTkxSs5diTZadzlu9xfWyyw3J3I1NJ95hWakdK2XVCIGM7IQUT6YooGPD+k3b5GBT2+5RTK9mvrXEyfuP+eYpUitlCfvY+7MBzGS/XcvMyRiUfo4D8YHb35OdjWF8YRPdhMlgoTKoiY4ES/v2eoSWVjIwSbD3tk1NpuRMTUbJABNINK21nCkUuE2e6EeIeb0hR9L0kCSiMzMJYyEVMPiMGZcYorKswhUn5mTEg30FDnnSOpwtVigDT73fgSMpFHwoqY1DOoJqalBV+9IRxFBC/TIw4DJw+f446znSb58Q+4oNnvdrw+AcfsFlvWV12jGPijQdzjm4cYusG1y5wTcvZ6VPSWQ1uhmKGMzX6MGAaDaO0h64uL4j9Be7iI8zlR+jomTcVv9Re8bWjI3J1wtC9ZLV6SXdxRr++IGX44Pt/jB7X3Hn3ayyO7nLojrjnDjmr3+Tj6m2epSOCa+DaZP7scQ1H2D9G3i+a+wd3h7TrSsuwIuGUweWINaBMwNDjwobZ2EPaYKJlQ0MwDSpGxv6SxgUaDCl3JDTOSCWPcU23fsnlxU9YX31IbCBbg25n3KhvUlcH1KbGGIeJmRwjtWlkswxbCXTTiG1OULVhO0DIHcRzMAM6dqiQUEONyWMx1bC4kHF+K3rpTDquiew9226NshZjLcqPMHZkNEkp5rM5WkF/cUbqLsm2xrQHiB+1p6Gj0q9oK0lEtrFF+1ek4Vuc9Z/Sh5GDZg7+Jamvib6inh1+0an8ucfeFVkWQV1kFqb7L4IBAqDr0lK0W+3KgnQdoPl5n1VexvW5vluLmVqKrlUmp99nqUyqUpjLJAF0bAEq2dlPvHaUWl7ZPKQSdJ2JrLV4NAggE8Cl3Wsmt+auGxiHSIoahaPvN5y+OsePnnZuOT+/wCrF8dFBAR+CAGa56DAXFrrWmnEcd9chhMB2uxEmVIqy2RbgVWlFCIEuJayRqvx0fauqEtC/SYQ0slpf8d/+t/8tb7/1NsfHxxwfz1BIN8TV1RU+eA4PD+n7LU1bsdmsZf3K0A0D84MDMQ8bR6ng7gAnCpAQpHhaQICJGTwZLQ7DQNM0LBYLmlYMjNb9Gj8GnG2IITBvGz74Uc+f/tG/4OT+PcYxcXF+vpNOMMaQU8A1TWHEhl0QMwVWOWceP37Mq9NTDo8O8f5aQKWKJJB6/b7vAz5hmb4W7F1jov88qPqzci7Xx/P151x/7tSdlVTajcHr57gf/QJyMZ1vYl/U+vIY+u4cdy1rpXMgBI+RyG13CimFncRHyoFEYdyWeZWV/Ky0yJioFAnjiB8GYlWDS4XBIiwXow3OViyXC5qqYrUeuNp6PIGAhWwwVmFVTdZiKhTDwBAiyntC0RmMMZY9SQruKCk6K2PFJ0BrYY1Oa0aamOJTIF8KGEoKgCqJKXUs1zUVI27YccuZ9Ol3gWe5lmli6Fy/piGSs8GaunhmCEgqHXiF7RCVSJEpabkHCWDJGesq0Rz2npxVCQATfgwcHyxomoaYIuvVWlpXpySGqVMA8rRmTMSAfL2jz6KV3QHl1hhhXZN3MgXzMm/bumEYRzKZatJ9toq6dlR1hTNuJ5+n1WTubnHGUFU1zllS8gLi5SlwT0z2PxOopiSLI2X5O8ZJjm3qZJzmlgT2fvQow+61KSWGYZCxAeXaCVB9eXnJ86fPadsZx8fH5Jzp+373d4wR7z3DMDD6EVQSAMqLjIn3Y1l7BHAMIePHVOQMRSU6+B7vR4ahZ7vt2KzWrDcd3meMrogRtkNP1iM5J5q6pqkbDg4OOT48Zj6b0287Li4uf6G5PXoYfELpQFIdtW5wWsA+V9eQI7HrUFoxjgNd32GtJedIXTvqphaphbLHuNrtOqCcEdmZnDJN7YjBFHNhIQDEMJYOTouKSJFbKZyxNAvHOIpcSvABaxR+6IUUoTNtXVM5K+SalIr+ZyI52Vd8jGJwZ8W01FhLDoHkRxROxlmW5DV6Ka6mGEFpnHXFt0NMURfzOZV10p0aKVJ8YVeg1NpiqoAyUlyaxnRTO1KKbMKARhGDIieNs6Jp37YzjNE4Z9Ba7Qo8ol2aqOsG7RyXF5elWCPMZmdd4SmkAsLaUmjLGOdwWRK83EAy4DvR7a1rg8FhtcZkGIctfbelqSqcMaQcaesKpWAoRsU+JSnyjJ75YkHdzlBK4cceH0UHX7DkTEoepRN1U6O1IQSZI/s9UmGrmrp2Yqja94QYdlKdCi1mp31PGAWMtk7a3lGgcsSisAqiH+m3HXGMaGuxzogxHXJPQpTChnMVKcEwBvSsIhtbPDsyTonkU87CHs8RTAVH7SFfefNN3rh7l+QDz85eErWmWVRkC36I2K4nhYw2mroV09oxSGEopojJkgP5QgCwRuOslfjSZDbbFblXO0a9Kd0wVjtSkC6W6/HalzkuXjmqWiTXuv6MuknUlWE2g8OTJdYobt48YbOKvHj2sayz1pCS5vJ05OWTDdG/oqqtGP8ly9Apnj4euHWvZtbMCCFhXAAtpLjz857D4xk3jg/ZXA3M5oqL0wGjxENsVlX024jSmR/+6GO+/qsVd+/OuXPrhMPmLX781x8Shsz68oqrM02zNNx/cMy2W/Phw0eQYe4cy0NH5TRv3m5IORCC5uJyoJk57t+/xfoqU1cDi5nl4cePuHh1QVtrtlc9IW34wQ/+guO7CZqBlCF6uLU4JA01yzdvcO/X73Pj2LGYDeRwzunVFcM2cHU1cDqsuHscePDWXVQawSk2viOenXK2WfH+177Gyewuf/XHf8l3vv995ncOefD2m8zbBuUj5+tzvvXdHxDSiCJy98GSw5PMZuixw5z58pivvPUWjZrx6vQhY9/R9efgemx1iKkGrq4+5WoDD58/5/DOMSTLoLb8P/7Jf8nRyQHBdYxDJmCojxKr7RlZW8g9/fmalX/Fs+cfcevGMUt3SNYJ3QRO159w9v1PqSrN8d0DPnr6HZ6dP2U2W/LoxXNumzuQ4HB+xOnlUz598RBlB/GXsh2u0vz1w+9yeHiDi5fn3L5zg9OLZ/Rx5PTlc7b9hq+//3VuvXnAn37nn6Ntz9PzpxyfLNj0W27fv8vVeuTRiw84urNg7S8433TM62N+9Mn30Rj+/PvnLJcLNqtLlM48f/mUbui4c+c2WSXJG5JnfqNl24/Mmzlj3pBUprIZnz0fPrngrQdf5fT8gicvH/Pw4SekqPjmyYKnnzzEWMu3vveK7Zvvc7Va8eLsFFUrXv3kBZ6exdBiSYQh0BzDeHXOh8++y3ZzxcFiTtArHr644uDgkBACT58+5f79+zz8F99mvliSG0Vz/Ivl3caKD5YQfTLJCHgtKgkCgO+7WYFJDvIaUpkLzpV02oHvRIXOe2mKCUyf5Ct22I6SiFmjd4QCAWsoHXW5PGfSiBbiiHRelpwqT4b20/uq3WfkJGaoRmWRAFQG4ypS0hAj3bBldXXF6IfyddQujtO6IicLRnK0tm05ODiQv5cLQDr/fBAfOVc1DMPAr//Gb/DeV9+XuFPrXbyYc+bTTz/l+z/4AdZari4vmYw3h6FHoRiysLXb2Yz5MBJD2pmd5rbBWOnGJsFmu90Z82pTiv0psbq6oqk1Z+eW09MZb9y/RdNWhNAVO59Jo366/mVvVEKOyRosYtYsBRIp8mkFJikMYBDVjKx06cIT2TFjgXIvcymYxKhKR+nkR3mNCJRE+iTuOhl0KQykHdB//ZiI1MICL8UAxY7Is+umzjJ+9dRRoQu/SQGqSEWCyOROuTs7JfLSLZ6LeOjUxSAEOKOF/26UwUhCwM7Pr3SKxpAIPpKCFB0mgpyQCEE7tTsfYwrGpKbCz3QvvhgR9AuD6JMWkmgjaaJCAtZMYSuVIGuSTSlJp0xKVXSZBIQvAhH7ioQS5F8J919y4CjO6pPGcbaZkMTwLGUZPMpYjNGlmiLaeVMLuDYGA7skd3LA3oEt+hroVxiAcdIbKmxPSdQykUDYDTJhXmktLDJnDQYxxIxksBpVGVL2XJx+SqMDV6dXbFcdxhquupHT0y1p9Nw+XnL/ZsvhckE7m4PJRKPx5pCcLX5IxLgmpjWrqDh55wa17ri6WrPdbvGbl/inn6A3F8zqGrsNLOaWk2YJ/pKzF2c8+/QZY9+RYmSzXvGDhxf8y+9+yKz9kLfee5t7b3yFm/ffZHF4yf35KcPs67yaPyCptoBWebcgJnIpl71elVBl8AkwdK0yyQRCQg6JOAYY17RxoDGGyihsWpFYcRUvSHkkjxmXR2rm5OomQQszR49XVKanSRqXa3wIGAUpO9bjgO8uiH6FTZEcNaMKrMeO42xBi+7q2gcxjfUaVTUkFF3/is5vcO1ypxkfNaU1OTFsfkLsFXb2PlodSvuryaA9OimaypCMJFDJD2Sf8JsLhu05ys1oZodS1QoeHyLVfIFranSGdUr05y+wdS2FIFMT+gtctaK5AdadktNPGP2cLWfE8SM2Y08fFLa3GHXFq9MneH0f4758oA4ChE6VSWEsCvAjm2ORLcjCXNw5IJcZvAO/1fUx8TqQCHuwcQ86KnZ8288832QRi1DaUPQXdszPvfmEVMUnjS6QAlv+3O7Za+dRPnf6vGm+h6h3zMKUY2mE0dR1zTB0jIPn8nLN5dUFYchcXghAXFct8/mStm44OT7GWc2HH3xU4p1M09SkMIqh17UgQYCMzGazEaC878hkjDJCUE2J5COQ0E6AOV3kDSYQKaXErG0FNEuaJ08+5b/4L/5LlosD/u7f/S3qArS/fPmSvh84OTlkGDqMNsxmLdteoZNIallXSduYsWgTCakjTMBWOTKvsw+0kQo3WrSz+3Fg82KLqxwHi5amqVDW0PuRvLpEaUNbz/j2t7/NO+NICBkwXJ5fQMrMmpZeZZxWRXc27r7zdL+ssYzjyPn5OV3X0bYNbat3Y2uqsr/eBrkHkkuksbuG++92bbR8wcT3Zz3vp1kanzG1LQGO6BaWdVIVNnoZnqkUtoyaikpf7kjFJFAkgkpHRymE6NJ2NWkchpQgynrgo5TO0HrPSkGRhWKAMRWoAR8S/RhofCQkRcgKkwwRg0+aiMG4BqOluC3a2gp0RUoKq1zpCBsISRNSBDwhToAK0qlVIn8RkZHAh5xRKUEKpRYg68Ik76MwEqBO61QOu9eRJWFQSmPUpDs/FewS5FjYFrIGhiCtgZPx5V72SO61s4q2dbRtJYaFORZgRaSV8hhEHqN2mFqMoCfW53y+oJ0tgEzbzjk5OWK5nHNycszde7dYLhdYa+l7Mcae5EuMMTsWuXWOuq4LQC4/2xIjmSKZJ5rqqngmFHC/MFCMkThtHILI9pVYyBpDJJKVzMUYM9GHEruVAD0rQvCkmBjiQCIQwrhjmIQ4kkMsYOmeyRRTYvCemKXbRIooAoQPhd0dQ8L7wOhFa3fbdWVcJ/q+3xX3BAAXMHi1WrFdi261tZbFYoFSir7vX5N0kbU4oe0+0E8TO18brBVN9Jw1fpTW3rqWa1g5ix9L5yKacZtIoccPEVwG5cr80lhXMZ/NqauaqmqECZ4V3kfW682XntsASVnGkLBOZE20VZATWlmqyhGCMPi1UWKTocu6o6QgbSZ2js4YLUweqxW2aQubW+5zVTtGVYwTc5ROjhRJKdJ15TuUQmdtK2ZNi9VibpmsjMmpeGdtRWUtVhsxmc1Fdz3FUhAqRW6jZT1KiaaVJDlFYTTHoFEZnLGk0RegXTpX68rgQ2YcR8iWtqnFtyYrXErQ9Ywhiq67km4D5YQhl9G0VcW8bamco++3OF1TLeuyXxic28sBxegJYSAm6SIb/QhZ0fcDoFku5mhr8b7DeCnEuEqMgnVW+BCkQ0ZJ/GGrCm2t7AFVJtliJqYUtBaVHTZaSaTjnJjEE6ofOmLxgko50Q2jdI9ES8oZW1fMlnPmB0tUznSrhM8JWxm53imiNLRthSpyduOYi1ye3Getzb4oqLK0kVtH09SYIjs3hpHoAxhDnLoENTKfjKK2UqR58eIFfe8BMey0ShOL0Z5IKYk2/hC9sOQV6CrhC8ustU6MULMmKU+OPYlEVSThmllDzImLywtevHzO8sYRR8cH5Ks1IffMDpfEPkoRpDH4IGtHKuthypKXSnEgk6Lo56IUMQT60QvDUWnqpiKW9SSqDNngqoaqsrsuui9zxFDTxUjKHtdElEvoIdJahdIrvvaNI7QO3Lp9wCePRNpFa8fLlz3z+Zyvfu2I7WbLthvQ2jCrj/Cj5urqJRenI86OHGqDD+IHdbVKdGt4/nRFVVneevOIg2XichYYukjdVNT/P9r+7MeW7c7vxD5rimkPOZ/pjuTl5ViqgSqVpFJJlrrRarcb7hf3i23A9pNh+48w0A8eAKNf+8kG9GYYMNpowLaEdktWV0uqkopVxSJZnO58zz1Tnpz2FMMa/bAidua5vGRRpBzEZZ7cuXPvyNgRK9b6/r6/z7fIYdJIgTGJ85fPaRrJP/h7v8HTx4Gnz96jKEq2W8/1ix3y2nP+5DkRj6nHIHsKdkNglwaKRmLDwMX5GiM1/RAoq4rosyN+125Yr3uaqkTGnI/V2R0xGuaLimA8XlmsLTDpkPV5z9WLn7A5+JA3vvKQ5fGSdv2Cbsi5F4vFPc7eOiY6hxUdrd9QqAYhJJ99/IzVbsfp8iEBi2oSfRyw2zVvvX4Pax3dZsd81vDa6w/44Q8/5ezklPunD3ny7AM++bQjuoFv//abrFPgvfPvsDgQ3Fz1WDfgY8fRgeHq8jld/5y6mtHHFS+un4AwPL74FKkjdrXFh4jWFX0IbNYdyMBm7Zg1gsPDA4a4Q+jI1fUVboj03YCpoWwSdvA5kFeUXF5ec/pggUgGGxQvrh5TFxWzWvOnf/EvefvtL+PFwNquKErBervh8OSQTz75gAcPT7haPaUoZrz30V9yeDznYGm4bB/zL//sKTc3O7y3zBaJhZfs1gEfzhmCp6gURVmi6xldK9h0a7wM+CGjID/87H0ePLjHxx99zPFJjSwcV5sXLA8OaO0GbUputiuUNkjv6GNLXVVs2xuU8mz7gc/OP8P7wNWPXzIMDiUVf/ydf07XdSyXS5azJYPZsE2X3AwvcL1ndlCT+sjF1TN0zL/Tup427ditc2bQ4vg+N+0NVW24Wj1nsZgzWyo6e82mvcHTc3r6BnZof+VrG0DqwOS9TYnc8T3+t3ei779OYvidNXSCKCKKiJzWHGHU1kYhXsgRozcZTCatbXLBZTzEaJyZ5mi5i2tiTN9hYCLE1IV+GzLKfvacO9gmM2UG9o0FgbEIIJVBG41Pkm4Uoo0xudA+zs1SShSFxhQFTVNTFAXHx4fMZrPcRbTZUBYlOduMjOXTmq+9+y6/9du/gymK0fx66wT87LPP+Mf/+B/zw7/8PpvNOpMoZP79mLK50nmbsXda0cxmOB/wITBYi/WOuqwpTQ0xsvMTeiYfx2l9YEUiRUXbmrGAHPB+QKpsHpCTiA4IkefZeZ2aWfNSZQZ7XogIfBSIlBEoSgo0Aq/UiHOR6EJRFlmDyuzzSW+R4zxTEXTWcMQouGYpPu2XQTn/cVqzqj1+7/Pb5AK/iyoFRmH+1tgznSeZ8jGtxRlNbml/3JMApEYpPeb0jDz8/ZvnLqxEnpsrNeZmMTrXJ4NzkjnXZwxldS6M6PEREz5RUtJ4vo+aphwzpsY/Ls+FGddDd/6+X7T90iK6mRZhahS1x6WmFGpfgZmcFkLlCzSOF7UYMSraZAFepNvk1HxwFGJ0Nu0XxvtSGfuKSEKSFGNXyz43eD82yNHdN9VEhLgVw6fq2LR4SoCfxIRReFeTk2Q8oFOarRQCmfyeGZpSHjKkGhlC0hNDXozIwqBLjVCK7rKlszt224HHH33GTEuqyqBUdnW9/sYhD790n+ZojjSK5C1ydkRsjokhgU/gcniALQ5Y7TpoB9quRwiw2zV9bxFqgTp6RJSwTT3aFszrhvX6KZ89fgYy4eyAtQMrl/hwJdg8b/nTpz/lYPGYN+8d8vWvv8vhgzdxB4+pTr5B9/DbMFuS5C07VU3H+guurlylvPuznNxMAOHywByDRQ87jvw1cxmZlwVF7BnYIsMGFQU6FIS4xpQKpQJeDBRpRyF26GhRYY1IGnxL8AIb58gk0VLSmAZVLQhiICDwqWDTWowO+NQjhGCwDnzFTAdkcnjX44NAywVC1OPvRVKQDL3FbZ4icSzSAZV5C2FKBLmDoVAgtSEUM1Al0Tv8+gp3/Snd5iWmPqUQgrKeIUuNKivKeoaSOruAvMX2G0LcYZZnIA0uCHobcGlGIZc4/1Nsd8W234CF7TaxXu3wzuLthsefPSEWlsH+esGiIeSJvhjxRYlpcLpFjkystF/XFfszAvs0wCL2xbdxdMlVyyyxjS7oHLqSEoQxnG9frcyv+gvec3Kfs3cET4PHnjccPDF6nBswuiSjqzLHM98sFZt1x/mzl/Stx7mI0DD0A8PQM/QdyWguXp7TblcYBbvNhs36Jid7K71vjbbW0rZtRqP4jNeQY6tXPkRThTU7caOKe0G5rmsAttst1g4IleiHgEgV3//eD/m//J//EXbo+Yf/4d9jt9vy/vvvv1JZHuzAcjknCoEuag4ODgkhjME+2a25WC65vnjJdg1d1+Ui2Z3PfhLPpVJY5zg4aBBO4kNH23W07TYjAqqGo8MTvvLVr6NVgSkqNm3H4cEBu23Pzc0GozVlWVBXFcEPpBBGnIzf36xvszAiXdex2+1GtrzYO9XFXtycxPFbR+v0X4xx72j//9d214X+V9+I86QhJpELsTHmjIKJgz4Gnvyq28RgFqJCiJR5zGQH3i1yZvqqiSnz9kTKVX6BRIrJ1ZJFNoFGKJNdgRFsABslHkUQmigSXhosmj5KbMwMbB9zAU7ILPKSFEkoQoRgAkprVFBomfatj4yuhf28Y5r4iOx00XJgZjRK5YIrMTs7cwHOEn1eVEgpSMFi+yksXKJMDimeGONaZ2FaSEnyEaWgMJnzm/m/hrIo82RfyT1iyWhDAuzwLn/zb/4mw9CiJRweHjCfL2iqGmMK6qpmNp9RlAVSZyGurmrm8+WYlVABYt+BUTclpTH7hUneJuF+Oq9vi5+5BfJ2gTWNHyBGh0kWw4KDlCYXVDYhOOfHOZwk+IzCm0TphMcGm9373pFiwruMj3HOY3tLP/QEF/Ahh3ZltGKk3e24uHzJbrPNmLPgc7eLy0F+Qmkiae8MDz7P/8KI9dPKUBQlCAgpd/UZY6iqal8I895T1/V+vldV1V7I1NpQjuxzpfICqTC5EDKNv2VVMJvVKCXo+xxcrJTMrcOmwtlICB2Q0FrR1DUpSlIaSDuH7XbE0CPwKJkFxj1mTGjKcsZ8fkRdVnm87AOu39G1A+G2RvkrbdV8njv4RKJuqoyuAZTQqHHBWtQlRhuKusKUZrzX5nqi1prU51Zw7/NnHL1DqzIXAcb7kHM2u/WD3ecUGJkF2r7t8TFfv0IIUpmDDlOKufNB6my6kRl5IfUYjCtztohWCjtYbNvlAr2U+BSxzhF9wrt8Hwgx5DEs5fOz0BoB+byMuQBtjEKp/DnlDvdAbhnOnaqMnazG5K7VZkKHhRw+m2KkLktKnRFANgiaqsHUsyzix7FbTeT1jpRgXZ9xLiGOnSJyzDfpqWtPWRYI5ShrjSRQFCZji6rEzfqGwQ1AzqWRZizaBYdzASV1dk8TGbDknpmCUhpMXVL6KgvXfcgmAzEGauHwoxPQlCVFWWB9z6YTHM6WVFWFjTl7J0SPcxYlE8WsgpQLdilFjFG5W1Pm1vHCjAGmY2eIysB6UhIMNhCGhE4Go0y+D8Q8PujSoILC20gfOgbnMxOeMaBN5HMlEAlBjAtuuS98CaMZBod3WXjKeRq5qCFEIAowRUFRlihtuL5Z0e962vWWvhuYhyzeFYVBqgFd6By+KsfG8eCRQKn1XhiRQiC0wUhJoQpUkvS9hZjQSeYQX+fRs5qg9JjXEtCqRpHwe8PJr7a5MHbJ+MCD1xck0WZGezCYOpHkhqJKlMWcB6/NuLnRvP7GI15evGS7bjlcCo7eMAQEZbnk5jrx2afrjDfSNc+fbrDecP/1Ai0SRlc8ebli3pSUB5Iffv8ZdWVo6grX7zhYzjk4jQQC6+tA3cxH1J+jNNec3Qv8z/+X73C0eJsf/3DDP/9nf5pDlg0IHakWGu8C62vLomlYHM64vGpxKeKCQguIwXN1fUm3LfjqVw84fOOY1ZHj2QcbClMik0QExbPHA9/4zXfphcHpa5SsOH8v4J/Bga44Poxcri540VqkCzy4/yY3N2vWTx0QSaFn6HfUC0G/e4GIJZ9+vOLhozN+8N2/5Nt/67c5e/OUmSwYJNysN/z4L3+EtAHbW+6/8Tr37815/bVj1tc7QlfytbfeZbvtuTp/xsv4GF0lpKxot/Daa2/x4vwjwqBZ2RWmstxsWkyjeXL+KVLV9KHL41LKbuCoCtabDUpHqlnBo0dn3Fw7Pn78Gffvzcf5XGDohlygl/lebUw5XhuCuqpzkTtEmmaGTonSKIRyoByffPY+SUMfLIcHM5QUtN2aBw+PWa8v6PrAcpFo5nB8b8lmd8Hl5gVNXVPNG3Y70GVDDIa/9e2/zoef/RSbdrg0cHP5jIPFCVqWyCLQ+wEh8zUkS8O233L64IgUBiSarre4sELIik8/3fLVr71OXRUoCe22I3rL8fGSXXtN2Rhudlf024Gz02OaZoa1PQeHh3Rd7pbb7F7w3R+vefToNYqlIPYRF1rAoWRks8vGniE4Xl6tODs7QUjFzbqjrhfs+h3WZvPobLZAKcVikQsWTz54Lwu5v8YmZYBxRBdk5vgIQmDimE8saCFHIxvsHeAJkCMWGSYDQCDErNuk0aA2gjPyOIsa19Vi1OgyOSKNXdMxjaH3o9aT552j2C/iKPxO66vIpM5lzSzdIkkm0TZmcThPXxXO9cQYULrm+OSYl5crrq93TNljZVmORrEszk/rZWMM1lrWqzXRe44OD6nqhqKsWCwWHB0d8wd/9+9xdu9eRpRojYwS6yxCCH7wgx/w7Nkzrq+v82v53A0+mcQmDdM6h1AprweKUfCPGd+MyIHYSgpKN6B2EuEDIUZiHA3EKq9Vz85OKApFCBZtJBCQKu5FZRglzsStQ1xkvU/IyeQnM/kg5uMn8/IHGeUkYqBMzvXI6MOxnJEm9HI2G4Ug99LIXd0j35oE2oNzE7XjVit5ZcE/bpPWl5dg+edSjejYSb0ZdZaMTrmVDcU+PPe2cCRG8VorlQs8Qt76nFLYi945n0egxRjGnk+/LKgrQRwd7iFEVIigxz/HuREtN3ZiyFtNSzJx2nM33i0uZuq0/XcooqtRZBbytmVaiMzSFEpmG13In/Jk3xfp9oNSEsy4WAcI04FiSg+eWJ+5ij8WDnItLI0fTZouKrGvfIQwoR3SrZQ2ivRC5CpUGn9fKrFneE6nho8gxO1BFeMHi1BjcGqu/JBusQXAXtTZ5xmn6TMVJKUpKkNx9AC3u6JQEeETH//gRywryaPffMS8LtEaRAVBRKQ0JKNRsyNkPUNKje0DjG2LRbPEhchms0JqTaGzAKIWp6iywc9Pcb6nG7aUuqE0JXp+xOX1LrfdhsjWOc53GUdx2FS8/mjJ6w9POTk+oigEu5tnqGFLOH9Od/Ec8Y1/n+boDCnSq60d4vYf02Vz9+FEQguogCpGSjuQnKPD42SgjDsav6ZJFRUJ1a+4j6QoZiRVcJEkVlSZsZh6jktPaWY5hMu3iKQp40CbSpKpUarCyIAejkl+Q+d7tDJIVbDrLc6/zIsXbRAcZv51N1CwySFW5hirj9mkGe0gWW1a1rse17W4bkOhoW/XSPM8c7Xmh2ij0UojRYFTZRaOrCUMPcHu8iJr6Ai2RTQlpqiQskZqhQ8+i4DbFev1S5rlAaWzSN0gREk31KxvDHF+QnBfI8gdEUfYXbNdP2Oz2tBu1wyt48XzwM32gqcHHwL/k1/2cv6CLYyC9ogzGq++6RqYqngpxbGT5FdbGPxct/p07qQ0csBCDhCVubVNirsi6OiiE6Nrbf/70z59QZFHTGNWvqnsxwUBkAtrMmSHVYjZLZfy1TwOzJmJ3ncDRVGBUGzWN3gvWM4OqOrM8N3tdrxYXzOfN7z24IxPP/4gu8WbJjucVA7F22w2bDabcQGiRlb6KBSON/ZpjHHO5cW1yP/OjtSeruvYbDYICVWj6ToPMVIawU9+8j7/1X/1/+S3f/tbCCX2yICbmxVKCbbbTR4jlaJtu5GLHkema66C13WDuXcfJTI6Zr2+wdpXg3SscyzrGh8DNzc3HB4dghC5nds7Qkxsdx2JFQcnp3z7278HQvLBe+8TfD7ntFLMmoaurehajdE5P6Jt230Y3sQ8zi37kd1uBylxeXlJ01SvCIqMgm9iKpjcusLviuk/D8fy72r75UV0xtDO3O2ThZyxYi/GRspfw4k+mzV07Q4hxntYyMUaFWXmoMsRx6ENRhVEJDIp0BLSiNTxU7aIQcuEVgahNELpjOQQAhsjNqYcriwFRA1aEqQmCYVP+fwSMk8AtcpFc+s83jp8PxBcT/IWhyNKQdT53uxDwHkPUlBXFaIq0EpSGcXxwYzT42OMMbRdn9v8TQ6cyW5tTVXV1HWF0SI7WWFEmCyomya7tk2BKfTegSCFwOiMppjwJTnQRtyGA8kcYoocwxyFIGPDEzG5HBqUxnnS2IIZYiSksMfS5YLO+NmQJ+shZY7fMARkyovgqZgW9mHqkyge9/9Novf074knOYnWzjmc9zjrGAaLtX7/vPx45njfHZecz4sKF11mSap879tt2zyvEhJnHZvNlrbtEBLO7p+wXM7xznHx8iWfPv6EzWpFVYwID8hCeFNTVg1yDAvNDqQicyx9HpOMLqiqfO8cbIcJBmMMdV0zobGcc0whzSklFosFQ+24urrJxTeR2fq6KHKhYOwCymz9bImJcZrnjK4a0r4zSQhJWWYnc0yebhhQ5LDl+XyGtwPbLRQmX0cBw65zOB8xoqTQC+rqgFnd5OwL36NkQquCpln8ytc2QD2vWaYlPthRIJfZCaua/VzZmFw8KIsCrbPBJQRPWekRoRUQKeGdgzBiEEUWHQc7sOtaIgFTaJARJSQyZQRXAtouOwHLMhe9rHRY70ZkSZ4nuuTwyeUuUGlyxlHMrdBGGXzyDD4SAwgPpMiu3UHMgWSDtfRDh9aChEaLmHNsQr6PSWMQMlEWhhhdvr8RMtpRGKTUIDQai3S52yxGR8KidEVdVAx2QKTM6PTOYWSOz5oCvvLaJo5FBkWMln7YEaPPHPLpHDaGoqiyo0xGpInURcF8XiOIFLqgMBWJhOrX9N0AURJcFs+NljkwLcR952wkYFN2l8vgSCmjeoSRECQuBXShxs9OUkiDcKC1QBWJgKXtO3Zui1ES7fM157wn4EnJo7XKi3bJeB/NTm1tctZAURhkkmP3SR7jrHV5gSwT/W7Ab0dHoQBpsmhqbR7znMhFkM5ZirrKXYPRImNCRxCpHMepMBbIYOgdVgSapSb6SHB5DuqFZ2u3FEWd5aEUM5ZHG3xIrNYtu9RjhEImxfZ6QwgClwLJO7ztc5cWkuQEyTkqpcauP4lSBiny/UJLhU65uKhDGAvLEiEt3jrcYPM+xOyUT8Ln9xj5wr/qFlwiREdRgRAeraGeVSgFDs/6quP+caLrdtx/o2RxVrDZvgQZsdazayNHp2e03Y73fvKC64ssBGmy6KCF4uJFj7UFJ2eHLOenLKrIH/yt3+Heo8j3f/B9njxZo1SirEoG7yhNwclxTbfb8Oz5FW9+5ZTD0zk//uSCsvLgO/67f/WYqxcKU1uWR+UoJimqCoJWlKlgt5O024BPJksHMuJDotsEum3gwdkMvOSnP7zhN7/1NuefSZ49e8G9owUpSDYvO777h8+4//qCPgaeP32B314zqzTv/NZDZvcC109fYtMNpMDlquHR/Tfpdi3r7SXLA0NdH/Hi4jOUDrih4+Frp4Ch7zw//suPuPfglG6zYtZUfPrR+2iROzp228CLZ5eUdQdyy4uXFxwfnvHwwT1urlcMQ8eu3XL+4prlQnHvfkPfrxAUDLYlxBZZKlRZcHxvRvA1Ly9WHBwfM7iWrnMcLY7Ybnf53NOGppqx26wyHm8Obbeiripa15NUoG4Muki4ICiqipQaoMD5HiEkdVMTY2R3vaGqalZtS3UwoyhKrq5vqKoF1xdrmsKgdIUHbAy8/c4bfPbxE6TUXN1cIGS+RxjyNT4fi2N///f+IX43cP3yBatuS1MIqBq0UnRdQLqALjT9kIvUDx8+QCZ4+eKc4BOHhwuKsuHwcMl6veGr7zQcLw8IIbK6ecmjh49oux2rm2tmszlawUV3w8HxHFVIetdiw0A55M6RMAyUUqJk4PGnHyBVNjeuVy2LZUaqxmQ4Wpzy/PwJ94+POFwe8PGnTwgph6svmoYHi4NcMIySdrPj4ckZvdpha0dR/Xoiujbj6DCtXVRibBVjmodM61qlJiPZq0a2lCafzShWjwYTKUDnF2cymSVyoDkpY0NFAiEVCkFSubMsyTCOZaBkGIszeXyXMov8E+Yucov7VEKhZGayJzL+OdMpAlVRIMiFfSEKnJeElFguF5ycnrBe9wSf536T2UNrOSLFJoOEZr1e8+L8HCUURVlT1jPOzu5xeHTE3/gbf4OvfOUro5t5XNeNYujFxQVXV1c8ffqUm5sV1g6E4FEiz42klAzjfNN5j9CBoqip6gpnHX3M82A1doh57/ZrAPB5XhgzKvTgcIlkYLfbgIjE5FA6d5QLOYrio3oec9hk7kAOt14pgR7novn3lIx5DhBvRey81gWl8s+NyTlV+fPJxlVUBANR365DxV5AT+PrCaJSmDvzF2BvWmMSwe9s+a2zUVmIrK3mztbpp+yLIPvfF9OfPbrQJ31YCKRMKBFH5/qozzBl+GTnvBQ58FMSUTKjXqRSY1FIEHz+u7UUDEZh7YAPASlNNgn5rBtJkabpTTYYpXEdLARa3pquJwrKX3kN/5XPGLd9uqoc3Yh3MA8xZfeD0HJfyciLK4HU+4IJSYyt4Iwu78nPPh7h6VK/G7inxq9xDCYVKeYJziQ4jdW1PTc5TfiR2xb4PPDcFlUS6ZaXBHungBirErnSF/cnmADUnQszM1bHDwEyw3g8GnGc+AVkdu8zkIYWu11xdb3i/mHBg7OG+bxgs2vxbkCpI4LrmC2OYHRbCUqEVnjXIrXB+p62za3LjSlyhUaCaQ5R9ZIhJKBgGKA7X2NP5lxuO95/tmboe3a9Zb3r6WzAR8fMGA7rgmVTcna6pHOJ+mBGWRe0zsPqU579+T8h/c5/yMHJ2W3W5H5gz2ehvPPJTY9nvx5UMTH3nqOhpQqWtYJLJbGq5qXt2VnLUgak7TiShkY19KriWi4Y9GF2hYRAXcCsqEhhzmq3JboN+IHIQ1zVoOsDvISwe8HOOfqhZVlm91oKguvtmtLB8eF9TFHgnGTVt0i7IhYF1BVKz4nyiNZ5Op/xN123I0WFThCSw9kNelhjmgbUDFWUuKBwcQxxAEKCwQm8l0iVOYl6DLxIoxaWyIt53Sww8xMg4ttrVIzIIAlB0LceYQVC3YfK5MW+aqlnl1j3BBdWbNeP6fpzCD1X509/2Uv5C7d9evfIsZquA0lEiekamq7rHC6TCKSx3esXCZE//2dTG9N4LUdyVTwaUihJ0uVOhhQIDGOVPC/IQ0q5pRymUQSBIdfQMrP+Vph/VUidTtWJTZ3/foVWBVI4fMy8VO+y49H7QFGUzGYzCl1wc5Unss18jnewvrlhGDwPxENuomc5X9BUhvVVdqM3VYXznpvNml3bshsxJWVRUNY1280G7z3KjK1wTDd/tRfVh8EBimYuEFohjULHAr9OyCRQuqBsDKvrLVGAS47v/fBHfPr0JV/96lsYU1CYGqUTUjmuV1v8ukfpCuc8i+UM53vO+2tOTk9RqmJ13SLHxOq6rti1CunF2O48jpE+Cy9NWdPFlvXNmtmsYTGfs91sRyE9YoeB/+8//ad88tEn/O3f/zs8evSA7WZHM5sTCUgtaJYzdrblxcV57rBB7lv7hABr8/inlKbvW9q2w9nI6mYHKKrKjIJZPrfiPkjx1oHO+P10b5zCFoUYx3Nxe26Osvvdq+QLz+3pPe4+V4zXupgKxWJyBMPtfXkq6EhEEnlSQK7oT2x0KSBwi1/5Vbbdbrv/u52ziJgF0uCz+IgMVCPDet/iKLJXQIwM9Sn1XZCDRoPzBD8gRRaptIIUHSEM+OgotKEoiszOrYrsroya+WLGI1OgqzllPUeojA1QSo0BOAIpEmWpqMuK2WJO0zQURYE2BWVdZqeKyUJqVZQ0haYuy4wNidm5MAn1ch+Yk0eIlHLyBONEaSwVkoNE7xT2ps8+3s4PpnPIOzeK57ddDtZbrMsM4Oy6iaTkSZPoHRMpJJz12OBIUtD1YwCvjwz9sGdwT45s5zwCSQppFOxGN/DI855E5uyOjRhj9mNcGDs5IIvVIQT6Pndjaa3HCX9B3w10fY8x2b3cdR3W2lEcnqOVwof8eZtCcf/0PsvlEsiL36HrqZsaIbK77YqAC4FZ03B6epoXIF3PfDbHdv0rqLzMnK5oZrPReXLbJZK7gsa537ioEuPvhjhds2LPtJxec8I/ZYZmQ4xX9H1P0zSoMai4KIo9YmEq0gkB3gWctSQixhQoLfHe4r3DmJK6qQg+0g+500arRF0oyqqiqiqMMTQziZQ1rU2sO491YSzAGMqixpiKvre0u24sJAli/PWs6DYOCE12AomAFBotSrSuUCpRlIYY8wJHakWpsojuXA7YE0BRGhQiYw2DJ4YwZt1rQuxw3uGiJQpDWZc4H0hJjkgngUQTPag6d1gkwE4FOzkusiQIPTqmSpO7FuyAlBpNZnej9LgIzNil4AUiCQqdkYlCSQIBJTO2LYpcmDZ1zvGRKuQ5ihRE77NIHnN3qNEVQhYEIQjbbb7HB0s/JGK0yKJksBaZsiAjI8zrBmcdne/BWaSSJHJgZEwa6wLOdZRVMbo/u7GjpMToEik1UkeiyKgcpQVGV0h0dlm73JEotUIkTfCWGBNK5wJd7CMyCISSOAIhBbRUJOkZQiK5cR6k8vrDE/dOOCMiUuf7oY89buwEkBq6fov2ZS7CiZDXb9k2mAPDEyN2KWRHXrpFH8U4Xjcy55eIlNdnwVr63qKCRkhN3zpkkQPgU9JYO+bMEGmtw8waEjl/I+ByEK2F3a4lGx7kWGjPxUstFHVZo3F0bc8QB4w0LJoCqSJt3+F9FuucC/ghYKTGi0hwic53JBTJCGyfXcjj6g6SQsSEFgKjDUpqyqpGIunbLjsBI+ikKSVsuw6XHIGI147NZoPSudM6BoeXfly8xy+aNvzy17b1CJkDnfvOkuttOb9hPlugTUdVZMHv4Fgykw1vmlPsVvOdP/kRz19s2e6yicq5UdhIec2aw1tzB/f2JuL6HUK2eJvQMvK1r96nXr5gdf2A7/35cwa/4+Jmw6NHx8znkm9885TDk4bVdk2zLfnw6XO+9Rtv82/+5BMqdZSBt1KxurYUSiNjjYmOolTUiyM+/eiSd795zGp3SVkEMIIwgEBxfCC4d7yg3QSefLQj7B7z23/9N/lvL685v1xTCoWI8MEPXvD+D54xqwxlAfe/1HD6Zc1rv3WP1m/4ylFDSprkAm7T8OTxOc+enPPVbx5jqjk+KaQ6RIqBEDes+x0pdKSk+cvvPwdexwdLuHqOFokks8h0em+GViVHp8dImXj7S69hh8jjJ59wc9NSlQ2HB0d86UunWL8iJcvQw1tvPSR36c94/vKcrndY22JMw8HyiJdXL6mqPL/tjafdWRaLipOTI4ahhehxvkUqjQ+e+vCEGDN2SCvJ0HYkKQgqslptqaoFUhYkBIPtOThcsH0caOYGU0pevHhJWWhurnbcP6vod5HQ90jjQUt6G1lv+tGImLi+vOLwYMlBdcpf+8pv8KO//Cll0pwdP+Dy6Zavv/11zmYf0JiSaiF5+Poj3n//E1K/ZrasWbc3GCk4PV7kNaSHRVOzPDhGaUU/7Fivrjk5OWYYPLvNKs+sReT5y6cslkukMux2uXvs/v1Trq6v8OTxMUnBrh9YNjMOFkcQ87XSdR036w1lXRK8Q2tD3Sy4uR4o+o77906JYaDfbXj48AybEjFYlFT045yh37UYJZlXFcJanPd0YfjVL24y1uvW4JMfmzqtmdze0xp1dKLL/XrjFlWZlzZi7CLNGkyUY+N4BO6uZtLEp07Z+5o8cRQnZfIoAmLkSkop0Hq8h4s0UhVzhmGQERknwRSkivl541pBTnpRgrLMRccY8h9YmIJdH1itbui6W5zLNL8FKEuD1ooHDx6wWCzo+xbvPX3fUZoK5xxt24KAv/bX/hpvvPlmNkpKnbu6Rc70ePnyJd/5znf4wz/8Qz766CN22w1tu0MrSVPXlGWe+4YQ8DHksPSUQKjcTSbAlCUyBgY7UGo9mrrYi68g0DrntPR9R4otIVZZHB5zUJS+xamMHyxpMuBO61HGYyYYtc3RziHCaPbNa0k1/dokoqs8780vn13VSU7rzld11v2WRlFesBfnb9dLkklD//yW9Vb2hRqxN13y6tdXxPPpfB1pHnF6qtgX4KWcOOppv4YPaewSS3lNJUdzkxQJJRiDVPM5n9EXAsgdbCSFVOD9uI85uT4XP3KkVp6TyVF3GZV1IcbHlRgNpb94++Wd6EZnJ/ooNsgIQWYZPN05CYAxZXWSFSS3n93tURVjVWHst8xQd3Eb4qWSyp/sfsu8nzA+pOTkah8dfDLdts2JfcwlY4GHPadnZC4yVevGhXWYxPf935JdGHtu1L6Mkl8/ppQnmTA6MrKLMIVADJEUNaYsEaUhzAT37hu+/e17JBcoRySDlGCKEmMkUpQIURGdx9mBkLLDzTmHSRDilpgqohA5Nd4PmTNY1nhEbt/0kcvdQLAbnj79hL/8oz/hg09fILXKi2UbWC4qjg7mXF1v+Omn12x6jwuO03un2E5htGFRS1K8pr1xbL/739D/jf8+zdHRnc9xotnfXpgy5TpUFLmYWvlE2Q5UbcvM7qiFQ+gCyppOHbFFM3QrHBvmRlJESaUNQs9p5JK+WNJL2PVrqugpK090A7Fv6Xbn7PqOoToiNmPbpzAMXnG16eg21yglqar5KLpKQszKaQo7tJjRukBvBcIo5tUxvjrE6gYbA6qxFH2NKY4YhudIvcCUpzjv0dbjQr5wUnCZ0xoTAo+KEaTGVEt0UWNmc1SzAFMRpEZEiM7ntlataeo5vpnB8ByTtkgnqPQB0cxoSoWIPTmAQ1POjpH6CCWOoXgNO6wZ7CkXz/5b5vM5dXX/l72Uv/j6nnAOYmJF3V6nQtzp8hAiE4lFGM+D22La57efV8X74scnITOHdpIKUsyLOkQkpCxQ5YkC+JTwKfM0JdldI+Ud7hfT/r9a4Jke//y/89+c299DzILWYC2FmWUBLMHBwQFHR8c09XM+eP8v6VqLViUhgNEFH334AYdnp6R4RLId2/UN3jlW6xsur2+ym290WhpjEFIQgs/uNylHQTeOxzSPVzHmhGltSsqqxhQlhyN65fLyisLkwkFTz+nFMLJ9E0lEfIx89PFjfvd3v8W7736F9977EK0iPu6YzSvWqx3OJWZVzbysWd6/T9M0dF2PtQERM090s7lmNpuhtWQYh0EhEiLlln3b9ZiZotC53W7oeoLOjl6URorIvGn48ttvce/0mB99/3u88+7XOTm5n5PV81KawQ5Y7/IkRuYq8+SIzZidMJ6XWcj66Xvvcf/+a1RVM/KN86RLaTPiNuCVGQrTvt9OYl75eqd17eeXhO6csa843L/g1Bq/H+dYTK549qXZ/PQsHOcMAInaFyvT2MIXxZ3X/xW2uq6Jo2PZGM3XvvI1wle/wieffMx6s0UozeHRMacnxxyfHFKU1T7kdaxMUxQlWmUGsNIKZx0hdkiRMKbCaIPWOXhysThCquzwKApNYTSlAZ0yaxtpkKbMbESlbyeDIs+Dckh4usWnjY6NKKZQG8mEnSJl4YNRyFHC3CmET8Vx7hTR8xwkxLEwnzK10Vo7unPjGHCZ+Yh+dG7nlHeH946+a3H+ltntfBaX7DBwdX3JbrvGhwGSZ+j73JXhQg7BQeKip3cW5x1aabQ2OBey20kIZrMZTT2jKEtmzYwH9x6wXCwy91Fmdm9VN5RFSSKjl0iJppkxm81IKdL1PUM/UBSGpmnwY/eLQFA3NVIqhsFzs1rnYOSmJsXEar3m6uoKKQTNrAGysD6hZe6N+9K2Oy7lBYPtMSYXQPIMyOOjpW13WbyuG+7du4fzAwLYrtfEGCmKYt9l45zL88t9Dk1ub1Yj2x1kduDHiKkyyuputoRSaix2+rHIlp39iYTzka4f2LVdPq6zOTEmrM2FB611LrbHhHN9DtlTYIo8PxumkNHg9ri6kCI+GVIITLlg1vmME1EaaUxGu6VIICGEIoaIc4HCeEJwhOgY+g5JxJS/Hoqt63N7dEKiVImWhkIZwCGDQEWBFBoXAs56RBIoBNpLUh9ISqCNQRlNu2sZQi7+yGhzgQdL0AkhNNEoktGZt209SerRdV1Sx0QznyO0ym5/Cc5ZSBGjFbIUOC8wRUWla1y0mUE6MvqlSRRKk2TEOj/y/DWkjI0sjaE0ApLLo+coKCmlcFiccCRr6TpHVZo8zkeFDwKPptIlMiZkSBAC3lmCypiD2DtUJcYwywQpUGqFCx0hZrY+LiBSLqoLIUgh76dEUBUVIDOfXSa0lFRlDrNPykLQ5NSkjLIChQ/gXEKEgrpoKGVF57ZoLahLRTdsSSmho6RSFS4EtCiQUlPMSqINbIddXrsYQ0qeITmMEigV0TF36dqQsD4ilKIoMk7G2j6L2yJhtEYoMq5JkoPbh7GQkm9aCNh3yymlsXFsS5eSQtd4H+m7HmctdT2jLCpsu82YHSFAGRKRsjYkAkEkirE7x8mIqQuEKYjt6H7XCkFA6cTB4ZzgQYiAwDFrDCJFnA3ZmV1rkIkuSpImdzRMY4NQtH4gJY+SGfuTBohOYGS1z2dwY6ZWDvR2ICH6gSQyWiN4P4r9iQHHkAZ2nSNJMFIikiT7mgQyRKSwFKXBx0Rv7a98bSc8Sgua+YKYBJtNz3pjOT41bDcRFUschpRqVpvI1kZePvuE9qZHqURTSWLoUVIQZcLo0QgWI9aORf1RGJMq51qUteQP/+W/4rOXD/kP/qOvc+++4ejsjD/6o+8RfE1gzbzRmEIhVE9RJHa7FWcnJRfPr1kUC9pVYLfaURrN5cuB7XVHqRr+zt9/jd5fUVYKXXiurs8p5zlvQyhJ0ZTs1p6DRUO/izx7vGJRlqwuI8+eXvGldx7x3g8+xnoYm7VAClad48FpRXVSsurW/PAnH3J8epqRYy7x6ccbfvy9D3nnzQdcXe14/z3P9vsf0cwa+t7z2qMD3v3aV0FmEa9tO0xd4MKaN15/jbquSATOzy8Zeo9WJV1rWa129EPL6ekpH330hLLQrNcd5alis3PcXLcsDxXWdRwdHnP+4iUpCapa462kNBUKxeVVT2F6xFiYNLrk/MU1Wuccj74bWG82LBYNcRdxYeSeD56Tk1M2NyvqegZRcrO6yTkBMaKUzGhKU+3NOM2sYrGY89njjymKsZBdO6ztcW5g3mS8bDdmr2y3G6y1VIuKg/qQ+8uHvPXwLX7rq7/D1SeWb33rd/jGt34TrKYxc/69P/hPefbyU87uHzCbl6j+O3x/9RcUdQ4IF3bD8nCR13Miouczjg4PuLy+JISMjer7nhgFoClrQ+vXDNajnSGIQL2oaduWft0xb5pRfPX4mFC6oI8R+p4UImVhUFWDso4hRKrZAhtg8IHje0dZ/yEyWzRYb6kqRXez4+byCj2uRUKqKJqCixcrTnHoZUOTFI+fPP617t11XeTrPN2uDbJGPi0k7q5dXjUGTevzV1GVWeCMnpw9ENPeaHAb2C72OluMGbEYYkTHRAiC4FVei8Y0GgFBqim7bMxSCvnxeCu5IUf+utivEcnO4gQ+5OBOSca/DUMgBsNiMePwwLLbObzLOJTpvJVSMp8vKMsydzkfHuCcy4YarbPJTEouLi65Wa1uu1zH4xZC4Ec//gn/j//y/86Pf/xjuq4jxsh6vdoXhAXke6KYsJGjjhhjNjXKnD0UJYS+Z7PZ0JTZUNfHiFJjPlmISJXXKS4FFvOMW12vrzk5vo8PghDdaNq7NR7u8bJjsWGvpaaUO6RIRJFNytOx34u9SKTM92+tIlKMuY8iG0/SaNxSOo1dCvm9JyPs3QzDvCSejMNypIOM96A7i899t/f+nGQU6uHzq2YxFeb3sm+6fW/x6utKJfZu+yn4dMLSRHIlSEwu8lGDyrjPcb/z6c1t97dCKIMK+VgLqUZxflpDirGjWO67NeV47HLnxZgzJRV/1fZLz96PFsUrh+gubiXevajhlarKXfPndNBhIukkpsCDfQ1BZBdZtvKnSbPdS9nT4gim9Nrbx/3dPRS3Vnw9tjffioSjE35028mQF0ZCCGQSObRU3GmdYZLh5N2zfJSS88mI3HtyIQboe4a0w+5eQnBUBZwdGAgCa3vWqw3Xq57TR49QWqDEjJjmhAH6tMvsd0R2qhOIAZJKmHKeq0VSILRGmxKXJFVZIYRCicSTj674zh/+Mc8fP2XnfQ44SlAqRa00rR4IKfH0YktSgjfeOEUrWF1eMvQD86MFTV1ypG64/OgHXATL6d/8j5mdnI4TrleVIkHO0apjDtlSCQ6s46Bfg10TQsdKK7wsqIVkKQtOjGHwAoOn1gJJAUWNUYYzWZNMw1oKNiFyvdkQdht0uKRtb7i4fsG6tcTllqraIk0FIRBlQ4hzNptAWWxpqh1ITfABWTQMQ5cXaEOPsxGUoZ49RDQnyOaAopgjbWCwPZQH6PIeffuCev4mUs3zzUQ7fHDE4EneErwiRkFRZ7dkmi/2nMx6Nqeo5yhtEEphh4Ew9LSrnhADoVuj3FN0+pRFtcRUx0hdECiyG2voGfoNSiSs8xhTIypDpRt0M2fpPLqY0ZRzDo5+TRFdTNzpXJQQTINZ7hjZ8+7j2Ax090b5BaL4XVTLL9zEq99MjtHJ6ZySIyWZedHREyP53ylzGUnZpSCQiGhQwmXH2Z1r83YfpoHpZxtsc/V1dGmmAmcz2mAwfb6chz5zwhcLlssFB8sl3t3Q9wMxgBQt9x88ZHF0yNB3uG7D1fUV1+trri6vaLuOkBJFWe4RDEIYUor7xel+bBHithCZcmjK4dEZx8ennJweo7Xk2bMnbDbr/c1gt9vS95amrlkujmjbAa0kT58+IyV49913MEbTdmuUTsznC0iK3XrIn300SNdw+axHqsTTp48JyVPWJUII1us1MWWW9vS5SpknD4yO8Ukcm5yj0/OUUlhr92E/Dx88IgnNo9deY73dYIxhs90A+eZfliWxH+hcLjAYo5kmHZOQLqVhs9nw6aefAol79045PFpS1yXzxUHGlag8Ycxor3HcZxJhv+C83N9bfvY8/qLtFUQMt/eJuz+bkDx3sS6fR7xMN30pVJ44jNdiHF0Ht/fLX237z/6z/y3LxYJZbbK4Y0zmrqfA4BymKPYTUiElWt7dv9vcEmDPiROAwkNyuYQWwbnMXSyKkpgiQuZOMp88KYbsWk0SHxMhCULyBB8g5kAeEX0Wt2IkxPy8KQx1jy1hlGq9x4UsyrdtfxtCGfI+eOfHEE5H8D67qX2gH1q2mw2r9Zp+sBnjog0hJpD5fAmjYB5SFp6ALPZaS4K9AJxGp/usmTGva4Ltubh4gR165vOaxWKG0TVeJISKNPWMqs6u63boQcB8Pqeqapz1DIMlJZjNZhhT4L3P7uZmxsHREVVZ0XUdgx0oipKqqkgx5wPEFKmqioPlQc4RUDmUqShKqjIjEqYCQeaCayKK+Xy5d7HnPBhBVddM4T/ee6qqQRtNWRhmzQIh5Li/A13XYocOSaIfuhywGDw3Nzdoo3n44CEnJyeYQmGU4tOPP6Frd/tQ1Iyucugih6LGmMWxlDKSJ7vm2YdOivH4p5SyCwn2YVSTE30KGY1J4rzHuvxfGSJK5/lqTFlQI0RUTKNlS40Lh5DblUeHuBDg3MBul4MVEwZlZohRBM7BU6MAJwQ+RvrBMThPEgLnLV3f0vUbqkpgCgHCst1dIkXkuD78Na5uGPqOsigpqxlNVRN9RIqQOyYihOhAKwZr8/iqTQ67TYI4eKKGqHIXnfMeF0LGN4kBTRgD6gRKa3RZInRBkgND6BFDDixcLBbU8xnlrM7ucCAFT6EThSaHm6mcuVBXC0o5w8aeAk2hCqQQOYRrDLmSAopCMasbBIIYMiLFqILgBtquxXmLwCCExro2B0O6QHIOmTJCKQWBTwnhIkZYVPBEb9FSUxmBkWUObk4CFSTe5QK00pJEpLfDuFDN820pNf2Qx0yCJPiERBF83He5GG3GoTNmRqmQGJF57ra3lCaj31xIpCgwuqaQJaUqEKbO+BWRIN4WA0tdMasKjCnouh5sQkSBt36cV2S8VvQRLaHUEuEzesi7zMhVKmNY3NDhbM+8KZnP5rlQmzISxxOxbiBYSxj8OA6QjSoxYXuHGZ31UirKoqQqGjbdljQEKlNRzYocvBoknkRVFsjCMDhL1ZRApDSKpizzOSrAmBKhNLqAetYQY+bIAxRFTd95un5NSANNWVGlksRAWZcgI4PvQZO7noSiSDC0Fhc9iYQpNVKM9w0biSG7ITvX4oLdSwshZtFVege0SCEpUkFwY8aDhqACQSec8EiR8aBlUSCEwHtLVYxhzngQeu/W/FW2w4f5HO6to722uM5z/40FKRnaTSL2iYv1lrPTE777o6e4AEIYKl0zO5RoExDSZTe7SngXSDG3FUsJPkQU4HxksD1CJAYHr73esNlt+Wf/7C/4vb/zVSyXvPYlBdFw80Lk80VmFFDvLEKUnJ2+yR/9i8cIt6BtN8gETZ2QJzl0udSCru+5XG2pO8eX3znEp57ObqmaisW8ZlYesa5WvP/ja5aNJLpEv8tdm++99yEnR0uqumJznQXSlARlpXntnSWvf9Xg5ZakPS8uL7m63hGGxAc/sRwt5xghefr0mqpeMAwOiUYJRWUqvvT2V0lcU5b5vrNYVrz19kMuLi757LNLvvq1t6nKhpgu+fCj5zy4f8p201KUFYeHZ1xe3FAYxXq9Y7FoKGud2bzGYJ2lqmsurzuMnlNqRUjZdTszczrrKQvFZ4+fcXxygLMepQxVCccnB6xWV6zXa8qqzK+pchDycn7Axcsrri9uODs74+NPnuZuTGXouoGyKAghi+MTH7vrexaLmq7dYq3n+OiIOIaEl2XBbNagtcrdrN6PxXWPkIJ5MyN0BfeXr3PSvMa9+dv8g7+5oGkOMcMcbQpSEszr+3zr3bcIfmC7fslx9TqPjlf0csvZ6X1e3Dzl8fOPKKpcgDWm4OLynMFZII24yy2Hh0fMDhdcXF3SzGesN+uM1DCK7W6DKQr6taX0eY5WVQVIjfOJ5ckxrnd0m5btZsfBwQE328hmd839B2eUpmQ5W7BrN6SU5z3LckbAY7TCqMiD04M8L1UCGz2r1ZaT1+5x0W0J1qIKgaqKX/nazvt8i4OZRPRb76j4nPcrMYWe7x8a1xJ3hXQQY1Ev4zr2IvrEoY7sTSgxRnzInddx7DKKYyBj8CGjXlTOBJpUrxQTMcoRK3NHRb+zT5ORRYys6+A8SkgKU+Nd1vyywaHDez92rQq6rt93IxZFQVWVnJ+fc35+zmq15uHDe/kzIZtvjCly/o1U1M0sd0eNx+Hy8oo//bM/48MPP9o/Nq0Rps7erusoitw5VhQFcQgI4fdIWTFmPnqfxtxEqOqak9MTNn2bUWfaoMZjhcg5Tt47fLAIwSieZyxL1hDDHfNxuj12YkIjZ3Rb7tLKIvKY/UpC3JJ+yIVPrTNCR4rbNTeMyB48gszdlxMDRshb39gdk5mcih5C3Oko+Fz20vS/9KoGtHeKfYGQnkX0268/Y2sU0zlzm2uW/eeCjAWKTFboW+vZ+DfsixLTYcyf0VjBQQrwQSCiHOfu+dyfzFj7gNT9RfjKx/HKtfbztl9aRF8WnukQ5AM6tkKPPfCfv7DvOgBzkSLv3e1B4HPPHx/OJa3xA7v7sNhXbOKdKlze5B3B/vZ9p/fL4YBp/9xX+MikXLUlMrGCELci/14oGf9vyiCetingId4RYITM7ZdaPkDHDf3qOYVxRCPYtWsuLi75yaee1ht+669XlKrBe8HN9YpYR1LdoosNQhjwHpFqopNEozDNAVKqzF9NCfxACpHWBYxSMLSczmf87m99kz8aLC+2TwgIBhfYBcdhY/jygyVloTi/7jg5bPjKu6+zXMzZdU+4vLiknNXMFg0ndcvT1PGjP/0jXj57xrv/0f+Mg0ePUFNYwJ2PTYdINXjq4GkknPiWU7GmExtW9PQqcz8PnOcoOoJMrI1iiJZEj1W5mlZqaHAYETCmwYQaLyu69pL+8jPWm5e0tuN63VGKDc2JRdiO5Aa01MyWj7i5Oef88slYUS8RMqF9RPvAgGO3DSi9oJzfp1ycoesDUtXgjcGHiCxL9PyQyj1EG4kyBiEGqjKijceogHADUiiEy20hUgp0YSgODvHzGUIkCpkTlAkO1++IfsDtOjYvH2O7l0i3JnUf0Cw8tbqhrpYIUeAj9GFkBQ8enwZME4kqh+5EbwneESMU1THCzAip+mUu41+wZTF0+vdUZBpLR/sb9WSSFhPP6I6o+vnti/jnv3CbKnDj1xxw4mBkkoaY23tChEgWVUAgyWJHQqFESTQ1QhZTSQ+YWpSm8WiU2O8InRMeQCkNVoyCYm7DCiGx2+3Y7XZstxu0ViyWcy6vrrFDj1I59K7vO+INpOho11f0IxrBjl0jwXqGIY5CWUOMU8BhDnJ5tSqc97soDLPZnKausMPAsyfPECScH6hMQT/sSBL6dkffDUhV4MqGvm1ZNEuaKjtQfu/3fod/9I8aum5LCGlsn1LUdYHEoSVsVztee/Qm/6v/9f+C/+K/+M/56JMfsxnRDn3fj5OC22M6FVA+j1PY4xfG5yqliTFyfn7Odrvl6dOnnN1/mJnGIy7CmIximc1mdO2M0GUHb+YyZ0RGFgE9SuWgV2st6/Wa6+trlBKYQtM01T73Iv9O2u+nGIumv3SBh1dvooKfff4rOBfBK+fZXSb6LxbQ5Rh6l93oOVRGMrKMRib6L3H9/JztN775FUJIucqeQIuEsxataxZZIcnZIHK6k4aRDehGB/BYrEpZhPAuo45E6Bm6NdZ6QhjZtdYjMv+JwXYMQ4/SElOU+Ag3qw0XV1f0vaMsqtyJ0XW4oSfYAe8s1gW6ITBYh3MWNwZcWusQWiKF2vPRy6rGlDN8TCPTWlGMHPQQcyChUjnEVIo8pqzWa9abNUIIlssD5vMlQkhMWaCVIQpBEDkUriqbLDgOA8Lka6GqaozRY6FIcHx8xOlyiW23kARtmzs9DpeLLGiLihRz4aqq6ixWix1ISV0vKIoC73f71lBjaoRQdF1P3+cFRlFV9EN2eMeYKJyj7VpSYu/IHqyl7bo95gWyS3kzFrimazMvVgWkMdA1hL3AClCW1R4HMwx2xK6UCGC33eLswM31FTdXV6zXK4ahxTuLVhKpM5vwprsmpYjRGqMVZVlwfHzMerXaT9yd8+jC7APoJ5xLHjvi/jrJJL/82YaQMoN5vCKBMRB1wlHksb7vh3z/dG50Shejw7DdF7aMyUGSfd8jyYs9o1QWHn1HTI6yNNR1g/cZsSNEoKgrqmpJtD22u8mIHpfZi1EoXPB0Q8fghrGVPhKCIyYLOISwpDSQ6DGlxphXcyb+bbdSa+ZNPfK7Y3ZqJ0/0js4FggAlDCG60akPUWqESoiQEB6CCwy+wzlLDBFvXV74Wk1IeWxTyqDQRJcoVEGqcqR3jIFiPmNeFQgpGHw+F9zgRtxGlR3eSMpSUZiaOED0idJUGKnHcHgDUuOJOQNF5MDLCbNRG43RBZ2X9G3PYHvirEbJGcFnK5LR4+JLCHprGWxAaBDKsrWOUgiiYAx4NRhToYTMznzI951giUbiREJLkMowrxqKak7bWpzboo0AoUk4lDb7+YMQ+Z5dFvV4XQcg55vgM5KsTZKqSoBGpIxciWN7vChVDjzzEQs4kTtmohSUdYUSit16y3a7I3mwnSVKiQxQFhVYRxocg81u/Sh0ZuLrHEDmo6MdOnwfWM7htUcP8dby4sULREhorSFYXMzRcUrmIC4tBLNqRvQ+s0JTzJipKCkxBFkhixwyXs8KpJZUUTN4jywSRSVJUo74IompcreTkILSNKhCYZ0nqcmvHynqfPzS6MwxVYEpS0xV4aND+YAqNTvX0Q8tqDwP0cpAkth+Ci69NWW44EY0nhxF+vH+rcjrRSGw3uV5htYU2hBdXmMVxlBUJSJGXBJZsFYCaSDJzD4POOpZRjt1XYsyJU3T/MrXtpJ5ZRqdw7WBZtawmB3y4skVq1WHDon7Bwf028iiKWi9wPYp/91JUxhJUQpCsohS4QaHHbIpRQpBVWkSkZQ5C2iTM77OL3ccHs5ZrRN//Md/zltfathtN6S4wwUoVM162xFjYDaveHD6Lt/5N+/x/NMttlvzxts1v/t7XyHEjvPn10hl2a09L6/P8QkOm5Lz8yuqek5KMy5XHnFacTlck5ylaRqshQePjonhhpvrjG04v7hiVtZUjabvMlYtaEF1CE5fEYQnyUil87WVvOTNR0tevhg4OZ0hpUQXMofSVxFTeGazihfnH/PGl+YIGRCioNALmuqI06M5q9VHPP7shrfefp3NVqBNyeAE2zYxI41z1MjNTc6dUUpxfX3NvbN7bLZDzoqznu3OcXp8xMXVDWUlOT05Q6AxOrLZWg4P5iShcG7Ahcjh4WHuwIuCYbCYIptulDJcX18zmzWkAEVdMgyWw+NDttuWw4MFq5sbhiF3UPVDjxizZGbzBe3Qo42gLgtODo/YbbdjF2nunjOmRpUF7TBQ1xV923J0dEC727GQc37zq79NJZdULHn3jQd0naPQFUhPFAGlS4iaZB03F1uOD874+3/33+fHH/2IYibY7FbURUndlLlLL9p99giQeecjMsPanhAdmoKTkxN2u5aXL68J3lHV+Z5Oyuu5dtdRNA0XFysOlodcnF/gXeDw8JibXYtLw3h/TnSdhSTo+i3D0KGV5upmje06vE+4oacqSl577XU+fvIZ203HbLlk03UjK1yyvb5mNqt/jTt3NkjdNeNM85f80N05fxZGJ1PJ/tFxLvN5t7BIYt9BPRlBJbl4lm/neZ0dQ8RFR0x+fP5taH1G46nxP8kd9TavW2MWVKd51zSWTmuaRH5vkUDUBUrovEZwlr6HrovYIePycvdn7iRMKebw9ibfQ99//z1+93d/l29985v89Kc/4S/+4nsIA8vlEmMMX/v613nzzTc5OT1jFAnZ7lr+1b/6Iz768MMx4DtxdXVF27bMF3PcMBBi4OjggKapCd7nTk8tSEpirWOyL03oxmwM0lxfX/Pi+TN2Q0cYDWTaZGRfShnbNgwbpJxxdHQ4GhLCXjd7VZie1qd3iB6jlrgX0AHkuPpMYqR25LKvknLPCRdpDI5N+fs0CthqErJHIX7SGZJgj9SeMiUZNfaMXcxGgpTGOXhK49fxidP5J26/v7tNWvGtYjCds/m5MTFi5KY40un/9f47meLYfRFhysBLghyWOuZNjfiWyO11IBhz9FSi0IKYJCFNf79gj0ia9n0SkIXYG8TzrPzfoRPd+PX+QOVjKO44925F89sulEkuF7dHb1/9uBUU8r6PzxkvUhFv2eoIcRscOD4WXxlb5PhWYs8oH2X3/Xvfji/54L0i2qfp4TFlWOQc42mf0/7g7uXE/THPP79zTIQcBxVAGYRpKMXrqOTZXm6xzrJab7hZDfzosePr3zyhKgxXVyt+9OElHzw+RzcN/71/8G2OzZIwQHIWKY8hVQgkShpCSjjbw9CS/EA7DKyvV9g+ixEHheb118/4++XvMvjAR08u2PWeYfDYMPDgbM7Deye8vOqo6xJvB6Dh7OyIrn3O5bMX3Lt3yPJkyTu7K55e9nzwvb/Apf8bf/1/+r+hmZV7tM/0x4uU0NZy4DpOhOdUdQi2bNmSlKfQNUYE6mHLjJae3C4cUsCnHYgBIRLW9QwyYvUMVG5ZL8oZfqtY3bS8/8EH+OQR0rA83lH7HdIWhJDFG6lLkiq42XS8fHnB2cPXKU1JEpH15gpZPEAUc2Q1oz68x2x+TGFKopB0KeGVJNUzXPQs9FuE/hjfX2N3H3NU3WN2cA9kQQg9iBIRBNFt8SIR5WJ02IkxmDKRgiM4hwwWJRLedUi3wV19DxG3VCbSlIdgz0mdp65eR8j7mFRQFApZL2h9olut8INFR0cMFtAsFie8/e7fxA0WG80vdyH/vC2NPCrB6OpjHNxGETCNnNGQyExGj5GTiP6LBfRftOWK+FRjham4lcYliQ82jwEpL65CSvixaj6J7ZnlnEBqUoYb7avhME0K4u1NK7/x+Ge/WgQwRuN9QQjDKCC6HFwlBKubDTerK87PX7DdbhCk0SUNzvY5vMJaipFP671lu9vgvMPHiI8eQqKqaoZhQGs9Thpug5NvhaAxQFlk1MTNzSVGFyyXx4CkMBVf/vI73Nxc8PzFE7y3Yyuk5ub6CimzC2W5nKG04P7De7zzzjusbq7pbBbRu65HkpjPC1xoeePRQ66vnvCf/5/+d7TdOW++fkQwhzx5fkHXdcgkR+Ew7T+76WOe3KwTp/gucxjS3sn8/nvv8e67X+PlRX7Ng6PD3FUQM5JncFm0E3LCCskxPX1E3YwL3cz4dHtnbVVVzOfz0cVr9vuYHfK3QnZmvqXbotAvvCQ+XyD62aLLXvxLt6nrXySa/6LrIR9HlXFCaNgXphlNA4L0BdfYL7v9N/+ff0qKgegdxMD11QWffvwxdVNT1Q1tbxFKU9UN1tl9OGXXdaPLObO4U0o4m/EbXd/Rtzva7TZfXzHzW402HB0eUNcV1nUYrXjttYc8evAQZx2ffPopP/3J+/SD5f7DR9TNDB+yQ0SNrpfdriWkiDaGlLKjNkFGB5QF2hhQo8u6mVPO5ng/4WoKmqZBa51F4H5AKY02uZDTu45i3qDcQPSZwYySSGVGJ8ndyW0iOJuDdZXCyOwaMip351R1xXy+4PDgEKMFu13AkghK0Qc4X+0ojGYI2V1rdy2i7fCj254Emz4L83HknAsp0bttfr61SKDttrx4fo4eA5CklCgpM14gZCSJGYsEPmTEVbF3dmdXthCCqirH6ykfK60MdVVTaEnfW+LofD9YLgjB03UKXxWUZcl81iAAa3Og5mBbumGXnW4uL0iE0iQEyihMEvTtisuXCWc3LOZzuq5Ha0lRVWw3LTEmTGmy+BwyB9u5sF9AOuvxPu9nHhNkHkftyD0fHcyJRG8z9knpEq01iTEEUEkKY6hG93oI2eNTVTVCjsVB51AqUpiSwhQMtscNAzHKsWBSgHAMdksClCwwUuHGri0fM1u0D56IJyRJUUBVCYbegTqgLJdoVSBEJIUWwY7FTFAUKt8vfo1tVpcsmpoUQkYBugFZaCQelwI25MKu97lV3rlEJ2PuDu7tGKAoCa3D9gPaSJL3eAT94PExZLyTNFgy9khrw3JWj2NVHv9C9Gil0AqUiNjg0EpQmQInNCEKhKkoTIVzA71PCJVzIETMbvDSaCjy4s12Le12gyLRFIZgEziP6yO2s/RDvrabGmTKKBWtNaLM997NrmUYAotiTqEM0Vqiyl01lc7O/aJsMKak1AW0G9qbFW3n8BaUUQitKLVhMTsAYdiFbJ6QYsQwRY8YnVJFkbMhxOhOzuNIRAqJUSUKgR8sQzeQgkSrIq8t1NiVk0BokTNEgqMPCUtCpUDvLarvESGjiERMucNUSApVUAhDKQx9EHif5xqWSDObUVYGXWgQkc3mBjfk8f3m5gXt7oR7J/ew3Zynz18QnaQQklQWBO1RSEptKKRiUVXYPs+LJu5lVVbMFk3Gr2iRXe8pZqyAFPs8bCUlTVUi0QwuIxKUrvbXKzKhRcxcY7a4GGiHLt9yfQ5hzuNXDhFThUZ5jSdghxwQ6nqL1pmN7W0cEWn5d511gEcKk0UCMYrooialYsS5JLQUSK9wXlGU5TiHjAgtEYXOPP1xDSq0wBQSIfwYLpvxnkUyuVs1JYxSSPVXL8R/3nYwr+hahzCauiipyjnPHl+yuWlRpUIkyde+8g1+9KMfURYSrxxCRAxwfT0wBHjzbQ0yIJIkhpKrl57tNjvrl01DWWl8cqhi7PxKHmRivd7Qd4KuTWgCZ/cbrNuyfPCQF892LJcHVJWirmZ877tPePKh50tvfpW33zU8eE1zfFbg7DFaH+GGgXbesuk+o2oESsNsXvH06YrjwwUyznnyeIuLjtIk7p0ec+8kO4Z1WTL85AlRCWZ1SRw8s2WBrhOn947oo+fofoHHZeyEEkgUrtM8/6zj7dfOOPv6gqvVM1zaoYxkphVD12WxJAJyzosXz3j46IymXFCVJ0RvCMHR1Ee899MP+OjDx3meojS7nWOxOEKryMcff4IQMJvldUJdN/TXPVdXNywWC5A9bbvh4OCI7bYljchKo0sIuQuoqTXOB+6dnfDT9z+gLKoRh1fw5Oklp6cNhSk4OT3m8eNP8+UXBUeHx7kbwAZOz05A5Hlc8JEYPF3Xcnp6xGq1IcaS1c2aelHSDy0HywUXLy/QSlGVBcH3+dwxGh89VZ27ZouioCwKBmf59m99m9Pj+xSxYdgOYHKhiegJ0WIqDSl3qVlvOX1wRqKn6zfcP3ydPq1ZXa6RSbFe3RCC5/DwBKEEPnpmsyUHB0e8fHlOVZWs1zfMmhpZFFzfrDK6a8hz7egVRwfHiDGg/fBozvXqBiki6+sLZHQYlQ2Sbd9SVLnDe7GY0+52GCUotYKoUbqkque8eP6C+cEx1y/PKeuatutp2wHrPWG95eDgKLO2B0fw0PfuF1/Af8WmTS7k3W63OTvT93nLa+S8juXO70xYYz73GuP6Zr9EF/uXubvsSTGhkyRFPf7sNo9pWttnR/JdKVRAvGVtQ3YxT28nYBSd5Uh1CBituLq85vn5S6ryiMFCCJKqmjGfS/o+0etEWSZidBSFQOJw/Yo3Xzvl6++8zl9850/41jff5dP3S6LSHBwsmc3n3H/4gDffeiMbW2L+A7tdx/s/fY/V9RopJetNj7W5OzV6z8FinpVhDzKAFIrVxTVmVjGfLdjEXT50IyImCE3AgTKsO8v11Q1SK3RZjos0nzs2yEEtdTVHJAUITGUyIkzk8vCkJwrYaxTArX6ZpkIIyPQqPgWRUXzZZBJQQpMkJJl1GRBE4Uly/GxERIxl6SwyyFGIH19XTudVgr3eI5mQEvvwTZEyJ3/EuO53SsQx5zKjdPe89Fuxl3Tna87kkDmHavyDM7hmqh5EUvR3zrdARgdPQQH5hXInf96PJHIHwF2qyShKIrnlpodxTb0/99Mk4MepR2P8/URCklCE9Fevu3/p6fvm5dNXxYQk9zt7V0bPSsKrQjnT49NR5fbnMV9x+wtvarkHUEIhlHz1NSZRXdy+9hQKll55L7nfH5Em8H1i5JGMQtot23s6qyfXwqvC/6uDHJ//0TQ+idv3DKnG6SWOEusUIcB63XKzaln1guve87U3Tujblv/yn/6I//o7j5kdHbO9eUofEv+j/+T3US4irYXKI2SJlCVCGKz12LbH3zynNDnc6PzpE95//yMKaTACzo4bzg4P+Nrrp6w3G16ud3TOc7lq8cOC09MZVVHT9gnbtVycZyxFKRNXFy9ZPT/g9Kvv8NbrCR8Fy4MD3vvsfVYvPmP25a/kiru8PRDC5LaS2lqasCKJHUPMSBukQqeOIoAKjt6t2CXPZujZpiuSXJOkZx1foNSCnXmTQRyiigWl0MiyodMzdkPBk+cblPLMmxq6l6j+JdIIIprQt4gYcO0Wu9vxPHSUdcO9ew8hGqzrmRnPvE5Uy0Pmy2PmVcNcgfQdWyoKrVmVJTI1JJkj55LbkpJDKvLCR2iMAusjKnrCzRbCkmQUUtUUwpNkwLmYq6x9Rwo2OzSjpTSW+khT1/cpZEQmC9YR4g1CLajqBhENIQyo6IiDxbYviEbl11AVsjqmaOaIk0cZ68GvlxLOXnge23qmK0sI9vEW4w08hhw+kkYh+/P64C8roN+OBV+wpRyw6+MoopMrsyHlRPBIDvYQY5dJCAKR+iyiq3TLlZ5G8LuFuzsC6N2v0z4ZU+7RBykF+r4dA0hyCKd1ltVqlVmFg8XZQNMsGIYOUxiMMjg7sN1u6PuOkMaWV3L4DyKNjN3suBJyEvezAC3HUNF9kKHP2Iz5rOL5syc0zZK/+wd/F63hxz8ZmDcLnLOUhRmrxnkhVJUlrz16iJSJ2azm93//9/n+977PxVVLM9McH59k4awALTRvfvkRH/odVxfPmM8URmvsGEiImATx7AoFMk9MqNExYl9xSEx4F631WCyIzOdzvvq1r7FcHCB1yUcffcTffvT7GSHgXRZFct8pSiq01vRDN7rcs2Cf3fu3bY2Tw6wsK4rC7B3tUgoSEyNf/My95ZfdXnGtf05zn9ok89eYg+/G599FuIg797fPv3YuGI04F6lIqHHiA1OBOPLqxPffdvs//h/+9/ncCpZFXXGwmHN9fcV2u0WaEm0KFodHnJzdY7E8zG4/o2nbjs0mY3aqKjv8UQJvA4NLbDrPajvgbG571FJR1DOa5UluaRSCsqo4OLnP2b2HDLsd11c3/Ma3voUpaqIQbHYdVT3n4PCY45NT2l3HT9/7CefnT0giX/M+BXzwKKPRosjzwAg+CXwS6LG1NMREcoHU9iMGJOJCxPqB1GeEVlIRxjlF2/XooqCpG2KwxOCRyP35JZUkOEvvLGZcSEqlxgAmSaFzov3Q79i6gfVuw+As3dCz2W1HTJPITnYpCJsdIQaUlBRFFrR3XQ5LymzuguRh62x2aWtFVZSEMDoqTJnzL+rsfrI2d4aUZUa7aK1xftgHG6oxU2AK28xMdbl3pGup9mn0d6/dCeUSD5b7a3jC/5RG0Uk4OTrixflzBttTiwapxuJWDCghqYvcOmptz5PPbrLrVEiGwWKHzOCUIjOvjSkQgizuh7h3PwUfM+4n5Q4ho3XmUO+Dp/I9Lxd9wsg4zHgepRRqcKO75rYYnMZgIx8SaizuKZ32HEnrcrCj1iUpGUIQWJsARVnO8nUfYLvdoMbQRak1LkLvAkJCUeYFntQFm40n9AqlCoqiwhhFtAmtEox5AcH/6qHBwNh6OzkiyUGVMn+OPkR66zMv1g4UpSaVhpQcCkmynuB6lCywvaPvB0yZBdEQIn5C6QiPFQN+5PabuUaQz22pEs4NWNfTDx6lBIXWpOARQmXH8tg94myPnhfURYErDJI0BqVFkg9Ik7sBEIogFb3PIdwqBqIdSEFlYW+Xw+5c5xhKm8MvZc7hcD4QETmM3UjqqqE2NS4mykKSkqAsa6TSyASlKSmMQYaK5WJJ5yx9GEhBgJRoWZIC7NpNbv82BaYocqBpUeRCP7mIjBakJLHWk1zuVKmamqYqCXG4LTCnLIY3s3kufo0da3EsCkuhCWNWCoIceIoiDh47DIgkMr5GSJpyRmUMGolBZSFt0bB1FqSiNJqqMHjvsLIgFgusG+h2az7+4D1KoThaHNCue67Xa4wpcNkmhpGaUhYsZ3OaouTy5QU7v8OLhDEFSRl8kqALUBbbB1ybcjCwN3iXKFOJsMW4VNJIn7m4QWmUzi7eFCNKGwigRIGPjm6bj1dZFCQkjkQQEq36LESKxK7fMtiOREKRHYTBgxs8dTnL96Ho6YceJSNa57lgTJ4QM0pGaY2LPrNQjSYJUIWmapoctCxcLkgqSQA6awkporQAGUkErLP7Odt2lwUYow0VEut+daGtlA1RbfF+GB2TmQMtoiR0mVW8ulnx4MEB57sdSeRi4jtvnVI1mqJyFHWPlIl2C88FrG4GDo4KvJfEIGh3FmESQkNVG6wVlGVFP6wgRoTXPP84EIeOr3xjxsPX7nN59T5PXrzk0YMDtpsOoxv+4T/8D7i+2PHDH/45//pPb6iaxGsPz3h47yt8+3d/kz/9sz/j8QvHaWmQOqHLSDOTmLLPnZWNpC6zg3zdrfng31zw6P59vvWNryLVgg8+fp9+2CFCRq4cH1WoyiKDw4fccSCFpi7nFGrO46trmnJJXZywun7Otl9RzRO7PlKZhqIwiBhRRWC97vny2SmlPqLQR6SguLh6zsefvkfvwIWIUAJTKvrBoaNivRlY3Wx5840lWhsuL9cAHB5KjKkQwuACuUirCwbr2XaeuspIt2fPzjlY5Pv5ZtiN8+AFy0WF1hXr9YY33nide2dLun5LjDa7aJ2nLmsKVVIVNbOZZrPbsd12IwYujvOLgpRyN5/WkvV6PYrAOXS56ztKXSEMROU5Pjqk3W4ZbE/ZNMyMwXnH8eEBIsScKSEFSghmTUMaFKTcteH9kIuAzFBGIk2iLAzWw2A9ZXNKVSx5eePQwnB6dMTlZqDtHd7ZvKaL2YU8DAO7XUddVwy2p7c9i4MzClVRliUP72m8c4QYaco5Tz97SllVGBcpdMG89IS+Y3PZ8vD1M1rXs92uWS6WqKogesusLGnqkr7taAePlBVd56nqA1obMM0cFwXXLy7wHpaLIza7jqPlKefPzqnLikEUtJv217p3K30rwd1Fs9ydj91ut4LnXYPP5ztr83zurw5EnLYYTK4kTaL4z5iHfmYX9s/7WaPR7SZUgfMJRA7wfn7+lK998zdY3+TwyJgcm5drbq7XrG5WdF0ipUCIA/cWB8xrw5e/9BZPH3/Km4/u0d1cs748p1QCypK6rvjyV97h+PSYqqnxIYuvfWf5wfe+x3a14vLFOUEltFZcXl4Qg+fs5IRgB4a+JyrL008+4sG9e9y7/4CXmzVqNFOEkOcR02chpMYnSUBRzg+YAjWDcDkDSimqYpbzJ1JHipkmIGSBMrf4m33QJ2OQ5+eO5ahnv/L9JDRmLMqr2NtstrvVMPZLxZTGxNDJgpU7FCZBWuxfPGUtfSqaiBziyWQ0S5N2Ou5jjLdi+V7vTbfvfKe48vnzIo0GwfFMfqVYJEfkzeT+zsbwgLhNIc3vsxfRBUlEooijPpDX0nsRfTxiSo1/VyIbtrlzDudPkIm5LoQYM+WytC5/iXX3Ly2i/+g738k7CuOHkqssmQM5HcxbRS0f2zGOcBS4b0NGJ2JxPpBxf0KIPeBfKUFVGqrC5JbC6QORkokZmw8i+/cQIjNdkXIvqE+veauRqdsTUIr9+06Yl9t9nbaponJH8BMwWe5vT4HbY5BMhVgYfFniUmY37q4veXn5kstNz4+eeVSpODutudn1/MV7F5ydHTAIRSLwne9+ysPjmm9/7SFNvSShiaIkSENMkq53dLsdabXGxYG1T7z/3sf88z/+CY+fnyOl4N5hwzffPOPeQYMgoo3GxcAn15YPnq0wOlHWhmZeslutuFxtef21N3j7rUfcf3BK1zmu15EvfelblM1nyA9fYhZL/MVT0pfeYaoI5YyDhI4RokPGDSle0PoNnpEjRYLYE3zP2u7Yhhs2/RVPLp8zxBsODxaoqmDrLtn6G/zsGF06KmcpFRggas2i1ty/dx/nPEfLJYbAsHqGcA6KCt+33Jx/jN1cIr3DD4Krly8plKJqaiQ982bDTFdo4yH2iLhlFiWN0DRxRhI1fUxY26LcDpMsJS1O9WCfE7sCYeagChQFJg2UtEg5oxCOMkma2GK0pJNw02+Rm2uE65GloXA9sX/OrFLM6rzoF7IiBo3rHbv1Nd5DiIHN6prLq5bBtZBWRKnRy7cQekEcOvD3CMNAt75G1Ee/7KX8hVvyY6KAGK9rxtBQkYgih+9mrrBARE8kgLlFZdzd7n7/s5inV575cxR08oBNyGI5Y2hvLs0CEZGmuGA5hlSIcRKWhe+7ouXtPt4RzvkiAT1vWhu09kxBn1Ipgh9D44TEOYspFNYN9H3P4cEJRZHb8mzf025uWK1usNaCyMK/jx6Fymx8O+xxJ9N/003hc58KEypEabi6es7h4UOImr6z/NZvfYuf/PTHLBaH9P0G5zxCGIJP9J3n7bfe5JvfeAdBnkT8wR/8bb7/vR/wj//JOSkN1HWNd9DUM2JwdLYHFVgcLEgx8OL5hk3Ycn19DUDd1KOoJulGbES+Fav9cZyE9qkdc/q7shO24ubmhq9/7Rtcrzacn7/g448/5uDwEKVzQIsPYS/Yl2WeREMaueb5mCgliRHsMGRG9JB5613b0c9qEBnVMN307046f5aG//O3z7vJX3WA3BZk9ucyr55Lr7jQPzfRvftvKSRJapAaIbIrL0VyPgduLBT924n/d7e6mqGUIPiCui45u3efo+Njnjx5xrZts5u7qCmLCqkUhSnQRqPU6JIeJ5GfTyqfihspZaa/Ntn9rI0hBr8P6PTO0w8dWgsOj5ZZpCxqpDJ86/QeDx6+Tt3M2e52PH36jIvLA87PP8E7Nzr8A1Lm69raREo+89XJIUjbTRzRDLkbJ2NIhn0wpfcea3PbaNkUNEXJTmmGtsMVJeZMUxZFdiV7S1lUzJdLlJaEFNBKc3hwwMHhIUVh2Kw3KKWYL+ZjkCcZ4VJXLOoZ2+02ZyCEjJKp6xptDN66XAxQirLMIrq1dh+WVJa5eGdtFtO0MZRFwbKZ0dQ1TdPsQ5XgLuuSvUM9puIOSzxfexN7fDpfJ4a4FhI75CC1vQg/Bn0Co/AtbvdRawqThevF4iAX73wupiqVMQjD0CMRlEV2xnd9i7WOts3oGe8jKSkkGqmy0H532LvtFrptU44xjWNJGifL8RXh//M4qT3nUNzipay1++MyCZhCyH3xgjGbIu7DpMYOIzfsixxlVQKJvrf0g6VU2YmqVeYwQ+7USkliTElR5MWiH8Oxtc7IGK80RldEP4b7pV89eBAg4rChx+gCoVXOgUk5CyYGlcPqe4cbBoQPlEpnlqbReB3pti3JZRRTFGC9xxiN0AIVGQtLmSUthUBLhW0V/eBZHC4wWuc7teuxfYsUUJsKEcku3JjPO9dbdp1DC03RVBSj4z26AaUMhRGQAtFGYgqEwRJcxFlHsjlmyw+QvEKmjBlxvcO2Qw7L1Qpvx6wEEtIUzIqx20JqymZOTDa7kEcES0qevt/Q9SAHSzFrOBIn7PqOSKIuKg4WS9p2y831Jb0LlM2MotTEFPJ6L+WAbSlz4HI+j0BJDUi0KTFlRbQhc+djDu2WMl8DKuUwzxDsWLwTSG3QukACtm+xYtgz0GPwxJBoQy4CykIQkiNZPwZrlywWM2ol6PoOER3zooCi5KBekoTk4vKK8/PnbG8CH/30BV96Y04lTkhtYBjzLbz3RC3RdUGSM7oO2htFlHOkltghsBkc16wQQhJCwlmNLmtEUZB8AGcRqaIfoO97RBwoiuz83l23FIWBFFEiUVclVgVE0JSqIaSBXbdFC4MA2jbPq5XQCKXHzA2AgFKS2jQIIfEuoLUZxz6NloYhbRE47t6/u67N3XuK/XibxjBVqfNC3kdPkJGos4jgnae3lomzPtiWlHIHYFmW2CEXomJMqHmVC6DxV793t+sum3VMREpHNSs5OKy4ucrmElMInjx9zDvv3GeeNEY5hAwk9ZJqPqNqEoEtZamYLZacnR1zcLjgh99bEXaB7W5LXWuMEAy9p+t6vINCBw6ONQJPsgYlDf2mxwjJdnfF628d8I1vvA0+8cmH53zw0RU/+Yv/N0Yf0NmOaiHYWstPV+e8/5MVSn0XoQPzRQXkjqnjk4qy9MyakuA0j5+uSCN+qKwNb7+74MVnV/zj//q/40tvv8HXv/5lHn/yCd55um7AlBoXO7reZ3ejVVRmwfoi8PzpS4YdHC8anjx+ydmDEicLkAN1VdC3kVlZodSA7fNn+vGHkrO/8TaCkiQSLy/PWW1v6Gzi0Wv32KxvUEoyNzojQLTi7N4cpQyr1QYhIodHB2ht8D6LMs7mLlRnHcvFnOPDBVoXQKRSkq7tQCTqpsR4jbU9QuZC6OtvvDZ2N2QcWFk22dDT1Ay7CEmyXm0pqpLlwRIhJV0/sN3uMCrhnUXIRFFoUhLUdUXTVGy21wgEzlmqosqdvEIz9EOeu6XI2b1T3v/wU05OD7GDZdh1PDx9yLNnT3n95Dkbtrx+7y1i8Hhr2XZrhKyJQWFKT1GXJJlz7ZAlu86yubykmde8+drb/OGf/BOOH87ougGpWpYHB1yv1hSFpWuv2ax3PHrtAc77zD7f9hwdHfPJJ48BweuvP+Ti4gKNZjY7wJQF292ao8M5JRCd4ytv30cVJZcvL7l3ekz0kaPDQ7abLS+vbvjKO28i64aLizXKJAK56GsdGGmywSEmijJnUdw/W9LtOq4udpweCVKUlPpXRzXBxJ3+nFmHW8PS3S2vbSdh9dV1zOfXFTmQ/QuE7S8Qu2PIRczxhX+uAAqjbrbXSm/35Yuem5JEaqjrGV3f8uabb2FtZLPpubxyPHtyw/nLFdttyzB4QsrIFEjM5jMEkceffcbZ2T3+/Lvfw1vHer2lmR/y9rvf4LXX3uCdd77CG6+/xXy+REnF8xfP+f5ffJ/v/fl32WzXhGCRSrBZX6J15Pj0mKY0vNxe8Z/8D/8HaCl5/uwJP/je93j2/BOq5THb7ZbFYkG/x4n6Vwoccpy/W2sZhj5jkBAoofDBomTuhPbe0nVbTHGCEBKp9B0z1C2//tXvs0Hortn71YLJ7SeRtzvUgLwE3a9xU0wkEW9fCJFd5Ezu9vS5/RmXqSORYI8pvSOkK1I2Me1l0emXXj0v0533uf1p3odEyHrNXsW+NbfsecHjPuXzMYxveDcDc9Rux9+fdGP26/Tb98udYLeq+bSWzn/WaIC7VaMZ6xRMGUd/1fZLi+j/4l/+GwodqQqBFmCtYPAZmRAFtD7iIihkXliTuWskgU8wRPABaiNYVBCiYGfzgTJG4KOgc7lSEYVAasHJvOB4Zii1RJFQApQUFDKHxuU2ionNPoroSmU+rtA5LEYJUGIvrjMuzrLjdgTMiywG7MP8hMrc+phbFfZCvZQgR1F++qiTuNXUhUAag1ocY96Yc1gLUr/FrT8hdM9JaY1LG3aD4eSwoKwj99/4Kv/j/zjxf/1//WsaYTk9a7haef74jz7g3YcVx0f3QBh8CoQkcCEx+Mh22yFbi/Q7/vBPfsgf/vlHfPrsOoevpcBuO7Ba97x5OuPwqCH4wGA9MQY2ncP7QEOJFFA2FcFGZmdv09ZnPHi9YX7yGjf6IVdHpyyXz/la82fsfviEZ9efsF3/NaLOragyBnSMLIeeeb+ibZ8S/BOS2oDRSLkgpnwTd87Tuw0+XrO+/pTPPvsY6wLptS9RzWfsnKNVhyTdEAePiCuMkIhhB7tLxO6G02rGoOH04B5FAnd9jnvxArQiJEu/vsK3a3CJtrXs1k948dlLzs4OeOvhIarRCO3pEdioSPaGQkdUXSKLQ8q0QO4G9M1LdOizuLx7SpHW+JsXrDfP0dUbmOoIoQz0DtW3pMIgqoowXGOMpdIgfMLtVrQ3T3L70waibwmbF+zMNY08pV7ew5RLnEtcbV7y6ZOfcnb/Idpo1ueP+cn3/pzjwzkHC4Mua0JIeCuJySEXJwSX2Hx2hZo9AP7TX/Zy/pnN+7zQl3eExzww50EtJo/znuAlMjm0CHduAncH979621dME+MAeifYgelmLjMDPWS3UBzpYIk88GWBaHxqCiAciIGYcqsyKt/wmcYFkcgD9D5R4Qtv/tNjGT8i8D7zuvs+hy4pna93ow11VeFsoOtahsFS103el5g5xiFmzlvwOVRFjeGNYRSppgCVyR06OUAR08/k6LzOjOC6Njw4W3JzueUvv/uvOX/2Mb/zW7/Fv/rjf5Hbo5VBioLgPc2s4u//e/+A5eEhU3fWo0dn/MEf/D5Pnn7I46cfMQwDi8Uxs/qQGAaePH1B2/UQIzIpOpedstN+TsfBmIKiKLi6uspcalNQltmdl4XWLHBNbD2tDWWp2Ww2SKn4+JOP+eY3f5O2c3zw/vuc3ruHVDKPt+OCw/m84NVaY+2wF9Py+ZNveCF4rq+vePDgASF49Ch8TQy/RBz3I4zO9YgQ6mc+79sTk7t3+/25ete9flc4z+eJZGrbR7waIvp5t8jnJ8r7f0+V2VH8YywWy5QIUZDiv72D/u7mQ3aTKilHRnPmoPqQ8A5AIYRGqYxsCCExDB1D71DSoFV+/+wYFBRFRVODtW5/Xmhd7IVbOwzjn6CIMdB2LV3X0tQFITpeXlwxmx/w+htvcXp2jJCRm9UVm82Wi6sLLq/OCaEjxiwuKq0oSkWICec6UnTM54fkfFSLDVBVueNhz23fhxWVd9zXAZkECoUkI1HmTcODs3scLg/2yJq6qpjP5pRVgdRivNZrTGHyZ/Xg/l5sndzeTVng5nM2dcNuNsucf53FnzC2+ZuxUDQ5UibBd9rvLF4LtFYjmiWjbKoyO9Dn83n+PMdJ/eRId87tsVCz2Sy3PI+PGZOLCxPjPISwd6o7l8XtCYtkjMFay3a73bdxF0VB27Z0uy6H8RUGZTQ3qy3GVJRFTW97fBjFcVmgBCQh8SmiTMHi4JD1esV2u2MY3BiaZjICxVlknxclWutx3z0p5nEjBxXnY9X3HcqUGJMF0L7vgSz2G2NwLnPLnXPUdb2fCFtr9wsibQxGCFzbE6K/w0aHYMP+OEyDwVSokEoyhVEJIbMDm0QKnpQExlTMZsvMZw+CzjnanWXoLSBxfshiRVIoaVCqYLu9YrvZjQWhX33zwrMbWsqUUNJgygZBdh6THDrF7FSXiQKJjopaV6jCEGwAKRl8oOss1awCGRFG5sVBAlNnHn4MIQexCvn/Y+1Pn23J0vM+7LemnPZwpjvX2NWN7ga6iaGbBESRNCl6ZNjhDwiGbH9VSI6w9f/4D7AjrG+UQlYorJBsmpIIEyRAgEAD6KG6uqvq1p3PtIec1uQPb+Y+59663Wh0O6Mqzr377rN37twrM9d63uf9PaQwsmt7iroAmxnCSIpeXPA5YZOlKRpKV2LQ+BggRRRZMl08kDw5CSOVqKiWDZFA9p7gA2M3EH0iBiQnKYGOiqaoyGTCxJ4f+xFjpnNLCSop6YwrSwpT4qzDJDHYeN+LS3pyeRsL7dDRjwO+i6zXx7imYVWVUgwyxc1agogmoibXJLOjOQdiVAx9P4UDK8qyoSor2la61WxRMIRE50dClLZvg6btO3LsycljVKaqy4nbi4T+jh37IHz6LnTSoaEgpEg04EpF1IF+bMle8HKusrRDR9GUFAZyiBTAsllTFAvQFb4t2BcO33quX8DHmyu0Krk4N7TdCEmRkkEbxc4GXvDyVjFaWsTDhFkRwUBCVY0roAJj5f4fgqLVAXIm+EQKHusCQSVxf08OvsJo+iLgiwHtElUlWQpd6ohDpnCa6CMpBvq8IylFs1pQFQ4fBCFlJpFC5nCGcfA4W1LXDVrVxAAh3KDncs4YqyElnFIYMsMwopwhZ/C+J5PQThGQfY9BsILej8KvjgFnzNT9UmCdwSUpPFtXEUI6zFt+ma0dPet1QyKBVnTRs37gsOvMonHEIRD2gg3cnkfOHhUcP9Sc3q+wriLTyz27rrBGo3Lk137jLt//y0tOzmrW0bDZ7lmtBCuRcqTrMn6AV08S1hhOTgrKReBbf+sRfnjFUsODew/50z/6gs8+vmDswPcedCCbFxS1CEqFaWj3A9iAzondJrA+S7y8SJzdPWK72RFDYnW3Zhw9d9Z3+PiHLeiRB48crs6894Hj/Dyw2X2K02vKInF8suTp854xJRSJk0VJoSCXksPw6knL/eMz8jKz3/Zsd9ccU1EvNONoKYxjzD3ayDUiDhJo2bYdf/wnf8rp2V3absvTZy8kT6s2bHcDu12mKg0xDDhrSB66YSSnNXfuLej6DbYwXG0vcJWl23U0uaEoKsDRtz1Vqbi63nB8fIaPCVeVxAR11aBtQluLcYG6cHRhK+KmkWO7220oioj3gXv3vsbQR/z2nOurS4yJfOUrX+PTn36Gip6isex2PVVVo3VBVSUWiwoY2W1b1usVR8dLnFW0XccYFd0ghilXGC4uL1BZ43tPYZzkFugClUdSalHGsNk9x+BQWlNWDqUzPlyRqQjBo5Rm9AOoRL+5JqTI+vgOv/bVb/P8/HOeXX0qc1ANl7s9YNDKUVYlp8dwcb2DsqI5PoFWOi7unJ3SNDXPnz8DFK9enbM+OxbRVUVyhHEIvP/oXaIfGMeRMmXoe6qyom/3VFVJsxq52m3RKnNy55TNdkdVFRwdnfDZZy/JcaSqDYtVxa7dgwqs1hXj6FkcGaLxaO0Oc7JferstQr7588tPljnN9Gu3xe4bcyqTAqgORs/XX+ItwviEpWN614Pf9G3PzX9DEV1pwTOmwGbbUroS7xX73Ug/ZPwI45AYQ2SMskZbLRfSlZQTRVUSkiaMAUNBvTjmd777bf7n/+R/w+ndu9x7eB9j3ZQdIB28P/zh93n+4hlPvvgc5yzD0OKHHUermug77MLx27/9G8TQ8i//4A/429/9bX73936T/8d/9d/w/tmdKffEHwoZ461csJTSIdheKbBWOr+YLMMpeawrqeqG4xPD+cULvt28Q86ROaB7OjpvEdHfPK7zGnL+++HI3gjC3BgF58fntWhO0smrDy8sqJSb8ZNvjaN8GHq3u6dvm9DyLVv2DXv8xgQ4i/CzdvR6oeemKiBm6Am/neY1sJ6kpBtSiBDDk+y3mvfvBu897wFM/PJ8s06eer+n17QHLUhw3sj7oSYUaT5w1990//8it+5fWET/4E6kKT2rxmAw7PeZ/W4KxUFx1Qe2bZB266zxgYkno9mOia7LDCFwcgZntaIfE+0WFpXmqLTsxsTmMtL7TJs122z4sdbi9lGZRmdKrahN5qiEZSlfQkjS0t1HSRpXSlEYhVWZSoOzEJiCGqMEqFmlMDpRGkNIGR/VYQBorUlKEVKGlCkKRVllYtTsemkhL60MnJBkYm+VEqFBK8rFkrsffsQ3Hn2bVVaY5DF6oKgs3/yNr+N+9Mf89JXn3kPHar3GFCO/87Vj3v2nv8eu92QfeXG+o2pK7p+9C3YBtgFqXFGSJu7yft/RnW8weeAHn77is6cXsjhXGasdCsX1ruO5AV2IS0Q4woEfv2j54O6So7Xm/r37VNWSr3zFcfT+V9m7B7SXe4xeY7or/uwHf8wP//AP+O1vfcDf+fbX+Ksn57z8+A+5zCtiytgYKcNA7loYtmz7F2j/DMweNFRujbNLUJl9d8VuvManLe1+w7MnF3RtIJxblkdLOn+Nd3vM3bvo0xqjHfuhpeqv2D75Ia+e/4T9tiVkS99ncmlpt5dcX7ySFHGb2Qw942bH2Hlpo8yZgQ4u9pz0kZd7haqgt09I7s8wuuBsteDu0TFHR+/R+0x38ZL9i8fo4EijJoyXmGKDT5doEjE2FI0stspiQdxHerUm3/2Q2kYwLeuTI+IYyZstut3Rtz0qKWIaUd0F7fiS/W7E3lGkRYSQaH/6GU9+8FeoDy45uXuX/PKC4sUe1QY4sgz5CtuMEB1Gd8TrT1A+Un4+kNIPftFT+a3b4EXMngMuZ0FXhDBpxxrCANmis4PU09iEdprbuuSb7V1vEw5f3/R0+wGIKB0lqCJbUhLHsdLxVsK1uHJTShKFqmYH8EhSmcwRcnnVgEGRpKp6u4KbRdR63Ul8e8uTk7oQp4WPFKUjxIGcItFLGGKOCauFvVsVFUZn+nFHDJ4QZOEsoVwaY5CAEnjNGfrau07CmtzEJrF/4t0Za1nVDTZs+fBeweZ6w3HdU0/BSD4OIm67JSntefe9d/nbf+c7lLWBHDEaVivHv/d3f4eLy5f8q3/zh/zZn/8po/esFwlnLF0/oqxwCHfba3F5j+LOLMsSZ0sGvAQ6KcHeiAtVU5SOsijJWWNtgTGWruunz6UOwX8PHtznyZPH/Pqvf5uHDx+y3G45Pz/HZ0lqjz5IcaLdE8P4GsN5TolHSciI0oKtkYR0O41ZcR8rfeNyFU50JOU4jTT31jF5q7Z967F5zNzMP28jW246CCSI9qZA9MZ5cPs11U0BZ55QMAvv8yRBTZ14KqOZUj9/yc0YR5qCw7yPdN2AHwfGIYBSEsqnrLCBy1IYm73gEpqmIWcJ1pVQQkNRFIyD4Hvi5DhXSlAcKsNyUVOWBdYaYoj4UXAo1hraruXlyxeMPrBYr8AammbFZrvl088e8/LVORcX55It4AzGapRWWGMxJmONQymLNUb62rSmLupJQMnTuHQHQXb+Hg7ubwXRB5KP1GXF2fExZyfHNHVNVZ5QTu7rGCLGKqq6oCwrkdCiBHiWRUFSSsKwfEArQ1U4dM70xlAXBcZamlo6N/ZtSwyBum6ENxojfdcTcqaqa6qqYugHcggH7Iozln7oaSd3dIxR0AKTIKy1Pnw3bdtKZoHWrFYLUDLxH8fxwEuPE5ZpxrM454ijZ5gYx7eLeeM4Hq7/81gNIZKCdOUoLSiJoqxZK814/oq2b9HG4FyN1RDjSAyCWCnLGlf05CzcYGPsIUQ1Tiz4oigO+wA3oVbzQmZ2ouv8+kT+9vl2+8+3z68vtUlrLQznpA/XXMiHzgUpjKXDsZpd7jM+x9qCorCoFBmHHd7LfUkrSwgj+33PvhuFJx6hdBDjILkVyWKdY9EsWa+OURQ4+6uFgqeciH6EPAejGgpbUjUVpc+kwkPw7LfXaJ3IIWFxmGwxypKm4FZjLXXTYBqFbaTgTM7COVcalZF8FxTDEFBjACLkSAwecpKOohjRxuLKCmdLjLZoqwlOisE5jhJ6NwyQEsGPdK0HqwkqSZhkAmccrimFkx9h6AYxblgpzqUghdVxjFgbqMqaorCEMeL9IOds8nSppVAlWWcCEWsNox8pCscwBrb7LWnqBmr9gFOFcPRdQb/dM/iILaTYIiChSD/sKW1ByCMpefZDotcDOUSMtjhjSdZODv6ecXSkKJ+rKCrBlimNDyPRj4KIshPSK2spTDdLkjdUTtPue/wYKcuawY9kBipbYwrNEAa0z6gIEYMqa/phZEwDlszQtqgAOtdcngdS6tlegh5OSW1Pvx3Z9C3B7+k6T06K0jq5VpIZpkU5IIHOIVJofShIKyYzU4ZRZUbnEZo7cg/GS3E9S6ZNzAORSCQJighwWtEsSvQqEBnobcYVmjAqdNL4kPCDFnZyziSdqKuKalkyxg6UFB+9T4L1GkfCxF7XVuZ/CUCDKQxpYtUXZUH0HpUUPkW6ocPhUDkIykdpolHy/j6QoyLmyOgHlHYYa7G2xBpLCCLe17Wb5q12wvj98gXw3SZidOLBozO6ccd+t0U7xdl9jVYei2b1zgl3j++RVMuH799BNYkwJvYbJEfDOK5CBgxGaTZXFxRVYr875/79Y8qmoO8HjE0UhcKVit0WfuNbX6fdd7x4+VPeOTviK199wNV2QGvHOESur1uuN3ssjn0bWa8q+q6nrCHFjLOaui5AJY5Pah480lztr6ibgpwVr16OPHhQstnshDs9HPPicU/KMGwhq4HjE0NVrzEEuv2I9yPD9YhxiuWqIowWozwKTYyw2/SQI+1+w3IhnSdlLWJMVZb0nSA6T0+X9H2HNoauC5RFJZ0XMXB+8YrlqsIVhq4Xk9j26oqT41PunN5lt72irhxdN1DeWVAvGl68eExMkXfeecjjx09QSnCUzlna/Z6zO2vaVu7fgKAkEFRkCIFViNjCYVPm5atL1usFq9WK/W5PjII6KguZE67XxxSFpe8HlM6cnKy5c+eU58++QCkp8BldId2f0mE4jp6ctXTalQ0pqWle4KcAQUFgNssFKct9b7WqyClRNhVH6yMe3nmEulT88K8+5sN3PiIPhrOTu2JuCBGjpSMga4ctxHmrDDKPqgqULhj8hv11y7e+8Zuc/+vnHK9PqBYVl1fSzbLZXPPw/jssVws23Za6rum7Hf11y/vvvs/6aM3nn32BDyPvv/c+5+cXhGFgGwPOWgyK7AraoScGTwyB45MTQAxBg4+s1mtOTs7Y7Vv80EpulHFUZcGrly84OWooy1OePvsc5SzOllOmjpz3y5Xj4mLDh195j91u+0uf2yD37lkcvP3zbduba4vXDUCKWyrmZDR6iwj+tkuRVofn6jcNPm/+PkoCLN/Yh7c9V/J6DMMgXVdNvUarBZcXT3n5csvFecd219P3EW0NmYDSYpi73GzJMbFaglIF4xB499FD/ul/+B/ynd/5Ls7V6Am9lcmHLtnjkyPe/+A9Pv3Jjzk+WXF9dU2KntViSfIe4yyls5y/esHZ6Yr/3f/+97Fa8af/7k/48KN3RM9Mid1ux9HREcDBHDTPj+fHYgy4wlJqKeBb7STUOiViGmkWx+z3Tzl/9ZxH755KJ/uET5yNV/MaZRZt5ZjKn8UxnV871q87yG8bHrn1nIkNnm9MICJiK1JSzG71268vIbG3RfTbAno6FKdzTgdxXnQXsRdoo96ybzc/X+sIn4xnOudbRYA8mf1u5vFSEIo3+z89Jvv3tvXwoaw0ifKzlJ4OQv1BYlcasnS1RpVQMU4OeTHKpNnd/v9PJvpHDxyrZUHdWHI07K893SLhSktEcbkd2W6lxSVnw+CnL0kpLnthhXc+8uGdxLt3NbtWQw/3ThynR5bdEDE+se8zl14TRs0rD1d9pCRxxym0M9QF1AbWhZQxQlSMUZMkew6tMxWJUieWRaIsNEMyhH1i0yaGlLE6sSpFjAkBrttIiJnSZgqjiVnRjgmrM+sqc3dh2PWKlxew6zPHtUYb6LxwsdeFoig11lhsrzFDR+0UlfUop9HLhzx88Bv07Y949sW/4sP7lm/8+n3uPfoKu5cfo9UV9+4ZzmhIueS9j+6jrKM6u49ef0S1fsjp6j7Ncg3Adrvl0+z5s598H9Xv+Pb7dzl/cYm2Gh8TV9uemBUfPLzD0cLyarOl97JgzWTOdwM/eHrJvdOaEEbWD+5zdPwurjmjyZZP+w3tVSRcXZJ+8IfUT/6Ev3z8R1x+/lt8/d//n3JUBH76p3/A0x98jzBeoXTgSef5SefJ0RPCnpgDSmmstiKAqMiiTvjo2Y2ZmDVdOxBTZvv0E4yW1vmEoVg8QVUrTE7oYYeJg/Achw6dMwbDi+dfYEsnSeY+sMkZjyIpjfeR5CMmTYnDJEIf+LR9wvmnV/RZs0sedEbnyNIZTpsFZydn1Lbg6uKc7faaHBQpiFBkXKSoEnoK1FVTOz8oxi7iQ0arkqIqKEuoVgucdoyDJwRP3/aQoXSG/X5Hyp7ubM/1yQUoTbvdcfnign4YePxsy/XxYzbXO/rrjmRHxlphbWZHh1GOEDqKxpOSor30Mv5/hW0YhAVrboUgKa0w2gjzLo0MY0/MGqsKbBGIacJ1GP0LLRPeelOeXLjzHGAeoyKWi+MqTy7iNDGzchI3pxjZ5TKpchKBgLdNHmYX8HzzvxFV3hTS5+fNgoox+SDivvfee7T7Lev1muvrDdfXO7z3aGXY7/e4wqGdML211uLYXSzoFdLmZQvmrrlZGJtFo1mMnX9WlUyC5xt3jJm2G3kRPBsjHOTL7Y6PP/kp1pUcH5/SdR6y8Cy/+53v8P77j6aq63wTUjx8dMZ3vvs7fPHsKRcXFzx58licWWUxHSk57v0wCHM1y4RhFuzmNHnvPSlmjDVTZ48BNGVZEkKgqurDzbMoikNA6Oeff87777/PH/3RH/E/+Yf/s0MY6OVmw8tXLyeBacHY7bi+vqDvO0LwzOGwxhhyjEAUh5UrqGthJarpe5sDTsOEcohRbg6zgPfzxuTfdBzfdqMfLBq3njO7lfVBnMuHx2/QQjcTA6WNZGvMMxrUoZDyy24hiuDgrEHnNAXsjMQsC2vJ17RU9YKyqontDpnrSIeWBD6GKbhWrg/e+0mo9YyDJ8Y8hV0JBqAsRbgNXrqehqGn70auNld88eQLHj99yvPzV4QIo49sdy3D6KXgEjxnRyuWzRJbGMIkXmtjcEWJ1o4QMilISKUrKmLm4LxeLBaH0N5hGA7OQ60KyczwEy5FaZy1WKMP3OCiELHLR1mAGS2TvX4Y6PsOUAcW9zD0spAF6maB1pb9ZkuIgaIo6RG0jGADItF7nHOEGNntdsJHHgf6vWMYh8Nrdfv9xDeX/c9A13UHh/mMMZmLBTcBu4Zh7F7DmcyZCvM4vS0IW6UPTirv5Rg75zg+PpY8gr6nbVvKsqQoRAwdxpF9u+eokud0fY8PkdFHlNEihpNEVJ3YhtL+Wr9WPDx0XOQvn08StJoPPHeYugdvIVvmzz9fv+eg5jkTwVphSs+i+CFU1XswZkLamAPqRWtF4dwBXzQf06oqqKqKGCP7dpD3tZVw3FNkHKTDLni5Rgef6PuRrhtBmWl/BBsS/ECMBYW1OFuwWKwp3OJwnH7ZLXmZpwyDJxpNTgZXlThXoqxC65KRgZi2oDUqJ7Kf8je8pTAVHXuq1ZJ6vRQBXUd87DFGkWMmKiisnVjQGqMSZdIQR7IHvLB30VC4gnK9whYVvQ8StOUKmrJBM+BTz353xdD1FKYQxElI7LqRbDSxH0CBLitMWZBNInRegraMIigRYTvfkxBfdMqaslyg7EjeB0Lbo4tEn3s69hS2JDtDuaxY1I6koQ+Rru/ohpGirrFFAUbwcDEH+jHQdhtMSihbyVxDS+uxNRmSx+QBrWFQgRhHamPQORDHRFAJozWWQNzLdUH5jEpQLCSzYBwHgjHEbMiuJCAFz8JoCJ6kAov1kqY+ZnvVC6s1XeHtiNUOnwMp9hRKYa0UDWKfGWPAeYVGsdsMbIHt/opur/GDIw8FYQ9+yIxDxo+RMHriOJLiTdFWOrlkXiVFMSkmjNN9YC4k5zwveiF7dViMAqTJ7RZTnLjmyPwtJ+lO0BmvFdFn1BjRNqMNGAeZghRkDqqTcMldhORGwcfEKBg0rXHOMI5e9l+LiDnGnt2AhLtP3X5VVePHkUik0BZPlEA6nVGlJhuZa1auQSkjwbw6kXScAkg9xsq9KGWwRYPOiqHbo9FYJ0W6rDNZCQ7sl92UsvRdZrvtsKXi9GxBUcPJqWNz1UkmgB84u1OzaxtGn3n8oy2Pv9hz/UpwgraA+48aQLPfX4rzvlAoDdebjuVSEGybTSsZJCFSNxHMlv/t7/9d/sv/8gX3HtXshy394Hn2+TO226f4cc97HxbkUHLxYkSliqKwtPsNrhC8YkwDy7XDFgPoyL27cr/pux0PHjTEOHJ13bNcFLz36F1++sPId77z22w3I3/0R/+Wo+UKzxQkiRT7wjTX2u5afuc3v8YnH/+Evg8o56hKRV8oYdXrwOk9QaCMPuA3njl7ZhwlxDslKar1faIsDdaUhBzouhGlNScnJc4o2t3A2AVePH1KU1X0+4H1suHJiyviyxcslhUhenbbduKSe9brBX0/sFgsKVxJKhOb7Q6Fpus7Qhi4e++Uly/PAWjbnqpSNE1F3weaRkwMXdviTIU10Heed9854/HnT1itjhDjkeb84hUxBPphpO9HVnGNVo7dpkUfFVTFQowJRUVdTTjGMZGnY7vZXPPg/gO63hMTDD5O3a2SKXC9uea9h+/zjW9+C9VrHCVhzPTdSFlX5CyIndPjU4qiJOYgxpks2ScpBC6vLlksC1b1imgUp0d3SfvEerFmt2uxlWUcArtuK9iOVQ1ZroXBZcaw44snl6zWK8YwgFKUVcnoPWEYsM6y2+9ZrZYUTcnjxy8hZ1brExHJlcaFxHazp6wW3Lt7j/3ukpQiTVMxjiN1LQXtH//4E4pKEbxmcbRiv98fBPOTkyOaxRm79pxdt/ulz23gMJd7Ex15cwG4/ce3rwNumwbmn2Ka+TJXfV5vvvn7t9/j5v3f9n4iRh6QIbNw+ZanaqOIKVAvGmJQvPveEd/78894/PgFT5/u6DvwXty+wsLvaBY1/TAweumcPb+85vJyw9e/9nX+j/+n/zPf+OZvEHxC6TkIdXJUG7n3Ga359re/xZ//6Z8Ifkkr2j5ilWPZrClLQU2WRc0Pvv8jnnz2OVVd0rY7To5POd/GA6pl7midDVx1XbPZbOi6jrIsuZGAZX1tjBSL/DDgCks/7PnmN79OsyhZLhty9of59yxwz9/Hz8L05NlZ/qXvezr6WXAnc2f9QahW6iB433SQ3aBObjuu53yxN40n0h1/44h//c8SXjuL7mnClmdEyJl/otSEp8mHwNq5Q30W5WdfuWAa0xvjT3Ce+dCFrm7WD7xeb8pTQUpM5xI0qpSS7Mb5MXVjlJPPK111TAQBVCLPZjqlfyHz2i98d29qR9NoykKCt2KlMNlQlA6foetHUpEJMTN6Qa9MRAQKLbB6ZwyL2rBsLGFMlDpSW82i0JAiJ0VioUGPmqukUUG+1EopjgvLndpyVGbuLiPrKiHmOsOYFFZn9iPUVrEoDZUONGWkLg1DAJLCe1jkTOMU69pSOMN+SGSfiDqxrCKlgxA1mwSlS3xwYnl0v+Bil7m8jDBm7jTgCmgHxTAozprMcZ1xWm7ey9SxtIHjZUHQZwzxGzTrgtB+TlM1nDaer33zW6iyoHAFveowxRJrLG5xh6QrkluhT36N+uhdjtZ3qZslOSaGfuR0dcyD3/277J59xo/+6F/QuMRvf/UhOcNiUVLXjiEktv3I0/NzlDWk5AWvgyzZ+jFwue14lEbc4oj6+A6r5SmXF3t+9Gd/xg+/92csXMaGVxw3MpH+4t/9a374R3/J4u5HFMpQ777gqLjGucj5AFeXgTGI28SnuRVCwirvrw1HK4Nyin4fuNiEuV+DOPl7NGCVxu+/ICkorWJZa+raUlaKxZlj4TQ6JkbviUqcezFZnl97fnI+0I0RrTRGKaxWB8JSCpl2O7DbjfRZs4kKn8GpzMoktnbL08+fSmBPyligdHIB6INc8JxRrEvNolZ0IfD50046FjJkNCHJgmGu5Mkl4KaSaBWCS0GKv8WTa4opSDHlOWxNgerg0wtxiE2LE6OhNADd4QIg32XG6rfdLv9mW9v3WB+EU8v0vWlBHkhwzEg3dsQMhXaUahQwys90mP/1m/zqDVf95v4x1Q+TCGw5ezJhEtHlO0mzGDkdZ5UTyoQDtuHNYv4sjs8/eXOy8vqzXxNHleLg5Kzr+oAOyFOgZNsNVFUjLc1Zqu9FUZBjpHAlg+rIWRF8JN2qDt+e9MwMO2uEBV7XE2svZ6JOdEPPfvCQLXZRY1RBDIoX59fYssIYR1lYyI7TkyXf+c7vsFwuDkcTbsJWT06PeOedd8hZkBxdK26Ytu3o+5a+HwghEWMiJUVdFwdxaXZ8z6zhMHjKUjiXs0Atxy0cFtjzRNEYw2az4bPPPuM3f/M7/OVf/gW/9Vu/hbGG9XqFD+OE83hFt9vR9x1tuz98f+IOzUAi5URSgbYV5rJSIm46Z99SSZfP//aug9vj43Unhhz+L//Oay7y+fOSvzTZmcfQfB7ddq6/hndhygtRBqUtiSDfVxY0mlI/e79/kU2aqjKj91SFCA5KC8IlZXHrgaIfemJOE8qjnMTJcWJCO5SSe/92uxd3aGY6B2RMSxisoyhKrHV02x3FJPTmaR+ur68Ft5CTuBoxdH3PGDzGWgkDpKFZHNMslhij6IaOEMOEEapQ2jHHxkIhrZ+T8OknoXrmjXddh1JKuNdKkbwnhUSOiTCOdPuWzdUVcRxoi0LCeVOibffEGKjq8iBUz07kmb0+nwNGG/woeId5zI19x9iLoG21xhmD0Wpy7GaOVktynsMgA3VZcrRaHcZIjJGlabCFQxs75R3Ie8+FzjnMV8a9m1jwEm5c1/VrPPj5GMxCfN/35JhoqlrE8MnlPjvc53E7C9JlWUlRM8eJDa4OoaDNYknVttP+TPc6V6K1BZUmlEnCOIcymkRm8B6tDU3THD7TvGhRSuEKK+GxQQo3zhUYZyfB9GZCPQvzh7DUSUhXSlBc8zF2Tgp5XdcJ19vqm/DZqdBTOPtaUeI2cx1uBMXRjyhj0SlKl8Tkqg8+EFMCNGVRgnIyxlXEOk2IEzM+GlCa09NTUoRx/NUq4MprQZ+EkbKQzpNow4RCkbE++hGfI84VpIzk1Uw/Tek4unNMUdUUdYF2maw1fdvdKvJmfI4kDT7LuVeWcyCsJ8XIMI44Z6lWNfWiIodE1+/pxkhcLCmtI03iyhhg33kGlUAZlHUknxnanpxGfIzEvqdsFlIMUFBVpZg0VCTpTFABrMbnEVUsUU6xubqmbztyzAy9hAKnLB0fOpfYXBFQ7NoeP4zs9y3ZaGwlHXF5wn+NKeG7jjQOlMbQDb0gaZQUjDWaPLFkbSni3dhKN0ahpHAzhh6UuCO7viemjNMGZRzOSGHK6gLVC6huUS2IKUgYvQqMQ88QOpaLNYV2VE1FigkfA9oKmij0kgMTszjNYohYJGAroQk+M3SWPFq6S08YNMkHsh/J3hB8IHjP2A/4cSRMxTQPry3Ah2E4nEvG2MlV93pR+NC5otRhjjOfp/Pfo1JEZkdbPAQtBxLjMGIGjSutFKS1HJeYB3wYKJzCFZlqoamPLVdXG6okAa9FISgVYzSqsrI29F4yVXQkDB0x+ENRzU0t+SlDPwh2yRUG5wxFaYkxTcVSg++EVS9dV2HKtJGCY+EqnHVkHyeDjZxvxjkyUGrNsv7lO02O71i+9mt3WR01DOPAk+cv5B4RLX1bcHUe+co7DYul4FHCpuPyaiDkkbJxDIOfckSgbjTGKun+jRKg1nYDm60EcT98eELb7sgMLJaamC95dfEF9VLxwUfv8Or8JZ99ds3Qy5rl9Eyu8aWr6drIk093OFtAtvgxsj5a0Q+Jew9qinLCFZqIswWrVYGxmsItUUoEt27c8bVvLvjhx9/j808vUMBud01RjViXSVHz/EXk0Xs1SQuC5fmLV3z44QM+f/wFfm+oKkNZGjKJdtyzXKxYn6xAD2hjCD7hfWS5XNC2W5kvWE1VlIJanAwF2+0VRS2YuEVZQYjUtaPbj+QMv/ZrX+cHP/g+VW0wbon34FzBixfnPHrnPm27oyxKrsM1ZWkYhhGtLUbLWA1TQSrGBNkw9CNZK7bbDfWi4fmzS1LKrFcNTbPg6ZNLHt47pSwN3//+T3j0zkNQGVc5yYKII+WiRBfSfXB5ec1isWK76UgRmdPHcbqXVlxeXlFVgvWzTtP3A8M4UpQVeYxyjwiBrm05OzklxsjHP/mYM3Wfk/IOTVMSxn4yVkWSymil6LuWnBLKOMFt9ZIfcHFxSc6wuQqsT0o+/ewJV+dbgsosF2ti+IwQAu+8+z6fffoFIQYerO/hw4D3PSGPXG0vcUXJfhCTXsiRSJZCpZa51/p4LRl03R5TWpq6IUaZp/oQ2O9b7t29z263J8XEcrng8ePPWCyW0hUUetbrIz748AGL5YLHTx6Tc+a9d9/l+Yun7Pc7NttryVgpS6rqV2Oihyz3fpXVJPSJ03vGPM6PA1Loe2O7EUhfF+J/1qb+mnyG+f0P7/22f3+bYp4nkX/6p3k9L/MHwSCfv7zipz/9gouLLW07Mg6aEBMh9MTRoyc10lpLURYYZdlsLvh7f/fv8R/9R/8x7334VTbbHdY4CgdGGbQREgYwFaUDDx8+5Dd/82/xl9/7HjEkQVSGhDWSWUOGcUwUxYqQFNcbTwiKmBI5KySgXtaq8/4AB5TLrAWA4F0lQ01EmK7rWC1ruv6Kr3zlG3z7b32FajlSVhqyOXw38/c2z+V/toh+8/iNgz3feo14I+Xfcn5PHvM3XkPmcbMTff7ecp7ut7fwMRnIaXbJp6noPe/b9NypijEL6jcmxS9/jvn+f3stnlIWHv8k8IOed5zDH/SkMc1j8TDEvnwuzAgaEekFu4dSaDPn+82CvXTFpQRaSSERlVAJTJ6DXucw0/il93lz+4VFdOuUTFKshRixRQKULA7GiDHgnDjXuiFPrehyIvkgYowiY7WeJqBZeHExkpJUPYyGyonTQKtpHgVU1rAsDItSsygzpU0Y0cHQWpG1wtmMC1n46QqMFreMmsA7WimcAWc0qzJTFUrYiFGxKkT4XVWZqjCMoyENibpUrBvLcmElsdmIW31RaRaNpnGZ5OCoSSzK+bMGUntB9+ILjr/79xmrYzi+C9051q5Z2kcU6gn18TtkY9HVkmpxSgoFpnoHs7iDzyVh8Q62eRdXHlM1De1+y77tJ35aTVmv+L1//L/m/MVzzv/i37A8Ulxv97y6vqZoNRf7kedXO1xVcHJ2Sk57Xp3viTlTmMQ/+MYDSD0MnuuXX/DwvV/Hh8j1xSu++fWvUO2v+fj7f8ZymViVkJNmtJlhu+XVx99DG8dZEzh+J9M0EHziWRhhYApPgjBVCq1RlEZxd+1QCl5dQhrEvTJXEzVgtaF0M187cVQZHtyrOTmpWNSWVaUpdSaNI2NvyErY+T5IB8LTq5EhZpwWwbucBKiUJdxxzFLBNBpU1oxBkotrC/eXmlUBajq5S6M4Wli8Vzy/DnR9ZlEo3rlvuHNWcbVNpMcDV3vhW3chsg2JTvQvzOQenQUsraE2itroScOVkAYzha9EL/iOQJ72GYaU5Vwmsy4VS6exGvohESZERuUUTaHpw68mtO277mcujFKUYJt+7Akp4XTJopDwpLc7v7+8vQ1fIhdEER1AAiC0KcRBHGH0ga7vSdkjCBZJYc5ZiXg3sTm1UnK8k3CAY4wTS3B6m8nxeFs4v32N/5IbeXIFO+sYtLzmbrfjyZOnvHzxnMvLS8qypKwq+t6TnNxIE1mEIxR9J67SEKK0Y2orl+QJ2fDmAlOEVhEcq7KW5+fMMMrkVri1ssCNKHQ7UjWKpUs4Y9A5U1UOa2oePXqHr33tI0FhzC50JR9aK8Xdu8fcuXPK1dU1zhV401OUJcNYcPHsCTGKA12clObAD52dvTNfea5Ix3DDPnfO0Xc9c2jfHFI4f8YZP/H06VPuPzBcXl5weueUttsxB8O+fPmCHKT7JITX+fHWGuYgEK0UV1dXPHnyhIcPH/Do0YODAGnMHLgiwvhc3Pp523wTPtym3zZm4TCZuRkrU8jhrQnS7U1r4fHerqDf3m7a8WTcaaYk96mY9LbX/JtsWovz2GiFKieuN4LCySpTFA7rLP0w0PU9dV1hjKVtW4ZhOHwG6TK4YWtXZc1iuUTtxaFtjZvCL+dQNxk7RVlgC4fvPUpLkGNCo7VFKYMrClwEpYyIP1ERgiLjRLTS0j2mjSNlC8mijMYoTc6yAMzkg9N6GIZD23RRSIdFSlKkNEpR1TV1U9O1LcYajo6OWK2XSPK7IAaW62YKCBMX/mJpKMsCrc3kChcu9izieh/IGaqqwmhDiOEQyDk74/u+x3txy7hCkBE+BLpWhO/lYkGMkbbrJtdYQVXXZDTDMByc9mVZHibTM+N87pxxhTu4XeZ9uz1Rv13Qun29nxcKOecDzmU+50IIpNxNIlnGFROSZujZ7neknKiqCpRinMJT66JA2UxK4lAWFJDF2kKKbzFQaMmkmUXvORfCGIOzTkJpQyDGhHMTBitl0i2W/FwsmfcVOLyOCF9xOk4Suqyn54QohdacpxwK1AGZA0yFEjt9//5wDsAUXDt6FqVwm+u6xg+ddNFoLYi3rIlZM46eooDloqKpZCx2fYfKEWeaw7X1V9my15hcUDgreQbjgMaQYmKMER+k4OsKg64keDoXGm0t5Ig1BevFAmWMMMmVjAdXFGitSFObtXGG0AX60ZNDorTFawVBYwyr9Zpm0RC9Z2g7xqEDLwuaODmAs1IkHCEb9u1AHKXlviwNvt1jnRQk9t1IQqGXFlKkLpw4LaNgilQ58fNzpM89u2HLfr+HKYC2G0YCmURkjCOMCTc2mH3L1XaLH8Q92SwWtO1IsSxQWDluvmfsenSMpOAhyILfGodKkEZPVgmfRxJSSNM5E/2IXSzEQdoHhjEyDh2VqzBZkbKiKGuMslhT0pQLaiPdNj4IUz6rBDbR9nv23Z6QMo1LLMo1WWVcWaDLkqwive+FO5wCKSsUWs4zHNo4xhEI4NuC2BqGXU9OktGSknSHRS/hgN57CXGd8mb86A8dKvMmPFgORb3bheL5XhlumQRuPz472xKTE326Jh+CynSGMTJacYInlYjZo7SEibu1oVk2LI4KdOmlq0IZmkWNcxKg66y4ibtuJDtH5iY0T1BvdsJEucM11PsguAvSJIg4rBUURtcOdG0rxeUgnVUxBsZhxFlLVVQkL8x6jXD6cxYnvDKKWinqX6GT7Hf/gUPpa/r+AlM6zu6UbHcj+12iHxT3zt7jxx8/5eL5v6YoFbaC0zsVttbouCDGzNVVy3YzknLEWMMHX3nIux+u6YZrirLkxYstMSbOziyuOOP8xZ6vffAt/tUf/is+e/xT7t5f0w0DP/jhc5arhjuLNVeXF/zm3/oqDx6dASXO/ZjLy08hgs6ao5MV9x+s0bak7S4k2yULR38cekLQ3H9wRsbQtgOr9Zq//IsXhG3k/GVPaZzkbex3KKCpCrSK3L1nMGbCRqnEyxcvaUrFcrHm8nov3ytSVNLGcr3foFvFauHIaSQmCCFzdlbQ9dAsKmJQ7Hee06NTLq82LFYlWhuCH9j6kVIZCuswCu7du8v55SWfPf4cVzn23Z7j0ztsNi0gYvmrlxcslgUpRZSWjrXj49NpTiX30UeP7jIMPXfv3iOGxGa3RRnD8fEJwzjyjW++Q9+3KBLD2HN6uiBPY9AazYuXz1mu1+y7jioLXqwfBjKBMYwU5QKlzIRnC3Rdx9nZEZvthrpe0zQ1V9cb7tw5ou32OFvQtR7tZK58enpCjJHFYkEKQbqnQ+QnTz9ht26pLl9SUeN0wVFxhJkcnikGum4nyDcfCRmKoubRo4e4UnNx8ZJX58/45JOf8t57H/LZy0+4uLiiLCv6YeD8/IpXFx3f/MYHlEXN2Pc44zg5PabvB7q+w48tTbPAh4F+6KiriuCls8A4w4tXlxSlY7FacH55wd37D7m63KDRLFYLXGnZPbtmvVqw34mTXE3f7U9/+hmjHyiLgq7fk1KgWZS8fPWcYehpmmpaVylQheAxfoXtphs13+iCkwFIFLN8yzn717zGrZ9v237ev01PEFF00t7S24TKDD8PcXFbyNfaEA/mFs2Tpy+4utoz+jh1w2ZsIQHdIQ0sFwv8MFAWFb/+jW/yg+9/zH/yH/8n/KN/+I/Ybnf8j//jf4/WGj96zo5OOT494ej0hLOzU7TVh06pDDTNkpPTM549eUZM0qmWseQsWMD5vJiXd/IcwXgczBLTPHjOybmZV94UiOf5hzGNzIGTYGyapuH+/XucnZ0S8iUKj57m27fvibdf68YN/rNF9duPiQv9Z4nXBzX6NRH9TRH+bWPjzf25vW64/Tu3X4s8O+phFvLffN7r7z2br+bXvrUuvj2+tAjcN5/py69/8xmmcTvhZ6SsI9008+vP+Jm5oJATGCMFKtGV5v396wtS8/YLi+iFtTgnnDAdEzEYlJbJah5FGCoLmaDoNqCni6rwcGchMU8hZRwcMuJYnwUGyDGh50GRM1rL40aLGGstGC3/NlfKjFLCJT+ktctBmVmPKWViErewVjPKQU2VPWnNNBGsllRao8BqKKzCOalMxZTwUSaXRgv2xTjIWlE6eY0QhIWU22uu/vxf8PJrX+PBr/8uKmv6/pyh3WNILKoCV67lGy8bXH1EiBVueZ9crMnqmNMH30ThaOqKnCJt33O97/AxM8ZI1TjWx8d89Nu/x48/+zGV29LFyNPrS/ZXnrbzjDFyuqwpK8ejh8c8e3HJOCSOSvjwrORrv/5b/OQnj7EBNpsdRycrdFHw5LOfcnkhbU5S2DDECXfjdMLkQBhGTA1NUbAoDQsHC6vRMaNjPlyQ5JhDYxXLQoTvUgcWWh8GcspyKaiNpp7QIInEUaW5d1Jz9/6KpjLULqPjiO8jwYlQoLRm8LAuNY3WRCtV6cIIT98YyClIcG1UxCncYZvk4lNpxf2l4atnmpOFEcdViJQWTtY1flRUDGyt53gJH72/5OzuiqurkdTCq0nnuugToYVx4vJbLcJtmE7uQivWRrMqLVZrrEoclZmj0hBjYgy3giUUxJzpA1N1NHHvSHF/7XDasNkF+lEuwKtKc7Qw7PpfzYu+70e0DoeL6Sx6aqXRSHvYGAJjCBSmF7xDFvTKz7rOvO2mfduBO9U7RbyaJxFKKoUpg4+Rrpew0JwDEnRqporp3Mgt7FynFVmNhMITJwfiLEwyvcvcqnbQJ3+GsDpdeg64mq7rDriEy8sLnjx5wna7YxziLcdipiorstLEKC7LPInn0kIqzFk1YU1ecynP7ihbUpaV/G6e0S6awhWkKcwsxMS+81hjifSM8VJwHWXFoql5+PAh//gf/2Pefe+d6fNnmGLRBH2TWSwrfu/f+y7rozVXmyt++Fc/YBg6jk/PeP7yOdebnnHax9ViwWKxOAhJt12Z8nMOABQ+eoxJkCTJMwwjfd+9JrJIUOjI4y8ec3R8yuPHn7M+WgEZHzyr1YrVesXVq5YQhNd444LTkwDnJXxxum/0fS+i6dQxEEIQt/J0jK21hChF3597T5yLLT/zn29++bYb/WdNTH+WS+TLz1cyCZkKbsJ7uykyMnVf/LKbcWZqCVb4aEiULFZLlkPPxeUVKUeUuc2NnkOF1UGgncN0jJlDZgv6fo8PA1rJdVMreZ5SegpxtECewh4D1jnqukEZy9h5QhAnxzgIpz1luZ4bLDFDRIkg5grq0kEGP6EGhCvsiCkxxn4Std3kRhkPjvSqqkgpifCfEkVZsFgtCVPo7+r4iHv377FcLQhjTz9MbPHlEmMsu10HqqeqJKBuHiI5Z4x1k5t5CthTUpjXRsm8T0FRFjg3idzWHMR+64xgsCIYJ9z3MUyoFiVhqlnB6EeMKQ7Xifmz3C50zoGaaXL4A7Rte8BFzc+Zr1UHTNQootnt5wGH691N0Ken7VpiDrgp2HP0nt1O2v9jSsRJsJ4LmCGIwy0fJqxTsVzrqRsnEKKc26UrDucLMLXQygR3DpAOIcAgofPzuTQMw+EcnLsDZte9MUb2bWK8t23HYjGFiAIhijgu2K0CrcD78bXgVa0VbbtjGAaMNSwWtVxvpmOdcyQraQnVBhaLGusK5vDemIw4YfFYCyoFfBjBj+KKygmVBXnwK21BU5gKbTTdZkNICW0n7qNFAlStRleaZGRu5nOUe4yegntrcSWqhGQyaE1dNYIbCR5tNFYZIoluGNBZMCUSBqcpy4qyrlguFlNRo2cchGGuVManyOhHYghYVwCSeRIidKPnjnVURckui4BuraOcio/WGdr9HldkBj+wD4Nk4GhFIIKCMQ1ct9ek5FFZ41xJgaYw8m/GiQMaMsoaTFGy7zwhZXI2dPselxXNoiGFzL5riX6AEIXxXjiS0jhlqIsKrTM+DvioGNoeozTWTAs2LTMaow166poqC+nkCQmK0mGsE6xOCizLmn5UtNst3diiSoVTRvLKmAUJhXXyvbjCEgn4MZD8JOBrwcSQLQpHDpYcS9KgCV3G7xyxVYT9SPSeTCRNc1Qm5FMMkRwjMYRDx9x8rQhT8V+6RW4hlm4JAgcR51Zh5XbROEbJzSDnA44vIXiXDNjCkLNHK4uxCmclqLQoHaMfqFclZVNSLgqqZYUuIkVZ0FQNKUeGcT/dOzMxBAlBtiLeemYhXZNTJkwOeGPsNC8U/qkx7tBho9B03VSkYJpJ5kzphHlfOIMiM05dPSolVJTrsw8jtnA0ZYn5FYpkp/dHyR+w4EeLMgW995yelGx3Pc8vP8VV4AM8euchT8+foVRkdWwozMg4BO4+LLC2RFvNBx/c5+SsIaotpmzIOfHOV5bE6AlxBEYevnOK3xjAsN1e8a3feo+ffvoJdx40+BGOjtbcv7ug77dklliruP9OQ9EYdtcdqYdm1bHvPIuVRWm4OPconVgvCz766BEfffXrfO97n7K5vqSsK9o2cPfshL/48XOsKmkWFaP31FVB3w4sl5rVicJHyTWYEQApwvNnl9y7d5/gdyiVpWyWIrYscIVjs+kJPnK0ajBakY2EZned4ugIRj+wPlqw3+8gJy6vrilKuYfEmABHTJ7l+piQEIzUPlKUlqwirnA0TU0IjqvrDdbCYlkAiePjNS9fnLNYRNp2oCxrrq4uWB8tOT4+4ZNPfsJ2u2UMgawVxycnxJjYbjf40KGQYHONpm93VGVJURb4JAWJ3W5kfXRCjCMZxTCMuMKyv95TngrfXEwvHqUR7EzoaRYlKQ9idtEF2TqsaSjKim27ox9GUoz0Xctuc401GmsNL4dntJc7zGA5Lc84rk+AQFMJ+gggZcmQKoqSOAZSCmw2HW4BY+o4vnPE7/77f5fNcM6TzRd89vgJx2cLdud7IiNf+eg9ynrJ5npLu+lYrWuudy3L1ZIYR05OlxgtQY+usNjKkUymKkogIfELiRwCpbM8ff5kuq5HlBF3c86By8tXpJSoqpq2bVFKcXJyRNu2aJ2leHG2QilZazaNBLlbK8gijGH0v1oX2UE8n1zomXxAV8yPcVjH3phqbl5g8hTn2bE7/XzbQuYX8LvdWhof1nivP+Fnr5FkN9XhZ/aB0pUURcX19QalHSEknj9/BWlBypHNdocrFffv3yHEwH6758FX7/Ljjz/mu9/5Dn/7u9/le9/7HtvthrbdH+b1H/uMcY7TO2d87etf5xvf/AaLhXReP3/1jI9/9Amb6x05KUY/yGHRk1MZ8HFel8haXU3zv0kWOpixgEOH6yyuz12dElYvnc/7fUeOsF6uqOuSlPe0bcvFxSX3HpYInnZaM76mb96SYPMNl/yw/jsIxrfF69tHXcyEN2L2zZg6CNbz32ed5dZr5jx1X09j5ubx6c9ZjJJfevyN15hz7d4U2W//+WZ/RC+ZEa1ZvEvTv982mSlQiazecIPn11/78NTJkZsP/z5jWURMv21Uk3Uu09pEQtRFcxFMrHw/+hfxh/7iIrqxGuOsLIx1wpUapcK0uJedFJeguMLntogxZqloZSXitBFOX04KrQQVUNYlQwCjw4QAySLAKmEd58mvmnI+LBJBJmVqbufIIqYbpdDTQIkTMz0kRZwE9SFmnAeVhTFNzlilSFoQNDlnYhIBThztamr9zsQo+yzvw3TCiZvYGE2KTK8RyUPLuHvJ9bO/pNCe/Rc/oHv1RHTz2uBTwKgKqmO0B6ePMNURQ65xy0c8vdryyU8/pa4L7hyfcHG95ZMnr9C24uGdM677jnfvnfDg/Q+oT+5w+fRaWnG1phs8vR9p6pKmtiwXFYW2vHt/xdPngaWDpoj89t/7Bzz44Bnu6D71Ox/SNHdQquDTk7/iT179Mem2CyWDSjcMKKUmZI9KqKzJIYizGoVhyqrIN05QpwXrkrIYhMtpkTtOybsa+XeNLC5R4KyiKgUhZJ1CmwQxkqI/nNxM6K+cMxrpdCi0oFPc9FMZzaiEpaemhcPcEl5axUmjubMuWFeWlDL94HE2saoto1EsC4/2ipOl4eSkZHVSEWPmuDYYb6ewJLgeM72X88EoGSsJWbTWRnHiDMeFmwoxiqMyc1LrQyuR3P05fH6fgGTIKXK21tw9LtBoXMrsVSYkOG40x43DGf+Lnspv3bpJkJjPq1lYMdpglBVcQhrwMRB1z+BHYorM6eJvXmt+vmg4PQemKqO5dd+fLnhoUswMoyclwbnISwaYID2aCNlidJQqm3EibhxefcJ6HKr5t0Pm1Gv7OG/zTUJrCZZwk/A3DP4g0M6YgMzU3lzZG8zJJCCFMECKhJhwriBGEZX1JPbBjbPxwPJVstC21h72SeuMMZYQB9SMzlCaMQVi6hhHj9MGpy1np2d885vfnFAuDXNrFtwEZCgBInD37jEhfoW2bTHK8G//7R9xeXWNK0oWiyWnZ2dcXp5PeIKbCdWMXLgd/HeDvTHTdzffIN9g7N9aVLftns12w71793j8xWMWqyVtK250YySoZPRTezmCovHBy/euFUq5w3dZliV937PZbISHOBXlxMUs4mXKeiq8/PLOkdcq73/D7XbxaP77vCXFtPifxPLZmfdam9wvvdvT9SRjnJWCXuG4d/8e1lkRi1Igk3GFdBLEIGGPs3g+C6n7idUt7chhcpUN02RTE+KE11ktcc4I5sQLUiXnTF0vWDQrtLKMvp0mO5oQs7T9JiSMt7CY0koAXAxgFLYUVrnce5SIgkaRo8Lkm0LUmw7seWwefhqDNpaqrimnNvsxiRiK5oCzGsNIHoWjq4045PtRFpZKG1KMjN5PSJpp0ZYS4344uCznsdJ2+y+Nm3lCPk/Kb/jfHPY7xIgfPWRzuCbcdkbPj82TfD0JbMboQ9eIMeaQY3BbRLfWorMiTufzzDqef87ntZ64qHVTgs6EGFFKszALTu6cQYauH7i8uiKES/kMWuFjJKSI1gllDMY6QtyJuDgVWKy1ZG6cPkVRTEUCEbO1MpOQbxE2f0RN4w9uRPSiKA5olttZE2o6z+bw0pmNr5QixPSaSKi1Qscv45bmItzt0NEyi+NUGSVhodFPHTsNRVGBsqRsycpSpxqtPeOwp9sNdO0OkyLRKGIcJdTtV6mQMblrtJair84UlUWXgmsAMDnjnEbpzOA7KUgoKU4opTBWoVQk5yi4uZyJY4AEw75jCD3aQN/viSEzjAFnpItinDjoJ6cnGCfCRtu1OOumcEErIZIaQs4EH/BDZBwlZFNrMVYUdUVZy32yDyPFomG5bEiKia+f8EGKIckZfMiMoxfR02q0mf6vK3LMGFfQlDVKZ8pk0VbY2U1VUjjD+mgphT80pXXCP9/uJAyZTI6Rvh+IfmRRlriyBGOpbMW6WZJToB0SZXPE1fYKmw3r1VrE7RAJ+w6DQ2Moi2LCeUl4Wk4DwXtSzvQxkacuthwnNIvPUoBAUZc1pXE4o4ihJ/hEGHq6sSfmTBqhamqMzaQQGULCDxB7YNT0e8Vwneg3A6FLxHHE9wNj7Inz4nu6X+rZlKRFHJ6vN/N146YzS02F/psg9PneqLWeivU397jbLjZFJqdIQji2WSdsIe37RW3JLmCLTLOsZTwkz2JZs++gXjZUy4rVUYUpEujIarmiKkq2u2vIicIVBB/p2h1VVVEUjSyWp7yf4ANkpvuana47FV03SpG3aqhKyVi5vt7SdYME6sYwzXOnoN1S5moa8DFgsswwy7pC9ZmoFYWVzpf8K4SCoweKanJJ4rGVZp0Dm/0Tfud33+Xi1Y7cK158uuXd9x/SxR26gTHtWC5HEWCVwhWOxaLiva8s2GxfoU2JUQ3GabSO+NAT+yRYgqA4OSn5X/yvvsPz84/pxnN04dm3nnHIDNuWpjJ89Gv3GKYOnOXK8K3fOYOx4c76Pv/mj/4tSkGMmeWyZrUytK1nt+0oC816VfP3/v5v8s/+8/+Bfux59bKlKY759d/4Ct//wQ+583DJgwcP+fTzH3Pn7gmPHt3hwYMztLX8y//vn7BrJbS+qqRo/+L5OSKgCl6r94JPWh8dc715iR8j3gv7OqZIXTfcv7/i/Q/u0/cdP/zhY06O1hjr2F90oAzLykHO+JjphoAtKzZXG6qm5mp7zXHZkJSg35rmiGdPn1GVJctlTduKi3lVr6RTyXu6dqAsHUdHa169Ouf583PilB8U9i3L9RqyIsZExmNMngwKiaurLc4UaBVIacQUJX3bce/+XbyXaxVqZLfvccaicqYs5dpzfLxAm8gwdKAyZ2dHXFy8YrWame0r9rsRP0JZO+qqpijKSdlKbK4v8TGSdeI6nhP0QHvRUd93HNsV/bAj+oGqXAjmj8ToBfc2hoRzDVmLwOvp+fSTT/F55NPnP6FYlZgiUZQlxlpSUgxj5Opyh8mJ9eqYuioxTcNPP/uUphG2f0yJVxfnrFZL0giuKPEEVBSjhEUxdh1FVbKPnt1+Q11WWFuTfJqY/SPaFBRFQd3UjOPIYtEwjj3D0AneBcE8TbU5hiEQIxhT4LOnG/e//LnNJGzOSwF16zH5w0F0lQWA4W0Kdn5TQM95toy/8V7w5ZX67WfNa8Tpuv0zILH6Z1zPZol01gOMlS7G6+2G0Wu2mz3bXUtd17RtJqmEdXB6tsJYwa+tlwu63Z6qbPjaVz7if/wX/z2fP35MWZesj5aMY8/J8Ql912O8o32856ef/YTnL5/ze7/3e5ydnjEOnpSgKhuuLq4JaSDnhGaaJ5HJGpJW+JQnFrbg/1B5ci7fHNtDLsh0v5M5ocO5appXRsncUZauGygKR4ieL754woNHFfWyoijEDCd4TjGWyuvfHL+bwypr9jxhPH9eZ8GXhW15bkr5tb/fmLm+LM7fDum80Qy49brzPr0ptN/8VOr27799n2WcTvpovpkTzKL3Qbi++YRkIlF9OTPoS+ZMplBSJvTMrEvP+WLczEfksTSt2bRozZMTPaUbc++sR/112y8somuTDhNBg8IWIvjFmG+qadNO29kFoyTJXE/gYqMFpyJt2BFnI4tlQd2U7PuA1SKI6+l3IU1yWiaS8EG+RNFh0uTkEWFdIdx1EVjkQiCtWxAiBxdqSAkfhaFukgi8hZPFuNUzm1odhHQNxKDoe8U4ZqwRAVTNg09Noi8zvV7avG2pefHp99HNgtOzJX7cE0sRZXfec705p1h8iCnPiLFGmxXJLvCc8OKq4//yf/0v+Df/9s9JsePe6TGuXPHcO9yy4rvf+Crf/uARf+cb7/PND9/hwTtf4dXjj6nrgtWi4mrbonXJ6mjB8nhFvaxZL2ruvGh49uKSRe24c7rGmJqvfuffx+cSffwOm6stw3bDs48/od1uKUzi8FXkGyFSBrG0QRg9X2jknqsOJ+A02Kcj44zCKD259dXhYp/J85GbnB5ysdJqejwnVIoQJ3ZRFFC4moZczpkYMjGKkD2Pw3xrH1RWh/2KMRNIxCR3RYNw9AurMUbGgzFTsUeL+C/FAqgqTd04isJglKLQhspafI4YLe4IwbOoaTyKpIjKNEazcIbKqCnQT95LZTXdGvX0fz7sg1GSbJ4SNIWmdJosmpKUlfLME7/Bx/yym4hBeRIh5AJjjBHnGhzcOzFl9OT0uIG53FS+3xQK33oxJd+6NN3w41EzG1ozY1tSFMECJQKa6KDSUZIRRxQZVM6Q4+Tcv+GsMwvo6mY83Ij+XxbRZZ+F/Szi7DgFCAoP/b333mNzfcV+vz8wQ0Xgkc6arvfE6BnHHlLAuokTOhVuVM4HXvHNDVnc6sFn+r7H2jjxZm8EQWs0RWGE75cnZ5hS1IsGazSLxYpf//Vv8ff//j/g5OQEN2GRUk4YpnCKicePSmijefjwLs3id7l79wHNouG/++/+W0KUY6iNZn10xLDfMwz9hMgQLvQs9s1iXFXV1FWDc+WEYIgH5yvwmlg1/45TmuvrK/btjpgFDXF1fUU/9LTtnqEXd/E4pgnFcHt8T25+DNFKaKVzjsvLS9599xFlWYjjNAdSMqQ8u4XjWzl/f5Pty21pf3N1+8uCOhyugFndmlTMYrp+29z5F95iGKnKQhaQ48AwyqTSB3Fsq8nJWDcVTb2g7/qDk3ku8Nx2J8/ntfeBfdvStcO06ILopXW4mtAVcUIgLJrVYZxPM1hiTGiLYLm0ISkFyqK0oWwK6mXNMPaEMDL6XlzqhZzTPg+MPshaYTqUt7EbVVVxE3qXD4JwCOKqFiTHyMvzV3zyk59wtF5IwUrLBMv7keClFdlZx+jHye2dKacQ3rmglHMWlJ1GuMJT18p8rFKUucrNua6EzT3t0xwifDvcc2a4K2soTImdHC/OCTLnNn5lxrdIKLOIo/aW2Hw7iHM+JiEErDK4W7zH+XniwtYHp461lropyCbTdR1+lK6CqlmglWW73YFS7PZ7EhmrDX4YGKculLIsSKnC+wZnpcXXaCtuXG7OBcHRFFO7rXTh2Am/IHe8zBxse9t9Pi905NhOrnU4CPYzM35m+ieyhJ0bEdRlwjw/JrMRWThlikK6g+SYSVFAG0VpLU5D10ZC8CgFy+UCV1R4nwlRYV2FcRatI+1ekcOIH+Tat1xUaJUZhl4Eul9hK5uSfujx2VMsHK62JIRTa5SlmgqOMXhCTMQQDuaG0hYYMmEUbNo0c0FnjcqaQjuyNoQo9wBtSkpXYJRkcOz3O2KQIghaQu+HcWBRLyYMT0lWiZClO7TvB4b9IPcxlUElisoSVGQIA9opdLKT0FkyJilU5Zzpu46UEsfr+7Tdnr4dWSwbVs1KckjKAgpZGJE0KsncvrEWo0eUNdhSo1WiXNaorCiUpVQOl2T+oBFDQ13XGKuJ3rOqa8rVEh+hxGIn1FphHGVTstvtOGlOqKuG1ndkAtvdHpUCy3qNVoIhyiT6bk/MnpgjxmpygqhLjLHUdcGwH2iHnjZnMAqDIeeAjx1D3EMC3/WMQ48yBUaVOFuiXSapyMCAHzWh1cStod9Gxr2gdYa2Z+xHwjgQc8CnQE5JzALGYI05CCRaK7jVnfI6gk7WgTec05ubU0oivbxpUDi0qpNRKqIMmFJRNI6iNlQLx/K4xhuFMpn1cc3xyRHdsKesCmwbKQqLUhFTRopSwrpXqxXeR8YhyHw4Z1II5BSZ8aFZGVJRUBQlwzBO14QKa6ecF2OpqobVckHTLCRcesr+AIXRehI3o3QqT4WvEIKE5VqF70Vsb6qasigYU5Qibc4M8Zcvkq1PHF0Lw5AxNoEOVItAUnB+9YwQI3fvrrH2jGJZcrnx0AZ8ivR9zaN31xydOZaLGj96nr06R5sOmwdsscB7uS/t93ucs+hprhj9BqMCy0XFfrjg5KTk9GzF2Jfsr7b07TX92LPdtZTVEuc0q2Vke/2CO4/u8vf+0Tf4i+/9GFAURY3SHnTk+CQzpmvy1Kb/jV8/5S++9xM+/OAUnRxN5fniUvHR3yr58EPPg49OOVo9xHDEZ5++4OOPPyErjyZRlJa+9xSuIE/heMYq9l3AWYtKGmsQdMtGcl76wbNaHWP0gpzOef78GRlYriq68QprHe9/cI/PPntOU63YtZHey/F8+uwpR0d3ePXynDvHd+jblsWiIWXJjHLOTl1FaipURrphh9EFKXtOTo/YbHfEkHnw4AM+++xTtJL7W1kW+KEjNY7CwG7fy9rBaCn6LRtWyxOapuKnP/mU88tWrh8RjEmUpcOHzHoxFfi2HRpYL2uC7zEqce/OAy7Od+w2sN/C+sihVeb46Ix2/3wa9x5joe8Gcsicndzji09ecf/+kuwzISXOry8pTcnzi+fcP7onqNd+pEZjU2IYB3wy6GXJqAbMUqGs4oef/YBx9Dw7f0EfW7wZ2O8jbZdpek9VO3btnqOjNTF67FS8TErRtnuOVsf0fUvwnna3Y93UjG0HaKqy4dWrF6yXDYvVglWz4Pmz54w+ce/Ofa6vz2n3O+pScJvGacbe44yhDz39rqcsK9658z6vLq9JWbM6PuJqc8F2NxJTYtnUxDRQaFlr+KQO2MBfZTt08ByKlLK9tuZRTFiqt69fXn8NJiH9F3jv+f/DGvzWv7zl9zMzweHt22EtD5ATMcFqtUYr6f7YbrdU9UKc6UrhKsPqqGZzfYnKGWdKFHB8dMT/55//c/7iL/+KcRy4/+Auq/WCmDx1VbNqjhhD5N6DB6yPjviTP/ljrDX8o3/0HxCmkPXFYkXTXBNo8VG6uFOUrsgQsiDU4njoIs0pU5VuMgPLnHI2X8x/nw0nOQvP2xorwdg+oJXBVY7dbsfZnRV919O1HUVxglYJP0rHojHqYPKYKQxMGoVoE+lQSLmto8lfpm5qcQfCZFoTzWvu6p/Mv1MxJR/WSOkg1M9iec63xfgv/9vt7/VGL7m1XwcxPN763Z+zLlYzQ/2Nz/dWER25d/9CmWCinqk8deMeBHDNnNMk6+7ZMR85uODzzH5ncqHPXPVbi8ufs/3iTnQziYlakZW0KeqkyVpYxjmHWbqUoIfZ0TWHjGYRJkUAzBgVWdRQ1QZb6GlSnyehRXAW4lpOgueYXQA24xy4rPH+ZhBYA9pqCuF3SFtqVoSYUUnETWfBRw5COypjJkYOSuGsxmc9eTXl32WRBYOX3y0dqGk/tVIH14Us/MRBb4qSk/e+SiqWnL+45O6Hv8Hpr51QnH5I9wNL0X0sPEClQDfkqgJd4PUR6FP+m//q/8U//5d/Sm4vSWFk8/njSaG20Cz5H754zGeP7nOs/yHvnK35xq9/m5/81R/T7l9ycromxkREkiirRYWrClbLmroqgMz9Vck7jx5iC4W98xEGRwiZHD37l894+fgzQvRUNgnGx2pynI5ZDtP1OUtBxOhDuKNRStpKo+A2wiQmKy2u8nnBGqI42ubQzPmSoScxOCdx92sSpEjygZDBONBJ3AaCtwByJkRp84/TAjmjpq6FiXWFuLZjgkg+OMcVglkpjEy2tBGBt3Aaa8FZzazMa60oKwkdMkYzx2sYI+0gqLlFVc6CmNOhiqwUlBZqK4iZYWKJzxfRlGEM4tSCTGHBMfFnjSbkhNEiXocpWFHrjMnTYie/fsH7ZbaYhT07cyulWqpB6ymMQwEFmTAZmz2JwGSUPVSv38RXTJWR6b/E3PYzly5nBy63n45B4XCqQUUDMYCOqGxgCsFQKiIp4QafDDGLABejiO9K1oBTMUWiR2TfknxDb1ROp79JcUiLOCNuXMdisSJnQ4znoDXL9YKqbuj7LUpZVIJuaAnRo5QECBoNykh5ZEYeSIgGKKRV3lh7EOJjjOQJlSEMYU+eQkoBXOEoyiV+cqLWi0YyHXJAWUtZHbNcH3P/wV3qxkzXUwD72n1AoTDKyq3XwMl6wde/+SHa/EN++KMf8eLlK8pSAo6sA2Ui7b5lGEdymgoSOcn11NkpfCvT9S3DOEzneAI136S+PDCVUhKKFgLPnz/nq1/9KilGwjDSbnakMRBDIMdECgGVbrn+p8lHjh6MI6cwhTVG7pyeoJWct0YjCwylydqBkRCfTCQSUJMQQpb7xO0Jys8Txt+GdLl9dOVOrG5NOtJrVfGbyckNWmZ+bs6OFC0x9SQCIUVS1Jgp8O6X3Yqssdri0HQps9+1PP78Cy4vrxiGQULWYsAPLcNcEMr5IBDfdiPOQm/f91PBhIknO/UgWOFIj17wAMZYqmqBUopx7BnHgZShKCqsFVa6KwxFXWBshTYlzmiWiwXr5ZJ9q9lsPeMQsU7hihKFtNH66LHaUhQOEAxXzmkSgQWdEkIUcWUaN77vUaXFkOjbPWPfMnbv4ctC5ixmclVGGUMyEwg4q3FGxO55dDRVgbFTgWqa+0jhS65xcXKxOydisbCpM9Y6qrqeTAji5NRmFoz14XoAgkrSSljjKSWcFca8UmpCsYig7Zx04ez3O+YApMViccBR5ZwPYvKcaaDSzWPjOB6c3VVVHYT5nPMU9FShrYFsUIjDvbAlCk1hC6qipKkq8oRryIUjpoAiyv2vcKyWC3KsiCEydAM5RIxx0z0wS8dfTGSlBIs1LVyMmsQ8pclaMFtKgZIKPjlHwX4Umto25Ax93+FznFp1DVVVC+PeRyJa3GpWBPsQIllFikJN7vg0IcEU1lRYJ50XQ7/HhxGjE3UpYnjQairw52lBEUkpTEGzAaUrjCopbCVBuTqSUw8TrbusNWT3S5/bwKFTI0bpOMkm048dKUdKVWJxWKPJSYtTryjQSrHf7QhFybIpsdbgZnybku85ZY1brth3kd7HiateoXVNGOOh7byqSorCMnhBaoUQ6bqRlKFqFAmDM5aclMzHonzXaEEZLZqGbbshxIF6WVM2dgpsV5RFyTD29D4Qhh5nKxbNkt12T45KxHIkv8Qai88BjCWPCmMFi+W0xhktRS7fE2OkNIa6cuSxpNYlR82CTKb3I4VWNOsp5HMcaMoC1yxoh0CZNKV2ZAXGNhK27iPNcU30CbISlEyR8J247kMa8VbCcrtuz+D3MBVjClfQI8WN07M7WKsJO08aPGiF1RrlCpLSWGPkMhMDpXFkbcmqwtmCrD3KGFRO+EEx7BN5o+g3wqDu99f0fcswBMIUCq2NuEXN9NrGGGntzjKPnrs8bl+P5sXm7aLbXCQHMVtk/YaATpZxpSdziHOUjaNaOWwF9dqxWDfU65JcNSKwO8udh2dcXst9u3QFhdHkGElqwLj6UETruh6SyAntfkcIMm6qskRlhBHfiIi+XKrXiqvj2KGN5GIsl/XhFt73w8R915NpSjIGrDYsmwVKaTbbLUplCTWcOpZT9KBk/kdO7LphQsn9ctvzx6c8ePgQlltQLwmppTEFy7qgdHJ9b/std95N9PknvPvRmo9/+BwUvDrvOHlQ0KSe5682jIOi70ZiHFmsAu9/2OKcYbk6YnW84vr6WvI7lKZSnv2+485yQTrvePFyJIw9z58OWDtw505J3yfadiDGgDEBpTqU2bILn5KsA+P54slA12bu3z9CZcUHH54xph1DvOBo8Q4PH7zPD//yGda0PHxnyXKxQhUrjk5GujFwducef/pHX/DD7/87ckg0C0tZS06DtpasM2OKWG1RSLCzKwzONDx9umXRJE7WDZVRtG1LVRXEAM+eXTL4ADbix8B6tWTft8TYklLBelXw7Ok5i6XFNZa6qXn+9ILgIw/v3efZk5cslxVOOxQSjrheL3i4vMtmc04/yHwgxMByuWQcPdfbS7QqqKoFjx+/ZBzg3r21FNf6lropOFotubw8ZzF1j62Wa168eMnDhw9YLJbstjuapsHYmqfPLjk9ziid6TspghgsdVNhs+Ls5JQn7XPBN8VAt+/xA5hkiMGidUVKnsePv2BzvePBgwqlFSF4nK3ZtS2+idy7d8pqWdL1O/ablhxkrmeXYhyIg6exDYXWjH6kcoayqelSz/PLx3zxyRPasGe5rBiGyEDgurti13e40hFGS9954WPvAn2/Z7FYEnNi20lGj7UWsqOwK4a+I4VAaUtc3eCTot93lNYSwkhdL9nsd4wporPm1fMrmqZgvVpxfnE+CXWO49Ud9uMVZI8pFLbU/NWP/pLdtiXGzL7vQEPVFJyuV7x4+ozFopbPWFfsLzuULn/pc1uujfO6+WYWebM4VbceVExX5ekBdesV1LSImHjav4h6fvjt+Q/Ta2R1C0n95uuo19Z0t4XV+ZXmtdQs9PrJkDWOe7548hN+9/e+zZ//u885P2+5d+8dqkWDsZpnT55x5/geXduz3e742q+t+fTTx8QU2O43XH98yYMH9/FjD0qx3ewJIfHo3Xd49913OT4+oW93fPTBe5AVXXfF5voFKfUUhSH1asq1SDeZkWS0CJKi1cx61nR/uI1QvM1DFzEWXKGF0d95wUtaCWperSq0ls7ZH//wMacna07vrqgWBUpJDmQkkLII73IUtYzLPBt0w5fWoaLLTMcYmDvyZxOjyrOtUTRKNQnts6NdPqdwOtQsYk8vnG+vR18fGa99/68NjQyoWYCe17Y/W0S/6UyLN+Nlfp/89m7rqJLogK+NxS+L2yJDiXI7z0vn4/RaPemwXjeHsTyb1Q56wIQfn//+122/uIiuRRBQSksFVE07q0UQUXo4BJlaJy1VKaaDQ1apTDExqpWGqjCsl5qytmg782oSCiOEjnnnc8YCVmWMUpRWUZcamzPBg48KjKYyCmMNThvCKLobsxPYQIHCxkxWCmsV1k58dOYajixAbjzLcryNmZoEZufy5FaW37tx0eaUiRFpd+0924tLfu1b36W6+w5+bLFJBAVb3+XOOwWmqPB5BGVQtpIluKnou4E/+cM/wFx8TBp6QAvLSxly2kH/inb7jM/O7/GDD8/43d/+Ne4/eMDJOx+y+/GGauG460p8jOiqwFWW5bKiqeUGZrVhtbSsTu9QNDVaOcasSQRsvWA3jFSLBcumpDJx+r7UVMESoTpPDn5rhVc/4zH0JNylOZIic0AcaS3ychIiC3O3hFRBFTqD08IzlwBlcaITE9EHkpdiSGkn16LO5CA4oRgnhEacTobpuxHxUk7TMWZ8ykQlTnHRqzOF1hRGnOjW6Mn0mXEWtAUzTvtmBCtTlJKwLhfSPCFqhLkfsmjugpaZP2OSgOGsMPrmAqymolDOIuoPURjoWoFVCmXFvW+NnNU3FbSEVlAYuUAaJaN3jH99ivDP23Kene0ytq21lK4UvIFiquDKMRJjztQZ8Mb2Jcb4bafSNB7edCm97syd2+uN4E2mdFANKG1R2SIrpiyiSrag5sKXnKtGq8Nxvn2xvXmLN6cZcxV/3kkO4or8Wb639XrN2dkZz54/xpUlWu0ZfBSRR1R6UOLkt1P4zDgOwjOd8AgzM9p74f7evknLOPaT81q+k0wQYUcVuKLCFpnjssQYcbM2TQPZYGzJBx9+SLOoMXa+OalDIeem0DEdc6YFsU6slhVf+9pX+Ke///vklPnen/87igKuNy8gCT/VWkVOst9aT8GEWh0EuRknccNIzYdjN4vNswCrlMIVskjouo4XL17QNA3bzYbgPVopjtdHOGNQvO74FefthJnQiqosSTFxtF5zcnzMoqmZryxaTWMpiTCkDvy4W+L+rUnq7TH81znM3xznhwnLYaDfdETdxkPMb3ibFavUfJfXIjSRSEo422mqzP8c88dfu5VWwvJSzBRGxMO+8/T9eBBJjRbX5WbYoPRN+1zf94fnzFiQWVQ3xlKW9eQalsmzKxx101CUBUEzib7ynaQ4kFOidCXWNtSLpZzHOmMs2MJhTIlV0rVUuIJUJtp9h08ZlR0qiVPdKMFTaDu7JTPWyncyo1DmY56mybPWiocP7vLO/fsynvzIMI588N77PLh//7XCQVkWKJ0ZR3EKiyNfghbnYNXbx2R+rCxLrLWvhVTOzvH5PFFKU9fL6fowHvIW5NqgDiiW2XUON5zyoigOYZQpRSnaGy0uuBipquqAR7FTQWM+L29jopRSqMmtMiNiZkF/FtOFTS4BS+FaFnAxCsZHqcDQe3KWYqPOULmSPrf044gyRhbFYbqmeXEHVlWNRtPZlm7fEbN0HqopEErumbKgkXCr6Rwx+iDsMxVBlWIqYsjixxUa5yaEjwqoHNFaus/m+YuPHpQ9XIckc0faQHOewnanEOuUDN5PBdgkRbAUI2O3YdwFCmdJOZGCJyu5lgkbPhFTAB8IYSC4NVozOf2c8FTjQFMXwoP/FcPJOt9TlFbmVyGRe0iDGAgGHWDqhlJZnLnOWHJODCnRh4gOYFXFcrXEWo0fR/w4MvSDnEsAuqRuKlksKkfPQPKRsi5olobInqwizbKhHSJjns7NwsrntCU2O7IuyeWeYegk4NJmXGnw40A/jiwXK6yu8J2n74KE17EghZEQwBYVfdsy9gNkSDFLEG/qWJoGp0u8j5iixjhH8j2oiCYx+pF9BKUrKXYbhS0GMIrSrqi0ZbPNuKqmqhYkEtENVIXBNgtQA8uioraG66tzOU/9iK0smJFu6PEaUJayKnHZElrPEAbGqKiyLKxVkvPPDyM5ZiLiahv2exyKSlkxhUS5h6WUiDpRNo4o4HMWzYLBZ/ZjT996SAGTCpxvGPea/srjr/d0u1YwPClhbSk4g5gnw8StfIS5GM00f1Vz55g+XC/mfKmcJxziNOfR81xlaiEPaeqgm1iL2oAtJIR3UWnqRlGsHHZhSS5ydKemXha4SuMWjqKsUMZweveYNmxJo7CxUzeiU0SbxJilQ81fR9pNIIwBrTz7viNjcGWFtgVjH2nKmpjBz6YEHdFmQgm5ihTyFGKa0MoxjLBvE9YuQSn6NqKVY1muBc9jSpJKqNyx37UsippqUUnxKE4F5Sn7Zd/tGf0vj1p8+rLne9/7PlYt+Nt/5+sUhefp0y84vqPxQ030mmWTQG8xds+9R0uK4h4/+sEl3/k7H3C5ecnnP73i1ctEDJkUFQ/fWfAbv3WHepkkv6cwtN0OW8HyuCJ6KTytihVGW9ZHx3z9qxW+X/L//m9+xCefPOG9h+/hhy3BB7r+ikRi3+65e/8EYzKb/oqzuxW768jLVzueP+v46Cv3uLjQPHp/SVVLf/miWaGM4eu/cUr0mR/+1SucWzC2Dds+8fn2ksefvaJwmm6MUgAiYsq5q1EK6mPvOTtdMI6BsqzZbQMPH9xBZWjbjmHwuNJQNZXck/xI8IkUFdfXe7JSPHr3HufnLwkxcXR8jI8v6YaAqyJ1swBERLu+2nFysubVq5dkXYOBFFvu3D1ht9swjD0xJZqmwE7C8jhKgLNSibKyPH22Yb1as9vvqSpHUVictZyfv0J0Bs04Bpqm5O7dO+z3e0IIXF5uuXvnjK73WKeoKsvF5QXLZUXXBcienKXrcO7mtK4kE7m+usCYJcO4o1lYikJzfr5ntWooK0dZGWJOdPuOR199F9+PbLtrAiP7IRKiJ8VAvxupa8eqXlGaCj0ZDDZbCUJtVgvGYaTPHS+fP2HTvaQ+qRl0yydPHvPg4TskHfj6N97n00+f8u6jBwyxpe2u0cpORe+G8/NLudUrxTB2GAOnJ8e8fDlKV6I1B+xC3/UslhVdv2O328k8yToUjqtXV9w5fkjVyH1/t+/xXnF0dsbmi3N2+z1H6zXGBhZLR1WJEfHqakMMgshr2z3r1Yrt9Bm1KnjvgzOurje/yq37NcH71vKTLz84/a/evuZWqEkHvzEEvU0AfPtMY7rOz+ug6fdvlkC3Xii/KeLf/nnzvJwg5CRmJaV49uIpj945o98PVFXLf/qf/h/wseQ/+7//F/RjZrE4xZU1F1c7nFP8L//JP+Gf/bP/nE8ff4aPkaoq6PrhIGaPXtYf56/OGYae+3fvEseOT370V2QyV+dPMGbE2ojvMtlnCKDitEbICpWCpIVNgn++hVec575vahVwIxCP4x6lE5g8hfpqjNaMoUerwLe/9SFnpxXL8h777Y6gB6xTpNhLRo5OZBWYO3NztqgMWgWUGhFhXDGHhx6+WoCsJ/3llgZ52MPE68vXWVSeET1vqh/z+PqyZsJbnjlvSt+st24E67nj7Gdv6i1/fk3ofm07qImv78nbRHqkixI9603yU7+pSR02c9jf2/z/nMWgesOC//nbL45zmcQ1lCwmtBJBOSOOhhtWnsaYjDJ5WvnPIWXgDLJYtoqy1GgnF6j5A+ck3GofxbM6v68EfSqMyjfhokneV0RyEcXLQmOUYSDhEQe5tPOBUwo9yoRPeN7inZuDJMkBlxGWaZo5qiLMa6PJspQQh4GS1w4xYXKcUFWTkIzG2oqjd76GT4aFdoT9Bp8Dne9YnD2gUPewjSOGAa08hV0AFmUqtI7YtCG2FxLCg53Er4SZhlPyHd31Sy62F+z6nuWiZnl8hl3fISVPSeKsKcl4rq6ucK6gsJYQEoU1+BypTu9QrY+JeEKfJDDl+Tm/8du/w4cffsj/8z/7v/HpX/whIA44HxI+iRM6M+NOZFLtc8THG77iPH7l5M4S+KrlwRDl+83kW0m4kyDKFFKWBbOijCywxTkjEzvUdB9JkGMiTzyjEEQwT5OAbRUT8mN2pE+C+oSfEXeMcNOdlmKJLSwpSmuOteLOCSTRbFWmLB1u4q9qbaaTVkTwkOb3B8iTkDsb2eeTmYNr/yCn3do3HzJGy+JXMgYk3DYrdaiwSZqwtCTPl9eQIuOv4HY5nN9aWO6gsMYeRBetZMdNyoeqppLUt2mSM7cUfXmbAz3fdKjfFs7f/HkQ0af/LVZCJBGHHIeCAmStYQqNEF1ynkDkg3AsbfyJ20JmPriFb7bDjXL6KNLyOy82Dd5L2+bJyTEPHwxsr3aMQydCSpbAS4ObBDhpQZT90gdm75z4PQtCRVEcBNWcE0VR0nWBcRzl+maEx1uVNWVVU9c1X//61zk9OeHjjz/m8vKS5fKYjz76kHffeUTdlGg9Cf9ojJmbvd6+KeTcbOqK3/md38Jaw39eWj7+8V/R91t2m25qlZ3P1emcm8Q34g3THV5HtxSFYxxv3LYzysUYQ11XFGVFjIHz85dcXZkDc3scBzTpIFLOIt8swoszLqG1vO5qtaRtW0FNeI8Lr+MRZmFfaz0JVvN96WcclDfHw+3x/Au61Ofnzu97O4DtTZyLngavFI9lLEU1C/16mjj+8ue3KSxD16GTtOy7oiCTMU6EycI5ign1EaIwX+eQTpg7MsxhDAtyyFLXFUdHa4rCHfIC6qo8dAxkY8g5ivs8JYy1WCfBbTFmUoyTiKlAzazcQFSaOH2H1lrKopCCppZWXBCmrdKTQB79YbzdBG0K4qcsS+SlIoWzrFcrzs7OUEpxcXFB3/fcOTtjvV5LgGbbknOmbhrK0tB1hq7rDiGlt4M658duT8Rmhvztc3zmac/ies6vj8m5ZXRGtMzBqPO5lHMW1vw4Hj7TTTdAeA23003Ii9vXnNsLghs3SBKROsQv/ZsI/TfjNiNzpJR5TZzv+54Zq6KUFLS01gzjiCucFGC1xo+RcfRolSm0pSgsdd0I37kf5B49ZVkcXK9KnOcpT86hoA7HdG5ZvukEkdB6uEG5KBAuv5Ow9xhbnKtxxfT3lCT0UymMtpP7KBIGLzg36zDaMg6ekWnO6DIYRRg9XXfNMKrpO5tdurJ4sabATC73oQ/AICgKIIZIu+8IvsOPI4UtIP0KzGSkwBBTJoYkqB0rOSY5SzeIKic82jSrTklwW2VZkpwjZvlu7VQ0N1k6AQie3ou4jdZUrpTFts5oq7DaUeiM0iP79gptK5arU0Zv2bVbikpana0T401hC5S26LJkSaLtdsTQ0TQl7V7wKymLk93HIPdoDDEIQ90HRZEUm+sNfT/gXEHTNJS1JauBcewpKwkTxGpc7Qh6wAePzYGQRvohYo1hoJ/uHwllS/qxxZUVVW2xlXQeGGvkmpMyafQ4ZThaLymM4nrzil27I5nMctEgCJcrclEwoihMSWkKYvak6NHGULmGNCpSFsd/UJmYpANF8JhS7F3VK1KC0QesLRB0yCDzbldQlpmqLrGlxl/v2V3u2V+1KF/i0gq/L+iuB4bdSN929H1/GCvzOTMjTbRWU6HbEGMgpSjZcWo2D02ZGllN7qwbceUAeNQalRLodBhnWkWUzlLYKgVFV1aWRa1YrAzVcY1qNKaxZAvZeqplSbMsqOuGjKEuC5w1oEtsYdh0A1VdYmrFmEZCHGnbAeUL4hDp04iPGe2kM3UYPDZLYTr4SBw8MY04pylKQ1FU0zzbMqaBru+kuxVNSkqOPTCMsm66c/cBy2pBu9vT9jtsMsTkMbWmKEq0TWz3LTFFNFL43Le7v7Yg//O2owdbqqOE7wZ+8mmidmfcPfkqFfC9P/sR2hrKRvGVr52R0x6lBj748A5H6wX/8g8+43q74zf+1jFf/5pmDAP1ouLf+7vf4sXFjxm8dJP5iw1aWZxruL6G9fqIIUa224H2ekQny1/9+SdsXhnO1u/SXj/lh99/zje+1XB5dYF2HXW9JEfFevGQxfKY/fZTPviw4m9/9xtcX+04Xt0jRcWLlz+maxVNdcaTZ5/Rdlf87t9bk3zkj//1E14913z9197n6eeeH/3wKYumxpqSvu8oXCnXYa0he4Z+xFgHY8RaySSr64LC1Xz+2RMKV7Lb7Tk+LaFKFKVjv9+yWq7ousRisWC7eYUrCtpupG17rCtIwWCLguVyhSs05xcbhn5ktVpwcnLCi6cXkJHwWSXc57OzE8ZxQClBwa3XR2yud5yeVJRFxThsUAqaRUXf7zk+qVjUBTGC91IwPzo+ou12pBQ5PT3h8vKK45Mjdrs9+/12QlTUXFxc4MqK3W7PcnUqHHuVuH//lGHwbLfXGKO5uL6mWSxRKnBx/Yp7dx+Ro+Ni84q6dpzdOeLq+gXaZN597y7X11tANJXN9SVtv6Wua16+esly2VBXBSerUzbdNSUFsU84W4FWUmRal6QUaJolUXnGbuTrX/0q5qmHJrG3I+WRox33LI+WfPb5Z4SQcUvFxcud3Gv6ga9+dJenz54d7D4xSXjr0HdcXl1hjGG5WjIMvcxHh0RZC9PcGMNu1zKOV1hbsKoLPnjnXYzSnBzd4Wrb4grHyZ0jvnj+HFeUrFcrYkzstt2Uj9AD0jlonMWHnqqo2W9a7t+/SwqZ73//YwqfOb17/Euf20yf783tBpF1+7oxi+hvEQ/fto7+GdjKt61p5LV/8fXFDTrm5vXeeo1TgEo8e/6Euin4+te+xh/+wR/z+7//T3jxouW//q//W169vGaxPmG5WBNT5PjkiLOzu3z01f8fc//1ZEmapvlhv0+5ODJEisrM6lLdPa2mxaidnRXAAjTQaEaAF7zmNf84gmbkBQwkjWsLDGaxWOzMzs5MT4tqVd2lUoY8wsWnePF+fiIyK7u7utpoBi8ri4yIE378uH/yeR/xHt/+zh9yeXVBCIFnz57RdbuyjnY4V9H1exZLIZlsN9d867/8V2w21zx9+oRnz57ivayj/TgSfChktVzWx7ImV7dwE1UY6MBLOSCTHeTkjQ4FOC73zhpDP0ougbWGptZcXV3x/vs/4T/7l39G5VramSOZTtw2bEWIHVYjVoBZAaIiE2Z3PJB0OSArkt9ygzpP/POyl+Xl1nL7+4NLALLH+nXP8+YvDw/vs4/0deSx17zmN++T9Wd+r7X6zLkmEFvzeeZPsXieiKlw0yZf3+Y5vOa2FdJtctvtc/ym43OD6HKZhbGdFSSxbxEC3uRjLcCUMbcvrti5IICgtQpjMq4yaDRai7VGSomUIyEaxqgIt6pgpkjqJsaQ2DHIBtwo2YBrLWwjmUwtOQVCnJjRsidLWdji2SomD/OYxBZEIyYSN+Cu2MNYZ4rkQ9BPoyYbDsUYApUS4Lg8LlAaUzfMjk/Q1hF8R1Ka+fqEZDzRJHT0khpPIseeZDLoGSobrA588xtf4z/+h79l6PubbpQ/O6Cu1it2ux27eU0yGrtYse921JVhsZqx256htWZe11RK09YGrWC72fHJs2c8GnvM2b9lpVt2Q2adMt3ZnibP+NYff5vLT35EDs/xQ5ANbxDQ1CjxLBe/agF3vU8HEHm6Ul3umVVghI4tFgAxH4I9Y2GLJyUhSyEpxhRpSlXVFPa3IuNcRuUSbOgDKcigE6OkLZNzCfSUKtkkTc0TAJ0TKYs4RHh1Ukyxsl+XDXwZKLUSS4yYMj4K+9s4jbJGqpbA5DEdU7GLKUOaVQpDCQPK+RCoWzmDNVo83qeBL9+udpXrPQzcEqaqY75lFzSxgqbwA2nTPvx+ILopcnmlppC+4pGptRT2ssZkCMkfPKZy0UVJIffXAIhKPMa59ZqXmbm89nfCNLZYW0FyEiqbLTnZQzU+ZdAYmV5MsZrKkZzDoWY6geYvTfpFJnd7EfDSNWsjbPtcPJ2tpe+FRSvszSTjgha5uEy48VDxFJ/pwogtFhMHH+LiVXwbVBUQSx+KAIf7gzqw1L0fMcby8Mtf5l/883/Bm29+ibfe+jE//enPuH//AQ8fvMPpnWMmGdeNx1gBKH/DXKABa2C9mvHtb32Df/i7t3n65COWsxVDt2Ec5DkbY6WaXYpfoXi6315cgHzuyYM4hPgSiD4Bis45lsslXddhjGGzuWa32x/sbMZe/j2O4wEYe2lRY6aE7RvmwAQq5lvt6Xaw5MS+na7/IJP5Dcfr2sdvWiTcXlTebucHEP/W7156ADKRklMm5UgiSs/KEpD9e4UPWoTZriVbJKZQ1EXCANYF4K2rCuss2tz4njdNc/DIngpAE4g7gbMxRnzwxBCIUfpnRp5RClIQCjHQ1I75fI7Sim6zlzA5U0FOJcx0jyIKU7ZsGHQJCtNKbBtCEFGr+AoaQoKcDVqrz3h+T+1tut9t27I+WjObt4zjQCbhw0Df7/FhIYzp5Mt8NhCjoiv+6d77wzkn9vgwDAemft/3B3b6dG+mdrvZbITteQi4zFxeblDqJlB0aqM3G6iX29zUv263vQmk3+12hz42FamGYWCz2UhoVtse7sk0BtV1jc4Kr8YD+D8x4ScG/fSznDODFx9nZ0v4bIrlMyecs6WgUDH6gf2wZ/QReyBCaJQypOwZvD+M67PFEmUdez8cxsCpX4inrHjCgjoUaZpGwmCFFTuNnZSCz03BSQHaZJTyxDSQco+rG2auIlFCW/0gn7OuUblm6BGm8GHsENJHzpGcAtZGXAWpyYRxEHsuKmxlUdphrKFyNcYIWJxSRjEUi5jIMA4Mw0DwAgoPvSeOG/z4+6nItLH0fY8PsqpJOZNIKJMxylA3zWFDBjDsO/zoWa5bVGGppziy32+hhGlnLMo5gu9JCqzRJTRbbBS0URhbcghSIKaIVVDZhtOjOYqEcZnoe/a+J1cjdibWj01rWSxXbLcVu90lVVX6aNZcXV6jjCzcc8rs91v2ewkrTSlTR0fwI34MYkVkHU1To4xjHPfsxz22mqFQxCzBlT4HSCN935OC5CLs+oi1QsrJww5Dw4XfMV+u0CYwhAA9mCzrOidsIYhCGBmHAaM1TSW+wkqL5VJMmd3oqeqAmVuSU6Tek0ePyXMWzYwUpCgVoqZqGmzTAtBHiOOAUYZ5O2PeGFw9YxgHrq7P6PsRV8bFXCwrGUAPLaqv6K4SvYccAr4fit1TPMzPk9pkKrLVdV2IAq4Ugwo5yHspuChdilVww2Yr81YJo5DnJjrA4v5HjcjRtUGA86Vlvqyo24pmllmcNCxO1vQEks3sxx6swThRhM7qiq73EAOVsaAiOSZMZcEasoLKVoQxUmnFYjlnm/dkdwe0Yt/3dH1P53taWzGakZCnvKtEGj0xa3xIGFNJoGEY2e8GQoC6EtuulJOEGhtN3dSsVisW1QwVpV2vmhlDkDVHTImqbqSgud3i991NQe+3Vet/w7E6qnF3LDbNqXnE018a/vrf/YrL82tCHKnqCudqrp5n3ng04+nZC1arj/nSo7dp9YKv/OGXePNdQ7LPGNIeW8GHH39IpCekPX6U9dzYK7rdNRfnz/BjhU4NL55c0m0Gjlc1D+7dpbKRcbjkK1+7zz/5F+8wxE/Yddfsdj0qz4i+5cNfXTBf7njj4TF17UnqksgVz84uicGxnB+RSGiT2A0fo6uR7rriR3+3oa3u8vZbgQ9+8ZxPP77gvfcecXV1BWqU3IhgiSmQiFRG9hXBZ7wXHluMiXHwzGYrcrJYa6hrV/Z9keVygfc966OWsxdPqaqlzGU5YpwR9ZpsJLi+3oiCyQTpFxE21571KpBzAEYePLhDzJGL63P23Y4YB5QWooyrLCcnRwzdgDMaayrquuHFizPBRIzh8vKMGCP37p3S9TtSitS1Y78fubg4YxhGHn/6KZnIbNbS9wPX1z1N0+BHz507x2ituHP3iOvrK87OX3Dv7n0otjYhBS6vr4ixQzvF9e4KlWti6hnGkc3mgqaQHs7PZXwxxrGYr7i6vCCMe853G2Z1w3tf+jLb6x26M6zdHf7VX/wr7i7u4mKNri0uy/jbtEv2+46qqWirBcZkjucrnu4ecxEvBK9IHkWiaSTg3Yc9y+WCeO258+Zdnj1/xvXVJad3TmS+jA6jNP040A+F1BPlnmitmS9arKuoas315oK6rjBG0/de9hSN5fnZlvXxEUY7Vsua5XrF+z/9FW/cW+LMmn6/I3iFsw1hlH57tL7Hs7PniO3TwGzWst/v2W33rNZzTGNpqt8JQnvN8fnGBiExvWadcANNveT+8ur++uYcr9u3/G5FPpkT1OHrdI7PnFrJemqxaJidnrDfbjk9PUEpxbNnzxiGwKNHD+kGaGcLhr7D+8Dx8THvvfcu3/3ud/jLv/xLTk9P2Gyuiyo0sVg0vPfel8t+JPHs2TO++93v8vVvfIPKGT7+WOzRfAHOp/0/haDhvcc6Gfen/cK0rn5dzgevuZ9aT3v3so9UAe9HUhJ1hMYw9Jqrqw0fffgJd7+0xrPHVYrZ3KIwQhpKqeAnkFUJxlSH2wcHJKOQNKbrma7v8NoJar8NmlO+m8iBk93LZ5/3AXDm9pt/dn+spvNNp2Ry77ixksm/pT1lbsiYv/F1E2HzM+dTBxb87Z8VFk6xtbm5R7/2nSamL+X5vnS62yvm33x87hEgZU+KhlRCkA4ADdO1y/faaHTO+CCptaoEXSkFtc04qwrgdfNBUvFiVhpyVEQQj+NcAPOySEMVWxAl/nfWQGUpHnwyfBijcTWobFFjPKTgqlRqK1mh8wSwTvdLYQu7Xh6GEqmpsZhiyyDtpjDZUTfMYZuggIUKASOrpqWeHePaBWkYmN1/iyENJKVBGwyJShsCAz7uEIWJJUZPCHv+6Z//Cf/23/4HfvnBh9Ix/XioysqhWayWLJdzNtsNRysJ0wiAz3A0a2malufPepyFeeOIo0xcy9Zx73hBd/6M808+YbU+Yjh/wpA0O1acXW7YDlt2V5fMKi2eJjkUnXXGGYU3AqRTKlshlIDZF2RIQwABAABJREFUlMrLZPFtFWKfoyWQVXzDJdwVBMj2SaxMlMrYIPY/Q0w4p6hqTds4ZosGY8HlQB4jYxcIXoIitNXFZ12eiy4gei6bH5WFDR6z8LAiEo6SkhRgnAFj5NqDRxgrBSxFFQlrzOgqS8VdG5LiIIeNKTGGWDzZp6czNe3CTC/tHyXS3Uieoh5vXoMqm7Fc6OqpBFIVZmrKskme/Lwyt9juxR/+9zicc6TIAbzWWpj42hhslnvrUyYFuW9Wi20KqYAWvyax+3Vg+a9joU8TmVIS5qSNbOiTl0AppSwpSwFMaenLOU9sz4lZLgvm22D0BJYfvi99XObVmwnzNng9PT/gYOegtGLWznBO2NLGFJsjlQvj2xSQORW7I1smW5mYp5C+KUhwAtYmb1GxcxFvevm++L5nUf20s4pvfvPrfOObX8Oaij/6oz/mjTceMZ8vePDgTazTYgVTAgWnTbG6Bc5Pn+uGdS8TmpnCJduK733vu/zgB//AE+uoq4ZUQD5rqwOQeFvqBrwEWk7gntbCtL8Ntk6/G4Ze7lmOpAM7PB0sbYZh+AwYOT2PiWmfUsI5YSU0TcN8Pj8wqCfW721GrdbCSvtti8bbgOWrbfn2cfs+vgqsv744o38NEF+KiuVnMSaSmmR3U/v54h18jIFslPhPhsAQRqyxh+sMXgJjMzCfzQhFMSHPS2OMZSiBr0oJ+KKUgBLjOAj7OYtnsveGYRypKocxurBehxL6aQvQfRNKp5X4/hqtyKFIjEgSeFeemYTwHiZwyKW9aUliD8FLUaVPZeEusn1XOXn/4A+sjc1mwxMS+/2eTz/9hKurK46Pj7m6vnppXMovitf2rcXeNFZMzOvbfddaUe7s9/tDX5iUPLcDT0W1ITYhU/ttWwHRJLhUAO2J4T4VhubzOW3bEmNkt9uRUmKxWOCcK6G/U2DeTSjoxKBfLBalzw0HexfnHBp1YKJP/TbnzGw2OwBtWgsbvGrql8hNDncYx6YiAMDR0Zrdfsf55RUxBLSWz5hTZBgCXd/jx0hTtxg0SpvDPTdWVEZCqAhkVCno2puCGaX4o8oTLf1iAlRT0sQUxXeyAIYUqz5joKoMtq5IWdrqpADQ1LSNI6aWnH3ZECqUModiaMwjyXfE3IPuSdETYsCqhsbVaOPQumyQssbZCjuXULlhGPFjR4oR5xpygv1uz27j6bvfL5xsHMcSXGuZAlHHccA5hatatHOEcSQrUQtYJ/ZOJb1FClhDXyy5LKv1MVK4SIR84w3qrKUfBpn7tCqWUyNKSYHfJo0zUlRo65qUh1Ic68lxlKyAbNC6QllF1VT4UBVf/wqVNUo75os5IQS6ri9g80jTig1a2zYMSlEHUQuOvscFAV5jtHT9nlor6kqz70Y0kdopyI7QicLODxHfD7S1plnMSKMExg55oCWw77dktLD6k8IZS9MqYvJcXp0RfGAMvvQrYZLuuk4KGEnWkuM44tuAm9XE7prQdVxdZFbLIxaLJS8ur9jvvKhPXSjr58iwHxBGhmI1X2C1g1rTNDP6ccfQj2hT1Da7xPWzHeOugnGB8ZbYJ1IYigQyvTTvmEKKmKyg+mHPMHSHgF1rK4yxaGWEQV/C41NKQrK4tQtRgCxGk7DrVNl7GaidMLOr1lAvDPN1zXxVk20k20B7pGmODd2+k7yfRvqljx3dbqRxFW0zI8Ug7aj3bHc7Ach9T5WLx33K4t9dGcx6gZnN2XcdV1d7Qu8JKWFayXiomorNdoux4s2eVMJM43hRsnofAU8udjSTuq5uxV7menMNVSIMI2kMNLYiE9mFnpzB1g22cpK9kBJWO2yWos0XPRb62xBhf6X4N//6F3z0wYZZq1idZHaXga7LxBEuXli6feTeo7scH2VGv+Htr3S8+eBdVos3Cfk9Nv4jAi84v3iGMgE/Rq4uM37syNGSoiMMkOPAi/MtMThyMuy3iXGd8X3g6dnH/MX/7iu0854KSz075sMPn7KcGR49esCvPvwVblGR1IAyDZ988pyLs57z55r9BtaLPd/67gk/+cX3Obs4x6lj/upfP2dmlxwtFffeuMvXv/aIzfXA+fkVi6Xm8ZOO5aoiDA7hJQbGIWBcwliDwmC0odsPWBfY7j7m4cMlzimxcNoPVFXF1dWG1bpm9Hvu3V9xdn7FYmlwtaMb9jx7nlgsDMEbNDXGKoYx0raiAN1eb2nqlvm8YT5v6Pod7bylri1dt6VuDKenJ1xddYx9z3I+ZzsEUmsIQbHZdHivWa0qUgoc3Tthv9ux3W1omgqlMicnp+z3W1F4e89uvzmE3e73HUrJeuf4ZI33nuvrS46OFsxmLdfX20NGy/X1nq9+5W2ePX1GTEHIWEYU/REhCV1eXhFCpG0dw+BZLOYMY8TVFX7bUTvDup3z1sMvE3vLanWPe6u7vPvWu9xZn1LhUGPxLc4aVay5lrbm4uKC2WpFv9uyqNacb88IfURHQ+1m+DhitOaTT55z726iHyPvvvcuT58+pu871kcLmlaA+ZQ8GlEYdfs9s1mLVYoQRt59910+/vg5Xd+z70as03g/Mp/PqaqI7z2utdgBnr94BmS8H3n29DEn6xqnK148vz6oKxXib73Z7FiuFrzz9pfw3rNYLkk+cn52QeUc+71n2a4wnxME//XH5/t72S8IaeulI09fXgY3b43St87xChfzcBxMwj/H9eUCoN98hZcLrdOhlZLqlpIsKms1q9WSqqqwuhJ7RBpWqxOuN1v6bktVG77znT+kbRu+/OV3+dM//WP+2//2/4ZzFX0v4a9XV9c0TYtzlsePH/PNb36D/+q/+q84Wq+5c+eUX/7yVwCi1JgvGVUoReQg6zAvxW5XQPVpbTn92xjzkn3otD6e9rzybykaBB+EiKQ1SkVilLBubxMXFz0vnr/gK++9RRghGYNWlRClkiaEEWWmQNAIRMiy51evaRc3JZTb+8yb/f3Nr28wy/IkypPMvE5xkLlhY3ObaPg6dcLh9y9/vY2j/Db29q/9/et+/Opnm174GqXFTTnnla+/jU1+Cxe5fa7bZLzfdHxuED3EgRA1OlaYYiY/vd0kk5gelTxuYUwoXexcgNoI+CbmFImMLo07Er28TpvCXpVVmoTeKLEcMKpYYpTbY22maQ1ZGUYvViFKZQnEK0xdUxZbpli7JC1A/iT/kpzGjHYKY7MwiMuNFcA+kbIixoKGUgDZCEOAxgqYKIF2hmQds9MTcvLMj9YY48haU5sGo48ge/xwTvBbsoGUHal484QwMHRXrNc1/9l//md8+MGvSH5iVt08TDef8+Ar7/Li4oxdt+fp+Tkhw3q9phv2OGtQKmD0yHxWQRzothvuLBuq+2u++413efDOn6CaI2J9gnrjLsOLM7pxpD25Q7i4JsVMO2sIu4xVAvAaLcwdVybjtoLaamKYQKdygTmXgoI8R1MSiUFsT3wU9nnMGZ+gj7II90667ZDEPMdahXVGPPadwqKJUZU2V2Q58vjEjz6V4sw0ICSI6sYiJuVEzIpAYYgDVov1SmYK+Zw2HqBUAa9UljBNRIFAUoQg7XYMSZhJ6abTTvUZcYSUdpNyphtFPtyHSGthask5i42LM6WGlzkw3RMCouesygb5RmJrDNhKGFefrcz9bkftKrDFVeqWlQrcWNBopA8aKyx1o8UD7CXw6TUV3Fe//zzAujYGa0RSP9lH5awEpFCpgC3CREffmsxfqrTegOgCZhcQ8wCS85n3VUpJsMSkSCjAV1U1DP3AxaW0SRnfBDwLo4DjMYmP9W1GqGBsNzKxCYB2zh3YrdPknBLFG9mVwtQU6BiJccRVsD5q8aGnrmvu3b/DURlj2tmiMCdlbMs5k6MU+rQW5c+rz+QA6JJBSXits4rvfudb/PV/+AMef/qYzfWLw2cha2azGSGIpYd4Z09y8Hyw/5jOnTMHYG9iwU1MWLGuqViv12x328PnPCxyCgA2+W/fBqKlTcrE3Q/CzFmt1i+xaW9Ls26D3RMbY1JJve74dUzz20WeV9vvb7N5eR0z5LD4KG2VnA+BJmI9KG05IXGoX/QY/IDSiqqAh2KdcXNPhmGk23f4pcc5zxgC+313KPpMwZPSHyClaYEpPsNdvy+FLwml8uOA9w6tKlKOxPSyGkMs0xLj2GNH2QyJss0goMw0vkkxxFhNGIYyHxsoxZWCc8ocryGW65L+6tBFLplSKG3Dic1MAV6ngstU3Gma5uBBvtttidEzm7XUdX1gnscYWSwWWGvx3h+sEtq2xVpL13UH25XZbHZg8N8uoBlj2e+HsimoD0zxqXAxve5VG6Tp63K5fGlB3zTNwQrmNmA2n88PTJvpZze+4lKEnAoCcDN2T+c6bCq05B8orQ5sZBCm+vSZJ1uZ1WrJbn/EdtfRhUHGAG3Q1pB6GMbAkALeS5HYh0jWqSiN8it9ZWLgx4P8NsaEQWz2lOZQgMtZipA5S9F7HD1hHIkRtLZo5YgBhsEzxlSevVixeB8wKlLZBudqQoQQPWQlbN2qZvSG7W7P0HX4sKeqcykQdaJWci3LWU1dtXgPwygbIms1KotdIFnWs03TCtisGxrriV8cYwMg5L7YflWknGnbmqp1GJuKvzSkIEpN7TTzxZKYElcX51CKLyln+v0eZSxVXcnOzMlafMrkaduaMHq2+z2JyMyJtaLOSIE7JJzRbPc9OUwBqyMxedq2LmtpmZMury6AREiR4D3LxYoY4N79lqqxnJ2fUbdSlKjbWjy9jaNyFfNZS90aMgFjE0oFtK7Kdcu4oUmkFEgp0IdApSEqg9NW1nEpo42jrVtMVWPNnDFXEDXj0KGdpXKONCb86NltN7TzhsEP9EMvAbeFDZ4SbDY7YsxUrsIXNuru+oqTo2OM1fRENvs98+Uxq+WakA27PqKVxYeRMWSgQlcNY9gzBgEeQhhJSrFYzOnPd3RdX2xYIEdDrZeMY0aHFoLGD12RrlOIAuolq7Vp3KjqipRUYVHLuNP1O4zWWFfR1pKTobXBBwHuFMi+SL4c1vVaZerKUjmDM9BU4GpNu6yolobFccviuMUrTx97koOr3SWdHxhSAAV1Y0oAXcdme8nd2RwfxsOYMzFrIUPKwkz2CZUVIfTUswVuPuP58xdsLq/wYy9j/lrAN5+m0Fhhj4YQMJUlJUWcVCEhk9LIMIyHcdU5RzufoRLs9nvCfqTb7BiHPdrAEAaGNBJixNYNSmtcXaEmpVwcubUs+p2Pf/irzK8+/BFVC1fXO5TVLI6PuHMf6kZj44punzFu5L0v3+XR25lul8nJ8uBeTR6X7M8esh8GQtUzuCdUi4jvII8tizox4CFDVVlSEkLU3dM1Obb0m8g4dAQuaNeG+28e0YdnNLN3ubqW4NWjozXX2wucrbj7RsVb7z3k6eNLZm+s+eXPP8JZy3vvvInRlpz3DOGcdXtKv3vE3//NBU8+DKzmns3FOT/64cec3vkAax3vvvs23/rWH6K15vnzpySfOD09Ztdd40NmHCNVVqQoAVY5B1JUoCLnl9c8eMOwOqrZbDuaxnF1vWO1btlcXzOfrXnzzbv4sOP8+hptNXVjGcYRrSz7fsA6zcnJGpUN7733Hv/pP/4tL15csFqKtcl+vyOEHm0z2YsXe9d1nJ9tOT09IgbNfLbi8nyDdQ4/drzxxh36fgcIqNU0Nbu92NFtNhKuPJF2dMmPWiwW7PcdbVvTdb1gHNZweXlOiJ7tbst6vQIUu+2GYRzIKvH46ROcrYgxUVWOXbcjho7F3EqewJhxtqHbD8QoYboZWa/vt4lZVTEzDf7KM3fH/Jf/4v/IrJ5jlIaYUUGsumLygpU6izKZfueZz9Y0zqHXDxhTx8Kd0XJJqzSn6/t8/OQjqOHktGHXX1HXS3a7a0LwNK0rHtgbZnOH0nB1fUVdV3R9j9aKoe84Wq84P79AKYp9n6iBxP5KFASDH+jCnuM7R+y7TlRZMRKGDkMgBM/p6SlnZ4HTkxM++fRTjo/XvPfeIx4/fYK2ToiMIXH/7l3u3bvD82dP0Trx/MkZDx7c+eKd+zccryfs3BQyX/fa1+2pP3uOz/59nnzOXwMhvPryCSzP3OAc8sLP/q2PvpAyGsnEKeSQZ89e8IsPfok1lpwM19d7Nts9RicWiyPu379HjCPXmyv+8Nvf5K//5u3ifS7KwZQSn376Mc45vvOd7/AXf/EXLJZzrDM3e7HEgQz4Kst8ImBMBJjpnBP55VVQ+PZrpuLqtIcIIaFNkLWXgZwT1hhi8KxOjvjow085PV5zdHfBG1+6w2rRMHR7JDfDScC6isTiY5Bz2QcpyXm6DQlP2U/c9jWXDe2t18m62RpzCzMsYLjKL9nXvHrctjWRe/TZ9vYq6XD6+joF7e96vHbPDZ/Btl73bKbX6lf2Ml/kOg7n+xwAOvwOIHouDZCpwjEx5wpYHWPEKIsq0nORLMn/BdPHGSU+nIW1nJWEVXgfiTHjjCEVAJMJzFSpgHUKo6fKUCangFJWFigKQmEliE+7Ihf/deOAJKGiYpWoaSpN5Qy+ABOJhLGRyuliI1O8eLRI60NQjCGVooCA7DEmfJhAz4BWsjnWzmIqg1kt2I49x8cLovdgDM42ROdk4RbFL0Iu3+LTyOj3qJzYba65f7rge3/6Nf7mb99n3EVZEBuDmzWsTk/wfQdRJJo+BPp+ABSr5YLVvCYPV6g44LLG7zYM2y1HixlzwDaG7bjjhEjye1IwGJ1RQ8foM66tWRwtWd+7i7/cEPdbht6X9Wth41mYt5amsYwhoCj2OkkA8ZuqaC5MFWkrY5Bix2T9khDSjBRHJFBOFfATJcBIiBplwKjEJCnQGpwyZTOdCiCfxCNfabGPyRx80m91jVssswIQ66kDFhDiYFdCYXhRQNwyAGVFCJlxTHif8AF85GBBNLX4SXyjUaSYGcYSzpYTysqAKJJvCrtbCgHCfi8bvVI+1gXsjDGXQgJUDtpWE6LCmZd9oH/Xw1lh0MkkogvAIJJqAQGk6ztrxS/WKInxVJ+dxF+u6KmXxuHfxEK//bvJ+0zridVsUDjAIuxlL7Yc03scZCUCCL/8Prpcp4Di5FTAF16aEG4Gzds+1sIuPzs7B2C+mFPXlYBrlaMziqp2aKUYRhnwnRVWiSzkhPkw2T3UtaS5C7sxvMQO1VqYszL2aHLWUpKMCR8HVusZxsIw7Dg9PaFpLLNZK6F3WokHsFGkHEsxREAoHzzacAAMDyxgrQ8MfrHfAFcZlmbBf/Pf/Nfs9z3bvzxDb0q4p5f2PwUPxlt2LtMxLUamAMXbYLZzjqqqxNJDiZ1V09aE6NnttgLsasm6cM4d2AC32a7yvBTG2sMiaRzFp/7k5ISmbQXMv7UYum2dA/nWeV4PlL/u37+u/f6mAtL0moOVzEsLlM9O9DmXeTELyHyoh5dx6Ysettic+aE7KLNSDEzBwM5VtO0cpWQjhxZrIWOUAGFeGIdVJXY+ohS4CRwVYDZTOYerjHgyGw1FtlzXtbBOjMUYR920PHg44+69U9pFRSIyXy6YtUf0fWLoB9rGsVwtAMXFpVj8WJupKlHHBBUJQRQwrnIoxaHgAhz679Q227bl5OSE9bJlOW+oKsfx8ZqUAnXtmM1ulAwCTitSijhnDyyUCcieGJ23wzqn59O27YGhPT3/uq6pDuoQDuPANBZMbaJpmgPQNQHobdse2uttZcXElum67vD9y31P+lBKwrpXSh0KTFO+ADFRV/XB/mUKB56e2TiO7Pf7YsFRg4J91xUViaFtW5RSB8sl58QbXyuoa0ffS3E6lLDNEBMhRmLI7DsPKJyrQEkBTSklQVTTxuDW5kfsrsRLeroXBn2478MwHp57CFGY40GjaIihJ3iNHzXbbY+2mnYGdVOTsxJf4OTJbqBppS2nFA/FGglztaVQIwQRYbWXsNgAdRMRWa+EvBor82gIAyl6nFEsZi2D0XT7DuMcdTUnzUH/bq6KnzmyHrGVE8AkK9r5EtQMpQLVzOKTB23LGJTAZVSIpByosKwXK7phpBtHYk4Mw552Phd1R4iomERBGhIEyabZ9h3JtSSliCHgjHhq73dbri6v2W6uUSrj40DVGJq6EZJCSgxjYNePYommYPQjp01DtxtYLlcMscNUGmUd1lRoKyHIyiAqM5VYrWbEPIAOzOaOprUED5aZMO1zoqlbUo5srq/YBE8/DizmDtfMyMkUv9uAq+bcvXMfqzTnVy9ICGDv6goqxfmLc3q/g2qFrRzZQR88280Opw05Jrox4EOkRlEphR89QzcwCNKMnwp7dQ3GMJ8vuHOSUDYz5oirLNotAMNytcDEEV0rhrBnGDxVI8DvOHgUFZCwLGhdzQ4JDo0xEJIvpBFQWgPxoJi5zbaLkbIm0Cgt99SWtVNKI0OvWMxqaldRFcKH9172LEaLZaFRGJOonKapoWkMy3lL21psY6gXNTQQbMCbEU8gxAxe/Mr76OlDz+B7amdZtkcY69gPeza7DcpUJK0kZNga1CC5CiYrauPo+r2M8SqSTSTkAR87jEnoyrFarXDOcX5xSVCaqDKb/Q7sjJQigYytHGM/ik1YmgpG+bAms9bQNDWVrUh9pN/2XO93hDCgDLKHTRGUOrCAtYaqaVAKoleY8MUJLj/80d/w1pcbCcZ0hnpm2XQbTrXhwVs127NAuwwcn8J3/qSlqhO/+mXHf/z3L3j60Z6TI8u3vn3KnUeax5e/ROmO/UahfI3yFj9sydnTzgyu6spnFoVgHGC36ZgtGva7SNOc0NbHdPsz/uHvf0Rm5OGbx5wcL/ngl2fkNrJanvI3f/WUFy92uD+/z+460848mA5TR8bxjL63xO6r/OLHz/jpjzbMGkO38QRrWSxmdDtP3Wb+/f/69/zd3/+I5OHO3QXGjTx++pyvfvUhWUU+/Ogxdd0SvOXrX3+Pn7z/fbwPLFY1xnouri5448GCe/dXjL3FGEXfjyyWC54+eY4xjgcPj7BWwNmT0yN2u8g4iOWYVorzs3PefPSIH/3gBwD4MFDXK3a7LSD4QNNolLbMZhUpad57700UlquLLZWzzOdz+n7PbDbDh1EyrHxiaS39fi/8iUL2GsaRwSe6vkPpjO86Rj+iteKdd9/mlx/8kuVqJesgndFZ7CWvrq5Yr485v7hkuwucnB7hh8joe46PT0hpQFOxunNM8DtRdpBYro64vLikqRusUazXx4Rh4N7RXVLvub9+yMKc8mff+1fcWb6BNY1Yo4aAspk4DGSTqSpRH6AjTVtRV4YQPTYFZtUJyrd89cEf8NRdYusZ8fQujy8+RNeKRGQ/7FiyLsr/xG63Zb/fUTc1PkTads3jT19gjNgJjsPIOI6Hdb73HmMN80WNc46+C8SouHP3Pq52nJ1dsJjPUEqx3WzJOWGUxrnEi7NPxbaw1dS1YrVqGMYrhn7D3Xtvstl4Zm3Dftfz6afPuXNnRlaBtq3Iv4fKRI4p0HOyQ7kha772tUxZTurwFW4RwMq+5gZwf/W86tbeZzpHOV7ZvtwwkuVP1auv5zWvp+xjCklHHaxyhe1e1xXWRjabDUrVbLZ7YmoIPnJ8d80w9Jyfn/P4yWOGoef58yd8+9vf4smTZ/z8Zx/QdT2npwuqynF6eszR0ZoQRq6vr2jqip/97Oc8f37G6APD4Dk+bui64VA4HrysU61zoCS8fiKvTNd/2wLt9lFV1UEhPVneUhR7opTP07KV+WLBl7/8VR49uMv9e0col1Cp5vqiZ9ddYyswNnN0smS3v6aZzdhuNxirEUtaj3WWcRhomhpXWXkfLTbBMXhUsbosLUD2jYVwklWxqs43ilKNJpFfwnJ1ySI8kAsnPCcLCXbaL5cWekNAlAd92I+/1Hh+G26tXsez5zXkcjl3fq3NwmfbYkYIzhRi2u8FpOf82z9HOT736t0YWaQbLc7hqdBlc2HvggBIkyevQljLk9TAaLFxEXZRJIYRZaQiGkOGnHBWiyWGnkJsCnBN8arOGXIUOUSIBRATz2ujxLPaVU7kcwq00VgjKeCZwvbVitoZKifSGGMgx4xziarSjGlqUGIhI+AnjKPwhp0Rf+94GJiKFCZLoSEFT//8Vzz52/8JvvR1wtuR2WLFfFExjltiGDFYchIw1dUrkqqghNGEkBj6nh/83Y/YXHUsliuuIlg0zaKBIuWsjSWGwPPnT9hsL2hnLYrMvGlpq4Zx3GBiRo2BoRvIIdA0lkenb9C0ht3mOWO/p1k/JDtNd7VhCBHbLmhmJ5y/uMat7+HCBZt+IMaREAUE91GeSeUM1pZgOBTWZEJW6Ag6yf1JE3OlAMAhFBa4lo6qY6ZSikpBrcApwCpaKwNcDJFxkEVEtgKyGStMF6Nlca33noQUUkx5LE5PitZ8i+0pg4gMFAKaG11Afy3txQGiiCgnSoWJTi4yc3nkKVA2LsUPfQpeQixrUDcDzeTpJOx7AQmtFnZ1CmJFkwqgnnMmZiX2MEH825WSYCZrNOMglhdaZWatZdYaRp9wvwfbBaCeQsMOepIy4Wp5v0xCaZFNGqWZgqYO91NPQ/mtSbRc+6uVxN9WORdWq4R6mQKiKz0VEoq0PiWx5lBIgamMwOLJP53n5fc62GlQGHZTv731VTGpECZv43RghP/8F7/genPGdrehbWbcvXuP3baXzbL3aKVBCYhtDix4dSgyThPzOI6HkMLbA3ws7L1MwBiFq8TbL0bxALRaHOAvzp+RkufRo7dZLo6wtmGyHZBJR7HZ7hj7IOCllrbb1A37/chut8Vax3zeFpl8Iqco7VFBiIE333yL7373e/zd3/972rbm/PyCi+6SvheGl3QNaftwE5Q4AfSyKJG2ZK3I36ZAwpQioOh2HZ0S/+Wh70kxkmLAaE0sNhBTOGLXdbdAa2Ts0ALO1HUlDGgvElFrTQllntoAh3XptH5U3CwQD63wNUD3647XFYKm43VKDJkPbxWKpgvLZdE19ZlprEqi4JJhJGGUIn0O//Zfd9w/PUGYF4n9XkKTrBVZoQLm8xnL1VJ8JH3Aan1g9d+2CpkY01OgpDGa5XIhzKChl8XjrGU+n9E0AhCLfYUEA0s+ABwfHbE+OuGNh/dYHy8wDmazOc7N8V5UXyl7+mHg+fPnqLL4VsqUvpqlkGcnVsLEtFai/MqZcZTPKeCx+CY7J4FUTSWe4LPZjLOzM5bLJev1+sAun8DrlG4CXSdWPHAo3AAv+cNPC/EJMO+67mCpMoHeUhiCqhI/z77vD/dTbHIElB7Hkcm3eBozpmubrF6msNEpZ0EpJb7P6eaagYMPZF3Xh0JajBGVpCg62bhMfzc990P7zlm+V+Ccpa5Xh7autaYthauqqui6jm2xm3GVhCKOY48vn1tyN6R9xyDh7M7e+LsfNi9THytj+KHwpThsEFMS/0Oxdym2bmEsY13L4ugui2bF2dkZMSiGPonCx0tuRggTW1QYQN54+gGUnkB0gzGZ4CPGZrSSMEaPgyyZKdZUxGQJPtN3A9bMcU4s0VIqxd44UjlL29Q4a/GDx6coAaSVw+qbAssXOebLCmccRju8R8IWdcI5qLUiJQ+mMKdyIGRZl1eN5cisWS5W+HCOMaFkowS0LcqRYcQpzbJpqY0jNS1121A1NXXjsNaw315L28yJy8sX7LY7hs6LrUUIuMqglKHvR7SxxCwWeNbVUqxX0zgpPo85JppG1EspBpSSgl5VO3RW7Pst2hramcHHAW0jKQmJxBqHyYYcPM4YlLH4asY+dox+i8+Z1dGaGK+IOdKPnhT2PFCaN1YPUCMM+09oFw2uFqXFdrtlDJkxDuy7gQB0Yw8pYYxju99h6hqXFWM3oGLkZLbE9wO5G/CNJVUNylbgLGMU4ktVWdABlWG2XmFmx1xtO3ROaJ/o4h4VNBeXFxhrSl6PJidD3yVczIRRl/3eSGQk5p6oMhIU6g9jycSym/qtWBZqmqamaS1KJ3zoUCpSVxYbM6RIU0sB2wdP44Rc4aylMhrJDS7trDLMWsNy6ZgtZ9i2QtWaXo2EPLL3gS70bK4HmnqJm1UoMiobxt1A3+4YZg3zZUv2nsvdJagKWzVo63BVxdDtaayh1o5FM0cnyQsZ0kCOGhMy84WDuMRkCxjOzs643u1YHJ+wPFry/PkTTK+I0ePqHfO5oeuHQk6RNXhVmVLUk3s1BW3vO88QA8lqutETU2S+aGi1WLi0sxkhRHq/J/ogJIboea2X8ec8Hr5jCDmyHzNeJVxrGHaen/2j58/+2Yqv/XFNpRU5Dvzd3/6UxVHNe1/9A/4v/9c/4R/+0zP+h3/z7/n+B5/wJY7QruN40ZKVY7PpqCzonKlqx3ze8ODRQ6ybc3W+Z3Pe4UPNdveU866j0Uveevsv+NKdb3Hm/46L4SdoO5KDgVRxcvQuq/WaH/3Dh1x8uienyI//8We4Cu7cO0FXmdm6xW3u0e81f/k//h3f+cM/YWZmvP+PHzIwkrNme7WjXVq0rcBmlMn4IfLs+TXr0zk+B3758SfcvzeXQmgLm92GT5/8ii9/9R2enz1hjD2uMlQ1PD+75sGDO+z2WxbzCmctdS0F/mEYyMmwnK/xA1yfe3K2xJRYrAwZzzgGahdxdmQf91jTMPoe42Turl2CHFnMKva7a+pqzundOVdXe8g982YOOuNqh2sqzq8usHXFGAd+9eknnBwdsd9uCQQSgaP2hP3Q0zQVMUWqxrHZdTirefL0MUrD+fkLjo9XzOdOQjWblqw0ISVmyyVHpxXD2JOyJyfou54cEpVt0NFg1YzFrGaoPcaJwij7iImwf3HF0fqUd+79Ad94+3vcO3qLRbNGWUdJESMV8gQ5obSVTIecUBjIETfTYhcWDXjNG/dWHK0fcX71S1z/AT/7+OcsVxXD4phB9/g84n1mHDOz2ZKcEue7M4ydYd2clD3jTqxgnFX0ux05BmIwPH38jJgV3bCnGyPVbEltjfju64YPf/Wc2bLGj56j9V2xPGPD5eUV9x+cMAyX4oMeMsOQWa1OySExd5q7RwvCcEmtR66eP+HBg/dIMdGHjJlZGBOJ34+8NgGXE/g6EfxeRfAmAFupXEDIG3La7T1FLuDfrd14OX++9X43oOm0fbnxNr9579sWrZMycAKKP/M5XkXjs+TTZTWB0Tcqw7OzswPhw+iWvih/hnHk+vqcH7//I/7Z2Z9z9+4xq9US5wwnJ0dU3/ga52fXGGvwXmwOd7stL1684P69eywXS/b7PR988AHj6KnrhuvrzWFtq7U+EFxSLm4I4abIPBFBXvfZbpOxpv8nEpcvlohKa6wRZW3bHDGfzVmvjvjzP/tnXO0vySrw6eOPGPcGPyZOTo/ZbyM+tOy7kd0+yVqt4JJVJcpbqx1+FNvrKbsvZ3CVRZV5Xh5VAcutoRvFZs9aI6RPH8Tmsli3TQ8+lL24Ui+D5CBKu8MTzbcsX279TL7+blajr4fQX3/8ulDRm77y8iHb+wkAz2Vt9AVA9N/h+NwgujPiMWorS0wIW1cVtm25udoolFFiQSmoMvKKjCYWz0thcqucJjMLIJNVQNuMTuLbMoFwWomFBEmsVcQOBiip6zkpMeJXCmM11mlyupHxy1dFiAmyWB0YozCOwpqOaB2onKKqNNpLurxC7GlyknT2wd/4sFst4JgzCk0SaDZFRj8KK2ebSMOW3fMLHrw1BaAJWzqTCXEEXVHVx4xmTecDWmlizGyuN/zkJx/ws1895unzc/zgcbUj+cgwetCKpq1JOfPixRlta7lz55jZoiUGz/FyLswnV9MaQ50VfYZ21rCez5mZxHzd4qoFQ9czdHtcPadanuKyFYZijKjZEtXMCallu4PNPrHrM4MXEF1Y+UnsRmK6Aa6N2OCYUoGMlODOMviGNHGHxa/cKKhVptIKp6DRmcooVrWm0oacII4RnMY4jXaG1NgSNCulDgkZVIdK1DRdTKoFaYHCu4xJHcQwlQZnFK4ywmw1GpQhA9oYlC5VutvVuFysXEaxWFEKhLR26/dIw5mY6GoKIM1yfU4pKg1GaULZ0MQkTHZhIcvfTcwtW0FTW5zS7Ir0UqnMYmaZzxyqD6Uo9MUPoysm0GnqO9NnSgSymoKmJgC9BKlOBQmtDtXNUgaUyV5/lmV++7htj3HbM1oXKbH4zE4VUc9kyVKqZKisxFs35WK7BAFNfuWclDahlKTPo7VIsVWUYUpl1OSTjnrJBsG5ivv37xNj5G/+4xkff/SU7WbHbrNHwjYVKSmsdaQ0AhmtXXlfg/hIi4dbVbkDQDYFIU4gl1EiQzcOqsZgrcZaR1UvyBFCH9heXzIM1/TdXWatgBBHRzPE6kbLeBU8L55f8vGHj3nj/kOOThZ053v6PnB5cc3z58+5c/cOb7/9iNUKZo0pTBhhXymd0EbzxqP7rBZrog9YW2PrBhNF7RFHLxvPg7WQKYxNd7C5uGFPy/33XvIxhmHAWsf15TUZYeqTMjkGTAFnnWkOC5uc88HeQtrHtOibmBXiDy32CgJY6dJ3tTH4LO9biBwSaD1J1Q4F5zJncCNpk7776yf81y0sXnfIgkuKftPMr5UiKU1WEZ2lrwsRUKOSPlgpCXBsiyftFztO5zOy1sLe2Xb4GFDGSSaEc9R1Q10sQVJh/FZVXfzQy30sn3HyuRcgpqiPjC5rsul1N4uWlOIBxJm3LaenJywWx6yPjrlz95STO0uq2lDVNSkZSA7rarpxz5OnT3n8+PEBYAax7QrF89xoYTuF4FGKl8Bq72+8v8XaRTGMA9fX17R3Ttjvd4dAzqurK66vr9Bas9vtEQmq9NfJW/y2bcpms2Ecx5cA88nqZfJGjzG+FMQqgbn+UFyyVgDzqbg2AeDTzyZAfgr5vP2zYRgOfWsCwKdCx2GTUFinWosF020/5IlNb9DE0r+mgtX0t9OGYjabkXNm8AMxBepGilYxSmEr53zo99JvEs466qpiGLubOSRPEmAt60RtQEtpSOmJ5a8P+QtT0Xqy9RPf7Ug2UwAziGe9OihcjDHs9zvGccAaW4qXiXGMhBSECOIaVM6kaOi7yXrLYY1YOez3HcYqqqqW/hBhP/RYp6gbS13NIW7xw57gs7RX02B0DdkexmFtQSUl1oFarEGMFmZR38v/bQNNo4nq99uILxYtKWhyiigd2VzviDGwWM1Ru4hPA1olKivWIX1OEqxVOVlPqUjIkVyIJ0lJLkUMAYoao2mkcFo5Q+2kKLUPowQ+6lpC+VQWZWnsMXXLnbtvEOJIzCOomkzC2oaYRoZ+IIyaRTtHzQwhgnYSKJmVoqpajKnpx56kxH+6bSw6QW3mAhiHDlQkhZE+eLRyzHTN0I+M48DY9WhrCD5gs6WuWlGsGkPbNKggntvRZ56fv0B3skk+mh/J2jIgti7tnDhmtv2OYQzoyiEqMY02FWhLs1xgbMfls2dUGVarI0Zbsdlt6QePaRuMs0SdSSbTxxGfAioncAqfR1LYMfTXqAyNMWz3Ha2bE3KiH3uMM5hKA47NWWK43tEETfKlMBS9FId8wPtIGEd8GYNe9XlVZKxV1AvD+m5L1Spms2Pi2BPHwMo5DEpAhjESQ6Df9+QElYHGaarKUtcG4zJ1bXC1pqqFeIHk/BZykZYg4dqSW8tu22GcZbU6wo6OfrcjxUjX7UmMKGvxYUeIPbnf0zZztHKSnWVKUJz2VLWmC5FdGBgH0N0eow3OWfrdwH43sNt1+H4kLzxtVQMK7xOjD2w2O4yxDGNH34l6ReYyUTbFENntNqATYfTst52oQ53BVBXDuCMlWCxEMeXLui6nWPJDoLY3tohf5LAu0w0jMWWMtYydZ7Fo2F/3/PD7L7i4OOEPvvIlfL8hjIYHd/+AH//gE1bHms1+w1e+sSKmjqM78Md/8ic8e/6U/+XffZ/cGXQ23H94l69/7R3Wx2tQjp/+/FP+/f/8AbNak/XIYq3wVcU33/02X3vvW5h4zLNnDRdngbpJtG3Lz3++4dNPH/PwwQXGZN756ozTuy2jH8hqRQyWn/7ojPsPZjx/0rHbBIJ3/Jt//Y/8y3/25/zn/8Vdvv/3f8+zJxfkLpdC/cBy2eDHQMoBQkVKicWyxfs9+11X1tI9J6dzfNwyRsvDN9c8ebKj70fm82OePQmsFoqmqQR4i4mUIqtVRd8ntFE8+eiCBw9P2GyucJXlaNHifUfOiD3Zriv9B5pG1Hliy6bJDGhtJFQ4ikopJk/f70h5pBsSXd9TtQ3L4yXDswGMYTabM2bPfi92YLP5kmHoGPqB5bKFDM9fnHF0tMBlV5T6mq9//Wt88MEHEhpeVzR1g4+RqnZ4H3B1TYhiY9U2FcdHJ3zy0aeQYOgHnDWElBnDwK7b0dQGlRL3Tx4Sdok37r9JW6/5w6/+MY9Ov4zNMykU5yRCK3JxB1CQIwoHORG8x48yx2al0ZUWu9RalONVveT4+CHvfeWf8N3tY3719Kf84Bd/y/X4gmG/kxDkMOBMRTOrMBvDdtfhXANJ8+TJEx48OGU2c+y7DYvFnFBsxrTWrOwxm82W5BuGTjP0kefPnogVzOXIarng/OyS1fqIL3/5q/z85z+l63qstkJeygpjKqwJeN+hc0TrTBx0yW/JNPWc2rXstj3tEvrtQFv/fuy1Sa37WfLO64hmv52ABi/v3csPDgXBiRQxvVyAdE0+ZDDx8nkL6K4m5iEFyH8NEKqmtX/Zz0AQimmmkG8NfS/Ekjund/jwo0uxb9aKumnpuw1N03B1eUnX7bAukwmEMNL3QqKqatmXWAtNK8S/cRzRSvHixRk//9nPyTmz2+05OZZg2ml9mHMmGBlfvPeMPuLLmvq2SvvVezipM6fXyRo9HdaeqljfTr/PRnNyesrp6V3qesZivubZ2Qu+8Z1v8cN/fB8foRt6Pvro5xydrAklLHnfdXR9x6ytxL1BD7jKcXxUCeNcO1GsaQn/HvYRY8BVrVg4a5l7U4bopcCdMWRd3AV0KYAcCifiBiGP9Xa7u8GAPtOeXoNc34DWn/N4Hfr96w71clv/jaflwIs7AOe3sbv/fx2fG0TXRqFtYZUWmQBZ2HyyOVIHa5Vcgg7VxBJFgigrIzJYgS9kwarL32kjIFZMEJIqAYIJozLOpOIZDXWlsTYRfSYlGH0Sw3StcLrYNuhbm/gooTEhgk9gcuGO5wzFHkRrkasKyFHC+JQujKRMNyZ6XyxeUDde97lYlUw+7ylJaOUY2Dx9QVi8R/SRq6sNR9UDhjSgTEs9a1BuxpgVMVl+8YsPOD26g1WJJ88+4Sc/+ylBeZSNtFbjsqbvdQFYBeTYbrfUtSPEY5ISz7PFfEHtDDp68ZtVirapWNQVzimOl2tm85r5bE57/BCzuAf1MczXtDOFPXrI2dNnXF6cM4bIJ0/O+dLb36DpNfHiF4R4Rc7h5lmqhEqRFEvlUYPNUmSoDIcB1BoJFs054UMkpun53ABKViuMSrROQk/mdbGn0GJRobUUaUzWGBduKk3FIzwEYaEbFCknYXdneU+VpSCTUsYDxVkRaxStM1TFK9ZaQxgL88kZrBOm1ZRbq5X4QqeYGH06tOGUX67O5jwZJdwemgQYUPkGvJeJEwHP8k1haqraZqCPMFfCDLU5Q5YQMWugqkSSi8o4+/tJwgUgv5mMb6RMUupKOaMR1YkpYZrTfzfnUAcQ5vb3v4m5++rfvvR3Wpd+lW9ZOZWJLmckcLGka+sohYscD8yfl0H0TM5TqJaW56Eo9kzp5esoFWfxP1OkJOzG4+MTvvXNb9J1W37+s58XhvReJFMF/MokKRYV3+ApNFDuaXwJAJsm7QmwTzFhbGa5XvDGGyecnK5wzjKbz2mrWQlu3BCz4vHjHpRlsTghzgPGKawTgMqHzDiMPH/2guOje1xdj7z/k1/y/vs/ZbvdEfyINnB6uubL777Fn/7Rd5gtGurGksgH9vsbD+7y9jtvMY491ayhHwdZLA8CioaYwdjSR83hft/2e5+OyY95ssAQoFHAUHneheXmLFMIzMQgeFVeNy04hUUQCtjuWa1Xok5Bijei6kiHtq0LQF/0KkyFtsk+4tU2+kUm388sYA/titIG082YUb5POZEj5GRQucawwKiJaWLQtOj8xft3ozTGVmRbsb/ecr3ZkkcvQZ4Zur5js9+xmK+KV66E+cUkmrOqElB4v98drD6UrhjDePAKz1mepdifBQHYFeX5BUKMaGM5Oj0hJ0vl6hI4KWqeYUj4AcDRzLIUc8u9yjmJSiFT7MtSeb+KmJIUYcq9P/Sl0lamQsw4jvSdZqsiRLEu6bodl5fnnJ8vOTpa0TSNgIcxEqN/iXk+sc8nJUbTNMCNT+LtsOAp52C9Xh/6+tQmpB9YYuRgEVNV1YFtPv2sbVtCCGy32wPIPVm2TNYrUgCpD0UrY8wBML8N4k+qgokJP4H/zthDX56uYwolvbGzUuW6NdrYUgxPGKOoa1fY8TIWS7Gw4uhoxXa7YXN9SQrxYPmTcsbHBLkA6VgmgHzqIzd9D5lz9Y06qvSww/wgOTUld6FqMFaXe7OXAsno2W12dN0GtDmAidbY4u1ewl5NhatKQWOQ9qbLPQq+gNMUhQgOZ+foNOL7rVjgIPdC7LiybPhMxlgNWkJO+16Kjrtdx36/o+uKX7xzL3+8L3Do7IR5mDyutrRzxeVVjzJzQlBYXZPjOOWVE3IkqURCs0tbhrFnHzsSmkorCfPc7dA5UTeiyLr2O6IPqAQLvSD5wMXlBaCoXY1zC3IchZWkQDnLycM3sMZwdXlFzomqdmLTNWZiP+BjQq8Ns6YlZw3asO87KlPagLYk5bCNobIJlQZZP1UKMAxeY11b2ooUx01OJC/gcSATd0HGuZgxlUI5TfKeRdVKgT9DUiMvLp/x5JNPWc+XnJ4cs7IrPJkwgMJiKrlu0BAMhlJwyorF0TG5quhHj61qXMrsuh6fYTSSzbBsapq2AhPooifXCm0r8Y11TmyzLp+yu95glSO7GTlqUt1g2jl935FUIGVH2Eb8dQPbmpgVVtV0+w1+Hwi9p9/vZbxVN8He09gyFaGVRoDzU838IbRrTeUyc3dM3AXoOyG4mAZCJo+Jcdcy7gfwmVqLdVfVWlxrMbVBGQGYo7GMGXSStUi36+nDKIDAeg5xpN9vaecLrG6oqlYaJoFh9FjtyNrQe48fwO8HjhZ3BPDOHRWZMST6cSTbiqQ0KIvfbYla1JtX+w1956ldTexGZlpzNJtxvTxi2/WMYcR4z67bMPqe0SsUhqpaUFe12F/YSAgd+80Vvbb0e09dz8gk2fNFIGZqW9ENHSpr2tmMYei5upSCqHZLzO8Bop+cLvn4U/F9jilQ1Za2dmKjkTOPH59x/uKKo6OG9brh/OqMJ8+ecLl7wZtvntCHLSF6vvaNP0CbAe+3tC3cfbjgaHXMH/3Jt9hsr+j6c374ow/55JMz3vtyw2I2o249m23Ab04Zuoa//Kt/x8efvM/qPqzvNNiqIWfH86cX3D1doqiBREoDZ2cb7txbMoyJEAbeevsuMLBaG05PZ+S04OMP9/x3/6//gT//p+9x7y3H0xeKO3fmfPpsy7rSdH6gdo7KGUiJftjRNJYYFPtuYKYquj7it57VyvKLDz7mzTfX3Lt3ytXVluAzpydrXjzf8cYDV9q+pu8HmqYlhC0h7Dk5cVxeXpCSWCEM/cD6aM1mc8msbel2e+7du4sxN0XmXHKqJIfCo8gsF2JTeH29pWkaLi4uhKxjCuEwCSkiBcFNwhgJfuToaC7jlQ+slkecnV0wm81YLueHdTE5cXV1zXa7Payd97uO2WyGArp9zzB67s7FHqLb7ZidnHB1ecG9e3d48vgxq/WSrANPnj3n5PSIttZkP+KywY41//zP/gveefuboFtmboVjhk6icNNJiqRaB7AltyaEshYoBfCynzLWHpjQqmThaDJWz/Gx43jZ0s6POT+7orpq2KWBrPeErufhO484uzgDFVitZigU19sNR+s1dW0JYWQ+m3FxuZHibt1web1hPl/i9JzLF5GTk4pFa5m/65jPF3z6ySWnp3c4P7tk7Afe/9HPODqeMY4DqoDHKSu22w2rRQspsJw1+IsNDx4+InrP+fMLfviPP+fdr3yVs+tPaeeaO6sFVxebLz5xc7PXvQ2k3/7d7de8uj95HZB+26ry19lZvO49uLUnumGiT+xzbn39tZ+k/E3hzxUMRIiSBoUmjYndTiy43njwBs9fDPS92Ec1Tc3Qb6jrhl/+6pf8zd/8NX/47W/wt3/7HwkhMgw97//4F6SoeevtL7FezwlxFBWkFVXIZXfJP/zD93n+7AVjLwUSbTTDIHvJECL7/R6AECW/7NX7+7Jt6MvH7f3r9PrK1VT1ZN0XsMawPFrz+NPHDF//Gm8++hJ/9Vf/ju9879vsr/bMmiOePn/O//v/81c8ePiQ2XLL8cmJ3CvdcHU98rTbcLRclkySxOXTjRDqnGBUOQfx6c+RykkQe9tWuMpIgLDOJRvHoedVebQFI7E3GWwgGXOlVXAbQL/hdBeM51AfudV2ihqiUBw/N5D+uzDDM6qQsV89XkPIVEJufrXd/28GRBeblnxzn5UiIxc9+e8aoyVMsQAfRlOsXhRGQ2UV1ihI0uikeiMNxWhFVOBjZgjF00/J31RWGMvWiA+0dYqx1wKO+4xWUNUGVwubOCogZ/FrV+IvHWPGB4EER5uYMEetQBmNLkA+lJueQZfwwjhZmJRNXkzCyB5DYdkr8S7WWmO1sOZcuyAnOPvkGfP797nabKkrg65WKG2IKQARqzKbsxf84O//jj/6k29wdv1Lzq6e8PTpC7bbnhCihEIVZq+palQW1qh1CmPfYrmaMYwjH/zkl3zv219lvphxfvEptdVYlbhzNKd2lvXREbPj+yxXd5mtjpkdvUGzfoi2jbAoVODk/tvgZuzHAGrGf/rr93nn7UdUy2vqq5HR78g5UVvxqFeFkQjioeSAqtjdJDIkAdStLX5OB3uVYnJyGPClTRkNtYPaiqWOWKhoVGFyqixBZ8lACsVrNQgQVRmotEarwkjPUuCwaHxhhvuUieU5ugL2ayXPO0axLonFR6oqFUxrDVmHYksihZ4Yi40P0h/E7Vza06Fod+g7wnhVSMGosSKFpYB3VosfOlERizWRIjP4yHZILBaykMw+lnudcJXG1YCRz+Xs7+nnctOxmeRCh/8LZp1KQDCTGoT0EmA9ff1Nk/6r3s63fcdu/5343xZf9kMRtIDoWcDqFJMoDVLAmCQL1BzEE1y9DKJPCwsBUycW5+TZ/cqiQdtD4nZVqUOq93azYblc87U/+AZD79lcbcWv3HtyyqXgog+T1GSnMI2HMkFzYK5M9/vAhqwNq3XLw0cnPHh4wmpd45x47Na2KkBaICvFvuvZXF+w22yoqyuOT08OC8Kc5Zqsq/j00yf8u//1P7HZ9Tx79oy+H9AmMYx7drtLdA78i3/yT/mv/0//B/7ga+9RNRK8GGKPMYXxay2r9Qnnl1dsduK/SumL0327HboybdZvW7tMthQTY71pWuqqwVWWlGYMw0BK4if94sUL+n48AKDe+8N9nJ6RvLUseCbv07quCuv4xg87JcmxEIa9PvilC6v3pl1RxvwJfP+81e9Xj6mt3Z64J9b2ON74Tt9mCwsjIMqclxqsshgsJSUCHS0q/x5IWyz+rq6i0powemIWr7lUQPDtdgcYYbUWoHYCXCdLndtFH5iWARIGbkxVmOsC9MYQCDkd7rMxmtF7hn6A7Nirjv1+y76rMFba0NBHUrJUTYuPnq4fboG4hspa8ftjFOuO6MlAVVcTdwKQNgtSCDJGsVzMeePBA05Pjlm2jtALQ3qyqjk6Ej/9ydIFONinTEA1QN/3jONIVVUv2bgAJQhMH8BrEKsXay193x/OJcx1Rd/7A2AuQUU3BQDn3EF5ARwY7hM4rovdzmQx45w7nGOydbltjzK1twmIv81Imq572kCEEA4FL4BhLGFOORJzYPQju93upbF8ui9Tccxac/CX9yU1U/p8LISGzBRcrXIJb09SfJw2KjGVXAelD3YtuZAbJjADSjaKc4e+PF1XSonN5ord9prR9wLYOZHxpxQAyc+5Yd6LWqVualISqwZhljbUdSNjYuhBKayt0anFmiiFzZDJSfIzDpYyBKzTKAKEQBi9eK/7gZylSBPCSMpBAOTf40hBQ5IgKaMNdVXRNjXWCBtwZhuMrghhxMhNlDVTSgzJ0/fynLVxaOOwWYlaUgnTdLfdc3HVE8ZIW8+o6owPUQqyIbGYLZjPZmRkbElJEUofmADE/b47qMtCjIflRoweba0Uf8eRlL1YGvqBBNjGUdcWRWIcB3TSmCxsrKZumc1qhq5Da0cMiXHc48eusDJbeY5DJPlE42pa1xD6wDiM2Gwl0D6MjH7karPlanvFftzzSGfmqyU+jAz9HjszVNbhk1gYoIyEooaBk9NTukHm5fl6Tdx37EcvGTmuomodtm4xlWWIo3x0a0CX8SPIWmccOsCSsmHfedp6gbIZbRLONkLfyTUpVeRxRGXFOEh/7rue7XZ7GNNur0Fu94lpTLEWoh5pl2uWxzOO7s1IyVNRYWczUq8JQwc54bCoaLGtph4cakgoL8V2Uxl0ZcFoopI1mKotMWcSkT6MbPY7dt1eLGmUp1nMIWQ23YZu6Oh2OwkqXNS42kFSwqb1I6EPxBRotKGqFSGLYne77RhHj51ZIXLljLZiGWSrlmY+w7qICoph37E6WrGYz1kvl/R+wHghTXkvdjrWuBIuGsUSqLbUjWYYHX7cEnxgksfLWCl2TspAMpGsM66y2LoiK03WZe2KZhy/uJ1LXTtyVtRN8bBXmtZlhr1YmTx4+5jvfO9NmjawWFkuLi54EAfa2tJ1L6irQGMcs1liuznj2ZNPOV7X/NGffYW6Vpxff8TzFxecvdjQNBVf/cqxjIV4zi8uuT7LDFcLNuaC681HHL95zuy4ol6sMUbx/OyS1VLJ3mWwWDWjtg1JXUNqaCtFbTOX12ecnMx4496SECNPnl3y4J3M8b0V//jDD/mzP/0SMV1ggDt3FmLnp5PYs44jFLJa1w2cHM9ReaBuKqpWc3G1Z78faSrN+dmGrhv4ylfe5tmzc/qhY3VkOTu7om01z55eslw6FkvFbNaIMm22oh892+seV7XEspZ/4403iDFQVaK6Ojpa8vTZOUpF1us1fT9irEMlybxSSuH9QFU1xDCyWLYk5akaR9d3xOg5OTllHCL333jAi6fPuLw4I3hPO2uo5o5x8CznK4bRc+/efS4vLgkh4oyMndvtHucqqqou/d2LbanW1E0jQJ42vHH3lBhkno0+cuf0hHEc2HfXOAcqJyrtWDVr5u0J/+x7/3vur7/Cwt0lKovVDVpVaCUB3iqCaI9lrpI8M31QKIIVF4DCepV9WgmC12ArBWisaYhZMXeO/+Iv/s9oHTm/esF/+MFf0ukLrroLck7EKL70Cjg5WvPOO4948eIZu71nGAeMURyt1+x2PVUlHujbzcB247m+9Lz17pLEhs3mirY5ZnO9IaXEo4cP6Lodq+WS8/MRhTCJYxD7wMrBelHTzmo+/Ptf8SAfkcLIenWHJ598SrcNGGWJ48ioI0p91v7jdzl+0z7js3vpG0req8D2zWsy0zr4VVD+N73H6yv5r3+f33a+g30KVtb+2h72glorjo7WXOWEsQKQxhTIKdI0NbO2YRgTP37/R9y9d8z19RU//vH7fPrJU8gO52ZcXJxzdDJnvV7yh3/4h+x2Oz59/CnOOJyzhSzhuLra8PY7b/Hxxx8SguwVbkhrYgNolSoYUjjsxeAGeJ3Wh7dDuafPF0IQa+KihphUr6vVitQ2/PCHPyKOnnfffounT57z4/d/UuwENY8evMsvP/yER2+2OAfWVTSzlsV8ztXzj9gGU9bskq+jdcY6MGYiB2m0MXTA88fnWKcxVuEcVI1muWyIa0MKGVeJot1UipQkMFlreTbTuvvGHmhqZ9KG5B5MpGNeBspvtYnD7z/P8Ttg2r+LVUzOhdf9v1UQvbg2C+htNLqKkBTJy6BprUFrVYDIIAyzwt4BkbWK3VAqkl1E0svEFlaQwAcYC4vYaM3cwcxJOBpZvCmdm0ASCBGcUtQzRzNzGKUZh+IvHMuQn8TbPGUBwCXjQDZoRhmSorBrJ9BQfJGdESAjFhbcxL71Abox04cki71s0GS0Fla1tppsG1bHD9DOUR+tCXnE4AAjFtvKlI458K1vvMuLJx+wP/8lX//au/R9z3LV8MtfPOPZsz3b6xFZkAlwG4LH1rBaKVZHmqrO/PiHP+b7f/sPfPeb7xDHkaZtsXFNoyQw9fhkxmI1Y3F8h8Xpm8zWb9As7qJMA2hc7cipp9tdYauG1eoeD7/6TS6fPOf7//H76EEYt84IW99Z8TXPxUJDZCGq2HvIYKmzIqlM62TRLqFFZfOLVI6MVoQ4BVGANonKSvHEOS1eTyofBi6jSkBiSPiU8T4TvAzQrVNUWqxiBJNX6DJwaymTE4BQ2DhWQW0Vzkg1VCtN0gqLgBPG2sPkJUCXSAHTBO6FKDY9SX4n/ChVJCUCtmstyoUJdpJCkIDpKUjFUBeQKaoEUQoJMBV+xJ/daAhlYNMm0bQaWyvJJcnqIGP54kfxmD0Ai1OhgLJJEfZELjY6wqwQ+eKrbPTbFiqvguu3X/ebfq8n5rKyRR0yDYjCAknJy0Ju8s8tfS9G8VG9AVxvT/wvM9xfV5HPOYtNzAEk0hwdrdlstqAUH/ziF9R1zTtvv8unH3/K9dU1zlo219fSBtQNE38CH6drkfOZwkiZwEFLXYtNRDOrODlZ8tbb91kfOVADViesCWUyicIozqBV4uzFcy4vzpm1a/LJEeM4EJQijIl//ME/st96/v3/8t/z0w+e8OZb7xZv0QwqUqmKmBsef/gR/8//x3/H0fER77z7FsY5xjAQgqhZLi4v2e72eMDHyGy+wGqL7weGfpQ+XIC5CeC7Hbx4G0ifQDYJX6ypXE0InqvrK8QOKB6YvBOT3Xt/eE7TOW8zJZQS9mlV1SUrQB/Giuk1Ob2czK5uXdftSXoqDf2mxeftRe2ve82rk7csuCSMcxiGlzyoXz2HUk0BvyRElyzsSn6fhYCzJKtJRpGtTHY3DINi7VFk6MPQl2sWaxxpz8LamAByP3pZgDrDajUvG30pIFpraOsGY40Aq8awWK5o2hkhRV6cnxF8QmuLdYb0xDP6voDjGu9lcTtfzJkv5lircc6U/jEnhMR217HfS1CvFMAn+b0nxkRtFJUTVnXbNBytV9w5Pub05ARNIDpN1/VUdc18uRDbrCwtQEKTDTFmtLZUdYW1jpgCrqoxMpGhC8icSwFYgm4lhNi56lZbKbZUWgpcIUwMmHSwyvGTX3EhJviYCLt9Uf1oVAzE4AlhODSDKfS12w8FYNfEUJQxt8ZxAW5vGOkT0B+jWDS4SRUSxI4upYQ18rMQp40FGCvBeZP1FPCSdY33nqaRcFZrLdZVuKpGDUNZt2hUKYqmGAg5oFQsa4hEyupAuJi2gWTJEVFZFIwZse8LXmT4co9lTMhlXTQV6rTSOKNIcSRB8ZvX9ON4aHvWWjJKgk+Dp6ozlVPSjsKWGHZoO8faBVDuR0qQIEZFwoF2JAUxRxJJVJCpFMViQhFpbIPKipR6lEoYl6jbzGxhaBowv2f9e+gzwYv6JoYgLPzgsUZRobGheIUjRXqxqfH4GPAqlgwGmdfbtsaYiq7bAoG6NXQ99JueGKCqFd0owPd8IRL5s4sLhsHT1A6tEnU9o9t2vHj6jLqp2e527LZbfIycnJzgUyCLbBCfRlQyECmqLhjigPcjUSlMlrXA2PfEGOSPlNgsWTKLtoUgBUGTFNeba7bdnqwsSls0BlfC8BrTkIfI1fUVoQs462gqYXBZZ4k6sev3hKuImTlOdQStpQ15I0x8DNpqtLXs9z1GK4Y0stnvUeTCWB8LG9OQtSXrzG4YRdyuPNYZ/DiILaAP0jaibP6rukbj2G57XOkHzrQMXvPi8TWXT65JnSX3Cp0qohdWat/3B7uW28SB24C6ujVXKxOoWk0zr7lz75RqoRh9J2BoiKhZplm27K42+Ohp64bkINcKM2h0H9El8DNbkbD7CEOImBTQlWUMIz5HmnnD8nTF6Ec2/Za9CtSzufgL9ztRviTRXOkAQxjBamI/Qky0dU0KW5SuWK6W6JTYXFyRsfRdAOcY+pE0erph5P7JHaq2xfeB7eU1pnYsFnNyjmX9bZkhBXelEq6tSX0WgtBuQ91YTk+Pmc8bmqZG6xndfk/XjRjtGAbJtOn7Xckuibja0sxaslZgLLPFqtjDODSfHwh49dhu97hKbPO0hcom9tc7vv7tU/70z76Cqjz1TDPGwJg7jk4XHB2vGfYJcsVmc8YYrvn04w9xrqHbRdbrGV2/YdePXF32LOd3OP3yI1S2jKOM40oHHj/5BfuTkc2LGWfP9nzl0QNwilCdY5srKAqYzXWi2/bsN1dYV1G3mqw6/PuJsbdcvIg4m7hz1/FHf/yIN95ccO/+yLa7ptv1vHiSef7EMews2Y5sO8XDtxds9h4/DviQaJxYjymj2G73zFvLOCZmywZrd/hB0+0Mxibmc8uvfvUr3n7nEbaKnJ9vxdfetgQ7SBaHV9x/4w77/ZbL615Ap5R48rTjX/7L99jtLogRhqH4otcSZGvMyDgGxnEQW49ZRTsTT/mnz57inKFpoiiuQwaTWS4XbHaXhRAXqKuK7fU1bVXjmxn9sGHrPXVtWCxWxAjRj1ycX3Dv3j1OT4756MMPca6l7wacs8SocK4uc71mNluQMzx7dsbDN+6ybCpQlhcvLtCVQhnLbn+BMpJ5sGhmzM2SN4+/wp9861/yxtFXcGqNUhWmZIApncRaNyWyjqhCftNKbF9jzCgjOTVKl/kWynpRcvTUgVSWSLkToC2BHxVar6hURZMbXDjig09/RnA9Q5b1/DiOnKxWnD+/5PLyjPm8JuWavs8sFgvu3r3Hj3/0c/yYWa/mVHrGo/tLHjy8y4/e/2vaxYB1DcEHtHasVyvOz845PT7GGsPRei1ExSjEhcREErE09ZyqntP1IzlFCDvu3j1BIcozqw2XVxfweyhEQWgyIODfRAae1j63+KrltpV9T3mhKi++vb+9/fvpJK/udSmkvul3t0Hy20jpzf759rl/3XHrHLcwAKXMQcGutGE+X7Jerfm78w8Yes++j0Wh2NPOW64312QCP/3Jz1kul+Ss6PY9OcJi3jL0nqwN282ejz58zPnZFQ8fPmR7ucFax4cffsgwjMSYqKzj2fOnhCCWj9pYrJPA4H3XgdKYArpzUGC/vCd7tZAxWcF470kpCiJT7ExtUWA8fvKYk9WK9XzO48dPePb4CV/+6nt8+StfgWx489GM7aZnGBJjF3j/hz/hS2+9w9AFlss1bz58F2Li/PycoR/FgtpqtEo0bUVTzzAqkcqa3egFY+9FPes07Wi5eHFNyi8IcUDrxPHxmjv316yPHfNly2zeUFWWFHsheqiAsNVLFmSWDCB1uzXlKW0uwy1StTQPw42v/qt71dd9f6CYvvSFl35a3v21e9/PtsTMRGKT717++vmOG7/26VP/9uPzM9FvsW6tMTjjhPEdxONTF8l8zkkGXRJksVvQSlE58caOIRLGcACcZHzN5W9UCV+UTm60sHbbSglIbSLGFLD1AG7KU6xri6sd6pYfeopZQNMIiYnlrATEtJDjBEzHAvzIJlY2r2XjlgXIjElAghATWWX2PjOmSMwSUGBV+Xw5ksMgcj/naI6O6HY9urGYnCHK5hmtDg991mT+5T//NoO/IJqKf/LPvsG3vnOXZ0/P+PnPnvKzX10K8yKK8cTx8YrlsuWdN99guz/nf/mff8H5Jz1/+mdfY7GE7dkVRhva9R2c32NtYtYuaGYzqqbBuAq0JiYJqESB0lo2DMB2v+f51SX7oWccB168OMOGngqRdBmjqSuk8ozI1CZGeSqe47o8W2MUtYPKakJUhBQAdbBIiQVQtFoX8FzUBrOZZbZ0zBct2mV5XilIYqmSQTn4wDhkRh8hZxqrsDoT0hSRkA/dISNBoBPgLJsNea+2sdS1u2k3gLUybcUkgCymDCQxEr3He2nDPkZ8EvsimVwhggB3SowjFFIwAPEMNiVoMxUrg1SeQUyZrCZlgwBczkibNaaA6FILoG0dVVMX5nMgpC++UIcbwFxYnC9XtZU2KKE7lH4u15dzLIPpy0D4q9Xqz8NMvx3aATLGaG3R2goLPqYy9wt4JIzzaaibFA7CYEhZDHtuLGp0aZuTdcdnoy2mS54smeTfwlr0PnBycsJ2u2M2m/P++z8heM+DNx7w7OlTtptrUBk/DpgCwsGNpcv0+WTBezMZCyu7PoDMrm546513ePjmKeQ9KRpyGsQ3NSeCH4oHWsOssZxfbPnwV7/k4aN3GIeANZIV0PV7fvaz9/mHv/8Bz56esx8yIdzhjdOHLJf3OL94Tt2seP4i8uTDxDB6zs4uZEPfS/8CxW7XMfrA0fEpISe+9KW3WSwXPHv8lF/9/BfkvGH0/sBmnRi38/n8tc91AttkoWrK55fNycXFOVrDbrcDbmR1UxHikDBeLLtyhlnblsBJdQAHh3E4MANAFp4+CCCfiw3QYZq8pbiQ1p5fWZT+5uPXW7e8LM8UZcBI1+3Z7/cvWXwoJcoE8bQzsonBobNDKVvmpwHyF/dNDs4QFXS+Zxc8vvTZgIBns3bG0WqFdRXdsENrsaeY/PvF/1tUBgpd+kWkrioqt8b7QIzhANJMzO1cnv9ssUApix8D/dDhfaRyNUkZ8dbuA9pkqqoh5JEw9NhKs7ZLrKGMzZpZ2zB6YXp36mb8mEB0ibYSUEplRVs3nB4dc7Ra0biKNAa0E+WOcQafIrv9ntl8LpvBDF3XY4zFewkv7scpkC+VIqqMDfu+K8W8UpzJqhAGbtrG7cIN3ChSbhfZYhJW76RGEUFEliJSjLR1hdUZP46HooYrBQIJ0w2SIeAc4g/phR1TFvUTY0b6oLznVBBGZbRVWFfYZrY6rOOss6TkDkUo8b/Xh/Z9ewyfxrvpa98PZJSw9JQ5qPnI08Yvk6JHa0VSipwDFl2smRIhTbkYpoDrxUoNUIciZAkDLzZvoVgLeT8WixZL9NMcoUBbsjLEJOA2KqNMYehGX+bygFI9KW4h75H0hEBKHqVbrFHCdvaR4MWaxlSWrBVJJUbfoU0+tAupNGeGGCGr4isbqRtQRlHVCesi1vx+c/fQBcmgMJIPs9vsycXHdGZb4mYg6kzVVrgkYZhjEiunkMHZikplnHFUzmJtCaxUmYQvWUMO6xy6sngiKifWpysG77k8v6IfByBRV5Y7xyf4eMZ+e03X6ZIHFKTIULziq7bF+5HNbsvKGEIM9IPkEegcpE0CKXugFnAhRZIWBtauH8h+wBkHPpLHzKyZFeVipm5rMrpk5li0Em/b88tnXF5eU7uW46NWgumNsIpn6wVmVlM5R3BwtruS9ZCizB1l3QigMiEM6MoR0kBlAW0kMF7P6bueyjVoUzPGkcRI13dUjUJn2Pe7sunMWJ2JcRQ1bkgkVWwjs2LYwuNf7vjoFy/wG40KDTpqTM5YZ1HWMQy7w3w5Fa6n49V11YHgoKSfkRUqanJIjN5L/9SAGmmWCyrdkHymqmcY7bg6vyToUVS12qKMZUyZmCMBGEPAxEyrFRf7KzKJdtmyWguIbjrH2dUF/faa1WrNfNFiTcIqYYDnmCApDI7sFSrBvKkZ+0HmnONTjk9PyMHQDyPX2x06St/ab7cMo+f+I0VTz6idZC74YRTGox+pnGa+qGEvez6ywZqKbd6WNb9nv9+wWs2AFmMci0XFfD5jvxvo9qJQkqwaserJZR6t24a6mdOPkflqzeX5FRVgJ1bMFziuLmU+drUmE2nayB/90UO++fW36LuBiKIbrknGA5b9deJXP9nxkx+c4Sr48lcXLFawXrcolXjw4ISnT7f42HO9PycEx8X5NTudWC3WzOcLnG2Amj9474ToN6hs+clPnvH9f/yIt9495eh4hnWZxi758Oc/ZHM1MOwS3TazPMpkDSGOwgr1jta1NG7B7mzg3/5/n/On/9Tx8L27rJol2kdWi4G/+5ufgo8cvdHiU8f5xZb50jFkT1Vpus5TzzUP7p7y/OkLRhOZL1suL3e8/fYDzl70rOZHXFy8YPSR42XDdrtlNpsRo+Xi2Z7VsqWuWrpui9aZ58+uqBtYrys22y337rcsloZPH3/EcllT1S0+eOqqLuCKqA2cqwgh8s4773Jx+ZSu2xKiYT4XQlrwiYvzPU+fXfPgzRbTe5q2YRjERub66prlao1SmpPjI5483bNctmx3G2Zty9nZJXVVMRZF4IsXZ0IIaBsePnjAL37xCy4uthwfzwDF6CW0e7ffM5/PSSHhTMVPf/JLRp+o6oHZsmY+n9H116wXDQyJ9778Tb78xve4v/oaVi9BibpNkWW/qrWMfVkhI5IqxXspeuesBbPJYu1iMaASWbzqMAVdikU9nJQAps7VoCw5gk+Kqp7zvW/9OZe7Z/z8yQ/pTKRpRV0YhoG6MgxDB6qS/pYjp6cnpChrxspk1qsFei3s46ZJrJYtWWWOV0eMoysFMC3kzhRpasfVxRWL5Yrr7ZbKVURgGEauLrc8uv+Ao/UC6powjjx7es69k3u0bUPYbzh7cYlxFQ8ePvjCfRtAH2xUMmjZb4tSP92A5QWclHlnghc1suKVf6tDCxUMi2K7x7RHuYVFHoDzCaZX5f25/bWA94etUPl5Fkvag31HWYff/FmxOy7nnEDQpnaMClZHx+y3jo8+foa2DVpLv4wpEKKs0ZyzvDi74Kc/+YC33nyEToZKO1TKqBjRSuO7QOsari+2pPFj7LtOyBkp0PU72etTUUdHzpGUpOg7KUKVzlKUn6ybbzHObx/TnDopQaefKaUYRyG1HhwMCgBvjaZtZ+XvHPdOT8klO2M+X/CTn/yUe/dOefzkUz74xa9YrtfYHGiNgoId6kpzdDLn6uoK73tSkH2v3w4M40Bd18znc0JM+BBwVVNIKhCCJQVIuSGMA0M/sL3yvHi2oWkMVb3l9M6S49MZd+7OWKwsSvck1aH1SM6hPOcKsi7EtJuCymTfOAHvGVC6QhSV6VA0y1OjU/mVexo5IDfqVZXDK+qHlA8A/m+1jMk3BZB801x/w/Gafb7St9r85zs+fxmtGIErKL5XRpISCcUrktJhp4uTzhvLTZQQTvBDZBxGkfrom81mSpGcNCFlfJqEBAIsCjA7hbOVEEMkVEJkXkIjlwWg9PxcFrwxihVLLBLiyoJzAqjHNPkDlvpKKkz6lLFaHsjoI/0oG2qj1cG/dgzCXktZOlEyUpmxWkkA0Mkb7HY9djPQNkuUqcTvz9aycUyBkKTBphxo5w3aL8rCZs9s3nL/4Zp2DQ/+YEYfB5xpyckwb4+o6oanzz7lanjGV79xyun3vsbJ6ZLN/hOSytR2RmUt9axG6QHtFlTVAmEeNjg35+CHlFMpHmT6KEBzt93yo7/+Xzl79qw8h0TWAqDP25r1nRXzOzOi35H1Gdp6YlCELIz/iZ+sAOekIDIWb/pERh8GdNlEzJ1iXikaK5KVpjW0y5rZupX7qiIqRUo4NQA+iJTaF796q8XyZ5pOUhKmd0KqhenW1JTJVEbT1lZkmM5CygRTwMhi9RGjnMNqChsuCrBe7FxCkmBQpRROFwZgVgQlHuJOKSpk4yQDQVl0o8r1yECTgljNTLkCIWcqjcghi5Qz5xJ4ayPNXIpGKYhlhfdfXDIKAqLrIpt/eSOmBMAwAEbYR2oqS5TBdGL4vgKW/zqAcfr6Kph+24NXG4NWssGNMUEOGD15CQfZlE+gDJqUIyR9ANihqEK0XK9MfOmmQv4aFrr8G8jCOJwAdPESTiyXS7bbHaend/jJ++/z4a9+eQCkvB8Ok8UEik9S6qqqinewQqHJScYictk058xiOeP45A5/8LVvougZuoTTCqNa8YdOI5lADAmj5dxHqwWbzWUJqwsM/TWzeU0MgbfefsQPfvB9dvtzrrZbvv8Pz/h5u6SqGq6vryTcbdwzdp6KBX2/ZxhGstLEJOzwZ8/OAc2Dhw956913WB0fsd/t+OmP32d3LeGKrgQrThYX2+32wEYdhuHwbCdf5unnQ99DljDo9XrNMPRcX1+R0gQoWvb7/Q17vDyjKbyprmtmsxltO+Po6AjvfXl9fQDRp2BgASTlmap8W+Z4U3XOOXNIHJ5++hmW+I3n4O3/Pw/oLtYfwka/YdRT2gjkJEDh5B8obVeRC2Xzd/GRe/Uw1hFCpBs8+24gxEjt6gIiaBaLJUdHJ8QY8HHEGEmx11oY27FIlSXZfbIskcK3KRYr3uuiRDA3i/XJsk2XwhyKeTvDrh1VVZeAUHcohoUgDGnrpLhiraHrBlKM1E1NXbnCAhtRJLGwMka8LaEU16WN9Ckya2uqSnzXL85ecJ4SxmWslXHm+uqKq6srjDFcXFyw3+/p9mLZYbShql1h5IsaYrIq8V6AlMkLHSD4QAo3lkYgYaNww9iGmxyEaVEu6r4ppMge2or4y2ecMbS1LUz0G3ud22qfg3WDtaIIK+POxEyFmzlxsqkyxghjOgZSylhXHRjt3odyjVBVUhDpuj0xBuq6pq7r4pG/ZxzHoixpMMZI3x6FSVzX4ns/7saD7VXf98VDv/SdEiRuytjvfaAfPShN5WqUsbIBDKEUu1TxV0/4vifjaHNFCPlgRzO9rjRCKcLeUgSlBCFFVAilL0plOsRM9CNaeVzZ24boSb4T8oCrsUaRohR1UUpCeWuDs41I56MMItaUQkcaDu1V1JWiPJLQ01zG8y/ctaW9BFmDZ6XZdz0xCKO533lWdcIMClc7THD4/Q7lbFEgOFKeMiIEkBh7Txc6MgFlxEIlZnBti1JOCgZG1kAqau7eP5Y+FkQ5aq3i7t27LOYLQohcbTYorWlnLVXb0Mxaus4QvYWukzFamYN9ktFirZhywFixPtputigC+64HPO2iog+eOIw8e3HOqlmwu+7YbEf2aSSgmDczUlbsrzuG/R4CaFeCxyPYSmPaBu0MIXlyThgrQafNYoauKgnOHSSI1GUJiFXalucchQlfVyzaCq8C+/1O1oq1pbUtKUk/XM1axgDj6OVOxyCEC+uIMeCstM3a1ISYCaNhpud0Z54PP7nixcc7GGfMzQrK/NwPe2KI+NAfxpLb66dpvLpdhJ4K2UJeMmwve559csUbD+/hYsST0DZjnGYYR/z1BavFmmZZM2sW1K5lHwPUI6t6jqtqtvueoR8JUSw4NYaIJ9hEMInZTJ751e6aEDxN2/Kl5Ze42FyBitStxVUzKqvxSQgqylrAYGzDuN+Lv/U4ELMBXnB6/AZvvvkez56/4PKqY3e1FXaujwTvub68kiB3bZg1Dfu6ou/3VI0jJY/VMGtrtLEMnZdx2yoJsgwDPu64un5BOxMV0n4/0LY1R0drQjhnNmvRWtQtWmcqWwmQGSW8dzmfY6wjh8iw21A3X9zywXto5g3oEVtr3npnybf/6AGaa4bQsbne0HlhZz55vMPvDd1FxWJeUznFBz894ytfX/Dxh8+ZzRfcv3fM5fmO60v4+LHh+SdQqYGLZztUesHp3TUPHjUc3VV8+Z13WbaPSLHnj//4mGrW8j/+T//AvYcnvPnmQza7wG6jGXpRbdsFBA9+C23bFJAJnAt03Z75zHL9HP77//tPePPLC07fuMPquOXi2QW1i9hKMlXauaYPI027YrcbqOuK2jrGHKlrV/IrEtZUVK6nH3bMF46qHjm+U/HBB89ZrY7RNITBsLva8+jhPc7Pz3jw4CHX1zUx7bDG8eTxU1bHlqoRmzhrA1WliMmjdCL4kThqcg70447VuuHqsufunSPef//ntLOErSKOhq7r2Gx67pzeYz5bUFU7fMgcHZ0Uy8dEDKKUzdGz33vapkXsvcQaZLvdorXixfkl65NjRPXt2Icttqp4/PgJkGkbCSpv6hnOabbbPW3b4upKlP5RE71mMVvSzmfs+mvmjVgDVdrwpQdvc+/obR7d/QaVvovCHkBuiysEPw3K3OCsKFL2ZBTWVmhXyf0JAWML4SNKP9BACr7shQqztW7IMTCOEkifdCTGDuVgtVzzp9/951yPV5zHx2yGc7Ebiz05Z7puD0rs0Oqm5uryChDFyCcfPeb6+pqjoxXojqw6Tk5XbLbw/Nklj9589/9H3J/9WJqn+X3Y57e921liy8i9tq7u6n16ehnOwqEEShxRNKwF4EiyYNiABBCWL2zAvhT8T1gGZMCSfCNIkCj7xiIN2JSGpMgZznDYw5me3quruqoyK7fIWM72rr/FF8/7nois7jE51RD0FrIyMs6Jc06c89ue7/NdyJxnVzdiLZICeXY4WnAYhCw3YDPLk2dXtEVOns0YhgQ2cPv0FpUtaLcDV1cXmCwyDDWYyBDbTz235boB2u7Parzyvcn8ds9WZ/riJsqt9vibnPmCgOj7On2qc6c94hpE/3nhixMh9meAxhT3Zzc5mKkRWFXXr23EQLRSqCjWPpJrpHE244c//DHn5/L5WZfLedQZfFRsthucE9C63tX89m//22yu1vzgu98n9IFi5ogRWu+xztF1Lb01PH/ysaiGUuLo4ICLi3OIgWa3xRq7f80SWtyONfiEORh0nBRc1+QofeOsPP252Yye9takuFHjWrq24dmzZ6jTUwiRb/3VX+HNt14HDXfu3CXLcn7043fZbbcc3zpms97y+NEjFocHHB0eU82qUfU5jLWKG9nvfv/c3gtxYm8rOeIUIDmQ1jqMUliTk2djTtXgaZMieHi0e8n7729ZHjgevHbM3fsHLA/dON4U1gk2gQow1dNKc41jayBjwoTTCHqnpEa8ZRyjn2zC7MfWTdzl+u9Xa+lPkDGZPCZ+/iX4ys+vxX8Wi1I/8/jTY/x5rz8HiC6FcEoBhRRbjJ7VyUeSDyStifG6u5CSEvkrEuJIAu/V6LkmB/c0+qaGGAhJM8REH4TdPcRAOyi2XaTaMxuBPXtcCzM5JfwQRG5pJkYxoORtl4IljsWxgBPisxmvu3RegMzJf1xrAci7PlK38rsYLayCoCIJjVERrSaGLOL95QxFVbFarTi8VZLN5vjBU182HNw+liIkRSl6SITYEUJHjMKWyE1OChKiomIis4qjRcGu8QxtQ57NaNYfs+o3OJfzmTdPyc2M2FyxXW8hgtMlxhSUVUamDE5XYEpMcUg5P0YrQxwGUJPXnENZizEWrR3V8SlvfGXBnYevs375gu/9we/y8t0fooeWrCi5de8280XO/OERsb6iqv+Esn7Mbic+mDElwggUT40FpYR1PozWJ9ekvYjVitIpCqewRmxinBNw21hZ2Bi9t8VSZ/TXUoaEJGtHybMSSwEjAGo7AtM3mqhSVCtpxORWkzuR8qIY/cbTtV3J2HwRL0SFNhFUhBjQSTzgUUigCgmjJgZ+YnJEzBQ4lfaLE4xTV/Y8khJp+BDFx7008vuGmEgGnEtYm0Bfb4rOaooywzlD5ye7jF+MzWamdOkxp8DoBGZ862IY9+Jr8G+6ZAxHzFSofWIh/LOA9Ov7TGDkJzfryb/cyNOPdiYxTsE+N33OJb9gYvunGMYxdh0hOQ2CPXhzYzFOY2Nm33iLcQQPRbI4WTD4QeRW9+7dp20atpsV5y/PCD4wSZ2mrvUEYN1kJTtrQUkwrWzoslHnzjGfz3n7M5+jKucEr5hXGU5HumZL1zQEXxNTj6y5EuKb5xlt63n+7Cnl7BRjFOvVGj+03Ll7yr37t/n2t/+Q3rdcXe5GhpG8I5MnL9GgS5FJN01NNyTqZsfl5RXf/94POLl1yluf+QwPX38djJJU87bjxfMX1HXDerOmLAqZVzGyq2vx/b5x4JiAxbIsJQhpfE+GoacfBDDM8wLnamIURvv1z0qDdGLWTszuEBLVbMbR0SFlWe73kaEfCDqKksGIUmYC10KMGECbiVF7PTTS2EhUnxiH+/G7P8ByY+z92WP85435T349sW2tMdgxPHnaO2VMy8A0tvjneo4/6+o7CWDtO4/3QeZamoKqzd4OIMVEGDwaix+8WHz0g8ytmOj9cM1U1gLsh7F5l9kMLPvPQCvN9J/vPV0nPq23b9+WIM2U6HpPUY7gY0xsdztijFRVRZ6JTcFu04w+8j1DP0jRNoaOTqqEGKRBJbZcZkyUHvflGBj6biwsA8Yl8syhjeyBYRhomxbvB2HQj8xCWZ/lLGCNHB61UqgEZV5QZJOtkhzGTJahc71nq0CiKLL9uvxKo1DrEdxNFNYJ4zpOa4hky4jViCj3+q4hy3Pm87kUln0vjYU8x40NusnD3mY5LsshXed2OOcw2jB4TwwBhcZoi80sfdcT+h5jLC7L0UYTU0foxbtfW4cCrBNziQmMm9a5a2B6Uu/EfWMxRD+eL6Tx2XYNu92W7XbD4GV8aQ2Hi2r0GhYWe920oDRUiiwXFkxIYb9egoCswr6W9aUcmX513YzBroncyqTthwGlWopihrHSjBE/68n/3YoKIiSGATk32gybOYiGJK4e5JlFGYWzOcRMfs8AIGGl1jhh58K+IcHYZE/Ry3kiSrB13wdS9GhVQMo/9dwGYfboUYfR1A1dN5C5it26Y6023HJzjsojrnYbdlc1wSaopv3O4oOEfxlToJKAHEPfUC2sWDImTZYVaJsRkiekliE09I3n1skd7tw7YbPaoZVY9VTzkuViznoloaUBKGYV5WJGiJG8LKEoCVHGUFnNiGGLSi1EhE3WtxQzhc0MPkZi6NltW2ICVy7l/Kg1q3qH9+D7iO9bUp7IqxKPJpBow8Curwmdh6tIbi3zg0O0c0SrGYhCyElB1Ay+hUzjZjnOlnR1Q1AJHRJGokCEbZ4SJ4s5i9mM0Pf40BOHGudkjGTWYU2BH0ArT2BgXoovvdaWaHOCB6cyUB5nLKVb0LcJOsPTDy548tEFQ23xLcwzi5qK5DjgSmlGC5Ar68q+UT820KZiW8ZI2gceA2IbmM158WjH981P+eZf/AIqg2QHTK7pQ0Yk0QfJHAjK4qoZB7fukPqO3FqKskIPA2y2XG42o41VwvcKjKY8mFFVJTEErrZXkjEUMxbFEjSsmzW9D2inJT9DK3yKNJsdNk8olWOM7BO+E+Z/ipd88P6HvP3ZL3D/3uucvbzkxdlLlvMKHTWFzanXNa1picPA7aMTCmcYuo6XL56jckMXekyWoRIEH7FYZvMMYxTb7UDCYjNQKo7npBrvW44OJVC6LGYMvmM+zLBGcbA4YvAeazKSD5JF5TSHyxlXvqYoPr3lg0pOmmwu8vbnTzg6dqzrS4Z6x9XljtlyRqYjm7qjqnJmh6fc+9IbWCpeXr7k46dP6fxTMut59PEVSkNRZTQ7R+xLLl7u+PC9C5zJiFHxo/fOWB4pbBF4/f5zHt55k6pUfO6Lp3z+s19he1XwO3/vd/nw3ReiFgjQ1QlvwOnE6iqImmVZslxWvP/4Ba+/NufZkyuePo/cPblFJLBZRc5efojNE8uDA5oaFsuC1osVUgw9KkKeZWgUXT9wcjviXMvhYc76qh+VxBkKhTae5y+ekheKN986EPWQ6VkeVJydb3l5KSrJx08+4sGDe1SzJYNveG32kKurF3RNT1FpFouM3bbncHnA0A5CconiQbxcHGNNYrmwpBT5whff4smTx/R9y+rqksViTuZEdZhnmocPTrhz+64oP3MtOQQxspwtWFZLLvsNCTnHD33L0A9sdxHrDHmuiL4jDIbVVYdKhr6V83dZVXgf8CHRe03Xa0LoWc4KVO958eQl1IHXX3+Ny80Vt+4c0X28I6PguLzHV17/dX7jW3+Fg9lDUl8IUK71vpGdkh7tSbX8UYo4st60skIWioF+aEnBi+UqokyOURR0QgIQK0A/KsxS32GsEdcBJapro8dsKhwHs1N+41u/yd/+B/8NmZEzS9eKjV5MPc+eX/LGG68TY6IqK549e8bdu3eZzXIWy0KaK7GjbRvJflKW26d32FytWR4d4kd2bKYdZVVwenzEsxcvsM5wuCzp+oY3Xz/m4uyKbug4vXPIy9WatmlZHB7jw0uK0hAJnNgTOcfU6089t+X6WQb4q2f9ETsYz6OTSpg9CWgqTPSrj5QipDB+fYMwNAbiTkD6RDL4Gaz8z6o3UpLHfaUGGi1euckmlhqDkTQ2hIBzFe+9+wHf//4P2ay3hJjT7AaxkrOGppO6cbGYU+UOQqDIMv4Xv/1v85/v/nNW51fUuxqlNU0vZI3jg1OefPyM/+P//n/H+++/x+/8zn/PbtdweLikbnZiwWgMg5a6e6rNtdb4EOGG1ZkQXcI+N+ST1qQ3VV1aaxYLyXfw4XpfDaOqNHOOZ8+fMcsL/rP/7D/lP/w//Ye89bm3ISa+/JWv8OTpM46Ojmj7gRcvXrI8OKDve54+fcrtO3dYLOf712StJcsyuq4jhEBZlnRdR9u2+6DUiaznnADuE7llOhdIUz2jbmpSypjNZ8QucPbiks1my2ZzwpufucfxyRxrIfkBazsSAfD733uqHUkGsTKy4wjoR0LQ1PSRkbhvtty49mOPGzU46hPfv0G23A/Q9HN+7sbQfGUYp/39JsX3q9f/BCC6VobJLkH8b0Z5sh/ww0AKoxRisvYYgYkQBTB1qNGL0xCjFC43/WtjFPDVh0Q3ssFJgXaAttdYO9kzAEk6mcYoMBBDoNmKr6orzJ7lO9HTYlT4qEaDfgHCrdEkSZgkeKTQMooY7QgiDgKidYmmFUDUGYWzwrADAdAzo7EjAJnQ4HLsfEHwMvg2uzW2nFHdOibpjBAG8dFjAgBFWpxUko6PMlhtyVWJUoY+DQzbLUNd0/cDfdOyqzeIP3xL3zT4rMVgyewSnTIypcWbk0RVzSBqmN0mZUtQJTabofMCpaUzHWNExYg2mllV8NHTF6y3sli9/fVf4e5bb3N1eUG926FsQVEUfPT++5h7Rxy6QLC3aNd/m7Z9DoRrgDJJ6JybgOFRFXATkVKA00q8n6dUToVIk7UhIYWZin5kRwljNHixDVJJ2OBDELmtjeCswigBps1koazkc0vxGj41KuHGIEGRoEkjRjqSkRhG2w0tdipay4SO/lr2o6ajhkpYJd7cxCSy45TI9OhxPkpgxB/35pKg8BGaQZQZxdRJHhtAWoGzowduEr8qayVIV+sR6I9iM/SLXJmxMqcQqx1jLVGPoWuI7YDSY0MkWYgCcsU4MHJn0ZOsLIkUGcV422jHMobOiAWM+N17/I3uoTRD5HMHVNynVstCLhZC1onHKSkJ489IAI5WDmtK8kyAH3FBScQxADUE8dgWy5dr/+49eyvG0UN4YjXKgWMK19GZRZvA7dunNPWWzfqKsxcvCKMEa8/EDQKGIR+5jCFlUFqsE7rOj51wcJnDuUwYFs6iEhwsDljMc3a7S/q+w+YBbQLDIOz46AWUcU6sC549+wnHt05YLBYYY9jsNmAttx+8hjeWeiO+jjF0o1xTPpPcOObHRyib8+Az97msV3jfsVld8PzxY/rtBV98503uPbjH6Z0jjMvwPuIUXF6e8+LlGdumJWlLlgnbuZovabuWLC/Q9QYQdUWWG/LckmWGLM9wucFoM/qe91hrODw8oq53tG1PjIOAyEhwpDUGtfdzBmUc5azkjbdeZ7YoKeclWVFcq6SU2EqFKEHBMUoWx02VhWyuiTQ1U0HYNz/n/Ci+gq8C6NPfNz1oJ0DxJkN9UndoLY1KaVbKgcwYjdJGwsjG+xlt9vNpCt/+uaeGf86rbTu0NhgtFjEKP/5OmhBhs9lwfn6OcxlqDFnumo6maQAlfvMEfC9e2Hme46wbWZ3SLMrHw2rXdcRxjU8xEVKkbVp22w1GVRR5Ie9REAuuSbWRkjBZ94evGEd7FPm673uUVhwdHWKs4/zigrZpUUSMmhRAYgXgrKUsZhws52TOYI0mK3OsVqQwiCRYwaKajTY2jhQDWWbJiwMYwacw9FhtyLISrdVelVLkuTQCRrn1ZLEyhRpNTOuqqvaH2AngyvNcGDSDgMBOK8pKgiv7tpN1AWkkK6XwfUuey+HYWYsPAT2FIHFtoRK8l9vGdSeGKOtESgQCUYnsc7Ld8YOEIk0Hc2MMZjyQT4D8VEAwNip8378yDIU5fg3c5eP7opUmy4RlG2PAWIOxItONKbDerNlsRMmSGYNVp1RVhbNOws92W7R2FHmJ0ZoxGAU9fc5KPMwTkSnQXo97y1TgmHGf1aOEe7KRyrRY3xCUvF/7AsmQ6QKj5pDAWLE98VH2LWmyD+OeHCjL0WO97oVxaRn3ICM2gn0kGbG6i2NO0NCLbUwYgCTF0WJ+yPHx7U89t4HxPCnNfessdJ7gxeJi3e24fbKkKmacXVxhTYazOV0Uhl/fRzRarEGUIXqPSnoflKqYCkmHycVXvW5ruqElz3L80OEKRzUrICXa2hORANm+7yiLkgEBhmIIhBTGz9NSVHM0AtoYbcmzAj/0eB8pioqqmhNRKCzbTQuIHU9T16QE8+Uhja5JUZNsHBnmlurgCE8iKLGsKuclbmGoXI5VFq+gTwmTu9FWyNDudkTv6cLAoCLZoqSaz6R5M3iU0nTblhwDqcMaJY2+uhFWuApkOlEWjqTEyk2riNUw9A2+q4kj8ejo6ASWlrOzSxLg1YDTFUOT8/H7Z3z0wxf0G4NNM9SQWGQZyfegvTS+YkvnPUNKqEGAxJTEwmnKEbm5D01ZFtN5R2ylFLG3GBTPPrrkT/P3+PJfeJ35QYmbyVlFwr4NEUPTeYohoW3GECK7KOHublGRmcSiMPLZAmw83g+4wtKHnixzVPMKZy3OZhilJShRJXwcxI4qc5g8gxRorjboQZGUp6pm5NayTVc0zRUxNjx+9DEhGr7whS/x2oPXWa9WtE1N5iyzaoayI1OvrtFHJxwulmx3O85XV+jcoHNRMGWF5FOFFCjLDPDYDMqyoixnGKcYOlkz26ZDYdE6Yz4rGbYtzkn+R2FzrJZA86HrCDHK+HASXpllnx5Et8ow9C0npzmzuSUx8PLlJbGNWFMRY4GIIDVWB1LsWa0vMeop6+Y5b74zp5o95OpqxWwhgeIvXvRcXD3m/oOCz3x2wXa7ZbftISiSA51rFouKy6uWZ0++J4Sk/st8+N53sCzZvIwsTwyRyHrVcHIsWSXWeU5va4Y+A295+azjjfsP6bsL7tyteP11CcvedULaubU8Yls3xMFxcnzCxdUlLg8cHc0giUpqPtPcOj3i7OwF85mmyBVdo3n9jbtUleP8YktmjNgIGcNu13FwICS5tl8z14rbd2dcnO9YznNePN/y5PlT7t49xI5ZXcY4Qkh07UCRB26fHlOVC16enwMDy+Ux3kvT2hjJ5Dg+OWazucTZnHt33+CnP30Paxx25miaLdtdw8nxkjB4cmtp6zWLgxl115JnS+qdp647lIosFgfstldoHSjLgtVqI6xya1guCj7++ILl7AijCw5vHfHi7COsdfgQ2dWe2eyAFHOSTzTbHUVWEEOinDl0ueTRx+8TeygXd/nil77GV9/8l6nsPdKQCYM8xRHTGO0qtFh2yHkqjU1/jw8Jow0GwWoUSjJpVBoJRy3BS/5KHHNTjDEYq+n7FhUTICC61MFa3AzCSBrrPMrDG/df40/f/6fk8xw9m43ZdpHZvEJraTgPHspqzmpzRbXIWW8uOTx6nZOTE87OzkawdFT9GcVHH31I0IK6PLh3n7atqXdbQvC8/uZd2m6LwVNVFbfeeZtHjx8hyt+G9dqQ2RnL4wVNd8lqdUHfBTK9JNPHn3puw02w+hPA+d4BfQQhk9Qp+7vtwXTNJwsWIZyJBeh0qem50vSvCXTXI172s9e1YvfG60phbJZM5/Px4RjPaJMrxHgW832Qeicp1qstq6sNRmfMZgesVp2QHxMYa8hzQ9+17DZbul3Cac32asXnP/s5/vJv/iV+9IMfMXQDV6s1l5sr5os5m82ab/zSV/lr/7O/xn/6H//HPLx7h92u5i/95m/wgx/+gAcPX+fZy3OePX/Bhx9+hMmkEWqtg34QO2Y/MGWu7e0Ob1iIvvo58YrlSxq/3iuKx/etqRtUjBzePSIOA7//+7/PV772VY5v3+HxBx/yne98R85cg2e+mNP1A4U2WOvYbjdoo5jNZntQfBhtU0MIezVnlmXsdjuaRohGR0dHFEWxt41smmavEO+6TnKgXEHbdFhjQEmmyi4MPH28ZugV9+/f5tbpEdXc4lRA6TQev0eLbsVIgNbj+ButE9TA5D+hUhrtBF8ZlTe+1tOguTGep7HIiH1Jza32d/okbviz9fBkIzRWn/sGkvoE8fP6CT99TX3z+uf3RB+p/CF4Ymr2AU++7/asyxQn30/p8MQUuH4rI5MHNCPbN6ZA6KP4pPpAwhGSGn0wPXosimOUTlwkEiJjAJWcxUNMpCFSxx7jMmxWCINr9GWPI8DoY8RMacFGkTkN0WK13vtHBh9H6ZF0VMQOBDovb7ex8meSImRGkTtHnhmCFzasdRnFnc9w57WvU9x7E1tUuKoiGiXFxdT5YvQk1VZ+TwUT00srkb1ILzgjswUhn5PnCR89WZYJ+5UIo8y0zOekZDE4LE5kpyYT5lUxJ+iC+eHraG3pQ8QMA0YbIe0ltQdRm7qmb2uuLl9ycbFidXnBlz77NvcevsE//e53Ub7jj/7ku/z0vff47b/+r1HdvsvD2Qm78xWb3d9n3Z6jBo8ewXAzsipjgsEL4K0YAaYRaBIGNyOoMgLTU/EeI2HwEAasCaQRgJlsTLwP9INn8BGjFMlMxbZMM6s0Mb0yrcZrtBgSK3gpbiep0sh0T1EAHrR4shstbFs/9KMn+ij7GX9PPy4gBo0hYhVkWm6TxZU943Tyi1JKGj4DYBLjYq4xRiEy/zhhCGOXL+5tBZQaMwSCNJ9+oYVAV2PAmMEoO4brBTwenQqi8jDZoUSLSgUp6n2zLEVPiIo0Mvz6XqwMYpCsgesiTlj/WgnwISC6Rutx7o370zAMhEExy064e/JZUC3WSWNF4SBKuKZYTFimpDKtLIv8CKMtPgxAICSPH3q6vqVrW/oxaM85N1ojXEv+xW5B7RkYUowK07/reuqmYbPZsNlsCCFw9+4dmma7B9NikKI1M24PpGp9Lau+DrxkZNhq2rYRALJtWM7nlJVjuaxISQC/ttnge4VWkX4Yw1NjoGm25NmM3faSjz74MYvlIXlZMPjA5WrNwdGcz3/hs/zh7/8j8iIbA/fEKzp4L952sefrv/wNDg9nPPn4Q1IcuDh7wfs/fpd3Pvs2d+7c4uhoOVpjFPgYca8/RFsJMvw7q9+h73pSTGQuw+ditxH2hxJDUeYURY6xWgIaXcVyMSdGRmsMGS9NU7PbbYnR7wHWmwx/WRslqTwrKm7fvs29+/d47bXXxuBAaZDtgxq5PvzsO8yv7JsjSDsqPFJ65dz5ys/v5Wyf+P7N62b3/ObtamzgCRj1KrNhOnRO2oipgSbXdDC116//U1wxMoZKZmQup216sY4xia5rabtubz1SFO6auT9K/ydmRkyBFK8tdWIKTAyCqeGklMGYiZEsTIoiL8X7PwS6poZUSPBeZiEKuGusZVZVTDJKgCzTzMqKuqjpg8caAdGrqqLvGpp6C8lQZCI/9n4g+EDuNLMyY14VElBtLfNZSW4dOnqGvqX3A/PCsqhyfIIUPYoolkMuo24CySNASSVWJW3bktq4Hw9TSF9KwtZ0zu7n9NQcmMbCNO+HYZAxOozzJHhSCjht6LtOwN7g9+G7fgwYK8uKsipJKdE2LcPQUxvNerSWaEfgO8vyfXCuNPLZf4YxXe85agSn054Bcs0in+xapt8hpkg+WuwI8CrrZ1FIQ30aL9ZaqqoCpRhi3AN2SinKstwzgCZblxiFhCDnx3EORgnhVSqOBbmEqwIkLRk23nt86GHMaJHMgWEfLCqfjxy49QiaT3NosndilN/ubaIGCWCvZgfEaPF+KwxtU+xzQoahI6ZBGjfGUOQ5YUi0bYP3gaLQGONIUZhAEtgrrKRBBbpuasRpYpDGszEZZTH71HMbQJWlNKhJWDRLV5E5R996fIg0RaLJYecS0eSk0RKBZOjqNT4mSudIKtGnnjq0ZJUlhowUpLk1eC+B7MbRdyW+C5wsDvDDwPn6gtNbpxRZTmEzrDKiCOgD1mWjfD5nYEDbwND2xCHHuoy+b4mdp28blO/JUyLpjGo252CxZFdv8OIBSJYv0X1PfbmR5tYsozwoiSGyulqTlELnGVlZ4mMit46uHpi7ktODYxZVwfnlCxrfo4YOUk8YOhSJel3jbIFzFbu24+WLC255g1OWhZ4TQ08XW4KPdG1H7jRDG3CZRuuEURqT5RTZjLrt6JuadtfIWIwJ3/bUfQtWcXTrLouDBdm25nJzwWBqDBlnP93y4XfP8WvHycEtCbzNpA4pq6V4uCrABzJrKTOL7+N+HZqY5s658ayRMEbIDtbq/X2kcStnsDhEbCx48ZMVu80P+I1/9VvMD5YUM0PdP0Ppns7XxOCY+YEh9ehMs9304o2eEn0YKKuMIfQCkJUOkib4Bj/0zGcls+IufQspKra7FlsktPbkeUJZxWw+RymD71qKLMdFy67tGKylnBXkVUlTt9IsHFrWVy94/z3F7dsPODm6w8fbx8yqkuVyAZmi7moIntLlHBYHDMHArmEIA1kyqAhVVUDSNLuW3Bn6vqHIQNHhB0WrLCo5hiFS7zpC2FDkopqJsSfSkQw0I3s3NEIU6nxPz4Bxmry0mF+gVjdjzTArLDoGmk1DGjS5ycmKBX0HKeaEYcfQdzR9zTq8JM97kvHYXJHlS45PDvH9jtVl5PXX3uCLXzI8f/GY3faMr37tmIvznrOXW7TKiCHhDCS7ppxrjFLs2mc0w0uuLh6xPHakmDA24/TWnCI3rFc7+s5z98GSq01gfR6gL3j0wSXVrOeXvn5INY8kXfCZz36GP/r9F7x4uuLoRIJi796bc3X5nOXM0rc1ZaHo+prFwYwu9EQV8dGxawaOTk54+vSMXZOxOMhR1hN6j7WJnfSB6XuZm7tdz3yej4CS4u69Gd4n1psVRWEpykzwhQiHRwvKwrHd1hwcnNI0a7LMoHRCG/C+w5hczlPOjXMp54OffsxmPVA3DaenBxhjuHf/RJQilUZZRz1E8qKk6xQHhyd8549/zHw+Y76YMfQeHxXzxYIYwepsxK+T2Iy4wMvzM774ha/y43d/jDYtOaWo2WNEm8j9h/dZvXwpnGSjCdqw2m6oZgUP7tzj6kXDG3c+z+cefJ3D5S0yJ37HN8EoNZ6F09gMm84tNxtyRCfAY5DgamsMMQyk6Ekh4Hux6p1qqi60TMH1xiipuZwjeDn/dF037tFyfn/x4gWPH388hrRL7VUUM66udoQgqtWjoxM+/vgRfd9zcuuQerMR8tBmw3w+Hy1x9OgZ7SmKgrc/+yYfPnrEersjhJ7N1pNXGQtdcHH1khClCTN48KFjvaq5XF2CCRwdHOPDwMsXL7h1uyAmL9Zc1rLd/mJ1988yduIrX18zeeOrTPT9hzaB6dd1h/zMZFx7466KsYG95/Pumbp6zIq7HgtI4fBK3TGC+eMY2dvAJKThPp6vlBqz8sbvW+vY7lr8kFBIeG7fBtbrS4pCsAZlFIP3YCWcPIUBUuKP/+iP+eYvf4OqrLh/5y65K7i4vCAQyDLHrdNT/tJv/kVCXTO0DbcOD/jKFz7P/TunFM6ybRoODw/Z1Q3b3ZYslGPwvSbPc5q+G5tIaq+Ynsb6TQB9wgamf998r7U1Y32lRkvMxGw+Iw0Dv/Vbf4Xf+Tv/HWcvz/gf/oe/x9e+9nXatuMb3/wGf+tv/78JMZLlBUOo6foeb4R4pI3es86leTfaNxXF/vztRgV73/dcXFxQFAXHx8f72mSyXJxsVFPS7LaiXru6qjk8mnN4cJfLq5ecv2y4ujzj/Kzj/v2B23fm3L5vcbnG5QltPCmJxZVEzI1NmnGtMCnu8bV08yv1c+ZHGkHBdH1flaZ2zjT+Xt00byr6p39/8tJoAWe5nlVpxAF/Fi//nwBERwlIxRj6GaOwKWMQZqdMIi2TawoKTeJzrpUF0mjbkkhpgKhQPjKMRVlMY1GeHJNfrUbAVQEp0947cmL3a6XEHiREwj7YS4DGmAIhDqSY6IZI7xXWCjCqVBQJr1HkRvwRrREmuQ/gR/B8GIMven/dwFNjMyQlAVbzTHy1UlSElNA6orQXtssYNtA1DSa3aBVQThOVeILHFAm+I8RuDC3z+8926qIYbbA2o4glkYiNAdz1pqaTJkRZBLW1GOWwKseaHOUydFawWBxKqCea5ewEJnZjAqMUxtgRvFUordg1NbdOjliv1vzgBz9iu+t55/NfZHnrAdurM778lS/wa3/hlynKgqbvOLn7gN/8X/0fePNX/yrf+4e/ww/+4He5vDin6xrUaOziAzSDAOlw/T7CGAGaEiEKO8xGPUroIr4fSIPHqAApEIIEt4RBuqEhRnwQtYNKcojtlQDpMapxPIjHuE9ivZKUkoNpbihLizFiwxNjkNC5kYE3xIgPQcBjI/7NSgnQLeCoGtnZYLWijyMQh8BfVmtyq8mUIsRpfgv7Ve03O7XvEqcRyFMp7m2HjBLrhD2sprX4khuR1vgh4r3av6+f9lqU94hBFu8JbFX5BFxKkRaTH5mqBlKG1RW+D7SqZhjM2FBQo2dyD0phrQCfcWQ73Ayc1MbAnj0oQVXCRk8EHzCx5CC/R2kWoAas0WjtAItKCmMmBoUw51KUyZOZDJJiGGr6PtG0O/Hq7ZrRCkGYEHmei3eY1oQ4AYcRrRxFUeCcKDVSkjDgLMtIIzCTZ47FfEaR5zx7+jFNUVDXot5QWppA/dDvDzfD0KPHpJ1Jbn0NqIs/6uXlJVmWUZYFRV4QF0vKMmetoFVJfGhjJMVhz+aIyaNCz+NHH3B0fAJKQuC2ux2b3YavfulNXjz9KU8ef0xeOMq8kE15lLfduXOXb3zt87x88gGhH1ivLvnOn3yHo6NDTk6Omc8qiiIfD8ujf7Ix3Ltzh1//tV/lRz/8MY8fPSZEYSgYLZZZfT+guG5MLJdLyrLcd80HPzD0A3XdsNvtRoVA2L8v/diUmJi9N+0whB2d8fDhQ5xze1Zw3wsz39gbvs8IcLm/PnEGnQ5CMv/Yb//TbVMRMdms/FnXzU3+5p/xxj2Ibq195bbp954OcMbYn7O1T1kjn+7KsxKSBPKCfCb90MtnhqcscxYL8bGPQaR/zhrybCbKjF6scJxR2DzbSyKN0rjRk0+anRJ8aY0Go0nBs5hV3Llzm1snS6JvaJodzlkBTYyh3jWEGMisoxhB4hRlXhttSPPIZrtBtQJGlkWGtQLoqpEZo5WVNTFJE0+rhDVKGOokVIr0TY0rK7p6w8XLF4QUOTo+4pu/9BWGGFFa09UbVPL4TtM2HX3X4Z2l7zpIafT19rS2ZbfdCjg9rrvaiAopJvG+JzGGc04Bo3E/9pQyhCSgv9GaZruV/WVUQKXg8UOHNYfcv3eHtt5CFFLBm2++yfnLcxKJ84tzUpTD9HIxQyFg59SUKHKxdZHQxhajDEVZjLkEPYMP5EVBWYm3R9M09H2/9z2Po60VJIpC/OUnT8kYo7Dl3PX3QJo1IUacMyNDfPK69yPILU0DCShXRET+3ncteSYhUCIqELA8eA9Jocfg12H0WVVKGnTGaHwYCDGw3e7o+x5rJIhW67TfZ/IswzqxwCANuDyXfWKcD4xAGCi0ySBk0oTVmiLPhWDQtATfYbUoD2xWQNT0jUfFhNWawhka3xN9j8KSjX7VxhZiadNC33kmqXPwUbwnf4HLZRkuc8TQ0/cds3nO0eEtVldb6rBBlY5NLzYuMSSSFqVZioEpnEyCyMRWMamEyTKI0jjK8oymrQlJgcmoG4+zM5yZoxm4OFsTj6SJ6owh+sCu3TKkwDAE2t6TlTlZnhFix66tISS6QbHerFGAUwrf9qiQcLOSGGG73ZDG8TEMniKviENCqUHILd1ANXOyN1cZoWklV0AxeoWOxBZjmc8X3Do+pBsa2tUlCkVTtwxdS2EzynJOCpqsLGjbyPmLlxDg9vKEg9NT+qFj22zZtY2ATMYQfc/MlhROvLaVyGkxShNDoq5bfPCYpKm3Oxrf4WY5F6s1yWSSHeQVsTU8PTvno+9dorslB7MleVZwfFDSND2bzY71drsvqrOsGM8gBXXdjfJtObtkmfjIC8Cn5BzG9V42FeNJjSpR50je45uBy+c1/8P/99v85r/6TcxhoBtaYZtbi0YCBzUR5zJZM1TCj5kRWW7JtGPoJTxRqUDXDigtY6yqCvLMsl41XG02VIjH7KysCEmYfF3vSWiqsmJpF6S0Ztf3MkZCIM9LutixWMyZz0rOL17QDwNN48U71kDUEZc70hCIypM0RBWJyXN8dEDT1YTopQljHNXxgnN/Thx6ht6PAIshBOjajr5tSamTTJGmlvX0vJEmFIGmq1HWYJKjaTtAs9ltIZOwaafAafNnzNx/jrltDZmzDHXN5qJlsay4fe8uxAysY3X2DO8DzuX0qWFWOW7fqjA2UMwzZtWCGDK265r1KnJ+HsjcjsXikNsnJ1RFzvnFhqxoODmxXLzsIQpT7PbdHJPt8INi154xPwRTjLWXN3R1w+VFTVHkzOfw67/2db72rXcYYsvf+ds/5iffOyPzmiJzXJ61tDvD4W3Fhx++j7YZeS55WH1Xs75SfPmLD9nuzogq0vZifdV0LV0cUC7DuYq63jGbgcsUp3eWoFq6viEvxEr2+LjAhwHn5EwtTVvP4VHFbjuM9YjGe2HRK+VQSDN1t+u4f/8OFxcrnnz8RAhMRrHZXGFtRpbNGIaew8MFl5dXnL0452BxnzxbotWkwDMMA9R1g9aJq9VLZvOSg+USsJydbek7S0qKoQtc9WuWRzNQGUlVvPXGfT764ENIgdX2iiyP7LY7iuKQDz78KTCw29XkeUVZzfBph3WaYegpioLc3GK9qVGZoxsi7dkVn3n4Fif33uBXv/ovsSzvYylgFI0kFUdechyJbRKkHWIY104zZoB5UeQq+XffNajM4vsAU104Kn5TivRdwGWO2bySRnwQldTlxTnL5QHW2jEwPr1ytlcj+mq0QY0KJaJivapZLhf85Ccf8Oabol6tZgWLxQJiIs9LIHF1tebo+ARrHS9evJB9u2so5gV3756SnsXRhSBQzhcMtDTtFqWsKOUHqc/ruqWqKlbbCyCxXq9QKlLXEvQ7m5dcXbwk8It6ok/XBJa/+vV1zTFB4mOO377GGD3O0416Y3yIa/zyE/edCGKMHuCTGp90/RJGops8VLomye6JTXrEQafXOWIUSTFZqScSKSq6rh9Z0y2bzZa+H0ZiT44f88qiDxit6UNg8D3LRcUsz3j+7BlD3/OFL3yB1eWKH3zv+yzmC+7ev8s773yOz3zmLR6+9oB/+Hu/xwc//Sl3797h2dMndG3DZz/7WSKKxifm8zm3Tm7x+S99icePH9M0rQTDG0PfDRhj9+qFyUZl2jMnQtQEok913KT8ugbYZU8rFgvSEKjblu9993t89Ze+yma3YbFY8OLsBcfHt/ilr3+dZCw//MGP+PCjjwgh8eGHH3F5ecVyuRhtTKFtZRxOhJVJwTypPvM83xP6lFJjeHGxzyvSWo9knYDWTkguJPKs5Pzlitms4GBxysuLF2xWG7ablxg1Y72qqZs59x4ccefuksFvULqD2IGesiTDmA0o/vckcSuR74z2qILujexyUXSmKEz2m7aZwOheMP3MNY42jb9XG34/WyWHFEbUeMIXb9by4//2IPwNUly6rvonZcH1/f7ZQPufQ2emhOlhDNpaUJMPpiLGHo344kr3NKBGSt/+DVCMC+zIyoo9wRgio2xUe/CJ3gdCMqQkxbAZAVFrocgmJq+wh6cQp6TiPpBI68QkSWcEqlNS4qE4AfU6YTONSgnnGB9f40f2dCIRlTDY2yHS+8mlZrKGEUa10QprEHC1F6l4Ci39+Yc8uVAcfN5Q3b1HtlxCBGVF6hnT6Nkco1jaIJKSEIVN5YP4Z8Yo3T5tLMo5dPBopSVwaWQtpz172WBMhtUZlmxk1RQ4V5LnM3KbMfgenzxaOen63GBWYyztyNJ99wc/4Pbdu/zyV36Ji4t/yJ/8ybcJMXH/tddYrdbcOz3GKLGA2e06KeCqBcOd1/n6v/W/5sFf+Iuc/fD7fPT9f8r26owiOwc3MISWGP2eeZ7UCBpLi5SYxO5Fh4iPMkFjkM9EjeFfYRjoWukEp/H39+PirpWow4ZwzX5PjN+LkT6pqWSUgjdTFLmRJggTgw0J3TKWPnUSWDY9kFKT3a4UaJNfs0qYcTMI6dpv2+lEYUFF6BMM+/k5BfFO/5MG0V5gpSEbGfVGTyC6NLDsyGYVFnpkGCL9EPYM7k97LWf39xvztKhZY7DOYW2BQhPjgHhkKSZPrBQ1w9AR9usBexarMKTMnslwM/hKjWuJ0iIjl3kbp2Y3KYFVjspVlDZDm3Hhw6EQINLYmx1h9uxJeV7P1WZF3wuw0Pe9NERSwmiH01aae0mNh6dA1/V473HumqUeY5LPeRyneS5SMK0gc3epd1s+97l3eO+9n+zDCb3viVHk+9pIc2YYBhwQox0/Y71nrAKkFGk7kWkdHs7JiwJjYRga+rYV5scw4FwcD6k1KYj/YJYp+m7g6ZMdhwdHJBRt17JdXdDXG77ypc9xenwgbByX7dnoy+WSO6enhGbNRy+e0m1rHn34iPc//IC//Fd/iywXgFoDvu/RViy4YgoYozg5OeY3fu3X+G/P/tt9A0ADuXOQIoNzxCTdcGMMX/rSl3j69CnPnz/n6uqSYZCCoG2bPZAtKoZhbHSM/sc3bDOstRTlDK01l5eXnJ6eyvfGQ4a1YxDSDSb6q4D2zzLFp/uNXJyfuW1/YJV//CxI/on73fx62vxfKRTUtW9tXdc0TYNSivl8jrVuVFfsH3X/eJ/2KstC5O1dM7Jme4zVI2ivKcuM5XImcujthhglwKssS4ZhYL3e0O1anMvJMgEuQxDQzYp5LPWuox86rLXMqrmAwbnl4HDJrVvH4jnZCJAvjTlpgltrcMGMBBlZWLXWWG3pBwnTnOxc1qvV2IiPdF2DMfL++qEleSDKOUClyPrqgtA3zKsZWsG8Krk4e0a9ekm73ZLlGQfzgtO7d8mLknJWEiJ88Ogj1qvNvnmWTIH3kb6TwMosy0cfdJGk55UoPJIK9EOP04rlcr4PDRwGP4K9QiQQxnREW7cPPY3BY41hVhUS7Nl11PWO3CoOZgW/9s1vcn5+zm63pcoMh28+ZFfXmDTmLmhN5tyovlO0fS9FapbJfBoGZmUmQbCjLZJOhqrK0cahRplmmTvybAxBTcK+W8zLaXDv5+hUYEzr7eTZOJ3t2rYdQXWZeyGEsYnZ7v0ZhR2uZcZFL4BuSnvwfLLuUwjYLmNOwh+FkBFH6yb5nPq+o207CdhOGmNEpeRcJlkUqH0TRKExo/XONBeNUqPVqygElXLjemfla4WE8Wovtm8hopKoeez42oLvGLRY32glAdCy3lw3YcVOp92rm5q64+Li8lPPbUBsXKzB2gK9jOSu4GCxQCeD9lFAltiSRjVZP3Q4a0bSx5jRMzYrpdlt9g2fECNt16GVZgiBYWiJQTFbzCnzA7ROhFMIXtF2PZkzbLZrts2aiGYYIsHD0M+ZVwes6g76RFlYVps1u82alBTLxQJcxq7bMYsRBrHJyXM32uqNLCStyaq5NHatxivJHlgWS/y5J4zZGHluRImRImixDRlS4PD4Fi8uV6zXDW3XolHcPZ1TzeaEwTNbzNkNtTTyooQMlpUjtoFiMacds0Ty+YKmDgStGVJis1lRliUYTdP2NHVHiobooR8inde4Yo7LHbu6RasNVb6g0IHzjzo+/MEz/FXG/aMTDhdHRAYuVxds1jVDH/fN16k5XBQFRjuyTAJLQ4g4J6QACWkV8DqEMKpk3F4xrLXkOg0xSCMPhVWO0HlePlrzu/+fb/Plv/QQN1PMipw8s/Q7T+paYt/Q2Z7l8R1AfMqHoaGpG4oiwxrL0AcJpA2ipa2bmhBgVi4pKsliqJuGqDpiG9DW4n1NTIqj+QHaBnJy4sVK1rSuI0NRFRXzcsnB4RxU5HK1wpctvW85vDUDBobYYrUmqoBXA173tDTUzYr5YgHJ0naeNHgsljQktqstXbNDm0i1KJhVC7o2sFq3XFxs0MpTVWOjfmjph0geDUVp2dU19S4yK2ZYY+n6nrrb4jqNxdD5cA1MfYprtQncXWYcHlQczBJHBwvZW7yj6bcMccUwaKw+QKeSNx6+xe07S0Jqubq64Mc/OOfFsy1DD6tVzzAk8mygyBRVkZO7jFtHCxazOUYf8d0/fcaH769ZnyVOTw6pZjldVDx76slyw2J5wHp1RSJgMs3te5bMKQ6WS37t13+ZTXPJ7//Bd/jo0WOG0HF0q+TqvOb8O4EyrzCzDQdHS+rtBUWV0NZjo2Z11ZJCYn6YYawm8zk/fbzi7sxQVhkXlzVOi+J7V9doo9lsr5gtHEOQpuRyuaSuG6w1bLc1bVdTFIrTW0v6XuNcRd962gYOD+f0fUvXSqO2aWpee/2Qy8stVblgvXo2rt0JlzkePLjPxcU5frgmVWRZRtP0aFXx2usP+fDDd9FaU9cepXfcOq2IqSPPF7RNw+XFJQ/u3+bsxUaAt17yGIzNqLdrXA6PHj3l3p0HPHv6mIODgr5vybKcPHcMnSj5lTas1wNHR4mqKslzx2azot3UZNpydHRElyLtdsPx/AjjF/yFX/rLHBb3KewxaEbCojBFxaYhiPewiqMNWySNX097PChS8qPFqOzdg++IvhOlSwwYpSiyAlUomqZhfXU1WjEq7GjH2DQ1eS4KgGEYZG0aPEYpDg8PJF8kWba7jmpWcn6+IkWpxQ4PZ3R9w3whZ9NHjz7i1slt0piRkhUlTTdQmQylLUklcTp48ZJ7d+7ymTfeoqlrotVs65qmbdg1DVbnWJtT17t9PRiHnhDAGEs9NGR5RkpijehD5PNf+Mxewfnpr08uDhP6fTO3S33i/zdq6BEYV2OjLo0ETy0m6kwbuR4zDfcA+h5ET6SRQS5kgutXsq9lkDOCYKB6PBNNqtwJ37v+9/Q806sPIaC0Yblc8M4777BctPz4R8+5unpB1DK2+qGn7RryLGfXt/RtC37g4Ruv8e6773JydMJf/1/+u7SrDV3bc3J6TAgenWV89/s/4L333uP+g4esNiu0tWzrmr/79/8+2joutjUXl1dkecZ2u2U+nxNCpB88I+9uX2tOZ1q4VmbuFZkjoUqP41jqJwN6wj4kP2boOwrr+K3f+i0yY/j3/sbf4A/+4Hf58le/ymyxpJovyFzO2597h0cfPea//q//Jr/7e7/HyckJw2gVPFkqTjiE934fIloUxV7NmWUZdV3jnKOua4qi2IPswB5Al0uye0JIGKOpqjl935HSwOHyFlo5Ls4v+N53f8Lbb7/J6mrHk0cr3nr7Hq+/eQubZeRlhTIDMQqWl+VyrlDJjSB6EDvAJE4gApyP43mqiRFCzB5AV9f49is19I0/4g5xs6n0s7NI/B9uAGHqxq0T3Z3pOcTC8fr5x898z3v7HwVEDyjt0EaYHQqIPuyZpGr0fRapRxQmqxp9gtI1QzzEiB/GQE4nwDxKY7QiKPG29lEYvpMXtTEJZzy5G71HtUHrnpQ8MUrHOcsNWaYFjFTi46M04vOG3jOftBZGsXUaFQPOSu/CWAgeUALSi4RV0Q4ShpnGT3piVfdBZBeKKbk2oQhoWvzmJc1mx/MPz6juv8bynS9w5+3PMj86ZNCahMWYEtKANgmdeoyNxMFLsJmCIfT0XsAzkOAnjCalAWcszpbjZpcEHMSgdYZWFqstzhXkWUlWzLB5RVHMScrStD1lmWGzUhjCSJdZoeiHgdXqipPjAx5/+FMOD5Z8/Rtfgz+JfPuf/C7r7Vd47e4dfvSDn/Crv/LLdL2ABtvLFU+fnxNi5LyrMXnGW//iX+Gb//N/A0vkxQc/5OK9P+DZP/g9Eh+hlYyPMHamRgz9usuptASLOod1Ir20OqCjoh88fduJLZDWRBKiWlf7SRPH+TJa1xOSBIz6xF5JopQ0ZMy0t6QxAIy09+72IdEPkcFHfJQuVRg8XTsw+LDfWKSpM25E40RPQKYVmRq9OWMa/T8F7B9RWXm+sakweTrtm0GIb3YaGfZhpLNbI13fECJd7+kHsQn6Ra7jxQOmNO5pzmo9dVwzAfGJY2Mn7IFukryOvRXOuBhNbP6pQaKUbPKiVpGfm8iZWiMd7OkxmUCqhNWT/9a4+DGB7nq/3qDVfs2R4L0ddb+l8yN4PtoaFHmOtRm5y8mMGw8WaV+gyyItY2+ycVGjpUYcf6+2FTuXq8tLzl+e8fzZU3a7rfjsMtmTDPsFOvo4doEVPnjcDa81P9qOxFHxEKMUn2VZkrlcwBkfUMrisoo8BGLyhNDjnBUrraHHK/FPe/TRM378wx9wsDymKktSGghtS6bgrddek+5017G+WpGGnqHe8fEHOx53PXoIzFxB1ntKNPOiIMst1ipUEosFHSPWuZFp7nFW8/DhfbLMUe+2KDVHKwkFNkZRVRlXV5d0XcfV1RVHR0cAe9C477uRgW3oOml2aC3A6hBGq6fREuLmYebu3bu8/uZbLJdLTk9POTg4GG15Jj++nx3fE2BmbjQvbsrypvvsLZr0jTE3AeDj2H3lezf81j4Jrt9skNy0hZlumxQb6/V6zxKczebELKH3j/lKA/1TXf0gHt0hehIBlxlOT4+5fecOm82WmCL90BDaQNs2iB+jBzU2c5Mf16EBHwyJsTNIJMRh3IN6UpTQ0hCHG8WXhJELeBqFpRmjjAGUeNM6B1oOnyRFWc4o8xLvA13bSVhi3zJLM8muULBczDFasdtt6YZebAwyx6yqmJUFRZ6xnM85PTkhzxzr1Yqu2bGocu6dLFksFlSziqHZsL44o5xVnN6+w+fffI2X5+ekpFB5QRsCw+Cp65bgJWNEiMsK53KKXNjYPkoRkFLcM0bath3BTbtnbBtj6AaPUkZyJ/xADAIWh+DRaDJnIBdwu222HMwqDhcVoatZX77k/v37vNysIAzkmYDjhB6rFDbL0QgjJPkOAywrkUq3bUvfdSSlyK0h6SQMWkZyhB4taEIaLZGkgEwpiSrLS1izc46J9TId9EGKjmEYaEbPxmlOXNtkxX0hcu3lPjD0kEIYmyoWO7LOJSIl0jaN7DExwXiuSymIPVQIN+bXtew2hIAPev+1RJ6Mjd1R5dZ11+uPsmbMOwlEJUWftRaVDF0XUPhRcTaCCVGUj9YYyqIYwXsJwASPy5BA8NSDBuMsLhl8sDfWND8qF/ynn9xAioEwBGbLOfPZCUMfxnA7x/HJEUPyqBBQVkJyY/Ry1tUIEI0QNto2SIC6MnifsE5TzWbUzY6IqMy896JyTIoUDS7LODk27HZX+MGTjazIbmjpQ6RtArktGToJLG43PSoaFmVO11psgrptaVwm/tcLg80kaHny0VfK0ffymRjnxJs3y8lLR4gDGKkBykVJ3ymKomBeijVUvVpTVjl97Di7PGc5P+D09A6bbcPTx2dobbh/p+Dw5BTft9hccXR4SEgds1mOtbBtV/QR8kWFbjbYLKM6mOFKi0qazXrNruspqkqCMduOrutJKUMlyUrwyeJyyUQxKsOSc1jd4tnlFc9/WOPqI+7dusfx8hZt13K1veRidUaKhsyV+ybxNPcArMkA2TuN6ccmTRi9zCPey/duhnJP+0/ne5QTMD12Yfy8Cwgdq2c73vveB7z1pWOKoxlGQRs6gm9Roaduaw6Ob4GCrq1ZX63wfuDO3dtopWkakc4vF/Kau6Zh8AOdb0g48jKn3zWkJKHWq/WG+UKzmC9ZVgccHS84e3JGGsGb2AcOl0sW+YKDgxNcbrhcX4zZKhrjPVmZ8CEypJYsObLcYXNLE2qKKgMGnAFX5jTbDVpbZkXJy7MrtqstWgUJN9c5mS0ZdEcYdqQQGNJAls1HtVtDNSvRY8g6UbPbNIQusjhYEJWc0fquxeUzfIzs6t2nnttPXjTUg+HB60ecnCyI9LR9S4iKi+0Vs0XJVTtQr+H8qePyxU9554t32dYrNusN9TaRwpLQQ+F6wrDj5csdtw5nWBXH+a5ZzitIjnc+d5unH2740jfeYXEQCHHH6vIFyUS+9evfoJop3v/Jh3z2M5/h3R885vFH59w5vU9Z5rz//k9pGs8//vs/5e3P3GNtVvTdjnIG+AybFtQXG+ZZTmEOWO9eUs0UUVkurjoiUPua+SJneVSxXMzomsjywJCZRNs0o61aw+HxgtX6jKys9vVE13mqqkDpiHOao6MjttstzuXMZhVnLy5xLmc+F696rSPrTcNkm6iUZr3a4EzFfH7Iav1C5o6Gp0+fUJY5WS7EkGbXstnU/Mq3fpnf/73vcXL6gKK0JAInJyXGSO0y9D31uiYOgdgnPn7+hMX8FndPc0IHz87OyLKMoirxoWe3abh9dIt7d+/y8Yuf0nWe2WzBdt0zDIrFYsa27jk9ORL7zDTgnCF3lmU5Y3Oxpus7+hg5mp9y5+AhX//Cb3D3+HM4s8ToDB+HPYDK2EglhVEJf01AnPZoM9Yhosi1DIOQhKL3GB3pfcdu21IWkvnmfU/fDwyjMrFLiVk1Yxg6ikLChy8uzsc9EFnXbEZIg3zG2qCj4d6de1xc1dy+dU+Yw0OgKDNW60tCmDGbzajrSN8P/OQnP+Xhwwe0XY9WiqdPn1JVFTFFytmc7XrDbtvQtS0oRV6VI5YjymNrc4zJudico7Tm5PiI7W5FXh7jh8jVVc9rr91iU2/lLBs1mZmTlYefem7LlT7x9U0Gurpx+6RYn1QtGtJkxyc18qSWnRwKrkl6Iwsddf31SFKQsiQSuM52kud/teCYsqS0sqN1nLoGH0cm76vkIwkU1UqUqUOQM9owYkyZcxwdH3P+8lIysYzBoGjrGqsNVTUjhYGPPvqIx48e0zUds3LG5z73Dq+/9gb/xX/zX3Hr1jEPHjzgO9/5Dk+ePAESz8/OyLOMO3fusDg+xrmc5S3Nya0dl1dXLA8Px9ewBiUK8cnbfWoyT+fUm/YhN3O09udL78mLgizPxtvkczs5OSF0Pa89fI2vfvlLvP/uu3zzL/wqP/rxj/nGt35F/MiJGGt44zNv8W/8m/8mP/jRj/jxuz/h4OCQ58+ec5wfUBRCftrtdvuQUKWkOTWdA87PzyWnabyv1CXSSJ9q4un36roWbzxlWRGl6GAYpB7P8wPmsyXLxSFPnz7hvfcecfv0hNVVQ73zXF7UfPade5ycLkA1JJXIiowYO8G/kpHxmCb75AR74Pt6XMsQnAKM0368TsPtJtFycjYgpZHUdJNZ/gmi2zRn1M9Xfd3EpvSIOb/SMZoeR11nou2Jc/+M689h5zLOuGli7rsGBmsyFBqlJwBdpLRpBNfFnWn0UIriq6qsGgG1hEoBqxJeiS96RH6udJpZlnAmYZT4hmbWYicP8xCI0ZJZTeY01kQUAynJH1ISyWSaPGfl9TsrB3UVPVZF8TQ0EojIHtBUDMGIB3EcvX0UwnRJiDd7HP18lMgZFB6lBrp+i+49u7MeazWXzrI4PsG5gjgr0WVG8hGjZ4SUQEsCeyQQkmw8GE0cxjBNNW50KGLUGJuTZTO87/HBgxpBdBxaO2kqZLl0sIsZ88Up1lVj11GPTNlE4USmHMcBbbRm6AfWqzX/6Hf/Idt6xb/6r/xV/sVv/AW+9Nkv8l/9P/5rnty9hfbw4sE9Xn/7TcLQ8OTZM/7k+z+GmFjMK168fMbrr7/Bg3v3uXf7lNkbX0Xd+Sxfuf8rXMT/kpfvv8tuvdoX5X2IdB4Kq3AmUeaGcpaTz0SCraPHRg/9uIAN4gWv1MheFxXtyDyfNgmRH+kRxE2vdEcTRkHuJGB2kjgZq4nRYJ0AoEMfaHthx6skYFAYenw/EMeOpRYcSJ4PGevi7xTJjSY3iiYJ4z+oSfqSxkAjUTQEkTnskTKdxGZHQHopcMMQ8CO7yFj5vfsh0HV+tCP4BVA2oMhmr2yg08sR8NAyScGiiuNSEAlRgBGj9uj3jXXhVanZ9Lg3QUutNcqaa+A8TSnfk9zMs29Z7h9cGheB0YMZRvDfE9JA3e6o6y2dH+iHkT3qcsqqIs8KsXGybswAuPYuFiBjCnKcgNBridtN4LQoSvK8ZjGfE09PWa+uRqaGLNbG6H0zUX7u5sFIpOaTh/C06R0fH3Pv3r3RbzOO4FUc19ecYAPKtIAlAW3TEnwLQWygkorMKsflyzMevfeIw4NDnNOk5AlJU9eteBC20uXXSmwKnLEUyXBSLVi6DF3lbGdzkh/Y1VuO0y36rsNlGYwAqLGGInfEEDk5PkQhr8GM+ZdOabKsIC8zssyx2Wxo25bz83PKsmQ2m9F1LV3Xjp3wKYMijODrqx5103jx3qO05vXXX+Nb3/oWr732GsfHx5Rl+Up42k0g/eaYnho6f9aVRobrNDZv/tzNQ+UrFjA3Dlo3N+qbt4ti6NUDwDU7gP3z3TxA3Lx+EQAdoGl20gzTSlQ3SUIXfehp2i1d3+Jysd4gRpx1+KHlqt6ijZXQ0CqnqWvqXb/32NNa03cSeK2UoiiEmeEHkbiKLZQwxZu6R7ykNWoEW30QiwYJhDXsmhpRG8gctMYSR7Azs475bMbyYInWmm7o0Tqx3a2vLWhczmJeslwsKFzGvCop84zMWtRiTpUZYp1j0oBSnnkpnopaW54/f8HTD97j8PCYW8sFIYKZz7nYNexipMwzdr4h+IEsK7FGQvKsNpNH2OidbfZslinkZ1JYCIAMeZ7t1SIqJTLnxH4sBNqhRZPIrWE2K5nlGbFvOT1aomIvqg4ClsDhonzFI7EoCmJSOCvZJnVdk+cFi4O5jEHfo5MlzzPm8wVN23NxteJytaJrO2lcZxnr9RpnhTCRRnZ7HxNN7wG192i86Wk6MfOm2yarFT3akk0WPFkmKot+zNKJfmBRHEq4oDWEYHBW03ajCiFEdjvxi7RZRl5M9kGe5NVoEZPGnBBH3/l9kaP1aEMVhYFeVYEsF6JB5CaTftyjUpRGkxJ7H4MhRi3hq0NDkQfKHFB6L/vVegyd1gqXGZzTaKtGto80nb0fAMvQiww9hGvQPAT/C7PZfBKP7KQSQxhou47oW4yWeToMnQRcq55ExGW5MIbDMKpUAsEnuq4nBE1Z5ZSlWEu1saP3nq7vsDaTZodRBO+5urrk4GBOCNLEMrnYBGp9baOllCGExG5Xs9u2dL0HApvNFt8NLGYz+l5CIBOKvCiwmRnD2g1+gL4TdYFCLHzQQngoioKm8YQYaHcd1jpm1ZLD5TGHiyVD37KucuaLGT56Nu2WbgiUecnR4THn80uKquL41m1O79yjq9ds6yuOj4+wWUTriDaB1eYltlwSTYZxmiIriSoyXx7QNR50S5bPGIIi9QPKCLgxdB5nc/ohyHhOkEeD8RZlLRcfb3jvOx+jtgWnB7c5WByw2+3YNVv6oSfLChQSknx0dITWmrIs6bpuZKdFlLLjvJK/V6sakDOIc9d2S9N+M/2ttRE1hklg5XzrB1EcpOB5+ajm9M6c9nigmDuUhXbYoaJnCJ6ur1EoLi8vWK+uMFaz2azFftIZqqrEOst8Pme5PODl2cvRKkgLQ220/zw+OEbrFTEqZsWc0lXCWDaO5eEx3eactt2hUmToWw6Xc2xmWe+uOD09wRaa5CT7RhEgJOLgccYym1X4FMDB8mhJZixZ7rhaGfLckTs5szknuRqL+RyXZ4RhCmWOlIUE1x8cHHBxeYbSkSzTFGVO2zYEH+gaOcOUi4KARxlpMOZ9QVXOadvhZyftP+eVVxZsoG57NrstxcyQlxl9HWm6Aa0rcrvg7/6dx7z58IiTWwsef/yUkHp8m1jO7/HRB1sef3hJURYcn9zCD2tS9Az9FUZXOCPNXOsMR0eav/LXXsMZeO8nz3n48DUuVi/5zBuWpxffZtYtODia873vfZ+Xz1rqneIs9nz1q/f5/p8+55/+k/fYnis+TjVf/NJnefLsp1SF5JScPV2RWXj24Qvu3r9HpxfsNh22clRHgeIgww+J88uGzg989u27JB15ef6So8WMlxc70AFlHLt6jdKKuhbbyLLI2W12VLMDsf3pOtq24ehoIZZ0wxWHRxWPHj1nMS8YQqLpN7g8kDlLUwvRZTFfsFrVpBTJ8wrJqFes12tCqLDWslwcsN3smM8L3n33h9is58WLp+RFNlqUWHwQax+LJg2e0pZgPMtKE4aGX/7lX2Z1WRNTz2azpvMtucm5e/cuzbahqiyHB8f0L3t2uxalSoq8pG16jo9O8ENAGclaynJHs6vpO6kbzl++5OT0hKPFbb7x5X+B106/QGEO0drik+w9WpmRaR5GcPS6TjHaYkalf4xBwrLjSICKYsU2dC1KRawRcpka1xytRNEeg9iyTY3rtm3IkfPQbrd75YyUZY4URMFWFiX3797l0UXPx4+fcnh8l7Pn57z99tusNxdsti+Zz0tenp/TDx2Zy2m6jqOTI9CKoZdzQwReXlxyenoLpTSvPTyhXm/QybDabLBZSVZVqHpUh+WSGXS1arh1uiBpj80MmauoqiVa7fjyl77C7//hGV0bqMqSenfFbjLh/5TX9Xk/jaWJ1C6vfJ/x+1yX2VIj37RnMWOtPdbtCOqxrzvStYWL/N/sHz2pgElB3BZ4lcB7s/hQSpMm4pDS4+uRGjmGUdHA6C4wMpCNFeKRsPol67Aoctr2gnq3HWtlTQxy1puVFXW94fnzl/wr/8pfJp9V/NG3v43kiiX+6Pvf5Ze+/FXu3r2Lchnf/uPv8Pz5M9CapqnRWc4Hjx9ztd1SzWYcHx0TE9R1Tb3bkRUlm82Gpm1HG2OxsJ2UwTLm4yu13SdrwGnsxij5iTFFqqrCGL1XYn7581/k+Yvn/PXf/m3S0PPs5XO+8rWvkYCh7+kHL/mJ5YzPfu6z/Af/wf+W//N/9B/x9OlzlssFTdMQQmA+n+/DRbXWoxVjt88XMsZQVRV1Xe8bHVMtbIxY1pZlyaQ03Gw2aA1lVTL0QpoIIXF1dUFVVQxD4MGD+9R1zdXqisxlvDyrWa0anj+94t6DQ956+w4PXr9LStsRtA4Y7VDJ7D9/9mNhcj2/BtKnLNw0Dug0kkonxbZSU4NmtHOJo9Xx6P8/jbub14Tr6Z/rwT4iOZ9gmaeY9o81tatesUB6ZTL82dc/N4iulUG8q+NoozGCEwiAqIgiTZ5CMkd/zzgCzmn6WUaQWoM2Fh1kAZ78z6eVwirNYWk5LiOVHciMMIcnsDChCUET0wjWGVBaJOYxhhG4UPtJIZ6wcTTtl6IumZHxPtpm6HERMmMHYvCJ1jOJEgCE0RQFhDVKjz5hCmHvBdLYCPBDR9t4hg9/yjJELg8PKcs5ZVXifcSYXBoBSXzZYuiFla/FnibGHvYgHONiIzJjYzLxzlUW6FAqkKLCaIc1ObkryXPxTrTGEaJiVi5QSjH0HTFEMucwVj5TFQI+Btq+JyrFo8cfc7w8YHd5wZ/+4e/y1W/9ZbLZgn/n3/l3+U/+r/8XnHG4bM7i+IDjoyWL0vHs3R/z4eOP+cKXv8SP332X5nLNj7/7fW6dHnJ65wHl4hbL4zv81t/433D5p3/Mf/c3/yb+7EyK4IkBijCw81yTF1KIazMOJ8SKZbKwMQa0lY6oDwJGR0YVAgGr9b47pWD/Pk6fY24UlTO4TDwQ1Wj7M202whIdGIaRmWr0aBUg8nNnBFy1AWxM+8TymKTDa1WiyhRlrun2+RxTqCoQxSam95FhtIcxYwLREKD3AiwWVsCtaSnSRmGdjBE/eMJ4uPm5i8ef45oYh3ANIO43EzWZ4Ey5B2kEdzXKKojXAXbTdS3J0eNieQ1ITrnjqGsLGG4sXnpUAghg/InFUsXxcAAJO8rRBTDvupq62dG2Df1o9VSWFfPZkiwTuZ7WGqfNGHirRmArMfm1y+93zRYPIeEHAY+mLn42Jnxv1ordbsfV1RWXl5cSPHiDkT/JvkRenqP0GKIyNhKmYLuTkxO++tWv8vqbr9EPA+vNhpPjY4Z+IEXIsoIhSuApWtPUDSF6CawLDaTEMOyoqhkP7t3jx1cf0teebdfQdTUuy1it16RxLbRa4zKDCVC6nLmdUZmCg3zBfFbiiRSZZbU6J39RUmSlAC8uGzf+Cjt6kYuvvN37jk8hZ0fHRxRlwb1793j//feJMXJ0dERZlpyfn7Ner9jttux2u5HJOo09me1ZKWvc9B5OB5qyKHjrrc9w9+5dbt++TVmWe8D9egRdXz9zCPpnbIo3pXw3f/4aHL++38Sqne538/A13S5f3zhQjA8wvd6iKDg4kDCq6cCk1Sdf5M9ax/x5LgEuFSEq8aGOnrarWa0Tq9UFSSU2G4fWiqoosEaNKhcBppwbVSU6oceGx3RIkgZvGg90UohN6qmUvOQQdA14A0maICGI8iIhzJQQI9pZOewkOYQGM1qHGCt2HOPnUOQ5xhpC8qNiiNHn1FBmGUWeUxU5hcsoimwEMRMnJ0c0O8eHz96nsHB4cMC8Kghdy9mFWAvlRnH54in9bs29Bw+pFnPasZCoVcvV1Voav0FRFKL0aJqOmETJpi2EKEzQac5PHsQT0G1EWkfTdOzqmsxa5lUhljVdTfI9VmuqIsPZO+S5Q6tA5hRlbnGmIvQti3mxt1ApMyvqD+uE3Q57uac1hrKqaOqGzIDSAoI7m7FcLrh164TtdsuL5y84vzin3W0onQDtsg96UJrC5sy024/9yX5omp8Tq3s2E6uli9X1PJmaYdN6MbHYQwg4o8SvfGzKTIdZUiJ4PwKxUBY5eVmRlxkReb40WQamyOSxOAUqWns9/3wI7Ha1AI55JwxerfY1YtdqhsyQZ6Cd7DGp05Jt08PQdSTfMnQDfRYxI4PHGIdSmq7riTGgdEQAlJ6ur+WMYuR3EiC1o+uHMT9CGitd1+2lt5/2iiqhMsOABLt1Q0/fReKwpSpLlE6gEm2/w4eAdhqrHM44yllG03Rst/WobFSc3Doiy8Q3348+6TFGrDFUhYOosVrR9Rs224FhaEnJk2VLQNYaZzJ8u5V93wg7e7Vdk4zG9wMvLq7om5bFYsm8nLFpdnRdjXGavguopLA2G22U4t6KBxVH+X8HKdJ3wsSsd2J7UOQGZyxWG9bbHblznJwecXZ5Tlf3tG3PdrMhz0sevvaQ09t3Ob19l7ycEUOP7izWKBZVyRBqQuoZfIdOYndVVgXOQdc2OJNhTYazJW6Z4QexiMmzjNnMcLZZ4YwUzeurK6pZRm4WDD7w4vKCFx+uWT9tOMrvsd1s6LqGLHPYTLGsFhhzSN+LVV/XdSNjLN9LvJXSWGc4OJzxuXfeomkaLi9nMh/bDu8jm81WAvZGltqeMJASFgsq0aVOLCSTfM7ERNrA+nmPf93Q5YFgFS0dXbsh+MRus8K5jHq7IYaBPCu5PL9gVlUcnx5hiwLf9QwhUtoclKZuOhKBbb1jGMYgvz5idc5mvSYOke16S3/V0nWew6MjWnpIHp0g9BIOqQ34YUArg7ayvnXdDvqISmCSIncFBo0yiSF6XJZTVUtcZlgcLCmrORFPOXM8eO0OsW9lvbaOXd2gFZRFhrMGl42yfRIu02OY7NSQkD01EfBxIKqINgarM0IH2aLk9KT41HP79N4c71ckA1mZgxGC2GbbULoTTFzw6NEV7S5w587rxNjQ9TsuVh2z3GGWOR8/eoxSBVeXnouLM6yLPLgXOVgOZLkmDIaEITIQY0c/NPzpdwaef9zz8O7XuHiWcXQr0XWBZrtDhZ52lzg8uMOtw2OW1SG7teK9d9f0jaHK5ly+3PGPf/9dDo5yrjYbvv6tN0img51HJctHHz1nIDI7NiznkWIZ6cIlQ6eFTBE1q/WaN9+8T5krdrsN1czSND1NWzMzBq2gbXrywrK6alGE0Q6ww1pD3w+sVltms5xIh28G5nNNZKD3O05OFpydn5HllhggyzWZE6B3sZhzfvFE6oshQYp0bY2dL6jHsML1eoP3DbOFxmiPHwZUytmu15RlweXZmtuLA8IWsrLki2+9hR9Effji0VPKfM7JckmvB9CB3bZmpyM2QFHc4umTl1TzQjzQszGgcTOwmB/hXM4QAhdXA/W2FrZvDJwen7K6uCLWgc997fO8cf9zzO0tUhSWfNIBk6b8FcnWck4IOwIKphG0YsRERKGlx33W+wFtFLNZQYqerq25urxA8u8GinzMRSgLUhJlbogRjR5tG+VM4Kzk2EwZWkSIMYz5Roq+8+RZQfSR5WKOSkqy73xi8EFCKJUiKlhdXXB0eESeZ9T1bq9gj0nIZdY4srykZsfduw/YffBTzs8viReJYuZoukCiJ/hWLOOMYb2+ZLFc4vuBfui5ffuQ84uzERTUElLpOrFs+wUuAaThkyx0tf/eNYEsTdB4Sty0ZdFqtOi7wa7VCVS8WXOo6z9pkvMxYnqjhckNVvmEeE3YieB2iqTGjMMbdVFKCXRkyqBj/4pHUq1WVFXJMBgODjJ2G4vL1iQiXdfSNM1oGTrgSSwWC778lS/w7/37/z4/fvdHPH76hA8+/AhQrFZrhj/9E+If/TFaKfIi4/TWKVVZ8ub9+wyh5+79+5ydnfHi7AXr7XZUtXrKquLq6mpvgzIM0lCyI/Yh5CZ/o1Hxs8Som9/bq8hv2NdO59gnTz6mr2v+7//J/41f+43fICss7/34x9y9d5/s9h3Za5yQEowxfOWrX+G3/63f5r/5m/9P1ldXnO+E+NS2Ld57qqraP89k4aKUYrlcAuybYXVdc3V1hTFmb6M6nRmKokBpYZ7H6CjKDB8kW2CzWUu2UAjUTSTLM+7ePWW3FSu26B31VvPk8Y4PfvpHvPbGEd/8lXdYHh2gjUfHtB/Hk0UK4+f/au2b2CsYPvGe/gxhTY2gtrmux29+Lp+80h4l+8SlXr3/RBJN+mdfw9Q0+h/FzmXv5ZoQMCbF0Rt67EClhI/CjJ6AAwEVIUQ/WmWIf4PYXwiAK6CjGQuQ686WNYqjynA8S+RJkU02MmoKoZTOWUSDVmIzYyeAKo7PL5ISAWgnOwlGwF3twYopIBLU6H8dCVEThkTrp85aIkbwHkgKp6FwWuxA9oCJSJJTDKSgRAofWs7ef49gHZ/7lb9Is9vht4l8PsMZMe9OeEg9Kl6DRUqPbZjIfnGMeLS2mNHfyho7BqUGkk4YbclcSZ4tKPKKIq/IixJjrUxya7HGSaiQ0mNQhPjNRqXo2h0pdsLadYbu/Dl/+NFHzBeHvP7lb/Lkpx/yL/yl3+Q//y/+S9bbmjc/c5ey/DyrszVHiyWPeQR9y9F8xu/+/X/IYrGkOphz9/5TfvOv/MscLI/IzCEnf+kW69WaP/hb/y922w0pejIjXW1r5ZCckGYIQWHS1CMVVidK4TItObcp4cO1nKgPkZgUOZFs33C57nJN20OuobSInY8g4Ew2BlK0yUYujDPIR+90xgaLc2In4nwit4rMyoQLSUJjXQbzAqpMsWpl4ZCwUMbnkf9Pr1shzRsS1H1EayiyROYMWSaf+Z5tPsljbrJcf86i8ue6Rjb4K3v3+LAxRpHRj7fFsTM4BUZobiw24/uT9g8rLUfp8k2LVbzewJl82KQ/DpJbIEE313YwE5B4zSzWJLTYHg072n5D0+z2IZVFPmM+W1JVs9EXdwptnMJSry1cRMop7CitIzEysprN6A+sRr9yCTjz3qMQe5ijwyMePnhI3/VsNzsa6j3r1Iy2BFPHW43BO973TCnsR8dHfO6dz7I4mPPRR484OjjgYLlAaUVIEVk108hcz1BJbJtm1QFDX6Otw4cB71usUhwcFcwqAz7glMUHR2gDVgnrTJlE7jJUCJiQcElRGMsym7HMlywOlpiTBccP7uF9y0cfvMt8fiA2LtZSVjNund6jrBZoZUXeXOVkzlAUTlieMXJydEw1Kzm+dULXitfubDZns90y+EBSitl8gbGWYWQyGy2AY9M04/sUyfOCzDnarsVZy9HygOOjYwkUGsfdJ21VpvkuY43r82lSgGGvSpLRNm6c0hTTI9s+hdHaaZyTWk02MNeHqE8esKaRKncaQeb9i4hImtPNRpWhLCvyvEBr8ds3ZlxjlIxJmPzDr3+/P+9VlRU+DPR9DUiY6G63YbVe0bYdh4cH5HmGNcJ0SEy+u9f7qfdRFFyjVUkcfTOdU8Jc9wHvG/QIAIcoHuHDUBNDC7ZAk0b/5YA1GpdlEujZ91itmM+q0bZqIGphc+aFJc8tV6s1q6tzytLhnKWud/i+ldwIA9aANuJL7X1GdIrVqhHgr6q4+uCcjx9/BN0Fy9JhLShus1gsGLqMjx+fcf7yJaTE9lKRmUS5nHNYOozvBSDptgxDpKt3DNUMH8VmK6aEdhrMuCaPbM+2kwaA914acUn8y1Fi72VUGK25IppAYTRluSBzFhUDarRcUlYR+px5mVFWx6yvVugRoG/aHUWWsViUKKVFNYXmeCHeuTEGYko01jAMDqMgxIF6d8EQwdiC0sBBmaOXMqeMg5SCMGb6IA27zJLNKlJS7LqOECErMpR2oyfzGI5rHdbZUbYd9qxrrfR+DmklhZvW0kiMCLM4RFnvtBbgCiVB4865kaGcS3jgngkkarMYpJAZerFZS1GN6q1BVI5EUhCroq4XC5U8L0arEBhSxLeBIpfxZiwMwTMMAZUkB8LqRAqBrhGwMXgJWNRA17YCAOhIbisIA8kLKcI4h9VKmHlpQCdPZiHLDF3nCcH/jCrlz305C1oRiOjMokIkDh09gdQ1YnNoIaSBhNh8lEXF0ckt8iKy3T5GGmKj+gbNZrOlbhohnVjLPMvIsgKiZeiESKD0QJYXYv1DpG0bYhBv20xlxC6xq4XtabNAUAnjJu9R6HwkrdYYo7Fa1GUKz3qzwSqHVhnGSDaD8oqYeoLvwUvDpt3VNLsdejZDRY3vIvW2Zmdz8IHNao2xmrIsSFeRIfQSTtoFIPDg/j0Wh8e4PONqu0F5sfRou0uCH5Cg1YEhdNDVHFYHEpirArumIXnFwfw2RT4b655xX1GKsnSUVY5WcHx0QJZrZrOc5XxBu/I8+eiKJ++9wIYZrWpwmaOYu7F2UaI6DIrgI/0wMAzDGAome81bb73FxcU5h4dL/rV//V+n7zqePXvGfD7n2bNn/ON//I85e3FFjIk7d+6w2+24vLzEGAk/E6a/JsQBlTuGkAghjWMykHaaJ+9f8fCdU+bzjJQBKtLqXubZ0BH8gNHS2Dba0DYdlNC0wqQPXlREs2qO97DbtWR5KQAbwiI1JscoCTpzNufi7JLYDqAMy+KEg8MDnAHdNlgLm80FbTfQNjUqOrK5E6DMOXwjQWpWOWKX0GjyMqPrW3Iq8mJG19fkZUU5K6m7LbNFgXUQWoMfPN53o62FZIV0XYePAnw6ZxlCLYHDShobKhnyzDCkgSH2aGswNiMzJUZlxB7JDfmUV1HVhJhwRYbNC7xvaXcr9ADD7pC/9w8/5OLlhuBz/sHffw/lPO1QM5tpvvG1OXGIaDMwhIQtp3NuxAdpqocwoLUiRUuZOdCB2XxO0jkfffABZ2cfk2LDUNtxv64gaXSpefn8ktCu+d6zlnarCb3CqIxv/tovcXSnJ7Djww/PWXVbfviT5zx87YSzJ8/oDJhl5PhW4t7rS548vRTVGT22EHB86HvC4Li6ukDpQDlzFH1D5hyzeU7TNmilyZ3UFUMX0Bq224aisCyXFbN5gfdQlQWXly0nt455un3JclmxXm04OpijUsvzZzuWB4ngoZjP+PCDn5IX0iQ8skvSkFBDIC9y/DAQo+ZgecTqci3n6UKzKArqtWeuMy5bg1GOWXbIb3zltzhY3Ob8bMWv/upv8vzFU5ZHFb/3u3+Xl2fPeHhwD5MrBjpW9SV9bHhx/oIhHGBRPLh9j59++BNgRz90vPX221xebnFZR15W3L59h2fPHpOGnjvHtyhsxp3Ffe4fP+CLb3yZTGdIxEXcn4HDlLU2YjoxiQJTGNwJtBprNYFt/XjWC2O4+XZXC7FCyfpeFE5yELodvhOrz13oRqJNgYpjOPFo9ZS5yRpEUxSZkI9iIsscLhYMXtF0iaYbKOcGAlxcXLA8PGRT73DZjD6suNq13DqtMLnFlRkxedp2h7UzQorYXLHarZgXS7qhZVCBTnva2JHlGVerK1R+yKbuSUgo9ttvP+T27SOePXvM+rKjmOVcrJ6T5YZVrbG5hjqhrZH9PfvFGuBiD3cNQF5XzmM6581L630tfQ2iS/iq1EBiNSf4ZQAVX1XBJnnfxQ5vKvJHbCW9iiMIofFngXQm5jtq/7InHq9SGrFAnWocgx4fz1pLCpr5rCClK9brl6zXl2TZjKVb0DUtaQhUZUEk8OjRI46OD7l795Rf/41f5d79O7w8f8mTJ085ODimr/ux2TUjKzSzg5ykB/q25Ze+8TX+8B//Ex59/JQOyYwzRhNItL2oVjebzaiKgrQHw4PYBO8Z0RMB6ppkKDirGcl7gYQA2DKGI1WZc7Rc8MZrr3FyeMjp6S2cUXzmrTc4vnObFEWJqq1FGXkvtdIsljN+49d/nUcfPeLv/s7vcPv2bdquHfEWefz5fA4K+qGX/BVgtVpRFJJv1rYtk596NuYh7e3bRrXjYjmjLHO22y0gOFPT7JjPZ+x29dhQE4xCKcS6TAnRYrOuaVvN8ckRH7x/zuXlt/mrf+1fYLGco9Qa9KgYVBNQLdhwGh0VRsNjGWNqAsWnESQEt2vMJ13fmn7WaugG3i3jNf3/AdETY/7i9fOIhdWET40/e+Pxb/79z7r+HHYuAuboJEFYKSaihzBKbsU3dwxkiMIYuWmhYTQ3AiWF7RPD+KaPPHufoI8Szui04qBULEqDGgKG0ZcpSceqH5nbRiuMdVirMG60YrgBKMYEIV2DqTB5owMjDK+0ANECko+vJYL3iX4QRrsEECrGbEKsUTgjPplKCUNLIR4+Eq6aRkmxlrCNPCduV8yOTylnh3TJUw8D1s0w1qHVnDC8IAw7YhTWiDYGHa/9dA25hLaOxqBaT4Flk61OgcsK8kwA9Ko8oCgOyLIZ1uViM2IsATnw65Gqba3jarPm+ZNnbC8vefd7f0ypPV998xY/fvcpH/3oe7z21tuUVcbz7z3lm1//On/4T77No4+fcOf+XS5XW5q6ZlYUbDdryqLAZY4PP/oI6xy5y/jBd/+U+3fucHj7NqtnK5af+QLf+Jc2/PAf/D1WVxdkRu3DOafFKsVJgjGCzlEWeWWkGFNKj10mYS8LW30UFSVptsYJ9JqG8fi3VpIMb8xoORQiU2JoAqJPhJGFnlkoco1xhiHEcaOQSeyMSJaaCG4XqJE4BWcUVa7ILcLWVIlMi2c/aWrsiB3M3ssMaTr1PuKMwrkxD0DLHJnY+mlk5McYRVrrAz7+YiC6LErXXW+pBbXMXnUD/EZCaV5Z0JQoArS6wSS8fuT946b9Q4+77ysgucwfpdMEf+6Zw/JeT17qE5AJPrYMYUfbr9jVIq3zA5TFgoPZIVU1I3PZ3j9MMfqv32DtT4zhm2xneS5ZqEMIoBJ5nmGM+ByHoafrWszYzR96TwqJ5XxJ09QSXje+3jgeDsUuSTH0LX0nYNrR8RH3H97j4uqcF+cvOF6ecPvWKZvtll29QxlFJOCTBB9ppchcwWx2QAwtIQgjo4sdQ+jIDczmjrt35qxfNvTeEFRGF3oUhpCCSL9tRggdDoONicpZlkXJLJ9Tzg6pbhec3L9FHTdcXFyxWZ/h8gxtDNt6BUrx4EEhctrdmnIsWg6W89EbGG6dHJMXOacnJzSvvcauqaWZIKczbJZD12Fsjveybg7RS3ATGh/6cW0LDL2EnxllWMwWWG32DPibn6Mc9F5N8Z4abxK+LOvktJ0nNVqNqX2Lbp9bMI0LAEwi6Slk42dB9GtVlsznOB5K9xkHSgkgrhKogELY+hN7mxG0lvwBOzZ1p0OFvDfmF2Cil0WFDy3rdYfvdxglli0KCKPsfzlfYCy0jcd7Ya9nTuOHgeBlrTJAkTkZ8zEQQk/wIjP0g4RiTxYmpIQzkTJXzEpLWTqSH21NrKUoK4yxNI2oN5RO4hdsE32f0Fbum5THOBh8zXaXeHmexsDfSIoeayPd0IE2BAztsEXvAr3PCYOnKgpWm0uC9zx9/gTdXWFPDzEq8dGH77NZX+D7gTJX3DqasdtsWK9WPPvwXbpux2ufeYe02XJvOeetX/kaZ5crkpLAuPc/fMTh8S0JdTKKPg6AGhvpAtxGHykLR76cY7WhaRo2uzWZVpTzksxlWKXItKMqDimcwSlhwi5nM0iRuttQ7+Do8JjcSNjfBNzOMifvWb0iz3Kcyuj6Hq+1WFYpJ+NXgVEBP9RsL17QNFswjk3jMarEmYqDPKeaFWg7gBqom4HNeqDvPfQNhCuUcbikcVlFQuxUwhRCkoQ9HuNA9I2ErqtENwzjGhlRUfbeIrPjdPLjPhbpfSCisc6htMU6hyuyMTAxwzqHhEQKUK6ms2NQhD4ydJIAH9OYExM8loRJo4IiRYahpQ9ii0ZyOGvInaEsLYtFzsnxgqpyNPWK85dneN/ispw8n5FCRtP2+L4jxIHUN2TGoqInti1dGrAMMndjRGNRRAgDhIhRHq0GUvLCYIuermupG/ep5zZIgHn0gYgEqSY8yorabuh6lKnEm9+IH27oAiELxN6zbjvqbYcxmiIXP/rMWfxgaJuGSKDMCsq8xLmcrgsMOhBUJMscVVXQtDuMzlDG0LQ9eZahk2GWVTSbnu3VhrwMdN1A0lBUpRzMtKbzA6mXxp5xBt82xBZMXmBdxqyaU1QZ/aBp+0CKkWEMkAttS+wHfObJ8pKm69DG4oNYBmR5JoCnHB7JMiu5KLlDecOtkzsUVUmycLG6QkUja7jO6cOGoqzwoaHdbcAM9F0LMdAOO3a7FRTQ2h1FtsCHkuAjWkn9k2UFiwXUm475YcX8JOdgvmBZHPAybPj+i8f4WlHkBSBqnnq3lpoB8eI3heNoWXF8cszleouxOQ8fvsZyMePu3VO+9MV/jYPDI157+JDz83O+8mUJ7s5zR5FnXF1t+fCjj3jzzTf5W//t3yIRhM0p0mAyJyrMPkT6QYpmpRVWWXwH9VXLkw8ueOf2fUwOPg0o4wgMJAbqtsOJ3bioNWJCJU29bun6jjBEtOrZXDUURU6Iis57XJFTGkuZz9FohiFwcusWp7dP8U1PtANd07K+uiSb5VRZQdcPGGO5uFyz2TU0fqAqMxg9X3WyHGanpJQYBk80nqSlMWB1waw4YvCKZy/OyUtN73t2dc3t2xXNuiHSk7SiaXu0MaQobNmmqak3WwxzDpZL2t5iTU6KiaGV5mjbdGRzjTZC/LDaoXDEAfpa3qtPeykloYtD7wipIMQF7U7R7eZ8909fcH65HevSSNttqeaB0xPD668f8eD+fX707mNs5hn6KIFvMRIH8L1Y/cjhWnKg2iZJ0LRNHB1n/OZvvs3h4pjFQWB5rPCx5+VFy9mTLe987g6rlzXKRMLQMgyQmYwYe37/H/0DlscVX/raKd/6tRPe+dIhf/ztZ9SbiNYzjk9LNrsdeTVw78Gc8/MNTSNAKylwfHLEdrths+2ZLSJZrjk7W7E8LBh8jx8iJ8e3ePnynDy3BB+YzTL6fsBZy3bbcXC4ZLPe4JyAScMQWa/WdF0Y7QU7Hn30hDt37tA2WyFqNJ6da1Eq8eijjylmlpQ0Lg/MlzlROZo28uDhKR8/eUYKA0fHC/IsZ2YWvPXWVziYnfBLb88xzmIUfOEzX4boePPBHELBnVtvMPgd77z9S7z+8It87nOfo5oXvDh/yj/5o3/E7/3B72JYclLco7hd0l7V5CqX2tRozl884/nzSx6+cZuULP3gWR78/1j7ryfbsvvOE/sst92x6W5eU9eUhwcNmiRIsMHuCUocTceoQ2qFFJrQs/4kPUiv/dwPmo7QRCsC0T3TJEg2QRAECiiUN9emO3a75fSw9jmZVajioAHtils3b+Y5J8/Ze+211u/7+5qK7bLF+5b1esXtW/e4ffgyVT6HqPAu1ay7IPYdM9N7/xkFZd/3KJ0U4EQIPiK1pO9SblrT1CnDJTj6PqIktPUWYmqgEh2H8wmr1ZqApF+vODw6SSBjU9O1HRApy2rwbW7p+3YA2w1CCxor0EpQ5BmtbWjaDc5Fmq5hMh8zHo1ZLhb7ujkzGVfNirOn59y7cwcVNXdu3eXnb/+SrCjIy4Kzi0tciCkP5uoKoTVCKzpn0X1PjILVak1VlKwWS6xNoeh13bFcX1JViq6Lg+Iqcnh4yKiaUOYxNXV/m7Vb3Fj793XpAAZ+xhJU7MkDaV5IpDOJREWFjDr5Uu/IkSF52It4AyAnoKQA/I26ZWeTMeAf7N8CKQdtB7YPdQwpI2vvjTE82A8qBrHDIUTCSSUy4XMY8jIjlhnf+MarTCZzfvR3b/PLXz6hqXu6piPPFI1t0FrwlTff5P/1//x/8M1vfY16e8md0ynHhzm3jgvG4xnbxnFwMMe6Fm2gqnImkxmT8QHT8Ql/+19+BDIfcpXSPW2yHOcsMQgcibRldLJK6awdPpPGu4QLarNTxA+YZkpPRIqB6CQlQTikkoxHJaOioMw0/WaFbbc8evg13nzjdR6+/IjZbI5GIHOdcs1cQIREqIkxWc6+9NJt/of/6/+Zjz54l7/+67/l9PSUpmn2NdV6s0pZTlW5Z56fnt5iuVztc1BueqPv1M47BnsIHndlmUwmjEbJMkZKTT7kT2lt9ux8pVIDoSgkWju0Ts3gxeIK5ztmswnnzxr+p//3T/jTP/s97t7XSNnjXERJM+B3iSCdaoOBRBYTQUcMrZYd8vMZF4MBKIr7nw3fvzHkdkD9fgQOmOGvYOjDnubzh9j/77qe/7wP/v/fQfT0wQdmNx4fGIDiyI757ZyjHwIopIgDOCERIpBpcR32OZwE7x0uyH33wPpIn2ztElPYRIpMYN0A0kk1FKgp1FSQZPFaxSGAVOyZs3G40Qdh0sAoTMxvKYfHEon4gRmf3pQfpEwuQOvZv59I+rt3IoG0QaBUwOiA0VkKCPURIVOX1w+MMIVAKIkRis1yxZ03vjH4tHu8VyACUpYYPYZYo31N9JYQPCp4fEyTm0AkNttOUjNIiRPLFbRKgThaGTKjybKCopyTF1OMztLkJnf2FWkCEAPoKWVK7RXGIE3GX/zL74Nbccu0+E3Nu+9/xLtvv8/Jg3u8uHjBRx8/4/adO7z33nu8fP8O2+UZZ8+e8uTJMz7+5CnVdExelqAU27bl5794m/sPX+JvfvhDXnrjDU6mY0bHM/JvfItPf/r3rJYXSJmuxq5z6oMneAcRvLBIYUFEdK7JYtoQOJ/ChJwfrBVk6oIpkZo2IAiEwS99kHyL6xsugbS78RuHuk4ON3p6bUF6LTEoDrwNaVMxyMazTJDnGkegXEhWNgwAGeSZGN5HAsLEEFi7u3HDUPALkYruFBSyu7sF117RgWg9wfn9vZKA3wQ8dM7T/ZZktpsTx66LeO31vH/QjQ72/pnDOit+dQKDwXLhs5PdZyw2uBnwMGQLfA7M34GJaQOYvLNdcPSup+03bLYr6npLjJKqnDAZH1DlJcXA7L1p9bH7QKmuTuC5HZheXwSO7hajm6nd1agkM5rVckmeZ7z00j3u3bvLRx9+yKZe4wZ24V5uJ2XygHYOlQYEZVWS5zk/f+stlNHcvXuP2WzG8+fPiXhu3z7h7r3bNM0WIRITXgnITEaeF3gPznf0tkMJTd+n4EglFLdOjlB2xcq2BBNw0ZIrg4k6SZOFROhIbvKkVhkAi3JUpXCscUYUab6RQqBysw/cVCIFanVdR5Fruq7FZJrj40Pu3LnNixdnVNWI6WyCMorJZMT8YEZe5ljbsdls9qwX78PeCqLvOtq62YewpnDdoUAewkJ2oPl2u9kzX3fX9to+SHz+UrPvUcedvEzs7/mb43jHKrhZXMBnu+CfP65ZHl8CrN8c6+L6/Uh5PTaEUPsg3r3ia3cf7O+F37xJlixW3F4BkcCkjDp4iKnhkxgLHusstk/slNxkOJsCooxSeIYQZCLOW7p2i8CjjUmNpbYl2I7RaJTABK0wShCDxfVprTJGpSaKTgGHQpFYznqwfwsp7CzLFc4lz/am3aK0YDafMplU+waEMYbRuODs6hytNYcHh4zKkuA93nmMVkzGY/I8591332W1WjJWjmfPn7NcL9FaMapKqqpklM+ZjCvWiyUff/QRi6sFL54+xvnAaDJDScl8Puf+ndvovGTdNJydX6BITGlPpPdpc2iUJgho2g7vk28xqLRHl2GwsksBxlWZY7TGIKnyjNl4zLjIKTNDmWUQAzO1C09LknJvbQoVdg5jEgsTEbHdFuiJDlrbYWLA5Dne9qjY0/Vbnj/5iNXqHCECQSjWG0sMhnF1mCwObCA4i5CO6CyCBD6n+yKBs9IUtLbn8nLJtnWAIQbQUrE1Bgi42A5NIo33NTfDmpTSFEWBUqkh2/c93vvURNJiyCWQmCzf35NCJNZPY3ucS4ogLVO4ZxyaNjHuGrVpTfExIoYxhYiDbZvAdWl/4RwYLZOl2xA6r+Xgoe5syuWpEjCttcBbkXzbgyaSgp2DECitEblBqCRNl0IlO70qR8gUjOhjQGlNXmpc6Kgbiyckw7LPSVX/qw/bYJ2kcYLMlCkIOHqQPTFaIiVCJiZY6AV936LCBtdY1tsmkRKKQJElH/7cSKYnt1itrxLwbS0+qsRqFCA0SVFBqgEghYAqrVE642B2yNWz5wgEo6oCOmbjGXk54uLiit52BO3Jco3KDCCxvaMsC7x1KApyUTEqRoxHFS52gEfKMARD5xQyRzgwaFzrObxzTNhsiEKmusT1ZErSdS3eWkJnEQxgTF5y795Dbp/cQerAsj4n+C22TwrPvDA0XSArc6pxwdVqne5dEej6ls06ScKjUxg1QsuKqprjQ6CuA5mRnBwfczByfPDeh3hhUZlMFnMhY3nRsrioMboiNyUnt+bMDyaE0ONsz3Q8pW8dSmb86T//Z6y2KxbrjvnxHe4/eEhwPeNS8eCl29y5+4i+t8ymc37wgx/w7rvv8s//9E958NJDXOhp2pYf/vCHHJ8cwHnkYH7I8fEt6qbmydPHzKYzpJJcXl1yfnZO224gCpRKoWHPP1ny8Ksn5BoW60uk0iidE1Wk7reDelJgXY9IMAFZrAheYvsGlSUVbNN0WO8S2SpGNuuajWqRSOrthtm0ZNvOcEZQjqZ00fL08WOqquTg8AArJZkqWa8sl+stxXSEycpB7RvRQZHZEdtuTS979CiBN87C0eEt5tUJV+cX1HWLznI22zVNb1lttiw3a6pxgQugigKFoa1bBJK2tbiuo1OS8WTMqDoAQQLb2oAQedqTS0+Wp8B1YkaejVLo7RBo/JseD16Zcu/eV/Gu5uJySa7G/OA/fMqTD58itEIaRVQpN2w0hulUcHAgOD7KmM8PWa1/hosQkhkmRpVIYXj/3SXz+QHIHql6iixPa5U0hGBp+gUXVz3vv3PO62/O+fTTBdWoYrvxtE3HiydrtBiRl4HTW4LH3ZrcCKTUtO0WKQQ/+uFHXLw45c1vzviD7x7y/rtrPvmo4/Rkiskj07lAqh4pAkVWEUOPCy1d5zk4OOGDDz7F2QtO70y5c/sYlSkur66IEZQ03D69w3J5xbOzjocPD+j7JV3fM5lkrFZXzGZznA1st83er3g0EqzXHcbkKKX59NMXicDRw6PX7rNcrskyw8VlQ4iOeDLDUdP5wOHREWFlUaqirnsy7cBabp/c4aB8yHd/77/lYHwXbyPlKMe6NpHDZEYMGqUyvO9xvuTN145xXpPlFavFJbcPvsKj0y32azlf/dob5JnAuhU/e/sfyGLBYnPF0yeXHJ1OU9PRO1xX42OkrDTr2CDVlNlkzpuPvsUrt79JpqcQM2zvB/ZlInHFoZkCu31z2g8bY1A6qYGd94gY8dZDSBZmRWaQheH8/AVGSzarNX3bIETyQydA09TkecZiuWZTNyAEk8mMvu+wrqPrOupmw3x+wGQyZrFYYl2HtRJPz6q/SmTLEBhVBdu2RRmD0oL1ekHdbPHeYpRBeMHV+SXrxZpxMeLTDz9lXI7ZLjsqM2axaFgseu4+uEPbNui84MXlFZPJCNv32JCInIXJ2PR1sp1RhrrpmM0qbt+Z8uzpmtOTY66uzhkVFReXl5wcnKCE5OJ8sQ+V/E2Pz9Qr8UYZPRDFEjVw+NnuQbtn7UuP3Rc7NtCX1SphIBPdfF7cW9t95n3dUH1/9v0O5KDPPSMEkDF+5vV3NsfBWXqfVP4hWm6fvsRkcsLVZc3zZ0uUSspu36ex9Oqrr1MUOXfv3uXo6IjptOLps8e8++5TBPD8xVOkzHG+xmSSybRMzftcU5SnfPL4IzbbFABb1w3TeUlvO+omqcSV1Pvaa2cVF/wAzMZrUt8121neONlDNRkFTni0SUq3RCLN+MbXv8bp8RGz2YSTkyOmswlZltQGSil0loEAN1hLd61Fa4MXDk/g9PYt/s2/+Te88857LJdLRqPRHocIIbDZbCjLkslkglKKl+7f5969yDvvvENd1/T9NUtda723fNztvXe2i7uQ8p1dzE4NubNj3f27rhtGo0RWDcEzP5jz4sULQojM53POzpb8w49/wfHtR1Qmh9ijlNk3NH0IQ2NlZxMk92TML8So4+dG1h5A/3xdf4PEOXwthfySF/3VIxEGrl/r8/YyX/Q7v+z49ZnoJIZoiDu7gyRJThT5mNhYgj0Imti/iZGV/M4HGn8YPgFi2Oz6lI57owASAowEowJKRpxIGwBtQBmBtIooU5CpVJGUrm0Gn7okC755cvwAoO4kGjsP92TRsgO71J45GoHOQeMjiVh5HWDZuYiP4HxAyUCWgdYBIVzyjJVimKyG309i3BSzGZvlivOPPoZqhJWRLjqkEkzGZZKCC41SJV47pA9I6UFYiDIxvKJHyQKp9FDEJL8XpTRGl2RmNGwOCowZYfIKbQb/zxjAJ1BGDJYFDKFfuwCxt3/6M4zpeHR7Rq7HZPUVD++MeLxq+dF/+Vv+L1//Jnk+5urqilE1oq0b/u6vf4jqPRerBbLIWDw/ZzFMYGVRpRCWesWP/upv+KPvj/jg3Xf5m7NzTqaSiXOcvvo6q7NnqNAg2IHEce89HmJIALpMbAmpZPID9w5rk29953ZdUI9REi3TpJcmO/avyXA9hEg+cMYYpNyFZiarBTGsTmFoEDHYBchBAmWto+99spCJkSyXTMeGIDxV5hDNTkkhybREokiWEXFYgFJzJQRwfvC5hh1unho8MgXz7iYu5xy+S3IjxLXFhPOO3lo6F+n8b273sDtu+lHdBNJvfj+SrsMXSV9uWmpcv+aX/64EjqrPfG+XGr6T9CR/RD+of1KH2Pk+Aei2p2429H2PlIY8G1GVc4p8nKSBOwb6PzEZ3pxAbwI9u5/BdWp3+l7A9RbvLForDg7naCV5//33+fSTTxACsiyj7/s9ML9byNL5JIVlhjgA5jAvD4kx8tZbbw2+4RnPnj3j8HC2X/CUUjjXpWlCKYzMyfKSvm+BkJilQmByTTkrCa0g1JHYeXxQ2AFcUEgkGqMVVTliXI3JVYaSktFkxHg+YZM5OtfTO89qtaYsK6RWaKP3jOkYoe9bpIpMp2Pu3L3N3bu3uby84KWX7mKMIisypBJMp2Nu37mN1jkmL5NHc9uwWq5wzmN0llhspRzusR7vu+uE8WFh11rjrKXvLavVivl8vgffbo7Fzy7Dg4Ts2hiH3by5nxf2DxX7MbDbtPxTXenPAObxOqTk5piS8uZ4TmP65r2TgHO1B88/3wnfzUe/DYi+rTdYVxOiR2uJkIltpDVEArbvadqGGAfGcBR4J2iDxbuhyawitrc0dQrLHu4GrE3FUYzgXY+1LVqnALyiKFJIZJtCfbQ2g72SoWsd27qh6wf2upZcXp6l5kxZMGVK226p6zXWtmRGkWcarQUhpM1eZhTOGTJthuZ48vr01hFDYDxY/oQQyLOc45NjYrNgc/WMtmvIs5zVcoEEZpMps8mYi+cveO/dd7G95eDklMVqy4NHLzMajcl0koRqlWatTAn6roEY8CIiScoz79we+CUm+6e2S+M0EFBGYK2jbVokAlVV6FyTZxmTUcW4KpBx56ct0Jmm73v6vk+sxK4n+NRk3rYttktWSoJI26yJUdD3Pdgto9GIZrtBG9gsL3j68S/pbcN0OkLonHqzoirmEHvOXzxhfjADafGhBwFK6HROpQclcSjazlL7xETtbNqbeRuG0HeQOjWGYhjkxmrHhkvNmWSTlYDpED3Cx8F2xqN1lixebhaUItmzdH1PH4amOCkw3oWADSFJcolEkcZ3lIpA8s6PCJAJ4NVSYgeVYOcs2il8SASOrnWs13XyQvc9xmSDT3jyGg8hBZYH39M7m66PVogImUnKCmUkJjfMDg6oJhXW9bx49gm17YiDz7fOBdL0KO3QavDs+y2OICw+RuraEo0nzzKM0rRNgwyCvusJbiC59D3WeqoQ2GzWNF3L/HCa7n3rCSGnbVvm8znjUZUaYF1P4wNZmRSGaIkLqUHRdx0pI8EjhOHo6JCjw1v0XcvT8zNkZpjnBXdv3ebW8S3OJxc8OXvGNtbMD2YUZU5ZFVxcnFPkhrpuQEp87HBRIXVJt6nZ1EvKSqPKkk3XU3cRLSF6g5QGFXJmVca2WbO2awqZlINGaZSQRJ9URnXTMso842nFeFqyXF3S1FtGVcl5s6RpGpp2l4njKKscY6q0RkU/5Hg0hCAQwqBViVQGJx2myshiaoAZnVGM01pnYoGyBt9IrtZb3nv7I/rWcTIe8+1vfJP/w//xv+fWyQFSBJSEDz/4gGZbs1xteOWVh/zgf/6f+fbv/AEPXn6DpuupqpzjwymLy0ukkDx/9py6rnn08BF/9Id/xJMnT8hMxtHB4UAS8Lz26hvcunWbk5NbfPThR/zi7beRKuW7/OAHP6Bt2tSUrEZsNlsCgjwf09Y1Xe0xY0XbWBA9hSnQKoFzXkTKvCAvDEYVIBxaaaROVn15kZHlOW3XUi8amlXLdDbn4vkFfWeTsglPcAXed1xuO8bVCBUjrXP4tmamDhEykYW6tkcgOZgdMJtOsb7BoMlNRrCOPnZEmWpP70EJhY4Gb1OdNx6PkDLSdskm5fLyEmc9wcbUJMoybBeIUiGzDFNU6d6PAh+gKqohc8CidcZoNCXrO1TRIaVKNmcukI01Uhu6vuO36ZE9evAqxgysvbpBqsjXvnWXi/OPadt+sJwKTOYFo1IwKj3GOMoycnZ+Tu8cSDCZQATN+tIjXGqEP/448M3fPcFzRaDH5AKBR6qAjJ67L81YXp1zdDzmSBmW6wYWgfl4xPqy5uqyQwhFWYxQBjrXJfayTIr0sqj44J1LPv30nO/+2TFf+3aBlLd4560N5VhSjSxCeJq6Z3UZOTzOyDLNi+cXbNaO46MDrN/Qd0kZ98njZ0glcc4yqkZIKRhVUzJjWSxaijJDKgdEbB84e7FIbMq85PT0HsvlAqUkXecYjQqWi3VSFUSFsxlXV1uyTDGfT9FacLVYst1a5nNF2zUEBNpkLK7WFFnJdKLw3ZqR1rz58HXuHNxHxwOESZZgQXdISQoAVhqBQpkcpTIiAYUkRsN4dISMgbsnr3BrfofD+TTlgbiOs1HLNL/L5eaCb3+lQmWKT44+Zt09oe8brG1pNtvUcA+S0+P7vPLga8yz20SniFGQGU2IHqIfyMY3SSTXLExjzEAA9ETvqeuazKSQUWv7gcwkEtlgtSQGj/eWPNPYvqHerhmPkuVU36fPXpYFL86eMx6PMEaS5yPKsqRtWxaLy31oYgiBzjUIIsfzI642F7xYneFdR1SR8aTk4vKCW7dOeP6iJXrPdD5ns1kzHU+ILhEbDw+OEqkRw+nJmE+eveCX7zzm7r0j2m1L21ryIlkNhhjw1qOEItc5x0e32KxX9H2D85JmtabMC5wdCGudZTaZYa1ju+2QsuDF2eK3WrtvlhXXUOOvWkUma5Xrf+8Y6gMql/AdcT3R7ImBn8PCvpgMdKO+GPZd1zS2a3JSEq3sHvvZeijVPr9Cr9vnvMXBglbGyLNnj5lO7/Daq4/48INnfPjhU0aVQU0yppMx9166w0sv3eH111+jKHLu3bvDK6+8wsnJCR999BGPnzxBSMVkPmY2q5AqstmuuFpc8JOf/Iy/+9uf0neSvBhjnWS9WV27QiidgnNVqsec88NnHljnAwHyujbc2XgOGEHcsfj93r5kNptyPJ9TaEX0gdPTU779O9/k1q1j8iJnNB5hdI42JtWyQqBlUgls65b1ekvXdhQm5/joiO9+94/4V//qv+Pf/bt/NzSl8mRrOmRbbTZriqIgzzNWqyV/8sffA+DHP/4xu2BTpRRZlho87ka2kHV2sHKBPM/3mM2u7t2B6MnONn32tk17g11+z3w+p65rzs7OmM1mfPjBJ/zsHwW/8zuvJjtXnyzDED3JwnTXnNgRJgdb01+rpN2NtR2bks+MyMQovyZwflGd/kXoj5ADqZuBTB0HF5GwT+z7wud90fFrg+gipBMTkXt5fBg6edcDbNcRGIr/yJDcPAQf7uxwZBw6QBLn476G2MnWE0N1CCxTESGH8FGVZN5SRIxUqRMvFUIElIpINYBgIULYnWK5g2UTg95HbNvTZZK2buisw2SpT+/9AOxHQesjbQC7s8wV6bO4IHExdXPVzls8ugEQ9QiR2BhKKkalIStysvkRqhyTmZJPP/0UshI5LsjHJScnh5RZjvOp+BUiR2iLdD3CG6R0ON/hg0fLHKkydr5Byf7DI0WGFJo8qxiPTsizCcYkz+Ku7xB5kqCk7lIcbGBUYqanE09ZVTy894DHH7/D4ckx08IQmpzt8inLv36PtbQsr874g3/2HZ5erPjRX/0lCvDbNUbAarVmvU0JwY/PXqRAOtVS5jkoyeVmw4fv/YLD7WlSHGRHnHU9d+/e494rD7j86F3wgRiT45YcWMhC7qbxHVs8MU+8szivcG7HIBdIkmWGCzGFbbGD0tMtkZq0EakUhUmsSKVkWqCkZO/fBMnj1/ohnIQkNZSK6Hus33mzk7ygS8MoJv9zI5J3fWkSyGOdoPcMINugovAR78GHHbB2/U6VkmQScr0L/GAo6Dx28IclpuZT39v0WYew1d/m+MLJZw/qyc/8fAfsfcaKJc1me8DwmgE8hJIK9gDhfnOQJgh2XqzXIRTp9yTpZo8PDoYmGiRWbW97ejsoNlTquOZmhNEFWufoIQzxyz7b7vu7JGvn3PAe4q80A24Cm0op0B4pkkXLer3i2bOnvP/+eyyWV/vni4GxnQDnuF94d+en73vqumY2nyOF4OnTp+AF8/mcrk8SwefPX3B8cjgE5WgCDjd4tSfGbpoPVIxkKkeISO8dIQoOjufYrUWGBNz3PjV+hFIUWYEMcDiZMx2NqWSy6Gj7Fhsd5aRiHWucj+RFkUJDg0dISTVOIJdSctj8ek5OjijLnMm0oqxy7j+4h/eOLDdImUD20XiKNgUuRBbLFVonRn0MES8cSiq2bjP4oYf9hgeSn9vNkFatdUpYbxomk8lnrnMCtHcM7sjOizxGhw8W6dPGaLe5DDdDccV1I+nzjZQvG0M3u/ySuN+M3Awmvd70Xt8/u+elsR++nPmxaybF35zNtt2uQHiUEphM43xSLeyYIr3thqAaj1Im5S8Qadqa6BxVVaZGxdDwK4psCIWLbLftIB+USAXeJh9hpSuUlljXsV4v6fueTGdY15GZjBAjm22d/LSnU3xIgbrbuqGsUmhn1/U42yaGmvNYWyNI/vtpHrB435PpxCANvaOJDUIwsB8bZEwbxLIquXV6yjv/+BEmy8gyw4vzM4K1TEcjjo+OuLy84Mf/+A8czQ/5/vf/jLsvPaTpEli8vLxESkUxHuMWVyw3NbZrWG+3qeEqYgrBloKmben6PjVDzXVgs9J6UBxonBWs+hWjsqC8c4eD02MOpzMOJmMyrbB9D4Mkt20H0Nzt1FkeLSHLFVaoFPDVdPRdS7A9Ukj6rmPrtoQ2p+9qlBZcnT1nffUMISMui3RuhXARQofvtxSZRutI23es1gvWmxrvwEdDlBpZZESp6QLoaszJ7dv0NtB1lnZTY5RkOq4wRU7ddpydXe6biCEEtNUpzFyka2ItxOgpTAq1lTIFRRmT0dsEfutdOLoPRDpcUIlF5z0uhsToGTI6fEj5GkLKZDGmFAhFFBIXPF3fsVtrvbfEGOikoG4UfRuoN471IgFKUjmKQjMel8SQQLe+Sz7pm7ohaMV4NEIYTfBJjaGGIkmXGcV4TDmeILsWYUpsbOm6DuMF3sU9WWOnsvptjkiPj4HMKJQEjSAXOV1f42yHjD2tbSBC7yxGZ+RFgXQ9lg6tE9CTgt+STUueZ+RZRqMUqpAQJYGAlpFMKwqlBhDIIoSj7xuyTCXrolxjRhXz0xNC58iF4WAyYaQNj776dR7fOuTdyw8pRyOkUhwcHVJUBU3T4CI07ZroO+q2R648vetomg2jyZy266ltRyYNIFBKUuY5/WZLNRlTx+SnrMdjijynKLIhsyHNUcFHbN/y5PknONfS1Fuulhccncy5vLqgrhf44CjyjOBzfA+jYobOdMoSqBua2jIZzTiYnyLI0Lqg8S15ppnpGaL3tOsW1zmEExhbMCuOMFmOsIpvf/X3+f7vnfCVV7/KG699hVFpIAbGo5wQev70e39I1za0TcM7H37Kn/7p97l77xHIjKoaAQHvI8cnpwQkbW/pnefOvZc4Pj5mPJ2xXq8o8py6qSnygtM3btM0LdvthpdfecRkOmY8mfLWW2/x+uuv8/TpU4xJ/vNN09I2DpEHzLwjEyN0hEkxw0tLkRVoneF9wOSpIa+kYDIa0bWJ+KKNYjqbUJRlanBaRVFWbLctfWPJTYaMgcm4QAqP1hBch2s71p3jYDrh9uktnO3JtSGXGbGNlGXB7OiAg4NDpBJkCrauIwYPuaLKKrqQ1jOiYVyWXD2/Iswhzw2TyZimWw/e6RKlMsbVlM1yiSqz1BgTkrLK0JlmFiNlkSOIZEWJ1BkmCmbzQ9pNTVWOmMwmeLnChT4pXXrHerWiKspBwfDrluO/eiwvPmYynaXGdD7CeXj1tVssLyI//Ot3ECIynxvK0lOWUI4kUgc8nk8++Qgk5LmGmPHiScdm4RmZiqKa8N47Kxbrmj/87h1Gc40QjrZbp/raJGsrIZ9zefWU+y8foUxOlo345JcXnBwfUFZbHj85Z9t5VAHRSqTPwGmC2KCqDTIEhK54/qxFVZe8+pWv89ZPf0q32nIrmtSYiYqvf/0RH3z4IZOpIC8Uq9US70sCjvsPKp48fUqIkfWy4fBwwuXlFcZoZrMJd+9NcL7D+Z629eRFspWqygIlcy7O12w3kfv3T7G+Rqma5XJNXuRsN5ajg+kQeA1tt0WbtHcfjyY8fbJOvsuZwvZbtKzYrFbgArmYMhrl3L/1MneO74D1KB32VrFSSAgCJRLpZLe31DInEtBaDv62Kann1q1bBNsNlp+K6HO+/ycPCFhcbDi/esGzF0/5na98l5/+8n/hvY9/ji96FptnROs5PrrL3aPXKNQRIhqkTASlEF2qfof6OYFjYrf4pFpbMDSMA9Z2aK2TSssoonepdonQtz0iDqRGJdBaEKPj6vIcpSR1syX4gNSGsipwzjKbTzFakxlFJNK1DU+fPsV5x3QyYTabJaLfoAyushHjYsLF+gqlNb3rKWTOya1jgnfkQxi7ljCfTFiua4SUFHnJdDrn8eOnFGXFJ4+fcnxywrOLK85erJlOMgSSpm4Jg3WkGSeSznrZ4Hq4/+ARL86eEoVEmxzXBS4vrlBKDNkOlnrboHXOuu5Ybbrfau3eHYlwed3YCMHvv077pWuwL4a0bx9EQL9CHkqYgdsD6Z9XVn/mEOwtX24C5/v/PkMiSuMlfol17LVyfUe+iwPgJwZry+TpXJbpfrp954Q///M/4+OPn/PBex+hVOTg4IA333yDyXTCZDpJdk0+UFVjfvd3/hm/97v/jBdnL7haXXB2/ozzy+dcXl3iXEdRlEymJf/iX36PH//9z/nwg6cYU9A3NtUnUqC12rPRUzNpV9+leT8O9s5p4o77ev3ah/uajOmdJTc53lneeO01/vgP/xmnJydMJyOm0zFZng+1zHXTg5Cy15AKLRXzmaFvX5CZdM6WyyU+RP71v/7f8/bbv+Dtt98e9sawY80XRU7ftxTFIa+99hqLxYI///M/5/Lyko8//nhfJ3ddt68zE4iexlDXdZ8hbCml9o2sHXFsx1ovioK+7/fXtm2TwjTPc5xzrNdrtI783d/+gqPDY+4/OMbadaoFQ1IQJgKzRoShTuMa9N6P/y9F1K8B9FSiX4/11NTZJ+vtH/1l4/LzrypvwPERYMjpvFmHf5kC/ebxX2HnEvZvNfkF78AClyZNyRAaxhDSyWD5cv0aISbZhxgwDu8jziWWuIrJuzyd4GSDYXQKm1QygSGJLXwNdKZwg4hUGp0ZTKYJrt+DGqmrIBJzPATswCzvekvfysSE6xxFLpM9got4n0DYxkW6mABaJdME5EIgugSyix0IE9M52AWqCkALjVGaPDeMp2NuvfkGR6+/ihMZHkE2ryjHY8pJCSLQ2Q1CeLSqiPjUrFAdQnSJyRwkSpZ71rT3Fh9SEdE7h1E5UWqULimLOQfTu+wmPNt39G2DUpo8HxOiHQDnEmQ2XE+4uLrgpZfvcvv0gHHRM8olUbScvvSQk8P3+PlbH/Pi+af84e9/j/PLNT/9+x/x7MUL2m1JrgTbukFJxWqxRIlkw9C2LSJCZzvceMSP/uGnVO+8z6uvv8bl5QX9tuYnfc137k0pyzLJ7GQqjPYM4n0owTB+fAp3FUIOobUREcGIiJZpPNiQJgctdszy3c0w3CwxoohkMg7sH0Xcg+7XXtjdwCTWSqC0GDpV4OwgX9IRrSHLBYWXVEZQSFBCMi+hyKF3JPBy395NNi5+UGyEGIdxn66DklDkgsIIbBAD8y3iwmAvI4eE8s7Rdx7vIkKoBPL/FsfnLVd+9fs3wfLh7vtSlvfNie66g7577etOZ7oiIfprFvgNENKHnq6rsc6iZJLdG6PSYhIS40QIgZKGzBRkeYHRCZAJ3uNIndUvYhPfBMWvAc3Bey5cbz5ugqo7mZO1/WAh4Af1S5Kia62gu7YB2THhdwEfzvn97/IxUFYVQkq22y1N21LokhgD9TYlzD999pRI4OhoznQ64fmLLcZoQpQDYKTQukAgKKsJXd8gtGKxXHEyOeb0zgm50BQyo952tL1FZwVVOSJTGQfTA6ajMUaA29Z0rsNGhxYpgMX5HcibQgaVUmmzJSTBh2FR7xhPEnjetZYHD17i4GAIOSFQlBqjC8pqQgiC0WjMwfyAo8MjmrrG9hYCibFmcsajnm29oW3XewsdKdT+OgghuLg45+DWCRcXFxhjyLJsf9123fTU6N5tHkMKiLMt3ifvuxCTd9tuHMehU35TUXFzXOzvixtffyYxPCae+01Q/Rqov250XW+Sw1DI+M80wD4/RtMm57dTmUQCUkaUFigFvU1ezN7vbM7S4/IiQw+Fl5QyeShbgdak8M5SI4DcKJyzEF3ypRU5trd43w0N5MTWdrZltXQYYwgh0knNZrtKViyZSaGcwNVVR57lKVzSWTbrhq5bo7QBEVAa6qbm+YsntF09bOBSUyF5+CWrt7be7lkT4/GYut4Qnadpk6+gFiQ/XpPuWRs84/GIoiz5ux/9iF/+/C1eunuXlx7c5x9+8g/87Odv863f+T1Ob99BmIzeeerNhtW25nK5QkuFITWu9BBOLYWk1BIdFT6GxBCXMgnwZLL+UCKjVwKRF8zHFfPpmCo3yNBjuw3RSlxvB2udDoFDxGEM+WTxJbWiyDMynYCIrutxXUO0DVKC9g63XVE3qcjddA3nz54S+g1lVaKl52qzpKjmGBUh9ggCH7z3C6SRw1iQtNZiTIGpxujxGFMWtD7iRFIDjsYlSjW4vkEC2ghyo+itGvZ5idljjMIP4dxaK/I8w5ghVCs4tE7MmTzPsdZjnR8se9LrWOdTkRPjsG9MzQRre5zvIQqct4QoUXJYQxCpKWT0sKdIQLvrOpztEQicgEak/YCKnlp6hPQUhURqRd0m65GubgeFAwRniVLQW4tQSZ4anQOlCCo1c65WazZNR9u3dC5gY/LhZBfKrSUqS+/Jhd/OV3W1XkNQZKLCuZ7OgSkMRpRE10NIyr4QGZhEh4zGIyIlcdnT9x3OWfrOE0NSlVxeXpIZg0KjcoPUJgVcti3T6TSFdSuNdYHlqsa5jhhzNpsls9kBUitefvVVXNNDa5lmI0yM3D46oBgpztyLZO8B9E2DQkEQRA82tqmeEHB++RzrLIhAkRfU7QKMJS8zzCAVDrGhbWuqUQJ8pJCURUFZ5oyrKpEOrMMh0ELiQs/F1TOaZkFVlNi+pcwKppMRTbNO83SAIhuhZcl0PMLGjqbfIEVSvGlVIchomp5qHNCZRMTIxCRLqGAFOioenjzgYHqH44M7RCfQIuc7b05RIqPQBUWRrI12itKqKpASJpMxZZlz30uK0Zyu9eR5SYgCF1yyiopQWUtelgilODs/5+T0FKEUflhzBJJXX32Nvu+Zz+dIKWjblkePHvL4yTO01jx69IhXXnmFvu95/vw5dd2wWbc8OfuE89XHNFeOrDCM8gmyjIxGVZpnM01R5JhMkZmSyWhECJuUiyKTp3ggsNnUSKUZz6YYnSOiYD6bsN0uOZiNyXPNdrNCSk2RzZFRMqlKCIHV8grj01roZECNDOP5hCgCdd1ycDChqQVd11KMDVoZNquauu3RQrCxG3y3ZjyeUFSJadu2DT46hMr32QrElIskSPtraWRqhjpHUVZp3jJ6z6gq8xK7bWm2W2aHM7TJsW2q10QUNNstrmupxpPh+v5mx6e/iLTdBZNZxemdGeU0EvSCk9PAndsFPvaMykieS4yJ6EIjhWO9bdk02+Q3LSdcnlnWi8Ta7xqL7TzFCOp14D/8fx7z1a/dwTnLg0dT8ipg8oyugZWvKS8AAQAASURBVNunBVIFNttzlMlBaratw9kVJ6cV5BOa1uGd5vyiQceCR3dfhfxDTNnhrOHyaks+31K3iqvL9xnNYLuRWJdYoQfzkouzx9y6lbFtG+YHFTE0OOd46cFdtBHM5hM2246iSA3Zru8xmWC1XpFlGqkCWSbZbCJN47l75xZ9byEa1ustF+dbjo8UeZkzm00QMpJnBc/CJdpo7t+/Te/W9DYkayLp6ZOFNyJkrBYWyYKq9EzKgvMnDS4TlPM7lOoO09EpWimkdGlNGLJuCEPVKQCS3ZuQEqQmCjGozRTe9Qiph5I8olRO9AmgE1GiouBwdMr0wRwfel67+w3G2ZSfvf33FGHCaFTxh9/8Mx6e/i6GKVqnAG/nh5wQduQekrJ3T1ZkTzixg7o2Ad6G3Ci6tqbrGqRI+XfguLg4Y71acXp6BNGw3awYVSXb7RqvYLutefDwEU3X0bY1ZTWiaWu225Sbk+c5RZnRdelcr9bL4d6KyJhY7V3tKEyFdi1CpbDDzOTI3FDXG06OjvjwvQ8YjyfkRU7b9SzWC56+eMa2bRDKoLKMy8US21oOD+eMipzl5RWNCCglKI1hOhqjZEHfeTarNQ9ffkBRjslyQzHP6Lcdl5cXTKbVvi5z1lFkI549e0E5Ln+rtXuHqYU94B0+92fAy+QNFnqIxJhYvSnLz6NkGnvpMYmccF3/7P5c1+87UPyfau99mZ3L7p3fPAbx//C7d78jPS6RAeUATqukLM0c08mMIp9y794DvvvdP6TvWo6ODrHWopSi7y1N0zKZzBiPZ3jvkFJxeHhM72rU6Qlaebpmjcw1CHjw0l1sH/nen3yHrvlfePb8DIg4G+japEqJCKQRKdTdJdKtIBL8daZRwrvTOdsH8IaEQwbSvmNUFkymFX/0B9/hv/2LP2dUpDVwNhmRVzllWSZbZxXRMeC9Rasc5A6wT697dHTCRx9+SPSO2WyC1oYPP/yI73znO5ydnbFcDiSkgVleFAXj8RghBHlmeOmll3jjjTf4i7/4C/7tv/23rFarvU3tjlG+wy52TPqmSdapO7/1z4PpO9xis9mgtaau62FvbsmybJ9LlZjqNU8+2fDDv3yLk+M/S+9TpvoQPJJEwkiNPIn8AhK6+BJiuthhMFzTTXcj8Xr0Dj+/4Rcudn2Qzzznc68dbuJyqcEghdzfN3D9Gv/U8WsjbyFaBNm+ixl2TNN43cHZeS7J4RPsQM5rSUkCQKNPn9KHHbAekt+4J1nEAFpLikyQG+iVIPiIGAIRQkyDwYdBrmfShtmYjF64ayuV4cK4wXrGx8GLdAATdmA+JAsY79Xgn50e60O6JjsQzQbobKAPUMqwZ96n9zTIACKpUHMObyVXlws++cEPiP/5r7l99y7Z3UeMX3+Vwzt3iWJGXuQoExBY8mzHjjVImaFlhcUhZfKjT53lxMTdBQv44FOTQUikydFZlTzElGG9XqTJyHVstleMqhTiEUNAFiSm1m6IecsHb/+ER/cfIIdAiKycwuFt3nz9IT/+5Iz11QW9s/zxn3yXH//9j/iPP/gB0VruHB+QFznL5Ypdk2lngbFtaiKCy6ur1AUXkvPLS+7ff8BycUVnLVfPn/PfnJbAkh0wGoInuPTZo/REmWQMSmoybQZmZmLXhpgYUgJB7yOdByUDhZLJuCHu5N8QRELirW0JQYPIEDJnlxsoohg2GkOhzo4VK4buY8TZwepFBpSCLBN4JymNJJOpIJ9VgtxEQtpFoSWDbz9IGYexk3poIYrhZo0o6alKTaEh9gyWPsP4DCkgzTlH1ybJoveD/C789nYuN4/P20r86lT0Rd+7+f3E+N3JefY/vQFIp0n4OtjTe7/353Le4Vw9BMs4tDaUsSSEJEFK84+EqAY7oxyjs8HTzw9J2wnITrY912zlm39uMsdvfvbPA+g7T2Bre6yzeJdCpeq6pu1aiiJPnq4x0tt+Lx/bgbs7n+396w/ZEjuWtbWWKhuxWq1YrResViuKMuOtt37Gt771TabzOdt2Sz90goVQBAJRKKwLmLzAikBUoKucxnXcOjjENR1h2VHkI2p6snLEwcERmcqZVlMmkwlFrmiKFQ6Pw6XARqMxIUeR5siuvu5USylSZkHwCCkoigwpFU1zxqNHDxJrY1QSRUApidKSLDfEIBmN4Pj4mLbryYzhefWctq7xLgH152cXdF2/92CXIqWXE9OcghAsFkvatt2z+XfXeHd+d5udiMd5i3U9fd9QN/Ww0RNDN70c/PGS7DUOHhI7GduvgOgJmb9eYG/+PYDjN9UWzjl2+Rw7Zkkc1rv0vPRepfzsZnfX+LnZuPnNuWyQZSncpe3SNZND0IoPu3BTlcZvkYJcvbWJZZRl9F1irXsX0Tq9r95a6nqLd5bRuKAsCloN1iZQMssVPvR4D72FzGUDkG7pux6lNRUjtMkYokgIsacoS0wmWS4XdP2GLMtTgKwKiOiwvcP1OVpBDCkcNa9yhEiBrF3XQvSJkTIepSIkREyeAtSUFByfniCjZXV1hY+Jde36nqdPnlCUJfP5AX/5l3/J6ckp/5u/+N/x8NGrlKMxUid2dEAwqWum0xkohfNJ3da7nqavU9EqUpNmtV6xqbegJCbL0FkKc3Xeo2RGleXkRmG7mu3ygvHRISIINtsttkvsmb5ryJRI4M4gs1ZaEtEEFfHW0my3bDYbMqPwtkfg8L5HEfDR4r3l7MULFotLOmuZzUdYaxPrJWYIWdA2LR9+/Cl1U6MywcmtW8wPTjiYz5CiJOYVQSt667AB+phA45QXkZhqIUZsn0Iqe9slH37fD38sSQ0RqUY5xkiyAUhv622yCRkKLmd7rO1AqJRHE0mqq8G2zLqAt5YYLb0RhFCk+05GXNeDVBDAhwylUoiz0QolSISHvsFbl/axCrxL67AYQA1jVJqviMm6revTnkgk5UqmFVElsEGQrnVnezyRDINwPZt6k9hNUmAHJWFSEHmUlAhJsnIJSVH22xzN1hNdwOmOXCuUSuq3IAxFUaEzSYdFSMlkMmV+eDQ0EWG5AmtTkTsezzDaEPEJUPcCMDiXWLzRhWEeDnjb0BOQylGVerDlgnq7YrNaEV3asx3PD/BNR4HidH6EMhK77ejsFiGTQrKva7abhvVyw2a9SXtTI4gyEvrIer1BSUGwkbIo6FxHngvo03osRUAZDdJRbzdkJif6QPABrRSb1Yo8yxAuIjOViAiipQ8B7dOY6Jqe3BSU+QitdLK8ycdUxRgQiKCw3nN8dJvtqqZtHPW2QYqMerPhYDLFbTqadkNcOAw5uc6YFlNMLdi2SzJTIbSmbrY4v8HanquriwEkGDMqR2SdJjcq3SdlznR+TFlOufLLNOeZjFznzIpDlqsVShnyvGSzqVlvtrx4cc5oVCGVGZrsGUoZikIyGlWs1ysODmb0NimRbt26xXg85unTp3RdR5Zl1HXNeJxxX51y+3QMWYv2jq71lIUBF7hanKO1GOzBxOC/mhPjmrarAUl0Ausj2mSMJxNMVjCuwtBkaxjPNAezCWVZ8OxJZL2sOZjOKLOS+WTM2dNnZEIlhSoxBcxGjy4ldbel9z0uZrjQozNFNSpwEZQ2FOUIb2PaS1iPdR3OKdbrFavVgvF0Onhmp/1E19UUg62WDWn+absWKaCYlAgJWZ7CK5Oi2dL3HX3b4WPH5LBASY2zPUYX2NbSNl0iPv06lfiXHD/4Hz9BasV0NiavzlFlyytfnXH3pZyvf2PEZiMJHtquQ5vEcHQRlhuL1ALfBp58suLqhUfLkuhSveiDxXnJ4rInzzP+8e/OCFHx8QdbvvPdh0wPAm3ds9k0TGbJ176aZGiT8+bXS9775UdEHRgfeGSblGJ3pznBeoL5mG1dM68yqgOBGgWC2KKKis1FjfWGssoZlWMuzhxa5TjdE6JlPElKo9m8om0dm80VKpPcvjOnGo+5ffs2T5485eLiiq5b0/eBBw9us1idk2WKh4+OubpakGWGuu44P7tkMi7I5gc0tef23WPe/+AF1UhjMsHJ6RxFxgcffsR8rsmKSNNsGY1zTo4P+PTjJ3Rbz+E0R+AZl4pCVxyXJ3z7q3/EH/zu95lkU7TUBBexwpGcQTVRpqZdIhOlLJ5UH8eBQWkgyGQr4tOajpDE6EFIhE72e8EJZJDEoMhMQXCS+ycvc/70kq+9+ru899E/MpvPuXP4CibOyNWE4Fc4XKqZ9ZCfJlMTqLXdXoEZP79vDYnh3HUNccglE4C3jrZtENIxm044mE85e/GUertmMq7o2hohkif6ZDLm8ePHlKMKkyUrp+1mTZ6rARxtESIwnU72tU/aA4KWBhkUF88vKW9VXD1eYcaGajLn40+e8sZrD5hNpzz+5CkpHzwiJPShx+N554MPmIwmuBDZtpbJZIZtJFU2ozSCw9kBFxcvODg4wNqW5dWKs4tnPHx4j8VyzXK5Zr3eUoaCZ4/PeXT3Di8/esBiecXF+YqiTDVD3bSYTA37lt/82BGFdtdhd032IDqDMtvHz9QExGRZJeJ1fbGvYeM1s/fztjDw2RoYkgPEdfaiGKxXhtpXJJuLHbiVXuMaa7j5Op8FQePu1dL7EYEQBASLdR7vNFo7rMuYTo4ZjycQR4QQUm3hPSD2lp1pLc/YbrfY3iIF2K4lywxHRwe07SY5Bww2fHfv3uJf/+u/YLHYcH655uLiis16w8XFFefnV0ACmWMQw74snf9rYtM1LrCrB8NevZssoTOjKfOMl+7e4ZWXH1FmGcF5iNd4hhyyrkKIeOcQKhuUKgP4GAVGZ9y7d48Xz57y3rvv8uDhIz786AOs7XnllZf52c9+SgiRpqlRSjGZjPnGN77Oq6++ihCK27dPiTHyrW99i5///Of8x//4H/fXfdeQ+LyVz06pu7uGu793iu+dMn9X8++IgDtblx2+l76X413gnbef8N47j/n279ynd1skiaQqBq73zm9eIQlDyGj6vRB37+HGOPo8YH6NLF1/zY33nkbbrjHIfsyLPUh+g945uB7c/KbYnZ/93/9rbaZ0/FeA6CSrFaURMSKjYOfPkjzGRWKWDb8/ADYEfJQp4Gk4Ob0Hl+6PZNVCYn0ksFvi4pBWqwRVrilMoBYglEAMG5JdeEGIEaVC6sIXCqkVcX/i4lD07IJBBymVlpSVoSxMkr8qkbzWVbJ9CYN/tY+JGa9vnMQQBBZog0fukpPjLsg0XSYfQgKtm8hq0+GCYOsFUfdIAlMhqO6foo2g77dYu8XaBlPIwRcx3dTGFEQPbdeQaZFCA4dgEhgC8ELYu/1GMYQ0RkGIgb6tUVKxXlxgSQFhdbNE9wZj8oG1mmTSSmmOT06pbz9gVJVMJwXKNQgfMVng5Lji4uySD97/Jfff/BbFaM6f/Ol3+S9/89fUm5pnZ+eUWTZ4wSaAcOdL33uHFBoX02I8m04IMfD02WOKvGDbNLy/DTzIJ7wxnRL6DTYE+i7Q1g1SOUoT8Ps7LqCMJDpPGNjZYViAfITOp4C3fNi2hAAuRnz0+GT4kuxiREBKh5KDfGfn2S/EnvHmBqAzsenSQuK8p7ND+KDahdSC0gKtQAtBpiPjEqRK4LeE5OkuUn1PwgSJwxhzcbhjRPIQK3PQMiK9R+lrUD+EFPbYtJZIoOsDLiS2+2+FsnG96H5+0rienD577L4Tf+UZX7SUpkftF+0BoBRi8KsecgoQaUMkIygEznu03jWVzH6xE4CUGh93E7/Ze+mqfdd16NANn2EHpF8v+p+17EiAqR8AoeH7QRCGhUVJhUUghyBL7z2r5ZLLy0uSDGuF1gYpkzdm8v3VafH0wwWK19Iq5x3amATCDT5m02rCs2dPQQTeeutnfP3rX+Ojjz5mtVryz//se0ynEy77DqTCe0mIMnmsMVgYDee4HI+JjcMJz+Rggn/eMKkmNM4hTc7BwTEiKjKVMykqyiJjnOds/DaNaSXJTUlRlHT1GiGgbtvEQI/XTa4k/xyknlIxGo84PDrE2mQB4oMjyzKMSVJZZOp2y8FCZ7Vas93UdG3DerVOjEHvECQrlyjSuVISjDL7c9d2LZtNYh03TUOWZzjv0muLodkoIiG6IYSwHx67HVrLMtlNmWzw5d+pQnyShbOzJfqcHVDc/2+/cd0vtDGtHrvxlN5rmquFTGvjbm1OILpAiLSQ73JEhicPTZZAUIqgJF7IX2sx/7Ijy+XQsB4sbVQCnSMiMadETIz+PCPYmt63xJgC/MDhXYcNHjMAhyF6+m6bij0SC0ypiMkkQkqc6wC3P3dKRbJBvYBwQ8HU4ZwnLxJjIyLSfkCAUoHeNcPGzRFCn7JHTEaWSQQeJQV5plMork7sIO8iRabIMwExqVeUUWSFomlb2mZDMaqocoV1lvPLC/o+UJoMU+QcHsz59MljjFJ8+9vfZjY74HKxYuzh5OQUY3LqtkVJxdHhEUhJ23Vs6xohBaNRhclTw8BkOdY7Xpy9oO27ZMnlLNttTV8nsD9KgW06enrMNEe4hs5ukKRMGOd6chXRuCTB8kPGgjTkmaHINdFIQhixWK6ot1uEXZNJjxaBKD0iOi4vnvHpp5/QtD3TwyNChKePn/LiYsGqgdU7H/P06QW9dYzGJVmu6W1HRDCdCI6O5uTTCZfNlovzCy6Wa7zWFOMxkcRsjbYnN4bgOkyTY0MKB9Za0LQ9zrVYa/GxpywNo1FBUVSURU7fJlZNJAzBYw3buiEiUDojxmTp1HWeIApS1o4lhA5fSJRM96yWgVVXp3VD5XiXE6VK+9PowQd83+LaLcSI3DGt8cM6khrpaS+awimjtxgBVWYgRIJzKJGsjQwCNWzcrfM0tsNaPYAQbQLMtd6rNIFk/aJ0Aid2DeTfkonebQXWOaxxxMIjC0FuMiKKcZ4RQ7rXsjwxmEajEZt6iwvJ97ttewSKzIyZTg+I0XF+ccZ6kdY0lGByOGI0nbO8WnB1sYCQQPNqnCy+InIomCJNvcXVnuZqQ3YU0VFAUVBNK56ePePDZx+y2iyYjCfIYDAoKmnovaTpAj4P6Dwj1ylAW2qNt47NpkVnChkVwUX6psf1SQ02rSZs647lckWR5RxOxkyrAiklV1dXVGVJHhWrTYtSEZUJXOhouoiRJd5FopMYWUIU2K7G9gEzLdhutwQpCV4yHh2SqTGX/ooYJWVVIEVEuUhoA6Uv+Z03vkEesqReCInZXBYTEBnWRpq6QZiUIfXp48dE+ZyX7j5gPEqhota2VGXGdJYs/3SWmgEiQpnniCxj0zRkRcl6u6Eaj1htN5RuxGqzYTSdEIgUecmiWVBVmh3wkRcZVVWxfv5iAL0alstlYrXlOdPplNNbt7Ddhq6tWa2uuLh6RrvqKaYjsqBomwVdv2U0KshyjUTQNB2Z7jE6x4c1vW0BidQ5s/kck2VIlSHziPctV5fnGCPoQ47yGhdBZjmUGV2MeC2JuSZkGqugHJXoUqaGU5bWXaM1DksxyhPjKaRGuJYZeSFpY4/tW1Qu6V1P3USUEsOaoZmMp0BG23T0YUvjGkxRoo1ER0FhknVOURo2zRYjDFmhaWwKR+y6mvFojIuJYZuVGWWu2CwsMQS0hHa7xba/+f09vWUSq5CWTJTgC95/+4L5YY4ZWQ5GelDHJtVY1waEhzKvsHXP5rJhedYjnEEZSV6ppCIosj1Q5a3AqAyJZL3o+Z/+x58xnWaUZeDOS4mJnta1EmLGeKoQRuFDCnLLjEDIQCYMTz7dUtNQby3rTcdXvvaAu6d36PqGF2fPqArBK68ZjJJAS+8cz88aDufHeAJFVdDWFqEVV8stb3x1Tt2vWLWazaJhtdiSmYJCz3nl4ctcXj3H2Y6ymJEXgs16w+mtY/I85/AwY7vxOCuIYc3dewe89+4vuHV6QN9vKQtFZiKr5ZKyTCCZc5rFouWluzMuz5bcOjqgGguIJbHLmMa73J2/xr/67/4H5pNTYh8Gy1qIMtWhYmB872tJcc3QHfBJvPUYnQAKH1J4Z8qaSWM8BfUK9NCoRgS2rkMKjzESHQzf+b0/onctb7zyTfI8nROJp7cLpPHsFPtqQIpCDGn/u1NBD6rNMOTIKSlxyVwcby3gyUwCii0RK9M614e0nk9nJdat2NZX1PVqaEAP9ra2JfiMdptstLabNYfze3R9h7V9+lwy7RH6zhGcJ4pUg202C46ODlm5BqUqJIqP33/OeDLifLFlPB5h5SUYRWMtXdskC97aEYOgqTvyquTqYpEC548m/MNPfsIbbzwAUtbD0yeXzOYlWiru3b2F0hGpLJ98+hGIQJblXF5sOT1cs6k7Ntt1UgF5ECrQuw2m0pjytwPRrzO4hrEjBhuUzz0mHTdsW0UcVG3p8TvW+vDDYbz9KoAOX0QkE3sQ87piv2Zg7xDLnVXmjkzEzdfZPfNzQKT4zP9SvW+0Sba1WVqX8iwjMybZtPYp+NIYkwg02lBVFXme1DxKaaTU6J2VMzHZJgqFMYmwpmSqIe7ePeH+/Xt0PTRtR28d77/3EX/7N3/H48fP0FqhVEbX9RBDUnkPZCfvPWWZD/dIspTRWg9kOI3JNKPKcPvkiIPZNCFLQlCURbrv1LUdjthbSsdBwc4ABKkBUB/ss46OicFzdnbGV7/yFf7Tf/pPVFXJ3bt3efr0KePxaHB4aHj27Cl/8id/zDe/9bu0dbJfklLy53/+57zzzjtcXV1xcXEx2EemRlVZFkPDzu8B9p1yd3fsgPG9Ram8tj5tmmafzyaEuA4jlZqqOqDv1vzVX/2I2Uxy736FxAz2NWJwQ483xoX4DOb0Za4G1+N2B5zfgNbjNYAudk0Jbjaa0rndAfA3wfnruyEmGPvG2E81/a8Pqf3aILrcSd2FSuxYJfa3hfcugYE+dVXDcAMnVveu+zqwFwc7DgTkg0x8t8AESH7jCIyG3ECWJVZx8mHye/CGgckrlUCbJBNWUiQwlB1NX4IYvKalQMmI0WAGBgUk0NIYECIFQvoY8SQ2eYhhf6GT52bEEXAhgtqxagesLoJQCdi3LtDbxJQPQaYwShuIHpr+I1oXWF5cMD2YorSknE+ZnszxziFVWpCVMijlKPMxdbtCySxt7p1D6wy36ySTghdjSGFl26ZGSIN3PavVBVJA12+QQlGUo/Se+wTi6GxE31vadolzPUcvPSQTnrxSiC4Q1xtKU3D71iFvvPYKHzw55+7bP+OrD19B+J77Dx7w1k9/xmbT40yHlINUbWDmpwF7PelGIovVIoEmAjbbLePRDCkFT0TG//3/9m94969+gG8viTqjtx7lejIZ8Y6BJZ4CARCDV7hL1wXSuQ9x8EUdrnviAAwxEXssLKYEcDmAtsP3QwgImSTmu8k0AbYJWESIwZ5od8Mnawul5B5UVjJyWMFkFJAy4IbQUyXF3u5h1zFLnvZx8Lke5laxs8mLEANSJOsk7zwhMHQkPSk8LtK7gPWKm6Eiv/ExzBwiXocqRhET4PO5RTgOMo6b3cBfeS1Sdy/eeN7Nv9OmIG04pTaJIaOGrqfziNwnadDeakOzzzkQanitPjXCFBgl0DIxnluRGLSEND6E94hh06iUTjZJg7dfut6OGC0hduCShQgDczgO97WUMi1WRUWRlfStJc82XPQLpNRImdF1jswUEGXKJOhaBGqYt1Ka926C987tmffBOdaryxQeIiVvv/2LAWDq+esf/i3z2YRvfPMrlEVO09R70ElIgckzmm0LXhBsOmeq0FjpMSPNfDbnSB3QhEAvYDQeJ093H8mlYS4LXJHRxYDINSEm30aQjCcaoRV1lzzViCAJBNfQ9w1hYK0rrZgfzHHBI7QGbcAGjMmpqhFS5SiZkeXJEmkyGScWxxCUO5mMUULSlyXaaLbrJdb3TEbTwQrHIZWkcy3WOnyfWPyr1YIoHNV4nJotQpIbnTADCS44eu/xMeJcug/TvlCRWqSDHQN+YE8PjbQ4hMDEQBRxaKgOAabB4vuGvmkRaDBp3nPBk6xTBC5YrO+SFQEpaIuQfie4PVs+HYNyyTn0EHiiBwuXBOxl/Fq6si85xuOS1abD+x7nOkK0ZJlGiJwYA25g1eW5xPZNepyPBN+xs2zz1tLXDqWTEq0bgqO6ViFFuk+0Ekgh6boOoho2qQYlGdhVAWOSCsy7ns4nH+W0+Y1sm3WSGssE0EUsXdvTdRZICrIYLL3rybLkdzyqSryzdHVDcG0KDIoW5xI71egCbUzyDm49QQRUXjCajimqin67RQiRmJIqsTDefO11jo6OODu/YnZ4gtY5y9WazlqUNhwcHmO9Y73ZpKA7ISmLiqKqGI1GiGGeyPKM+/ce7ps/q9WK5y9e8PTFC87PL9gurzCjDFVULM+e0F4K8lwzn82QOqNrWnKdQF5nO4LtQQqEqcArunbLcl1ztVjx9MUZrusxoSaXniIXBFszKg0XFxdcXJzjAhyc3uFysebdDz9h2wsu3nvO4mpD01pm0ymbesNkViI1dF3NmkucjRwLmM8nhDDDBkfUBpVn1F2HUmBMzqSqMFLSWUfX+2TbUmjqJhCFxfoG5xsiNnmHa4HJEqASBhWIczapffqeXRQ3QN/3NE1HIKRgdXoIHQSFksmSzahI8MmjP5o0p6cdSCCGYZ7tNuBqtNLkWpPppBLbNbhDDClo1A17QOfQJksNVRvBi+TBHyWiDygBhTKILNB0Ha7rhrXbJ9921yL1rmm8Cy5KIbRivzf9jW9tANrNQKgwlug7OgseQ5ZVzKoR3m4RyhBjoO97VusV1gfKIh8a1IHgBPW2YzoFrQ1979msG6TqkVpy7MHIDBkkzbrB9S3TeQYxT4ChFrRtR986uhpyOUIhadZrMqXQMvLp8yd88sknbPr14HSQ8hi0UswnU2bZhDcevcb7m/dZ2y1FXuIsFEWJ1w7nQsKArKC1nnYbUlCejJyODnj64nnao3uH0go/gBNd11JNKkZFWiMat0XKAEoRbVKrBR8IXjIZzVITRmbMJgcUWUXfesASfINzgsl4zsnRbbx113PPxnN3focTfcRRcchEVzSipm5qZCFwPn1WgmJUjZKKBc+t26es646fv/1LvvLmNyjKGUqP6Fzg4qpJVkR5A94Tnef45BabrkObZFXgveVgfsB4PCaEwOXVJbfv3N6rHvI8G3xgE/NuMpns5eF939N3HUYr5nduJxKQ8zjXc3W2pV02KB8Ym4qj6YxtWGKbFoxEkuzsrOupilGav+sG7wVBBIJIigupBEJe7/da27BYvuBieUGRadquoczH2D5y6+49Wh/p256Vb1GzEYVO96QYFWACWhq60LOul1RlSdNa5pMRl2cXXJytMHmFNHmqM73HBUtRVljXUl8uGU8LyiKna1sWiyvK6gBT5hSyTEonb9FKkymND4mtV9eOuqsJ+MHCKNkfIWA8GtO6GvAQYFSO8F2HEooq17S1TZZ1v+GhConR6Zp4KbCd4rWHM46OJbK0uJACXjerlq6LyXda5SwvHedPPVfPYTLSfOObDxEyUFU5q/WaJ49rlldryqzk5PAeZX7AarOidufEpkGIHq0NTdMRRUWMEi0zdJ7z7PGSy3PL+dPIm6/fQssWQcA6Q6EFtunJtCTXGW/95Aknx3MQAecURSk5Ogo0TYuSkclM8LVv3SJThzjR8fGT56wXLbdPNcenOdODEc1ZTecs02mBQlIVJc9WC148P+fW7VPq+jkuJtVTURh8aEEolssrhISTWzkHBxUHh4af/6Ll/v0TmnpD23Q8fnLBvbszpNQ477larCnLMSfHt+m3T9muV4zKEQejE/7Vf/9/4v7xV5iUd4AKYQtibOn7dg84xhhhANy0vEE0HJD0PTNyqD2ECENDOIBLqku9J/ukkPLgE+EwL4q0fxMglUCJRLYr8zkhAFEk5Tbggh/ArqSWyzKzV0UOt2LaywuJ1BIG1aSIpEaUTkQLbzuU0YTgaNotUgSca9lurohe42zLdr1GSUHbpaZhVUayTFNkWSK6Wc/R/ADnAlrpdB5IJJNC5am+FoMl06hECHDecXZxgZI5t09vcfZ8TW8b1k3Hi8sF89mEer1EKkV9sSXPK06Ob6U9qtIs1isODjK08rhYc3Kr4OLyirLI6HqPdZGzsy2HB6kOt8FTVFC3NdvacTA/4fd/72s8e/Iut04PWG8XCKlwQbBerpkd5LSbGjHYH/6mhxA7AH2HL+6ylNL/Yri2pBWDqiE9Ng7OFZEbFfbw8wEY3+PZnyO2ff7fX/C93fcRiUi2H9sIRLzhG73DAWRymN5hJpBwBDlY5cJg1SwkYfCxJkaM0mRZCvpMQmO1B2p3xLiyLBMpyg85Am1qoiqlUNKkbK09Cc4ihBrU/j0MLOo8M1Rlwde+8ipaRH7843/k/PwK20cyXeCsp227QVmVrEqyzNANAdLOteR5Rts2jCcjJpMRIloOD2Z8+vGHnJ+/4MFLL7Hzqh+KzP25Tb7mcs+KTsD6AFDHlFMoBEymE6azKU+ePCPLMmKMTKfTvRf5yy+/zMnJybVdsoDZ4QEiJpD71dde49vf/jb//t//+/01sjatP1IqvLtm2WdZtne02AHPu3ksKUz9XkGvtaZpGqy1zOfzPTM9PdeTS0VVTTl7/oQf//1bzOffZjIxyMEmWXDtkS8Fw+e/Jjl+KYjO57z8hybkHk6P1z0aiRiIfskVJDUNY8Ludk/b3yniM/dCGrv8yuN+neO/ykg5xh0smXw2U1cjYl0C/EIYvPhiMnFILFvBzu5k54nuA3vQUAmB0iloEQR+CIUzKqJVRCmPVDHBHcMn3XlJ727q3QAN0Q9WJz1KDgD+sFpIJQY7jThYuewAUjBaJqCMMICkiSUS/DWEDoNBwNAc2EG0zkUkybpGwMBETyzmhEim5zvnaGNPiBv8J49pO8tVqZnMCqavPGQyr9is1szmChGShEYrQ1WOsa7Huo4+OJyPZEWGsz07hC+EQNOukCIjRkXbNygEF4vHSBkwUmNMjnMGkY+Gbp4iRmjrmt73LFfnBI6YlaOhazUhtCuUl5wcpaAiqxQfffwRKsDP3v6As+dn9L1NDQofOBhPWK42ic1/s/kQ++tzGALb7QYtFUqnzh9CUwfNybf+gOM7p1y8+IjYLNk8eZvYn4HoE+Q8XG9iSEWBC3R9ApflXnqUomSllAMbdTcC4n4cyqHbq8Tgh7jvVQ2vQ2Iyh5gUCkYnUGTnbTvgbAN7Lf0Rw9gsdOTugWQ2llifXuemnCQB54PlRLxx4wqxXyz3nsrRDUWuGNj2amhQiYGFD61NNkPit3VzGdbyoeeFGkDqYQ/4xY8X+xPxKxOPuPHFPzlJ3sD+hQY12Od4ZdGhIsTBnimkz81uokSCt+wkSEontrgUkjjIr3bs32Snsmt6RWIYAD+5m0jDwNJ1OGcRLhKkQ0UN0aOiQejECImYZPfkfGK7HB1ycX5BWVa0TcuoGg1WNHaY/JNdhnPXcoG9RUcQQ/GexmRdb0BUxBh5+uQJzlq+9rWvcfbijP/8n/8z0+mIk5PjvZRq75mtFUVe4Poe7wNd11PkhiAFMteM7h+xvbBUWUkVJJlzqBjxnUdITavAGUGHo8gSayuPibnvfKAoSmazA7ousQSct4S+SyFC0aOVHNgySTYmdVqIy6Ikz3OyPEepHO8Ftu1ou471ek293dJ1HW3TJOb1cB6UUkyn02smzcBM2MnK6jrZVzT1BISjblfMugNGkylaZ0hRsLNmkEIPPqUKgUpjBJGUVH4I6xmCR4N1OJssZIJUKKFQw40VvKePDu96+r5lW2/ZbrcYXVKV0wQ8uZ4Y00YvRE/ft7RNPazQEmebxHC0LtlVhRQUE0IEKdAq2U4kC7GQ2KpCgNJ80S346x5ZrrGXLTE6sszQ24FFPxRjMcZhQ+aQdEOYj6Opt0AkNxqVSRo3gKNKMmQxJ7sl3zMejSlKk+xffJIka2Moy4wYPV23GXwbJT56AklF1HU1XZfYxvV2je17xI4pal1i2LoeKTQxGqyLdG03NJIS0CuCZ7vZ0NnBt1+khqVUirIsky2NTRYb0btk5WUtKjP0K4ftOgwSL+HBgwfcf3AfKSWjyZTxZAZSI5RhVk3QWc7VYsF6s6bte8pqxPHBMaPxBKmzGzZAHhkluNTYK8qSaTXjcHrES3fvs1wt+PSj94j9mkJFYl/T+p5MjOg2EZUl24bYN9TNiq7esm0bfAxIk4HSoDMurja89+GnvP3L9/HWce9owvGs4Phwgu9WjEeGy8tLVqs16JyLqxVnVxsuly2Lrae2gqBHfPL0A56dLTg5HrNpl/jQUZYld28bmnrJ++8smZwcIUyOihaV5UgtcEESQyr8r7NMIkpFQuyI0aJ0QAiH8w19v6XrO2IM5JkhhAzXu72t0M39mlSGLM+JMXnfG+2pm57oPVI6JBYRHPieGCVGBQojsR4kHqIf7n1PtC3B9fi+SbZrmSbPdco6kTGxCJWEYQMehUZIhTY5WV5QZjmjsWFWVmTasNk29N4SJHgZEEhUntP2nq5vEdGhB3KMiLv9ocDFNO971w9Fys0i+Dc7RG/QJAsb6y2uhUI7CmPobE5Egazxbstmu+ZisUDnFcWde2TFDO83YDxd27BYnlFUY0w1QbmA6xp817F69hwxmsC2IWsF7bbBjRtCnNJ1Cu8z1hvHclGTm8jRXDEqM5puiVWCLm547+mHdC6QjQryfI6XMqnrfE/XrplmY8ppSd4VWByIiNKR2eGY9WZDE1smRYUMDi0yslixWm5RUmAmOaYxZGud9nC5wGlHK3vMxBCkw8WaslTERtF3EWUKlM4IwtAET2N7YnA41zAajZjORkQvyPQEETyjXOJCUp5oJSlMjhQ5YzPmbnGXOwd3OBhNMQHWL15w/vQJXb2hr9Ocb0YTZDVhdeZpgsBGeH5+idElf/LH3+PwaMbZ+WPqpmU6m/PS/ftkZUXjAzJGXN+y3CwoRmO6Jvm3bpst1bgixMh4MmF+cEhVjnC9J8aeqpJYVyOFZDQaY61js665OL/k6ZMnKOD06BBBoG8b3n3/bWzbMMoVt+cjwnSEDUdJEZU94PnVBa0YoXLLxm2oW4vXHS5AZZK/aiwFMjPYPiJ0xOPAeWK/wbqWtlkTvQef0deBvmkYjSdUoxHdeklZZWmeyzSTPAWyAqzXK4TM2K5XRBeIzhJEZLOJXFwsCJ0htw6dKYSU2E2H7Xvy2QzrN9TbBUZNcF1H8JHtZsuqSdkv1gaMkAgfKAqFD5azixd0rkeZiDKSZlvjVUb0EoRC5xOu6gahPCNVELxgMh5hZEW97eldRGUk1cFveORmhNQNmelpm55bt0/56jfeZDJeY8WGyxdLLi/W1NsObyPCG5TQrC4E20XOH333de6/YihGHZGOzWbLccj45u/f5qf/8D5XzwMCxWpVk5WaYp7TnvWQgZeCbR14/MmW01t3USiUglKPENZQX8Lf/KcXPHx0RDmWPH72gssry8mxZjyqaLc9de1Y65aLiw1ZDvde0kQvGY0y5vMJXetQasJrrz5CZjmPnz3jJz/6JUZFprOSulnz5EnNK6MMpxVCRq6WDcfHM5QWfPLJR5yclmwXK4SGg3nJ1WLJZrsmy3LuvVSm8PJQo80hr79xSoyB4+NjFstLjo8mSJVhvWO1bnjzq29CECzOLigzg/Yjwibj23/4B8yKY4TTuNZitMXFGob8jR3gtbNBuM7EGWT3w5Hs9FIWRvqa4Y8anheSGiokdnbftZjMEH2qTXNTgoC+24Lc5UqldUsNqivnA1qkXCipBJnWuCFUWytNFCkkescm3ZVlUqZ9HWEgoQ3q8batSetyQApLvbmk3ixYXbUoKVheXpBlBVIbQNK2ycYohKRML6tyn2MyHlcsFgtCdBRFTozZQCoyjEfH5JXhjdcPUOOKWgYK37DZbHj51fs8efYU1/dkSrNeLomhJTclk+kEETOIiaHc947ttmN+MME5S/Aek6kUDO5kCip3MfnGy1R35aWkNPlQWwuuFhc0myX4ZBlZbzua2lBUYHsITiEd+Kb9je9tuK6NE2ltBypeY4VSpvdz8/HXx7XX+e7fSVE/sAw/w7m9ucf4Ipjwi1nAO2D1Wimb/tqxk7+srgcQclf37/zbBQhNnlVAnnAok1HkOUZrrA1ImQD13WvvWM/O9TjncN4nq0DnEBG01FT5iEzrZMHVylRbeYvtAlEItCnp+4YYNSbLeOXV+0ymYz54/5PESFcZi8V6H+rpQ9znRSUwX+89wCFy794dRlXJ6vIFX/vKmxAjn3z4Aa5tuX3rNuWoQiiJzkyywpEKxW5OiOAHy+vduRHp3jZ5Rl2vqeua2XzC0fEhISaltzGK+cEBb37lDX7vd38Xk2V7G1OG183Lks3ZGd/5znf44Q9/yIsXL65tZaSkbuqBbZ/wJTWoXD5/LW8+xw1Ev915aJqGrkvkqp2FqiBZ4vmQ8kOeP7/k009e8I1vPIQgkMJfM9GFH4ytrt0Ndr93Z7n7RWPw8yP45qjdN232FMuEl6XbaYdNiV953g7r2jU6vmwc/68dvz6IHuMwwYR978uH5D3s3M46JQVn+CCQPtmfhKSixXlwiaTMoGYnxgHMFqB27YSYOlpaJA/TdF49WsrkeawGu4aYQkKLXehchOgD3jqC98nDOoqhQ0uSXEqxB89ScePRIqIlOJLndNx3SNIAlwxAISlYdTdodh2W9NlCaiSIFE5q3Q6MhzgYrqSbs6ULkZaaNpwznmYczO6gveXjt97m1isPqUbHqEITogWRQtnGoxl1s8I2K0bjCdb5lGQdAkoobOgJdkOIAesshZnSNFsmoxF1s0BkFVqn4LaubTHjOUblED2db1ltVkSZ8cmTT+H2ffJJSRQZYnyAqnuqwnAyK3nrkwuuLhb8zZNzHj9+znq9SnYqPnmFhzDYEPgdMhxvjPz0xQ6yjiGQ5QV//L0/4cOPH7NZX/GL98/5F7/3HaYnt9iu1gjf01/U6EGFIKXA+51/vcPaSOcSYCUYOp8CMiEx8oZgJO5aHslLW4pIpkXq+KeXTlYDOyo4Au9EauzEmHydjUpNFD/Ir2QKypC7hYJIJj1H48CtA8NklLFY91gbUEIm5jvJ1sHLuPf+34HnQiQZYBQJTAwBlJQUeZEmSmGJJA/q1CAAGxzWX4eU/lZHHED+GxYowx256xfeePB102F3z355+y7sGw5f9Duv/dXCEDBqh4XDDTYXOp0judtopAwG4k4GHwA5eFsPvldix/IdOgPiuoETQiCIpGxJoLgHhuDaQeUiBg+oPnZEQBtDLkaDDYZAYoghUhQFy+WSrm8Tq1JLRqOS84uLYcF0af4a/LeUlsPmNL2fnaxqv4ESiXG5Wzg+/fRTsiwFKX344Uf8h//w/+Vf/Is/YzobY223Z7XEGNFGU1Ylm41NHr19j5Q5QknEaUXMOpply8RrZG8RNhKtx6NZikDnPNP7x9hMEmRk3ayZTucDsyClcQOUZYFzNjGng0NLidQ7VUUK/9NSJSL4jmIZU+PVaEOewaiqmE6TH/tqtaJrEoMlhIB1btj4Jga31np/jnabufV6xXq9YL3K8aFAZxofLG3XMJ0ekAlNnuWoIa9DoxBREge5d4gRosc7Rwro8cRdA9ZFgpDXrPvhd6bNRE3bbqjrDZvtmr6zFMUI5zqUMiR7FEmIqTmYvJ37oWli2dZrlqsFTdMk+xHvBt/0dO8niaNKyqpMpg2xAKF+1U7pv+Z49vwZ5xdnONchJCidCgkhDDZLa3gY5lSdCYrcYF3yOGdo0GmpiGWGs5adzFjKiHM2ra2KxChWAq+GBnYMRO8GS4MU7AeATLZIShusbVksLgkhUjcJbFIa9DCvS2H2qrGm2QIp/Ma5dE2qIqfI072fmRSkFXwAlVgffd/QdMm6YtdYabo2yadNaoxa1zIeT6myjNF4RDUakRdFYhlJnTzRpWZT1yxfnNF2PVJJpvM509mcsigRyuwzXozO0nygdApx3das+83APxHIAJmWvProAe3qjNX5E9aLc8alIXSC2vXkVYUylvXVAjYLfN9hoydqBa7Foji/WvPD//IT/vGtd/jk8RlaGd546ZS7t2a89vJdpiPJanXF2eUVq/UWmQv6py9YbHpWtefFVcPV2rJYrHnxYkWuwWSRuTScnfc0zRbXOe6e3qf3kdV2gUPgpGZ66zbT45MhZFbgrKXrJWWRUxQ5nna4J2u6ribEHq3BZJKui/S2ZbnydF1DmY2GDXtq2qUAKYHR6V4QCJzLUqB2n8BnokXplJPjbJNsvIIg1yJZDg35IYn/FBDBIYWn0HEfYGryDCGT8idKgSXgXcDFiAgehCBTySpMKs2oHDOfzihNznzsafqOTbth1axouo6AJbo+AfahxyiJMQOjz6c9K86lJqzt6K3H+bAHUH7TQ8Tk153lOjWqtBoaFJanT58gRMDbNUaHFOJYt+RlmpOrajz4iGuIGd7ZQTFlGE+mUBXUywWu61hbj20tq8USTNof1ZvEVM9zgVIF4KjbhrHryKIi+J5t16KdZt00iCzHSDMobiRtb+n6yKZrWIctV4sVq7BlcjgdwoqT2mo8GQ3Wg4qizMl0wWwyJs8rJoczIhGTKcqRQQTQmcQU2eAnPefq8gLXD03JKFBSp8x6reidRdoOFwLb7ZrerZCF43zt6RpPcBJJTlGOMT7n/HJL/eKKaT7h1vwWr997mbJXrJ6dc9E+ptKKUaY5mk/wVcZ6ZWisZdX0uM0KJzN8VHQ2BcJ99c2vIIXg448/pLeOu3fv8eDhI5CCpm1RSjIuc8ZlQdv31P0CqVM2TF13nJ9f8eD+S0zGY7xz9F1DkScWqvf9oCzo2Ww3rFZrPv74Ey4vL2mbhtdffZWL8+f8zQ//io8/eB+jJS8/uk8mJ6yXiwQ4Raj7nheLK9Z9R9s7qukjypmnDQt0HnDdFeORYts1VGqEA6zyKDQahbUNRa5TxsAQJq60weiM7bYBPBcXZzgXmM/ndF1H9Km5lesiWbb1FVJItus1s/EB0BODp6l7pDRMZnOcDWzXW1wY1MshElxADiHEdV0DAms9GRC8o2la6s2G6aii1x1VnrNar7i8uqQLPeUoY6JH2Lanlz34ZKeXFSVXiyvKkUbbQAyeyVQRpaOzPdttUndNJqPf+N7+l9/715xffcQv3/0xa3vF2dMNf/d372DDiqgt03mRmpi64vKy5Wg2od5IpMh49fVTQrAsrjpuFRlSScajHOs82/Ypt+4Ebt+asr4U/NX/8g7HJyXTIjCZSZou0nQBGRR9K9Aco7yk0Dn9dsP5U4ehJDSRD95+QT4OTA4Kcm0JNhBsahaqmLM4byhMzvJyy5l2NE3H7dtTJD1FaTg7e8Fms+Dhqy9z+84Rp//b7/Cj//ITrq4WFEXG6Z0UMO5DoGssk8mIrEhEgJNbOZeX50Qi83GyCp2Mx0PWkCISKcsMgePjjz8ENNtNQArFelNzMJ/Sdh7rLePpBCEkdd3S1Y6xKbl9cpfXXnqTr7/yuxxUd9CU5LpAAK6rkblKpIlhzcozTbIDDXjXp/qQXf2TlHSItD+OwafVabBF6Ptk+5OYuLCpN5ydveDo6BClNKPRBGcDMUq8DcjsmiGspMTafgBgxR6g884RRCDl/gQiqW6Nzg/P3YVXXtdnavg8SfGf6rcoIs521N0V29Ul1jYEZ/FAcCHdp2WBzLJEoClL+r7fBxFeXV1RVBUxJla8H4Dh7XZL3/XcOj6kKMbU3Za62RKDxJiMzeqcZb3hYD5lOhmz2ayRQVJVw97NdownRzSbwOHhMTIovA9cLWs2W8/JrQmXV2cURYkxBQ8fPuDjjz9iOplRN1vKsiSEpF47P7NUkxTs3HctMhiCtcwQ1LVlPDpJNarXtHVglJfUm98ORN+xeYXYkdUS3iSHMbR7zI6QlYBzbjwekq3i8Nzhj5BDzfO5x8JnSW077OyLCowvBhbj9ftkB3Lu/r0jAu5IhYn4trPakCjiLmxXSfIsJ8tGKYfFkxoxA3mvt30KoC2SWnYHsoaQ0n611GiVYUWPVibVGiqSZ+xrrhgTlrKsa7KsSAQl4TCZ5PbtY46Pjzh+/xM2mwbz+BlSRnof2G4atNY4G/eh0ErJASTOef3115ObxaO7fO97fwIhMhtPyExGnmfJ3knKPb6za5rtz1dIRL6Ug7mzS0lKk/n8gMl0yqeffsp0OuGnP/1Hbp3eoigLqqri6OiQoiy4fedOuj4y3ee27zB5zmg8Jssyfv/3f5+//Mu/ZLlckmUZ6/V6YJ+bzwSI3rzON73Tdz+TUu5VaztrmL7vSYGibWoeBkdejGianoDn4nzFe+9+zBuvPyTXOSJahPAD5pr2c+HGnPNPkSx3P98pHL4M50pc5T1NfX/eY/xVw+Hr38keH/xtjl8bRN+xq0RIVhK7Ex13FOzhCAMDWwn2ReWgcEpBnWFvX5fkHTLR1q+BxAHLlDsQXYFIHUOl5MCilsna4gZGG0Mkkrq3kJjcQiaGewpkjMPryj3wKaUgMzLJHXcdqAguhIERnzBVJYauRUzdFCUFRgmMTs2D3qffIxXp93n28mSGbkgKTw0Eb2lCQ+cC0me8eK9j8eRTxrfvcPzwlKvLBffuneKiJeIQUidQxeeM4gyBwLYLXNcCAkxqFAjt6P0W0eu0sWxqnN0iJWS62APP3gwMa52x2lzx5NmH1DbJ5id5yQcfvYW6+wDBnFFVEpShKCfcuXPEf/ibX+CFxFs4P1+yWK1JbYLEprTODwqD665lakkkEGjXHx1+hADuP3zE9//Fn/GTf/wHPvzkGeKf/xlVdPhsy+T2q2zap8imR4RukBG5/cTtA/Qu7jcHYfAfV1JgBonYtRzk2g9bK5EAg6HjmUIiLdY6pNJEAn3vabqAEnHfqQreY/tU+Kp0WZOqQaaxNh1BXmgOZoa80Ii1TwB3ANQQOOsF/TBm3OdY3iHumNOkgjGT5HnyCxMk0FVp9hOydSHJzv3QfP5tjp2vXwyfmUDTebvxuOHC7fyidyB4jNdM+s90xj/7jc8ew72RFC6DbYRPAUOJKZtcZ3ee0XFQlgix83duByA+MVnTniMQEQMYbwf2624iTp8v2R9FvBeDL63fs9Ct7cGn+7S3PT548qIECSYrCR6C83RtS9d1bDYrBAGTKfJc44JDqwToBa8SKI5L4b/sAkr9cMr9wL5MjGRzI1Byt6H+9NNPE1O46fng/Y+B/8j3v/898kLvwz1Tc8CTFQVjEam3W2IMWO9o+hZZloxOKzyO7eUWKwRkkg09ywhFNuXwzinl0ZQsk6jg2DQ1bV9T6CzZvmSpmaOUpu/btMGLyQ9SybQ5in6wohq8hY1O4KZ1Duc7hEgWVzuG8C5tXClFZgx917FqGupmC0MAZtM07AJN9qng3vH8+WOUdpycHlKUOc7ne1Z8oQsypYhB4r2lWa9pNhu2/z/W/qtJtixNz8SepbZyFeLok7JUVld1V6MhuiEaGGBgNjYw0oyXvOGvo5G35DVpQ9I4GAhi0GhR1aVFyiNDudhyKV582z1ONYBhTxa9LC0rT8Tx8Nh77SXe7/2e97DHWtk0ysZLWvklyV6CTUOYQ0H9RJx/ntyPka67YX+44XDY0bZ7Qkw09YrgR4wRNIIUPcTJ7adhFuoz/SBOuKu3L7i+vhY+dJz5dOS5OKYoasti1bA5W1KUFqVn8NPvsNDvd1tiDBgjBV9jzNzKNwfeRHHtWqtRyOEvhUAxdwr5ccCDBC46Kx0DWtbCwhmcNfPfmUhZmIRWy7ieph7nCmxh5sBdWfytq2gqR1aZ7rAVYfzEhzwWw6CoHFZLoKUxjqquCKGka3tS9ngP1iRJq9eKYehnjrC4TiY/MXYtMSfpjCgqgvfEIM/sYrkgOoc1BSFlimrB+uySomp48PQ5y7PHxJS4vr6h7STk2BUFxglTvh9HtHGYbPBR2jJDzKicqcsKVxREF8mjp+t7hn7g0N3R9bcsSjCpx6mEKg1xGni9v2O5WlFWJYe7W7Y3V1xWlmwSKgTC5OmmwF038uOf/4Z//a//Pa+uD2RlUDpytTvgpw5rMt/86DFxGoACU67AlMRocc7RHt7y1Ytrbvae/eFAVVpQnl3fU9QJ60q8n/jqq89RCVzdsGs7VFFSrNYcdrdElSmaJX3X07YtRhnW6xWLxZKcJIHETyNtdyAnKXQXhWO9XiBoLoPRlhQm/JRJ8dg5dtzrKawoEBTWEovI5Dxj8iiChIYihRqUnUOdAs5YQhLcBCThVRootEXZCqOhdIrCIU2HVhMVxAA5SDFNByArPPJPnDeDSSnGOKFUpqgMLmtyNzFNBxIBRcQ5RY4KZxVFIfgwHyP4OHdTpvnQMhf1tfqvPLl/u5fShhwgK0VVl1hnaRqH9yNXb77EWoUfW0oHm80KhUFnRZw8VVVJwS9IkbksS4hxDmO1ZAV1VZNipB96usPAGCaqssTqmsLW4mqkoKkXpDPHft8yxYByBleU9NsDwQeadckUA8HvUSlS1gvS5KUIE7IEQIdEtJnC1UxjBDTRB1YrwZWUzhKmjqasqYqG1fKMxXrJ9fYK7wNVbXFaU1UyVxVFgdWO3Z2sKUrNOwQtAkPKgvDohkjKmqgDutAUjeV6+4px8NTlgmGXeKY+5mz5FMqV7DXawNNHT3nz86+odOIH3/seTx9esigcaRw5bO/Y3t3hUZRFxdNmTe8j7ZS4PfS8ub7lO998j/VqyW53x+WDR3z08cc8evyEu+2WN2+v2O0P3N3dcdjf8fzpUz755BNWyzU3dztiTFTlgmkMVKXcoxgEryRPkYjsfb/jcDiglKGpG7z33N3d8fTpU/7Nv/u3/MWf/xnLuuKDjz7ig+dPKUvLOLSURuGUnLeG62senp9RDwPBlyi1pNvfkveB29fXuGpB11pSXGIri7OGKmeaesHlZkM/7umGLUPrqYsFztanVv1+7kRrd3vKeoFCcXtzQwieqipRypN2ibLYUBQS7JeziDLDMEKCqlqQgpiIco60uz2r1ZJqsSaMnnJZ4kyNQvIw9vsD5WIDuoCQmYaeQ4pYZO+y3W8JOTBOE/WyoqwbdrsDKWWsKqiLJa4oWW/O8LFjmEa892zCRFKaerNCFSVj1zL8DjiXi9XHEB1vV3fc3XgW64bbm5aQPIszQ3eYqOuKoRtYrzc4u2a/veHm9ZZPw1fYAqyxbNaC6Pq933/Co6drrB24uCwheqp6y5/+tw/45S9u0cbQ2AXGQJpqtjc9u2Hkz/79K/7FP/8BU8r8xZ//OSl5Rq+pqxW2AOV6Yhy5uBDGPCjqukLhGca5m9AW7HceZ2u+/Dxwdp6I+Y6LiwVdN/Jnf/Yz3vvoAR+9/4zf/4Nv8T//hz8npIGqDmjjqAqNbSyLxZKmLthut1RFhXOaYfTsthMXFxu6fmSzWTJOPV07oU2mKsCHzM31gYcPLvjWNz/hRz/6lGEcBTtYWVII3Ly9Ynd1IBwSf+9P/ojvfeP7VGxwacnUZUwlDPGcPMfuY9EjBCMh+keYBZw4ty3L2U6hUCmikJDpGOMpYwigtAYfAvvtnWSeFI7Hjx7QtjuWyxUxTJAdGkNdLxj6vWRyzCYPNbvPY8oo4yiL6pThlFKSDssoSLtj/tTxbHZag9LR1CP/mXNkmkYKZ6ibmqZOxKml6yI3uzu2d1ucLVk0K8FuoVitVxSF43A4zPtcz9nZmt2hI8ZAWRY8fvyYruvwfsT7MIdbj2hrUFEzjh7nSqyxPHhwyTD2LBp5xh89uiQzsdtfEbWYOparFS9evGBsJ5bLFe+//5wpTLhSofUdMcD5+YKmKYhx9q9mGZNaO0YfSTFye7MnZUNdSSfaMEigZVXVLJcVh4Nn6APWwvmq4vK9zdd+toF75+zpv/Kso9+jW45fOdrajiZPOVG+60S/F6qOwvVJgFT3CJZ3Xb/62EX+jgD5rm1OJPDf/hzzX/itM8mxEHNfkDkWaO7fKysxvKUUIE2MasDqRvC1WnQsa+yM4YDCOXGoB481Dm3mYpVi1gT1qVik9Nwpnt38efLpM4QYaaqCcRgoq4a6dNzd7VksNjx98ogQMpvVin/7b/89Dy/PRW/Jhqu3t9RlQdeNQKKwluWy5r3nTyEHVk3JarkgJ1itViL6lxUhzIGi1t7ndBlBJymt507jow36PtBXG3Gjx77n8vKS6+trfvCDH/D27Vu+853vsN/vWa1W8vulWYXPihgirijJIVDXNc1igTGGR48fATAMA01dM3l/Cg8/Csx/85+jUG6tZZqm+XycT/8fpNhXliXjOJ4c+rvDltWyYRo9ZMdXX17z+Wev+fY3nyLdOGImFhH1mPH122fa+/H0zng7fdtvf38+ys6n8TsP/9N7zOPiFJL77g+SMa1IJy37+M/XcaP/rxLRc0rod3QxQbRIyJfivqopKIp5k4ps7kPMBA3mKHYdN3oKji5+re5RLUYbtLm/CKh0YmHDvTh//IOc0+zsm2ZIuTDNQ1KkbMS5muVXPjrRjVY4p1F6diAd3fP5KPzeO5kVEhRzVCtLraidsNvHIZ9wLkcRVKEwSpNmp30mkxJMORCSCPsOj8s952qFKw1f/uYL9FvHg4drrDl+zozVmsIWaIzgJmaHd1EUZBSltZCOmJrAOB1OomNGXEo2R/QcAFjVC+72t1zdvOZue0fvJ4zSvN7vMSrzmxgp7UfUq+czc8ry/pNzQgj0Q8d+N3G73ZHzzP8Nx+qnCMU5zwwmjjwkETbfnaIzEHzkq68+47//3/x3/OAP/gBDJluHWz3gwcVjKh359ObnpOkalTx5Rh8IJ8zM91feUcbj8UEwpzGV86lGeiqcOAOLUlj4x2CHMAsURVkSY2acImPIODW7hbMwKacxMk4JZxRllkBRYwxlaXhwbkEZNpuKsizQOqB0JJEIScZvSuCjws8iesozHz2L+zznPIvtgUoxoxPyifMkxaY0FxGytNNnNeODvv4rpglp5QmnDZkx9tSMqI6YgpmRdZxsZSKX73qXC3iqfquESuq3FnH9zs1RKs0i9kSMRxe6hOLcOziOk1ya3WgZH4fTxlUwAlq+pjUxeUIYGaceaxzOWdmcKTWjYRJxFlhyFt7kOA0MQ884DjPLdqTr9vjoqYNwjut5/mB2UVhrxGloFbGfePT4Afv9gWGYmauzoJLSRDv1Jzfs8RlIWRzvzPf1xKGfx3GMcT78yuYkhJZf/PzXWGv4J//kT5h8h7Hz4qKEg6itpV40+GkkxsDgJ+L+lmINzWWNWZREH8muoHKWerlks1hRL5eYusJYJ2qSORYA1Cxi2xmtE073NqVATuG+OwKNK6zgnd5p4TsumDHGmbW7F9HNWs7Pz4Vx+fIlr169ljVGz86FGfFwwtacAk0mxuHA7Y3CuMBytaCqxeVitcZqSxxbLIqcAtvtLVfbW+4O2zmERlEWNcF7qrqeC3OCDpG4DVlMpWgg41z462/ZH27oB8HQ5JTnOS5jTInWlqpuKFNJiml2zQYUimns2W5vefXqBdfXN3g/EUKc59C5pc/CctXg6oyyFa62s8tf8C9f+9nOgSkOTP2esrI4uyRnj1ElVhlC7iEPWB3RJIZ2koKEteicUVn4lSlIkJ4UMiI5eqwFq5W47uff1RYO45zMz0FCMauiwOU8M9cjzs1tpxHC1M57i+MBTsaSsobke9kMkSmcxahAJKAYBWlFoBtGfJqo6hJjDTkEhnEgpDk0NvXkNHdvxY6YAkobXAGYRETx7IOP+N4n32OzPmdR1ayWayYst7s9MQUO/YEpDNiyACNh6QZNViX9kAmpEydaXRGmkcNuj1KK5XojWBo/MXrPtm3Zb+9Q05bUDlRqQIcDftgzThPGVdTNEkIidi0Lq9gebhj6lkPXMfrEofe8vNryi19+Tj8Fsi1IuqBultSbBzx/fMajJ5dMk6epz6mWD7l4XhGzZRwD++2OL7oXmKAJXrApPgecBa8ih2lC9VJcTiFRFZbN5QOyEecwweHCSHfYM3jP6APD2KMAW8ieyvvINPaQEzor2m6Qr2uDLaR4k1IQ7J4K+DDifURh5jldcgBSVBRFjXKGMGWKwlOWIkBoI4HheeZrK5WwOpFCBxFiakhZDl9lYSl1wuRI8gNxGoimoKoayqYhuoapgzhMOK1YOI3JCqs0hYGqspjSMOrAoEZCbnHWMJUDk92jiwlrJTMoRsc0Co5EGTClJXpFCgFXlpLNYES4MDqeitZf91U1DRiwBaw3K5TK5CQuXWsjF+cbjKrpDjtyzGgr5o7gPeWiIuXMNE5Mw0BVVITJSxdI4fCjxxgJiPIxYkvDeXUuwZXViovzh3Iw0oBWuPMabTRTigQyhdFUiwbvJZNB1vtE7CZKV6HQVFXNbbdnUTecrTd0Y4vCkbOmcBXJt1SupCpLTCEIp8ePnwhDWDswlnEYySFSOEVVWlwpnTI5S3Cd0YV0yxSWRKbtWrLKTGEg5ATRY4uSsrEYvcLZBWO7J3oNeokZMzefD3y5+w2hN6yLJX/47W/wjYeP+cbf/QPO1wsInuwHXn/1Odev37Db7Ti/fMjl4yccQqYbA4d+5OXrK7548ZqM5ruffI/FasHFxQXnFxdcXlzw+uVLfvmrXzFOE+v1mh98/7tsdztevHjJX/7wr/m9732f84tzXr58QwyR58+eURYlfbsVXMLtW6w1XFycIXkbkdVqPa/ZmW984xtszs54+/YtH3/jG/zRH/4By2XD7vaG7d0NfhqY4sTYdWSTefroMd/+1jexriCS6fue/X7LmB4wpI95c7PlRz/8KZ/+7AAogrsj4NFO8+wDy1lZE0Pg9u0VbZtYbRbkPHI4dBKa6T1x8nRDj8ZSPSrww8h+f8fBKMLUgoIHlxVkCVgdu5GzzZLD9iCs/AR9d2AaPUYVXF5cUlcVVVUyDh3doWfRrEFl+mELytAfenzsyVmOozklBj+Re4g6YwqLTYa6WmB1QYpZgnmrFWHMxLnNPSbL4bAXcb7taJo1PgcCibvdgaaqv/azbXXNxdlzPvmWuBt/8su/BuNRJrN50NB3LeOwO5Ko+MVnNxglhVxXgFGOMFa8+iJTLx3/+v/xBY+e1fzJn35CZs9mU2KXgc1yzWGfudsPGGvY3WzpDxNDK+vyZ5++5v/6f/m3BB8lWDrLOQQ7MsYDxiRUlr0XRnPoWzabDTe7A3WlsWXBsw/P5r1Ej59GpqHAlY63bw4slo6zswtefHXD4a7n29/6gO9+99v85Q9/SFla0Jp+8KwXlhcv3vLgwQpjDNvdVkQfI4JV8JEXX+2JcUNViQv95nZHu49cXqx5/LhmGiKffvoZf+/vfZ//+B//EzEPVPVSGP2HAwtr+aM/+bt8cPEUMxne//AjClejbYl1lTDDTSb6SYRpwM7IxmPHpDYalRIxihmm6zqaphHTxIxRCFGCp+MscEsGTyAnT8agdUFZlSjVzIWtnrKQrpKy0BROulSPBecQJ5RWGLScIcM0BzLO3fNaCR/ZiCP16CoFOGIcUowYpWZXe6IfelL0xDDv3wdPSIasLM1qw+XDR3SHDuafuZyxjznnuesjMY49PkSMuUdQSNZAi3OWxWKB1gljM0kl3l694c/+/D8yFZHzZ5fsxwN1VWK1w6mC2q0oS8P11Q2Th5wH1stm/jkBH0r6u5526Hj2/Bn7beLBg5qb64GquuP58wf88hdfUFU1b95cE+KINprDHrSTM8w4Rogj45DY3o2Eyc3aj7Ckt9sDy4XDFP+riMj/2Uvn+4wsec3FlqxQSjCORzyumh3L95qYCNMJO7/HSVkkGz03iN+fJ1NOs7h+L4YrwB6FBXELnkyQ5sT2OSo2ag7EVJIRk+4F/CNR4l6ElPySoKQbW4Rjg9Zyhq+rEqVGwtQy9juKYoUf44zrFFRjqAz7oSWTKTZndF2LRss1iwk/+hNKeY7oQGWNxs4GRyuisSlw2jHGieQlw2vZLHDWcr5Z44qK9XrNT376Y5bLhs16Qbsf6dsOZx2qlmD7unS8/+wx3//ut3j75gUXmzNUyhKKmgIpOfaHPU2zlHMjGuJchFAG7FzQitOsqRRI1huoI8JGO+pG44Pn448/oigcT548pq4brC2oq5q6WqK0QxtHDqCNk7GgRYj/4IP32KxXkALr1YIwTZRLyTsS5GimLEv2+z1lWZ7Ot8fz9TiOlGXJETNUFMWM+5RzeAjh5Ew/HA7UdUnwI8OkMKoiBk17SPz0p1/x3nuPqOtEYRU5QcoWq8yxj0HG7ZxjecprOOo9RHkO8iyIcz8MjTJiqsxqpgofmfuiOEpRUM/jfx7sJwVSYYiofMyVnI2/6mj1/Zvf/7/8+lvPAOJChxwjGUVIkcmHufIvPz8lqYJJpWgWTuenLGZIYaZlaGY+ubhxVI6ywVMZn7kXJ9LsPs1zGwviEA0xCh4mKubtj7RGxUAMCT87mkLKRCTIgCRIlbl4hdIZayG6jNKFOGDT0emq7uXflInkGVovrvQYE6VN1CW4UpE7+ZxK5zmJW0HSOM18Q+9FXHImxsyUPZOJBG8Z+4G3n39FbD36fMnuD6558OjiVB3URlHaAqsTY44YFKUrsMaR58kihiCBgrrgMA7EFNE5Y1RBQsn1TxptHH3f8elXn/L6zVfElOl7aWcqXEkKE6U1dONIHAf5WTnwg+98yOXZApKi61qmqcdY4RQzFz9izicXv+Hdg6E6Dcl3a00pw93NDT//0Q/50z/+h9SrNZlMtTzDWIV99k3Khx8y7L8iB2GqBh9nJvSxE0EenHl0kAA3V2lkXVHvXHv5HKWFxiUKp08tf3P5F7KM7eAlrBUjAY5kSD7ifcJHgETWCuMkmMUVhtW6BDJlU6ILh7YSIhhzJGYo5ipLTIqYpCorhSZZRNPcDRFiYgqRRllx9qdMjsdHXZ2E9nQcH1rPbPev/+q7HffhHGlmvR/bzO8r2XHuPjHWYI1U67V1cBLWzW/93eO8+Dcrn0chPYR+ToeX9sGc49yiJPfviEIBEbt9kA3aGHoCk2xu9UzcSpmEhIv4IIE/2SZirMW1eGwJA2lFjJ5x7BmnnmEQ7MA4jRAiIfQcuq2ESKUJbSzaOJxtKG1N6RzTNNA0Nev1ipgmtMqMY4fR0FQF62XDdrdjHDoGPY/Xd3Aux2A5VJo7bcyJV/Yu8gUghgRKMU0jP/3pz6hqxyeffINuK4uYNrMIjyBQnLP4cV74QuD2+g15taFaLMA4dFmLQ7iuSdaRKgd2Fi5MhS4NhREX+rG1UKMxTth1kx+I0YtIpiUgcBg8Ks9xgIq/samag2DmkNabmxtev37Nl198QQqRRdPwyXe+w2634/r6an5uOQWbHNsej9zk4Aem0bC9zYQ4UA0dfd+x329p2ztWdcPSlViFtKF2V7x4+QXjMFK6UsQfEuNUijNCzY7+KBsHMlhnTw74cRyYxo4c/ZzEnglZkC2T6lHao43FOU2ymhAEF0MSLl8Mnu3dLdu7W8ahO7HmQApUMQVsoalXhXCaHWATag4lPCKJvs4rRj8fzI4BM+LOzfN40UrCu43JhGGg7yS41i0Wc9eHBMiJK/8U04w2sh8YYyCetkWQIkyIm1WC5STgNUYvQc5IG+52G+cxL3z2FMUFnDFYia9lShGlDNYVQJzX6UhGWLZxCgSfMIOmCRVlVRJCwHsJui5KhzZy/UIYmCY/h1sZ/AQhRPaHls++eMGiOedP/sEHNIsFwxRxTjjtbbcnKwnLAijKEudKFA7vI1FLS3dZutPGM5F5/eoVxc0N1jn5Ofs9XdcTfUvu7qj1gGs009ihFZydn2OLmrpZSBEqJ+Eg371lGge8j9ze7bjbdYxjZllanj95wGpI2HrN5cPH1Nnz/OljvvHhE4g9i6ZieXbB42cfgSm5fnvDT//6R/w4JxaFZl0b2iETVaZpHKtlg1KekARL1w8928OWbC3Veo21JUPXYsuaVVWTZyF80VRzsU3jwyRu/+ixRlMWBcOgZ1SVzOfeS4uoHOYNKU+zQcBgjRwuyAatHBpxz2mVyHnCFQVFWaBI4la3lsJZtJJw3hgEj6XndlltpPtQTB6JbuiwVmFKSDiS8iQ8qIgxYJWmtBqL4N4Kq2U+3ywpypKQFUMcaNs97WFPP3Xkd1rmFZmiLNBIAHc39BjjcLY87Udy1Eze46f0O+eZPHv/KW23Q6nA2dmKYegYOhEXq1KzuahYVg231xo/RpQRtMPkJ0oqCYmaUX/94YDSDusq/OjxPrJcNdSbFd1+hzaamBRT8JxtztHa0vU9MUWKuqCqC+q6Ik2DFDiyY7O+ZJo6Rt/Rjgd0lv6Aw6GnGzPljGNypWVzvmSjzxjHCbLB6oLSRpy2bFYrYdtOA5fn5zTlgu32wBgijx895vruipQTZakBwTVN48jt7R1+CqxXa5TRFDPSyYeJfuyBjHGGxaImhYqFW2BDzXlZkrNjuo0Mrw+MY0bngidnD/jo2WMen9Vsykjo3vLF21e8+vJzDrdvKY2mqmvOz89BG27vdgTtGCbP559/wS9+9RumkPnTf/bfUFeOpqr4wQ/+kK7v+fUvf8Hrt1d0XU/bdXz+m1/x85/8NX/09/8Bf/In/4ivXr3mz/78L/in/+RPUVpRFo5HDy8xCtbrFePQst/tuLy8IMbIZrPBGMuXX37FdrsnpczZ5pxHjx6Chk8//Q1/9aMf8cVnv+Hu9pocAkWhWa+XbJZLnlw+YL/dcvP2DdoaiqrElbBalpQY3r94zrpc8GT5EK1qlqsN2/6OH/30r/nRT37MX/6bn/PrH3/J6rLBFAlXl8Sy4G57SxcOxHPo25HGRMLgedO/5vzsjLqs6FtLVRYctgeqWgJBb2+35BikS0vNgKwMOQvOqCorClvx4PwRYfKSn1DUBB8IHmxhsEXNUksn2rjv0UYQZkVVgVUoJ87GytQoo6iLBhU0i3LJg/UlZMXdfk97ODAME5vzM9IkpqAUEm3b0vXTbBhoqW31tZ/ti/OSqrngw4+fsjwzvLz5OcolbKnRJpJG2dcWhWO9OmfYZ26vIgrN2IoIFtOBpqnQRpFy4up1y//wf/tLxnHig/c2lHbBJ9/+iEVp+MUv/grtImFIOFVgG00ikHWiG0eCV4yTCDdFUaFLcbNiISQFusQVmXHyHLot63NLWWq0SdztBVlmLCQiKQeUEgPS1VWPsh0xGYLP/Kc/+2seP76QLCQlHehVBctlw9nZZu7iSwQvXcbWZVabAu8nPvxowTCMrDcVIfY8f29N5ZYMfWDoI8PQkfOI9z2///vf4Sc//QmVKXCAVZYP3/uAx+tzXDA8WT3EZEXhCnRRklCMfsTmxG57Q5NW1KsVMd7n3CiMnL91Rs2oUashx0m6pZKGZAgx0bUHqrpGWYufAil4jJZQ1qmfQIlxpZ0GCtcwjR11tcFPe0ARfcBPEecE3ULO4kpNkEIgmSAC/SzMpigB79GHWTS7xzfI2pwwxokWktVc6LUEP87GLUuzuiArxWq9ZpoGXEh88fnnPHpiKaolKMXd3ZayLFBK0G3GKC4fPBSzWE6E4BmGgWHIPHz4kBQ8XT8yRc/DR+eUpWPXt1xf37B5dEbMA74PbG8P7K5/xQcfv8+TRx/y4s0XbHe3rBYbtMmsN0v2uwO2tITgpXhRrcjJsFgoDoc9KQc+/uZz7m63bO/2KAV1veD8rGEKnqwMwzBQWsdmVRJD5uxMOrhCGNhs1hRl5DC0NLH5ndZufRK17zcBes70Or6OBjLU0dF777jNWfSl33LSKkXWahbb7t9DuiE4CekoBIGbOel08r75tE+TP+SkJ85SrygR9zZh+dPfQnKo2WuWJZ9ofqM0mx3HsSWGCWclc6soa44fWc+i/Tj0ACwWCzHBBtEm2vbANE2zAXfGbXoJepYstIgxTlBkJhGmkRwTzjq2d3dcXDyELFjRFKWQs1otWK1WnG/WgGK1iLRtj8LRd72gKtPI3/ujH9A4w7OHD1k1S4w2GK1JIeL1hLGOcRqJOeEoZpOlmwNo9b1Ln4iKExjLsWvgqJOlrCjLisViydE5vF6vgeMZ2KGUhTk/8uRqR1zpVV3zzW99gx//+Edst1vcrCsURUHKUcZ2Wc5diPc6XQhBMsyK4sRBFyyWXOOjuH7MZBMEaJwLc0nO1kVB4ZbkPPDll695e33Ho4cO6ySrSitFzEcX+GwOzUckw1E8n0PwRE5Gnf5n5pF1NDjfjyydAT138nOvn0hX63ytuA8YVfNneHfsHkkH74rtf5vX31pEjymhYxCGV0pMMYvbyEPyInrPQbMnofpYYEsZYpwv3lyVEgwGMAuUOYNPEsaYZvQ8iFsp5SCVlhRntriESaZ0RDTIBBGTJPgeES4+yIBMRxGchDayuGmVKQq5V1qLMB6ikfeJc5WNmaOsOLWvK6RiZ3SiKrMUl2YBVJtjYKoIjlbnE2bjWN87VkbSzGuapsR+r5jGG65ftbiLhr//j55wdrmaJxSNVsKPPbI2tWJGGxSEGCisIyiF0U6EDjIxz8iRI5sJTVHUhBj44sVv2B9uuT3c4H2aF0tDDiNWadquY7/bkp49I7sGes37Ty75+L1H/Pyzt+wOB0KK5AhG3w/skAVZc5xW/+YQzH/j3z4kDjdbPv/1r/j88QUXDx7h+zOa998j5YBrLvjmP/k/8ObsCTc/+Z/wVy8I4xUxeWLws5s7n4b9saKbj/+feQzORdbjB9Lz+GMWko6VV23kUY1RRGujhTV3HMRSgZvd8HpOMtbHcFEwToRQY83poYxpFsHneVLLiicHeqXmoNIkYtv8/SHNbX4zdkjGeprnyTyztGWcn67o3+55/6++Doc7mRSTOO+PbUjpHbzLu6ysYyiFUhplCmE6Wzu7s93cSTE7SpX9rU3B/XtFQhxmTvWMhkJa4WWxSceyAdJZEkT0Hnu6cY+pDGVRze1QMqWSBJfig8d74afF6InRopXjNCUrxKk99BzaHbv9VjaKJFRMhDgwjh0+Ckd6nJb0fYs3oCojzo/gOWYe5BxxRYEC1qsVVVXOad+Goe/pZ6REIs9z1m9P4NbObf/en4oMzlmOC2POUsnNKdJ1Az/84Y8wFlariqvrgcWqoalrNJCcxWjmropAgfy9t7fXnD80FLahiJFSJUxMpEqRtGYK+SQaFVo2AD4JbsQ5h9aWcewYhp6+70jB01QV813m2GCotTqFsRyr2xkZ90enwc3NDV999ZW4d0Lk9evXTINUw8/Pz0nRs9sJ5uPoBg8hiDtay+/jw0Q/JNCJvmtxRYkxhq67ZlXWXCyWrOoa4wzWeO62L7m+umVZr1CIeBNDQwiC3MkkcjIcWwGLWJBTgaIUHIWSgoLRUhXX8yKX5+Isc5E3hnHeA8xzUBR8UAriOHLWze8zu+yN8L/NzHO2zqCMkgDO5IkpSPve13wFP81CujDfPQPKNjgjnU4pToxjpmtHnBYEBUqRc5wd8AnntMw9aSJlYaJXrhS28ziegnfT3D4Zg8dYe0L2jONInO+lsXIvh9n9IGNLE2LAe48KnoydczuOc7klExmnibbr6buOrDSuKBkGCR3qenE1CfNvJCZPOZUzB1ATgsxffZioXEOKItI+f/o+//Qf/3O+993vE/2M2MmK6+trtHGUVU1I8vssl0uaxQqlLMMQGPuecTzMbmeHcVL867qW3W6H94LMQCkOh1bwRMmzLksuNkt0PJBDyaMHZ6w2G6Ykjp0wjbLvU4nz1ZJtmHj95Ze4subRxZq2j5yfnbFcbrnej5hyyWpd83B5wQdPznj2aMOivGC9WVOv1myePCcng06Rq/MV3/r4OeX6jsX1Lds2yhpkNERxPKYp402C5HH9gK16sI4xtCRjWW7OyWkStrEysyECNEEyUuZCh3VQVgbbQd8NZORZCXEgxklcoFbPRc6IhEkWOKfmPaJkX/jgCXHA+wFtjk7ujEoKcqAsKxQZ7/XpuUsp4KMcrrJShNkliDUknQlAAGwWUd+PIzlNlGXNZtWwqBrM/IyvVjXrVUNVlXhlqJNmGHpSEhOD0SKge+8hZ4rSohHXjsrghwkw1NWSRXOGwjD5OKNwDl/72QbQReKsXpLjRFUX+KknxSz3xSnG2GIDwko3BdqUTDEzTBML5PCkY6Bwhexz/ISzlTwLIWKtoywLmrqgWS44dD37w56qtrTtgTdvr9Ha8OjZI0B4uKaSEHSjLaWpQCc5UCrJr+jCwHY3sO8C1kasMUzjgZvryJPn3yZlgzUl6ETZLAUpZgrquuTD9z9k1SyQTgXpUDw/O6MbpPCttTjzisJx2HfE5AXtmAIhIsUuI0GsMU24ws7dEwkVHcFrhu1APhiGmx4THWd5TZg7sW5ff8pw8xs++Vf/gqHd8su//jVjn3jvySO+8cGHqBRIStFOmdevr1hdPuLm6paf/+KXfPrFl3ST55vf/gStEilOfPvb3yTniDGaR48ecHZ+xnK5xhUFBAky/fHPfsWPhp/wB3/nj0ALrmi5bKgLh7PyDOSUCT5glKWwJdM4cTgcWK/P2Gw23N5uiTFyd3dHOVQs12vutlv+/X/4/zB2B8mM0oqiXGKUxg8jL1+85PJsw9nZmmpR0w4deTIYs2ZRl7z66gVX128pSkfRNAzjNX0bWRaW9x48oDSabbvl5vNrbKlpzksWyw0XZ89w/hbnFNSK9fKc5AJv797w9tVrzs42+EbQUH0vjvU4eWIYaJoCYzLT2JOJNE1N27ZYZ3j88CmlqfBD5Pb6hu5wIEyeatFQlwtyznPWS+KwbbEMJB8lp8hMJBQ6GzBy4K9q6daKQ2RZNDRFjc6K3CRiF+jGQHt3YN0smWKBSopDu+fQ9RhTYBGO+Nd9/fI3f81HH3/Mv/t3/45f/PqvCQHef+8MipaQMoMXs5orCj7/7Jr9NmBNSUyKFByZAe0ipuhJSmGsJmfL2E9Yk/ns17eo2PGLH0mnVbVKNEtFaQ3T6OfznJyJRwI5WYwuaJol0zDiqoKqWdJ2e2LSjOPEau1oh8iyMlirGIeIU4msI8pajNXsuohyiaIuhZMLRO95+WrPfmFYNIYvv3yDK5x0bRvF4RBZ1LIPe/Fyy9mmBkQ0W583WJcoS2HsV3Ug5onVpmYaJ169vKU9jASfWG1W7LuO//Tnf8W//Jf/mIdvzxnagVW55PGTZ/zx7/89VmZDmao5O0bECoUipUDfHqgKWfNyikwzZ18pPZsqZ3Sqks4+BVhnhLOtBalCDjI+rIi9ZenoZ7dnzoH97o7ClsQ4MfiOGDx2U6JyZBgOVHVJu7tlHDv2hy1lVbBaLeUMlWc9Zl7zjvurlDLKyDldPGP6JKoezTp6xohorWgPnZhkrEJHhXOWstxwc/MWULx5+xZXWG63O6LShKz4/KuvuLy8wBUl2gq64ny9ZJomlsulBENOIykmqmo5n8MUVVkJIqjt0Frz5NkDhivPy+0Nr7c3PLhc4rTje7/3XX7zq8/4za+/4u/98e/z/JklfRW4unrDcrnE6pLLBw/o+o6qtnz7m9/g37/5DaDouz0xec4vlrx6+SXee84vznj96g5rYfJitDx0LRcXGzQF5JL9bs9qbWWcrTcoE0i0tMPI79gAPnfsyhlXtFB1ykcCZj0jk5MmKyXm1ON5+Siav2NPPP59cavcZzvl2WF+MhTKN4uI/06m2vFcmknvnFHfPavKe0s23P2f53w0Tb2Df81KOu2TnOQlTFMMrz6OxBABi/ctOa7RqsQ5S9PcG+O22zuslWvf9wMpJbquxTpBjhCPTH+NsepkTDLazB0aYuxLSYo30zSRknRnLRYNV1c3lFWDzprVYsGH7z9Hac2rN9c8fvaAaUrc/PKawlkers74/W9/i3VR4OoSZSxhGDGALQqCDyilcWWBNoY46wLEgNMaq+Qcm46a5dw5/m4h4nhjtLEYW3J2fgkoiqJEKf1bJAaUZMWdShtKieBfWJ6/95ycM6vVirftW3KWM7yxhtweTp3WXdedCAJH7eHoQj/OCd77k0P9GCZ6MvfFyDhmTCG0D0PEOovWmdvbLZ999iVPnn5C20l3fl3WMAvnQl84akqz9jMX2ciC5D4K6QBKi6FKI9mHIpaH46hHnzLzjqUeI+asdFQI78VxheSk3FeTTo/b6bP85/flv/z6W4voIUwYVZNiwIfElBTTJEL6rHEzF0JnERNi0vio5/RzCAaM46T5HTtQAHyAYVIMcWaXK+YWkGObiPxeMviOn+oIm0+zMMnsJFT4kAhhbvWYBX3hZQvP1epMWRo0DmVF9T8yqsPJxR5F7Mgzh+p4Q5XCGnB2DoaYHbrH8Swu+IhTIGGW9wUGmXTvBd4UI203SNBX8rgxMt7tubt6w/rsfHZY2RkoLuJe3dTkbMlZSwiesdJaznE6TjPaRMlhSmdCiBSu5m57x6df/BqtJa1aaYN1FrifHJ2zqBhIUw+FJruSZVPy/pNH/Kcff04Mcd5QHKs5switFIGE5r8u+NxDCaTa+ma3ZRhaxt0N9dmSqYdpuqSsKrJS1OvHPPvB/5Zm/T4v/+r/SQg/ZOpHfAoMPjOF2ZGd753Zx0LHUTw/FlmO90FC6TQhJOGgz8K0Qlj7foozo1rCIrU6CoCJcYzzzzmmqt/fT2M4JX6Tj6Gzc5DhXIXTGkjqVFxKWRYYIyd9JFwxEVLGOrlGMcTZdX9kwce56HOsninIX19kA9ht78RJkf286XNoJQfhEO4ntjxXCGXiRa63lTAdax12du46JwuJURarHWb+HlDCcEUWkZzG0xyg5Cqc7pmeN3l53mDo+UHu+55Dd6DOBSwzKmtSYK5SyAZ2mga8nzDakWImh0xScRY+NTkV5DwxjD13uytubl4z+V5Ev2TJeLphRwwepQ3j0GP1lsJFSm0JKhPDRGENZekojKZwhtWqxofAarkkTJ6b64hTkXVdohUMIRKzPCXMrfzyvEIcpR1LBHUluKYM3gcYA9PosVrh+4lD9PzoL37Ex9/8iJgjn3/2gssHl6xXC5bLhtJpgg1UZYGaXZhTGLm9fcuDy0cicOfAsN9TlyW6lmDIHGR8Fa6QOdFIcKRzhnFsOex3jH1LCgMpZpwtcVbETucKuf+2wBoz51cI+zhlhY+Rtj3w+vUrttstDy4uuXr7lrvdfsYIefb7Ldu7G1arBmutuBrnBVfNzoZMIiSPDwaXjHCwo8e5AmMNw2jxTUPoDpRPnlCaEgiQZiEuOXwaGKceoxUhDMTYz8+XxmiHMY7oMz4n9FxIUBmsrchJoXUmxFE6fJTM/YUVB22c3ftHHFnI0nJb1cLMOxYGpHgpa0+cRfTClZCVBI+qTFZxRgN9/d36NA1oDaVzlKWEuamkpMsiTMQ0yVwYDMY5qrqcW38DWUWq2pGSnvEz0gWk5u4bCdlNKCPtyCkl+nE4FVOPYTXee8LM1TMzp/NdxNPR1SD/9iidcFWBddKtE+JEHMLsmPJoIyGiw+AF6xI8IUouSl3XHAvwwxBJyc0cQEOOmeRlHfJD4OHFI/7xP/xnfOdbv4cfgszxg6B6mmZBSJFuGFAaVus1m7MLjHH4kLGFoWkUPuzp+o79wZ+KR7e3t7/luhbuvyFMihAiVSXc0FobHj5/wtlmwRQijCIYow1FU2Goudu+Znd7S+UcTdPQD54w9vikaCysnlzw5L0PKasKG0ca6ykYOV+eUVeGYWjptjcoU1EWmg/ff8rC/Qm3hy0vb6642R7ox8jdYeD162uub++Y/Egkz+M340NgmkZiUvT9QLvf0azWGCNjwvuIV3NoZ1FCCuQ4kuMIacLoBFnc6VlrVJ6wJlE4qEsDyUnoWlZUlaEq3czHlEAzaYH1s2s5yB7EHYvVc36MCvJz7FwWn9tEbWFxlYPZZetqQX5pJx0O2koYpdPgFhUPNmd88Pwpl5tzFJxaWQsnz71RSOsw+nTYk6wLjw8BMxsaSleQbKSd2a9VVfHg/AEfvP8N1qtzrCvp2oFXr15+7Wcb4Hb7lqoqMSpT1yXjOBF8ZL1aUdeW7fCWvu8pTUNdVShl8X5A+QltDcZahr7jcr2iciW3N1sOuz3RWJTRdF1HaWF9ecbFw0u4uUVZMDahdKKqCqpqwfnZJc2yROlMO+3Zbnc4Y2l3Le2hZbGo2SwaQg5kWrrR4KaRmLJ03WTP7u5A2VxydvZYeLUpUpcFceopneNsvcGUlr7rGLqBm9sbYlIUPswdU0DW1FUtoccarJXumHEaCVHNGTYKazWLRYMtpFOz3UcWxZLrN1vyLvB89YQnlwtSF2gPIxOR/eGOD55f8if/4O+yWdZUtuJP/uF/y+XFOTZ70rDn+vVLXr655sXbG7wqefXTX/D27VvatuWj956xH0c++ug93nv/Cd/+zu9RuoLDfk9Z1ZxtNhK4HCJ939PeXdMsFvyzf/bP+LMf/oSf/fyXPHjwAK0NSkVsZZimHnJmHAasNoKimgL9KMGE3/29hsViwXa75e5ui9GW8wcX6IPl+Xvv8d//q3/FzdUbdne3lM7ih47Y91gUi6rGGUM/9Ex54vmH71NVFYfdgc+/eMUPf/RXXL19Q9nIXK2toqyXKOW4vLCsVg/ZHmpevX5FP3SM+5F26zl/b0MwkaLWbJozzpozQj+STRCEBYrzzQUpZR5dPiPlzDAMaCPdJ227kxNPipiTQUWhlRgQunaHnwJxkgL2OHhcnfBMrMoCYy1aaTbLFfv9Dm0Vo5+YYobSUDcVfR+wSjO0HQXSjebrkf3dDq0NJkNjC/yYMLXmwfkDbGk5HFrJVkiBi9UZm8X6az/b/6f/8/8R4+DhxQV/5wd/h8+/qkntLSklDqFjcVbgLGTv6VpPfwAVE9GnWRhIpAhxMliniDPiSeatRFEmcprwfcZPAXNwBDTGRQprIXg5s87d3FWlySmzWGSmqed2e6AJhqIQFIm24MqCnCKlc/hppKlruj4wjoo37cTTZ2dgRqaU8VmTp0DlxGxQLTTKQtaRshQE1qIqGYeRMCiaZkXXv+X5e2t22146yo3g5NCaKWX2+4n1WjIVXh+2VKVlVS8ZO0/Qid2h5ex8SU6BX//mZ9SlZdppvvPed/neh79HOASqsw0qZ8YYYYrYKuOKjM2B2kC/3wrGNnnG3pOSoiolIFuljEaCbD2zIUkuIsGPqDxjT1XC2IKQPZ3uZlZ0moXVRHfYoY1itVrSdR3Be+qqpus6nIWqKhiHLatlITk9ecK5ijAJQ96UDVlrgp5RN8mTrT2hi8gimueU0EmMRSklQgrked4dh0B/OMjn6fYUlcGPW6ZuS2EU12+vsGVDuThjzBlVGu7aPeebtbicleauFcZzP07SERIiPijKckVZSgdb4QwpjRRFx8s3nzOELdf7N3iV0bag3R2kC90nNg+X1KPn9vYG6yybxYJd9ry92rFcLnn4+DnKWBaLhpur1zx5bLFOsT9Ybq53xFWJs1BXFTEH+lGKHF03MU4jy5XFGcvl+jk/+otf8fG31zhnyNkyTJnlqsaagtJY4jD9Tmu3QbrVtFQ1pOv2na55kYtmgVzNgFyV0XPa3Luv+4BG0SxEaOUkKt67x+8Z0QpN1mk+h8rPOnXmn97/HW1BicYkSNn8W/v3dwMicxbjqvxbuiqPHmw9s6ptodBKMlImvwJlyPkYoqpo256uG1guxRh3zDZrmhpjNcOQCHHCzHgi6TYVB7U2c9FIqRO3e+gH6roWY9nsys45U1Uld9sdDx9e8s1vfcwXX3whuT3W8avffEazKkg+8Pvf+4TaaFyMlFlRVA1dCAxdS5kyrixBSSe4subkJk46k36rk/goeIhR8L4Vcba7aoNzJWRNWS7kqs3oHKUM+rjgzcpvnvUvpRSTn9htbymc4fGTR3zxxRdcXFxwdXUtRk3SKedLrqk+3TsQ7FRKibquZb2dkU/HDtujM/3YMQ+iTySlMEo6Y/qQqCpIKH74Vz/h7/+D76K1JeWRKYyyL89HNHM6FRUk5PYISBUjtJzUROxWeTaeaTEDkY92GXlmUk4z2uWYf5hQWQpP9x7T2RQ7G37V6c/U6X3esUufrvH/0utvLaLv24m6kQihaYqEBH6CGC1pdhRNkp2ETfJrT7L+kKLCa0VUeeaFM7trM0EKNXQeDiP0XsKErNFYI6m4+XSpooi2x0IBiEsvBvTMxklJnVAvMb2DdclZAr84ipkKW2jAzQK7EZRGVsR5OKf5YsYs7QLmnefA6HsRPPjZcYwIpz7Oorl5Vyw9OlDlv/VJ7IaYFb0PDDHjtSKNI+PhjrBcY40wLCUgYcAV4gglWcYQMMHK5zdGNkEmz20vAbJi8iNaO7JV7Pa3XN9c03tpK0s5Y2f+LLOoqTMMPtKDhGnND/qiUDx9eMHowzvVz3vekNZzi898ff5rLvTf+krOXDQNpj8Q2zt8eIxLit3dDQ+fPSenAFqjywVm8wxVnpGznUOoYAwwzZXIzDHUArJKZCXiTeIIelHiqszmt4sumZmVG4kx03eerg20Q0CpDDqhrUwYwQf6OVjUGxBunZ6rswjixErlM0YIEaaUSXOh61hpPmJbpDqqZpyLXJeYYYgSNVJYcdv5KGxhHxNqDq8NOROSmcd4eufKf72XH4d5UfFAJoYRQeLIpCYOaslAODriFbOz3ssEq7QRId0WIk7MfOqyKKSSqjVZSVtTQvjqOh8xK7J4qBlxkwEi5BQJ0YMWQdH7ka47sN/d4oNl2WxYNeP8viMpBybfsm+3jMNASpnSVaSQ5opmIuPwwdCPO+7211zdfMXV9VcMY0tV1WRWaBMIYY/VlgYzM9cmkuoYBjtXAMVpUloRfeqqoqodVXYsFxWTUbz1I2frJU7LT07DRCCjjMGHiWEMZCLeR7SSEEVtEsvVgsWiEXHKR9h1RO/RgEWhouKw7fnhX/yE5x+8RwiRF/0b3haas82K87MldV1I0KYznJ8t0cDUtwztjqYsydaRUXT7A4tqSWkLdOkYx0DIc0tcASkGQgz0XUfftqQ4ieCOiEXOlvd5BByd2gqtLFpLMUaj54KoBNaN48hnn35GU1ecn51zdfWGoW9RZGEyhkme2/SO40K5OVD2fr7Oc5uW1pbJj+BhHA1xGKkePaIfJ5Qz4oJm7oBygM7SluscOXoRIkgoPXPQlbS/5pDweYIcyVi0LqRjxSTi6ElI1gQ5UmDJc8FA3CAJbTVGCS6gKB220Bh7nDfM7PyZC5ImC5orRklARw6TWt+3cX6dV1NLIcUWwgXOyZKjsDOdhUmBypEUE2me11AaNQuGSuW5jRBJVk8ZpWVOyrMj4RhuJS7uo6tAxHOtNTFFQpbWSq0kzNs4K2gVrYQfj3TshBxxSYv7am718kdMx9zxUhlH1weGvkcpg5kfScG4ZOnimPnL4yjdbIWtMJTUrmGzPufj3/82n3z7+zx/+iHX13dM/cQ4TJSFFGPudltuZ+bq4yePqaqGvh+ZQkfOGmdLjLOzoAXH1PtxHE8O/qoqZEOHzP2lMzhV0SxWLJYlj88ecrmpyXFkSv3Mq1ZkHQgp0/Yjd/uRKVqa1SX7tuP65g5b1Vycn/O0WXF+8YDnH3xEjIHD9VdM7S399g236UDOT5hwfPriR1SLM5bNkuVqweXmG7x6/RnLVeDm1nG796yqhYzhkLnZbZniwOgj/RRYzOHFOWnGYeCLLz6jmyYuHz9Cac0wSp5E09SsNxsiinFq6foD0zRiTKKsDQweCLhCmJ9lZWcBxpKSI0bpEFyuCsqiZvKRcZAgyKIsKWvDOA0S6lsVKJVABVIcyTmiVMQZObwbp3CFRmfHVDn8OGJKQ4wag9xjZQ0+ydgqLDT1ggcXGy7PVpytG7TS9J1mnCaG8UDsE57AlKYZNTUL6VoCdq0xcjBI8/OREoU1rC4uWa3OWS5XNHXDg8tLVssz2nbAqt+Nq2qVYWg7YvRs1ivqpqZrR6YYuFyds+t3DN0ANmCIpDQwHnqMsmQ0m7MLls7y/pPH5BgZxpGb3R5nahbVQnJQTIEpGsp6Tdl4TFXifcvK1Ci3QGuLcQVlvWSaOogGHQEzsd3dCc/eFSwWhrIoqKlYh4oQ7+j7lqI0hOypFyVVZdAqoHVkTJ6YBbGRrAIDu8Mdb6+u0LpgPw6kpFFjZLVYcFZeUjrNomjo+w5yZPQDzhVyXxQc2jti6FmtFlRNwxQTfTuiU8l4HRlfTTxbX3BWOHx7S7u75fZuz812x2a9YFFuaHdveLipaZYlY+j54strsp+4ffuGm6u3fPnFC7IuwJS8eHPHcnPOo/WGfX/gW598m4+/+Q0eP31OXS95++YNDx894vr2ltvbHePk6YaRqqp5eHHJL379Fe/nih/8/t/h//2v/zVNWTG1neBLFiXbtpeDvLGsVyuKQvHlF1+QVeLFl5/z4ftPyBiqsqIfr0l5Yvdlx2K55GxzxvmDRzK/K9jfXbNZLVg8vKSwlpvXbxinkbPFiilMvPzqC1xRkELkL/7yL7m+vZaQZaNxTU1UiTbt53OPtHy6jeLcLFC7wDRExpuOO2VJjaUqSqra4FwBIbEsNkAmx8R6uWa/P6AUPHhwyc3NG4ZxoO2i7IOsIyQIQ6JQJZnM/mZPr0fiJIi40hUY6xjGyDQlppRYJEW371HW8ujhM+r9lqhG7vZbVPQsixqjLB4zF4EORFPTth3Ncs1vvnyJ1Za6blBOURRqFkQNhSvZLDe0W2FJNxeXFNXXd6JXRUVZGr71zfd4e/NLvP4S4zRdl0k4KguujHgmnr/f8KX37O9GslGCQ01yPpqmiDaz61VJxpMx0hUUfSIlcSGOw4CgSaFZWFxVojVMYWScoKozxii6rmOxrGjbjmOHoXPHc2SeEQBzV+8UsNYR48iqdjSu4CZkXKEYRhGOH18KQk2pSAgZV1RYa5hiIkaFMSVnZwWHw4HRB1wppo2+a8kopjByVqxmnEPg+npHWWoWS0vTrAg9+BBxheP6pufBQ0thLddX17z36EP+5T//RzxYXLC92fHx049YlAvurm8koLO05Bxp2z127hrc7basV0vGEZS2OFcKMk+pGTvZi1CVFLpwsvcPkRQ9KUp3ktYJlQPlYnac9wfGvqeyUkA+7A88fPSAbr8VDWKWa4qiYPQjVeWolw373TVGZ0geFUWDiKYiz6x4yX0SE6Cd9yohMI+NeX9NngMuZXxImKBnmibathVkpNbc3Fwz9R0pRqw17LZbom65fPgEyPStZMFEP/L40UOur28l0LDvKV3NarXhcNij1NyxPHfg3t1tUSqIaaHrefXqteA+yhJdGPLkOTtfE2ZM3xHP9urlS5TpZsNNoqwK2k7OEU+ePOV/+h//ksVKwRBnd3diGj2r5XLGyXQ8fLSkO0SaRYkrRGgsK8PlwxXvf7hkuXLEELC2mJF0kbY7oIzsBX6XlzkK3O8K6Ccn7cmeSCITc7gX1LkXx5m/T5/eQ7jgAvE55pTJd73rRD/qOBHB3x3/d3/W+BtiuuJkWFR5FsDeMVAe31uhyOrdz3n/26ijkK/AzF2w07hjmlZUzYYY5PofndFwNNgErBVWeFG60zzzrhs/Z8F9FkWBwjBNfj5fidAbYqAsJcj642884/Xr11xcbKibkrdXIx9/4wPqqiT4iYvLM1w98NUrx3c++Sa7q2v+8Pd/D0tk/+Y1ZrEiZ4WuJEh0GgeqU15UwDiDns2594iRKM7rWSh/F5c730HgPsswJsl4U2o+T2b1juD+zm2Zr29MYkTb7e7QCi4vL+bzslyzKYTZLa84UgWO1/ndPLau604dxEcXuoz7PKNyzKmAcsprG5MUCxyA6MPGOD77/Ct+9rNP+eS7z0EppuhBhVl/OxZhjqK1Fg6/FuZ9mokLOYd5fGuSShhEyxXKgdAg0Ho2WYqmxizMi5Zs+JuKZJ6xxEeUjkLulZn1qeNL/Y2/9196/a137z9+5Xl61lI74eROUdF2gWESJ3qI0E/IxQvyOUYvIYo5KeGXihaFSvNEH2QAeC9O9CkcHwz5+MKGOiI1kuBcOAYwzp7E5MkUxBxmTloWnEvMM+5FDqVxFu3loZZDj7VORLog7+lnDIw43Y9TmJpFyiztNO/8eUzyuX3IpChm4BSlPSnn43fOiuBcXTzdxNNkOG82UiIlRWE0jx6csSjs3J6AhFWRsdaRcoFWFmMdITNjDfz8UFq6TpLbQwr4KAzZu+2OzdLzcnpFCgmrC9q4wzmLtW5uf5UH3CjhL11cPkIZI0Kbdmir+YNPns6ilhQ3TgN1xlGAtIxMMfwXfZPvNAUB4ng9tAcePTrn4bMnLM4uKZszQbSEAIWDLK53XEWIga7d0g+Ck5FihUYxCzfquJDo012a66+ntquMCENuRrAIS18q3MMYObSB7TZxtZPNnbhAmZ28ntF7AhmbEjkfH5+54uncHHI5I49CYgzH0oIIiHkumggXHRmf80ojIpB0DzgjGR5KCWsxxFl4zzK+fVL4IO16WemTO/7rvqYoKBMQLnkK965cfaxcpiAHai1VPhCeGlmYhSl6hmlCqZ5jkUVr4cPWdS3uH2MJSTojUkrorGdEh57b1O7LAXFmG/djh7T5K3zybNtbtocrdoOEPyXA2WYW7UbGqWXXXYv7rliwbW+pXC3FuyjXcvKZYdxzc/eSV68/5/buNd6POFfi3AbrMtYmzpbnWFNglCNFxRQnwrTlmJoQgwTKVWWBRhzb5EzhDPtdh9KJiwdnhDDiesNCV2TEFRejPEcaNbvkARWp6ooHD8+pypJx8oxjYBw8nepOYp2zGh8jZPjVr37Ds2dPcc4yjhNd30MO9L1lUVe8Oeww5oMT39DHyOQjxkSqeoVH3NygWTRO8BMzH0wbcVb7cWQcBmIMM5JE40PCGkH4KGUwKaGtm5nsjrIo0UY6ZnyIdH0/Iy0kLXyzWfP2zRvqqmSzXlM4w831tUyk+V1WYz49U+k4P2dLiooYxYFWljUpOVKWAuI4jOx3e66dpRtauqGlsAWLZiFFLi8ClwYRd4OMe+uMVP21zNtx5lzmLIdQbaRN0BlxTPjgpXim7cx0BnCng4gI5EcnvTrhS46OEbF2iGAtXPaM94Fp8uJodXK8Sb9D32hKnqo0FIXGWCA7pl7WNa1kDNZNxWpl5jCscOr2kowE4Zlroymt5D7EGOmmFp04Pbs+CI7FODsXv+83YcbIcz6OI35es7WRLqgY45wRgYjk8+bSB48JZnaSREIMmGzeKa4IxzqmQDFjnWKIkptwKoXPjvcpkyOUyuH9yMW64Lvf+T7Pnn3Am1dXjENgv91TlzXWOHFt7nakFCnKUjaAk4yRkBJGF2gjKDTrDK5wHNtbvZ8oy0I20/OBL/qA7yUv4fzynIePH/Peww1nlSIOEmCsTY3ViYCR4K0p0IdMtXlMs3nMOE2kqmf16H3qZkFVL1it1yLMLhYc9lt2Ywu+IyTN2zcHIo4uF/zw55+iXcOTx494dLHh4aah7W7x/ZY4DvS7jn4fsMB6uZR1ro8M48ShHVitIlWEfhykjVYpQWsNPbawBD/gp57JRNpDJilN1x0Yh252P+q5u0Sc+VoLAsVaLY5aiwRz6iyieJpQyqJUJKZeHM9OsCJmkDVwtazmLotIZhLsHwFjpTNPWXGpp5yIKoLJJC37Fa0VRVXgCsc4eSCyWtacrZcslwVFAQrPNAVCGumGA+M0MowjgURQmbHv8V4yOpSW7ACUYMK0VvicKAtHXS14+uQ5Dy4fUxQNy8UG5+QgmOcOlN/l9fjxM25u3rLdXQtCqYJyUeK9ZxoitV3icyZFQ9+NEEGnTGUKchJxu7m45P33nsvvUBjM1RWVqzAohmGUPZ4r5xbjB9xub/A50DQVUXd0bc9h6Ek7CQ9udzuin0ihoxt3OLtijIF+6rGqpO81xlas1ysykwh6haVsFlityHGgKjRaFTJ3GkVUgavba26310QUIQXa4FHJUCpNUzXUWoo3u5s9GS9/z0fOz9agC/phZHu3o9vfsawdKEO/97T7iFMB/6pnExasVMPN27eEsKepHSvvJJA5Rl59/jm3L19y+/Il69V63gtLMP1uu+fm7Q2rxYZnDx9yfXVDWTb0UTHs9lw8umDf9fzoJz/j+3/nT0hZ8/TpU7744gv2h5akNLZwXC6WZKXIpuC73/9DfvLTn1EUDf/4j/+YP/+LPyeGibquWV+e0dQNd7c3fOPDj0Rk3u25vbtjGkesMbSHO8pywdOnz1iePSabTNe37O623NxtOT8/5+zyIT/+0V/x13/+H6msYrlc8OH77/P48hJSZHt3R1E5QvS8fvGCLz7/AlRmvV5QLiq0M8Sc0MaijGWYBu52W7ppYLle4qtIWRTYfSANntgGyAZ7XuDHHpYNxbJCK8vQtihEmMg5MI4t7SCdKo0rCCpS1wuCT7T+DmM1m/UGpRVXr98wdrcYZVhvzkE5sjEsyyW2cIyho6pKbm7vuLh8wPLiElPXtIdr1H6PyZqSQgJ1syFMgXbsGQjECGNSDElBDFRnjkVTMXQHxjhy6DvQmqZeCk/catZ1jf0dukQvzwv+5T//37FcnHM7/IoXdyO78Yr+kBiHxLMnS6w+sDpzlA/OyD7xk+3L+6OmujetAYJUsnJWUkJoE7OLM7giEX2cfVOGcYBx8hSVODtjkL2JUpqrq56HjxoWSxH5D4eWEDJV4chZUZYQfMAay74dqWrEkDNzwDfLkm6c6NNEXWuC0kQVBZNaWdlretkrxajZbg+cbRxtP7A5r/FhYvJilipKM5tNEorAfpd48PAMONC2AwrF2fKC84szunHkgw9WOOuIYcRiePb4Kd3uwF/87Av+8Ds/wGhNnDzOGumC8YN0ZRmNqxpikPPy5CcqY+ew+kxKE6iM9z0pT2hlGLuRMBoKa9FzALvkMUX8NOK9FKKLqkLFiaHd0U4TDy4uUWTGccA5y9B2KKRjKEZQThOHIKYTa4hT5vb6FaTI5uwCV2tycpCkAxNkH6Xm8zs5CgdbKQn3njEROd2vYTlHnHOsV2usE3FyvVnw61/+lMVyydXVG95eveFu33J9e8tqvWIaerwPNIsFMXqWiwVPHz+eXcGZq6s3nG3OsNbRD73sJRhpmgZBX0TONhfEkChtwdXhQLPOnK0XDH3H2dklm/U5tzc7nCv48MOPubn7knbYo5RntWp4e/WGFAKvXryi67YoU5CS59mzJ8R4jvee5XKNUppt26K058OPPmC73XF1c+Dps3OaqubswnH2QPY5Kcke9PxsyaF9y5MnD2iHLT78jk50c8RH/rYofo+kuJdX49yQevzKiW3+zt85itgawVLNcYz3mtr89ZM8eHS9K04i+slkyH3+kVL3P1vkrNkRmtNJKL53t99/KKuO+obc/+NX5D1EIA0h4qcObXtSMqQIONDaUFX1XBjjxNJ3zhJCZBxH1Iz5jFGC3LWW4Fc/hfl3ko/Zti1lWTEMA3XdkFJiv9/x/vvvM06Ce3v6+DH9oeXRo4fUiwW8ueIH3/sEZyz6mx/zjfefEbZ7vvjNp/SrDe78nNXjR1SLRkRcpU4/89g1fcwMNEr/1n1Wer4uMyFBzVif0708mtHeNawex8ORuHC8GTmTlWgfPkzc3d0y9GJqee+99/j5T37Oen2Gikd8SebIjj/+E0I4jZ9xHOn7HmPM6d9HDvpxXB7xqsffJ0fRqYwOmGIe08qidc1f/sVPee/9R2gXqCpmNGk6/ab5ON6VBiP4atFXJWMxzQGgYugSc5pcKXkPnUU7M1owLZJtlk6IYGNmnU7l+0t8NN8di0qzez3mdwV39VvP5H/t9bcW0f/HLzXf2g88WXgWWsLAdodI30rg6BQUh2l+vLXIcaNXTFGhsrCrYxIX7ZyZJSKih2kW0fOcfvxuq8ApZEElUjYQ5XtDPD6/WZyBcxUlhHT6OXEOGD2FGSo1s5qNMIecJSXNFCIhZcYgrnXRK/5G2lOWgR8TJ0E2RfnskhA9/z5BigJS2TH3eXD5Hfs8x8RkEe2FlaxIOVM3FZuLZzSLgmT16WFUaIyqsFreVw6hgaBEDEoIn6gbOvphAOUY/cA4jlxf33J9uyVlqOuaRbXGGGHxai1TnjGWqBImS1taWSxJFGTjyLbCWMP7D1bUpWO3FzH4yHezRqpB4zTOjPFjTe0/F87fnfVzTpSF5uXrl3z15SOqxQOKZFmsGvw4ngLZpnHE714y7V+R/B5r8kyaEzeqMLHn91b3A184YPO1Zq72orB6DnpJieA9wXv8FOi7wM1t5NV1pBs8Vie08oSg5tTvQIzz43scV7MgHEOYK4cSvhdCZJxE6D/e+nk5kvDVBH5uTcinKrASznvI6ArsnNgsHRv51M0QE4wTDD4TkuJU0PwdXr3vQCWOXNqUkgRFIu3e8jvHWQzMJ+67BOXIdZUCkkxgwvHOKB0xhaaua+p6QcxJhGHvSREsbn7/TAxhThCXOSKEiX5oObQS7Fc1BWhouwN3uysSgWFq2e5vMaacUTwjkx/Y9TeM44g1FZVbiCtaCdMwBgkNORzu6Icd/XDA+3FeRAyubCkrw9lmiStKSlvhTI0mEcOIjwPMjumUIlol6qpgv9tSl5bdbkdvIISRR48fcHNzDSZTNwWbSric1zc3uEJwORLMJhuExaLi0eNLzi9WpJRmpA8IroCTS/pYhQ8xkVRmu91xdrZhtWpwtpBKcpZOFz1XV62zwofWjhASOWu0djRVI3NSyuz3exaLpbTSaz0HBMomJeVEUZRoldAqY2feuaB8rLjDivKE9THGYp2w3LRJtO3I3d0dv/jFL/nVr35NmCaWSwmvbA97uvYgc3kWt/FRRP+bPH2SRmVLjpoUFEkpkpV5CzWLqykwTCN3uy3jVNB1B2GcukI6bJLgWVQWd0WYf8eMwZiAVlHCkbP0YchecC7LpTiHciaCn/BGihMqHx3IGelwOjp6JoZhPDkrzKkyfvxnPuiq2XUR5B+sRrv52YtfX0Q3Ws+bdU8IURLsdUld1/RdRA0Ko+UepjnwMaU0t7BKISHEgM6asjw6FCbiKO3uRVGQtTptyFLOKCv38Hjv3m31TCmdrsXx60eXQ855ng8kOGwcx3c2MnLDQvBM04iPAe8nlNLEOHEsWLzbFiqBZKXciwCL5ZL3Pv6Y7/3e7/Pk8VOmYaLrerp9zzR6zjfnZLIEDKuMcRZXSJbCOHpCks29KiSUNMQoIWLGkBIUhUMpmKaesnRoMtZosjWUTrNcLnjy5Cmr1YpFaQhTK58VS1Yy1m1pqBaCYiree5/SNSjuN73HA8s0jTRVSeEs4zjMYpmlvxvoJo9yFdWUuDp0bNvA9e4VhzFyfbvlL/dXDIcXpHQAVeK9PK/nZyuq5ZopRyg0qjOMc8fM2driKglqj7NjKsaAQwKDy8qgVKJtt2Rl8ZO4w62RQoPNmhSPGCPBach6knDWUs3BrCEkYuiZpnnt891pjlHaoLQlo6hrhzFB3t8mcogkHSnKAmccOIMychiZBkO0wmoMQfBM2hQUlUUZGPDUtWW5KFEqMIwHgu+lQyJn+rFlu9/jgydrhZ/3DinNbcaA92L0EPePdJBZA/V6TVUVGCti1eQ7Xr46YLSj70eur6++9rMNsFotaNu7U/EypSSceWto2z05ZKqixNpC1g2jKK2jaIoTN9RaxRQjVVWyPttQLBfUZcX1m7dzB2ygrBxVU6LHY9hxEMclHldAxnO33RPDRL/viKEH7dGFxZYOT2Lne1Qc2W5FEdA6s1jUDGNHVdYoZXj7+oqqaKiqgsuzC/aHLU1TUjnLdrdlGDyL1YZut0eRKQqNzplEwOfA3WGPvx3ZbFZcXFxgck/oFcpldvs91pQ4W7G9a1Emsj94hhGC32NGQ12W7A4HpjRw+eCM1E8UVY1KisW6QaN4eHHBplnTbVuM1vz05z+m7VqqsuGjj77NwwdP2O5b3m4HrnvPIez4+FsfcXWzpVkt+Zd/+i8w1mG0Zbu9pSwLnjx9iraWvh+YZhb9YrWhbXv+4Hvf4tXrl1SPH/PJN9/nZz//GddXL+i7axaLBWdn51y/fcUwjnz2+ZeQYb1seHj5mGaxoO8nXr18we1+ZNvtePDwkufPn+OKkt/85lN8zHz/B3/E9vaG7fVbtLW8ev2GME08PD/DWUW37cGAdZrvff+bFFWJsY6b2ztutlumfhQbhjIUZcXFylKGDopEN7Qc/IH12QbjK3muhgSjRrvEtt2hrKMuC5S2DGHEzwYhlSP99VtM9izKAqMNkUA79CgD1bKiXtQolaiaksIadrc7fBiZYkRlS3O2QBs4f/QI4zT90HNxdiEGq5jwIYjRK2XutlvBc1otCKuQCGnC2VowHBnp8DWyr03R03YdXddSuIKQE4V1rKqay7JiVdVf+9n+R3//j/lHf/+/w7pnDPk1//d/t+P2MLK7eUMeM8NtxdP3Fyg9kb1j7DqMtoxjZIY/HKkCCH5q3vSoORtKKTAFYZCsgGxHXGEIk2EaJzCSb7FcO6yVInnhanLSkvdhxQQWQqbrBNFwc7Mn+Sh4qcpSOIX3kbq2WCV7qUVVsFzVXN/dUTcVt/uBi/OG8/MlZVXw6tUrwVYOSDBeFtyocyLSa+NYLBsWC0Xb32FMjZ9guax5/BhQnuVyw/X1DcMQiYvEct0w3nq0lX1I9JGqKri7vqNc1/yTP/7HbMo1pS6xaIKXzuJpaCFHfAgUheHm+i2r1YqyLNkfWooYsMbMXVcDSiesNbSHluwVz589BZD8huQpSktKiqG7I4aJu5srcZorxbJp2I89fXtA25oXX37Jw0cPGPuBaRLevSsaVGkEJ+glmFHFkbPNirvrN/hhT1E3gqWMIzkawHAK3FNKAneVaDOCIBNncYoeax1Ki+M9+ImiWRBjoO967rav0Ur2IofDjv1hR0yJQ7clpxGtFMM0sj5fEtJEvbjk1csXlK4ErXBFSVFalssVb6/e4qyhKEukiwym0XN9dUdpK2xVo+uSIXlS8hSFBEB+/sVL3n/vOddXt3z55QuWK0fwkfPzNW/eviTngtWiodtPXJwvMQWU5ZK6rmfn7cTLF69EjPWKotKMU8fTZxdEbinLRNU4trvXoFr2+4lHDz/kyeMP+PLVF2IQSZGFrein382J/i7+BO7Nlf+l3b5R79zDd/7u0W39W++b788rJyUZEGzvcT6YBdFT7OfctX+swOWj4D4rF1qfTisZMRqJU/i3CwDyHcfPd+x+Oc5D8x4dwbpg5Gspevw0Yl2D0qJlVVU9C+Ri7AkxEoLsuaZJgmnrppCcIQ2usKdr6WfH9TAMHA6t7Ju1uK6fP7/g1csXXFycYYymHzrW65UYenLk7PwMrQ2ruuaDJ88Yup5KWzZ1w+s3V5TWMU0Ti0KKykqLlpbJJxMl+ViIkvVSHQsJWvZl965z0XWMubd9MlMUjjie4znx6Dg/XevMrNXE2bQFfvJcXV9x2G2lI+fqirOzM66vbqiXSzF3TPHkKD8WccJsgDoK5YfDgeVyydGpfjRAHf/eu2PWaENCy7XwkmlksUxTwpiaq6sDbRuplxk/G3VlzB1d4KK7iNg9m2CVEgPrMfcS0W+TQnIfZtOpYFlApYTRzNmccTbcpdmoGeCdcXwcoXOpifvsPikovfsd/38V0f/nW8WXLXyj8Tx2nkZlhiEzDQqdFSFl+lGqASiIWdN6wbkYxOUdZuFci/mOrGYEzCyKxyyO4RxnXjJHMT0J8ztqfMhMI0yTIGO0RgQfM7t83xHgU0Y2RlkGrLUK4xxYi3IW7d3cUhFISc3VLjUjVmYReA7oFEf43Bozv/fg5VGYoojFId7/eZgZ5eRMnH1xKWdinonhWYoKPgmDPSZxuj98fEFz8Zyi1CSV8dGTVQBVkHWBMhFmESen43WS9xvHUUQd77GFY3+4o+9Grq/f4n2iWS7ZXH6HmAJKQ2kt1krri5pDJEnzpJayMOJjIuuCmDX1ouHyYsOb6z1HVApGgrpilnZvleeqKe8+/HOFLt93GQCUztCYyMcXBU3c0d1+KqJCekpfl1grFaJ++4qbv/5/0b34KXk6UBaWMEUSQdrkjqW/dB9CCVqY6HNRJB1lajUnBKdEDJkwJYZ+oh9m1r8Xh4GECMp4OLU5RQ3KYnWcFwVxgIdpou/G2aVboa2koI9TmpntSULGcsRH8FEe/JilkKBnQV4aOBTOiMBgjFw3H/2puJPmroopKMYZF1No+F1xLreHq5nHrShKSZjeHQ6Mg6eq6rnKCzlHyqrEIW7TcZro9iN9N+BHP6fFe5KXCSzmgeWyYn12hitKJi+O5LbtZiyHOFunaRJXoWF2ngaSCvg4EaIXt96MGUNlRj+QcmDf7qhefomzJX4KJ/FLORHru3Zi6iMG807420Tf9YxTh3WKqnIURTUz4D1TiihbofWKsqjEia4NKmcmPxJSL5OzkvYfBVSlobPSymm04HCWS+GKoROLVU29aCiKirvtjsWimjn3ag5TjJQLy8NHDzg/W7FY1LJomcQwDIQw4Qozu0cyxjjA4KcBbaxgsmYO3jRNlIWVIoDVrB+vKOtm7jyRDhxtLAmFMlY43CiKQtpz2/bAer2hdAX9GOaA1jQ7WQLOOhHovSy8zWIxV6OPgUXHLiL5WQpDTIHlcsXTJ8/YbDZ479ltbzFac7ZZs1otyDlw2E+kGMhJhPijsHpc6CW88sjVM3AsEszVaGssWsE0ydwwTCMhTuQUKJ0j54qEiEgqiXDrvWfynpQCSllOieXHOR8Z9+RMjn52yksVV+VZSNcG7RxaSzcJzMy3OczG++lUub8vChwzDWT+0rN7PcbENEWqwkqHU0yC9Pm6ryz3JispgAHz5iTPBycJONRmhCwFwXvRe55Dk+Q9CH88iDP+VLgOsqgjWLGYIn7wVLVisVgInsZPMosqQCtCioQxElKc2f8ZH8XtIKKsISfN0E9yAG8aVitxpbRdS9e377QWMhd65u6YOUQr+EAyirIQ10EIkfeevcff/8Hf5YMPPgagOxzw40jXdayWK8qqYH/YY5yhVOWpzXEKgYSHbPBkirJBG0ucJsG3zPzznCNKI90GZGm3m7m9VV2wbCqcFeTHNEXUNGFQGFuSkLkdZVmsHaVNOK0xZoHCEGKYDxUeTaRKAZUCQ3+gbw8ctlvatkPbkmkIWEq+erPn5dazvHyf23DN613g15//ml/89Z9xsYJFBVXZyOHFlNRN5oNvfJvlxYa/+slPsVVJzDCOPf0UsGWJMfZUcPQhoIORoPEknMHJe7QpJLg2y4E7B4+xmsJqSHYuCCqsdTx7+j7eR25vb5mmicOhI4aJYeYeaqRjIsdpxiYIx3Iae4yR9t8cE1pHrE0YHbEugY5UjaWuCmIo6fuRlD1VKQUuZSSYyDkJ2xPX+8g4jdyFAbIE4tZNTb2oebvt8SnQdSMhJhEitKYspfsm5cw49rjCkWKcOa+e0feMYaCfWrqhY5qi4FYmT05ZsCO/w2sMPc2qwRSPKKoSHyNVU1EUJYe7HaCoq5qYE82yQRlLWdeYeR5fLRdsmpp919NPI9rJ2rGoa4axpV5JIXR11lCUZp4fJBw2xcw4SiHL4FAqME7tKcy4aAyL9RrrlkSlmULL0HdMU6asCjIiZI2ToBh3247+bmLVTCyWFQ8uz7kLAeekQGqsYXP2AKUNxvRYHSnnDqzr3RXTNEomg49c2IbV8gF3bqTdjoxpz67bUVrNNERxYOaREBQagwqROEa2wy3GZJoLy4vbK5gSF9Wa9cOHHLtdDiHy5jefUhjL2La0+5Y//MM/5KOPv83g4W7bcZgUuVpxcVbxqK5o+45vfOs7PHn2jMuHj+f1IfDg4SVVWbLfH8S92VRzF89EGPecrWraw46Pnp2R8sCkej751ntApt/f8dWLl3QqomzJq9fX/PgnP+Ob3/o2KM2jRw/ZrNd0h9c8urxgvVG0/pz9Yctvfv1Lzs8v+e53P+GXv/gV0zjw4bc+4U2zODGPh8mzb1uePHnI2fICNBzaLa9efcXZ+TlKWciOy7PHXJ5JaFo7DOz6HSEHmqakzQf2fkuwntwsKLQh3Ewcrm+obwYeLM7BaKaUyN2IH0d2+wNNU1FWJZHI3X7HotD4qcNPgapqOHQDi2ZJInHo9giODZ49f4ZWmkPXYsuCstZgPYPvWVJBMjx+8AC0O5ljtLY8f+8DEpnb2xvpzIuJoRvIWc511jm8DyxWS5ZqgTLQ9q0I/VqyA6bR43SJ05boA93ujrX9z0Wuv+3rX/7T/z1Tb2n7W17vPuXFi1t+8ZMb9ncWcuLP/8Mrfv5jyyffe8DFxZqXv35F7A1GabTxJxe6D5mOxKa05OwpCj13mUm4oQQFKoZR1i9t7IwGSThtBUEZj9k4nro2p85Aay1VbQghMgxB2OtBsqfKsqLrJwmu1pC1ph899aLi6s0ttlQ0VcPdm1vaYmDqB8pSXEE5yc8LMaGMIZGYQoAhk5OSDsbNhn7cEUKgsDXeB+q64Or6RrrYjWUaJYS+LgvWmzUxyJ7HasfQjuQp8/zRc3TSELJk3MzO62kciFEcjmVVMbZ7prHHKyOCmFL0hz2TH2maCqsVw+C5a7dEH3j/yWN2N9eUVSFZJwpAOibruuDm5orbr14zjIH3nr+PUZbV6oz2cGDhKkFFVoLC7HtP2x542KxIJKzReK8IPlNVS/zQUzcLbre3mLKiWRlSHNHRzedvEbxCOuIBNUPX07Utq9VKOv7JhDhxOOxlzU0Jq4/dwrL3zsButyWEwHJZs+t62VeqSN1UBOW52l7ThQntDONhIJaBalljLGx3t1RVwdnZCusMIUzs2y0PHlyyWDZ88+NvMjDx409/BlpTWMdiUdK2HZ999hv2u5GqFNzbMETabuTiYU2IEyEOaO0JwXG2Oef25hatJ7qulY7AmDg72/DkyVO2ux3ZOA7tnHlSGqra8Obqiie6FBOd1vT9xN3tlhcv/yPnl0u0c2xvO95/tGTatr/T2h3uA/5O55wj/g84WppnJeVekJYvvSvyHUX4+/dC5dN/HwVb2fLP4vsc4heTesdsGt/5HPn0PgpmpvY7hkVxJs7P6m8bZd51xGeETa3mQpGdkZUqq1l/y8Tg8fnAumhmVE+mqurZ0CPngsmPhDBinWWahIV+ZHgfjalaa8ZxPJlypPO5p2lquraXwtd+R1mVbM5W9H07G0AqJj9RLxZURUWaAh8/f4/aOlo0/e7AT//iL1nWDeeXl/TDQLlqoCrQ7p2Mtyw6o9KiM2V11KTyfHPks57E8Xy8d0c06emWy3X7LV3nv6zxCI5JOrK7ruXm5prt7a3ocVoQl4tlQ9u3uKKkaRr2+z1935/46EcU5TGAtO/7k5nqKJgfr2kIgWMY6bFgcMQrZbyYgmYj09BnDkXmsy/e8v0fPCPTz3jX3/69hONvueeZy/U0xpCVnTuxNTndi9z3Y3vWZzMkpWbSgxR/xTw1F7rUUUQ/FodmBEySs4ySdtKT2fVItfj/9fpbi+gHD5/5xKE3vGfhoYmkoPBe2GsKCElTGKl2xwxTEmFcIaJ6zOLATcihPGVOGJXRK+FH5+NzGUkpEGbEC0a+d5rSybme5oXhPoFYHHtpZuXGBDHJ15SWMFBjxYWutJXKmpELlTASoJUjY1L4POsC+XRJOQrCeRZuRy/O83GSb/MJ+qg4JMWQMxPgUaSkRRScxVzRHwSXkZIi5CzXyRief/SEcnGONZEpdBJIkrNYc3GghKGcUyBlL19CRIQUBU2jlKHvW9pDx/buwH7X0jQNhTWM7YGyqqiLmsIKmsBqS1ZaNkzGslovsWWBcxXZaKLboPQ1l2dLmsJS1Y5x8OI41xJMorK4sJWCGCJaC5/x4cU5zlkWiwptDX/+Fz8ixEjlLM8eNDw7L3h7fU2zXrF+HtEqU5cNVktVKEw9r3/4P3D103/D8PYr4jRgS6lmCuJExIv0jjgPR9Gc+0DRefG4n7MUfsr0KrBvxYmutaKpE8sq0495Dvs4VlnN3B4i40LcnfLzvI/03YQP4uKwpcWHwDhJ63/MEQmzlDDTEI889nmRy1KwUDFTWUNdaOoy4eY27+OYVgq0kUnAR804jx2V5oLG7/D67MWnlJWjrgvW6xUxRm4P11xf3dHUNdbOjGetWa2WLBY1xhi6vmd76Lm9vuP25obD3YEwenGix8SiKTg/3/D69RuMKxgnz+12y2HfgjKkaCQUMHjSLGgsFg3awJgmlIXFomG5asghCRdO5RkpA9v9jpu7LVrLJp4MTdNQqkIWtbltsvcRO1v2Y5ydtdrgnHR1pIRs9FGoIqOtwRUVzlUUrhK2YPTkPBJjLwKekrkkBBkXhRMURQz6HrFkM2XtKKPDYFjUS7puT105jC3Y71uqsiAXimbVsFquyBmKoiTPAvGb19fSMu4UOnFqDZNFQuOMBLmGEDgcWupKmIpFUVJXBXVdYW0pYaLmGABbYa38uXPlaSGtqnIWe2XTMk49SiWKsiBMA1kbCVacRgktNnb+2SJGY6RzIcaI4yimW0yCvtvx4uVrbq7vqOoKxZrgPf3QEUMg+AlrzemZPYqkItoKcsVaSxSgo7R3HReRqCAJ914KjIlEop8idWGxWjZ0dVHOfEkI3jONI9MkgUcSRCivvxnaI10nUtg9Ik5SCHPCPCQdCEr430qrGfuW53nqGIh55P/reVMy+zxmZ8jxJ8a5GyXGjAoym4Xw9UX04wZEaxGng5dwS6NnJlxSjMOIsbJ5qasK59yJcQ4KaySQVCszu4bk2VFzW1ZC5kkz23KPtYHjRizGOLvf9OmeHjdj4zjOn1M2bDFJ0VFl8D7jw4QrSpa2mDf+nHh22sjvJW6VI3pLNvUxSvHD+4gxFqMtP//pz1kWGxSG1fKcrhvpDu3sIE7c3t0K6qN0tOOBlBXFzHf1Xj5fUdRobXFFQUbE/77rZozHHDhtFDZpUAaFIgdPe+iECa8Vlw8fgTXYwlEoyTWJXsaINY6yVBQ6yphTjjQX3yNJuptiwBDIU8fLrz7j6uWXXL15QZp61qsFSZdc7Xp+/dUrfv5ii1pccJgiXXfg5s1XHG62PL8sOF8XLBcRazugw+xbVFmwPL/g6dPHfPnyDT7BYrEELZzjGBOuLHFVScqKcZqY/IDKAWdlbdZG8HN5bsHNWQRzKOZDjxTlyrKiaRb0/XCag8qymENsZWyV2s4b5kCKad7LlXRhxBjBFWllKFwh80P0+Glucx8PTFgUE8XM4M0zGjAEj1ZJApFNJinPMB6kC8MYnHWCORsj1apEOxlPU+hml1DCWIdzisLN86LXLBopvlRlOWsVmV27Y0rSteWnyDCMtF2LVoqi/N2Y6D4ONItSHE+z00lbEQUWi5rCFCht2O53FFVJs1xgC4fSmrp02LJk0VTstnfcHfY0qwqCxgTQhaw3ZVlIPkYKoCQnYxx7xnGSvWBWVHVJ05SkvWTZVMWCZlXiygZlCim6MJHUwGqzpK7cvIeVDre+H/FTlG4ZBbd3d1zuzpimiRgTXdfLPk4rEfKR+V+KlFIIDcEzhYjRluXqAlRBWS7oDjcMfsBPAyYXBJ/RVpOzhgRN1dAf9hKcpUQ4QpXYhQMLV0PLVCpWmw05RLYxUj/ZYLOmiwP/8E//Mc+fvoctF0zthC4T2U1QZkxV8Pr6NR988CEZRdcNaKUpS9kjp8mTgqep3DzPJ3HclhX90NPdvSSOHVeHrQi1IePT/5e1/3qybD3T/LDfZ5fZLk2ZU+cc4AA4ABroxkz3dE9wZigFgyKDoZBCF/yz9FfoRhe6V4xCCppQBIMih2O70Q4Nf0yZ9Lndcp/Txbt2VgEcSj0Ad0VFVWVWZu699rc+87zP+3s0u/2ei/WCdWsxOoHK/KM//hH/9J/9MxaLFZVRtHXh7vZrjts7KrvCKM1queSjyxUhZB52B/76L3/MD374R6Q4sX/+QsbB/TVDTizqiv2xo//iS+q24vLlM5aLNRfPFNvtjsVqgfGyl+kHKWBMacLUhsPjgaubt6xeLnn58TOmMuCdZzr0JJNwtRIMZxF0g7EV/d2Wvh9Q2krHmDVUzlOFAaUyu91O9sMYjn0HyrLdH0XIrRxxDNzePdAu1+y6I4VENx4JRkTDN2+/ojI1ddUwxYRftIQ4cehHyjFx8eyS589eMow9TeV5/fo128MeAKUMBcVyJXkzSieO+z1xCmjjGKeJw9ix8C2X60tSiNwfdqQP9hX/oY/bu443tz/jze0vudl/wePDHVNXaGwj5weVGYbIn//rd+hyI5i2kOWM5Oeut/mMMY2F7pi5fNaSyvC0H0jBMPSBygsq1FotnSlB9rXGnhyv70U4lAS8VpWVorHVtM1Tgz1ag3f2SRhKSfJBUomcXazIRJra0rSOZdPSuAdKBOVgGiNhzFKwy0nemwLDoFmtJUDQVy0FTT+MxBQlp61MKBUxVtaRnAvWeOlkzolpv8W5FucrDo8D076nsppVs8QWS2NrchD018P+QDN3vuYYaddL+v4oIZ1asT8eMM4SRyn6OG/Y7x6x1vN4v+OLL96wfbjn6vILrNU8e/GMxXLBMA1cPrvk0B24v73l9u6R9eYFv/jV39F1hT/84ffRJXB3e8Xnn7copbi/u2WxXKEoOCNt/CmMDEPPol2w6zr8oma1vhTxMYzst/dYV6GLwbkahRQ94jQyTEE4/VqK+ev1Su7LUYLmY4wi4CnhiaZciMNECJnlYsX24Zarqyu6bjvjAIXv7ipDu2q473dsDzuS03B/i54EZ7g8W3I47Ekp8bc/uUUpw8X5JW27wFWKw/EBV0l3zg++/4c89B370LMbH9ntH0gxUVUN91ECkz96+Snf++53ePvua77+6mvatUbpRFVVlJJZrTZcnF3ylz/5V5yfrwlhom2XHA5H3ry54aNXH/HixSv2v/w1pcD9/S0hBLoucnu75bjrWC1avv3ZS25uH1gsDN2w4wff/gPOVj/i5z/5NxyH8Hut3WlGPZ2wvyKIvheh34vpJ6HwPf7ivTkR5i9HlZPlb3bozl98kgafAkBR5DxrX3nGPc2mPjGE5JnoWWZbuzyB/IHx55RHdfq6UmaT25ODd34FZe5GzfJ1xroZP6vE2DarF6VkUg4426CVw8zc8JyTdKdGyUBUiRkDBMfH/fxz1dN5EQRDOY5y9mjnTophHFl5zzhOnJ2dz8LxhLDUJWfOVRVaWarKctGu6O4fqaNiGiIr3/A//g//gu/94A/4/Pvfo92siUZhvRNT5rzXNdlhkeJiKnKuyUUMPTlp0GpGm5+C0E+nzQ9Y9Gr+8+khZ7gPndSnN18bTU5B8iFL4u7uVnAuw8BiseDx7pG+H6ialt1ux+WzZ09dGafzl3Nu7pgecM49nf9OhYlTsebpLAdP59lcsmiIgFZZ0KSUGTstDPpf/+ot/+BPP8dYMSXr35KrlDrxyOcOCQVJaQkGzRpVRIPVStzqJX9wv6gT03y+LKKmz9WIJ8n9/d9VnvXnPJvkpEhKeX8un6+wzIH/fx5/7917SYqYFV0pPBqEgZ1E1Ba3kNxvfk7f1ErNLmt58iErYoZJ9teECMoKsitGGCb5Hct8AHVwwkqU+SXlVAghM04QopQeSvlArIhh5psL7iBnLW5qlHCwDVglU4qkBc8TmFLEbOhDJGT95KLWSp0Mdpw4UlJ1kU1CjOKs72eMzS7AfTZcJ82AosoFlSCckmRnx35KRZA3uZC1mg/70hb/7PKjeWBGYpqFOCUCfy4WVAU6kOM0T27CXIpxFB7wjKvY7/fc3+04HoRh6itP5T2Vt3MA1pxoXKTiBeCcpJg7K63dypi5umTBNmyWC7718Uv+8mdfivAj8yAxBChJDspGo6zhH/6DP+L/8J/9b/mjH/whV3fX/O1f/A1vb2/46MUFDze3NF5x1mg+fb6gFJjw+MWGs80Zx+0NXz2+5tPPPgNj6K6+Yry/YewPZBUxxYsbO8wFltlh/tuO9/cC+omLfrpeItCkkBlzYeyTCGytpSzgcMj0QxCqbpEKsBRoToKU3IASlCii+kk4kXYrCbKYZta+Oo25p2rwfOHmm7sUmT4zMt680dQzu1AWljmsRymskXE0JphSIcjtR/49ggcBvn77Bau1iOOYhNaGbjxy93jD/mjn1vuEdZasLtF2gzaabho4TD1d6Oimjv24Z+pGhoNs8M78imEImFCYpiOPuz2P260IUEWTi2GaxOnsvLRPKi1jspQiwZW+wlUVMQXCXFWsqxpjNImekOTrfe2xRvAw1jis8hRnSFahc5w3JwWjPMa2QMJYqbhLq5iEaRqlsc7jXS1IEmuxZg5UVImUhzkI0VCsn+c/TeWthB+2NWOYyCUzRtlc+sngbYUi45ymVRVx3gycbTYUFMoa9vsDz56fo7XFe8UpM2GxXDL2I3GYSDk9OYy0EtetbBgideVYLVuqqibPPDBh2QpOQNBYGmM9znusFRf+yX1d1/UcNJLmzgP5uGQeiNNYKfXk+pX2NWakhf7ApXpyOivcfEC+uNB88vEnPHv+jK9ffyHtZFoRplGKLvM9os1cJCvlPS+/vL93Sk7kGMDbeY0Q5EwIEnyY40RIE1kXCgmtPLbyUkRREEfpeihJAjSnMIobzfK0WZAlYkYtlTzfh0nEzhhJQTokco6CEihJ2shKEufSvIDnWbCTlrtToEzilLtRKE97VMqJA1+E6TYjyQTD8rtv1k/FZWG9m1lgPu2cNUqbuYNJxCOtJBy4lDlHRJ024O9xTSVnstZzl5bMW0W9d6VYrWYk1PAe2UL5jVbAlNKMjXFPjD3B9xhKVk/udeEeZsYxgJKCgrWOAlKgRD1d29PPj6eCmNLkJEVP7yu++dFnfPrxJxitub+7k6L8NDIO0sYcwsjZ2YZ+GDgeD/imRT9tzKXtcrlc4asK7y3KaKYonQYSLGcoOTEOvTwRMk4b8JKfUEikHAhpJBVPbZSg1GJBGbB6Dl0mM4RIGoM4WrIYAbTWVN4LQq8bOO4e6LZ35HHPy/M1xj/jfrtnKoXDOPHu/sif/+QLrnY/Y4hiRiBOtKZQVCSpmqAUtc9YWyAeub1/R1SZ9eY5L5Pi6vZRBpLWuLpBU9DeSeFfzQeWIPgzg50PMCNqdjhCwftqRo2k+b2PT/f23d0tfT+y2+1o25a2bQSVE98fik7BRtpYcfeoTLuoSHGcnYUR1zYo9MznnfDFUlKm7xLj1OEqK0XQOXjdFUcxmhgyMQdysVTeYqyWcHKdaVfi4Ll9vGXKExhoV40EVWcZz9pAQYKvmlaC+Lx3OCfdMlXdEmJku3sgRpnTjNW4GmIciL+NDvwPfBhXSASKKtLZUXvMKOG0ymq081R1JfOhKnhvSTmQIiTrsV66omIBZS1DmBj2A/uukOKA0oWiA12/lc1GcoLCmzNLYsxzZ5TMldZZpi6xOXuOdon9occ6R93WrNszau2oXUNMI11/nFu0I+OQqPySal2hK0Xf9by+foe1hm4M5JIYwyis8JQJY0SjGftROqp8hbOO2oPKhnGMbPcHApE+9liveLG+YNUs+cVPf0nIkaquOXYdy6VmedZio2JVLckmMdmA8hV11WDWhrNXz2lXCwn2O/aYKWGnxLdf/AEvVs8JRfH4+Mh23/Pu+oHjGOnGibe/es3Hn37C+dkaox0fvXjO84szzlct3XE/hytrSo4S/Jwij3f3fP3VV3T9Hp0njg+3EAesc6w2l/RTJo8j//Zf/yWXz1/iqwWvPvkM0oDTLSUOkn8y9Kway+rb32AaNTErQp6kIyIXqvM1L5495/r+mk8//ZjHhztevHjJcf/A/dVbhr5js1xQVZakC/Hmht2+RxXDkDR3V9eMaeD64YbjeMTXHnRgsa5pNw3P1hd06YCvai7PX+Kd4+F6z9QfaS+W+GUFxtEdJzmXJI3Hko2Ehu12O1arBY2vOB625KxJuTCGTJ4dbyln+kPPw8M9i2rJcHikrhqcq8ll4tAd0CXQNkuGXce6LqgMx6GjVhP7rmcYM7e3N+yPHevVksp71s/OaL7T8ubmmiFE4hTJqjBMA1Vref7skuubK/pjz/n5JdY7nHHkSRxi3nmG2GF+j/DB/8v/7f/MH//jH3FMO15//RVvXr+TMwAHrNWkYim5kAKEQaGxOB1wBsY0Yy+LrEnGFoYhcjxqFkuP0pmSFHe3R8KkUCU+5VIYk4m54CsRSQGU0oxDml2dQXI/SqbrJjn3KQhToqobQoyzOCdirLMz4lFnHvd7zpcNz18857jv6A5HrDKM3YRfOHxtoRbMaUwJbeZuQ21YLJeE2NP3I0N/5OLyEldVdMcDThV87QlxYBwnqsrhXcXHH19yGG7IpTCMCU2PKYLW+u53voN3FbWt0GhWiwVpSCwvzjkcd3jnGMJAf9yx2x+ompaYChfnZ+J2TBGlCtM0kHPm66/f8PWX7/jpT7+g3+/Z/KPPaRqP0xKUftjvaduappHu3EOf+Orrd4RJ8dd/83M+evkxf/Hv/iWbVc2ibdicr4l9mIsZFdMwkMIEKlJbS7c7oIvmuB+wTuGaNRtd2N6/pev2VDhyKz3OM+78CcmXYpjDZnnCZDhrKTljKj9reRarLfv9/ulsvV6t+Dol3rx9S9XIvC8ZOJl+6uimnrvdA7pu5OxialDw+PhAzoWq8kzTiHOeYehYLFv6fk+ME027wNU14xh4fv6SL/7mx+ynHcdwx2LR0PeB5bKhqVaUUnj9+i27/WHOGRLT2jSNWOV5d3XD59/+nB/96Ifc3t7Sp1E6VHNisWgpGa7e3pISHA9HtAu0i4amHiklUErF29c7Fo0YKOtWc/niOX/3dz/jD7675vLVp/zqzZvfZ+kmlVnfetIFABQq/ZazG4X5UAdUJ0VKzhknReFkEhRA8PsQz5LLe2NQmTtstUIVJYXkJwFxFtDLbN4k8/TN5+5DddqvzG5q6VY9/X1+RuUJWCKCJ4LiUAUSBYp5v69VBms92jqMUbRtg7UtzKbDIXSzvnHq7B0xun7iep90lTCFJ2f1qUP2FPQ5jiM5Jdq2ZbVagRL2t55NPyLUO6z36GJQU8ZlRdp13H79hm5/wPqKH/7gh1B7Rgpni0YQNN5i52BKyY6IZGvR1qGUJuZEJpGUQmeNyidD7uygP2lDlA/ez98W0D8UhHn694kvXkpGazVndhke7u+fBGYxplQ8Pj5S1Q339/ezYa5+cpyfzmg5S+e09/7JmX4S0Y0xv+FAP53lpINbfr5zCjOfo3PWNPWaMRTevLunHyLnTSPZafmDDgol4+wUHaJ0mY1wae5gmI3QvB97p9rOqUAj53VF0ScyhX4qAr7HHeWnP7M6jUuEcPTBuV9E5fka/7ba/+95/L1FdBULtsgXZBSjEmF9jApyeRI6pvlg7jSEAiEJykSXwhBBWYXKCj0Woi6QBIMyRMUQFalorIbagdOaE0yD2aE2hMwQZYOgFSRVyEWCNKdYmKJhShO5KGLWhCS4FdG/Z97OPD+UchI6CmMQLnpBzplZyWs9RVTONTdhltoiHOpZSO+Spi+F6ai5yZo3vdyAPsnvULQEojIHquaC1YJ7iSkTsiJrLQm6KTCliNFBbowZJZFikXDDYihlJKpOcBJGigtlnmRSiUwpcNh3HPY9SmnquqbyDVVdY4zCe4t14ho/OfuMthhXz5wwCbZTyqDsgmwcsdP4quYffv8z/u//7385YyVm/EsWV1CaB+Ry6fjO55f88Y8+Jxy3/Oonf8O/+Yu/IKbI2G05XzuW3nHWymh69eoFP/qj7/Hx849plxvim9cc768JLy9YLy9Yn3/MA5mi5GalZGLIEkRbTs7ckyP9vZB9aqQR0fxU2S04DVLILJRUsEqhbaFpNMpYnAvi8C4fsOtjEeRQkok8k1C6oK3CVYZm6amypWrELSnYA5kOldIYPQcf6BPWRybG02SZZvdbKVkKPyWL6zOL0z0VhTkFjWbFEGBMcpBIWZBBv89jezxQDMQS8K0TF52ROWqYBlSQidpGQ330WK/RRjGNkTEGkirU7bywrSO6QO0rKlNhlCVMkcNxx+HQ42wtgvLs7LFaNuBV7WhaLy1+aSJpjfMeYz2gibkQYp5xCRXWSweF9f4JHyItQVIkAtBKGNBWO0FazC1pSjusdRgLuQTClCSMZOaqiRg8V+mLuMO0LnOlPRHiALmQc0VTL6icOO1iAGOl2jyGCaWVpNB7e8pKp21qDscehRYXOiLujlFckc5VHA89i8ViDuFsMdpyVIr9KGLqk2Nh5rxNk7gRl4uGxaIV9IqSNrUnd/RcVdXazT+nxvsa5zwpJcZxwFpxL41Dj1suqLxn6PvZTSq8+hQjJWWsNpJ8nU9VoEIMAWct3oo7Xq6hcPbCzLKOczhVjBFVMtYYdF3PKdoVQ9cxjBGl9dPmQAKjBWVRjJ47bwIaN3cMzYziCCmM4obzBuvmwNqYCFJ1YBxHpjFTNwbzATLmN3MwTrw6UPMm5SQqpzSL2mlGvMzrk5rn9A+DRGXxZL52RTIolHRrnTo5ZSaYq+klP+2VTpur9EH1/3d5KCTTIZUgIm+Zee3kuTBnKEpJC7hWjMNAznEuHokoetqwlZmvX7KsaFrPmytr5jEm3V3iTp7eOwXmMSjjUj3x+GKMbDYb6rpmu90yTRNt00CC7W735Eyu5k2dFC4K9Tyuu2GYnasa783slJdXbY2jqdsnVFPjGz796GM2y7UgtGIkhsAwHEkpsz8kFoslU5gYp/H95l7rGb0k90bdNNR1TSFJ238+dRgomqaWXIZBPjYNEkQp05EEt3XHPcv1ilEXKRzOmSJzZg4lZcYwQYqoXMSpnLIEuzqLMY40wfA4YHLk+dmaRk14a7jZdewOI9YvCBRce4Zv1gz3t0SsiBEqkbSW0NGg0X1ijJG6mouE9zco5zBVS1N7Fm3DruuI04SrG5Q1tE1LvWgYpo6ipMPPGCNBjnOhK8Y4u4gTTaP4kIcfwvTkJgohPuUunBwj4g4SB7QUXWQT56qKlAK5BGGBHxJKSWHHWGnP1LqgnAYi2kigWyqK1apmsbTcPdwDiqb1rNr2aV+pNcQcKHMLedVIR89UHCEFrNcM08RytaAkEdHVfIJMUX5W3ch86U3DOPU46wnJsViuyLlwc3PLsZ9YrZasNi3jCLv99ne+t0GYmxRYLheAdBcuVy05R4Ip8l4rhW8qUorU3qK1OI9KinTdEWUstqqpVw1T7OiGHfvdI1PoefbsjG7cc/eoGavEorrAOdmXhpBISdbFnArTFKiqmsklmnbDsXug248YYzlbPGPZeB6jdJkOw4F+OFLyjEoYRnH2V4WsI7a2dGEgdpFmtcI5Tz8dSbnHFIPTnkWzpBx7MgbravI4oABfN6SUub275WF7y3HY8XJ9zvnZkkW1oF3U7B87Xr78BGMdu/0WbWD9cs3LixfoWtPnjqEbeLl+iTOO4sTd69c158/OeHj9jrE/cpgCd6XCas/Ddsft3QN3DzummNnuD3z2rU/4/LvfYbWo+eybn/Otb32bi/WSsdtROScHPcRl13dH7m5vePvma3JMlNizu31HOD4S+j3OWY73tzRnzxk7wRpsHx949syjVWbYP0KOrNdnFN1g9IhGsiNssqisSLFnVWmCNQyhcJw6vv2NT7i+uedis+buxrI5O+Pu5kqcdEqhrcXMmQT9JMHAQxk5pC3JTVx+vqCOmc3lhqvrK7bDI/v+kZgDy0VLCpHt7ZaXzy8kk6jxWL/ANo6xQNiPFK25bBc0znO7v2N/2JIIjH0neTQ5sdmcczx0GFexdC05K7qhp+97+uOOwWdeXL5inDKLZQPOsh/3wrgPGYaML6PkJ6hACIXd4ZGqecb55TOc1nSHjofhjhQiH796xatXn9CPE9vtnsPhwDAOsE28/OgZxgrSKs2t585acslMw0BRhmOZqNTid763v7r9BV/8P38JShODmK2sKSgdcUo/CQRJZ7RXeJNJYXahajGPFTLouVifNcfHTJ4Ui9YzDCNnyyX7PFDKiHNKHKi+UOZ1KQRwDlRRpAAqRBonKMyqsaQS0UbhrZ73AmAKBAKYCl9XDOOAtYV+gEDhOAxcPltw3tbsdgfqxhF3maEvLNaaHBNx1OToGfrEculYLDzv3t2zWokIPE6Jd2+vefXJBlLFGA3KJIrJVK3BGs+rZ99h+9Chc0fKg2RFKIXJBbSjcZ71ci3vWZiIpkIpxW6/J5MI3cgwHgUficFTkVMgDoU4t6CHaWK33/PFF1/y//nv/wV1fcbVu0cWjWd9ccHZ2ZrLFx9hrKFdblitVhTAuyXPnn3KX5W/5e/+8u/46svX/DfHPZ++WuOV4f7uLW1rqeuG26t3vPr4U3Ic6Q4P1OsFKSfGsUMrsM5iTIWyHmMd9TiRi0Fp2d9qI2PCWId3TjJlQpg1ACvnUSNnTpvdLJpmSpGMD2MkNyinIExnYwTZgMYojbeG1WpJzAFvDMu25XG/RWknhdd+izKZzWbDw+OOupbg9e3jHYftA9ZmfNOQaVm0K6qY+Kr/mm5/4OyjC/Q20h8nMB5jKj569Q1+/ctfsGwWdPst3/jWKzaXlpvbG0KSDoScIj/+m7/gBz/8Ft/5zh/wl3/518Qxsly2hDHy1Rdv+Ow736IfDkCHVoah13i/YAqJxWrJ/vGR7WFL3wfa5Rlf/vqKt68jh7u/4p/8sx/y0fOPfq+1O30gnj/9eXKLf/BbP2kY8tYI0nIWVk/6X3nvty3zx98HDIvOJeqjIFZ0+S1chTrZ0eG0B3svPIoCroqef+780fnvyv77HLvlKSPv9G1VEdFVa01BzmanbuU4C7bO+hkD6qjrzPG4k2IVau5ETkzThHNSOE9JzQ768hvCrzEW5k7Jx/sHvvXtb7FarWbxXTrqtNaS7YVCFzBZ4TAc7h8IG40rcP3mLaqAaxP1+YZ2veHFJ59IgVIrjLOzsaw8ve4UIkEFjHNYY2a9B055FOrkyc3vjcDzFZtVoTyrVfO1nz/7/u+zSUjP83KRs+V2u6Wqqtld7rm/v2d9tuGXv/glbbtkmMLTefe0Ny+lcDweZ23TPuFwTrz00/U8ff5DMV3G6yzWW3BuDjBWmhIVCkvdrJmmHY+PR87PzzgZlJ9ek5rJISnOY3vW3VJGKSFl5FzQRQkDGSDxdN2UkjUrxQw6vz9Dz5qfPiVoPzn5tfy/2QV8OpPD/HOf/KgnXvr/78ffX0RHY5hfSIEpS5hISEVE9BOTVmv6WTg/IV3IIko7pZ+E62I0lU7i4ktQkHZlZRTWzoydrIjpfSUs5cxwwrkkMB4GoI9HfPAMY+Q4Zg6TIhfFEDRDhCHJRjAmPhCz55b8nMhJKt45F6kYa3EqyI2eZwez/mDwK3H4GChWEZXmJipu+sJd0BxCoVaFOmrqrJjyzOkp0vpwmstyKYRSiMVQlKZpGxbLFaXo2akljnBx9hlKngUCXWF0QzEFkxJBKbS2KB1lgpzbNIwWrltd1yyXEhYozPKMURljmVnPFmUcbg6mss4yhZFpHKFAVRWKX0Hq+U/+yY9Y/F//ObuuE7dkiB+MkUJTa149twz3v+Bnv/gxF+ef0O0fmPot/dBxsTTURmPnKubDNvBZEWTH2cUlvlqRLgZyiHhfQxlRzoB1cyVUMBYhZGJ8P+k/cctnOQp1YtzPz+xp/ShUVlNZhdKzA98qjFNYp/BoKn8Kb3x/P0r4UJ6FL41SEpjmrBGHm67RBXzl5glnJBc9F3DV7Fg/iWrqaWkCGRNGFbSawxN0mREP8z0w/1xjZMLqp8RhkgLR+1DS309Ez9SEbOjGkfvdA2ebDVlptK/pDj1aKWJKeJU5jAMcLApFdxwok0ElabE3jUUvClVtaFqHN0tyUFy/uyGSaNoGp2bHThLoTm0alFZUbSWYDwPDKCF1vnJY46RQFIskQBsFOVM7Te0bUm6Em6iFw6WUVIOnNMkhvAQo0gKeQyQrQ1KJZMF5BQRymiBHjMo4XbBKS7B9Skyhk/BIZUB7EYTzSCkjxiS0azGuRlMTEe5r0Z6sEtbVpNzjrSdFEfC1cdQ1jGOk8rKJHaeA1pmqbjjsO4y1VFWL1nDxbM1w6PAU+u2BaUrkoolkooJUIl5rlu2S9WpF09ZkEr6pUXYOd9WzW6RoFE7cSbNLXMT1zDQG4hSwRpiapMJwHERYHHoMmu3ukX6/p61rNps1VVEc7h44HDuU9SxWG7RfzOuECPfGWKw2NHVFu2ioakeIE1orrLakcSAHuf5KAzrN7HERZsW16QSXQYRi5rbTGRvjZqb2GKTTQEtBalE1ktFREnEcyVoxERnCJM6LHMhhlEBUJUWHTIYi83/JCW0VhUgqIhwUlcllIqeJXCbQCYwhz4tynsVuPQeiiqgu97aa8w+ssiSEyZzJqKJR2aBndryK888nEomM00T8fZjoeZqxK5lQkrxe7aFMaJWpvAFVs17WlDJy2G+heJxpJXDUe8k5yJGz9RnH45FsDcZLKCNzO2LOibqpcN6yMAv2u93M8Q5Ya1gsVpxCgdbrJW3bcjgc0FrTtvXcbtuglIjVochm7mwjoT/jOMocNAfknkRYimYYR6zTggPKibpuSSHz7OycHArPn73ie9/6Pp+/+i7jcSTGiFOZx+MDcTpIoHMB7TTV1DJOEJPGuxaFYRhGFos152cbrIGS5w1oSnSHI4fDnspbRqVQJEqaUCWgVCTFkRjCjIlQwgTPSYKJS2GMPdPYMw6BkAo5K6YQMdpycXZBCiOqFLy1GBKh6+gPW/leVc1wNBRV8dh1fPn2isdjT3d35MurLQ/byPPLC97dbemnREw9hYRxjmILQx7QUcZjzBmjhTs7vf6aMSZWm3OaRpPQ3D0cmUaDKZYUNN5U+GXFdj9QeSeMcQrKWuIQf4OneNqwg5q7MiDGyDB01LVgys4v1nNxJ4qbhYKxMm61ke+lTaSozDiM5Oypm5a+B+clzHCaBqrWS7AUEd8qyYHIDWcXG2KaOPZ7ctFsNgt8JUGcpROcWFETY0yEpLC1YoxZBHTnWG3WeO/ZPuyoqwXjMEho6TQxKXFMppAYwzCHSSn6MFCsZWlWtIua1dTy+vUDdgRlZgf879lFZq1CKQnQDZMUA4zREsye5HAwpYCGeS9bODtbE0Jg0Zyz2w8UBNNWNYYxOHIeuLvtOBx3tI1Hm8x2d8doA2rj8a7CGCNrRgSKmREKmdV6yXKtKUrR9yPO1rTNCqs0w7Hj4faWUCLWSXffOAWWiwXaBGKcyGoCItppMonb+ztevvqEVbMkpMA4JXSR9bhkWT9DSMSQSYME4LVLGYcPuwfut/dok+n6PbVNDIeOxbJBY1htlpxdnPOLX/2SLh5RVcYsHS8+fsH14xUhRa4frnn14iUpZ3GRGUU3dUQmtFdUumJ76Hj3+pfc3z9wOByZpkiMmc++/S2+/91v89HHH/Hpx5/x0ctP2axaxv6Ar8Ulp52npMjxsOP67WtSmFg1FX3XUULiYtHQxyOHIfPu9Vd8/E3Hu9cD2y6yvHzBH/3oD0kx8u71F2htODs7J11cEhZLztY1cdwKh74vuLrFVRZVNA6P8R5tLFN/4OWzc4a+482br1mfrXn24jnvvv6a+/sHhr7i7GKDSempaNanA9mPbOM1Z+2SKewZK01vD3SqQxU5X9h6TQyJoRswKZEUtKs12rcU4PH+ARsN52fnGK2F0TsXjrVRdIcDcRpxVc3ZxrJcbKh9QwiJu7tHhm7CGMfz5y8IfaL2NZWrMV6R9Yh1NY0qDMOEx/Lw+IDTirNnG2IYOR735NJQecl7iHMQ+PXVNYtmweJiw7HrCCni64qLywvGqWO721I3Deu1FOe10XjnODtbU4ZMGgNXNzd8oDn9hz/MjHAoGsjklBjGTL1wM9pK5kXnFVFlMgHfSFZTTnL4Nka+1hpDTOJCjWNmN3ZiELI9VY1gCRFTmwaqVlEiaAOuskxTwCiD0wbtNcYmnLdY7+gPieMhst5I9kJWhXbhCHMgYNs6Pv30OT/72R0xZHb7CeMtD/f3DGNk1VbCZ9fSgda0npwc4wDPLy8Zw1E6lZoa5syUpgHIHPY9q9U5t9cDn330gq/e/YK28eQMf/7v/pbPv/UHrFdn2EoKJtMw0PiGyhsWCwkwvr29Zf2NlQhIUUQs5wzjNGKtZbne0I+BQsE7w9Dt6LaPVM1CNI0UKDHQH3uu3x2IIfHdz/+Ej7/zPT569Yo8B65XboWRJ45KIylG/vAP/4h1u+K//a//K8ZuhzNL1suaymke72+p2wUxZMI0sl6fk9JIKQtyKSidabyXIsWiJWNIMeOrFf0wzmaVhCFhfUXMmePxOK/FEvKqVGYYe5RSeFtjlYjo4ygMalSmkObwdhFXxylQ1Q1KFbSOWGVoq5qYDd/86GPUzS1//cXXFBzNeYuvW6xVrM7OxAmfk4QAG4sqmdWyYkgRlKbvRyrrefXsBc8vz9lNPZvVM/aHO1KUotAUFV0/8snLl7x8foku0O17do8dzWIN2lA3Gu0Mf/lXf8cnH0uGBEr2HsPQcXGx5uH+kefPLpniDmU0d7dHrKtZLMQksT5bcNgdubhcgTK0zYr/9D/5IY/3gX/1P/0N3//B89/j5j49TiGNsyB96mL9oMs+KwXz2VbPppQTzvjExBaj6GwWRbK63ssCsyN9NuWIOUljjcIU8/6LZ2jgyVV+em55zuCT/6Fmw8x7reXUkfwk/D7RAOQMnbOcj4zWs5FVo5IgNKzSTDFiWzF8amMxxs9ru5J91STdFOApOeCMnOerqmacBN05HgM5Q9u0jENg2azYPWx5fHPL83bDs5WcXbCGMUy8+OgVKSRMUTjtsFFxMSroe9793Zc8vOwI04h2htfX7wiP0Hbn/Oknr2TezBnjPMN8L7jKkZFuSYwlMqMNtZYOdyMmH+ZrJCbN/FTIeAq2LCf75wk1cgoiff9WiuYlNI0YBygSFJxjZLXcSPefDdTNkqura5p2xf3DAy9evmS/388FNPUURj/OWNOTKB1jFBPlLDRb49EizKK0QRl5HUYQDHK2VQWtPSUL2rDM50I9ZwJev7ni8289J5d+7sCYxfTZ0Jzn7EqSomBQukbrBFlMezm/14JP+ph6uhZl1qd50vpO6+bJQDszdKQo8DT3ian6/T34/vqKueF/TRHdg8pKnNcRppAlMyvOGadzCcyXLIfDeRCEWdnPKA5TIWaJnUwailbMeYGcjIDKaYwvTLmw6wI2Bcao8DYznVAuCZKFXMNoYMyJYZjoB8V+gG2vSMXQT4pDUHQxkSiEZMgJYpjEURgicUpMIRKCPAEt5SG0Nk8e9KegUhQJmCgccmabgKTYFsd1KFyVxEMs4lpW8Bg0bRFXcppdjCf2FOUk8Eo1Usycmevraw7dkfVi/pmnquSs4xulydmjTYMpEasT1iSSlbBNozWVczRNS95YrHW07YLV6oyc5WCek4QrAhgVkHCgQMoBjXzOGMUmrFEKnHUi6tgF3/nsBd/71nP+7d9+IVWj0yCeCwztQvH5typeXYzcXf9L7l+v2bSOF5tMWYlbpkRxpraVouSRVAJN01BKQlvL11fX/Pwv/y2vPr7Atc9pLz6mOv+U/vBACg/S0jkzcGWiznNwq5od8rN4NRcsnlzfWW4cqwp2Dg7V8NQR8l7wPi0ezIUTETtCzFIIQXjMwg7UGK1xVtzE2mgKam5FlOqvoI7eC/kpn1AzPBVpDBIoWlcabzLevkdixPSBa7Uopljoo2QQmBkLpE7cod/xcao0pjLR9z3eVyg8zjvqupDiRMwTeW6TEkyDJSWYjiNhkgKO9w7vLE3tWC4anGsIsXDJBc2iZuoCw2EUwTMIzx2lqJqaZtliZiePArIF1wqGJIThiSttjLgpFnWL0oYQ5R42xklbdIF9ODJOQa6tnS38CQktjoWQR3Jixmuc3NCyOXfaUmImjlHYYn6gMg5vrFTPtZFQ2DyPOW2wtkKpipJ6KSACKC34m+KxM8pBa4etnPBxY8EbYY8ZrzBBRMDb2zvhuMXM5bNLpqlnpzS6GLS+QXHi2kFKE4XMYr1kuVxwdr6RMCBbUde1bLqMfmq9OgV35JyZxoGYIo1WhCys7xgjU8ks2ob+eKA/HhmHkfPNii4l9rs9cRhZL5ekkLi+ugalcFUjwlwpImj6gDMR1DQvpprj8cDt7S3X19eMkiQHSVziZV4gyxxMmGe+ebZmdkxIWKMxsoErar7vi2QOKGTB11pjXYWdRaQyu6hjKsRpJM0OcIMUGXNMGGfASsidypBjIqvZLTFvdE4dSeWUOp1F/H76VSSU5NSahj6FwhThuRWN0Q5jKtkcnzZB5QMXfJYCpyBoZB2apkAYp/lA/Ls9qto+MVzHcSDFgnPQVDXuouFwqOiHPev1AmsaQjhQSkCpxGLRcDwesM6ilbiAl8tWRA4yU5JsDO8t05QwTjpUtC5Yr2lci3ceUDMzWkIJ67pms9mQc+JwOND33TyfOlIKVLVFm7W0YRLnUOKCn7tOTnNB2zQoHMvlkt3+UVq4fY01jjBlwjhQ25Z1tWDTrjFFCn4GTciZMPaE0DFMiSFk0J6m3WCrmotmQdMsgDK3gS6w1qDnVrYyj1GNHAQUZQ7e1TgjAWreGbIxTONEjpMU08PIYfuIKZFeRcZuxzjs0UDXDWx3HRcXL/nmN7/DNHWEfk/bVExjZvt4pISRpnLENPDrr37FfvfIbvvA1dUVb66u2R5HHg8Tb24O7LrCkD2mTPLbiFHBuSLBzaUwpURBiqRGZ+piSaXn4fEO5QRfU9WSAZJSwClNThPH4yN12+C8pmo8KSdijthkyUH4jNZqtHEUhDl/crPIpjmRUuDYSdt8XYsLcJwmwjRR5vZhrTV1Iw4Y4wz3DzsK0C5qdrs9Sss8e+r88Y0jlkjlHc9eXtA0NSnJ83m4usZYuNicoxTCWk2Ri4sLoEgQNNLe/vh4R1VVAAyHDkg09QVnZwv22z3b7ZbFQsZEHiNGKawzoAqH7kBVVdR1Q0gju8MDx6sjSimevbiQYC1TOOz2v1eXCYgg0DQLjt0OZysKit1+Kzk0ClIcpPA4M/gOwwE/SmFDqwWVb9HK0VYtSkVyPqBTxISEi5bpkLFNIrsjcTji7YK2fg4Zmqqhz4GUoO8F6VW3hmahedi+Y3c4cn72nNVmwX64oxse2HePdL3c0wrDNGmMSvjKEuMR+oCzmrr2xDihUqCEgcYZoltgVC0htjES0NiqIiGB1FkZgoIuDZRJ0U1HvK3wyuKSYrcT0a5uljQXS0yleHZ+Tkqf8PB4xRQjQxpJOHKscaal45FRHSnKME2WJsPxuAU3sR1vebuLPFeXHMY9KQ28uNjQ1A1VVfH85Ue8enHJy5cvKCUSwoHHxwGNZhoc+/2R568+ZXt3xcPVa1K3k3UqJeIworselwaG0GPnQs2uPxL0gqvrO/7sz/5jfvW3P+HHf/kXlBKpvOGP/uD7xPNzqnbFW2cxVlB5F89fsth4xiHO5oyeokYMGlscaVBUvqJdX5DUAExUznB/fcvdzS2HY8/li0uqRcNx6tj2j+jVQL0B/MTQT7ix0DiPXWaqxtOPA7Yy7O5G+m1kDDvOXi6oXOG4fYTsSEcR6PSzxG33yKE/EFXGNTXWKLq+F1dtMjAUztol62bN/fGBjao5e7YCC8VObMsORaHyjq7viWTGA+haUek56DcP7PqRy4+fs9tuMdGgYmB/OLBcrkBr1psLwQFULfvtyOGx43G/pW0rXGWxbkFRiqZuSYNwYFWtRNAwC9bnC2I34Xeaftj/zve20bJNiiHOrk0wlRh9Yo5S8EeLA1BrDocIWsJ6lSlMU5z3zCdsgYgR1kpxzWgp3Oc5V8nYOfupCMO87yasV4QwMYWMLpr9caRuFeRETgVfOzoiISJ7KS1oVq0rUNKBZ+YMIzWfq7TSWNMwDpEwBdxmwYuPNmw2C67u3rCpWrbbwIsXz/nJ373Busj5mThWt48dl5fn9N2BFy9k3m7rmmkSQ5fVDUM/sagg5o7H/S9pjKUxFSkNFAK5GGKyWKd59+4t3/vkDwhhQqOoXM2iXTJOPU2zYLN4wc39PU2zEI9oCuz3D1y9+5pPXn3KYrkhOI0i8o//7B9ydfWIsy3/p//yv+R7/+CHDINkNtTWg5KOWe8coZfCO6Vw+fIVYxyZ+h1aR5wTNKqrJBvJVxVtu5DOVuvm7uiIVlq6x6MhhkAiQkl4a4k2EUNA6QljPeM4yJlkRvpZI10EOaXZICuu+lgShcR+v2MYjzOezBJjYOiPXF+/w3s3d4hGnJtRbSFxtlqRChymxPn5kS5OdHmkZ2JdL7jbPvLs4gKdM/eHe370R3/INPQ8PtygvcfV9Yx8kzD6u/s7mpcrvnrzGlcveX55yeXlC3EcY9gdEyFVLL2naiqa5ojVDWNfgERIEzEO/OrXP+Wzb37K9nFLCCPnFxsoil//+pbPv/9NTIi8eXtNXS24uz/w2XeeM00DL1++ou9GXry45OrqLblk3nz9r7h6m/jhD8+f8nx+54c6qUzqvcmYWQ/7wBhXKDPmuQhTW713Lv/PDHQnXeMkyp8Y5rMgevI6zzLUHH45K5OzoPhhKGmeu1YpZYbERFR+j/U43dPvXabz85356mkeT0oV8kkwLhZNwegK4xzG2RkfKghX72tSzGgzzLQEwT9qjASRD/2TOO9cxbHb44zj4uyS+5s7Xj1/xc27G26/viYdAy9efcrbL75mdXlGiJGzi3O6/ZHNco3Kmmnfo/GYNHH/1Ttuv/ya6binWMVYEmcvnrE8P+fs+TPOnl3MBSwxXommIwUaX1XvRWatnhzoqpS5c1H0IKVmrKrSov/NYjunoog+dR3IPP0bg4P3Y6NkMchlrdjeP/K9732fv/jzv+Dly1f8+Z//ORcXF0xTpGkatDHc399zfn7+xEA/ieanPeiJgy5jsMyfh/cudNEUtTZPDnWSwmpEvzMGheTMWSPrlbWSEfjwsGMcA05HMmEe/mY+J+sPzCTz61VS/H3CND95UOe15Emrg5Izeh7Zpzsqn65fOYnl74tCqPcudfXBmP3wHgI+QMH8Lz/+3iK6N9I2P4VIiBkdFDmIQKBPTlsKwShCUbis0UqCQcjiTKdAFzNGK5KGrAxeZ6yGPiumUsi2QF14iImvdgodInmaJ48SOAxwQDHWBdtAdBJKOiRF3xX2g2Y7KVJSdKFwiIUhi+gyRJimwjSHB8UxkIIErkzBEKLgVeQayxuU5okj5ATZEFRhLHA9OMrOMAX4asxcx8I2y+srM9T+ccoMpWDj7NRPhZiFA5Wy/F0riEBMkcO+583Xbxj6nmWr37chKI0yTtyxOZMxZCwlO7QJmJKwSSrCTluyKazaFRapzDVNK4GBM7vdaOFGhWkiKqn+5tJTiuKgHYt6Q1VVTDHgvQcMSdVEJQEm/9l//sd8dbPj6uruabL2tWaxclxeaD7+tOZyY3BmYn/1luM+8+1PPGnSdMPEOGUOXcA6z+XFGWfrFYfdkcvnE2jNl9c3/Hf/43/Pf/y/+TPa5RnTzEt+GuolCa6naHGPlUIsipDlf1UZyuxET7lQitxewn3I6CIBlGp29pci6dElQ46ZGAohnUJEZaN5KiBNIZGyYFz0XFVFy1inFPm5MTPNokzKJ1e5OAwTs3aYi7DO59eEUlhd8DZLYcSCUTP7M2ZyEZRMSDDGU+Elze1V5Wnx+l0fzglaJ4aIVsIXaxrBg3jjGAeF0pGUw7x4zQgDpJKXw8xKDhmdC4u6gqhIJqKsYnOxZLlq6A4D+8c93b5j6jNqksC5xarB+5lznBVZg3GWyjthZBeDKZrG1KxWCxbNgsrUs1BdyKGjpIIzEnDhjBNRUmu0N6isxalTFKSJGIK8Vm2pGjejaCS0wlkvPOqYmcaJcZhILpK9bDKUtmgrzpmMBeXAiDCUsoSWpiT4k9Na4CuPJFcXsKCdtI0JZ7ngtMVGTwiZvheUi1KK5XLNMMzXOBZBFpg5IT2Lkl/XnuWipW6ENWhdRV1XeO9xzuKslTF8qhyXTIqBkhKH3cNTaFFJkd32gbrypDCQUqBpl8SYuL+5paoci2ZBNG52jAyApm5a6rqlWa5pFkuaxRLfLHD1UgS0uRCEUiyXSy4vL6h8RZgKYi35sAIvGI6kixTKYpwXceHlWytVbuustHDlSIjCl0tZMBUoC9rgtSVTiFNgGuWwYCpD7SucEk6l1QZjHRpDVgmycPaiCSgtjm2jLCVLYViWe9BFY2emvykGUxQqyca96CLV+SdhFcCgdIVzs5jIXATIEnQoJJc5TXzujEwhEstEieK8/F0f1jmKgnbRsD5bMQ6BaVLUrWPRthhfYNdjfMFZQUHEGDFOs1oviWlGbyhBgdV1zdu371DGUDW1rCUhzEWHTEpgneckSq5WK6y1TOMkh7GkhXtXInVT0fUHQhxpmxZgxrOcQiJPifeSTSDrEU9hOMzjRRvFoVPUdU3TeGJIRFUY+p562dIPA/f395zVlzMHNBP7KAHQ48RufyQVi3MD4zjybH1G0y6fGIFt2z6F8QBPmzhrLYtFi7EKkA2tm4vY09ijdcZocV4NUToKhjKQ8w0xHMlx4Prd10z9nuWipesG2sWaF8/PUDqye3jAlIFDzHTdgcN+hzWK2lu2D/fs+3uSCtzvb/jq7RfcPOzYHkbutwO328AwaYprWa1rzCCuwqKyuMrkBPU+O2Qu+KQixYEpTvTDQNO2WFfhGy8t3aWAFjfqodvjvME5LfdyluJndgqNYDqqqmKaRvq+I2VxmRtjBeekBY9UAGP9vAYIskdrhZoDE5USV6rzjrat2O8OdP2eUiJnZ2vBEI3imgOoKgnJdk4zTr3gRrIYBJq2RhtFjBN17TgeJwnOnDsbYox0XccwDIQgBf6UpMD4+PjIol3MDNkJrVsg03V7rJVAppQmpimQkoj62jgeHu7RWjEMA3VdI5t5j/eWvv/dmckAXX/AOc9h37PZXGCt5eb6ln3jpQhULEbLmrZYtlSVpRs6uuORw33h+eUnvPzo47krM9B3R4b+SF07CVwukruRY8GQOXR35FRISdiZKSmO3Qgps1qvmKYJYzQPj7eULMWRcRzY7bccukfJ70mZw3ZEa4dCUxrwpiJOgTRBWzU448lxoKkrxn6ErObuqTkPJcnakHNmt9vT9z3LVli5J6flarEk6IllveRs7XnYSX7GEBJV69gfjzy/+IicCuMQUdaD1ry7eict43M2TSoTSlmGseNw3MucqKS71NqK89Wal+sNZ+2K8dCxWq7IObPcrHn+4jkxTKw2LVN/AFfR1gtqZzmQ6botfbfj+cWKQ9zz5Ze/5u7qihgiXntx3aWeYdhjtWLoB0rd4HzFF19+wfXNDY+P9zireLw5EPePOAWffvoN1ucb1mdr/tE//lOqtuH6zVtCkqPmKbzb+hrlG3SVeP7sHPvzgneOzfqMNEzU1qLIXF295fWbrzh//pz74xa/1PiccKYQpp6xP6KpaZ3HqIJzVkwiOVB5hV3VFCNZEbuHOw7bxPnqBY11DNPIzfU7dt2eogu6smhrwFm09mQKVnnyBP6sJWdNN4z4pqbdbMBCN+7AHBnjSBVHxrEj5AlKJIaIqyzdscd4i0mZgmG77TDayaZ6imzvHmiXS1589AplDFOMHA8dFEXtvARlE8klytot3BT6sWe9WTLmgYf9I/Vlw/JsxeZYM4bhd763FZoUMyHI8aUU8F6RS5CCHTJP6hkT4BzEUc4XVeUYhogx4hQV1IHMb9Y5vPd03UDTiMtcz+jBIUxzFkySTkA3d946cMZDghBHnJeQ3KZd0bSevu8Zw4jxiuXSobXHW8XuYYv3cr5YLBuG+y2LumLoxYi3Xjfs9wfOL9e4BlCWEDXGFnKJvPpoRZhGjBbOusyxmWFQ3N4+8Pz5M2LqWSzFxHR+/pKHOSTym58taFs4jB0hZkKcWLQNeYqMo+DGLjfPKCURpknOexiOR+luWq/PGA4HFJamWdAPB1IM3N/fcnX1hrPlCpJkkF2eb/jo5Su+/W148eIb/OCP/oAxBB7v72UNwdDMxhZVLNq2FKVwtSUet/zgH/4h3eGR7fZGOpOL5L/0/cRqvUQbze5wYLlaUxsjSFugO0pxdhw6sjJoEkKULHNWixexTckejiIfV8rOOXKBqqqeskwWzZIpBLy3XDz7mN32gWO3BzJ9f6QUMdyIwU9MS30/sqgrVNtS2QoivLu6YzKF5WolId6VEVRGd8QrzXq15N3VFTFMbFYt28OBoj1ts0IbzeGwZbVuGUvg2csLlLZszhbc3t7z9u2NGGWqhqqNvH13Td0qVqsz6uqMH//453znux8BE6tVxeEwoBS8fPmSm+srKJHzMxEYr95dc3ZZsd8lVh87lsuWse9RwDROtO2aqvLUref162u+950f4O2B2/s9tvr91u5MfhLsTkGPT7lpT7hHubcz6b1rmfefO4l/pTx52Z/OKk/i/MmqPucblVLIKs0O99nuo6QzXHLwfuv3E7s9z1rMySU9/yonIf7pic0ZfWn+HpInVebMI4jENGfGaUUxjmISYSYbaCUIZKUU3nt88HLeVKIXdPmAMhoiczhyYlEtsEmz9ktyH3n9868oIZLHyLs3VyivWaxlDjJZU1c1sZsIQ2BRL0nDxHYI3O0eCGS23YGLF5d885OPCGSWZ2fUqyWu9hIIO5tPTrk9KaVZC33Phj/pMicXtZINi2BmFZI7GMVI+R6R8gQi5nRBf1veOelhRSviGDFWcgtu7+6FOjFjSne7HTlnuq57cpzf3d1R1zXee4wxDMMgc/cHBSE1Z1rN/3pCuIjp8P2TOWEsT8UXMTgx56pJ4dYYwxRhv9uLicoEFKOMTnU6Ub83r0rHhbxKpfLTePywqHRi2T89E30q5JwKRbMD9unmOonwc2EAQZXxW7iWpyy3+e9/H13t7y2i66O8cSlqUgSChGYarbFmdvUBXc4cQ0ElQQIoNKZEvC4EwCAVmTDBVMBrjfUwKcekNYlIsIW3Cfr7jJ00TSp8YwFrV7geNG9LJlpYa5gmGI9SEdodYT/CEBwhZfok7PQJjcnCZ5+mxDgmjErEMZIT9EOmGyT0NObT3CX26jAzi+JcIRqKMM5vOsdVKHQx8zBmurkIkGZxa5C3i4jCZYVLBdJ7s2LMEgxptAQtJDLjceD1F9e8e/2Gy4tPZ9eoQeuaUsyTyKSVFvyA9mQT0SnNvOwRYzRGZ1arBVqNDP1I4zzeaIy3T25oa0RYizESYpirappcElMc6IY9++MDTV2hlCUkCUXw1Zrvf/9HrNf/lqvrG7RVNIuKzZnm0294VhVcPGtYr500QkRhli79Gd1+wu4LwxAYp8KxPzKmBcuLcx4f3/FNvo9W8Kxd89nHzzHWk7UhUZi6HeM4ULJCzYwkcXTKzZRymVsVCzFL9SxkadFPv3H7nazzhRgTOouoJd8zM06Rw5AZgmBWVoXZoS3dF1MUkcHDzFJS802tZr6VBNyOY6QPmZQzzFW7E7n+5EQ/ueVP96qauWU5J7SaQxtSJqXCFGcKlpYx4PVsrp47GtzvKaKTA0Z5UIY4BYauZ9kuWS4aJhOwuqBVoh9EjMwhM+tGEtKDpmhx08Yx0e0HwUZgMI3De4PSGp8NdfZkHTA+w4C4ypcS4jmOQQQ5hDXmnBVnq60lJIdE2wq73yoJEe3HgThFESy8Q81OD6MN2Qgr12BRKkuwWypUKTDNIaVV5dlsNjjvSXlenObWipQzUxgYQ4XRs+ipwTgv1XosCU0ucwtymd0Ps4guBvgiBQGjhXU8FUylUM5IonhxxJhZNC23Nw8sFu3TgrVYyAa0ajJV07E6WxPGR1RW5JipjWO9XlE3DjNXfjebFc67mS/ncMZ8sLiXWUQfKdrwuH2gXazmMBfFNPWQxQnaNDWP93f4WRzaP25RMIuJmqqpWS7XLFcbfN1gq5qmXVE1La5qKNqC0nPlt8zhP+9zCnIWvMnJneOsxXpBW5UE4zA9FbnK3C7inATNMi+iMUeIcpAszK1x1hApjDPDcRgHpnHAGEWNwzpHWzc47aRdywDKzKtsIZXAVDLohLca72pKMMQiCegYfiNw2qBF5M4J5sKXJIrPJcYZsqaNkfCcOYSZEp/c9yWXWSQqIsLPrdu5JGHO/R4i+kevXpHzKPgZlWlaz3JZY7RFmUjdKDKCbBj6A01bkbMI2MN4RBtxWVRVPbOsG27vrvn2558TovB2l8vljJEw7HaPjJPBOTePK8PhsKeUwnq9Rula2gi9FK6dk01ou6ilQ2HiScCcponnz58TQsDNnQ45Z9lYe0937FAoyRiYx/g4nsZNwVpH5WvONuc8f/4CgJgTUxg5dkcZG5MUm2MMhClAKVycn6O0Ybvbz24MeU+0sU+ukZPosFwuqWpHKZEURoxS1E1DCguOx51cF2txrmLKA9M0kkJg7B6JU8dxd0/lDNfvXuOd59XL53SHex7ubxjHnpt3P6NkaSXPORHCSN91KArr9Yrd/pFffPULrh9veDwGdofAwz7RT/P6EEaU9XgHrjJQJJj9qZCbkyyjs9iSS4J5PzdOA/vDAawIKtbYOYtCrolz0LQOayU7A6WwrtBU0qmiNBibif1ASD1GKwqBul5gTEWIgdpUssbmSTJOLPN8ZjiFkCotAbS5jDgPRU1MoePy8hmg2WzWXF29RWkp1G23jxgDKU/c3NywXi+pmxpfGarKEePIMHYsWmnn77oDpZQ5/FQwMyI6WULQMu84Qwgj06SJaSSmkXHqxCFvwVdG5pjGydgOE/v9PavNBcZq2qZBa+iHgc1mDaXQLhqc/3tvw/+9j6qq6LojNzd31PUS52pAM/SRMO1wzlP5JcdDL9ehrjn2R8yMvFpvaqzNPD4+kvLI4/aOu7tbnGwHCDFS05BTITOS1YFhPAlyJ8ZmntEWa6YgobwpJdpmwTj1lAJ1U7PbI8UGDV0ecbbC+4bKVCigNgsm47G6pfINYYKm2ZCz5nDoCSFSNRXeV8LWn0YeHh+4ubkmJsnjiGnCUWFQTLNbc71e07SOMZ6j7ZJYFKUYjoeREAvb7YFpzFys14Rp4vr2lsWioV7VtAtHjB2pGIbJ8vCoUQS8AoWl8hUvLy/5/JNvkMcJFWQPuN/vsaZw2N2TtWVZe0LMNOcXVAbS2NF6jVMTjS381b/7N/zyb/6a29dfEft5HqkqLi821JXm8eGBZB1JRQITzXLFECe+evMaX1eoHKm9J08BX1dM+3smE1m+vODnP/sZd49H6sUZF5evmEIg5ZGcE8M00a42rC4/Aqt4ftHyeNdjgOWiJfYHXr4449vf+YR2tcAtlvzVz3/Gl29/yeVmTdJ3UBKrtiHFhHae++0BN0hB1hmFclncX9ajS0LlQm0tVoH1juN24u7mQD9NNMsFaRoZ84TznrauaBYtq3qFwXI4jBiXwHuCgS4PGCxJG+rlAqcMKY+k3BPiQFMbBgJFRbkP6wXLxZkw0pWjJKiMxvmah92enp6xH1huzri7e+B4OBCThA56W5NUYuj6p25MaxwlGrRxKBPop5GxRC43C8avRor+3YW2koRDXqJ0/SoNlBkBh5o7UOf9BAVfaTECxYgy0LaOvpfOYq2lQ+zpeyNoh5izoIWyFATqxpHUhDaSGzGFic2mAqXo9hMlF+paOsr1vKcMKfDy4xWP2wNp0qw2gslK0bHbBZZLEdWa1tK9jjx/uWa3DRyPieWyoqogxB5latbrmuOx48WL52wfDlgjZznnPVo7DocJY0bqOuFc9VTozikAZ8Qp0zZLHu+uMCqxXC3nor24YSUQ1bBoWipfc9gfuKgvUVrNCCEzh95bUsxstztWZ2sOh8Ps0N7jvGcaJ+7ubjgcDpydn7NYVRjfUtUbnj3/hMN+yxh6Nk2F99Jxeri/QytL3SzEWNh4VJAu3rPLZ7SrGtd4wngkdgemcOTtu7ecXz4npIRxszEkRozSTDO2TBtNzBllrQh8RboUrLWcMjuq2kuGRUyUFBn7IE5RIxk3MYiItj/sxaxiDcfjAecsxgi2zFiNsRIGrpSW4vjQs2hbhnFk6HtWq5raNaxXS0KtuXq85XyxxD5aXpxdslitaI2hOxwEGasluyrGROo6tNpxcX5G3x958+aK9TdXnJ2fUVct11fv6I6Z7f2B5y9qfv6Ln/MHf/AdjoPh0HVUviWOge9+/hkPD7f4Rgr1z19ckFJmGhPf/e73ebi/IufIj370fW7vrtjvttSVJafEalHRtpUEhdc1u13Hzc01z1+csd3ecnX9ju99/0f86oufEH/PLrKQTo5c0XWeWOYUssrvEROzSPkkj58+fKIkzOgX8QrqJ13hJE6exHUFEqo4+3RzyYLbKO81iicRvcyC++lXec/q/vAXBXI6OdPnp1ZEOE5Z7rmZmTG71kXDKEjWEAniZCnxiPcBrQUBmpOYVOq6Ihfpfh1H2W9kCtY5TErklPCmYlkvyEPER8u7L17ziz//KbfX1/zZf/QPuL+549vf+w7d9sDZ2QUOy/5+hymG88UZw7ZHJbjdHrna3dNcrHnx8UuaVUu2Igr3acJrcE2F9TWH/ohCdJoTyi6EgPNmNtCZp/fnhFf+kCNeSDOloHxgzpmd/Zzez9/qMng/GJ7+lNwfzWK54J//8/8Hzjmurq548eIFX375JVpbuq6bDT+LGdMkuMWqqmbczCkk9H1Q14eO9A+DRMsHnxMR3Ug+GHI+D5Mw60s21K5+crLvdkcO+47VakKp6enrTxiLk+alPhTVi4SyZvJ85hDlPJE4jWrmUapPVac5sFUKQ4UT7/y9SD+/rg+c7POTedImTv/+X1VEDzsRqYS1Ju46NQeRaWOpvKegOITMfkjEqMSNkCON01SVY9VYFlZJG4fKWFUwSgSlbDRKZ3QsRBW4C4W3YyEPsAIO0dBaeJzg1gmK9pMRHrJiMXgqpbjeZ7a9oo+ZVDRTLqSi5uEofJw8O4qLEhRNCoVxUBxGGGOSr5GynyA58ozjUIaEYsow5sKxwD4WDgnGfJrETm+SOIcHFEkVqlyo8hz0WoqgAfIsguZMQpKLY0g83Gz5yU/+ih/86COsaii6JmUryINTtU8JSxjj8KUhMlFUlgOutcQUZ1eVoAiq2glz2ooIUtc1J0aQ0xbiJIicKG1UoOgHx8Puhso72maNrhZMu1u8VnTdkUN/YPOsxnrFq09qXj6zfPKpw5fM4sxRNSLKaRTLZYWNNcYOKGPZ3h84W2mMqWhqy/bxGq0jUxqgwDc/+5i773yXYRyo2yWbi+eYeiUOmihO3BAkuFMeer7REFb0nPwbZ4xOIpOyliRofRJJy+wuGyklY9wpDLRwHDJDgEwhJCUtTEUxBQmpFRqDCJZqXvCebswsG5xxykgG5ElkFyDEKVT0hHORCUBu9gxMcQ5JKOLEHVNimCRMVxU51NYWlhUc4uxKz78Zifi7PHxlOCUqqNMENLfje2vAO1KwZOflNacsAcFagym4SkSiU+CJsRIaGmMiHAOhH9DGEEMk5gHXaOpmiZoKRhuq2qOVIZaIjuK8NlZEHVdZUFbeI5XRRpivaJ6Y8NZr4S/nQN0YfHHSsqoUlbOAkbDgZNFK440iJEPVGlarFZvzDXXTUJSm646UBLoyJCJZBcbUoWIgEphSkJZHJWNjipExTkgatBwWxmlEayV4CeWktS1LK6lbKIhmLkYqcgE9JXKMs2A8O8SsBKPkDK7yNMuWFy9fcHiUA79SsGgaVosW58XdW9XCij6FsGgtGIQcM9YaSUdPIhQqW0ihcHX1lratuby4QBVho4cwklOgrVt2j4+slyvSFEgxSct327JZn1FVLc55tPVYW+OrhqpeYJ0jKzMXik6MScdiseBsDpKM44iZW/jSLJaVMCN25hDU00bwhBuSkNI8cwGZb4r3BalUEqFEDAqV564VXYgqiZu9dtjao73GWtnxpTJn2RcZVyH3jDmS8DgNtXVY48WBEwNJBaISvFHKBYsll4AEdZqnopuakSOJACqibMF6RVFG2gGzhHDmLJkLeQ5sLUpYdzkXSkoYzOze+N0ed/e3nJ8vWSxbptAzjZM4/3MhpAGphg+kErBeNkUpKjaLNYfDEXQhpAmdNMfuIAz+ymKdMOsXiwXf+tZndN2Rt2/f8K1vfYuvvv6KYehpGhHMD4c9MQWqys2u9UJKhmka0LrM7ZGyo64qN4fdCr/zs8++yc9//nPqWkT2lNIsCGjpdIsnVql7Cp/1TjoxNpsLltWa8/NzqqomhsA4DnTDkXGaKEVhrKNtLYfjwHq94sXzZywXCw59P7tdTweVD10Rp8e82S1SQDDKE+dwIxFePSmOgBZ2f8qMsaOUgK8d3nri6Li9fkvXHbi4uOTxUcTzu4ctP/27v2G1KCwXIhwOQ89298gwTjRNy8P+gTdv3vD1u7eMU2SImjEpQgZlNUZ5UlSUeZNbkqATciynmJen5z9P/k+opFwSISUOfYfxYOsl1ho2F2t8ZWlaT8wWZQrDeKSbgw9LzlgH2sqeIhdwHlycHZExUdWG5WrFw/0dVW0l66PvKEgmg7UKbcrMwY+C2fKamCdi7NmcLWnqBd/+zjf54osvCXGYnZmZpqno+1N42kQII1V1QVUJDspXFu81qABKGN7jOBFToq7rJ1HGGPPk5mmahhCkM69tGura8/CQ6LoDSin6vsc5I2M5n8anIiZx01RVxf6wnVnP08yCrwhh+L3ubZBxd393x3a749VHkTAJ2kwKi5Fh7Eip0PUjVe2hFLrjwPn5hlfPXuKd4+rmNbc316w3Sx4e7nl83FI1RjCDJVO0pW1b2qqmP96Q3YCya3KOFGY803olzvohztdDcoPG8UjTLuaWYMc09agcMdrQ1A11tUApQ+U9F+fPQImIVdd25qkL83roA8ZXlAx9N/Hw8EDfdxyO2znfSDrlVssWSqI/7hn7kbP1Mwl2jQPeLxhjT46Jw2GkrReEmKnqBUppFosVb6+v0EbRDztUOeBNxvmG/e4IZsF+/0CMPZUyLFjgXcPZas2qaYhKM4Q9ThVevbjk+u6G/f3As5evGPaPrJYrhsOOxmiGYeT88pyf/92/41c//Tm3N1t+8fUtX3655f5+T8nw6YXnG8cji9qgtWAsh9ihNxvOXlzw+vqW8xfPMRTy2BGOB9beU2uFKR2rRcvPf/4LrrYHfHvO+bPMF1/d0XUHrA0s2hZUYQiRs+cf8fIb36Kxjm98/Ipf/OynhOFI7TXO1bz65CNefeMb7IbIj3/2c0LMbLd7Xn7W0qWREiNGVWLECBKmXNsKpy2JgUP/QFaR9WYBKDbrDa2XkPmh23Pci/vPGk/UBVs56qoVV12MaCQ4fn/sqJc1ftkK+iWMNF5ERFdVGGAcevrYSUEwKem4jcISRikWiyV9Jzx/i8JpS0yZVbMkasVud2BMhe1+x6KpiXEkjCNKOQqFpmrnXK3CGDLGOKq6oR96Ykrc7R7xdc3uMODs7747T1ERJyhh9gtp6fTBnoSYgrWSR0IRxKR3iikUVBSsy2pVzWuXeRLbZA4WVKLMSWqex0batcV66ag12hGGzBQDxmoWq4pxDGAS1mkWC4sxGVRgiKC9dKiXnDllDUqxTSCH55eeza3l8nLJu9d7yTFIBVcbyR7KEWNrhv5I21QcD48oLHESfu73vvct/uqvfiX33zyfhzBytllwd3dLKZlh7DBaBMnumBhX6UkQzwlurh94dfmSplpyf7+jKR5rleBMi34qvhtjGYaRx+09vvV0/UC7aAHpVl0sVhjrJYtJadYXF7hmibYtuqllb2g0KQZsu+Lm3RWPj3uOh4HFck29aNmcr6kajdIRW0CbirOLlzzcXhE5UoDVZo31jjFMVLWYEoZuYNE0jMc9KQbWZ0sed1s0EqgZc6AUEaQki0tMGcYoKlvReIexhmHoeXi4l45VZ/GVJ85BjcvlUsJ9c5zFs0wIYtSLMZJSIqUMRc55xjk5w4aEdzV3NwP6XNGFXjLYlGO1WPHu+gqfCqu2ZciJelGz2+9xrqJdVDx//ozt4z1TGFmvpaNxuVjy+quvaLxj/azG64CrajbfecbbqytC0gxDoB8HSqi4vFjQ1objsMe1Em469OI6Xy1XvHzxiuvrN1gTMToTpsDZxuGdpxTHcdfz/PkFSilCGHj2/JzlsuHly0vOzz7m+u07Xr56Sdcffp+lmxCnWUCU1EEzC4lyYnrvAFfzffRe11NPojfwlC92crNLttN7UVYx86NPTtv5zKRmBK6ez50nEf0kyFNOMnp+EnZPDvcPZHQ57/yWuH8KzZSnexLRoWTpABcDcyIWhSmJtq5ZLpaCLVJa7kd1OsOe9tLyPbWxMt97zzQMrJolDkNMiV//4gv+zf/wr7l/dw8pcfv2huXFmt3DnuX5isZVfP31a25ubvizf/hnPNzc8eLsOb/+xa/wznH+6gWrzRLtDVFlbF1jncFVNfWyRXvLlCIZaObOWKXUEyJFGyfaKPN687/wOInWZu6ezzmj9OyQ/p9p56dR8OG/32dwhTBhtGG73fLrX/8aY+SsfQrGPLncD4fDXJioiTE+4VvkXhaN8SSWhxye3svT8z3tVVP6za+prKyfMHcgnLLBlCbOXS/HQ8d2u6ddGJSaxA2uYBZyJPyaGTk7H0Skky094TufnO+nUThr4qXI/aJPHRdPf8z8+dONU37z0uYPChhPHy/vx+rfx7r29xfRi6eEJEJbUScjIFEpjNLEeeNLLuAlpGQMExZBXeiqpVo4WpvxM8NWzZMERp6JbRUuBAZdGEJiPyimAe5RbCPUutCTiQXWEzyM8FU2dKZgSGyPhZu+0GV5UyQAAazKWKOYkmF3hHYb8FaRA6Ro2Q2F/ZgYUhQn+tyilLLwza3VpBlQPxbogEPK7HNimPm278f4/FYkmYyCUuI8VgqLgO/V/E6qJCJz1vqJ6R1i4Wc/eUs/ZHEUZScCehKhQCGuTaU0xtTCfDaOYeiFI1U0hSCCXqU4HhNhOpJSz3IpreylZKqqmYsKiQKC6UCKBioE9scDBk1rPeebZ6ybJaNpUSby7Pwl3/z2kn4MQOGTbxpePWt5+bzBOXEUOu9xVrjaZtWSR8fK1VjbYzG0Teb2Yc9wPHL79Ze0VUU4dJTLiFKJs7OWkpMIzrrFVEuMrgk5EAqMUTGlLKnE82U/idRTkkUgFkHlCG+eJz75SfiOKZGmgMKQkjjbY5IN5pQlMTgUEalDyAwhM6U0i+jiwCg5kkJ+YliVIvzCfsyMiafqrFRm58/njLCWFVnNhLJZLJ1mx0kumTGM6FiYQiEUKfrUpXDeFC6U4nE09KFIqnX5UNj5D3/UlZ3b8wSX4azwpHNSs0BlKW2NtQh7sAj3qqiEsoWm8TTNkjxzm5zXGAMYcU4mIlnnuUKZqWvh6AnizeCspyQw0dKHeJStAAEAAElEQVQ6EY9VJd0TUolOFJVFWLcGZ4QJqXJCe4VXljQEcUzajM2a5bJmilLsSDGiLVTLChzkDio07bpmtV7Sts2cDg/FSo5D0ZmRkS71lCkwZiMu5TyScpg3EJkhDCh1wLuKNDtGtZHnab2dOZRp7pAIGC0C6jSBcpYwZSpbEfaRUiIgTqJpCvT9QN/1+EazXC346NVL3n31jjQdhJlojfx28ruq/BwkJROS0hqjDCkLvsZ7NzuHPfvdluVqQ0oTtzc7pqFjvVpwd3tN5R1jP5CNQ6XEOPSM48RqtaauGuq6nbtXDEpbnKupqgZjxb5YlBE+mpLilFaF5XLJxx9/zJ/8yT9iGCb+9q//msNuS5jbY2JKxBAJOXFCZXxYBZeiwIwMkbaMec7MM/ddSRCoSeAMOMFTVM5iqgatwNYG0xgSgWiyhNbGwpQKVS1BpP040ecdU7JYoNYeimbKI5MemMpEVIFsEkVJ/KcBlBIHrzKWpBIxDiKEl0xSEWUSzme0E3HNFNBWU7IiTA6ZTfMcXiqs0ILgQJxxv/O9PU0jX339QLMwrNeL+WMTzjUiFMaJEDtymVg0aw77npwzL1+9ZAoTKWlMNhyOe5x3XF2/o2lrxnHg8vI5+/2er776kpcfvUAbxf3DPSDticPQA4W+P6INDGPHOI6E6JiCBId6b5lCYJoGxqmn8hVV5ecNW2Gz2Ty53VOKgKBAtGYWLjWL5XoOq4VhiFjncaZ6cn+s1ps5hGgkZmkb1cZgnWdhLLtjR123fPzqFS9fPCOHiRTCbxRx4CSQils7zwXTEALj1GGtEQZ6zsQQyAXqqmEshRAyCnHYGGMgBYZhYhqO9H1HdzyIg3/o2G0f2R8O3Nzd453mfHmBM5bDfs+bd2/pul4yOtIdx75jt98Tk2RkTFFLoZdMURV1s6Z/PNB1vRQVK433lkWtxc2hJVhbK+H6Cs9eipMYJUUelYGI05lu7NmwZLVZszlbsj88EtJIPw3kMuErj9Oe6qkIMpBSRutM01iUVsLOteKSfHxMQKSQSXmkhADIHFaSJkbpSIhRUVWGkEf68cBmfc409VSVYbGoeffuHZ988jH393cShrlosDZJEdZbVusldV2x37eEOJGzfK6uBSHT9524n2bki+wPxIwQxkCVK47HI1prxjCQplFcfM5QsjiPu+4kjI9oLY735XLB7nh8EuCnKVFVlu3ugbp+Idifafqd722A7jgxTbIPjDEzBQmAG8eBs/Mlh8PEsdsLTi5FhjFyPB558eIZpRTu7m95/fUbDscdl8/OCFOk8o3cv2NHiD1TLJDPeXnxijQc6fpH6tUaN69vqcjh5nTtjJkxSzrPrHPhfCtlGAbpbFNFczh0HA8T3lecnZ1hc5oRMZHdrifGjFaW5eKMFBXOVDxuHygFbm8fOR72WC+O/uNxouv3NM05Uz9ilGbVtKwWCykcDBPjFOj6iX6cqJsW6xzb3RbnPX5eP2OUMMEwdQzdQGU1bdswjhPaefowMQ17svVopVi4mtVyjfc1y6oheI/RwvwfQo+bAnk6kpIjaDBVw/5BkBpf/PRv+PLH/5rjMXB71/Pvfvqau4Pms+/9U67vHviLr37MGBPnC8ezyw39EMgtHG63xOaci8sXksdTImnoKEOPL5nbN29YLyw3jweqzSUbf8HjMfAv/s3f8HD7yGphaeuEVoVvfOMTPvr4Y+JwZHd/Rbva0O0fuDxf8NXuLTmPhBhYrRfzWm4hW+7vdrhFjTYLvG053m7xVY1LmaZpUTnRuiVGK/bTUcaJmtB2KQW6OPI43ONMxfnFGc4ZlHKEUjCNo14vKEqEplzkHOasQWXPceiIgyBVqroSg0eSQnxVVeimolVryVdRihQ1MU342s/7FIc2iVQyta8x1tNPHXre9+73e0KWAONF47k83zDFIAWl/KFgZVBktC7EKbGoFxgk6Prt7Q1jMOT8u3eaTH2mBJ7QhAqYBkGgOmsoSvJkvJfiZckKXzmEKR1RaqJpqlmskVZ7baSRLyRx5mhjxYQSBdXaj4GmOXX3JaoGQkw0lWYYJqq2JqUOY2Vej2mgXWr2/YSejRoxFbYPA59/+wLKjeSC5YBSE9/6vGW5cuSv+znXStE0Ldf3HeMU0HaF0jXX1w9UtWHKHctlTT9JQfDb33nG669fgxIkqa8MMUqf+363xVeWu9sH2qahsStKMpL1NHS0i4Zdt2O7PbB6tuGw61lfrKSrMQQqJwYzYyxVVfPmzTVt6zk7W3L/eM9yJffA82cf4ZTh9VdfUzcVIUM/BYKe8KZlYS26GHAVMST+2//uf+Kv/vInHPY9Wjucrcix44//+If8k3/6p/haURKAQyvP+fkL7qYdPky0C8mKKhSUFse1ypZlXbNoW96+ueHyxTnOiiljCAFipKBIIWFtJbk6WRFTZlk3lFzY7bdorSXoUxXpso2aEAvD0BHTQM5SLBFDkOfhfuL6+h2lpCdTi9aSUaaNIZaMsZb1umW1rJlsZFXXTGkCq9kdD7TKEoeJZdNQNzWrzRqmkWkKVN5z7I44J6z+cRo43h/RGPr9nuefvMQ7DWvHvusY+wfG6SBdyPakMnd89dUj//v/4j/i51/8lO1whOIYQ6JkjVaO47EXo5JJlBK5OF9x6EeeP3vBj//iNR99vOD6+p5piqxWS6y1HI97FsuGUqKYZsoH4fa/4yPmKM5jrWdD2Hs29skwBB+K5+/1pnL6uxLDn2Q3yflI9n8n7fqEXPmAIf2BmigfEXHiQ2H+w//73mMhVPQnIX0W6U+4kveCf+E35HZ1EsnmoFRE2ykpkbOcA0Js2awmYkxYI8/n5B7OOWO0wbmKlDPOefqxxzqLNZbKObbXd/zVv/5z6uIZDj2fvHiJc47DYcdys+TNV1/zJx/9Kdu7B/7r/9d/xf/uP/1P+as//zE/+O4P+OlP/g5VDC8+fUUkiAZjCkUrnNNY5/Ftg2sq0SuRDsCqqjghQD7EuhgjWQdP90dO5DR3sFIwRugZSuu5A0HGQs55LnqJaCTG2UThQ6PF+2uptZwf8wcC+Nu3b/nBD37Izc0tpci9nHN5Es/3e+muFVPS+Bui+jC8z57D8PTaOL2nM3YmhEnOObOj3S1aUioMgwSUpsSTGUWZCevUjKodUWqJAL7zXOgTBvxTNhhZ/lRqPmfNoaLI53N5fx47EdDn6oz0V8zO9hNVHnjiqX/gcxadRtQ33rv/T1e48PelJP+9Z4B/9F/8H4njwDRIovRTJUDLADOuElaslL+YhpHdYQ9K0XrHpvGcNR49deQwUHuLSpnhuCfEkagzVkequGecHunHgUkyMhiBqzlUpVjFqhTioHgYDMfe8nUqwERMipAE5WEUOKUxSmFmIfOhV/zyGnZ9ws0xrgW4Oxgex0LWwgw7TTzoDFbNAQFy4UOGsUBAkdJpQlMfjO0CQZiip08lpejntilHxszu9vmcKlWop0mz8PqLB371yyv+wR+/Qs+CSsoTJQsTy2CkhZAydwNIAJ51HhsTVcUsCkM/bBn6LZLWrKiqlmPYE1JLU28w1su1KiIykwNTipQhY4tiZz2HwwO10xRlKEyU0vGnf/xNbreBaex49WrJculZrj3OCsPKuArnVhQ9oNUSvMUrR8GgtaM6TPiqIsYRVSK1r7HOQon86qc/YdG2NN6gbAW6xroV2rjZHZ4Zowj+BUglzWE48prDXH06vT15TuktcwFDo0TUDLO4oRRVzk+LgriJERE9CeJF3PmRlOeK3Hx3pZSIYQ5UUJBTYRoD/SjidynSpZC1mnncwgF7mpTU+2pXLtKZcOKnx5DJU2QKp0KAhKG2tSYZi9vJJlfGwe/nZvPWIKuDIFKMEbd5QRxAvhJciNEKowMxIDxpb9Am4mxF29YoZQRhQ5JCW9GYAsZK1c1bK90XWULItDM4J9zpHBS1koOyRklvgZawIms96MwQZWOXZieqcNxEmPTGYaxGuYItitWqYZjkoJWcloOTbig9ZOtARRabhnopY884jdWWSU2EEBnzCKVwnAoxG7zTFKKw6LV0g6QcGMOIUv3cTZAF8+Ad3lthDZYkzzkptIokCjpbbDFElKwX2ZCztFmdhOe+79nv93T9QDGWs7MVTVs4O9/Q7Tsq7WiqmqaucF5R1x5j5NBpnZZDVRGXAgVKEaahc47uuGexqOm7PY2y1FXF8XCgbSr6vqPvC01d03cHmYOLbI/GYcTZWhjnaLQxVHWDrxu8rzDW8Z5zJg+t5+6ZIm7Glx+95E/+5E8wSvOrn/+Mh9sbjod5UVWGEgO48rRJt25urzXzOHROqtbqlEsg/Mqqcvi6Isw8c1cZjNazm6Eip4iymqnInBOjwisvrl1XoyvNnL/FNA5MKVFlSxVrYoYhd3SxI6ZJ+OkOdFYUncBqioaspdslzhsOpWWOCHmS+9dpYazHjC4F46QVMp7aIY0gvJTzlORQpWAx+N9DRO/6A/2wpx8zw7BnvV6htSOPhcVyydT3TLFnCj2Vq2eB3XF3d0fTNHRd99QJME0Tr169Yr/f8+zykilMLBYtj9sHxlEc37e3NyyX4mhdLMQ5VTc1KU2zk0mx2z3S9z2vXr0SPl/l6Loj3leUUp7wLx9//DH7/RbvHdM0sN/vngJJc05YK7kWIQgjtq4rDocDOWWsFyTN5uyMly9eYvFs77YM44DSGqfFFb85v+D+cY9SlrvbO6zxXD5/ORdjmQ9KeW5LVDhnntw6pRRiFOzMOESCe795pqg5AAuU0sRYBBueEXeaERFmHCdAXLrDMHL/cM/9wwPaODavXhKT4uHhwN39Pdc3j0xjIJZCP41SpFGeEAPHrickRy4aZQ3jAT799CMO3RuGhyOly8RGs/nkjEXjOY4HlE6YEyJAza9VZaY4sbQWVzm0NSSVqRcWEwsh9xyOD/TTls3ZEo1le5hQJUv3ymopaKrbe1IOhDhS157ae47H4xxWu6OqLYulbOTHacR5mTNCHDgcR87OzticLQkhsN1uORwfMV5jTCHlgXax4suvfs1HLz/m6uqtHB5n9FPT1Dx7tiIG4erGGBnHwmq15s3br2maat4TiHgQY2SxaMhF1qXvfe+7/OpXv5LQ0JzYH2TcWWcIk2BEJPBaDoCLpTjVUzZoo+i6A9osaRcLxvsHYgxsztbSDTJNHLsjMUVCFHHn93kcD4G23hBGxTBIfkHbVjw87lC6wVeWw75nvb5gt9/NmQGRvht5113Rdf08npUUl7TDmnbOswgUenGJTiusWvDxy8/5+s1PGYaerpswpsEacTW64t9j7ub15lQ86PsRYZpXTJOsscfDEYVmuVoy5Ynn5pIpDLx+/VpyD2wlnHe9YBgybVLzIbuag5o91gpWaLGsORyOPDxkKmPwVUPtPGPf8/iwRxlFSANZa5TRs4Cz4u7mjrPVWgwVOtEPe6bjxDAc8Ebz4uKC/a4jhoI1meNxT8k958uGpav57NVnDGPg2I/shgGvFdZAjBPGWnzOxGkAXxi7jCmyziVtePurXxD7ia+/eMt/8y/+ll9cD0S/wt3taNfPGM9e8dPrN3zj+YY+d9TLJetn54x95OzyI9ZNS9cfWCw39Npy/uIld2/fcH3oWF9+wv0wcnv7jmPS7LvI11+85e3Xb7lcOX7w3RdcXqy5evuOu7s7vv3d7wojte/pjx0hDFBGjM28ev6CUmAaI90RdPb020h/SBhdU9cFW+849CPTlKmdkyD0bBn7iRw1WlcUIzi0elGTpsI4DHz9+jWff/vbrDZLvvXpt9kdOx66HV2ciFnW8qpyxDwSugPKWfbHHblk2qYl9SP9FFGuQhcpEBsNq7MLhphJaKp0JIwdzlWoLFg9+/8l7b+aLMvS9EzsWWLro1yFjsysLNFd3WgMBAmbIYbGCw6HHDMOjWakGf8hL3hFozJyLkkjRnAGwKDRXT1dlZWVmSFdHrXlUrz49vGIajSAsuxtFVUVx8Pdj1hr7bXe7/2eN7Os1guaspaA0MzgQiT6wJQE91jmJd3xiNX1HBLscbPQMDlHVtQEF2mKCuXljKeKkmaxZBgdISlU/EP8bH/75Yb5vmPMrBOIM9W5iMoEw5IIVE1Jip7jYUJZTV4asnkLNrmRsihAwXJTkIgMvccag0OhdJwLjJ7FomCK42ySU7QHhwJx0+cS6rfftZyd5Rg7YWwEJZxjbeQsZef8mrQQh/qzFzWrtSVGMJmmybUE8D5d8P3vEtaWeC+GhnGIFDZgjeXh4cCf/dlLbq49fYh03YibZDyE6HEuEpMmBNjtW7SSrJSYJrpjjx8U9dW5oKDaAaWhaz19DzY4whmsmobVomEaBlKKVKYSrFhwvP3hmnHqWCyl20o6ijKSVDSo6gWLzRl53VA1DQ+7I5usIc5dTcvVmkTOr//qN/wf/0//F8HImgLnE9M4ofyed+++4+uvX7I5ExzeYrESNIuLxCCZMgk4tnuqaoF3E8djx6JaElKkqCryvOTh7gGTGbwbKTLDsT0SEQOfsRn1Yo2Kgeg807xnjsEzjfJ+ioN/nAv9BcF73v7wkWZRok2a914PDMOR43FPURRSUCnLWTQ07PY7XJUxTJHvbx9QGWgrnbqZVsSgeHL1jIXOyArpeF+vN+R5IefbAMMwUhYNZVFxeX5Fmdfsx5Zx6GdBu6Euc0JwLM/W3NzvKEvFbr8nzwrBueIZ3cQ3332DyTVPVld8eHeLsYqnzy65395R1zl39zuuLgw+DAxjQd/J+f3iooCQ6I4dT55e8u7dPU+uNoxOuqbfv3/P2foZveu4uLj4O9271ewAFOOMOItVnPP4ZmFUz0LrSdD83JV7ekxpwWGkNAd6ztimiDjCT9fJmQyfXOok/ymPKZ1Y3jyK7ifkRkqPX5DzpPqsm1HNHccn/IxidhLP6MtPujAoQQmCYHS1MaRoEHSxEbE5sygVBD8U4vw+WDG0ThPWZlRakbxHciMCWZFjrOX7337PH/3xL/jX//zP+dnXX2MVTMNI0onjfs+bX7/nyy++4vbmBp0Mv/qrv+R3v/6Of/o/+Y9xBGxVCEqHKGuNTmRlgS0syugZ4aOxmZzJsixDzTiXsqw+c2Sn2ZioyG3+SV86nQ3V7EZPAZTGmhNueRbSFaQon+enz+2TgH4SjlMU5/exPXJ+fs7xeOR4bD8bI4LeSkk6O8uyfDR1KCU5PY8dDPPzPV3mMxTsKfsqyzK6rnvMClJK0R5btFZyxtazm1zJOp1bI5hcn+i7CWMsLqQZOSQytlFGhHV1KsSkecyJqQw9jy0lmkZ65LV/dj2ikdWcJ6lnJfAkuJ9Ed3lIcgHio+Z3etkpfdKG/xAr+h8sov/yH38tlX4vbWFEMCZH6VzCeJS0Z5TakBmLSiLmhCSWfmM0CS1hfkSM0rJZmQaCF4H4tY4cxge+ffev+etv/oJxGghOJjhAsprVumRTN5hRMXlHP02koCjqhrypMMaio0f5ERVlcSd5cbYExXf7jLcHYYKdBPYxwNElaQdO6ROGKEm9RM1vVIyKMgoVyibIMLQp0M+PkUDFNPe2iBifFGAMUSmiSUyjx8Q4u6fT4yIXlFiVVUoMneN337zjZ3/0S5ZNxYljHFOAJO6xyc8u/pldZTOL0nKAs8qgYmDwIy60TOPI2eqS2/s3rFfnWKtwaSQQsaaUxRf9yEYKQaRZFx0hiTMrTI7gJihgs1KsVwNFs2IYFMtFxmpdSpt0Ji24ma0o8wZlS0KyIi4lhVUWnS0p6omVm9je3IorLNPYXAI7N1cv+OEv/mt+8ad/hvOOrj3QH6X1zwUYpsg0zfz6edGWAFhZgFKUINFT9uLJhT5/LGgVQXmcSnhAJxG7jZHXLxWwJOJ71BLo6QLTLLgrnbD2dKOTlhc9T+oYE+MY6MbI6CI+CqfdRHm+KUpBRpju843JJE4dPDFClGE4B5RGpjCT0BRkFopMCjnqcaAqrP4DZvu/4zIqCSffZOIwT+LSN3lGWcrnGq3GGgOpIwSHyfTMqZVijikimbXEKO5gCdHQKCVCTJYZSpWY3CQtPlaRck1RlGSxQGtFyCM61yQVGKcepSxlXlOXK6bgSMOOwe1RFiS6biIS0NqSk4kzXkNWZPJZ24DWwpBMIUelErPI8IuBkEaKZYEtNEGdKsGBFD1EYT9iAkk5krZzcG0iaXHPKxJKWVIa8cmQKUXSEZ1JN0JQEay0CE5TgCnD2gKFhNdKQUcCZEOIxKDRJqcsBe/R9y03N9e4cSTGgmXTEFMkK3PJMo0KUiCFQFU2ZNaSgsePA+SGGCZ0KiQocK5au8GJs1tpYohkNmPqe0wlz8ONjqpacHtzQ3DSpmW0kQ6DpIlpwmYT1npCmWZkkqw/RZVjMistbNJy8+gkUIjDNc8KVssVl5dXvHr1iof7Ow77HdWiwTnLOHaEKQh7XCV0pjCZwmYamxtsbjBWU+aVuGV1JCs0JtdkucFYhdWFuESthGspLUJZCAalRVRz3jGGyKQiWmfUmYRyGR2wMYFP+OToXU9nWnyMTHFiTNJpYLJMWOdenoOy4oZXWm7Xwu3zgndQDpcmZpIfigyUOJPV3OqtdMAWSjo8jOB7siD4Cu1lXv3Y6+xiQz1mHNo7kgrsDg+URUNKGm0VtjA0qw33W0+9WJDlDdfX1/gQSAqKsuDm5oaUpLXYGEPd1BRVyXZ3zXK1oCgKPnz4wLNnz8iyjKIQ7FOWZdRNhbqHphFHzzCMhBDZ74+cn088efKU29s7mmZBCBJufDhIWNbl5SXffPMN+/2e1WpFWZaPqA1rLauVsEqVVnR9Ox/o5T3N85yL1VOeXD1BKRnDPni6QTaA3XGkrhfkWYFzEvTTtkeGoWMcetphRGUZWW7lkDn0wPDoNLFWkFE2s+QxJ8Y55DnN63yM5EXBNKY5IFdwBNbkpJhJoSB4lMpQOiPLKg6HloeHHeM0YbPEw3bL0HaMw0Db9wxjEEfHNOFDxFgzt1AnfJDDTlHk4poMPR8+fsQYy2q9Zug78lLT9yNdd8RkiaLQoBXaaKyVYLakZ+FHRfIyZ7nZ4IlEPVLVOShPNx3FUWAc1krBTmlmFvSe6KUoBhJA6b2a0R8epTOcHwnBUdcVMXq6bgIUWWbxfmKaep4+/Rl5XrDf7zge90zTwPn6DOclryHLLLe3N8K9Lwpub29nfMvEer1iu91TZBXT5Hjz5o04bp89483b72nbjqoqZvdNL+t+iozjxDAMVFVFVVXc3d0Rwhy41si8CN6zqCVQd5omEdGbhXzmc7C8c8IpNjOvsqpq3OTI8wKFJs8Kbm/u2JxtHh1hP/ZSFLgpUhYN3nv6oWOxqHC+xLlh3h8anj59yvZhR4g5KMVut8POIetFUeGDZ7vdEQMcjz1ZaVAqp6wC4zTJockZimJDVZyz3X/geGxZLARL1A89kxeWcte3ZEZLoXzOq2jbFqUs3iX6UVAYPkk3H0bRrGpWZ0s+vn/Pw/aGxWJJQjo+Pn68ZfvQsVjXXDxZobVhs77gbHNGWStc2KNNQ0wfGbqOsl4wDSN+CIRouNseMJlhc74gaU1VV0x+omkqtveAFqfs8bjD+ZF+aGnbI3VeUFULUrI0dUnRLLnfPTC5kdH1dOHAze1H/v7VT4kYiqohTgPDOODn3JWmqdEJ2sMBWxim9oCtan77m+/oDkf6AdZnT/nqqwG7mrhtR4w/Eg8T9WqDXdboHLxNfLxv+dP/+I9RuwPDFCk3C9p+JCSDyiry5RntD+/wNueu89xsO+47x92+5/buwMd3H7EhMQwSom2UhEtv91t+9Rd/weuvfsLq/JwpTrTtHsVE8J6yzIkh0rYjD3cjdx8e6I8D73448vJ2gVlMmKyAce7EU4bNaokfE3c3O/K5VX/Ss8PJKqzKIOWYIuPD7Ue+evWKlAJXV+fcf/tAe9xRLmqM1Qxjj/I9YeqpFgtiFMciWc7YjSitSdkEKpCXFhSYekHRLPEYph5SEBSpYAYSKgWqKqeoCqaUUEWO73p0ZllkK5yT4OkUPF3bzoi9jBgkHySEhB+9cF+jkmDxlAjeUdY1MUGzqD5xZn/ENcejoFUShqs2xOiZfCJ1niLLMIWlnxyrTUHsZP0LLpGXitxaXIgEFykbzRA99bqge+8gSUhplieMceRFIoQRmyWSQ3K0fCCFDILBaYXJJQstx2BMJK/AJYVWgonSRgHSHVpngWj21CuFT5GutdSrClKk63sg8tVPFjR1xseP9zRLRWkKSAHvO5wLfPf9PUoF6mXNdjtBkEJvbi1dPzFOkSyLFNZSl2LQImlePq2YRsGDJW3o+5HlsqE9BPA1Rb0iBUWKEyo4XDug84TzLd4r+s6j0ohOnrbzXF9f0zQlbbvFmEQ/tQQdWF+c07cTxhbUVQ5BUdqM2w/v0DFBKul2B27e37NvB+lotxmTH1lkjs1aOpea6oKqtHRdz+hajE5sVue8/eENXX9k+3DLH/3x3yMGwWBNfmAKIyarqFfnHA/3rLKS/faOxaLGGBg6wcGMfaLIM4wpsWhiGCXg3ffzXklJHsw4YLQgMxZNTfn0gu32mvZ4JMSJvt/i3YDRkb47onWOUlpQoEQ0kTzX3I073vW3XP3kEpfghzc/kJuCxtQsyxU2RgjiVm0PHfc3d6yXgp7wzqOVweqC880zzpbPCeWO3t+TVZq7/Zbv37QoY6iCweY1mwy86+bnYnFhYnWpef9wzWJRsckNi1VBVddcf7xl8COhy6iWS7q+5eLsihevfsr+eODu/iOrtXRe+qHCKM1XX16xXNRsTMNvf/sDbgjUzyw/fPjwdw4WRYsDVgTs+Ht7gVNh+ySbxsinteskjs6onlmN//T3R5H133QTyz87YV2YjXgiuH9Su0W3E+Y3iNCfHh3wJ7GSWWA9IdnlB87glyhfT0ohHj95QiHMuleakc/WUmQN5+srFosleZE/CrkxhE/dT1oMatZkoBLKK4JJRGPQJlE2Ba9+8ppFVbK/2XL1+inv7z/w9devGP1E5wd+993vUJnm6tkT+n7gJz/5mv/b//n/yv/6P/vPQWtUbkj5THvQc0FCzc5+wHmHBbJccJEnF7mZzxyCk5R9oaBZtCCW52wWa+0j4iVG6d74m9dJ3NVJP74PMhZO+BP4rB0BpTTjNLF92NJ1Hc+fP+fNmx948uQpp1DRpmkeXfHSlWTo+34OFI5i9HLu0U1/+v8yRtTjmFEqPTrZT8jZlBJd2yLOeLCZZdGsyPNKdKzoSTGhrabrpSPx0z1xHuOzoeVTYUgKFWJUOmmdcKrRpL81I2x25yPoZikEp3lqfAoePaGGYvTyGc0ma8UJv/N558W/X1f7g0/mf/If7kVQ9JrgDSFkiOdXc2KNx4AgS5RCRYPzGSEqYjRzq5KBWHKaqGlmwRLnhHESw3hFscqpV2tu7h64fdjxsNvipo7VuuGPf/rHPDv/EjvldLueoR/QJqNZndE0C7LMEr0jDi0qeFSMqOgYhxarocpyonMMx71wQmOkTJ6V98IyjQqtwLtJeLRKiwO/76RIMPYU00CdEsW8cE1z+Nts95zF8ySf4unP7300smDJUqdmsV6+qqXgz/e//pbb+1sW9Wup7GgkDTgFCQU6cc9Twic/s3WlxV1pTfATw3SPtQNJRfLasDvco46Jqqwo6pqQejJtRbz0wuD2XgI+jNFQKDo3sO9bFtUKG2UDsL66RJuR3CqsqanrQoRWm1FkBZlpqPIVVXPFwq7Z9Xd0HEmlxuqClGXYdcFw/ZEnL3O21x/k4HHcky4DT548oeZPya0lU44yTexub2i7gWnyjFOimwx+fs9jmgsYSaaQONCjtEglGWMhIYUOlSRD0EQmk3AGinmR1tqiNcSohRGfBB0kE08CSk+BvnZ2mE5DEJa/Do+Tc+wT/ajwUVzzxijsjIpJQVAhYeboJ5J8dmlePNPM5J8nu/PggiZxauFMGD0/n8/25vpvDaD4w6/cSKErsznGZIzDQPARbUHFSGZA21yabYoCETyEUxvJZyyQf7xhnm4NJrfzGiGLmVaaJqshVUzRkXLDsl5SxBKcwuNwZqSPA94P5KakKErqakGeIlMYaF0g0xaDFPPSvHBqU2JMQmlHcNJxIE7tDIWGkGNNjUkZY9fjQo8uFCqThikXEt5HMpTcFFVE4zBGYYxURSOJOAdVRJWws4szqImoRDAxhRZXphKMB1iSMWR5iafCKAk2Viky+YBOCTWHcBpjCSmQfGToe/bz+PNxYr1aysFRCYNexdnlnSJFLsKlBMJ4WSemAR8KlLHEGCizkrzM8ZPwBIP31PUC72DsB+q6EZFHWaqqZxhGcmtmbIulyEsWizXLxZqmWVHVNVlV0CwWVHWFzbJZSNagElGdgmbU483+ZFtIc8p7Pwxs9ztCEPcYM8YlJsFkGObOkLmOHFNEhUAMnqSiONDrHCN1XJRK5NpSlhkhDpSlzJ1IRsSi5oLL6DWDk66AIrMiICqPNZDnljqVDKIv4aPH4HB4ok4oa8hUQZFrvHOEFKRryTBvZsWNJZ0J4tCVj0lL8cw7MAqjrXRoJJnXGE1e5lgDSkdCNKRUYtWMUfuRVwgTPo54P7E52/Bwv6UsFywXS9lIlZb1Zs3CLVku1xwPPaC4uLyiLAq8d2R5gXeezfqMw2EPaELwXF1dcHZ+xvX1Nff39/R9Rwie7VZE8GHssb0mxYTRlml0dG3PYrGibXtARN2+H7i8vGK325HnBTYbiCFydrZht9+y229ZLBqWq6U45MuKfujI8oyLiwvut/fzBjDRNA11teBwOKB9zvPL1xJa5YTjqYwUJSc3sdmc0/Uj7bHl7OwCow3f/vZbYoSiWTwG74jjXPiBp0C5PM9FVHlsNZR7gkZcQRjDNA4EL0zWellhtMb7joettDJ6J6+96z3eBZIyZEVN0obBDdxtd6xqCemU4HMlXVhBCpcRNXNJmdnBUvbP80hZGx62t+Rlg81hkRVAYH9sqSpNps28C9EYredwctn0GmuwWUZZlzTLBpcm9t2WqqkZhwFlJNT05uZG2p0zSyInm905XddRFPkslNdza7YwyrPMcHa2oSwL+pk7f+IrOjc9tojudsIQL4qC9VrQIYd2D0TB8vhE13V8/HgtLb1FQdsemdzEq1evmMaWzJRkNuehf5gPN066xdxIllnGcZgF8gXb7QMnIX+7fWCzWdN1Mp7lOYXH8aWVmp9bhbAmW+q6oq4b3r59y4vnL2dnTuKnX/8MpTTb7ZYsy8nzgrOzcz5+/IDWmjwvfvTcBjC64PrjG16+egoqkHA4H6nrFX1/JDjJ52gWDbf392wPe/JC0ztHZYCUKA3YXHPstvRuz+gP+NHTNI3sJ4OmyWtU9GzvWqY+4ifhecrB2OH8wDgJ1zMByljyrMS5JFz+KAWk+/sHsqwkMwlVKiKeusl5/vwZ0Sdub/bkxZKL82fsHnbs7h8YusjYB1ATi00m8yxaNusN67OGfszxceDi3PHROQKacfRMXU9dNpTWCDrP6hnloxiGTrqvmiUhaTA5h+6INppmMTtSo+TSlFajo2VVLnh19ZTvPhy4b3foZcHH4Zb/4l/9f/j7X/+Snz17zSJfoaaaaexRWgIIiZ5VvcR1R0Lf8f7Nt4zbPd2x49gnosr4469ecf/wP7DJNWOYmCI8Pa9YliXLRcU49uRlgVewXizRydH2e84vzmn7AUxNpKR3huu7lmmI9OPEbttye7Nj9IGXT85ZLRas8oBNE2nssVVBU9a8ff+Gv9w+8Md/+se0ukNrixsG8iznsH0gpRwXE9/97juOhweKImOcBq7f36GbibypeLK5ZDzu0DGS2RIwHDtHbXJWixJLgRsHtBLE0ja0nD3dsN9vcVng3e4N67hhs6nJczh2HbkxuOQJBKIKjNPAcX/g8uyCpihpx0mwLEVJSJGmqYlKzpTRjYz9wNAfUVHOaVHJvXZoew7XW37y5Ve8f7hjmka6tqPIcqzKSF7R9wNFI1Fn03CcV3mNCppCF+zaA/XC4GNk302yx9UKY0aUmYPcpuFHz+2iMDg3t7QHKUBoKwJP8Il2P1Jv7IxomcOPe8ky8i6QvCO3FqMhLzMxd6SJzUVJfxxAJ0xm5Nw8h4ZmRs7A0+goiww3RYZxoFk36Fw42yFoCitZW2GS0LcYEzpCXhrJPVEQk+P6tmO1MERriKGmbXuGQXHzvuPZ0wXLRc6zJ5cs1hXXt9csFiL4WMvM1g30w8jXX7/gcNzxbPmEqi7YtyN/9MevWS3X/PDtt7x89SU//fpnHA4tRZGz293z/Q/fkpB1LAbp1KnyhmGQ/K3N5hzvAzELjMOAy0ti7um6g4TOK0Fc3t89sF5vmIJgFlMS17tKimHsqH1NUdWgA871lHnB/e07iqxmuVD8o3/wR/z5X/6G3WHg2LeUVcXlec3f/3s/Zxz2XH+YqOscjyKpjLoqBaniJrLMorRw/rXKWDRLQpwY+j2agNKecWwZ8gAq8LC9pcoLfBgIMc75DgeC72SMFDnjKB0bZVnKnjnPQNUQpVCTmJimloQEEBubqOuSTiua5Ybxh49kVnKOSAFr1XxGtnRjx7Ht+HA4MgYJbX9ytuH2/prAT/HjgHaOwlj6oZNCq5F9/mLZoHUixImXL56z/OuGb7//npB5qsJw7HpGP6FTRux7lC0oCsNiUbLdD6ipI8bI1dMnHA47pilw/fGBui7ZbSWz5Xg8PmaSZMawWKz5zW9+Szd0DGNLU5W024kvXn/Fh7sPbDZLht5zdrahrhYEp9msz7m6OGdo27/TvfukfYup7tTdOOuW6sQxZ9am1eNjSs98agUpzGL1p5+KVlK4PmEqHnEV6ZMw+CgQfibgy3FT5riajXpyZDsFXgqGVgzlgiNOShAysxIpSufJQJo0SUnOGzO6Vl6wFDXDfFayeUFZlmhtZF/sxYGulBj1lIqPa4GYDxM6M8QZTxz9RNcObJ6e0SwqqlVNtan5r/7f/1/6scNkhvfv32Kbgr/3j/4BKtM8u3zBv/7rv+CLX3yN05G8rqAwYA3aaqxRuDDifCRXkBUWbUSEHud7mJrNp6hPmV1q7hrQc/5GCKLuPQZjaoWxn/jif9t1cmCf+OAxffbenQoks+jl3ISbJtqu41e/+hWLxYK7uwcJH54d8ofDAaUURVEQQqAoJJj9JK6fzjanToeTAe4kbJ8ePznXrc0f/y4ivJhlYgroUcLjq2qBMZZxEsNE8onDrn0kSJwKQXOdeH7dp8LAjABS8bHI9DfeoX/zPeNU5IlzYOv8c2Z9MD3+EUNyStL1oT8rBqlwmgPzz/8DeC5/sIj++uutiBExEYMiRi3CFHKQi9HK5EpANLODVxG8IgZN9JqULERDTIboDSGIKB+jfC1FjZ0Gni4jxdUZz/uah/2Kh11D2x5Y1hteP/2Cy+YVNtXEIIdzrTPyvJzTu+1ctgsYEEEfUS3VvJjEFAnOS5XJBxmcMRJnsRWSCNYxEH3ADwNhGEjeMR33tPt7vJ/YDh3f3nzkePsRwjwIZ2c4VhYXUU/kAzExkUWFkVoJJ+hOQOERDaa0mrPSsMkUUzfhfMBqJRPWOKKb8H7CeXF3MU9cHx0xzm09SdOPDxyHD2BHuUGakcW6JCYP2hOjI6UoHNlM+MV5odE6Z9T+0U03ec/N/Q3KJZ5kPXH5kvPzFywXNb0/knJNVQpDOysqcrugskvq8pz18icUi+fY7Q9M069IuQZjCTojOENz9Zz23XeUZY3rO6L3KH/k8nzNpv4JhU2YJOwsXa4IyeC8OLxHr2UynipSs0NcFvj0OBZTgk+tSHNYKIkhSrFjUrNzNM1FBJfoXWIM84KCAqWJaEKUm4Rwx4QN3ndJnNDqE8fscIi0fcAF4ZKJQ149uubj/LxOXVYxCnImzp0PViFCmpLHfRRcjTUKycpUc2CuiO2fatU//lrVDdJqXQqWI0EfHVZpLLK4KJ0obY6pDZnJpQJrDVOQALoTk+skpJy4xcZIu5VsjDMW9QKlFJMXLEaTFeSqgFzjZt7x0DqcnzBoUhpBORSRyAQEtIJMZ+SZLMHG1GSmQumI8x3eC0JHG01mLDbLSTHHGhHrs3pBSDnKJpIWl09IiWFyBA0p2jk/QNr742ns+ABK2NtKa4w1sltQARdGqZIbhc5mcSFGonNonUswCgXJRhG7oiNkkHrPMHTCXyusBCwqYfH2g4ztfuq5v9+iUGR5ztn5OTy29cmYyIuCYd6U+iDhblprijwnz3KGaaTIBLfig6cqK2nvqhZoo+n6jhAjTbPk/OKS/X7PslkSQqIoKpp6wXp1JmGNiyVFVZLllrwUjMtJQFbzhu7Tfe+ENRK3Z4wBN010xyPt8UBCnNbWSkunn0a8i5CE4TlNsrZlWaAsK+Hzp0BCAr2MUeKGVcL9zDONsYnMZKTkQScKKx0Wau5U0F7CTRInl9fsCFZG3PJ2jR5lkyT5GrMrAQlo1UbLOp5bfBB+clKyudHKYoxGacM4gUZRZwu0kuDLUU0zo05Cn1yMmFK6YBaLiizTuDASvBQdlA549+O5ycPQceyOQGS9XtH3PUoxF0stxRyap9TpwC5txWVRURSFuGnVgSwzHA5HUtIsFiuur2/44ovXOOd49+4tr1+/5s0bQWUMwzBjHEacE+FQUuKPFEXJ61df8P7dnfDXw424XH2ka3tOiJSrqyucn7i/vxPWbZlLqNbMWRdGvqwvq9WS65sO7yNNvZSDUjfx9Owli2ZJDIm+H3BeHBaTm7BZRrNY8O7dR6Zh4umTp+wPB+7v7ri6uiKv6kcnSIhBupW0cJNjlNZG505hwBIGG7yE72qkBb5pGkgVwXuid7jJEUKiqlekpCQMdIokJbgqm9WUdSL0YiqwxuAjoA0RcQSP0whIq6UL03y4iILgSo4QJqw1LBrD4CZ87OhHj/NQVVAsNHluQM3ZD/OpLTEXhr2jWlRUi4a6aYQjnFuGuKcfpJW7rCT/YRx7FgsJFwYR8r1zjMMgCAYvjOJjO3EKUXLO0bbSbrpcLpmmgSzLOLWbGmPYbDbc3d3hvRdW9sxtvNveURQiOmuluLi4YLFYcTy0eO+5vLzi+voj9/cPeKe4umxYrdZst1u8i/S9cPzb9kA/dBwOR8pS2Jbb7Za6rlkulxRFzna7ZbVakOc5q9VybmW1sm72I13XcnFx8dhd0TRLFosFy+WRPC8Yx5Fp6knKzs4fKVSen5/hnKPvx/lQ8+NFNhBsiCBrRpbrijzP2e32bNbnTONE1IG6EqauzS0Pt/dc1udM3hPGvQS55olju2dyA4fDPcZA02RUVU53GImjZlUvyQw8bPccuxbvw+xGE56+0RIKHVJCYYhJMwye4D2ZLnj+5CUxKHa3A4UpUVmEzFAtcs7Pz8lNzs3NHYvmjLIsKTLLYf8OQsLqEu9GtG4oqgKL5Xi/Zx8jyQ9045Ex9qg8URYl7WHAkjMMjqJILJcNtpxdZjExTeIs/uHNd1Tlii9ef8n9vaMbHFVd0yxK8jzjcDgyxcDUtRQ6sru752yzYri64u3NO9qhx4aWYehZVDWVznm1fMJZtaCuFiQV6bojY3sguZ6paxl2D7QP9xwftgQfKVXBOA2c5Q1/7/UZ//q3N4wm4+zpFYvcU2SRsil4/ZMvaNYbDm3HkyfP2G+3DGMnYZ/9RyaX6HpPSpb/1X/2v2FRKNrDkYeHPT+8ec+7d+/Z7rbc377l+dfPyU3CTyND12OM5usvv+LDx7d8ePs7Vi/XHI8HctVQ2JxMibP14XBkt39g3+1QWWS5KinKgl175Ozsgidna94e7lFG8bC/pyiWrC/OOR4PPDw8oLKevExkOZRVQTGMjFPH2dUacjncXm8/sCyXfPHyJb/6q1+xqlcUpebj7UeGyUmH1hRYrzasFgvqWro7bLEkKHDRM4aRh/t7hr6TTtO5pT5TWj7PFMiygqaoMAGapiL4Eashuoksz7BZwXQYcYPDB8mDGYaRGAP77YEiL0kqsQ8jq1WNCxPReazJWaAFgzdIBsePvbIix8dRwp3ETUD0CXzE6pwpRNI+kMeExjL0Qdb0KKYVYV9P5IWiXFqaJiNohy41KrMcDxNlnQOK4KEqpLuKJHsyOV5p0BnKKLT1+Bhou4n1kwrphHZScC0DRSH4q7I0TJPn/HxFe5zoWsdyUWJNzmHv6DvNV1+9YBpGulbxm7++43/8H31B1w8UZS5nkcxKiPAwsckLmrpkt9syuYHFoqYbBura8P0P3+Cc5ze//YZvv/vAmzc7/pf/6T9it5dxqJWlqSuOx5Hnz59x+/GB1eUZXdez3e4xRcMUJ3GVpsAwtBhlJBhQGTabS7a7nbynSgrk682au7s7FouGb779Dd3Y8vqLLyV3ZZgI3oqLdtrR7kd+9vUT3rz5gdubO6zS6BRY1UueX53RH+65WF/Q9UeS0fikSKnmcHPDt99+w+Zsw+vXrwle7rchBrJMwqGHriDPMhIj4+SwmaZte1IYScj53gRN5nNitLIfizlFUZAIfLz+QIyRFy+ek+eW3W4nxWXnub39gDFxLo6ODNPIsUuUzUpSw+dgvBgEAYbJ2PWO7958pO0mskXJFCObdUGMDo/n2+9/w8VixSIryKucvChkD9AdsJklJI8ximnsCdrz4vkT/su//O+wq4RPmu3hyNnqjLbrsdXMTT8cqQpDVVrKsuLLr57z4cM1RVnO9+EJkqbrjjRNTQieLLcMY8/9Q0978NzcHymrktVamPjew3/5z/6af/hPXpFS5HjcM44d3k+cX2zQWnG2WvPmze7vdO+ezbacTOWzOZxH3vjJiKTSI7YX1GysNI9u809q+8nIKfl5nwvoik+i6EkumJPc5k74WU9Rs4MX+d9PjG7J9juhceGkbcgLOJ1DT7iNNBcC0u851U9hjhpthU+f5SVZUUjnj3MzfjSK8cZaUDlKiWM5BkWemTmzzTMFMeq44Ag6kEzg+dcvKJYFh+nAn/6TPyPcHcmsYb1Z8u7+huvbj/zkj37OmCa+ffc9/4v/+X/Koe15evGCZNQc4KywuSX5QIjiuJaOxIEiL8lMNuct6UchOZ2IBtZgjKBOT3tqZjyW917kAm1O3tp/6xWil8BZ9Yll/+gWnS+lNW3XcX9/j5sm+r7neDyiteL29vZRzO/7/pHfn1Iiy4Tnf/r652L5Cc90usysMTrnZtPQJ9Y78Cimex+ISZK9jscjZbGgaRYoJcjNGBJdJ+ao9BkGSMZLEg0Nxeco2MeXrT8bu6cx9zeuzx+a603zuUZE+5hO550Z1aOijM35PZZxGh8d64AI8P+e6w8W0evFySud5lfO/GH6+QUI1zPxmXg5w/TTo9g5C43wiLIIIQlIFsFABB/wKRKjIYaC0SvGKWeazojBkhHQ6Y4Y9qRkIFhSKCBlkCShNAZDjBowEu6RrHS8RFkMtFBm5blGkDYJJVVY5upISnMriQjtBOFt+mlk4yfwjhdjz+LtD8S/+Od8/PADfppwKuGsAqM/VfAU6CjtgzqfA+UUj5NQWAkarZN0OgJxHOh3W0ISMUgDKsqGzIcR70c5FGUlEnA2kVIUEdz17Pp3hNQL89RYkhpolhkxCJMyRVkgjdHi7pRPkZM/PkZp8fI+o+tb/PKCYnmJyResbcVycc60bwXjUOTkeSXu82xFla9Y1E9YL19TLZ7h/cTd7jdo7UjREMgZXGKKiXxxhk6KvFySgiP6I1ndkFeXMB3R1nB2dcaLn/6C/Q9/TddtcSEyeeGVy4bjk5g+F7I+tW5wqojKIha0oouR3QTOwBiFvnMR5yDHIdKNQYSLeTKHJIv35NLsppUyq/eRcZJNZGYyUhQ3c9cH+lGc4icxPyK8dk6tTukUZHASaWCKcruzRlLdxe0m33diOZ0WlZBEYD+9yj+gYPbvvDarjaCGtAhestGLxORQSZFcJBkRcGOIVHlGmquqLk6PVUz4lAQdgqBRlFUYI7fxPMsktExpcpvjw4COCW0SppAQQFoYp2mePxEfRybXSTFFR+GeK6SV1crGN7c1hRWu4DgiLncTCEjwq1YS4hgIZFlOaS0hGkLyM3dd2qxyMzAR8U5CqNAKFzyTd4Qogcm5tRh1wjnMLU9RnmdIgvdRVpHbQtaWpOe4DCnIoGUPavNEETVuSsKK1nZuMZLP0zlHUgk3OYzRjIPDWsPl5RWLpuFw2In7xyhcFFhIVuSgYHRORP8YqPKcsiioqpIQHYUV1MbkRoqyenwNIXqO7ZFxnMjzkrKoGadEUy+pmwVXT55xcX7JarWmLKTNOiovyKT51qBO7DJOsJq5CydFYgj0fcf9/R0fPrzjhx++F/dnFGxIDLL5UkpQHJ9aD8HaT8w8Yb8LR1AbpNChBZllM4PNFBFHiB6lIiZJIFlucnG5EPAhkFlDSHH+bILwV72MkzKrmHxAoclsAZ5HsTSlICgsK+G6IXiYZvagks4SPYeyGJUAQ5kvKPKKEB390AoOyVjZPEQos4yqKimKDAnHdEyndHIlmKsfewUVGd2AsSImmDyjHweSygRLUhQcji1aybozDgN5JozyU8fT2fkG7yQbYr/fc3V1xeZszcXFBX/+538OiDPLOUlBK6uCrmupqorlcsnx0FLXDc55bm5uqOqGmBI3N7d89dWX5HnJODr2+yOLRS0iWlHw/v0HttsdVVXR9/3jBlCY6LLOfPPtN/zRH/0CbhJd35FlGW13ZLO8IC8KDscjbngvwcVW3Ndu8JRlyTCM3N3dc35xztn5OeMkTui+7yn6nmRmVIlSxBDn9swMpfTj/Gd256A0wcM0jrI+5QuszWiaijKXDpDgHTE6dscHEomhG8izmqIcGPqEnsNnrffkiEPMTxN+coxhJBDAzOxdEioErNR5JFBSCVdQG48yhvXG4qOlThk+BJQO0qURJqKL6Dng+vNCV93UrDYrVisplIGg+ZIRVuViLY61aZrIcju7WGSsTeMkm3iZueS58K9jSLNzOyfLLG175HA4UFXlI0buhOjJ85z1es3t7S1XV1cz4/HIdvvAcrPkcNyjdaJZL0lJOlfKqmK/O/DkyTkpJe7vHyjyhmEYqWtNURQsFgtAii4XFxd8vJZOyjyXPIfVasX5+fnjQeRwOApTf20oCnkd6/WaIi94cvmU6+trVqsVt7e3vHr1esaVaK6unnB9fT0H4g6AYrlc41ygbTuKoqQsS548eTqvtT8+7wAgyxOrdc3kRhQNeVYyjVvclACLMjllXslh0GjcNOHGaTbFSJBrP0jGQPIRN0pnU24K6qph99AiwYCRbujx0YMSzFM/DHJwNJKFslyuGOciip8Cw9BTFQvOzi6o8objoePF82e4ybFrH6iKnCdXV1RFxcf3H5k6z7OrZ1zf3OCGaS6KNRgyilLmX1nWVFlBHBzejTxsW24ePjJMLc0yZ1Ev2Q0HktFMPjBMjqfPnxKNxzFgjOJ4PAJy+O0fRs7ONsQgn0+zqFguNgzDxDgEOfz1O4q1oe0DPrTSAaUsdSpYpJKXT5/x5dULXlw85axas6kWaKOIUbIaaCp8fyQjYGJkGibKMRAmT6E1IRcjzC9fnPHl0yfct55t77nZ7eltxpMXr9nvDuyOPWfnl1iVoVRGvdygTCZBa/3I9z/8gDKGvKpmPNPA2fkZMQTWTcE3v/krnp0/58n5Ct/v54KiA5ewych43t3Ag8Z5RbFcEWLCx0DyA9PoQUUcE5fP12yeF7z68oz21/cU1jC0e3SRyHLD8dBxf73jl7/8U8yN4s3bNyw3ORio6pIYEkZrpqnnanOBJmKLDI3i7v6O87NzbG7I8oyr80uOXcfkjwyjZ31+SdEs8AqS1pTlzOWfHPvdA/3Yo7Ri7HoyY7BljjWaZVkzeM+6qhiHiZUpqOuKomg42zS0+wP9ocMkS3sYqOoMZyJTcsQUqOqKGKXo6L2nbY+cX67QBjaLBcPQE0KSfUbwEAKF/fHzW9z3ac6rmruao+yvRMAx+GkkdBE3BlIAayxlWTwiU8rasjlbsm93PF01BDcSGWnWuYiWuWLqPNokrFHkeSbHae/px4knlwu+/tlr3n18h0+K5Saja3uUyZj8gMISA1SlmLQkFFzWk2EcePZ8QXv0kocUpBh9dr4gxsCLF0/QKuf7/C3b+4P87pTIsmI2XUBdL+jakXfvP5Jlmr7vyAvD+UXDOLWE6Hj67IKbjztevz4DJrb7W/b77WMAnTEZy6Ug14ZR9nBFIYitlAl+QX53YHIDhIjVwii+UFeoJN2abduyXC6xWlNkkv8wuZG7uxsur84xOjGMvYQgWsPkJ96++Uh0Iz//6RcMg+fu4Uiz3LBZLbj58J7Ls5z7O0deKaKGqDVu2NMe91RlTtdKKG+eWaxJTGOLGye0SuzbLXVTE+LIOCmqusJkYhxM3iH6jCfiQSnJVxvDHOKbJOR6t+OHH77DzllY19cPNMsabQLdcMQYCacPwVOWS77//i1aZ2hjcUHyQZRNJJVx7BxRGczclZ5XOf3QcvCB0haMbmAKJZOSc9V5c8axa8m0YX12xuA8wziA89zurgluoChy9n1HWUGe16AMymQkoO17NquKw+5eDHbjA9e3B5o6x6aMYRhYr87Zbh+4f3ggy2QvOwwtZVFhvWW12vD2w57lGnxwuDHw5Rdf8NOf5AR7EJe1dnSDrD3GRK5vetbrNRdn53+ne3dSkouWTg7xk1jBSb2ASJwdsxbmru4Umc/DM3KFkwtcvuMT0uX0w9Wj9nD6++/p75+e0WeC7Rzw+Nm/0Whxns/ZDGk+9xmjZyOjfhRhlIITF11Mg2HW1cT8pFQiJcmFcZNjiAPjjEEjacLpd5z+pCQdknOXjYue4GfzXmbJU8Fms+J4OOBN5OVPv2S5XJBdecaupzpfcPOvdhz6I4Mb+f67d/zZP/j7RKt49voFXZioy1oyXXTC5JmcvUJAGyuo5tmx78Zp7vwp5rOB5OJZa2fhPJKQx4yRsOWTyfAkunvvyf8WnItWmsjsvDd6drwrPvm1f+/jIoTA9fU1v/rVX1GWJR8/fsRaMYFYmzGOI1VV4b2fjY0SIl3X9SMbfRiGx6+dxPSTaH0S2cdRMpiU4hHlImPj9HocNpP7UghybwRx3Ycg2Rli3MpQJptztoSBf0ICPXZJzANOKz3Pj5OoLIMxhr+lMK0+fe+J96+QgshMwxFBPYrhTgxvJ7FdXtcpcPSkzX3e3/Fvu/5gET3iPk1G9YldI3p6fCyOfEpEnSftKV0YJYufkpuauNiViGzzApCSIsUogYLzYI0xEVNGjFbwF/FAjAdSPFW8jKTTxtPv1LOIbghRE4MW9EyU8IIYFClZQcykTL4vaRQFKWQEP4ccJGGahiDOe6I46nVIhKAhFugR1jS8GC8IxQPHY0fbeVTUKJ1jtKHMxMFZZg2ZzlFJoTEYpSiyDKs1ITjhJ44daWhR65y0eCKO7SBBWWl20KdZ7JlCh8JibEFKnhBHnJ9QemTbfmAKR3xyc2uWY5wONNUGa6XlDyykiFYRa+Tno8AmI+TeomBZLlAYirzEZJaiLlHRU+Qbrp58zX68Jc+0uFSLM5pyQ1UuKExDXZ1TVecUWYlO0BQ1rW9xU0DbhM0zMFYqkirjyZe/oFycIbxg0CZHFTXJgM4a2t3u0c3sg2L0CRdg+rzEOVdcZZwo0FI5BQQzMgc2tAFue0WdEpOBDjiPE4tQMDjFGCQEVD0uEBLQ4oJUzCIioqd5zAnT1p5KavgoAv8pWTslCdLwWqHiqYA0q/1azS5omDxEnWaBUJ6zCwk3419ClPZw7yUQdQpz2SN9YnL92KvIC0HtJEUx875C8Dg3iKCvDVrpx7ATpRTGavwcdiuu0N9fgEE2/zpJuE1IAaOlPSmzGdMY0CHhwkDQgcoqTGY48W5O7WjTNNCmHSbLZjrSKXQlzBV0hVWKTAubL9cZJoeQzCOXPgZmYfMTkievakIMOO8gRaoig+RRPicz0p0RFTL/rMGYRGYCRolLRly6csOQG7onqohRhqpoyPICoiZ6Cf/AiLB8+n0RuflXTSWhuVNiGFuUj2RFRpWSsNznSu3xKBzfxQKa5QIXHTFF8lzadUMMcwVcHAtRqpR0fYf3jrZrKYuCPBMsj4RNifiXZyVVU9Meew7HlrKELMtJGLJyweXVcy6vnrJcLimLSkQvqyUnI3gS0j6tkCyIE55IKxGWnXP0gwRDXt984He/+5b3797hpxFrzIwjkoOXVjy2oRmjyTJLls/YqeDJ8oo8l74En9z8u+TzyqyVMGbSHB6myU2GVYbK5PJcoiM3OUl9KiDGIG3iyorTSGlLlqTbQiNBjXHqSOEU4qOkFdYmXDuAkfAvrRQk6VYw2mCUxRhLlTeURcnkO6LXYC0i4mgJyLM5TVNjjJpRFRMxBnxwhORkXvzYuT0H99ZlwX7XynPKASbyzPDw8IHFouRw2HNz+4FpCuRFTl4YQuzpDgN5XjBME6vlGW17pCxLXr58RdcdKcuKruslBGvu6AleXL+nzdvkHKN3mFxTLTJ+ePtb0BPD1DN5wVb1w5HzywuyTGOt4dWrr9jvD5AM4+h42D6wXFYoEzHazo0Pko1yt31gdb6hmzpGN6AtbM5XHLsD2/0dV2cFIUiruvMDympGP+LGxGHo+OnXPyOqxHK1kiJ6iHTHA4lEmWWgJQ8iJE82jyNrElpFJu9ESI6C6xIHVM5qvWa9WlEWGYpIr1rpenEBqwx1UTGUlbRJt4ah79jv95SLkiyrQRfkuYZ0YJrEAamNpjA5Ej594iZKV4EcOtSnNU4FygyS8vg5EDuhMMaTrBR8dUqokNDkaC0Il6quKUpxZOe5RdvEME6oBK+evaCqKq6vb7Em42yzmTsYHEWRM44TITqMzRmnkUik6ztiCvjguVjJ+LG5ph+PXN9G1usztLWEcSQCm/Nzjl1LXhZkRcYm3/D+/QdOU6suFvgQ8H5iGCaaRYMPPctVxbHdcX51yYePH3j69BkxjQxjYLmq8LHHKMWx3fHs6XMUmqdPnwGaaXKEEFguVzPSLnF+fsnd7S3BS3GwbYU3CUhLbmbZH/a44FBGcWgPZHnGOI0UVcnT58/ZPuykOJtLMDfA9c07Xr58gbGRAs3Vk79bOFlZa7StcZMnhEh7HNAqp20HYhLUQ0TRdR3TJJ2JJ5e9tuBDT+wnFnWDToplvYIEQz9SVpGzzTlxEdnutwzdEWMSh+6AbU6t1hKgy1zgCjGR8gI3edzkOV9XFHkuTsqhY7GsiT4w+gNGQZUXbG8fePO7DySvuTh7Tm4y7h/uRCic9zbrzZr1+ZqyLMiNEYFzkvE1ji2TG9CHiVxluHHCa3AhcOw7Ru+pKoObpAO1KC3eB/I8Z8Tz/fe/Y+hGhlFE9sOhJfj0GHQ8TC1tb2h0w6F1qClSkNHokpVu+PmzL/jFq59QJku3P8AwkmLAx0nuFyRKI2x8c3Ym+TNZye7uHt91TMPI8dgSksZWS540K0LfklZLstUZV5dX/L1Xr6iWa4bB8fCwY7M5ZwpgMylM3d/fUOUFX/3iZ9RlwXjYcXN/z7s3b7n5cE1lI1O748nlikVpyBaXco5CYazl2Ml69+TyimiNdCHqnO32QNje0azO2d4NfP/dG9rhwJOvrzh/XqHzGbOWAtu7a6ghas9iU87YgcTmbEXbbtBZwljZE/ddi9UJoxJ+HGnKBff3D9IB1WT89offkNlcOhlMxnp1xmJ1wfX1LXW1IM57wxAmKluTGc1u37JeL2hiKW3nczzLkBzD0LHOa15cPKM+2/Dr3/6GbuwpcktRlpxvVqzrkniV8GPgr/7yN2JGUOmRj6tJFGVBWVUy3+JEUeTUdSkdZF7h3EQ/tBidUeUFdV3/6Lkd8MIT1iAHhlOreiKFgM0ykpkFh6SxWjF2DhUUr19ecL/b0/YeWyrUGGiPHYtVRTAT49RzflGTnGKYPHlhSSYSkmcaAssmx2hNXnqO3R1Fqej2jqgt9aphih7vZf9UzMLW5ALr1YKH3T1NnbPfdzS1ZbNZ4EdH13cUheb8oubwcCSEif/mv/0tm1VD27asL6ST6enTS25u7kVIUoosk3vq4dDSLOX9tFbTjwMQORwPZLnc64oq8s2333BxcUaRZwzDxDRNLFdrlss1VZGzv91ze9dz9uVXhOixZUlVlSQkzDA6hy4LYlRMU49zI1VZctsdqcqc4Eva9kCzfMpyucK5ibubG+5vbyjzbBakFF3X8+vffM9uF/n+7Y5hEEzFOE58/HADboubdnz5xRnPnp8z+p58LvBcv70GDEoLXjMtl3g/MQ4dq0VJ3x+JwdF10lGJyqULThuIE2LdCkzTiDYS6D5NYrbJvcVozXJVE5NjuVhirOH29ponTy/xYaKqGrp+zzBM9MOAsJc7iFJw77pBMC7a0s5hqvfdyKHvCUlx2DrqdUZhS5SC9tjyk5eveXJ5hfWJs/WGRb3Aj4FFU1MUNTpL5MaSlKYMOb/719/y4ukZdRTmvjUWkmZRLXApcrZZc3d9TXQBlcFiUVID/dBz3O2pqwX7/ZayLKiqiuOxxXsnOT0kilIKJxcXNS9ePkGbyPu31xKImFtMAQ/bB1IKXF6dU1WG3W7HYrFmf9yfJuaPv5L55L2d3eDyn/AoKMaTFqYiau52jUCKQfQkpeU+ObvU08lonj4J8UJJOCFjPgmNJzWOR33uk7EvMZuiZqzG42NpVlrmn5nSJ9MfxDn0Mc1M9Vn0TwGVAsRAprWYbpPgS1MY2O9vOCTHcnmGi538fKXm8Z+jYkIZwcYR/WwmFexfZjOqrMAsN2gNB3+kKpdEr1msz1F1RPcDOqx5cXygqmvWmwv+g4snKG1omgVGq5lXLpvnLM/JixrnI8YWhKSwWYYx2WOG0DCNZHlJQnIaYoxkWY5zHpXAzqwejXosMugZTqFOVI9oUHMOVkJy8GwmIaRGHGKE4GZtZRYwPn24gIRz933Pv/iX/5Lb21sR5/OSruseTSl931OWJcaYR3TLyXF+MrGcWOkyPpgtcGJ89D4QvSdpg1GaECTXsshztl1P33dyXvVgjJxrx3HA2oyzeo33idGNoAKZVdjHwrIWo1I6IYBPLvHZkMZ885u1ZubxdpK7fo/PPxtWP3VwiMas0bOTfS7GqPSod4r+ePrZzHMnzt0Sc0Dpv+f6g0X03wPMf/ZfGuGQxtML58S7jY8vmiS2efn858ApJd+Hlr8r5UlJo3RCJ2FPKmWwnKpQJy5smitvZh5HcXb7+nnBkI/+hMuI8VQtM7PwLuKmuONP7TEnMV5QMyT9mUgvLMcwfz148HPAanCJ5VcHrv4k8mfthqEvaI8tw5DwU4afSlRco/UZmT7HqCXEEq0KtDKUWYnV8gpjDPhpIriRstBcPl2jmxWjUxS5DJCYIj4MTG4QwYcMbTKc75jcQD8cQAWm6R6NONSkJCgCYlJOhCfVUeYbVFLENII2ZFYC0LRWIrTZCqstVbmgrhcsF2tsuQK/QxWGdfOU0hbkRUZdrGjKcxb1OXW5weicIt+Q5Q1GS6vloqxJw0gXHVPw2MLgx4jOMurNOXmT0TRLEcRO4lYmvNHqrOLyqz/m+//+v8JHy+A9Y1CP4rKaJ1JSp+6G+aahIBmNMlZCf4IIoFPSXA+JzEcoEnUGByKtDxxHzeDjY/VrCop2kI6JKQj25VNBTD0KyEZrcTUj486lOL8WuRW6KH0aZhZ0Y1REBUQJvQwR3FxICklJexwW5x1TSEwRBp+YAvQeehcZQ8JrCFqTih8fPAgwTo4sK6V2pSUENC8lvLEo88eWHh+8bM1iwEZhh+Uqmxcz9bgIS7uPhAxnWUmR5QQjTMrgJxGno7gjpmnChZ5kEpVpyJSmtgUYK6xGF8U1baTSrTE4HzgOe4ooBQSjCjItOQYhnEIcpXpvtThRykwwDCGMhFJRFJbkIDjZiLf9Udx8mpkNrgkxYVRGMQfwhDiR0vRpA5JOn5nGRwiM6KIUp3KWkZkCN4rzbwoR7yOT81I8nF2UeZGREkzJYSwUpSUGKewZnTEMO6ZxIjjhNqLg+YunbC7OMFbTdy3F7Gac3CAdJtbIoc8Y3DTghomqKGh7h/cFRZ4z7Dr2hwPWFpyfXVDXq1m0kFbclDRVveT88oqrZ89YLJdSWS8ET6AA5kLSaWyEKTy26ymVzWw+cefs93vev3/Lt99+w7fffsP24V5a1bSSQp60bBCUdBzYTNLYBREkwmqWiyhUlpaiKhjnz9oFR1XNXHg/EZNCG3EFZLagymoqWwNJOmKUheCISu4pbhqZ3CQoggh4haUgIezRGCd00hSmIleWRbWgrmowiW5swbs51EoyEE53x8yUFFlJXTUiJPqJmNzcCWtm5rYE/VkzF6BClPHnxP2cVCTTP97NNvQji2YJJG4+3vPVV1/g/EDbHlCV5e7uLX1f8uTJEwzCMXUuMAx7+WyskUN2XrDfb3n27CkPD1uePHlKSvDyxSsWzQLnJ168eMHvfvct79695eLynMvLS2IUpur5xRl//Zu/ZHAdN3cjJpdqVkRxOPZkhaEqMobhwM+/+Bn1oubP//VfUlYL9vsdq9WSEAfGqWe9PifPKiJSUNoedjx9donScPHknH/5z/8V55sLrtbPicljMo1W0B62DH0PytB2kxRyM8XzL16S6Yy+7amqmqdPnvKw39K3O9bLmqZpaBYNzvtZMC5JSPjv5OT96rqRIs+5ungimJGmIaVI149MY8c0DWidsAaKMsdNGUVuqaty7liApmloh57JBZbrNePQYbSZObsapSzaSDFpck4CsEOUTa21RD9nbcRImQmPEiUuYj+HaGgl7hiMbOZTTJAC2gjCIKEpyoqiLjFW3GuZMayaNbvtEYVFK/PIVzwcDjRNIygX76mqivPzc8k0sLIJnpynagqyPCN1EWXEJTpMA5uzKxKa9UYwJzazlFRst9tHfE2zbKirmkznwrWf95Zv7t4QCTSLJU+fPuXDhxvavsMUOSE5+sGx220FI1TmXJxfcDzuePL3/x6/++53uM7TNEu69oi1gisiaTbrDX0/UpYNfT8AifV6Q9M0PDw8iBiRImjF+cUFXd9SVsW8DtXc3t4xTY58Rs8sFjUfr4MIyPuR9++/pygEvfL+3ZsfPbcBikKTZTW9mpimQIojKcI4TCQdQUtXk9KdIGmspW0HVFKMk5PQ26jZLDaYPMMuMmGTH7a4hZewL5vz/t1bhuFI0+QklWi7jsViMZvcEvkcUuW9J89y3OAo8hyVAsf9luACwXu6sRdk2NmKbjzycHvH7YcHDvdHTMrY3j3w4vlzrm8+4P2E95ayrNlsljx5fgUq4PxEUWnGsadtd5RlTlnnFBaqOqeqCsYpsl4vKKoC50fUoJnCRFkVs6M1Ya3B1BltO+DCSEoSIrvdeoxhRqON5IWmGw7Cby8rRjeBUqyXa3729Cdsyorf/fqvWeQ1lS14iAljoCgsISZUjDD2mDDhug6VxOxh8oosawjmiEmGqqjoRumW09Zwub5gdfWcH354w/Z4ZHN5xXpzzuXFOcdhYP+wxU0jfhqIbuDrP/op3nvevrsmTSN5UfDs+TNeP3/G4eGa/f0H2sOBs1WFNRVKSSF8GiUj4NgeuTy/xJEwWS3YKaPYtfccDkeiU9x+vKeLPSpzDN7R3gUxgUwOo6EfHFpZ8sqgFxXaREJ0XD25pB1bItKyPwwtVVFQWEtynu7QMUw9Jjdg4bDbsazXKJVEIIoiVlxdXFIUFSg5d+pMozNQOhCYWDcNq7Nn/Po3vybLLWVe0m5v8IPD2ZHnP3tKyCx1WTMVHWN0PFxvsVbz4vkzLi7PsbrgYd/x9s17sgKUhaquOOEVrJE90MXlBYfjAyGMNItKXJXe0fc9eQZ2Luz/2CvpRF4p6diW9tPZ3AMgzsLlJscpyewQfGrChYlhOPL02ZrvfriXdXRUKDxGe9ZXBTF6/BTIbYHSUDYWN00Ui4xum7A2YkyBtgmlpUtRKUM/dtSLnGQEJZXZDOc8u+3A8xcrYlSMQ6AuTxlT8PHjHWerBbd3Oy4vl4zTjqJKDNOexXICBZuzM5I6sNls2O2OtMdehP2kWGwWYrrRhqqsWa4rPly/R2tD09SoZChzw8ebG0KYWCxz6iaXs5pShBhAee7u33PYdqwWKy6WNdpIF6iul2j9KWCzsBYfJLz2YXuLc57NuqEoNMN4hJ1HKTHgbDbnvH/3jof7LXVV4MeRcRyYpon3H+44doEf3txxfvkltlR47nj1xVesq8hv/+q/5+b6muhf4KcWa0XFyPOSGBwhOrJMczwcMPP+N/iJ0USGrgMVqbKCybk5dFbY+Cn62TUqj4UoPOksyzC54Xjco7UYgKqqZhhbYp+IyXN7J0gXiBR5yTD0lGXNYb8XQcpP7Lb3FGXN7thjCkOzqrhvd4xETG7ojiOjcyhTcWg7VssSHRT313c8bc6wtqA/DDz4vWQUDYIIStqgbKLSGSkqMp2xe/jAmE1Yk3EcPc++foHNMq5vP3K3u+OrL3/Kd99+R4w9w9BTLypsFmmsZCA9bO+4uLjg6uqMECLv31+T59K14buR5smSZ88uWK5KjscH1uuKJxcXnJ1d8t/9q39GP448fXpO1zqm0VEUDSQDynDo+r/TvVsjIcgSajg7z2OUrnU+N6DK308d6XOzr7h3TxbazwS/T27ak5AuhjYxun/WA6/UjC36t6xRas7jOxkV0/yMTmb108NzCOpJ2NVKBNiTkB8JM1YrzebN+XkEQaQqpVnUOS5OkqkwRTBWRGuv0QlUCqjkiSqR5VaEfVVQkM9Cr6PvW6yaQz9TRlMtBW3a9Zhx5J/8R/9TyVOZw8WttbOx1KKNRVuLzUR4lrwXjTG5YEWdmP4SUNc10cXHAoee9z3eOzKbSxd5SoKjjREXTpqmxubZHEiq0VX1aDyMITCNkwi7xszGyM/f6MdPn8/JC5vNGb/+9a8fs4WmacJ7md/TND2Kyt77R9d4jFE6SWesi3ShmEcjpJrxxHoWklMU4X2aJoq8wBFoj0eyzQaQTnGUJYTxURPRys3PadZmlSCJT/3pgtE/abBiTkunApKWjnRBHpvP3oU0C+yKk6CulNAPPmWviV6q1KkfdjZvKzVnd3wK3n00g89XIJCSmgtFv/+1f9v1B9/djc7E1ZvCo8VdK0umhYXpk1Q+1VyZi2oOKlCnCE3Fqe4V5g/kdBDSczBB+jQjRTTXj3N2rkpEVJJWFaXDpwmdEnF2/801BknxVXKgVAI5kh/0WeKxmll16fG/tAjonFoHPqFC4unrUc1VQC1iaArEWBLCM1w4Vcc8eEUMBucygi/EUTxN+Am8T4QgNwliJHmp3ARfEL0waJfLCrPICXObbJpdiZOfcH6Um73rBV/hOoZxL9U+LaJRSiL++eA5YRBUEr56hofQkWVLWcwYiUkSh2UTJsFLZVWxWV6wWV+yXmwwNicNHwDP5eXXlG8b8jyjqpY09Yq6OmNRPwUgyxaCOogJrTIKawlWOObOO3QUbEJmKlbliuX6CUW15HD/DtPt2Fx9AaZABYe2C57//M9Yf/FL7ncT0+6ekATrkYJ0O0gwxic6eEIKIDFBnFumFGDmytN2iiin0TEwALdTooyOu9YyeBlFEegdbA8iQIwuEsMnvhIwC8cWoxUpSmUvaakjB9TMJpSbTpiLOi4mfJJbolZzmIFOj+NSFu4ChWEKPWNABNqYGD0MLjEEmHKDMwpnFbvi3z/Z/13XGALJyOvLtJLCgzUkr9EzviApLwFGQZjFQYkLxc2VTWnP0Y9ielEUFLagyisya3BqnFv7J4KeK43GgNG4qacbjtK+qzTLvGJSfp6DAZIEcdg8px17/HTkGAc8UJcVIU2MbiB6TwwDDgk6iVGcxDmaXBlcdDg/MkWwKWPyjsENGAsEj48j1pQoI3Omykpxr4V5g6MjLg5EEt4zh9hqlLbYQkIDA56Qgqx6WoujOwSGaSREOTyfdjExBlxIhCg8w2pZSeATCE4marI8E/QCEnQ13UycX55x9fSSwfWyQYmJLLPYud31hEf41JJlmLynzHNiEvdenmU4PxGV4t3H95TFnouLZ2RFwTh4qrzg1evXvHr9mtVKgv0kQFCQKCnOhcqUcCHgfECcsEawGSFKOK/3tH3Pw8MDHz9+5M2b77m/u2Uah7kTKZFZYcxluiD4iRQ8Siuy3JJZQ1UXNE3NNE1YqyirjLIuKFROP42kKT4iYKSKb8mNoHPyvKbMGgotzjgVHagMtMMrYatLYdZisLJWoh+7J/z8WZZZg9WWXGXUWU2mMnycSEGESW0K5vQEUghzSHVGpitSTPTDkXFs5XucHFCtNYIH0bLJERwK5DaXAkWmSWTYv6Xt7w+e2zNepG07Xr58yq9+9dd8+eULqqomhMBqtWaaJEw0psTV1RN2uwe22wfOL85YLldUZcVue2CxaCiKil//9W/5xS9+zje//Q0PDw/8yZ/8kjdvfuD+/o7b21uWqwWXlxd4L5u4uq44HPb0fUdVSfvgT3/6Nb8avuF4PPD61RdszhYoFdAmUJY5b9/+QJbJxm5yimfPnzMMB7a7e1JUFEXF9mFHvazZH3bsHrZYK8zti4sLuq5lp3b4AS7WV6hkaI8HvHMMkycmw+R7NpsNmbW4cZpdRhWvXj0nvQ2MbsCaRIziop2C5F90XcfkHF3Xsd3u2G33jKPj/Pycsiipm4rggwhAJNmExogPEnDK3DWiFDg3AgmbaYax4+Hhnrwo2e+3GD1nuqDn4OsZwTYjZiSIKc0ucoMLg+xJSOR5iTZmDiOOElgcQSkxIRgra6RCwqC1VWSlZbGqKeuc1dkSFwYOxxZjNWdnF3z33XfEGFmv13jv2e/3jwzzaZrQWjOOI1dXT9nttqQUCUEyH7Yzq9FmGdZ6tLaPr4OkONtcsD9siRGqqp5Z5Csetg8sFtLJ0R1Hhv0B50aePLni2dPnTG5it9tiTcFiseL9h48URcHTp094uL9lnDqMFdcqKrLbb/n48QMpRVartRzgFg11UzKOXoSGeb4dDgcuLs6JUd7DE04oJdlgn4oFzjnOzy/RWuMmz3otHPaiKDkcDmw2a4pC0DjiXHQyfrc7hn780XMbkEKXthgjLbfj4HGTw3lPVFAvFjg3kZeSyRC9QyXojgMuevG0BDjkPWfLDUVZ0bWddOvExH5/oMhzbC7dUsmIUBfnA1tZ1fggeRV939N1HYu6gSQc5BQdd3c7NAprMvpeeL15adEK9tsdJmlW1YI0GYauI4aJy/MV1nqU1izWJc9fXZHXOQ+HW/ATlTX0w4HDcYuxBevVGauNHLhe18/ZPrSQFIvFQrrZjg6yiDWC3TgdLI2W8ODNesHDdjs70ffkuaWqLT6MNHVF3w60XQ9aEa10TLX9ke9/+B3jzTV1VrL1YJNmWZVsVjXTkEgBkg9M7ZHD7Q3dfkdwAWMtNqtxqcBoS7E+x9gcHw4kNJcvX/Hk5Zd0zvMfPPkTDtPI9f09N7fXfPHySy6fv6QPkeNhx3Dc8tMvnnP9/ntCjGR5LqHucyaKHwZWmzN+8tOfopWbkXFyQM7zfM62Gdnu78TFT6DIa6ZpJMszFnrD0AdsU3C+PGO/PTAOB2xQfLzfUxQLQjKUeUGapLW+ezgyOYfVOWCYnLicu/FIGTNyq0neSRBzNLS7DlNqAg6lFdWywLuJY3tg6gcqW7DvW7TSaARb1TQFx3ZL2+0YlcXhOI4tRajJqpzcZEQfGKaBOst59fwFaopsH+65vLikqDN223uGw8Tu9gEXIwGN0hmqyKjO1lS1ISlhwhZlibUZ3smetMkXHI5bum4gyzP6fsAYS5YVEowH9P2PF9qUTuSFlaDs+w43zfMVgEReGMrSQPBYqzAzZiSGQD8MrHXDi9drHnYHTK6JCe5vHWeXBXWRczhMLKol51eam9sdeQ7JBOpGU9aWsjSkOGEzxTAEXNAs1jl5KQjGqs4IkxgdtDLz/tIyjRlKWbIs4Vwgzy3Htqeuc84vltzd3dKUEi58flESnOH27p7zK+mSadsRyRlRVFVNjIm6buj7lu32QFWXBK/4eHsrgc6lBOQNQ880jWzOSo7tnkWzknOzG+n7I+cXZ2iVOGtqtBdH42ohGRgxeLrRkRtNTGIiidEzDkeyrCAlEbRTCrx//wNlVZG3DdYWjJNj6FpIiedPr2YEWyCpnBAmuiHQv7/h/MlLfv5Hv+D+/oEfvv2OTVNwdb6mrhaM3YQqNNZofBqBIM5UGwne48ZJAo9NxDtPdB4fHatVwzBO5FkxdwYY6QZKSJ6NFtRoUZVktqQfJcw7z0vadng8I8wjTsxDIchrHweyrGC7feDhfosfA7e3W6zWdF3P7f7A1fNL9oMjZVbE9bzmF7/4gh/efMCFQJ7lXF48Ydhv+flPf8YXL75k2ndkZFiVU9cNPnqyrManhDE5WiVWyzVfffkVN24HaY8fI01ZE8bAZnXGsTrw9NkVma3JbEm9XHBz/w70wNNn57z/eIubPNpEju2elBrJo6kqXrz4gu+/+571smG1EuOCdDPJXL27v+Ff/atvefrygtXynNxmXF9/pGkadvc9z55fMriBYd6v/9jrxJVWSj2uySDnQvnfv4nw+JtcaPU3Pr/ZQPiZ0U++Cz7HYajf+yOayN/kTP9NgfFvfPlv+Z2fESo4BZHOz1kUS07o5BMn2zs5C1trKMqKiwvZQ8UU0Vr0kxiFSx5DmJnVcqbUxlDVtXSjzzgRow1N04jTOgti5HJechEWS3FkFwV10zD0w2coQUFlKS3hm6dct5OmMY7j4/tz4r6L0G4eH5e9n8NWuey5kbEhe0nB8hojv8+aTDSmENBG/d45Xf5tfPzdj+D8v+VSWtMsGs7Ozvjuu+8IXsJ++/m1nQoyJ6b7Sdw/OdFPr+dzAf2TkM5n4zHO3Pco1AJrZa85jmitJfPKg9fyM62VPXSYEXlJSeHCZpnoBvGT0H2ik8gVOXnSP2nGJwb/aZyLAC628zjryCd89r85Nh/H4/zYo+CO/r3XeApP/RxN/Lex1//m9YfjXNRpSpw2X/axjQQFJhXEFFBztUlxasOcBx5SEYizk1wc6RLqJmykU9UKsd7P3LfTm2KUvKFJBR4VKBUhhcff8BjsyalifwqU9HOF4tS8Mrvitfu9xz+vpJ2KAac3Us+PCWvnxKZSj78vxUg68dfn9yklcUrH6IixJQRpEQ1RQpd8QJKVo5Ww1SgOVCjIdUluStphQW2/QqX5HVPy/k5ODiviahTOeUgTKXmMsTR2jfMO5yaMsXg/YbQIRJk2WA2FUXOQh1SdQxwIGDJbzeKFochrVssL6rKGzJKKC3A9F+dfsl6+IKaOqlhTFWcsqkuq6kqEA1VidEbX9yg1oVMiR2O1Ics8k+/JyxztxYHtfMQ253Qfvmfa3ZMXDQt7STIarSYuX37Bn/zP/nek/Cn9v/wXxObIcnDsj63wN+cEZB9E0AtRQmLd7PgJXqpYRmkKpeiJdM6hgGDg4wBEz+6ghLNuZFwEpWjHxORE7GQW60OMuBAJYU6mTuLg1TbO4VriBEizg0YZRdQJdRL2ScRTynWUn6kfi0qndG2Ni4rTiH+ci0ExoZgKTZuBz2DM//2T/d91aWMlaCxEaqNJmSZINQo1u2ATTtqo5gKXsfl8A074ubsjBo8xspEvipIyq8lMLh0Q8hOIIRKVm6vvOcpqok70U4fpNUZpSJ6YPC4lptGJeFnJnKjyimnqmVxLSjmFLTA6EfwgHSIpMuFkXkcl+JU5CQEFTnt88oxe3BJJJTDi2JxGYTzaohEueF6TaU/fChLFq5ExdHP11OKjxmYleV4K3mY8Spu4kopJpjQYYfErRGzNbIFKUVK1vSOGTx0gWmumUeaqShnRK4qyZBo8fScCWbtvuXu45/zJxXwQz0jOo5SmrhaUZUnft9KmagxgToXbTwUmpfCzkzIFh8LQDT3Z/sBqdU6zqHj9+ku+/MlXnF+ci6Mqz/FBQj6jkwyBGOb7gbEUJkebDKVPwbsON3nGaaTrW/aHPbe3N7x//5672xumfkQrKHIrhxMiRW5ZrVfo4TgLgZqyzLCZIs8NZVWDciQj66AP4sYKQZzwSs9MaitIjqIsMSZHaStwoLn7KM0OiRQjMXo0ghUwUYQHlCHN741FnAGFyYSvrjNI4J2nGwf8GLG2pMpqrFEEJ1y/4KX9OybELR89Pol7KcZIYQoCidEHPvHWpaisjSUvLT5EUghSCP6RV2YL9vs93357z+vXL/jqy59wd3/NxcWGYRgpCuHTtu3I1cU5McB2K62qu+2BzfqcYZhgFly+//57zs/P2O23aA1tu+ft2zf87Gdf89/+t/8/Xr56xt3dHcfjnpQSm82Gs/MzHrYP7HY7yvKc5WrJdvvAMPZ8+eVPWa4ayjLnf/jrv+Cf/tN/wre//YYUDavVktdfvKTtOtlSacN6dYZzCa0s0+i5uKqYpkE6CUJgvV6zbNZ88z/8lkJ3PL94gSKyu9/SHvcopdjtjlT1imlwXFw+JXrP8bDH+5HVsiH4iY/XP7A/7ujHA5nNKcqKvKhA23mTPGO6nJ9dHZarq0suLy4oixI3TQxdj1IJo8UY4NyIdx0pjLi+YxhahrGj7Q4cjzs+Xl/TDgNJwfXtDReXZxjl8CGh9DyPFTCPEa01ygvn+sTq1bPIqYzGWEuIkGUFWT5/j5q7VLIWY8BmglAr64rFuqFZ1eRNxhR7Ih5lAr0bmW4D0yTOtbbtaZp6Foalbf14bLE2Z9Gs2O8POOc5HluapmK93jyy0+tKWIxlWc6Ou0TTLCWcs15SViW73QPr9Zq3b98CsNvuePX6NcF7xlFCPcuypK5r/CFwe33D1eVzCaaKidevX+Oc5MSUZUEIjmGYkPwE2O7uGYaOVy9f0bY92mR8+PCBs7NLFssl93cPOOd49eoVkLi5uZb7kTvncNhxdfWUt29/4KuvvuRwOPLu3TvJF7CWvhvmQtTu0b3z8eMHvPd0nZO2YSBm8OzpS55cPf/RcxvEZT6NI33n6VonrbdR7p+TU+jRPx5nY/R0XcuiXnA4tESVYbUlTI7bmy2LbMm6qRlij1WGvu3p+pGssFw9ucCnmtEPBDzWCr/4eDyitHBwp2kSV+runqYqyTPpMupbKRBqJffqrtuTVElmNH4O53p+dcXhviO4iffvf8CYxHpVUzYV55dXVMuMY79lcgdMOuXOjFhj2R868nLB6lJEb21ynr+45Pr9NVYFuqElaBFonBMMhWRSCM6k6wZWq5yysuSF4XCUsVs1G7SRA73RlmQTu65lUgOLsuLd3Xt0HthcPWccOlSAymZURWKaIipMhCmSpsD9xw8sy5yrsxWHQ0vb9pRVQ7NcUuQ5m/WaY9vz9NVrdLEgqIx6sQKb8bu3P/D+/VtGHzl2PX3XMqXA+vIp7e6BUicuVhV//09/xrHrefv+A8d9h84zVEi0xyPb/RFdVDT1isJAmgb2uyPjtMMHT1HlaKXZ7bYUVcb24ZbN1cu55X3Jqimxcc0//SeXbP/Z/52yUNSNZbqVsOHF5gITjpgp8HzzjO/efUc/dPz219/xxU9eY6yl6wc5+2lpH/cpUOQVU+vpB0dkpHMt682arDQc2479CDf6hqIsyIAUA/1hxxRGqkVBjANd3zONkcF7ujgyIIi9erFge3dPSIHLiwsuN+e8+fYHPuy3vPqTr6maisk3ZG1L7hwf7+7onResQIDN5RXWiKs7RukIWi4XDP3EMAwYbeYOPslPKfJ67nwWseTYHtB/hxqZd4ZUR4oGzs9fcdgd2T7c4UcpzjgfyYqCn/7kK65v3pFpUFaTL0pS6dgPLc2qIBtHVrlBk3PcTfiYyMqALSL7w56nzy6Z3JHMZgxdwXJlIesJdkJrBbmmHRM2i1jjWS1LTDJkpuLLn31N391zsXggKzyt93gSpiwhwdAe+Pqrpxx3DyhjUMrinSJVOT+8fSDTJcfjwNOnlxADVaU4tj1ZLt3Xi2VGTCO9c6jMcti3PLl8QUaD8ncopViuarpdR1OVpHnO+amnKevZWZ7hfOD64UFExL5lkUoWZ2uszhjdQPKei82G5D1De8TBHHhnMVYzTB0mk0LEYr0S8SV6jBKUyDH03N/uoFDUNsMWltVZxWKzxuQFv/3tez788FeMk6DPnq41X3/xhC9eX5FnHsVEWWV45zl2Pbu2o2kWoCMu9IxOM/TiHl9Va5QuWNZLcrugKSwmapQGHyM+JDn3Ry+5FyEydge6cCTi0UT6dmKzXnA8dvgg62FTV6Roub2/4WF7R5ZZyrzguD/QHXuGMYEpKK3l+uNH9sNIPG4ZxbaE0jkuIkGtRSQHTDIE1VOtDW8//MAqK3m2fMIib4gp4qYBW9QYVVJmGtxECANT7Dm0eybvGQdZu9frC6ISlOM0jXx4v+enP/85F0822Exz6PYcjy3mbuLp0y/ohx5SZPcwUpUSEPuTn/yEN28+cjh4np8/Yegnqrri/fUHlNZs748sqoJ/+I9+ybv3b1lvXvL2zTXEDR8/PhAZGH1H23fSzfh3uKSr7sRwTo8MfxH0wu+J6Fp/Lmr/vmnuhE89XSeh+nNR/jOYxGf/Zja+MR8QOQnw6dHJe/r58lw+F/A//Zx/U3j8hMyY/xEkCe9RWhC+oMlsRlFUrJZPODt7yXa7p+8S6+UVWqsZURfk7B2dGFRJMz5F3PYhJfxsZPAhPHZE5nk+B91Lp2bXdbx6+QVFUXA4HFgui99/DSR88r9XPDgVObz3j4H2J/53VRWPjPMQ5DkJClBypFJK9H1PlmWzoC0oXG0EY6OVsMkFQytItUIZ0vx4CIEsU1JMeDQd//4Vg2ccev7D//A/4v/1//wv+Oab34qY7/zjZxLj58WZT4GgJxHdOffJDT+LyPLazePrO13WmNnIlj8y1cPsskedRPFP71uMUQpxhZhJ81ywmie9VZ20rgRppjfIGJoFbm3mcfNJD/40BU668elrn4/ByAntIqQT+By9dGKfn4zcp/fq9H59bvL+911/sIhulSURiLMrKqVEwJFimAMPJNwgzpPw9KFoDSrJAjEDFtC6AAIpeZjZN8y1AfnmOZzw9FclYaBan06QABIOIvb8gCCwT67k2UmuTnUwL4eWmVFt1Mxdnmnzjxx39VkLTPLzUzF8EtUjCanaCbMpPn4AUQcSXqrn6vRvE3oOVNXKQjKcWmvS7J4/VRPjXD1M8eTnL7CpQU8vYRjAZWgt2BA1/1tjDc6P5FlJZitMsvTjDlBkRYPRgcx4PhUOFJkpyG1JU67JTImLisFNEtxnrFRDkwT+lHlFQiZZyAIxZqj6C5juWFQb/uyP//f8+tv/B7mpKfMVRXWJtY2IaMifh901KbSkGEjBk+lEbiEYT/BbsqzmfPOMvGxwPvEXv/oV3/7FP+c//9/+H1hsrlBWoUKHyXLOXn/BL/R/QvXsC1RuSGg+/vA7vvnzf8HNmx8Y+0HCRqMRd2xMHF3ievB0bnbzoSmNfA77FJgchCnysWcOz1JQWNTMJTelJWojAlImnQNaMy+cBufBughGxleM4GdG/+NkRIl7C+SzkxHwmag5fz7IMIwkxmmkNYlhSoT42VxWihHojaG30BLxMaH9HzqT//arygvGIMJy51s0Sm5aIRBcZNEssCgO/YgbB3QS3r8xJRQOOBJDT/ITKYojJosJqxU+BUY/0LqO0fdoHSiiYFmymPApMUWHyhXHeCQFjwsDQ0r4OBdYyKh8TZNqKp0xIgt04QvqtCDH4JLD5gWBBhePkCYypSi0QccEjseNWh874hQ+sYRDROeeoCesjlgUla7IqNBZoDUdx+nImI642KOUIbenQlNBmVfiqqTEqoCJwDBgrGBhVIyYCLUuwBQ4LyGWQc8bAyPOHm2hrHOC0+RZxdh5dK/JqgKMoe0OWF1wd7+levOO58+fkpliZsQFsqRp6jUpJLwTZmGWaVxwEtypLXEOfwmAUknw3FGRgqYdJooq8stf/oxf/PyPuDi7EIa6Fg5amAJ9J+4Vm2ckrUTQ08KLM3PBRVyQI1Oa8MkxTj193+KGiUxZ6rIkjAPB+zk4RYQKHzzGaoxWEliExRoFyeNcT51V2KZANzlRaVTKsCZiUkBZxWQCnfFzcDKYmJhi5BhHeiYKLW51hxzYExMqSfhoqQs0GUlbgs0YUyCkQIgSphbTiDGaKTqMykheE5KmzFeUtqTWBcYoHBl97JniJDdiG+jjxJggKYvSBUZHkjE4FQlhgqAxVgJJfQrk1uCc5CjEwNyB9eOuxXJNAoy9Ybs7slotKMoa5xNVvaTvW5TOubp6TmFLPny4pq4kwHFygafPXvLf/Nf/Df/4H/+PePvmPevNhuADbXfkV3/1F+R5xvfff8uXX73i5asXaKO4f7ijWTTc3NzSeA+z46OqKhFclg3Hw4D3Ezc3H/lH/+gf8uHjW372869p2z03Nx95cvWCGGVTHILl+YvnHA4PvH37Fu/l3vzLP/4l766/J88tdbPkN9/8mvPzc4Z2nIuRUBYZTZmRmpLuIPd4reTwFqNgUu7ubmiPLVVpOVuvGMYDH6/fcH37kb4/sFyt2Zxd0CAdBHWzxLvIcGwpioKvvvqC5XLBxcUVZZnPrZmD7E2Cx3HKb5CF/HjYs9/ezS7zyH4vzv8Tmux4kPv4h+uPVLmWQnlmZ4dMQlgsRjaRSu5P3gfBcaicYXScrAtpzh0xNiPLy9kFk7DBYAtoFg0heaqmZLVeYAvD5nxFP7WgA9Wy5Hj9QLlY8PLlS3a7PavVirKsaNsO76VjyHsvQV7rDW5yNPWC3/3udzSNdKzV9WIWvysym3NxfkXTLDjsR372sy958+YNDw87yn7AGMP19Q2HQysheV3PNLrZJRd58eIlfd8yttOj8+j+/p7l6oyqqmiaBbuHa7JcgqgWyxVX+QXjOKBUwrmJuim5u7+hqhYUeSHnvJSoq5oHtcU5NwfaCrbEGMVyueTd+zeEMPH69cvZqSfhTavlio8fr0kp0bYtWmvu7+9nPMGWxXJBrsW9tN3uuL8/sNl46vLsR89tgK7t2e1a3JiIUaOVFhOEkeR0P43UTSZ8UyJZLuHenXaAOEhj9Bz2R3bNjjovGIae7X6HKjNA4dqR3dZSL0vcsZesCpMw2jKOTtzLtpeDtoocDkeaqsIoy9iNlLN7KaaE1YY4JghSYIwu0XYD0exph0Eyc0gUpWG5bmgWBUoFvv/hO4apZbHKyYuCdnek7UYihrpaoZTFhYlx6uiPWy7WF8TkOBwe6No9yRqKzEA0FLZmvToTQ8LkKW2LUh5rE8Yqrp6ci5N1UXHsxDk/OYdSlq4f8WkiDYmiqPj/k/ZnzZZk6Zke9qzJpz2e+cSUc2ahqjCo0QWqBxrVvKBaokk3Mula+gP6SfoFMlMbTTSaKJEUTWZSE0ADjSpUFaoqh8iM8cxnTz6uSRfL94lIoLuBrnKzzIg4EXtv3+7L3dd6v/d73q1tuaqvkUFwvDhMuDvl6W2LtD2Zyri+v0LLgAL6tsUozcnxCVk1Q5UFy+UBMcKTJ4+529Rp3ktkdX/Ntm44OT3h9PSYL795zqu3PdZ1bO+v8c7jXcvJ8ZyzsyO+/NXf0PUdRZlzfjjj7m5N27X03Y4sk3iRcXV9g4qWTAmiT63mt3d3KC05Ol5icoXJJZ311M0akRW4Hg5nM3xvOZ0v+PHnnzH5wNFMtih5w4dPHjOvZtguIpWjNDNODs9ASu7Wl2w3Oz7+/DPW367IjEaMXcJucMwXB9imRTLO24fUEZTrHBc9hRI0XUc39Nhg8SGy2e2IwnFzP1DNDEWZk03mNLuEC+p9CjArix4XI9PpjIjk+auXbNY7tvWWvm1oQ80w9GM3lmMYAm3nWSwNcTRYSKVp29RRFUIKSD85PeLVi9fYPoUmL5ZLmmbLwcExEFmt1yipsIMly7Pf+toWOiPElrv7Dbe24/HZEUcnJ1y82TC0oDPBtlnz9soxnWbsttC0A23vyJQk2shBLkchxqN15Ph0iQ89VS4xPdSdG13amquLlr42ONfx+ANFiAlRFpEcHEyS0zJTFEYRXEa7HfjpX/6cP/zDj8izKY1dY6Tl0WOV8oaGhHLo+4bJVONdztvXt+RFQdN42s6zfJQ6P1++uuDsdE5mU0dvZjTWRm5vb6imChc1eIe3mpODU+5v7xFo8iyj3tUMXc9sNh07HCNN4wjLUfAyU4JIZsCyKmhXLfN8gvQCVGTX7ljMZkgluby4JtcK5wNaG9q2SZkoRrPdbYkIimrCm9dvODkUbLdbiipjJhasN2u+e/OSRwdHTMqSeQlCZBwcPOLjj065uVnhrENKODuYkcmA0p680FjnGLyn6wd664ijLlBNqoQKsz3b7YYiL1BGIjpBlueJgy4kxIAfjXNaJ9xF33YoKZPA6CJFXiFioO+6kc1uiSRsZVFUCClxbkBridYCISLr7YauH9hsG/JywXa9ppzmFNUUt91wu9kgSomSmlIZFos5by/fYJ2jd0PqQmgiMkSeHD1lPlsydJYBy8HhknpomE4mCCJKJr1Gjrl4t3d3uOBZLhMCRmcZ290OqWEyKbBOpM7E6NluG4pyyuAifQ/1zmGUJpOe6uAUJadcb2+5v7jmoCr5g8/+KVVp+PLrX4NWtO0A0hBCxvNv31AWJceHR/z5n/0lIRjmBxlvL2759PMzBusYnOfDj578Ts/uvYCZwixHl2xMrl/v/ffcwklX+n7QJ/DvdKIncfydmJl+/n3x+8FxjPg7/+Z73OlRXE0C6950+ne394XIOJpp5Whc40HQlA//KZVR5FOKvCJ4wfNvnpNlUx6dl+x2G8oyGRNDTGvkOAq8yUiyz46yaV7tQxKno0JpTT8MSTAOiYPhQ+T45BRjzEP33D4wc3+sAwH1sJ88uLIfMuBG1/nDa5wj6OT87rpunJsns0AypibxPYTwEJgcgkUoidYGqdXDZ3ifuNMPmqkQD+c/udP/fVtip282ax4/ecKXX35FN17b7xc89qGm7/PQ998r7a/6W7z85KBPmJd351sphbUeSB2b+2O3fy/98J3Ew2emtYsiCkVRFHgfE6FBJOyPUPv93Ou7oz4rxGi4Tt0/kLTk/eD/Hn0igpDvdOf3x2TKp9jz/uGdaJ6wze8fo71m935R6O/b/iNwLpoYZQpdEyPGJI5cyL2zO+7b+ZM4k8IXI+DHgL7kgkrX1N7RvT8YYTzlIS0shUpNDOKdACyEfhC1I2mRKCJEpRBhRMk8COFhdKoDGB6w9SOOJh36VBjgvQOZ9kyll4kUXviOF7U/eeNrxF5gJ3123CcWe5Kkv2e4p8ULY9CCYGxDIAncPAiochT7I0QHHqJYE0MDfjZmy6RBkga3frhhhhBTGJ4uk0CHxOiMzAicH9hXfTKV3MFlPqXMpgzeE9nS9TVxbGNUOifLS6pqxsHyhEk1Se7Z8SYusmPwHecnP2ZXv6XbvUSqDK1LTDbBh3TM6rbFek8cOoSzCNdjZMQrhTOSECyee2QpWB4+YVu3vHl9z89++huePfn/cfr4Q4rDM1QcILTUu0t2Xc0P/pM/IZ9OODxMCJ1f//TP+Iv/9r/m67/+GRdX1+AiWghUhIGItoB0+BAQOrGVcylQLjnsbCe5EIGoJLlRDAhqYVFC0klJh0LJSFmohOGSESTshkjYQd5DptL49RE6m4LEFtN3YQZSgBo7JYSM6dw+/N2eO5Zu9tbDth4I1rJtXWK8k+4dNkLtNVdesHGRNqZijvodRXQvUsilsALvLNG5FDoXE2okqNQiLJ2g33aIoCiCpcpKcpWjVGSw4IYwBncmHpcn8WlDdKklyyWebWrpCViRnO1DbymzEhEFQ+/oO0sUaQwnZyX0g6MbhjS+lMHokhAM3iuE0WQZmKICUaK8ZhhapHfIKLDBI7CjCzsxNq11OJsQSdgBNSKSBlUx0CNsg06JfONVrvBW4J3AGEOWF2iRIQPp4RtTx0wYBWEEOO+IQY5ImJBCqiKUeU6ZZXhrE0fTWhARJTxlYQiZwCiF7TuqqYboGQaBziYADMPAmzdvEELw+NEZYpwoByJKG6rJhL5N4JXUvpR4X2Ls/BHjosr5QAw6TcxlzmxywB/8wR/z4x//PpNqQm5S+G8MiamXUrbFGDrKWDQcUTsjJzTGfcU7IY7qesfNzQ3X19ds1muCD5RFiR8Guq4hEslyg9KCvmsTTsoYdEyOmsG59FB3gabrOVwukcqQmQJVKEQriC4F3QodCcJBTKxoaxXOFVgl8ahUQEVi3QC6JDNVGuO9Q+ocLSRBycTK9mFkMu9QMhCMYghJkCyzHBEVRkry3JApjYqpECuiJstLTJ5jvcMTabqOfuhRIgXBheCTG0GplOWATM76kBwIyVSsUcZgY0In/bbbo8dPmO3m/OznX7Na7xicZzatuLm94/zROTFqFoslTWuZnxymMO0QkCrj008/YnW/oZrM6PqBi6tLirxgvkht0p9/8Rk3NzcIIXjx8gVSSjbbFccnx6w3G/qh5/jklIvLK6x1nJyckheK3a7myZMP+NlPv0vXibPU9ZYnT89YrS6BwNX1Jcv5MSEohLA0dT26IzRKaoqi5PDwmH/70/+JTz/7iLquWS6X/Pmf/xu++PQLqknF8eESnKXdbqjXa9pmi1AKJQV1U2OyJMBeXr7FDZbZtKQfNDc3LXe3F1xevKbras4fpeLAMAwcHZ3hbZ+cUfMpWmfkeRJHu7aBGKnKKUqm0BytDFJoYgx0naXe7dhuNwnL4sMYwKOZTiesNx7t070zjiKs0alLZTqbEsc5AlIhVHKtgUKaAiUsSgmUMXg6nA+EkQUopCYFRxm0KdBaU+UF1UKxWC4I0ZEXhvlyytuL1zT9jvlywt3dTcIJVCVNU/ODH/wetze3NE3DcpkE69lsTte1D66VIi+4u7tnuTwgMyVZVrJe3+NcwoVZ6ynLgsmkoiwn7La3/OIXv+Ts7Ayl1BiIZAg+cnJ8yt3dLcfHJ+RZQbCBpm6ZTHr23YeXl1dJJIuC1WpNXpSE4Kl3NaenS07PTri7u2W1uuWTTz7h9vaOzWaV2vD7nsXikNXqjvPzRzx//oKyKJnNZolLLSU3NzV9n1Aw8+2MGDy7es2zZx/x87/+Jf/oH/2ESTVlPl/S95ZXr17x+PETmqZhMpngnGMySQJU0zScn59jTAHRs1wccXe3/p2e3fVuh7eOGBLWRUrFpCxT8c8N9HYgusAwRJz3zBeHVKai7xxt31LmBicUMRNIHdjs1tyubtnZLZkukDGFTK5urtH6aAyF8lTGoVEoEt6k73ZEmZ4VEYVUFb6TdCufBHQlsSqQZwZpB4amp+scLkSysmTbbGEiUFVAFSRkhGjoaku93bHa9ARhmUwFIWTsas/VVYPE8OTxY87Oj2l5jpIeKR2b7T1CSpq6HVmwAhUkhawosgnL8gAB7HZrQpXTs8WoFMhbTacIFehtj1CS3g/UQ4OKOdEJRJel4PJpxB1JfrN7zfH8gOu393x6+owjPadveyqpubq7pPM1J5Ml7aql3w3crbZ4ocmnMz7+4guizKgmM3prWW22zBaK6+sb8rIkI/L2N7+iqCo+OzllajTPLy4R0dHeXVPMNbs28vWLnsODY0y0hGGNig3r7pbYNxzMFF0bGYhUx4dcXV+yG3rKogQB88MlddPw6s0lH370Ia21TKoKHyyCjCyfgBeUAWS/4/PDBVeyY9fumGUZPzw/IlMlX653OCm52qypW4s0U87OUhBbs2mYFlPqdoMnMtgUVtx7lwKno8GpAhHB9Z52GCiKCUoX1NZSrxvSAlgwOE81yVMXhrMEK8iKAl+3zBcz3JD4sve3d4gAOmTcb2peb284Xh6nwNT768T63m6YHZxzdnhOZVIXzfZ2m/K5vGVenXN+XnJzc8H93R0Ez9HBAU/OH/Hi5Su0USwWS/I85351x2xWEGgZBsvQBYL9d1gI/4Gb0SmHy2joe8ft3R2dlTx9dkaMCseOXe/RZaT1LeU8IToS09uxXCwwxjCbV+x2LUM/sN4FPvpsgSQynSrur5KIImRgOi3Y3TqOjieEoWF5VBK85/ZiR6ErDqaG1t7SdwPBhhSIPtW8ePWawyOFMgJ8IDNJJ4gKglJkeUISDoPn7q7jydMFNzc1kLAHg9uR5VA3DVHCbFbgnaQsc3a7O9raY/Kc3BScPfqAdmh58uwRrduQZ5q72zVapnydyXRO37d8+PEZTbMjRokyhoinKgpynbHd3jL4AhE8zlrqpuHR+fmYZTFQ5VOapqVItj+GoR8F9GRgq+tdyqtyKZMHSDkkWY6Qhm6wZEYzKSRaQ73bUhSaJ48TpkxKsE1H9B6pIZC62eq6xblU9JHKkpuc6Yg22+02eCko5rOU6xQ7fNB0fQq3FYgHJr/EjGHWaT6+X48pmVBDMQjqXctssUArQ5ll3N/f8+zZI2bTCd4P1Fme3LHFlLr1fPf6lsVRBKnZdR3rbT06RSF4yWI6Aee4u7khzwv6bodtBIXRKDK0yinyJU0beHp4xub2niDWVNOCzGiUTKx2IyPTLKduWoIXHC6PccqR5ykscT6fUjc7qklC39oxFHy73ZIXKe/JOk9RFNTre+bTgtX9CmLDf/mf/3NevP4G7wLdekNpDpCi5OJtQvZsdjuc0+T5jLKoePPmgsePTzl/fMbN/WuOTz/i4PCUq6sdPjrut/Xv9Ozec6rfF9Hh+8I1MBbwInJEvuz1vf2d5ftO9DFcXiaBNfJOINxrVHvNPTmw00u/L4K/E9zDiCB9J6K/E5r32/vi5V74R6RQ6b8rRsrRMCpwzrMdGkLoybI5Waboh5Y8rxhsQwwC7/yIMB5AJP1gcCnva1LkGCXp25QT6LxPRlORwkG1TsjS6WyO0prBOrTJHrCP3nt8jAmzGkNah47z2PcxH/sA9b/9ffdC8TvUi6TrOqRMgvFeaH4n1qfu7zS31KlQNp7nd05/8YBCeTjPD4WL7x9LIZKZtu97urbl6OiIy8tLhmHUR99zoO8Rhe+L6Pt93xdU3v/9vwvzEmLCSXrvH8JKQwjUdYOUEW0SOmb/+hRsqskzgYsdmckIXqRwVvmuKxz2BaS9Q3w8B7xDryTjdBLS4/j7d0MraToPQnoQD8fK+zBmRbx/7ALvD+H9++zf831U0t+3/cNxLqNDSgk9Omf3LeZ7oPuYqkpqf5LIhKzYsyQhidoiVQ+IIGWeBOZok/AShzSExvbOJEjsB+d4oMZ+5hQKasdKQkgg+rFKtsfGxPFAyVGwjxGicESScyrdnxJ0Pn2nlICbsDAJML8PhhyP6ejgHNIJE2OgZXyv6rPH1JAY7Ml/byCm6k0qMth03PZ2yXFL4noS3VNxYiDKNaH8BiUeI21Grkt2pIs6yyq8BR8SwN8NPVLl6LFwJYQkM/n43qk9ItcFeTYhyyqyrCTYPRdK0/cNCEuhK5xtyIyiyEryLE/hZZ3F5HlKDZYGEQNFtiRka4p8iRRZGvxS4hFc3L5g3Vyg/ZqZTMULHRxGgso00aWg1IvbX7OcHDObnuD7gHWWX/36V/zou9/wkfDIgyOiEuT5wGRmyI1GRQe0LJYn/PF/+i959PEP+R/+6/8bz//Vv+LNm0sIqcI7+Eg9uo1lpglaM0jFQCAIgQ8COwQ8AplBFhwWqJ3DIMiFZiolEynIS5VKMTHiJdy3gbs2oGUkk6BHhv+68+RaczLX6YKPiYdFTA8YH1L6796NLohkSo6ubUeuA9YF6hiJQqMNKDxGCXokt17xZohshpCY8zIixd9/sf+HtiF4yqoi9zkyQrSOXBmkCjjXs+s2GCEZcDgRca7H9DUIhTFp4qJkwCuJUqm4lbh6FhEFPjgUgkKXFNKQK020IaFqQqTQJZNshskUDBGpBFIZlC4IKgWvSqHp+sQBRBpys0B4w2BB64AQHhkcWSYoKUFGfOxpuzZVdKMjMhZR8tTVoVUKTbS2RQVLFJ6drfFCYqKlMnPyvMRkGcYWGGfR2lBkBaVOC4zgI77vU2SC61NIVybpRXJmK1KifJrmpEmRVhotFQ6VmKG+wfoOPAgfyXXGYAekDCgVqKaaImikLEfcxpoY4fbulsOjJVpLhJb4MFavlUJqNSIeGCczIzKEd/crKTTB53hrWC6O+JOf/HN++MMfM5vN0VojxxYpH31iqxlDlpfEAINzqPGhK5VEqPQQi/s2O+9om4brq0uef/M13z7/ljevX3N3e0dwAyF4siJPInSZMZtPWK0Sr91owzSfEsO+syeJbwhJ2w5MljOKUQCYGonxGa1tEDGQGdI5CZ4QbSokqCSgB6WQIgM/oIxEG5lSx5XFCYXSIxtv7DYiuBRUGx1OGhQGJQRYi5ECGVNnkvWeIdixaAxKa5QReC9ou5Zdu8P2A9Myx3tAKJxPafVCZml/vcO5gb5PXNWimBBjCiD+Xa7uuu3wUfDk6SkhBo6OTrlf3bJYnlDXA48ePeL+/o48z9GPcg4Pj1it1iSmaRJc57Mlmcl5/OgxP/3pX/Gj2Q/5+uuv+OKLz/juu++YzWYPzl0pNKcn57x989MUyDwWxc7OHvPq1bd0XcOjR6cpqC4MZFnGq1evcM7y8uV3XN+8pd3tODg4IwJVVXJyMqdpGqzrqeuaPJuwWq34n/3RkqrI2axXfPzpJ1xcXdFkHdZaMq0pTMbJ8QHb6zXb1Q27zR2mnHB3v6Wolmy3axbzJdvNBqMlNzcbchPoRaSu17TdjvWLe6RKKBuB4Ob6LWU1ZTpbYkxBCC4FcwXPYr6kKgvaekeRFRTVhKHvGYaWrmtomh0hRE5PT9mu77i7SXiZ+Xz+IKb7xjOZTLi8uQGgGxxSZwQkq82OPM/JMkPd7rB2eHCnd70jSpBuP2k07AORYgzkhQap8VFSFiXZLGeykCyO5hweLXn1+iUHR0uySnB1fcl0OmW323B7f8+T82f81V/+ij/6w4Rm+eabbxBInLNkJoWHT6rpGCw6QYgtw+A4PDzi+uqGojAopTk6OqYoi5GND8dHp6zuW/KspCwqNustH3/8Cb/5zW+QUiZBKK/oO5vczcU0dYyN4aIhRLouFWqapufDDz/m+Xcv2W5rBjvQdT1935LnOUVx/MDAdM4TQ2AyLbm6ukQpxXq9xlrLixcvefLkKYeHR7x58wZjDJvNmmcfPEmBgUVGUea03Y5qUtJ1HZvNlhgFzga0zri7u8fanqZpOD4+QWvN1dUVIcDNzR1Pnz7j5uaeoR+4vr7/Ha5u6HtLnlX0I/O+KAqKsuDu7jY1XHpB31t0blAy52B5TKEKdpuWuq0pyoI+wuHxAZnKuL+5o+1biiqn7hoyYZAmo20auA4cnR8mHAoSGQW5zojC09Y7hBFJtNSKqizZ3azp+5SF4lWEXKBlasHtXc+m2YEQPDs7QOYhBdSqhGablYbcKKwNrOqGLEh0lRGGnm6IuL4H5yjKCQezKaeHB7xdvcRFSZUXDI1j6C3BeZQ0SKMf2uUhOb6UTMHc1qZ8jMSzNtR1jQ+eKBIS7PpmTQgKgsCYPOHNhEyhf90GPdcELZhOJsxNAa0lDp7G9zjrmBQVu23N+mbF0DkGGzk4O+bJhx+yPFgymU5T6KsxfPb5Z6w3W548eUxdb9lttmQ6srm/ZV23hGLC0fkToix5/eoaV9dpJRcEHz+B2G5YXT3HtTtOTk+pigKdZRR5znV7hVaKxXzOgUkus+1ux3XT4IKnmEy4uLri/MkRza5heXAw4hYHokps6UkDX+THbG+e4+2Gp4sTmm3DrdtxvV0jLPTOUtcp42YxneD8wPb+GpVl7FGYUqmUFdL32MaOODuNNjnOdgzWkeekgMKg2WzXZFnqjKimFdNZhah7TBYxeU70iZUbQwpK7ZoWEVNr/Gw2TWK7dLS+QRvFerchxORY3NVbptMF5aTk+vYaFxTGCHwc2GzXzOYzjk9O0Bq8s1xdXvHRh58w32wQWqAzw8RoLq7eQpMMU0prqqlO3N/fdnMWUyYxYhCRH/zoCXV3z3cvrvj440f4MFAYSZAd2hjysuBxPuObX79FlZG6bvBRM5sXKY9C5bx+sePubsfZuWEyqZgttuR5Rmc3HJ/MiVYi4oBRkllVsll3dHXP6v6a09MpTz885fbuFlMVHB+d8/r1m2TUEprBDux2LTrT+GDZ1fD5Z+coFbl6u6JtkjiS54aDA8NqZYlYikLhLEymcxaLgJCC1V3D+fmS9UqgjcbbHkzGrr7BubTPeW7ou57lcoK3gaZrHzSEtulRMuPxk0NurpNYW5iSMDgyZXBDmtut6y3LxYK6adjsthwtlzT1FmU0VVXhgbLMub27Iy8K+sExmy9SYJ4INF2TggkBGcFkGb53dN2AHQaKImXiSOmoJhOUCinE1wbaoUuhvLuevh+wNqJVTp5PECRn/XazZTqb0PcdRZHhXM+qXSOEoG09Xdci5Yx+SF1tIUSMTCgLazO6rifLcoK3I8M4I8tKJlORuP0yrQuKIuOrr75Ej2s+ay33qw1BaurBEQvDtxeXHBwcMpnMESZHeY8XyXgYXEz3fmVwNlJmE4y2CJHQQ3frFvV7FYen59zdb/DecTSZUFbF2COecoX2XfGPnh7zE5Px//6L/w9DaAgisjxccHV1RdO2mOwQ5y3z+ZLd9p6u6Tg7O+f1m0sm0wl90zP0kdthx+HyhPV6x998+VOKMufk7IxmF3EusN2t+eijczbNhuu7e06PFsSlZL3ZcHpyTjkp+O7l15w/WfL27SX1zvP1lytOnhlM8duHBsM7F3kIgTgKnfs/w15Y3IvZ73jNyVk7unIfBPC9oVUkV++oz+39vVKkkhWjSL43h+2xyyHsxUQx7sN+Lx/egT2q428zo98XaN9x2CNun3lDMocmB/J7eUA+jEKrQasIwrJaXeFsYuQbnUMcTSXRo40c16X7ffR0wdF0CXE2nVSJm04ky1LAZ1mVo6025QKk9WwSs4NLWgwyoT6Vfuf63udTJSd5CrTfc9Dfd6kD47XYkmUZeZ7T98PDudqLyem8pXNorUWr1L1trUWOeDMpEw5aSjnO6+V7x3QvQr5/3iF4z+npKXWTukKPjo549epNWl9nGeAf8or2Y2s/D96HrO4RNHtxff/3KWeMh+/v91rawzw6OdurqsR7i9Lja6Ua10YhrQmVJcaErXHOM5tVaCVHbULg/T5bQDycW5EGW2Ji8/0ij5Dv/ryfz+3NZ0qp0cUuHooHSUgP7xU//IOILmXq3gSIImGh43jc91l4/6HtH3wHGHyLEqmylb7QHo+RpcsluvGrJGe0B0J0JMBLuoAEScSKoxi+F7NTCN9AEEMaQJjkUo/7z4kQffKHi+TwjNElHEry/6ZLJLpRgC/HA+iJeJQwo5V/7zRPAvb7bRLJ7a6QIie10wwpFDXuv9UoPiXr+8MxiGP7R3LIv/t1zyEWjA6wuBfG3yFjkgAPe0xMws6IVGRAgBySE10NgEc3n6QAIlWiZUGelQSduMBtt02dACE8VHFidEiZgk+U0ggERhfJwakzhBRjxaxM7ScxYN2A9y1Dt2a73fL0TOOGgfX6jratOT45YzKZJx6+VDx58o8Zhh8klEdI3Nbge7bDNY35FXX1a47lI+hAO4MOiRmc5TlDJomDwLuW9e6a3kkmswVSCX7xyy+Z/jf/Ff/H/9P/GT0dEFnOZDLnZz/9KzywXMzJjSLLJuhizpMPPuaf/i//N/zq4pr1v/1zds2Gut7R7Hb0PgV56kzQhMD1EHHR0sVU9gle0A1wHQIah3fQe8hlwESPsI6ZSG1hat/qk2y4D07yTAq0EsSYcDJGCaa5TMMlRsY83JF9P+YAjFymGBOSmxhxQSKkZ2MF0kU6AmhQSLyGLYI7J9i5VD2N4w33fYb/b7NFndoVjdIIH5EhjWZpMnxoaYcN28HTDi1+vOJ8TKRz7ySCkMIcC0OMIVWFA8iQFowgyU1BFJFJVjHNKgiBzvUYb8lCTllNkUrgc08mcrRI4nOQkiAUMjPITNE7iXeS3GhiMEShRgEzhWQqWRCdBiew1lN3LS5aMhnGu4skMxPyrBgLaIG29/RuwHoLWYOLUMiI1hk6mhRiMqlS5dZ7yqJAK4XtU2t1sA5lBCqKxAuPPcIrjCrS4l1rtNDpnhXGe0eQRC/QGDJRQtAoHSgmILVOzFQzo2lbAhbnB4zOE9tXTmmatEC/vr3myeOTcVKUsEIx+Id7pZAKLWAfjBd8wHsQQqFlQfAF08kx/+R//r/gD//gH5Hn5XudEenhY50bJxsKPxaFlDbI91rfYkyhwd6HFOTU1Gy3ay7evuHb59/w6sVLttstQ5+KGkpBZjJ0rjGFYXY4QxUQbhzBeopxwiPGqnXbthgyvE+hiTozDG5AZhIdFMKlToDcaLwK9M4BIbWLybErQLYo5YjaERBYl8L4rB3ohQRTketyvLd7ijwjiAwbIjaOQaltSycDs2KKJiK0wgeHjwMyJuextw4RYYiWxqYAv7IqKasKBbRtCqYMKrX2Bp+yM5y3hOhxQ3o2GlOiv1dx/4/f1ust19dXXF3ecHC4JM9LDg+Oub6+IssMq9UO7yUhSHrr6AbH6fljdrsdf/XTn/OPf/ITTk4f8eXXX6OUTO3k11eEYHn99i3KZIkNHtL9NMsrdk3P8cl5ysSQhtVqS9ta7lb3HBxO8R6apmM2m9B3A2/evKWaGCID3geUMtS7hqpYcHR0zHfffcf9/T3O9+x2Oz79oy+4v93x+vULHj9+xPn5Ma/evkEqxcnxMc2uoetavOt58c3XLPIZ7XZNcAPDoFL2geyQMk308tywWd3TNWuWi5z13Q1tu0Urwa7v2Kzv2S2XSCGZTDwmM8Qw0HWW3a5Ozo8I11dXzKcLzk8fU54+IniP0TqxbX1gNpkxmRyyXl8xDJY8z3FDyU4pjEnOmSzLUcpgdI6SFutT/kY3ePre4iP01rHZNQTv0CZxvofBpntN3HMQGe8JMi3UnScKS1FNqWYzpkcSXQ70ricQyPIkAv2jP/4j+qHDDo7ZbEkIqTsoBri4SGzv09NTqqpit9txcXHBcrmkKFLB5aOPPsL7JOau188xJntwqw+2Q8qBSTXFWs/Pf/4LjE7CvBCJ5Xh5cUlmcsqq4OrqioODJU1Ts1weMp/Mef36DSfHpzTtjuaupShK+r4nMxlFWbFYLLi/u+f05IzXr5+jlOT8/BQ/IkuyLCfPBLPpAu/h0aMjvHdcXLzh8ePH46Ihtegm7nrJdrvm9vaW+XzG0PeU5TG3tzcsF3NWqxVZlrPd7Dg5OaPrepqmZbmcjSHPelwkRW5vb3n8+DEhBLabLTFIDg4Wv/3FDYiY3IXW9mMH4SQF0TsLQaKUSZxkM2E6yZKD1Y2FfVIBsLfJEd51NavdFrSirEr67TB29XiETJ1Ctu9BBEJQhMFDSC3bRpkEewyBWVXh3MB2t8VZR2ZSODkCBjsQrEdoRVaN6DzdoauAqSLzbIYKU8pcYrTg9vYu4di8Q3iJbXqi7ylVxsnBgkyXhKFhfXtFUzdEAkpIvHU0uwZvBVJCIRS77RaBYjaxRO/Js+yhGGOdB5XwlN2ID5nNk1Flt+4weoZAJTRRljMtM2KwaW4xn3J7vWKiDiiiwq5rhBA4bylNCc6x2dQIrVG5BOE4eXLO0dkJ69WG+/t7prMF5WRCbJqUI9J0GBkpjUQGhQ8ZcjJnR05QFTcbSy1m3K4aNvUKXOTVyxV5qHlyXDA1BZk2qUtMa4bBoY1hfXeHjLBeJUb1o6ePkUpyc3+HHQPt7ODIlaJve1Ql8Qx0rqXUKQ/E3g0cz2a8qBUnT07oOstvXr7gru8oRMZpcTyGi1tEMMzKnNX9jm2zw2cGrTK0MSkA13tsjGAdWWUQMeJ6QWctsa7BtszKOS44VBixnNFAsGRGUeYGhMT1A4XJWN/d07U90YcUnrir0Ul7J6sMQVqCAmU0F68vqCYVqkzIuaqaMZ1V7LYb8iLHB0fT14R1TGHmRUVd73j99i3zxQJtJNWkoutb+r7n9OyM7W6N82CynJOTM7z97bvIhmFAKDBSI4VncDUHx4pvvuz48qvvePbRlGKiaeyAx9M2O2bVIadP56w3G+aLin5oWa08p6cVzkmefTzj5noN5NS7FjMWU+IguLndcXr6Ic3ulqpKQfYhWiYTgXOaw6MpVZmxVpqz0yN8sHzxgw8ZbMvt3TXVJENtBVWmudt0LBYlzlsWsznTSUYIgrzUFEWWcCHS8fjxY+7u7ilyy+FyRm8b+r6j6waur28TqieP7LYDj86mvHj1lqLMWS6mZJlmMs0Z7MDl/U0y8AjBfL4kALvdQJZHZosJdZOwoHdXGxZ5AS7QhzR/19kRb96+4WC+4ObuhkKnjIcoU/ZNiCm0VI7OTOcG6nrL4dEPubq9YTFZsN5uaJ3DyMR5zrMM7IDrA1KJlL8UWiLQ1DWCwB4x0PcDxmSUJiN4SdNYjJIElzAVu+0WISLODjjb03YNxkhc3SEEY7jyuy61EBNPO+VWpJyMwabnUtv0nJyds1iesFqv6IeBKitYrbest3dMKkNmDLvNhqZ3DGguVlvumwGrBFfrDXrXYbIKZUoUnt5GtCl4+/qSxaIiy0sG23NweMCu3dK0PdVkxuAs+SQnNwuuXtaU05K723s+++BzYlTMlzNyo3FDz1fPX/F//W/+7/SFQ89atFFM5imTo+0b7taS2WzOel2zXm0x2jB0A1LA5cUVZVVS5DmDlNz3DV57dqHh6uaK52/f8unHf8jbV6+pd8nYMFiPMUlQPjo64Ne//Jrduubw+JDJpOT27g7vPd999xrv4O52zeJg8ts/uHk/ODQZKPeCaRI81Xt/zzvh/L0t7tvZ/9bPklj6fTf1OyTLO+H33Xu8J1CKvxtW+v0g0u+/7/u/fj+MNAn5D673h/+FUWMbO8PHAoC1HTu/wagKrdWo/XkECUUnxcgSlzphUnxqv6/rLcEHDo8OiT51PkspQSbOeBLBNcbo8Tj4h88MwY3iKuM8IKFX9uvNJHC/6w4QQpBl2Xvs85R5U5ZFurbaDqJIuJYxGHTPN08BpmosJqSgamn094ofe2xPfO/8vHdm3znTEQ+nvWlbJpMJP/7Rj7i+uqZpGn70ox9xcXHB9fU1ShmqqqJpGowxD2Nh/3l7Lv+7As1eQE8IPjmuv/cc+jDy1vc/0zoJ//Gh0z0dzDzPKMsJ3icDj4jZiIjJknkaIIZR4HYIuUcL7Z3vOl0T3xPMeTCfPiBZxv8QcR8v+j0hfl8Y2p/DvW67N2W/P3b/Nsbl756Dv7v9w53oISWmE/x75mlBiAMPfPIoktM8JDd4hAehWrDHrYAa3eJ7MVtIiYxpcpYwLSNXfX/Axpaq5NkdA7T2qJT3DyaeGBRB9ESZ9hlSqB9xH2QqxwqdIfF07LuLfKzuMHJG9+7tdPD3vPMR7RAT05xkb4cRqJ/aD5JgLUYeVPreCUeT3iMdkXS8/HgR8SCei/H1UQZktCmAMjqkLEBBZipAYXRatDB0KN2PNwf/AOcX4w1LSUmM6YI2yjykBbvgkMpwUB3SNKtUWScihEYKsLandx3Rea5u3tK0OyBQFBUej1Y5ShWURZHezw8p8LS74+eX/z2vN/8jRaY4mJ4gAG0KVAz0uzUFil6rJBAMW77+9i+QcknMLIdHc968fsvt7R3rzZrqaIkUgvPjQ/7ox7/HxWbD7r4ndB2RyOLkGSqfcf7ojH/5v/vPOPmDms69oRnuWW2uWa82rDcD65WlXkfaOhJsQPqItoLgEiPaxbSQCmOlrxeSVXAMfUB7hRmrZHI8S3LscpAiYoRAjQ6rQCTXily9a0TZZ1WLGMfX8RAgiohjkGgkKIlAc9lHfAj0LqRARFJVbBgC9zI9WrTeV4Z5cMD+tts+/d2I1NaphQBjCCqQBcvgOna7HTYElDbkWU5elBRFSaFMKlqN7nPrUykMKUfnfXKLpQ4GQVaU5HlFcB6nUiBTGBIuRJKSnl0UZMKghMG6gI2RPC/IJzm9kwy9otA5UhSjwBsIDnxwxODJVElQgd4mbmCVl0iVFm1uGLCyI9N5KiipDKM1uxbEkFq3bRwoy4D1DSFmZGaSuMJSYbshpb5nqd1pGCchzgZkkBSmpCgqpFBEGTGZTi59L2lbi7cBTVrYVtmEaTHDVQE7JGROINK0LQfzyK6pybKaIbRsdndIAwaFHCJ5KVFG4UJHbwdMlhFDKm6kVp90P3vHeIMY0+TKe0mmJ4hY8tGHn/Kf/4v/gmdPP8KYfHz4pPupD37kKYbxHKcipDaJsexjCq3xY1ustQlN0zQ19/c3vHn1iq+++pKXL16w3Wzo2hY32NEtkEKETCFp+hqRHTErp3SxoV5t0Nm7NjalNIgciHgX6DtHNZF4kQql1qf3NEpRZbPE8u9Tocl5z+CG1M7qa3TQuN6Nk890H+xtl+7YIhJVwtY4a3HepStYKJy1CO9o2x6RKUIukFrhiXTDQBQDuS7wzhN9usdYORCEJ8sN08mMWTXF9QPWedwo3vRDmxbaMmKMpDAFbdvTDR1SaSYmQ/4OKnrwgY8++oTNuuP+fsfr1xecPzrD2uSIev7NS/7xP/5jdrstb15fYjLNr3/9JZ988ildb/nLv/gr/vFP/pgvv/yG3/vhF0xnU548eczV9SXDMLBYHozPYEnT9AghWK025PmEt29vEB8bDg5PqMqKvMjIckFZTrEWjCnY7Xp+/OMPuL5+y2Q6wbmBu+0tz56eIKXmq6++Tm74+YJXr7/lx7//Q7bbDQeHBykwW8B2s2a32+JiCjy+uLjARMnTx4+4+OYlg4+0uzV121JJhbUDPtScnC6ZTioEcN03hOh49eI5b1+/wHuLtxZioGl2vHn9kuF4oO1aNtsV1zcXCKFo246+c3if2hA/ePYRH334McaM3Q8hkBlDtjxACBhsg1KGw8Nj3rx+SQpsLdNYj5CZghBTgRshE4qFxGjteosPoJSjH9KcJpIm6EhNjBrvxYPTKL1e0FsHypBrmViSwaO0QRvNyckh/dDSDy2TScl6veL46IS+t6zut3z60Q9YrdZImSb+77uVDg4OuL6+pihK8jxjGFJR4emTZ/zpn/5Z6sAJgul0CSSO5Xx+wHq9Iss0IFgsDrDWkWU5n3zyKb/61a84Pz9Fa82TJ0+JMbBYLNkzEx8/fsLt7T11vaHpGo5PTzk9O2MxP+D120u6tqeuW5bzKc4GnE3X7HQ64bs337FYLLm6umY2O6SqKg6Whwy248svf421Aa0008mM+/trDg4OMEbz6PFjuq6h73varub65oKqnGOdRUnNfJ6KU6en53zzzTccH51wenZE23ZcX9/ifeouqOsdkIJKtZEsD+bUu+a3vrYBlDIMvYMoxw4iiXMhha4OAaUKdJ6lgrDOaZuWzvUJjaRNQjPEyHq3w1uPA6bTGUJG5tMZXd0RfECS+PJx7PKxUjI0o9NbRbIqw8uEZJjOS7qmZbCWru0JEXKdoaPEDZ4YJFWV4egxuaDuNkitaPodwmuqvERFyW7bsOl6+ujwRuKGhiIEnIXCZFRZhojQbDYM3RafOUxh8C4xO/O8oLYDm21LlDo9E9khUQxdz9Bbzs/OKYspcYh0Q8PQW7x19G3HdDKhHyxuEGiliUhcCAnZliUn53RS0UtSyH1eQUgidAyB6axCiogQhiBkCsSOgsfnj1jvtlz+7K949vgZ88WcqqrQWYbSBiGTY2t3f02zWSd8yeKIdV3zs2+f48tDYrbkbmNZ1QNClly+fcvm+pLDMnB88EPy6Pj2+XOmyyWnjx4xm82ZVlPKIufy9RtyowkC3r55RVYWnJ2f0NmBzW6LC1AYTfTgXABjcALMvKK3gbf3G77dXmELw9fPX3P65CgVUQdHl9oryfIMbwNt23J8uKTdbaiHAS0UTd2SlzmDtSgEk+kE27txbaeYTKYorbGDpx8G6tggpMLHgFaSLDdpveI8ImT0/YCuCuZFxdWmRoZkl9qbvhIiShBChxCKqpwQo8QUBT55POj6Fm0MT54+5uc/v6XpAgcHc/rBEUSakzX1DklkMptinaUbOoqyoOsGnE9jThtD16dcGKUFeVH+1td2VRiMAa0yEC2BnnKaUU4UfR14+13NZ79/QMgcnQvklad3d5w9mXFwskTpnuUkZWfNFwWbzY4iL7m/S+5ObQRKGW5vdyyPNNt1z1psOD09ZHA3KJOTSZBKEmXH+dMFtmu5fFPjhks++fQJzne8uXiDjwPVZEqRGbQUHCxLFssDyqrCDi4ViETHZFLR9TvKMmexmLJe35PnBWU+Yxgs1zcbnj07QCBHgaUnBIdWGdvtjqxQuNCDmgKRSVnRti1xDF+vm56iSAYI7x3WebRO7sgYYLNuKQ6KtL5SjuXxEuscLnjqtsYNljIzIATWWdqmIcszHrrhJazXKz7//DPu13cUZZ6wZzp14HdtT2YgMyXOJSSBiKm7WChB23Q0jSdTYkQMBITQbDYN3rXEoDBas5xVDH3Px598wGp1Q1kWtG2NlB5tVOq2y5KwNQwWgxkRHkkITmGIFX3nGQZL3/V4b1mvtrjoWS7n+DHs/O6+YbW+Ic/TM8l2CfUkjeB61+GEppeSITiG3mFbSwgt1kdUZlgeTxgcRKlwQdHXA3lZsG1bzp+ccnl5S7SB+80lLy6+ZlFWmErQDQ2PHp2na2bwvL685MkHj/n5l7/k//Hf/3dsaVmeL7C0dEPDen1LiBZkpChKvIdJWbCYzfn5L18gRcZkNuH+fsvdzY6PP10yOE8/Zr8123vKyiBVxr/+N1/ygw9PqHeWL3/znNlhxenJCdNqSgye09NDbO+4vt5wer6gHxy7XRJKhdB0TUPX/m44l+8J0sTv/ezvCtl7o9y/Q0h/+HX/OpXWcHH/b8LYwRfee7+0dnv/3R4+lz194p0gvidE7HnXf3sf3g/jHGNSiXFcS+7/vRzpFAECnhgtUmh81GlfokZrQdvdg4gUZZZQwhj2mlmMYDKDEBneOcqyRIzX6ma9oshzZtPpeE+IKK3T89S7MXg+FROcS1mJ+8JYjIFhSEZe4MG9nef5Q0Do/hgBOO8JwdO2qfsy8e0TurWq1IO7Ox0fRge7xMfw4JIehiFliSn94DaPyZnLnrqxP3/pWO5N6ONYGUNE9/PLPM8xxnB9fcV8PkfKhIUDyPN85K/77427vaC+3/Yu+H0xYc8J34vrSXgPD9ievfCstR47TlMHhDEZZVnS9+14PHu22y0hnNH3PYiIEAGpQIgw0j1G08fI9A9jcen9vErx0EkfH8zUad4QHwpHciy6vOPwv180ev+4vrsW4t8yq72PTfoPbf/wYFGZIxiN2KPrOxJwcRjFQJnEviBAOpQUyDiyxccwUrEXpomoJAsmETgmJ3lCUuxh8kk4HhtOUGJ8GAkFJI7RqHWOorhCPezb6JUd01s9EcYWDx3zMQh1PA3RjzeLcXDHdjzIMr1HTANVCDm6zk1C2sSIjO/dHAiEOCCTN3EMy5OM8X1jm8ZeZN+74d/d/OLopt+L6Cl0NuEliDJVY0xE6Iw8KyBKTJbTO4eUlkwX2KEHIlqlh7aUIoU0aY0dXGpplanQEeKAiBlFPqfMl/T9BiFlCu8QGd5Fsizn/v6Cfui5Xd+wmC4pqxlKSNr6jpgXmHzGnjevpCbPUvjjdO44yaeoYAhyh88KcjUh8wOlVBhnGbTAKUmQkabfcfvmG/rdmsncMFsZXr58wZ/+6/+O//Lk/8CkKEBEnjxeYjPBi1//ijvn6WNPMVtQakNeZZw8lXxaSmysQBTEcJhaDPue9a7h9v6e9WrD0EVsEKTMO0FhDMHD/Wrg8tqxvksVMhMVsZH0NfSdYG/DFqNoLfEQA9KDiILgk9ite4WMKVhWCcG7x0oaWUqmn6tkFmafjh1HR3qM7zhdIUQcESsDNlN4nXYj4ZTG4lX83ZzoBonvLaU2RJJDOEpBN/RoDCZqjMpQSlIUE6piQmkKCm2YFgatM5rWslpv6AeLMopyUuCtJRLJTJFSp6MHY/BaMtiO3vcIrUALet+jsgqVZQzWkRABFYPzDN5jtCKTEETEC49SIeE4JASXmPT7MEGjBc5LyiJPIn8KWCCQsekG+q5BCcNsPqGaLCjyyVg8KlBhi7NDcgC7ge3uPu2bych0jtKRZuiQPomQWVEQQ0iBPDGjqCYUVUlvezyBKANKR4IbEtctpvahSTXjcHZEqSc0dYuZp4f9ersmFppNvUabwFQbLm/viESs6+ntgM6gnJQMNgn6Xd+xmFfYzjJYCziUiPsb5Lv7XUhMdykqpuUZ//yf/Qv++B/9cWrPHEenECG5of2AcwFrU7iJlCo1XaqRk44gxEBvU5uoc0NydPctm+2Gizev+OrL3/DN119xdXlB9CmUa480ElHgQk8x1QQp2OzWHJ0eMFlUBNuQ6XdBJkIEilITo6DvB5p6oFo46q5NiK598rlQFGaCqQyZmlC3NSBx3tLbjmgH9CCRQSLRKKEwWuKVZHCW1u2SM9LB0Fu6oacPPRiBjhrvPCokhEFuMrKsILhU/wzBoxDgkisz6gjCU1uLCwLnfVqQexAqw8iIDelacT7QNDWTaWJ2Wmfpe5uQWyqNi992e/nmLeb6hs9+8ClSSm5vb7hfbTg5e4wxmqubW9re0Q2eb77+itPTI66vbzk9eURZTDg7O+Obr7/l2Qcf4p2nqiZ0fcflxSVSah4/fsx2u0PgUamlBqNzvIt89tkP6DvHq5cXnJ8/4vPPf0A3bPnFL37JD3/4A9rmb/iTn/wTvvzN1/zBH/2Q3e6eYbjm5PiMxeKQ2+sVISRR7+r6CqUku92Guu45WB5hXc/q/pa4itxv1gQEp2ePEq+vmrNer1BS4G2PHzqcs9ze3TL0nsdPHzOfT9lutwnf4C0315cE37JZ31LN5oSYHNWZNtTbLWVZsdttqSYTZvMFUirarsMOgcODEz788FOePv6Ag+Ui3eOlxGgNweNcRGuFMROIPZdv3uBtpGsH3r65pB8sWudUOme3q8cJnEIbiVCaSMrMwHlClCMjU7APu9c6I6LSLGLPF4wglSa6AetSLkjbNUglmExKXl2+YFJlTKclk6rEaIVRGlVOub54Qa5K1rc7rq5vOTo6JcsKoEkoJSRN0+Fc4PLyiqIomE6nXF/fcHb6iCzL6LqWm5tbjk+O8N5zcHBIWVbc3NywXq8RQvH5Z4fECNvtlr5PLakXF1d88cUX/PrXf8PJyQlVqVlv1hiVXEiJZb5kcbBktV4z/XTGL3/5N/goODs/5/WrN6zXK05OThOqKCtYLA7w/luU0miVYQePlgHnPNdX19T1js8++wFX4++VUnz77bdjYOwhQsDt7Q1GG7z3PHp0xtXlPUZrPvroYx4/esqrV29wznN3d8fh0eJhUXV8fMz9KrHkGY0hi8WcYWi5vb36ra9tgKbpyTOZguFGt5ZzKdQ48Ssd3W5AdIYsG1ASbNtCTE7FEAJCGxwgjcGUkE8neNczn865aq5QWnJ4cICXnsmsSMglGfBB42zA9z1FlMyPZmhtcUPEDp7lwTFX/Q299dhdh7GK4MG2KQcJJZLbOVNID+vmnjoX5FnPwXyKHQa8VKiJStksXURLQzWtUBTUQ4/RhrIoE0JNQueaxIWfl8yXx1xdbej9hrYL5HkqSHkfCXYAJHc3G6RS5EXFbldTGk0MbVq7IXFDyjiQMhXRm3qHkAHVeIZ+y9n0HNdZ2qbDFjBIwXa34fHyECk1VVmw3WyYHRwydAOD9Vzf35EXFR9//DFGpqBNk6kk2nUdQiqMSV2ieIeylsXBglffXnJ5dU1xNOFq/ZLVrmPbbceMpIzJdMl69YJNXfPJh+es766pipLgA6vVCikkUmtOzk9Z3V1zc3vLZD5lu9tS31xx/uQxhwcHbDY7Igmfl5cV88MTds3Add3y5mrFT59/RW8asmNBW2+477cgPMtpxXbXJXdekSNlydA13K521F3k+OgcT+Ti9pLa7kBIUJLGd9huQPYxYdYKlfj+RiEyQXSOqizxwXKwmHN+dkpX7+gQhEHQ7SwmNMQ8pzAZ0Xa4ECEEmqZDNJ75rMJ5T9f39N1AliUDRQiRpu2SSLtrOD6Zcnh8zPX1Bd1gyYspTVszn81TBy+RxXJG07c4P7DadBiTU00m+BBYLA9QRnJ/f0s/tBS/Q7DopMpZrXeIJjI7KCmqiFSWg5OCN+uawXo2dwOnz+b0tEgl6NqBvLSYPDAMfUIVyCRYDbZHiEhZKeqm5/Cooih0wiVKz2JZ8fblLUL3ZEVLFwaKwuCjQFeBlxff8uT4lEePM4pCcXH5mkBks9sitaDtOs6PF0QlaOzA/Wrguxf3HB9m3N/sODgoWW3uyDJF1wUgXVNEwcXFFoHj7GzGq9evOT8/JQbB3d2GYqIoM0VeGGxUtIMntctK3r55y+AHXExOxKLMKSdFEjz7gdu7e4wSnJ2d46Pgg4/OcUNHX2/4+uIFf/TxD5NbMs/ZbnccLZcg0nxtu9sxrSZ0XYvWqfhu29S18+bNa3Zdx3Q2p6pK7u/WZNrgRTJ9rLdb6u2K5TIVFpq6RmjJEAJ36y2zsiK4gaOjAyDQ9TUSPYqVmr7r0726H8i0wfY9RWbo2pacPK3l9zqF2KMLkjPUugEfwDlBVU2AhmpSYazk4OCIV69fM9gak2tCDDRdzWw+pe97Vqs7XGcTciKf8eLVN3x3u+FqNTA/UCyOZ6xutmg0B4cnPH9xweXFhuniESavWG8txWRCu62ZLhT1sOPxBwe8eX7NtrnmmxeWJ6fnHJYLOteSF2foXKB15PqbNzQv73HZjukZbHY95B25yOi3DYNznJwd8/LlG6Q2XL695/QooJXBaE3TtByfHFKVOUZ7tMy4uLzj5HTJYjLn7GTKdrdGxoqzL56io6ealPR9x3Dr+PTzjwjO0vUtZ+fHrO42fPrZMy6v7ljftxydHFEWHmcN663/O2Lyf+y2R2wkkXovoH/feQvvxO30e3hfSP/bYYrAg6goxDsUxp5t/n2H7iiA70VF3mlS+2DS94X0yMhW5/vi/fui/757eLRTs08gjKPxVIwa1H6CGkUiPUiZeNvO12lO3rTEAFV5QJ5P0Cqx/J336bGhFIMdUDrNCXd1jcky8iJP+Dg/dmNkGc7b0QzmHr6Pj2409kVCdKnolGUPTm3ggQkOYIx5wJfI0TDZdf14rsSIe1EPnPG963/v5n5gjO+pFXuEi3pXPEnZhu/TJMR7x/nhjPMwBCIslgv+8i++JC/yMTthx2KxYLerH+axKVxTPoy5vSAu91lq7+GD9o75d+PmXcitgBFNkwT0/c9DSJ2sxhiM0SMaKJluUqC8JcY4uvsHQugTQ10nE6lSo0EvjsgZKRHBE0iFimQQ3nfujI7xPR2ERHbYi/2QdI4ov893f9fhIfD+3THfH5d9ttv3swH+/uv7HyyiZyotmgSSEHsCHSEGVBRj2KhCRpG42KOtHxSBMTgBxmqCxYZA8ruwl9HHEeIRexfl+HchhpG67sb3Hh8yjPD66B/eCylR42fugzxlSG6uIAIipJuKEIypwRIxsooDEeIwtiTo8UTJ5IyP8mGfBImpG8b06HctABbGi0MKiRTZGCaWWg8kahT444hAGCss6csnjrwUY3vJKKqLVLGSMoJpkcai9BwnapTUKJkhRXLyyqgxMsOHHiUUUgTIHWbhyMoeEzKGdSCi8CTMjhSQ5VMae4Vjh1SeLGrEyHC+vHtF0/fYPvGKprM5i8UR3nXE4Nknvu8rpEl4a0F3nBwfU7gTiC3Gb1C9IO/OodkwmZ3QdncUBFoVCUqADphJztCmAMQsU2x3Pf+v//b/yYfPPuQP//gnFEenmNwxzQvK6YJhfctmdY2zDd5P0WZCzjmEOX18iTIDeZ7E9UksODqc8+jRkt4m9ExAEGJCtOQ6seO3Tc/l3T039zVd4+iHSL2NbO6g3pEqXU7gXXoIaCWT6DDykYcOdhtP10SsjTCAt4CLEATRB6Ifz+u4jhT71pKYxPkkZUr2LQUp1DGFf8YQiCOeAiJCCaIU74bSb7lVWY4Skmx8qCitQKvkzA0B2UvmkzmmmJCZksKUGKnpmx1t27CY5yilqKopSInOFdNpiXeJGa4zQ+8tg/PUtkuFLSxhDO+NIjK4HmElWiqCCLjgiDIym08JEQY/UG/XNO2aKDxuaNGZA6UIwqdjEgVd12NDJIiIziQqN/T9gPCQRc00r+hDO156YwEiaGaTY4wuWW8djbcE6ymrHB8DXb/D9TAppmRSY0OHbTpm01kK+CU5YI3JkSZ1enTDgIgeJTyFTsJGGBNl+76n1S2d7nEx0uxqtHYMtqMbiwu7bosXgbZradsG51NbuhSQlYayLOj6wNYPY7ElkBV5ctXYjrJIi3PnXUrBlip1PETF2ekH/K//5f+ezz79AVqP3DC3f5iFcdLRY21iK++RBHtWXvABFwOD87RdlxzmfsD7NAGtmy3b7Ya6rrHDMHbbhNQ2HAM+BNzo9p7FKSY3bJstWZOQQLPlBDsMKDXCurzDZAlL5WrLMCSxobcDQgeMkhhhkEJiVEVhckSliSItMqwbcH5AEIhRomWGkRlKapBgRSAGS+9bCrI0JuIYqO0duS4ojaKta6SSVHlJVVRIaWjdMC5ikvuw0CWZznDS0vvI0HU4J+mNRcae6ONDAVkIyPKc4NPEM8TEZ93WNcSUkbDtU7Hmt90yk5wU9a6hruux7Tey2zUpqb6YcH+35rsX3+JswwcffkBZ1ljvadqONxcXTKcVf/hHv88vf/nXIAPd0FBUE6x1XN/ccnh4hHOeyXTO5cUVi0XJ9eU1j588ZrCBH/zgh3z15Vc8++Ax1qauhS9/8zWZyem6nru7NQcHR7x8+Zwf/t6PETG1b5flhDdv3nB6dkyIlvNHZ3z33XM+//wH1M2Weruj6xqst8wmU+q25+ryisODQ04Wh7z87gVLUyKjSnz/4FhtN5yePuXxo0esth31bk0k0jQ1F2/fkOcQg8M6hxQKG1JIVVmWKCkoyoKyKnCuB+Ds9Jjzs6cs5idMJ/ORqbxDVBJdJGeid6nDTmtJ0zRcXl4TI+RFyZvXrzg+PgUid/f3CK3o+9cIoVBS44InhvhQ0EoTuzHjJYrkQYjpvfdzNKVMug94h9x3pSDo7ZBCLl2fslSc5dWrl3z40TOEAO88ZV7x61/9hpOjMy7eXvHq8i0Hx0uePnvC3e0tXdfx7NkzNpsNZ2dnSCnZbDYopcjzPInjKD766CN++tO/4sMPP+Tm5pYQPI8enTKbzmialqqaMJ8v2G529P3A7d0dz549GV07aUG8Wm344IMPODg4pOt62qYFkVw1VVWxrXdMJlOur9LxNNoQA8znc5azirOzY25ubrHWMwyO4FPHUNcNtE3H+dlTvv76OUdHSz7//Atub28pi5L5fMGLFy9HvmXPxcVFmo8JwXK5YHFQcnt7g7WBVy+/5T/5k39O3/es1yum0ylv3rzi9esstdHWQwpGLQx1nQo2P/jBF2w2a/q+YbGc/dbXNkDXWrQqKIvJiARSNK2nrmuiT2NlcAFhckKIFLkeOxtTu3P0Hu8C0fkUJCkk0qTWWzsEolcgFfPZAUOw5MbgvGQ3ODJZEDUEa1ntOuanC5Qx9F0S0o+fnBB8xtXVFf0wJE79rmdoA0PfcnSS0E6Hy0O225bYJ8ZuO7RUOsP2PUJ4VCGQpAVxVUw5WJ7Sd4GhXyNicovN5lMudi3bumV5OMcLwfLwgIOzDzi+rnn76oLteo0dIOTjHCsqrq9XqZvgMCd6mFYznOuTBhAFXZu4/zEqyrJIzyIj6YYhudJ9IDQ9Wiic0VwONbODOcV8hrAOneXIPMe2HdEYJJJFUXJ8eIwSifdcFDnb7TatWbRhu9my3e3ItQGdUYx818V8ztD3/E//w//I9SaF3cpM4WMK1y61ZmJakJGubZhWE26ub+i9JwqJ1obddoskYsqck/MT6rqmmpQoo9iuVhwdHzFbLNittwTrkVHy+vKO11crsJK7jWWYTWl7S7utEZWi2Ww5OpwwKQtCSK7AmG6PqKxi03r6aFjIgjDUTLISGx39GO623d3hB4/zjvNHR3gcTVtT5gXCezKVU00XbHfb5NwTCm8j02LG0DmibVnd3KG0YrZY4F0KPgNBsGNRXITUVOahqxu00IiYjt1gR2aqhN2upZrMmduBGKFudgw2UJYTsrxMJjClubm9ZL6YYgdL3e4o8gnT6Rxr+9R5uVkzNA02++1F9NX9dkTnRerdFqnmIOH4bMrVq5pMpS9kMsPhYYXJBNtNy4vnK05OCyKSEFP20K6tUTLdHw+Op6w3nsEnDn+zGcidoNkNaCPZ1DsmwiNcIC8EaPDWUWUGqQWHh3N29RZtFE3bM5/lZHnGzXVNrkqmC4nJBf39Cq0kboCymtL3LY8en3B3d8+u7Tg/X5BlBa9fXTKdGTKTjecthUMrJckLRQgeqSMffviMr779mvnBjKqqWF2viEFCMCxmyXyQTDQKCUzLCZnR9E3NzeUVOis5Ozul6zR2qOlj4PXVBUtVsZjOyX1JIIW/275Dj87NwXa0fUcUpO7LEaHlEGzalqVO7GCFSDKAjLjQc3p+yHI5J8ZAXha4McPMuXuqyQFu6KnrHong8vWWH/7eD2i7jmFomBYVB4cLVusV8/kEKVK3t8kM1jqM0XifNIUsMxRFNvJ9BabI0apgGODu/p6qqpjP59S75KBVSrHZrBN+kzTfv3M93jqCD0wnS6IQvHx9xbcvVrQajk4WdH2ND47FQYXtHSqLfPLpOa9eX9N3PQcHc37x5i2VdRyeLHlzscLJBh+mPH6yJNOKbb/i3/zsFf/in/5n6FJzt77Fjd0eT5+e8q//8v9L5xvMTPLFRx/y7cVrNJKynFLXLUpHprOS9WbD8vAAkwUIA+fnU4bBcnr0FNtmvHr1Ft8KykLy9vUrjk9nvHm9Y74w2L4hDDv80JKXGcIHbPA8f/4CrQRGSVarLQfLQ+7XN/S2TkGWkzknpxWvX1+wpEKr/Hd6du+xyElf2geAvkNU7IXp/borCe3yHeFhxH/sRdD3hfH3UTD7n73/fjDiZPb6gxAPr9uzpN8Pmny3v++LuvE9l3XSKqJ4T3wU+38THtaoqXt+fA9IplGSGBtxBNcjZIZUju3uDuciCyHRZf7wfSOBvu/x3tN1A0PfJT0KRkd4wsaVVUXb1qOo7kYzmBy/U7oWg/WjuXQU5odhRCHp8f27h2PhvR/RaKkjc7PZkOc5IB/CSpVMSFul1MNxFEIm8V9IhEpdMUppXGK74lzSMLU27xVS3ndI749pWrvxYDhOJ+LZBx/wr/7Vf/Vw7qx15HnOdDql7+8fzvmeg/63Xej777cXlfci+vtjJ4SQ1tfGPIj8exE9jt8hceRTJwQxnaM8Nwxdh1SKum5Yr7cUhWMyyZFKEULSL5RKBmN8Kg5GCR5LCHuRf3SQv4/mFu8c48no/I6T/u73qSjxgNN5uFbeIW0SknuUgsI+W+J9k/S/f/sHr8yVgvdAFuhYgEiBgSEmQTI5a0emOMmuL0mscSUUqYEjMYGD2NNrYmKWSzlWLzxCGKRQacDE5FAXo1AuZXIjSiEJ0aYKR/AEIQl4VIo7QpAloVju3fCj05exHVnsE3XFw+ckUWN0u4/CeXL5vnN0xuhx0bFvlxZ7H3E0qSYsXBJuRMK3CBET3kbI0XGfREnkyHsS6r0SU3qvEB1R+HTsJKk6LQUonxwqusCobPy+EhlSQEKmcjpn00UqBHLWkR0OBGXRWqHKArstCEOeAg31hKg8jX2LL2qk98huQQwKoRR903Cx+4ZST5iUB8ynC7IsZ333GqNicvY6h1SCiGfwW4Z4Q5SSnIqN3yEkeL3FEVDyQ3CGzBwy+C15GMbUXhBZRJWabJKTVYqsUtAFVts1P/vrn/LFZz+mXPREJTA4nB24Xd1yVE4IzqX2WiE4mj2l2s7pLEjlEDIxLpVU5LlhGgtC1LjoU6XeO0IELTNiiHgCU1vipaKfWbreIcuOmPeYPlW8RZR0vaPvLTEYtDQJ0SIgGwTlQiA6cDYQHVgH0cvUpeFlYpmHFMISPNg9fSOmCll0IL1EhYR+IQA2jZuASF0YMaJlChVlZCr/LpuXHQEJ0oxjNzHOlUkLWRtCCtObTFO1MKbQiq517PoOdgpNxrTI0NIzxB5lPEpnyUFJi9EepKDvOna+ZTGbgCyxrkUpjwkOO2wJMgchGbyjswN5mRySg3c03YZNu6asJmQhww095axM11ow5FTEXuA7myazSmBtapvSXlBlU0x+gI7TUZDuaLotu7ZBFxk2evroCFKlro9inhLgnaVtW/IoyZAMClSp8LTYHgwZlZkgdIZXgcbXdG4guCF1KhRTJAElPF5GVs2Ky7tbJuYNhcrwvU0uG98jdMTkEi+TE3pTbwneILyiqjK0ARd6hHCjg02DtITYkyuDcJKuD/QikOUK7wdykxFdDsHw7PET/lf/8n/Lxx99gtIRnSX+nMmTY9t5Oy4cPDEmfmkK65APlfjgkwjunQefrj0lZOpmEYJcGw4ODlNo5d0dbddR73apgCglQ/DYweFwqasg0ygTsd3AtFyiZsfYzR1eOEwlCT6QaY0ShtwHervBdkkIN1Izn0+I0dI0Da1r8SIlj5fFPPGBhw5TZBA8wSe+s84NSmt89GQxx/mQEAB6ntpjFRS5IfMDWanROjB0HUpF8rzCZAXWpfAc5wPBRrzyxCIwyJ7GNezslt53VHHGVJWYPKezA84H7JDO32I6YZpN0CowuAbrB5SBGA3KlOgQkH//s/zfuxW5wQ4dRM90UjKdTPjq669RSjGfzZEZDP3A47NH3Ny8Zbvdsdluub29RshAURqms4oXL75DSoV1kbIsR2ZqSwq/UTgb0bJiPjvi66+f8/TpY4ZhIMsMTdvjQ+Dy8hLnW774vS8YOktVLbi4vOTRo8SifvT4KcPgOFoecn/7kl29BQaUHhDB42zL+fkx4Dg4nEPsefN6zcnpKSFKPnjyARdvriiyikV1xHR6Tnu3RgXQ+Yzbl1c0g6N4VlDv1nz3/FsOj49BSOr6HmTAukie5QzOYwNkWvPRoyf86Ec/YrADPgZevHlNiIHj42OUlGzvb1jd3HN0dMb5ow8YdE+cpJAxgLzImGclXVNztV1RlhWnp+fcXN/wqSyYzipev/6WYDTGKO63G8T1DVIbwoiGiDZNcl3v0EKNcyQYqz0pKGksUBmdJaQQPLRdmkKjNGQ5tN2KFy9XTBdJrHjz5i2np2ecnZ1xdX1Pnk/55vm3SKHIyoybmyuyDIIfeHR2gh1a+rZmdSfQCmbTirvrK7QQ+OARwvKLX/wKa1vW68B0OqWua4q84rvnL1ndr8mzjJvra2IwLOZLyqLk6uqGp0+eIaTkm2++IcsyqmrK69dvgRQyq5zm6uaak7MTblb3PHv6IU3bsFwe8Pr1Ww4Ojvng6VNevXyBEoKh9xwsjrC9Zz494NH5E2JQxCBGN3rOYnaAVor1/csk2BUV3jpUUbLd7ogxMJlOWMwXtM2KvBAMfUCgmU0L+n7D9fUrqgkUZY6Qxxgtmc1mNI0jLypCCOx2dWJ62oHtdpO6FP5We/Z/7OadTyaNEBk6S9e2qTNECPIyT0iTzoE0I5s9dYxoJZJTeLBA4ua2XY/WWeKk7xra9Q584kfvth1BeJq2YehHh5ByiCAxeQbG0nSpU6pvJEMbEFEzmy25vVkljEuMeBvRUeH7gAgKvER4TZnNyRdL+raj3m2wMiO4HqTDCAO+B6WoO0vRWQSG6WyREG0B2n6g7S1126HbjKwwXFxfc3icszg+Zjo55M23L7m5vWKzaZhNJtzd3NHVKRA5rySz2QKtR7GcNDW3g6eapEBt73xyW2mB0gozdpT2m5rpdIqZlKyD5fTkETqrmDrIy4rm5g6hDTGkub2IkcP5PGUFuIF+6EZkmUz3PCHIMsPtzYrppIIYsEOPUpqToyOWi5reRga7TZksQYDXKFMynZYsFino+fbuFl2W3N7csDw84uLujrZuUsE39sQQmc9mCVFSFljrkAEmsykxwqsXr7GbCRdNx9cvLjFqgrMKofNUbBYeLwLkBfOjA4pJhQ2pi9V6n4o3TjKZzplUaexF66lMzoCmyEv6ztEzsN3VuCGyKxtmhxUyCKILRJfmJkSISaGk7y1t0zJgwSu0MJQmx0WP7YbUMR0iQz+glCRTFYUuCbFPz1edTCpqXIBnpiDGtAZt2h5jUtGo62va3Sb9vGmYzxdoKblfr9luN2SFwuQZPljWmzXKGGbTKavVHSFC17bsxG9/fWdKkZUVvespptA0LUf6iGoeOPtEkZuCzdZxfXNDVi3Ge6RhuSi5u+5YHuV46TAqsNpsqPIK3wkWM0MxEwTnEapluhAMneP87JQ7s8LGgWHwZJmmXjums4yhjYgq5QVkpWF701OWaV2cac1u1VBkBV99u+XzH5ZUUzg8qLh6O7AZPIdHUzarjnbXUuUl26Fjt2n48Y+fMa1KvvnmObu6Zzqr+OLRGU1TYzLFah2YVSXODlzevObwoGK92mKjpMxnrO9bNjvHYiHJtCbPFa5vIHiWswXBRapJwp8M9Y6XX2/4/T/4EYaAtQPbboepBP3GsayW6GqauMVSkRU5SmTs1muEDuy6HdbD8ekz6tZz26W5WjERRCFQRhG8IzCgCzBaU29bymJKux24v98h0AydYAgSUxY8OX6KiZrf/PQtX/3iNR9++pjZQfoeg+vph5Z8YjBGIY1mcHYsAEqUTF22xmRIoUYDn8QOAz2Otk1da3Vb461FxEA1mXN+pumHjulswtXNJZvtGoGk7xoEivtNy7aFi5VHVxX1es1QB/IsEgvPZJGlTvhuTbQaIRx+aMmU4ezEUExyihJOf/SENxdv2KwatBUMLqAKQzHP+fL1NyymSywWFTKUNBidcX7yiD/9xV8Qi5zYa7qdZOg7FouKAGx3DVKkdXDvLNqAFp7ZMmLkId3Wcjj5kF0pOT1aUCx7qm2gLCa0PTiXQq1XF19xen6OVIp5MePbb294dH7K7c0tJnPUu543lzWn5wUhdDgBm3pHOStZHOZsQxiTOn+HZ/forn0QA0XKDwx7ATCODyERE0pZMLplYQ+K3QvZe818b2r89yFX3ueZSymTFrFvW4zxXUc3fO/nSY707BHL74JLR8Ppg+i7pyo4hEom1RhSd/M++NaF0fgaARJ3XLLHhAik9MggQFqiVAg1QyqIPnUCSTFgQ4f3lmHo6PuWGANZbghkyeSpI7t6BTGOc5zUmSfTw4EU8OmSI9skA9ZghzF8Vz4IriEkNN7ecZ2MRwVD7wjB4Vzq/ksGsYhz9qGo4Fx6fyEFg7MgJJlOWDIlJZnSOJcc3ilcNjLGQSLEmG/GHkUyisN79zVyLIB5irzgn/3Tf8r/5Zd/w2q1BkQKEw6Roige1gB7nMteYP53udCTmzyha7yPeJf0Ku8Txi5919E1PxZJkkC913BH5A8e5wYijmHokCpyf+d482bDs2dzQsxwIZKZLJExxu7ZNIjSZwd8MlruiwYyIcGDHwXwB3TQe8WGEIijlhpH/Zk9BweRzNMj1lsJnTrrx88OvoeYuraSEfzvzzP5B4voPtQooZPjgVEcB4SI46BMDloh+rSjD3eXxNjWskwifBiIYn/RBEK0SGGSQ1ya8Qu++8KaIjnD4/4zwxhVShLWRUDuqyZxvBmNeBEpQKESUiFY4lhp8zGxtIVQiJiN5ywNyvTnfBTbkws9ipHJLiDiibEDRse2SOGRiVsqIZq0kI0R8VBtc8ggQewviHF/ZUJQIBVxbJlITHj/cBXFGIgipOOaXSPkEVWZhCEfdQr0HI94FCqJ/kEgjEQZh8y2JNaIRE5ajC4IXUEYDFqeUsfnuOoSoWuUn8AwR0mD9RopPYR0kTo3OpSEwIeUPFwIgbM90g1EWXPnf8at+zP6uGWiP2CW/YjGf41nh1Q93q+p8jkSS+ZK7K6jUjkhgyGDpgjIUpAdZJQuY0YGBH72yz/lD/7wB/yT4/8UqgNE7OmHW/qhpu3WBBHGYk1kUhQ8OllghgzU/kLQ48PBk0ZdEq59iHgvCDEJ3N5HSheZ5JIQBZM8YygElQnMCo9zAmJGbz3dMCR8UUwP+N3WUjcwWAG5Srxs7xFRYHgXXEGUeJfGqRxbnZSG3Ci0jng7rhWFIVMCUHgHcYjEIBgcdK3H9un61jG9h3W/Y7AojqbtkMJQlVOEcwifKqODtfjocMGSZwrvAl3bomRAqIgWisH2oNL+hGDZ7laIzFGUB3gfGWxPUZbkWYW3NX2/w4cUINgPDVpDnmm2u+SAzfQUk2coLem7lpAZXLAEGbDSM89zKjWjG2okPt1Uo6EoJwilsTREGbHKUbc7uq5jYsqxDUsxyQucb6ibewbX0vmEN+qcpxvakS0ek1MngpaaqqgoTY6MIrVXhoGu78hVTp5VFFmGl4YoHEZlTGZT2maHUQkDo2NgCB2DF2S5wHsIwhEEqFzghURoAyoVG4JPsxKpErs2NxlVmYN0CO/wo1OnrPLEO4seO7QJZ6J14nQGRZFJ7BBRIuf0+Bn//J/9C54++RApQZs0Edtz7vYhIj4EICVsC6HwY4EzTdZiQkOM4zszhsxonLNY1yEF5FnGpEzhrXleMp0uePXqFXVTp7v4zS29XeNcIAqYLWbj+E+BYNVkgfMO7xtms4K+61LwqsjRPrmEu6ZlvlyynC+oZjlCOkJ0WJ8mnTFGlNDkRUE2neP9QFPvIAa0UlSzCYOz6TW1ZVpNKMsZmcyJQ0NWZEgl8fWA9wPbZsPV6prD5RlFVeJj6hhyfqDr6uSnEDE57LoOh2XwSXRVSqK0IAgHOuJd4q1XeYESkkDqFoghpsWEEQhZEMew56h+++t7u75PDtB2R9u0BHfAcjFPE2MfaOuG3XbH6ckxJ6enrNcr6qammhR0fc1yOaUsDV9//RznAp9++hnNrqGsCszM0PcBO0Q+/PAznA3MZhk//tGMfqj55d/8jB/9+AvqumGxnLPZrpkvUkCvyQRNdzc62KcoZRJvMgbe1BcMQ8+nnzzjz/78JXl+zK9+9SWffvoxZ2fnfPvtt6xm9xhjWBwcsFrvOD1+xKOjx5zNH3P16gY1GNrB07Uwmc85OJKcrgd+/fXXfPv8OwbnaNp71NZzv1phg0UoGAY3tq8bVCEJjeWDR494tDwAIeijY7fbcHN/x4vnX4MLhCh5/OQjhsmMb7/9ik8+/SFZnhElZEajUXjb0jUblIJHT59Q73pOzp9xfPaMrt3yVz//KbrImc4rDo4PqS6uGUJExgERA0NvkYx8c5MjhcL5tFARyuAQOCxRCoZgscGN7tM0wRcyUpSKftgwXSxAwP1qm3jkpqBpHIcH51gHd+sty6Mj3rx+xWI5H513gswIri/forSmrmuIPmVVmOwh9EwZwbff/orLy7ccHZ7w0QdPmM8P+dWvfk2mM9arNUpINqs18/mCpmnITMGrV6/54IMPODo64auvviIEwaNHT3j79oIQAovFgrvVPesxDM2FyGy2YFc342JMMKkmdE3D6dERp8cn9G0LHo6PTvjuu285Pztnu95xd33HJ598Tld3vPj2BUoKqqrgcHmElJKvvvyKPMv4+MMP+eu//hmTyYS72zt+9KMfcn3jyJRm09xzeHDKtNLc3rymbtYsD+bc3N7gBsHs6An3d/f85Cd/wr/9tz/l0fkJz559iPcD6/WaSVkxnUx49fL17/TsXuoSY8FuLbWzoCUmP2AyOSbG1FEkQ+qIiyKFOEUZGLxDSDkGqmUEPxC7gBMdrdmxu2+w3ZAEd6O42VyhjSKMDF/pBP3QIUV6PmXZhESPyGjbnjAI6vWOerPFrhuGbiArSso8p/VbBivY1ZbeDmTVjqzImSxmWN8SoqXtt2gZUVIiZQY+0jQWyYC19xT5JD3vy4KgFfe2w7rEW99te6ayoCpzrt5eENwlh8szTh6dkxclm/t7mm1Hu97R1i0ZDttCKzu0aZHGE0VP0wdMXjFbHCa0zcs3FF5xtFhgGVjVlm3TsbE9j06PKBYzhDXoyZLZ8jHFIPj5X/8F2kSC9SyrikmeURUV1g98/e1XvHnzhj/5yR/j7DAGmiZE3/XVHQeTOW+/e4XISpq2w1RTPn50zIurFV5Jrq8sfZe6ixZHU45nJZ8/PWZze8v17Q2TvGD7+pKyKlPoXqbpXU/bbDBSEKxku7nh6dMzpIrIoOj7wHwuiVKQHRxxse1YNRbbRXq7IwZYHh3ic4WclDDNcDqjKBYM3qZ5DeCFJGiJCOC6nsPJEhF6uqCIekRLKomwLXkZCTE9Q6/f7ghANavI85Iyk/jeYQeLUgIXHJtmhyPSt00KcNeGrpOUWUkeDXEI2JjRDg5VGMIA1ntCADsIDIqucWgZcYPF655iUuJjhwsDvgNcwO4aqrFA07kt08ksGWwGh4yKbtsTvMTonEF4ri8uWX4+YzqpWN0JwuAfwtd/m+2jT85ZrwaW5YKBNcPQIpDkucR2EbcTFFlFu+m4ebvm4FiRF5oPPljyi1+0dL1ldpgl8VJNqdctlcnZbDa0rufocIYU4IZAWRRs1isQKRdCqowQkpi+um+ZzXKMzpCq4stvXpHnAucTPmxXBwI51mqOTwt2dQrxnM2W3N+uOFgegHDMZyXORYahY7mc0XU9f/pnf875+RHnjw7o+hajM7puR54bIknI324dEseQWw4ODrm/XdE0DbPpIYO1nJ/OyTN4e3HN7PFBmlPOSzabDZ9+/Cn3txcMdUueZdRNzS9/9Tcopel7x9F8yqrtqXJNe3eHlIaj6QwZApumRytYdz5x8IWij4Gb7YAyGi8E3jvuVytKJLnWRBHxRLRJrtftZsd6teP05DHPnn3C869fcHX5cy5vb/i9H3zCDz7+iOe//o6Lqw0fPMtQKrk+Z4sDZrMKISJ3d9d0fTLeFWUOQrFab+n7gels8iASZUYmdUYIYhCU0wlSZWxXa3a7HdE7dnWHD5EsN1xf31I3LSYr8MFxd7/h7OwJ0kz587/+a6woCQLyIqcwhr7d0bYt80W6fk2Wuu/Pzhe0bUs/WD744DFnj56kcORhx2effMJ3377krm2ZHWRst2vM4ZxVveFPf/rnfHj6jB9/PuP+/oq73S2vrl8zO1lwt9uRFZoPP/qQzX2NkIHB7pAytW+/+PqO2cwhzjRKdQghabpbdjvHD7/4Y+TVlp/9zXcszjzL5RFvXq84WC6wNoW1Hh0tkdrT9TURw/nZlM2qxagFl28uCFEyXxi2m5ayihwdniAwvHl1hZSR+XzK+n79Oz27o9iHSsoRc/K+iD7+m5hEXzmigRnF09RFHUYB8T3kysghf98xvnfnvhPR939O7GnC3mQ6On1jElUZXdGpgDm6e0VIGpuQ7BGgeyNAoi0ktOX+u+157HtnvdyLpfvvRvp8PRaBEJEQLT7WRN/TWRjcjEqcoFSeTLgMBO8SwjgmDEyIAe8t/eAYbE9d1wggk+bBEe7saMeNPLipBYIY0r57l3TL5PwPD4Lz+0Gcu10Kpwce8gVjDDiX0CVCqNTJz94lnboNfAhY7zBk47GKowM8dV4mATiFuCOTiVGKZIxJ7zXu+J7jMgq8WqWO1bOzc37849/nV7/6NVdXV9zcXDOfL5nP56xWqwdBf79f+w6DvZAuhEBrTZZlKYA0gA/745GK2D6EFAgtBM4NKeA1+LGAoxhI+BpjUneEdZYQFUql77zbBC4utsyXE3QuqAQpZ0copEodAgRFcKMjXMWke+8F8HTUky4iJVKZd0cjhGSofiAzJMF8P/69Hw3cMRB9MmILDErmqZsjJtNqGNdbPDj+/8PbP1xE9xClJ0SBEgXIiI8J3C/FnvKzr5QkxlkkiRqBxMPRckKMihiHFA6BI+DwwY3VMwfYxJYeneBCOGTMR5RLqm4wIlckGlRyxvtok0deSIgjs3y8aRDD6Nr6/7P2Z02yXfmVJ/bbw5l99pjviBlgJplZLFKssqZkautHPehr6MvpRWZtJpOZrKV+qKpmVSaTZM4ALnCnmMPHM5+z99bDPh4XWV3VlZ0pf7lA4CLc/cx7/df6LY11DoT14rvQeBp0hBtOXIkvM7RCY9AefSIdQrQ+SOH8Qe1LEQBr0QR+OufVdH+C2JbOHmzEfuo2SKYoQoQYdjj+szksxjY4Ws+NQmDcUDYgehwFiGtC+blfMA984UDHQ9wkGHh3XsyXKkQHmiBwOOld40oGOC3pdA2do++vEeEDKshxokZGGiP89FHrCCEanGtpu45QeYSCtYY40sTxDK0zlOi9kC8djbmn4oqWFWX7LaGeDNdqgdZjnAYRx2gkYRfR9TGpE1htaVNB3li6VBIdzbHKeLHJGjbbFf/z//R/54tPXzBPFjTGcX372jtaA0c7DFakM4AXemOpsaIfLpL9cJMZLgjOIj3dGTdc5MUwIc2ymOkkpTcMDDF/Ie16Q9c5ut5SNQ1t1yGVQOuQvu/Z7LasNgVl2eOApjXUnfMuvSQmS2KEcLRNQ1n19HY4VJRBKEGoQ39hMpV31o4nJFGEQPm4cGfojaHrLUXVUeQ1XefZ0r3tKMv/9sTsf+s1n8xJwpaybAl1TByn9J0jiUbU7X6IDjZY5wuUur5G4LeNdzJJ9ts9SgmKKmdXbAlGCqlibG/pe4cMU7ScMkkzXDYmDH3cp7cG23rOmUDBcBM2WIxtsbZHBgKpfZGjxVF3DZkGHfiblnXeYSwGYbgRBqEGl3brv6NzvulZKocRBb2tqOuSvjegNF0DfeeQDvq+palzCmEHt9aUNE6JdIRte7QOqOqStm0YT8dk45RIRFR9jzG9L74JQnSakcjAt8R3LUGrkUqRjmJO5ynSgGk7tNToYExe7mm6Aq0dXd9hel9oksUpSofo0D8sawRIyPOCSZKBg67qh2tJQxQrmtxQlhWBzMCAjhRPnz7l6dOnpGniWcuCx4eDP4wACgLtP6t/KLLDfeDDRPvDTc1f95USSAVaKwSCyWjKdLLg5PiC05MnvHr1isvL99R1w2Sy4PvX37MpHLPZnMViRt9X7Hcl1jaM4pgkPKbpcuJUU+qAqupwTqAC/7kQjjTxRZXW1uTFmrYtCfSYQILte9JkxHiUAYa2dfRtjJIB0/mU3jR0xrDd7uj6hiiK6EwH1hHF3jWYVwV5vaVzDbs6x2qBjDVVX6FFhxCCQBqiSPneCyVp+s7zrQNIgwRS79TfditsJwjCGCctQRggledX1s2Wtu2RIiFUEqfboXClpTQlgfzTI+Hb7Y5nz56hlGJVramqK46PT6iqmuPjY+7u7hDCu8qd8JHDqvIMQK0Vb9++4ZNPPuHs7Ixf//o31FVNXbfs9xt8gXrA0dGEj15+zM9//gu/Hbvel4AFEZfvrzg+OiNJJpTVFmMM9/drRqMxX33xI37+83/m+fOX3N09sN3mhFrQVhVPn15we3vD8fEJTdMyny9Zrbb4gm5fWrR62ILUGCNpG8NuW/Dpi8/4+MlnHE8W7FcbxkmCcv77rHb/D379zbe8+v47tlXO4njOu6v3A8vdJyTG04lfkPUNT04uWCRjjo4WSAFplvL222+5ub3j+vaWrmk4PT4llAHb3Y6Hdc7Ljz/n6OgIPWCdnDXkZUFT5igpODleInVAYXKOjo5o2471wxXTyQiherQU2L7jeDmnKmtC7ct6+ran7Qz+gfzwrKFwTvphMI7JfEaapVxdXg/OSoMxEMUxi+WYqt4znY5IswgRGHarmt224G//9lNub+/57tUrnj17jul6bq6v+du//VtWq3u+267I98VQHNxzdnY+FPz6ToZDZFVKRZamvH7zHePxiDRNHx/Wv/jic5xz7HY7RqPR4+AuikLu7m54+vQJQsA+3/HxJy+5urpCCEGapqzXa4JAoxpBlsYU+Y4wCFCjEUopNust6/WW+XxOmia8evUN0/GUh4cHxuMRt7e33N3dcnR0RFGUfrHVVLRtz1/95EdorSmKnNFozGKxYL3esFzO+ed/+Se++OJzbm9vOTpaPi42tts9SZIN/HHDeDJiMssoyx1fffkVm3XJ8dETrAu5v/fpi7oth2I4ePnyJZv1mixNmU4mf/K5DXA8P6KsOzb3ayaLI8aTKU45yrpgvbqnzPeooZyrazuSNEQqyz7PkdQ4ZxmPxwhn0QJ605FvN5heEEURKnBY0XmuuRVEScpsOmO32VEVOUEYsO87wjAgzRKiKCEMHcko4/b6hnK7x/WOvrVMpxH7skBoQZLGQ9Kipuk72qojykLGsxF9VyOFLyELdUDT91SNARcShBPquqOstqR9h9AQRIrxJCPLjoiTkF2RE+qEUTZhdJxx/f6Kt29egYg4mh9x8fQJm/t7uqYkDB2jscapjs4aigZCKQcxXxDFI+IoIo1jnLGESBId0VYVXdsio4jpfMZ4OsE5PCqo6TEq5pevviGazBGmYD6dM9EaZQ2r1QO7wpdl/ujLH7G6W7Pdb4mTlChMyfc1k3TO73/9O65uV9RGeU7v+Qmzi6fMJwn/8Z9/RdV0IKU3QWzXLJ8sOZlOuH/3ht1qg3CS2WhEW1fc3m6IsgSVhCRpRl9XGOMXj5vtjrMnRxjXoIOIru2o255NWVM0jt54t3bblrjh+chq//yprKCtDG3jET191WPoaJylNYZpOvZpRlmgA8W+aMimE4xzlHlNVdRIIZhMxpSbAoSjaQ2ZChHK42+sERz6rrq+Q/aO3vmixjj0GJ71asd8NCFIFfPRjMT1vF3dgYAkSTBVz3abg3OkceJXADbAdR31LvcdMoHA9o6u7dEouqYlchALhdWapqnJwtDf652krTr6HtIUhLXs1g9887uW84tTAgW1NYg/YwD+3Ztr5oslo2nG16+ueDoNfBoiUtg24P6yIopaRtOAMjQIUXF0PKJTLS8+mvHq9R0ytwhpePb0hNftDR+9fMqu3PP28hpnIYgkoQ5wVjELQvbbgu2+YTLNcBbKoiCKNEXRkSYxvbU4KUiyMQjY70vqxpKmERbJ6emU95c7BBAfC05Opuz3JUVR8fknT9luN+R5RV3XtG1DmoZUVcF6fctsMSaMQ6q6YTGd8u7tO286kg5hBX1nyJIRaRojhU8ujccR6/t7Pv/sJTfunqbpSEcZ8+WSzhq+fv2WprwniRXr+4bZdITF0dYeF1dbiYgTVkXLzdsH7u53/PWPvyJEUHXeTFdIycN9zvL4lG1+h3IVN3fvqZqGWMeMn71EaM/1DaTyphLr0z3z2YyybNluN6zXWx5WW87Oj8gmY548OePrb77j7vaBr/7yE7I0ZjyfcPF0TtvWJGmGtYb58hilxFDS532Wo5F/vtju8gFvZoGYSGnflyUUQgTEcUou9+yLnEBKjO3Y7fckaYKQgqqpaNqWTz/7FK1T7ld7pqOUomm5XD1glSTMEpAWqSVxrOj6liwYs92uieOpN0YEkIximrbin/75H9E65uzimJPlMRd/d8H777/mYXvLeD5ml1cE8xCXCL65+Zab/IHrmzXPPz7jprxhspyTBQnfv7vk008+oilbVusVSjl0HCKE4/h4hBBQN75rKEkDsA2m3fIP//Tvef78lHiWUvcl335zz3QSEscJb9685fx8St3U0HSEkWA5n/Hd6oar9yV9q5gvMnSgOD5JEKpmu9vTVDVpMma3LghDzZefPKGtqz/r3g0f1l/A4xrsP0exeAKEGnSygZQg/tfi+A/F6oPI+oGlfpAbD6/h7wzpBf8Td3i7x9/pNajBlS688O/cfy6gexPjI/VBDCQHC9b5z2XNIMKr4bMMJIg/lIW9mdUL7R/E6bqpMKYlCMC1FmN6+t5gBkdyEPg+oK5rqKqaqi6HHgNN1zeDuB1iTPe4dhWDQc05j/rz3/UDC7wfjGIHxIsvKo+o63pIEfqiTs88/4ASiaIQhv2olBq0QYGQ4lGUP7jc/f8vHj+PQGCNxVmBCOwfHBc/PE4O211gkErSmZY833N8fMT9/d2j6bUsy8ei0AOe5ofH3X/+8zAMP5SPIodhg8e9hGH4yFY/fJZH7rzWBIEetvOh7LQfRHZLkiSEYYi1lvVqx+V7X1KvVQKiRwcWMWw7IdVjqa1z0m8L/NBDSY8O1/rwGT6ghixm6OW0w7kxiP0HBKbFJ3b9pArjjDfeCe1JK86hhMRJCcartdb+twfgf7SILuRiEHJL3PChjbNYB3IosfNBu8Bb751EigiH8+KwrfxGEZ7JLW0PsvdTKieQj+27fhIl/OgLQ0nnKl8CJ/ERBuc5ve4wofGbjkOplnPGi+1CIpz/ikom+MJOP811on08cd0giHvkjMPR40SAkyGCGkSNEGY4kMNB6BuEIwFK+uhD7/yJqJx3EHuhvcfhp2XCeSa5FQZpD0x1z2tHOITsAYuwdhgiWKRTA3+9Rbpu+F4hxjqauhlcZoF3bfYWQUCgfVmL1AY7/E6EwdABGqm949+Fd6C3SFn7i1/gXaEK6U9C55BS0XQH97/D2paqWiGVRgaHYp4G5To0HVbusLb0DxRyj3Jj+r6iMoKZ1IggxLgaEYYoE6Pbjtha0tCSpZK2A1TMJDxDDxc4XM/t6pZ3r37B9MlfkS6ekmZTrDboSYIbOM7WdTgs1rYIaR8jSP5K4G8shyiMktpvF+fZ8+CLNJIgJI4Ad+DiD/vChVjnC8V640Vw8DEQYaHvT/0is2vphgt8b3q0CkjCMYEKMLajbRuqpqNuhoJFapx0aKmHCIuPC6eZ5zof0gjC+Sl03zvyqmGfV3RdT4fBWEPTtX/sqfxffCVRgmkdip7JaMxksqCpOs8x7DeUnaRta9bbewLpXYi18KiPLvQ3jbotUaFARiCUoO06UmqkhmjgUSnhB18y6FFBR9v7go26qdBKMUonSKFpagOuw5jGD65Kgwg+3ITWmxWhjRlPRujAF4xYY2m7hrZvqU3hGb0q8aKu9JzAMI2Q2rItN7RdjUAQ6BipQxpr0TGkUcZuvwbhkMr4GNahNNFYurpHKD89ts5zNh0O43qM6BHKoqVCEiCN9S7A3g/h0jjB9AIlI8IgpmoqTNsxnoxRQTbwLwViSFFYa5Fty3iU+XhwWxFGmjDOaNqGKI780KBuCEJJrBS2NYRCkSQRzbah7ySKgCQe8cUXXzKfz34gnvvBxWOxrbX03YAtEvLxAehwEz+0jR/azT1f7cNx5At+wiHypRkDs3nP8uiEp89e8ObNW969fcub1++4vr5H13tG2YQ0zSjKligKPXYpiXAipO0lVvREkaUzUDU9YRQThwlxEpKNExAd682Kst4ipSQUMVJqjLU411HXxVDWwpD8iId0hGG3H9izqcZJR17uCFXEfDKl2Nfsig1Vt/cDPWlxUtFjafoGHQmwligCHcbUPZjO0ZQ1gZKEQUgymjBxY3JTcvdwjxSKaRL4KHmgGY3HVOWGtqsIw4QkmRPqhKopyYt7pHIIDa3r/uRzu2t7Hu5X/PVf/zVN3aKDgDRJPe+763jx4gX/8A+/IAg1L14+w+GHU7vdnu12z9/93d/w5s0b2rYHB+/evWM+X1CUJT/96U948/o9OMG///f/gavLG/7mb/+Gu7tbBBIdhFxf37JcnnN7e8diOebjj5/zm9/+miKvCFTF0dEJTdPz7atXtF1BloSs7+9YLucIoTg+PuHh4Y5Ae95/FI5QIgUnqKotH3/yEV9//Q0nJxfYDuIgZjmdo4Xio+fP0U7wq3/5J/K84OLpUwySvGkY254wSRiHkqoq0TpgPBuRRClZNuLJOGExm3A8muMwvL98x0cffcLt3S2/+e1vqdqeo+UR17crTG8wwMnJU6RS1HXNUvnF9KE7RQee09q1Nfl6xXy6QAnLOIvJ4gBMx/v3r/nss4/p25qT5YJ8n7O/2SMRvtBSWlSgh+cDwaEE3TmPJ9FD8XhZNoxGE5zyqackiQjDgLq1pFnAkyenXN285fzslPXqa+7vV8MiACaTMXd3AalI+F/+l//AZ599wmQ8IU1SxifnNEc17969RwjByckpaZpyd3dLEIRst1seHu4GNmI/PMBrdvsNaZJRlgVhGDwuJKIoYjwe0ff9kIpx5PmO0ShFSkGSxLx795aTkxNOTk64f7gGHOfn5xRFzosXLxBCcnl5RdvWCOE/v5SghCZNE6ztPVs3CvG4QMP5xSnG+kFV21Xs9y1ZNvKDJOcwpieO42ER13N2dsbl5Xv6vifLxmx3DThJXXfUdUPbNZycLFivNlRlR1l0nJ285OrdNePZhKbO2W0Fp2dLHu7uODleUhY5Sqqh+O1PfwmpOT1dIoMV42mKpWW3WbPZrWjr0hfRWgfCsN4UCD1Fh5q2h9k4JI5CdKCoqpYo1kyilN1+KJoP/f1Ih4owiumdIYpDgiiiagx52REZ/8zNSNH3JWkKx0czurrjZr1BOcV0NiVoa0YzxaYsmUynnJwd05mGdBz69FakMZ3vzwjCgLYu0UM5dlHVbLY7pEhxWeIFVdFTVltUCMYJzs8niE4jxZTeWqLQ85UlgiwbYRxcXt/y6vUDT84vSKcpT0YX1NWeJA3ZlQ+EscLSUDX+OUMr6dM65Z5JkjCdjkhUyC7fcvNwjYhglMyJdQStQUSgdMBmt+d/ufoZ02jMOB0xCkeMJNTbNfV+y+vvXxNEMePJnPdvr7m/v2M6m7K+v0PKkCgc8+rbb/n+7Vu2Rc38+BxjFO/fXnG32WODjL/47AVSKW7u7nny9CmnyznTKOT+7WuK1Z7trmaf12zSnvOTORCCCVld7UD1HB1NUTom3+9QoSUvSuIkJY4TOguj6ZzyzS3365xsPGc8ztgbQ101fnEsDKbucPRs8w3zmUfAtI3FCYOKQzB+jSekYJvvsMZQNQ09e9LxCDcgDOu2wVmQYcD5+QmNbSjqmqKuCHWEcJKqKoiTkKqqaRqfogplgDQ9RV56h21rWa82pEnC0fkxaRCzyveEU00rLFmceraw0TSVgd7QtAYhGgQOGSlQwqdUsQip6JrOO9C1JktSsIY0jtncddD3SNXQ1RVKge1aLt9d0TZ7X2jtPDriT30ZYfn8R+ekWUJtxhhb0XaGdBxweh6TxXB/WzIepQQSit2Gpmw5PkmYnUacn2dsdwVHRxNs35KlirfvLnn6/IKLM8N2u2c+n1GVNdtdx2SSESYBc+3XHkXt1xVNY3jxYknfNezyrR9o7Su++PwZbXcN0hGElvFUIaQlCCLu72oEW2azCUkqKSvHer3zpdsqIQg62jZgsZzRdS3b3Z7NZkeeV/zrv/4Jr169Jwy9M/Lqcs0o0X6t7xxaSi++l9oXvC+nSLyp4nh5zP16zXhmmC1mXN3+nuNlitIg8xqhBLaHvoe27jFhQO963n5/ial7rGl4e3+LbVq6rsaohqqGKJjT7lY00nD1/g1BZAmzmESnTOczRNUwiRPqusIKjylAQBTFZNmE1WpLVTYcH8+ZTEYIKf0QMg44Pj8j3zYsFnN0BJvdllGWkBclWks/sNUaHfhC0UBLijwny1Ka1mMtwijyvUE6QgcxTngkZV74Y6apW4gCtA4ZT8a+lFEGxGlGnGYEUYLUHferS642DUaFyCgjrwpcV3N6egRijKOjanxaMU5Dbm/2nJ/FTBcxdb/n6UfPmO5G/O633/HNNyusq6jKlqNFQNonhCqgUz37MifLYuJJyG1+y+z5nJv8HhcrOnqmiwn3mxV1u6PptmQjyX5fQNsyylKmM+8w3u0BMWe7K4gChnVWxKvv3vDZF09pNzVadzSt70kYjTImkwlXVw8cH52gteD2OqcsLYt5ihA+HZ0kIbc3t1w8nbKYp+T7jvX9JVLErB+2NF8Itpv+z7p3q4HRfcBSeA4zf6h1AyAQUiMOrvVB4PQuW8vQjfgDI9SHNO7Bce6FWryl+fFNBpF2+PmhYPTD2x7wF/5fpfMcCq9S/qELffiUg+7mu7cQB/PoBwf8ozjL4T0Hf/3w/lIM61H3gXjRtQ1tWxEpvw7yKWxfvC0lGNP7xO4wMIijGIC+7whCjdYKqcRQSCp/YBrzWo21vk/HI0k+mCeFEI/ir7WWMAwJw5C2bZEyeOSrH7Y9DGvjzu/HJEn+YF8ffrdSPIruB/yOF9Gl19rkh2NBCvnoWH8Uzw/iOl5c9j1HHgsH8PLlR2TZPev1iq7zXPe69v/NG17E4/sDj8OAJEkQQtA2LQcu+EEoPwjthz9/+LODuK70H7LHD3pN2zbegJsE1LVhs8mZTDICbYlij9uSycBaZyhLZqBqCD88UlKjhR62tzwcQEMZqd9OVoohbfk4DkK4Awr5g65sjacXIN3QIhX534FDOuvN1gzS4H/j9b8L5+JEi3MFjhLD4AZDIoQjEF6AgxoPI9b+g6HQIvUsGyqgGazyApyPGUvrcKLxB5D1JZuPJ5xVw2St9wtFBJ3pwTTDF7UDFkXwWNAwRFGsGBAy9Dh6QD1GI3DGu4+doXfCF3sSIJxvekUm3uHlKqDHOIO1DbgahMINXmak9gc8A8964AOLAwKGofSHw44xQynr4VKkfOGkn3v4g0UppBWIAY5k8C57f7GRXpAXEmTIdr9BCkHXWHABWvrFtlTmsXXY0SORGOtjogKN1PUg4Dd+UuYAq9FOo6WkV/4kFVL6NMFwYW7airouCKMK3ZVY0dHWK1I2ZEEM1n9v6zp6OyQGJDjZ0cueThrK7hqnekSgEc4QdBBiSBKoe00pLUKlZO6Yrm3YbnL2ZcvNesVXriNLM56+eMJ9e00yjlCB8A4aFIihhdiCOEQ5Bo6XEG6Y0Pst+aGoQQxniwVa32aPP6EZhkT+eB3agJ3DESEIYSjO9QkLiXXBh0ICp3wpo4iQQmJsg7EB1niGuR9CtYOw728cbkASPUZbhtIQMVxUnFU0bUrdjj0zyvrBgHF/3s18v98BEmt7L6daMzjtJVEUA95lURQ5s8kEqbxjJ4xDLyz3LUGkcNoiFYRpQGc62q7EGzIVzu1pWl+OZ8o9QQydVSAUfQ9915PFIUmckkTQ9yWm73CdLzVqioYw0EQuoGhLetPSmRbpYqTS+NyH30cuMLS2hd4/OKRpQpiE1Kal70q6vvEX+zAiicY4Jan7GhVIlOvodITVhjDx4irCUpUlOImW2ncfOEs38PjbvsEKi1WCvmtRQqGFpKk62qZGZxkyhED77SWdQvQO1x8iRhBpRZYmjHRA2xes1hVtM5Qbti1VWdFbgxYj0JJIxVjlqIrC43J0hNAGpbXfN1FEGCZYE6BVzJdf/CVPnjz1N2sphsbwg6NguM4bO0zVvYvVGn+t9sM69Tih7/veLxasGabP4lFkD8MIrYPHwlVrDHGcMBpPOD0759PPPuP+5oGPP/6UX/z6PxJPBGk6AXqSKCRUGobURu8MBkMUJeggRuxzpAsZJ2NUgC8yrffUTUmgfbeGEL4IVoqQ7X5N17XM5hOU9kxGRECx2iOVIy9zZCDIstSXilUFDkPVBfR0yFAQhwlVW1KVLUk6JtIhgdKUxZ4k0ASaoczFoIV/sIrjlCSO6PsOK2C921J3DVjBZGSQQvnhjobGeDdhnKRoGTLOxkRRSFvv6U3trwGH2OOf8JrPF1xeXvKP//hPPHlyQV1XvHn7lvFoxHbb8ZOf/BWL5TdEUcTV1SVHx0sW8ylv371FCI/ROLgZnj1/Rl21PH/+nIeHhH/8x3/mi8+/8qWAVvDlV1/y7u07qqomTSP2u4L5/IiL86fc3jywWEwBPKrIOS4vr0iTKU3dU+QVH338AtvXRFrx6tUr78KaT+n6jq7tKauKNCkRSAIdcnryjLIy1LUljjP2xY5//Pkv+Pu//bdk0xldVfOr3/yGcr+n7Xp++etfU5uGphfoKOT5xy+YzMbsdjvu7++5ePLEY0CaliAJmE/HvDi+4OrVG6SR/Pt/+A+8vbrmu+9uCbNgSDd4F+6Tp895+fFHfP755zx79tQfjwLaqqEuC0xf4UxHXeekSUTXlGRxSlVsUVjKfIcWjukoRQtHXeQ8OTtjtd+y2e68E0P5QlFjBoydYHCchfjFkvrBuc3gCmkwpiMIJdNpxtHxDERP09ZYHD/96V/ycL+i63r6ScdqtSJJIs6mS+7vrvnuu2+Yjifc3d3jnODkxBfDbbdbmsbHZvveoHVAmmYImdC7BikVcRzy69/8iouLpyyXc75//Z1fnMhhCDnEQ1+8eMJ+v6eqa07PTthu1/R9x2ZTE0UB0+mYPN+hlX/Yv3jylO1mi5Q+joqzzOdzPvvsE77//jVt0xBHXmwNo5DV+oGXHz2n61rWm3tfBuUsT5884/7+AWcVVVUyn8/5/PPPODk54re/+w1pmvL73/+O4+NjsixDCMF0OuPy8g1HRxNM75/h5rMltzd3JMmYvjPEcUZVtYxGY6RyJGnI5dUbXrx4hhCG3W5NmsaAI47/vHKy+/WG2dExs/mEuilZbx7oTEsWayIR0dQtxuG7crTHAYkgYn50zPOzJWApy8IPHJueMFJMREpd9khhsfQeXxGmTCZzpNJsN3t6I4jijDSOSZKYNI2py4oqr1hMx9RdQ1m1RGGEFY5sGhKmlvkyYDQeobSjd47JeIQxnY8JS6iqlrarQUBV18TJgH+KEnACS0WchiAlUeyPofFoTCASNrsH8nzPertmikAqTZkXpEmGkIb5Ima/K1ltb9gVmvl8yvnLM4xtWb+/pBuQZsY6AuXxFbu8otgX0HeMoozFcsFvf3NF0zQ8Ob9gnGb0RUtVFhxdnNBUHZc3N6Qu4ezogryqkEYOAr3g7dt3/voSJnz/3WvKfc58PmezWXG0OOLhfss//+NvKYuKxoSscstDfkUaKKapoixr0ouI58cLjk+WrM+OmM8W5Jsdm/fXULVkQUI7FoRZSpm3dM6QRJqqzAm1oqhK3r2/xtiIJAsRgaJoa6LRiKrryWZHaKHJJjPSuqdtar9uUBKlNQiB6Q2d6/0AOAgp8xopHX0nsVgSLbGdoy5rEq2pu5q2ajHG0QtQYQT4bV3kJaYzpNmE+dERebnj5t5jo4Q0OGPI8xJrer/eUwKtFJ1pWBd7bOefW8q6I4sCirImWO/RQUgkIh5uH0ijhDiI6VpHuWvpDbRlTVnlZCNQqkE0giAJEFJRtQ1pkgGOoqgJ0oQkCL3AOfQjPNzdEiBw0mBcTxJLrPHX7iSNcErS86cPwD/78YRs1rPevGOySHnzZk9voCh3fPRZRrWXnJyPuL5ac33b8H/+7/+Km+t7NtsKl+RMZxmBdswnKb0RKKXRWcib9pmYTwABAABJREFU1+/55JMXLGZTVpsV+32FVIrOtKw2JaGOODoaUxZ4rIvxZc3XV+/prOHjj2YY47i5vkYKw3QcDwNdy2gU8v13PfPpHIGjNw1SGsZjjdLGX1+UTxZWVU/X9pycnHF5eYuk5/hoSdu2dF2JlIIoSoiimixTzKcZpmtZzGdsVg2KiOlkyma1ou1bRuMRD6sdz54/pxrKBrUSCOWfdcNwSMn3guP5gs26QEjFPt+zWE44WsxItOSh2LBdbRmPUuouBxTXt3t0kPHFlx/R3wuWx3M2t3tmmU+ZxgLKovTO2a7xrk6naGQ7FIEqTBQwGU/pe5+AjOOYOEn5/vsbxoslo9mY07OE9fo9ZhjiGQezxZK6rgnCiDiOEM4gpfLP21FM2zUDMjDA40sDnPD9aVkWsl6vSbKM7WZFknpBvjO97y9LUqxzvHnzHkfA1d2Wq3VLugz48WefUbcV1zc3rLf3TKYzwiggcIHH41rD6XlM3e6IJylZGnF58x0XZ6d89eMlbe2YTjSCgsl0yr7MeXgoOb+Y07sGoTSNadFRxNXNHToISeKE7bZgNl8wn0+o6xzjCubzJY6Wrm3Jix1pmtB2HVE4pmktF6dLri7f44zj/PiYd28L+k5ycnLCfG55uN8yHmcDI7pnsRjx9vWKFy+eMJvE2D5kMV+S5zVSV3z7zTXOOR4edhwdxzx/ds7r7zZsVjVPLs64vX4gS/+8UnClPgixBxMDB33sIHof/hmN4NAl+MG77YVZv/7/UOhpvXBoD2WRPArpHzAVw5/ug5zuHt/zYLKSw7Pk8K4DzhnAPbp0xeP7OsQwAFBeRxs+uRAOpQ409cHZ7t9kEM49Ftr3HXquhTG9Twa7FmdyymjPOPFDfGMV4tC/aPvheuGTyloHgMVYXxiaReGgIYIQjq5vvanAHfSWQXNTPgHf9/ZxnxwE4cM/H5zphyLKw3b64fY6lG0CPsn3A9f5Ydt6Ad9484gOBmOwHHRDAXJY8zn7QSwWB1TP4XgQA0rF0LYtx8fHj+meb799xdnZ2WA2KVmvPQYxDKPH9/kD0X8QwQ+u+r7rHxPoh+/jjTEf/v7hewghHp9hu7ajUZI4ThBDkXkUhUjpt3vXW4wJCXSMUtHj9Wk00oShQCuNs2J4jvCqqEKjpCaQGq1CpNZeqhuoIBy0VtVih8/scMP2E8hAepOTdbh+EFucN30LZ3Gm9wiY4fs4Y8AOVPU/Ytn9R4votblDyQzhYoyzdM7HnKRUjyeDl996pKiHnZwhhHcdaeKB2e3FTOs6jO0HB7odYgPeyW5Mh6XGx0N6rAtQIsYNsRPNwZ19EMQPIuKHiY49iM0Mv8t5R2tvLbiBWzT8HTNcOOwgOoNDmBbhHMZ2HE59O9QjH9hEviBogNEP7KLe9N4tji+M8MgQ74wX4kDkht75kshDLzNOIFH4ULYZtukQAzlsGz1MjYZiVa1CjJVUTYU1AiVjdODB/04O/GIjUEog6Py7SQc03gHtzxKEVAgbIdoUhPwwHRXenS2leKwyqJuS8fiMKBrTNDmgUU7S5DuW8yfMxAtu+QUg0DLA0WFEjRZgRIXTgoodtBapA88V1hJpWhIt6dMIJR1tLemdIpwdkSw2OFXg5HAxUpJ0FiJziMYSpzoM/cC6bzCUw1ErcW4oksXC0Fx+uAD5jSwfC9oczTBsccOxw5CK0ODHIF7MFnaYlnpx1d9W3HCcf4hF+dGKT0o44ZC2Q2Pwzl89zMXUYzRKiCH6wyH14B1e1nZDKYdAuIg0kTgXDkMewEoQf95CfLV9IIpi6q7hfn1H2dRoHdF0DQ5I4rGf2rY1WmvmswVVYThenqJsy7bYYFxH0ewxwlG2FaazaCRxKHEU1HWFEpkXzZqS1jhUPCZOEs/47zx2p2s6wiBEJym77Y5QhSRxTOVKglCiEhjphDiLsUOxRpQkdHUHwtGZhqrzseC+soRBSpplaK3ZbTfU9Y44VAirScIxk8ncO+E6A7Kn21kiFWADf7NM0gRhJcWuQuqQJEsJk4C6rFDas7x06KekPQ1tWaI6TaACmqLCdQ0ujmk7S98aYp0R65hARsixpqgrgjgC0QIdSRwQ2JjVynmEQ2+4u77DdD1ZOqbsFYGImM2nhDLEDnGlKIwIwxDXC6rOolVEGGTYNmA5P+err35Mlo48FuEQyXocPsKB3ebFNwb34ge3xOHmWVUVTdPQNDXG+phbEqdEUUwQRIRh9FjW4fFZwkeapef+SSWJBkHm7NkJd9v36NiwD0PA0FYlm/2esizIxt4VPJ1NPGfu8pKmtBjjeWld2bDdrwmSgDSdkGUppm0oq5ooCtlu92zLLa2osfRkevoBxSS8qBRoj/PSYUAyiijziqoNaduOJBsjNOyuS1zrmC9njMIRfddRFxXZfIJzhqauCVVCpCMCF5ClIyyWpqtZ7zfsiwKttR+yWkEcxRRFRdOvPOvVSi9Ehh1SWWh9mV7XNqRxhvkjCk7+a6+iKJBS8vTpE+q6pqwK+r5ln+8oipzbu2uWyymr1QNf/cUXNG1Dko5YLheA5Re/+AVlWQ7OihE3Nys+/dQPfMMg4vPPv+B//B//n0wnc5yzbLZr5vMZYaTZ7/dMJp59KqVkvdpwf39NHMeUZcticcxmnfvvrkJG2YTL9w80dcHDasVf/uVfcn19xWg0Jo7H9GaN1jFaa6bTOUVZYowlSce8fv2OJ0en/P5ffs1HZ08wRcnDzS2/+eUv+fjFS/7f/9P/h+z8wt9zpGBxckzVt1DkiECyOFn4pMBixt39Hbt8g6RDXzzh409e0uYt//Qvv+TmYc38aMzl3T33D3ukE/ybf/u3/PRf/zU//cm/5uzs1IuCyjtr6ypnn2/omsoP7vuKttoRBr5joaka2rrgo2dPkOqMQAo+fvGC//Dv/hPj2YJn5xfkuxwrwSjvZjHDgyJCIZUvzka6QXSImM9ntG03LBAElp62LRlNIo5P5uz3G87OTnn//ookSojDiMVsQaAU15eXPH16zi9+/jPOz0/Y7/esVyvqqiPLxlxdXfPkyRMmkwlFURBFIUkSsdluaZuG8XRElmXkeUFZlVxcnGOtIc/3jEYJMGOz2eCc46OPXqJCTV0XvPruG6bTKd+92vDJp5/6h+62BTTWGl5+9IK29QOI/XaNAMp8R74vOFrMubq65uf/6T9xcXHB9eV7RqMRXdcRhhPOz88IAknXVYShLxmPooTdfkPTVszmS4q8QgjIsnSIACuOjpZstxum0ynW9mw2a7o+YTpd0HcWnGKUTehaQ9d5R1OSpHz08kv+8We/wlhHnPmeEJxhnCXcYvn1r3/Fi2cvkUoyGf15OBcnBdd31zjrB7ahlv46pDUuiMlFSVHXIB3jceTjqkIwny2Jk5jN+oGyzBHSL8rquvR4gKMRfee7ULp+EILCiLox7PIKJxWT6ZSLs1OOFjO0kuy2W9arB7arnb9uzzNf0usqoiAgiBTZJCYbJ+RF7ns5dEIQhBT5HrD0xmEPXUvSdxJlownJeEEUa4LAYyCbuvELNCKScEK5c9T7kmK/pykq2rRmPJvhlE8laSWItENP44GbnpNvDUfzDB0KhOn9AloFCBtijb9Ol6KlN4bNakWhCubTBTKIePnRJxydLui6hropaJuOrvOpn93tnmdHz6nalsAp3ry/psoCdtevOcSoy6Jmn+dMZiM61xBpyeXta4RV/MWPnrBcnNCKlH/3s1/z9s0dP/3qR8i+4u7hlt5Kyn0JSUR1d0f3sGU8nqAd5FVNEgfMZgnpLCMKfdpAOIfGsd1s6G53rPcNRVHSuxEEMWXdo8qGQIfEhKx2OVGUMJ/PyfOKtrW+M0ULhLQDPs8RaIUKwLY9xvU4FG3f0vWGtm9oy5pKK8JQI6RCWoc1UJU1Ogrp24GpKiRxEtMbvzKKg5goib3zMdLMhQbTkVd7XxsV+OPcuI6u9+JKOk1I45QsyairhqpsoHKYzlLRoQOJUiFNXyFlQF7WeP+ZoG8t9Ja6qlken2DoCVVE2XTsd1vOnyW0ZcXm4YFACY6XR7R1Q5XvCYIIIR1KB2gNQRAhpaKyFe7PMLgk0x7CPb3YoqMY6zRV0RCGLS4TpNMRd6s9Lz45ZrNZM5qG3K8Mf/83/4qf//Y/YJ1Dq4A4TNnuSrabjhcfnbPfGl6/uuLJswtm0zmrVUvX98yXIxw7P7hwHccnM3CGIBBYVzOdJpxfnCOE5vZmS1tUhMrRFKW/p1c1jh6lIh7uK45OYu7utswXAUIK+r6m60qSJMXakI8/XnJ/vyKOxzgb4JxiMT9hs8nZ7UriOMDaiidPF9T7B/b5Fikt8/mSIGj58otP2O8qRpMJVzc3nJ4es15vuXx/zXQ+pipLxknMw82e2WxMpFK6ynCyOKUrW6ZpwmpX4kyP0YKyyEmPx7gWzj9+xu3dPQif+rSBI2+2fP/+LVJLbm5ylBMIrYiThPL2nlhqkjBkPB5hrMX03nSSJClgmc2SQTA1LOZTOuPNZrOTY2wbeLufM8RJTJpk7Hd7uq6lLBu6vsdZR5KMWa1u6LuG0XiEtT1lWbNYzJFaoWWAkD4JGsUJQeh737q6RAjLbr9HuZAkTXn95h2z+ZIoTpBKc3V9C1qRNzvKXUh5/Yb5YsyTZ3N+9cstQsJqs+XkdEJZ7b2BMpIEkTc3xlHM8ugEiyUIJPNphu0ck3FM50rG04B373Omk47lcswkyVhvHuiNQtmYSCZcvlnx0ScXNGVHvtvTtAVRqFmvNkzGM1wKD6t7RqMJUknev2lwpifQmjhMyJIpTWX4qx9/wfevvubpR1PiOGY6nXJ7cz+UJ2ri2Av2gRYU+RYpHN98/R6tBPNFxMcvX9KbCuM2WNuw2dySJjGjZ0fstwXRyYiq+tNTJgBSDkbSYX3lRe9DSecPxGshB93gh2go9+hE92nhH6I2D07nD4L8B3v7HyJdHgV098Gk/ijU/mfIDHlwMA9W8oPT/fA7Pwjx3o0u1QckyOH16JR//P1eZPf4Nm9K8Jhdn3SWsseZmrLc0/c1cRQShhFti0+et92QILQEQfD4DGONLwTt6hJfpu7PiQMKxtMIDgMGaNsWgcYY98gGP7C0DziT/X4/oEv89cz1Bzf5BzHe/075aKZ0eCe0CoJh37rHgYmxFuW8GfKDQ9zvby0FVthhHzMcB34rH44D///0JEns3fZSMJmO6b/r+P3vf0+WjdBa0/c9y+WC3S5/RAVJKQnD8PFYOQjlWmuMNZj+A1boMEg4rP8PeJvH40J5k5VSikD77XYo/7S2R0hvyIzjAOcMYRgSRV4rSOKYMNQEAUjpvEYgJEIYbMdgRVYoEaBl4PfbsJ2cHZIW0ndOWmsH9XUwVh/S9lJ4aohSgznW43/dUOzrTPdoVHPWPLb0CvfhuP2vvf54nItwCOHjDcJCILwACG4QFP0XU2ic8621RlQ4V/pYhnBIogGDAU5ojK3BdmilkIQY16OA3rUYWzyy/oT1ETucQrrObyxncFj/megReF67b6yVg2PFguuGnaKxAxPbYbybWCiUjPGQJjngVw5TGh/VFyic8+KcVCFSBGgZY2yLsQ2dccjBQS4G4RiiQTT3vHjjOqxrvTArY4SLka7Bl4ja4ao1OJGt57U/Fp2isfiGW0vjBdxBDBaAViE9LdY5ojBB6h7oQXYorZHECNf6A4nec92FxrgW53ov3DoDTiBMOohr/qIukEM5q0BKTVmXRGFGGMaU9Z622vjFmbPMA4j2LaeTL1n331Jzj3QGJSI8013QOUMjGkpXIaUCOSAyAoXsDIl1mOhwk3AYJwgnGdPjBSpNSadnyOGYS8ch0jXIcUkvS6wcYZD0tsIfGRrnNEIMDCh3mOAOY4uhHMPHOCSOFg+yCYYhyIeblRCHpMDhJREDIsc5/BDCHUR5M5wTcnh4NgMORvzgTtUPIy6PLDmI+YebzwcR3+GEx3pIZ4f90T9+BueEn4lYiXAfYkN/yiuvdvSuQyhJ3RW0+475/IjOSAyWJBn5gVIQMB5NGI8WrG4LppMFsi8o6h37pqVoc6wU1G1NICPicEkWJRi3YrO7J1AwGS0IdErd1kQjTRwnREGIaXvqoqTrCkTmeWrGOFSoGcVjEiKEs97Vt7qkqktOLs6ROqRve89qFz5V0Jqa3nQE2hGEiReDtSRMQoJ4hkbRFj690dY9QWxQsiMvV6g+JVIRLgKjLMb6BZ8kZDqaksUprW6Iw4gwCCirkkjFfrghBpeU0CQ6Rs8WNFXuH2y0xCAJw4hABWgR4IQk1RKhBbiGttnTGj+lBn8jyjc524cHxumI8XyMNZJqU5OGGVEc8+TkCaWrUECsIkwLUhjK0hFoSdcpfvzjn/LyxaeEoUbpoVTk8eHKv354cz38aYebd9d1dF336EL3TvQS6zrU4A6NYy+GSKkezxmE9QXLTqACiUMRuci7KpzigqfIyFG0W3CCOApp6pLL+1eEImZ5dEwYhkwnY+qqxLUdVVEzmSQs5lOqZkdZ58RhQhrPCHSIkJZYgg4CYpvQbu8odwUqEIyWEwLnGZu96wgiSW99zLOzHZ2tkYG/CUdxTBAl1F2LIyILYR7PCKxGOEcgNKbv6VyDEoIkTkjjETYWtF3v3eXZCLPfkkZTJmlKqDV92yG0ozcePyVUQBikRGGAkB3b3Q1t1aGE5uz4Oa3s2ZW7P/ncttYwnU3pTcvt3TXPnj3DO1gM4/Ex9w83tF1NmgXs8z1ffPE5v/zVPw8Pi+LxwSlNR1RVyXQ6omka7u9X/N3f/Rt+/vOfkyYJFxdnNE1J1zUEgecITiYTJpMJv/rVr3hy8YQkDLBOcHt3izWCxewMISS3N3e8fPkJpndcXl3z8vkpQfAc5+D8/CnX17d88cVL1qsdOMWbN+85O+0xwOnZE+Io49vf/A7dW/7qL3/Ebrum3m6p8pyvv/2a58+e8X/4t/+GX75+w2a3J058Afi7d+94+ckLsiRBWcV6teLt27d88cXnrLa3FMry7t1b/vqrn9AlhuXJMf/xl7+mEwrjHEEccn5yTmccRV0j1OE5oqcsNpi2o65KmjrHdA1NUyFczzSLWN3f0BQVk+mc/WZNVeRcPDmmKHImWcb/6e//nvvVll9/+zWhkuRV5R08ElSgQEpM32OtX4z1rkcpaJoKqUBpqKqK6WTEZJwileDs/Jjdfjtwuh3Pnj1ls95RliVt23F6ekaSROx2W5QeIpm2YzQekcSOsiyAZHjgbrm4OOfq+j37ve/DeP7iCa/fvGZ5dETfGxaLOV9//TWff/4lWkum0wknJyf87Gc/4+ho6cVp0bHZ3LHfbwhDxWQy5f37t1xfX7NcHvHFl19S1zXfffeKMNR0XUOee5F2tbrHWsfDw5ow0lxdXqO1fCx5OjiJbm+vSZKIqiq8C69rCYKQN2/eEEUpk/GC+XzO3f0tt3c3PDzcM5/PH6/Ds9nkcTGlA8l6veLjj1/y9s175vMFbVchhOToaIG1jqqqsM4/tNdVyXI5I9R+kRUGIcUux1nL/HjO61dv/uRzGwAlWG/XaAmjNMZ2PU3VMlnOSKdjdLxH7LYUVUGgJUJr+q4l1JqqrLi7u6frWsIoeFywRVHMaDTGOdjuN6hAUzc9vRHs9hV9L9CBxwIsj5ZMxhnlfk/fdZjO8PbVAx99kvHixVOqrqCsd4SpRocRKdI/49MNCy5D29TUVUnXtURZRpLGVPs9URLiACkVUoeESUgc9x7RpQx1vcP2mvHxlCr3ohbGMptMSaL4cZEe6BAlINKxTwG2jpGeo4Sk35Uk44RUJINbSVPXFmMcJQ0SwXQ8YbPNwRr2RcVHH31KNkopqj1NVZKOJqSpIN/7wnRrHMIJVvcrrt5ccTIb02wfCE1Hs90gDNRVjxQB671HB0zCkMU4JY1jlFA4UyNxvLiYsbtfsRhnqC6iyStu9g2dtaybO1RnafoCV/XkZY4KffeRDjr6do+xmqL2mMBJmpBOQl6MLkgfSt69uyeIIo8pcwpDSNfAw3fvyeuS3X6LUAoh8J0n0rvEEH7Yq0OJlRKlHQpH27W4AW/Z1oayroiikN44jJPEKqDvapqmJUKwGE1w+GJOBufYbrOjzLfe0BJY4jShMZY0yXCmp6wrmr71gngQoiNDXXtH33Q+Q6I4Wp4gnOS7b79H9Q0OjYwSOuMdalXXoLUjGseMRgltu6dqK7IoIAljxmFKJAyRTqlpKIuGLBmx32xxfc92vWH+0UfM5gv2+w1WWMJQ0zctKkhYLM+oqoa+6XD2TxfRn71cUNU7UBVdBxdPR2jlcK6l6VqyTOCk4/ff3vLJZ1OC1OBC2NcPgCPPKyZpgnCCIq+QwvH960uE0Wx2OZvd1zx/cczJyZTXb+5o2544DpD0jMYp+X4DwvjSQxdwenpMW0lurq4BxSSd8ub1nZfQJj1B6DtxkjhCOoE1PXEU0nU1STxmv93yNDsF59MvXWcIg4R/+eff0LU9i/kRuJBvv7kiSTVN0zAepey2OdJ1WFuz2VpOTo55eNjxcP+aH//oM3p6JosJlze3mN7hioooirC2J1IB42xKmTeMkoz9PieNEh42FX/z1z/l7mHD/fqBu/UNFxdHrLb3NKaFTrHJW/7ii5/yH//hZyyPMnpbst3mzCbH3FytULQ8O3ZstlvOlkdM44Q40HQDzjMMUzabDeOxdyxPJlPyPGe1WhOFEdN0yt1my/HpOQ93G+q2Yb44p28lu9WOvjdEcUJRVIzHY/re85LjJMNGIUEY0jSOJB1x/7BmuVwQxJogiHFSUVctRVkxGY9JswyBoe06xJASTbOUh/WKk9Nz5tMxRyeCd6sNZ09m3Fc5+ypn+3bLcpby4sWYphesL1uev0yIEk1VFWz3O8bjEcW+RoiExWxKnq8wvaPKd5wen5DpgHV5Cw4++WhEFGikcQRoRlFK2wZcXr/nxUdz+lqyvss99geFaS1F09M0LVmyIAwSAtnwcFdyfLxglCnSJMCZmi+/eMp23fLt11ekSYQVPd+9es3JyRHz6REff/SC95fXCCEYTyakcc96fUMYal5evKSr73nzest0krArdmQjhZKSKI6oq575YkxdOZLTCUK0lNX+z7p1+/vtYR12cBqrxwLIw5/W+kGg/5l4TAV7t7lfJx/czCC82C0O7nP/Hl6g94LpIYXs3ME5/l/6bO7wAf2nG4T1A5Lkg7vaJ5AfCzgHYV8gkU49/p3DyzIk9QcEh79HW5zt0MKbCp1zhErjpBeGA6WIAom1NYJsSGQ2mN4PCQ7oPecMxvTDGlSS5wWx9viPruvpOjP0JnrTkhx+vxCepx6FAWmaPArEP/yexnjn+IEj3jYdPxw0+JR2h5SaJEkpBuOUMQYn8Ea/7oPj2xhvSsvSEXVdI4Y0ghDCl4sKr7sdWPfqwPN+XLMPArKQaKWYzWZkWcpuF/JXP/EFo6b396UoCinLiqOjI7bb7eOz7EE0T5Lk8bv2fY8ZEIBBEAxr/foDYWE4Fg5u9DAMhm3uKRhJ4tcHdVPTtq3fnwCiZzKbcnQ0Ii+2KHVCHHtnfBAGBNrrzDjnE2f47a6kd7QfRHpftuq3guVwzLmhQ0z84NhmOF8C8I8kg1nV739rfNrhkArw2JoP389/1/8/iuiOjs4MbmYEQgQIJ704rEKUjJBi2AhWDjKkj1N45nhOT0WPn/wLoUA5b9UXht42tFYQSkD4dlpLhxMGSUxvG+/tdR1C6iGKYUC0CNsPXmB/skgREgg9uH01CoWgQVANn18NAuqBC2Q4cNSF0Ejh/5uVPkyAPfiwFUqEaKEHNz1enJVyYO54LIciQMlwaFsW4HrvbKFHqTHahUiEL0OVynMehcFZH933Tc0O4WIECusO392B9EWHAn8BCnRIIyTGWYTy3CeLQaiWIHIDbwuEUFjbIESAkhnYAGSNAIxtvDdb94PrWj0KYTjvehPCsN3dcn76I8p6TV7esn54Rd85wihm11S0Y8Vi/JSZ/IR7W9LbPU5YzwY0ewr1jrz5jqZvCOQcAosWAtlHuFigOoishQgfjxSCzoWM3Clhp3GzF1QuIhaS5ew5V+KX2PASpz6mti2dSXGiwboe5WJffCu6xxuK34cBDLMq6BHEw01kYPgPU0B/7gwDFBiGMHYYoHoMjOfd28eIjef1m8GVfihwGIozxOHYlH6oMJR/2IGj75EuvuzUl3IMx5zziQAOTDPUcEIKL7A7O3jeH/7YU/m/+GrooBNMR1NCHVAWA3ZJOGwHaTqi70vP69JeJK+7kravEKZDao2owXYdTjrSOCKJ5hwvnhAHEVUjfQO28yKykBGNdQg18PuDgEBKyt0OhG+Dr4WhcYYu33o4lFPYpsV1LaFUPFQbxk1BhCUvKkIXMIlHSCy13ZM3hjDWpFlKqFM8/1yhtMR0DiEsUjryasMoiAjjGFkEhKFPZBBBK2qKqsQCaTYhm0/BtJSlP7Z1ELDb71BCE8aeBW66ntoYolSTJDFYXwobBglxFBEKjbASqQShDqjKmrotwJaoIKA3jqLYIZQgCAOmsyVZMoLeO6izZIKUEcJFvjxFWmpXooUkDiOSLCVOEsxtQdO1PD9/yd/+7d8N0TKBksONWPjjCBja4L0V4eA28A8+/sGhaXxczhhDVZU0TfvIMfbu8/AH4vvwQCjsD57PhocxKVCBIjCCNIuxaoZVhqIeUdcFzlmU3JMmK6KoQRAQBrE//4xDSTVEzwRhECMkPLvICKKAfZ6zrbYEaUfTt2zyPXleILRP7ygtKJsto2iEwl+zu7bl5PwEqSV3q5uhYNrR94bRZI5WAUj/oKBCX5baDtuh6TpM0YHo0DqgL3c0TYvWMQ7FcrFktVsT6pCLoxPioYz28uEdm+0KKwWT2RItY0ZhRpIEdH1B1zYEUYiIAuIspdzeI35QCPO/96W04PT0CGs7uq5mu1vh8DH8o+ML1usVcaw4OTlnuy24vb0lDEPm8xld1/DwcE+apuR5ztHyGKUCVqsVJycn/OY3v6EocoIgQmn45ptvOVoes9k+MJ/Pefr0gsViwWKx5Ohoydt33/PwcI11htOTJ1jX83B/z09++q+4v79mtb7l7PyMqq5o244wSqjrlqbtuLq+pahq7u7vcc6x2W0I45iyzFk9rNhsHnhytGSzfuBoMuXqbsX1+yt2ZUFlOr5+8z136y11A/Nlxmw+o2pL3r57w2Ixw1rHYj5nt9vx+9//jrLcs/j0EzrT8/3bN8heUtW1j/nPlzT3O6SwlG3D3WbFqzff8/zlc05Ol7RN7ofwzrHb3QOW29v3dH3LZ5+85PrdG7753Xf8/d//9+T7LUkS89WXX7Ld3zMej7HGcXF2wulZw/vbG+IwYLV+QAYBKIWzhs5Y4iQZEg970iwligOsa1EKdrsCYyxREnLx5IzTswV58cDt/XuMaT1uJJsyn824v19xcX7B3e0Ny+WCJ0/PmIxH3N5ecXZ2wnazx1rJKBtxc3ODlLDZrD2uK9RY1zGbzui6mru7GybTCfNhW5ZlwWSS0bQVSZwSxxHPnj1jNptxdXXFaBaitGA8SZjOPNqjKErG44z5fEpdV9ze3vL06VOKPEdgWS4X/O53vyOOE9IkBWfp2o7z81OauvbDCuGTgZut58pKNUZpTbXZkGVj4jglDBM2mx2ffpKS5+VwfStYr1dcXl7y/Pkznjy5YL32AkWSxMRxyMXFOWdnZ9gePv/8M/7Tz/6Bjz/+iLdvv+fk5Iz379+T7/ekaeqxer0lDCJOjk/Zb3acn5wThjFN3XB2dvYnn9vDBZcwCkmigKaqWN+umY/nLCenFLajs56Nq4KW3tYkkaboGup8Ry18glMpTdv0CAHT6ZTF4pjdfkcUx7RdT9v1SBnSdY6mcYjh+da4nrzYUxV7yv2e68tLmqpiu2q4ijb8xV9/hOtqeiUwdFRVQNN0VCb3gmxniUON6bvHewRDjrYz3mBTliVlYxAqYleEhKFEqQ7TN5iupm060lgRBxkq8IK31QojJNZ4vnoURYTa517roqWuGibJ2EeEpSDTCVW8xDpFFE95f3WHjBV1U9O0rS/VbHqy+Yzjk3PmswX3D3e0tWWczUhHmnJfc/nmGnrFfLZAS839zR3n5085mo5ot1fIyuDCkL72i+eyhVdv3vPpZy/JJicE2rHdrlhMJ2RZQtfUXBynvBuFTJOYusmZJBM2dU7bNhzNZjRtR9m01H1H3bScn5/jlCGkJYw02yqnqGuM8wOki5MTrt5fkqQhXd8TC0E6GqODBEfI3TrnflsSJRGjyZS2rR8Lwqyznl0vmiFVa4dOJN8NZfoWESqyLKWqG4QTtK1BOekfNbqOrm5xUtJ3lqbpyNIR+85hel9GXe53uL5hMklpixpjoDWWXga0jV8/JnGK0pq8qLC9F9qt9T1YRVGxkhs+evYxT8+ecivuGKea7HjBer1GCNBRSFmWaK2ZLMb0JqLJ99B6oXWWjtluC9qiIUsnLBZe4Hl3fQ3KISQ8PDwwmc0Y7yY40WElxKMRzmry0lLkhiQJifV/eyH+X3sloWO7rQnjiLbv+df/6kfcXr0l3+cUhSPNBM+eTenrFV1Z0lQF02nAu6vvCLSjKium4wydKO63OScnY+J4xqvfXiKcIy8MOtI40XByGlPlG2Il6WtHk9dEgS/5bKqAs6Mjvv3dW+q2Y3k0oshLkIqPPnlKkknuH24YTWI2m4Yszcg3W46PJWdPlmz2O/Z5h3GKujNI0TOaprR1y/3qAS0VaQqdKfmXf/4tEFDkHWkWEOoRb67uOD6eouI9KjKgHfu8o++2tLYmr/ecP3nCaDJi9XBH27bc3N5ytFzy9OkTHja3flhoDdlixKbc8cVPvuR3r9/y5NmSiYxQ8YxRHHL1LscKiYgkdVHz65/9ikRqTC05np3ysLpnv7tjufBp4IftHV1ec/zFgvvditkoZTKek+mEm8s7lssjEHbgNvti0MX8iLbrMQiOlkf8f//nf8d8PuH8/MQbMFxElI6J0gmjUUYSp1SVN4i11hHFE5bLBXm+I4oN48FkorTCGkEYjamqiigJMbajKirGI19Ufn5yznq/4/3tLbkx3BQ55fqe26bh2++u2e5zjIDRKKM2Na0VWKkoTcd0pPji8xldW/Ls2Rn7PCSNI/rOIGLF0WzCzeUlX/7oY/JiR1GV/PLbb5hMM4QrCbXGWEeIZDTJKKsHjO04OnnCaPIJRVFxfpFwv1qzXmV8/90Dp2cTisrQ9XvW61suzl9Cn5AlEeXGMhlppPZrGKGgdwWTZYcTHeMkIo4ykiBGG8P+/p7FOMMEAaW1bFZ3tGXDp6cf8e79jjiN+clfz3A0RJ0hUB11o2gKSJMZ6/uc4+Mlv/rlN/z4q6ckwZ936/auWOEHPw5vZvwDt/mQDMYN5j43/OzgUtfDmk1jnRnEbjm4qwdn7SBue83Bozy9eP5hHegGTMzBKvgo01qvrIkBd4GHFzz2ZFnrB2yeHX0QlNUjoeKH4vnhs3z4lyGxLwa8yyMlQD7qLFpLj7wRkt40Pk2ZTj2WajBJWGcQ4oBa0cSxHLjjhjAM6KoKP1jwrY3g16hKyQEpyGDg8qXoB3TLwXnthw6Stm0f39MjCT21wfeEycd7ZBT57ryDA1xIj0b1eBn5KM4D3oTVdwRhSN91PtGkFM543Q0MUh444B+gzvxwLw3DgziOmM1mXF9fU1UV52dnNG1LWXjn+KEg9fAZfshGhw/CcX/4TDp8RLiEYTikz5tHAR54RL30/YdBStv68tYoigCLUg6lLEprsiziy68+oigfkNoz0sPowED3x7o/3jwFw4vvDovBuO7D8eoPLq/fimE4dDBXHxgsgw6hh3SAsR5x5Kw39z36aQ+pDzxWaBgZeWOs++G5+F9+/dEiunUetaCGaYkxHr0iiLxbFIdUit62YJtBOFee/S2iYRrhkDJ+FBCF8BZ9KSRKOqQL/SRBOHARuM77sUWIEgEO6Gw/lIX6k9WhB+HFs9SNdYBGEHp3uXX0tkdwsOv7qICU2k/KnY8iCCpADViQFoTDDN9BEAwOsAEZYnw8xDFGiATnhu+Lb4g9HACezQNOSM//ccHg6O9RyIEL67+fda13rAuBdBJhDVCBUx9EWOmFeiEUgYJWdISBRydI5UtVlRxOtLBHaF+w6aj9tNJJetuDy8F5F75zHQJfEHoQggViuJB414ixlrrOkcJHxevWYYUgm57Stx2uy3n9/h8pxxNIJhyd/Q2r/nf0zuFkBy5CSUFlvgYZIMQIQ4vQHb3okP0Ei3e/h86X88TKgPQFEtgY5SbsneJ+s+ZcpCShIkklbVDSix178x04iZSOxtzi0Sx2uAnYYTjhbzagfbmpAEPxISqDQrghjjNwuSwMZ6xH/7jDQu+A6EEN+8iL8gcB/SB2+1lhjyD0xbXCDZFXgxtY/X7dqHyiQwTD7x1uksKL8I8XOob4lRgiLtIPOtyfUTwIEI1GiM5jKdJRzPliSSAVURjTAm1dorQlSjWVKbnfrFFS0Ks9UgagQtJwRCAtVVPQC80onZOMQgIJVaWZjc7pbc++yLFa0tJxFI/AdhT7DdL1IBqSLKPsamoFjQZMx131QGhAdw5pHE4Jgihik69I+4RRNGekJugepO0RJiAME3QoSNKMWC38+MzVVN2aquppyhaV9hBDbmuyfsIofkocJnR9h5E9+/2WpinobUjjBOt+h+nv6dVQDGs78nLPdD5BRpLQhIyj1Lc9i56qayn6hjQZI8UI24ILBEJbjCyxwtKpnJvtNTLoGcUnIBN6rYljmDJBmDFSaHbbW/L9hlF8xHL5nKbv2Wwf6IoHNs0K5TTt5IL56JxYjwnjjMxl/Hd//99xenr0A0fAB+6dG+J7xho/iJPiB0NBHwdsu5q2awa3qo/1h2HAaDwjTbJHfMthIGnM8OA2vMmH+Jq/3gknkBp0LEhEzFIuydoRnelp2oYw3CKEpGkKH5EWUBUdbSsZT07oTIAxlvUq92WQ4ynOdkjnwPbUbUtZFjys1uAcSvt2CtO1NEoQxwFxPCHGx8PqqqRsd3S2wSFx1vnCGifB9gjbIFyJUxqZ+GSQMZZWdBRlg1TQ9DuEEMwmSzJpWUyPKMo977//jixL0cY7v50TqEiS53u0iBnHE0bhDDrP7w/VCKsV6/0Gg2VvC5o8x5n/il3kj3h1XUOS+AHK8mhGUWzJsgSpQtquIM0CirKgbguMM+x2O3BQ1zWLxYyyLGjb9pHvt9vtCEPPc76+vuLk5ITZbMrr1995cXX3QBRGFOWOPN/z1Vdf8u23rzCmYbW+o2lLFosFWivu7+949vwp1vrBYtPUfPrZZ/zzP/0nmrZlNJ6x3W5J0hGv37ylbRuyUUoQa6bzEWVV4egZT1LmizHzxYhZOsZ0Pdv9hqIt2VYlv3/9Pd++eUPRWgLliGPvJrG1fwDdbDZ8+ukn3N/dMx5lzOcz7m5vUSqgbXtWqxXFtuT+4R5joa47JvMpV5cPRPmWMAu4W93y29//miwJefHkKYFU7FYr+q6jqWtWmxtevnjO3d0Vv/rVL5mM5jx9+oSyrAkjzddf/4b1as3nX3yOkj71drSc8NGLF5RVQec6VpsNne3RUYhWmrOLIzabDVVZMp2FGFOiAo3UHpm3PFpyenKEVJJ3795iXY0Qgu12jzW+QCiJE8o8xxrDeDQi3+/4/e+2fPzxC+JYst6suLy85tOPvyCOPXLp1XffsNvt+IvsC2+w6Gs22wc++vglz55dcHx8jFKa9+/fMZmM6PuWu7tbjo9Pcc5xdnZK1xnW6xV548+NNIsII01ZVgSBL1K6u7sFBMvlgrIsKIucOArJ93vGo4yyrOi0pixyzs/P+eab12RZytFygXOGFy+e8OrVtwRBSJamQxw3ZLPZcXb6jOOjM969vaUsa6IoIkkS8txHXq+vr3ny5JyvvvqKq6tL7h/ufFGUiIiikNuba+7ub7l4csHJyTHG+Gi3GqKtT55csNsVpFlCFKZMp2Nm0yWnJzkgaJuO6XhKvi/+5HMboOlbIqXoG0MiE4J5yHK+xDrH5dUtt5sVlh6t/HN3FmeY1lHsdv6abAVt29N2Dcv5nHEywTaO1d2eyUzQNn5InSSKMq/oao8V6PuWvbE0eUlb14zilP16T1t3SCfI9yWb9Ro99i6hpmloa0e+rxHK+v1YViRBSKACRKzY1Dtc7xDao6LqusM5RW8rpAZXdESRZDQR4MxwHsM2v8VlI9Jgge0ACXVdcDQ5JYhitA4wfY3Ap90iImbhFIUjTkKssAQq4+Zuy8mJJgk8+3y7e+CuLglESF11RKcp08mMq+tLvnv1LZPxmOT4hLrquLu/Z715IBUjEh3T1h0ns2OOjk+QwqHSMWEsaZqaKDQ8Pfd9Dr97fYdAM5ssePv917x4dkEUatq2BiymqziejglQdETEoWQS++LcWZzRaoOWHX2Z4xBUVcFkGmNaw37fMJnPqTtfAL7d15ycasbLY+q6IZ0kKC2ZjGdk6ZS2h6btyXd7hIT5NEOHDtsbqnrv+4dwKKF8J03fgLYoqSiLwt9vB85u1/TDM68XirrWdztFKsJKqIoKYwzL5dLfo4WirSoslixJSaKUsqko9iW9s8SLBX1lECpAKk1d+UH+bDRlNplw9fo1tndEMqTalzzcPRCPRny6XLB1PZs6J8lSj7MLfBK2qWv2+w2z8Yyj8ydcvfmevveijLPQ1y0ygLPTI++4aw2tNdRdjg5j0smY5ekxu90K4QxY2G72ONf44YE2hFH4J5/bvem8ezIIKZuSh+0tx2dzRmPB7f09jp7pNOPps4Q4Sujalr535EXBYpYSBb7oU2jHZCZYLFP22xyteqIkZHoS09sWY333V7GvODs6Yd933N2uOX0yxUnL04szLt9fU9cNz14eM56NePf2hvlyjB0MBR9/8hHfv/6OpreMohHbbc2LlyPubu85PlvStHvawA/kwLNp27ah71vG0wmrdU5MRJqmTKcTepPTW49MFDLC2RAhW45ORrR9R1F1zKcndH3LPt/y5t0Dy+MJJ6cLdrstTWi5u3ugLDuEbhmNU3CSqqoIRgH/8utfo6XiyKTUfcl6c0/fNH7N0/eEQUgcBZh9RRAGrB42REmM0oq+bRCJF+Nu7tbEy5TVZss8DTw+BcXt9QNlWXNyekTdFICjaRqvi1hLnMS0XUeoJT/9y4+RA0t4t9kgpSBNYpIkGfogIjpjEI8O5YBAhUg0URxjektVNQRaEYQpQRDQ9b1P+QtHj2GxOOb9+0tM21A3BhnG7Pc5rYC8b3n75lveX2+IwgTrHMkkZJ5OWO33hEFI30PT1BwfLbi8vOH+7o7FYsnL5y94/+49ZW1ou4LPv3xJ39f0piWKNEfHYybTMbdXW985ITRlXdH1PVVV0rY1RfmaomzJsglBIHn+4owg0Dy5OGK7K5AqIdAt682WyXjDbDbjn352Sd83/B//h4/Z7B9wzvH+/T1Pnp0iA+GTDk2HaRzHT06JhECokjBMyE1PawzHJwvub+55/fZ7mkaQxDHIAqEsfV9hOo8sLqsaJVo+/fQFfW/48V9+isSfQ3/OS4pD59+HksmDWxwOsql3ah96AhGDOVJIlHJoF2KMTz8d8Cpae0wJfGBYG+O73ZyTH5Lzztd+MhjSH8Vzx+PvEs4hB1OVcGJ4H+82BwbnuSdSeN1IDyL9gFw+rP/44G4/rGbE8DPh3KNALITASe8ktqbH2R4hGky3JVArRtkcpUeDkB4Mn8MRBB7V0nUNaZoRRb5YPMvGg3veDFqPGBCl/jxUUgwdXurR4Q8M4viwnwYnehAEH3jnYfRIbTDDtj+4zOu6JQxD6romzbJHtrgeGPgHwV9KSV1VjCbTwX0uB0r1gZ8+rNkPQ4//fCjBhyFJFEVMp1Pquh7wg36/v393RRgG3N7ee91WfhCGf4gM+mER6AHv4ocRfrscMDYHof2wjcIoHPSOw+/xx02gQw5DdoRjNE44OpkRhD2fPLlASkcY+V5AKSOs7fxx4bySKqWDoYDV0MEggNvh3iyEwDjPOpdSgOmxrnk0BjIkBDxWZkhkCOf/8zAA8oN3QBiPzjkEM9yAJJLqv3kO/9EieqLHtNYL31iBJcYAWii08OOpHmhs70svhPZObHqs8wxpL0gGnu88IF6cUFi0j1UIR28bpDgUM2q/mJQhSsSDEKoJVITDlylYq0FFKOkLPTs8k00qh6PG2B5fVmo871yG3mmJwIgOKRNA4ShxzjvicZZe4N3ELiCUCeAPfL97NcKBRiGkprO9n5IIi5IgREtnB067O1SQjrwo1RfesY/0F0/haO09jpbetf6kIfRsKCHwZaOhn6pY3yArhMZh6Xt/wCvl4xxKej6SdR0qrnHKIEVMZ0scFsEIIaB3OdByiBFJoYf+gsEFP5QN6qFAyJ8ckpOjcwQO60p6m9OYnrZvSKIRi6NPuHnzH6n29/z47/6vHE3/LTf9/wslvMMa0YGokdJ/H++6zzFBi+klMozoa4uwPrZhW4/QcU7ibEddVbxff0/gJMVNy5d/9VccRU94Y/+Zzq6o3Df0rvY4IHY4qmFgofDYnP7xn8XjdFc/RoA8msVfGBQhbkgpeIXbDif4B9SMv7gFPI6yONwo/LRYoPy248BncjhhH+NRPG57X67kX97JfigblU4PcZ7DDVUOIr45vAMIP2FG/OlOVYBFkhHPE3a7HTUdJ4sxpnKMooRd5x2+ndwRZhmmM2ilGaUZVVNgrKWzYFA4IhANQgmkNnSuxFpFT+/P3SDk5v4eoTVxlqC1IN/saZsKrF/IaxN51EUwYhIfIbRBdAbheoJEQ2dobU0SCqxoKPKKbDbHSU3d1ISBpOsNtenASuq0Q+iGNIkpy4q6dYyiC8ZTR9cVWFlzefeGNEw4Hp3SFDlNY8nGE+oarFWMRiPiJGC1vkOQ+yl5EBCGIUEYUDc1292GrmiwpmMyHWH6nrbpaY1Fowl1SFHm7MqKOJIIZfywoG9xssWh6PqevvOFqoHVjCZjMCl925FNYooSUI5slCC6ClWBimNGesTmbkehKlJlENqgZcxffPkZX3751WNRygcnwEFF/8Dzh/+1c8BY3+y92+3YbrePN+s0TYjjhCQZDVy1Q8u4G65LYH/ghDg423/YBu4fOBSRC3FCILp2+LlCSzWgQWrqtsTYhlArdJahVEpeFoCg63uKqkRKnwICibKSQATEQUTdNlgLOohoa0PdtAShL4jta/+Z6qKibLYI7ct7Lk5eMB8dU+4q6rLB4Jhkc5rQsC3vCbSfnoeBIlARYRBiyGiNfwjojcHYnraqGI0SxqOEy+t3jMYzwiwhzGISkdC3FhVIpPZD1yDU7Iucm/tbrIBsPPKuj06TRH86rqluKm7vrkjThMlkzG7/QG/EIFxOfOeFq6nrgrvbLcfHF1jrO0uWyzlBELDb7Vgsjh4b3pMkZrNZM1/M6E2DDhRNW/LZZ59xfX3te02s5osvP+fy6j15saVuFLPZiO++uyJJz7i/vyGOplRVQVVVgHcBv379muXyyBc+OsjzkjDsefnyBXd3N4wnI+7vr2i7iLar2GzvaaqK+WJEb2rCeEpZF6x299yu7mno2HUN27qhN6BUxnRyRFWUvHzxgtv7K6wzNHVJFCq6tuP1q1fs8oIqr3i6PMPFI96+fUcQhjx/+ZR39yuOThasd2uiJKAo9xTlBmdb6mLLu4unzMcjkiikLku2mw1BGFBWe77+3e+ZTCfejWYMi6Mj1psHbm7vOD49pm5bwjih63zp3tFywXiU8uzJOSoUXD/coALF4nhKNtZc3ew4Pplxfr7g+vaa0XTBbmdIkoDlcgrCkCQxs1nK5fVrtAo4Ojri66/f8wxB31nSNKMsPb/8V7/+Jc71SGWYzkZYa/jpT/6Kuu5Ybx5AWNI0ZDw5xtHRNB1dXzEah/zmN//Cy5efEgQR9/d3TKcT1usHijJnuVwQRcGAfwrY70um0wmN3dKbljBUrFZ3dJ2hqhpm0xnpIHpZaxmNUmwfU+Q78rygqmqm0xnT6YT7+zvSLOHpsxMEkjiOeX/1ntlijFSCOAm9yLna4J/lBFGY8ZO/+oztpubr37/ib/7mp7Rtzf39PVmWkqYx1lqmswnv3r+hLEuqqmK329B1DUdHZ0ynE7799hsmk4z15p7pbMLN7TXT8QnLxYz53LO4tVYgLL/65W9ZzuccHS25v71lvV5/eOj/E19d09JULSMZ8+zFJ6RnGet8w+XqEh0oqrymriukEozHKcJ5kaerW0zdYYX1vTrAZDSnLy33Dzds9xVNDaNxRmlqurLGtWBqj2xR+MLF3AmapmP6ckFXS0QfojAECq7fX7G4SLB6SENi0VLRlY7e9EiFJxoKRdtYmkoQxQrXOuIwo2ty2s4iZYgzYPuGZBSh8PztrtVYE9E0FaYvaEcpKIFzrTfDYEjCAJBUfcvd7ha3Myykpuh31E1FMA4wIbQqpTA9myKnaWrKzY6ny2PcPqdpDONswmQ05f3bN/zud7+gbguUeEo9mpEXObvtDq0NoXI4Y5nM58xnRyitubq5BNeSRiF2dkwsHeF0xPw05kdfPeHq7WvepPDs7BSTF4hJhut7hNVoG/Ly7CWRSOjDiNZUjOKOQBuWizFVYXCupRSCbnuFFBbRFkRxgGktN7e3SB15oUIqbtcbRpMJ7y/vqKxBCegai7AhcRQRBAWTUYZtOsrNzhePOkPTFJghWelZt4MTzBra1tLW7RC1bgl1gsKn8pDGYzu1ACvR4YggDJBAXdU409F15eBc9MjQeJRilKJoOtqu86x0LMEkpWsMvZE4G6JVwHx5zGIy5t3vX1OuChajGUmScHt7Q3I05+T4gnKT8803v2M6mTIeelOE80nYNq9oTMA8yZgujijLgn3T0vQWHYTIoKfId0jRc3Z8QV5XvL+/oapqirokzRKSdkS52WJ7Q70v/fPmJKNzjrz503EueV3jtH/uT9KA79+8pTlb8MnLJzSuZbvfcrSYkk0iJHIomIQsGZGmCU1bESW+NC8MoG0alBZcPDlGoJgsUyqzp6odSgnOzud8/vEL7m5uub5zGNdyerbk8vqarimRyjKepJTlniQNWa3uaBvBdts8JozCWPNws+Vf/atnrNfXTBeS/a7geLnE9ncUuU8cJlFCVTVoHQy4kgmT8Yy6stw/bEhSg9KWvFhxehYSBxFx5tjvGor9lpcvnnJ2ekHTbpnP5rTdlrube5xJGY9ShHXcVBv2+46Ts8RjUOIRb64uCVA0VY0LQ371698TJ4Lzp+fkuy1RECA6hdSwPJ4gpwEPqx1Ix/XdHcfLKW3d0dSGumqwveT2bs1IxQSnS5aTKW3ZEYiA2WxGHKdYa+g63wtj8QOlOA4JFbi+ZT4ZkWbpcH/ZeXFOQJJ4tMQBDXHoIbm5eg+2ZzIe0/UtGkmV59RSMJ0rLC1KWeqmoOtatFbUdY3WIauVT+k+7Ct2Re3TE8oRpyGTqUZgCMKAOJaoQNNVDpv463M4gqZtmM1GA46npapKPvnkY757+57bhxs2v9/w5efPSeKAIi+ItaIrfcpLS01VGoTT3N1umM+mFI13jfvkN5RlySIZ0XcOazW3Nw3Pni3IMkmet+z3O+QoZnmiGI2W1G3JfLHg+vqazjT89rdvOD+fUJUNwkbEekwgMtqqYJrOqeqKpihJxmPaasViEbHb1igVIETD1c01x8dTZtMpq7sSXM9kMkGrwPdf1A11U+GEpVd/3r3bl04enMk/NMoc1mMH1zlDT4H/bwckC494Cu+fPTjOPangw2/0PPUD/1w+ivOP1rwfrM8Oa8BH49Pwe6VUCCcfCcT+Hc1jNaFQBxSwwlk5mAk/aCMM+Jk/cD6Lgwnwg0nwDx3XHnGsFAhn6buWrvP3dRDD9tNIJeh7byQMgtB3Xmkv6pu6wxrvEpfCs80979xjSoPAG1LMsO3gw/r0IKpLKYdCWl/ieVj3H777YV8e3Ng4/+/7/Z7xePzoPnfyh9vT41LqusX0/aNob60djLcDvxs5/P3DO/0A5zIMOrzzO+ToaEGapnz//ffDgMExmY6oq4bxeExZ1t40Z8zjvj7s74NofhgE9J15ZLu3bfsonEspSZKEKIq8gO88qcIYn7iX0njmvBJEYUw3POdNJmOOj+eEkSPNQCqB1B5p3RtvGBDmQ+LhQxmkH7gavJNcorBmcPLbgb3+qDV3HPoEDwMIrQJ/TAk5DAoGMd44ODDRh9/tDoZZfM+U5+b/b7/+aBFdipRAqIHRDFKmtK4m0BGhjHCu9k5z14MIcbbFmQqhwNgDRbr3gi2esWdciTEd4AvphOgxziGEBixqiHU66+hs6236zsuHDkFvWqwz3j/ueiwdEo8jMHRYV+CEL7qxrschUU7hXEPvvPDS2xzr1OMGts4O8ZHAo2MA4xoUI6xNMLQoERGIEbgOIVrPQTcWRA6iwjiBdAnSGazr/HsP4rdxDb3zsdnOVfhW5dLHUZxACYkVYoB7eDerEB1yYL0b6mGK4qd3MCxGtBr2i/TTmXBHZR7o6XBUfnhhBFL6Ej1ER2dq7+oX3uMS0iGGCadD4KxPF1jXMhkvmU2WCKGJ4oy4T5DSOy9uH94ROO+Sv3r7T/TNjn/zf/m/sePX7PpfoVWCVgIpg4FpFSG1F8w600C4oxeR32adBtmhkwYKEIEiSlIuL3O2+WteVQ1fX/+eZ89PuVj+a/ZmRc139PKW3u1xBCg98MIGYe/gtnXOF1L5BETsBwpmKCB4PBUGIdwNiJUhXuIeJ8JePHfOPt6SHi9rQvjUwnASy4EN+XjMWj/IEYMTWBB4sZxomD57MdxfRIab4/DZDxilw7zYOfODY0QM58yf/proiCiJ6WjZ5gVv7t6RmoRMJuw3DxhRUjd7zENNbxSBGuEcbPMthhYtxoSRL7Eo25I41aRjjRGeEWmkoy4LAqWZz+ckSUZvO7brO+5urhinMVEY+Pi46ehMzyRLiIKRjyLORoRKE2mN7XuqrmDb3dHaEhtamt6Aqwmkx1cIrYlkRhJm0Cn25Z4g8ExyKWIWkzP6oqYQcLm+Yb3f0KUdgQzpqpamlpz+/0j7r2dLsvw6E/y2cH30uTJERopSqCoUAAIEBZrDbrLZ7G4a53Ee5i8cs+kea5vhw9iQwwc2G02QkEQVKrOqMjP0lUe79i3mYZ97I8FBE2CVm4VlWNoVJ85x3759/db6VpQiZUqRRehIgujo+opYc4xoEdA0iGPZxjVt0yKFoBoakjhGRzFOC9blFiMc3g+h28CBcwPt0FH3HWXdEosMEx3Aa/quIYpGTCYJ1aFlX69JE83ZxRlZljH4it6WWNEzLnKyUY4cUubZObPxGV2luDh9yt//O79HnuWPt92/6nhgocEHEf2hZKRpGu7v77m/vwdCrMsYQ13XWOsxxpPnBXEcptEPnL4HZ/s3N2Lf3KSFcp/oMTbXH6f79jgNlzIKw0MsUkQIdYxmyZhuGJA6OOLbtuXQlGRZjFKe3bYiH0HfdmihGOcj6q7DGBiPl8howLmOzoQOhCIraMuG3hiU1CgHpjdsVxvaagioCARJJEkKg6Vh6Cu88WRpTB5nzCYzdoc9hz4wPtuqwo8mCBxJJJAYBtvS2BbvNDLSoCSD7SmrAyITFJlkV5ccqpqqachHI5RKMYPlZD5nOV3+0td2nsdYO9B2DlkFRl5wpS85HHbHtnpDlo4YT8KGz1pL17e8e/cOrRVnZ2dcXFzyi198RVEU3N/fc3p2ghCOLEtxrmc6G3F/f01Z7jg9PSVNE9q25HDYYcxA39cImXB6dsLbt29YLi6CA1GGBNSrV6/49d/4dT777Dn3qxvW2y37w56yqphqzSeffoJ1A11fESWawbQgB4zvODtf8u7lS/b3N/iuZb/ZUdYlvTP0znO/39IYg7CQZwJjGsqD5u72lkN1YDob8eM//zFPn1xw/f6KNM7I0ozNesunly/YrHd88fkXDAgq5yhGOZNpwXw5pe8HdFyQJJL71S3C9DT7DZ+9eEGiFbGOUMrRtCXv3r/m4vKMLMpJkxHb7ZZIp6EILIro+4Hr61vOz5+R5xkvX72ia9uAScAxm43pfEPne5JMcbe6pW4apvMnSG0wtsV7QxRJlicLpAqx1fv7O+7ur4hjyWff+og3b1/z8YtLZrMZb16/5eLigqIoGIae+XyGZ+D29hYIg00hA2u9bYNj78mTC5QS3N5dk+cBqTOdFaRZxOvXr/jOd+Yf8FFK0LY1k8mYrmupTChCt4ZQPlu1R96jpe9bhNCcnCwZesNmsyHPiuOaEDb8dVNzcrLk88+/4OLinK7rg2DUtcznM6RU7Hd7FovZUbgI61JVlrx48TFffvkKZwXD4Li5XvM//NN/zv/y//y/c3NzjXX2GOF1pGnKfD4/uq5DMdP19RXj8ZIoUhwOe4be07YBPTGdTthsV2y3G7wLaciT5SVxlFDkOVV9IIkT2qbi/Oyc9f09d3d3/OD7P/ylr22ApIeuGXj2yRNOT0ZsDgfW+w3VYDi7fMpHzx2vXr0KXE0ZsVnvOBwOoeTYgtDgnQxdHiphdbPi+v0tMomxZclolAUBwQxI6Yi1xkmF6Vr6rqftbRiGpjHWhV4jHQkiLQOmxR6NGc6DccQ6wmvBMLTE8qEgSx5LYGMUGi0UaZbiHXTNFi0leZJTmhopUtzQ4Zxjt21IkzF1PZBkijQN8e79fou1PVJJPJY4TjGmo2s7YiJ607NqN1RtjTCKbDbCSRhPAk6obVuE9ZydLHj+0cdcr29BwVDv+JPPf4yjZHE6Jk48wxAMOHEcMUqnKJOQuQwpBF1XoSXYvsF7y6qsyWJFmqd0IiZVMX//936bP/8jxdAF1Nb56Qm73Zoo0uhBskjGoCaYLsJGgsFBYjuUU6h4hBoGimnBRbqgGzl6uyYaSbq2Js0SemNp2uBIL5uS9WpFnBVYr5gtT/EGdlUD8YHRXLM8PWM0mTP0LXZoiLSiyDPUvUBpTW9tKDkUoWjUy4C5ipQG53FGUO5q8mLMoC1NW4FyYY0TgcNqnaWYjElcQBpZG5JiyTFiDoKuDaVxXd8zXyyOTjhF3x6COBdn4DxSCNqmZZSmFFkexMi2ouz2JKrg5dsvMb1lnCZ0VclsVDB0A2bwx2dRT28G7rdrTi9OSIqEuu/ovSFLcopRSt3WrNcbnj17jk4TfCzIJzlt06IjRZ6PqXYV1lqSpEAnEWmSYn3FB7Tjf/nR2j7YcmzAdSrhuL69Z7A1Ty9Psabl5u6aNMlomyo8/6iIPMuRMiKOB+I4rMGj0YSqrknTBO81s9mYfbXGipY0i9g0ey6fXPDu/S1p4rl8Oub86cWxyLYMZZGpRCrPYHrAYUzLaDxhNst59eolZ+cnTKYJz5895c//5CfYweFMRFN3TGeh0LGqOpwzRCqYUN6+2TGfxbz4+DnX1zcMg0GpwDk21mFdy+XTEw7bnrvbLYuTjCxNKcuKW3nFfJGz2x6YTnIibcnTFEHgCp8sp7x7t0VrjVYK7SGWEV3ZkKUph2pgeV5g6THO0JoegUTpKHRt3O84mS7J8oTeGqqmR6LI04TDdo9EI72iLi0eyXy+wFlH2/XEumCyGGFtWH/u7laMRuPgvh8MsgwiY993IAR5GoEznC5m9MbifCiFb9v2MUU6DMMjhqg8lJwsl8Sxpq5D71IUabq+ZX/1miRJmEymICPyrGCz3WMsbMqeH3/xFavynl3fc/7ihNOLEXGuOX8yI1IRUZJye3OPkJIn50sGK4jyCdnIs95uSdOYwTm0hM40XN+95/RizmSZsdmsjs+5liyJ6cqgDY2KjKF3jMc5Q6epS8e4EBz2fSiPHs85P3vC2/dvkVJyc7NGRxknizmvvrrl4mnBfDkDLxhPRrx69ZZFmtKZHmkEUmlm0wnpeQYe9CSl2kuKZM7QefI0D1i8+xW//YMf8eMvfkbVNxg6ptM5Ty6fcXe3omm2eAfbzY7JeMrhUAa9ygV8Ja2jbhqc8mR5+ivduzkm8IXw33CkQ8C5ig8YD8GR8hq+znkH1j2ulQ+Ykm8iWR4Y1s5y/Bk6fKWUj8jNoMOFZ7//1Oj0AQPzQViXj47ooOh9cKQfZW/xoFscne5HoTeUn7pvYGXCi5WPPPEjy/rRquiPRtngGJYyqF9NW1GWO05OZiAkxhwd686hlCaK9ONnNQw+mBCjQJ6I/dE5HYcSzmEwJHFOFEc4648icI+UAh0FPeXhWVXrgI9pmuZRXA9Crnws9AzCNI/oFAjpO6kk8ijGP7jZv/l3IQR1VVEUBd70SB0fO/U+GM//srntm0Y4iVASM7T0fcdiseB73/seWZbxp3/6ZwS8Ss98PqdtO/rePKJYAz88mIMfPnfn3OPvetATgYBh/YZLfzQacXJyghCCsirZbO4e15+Hc8f7YHKTSh6RLmDtwGgyojcVkVAYK4+dKV2gKjwm5R9IDC6YT8WxI89brBGPJaDOBaOf9OKIuh4+oGmPQxjnXNA1kTjvHlMbzoDHIkXAs7lvECSkAKEUOv7reU1/Y+WtHu5QMkIQobFo1WKMZbApijmhmPHhDfAgO8AhSJEiDcq/B+NbtJSgIpxLkHQgejqzDggUb8N15yASYUonPCDC1Mf5lsFUOA+OFiU1jihc2HLA2PCwKZzF+gqQWK/wrgIUSuZoESGEw/uI3uxwCLRMsL5FiRFKBsnSC4kU+gi3t3gXoiwcxVXre3AGKSN01OPsgLEGBCiRopWmP7ZvG1GiRYLw4MSA9T3GOawfUEKiRYwUw7FE1OG9xvkKiUKro1vZBzeTtT5MU+zxNfkwBXQ+LALELbJoQPVYUQIt3kuQFku4IT+WHwFKWTwaHx+OJ5sKnwEh+qCjiMlkfuSkQxbPsCNHWR6odjvSaIq1jsn0I3bv/oLd/U+x+3smi8+o+1uEDBxDKQTGt1i3JVUpWo9Q0RxvoXcGpXpso/DSgLJEQiCNIcvmGG+5elVSbq+IhGB795bJ/BPO5W/zs+F/ovUrtFBIFYpHw0kT8CrOByf5B9aYwNEToPbieHGZR8f9h3moPLp0HwRzA4/vTSgF9PhQfCuOUz30kQU9gD+WxeKPDh33OF0LeCKH9wp5nJi5o5D+IIgL8dDGfZzm4sDLxyRFaAEIsX8nyr/ppfxXHqlMqcqaQ3tgwLKv9iit6brhOEgwOGtpS4NSBcvLJSqSXN/uqc2BUQIfP3lKkWYMrqEzB7w3bLdr4qRAKJBJmOyFcYSjbxs6V5NECq00TdMznkwY5zlmv6epdojY0lQteRrjIoGRIFKHVAIxSJyxjOdTyoOh6hqyOKLaHzDeMJ0uUTbGG4F3hn6oscITZTFSNQjZM5qkuL3DOoXzKU0naduKrvUs3ECcJDjRsFrfh8/DS0znsT64O+MsIC7G4zFlVaJ0itYxKsrxUpPmE5TUHMqSuhvAG5QgFFoIRzt4hAoO4e4w4D1keYR1HXXbI3Ybqrqi6Q4IPabIFyitcaJnX9/T24YomaBIWE5GjOMTbKs5mV3w9373v+L85PR4XsJ/6jKAwBH85gT64UbadR2Hw4Gbmxuurq6IoojRaPR40/feUzc17WDImpo8z0OsSyqkksdiEvdXCvTwYdIvpURpzWhUhI1LXdF1Hca01M2evg+4Dud6ur453ogNcZQcS14UbdsFESeJiKKExWxEVUd46ThUVShGUwlZNgJd0bYe6SQqjpFS09Q9ZvDM51MSVdDXPUpHpFFMnmRko4ymq3m3+QpDE9ZkIqRXDF1HW9V447g4PWO1WnOzuqPMRmghcL2hHnq89Nyu7ij6gWSk6QdL23bs9ltcbxl6Sd20WCdQUlPuG5TMuTi7RFSecvvLX99RFJMXGWC4u78mSSJSGbHdbiiKnDRLaNsKpQXb7Rqmkm9/51u8f/+O7XbN8+fPub9fhbKYtmOxOGEYepwbiCLF6dmSzz//6TF6LIliiXUdOspompqmPSCEJ44j2rbjydNL+t5QFAXlocEMDWXZcHo2Y7265dNPn/Hy69d0TY8UmjiKSNKIP/2zP+HsbMl2d4/3IV2QZTmr9ZZxVtD3ljhOGJxHRDGr3Y66HciKlChWTKYpb1/dMF/MyUcZVzc3ODVg3cB4kj+6gqKo4PZuw9Pnz3l39QrN5/RlQy9FKEZUkj/76WvOmpqPP7rg5voWT481numoYDzN+fjT54xGBcoJJuMxXdcxVZK8yFjOl6RRRlX19KZBasPNu7dkecJgBoy1OG+JkwitJW3XsFjMKds9Yhg4O5lQmwYpB6pyx3gcE8Wa29tbdOSJY8fFxZzZdEld90yngZM4HU+4un7LYb/kZHHKm7dX7A8Vxnq++voNT56cMV9MkUpwdXWHF+FhcTIdc3V1TZGPWC6Xxw24Yr3e0vcmOGzzGd/67Pv8wR/8KVHkSTPBH//xT0nSjIvzC2azCUmSUNebMGhUEZcXzxiGnkOrmc0ntE2HQDP0Fu/CcGa5LOj7BmgoD2vSOGboLZvNnqdPn+E9GDOQ5xlNEx6whmHg7Pwc5xJ0FJFlOcNg6bqQSnz27BlDHzbWr968oh96Pv7kI16//gXT6YQ8TzFmeHTbNE1LmqSPD5QXFxdUVc3NzS1JEpNlCZvtir5vw+cWZbRNj1oGfubZ2SmvXr4kyxNW9yvmixk//fxnSJ3w7NkL+v5XQ7FdJmOK5Qm/9r3PqLqSN1dfs6oMRqX42zuenp/T1FVAozjPzfsVVdmhtWA6KsKaLxXeCpp2YL3ZMwwerS3CS3bbHfP5jL7vsNbjZChO7wLwkNY4Ts8WICz5KKEra6TyODcwLQoilSBUjBOGVbnBu54kKcgihY4e3HiOqqo5WQZEoJYJ09EUhWRzu6GrGpRLMO3A3fWGYiIoihQ39MxOL1nOFG1fYnsXmNUiPDC1bcXt3UCapEjlwXpUpOg7S1FkLM9m3O7vMVpS7beM8il9F1I4zie8v7rjZDnn8uyctmv44mdfUe92xLkgVhLhLe/fvwUv6Yc9s+kYIaAdOuZPxyyyMb43FLFiX9WhaHawzM+fYD2UvUHVB5CObJSBEpRtwJhx7FgajKOvK9K4YDGbkrQtzg70NibWBT4OmBSReTavNiQjx66umI9HDIMjijT7sgkOvT5iNM45HGqiZMRgG6RWOBVRG0shNVIrMq1IEomzCm97pPHMJmOsF3TeUQ41TVMyMCAjz3yxYGg7uja40Nu2p1c9oywDP9B1DUPTI7XCekPTNbRDz+nFKUKDcY7dEUVizIC93wRnoAhoRefAGosQniRJ6BtzfE503N3ckMcp02JKlsRk44Tr7XsmZ2MGWm7vbklImGQjbBLOgb4dqA8N7ogojZVjcIZDfWA8HSO7gdV2R911dDYMvbNRStOF56ez01Na07HarDHG8OTiksXilPX9mkxE9EM4TycLidTur7uE/w8PnUSkePpWsJhMWK03mMFzv6pom57z5ZTBdOzrA9aETHUcRVycP+Wrr15xehGjj+XgaTI+msMaBrMnL0459A7rOrwXnJ7O2W5XaJFgbHBBj6cph1qxOClY3x6Iowhx5AMHB2DoJYuTlMkkZ7PZ0/UwSuZ8/NGnmKFjW10xGknevXmHwzMaZaxWG5SSjEdT7u93eB/W/M8+/Tbv3r9ltV4xk2MiIajrhqbdkaYF0+mU+9s9k3FCXTcIYYlix2a94+wyDE8jBUZq7m+3TEZjfu3XnuEkCAe+7YkQaEIv0fnZnH25ZjzJOewOZFnGarUlzyRKWA67FjmsmM5mFHnGbDpnOhpTbqCnJVaaNCr44W/9kKfnC/LUkyqFihPGxYJimtK0bbguoozTk/PwHIfD9j1v37xCSEmkNXXT0Hcd5aEkTlNOzy/puu6RXZwkSRDGx2Nsb7m5uubm9j6UhuYT8mJMlqY0Q8tmv6UsS7quJ9IxbdczHk95+eotd5s967Km9ZJ4lGJoGVwKytG7jjiNePX2CmsMI6UpRhmH+z0vPnlGMoLbuw1pLji7WFLkOe/eX9PdXvNRFqEjzXQ8QSkww8BiOsd1ntu7O4gly+WCrrFEUqKLgpOTBbvdgcFIujYhigq+/e3P8FIwnU7ZHZpw3XvPj379E27vXrLbl9zc3oAUdINFIambBqE009GI2+stm7Xh9ERjesmb+zfw7JQ+9WwPAwM9//6P/gO39yXPv71ERtD1Fm8Vpvck0YjFbMn93T2j0xEAxhjOzkIR4uGw58nlKbfbDU+ePv2V7t0PLuKQQhePz2IA33QJi0d90fHQx+YI5dVCBF0gfFG4b4Qf8QGjGQRZ8Y2f+6Hs0zv/l4T0h/9+KJIUPCBmHDY42R+1koC9BYUjiJvePjjmjz8T8EJgxfG//kHqF4/mxAcsyWNin6MWKFxIlkmLFz2D37KtJESCPFviiYn0KHDe3YBxNUI40iRBy4jGtugsYF6cdyipHx3aOgqDTvB0vSGJJbKXII+vSkq8M0iliI/7P0QolhdSY13QxqSOjpqQZbAWKRxxHIe/a8VgAj8d/2Ak+/D+WutQ6oG3HoUUtlKPiJyANXmgenzzCO+hJ4jLUZxiTHh9bdvinec3fvQj6qbhsD9QHkvf+34I6S/vgygug5P7Ua3nA6/9QfuK4wgpQWlNnud47xmPxpydnyGAzXZLVe0ZhgHnDFJ6rPMoFaGVxAvPaBJzclYQJR3GSOJYEqskJASsOw5gjsXl4pvn29FV7o9MB/fhfJZChqHLER8XhjsqkBoe3j8PvRseByGPKBtv8Mri3cN19PDPD3qgVDwy8/+6429uX/Ueaw1aWpD5cftskaIJ7GsP1g0MrkNJi5Yd1sUgLFIawKCkxTmDJSIicPzC5xUY30paBBGODk+KkgnGBAER7xASvBUIkZHIHOdrHPb4Dw+4CGT44JVIwY/w0iB8d8RiRFjn0FIfNxOCSE4YaPAiuFilGIIY7UH4GC9SvJcMrmbwQUAS3mBpsaLGI4lkDH7A+BrjLXiJ9RuU1nhhEUJiOMZLvce6HiEMWufEjPBHljduwLoILVKsFI8LX3BVRwhijNph3Tl2MAyDARH4U0M3IFOPcAPEB3RU44QNTgtSpIxQIn5cmC0RQrujEBtwLoYWJ1xgFsuj0IUkjWLSOEHr+Phw4SjiU8aLZ4ziM968+zHvt18zmpzx4pPfoz38nDhN6NwtXjYY1wAFTgkcgX/as2ZwWyKdIXyMsQe8iHByFkQqHVA+yoKMLcvLHNtFbAeH8ye83TZcEDOzn/I0/h952f3POLEKWJ3jsMGLDkF8xKMMR865C1K6s8eJrcJ6e0T+6EDNOU5DgwBvHxcshwvvFfZDooawePqjwzxc0CKcw8fv/fD1weUujvJ74LOHh03vxaMDPtyYAtM2TI9DMkCIY7xHhEVPHKd9zjss3d/4Uv6rjrruqGxD17V0zlDEY+bzJaPxgnW9p2l6pAgu+9P5GVpotrstCIFzkt2uwp45DvsdWMsoH5PIlH1dsb6/YTaf03YNOM/pYoFxLe/ev2JxOuPp5VO8Fdze3GGNoDMdHgX0GFcxmAbnx9RdR7OrSFIZHuido2wqatvSdYKh9Tw5OyUpIu7e3yCcZpqdYb0hSeH99UtaP5DnMfXhlkW+ZGjCA12ajBjlJ2ib0g235KMRaR7RdjX7w5puaBiPFginqPYlUgeOrI4jsjwLa45UiCgj0ilJMmYYHOXBEkWSw6EF7yiyBBt2DCAEkYrRaYaOUhLXYF1HWW7DTT3J2R72GFdDNKASjYpS6mYAXbI53NH5jtXqQKFixskpnz3/AZ88/S7nZ8+YTKY8+BQQHwpDPyzpfzle98BEexDQr6+vg1M2Co3bD26Yh/+2bYNtKwbT0g0Br5IkCWmaEkURoQH7g0j/zcn3N5lqUimkDty1UVGglCSOPVJZ1ps2DHeqLUIEt58ZBoyxYA3edjjT0FaOWE+JlObq9hodQxRLjGlJohTrPNX2gJUHkAMRwbUgvUb6CEVMJDNimdEODVmRIT1o6Ykk7NqGpjyQT1K8cbRNzTifIjm68OKILIlIdUyiYpqyoUhyhI9Y3d9hYx/a3GVL1ViM6UiijHpfEbuYw77CO8jzCUU6Zrs5kOmMWMQ4PbAvt7/0ta3jJAzutMQLR5SEtFKkEzwapWPSLGxaZ/OcoW9ZLCbc3b3H2IH3V+/I85yub0nTmIvLE87P5/zZf/zTY9v7hqapePbsCWV5YD4/xdiBN29fMptNGI1j+s6iZcT19YY7FR7U3715z2g04nDYMyrGLBYj9vsdf/Qf/pDxeE6RHxiGlqLIOFnOOD094Z/8k/+Wf/Ev/gVffPEzFtMzrLXcdnvayjGenJCoCCMTvBbExZgo9VxeXIaEzGGH0Jqz55eMxinr8p58lDIaL1it7gFFWQ20nWC16UnSEqlTjIxYfnTCEs+XX39FFAtmiwQpPU+eLCmreyKVctgeGPoWIU3ogshiUl0wmS4YZQXD0HB3d4XSAo9hGEqGIWa9EVTVivPzBT/58U+CWyZSDEPFs2dnbA8rnDRY2aJXA/t6zWhW4KWkawtUlFNVhiyfMptLlosRdd1Tlfeh+PDJOW+7hjzNKNKCty/fc3Z+Snlo+PrlW0ajgvPLc1abNdPllNF0QrrfsNsfcEJwqGrGxYTpfMl+d+DjT77NYrHgD//wD49ChsIYwVdf3mEGwWI5pbc7ensgOXbmWCNwFhaLKTe3Nd53VPU9m802lEehcIMDK2jKht1uj9aKi8vgeO/7NkRjizlN3bNeb0nThHCfNOR5wmQy4vr2LVmaYF1OeTBkyQSBJE0S0jRHqZiqKZkt5tyvb3Ci427znuur14yKmEO5RamIs9MLjHFEUcLbt285HCriRKNURBJnrO73CBTnF+cI4amqkpublsuLj9hsDqzut7TNwOnpCV9++Tm7/Zbz+BwVSax1DMPAfDZH64TzX7FY9Nc+/pjl+QynDD//+h2b3Z5t4+ncwHZ9QFm4vHjKy69fh4dbFwrplRLU7YDDkIwSlFbsy5rdviHRMeNxjopDTFqriGySc3t7i/cuoBwjzSxPyccDzz9+ivSwPJ1yb1uyPCYrMkbTCdY5iihHK8HG78lHOQjFeJwy2Ia27RBC0veG+WzBF198ifeCycTirMNbR73v2N51ZEWCTC2ygfE4PADXVcnzZ8+J0wVVWwf8lx2IIhkG0nagbiqU9DR1TzbKIRKUpmGejGi9RXuPVALvQxlt2/f0rWW321NWJZ9+9JTpaMFivKcsG5zuwUXcXG+4vd2jJajYEsWKeVEwno4oqx3LPGFSJNy+LfFDx3g+oxiNKZue7WpFKjzd6g12GPjk29+mPGwp24YsiRFKIZOMgzX8xdef49uvmSZLMp2RqIjFdIofjrYmO9DvD/T7A2kSY71ha/fMFksymTAhou0NOs7Yly1RkpPnE5JkRNW0QSjIcnpjSLQiSxX4GNNZhq6n73qKPMMLhbIDdzf3DKbHCINzFt20jPMRzgq63iLkw8O8JolivLU0dYtNPM4G49Hqfof1jsl8gtLBoNU0DQIwtg0P7UWBMTVN3dG7nmycgYMszRjaga7raKqS5WTK04uPWW/uOXQ1pBIbeW5uroJLMUpx/cBstiSOM/rC0JUDQgVc4mA6iiin7VuGnQ34hjTisD3QWYsbDGfLc7Ki4H69YppPsdagROAL991AXozpe8vVuyuc9fT1APOAuvtlj6HvybMJiRBMx3MioXl/e0cUWxKVcPW24tknC5q+YugbjAdrBH3XcX72nLZ7w3SWYY/rcxzDk2cZUQxfv3zPd394wbu7jv2hwbuG2SjH9i06GeGF56uvv0Qo0LFERZ6ziyVdN+BshI4sZVUiOk+SRHh6BlMxmSy4ev+eUXKOFJKLs0t29R1t04G2FKOUJBUMpiGOC7xXWGep64rNpuJutcE5iRIJVzdbhPTc3d9TJAOL2TlFfsLN9S2zWUHX91RVQ1Gk4ENXj1YxSkiePXkSXJ51DZFGOkiU4pNnZwghqbqeJI7Ih5yT8ZLr+xum0znjwpGolL42FBGksaTc7xiN5jRVR+c0k2QMieHjJy/4jR/+iI8/fsFPP/8TXr97y9/+zd9EdDFN02EPPUmcMh5NkEKjdUzfN+RFhok0cRZSnE3dUDc9682Wvu958eIFVVVxOByYzWZMJhP6vg+u0WJEHGdMF2ckeUY2nqKPJrpt2ZDmGXHiyLzGOokzDu8NeZHzvR98ny4u+OL6DUUyZn5RcLu9ovc9bjDUg4HOYIWm94Z0lPHu+pq29fxwllK3e05OR8SJZjQe0dQlWa6pm5rysCNPC5JYM/Q9zgruVlu+/Oqezz57yvpwzep+zXJxzvL8hLu7e16//ppilHB/3/Cj3/wWea4o65Kqq3n16j3WWeaTU5azC2ItqOoh7AsuLrEuAhkc+13bIn1IBz95csFv/+YTyv2B6/c3vNleUXZbJiczmrYjGWV0u454pBAypCHKcktdNjRVy8XZE/aHkrPTcwTi6Bx2NE3J+/dvWCyWXD45p5iMkOKvLx78zx4uGDKllEerXMC/ekDJI5LF+yOaxYdE7lFcDcn14GJHyCAESo9w/sil/lBAKh6+7RvHI87jWAz68Iz2IPA+GKuk/OCI/5DwF0fNQh41iYDiGaxDHF30D78j/EXghTpKGw8cavH4x/HgaPdIEZ4NrTNBZpcS7zqEDEbIfdNBrBlNpqAK+kbh+lDAmSY5XnTYfsAOniTKibOErmtxzoVBspBH7IhCRnHADEUhJSOUDBqltURJAjKUZEZJFpB4OmBghFKoKELJKBjRTEiieOFBKpCCfjDEaUpZVaRZBniiSDwKwUIQzLDHQYc1DpVozNAH45eURzHdfmPocSyF5SHdDc4LpIhQOkFHHzA1cRQTKc1+s6Wua4ZhOHZriUcznHNHWsPDsIQjgUIK4jg67ocGtNbMZhPm89BlNZlMmM1mvHv3ntevX7JcLtnttoDD2pAolFIQJ4HokSSC5y8WnF+OyGJBnoaeRJRGHv9dIWl67KiUH7jrxyjpsfo2DGJwHJ3x4EzQMZACLyROKJxwx2uAb1wD8lFE9+qYwrAcuegeKdWRInEU7o/pkL/u+JuL6CJHCIfxA8apIDbKDilASxmEL1kdZ0oaR47xEf3QI0WPEhIlHVoKAloEYh3jXBewIiJBUiBIEWLAC4ux4LwikhleDFg6hLRoDFp2OOexxAhiPLuwmZPZsYg0xYgaJ1oUE6Q4MnR9fIyMBMyKEhlC5OAsQhjwR8eyd1hqnG/xTmDdgBBjlFA4TLDKI/FCYY9xeOstxlusN2hSGEyIv8ocj8f4EolCyMAql1IiPXir8N7ihEHJ9Oh4PYremCACew/OYuVLYp7ivKTrOrQG0/dYH9rtdeSQeYXQHiGiwNT1PVpOwkXumzCQ8COEF1i7Ooq5EpIdMnLQ8yFKY8PipI6sqyD4eiw1iuCsni8v6Po1b9/9O+aTKZ88/R+JJy8wnQC5x9sWI0JQQsoWKyz4BCsGTL8lkjMglPo50eN8ghKSxm2OiJ0KkSgW5+fYg6Kx8OXVa5Yv/5hnJ9/hNP0NTFrxpv+/4f2Ax4SF2is85rgAxce0hMH59Ch4KwTqeMtyj+57gQqTTf6Tsgv/0O7sjzeFwLsPg5uHCdexfBR9xBeFyZg78u49A84fxfojusXxgXUWSnODSG59SFVIcWQ6IY/udXOc3hqsD7x779q/8aX8Vx3pZMHQSArXEw0t89GcJCmoDETZhEL2VIc7ppMJicop9zX7csdssYB9ipGSm+srFAPWVGTJknl2gcl33Lz7GX1nkJEIpbt7h3AGtMUay35fkUYFSqYMg8ApRdt3FKkKETYpGE9naK25vmnoulA+NeA4tCXKKewgiaOM69s3TMYxxjbUdcU4cqRJRNttiFOPsZLWdLj2QJEW7A+HsP4owbjIoI1I0jHZuKBstuwO96EcKg5s2cloSZ4DwrMvtxhnyIoEHQUcVFX3xFES3tMkZbPdEkUhYt7WFRpLGiviSCMkDM5RlzVCGqbplLppKasdOsrQUULV7BlcHdIiAg5lS1P2eJ1gXM++PHDYG85HE37wO7/G3//df0wRT443mJCIeBjE/KXJDx+m4RBuNMYEPEpZlrx584a6rhmNRgG7NAyPTd1ZluGco2lrmqHGCYMTKcMwYHaBHzgZTx+7JB5+V5hUuw+DvCNfTkiJ1Pox9oaHSKdk6YSL85RDPiLepbRtiY4k3lUM7Z4i0ZCmSN+gNeSxRoxzvrp5Q3soyVLNKM+JpEcMHjtAlKW0w4BSkkzlRCJhnEwxg4VBo1XC0B4QzhFpiWDgcFghfE+eZKQqwXlHZ0viKEZ4RVnWKNNhblqqQ8tsMgcjSVSOVGOYxpQccCqlHySRTFgup2SRpt52PDl/yvXtNXlWkKQZkUqYFycsF6cslgvqZsu+uvulr+1RnjH0DVGsmU1zkiTGuRDxjrXG9D1xotGRZlLMWK8OvPz6S1589BxnzWP8sOs6dBRRHjYslycoKUjTJGAHtGa722CtIerU0ekgKMvwEPjq5WtiFWH6HimmtHWJkJIkDl0oH3/8jO16S56mfP75z/jhD39EnsWs6i1PLp9xdrJkMh7x7//dvyNSmnE+4ubqlvOzC2aTKYf9nlEx4uz8gvXdHV3Tcn5xzv3tLZ9/8VNOFgturm9I85zpfMzLlz/n4nLJYjljvb5Da0VVlaxWO+Iox5iBpq25vdsxm02YikkYIuQxX3/9GiklH310yu/+7u+AcNxe3+Gs4cnlKVmm2OzX7PcH2sowKeb86Ic/JEkUTd9QNvWR75oS5zH36zt2hw390LLZrhiPJigBaaLZVxXT2YjJPGd6kjJdJ7x+JxgY6J3l177/HQarsFYxnUxRusS6lv1hz9s3N3zrs+9hnaVqGq5v79huS+JYIJRiuTwJJYFahZSUDGWyUaTQUcx8HqLMy+USITVNHR5EptMp5+fnjMcj3r17S1VVx2SAJo4TTk+XbNZ3ZFlMHCuM6TG253Do2O7uGExLUYxYb+7pu4EiytjvdsG92oc0n/eOYQib2PF4xt1dTVmWnJ+dsT+s0JGjabdYa2i7hkMpUfoCRM9gLQiD1pK6rhiNR6RJSpYGh3lZlo+/6/LJE6SUjEYjJuOM29s7oigJnEzdc3V1xX6/Qx9LqwI3+Jyrq1vOzs4QImAHXr9+xenJKVrHnJ2eh/LM4zq3Wt8xGuVICdPphN32QNM05HnOaDShLH+1YtHpSY7Qgl989ZovvnxH5yR9Z2j6Fj84XjXvuTh9SqJTrq+v0UJRTIqwjg0tA54ojihGGWVVYZyliDJGRY5OgjDRtT1KaqyxQHBWTWZzkIJFFiNUYGMvTsZIeqI4RuoYlWQY74mSHCU9p+dnnJycHBNuBusiDoct1lqKPCWKY6I4J87GdEM4b5XWGNvTVJY0gzxLELLDO0McS6pyy+2dYrGYoqKc+fwUITz3qw68CiklJQMiTUdhb+ehGzqaPnCCrfH0nSGPBPt9QEhFMsY7x+Jkxv12z2ZTM19csK8ben9AklDtS/rakkwUs9mEwQy0fc/b27ds7S0fLSeYbsB0FWeLBcuLc7aHkut3b2mqhvm4YFsPXF4+weYTxuMJu9U9bd8R5wU21sjI81v/+O/yB//mT3m7vmEeT8hEzKY5YOKYT5+/IDaQGM+z81MqsyGJE6wW3KzWSJURZ2PW2wPZaIRK4/DALWOiKAgJvTEICUkckUYaxYDwA8Z0VOWBpuvwUoOWVGVN01WgQ/Gl0AGxZAfHZDShGyrapkZp8INBWUWaFPSdZ+g8Xd8TpxHWesqyoWobFicLJpMp+8Ee0U1h0COlJoqSkBhrSqJEo0SEsxZnLUoquqGmOpSoZzGtHejqgXiSsd7t6VvDOB8zzqehFLQbSKOCs/mSw2qP8JJ2aOlsQ9/E9M1AOsqJ4pjRpGC33WHxCCXZ7HdMpjNmszn7wx4nLFop4kiy2a5Ynpwync95++YKLRST8RQhO4T85Z3oQ2dIioibTcUsV3zy/CnTacKu3mE7xdtVTTIaWJwXDM7R1R1SRPzkJz/jxfMfsDw7R4iSvocnFxdc3axYrfYUcUaaKAYTcFfn52Our1e0jWA+XSClQkWa1WrNYhlQAHESMQwD+XjM0K/C+d63bHc1sEUKwZMnk9Dfk89Z3+yYjBMGG/BjWgtkrLGuJ04Ufd+w3d5zchrTVDX7/Q6lRoyKiMl0TFVaqr2nmARucB55fvrTV6TxnOfPn5Llgi+/esV2u+XsYsGhXBFHYc/nrOftm1s++fgp3qe8ubpGOc88HfGb3/91JpMpt9stP/nZF0yKCaNkzNBcIZ3isGnotUe7GFOH5GuRZPjB4HuHTjT/p7/zD3jx9AmLbERbt0RxGDq+ebejMTWz0SggHXyPlIooTkkM3N/fBxdmEtG0AyrOGE0moGIOhwOLkzMOZYlxnru7O7z3LBYL1us18/mc7XZLmmaQaBZnF+RFTt91OA9pWpDpGGfDGjmfXRxNQFv2+w3DYLi9v+XPfvpnWGWIs4QBQz4u2Ox3LBdjfJNwaAUff+sjrq+vWe8PnD+bMBpl3O/eU1YtXdeDlKGLSAu8sDx99gSM4/7qltPT09ABIiVJknF+qbi+W1FMEpI4QypBnGguLk5Z3d9yv7pHMOH1y/d89PEJUoFzAf9jnETKntksYzYr+OzTC16/ueblq7dEcc5kNsHKmr4pSaMIhOTuds3qek2sckajjOVZikx7TNSwq3ZMowg5TshlRJLHPLs8Y3V3xW53x2a9JY4En7z4mNevX7Mf2pA4jqMgRJqBtqvY7dZIr7m/++X35RAy8cIfDebeI7xDHgVS6UOhp3/4SnF8hhMfvvvBwR6MxB9c6+G/D8Lkh+e+DwL5kW+t5OPXfdA5+KBHCI4i+oduvAcx3VnLAzL3oX8xoGY+uNn/ctfWkXd95IKLo2j68PK8+fB9D1/vvAdrA5HiG+WqXV8zmIZIjUAKhOTYndNTNxXCQxKPiHRCb7sjIz1CKY21QTCNohghJFGkEHEYFOhIPaJM4zhBPJAoVEQUeQK33z/+P2NCB54X4X3VOny/8wqPQyqJIwzmBeLxGdh70FpgbSiuBRGehYVA6mCilASSwYchSECPBQH96ET3ge/tpSOOE548ecIPfvADdpstv/jFL8Le3FjSJKEfLDA8CskPxaMPCfSH9905Rz8YkiQmSQI/PkkSFovAW5/NZo+4mvfv3zEMA/t9cKKPRgW3t7eMRooiD+iqwbac6CllWXFOQZblKPWAT4EHBNCDYO3csUMQj7EBwB+wr0czqw3PLGF9DTqcD8zjx0SDe3hfvmEQxB3PZyFA6uPg5uGCCZ2aPOKRAr3CmP80AfD/f/yNRfTO1ChtA0dRCbxP8CQoNUMKgzo2myopj3b8GGGbwAPVp3hTE0uN1vI4eR6Dt0hxjxQVzg8YlwSWOh7hEx7QML3pQAwB8u4BoeiNwfieREdIMpyrsL4FAU4YBCWOis7usTYnkj2RlEg01ksgQkuP8BlC5gyuxRlBpEFIhzMBpQKSwVmcS1AiOZaRDmFR8RG97Y9g/VB6GhZB+cj65YGJDkgkWlu8iwPaxrR4Z4Ho2KC9QTiDF+cIOT2K/QqlilDUoATEfXAT6gxBSVlu6JoKHSeYYSAdKWTWo2SPtwPOS7QoAr7FyWPcRyGJj1OzFONAKSCqAmrmiHox9qEkU5CkMUpJ2vYQFiQZXMlSaTrjKPsNrd+xp+Nb57/FjfkTGvcWKyokBVpGDG6LwCJFjlQJkiREKaQjEk2IZ/iEfoDBAzJwkhwbnOiQ45Js+Sl9nbPdlPz+H/4b/tHvLZiLCybZJ0RieixNDe83XuKcDcKt0CByvLPHCzZgNaTUSCK8tOCO8SKveEAqCYIAKmTgiXoncYRyhjAlfVjQHgR2h/Dy+H1hIqtEfozGGB6KQLwPw5GwOgYxP4ieBi8skjjccrw9fk1wjwYn+wMQpQPfPQZ7fpXjbnsgiSTzfE7bVNA5rq/uaM0OiSNPNdPxKVoIhNPIY5Hp0A/YQZNGKW1Vk8WOLI5hECifcjJL+OhZR2NarjfXWN+jtGPoavAO0ztkothvSowlbIJHOfXQsz9UxDrBWEHfe4bBEOkCBs/+sMZEhrIrmeVTZpMx0irur1YMSUIxSjG1Q6uQPhiGmoPZUBmPUBH0nsGFSaiOJNa1bNbvkF2MsZ7driSKSnqzoytLkmiM6wTSN0wmOUIFB2TVHEizmKZqKKtQdudMj5QebyXz2TicEzR0VU9T92iZEumI8WjCerPlsD+QZVM6WVBWNUIKrHOs1hu8GEhHEi0FVd1gak2qi8CN1JrxeIrpUl48/S5/52/9A5TIQrQvJLuOGxT/eL4+HGEz9YA6Cjevh5vh+/fvWa/XnJ+fc3JyErAjux2H8oCxoXTIGENZ7SnbHXEiyUcJwjlWd7dsdyvOLy6Y5afEUfYBE2MdTduEyb9Wx80DeGuxfQ8CdHzkpJMRKYEdapTIKXJPEhV0QwO2IYtjlssxh3JNHyuMGdisbjk0JVVXoSOIkwjvLHVVksczVBQhtaY67HAx5HFOpDPGmeFQHmgPHUO5xxhHXVe0GGazMaM8w4mI1U7SVwalIsbFjKYJBTPWO+qywbmI6WSB8imJHhHLlFinzKen3LZXdHe3aBFzeroEWrqq5NMXn9KUPbHMGedz0jQNkUMVEUtNW1a8u/madtj90tf2YBq6vmKip8fNoGYYOBb2SOqqRkcJ3gvavmYyLTDGYF3PfBFEPu8tOhLMZmO0FgxDw3ics17f4Z0hTWPwjjjSPH/+jHdv37CYz7i+vuL9+4o80+AteZ7RDxVCDpwsTxHC8eKjp8xnE5yxjMZjXr16iZQWa1s+++wFZXUgy2K+/PLnfPrpZ3Rty+GwBySXF0/49re/RV1X3N7cMh4XvH75Nabr+ezTT7F2YDab0rUdxTjl5MkzkkTxox/9gJ9+8ed8+eWeb3/7M7788mvSNOHl1+8ZjXqkstzd36D0QNPucX5OVe9puxodCe7uWrIs582bN9zd3nG/uqftai7ElKv7DTf374hEzPr2wKcvvoOKFcv5mF194M0fv0F6yXKx5Pvf/z5SKJI8YbPdoLSkKDLu7m4wxtB0HSKJyEYJo+UFm/KK+cmMV29fYvA4IVBRzre+/X2GwVAeau7u1uz3NUmac/nkKV99+Yr71ZoiLzi/WPLJJ5/w8uXXjMYFVVvTtCWeBGsDg3w2mwRmq5IcDjVFMeGTFxfsd1uMMaxW90SRIklizs5O2WxXCCEoiuA+2R+2XF2/YjKZ423ortls7zkcNvRDhVQiFN4lBaNiRFUdqOoafMChPRQb7fcHDoeKk5NTFosTlFakmcD6A6PRGGMkEJP2nv1hz2ArprOCrmu5vbvGmxRBRpzESFGTZSGKvd2s0VHMeDylqkqiKObFi4+5u7lmuTgjzwsOh5LDocQ5x36/Q0jHZDoiTnLatqbvW/aHHXme8fTpE05OTtEqPjohU2azGd45qqrk7OyUn//8Z8xmM6SUVFWJEJLPP/+cb3/7O/ynw83/0qP0e65ebvj8F+/YlQKvYqb5FCkq9nVJ7w2/+PnLcN33QXh2zuBx5EWKiCWD65nOp9ze3KMiAdJjTDBnKBnTtQNN2ZDGCX0fSttPThZYHL3tGUyLEmH/PFuOMTic0Fw+fYZxiiRK6Jo9+92aOE6o65amPRAnwQwyGk1QMqUoCs6fPGe+mNJ2G04vzgDBdv0WFUviOKVrerIiQomEyShF+IS2qtlgUfGAWRnmyzHn509puwozDMRpTqRSfNqiZRDKIi1p65o8SmjqLhTKO8lmc8BZWJ5PyLOUzjbs7jbUZce3v/U9DIbB9rStI0sUejllPIk4fbJkvdvQDR1KREjpyFJNtz7gTc/Z6QJne+r9BmF7npwtub27I5qesneaL++2jEYZUTJCyITd4EgZ6IYe0pi/9V//Dv+v/+VfcrM/cL44o64adtcHfvrqp1yenzE+n1HMcg7rLa0xGBTT00uyfMZu36AygdcpWZGhvCcRGplExCLB+qOw4g2RiIgI6YquqTF9S9M2GKXZb3dcr25oTRu4otKjpUQqTd10tO2KZy+ecXV9Ffa7xyLFrnck8QRrQlGklyGF5p2g7wzr9YEXHz05xtkjuqFHCEVV1eR5gXcw9IamrjldnuNNh9ch/h9wXylRntI6w9ANTEdjBuuZFAsiFyFJ2Nc71qsDbgmzYsw8y8mTlLKteb8pEd5hzUBThxLHNM6YTif0Q41WiiIZsd5s+Na3PqMeajbbHTKCoRfsywqL46PnH3Px5Jz1zYq+G9BmQP71WNX/w0OgaNsGHUX86Z/+BU8uzykmaUiM1p7nzybUbQ02f+wDOz2dY/s9eZ7y5vUVs6Xnpz9Z8cnHnzIqNFla0LeWPA0J7yxNKZs27MkIe8O7+w0Xz86ZzWahRyCOMYPlzdUNL55/i88/f0/b9EynI3QUXH5JnAUusHOst/f8zu/+Hv/2f/3fOL0YE8cRqpXgHKYfiHVEVVcchooszZnPTulaS1EkrDc7ht7wk7+44jvf/oyLy4iXr3+KUprFQvPmzQoVG07Pxnzney/44otfkKYZb9/6YCaya/I8J8lijHfEacLp8pRqv6c1A+/vr9l3Jdtyj5M98/mYr776BR9/9JR2GDhdLLh6f8d8HHFxcUJd1ZjBMC4yfu3TT/hHf/+/5mx+8lhyl8XQtCXjUY4T0JqBrm+IyXFE/PTrN/zb3/9z/i//5/+Ooe7BSPa6wnhPby1fff2Gtm3p+57xpKCYjALnO4pJk5Sybjk7Ozty4GvKqiErTsjyMZHWpGkY0LZdSxSlEEuquqGqG9quIU0zDuWWzXbN/XrFL16+QRYKqTzz2RRHzt3K8+71nm9961M2u4YkHfPpt8d88cXnpCMFUU9fd2R5xmB7dKwYT2ZcX13x9NlTrt6/IVUpphe0jWMyTdnttljXcnK6oBt2xz4lR5LmOCST2ZSuhzgp+MUv7tjsbhm+vuPFp6cUhaLrNaYaSLMIpXsQPXjHcjGnazeksSeJJHXnEQa8CEbCUZHieo8WgiRWPP3onFdvv6LtEgZjubq9YjK9oJhOyLOIqtzS1A1d15OOJIPv+Prdl9zd7zg/WaK05vz8lNu7WxwQJzmbzQHZS7b3v1oCXHiPcO6x7y8YHI9Uc/8g/B3RrfKIqv3G6vDIVH90fvMNQZRHpzlH4sXDr3nQJh5LGMURgXt8XgziOcGVK+ED5jxc69b+5weDHzAyD4K4f3zJH5zBH1LNggAQAB61MyklxgyPzu0PBixHXZXc3F0xzgRxNEeiaJqefiixzpAnCVoHIds6c+znCRx0IQVa6uNr8kfeeViknTsWo3qOgnvQ9pTSaB20nkfciQ9Ypg8it8VafxTD/RHVonDOBh1KSMyx4PThT54XPOic1hn6ATQxSIH0/pGWE0xJ8uhChwchXRwLSDmaXjkK/D/84Q9ZLEIiaHV3T13X7MuaBxa7MeYRE/VNDv4jQshzZKnPWC6XjEYjsiwDYDKZsNlsOD8/f0zHbLdbtFYIWRDHCX0fxPnlcknVQJ4XPHv2nCgKxcREYRikVNDonA+otgfzlRD+iKf2IZJBENsfBhAfXusHk557uH6EDEx54Y7/3qNB1ruQkRAyuNqFAiWQ/sGJ/4A9AoQLJaV/zXkO/wUi+jS6pHE7kJZIJCid0Lug6lvf4uVApjOc8/TGhRgREVqWSDHCS3vk3XjqboOXilhlRCrCuPSI3vChWR5JpFOkVzjRUw0tjg5xFBARCi0sSnm0UkjR432M7VMECusqvFwR6QXeZJRmhcLgKRDSACOsT9EqQXgH7PHK4WWBIUYYibMC5zXWt1gSIrkkVp7e32GtRJIiRYwWDieOWAQcnp5EJkGYdR6OJaneGZwPURXvYqzrgrjuVbighUL5BYgeqVLAHB3f9oghkSAMXodSODt4nHEM7YC0Amd6qqEknuYkOpQYZTpB2mP8xHukiGnNAYTHsYLgsThykRs0yaPz2toQ08RZlIY8H4Voh6sQQqD1DEkEvkMrQZzmiEgz+JYoHVHxDiMO4EZoPQpitZ8eReEI5wyCCC0zHAKExtga8Ag5oFVGkoxxtqe3Ld4lqMRTnBzot1P6rmB9f8fv/+G/5h//N/9XUrFA+hmDXQd3+HH1UUKHmy8g0OFu4B8akBXqOHUSKORDE+9xIOUJNxVJGJg8uNAFmtAOHQYMocjgIQpljwtcKKvleIOSj8UfaXD9YvCIMG0UUVgEvQ/RHcSRC3aMTB3jWw9xG0l4LQIXprN4vI//ppfyX3kYY2AQTMYL8nRJWbV4IZiNM3CGIolxdmC723B6OkGYml5McNqiVcVQN8Rk+M4SZRlCR1yt3qMyTUPFpjzgjGCUzdGDotAZJydL8CnFaMLG7TjUe9JUI+WA8DX7ww3j8YQozrndvDmKzylRPAfRoHXFqChomwNFpilmEw4VtKIhi3OGruTgrhGNw9iSvmuRIiFJY9AxKp+ifUzhe3R3INdTlJwyiRVtf8BSEcUTDlVD0/XkuaJ2DdpYUiXJC4l3ClsPRGSc50s2/pb94YDwEWfnT1hMFwxNT0bGWX5CP1TcrN5iERhlqLoebxV5OqL2O7ZdjTMaJTTVoUKInnE24WS6YLvtQHVsDyWX6oQTdUk0yllvWr7z7Acol6CERuoHt4JEEB3PnwfsVfA5WOeP7OVwrhtjjw70t1xf3ZCkCZPxjCQtQMZUTU9ndigsjaup2h2tW9OZHftSMJ1Njm3WkvdXrxhsiTgX5NmUOMqIdIw4bgKN7WiaLiCmlArYKylxztN0LXiBkhFxFBHFManPcd6ilUZIhTzRtH3FzWrN0DnieEprVry7eYnTjiRJKJIU1QvqukOrlMXlU9rBMvSOVDvKasVhtGU+ScnzgkmRYYeWSE1pleP97oaT8RxvE2I54VDvaLsQ5+v2lsX8jIvzE4ztiVWPGQak6JmOTjEDKDTeO3pbM1tMuXoNs2JGrGJmWcZ6daBIMpI04ub2ntPTM3QcuO5t36JTRWUq7t/fUW6uUb+Cm03IIJx5b6ibA7qPSJMxXgqqugTviRNF05Tsti2j0ZTTk1Pev39DYAdqkjRhv9shpGW1vuX21jIMPZPxBGM6vLN867NP2O22vHn9kr7vKYqcs7MT3r9/z6HcslxMUMqSxDFSRjgGzk8vaZuOr776BWdnl7x585o4VnR9iVKOwbR4P/Dzn3/BbrdjMh6z3qz5u3/37/EHf/DvyfOcYeip65JilLPbb5nOJrz86mt++vlfMJ9MkXEUWJx9h/M91zdv+Dt/92/z6k1CFCuiWFGMMuqq4eNPLjCDJz0/5X51R9cpVusdH39iGI1SNpsBvOPycszJ6YLxeBTOt3GKiHo6W1I1e7yzZGlBOk4QMXSu5X7XcTisKasDJ8tz9m3Fm6u3LGaLwPUe5zxLnpKoCEdPmsZIDS0WGXlev/2atm/YVzuq5gBa4KTgbD7H2BZjDWmWU+QT3FyjVcxiseD2ZsXl5TltM7A/7ImiiP3hwNmTU+JEcihbZrMRQkBd1yRJ4IgnSYY1njQpuLh8wv1xMx7HMW3XECea+WJKWa2DecF39N2A31XkeUGWZlTlgNbi6EZvSTNN2zXc3e3I8yld24KwGDOAD3ubLMvIs/yR33h/f08UhVKnKOowtsI6GfBEHibTDCGDc8Z7z+FQoVSLsBmjQiMl1E2F8553764oy5rpbB7c7n3PfrsnjSKm0zl5XvDFF19Q1w2ffPIJp6dLhLDc3l3RdQopPV/87KdkeUbbhTKn3W7H5eUTdts9ZVlzeTHn448/5j/+x59we3dLXe2ZTqeBkzlUzBczoijm/v6eYeiDs/BXOK52t9wdamoDKp5Tli1aSy5PTunLDq0T3ry6phglOBuiunVdIyRkUcY0L9gcGowZmM/GHDYh1VnXFR6LIgp4NSGI4owojhhsRzd06FjT9x1pGqOVYGjbsB+REGcJ4/mMvpdMJzM2955dWZKs19zd3NIPNZNJSts2jEcThBCMx2PSnWUyX5B2EtMfSIucfJyTJQnTyZLt/p4sLvAmIlIxbW0Yj6bMTybMTi55+eo1u82e2XxEPs1o2i4MaCJN13Vh/xp5zDCgdYw3lvpQMp+eUJUNw+CZjCeMJxMQlv26xLgemQkGWaNzT0Qow8qyIAwWaUqWJpyl56zuSpCC1f09w9Di3cBnn37Ccj7lJ59/zuXZGaNPRlT7kqo8UDrNtjXsTYlbbcjjCO0dylqeTSKaak87cvhkxG/8o7/Nv/xX/5YvXv0EMxhir5jHMWqI2B96fK6ofCj0e/LiI9J8wnhywnyQ1O3A++sbUApnegTBeZ6NR4E76wyuN8QKtPMw9EjvUCogA+9W9+x7Q2sGPCFZphKFFIphsDRNwGPcb9ZcPr/g/bt3CCXxVnE41AgfIwVIpSmKETrWDLZHmnDOdF1/fNaBpulAhL1KUUy4vbsFCeW+YjEJbvXpZEJ1OJBnCVmSIeMIHylULGiGgSTJKeSE3fWWw+6edmjQQrDb7nBNF5K9XcM4TciTNNyPs4RNuQ1896pjOV/S+yzE062kKRtevX7DydkJ290GM3Q0tcF7Q1UfuL27IcszsjzB9O2HmPgveZhOYfOeyYnmF1853ryvOLUKlVhGE0Fbt3zns8/YbO5pDi1xKuj7mtOzKa/f/Zz9vkLIlLPzgrK6oh86hp3ENA2jS8l2vSKdJDR16IiKk5jV3Y591WPsLc8/umB7ODCeFYi2ISk06+2W85PiiMUcyFJNHMXUJTStJ4p7xrOEN7c/5fRZwWZzx3Q2QeuEtu1QKIq8QOYS08PubuCw2/LRp+d40WMtbDc145FgMlVkhURrQde16Cjlu98/4+LpU372s5+xPaw4lCXrbUFZS7arHikMOoJ2sLy9uUdFklk2BRkxWqSshi1lXbHa3OOwrA+3lE3FzfUdUayYTpdgbwNDOVIsxjO+9/0f8Pd+9/coXMQsGmF6j401/59/+a/4R//g9/AOtI5wQlPWPYvIY5saPTrh93/yU+79jkFKpukIoSz/8aufUQ0D89GCKMpY5kviOGIyy3h1/SW73ZZpsSQtIvZVhd7uSS8Knj3/OHTcJOOQejb2UfTyg8FJgfEWFcHgDKMioWkqlITBDSzOTzh9csLV6hZ36GnLgo8//ojV1YpZPuPf/ZuX/Df/5Lcod3u++4OPMeYJh2rFYCyWnqIomC5PKYqCuuxomoEsGSNlKBhUMme1Ca7z2WzEL758xSefnHFzNyB0QlGMMIPn/c09+m7P6fKS1erAr//Wt9nubkA0rNZvOTk55cXTF+TpKUVe8OMv/neG2z1KRpyfLhnlOevNhizqUTah6jVNHYyaWZLw7MUlf/Tv/gzx9JTpeYEdEvpSkvoMXIPoG5JkzCiLcMOA9zJg/LZ70nHF+dMlJ0/GRAK+fvWSumsBwc11jTEb5vMpqU8Y5fNf6d4tvMN/sNAGAf2hg806nBRHzUugjriKb3z3BxHVf8CwBAHDfnB4+4cC0A+C+Qd3evj94Ws+COseG5zi8vg1R3SsFB8c7ko9lFKG8s8Px4ef/w0TPN9EkDx8zYPDXYqQQg5FnvrREf2A/wylnUE4Dk5qS9PuyJNZMMH2nrZtwVuSNHncU/V9g/MW7x+6BA0ChVBBH9L6iA5W8ugwj45CajByPAiwQiiUCvxx7wPH29ghaHE2oJWFgL4PBsu+7+j7niRJQpGlc2gdIQQ4ETrDQrpXH13p+igIh0SfdSAfi2aDJipijprSw+BBHIkRwQwlBEymU374wx/y//if/mf+1b/6Vxhj0FJRVxVCaqI4wjpLPwyPA4yHz857j3VBf4qiiPGkCOXOxnB6esowDBwOB7qu4+bmhrOzM6qqekyoSyk47A/hmV5GVFUNOOIsOVIsHHGUI0WFD6ofAMY4hiEUsAYdLfwsjwu1hT6I55YPKYkHl7mSCtwHdFD4f+GccsijMfXx8jqyz4PooVQc9F/vjthm9+H6OWKnrfvrzal/YxFdyJxYKpzfkCiJpUd4h7MOJUORpvISSYSjAmVQymGcAbdGK0/rWrRXoAYG51BiglYN0qYY5xmokV6iZYGXe6wzSBEhhULJNEwPfBDtYx1h8UgXB5ey8GgyIpEzCIcXMd5qIjFlFGV4sQYf07sOQ43AIX2GMwPWNVgf6kYiqYK4LTSeJEzYtScVKULUDKbHS4nx3REo0YFPgBiJxzJg/YAQgsFYlBIo5UNrvZUYFBAibUqYEEVA45FIUeC8BiKs2+PcACJFeB2mU04GJjwx1jnMYFBSI+MwVS4yjfErMqUxLgEOeEqsD+WkAkeECK9VZHhRMjiB9RatBqRPwT+0HYcTUuuYIhsxymcBfxPF7IbXtM0d4/QpWkum+YJrnwIZaZwiYyjtV+G1k+BcmDb1zqKFAt+RqOURr+KRPmawIZGg9JRIxihpGWyFIDRiS3uK9Jood2RjR99mdPWYdbnny69/wg+K32Gqvk/n3yOERfjw2XkRmqydN0eGeFj0wxQrAxKca0Oxq0zD9BeH8zXWd2EiKpJQjkuLFx1KJAQ0TvgshTdHMdvieGibdiA04sg39/RI6VCE9mh8clwQPhSahuLrIK0/TtiO5Qnhhnm8FoV+xL4oUjwGx69WTvbik4+xjcU3FjsYoighSRW96EizCCkAqcknY+qhY1/vQELXNhjfkuUzYpMyyQrysWLf7qmbLbGI6LoDXbsnjXPGecJyOiXXkiIfEakZXgiEklTdgbfv3xBlkrY/ILTHK8PiZEzfGg77NeN8gU41SZSyOtwzLmaUVXAOx4lkPM+5u7vnsK6I1YjV7hY/DCxmY2bTKc6nZNkYvKBe14yKMUnm6ETC6fwjpJvQ+4Z0fMrt6g37ds35yRPawbPbtZw+uSCLPYf9liLNWU5PqA4dqR6TZ6GDIYoStocSMwyU+z2JijFtT5Yk4BR9NzCeTxAIJqMpkXNEaLbNIbC9vOL05IxUrWnqDbGM8T3gPGW1pq16hqRgVJxi6piz6RkfPX3BqCiIk4DAEY+3KEEoxP3gYXiYzloXxn6KsAF5/fo1b968RgjF2eTssfDNH8ezcRJj6bhb3eJ8R9tV7A9bvFMs5hWT8YKimBBKB9fk0Q124tC6YVzMSZKELM0DZgGBtccG72HACc9gTODfeXC2oiJMqrWWxHHEfl+zWW9pbQnaoBONkiOaZk+UjFFRjpCGs8WcoemIdIzvNdPZAucFTTegZUYcj1CxwTrD/eqe8WhCkSXsuwODqellRd0fcHIKImB59vUOlMcKQ91Z5kKgY81hvSWOcibFlMNuhR2gbTqKXOOFR0eStzev2O23vHj+EWkU48zAZDymamu6oef04oS26dlt9lxcnhNnAqEsn//kJ3Rdz3QSU7W/fLHo6XLCen2P6UOJr0BgTR/Y/XFEPxiEt5wu52B3PD0iLoZRBj4gPvIk4qPvfy9sTrzj1avXLOfTo3Nesbq7pa1LvB341icv+PLLL3n3+iWLxYKz5RwleibjUIzV1D1KWvqupGtLtpsdVdXx5PKSq/fv+PTTjxhMw253h1KWuqn5nd/+be7u7qiqHW1dsrq74b//7/4pv/jFz0nzmPXq/nHj/vTyCU+fXaJE2IBv1mt0JInzmChL6UzHj3/8J0ymOXme8dVXP+fFi0/46quvWEwmfPnla7Is5jvffcHt3S3DYJnOct6/v+LJ0zOkhPF4SlGkGNvjsWR5zHR5yv36lmZoKLIIlEWphK/efMm23DAZF8RxQB801iC94M3Va7777e9yeRp6TjAOmWVMJ0um4zG3d3c40/HH//HPiDNNZxvariRKFXGakhY5p+dzrG0p65KhG0jSgpPTS6bTOW3T0XYdcRrz5u172nbg7bsrhsGSpjHKOaJYcn4RnK1fffUV7983fPTRC5SKmM9PefLkkjhKKcuS3W5LksZ0XctkOsL5niiOSFNNMUoYeojTiNFoxsuv3xDpjKIYkaSKicyxrgURMR6PET5wY+Mk4FS61uBscAalSU6apiyXS+azBWfnp9zfF7x++xcYZynriqIY45xjnOVs9xVl1ZKmGf0gsE3HpCjwWCItGPoh/K4oRGLdsawsiiL0ODo6kQS3t3ecnJxyfX3N+/fviWPNeDKiGH1M11fs91vW6xXnZ8+OD2aBZ6pVzNnZJe/evT+us8FRs92u8a7no4+ekyTJMaIP49GIs/OTx1TOr3L87LblbPGEaHvFPJ2y/emXbDcN0+kLptMl1gf2+aFqiaIolGjjwHpMb5BWssxPcAdPEiU8mV9QHkpM3WH6HuUNMo5Rk5zGeyIdYY1hsz6QJAqhQt9BEkWUuy64IWWG1GPadiCOUpxt2W7vccB2X1KWNSfLlHEhyJOYInb0IkJ5Af2GZqMYjZe82++pmoFslKHsiDwf0Zoa6xVX11vyPKbtavLpJdb1jEeaH3z3W+w3e4a+xrkGlYR/c905ytqQJQYvDF4qUBFt39HTk+Yx9/e35HnMs2dPWS6XXF2/I4pzZvMpSnlOziaMpxrnLOvVikFZ8kwx0hF313ecfvqCaOpxvud+t6M6NMyTiNPFgvdX7/no2UeMp3OMs2z3OwYxMNgBJSWmbcEYjOlp2hbpPT+574iTmInoEd2eKMn4r37vR7x6/Q6HY191nJ8siWVM2w10xnD20ceMJxOy8Si4PnUKSqKFRumIRGu6wWAjhRMKKTTjIj8WpRrs0FIfKpT2KA+ZTBm2G1b7NYOSICXSKpR3JCJB9CKkkJRCS0WzOzCkOSejBfv9ATv0KOkwpgEV4WMQiQfpUC4mQiD8QL3bURQF2/2OOFG0bR1QF17QVCYYeoTF1xbaHqc6lssT8vGEsqrZtg0IiRbgu4G2bEhHKYO07LY7tJaMJmPQnto19H2LljBJZ4zGE5RIcMYRDRndukXHiu1wx/Nvf4bxnqZuEUnM/fUdWVbw2ZNv8fLtl7RtidCSWAgOuxXJ8pRiXlAPFSj90ML0Sx1pmpHnmr6vef6i4KufH9CJ5bPv5ghhkEju765ou46mbSlGE3bblukkY3GimM5TPBbXh4HXYBxKpaGoV2kGG56xObqqu9bw5OI58r2l2pXUW4VyOaZ2KCTLWc5m8x7vBPt9y/nFCdv9nvH5lOViweefvyZNs7CnS4I48ZCYn4wndO0WrRMEEUkiKfdbQDGfzcFp2qbEe7i62jGdRiBaYEReFIzzOff3K8azEdvtLUNX4XqHtgJhFGmS8M//+W+z2254+fIVn3z2lO1+S1YkHDZr0kwHsUYKht4g0FRljavXpPEYKVK29yWLUcInTz8DJ7h8ccHv/sbfZjKdB8FtcAjn0LHn/eotX718xf/w3/4z+r4jS6YIUqrSUKkB3cOhesO7dy/JpkVwYwvLZnfHu5tX3B0Mm7sv+Id//x8yXcxp6op//f/+/wZ3/GxM36z59NNv8d1vfzcIUzygN8IeP9KarmuxtjsiR0KaWaugDzjf0dUNXdex3e/4i5/9jOv1HVJ7sizi6bMzrB84HHakWcp+V5FlgjhK+M63v0Vftwin0SLD2J4k0nR9xen59OhUDZjJ/X7NeDRm128Z5zFFkVPVW05PZ3z8yYLV5ooskySpousamqrDmRgR9ZT1eyYzwWKRI+SIsuzpq4HbqyvakaWr2nAPLyI265LFYsHLl6/J84z5bMrJ8oQ/+g9f0LUGGYdyyfEk9Pn81t/6DnmRcb25ZTwZc6gsy7MR+7sS0Q3MZzGbzZ62rnn29DleJsQ3txjfIZzi9n7Ns8tLVByx3becnUzJs5j37w5MCsnsfMbrr69/pXt3UGWPf32w2Hp3NGWCcEe3thRHQ+Y3jw+Obv+NnxFwWA8/1j2K6w8YEggu84evfxDQ/5KI7t0HfrT/4P71wj/iN46oaoIL+kG0/6BTBCf7B1TIN/7Rx9cd+NPBXAvqiNJQKpzj1gYToRAWMH+pV0tFAuda6nYDCCKRE8caQUSkJdaBGWq6riXJM7quPX5v0F+ccyRJRJIkSCmPzvAw8JRCoqQ+ki/EES8SDGgg8I5HYV8fr8G+78my7OicD4anUAosj2K5P/58GfrCjq5raweGwQIJSgVcljt+Tt6H12SPaQTz6NwPry9oh/7oguf4mSmKouDTzz7j4uKC29tb1FEUP9RN6LPwQXTXUdh3Cgf6OLQIgrwiTVOePAldPUVRsF6vubm54enTp+x2O/b7PbvdDu89TfOQLh/R9wErM1/MKQ91EOs1eB+z2+0YTyWRGo5ECIDo6Gv1R8e+e0QLHb3huON7bX04v+ADwsXJh3LdoHhIEdJYUkqEc7iHSMXx/HtIxVtCx6B60OqcOGpxH5BCzttH7NF/7vgbi+hN36EjH1ADosO6hOTo5DauxbkxrW5DXMsLBDXG7lFYIpUQKU1j1kHEkpc4oendAYdgsCWxmoOowzROJiH6byq0qIOI6QWRUrROgZMY4RB+gZJTjN+jcaRSMjiJ8RnePTlOZ4JDpjeaWOYIWdENe5xMsMYFRIbXR6cNIDc4oxGiIBIJQsRYcU9rXxLJFCEM3g4MRpIkY4SLcFaiVIaSCmctxu1xlKE0VSYY05OoGfg2MNiRgTMuawQRxoUTxnqDFI623+JsidYOhEOKaXiPrQIbI1yMigRxHB+nJWHRVFrg8oYojnEybNQFOQKJdTuc71HkODEgpMF5A74hikDJFmcKhFePOBdHiNbEcUQchVMlVjnCOQZxSzU4fJ/SN5LF+Dlt9+t43+CzmAO3DLbEexkWHwFapAH74y3tsDoKzQbhc7Seg8ywvsT7DuMLhqFCC4JT3r8hUxd4qVGjhrSJyesJXd3y85c/oSg0J9/5Hqvh9zGsiVSCFBnWD2gRpvBS5Civ8bIP6QkA3PGOcpw+YcKE3wNInAtTXe9bvGjwDDy2TvsQC/F8iPNAFP6EfBkcFydE/MhrCgW6x5umD8gW/1B64MMiEcwFAiWi8NKOxaPe25Cm8BasAFmAN3jxy4tsAFXTkYiIKA/laoMdiLxms9/w/NkzmqbEWEc2yrnb3mFcS5IpBjMw2I4ksqSjOPDFo5xMJZRDC4MjcZJZVjBdzAPDtt+CV7S7A9gd96st08WC5cmc1tW8ef+Kze6e+WlBWqR0Q0esM4S1uKElSguW4ylfv/052IxI5RRphOstWiUor+mtJc0zhsrhOkMSJUzHM/pOkMUFq/s1QwsGxclyjokzYuIgMvQH6rLF9gO+dyRxSpIpbAfaWbp9hRgkk9kJo9EUO6wwZqAedkSJZhJP2R1KuqpkVuRsNzeYpqOPQ9yzqXvKbc1kOmZoG2KXItqe4dCQIIPb2g6M0phxvqDIYvqux1pD3eyZTmYoDftdRT6a87d+8x/w4uOPiaNv5Ia/4QAQxyIS7x9uVu5xUxXY5I77+3vevHnDbrfj5OQkIGrairqtwtpkOqRw7LY7elvTDTXXt7c4O+DNgd32wGx6SlGMGI9ndF1DVR0wgyGJC+qqZjpdUOShFMk7Rd8PWBewNAZD1zfc3d2hlCDPwpDJmLB5iuOENEtYLhdsyoHNbk15KFFI0jQhjmI+evot1rtb8jSHOEWJ4MaNkoj1ZhWKbaRgOh1T1z373Y75NCNNIiCnLmOkhwEP3tL1DYfDHmcMTluyIuX6/p75yROyccShWYMyHJodddXwyZPnHA4HJIIkTWj7hsa2rA/3HKo1m13G04tLlI6IbEy72zJ2gt72tKbk0Gx4Fp+RZwX7bcliMsV7yd7ueLfd/vIXtzDkRUzTNIzGKVJoRqMFXRMifdIFDvDZ2ZzoaU5RZLx+/RrnHJ99+inv3r3HeUNZ7qiqiidPn6LV8X2sasazgh9+/3ucnix49epruqbBDC1tW9J3GbPZjNlszNnZnPV6TduVTCcLhFB8/fIVH3/0Efv9e9bre4oiDrgMU7FcThmPc/Ii5t37Vzx9+pw0yVHqDTe312gd8fEnL3h/9YYXH79gdb8Km73bG5I4RinFYjSibRveXoUS0/v1DXmRI6REi1A6mGYJF5entG2D9/DxJ8/p2hbne05OJkgpSbMoDBOl5uLyBCEkeZFgXc+z55estndEGWwPd6QIiklKEue4QVOMJvRdT+dalIzoXUfZVsH5F0Vsqy390NIcSpazOfcrw1df/Yy+6Ti7PCNbjnj17ktmyzE3d+9BWuYnM4RSOC84lHuMgf2+RPiIp0/OOTs7oygK/viP/wwdKZbLBa9evUL04QFpvpgxmY558+6e+XxGURQ0TUscxxTFmCzLSOKUrhuYTRc0Tct4PD4myGC327A8WVAUOT/4wfcQIggmUkIUKw77mq4bSJNQQtYPPVqDHSxJoplOZ/QtHA41CEffD0fDhH58QAGODw0ckR9jhNA4q+gtpImk7y3lwbBZ9aQJTKcZSTRmvV+jadFCs9uuQUj2uy1KxRR5BsJTliVSap49e4azjvVmQ9M0/NZv/RZ5nvNHf/SHTKZjptOcNIsYjMJYw3QyATzT6QStI+bzBc55uq5GIDnsD0ipmU6npGnM3e2e1WoFEMqYraMocrIsxdjhWJD6yx9l6XnxfE6Sbrm4OOPnX3xJ1/dc392xWJxgBkvdhL1ilqah4JtQpAmC/b5kPpuz3+7Zmh2L+YKzkzFv3r3E9j4gHL3HG3BOEAkNvqPrOpyTLE4mDP1ApEPJfd8bIg1KSFzf44Xn5v0V79+8JIkUiVIkUrEYz1kuU+JI0g+eq5s9bdeQRI7V3RXqmFTt+w6P41DuEEhGk4Sur+mGhlwpxtOEwVbsy5ZXr2ExPSdLUy5PF6x31wzO0A2Cum1AOIwbWG+2ZMkIqQYckI1ylJJMpwUX5xf88Ae/jtaayWTE23evmS8mKBX4xEqcsLpbMWQOoyzCOcZ5QrUdWF/dc3pxyaHZsDybcLNdk8xm1H0NMjyQ4mG73jD0PaM8pxqq4IruWxKtiYRExzFNXdM7iVQp26oDOoSs6fqWLI+QSqOyBIOlKrehjDFKmM4XobBLRwgdIbQODFihOT05QVjHqmvoTY8aJN4moQ9ERkhnUUAymwOe3aGk3O0JMXJJ04XkRKQjbG/DH29D+aRSGGOItGK32YVUQZzSupaoCEkSFYfP37mWvvW0FaRxhpAeFxmkEkymBcOgWW8OTKeTEHU3Njgzvacqa9JRyvXtHb9++RSdJKjWUDcNSocOoraq6buBeBGhY81kNg789kiiE401PTJVpHlGMs6p9iX7rqFtQ6eCkCFFvqv28PoVy7NQUG+xRFnC11+/5Ld/9JssF6dcb3uUEqHcUEk26xWnZ5dM5jOaLgxFftnj009f8O7qF+hIMZ5onjyDOBZIaQNXVnvauqXrQp+SHcA7ycuv7/nBDy/pB0ldlzhnONQdUoRn6zjJGE+mrHdb9puGtjIkcYKUEdc3G37jB/+Q3//f/nf+9A9e83v/8Fvc3L9iNMloyj1prGk6h9YZdWWJVIExHut68lwwncyJooSuNeR5zs3NHRwTu88/esZ+X6IiRZxEPP/oGdfvdvxXf//3+PFf/Dn1UDIej/ne9+ZY68iylNVqjRSa29s1xSgjigVVueewqViMZrx49hnvVndAz7t3r/nOtz7DuZ7tdst4NiJJY+6vrkmTEaNsDE7SNh2L0QX7+47WSGQUCukuT55xuK/5Z//0n9FUDU9OL1hk8+CItA4twNqO1nf8+Gd/hogFZVvT9D21adlu9xR5AaVhGhf8h5/8hL6vGMsZnanpq4qqr8jHOUO1Zkgc//rf/6+cLy/x1rEvS6zvSPOcv/NbP+Ts9Jw4ConN6JhokFJi3cDRrooxPTd3G/I8I44jhq4JGWchuN/tKNs9MtG8u79h19TkJxNyHyF1GIjuyx3Lkwld3xDFkv/wB39GXoBxjrodePbRjPW2YraIef7xHCk061VJUw1MxgWb1YahNzgrEP8/1v7r2bYsv87EvmmW3/b469OWQ8EUKRJsdKBFqhUdUij0pH9U0qMU0aIUNEESAIEqVKWrzLz++G2Xn04Pc5+bBakDRFT1iqg0dfOeve/ey8w5fmN8QwmMM0xnFfumjomVcQARKPKScehIE/ju5S0//vE5m80N83nB5eX3LBZHvH19T1EkSBTnRyfslePlqzWzJMVZibWOqiqZzaYopRgGwx//8c+5u13z/voVFxenMd1Ub3j39hsePX5EmmrKSU7d7/AyQaWS5dGS2XxBt3Vsdnf0fc3bdz3HJzlN33B/2/LJZ6fcb7eU04pEai4uzvnZj3/C+zfvePf2HavdNcnkDzOvGTN8QLVEtEWILl35A1ojDkaSSDwIh861g3j94M4GcActKOqMAu/NAUcCHNzm1sahfXR7c0glu0hN4AdH+Q8Y0N8R0AMoFXUIDn14CP9DIWn8LfgQzU/iQQg+OKoFES3jfeRPPzjdhYjOd+cPe9TgUEof3rM7uJPDB+d0dKePCCUYxg0heGblKVpNcFbgHNhxoO120TDRcXCbK8CCBCUVQkYR21pIktgP4lx8v0rH/jmpooYjpMCOsTfEB/+hH6cfOrq+RQjo+oYYKnD0XY8Pnu3OUJUlSaowZqA4lJzG7zsiVLXWh+/GkGUZ1gykaYbzELw/iMMKGw5EBalItIs4FhUTgkJw+MyiMeTTTz9lOp1yeXn5YR8dOe4Ce3D1S0kcyIkD5kYEjo4WhBD4+OOPyfOc1WqFtZZXr15xf3/P48ePef/+PVmWcXl5yUO3WVGUcQAh47nYdT15kZFmGudr8iLFB8Nme8t8qtEqJsZjl2c4aLUPg5UH/v4DcvbhBIQHET1+ZjFZELHbBwXvUNArHvAsBKz/YZDrfkdMfzinH9Y2QujoSvf+QOIwB4f6P378k0V0qXc4F7C+w4kJzlsat8MLRRKOkAI6s0aJlFSWH15c6xQbEpwf0EGj5TMUCaUyjBYkOSKxwArvNcJrrOhJlGKRfkZvbxmDJ02n8eGiNUEKnLcEz6E0YEIqA0reY/wQI4hijvMNQhdICjIxQ3pBnh6Bv6OxO0JQKDkBkWNDj8ASvCLTFSEMjHZAyCla5lhqgrTgYhtxmc1J9QykxOkRHpArTpDIAilzBtFErpGY4rxAyZxERqemciVKzHAYtByQWuBcTwgJUiZIURIkCJFjbIjiYOLwoQAUiVKUeYE1/YeLyMoeOd8TpCewQyczhJ/g/RZweJPggkMlDiEGvPUINFKYeEMLBSokICKT2VmHQKLTHK0TnO8wvqUQFdfu31GHDcnwb2h2HfgMrTRl/hif1rh+hxA9mZqhRIUXzQHZEtA6IxEjkBxiOhDEhhAkSkiQkWEtpcCGEa2WEOJmB2nR1YaszclmEzwpu92KVy+/5KPP/y8U6ikb/w7nByTtYdqkPmiKTkReuSDy/6QIaJHEyd4hJvJQ7CnRCBHjqrEEK16ALnSH/78AEZ2I3qsYd0ITvAYU/gFnLjxCKZyPC3wdTMQWhchb95iYjEBHvAYaSRofIoQfciiHSXG8uC2gIChCGOMN4A842qYnn2cxSpOD8p7e7FCJxdCicsX93ZqBkW5oURKOyxnWdLSjoOn2XCwfcfX2inYUTJdzsiylFBPm8wWDH1G5oh9b8lwxDi3GDoxdx2hatluP8TMix02hdfZhoLDf7al0LAiVztJtN8gk5Wh6ilKBpr2nPFoyjnEIVyQVxnnyrOR0MqNKUhLhabc12+3A8alGpZAVGiEG6t7SNwNVFlBqwNJH/nHbEWwsIJvNlzw6LujrNblOCB6kz8jUhMUM6u6eutnQm0BZTmj3O1IhaFJNKsCF+DD1KM6OLwhiwA+BsR5IhSRRKfO0xAfoupGh3iOkpyhjXL7ZGQYXsMaSHAr8ZJhxevycP/n5Lw7s97ig+odTfw5rrIdx9eG/OkToPLDd7njz5g13d7dAYDIpyfOEut5E1lmSgJR0XXRvBRxNvQcfSJKM9WbL2VlL07TkWcFsesTGraL44R37/Q6tCrquYzZbMpnMyNIcIXSMuwuHloq8SNEa3l++IcsOpZWH82AYFd5DWUyYTeYoCe3+NdvdJi4+vGQ6z2A9cn/TRb6nUOR5gVIKZy1SEdETYULfj1ycPcJbz9A3eOcp8ym5rsAJemd49uRT3D4wjpYsyzE2buJ39ZaqKtk3XRzsDoFleYIbLZNqyr5pabuWbbsB7WhsQzfsub27xNqB2WyONR5P4PL9Je1YE9TI0HdcXb9lWs4YWsusKJnNj/j1610c5v2ex3RSUhSa3W5L18XS1rKoMENN1w2cnp2wr2+5urri009+zLt317x9847Hj88pyoLJtKLvug8N733fkSSKtt5TZDl5mrCqt/RdjRLQd3u6tuZ4uSTPNG2zI8sV3kdE2NFyyTgahAi8ePEIY0YWiyldVyMFpKlEqoS62bPbW9I0RSm4vb0i0RllmbDdeNp+x5s337NYzMnzgqosyPKU77/7jn/9r/81L1++pO4aTi7OGL3l8eNH5JschMcYi/eWu7sNi+Uc5xx937HZxCHS48dnZHnC7e01Sil2+xVHRzP2+xohLcvlMUkKTd1QFJqL4pR+rEmSiLxIs5TJpGLsPHg4Oj5luZxyv7omzWPZ4e31HfPZjNGNdLuGsemYLyp29YajxYLp8ohm3PP1l1+TlIL77Q3d2CAU9JuBxfLogxvbucBiMUfLEqUU682GcRzJi5TPzz7m1cs3aC04OZmTFwnV9ISyKiO2aTZnu90CcHx8wmQyBQR93yOlxnvPdrvn7OyE84tT7u6uefPmLdYZzs/PUKpgvbmPbjmlWa+2OOcpixlZVjKpKowX3N1tMGbAWocQKYKSvhuRJpBnGcvjJUNv6PuRth1YrVakacIw9lxevWM2m9DUluALZvM5icoYXENVnFDmA/f3K5R0HB89xo4JZtzF0ndnEEphbYzYpmmM+u53W5RUjMPAfl/Tti0XFxdst1t++9vfAoJJNUFryep+TdvtcTYu/oehpywrlEy4u12R5wVSaNq2w5g9k0n8fU1Tk2VxOFvX9SG269luIyM2z8sPcdjf9zhdnIGVFFlJluU8e/GIy5sbetsh00CRZWSNZhgsWanQWY73nrppMN1INwzkeUnb9ex3DU0z8uzpU2azY+6ub2LHi0jpWkuRZCRFhhM9SkQciAyCzbZGy4TZZE7fD2RaojCYfktXj9xcXYEbUMDJcsZRXnJ+dsKPPv8MROCLr7/m7v4tJxdbBjMiteTm5jXWG6aTEjf29GZLEVKqfIJDMF8WlFXC6dkRSSqp6z11s6NvHaeLc6oyjRt5D9vdPiIwtSdNBToBa010qCnN2fk5Y2epphPm8yPWu00s1CozJvMpOs0QBJyV1LuO7V1LuzV0dUNZZrzfbUmLDDE6wqZhnhWwnPPy9h0yjFFAL+fsmx2mN2zWG8qy5GR5TD1ywCylTKqKRErGvidJLUGLWOgpNd3Q03Y9OlEEkaBE7ALJEs1kmsdC3Kwgy3K8jGxXgST4mLyQSI6WS+rthqoq6Lo9dmio1wNmJ1h7R3pArjnhCVKQ51NKAy6scE5AUEjiZtl7j7E2uuF0FDf63pJOE8bRsN1uybKM6XQat7Mh0A49k6KkKAt26wYjHHa0lFWCxNHblsmkxNQDWZ6gREASmFY5beOwzrPrGmZnS8wdjH5k6B3G9Qx9NMAIxCFBm6ClosgLhFIorxmDw/ZtfFaonLwqGLzBa4+WmmAHvHekmUZlmlxlrNa3GG/Iq4okyyhnBU2946vvv+JP/vSP2Q9b+qEluAhH2O1qlF6xXB4z2i12/P2v78vLd6RpTjUpuLvZcHpeIIQjy1PyRHG/a3AmMCkU+xqKLMM7hV6m3F3vQAjm8wVdbfDGs9v3LJcaraAfHM4ItrsdWRILza9vGlIlePnyb/jX/7uf8+/+/d9xc/2Ok5MjBjOQ6znObaLhpUkYesdoBoaxYbttmU3L+IwUIxdnF/gAR0fbD3zim9sr2n5ktTH87KefYo3l+ccn/PXf/SeGwZIWmrreo1VKXky4ublmvpigVErwcejfNA27bUNVZiSiYFoecewD9De8ffeaqkx5/vwpdTvndnXHMIycnp+hg0LYDBVyfv7Zn/L67Ttst+LZxQXaJ8wnBfPJjOV0QSVLfvzjzwmjIw8pTgVMiEhZg8P4kav7d+zMjv/y9/+Bj56ekpSAGvntq695tnzEf/zykp0ZybKKi/Nj3t/eoazh7v6WdVeTTzMWhSRVBa9fvcIOIzjBjz57wb/88/8Nz549RypFmuZxCX9wqKZpGg1z3gORBz5NZXyG94FJUSLw1M0OlUGza1i1W7rQc1s3HE9BakPT7UjTlOPjY969e09VpZydBbwtQAak77lfdbT1lOX8GCkcZTZnt7K8fbVhUqbU3T4iwkTF8fEpo+nZ1R15OUdpTZJqoEGFgBsVmoI8T/jv/tUzfvXLX5Gmjr6rOT45IkkV09mUIksQXtLua+6v12S65MmjZ3z36huGYSDNFEmqGXrDX/3VrzlbzhgHwenpCVWZc3t1TZFrLo6e03Ydd/f3NL1nvpyQpAk6TXEu8O031zw6y8iKkumsZD63OBcomoSiyMl0iis9H794gekGTo6PuHz7Dik8T5+cszdbzH9bY/tHD6FEvD8HF9XnAEKKDyIfHNAVRFdyLEP8YR/ngzi4ZX9wEkf3NzhvPziLtVYgJP4gqocHPKwgdhwiY2LfHTjmEKkwwIPVOQrr8Zw7vLPDfh28dx9MWQ+loA+i/AcnuhAQ5MEF/4PzmIPnWAkO708eEFjRgHmAwBwwH/IgqHs8A4OJGGXrKqTK8C6JeytrDtzvaFxI07gGexCvIzYmTkpDACkt3gsOJnIe2OPeu4ORKwE8+31NkugopnvLOPYMQ0eWZR9c933f0fUdSZIgRERmTqUC1OFnPSQCAuNBRLfWHBjsEeNsTCwkRqnDIIIPn688fNcfCmEPrxu/pphEOD4+5i//8i958+YNwcU16/vruJdJDma7JEliElNrmqZByhTnHEdHRx8Y6CFEs8k4juR5zjAM9H1P0zQ0TUNd14dhVhT48zwFAn1fY4zGOk01heWyYrEsWSwlIgzxnPcP6Qc+cN2F+IHT7j+gWA6YooNmoaT+geeOPJgEH9zjcegjD5zaEMJhmPOASz4MdEKIvXUP8tqBry7QCGEOZ/cPf/3Hjn+y8qYOjbqjtVg0SuZkOkOQMTiHUIIqeQS+RyKwTCAEnIgfqhQZiTpl8II+WJSoo8PXCkTIsd6TiGO8eI9UPYMrsXxLogq0T7C+j1x1KswoCLKh0BWj6UiR4GcMIvJ0hY+uJ0aPdyMGh1ZzBrOiHwKDNzgrqFKN8zYGB4JCSosQKb3pY8ml0oRgCD7Bjgq0R4QURcnoAp4u1kkqiwkd3g3k+hTBEuN2JEohpccGi/VrtATnh/hlhZxAivEB5ztU0GQqw3mJxSC0Q4YcwRKERSuHFBVBlR/iMt65w21GgARftVSLHiEd3t8x9B7nNEKE2ESsDM4MSK+iw12OBFKE14ze4MYUKbJYtqgjm9r7OAWVMgq8jbmiTCsW8s+5bv5n+u4+uuj6PZv6kuXiLzC0eC/ReoHWKT60eN+iZBq/H6lQMo2CtcrwIZCo7IAGytAqFgNYL/BhQMrZ4QY7IhAkkxHqnrKaYU2BHSVN4/BDwSz9U9b9V5iwJVGRUxycQonywO9vCT5OxgUZSuTxYSFiqZlSKZ4eHwQhpEji5tiHNqYSQof1DcgDt+kgeguRxyEMDlBxIKGy6NCjIwgLIZaNumCILRoejwMhUSIhYBEiRn3itDZBkmB8fXDzS3zoDtEUAYyHstFYVvmHHFVeQvDIVCGSQJEk2LolQTIODUle0I4NJAqRSKQH7SQpCUM3UKUFSsBiPuFudclkPmUxP8LeS+bVMWmV4LTlt99/xcn8GCMyrq8ukSLw9PEjhMpZrbdoFBfHZ3zy0QvSMk53211HushIdcrt+zsSmTGZL/jJpz9mvb1laLckpExmBdvthrPlOeP1SLutyaucyewYN7acnc4oC8/tbk0bNmSpJE8TNnvLpFhy01xjjeHk+AiEZzqdY0x0RZ4dn3J59R7TGxbLBZOiZFLMaHYNzo+kSpNnGb0ZGIeBxWTG2fIY03Q044A1ltnUM4yOMp+wb+M5sNvWFNoxLSZM9ITejgx+II7HoR8GRjfQNXB9u0XlCV0jaH1gnpb883/2LyjLIrajhw/0r38gpIvDgyoc3ALhQ0FLwIyG25tb1us1SZJwenrMxcUpTbvj+uaa7WYdC/3yEhcCzhr6sUaE6KJtuza6sIaO/T6KUmUxoakbtusVSRLF7O12e3jw1izmJxRFRV7EzYF1FuMH7lY3VJOKPE/Z72/Y7T1lEYtK8qzAe2hag3carae8ePI5t8UVq817XLsnL3L2uxuWy3Ok0jRty2S24G51TwiBMitIE8VyPkednHD1/j1JIvBuINUZk3LBUI/Udc9icU7XCio1xQvHan3HbHqCdQI7ish/T2LvxdB1mMQyW8RYedu1ZJOcxdGCfbuOi7gksrd/+93XPHv2gvl8yfnjMy7fX4F3JKmidw5nDfd3dyQq5+x4iR1GzqYniPPf/9oehpazszNW93fkec5ifkqelnTaklQ5u82eLE/JMsXXX32DQJMmiizN2G22bNcbsiw7oIYC9W6PEor5dMp+t8XbglRLZPCYoePi/JRJWXxgJpZlQVZoNusNiY5lWVkqsNbz+OKCX/3q1xRFxeNH5/zq7pZ3b/d89PEzVvd3VOWcxXzKanWPGTuePnlO2/V89tkLbm7uaZstXVczmU4P3L6B+WLGF1/+htPTU16/fs10OuX5R88ghNiHgOf16xgJds4xDpZ3b9+zWCzw3pPnGaenx0gFb9++oigKNpsNR0dHTGfRVbJcTtjtVlxdXyGE5OnT5wxj5PNNpxWL+ZyqmmELmEwWXFw8oq43vH2/I81zqqKka0vqZsfN3TV2HCiynOv1DXYceX70jOPlkpdvvuPN1feUkxydalSqkEqitKYfRozZMJ0tOTm5iM8Xr2naLg7asliabOxAUaWcnh+jVEJeZpycnJIkCefnF8xmc96/vyTLMibVhKqastvtY6lfnjOMA11bkyQQrKeu9zF26nzknrc1XTswnU6xpme7aVgsjjg5WlJWBZPJhLodGc1A33cYYyHkTMqS5fKEd+9f463g+EjhnKFrO6RUFEWO0pK+j2zuskqxXpHnC87PXzCOln3tKMtjFkvHat0BOSqpmC8C6/sW6waSRGKMQ0lBXTecnJxhbWAYLM4aLi8vAQ588pT9fkfXdRwdHdE0cTMwDCPb7Z6+b5jPj5hMZpEPn+Tsdg1N03F6ch5duolHa8Vicca3330Thfxg2e9ayqJCSk/TtDRNw3Tq/8GG+fc5jhcn+MExtAPv373lo48/RqSKZuioZgUqKLKtou1GmnbHbD5lMpkxGQraLpYG6kxTzSbsm4G67Xn5+i3Pnz9DsInYrRFMcExVQpGVhKE7rJehbTvs6OjauNEKHrwd6ZsBXELb7Om7DVVRYo3n9PgIbSVZkjObnLDZ7mm7wOL4lGo+QUhH33ZcXl1T13tOTo85Oloiidif5dEpQgTSTBNwzGeT6P7qLf3Qk+eKyWzCq9evsL5ns98xjDA7LuPaOBgmk5Sx03RdTzmdQZCMpmfse8pyzvXuCiFhPp+R5Ql935KolC9+8w2zYs67VzfU6z2LyQSRKfrRgJAczSrGzZ4kyXDe8fr2BtM2zKYz6rYnQTPJJkyKnKqs8EIxnxhEEGRpQpYmKCFpEFjr8E4wmS1JsozCGJK2JSBIsxwXPCLZU2YF02qKUhqVZvTGIvWBk+ti2d5oAsIHvJCARyuBlgHX9/TdgHGWIklRacZsPmcUksHZWOZ4coZ4/Z6uNaCjOaQfB7yPyWTBg/tQkaYBlcRCUO0Vx6cnhzLdhmo6wXjLOHQUmaIsMpptQ1s3SJGQlzlpprDOUNct08kEM45kacGzZxds1ls22w2jtchUMT+d0JqO9WpDlZUMTUOWRTeeQOKsZXW/RsksFrqJENFWiUYkCqk0gzXc3t4ilaQsKjItqfIpaSIjRiDJUYCzI/XOkk8mVFVFVqUELVhvt8znJ+zefB+FeiEJ0rO6W5OlJcGLOHz4PY+221PJglcvr5EqZTrLWa/XXF2OfPbRIx5dTGkbx35fk59Ek44xljSZ8NlnP+Hv/vbv6Oo1RVmQKljMQAnFdt9yfb3m048/RYlb3r69Z3kCk0nG9fuGs6PA3Vrw3//lR3z1zau4rr54wi9/9R1PnmlWqw2KCUmSEQ7oUmNa7AhaZyyXJavVBikTnAuUZckwdozGkGVx7/Orv/+Kzz99znJxRNM47u5veTRfUlVwf7/j0eNzXr9ecX5xwrt3NyiRYI2lOdxHUzUhDJL/+su/4elnjzi/OGXsB6y1fP/9S55/9IyqmtD1HQGwRrC7r/nf/qt/zl/8q/+B//nf/lt+9Mkf8T/8q79AG/9hL+3GkWA9VZIyuoF3r19yu93w45//iLapESIwGMv9ds+oel7ef8m3b/6K2aLEKoNLAr99/4Z8PuX+lUFpR5IWfPPmNfvNBp1oVKJxvieoASssL350xOZ2hw6aTz95zvMnT0mTlCTJ0Doh0TrqKsYQPCiVoDRYO+I9ZFlJXhjev3/HRq4JwvL67VuCsKTTjLvmFlEI/vwvf8L9dsWkuuDdu0uEUmz3NUqnCC2oqoqTowvabsSFjqfPTunHlvNHi1iya2F1e8uzi8+BDltl9N3It1+tkCLjxz99wf1qRb03BBxSDtzf38WE34sj/uo//5Ifffacj579iP/z/+kFu917NrsrBmfZNltQCqk1prXks5wnTy64W+/xWB4/OWO9uaEsc969e41zkl/82ceYzvF3f/clJxcvMKZHKU2RlrS15+TkHLTmerUjy1OEhCzNWd1smKSCV682nJ5VKCk5O32EHQ2314KqLHh3c0W2yPjqy+/4/JNnXF6/5/3l24g404pqUVBVf9i+WyoZVQAbEz1SSLTUkQ9NIPjIqfYuctJjAemhh80/YF4l6oDBjGK1xHsObtroqlY6stN/4Iu7g3gsD68lED4iQR5SgAr1wWz1Q7LZHxL3D2WPB4fv4bUeSuFjSeSDQP4g+P6gmUTETOyP8wfjYpAhdrMFDmKqRMmoPT0gbqMTXxC8I2AI3iCdwtiWLFmgtGZsR4a+x9qohQmdfBDytU4+DBsiqjey0a01ByH6wREdE9rWxuLMptnT9z3OOYahQeuErq8Zx55x7A8Ctj+gW3q6riEcujT6fiDPYwfjQyeLOojjzlkQh046OHDD02j6SgQPPXxSHXAyh8/NPlBGHgYcH/jmHmstm82Gzz//nH/+z/4Zf/1f/gopDwP0pse3LdZanHNMJhPKskQIwX6/pyxLzs7OSNOU1WqFlPJDinI+n3Nzc0Pbttzf3zOdTg9GG4Vz8bNLkpQ0jbg760barmV5PCPJAkpbVKIo0jziFH8H9fMD0vCBxHAQyB/6Jh8c5EqidETuxELdKKJHZrrHPQjkyMN5FIcvkWYRUwYigCR2SsaPMBqjOZxjIiikcIQDV/2/dfyTRXQfPIlKyUjJlCSybBzG78iVQ7BAiwqpBFocg09wbLBhZLABzw58RpCBRKdIJkQWuCCTJYSO4AVCVXhSUp2hafDEyIhxHbiUzhuk1AQf2NgbFBn9uCLTjwgYbLDAgLADWs3wwqCFIeESUkvwE6RfgIQiMwyjjbFTkWFdQGDQIiDUBhs0UpRIkaNVhXF7FHESkiiHpMeFln1/Q6qOKBKFCB0BR5I4CnXEaG4JPqASjXcdUgacFwgSBtOjlEVxjPcNnR9jsYSMN0nrN0iRoMQCY1oypUgOX+rDxRJPRIuUgWzSIPUe50YUijTJGKUilkxEodyG6OCPSJ57vF+S8wjjA0ko0UIxHk5uqQ7tyGbAO4NUCYTAXfMl0+onLJL/iZ27J4QOH0Y8LV4ONOE1qS7ROsG4FT7sSVWMiShhDkUACufrA/c+w3gLYU+WTFHEMj+IBXQCgwgKLROC3KGyPcwr0taS5xU+nKJVQjdYjss/5Zr/jBTvkUic3+Fchw2Q6gRwURxXsdjO4pFeYKnxzqFkij9gUxIVgAxCifURZyFEGblKNLgwxqGA0EgR/0wCiSc6/eO01EAYEMJFfr30hKB44JiJkBCCjfUeIj08rB6mxSMBgxAWQrzmQCBFFILi4yQBNCH8t9lN/9jx5OyC12++p2ssZuyjo3gyI0kS1vt7ru9u6XvDZLqgyidkXrIslyRe8F5eM80rNvf3YAfOj07IdUq768j9DI3C9APpJCERCtOO5GmFMHET3O8GZosJy8mS0Y0MbUcqFLlMaLqajAzhFVqnpGmG8JoszRj7hmBHZAiMbQ82YVHFgp5dvkGLnFxmaDSjDRgLxgWM9bRjTzkpMb5jVw/0DZyenDH2O0bXsm1qZtNjlken9H3P/f36sI5IkWJCnlc4O9I1W4wdcLYnKTSBwHq95un5IyZZQWMETdNSlBXNrmMwHsuI9YLv3r+hzHOsgNV2i3cemSiCFQzeMHQD+SRl1zR0WwW+4tnjH2E6x/HiU/7NX/zv+dFPPv+QUvj/b0SPhyQ+4H4YvsSHTiDyrne7Pd7Dcrnk+fNnjGbg+vo996t7xr6Lk3IzRuSQigs27x1917Kvt+R5xXZ3R1mUAORZxWy2oGs27Hdr5os5o+lpmja614xhOl1QDDllWSBkLDRMdCw0rKopXXePD4H15haBoKwq1usty8WCeXWKsOClZz5fUE012+171nc3LCZLqqIiLyuaruf7Vy8J3nN+corW8X7WNDu0KFkujlFq5PrqLbXVZGqCNz2nx0tmp2e0W8NkNqVuap69eM7gd9gBQirJZEbX9ZjRkKuS+WTB26s3XN/e4REM48DxYsF660iE5uLJc4QQbPY7js6OY2mdCzx//gjBI9quY5dPOT8549d//2tOjs85Pjvmzct33F/ecrRY/N7X9tXVPbPZLLrBDpgqd0gELRdHbHe3JJljGPfcXN9yevqI6XROXbek6YbZbEGaJljryLIsulj3e5yzzGcTVve37Pe7mLLRku36HknA2gElYynY6n7FfLlAKcV+X3Nycsp0Msd7z+npCVprjo+XnJxG4dJ7T5LExdjd3R1ZFkt53r57xWKxJC8UZZUw9HHxO5tO2G7XPH/+lFevXrHf1+x2W6wzHB0v2e62VGVFUeScnp7w9dff4Jzjo48+pm07tts9z5494ejoCO893333LUI+OF8EWZZwefkuMnjLktG07Os9YBhHS9+3pCrneHFKmsF8tsA6T17kLBZzVqtbrq7f4MKIlBlVVXB8vOC36ztW23u6uubs5BQXBqwxfPvmW+62M3rToTJwDMxmFYPr0TplMplyf7/F+8Bu16HknkTnCCGp65pnz56yWMywbuD+/p40k7z4+OmHTcnR8oggPEmSkaY5WZaT6ISjoyN2u+jKrqqCxXJ2YMBPcK7j9vae7XZ7cNlI7u6iA306WVDXNev1jvnsiKqcM58fI4SnaXpW6xUPXQzBC/puRPiex49eMI4jt7e3rFdbiqIkzwu01kxnUXDe7RucN7TtHik0Z2dPmE6WvHz5mu2mYRhcTLfp6AYa+pGiKHHTCjtuDygyyLIELcWhDMmTpglVVWHHEZ0q8jzh/v6W7XbHs2dP6fuezWZLVRUcHR8zDAP1vqbvB9JkoOtGxnELQVGWinG05HmB9571+v4Qm1ZU5Yyu6+L5aAxaJ2RZLBb13nN0tPy9r22Aruso85wAfPPNSx49eUJZVoyHDVoQnmpWkZVpjKsGj0wkx/MjTg8FW4lOyZKC6XzG3c2K6+t7rm7vaLoRjEMHz/L0lLLI6doGERzBW9o6Cq1pUoCXtHVP37fI3lNN0sP5BVWVMptMkSSx9MvCrm64ub+L3H8RePL8KSfn52y1Jsv2XF2/YzqNPTvPnj1jfNyx27d8/MlnAIdki8U5ExEnxZyszDk9OkcnsQDR+p6275GqwlqP92AOuMKiKBgGgxkN6/WheyQEjDUMZsB5y2RasDhe8PWX7/AGdpuGKpmiVEbT3PLo7DyKEUnK8ekZi8mMt9+/ZrfdI5XCesP7m1u2z7dUOmdSppydLAhegtB0JnC0OIqINiXpu4a2bQlCoJKUVEuyNEfpBKVTkrwiSInUsQAty0uC9SRJ8aGgW6gAUuJtIMgoUjhjEQHqektfb7FjTzA9YWjRfmSSZ1RZgtJxfzGtKpSxhDTBpymPnz7hi+9fYoJn6HoCFqljzF1qiQtRRCnLHJVoykncmA/jiD+4wLRSzBYLNtt79m3LbLKgqDxdO2L9iE4r0jyhafrY1aKiq9AMI0IHHp2dM59NeXX9jn7sOX9yTqpT7m9GOu/QKuIRCAERYOgMQSuSRJKkKe3YgRQUZXm4D43RQOU9qVDM0wKZRuH+eDGn3q8ZTU9vAkGAE556u8Nay/L4mGbf8P7ymuV0iZYR7YUXCC/QSnFzdU0+0fg/QESXKrDZ7vAu4Lzh++/3KJlQlSVfffGGaTFBoMjylIDBWYMI0DU1N1c3/OLP/pR//x/+35TlFJ0Ehn5AaYW1sLof+emPpthxS5ZqjHVMZxlZCaNxvH57iQ+az3/0MV989S1Nv2VxLD+sva3vKScpro8F9V0LYz+SF4FxsJyeHsfhU6J49+6aNBOs1yNta/n5H3+EFHvevduxWWmePHqKFBmOkTRdHAQhg9bqcF5ptDzse41i1Y0UeQHe8OSzE7KpwASDc3B5dYlAkqSaJI/DLJkE9usV/+Z/+B/56Uef8vUXv+Tt698yP50zNj/FtZCmGUVVYoBu7OhE3GMfnU/5v/4//+9cbd/yz/70F+ACRTUnSac433PfrVB2pFk1WKnYdvDd17d8/qLg8aeP2Kxv2LYbvHa0iSUEQe4dSYicf+MHbO84OzmhkCW/+KNfcDw7AevJ0uyQNPPoJKUoYnpbqQTjbEy4q5SuG5lOFxwdDfy7//T/4Wb3jtFYfvpHn3Fb33Czu4LUc7O9whpHaKDtLDc3DZ9/fsZnnz/j9euXVJOM/X6L8wU6FRwdVdgg8X6P85L12nF7vWGanzCZ57T9LZ88PkOzICtTvvjqGzbbgZ//8VPqukFKz/xoHnnXYuTJ4wVt2/LFr3/F8xcXPH9xgvVrrl5fM1kecbuumTw/o+l3rDZrJvkMhKRp16jE0/V7TtMZUnmSJMVax3SW84tffIxKI/bGjp5Hzz/mzfc33L7fEZKB5WRC4ywiKCZ5gRGWT56+4OXVa3Z1TXCWxxenzI5POJ5NafYNZZXx/dUbFJ6b62uOlguKSUHfRdF4uxoYhz8Mo+qFJ4iAIyJVlIrJeUcUsj2e0Y0ILxDqBzSL9w9rqQOaQsf12ENy3YffdYE/OH7l7zjHH5C0RFd7iOL4Q1o0/r4DMkQ86BUhYo2wH9bFhz8FD+J63Hs+pKIj8/xBlI44EfkDfSD8gIgRUoIfARuZ4Hg46IMhhNgNF/SH9+owJIlC6QTkwGhbrB3QlIfyT0CA8wbpJPbwnh/+zErZHxAvIQ4iHnjcD0iS0RiMMYch9MjQj/FzOuB3ttsVSRKxItYajIlu7Fg0aiIq8+FnjSMyS2LJr3pwUnPoHYvImodEopJRGLeBKDangVQl0TkPsc/MmqgZxbI65OH7OWzw2GzW/Id//x8AyPNDn9Fud8D4BLy3dJ1Ba8lsNuHi4oyuazg7O6Us8w/Dg/v7e+q6Zr/fs9lsPvDSHwT7h/NLyTQOrp2LOqqAECw68fzs55/z2ecXZGWNsWukmhyGPuIDRigm237QLh6SAP+Lxw8Eo/g6B+SL9R4X/OH79B/OO+DD4AQiNz2aaN0HF/vDUCiEw2uLiNwW/9j7OBz/9GJRWnzQkTPtOqTqopvHbciUIrDFhY8IYmQUl0zUY6yND+tUpSALvHkL8gwlFIQKQoZ3d7jEELxEq+j2hWMGZ9Cyx4oEH1IKHYscQwj09oY0KVEhpxm32DDifBQrfJCokJJlE4xzWBswLlBqgVYpIsSF6ehqRtuiZInxW4Iv0KJC6DuC0mgV8DYniBzrehLt0T5H4HDhini7mhBx3zMkJZ1ZkSgNtidJBMZ5pNBoZUG0CPpYhsoS57uIFBEBlUwipkUYwOFMj8cwWEWu5kgVxb7Rbek7R+l9vLikjCeNkIBlMhGgPdYJtHqBDAYpOhw1So5okaKExrk9LjhggpICqSypOSGV1YfvOzYZuwNCJXKXjO1RqsSw4uXu/8aJ+jfgY7wztg8XTJcLbs2/w4Y7BCUu1CSyIoSHWI6JhZ1MQCQxTm3XFHoaBy7jCiWJqBNf4jE47pCiwrkBoQKePWlyQ5o/YmJlLD/RBcYJylCRiClBJtiwRyDIkznWRkFGyIBUHoKN0Ty7x1oDIrIgrRtib7D0IAa87xB0sZhUGKRKgRgt9DzwwxxBjEgRMS4CEd+3byO7Dh9fVyTxDhBicYbHIYLGhoA7fI3xmn9oKY4FWAKJFx4fDCLoOEQKD3Gsw8PtD4yV+dGwmE65vHrFbFESXODs+Jy+G1jZe/qmRaiMRKc8OnuM2bZUScV0mXPTrFhOT9leb8jTgiLX9L2lqXsuLh6TpCCCi5PhMHB/U3Nxes4sLyiKyMgejCPXCXmasQp3FDrDtoZKTzg6OWbsDfUuJhykENTNHtd1KA1PLs7JsoSmbdlvap4/e04WEmblJJb9eIExDp0HnBBAyjRZkHqB1hk324bZfM7p5BG7ILjZX3K7XiNUAaKgaTu6bsukSvBBU1anKGV5//ZblHB0dU2W5OAEt/e3kY9ezaDzjK0hUyW2j13aaZLRtz1eSQYTh4PXzR3TYso8XzJflKRp4GZ9jVMOWabINMf5wPOnP+EXf/KXPD79iB89/pgnF2fxvMAyjP5D9E5+iNPFI4TwYeQSHkJKIeC8Z7ffMwwj08mE5dGMm5tr7u6v2WxWtF2DlpAmWWz5TlIylZHnGdud4eb2mvX2jmfPXrDZ3FEUJaMxKJkzncw5PbmITgoXh3xCOFbrmxhpCxbno+swTuINSZJgR8dyecJ685btak3T7FksZuz3K+pmzfvLbzk7umA5P2Uw/sAq9cymF+RqwnJe8v7u1YHtXPHdq5cIAY/Ozzg9PuLu7pLgPavVhulkQttfY8YeJSYcHx1xtFRc1beMZkMQkiAHlicl05Ocv/3b7yj1gt4M3L6/ZzpNaTd7nj5+znx2xPfvfk3dbzk+PmO33xJwuMHx5OIZzli++OoLqsWUuqvJq4xgLRrB9bs7zo6fEjLPrJxTFhWT2YzffPkbmnpgNl/w8cef/d7X9nI5Jcuio2noDWa0lLmkLCcURYmxE27uX2NtR987bm5ueP78OVVVsdvtyLKci4tHrFYrhmGIWJ75DNO3aJmQ5xl1LZhOJ6zXK9brFft6R9vWnJ2dIYib91Rn6CRl9nhJluXU+5oXL15AgP1+z2634fGjC3bbLTc31zy6eETT7gkSfvbTn2Cs4ebmiumkYBxaqirFjD0CzX6/o5pMaJqaNE04OT0mSRLSOmFf7w8sast+u+Xi4pyTk2M2m91BsE2p6z03N3fkeYZSEmMsWZZyfn7BanWPlJKiKCiKDCkDXR+xLkWZ0N7veP/uPcv5OdNqjpAmipKZ4uQ03kOvry8ZhhalwPnxgCpK6LpHeO8wpqc3PdVsybbe8t2b73nx4jm7ek037Cl0xv36jrbtcBb6wTIMljzLSZP8kIaKfSZ1W9MNHde3bRyCB0Pbdugk5aMXL2LaygXqtkEKxWYdB3gikVTV5BCljQWfSSIPQ16HUpLz8zO22y3jOMbzaRhp246qqphOlvSdRauEPHvgU1rqZsX9ZkUQ5uBgl2SpJE1y8rzi6ZOP6NqRcXRMp2l8phFIM8Vmcx+FmFSyr7ecn3/Cz372c3a7Pev1htFYhrFnGAfqtqaaVCADWZYSqoIm1Nze3jCbLZnPj9E6ZRjjBnEymZFlGca0sRiuzNhs7+n7kZ8++il9P3J3d0dVlSzmR/R9R9t2DP3IkA3M53Mu398yDgNFPonOXBHdM8YYhmHg0aMLFvPIpuy6GPPVyqGU4aHw6/b29g95dLOuVxyffcriaM5vv/mOm6trgvPs1zVdM5CmOgp/ywWz+Yw00Qxjj1BQzQvu7u5xgNAZT54/Znl0hM4Sbq7vCNKBcFRVyvn5EYNruL95x/GsJE1iUaFzHmsGvIWyKtAqwfoOoQR5mZGlJVIJvPMslkuSoqSuGzbdlsv1a0bfI5IRZEKSZEiZM51KHj8+J08Ljo8ec3J+xnp3xd3mjm19T5IkBzRP/LNZH3nVVVUwmVTsNy1pnrJfrUnTjNECJChZEFygbTqKzBF5oAazNxzPj0mz7INLbhh6+qFltwfPwP3qHi1K1rsVT188o+0aRK5ox5oBiwFClkCRxqSnhyrPwMdEZ5GnlEWGVoBWDDZggydPc6RSbLebmHYYOhKtSIucYAPBjAgpKasJaVmi0hx7YHnWdU5XtwQk1sXNcQiQ6IOgZMIhAi3B+ZgQCgaRpbQbh9ApmZRkWiKlQOcFXmuyLCOrSu7bBp1pTs5OePHiKW8vr2jaJiIKw0OXUHy9hyi5cJI0S4HIIE3TBCkytrstRyenrOs9gzMEJVG5ploUKG2p5hmdMQQhKLISJVOGYcRpT93uMV1MAf7os0/Z9OuY9CXgzECapAgl6IeB4GBSTJjPFvTtgEQwGMOkqiIm00UHoJCKsipIpML1BmsCwRlkEAQDk3xKMp2jxJo319fkswmkKUPv0NMUZxv2ux3NumM2r1BC4YPB2hGlIr7Jy5J/wj78Hz3KoqCuB7JUYU3LdFpR7wxPzx+zursnS2XkcAtJ1wcaM9APht9+8z11vY7FeWFAiB6deorCMZvnXL631HvHbFLy/p1F54qA4Oa6Z5ZOKHLFF19e8Sf5jOcvnvLbb7/HY0nSOdY07PcNRyczJmnG0Ce0+5HZbMF8mXN/d4/38MUXN3zy6QwhPJt1DyEnzzR//8v3PLqIz+i+g7/6L1/wk5/+mDTZUtcNT58+ZjQ9Wa7ZbXekSUpb1xwfzcnSAp3NsM7Qdbe8+PFjvvn2G9K0ZHmUIULGcrEkEGjbAZ0pbm/eMi0yXr76gkopvv3mO4RqaduOr7/6O/70xZ+wqHJu7655++5NxKyZkaPjI+7uL/n5Lz7nP/ynv6IoC5pdh64mvH93A4tAa0ZKkcRrEImhY3l+xq++vuanf+o5eTSnHtekE0V1mkWB32kUjtFIrIf9rqMSnv/j/+F/5MnJC3KdIjMXcVmHNb0I8bkohKDvRwYzRqNBcAyjpe16pvMjPv7RZ/zN/+NvSDPJX//6bxisgTSW8AUp2O5qZtPYZ7U8ykjSnO++f8lsVvL+9RU//+nP6IaGrnes19eUs5Smq+nHPduN5PmzY95fvuRCpFzfv2Mct0znj0mykk+XT9jtNxjbsNnesTyagnTcrdfU9Zaf/vxj7m72zCcFV9evuLn7imoicFhW2zWzRU4/GD769FNsZ5lkFV999wrZOZ4+O2c+r3j9+nuyrGS92nF28oSyTBhHwfur93z2+U/QsuM3v/kO32smlWbXbFiePWLbtswWS5yUhKmIa8HHx9zcvWccAl9/8wWff/QZwgm+++13dHagcSNVVVIWJft6z3a/xzlHkiQEpzlZXPxB17b1h0G3PKBMEHgRDiIfBBHwIkTc6wPW9aAPhGAj5sNLhA9IoQ5CYSxN/F1TlXM/4FYexE/ggAUJB366PTjKXUTHeIDkgFERB8f4Q1/cDyWND//84GB/EC7/f7GiDwWp6qAXPSBJ4q89iNPRvS6litqUjE5iQWS6ew8+WOKOVkTjrO3pXUMmOmQ6R0pNmqQEMgYzxvT/ByxL5MLH1FTKA/pGCnEYPNgPqLJxHDEmYmGMMRgTP588z2maPcPYI1WOsSNCxrJRcxgaR867x9i4Tu77niwtGIYRpQ7nTwgfcH4PzvckSSKGRko4uPnjZxXRN2mafcDEPKwfxaEY9eGz11oz9AN/+7d/ezDaxoTJ6Cxt23/4zMdxZBxH6npPVVU8fvyYNE1o25ZhGDAmGpiGYcA5FztNlGK32/HAQo+vlyJFEjE3h/NLa0lepEymCY+fnJJmgnKSgigJ9kEDi+MXEcSHc0k8OMxFxAs9DF+EiBBmiIZu4Q8ah4hcI+8cPngs/ncc6PFnSSnwgAs/lOsmIqAeOP6/83z+gHdBgvhhCPKPHf9kEd25HKF7vNB0zpBLgZRLgh1phpEsPcb6K1qzpZAVVt/FiYpKqcdbtJIk4YQQNnjb4VgTmJCojBAsoy0haAgazzVC39K5KUKUeCTOTejZgnjH6CzGnZJnDq0swTkyHRA+Tu6krGLnhhpJpUah0Tq6mXSyIfhYMOf8gAs9SsaFs3F7hE1xXuPMBUEIhOwIfqAzl0zSj/GMEJZ4MeJCjvc1VXpKphfUpkAIj9Y9gT2Qkco5o+9RcoFQNdZOGcKeIknoHdRWM9V7lCqQEoy1KFFiQk2hliTJQJ5o5GjwYaTIn8AOwkE1dd7HSKUWOFVDsLjgGPr30cEtM0YrDhzQlhBqhNQEnzKYNWWuQO/QGBJ3HssMDvc+7+MJWeQVUmq00IgAVfIJ3XAX29GtwdiezuyYLI/x5WuC2yIB7yGVCwTRRavkgJMDxjZkcsYwWiSehCOsSUAoRmujqyMMePoYix09Sh08+AEkEsNAogJ5Ghi1x7iR1XbNfHEGYqRzN1gXi3R0OgUxIkOBElOMv8H7beSgA0GMCOJUM/j4s1SQWAdBDCix5+FaE4epnySPNzmiAzwK6A8FBYe0gUgIPrYhy3CIbgWHd+HAsfJ4bxAiQUh/QLzE+E4sNR1jiaiQeCcIxBIS5wVBGPzDQ0vGiO4fciyrCZqBWxxH0wmjcTEWNQzYwTMvj9k3PcoKpnnBdtdyv7nnzZuXhFJw9GyGqxtyrVhvVswXJzz7/FOOFqfc391gbM/tzRWIGKe+X91SJAUKSZJnrPsNzx49ZbfboAi4ccBjOT07wjnP/W5LIgpmsyPmVYXxLT7RDE3NyWxGPw5s+oFhtNxd3zCp5giluLp6R9s2lFWBF4KsWlLlBm8cuZAIGxi2I4vHc4QJtLuWtu8xzpOVFebA/RzGEetblvML8irBdB3eDqRakiAQNuBGRz0OKC25Wt+So5FGspyeYMaATkvSKqc2Dfu+QZQZm80aBYzGMj1dADB0A33X04w9XqYU1ZynTx/zF//if+InP/5jjqbHLLMK4cAHxziOCB8faFKp6Eb7HSe6C4KHnVxceByYeS5G8X3w9EPHb754w3Z7T9ftads9zhsmVUEIhjTJSZQAr3GDp8wmOBPQiaIoM7abhrreMPQGLUvafcvR0Yzl0fJQbAW77TXGBBI94Pwa4wfyIqXe14zGcnp2HhvZvWAynZBlCe2+QwpJFxqOF5qm3pMWis3+lqwosXYfr2Oj8Abubne0zYB1K4Zh5OTolEQrhn7k+vaWpm6YTBes6hX3uzseXcxJfEFdd/Sh5+3lmkEYrFBkyZym27Padnzx3S19s+fFT56wXfVMspJ9e4dD0IwD9y+/4/XldyRpxlSc8Ozjjzg7Oeb199+Ras23X39DnmQI70iUoq07lBAkRYZ1jnF0LGZHXL654WhxzpMnL2IkXO2ZTBY0Xft7X9tSpsymS6x9Q1lOWSyO6LsR7w3fre4JjLF0836PALIkoWn2zGdTFosFzjlW9xvyokQIxfXNDVLBYjKB4MiKgpPTE1br+1jE6h2L5YIszwgEur5DygRrYTqp6LoWZ+Pi/ovf/IYsz/nJT37C9999HwedeGbTCYlWFFnOYrng6vKG05Nj6n1LVUwQGurdDq2jU2Y2m4AI9EPPZnvPcrnkzbu3nJ6csdncx0LjXhCE5+7+hul0ijGO1WqNUnFD2bZ7drtwcJEkLBZPkVKxWCwPscY7hIhOjP3exgG9MXBw05RljlIZxnY465jPF6Q64a6+ZzKtSLLAat2SZyXWOk5PTkA4fvOb35BkEqEcTbfDuJHZ4ojZYs6mXuGDpe8d3kuGwdF1Dms0SmUcL6eEILi7u+P4+BSUQGmJkHB1cwUiDrGEEnRDx2qz4rNPf8R2exieTedsNpsowLvIfDw7OyHLUq6vrz/wJEfTI5XnaLlkcbSkbTqUSmibFikSvJcEr6jK+YHtPMd5i3Ox48EaUFpF/F8/IvGEUvDm9WuyPOLjAPp+ZBwHtJYHDv4NYPjo/AlFXvLJJ58zny94f/mevExAlFzfvMeMcfMjlaBp9yjhGLoe56KrcjpbMJ3NcB6GsUUKgVYJ4xALwJWK3RN13TCbLvDOkyjN0XLJ2ekZKpHs64rJZErT7JByTt939ENP3xnSfc3R0SlKRbdVWcZNt/N5xKQpzWJ5dCgBHNhsNlFA6nt2u93vfW0DVPOStEyYLacRM7TdURQTJtmUdb1hGEUsfEwTpFYcHx9TlBUqEUxmKde3A9dXV8igmU2XzKdHfPr5x8zmU+4urzF1xyefPePo0ZL3VzsmkwSlHVqlNHWDcwq8ZFLMSHVkhxsJ/ThivSVBMlvM2K0H9k1HMXXs7YgvAr3Y0todrRmwQ8tT+xl5MUOJjtPTE2zvEF4ydIZ9vacdtrx68xu01qRpwtHxcSy09B3W1SBnGBcRHcYYgg9Y5w/uhAQtK7q+h6Bo2wEhCnzwdH1LqjI+evEpWZaz2W2x3uBxfP31l4gworVj6PaIoDh/ekY31hjTYZwlTws2bc2ooFGevMyYJzmfPn5Ct6qpipJUKZwZGIYepKe14ILGWYfpelarNdvtjrzMKKsJxoyxTMwZElWQp5qiKEiKCUEqjHWEIDCDxRkT95RKoROJShKmaYVAYz0M40i7r7HOcny0RBrHqDPE2ODHHVoFZJYh85x8Oqcfe2blHDkI+rGj7vacnZ9yt1oTcFhvCHh0oiNTV0ahph8Gci1QQX0QTUZjmM1m5KagGwzz5QlNu0bnOWbTUi1KqqnEK8tu00BQZLrEWUmwEmcCzgZGO2DyARdgNplihoG71YpgxphYDoLgHEVWReOJDey3e5TusCpwMbmgyDN22w1VVZIlCdaOdF2PCyoirLY1qU6w3uPdQJmnPH30nCypuNlugIQkL9kfOgC0SqOoqhWLownGiUM0PyZNx9Gi09+/r+ij8+cMoyeYLb2JCIK+i9z3vrM8ffyYzfaO9eaGFx99TECT5ob97pZpfs52ZVFJhRWGtEg4LSu6TcdHj0/J0h1OXPL5z5+jin0UgGxg+elTpExjWelmwzdfvOTZi0fMygnXd3uG1jKOI0jB1U3Dk8cn4BpePD1h3zS8v70k1YKmvednPz2jrw2pm3B33xLSERdGzk8nPH405+tvXvHnf/4z1iv48pvf8Bf/3R+xXu8PXRIJ49ijNehUU02nqDTB9gNNc4sxI1kmuLvfEFCxryc4yqKimpQY67i/umW0PQkZu42nSlr+/uWX3NxfslhM8cD3V7+hyGCyXVDvdnz78kumZcFyeszV9evYSeAD/+pf/nN++dtfsusHhMzomz1lCoWe0t4b6q5n/mzBcqlYzDXnxyfYZmRrA8mR5n6/5vHZnG7ds13DYp4gFWQ6JWSKf/ln/4IXF8/RQTKvZvS2J2ppAS1jklQCMtV0xiGMRAsNIqb2vv7+awY/ctfd4HJByBOSSYEKntX6nuXJgsvra8pqTpCa47NTnjx+Tp6k7NYrrt+84eJiyvXmSz766BNevxlAaazXzOYR8dDXt1RV4Bf//DFJHtg0b9m3gcHsebF8jsDjio717p6j41NmiwUuNEzDiG9zvvryt/gAd6v35EVOUSY060CZzRiGAaUUXdtRFy0Yj7OW0/Mpq7bh+1fXCBLKfELXDBSp4OV3X/L5j3/GSMbtpmb46iuGzvDpRz/m7mbPZbMjn+dcbu4ITpCKFK8EybHG2JpdbVit7ymrQGcFv/7tlxwvH/Hpn/2U//Iff8Vy8jkkDUon7OqGRC/xxvL+9T2Tac/5o6M/6NltfbyWow4hCIIPwt7D34U8CMhYJCAij4LgiO7vQymnF4aHIlApNeHh9xJ1Ih8eUsgHyriIGg0HkxX4KGwe8CreR61BBUEIUZcI+IPwGcDHFFv8WYcODiE+CJH/QEQ/lIcKAQeCSnxF95CalkihY9pfcUjtPxSg/uAgx3qCi78evI2mRz/gQkKZnjJJJIlTIBNGE7DBRVNICFgXtcKhH0mShGLMDsL/D6gY7w0PSJtxHD9gVtq2w7qRyaSk6/d0XXsQx1uGoSeEAmstw9BHd7sIh8/SAJJ+GJmGOJjT3n/Ax2RZhrX+gHc5uPelQiiJTqPJNBouo1lB+NiRkMiYDA7ex6R54MD5jhjqZ89e8Nlnn/Nv/+3/i/k0ppCLokDIlGGMgwGpJJNpiXVRw/vs809YrTYQAsZaLq+uWa3XrFdr0jTFOotUCp3E99V1Ec2odXowgEYsincWi0Nqz3RWUk0kSrvIIEdGNI06oFNcHIwEKQ4YlcNJLyUyKLyQ6PBAxeeQRAAvIjozCAgerI8D+yAO6QwRE3JCCJTQBGFwAcJheOO1Iij1AyZIHM7uEIczEPeV/6uK6INb0Y8DWVHgRcWunhOSK2z4mlz9COstvb0iVQNBaHbjjiJJSeU5icgRIiFTFdLn0VVtDCIZsS4h03OEFJggcG6IjOjBUmYnDEaBNPShxgmDoCeIjMDIaLco0RNUwAcRURy+oQtv6KwjUSXBl6Q6x8i3GJ/j/RRnb0HMcCEwSSpc8PFitQPWBrQ0SDnSW5C+R+DI1TlKZDiXEZjH/8Y29MHFB7jbkaoC5Ue0DgxOo0JGZ6KDHlEggFKfHco2BYNb44WgCx2pc8hQA5oknWL6LVq1jM6A6LFhgwgV4XAiEqKnVKlYlEHaYeSIDBYhU6T0aDXFY0hI8DZFqA7jG7TMkHJGppcIPAIbmfdjjiAWRUkhDjETSNI8uklVgRQJvbvjvPjvaTcK/D29rTH0nJ3/MRv71wxui0oKrPeMLgozWt1TyBMCGqUNXlyTqGmcGLmEdrSkWYlSjwnESKYSPXhJomeIIFBCMjhP8JJMV4jEoowgyzT13vHu/SuOFgXJtEQYQabzKDzYXYwKiS3C7/G+idOrMKJVhRbZIaYU3T1SZvE9y+juCwRc6AgiOTxqDAgV5UiheeBoEQQBj/MG40YSlZOqCcZ1BwE+HHjUinAosQhBoVSJFIEgYut1wCMFeJ/hvTjErEBIFT8boQ9ifhId7lLQm/t/6qX8v3i09Y7t+gY37KmyF5ihR6UJfb3GO8nZ4hm+uyT3mu3dLb1tub274ba9w7eBvm/JleL8+JQ0zynLCoLjdn1FYzqsMWzrjsViRjcYbjYrzpbHdG2LG9aoacFte8XN5VuqKomCUr3j3RevSKqCLF8ydpLgFFUykGbQuJH9/YqzcgrOI5WiqBJubq+ZHM/Zb9bgLEmmYjT/+zc8fp6y2d4yqRR5fsKbV5d4Iejtjm/fXDGdVRRixnobN2vz+Yx9W1POJxS5ZLO5Y7IrSIPADgYlJ1RZLOPdjT1WaDo74nBsR0NiNcYrUlGydVtmaaBzA6v9Bp8JQpYghoA3gZv7W+bziumkZLPRDDahkkcoN+Ps6DNOl09Z5BVFGDHdiB8zAjpywJQhEFBKI5X6wEgDEBwwHr/jELAe+sFwv1rxxde/5u3bV4Al+IFhaLG2R2uJUAbrBuYVoDXS5bggKJIps8kRqY8LHGN6+q7BSk+ZS25Wa65vXvH48aMo7qEpq5E0yWjbgc7UmDAyn89Z3d9xdnJOqhPafmS0hoBEyZzFZIawgkIN6DyQ5hnbtsUngnIywQw1/TjgrMPayERLs5wiLXAWTqaK0Y4QFE07YL3k5bu33NRrZtMJQR0zmMCTT17w7voK2oGL88fooKiKOUJKRhdQumBxYrm8+5YyeYQWCX3Xo1OFU54kzZgdLyiqCTJPuby/5fb2juNqQhhGThYnnJ8d0do9j85O+e779zx7+hHeDcg8ZRQWqRNSHfEfdW1wRvDZ0095f3/F/X7ze1/bUhacnD4j+K+wTnG/WvP06VO+++57rBsRAoaNYzKZM80XaC3Y1zu87SmqKe/e33B3u+fJ02c4LFlRxOi0FNghUCSKbVNj7cj8aInWkulsyv3dHcNo2e32zGanrNY9xycTjk7m7Osd52fn/P3f/xKVakbXMzuqePv2Nb3t8MpRTI65W93xyWc/5vrqlu3eMZ08Ikmm7Os9TWcYhwYhoziYppo3b77HB8swKpQ0vHnzDbPZgrLMSZKEYezY1WuKKqcyFW3fIISjmuQoY1mtI0JNe08/xuSRMYGuayiKJefnF9R1HXFfOidJAk0T79lXty9ZLucYMzKbzZFScn19xXp1H3njxQLhFVKmFPkEAGt7sjwglUMIiw0DUjuyIkenOUU5JdlrwDMOjkTl6DKlyGYUh5LdptmzqtfkuUQlKbN5ZD9PpxO01tze3pJlObNZgRCC65srnAvEyW5gMquwPsZP19sVUosYYw2epmtRiSYvK27vL6mmc370kz/i+uqG+7s7bldbPvvkM+azOZeXl7H7YXaECZ7dbodzliSb8OnpBW27Y71ZId0OkFycnSOV4N3lK+aLCiVz6v3Aftcf0gAGJTWOkcks4dHFKalKeff6Pfc3V5yfTlFqxq9/8xVDHzdIzgzkqaSpN2RZwWZTI2RFmk8QWlO3DaM1aF2QpgUnx08QQnFzd4kLLZMqIdMlq9s1RZ6jdNwURLSPRycBnUm6rmG92uK9YDZbxF6YribPY8Q8hMDQj2idkhUKFxRS5WSFZrtrcCEwK0uauv4nLdT/0etbB7qhJi0Ux+dLMlWwW+84WZ4y2gEvDKbv2a+32Hagvd+hhGQyr9jXmiKtWFSeeldzd33N0I5U1Yyf/8nP+I3w7Ncb5idzpAgUeU6wJWPXIKwhr0ogpelG8kWFzjVKaGxtGfqGcbR0bcf58SMWk5L9Zk8wHULCyek5Ze7ZbO6ikWboGPqOSXlE29ZYb2j6mq73bNsdaSlRQqBkYBw6nB3Ypx6dCpz1JJnGdx6jeu5vrrm9vkeoBKkFx+fH5EVBb0asVYSQoGTEh+2bljytmM2PyIoCZ+O6Fy+5u7pndb1jNk3RSpPoQF4kGD+yOD1iNDWdr8nygnI25e3Ve8Z+RJdTRCJ48eQ52RmUQpIGj+s67Niya3eMKsO5hPq+pu07vAjkeUaWZjgXGPo47E8PrvIPzjMhUEmG8QNaqWhSIcTNdZ6AFAgtCUKjkwyJZLpYIE6OcV1Du11zf/+ObnVPIh15cti0asVkPkfmBVZ6nHAoBXe31+zbDu8t86qgzDReBtabLXgZObMHxq4bDGMYES6yS4OJeJxMp0ghaZue5ckJo+nxXuCCJS1yZtOS1fr6IMxE4WA5m9O4HW4MOOuQyqO0oO0bptNZLNyznkwlcUONJMtzqnJOvYtO6TE4hPOUZUWWpQxDLEfPEsXYtYyj4fp2RbU4IUtz6nGL3cfv4ng5Yz8avvnueyZlxePzC27W97jRAIG264GIuNuttywmk+iycyAyhc6SA/5O/WOX7z96vH51yZ/+4s9ohsC42bNZj+TZyGIai2FPf/SUdrhhux344ouX/Nmf/YT3l68oSsWT4zm/+uIlTz6+oJwbVnevscZwcfaEt5c7fvrjz1lvb1GpwAtLCAN4S6agbhseP3mGFD33qy27zYrl8QQhTri7rZlMSqqDULXd9CRCU5UFIXhWzYbBQbAdiSxZLqeEUfLRs0/42y9+ycnFCSFYrq+vePp0xsuXX/P4ySM+/WxG3WzI85j6ur6+IU1i+fJms+HJk6fc3d0jUKzu10wmJWky4/XLS2QiSNJYIliVE6qq4PLqmuXRhPeXdbzWqznt4Li6/ZbptMJIT73d0Nc1Q2fJkxnXb2+4OJ+y2q0YR89+O5LpjGlZkE9L+rrhvt7T7EbsYOkUuMRw37QwKuxqw49/9gyzbxlLRXq25O1NS0LOo1nKRDhkZhhTw+16R/CBWTHjxaNPeHzxmEQnlFkZOdgBRutJpEAojTMjHo8fLM4HpAjU9Zabu0uKWUHratbtFptY0lSz23c8efoYY+Iar8gzTpfH6Dxjt28pyglv3r0hUQ6des4vlvRmD4nj9dtXSJVhg+T2bsfZxTEew+JohsDR9BsKlTOZLrm+XaHGPX/1t/+FH3/+GJF0BHqkLJlPj/n1F+84PsnRE81u6/AuZ71xnD3SVPMUT0OelZwtzri8ucLZkZubWy5Oj0kzydPZBeL2nvV6Q6IlF+dHrDf3dG3HfD7l7379a6bzBZ989ild29I097TDiEw1pydPacwKb2p2m5pMXZGkGpVqlicLurt7ilIj5EhW5mR6Tm8tX337WybLikR5RBYI0hJEYLlcct2sOZ5PYkGpLf6gZ7ez4eA4/gEl8SBiP2BXhDgIwPTIA04YoRBa/CD+isgXR7iDBjFGEVBInPAHpK/E2egyl/Lwa94BB9SKlugkYxxFHJIhUOrg9hWHYtEQgBEfFNGlHrWI+DP04RkV4ENK/8EU6A/ucfcP0DKHv3xAZ2j94HyP+0p5uHc+CPJCRvwWPnZnIHqkUAQaBrND6YBPJF4kmC4ikKSIjnNjBkbTY71BBk/b7xEi4g2lSAhBkCaCcYyJyXBgwkRBPRZdh+AwpgPhCViapkYpTdPsGEdDkmice0CHJIQQuer+A199jOJ6cLjRMQwJWZbjnMF72Nc9y+USFzzGRaE7SBisoSxLgveMZiBJkoiI9lErOpDEIcQ+wcl0zr/4F3/O9fU179+/I4hAWVUY2yJE5OPnRYpS4pCujQW1aaoPJo81/TjSdgNSJyRZTr/fEwZDlmVInSCUiVsIrdEqvgkpEnyQhNCTJPDoyZLRbkFm0WmfZrgwEnAQfBz6aw3IQ6oiIoNkiIK8Eupw7kkIDykKYgmrjQgkIQRB6EiXCDGRL4UmkQlKRQa+DHHtCMTrw0kIEq00cT4ZCN5GEf4DmlnFn/vfOP7JInoqCqyeM5p7cjnF6j1C9wT/giD3BKeQTEm4pXeSgEcERTt+T5Wc0QVHO36NDscU8phEGIJbMXoDB9i/UAVadbSjJxFLGtMghUVLhx9jFHc0illxjPM9u35DlTwjFZbRK6SqsW44RJsF6/YSJRKSZIe2BUq2OL+IpRO2w1vPgCIIRWfXCOHxciARKT50BCRaFBR6gXlwJatbfMiR4gSdZDirUKrBmzlSxXIEzympyJAo+vAyPvxUBjh6ewUSKpkhxCWFmEEwIFP6QRF8g02vSdIS7+/xviLgSNQ5SsxxNSiRERJHnkfRKCtynHKotECqSUSDqJwimTLYW7SG0QgcgkSf4L1BkpAkghA6RrMnkzMSMUeiccSbwOhihKTruwPfSTJLH8PYMjQehcXZHhs6ympBOYXOdkgMSia4MJKoKcHnaDzSz/F+j+YRo1+hg8OKPGJwpMLbDq1zkA3WNxTqMYlK6b0neIeWEi0EyASFQ1U76B+Tp5pBg3EGlRSIkCN8bGJOUsXo9vGmLhJ82AB7YBGRPN5EJrtbg5UoEYcuEoFAYFyDVunhAZHiQxwBa6HwfkByiLsHH78XHCFIlJggSMhVyUQq+rDBiQ1pAt5LYpFtglQFCBXZUGJEH/bTAY+QnhAkQqQE5fG0hJB8eKj4YKK7yfcHNvrvf7x7f0nTr+id5fLull09sjw+wXjH0fKIVGoWsxlZGjdDzkSeVKIzjJDcbRuWkwnbuqMsKrZdx+urO2QiGE0PAZI0ReuUophiBk+SFiQZ3A1r2s0N764aZrpEJQv6rke6gmF/h04dZ08X7FYjrhvpncLbwG6/x4yW3b6hF4HBCyaTKbv3N/TBk2h1cC10SFmwPJojRUGSFMyOMvZdw+36HpEk9NbRGc9xMQHf4YDNdsu+7g+ss5H7zR47Opy9RDnIdMa66Tk5OmEYRnSumQ4N7dBwNJ3gBkNKwsnsnExPaczAy+++ZT6foJHsmpbgA4vZnPPJMavNnm7o8V5xcXrBfGpIs4pAwizNSIFut2M/tGipqCYz0nyC8wYlBpACLzXIWDzIgfOW6BQhdFyYHBY1IQT6oeXy6h33q9uDW9zQd3uGoUaKQF5kKC0QBZh0hDzGpKSITreT4xNuti3v3ryhKEusHWiHWL4SiGzWl6+/pe0bjo5OOT6dsV7vcPR4HKP1h0Wf4/h0yWRWUc0nfP3br7l+u+P0+AihwNAxhBZkgkg0vRlIkwyCjsKiqHCho2nXUQwcU4KQ5HnJIlvicbRtQ5amjGHkq5ffIbKE45NT8LCcLbG9YT6d8vLqkulygRCaiZ7h8ahMkiUpX716Q1VO+eT8hGmZM++nTKszlsvHfPndN7gwMI4Sbys0KSkZj04esb6+5fijM+anJf/pb/89f/f3vyIvlhyfnjKMLbfbe15fv2Jzd0OVV+hZyfXdDSezI4IQ1M2Wx88e/d7Xdr3vePP6HWlakKYJXd+BgCxPGHYNRZlHQcN5Pn7xMWmi+P7lt3z7/becnV1wdHTC69dX3Nzc0LQ7zs4XjGPL7e0NVTFhs11zdLzk7u6au/s7yipnV+8iDszH0jnjLIjY4l4UU+7u7tls1+zrmqPjBTc31ygtf2ehbbHOHCJ98Md//Kf89rffkecqpqJ0irM+OjsSQV3vefzkgsm0ZL2+x9oRISL/2lrH8fEZs+mcutlxfLykLGZsNg0hePIiYxi6yECeVLFzImj6PhYGeefZ7ja0XcpkWkYOsBIMQ2S2pmmC95ZES8ZxoGlasiynaWqub64Y+hEIB8dKYDEvCcEf/t2RF9FNJlUUA9t2pG33h9JeTZYVjMPAfDaBkFEWCz7+6HPu7lZ0XUee52SZxgdDkeScn58dnLrp4fU8OklYLo4OxUqw39WMw4AxcVGc5znGGLTWXF1doZQiyzLSNKVpGvpRYp1nV9dkWU5VlazX+jC0iwO7sqiwiY2sVuNo2sgPX8xnTKo5n3z8CXd3t/zXv/kbuq6jaVom05KynLBYHJGlE67FPdvtnv1+T9eDUILZdEJVTEjTnK7rub25J9ERbaB1ynIx5f3bW4KPhYlFkeJc3JQKkVJMUpTOoiPeg5QJJydnPHv6McvFOd4pjB+5X48UeYo3sehLK0Xf9WzWa8pJRts0NF1DnmeUZUm9byMSKKvIshJjRna7HQ+lVZPphMl0EnmN1h0iwXFzVpYV1WSCs45hGP6gZ7c1A7d3V1RFweJkig4p795dHYY1sbhdlQXBBuZZxeZ2Rdd07MoSU2oW8ymPz055unjCzd0dMs14e3XD+eNHfPzjz/jm66/Yjy3ryzuOlgtML5CqIC0KPvnkM7rB8usvv2Q31BRJTjUtoBbRJekEY+vYi57HJxckpaPSntY7Sj1jWU3pZi1XN++oyiKWxwvHdrfmZnUN3pHpWHwuvGdoDLpKES4hz3N2uw0ysaRJjhIlyljKWcLxbMLNu3vcmKCzHCwEb+m6niyr0EEhgidLU0BzcvaIoiwwxnF1dU3bdmACXdvhdo7ee7IqpiPSTFI3W05Ozrm6a0hLxbSacHJ6xjiOvPn+FWmpcIOlqRtOZ8eUzpNJaPqWZrdmVxvakOJ9glnViEQhEoV1nu2uJ1EJfduj05S0qLDewzgybrcYFDLzWOfAWxKt0ColLyZkRYH1FnUo6lJJQlmV5FlCMAO963AKtOuZ5oZx6PBeI0NEyjwkQnSV4oNnv91wfXVJKwSDcdEVC5gQcGN8D24U6ERhrSeYACGK2X3dIfJoOZRBYvqBsRvYrbZUk2mMkw8jSgrafc+wc2TlBC8FwXlO5jOK0bG1HY0xJFlGOzSUsxK8YLfZxp6uKsFZC1mGIiFJSggDeV5ydH6MTKKRYBh6Vve3WDvSNjVFXlLmFZNqQTikKeazGePQc7/a0XUjn3wc3bWvb675KHvGrCwYrUMawyBieDSrUsba4QZPIlO89BSzkj4M0Sn6B/QVpWXC2+t3DKYjzQTzqWboDGdHc4SQ3K+vsH5keVwxnRzz5Zdv+fzzC8ahox5uQRlubm94XJax18QBsgAPr77b0w4jj54Yzs8u+OarXzItNMbU7LeC36y+40/+7AXnj6aMY8+uXlNWE350tOTduyuyYkLbbLjddlycHvP+/TriFQbNi+eP2W3uePNmxeefKD766AVmiLzjo8WU95fvOT0+JkkV797V7DcDZZViXM37yy2L5ZwnTx+xXm3j2lUM3N7cMpqRrjUURUbXdaRpXM9bP+CdoyhyijxlGHp8sHhLdFsOnsdPz9jvtughBSlIshy7U7RjoO4jEupmayhmI2PX8td/95qnj59Qij31VvNR9QmPHz3jN//xv3J2ds7x0xl1uWHdbTlaVqRCs222vHr5lmmeg4frqz2z4phPHz0nhJbV7g1lmbI4nZOlU67e3vP6t/dM0z2SmNaziccOPWmeHcxUjsFYtFJ4DzLRaDsy9B3vrl7xn//6P7Lpdjz5/CnX2xsWJ3NOlkc8vsioNxuEDNhhIJtOyWRGIgQhT2mHPbPZhCdPn9HUe+rdhu7OUqU5x8cXvHr1hjQvKMsErcH5AaUd7y/vsNbz5MkzlkcnXN7cEUJPNVes27dMJilZrsiTBDeOjI3lbhh5/uwJx0ea3WbgR58/ozNrjBlwDKzaLSZPQGguLh4xjB2rzQ1aBM7Pzvn4+QWp8gxjz77ecr9aUU0LXr+/ZjCBfd0wm004Wh7x+PETBuOpJjMu390wmWVM85Sj53NEMPS95Ytf37NY7liclORpRdtaqrxgPp1DyFnfr9AprPs7ZkVBO+S8e3fPOmn4+OkT5tMl5fQR++YPe3b3Q0+SHFI74gG74g7/HnUA720U0q2NjnAlDszp30F6cECIhCiixw61gzgYYtGoEL+LgI3OZucOTm3kB3TGAy7kgwnr8PeH9wYxefnAspbigbUe95UH+m98beyH14t9WHzAdsTUvTogah5wJA/u9QfcjP0HPOuHP5N3niDivTWICFIydkBpQXAKYwQ6yWISarAoJRjGnnEcSJJYPt0P7QErkpBlZVxnj8PhdaPZUWuFMXEdnqYpxgwfePPOWXxwmGE8/NksQmQYa0jTlGEcDmtBg1ZpTNkKeGDPm9GSJMnhM88QIiIU+74nSQ84NO9BeLTWB0xKcihDjWQDnUS074diTOIePy9LlkdHfPrZpzRdgzEDw2CQErRWzBezA3LGUVYZi+UUqUCpiO5p2yYiaLKMvo/u+rIsaZPbLCgAAQAASURBVNuWvu8j3lVr8jwHDmnXw3mglEBqxfJoypOn50ymOWkau7GECL/juufDAEYIiRYyDmBcOCTnwz/8n4jnVjzv3UFcj9eC1ALlo3tdyrhHVDKilcPBISuFIMgH82q8brx0hwG8I0TMRew1EQ9Uif8Vneh52rFpV+SZRIkZaXqLoECr53T+LSQlop/g3Mg0mZHIEuNXtLamDjuMy6nKHKxBaY1G40lJ9B5n5iilqPsNReJIxZSgeqwfSHUeB2EyXuBZekJv1xgPRXaMM1u0rhAu0NsmclCljZOMFDwWLZ5gnEeEgZ43SBL6YUqiawZ7T56DHQ1andAPAy33ZKpAixmD3VKPl0gWSBXQamAw9/S0lFmgTI8YTcc4blHpgiop6caO4DtyHYs683yB1DlDn6OEJlUKazvcqMmzJSGM9P07hCwQ0uHCGxJ5BKSI0OLROB9QYkoVfkqqSmTmMdaReI1OJKOR9LUm1ZYyzdEhY3QblJigFQgxMBpNxhFGtgyuIxUVWkzx5oIQKqSbRsyI8PgIimIYRoahx9ghcnVFRc4TRr/B2ttDM/vA+dmnbMN/xYlVHCaEATjcPEKFp6ZzNUKYWBYacoQwGOsRIkGpgBIeH25i0Zc8ZbApnenwIifTCb1dE4QmkZOIdtHRLYuUZKlChoxETSnCJ4z23xEk6GDQfhqZzDLBuRQXHpElBVpuGW2NkoI8OcP6Jl7QWEJIMS46TbwXiANjHVTsBQjN4QLOGF0HPDRmm8N0s8KHkcbEeKoPPl6g0uOtRcuAVikhdFjX4YNBCkmQGUoUh6niYUqrfYyn+od4bhsfOuLg5nGHh9EfcHTDQDeOGGDV1Nzcbtl2PcvjaeTWt9FNL5Xi/eUlaZZwdnpOUU5JJwuCs5i2ZWxblo/OMMZhrIDEkglJW3dIJTDGslyeUOUziiSjqTd4Edg2K1LpGU3gejtycvwRP/vZn3C8OONq+z2r20sgIy/nqExxdf2e09MFiRW4oGjNSOs8rh3YtgbZ9PzJH/8RQ9fTeMv54+ecnT9jvW6QasdgR95dX7FvG6wR7L76KpYedg2UHpXGqXrXdbEcMUtIROQhS6FZHi3/v6z9149lWZ6liX1bHH2VXVNuLkJnRmZWZXVVTVUr9HDQ8zAgwGf+owT5QGIGnGaze7qnRYlUFRkRrt20XXn0VnzY1zyyhsPuQiYv4EDAzePaFUfsvX5rfYtUaMLoQSUMZsQCy2LO/mFNKATTYsY4WJp+pKWm7XYUWnGyiJiQh3qD857lxRIdNJP5lFRLFpOSzf0K6QT7+zWfffZjFllKs7rDNZsoqC3mZGWBG/bYcUAxHKJcCqkS0rzAOo+QMTsnpUOFNCKFfFyM3t9fU9drZvMJ2809q9WOcWxxdkRJ0EnkxYpSHJh08VhJkvj808mM717tqOuG+WyGD4ZdvaVu95yfXYD0bOsHNvsV05sZZ2dPmMwqghzZbPeMTaDrGx7WNzh+FCP5IfDu/WuWk6dM8oK3H35D0C2b/QMn+oLVfsu23jApK/YiY5LMscZR1zXdMKCSwG634+jkDKUV42gosox2G7/XZJrhgPGwQJgmFZubW5YnJ/G1m4a/e/kbzs+f09qByWxGUebcXd5hrMcH6G1D7jRJqrm8vsI6yWxRcDc6tAY7DmgU01mF7z2pzPBBYAOcPjnnw+qOZ2fHPOy2TKuS67s7EB6vU3SZUA81vRtxIrDvGwbX8bC+/gPO7sD19TXWxkXUer1itVodmOcbEp0gZGTpXV5fcXZ2wsWzC1abe9q+4bPPv6KuOzabLVmmadsdx8sZ1kTe7c3NjqLMIifPO+q6IUk0ZTlhtdqgZGQgBi94+fJ7fvz1j9Ba07XRhTsMA8PYHVwacRGfTYq40DQDl5cf+OyzL1kczXj79i3bXU9V5ezrLYjA/uAMv7y0h4h3Rl039P1A2w4Er/EO+s6QJDl13dA2Fik008mcgKHrLMNgIMTrf54XNM2e0dQHB/ueujFIZVgsFuhEYp0FYSmrlK61eB8+luc6b7m+ueLu7paiyAkkjGY84CUmgGU00VUiJWRFgRCBvh9BBNq2Yb/boLRkNjmmEQ1nZ8+ZVkuG3jOOls1mi5SBslxQlNFlXlYV+32DVoqqmnJ3f0eel/T9wDhagpcf+ZBt2x4Gm4pHlmLkP44MQ1zQl2Us0u6GQ+FhCEyriiSNKagsS3GHGOtkWqKkphs72npPCI5haNjXnptbQVGmsYR0OmM+X7DZ7Hj77j3L42PaJN4T0zQ99FsYum5gvihYHh2jZEmzt/TNltXqDuTIenPFxcUFk2pCXuzwfmAcB6wZURoImrSYMJlVCJUyGks5WRC8YLE8Y3lyzmJ+wjh4yu2Uze4upmHGBmNisZIxI5XMcd7Rdh11XdMPHVVRsVgs6LohOoOVxh3KcLXWOO/o+/bA43yM77qPJa1a6wNzPiJd/pBHluV0Xc3OGKZlhR0s5UTT9CuKMicEz+nZCc44iiQDPyPNEpp2wOwkfbC0qaE8nZIlFV5IppM53337Pf/4n/wFL569YL1+4OFhdShfhy+++hGknrKaIeVIEjTDtmOsOyqZcLY85uFhiHHn0TAkhu1uz+nRIuLtZFzbDP3IcnmK9QYh8+hc7/fsmw2xsCrev4q85O72CoFDaUuic7QSCBMY+xFJQpprzi7OmR3P+En5dVzDbC3TaYHyBt93aBeH/2VZURYFu7rj+OQ4so9v7+i7jqFtGbseN1hsM7Ccz1G5wMoOY0e22weETFieHGFdR1GmTKcL6rpjeXSM7QYyFPOiJE0lBI9GMvYdzvqIAZGBZrUHkTItcxyBZhzprGEYogvQDIbJYs5oB+gtiTUEodA6QbnIQh2HDiE8eV5RVhUqSdHBRa66d0g/Mgx7ajMg7EghFWa3YZJnWFkeUEPy8EcdNscxjdrVLft9Q9N23PUtow+sHx5QEpyE0Y2MxhPMSJJrnLcoLXDBkRcJHos9sHi9E4ggsaNjYzYsT7/g4eo+bqa9iPcWnSKNZzqLAnuV5YiyxDrHQwdCyZgisY5MFmzWe+azkrzI8UlCWkzI05J6XZNnGdPphGJSsO92ka3uAlU15fLyilRLtHLUtqPMq0PiyFLmOcfLBZd4tpuah+2G5fKIi08+ZbNv6LuWo6MjJmVBPzpEomlCj840oxkQBHQuWUxmNO2eXbM7xMN/v4dK9AHDNJBKOD9bgHH89MdfcL9aY4XDecHd3Z7dSvPi+TN265GnFy/Y7u549knBaEFJSJI4TBqt4cdff8a3L98yqQq++/ZV7DkjxRrJk5ML/vt/8S94+fp7Ts5KXr59SVUWjKPn8v2Kr3/6KdVswmy6RAnNdHJEogRdIxlHqNsEO0DXjjy5qCgnGW/efcfR0ZLlccFut6EsooN3t2nIkiPwKWVRsW9v+OnPTnn16oqy0MymM6zRfPLiK16++jVSavIiMJtOaZqet28u+eSTTzD9CEhOjk9o6z1pmpCmKR+ubzk6ntPYkVdvX8fjXYDQin60jBbqfcvRVBF0yvRkyraPRpAxkazHnsmLE25v7rj85V/RIfn6T3/K3/3mHV98+TnDvsHe7/n65AlqsIxILk4qdncrsJ7Pq4yXL79n1PByNfD0pydcrl7z6WLKar1ieXyCsiU///JPWczikLsoS5TkgOaI92EhBRzKJK11CDfQ9zv+43/+t1htqY5zfvHbv8FISKc/4AJn0wkvX75ECMHx0RHz6RTvDNum4+6h5dlnBa/fvKWuO44WFVle0tQjK7Xhx1//hNVmRdvV3N5dc3FxxmBaFscVQyd4+eo988UkFg1Kx76O64mmFnijqBYZ6/sHnj85w1nPy99uOT07ojc7cn+NSg3WKvLiiDzN6RrHu6sb/tHxLLpwtWK3XTMOA8uFITiYTZc8rDcUxZy+N8wWx9zf7MnzHCXSiDJRgrev3vHZpxc8/+SMb355y5//2Qu+/e5vmU1Tnl18wpef/ozf/PoND3f3JIUlL3MSnaNUwtXlLV3XkQmPSC29DQyj49nTC3wv+PDulvyLOevmjiTP/6B7t/fmwB4PBzE09qM5/ygefgwQ84iIeuzs8I5DDwsfeflRSOXjNedRLIzPI1AyOTxP+MgwD4GPg//HdeDH3/g76eUf/p9YJCqkP+yHYwI/OoDFwdnrf+f1h+jc/sj3fkTOyIiyDRIb7GFg9Oh4Fx8/l8hq9x9FWsIBW0pEtTnnEcEg0vi7YhFvQJDQNvWBqDAc3PVxKDWO5mNPS3RB28hlD+JgOnssQ1UEz8F8IrDWk+iYQvYh9gS1bfMD473I6PuOJNEYE9eG3guU1Bg7fiwuFRH0jjEmuvCFRKnk0NFjDn+vDvz9x6HDgZuv5EdTK0IeOtAehxIChMAZy9Nnz7A2fvfDMB5EfsjzFCEDxvQURc7p6QlJomOXh5aMpidJo5j/aPB4FM2zQ2fM4x+I2qwzlhBURPYQUFowmeQsFlOk8jhnkIcCUCnCx+NKfmSeRzpDLKh9RErHZEU8jtzHE0GIKIxLdeCih1jbyMFBLmU093AYMkVEDnEt6fzH62rAEjh0xYXfHdTEdEe85P7XMcn/YBHdC0ueZCTCI+QHbG8RumQ3vsGpe4SfMkmfosVnDOMaKwxlcUzvW3J1grGgaEmSjNa+Q6q4MZMiRbIBZyjzkyjihjc4LxHylN5sSJKSIG+RwTDYI5SUeL8jTSO3hnCETla0RqGZk6s1nYVMT5FM8W5CnlZY30XGGBof9gympdAzrLXk6pTBdZT6p3T2r6IAqLcolSFlhQ33IDXWpSRK0Y8P+OFHuFBi3UCSaqT6QG8WJGLG3tyjVYWSmlQUeJuSJ5bRNkg0Qa2jW1GW3HU18+yMLJnQjRus26NNiSNHC4ELdWSLhvdMrSKYPl58hcMFgwuxkTINM4JLGcY1goHBORK9xfnmUIIp0bLEBUeiAgSD0gXBbwlBIm2JwOODxxOi0+XAAPfBIsgJXpLqOdLvUIei2UxPOV9+wWX49xgeMM7hbI0PO7w6AWHxrsWHkio7QkuLpMR7h5axgEmJlkROcWEAmcXppIxOJMUSO0q0PCLQYl2NIVClCqltFCFShbOKut4zPfmCYnxGzxrhW5SsCOKI4G/IVMGAI7BldB0eGNyeRFcRFRQEhJwQMqBEq4ASGk+Pc4YgevKkQFPGCKtIEdKTSIt1Hh9cRBPZHUJ5vIiNzoma44MmOB9bhGmQUh9uGgmpqggMP0xtQywjBfDhUELmIvNJS4V1FkGCsR7rUkb3h0XCX3zyCX/3/RpjHO04sDg9ocxLdtsVAz3Ka3Se0gw96ITJbEEwlqPZMUGkSBm4azZkJEx0iSXQuoHb+o7yKCPJEzJdMK8WeCuwZuB+syXJUmZH59w0dxRlzkxNWZ4/x7sJSlTc3azZNjtE5ggkSFXz9MlTFqcLvHORoR8CTits0HR1Rz96Qt+xvt8yjCNpXuCF5O27DxTFgmoyZ7t/hUpSkrxgNp/Q1rF48PXbN5x+coLSCeMw8JMf/zF1vePm4RJjLZ89+xQ9KFKV0jUDCsXQGXJdsGv2qEQwK6as79aY6QSdZqSFIE0VhUwRXrFbb3DWkeiMsetou54sV2zqLc4azDAjGAceJmXJ0/MzpDc0m3vGJDrElscLzNBFIc8ZsF3E/QhJNZ1hpWS0MS4VQmyyF1od3MA91zcf+OWv/4q6XZNlijSLN/1waAp3Dqx1sXPhdwo60iShnE5Jspy0eM7d/af85ptfxrKWRIHyWN+x3t9xenJMNYmC5s3dBx7WkUW9WCwpJwn7uuHq9h11u2G0Ndvre+43K+5Xl/zj/+5fkJeSvfnAr7/7Lev6nnRRsN5tuL694un5OcplzE/P+OzFc5r2hH13xMPmhsVZwfRozna1Yv1wz/OLC37y05/y/XffsrrZEll0OZ99+jnt9QPzyZzdZsv9wx0Gi7eWTb3mJM8wbuSonNEPHTM95UhNSJyFMLBpakJRkp8vefnbXzAay9G8JBcZYggUWcar16+YT6YY03O1uWQ33mHMQN02pOmUX//q1/ggSLOUcjLBekuz29B0A/u2okgSBtOj+vBfP4n/fzy22w3jGMthFkcziqKgbVvm89khYQN5mlOWJZvtirfv31BWOaMdcU3g7ds3OCco8pTz88iz3W4eSNKKs7MLfvqzn3Jzc804Gh6jod5DkmTMZwuyLGLCsnTCwyq2vXvvybKM0fSs1xtm84r1esOXX36J0rBe3VIPe8qqZLfb8q//9b9iNltQFAnj2HD/cI0PI8ujI/LCkiQKa8cDV1OSJCnOxaSMUilaRRFASI0xluDGj4utro/D0qqcEjchCXlWYVyLDwOjaQmMON+zrx+YTDPyJCNgDm6NAgh0XY/3nuVyQVFkbLcrpBKkmSbNFAhJ17Xs9g+AQMhA3ewwdmSazhiGDmtHEJ5hbNnXG+bzBcujc4p8wAxgs0CeVwdRPBYrKq04Xh5zeX3Fze0d5+dP48JdJVTVnKqaEMKWrhvZDvs4RBsGur4jy7OPBUNpmtK2zWEBH8/3JNEoJXGMlGVBenDIDMOAEHB8fIQPUTDO8wKl5IHR2YOwjGNH38fulIeHO8qiIssyqmpK0zRonWFGz831A107kOcJs0VOEAVt5ynKjCwr2G5brGnwtkBKcah4CIfXlzCfLxj6B+p6T5IKqklOkpaU5Yw0j6XjQqbM5hOUTEiyAo9EqIQkExRlSVEUKCGhCiRaHxxD8X6+3+8YzchkOiWEuOHM8oQkSTHWIIf+cE2PBUvDGI+F6XxGqiTeO6SIgyYOm8hH3Itzv/+5DTCbTUi0YL1eYawCAYvTkr6NjqFMZshEoLSmaWqMMDz/8jmX727wDyOm7Wnrml0qsHak7numkym7dsfN1TUvXrxgu12zOFowDANlmVFOJgQduPxwgwyKKp3QNTW77ZrnixOePDlhv76JwwPv0WnCrql5+vSUwTW4MaYz2qZDSEOeTZA6px9HmmZNCCNpFu+Xs+oIpROct8znJVkGBIFONEV5Rl4plEpZr1rKWcXd5j6WxSceKS3BdhxPTzg5W/Dmfcu+aZkcHdH3Bp2klFXFw8Md97c38ZrU91R5Tm8aJrOKi5NTvLLUdkM/eIZxoB8aHlbXjCYmuDbrHVInuGA4OTml26xRGpp2h8ymSKHYtzF6nqYJkzLj2CUIlWHayEn3xuDHEdP3jKNBCIn3BmdHBhPo3J6yrHB9hh1HZJLQDR1CpigZUAqCMwgBiRDYeo0UNjq47EiZpezv1ojBsJiUtF0g+OjIS9MMAGMNwWqsHdisN2w3W/phoDMjvfc0tidPkjhIr1KEcRzAzYggkTL+7rZvmB/Nubm5Q+ss4jLyCUXhY7pLyoi9kRozWLIk59nZOX4YmS4XXN/e4AaDCDAMPUmeMowjUqTsd/tYK+c9wcUCPCk1fTMwy2dUeU5SJWR5xu3qDiUVy8UpSiuKPKNtBszQk2Ul+03DbtMyrSYIJZFaonXJ8ckSlKAZe5qrK54/e0F1tOR+tWe1fs+XX3xGlVV0rkfIgFACGyxmGPj84gX7hzWTtGC/3yL8739+l1nFcrHk5mZF3zcczSZ88uIpD/d3jFZxt95T1w47aoT0PNy2fP/9K56+yLn4LKdAY3eOy3c1p6cR9bPebunHwPFZwW+++Z6sFJSFpt15/uSnP2VeHfOrv/qGT794wXb9wOWbPcuzCVpVnJ1KjLfsdg37bc/Z8RFtveJoecR0UeKc4v7ecnZ6gk4sggGpoZoVdEMNwTGpSqwNPDw8YK2l7zWL2ZzgY1q162q+/PKE9+/u2W53TKcnfP/9K8bR8+KTc6oqZ7V+wBjLfD5jsTjCbxzDWHN788DJ8RIzRpNZWcRC9SyT9KGjrQesseRZFsUrEoTy9LZnminySSxK7/uGTycztk3HB79BLRLEIMAOqNzyR3/ylK5fsb65oRgHzl3Lnz475iw9YiEgrxKkt6yV4G1R8u8uXyOM4rdvBxbHS27erRFKsbp+z//x//B/4quLH3M6PyaRCmsG0AEzDIdhQB6PfanQWtJ2PWO3Y7W949kXL9iZmqv1NUfnZ2ybaGr58Y+/ZjQts1l0j3Z9gzGWD++vePH0GecnT5jMj3A60O62XN9sSdOSSbkk2Jq3b67wIaGaJUgZOf9ZnpOV0HcDSnrSrWDsB1KtKIqKemsYmhJpJ/zJH/8JdmyZT3PaZse7d1c8f36OVPDpp2eMrma92x9SfBIrJCdnR4xhgU5Tcire366piglaZux28f0b6xg6xX4XUEnJbtNzcfI1wXuKiUYlnsGsefFZiS5avn/9gXoX2G5nnJ4f8+7dO+4f9jw9e8Hp8QK0oFok3Ny952q/Zlsazs8u2O9WlIUkJIbROBQp0+oY0oQff/oFadqyv/0t6/ubP+jerVS8ZgpxSGASuc3hUKoYncscBPJHF3kUmGPS89HZ+4hOicL445D675V8hpjsE+LvF1HCo/DsPorHj8WVv/vfUeiPXW6RxBLFcYQ9ONR/2CuGx6EAsSAyBHsQLmPJZxRhVbxnIaNJ9CD8P+4fnPMfhfsQ4vs8QHGj41kCQuEcaJGQ5SlNs0eiGMcRF6BtDM73dF1LkkiEDIxjj9Jx7fhY2DmaFut6qmKBdIfUNuKjeC+lwhqLQBzEbY91I1mWYsz4cVgRgqfrmlhsHwJ5nh/eQ4IxA2XxWEBPLP62liyDruvJc0GapjFFzt8fbEgpP77/x+4zHlE3HwH6j0xikEpTlRO++upH/L//zb+OeKddLGhOs4gwyXLJ+fkZi8UsJj+7OLy/vr5ifygPjanJ8uOxodTBEPUooIu4R9AixNeEi92TIZDlmnKSkSQ9QvqD014RHo+9A4Qmvid+p4A7/DA4Ep6AiaZe//EvEeIH/r4PsZA36mXicOyHwzEYn+uxn+Wx8NXZOAh6/P2IOBSKPZAxre/DD1ih/9LjHyyid6NjtBucXFI3W6zxTNMCKVKCL5klU5xraENPktb0pqdvcrJE4wkEqfEuZfCBMn1GZ2uSvAQLKrRo8QTkGmtyAmuK9AXduEGHS9qxIMsvaHpLkSb0bk8QW/a9x4UcJQYKHTBii3cdWktcmOI8aBGZ5sZvCUGjWWDDSKo1bfuEUQoSzIFyoOntK7ToGLwn0RWZWDKEHWasyA/vox+OKXSKoGc0h6JOtijZ04zXBGEQah+jwyJBqBTtA0Hdx8lRovBuQGUpo98zrSSZvsF7zSRf4m3BMD6gk0AiS6yUpEoRXImzFu9rgn+c+KjIRJaKRINPQDJFM8PyEBEjIUXiqdJj+nCJlineDbjgGaxBCtDmBG2OYgQuuI8tzVJr8qJEiQOPiICxI2k2YVNfM5qBo+IJJG8h1GTJhNFuSdQRQUxJZIbxhkTpw6Rxiz0UpHj2BOEQMiMIy8hLEpkTxIgUn6DUjNFeI8UaRAmyR8k5vYubLCO2pLpGqNN4PfGeYWiYySfkiYqIi9ATA04eGRQBGVmw7NEionwQKd5bJPkh2mTQSsbPKUSXmCQl16c40ccbXYhTQYVDS4OSNnK1xILRtfgQy9Ngi/cd7uBWCyhSpQmOKDTAYcMeDgAZBeIgXgZDQDC0h+INafDC01mJsbFEoh96egPbpoFP/6Fn8//347vffIsuEsqywop4U0uSlN1mD6NjMTuOBbLWoLMEGyBTmrHbQBAUaUqRZUgPN5c3aF1ge0/Tdjy090zKKYtK4YYV+23LcnqMUJL58THpScqH9TVtXfP0ySl2VDgXXconp8cYuaMfWyazIza7mpu7DyyPZkgvsMJRm461GaNraQgED0VSkCYZRRXLQeuhwY4wOofQhs2mYRwcVTnly8+/oqkb3rx+xV7GSXqRZuQyod3u6dsG0w9YP7JbP1CRIbKSYexp6p6n589xvUGKwHqzJkkT5tkRo4su6LRM2O7WjPsdT4+XGOvJXUniHe16x4ebG/KLp2RlxmbVsG1qJrpEJ0mMdWlomh1Zkh6OX4UZBuqwBm8JdsT0dWShqyhyJdbigozXpjy2gcf4mWC9WfGLX/w1//bf/iuyMuH05JQklSSJYgwC4cWBtSo+Rv4CHFzCKUmSoHVCkqX85Mc/5+FhjRl7dvU913fXVJMJ/djh/BixEllC2zU0bYtOYPNmRZrnnJycg7QI5fnm219ig+PV29csl6dMlpJyknLWH/Gv/vMdg+m4vP5AoiYIAdVsgusMSQrGdeRFyvdvHqjmE0Ii2dVbklzz9R99zW675fLhkrc376jbhidPnnC3fuBv/uqvoR4wu4aL5894/uITrn9zg3WOTz79DGcD11dX5GnKtKhQVPz8869RqaMZDU0/EvKSt6t7NmOH9CllPicjw3lHViSExDKqATNK5ssFd+/eopVCEhi6Dm8Dx0dnpHlCJjRjO3B1eY1IUsRx4NXL71meHoP8/YuDA56ub0jTFOfcITLYUZbFwXnsmE4jy3m13jCZFSyWC+4ermOhbKpw1jItc4pUg0+52d8RvGS1WjGanrZtkVLRNQ1KS8xoscaTpgVFUVGVE+5u16RJcnA6D4ymo207kiSWSbVtxzgaplnkEmdZwdC3GDtGtnaiuLq6ZzotCSSsNwNllX8cPsYYX3S2W+vwTpCm+cHpEUuK9u2eNM2QKonClfdY65FCHhZfkrKY4Jyg63qEHGMJpVY4p/DBsa+3OF+RphnDMNI2bXRpe0Hfd3g/o+viUE5rhff2wEzsMXaMos+hz8LaAQgH90kcuGqtsWN0s2dZhpQ58/mcYbAkSRkdK7mimkwZhpa6rtFp3LgMY4yWjqPF2cBifkQIgtk0uqZDGNE6YRxH8iz/uEkZx/id1HWNMYbJZEKapnFDFAJS/HDu90PHerVGSsnR4ojdZsNoRpwz7PcxIumcQWlIc0XfWcoyQwgZXTp6GhMuQiCIfPEoUu9ZHk9JM8dimVGMnqoscG7g5mYNQTEpzxBIhrFnGEZWqw3HyzNOT07ZrPcxnloW5EWGDxKd5fggaLoxlpWio3DvYL3ZEUjJsxJPQCqNc5bl8dHhHAUbDMYOtF2DdYcIaxbF8KZugLiZ3W63WGvp+shz996RJIqiytFpRts25HkeHU3OxTIka8kO58cf8ui6eG4H71mtV0yqCYNtSSoFgyCRSXQNjyPD2KGkwGvDp19d8EHcggiUE0FWwiSrOGJCP4xIVbHfbkk//zxuvtKUsizwwfHu/QdQ0DU99bYhQROMjcXjaUWzqdmtdhgxRHasj6iFwXZYzEcXV9MMdO2Wtt9zev6UvjdYGz/zPM8o0znzyTI6qAqFVClts0WJhKzIOTldohNwHu6va96/f8d2t6JuWoy36Ezx9Okpn3/yhCAtRZ6yXhuMdRjnmc6nh4RrgrMG0jRG4wlkacLp0THL+YLL+/dYbz4WjfkwsNs9gBpxwbC6veXZJ5/hjD04+WKqQklJUWRoG9fpUqVImVGVCWk+p+tH1v2AVpJUSQbv8eOI60eq6RRrLH3XgbOHBKOklVu8SNB5zoijyBOk8HT7FWa0DENHVeQUjOTC4b0hTQT15gbfdXz67FNGJ9jVNU3bU1UVSZoxehfvS2akqWtW6w2b3Y4gBOW0JBGCPEtQPortvu0QCpx3H1MYSkkSpTDOEIgOvpjoG0l1hjeWkyenBB9TBpNqSpkXHE2nTIoSL1vqzRY3GuxoaNqW7X5Hupiw2axI04ShjnjAPMnjzlsohFTU6y1i8BxN51jTs5hO2NyvODo+Zjads91vkUryxRefM3Q9SirmkxEzGhItD+YhDuevj4gra+iGnuvbWy7On/Hplz/i229+y+XtPYtpiQTyTNNbi04Uico5Olpw880leZVSJBmjNb/3uV0mFd//3TtmVcanL87ZbLYUVUVVTrl5qPlwveVoljI9Knhy+gnf/t0d/+0//zO+e/trXr+7ochz9rUkzyIycrSeoet4WDcEEbh4NqXrPc+fn/Pqt2/YbkZ+/Vd/Q7ftefX9S1589oJ6M1CUBa/fX/HJ5zNG4ykmAdePBD8ymxbcXL/j7OIM5+GLLxf0w4bz8wUPqzsuLx84PZ0wjiNJqthsNjzc7zk5mWO15C//8udYa7lfXbPa1hwt51xd3fDJpy8OBeSC2Szn6OgFUsI4NhACbbenzCdMpxWv375GiJj0G/qRzeYOnUom8yneBqbllNXNHZNpjrcSbxzBBjKV8cXXz3n/7g3eCz58uObp8yP6vmExPyOfpfRJS1/3TF3ORZVj6hqGkc27e34cJE/lgv/uaMq5dEyERgmJ0ALpYeo8n5wUnBVnXFjN/+X7K2yWcHZ8TN3u6OqGqzdv+Ysv/ow0gPQGnEElkn7YYaxGCxufE7AHZNmqr3l7+4HeW37xzVumxynGey7OntLUDb/41a+ZzjJuHzwn58d8eN/S9gNHx6estyMmbLHCsh8asiLn9PgIKSRFVvLr1y85Ppnw7beXfPX1EmMt+/2ANbFgUsuM4PZMSs1nn3zBt3/3krv7mkkxY/NguNzd8nD57yhKz2xR8vz5KecXpyR54PZ6g9YTTuef8uMfJ3y4/g2duSeTChd6licLmqZH6ZTj4wuaXc2uHzh/ckySZIyj51e/eM/F04rl4oR/+ud/zPXlmm+++RXhbiSfaZ48qxDacr+6ZXmyIPWBzuzxYmT0htPljJv7G5azgf3OkVdLptWcJFHc3Dyw1huWyxOms4S3719SzSZYGxiGjmlR8N2rlxAaVAFD918X2f5LDyFDZJFribV8TMABKC2j0TY8CtyeEHR0nh8QGEL4H8ox+cG1Ht3Of/+1BeJaTyn+nhD66MyNIng4iNix7DLiVR4FdI/AI0V06cZKHR9NlnZEiICQ8oAbibK/F/G+FYXwHwYDj+89utajkI343TT9oyvZHxAgsbD9kbkuhEPIOAT2Hrx0jGPL5fUbzo4T6mYHBKz1NG3LbrdhMikZTTR1TKY5Mm6lcd4iB4GxHVomWOnRKkEIibUmlrPa+Bk9usOBmFJXETkTQojGlC6haerDut+jVGTcS6EPa+LYpRaTiQLnopi/30e2el03VFX1MTEev0v5cZ2eJhlSSqyJ62upkojveUw7HXR05xw6y/jzv/xL/uNf/Ud+9atfkBU5IsT0qFJQVrF/yzlLkiq0l1gb+5/u7m4ZbSzeLIri477xd8X0NI3J0rg3iyx2QYgJ1VyTFwqtQaqA8wZMdI/rx2GI5+DgjymA2D/4uwkGEfe9wR+GHeEwLHpMC0T8i/f28DlFjnn83A7HbTgMIVTk+wsBPtjDoCocjtkA4YdjLaYqBEL4///iXEKYIaRHuRlTndBLhwOMvEMFyaa7JUueMoYNTX/JIv+M8dAC69Wa3gRKKTCUhGFk8DsSLzjJHXU3w+t3NENP8CW5+hrlDdMsQYQj6g4SMSUpO5rhDamcMlHPaPstiS5B1thgEO4IlfSs9rck0qCTARue0423CNWghaL3t5GrHRx5OcfYe3AltWtJZImQkKUTEuY4V9P616AyQrA4v2ae/RGlHmjsHu9KpHYIeYMPBcFfoOUeZyDTCuN6pGrYjLckytD1A5V+AragH0s616DDLTrxNG6LpMKrBhMSjITEpxj2WGcIVpKlPQRJEAGUQipPkRX044hSgiBGjIux5iLpSJM9weXo5Jh2eKBTvyGQMpiaQIX1LT7UlPopefcpwkbsSHSex02OChotNRKBNR1CZrjRg4MyP+J+8y3T+ZQN/4EhrNGuQSUBzQQHDHZNpo+wRiCkxAuLcQNKSpSAVCo8OdbGyIqVgcAFo1uRIEjVKS7sMbxBigwf9vhQIqTFMWB1TSokXkqsiGLjubtAM6cPv8X5kVSeI0Qbp1cIEmkYXI2QE6QQWD8ihCb4WDI7+g1ZMsO5AS2TQ2zm0V3vSZUCepRIUZE1hPclLgykwlElR4QUkI7eekR2gvV7jBtRcgoHRmJgJNFFTFP4lCAP0ddgkTLDeodzgd4kgMOaDmMr+lHTdp5m2LOqr9jWI7s6hX/6Dz2b/3fOb+PIqoTORn7xZJIytAN5kvHk6RmjCxgJ97fX+HYgqyoypbl48YKbh0vasSVF4gbP6WROWsyQXrEsn3C7+0A3DIzdHcfzE6bTCfP5hPknM67XV6zvDW0/sJgcoWSGTiTb3T2r7y4xrsX5QJrMWEwvUHJNNYf79TXWCl5cPOdhE0s9sCO77ZYiTVlkBedn57x8/4r90FJOJ8ynR+w29xwdzZhVx4Qc1g9rtustWkm8Gzk5ntHWDUlI+NGXX3H34Z62a5jMCrCWdx9eM8ly2n5kcXxKZw3H6RkQUCGhXtcIH5uwkyLjZnXL1fqaut6TESgSSJIS62JxYZCSTb3nfrdB5QnWO27vHsgvSi4uLnj1+i2vr94zT6cczRcMZsBYz91tYFaVBNPjxi5yzJWmmi0QSiONPbgsC4oqJQRHNxicU7x+/Yq//du/5rff/Rqh4MWzT5jPZgSiQzVN1OGm7T+2WQvihjjy8uIxo0TC8eIJzy8+4/t339C1I9vtjvlyQTu0dDc9VTlhMp0ymU7QqWI0A2meMl9M+dXf/YKnZ2eUec52/8AXP/qCX37z1ySt4HL1a97+4i0v331HUPDk9Dnr+y1eCbIkwxpHWaTcrd7wcP+O6WTKs+cveHf5jnVzy/MXzxBJyst3LxnMwP3DPXfbW7588TlVNeX+4QE7GE5mc7aDpZhUXG/vcM5T5BXBAy7w1edf4Y3hZH5MdjTjZrUnyIbVdkO96/ni+Y9Jp5rb60vSdMJ+PbIbW/7oJ19ztbridn+F3Rr+5Ef/nOOjGX/zqz3Ce4o0Q6M4PTrl65/9lFdv3nC2WLJlxZef5Vzd3yDxnBwvUEnCarf9/U9u4sIzzzNi0Y0lBE/btlhrsdailKLpWqx/TCFZun7AWc8qrMh0ybpuyHPNYGpEcOy2G6rJkizNKMuCut4hJXRdS5bF61rfjzR1z/nZcy4vb5hMF4TguLm5oihzxnFA6xJj4iLp/fsPHB/PKYrycLwZnJO/45ABRHSz5EXK7d01QqRUVYUxA5NJhTEpr1+/ZRgMi/kxgiiOSplijY3DTBfxK1ICQTCbzQk4xsEyn88xI2zW9wTgaDmjOLzfgKdre5wJPH26QM4Sbm/v8V5S5JGDvl6v6YeccTSkmcL7WHo7miiYV1WOFNHNHZ3d8iC6O7RKyPMEpwJpqpjPZ4yjYrPZR+6zSmJUVcBsNqXrJdaOjM3AfD5nsTyl3nf0/UieVVQVFHnFdFKRpv3HmOkwdKRZirUGnWjSLKXe73He0rR1dJ87exA0I5tea4VSsaS57ztOTk7QWn7ElCAcbd9g3MBgOvIsI0kinxIB0+mUob9nGHqOjpZIKXn16jXD2CCVBRlwQZAkAnzLaGsKJN4eyo2nxzjraJs93bBjGMyhyHROnzim0xlSwvHxMVJB3QZcAPsxwqlJ0+jQT5LIrlxvNkynsZQaIej6niLLsGakaxuGsaO0BR6PTmNps0o0XddT1w1KKZJEs92suL+PKYvZbEpRZAiZsN6s8D6WVeVZhtIabyzeBbROmFZTiuIPi4QPfUvXNZRVyYcPa1SSYoJj7AeKbHoYzgtkiJzyLEnoXItKJRefLXHeIhQEbcnKnOPlaeS973r2bc9qtYpsUGsoJyWz+ZT9bsfD/X3sPapbJkWFEpLnz56yXB7x299+w+6+RuSe4ydTjB1RWjD4DrQjy+Mx14QhcoBtYBzjOfkYoS2LCfPqiDzJMaZFJw4zGqST5HmBH2Fzv0elcdPV1T3tvsNLCwTK6QSr4MXzp8ynBb/67S9RWULQkiEEytmMIBW7/ZanTy54qyQqkXgCxlm6vmU6lDx78Ud8uHvPOEbuZ5pFzFnXtaRlwNmRNCto25pqEssVzWgBQVFWbPZ70AVBaQySQMQuicFhrGcyq9CdxluHDpALhReCRArMaBiHETP2iMOAbXAeleUo4QhKUgrP2O0Y2556s0Z6y6rZcbqY8tnzpygNfbNFuI4vPn1GkqTcXz1QNz2TyZzJtMTLONTxBPph4G6z4fr+gdE58mmJLDRGBIzWSBfwLsasdZIgVcQBGhPXr/aw0R6N5eTkmN12z6Qq0BLM0JMmmv12iwhQFRWTvMCbwPfvX7KclWy7Gusd+7ambpvYkmEMHJ6zrtuIq5pWBEk8p1xgaAZM3ZOG6HGr05RUaNzg2KxWvL16x/HRAiEcR/MFk8kMP40ixTAeCte0xrhYLlYe7o3GjAyj4X694uw054uf/Jir92/p7IBzHeU0J59WpDLFWcfl1RVKq4O7MMOa319EH3Zg2uh4jEPPNZvdjkk5ZV+37Oua45MJiAEhe/70Ty/49a++4eQi4aYGYz2fffKC9aon0Y6T8yW//tUlJ8sTVusNq/sN50/Pmc2mfP2TL3j/8pZ/9s+/Iifw8uUVf/onf4LUBXfbNzx/kbBvrwnlNF4XsxIlPGZs6Yce64aYyvE9TbsHUXB8PEEnmvuHNV1n+fT5M7qm4+x0yu3tHXme4kwXETpXI9vNyHymWa8GxvEtT56c0HUxEXZ9teLFi09QGYRgOD09Yugt01lJ0wxMqhlDH1g9rMnyDCkE797eEkRBlVgmRUUiPCrPsWOgTEteX6/4058vefPmG3RiKUpN120Z7UBQFjdYzGbFicp5lmrkrifzGQtV8aO/eMYXOiVd31LYFTQeqhS5OMFnEut61H6H3G/4MaCThLe5ZphnyCJgvORP/ujH/JM//3Ns22KsQsm4Luu0Q3nPtutxZo6xPpb5SQVSIFNFZw2btuZn/+hrBmt5//49Kkx48+oVX339nKZdY4Lj5ZvvcU5wfvYCrTIyranbLVcPl2TSo5MoFGppUTLw5ZdPmExyjo73VFXFar3heHmMJMOMgd3uBilHjmYJ71+/4qg64vOLZ1zdfss//ovPePuq5W/++jWjcVSzgtWm5tXL7/njnx8zmNiN89tv/4bl6RQXeu4fdpweKbJJydAPLJbnrFYNfQ3T6hO+u/o1rXnN8fERx8tT/uy/eUqelly+u+ff/L/+F/7lv/ynLBY/4+bhMh4bynN9u8GGnElZ8PSTlOvbG8rZFLRiXe84Xc5YLDJ+8ctLvv32gS9/POX4pOLZk6cY49huV1TFnHk1Zxg7+rGPA9Fcc3qxpGslo99TFNkfdO+OWDf1A+KDRx43PLpkH0XByDV/RKvEn/9Q3CnwPl6TpXzEovxvROuPa+go2gshPrq8HwXbj1iS/40bHX5AughAhMhDVzK2xykZn8tZhyC+Dg+oRB0QKxHJkqbpx9csxA8s6sfXqLX++HsfhWkp5d9DzMQ3E3tmrIkJ/dEM7N0a4QOXVy/pu4AZHUor3r97RzUpqeuWrqvZ1xuMnZDnCUoHjBnQScQcyrCiKhfkeYEUPwwXpPwd9//BVQ+Bvm+RMuJSvLeHZOnAbr89JFraw/coKauIu7bWgDgMAA7lpUmSHlzpOdYecNRCMo7jwdQyi7/bBdI0i4K2tUilD5rzI8M7Yl2SJDLY8yznL/7iL7m5ueb169fYMSZYj5ZTyrLkEcE5m80Yx5G26Q/fdcRvOu/5AfsTDuvd2GEZGfHRDDP2EfestD6gXz1ZltK2NTJxcFiTWSP/XhoiMvn1wWV/wASJRwk8diFG5NGj0//xuAcpVHx93h0GNzImI8Ij9uXwRz52ADwOiqIWKYlluuKAEXocFEWTU2S4R0TOf+Uc/q/+i4/HbMlULTlSn9HbB7ZhixMd3s6Y6ydodcHOf4NSHuvPIBxTaOjsFd5pJA2pWMYPV6d0fUqVFRjTMMuf8qFZkWpD0HsKVTB6izMdWub03iKGexT3KDlHh8BgonCaqjXBpxgHlQ5ICkZ1SpoYrB8IwdB1GVUR6MOGTCqyNKUdA1rtSGRJCJI8lbFNXBUY4zHiDuUluTrFO4cPN8ggaPs70uSIxHuc2qKCJucnIDsGPxIwFPmU0aQI4SBYpFrgnScjp233TKs7JHPKFPwoUCEjsMQ4Sd8PeP2OWTanG6/px5oqPUeIinZnOHGTQ7TVI4VC6QTrA6NvI1ebMzxrWvOeJIwk4ktCsAhpMb7Ahw6lZxA0IZSkSQ4+RfZzfAgIpRHOgYAiycE/HuyGtt+SJVOUSri/u6U2b5nMFmSzngdekiSOrrvHhBmZ7JEE8D3jeE+SnB4mmQGhaxIlUWFO8GckypOwxPsjBrvByilaTklES28DDo3E40OGt0PkncoCIdaQroEYBXLK0w01boDj7C/Y9P8zUiwIocP4Nd4JgvckucF4i1KGkY5UPWW0N2TJMcLPCXyFEC1BjIwBbNhFlr/MkTisawmkWKdJZY+U1wjxhIDGBsNMXyBEwsAerT1CdEg/I5iAJIrtWSIPg5kRFyoEJ/TjWyDB+RZjtlgL7dji3YTO7tjXW/a7gqv7mvXOstt7dl3Aupxh/Ieeyf/7j4uLc0JqMXXPJM9BZEgXeHZxwWazYXl+wSAC7g5kkuKFwAkY3EgXejbNGtMOJGQk8yV1s2dwHp9otEw4fnJCs2tASaw3XN1cMpiazm/4q29+DTqF3rKQBbnSPPt0ST4tuXu45+W3rxEuIZUF82mg7j5gbEeQJV4LZidHbG4tJjgunpzhtx2+67i+vmS73yBzCdKw3t0ggqCtBW500UGfFzRNjcBxdrrk9PyY9zeXKK8xXc/64Z75fIZWmn7X4zW0cmAfWrargSKbcNPeo4Jis1kjC81qtSGfV9RdTVamNG2DDYbt+oGTxZwMhQfqtsUGj/COJM8IKqASjfWe9X7P/tvf0vY9wzvHT55/RdPt2TcD221NmRV8/vQpptljhpreNMwXx6g0w3iQ2YBMciok1uZsNmu8l4wm8O//13/Ht9//HVJD3ex49/4NN0nysSBxGC2J0h+jhI/xsSRN0ImOzFQVb9hapnzy4ktevXvJftdydvYEpRW71Z5n5y+oygmBwGhGhnFkHAe+unjCN999x77eYpZzmtbg/Mj7D++oqpw377/n//4/39EOPcYLynIOQTJ0I08uzsjSOI1XGk6Op7hx5Gg+ZTY/5dW7N6w29/zs519zv7pnU2/QmcZKR2t7rDWcHi1xwbPbbrHec/HiOTd3d3Su5/TojKIouHp/yWwyZ1bOKPJYGti4QDGdYE2PDbDftnQPNSezC5b5HC1LgjEEKXh/eYlMLaMcuLz/wPT199jhgvSAe1BCcn93z3J+zOX7S2bVEU/PX/BwfUeeZJwujzHDSJ6m5FlFqn//E1zKmIJ5dJ845+i6jhACxtgDL9ZTlROKrESLhL4bD4vX9OMC21qL847l8piu35EGQde1HC0vWC4XvHrVs9ms0TrBWs9suiBJMrouLoSfPDnj7du3HB0tyLL0sDD19H1PUSyYTRfs9zvads/52ZLpdI6xNjLNDxHIvu/wYaTvG8ZxRDkBwkXUUKopqxxjJNWkJISGyaRkGBzWjnifM4zDodndMwyGtm1xricvJXkeC6TLsqQXjuOTc3a7FSHEfx+CROsEYw0PDxuUKsizAmcFeV4ynVZR0D6wEOMmJeDcGEXVw+ahKAq803R9dF94L7DmsHhL5MEVEej7ASEls9niI3O973uciwVSSsvD0ECz221p25blSYwstm3N7Z3gqU5ZLk9xNkZLhfA0jWUynbCvd1jnyfMsulETxTDGMtVHvrFUAuECiVYf47ajMR8XvX0fkwdlkUOAYejoRoMPFuuikydNEw4+KJI0Cmh5njGbzaODRVmmZUpeSoSKiJx+aDFmoB8USqYsFnEQ0NWaoddolZKlJZAw9HFA8uT8GQEXS51MC8Tf/8PGjNjrciiDjag6RwgjSaIOnEnBaAe0hH5o2WxXpPk5yBi1TbI0iprGMo4xVZBlCiHA2iis5kWKTmJEdLtZk+VTkiRlGAymbhiHAYHgZHmCsQPj+Icx0aVwtE1HUc6YThd4L8nyin5scSFQTSJuKM0Vo4kuYSsM227DyewMMzjavsN2jsF4gkkIAyQqwZk964cVWZKyWESeubGWk/MTslyx39bIICizAgkcn5+AFux3HamqWByXXDx9Qt0N5GWCl+PhfM1omoh1qqop1bQkLyp0CsbtyNKSopjinaAeaoZxSzdsMLZDkuFGRVFV1JsdQY0xomssRVZhGfFyZLSWqppTTirevnvDw27N0YvnjAqqIqMxI2a3QwZHlmiklgzWMFqDFYKsyMnLAi8Ex2dntFcNSIF1DqU0oxmpkpLtboUfE8zGYWzFcj6P4nQ1p+8MO28ZfYtzUOQpZTVFpRmMHTpRVLpCK40fDMO+wWlNH2DoejoHg7F4P1JWBRZB0zWoENAhgBIUWY6pG/Z39wzbFcr2TMuMZ08+QycCFQKJDiyfnpPmOferHS4EjpYnzKYVCEtvR+w4YAmMznC3XnO72eDxeGGpe08/DgjjSb3k7OyUo/mMm7trNpsVyBSJJAh56MCIfF7vA0+enNO3LQLLcrnAWhPTM1JTFhOEh/u7u9htMvZYCQ7B7eoBYeJw7HK3YnSWNElwztN3PYvFgqBgMpnRtT1FWtJs19H8kebgAi+ePufy5o5td03ft+z2PjoQD8IITqLShN4NWGsRRG772MXS5bKsmBQpOs0QScm+GQgyZXG2ZOx2dJsesRm4ePqcs7Mn3D+sePP2TdyAi4Af/R9ARIe7+zvOny9BO9brNUpphNTc363xbuCrL45ZbzaYLOPEB968f8ezF6fs2ysmk4L9zuGCpZoIrj5sSRLBz//RJ2Q64Y9//in/7t//LdvthoeHB/7yz/6IP/qTFyynBS/OpuSF5PuXv2K6yPjuw57zTwpC37J+qKnKEucst+tbqmpCMUtY7dax8N04yklJb1qa2y3TyRypwFrJ02dn1PuGX//6HcenRwympXc1r97fM19OGYWkHXsm84LZZBLdsAjSJOXJ8gl/+5++5Y//5DOq/IhU59w93PCwuqTMA59enPGbX13yk5+coRLNy1eXVMUUnWk+e/aC9+++w3vHyckRY++RKsN5wWqzJS0Sru+v47FwGCx///INs0lBERqWwvHf/eRrPp0fMe62pKMlqXv0OIDosIfiW6EycBLfK6yXECyZNejB8DQE/nRa8Feu5e5+Q9OCL1K2mzvKSpC5kfl8hh0sZmwJwTKfRJTg3fqBm/UdT549YzKfc7e/Zd2vud3cE/Z3BCR5NuE//Ye/5mG94cXnT5Fasd7fIlVAqIT/9J9/zY9/9AXVDG42l/RupO160jTj/Oyc/WYfrzN5xTgOmNFyc33HarPny6++QkpNlZfcXlkEIy6NKb3RNuz3a3789ae8e/9b6i7hJz9/St3WHC0n7HYti3mFMhnX7z6wON7x5HwCBAY7QTHn5m6PcZBmE5yH6XSBxvLLv/2WPJfMj0v23YbusuH5xaf0rSPNUnRh+Df/9n9keXrEfHFMN45cXl4zevAEGrWnsT1K5ciQkeoZZZHysFqhleLP//I533zzit1+TV54buuOt28f+NnPnvL+7Zrjk4J3lzdMFwmb3QMEqPKKcWhYLCv87vcfkAEHwVH8jkgYDii96OyNA+UfChsFklis+PHufxALA8FH82NcrtuPCJBYAiqjMfex/PPwM3EgDDwK9dFsEAeLj6krddjrSSnAy8hfPyTx/WMhKABxuGqDA+HwAayTh06YiEUJgVj0KMLB5f4oVEu0FAcSQBwSPArp1jqi0BxfwyPaNhZEHgpUCXT9Hi0FgzNcvl/TNoYsLei6nul0xma94ebmGpVIum4gyxXykAYtypSqGugbT1jGzzpNYybNucgn5+BSjoMKgVISY4f4mYc4DBiGuMe01jCO0WySpill4SiKAq0lfTcekJNF/F4Ih0FK5NU/9kJJGb8fY6KTP8/zQzr80RXu0c4fcGoSQkRG/jB4EZTlhOPjU87Oztlutzzc3R8GNwlFUVIU5UHwzgHJOLjDzyqQgaEfP34P8ViIDnTv/UcxHRHQIv1IaEizlCTxKCUZxoHMBISyh2PWEBSH/QOH1+kIQR2GNf5wDMfj0LkedxDRgY9pYxCHgtd43B8OAx5Z84Q4CPAhIA7/zcElH8u+/cdhDfyQxvjYORAcznmU/K+j2P7BIrr0GULV3I5/SyEuUKJgohVeHrPQP6UPLzkOT0FesOrvkSFBSkfKGT0rcBNGL1Fpj2KLAm7rK2ZZSe3+A6W+IKgJ0jUEkSEDpHJJ79cIn+CdwPgzlIiRyDRJSFyKsdck8hiVlhA+0I1TisQzOEcaCsbwQFYkaCkp5YQgtmgEmQ6g7hn6I5zL0D4l1wYhppHpo3MmaULwkkHUZEwQckTIiq29J2FAMSURCuvW7Md7pEoQImXwG5Q+QoQMbU8QcocTjirN8cFQDx78mqmqkMmCNtwitcL6hCAzcr3BBcVgBryz+GRHOwhy93Oky0CoA5A/cq7SJAGv6ZyFsAUEWfYZ1r7Hyy2j8VgHQqQIMcH6EcIWIQuc7xDmCGwRRXmhGENcTEoZyLNYguVD4Hr1hjydcHL8KdUyY7e9ps9eQiYI9gYREhK9IKEEJDJohFiAGHA+iglZmhD8HJRjDO8RQjOM16RKkegLJskzRvdAkCUDLaBJVYfzCcEn6BCLzkY/kKgKoTVCgdKKRAhwlr6tOS7/OZfhj3DJhtE+oIQmqB1FLkDsCOFHpKqmswYvhui0cndo3eO9JjiJlCWpDFgxQYo9icoZTM/gNkgmeKexaEp1xuA6AjleWHbjmjz9EkOKCDu875EsmSQKHxReb7G+x/iOvrcIWRN8y+gC+2aNsQ1N2+B9yt3+is0G9oNns3HU9Z71PhBQhKAYRodSAfjDbuZj0nF6fEbTNSwmR6Akt5sb1t2eh/2WpJoyArNiynQ+x48DbuzZPrRMdU5SHrG2e8pyTjsMpArsaFkoycnxJ9x1O252OyZzybDrmcic4+SCdtfz5uUDp6dHONfxd1e3zI4mpLuS2cmM3XpHkiWUUpOqkXq/xuKYTM5RcsGuEVzfXeKFZzKZ4MeB2/UNn316Sm13kAiSLKVt9kzzGZtVh54qpOp42G5QaUpWVCRpjneBD1cPzMolgoBhJJQOZoLb3YqknLDfDHjryYrofDXDnvX6EmcCprdcnH/BdL6MTt6m4Wg6ZahrvAgUyymvrt9zunzGxcXnzNc1frBY4bir19GhnBTMFgt0krLb7zHB0+8faIYTPlwP3N5tuLldczRbkulAv10ztA0owdHpBaMd2dc7snJGVs5xQqLTlL7vSfOU91dv+Vf//v/Brnlg9B1KKx72D8ymM1Q2jQUvQZDnGcZavAOtMvJiilBpRMYEkD6gAJ2knJ8+5cWzH3G3WrFt77m8v+HsyZIySxn6lrvtimJSgiKKic5wMptxfnbG61evyIuMP/+zP+Pq+orLyyu8cFytPzCdL/GDIlEFQ+u5ePIUh6Ndr/jZ+U84eXLM/faOXbejDx2qEFzefoco4PrukiRojuZLHrYbtEw5nS7p+prL27d8//I1NgRus3smWcGnywuOkoqses78rOS377/nZnWL05KqqvDBMp0G7rcrqjzn/Onn/PqX79hv7yjkM2ZJzvXDLU/PLxgHwdHxgm275na1YbSe0TVc3rzn/fU9LvFk8xM63/LQwHA/8MnzF/z2+x3bdkuW5+Ra0XYtd+2Wsu1Rye/veDk6mtN1HdvtlqqKTOroQHdxUZRkmNExny3I0wKcY7+pqbJJdEUoTVlNGQbDarMFVZKXJR5H0zQfo+/n52dsNqsDDz1lMpkxmx3x4cMlV1dXZJlmt9uglGA+n3N7d8PR0TI6GUbD+fk5UkpWqzuGwZFllqbuef7sE5Ikx7kQGeZtR5qnGKcZxz7GMaVEqYIQHEIEskxBiIVXIYz4MLBaX9M0uxg/1NUBVWRJswQpIoolz2NXx36/Qx7KftwB9zKfz2McfR1RJk3TYk3A2RhbtdZEMXpUH10Ww2Cw7pExrg+bAEmW5RwpRdvV1HVLkqiPJZ59138Umq6vr/niixOmixnX11fUXU1VTUjTBCkTrDPYscN6h9RJdIZj2WzXOHdgYR6isc7ZAx+1Q+vIau+H9sAzT1FKkCSKqirQWkXEyjiQ53l0rXt7KBvqgVh+GrwnS5IowA89w9gxjj1CxfIfrRVZljGZlDhnGMaOYezYbB8QwpOkkKaRz5hlCutH2i4OSIQIOGtxBM7OluRpQnA5XWsxdqRKNEU+QYqc6fSI+eyEtt3R9puPfEoZFCJYJCq6JvsDGsMYlIr8WzMEBtPTjT3WRQ56OZ3g3MB6GxjGnjTP0GlCCAJjHNaawwBoPKQoohu/7zvqvUTp2HVinEWIAq1i2q7vDXg4Xi6Yzid0TRs5+H/Aw1iD0ophGJhO5hEFJwPz+YIyL1nM5ggJm+2Gu7sbxn7EBYEznqq0lLMF67qjq0f6fWB/Y8CAKjWDH1gsFiRZxtnZGd+//B6pJUWZcXZ6ynQyY32/wRkbj5tCY4Xj5PQUM044f7Zgupigkpo0VxjboKXAjhKlRqbVnCxNETrgQsDUHWawKJEQXOD2/uYgyu4ZxgaFItiU0Xuq0zld37Ord3gsWZJyNDtis19DoqLjC7i+u+Xy3Vt6DLu25ej4hADsdzuwjlxr9tsdSRK/L50nqBCYTSakVcr7m3e8+PQTTBhou210R4coJgyHgUhTb5hMFzivWa9WnEyO6XvH5mEFRcmmHZjOjqiKgqAUNnhkotDEUmdvHVpLCIE8S6mbA5bKa8bBEIQDqai7jlXdIIuGxfKIVCnGNkGakVkukapCGcknT8+p8jQef9KTF8XHktskyygqTeo1s/mEut4dXNgJ7Tiyq2tu12vW9R4E5JkkK3LquiYJUNc19X7HdFZyfnHK2dmS9XrP3d0DxnrQZRwOukAzthwt5tzWW7yznJ1f0HQ9idIsl3OcsXT9iFIZk0VFZ/fYx4IvAV4EZos5d/0e0zmsie46JePQS+cJ/dDHVMd0xvbuns3Dik9fPGdalmzqJg5e00BZZPFalmikkmx3m4gFomB0hu1mS7uvET7Gxb0LZDolSxOyVOEVIBy7ekNZapJccXS8YKoT5tMJRZYzqapoGrCRESx9+ChG/T6PYqm4290wO5oiQ1QHxsFwc3nP8+enHM1Lhs5wfdeR5S2TYsK+tazXntPnE4pjgbUtSZLw5VcXTGc5q9U1zuzgVoMYSdOc+ULx/atfcnH8nBTJN/U7ivKIbrNmdbvhZ3/+lME6bl/vCEYwtpDmKTIXbLoNR8fHcRiP4sPlCucH5ouEJ+czbu4emE0mPDlf0g4P3NzfojNHNxp05hip6f3Iu5evUWnKdDLB0TM6SV4s2a5rusZQnQT++//2j/m//t/+LV9+9RlPPzlmUrbs9zcs54779y85mxcs5ymj83zy7MeoVPPq/d/wV3/91yyXFU0zUL9+T1VN6caUYlZxt63J5wsedleEEYbOUpWK07MF97cPZHieLXNehIF5dw+Zw9R7xL4jWcxpBeRqgggKsCBqtJPIocONNXgXxaWm46dlybfDwNVVQ1ZM2bQrfvGr/wxf/oixmnO/krSNQWJxZsPp+aeIrODV1SvWwwOXwweELmhdy6ub70hSjTOespjz+s233D7EPfNuW7M8z6hERHsN7cjZ+ZTNZo1NFLXpsVaT5zOU0KzuWvY7g+1WrO9aTs5m/Oznf8L3r79lNlvEzo3VDadHCybFjHEskMJiaUAOvL/9jt1QkBYZ67rBe8uf/unPkQROjmbU6zuSvudsXpJkioKcVC54dXODCxIbJHf3e05OKrQa6Jo902pOUWj6vgY3YzGbY0fLbr9CywSVdkxnBebBs93VBErMCM+efMl2f4PxNcGNzKdHTKoThk5yud7R3Ld4IfG25ezE8fx5jveCly9XfP7Fp7RtztXljvOzOX038NUXP+Hd5TtW2z2amma753gxQ1GRqD8M5wIyYi3gIH7LAydd/o5Y/Ch6RzTWI70j9rfxkfUshMe7WMQZgvsokv9Q9ElMvR+46Y+s7YjnkB9F/Ecx0Rh7+LlGqYjw0FISXGRKexdwRAHfOR9F9YP4GIipLZQ+dOsJCBJr4ouPidJAZKV7BNEQKn9HtIz8dYkxBucCUmpAfkTDeP8opAuCd8BAP6xRckQlHdU05fbmGkLO3d2KN2/eEk0sGeuuYXE0xfkBY/povhkdUrQHbLFjOl0ghf74OXgfcM7w6JNWWmLs49ovfgfWmo9pgb7vGIaePM+RQrHbb4gDAkHfp0iRkCQ5PjisdUynE8xoKcvio9Adeff9IT0cDgJzOKABYyrl8GF+xHtDHII4HxNhJ6enzOYL1us1Qx+7jPpu5P7+gaoqybIcZwNSqINw/QNG5vH4eHwIEfsRHv8+HqOOVEejEwfUi9bicBwfUgWeg4HLQvL3UxePyQut9SG1az4mJKwzH49LJTVCxR4rKTXBE/FC8lEI/+G9P+ISrXWH13r4mA5opODj2gLnPzrRHwdZcZ50EPf/Aef3P1hEn2YSg8f072mkiYKPDmgWNO4DzvcEeUPGC1IsnrckdkEIzyhUwlElsUYyVU95CN9S5C0zO2ftrpAqo0wqanPJdn/D+UQSGMl1QjO0LEpJNyqq7Cm97dkOW5bJBFSgtYHCBoTvQB1TjykzLZA2pbZrgtqQyk9o2wZVzshUHgtNlaMzX6DDC4r0Aec6VNhjeaDMzhn7HCfABAvBsu8fOJqUeCTWK2rTopM+ivp2jqSgaS1IR1APZPoBVELrM1IPweYxHiIusWKkTDs6b9HsgYGmnrKcJAzDPdZZjG0JeJTKGY0jEQO4GVJNwQtCGD4egEHGyY4MI4E1eXpEZ3oEFb29Q8kzhGrRKqfpt5QYBDssZ7R9j287Fo7IYTrE/ePkMZAmkRkqlaScZ+ztb8jCgFEj8xNJUCMr8xKtjjF2QuAehcX6DscDWgW8nZCpBYEdIRg8A96dosQTDGusDeSc0fX3iLQl0x9I5AmdbZF6ghQa6wNFltB1FuEDwjd4l+B8HSdTUqE9sTzRBzJOmOk/Y2P/JwRTEm3wNkY1CQOKDulzRDA4YUh0CmHE+2uCOkKEBUGsIxPX36HEKa0ZsQ6y5JzgDVJoUnUGwoG4QSNJEo/1D+z6gNQKIXdoFKNv40U2KMx4TyZe4ExJN1zTjq9wNqFuWzZ7za5uub5u2O1THrYjnRF0VrHewmSSMo6GLFXxAgPMU812V/9DT+X/3cddfU848LvW9xt0rrHOsO9qBjMgEZwdLRlHR6kzjPN09Z4yT1hM50zKirycMBjPfDbDDo5ECt6vPzCdl9T7BtlZ5suCtkowTrD2jvFwMzaDpdvvcW3L9f0dcpbxo/QzzNhT33c8P/4E64fIHJ8m9INhUgjEofTGBcPQS24+XHOyWLJrt9ze3nN+8SmjGREh8OT8gmnqCG7k+v6GrMjwUlF3LWVQzKoZfT2QJjmj6bm6vaacl7S2px0HdICzJxdY23F/d4MkRqHsMFKVMzZdzYfLS86eP2N1t6GcTNjv9jjvaW1Pmia09Z48a0nu78mThKrISScTuqFm2LXU1EyLOUVRMZgRlWr6oWdT79iPezbbhmZoCVvPy3eaJAS26xVFtWAMsLq/Z73dMl30zOYO6x7dxgOL4xn/0//zf2S1ucWJEUcUX0IItG0bb2LGUhUlzvuIJpjOAUkQj43o8VrTtz2TSWT86izh53/0j7i8u+T+m3uytKLMJ0yqKVfXVzw8PLDUIoocpyf0XYdzhuP5KZ9+9gl3dzesNyuMHfmn/+Qvefnme/Z+hUSTJwUni1PqbY0dBz5cvWcxrVAqsN2vubr9gNCStmu5vb3DY9Bpytu373h6fIENEqUzEmX59NmneB/ZwGWe0RqLJWClRCSK5+fPacxAkiWcnB5z9bDCh0A3DPTdnt1+y2J+RN10jL3jR19/SZ5o8lJzcrqgMS2j7ZnOpnz7/bfkkxydpPhe8LBfcbI8IZ9OMBiMM6S55uHhnq8+/5Kj5ZT1wwadKLqu5eRkSdvuSLOUybzC/iHlZFVB2zX4YNnvd3g/oaoqrHWEIMiyjGEYuL29p2v6Ay9QUMwzNpsNWmmatkUqjU4kt/cPJIlHkFEUOc5Z7u/vmU6nnJyccXl5hRBwf//A0FuUTNAqcv+MMez3e54+fYLSki+++IKu63j79g1FUVCWFfv9Fq1TNusagWK/b3nx/PTgoNEMo0EohVYaVRQImQCw220Yxo48z5AqoFNiMqxKmFQZtzd3hGApioyqnLDdNDRNhw+StqsBz2RSRUd7mX9kHupEkqSKLI9NHuOYYF2KMT22NQSv8N4A5mN8VhwitcbEDYtWOWkWWeT1vqMsIwZEHljtSmnSJD/c08WB/2m4uromTWcEFMYODMNAlicsyglCSpomuun7YTzwDQ0Bh5TRLTOOA01dH/AKPbvdhq6vqaqCPM/Y7eDu7o7dbkee50ynU/I8/8hIDyEcXOoJQzdQN3vauiZNkujoPJT49H3LZrNmt1vTm5GiLEGJj+8tutY7vDfkRYKQnn6MLvbJtAIRkTfDaAlOkyYJXdfSOUdV5QeHCAgkSqUUxYSyzJhO5mRZhRTRHeMsjIONC+zg4prLx/WBCAl9VzMOMopkIUZSp5MZHjDjSNe1WNdFZnWaUFUF4CNSL0T8xjgMaCGpqgnW7mnbDmujA340Y/y33hHQGGNQ0jGpEoI3WOtZTKfRRRs8TVPTD80fcuvGeEleRCdRpnMSlZLqnCwt0CkIFRnfIgi88WgnUCiaXUejWo6mJ4QefB+wOPphhzqg79CBJFXgLUoIgnGsru9jilSnZEnO0WLBbrNiOsuwNMgy5+SzGficspJ4U3M8qUiUwvlAmeX0QeBMIJEWaSX7XYMLjru7W7w3VLMMa/bcrb/leHGE9ppl9gw/ajauZ7ZYsO921H0NQpHrWCBIkqKSAi3A+jWmXnE5bMlKjeklhZ9yNkuxQ0uh9/TWk+qMRBToRHKzviLVgbSQ6NxhdMtVvaa/HimqKcMwUqTRZTyaDjsqyqxCpi1FZui6Neu9RpkZi1Qx+BqrKpL5CaGaMpBQhPh6hwBegbAxsjy6EZkqnAlxaGo7lJRMZxW7bsuua5HWs9qvEXvD0/OS83JGZRrmaYrPC3xSkMhAleSM2xUyy7FSYF1gPilJSPGuQSWCXEmcbWI5KgJDYNMNfPv2A/e7lnU3IiQcjZbKwbwo8M4wGkjynP1g2L26JlGas7NjPv/iBU29Z9dI2m6g63uyPGXd7CnmM7a7HUEq8jRnNIax7xgArGBSTjHdQNfGYdQ4NFQvZvgiRWaak+MTHu5uSWRCQHJ8skQpSDPF2w/vWS4vKJczTvdnFGPgZHKEUIIP6/e4fMCZKMrYXiCcRuQZxg4obbA9dO2A7wZ0N/Ds/AwroBeQLHJ0kdGNA/V6hRaKxA/IQbGYTtF6RpFXWGN4//4Du90OiWRwDqQg8ZIsSX/vc3sYa1Bg7UCqs5i4ShRFEQVw60ekcvzTf/LHCKn5/re/xfRtFHLGHKUV4zAikZSLjJv7a0bbo4Ll+GjBdrvi8nrN82cFqVZ0/cBvPrzm65+dMw6W1o2QeIwb+O67S/Z7j/IpX/zkM+4eLkmUIikrgrPMJiX1fgBhqSYRgSqEpu8Dx0cVWVbx9s33OGeZTGa8frMjrzQ60Wyu7xn7gBot+6Hl7GzBuw8fMKNlUlWYxvHh9pLnzzR/9pc/5tXLDSb0vPh0znpb431KUmmeXDzj8u4eqQsWs4rXb94wqVK8jaXVfa+oa4OUluNlzsnxKXW7I9Fz8mxgPlvw229eMfQdeZYznSwwqy0T5uQbCPUN+mxCc/OePCuQ+ZTMKoIbcVi8DcgRlCyQQYGZInqNwiCV4RMh+T9/8in/unnFf9oNPAyO2/KB9SdPufxwh2sdnzz9jGmZM+4117sNQTc0bmQ71LEjwQh62xFUTAPXrWMYG5AJ/+xf/CWp1iTZgFMt0misFRwdHTMvjvnw9oausRR6QdOOqFAwmy7QUvH0JOG3v3lNvQv88R+94MObKz7/5DOsH/jlb35Jlmb4YQSXomVJ02xYHi9p25Ysi8O5fmyRCp6cnyK84PbympPFjOPqBJ0Z5mcTrHaY1tLVDU9OlritQ1rJOFrur7bI04Su66g3K/77f/lnjGNP0625f7inaw35IsO6gbRwCO2ZzRPu7nZ4HyiKnPdvb+n6hK9/8gW7/T27ukYngX//v/6aJ6cXrFfXKO1pt3sSlzKppvRDy+nRFDs2PH8+Y79v6M0tWEhLxWxWEYTj+PiIu5sHFssFxgxs1nd/0L07CsPRnR28RMn0wGWG4O1Hce+gmkfR/LFfzYvfETzhd4s9H4X3v1csyqOQ6f6e4x3+vqD+KKQ/Yjwek6j6sOaGyEKPRkuNAJwXiCDAhcNaySK0ipxsETnpQmhCEEghI6ojRDHTeR/d6c4Cscg2vi5JojMEikCI+wgdBVTnHEqKSDI5iP5SSozxpNojZI4ZFK9fr9itBauHOt67i4Kb6xVVleNczTi2jKZjOs3pu5E0DXg/YkzP8dBTlpMfXnMQeG+j6fsg3j6KsxF/GD+7JNGHBHcUcsdxYF/vaJqGpm6YTudoFQt1F/OMcRgYBsekCh+FYGMMSZIePoc4iB2GIQ4mgjjw1jU/KOePwnRMK/hDj5HzlvPzc/7iL/+S77//nrcvX+O9QwjF2ekFs9kRxsQ9MQi0ypAiCvjdAW/wWCb6eKzEcvnyo1HIe8/gojNdCo21jnH8QQgPAZTUJPoH5v0jPuV3B8whRHa6dfbj80bWvzqkohMSlRzedywz1+og5Id4PEWkkSB4cUCx2UMaIyYyHjE8j/ygGPR4NBrF80FJCSik0Id0wH/58Q8W0dfuHdYGatdg+zVn0zMGvyf4HZ37VWTWpWuGdstpkjAOPcgtQUh24wcGK5jIJaPP6McEG3IWyRGpHOjanFY3JLkk008Y7QBpymYcEUwYjSCRiqWuGJOaVgkSsceh6ceSIh9QXmODQCctTSexwtMay0X2BUpMqMoSqTVSbCjTJcgdU/2EwRuEnOLFOSIs8EbQh4ZJMscxYPxbTrIfY5yn7WAUe6pSoXSKp48OI1KkSJmnGcIbRpeS6sAYLEiNYIOkwLgWKRLquqaqjgjKQFCEkKFEjrV7nOjQytIah8GRS0mRS6wRzIanyJASDmwkY0x0WyVR3BqNwlqBN5f40GHDiPAKFzqm+Zx1UyM44c3dax4e9qzahndvDKel4/k/AqEjKxYEKtFo62jaLUmiSLKMqahQ/pj79t8wzX7Eyt7wfv+O6dQjZCxT0iiCHUl1hQsNkies23so9gg0+/4tCRfUXQM0dGbPvh5p+l9xslSURUKVSya5QJAhhAZxhPMjTf+ARRBERioko5WIw5BTCIHSEhkUbbNlYU45T/4H2uFbLK/pxjekqmL0JVplBD9SDzVSJeBjNFEJBeKYNMyI284c7zNksAglUF4RpEMCKplirCOwwpPgvSNLKpy1CKlxXNL2NUpWYAactIzOYB1463DmhqZuMWFg1zi2u5671Z6re8/9Bla7gPU93QBVqTGeKDyvolPJywERAtbAeqjR+vcvHgTYjw1hpwgdzKcLtnVDyCJyJE01bbNHJQnSeYT1PDt/wtjP0Ao2+wfUoUB4GHvS7JTbqyvm0zk70XOzWnGeLXgyW1I5xfLsjF++e8Nts2ZYX/Pk4pivvvoRzd2ab3/5G4wJaJsxGo/OFJPFlMEb1s0ekWaorODy7Xd8/umUoR/wYSTNNF1X453l/v6OLIvT1purWzyWk+M5Q98zn52wr3fMT86ZLRb83TffkqY5YzcyOZ7AJFC3HavNPev9midPz+iGkRBgt2/I8gQhIu9LALv1DpGn7DYdzX7g6HTO5e0lQkm26w2fXTzj3S/+lq3tIppD53gfqOs9ATg/P+bD3TtkIggCqqpCiVi+ZI3h6HiJaAX3mz1pSGmGERs8226PvAvkiYw3ZlVxvdry3cvf4r3nkyDROjsIoA3dMLCpH/j2u28YhxGVgQjQNS2JLNhtt/Rdx7Sq0EqRaI0NnmGMpXq73YaT4xMeW67X2zXVZE6apNjgOTt9yj/+b/4Z7y7fcnn7BuFTyqKizEvOT+Jn6EfDLs2Z5jnb3ZbWDBwfHyGE5z/8x//lgFqIyIXVytPZhuP5HNMOCBvwoyNVKbPFnPv9A+19g8ViBktZVDxs1hyfn7IfW77+4iuEEVw/rMmyDD9alkcn3N1fI1As5seclAVXqwfSsmA/DKyHlj44tEwZvIuGjwB+dOA8AUHbdZyfPOFnP/0Zv/qbv+Xu7op3V99jgyeb5CQqxoLXuw2qV0ymE64frumKge1Qc/b0DBcs11eXzKYLnpyfMo7dYSETsSeJ0syqKfvNKrL/MDz7/PdvDd7vd1hrmE4q2ran73smk0mM4zmDEBJjbLweDYbieInWgtX6js1qx9EyxfuOclJRVDm3Dx8QynK6fEI1mdA0DRDIsoyui8+fpgV9P2DGDcfHx0wmU3b7g4O7rWm7lidPnsTyNhORG8MwUFUTnjx5xnRScXV5Q55XtM1wcKJHx0bwgbZpqSYZQmpGYw6u74gB6brk4F5xjEnPbLoAYUBadCLJspQ01UymJUqDD5G5necJzlu6vkFpgU5gNi8Q0uN8j1JxcOlCR5K6+HstCKGRSh8c9SNpmsYeEaGIhaYpRX4Qp/s1D/dbunJkOp0ilaTIJ2itKPKSvu8xZkeiEyZVjtIplzdXlEWFThRSRUxINPkIirJgNpvhnEdKxXrzQNu1ZFlKUWR0fcduv40uWxNd+0mqyIskpkoOXMw4MIhu+KIo6LoOgMlkwmQSkUxmNFH07VqcTdB6QpFHkXm/r9ntN1g3onR0TQkhozt9GPHGkGYpWZYSksfiH0dVVRQho+u66Lwd9cekYbv3FNOK5dExhLjhzNKC42UVeZ8yCubXV3dIsYnOHR2HENZ6CO7A6vRIlZKn8VgLTuCFi0VQjadvd6i0iKkBaxnNyP1KUBYZ1bTEBY8PnnEw9IOJSRwhSJKEPM/p+wEpY2FT9Gb5Q1nrwDAGtAJr42ty1iOkxFhD29bUzfZQLvv7P6zxJJOMPMmRKBKV4S1sVlucGBDa0dYNxo4fHW2PBVHGxXvnrt5GjNMw0LUDeVaQaEmRpHR9x3Q65erqCu8cTV3zcH+H9QOJ1ORJilKS9WoN2iETgZKa4+WUtn1AScksTxm6gSTVSCnp2oa+d+SFY7tv6fqBfhzYN3tmkxlpkpGlmmlV4Z2lbQzTxTH7fYNONdU05/Luin6sY3TXeSSa/cOGLE0Yux45Oo4mU85OT8jyjNWuw4qETCq0UiynU4Y8IFT1/2HtP3tly/I8PexZZvuwx19/01Z2V9vqmmkzMyRIgoAEARTEV/oC+l76BAIBQqAAUpwZkjPUTE+b6uryaa87Nvz2y+nFintrKFCcRhWjUEhkZt1TcSL2jljrt37/56HuRpazGT+5+TE6g/OrKcl0Qt+NNE3H5vZbKl2RJJH56YJhezgwkRVSwMnpgoClrRt2d1uGtUP5jjTpyOo95/M5vbXMCklZFmQyukXs6EGD9Q5zZGd7HwWoxlmcgcmkxGvLarfH9gZnLOcnE66Wcy7SimmaYZoGKRVJogjOsd5smJYzgjEY70mSnODBjMfDEG/ogqPrWwY70vvA9fbA3335DW/uVgih8cdRdqWTWKwJsS1WTip0muHsQNcb2rbjUH/LyaJgvpgwnVVkeU7d7OmHFhsMk+mEalYxWsO0mjC4gaaLsl3vPe3hAF6gpf5w4COFZLFcMFjH6eIEN/S40TEeg4Z+6ClnGW3dkegDi6cLfvgnP2DqFEmW8G9//iNWmy3VPGc0MUgXKMzoaboh3qvCoZM4jRVc4HS5ZDab0TsTg5JEo5UkSEGVp2RJitUK0w802xqEpE469rsdzf4QQygBZVZQTEqEElRF8Rvf2yFYpFSM44DDI4Og3tVoqTBuIEjP737/c7rW8803XzKOLVJ5zk5meBNItebZkyve3Vyzb/Y0Q0QL+s5yeZbx+MkVIlkTfTiBUEo++uwlP/vqjsWpopxVjE2LcSNVVXJ19ohnVy/48Y9/wun5E4xbM5gWbwwiDaSJ5MnTE7759p75PGCN5WRZMZsu+NlP34DcUuYFUjoeP14gtKNrexKdcb6ckOsK4zpEgETDvl4hEyhnCqEDbx/eIHzKbBkl5kObo33G2WnBru357voN9w8bPvv0E97dfIdOLNu9Qyuomx1JoimKyHH2YWQ0cW2UUaJ1TtO0ZKmm7Qxd13NyMkeHnLevDtx8O/B47Aj3A03bID6ZkVqHdg4zUfH9GUdkHWCW4AuNGBNIRsbEMS5SCp3zuO34T5+eMe5HpiJhPTb8+NU3+JDg+4E/+os/5fb1K755/Yq2sxSzOT/58kecP53SuYHpfMG316/jdbqzPHr0BGsk3djyV3/7Y548PuPiLGVxPkGQMJsskERP02IxjwWjdxvMAI8uF1ydXvKrX31FVWbIUHBxNmccYkiXKIW1gWAMXgTmF8+5fbfFWsv5+WMeVtfoRDOdnKK0Z7VdcXH+mMuzM+rNnt39gcJmPOzvmM80oYBkJhgby0l5yY///ltkJXj20eestwe+++YrPns+IxNwdn7F6TLny6/e8u7dPYf6QFGmhKBo+xapBNv9ltPZgiePMvphwJiOLIOz0wVu9BTFAp/C3fqes8spi2VGWVyBh9mk5ObtLV+/u+EHP/wDNts7pDKUBVRVxtffrXj6/CXb/RYfAmmiefX6lvOziiBgtbrHmO63+u4WQn3gNcfg9YgnOQaIoHjPR7c2rpuSJCNNM7wLhGCOPycWTBIdiIJO/78Ixv/9x69Z1L9GusRQ+H0D3hOObGohiagSH/FXwStEiMiuX7OsjxgMNNZKGGPb+j1iIwq2k7hOFhopFSE4nB+jL8cGPBEdGYWRMRD2ErzgyHh/j4QJxyA5/lXwXr4akaBpqsiy9wgcz5On56zur6nrht3uQKKzIxpSM/SGfmyxrsf7gHWeyUQA5kPzezaOKHUM74WKmEMVW9/vvSBC/hpRE//dUcwaJAiFc/7Yrh4/vKdFPkFKwzD0aJ3hXdw7KKUZR4OxI1mWfzgQiZJTS6LT415nQCmPzgRJ8t6pE903IHAuCs6VCmil+cEf/THnZ+f89b/9a/6r/+r/gUCQ5wVpEjOCoR9puxYzWrpuiGH+8aDlvVD0fWnXOXd0o8XrSmsNPko8Y+McvDsy623EC6VSk2UZUiqcNQjhIkP/fdBOwHlLFIj+eiJCIJAyom2U1AQkzoojjuXXrHxvLM6bX08oOPHh5wshIi5HCbwXvBejEt7fW/H+ESpe11rHwwDv5XFy43/78Q8O0SvZsXE91pyQJQmFvGPbWKblnqY9pShGkjChKEp6H0Njpywi3BACdMYy0QlWWmbqjBbL4BzGwrKo0GLE2SVVmlMUDZ1vSMWMUQVa0zLV53y3/4YyM+yGgHYbvPJoUeJshtCCQjgOrkGJjGXp0U5T+Jf04ZbOvaJUT8i1wJqBNFQUJIiwpzcPBD0jk+fkomWwJYm6B+dQPmcIHZ4JWhmkT6n7e2ToGV0AvWMYR6zKUBLm6YJmHLCmw4VArp6hZEI93qPTC7Jkx6x6xjSbEuQD7VCDX5AoiU4OdA0kdkGeWoJP8W6DciW7NuXELJEyhpXvIf/+CNE3ruPQtDSiZ30/0HWOu0NNVWXs6w2JWPDuoWdor+OppZ+DMDw5veLzp39Ili6I4s34YayVwnlHPR64uX/D7KTkdvwXmLCnzF+yNv8KnZ0xtY6hf0AWlxh/i1ZrmiZh246UamS9u+N/+rc3nJ+sCCic6BGiY3WzpygUk2rBoycL0nRGlRumVY4Qk9iQtRKJpTMNSi0R7o7RDUjxiM5vkCJFqIbgJLiA8x5koO03rFfXnF58xHn2F7zzXyHFBMIdqdJIcYYNEi9GtC4IYoezhkD8QhzNHWkyI9CQJSXGn2HcA1pIVBJQao01AckcrWKYnsgJ3bAl4DE4hqFBhhP64DDjHd24pR80+6bDGsnt/TvuH0aMUdw+eLoh0I4BmaT0TmFCQEtPkUKRl2zXB5YzjRCaRGv2+wG8OF4LHuN+86YqgEgl++7Ak5PnNIee69s70qkiCBjHkSJPOT9ZInxkq2U6ZVevyMuU3hhO5jNu375lMpnx8HCL1tB3BySOvm9x+YLiZEFRLLi9XxFGS2g79psHhHDsDltmsykvPv4MoRLmj84Z9T3ONpDAdD5nf2hIspyf/uSXzOZz0kQyOkdZxEQ4yQvKRzEQzzJJmpTcr/bM5hXbzQ6GbzmdGXpjEKXi3c2Ks9NLqryiyirwcdHfmp7OGOaLBUU1YbA7hrqjbTpu3UhRSMZhYD6dErwkTytWuz1FPuPQHBhVtHGP3vHNq1cUZYkZBV3TMZlUaJVg7IAxI6eXM9JUsTxbspc1rnd4N3J+cs56vWa33WKCpd2PaDKyNMWIyOoy8hh8Wk9/OPDTr7/izZvXpIlmcbpkOszoW839bo/OUsx+xLqRybTiUG9BRHFomqkPo9Xj0GNNlMBJLRjMwKE+0LQdbdvzh7//RwRgu9vw9OlLrHfoJEURePn8Y/6L/9N/yb/+N/+S7f6Bv/q3f80wtHR2JJ8UfO+Lz1nd3bNbb5hMKh4291xcnlBNCsq6oKyKiNJSivPlJW0zMK9mnMwX9LpncnnFV8Zwe3fD5LSgnFSU2YTNeoMx/sjbt+AF2+2OWTYjS1K6Y0i73Ww57Bo22w2ZTpkcxwvvH1Ysn8/Yjz0P7YG933Nz+5rFbM7JbMn6Yc3p7AQrPHXXIST89Gd/T2davnv9DXW74vnLl/QG1ts1h/WO2WzK6EbWqzWEQNM3DNZwevU9mrqm70Yuz3PW6wcSpagPW04X55ydnbK6W7HbbEhlQtvVvN2uWK9X8H/8ze7tzWbHOMYTeXk8bFRac3FxSb1vMDaGqFpqUp3z5OoZJ6dz/v6nf8vhUB+lnUeRjItYgfdya60VQz/gbODu7gF4z1yE2XyKNWCM42G1ous7sjxhkpTsdjuWyyX393cYY8iynGEYmM3mzGcLppOS4CV13ZAksSk+qSaRj5rG6aw8L+mH7hgyjMcQKY0jpd6idNwQ9EPHMA5YO6CUom3j619N5pyeLmm7PT4oFotZXJQTmEwq2nZPlssYRg8WGBnaHT70ZHmCMYExvBfZHDcvIrIRYxNII4X70GhQKiUESd/H5zqpAipNSZKUqiqOCzpJVU6BGBpLlbBvWsqywHpLVmiyXDPaHuEUIQjmyyVFOcF7x35/FwVFeTwoGIZ4QJMkGgQRA6MT0jw7ct79Mcgv4mGTj4tfISTT6ZTT01OEgKat8d7C0WA/jB2FTdBJdRR/RmTNZDJBJilJUkKIDU/nLWmak6UJAmIoFTzeW7SSuHBE3QiPsxIzBkSQTCdnPH/+lKLI2O3XsamlIzdyNEMMsuqOph0RqDg1Vyi86GMDHRtfUx9IVEFVFgyDpO97BJI8S44HSIbDYUs3BvI0wTvPoW7gKEeSxymciGgICKno2o6eAZCUZckgzQdm5vvxUmMs1oA1lq7tCD6QZTnexSmfoW8IwZOm/+Bl+P/qo0hThBMMdkSrhGo+Z/RDbOoWOaPvqPuGoe85WSxIs4zddsujJ4+pm4ZHzy45/WbJ/e0GpMIjsAEmecZiESWp49iz3a5xzjKbTUhSjfeGfX3AJDmp1pgQD3T78YZqWlJNUzozUOYFI5bD0JCJQGsb3q1WWCs4SxL27Z6sKCiygs60TGZTJpM5ZaHQqmB1c4cmZxNqpvMZ+SSh6Xe40FM3W8zgSEQGU0EiQMkKNwzMJ3OePXnCbDrFW48+nXCzWVEfGlb3t2RpjgsaYw+MFp5e5Ty6PKNua/rDSJ/GDV6/99g2YH1HUSToBJyw7OuBzgfKKkWMFo1AWUElFUJY9t0WxcBMLTj65xFa0I89if61sKtua4ZxIPgo0hIhkGcZvTGgJOMYD5v3fYtwkjQpOJ3OeHlxxcwJtmOLXk5YpAXUHbf37wgyIKTgsN2QZjlJVhynanm/y2VwlofNGi8ljbWsD3vW2y3D2BOQ6FQhQxTN93WH1IK8yNEqRaKpCk0iPW3b4B3sDjXGGVRqUEqzWM7oh56mb9gfdjx7/oR3b25igyzVZMQWXrtraTd9nMqbZkwncWpXCY0ZLEEI9tsD+MB+v0fJjKat0Xmcmk1lghtGvO0p8oTvf/QFf/23P6JtetwIMuTMphnOOYpygrWBphtou468yBg6ixaaRCckec5qt6U1A70ANQwUmUSGGOw3TU+qM0zv2G8PGGOZLmZ4a5lVM5SQNG1LkRaczE/px5720P7G9/ZkWrA99FRpSpZmtIeWSVWRqwydCbyUjEPg9bfvwKbk2RwhLSEoXjx7RpIm/N3ff00xi/vEvMxpOoPUirfXd8yXMy6uFngrOJ2fcvP6Hch7Rtmjiwt2TU3TdmRFRJ7dXN/gusCzx1f8m3/3d/xH/8n3aPst6+0KKWJTsygzXryc09Qtw2g4mZ/Rti3v3j7w5Lkiy3Nmszn7w4qk0PTDSNda6rXFDz2ffe8xJtSkSUY/jtzfbTk9O0MlUc79cPuOMq34L/7P/wf+/kc/Yjk7xUrJprtFpB2Pnp7Tj45qmvP27R1ZGvnNk0mFkpr9vok8ZuE41DXvrnd89nnJZJJibMfV4wk6mdK3A0I59GLOv/nyFV/dNfzeWQJfbjg5O2X+zvLpZsUkNJx88oxECVzTQpLgUjCM+LymrGJ4lOYZ4uAJO8vbm2vedp7tNEctJmxNzeL0hHRa8F/9d/81yjqur+84OTln/fWKtw8t6YUiKKiU4uT8hK4bqLuB3aEGUp68fMqbm7+nmuVYFw+vQ5DcvFshgOUkliNePn/K3c0dSaGZTDTOtnz84gIp4eLsir//6YEgC/p+zWr1wHxx/O4WMbSdzmasHzrGwfHRy094eLijqR1ny4rlcsZms+PZo0+ZzxKKFwU//etveHY55dXrt7z4/mPu12t8H5BjzuPLc97evePf/Ou/RycLnj19ymGz48mTBRfnp9zdvWO323xw3uRZya9+dcPjJ1OauqFuBqSZMJ9OaQ4th6YhL1N2h2v8TqOyjGypWR/uqco5T55PCWbC0I7Uu4ZHTxa8/Ogxdb3j7OyMtj/QtQ0ujDx5+pjXb2+YTPMoVNSKsiwYh4HN/kCep1j724XoMTS0WBdDW/GBOxH54IpjcC3BufH4r2ITXYiAdLHFLY5FQhGH7o5t7/doFs/7rq2QEnxEgrwPKqWMobfQ8uhO4kPI/v7neO/jmloohIf3jPSI64A8zxAijcVGEYN3oSQ6zcizghAU3vEhkPbeYd3IeEQIewlVPiVNswgf9D5SZxTx37sYrApPXMsGEWWSR4529BUplFYY06B1QVkmPH66ZHVnubvdYq3D2ZFEZ+z3LWmq6PqeYezxXtJ1Buc1xjkGI0DomO3olDwvjlg5d2zDK3ww+HCUqPojZ1sKEqUjjlCClgohj1MEztEPDdZaiiJHa03ft2RZjlYFxsT9SZSJxhZ49HQEtJbH4Fxh7Yix0TOknAStgePkIAABfWyPCyEJIqB0zscff87Txy/Iy4L/8X/47/niiy84OZlzOByiB+z6htq3cco3zRCdQfMejxJdONZF/rtS8ohasSRJRpIcEaHOoROJPF5HUqiIbOc91sWjpf7Qpj86aGNudmSYx2s1yj8JAo7Bt/cQnCN4SwgCpXUU2woZ7yP3Xqjr8P7X0lzgeOASrU5SSuIJT+Tcv8fzCI7TrkKhVHx98f/hcuo/ePW+Gz3tOCERglwWjC5HJzU2DFxMZnSjQ6cFifCgTpFyRmN2XKYWlXnuDreseUOWaESIIPwqtxQBvG/B1SCuQK05jJrB1sj8NcM4o7YSpdvItfQJheoQUuL9wDSf05gbhKkgnzFLWgbfoX2FVoaBHUI5JBN6d6CzO/AlRaIZ1AEznDNJniLCjnp8Q64KXKiwrkGGC4Tfse82JLICOZDLGdKf0oe3JLQIZpS6RyjwrmYYLZme8XBomeQpg3tHkUxRiaDUIxKN9wc2Q8M4bsn1gkQtafxXaBvQOqE3GYe6oUguKHRB1weG1pMlZ8igI//0eOporMELSz/s+eXXO/7ll+94uA4Ebag7STVtGDuYzXdcnSdUecGf/5OnnE4UidxBoTgZL9GHJPKp8McLP4YJHoexI525Y2/+jsY1zPIVWs5p2+9I/Dmr7Z6H7o7DuOLmoeW7Lw23d45uCIxdwEv4q6ZjfqJZzhJ++IMlJ89mGG35g997hhQNRWVIlEVQI9SG0SoGYyk4QZETcASpKBJJGvYoIZH5lNAqvFOoUJDrkjwvKcsZzgRwgUXyJ7zq/u+IosD7M6T3jESW/iRf4sIhchxFg1JTJCkiKNKspDMr6m6HEAvyIy/Tu5YslAzmLnbpzIgPFmdrMq4YjCFQ0QwN0nnu9l/RjQPrneHmfk/baB62gqGH4CXtKLmvPcu5pBklp1XJ0I4oHdBKUuQ5621LlgAytiiHYSQoyaRUkUNoJWXx2+iLYLW7Z16cUk4nKDIWyyUuGXl9+xUiQFZmeO8o0pSqmjL0I0JpRuuZLpYUkynOg7GW5XxBPl/wcHPH1Gs8Uap2vd/xuJpz6BqeTGdMi4THv/O7vF3fM3jP5rDl0dMr5osT0nnFL7+7pesGHp0/RQqNygL3qzUQR4Jvb27IqhQlBc9evOT+5p7N/YbFbMl8VvKr7dfYwTKpplyczZlXC959u+LiyRO+vvkaLSW2G2H0UHqSJKWoSjIZN5zWDrjBIrzg4vScixPJZreha/akScLQWabVHBEUJ/MTbu8ecIlBTQtkosmrgu27O2bVhIuypE5aTuYL5pMpN9e3LE5mrNd3nJ+f0/QtKklo9i1PL5+TqISyKLCjweMoJye0zcjY1kyrAoWn7Tucd/TW0bf3rJsVwRtmsqIdWrqxxXnH199+w/zk5IiD6D9cV8PYkWUZRXa0bGtJb0eMDWz2W9IkIU1S7tcPzCczXr99xWw256MXH9P0Db3pmKiAVHFUusgKnj56Snfo2a9ryixh03S0Q8/5+Tl+dNTHcEpqzbNnz0nTjDRN+eyzz3j8+BH3D3c473n54iVN3fLZR9/j5z/5BUmaMKkK/uxP/5S32+/4+vorirLi8uwS2wfyvOBkfk41q7i+vwHvuLw4w949sF/V1PsdPYL58oTpdMmbb76l2TUkOiGrck4WS2xv2OwPoDLsaNgdtogh5er8kovLE3SZ8rNf/IIizagPNev7FX3fc3r2CUopmt2GLM2pqpIiy/CtpUgyMpHEkUMfGJqBetPwxSdfoBLNd9++YlKVpKnGB8dut0EAWZJxMptz/e4Nm92KbDL7je/tQ2MRCBIhSdMEnWYoneECCJ3grefQ9jjjkE7z+s09h7ZjvTkwm8/QCVg/xoVLSKnyRWzNe42U8PCwYb2qmS9mJKlGKEeQA/OTktubPSooNve3WHNgUk2YTmdMJpOIGmljEyyO38UAbDFb0PcjddOiEoHzPW3/QNOvENqhvCBPiij98YE8K/EORgRVNcGYHiWT4/+H5eFhTZJkZFkVpwB1oK5XWN8hFwu8G9BJglYpZTbh4vwRh0PD2I8IoSJLN00ZhiicLBOBMSOpsAgZ8MbjhMJ7SZYdOYYelE5xfkAqSW8sSRbQeU6aZnFBbh1ZknJ2do6QsNvvcE6SZVPskdvnLAgCbbNFak+WJbSNoe8KlMhJdJSbpirBhnBsUSjSPAcRN0Nt3yF0PBhPiymIQD9YrBsYXI9xI7N8gdQZIOlHgxQJi+UlQmkIAee2SDlSZAElBeNoQfYgelywBOnRaUJsTiV0vWXsR4pyTlVNYvtQBLJ0JE0kxrQ07Y6hb5C6RCAjdkt6pAgIaTk/P2c2m8aGugmMQ0dggzECZ48LZC3Jyij7bIYdnYWySo+ekBFnB7J0wun5KZNqTte2SK3wzqJ1Qp5VtO1As37A2RaS2DYeehsPTmTkprddE+VxWtJ3A91g8DaQJCnwvvEFWXoU5h65nUpqgu/o2tgMS9OEvm+wVqF0lDCp3y5D53x5znffvUaKhKqa8fzJHF86ZrMJuhDcPFwzjvFArx9jq5cQBU3GGw7tlo8/e8lqtaWoKrKiwluBkLGoMZtNubm5IUmSD+9Jkmm6YY/tO+7vbjlZLKkmFXq2IEiP0iK+Zt7SjR2JKfEK6qHFeUM2LXD1yLbeYYKlUILFco7UMJ8tmExnrB6uWd/tokQ0L4GU0/ML3t59zeawwvoRlYKzAS0URVGSarB4OmcoywUHY7h99RqsJy9zhramrHKaro8BurG8er0CBHkaWJ7Mmc2nHw6UDoeaw2Zkc1szy0uGvOfpi0fcbh4wo2AMAakF8zyDEYo0w/Zv8crSO4FEsj7UTNghRk2RKop8yjRL8TYwjob1fos1Q2xge486bpqVkuA8k0mJThWTaUnbjHhjuVwsOMlz7HrHbDFhO7Tc79eMtxtcPzA7P6Gpa5r9nmQRZciWOJocnMU0DXcP9xjvyWcL6t2Wvu5IERRK0fsojUuS5MNnSJqnqMRRNy1ZUpCm8WCqyHOsG3B+oO9bhraJI92mp5pMmE7jJIsQgYvLc5pDE9eUmcaOURyWpIJ+aBlVz2h6FpMKvKTdd/TW4AjsdluGvo+fEZlgPsnZb7c8uXjEdteg8NzdvkN/9n32+wNKZ5wuL5BCkqYeVHQF1E1D348oqQhe4j0M7YDKE+ph4LDfYoJj8AFZtxQpKBHIkiJu6HOFQDObLhFKUkxy8iSlynJm1SSG82VBVuTstlu+We9/8+/u+oAxcHb+nMOhRQjN7e2ay5MFqSwoqoKbd2sUKYfaMNqRoCxFnrA97NjtDtzeP/Dx/IKh78nLiF8bjWU6mzDYjirLKPMZr17d4oaR5ZnkvJiQpSn/8p9/y9NnisuLlLIM2H7kq2++5nufP+P3//ARr958i/EWHwyz5RxlLW1/QOnAi5cX3Lx9wLqew6HlH//ZC+rmjtmsYrUeo6A3habrGceMJM159fotTXfgB//oe5ydwN3dOk541YIkU+g0MFvkYA3/zX/735AKwXZ94PTiKdV8CkQE3NdfrvjzP/+ch41jHBxmCOz3A1ePEtLRkSaKrtvhvaBuGpq2Jss8m92Koe8o8oRpFb8rVSnZXia81prXVYK5t1Q3LS+7kt1HFc/GjvTrFbPljGGewrIiNwr5sOVm3DLJKqY2p/7lPZqcX7265q/0yN1Uky0XzK5OaFcP/Pgnv+L3v3hJWilMPxIqy87WfHO95+PPn6DSjtEbrm/XzBcFAoPWkuubB54/e0qWCf78L77H2LWcLy7ozYiziiePn2HGIU7yuYF+gO998ZL1w5o0DbTdmufPnlDkKW+u9/zOH51xf//AdndPUsz45POnXK5O2R8O3N6+ZTl/zPPnl1zfvKP+dsXlo3N+/vOv0cWMJE9wLvA3P/o7/uKH/wjyCT/4xzm3b645f/yYh/2B1/drnl48Qmcpdhh5+uSc7GTkdrXn44+fUeUSEQaur6+p+47RwfX9Bikdm52h7yRVPqfZDVwulpydPuNf/asf8+TZhOXpBU23QyQCbwfu73dkXcZ8UeBtw6G7pm9iAa2YlfFwNVFcnZ6SZRN2r3uMSxmNJZ+WfPTphKHv8dYz9ANN3TJbLMmKCZPE0dbr3+q724sB44eY6QiFDQ7hYjtcyQR9RJfoRCOcOIa1cd3rvcCH2MRWKkoXAyq22IU+yhUNzsN7V5CMI30okZCmxXHdPcRDS5nFJjiewfckSYJSIobdY48zHp8G4teSAz+CSpBKHp+/IBA56EokaJWQ6ZxMR+a8I0RZZpBHLJ9CpQpFgidQqoIyLZFCMow9SkGQyZGNbfHH10EnCaN0x+jz2I4+ImFC8KRK46zBu4EsTTk9K7m4mLNZ7+i7mJkN1pGkkn7o6PsOqQuaZmR3qDm/XDCbJYx2T92PlEXOfDqFAEpF1EmqU5IsYtKsjS156wLuiN0UUkfRdghRThksEV8yMhjD7d0bvLdcnEt8mOC8iFOLiULrFDOOmHGkPhw4PT2LaN39ljzTHA4brI2s9UmYoQlx7S4VsemuI2pHvP97UO9Z6crzn/5n/zn39zcAVGVOfdhx2G/Z7Xdc395z97BiHCy4iPDx1qCVZMQTnCFLNXiDG3vwBrzEEUjSiBo+1C1ZWdK2PWZ0eCexziNNIE2yWP5xAmE9OmQ4f2yRA8Z7nPMIGUsxWsTr2brI/AewzuO8IM0kAQPhKID14siBl8drJrbahdMwWobRxYMppY9t9VhiUFrHSQsBiZYEG1vxhF//b/63Hv/g5XvinlGoDc72KD+w1BPu7Bn39U+Z6RsE0HU7pmVFPzomWXwSb5q3kOSURYqzCc24Rbp3pEozSk2RTFCh4hBy2uEenVpEeEYhUrbtgE4yEtHRdSNmTCGJzMlZOUEHRXAFqThHyT2b7ppUjxQqIfUVk2zKaL9lqmdI5bCuRoQKLx7TjA1K9YjU8qa+pdJTrG+hTI989wLvLYMzWNOQ5J5MTlFCAvdYuUWLnMCENB0x3pDpCw7DmnnueXY2pTUZ7TgyekOVpKQiJ4SKIrlhNQbK7JLgS6xfIcdTxn6CTlK6oWbd3jMtR940Kd++2vLp5HOSx6eEENEtUkoCCiXjZu3N/Td8c33Pq68kzlnyMkU7iR88p1cpn39W8fnLwIuLc5Jco3yPTh1325ZFSGMj7Ti6IITAuuPpKBErMpqW65trvr15w9C8IgmnbA8HtoeaV9d31PWA9ZLRxAWK84p97SnmnnIiePZiyp/9+Sd8/HxKmd6RphJjlqTaQGjog8K5HO+nJMHhlGE0G4R4wLmCPP0YY88ISEa9IxGKpu858VNEmPH02e8xn50e2/QiCh7wlOIJT/Q/RQ4/w6J50G/p3IBQGmseEJS4kJPol8cbdIOSmtGs0fIRTu0ILsP7Gb2t6UeDkp6hB6UcnbnGOU3da7z9GX0b2NfQDY62tmwOjoeVxFrF7V0KmcQFT5Erut6jtUJjOewgT1PadmTsHUJHcc+hGRhGT1VJRFBsty1ZqkhUQlMHijLFWM9u89uF6HW745Pnn/Hs6VPu3mwoipR1fUuRFVF8pgU393ckIqXterKijI1uJK632G2N9YK2HRi6O55eXKGVppIJbS/46Plz3tyuMLuGdrPh+cdzFrOSzWFHUVSsH2757NOPUTbgVcdgLHmSYrqMvh2ZnZ3w0cvP+au//BuqomS32fLRR08xbqQZO/bbA/tdzXJ+xpOrJ9zevGE6WfCnf/Yf87C+pe92LBcLXpkbhq4n0ZpZVWG6nv1uy83NO569eIFQEts5ijSn844qL5FBxo39ZM72cMbf/t3fsDg5Z7/b8ezZM54/ecH93ZpxsKya+/hlL+O4/9PHTzibnXC3XnH69CS2HwPoRLBYzLi56zjULe8HCb7/B7/PJJmwvlkxn815WD3w7PlT5mePub6+xw0d54sZSgge7les1lt0loNr0HnG2Bs6W3O/uSPNM7yDb998i3/zbTSDDx0Ez6QsseNAojTCxxPXsijil5DzjOOA9ZZEJiwXGVmZgxd8/c3XnJ2ek2Ypw9jhGeEYYHTbml/87Oecn1zwO198zr/4f/8/yZOc05NTTucn5GnOYr5AKMnF1RVSxI3xdnvg5ctT0rTg5vqOs4tzNpsdRVlyc39N0+8p0gKVnhBEHFV3zjOfLmh3PSeTM7quo5iU9E1PnmnsoQccWnqm04IsFWQ+IHSCSnN+94vf4/Lqkr/56d9hvEU7wW67RyEYmo55MeH80SV97bCDQQmJHwcen5+TiYTWCrCCq9PHPLl4zr7dURUZOJAhsH1Ysd1tefr8MX4ceFhtWcxLtjdbFvMFhSqxzvHpJ5/TdAeMHfn6m6/43se/y2xRMjY9XqeUecWh7vj4s9Pf+N4e+kBRJiQ6/yB62R/2WGeZzRZkR+u7sZZcZ2z3e7qxoe8NwhhUIjgconhvMT9lUk6hmOFsxzgMJElK8DpiNJwjyxOsH9ketgip6YfYNC/LgBJxkaiUZrN5ABElSMNgcCZhPk3Isgmr1R1N3ZIWASENu0PPbtcQnCJJMgKxPW+NI59XgKZbDQy9IV7OEilTkkSjZErwcZQvzzKQgfl8xjgM7HYriryibRrM4FjOIVEb+m5EcBQheUGe5TjTRoGOlIymRxPI8hSrBb1xjLbHe0+WFWid0x2lk6PpESqO/hvr8EIgQkBJRapTtE6iDMqLX//XSaRKYuPDj5ixxXYdTe1Ikgx8jhIF89kFYQpCBoahxTpLluWx9e4jW3GzWSO1ZrE4iQgXKRiGBmN9bHB8MNOL+FysiOPCXjL0lmHo2Gw3hDCQ5RqdeJR2FLlGKRiPpiulEqyFvhuoux4tU4p8QlXMYkOKgM40iYauOzD0Q1xzuDg63zQHQvARB5Gl+GBYr1cRF3Go8UGSpoDQx1HOlDRNkVLRNA3bbTzQFipHJXH82QeDUpIsK8jzKDx33jOaDudCPEBHo1WCCI62PZAkaRwdHjxZBtaGeH06S5LGSazYUhpwPiJStBDH912ihKDrWsZxQIiA9z39EFFNiPwoqhIURYZOAqn+zZnJADII7t7dUxRThE/ZbHb4YPjok+f0tmNazajyCbby9O3A5mF/DJwl83nFj/72b/joxSc8enTB+mHPpJodeastxgx0RzzQRx+94OTkhKIoSDPNd6++4iA1pycnBBf5ltWsIJ9k6Ewx2j1ZmjCMI01bUxQl9a5nNB2JmmCDpzkcKPIKKRVVOaHve/b1joDn/u6eYOFseU6eTsmzGXXb8/r6hrrbMpnlVNOCVHsyJkymU6zrkKmir9fsXcdmU9PXHX60ZGuJ6TrOLi+Zz085ND3GBYSKm++79T1Kwnp14MXz55ydnpOlBdNqxs/aryMRSkhWD2ums4JDe0ConL4NPIwHSlmwLOe8+PhT3t1fk6aarjesdzsmYUIuy4jNGwawjrPlBWHsI9d9tNhxiJBO78mSFEEgS+NBpUoEZZnRNj25kjy7OGM47Jimmof9js4PbA87umbHeTnH9Ia6qREioPAYbzBuiMzRfcPu5h7pPUVWcP36He/uHxiCYFFOGLoeowWjcWRpxmgNQkmkTuiHkb6x1OOW6TxQVRKl470og8AFME17bNkNDMZwcnYSJ2msYzmfo0Pg+vYGtKDIM6bzCT6LIZIJA4d9S5Vm2NGjVML67o6T8xPyJMU7x67pkH0gGzwqCD7//Pfo9l8RrOfQHXjz5g2PHj/h7X7LgOdQ72NDdZIQENR1jXUgVYYLIaK3pGKz2SF1ABkoypJCxwMyLSx4ix0toKjbFq1SmjoeLLSm52x5gh1rnPGYcaTtBpx3qOCZpvl/6Bb+//vou5Hzyyv61rJa7dBCcnZ6xpPHV3z37g3rw5q+gSqb8E/+yR/yl3/175gsJ3z0yVPq3S27+sB0EcXQ682KuZowL+d8d/uAtw06t6i0ozkEvvrqHS+enpHnGnyHDI6Xzy756KMT7LCnnCToRLKvd9yu3/E73/+U+t2ah5sds7nm61ff8fTpJbaJbNrtdoXSYGzPMBywleL0dMHqYYcQCXkGPngIkCaCvJwymaecnU94d/2OR5ePmFcJzx5/xk9+8g1j01FMNEqC8QPeDUznS57OHoGb8KN/91O++L2ScTBsVj39YEiyhNvbA5M8IU0EiJFy4kkST3NoGfv4nVXXO8pywWQy4ez0hN12R9d6tNYE0TKdpjw9fUbSZ+zGFd4HvlMZSR5YWY+qB75fQJon2M2I6Czj2x19NeNBFmx/cc3uTcP9ENgquH6UYy5zJmc5MpOczuZw5ciVxAw9KhfMLqa0TeB3/+gRp+dz3l7v8B7KsmR9v0EKQZkVLKZTtpsNL56ds15fM3Ydq9t3pMWMq8ePqY9TZMMwMplUDK4nCMFssWS5OOHh7o6b+3t22y26KMnmnvlFoFiccditeHN9TTVdsNrsUIllMAeCVxRFSjWtOBxWfP/3zhndSNv2lFUMe3/8i7/jT/7wT7m7v+flFx+TpPCL659R2JxHLx+R9prz5SNWm1tcf2BxKensCntwuNExW57w9ZsHnn/yks4FEp3gRs9ybml3hkV5ie0tt3c3PHo+Y3uoedi2XFydstluWe9a2nbkB8+fUzcP8B5fVkkm5Yxvvn6D9BJvJUU28Ksv/x6k5Qc//AKdLtgdNszSGbtdQ1+3TCczHro9i5OMLJ8wq+Bt+O0wqt67I+7DIUU4ShMFzkVpplYJcER6HP8TUXEjzoK1USDqZWyG/xpvchw48kfpp3fHScejBynNKfIS76FtLerInI4yT4/WKVmWk6aafmgYhh7jLEkSMT/OWcwYJxCTNCJYhj5mRvEAQB5Z5R7vLN4R11SImNFIYijqPUoIlI6H+t77WHSRAp1o3LHhnShNCEfpqlJI73nfIUbG5n4Iivc87PdIECkCl5cnaP0taZpgTaDrLTpRjIOhaTqEELTNQNf16MTRdj1Pn12hdML6YUeT17E5P5hjSccyn86oyPDBRiebc0eRpf2AyHnPEo/r61+z0713dF3DerNiMpmT7LdonVEWE5JE07YH6rqjH1qEEJRVzsPDA9Y6iiIjy1Kapmaz2fDyRcYgFKknIiV1ipBHdv6Hq+zI/yYglSbLc/7xn/5jVvfXLBZLrDXc39/R9R3r9ZoQQOsE6/99oer4Qej664mJOHlpnSFTOq7xpUSLiLwTIe7BhqFDqKP8FhcLHEEBGuvGyG9XmiBAuIDwAik0BIWP1EcQIRaWfMDagCdgTWychxD/eSDy6uNkQvjQTHfC41z0EUX3VFxvj+OI956iKNBH39t7QS6YiM3+37OJPskUmU8ZbI61lkYajPPM8iVn6nvszM8IuiTTB9a9JLVxTPhgLacyR6AgDejx93B6RyI7pO7Y2x7tU0TI2NuR02TJEGqE9kyTpxxGw6IsacyGYEtOdcar+hU1jkIrCrHDSY9xObgThD5gxxwjEkZ5j7OBg9+jUkPd5VTZgJQ7CmpUUrHqXzMtchIhqHTFpqvJRYlLHQM1QcqjZKZD+BEtB0YsIlySqUAfcnq/AKfwqiNJHEIpBrfGGIEOFYt8gQzXBJcyBIMbIluvN4b91lLKCwbzlnerA4QS6+C7NWw2b9itLd0hYf7ZU+TjHO+Pll7v4iiCgEO/5p//4n/im9t7kuDJC8hzz9XLBemiYHmlePE4wdQr/s1fr5CFZn29o5hVuENAXGiuPomnVkIEhJQf2EqJThhNje2f8c//X4FfXq/wfoPw7whOROayEWRlwXSaUmUJSZiyNz2/+3uaq+eeWVXx2aclVXVLlfQYYUikJSksdV2RpzmpqLC8RYgJbsw51APGv2S17cllzmpMuV8PHJoWKVrebQJp4fj9Rc8fLjTWdAQiMyyEeMNKIZGiIuXPuW3+Zx6bGbv5FKU6MhmQXoOsEcOC1nistwg1YRjuOJt8QTN2dKNBqT1C7hBUOFOx6+/xXtN1B7zNuXtYY5zCGs12N/LmxrDexi9FY8B5yaTSLE5KdkOL7SWjUQxDTze4KA4aoDcjWS5JdUKWaowZaWuDFgJNQtdF7q3WabxxQ2S0Ki2x8rcL0dNEgvesHh5o9l0U4QaB9IF+HLjbPDDL51xcXtI2I7f39zgJfTtwfnmG1gnjaBl84PL0lPVqSyokDs3F2SO2X9/QvrljNXzFo0+esGlW9LIlD/lx5N9xv77DtjVn50vypGBaTil1xeFwwDnHzbs3aAnL+ZKxafnk40+5vb+B/TZuaHQeA7OgOT+/oihb3rx+QzXLUVLy5vUbpBDkWcIslBRJysViSXM4cHZ+RjO2BCHQVpFIydj0NMmBqqxYThfMJnNef/eOSbmkbcbIaTaOvus4XS6xoyHcDXx9+4bz4glFlrGcVNjWcL44Q2SBQ7vj3eu3sZl9PDVt6p5yOqGcFDFYsnHMTGvNxfk5Qgi+++4bsqwgy1TkpXYD64cN09kJXgqmi4zzi1O+/fYb+m7gdn0LOqFrDZv99mhwhzTPsEN/FHoIuqbBafOBSaYTBUrE4DPJCCIcQ7eBV9+94uzknJ/97Kd89tnnfPnVL3He8fLFJwQizuq7b77mh3/yA8oy5Uf/7l/x9Nkj6rZht9kynU+5uLxC5wlaSerdnjyfMA63XL+7pSwmx/BPMFrLosz59ttvOD89RQnF3//0R5ydnXN9c83543OwkMmC2WLGXtcUk4K77S0CT9PsuL+/RumUV6++QQkw+wOffvHH9L3l+x9/yvnpCV9/+w2ffPIx92/fcT5dEsYWKXv6/Zpcptxvbjk5P2NSTfj6219ydnrObr3l8vSCy7MrvvzyS27fruhtg65gv94xyaYYAYuqIteRGfzx0xdU+YzPPvmCk+UZ//2/+OdMTmacPD7H3I/0Lkpz7x/uuFhecbo44dH5Ob9qGtog+Luvvv7Nb+4AZoiHilmmCV5hxth8XC6WkW8oBOMwkqkEH0bKcsls9pTrmzeYEcwYmM/iuGXT1MxmsxgI41AK2r6HxpOXmjQpEF4xDpbl8pLtxlDkFc6OjGOUzmRZlL4MY0Oaqsiv83E0EwKTaWxfHg5rlHZorejaliQpyWRsLYcQOewX51ccDg031zeMY02WxVZ/Xbe/Hr30cXEkpSYQYihuoe9aivwoaPIjSaLY7jY0hzZy9HQ4jhJndJ2KY6RSx5BZCQSR7yisw1mLMY4siy1NrRRpmtK0PW3boXUaxxW9IIQoTgpBxA2UkqRphrWBfjAY65HOEpJ4YGqNwBiH8wNDbxE4siSKg5xzeGvo+w6hA2VZ4Bz0ncW4ka4zzE9OSNLIMdSJxtiecLTRhyPChRBxIGmaUpYFxoyE4Nntt6xXG4oioLWKDSgHHDd0Sul4KH5EiyWJpPCQ5xVlWZAkimEYGMwQg4zwXj7kQIwE4HCo6bqeqiqZVFOkEjR1y2bcM46G4EGnOWmaImTc7EkR+YVpmqKUwpgB60bKIiPJJGPf0zrDOLoothZx4mYYeoyNBxzeBSA5ymMThn6k67oj7qU4/lwTJ3UEseEjw1Gq+n5ij6M8SVGVFWkSWaF1HTdzQvoo3RJgTI8xNjo1xHs2/G+3EX+4uccbT2Na2trQ9QPTeckf/fH3GbsB13va/YBtPVU6Y3W/ZqwbPvr4GY+enPCv//W/4le945OPPmeo+w8S3uXJjOmsYrvdkCQJ8/ksbtSlwIwGgWa3rXnx9BlSCJw3gI/SdcD0PcvlgqY+MKKpqhJjRrq+R5VT7OjJs4Iyn+CMxw6WZtewOmzo+zlSCk6Xp9je0fsRrS2H1qOSEt8eIi6xyMi1ZJ6fMSsq7lcHZCJRWjG4nmEckCrgRU/XegpVMClmlIs5t/cPPKw3lFXEpFl69ocWmWoObUs/vOH8/JSrx2cE5wkjmLFhGFtOTpbU7cC2HlBpSgBuNg23YQRluHhUkWjJ+rZnpiZs6xopezrfsFQ5ZZJx/+WvUGnK0xfP2Ly7Y3fo0EpGAewRnaSSBHAgFGmWkCWaz548ZZZluHFg3e/ZbNbc9DVcLXFPlnhyhoeOfujIVMI4DnibcLCBwXpE20ZEizHs65rX92vutnu8TpidLJnOp1jTM4aIxPLWUpYTfAj4EKiqCX0YsYPh4PYkWeSdpnlOoRIIKoYsZsCHeAAVm89TDtsdtuvo6ga0psjTOLJfpEgEIngql+Gsoz50nJ6c0tUdO7WlnCQUeaB3DmT07xQyw/aGs9kZu9UtpdP8+Mc/5c/+6T8j/fpXiL5hUhTUxjL0I+WkYDarMD76hNKsYOxG+qaj6VvKWcnJ2QlZmn6YcMAbnHe03Z4kTbDWUE5nNGOPkwGhFO040O1q3vXvyNKUYRypm5rgLbPJb85El1rjfDwUHoeA8YZZJdnv69jKnuTY4cCnnz9lGDYsz8roQBi2fPTZC0Zv2az3bDYHNqsWgeDiIufR1TmD6Rh9DMi0LAmkBBRtV+NFi/E5f/C7HyNk4Ha1Iq8yrB+ZLlN2m4bV7pZqlvK8WLLZ7knzjP2hQWCAgNIJy8WMrhlIc8HJ6YSujfekOq4xi0RHcbVa0BwCV09PGYeGk9Nz3r655YtP/5hf/uSW0+kTrrd/x3Q2J8symnpHqhWDjWH5H3zvD3jz3RvKRFDvev7kBx9zd3/Dk2dnUUI6xtDEOU8QPcMw4FxCkkwpMosICVoVtM2Wtu4QISNVBfVuQB5q1ruRy9+5ot/VcJIwn+UUyzmbesPk0Qk/3hkeXt/wpw8lsyDpbM/NvuN/eHPg1fcvOf/oU/7t/U/Z5Zb5tIJJigstpt6S7htSn1BKRRgNRZFCGhhkT44gTRKEEjgnePrkJV/96ivKXKOU5PZ2zw9+8BTEhNXqHYIOnUA6mfD2+oHBeBaLGVVRcnv7luk0o+4a0nTC1cUTnIfJbMnrV9+xWEx59e4dyzCSJp6mPmDGkbuHB85OL+gHz2wm8PScnk/5+c9/ycOm5+rqBK0j9mlSThhdnFZphgP/4//nX/I7n/4eP/3qOz75+AonNTqruF1tmVFi5EDTH8hKQZrBvr1jonPWqz03qw2bnaHaNKgspWl6+sPA+fKCR8tLrl/dkZCyN+8YsZSzCd9+s2W1u6ecJHz2+afstjUqzOnrNeU04lBH4/j2zQ3nV48wrWWzajDB8Yc/+ATjRg79FlPXTOfz6JLa7lhOF9hRcHfT88lnC/b7jqv5kvnsNy+3wDHs9u+Z4uIo9uTIcw4IEeWcMcwLH/6MtR7vwoeQTyrwXsVD+yNi0LnIWvcuBrnxz7pjQPleIuqRQqC1RmvNOMYg+D0aJgoZ5TFziq3vIN8H8h7jPH6IoaW1BiHjlLMQAusNwTh8eD9p+mtxphAcueixRKNkAkEwmhGpxIew1h8/64WIIbB3FuEdxv/69wnBHX8/EZFSgeP6OiJZlJIkqT628QU6UVjr6Lr2uF7PGIaI3vM+MI49t8n2SDVIqA8d3m/wdvwg+Ey0QmfxNXh/SBFf7/fS1ngoAEdeOiIiRY7MEGNH2rZmtbqj7zqyvKQsqujIMfG9L01FVVXcP7xjt4tTvOv1A0VRkmUZfd+z3W6ZT4BCkqQCScToxAzv/wfxe2TuhwCXl1e09R6lktgeP9Tc3t5zc31L1/fRTyLUEaMS/T7vJaDvrxU5yg9OJe8d1hpAHqWp0PUGJROyVCGx9F0PPuBFIJFxaiEQX6eIB/R4K5AqtsOV0HjnYjEGFTFD9hieCwHCIT0fXnchHVIexaLHA6R4eBE+/N6xo/D+erUYY+Jhh9YE71Di16+ZOF6D/6HHPxzn0q8pZRTNdaMg0YJELenHjLX+JTtzixenGL9BiVNMCOy7mmX2FGe3JCJjsALEW5wT5KmmMymH3rHISoSuyVVgz7fgS3J/yRC6OIakc7Q4R2cJw7hmUeR0vsG4gkT1BDNj279jWhgSUWBlRwiOKoO9NQQ50AwzqlISRsmoQGCwY0tKQhCGwT2w7gd0IjHcIsOSZhQUuuB8kuJIIp7HWUa7JZVnNKNjmTlWRtP0LSfVHCM6hv6EofWU2ZJR3TJag3Mz6q6hbq4QAt7cveb6fuD6bc+40RSpZN1rurCjPVjaZowc2jIlbTOufvgJhGgyjh9C8eREqYS363f88ts7DvsWM2gm8wKVK0yw3Hy74etvHX/ZBrR39H0gKEESBIEd4y7hP/m/Pou4Gjw2WAhjZBEJQaJi+/DhsOV0+Tv411/h056TuUarQAiaP/r9KfQSLQaePi4xTcbkYsbkzFBVoJQH0yB8QeIFfbBYenrnGAy8Xe857Gs663Byw7Bp+O6VoTbXNA8K4QfSdE6wPaksUGnCyUzz6dWUZHKg9q/YrCFL4ig8QZJkBT6AQnJW/DH3/nO+6/4SLwTBWTpnECFDSgg4pBeIYAk+IMOEm/tXCJlR15qkHGkby/1+z0xP2A0NgkBzUNxdt5gx4dB1PGw9bavpGoFQKaenCfumR+SC3eBxQ0/TWrKJxlpPWWSsty1BBnKVURQaFTR1H8Ofzd5gmsDiNOFwMLR7R7VIsYmnay2XlxO2+x0EyfLkt2uzpTKOxh7SLTrPIREMnaWaLuidRSE4Pzlnc79BCM2jy3MO447npxcEq3nz5g2PLx4xm0/Zbdbc3t2xmMzwXvBws0IdHI9nZ3z51ZfUdxtkyBmN5W53z9mzC5Ae5RXrTU+WDrgAGSknk1NenL8geMl+d+Af/cGf0PUtp8sJD+sbvLScnZ/wWGWs79acnl2i04Jh6/no0TnX99fMlxXifMJ3X79ifjqlbneMpqVvDtzd3pDEeiNt3bM4PUFKjwKePnuCswbjDIPr+fZ6w2g7TiZTnHOcX1zgg2Vf12T5SDvuUVIxncxIdNwUHg575vmE5emSpAwkW8fTqz/l+vqe3a4mKwqsgKHtwTpC6mnDwOhsXDCEQL/dYZzHjSMKuLVRYJZPCqpZiRegcsc3r79lX+85W57y8cuPebhb8+knL+jaHevNNp70B8np+SnD0NPUNc4GiklJkI5Dd+CjFx9RVhW/+sXXaJnQmwHjLevNmuZQ8xd/9uesthvkJHD97i2z3ZST/Yw0g85sefLRBV70EODRkysm0ymPnz/hu9ffoVNNWmTIVDL041EUmJAkUXp0ONTU+57d7i3PvveIRJ6QBEOmIqN4U29pxw6hNEPvUTK+xuvt6nhwOdL2PdVkzkHuOdQ1gzH0bc3ZySkvP3/Ko8fPqDcDF7MFh9UK6R373Yb72xv+4z//p5zantuHd7SdR4+SxckJoUi4XT2wmC8xxvDu7RvevX7LX/yT/wiB4LPvfcFPf/5jTL3H9iOnj0+YzafkecbD6p6zizNSV/HLn3xNKZfU2x6popzu7uGBLJvQ7TuWyzN6Z7hvHhjFwKhatuOGw6GhH35z50FZRpxS31vKMqMoc4ahj9NG1pBlReSBC9DCk6SexXJCnhWs1iu89UiRkeicqiqQypNmII/CuSxL6DpzlOcInA0kSYkkJc9KrF0RfEeWSaR11PX+GHja46IzcqnjiKfAmIFx6EgShbUDTdtSllXEyShFnucIGRhNxBX1naFteto2Ch7LoqDIIpYjTVPcJNB1XUTZqAKlE4K3pElJluZIERdWUkZZpTMjUsVGuQuGoTeEYBl6S99ZrAskaRkX3MNIEJ4013gd2zij6cm9OeJkHP0Q4Y5JkqGThDIvMF1L243ouiHNc6azKVU1JaCwjshWDzKyAocOYx1KJSgVMX1KJkcRryYQF7A+WFKVkKYJbTswDgPeuQ8NkbiYBIhooXGMQVhwHjuMOAKZzijyjEQq3DgS8HR1w9ANcRQTQd9ZxtGg1ICUCZNqTqIE6/6As4E8zymqEhEEzo2MY8Bby9A39J1lUpXkRUUx9lHm6QbG0ZAkKbPZjDTNMMbQ9yP7/QHnAonOjlLYAoTG2bhpckeBkdaasqoYx7hRsTayGpVIaduB1WqLNTEAH8YBY4+bghDlfWaMrXilErq2iaiXPCdNc/q+iWJT9X7jKfEuTgRkaZzis+OI1oo8y8nzHBDxve9bpPSkaYIxsYGkdcpiWZHnGV3Xxkmv3+Lx7s1bgossyaosaZqOcpJwc3tN3w6sH3bcv31g6EZOFgrpNG3TM7aO+eSESTnl/mbFyycDH714SnNoKHOJ1oE01TglsNZyf397bKFn7Pd7TpZnnJ1eUJUTgrfs9jXW9Biv8S5HErBDT992pEmJs7HxFpynzCr2rmc6X0CQ1PsDfTVgOkOWJpRVju0GhPf0ZqBuVqR5ifcx+MqLDqUsTdNRpRVVVbK6f2B/v6aaVxRSIZVkMB15mtCYhmAkz559QVnNabtYAjJ2BBxKB1a7A21nWM7mpGUe/1xfc/fzNyynpzx5+Yyf/PjvUNKx3awpkoKbw4GmrTlZaoryhDEoDt2WvK1JVKAICWawNPWWVGWMosfKlqf2Mb/65hs22x1nV3P+4JPPka2he9jjgLZtUUpinEWg8SE2vp49e0oYDSpAXR/ouhrqjlmaci8D68wTmo7LusF5h1eabuixo+ZNv0amGbJuSJ1jM9ZsupHbZs9BeC4enVP3HYN05NOS2vT4EMiKnCTJ6IYOkNE7kQisM1Ee5i3GWiopo1RZZQQvkDKOYFvjUVrirMcYSy4ly1mJEZ4k1SAlgoT6UKOCjZ9rPqHrBppDR1VUBOtxNoYls+mEkQHwCOD1t685P32CGhLapuft/pphGDk5PWFbb/FBUOYVg0+wxqASxcX5WQzlBkcwgsHH9vn89IS0LLDGELzHI+m7MXojJjO894yuZ/ADaRmxD0mS0o5DxGZ5T6ELEplxWhUMoePk/OQ3vrc76fDdjsJGFq03ga++vCah4os/vKQZ92TVhO+uv+b586d87/tP+elPf8LNmy271Yq/+Kc/5Be//BnffrMGUXBoAmG14erikqatMdZhxobTZc4//vOX9J3BIOhHw7vbL5lP1jx/9pRh7DBOY0Jg9AYnPGPfcHY+we0sy9mUh1XDYWy5OEvohx4w7PYbJtUclSzYNz1NfUClFe9eR2fY848kVSaZThKywlC3nu3KcXO7ZjE7RSYekT8wv0poEsHoakwwdA6KSRoPyYLkxz/5e7IiThpMpmDsHoXC9DIe+DcHhNB4pzBOYI0ly0pk8Dx/tiDVOWNnUToWWC5Ol0if40ZBNVX87Jdf8fKjloGRXg+cn57w+u0NV1Xgrh6YPn3Od1Yzvus42XQo4Jet5+/nOV3Y8atXbzn940uauwOtE5zMK7xM2Lzb8elHZ4xdR5JpZApGWb761Q2p0kwXCYfuln2r6Xt4+/pAlZ/g+gOPH0+5XJ7w85/8gssnp5xdzsiVpt55vvrlgctH0+hVWm9xVvL59y5YbV5z6GvOT2e8/m7HclGiU4tTjp9/9Y40K7n9LjLJLxePePv2DXZwvPnuNZOqYnE+5e7unm+vv6QetyRKcb/aRWfD/sCjyysSKdCThH440HYND/tvqU4KbrZbEpkx0xmHh1v07JS7JsqBdVYhpGGzeoAq5+LRJ/zV3/6Es8enXN9+x/OnV9QPlsX0HBc61HTg/OMF/+K/+wnVycjVkzmT6YyT0ym/+uodj55UpPnI+ZXGjmtEOtKOA7td9JI4M0AY+fTTz/hx+0vKKsfJmmcvT+hHyes3B7wPJEnAi5GgPG9f72jqiu0Kymlg/WAw7fS3+u6OPh2JFBw55+rYIhcfAlcf/JGF/oFsHsO9o4DTOYe00csTufX+KH18z4OOZQ+Ik1RSRt/dMAzHBi/HsNR/aFQLEYtTQoTj4ZM7BpMurk8BpWOBwLkoxfbek6rkGHi76P3yDmPTWF45BpsCcUQNGoyJn+OJygj2+FpIEWWix98jKmZCfA7EgwX/QZZ6bLsf+d8IeWy6yyPIJCXLo3PIe0vwikRHJC7EcgworPHHz4dYTH2435NnBU+enAOS+tDFgJo9aarZ79ckKXGy838hbo3vTzyEOIbmCAgC68OHIF/KKLU81Ducs2RjT9fVpGkWJxpViveavg90XUNVTbBuREhJ1zW8dyJaa+mHLmIclcQjkC5OfPLvP61/7/l555lN53zxxRdoLbB25Msvv/rwPiqlI/JwtB+8Lb8+HBBHYWnk0Uspo+h0HBBYlErJstj4N6MnyyY4WxOCJNETnBOxJqWOB0JCfbjuhfAoBYlS0X0UwAVzfB0lPgQcsfwnkMdr+ijCDSCCOE5S/HoSAyL/XOvoPFEqFomstR/Eut772Ep3FilCdCBJ+QGB8x96/IND9N43cUFtPVWZ0tiBMruht55MVBifYu2BbZ2xLBsm2YTcpUh5IIgUpSwaifOSxnTkYcKhd8wTiQsPBNHxaPYRr5qfcZpbDt1XeBROFOyNZZlfEBgZTIsNUzJm5MIgVc+qK9gNcDqdc7Nds5gWWDeSmQElOqRXeN/QDRPCMKOqLOu+Z547UlkRQodlINMpWRrD63VzQIgTnMip5Q4l9wg3Z9duEF7T6TVaCTprsWZGmQekGJhxxW33wMUkZbU13LQBEba03Yyfvdqx3a0wdcXt/RjtwFuQwjMMI1VZMg4BWWZ4GbBjIKtSZtUpXzz5XYTzx/GW2GAVHlAJfZcwjppqNmFsIEgPUvH6yxEvB/AJ/SG2SELwZImgsx43aH7/o0/5/OXHBBHNuv69UlkIEpWilaK1Nf/tv/mvKU4S/viHL1Dza15ezUjyCbMysoSC0JA8oHTKOAzUdg/hhNt1g7GCedlwdxPHW0PiMYeU/a3j2+s73q0tiR0wMiGYlrOp4uyiokotpy9KqjTh6dkjHj31zOeWQ7emnHuC6ajKLTT/DbX9Pb7Z/gu+uPq/odMZXh4Y3TtEqBhcS2M2DFmHkiXBl3Rjiya2xLa7MYa4WjI2e7CSugn0xlLXG5IsY98mrA/w5bDFG8nQOtbrkaFzFBW0g+LQBZLUM8uh3gW0LClnnp2xSATWS2QmIHF0LZSFYlpNIguqNzgrQHqCFLjQMslyDmakbhxKaDKdMNaWNAmYJvDm7Yb5PIMgGYffrs2W6ymTyZL5yYJ63/Htu2+QmcYGx+988X3WNzeM7cCsnOGsQ6uAG3ZIpxlqTephWVbYccTZkTxP2Rw2PDt/Smgc3//DL9Be0AnH16+/5uPLT+k3LS5YXNfw2ZNnTKbnvJ08Jq9SFqc5pZTcvLol+Bii/M7LZ1w/vGJ6miP6jm7fYJwH4cmCoEpTtpsVgxA8nlwwqyZ4eUIykfz4p3/H/SYuzM0wkqSadjQIpcjzCik1N9drXj75hF3zEL+s+o6mqTm/Oqe1HW3oKKqM8+qM3e5AsB4bokhOYji9OmWz2XIyP2Wzb5hkOSflhCRJ2R22LLOcJ48uyZJz0nTJ67ff8urt1wgJk8mMWVGxWJ5RTpbsdgfWqw3BjPTbnrEfubq8Yrlcsnp4IJ9M6LqO+/VNlASOYIUkryYUZUkIlkmV8OrVLwh+4PL8hPV2Qz/2bHZbEq3Jyog3kEpDEr981/s7VHJBc9gzK08QmcIQ2K3XeOf45a9+jp5o/voXf4kUmn/5P//3eNvy5deC1w83XDy9onFrVm8PXFycc3t3y/qwphtbxCi4Wb3j4uqcrh3Yb/aUZcbVozPOz87j4WpeIrTi8ZNzxm7P05MlzXZP3XboNOHFRx/xsNnSdgP7/QEZBKcnSzb7HXd399zePzDfz/md7/0RSgd+/oufcL5ccnV5RZFNCV5QJIpvvvwFl1dnXDw6w8vAH/3B7/Hw9i21H3ny5AmNKTibLDh5WfDLzVt2bcPF6ZKf/+Ln9GNPKlO22wc+/+Jz7lb3ZHnBi9MrrDe8u7+l6RvOL05x0rNZbZh4xx9+/4dcXT7hYXvDixdP+OrtVygqtusaaQSffPY7vL5+zXrY8Wb9itkqp0xznp1dgU9+43u7yNMYSBuDVkd1pnMEZ3HWYKUkSxNOl3O865DSRmng4ShszHLOzy8R0lGWFWWVslrf4n0cQy2rKSFE7p3SiiyLiCmtC4RQaC3Y7/dorQghcDjs2W7jeN50lgGxJZYojZCCcezo+4Y80yyXC/Z7T55lkV/rYpgiidzp3ljevbvGjJaqqEhSRZZmlGX+ITyWQsTwRCZolVMUE8zY4V2I3MVjcDMO4wdWYKpTxiHucQ5jj2z9cWEfre6RhWgwzhJkFPmkMsokwSGlIM9zqtKzT3t0mjGbzimrCUrAbrViGLo4mhhCHINMNKmxFKWgmsbft+taumaPGT1J+mvES5qU5HlBliXHkcu4+ZEutnmsdRgz4jxkafZhM5TlcZKwbg4c9nuapsHaOPIbfGyeSykjQ9nGv2+aA8FLpEiPDWxNIApS09RGzqBMULJDCkmS5KSZZhyizDXVgrzQWBeFnNZZ0rwkSUuGsSYER54XJIkkz8qj9EcihI7vufUMzpKkHjPayONExDDPO9rWkCTqyOR0dP2IkIE84diaj0G7c/646QFjRsZxIG5w4nh0+CDnij87zwvSJKXvuqNIK16/cbPhkEhEJkiS6JV4P2rsnIsL9+N7JZUHaQnEVlaSZkwmJWmW0vXN8Xn85o9qcYLMR7RKKaqK1fqBru/58ldf025b9vd7Hu5WcXLEQhAeLz33u3dMHyRlOaEoeh4e1vyjf/SPuHn7DqmJjXEEioRUJPRDi/EdZZ7Qj3ump08wYcZ0MmGzWtOOA+DI8hwzdMDA6qFjHEfAs9tuSJBU2QRnA94F6t0evKSuD6hnj9C5YjpZMqsqWhNomob5fM5gNlgcm+0WoQSTfIGSgf16RTKH3jZ8e/sV0ka2qukG8kwiXMCMjiQpccqxXJywqmu2fceh7fAhkCjBtEjZ1QpVaM5OlojgOewPWDvysFqzWrc8ffYZP/jhn/Gv/uX/QJ4GxsGQobBo/KjJFzmnkxkvijPOz0p837MZN2ChtT31piUvzmnGkbv9jk1zoDeGw/bAbrdHS8Hi4ozm0NFsdqRpjhscIihkkCRSsVuvOM8nLCYz3m1uIHiq5RzpPJsR9rRYF0h9R6Y0KoDxlo0e2TpL0cHMSQ6HPbXruWsabFnw4tlLNtsdm66hKDNGawkiELQnCAhIhs4ihEemirQskEYxDj1aaYauY2cMk9kMoQRBRVGXej/NoyTb7Q45Wk4ncb0lGVFCMNrA2I/4NjApK4QILBcn3F3fUe93pDrBBEOuc/wQ5aBOFKSFRjjY7xrKcUeuNb0u2R0GvvryFZM8oZqkbFuLFBrf9hyaGh88L158TNca1qt7qqzg8tE5TgZ8CofDgcV8hhOWoe3x0jOakelsinUWLVK6sQch8ARMP0b0TpWSaoXUksQLqukEmZ8g/qG78f+1e3taYIdAN9Qg4vfrfD7hb/7ygcVlweQkMJ1X7Hcrfvrzv2daTRA4lJQ0h443r9/y8UefMvTvuH33HZt9x2AkhAcmkwJpLR6H9Q3GK4TSqLREy4q0cExmBaPpkDo2O4fRsFie4M2OvvO4UZOICh8C82lK0/VY2yGIoYUQmrYdWCzOGTrLYLcEr/neF9/nb//2p+x3A1muyPOWuqnZ73fkRc60mnB38xalLe2woh00JyfRNVWUKdNZiXOGPEtJZYYdatLKIZOcaV6iVEXfeXbrDZ9/esGP//aAUoF+GFieLBhHQ996pArkqWJ13/P8ozMeTTK2+1vqZsvYlHzvkxc8bG958iRjOsl4uNtzcRG9K1UF6aMlr9dv2bgHTl6UbM4KHn58z5svN4za0YnAVZ5RZoL1bsXvfPEZP/7RNZ9++gmv3v6SJ5++5LDfM5/NKKoCLx2r3ZoiK3F9SSJTyGoeVht8UGRZz2efvOQnf/kVdpjT9gOX50+5fveOtMhYnGbMFhl/+k+e8zd/+++oJoqzsyesHvZ8/e1rUB0yybh/uKVIZ0wtfPXNW66eXLHdv6KuG5azCVpkvH39QJJkGAN4z2ANISyPThJDNVngRsP1dctsNkWKhC9/9Y7Hj59xaAYePbmg6TYo5Smrgs2m5fxkwszOCYsZN7drzh6ds9rds1kdODuZkSYzHh4anj/7A56/cLy7f83TF1PSRCPQZFnJbJHz9auf89mnv8sf/PBjdoc1SgVG03N2fsLp5West2uc6/HeMVhDMYnTa8Ng2G72lEXJbrvlO/sNs1nFu+tblHY4v+fy6orLk6ds1jVNvebiMifRnsvHS16/vqHrGs4uE3b9Dpf8dvtuawNwxJR8CJr9BxxICLFFrtSvxZHeR0Hj+8ZtCCGWKx1EyaUlSd7LSeM6SgaOP1OjVBJ9O0d5KCGuDZWKrer3wtG+H7DWMJr22EL2GKs+sKZjySAlCBvlPQiMc/i+gxCwtse7kUSnJDpDHvnm7ihCtXaMny0yNt6tH+I687hHiWgUiTyyro0xH1reQqXH8PrXz1cIdeS46/h57wX4hH3Tsj9sI1c+ZBGRO8TWPEiscUTxqUSIhBDipOXd7RYpFI8eL4lF9zgN8NHHL7i/vaHve6RSx7a++tBwjkWL8OF9lFLiXZxAFSi0loTwHmPZAp5+aJFSMJ3O0Bq8NyRWYuwQD7B7QZpkhBDQOrbQhRDkWWzGK9nhgSyX6DQ9BujhePACHA8uJJAkKV3XHdftkCRJdL8c6uM+IMTv/mNbO04N/PpARkr5YUrg/XTC0A+EIMnz+POM8ex3LSKknJ0+pW42cWLAAdIzWEc8BBd4G7DeooRABE2SpGiVxulYKVEiHpB4AsFLnCP+8+N+8dfP6/j7hl9PPMTfOF4bSim01h8mMN6/b1JKxiE6I6SKQtN4X/j/fXEunTOYRqOlZF3vmCUTdrZhUpyi8Hi7YFlZzm1G61pWTU2uE/aj5DRLOHQCocGNgnkOAsdptY58yuECxcjNeM3QS2pShAoI26C0Z9fM0KFBCYE1Z4z0FKpnbXqCHZnonJOTM3b9K6ye0dkHpiKnG6CzBZezC9wwUoqXbJPXdL2j1FeIsGO0gsFUnM5OqN0K60YSdUmp8+NIQ4iiyN5TJXumRcpmbyMbXWQYcqSFzS7l7bjmYXdAiJG3+ZSffLXj739WY4eUpLhnewh4L9GmYzYt8INgcamotwOFLLHGkaQJdTviMJgRhr3m4uIpJ9MleAveET7wniIDat8OPOwMwgfKhUCrwMODYXc/khUBlRtIJcoluMHiR0lWOarJhP/yP/q/UOTFh4a7DwHn4tiykoppdcov3v2Ef/fVv2W+zBmGHR/NOkRWcd/e8fAQOYvrdQ7KgDhQTC02CN58dcD0Peu95/zRlENX4/qUrJjwdHrKIsn5Zz8QbLseqSzTqWBWLlDZgcn5GyqVMrqBbDwlV4I0rxnEiNXx82GwBTktXfIjTHlNQc9XtsMYjxSGPvwUH2YcOskYdujilE1zSyJi0xgB92uDFwNubBiMw3fx1PFhLblbtSDhYAYO25Sxc9RNZKFXpWa3UQjdszsYRu/I0hwvYLCWJBWkxcjd1iAyjRSgU8doBdu7kbRIECFDa8lmtWc2zZBCs98YnFRUhUJmnrRW2D6gK83p44Kmb1hMp7j+wDDCeBBMTlKc+g8LEP63HtPFFacXT3l3vUIhyfKC0Y1Uk4LetIhc4JxleXFCUzes1jcQJHlaMCpLmqVURcHd3Q3TtGB+UVEUObP5nFfCcd/eY5qRy48f0acDj14+4vrulh/+4E8I0pKXJbv9QFmW9GPDOMJ0OmO2XBCspppUrPb3eG24Wd/j/BBHe5zj1bdv+OMv/pBpOaFebcjmM9Q052+//gn5VOMaw6o5QJry+PlL6u2O7fqe87MLJtWUECJi4Oz0nMePn/DNa8u7N69pDjWTqkLLjPOzS8zdHT0DSTLh/LLk6skF3735lm4cmJRL8ILPX3zOKAI//+5rzuZLnp1d8tWvvsQrQdbF8bNOJZTFjJPlgrz4hO9efUeZFygVW9XLswvEQYIQ3K82BGsI3rHartkddigpefniBXY0tHVDkqe8Xd0hVDSL51XJoWtZb1YEH7h8+ojm0KEOkklVAjCfTZlUJWU1YXX3QFJIRA4uWK5v70BJdJaQTAq2mxVlWVCmKScnS+73d7x59YbPvvc9ZF7zdz/7W+7ubrAKvkh+DxkC3XrPNF2QZzlBOh4/ueSXX/8CY1uUOMG7nqbZkuVL8mJOkmrqzY4f/vCPMN4hleHLn/6Sqci5uHhEnkuen5xjBsOTqytev33N27ffcXF6wWrt6YeRum64OnvEyWzOp59+ghk7vv76V7jUUh9qtEwR9Kx3d/ix49nZM2ztCKNDSs/9q3e0wvLm9VtOkykvvn/F7WaL956ubeknE7IyBkDf/97vMJvP2W63SG9JlSAoRVFV+IcHlEpZ3a3BOmbplKvlFS+fvKRtRoauY1qkKCdg8CgbWM4WZCJBB8HDwxopHel0RiI1n7x8jDfyN763F/MZeZ7RtDEUMmNP8JYsnSAFtE1N1/bHRYY9sgYjQ66syg/YjEBgNAMwYsxAmqYMfeTNCeGOi2FJmubUzQ4p4/hfDN07jEmRMkHr+AEuZTiy0jNAkCaTOMosPGkmUZ0g0Zo0SSjynFQX1PXAfr+PLUcpyFKNEoK0KlkspggRoqzU2A9NGTja3kU0skuRkqWSpmlo2watA33fYs2IMT1lWZEoQdePJGmKcxZsYDItKEqBt4a2a+Kf8Sb+/OCj8MZZlEyO7ZjYbizLCqk0RRGlqqlO0Cj2+y1llVEUJTpNYktISdIsJcsmR+yORasUQRR/CyRJkjCpKmbTBdPpHOccfX/A2IEw2NhYdHHj4X1AZcWRozkcR1d7Dvs9h/2B+tCiZIKzkdEuZWwsjeOAGQesM/R9H0U8Pm7kEp0xyg4z9ozjiDGGIs+ZTudkaThu/gzOmRiySY+UGUJaEIHRGGRv4uY86IjYKlN0ckTFHFmgic5IkoLgDd4LlEqxNmCOY6VCxI1L33fRLZIcG1oEEq1xbsBZf/SkxIaXNREnVNf1sWmlSbQ6bqhiS6ooSubzRXzf/r1F+vvmu7MujtzaAaUky2XGbFIRgqfvetq+x7u4WUGWBFnjvUFpQZYnkcGY6A+blGEwv/G9DTBZLknHnr5vUanFiwGhCg6Hls3tA/ev73AeVJpincEwoDNoDbx6HQCNTjOMBxdgeb6kszVBpqRa09U92mtKmTO4FqU8T56fU84Fr99tOCuX1NcHBjsym0yYFFN2hzf0455yNiVJcg6HA0PfU+Ul8/mSb9+8Q5PQ7A8kKsU7g9CQVSlFVpLrgtn5FLvwvH7zFpVojDe4MCAcpBS4JtBvID+dMvqe5eMMNyps4qg3NaOPkyOz2YQBw9nlCTrNONS3BK0IQXB2esZ40JSp5ny6IM1ytE45NC2HfX1sMGUYK/kf//Vf8p/9s/+EyeSCer9nvz2QaEmRSRbzOViLaQ48XG/xzTmTqqSan0SE1hqKYAlBUI8DJtHULjL8pZpgXAAl2O92DL1hPwwsTy8wvkcoaJqaoirZ3m85f/ERvRmPPgVF3Y8IAxeq5K3r6RlgmTEJBWr02EqzV4ZuNBSjZL+uMc1IUiXIrODq0VPe3a45bLfkWUo39Dh3FNYlgiRLaesR7yVpIggSdBaFYAiw40iqE9IyJdGCkEjQiuS4sQ1A8HHD3PYjdrScXs6pdy2q6+lrQ+g12mlsY0mUZqi7OL3iLIM1OBFYzGeUxZTZZEFn4qa3H0bW9YblyQT6jqQ4Y/dmz49+9BP+2X/8B8jgUUVBVeYkWtC1TeS7+wRvok+AAJP5hH1bM3QtxhmSNMEMPcYNCAleOAZzDM6Fj3/1Eang+oFEC6oqIy00iKPnIAHnAtdvrn/je/t0sWC/a8DBpmnIVODq0ZT//GyKSCwqU4DHjOH/y9p/9cqa3Wme2G+Z14ff9vj0hmSxyCpW13S1pqd7IIwgNaALXekL6APqQpA0A/T0jDRtyrDLkEwm0x67Xezwr11OFyvOIRsaTDXIDuAgEyd3RuyIeM1az/95fg8hxGK3ajQGB61wPP/+mvff/5gffD7j69/sqG97TK9pmpa8AOctZ+djpAps1mvGoxm7w5I0yyjLMW3Tx74D49jt9kzmE/bblpPTOZu7HW9exoGDlIHDYc3dckuqNGWRkuqcvhsYOs9u1TCbzhmPcqQoefHd93z6yfss1y/p2kCb1eR5yvVVIJ8JpOw5OS2xdkcInuVyg0pBpR6VCqYTzWHXcTjU5Nry/gcfsN1vCGqgGI14+fyW/Ubx0598zGZzyw9+dEqWObb7JRcXZ0dHbMnV6y0YwelixKuXN8xPNVrCaJSybje03Yy63VKOU7aHNcU4EJSjNT2zkzHjaky6SqmbPT7vqZUm/9GM6smMYX2gub5jaz1SGaROeHP9kidPFyAGZtMJWZJxva357MNP2O53HLYtp4tLhN9yf3dPXig2dy15OmJwivYgGQ4p/+yf/XfcL9fY+xvWuyVDJ2IfUqOioMwNZw/H3C+X3K1vKcopJhQgNeNxyd3tiip3BHoePT5DZ5Kn711wf7tBuJRXL68YjUuUDhjXkucKPFxd3XJ7M/Dg/CFpUnLo17z3bAx4njx9j3/7P7/kcFjz0cfnPHjwmG++a9gfWurmBqEkddNxdnrC99/cUteWc53y3pPHzKcdr19dM5tW5FXJi5cvefL0ESQdVak51AceP71ks6oZTU64WQXeXL/k0ZOPEVcdy/vrI/9+YDKfkaYph8OeLMu5X+65vDzHWkPXxDVfvT9wfj5hZVfcXtdURUGRlexXHb5d8+jyI3qVILD0/Q3ObBhPUv7pXzwAMdB1A6N5xXL3+5/bELEsQNRywm/5zm/dzdE0Gd4JlxCTiBHlEo541zisdy4yypUS74TlKBImR0E3ioYROv67iJG4t3fOw7HwEyAEjzHmHcbjrTgcQjji7aKgKmV0+kYETRS74+/pj0lIG5nXHhAqokOdxdghus0R+BBQKqJWkkTH15TRxDK4+DtYE7FaUiqk8rwtkFTqyFmHd1icd459IUmzjKE3737v2N+jfqcPiPhCIXLVA9FhXtct11d3FEXC+eUUZy1ZnnFxcU7wA9tdTaTsHFn0RCe9cxHRFx33grek3bgujoaVt6xxKT3G9IQQjuJzT9MeSJMS7+Pw15iYmkyn2XHNKmM/wFuMScp/go4RUkSUi3hLjRfvXOlvteU0yUAL9ts1b95c8c2337G8WzKYiIu2ESiOkPKIRBHvvu+33605Jhbi84Z3Q4Q4iHF0rWAyPuVPfvLH3K9ueP78e+q6Bm0wriX4gLMGpUBYh0g0Iki8i8NpeaRtCCGjWB6I+0whCUIQeCuYx+FG7En6LWIxHo9vfyd3HFS9Xc/z7ufeDTp8/GccxCiCs/9lcS5ZmOPalNl4RVWUVOoBpNcMvsV4y9noEiX2jNJApSbcdwKlcoSt2NRryqzAUjMZj2hNgR1mDGLJRJ3jBSQhJQ9TOhkI9kCZZMikoPcdjWw4tBmT2T1KnTDUt0hfUKSXeL+jMS1CD8z0glKs6IeMIRkwKidRktXhCsiZjQakUUyqE5y9x/ucVD8GsSFYQxqeMMkMg+0pigJjrzlLPuJ2SEmkofdfMdUPaIPm26sDr7/7Hs89PhG8fj7ifrvBmB6pJPVqiywSjCvY3PZcPi443A6RXzwVODugtSBYhWkkJ6cZ602DCz2lTLEhoKtAKVP+xc/+ebxgWIf3by920UXnreHm/pa0DCQiZ+gb+lrS7w3BBZyX2Nohk8B4UtAHj9fQN4b/w5//BX/8yQ+PosnxAh4giMi9TNA8OHvC//jX/4am37N/WceT6leaX//8CoNjtwSdSZq6JVcJZhjiwa8DWjhcp3jvj0b84MPHCN0xm+SQaC4WGVW6QcmWwUOZ5hhzYJRZajNQpRWrfkVWSoK8wyV3bG1AJCekqkHpnDJ9iBFfQ+I42BU7r9iKv0KJQOxWaHBiR+8s2nzEuv+O/eAotUSJjn6w7GvNvm7Y7Sz7Q87zF3sSKdluJEakIO1xQZ3RdYbB9TSrQJsEbAioRLLbWSZzjW0ULnf0vUdLwdXVwP4QmBQafEBoT7+FqqzwwWB6S5KlpHlC01qmk5TRPEEET5FJ1quW6anGWYX1Hieio2uz2TN7oNneByajkq5vUOo/+1T+X308ePQhu13D1fWSUVHS9zVP3nvIZnfPoTUsV7fMx+d8+/pbZqM5RVXRrvY0+44gFB9/8hG/+fWX1Ls946rk5GzB4nTO680NB/bc7zuqpORuc8ezH7zP6cNTxmeTyLi+eYm1luvra+aLc37w4Uesd1fc3twgXIKWktv1DUWVcrN5A2mHlhIVCtzgqLKSZt9ysbhgVBh6J3BYZC5wwrJv9sxnczJdomXCpJrijcE5x83NLU8eP6E6lql9++3XDK7l2bNnBOupyorLBw/5zbffkOmC5W7Pvd9TjCVBDCxOx3z97YY8DyhSQuf54te/5MlHH/BHP/whyzc3nC3Oqe3AMEQxpW8HPv/8nKbNMEYwripOZjNmsxHfv3hN0+zRiSIvC372T/4Jv/niCybjksV8xqsXL5lNJqRJwnZ1z5/88U8xzrLebam7GimjCKdTzc23v+G/+q/+KZ+89yl/+W//klE7YlxWzOYzNpsNxgqEhAdPHnB+ecpX334Z47fTKYkqGTqLJt7kd23H2XTOze0VFkciU5wxNLZhu77j7PSEg2lAe4K1BGmRWvBnP/sZX/zmlxz2W+zQ8t6zR6SJZOt7/uiPP2e9XjEMLTe3V5TFiPvVLYvTU7rOMRudYrY1+33LZz/+EberNbvVhjzRpCrQHuPgxgwxRWMCmUrZ3C959fw596s7FrMFbZYxDAYRHEUhef+TB/zN3/2cv//+l+w2O87yOcO+QRpPMB1lUZALzd/+1c/xj2fokxyEou8MaVbCRJGUBW9urvDDQFvXNH1HNhmxqmuGweFMYFbOKJTm/uqGD//0fU5mC4bW07ZrerNDebg8uUTpjKFraPcHbNNj9i2z+QhpAypR5FVBno9+73O7LCNi4l0RohDkRUY1Ko5lPg6OTHzbNygF96sl0/EJ43GFQMZ4punZrA8gIxtw6A2g6Ic2RvoHDyIu8AUR0eJcj3UNRRlLZJSEPM8pyxFCeNJMgbAIYryz7Zp3GBcfDHWzP/KrI35lPMrY7w9kmSLLRSyNTMpjEZImy1PUOnBzc0WWZ4zHY6aTKVmaY2w48gkhTTOKomAYohM4BM8wdCSJJoSUrvMEEfl8OokbhKbuyLIMIUUcEKj+yEO0yONi9e1m4nCoybIJSubkecZg4mdcliWJ0thhwNiBxcmUssqw1mC9w3qHTtJ3cV0fovu9yEd4WsBEgTlNyfP43HVd43zEP/WDYTBDXIiGcCxxVe+YhU17oO1iJ0JsqI+lwiLAeDSmyHIIHhECgoBWkvlsStf2iCCx1hN8dKZYZ6jrPUVRoVSMxb7deIlgMF3HMHQE3+NCztD3tG0bWeRB4oNCJSVSxvJPpWJPiJSSqqoo8sBexufQKidLK8qiwoVA07TH9+Dp+57BBBIn0ToukK0zsfvBeZTKkUofBf8e6/t3vETnPNkxvdB0HdZ6xuMpZ2cXVNWIw2FH3HRplBQ4748bNY6Odc1oPGI0KgjB444M+ij6a4RMqYfoOhpVE9JEMB6PEQj2u5rdNn53f8jDmNgrcKgbnA+cnc2YjOcELzm7OMP3Bms8TdeDcxRphuk60iplu9wznc0ZFRMSnbBer5iM8zhMykpG5YjD8nBE81i27Y7iNEfouAmxdkBpQdPsSFNIUo9QBiEDWVGSpjmJ1BGnpEqqfIoSOYlQnM1PwAmkkOjkFBkCp/MTpMiZz88Y5TnBe757/pKiLKP7KjikEBz2G9p9z9n5CWdnJ6zrN+RpgfEullV5T103FPkIpXKKPGc6WZCkKUmaYoMjzzKyNKOYLRiaBikKhj4W0kqRxuPDe2azGavVjrZp+J/+P/+G09mM1rS03qC0QhSSoR9o2o6yKkBLbtb3GBkYjSv29Q6rBOPZgmHoaGrL3vTIUcmuvSfsNyyGMx6engGBtIR6u2OkJaHQWGnYtjVSZ4hSUSwqDq7FasHqdh/XNFlJqjJmXnM1tIjpggf5OZuup6sk+xfPGZGwvV3hdh3aAyEjKMWXv/kNxgkSIku1PXSMigm1sSSyiIIULUmRkSSB3vQI37GpN/ghdm3keRpL+gZPsDImgbMEoeMmPElSUi3J05QQHGmVQa2om4Y8jQOfKh9zaHbkScZmtyLYgHUBlaakeYEJnqLMQEnmkwU3Nzcsr++im73tyDPB/eqWITRc3ezZrz9AGc1sVHF99ZqnTx8wGo1o2ja6f8exXPpkcYIZBrabDSpLyNI83tsCJFrTdj3VaHx01SsqnYAQsdh6cJgwYG1PZwa8cEynUwLi2G0x0Jvf//x2g+F0PmW72ZLoWPDWdVtm0wlpWWG85/Xraw67Dmc0WguEsHz0/lOapuXb58959eols/kp85ME1AStIEnitVAngfV6w8XlnMk0RwqDMBYpc9quZ1KOkELx2Wef84tf/z3yoPj4k/f59jff8+DBOTevt/zgsx/xt1/8DUUJSQpD50h17OTwDsq8wpqAkhnew3ha8OrVa0RISfMJy7sVJ6cVEFhMR/Sdp5ynBN/Rt4aLsyk60Tx/sWJ+pkkzjfWWqso5mBatNR7PaF6wWm3RveLp01PeUPMPf/cr3v9gznhqsXbAeRMHmF6yut+z2++4OLlgfVfTtQd264KHjwoGO1CUsNpdMz8Z0Q6WwTZUo4TJNBa1btdbbr8dsLct41NIQo9SHpmA9YFnH3zEg/cWXN99Q5JGx2s1LpAhsN3fIwPcXF1zcXZKoiSff/Ip//f/x/+TNM94/t2GH/7oApShqjTrleXl9zvef/Ih3327QcmUbbPm5GLEcveGskqpa8Pd/YHzywlpMVCOUr79zjEs7zlZWMbjMeVIo3RLlpb8t//N/5lvv/57VofvaPqeJDnh4vwR29UumgF0TpI6hq7DS4dUgXZjubttaXZXfPrRD3n86GO+f/4FxnaMR44/+dMf8PLFis3Gstv19EOg6QaqKkOIiL4cTM/yvia4jLvrNb3Zc3I2R+nAanNDVVZsa0e4scxmJfvDihAk+3qN8Zamzbh8cMl+t8X4LcZ1ETOWaw71jjdXO5I0PRZ9J5ycnCCEosgzTDewmC/Ybw1DL8hSwXRakqgxVT6G4Hh8+YTvvr5lc+g5fVJiB01WeOp6hbNZHDRtdqSZwnR/WIosssmjIeCtUB7Xfr91/r4dtls7/BZt4TiKfnEtq7Ui4JHB/dYp7P0RvRF5z8EL3JHzrFQUEK21v1MsH965c39XzH9bdhrCb416Qkqsc8d+lwzE73Koj8lKnRDkW5SgwoW4k1Aq9hIJqZDEPYPzDryJieigsN7FtpijoBsICC0R1oMSxxLIyBbXSpEkEYljrMO5iGgTQqFlgndQ120UvKOd5uhUfiuoRlRI1JrjYEGKuIfpe8NqtWWxGJOkKaNqRJIoJpPxEdcXu53eDiPeitgh/BZHE4RASXEU/X83ZQBKvXV2Q5bl0SlNdFmbIaZaQVBVk2NRvaLruohhLYvjGtgh5Nvngt/27Kj/v8Mtpm8lSieYvuf+fsW/+3f/nv/48//IZrNFJ3n8PY+C+G+d2P53jgv/rofI2rdDlpg2E2+d/TjkwfH3f/cFP/vZn/DJhz+lKk75h1/8Pc2wjO9RxzWlUhKUPB5DIiYdAiipj8PquEdxx89OJxr3trA2uKNhCYTQKJWRZdm7vYF8+114h3MiMtiF/E9QLhDd80JAksQ0g5Ty3ev9Y4//bOVNyD19sOwHRaIDX7e/4jQIEveUIQxkWUulJcEdcCZjnvyEnl9j1B3C51i5R8kD66bFMkIOOzZO4YuCLDyiY48Ilkk+4dCklLKiyApUU9DpJUO4p7cGwZZxckmuAz0bpIJcJGRyjDXN8WKYs/ctmZ4ibEtapNiQ0Ps9WboHsSJRkCRPObgBgkGqntBLtkNPEkb4sCbXOQNbLtOHvK5vubs95W4o+Kt/+I7v75bcvjBIWeAKjeLAoTZkaUZoNPlIsz4cSLMYJdguLakqMNRUWcH+2hC0Ja0Eo3HG3auGtakpC8l8lKJsgbM9D6bP+Cef/Tm8u4CJuBluDqSpRsiAaXryXFN3NcHDNM8JriU4QRt6EhLGWtLvW0JiEV4i0ZxO5/TNhmxyEoWS4JBSo6RDCkWaZFTFjEN7Hw+wQuGt4rBV7LcGIVP6rsPbDCGaOMULKUJ5fvqn51RTy6fvP2Uyu2Z2uiV4yELObjgwn0DTWYSeEJpbyHuEzul9g/TQtoLgM4IKmNBTiAlFUFgDRVpQD5LOfodOE4TweMbkqaRtb3FhTyo1fa8ps0uUlax2rxicpHY5nUzYbWu2O8vQa6yF5X3KetOxryVaZgytIi1TfOhY33mqIqVteoSK4o0TjjLNsCHw5PGUduii0D1IEllgmkCWZJQThzOOTGXUpossvlThehBG0B06xouUobfUfUsSBDINHNYSZyWmV2RJ5M+vbnp0LmjWPdZ5Eq0w/UCwCpn8YcWi33z5LWVe4AZPH1oePnwAPlAUGcu7WybTGdYFsixhcXbCdrViXE7xNjA9mXB9c8VuvSHXKZlOmM+n7HYretciC4VtPXXoGZ1OMVi2hx3zyRwpJW3XcbdakhcleZHS9x1ppnny/hPub7cc1i2nFxc03Y5nHzzl65e/wPaQ2Zxcl5y/94h5PqHeHHCDQVea++srpuOSztSYQ82f/ORPWS93KC84e/CIer+j3m8QCNarDX03oHXK06eP2TQrtFSczU9pDy279Y5U5gzG8dOf/ikiwL6+5WZ5jXEDSZLgfSBVmqwY8ePPf4hPVBTQZwsuzs755s0L3izvSZJAVY0pywzvBkzf4QZDe2iYlBmpUgTvUVLS1geSywseP3nI2cmcV69ecH5+yqMHD9is1+R5Rl5mnI/PWHwz44P33+PV1WsGM7Br9iRpitQKlSVM5hMSJen2NX1Tk6cpSkt29R6hFONpxfnJGdPRmDTNed6/pMhLmj7G7f7sz3/G6u6Ovu0YZyU2eLx1JGXC+ckj3NBz2O1Zrm45ny+YTEpwnuevv+fq5g0ox2w2YTYdc7+5J88ThHA8evSAN2+uub295fxMMJlMURpCHXj6+D3KJwmvrq44HBqGbkALxfr2jkREdvX7z57y+tUVN9t7zuYLPnz2jFcvv+Xm6g3390s+/PADzuYLEIFDvWO1vMJKRzHLudvcQOMYTEJWVXjvyHSCTjP6uqHeHpDzFDXPGLqe6rREJSmbsOPFmzd42/Pk/IIEmMym9Immqw/MZnMW4wnn0wWiM/z5D/6EdJxzd3fF9r4mSRTt4PjgvQ+4fPQer9/ccHV7x+XDC9579IRPP/iAX3/1az796HPu1yu64Miy339IlqUJaaoZj0qsbRl6ixABM/SErEAQnQNaKmwAYwzb7QozWCbjOdPpDGs9TdtT1zuyXL9bgIzHFdUoZ7vd0HY7hI3FQrPZiMEO3K+uGFyD0hbvTFwn6xEPHjzEmJ62O0RnseC4APQ4b7CmJ1GCqshwQ0vXtIg8ZTpZkCUZSgeSNMRIaH9AUhCSNEYR/RDFNgJllkenjpS0rSXNMtI0RYoYJ0wShRCKfhjQiSDPE3Qi8MYcE5ESbx3G+ncxTyQ07ZF1LjR5WZClmmHo6bsmurqVpes6svQtHi1EnIe3oBRpmpBmaWSwe8vusKEfekBRFBNsiFHato8RV60zPBE7MgwWZ6Prp21bDocdbRvFWOscfT9grSc7sqLTNIryfd/T9d2RVwtFljMuRxjjSXTCuKpIlUZISWcNWinyrCBPNAfdYl3k0Pemjzx1O+CcYbdb4yx4l+BMxOeIcGTk2/7o+OlASIwb6IwjiIQ8n5BnCW6IXPC+i6x1KRSjSsX37MA7CUrhXSx1VepYmmU93g/xdbzBWhETDEoSMMgjVkeK+McYy+FQgzBMp/OjayUlUVE06YbhXSGUOC60jYmmhbdDGmN78jw6n1KtmM3GjMbR5ToYh3U9zg1onZNozWBNJDjKuDmqquMgwEJdd7TNscz5D3jU9ZrxpCTLAQbOzi7IsxGCjOI042Q6o2sH1qs1q/s1tjPY3qFnKSZAexiYTMZYN/D61Uuqz95HJZLB9GilmE6nrG83DL6n6VvSIsfI/jigiN+HsR3BWabTE7TyFEURN74kaJ0TdHyPmagIfeD9h08ZlaN3rNaub5EuoKRgPD9hfnqJ7Xt263vmJ6dwTGIpkcVYdR5IpOTDj59yqGMRsEo8yJhiiHgmT5qWBCcpihFSZTgCTdfQGMPsZI53FqUz2uHA/X0Ty5VzjfGWQ1OTpJqLiwesVjvq5kCRaZqhoRjnJOMUtKIqSlSnaJuOIMFi2Ry29Ks7LnPNerPCe4tNBFmWosqcV6s7OiwbNxAyyb3rGclAn8oovGqJL1JGk4rOHSiKEYN26HHOzjRIAuvdCuMFGYJcKYQPZA4IhpdmxQ+mjzFZxhe3L9FOU/WKXW3Zdz1lktIcGp7fL3FFxLqF4KkPh7hZti5+HwjMMJBlGW3bM3iHFY6h2zO4DryLJZGmx2ARShIQZFmCc3H92LYdQUXHmM5ThAYroByV9HUXxfb5iEQmCF1iBwtKxjRJZ3EepEpojSEYsC0EJZmMp6z1CuUCQzMwP53zZnkNqUEMgm+/esEnP/yUF3e3tIeaN1dXfPzxx/Rm4NWrVxRVRZqkzGYzfv3LL+i7nlwrpA844xh6gzOOvjNkeXRpSiVjiWqWUpY5UiR0dc/+sKUfatAKkSTc362QMrKAZ4vfn4kuvaKrO9pDR5mnWONZb/akeUY5LQkOtNKURUHtevKspCxyEPH+6AP85qtf8clnH/HehxnJG0nXBJTIMLZ7h3O4u13x4OEpfd9RlDI6q3E4P5ClCffLJZPxjPV+zVdff0FZ5PSHLQ8fj3j++ksGu6OoBA8elEiTkB73JrPZlJcvNnSN4L0nn/H195aL85z5YsJyteHZsxMmSUXXmuP9N+PF8yWbpeXjjx6gfEt92HNyOiORmmZr8GGgGGl8CMznE5p9TEs5GTtANpsVu9BydnZOmkA/HJhOBdtdE4fJxjH0DmddLNqsN/z0T3/Cb379XUSaWUOiFONJyXrTMbgOnbYEJKvNjrxMycuE3rbc7jpINbY3JNqSVQpdpaRjyRdfvuLk0cBkUeFMi5bQ9R1niynODww9PHnyAcubN2y2a/IyZb4Ys15v+MHnH2Fbg6GmOwQSJfnk0wWjouXyomDffUUvWpStmCwUWi9YLg9Myjn3y3vmp4Kqyjk7fUiRp+z2GxYnY5Iko+sFaRr47/+H/57/3V/8mFd/9/c4LINJ8aYlSXMOy4a6NVw8HJHlOVI71pst3mU8e/YY4TV//w+/5NGDOZcPJjg8680KLSRllXPxYM6r1yteXbVM57Babeh6y+MnI3rbc3Y5R1Fxe3WNGWrKIuHsbIzUOf3Q07cDWeFRyqGk5+p6zWi0IMlSuqHnbrlGp4bRsGY0zjFpxma/RMpoYDO2J00zlIqFzIKIcBuNJuSpIU978ixHKMPQt9xeb1nJlsvzGX/z83+gPcChGTh5+JQ8WXDz5iWz2ZjJaMRqtUagMDtPe/uHpciSJCOmMsU7Lvlbh7U4Yk5iinA49tCoozFCHx3Y8ndcwgolo+DtXPy93vLVo0M64jfe/r+RgW7jzwb/n5Zhht+K4VJpQojFtrEoXZGkaRRaEQilY2eFt5hY6EOqFDpJESIFJEpqvHF4outeKIFCgNP4YHEhMtyDdUeTQ2RWh7dpShXT/Bz57j4kx56QKMC7YGKXmfdI4Y/YlAwpI/3hsG+wxhOOn8NbDA4cXckiJqwiY/3te4978fpQ03Ydo1HJfDGPBrc8o6oqvI+4ut/FuLz9Ht/iXGLy9revF4cK0bn+1gWtpIq9PzqLOMQkj0nGt4x1BF3XH93TAllJtK4IweKO3VZpyFFaHvctv+tEj484IHg7HImpzPV6w93d8t2xZa2jNx5CvPf7EEtT35bMvk0daK1xx2LOt8efUsnRWOHxwaF6wb/+1/8LUgr+5X/7L/jkk8/46IMf8utv/gPG9SSJAu+Og1+FFDGZYV2MRCof0DpiVZxzUdSWoI/vsWk7rDVIJSHAMNjfQbYEtI6pi8EMOGux/UDAkqbZ0UDj3n1fUorY53Lcrv0umucfe/xn78wflE/pVYtIW/rBontPEp4hQsJEVByGn3MzKFSAKtWsmnvGVUEwlk2zYZo6EqHwIce5LeXoM6p6wjgbszvcM8tm8aYdNKgVB9+S+IKQFZz6Z2zDtyTyfXo3UOYjuuGWjIr7Q0uWNVitUcmEQo0p5RNU+BonOgafoUNKpkc4b5iqT0nlhNZ/TWO2SOU5yWFVZ1gKUl1xtdvT7kuwOw7mDd3+Bc+vBV9/k9DX39I3ntaCHUrSEtJgGYwBp+hax7hICakndxm29/FiESTjuaRts8jeSTQuCNptoNUtYZRQ9gXay1huNVjKpOD/+i//b4yLGcE3x9MplkLcX7/Cmz35Ysq3r77GOYPv44myWtU4K0gKQaY0iUvQOhbCtCtwGCaLhE1/R5JW7+I+cXIT2Z+JShmXM6TQNPUW4aFuBrK8YHnbURQjlITpOOXBszFVdskPPn7Ei9vvAMG/+JcAC/K0w9sLcFtEsLHkQQ4kdkGiDYlISPQZ9bAiVR298wifUxUTTJfRNS1FmVC7FJ0GerOjNZ6F/IhpZlh2r3HBUXcDvmppOkuwY+phhVATDtstQxNoO8O2Htg3lt5kfPOlwQvY3bQxkn2RkiYa7S2TuWI1QNNb6AuywoKJNy7nB6pxgpYJWS45rFKGzlCUCQffob1AKEE1ShkvCm5WBkScpPYbT5WUdNuWcVGhMk2/9+zuOoIPjGYJ0kg29x15LpFasr0yXDwa0ffxZzJdoGYK1yoOpifJoO86kv3vj3sA0MZyffU942rE+fljPv7wA/7ui59zu7qmyDPoFednlyiVsdns2G12TMoRLhjW2xXXb25w1qKzkqIskInE6kBw0VGZZjmPz5+wW20BjR9imentzS27XVywzyfFkUF6gxMHvHM0B0OqSw71gWFoQQuapiMJCSfVgsEHDtsWvzMkSKTwLOYT3JAzm1Y8v/qWTz/8kNPFjBffPKfKJtzeXCOF5P33PuDZs2ekacrNzQ37/Z6XL75n16+4uDijaVMmoxm7bcMPPv2cX/zq1zx58pBXVy+QmcB5aNqepm24vBgxH59Qhjl1W3O1vGNU5Dw8P6cxPWmiuby44PrmJZ/+5BO2m3sEjixRPHv8mMuzxyxvrrmcn2OMx3lPqgTfff0lDx9e0jU77t68pixLnj1+AMKzb7as9ytUqlBCsL5fMS4qOtORIPns40847Pbs6rjpEt4zr8ZkOmEym3LoW/7yb/6aLM+ZjEYMnebRh5+SFTnXb66x3qOl5PGjhyxOZqyWNzx+8ohEpQy9oT205KOc5y+ec75Y4Jzj/Wfv8fDsFNk77u62TOdz/vbXO/78z39G17V89c2X74Z/WZay2+1Jk5w0KQDJfD4neE8iJKeLE3Z3K7I8ZX/YMp9PWXctu+2e8XzEwx8/4vXrV7x+c03f9Hz+0Seczsb8/K/fUOQFfdvSdz2CgLE9fVvT9Q3JKGc2n7P95lt++t4PSBvBrJqwWkxZb1Ys1yvSLrrA8hDo6pbFZAymh+B58ugBb26v6axhdziA9Zydzli6FmsNRZlSFAVd0/Ls7AHSBb784hfU7cDp/AEIhegkq+0Wp29Z7/akecrJfI41A4M1eOvJixF5NzB4Rdv/IeWDAWuGKGa6yDYsioJER+dv27Y46ymLglRA2zWE4LGuo+22IIfjwht8cAxDTNNcXCyYzkYMw4H1egAiM7eu95ycVpRpxmZ3xeD2aBXABawFLT2Hwx5rLc5HNuF0OqMqR2RHUd+7gSSRjEclbVNT1wN5GjmMWukokkobkQ19RwgD1iuMTfG+jw3vKi64nHO0dYcPkjRNENLTNHuado/3JnJ+ZXjnPJcCZCKi4C3scdELwxDLmHSS0PWWYYAk1aRpyWhU0ncdzcERnIcQC3yMiRiPMs1IEg3CR5eGjLiV/WFHaiX7ekPbdWRpSZYXWBvouyh8BwRap3gsfR+HHE3TIcWOJOlo2hofPFme4to24meO7zVNE5SOi9zD4UAgoBNNXuRgLVlaEHyPkpIszd8tkCEunpUSKJ0hVULddLT9/sgPNO8GA21XYy3gMgQZmU7IUkmiJdbCYDoG65BKYVzA+8idTPRxo2ANnW2PrpDfYneSJKcsJzR1FD3apqdpOtI8jRsxrUBEF0kw/shrjxuFWIjk4uftHV0XnUjDYCLbv6iYz+d4L2kbw2AMSiqcDFgbi2jfstGzLAfhECJAiGVUo9GYPEvIc40Qnq5v6fuWtjswmC4uyBOJjanYo3MnpSxGaJ1TH3qGLkDQaJX/QfduaxuaZiBNI2+zaw/YXjCqcvZti04SqjRlNpvy6NFD7u/uuX59RXfokJlGBYEWmiAc69Wa5fIe5z1t09ONeibTKc0uOtNTpymrkn1vGQZLolPqQ4uSmmEwjCcTuuZAoku6dsD1AYWi2w2oIFlulpwsTsh0ShpSdJZQVWUcfrlwdBjB4VBz9eY1y5tr5uMRpm+ZpAXIlEO/h1yiRhl1v+Hu/oaAQwlFUiQM5lg+JTTNoeXhwwuqKpZ932+2bLZbemvIioyyKFlvtlxdr9iuW+bzKfv9irptSfJYerXb7inLMU3T0w0DuTHoTNP3Dc5bKlUymU44OzunGJU46THeEhS44Bi8YX/YsdruqKocJTT5eExQOadZTrvbEKqcte1o+4bbqxu8h1W9xw+Gpl+z6e45e3iBUJL79QqfghMCOZ8hVEKXp3S+ZTIekdiEq9Dw1eYKV41Y1gc+nzzGXTc0jeG+bemRqLJAVxW9j4YM38d0jEijw2xUleR5Tm1i2uPtpjJNU4xzLE6mqOOQUchYGCaUJNVvN7IS4SV28O+GaqUeYTqHt4IyrUiCpsxKHlxOWa3WBBVj4mVVghOEoKmblv12T6FT8llK2xuEatndbxiXY3bLNV09MATP6aM5r+013jmurm/5i3/2z/n+1RtOFie8Xr7m9PyMyXTKdD7n+uaG07Mz6qZhv9uDjuxa0xsO7hDvxV2PNREPVI1HiCA4mB1KS3SqY7lwkZNkC3yYUBTZETO4RgPOxgK73/dxeXbJ61evSVVB28UOh2dPPzmmP2ocjpPTBZtlTZlXnJ6ccDisePH8JSenUx4+uODFy5fcLa9Ic0jzATNIUlWx3x2wVnJyUrHZrbh6s+PpsxM2uw3jiaY5eIxp2e8tQkomixlN33PYbzlsD0zLCdv9LaPRCOc6uk5T5AUfffwJv/jlF+jEkybRdHe7a2ibATdojPH86Z9d8KtfPkdKizHQtVCKQJZqfvT5B5gu5/rljocPLxhkYL/peP/JY755/i04iekNeVkwKWds7q8p8jFdD6bzXF6eoUTJZr1HJh4TOowTx9QdWANNE9NI+/0BKTzL+9dUVUKVT3B+z7Y+IARMpgVt3x5Z+JK80PhgQHiSTJONRnS1ww2BIAYGPzCfSWYTxfzzKS9Wr9CVIlUlRVVy2A7c3K7xgO8dXSv5ox99wHpzw9fPf0Nv99xvD5xezBm6mvPLGWcPPub599cEp5lUI5q6xsk92QiM21JNU0wfBwJmgHE5pq03zKoxk6rjo4+fsd3d0feW775+wdVVzY9//Izt5poXL7+iyE55+eaWk3OLUD2TccH1jUAlAutiQq8YJ1jvCaHn0aMc12eMx8/4/NNnvH79Dfe3hqL0pOXA7m7LqE14+HjG9OQJL998w+7QUaQVkJFm8ihqdQTrCSbgeqD05KOMznTIRKOThNub+9hzZROcP3B+dsJ3373k4nJGkg003ZpMnUcRT4o4CE8UEsEwDFxfX3N5eUFZxETldrUhTVNO5idorbm5fYmUisUiI81yROLIJjCaF3SvejbbW0ajjOASnJUsl/fR6ToETB2xYn/II89LQnBHh2/su3sn4B2Fda0V3kuMke9cynlWRIa6F0d8i4t9Qf63qCml5BFVwjtMX0SW8DsC4fF134n3HBN1bwvpBUKqiJDxb3EvKUmS4qw7urc1Ijii81kfRVWNlAlKBmJJqjwaNUVklXM0WyiFDxoQSBHF+sGaoxYV9ai4KD9iRDhiPYj3mOh890e0SXSEK83RgeyxzsZiey+RQtMPFnvEqcT3+Lbg0h9xMG8563E9GTyYYwllkiTM5zPk8fONf69JEo6JVI+U8b2E8Fu3e0wuHnkPvyOeC+Q7QT1NM5IkQ6sMrXK0Lo4MdHAurrX6fojdUUlEAvZ9B0ed/e0x81sdL0Jt3jLC4+Pt7xQHBEVV8eDBA85Oz2Pqsu3BB7TSx16f332+eA97mxgNIWBsTPPHtIM8DmF+y38fBodsPV/95jlZ+tdICi4uTxlXY/b1gUQpvFLEdXpAJQkei8cf3w8E61AyDpTw/pialiA5mnMsEvXOsd51DUJAnmdIJY7rlpji6PuOvhdkWR6TEe7tYCGeY1opzFFYB5D8FxbRa3MPak8iczJ9zqPxx3T+Bb2/ohMZ1mjSbExt1pihYZSs6ZsWmfWUI8XQL+idoZIL8uyOxr/Bk3NzeE2RKtpB4vya2WxMokCrEcvmG7R6BV6QphUhBFKXI0VPKhW5cDw4mWDNKVqUBG4J2rI2ghAshcxxWtMykHSekZ6iZYkLWwKPCOYeKQKdX5LKp3z74orb9YZffHVHmoy5+tZBWaNTw9V3KUVaMZpqNtsWEwJYmE1n3L/eQOGpJppgJcUClBnh9YZC5gjT0veerhlIEs/QxZj+aJIT+orHjz7jwwc/xGwP3F/f4UTPdrvjX/3v/y/8+MOfYV2HPE6GnIsczmIyY7c80BqPE7C+HvA+oCswRjBKcspJvBBjwTrL/KRiNI6sdaU1T84+oyzG8YD1xziLkLgQ4o1m8ZAsKzg/nRHejHl2ntMPlo9/+oTFiSBTe2RS8PkfXbI9HBjNrvn0jx9hzIZMbah0yq4bo0sDoSCXLd4NzGUFdoSVFue2SLGnUBrja4zJyJTCugGdSHzQNFtJUEUsMVOSEG65PnyBJmD0QNtP2G87mn5PEiqGOmG5TdmtB7zp2K8KWlNzuzYM1iOco20FUmlMB5PLBBkk17/xZGPF/RvBvhkQQaAzS6Ykt29WXD4rMU7TNBbHAEjSMieIgDE9fRd4+KhieVfTi5bQRiElVQVmMJyfj3DeMhqNWCxG3NztCXkgMxlSQ1WmyFzT9g6dacq0Ik07uq7GGkiFxA+WUTFi2WywjUf2nrTMcPxhE3HqPZ88fsijh48ZTcbcXr3BDkO8QPqUs+oBYtDcru44PTnh/fc/5LBd03WOujlwf3/H2eSc07MzVrt7tt/vIA84ArkuyMeR8VtmUx5fPsQNPdJJdnXNbrtlupiitOJ+tWRzWDKeq1hGSM7HH1zSN/Em/fz5S/rW8ejxBzxbfEJQsa069Y4yTXh9f41zPefnlzTtgZPTczbNir/+xX9kuV8xCZZH5894Nn3GYjFnu9uRaM3Vm9cs5guUBJ3CxeUpi/GC7796jhYF129eMx2Pef7iK/ZmzagcoVVOmlbkWcODByeY3lA3PS6BJ+894nJ+ymG5YjKb8rM//Qlfv/o1AUPfd9zcXKGU5eOPPiRPx7z+/pYPHr3PeDLjm5cvMXiqxzn39zfI0ON6z8OzU+YnC/b7LbvDltVuxZfff0VS5pyfndF2LbvdluTYUH42mXG/WnHY75mdzsnDCc/OH/Lq+QvqtmE+n3KyWDCdTphVIw42sLq+4/MffM4HT57y6vqKVgTOz8/41Re/4Pz0hJPTObkuqPIJr9a33HRLRtWIru3om45RXpDJlFdvvoM8YV0vOX1wgkgEr79/jZAa5wW319ckacZ0ekqRZWzWNfIoxAgZaOqaq9evyXUCIrBc3mGtYT6dsHzzGjWbUuYVu32NlIKzswVSWLarO4S3zCZjmkON7Q2NMVjXYWwbF0WD5/JiQXphGW43FHqMG3me/egT3JdfMZaa+nrN8vUNZVyhkEpN6HoCFmsL0jI7CiUJk3HF+n5NmxiKXKO15OrNK+bllO/rnvcfPaEqMwKwWi9J84zZ2Qkm01il0GWBGXZIpZgUE/7DX/4liU558eIlKssY51Nc+wdEwp2NLuKhQyeKssopilgw2bZ1jGSmBaPRiCSVuOVtdKJ5i/c9LjRU1QidpEdXs+Tk5JJHjx4TGHj9ZkXbtaSZoqpy6r1Ha0VRJezbgDNNXFhZibMarQx3d8voElYB6xrSNBanWheOpX5r0hSUjqWZcSMgY7GkcyAcXV/TDXV0MkpL2zUYe+R2J/LoMo/JrWEwZHlFlqmjGN0QgkUnkhAkpcwZug7nHNYZtJToRJKmEdkhpDxGan3kWAaNSgp0qpAqB5FE9qOIm2xnY9mT9I4kiQiJNFVxLyDifdl7y3q7ITca63vs0Rk4mB5jHb2JIrq3HidFVPdD3IAOg6ERDUrFiLqS4hgj9SRJQpYWTCYT0rQgIBmGnsPhgE40STqOzpdEk6ZxFa518i7OK4QgS1MCbzcmkOcp/RA3W+4YH9UibobA4+yAQKOO701JhZcKgcfYDustnoAQKXlaUlUjJpMpQiga10Pz1pWjcS6iOJRMKcuKohiwpsFZjzGOJD1yEgUoVBSotYjFqzLiPIQQx2Igi3cDzrV4FzfWWh+56zoDNPU+blJCOCJYhIis8m4gTXU87uxA17VYN1BVU2bT6RHv0tE0e8xQ0/cdTbuPyJvg37lbMlOQZRlZVqJ1jiBh6Bu8l2TZiCz9wzbiSSJjqRKONE25Xy457JfMxjH+671F4BmVOR998AEPHj5gPJnw5vUbCJ40SXHHjaLWKZv1HpUJGjOw3G2YZIG6b5mXUyblhMN2Rx8i913rnPVqj3easpiDz+iaPaHXiAGE96AkZm9QKqFUKW5v6L1F+Z7MQRc6xqMJZ6fntE3L3373Fdluzcvn34NzpMEjBoMcDDIBLaDzHVkieX31HX07MC4nTEcnNLZjt41FvPWhZ7G4YD6f4b2gH3rq/Z7JZMLgBoJ3ZFlG0w6sdzVKKqazMTqxCB0QOjJGN5sDJ2cXrNY7mranq2+ZzccYZ+jtAMajSo0xG5IsYXE6x3iLDY7T81M++/Sz2C1QH2jqA6vVmru7JSdnl0xmc/SDC0bTMc45Vs2Bu+2GUpS0Tc+wOzAa59i0xBx65qM5lc4YpZJBKpZdT29rXC841Guens0pnMKmkms1YO7vOC2mGBu43e252x9Yti0+L6DpkWmG6xqUtzgzAIGhPx73SYJxljQr2O1rqnKElA6pPUiN9IJEKOzxmhwQ6DSBYOi6nq4bsMYhlcQNBikFygvul1skgYuzExKRMZ3OmS7mLDcbnINEp4TB4waLMxYtBdYZ+jYghaDpOi7Oznjx9XNm5RQhEtabGr3KWDycMz6tqE1N33lW2y06VSQ64eTijHroWb95zXg6QRcZl08f8cu/+0V0cCqNHQxBgneKRCm0VORpTt/1saMojQXdaZpi+4HVektRTZnNJlSjEVVZcX19x9n8DOcC2/Warmt/73P7/mZN3xoSlaHynMFZrq6WKFVw/kBjQ8fQR6TAejWwXu3IC5iOCr768oYPP/6EcdVx9eaex8/GeG8YBkk1LhBMub/bUuSBLCk5HFpevljz9IM528MtRaHY1A1D40HEtF6S5HSdwfQNq27HZJRhTI8ZAjdvWj76YMaL5y/YbbfM5hltW/Po0SXfff0d11dLZtNTdtuGswcZ730wJriC1bIDrzF9w2iW8c13dzSbOVcvW25fv+Gf/fMPuF9fMS4WfPSe4mbzPKaQXBx4Kplyf7NlVKWkMmc2mXO33ND2LbPFlP1hjxDZEV8mqeuYenLOohNQUvHi1XdczB9y9XrJf/1f/4xf/qZmuVljg0BohXWevCiOQpajH2I/yTTP+Ojkgu31NXf7O3on2W73nGYFdhiQSMq0QOMJA4ggGYxBpIq8HLE77Lm6veX0vELIjP2wR7SO1faWBycXdO2O529eYwZNpmdY07JcXjN/UJElCdko4/bmllQVHJotUmS4rkLKjGn5gFI7/BDIEsmL756Dh4cPF2w3A3k6Z7tp0Jnn9LSibQdOTqZo7fjxjz9kva25Xa5obYfMJEJLiiSjGkl6VVOOU7765m8Yj8Z0fc/5xQnGHliclRjXMBioxhGlOBnnCDehKhes9i8YuoGL2TOydMLJ6JTdbkc1rXj+/SsmizFd72hqz27d4QbJ6cVlLEOva7K8YL3ZI3THxYOU5d09zihQNqafRBT7zGBomoH7+3vUhSQ4ifUGbzyvX91QFBlKGhJdse9bbu93nJynIAec7Pjsj2fsNnfs9orpZELT7ElSj9bw3tOnVOocLf8wjKpWsQDTh7fYDPvO2QvhXflnXKdFjrZWsXxRyeSI9o0PKRXSxzWPOGo54ViOqFSGVgIvAu9K6KV458L2b9nWv8OAFm8RLS6Kl947EOoojqeg4t85GwAVu3vSI8eaiJzBv+VOvxXoIfho0ji+CvKIskmSFOc81kVR1R2Z3EIEAm/fS3z2LMmRx31uCA7nbUw6aY1UIrKycThn2e87QpDHAZo94ik9UoljWWV8zrfvXOmIinn7M2//SwixN2i92bDfbrDGAbEM1h754G9Z52/f81t3Ou++07cmEXEciKTH4yBFioTgFUIkEBR977AmIkvepgiiWN+z3+9RKt7X3jrefXj7HR056O++yGj+CLw1mSgIEXeSFznj8ZjxeMyhaRmMx9hAXTe/49QX7/787gAmrpUHnHPkeUR1Oe8Jxzr64OMAZb0+cL/csrrfM5nMKbOSw97jrY/O+BDZ5UJLggygJEFE1GRw7ohfiYOi+DkHHB5EdLG/3VMhBCIQkddeYe1AXUdiRwgW5wesiy55gYagjqiiY99APNixNvLz35oB/tFz+B/9ieOjUQ1uEKS2ZWe+ZbApqd5TJQVSWaqkZGCH8E2MeGQ9gQQfxihVUZUeGQxt25BkJdLEqI5OHMZntMZTVD3bYU2uS/BQphXrYctEPMDanCov8a7HWYfMG5yEod/Sdik6iV94Lj3eH1DignrYIbREytfkyYIQAq2/xXYfkIkPwBl2W8t391+z3Db84h8M1286kmnCcBgQosC1IINCOUPoA0PjWJzk6CSwawaEbskz8CTYOmCtZyt76MAOgZNzjZqOuXmz5/6mYXGWYjvJeCrpm8BifM5P3v+X/MmzH3OaZ8dm2OSdaCCCRYbYsOxcj8MTlKKazgkaXty/xgQDUrA4yeJFdOQYq4LRQrI/dHTUFFWCzgzWay4uUmRb8tHlx3GiNJh3k5yYaBHkaUWexij4f/d/fMjDv3hClgqMHRiNJIKWUTWj68eMMkc2uiNXkm64w3pJks64PRxAasqhxto9yWgg8Wf0wYN+jdYfo0NGpgrMYNmENZl+xGA3rJp7TosFgwmkaclyu2RcKoINeDnQ2oLBK3atAm8RwdItT7h62bDbtWTllHp/4O77gWYXCDKFpMAzkGTxRit8SpJYcAn7jWN2OoruUKkYzXMmk4yb6z0+KNIiINEIF0hE4P7WkxcCpRtEUCQjR5Gm3F3toziEpu8Mtg1Y13P2pOSw9vRNYDRJOewtpnekhcdpQZGVmNYSvKMcR8YfyiNxeOnpD/FikqoEy4AfJJMnEqUC/TYlzf+wm/mn7z2jaWLhYZLETYuxhjTN+PjJJ2ibMljHqKjQWlONK3abJUILlvd3jCYVz5495fzkAissL9cvwQTKfMTJ5JTdJgqa88mC2XhGlRUQAre3Sx49eEyQnq5umE4n7NsVATjsa+ajnFQnnD+65PXVFe89+YhffXng/OQhjx4949XVa8qq5GRU0td7vB1wg+NmdYtOExpj2PcDzdBTnEzJyhHrw5rp2YS6rplNp/z85z9HyhjT/fyzz3iz+ZbDYYPy8UakE42Il12c6GnaNUPX4p3kweVDQghstku0lGy2Oz79wQ+pd3vevHnF7nbJjxZ/xKFtWN7esVic0LQNWZbS1A2L2Zxvv3qJN0e8T90xG48QqeL67jWSgVTnTEczLqYnnJ6fcb28Y9/UjKdTTk7PEEf7bFmWPLi8ZLNZcXd3y1C3PDq/pDeG2WyGMg6ZCCazMU549ocdP/3pH3N/v6TvOn74+ed4Y+n6nj//J/+E7Fd/z/OXLyjLjPc/fA9vDLPZlPXdlmAlWZLw4eX7bG/uWN0u+fEPfsxsNOXuzQ25TmjCQD9YxtMKFxxnl5dsNluctfgjGiPPCpKk5Pz8Ae89e0KaSr76zZcE58hnKV3bYoNjPKm4PD8n94Lzs3NmswWHfc352QXexbhrvd/y5uX3pIliOh4jHymWt3coLXG+ZzwpOT09I2hFfbflLJ/S369IlODl61dcfvwB1cmELC+ZppHB+/DpE17ulwTjuby4QJUpv3zxLXfNjtF4wq4+UI7nbA87atUwm81p9zt0UJydnDAvJ8hE0g09Xd9SVAtu1/d02xuMEGTVMWYpBHXX8/z5C4IQWB9o2o4qSaAduJie/t7ndtd3qKMIPBqNmE7Hx8XlHkTk2WZZilRRTE3TnLo50PUtPgwkmcC5iLooy4q2MXgn6NoOY+ujcz26eyeTCUpYkkSgZIBg0EnADQbvNEJohICiyHHOUTdb9oc1aRodHKNKkB75ht55pIjroyxNjyzkPVpr8kKyb1v6oUOVcZFohpbgojBc5Cl5lkLwJFpT5DlJGss5I0PQUVYpSoWI6YomaAKxb0QnGqU9yMiKd/5tsZPAWlBJRprkZLkmz2ORqDE2cv28x4a4qBdSoHR0uAeOccREIoMgTRM2uwGUQScQcHRDi6ijKDn0gratwYJzoLTB+bi5GAZDCF3cUKUydgO4WKaaZhl5NiLLsuMC32Gsfce5jOWnRNf5cUAA0an+NuqLiCx9ITJ8sAwmRlzd0S0VW+zVkZMZF9WxBDYl0ymCo3vluJYXIqCVIstKimLMeDJjNJ5ibXTjyeO6R2sVxRHj8Co6i9IsQycG6zxd16ESieBtoZE/MtQnZLnG2pa6sYACHwcOQiRMpxOCV6zXa8DH2OxRRM8yS256tI9urYi0i7iituvoug5jOuqmJi8yppMpWkus7dkfNhwOa5yNCQBjonAtZDgOYUq8MHFjFSR9bxFpFJ9nU33cBP7j5UX/W48iyxDCEYJh6Hq6pqXdC2yzoSjH6FRQN3tevn5NVqY8vHxANs6oZhXKORBR2EmKjNl0Tts1LCZzRNKxbvZsVwfWNyvqw458pBjNM0ShcCaQpSWb1R6tKk4XZ9R7OOygCCnSKrwzUbROG0qteHR+zu3tmsOyZr3cMT2ZsDiZc3/9Eh5ptNQE07FdbdHS0XcdfS0ZqxzXe8wwQGYIwWBsQz/UpGnBuBohfCwbNAOMqhmJtkynUw6HQ9ykISmqMnbQdA1d13O/vKcazWi675iVmrbbU40zyskJd/drlssarSzVxJGkFattwyjLkEEjzECV5iQcEUh9zW6/4/zijKHv+O7bb9guV6hcszvsSYTk/fcec/7BxzS9oekd3377JX/x539GXmR0TY3OMlSSIqzg4vQcryuaZk1BSrNpaXtNXpxA3WP7luVyTXW6iGJ61zJPnuD7hu8Oa15PaoqDIdxb3uzfsGksJk1xiWI1tAijqLHYYEmSjOA9WZ5H9IyAtjmAgdkiYzGfs11tyVI5gFI5AAEAAElEQVSBMx4fLApJ01tMb2KBr9KoRFGNE4SQdG2PtVHYKKuM0ahgaAbqZUOVpzSqZTwfk2cVQSiK0RjlEsg8zToOaqZVifeB7W4HwZNpjZ6MyJK3ydCATnNc37HZNeSLnMnZlM3NgaHv+NU33/DRDx9z/3LLdHLCerNmu9shMs3D954gUk1QEYWQJgmtHQgErLERbzY4Tudn7O02CvttH4U4JamKjKTSyDTD9gOyKmnrhs39Gq1zynyEHAcaefi9z+31dkuSJCSJ5sOPPuLN6yvul2vq3YZ2UjE7K9naA+NpgrWwWe1Is4z79cC4mvLq5Yr5YkpnLWYQZGnFyVnGfDzigw8ecXNzx2p1FzsjTGC96hFiw+XDkt63VCVooTjsHbvNhtn8BKs8QUh6U7M5OCYTgUoyzk4zVqsNi3mOTAPr1YCWA+/9aITWCddXNzx4dMputePp0wcoAft6y3bjEaQxAaZhNJF88Owx//SfnvH8+9e8udqyOK342y/+HuMcKEc11iiV4BLBe+8/Zn2/RiU552eX3N/VSJERqGMKsxgd0TUJ46qAEEuqhVCkiWW32zIuc5arG0bFhP/vv//35JVGSE1bR4zqfg2L6ZjB76Pr3Dgm5Zi9A5VqajvQ2EDbGNSuJl0EZOIwJkBIMLZDS001zpicpFHM7AWzccry/hXLdeBkfsZHH3zOav3vuHrzhs3NwNn5BD8ovLUUY8M//5d/zF/99T0G2G87NtbgfcquqTk7GyNEzicff8hf/fu/4vWr1/zZn/0pq80rcAHTx8LxB48Tvvl6w7gYcdusePgsISiLC5Km6Tns10i14/Hj96Kx7b5muz5QjQpGoxykwVJze/ua2WhOQDAbTVmvDY+eLRDkpFlJWWZstreYzpOnCT/6wafU7ZJxOmbftRRJQZq2CJXgfMp0es6muWFoDcMhYXnQvPi+Ji8sKt8wnZxxe3fDZJpjTKCtLXWRsd3t0SqjGqcM7QAqisZ1bUiTjL7bs95ERnQUIqGaZLSHnvliynbbUI5yDu2OYXCkpURowfawIs0Ck8mYeu/oupgOPdRbbtevOJnk7O3vPyAD8MG8E9CFCMe1V0Rb+KMjPARPmiaMR5O4xkS8MzvgPc5aklSjdII0gWGwQHQGv11bKCkISh5TegPWvEXDiMiyluIoSot3bHDgnbvZeY9z4Yh/+l2XtWAY/PG5FEr5d+5lZz3+KOgqGWKxKAKlVRS13W9F+yAkUkXDlDqu+7z30b8dOArdAcSRlJCkcGRdCylIpf6tmOw9/liSqkXgcIgIS2uP63Ekxroo0IejE5roaBchIEWC51joqiBJFWWZMZpU5GXK4XBAakk5yukbjzURQai0Roqjwx2JOG4qgocgeefG51hCGhBHwwUoFUX7EBxKGpS0pGmKkALTWQ6HA2maMh5X7Pc7bm9vyPOMsqiwxhxF4KPAfUzOxkFFHMb8FuvyNn0QizsXp+f8n/7VvyIIwb/5n/5nvvr6u7hfVBJvHJLjsXEEHWitosN7iEjHt7iat+hDEezxvUVDjjGG4GG5XPHll1/z5OljUp0e8SwBoeJ+y7uYEEBIkG/7GaPIjgCPx3qHwyMIDKZHavGuBDc+hwMh4nDkOCh6y+kXUhyTCzFh8ZbvL2VyHBBEDJ2QkY9vnY3v4r+kE33Xr0iANBHM0oANW9qhxAZB33lm6ZpEBgyaMpEQElDRVSDsFi0HOueQSQLDgkd5yb3+BbkSaFGxVK+RIiWVCb1v0RTI9IZKJsxSjbU5PsQiFGs0q9YwTXL88JhEdXThBcE3pP70+EFe40THvpfMy4ybwz0DA9++ktB8wSJv2DY9X918xRdfrkh1QRBjWpfQrAOhT3CiwfWBvLCkI4/rJKZRIHqMAjEopk8STDuwurUgAqcPC5y1dJ1lsIHttmNU5IzyjNMHCW1r0TkMfUeZLfijD/+CD8/eJ8fQtQfyJDZoK60JxuK8QOiE4AfwPRDjk956tt2ef/23/yO39S2nJznnTxR+8AhykiCZniom6QnLxpLn8PTBKdu956P33+fTB894cPII08dCMmQsIrMiRFdiPqJMc6QQ6Mzz/tMdNni2vUJgUG6HoWY6HujthlE2UCYVeQrenxNCTT4SWCq6rkeIgk2vENyQ+BHj3HF/+HfM8jN2tWKkZvR2h7MvsMOCPPmcdbuKPDDj2a4G6lbEskMtYpGdsNwuLbf3NSNZIfHc3Xrulo6yUNy89KhCk8gp3lvM0GBagTqB0/Mx9UaQiogUGE9G2FawOC3wGbEACc9snDF4qKqC7bphfq6ZzSZ07RZCFFuClThjSTNQMkGXCkTCrj1QFSX9YOkaQ7sJWAyHg8N5jxY5Q13jnKAsIwtYIgiDRmWKZjggpaTZxRtFvMwKyhFcPCnx0tNsPHiOPN3f/9EO8OjxU3SiudnccbNfkhQFi7Tk/GRKXe958/wFOku5ub1lcPcY13G/W/F8+YJPn31Kkim0gqcPH1CbDTeHO2anj2n2B2aTEm8tu/qOb1/WPLx8ymJ6ghkCMkjKMkOmcHV3g/BguoRRccHQB0ajGU3bk4iUjz/6IfVdy9V3t9xe/b/4/Ief8+23X3O/y9BBc3JxydnpOb+8+hW9b1ittiSi4CyfI5VnsZhxc3XN/W7FdDyFtkWpBK0kqU5Y3S5p6hqdpuzurqjKGY+ePuPrr77l9OyMXgnM2uK7htPJAhlqPFuatmZSnHA6u6AQJcvdHZPRnCKr6Cz4znMyPsFLRzWu6OqOjz74IWVywknZQSroekeaDowWKb0/0Ax3yMTQdzUXTz6k3/f0bcuoqpi2E04m53z4wSf8+Cc/4X/4N/9v9vs9g/DINOXDDz+KjNQk5bvb17jFhOX2Hl3CoHoOPpaQLk5m3K1vccqhy4hQePPtN2hybBL46POncbDtNcloTm9gfdhTFoZqUqBcxx/98HOG9y1DN7B7s+F8fsqb189xyrBaLsmLiiIpKRdV5MkftpzM5rgulhQaZ3DBMClLRlqzTUvWhxV3qyuyyYjWDuRJxeZmw0V1Qt95nr9+zQ8++4w3L19Da1jt9zgR0Knm5PyM/eHA08fv4TrPs2fPeP7mOV3fgFRkCH72yR/x/d9+gQ8Jy/WORsNqd0+b77i7e4kyCfrJmNthj7eCf/tv/5pfZAU/+dlPePHNd8iRYpJrdJawMUvCyJMnCUoIEiN5cPKAVGT87a/+IXZJdJYgJIfDBl+krO63OA+PJgLhJXsbeHF7z3g0JwsCaSK2Au8oRpLZ/Pd3q1orQR8ROvmIvKjoTYvqE5SzR/SEojcHfEiQKpDnGV3nUTJFhJK+k7EIOYmFOMv7Wwazj9gpJFUxY1SMyZOKQR1omy3BKxTRxW8AIRO0KtAqRueCDwy9xVsBWpOoDCkF1joEKVpHrIAQMZo32B585PPqLEMoiXWepmnjYinEghtM7CFxwYNQ6CQnKRLKosB7w353jw0deS4IWKR22M7gfLx+ezSo2O3hRcCYwDB4fIixQa0UQkKeK4oyQSdRqA7C47EE6UjSFJ2BTsW7z8xZkGiUSFGpICtGpFmFEBatNBKLsZa+bQle0LUWY5rotnOCJEgQEiFTrI9RRw8kKifJC5SLKQ+lNFKryBAmABqhE4pSQQjIcCz/9R6pJUFExFszNPidpyhzsjSjd5ayKNAiwbgmInNshw8eLRRCgHcx8i3TGDNWWkVUjAfhJHpokbpHCodSCVU5ZzJaUGQlEoUU0dmaZBqtc7K0eFec7ryLJanCESQ4PE3fgRbv2J9KKrTKybOCNJUoPEYlmOAwXiJlRlVOOD05g6AIHtq2pcrHpDonuMC4KlBJoDP9cUOjiMMSG91sbdwwFMWExWJBUVb0w8C+2bPZruj6AxILIQ7nCUf+87GkSw2KYejouwOjSpNo+25TjBC0bfMH3butd++iz10XnZJS5jGybC1CKmazKUo7toc191/cIUVCIhLUsZQZlYCHVBeAINMlk/GczXYX911Sst3t8CLj7OGCs4dnhDTHJHDYL/FWMqoWfPfNt2ghmBQ5w9AynkwpyxHz+YLMB86mJ1w/v2O7arjb3TOpT3n0/jN65/ji178iVQnjRwX0PbrK2fYDRZqSSE099JB4uqYBCdvdjizLyFWOd4Gb+1vW7Z75yQmjakIIB7yL/PxoFulJlSbNM5abOw77GmPhwaP3mC6mpN7SNgcWJ+e0fUte5BSlZ18brm+umE1PIUhGVUmZZZjmgPaavu7Zui3lqOTufsl6tSbPchbjBdIG+n3NYbMjWMO0LNiINdt9g1cJ02pMqhV917G8vaXeHZhNphQ+o21bJlmOMxl919DsakZJRXs4oEWg3taEZuD2+gazKCMq3ism5IiDY1t27JZr7PcNan4BoxG7q3vycRSxEiTWxjLjfXugkPrY76LozcBg40by+uqW9558QLPb0QmPSiVCJUyqMW4wNLsDIJE6wbUej+Ts7JTV2qMSic4008WUPE/Y3m25PDnDdAOJzNAieTcYDF7gBo+pW7RUpEdRejwaUZUZd/UqrrmVBgE61QzOUmUli+yMgzuw3zdcPDyhWow4tC23m2v+6eVPKVcvadxA07fk44JiUlFNJqy3O7wXbLcHFsmULElASvp+QBPFrFQpxkWFVBFRlYsEYUEHRZJmWA/36zvGVUHfDWQqpWsHrDSczOfMRtXvfW7Xg2GcKpw03C1fUxSSNIFsnhKcwzuHMTuSJGUyTXAuo20Nfe84OZlzfb2iqKa4AC9f7Hj8bMFqvcYMe66vB6pyzoNHE/b7NVk+4sHle2zXB26ev+b8QYkeBZSw7LcO1w8sr65YzC5oDi1pOmZz2GMx5JlEip7JYszBHkgKxfpGcHk6Yr+/46c/m/NX/8uGJ087XF3TH1pOZjOWN1f85Cfvs161XN9ek48SspHn9f1/5FI+ReQDduhprYSRp9sFqnzKZjMwqiY0dct8lqOynt4LuoPGDbFQ+eZujUw9m92Bk1lFay3T8UPadh2NAse0U99oMh2Fv0mhGQ6Szf0BhKOqZtxfD5haUyVTjDDoIKEP6Fzx4GzBi5dvqE5L7uodWUhwW0sYCxwBnUmcUHihGZzB257QN6RJSplJzNCQZJLL82fUB89Xv3rD2WzBihuEbMmrORf5GdvNnvE44Ysvfsn+cMAEj9ZjvvzVPT/948/ohg2Heo33e+7XJRcPZuy2d7x4/ffMpjNGyYTZfErddAx2xfkDT9ssqbKUQyMYjSuKskPnEiGmdE3Hm9dfgxi4OMu4W3qUz7HWU1Yly/USJxy7pqWQJZk+Q1fQm57RKND1K6zReBOYlDMO25pvv/obEi1ZrWrmi1N2m4YPP3nG1998SzWd8uVvfkNS7UlFxdn8A/r9OYobpjODVo4waObjC3a718znE1Y3kqVzWGFR44jYUyoWF8drfcFhMzAaC7rO4URPmiWoPKNeDxTliNZazh6esFttubycx4JCqVneNzx9dkKwO+puz2rT0fc588kn3K6+ZLPbcTi84NC5f/wk/t94KD0c13s+mgWERwXQOo0CZPit+cF7UCpDKXkUiw3Gdjg7oJIccFjX4oMjSysSnb9D9ArpUUkgWINQDmt6EJJgxREZEo0QRSFIkxQZoqt6GGIxspSSJC1RKiFN0neucil0vPYLTZJInLf0fYsQxHSj8wTv8Tb2+AQfwPm4htcJ1r41qqUolWHNgCAl0Yq3/UhChiP726ETiRISJQLOeZQMR5SIBBWd6sZ5rLVoFZOzh7qnt560SAnC07YxYe+cI7jI05bHMk4pBPpoYsrSDKENj56cUoxgcZ6TjyRZNeewzzEtZBJkiOsuANNbpJBAQEgdTR4Chv7otPaeLE2wLqYGu36AIDGuQytLWUp8aOjNwGQyiYnIoWewPb3pSDNF0x2o65rxtGI6nb3r7Xk7eLDGxL0j8FsB3SOCOiJrYsfIWzd8Xo744ssv+Zuf/xylNAQRhwlvcTMyHAcC7vjvgbreo7Vi6B2E455NgBAe52LnkpQpCMEwWB4+fMQwNHzzzZe89/GYfDTFhw7j6lgEq6MRWEl9RP6E6EaXktrHfiqdZTFDYA1aBLwfjkYlMEMfhygqAZUg9LF3UCV4JFIpsiI9iuce52P5LMLjfexGyvKCEAZ6a3B2YCCmF/6xx3+2iD7SmlHRo7HsOkeqUnaho1QFTktCLqMgpkoaf0+RC0ZqgpQKLyRdZynUCTpvsJ2kDd9gB4/NA/eHe2aVZNds8GmCEjNK+WNq8w9M9IIknLDzz0lFR6pbpJryXviAQ9hQMKDyBW27IJM9Tq9pTErbHtBiSurg9n7F9X3JX3514MVXDV0dkLwkeAUuYXdImJ1JerenGGfUB8iSFBcEg+pABLI8QRSAgbYN+DYhyWC73LHbxkmQqaFpa3JKRpViuaoxgycUmtMnnptXFmck84uE+7ue2SQlOMvLV39LcvKIBxfPUCrDmI7B7Nku3zA/fS/GUuwBbzo4xr173/HN/d+ws68oxo7LC82Dixmr64HRtOfBozGbpuXBj3LK8SeIruXDDxcomZEJxQV/jjhkhBBPYrxA6oTp9JTJaIbwnkRnCCGYVQv6cGBrUppuR0aKShMGa7DJG5TQyCTHhA7h4+a1rALbg6XUgbNpQu/G3ByuKbMRWoxJheFBKTiYLW3vSIspm7VhlM+xQ8JILDh4gwgGuooQLnlxvUEkjv2dQpOzaTasbxOGXnBHjWlrpBpxODQ4A5NFyWbXMZ4BNiMvE+5va+gl7SEwPU3YLHvytKDe9BQTgcCSqIT1VUueFowvMm6vDJtVx3xWsLkKCNEjg6KaxCiWzCDIiu2yZ3aeMeDpdpYBwWQeUCpnd1szO0tpeoEbBKN0SmN62q1Dj4E+3lhsMPRbizYSVShoFFp5VA5CgTUG16QMvScrNdOJ4tC1JFn5n3sq/68+nnzwIe1+i7SO+9WG+ekpQkhyp0gThS01xrYYDNYbZDolDJ59s+P09JzxeEo/GMxguF8tsYPDtDFF4Y2nb3uePHrMN19/i+0je8z5wGc/+IzNeokIBpXAg4sLdm1Lawzz8YI3r17jvaMfWmbzMW1Ts5jPeX39KpYFdgdcsOybgY/f+4QwwLdff8t9s0SmCdJKsiTjYnbBqzfPOcg9wQXaoafCUySCJ+8/Zug6WlOza2IBrlYpBoNzntvbGybTiq7f8/3yO9qh5bw8RYjA/f0N+37HyfQslppYx/3VNZXOef/xM16+fomSKhbmkXB7d4/zK7wPVKMR1sWeAG882ahCZJ5dfc+2vsd4i9IJSVJQlWPm1YKvX3zDEBwXF2esl1v22zXXr15SVRXOO9pDQ5FnKKlYLGZ88YtfcXJxwjgr2AsRi4CdY1SVFEmB6Qfef/8DXr94yc//498xm8+YzGeIREfmnnCkSUKRjhkVM549fsbpfM7N9RXjacnN6ga7MORZRioTHl1c4kzP894wP5+wXW/Jdcbqbs3T95/S2wEbDNVoRKEK/tlf/AUvr94wdB2/+fIL/vjjz5Be8PjsIff1ljzLCUqS6wLZQpqlnJycsO33TIoxt0EiVMbTz54yBMfP/+4/IoTn8vxBdDAnGevVhs8++ZzffPMlo7ziyeUlq/s1+37gzd0dxWzKaLbg5vYWMW7RuaDualQ5o25r6k1DW3esXt/RW8PHf/QRs8sZ6+aevu2pnUUoGFdTnn/7iomckD7IuV3eIBNBM9Q025Y8LfGJIsvGFEXFzc0tZhgiGug8uvS9VJxePGR5d0U39BTlmKZv+P7VN7/3uR1ICEITQsA6cWSD55SVOzqFA84PeARaCbJM432Bc5AlJXk6PS68JEmijyKjJ9DjgyLRKfPpiDwvEcRjDGGRRU6epthagk/I0oo0yfFe0nUNIEh0gtU5WiVIofDO0rYtidQkaRL53jIB7zC2JwRPMAOhjYvWJMlxZohxzvhOsC46WJS2BKlQg0GpDKGBoadr98hkQEiNMd2RsxcdSsZGjqLQniqJTvam7eg6T6LLo7s7Lup56/7wIWJDgsMLDyqgM4HUAaSLfHYZnZMhRLE4NtxH5rdzPVoK8gyU7PHB05ueptnTdg1SaYRIkapAqSJeO4+IHp1KlE7QSYolkCYpQkpcIPa0CJCJQklFmifHeGRA+DjIRR7jusTF8L7eMriBUTXCBX9kgmcEAoFY4Bl8OKJbNMgEayLaLIRjRFVnCCGRXpOagWSI5VVKJmiVI0QS0SyDIWBJUkmWJxASQlBIGZ3c+BA3RsKDApVohJbRmSIid1EeC4+CFwgvSFRCIhPafo9zkiwZMx7NGI8neCcYj3pwgjTJ0ULhpSNLwJHAUfTWWmOMoa5r2qalbTu0VsxmJ0wm8Vzoho7dfkPdNgxDiwiWRMffQxLNI1mS4YKnbVr2+z1CJIyqWTzWk4QkSY5Os98f1QRQ0zPKSnCSoW9wFhCOoBqKUtMe9oikYlJoTLOhrmtEGDEdn7Gv9yQ6oawSCJ7NchXdyHWgSkuM6mO5e1HQEZBJirFASCn1GFmlzGc7Xr96DVh2uw2TckxaSkwwjBZn6HmJlpaw7zmZP+JsvOXN1TcYF+hkx8pvyc4Vrm+4X28I3WVkm3pJio4s6EnC992ak7wi8QmH3qCzEWVa0O8bDv0t+65DphJwbDZb2sayXQ+MR5HbabsG3/VUZzOcEAQhSJTkzdVrPvrsI25+85zNasPZmWC/M5ggOH88Ie/v2a9qrCkZl5I8MWTaMhoVtL2j7wPeNlw8mPP0/TNevrkilROqLKU3KwY3IL1gNJsSpKQYjbi+WXLYNHzy+WdkmWK72nD3+jW2MUzKOduupu4tCZ4USTd4eiUYyp5be0fvcqTL2W5XdJMUeTBMUaz3e5btjsRLbGeos4R1pjnTBd9/9RpPoFA5xvSEYJBeIK3AGWhwpHogBBuTFCIiTZzw2H7PbFpCiINNFyz4gbJS1I2N0f1gKYucRMZSwEfvz+mdJ8kirnC1W9H0PZnNCEFgg6P2B8ZyhK8NsjYk3tD2B5xxnJ+es9muSU3Kw0cPsalnu90itaQY5QRlqNsBnWmEF/jGYPeSbt9xcjGmrQ+QDpjGMUnGrDcvqMqMdJwxmuS0dc3qzZr9smOwIFQgSwXBZnRtQGaas8WYohTkqsDZEE0uQpBkmsO+Bh/IdMqsqmJZtfVooxDGcmh2eFuTqN+/ryjLMvpuQMqU3X7HbDqhGqXcL1fokDCyKUVexPuWDWRpzt1Ny+nZiPvllkcP50gtSGWGZ2DoLFIIdruaIk14/v01RZlydl4wmY64W75ht+nRMuWwD0znFUb3FEUsDayqHB8so3HFerdlPK5Yr7eIcUKmQQhHqseY3nP5yDMaTXjx/AU//dlHfP3r2OE0ngsGP1BWI9Jc8Ktff8XlxVM++uhzDs0SKSWb7RWH3VeMR2OS1NENA8+enhN8geliN9R8PmK5fE5dH/De0bY1ZTrDOcnf/e23TE4URVGyORzI0oJOHnDBsD/sOTlJyXLNBx+8x831hqIYsd+1aJXhbMd0MsfYFnzgdrnh2dMzhIjdFkmicE4SvMZbQ6Id00lJmUu2m55mHzhbLBiNC+7be1wvkSqWCvfCcLdec7KYcrdZ4Q08uvyc0/NnXN/9B2xo6BqJ1nOSQlL3Dav7HbPZGJlKnr95CRhssAiZ88MfPcZ4w3a/BTyfffoD9tstFyfnjPKGNy9fkirFZDri0eUp2/2aw2BY7Tcc9gNZFs1TUtVo/XaAnGCCpUhyBrPHDpZHl2fc3hicBKwgBtAUddtTTD2Tac5yfcPuiJSczyYoKfDBkmcKkyccmjamdnct1TTw0fsP+Oqr37BrN5wsTjhfzNnV9wQ81vWcXuRMT37Kr7/+d9h9w8vlVzx+fMZ8kVEfBv7sZz/kfn3Fy+s9QwcDlqLKsXZPUZTMJid89eVzyvuM08sCXVjKbMx6eeCj9z7G9o6BHW1Xo1XGel1zc70lyQWtsSRJyulCcti3gOSw7/jlL75mfhpL2N+83PD++x/+QfduhEcpcAScM4QQi7EjgzoQSSDimNR7iyQxR1RJLDIHjzEdAYsxEa+RJkUUNSVE0TAmMKOj2AExrRcRmpa+NyglyfMMiCJ0lmVHZngsPlVKRf46sZgzllweuesiOoetNVhnUTK+eMBHMfTIeEeCUIJwxG+Et5gTIfDuLQ8+omm0TvFe4cPRpUxkfwsZ8Hh0qnE+vibeHj8zwf+PvT/bmSzLs/yw357OaOM3+hweGUPOVV3ZVd1FQSS7CRACJJCQwBtd6EKAnkJXegg9gi71AoIu1JREUiS7q6u6corIzBh8/kabz7gnXWxzz6ZIqAqZDTZBxgYCAQ93t8/C7Byz/157rd9CSpTRaG3QxqB0jh2TwcYHQWY04xh+j1dJHGM4upwFqeBcKcFkNuH8bMliOaGoDEJ5BFBUGUZA0An3YkdL1w6AorcOHxxlUTDaEW0UniNiJkIIKeG/27Ws1xvOzs7ZrdaEEFksFiwXcwR9MpgdZ/h0aOLY7TesN/dIKdgfNlg3Mlqb9kkc798j114I+a/x0P+bBaOJuSNBRLRJpruLiwvu79cYoxmHDmJ6b5OHIjm5syxDCIG1w4cOoxBSEWkMIGU6VBLH11MrRd/3bDYb/sGf/XsUpeRudU8Uaby3LmHerHeI43b0PU/+PfLSBZtIAVlKKlobwCv8+PskrBSAEik57NP1ndzn5gNSJ8ZAFHy4dhM3XoI0aJ0lnJu3SC0RXiQTVvy7VfS/t4guIqx3gTKLDL5i3UOtZwQZqJTmft9TaM0kCyCKFEXPPDJc42xyN4/hlneHHWemJ1cVkyLxS4VsEFEzRgOjpTR7ovoVfhixQVAVn/PWdWj1FuE/QgkYuCPGGUFZ5Fgy8w/Zj695a1/y9ioixxnvbm44jI7dzvHmVeTgBuyqpDoP3L0KFFVE9BalBLv7kdPHc9Z3B6QUtG7LLJ/RdI629ZiyIy8ylE7uvP3Oc3ZeEHxPaUqqTDBOO5SXDH7EDZpiFjl9JPGto+0sJldpQ2wc06VEqwP/5S/+LxSF5Sdn/5RPHv2Y6UQTYoe2ewQWN2kRh0iZl9yt3rGcPuXl5jd8e/8v6NRX/Mk/Mmw2Z1x+pHj6tCQccmR24PxM89srQQkszxqW5cChe0sbHMvszxCbh0e+VsZsccl8doY8loppqdJNJ1JxlxRPWI01KihOTUWdH7CiYBg9zlbk4iFbd0MpBwZfEccdqlYooRCsOHQCp2BWO7p2QOuHHAbLfghEvadzDbvDvyCaBU4GnL+nGQVt4ygL+OrNG27XApl51neeV68aHjz0DL1GECiyivvbhrIw6CzDhoA/9Jyfljy4zIi0HDYC+gwbBmbVhNXVyND32N4SjEAZw+a+IxCZnGYUVcGh33G4UShy5mXFYl7y7vaACgpVS7qDRAlN9BafO1QuEVGAlfgBTCFYvxswlWM+LfFdJIhIOckIjWc+zZjNKg7HjZgB9juLGBTlqaZrLItTwXgv0VbjosN1kW07MDSefAHVJBXN6mj+vrfyf+d6c33D+WLOy9evCALublecL085mS+5u7ljfjHB5JrNbk/vRk7HU0ZnkVLx+OKSTOScnVzges/J8pyD7Ql3N9xe3eGHyD/8J/8+dhj56Y/+jJffvjleG7DrtgTlGPuOWVlT6YqynrHe7MlKRfG9j9ht17x584bT5Skbs6KcFDx99pi77Q3jMCIQFHkqHLy+u+H1yzeMheOj7z3l3e6G733vOcMwIj3EI0NxtdkSJXz17W8pc0OeGZbLOScPl7z4Zk8tMx4+vkSQNo1df8B7i48jyijuNyuGrGR9uAEdyUVOc92g+5zLy0cg4JfNgawoODk5obYTyBW9T6xUpRWv372l3e54dHGRnLY2uVD33RpTaE5PLtltGqp6AVJzc3uHNoY8Lzg9nfPlr3+JHTtOTmY8ffyEN2/eoINgc7+iPr9gsUhs1kcXD+jHjh98/CmjG7i9uaHdtahCMDubUUyntLOG33z5JSB59tH3uN+smZZTNocbekYaBva64eH5BecnJ7TbLcKDHx1vXr7k8vwhmcrQRtM1B7p+4P53X6OznPlkyhigbTt2+z2HtmO+WJBHw9vXr2mbA0oEejtgveP07JxJUWE97HYt9aKmazua9Z55UfH5p5/iQ2Q5n/PWvEYVih9+/iN2bcPXX32D1pEffP59tvd78ixjvV7z8acfJxepztndbemHET+psNOSAPw7f/EXvL15xa/f/AvGGIi5xMuQsD3jSPAeLyLL81POLy8YGanLGW3fEoNgvlyyOzTs9wPnD2u+evkN9SRntbunnGY8fPSQGCTbtkEJwfnJKV3XI5VKg5bWnJ+dYbKCGKGaLtjt1xTVnHZ9y827N3/wvS0iZErj8PjR4m1CmZAnh0jf99jxyHn1CX9idEaexQ+RUK1VcvEoQSS5Too8FRkm/EUSRrqux/tAPamoqgrne9ruQAgD0giEhOgtLniMySmrHETi3I22w7rIMPaIssKjiTIg1NFNEdMAGAX0w5CYgHlOXVakEqFUjJeipx4RIl3XE8LIfGYS09yOycEQLCmOmtzOoAkxMA6eEYfQCVeklDpyIMFkkswIhsHi/UDfRyImbaqjJ835Cq0MSpnjZ4fHDg2TOkMWyX0vpca5EWMy6nqKtRmIVLCqtT4+z4hAfcC6GWPI8oI8K/A+0HXtB+52GhIFISbmI0LggmD0qY9JC43JTcLZWEFw6XDw/eYphIjRCkGKorbN+wFaUBQjQqQCKyE0eVYAEaXeu1MS/kSgkVKhpEGIJEZnmSbEGT54+r45sgeh73tEbJEilbymQiJN31mcHRObO5NE4YjRJ0SXG8lMRpZpTJYc5d75DwO3947gDUqnGP44pBInnRcYXZCG/RSN1pn+wHaP3hKChegxSlJW6Xtk50a8HWmbA8GNFNWcSVWipWTfHNht9gxd6k1x1kP0BJf4mnUxpSprtM4Y+45hsDRNT1Ulhql1jrLIj5uRxI/+o5YX1PkUXOTusE0pgUyjtECpSJaZtIkUFusP6LRrRxDJsuRsG8eBPMvY73es1+tUVl5ntG3CsA39QIiOtm+5X98xhJHZ/Iyzs3OePHrMfr2j6zsWizkySvJZxRB7ZidzBp9KJbfre17dXfPsB99jL0fKjSJWnt3ta0wOUg2YKnC/XaFUhutGml3LdDJF2ZzTs1Pi2CGEZDKZ4aXAW89+12CdTVzXAKMdePPqnjKfMQ7QHkbyoqAuDGPwyHEgHjdWQ9Nzt2l4/ORjzi8f0LQHRgdv3q7QueKyWnB5ck7BAdcPVLWmrEtUljPPK4b7NT50hNHR7LZcPj5hv++4uXpHXJ4AjnoyQznPycWCelpBFNSzKVIYnj1/Rl5U3Fx/wX5/oFCG3X6NwDCb5UxnC8QYcERKPdK6kcePLnG3A9ZadsOAkBnBDdQm491uxd43XD56zIvdFfmsomXLN29f048D9XyKKTJil/onhsHhRk9ZFEgRGcdUxFXOJ3h0KgEXkbosqIqKcYjsd3ucHZCk6PTydEbX94Aizw1FkeGsY76co/IMqQ3DMLDdrtk1e0S7o85KbDNyejLF2p79uk/J3tkUZ0ecc6Dg8ZPHhBCZzmbctSuCjEQVObQ7dC4powblaNqW4C12DKzXlqcfPSIKaLqOb198y+Jkzou3nnJiUuonRq7evqVZD8zrCtsWBO9xI+AdbhyJVnF5doaMkn3Xst3tKfIKlWWM48DV6o5MGSZ5wWRSM9iBw7YhNALlPFoIWjcc00B/2LKDx1pP1/U8fDhhf9gxm07ZtSpFz0Pk7OyCX/7yG/yoKfM55+cLnj59QKYDbbujKPKE+pKSsXcEL1jOZ2zXPZu1YL22WG9ZLg9HprBCUZFnGbtNKh+dzqbstgeEDIyhJwrFycmc7W7PySIhBurTCYdmZLVp8EPG6dzw8tUb8iIhIaoaHB3FJONuveJhmTM/XbBrtry7uWO93TOdGabTkmGM5LVgu9+jVEKIyv2W2dSwbXbEIrJYnGPthNXmirYfcVZwOp/w7vUr3r7ZEtUMVUQEjuvre4rc8M23v2W2zDk5rciyJYfmDiEsq3XDpJ5ivaCuas7PT9ntVqw3G37y0+cYo5kvJhSlYrfd0TQjMRb0/Qpv9xAN//6/+5f8p//s53zz9S2vvtnwD//8cy5PNH3XsjzNGf1AXhqef/KY+9Wa3lmKrKYZBrbNjqA8gUjfw5Mnz2jaa5RSjG4EJXAxIIzC+SQy+ugIoaXvd+wODYtFxas3V0zyitXdhqoqefbkMeKYPJNAu2849EDM0AqInv1uzWaz5fGTKd566qKmyhQqwsXJjG5sQOR8/7NHDO2aMEbGzqNlzrrrePhAgxk5vzzl+uaKrht49LBiHLokeMaRySynKk55++6GYhIpJhk//9XfEhgRmeftzTfkhWW6EPTjgAo79rtv6Ic1Tz96QFH2NOclUkUGt8O5yP6w5+rqmuXiDKRhHAeCtwhh6PoeKQZ++tPnDIeaV2+uOb2AsRTcX++ZZjvubnfMzgdms5pDFzhdPiO6DYdhy7OPl4yuRYrIbDbjIGC/3yCB/c6TlxmPH1V8MPz+gatpmg9Oc/CJK65FEshdEqu1PiaEjv/N+0Q9SIWkqVzRD8MHQ4wQMYmBbjziUiK41G+jtSEEi9YSpdPhjlLiwxwpROJ7932XBHHnkqtba7RWeO/pujbNVio7zl8RpQzWBoaxZxw7sswgZf5exSfEkGYzkbAxkXjEpcgPKJgQU5dM4l5LpEyMbWtT749zjhAtmZE4YQkypNeBiDuKsVmWY5RCuiT8p99zR6PFse8vz/E+FQuLDwJ1Kq8UBEIYKQrDfD7lwZMTLh+eMZtpykqDcEeh36FMSuFmDgbbcXV9T/CKw75lOpuizgr63lFrA0hurtcYY5hOZzg78OLFK7IsZ7Pec/Xuhul0xnr1ipvqjocPLolBsd/vaNsDeWF4+PAC6wbAM53N2B/WrNd3TB4sjtdQSgxInZzwCU2pjkx38YGN/v+7sizn7Oycs7PzhHM6Yh3T9ed5jw/K8uwDhrPv+w9lqiHE48GKSJ8pvE9KBIQ0KCX53Ve/5Uc/+px/99//S+73v2PX9AiZXstw/BkpMeiJUSajjlBIFRAhFVsrHVBK4ENAAWhB8GlPKY1BKpWA9lEe96z6uKdJ11XS+gUxpsMfKSRZVqJEdcS68AH3khLRKUXxd62/t4ieZzlCehrnECJnWWiqvMQ7yzRbUpgM53Z4n6LD0Z4gtWQ33KIZ2VvHvA5MTc0oHN9ut0wLQdc7ijxnOwbm2QwpRlrf0dkXSHmCH1reFf9XhtCx2pQsSs267xByYJLXRB9Yr3/D69e/4Js3K0TuuX4Hq6s1+61jcZazb1IkXNc5s4uCrG6Zzw0ESX4ZcYMhjobQKkIvMRODMZ7DpkV5w+gDfa+Rc4HTChUDonSJW3slmcxzshzGzQBFJHrF/DQnahh2Cj8mjlPAgXZEXxAHRXHuiQSCEvzzr/+fbLqv+fHP6hRhyST7dYeYfEUWet7uDa1xtFmOf2gZhmsW5zkX8ymfLv6cW/svoWwIpWPnruj8jIulS46rIPnn/8px+8bwvY++z8XZf0C0OfPZlOXygqqaIaTAR9IHXPDE4+lOJILYMSsOyb1mFkQuUGGLZU8W53T9SFUafNAgC+qJwHkYwwZYYEUBtsOrVEBxaO7QckrbQxM6lDd4Kelaybfdt4Q+Z9wP+LijnAS++lZx/U5QVwZvFd5mXL0QGJ3KB3zcE5TBRkXTtai8YlJovI1kRtMNI6MdEXguHs447FqyDFZvBi6fzAhOsV+PPP/8lNev1zR7Rzn1NFeObBppt4H5icFMLKYVFFKSTzX9Km3+m75DZ5pD01I5SXcQFLVksOmDYVLmjINlPEQGL5k+MFjt6bcBmUvKTGEd7O5aJosaXWnW17vESrZQ5Qap3XGz4DmZVczmqbDFZIpMF1j/x7nZUJp92zJbLHnw9CFfvfqWw66hlTXbw4rW7ZlMpvz8l78mrwq6tufQbqnLinHfM5lMefXNax5dPsbIVD4okfRNx09//KfMp3NaOgSS508/Qqvk8jy0O+7urpgvplzft/TdwGef/AAtW3abDbv9huVySW4UQjiqesKh2XFzc8OTp0+YTWes12u8C6zuViihuDg7Z+sOrK83PL54xMl8wTiOOP+I9W5F1/Zc395STY8cPC+pRcnv/uZLiqIkDgpBxvn5I168eEE9KZjNKl69/iYJLgjyPGc2XzCIjs32nqHrmTJFE9nd3XD24AF+GCjmc969fYuPkZvVPbkpGbsD8/mE+75jc9gwmeacLE5YX60SC18oghMspkuqbME4OF6/eUu735HXOQrJenXP9z//FBxstytOHz3mdHHK6WRBs9rRNz1v315xefmIsRlYzqY8ffiE9W7N7asrpkXNbrWlzidMJnNOFiecnV6QF0Vyk6KQHmb1gpubKx4/eESwgXdvX/P5J59xujzh6uYdp/MTLs7PaZuOput59/YN3nuWJ2dc5OfsDnu+9/H32LU9b67e4UbHbrvj+Q9+zDyrefXiBT5aFHCynBNioKxrXBfIVYkYWjb3K05Ol7i2Z9uu+Xz6OZksCDFgVMZoLVVRc3+/4eHFQ2anBddX1zSbNvFYiwKtFM+ffcTq6prNbsvi8oJXX3/Nj//xn/PVb79h37Z89NHHvLz7kia2yEozjCNaZpiq5PzynPyjJ3z2g09Z7dYc+gM/+NH307A7evZNS7u1lMWUt++u+Yc/+we07RYbLP2m4fzpAx6cP2L3xReczBdkZYnzAZ1lNPsDQnuE0GiT7h07Bh48eIJzA6vNPYP9w9mLQhwLDoXCR3t0vyZXhx3dUQRM7D6CgKioqoq6nh3xEANSGSIRH9Jwqk1GlmdUZYmUhu44fHkfOD09oShzjI4YXVDkVRJI3UAkpCKiKMgyidYCk5FEll2HUjIN1EEQkUgZ0CYVWCqVxGQhUut6COHI9c4Tf88N+BZ8tInZLRTWpjIeIdPgbv2AjwPRjth0K38QoZXKUMpjrWUYHH3vyYyEqNAajAFlAip4rLc4S0KYSJU42DgEKnViCHXkYAv2uwMxZkwmp2lzIDRKBrIsp64ndB1YGxM3UsVU2KgMWVZSFMk9oY8/J8+LxFiMiW3+nsXubHpPxyEdItgA1oPSOaVQyKJACfCCI0ZnxPkUESa+5zSmOUUIgXcJtdM2AyEoCJEir1guzhlscl8PwwE7jghhMLogM/WxrFMcSyo1dVUDgTw3WJvKO4P3iXcoU3Fs9AohJNYOiU8voAjJsBCPKQEJGCXJtCTTiuA8kdQToo9cxMSFfM9WT9FzHyQ2pIMPay0+epQSdEODajyQNqExSoTOkDGkQwY74MYeJSK50RR5Kk21Q0e739M1HVpogjREn5x5LowQJNPaYLIC6wJt2+NsQAh1TPapoyMpYq2lbVva9o/jqkqrabcjhc7pD2MqRCoCQlhi8CznJwghOLQrinKOkJ6+AyEjJjNorRn6kX4YOD07Ybvd0vUdh6FJnxHDiB168lwdi59gs1vTdqmkazGZ89GTjyDCcrHEe8/s0Tn5SYXJDfdXt9y8fcvYjvzm7QuePX3K2dMl509rnGs4dGvaQ5M+m2JgjA5nHYUxiEIyhhHVt5zUU7oQ8K6jLGvWhz3r23syqVASsipnvT1QlanQ0knP0DsEnhhSBLyc5+y7Fh8j3kbabUO/bXnx25f89Mc/YtesqWYlg4u0XYe38OMfPWVeQ6wDQyNQytCNDqUik3lJP3bYPdihh2D55NMHLE97hCgw6oJHDx4SZURmIrFWiWRlSaYyzh89pO8Hsqzi8vIBynv6Ll0zWQH5JMe2DlpDXS44DHcEKWntyLY54DKJP2KgchPZ3615+ukzfJGTuwoRDDJX4CVTPcXkOYf+QJSC8f11aQRVWRCjoyglo03iRp5lIBIOQBvFOKRNbt/1x+4k6McBaWCyqJFap7Ix79nuG8rlEmv9EUFX8vHzZ5Qm5/rlLeMwoKIjKxZ03YF215K5ijEXbJs98+mMXXvg8vyCw+5AO3a8u77h8fceElWk7Q7kpWY6qwgusfn7vqOoFFmp8N7y5Nklty+3/PyLX/O/+A//KUPTcXF6QW97xranPxxo9w1nywvKwuCGgTh6jFLkBs7PTpjVU5q2T/O1EozBMXYDY++ImULlGbumQQcNQ6RrG9y9o9QZJmq6MKCyP9zgsl51TKea6UxjckXXHhhDRjktcWNgv99xcjphUhdcbQ8sZo/Jc7i+uuHspOLs7JSyyri5uyLTyQRT13OkiDx6OOf8VGHdyDCuCaHHWosxOYWZ8u7qGqk8+/3IYiYp65p+3BOGASENZTGhzAzDOFKVJX2f3LvdocUOljDWzKYS147cX0X+4i8+4t3qBbPFlJevr45l5IZhcGR54qirPF1Ps6Wm3Y1U+QQhDEplDL3jpn0LSMpqwq+//HmaTTS0reX87AFdN5CZtAedTk/Ic4vqDESNlJFqkr7HhYAiL3j1+msms4q2dZT1lL63jF2HW3iu3t6gjURPIlGM+Diy2qywg8UOgfv7Ax99tETrSNttePPmaz753iPsELi966jLGa9fveYHP3yGNIE3V2/Is5yrm2uEkMwXF6zXLTq/Yt1IpotTXrx4Q57nXN9dcbJIRaaz2TJxrjtHVhS4fiDimU5n9G1Ah4jzya0vRANWMD095fb2jum0ZjariTg2mzVSKoZO0B4GPnr+CG0id7d3LOZZMoQdD+LrcsGsrnl39TXnDxZkRcXd/Qa8Zb+1ZHpOVJ6Pni0pypyqNnz561c8fvKIu7t3vHr1hqrMKMsSax1t29IXkWoy4eGTJ1xdX+PjwOWjE27vrykqjzGGcbRYG5lVkZOTHDhjOves11u6zjM/Hu7luedv//bX1BPNMIzUkynTkwVtt6J3PXmmuLl9x+XZcx5cTDi5DFibUmWffPScyeSCoYOm+Zb5vGYcHP/yv/o5RTFheW4wJmOxrOiaLUVhuHp7x4PLU4Y2Y7lY4MYVuYarq7d/1Hd3CJH33HGAIFLfjXOeYUj4XqX0UQhMyMKIh5hE8VSimJC0xiSxPR5LGMOxbDOhRVJKcxgdEFIXjkjGgtRzkw76nHP/jX//630tIXiGoaPr2iTG68RJT3z25H72fsT7kRBTAjZ1MaViUq3V0UgRCDEd4stjcWjwjsEeix+DIwQJKLyzjONwxIikQ/L3JZfWjscep1QimZjo793Yae4f3MD9/QqlDc6mgwpnHVKmNK2UqWReSpJRhEBVlzx4+IjTsyXnD2fMF5KqhqoySOXxfsD6AYVEKE1U4INjuz+w21jyrKYOGd98+5bTszk1mpvrO7b7lpOTU/rOcb9acdgPnJ3N+ebrV4QQORx6+n7gweUlRm0QaHb7hqur10ymFVVtWCxmzOYThPAcDjt2uw3hwn/YB3zA9xwZ74i0j/pvSefi94x0rQ2Hw4H1ev2BYy5QxwMGjqKyJDPp+30YBsbRHoud03Ub8QgEQbwvkk0HM+M4UBSaqip5++4NTbNnsZjx6l2LUun97weLMSm5EEK6zjhiFZWOmCPf3fue4OWRdR4RvC83Fcl1bgzS6MTiDwJ5/DxLBppjwet7tn6UgEarnCwriEHg/BEPI9L9kjBB/waZ6IUsyHSFiyui8mQmIryAsKW3W0LM0VmO7QRt55lXDqkHlD2hyBUTd87B/YIYUuSsVJF92xP9EhEjmd5g9Q2HvqcwM4I3FGZgb2+I1tD5GT//bUulPHe3LbrssaNjv7IMAd6+cuz6kcyWnD4yuNESrOCb3zYsH0uKqWRsAoPy3L90FJnAtZFiUtNsLNN5xJjEni4nEt9AP3omS43oHbkqiMWI8JooI2UUjE5QzhRRRN59uyWfBEozIchIf+iRRcDZAXvI0ZWjnCbhfjrJ+NlPHvHRR+B7z8lJxup+x6efKOaTHUY+4/6wwg+ai7Nrehcp5pcskRT5HWOckFcLJtMOO0Z8+c/J5BV1NuPtXYYyF9y+jfz2d3tOHnS8em1497blYn5KdviUqv4Yk6cPuExF2t0r8moJMkPoAhcD3o3kGSCgGV4y2ogwLc4XZFoTY8bUnJKbBevBEweJNCVB9IxEmt2ewJRVb6nKkX6YMYwaLxqwGzabyP4QefrQsN/0vFs5BhuxXnLzesdhC9PJEhvX3LwKeBTbe8/yLBLjQFFXuFZw9dKxeByYVCUhWiaFJDaCQ++YZJZuY+mawOJkgpSB1U1Lu+8ZW8PzT89R+YgfDF0/0PT3xAh1lbO7axEChk1keaFp9iNVWzPLa4K3DNtU8OmGnnwq0DLnex+fsr3fcfkoQwjP/uCJVU5wIzoXhDJDbAOXjyZ885sVMRgKOaEZtkzLBZ3b4hhABrIso7PptM8UkraP4BUnswLfRjZ9w8MnE7QqcL4n/HHdZCxmM0qjGPqG+9Ud/dDR9x3WWYq85Or1G07OLpBB8Q9+8jNMoXDDgPeBKq9Y1lN6b1kuluhc41466rJmPp0xKQvub24pTMU3X39L8IHPP/uM7X7DfDal6w9IJdFZDkrx7uYd0Qv2210SzUL6QBNScL+55/b2lrqu6do+ueFFEt7W92tKVXGyPMUfJKvNmpOPloxDx9n5Oe9u37Bar9g2B8o8Z71aMZtN8HHA+pHVbsssCEpVMTrLF19+wcnJkucfP+PQbDG5Jh4CmS7p25EX29fIEnSeUhmXyynODZRVwX6/TUKpHbm/u2d5ekqG5tHlo+S0yxQrIm3b8NXXX7E732PMlJOTM0T0qVCzdSwWC1zhuTw55+vf/RatDbP5nHEcGHyfEBYu0Lc9D84vkS5Q/YOctmt5+fYNo3fICMoLog20uwYlJB89+YhreXOMIk+YT6f0XUeADxzQQhvOHjyk3R4I1pFpjXcjV9fvuHzwgCgjL159w2G3Zz6fU2QF69WWPC85PTtHmYDUCQujhOJPf/on/O7r3/Lu7RtkEDjr6A47Xr99gdGas9Nzvvf0eSoiqmuenC95Vn7Mv/r1X+OdY7AD03rC3e0twgryskzsNyXY7HfsD3tOFguqWcb9zT2npyc8efQRt3e3LOcLrlfXtF2HjZ7ODlw8fogpcz75/FNev3vLb3/zBYddiy/AZ4nvryQYbfj0Rx9zcnbCP/rLv+Dnv/45v/v6t8SQIq9FWfLZ977PL3/3W969ecP8YsJHHz3j11/8nNOTU7I8DfovXrxgVk+w/ZCY2UIexdCRvh/YbA4sluc8fvIR75oDp6cLrFA0hwP9H8FNfu8i0Voz2sQHjkeBV4pU0phmbUWWV9RVzWw2JctyttsN9/e3hJDcskqJ5ASOFmsVIS9QKjlqnXPUdc3JyRLnR5pmc3Qz12jdpT8z2IT9EIIQE2JmHA/JWX0UWyMK6RyZS6KuVAIZBcGnCKiQEqmSq1kpxTBYpHR0XUvbdZjMkBUVIR4ZkPJ9SVJiLCol8MfIaiouSoK7VgYvPV6QxHCRo0ROkBKtLWWVURQZkQFrIXqRelzMFKHAeo8Q3RHxkoRoKdPjO+uxo/1QppTnBconx8gw9MfSJo0kpMSLS++HVjlRJIE8OXQEeZEDHPmTLdamjQYiglRED97aJFr7iNOGYEfGCOK4KfI+EqL44NIGhRSaPMuPOBPHOHqapsN7QZ4rsqxkNpNYV9K0u4QHcklUNDo5vrOsPLrRj1FQociy4uiyH9PBQlCpzLbQQKRpLUOfhFPvPYjIOAQYx4SOsEPaQMWAEhEtI04GpAwYLSiKnDzLyXMDR3G6KCYc2o4QJM4GhjHFlauqZOg7hqFBiAEhIlKl10ESsUOP8w479CgBk7pESUldVxBT0eXY98goQGiiSxxxHyXWp4IiiWEcPUNo6buREBTz2SnT6ZxJPaUsJ0dH/kDf94zj+Aff2wCanNXVlrPFKSKkA6gQIuKI5YgE8qxirpcUVeDQbBm6IbmOCYzWfmBjm9zw8PED1ut12hBFQcgtmZ4ScJhSYfHYECiNotnvGfcD33v+KUWe863/BmkUcpojjeN2s6LZ75nWNZ3OCIXi3fae2kg+fviALMxo9gW7bs/VNrkzvQj0tkdoyexsQRgs+9UG6SCThpBBZx3r9ZbDvmdZT5lMJohMM4wDQzciEQTn6do+OeYChKiYXUzgiLMSReDg9+Qi48XvXvPs6XOePv+YQE81qVldH7h48pDdnaOcS8qZYlrP2G56bu/X5Lnh2fNH6ExxuB1YLOdIIVmezUFJXMjpe8HgI7brqXXJrjswuhHvAj/8/Bl5mfPqxRvcEDhdnKFjIHqX+PRFSVaVqDonkyU+dExMgchy9LRCd4HZpMRPFDf7G1b9nmgtnyzmfHP1mvnDJXd3a6YnExosVTaj73qGpkvxeKXJlEILzcXFHCEju/02lYmpVKheljllWSCEZLs5cDh0CCJt2zI4yCeGejLFlCadvwZBFnN0UaCUweG4v7vnsE0HaqdnC4wUrN6tUUYwuI5D25NREMLA7XpPOw7Q7inzkkPf8e72hm4c0EbTdB1Pnj9g1wlMljb0hIiZVcAEaQRepM+NalJSLweuX63Y7tbU5ZShHdGVpt0fKIxGzSqa3T1FodisHZnJMSpSlIrnzx8jhGN3aBiDReeGbhhZ73boSUZW56Dg0O85z5f0rse5kUlRITpH5RUnywfs2z/8u/uzTx+x2awp6wytJeU04UmKvCTKnsxI2m7P4yeXrO4sv/jF15yfnWLMyPU7S/CGB48SfkyKQFVkaBnZrLbc2QO5OUnvcz0FqbF2wLlI41oePXlElsPbN7es7juyHsoKbHT4wVFXVSqolgon5LETRPLowQO61vHq25aT5ZT2YPnr/3rFf/S//iEvr76mrmqCD2zWG05PH1CVGW1rqSrD7f09LtQcGs+knvPZxz/Gu8DLV1+gTXKiKq1phx03d/doU7Dbd4gY2G2vef70CRcXl5jswOu313y+PKNrR+K0RmkoCsmDh3OqcsbtzQY3BgKwOFky2p7gIk3bstlsmFRVwqr1O7LccLduGFyL9w6hMm7uGh4+KokiMNqR7a5luTB89L0HnFy0fPm7XzGdK1abW548e4iUSSAK1nC/WnN5XvDw4QQb1rx684LPvvcjlCgJPpW552qOUIrlxRSVa7aHO4QeoW8YB8/G7tE6o55qlmca7xw+WnSumcwWqTzZWkS7Z3ZSs9ru+fTTj7l8nPH23RuWy4ys0Az9hslkxnp1YL6cESOcX5xztjxnu7/n9dt3LM+WuOgxOtI1nl9/seIv/vHH6MwwrQqCt7y7uqOqa/re8vHHT9luVgSvKMsT1qsV682KECRKFZyfX/DFb75C6nUqT3Setml5+PgEO+65X93x8uU1T5+dM1nOMSYJyW03Uk0KPv7oY2aTHa/fXLHdHuiGG4qipCjTvkJpwWSSsdm9Y740PH42Y7vqGfuB6Htevbziy9+84pMfC66vb9hvJI8eL/joo8eoTBBExzh67m5aZnPBkycP+PKXNzifcXZ6xt/+8iVPnhqePX/8R313vxfQ3xc9vi9CfI81Se7YgLX2XxPFOYqgyQAQgj+6zN8XhyZB03vL++J1rfQxAenwHhAB531C+UWObnSOYnRM12kQRwH1vYA+0PUt/dBRKYn3I8NgKcuSEOXRRZxE+mS/9IQocMEy2AGhRDKgJsU6uaIDxGNpvTy2VwoZQYSEgiHtV/SxSDUE90E09TFgR4/14LxN5o8YUDJppN4FnB3ZbjYIJM4OaG3oO0uWFR8OLsTRLCqOr+mTZw/5i7/4C7JcIfRIlDuKEkweCLHHhQ7n+9TLITOEVBSVIc9zmmaPEhO++foNSsHZ2Tm3tzu++uo1RTlB6w5jHF1jCV7x9s012+2W6XTK7e0teZ6wYZvN/igWe9q2JS+TSWgYW2L0OO9x3vL69Usenn3MYnGCcxZrR4yxCJPANBCOiVHJ8QTi/ZXHe2VdKcOPf/RTLi4e8OWXX6bX2Xu00h/EdyFSISzEZEh0nhhSF9T79y0S/7XXNCKkociLpO1oyc3tFTe3V3y0mCBEwLkRqQAEzkcyY9L1Hf2Hx+AolENC2wSfsDHvzUtE9a+lUcWxbDThkLTSqUxWCEKQv8f3hNT/yJF5nkpr08GD1ip1Y6lUNC7l3x01+XuL6H244WBHKi7Y9Qf2PjLRW4RSbIIj9BqT3bHMl7TO0zlHrQWNayjNCVF2OK8oxIyJOGfgmmlRkumcQ78mxJLVviVTS+gvce6WX7x1jKGiGQ781X+9oztoon/F9jZw+lSyu2vpdyB0RgiCyaKGGOi2MX05jJ45OYUqOdx3nNQTDvctxURRzzXmsuD1Vxv6rWN5MaXtQAmJjopu7JG1o3eWPDNEOeKGSLsfCT6wfFiwW3cslxXtYeTs4ZQgHCr37N8N9AfIp5HHj2estGUYHYvTks8vn/LDP8lQwvLZkwmeHZJ3/ODRT2nDHat+zbz4lqwQyKxmNwgkObNKYPsKb3uqYsZKfMO+BS80flvy7u0JW3dPe9jTdj1f/sKxXQeqosbpA7NS07SG4tEpSgmUkVi/Y321Jq9zsqJCqQwfeoQ4cm9FuqDL7Iys/4xBvIIYCf3ApveUmWHfX1MWM4Kd4NzAYAWDHwk2Z90E+mho94H9RiMKixtHNgfFdvsOneV8+//xiF5yaBy7LaAFriuI0dOtG/b3ErLk7nPR4osl2zdreBRQIcVksixnf2gYW09ZG1QtUDayWQm6PWgd2NxZJnPNZJKjM+i1xsUO2gpLz+K0IHQSY0b6xnL5YMF+7PCjYrqUlJMqxYJkwI8wm+aEmNH3nn2/pg0DxSTFK5PzbGQyVazWe5bnhvvrQGwMD57WDJ1jeVpzspzT3PS4Ln3oZVKjlUpFf15STUqc7KiKKdYZ+r5FBFg8qMhGnSL/3tP3Hj60FP9hy/Ud/RBp2z0tA+3QYoPlZn3D2WzK0wcfkeUFP/r0x5SqZOwHbOvJMo3tGnrdMJ0sESJS5BmTqqScPGR9d8v11St+9P0/YewtVVFwc3PDF7/6BW3fMF1OqasKkcHusOXV69dUWUWRFSileP70Y774zRdHtm9iJD988gg3OPq2R44S6y1Ex/nynGHXs75bs+t35JmhKDPa/sDoJ2z2K2Qu+P7Tzxl7yy9+/Tcs5g+oiwn36w3OCkLQfP/H32c5X/Dlb37HydkJPgR2+z3Oe+b1nOA0RVUzOMsgu/QFUM449COzOmPb7LDOk/Vl2qhmGYfdgbP5HDE6Qt+z3uyx3YBWhq7ZY3TBbHqCDBojDFlu6No907zg7MkZzaGlHwcmiwnLk1O++u3v6Pc9P/r8x0iZ4axn6AZs0zIpa/I85zD0vL25pmt7FmXN2I/c394xn83R2nByegr+KBzFdLL94NEjILC+v2VSFJig+OTxx/g4sj9sUwRQCwZvUdpwfnLOZFJyd3eL0oZHD58ghUkJDbsjKMnrt+8oi5oqr3j28DF1ltE1Hav1PYUxLCY1eZGz26y5vb/j4vwxRVlzfXfFiy+/5dDu07BoPSYr2G3XzMtZSgiMU97dXPPzX/0tZVHw2eef8uVXv6RrWybZhKooEBHu7++4v7/j3f0VvR14Pq+Y1yXddkuRV3Rdwk2s3+woHk9pfEeRaYiR6aQEYRhFz7/61V8ljnAu8MHy13/1C6bFjP/4P/7fsCwnLD79nKrK+S//i/+cqi5YTBf87c//lvPlJT/94U8A2O4P3N/cEUhO0JP5FLcbuWt3dErT75fI4NmvNxwOO8qs5NHZ2R98b2dGJ2dCODLeYiq7ybOcWE/TtdOPYECSHLNZlsRxrQVZngqGTCYpywxtJH3fpc8jFHmenMVlWbBYzCnLgt1+oO9HQgCt3qMrknvcGHPkFaYhzLoBHz1FIdEanB8Q1jM6g9E5kJAy1jl8eM8yTHzSvCzIVc44jBwOe6z3lKZmMpklh7ZriEA/jh82G1mm8WTHxww4G5GkGGCKF6ZBzDtBUIqiqFDaYTII0X5w+fiQNj2SLDnoVYZAfogOep8E3cxkqSC6bSmKA1VVURTFMQb7nsNtUgzXuSOGT5KZ5FwebHt0LnvisTjUmAzhwTp5FLPDcQOTNhhCgJISoxVGCOww4K3HaJ2K+aTCDykmqZVCCkWelxRFeYzwJmdLDCNSJI670umgASJ5ZnGFSyiXqFIckvQaGJMKfIIP+OMGEQSZyY+pUklRmFQQ7vzRMQ5SaYpCHofngWEYcG4EAkYaFBERPRKPILHdlRbkmWZSlwkf5BzWWrROSQgpU2+JdS4lPbQi5CqZG1xIbhiV8ERKybQRGUeI6UBHynSd1XWVNmneEb1PPRfDQPSC0/kp1g00h+YYgVX03ZgOZ6Ti5OSM6XSaOJ/KEKLHjr/nl/6xqy4nrG72x7nDoLRCaslkUVFlJbv7NcMwUBQZ/siDn81L+jYgtKbrOpy36EzT9k0qYBKJNamUIjeavNBYNxJ1+giRRpEXBj86XO+J1jNdTjhZLjkMB/bNjvv7O9r1illRcLY8p2l73PHAarServc46wk2RwfBsqzwhz2b/oYoIl4GnHJUs5J+17K6vWNRTykXFdvNPcOQrve+HxP2RBsm1Yyxc1yezdlve3JtqaoSiGS5pm1bFuenNLsDDB4RJCJIFPCbL77iZ//4R0QlWSwnZLLidHnJ7du3CBlxcWQyKykmBbWt6NqGpm+ZncwpzIhShv1u5MmzOftmYHO/4c3bHb/51e+QMfD5jz9l1215d3vFYj7nJz/5KU1z4De//A3NesW00MjgMFJSFiU6yxmB0kyZ6Am6kFgRaezIpjvQJWbgUfiw6FLx+Nlz2nFkdbNBTzK0UtTTim6/RReCXGimrmS0gbqc0u325BLOL2aMdmS1v0PotOFUSjKbTQghsN/vkSqJLFEKdKER2lPPKs4uT9kddhyaA1plLE8umM7mvL29RmnFYrak7zrub1b0Y8fJfMZsPgWZhBYXLUZmeDzt2GNj5DD06DwnSDBVji4z5vmU2cmU6XxGvawRCva7HbZ3GKUJ0WMKQ9OnFKcVjvnZAh8Fb67f8PTJA67WNwQfGcb0fXE6WyCF59D16CwnMyUqk0xqQ14ZVvcHDl1LZ3tSdYFKTkkrmVQLwjgymRTpcFVmFBcl+c4Q9UhhNOfVCSfZ/A++t00W+eTTR7x6+4LBtWR5jvMwWE+WGfI8cmh2nCwLPv/8GXZ4R9NuUdJTZwX73RbrBk7PDeBpu4bdrj+WJgf22z3OKXTWcHqeMZvWuAgm16w3t6mTQ6RS6sTfDUhDErDsQLPrKIspu52lqCVlqTg0e16/7KjrU+rJBZW55OrVFW7Q9IeMXOVcnCwIUdDsDpyfTtkbWC5ruustd7cHyiLj/r7hbLnliy++ZTIbEYPjk0+f0nQH7rc7nIgMo2fbWOZ1jlaB/e7AIJOYc3pacXe7wlrPbt8wXZRkhWaxmKB1yc//9pc8eGRo2pbZwuBt5O72BhEEd7fXaCJ5lbHZ7yiDYbmYYUxBc2ixQ8/lo5gSJqrAhRaZO67vb5lO58jC4seWzgt2qxFdQF1XhCCY1ZanD3/MV1+9YD5TVFXG5IFh6O8QHJjWSz55/pxvf/M1Rtfs+j0Pnz7GFTN27R2FqSGYI8It4ELP2UXB7c2eGEeQgv/qr37O6WmNziJGwou3rxmj5c31W4SQDOOWwY7YECkrQd8d0gFSZji0Db/79muaZkCa9OvC5tgQuL1vmM+n/Pk/epIO1Scz6tLwX/4X/znB9ex2Wz7++COGfkyM+XnFtD7hT37yhFevv+W3v7tmtxmYzebgJX3jKAqFiKkseLWytG0gywKnlzOmC4OQHm0M6/U911++5h/95RNW91t++IM/5fvf/z5fvfolt/fXZCYxq+OQBMMYUtLmN7/9HY8vz4k2sJzlmKzk+cePqGeS9eElm92BH/3gRxSm5uoqGW7QmulkQRgyvnn5S77/+YTJbMZyPmez2fGzn33OrnnDNy++/aO+u53zjMN4RPalPbxS6uiUlqkb5Mi57roWIWPiUsMHgdx7f5xNwgcR3ruRaNLroJQ6ltvHI1fdMY7jB462MQXiKFR674+Fpinp+J7FnoTScBTxw7FHSByFW31EpqT52QePtSNaK9TxO8N5i7USC2l2PbqOExIkJQrruoYj4u79rP5+Rhakw4RISl0IEvLLeUscLQBZZo6lpvLoRg4ErxnG95FTiVIGpfjAeHcupVaTo16gpeR7Hz/jk0+e48PArrmhd5GqzpDSE6JHSIvJwNsupSF1TlEqJtMS7x1v3rwhHsu1X758h8k0eT4leMlu25Flnr533N7ec7I8JTMVd3cbrI3M5xV9N6R+ms2A0jDajjxfMptNj+alAZOlDqnb2zt2+y1t21JPpliX+OhCHtMB4ng4IcQR65I45sARk54OnS4vHzKbLVitEhM9yw0yyg/GnffX5jiOx31BPIrXCY+SHOvJoJOSvXxICsfoyfOM6bTm5ctvmJ09QOmIGxzexmMpKsjcJLzmmNA8MQaiFyBUeuY+HstgQUkDaIwqEELjrMe7HisDxsgPwn56HHFMHMTjfZWY8FJAPB5QCeGRMiZ9w8uEplUZgr87Rfb3FtEVNW0fEUaxbzTPzws27Z66Ksi5BCrGsceUI1X+gHbYoSxoObIdX+N9YscZoRldS6Zy5sWOzrd0Q89V19Pv5hyuDS/fvCE3ijfvDjQHixcN968CJxcTxlGgcoO3njKvcRNLs7cUyqC04NB2HDYRhyFG+PizU7ZNg+5lKohCoMpAEIK7Nz0Ry2SecffaUj1Ozexdu6Wzjq7xFEZweqG5fjcSANeCC545JSEG+rFhelJz2HWMjYJ+xNSC0/Oa1V1DNnOcVznnlxN++tmCh9OcR09m7FvFNFswWEPnDI274rx8iLMbTJhQGs31/YLBtXR2y9DvkAFevL5DVS037Z6ZyLm/XSG8YLMRvHzbErzn7KJGhyXzRctf/sX3+eWvv+GjTy/INiVy/iXxNKMdl+wOvyPvnvHJ2Z8xco9kTXCGPr5E65JV+5+hpeRq+H9RVGeMbk/bdFRak6uWxspjtOievjlgvSQvpjSbMrGRPNRKc3+35W57TxQWYuDttaBvLAHL/q5A9JFATr+PLD72iFjgxgFTSpSa0bouxdHeWcrQkxtFewd57aiqjPYeFmc1O+/o/R63gdxMiCHy8eNHvHlzS6wC85OK9dUBgWE6kYQgGEaLLBwnp3Nu3xxPUouB7WGPGzT+EDgESz7N2K8cMKILg7I9UQiKUrNeR6qlYRx7Fmdzdus91TSdmIkxY38XmZcLxsyTzx03247HDy7YtVucjCiT4rPL+RQpJS46vAmYTFLkhvV1w/xywnSaE8TAg4cVt+8M1vWITNLcOor6j4OzzaoSP/bsbc/t+ho1Kwki0LuB58//hDcvXvLm7Vvm0wVff/kNi9M5hoyb1+8wEnQw7FZ7bm5uuHx4yTB0eAaGvsGNU67evabMZ8xnE4pMs9uu6NqACoo6r9n0G7747W/Y7rc8ffyY3ORUouZf/vO/ZnAjTz96ihSG6/tbJrMp/TBQZlUq8/OB+WTO4XBAOkVZVZxODTaM3Nxd07Y7fvftF+y6A+ePHlJPK8Z2jYwB2/fU9Zzd5sBscspf/uN/Dy1HfvGrn3N+fklWZOwOB5quY7SeWT0hesXZo4fcH7ZcH665vDyj3/RISnSl2a/vMCqnntWJxVpNGAZLphSbu2sO+y2nFyecX5zStQ1aKB5cPGS9HpgtFzy5OGd9e8PZdMa+2TBOE4e46ztCDGy3B8Yx8PjRcx4//Jj1ZseoAhLB9bsbyizj2fPnzGcLrtdrmn2LvrxEIHh3dU039Ex/tmTfNJwtT3BjxxiSMDWZFtze3PL5Zx9TZSUvX74lE4q8nmIU5FWB0JK7zYr17ZpZnkGIZFozmU2BSFXVFGXN3Zt7emcps4xhHBm6nrHpYXRMypL+0KRN8NDz6OEFwaXNz/aw53a/weSKB88e493I7dUNfdNjTIFCEYJld9jQDA1ZnROC5/TilF//5les13eUWcV8vmC9WmHtyPpqxd3mjt3Q0AwHqpu3CCc4mZ/iu4HlYobwcD45p42Rd1evefL4jGlVgXJ03rHbDeyaVMbSDS0///nf8Op3r/nk6SdsVytOJlOEljTdgSLLKLMcLQzSa54+eXbkwianfvSevCjYNQ0ZI2JomBYa/EiVax6cn/Lti5cUVSpAzHX+B9/bjx5dstmscN5ijErloDrFGQuRUY3lhyIhk2VYZ9nvd2S5xIcBhMNkAm0gYmnbIQnao2AcPEVRYoyhrqYURcY49uz3uyNeI8UJi6KkbfdY645u9py8UJgQsd4gG3t0UnukEIyjpe97dJ1/YBZa6wnvS2Zyc3TXC5QWhNEhjaIUJUqnDabSGdpYrPOM9sj2IzEAo1SMY0cIASVzpMjxThCDSwcKUpCpmiJ7jyZJpTlp4xI/MCRDIEVkR89g3wu1krKskTJheaJIQ+Y4DjTNgbqq8MEdWdQ9XdchpcAYc9wUQUrTahAjLljGMUUN+75PLnalGe2Q2ODB0XU+DbpSIaJAH0veRIy4cUAkPCKjD8RMUpQVUjn6bmQcHcbkaJ1mpmEYMTqj70b2u81xqJWIIh2g5HkB+MQPRCZESUjswXFwCdUlknNkGPv0nqnkaDLHQk2TSSKWvm/xIVKWE4wJeGvJDMQoOYievnd46xEiOYy0ACXisQs9IgkIPFmmyPMcpTTTyZz9rkWplNN1/sjZlJEYXBqYNfRjR55rqrxKvE5+jwl67xCrqupDmVLfp/fKOUf0grqsODtZIhS07QFnA0p5hJD0/UBZlpyenpMVBTGGYwy2R9i0yei6A8PQIuUfFyMrq4wsk4wupAJhaUBL5qdn2KahbTf4LAAVTZei7ovFJVWh8cId91Q5z58/59XrF6w2d1RVlTYacLzeA9YPoHLmi3OEktSFplt3aKXp9h2HfIezlt1uw7hbMTqLzgyj8Bxsm5IhQTCbTBAmsO9aXNPS7xuUMAiZU8sZYrxB5ile29hAlmlklriph/2BaGQqQhMaUxlCN2JHT5YV7DcDzgYW8xmCkRgCZZkhZSQvDLvNjvMHD7i5usVuetze0/UBWUx49+qa+48vqE405cxQVSXvbq5Z321YBENoOrbNyMnlnNMHEzabSNN3TKYz8rlmfdPw9ZfvgIKLxwsIG/a7NcPW43rHt+ULnn7yGK0Uj588ZjKp+O0vfsv1i3csJzU5OYd9g4sCqxyBBpUrRtEja9CVZsxbVFFjFdzs1iymp4jo0SEgheL08ow3375BO8lhtWN6uoCpQVxIFtNTNJqh6xmcpaxLxsOBeVHy0ZNn7A4N+bRkcAFnHVWRo7Vku9vS9z3ORrTR+BiSMzpXnD88Q2cK7x22t0xOpiADg+vSbGYji/kJb3ZvEcHg+p67fsMkq7FxIFc6peXCiIqByWKKFYFm19IOLVmRk9cFTd/ihaUOFdvdFkwqGe86x9CNGBkZ/YjuR3aHHVmu8aFjvjzBTAxRBaQSFFXJ7W6FkLDZbTECZrMpA54sZEhR4MVANc8ZfM/m0LA5HNC5IcpIsJHldMYYRnIEQmsmizlGC6LRXHzvAfba095s6fZ7bq5u+dH3f/wH39uvX93xD//8M+q6oncDd/cbmgOcnNZMKoXRgb5vedu+Ic9O+LOf/YCf/+1v0Aom+Zw8i6zub9GmYDIrmEwm+NAkDq0QTCcZ52cXfPGb3zB2wKSkrgzteAAi00mFdyqhwIInLwy963jfQZLnGYdDw8fPH3DoOxA9UljOTx/w+vqGf/b/uGWenzKtJN9+dcPl2Yyxd9RZxaFrOez2nJ8/Ymh39E1G35ojfq5AqpZfffEF3/veBU23Yrf33NzeEmVk1/Zoo9gcBkYvQBrCOHJ3ewDvkVqxOJU0g0XLxG4GuLhYoLVmt2m4vWl58uwxXb9htV7hbfp+HTvITyts8IgxCYvbbY9SsFxMKGqN8yP1NEvFiCqjaXZUdY7O0nf2Yb9jOi+QEmwMbPcrlNCU+RQRJPfXa/70Tx+x298iQsE4tOTzA5cPNNdvtmzqnrurFZeXJZP5CZv7Db1vExddaQ6HA0VRgPCYGJEqcnZR0zeK1eae1c6iS0GJoF/tsa4nL3MsjiozVKICGVA6fX9eXFzgbMbbm2uiDpwtznh3c40QA6NPCbdJlaN0En3LPKMqK4zW/Iv/6m/QKDKdOlOiB4Ehes3qbsdsajjsW77/2SeMfc/F5ZLu0HLYJoPB9PQUqXOG0bJar/j+979P3zeYjNTj4WC9PvDs2VMWyylt27Ne3fLlF/8pn33+EafnU4LscGPqlZJHM0LTevrOo6WmPXii80zPFmy3W1yAonTo3uBc5O7+mvl0wnxp2B/uKaqKq7c3ZKpESc0Xv37DZ59+zmYV+Ku/+oK/+HceErWlnJZ/1Hf3B+wfiYvufZov35cfCnHsfwmeYehJRaRJWPc+Ob611siQ+N+pnybi7EhmBD6kpOt7dEuMx4r44wG+MSm9916I/9fRMu8P+I3RaS49GpnT3iGtENI9kzCL+gNqJQSXekpCTMKzSD0+IQTyPCUT36M2pJAEAm2X+gj7fjwK7e/Tl/H4/3FEhniPi8mVr5UgxOSQTk56yzD0R5SixAdPZnKUHCmLlNIzRh/F+YR41FLhfeTkdElVRJ5+9BhtBMFGhEqpbKki/dCgTcIQCuWI3hIl5GVJPdMszybMFgX7/Ropc3a7Pf1gOD07xTtJP4zMZgX399uU+vaw3x+SWcR6pFB0bX/E3JRoQxLgC01ZZYQjssday+p+x263p9kN3N3d8vjRU5x/zzAPvGe9J3d4KpSNBIgSIVP6GFLJZio53bHb7nj86AnD0LPdbshNKpY1JvVhCcGH5KT38cN7lx4nfjC5v0f0WDui1ISyKhltz4MHF+kze3ufkDh2OCJoInVdIlAJAxRFSkwEjxQGpErFocIkvL9zCALSKM7OHrKYn9G1ferGGNeMoz+K4vLYzZQObfb7AyAp8gznhyMaKRX6Kq3RJn2uKynRRqNEjvd/tzn17y2iz4qMzlZIEclVT4iOqhBs+h0yDMz0OQt9RuPvkAEyBaUWxFhQZ4rgp0QkzThSKknXO7aNwtqO7eaSV6/vuXqt+ObLHZv7kTyPCOMIOtIdUtznsOvpOsd0nvPueuD81BCDoJ4qzi8kXaNQRkBWMFnkHNYOHwQxKCpd0dz0xJOBeJDsX3nkZKDIci5mc4Y+0A4j+3VDnOfUqma/3uBmsLsfcdaxu3MUE4W3kvVNiwqC3Tvo85FusGSVo9mm16uuA4tlyXw252zZ8+PPFB8/XRGdw4SSh5Nn3Az/HGkVQ7tI8dgO5Pg9fnf1hgdnH7HevOPFy1e8vRt48zINFru9I5/eYG1PPkZEMXD/diSrcioqvIaTk5rFckmlZpyfDPzJT0757EeeWTkQxC/oi6/BQywyrPwV3/j/HK9OGboX4M/YuRc4mVHKe3SQaP0RN7tvaXpNNb3gfneTOGfOk2UDQ78g+J4waqBh7EtuV++YzjOQgsOhRquK+5s9bZ/hY8PN68CkntIcDizrGTfrBmkMr760SAwXD2q6bo8fLHs7UGjDk88nSRDAIXyB7SxZCcPgsDIiM8nQGfarkcunA6qOiNxz+eiEZthz9fLAYlamU1YxEmxgv7GgPIoBVWhqU7LuLHmW4zrHIEZOZie8fX1PVgsqU2LjwP4wIIPCZhYRc2SM+EGx74+8VFPivWV5IYmhYLtpEZng7i41AgcX2VyPzBYFWku0hKYbwAnKhWJRVbjQU2UV6tICI3WR0YSRuxuZ+Jp+pC5rHjydpsOMP2L97/63/8c/6u//j279r/5tP4H/9vpP/pd/jz/0P////9v/9J/+H/6NPJe/z/r3/nv7Sf8DX//7f7s/fjarsK5lv+sSIy8ImsPuw+m8swMxegTJZeK9petGhjEQokWq90O+/VD8aa3DW4hhZBi6o5CuaDvNarViv9+ijTq6WBIjXClDCMnJIEQgLxQxCEZrcE7iXBoAE7IFrPO4EMh0cimFMOCcR2mBVCkR4aNj3xzS5kkHcm0oyhypE+YlxHTokZrqJRGVmNwBgkvoE6IB1HHoMkghsTYVruamQGcGFQLeJWd1cn2naKB3nqY54PyAi8OHoS1x3VMsNc9yMpPhnKdt92y2hmmYYoz64CJNzs8MqTJym9zZid0o6MeO7ChyK5UcRcmZ7hhHiw+WLEuHI0ZnSCMIWWDshyPWzmJDxAmH0jnGJG65EOq4cUrOoWHoUcoQY6QbOtq2YRhGsixjHIY0uNqINsnpnkpOjzx9+Xsn/zCOGCOP/HMYbTqcruuSPM/RRgOB0Xb4YKmrGVIaYoh4ZzHK44Mkxi6VBQ7JLVQeGfzGaEQcQacAVgiW0fbUTMhyQ4hQ1jWd9fgEfcfHgPCBGC1GH1nsUSWOokzvlVaKLC8YhiEVOMWI8wHhkut9tC79WgqqrKAqixRlJhyfd3LjZ1lGWeZMpjX1tAbUMbnRMQzDEXUT6IcGH+zRHfOHr//b//lXf9Tf/279W1z/CfB/+rf9JL5b/0Ndc32CcBE3jhhlmFaCPFacVDPyfMDZDYU2DF3OELasQ+Af/Owzfv3rb2n6kmEMPH74MV/+9nd88lmJzg1FVWBxuM7h+jXrTcPTBwv8WCGGCe24Yt30iKCpq5TKETJjsTCMzhN6RbeHqpBs9wOTOuft9WuMyLi8LKhPTtkqRVA580nJ7nbg1QvH3/zNHf/kP7pgf1iRUROGkXo2YwwDP/j+c67erpHe4yN4FZkuq+SUDw3zWUEMfZoBvMI5gXORobNkSuP7iOtTB4rRjpcv7vhEzLh8cErXDdjhgJYl06pGBMVud0AXEBTklSY4h+1H3OjI85Jd2zNfTNkNHTEqvAM7GrohsNoeEELRHBxetJxXCz775E/Z3DnG+DXOe8rcIKJGCE/A4kUkCsH9tqU0Z/z6b19DPuHx0yXr9Za8LFlveubVjP/gn/wJ12/WzE/P2bYHVk3D0+fnZFqimaBzTWdbhnFEktG3Hu8Cs9mEgY7JxFB8MmO/3+JbgVaRoiywvkMowaHLKYpTJouSbrVmFqaYNqHlGAw+7tkfrthva9p+w2yZc/N2oCwjFw+WELfU2RThIqO/4+HpJXfXO5QfCcPAm29eoOM59aTk5m7LdOrpww3bZk+9GDD1gZv1iqfPl+y2I2/f7fjs02dM65L5NEfGwOXJBfer++RmrbNk8tAHzs8nrDeWz77/nN9++5JfvvhrlocMraEuah6cnNA1G7wFJQVlabCjIgq4utlyf3XC5z/4Ifvud9zc7VnfRrKsJEbDNy/uOV3OqXLDTE7Z2I6zhxOEecbN9Q2j3fDJZw958Og5q/U1qJg6rP6Ildzh+oOA7X3EuZ7gwzFxmJKGfT/gvT/iCNWHJKP3gTzP8P4oYsaje5HUIyRQx78bjqnHJHp6F9O8HSWC1A3zPuGZSiLjB2620hKlBeM4MgwDMQaMBo7O9izLklsecZxjFT5Yuq4jHAselVZ0fYNUitENOJdc9EVRgoxYZxNiLEpG644zuzgyvsX7flKOJUYEAlIrTGbw3uLcyOhHlExpuiwrjvhGTz2pkbL94Hq31h0d0hF5FJrn8yn/+B/9I6oy8uzpU0xmaLsdQgSqqkBpi4uWgEcJSSShVmwAlXmqac7Z5ZQHT5e8vb7F2ogMmmY7InVJCIG2GbB2y26bEEZZlrHfN8lYIZJZR0iBlAFESufO5hecnj+iPPYLhBBwFg77gbvbPdvVgV//+kt++MMf07UN9SQlx47GeyTyeC0cXeLyvRtdfng9Y4RxdNzfr+j7JKBnmca5kaIoqOvyw74j9UUNH1zdMSY3d3LpiKN4nwpAORa+zuenhDiS5ZpHjy5Y739DoEcpyXR6yunJKXU9YbvdstneIUTaC4oIUeoEWpEQvDu62yXjOKCloaoqzs7O2e8ObPd3qctptET80VSjOD07IfhI23bkWcZ8vqAbdvTDHm/jsUxUImXEjzYV3IZ0QE78uyXyvz/OxTUk95GjMoYYcrK8ofYVeZ5j7cAgIjJMiWrNvHYMQ4NRc7adQIbIF2827LqGzdYT7QI39uy3G/brLXe3ktvbDe1toJ7mkBnarWRQI7nO6e4dKjpmS0OzijSdRzYd+ULjdpprN3Lod0ymJWu7JXiDmWXc3wYwkE8EXddTlwo/5GReEZ0gCMu27VDWsD+0PHt+wdWrHXI6Mj9XtJvI1lkunkxQk5GqEOk5nhq6uwHXS6pcUOaSvIqYPGd/A/Ol4ntPJd//acngPaenG2I8w4aRt1e3rDYrrm52FMpjx3t6O2c4vGGw0PYddfYvsNayWVve3O9wneb0oacsJLlS/OmPPmV91dI2BfPvSYTI+OT7C04fSdCO5eSU07JFyJ4/Kxd4fUXjIr275T54tPCgFb2b8WafI8ueLFRk8gotM/rhli4cGGPNTN4TmifI/I5hGPDjnL7fM0ZHe9Bs1xvOLuBupQghp+ne0TWG21XPZtUSQ09vBX0bGXpBuwePRtiQyp2ynlg6dAZTmYE7Njoj8SpSKoXbB+xEcNgInBUUmadZCy6eaoQ33N0fmE5rgpOcPZiAC2TGsW7uGAZPrqYYVVPOPEUZeffS0+7g9LJiGCIDjnFokU5z/65leRFZnuds14LtXUNZZ5QTweF1pDzLaMcBHy2bO0011WzuenIdKCtNoUv6rSafRw77lnqiOfQ9l8sZN29aTmYLbl9viPR45Rm2AlOl6Ox0WXB1vaXrdpwvFqxWe/pxZLY0DNExdI7N/T2FqShrxX51YDKZ8PbFH85d/G59t75b/+Ndxgim05LN5opu3zPmmhDSIAnQdUPiB7oRa7skKkaHcx1Il1zNMuJDYhgOw0DbdoioUVLjfRI52+7AMHbs93uGYaBSFe8dNu+jfan1/Dg0OwsiCfUJ+wHv7a9CaJyLR057CTEwZhBjd+Q8w2h7+mHE2TYNocpgdIXJ1DHO6FNju0jRWB8iUug0iHp/xDcpRDTHFnlDlkWKoqB3AzE6lAajNUpBjBZnHYKczBiUVAQvGfqObmiIwlOWJVKSeJQRskyxWEwp8glt0zMMjq4/kOWKENOgGqJFRPlBUJUyRUuDlx+cPUkMlmhtjuVMEaMNPfJDtBIExmQoKVMU1jqi9PjoIQRcGAlBYo1LBxAu4L1nHAecS0z75ABKs9Iw9sdioogPDucEIgRCjEiZ2OVZZlJCwIWjkyTibHKjK2UwJsc5x2Dt8fAhbVysGxnHnhjl0VmfJ0eUG4EOnEAbiZLJbS61pC4LZvUs4aSkxvgRqSRCpA6XYWwxWWLUZnlGWRZ040AI9shfTI+njEJLgdHyA1pIiQyTFyA1Uhms8/TDQBhdKmaSgdEmxqfWmkmVUxTlMUI8gA9Hl32GzjJMnpBciSeaXFi73S6lNY0GEbGuIyKOG9Dv1nfru/Xd+m+uV9/e8eBpxeYeTOlABmwveP3tNX/573xKM4z0XeRu12Iqjwvw9t1rfvInT/nNrxoO2wN39xnPP3rAN9+847MfXmCyjCKW2MESZaQoBK7zvH21oTlEnnxSwH7kzcsVi6VgPi3Z7bdUi5HRg/AaJQsiIKRgdCN5ERl2A3aQnJ0+4d3VG6bTnO7Q8mf/8CfQzzi7GNnuX7Jdr3hy8py2HanrKf3Y8y//+m+osglaCXJT4AAfXTr8FDD0PY8ezXEBmgF6C/d3PQKwg6Ozlk+ffspu2/Pnf/En/PW/+ucUZWS/7anrGUJ5MlNSFhUvXtzQNlPKSuH8yHSqadaWTz75mDcvryjynN0hlQ9LbSCAd5b1aofUFVVdkkoPNQiLkIr7+4aPHv6EL7/9kkDEu0BRFuTlBKkibbeiyFIJ+vXNOx4/mad05uiZLWZstmvqegJRcdjfs1pfc3K25Jtvvubxs0sCPSF6qkkSm2azmvv1jmk9T71ceXJfn52c8ObtisMxOVsWFc6NKJlhbUJcKJ2z2jTsmjUnWcGz08fE2HF32CCFYeh7lITp9AHPnl/y5upv8c4iYypeTYxtw3/x//45P/3pOW6UKJlQflIIHjx4wLz4nC+++s/ompbV+p7pCaw2NwR6mm5DUSnGraIfIt978oD9riEEy3xpMDp9b1ZlQT8MeOdTZ40RDGMHCH795ZfcbFYsz2cUdYkdO4ahZ7W6J88LVqstUqeSxH7wBHLyXPLtF1eYrObyuebiYkm0B8Y+YU3mixrnBoJSrG7vuV+vOfg1Dx895NNPP6Jtdnz74kucHXDeEcZjP+EfsYZhPM548ujAtQx9Mou8Txz+/pBdIoXCWo+SaX6XWhM82NEjBGQmkpkC79I8KKUmzytCSIWaQqTHEuLocj8aMt4L6CGkmTCEQFEUx1RIdkSfuGRGObqYkxs5CeGpr+j9Y0liSDOe8w6lqoTXywxZllIhznfHn5Mc5zH6VOQo1BEn6DA6CfDJQON+z9uOAhfc0ayiAY91CcsRdWQYUjmyEMmJnlzX7wXe36M+Ykx9OCF4Li7O+Qd/9qfUpWC+mHA47BEioqQ8/iMQUuPCiLV94tVLSXApkSOkpp4qLh/Omc40t9ctWhYMvePuZsNkMkkl28MhpWSlpu8HtP7965m6qjxImM0nWNdhcsFsVlFVCdu227Ws7ncc9iPBJWb3u3dv2e/3TOftEWc4HlO4AqEkgnRAltj7yfQilfnwXoYAVVXzvsfIGINzFi0FZZmTF9nx8CF156QEhEQKSRSRGN+jft4X0b4/XNH0Q0dR5EymM7x3TGcTbvcDUYyUxYTz8zMuzh8So2C12jKO7nhdeJSUaa8XPaMdcW5Aq3Ro471jHA9c3bxiGHr61rLbbY/dS5qm7RJLXyesTXLjSxaLJaenZ9yvLKPdE1X6fwnB0Q/uQ7dX8AIp6n+zTPR15wmjQKmeUQx4NUcFz2flU9bxLZvQ0PqI4ASF5v7QEcYCR8evv/YMO80Xv4YRT9PtGfc7sixHy4yrryPBOOrMcPGDnF3veX2943JxwjhEZucT8liw37V4qyknis6NiDydvg1hYFhZjCnZ3o9EK3n7y4HZA8fcLCF61v3AYd+hVUHoHZO54MHTCdfXgaY9UJkJDx+f0jQNUTpUbtClZjGbsba39G4g8zm1zGj9ljrPGQSIKrJrOkpjOLyE5aeWT35wzj/5px8x2q+oi5K7dz3/97+OlPktbV+z2V5hR8mL3zkuHmkOK0MILTbsqOYZoZfs95bzk4ogLNPJgig1ZRV48klOnXv+Zz+b4E3H2ObUyxrbTdAikskVIwEtO5alwPoTNt2IrnqutiNQsxCAcKiYM1rDvK7YtDs6H1GhQyIJQmNjQfSGIV5h+y3t6op+1BiWeJdzdR8hwGYDb990vH3pE7OIkmGAstTsD2Db5Ei73+5ZnqQm7HIp2a0bTi4zfFeigmR0AwpPpgydP1BWit5EmrVjtigQQYAMPLiYMVpHXiu6eEBIw6SYUNcV0Xd0ocV2mmELizPF4jRL2Ijg6FyG7RuE9oCiGyQ+OrKoiMGzvh8RXnJyOmX7rkHnhtxpxujotgPPfnDO1bsVs3LCetOjiETVU5AjSkvXBvo+wmgwC4HrSjbNSO4nDGvN+XJB1+9ZnpXMZqdcvXOI6BJGAEvY9fhhYHaSs9+26EwhrCQvCoYD9JsRKQyFyhGZhyDph5bzx8Xf91b+bn23vlv/E1o+9BSFRKpI024ZbWLeaSM/iOcxprb74b3Ih8f6AT8OSOlRWhGjPyJK7DF2KKnKirzIMUYCjrZtjo+X3ObvRWCtDUVRJ0F17I7umh4hPdYOeJ+EcSUhtc1kCDRKpUJzsoCSI0KMCBFSXNRGfOhBdEiR+KA+JBeMkgapNSYInA94H47CaI+IIz6m0iWlMrTOUTL9vDxL2BE1HsiMSQ7vzCBlciV7p1BSg/JIkaLaIabSJ6Eizo+EwSJEJM9zyjynnpTUVY0xhsOhSRzocIy6hvRnEQHrxiTMJ3s8kbTpFUIcmZPuGJWUH5iUUik0hjxPThWlFDEkHmiMHi1BC41AMhw3Gl3XoXQgxHA8iEhYm9EO5FmB1qnEyxh9dNanzZbzNn3fuYA64hHS80nmingsgFZSERHkmTmyFAPOJzyP3Ed8SDgY7y1KFShtkEIfn3dkGDqGYc/Qd8RgkwMe0FJiVMLBaKkYrDyWTyWXymG/Q6iOECXOJ4a/jQJr4zG6ntz6Wqci6uBDcsCjiCKR1l0ApAZpGG0HgFCR5NcSCGUotKIqE+oieHcs43MQI0IJODq7pIqE6BiGlFbY7TaE6Cj5/eYvlVp9J6J/t75b363/9vqrf9b8234K363v1n/3+u8v1Po/2jUMI3mesBnxiFUZR8t45HgXRXnsj/m9mJ5cy5osS87kcRyTKzgkDGBRlMQgj45kgZJHTroH23QoJcnznKqs8Udx+r2Q/l5MTz+HI17QHQtJ/Qdeu5KGbhjouoHMlGRZhpSJLR68hSiTocAopJKEmPYQiFQEb12aB60dEDI//l2PypLL3lrPaAd00MkIw/vXxxFxhJhc21lU+BA+9CVF5xDHIlSlNV3Xs9nuEqPb86GUMsbfI0+yzPDDH/6QBw8umVaK0R4xj1ojo/yAwpFaEL1nGHsE6njYk9z9PjryUnD5cMKDxws26wPWDwiRMQyeSINS8ShQuyOH+1iWGRJrvixTCacUgbOLKW2bOjG0gbLO2e233N5s2W0Hxl4SQ0amA6v7Fd988zXT2ZxmnroNstwRY8b7ys/35bTjOACKIi8xWQFItBI8uHzEp59+ztXVO/I8J6IxUlKUyc09jgNd12GtfX91HK+RY4dRikF8cNRLqY6pBMF0NuX8fM58PsV7C6Q+jSwraJuWN+M77OgTUitElFTILDvSFTz2aJAJYSBkElzGYAds7+n6A3e371AiZ7ZYMJmfsVrd0bU91oxICV9//TUhgB0jRT5Byg2b7Ya22wMKJVI5rB3cEVPmwAlmk5LF/MHfeQ//vUX0ECWTStJ2inl2Qts5OhW5H37OyTTSDjOCKHHijhevIv2u4s2bHTdrT3fI2N2+psin7JqRiwc5wgSclzQHx/KZoR1HZPQMQbG6bpABhnCAMrJe73B9YHqe03QDeZlT1IZoPH0XsA4GGyjPIn6vwEeKmULNPLvDDhMkudDUsqK9czz6fk02VNze7WkOPX7wRNtzc98wyTWnjwztMNLcey7PBbNiirfQSksWS5YXlt2tZdgEZOkoJwLhBH424p2gkJFvv9hwv/f03R1CevZXE+5u9yht6a2gzKCIFd/8ukNPBrJSEb1iNpmw7h2L05xHj2Z8+7uGP/+HS37wk4xy3uGU5rDtqUuNVx4le2IYkHT4MbK8uGM3RKLM2I6CSg/kpcILyflEEMacUimiHAmhYl6cE+1bvHcEl2MbjfOKIXoO455JGYCCttux2QRurzqkPBD6mtevD1SVZr0fCaMk44T7mwPSRJQOvPm6Y3lZ0qw9AlBRoTMIW0mWC8zJhP2uRwSXsC+dYFJlWGup51mK9OA5mU8JamT7zjMIxxAD9bKi2Q/024ApkpMyP0yJg0QGQ5ZJirOMrI4Qc2SQqNLT7wO7vefh45I4Rtp2oDuMZIWk7Qd2q5HFQ0O3tQzB0q1apnVF2zaEXuDcFfPZjKbpKKcWV1uqfMliUvDm7h4XPbNZRXMfyIaSXGt82THRE27XG+7agSIz3N52+F5wf31gdlbjZOJMudHhe0ezDww2UOuK04uSu9sVOky4eDhhv3Y8+fSEr796SzXNGJuOycnk73srf7e+W9+t/wmt4BJv3CgB0eOtx8vASISYfl8pgxDhWIrpQTiIDu8tox0RFoaxP6JOEtKE6KAq0FogZBKTrUuOqUiKnsYY0DpDqcTLdMWQEDECfLDJIe0SA1LAMbqpQWQImSNkhosS5z3WB1yMqBixIUAEpVNsMYmZAR8ESpdEEiImHF3aqWwpYDtLjI4QHVKByBVCGJTMAIkxgqLMyIMmOP+B5S2ERKCRR+5jEEksFSJ+cI7Ho6PG2SSAF3lOWRYURU5RvGeOR3bbHdYNCBnwwSWHtBJ4/75Q6Vhc6u1x6IQszz8M4OOYXtck3JIKK3USl+OxNEdEwfEJfeAfSpEOSkYXyAqAyDgOjONwLJmKR/57EprTJkqlOK5KYri1Fh9GtEmM6VSKqnCOo9s8xUbzmKN1hhCBYeggwmiT6z7iyfI0fGuTnjvHEqNhHNjvt3TdmhgTH1JLSfQeZy1DPybzhTbp8CUm1uNoB8I4ENGEKHA+figUynNDUaaZQIrwoWzI+7SzEjic1ASR8EHeB0brsD5FjgMJRySVIZOa3KSi8+DHtOEd+sTY1GnDiEjcUQh45xiGdHDRdS0IhzYRRDpISUW5/71/JHy3vlvfre/Wd+u79d36t7iyLD8WsScHd3J8J+RIcv0mTKuUirKsjgaViDE5IcDh0B5RiVkq+x0tdrRHRExKJ7Rtj/OWLDMkd7qgqiZUVUk/dBwOu2OJqDkWbsrjjJSe43t3bnJxJxNJluU4F2iaFuuGlP4UmveCrRASbRRRJjd4PzikUsd5t0hlnsEzuAGdGTKjceOI1prpdE7wsFptPnQAFWWB1oa+bxOTOyTcjXMhdd5Yn2YvwGQZIfkmaNuetu3ROktmCsIxDeuPzxNOz0742c/+lOm0QpEE1OSKj8mNriXWOSSOJDqbI+7RYnSWDEYhEKWjnmqePjvn6u2K+xsLwiClxo4eJx0haLz3aX48FrUKkQ4qsvxoOpEj9aQgxJ4s04nRLXVCj3jx/2Xvv54sy9LsTuy39TnnKpchU5fGdDXQaACDwTQGA/KJNi/8L/nMV9KMNJu3gYEGQtZ0yRSRITzC3a88Yks+7BNeDTQGXWYDA4xTvsrSKjMjI/zGFR7f/vZav4USthp4ptpntD8c+e1vvuarH/yYYRg4Py+EWN3xQinkjG2ss33tixIfCzbn16tbdHz51Rf8z//z/wsfRlrnaLsGrdXD5c40TbMLvWJ2ssjk/HF5Ph/g+P372FrNatWyXLY8fXaFNoK7+xukEpDq++rN29cMQ8CZlrar3W+n0xFlMtYqhCgImZAyI7WoyJU0VYyjAdcoliuHVg5jxZymnmaDj5hNXQnnWiiam5t3vH37jiJ6tAnkLFCyIEQtDBcC/BSh1FTv5cXl3/gZ/oOX6BurWJoOkR1DMMSQEXLAdB0lPeOwF7zvt3z9euBf/XPPcbtHicJ+Lzg/FxSr2IeeZCe+/TbTNJbTLnDcJRoFrAuyKPwhMN4l9Lkk+krusY3Ey4lpFCzOLBLFcTcxnRLtShJTISeIo6R1IM4y4yTJHxQTnnh0hLOJ9dmCpluyvT9gZYRFZDxm5KTxRiPLfFBiw93dB8iKYz7wZLnmeDcyHRKjjoQUmXxk3a1oziGKSDaZZdvi94LXvOH7d1tSsohU3Uwl1cj488/PefVrS5Y7vBrpzjpsA3YBLy9X/NmfX3PzveaTz7f88IuG3/y7BZ/9IGE3r1mbK17dt1xdKEb1hrXVtHbJFAyT6ZD6wGmCdbPg9aFHq0u2vkdKzbIZGPoljRP0cc/YC3Rzj8wTS6tw6QkpSYQtTKFw2EWmqTDuB/Y70KqwP2Vso/j214LTcUfnVtx8X5jUgD8JZOm5fLbh61/c8NXf2nDYJVzjaF4Kbt/vud4smE6J5Voy3lsCI1ooplTb4uMIb7YHzs4tohQW6YLpdCKEI33ynK3PiOXEzm8pk+bDt9C4hqwybdPRT0fG6Olch9ERocFqx+E4EUaBW8Kqlah8yf32nidPV7CNLM9bcsqYOw1PB9ozibagLCykxdJw8anmd7/YY4zlNCROQ6AVkpw6TtFzuO+ZpsLFkw3TNHHxmWU4nujaBfv7nrRKhBKxTrG+0jR5zftXey43Z+z6HeLo6NYS21jkOexuB55/1hG2gmnsSUmgbeJ4m9BO8O23r9kd92BapHfk/L8zV/aoRz3q/5AaB00uEa3XSLkFAikrsk+za0GhjWO5WIGCEEZinChzQU0phRQS0VeGovc1VumswVqN1oqUPDFVV3OaF70h1OHVaIcQanYn1CG8lLqsLCWTUyHnj5Q+hZQGhEZJgyiKFDLTGEgxI2aGX46ZrEBrgRTUgXQuiYlhgmxIUZLnWKuWNboomoYUNanUAqbKvKuHg/rza3y1tQ3Be1xjMdZWp5DIyKgqu1HIyhxUCe0KtkhKEogiyaX+uKLF6Q3OrLB6gdEQveB0HAh+QJBQUiGKQhZdC0VzJkfIqXK2Y4xIYXBtXVjnnJimHqiFUTmn+rhjwft6YUFRoEDqVGPxOZFTIqRELPVQFuJELqmWP5Vc9+3z//joSMrlIbZqnIFSC0yHcUCMCURH23YP7M5pqq7sQqFtuxlPU93YRXhS9pQg8F6jdIvRGoEjp1zLsFLPFHachi3jeMAaiVEaoSUpC07ThDodUK4WvcY8EUskl4+IGTNHUuUcXS1oUQ9VjbVoXYtjU4hIVYu/Qwp1aMajJ0/JtQD2eDjU4i3XoJVECrBGI4VEkBljIsVpZnH2CBXpGlNLu2KuRUjoynanHtpSjhQCwdcfizHM7/vHLfqjHvWoRz3qUX9MWnSrujhPCe89q+UKJQ1aGYZxIMZQS0BzxGiJVo4YIo1tGIaRZnajSykwuhod6i55XpTmgjUGKIQp4JylaRq6doHRtWS65DI7zqsLvm1bvK/dLfWxWIyuy9YYIznHGWGSWCwN1hViOiGkA+rFAIBSBaEF4xhIsZo0EIIP796gjWHZWfq+p2RNTu7h15dSsOgWdUmvBpTUNG5BToWxL3MJeUBkCVFTfEQWjRGV3177CyJZOva3IwpNiYna3y6ry7+AFKCN4E/+1g95+fIcJT1kVc8ZylPkAWWHaijKtd8RNEoYpI0M48jgPeI01CV98gil+OSLDeP4Of/v/+cv6r/vA2VGpQRiNeHM5x5QcyoyUHJG6cR63dRCdNdxuXnCNEb2Hyb2dxOhh+kYGPqJMNX3RuOW/OVf/pr/7h/990zTOBtv7NxllShlPncpizEtM4froXgUkbFW8uXnn9A2Gq0UbatoWkkpEyHUJKX3Hqg9VvV4UM8ShYSU9RJDUhEwWmu6rsFYxcXlmifPNuwOb/CxvheUdIQQUEqyOdMIEQnx/vcXOkkADqninF5okbKy0mNMONsgREAog2kacsp8+/p3iBlB03QSHyprXqhqvhJFEbOiFIExouJJEQglGMeRnAqr1YZONdzd7ri9u2W5Wv+Nn+E/eIl+c7ol2sgkLEVLLlvDMSv6bcNfvvmOf/sLy29+cyIkyekeUpCVbZpASs1EYrsLtKrhdOzR60JzLRlDxjhYb54wnI7YprD4iUS4QgkKi2N72mGt4XSMjCmQe8nQe3QjSDea5kwRTzDuPMFOSARhLPT7zPJqPjwLgV1njsOAyg3HcWI6jPWNtTIsVor3Nz1PLxXZnAg2YJXDx8zr+w8M7yQvnl+y3W0RSuC62u48Th4pKnvIKsNh8ty8z5y9jIzbgtOaLCaaxYhyG4QV/OBvrZFGc/cm8eOfXdOpJZ20fPLjd3z61Z7tDxyXzS2ukbg/F8QsGIrkgx9ZtgUfEtp8YIgNY0hsbKHoBbvUcDd6rqVDlCX9oEGsUVkigkGkllEe2Q2O7SFzXgzBv+fea46TZumuOewSUzmx23re3sBypbj/MCFFZPSBNzeB472lazpe3XjWXcPF+gnvpwNSCF6/vefqMwtKIFLiuBt5/uwJUzei2sRZ13F341mctdweRoywtMvI+9cTl09bfFDoqNFBMwXB6Rh58qXhQq44fPB8cv6M25sTaRS41vP8y47DISG8QljIg2OxaDmEd+jsKD6RfGG56WhEARFJbqQTZ8To2W09yIRbC9ZfWs70it39yP4uoqUl64SUkWmruHjpmA4JkTXdQpOmTDGJ6DNWWjZXEt0qzs4uePX6PX4IZK8xwnLsR0SWPP90w/u3R7oLCGUEOSKSoojA8SAQB8E4jTz7tLaQSye4uxuRWqEQjKFnuVScthNn5yukKsiVZLuf/tCP8qMe9ag/IoXQIlWm6yTL5UhhQsoaW3twJqAwtkOazBT66j6XH0uTagmlVhYlDX46QQat7ENxaEx1oMvZ17+PgZQibVv5f9W9INDazgvWiRwLMaVawFmYXdQGiSbnuRhHzES/nBGlFuWUFEHW5aNA1mU89esoaaorvCTyzLbTqjrCg9KknOYC05lBWEAqU9l+5IcG+0WzRnQSbSTaZFIe8SmBqmkhISTIQs4RZKImVQ1CGHIuKOVQLBDZQWoopbLKrUkYbRj6IxHQzQKyBKHRUkOO5OIhQ07V6b5cbCrHUVTnesWH+MqcjPkhHaDGjGo1QlQsTrGZoqDEQMwT4+xKUEqR0lSfezJCCUSWZCCVTMwJKTXIyulMuVS2ZEnkMkdnk8f6BudAqIqVMaY6dEIMjL5HjhmpCiEPIBOoDLKy6YMvKGGJQZLTiBSFcdoR0gGErwcX6RDS1OecxGE8kVWm6ICx8sHVnkodqptuibVLUtRMPhH9QCmhOopyqZSVDNEXjCmIIvBzgawQGuMDOVV3fo4RoxRWS9zM5dRzlHmcJpIQhDKSiSRGEBEhIzFCzhKtzmiaFUo3xHRAKoUUklRqca8I9QKpHjv+95WTPepRj3rUox71qP//UuWSC0quhsBSCk3jqhNaMZee1y4eZy1aWY4hcjqdEEKxXm/qnChrCnLyIylX5KsQGqVlLXSPhmEYarpQCIL3xODnwlFDLnFObdavaW1DrlU6tbDe1O6XGKtp42N60ViFsZL+NNbfDxqlDFo7hIj1jKEcWQlyzFR2tkSisMYhOoUzdl6ORmDicDgQQy2Eb5zDGMdiUTHAxkwVN1lUxaCkjESx7Ba4RhGC5zQNkC0la06HwNhPxCiraSLMxgVZe4ZePnvKn/+9v4O1qpanDmXu4MzkMoKYEHLuWC3VaQ/zjK0VOc59RSlV05EUrDcNX/3gJf/8f/lL9ncebRbVBS/quQSRa0JUSKSwpFjLjH2c6BZrjKnIzNViRQyZ/fbINCb600iYCnEK5JgQJaOVJGcYx4nf/e53PH3+lBAmXGlmtHJEz/zziutR/N6zIf4KL1zw2aefsF4vOOwj1tRuphAi0zQxTb6Wfc5WJylnnIv8uJOv86wyhqZ1WGNpO0fbKbqFQelAoa/9TzJDhhBrokAqSymJU3+sGB6l64VDiKQpUigslysomv3uiPeRs43DNS1Q6IcBrRVNawnRE+KEVKByLcNNKaNKmefsVE1dsp5HjTEV3aMUV5cX/PjHP+X+bsfp+Avu7j9g7H/GYlHLC45jC/pAKHtudgOv3wrevt7yr/9lYHub8DlgG42Pieurc253d5hGsnsfyWqCopimwOrCcDp4slf0bwrq88TBf6AoxRQT4RApR1ivOkIYWFwaxl0gm1pmdf3JiuOh4JbgP0hKK9g8ceAmgoAnF2u+/csT3VKzKmvs80wxJ2LKHA8T3Toy3UeGE2w2Cm0lp9MJZOZwHBlOntM+UpaF3XeFdiXRKvPm6zs4PyFHTYqCLiu6tmUcA0MOrFZLhnAgK0HxtVX2if0BB/mK9VKzPs/84MeWn3wu0c5x1w88WUtWeSB5iW8nhnigcQLXZY7TwHbqWLjEQjqOecTHAbTA5xNOJdL4lLd3O67ODE6OYBdMoZDSWXX7cWIcDUZs2B3uGNSBLJfcHwZC6JiOgt2dYIqR25vvkVrQj4EwKlqn+Nf/7pYnV0t2NwlhHbkoRh8RSrC+Mgy7AZsMV591HLZHNq1FiomQAhfPOkwjOJ4OHE8FGzViTCwvGnwsWOVIp4hOSzoExw+Jl58v2R4G7m9HNufQXSiUEagRbr8/0pQF/W0kqcDStISgOfR7Wtsgs+DzH19xv7tH0lCEwmI57Ca+e3fL2ZXh2ZMNoR9JZsB2CisaDv2Bq08X7PcT/X3PctFw9rQhHQs3u1sCnqYx2IWhDxPtqmAbxRBG1oslvhnpVhqRHNvdQLIJ02aahWPlVuy3PdJDKfD+3YlFu6DkwtXFmsWZJV4GhCmc+ol0KhjZYo3i7r1CO0FbFjx7usJHjw2Cw/7I8+fnHPoDOSvuPwxcPH/EuTzqUY/66/JhYt12hBh59vwpw7gDEjF6hmGa46EaIWVFl8iKQckpIsgPHMG2aXFugZIWP0wzB515uPazc70uXqXQSGEAQykaUAgUSimUFIQEOU+UDJSPxGkFpf78UnhAhSit0UbSNI6UmcvGAkIKhBSUKMmpIGYXu1KWklQd2FBY62ibFq0icY7H1kVymZvma5lSyeJjiT0CUYudtALhCWlECIW1gpKrGyPlUIfKUg8wEFFaodBopTFWz89HJsUMqlDjtJJcCtPkMabBGvtwgKmDrZwZ8g0mF1arM5AV9zJNfXWgl0QpdRivDvr04OLRWs8HKChTfhgSpZLVaz7ndJVWiCIq0z2mOe4qMNphbXVd51JmV1Ki5MA0DYzjQEweberBw9kGpWoUWAiIMVbuuipoIyrHslTGeuW51/KnkidSEiiVyTnQ9wemMFZEjqrlqSlnUk74EMgpoFRF5rumstqFqq79GOtFijEWaxxKR6IqjGN9XqIo5CQfGPMwvy7z85aSx3tPzuUhuqyUekhQGFMLzz463VOpz0ciorSdy+dqIlKIyitt2yVaN8QYaZsWax0+RpgTHlIqlPiIynnUox71qEc96lF/LAqhFmRWCY7HI23XImTGGDXPCXVRZ4xmHMeZTW5QSjJNYy1x1LoWPM4JT1SBUuc+5yylWIRMDMP4gGz8yD8XVFRh46pJpv4FU6romI8zsvd1kWqMIqZYixupzu2aMq0l9yBRUlfDRSjVma4NU5qAglEWKQwlK8gCrZYomcj5hBAwDiPT6NHaIqVGZon3A8M4kstI2zSUohiGyDCeSCnRdeYBRyOlQZmWklwtiJQSNc+TFcNXTQtKSX7+85/z2WefVTe/MHVJI6o5hxnnUi8RJD56fKjGDW2oxaYGRKkmIa3NnJ41XF6tuX5yxd37Wyw15Slm3MlHTk6dJQshJKyCEDzWakoJhBAq2nB/YhwD5NqDpI2iWzikkoSgSDnRdJqUM19/+1v+7O/9bfq+f3BQl9l9/1c2539NH1GNL1684MWLF7xKAWPkXCaaGIe6iBZCIZAP54e6lK+XAc51xBhomo71ellRLuuW9ZllsbTkMpHLRMkeoTJWK0ATk3+4kCml4FxbL3XmfqXRD5xOe7Q2SJkZhgEpa/lsKxtKSYzjiLUVm0OJ9f1oNWIu4bXW4mxHKQLvU/21i0Dr3xt/tFYsFgsApJIsl7Vo9cmT67/xM/wHL9EPoac/BD7cJ97stoynwulgefU7OOwyy7ViCooYCxfXlsNuYNF0fLjtWVtR3VZD5umPlpy2EPcFZQyf/NSR3ECz0rz57UCJtZjKFkOcKusIH5jGRDwpEIl7vWfaZoaQsa4gVWG7SzSXBZElu/eByxdrijwyHg4sO8vQJ/otOAf795EwJpSXFB3pfWJ5IZl2gt2NZzoWmtaQtMLpxNWnGr/PLC4tr14dycfEolsw6YA1EmzApRrbMFLz/IslhzcKHQUXXxb+yT/5En32hmftE4Q4YBc3XNgV63NDiXt0PoPmt4icWNr6pjTKkVWLVD0LBT4X1OQYpiO2bXHqE1I5gNghG0EfHHDP/W3D/lhQZgd5idSR++0Nn12dE7NnSpG37wKlLHj7eocNmXdvYBgMx5Ngmk6g642eM4bjjeSsa9jebjl/oshBc3YhgMxh5xnHwvB+z/BdptGJzUXhw/uRy0vL5gKO24nYW7S1tZSNzP4w4gcwV/ViRJ95VmvQfkkIgg9vTpx90tCuFHqy9LvI6fvE1bMVKil6P7FYWE4nz/TqHtlBP3l0Sby5/w6Fq83vTYNQsPhEk3eCjdvw6vstsiiefV5dbi++ctztMnf3NZWgjCIXMJ1nfx+IvWTddiituN/tOA7w/OlThJqgq3zTXBxt27JZNvgJlO256lp+95dHzn8yog+CYktdbJgJP40M+4n1Ykk/TRTvKCJSiuPw7kRS8O77wOZsyXjwLNeC8X6kP3q+/OqSr19lZMqo3pFV4nKzol2H/+Tn91GPetQfp3a7D6zWL5Aqs2gbpPLz0jKSUiSlgtGWaRoxQj4sdCtTWs2FjBatLdZY9NoSm4jUCaEi0zQyTR6pqtuBUouHpHRI4RBU3njOBWKpy29kXZ4LU4s81bxkl6a6c3J+OEBIWdBaUpyiYGqBEKB0qQcM5Sg5zMvtWpZKrkt1IRXOOax1dYE7eYQQOOdIqVT+tQ+EkKD8/p+HYSSkgrUaoVJlZIdYo4DaVcdOEjNf3UNKpBhIWSDIGGMxthYOhRiRYSKXGtstpVByIc6FTVrn+WsXlFK0bUvbOWJsCCnTuBapqyMJUct0BMyscpBiLsw0FZMDtfTz4zI/KYVSlWMZQpzneFkLn3J19idRKDnNsdKInPnoOWWC9+Q0kbPH+5FpGolpQqlaHAQFZ3lY7KcU8F7iveIj8FvryiovufIUY6oscilrYVVOAe99TTCEUA86JZHjRAgfDz1itgTl6hw3BqlBSIFWuj7WEDDG1ksSKeaDgKeUynavF0YG+FjQVZfkH7mfIGZ3lpp5lTNPXmqkqux2pQqCiFYLUhBoLQhTXw+YQoJ0WNugZL00qI9d1cOyDEgSSgm0UrNL6HGJ/qhHPepRj3rUH5OmMWJMQyl1gT5N4zzn5DrriEwhIWTB+4qnWCwXpFj3RB+LHmsxacKHWqYIiYIHYSofXMwGGVHLOYUQ8wIx4X1CSMlmvSSayDiND45grRWn04kQAlLKv8Ju18ToH+ZlKXUt7izUctFcsXb9aaTk+t9DXbRrVZGO0SeGPrPsHKtV+1BimlJdjCpV0TLTFPC+x/vKrG67FUPvEaI6i0sJM+O7IIWicZYcG3w0pKgQiNkQJHHWQKmz5pOnF/zFP/pHrBYrpKzPiVRzh5AoSFHIIiNkwTlDOI2zWSigTE0nSqEo8zlCq1oyqqUhecGzZ0/45tcHCGCMnHF+hdm7PXdASYzR5DzRNKqed1QtE0VElCts2o6S4OJ8zTRF+tOENhCjIub63pj8xG73nnE8stvdc/3kKUZbhHJIIQH5sLz/DyWkJHiPNobPPvuM3fYe78c6qyePn+KMnpH19ywVpdSUrFL1gqJtWoahZ7lcsDlboVTh/LLj+Ysz2kXBxz1FDCA8Umpc02CsZBxhmgZSirMZx1KKwM7ph5jDPKNLjNa0bYvWBmvtfNbh4aKn4kEFxlQTUH2PB0qez7HKkFOo6V1Vz0gfneqlCD58+MD9/a6afPyAFJrt9u5v/Az/wUv0/9v//Y7hvtAPmXHXMfkTxglSUOiiESvPdMjYZcB1S959EyiDQK0kuYOYCkJBOhbe/erE2UtFs1H0u4H+dkLanuVGMxlgMHz24xXffL3l+eUTdnwAmbArzbDzpB6apaZ/HZhaz+ZTzZMvG+7eTkgNXnuONxnbwfE24PvM1ectQk8M+4wYDD/90zNuPhxJodCtHa3WiCc9ux20T+pBWOhE+6zg/YCQiuNp4MnlBb3yBALFKG7fB5qF5ulizRQPXD5d0OYGcXHi5396xZ/8nRHh3vHJ1YKNHDh4zW6KvA8HjGzIOTLKhNORMfSEY8OiC+xOAWEcuXhuRsX2tufs3GLsOa1YEMOWk5f4fslxukfme/aHQrtYsl46fPL0p0IeFFfNl9zuXnPqnzBOW37925HDKbFaJsqQmVJi+05i1hG5ibz/WtG5jo3bsLi45eb2Dtd07F9BajLFRpRUNGKNPUuMvadZJC5WK1LylMlQTMvt9oiKCrVIrKRhSjDejmzWS/SVZnUJt28mZBfxfaR7kkEHLvsF+/eRnHfEIWFNQ7uWjIeIuRh48cUZRU2c7hLnFy37waO14Hjq6XvYrDVWLrh7NeG+jEx9ohDY+QORkavrM/a7ibNNZutPZFs4HQZcm2mWDdN+xPeKbm2YxoLtGmQXUFvHsyvN2+0NLz8xvP7mwPqsRaPxB82r4z1+SuRj4PDK0JmGGAIxFaRLHO48qghsUlxdasa8oz9JLi80+/vE3due5Vld0NAruktNkgkfI23Xog+K++OexsCT8wUqeLbsaZvCuvmb2U2PetSj/vjUjzsmv6JpNTlHpAQfAmkuUqnu80IIE1OIjL46xI22OGexxtQlaMzkDM41LBpFER6fTgxDTwgJXaq7IkZqASeWkg2ZjziXSJEFqQQFVXsc5j4aQZ6XiZpS6mL2YzmpzAohM0oViqjL4cryi7OL3QICKQzGtBi1IAYIfkTrymhsmhZtDKnuq5GyLlSV0ihViLEiS0JIFMZ68FATzhmULoTUk2b8TevMg0M5l4gPmiKoqLCcERiKaFC6cstDnGr5kKAuznMhUyO8dZHuoShKAW001lmUkoSgET4gdT0AzSaZediXs+umHkg+XnIwO81LljOuRjxchljrUErPS+vq9MilkJMixUQRoGTFi0hZKte7ZGIMpJRJeSKEsfLvY8D7Ee/rQKuUoZR64KtOp0iMAaUrO945R0yBFAEKucj5PTEjcUqsHPOUCLH+XCECOUhSAiENRtsa22wcXdfQNAaEQMpIyYJhHIlB4FxGSE2YxvmxB2L0aF3fx0CNAs/Pj7U13hnj9FA4BQqpatIhxUKUdalf48gCbWy9pClAEZQkoESKUBjT4poFQuoH1unD8l44Svb1tVPVLZPTf/FvCY961KMe9ahHPeq/ooxxBJ/mdJ7HOlvn41KXuDGFufumkFWDtQ3GKIb+RNsqlBIPKJSYJrz3OKcrrzpDyp6UplpuGVOdI0kzerG6y0suNc0XMiEmptGj5pQneU5lWos2iv40IJAYpcgxzkBFiRISKVQ14ORCipGS5nnOe6QDUcq8OBZIUU0E1liMNjOSsKL1SslMU8W2aK1mp3JFCRpjmILDhwEh63K6lGqWqHMlta+x1PNHSpLga0mrlGo27tT59ic/+RFPnz0B6o/54JFCk2bmO6LOsfWSwNU5df53IZSKMLQVBQkJrXU9W3lPCGq+cCj4OCGVrZcipcyJXT33O81J0lzn3344cnm9YbWxWGfmC4DCNAZEjhAjri10i4aca5IhA0MvSKWw3d1zcX5diR9riRS18+lj4ef/ltR8wFitVmij2e9HCoIQ0oN5qJ7P5vTCzJ/vugbnKgM9pchms+Tycg0icHm55MmzNVJ7Rr9DSA8iUKAiimL9SypBo1sa15Iz5CSwtkUpifGWRddhdE2CrpbLmgg2rl645EhOFRuklEZpg5QVPVod8onTsZ5PhdBobYCa8nBOzZdQlVc/DANah4dzwjhO3Lx/+zd+hv/gJfr7G8Pxth6ub349cPHJAj9Gnj11bLcnZFxx/+6OJ40llwG5qGDO49uC0AMxAjpx927i4pnDXIwYbRmnkTRInFqwXBf270cYCr/5t1vkMnG3vWOkx0fww4hIEm0bLi4WGE4EBdFn5DLRugaVDesL2AXP8ZRQBjaXLffvJ6YyQXAsrzRTOs6RF4mTivv3PSFnUOA2gmlb+U6xT4Q+cbluuPkw4LrC4ANXLwS+T7z4ymEby8unmt/9m8Iqd/x3f+8r7PqGH3yxIqRvaRaGRm4ZkuH26FmaTwn5HeddZj9pvr47cLW6YAiRjVtye9DEvGOtIPol3aLh6nzD/ekV1rQc+4kUDY39hJM/EOI5MgiW7QZrW27vRmJIZHpyitzsM/enxN37LX48sNtJbt5MnC5g6gVaGpZrxavfTTTthot1iy89x3KLVpalc+To0JeJm9ue4ZR5/mKB732N2CDp32dsrNGPq+fnbO8OLHSkYFicG1rb8ubmnqefX9dIiAgMfUR1ljBKkO/ZHbdY41heNOhOESbB+tzglp6YEq5ZMvQn3Ebhs+DTH2lSnNjdwfY00q0W7N5OLJxEZNBL0F1ivO+ZTgrbTAx3hb06cfm0YYiF4wmiGjnv1vTTCdVopBTcfDfQdYZupTgOB8peQ1YIDcP7zK/7I8uNZb3qSLlw/rTw6lXkw6uJT77aoNaGlDShHxBSwtQy+S3LjWXhNDSS7Te16fjOj/gp0qwFRSYaaZEbiR8mPrztcWcFo3uG5gCTZRgCT/B8c/OaqARPXsLo/+CP8qMe9ag/IpUysT/cocxq5mlP7Pbbhx//GMMMcWL0A6lEtFRY29K1C9rGwRyzzFnMC2RJKsxlQGmO+9XFLkXXVM8c2xSYGQ8zu1tURbIkX53X1c0MAoNSpi4uc5p/7VALcEoilQAlP7hFcil1eC4VzVIHRosQ88AqAkUohDQIqRC5oLQhpkzMEGLCx0JGVQydDxSfCDHB5LFGzQv2wBQqZ8/oBiF0jQ4ClI9lNwEpIlIZpKiXEolUF6tkiigPruY6OEORkpgzU6zDuxCqFqSKgtQKqxqksaRUESPTNBLnMh5rDUKA1h8TAm5mCYrZeQI5l9ll8fFCRD6UN6WUHjA2Sku0npf4Sj1gWYSsh4NcEmIuJ81zCoAZIcPsrBHi9xHVigPKxBRQsTIzjdFIJYizwyfkeomQYnVc5ZxIqXI5pVSVXxlrjBVqwiHlCWs7lssFi2U3HwiBIBj6CR9zRekhEVKR/AhUtqb3EzFGnGvm10E8OKqcc7Q4Jj+7rnxACKrTXdTCrRAiaXZaFVEg1jLUyk53NK0m6lS7A4xDqYYQMsM0EEKgUC9sUta/v0womUSe+Z+PetSjHvWoRz3qj0XW1nLNnKsTetEtUJp5KahI8wycc00kDkM/41XEvGz2syEgkktFxSld+2LiPE+NaUDPDt+6RIZh7Oc+nZpSDD7S96e6jA8B5VzFqoiaSHTOoY3CTwElJUorhmFOXGozY/SaeU4Sc4dooWkMQmSUAuHmuXR2wmst0QuL1uD9SIzlwfBhjGGx6FBa0g/HB8NPTJlhOOCn+IDbS9kzDB6tJSlJjsdIa9c4u2LRrJHiHU3jKDmTUsXcfPLJM/7RP/qHGF1RKCUXlNCEUp3tMY6EOJLKyDAGUh5JpSZfcxHVmZ8LQtbS0ZrerejEXAoIRYx+5trXNG1MFUddckFQELLiJ3NJWG0QYiSmwPMXl5xd2tp7FDylJGwLje1Ynzm0akgRdvd7hjGSs2SaEsEnfvWrX3J1+ZTD4cDFRa4GqYeJfXbZ/wdKMc5oFkHf9wTv6fseHxLB17m3viYWKQXej0hZcE6zXtdZ3E+BUhxX12c8fXZGKgMX1y3dUhDigRCPc0loRmqB9xO73Y5CYr1eI6WazT0CqGeR4NMDMmccJqRINE1NeJYi6hI+1I6oXDJKG0pmxmRqlKpISmvtfIakLtFLfmC65/k9oVRNFjygLlWlbXzsz/pP6Q/evPV9pF1pbr6OaCvZbQdcVnztd7URd3cApfjwNiC2mcMbsMbinODiWcYfHcNx4jQGVDvSTQpS4uJpw7tjQCwDw5h59gPY3yQQie7CcPtNT3Oe6QeFsAW7EEglePf6QDGBBsO487y/SwglyelE1A0+jOQkaBaOg59QNkEvKGQO+4GiE8Y41nbJ/sPA1ZM1NzdHGAV33wxIr0jXI00RECVRFT756oz7tyOfPl3xw59d8v3Nr/mLf/gVy2XD5fnE//SPV2zOJF27YwyFpnuHj4Gbu8ini5Hvjx1OgjQZJyVT3nHwiak/pz0/0Y+SKW0Jac3gE7tTRJQVYx5JMZKS43YbUKLjeJA0tidRSwDCmNjdveP6acPXXwv2u4mUe5pO8+2v7ji/2vDtr0fKYHjyWcez6zVj7tnennj2mWK1kaxWC3KQtI1gvC8snzmmo+L6yrK9T7x+c0BJg3OC7kJyezMSdSYlz4sfL7g+u+K0u+P+3YmlckwhsWwM97sjJx2wqiX7idNYOPVHpIt0qxWqc7R5BSXTinP2+yO6izw7vybKwOrM8stf3HN+lSjFEceJ3X5g7a6J6p7Lzy2L2w1tB12rSBT29z2qhelWoxvNerNk2gVGXdgdR6bg8SMUJIvLzGGcGHvwamIKiaeXDpUloS84Z1leaOIIx2nky59ec3N3w/LMsH8/Urzm23jg7maiWziaTeHwYWSKHqXAOYMQgetpjVvC1ZXm7ZsRlVrefrPl6kcdWUbOFisGP7HfesiJ1XVhsVScxsDb10e6laFxaw5pz29+vYciyUUwDj2NfIyEP+pRj/qPSAZiGjid6qA3jCf85Ikp07iulquk8oAxSTlijaXrlljb4lxbh0MjSDEyDp6pZGIZGKcTw+BJCawxSKEqeqUIpFA4281x1ep0j2miUNDa0bXUcshcULIWMiqpqRboeeE8O6tzqdy6GiOd5rFQMk0BJR0laYq05KiIQAyFUiq7fBw91tZlPEIjZCblwjgFfIhIKdFS0bTd/P10IBUxc74VKQVKBqnV7Lqo/L1piky+xgMRUERGioLUEtOYGie0FbNCKYScSBRiro6fmGrzPLIiQZQQxBzxwaPmg4nS1Nck1cLOlOsgrvXvMTVKaZRU8wFhdhBRUEXhXDOzyKnOalEPAMNwIvhEyQktBVhNzgWjDY2zc7GQglw5kRmQyiJkmbEtuj5vs/OjMiEV1lZjQgjVqS6kQiqLVKay7RUEXy9USpYVsVMSOeU5cln7Zj7mEx4W9CJjrGS56nCNw1qLMWouBvWAfCihEkIRYp4vAipmpi78xfw+DBhT2fFaGxrnQAYQsXI/TcZaS9M0UOoyP8ZMjIWcCtJIhK7OKqMVjbUYpZHN7MQSCpAPiJiKiakJjpQiPo1oI+uC/sH5/qhHPepRj3rUo/5YNAwDIdTyyqZxtQ9GCkr8uPAUc8liwE8TXbskhDj3zvR1iZwiQkqUEnMqUJFCxbZ8TFwKURe9Q9+jtZ1RMImuXQB1tqoOXYnWjpwzIUyE4GmalmEYMNHMaboZr1g+IhfrQt+6OhfnXFGQ0zThmspu/z0iRZBloRTPMPqK6hgSIGd++xIpJdM0zAYbiTEGNS//y8Mslyml4k/q71mSS57d1hIlDV275mxzyXL1FpJgs96gteZ06vkn/+R/4IdffVlZ8gjGcagzephAJEIamaYTwgZE8eSSa+dNdVFURIiu856a+3NyruWV4+CJPmNsTTNaq2sCloSzllC9QPNi92MXUpkZ3ZJuYTA2UYRHmYxzjsY6hNAEn4nTyGl7IuQeITVDH9FGk4rkl7/8X/mzv/P3OZ1OTH6COZ0Ldeau5p9/f5Fe8Zr18dx+uGUYhtpHNBtwKjZSY201T3mfabuGs7Mlz19c07YNr998z8ViySefXXP1ZMnoM8uVxMcdU7/FulwvOXJ9Dvo4cTqd0FpVg0yMeF8d722zql8nemLyOKcesDdKqYfeqtpdVF//mu0VSCmIc9dV01TDTNM0NUkbI8qaGUkjZzTlRIyZxWKBEGV+33vk3A/2n3WJHqLH0tA4y8ULwfF9ZvU0MZw0m/Wa3f5Ac5aRvuPDzZHmQnF+Xd1Kr3+XMK6gtGB5BVkZcoKxjwxjdTM5pzDWcdxPKFfwvSCe6kElJ0WOiaZTrC4E4TgSIiipaJcSu9IkLzBa0rRrxlNh2DdMJ8/Tzy27/ZH9u8Jy43jyU8Pb306UnJEUaA+cfwrWjDzjnPcftrTRsTgz2EVhPCWWK8FSZP6v/5fP+fVvD/zjf3BGs1kxZri+zLTmyOj3dLYnBEER0DWKkDXDdEbrDhQzsj6T5KxxsueQBLsQcI3lxeUZ77bfsuoc3m9I5YCzgvtT5mwRePNO8PRpRCFJcsAaTUqK7S7QLR1fv77neN+x31nefOghN+z3juMY2L6fON0njAW5dOiV4phOPH92xt3Wc37ecton1peJy68C79/Ah0NPvrMMrWa7HRhOkfevTzgnac8dx8PI4bgnJYlsCpvVitM+EPfvsRpOMfLiXPLm33nuzAeunihOBzh/VnDdBUHvCdEiGihkUCOnvQcKzdWBpdUcdidOac9x6zkOEmng5ts9T1+sEWKkHDxZRIaTZCwD+7sDL84M8SjZXLS4laTLlsWV5e6Xiq3vKQTWlw3Pv9jw/tUJudGIrFFW8c3rW37086cc7gNsYbNa0ucD001gmATb+0QKmWZp2H/oOXNrTscdTdtw9yFhrxLFBPSZxI8a0wZ8X79phRFUC9dfKWzTkEIhDpH9Yb5BU5niDTfvdzz/UqPOWk7HSLaZpOqS53jbM/USWTLWGu5uDmzO1pxfS969vodV/4d+lB/1qEf9EcmHgWmSCJFmznXl25VcI35iLlmsOJA6uEuhMLqhdUuWi1pa3MuecTwRvK9L3ZBm14kkz/+stUBJjVSapmlYLesiPoRI32dSqiVFWku0a8i5lhT5qS4TxczidrbBaFu50lIgi5wPGNUF4+eIZoqCWAola5SUeMGDGyYXhciyLruniDIaIRU5VgdPLWkqNI1FGwtlLgfScS4OzaSYiLm6yWtxkXoYfr338xBWncu17Kkuv42py/aYci0jKgJyjamO01QfY04oVYc0+bDsrV+3Yk7y/GOzw5+KS8k5IaXAWkPXtXNEUTCNgZzrMC6lxllIs8O7HqjE7C4veK8IZYKckQKMVuRUMFpitCTFQJEZoxU0lhArguejo8Nai7UVI1MZ9nLmXRqKETUSGyMhVDeMi3PKQMgZRQMxVbd8Lml2flukypQ0EaMGAh9LOLXWNI2eD1VqvhiY36tS45yizIidigPK5JIeXh+tf89EF6JekHRth3MtWitiHmfmqJrjnKZe3qTq8EnJE0Iip4JuDDILSkokbTHSIRuFcx1CKgqFLETlVYrq6nfWkXVF6QgK2tRDa2D6L/vN4FGPetSjHvWoR/1XVymxLltlTVuGWPne3k94nxGy4uamyXM8HudEYYuUui51c6r/bchYW/EnKYkHJJ3RjhR5wHLkXAsu68wqK65kOrJeLdFa4Oei02mMaAmu62iati46fZxNFBGjGhZtQwgTKULjGnL0iDIjYIpACpDoumTW1eTnJw8lEaap9u6QCVNCmRZrOqSqWBBrHd5PhBDo2gU+eCY/zoacjwXxAAIzdwHNQU+sqeeYzeaCH//oZ/zoiz9FFLi+vmIcR4Zh5Oc//xOUqovUXKqBY5oqwiWknhh7hAwolRD6I14y1+4kBNa11TiTC0LX+Tx4z9D3lKzZbNZ0XYMQhZg8Qmq6rsH7CSklIRayKLNLXZBSBFFYLFqKSEgDVkukEFgjUCYjci02TSJjbaFZSFIwaLNg7CPvP7zjcJh49eo7nj37nN1uy+ZMz/0+AqXsf/Q9+DEhq5Tk+Yvn/PKXv6gGkwQxTiAybWvrBUR/wDWS6ydnfPrZC549u+b27j3nFw0vXj7j08+v6gWAinRLS0iJPE2AQkhJSTCOnlIkq9WacRwY+pFu0aEzsyu8FpZ6PzAOB1Ko54xpGOk2HVIaDocjOUSaxmJsLdzNArRRpBw5nQ6EWDsEGlffv9MU5rOnQitdLzNK7QZQqjrhfRgB6LqOtnO1m+Bv0B+8RL+8tAy3ls2VINmR5ZXm+6/HuSzpyOkwsVhI7DKxEoYi6hvOrqArmicvJPtdJgyFTz5Z8Zf/ekebWkiZ9XUiCk8jDLlXDD4iRMJmcK0hZs/Lz1dMvid7TZyj1jILsohIHTm7ahkPkIKnO4NuaXj/PnO3G1itoZOW5kxhu4Q0hbh36FVLn0a6VuEax/pqAutov7L8yY/P+PyHC37zq56f/mRNozxPNoHnn0C7+oZ1d8WZMJzijlImrBsAi0eQisCqCVsEXVu46Qf2yXI3vCf4zNI9I+aCki37U41TtGbDepHZ9Ym+L4yngZQMU28J/cB+HxjCgeGoeXU3UtKyxnmS5Je/nWiUYX3u+M2vB5brid0BwhHitkGgGMeMdInpmLhYLHn16yPZStLB0l4WdttIFhPHHuxKIa89aINtM8N4YHPtWGwUrWrYHQ4Me41sIp9+es6wV2zvblGl49mPOrTLHPcHrl82SC3ZvYsIGRkHR2lvON1FFmeRD/eZRmVUMjinON0F5KVgfbbCOMHbb47cvxowZ4LlssEKi1wExvvE1fUK6xxTL2mbE+KJ43ATSCUwHhxuAZtzy+2hx7YFgsVrSfCFaR/JXiJN4snzBbsPkqdfbKDUPzi0TOwOI8o1THJkvWppwhnvbm6RSXLqR9IGDvuCViOX1y37D4ESDOMtfP/mRHcVOLyXNMvE2YVk92FEGM8mdxzvRnRT2DzXnF23LK894ymzKR0x9pU124MI0NmGdScxWfPy8wXRZzKFcaVpN4rzs5Z4yijl/9CP8qMe9ag/IilVl5o+TEgh5+Zzg9EtdnbuSplomnqjP4UJiZoXio6uXVVHs9r/vu29FFTWtLoFeOAygkDoNC+TFdZpjJlLf+ZiohTjg6tAiETKv3dGfCw8MrZDKTUvhOsi8mPkVUlDyRPTFEgJclEoWUt0CgZQ1awwW5FjykwhYBAoXR3X1c2T6uLX2Ll0M2G0pTiIsS6yfZjIJSAkD49Fa0kIhRDDHLfMSKVQurLBQZNSBio2RaoebSJSaCY/EVOkIOaCzxllkitdMpeZHV/KvDxX8/Or5q8r50JMMbtDLMZYci7EUGO99TnUKGlmFEkg61jLp0jEqP6Kq8ejZL2AkNSDj6gPDFEq8sTq6ghCQMo1vovI9bXVBq3MA4NQUBFlIY5zaVWgEDGjwjmBVnZ2GJX5EJgp5NntIioP3evZOVU55FIIpKpDPuSZBS9md3iaLxwKQtb3z3y2mp1U08MFkZqLb41xGG1mvqid+e6R4FNNFcwXSSnWA1O93Mj1oIOoyB4FMVcsT8oFKTXauPpakBmDJ0VPpl54NI2bLxUcUoBS9fFV59Rfj9g+6lGPetSjHvWo/+PKh77OcroWmRth5wLQuvSUUlDQdF3mdDhyOB4AwWKxJITCOIxzaXuckXUC16wpuWIG7cwM91Oc+c+CafIIIVksmrmcMRJST0HPM3zAugbnztDasFyuoAgOxxPjeJgxI5VHPo6eQiGq2injfWIaA123oGsXhODnaapy0CmJFCJaWbTR9P2RYehxDXgBi+WKnDLGaqyFaZpo2yVwqkgPo4l/hdH9sfx0nI4kH9FK0rYtOUpevnjJn/zkJbo0LBcrxmFgGE4MwwDMPPiSoYAUYnbwT0z+SMzDnE6s+MRS8u87lKTFaMsUBorIqJyrOSnXWbttm9lhXVAa2tbx6adfoJ3il7/8JUM/zY/bVDwmglwqBma9XqG0QkiBFiBEAgE51bNK27VolclxoimSrC37KbDdboHCcrnk17/6DX/6p3+fGEN11rdtxcggEfx1YsHHJbp1DU+ePJkNMhYfquO/6xrOzjd4PzFOcH5xxg9/9CWff/GSYTgQY89XP3zJk6cXLFaVw9/IgmtAJsnkFSFGyiiqiSnVTqqLi2umqa8XOtTUrkWhtCSmiZhqAmCapmpukoqmbUkxz2kEquFomggxILUixsA41h0aYr5YKQUpzWzC4uE8qZSm6yo6SEpJCBWfo1Q16Rij55n/P60/eImeApxOgcsrwetfe9QqIo3g8gsDMuC9Zhwj8nIg7BVKQxKZjGa5NhxPPbtbKFHy/u6erivIkEBFdtvA5aeSYz+yP0TW54aY4HifefZpw91+Ynfj0UvLammQTaJJIAiA4fAh028nhAG3TEgcoVecnRvGqWCkwhvPZm057uHqOvNP/+kL+l3i3dsljWv50Y86tncfGKPgk89afviTgaU98NnTDtcd6GRDlO9YlRXKZEpakHJhYTeo0vNhP3DeKQKFYZSU9kCjJXGYUGbBPt4T8gprJEPYkyJkeU8/XFCmEdHC968PHPzA/f2Kvg84teCYE0InPrzRbI8LUooMU10GDPvIXR753a96Li4DN3uF61bs7jOlB2V71mcd7UXH7nCEYPA+8u5VIibPpmlpLjMnMdBZS//BcNYuWG1a9uPIPp64uz9xdqYZvYdJEsqJzXlLGAvLCxjGyNvvRp5/sURlx/44EXzm0592RDFwuLHYJoItpJPhw3cDi4XCx4BKGmMFUmTUsaVZRnISbHc77m5G+kNELy1aJa5etJxuA+9fTyzcGVdPFtzubinLSJocyhnyXrE89wiVyMnydnfPYVs43kc2G8P5umHjNpz6LQlF2BWmdoFcHDmLS3Z3O5qF5smXjmGvuNvtWC5brDW0qwz3mWapuPEHFk2LuGlYP1W4NcixIb2PaGkJ8UjHNcFMKDmx60/s72A6Fk7tifMXhmapaUZD9pJhl6EbOdxnplOkawTPnl2zG7aUSaNUIMfE7dcTYxwoOmCXmt1wx7le4hTcHR+X6I961KP+uoTItbySujyXQtfFp3bU8VbNA0pDt3Dsj3vGvnKbBYqmWeBcQ/ARrU5kHSFFhOowTmKsI8bqiCkl44MHxOyA70kzA7qUyteOKeBDQmlLiGFGeUis1VjjcE2LsQ1S1UKcUhJSVEzGx/hjSoXgE6ChaKRyaNWihAGhEBSULPOiXhBDQspEEZIECG3Qrl4gGNugjKWQEMqgrCOGSA61+KbGXQ1N67CuRlpLCXMZam2od66p+A8UIWTGMVCnc03KR7SxaFPdzcZqSnaEWFfOISdyyCilEblQhEQZj2maikChFo5qrbHWkIuhYk74KyzxTMp12Y/IyPky4uMSPmdJyrVk82MhT0qRnAKCuqCunPo6nDrXUMh4P2GMpWk6tLHkUv9dzhFt6lL643tJSolSpR4AQi1/jSGQcj2ISKGQTs8r4+qCSjnVElMp5ksW8Vcc+OrhcKNUXaRDfmC5p5RrIVbMBB9QBgqClBLDODCcjlAqhz/nRBISa/SMB6qXF3GOkU5jwPtUPxv2I/9Rz8Ws9bktJaG0QeuK4DF6duJLiZQaga4XCUJUjEyOhDQ9xEc/qqYYEtPk8ZOvhVWPetSjHvWoRz3qj0ciE1MkZTFf8pvfG0mMqYtQn2ZWuJxdxYXj8QhF4FyLdYacA6fTEe9nRnoWpChhNpgYUxfOx+MRIRLL5ZK2bebZcSDnEeU6lNYz31rjGkfJUNN+Col84K8Pg6dpHIgyp0ehECk5EsJIzg6jHYv2nNVqxatX3/H+/pbFokVKWCxaILPfH2pxPJLC73t1Kru9JgulNCjl0LqZUTMVxVdTkfV5ibEmLYe+x+qWReO4vr5m012SJ4EScuazt5SSOJ2O1dzg6tllmhICQZgGYuorHYBMEQkhIYWCUq4+FjkbV+a+oJTq2Wcax9kYoTFi+Egdoeta/u6f/xmLVUdKkf/1F78kJ/mwvM6lzJgSz/WTKzbrC5ToyUWQMvUiImcWyxVNs6QkT4j1wiSFTEqZtm0xtuNw8PzyV7/i3bu3fPb5V6TkKUSkVOTkkQ9p47+utm0xxtC2XXXMDxWJ8uTJE7quZbffcq7X/Oxv/Zi/+3d/DiLx29+94+WnT/ns8ydIlRinHdomlA6kMqE1GKsrsz3karqxDU3T0raOmDqOxx3jOJByxGg3z8f1rAIFay1t29K1awSw3++rwUqpyrnPEUrBaI1QIGRF6VirayltjhQKwzAhpWK5WKONIiaJ1hpjqtnIGF3f09RFe0VS/s1J0T94iT4kT1+gHR1nF0tu9jsunjuuzlvut5G2U2yu13zYHTAu0SwEIhumg6C5KEy9IW9BtDDlQGgLpAElJNHWbxKH7wVpbxi6gDOS7BXf/boHJIvzgCqO735x4NOfO1KOaJtYdobjTrA5ayCBzArZjzw56xjLRDhG1FHzD/72E/7iHz7lm3d7Pnu65vNPAuOQUfkK3fRs2gvC9DlR/QJrWm63knaR8cuBIW+JXmNtRpsPgCSkgWO4p3EBcsH3TzmUA80yYwz4qeU+Bd5t95ydtfhxyYdbjZVHtFhBtiAkKWl88uy2I+GoOYwrTncd7Znh6zcj4+nE7nbCrQq9F5y2cHXtmMYDN28CV9cWJw2Dj6SDoqwjMQpEL3j6uURLzXGf6ftIf5z45NMzjseB841j2AaEzTU6Lw1aFtqLzGF/5PLlkn/zb+7oloLNVcPUS3zwnNLAk2eGfpvxOZKQnD+THIaJ1ghEyLgNHA6ZcSikKXHyASaQubJGE57kJSkJbt9PvPhyxf2xZ3mtGdOB0weNwiLNWHlg2RNCYPQTQnjsUvLLX26xTrPfB0IMrJYNi1XCOM1wCywFfijI4lnYhhdPl7x9c0Jv9pSiGPYjRIH5YeT9fkvaWxamoXWabgPD8cRnX13w4fuR9bmgUR1Pf3BCBM2LZ+d0a81mM1KA7d2AkpLNpQMpOIbIKHZ88vkzvn/zhjQKfvi3lxzfJ4bTQD9m8jEiZKBrW3b3E85LDncjq9UahyEcI/0uUMREGiTIxEjP5Yslh8OecQxMXvL2t4EUC+P+kav6qEc96j+unDNKKvSMHNGqLpDHMaCVmheVGuPqBWf0czFlLqRU0SXxoam9DrpKarSSGGNxzj3wClMKxCQJYcKHgZzTXIRU3dUiMeNgqtO7FmU6lKolmUrXks2PcdMQErnEB+5hSnnmLtZBW2mHUpUZmBE1SirrAG6dAaqbowAxV+cy1FJOoyv7uiI8ysywTiipSVIAE7lInDM4Z7HWoGYndHWDSyTzgtu05CzmGGoiz0VEMUGDQOmP3EdLThU38rH4sj6mulBOSRNimEujwKq6gFVKobTCFkvOcY7y1ognBbwPD+xLrQJOdA+lOiFUvmAI1T2fYsWKfMSNAJScCSH+FZe+glIv7JWq0WAh61I7Rv/w86CWb5Yi5wuagpJqXhx/LEUdMNoihEYUmKaM9/X3IFWkpn3LAyO8OrEkav56tcg04EOg7/vZiSIJMRO8r5c4McE4ElNi8iPRTyjxEYuj0BoogpSq63+QI0rFOQ2QUcqyXDSVESkqxzOEML9XR0KImJJQQZAT2PlwKqWulxUCQkogK/tTSOaLikhJuSYCciaGhA8T3geC93wsZX3Uox71qEc96lF/HHKudusgavdLiBVrG+alcCkV6SLndGGKc2fOjK2Tyc+uWkXXLWY+uMY6Rc4fk27VFW2MeuCHN42rC/eSUAq0kgQ/1HnTNqTkGQdB8In+NCLmsseS62xz6nsoeUZv1LVsThGlJc4apmnkzu94er3h5fOvsLpDya/xYcDoQmNb+uFISYLWdVjTELNg8hNK1p4c7wOlQE5lxu8t58dvMUoxTRN+GlGqGhOUVkxjIKZQ8R65YgJd4ygpEVM15FT2uCamiueTUtUi+pyYQo+QCdcaRFTVcCMEKQVcs2QaClOYWCwW5KIJKTN5zzgM1ViiVUUgUlCivm6ffPKCTz99iTKSH/zgS3732685nQIpR6Q0lJiRwDBMKKVomo4sMjnVeThGKKmwEpYUYRgiOat6TkLTdoahjyRfMZW3H275xS/+kp/+7E9YrJaEOOFsUwuC/j3NhaO5vvestYzjWDn2J8lq3dI0LZ9++inD0KNt5vLyc37+pz9ltWm4375nfdZwebWh6Qo+DPg0oIWmEJh8mo02BeccKWViyChT0Y0pFYZhZJo8wzBWZGWpDP6mcfX9nw2b1ZqSQUnJ8Xik73u0Mvjgcc7iXEOMHq0kyina5Egp1nJQUR5el/rbr2nclMSczODhvNG2HaUUDofD3Dcgmab/jEv07a4gnUAuC9YILmWDayT3d/dcX674UAbGMhJ8xC0FtivobDizjiJH7EXm9EYhGhgOEqOhbc+5vdlWNs1yQZQ9F18KjiOkosimcLYGdy65+S6gtpFmIVBtJg6Z+7cS80TxyYtLEiNXa8fL6zO+/v4df/Hna4ZpQLKiiMynLwM//lxwdlWYwntU6vj0/CnK3lD0O0L+Hsw5x+ktjbzgyfmXjOVbVDnQqRXHvmfdekY/kcslJX2HlI6YM40+4GxHyCO3HwbGqbB2DUoPUDq+/u7Es0tFNzuGx7BDCfj6d5mpBBqtOZ4UWkZSUgyHe/q7yP0rUw+kpeXb3w4kWzACjvcjyhp228Jm5Tj1nrDL5ByYYsY0NWaNbvjFX97x2ScbppR48rnhOG1JSFCJixeO41CddtNhxCwCN+96LroNp/eZ5592jEfPcKoInP3rwvWngtffnthslogo8GNBaQle0OcTSoKYLK9/17PpGp78sMPuLLffe/bhxJPrDUlCs5L4oydPhdu7Hq8Sz87O2X8YOH6IdBsJOvPyc8vN60jwIyknfM6YY6bfZcyVwO8yohO4RpNSQMuGKfbgBw7byPWngvfHwHEbkC5wu+tZdWd8+qMNi9WCbI9sxIrfbnekmDke4Llaszo3xBM8Ob9EhIn98cTxLiOl5+Unaz7c9Cw2gvvdERsv6NaSIY0gC85JTCMIokfZQr9L7E8HLl5uMMcjn3yy4tvvBvoPMI07XGPxvWe5aSk5kjzcniKqONCw/zCQdCG3HnmU9PeJs+sFTz9v+HBzYrfvEeEP/ig/6lGP+iPSYrGkFHC2q0tqabG2QQpNzqcZgVEjhca0tE3ED4lSCsMwcDjscc7SjydGf0SQyMwO8lhdvkpLtKmR1LqkFBQhKNS0mRQaVSqzLpc6ZMX4sRld1YW2qegxiiCmhKIOODHWmKRrqosip48L9IosaVyL0e18KZ2ro0WA0hJjDaVIUqku9BIlOSZKEsiisNrRuu7B4eOFp6QJITNoicwKgUZbjXYaZSQlxdmhIhDCQJaQFDFUJ3WKguDD7C4CjUA40PNiWRSqgx0ooi79BQJZBFo4jGrQwpBjwQdfCytT7Y5RSpDLjOoTlTmec6GUGt9NKSGEhqKw5iNvs14k5CwqGzPWBb9QGq3TjI3JIAKZQAyJaTohpaVrVwhRf1/IOrjPGcmKnaEQUyQHj1IJZ92Dq1wrTRSSlCGGukSWwgMZP3Pqcy5IkZACbAFm1mUpiZxnN7+ItZhpTORceZ7ONmhdB/FxjKRUiLG6+OvjBVEUPoyVO2ocFEmKE8YA84K8lBrrVFqhraExFWmUc6JQkJnKAc2JEHxFx0hIoiDbetFjjEFaA1KSSkEWiZIOKTwl97U7IAZCjCQglYqCSSmTC2j1+Gf3ox71qEc96lF/TLJGkZMkxlBLMo3AmIoBKWUi5UA1gQSMkQQiUhaWq2bGkVSjhTENbdsi5ZKUEjkPKJUREoxRTH5AAU1bO2ma1jCNiRwl0wTLZUs/HiqDu1H0/YmS+zr76kzOghQL1slq8PCeQmax6qp5IAd8H0kJnOuYBk9KJ8z+juO45+x6zc29YdrumcJE6md0hrGgVDUfiIyfeqBy4IWU5JSIcaqmD1GNFEoWUjyhTWW4708DSoHSprquha7GoEJ1J8dMEgUhM0JlMpFcIBdZv3YcKKoQxchY7hnjDmki2jaQc8WCSMnkT8z+dKZQ8S1+nJjGAaMEsSSclrRWUVLAGcWPvvyS//F/+KesFktyiWxWS6wWjLKaRyiFIjxSFyYPPnpCGoh5hxAVUWhdU7t5EvgwMfoJ09TLkeN+YN8fcd2i9vC8HxDS8c/+2T/nn/6f/s9cX1+TY0RYULJ2J9Xl+UfTpUTIaoKRUvHi05ecXa3AVOTPixfPaVrLdpvYIPjBD1/SbiKH8RXYibNriWl7hrRD2YIhM6aI1oIsHT4Wpjj+vkfJKBCRXALTODEMIylXY4s2DdpYmmYFQqBNxMzlsuM4sj0cqrlHC4rIIME4gzYaPKAyIU3ENNTCVxzGaKbRo5TGufrPp9OJxWJZ+5syhJhqujhDjIVp8mhlWCyXONf+jZ/hP3h6X7UN+9sae7Va4FPA32emobBenwghEL2kWwkunhrGwXPcZsY8oISnaQtnLzXaCvqjZneT0ZcnFheFhQa39FwUx8WLlsvRse1vaS8jyzZzd5P57AdrDscIAq6Xlqxb/vI3PbjMP/yLS26ON/zFP3hBpwtn//aan/14Tds0NKbjOChcdw/qe86Xhv4YUFpy308sRP1ACHVHpw2pv+ag3uLDB5xcoXWin7ZkIzhMhWWXOfYjJa659wek9oSxMPlXlKQxJtEPmuMhU0QkxAkjMoeD5NtXR3a94m47Mg6F119LXn5iOZjAt9+P5FDorEHT0vcWf8pIM9IYzZNPOt7fnhj6ugDQZAiK/X3k/Lzjzfsj2lbbm2nq4fTVd5EQC7v+hI+Bu3d1AXt91eJ7MCYz9pF2UTgcE9ZkxjtNT4/RinGM+Klw3Hs+++EFH8qBt1+PtGvL6T4gjWH3tufJjxJDiNhkWbZrZChcXBWEjLx6fYdA0Z1ZuG253255/sWScRhxq8JnP17x7/7Vjq6x9P2J/iR58pXgd7+5o9Ga/aFHpYbiV8SwRSTFerVgfBFZrRRSW6IE2U74e0EKgvW5o4jE4jyyXKxRzwXH3UhMhdV6w9Xlml3/ge/fnrj8pGHKB86vJctOc3yfkDojhWYIiUO4QYdISYavfvySf/n/+ZazJcQ0cb/1pJRpFpKmUXy46ekPkevrM5SNuEXgWXNBGXt0ybx+u+NiY4hp4t2biReftxzeRfb7gaunDTkpUhpgEdmct5BhOBZW2XK5Ouc2fKCzLaJpOe4DRdYiuXXXsg+PkfBHPepR/xF9ZAeahkW3xrmuLtBLQVv7gCxRujqHjXIoNTKOI5OfKvtPBsawJ+QTOXsQCYkmTJFQJqSsjpakZU34uA7rGoQWKAdGGXKQIAtCFWKs5Y9S1kSSD1Qshqj4ESEzWVTHe2UHJmLIGFMHZGsKAoUUGqsNbdMglWKcPD5EcomkUpjizMmWgoIgRyhBzbzvgkRhpMHoih8RuZBUYAq5ut+tQWmJ0Kp6xmV+KAXNWVCSI4YCWZDTvABOkpIl5IIQGVUyImfqJhhy8pA8pUTIkEstZy1SYJVFZksJEqk0UJhGTy4RKcGIypXPIiNkIidIKTONgRDrUlZphwoOM000aEARsyQmgQ+FcYr4ECmpzMmEQhEBZQJWRITMhCnPzEmJFLIicoSkiIoyEVoSZtdP5WiClQmZM2LmuzvTUlJGzAz9FDNR1N93dfwHUszIUkmNRhqMkgRZECSESMSUKVSG+ORDLXP1PV27pHELcpbEWAi+QFFoY9DSVlZ6qa8Rs5spzi6kj+5yZid+2zZot0RqRVFzWqKU6nYP6cEpLpFEn5imI0kUVNF0rWexUCijyIJ6WaQqBkbkQImGEAtTqJdOPkRCKShTS2e1VDjn/kt/R3jUox71qEc96lH/FeWcRWtBmJN9ztXenRwySgukkpxOPSlFtGpoGleLEU3lSxtjGMcJ7wcIHkH9d8tly0mm2aFraBpDLp62rWnCbuFQCo77CaUaYhTkomm7ts6dAjIZYyQxBXKGjEIog8QhdUMRtY8op4ifBqxtGHpPLhopLVoaTuOBf/Vv/wUxevphV1fQJZGojvnlZkWMmWkacbb2w9QkpKBp6tdIqaYnKwLQI2WCMiK0JhMRovbSHI4DMRXG6MljX42kc2JRkFEKpKpO5CJEnWmVosyO/1gm9qf3HMcbmlahDcSUkTnVDiPijN+ZOBwDWpjam5Mj0ig0uZazTgNWd4gC/+Dv/T2+/OwLEJLt/paPTuhqaglorWnblilsAdhu98AVPgxIJUgx41yDkJJhrGxw0zimaSJnT9MZnj2/YhoN09TPz5HgzZs3/H//xb/g5YvnXF1ekmJCqjInE/5DakFNr0op+PyLz/nBj7/k9s5yeXXBctnx4cNbXm7WSBVR7sRp2mIdLDtN35/wORLLAQN4n+j7HmctC5Z11o4zT95UfIqfBk4DeO9B1HSE92HmlmtirMacGAK5MWQfGYaBkktFzjSG4+GINBJlNUJJjHKUMnI6nfBhQms7JxFqKnkcJ5qmQyoBoRCCp3ELhmF4wOp4H+tnxS0qJumjqetv0B+8RA8jqKy5/X7kyx877gk8feLoT4njqbqkhIpIpbj/4LFOcH87Yhfw5Lqwv1X1BugkCD2EPlMuE9MpEyKIMKKs5HgsfPhmi2kU51ctd28PXH8Ch9ue023iz/58w9//u4kSL7l0Lf/tf7viB59fIt2CiR1nxvOzlz9hpX/KLv0/2LhvGTcd744KlGZlM+fmS8Yy0oc1qbQM4VuyiBT3lk1zzdtjjbLj9vSTZ9me0x8T92MmKUEaRoyR9KeeJCVvbzzLzrA9Fc6XhsNOYkzhNCX2u8I3vwl89dWCf/lvJ3RRSJUIUXDYT/z2tyNZSbYfBM5qumdLvnu948nTJU3S9CdDd7bm7niHDA2LRUIvA2GINMkQ5B6ZLWkSZA+jSGTg+gUM28jnn1lubxJxAm3h8mlHtzIcPgSIhcVG058SjYPoJctzw+I80u+OSFVoltAuFbvdgWc/akhBsV4uiSGjjWB1NiI0JN/R7yYSmSHsCTGxuIJPnp9Dttze78ntiEow+R5TFtjrHYfhxNnKslxbtE1oEXnzm4k0FYSF+9vIdbPgsJuIJfHiK0fIPSUkTsfC/lawOFMoK7GNot14hj5ydxNwjeTm3T0XV0vcaEBODOGeV7db2nIGJ8PpPZTc4hYeJWHYB74f3/PyswtefGn53Tc9olTH4LHf8/lPVpxtCvvvCiasaNYTx/EDx3eG+zcBawTTOKJTYX8PWQRizmxfQ/tUkIPg/RvP+WpBurfIIvn0S8X9qaffJRZLyfpCs7s74azgwy6SJ2iXmpIr//WzLy753e33HPaRkANf/mCJNs0f+lF+1KMe9UckOWNcrG2w1s0ljgat9Yyq8HXZqyvzUEpFOzpOpyPjeGKcFkw+E+KID56cPUqBkIaUK64jxDgjNmrsVMiP/MRUXR8EhLBYqxDSEkIhpUgpcS7FTFA8Ss6oFavRc+HLx0Eo5zSXLkEpDiUrg7q1DU3rKv6lJEL0D0VAMc4sSaEp5JmXDsysbWvMXBhay4OklBVFohSiKJQWGGsxupBTZhwncijEeTCkQIrVIRNDmp3iBXLhY2WkQqClRM+ll7mIutQN1f2hZHVrpxgpTUFJMRdi1ohlEXF2Rof6V6nL+n8PxUJd+saUiDmj5mHQaDfHVw0h1PLPlAolgxQSJUHKQhH1dVeqFmuKmTUP4INH+AltHFpCHcLzXFLqHzicOdf0gpK1vFUIiVIamWJ9H6RMFBUjU//b6uuxxmG0ro7s+XFUHroilUxO1RBUO34iOQ0zlqVAMaRYC5OUsqgZQaOVIZdYh2ABoAAxM/VLddZTY5wxJmKKqJxAyHooS2n+XNTSqfqeq7HhECIhBVIX5mKqREmRRL14ENTLH4nAKEMREu9jPXCVj59HOz/GjKvW+Ec96lGPetSjHvVHopwD2igQmhg90zTRLTqapuF0OhJjYBo9xljapkNrQ51beJgDnTOE0FNyxcEYKx+wit5nhJMYo/Ch9vGM48D+sCNHRYhlxo9knHM0TTvjLwyLhSMlOOx7um7JolujlGEcT5Qs0HrGwgRPmHraZk30PUYvcLatc27J3N5+YPIjbWswRuGalpwz4ziS4ljTmymgVUOd0XhICKZYCL6mFqVUs2EjkWKPbVqKUFhrGKepFqnaBSJaZFY411FXnAL4PcpEIOvMi6IwByvJ+DASwkBMlekuRH1e6rkmz2lV5q4cSB9xjCJV00YKGOXQ0pCT5OL8mr/9p3/GanGOj5H98Q5j1Mxyr+ecnDLWOYaxvkZvXr9nGr/AGIexmmM4knPtsupPA0LEymNPIISiWywoUTANE6fjiWEYiTFijOFf/at/zT/+x/+4XoAUUEJDmZ8LAdWRLuZZXSCEwDWWxaIllTXdopDyjinec31+jnWW4+kepQoZScwK5IT3I1KBFAZKfvgaJUPw8QExWTGccca49BUd41p002KMRqn6niqkei4skZgKOdWC1JIL2kg0asZ7KnIJ+DHNL2wkhvQwW3sfsNbOvUehXkYI6DpXPxcz97yiSMvMRTc0TYMQsiJM43/GYlHZ9oReUJLATxkfEx/uAgrJNBru3weaDsa7hG0lbqkQSRHHyOFO8P4XBXlW6K6gKLj6zOFDZBgzKUKzLMSQ8YcTMSr894XpGLj+QnBxbvjv//Qr/tn/8jt++EXiT3+2YOgl/80PrzDdLZQTiyWYLGj1e2TSNGbNfb7h7ZAYQ8t5Z3DK08cTA98wJYtPA6Ax1rMfRpCSjRpoxDlvj7ekVEvVBj9S/DOMTry6+ZpF1jQms783mIWgpMirNxPLheLNOxj6yLvXkeOkeP4y4sn88jc9xMT97UiMmW7dkXJmLIU41huy6Os3wZIyWiVGE4gy8+7mCE3ik8/XfPfdDttACoLDh4Qyme7KovGUJrPSK548Twg14tqGV/+mYM8Lm0tJYyszahgFYx4wEuTgONNXHLhDOsH7u56zNqOyJAVB4zRxsOzvImExslotUQb2256YM7axWGVwTtJ+kjgc3mOd40ysyWrP+5uBHAPffb/l8pnk9LrQri0Hf6Q1BZfOOX82YHzLu+93hJxplw16o2i1ZrVeczwOXDwTHLaa4OHN7ybCkLh8agknT1kF3vw28fSzjiwS23eK/dZzfm0xqqNTV8SLe/bvQEtH9plgPd/ff+BMNqyfCqxS3L0fEbJQgqPfJi6vDFMYKUqQwoBPEZTgt1/v6fSSIhKb9RWvj294/qkmYbl+sSCfHDe39xQjOG0nrDKgJo73hfZakwfD0hn6DwmpDMkncq9wvePudsC1BVE0UjienBlU0qhNYvv2SCw9wbaMvSelgtZwcz9w2qY/9KP8qEc96o9IQtTInhC1hCVPBSkDWltqcWdBzo3kQjAPHAbnDDEF+r6W8CBidRPHRDGVMSdSwYfE5KeZr2iQsg7Fk6e2sUtZnRC5okO0FrXoMjEvr6thpOREnIc5qXUtohSCIuqgn7KcG9YFtewoUorAOosxEoREhzqExRAQSWIsCOmQov76hRpnLSWjtUbrinGpboSPkO/6NbTWKKtwTiNEJMYRP02zS5mHRXCIYx2gdaGYeaSaf82Pv2JdrIr57wVKGgQJow3OtXgf8ZNnGPv58dQLAakKpURSDuTiyaUWBUGsz/9cZqmVoRhJyj2URIoTMU7kFEAZBFQHv3Y0tiVKBdkjSnXFUGo8FApKQpaFnCCmgI6BEEa8NyANQuT6XOY0L8cjfnacpJix2qJnTrjWdubY+5m7X8uzTFGUIsm5YKTGWj2XmwqE0EhhgYCc0TwUoKSKbMkJIQKCWoqac11cS1HIsiAplQtqHLlUF5UUgkJ5WKLLGVFTI8JQUuVOxhgpUtT/zx9ROfkjPnLepOf59axYnpwjPgzAxyRFnpf3Aa3FXIBUD4dGWYQGa3R1nMV6MfSoRz3qUY961KP+eJSyRwsH1FlymjwherSuy9+mWdT5RSqapqIlKu9ZM00jx9N+NkkIlNKcTgMh9oQYKUDTOqxRxBjIOdZFvR8RJVKypRRNoZpJjK1LxmnyCKEIITMOE957lktFLpkwjnMBu8WaioUsRaNkpmSDlA5rWkKorPBu0RJjLW303pOSJM1z8zAMxFCd1tZ+LLwUFdkXKvIvxtoZ5P2cTIwJYwTTmNBGzB1HgtNpousWtM0Cldecdy+xtq24xVQQIj0sij8WpZYiSCXPM1xgGA4ok9E5o3RGKUlMczdUqn1QMVRXupSFmMdapBqrqSjHQtIZrQ2H3cDPfvZ3efr0GeSOfDrStg2LxYLNZs1u19dlfJbkJFGqRWvF29e3fP/9Oz75opnPZ3Vn13W16P546FEqUs1KBqk0wdeuHR8CwzBhTH0uv/nmG169es0Pf/gTjG7JicoJfxioq+pzUfBx5OLijMuLc3aH7zgOByCw3BSaLtJ0hsTM8fc92+2RxbJFyNqfpLXDWvmQ/jwdB0IMGG0BmEZPKbBarWqXkx+r6UYJpAJtBNooUkyk5IkxIKREKbBWza/ByDDURKrWBkRiGsfaA6VrP9FyuSLFyNB7xPzzck70/ZFxHOi6BTnX3qOYAvbjRYuoHV0xenyoZp4/RH/wEv24hcW6EFTBhwNSF3KAvi8gAm1n6Q8R2WVw0G8jgULeK4bvCu5JoVmDXRWs0ihVkMeWuJEM/YRZCHwPOsOzzwQEwX/z8ysWZyd+9rMznjbPWdnAz38q0KJn4yzavWd3KjXqGwwxjUwqE/OGb07/hiADObcUsaM1J3I5cRh9/calHVJd0sdbnFzQdCP9SdJnzxhGnF1QiiGleyKRD9v3KJnoJ0NRG1IyBL/nV994UjBsnnqGXkNyvHl94OuvMzIZTseRMQqEguNdZrwp9bx6ltg8W/H2/R4lCsNBkoRHfUisrhJCD+QiiSLhQ6H0nl/d3lKAtSh0nWGk0F0Kzi873rwauXrpcEKxvxtxK4OxkWZdkKpgly2n+4RQguWqY3PR4GPPeJf4cPsGdy7JQXN6l3mnBJsrw3rRcNpmfMwoG3j6mSFOkTffez777JLhdGI/ZUoQPHvR8fXXe9brlv0pMA07VEpcP0kc7gqLTnC4AWt1/QbYKUS2tCvP3XvPYiE4O1f4wbA7BcZjYvEi8+b1hD2DYe9QseP1tzuWy4bl5ortacuLL9fshi2XLy2n04gdBVkXlssFTy8v2O63yMYzHU5oa7g8W/P2wy3pOHF20bBcNaQduKcaYzwHsePixYa77wb6obBeL2gXkndvDvhxwrUKZx3TNjIxsP13HtNIdvc9QcA3b+5pgsMsqov/6bMn7O6PpEnw+v2B/W5kcwVaWtqnkrubieF1QLYSdwlLc0HnKh8tkhFJoFvJ/WFLs3FENfLrr9+jVMPlpWB/nAgnQVaPxaKPetSj/rqmyc+uXUEImVJkvdWfCyOVUrRtgzG1QHMYehC5chMnzzD0OGdnELcENCUJYqqIkhgS3leHiEBVhzmpxkCLQKaCkRanI841lcGtxMMC/aMToswObcTsqo6pHiaKoIjqTPm4cFSK2dmQZze9mNl+ocZFs4ckkEpgrUGrOjRRavwvpYyubZZzeUxdrqZcS56UkkhTo4NGS1Keai9HmC8rS21wVxoKcS7aFDMnXTwszIGH4sqU6mAOEqUc1gqstRjjEKJiUY7HA8fjkaapjMtSAkprSqlu6RDHGmlVZXZvzK5nCVJWd1JOkSAkMYyUHOZFuUAisKYy4MdSDyAlR3Kpf3YoaeoyvoCS1cUvEJUJXgLB94ChzpYJQUZSEFTnfc6RSGW+a6fnBEF19gthZlZ6vRyplnYxswtLjVlSyKnUhXOW5KQAi5ydMwKPlH7Gw8haZCo1UkhSLEQREaSK4FESJS2pNA9lpXl2xAMwu2MQoGNF9Ag5VZ6/pG7V55RFDDUJAB8vQCTSyMpipNRhXzC/rtVNL5CkHBEiIYXCakuW9dJGicrqr3/G+xkt86hHPepRj3rUo/5oJD46oAPWWFxjybnQOEcuhdPxSIyZ1XI1p+eYS9wz2ijiODJN1U2bS+D/x96f9Vi2ZdmZ2Le63Z3OWjf367eLGx0zMpNJFrtiSclSkmKBD5IAUX9Pv0PPAgQ9qKCiKjsmMyMyIm7jnbmZnXY3q9fD2mYRBAHm5UsRYNgEHNftutmx0+xjNteYY34jE5jsVLAtiGKGqWr6wZOJ7HYPQKauloQYyKls2Wldz+71ItAqqRmGkRASdd2WfKRhBMTMzxazmaW4qVPKWGufej7nRobhhKklSou511ZUlWayI0IURraSqZh7oqeuqrIJK4pb2/tyX8o5IeOsJ8RI06xYrc4QSjJMI5UwkBVKGVKSEDNNvUBSBPTfONHLHyGKoQghSDGSCcRkmeyJEEZc6BGTJ1PjfdlqlUrjnC2B8pUprn8fCHEiREerK+Tc2wcvULLlyy9+TNct8bYYjtpFCcx8+fIlHz5s8b44zGPMaFWRksPoFu8ibbMgxYTRNd7H2VhSTEoF2ZKxU0Ay4KfAOHpiTDjrEDTl7y7y13/91/zx//5/RCo9y+azgJ7lY7BSyVGSEkNBy9RN2ez1+QEhPFLBZA8IVROTJVPPjvFUzooIjG6RoipMfSOw1nI6nei6FmMalHrcAk3z8y9mZ3+ah0KSlAIpeSY7FpSoddRRzUihOOcniRIcawdMNPP5K85nN4FSNSlKUpLUdYMxFev1BmsLnhRK3pKShtPpWFz9KRFFoak4Z4shLKXyXHwP1OL3F9G/hs3vK4RJjENCZon35UF5X16PlDPGCC4uawbtC7JkH9AvysEouESTYL+PtF2ikoJ2BculJozwxVeCdS3IMjLmxKvXFZ9e/4Bl/Ybz9cj/9C8WRPEWP12wOlMkccG6+YTdeE9v39PIFnIiqRUJg8hfU1UDOXn2U00SmUimroqTyYUD5IpTH8hag1jxMFp0Xs5rHScOg0DKxLKe+LhVCNGy7+8xomO7n9j2E8tO86tfa6bR0WjHNCi6ukxWbh80zhUW7PhR4XeBFz9ccdo6Pv7iQFSRplFkJzh70RXxPGkOe5jGic9/2HD7Fg7HiD4TRJvZbjMqCuQKdC3YbY9cfZZpFiPeBcaTI55ymd5UEpczD387cXN5DjLw8HDgxz9+wdt3B/a7QL1UXFwv8Vhs0ESbSQTefz3gxsgXP7rk4T6y33liOLDszjn0e6TyaKnxKdMPI1lk6logtoLLTzLb+5Zvfz1xuV6wWC7waqJZFsf3i00k+Y7b93uSyLg7izaS6BW68ixymSAh4dOXa7YPPZjIQixRQqJ05uyswTMxnSI/+vEZHx+OJFeD8iwWgtW64f5g+fbX9+xPHhUzoT/QHzNf/OCKj8c7xlPk008v2A8PuHQkisipH4lKI6uIjgq8IPpcMCwXC3a3hUW+PDN89uMrjoee4CaSH9msztkOB27qDbaPRHvi7vbEojXEnWJIiW6VaPSCb75+w6pryFVCZEWUgesbSAHcWLMfTxyHPXmX6a4hWYnrJdllLr8wpJNmPA6sL2tevL78vm/l53qu5/odqhBmB7kPSDkVnEdOKKUxxlBVFSmHuamKWFuS5h+DfJwr+I4SJpmLQyLPCJY0C9khFiSIKqJizBmVRGncfESKiWAiKYWCKMl5vr1ITsUdURzpiZjSbAoXdF2L1hpBCQlSSs5fKxCiNP3DNKCDRkpZ1u9SQOREAoK3RKPQbUNd1VC1CPST6A0wjgPeO6QqImlxy2uUqQpzT0TyFCApyBIlZXGB14LoDKMSRAEShZbmPxpOeO/JCaJPJJOKIJwVOUW0qougnEURWquaYRiYpgHvR1LyKLV+SpWPMTBNE84P8yptQ13r2VGvESKxEB3O2RllU1xHUhqMMQgBi25JVVX0pwPOSpwVWDcRw3zEUBIpBaZRpJRxLuDDgMgeRCShkI9rlzk+DS2kLI9BSjGjVtKMRUmUzdVyeFFSzcMQUQJqcyKHCSEKzz3GwnmXsqauyrDn8RBFnhCxbCMUUb7cFlKRUsL7iJRl7dYYUwYv3hPJT1zNzMw8F3FeiZVIEUhMWOeoaoVRqrj5YySGgHOBFOI8UBAoFEKBzIUxk4IrwVVFOic+vldCae5zymVQQumXpZQFIyQlY8g46/43/XnwXM/1XM/1XM/1XP91y3uL98VZe3FxxXq95nQ8oZTmbLV6Cjt87ClzzqRcxD4hCu7Pe4eUAmtL3kwxCmROpxMxQF01lPwX8MGX/slbUtQsuhVQjB7jWAIZm6adjQ8OIXLBZqSEMTVSZvp+IEVwPhGi43Q64ZwnBItSpgR14onJEaOnqsyTE36aBkxlnnCLUhURP4SAdx4o4rqURZAu+BSJnPGQMcTSE2rF6TRyPA5cXLYYU5dzRJZMo8P5CELPG5bMj7F0aKXLLzi/gg3xxDRh3ZFxOjJOfelrZ4OElIpoPc55tDJIoQkEtKnJ2RPTIxJGkKJEi5rXr37I9eUnGLnkGCzGKIzXIDKbzYamaZimCUEixFgCT3Pm+vqai4tLmqZjt9uSM3gf6E8loF6pCmMq3Hx/ptFhB4e1ufTw0swcb4mSmj/7s7/g/v6BulnSNI/IXzE/H+XvZRiSyRm6pmOz2cx9scSHwNSP5BxZuo6+P7FYtCwWHU294O7uruBPcsS7YXaIK4IHO0VWq3p2dAucK8MiY6YSoltVs4Hq8fobiqPe2XKGjI7JgnVlQPOIqamqqgxlRDlDljNhLNumWdD3A1JKVqs1QsDNzUu22wc+frzFe8/xtKcyC5xNLBaLGTuan86Dj6anx8ykv6u+t4jevobXPxR882uQFqZdwnTQnoHaS8ZDoFtlrj9XKOnpp0h0gvOvBLFKyABKag7vQWlYfgLn5wKC4EdfaS7WHXWdUFnStBvGcODVpWGhTpytzqlqh9B7ggsobej9RKXOaPUabU7kWDOFI03OTOHfk8InfHvr+eKlo9E1zi+w6UgkcfITwxg5DQNddYFLR8Lo6IcdEUUtDmhVLqDTriGJhq97z/EoWSwdznn6/Zb7j4nRCw6Voz8IhsFz9aIEdmpjGKMjIEk50ZmKvBy5/kpQG8H+VxFpBYuXNZlIcy25eVkzec/Zecvt+wlVwd3HwPYuUa8TMQv0SjKdMkZ4tKzYPUSSGKiVxKwS+4OAqNndTVy/1PS7yHhILFaGMZ+42jR0NXy4f0t/yiyuFFULLo5Ilbl8JckR7BBoWk13JjmNYwkP2BXH267fsbnSyCxJPnJ7e+D6pUJrzfajZ9k1HPsDWQpQHsyAjjXtp7C9HdFSIZLkdJ+QjcC5jAoUF2BKJASb8zVZe9TGFh79fcQ0lquXS97+4sTmp47zy45f/GXAVIqPHyyHXUA1oI1kiD0Hu8M0mrvvHDcvrxhuPY6Jqknkuue8FezvAw+HLUZ07D8OhBS4dQOLRmG2HYfTkS//uyWuV1RNTWuWvBseiDlhOk1gh4gdUjqWzYK20hyEpj84hn1gfd7SVS0pRRptWF8k/JR4v73F7iJ+N3D+hUBoS5gM+8OJbOGw62lWLVI7kshUTUW9rnCmIeqRKDwyN3xyc81pHFHf483+XM/1XL97NU228L8pa35CPK4nOmIsk/jiBC8hPtNkZ1dKacZTjoToZi9zaUIfhexHB7eQmRgi02RnlMWMaUmJXH4L4rLFmILukFLikufxZlIsDWuOpen13iMniVZljVFr9cRxhyLcF8RHacALy/2RBZmevCciQ44RSabSBqVrpDCEoOfGzJdQ9OCQqjgktFZUVUtlKqRShSkYC6okJ0EWsxtZK6oqUNdVwdUIhdEVpqrQqnDVHxvVFIuTJsVMpjin48yfz7/1GJQqSJUQfHEW5TIoEELMr1lxrMdYbteYBrJECo3UIFVbehefyCmUIYKuMFphtJoxJxoJ+EoxqOLIThFyKqxDreXseAmAKwOHFAvqDI1GzQFFlMAmKUElyHI+nghyymQxu3WCL8K8qebXsCJngZyHIBlZBPrEfF0ptJIYXZf79MgrzGJm4pdBTrk2FDkptFKEVHjspck2c1hXuVadd3hvKWxLg1J6Zl5SeJapOLu0qoizuz6EgMiZMD/nSlTz4RKELkOBlNLsRC8ImTI0KA6qIsIXMV3J8roKMkpKKm3Keydlnn9zP9dzPddzPddz/W5VVdXUdc0wTPP2ZSDnPGfYeM7Pz6mqEiTpnMc5y2LRgEgM45HHXJfHfr30r+ZJGFTqMbjRMI7T7PhNHI8nlt3Fk0HBOkvbLErPnwVCKKqqQQjParXCGMPpdGKcekwlyplBKjKFU41Q9KcerTUhGdabDqUzSguGoYQ3DsNYsC0z81wqyTgOTxupPjiapuHi4pztw47jcVeEdqnR2rBcLRiGke1uy/XFGVXVcHHRslyucN4yjZbGSJwPj9Q9SJQspMdeS6gZuVc2N5mNQqf+wMP2jof9PbqOTz3hNDrW67OCRcyJnAWn04C1lvOzFUnVNAqC9yhR42zG2sA//Ye/z6I7pz85lBZUtSH3Ga009/d3ZWuUQMoOKcG6E1UTCXHAGMn2YYuQgrpumCaHtYWekeY8I+c8SikECilgvVzwZ3ffzBuaJZfv7//9P2C1XnD78Zbr6xvajkcMOr/FJySlIqQLUc5YZ2eXFEc9HI+W5XJF3594uB8YxxGtG/rTjqapiV7TnV+glGa/O3A47ufrRaN1xThYxsESY6Cqi/h/FCdWqyXex6ftBWM04zhg7fTEJx/HAectbVtyvEIIswhfhHRrLeM4PpmyivCvcd6RZWSxWHI4HHh4eOB4PHJ9fcPDw13p0VM5PLZtw+FwQEpJXdfzwKoYdUqYbebvqu8toosa3n0M7PcQJ1D1I6xeEZVHNWDOBB8+RsIpUneCZl0+p6kF1UJwuBc0S8kPf9wiTM+XP5B0tPzsJ0uWTYPSliQiHx92/PizF7j4AZWvWHevEeqeh+GBECWdqoGAT/fY7ND6gkV9xjAeOQyZYbLY8Zbz5ooQAh+HI+tKMXpPiBkbBQ9bwcXqmv0u0EdFFBW1qRGx4v194jAeubm4YXL3vNt6XJbY8cTdViKSot/CaYr4ALGpGMdM20lu35fVgMWyJUtJsIGUBdMRXv5AMFrB9uFI8ILupeHsumW73aGXifEU0evAtw97Ljctb97BeEhoMlJ1TPsIMeJHgZskIibajUIkgVxKHu4S/TbR1IlhhMPJ0l1o2oWm0hldSyYOuFPkcCcZttAsIu1S0NueL37aopzkcF/CvDZXmbOzJf/hz490a8lipTmdLDevK7ql5ri3nPaesyuFyDWnbWDYeVYbS7uu2L11bF4p7j8ERB0YPsCy2iCFR+mEbhLLTUO/9/g8omXFYl0zTYnRHqlMIovMsU9c3iwQOhKC5bOf1rjsOB0tslFIAt05pLBkxLNcKXbbwDdff0QpzWa1ol16tBDUmzPu9zuqBt5+8MhQs9wopPFspo7dnUcKkElyOO44u1CcTnuUAVUJDjtX2FdWINYV97eW4RAwJmMMfHjzkXrR0tQVqQUbJq4+LY6/GBLeKpYrjbkQ3D4ENpuGuhGkoaIPAw8/D/zBPzxnCpHLlxXuviARchbsbkuAg7E1QUeai4HzdsMqGH79t/ff+xf3cz3Xc/3ulNFFfNamiNfkR7EyI0SeV+oKlmVICev8HAAzO4yVBPKsSJcVyBgTIueCA0mBIrZDigGt1JOToSCt1eyoKI6buq4LQzAVxjXzvxfBtDTnAjmvJHqMqWiagj0xpjTA5fbizLULT8GPUNiQ5ZOK65eYySGjZWE/ClmcDCF4rJ2xMynODuYirgoBMXnyHDrpnCvOewRV3WB0+RySwC8CWpbQzrquqYyB+WBTmbogQUIEfOHJZ4GdE+gLV70uGJY5mPM3lZ8OQzn/toBeXpcQIt5FtCyHJSEEIs+hnOTCKncjWs5udVXNbO7i3hCmIpgaoz0xSvLT9yq/qzIJKUFrSDmUBl5VkBUpzlsGFAcIQj4FqqacCDmCyE+PXSkxN66lpJRk+XgYK9dCSnle2VSkXA4LRXhOxcUtNYjC8ZfSIITmMTBUCEndGFarJV23KO6XJ8ahx9oB7ycQmabJZRwk5exIKicLJWUJEZ1RPVIUtqMQAiXVLHxLiII0b2V4F1DSPb1WQmTE/LUxRZydmesZckooPfNHjS7c/lxE9ed6rud6rud6ruf63SkpFFVdXNSnUz87wZsiME9jyeCZDS7ee8axuILbrrh4x7F/cuhCCViPKTAMjpwz02Q5nXqsK0LsetNhbcFaVHWN96WfVAq01nTdgnGcMLom6Mx6fUbbNEx2xBhNyooQC88bEsZoci5ZSplE01Zz9pJlsWwIEeq6Kq5rUTjeITiqyiCVKNg9LQsid/QoJRiGEz5YhvHEYrGYzycOMMRYzBBla7BDInHWcRpOTKNDyQ5BQ9d28xZgQShmCu5R5sLNFiSMMQxTwLmJ3e6O43FbmOuNYrFYUVct/TEwDoGmachZzo7zzDhaDocR7weWq4px7FFIVFpyvrzm5c0XGLPAaBiHnhgCWhk2mzN+8pMfs1xt+LM//w8c9j1t12CSQZnI2UVNSGPpemPp643WeGdRStK2Nf2pL8+fVEQvWC7W/OJv3rHbnli0Vwy9R5uKP/mTf8lkT9zevuOnP/kJ3k3I+tGN/tiLz7icVOz0SlW8uH6FVg32CG6CgUiMmsOhL0OEgwcy24eHcvbLe+qmZhod0QucLVuezka8H1BKobXEu1Cu2cFB7kuwbCqImsLLLxsWpY9m3rwdZ457YalrrZmmEp56PB65ubl5co2H4MlZoY0sjvPjEaUU+/0eMWcgde2SYZxompZpLO72vu+BR0xSec4fNx+c+7u3RL+3iP760+KovnkFrlccThFr4fjgWVzB1U/h7i1Mg6CpMj/6acb2Cpk1iYCzivpC8Md/fEnIt6w2hvUSrrqGy7MlbrR0VaR3I7WOCHGiNtDVAWH+HVpNEBLOLTBqwJgR6/f0wy119ZEQM0IJrIParJjGO7S5ZLuPnGyFOAv0oyGmBusE0yD4m/0tbooo0dFWNbnqOJ4s/+6v9kijqWTNdu/45utI76BbQBoSq43EEenOIXgYdp7eSk6DoL8TfPHTlg9vJiabMOvEcKsQ1pPIhCQZHopzz+M49KEctDr47n7PwiaSkJyER9WKT14IwknQbWrq9cT+fcBoiWgTKcHlTcP7DyNqKbHfCCotqJYBcxQIA7qCKCxV2yBEoO89i/aMbmEZj47lhSbpwPUryde/tLw4WyGt4f3bOy6/crzdnfBa0/uMdnDaB6o6YeKEdWBayesvan7x5yNVLVlfa5KMHO4jVS2QWrDaKKwLiCpyGkZytixtxdRDzI5Ga+LsDtzvPNnXLNYV1m65uX5FEAN2dNzfe1QFFzevSP09hwdHGCMuR958M3D1oqWWiZvrF3i7Yx+O3NyscGOiHyzdElzIDDZR7QXLdoUwsH0fWV5IXl6uETawulLcf+foXiRWN4rDnUOr8sthe3viqx/f4MOEEhUn+4A0mvWF5v23E6fR89kfXjHtLC9/oHh7a1FGEwdP1WT6rSetAwebWF9L6mVC5gZRZ5gkphH0gydG+Hg7YlqQwbDf9dzcXJCl5+1fH1m3GrMRnIYj/Z1ArZ65qs/1XM/1n1bTNGijMXPoZUoRpQUp6iJaSjk3OnoOUpSEME/iRUYb9dQ85lksTCmSYyRHW0Iv028yGXIuaBMhFEoqshAlgHLGg1jrqKqqOGiSL+JtgpgEOc1+d23I8/BQSoXRNVqZcntklDJUVY2QEoL/zXqoUCiZC6cxzrfrI3Z0sCnirdIl1LKqKqybwD+K1UXsLYeWhA/2yWnsrCXFkuguS7uMFAKjK7p2UX4/IKjrmq7rkFJirWWaJk6nE9Y50uwU8iESsoD5eck5Fc7iPNx4YqrnR2b8fB98afrS/DkpJqbJIkVFU5cQouJUyZAoq69hQqSCoEk6lDAeBCLnGVtTU9eJlBTehzIgCQGfPUIGhCyZKnneShCUaUWORQAnza+1lEgkOZX7FVIg5/Tk6BFCPDlJSgAnxeE/24VSDDNmiOJMR8yvabnumANlHzcVyAWLI0QZGlSmoqk7Vqs1TVPCrIpDK2DdgHVDwePMobaFp66QsmxIKqWp64pKq5nXX2ZG3ntqU1HrmbEeMnEOG00zIimo4m6KIcyO9MJVL+J9Kjg/ylCpnkNUjdaInKjrEtT6XM/1XM/1XM/1XL87lbNiGh0hlF5nGqenIERrJ6y1LFeL0md2Dc5NHI8HhrH0R877GYEnUUqilSSFgh1s6poUBX0/ziYTibMBgcaY4mJ+3FJFZPq+J4SEd4GuXSOEYdF1pBSeDBzWWobpVMLpdUbrx01Iw/n5Fcaokp9EZru9nznVxVVfVVVht6dIVVfz9mfptbquYTj1TNPAw8M9xlQslx2r1Yr9fs8wWqqqxlSKpt3gnUdXBYVinSubhsExjQPrbs2Lm2ukEKSnnrr0XwUBqIpgPG8VFg73Cakyy8USqROVbiEb6npBCJHt9sjZ2YaUAtaOSKGwY0KqmpQkAkPXbsh+xWevf8zVxae4Mc/87zwjGQWLxYJ/8A/+Ab/3M4d1jv/1T/+U9ZlGmQWqktx80uHDkZQ6pFS/hSAsG5hSChaLDiEU9w8P1GrN8TDxy198i1Yd/cnhXOBnP/spX3z5BVImPt59YLu7Y3129ZQJJMRvDC0CSnaV0IDhYvOC66vXfPfmb4hBE4IkJY2dUhmIjJG6rjgei0Ae456cM5Wp0UbhXWboe6ybMEYhVaSuzYwbKmii0+mhbHRKURDRyyUpF4c9QNe1VFX9dG0Ow4hSZcgzDONTiOij+UppwzhMCOFp2yXOWd6/f48xhqZuAEF/GmnbFjlFKtMwjY7j8UAI/mnzwxhdhhNzTtH3ATx8bxE9uiLa6hrkElSluP0u8vpzeNjBQsLv/bOKtlnycB/57IuGV+cbuuac79685e27PV/+aMOPfnDBOClSdlytrmmqI60658GdaFSHbi5odc/54prDuGUKb6iSxsYFMfWs2gqRykFP5JaUthz7e6LvqBsDSaOrhqpueOi3CG3Z9RCSZRwtKa0YJo93kdt7zzhAU408bAMpndj2iSwkw9Hz/ld/i4+C7RbGCW4+g9OHzNWnFtvDdK9ZdYbtYaSuBOMpUy8E/S7ihwxVxu8Fqoq8+JHieF8OlGYDdauo2kz0memQSSQwif4g6FaS260nRUG3Ttw+SM7lDjtKqCKtMthjYrHOiG6kWknuP2bckOg2CZcSixcgrObi84qHXaDvQ0lDtor6k56Uy0r8MI6YhWC1rPgoylry6dDTrRTvPgYO94KzF5H+jeBhCHRrhRI1X//8xHKtwSuMXlCfTbR1ppUb3nx3x+nB8vnPWtbXCZUSh73C+kTdaj68tXz8LrNaLbj/eODzH2XuP2TqNnB+0XH/YeD9d4HFpeLrb+9YrcAHyXRMrM5qwHL/ceTiqqbp1tzfj5jKYdrMeEp8/fV3jNawXCju3428+nyFDZFI4P7+hAw1QWbOzmpStGzfOMaHhPYWuQxsdxPdQiMazf0HR70QyBjIKdGuBIPbEX1C4nAp8uoLzXE7kk1i9UqDHBn8SD1IlgvJ17+w2KOnbho+e/0Jt/d3JUV4nenaGhcsx9PI+spgJ8Wbr480ixVGtXTdwGkYMJVk0Qm++Xbg7LohjIkwCd5/t6UNF+T1934rP9dzPdfvUFVVaW6KiJ4IIRPjHObpS8BmU9c0TYv3EeciOfnZeVFcAb8tohc+eppxI47MIzuxNEViFiiVNEgxL1PmImg657HWPSXCSykIMWBdIIYStKNkEXq1NjMCRPMYQFrwL2IO09RoVe5NOUgUkdsoSVbMoZARHyPDaWA8DSjdgdAoragbg/OFKSlm13SMgRiLMzvl9IR7iTE8hWR6HwqXUQhSegzknJ9rU9O1i7JCWzukODKNlilZfCjOcxtCQcHUj4E7kRAgzuGVUhZnT3H7CLx3+OCJ4bEBLkJyjAk7WciaGCRCKHKy5Fzc3yFEBH7G8kBVOZQySCFnMb3cljE1wQtSdPNmQSpBtCkgZSRmTyY9oXzKFxYuvJxZ9iJJspQkJCKX0Nmcy2uSKW5s5zw5FxFbq/J6phyReQ7+nJ3oZPFbXMJyO5k0C+nlORHzgUjOK6hN01JXJUhISlUOCQJCcDg3EqNDyhmfQ0DIhNICow2VadBa09RlQ6EcVopbXWSoVFVc70lh8UgfCAlSKtsSRbAvAwchysovIuG9w04BZ0vwkq5VEdjJQBlkmUrNmQDP9VzP9VzP9VzP9btSSmomOxFCQmuDtSMhBh57bIhM04AVlqZaYIzBOTWbUorgrbUmJ0nMICl9kZ0m2lZT1YsZnxeoqxrvLVVlWHRFHA4+4b1HqURVteRsIctZzJT0/YAPdsbMlP6p0g1aNXiXCB6mMTJntLNoF0jBnOsTC3Exl398DGk8HI5AMZJqrRnH07zxKkpovUhkIl1b8n28txgjqetyDqgqQ2/d00asc8Xs03UtUhYXstYa6wZE7FDIp01VKUr4u9YaqUo4vPeusNqNJIoabaBpVgy9LUYNBDkF+tM4O+3V7IR2nJ2tCS4gRUUIgs4s+f2f/UOqasl4OhFCnDOENE3bYb3Dectidcbv/ezH/O0v/wOrTcVyXVN3FeszyTjtqKdMXbWzCQUQGesm+r6nbRcIIXHOUjVw+2HL6WSJQeNdRAjFH/zBH3J5eYHSiVO/ZRgPxGRR2RdhnwzzpjAwu70zKYGpWj795HP+l38ny3UVwDmHs498+4jW4cns431gmhyVKeJ6uZYFlamJqZx5yAWzLObnnyxo2nZ+Hk8IodBaEUPZerXWz+dVMQfNBoJPOOsLG14aunZBCKmI91oz5JEQHItFoqo03s/4F13P128R2LWueTTpWOtpmobFYoFz7mmj17kZ8aj/bl3teytvv/5L0CtoFpBFJE+wXMJqCWGEsxX84IeRL29WRHtDYOT6asBNAzdXn/MPfxZpFwekPJI6R8onLpYvmSaBECe6LqFVTcUKrQt/c7l4wa4fcP6GcdJotWXy3xCDR8pMzBMIjwuCkB1+WOBs4DAeOFrY7gXCJLLP3D9EjgMcDyfsCF0n+HAL21t48ULw4aGAmk6DRJDwE4wnOH5T1nnbH2Tu3gjsFqLOnJ8Z/tHPXvKwfcCNEqM0t+8tL39guL+LoATNCg7fZjafwfI8oYzg47vMYgXRSqII+D5TN4KL65rdfgKn6JaCNAhcToQAQmX6MfP6U0UK8O6XkW4l0C3c3SaWa0UkYIwgB+iHTNvA6RSptgPDMRGOHhEln361JKmJcIwkElJAdPDwwWNQfPPznnoBqoHkBZXTNGbBgT3SWM6uNDFbqhWoJnP1KQzTAzlmpFL07oF6FcnUnHrH+kog5IJx6Mm5Yhw9tcl89iPB4WFAS4kLE5evS6qurgCVWJ+1TNlxuHW40WBHT9dowgT7wdOP8MW6ISaJrjV12yKS5Jtf3xNT4sXrlhwzn35Z09Rwej8/nxmUSYTgeP9Nz+KFIk+ZWGesC9RTYdY3tWLsJcdbx/oPK+q64bu/6Vld5BKsqyMvX6952CWCDRz3EZEEvld8fGORSnJ/74k+s+8j6VTEkl36iFpkjIJ+8thd4Oy8oIT2d5bho0B3MMoDzTIScmA8RG4+hygOtFXF3Xc9y5dQm0vazrE2LfL8704Rfq7neq7fxRJPgZmPAUMlTAaULiGMddNQVQ0xelIcysBcaYR6DBQteI48u3BzLqxzKL+rlQCZIeciNiqlkerRZVuc6zEwN/4RpfzclBfLb0qBEMv3emxyfhMoVIJlUipNc1nPBK0L2zo5WUR/QmFv6wolSqinzQ4vPHac2O8OqKojpNLQFxf0I9amNLpyRozEGMnEWYS1pbnUBqOKU4EMMZVhRE6iCNOqoDoKJqQEobZNw3KxIIZ5iB0TOSVSE1shWgABAABJREFULmz6go8pgv2jC13KEqCqpH7ibpN/szHwKJCTy3OBdaRYxGSRIzF5vPNPq4hauycnkJTFzV/XDVLqebVVo1RGitIPMIvBIXkyHqVAqhIaWsgjsytdSPKMPEk5FvxKlvNjUEWIF2IOOkp4HwCJlGEW5NWMj4HC1i9CerkGy2uASIg5sFYK5gOhnK8NjVZmZpuLJ5fRY6NervFUQnPJCMUsnpdcHq0FWguULiiXEiL0eD9m/IosfHitKnIsmQAhQnKhYJIerwcoqKM8h+VSXFvWemI0KOlJSRKCwFpBzmEeTMXv5XZ5rud6rud6rud6rv92ylrPcrFmsgPTNJYecM5TKcgTyTj0gGDZrZhsYBwH2q5luVzAnFMkhJpNKBVdt0BrQV0VdrRAg1C8uL7heNrhvWOxWHM6lEDSlBNSlZ59HEeUrNjvDjRNi5BizpKJhJiQQqN1ixAVduoZVOB0nHAuslot2W57UnK0bQmUVFpxOvXlfnZLnPMzzlFh3UBrqhnP4VktlnNflBmGI3WlOfUDMQbadoVSRdQvjvgJU7ezwSZh7UTbdoTk0TLSNm3p4YMlpbJdyCNqT6qSmRM9UkpGOzBOY8kxHDxCVghR4f2AlAajFc4FTqcTSheDR9d1uLE40Id+QGlBtIl//N//Q7788ie4KaDmXnu1LJpm8ENB9+VECJaqErx4uWF11nB+WUyVpnJMp5FxrAg+4nxAK800uXnI4maDCRhj2G33vHt7S/AQfAYkL65f8D/8D/98DrEXLJcLDoct09RTmTWZNAMMS+P5OAApnHpDip4XLz7l9evP+MXf/hWH+3u8t4RY0D8+eMbxiA8ls6mYi8pZJcYMzqN1jdaGYAuGchh6uq4DUTjzbdsh0KQYAcV+f2C1WlJVFTHJ2czlOZ2OtG09i/Ue7wNt03I8HmnbljDjiOxkSTnOeCCB92nOBoDdbkdVNWhtcC4QQ+J4PP4Wj92QUmSaJparZXHHpyKiq6z+zvfw9xbRmytYreG0hfNz+HgHyzOojeTl54mLl/DmLqD4wKY7cbaq2O9vaasvafVLAgPJ1yzNHxDlr7Hpa/rxF5AFIU3ISoFqEKGmkgqyoRKfUonIqT8wugNNdaA2iv0wURnJMJUfQIM94H3iOB5pK8ntu8zgJD4ndqfM6QDTCAjB7dvMdguqyriBwr+uHKOF/V6wXgmO9xJcppWGgcDl75Xbm8YElSB7gUuRb27f406By2vB9jbCCCC5/qorE6N6xeInA/UCDsfI+UZwt8v4CZKNGC1pN4UjOoYJoTKpSlxcNGwuMsejYr1JMMH9+8hpDadtZjpGVq8K50lrqOvM9LXiJ7+35m53YLyDdgPd0iCzoV4NMJYL/Oam4c/+9MRyJbh5XRdme4TsNME7qDJ0is215nASuDqR3MDV9YL+6LGjQbeRzQsFXvPLP59IOXN2regPgXaR6VaZFzeSGFvG4cRk96TUcjpMXN105Gx5+yZRacn6THK6EyRGRJQEb7n5vMJNFjPB5Q8rdK24fZsQ0rM8K1ysl6811lns4Hj1RcfpkPnwZqSpFlx/pnE+0FRrYnD8zS+3xAyqAiS8/Crj9poxSZpacvYJeJd59+HAxfmS084jlxM+GJbXDV+8uOJXf3vk/LLl1acrjvsJqWG7nagaEFJxdlUhUHz99RHft+WHrczsTiOqVbx+ueLiquW7Nx9YLWr6vUWQaTvFxVXHeNqipKS7hs3Zmn6wBBkZh8KGPZ0EdaX47Kslx48TqzODEYXNn8yJq+X6v+iX93M913P9blRBlchZpE1zMwlKFHFdaz03QQKjZhdvTgiZSdFhU0DKgtSYsdfEXFwyQuTCPqc0cHnmZSs5B/cUC/nMmZazq/3R0Ty7mEXhrotQxMcYHVWVZ2a6x7kiOjpX7mtVmTkcUiGVRKnCdE+i4Fy0qtB6FqBDCWkah4FhHFi4iSTFU9iQc65gazJUdTWHyoj/CDOSHsOWACUlRpcAyeACwXmCD0iREUrNTO1i79BaU1cVbdsWJ4cPyBBRZEJKxBRLMJMWyCzmpPri6NdSFDe0LGuWgkxKnogiJwi+OE6UkAULk5jZ9Yoc5VNoDznjtSMEi3NVCSCVurAfTY0xNVr9RviWkplxLlBzyLc2VRnQA2l2quecyus3u9ZTLH8XgBAGrRQkSRaqCOwZHp1VgjgPV+bHncUcLgo5l00Aax3Bl0BSrQWV0Aj5uMI5C+ZPormeA7QMWhfBPgY3O/gDOc3XjVFoU75GzY72nAIxWJAeQUWaHTlC5LJtJiVaUL7PvD7aZokyiaquSoCSEU9BRD4+DoQ8znt8jGQ0bkYOpTRfKyS00iihEMb8b/jT4Lme67me67me67n+a1dBXugiCIZAXXczDqVsYlZGE73GOTsH0YPSAq1lcTc3RUg0pmaIFmMKqiVTgYyk7BEikGNG6aIBad0w9JacBUopcozEEEk6cTgcqExDVbWzwaaaWdUeYxTOTWThialsGPbDwGQ9m/U5mRJYGlOgahqGcaBbVFjXE3zppVerTdn2k4IYE97HIrY6xyQmtFLkEIjeEZzFDgPr9TlN1Rbmts2oWlK3hTaRkmK5FKTjRIoWbwXnl2ecbV4RYoZsiT6hc/meWlTz9mcu98MWZORytcLGwP4kca6IsiHEYgZJRQhumo7TqUdrSV1XXL3YcNgNXF58iRskr66/4h/90f+E0R02DeTkWK9WpFiwipqR7AMiJ3b3H/n5X/8pV1ctm8sWoSeahSamjFI13oIbLc57FosFm/U593dbSuBnQT2mBPudxVuNmxKgECLys5/9mKurCypjynkqZL777jv+6O//03mzWM05VWLu1+eLUcx9uGy4vHjNenXJoltxPO3IOTJNI0KUMM6h90jFnIeVqKoF2pT+O8bIOFmaxszbw4Jx8iAses4EShnGqYTlNs3itzj+EqX1fMYoZw9rPcvlcmaXC6qqoFcQBb/jrGOyE13X4b0nJ0EIZXM2hIhzgbaVc67UgAsTKUasHanqsqUdoi9DhrpgTZfLlsNh/z1iRf8LRHRrIdzBy5fFcfaT3xcs21wOr8DDQ3EvLxqPUUfWywYlFafxA3X8PTbrM6b4HUatMfJzpBCM8W+QagDOOR4PKDnR6APaSOx0JIWPYAIxDDQasv8h2+Ejt9u3JVAg1CjxJSkceHP7NR/vEt0yEkZFfzD86s1I72FMJRwr9pmH92AT1DWEADfX4DwcTxBcJubIq68Ui0WiHzy2zmQNr15ldlvot5kwQgyZYScwphw4nYgsbiRJJEZnWa+X9EdHu8wMfcXJR2pT7kfVCZYvJLstXL+G3TZx2EK9gMUmUbWZj/eJ7TaTRCZqwflLwXKtiWOgea3wQ2B9IRkOgm+/jXgnOW4Tp2N5bFMPbrJ8+ROPqhRc1Lz55cBhOPKzn204nQLNEtrqkrfffeTqZctf/5XnxRewXJa17Ls/t1x+Zmg3GT1J7AmUdAyHRNUqllXHhxjwLmKnSE6KmASHe0ibwDRYrj6TvNhUHLY1w5BQRiFMze5+4rPPa/xE4ROFkSQEqkoYI3j3LpJdRg+C+sxz9qLCToYXn3RoDb/+pcWoTN3Bd7+wrM5akvHYlKjqSxZnhZW0WDZM0dP3QJVJQXC4hfHk+OTLjsVqgZHw9sMtX/zkgipL0spyc7PiNFgS8ObNjoc7y+ZCoFWmXdY83A3cvRu4el1zca5oW8HuLvGHP3vF4ZS5f9jRGsXWCpbrCqMhxonNWtOfLNMo6LoGe0rs7k8478ta+kaxOjOM1mNqwbCTCOG5++DQdaLSiigmbt86fC242qy4eFWz3R3/C351P9dzPdfvSgUXyClCjmhDEXlzonCaS+BjChmUgOyQjJjKk9KAdQeEjDMeQ4AovMWERKtMFiCzQsgSDhpDoKoUSlmkLKt8InkEqYjDypQAR0rIaQiR5BPEjJKPQUMJIWxBrPhACD1yDgsVQlM1LcvFGoMixxLcDYU7nRDEnBG5iNooQRQJYQSjH9jubum6JW3bkZMozbIt/LsYIouupW2b2fUOIinkPIAgeVKI+FRY6dFHUghFOJ8d4y548iSoUqZuGoSuUFWNNDXIkSgkIWWSzDMPvYTt5OghOZQISBXRWqE1KCHRaoVCI5KACJFMmMMqtWlY1CvqunDZlcocT3u8DYjUM9keRCIYTaxalKxRsiqPa6kJ2UIOaNVQNR065XnA4MmiRggPwgMBQSLJMLvzJ7yfZo65QImKnEsAaGUUWhnaaoG1haUZ4kSIAyk4HCM5FURKzooo1PwaeqJIhDTh/UDwgUpXSNlgco2mQskaqdW8kmvm7QGJMQZTFQHdWks/DPR9T0bSNhtiXKB1CWkqQ5gy6BFkoh8IyeNVpNIVSldlhTQrtGxRojDllTTzoCFSxYw2BqkFptKE5BndRESQkPgEPkMQAmcnlNKEJAnJlzVYFLLWaKXRuvqv8nPhuZ7ruZ7ruZ7ruf7rVNMYTv0O66pibNA1Ws1b5VmSs0HJtmAEXUBKaJqWw+EASJaLJYfDkXEcS6ZRDuwPDywWNc67JxRj35+oqgbvA5Ux2OARWaJ1Q7dYlz5dSJqmoa4aUkr0/Ym6bqiqmhgTXbvg2G8R2rPoGnyIeGtpFzVSg3OBzdmGvj/NbPeEUoLFomUYJqybWOQFh8MBcRT4YHm1eMXLm9cc93uYN1EFkcvzC2KMLBdLKl1hB8fNi9dMS8e7D2/ZXHYFN4nB6LLZH4Ij55qLzSukWDBNAZkdpEhKqth8ZEVMAu8jQpWw0c1mw/s7jbeR8/MLnJ+e0B5VVTGOQ9kuzJL+NPLFl58xjiPDcKJt1xh5wcXF5/wf/+Tfcrb8lGgD0ReEYE41Yx8Y+5FgLW4aCXbi53/z7zn193z1009pOsE0mz5GZzGyYdGcFxE3JUiStl7Q1I4YEjkWETyFjB1h9zCiVUGaKKX4wz/8PbQGKRUpZdp6wXCa2O32NO0aY7p5Q7QMEH4jojN/rGnaM6zNTDPuJMQMouQWpSTKxnHODMOB9bpDzYORMPP8U4iza92htaRpSk5UzoKqbsq1oTVSpnlbs2y3jlNBC+VcNoSVqhCq5Hodj0emyeKcxRgzi/rMQaDz5rLQ5czhy0ZrGf4U0V2qzDBG+v5AVdWEGFBRMk09SilSDvT9AaUUy9UCqRIh2r/zPfy9RXRdQ9WAjVCX7RCcA2XKefzqQrJeCF5saq4vGmQ2tPqaadzzEH9JvTKgv2GUe3LsGN3PqavEqW8QacFV9a+w/IKP/f+bi+UP6f3Ih+3/TFfDsl5yGraMk8PFiJQb+qGERR5PXyOk5f1d5vYjkGG1jtzdOr57Dw/30Gxg2gMe6laSY2LaQ3sGH96C60tOFQJW17C+grv3oE3GLOHsAtoG7idBIqM30NYFxiqU5O2HRCUkUcMwRW5eKrQaGd56zs5bdrcelzLbDFUtWBpBJnO4jaAgRsHlC4EPiaaF29uR00GwWhq6zqEruLvPsB256DTIivttYDxlkhN0pmZYRL77uifViTCVA3ZzDm+/zawvEgyO3S38Wg78vZ9GPp4SdhsxqUcLmGziJ394wZs3Dwgyts80Z4kvvirM7nGK9NvE0CfMSkGKDKnn1VcdcYT7DwPXryt8GhACDofA6jyhlOZ48DzsLEIpPn47snrhefWlRCrP/g1c3NSsz2uOdxP7e+i6iqqOBDxa1SwWmnEK7HaBadrxyactbVvz9vbEy1cVKcH7N0euXhsyluXG8/b+yOpSs3MjZ5dn+Nzz/sGxWS0IObK8aHl/f6C9g24huTpb83A7YM4Fq6ZBaZAqsl7X3L71fP7FC8bB8ebbns2muNVvPmmpO8FoE22tME3g3f0H2q6lqgR9H7k571CNQJmelGrGg6RuWjafFYGibhq8BW0Sn7ze8Dd/856HhxPrM4OfFEpBpTSHh0iqEm/dDhTcXKz58vWG2487pJm4un7GuTzXcz3Xf1oxBYiiiMKyMOdyLuxpskYKP3PBJcH1xDiSsiWmqTDP04ycmMMjxZx0X5zFzKLg3IwpUAogEuPsWk5xdqEXcfU3zndZxEwVZ8Z1CWXUShS+uojEFAurPJUGT2lDFoqmjijNjPLICGYHPXlGapSGXCowlcSEgkGxbgQKL12gZzRKmNcjC+9cKotS5XEqqVEyEoIlhjh/DFoI0HPfn8r3Ld+7YEi8dyRyaWQz5YkXReD3qaBYvPc4p4GIyHles4zlYyGQsytdqcJIz2RCDGgdqKpy24vFisViRV11JRgoJbQcETzyusOMajEF1UJxkaecGMeBrBRGNxhdwmeLwyUglSZlRYyCRGFJkj0phJmjWRzo3ofSPOk0u9zLIERLhVY1AkNQHh8y2U1k7MzJFMhCgiFlSYiBFC0xWEK0ZAqyT8mMUsX9X5kadIWuTBHNjUEI8XTYiTHinMNaiwtuDjRVaF24+nr+Y4xBiQQiFHd/8OQ8kfI8aMoRpeqCjwG00TRtg6DCWgcizBgY5sCoki8jpCTHPLu0BBlV/r9SpFyesxQzSli01BhtELrgYp7ruZ7ruZ7ruZ7rd6ecm576ZK0NQoLzlqZuS+aNnYix8KFjdGx3WxbdAq1rBLLk88yov/V6VXAsqvRTx2NPVTnqpp4F+owxFSEEtK6ZRkfOnqatcM6Sc55RiQkfXOm3Ztxc3x+Lwz0znyWKs1xIsO5E1zW0rWG9WRJjYVELFLe3dywWi4IN1OBdIMYSNF9VDVJIdrs9+92Os9UayISYMFXNsD8ghKRuGqwbOByPhJAwpmIYJpyzLLpzMoGmadnv9zjrOT+/JKXynJBKsGeKuSA6ciQmQYwSo2vquirBnaoEs+ZYmOulMtM0st1uZ4xiYLFY4mzk4X5PZVpevf4cP3X863/9f+KTl5+jdYWfTggkVV0zDo7j6cg4Doy2J+fAZAfuH+64vDxnsei4270D6QhpwlSStlmwXK7Y7fYFTakM3kcqU2FTQCtDCCXDab87kFOmbQvi5B/8gz/in/yTf0LTNDwCz9u2pWla7OSIIaJ1+q1g0cf/it/6j0ArzfnZJfvtEV1nTqcTbVvh3PjEnFcqE/vCRn8MDPVz0G0RwSM5ZcZxomlqrC2DCWMq+v5UNnXrej5/JabJFrSQlPPty/ncaJ6Cd1Mq2wtSlvPc0I9F8JaKFDNdt3jKgEqp4F3quimuc6nn71WeOyh5X+M40LYNmVS2YN3IqT8UzvrQ/53v4e8tomcHk4fTd8Ux/VFkNi388KegIlQq8/qFYtkqKqUh1FS65sXFK/qxJ8U7QtrxYLcce8HZQpP4kmHKpAAv6tc8DD9nCoGH4xucV0DgYd8z1ic+3CdinDhMia42xCT4268dhweoK9iP8OYdaFU+PuwjtoccQGS4uAFvQYmEu4XPflgV59gikq4zxhTky6KDD28iD+/KcMBsilM9eVhtYMrFcV8vInUt+eVfwtgnmlUiS8EwwnE3cXm+5MWnkodDz+GQOf8Ezs5hv4OERknB8jJxcyMIQXG/94QR+nuBEoLgBefnsNvC3ftMmiCcw+oMxtGy2wnWWjHsMt3KEV0m68z6BvZfw+Ja0O8oaJoUaUSmW4Oo4MPDhBeC8yvFw8fAYEHuB1aj4fAgOd4luovA4koQxMDt+0S3dFz/uMJFxXFfDtJBZpYrw3bvaJYZnxwPu8zNzYrdxwmSQlJxuHO4CaZ9IgmBDxlvMx5B1IrdMFB7garK4MKGPe0a1ueZ4Ri5vlb84mtHkoLJR1QlePPziaorLNof/vSCX/9qz9uvHYetQ6l7NjeKu/cjWgn6PJIRCAsTIzZGVmeR4AQnscWPmbd/mdhcVYhrw+E+UpuCCFgaw8l4lgvNt29u8dGzvgj88PdamqpiCI7L6yV2cLz59p7NekE/JBqtuP5SkRmRsuPjncdtPVMf+cMf3vD29p6qjrQLw+kQaVr45ld3yARXL1ti8rSN4eaLFQ/vBzoT6S4FYUpUDSy6iiSP+ADTLtG14fu+lZ/ruZ7rd6oi8MhrLit4McUSgJhD+dk4aXyYiP5IiD0hO1JyBWsym9AlsiBLpEEKA8khZHxieZcV1NL4FCHZz2t5+SmUs4SF1hit57DR+IR0ecRzKC3J+LnhLdtuMRVWel011FX9xCXPUMRj/YiHKU1/cY4UB31VaWIyCAsxCZzzeF/W/fKMUUkpU9qhSEyarivuHykF2kh8KIK0lCWERyKfgnWKcB6KMydFjK6QqjgzSgMccM4RY2GZ55QwtS5BoLlgcKQs65gAOZYhQAl3srRdQEiNUPPzrxWVLAz7xWrJYrHEmAayxA7jzGSX87qmRErxFEZa2IcFmZJIkDUCOR/WckHESFVcIkIRYsHJSFECYEMA5xLBF0dKDMwYlwySmS1ekDHGVAVdIiUp27J5MLPOtVazAj2z+qMnJkvKDkREyMKBZw75VLq8DrqpkLoI6I/Om8L6Z35+S1NvtMZrjfBz6KcozXs9I1+ELEHhMUVCdITggIQxJTC0ruv5YKUxVY02dQkAnZceMrGsVyNJXsx8yMK7zymjlSmbGT4Tmdel5yGMFgqrDHXVIBrxvcKLnuu5nuu5nuu5nuu/nUopsV4XFGtBC5Ytx6oqnGYfihmlBHEKlt2GzdmGwzffcIw9i0WhWz8aUKQsuAvnXBF0fUAIjxSa4/FEZQo2RqvfcKbHceRwONC2DVWl6fuRuq5ZLFp2+/0cyjgjHbNEqdILKa3I3iJkRGqPmyLTVDCLMQqqqmOc9ihlyCnhY+R0GijYETC64njq6fsjXdvhXUDpcr7Ybg9Y69DKkFIJlzwej4BEasXoBtwk0MqRsmW1qtG6og+R5WJdelijIAncNBJDmDciM2nGTqqk0bWiH44cjztcmsi5YEr6fmAYBuq6KiYOYzgeT5ydnZESxCC4vPqcq80P+O/+xR/z2esfI3LFNNqyVasM3k9kEqdhR9/3HE8PjNOJb775FXVboSvNx48fudve8erTK7RUeD/hXKE7jEPB8yjpieFISplxcBgD3he2eN8P1HVDCIkvv/yCf/Nv/g03L18QY0BrRQj5Canyzdff8IMf/IiSKJUKovIJrVl00sfSSvGDL3/E+fkV7QJimkjJz6iTJcMwkFKgqiqGYUDIQF03Tw7xIqpX1HXN8XiYXfEeIeB4PHE8HuaQ2AohJFVVtjGFLO+DnDPew/nZghgzp+Mw5y3J2TkvWXTLcm7L5fWRUrNcrsoZN45zwKlGSsg5MQ4TVa1o2+5JHDdGz5lXAe8t9XJJJmHdSFWrebv1P1/fu3tXGqYDpB68hqaFFzfw6eviPHv9wnDWfEqMjugkpkoEtkwuEcKIlGuiXyFwvL0fmWzibAGT3eFDz//nl/93DtM9WQ9oZRn6hHWZ/gDdKvLuPcSYsQ6ahUdkuL2DX/8KXpzB3R5ODkwGf4BqCZulYPTlpN00hVW628PFF3B5pYnJ0p8yD3tQCfwAYgGVBlPBcV9c6snDp5/CYi14/z4z9bBqgJxxIbO+hk9/AMdjZncH/QmG4cjVjUCojNnA/giqLj8I29azvS+ivKklzkbGAQgwfZ25+WkFOL7+28jnfw/cBDlKPvki04SOu21P9JnTLpS14ShYbDKL8xJudlx4vMy0K0laZLSQfP6Tmr/+Mwd1RjUVqc80pgP26Fphg+dwt2W5aMlixLrA8lzz/oPnuBdlwHBuCV4SYmZ90XHcOj7/EiZbAsy2B4mzgmM/IYzn4y2cXVTUXQJVMAIulLDU5OB0yrz4SnDYOZKQdAvY3Qvufw1GSU47SZZlSpWAT35Qk2NEKouznsuXhqqSrC8k3VagmwXXn0C3EozTRDhKbu8Ddj9y87pDq5rDfUQ0mWGaqJTh7LLm7bcT608Fy03Nx4cjN+ctrz85ZxonjscDm7M1v/ruO158ksjJEGJgcj3v3x05u6zY7yLLVnN51dJ1DXe7B8ZDpD3THI+Rw8OBiURjBNevlvzlX7whSs+L15pMQ4gDTavQlWJ51rLd9cQsGB5Gtg8nNsuGF5/V3H0YECayutZ8++6BqmowRrG/FVQmfd+38nM913P9DpWUqRihJYVrTiip6TEDqoioWeA85Hgi5mF2K7vfciyIWRw1SFGBNJR9qjSHQz6GYsrZmVBCQGMs/65kLPkjUmG0oarqWUTPSFkCfjIJpRRaSZCBlETBqqRMRmO0pmkamqbFmOIuZg6DlKKEeabEU7hmimEOpYSq0mU1MpTvWVYtxdyUPYqwoThtRCJGU4J5xKNYDjGVQGopQUtJTpBzCaBJKRNTYcUXkViTEYQwEOaBQoyxOLV1CXxSM4c8z4GUIRSnTFmzzMQYy+9WOSGVIaZAzAmhFZWsqJsW09QoUwR2cuGwG9NgTF1WPhPz481o/fh6QUwOKKx1gYQ8lTBZXdE09YzaKdi6wghPxJDwLhG9IAVVhi+5sNJzUiAVAoOSVVlL1hXkhAyPg5bH58sVfIuYByExkZIlJkdOlpxdYTfKjJDp6e9yXuuU2jwFwT4K0M4V57lSZRBTgmcrtEoEZreWeHS4CIRUJBEQIRcxPZSsACUFaDWHdcl5C0CRYsK5iLMeHzwujsQYn67BTJ65+Xlu3DVkgVeRbOc13JSRogyyvC/Bo77uMOr5d/dzPddzPddzPdfvUuV5a1KIEmjfde0cnDmSc/ERZALjONF1S+p6QQySrl3Ng3vBcrmi7wvSZblazDzvULjXKRJDRs2CKtnjXCQny3K5xrswm0OKoF5MHSWw3nnL3d0t6/Wa9XqD0oJpcizUAmEUKUYOpy0XF+f40BNSZhw1Tb1BioJ8dGFJZWq0ElRGMI6WcpYQs3PZcTwM5AQXm/OydRrLbmlVtTgX6IeJ9fqMjx/v8d6hq4IpSbncXqYYfoyuaBtD07Q0TYezoWRBJeagSsdkPVlIqqrlcNjRLgUxON6/f0NWJ7TJdF2D1mVLFjKLrmOxWLF92HPcD1xcXFGbNefLT/mTP/6/cnnxCTkWl75EkoRAKkWYIuN0woeB7f4D+/2WYTrx7uNbqkbRjz2qKq4MO3myiAW3o2C3PTJNHufKlixYyILjaUDJCWcD4+Bmg1Hk937vZ/zb/9u/5R//439EShEpzYyE1EyTpW1b/tc//VP+xz/5E4qAXnCe5Zn+jR/9Nx8LXly/Yrk4Z7B3JQQ0OS4uLkuoqnPla0RDTB4QOOdYLpdPJqoYUzl7CgVZPZ33xtEihCYlmCZLXVfzOaj06FIKjClbrYjfynciE21gseiIMWK0BgLH40Bd1/N54ITWFeM4zlvGA69evcLaqfy7knOOkkBKCNHjgyUTZ6QRaC0xprjQy0bzf76+t4geXXF1U8PZNVyt4ac/k1QVXF40nJ15FkvJNCnG6Z7BO0yEu53nrNUMtuE0ObKcaCu432cImd2w5XAItFVL31uGCe4eiqgcbNmUzhJMA7WBb34NzaIIAsMAQsMUYDqWz3MjIODlZ5JaCKb7yOkB3DGz2PBoyuPXbwd+/EODkJHjCG/fw7qG9rzwxNdXEB4/XcKbb+HyJeQIaQu3A6hFplrCqy/Ah7Lc3q2KWB88vH+bWZ3BZgP393D7NVy+VDzcQrMMBAtvvgt88olEfIDVJRgFNluoIJ7gtIduAXcfE9MEh9MB2UJ6gP4d6DOwU8ZZ6F1Cpky3gOxh0SbqBVy/qPjrvxgZHMgIb38ZcCEjfE+IghfXCt0m9BUsTM3bN5bLM8H51ZKf/2KHGwU+Jtp1EYV/+IOO86bi7KxcgPttpqkk67UgkFFVxtQZOwkmOyFNps6Ji88bhsFhZxfbMRe0QEyQciYPkssbjf02IWIJORAycTyVELKrywXTfmTZCs5fSI5Hy7gP3H98g9Sas3Wg7SLHPrFYNoRNQjcwbSo+vhu4+WmDycU5qI1gOGY2usGeTrCC3W4kR0k6z/zZn73ly68Mk/cctx9IoTjq7ATrS8P7bweESKw2imnsiVWLFIKqjois6ZYCZSru73tWC4mhJaVxFjVS+RkWEv3RUtctk+vprmouLytsHJjGhPfQVIq+H/nqB5LtLqAXiXpZsRjh3ZuRzz+9YFw47OS+71v5uZ7ruX6HSshHp/i8EpdT4dalPIvDc+J7jkgGhAqQYwFS/BYnT0qDlBVKVCAMyASiuG9DKCt04lEYzbPLOxZhO8uEkiX405iapulmB7qc71ckxOJUQBQxOcRYMCBZFU61Uk9udqU0QsqZRV54eo/Ik5Q8IVhCdCjxm0BTqQQySVKanRdCzozrgoIpDv1MSo+3qYuLI5UVQyGZ1wBBGPMksCPmUE8JLoQitsfy/JaDSeljlBJlbVBAVRtEzkhVXpcUKGE4Iv9H7ongA1EMKFOY4zEHlDaYusZUNUIrksgIEWeXkKKuKqrZrZ8SEDJRF/wKshySUiqObXIRnIUsa6xN06KUxHvPNE0474gpkJMjR4sPYV6tLMHmQhgUoFVNZRqapqFrFzR1h5YVU/b44J4GEZDnLYESMoqYWZYpIIlkMQfEqoLKe8TwCyUQ6jEcV8zBsWI+dMWn/z42yCmmp2st12UzQUhZ1nyVQDIj/uehj1QaoRTK1EhVkVCECCFlYgYfE6O1DNOE80MJsw/uKeC2qpqC+lEapUz5upjIMSNRyML+Q4syzHHOMSpNZcY5dPW5nuu5nuu5nuu5fleqquqnQPnSx5RQ9WEobnBjzG+CFBEsFiv6vqcgpAU5CVISGFOx2+2KeSNnFos1p1OPlAnvA9NkZ0yGwfvEOI5cXFxCFvTDiDGG0+k0Y2HWCFH6wdevP0FKiXOWcZR0XcdicYbzPVIprLWMY896vUEqwel0JHhFpZcgBMGnIpqiUaoixWKE8N5xOo4slwvOzi6w04QPmcOhxzrHp59+ippzlkKIVHWD84VxHZMiy2JaUFLTdi0+bKnqBm9KP6ZNTQylt26aDnIxwQgXngTjmCKH44mH7R26EkSAnLDTRM4ZZydi0IhGEXzk8vyazeaSFAU/+sEX/Mkf/5+5OP+cHDJSa4JzmFrR947oLdaN3H58x+iOfPj4HYfTnmHsSXj6aWAcT5hYzjPT6AlxIubE2cpgp4BWjzlDuuBzUsaomrZdIoXjeJjIKaOU4l/+y3/J7//+zxCCskUpSw8vpeDy8oqf/OTv8c2332HtiGhA6ZIfVP6UjdU8L1pmMiILLs5fcPPiNW/fD3TtgofdkYK4mVgsOparloftLSkFTNUQQjEPtW2L9x6l1Bz2WRA+UjwG6HrapkZJM/fQGq0V0zQVA0zWs4guGfqRabLlLBQCzllWq/WT6G50ceEbXZPJDL1luTTzJoXheDySUslJOj8/fzor1HVDzp5pGhj6gaZtUKqgQZVWbDYb9vstj9vB/7n6/k70PLNOK7i8gPM1GJNYdoKbqzPaeuQ03NE0Kyoy0bcYXtFVDxgd8FFxsiNCwmmE0UZE/hWCTMrw81+PkKAfYfcA9gDrdY2uan75zYHNRUHKuAjjbbnn+6FgXR9GiBroAVlc8kYnYoSzNcQ9DDvIM7+90UWUH3pP9tDvQCrQK/jmW2gaWC3AR3j1SeGh//wv4TglkoXuHEQNScFyCd9+U74nCCqt6SfPxRmsz2BzNh/WcuGzt23g4wNUG3jYQ9dB02Sur4rb/+wzxYcPkW5VhgSnfRHvhYZpAhvgeITmEuIa1pewu50deA7qOrMwBUGjM0gPX/+Vw0+CzQuBPyaG+8SXf8QsXGR060kx07tAbu5ZXgpevtgglOTmuuLmqqEfTpi6uAq9tYilxfaRw1bQtS3gmKYSmlXW0SVGZ6RKnIYiErxcC1KG0zGTZcamwqTvlgJrYTomgk1kn7j+0hDxRJfYHTPVUrDbn6ii4LhL6CphJ4kNgrZTVFqTgaatud+duOxqPI7j/ciUAl5Emnbk47dw/koitUCIyN37A6bLVLXk8FB4ue/eTry46tgdLNoItrtEFoKPt4abTzU+92SZqGvBd28seHi4n2gqzbv/cOSTTxa44NntJpyFxY3i/i4hk8bJiR/8aMHkIk0Ddx8m2qZC1JLT5Dn+Yo9IAlMnXr6q8S5w2ErsJLh50dGPIx+/Kc/T9bnh4X7iw/2JV5//3W/253qu5/pdrLLyWZhyuQzwchGSy6Tdk0XheQsZCoIDSliL0EV0znpOdS9iekZCViUcUuZZ9GVeuStfJ9DF0Z5ycT/HIlwbU1PXLQLmxiWRcgCb5wT1REizyyLI3wqRrFHalPshmG+7/E4qaSWJnAMhFq52jH52qmsevRYFVyKQRqGUeWrGYiwCuiA/CfoqFIH90W0vhSDN3zfnTJ4Z7OW5kvM6ZH7iSxaRWZJyCaFsq5ZUJgcobYBMTmUrQKT8xCosdJJMikWMF9ISUuARfSKVehoipJQIMRQhXyhyyiBAa0Ndt9S2JSZLDImoMtLM/Qii8OmrBm1qmrqlbUojae3IOE44VzjhMXpicECcrx9BjOX6kVLPoT4VRlc0TUfbLqhNg5SanCUhujJk8AXP8siuL0K6QEuJNJqoIjkmgkikOOOBlEHNv9szogS3zpsD8LgCXbBBMcbCO1fFqSSkoqrKsKIwOsvwqKyxzlsGlG0AYyRC6hK6Kqp5SCDJWZbBNxnnItM0MdqBkz3hnZ0RMeWxqrpcTwApxtnwk1FSEZhfo3mDIaSItY5Rj4U3/1zP9VzP9VzP9Vy/M+VcCZcv5oKMUsWQYq1js9nw8eNHzs/PMcZAFgy9ZeineROzRmvF8XBESOi6FdvtjrbtWC4XxJAxpqU/PaCNYrU6K//edChZXM+Hw5GUi7v35uYlb958R86ZcRznDcvEMAyzw1hwfnaBEDV1rXn3YUfbLOjaJc4Fgs8o1fDwsKXSjtLiKirToZShbRacju8ZB8tisSgbfkmQk8S6wDBMVFVLXXU4G0kxsl6vmcYjQmhizFxdXfP+/VtWZ0tsLP31OFqMkRwPJ6Jfc352CSnTtgvIhiDH+ayhadoO5zzWRepK8/bjPT5MCJFoas3pONB13dPjzhnqagFZkaKiNivOr6/53/2zf8VnL/8eZIN3PZpIIuB8Ku5zP3F7/56//frnHA4PfP3mV8QZGZJVYJpGUMX9vznfcDztiUkgpeGwH1CyouvWCCE4HveEEGmaluOxp+89StYsl2c83N6yWnb85Kc/oqrMjEcRGGOeNjMFiuurG3741Q9x00ily0Ajp4BQj2ejVDZgeaKjs+jO+dEPf8abd78kJUFdN9R1zfl5CT2VUlJXVdHl6paua3HOorWeueTl9VkuVwgh2O/3GFNhTI33lnGcuLy6eAoIXS6XhOCoFxWQWSwW9McTlalJKXHqTyyXa+wUngxNy+WCtlnQNA3TNNG2HSHEJ0H85uZFMRbVNdZaUsqsVitWq479oWw2PDxsWW/Wc/ApvH/7nh/+6Cuur27m4dV/vr63iP6H/xiOW1guwNSwXsGqbVi1HSkGSFcY1dIPe6w35NgRgqIxHTkl3t5v+bAD64vru64grQuO5M0buHsHqoJpxppUXTm4tJVh8jDdwssLuLosKJDdfnaqS6AFVAkKDR7W13B7C9bCYl3wM+qi8M6PD3DYwuUVT1iRbgGVgMMd2C2MF0X0NhUMU+Gsmwbw8OlXIAX0rgwDpIKLiyJwD2Nmyp7VCra7cjEmAXaCu4fyPKYMi0s4HIpInzM83GXGoeBdkJHuHM5vYPsRqgrONvD+TblP+y3lh9Mc7Hq/g7opCJrDHoYtvP4UxDl880sQCaIEs8wMfUY5aK9AK4H3ma4T1FVBpxz2nkEJRp9o2oQLE8eD4IsvG4QY8CnQCDjtA8tW8O47kBI++RSOx8jbXwk8mfMvDQJH9In9NiM1pKBwo2DqDX6SZBHoFpkXryvaBrb3iXe/Ttg+8umPM8uV5e69RGTB4jKxWip2x5FVpcjOlCC1VeDhziFbR1Un3r+Db7/zaG24//DA5qXgwzcQIsiqTFFNfWJ9pjjZSDaJwZQf9u3S0Daavp8gRCKO48GwWTXloC4SZ2eZ9w8jZ11itYC7W9BdRo6J0y7y+itNRDE5y/1Hj9KKL75s2O0nhIGLM8U4ZPY7BxouXy05HD2RAWc1y3WFaRXRR1JuSG5BjnsWK8XNywusD4j3a+7eP/D3/uAFWh5xQ439bs9h/33fyc/1XM/1u1UzSw5ZmM1Zza7gNOM18uxWB60MUhZut1QaIXUJREyCFEXR3hVzMGnZzpECtP6NuxsySuXZpZtIyc9udomUGq0MxlQIyrpeCJ7KV6RUXM4xxtkSUViIZS1VItClmX1kmIt5CCAKPzunQEqFb13YkoEsH5l/xRn/+HdjFHXdAqKEY5LIWSNkCd703hYMXNWScyYET0oeciblUB6z1LOLSCCVQAlFJM+fH2eXuySmRNPUtG07o2kEQj46kjMp+vIaoVCyBKqW2y0DBmcn8ox+MboiSF9QIS4gZWlam6ahMoZKmpnhqOa1y4lxLM5z5x7Z4JqqaumWG0zdFSG4bpFCM44T1tp5qBDmx11c61oakkiEXNY1yRQkjir8dClL+GddF1SNUhrRCKxTs1mivK5qFo0LK19QV9U8DMh4IjLK0mTlR6e8wkcwqaDp8iyKPwaJPgrpOeentU4189IThRUaQhG084yfKdepemK35yyJWTxtUORUhP2cBc6VzQxryzrwZB3OBZwPgMLocnta6SfOKCljlMYJRcoRJdS8JVCXgcbsiJkmi9bPwaLP9VzP9VzP9Vy/S2VtYLlckmIgENG65nQcSDGhlKHrloQQqesGJQ1iRqH44FGqYhwnlssVh8MB7xwIxcPDnmksIvB6vQIkOQk+3t7PgmKD9wPjWEyDRtd4H/jkk9fc398jhCSEMPPUeQpq9N6zWDRsNjf8xV/8/3AuIGWN1i339w9oVdNUiq415CxZrRb4WETuVy+vOJ1G2naJd4HFYkU8ZOraMI4T6+WGbtHN+Uv6CbE4Tp62XbPd7p+weYvlomy+Zk1V1Uw2cDqNDH3PotlwdnaJlIbiWihovabuaJqWEAOnvifEgcPhyDQOhfVtJEJmzs7OWXYLdrsdSmiuX7xAiYq2XuArhRvhn/2r/wOff/pjctBIAad+z7d/8wsuL88wlcZay2Qn9oct7z+84e7+Lcdhhw0OoxVZRiY70HUtpqq4uLjk493HOdgVjm5g0ZXt2NOpuL9BlhzCkMoZQRXTjFKan/z0J1xfX2GMJgQ/u6wzWpv5NhVt25ZsJHLJIyrNO+T0W/1wfhLQS9aU4cc/+j3+n/+v/wdnZxdMH4qY75wjhMA49YxjQamsVoVP/ohVNKaibVqEkJymnouLC4RQBB9mTGLF+fmSpq7pTwOy0eQcUEoxDANNU3F/f8fZ5qIYfPu+GH3aIpanGNHKcDwOSAltu0SrCiU1Pjj2+wOXl5e0bctut8PaYngp13ZkGC3GNFjrONtcEnxktSqYmrOzC46HE03TzmfE/3x9bxFdKri8FvzBTys+3FmUFFyev2TdXrKoV0zuPZO7Z7e7Y9l+isgdUxoYw3fUdcXbu5HbLTxsC5YF4NdvoD8UcR4DMcPQQ4pFGEY4dscHwlj+bazL17YdiB6++BJCgsMITsCnn8H2HryDbgU+FdEaCXU3O+lNOUYf9uVCubqAbl1c4n0CThBqeBDgezg9QLwpAr9QxQWuVLm9FUU8j7qI4jEU4f3uY/meMYKzRfQ2TWHKkwVdk9msIQXQunydFAUVE2N5rt99B/YEn7yGpgZdgXXluk8RVufz4zIwjEWs1x3oNez64kxPFUz3gqQz6w3cXJbPtwM83OfiuN9IEoIXrxbUzchyVfPn//7Af/jFHtfDi1eayfY45/l4B8sG7Ah378vAwvmEqgVrU1O/d0iTOQaL3WUaBadtGWQoLfjmW8vDXeTiQrC5FCTXYA8Ro2N5TJeZl59phlNA64a2m2iaju/uJ77bRtoWzrrEei3Y7SIpKuQi0wc4u1CM1qJrzeeft/z8l4HvvkllrXorkAu4/Ri5eGmoDKSDZDwkVJO5vKxYrwTjwXP+csWHdyPbj4GLc827dwcmn+muMj5OHMZMIyWtluw/RlYvQYpEtVYg4eKs4uJ8gRKOus1MY+K8ari8bNk/7Lh+tWT7EPhwd+LhNJIr2N+C6RLn5x0iwP37E4d9pjGB65c3HPd3TNPA8TTx619api1Mp8hiKeiHE9cvDKP9vu/k53qu5/pdqhTL2mcx/kpy1vPHgZxAyCIsC8Ec7CkKqUUqhJTkFAk5E3MJOZJydgPnhEjMzugiGJfVT4cQZU2xCN7FAV7XLZVpiwNczFw6EZ++rvwp/I48u4TJFUo2GNOhZEXOYkbH+OJ+F7k4eXIkRov1Pd4NTwFNRtWQFVIUN72S5U9VVU8NklIea4sYL4QgpvI4japmh7dAqeJIDqGErQbnqUxNZeqn+y2VxMyud6UiyPK8m6qauX8SqQRCGVKSxBBgdjvnVJpLrRVKamKKeG9nLvqEVJqqakhEQg6kmVUvpaSqNFpCpQqWxJiCi6nrmqZp5hXhjHMJCBjdsOhWbDZXmGZRgjmzwHuHdRPDeCLnVDib00jbNiy7FvW4bUAkJ09KorzOyqB1RVVVNE1F01YoIZFCYUlI+eh+B/KMbsvMGwYNZEPMJUhWJI0gIQWkLIhJEpOikjUZMwfBpich3TnHNE0MwwCUA195bWvgEbUD2ghimMNVRVlfVcpA3WFkQbe4UER2IVVxoaNwPuLcSEpFHPe+bO8JWUGOpKxAmqfrmixQomzapRjmkY1g0ZZhBQg8M+InJawLyOdf3s/1XM/1XM/1XL9TFXzpUbfHoXCblSHETNMsMLpGzqLfoisD+tPxyPF0YrHo0NrgnOfy8hrvE/t9wbn0J4sSLXYKsFYzfzoxTY5xHAihbId2rZgFa8X52YZpDKQoOZ0GLi+vOZ1OnE4nhBBM08Q0WXY7y2bdU5matr3GuYkcDW19TlOXHqiuBTHCYtkRk+Lu4z3Wlt5puVjDomwwpgS77YEM1G1H163Y7fZoVQwP2mi0aogpYu1E2y34ePeR8/Mz+r4nxExZwnzEMmpeXN+wWp3BfOYpQnERjVP0xJhAFKSvtVPZAhWZFy8usUGic8Xx2EMWpbefMZY5SZbdBf/6X/5feP3qB8SgkUIy9luWK8nDw6/5q79+y49//BNMteCbb77l3Yc3hDxxmnb00wGlJc1y9XTW8MFxdr7hdOrn7KFyRru6PCf4xOnYU9dNQSrGOCN81JOZCKGpq4Y/+qO/T1UZYvRIqaiq6imXybuywVp6T/CubOiKRMm24pH5LR4Xlp8+FsLw+adf8dMf/4y//tv/L8ZUhBC4v7/H2on1ZknTnDPZASkFfT/gXMEKbzZnNHXDbncA5NMgQzSC7XbLcrmELDkeT4DgcDiwXHZPzu+SpxVxzmNUg1aGrDTBJ4ZhwhjNcrkmhIIAjaEgoofBgshF5B9HlBKM40BVlSDdcXQ0dYezHqXAmIauU9zf36FUeT8pJej9hBQV3k1/53v4e4vo11eGTbtm0XWsL77l1fkKrQcepjscnxPjiWP/kZN1jOFbopcch8BgA6oOJOA4wHZfDjVtX2gzzkJvIVlYXsDOF563O8In16BUhgayhbtjEaSHPXz4AC+/KmalWsGUym1l4LCDF6+grguaJfqCNzn0EIcixosA0hQh+Oyq/D12wMtyISkNvoY0gvdwdl6CTPstKAnVAONHyF0RwBdnhafuehj3oJZFbG+Xs1AeoVJgdAk6NRIuVlDVRXiXojyms3NYb+D0rjxWH+DhAYKD7QTLDU9fr4BlC4umDB22h4KImd4W4frFDfizzPYILz+BuzsQFXz5SvPhTaTbZLTJDGPkFx92ZARDcNR1GU74CaYp8N13J5YLWK8EbZeLCC0ypw9gM/ztm4mqSVTnmbATdFozCcfHe0GymWpdDow+C8xScrKJjZLcNEsGrzjaW4SGep0JQXL7VnJ2logSbt87dg8R0YCWsDnrSDniCdjoWa8V7z8mkh8JAtyY+auvDwQSyQhMUzYnPv2so20U7z5ONF3GTwUPq6Rgv/X0NtM2cLVZI24Fg4dzk/AyI2pBP2Y2eg7WXSVOu4xZKExXY8PEq1eS80VCpA6XBN899Kw3gbYTrJqW47BntJHxrqxerc4FD3flh3+QnnGX6fvA/uPA1QvBhGe9trjk+eGPb/jFd99i1IbRj1RXkm/uHpAfIpcXBqRkc/m8Ev5cz/Vc/2nlpIqpN4mZTf3otp3RJEmUX2q5uM2lVAgKb1BIiZK+hDCKNAeOFq6gSIksJSkWF3ua8RxQUBlKKrTSZANKVrTN4onVnWekxuPKaJ6F0VICJavibqZGyRZjOqQwpAjBF6e8UAJELO7x5PFhxPmR4G0Ji8mgMESRQUqkNChVo1SFMQaty+FCyjIECLFgXR4DToUsm1ZSlnVApcCJjPMWbz3Be2IdqU2DRjy5+csgYHbrk9GmrFjGGJFKIWasToq/Qd0IoZBSFGyNAoLDOUv0AW8dpgJ0CdjMIeJTJOU5CDMZ6kqTjCJLVQIyFbPjvKZpIlLqwpasDE2zoG2XtM2SqlkipCAEj3OWEDzeW4ah53DYkcm0bYU2EqNLQJBSJVAzzeu0RQwvGJ8ikCe0KZgcRAIRCxJndumXBl3NAaQNIhsCihgTgjCH0kZAUpuWulpS18viLppF9DiHtf72n8eAruJwl+hKFuRNTAhRng+tCm9RqXLJS6GISLLIqJzK9oOU8/WpSDETYgkFTakMLaQwSMAYRVPXZStOVQVtlFMJG82JGCJaKYyq0XMAqQ8BKeXMfiyhYt9nZfS5nuu5nuu5nuu5/tupynSkJBkHR0qZafI0dUuMgdvbjyiluDi/mtnj+xJG7i1CLBBCsNlsgHmrMh/JSVOZjmGwnJ9fIdA0TUtVaY7HI3Vd45x72n57dCyPo8PagFIV0zSyfdjPIaeJujYFPTdOnG3Wcy6NxNlIVa1YL1cc929wwHJZ4+dN0GmSDMOAnHOMyG7OszFYa9lszri9nVgtN6QUOR0Hht5yfrbAeo8xGuc8WqsZq1KzWCwAWK839MfCyRZSUpkKsZT87Pf/4AlLkjOkENFKE0PZILRuIqSIlIKmqdl/u+PDh/dQbfHxgIg1tWkxuuby4hqlNPttz6K95J//9/+C6+tPGIdITiNnq5qxv+M4fuD8QvM//y9/wRc/eMFxOLA73PH1t79C6kI16FYVpqqY7Mh+f8DUqrjSc+Z4PBFCZrPZUOmK/jTRNh2Xl9cFKZM8ZEmKJZ8qZ4H3CSjXzMX5GTF6mmaJlAWlIoUuuUHzFrLWmvV6jTaaHBNSF3ynyr8loovfFtLLIERKwz/9J/+cX3/7l2w2Z+wPd1hbwkC1UmQxb+JKgdaKul4xDAMheI4+kFJmuVwxDBPWOhaLBdfXN5xOx/msVMxU/ak4yq0d6RYN1toigA8TsquQ0syCueDy4gptFMYYci4moRAiOcMwjCyXHV23IATPw/aBnDOb9RneB46HkapqyESmaSwhpSHR1B0pAkgW/3/2/qzJ0ixLz8Oevfc3n9HHCI+InKtraIAgmqQRNBIkRYHDjf6TfpAuaJRMMklGM12INBFiA41ugN1V1VWZWZEx+Hjm84170MXa53gUGsaqC90QeZaZp2e4n+GbTsT+3vWu5x1NaJo6Tu4q/lD90SL6bJTRNBtun7Z0FuxsQ1dvSNOED8vfsNoNXM4CDnCuZ1vDwxKaDrY7cVsfECZlBk2A0USCQDcb8ImI6Odn4iRvGwhaHOhJBlUhaJXeQ6ng+gxGuTiwbSJ4mKEHPLhO3Nl1K+J5kUlwlk/EdT6ewHIpmI/HB0GvTKfyPBxkc0HBqNjwGU9FpNYayAQj4wbhkpdTeW7XyjW4a0S47zoRxL/6UrFaBtZP8piuEETLZinNBOfleCQ9zCby3AND3Sh5n6e9YG6yDH769Yj1pibLNZNSOmrOemlG1ODXMDoTjvu+lvcYn4nLff0O9AXsX1rOLhXOgw4ltt7z+BbSSaDr4eWl5v3bgN8HwiBomaGDV59lNE2PTgL399AbwGvuv3eoEpIcQhv48HGg20NSwvkLaVI8PHiaQZHlinYbeP/oGUYLXt3MOJtd81d/uUSlHbV2vHij6VxPu0v5+KsBLhSXl4HrK1jtahnJUDC7TsiNobMDZZWQjxz7xrPZBHwn9+83X8FmFwimJh8rso2EsAYN9QAzpRkCrB4dWQaLhycmowrrBh43NSoNeKtwFiafBzoHzQDjuWZ6Act1z27lufOB7DPH453lcW0ZLKQVbO5hNK4ZFSmkKUPoqQqN6hXtypFliosXhqH33C6WJBpWW1juQGc1aZJzWz+xbD3nk4TyKpBlltXCk6aw7npmoywG1J3qVKc61e+X1nkUq0NcK6nIN9eoYEGFuIAA6xUKCVlMlYy/Ga1xGoz28g8TInxL6neIbvC4mHIHIRMgisJJSpIUcTGfHQMhtRaUyjAMWOeOIrpSBq1SUClGlSSmJEkKEdaNEee8EhRNCA5Cj/Md/VDT9w3D0OKdRauUYMLRga91SprmpGkWRx1BnBjhKHJbJ4uIJBGneN/3GCNxO8ZEHrmXf3+6tsdaj80sWVqQuByMjmFAgm0JkrJ6dGjoJEHZACrFWsfQC9ZDB0eWC/vdaMHHBA/WOZztMVrjB4sLYgcPQRw4wUBwGjcMDJ3GEAjO0w8O761wIIuSNM3wLpBlGWVZkmUjsmxEno/jYraVwCo70LZ71psl+/2WqirQJmASCZoyJiV4WcQHr+OUARADQ/u+o+sMWskEwDAIXsd7i4/nWGtpsBiTYHSKUeL48S4wKIu3/ZEBORlfkOUj0qRCSO7DkVF/+AKOwvpBSE8SEycVHFp7VKJJE+GXy5UJSkVkTtAY70iTcGwySRhSYOg7Diz6A5IoSXIUGVkGVVlSFMLq19rgvAQqGQXeecjAJLmcS2uPr58YuWFxPggn/1SnOtWpTnWqU/1oSquEzXqHc3B2dsFut2U0kvDFptlzfX0l6wrnj2GRMqE48PS0E0Rh0Ay9oGCKoqAsRuz3NVU1om0lQNTanqoaxXWRpu8sbdszHk+4OL/k/uGRq8tLnh4XTKcz6nrHaDRFaUPXtVFst2ijKauczdpSFCM+vL9l6DWJGbHdbMnzkn29BzxpOj2udx4fFmRZQdcO7F2L0YZyVjEaTeJ6NGW9XlOWJUmSslrtODu74vHxkSxLGIaBNDWMx1M2mxVn8wpXKDabmiSJE6Iefv6TP8V5j1EGZRRGZeAc2kjouw+BruvY7Das1yt2+w2T6Yjbpx9QxoqZt5jx9PQojYLecn31mn/yn//XnJ+94bvffsdk/IKvvvgZi8c77u+/Z7V9x7/4l/8jNuz5q3/5P6P0iL4LdH1Nu9sz2A6PxeFQAabTMUmSsVqu2Ww3bDc7qqoizwryTNbCk8mM29tb6npPVVV0XUPfe7nn0NB1nr6zKBIuri6AQ0aU3MhpLWYScaAHTJpydXmNMZqmbRhVCdZ2BA9Jmkft3BwW8xzQLs4Fbl6+5uuvfsJf/stbhsGSJKkwyLsW53ryIo3Bp56iSHHOsdvtsYNgiIqikCZIXrDd7uJUg4jdg+3J84TziwuUcuR5Kkah4GnbFk1Aq5wsk3DVEDyz2QxjNPt6T5omdF1PlmURM+PihLOhKEr2tTSOqqpitdrQ9z0hQNN0jCcjvA84F7i4uKJpG9rNmr6zTCZzrO1lSvcP1B8torduz8MDfPX5jNVu4P1tIE9Ba8tvfwCroGlgv4WigvsHEczXK3h/L66fvociFYxJ10M7ABZcFJPrndyj5ymUpYjT+y3Yp8gw93D7BGUOLy8hG8FyIa8VJ6MZjyEphGPuLAwNuBSubqQztd5C08pzhhbsWtzkwYEGZucimD8twLVAJvvlHLy8gYd72A7iKn/9hTw2yeDDW8GcaC1BoB9uJRj04SmwWcH6AUIKT2s4aBFZDm+uwKSw2cpxjphWSODVpQjRm4/ys+k57JoGpwJPS0eWwLjK2A8DxgRmM9mutofza3h6kODO8SUUGnQOuYbNArwJXJ1BmjWkBWQTyMZyjJZPgd2DTACklYj/aQbLp0G471bCX+0D0AVUKkz22TW0OSR5oG9BmUAxlwmE8RQuLxWTasSvfr1jtwlUlefDasHI5bikZz6BZm9JK6jGhvSVYrlUTK891VgaH6NRYLVx2E6x3FjmI0tRaBm37yxaKZIkUFWw+G10/SvFu28DT2Vgu5PznM5gPIebl4rxJOOXv6qpG+hcIDUdZIFqHAg+oXoNy1vAW16/SvlwOzCdwdBZJhNot7B5Cvx2D8Z4+YspUzwtA24beLyD2dzz859Peft+yYcPgYtzce4lIfD6yvDdb2Q/slTTu0CaBO6fPIaGbd8y9Jp6v+XivKQYtSQqo+t6zs8ydhuLtaeR8FOd6lR/t4zOIFhssPIXvAatZApG0BguhocGBq9QWpPqRBy32uBNQFsroYkqoDRgRTRVIH+hHisceYYgzt0kSUjTnCRN4hincMWVClhncd5HN7aM/YmkaNCkaJ2hdS6ieuSaqygk++DxDAQGnBP3S993MoIXIE0SCQ31h/BPInNRyVrxYFeBGE6qjk7nru0g0yjlcC6glCTZOydJ80prUJqu7wk+CDdb12R5jtYJRN674GG8PEcpNLLPSjncYKNYHtnyOokhkyHiPmRdkuiERGk0CuVjoGd0gBujSZMEo2R0te9bnHWCHbESWGUis9ypQJJkJElOkgg/UKtEJgy8x9qOtmvY73c424n4rAJahfjvakImlniyDLl5GxwheKwdEMyLxQ49nXYQDP1gGWwfETle3NzGYEyGNilKGxKToZTDJEMMkAJ8SlVMGY1nKDK0yuLkgoMgfH8QZMyBCd/3suhVSlw13nswIbrkU1KTHkOPgvdxykImMJLg0Ib4mZDzKdeSi9MTSQz+MugkwSFOhyyTpkyaZGgVcDiMFmd+YmVbURpPiNe/ZnAOZ+XYikv/JKKf6lSnOtWpTvVjqiQzhMEzGo8iq9yCEgRLWY0YrGW9WVLXNWUx4urqmhACu+0epQxZmtP3lu12T54VVOWYEALz+RznnLiGs5K27bi+uqFuakHquZbdrubFixsxBaiUphWs4XQ6Jp9P2Kw6wNDsHBcX5yifsl23zGcF280Ds1nON9/8CavViqLIKcqU7W5DXe+4ublmvd6gdUJZFmw3O15cX/Dx4x3T6RTvYbdtqPc9203D9YsrimJM0yzxXlGWI6pqRJpuaOoWH2AcCobOURXntF2gafcEOgbnMaYgS2Yk+hrCmKAStPagA0p7MJrClIJZ9g6lHcvNLdY35GXGeHzOy5tL1o973rz6giyb0jYd+33Nf/1f/h8YT6csN4/8D/+f/56f/fRPGfwjRZHx/d3f8OHjd7S+Q6WG++UDdf2R4DUm1yjrZIo1LRiCx1tPVUmTIxAwqWFwA9fn12zWW/bNhvFoRDdsCfRoIzzwfnC0jaVtLEZr1qs9XdujMPzww/e8unlJkkzi+tkTvCWTRbrcr+EYT0vWq0WcdtVoI+x65wLaZKBk+lOWo5LHZAycza74z/+T/4Z9veXb7/+azeYRHwI+QNc58mLMZrMn+MCq24hLPiiKYnTc19FoRFnmKAXL5SMhdOx2PYMdgBEhOEyiubo6Zxh68qykyAeslaaHNooyBpd2fR8nog/Il568yI/Ym/V6S1akKAVt08v+6S2BQJonEe+oyfOUzWZDWYqxJnjPbDojhEBVVux2IuT/wc/wH/thN5lwwAfvcAFu7yXwcreD+3ciXK8f5J40ycRdvl6KyJ1XMBpzdAd3tbDQmy2YHGyQgM+2kucPg2BM9hsR18fjiFDRMBnBdC4M9LaGYOFsKm73xgpCZTYTjvh8Jrz1bi+O7n0T3xdQKRDdd94LLuX6hbjUHx6Ec65KCAm0YkYid3A+lffYruSmvCxgvxdWeruQ9x1aCV4djaCSrATWK2G5v3gB778V3vrsK3HIN7W4j7sN7DaQT+V4973haeOwTvZ/lMLf/qWHcWS8A4tNj1ESSnp1DR8GcdQPVvaLICJ4X8D8BYwruH2ExIOdQ5t6dh1Mz4Acnp6gs4HiUu7/ikK475sNTMeeroPVRpzp6RkUo8DllQj8y5U0G84uhNvet/DwCP0OfvInkBpPkrZcXhref7B0XWDxCDrtqMYwv5DrqR1gv3fc3TvUFFoL9SPso5gfPAQM3ln2Lexqz9B1vDw3tBtLqkEFBUng9haGTYBBBInpGaw/QnEGfQ3vP1hevbTMRgl1Z8nH0HSW+RkkJuHDrePqCsbnmqFPeHGjWK6hHxzNXiYmqrHi8SFgW/jyH0BZQ54ldNZy+WXKx3cOk2qGvsN2gb6R64YCssSjgmM6hmpWcnfXMPjAZBLY/hY6rTGZBOe9epMzKgqWW0tIHJM4tWB7j05PN+KnOtWp/m4FJwJxYhQBJ4slJT5pYzzOiRNdKU3QiXCelYkBmJBlCT4khN6LY90rtDeEIO4K59wzxiRiLJwXBIYxCSbJyLICk2QEwHnP4Cw2cssH67Fe4b2OYnJK8AbrROgHg1IpRhtARTO8FyE1aAbv6O1A23Yxgd2RaBkBDV5Y5UabKJhaETBJY4L7gB06XN+jg8KEBBU0ygYCGq/AZAng6fuGrncMVvZZDoWiHix4K2ufbhBRNZOAS6UUru9jNotCRRHY49FKEZQnKE+S5eRFLuK89yQ6J9EDeVKgrCNVOWnIICSkSYpJM+FAGkWSJZjE4FHYvqfrOvp+wA5OhHh/wOaAsoYsInScs3jbERgY+i3O7iDUEBq0GoSzniRkpiBLRiRpRZoemhmxQZ1ICBUhvp9V2GHAh14aGNYR6PDBo0xCokbC2acEXZLkJShhuashwfsErUeURUVZnqEpSJNMQqW8J+hE3idokjQjoLDeYZKUJASUUSiDsC6tJk9SsjRHxYYG2uOtA63jtQieyOH3nr53EkDqD7gjQfJoDcboOJWgCFpGhlOjSbVGK4V3njTJSFIJtTIJ+DDEqQF/bNTEXg5+EPa+MScU26lOdapTnepUP6babJfMZmc479nudjH8UNYYznmss2K88BJCrkiYTKZ4ZzBGTAxJkkGo2dc14/FEgtbtIMJ2UWCtZ+gFaTeqprRty9nZBO+fMDrh9u4O5wTd++azrxl6x3az5eWL17TNnnrXk6iSz1694K//5m/JkhF/8ie/4OPHD9zdfeTNm1c4b3laPFBVOWUpAfXD4ESsjWaD/X7PMAxMp3MeH0XMHVVzqqriw4f3vHnziu2mpm89b958Tr1vIirSEFxgv++pqpIkzeltR9svqEaK/b4FO+Mf/Dv/mPH4C4yeR5RgI2xo5WQ6sxtw1oOG3u5phiWPq3dkheLli89o6pbJ9JI3b77CDmLSePv2B/6/f/7PGY8r5mczrm8q/od/+n/m8rfnvHj1is1my9u3vxN8ow9stju0kgnNJEnp7cDl5QXb3Q6lEupmG0MtU9I8QxnF5YsL2qGhdy2jUYWnZ7PbYlLN+WSGd4amtjz0a8qyQpGTppCYMV275Te//Vv+g3/v38d5h1aaxIgQbrSRdbcCrRxVmfH9d98xn89RiYRxZiqgVUqWKIKS+xyCilOaIYa5aubTl5T5BUaVvHjxmt7tWK83tF3gLB3Rb3cyqasNs+mYpm4jv90QguReNc2eEBxZrkjSBGsteVkAMsHqWkdVVRijsdaTZSWJga5d03YtRVmQpOlRPC/LnGEYZDpZicgOZfxdKVhID+O8ZLPdkucZk+kY63tQnuVywTBYynLOarU6hrDOZjO6rqfrehKT/sHP8B8tom/28LiEt2937PYiUCsPP3wf8SYrcS3rVFAmeBGHlQHbwDZAs4CmR5xrpQjnIYFJBY0SUfr8QkTf1QJu30E1h7NzcXmjhe1dD4JhGUby3DSV9+o92FIcxuUIKheF10RQL7s7EVJdLoL5bCIBnsFH7MtERPw8ho9SxmsKcdHfPcCkFMyKj+K0PpP3ns6hdzA/F5f5eCROc41hUqYo0zKdK66uUrwdWDwEQiJu9TQRfruNjHa7Ex768tbhFbx4I+8zu4CHW+gGwbTMZ3JcdnsRutVaXP6ukYBVAhQjEbw3W7g4F278bA6pEmF8+xFWqxhemoPqoCjlOD/cS8NgPJd9JIUqyI3jk/acz8VxHXwgL6RRcv8RLq/lvYKHpw3Yvbzv7RN8/qVlQNixvRVmfFPDZKJ5fPS0A1SliNNtJyJx00JqpOEwWEH12M4zOxO34ago2PgabxyDg2mheHkzYbva0tcBtZfrzHvpm1TXcH0ljY0ih+A1Ny9nPG6fIgP30DiwzC7kGk4rx3oDw/fSNNnvoF7F5kMSmH8pnH3bK7I8UFSWkkBVOYL2ZJXHozg/10xGCrzi6T7w6o0mSxMUA7fvaja7QDaGF3O48wqywHQqozrrzY6PtzvyVHP3Q8/8UjMtPGdTQ9+7P/ajfKpTnepHVCFyqgM+ug0GQDgcR3FRCf9OGwkaOgQfeh/HpqI7QUIZFS4ItiQECck5vhfy97SWeEgJJ1XmGFLqkW1xToR2ay3WitjrvAjkISBucwzOERdhDqNTcRrrAGgRQH3AeYd1jsFa4QFqhUkMiTESTqQ1SaIwWrjlAY8PA36wDP3A0ElokrdOEBwojNLYEPC+p+8h4HC+lwaAFSY8Slz7QYH3Du8CLgxY5xmsw5jI7VOKEJEgEoAEyhyCXMX9nGYZaZJKoCYKo1NSk5GnJdrL8dQqIkmKKh5LMKkmTdOIG3FY6+l7S9f1EbFj0Ca6oYPDB0c/tNTNNt6wBZRyeNehsCRGpvjIUoq8YDyaCjs9LUlMIZz8iO8hBEymaVtH3zm8H+jaAe97ipEsPq21DINw8rQ2uKDE/eKlUSP2bzl+1oNJC6bTMUVexSDagjSVZapzjr7txKmOjm78gDEpSRIiykW+Cy4oFZQPKk45yLWu4oimD+EYYORtiOFE/vj8YbCfIIaUYFoMESckEwFJIjezKn7OlDGfTCI4AsKDFxa+4A5VRM7ES+h5gvZUpzrVqU51qlP9KGq12jAez/AuMJlMBfOxWrDd7pnORuR5xna7osgrsqSga3u83zAajTHGsNk88vj4xM3NDXlesFwsybKMoe/57W9+yzfffEORl9T7j1RVhcLw8sUr+r5n8bRkv6/Z1y15XnJ3d8f11QvKomJUVSgFk8mEptmxXK04Pz/j4vKcxXLBz372M1arJft6AwratpGJyEQznZ6xXq+ZTAT/d3//iMKw2+1Ik5ztds1kMqZrJQum63pevXpN3Qg/XWnNx48fmUwmpGnKcrng/PyCqhqhVKAoM3aLLZPJhMFumc8v+Jt/+Y7/6j9/ybic4Z0XnIn3R7538DAMlj7mzxSFHOfl8omLqxnv3r9jGAZe39zw3/+//p+s1xu+/uonrHYL/up/+Rd8/fUXrDdLvvjyDXW/5nHVcb+453BPNJ3OeXp6oixLmlrMQev1mjwv8D7QNC1lVTIqS7bbLZP5nL7vaZqG8XgsmJGqxA4DKkkoihzn4OLinGGAzeYDeZ7QNJbEZJyfj1mtthit6PYtwQdcP4DWFEUm69rBkiTmeHs2Gk3o2oH1eoNJcrTOyfNK8p+U4jDp+1wepRKs9YwnU6aTGX1vyQpF2zbkeUbT9ngvZpAsy7m8vKTrhOEuPHwd3ecLssyQJAbnHGmWEAiMRiN2ux3DMFAUBavViiyTANPpdMrQ2zgwLI505yxlWRBCoO8HnLOkUVhv2xbvPWVZsl6vGI+FEd/3kjMwnozZ77dA4MWLa373/fdkWXbMJVJKsdvtsBHjeQhn/UP1R4vo330P73+A/YNwwtMU7j4CXhAk+62IyutHcZbnObQbIMCwAn8F5VyEc+dFsG6t3H8PVkTduhYBdbsTsdtvYFfL/0+imz0Aj3fiTB59Ln9+eBB3dDkV4ToEeU03iOBdTuDyEjorwm7dyu92W7i4kpv+zQZWa8gbWN7FJpZ8/qifoLqE2UiwJR8+iGBKgLv3sNkJ/3y3l9chiIi830PdeHZPPb6GcaUxKnB2rgGHDdJwWD7J8cpfwfVLWD5KOCoJYCFLIR/B40Ien5Rw80qO/XYd3cgDfFgBTkJKd3KtkOcwGQseZ1TB794KPufmGuoO0FCNxDV+NhV0TecR5nkv4vfQRWG7h88/T6nGKb/6Vc12C9M8MBlLI6DdwmQSGxZb4dTbQY5jN0j4674R93kI4qYvs+iO/8Fz9gKuLhVdE9jv4exMXkdpuDoHdMZ27bg6y5lOSwa7oR80632Lc54kExzP0Afevt9ydgGDh/wLePhbGLawN4Lg2e2F1w8wnQa2uxrlBPMTnIgbNsAXV4IZ2m7EYa8CvL4u2e979hPHeCrHab2G6+sKrzWr7Y7mMZCV0HQOnSr8EKh3nVzXTpA6EpbhSfSY8Tgn6JZ0ZHFKtIWrLzQOT6o1QSnef+wZBrg8F/yO1hJGWlWe3Pyxn+RTnepUP6byXkbfQrASxKkkoFJrCYQBRYgolSRJ4qIqiow2gAoxGV5F1p46OpFB2NzSfNTRjRBDSj1HkT5EZvYwiAAr/GpBjrhPQkUlbNGhMOgY1GiHHp8WEJuvOm6b94JDcaHHDZ08LlhSk5AmRoI1U0NqEozSyK6Ks9hZHx06fRwDtAQnWBmtFUlq0CaJaI8OHwaIx01cLuqZlxfxLkmiY2NCHXndNgruIYb7HBaVRh/42ilaKQnqSRLBzijhzadZhg8VeZqII0ml5EVJmqZY78H74zEX1EkgSeQcHsI2pdEhC0QfAlo2F2tbmnYXWZdgvUVpBNmSFRjjSUzOeDSjKsdkmXDVjRHHtvcDIS4+27ZlGHo48N8Hi+48Siv6vqfvW3yINzYBPM9c/aNoPVi89yRpSpFW5Fl5xNUcjqdSco1ofQjIVQR8bCIo4LkpA2CMOYrkWifRA64iY9RhrYuPtyK2B/13WOs6Xs8HLJExBpOYeB09N6G8DzEg9zARFj754hkzAzE0leM+nOpUpzrVqU51qh9XXV5c07UD2+1WMmAmM+o6J8typpMZAcd8ds5uv2OwA0oRpy1l3ZOmKWdnOSBGhOl0EgPkcy4vr/j48Y79vmYymbLb7RiNxgyDY7erub5+KWuQmE0zDI9sdxuKokBr2O03tHVDlhomlxcsF0+8evWSh6cVfd8wHpecnf+E5XJB0zSUVU7bNtzf39O2LWVZYJJEUH1GY4zgIet6L3iZ61fc3d7T9wMQaJuWm5evsE5ykpq2ZjKZUI0KMc4ox2AHHh52zM8uuH94h9aKvvOMyhlffvG13BfE9bjzXqb8fMBaD0GTZTlD21I3NeOJYESaZk/btaRZxv39RwmppKcbdvjQcvP6EutbRuOMDx9/h/c9u11PUU4hKNI0p2ka9vuasqxiaDzkeU5ZlqxWK9IspW1bRqPRMVQ+xKykxWJBlmXUdc35+TlaKfquJU1zVqsl8/kF43GBHXb89Ke/4N27W7RK+S/+yT/mbDLn6zdfMB3PUEGMNi6akggBnQnWJHhLmlVkecFqtaYoJozGMnGpSRCxUe5NQohISf9sONFKUxQFTdNQjDJGI2nwdL1lv98x2B4zwP39Hcak3Ny8ZrkU444YRQKD7el6ma5IgmY0GlHXNev1mqIQpEqapozHY7bbrTDRlaEsCpy38d7QHzO1rBWjUFEUx7X0wVC03zdIByUwGo1xzrLb7miaJh5rOV8hwGQyxUbTk9aG9XrDaDRiPpcw0j9Uf7SIvrgT1IiXfCw+voUkiCCZFTI1seqgcyLY7lcScokDNYLZmYjB2gi/XGtQLYxKWOxgqOHiQhAtmw1cXsDkjbynSYRpvt8Jq3zoBceSaHhawnonruzxVF53ETnpIAKtEYQq53N5vO8EM3N+Jo/f7OS793G7RiJSd4O8TrMRIdU6mE8ElzKZi4v5219CdS6C8Os3cPtBmgntABdnCauVZrvqyWdQlJreOZrWs20EXRJqIIsieglPd/DwHhjDaC6C62wupq23byEU4oLva2G+L5fiug8d9I9w+Vpz+Tphse4JXiYDxiNByWyWsP8IqoBhDrOxvP5qDe1e8C3nZ4KqaQZwSo7N6xvBoyRJYN8MqGSgKgCnUCGQ5YrVE9ghMBpLU2DzBBcv5JipVARru4U7p7i6yNk2HV0T8BbOx8K5LxJF10LTKNIskGYi/AcFRZlRZGO064Vbqrb0duD2URoi11eQBkU3BGwPOg2sGxH15xdyPe5jOGvXQNbB9aXgZj7eBvzQMJ3IecyMZrvUbGpL30KSQJ4r5tMUb3tGo8D9vbCm6r1cn8MAo2nLYh14fIR+AfMb+MXPCtRFyu27HW4fcFau18lEEDiDhbvHPfPLnC5zZAqaPXz/G9juHRdnmlcvL7hfbshLhxvg9iN89tkUGMgSz37oGI3/2E/yqU51qh9TeS+CtfMW9IDWXuTEuPAQ1rlBKwl7VErEREmDJ7p75UuJteM5SFQnKC1NxzQ1R1eDCItAEPe1cMP7GOTinl3s8mrPomiI7xM8aCfhmZ7oXJfOtQ/iJB/sgLMdg20Y+hpnW1AOo0REzzJDkacYnUR+hjvy133cnqG34j5XAW3iQixJSZOULLpI6ALD4PHI9hojATxHwdV7rBWIufpUXI/H+CAAq6AxiICutcLEGxitNVli0MepAENIFRmKNEvRwcb0eWl4DE6c94Aw2q2Oor8CncZwckXXdUeB++D+HwbFMBiUsvE8OrIsQWkZBU3TjLKs6HsR0YtiTFmOKcuRhKKGQG9bcYXYga7vsENH8FaOXwj44Oh6i9KKobeRzS7HTml1dG0PQ4/uYminJ7LFDYlOYnPA/N71670T1ngS+ejBxSkIEdF9ZM9753CIy3wYJFjUaBAAi9xgiNt8YBji9sUbkBBZ+yhPmpnYVEhIM0OWJWSZNDsO7pgQiGJ8F7FGCuMGFPp5koPfF9EP18VBnD8I9qc61alOdapTnerHURL2mTCdTsnzgsfHB5RSvHz5kqapybKU6XQqIjCBrMwIIa6Hnadpas7OzpnPZzw9LXDOcTY/p2k6Xry44e7ujqbpuL5+Sdv2tG1HlokY+uLFC4ZhYDIeo7Tm888/IwRFXuTRDBAoiozz8zlVVVDXO9q2YTIdsd/vWCyfuLw8J0kMo1FJ2zW0bcNoVNF1LSbRrFdPVNUIay3j8ZQsLdnvGtI04f7u7mjY0VrT9wPTmaGscp6entjvt1xdnTGdjTFac3Ex5fb2lsG2rNcLMcv2sKl3/Ad/9o95c/MVSpko3Ms9BUEc1goR0JWBXeNYrRZ8++3fkuUJeZGyWD5yNX5BbweaumawAx8/vuft27d8+dWX7Osd47HgDDebNd57tC6BwGaz42CoyNJMsDHOkSTS5Njt9sK8H+UMfc8wDMzPzui7nuA98/kcawV5UpYlbhjokQDSd+8+kOc5aaaYnVUMbseLmyn1vqUcBf7hn/0p42TKbrtjMplS5Fryfg7YywBKi2ifIKz5h4cnMQupRDKmVCJO1WgyiTOSHKKtjBE37+eff06eZ+y2WyZnOR8+fKAfHCbJqKqCEDxd33B1OSNNE+p6jzGGs7MZIRQEHJvNmjwvaTtpKDRNcxSskyRhv98fOeR1XZOluZhhvI3ZWgl5ntO2LdYKqsZ7T5LkGJOQJglt2zGdTiMq0bBcrphMxmy3O4oiZ73esFwsGY1GxzV7VVUArNdr+r6nqirquvmjTC5/tIg+nwnypEmF3d1u5Nn7HnQNuRH2OApUJcxvNQJSEYOLCh7vRbS9eCHM8KaXe1vtIWyhLkRYzhKYTMXJ3Lbijr5/ELH+4hzqdXSaR8dwWcF8Lo7gto3bEAT7Yozcvu13sh8BUIn8vijE1d6ugVLE9jwVJ/J6DTqBLz4TkbmPIV9aSUCn0/L78Zmw1jdrcaNrI67ujYIvPs8YjVK2m56ra3D9wGIvTYR6Bf076ZSoa3GTTyoR0D2y74kBn0Caw9OjsNaLEhIlYjsWfAM+l31IUpifa3SisU5c4ImDVS/bt1nL+QiI0/z1a5kgWNzLMbEeNg/SnPjsJ+LWHuT+ksVD4M3nI4Lt2S4H0gI29zCfaNYbz3ojn8FxAtOxfJ3NoNyKg33bQFbCqEp48/qS73/3wHbXMZmKyN072GwDVVkwmyiaoaPZezaLeL29sFTVhvXa03aBl28C1glXvyxgXEBXB8ZjEdLHE0haOR9dA62TczWdyEVQVaADVLm4/rMcrm9kcmG/VejUk6TQN+LUz0yQzhfw179qeVpApqRBYLxMZzw+eJYLwRdlEeuDG9isLNNpYFTKFITJZarg/EyCZvsnx8Ompu7FZX5ApI5GiusXInDstgPrJ5iMDRdnhp/+5HP6vuVp+cjbdx3J4o/9JJ/qVKf6MZUPHuetiNDBcXDHipAnwvjRMe69OIW9xwk6Gq1FULfWo3gWhokBnVqp6Go3sZsfxUGvYyinOqIzhmGIIno4LnQFsSHbehDRgx8Igxdch85xTpwHTimU8vjQY10ngaJ9jbMdIVj59zk6vQXnYjBaRdSKZehFPHbOi6jvRKhX6oDmMCQmwWgto5A6QSWKvtd0Q411A1pr/CduZaU12kA47otCGWGw+09c9lqBNxoTTOTRy/uZKLo77zHaHI9xmmUYnaOVcOa7tqPtBgbrZLKAgOssIXi0yUkzg/dgkoScApOIazrpDF3fYtuO3jp6Kxx2k2QkaYIyGYkCtISVFkWF1oEsLamqCXk+wiQZ2mjCUbwX17ciCG4laFx0pwcC1jn8cMDjHErc2+JIFxHdBy/Od2XEqaQSdEg+uRYO5ytev0phEgkE9YHYjBFEkAjVGllBKYJXOBuw2uMS+ZmPTnEb0T8Hkfsgnh+E74OrpSiKo+NLFvEpSZrI8fdyTYdwuK6f91VrLccg+BgWe9if58d82jg61alOdapTnepUP54ahoEsy49N/aenB8bjMVVV4JzDmJIf3r7n7GxONcpw1tJ1HThk0jLLaNuWH354S5pm7HZ78jznb/76V3zzzU/54osveffuHft9Tdt21HXDZDIjTXOUMtT1hul8Qp7njEdjdrtGsv7KXNy+Q48xirLMeHrco5OUPEt5WjzgveXx6Z6u6xhVFXW9xznLZDJmsXiiLHO8H7Hd7phO57J+d47xZIp3ilW9ZDaTIMlXr15S1xuSRNF1DSE4zi/m7PdbhqGjD4EkvULFbRsGR/BgdEGeVvzpL/5MxBgEayhM8ITgB7z1MqGZapSB0bjEB8tuv2WzX4Iec319xWg8ZrPaoFSCMYq66Tg7v+Tu7pHJZMy33/6Oq+srlMpRWu6PxuOKEKDIq+O9S1nKz7quR+uOqqrou56yKLAImiE4T5okvLh+Qd/3TCcTrq+u2O/3x/un6XTC+fkck0CaKa6urqjrltl0ig8dH26/5Ze/mnAz/5zz+RVn8xngCKEnTcTZ7e0QWwgaaweUMqSJGGUks8lEt7k0GyL/Jk4d63i/IwaVVzev+PnPf84//xf/b0Y+YTabsK875mfnbNb3Eevi2GxWx8ylw9fT4oGzsxnGaNKY4SSoniVffPEFVVVFw89AlmWcnZ1xd3cXeefPhpTDxGdZlrStIF52ux0Bj7OWLM2o9zVfff0Vi8VTxLIMNE3LaDQmBMdu1zKfzQDYbrcopcjz/BjI27ZiiKmq6iiu/6/VHx8sCrx5LW7mu0bEaj0WETLXIkSGPZCKg5tS0Ck4YVp3XkRgrCBJhkoet6/BRt52kkmIaFGKiK1C3MIE7u8E75EAOoPyTBzYXXdgUcN6Idzws3NYPICx8jp+EOG7LCBPYEjkUslyEai7Spz0uhI3NkEwIvNrqKLDV6fgelgtZTt6J+LqT/8e/PBWtq+pxUVclMJ2N1nDctVAKfv69F7wIhcX4vAz54Ekctj7PTwE0FM5pvs2ivbAd7+B3T2EDTQ5UMTrHdnW5kmE2bMLuLu3pIVlNBa0zn4P1VQaFmeXcB/At2BbaQ6sb+X9fSuu6pBIk2MyFWf+Yin7lBWwXPdsdpbdTtzZ/RDAy3FNjKIo4Wxestu0FJVn/RBd8qK3EEYwGVs+PnxE545qJKLyvoHJuTDdcY77x0BVyeh3uwW3BvPaM7/0IvY7OQ92EDE8SWG9hczAbh8Yj+VYbDfy9dSIUzw08MUbTV4G6l1Ao/jyi4x//qHDB5l6sD189eWEu7uG203H7l6aGwaYnMHiCVaRz9/cgTkHt4TqtTQcsgJeT+V6kr9cC4pLw8cfNmS5Jis9+SCfjc0a1nVEGzlpFjkfcTh7uHoVKCaO9+/X3H7ncS3MXqWk2cAv//ZbilxRDx06U6dg0VOd6lT/xvLeypezoP0RiyGO84NzWqOUwbqDezaiU8zz7/2RT/6pKBgF4gP7LOIytDbCAg8SaHlAuBzESxCBXr4LZ/0Z8xJwVuGdJU01KklFLB8UWhM50wPexy/XgR/QIWCik0KhRNyPQj9KmPDO9cJfd7LgIwjWxpgkYmAkwFR424YkyUgyQ5oZTAdd39K0NiJNnp3pCoX3z5iR58aBIzjhYquQHBE64pYBo4Wn7b00KHxkp6sYhJokBq09qAHf9ljnDy0QrBW2vdaBEOIkQZagrSyMlRNBXxuF0h4fLH3fEoJGm0BgwIWBEDQBcX0rnZJlwpAv8jGj0ZQsy5+d1DH06uCklhsOIEhArI9Il8FZCRUNEenDAX8Ch2BSax3OBwgGowJJIsxExfN1YIOPAUEO7y06MaBStE4jp1wTgsbagyit45chRKSQj+O8z5z/EJ3tnzQ4YjjoYReTJKUocvI8P455HlA5xhjwAR8U4GKDx+K9i9d7+D2kT0D9nlgvn71PkDQnEf1UpzrVqU51qh9VZVlGlqf88PYHrq8vmU4nhIiBO2BNyrKiaVqcaynLnM1mxTA4rq9fkvQ9fd8xGk24u7tjMpnw8eMHLi4uMEbx8HDHeFzR9x1KBUajkvv7W5xznJ/PaNuG/qEV1vliyXx+yXKxoCorxuMxCs96vWK9WWCUou0a6q7FugGTCAt9GAb6QZNlKUUxxntPXdey9kUmHcuy4OFhQZYWBA/T6Tl1vWU2n6K04+PtW6pxTtfVXFye43wXGejQNJrtdsNut6EsMwbnyYuEPC+pdwN5VvHzP/mHGFWKIcboKD5KJpBnoO8G0iwH5WVtW6RyLPee/X7P2dkFi8UaFQz73UAIEiafJjkP2ycuzq94dfMlq9WK8XjGaDTi4eGBNOkZj6YURcGHDx+5vLxisViwXK6OTuvpZMYw9Fgrbuq+79ntdlxfX7NYLHj//j1/7+/9vcj8djg3kOUpHz++j8xuy3hcgHKkmWJXL1mubgmcsdk+Mk6npGnK4Fq2izU3N69ZrO64uHjBYDt0KkK0Coqrq0se7u/QSkszRqUU1QgVJCtKEcMHkbWqVknEZjr2dYO1FpMYnHXU9Q6T5CyWT8ynI6zrqaoR+32Ni+J7kmjqek/fd1jb44OjbW2cvMh59Ur4/Ov1mjRNef36Ndvtlv1+z/n5OU9PS4zWTKbjo5C+Wq2Ehz8MlGXJ09MTTVvz+vUbFk9LXr9+zdPjE4Md2O12vHnzmqenJ6bTiTDds4K27dAaptM52+2GoqhompqiKBgGizEy8SoTIP/r9UeL6O++hVffiICZVxL66BsRQdtCfk4OOjq69UyczX4lz9cazq5FfD3wyselOHGVFvHaGxhPYL+RUNHpuQiY653gSpZr4VRfX4swX1WiJesgIul+B2khN0HjUlzrywXcvQOfgkmhTMVBrnUUPF+n/O7dEMMt5fdtAlSClKl38HQPo6kEYa524lb3LdzfRoSMEaH+ICjf3Mhrb7aB+ydB3Nw+yedaeRgmcHEDyUQ48Ms7JPHSwYsosLeN+KkyDdu9uO71BEYzOc5pIuJzdw/sIFyKqzx4OR5WiTPfBRHk8fJ65UgaEA45D97AaCKBpJMzcb+PJ4IoQQmypmvknD7tBnqn+OYnIx4ea+r7wPkbw1efT/j4oWe7bxiXFdt1x9CLO/vdezneF3MJpt03gWZwzMaxkWBg18HLqUwk/O67gTJiZra17Gt+Iwz+oRfsjRngbgEvbuCzL0RQ/3AL8xew89Ko2G7F+dd28PqlTAecXyouLipuH2qWy8DVBdjQk5TSFHr7LZxfw65bs9oFVC/bmI4F47NZwvJB9ifNoL0QtE91CbOpbGOWC7aor+Xa+fC+xvaBxQJGV4b1yrO6g2kp50Pn8To2cqzvbwUfZPegXkHbCTt/ci4M+yTrGGzg7rFmPtH89E/OuJjteXhq/9iP8qlOdaofUQm2osP5jiRR0VkASvlPRM6EEDiKw+LGBRMSgj5wosXl670Iz8I9l0WG99L0lK8oIvqDY1lcwDa6l2WBquNIpBZR+BMWtbiXBQMjv/JReJcbAZRDaYcPPc61aALKqOgoV3yKzrDOYoJCQkotPkgA6TAMBA+JSdBGkRgt6xYNSeSoK20gjpqmaUIIOUorAo6mbbFxFFDF4NCAiPGJMc/uZGvBu8gldHg34KzGDT3BJPijmHpgm4sArBNB6xitCKEHpUiylNwHht7RtA3D0JPn4ow+sNaBo8PdeXEEWDuQpCm5zz8RiZMjq/DQKHHOC/omMSiVUhSVMNt1AnicG/BWOPQKwa+E4JD0dUWaZHilCFYWIFqnHPRhpRK08vgALsg1YVAkqTxG8ChAEA65XDuW4CQMVf61DIJq0UrE9k+Y4nLNabIswzkTcULxurNy7RyaP3LdHRoB0hSQCQAVr3G5uc3zZxH9059JxoAiYOmHjq5vsLbHWhfPo43HLJ6PmBdwEOE/xbz86+70U53qVKc61alO9W9/TWcTCdqcVigdqEYx6NC2NM2Oru04Oztjvd6glGG32wBiPNltN4zHk7g+8YzHEjY6mUzQOmG7Xcf1iuPh4Y40NVTVXELT7YD3Tt57NuHx8YHLy2u+++473rz6jO1mR5olQKBpaqpRDkpMJCZJmM+nrNfL4wTnbDbj7u6WLEsZhoE3n71hvV4TsGRpzsPDPfP5uRgznxaEELh+cQEMrFYPnF+c0fdOUBzzism04v7unizL+clPvuGXv/xlNDaUrDYb2q5l6BWEjJuXX1IUM0BQlAQIKhwzmwKBfmhwfmC5XrDvnvjw8R11s2MynZIXKcPgWCzWfP3ln9C1jtV6Rds2nJ+f8cUXX6O0oSoK+j6gTYbWJdqkpKlMAiRJSpaJm/ns7IyqGvPdt99RliWTVxPqesd2syHLMrabDa9eveLh/p7NZkOeZTR1g3MWBQxDR93suLy8AgJd11CUKU+LO7wLXF5d8NXXn7NaLrl//ECZThmNS9b7RwlpTa55WLzHZIr52TXdUJMVI1SA8XgsYZ77LUmWoVRL17bkeYUy4uJXBIIP0QEeUFpjkoTVakVVVTGUdSmOcqNo+4G6lrXwq5s3koM0dEynYwIyYXF1dYHSgRRZ/y4WC25ubpjNZmw2G5qmYTabHU1WIYSYE1BIWGuzp6oq1usdL15cU9c1bdvS9z1KKZqmpd7XFEVJXTcS5FoWXFxc0jRt3O6Spmm4urri9vYjwTuKouT8/AJrLWVZHk0t6/WG2Wx+NHz9r9UfLaLvtvD+I+weRdDGy7ODEUGxGsnNWRHxOrsYFtpV4q4NDq4uRGj8/i08DSIM51Nhdm93IjoujAjIKolhlnsReEcXEkS6XwvTvG1ESC5zcX53LRRTcVCnkbue5SJydzci/A8N7Fp49ZmIzW0t4w59LeJ2XgriZTySENIDSz1LxaFcjuR3i4WEVDYOJjVsl8Larqr4fSRO4uUaqkTCLttBBP1+AbsVXL4UBIsKkGpx9U8nIpq3nQR/rjci6mcpLJX8virEgd02EGw8g6Xw5+ulQuVBRstLePlGGgddJ+fPGHhxJc+vexHbuxjYmkRRPk0EJfNuLcf0bA4PH+HyWhzfZZVyPslYLfaMS3GWN92GxaNHJ/D+3SOLpUwUpJkcvyFOAlQj2beLiabee8pM+PnLtRz/zRPUG3jzheyWuwObyXNsJyGq44kI6TqV41oUwuV//UYwQM7Dh3v5brScj6yUa6nIAz+83/P+Y2A6UWid8f59Rx8gmcq5SjLo+0CWER128hofPsj+D04c+qGNuK0CZtdyXbz7TUQLnUG3E3xRNwR2WzifGb7/YRDG/gqSl4JHCg5Wj3J8igS2dxByyM9guZJrezqXfXz7Fu7vA1l+cH6C0R2jynB/q//uh/ZUpzrVj76GoWOwPSEIp1q55zBKpdQxcJHAEVNy4G97f/B1C0NPRFeiEG/xIbp3OXCmiQKmFZxGdLb7wPH/iY995haGGBQqzmClNRoD4bCNgBKnhPcWlEXhCQjKJImOaHQqAZ8mw6gUo5LIj/SAZ7A91okILC5uQdGEoLEuoFWKUsJ4P/DRhaQtYqw2moQkIl8MVh/GDB2QCDPcJGijxdEBBDuAllDTQMA7ix2iYxqFKhRZlnFQwI8oGJNgkhSlFbbvUUlKWmgcGusatNGkmQjoEuYqSBydGJyLHHFAGUOSZaAVQSm0kTE8YwxJlhO0xvpAsB7nFegEjULpBLTBBU9vOwKxadHHwFIEUyKBneHItw8+CvNaFuNyDCVwNSTi3rbeYbQXwVlF/I9KIhNeY7TB43Euqurx9UP4/RBRMMeGjYS2ynGz1uFjcGjwktUSvEMpJ9eyDvBJw0VrTZqkR+zKpygXaaCkvxcsesDH9H13HEH14cD5l3+Yn93lGqOT3xP8Pw1U/VRQP9WpTnWqU53qVD+O6vuG3W7Ni5fX9H1Hkip2u4bV/Zqrqxc4Bw8PD4xGI/q+I8sy1us1o9EI6wb2+x0BRZmWVFXJw8NDDErsuLg4YzyZsN/vuX5xKWL3fMJ2s8X5gfuHW9589prbu4/M5nOcs6SpYbNdY62jbWvhcw8d3XIva17v6GuLMpAXOUopdrtdRLAEZrMZHz68Zzafk6Yp1kLbdhRFyXa7RuuUy8tzuq5nVJbc398zuJqz8894Wjzw8uaC29sfmM3mjCclSmk+fvwoaJFoUhCn+56z2Q17r/n5z/5dkqQihOj8PJSS/wTvWG+WbNYbtvs1zbBi8fSI1prZ/IzJdMrbt+9Jk4LNZosPnvOLc+7uPrLerMmLDB8gLwqyvCdLC5Q2vHr1hrbZcn9/T5rmvH79msViyc9//nP+5m9+xcuXN9ze3vLhw0f6rsUYRRKnXbebLcYYirxgPptjh4GiKKjrPWVZstkueXy8ZzabRY73imHoybIsGnREqHZ+YNcsuXtSJEVgfnbOr7/7X7B4frj9lsl8TGsdrrWUVYnSgfnZlPv7R9JcOOLr9ZLZXJEXFUcuugI7WHQ0kFjrefv2LR8/fuTi4oJk67BOAmHzPCMvFLv9Bu9l2nUYHDc3NywWT2L0SQzbnfDGLy4uUEpFrrmlqiqWyyVpmrJer9lsNtzc3LBarVDK0MYwUPnZkmEYBCE0GsUJUM/11TXWOooiZ7Vac3l5SQg+fj48xkhgaNd13N3dMZ/N2W034lR//Yb1Zh0nMWZy7xWzvXa73R/8DP/RIrrPYPsItCJwh0zwJ/OpCKVdLxzsxMSAzk446p2B+6UIn6NKxsf7lWNogBSqKwmPHFVwdyfC+nQKKoufAQ3jOUxKGNbi8N2twVSCT2lrEVYJcD6T9+8GcW/vaug7QavMzoUjvq5FHO2tuMB1fA+cvLd1Iv5qI85nk4sQPb8UkX23lhDUfAYXVyJs9w1MxiLilpW878NC3NSjqQi4SQd1CkPEwhgD7Uq+VwWcv5D3f1oIl3tsoH6EOpHXQMn+7pbCYw87oIf8StxzwxKMCpRngrTRybMzWikRsAcLJhNsyLqBKgrFAXns7gdICgmMXazFyT/UInRfXoMFXOhZbHuyAkwND/fwpByX14pqlPKrX/VkOTFUDm4i5mS9k/eZTxO++fKGv/yrD3S94+IcNlu4f4RJDrMrcZB/vJUph88/k4ZBlkeBu4sNk1F0q6/lZ+VUhPhRlWNtR9OIs7tewm4ML1/CD+9gPwTsFkZF4O27Tl67kvOYZ5rN3mNWMWC2l4ZMoqUR0fcidrOAsI5/SafCN3c30D7C+AsJZlXAdCZM+3wOZ1cimm+2kMxh08r0xflcPju9lWtOBWlCFRU0Vq6H3U6ua+dgs4fLQo7Hvvas15Zd47i7P92In+pUp/q71Q/ChQt4lFZILqb5hCEdxeKDE/wYImpivrkGLZxxY5S42Q1ASuAZY3Fw1Q6DsNcFn0HkkT+jX0T4lG17DhP9NITxEPJoBLGhD2z26FZW8f2VBpOgfIYiirImRZsEo0UQJyhcfG0JIv1URBeMjLiSNV4pfJIQgsN7jfcDSh+W5YfjIjibNMsICFvbKE1QKT5El3EIpMZQFAWJkka998IPT6Kg7uzAgISYhjSVRauzqAAmonWUP4RxGgiCo/HRYKO0IlHJ7znotVLRgR6Z6SFEDElGCIo00WhlCVHc9l7RdxbvAsZkIogrQ5KlKCXCvPMO21m873G+w9uBEIiCsgjLRN69d4dRBEdicrTRx6BalCIEeb3eOqyWbZTrTvbJ+wP+5Bl3gpZ/16W5QmwYCOcf/zy9IAK6JjEBox3DYCO2KHAIkj242XXE0ZjowE/TZ4zPwZ1+4KADpGka3/eAoZFJhraVgFXvHcboyPIP8fNwwLZ4ID9e6wD/uhP9VKc61alOdapT/bhKKU+SatabJV3XHEMUx5OS5eqRspjw9Tdf0zQND/fv8F4zGlVx4s6jjcZaS1EWtE2L1sIUL4qCfb3j8emBPM9QSswA7979Duc8NzcvGYaBqsqZzsZY23P/cMfPfvZTHu4faenouoZ6v2c6GbFcLhidTTkbz7l/fGSxeCTPc5xzTKYT+qGjH9potnC0TUua5CilGQZB8wkyoyFJNT4oitJQlAalUza7J7xv8aHl+sUZxqR0nWU2PeOHHz6QZTnWOlbLNVmRUZYFPnhG1Zy//4s/Q1MQvCKYg8ECDuYNHwZ29Zq//pt/yXg2xoY9AU9VVWw2W84vrnFW8eL6FfcPH5hMKnxwnJ0Lm3673dJ1A9WoxIeOsprgPdihp65b0jRnsVhgrWW93mCM4ePHj4yqEWdnZzw+PvH69U0U0g2jkfz8sAZcLBZsNhtevnwZ1+WON29eMdiB/X7PxcU5Xddyfn72PGFrLRcXl3hr8Wagtms+Pjmsbri9vcdbOD97yWdffMaH93e8evUlaW5Yrxe07V7Ok7f0fSO5SCqI+BR8nAiNyEmdAI40Tbm5ueGf/nmHSgaU1tjOkqSGruuoyoKiyMiLPIZ47o9i//n5nP1+gzEyGdu2LePxmLZtqes6ZgBU0YRl+eqrr9Ba07Ytk8mM3/zmN0ynYx4fH3n9+jXL5eKIDFJK8ebNG+qmZrvZs1wu6TpxqHvvjq8rCE1pCEjWQEKWFez3e9brLYuFuOwJmqG3VOWYLC2oyj/8Gf6jRXRaYJCw0PmFCIKDk/vY9VpE5Itz+f/lRly6Tu5zSAxMRyIa2kG41sVc45QwRlINSQ4Xl/I6doDlFu5bwWhcngk/2geghGwu2BSvYFnDbpCwzXEl773bixMYLwLyaCoO9rYTZ/fqSZzXZSZYmHIsbulmEBe2QwJTUdDtDLvGY8qAiwLn+FxE6iwTod16EZ3rFt6UwtZuNuLizqP4e3cvYrKKWJmhkZvzzkIZw0NdD9PLeJwRPvngwMWw1GEAvwMq5L5MCTZmPofqCwm0DAjSxnlYrKDpxK1/cQmrlWx/ruBqJv9fFvIe5+ciGIf4vKKA84k8ZzKXY7RYSHOiaQTxstmKsP+TP4HJPLB86OkGwfAMvQj2gxfxeVpBNdZsV46mX3Fx5UHBfGYYeo/ZBaaVNDGyLKMqLYPz2EEaE9VYkWea79869p0cfxtEfG86mAwyffB01zHUsdFQQfc7xb0NVD8Xsd0HERzqvZyHqjKsFo6nBpLSo4G7pWBz0ok4zNtWmOjTmZzPD79FgitmUJ1B6KUBsJlHN7yD6zfSzLm7Exf5Dx8dWsObzxX1LtD1UDew2MprWSvHvLwUoWA0ykltz/LB4QaYX8HLKxHVz+aK9TpgtKbe96xWEnJxqlOd6lT/evlDgGJEeYhAGr+UIEuCDxHncnD9PmdZKCXBNAcGtTEHVjqAwQd7DH48oFe8Eze7UubIRAdxQB+CG80nbu9PnbpaHRzoWtjkh23VgkwRV7gIxoQE7VPwHhUFY6UjiiUysZ3zOB+36yj4exF1ncUqRZKkRxal7IMFnkM+Q2wnHLZdcCjyXaz5Ob1VdF0rnHWjSRODCRlWBQlY9R6jhbXuwoGF7gSPogMoI6HSQQjiPgThsUUXTh9d4NY7aSR8ch5NPD7Pzu1nJ7SIwsJW93EaIBDi6KSgc5QKIsiblDTNj+fBe8cw9HTDDmtblA/CiY8L0k+DiEKQMFlFQpomJIkI0Vq6NgQf6K0FNaCVk6kAZyNH3EOIfHdDvD4MQYXYPFGgPNooUOE46XC4TrUW979KVGwQJREl5GLgkY03LV7c+Dyz0OWalmP4Kav8cO0fvh+Y/YPt6fueYRAnjlzTQQJgef78HF7/yEb/NwjmJyb6qU51qlOd6lQ/vnp4vCPPc9pmL0iO9RprBybjGV2vmEzHfP/dt1jnmIzTKETKuiHLUqx1JEnC0+MDJkk5OzuLzf2W0agkScWtO52Mubu/Jy9y2qYBFRgGMRumSYK1LcZonp6eeHx8oMhLrq+vYi5RQlHkMdS0xQdHnqdAiKzvAa01L1++pOu6uC6qOZuf8fjxiSxNo8HFkGUJIXjatubpqUfrwNX1OZPJmA8ft3R9jdIlq9WSF9evGYae8XjEfHbBaDTht7/5jv2uRiWeh+UD/+V/9r9nPr3AORXnZUFY6OG4jn58vGNU5QQcxsDNzWt2zR37fklRJazXW+bzCx4fF6R5wrZeUlYljg6Hphgl7JotTb8jKxK8sgzOkSaGx8dHzs/Pubm5oWka9vua1XJN33XMpnPSNKMoaq6urmj2W5xzdF3HZCyicFEUGKV4eX1NliSs9juSXEw0FxfnDMPA+w/vybOCNM3ouo6+b+j7Aa0TurZnsA2bhwX3T/Dr3/w15+eXPNw/8adFQZIrfvXbX5JXJeNRyWLxyGa7jlOWgdG4ohqNSFPJIiLmWFkX0EGmcIPyGKP45utveP36Nf/j//R/pxwbiiIjhMBge5QqSBKD9yKsl2WBUnB5eUFZFljXkeUm3ocIF15rHd33NdPplN1uF6/rjOVyyX6/pygqXr96xXQ24e3bt7x8+eL4PK2JYrlHK0WeZywWS7788ivqWl5rPB6z2awZhp6z8zOyLKVtfWzqFAzDgFKKoijwXhzrTdOQpilPT0/Udf0HP8N/tIiejkX4TQ4uYy1s8LaFMET0SicidrODbAxPT8KizkciqNa1Z7MBlcP1pTBXjw5cRLDcrAVjYT1cnsNsJhgNH5nexQSuJhKUuRuiiTyiV/AiqB71xE7E/L6HUSGvsW/ARq65CvI6WsHZFIqxiOqbvTQJXA/tKqASccArK9s5KmW7t2vZXmUiXkPu7xH+qYjSeS6vuV5Jc0CXEoy62YoIf3OpGPqAtWAkO4CntYiy1y/hqYZyBiaAq2HZAVtIr8FMBVOjc3nN7ZMcN7cRl3ZnBTEzn8HTg+BkTAGvLmW7NlvZ5MyIWF4UkQm7ETH43UfZn8m5uKXrjTjx9/GYlVnE56TweCci9dW1hJm2DRQZPD6KoJ8Y0NpTVfDtd1u8h6sr2G0dTytpFLy/EzRK11nazrPbyTWVJPCnv0hodprHB4fKhKmfKqhXIuSvKhGrg5YmwmwK9V5DYUjygcVSRO2QCOZGB7kWmp2cF5xcQ9UM6nidjCoRur2PCKMB5i/h/lJE7/RMJhR2K9h38NWfTEiynsF2tDEcdN+A6aBfybHcnwX2teB9FBJSq5H3qKpndNDycWAIDtfIxIc2iqIKJFs5HtOJdLH3nZyzMv9jP8mnOtWpflSlQhQaiSGaWpAjSkcUixamdwj4GIwodA75HiI7SqGP7G2l5LVQKjqCY2BjdEBbF46PP7iNxQVvjq8hNwTCgolDhIIFUQfXbuRWG02S6OgkUAQkcNMk4pYPfcBbi1JGnOg6waOjSK5iM1+c0D664LU2UVz1EIbj+J42GpNoUp0SSAhBMVhxYh/Fa23QR4FVgTaoJEMlGqXlH/9Ui0McLTc7SqUo745OD2MMOkmOWBBtAkkaPe8R/eKiq90PEgzrvGBItJYQGRObGgfWdpKmDCHg8fIaIb5WkH3SWiajjk2VeM7lnBxQNPI6EPn4ztINLXW9o+/3JNpQFgHvUoYw4JwXHFDkzmsNRsm0QpZnZFkujQal8R5M36HUwGAsqu8F+ROGT3j6su+HYFe0sPiVDijlCUoaGdKoEXFdxwBWxSEA9ODWcQy98D8/bS4crq1DiYvdHJ3o8MwqP6Bc5Gf+KPh/2kwRcV2QLj742ECySHBvEl32cZsjrgglExISmHoS0U91qlOd6lSn+jFVlhuSVPO0ED63tYGqmvKLX/x9/uZvfiUs6FFKCMlx3brd7ji/KBmPxtzfP5DlmtV6w3Q6O7p5jUnwgejYbXl6WvL1N1/x61/9mhcvXrBarUWnijky0zxnu9lJYGiakOQJd/f3aK1Y7/ZU4ymJ0aAD2hiGQcLUsyyn6wfapo0Tj4GiGKG1ph8sWZZxdjbHOkuSJoSQEQiUVSVGFGC/r8nzguAM7b5nu96TmJTF0xNlMabMK7wNfPH6a95+e8e+t7TbjpfXX/Of/aN/QqISdt2e8egMvEzGSkYSgKGazLj/+JY01wy+ZrUZUInj7HyC95a3P7xnva4ZT6aU4xylC5JU41xLkiRsNhtGo5LtZiWIHd+y3W745stvuLl5yWg0Zrvdslgs6Pue9x/e8/LlS6qqYhgGXr9+hfeW0aQS97TO+Xj7QSZPNbjgaIeOvMoJBKazKeNJxWq1wg6e4GG/35MkiUwg+IC1PZCT5xmb7YqiKHj3/h3z+YzHxR1oWG/v+O7tv+LXv/lzdvUdk//qv+HXb/8ZQ+8oshGv06+4yW8wcRJX1sQHs1REWCqZTgXPZPyCf//P/hN+9bd/CWaHdR1D25EZTdd3DM5iEk1e5Pjg2WzXXF1dsW9qlNaslju0Urx8+YLlckXwntFojLNrZtMZzjl2+z0KxdAPjMcT8jxltXrCuoHRqKIoCrIsQ2sJLA0BlsvVcaL0xYtLkgTads9oPKIsc/a1YrtrReBPEvb7J6bTMbvdntF4zHa3I4RAlmWCA03To4lpsPYPfob/aBH9/FwEzelYwjUDEt65WYHfSrjjNoH9Emih90APWOijO7dtRKy1VsIunY9u9Ri4aYOI8K4XsV4bMWK5VkRt30CfQpPJe3dLcVN/9gpsK8/tGgnLBKAQ5/dgRTRWWpzOwYngjAMqEZbTVITjhRfnetGDcdDgyQpBgzR7wXqMpjBGHNtDbBKcnYsIu9/FJkB0GtuFCO12JcfCK8G90MH0GhSGura0LQyxETAMglopSyg8jHMRpvcBmAITccMbFYXkFjY7YbNbK7zws5Hs19kUthtxyuPB78GdwXIJq63gRHDSDClKEaRTDRjZ/qKQ41fXcgzHE9CtOPbxgr65fSdBpOfX8rh+JQ2RwUoDQ3jk0DWKp/uARUT+upXmwG4L9Aq3D/Q9/PDBM8qkydAtgByWTwMPCxHvX57lOD/ges/QQzVRjKYZN9cTktyxr3dYm5IWI3xaY7IBq+RaSFO5FpUW1EuvHJTARq6h5EL2eb0GPRLHeh+v46YHNHz+cwlMHZDfn59DvQWnW2ZzxeMTfP+dnOvQiKP9/I18VvJSHv/9D4JkubmRnw+DvI/R0HkPiXSNkglcXObg4f6+I0tg6DWLpcNoWHfwxUtDnpxwLqc61an+bgnGRf7OQymU1jE0UwkeJDrQ/adu2WhqDiEQfGR4Ky3uX010fSiCt9HZLGOc1g7H54SgICiUSjAKUD4K6/Ls4EQwDyGCVVQSueeBoDWJMVEg1iQJ0bkuAqQxQbjuOmC9MD9CCLgQMTAmiWgOEVoFp2JQ2qCCQhtPQEehWQORqW4KlJHFRwgaaz39IE77A2bEhYD+NMBUKel+64BOI+s8HDjcAZMkpPGGwTuHizx2dHTM4AnBCpPcKojNDVR0MTsfA8MVGoVR+jCpKjcLiUElIiArG9AYacwGT4gnWJolsRmiNanW5EkandgiopvIYj+wwYOWJVKwDjc4/BAICTgbcNahEhN58B4V+fmBEM0EKr52gtEyThwU6NRgGGhpYHB4NO6wYAs6XnNyAWoNCmHUH46T87KodV645+GTxo5JTAzZUngfyGI4bOgCLgwHsr00GWJgkjIpJinQSSqBpofjHZsYJgScdxLwijSJjNbkaYaPQbnOOsEOxVwBQohseDj0F1wU510IOMD6wOA9NuKJTnWqU53qVKc61Y+niiKj6yxVNcZbBT4lS8a8++GOsqhAebq+kcm/kGCtJy9KhsGx2e4pylHEuYyoRmPu7u4ZVSNMotntG3a7HbPZjMenJ/recXX9giTNqJstZVmw2eyYTMYy8ZdIFk2aS64PWnF5/YIP7z8SVAImZV9vyLKCapSy3eyoqoqmaWm7HjrLm9dvWCwXaA/zswploO1FjHbeY70nRHG9HwaM0XS9Y71sOJu8YOh7bL9mNB7TtR04BYWhnE549eIz/tP/uOSf/flfYXTJf/SP/lOcU6zrJ7oO8iwhOJk8LcoqogE9SVpw9/jA9atLfvv9X/PufkXXb6m7HXmeMJlWrNZLoKMqz+n7lkQlKJ9gSDEqZTQacXd3T5YkwuLuO+4fb7m8nJNmOe/eLSmrnM8//4y//Ku/5Gx+zmhcUjc7jEkYqZS//uWv6Puey4sLnhZPAHz5xZcEHVhv11xen1OOC4beslnVpEkF7EkSTdftmEymbHcbsixhPClwvkeRsl5vSNOUUTWiKmVfqqpku3vi//J//T/x9PRIZx/5539l+Jvv/pLH+w3n05dcXl2wWj3x4moEwUA0DSmtjyx0FSSbChRGjRhXV1ycv2S5+R7v9sLN955yNKdpWx6entBawm2bpuNx8chsdkZd71mutnz11Vfc3T5wc3PDZrNh6C1ffPEVi8WC8XjMdDLDOZl2tc5ijGYyGTMej1mv16xWS+q6FlpFNTpOej48PHBxOWcyLXn//necn5/TNDXb3ZIQHF999RmLxYIkSbi8PKfvO54WS16/+QxtEprIXQ8oxpMpj4+PZLklSbM/+Bn+o0X06RTOMxEhk0QEQB9EPCfiQwojIY59QPgXGTCIM/opAxMF4pDCfhB3t29glwBaEB1FIc9relgsEWyGjuzyDnwNbRS12z3kE3mON+L43heQa2i9bOuojEzrXsThF3NxZQ+9vE+SyWM0EuRZbyXk0yPu3mws4v56Jds0eOhWIv4enP7dHrapCLxpKm4vCqgHWNyBXwJroBQhN7SyL9tH+E1tcVZc1H3kiFcjEb1dFJCbvRyrEG/KMHLspxPYLGHdg/bi9ncIGmZ2EW+CB1juYTQSlni/h4dbQb/kZ+Ikn8Zg0aGXRkk1kvO7WMDQyT7e30Gw0V1v4vdJbCz0gohpWzm/44hlsQPMxuLav/0I9YPCm8DN1+K4zjM5ruUI8nmg3gvrPBCd7pVs5/RcuOWXqWB7piODtQObFbz6QuFDSjUqGFUl6/WOpnY0g2O97+n2llZJQ2E2lebG40MUju6ACznm7CFMpcFRL6WBY4w4/ImTF4OV7bu4gMeVPG8/wLyUhsvT00BeybEbTeRYBi0oHJXA9EoaRpsNnI1hu4XtQs5rNRJUy76DeQZFnnC3sFxeplzP53y8X9J2cl03rSNJwSVwcaG4eT3mu99s/tiP8qlOdaofUfkQno23UeBUiNgcRBEXAZ3fR7gcBcFP8BTPzHQRro8YF6w0Nr24ixG5Nzp/jWA4cII6ic5e7wPeHdjdgtQIGMF26JQsyyIOJDrPNRBZ3cIxl21K0oSQCJ7EO48PGqNEFNYhoHB4p0g0YJRg5PAo5SJyRqNNjkkLkkwEVQ9YGxhcxI5Yh/NWGNcKtHa/hwKRZHuNU1aEcudwg6BK1KchlhEPIsGfgRBkbk4pfxydRLl4buS466AI1mP7Ae8GEecPgvgRgyMO/SLPSROwQ0S/xOCdwwlMouvkgPJJTUYSA0wP7u9nNr7HaUtqEjKTodOIH/EKa+X4xZXZkaOolJKJg0RFUV4aIRrhxaMVQXuSoDAhkCqFValco4cLVAmuR3A1RC798/7aIDx774Xxf8SvaBUnLKILKciEhfU92glrPsTXCV6aCcakGJOjtIlNJI87wPwRsd54mT6QwNSASRRpSJ+vN+Ux2pCYRCY3fDhigEBhD8cS4TF21tJ7h/UiqKuTiH6qU53qVKc61Y+s5N/+UTWi64Qh7Jxns9lSFClFmVM3O8FRzK8wOmXohxjIOKJtOz58+MA33/zkGKjoXaDrGsqqZDwWl/QwWBaLBWmaMgyWoiho244XL17w8HhPWZXUdcPFxQVd27FcLnn16rUwqyfjGMy4Ji8MZVVwf//A09MTV1fXTMYTvPfs93uatj6uxYqioB8kN+b169c0TcNmsyHPZTqxaRogoSwr6l3DdHzGYB2vXr9hsXzEBsfQ1Vy/eMXj4wOr9YqgPF9+8zn/4E//A969u2W5uePxccVm3fLv/3v/iK4fMDrlw+0d19fXeDew3z2yWj/QD1sGW7PePDGdVXidsd1tURjG4xEQ2O12YgoKimHweN9idEaWFVjrWS7XgKIsK7abFffN9ojWGYYBpXsuLiaAZbW+ZzKdsl5vaFrDN998zWq1oq5rQgjkeS6TqwS++OIzbm9v+fLLL7m7u6dKMtLUkOc5q9UyYnoajNFyLZzNWS7l5z+8vSVNJSR0u91xfn7Ofr/nd29/x2QyBjzL5ZL/9r/7bynKnPn0infvP/D9d7/js5ufy/XgA8rEtXEAVPgEj3O4TjXz2QXBa5yFqpoIumcYMNpwdX7FeDQmSTPapuVsMuf9+w+oiWKz3HJ5dkXf9EwmE5arBZeXl3z48B4fLNOZ4G2ur19wd3fL9fU1Hz9+ZL1eHY1dWZbhvefNmzf89re/5bPPPqPv+/hzx363Y+06yrIgBE9R5AzDQAieruuOTPWmqSmKktGo5Pb2o0w/o0gSzXq9BODi4ow0Tbm9vfuDn+A/WkQfjUVYbBrBl3hEdMbA7FIERw94TeRTENOigL0g1Q9BjDqF+RhcAY8fBQejK3EsT8biUrabKPr2guHY14KBUUqcxNtaxO3KyOOyVLAvZQlFIq5yrWESmdaLpVwcgxdx05yJMOu9iLzdICK83cGqg8kshokmIgzniTyv2UJ9mAYeJIjTdbDfwHgqwnvvZFvLVJzxOGAC6qUgQPqtHAcQHjkNuFSc4PsFbL2ItyoRx/1+CWER33Msr10V4kTfLaLjOQUfp5ZVBo0T3nlTi6j76hU8fYRVDcN7Od5nE8i0BKNuekgTCRA1GXz4ANO5CNl9JyI4VhA4ZxdwWUrzYdjL+64EBUvnxJ3d9dKgwIn4HHZAFzj/Upoig4XUy/csExH5/Er+3zkJjPVeWOBlDA2ezgQn88P7mhBku85mAe8l7HRo17z9W5hcQTWGYeehQ5oXyDFzWqYJfIAhHsvRRMI7QxYnKRwUc6h3ka0veQj4iAbKDFydCfZn30ozZRWvV4ccpxARBTpic4YYhrrfwXwkTYjVUrZlei4flTay5usAs4mjzKFI4WGxJCt6bqbw8CgTAa9fZTw+DczGsN/VdPvTSPipTnWqv1v+4IqF6PQVxreO7vJDuOYh8PMgpB+kSx/EbXzAoQQn/699EPHcWXwQ8fiAdVFKwmoOoaDGyCLsUxSGhOj43+Ol6yiKJml6xJQIruPgmI7CamT3yWuFqBEfBG4TxdEkvraLiA8tvHQn42gOF/dVYZKUJEljYKbBORcd3M+s9gNb2xiF1yG64eXxSZYTUAxBHMYuPvbQMDiUUhqjDYL4djgfUHgIwiBPtMEohdKCBfEugAdrB4ahw7lBHP2KYzCswqCUfGVpjjcHnvlzSOxzo+JwDKXRkKV5DM48hGrqI/vbWkuWZpRFAfj4/hbv5fcAWifH86dj48Ik/2Zz9fFYfnK+0wDOBKwTVIrWmkSr6PSPz0MEb3lNYbt/+nqH/QIiKkdCT2VBakl9gvcZ1qnnQE/P8VzLjUxsNh347JF/f2goHAJHQwgYDV772CB4bjoYczjenuCfJ8ME43L4LArWRVxSz42BU53qVKc61alO9eOpd+/e87Of/oLHxwV13YobNgSm0ynL5SOD7WmbjrzIybKMei9hjMYY+l7CzW9ubphMpqxWa7bbDYnJGE9EPP/888/ZbrdcXV0BcHV1xWKxOK6DQNak+31NXTdcXV2htKaua+Gfe1njLhaPtG1HXswJQRjZ19eX5HlGWRUMdqDrO6wdmE4nNE3N0+KJ7XbNbDbj48eP1HXN69evsdbSdR0hBLquwzlHWY0AxW63Y7F85PLyHPDkeUHdbHn15pr/2//jv+Prr39KWUz4V7/8c1arLXW3pms9ihTrN2ij6bsaT81m5+n7lsfFO8qR5u7tPb3dslw9opIZ682Sq+sXPD0uefHiBU+LBX0/0HUdi8WSPM/IsiwGqHrG4wlFUcb7GyjLHGtbiqKkHwZG4xG7/QptAtYOjKoRs9mYLDe8/d07/uE//DOSxHB3Z5nNpsxmMyCQpslRGD/gNosy5/HxAaUgz3Pm8zn7/Y66aUgSmZB1LlCWFS9evOD29pZf/OIX3N19pG16rq6uj0K9uKwL9vuarnVkFyNGVcqvf/23/Mf/6L9iGOTeTSUh3g95mbpF/53r9fz8Upzo63d41/H61Wc8PN3T9wMXF+eUZcV8dh7Pd8tkMgVgNBqTJAl938X1sqeuBVFTVRXffvsts9mMthUeeV3XvHz5Emt7Hh4e6PteQmwnE/b7Pa9fv2YYBqbT6ZGjnuU52j7fFxVFcWyKNE0b75G0IIi6jrIs+eGHj3z55Su22y0vXlzR9x0fP35kNpsxHo/Qf/cQ/J36o0X0v/g/nkS6U53qVKc61an+t1SfusuVEna0QnAuR0yLE2SGIM6dBEQqS4hoDOvUEQEj3PKAdg6U8Lqt6/FhADwHD4M2YLRC60/CIWMdBMrD90/FZnUMFhU0ByDudqOOLnR5jdgYH5xkpjhZ+imTRBFfgkGDVhgtiBgfHAr7yXsRRWXze+xrwZQ8//7ZlCFOIY0YByQwKcFogz0ItCFEPE3c9vgz+TlHhTmg4+snGJOQmIQkureVMgRlcTgRznFoI8xtYV7qo3Au+ypfMgEgmDitEhHo8UdAevCCiyHo6NwGk0R0iorYlHAYeFMEDHnIBJtjDX3fY617xv58IgQfw2o1RzHaWgvBYA6if3ysMYYsivf4gHVOXN/mIJaHIys8ACZep1oZtHZH17eKyBgOeKEjU11BdKYf3PABH5tEHo7nXJxAgqEJcXeEn67U4Zo8nPzIldcqutL1MVTUOdl25/4Nq27vo7FdJi/wARUChufw3FOd6lSnOtWpTvXjqSwr+f7735HnZRRsS+q6Zrfb0veCO8nzgsl4wmazpW1E/BOhfUIIgbZteXx8jFzolPFoyv3DHWVZcHt7G8Moe16+fMlyuWS73ZIkCWdnZ3z//feMpxOGweKcY7VaMZ3OeDRPR9f64+MDWmvefPaK3X5N33uck2DLzWZNXe9lfRMCVVWiVKBtG16/eQWIMaLrOkG49D0hhOgQlu9d1zGbnrPdbTHGMEor2rZFa0Vd7xmNRvzu7W+5ur7hr//mL3j16g1952iagST11PuBxWLDn/z0c9I055e//BV/8vXP+PVvf8nl5Tl198Bi857v3/6Kr7/5jMmsBOX46usveXhYUDc1TdNxHY/PZDIh+IDWcv9R1w113fDFF18SQuDD+/dcX1/RdR3b3YqrTETwNE3Y73ecnc2i+F3Sdnum0ynXL66O07M/+ck3LJfieJY1ckLbNnz99dd8++23lGVF33f0vUwKvH//jsfHhyO32ySG29s7RqMqok3yo/FnNBqLu9qf0bUDSmmGwVPXG5arDT//+d/jw4d7Xly8IXhZO/d9j3UDymcRl/nJ7Q7Iujj+JEsq/rN//E/461/+M1wYePFiRJpkrNaLOBnRs9s2eC/rfRGsxS1+d3fH1dUlIE2GLMsoy5J3796R5xl1vSfPc7IsxXu5x/j48TY2ZRqSJGG1WmGM4fLykmEYuL295ebm5mhOGs/PWSyeyLIcUOz39VFEf/Xq1dFkI1kBO66u5qRpgkwhbFmtliSJkC7W6xV9//9HJvqpTnWqU53qVKf6314pRKzVKjuGxeij0KzROsSQRsVgRbg1Ojm6xa0FLTyPKKAGrOuBAR+ssNBxaK1IEhGkRRA+iKLPrOuDoGztc6jiIVTyiCYJz48TJ73HWRFAn18vvs4g2BPnQhRFVeRrx22NHDR14LwfkRs6uuT1ccH8LOyLm3ywLmJRhmcXcwiIoJpE17OIuCKm+uM+HbApChWZ8+LwcBEZchCTkyQjTfMjlzyNQZwB4ao7HdAIn9v7Axv+mcmulEKToEgii54YsJkej7lzFuf88XuSGPSBuf5JM+N5/6QOwaUg7iNZrD+7qQ84F5lmSOI2CT7Ie3GzB69w+OiaP7yuLFzxjsS56Gg/oGaepwxAQsCfEUNyc5NmGVqL5d3EaQV05MKrT8498bn64CQ/XIchTjBIw0R98ucDmP2A0xGDuiKo2Lg5nt9nlMyhjsczBEzE5R+mLz51tnv3vH98crxPdapTnepUpzrVv/1V5CKYfvHFF/yzP/8Lbm5ecnY2o65rcd3We5loVAmr9ZrJeAJA3/fH5n2SJEd033g85hAWf3jcfD4/itVlWVKWJc45mqYRgf2pp4jCfJqmXF1dst/veHx8JEkSPvvsDdvtBm0U69WKs/MZ4/GYvu8pyynDYIW9Pp9j3UBRFEwmE2azOU9Pj6xWK87PzxmGgQ8fPvD69WumU+FOH6b8FosnpuMzkiylafZY13F1dU63aVmuFyit2eyeyErNavtAkiRs6y3fve15dfM5Luy4vf2Ox+UTf/EXf8Fi846zszmPqw0//PAt/bCnGht2zZrxpORvfvk3NF2N0ik//8XPaZpe1o5ohsHStYMYPbKM9XrNdDJjvdpQliVFUdH3ltFoTNPuWCzWnM3PIGhevnxNmqZkWc733/2Os/Nz8rxi6O9oW2HDLxYLtNaMRiOGYeDm5obf/OY3tG3LYrHg5iZnMhmx2azE6V+WZJlw2VerFWEIjEcTrq+vub9/xOiCoij4/vvv+eyzz3AucH//SAhQlSP6bmDdr0l0RlmMmc9Tnp6W/Nm/8x8ym82od3J/k3iHMvoTjIuKxpLnP4egeHn9JXk2ZbPb8/S4Is9yXt2IM7zrO/rGkufi9t5stsznc4xJ6JqeUTnm+7d/yxdffsb9/W10+Tu8PzRbWgCGYWAYerIsJU1T2rZltVodHeibjeCLr6+vGQYRvK+vL1A6MJlMsNayWq24urqirmtGo1FsCg3sdjuGYeDq6oKyrFgul4zHE3a7HdPpFK0Nd3d3OOf48svP/uBn+GSBOdWpTnWqU53q39IS969GqwStE5RKUaRAcuR1A0f8SggO53qcF4E84HBe/hyCO34pDrgWESgPXGitDUYbtNFH94WIh7I1Smm8lwCgw59/z0l8FNOPWw/I44fB0vcDfT+ICO/8s3AKnyBM/CcoEz4RTzUxnjOK1inGZMeGgTxOmgQHgVVcEf4oegNo9cz8Pmy3CLfyPv64H5rD4lP2OUQR2h8F4DQ7YGSEmW70J19GkWYakxADYkNE20TxOuJcxDeuj42Jw3uLKB2d7QGsdQyDxdpnQffIP48Il8MN2eEG51NsymG8VRwjGUmSkKYpafaMhPHe46yVm5Guo2072lbYmP1waEaE45k1Snj1qTEkRqGCOOc/IRDFfVXHMdPEyOL6gP0xJnnG/6jncyLM+cP5DPEa8fEchYjocfjI9vfeSmipf8a6HJ4b4rTDoUnyaX16LA9fh+v3cFwPX+7AzHefvs+pTnWqU53qVKf6sZTRKVU5YbPeYYxms9lQ1/XzemFwaGWYTGZcnF9gjKEoRDS9u7uTNVAU0QUFw3Gys+s6ptPpkSXdNM1xrbdYLMiyjPl8zuXlFVdXVxRFwbt379jtdlhr0Ubx5VdfUDc7yqrA2o6z8xnWDlxcXBzX73leMAyOoXc0dUvw8ObNZ3z37bdHXMdh/VhVFZvNhu12y8ePH5lOp1xcXJBlOd570iRBKciynL4/OKkHkiThafFAP9SARWvLZvtIUSr++pd/gQt7/va7f8nbd79m8Dv+xV/9U+4ffsevf/NXfLz/nt7VtP2OtttjUs31iyumU0GNbPc70LBvapx3VKOSr77+krarKcqMV69vGI1L+qFlu13z+s0N2ijm8wvubrf85tfvubi4YT67wlnFZt3grOH9u3uc1ZzN5PgeUDfOWabTCcYYFosnvHdcXl4wDD15nmGtJUmE0940NaNRxWeffUaSJHRdz5dfShDnixcvBd+sAhcXZygV6PuWNDUkiWY8HnF3dwdo/vRP/z6z2ZzHxycUijRNuL6+ivcXCT6IAcp5mXb1x/uZEJGeAR/AB01RTPnpT/4Bs+klKCO/94LOubg4x3mLNrDf7yiK4ogGKoqMyWTMxcUF2+2W2WzGZrM5rokPQvpms0FrTd/3jEYj9vs9IQTevHlDnudcXFwcr2FjDMMw8OrVa8qq4vz8gjTNBW2ZFRiTMh5P2Gx2JEkap2hletmYlMViRdv2pGmGMRltO1BVY5IkpWl6Fov1H/wMn0T0U53qVKc61an+LS3BVqRonaF1hlIJwRuCfw6ZPIif3rm44CUK6T1KOZxrGYaGYWhFaIwi4GEBdBBrQUc2dhr/THSnJ6RJhtGpoEQwJCYVAThoyUEZvCA6eBbVnfNR2HXHmwpnA33fR2F2ODpyDoLkwXVzGGPtuu73QjZ9dIU/i5mWYeg/EU1FgBaBOIrbkckdgotc8QSFwrlA31u6Vt5DnqtI4s3OQWw+MrtjaJFJM3SSkSQ5WiXxeUiAqokgFQVZYtA64Jw0MLSGJDXHx+nIYXl2k/++kxyIrn/7e2KtuKX9cd+HoaPvW/q+xdoe5wacG7C2P6JLDteJ1oo0TcnznKIoGI1GlGVJnuekqTDSnZcwn7quaZqGvu8ZBuEaDsMQ8Sfiui8yQ54aWfwbCSDx3jIMHYPtsNbRdZZ+sPGmQUVOvlxTaZqTJhlamziR4CJ2po/O++dGiLjje+CAjlFx6kCmKfqhl6mDGPgq1/jz1+HQPrvK+aQB4Y4OfRPRQIeAWTsMDH3P0PfYYcBZe/zykS9/qlOd6lSnOtWpfhw1DJ4sK9hud9zc3FCNKgY7RJyEhFiGoLm/fyTPC5xzZFnGfr9nGPq4zrFHg0fbNnH6zzMajajr+hi+2LYtaZqyXC7JsgyAs7Mzbm5uCAGyNAdgt9tR1zWz2YS7u494b6nrHW1bc0BkvHv3Ee9hOp3RNh2j0YTJZEpZVuz3DU3kYR/W3c45uq47hmEqpfj666+xVgJPrbU0zR6TGpz3jKoxbTfEwHd4+8MPXF5d4oPD+p6n5QPKeG7v36ONxzPw3dtf87S8ZTzJqesFH27f4nxHN9ToBFCeoCTfqaxKrHN8+PgBHxxd32KdpapyVqsF3lu++OILksSQZQmvXt2QJAbnBW9zdXXFF59/yTdff83rV2+Yzy5J04oin+CsYj674O///X+X9WrLMASapqOuGz5+/Eie59zf37PdbiRkc7nk48ePEccyIstSsjzFOctsNiNJknh8WrIs5cOHD/zsZz+nLCuKsqAoUwKWosxo2h1JYri4vEBrxWQyZjSqWK9XRz55NSq5urrgf/7z/4lf/vJf0fc1BMdge4xWMiDqA893ESFOgoJWGWky5j/6D/93JGaEd4oiL+g6adC8f/8DNy+vaZqa169vMBq++PwzEmMI3vPhw3tW6xXDMPD+/TsuLs5Zr9dHRGdd7ymKgsvLCzHnpAmz2Yy7O3Hy13XNDz/8gFLq9ww/ZVnS1C31vsEYQ9d2TOP1WJYVo9GYxWJJ3w2Mx1OKvEBhSJOcV68+Yxg8Td0xDJ7fff8DX3zxNVlW0jbDH/wMn0T0U53qVKc61an+LS2tjbCklUFF7IdSmhD0UXQ9YCz80WnuCVhCGAgMoBwBiw9O3OnhIFpH1EXQEcGiIBgIhuB1/HN0M+tn1/uzM/25RAj+113k4SiM9/0Qneg9Q+/E/TKI+CmBlyKWys2FMAVFsD24gAeGoY3C7EEotoRgj9iSg5v64H4XAVnCfowxcVQzI03T6O6OzHgrLnlBkpiINkmOLvCD494YCTA9MLmV1sfjIkz2gCKQJuImUXwa4vnpdojAf0CfDLan61v6QfbtcEwO5zVJRKTOskQW6Vl65KCH32soPDdGPm1MHIRi+dmz2/qZg67j9stxs9GFXtc1db2nbVuGYTgKxs8YlGdHeqIUqdZHR/5B/O8HEbf74dnJfXCD68gUPzRxDlMJSqmITBFnzadTBc/nWVjoci3L1+8Ftir1yfX4jHY5hoeGw7TCs6v/+VqOGBfrsP2AHQZs3zN0PUPX4YaB4OT33p5E9FOd6lSnOtWpfkzVtT3Wetq2xZiE/X7LMHSYRDMej4+hlp+9+RxjZFqy73u01pydnbPZbGia5igmHpy9WTQ4NE3Dhw8fotFF0TQSHlpVFdvtNqIwNvTdwMXFJd988w0fPnxAqcAPP7zl/ft3MQxS7gnquuazz74kBM352SUhSHpO23Tsdw0vX74mz0uatufDh1tG1ei4dpxMJiil2O8lULIsS+7u7gRrYjuarub27pb52Zy7h3uqakzbDpTlGFRCCIJQTLMMbQyff/EFj0+P5EVGINANHU+LR5q2ZrvfUFQZ+2bHbr9hs12RlznX19d0/cDNzSvm8znf/MlPAMiylKJIqZs9w9Cy2Sy5uPj/sfdnvbJkCZYe9u1ts8/uZ75jTBmRWd3VVdVNNiGSrSYFgdCLfq2e9EABFMUGWGx2NWvIyozIiLjDuWf22W3egx62mbufyKxOPkiQEGkLMJwbZ3A322YWMF977W9N8H3JcNhjs13x6tULBoMeUlqiyPG5fT/g7WdvWa82lEXZpKcHnJycMp3O6PcHbDYb6lohpSCKIm5vbzk/P2e5XOJ5HoPBgNFoxGAwoN/vY9GUZYHF0O/39on5v/iLv+Ds7Jzr609I6fPu3TuCwGe7XZEXO4Q0PDzcoXRJvx9zfnFGWRXUdYkxmtnJhK9+8RlSGvxAkCQ+/+P/9D/w6eYDfujhEJ2aXZo1CEPXFgTmsCrUCoz2+dU3f8kvvvwVVakoy4pXL694++YVvSQiz1OqMmO1mjMa9RkMYoJANjhIwetXLzFGAY6hHwQeWZZyejpjOBygdc1ut2W9XrJaOcO91+txe3tLkiRst1uklIzH42eraPM85/HxCWtgOp2xXm/I0oyH+0c26w2BHzIYDLn+eM1ul3J//0gQRNzdPjAZzzg7uyCJ+9S1xhpBEvcZDEZ/9B7umOidOnXq1KnTz1QOWeI3Zroz0R0zXGKNwLS4ChrMhgAhLcKt6QMMILFWYRrD3P23S3W0OBeLB1ZgjMSYxiS1rtpSCoEUEiPMM470c540TVpaHnH42DO43b6Jhl3dmJYeCKlwJrVjURttUbpu9stziXGj0cqgjdtaVAnC8cc939sb1Mf8arePHF7rCOlirUU36WRjDbYtb2257zQomcaAdeU37mHSYpGe78pam5S7kC2Sxi2txLjiJWNsUxolaYtM26Z59166YbELBArV4HRcotsghEPAGOtSJrIxqp8n7J+jSI6NYWeym8aQP2K9C3FglbsRafbforSiKAuqUuFJhdVuEiHwA/zmw6CQEtGWmApANqWnur0OGiO6WV5qcCWxbfmq5/k4is+hIJdmAsQcHccB5eKu7TbBbxuci8WAbCZLZDsZ1O6DQCmNQHBAmD+fdDhcL4fr5jAB5K5faw4c9P21d3zhd+rUqVOnTp3+ZHR6eo6xyjGjy3JvdgdBwMnJKYvFil7SQwif7Xa7DzlMJhOstXu0i2N09/crKMeTmeOYNyv2drsdSZLsy9DX6zVRFPHx40eHwAgjNpsNo/GAs7NT7u/vSJIYaw1hFDCZjPn2228J/IQwiHl6XKBq90Dk+yGBH1GWijwrsUYyGc/Y7baUVUUQBPR6PVarFScnJ7SFom2qvigKojAkUwX9fo+6rjk/v2C93pDEfZZLZ4De3NzhNQEHZ3bW/OIXX5NnFWEQ8s3Xv+TXv/41SRLz4sVL0jSjKHKsdStasyzj4uKK60+3KPPEZDpjNBphgdV6BRZ8X9IfJKzWC5JehDaKvEjdZx8Lg2FMVVV8+PiOXtxH64qqyrm8PGW9ztlsF7x9+5btdsHt7TWT6ZTdbg0YZiczfvvb3xKEHuvNEs8X1Kpks60ZjQdEcUCchAwGMyaTMVVVsFovODu7IEsdv/7q6gWj0cR1KXkBxihOz2aUZcF4PMRaxcPDE6vVgu12y+vXL1mt1kynU4qi4K//l3/HV1/8giQKCb2Q648/8Ot/POWf/bO/JAx9ttuch6cVn3/+C7TSDnwppXtGtu7zRhiEGCz/l//u/8rD4lvuH79jsahIejFxHFJVOV999QWPj08gNKv1gtevX/Dd735HkoTsds4EHw6HhGHIxcUFeZ5zd3fH119/jVLuWh4MBtzeugT6drvl1atX+88dm82G6XRKWZYopSiKAq0V/X4PYwzT6ZSqufaklHz++ef7FclFUTAaTcmyCk+GBH7M2dkln64/UVea6fSMT9d31LXi1atXf/Qe7pLonTp16tSp089UvueMy9ZId8noJkV9VKS5R5m0nHPPIqQBYfZJ9Da9vWdJG1cu6QpKm9c1co+LsUec8GMm+nFy+Dg97DLJjelpBXZvXB4nfeXezN4bwj5ITyCldcs2rUKpiqoqKMqMssyo6pxa5WhTYlEI4QxmIQ7ca2cWt/t5MPp/uhlDkz53GA+ah812X7UxBwb2kXEqPYn0fIIgcsgbz60KMEfHaIxBVQ7/YbTG93zCMCYMI4IgaPjfHp5sTWwLuBUE7UoCx7Jvz50FYWm6N/F8gfTc/jpmffs6h7S7EOxRLw6bUzxbmntsFrskyE+MZa2fmcZuHNl/iHOYHFduKht0jS8FDeq94fg3ExK0iJaSsqwaNEzd4FOOilyR+4mF/YvAkfnvjhvhJob215RwEyQtw5+20LU5TvfeLllfFlWzD3XzQfDYOH8+JkqpZmm22mNefn+C4nmCvVOnTp06der089dqtcZoS57lbLdboihiMBjsk+JpmtHvj0jTHXnuUC1ZljVG4pper4dSCnAlor7vkaYpi/mcu7s7qqqi1+vt07z9fp/5fI4QgrIsOTs7Y9ikoMMwJM9yNptNwyl3OI3NZkWWpZyfnxH4AUncZzQcYzT0kiGD/ojz8yuU0mzWO3q9AYP+kJcvX+JJSRAE7HY7+v0+0+l0v79lWe7DDAiYzCbUSpHmGUVVsliuWW8z0qwg6Q15/eZz+sMxSltqbdmlBRZJXlaUdc31za2bPNhlbHY7ggZZI6XHp0+3SOnxP/27f4e1gu1mx3bjsDV1XTObzbA4w7coMs7PT5ESsmzHZrOiKDI2myVJEpPnGZeXZxhb8mf/7BeMxj2kZ5gvbpGeYb2Z0+uHaJPjeZq83NDrOeb769ev+eKLL5jP57x+/Zp+v78vFLXWMpvN8H2f4XDAxcUFZVmRJAmvXr1Ca90w5E+J4wTpeYRhQFll7NI10rOMRn1AIz0XilKqZrNZMRj0eJo/8PLlJXmxI04CVus5v/zlLwhCidb1/rk53WUNRtNgtGnS6LZBJYJoPsPNZmdMJycMh0PKMmfx9MCXX3yOL6EsMkbDPpPREE/C/OmeVy8u2G1XeJ5gPB5x2kzWRHFEXVcIKbi7vyUIfN6/fwdAEPg8PDzsi2izLGs+A3n7TqaW+z8ej4njmNFohNa6KWQNOT8/R2vNb37zG7TWfPXVV7x48RIpfHa7nKpS3N7cc3//xGaTku5yfD9iMBixXu/+6D3cmeidOnXq1KnTz1S+f0CLHBd1upR0Y+wdpY6fm3oGiz5Kf7d4D30wm3GYGNkWltoWFXPAuZifvHaLL2mN8EOy2e3b3oRsSjCdDgWXUnoNgsVhT5wZ3KaNXeIYDEpV1HXL+i5RqkDrEmscokYI05juLXPd8aqBJsXjkuPSO+zfYRz0PjHtsCz+Hu/iAtDSTQI0y1B9P9hPaLSFmG0hqjPwDxiVuq6wVhP4znCPooQoSvAb9rfnefiB3xw7WJxx7saWfaGrG4dmXHj+fYvhp6n7dl/+U4Wjxyb6HgGjldvaQiytXdpeyn3pp+f5z3A4vnc8ESCQzb5aq7HGrTyQwj3aK62pSofzcRgf3fz7UIJ6fNU2wXOsPUxmWA6rIFq2e1uM6vve/ty2g+iQR7YptK0pi4qiKCmLw4TC/vwfYYocFqlB4Cj9n7jmxbNrvlOnTp06der0pyFrDVcvrvADvylnVKzXW1arLRa4vLxAa/dclyQRw1Gf09MTsiyl10v2Je++7+2Z2mVZEkYRZ2fnzGYngGA0GlOWFVmWo5QmCALSXYrnuZQ6OBPeGIijhMlkQhhGuBJGx5ze7VIGgzGL1Y7T8xcIGYD0qJTi/OICZQwn52eMxmMen54oipI8r7BGUOQVWVpQ5BXDgTPgT2Zn1JVGCp/J5IT1esvd/T3bbcrT04LLqyuuXrzgX/7Lv8L3PSwWz/c4OZ25saprsjzD8wQGRVll1KpgOB7y4uUrRuMpWkPSTB7EcdKUTG7QWnF3d+MCHZ5HnmWEvk8YeQShIIolabYmzTaMxn20qdnuNnieYDab8unTNReXF/z9r/+W/iAiy7dETXGm70v+9u/+N84vzlC6IopDqrpESri9vcMYwy9+8RWDwYC6rjg/P0OpCmNqRqM+RbGjKAvyPOfly5dcXlwxny8AwXw+J8t23N3dsF7P2W43WAv93qDpBLJMJmNcOauH5ws+++IN682SV68u8T3D6emY7WbJZDpgvV1QVSmeb124yGrSdE2e7bBGY42B9vOOrvE8izYWKSSh3+PP/+y/QOoRo8Epg/4IozUXlxesVgvG4wGDQcz9/TVBKPl08xHPF2itmEymzJ+e6PV6e15+v9dnPJpgjCWKY4qyJAwDZrMpvV6P4XDIarWi/Qy7XK5Zr7fUtaYsXeho/vTEcDBgPp/jee6e2O525EXB28/dJMzTcsV8vuTzt1/gCYkvfcq8xCjNn/3yVxRZzsf375mOx6TbzR+9hzsTvVOnTp06dfrZqimfbP6rAY00zyIS64AaYH0gwNoAaz2slU0ju93/jdmXKYIxEmsCbPN3UrqG85Zf2KJfLC6V7vzGlg1+SMa3SXRXotTuo3LFplphtQErkLjiUrfrbTrCpcDd34p9aalo0DJYgzWOe25N7cxzU2FtBVYhUEgMGI1RzkC32iCsbdLJAdIP3f4GAdL3m3d2CQ0hwfMlUlqsrdGmRunamajCoozBIhDSBxmADBFeiO+H+F6A5xqPsIamvNIVehqrkL4ljALCICLwYnwZ4ssAX/p40sOT/p4rbo3FGoMQBk9aPCnwhMRDIl1GGykEnhBIYRHoPZbk2Bh/jic5SksLixYGjUGjMcKNgLaGStWOW65qKlNTG42b9/CQnudQOYGHH7ivwnPXoxECIyRWehgh0IDGogUY0YKELEYZhDZYpZvjZF9Eq+sGlWJw59pqjK0x2qX4tTIYZdzY2BYr5FL4fvMBKvSbMW1KboWVeNJvJjncdaSUpao1VV1TK4vWYIxo7o9mX9v3NxptFNpoELjX8nwCP3CTCEFA4HsNk14Df7y8qFOnTp06der085EMLGm+BQnnl5eMJ1NqZVhvtlR1TdyLKVWOpiIvU8Dy8HiPxTAc9dlsVvtgRFHke3TK2ek5gR+y3ezYbVNOT86wBrI0Z7vZEYUJ4/GUJO7RS/pkWQ4IwjCiKCqwHkVRE0V9BAFFrjiZXVBVUJSWpDcmzSuEF3Bze8fDfI70PdJix9/87d9wP38kzXKmkxm+HzIcjpFNyKQsa6IwIcsKLs6vGI+mREGP8eCEJOzj4RN6AR6C3XpJGHhMJ0O+++0/UBU7yiIliQImwwHCKqRU+FJRlRsW81tOZ2PqooDGqM+ygjCKmc5O9isaR8MBEsuHH38Ao6nynO1qTVWlCFmzy1aMJz3OLyZoU2BMyUXDGA9Cj9lsSp7nSOlT1ZrtLqMsa3r9AYPhhP5ghNIQhD2Wix1hGJJlW/7iL/4Mz3PBptVq5crsq4KkF7oE+8AnTnxuPn3k4uKc7XbD+/fvsVYwny+4v79jl27YpSvCCJJeBAQs1ylh1CdK+nz2+ZcUZcX0ZEZeZkgPHp5uSdMVghpJTV2lWFuRFRsMNUiFUil1veXHH/4BSYknDRKoyop0uyHdrTA6x5rU/b32+Be//Lf8s6//G0bDF5yevWKXlaw3W2anMx7nd9S2RNkSZQt6gwDpWa6urjDGkiR9FotVg+50n//6fTcZ0EsGTMZTttst/UEPKeH8/BylNC9evGQ4GBOFPcIgod8b8eUXX/PF51/y+du3VEWJVor1es2nT7eEcYIXRGRFxafbO6QfcXX1krossVpze/2RX3zxOWezKb6EX33zFV9/9QV1mfJ4/+mP3sMdE71Tp06dOnX6mcoisMYihEELjTUNVkP6TeLWBzxEi1exJdY6RLWx1pmStsW2CGcK2xYJ42Os74x04e8LIR1vvC1obE3yFo1y2LfWuG0LLts9tkZhkAhxYLlL90vQTgBY4wxVE2CMACMRDUtdYrDCEvoWLSxKG4zWOEI7CKNAS6wUWOFhZY3BTSiIQLjXsWDxHOIjkAdDGYsUHn7z4cXzJAiLVjW1USjjCkYxjYUtcV+tjxE+nvSdqS2la++hTTw3BrYWhKGHkAHCAykCV84qBQLpEv0409xtdr8UUwqzx6Mg3IRDu9rguMTTGo0xAqUdq7tNnRtjCIJgf6xtqamLi7cofYFuJzHQ6PqQaFdKoa0BESB9Dyx4vo/0JZ7vxsIKi26T7ri1DO2iUYTAeh7WM1jZjIdSSGPxLAht23kUd+141mGDjG1WEBi0rlC6csfVONwSiSfBa3AtvicIPM8xNsEV70qBMu7I3Hi6NzJWOzzPUSmqm3QChMPJ6KbA1GBQRqG12WNifBEermvrVnY0g4oxhzR7p06dOnXq1OlPQ0kSk6YOdZLnOa9evdr3qZRlyXq9Jo4jNps1WFBNWXwYhqjalXVut1tWqw1BEHJ9fY2qNdvtltvbe77++hd8+vSJPC+oa8ea7vcHCCHIsozhcMjdzSeiKMJaSxD4hGFEmuYUecVsesZquaEq3XN6USjm81u++eYbXr58SZ5nTCYT8ixjOpmSpTkvX1whhKCua1KjwQqeHp8Yj8dEYUyapmgM9/f3XF5e0q4wVUrT7w84OztjtVrR7/cxRvHhwwe0VownY3bpDmMtURgBgt0u5ZtffsPtzS3GGCYTh4upqpq6qvG9kDBMWCye8LyAqqoZjSZI6WEMFEXJ09OCk5MT0t2O5WLNxeUZZVHBUKCVoSpzrq5esl5vWK3W9Hp9wjDEGM3JyYxer8/vfvcDWruC2KpyjPssS5lOTri8vCKOBYvFA2mao7V1oZ7mc1IcRzzNC4bDHh8/vmcwGDCZjCnLlMfHe/7z/+xfozX88MMPbtWp1YzHI5RypbBn5y9QyiDw0apkMB0RBBHrzZLvv/+ef/Nv/ium0wlYTZwkLJZLrJFcvnjN7374NSdnl3z7/T8giJnPN+AVZMUCz4soyhIh3LP2YjlnsXpCSEm/32MwGNIfzPjX//rf8D/+9Qdu7n7LeNLH82CzXTarLWE4HCClIE5i4tgxy9frDfP5E5PJjMFgSBDMOTk5pSwrpPTxPMl6vSEMI15cveDG3hAE/n7y4dXLNzw9zdntdtS1Amt5+fKM3XbHerMhzTNOz84ZjiRJkqCVIQwikqTXoEXdatE8z3n5+hV+GDAYDbl/uOPFi0vefvaav//7v+Prb37xR+/hzkTv1KlTp06dfqbS2u4Ty6AbbMUBXSEblnf71aIAD22caYt1v++wG9KVk4qgQbj4Tcmj1xiPjlV+wK4coVr+wL79YZSFQWvV4Edc0lxgsY0B6v7kGP3StE7ujWQBnuNfCyNRAqQELQXaeo1dK0A2vG2rqbXCCjeRIJqEMtYimmImY0Vjcuv9WDl+doM/MYfyTdt8zyHeHTvF4V7cJlv0R1N6aewh9S2wCNn8jfOtHTe8KTPVDTLFGPWMp91iQVpUCM1bu7FwyxptWyCrDcYqqtpQ1z/ldWtq5T97zWaGAwH4zUQLLX5GGbc6wbTMRIPRDuMiPA+JKwKViL3Rbo7NaNF8vymvFRze8zk3/Nj8Vw1T39+XuBqtsVqjrRsbh7Y58PXdOIk9g1MIf1/kup8oOLomrQXbciGN3XPX3XmztB0CQj5Hx+z57A3f35OuAKrF6rikvGjQRq5A1to/dA906tSpU6dOnX6uqqoKrTWe51GWJfP5nKqqeP369b5Qsd/vM5lMKIqUqiyoKtdT8/S0oJf0uLy8ZLPZslotefPmLZvNjo8fbijymu++/YGLiwvqWnF6evbs+fXi4oIsS7HNc9nZ2RnX19dEUUyeF3gy4NP1HW/ffMlyueL6+pqLy5dsd2uWyzlVXeJ7HsPZjKqu6SUJ682aMAwd/mWzoSoLPM/jzZs3XF9fc3l5SVEUxHHM5eVl0y/kSj+lFERRtC+L9H2fh8c50+mEMBxwEZ3z7t27Jk2f8fLFS7Ra8e7H95yfnRMGMR8/3lCVH/nqq695/+4arS1+4AIRJyenbDY7osil7RfzJReXlzw9LgDJarXl4uIlYRAQRTGqFsTRkIfHB/L8hiRJiOOYMAyZTCZ40mOxWFLXNdYa+r0ecZKwXm1YLldcXl7x7t07RuMxnz7dMhi49P/T4xO9Xp/VakmWZShVEYYBvX5MmjpkDFiqUjEcxbz/8D1ZWjZcd8loPGIwGHJ3d9swuxfUtUXVsFptkdKnKDLW6yXTyZSHh0fG4zFSQD+J+f77Hzg/u2K72/DZ56/5dPOOm7sfqWtLkSt2+Yb56iOTyTmfbm4YDIcIIfiP//F/5Ve/+hU3N7f8l//lf4nDZZb4vgugbDZrXrw8J8+3hGHE+cUJy+WcKIrQ2tDvD6iqmvX6nv5gxGg0Yr1e7bnmDrt46Np6fJzzL/7FP+f6+gNlWeF5QdOnFOAHPmC4urpqJiISlst7sJqzszNUU4w7m52w2W3Ryn3GOpmdcHt7R65zrs7O8TzJ27dv8DwPsLx48YLdbkuWZ/zLf/Wv2O22f/Qe7nAunTp16tSp089UWtum+NGgtW2SLu5nBz60bMxweWR+e4fCUOshkE1i19vjLhzr2ntmhh8b6D/9/vHPWvO93Q7vK45Y4QZjFEo3mJQWNWKABkNjtMVoV0TaJt9bBrn0GwyLFyD9AM8P8P2QKIyJox5RlBCGcXOsB+TNHi3TFoLKQ/HpMerkwG6vm4c/Z/m3r3c4HmfkC3kobgW7N8XrWrn0f8sQ94IGd+P/3ngdj+nxRIVsDHopD9x4+RPm/DHn3JjapbZVSV27TamKssybLaMsc+qqxGjlFgDsVxI0yfkGX+JMYeMmAdz8hCsK3Zd1Ngx5rVwxraqxWoFpy0GbNHy7tSn7o1LSdpLAkw4REwSu+KqdNHBlrnWDxHFIFcetbze737Q2qGYSoa7rZ2z1VsaY5+/bjKWQxxzz1vSX+4RLe617XlNW6rlx8Dx5xNZvx9ItY+3UqVOnTp06/elIa+1CGsYwm83YbDZ4nsfd3R1CCPI85+7ujqenJ1Ttnn+n09me97xer8nznIuLC/r9PlEUEgYhRVExHk9Zr3dEUY/Nesdum2EMbDYbfN/n9vaGsip48/bNfgVimmbMZif0kj4XFy/o9YYMBhMCPyEMesyfFgwHPWpV7osdQTCbzthut9R1tT+uoiiYTCYIIRgMBsRxjNaa8/NziqKgKArSNCVJEqQUpOmOwWCAMQbP85jP58hmDLbbHb/97bdsNjv6/SGBn1AUirJUxFGf779/R5L0OT09R0qPzXqHUposy8nSgjwvwXpYA9YIpAhI04JeMiAMY+pK0e+NuL15oq6gKg0/fP8BYwTNYyq7XUYQhNR1zadPn6jrmqqqGI+HnJ2dYqymrkvyPNsfY1VVLOZzh7EpFP/469+w3abMnxYo5YzlOI6R0n2uWC6XfLx+z3o9J8u3jMZ9hNSUdcZ6syAIfZSq2W53jMdTRsMpWI9Bf8JqucUaiTWC29s7fD8gSXpstzuEEFS1agIcgtu7W4oi5/LqjN7AZ7m+5f7xHU+Ld0g/4/rmN3z/49/x8fq3ZNmc73/4e27v3xHGUFRrbm5/pKy2CKlYrR+5ujqjP0h4fLxvilGHPNw/opQmjnskcYJuDPI4iVBaMZlMGA4H7HY7hsMh8/mcp6c5s9mJ4/b7Idvtzn2mErJJ/5uGoZ4TJwmnpzOUqvn++99R1zVBGDKZTRkMh0jhAlBREJKmKaPBgMV8ztu3bynLgjB0XUg//vgj//1//99jjGOzWwxxHHNzc8O33377R+/hzkTv1KlTp06dfqYy2qKUaWb5D2nZn+pZ4tcCLU27MaWlDFyCV/iNWX0o2zwuSPxpCr158WcG5U//7nnBokCK1vRlX2TaGs5tUsFosWdYC+E1qXjHGw+CCN8L8fyo2UI8P9z/ThDERFFCkvSI44QwjPD84Mhotg3b+sAJbwtFj0skj4s3ldKN0ewdFVa2Ew2y8Upbbnd7PAcD16XsvT0vvmXGPzfx2xF6Xkh5bOqDwG9M5naS46eTHaYxnOu6oqoKqipvvpZUdUFZ5RSNmV5VeVN8pDD6YJq7/XDn1hgN1hx441LiSbkvDm3RNVorVF1R1xXqqJyzHUetnpeTGtOy52VT9Orj+d7++DzfJUhcKWuNMRqldcOaLJtC2Xp/jqqqRilDXSnKsqSqqmf74Da7P99Ses8mi6Rw94Q7H66o1xX0HiZW2skGh0zyXDmqEHvzvZ2EOL7nOnXq1KlTp05/OmqRLVEUsdlskFIyHA5RSiGE4OzsjPF43CS4S3rJACl8yqJG4IrRN5sN1hrOz0+p65IwChBScnvzwGeffdE8S3pu1aXWxHFCUZSMxxPOzy8QArbbNcaYfYFoUVQURYUUHr/9zXcURcWLq9dcXV4y6McYVeEL6McxcRBTpDmDpMew12cyHJFu1qi6IssyiqJAKcX5+TlVVXF3d4fneU0hqis27fX6xHGM53nkeU6apuR5zm6XUVeaxXxFkdd40mcxX9FLBtSV5mR2jhQhk8kJ3/72e1TtgixKua+uuNLQ7w15fHwijvsI4bFabbi8fME//uO39PtDdruMOO7R740pco3vJQR+j8eHFavVjt2uaMpRc+pKsdlsSHoJRVkgpSRNU9brNUmScH5+xunpjPV6ydff/ILxeOyM5fEYbUEZw+9++IE4TlDaYAX0h0OeFk8k/R5pviNMAoLIww89Hp7umJ4M+eKrz5jMJvzlX/0VVa3wg4jPP/+K2ewMzwu5u3tkNjtpTOwRURTx+edfEMcJq9UGpRR5XqINSM9jtVlgRc30ZMj90wc22QPr9JbRTPLv/7f/gXX6icrO+Xj3D9wvvicZav6X//D/YDgVvPv0D/yHv/1/8rj6kev7b5kv7ggC91kvjELSNGM6nREGMViJ1mZ/PRd5zmq1AAFv374lDH3W6xX9/pCTkxMeHh7pJX1msxOur28oSzch5PshX375FcOhK4dVquLx8bG5lkJ6/R5IicWd91opqqpiNBoxm05ZrVYkScL7d+948+YNtXafMbTVvHh1RdLvUdYFRVkgPFCm/t/lkHc4l06dOnXq1OlnKlew7sw9K10xYxu5PkaCmAaV0S75bEtApWjN3IZvzqEE1PLcATxO7XoNgqQ1d/es9H9CLbfb/T44U7lNl8u9yQneHlVikQRe2Jjuco8RERKwBmMlwgLCIrREeq6UMwhCgiDcG9UthsRaUFpjqRHyOQe9xXU4/Elr7B8KOZ3p6gxs3/cb3Ig8JJiFew1tDFppt4xWGyzmgBA5WiGAbbEuh1T+8UTE81R5Y/oKDxsczF+B3pu6FrNHh7QTApY2sX0wgdtjac+lkIGbqNAKIwXCWjfWNKWaWjvefDP+gddy8N17ecIhXdw1Zhxn3BiMdO8vRIO0aa4lrQ36qOyUphDUkx4INwnRJt+11g2OxaCtRhvdsOlrVFU1fy8cdkVpjJYEQYTGYAxNyZN8Np5if+23+BdoL1u7hwEdJiPchERzLtvJp3ZFhCeR7SSCcFWprnC3xQCJZ+e0U6dOnTp16vTzV5tmns1maK0d61wpwKFe4jhmtVpRVRWBH7FcrghD17EyHo/3KJgoikjTlLIsEcLj4vyUIr+h14upqpK6rpnGE/7hH/6Ot2/fUJYFs9mMKAopq5KXr17R7/f58ssvKfKS5XKD1pr+YEAQxBRFief5RIlPpAVZlpMXqQsyeAFRFPH09Mhg0GOzcX/76tVLfvjhR4wxbLdbqqoiSRLKsuT29pYwDBkMBgwGA7TW9HoJ2+2GXi9x+zudIaVkuVwSRTHGQJal/OIXv2SzTrm7u+PVq5cYY1mvNuR5ThjGLlneTELUdU2SJFxeXjKdTvnw4QPGGF69etUEchybPYoi8jzHGsFsdsZ6vWQwGOJ5Hq/7fW5uPzWhCcnHj9dMZ1P+w//6H/j6m6/3oYinp0f+7u/+gX/7b/+PvDx7zceP12RZymQ6Rki7D35cXb4gjhPu7u8ZDvsEgc92uyDNMs4vTgmDiMfHOWdnZ+x2GwQe3333HcPBhNFows3NDaORM8mzPGc8HnFz8y15XjR89m85Pz8hCN2EjOe/Js9S4jgiSzN3zssS3/dZLJ8YDAZsthmj0RAQPD7dMF9e8+/+5/87RV5zdXVFnhdkWc7dY83lq4Siyvn2u29Rds5q/UBZr1GqditYqwxrLYvFEs+TbDZbTk9P3LVeV8RJRFGV+L6kqksAlHKhlvF4AlbgeQHz+YJ+v89w2Ge9XuP5cr9KoSgKhsMhWmt26ZZalQg5RFvrjHGj3T3TpNAHgxF1rRgOBggrMFrx8fqaL7/8kr/927/l4uIMpap9P9dyuWS9XrHb7f7oPdyZ6J06derUqdPPVG2SXAhXDkpbL2nFwUxtmM6OTCGw2iXY92Wi1hWQOh66458fp6FbwxEOiJE2XW6M2aNGWh0XWR6b0O5nz/e/NY8dhsbxuaUA4fv4XkjgBXjCw2tSytLzsViM1SjrIbRDcHhGu3S013Lc/Zaiju+JZzy+WjUm+j5FzD7NrHWNS5QfjqE1Tx12JXAoGSkbxMyB5e7w4nZvAtu26PToZ1q7VQNu4gLgMAnRjrWQAqvsHiXTfh/h7/ncrREupfy98srj9H/LCT8+H8fnSSnVcN7d94zXMM+RhzJU48xt3w8IgsClr3HLMD0hafnu1iiUsdAUqDqH3GKEm9wx1lIbiz4+3n2hasOyhyZxXiOtcWNhLbWu0aZC65pKlai6avjr7XE6FIy1dZP4BzcZVO8nOtyYud9z17Pdr4Y4rNQ4dAoYbVANV769p9oHcd9vkC5CNPfe4bpXSlPX7dj977yRO3Xq1KlTp04/C52entLr9VBKEccxeZ4TxzHGGFarFRcXFwSBM6mLLCfLiv3zaxzHAIxGI3apM/v0HtEHJydjhsMe3373W6IwpCwzvvnma4LAZ7VaNrg6yXq1wfM9VqsVRVE2WEFBv99Ha8NsNuHm5o7+IKGqtyA0QlrevnnNdpvje44NniQ9giDk4fGOs7MZZemMzqIoeHh4oCgKwjCk1+txfn5OEASNed6jLEsGg37De3fG6tXVFVmWUxQFi8WyMf9ryqLi6vIlcdwHLGWZAxDHCWm6Y7lc8+Z1QlkWjbEecnd3RxzH+88rSRLz/v0HfN+j1xs6Nry1eJ7H48MD88Wck5MZg8GAIAiJoxhtNFeXL1ksFmhlCYKAT58+MZlMOD09Ybvd8m/+zb91AYrQMhwOuL+/4+Liktlsug+JDIYDHp+eGAyH9Ad9Vqs54PZDKUOSDPG8kDx3Sf5e0ieKI9bbNW8/+5ynxxUvX73i/u4RYwx+4HF5dcpk2icMfa6uLlguF5yezahrzccP13zxxed8+PCO4XAAAmYnMz5dXxOGIcvlnMViwWDQI88zrNV8+eVb/vHXv+bVqzfUekltCtJ8yZu3b/gf/l//N6aTKdt0Rdj7gqcffmQx/0StKrbbLaPRiCROKMuaOI44PT0jy1LevHnNar1ESsHZ+RlFkVPXNZ7vcf3xE6enF9SVpihKtLZNgt9js9mxXm84OZkihGC73TbXrmA+n5OmKScnM3bpjqSfUNYVQkpmJ6esV2v6/T7L5YLLyyvW6zWnJ2d8//3v8HzJYNjnxYtLhqMhWZriBx5RHPDw8ADA2fn5H72HO5xLp06dOnXq9DOVwGEonJHu0hRwKG5sCzMdpkPTJsCdiesjRYgnQ6QIms19//g1jhPt+5LMY4P9yEA/5ojXR0iP479rmdGtjg1kZ9IHSBniyQBP+PjSccT95nv+HusSE4QxUdQjinvua9QjjvuEYUIQxHieD8KVfnLEDVfmkHRuMTKHr/qAHWn2v03NS3kwqD3fpeItrUHtxtf3W/78c8a4btIxx/zuAzP7MM4HxM3BbHbGttifU8csf148KqXcc9Jbvni7ygAEtuGFG9OsXLDuChK2KXc1xiW9q5KiyPfFWGAdj73BuXhS4ns+ge8RBO6rlAKzP0a9n7QBh0PZs+WVakpXnaSQRyiU9tpoefnN+dE1tSodnqYumvRVQV0XVFXRYGsqqkpRFBVVWVFVLgXWpsHaYzFHqwt+ykzfrz7QbrWEO0c0RrttViPIBn/kHeFb2N9j7XG215LWnYveqVOnTp06/SkpCAKKwpm9vu+TJAlZlgGw2+2Yz+cYYyjLEil9Xrx4SZL0OD09p9frYYxxvOfRgKQXsd1u8aQgDD2ixKOsU6Q0eIEzjheLOb7vSkyjKMZamEynvH37mQsi1DWT8YTpdMJqtcDzYL1ZkiQh19fvEMKw3ayYTcbc3d4yHAyIwghVa6SQVGUFxqLqmseHx72B3u/3OT8/ZzQaIYRgNBohpaTf7+957MfPRWVZslgsnIncG/Dy5SvCMGG3TZHSh4az/vBwz3Q6IY5jRqMR/X4faw3b3ZogkIxGQ168uOLNm5cIYTg7mzGfP6J0zYsXF5yfnwKa6XSM1grPF/QHPf7qr/6Ct2/fEASOV//u3QeqsubTp1v+xZ//FbtdSppl1LVDAiIs//yf/zNOZjN+/etfE0ZutcDnn3/OarXk5u4TQeQzGA/4dPuJbbrl6sULTk7PiJI+lTIoIzg5e0GUjHiab3nx8g3T2RnCCwiiqEG9ZJxdnKONIYwjtNUgat68ueLkZIQ2JdPZmNdvXpLnGVJKxuMJVaVYrTfUqiIIA6q6YnZ6Qpbn+EHEN7/8Fbu0QGmDF0R8vP7EL//sF2TFmr//x79BmZTZWY/F6gZkjgwL/KTi2x/+Izf331HWKcNhj8tLh+yxVqCVYf60IM8KkiTh7u6ePM8pq5I8T8nzlDRNCYOAN2/eUJYl2hjevPmMwWDIm9dvMcaSpilRFCGEh6rd8/PZ2RlCQBB4aF3z8HDPbDal1++xWLhU/3bnzPY4jpmOJxilwVjqquLzzz+nPxhw/3CHtpqqLgnjgM12g/Q9BqMBcS/h8y8+/6P3cJdE79SpU6dOnX62arEscl86eVwy2RqSLhmtGjPdNkiK1oBtjXjHgW5yxRzwK84IbFPPxwn0Y7UGutobqb/PSW9f/5j80jK45T5R7OPJAClcGt2XDbe94bUjQUgfKSSelWjrYzFI2DOu233UWoPSWEA2xZZYV2y5N6HlAd3h9lvv998Yi5SOe96a0QfT+jDRYGyDJsGdhxYR4wo5f5rOb1jZVnBIQ9sj/rqiVgf0TmuK/97EQ4MNOawUcJvxPKT28TyD7zmsiBsB0xTJgrUuSW+NwAiQxmKs2p9HZx6Lo1JYf59Al9Il0Fusjy+le7g2pkmHszfbNRprOExYtKWuzetIK/GEh2gmH4RsMTdg9ogdt1S0rkuUrihVgVGVQ78YAXhuIkja5t8evq/xg+eltkEQIHzvaHJJHZnlzYSFlU1qXjcFpm4lgrseaJJiAZ7096s12tc7LqFtVz10OJdOnTp16tTpT0ue57FerwH2Zjm07PJ4vxJwOBwicYEHKW3TueM5wzBN2WzXDIcDTk5mZFnGKPHYbnfEyRmImtnsnIeHB8aTAUmScHp6Rp7neJ5PXSseHx/Z7XbEcUxR5my3W9Jdhue555c0y4iTkKrKiWOfpBez2exI05TPP/uCqlIsFnO0qQnDsGFWK2fKeh69Xo+/+Zu/5xe/+AxjTJPwDlyKfrdjt9s1JaXsMZB1XTMcjrBWsN1sHYYSiTGWoijYbre8ePFiz8S+urri7//+71ivF8xmM6QneP36FWHoCjY/fHjfdOoI8nxHUbhU/3gyRAhBGAUEfsD5+SU3NzcMh0N6vZjhcMif//mfMxqNG1xjyH/2r/417z58S78fMx4PCYKIQX/Ad7/7Lf1+j4f7e168uGq6dF4xnPT58d0PDAcjtKmZTic8Pj7y6tVrppMZeZ7T68WEfp/tpiBLK4z2iOMBQpRsNhvOTs8dN19EBH7E7373PZ+9fc3t7Uem0wnv3v/gyjpHI+pKcXp62uB/Inw/4Ksvv0TpgtFoyIcPH/jFL77mxx/foZRmOJxSVxZBhK49srTm/u6B/mDAhw/v8XyPh8cH3r59jfA0y/Uj1hqe5jeEscDUlqLMOT+75P37j4xHU8bjCVEUMxwNKIqUIPDZbDPOz864vvlE4IcYo3l8fOTrr3/JaDTF92Pubu8x2jL78xk3t584Ozsnz3OSJKHf75MkCUopHh7uCIKAFy8uXbJ/NOT27pa4l5BEbqXB5eUVdV3THwwosoLNao2qNd/86hv8QDa9SRV5XhDHIW/evuHTp2uEEHz++Vvu72//6D3cJdE7derUqVOnn63actDnyePj4k6tFbWqUKp+xkQHCdZrzOkG5dIa1Tznch9vz4san2Nbjo3g5ziQ54a63CM1jtjtosXFtPvh4+EhheeQLrL9oOHvf0d6foN6CZCeKxcVbWkq7th8PyTwIwI/fFbCeTi+n/LI2RujrcGPFc28gsAa3Hbkj7avVdcVRVFQFsXeUD0cvytLdUatOz91rZ6l3n+a4teNKb2fhBCHVPRPmfWH8yoa4z/E9yN83yX3XelqhO+75IcxAq3Zp71/ep5ahr57LwPCAAZrNEKC5wl86SYT2pS6lO57nmzS/+0u0QzfUbLe8xxHMggCgqastS1MdefHOAO9eRh2W9kk0dtkuvtZWVaURUVZlPtS0eMx/MO8efPM/K5rV05altX+/dxruCIr0Uwc+J7fTFbJ/ThpXTuTvyk6/ekKjk6dOnXq1KnTn4Zub2/xfd9hLRrW89u3b/fhg7qu2e12lGVJGIZYC2nqErz3948Nk1wSRSHL5ZKyKtFGoXVJfxDy6eY9w1HC7e01g0GCMZqHhweyLENKj48fr/Gkt+eTt+a073tcXJ2hTYWlJs83JL2A2/trlKqwzWeE3W7Lt99+hxCCk5MTfN8jSZLmmck9W11dXWGt5e3bF2RZhlKK6+trlssl79694/Hx8ehZ2pJlGZeXl4RhRFXV1FXNbHayx89I6THou/LVJIkoq5zBcMDH6/ecX5zy5VdfMhgkXF6esdksybIdT08PKFWRZTvOz08IQ5/VauXY4U2a+fHxjnfvv+fu/iNJzyfNNmT5jt1uizGa4XCItYLNZkuel/h+wHg8Zr1eU5Yl9w/3AJydndLrJ7x8+YKqqhxL/PaW6WyC8AwvXl5SqZKkl5D0+nhewFdffcPF5UssktPTK/7b/+a/4+bmicnkjMl4xnQ6I8szkl6fH378npvbG8aTMYvlE2m2RHqastphURRFun8evry8RAhJHLkS0yDy2aYbZicz0jyjqhVx3Of29pHBcEYUj1gsUi4v34KIqCpDEPbZbDLKSiGkx2a7ZbPZIKSg1hUXF+cMhgPOzk6wVvPNN1/v0+JJkgA013GK1mqP9fF8ycuXL3n16hV57tAuAsH52XnzfO1W5YZhSJ7nzarcw2fB9XpDFIVorbm6uiQvchbLJXEco7Tm4vKCk5OThm++ZrfbufdKM1bLJePJmOVqRa/fo6xKXr1+ze3dLf3hgG26Y7la8u79+z96D3cmeqdOnTp16vQzlWiT6HvMhDwy0t3vuKStPsKlNJtojcADBgYOmI/nXHD77L+PzfJj7MlPUS9/WMeJbtijTNqEdsNkB4Fpvg8C2fDfHbLkeMLAc+llKTEIlLEo0xDJpQQpEJ5EeA3SRbRp5+N9oEmitKnzw1i5MWwxLL/Pem9fw1qN0oq6ro4292HjGIdzPH6tYf4cAaKfTUIcjHhXcHn88z80ueES1WI/ieDJwCX7ZdAkyo8mTtpz4fZuf96EOKSGjidIzJEpLMWBj+95Et/3iRojvB1D0fye7/v4XoO5acbW3xvnPkHg71/rkO5ur60arZ2hb5rUv7GHotz9GBi9P09uIkQcHU+zauDZxI/+yXk4mOdV2eBgqrrhoLf3VXvNHQphn080mZ+cj85E79SpU6dOnf6UJKVkNjvh888/5/T0jF5vwGazZTAY0usN0Nry8uVrtDZUdUVZ5VR1wWa7IU1TPN8nzXKiMKE/GHB+folSmtVmSRh5RFGAkILBcNAYkaZ5hnLPJRcXZ82zjKauDI+PT5RlyePDE/OnBbtdipQeQRAShhGe9DEGFgtX+rjZrInjiLousRh2uy2r9ZLVaslwNCKMfKx1paFtmWUURS7xXhTkRUqvF9MfOCNTSjchkOc5j4+PLOZzjNWs1yuKIufi4pQ4DllvlsRxSBSFPDzck+fp/vny6uqC/qBPVRUoVYMwlFXGerMky3dst2vOzk94+9lL5vMnPF8SBD69XsJoNEJrxWaz4fHxgdlsyna7aXAzFXmeNVzuDS9evGS3y1ivtwyHztT3fAkYttsln24+EkY+2lZs0zVptnWoHmG5uLxkMpnw/fffkWYp1hrevHlNuttxeXHJx4+fmM+XeNKnqhUgUKp2yffJAM+3SE/R6/sMhjFhJHn72WukNOx2axCa4ahHHEeAZbVe4nuS9WZNGIdkRc5mu+Hi8gIk9Ad9prMpRVlw9eKK2ekpWluGowlKQxD2qGrD3f0jnh9R1YbHxwVBkHB7d08QBKRZxtN8zny+5OLyku1uh5Q+RhuqSiE9yRdffEFV1wg8dtucsiwJmqJcpWqUditJzy9O8QPBZDJit9u6j2hS7MNeSRLzl3/5F/si3iDwSdPt/nn/+uNH+r0+2+2Wfn+AUhrP9/ADn8l0zOPTE7s0x/MDjIW8LLi+uWE0nqC15dWr17x7/4Gy6Zv6T97D/1/9P0SnTp06derU6f9nEsjG4GtMSBySRQivST27xLmwHlIGDkXhtaaq3yBcOPCr98WYGovGorC2TSSbnxjpFqUtSoPRwpVJWg+Bjy9DBD7SesjmqzBuO/4eVjbFko1533iOFpf21ga0bYz+JmHfFqLKxkDfp84bTrWxoPeG/H6gmk00xrdLUrcIHE/KxqR3Rv2x8SyQTYpa7DeswOjG4Le43zAaa2qEVQirwNRYXaF0tU9073ncSu2T1XVdobQzil0qvDWFaXAjAq0ttbLUyqC0QbWTGEajjEIbhx7RtkGxtDn1ZnLBrVSQ+/EVQuDJBsfSJKNkMx7tiWgLQ41x5r0xDoNDU1CL5zYpJdL33Bb4DpkiRRO8l0hPNu/l49Ew1aUk8H18X+L5EuH7e+CMaQ10rajrEm10w1F38X/PSoT1wfr7a7w1yI95+8IeTcdYV6irlUYrjdGgtUUrVx7aptCrZjLDrdxwprjRlkOB7zHyqJ1oalnz5tk5bvsIOnXq1KlTp05/Onrx4hVpmuH7EaPhhBdXr1jM11SlIUsrhPURBAR+Qq1LKpNjpCavUkQgWG3XBHFMqQybXclmV+H5feKoz3q1Jk0zbm/uefP6c7dqU4ZMJhPKMifLt2x3C3zfI/QjLs6ukDZElTAanKBKQeQPCL0+0kYIE3Jx9ppsp4miAWmaYaxivZ2T5SuybEUcB8xmU6IowRrNeNRn/vTAerVkPBrgex7DwYCH+zu0qjg7OeHTp/esG/56mm3RpqYoUobDBIRiNO6xSxdIT4Go+PDxdwwGCUHgUVUl5+dn5HnGyYkrMwWHCoyimKSX0OtFxIlPrxfw9rOXXL04p9+PmU5HWFzwRBtFUeZNmWXEZDLjzZvPqKqa4WhIrx9z/3DL51+8pa4L0mzLw+OCsrYUlWKX5czOpviRxyZb8B/+9q/J6w2F2rLLVyRJxKg/xBcexS5FGkOe7gh9yX/xn/8lmJKTaY/xKCJLVzw93PP66gWLxycWj4/4UvLqxSXr1SNJBFJm9Psa6RUok7LdzfFDA1IRxZLN9gkhFHEiMbZgtXqkqh3iMN9tUFXOZjWnKnd4omazfuD+7h39niSKFIuna5LYx6iKLz97TRL5/MU//+dMR1NOJqf8q7/8z5mOTrC15GRygfQCev0hCJ9Xb95S1oqn+RLpe3y6u8UIuHr5kn/87W9J04rJ+Iy6gtVqx2a9wfMkoOj1fMp6Qxgrrj99i9IptUo5PRuzS5esNwuEsJRlxu3tJ8qyYDabUVeKKs8IPItE04sj7m9uCb2Autbsdhm7LKXSFUHsc3t7x+31Pdm2BO2hK0G6ySnSCl1p6lIxGU4Z9Ud/9B7umOidOnXq1KnTz1S+L/GEwG+MYCxYDQhnEjpKiivnlABtUvwnIfFDItzhXwQKaHAmwnPsbEAhm4SvhxRgGpPeGfXuZT1hsdJCrRo2CvtEMkIgPOH80DapbASi4VNrYVGeRsoahMDzAqTnodu0dNP0aIxGSIMVzuD0hIew2hnaFtwbuAPzZEticblrT+AS7lhc36hoXrYxTK3EEz6CJvXcTkog9zgWjESIJp0vFMZWWF0jTQ3WmdzGaDeJISRGgNbNuGtQDSZGCOPQJ0KilKKq1L4E0/M8hCfRBmxtMNYj8EWTpjd7cj1YrNENQ7xheFvQaLf812isMCCsK0LVrijU8xx2BWGQwn042XO8rUXVCmubElrfnV9rjCtStdp9RYCwWAn4DnXj0CftdeU3zZwKazQSgQWCJrV0vFpB1QZXgKqwwrqJnGZCR4i24dMiESCi5q+Mmy6wDadegBQGrHaTDwqkdaWqxneXBTj+u2P4uyJZrQy6SbnbZuXAgTffJvr9PWO9mV4ABMZYlNLU9fMVBJ06derUqVOnPz2NRiPW6zUPDw8YbZlMpgjh+mPiOKEsSrCCJE5I8yVSCsbDMWVZUhQVYRgRhhFGg8SnzGuKvCT0XQI5CCSe57Pb7ZASsnyHfqp58fIKpWqKIiMMwgaz4j4UvHjxEik9vvvuO05PzyiKiun0hMViRZ67Yk+l3KrV09MTVqvVs1WUg2G/QdEopAfnFxdkaYa1gpPZKYv5gslkChjW67Xrogl95vMVr169Yr1eMxgM0Np11tzefmIw7FNVJZvtmrIs2GxWWCyen1DvKsCidE0YBuR5RpL0WG/WLJdzZie/oq4L/EDy9HRPHCf89rdLptMTlCoQIiJJYura4Qpfvrzcc+Kn0+m+wFJKwcPDDSenE7a7NQLw/ZCiKImigDTdNZ+FDKdnJ/iBxPclUoJWit12y2q5Jgxj6nrDbHaC78PNzTVFsePb3/4GazWb9YJeLySKfZQqXZDEkzzc37NLN0ynYwJfopRhs1mRJDFptqPfG9DrRWgF2+2O2cmE+/tblKq5vDrn6emBKIrwPEmW5QSBz2g0QCnDdDplvd4QxwHL5Zyk13MBEl0hJKTpDmPcCtrZbMpisaSqak5OTqnrirzI+LNfvWW12uF5Hv/1f/1/4K//+n8hDEN+8fXXfPz4nvfv3wOSIq8Igph+fwBW4PkecZxQ1zVZnhHHEdYa0mzHZDLj/r4mzzO3qgBLGPrUtSaKIoIgYLlcUtUVUrrOq2yXApYkSXh4eKTfG6K1Ii8Kbm8rrLXMZlOWyyU0PUu9JMH3gj1CSSC4vHzB/56ceWeid+rUqVOnTj9THQo7DyWTLdPbbYdkrjX2qLixRZC0yXL3oNz+W+A2614YvMYMb1LYWAnCw2uS4VJ4iDZFbizi6MG7NWVtE+C1WETruIu2GNVz79jwE+u6xgpcWl1YkBqNj/AEQlqEbF+zeR0sYm9Mi2eJZN2kwB1fvDVGBcYcFYPuMSDt/kLLjReiOWbaAlCLRbtjwBm2lhpr3cQD2pnXSrlSTSsEvgfGA6E0nqfxfB/hSTwsutnP1kRvH/gdPgawniPueE0Kvhknux9fhyYx5ohXb21TitmelGNUywFzIpukuqOTSKzVzfu2mBq555cfl9a2YyalbPA47bl9/rXVMR/cb1joB/Z5+14Wa0BpkLI+YIqalQhSGqQ0WOs1CXPr3la2nP0DXqY9RmMMGkMD+sE215mxLRPdNB8oDAZ33NYcynS9I3TQAR/UolzYp9VbM77973a1w+/NVnXq1KlTp06dftZq2ee9JMEYUKraIwNdCERgjOb27pHROKafDLm4uOD+/pFB35LnBQLpgh3AerXi88++5OH+BilDsA6Ft94sWKyciSqVZrOV+F5AWdakaUZVKqqq5PLynOvrj/zyl7/CGZYh2+12X3qqVE0UB6Sp41vneU4YBniej7WW1WrF09MjWiuuri54eLgjz0uCIHQrJWv37OMMS4mQlrIs0bWh10tYr1ekqUOmvH79mh9//JHpdIoxms1mQ5IkVHXBcvXEZDKlKHak6WZvOFd1QRL3yIuUqsrJix1gULrG8z3mT0+cnvkMB8Pmbyq0rpFScnZ2QhQHbLcbjNEkvQhjVGPewmw2YbNZU5Q51hqiOOTy6pL54p5aVRRFRlHmzTl1KzvLokIpxWgwpCprwihEqxrfD8iyHcPhkHfvfuCrr74gjAJW6wVBIBkMYqoqpdfr8fH6lutPJf/yX/4lo9GALM/44fsfscDbt28wRrNarxgOPabTCfOnJScnM379618zGIy4uDjn4eGx4ZBbbm4+MR5PCIKQ1WqF5/mEgTOkHXdeYK0mDCOUUuRFynK5ZDabMp8/MRj2KYoCYwzb7ZayLImiiO+++8Dl5UuWixX//t//B3a7lO32jn/xL/4Zv/nNb7i6OqffH/H4sKKuFMPhGGsNVVWRJD0eHx+JoghtDavVmuFwyHKx5OLiAmttMwHgYa1l2bDP3fUXslqvODmZsrxbkqU1UvgM+hOMqalVSX/Qo5jvGI6mbLdbev2IxWLJ5eUFWivCMMBYQ1EoJpMJeZYTRTHj0fSP3sOdid6pU6dOnTr9jNUamq5QskmjY/ZG4J4JDY3x2JqMjXFpTZPGdssfXSK9RYK0PGlnmtsmie142wFS+I6N3RiLrbFthWmQJ006WmuMtiDcvjrj0yWjkSAb/rbw2gkBg0Lh2wqhDUJpjFBIZJN7b/ZPtKWXgKFJWLtkSWvQKqUA9uMhpdyjXo5LJ4+Nd9xcAxIwR5MU1lqM1k3I3WKNwhiFpQY0oLBKNYgVl5BGSpRVaG1BSDzP4BuD9D0M9mg/9b5o1M07aEDiSbPnk+/P9dEESJvWPmaot6lqtDtmYw/GsDi6Jtx1YdzEh+DAtW+vFCH3RrLn+fuy0HZS4tgEPzbXDxen/L1y2WPW/HHxpuf5GGEQzeRGczm567ll5QuX/nGTG/zeNX4w0du0eAN0aUthW/4+9tk+AY157s7vsYl+SJ8/v+estdS12vPsj7sB2uP8J2sBOnXq1KlTp04/S7kSTx8ELJZP9HtDJpMJVVVRFCVnZ6fUdc3pySle4J5Fs6xECI8o9ImjHkVRUpYVFxeXrFYrlzL2e4yGA9J84xLK/ZjhMEYISRB4zjxWGt8PSZKYXk/y9DRv2Oc+VVXw5s0bNps1xlh6vR6uHPQVT/M7rDV8/PiB6XTKw0PKN998w263a1YEQhgGCCE4OztnuVyTxD0+fbptnhUDhHAhCWMdIm+32+GHAf1+nw8fPnBxccF2u2G325IkMbPZjOFwgJSSwaCPNrVbZQr0+jFllbNLNwghuX+8YzadMRoP6fdjyjInz3cMhyPiJGoS+RnWWibTiXuGl4LBMHGdS2iEtISej1Y1Z+en3N7ecv9wS7/XY71eNgWeGVm2I+lFzOdP9AcxEWHDfo9YzOcMh2PKsiAMfbCastSEUcB2swMEvi+5ufnE6dkUpSrKMmd6MuLDhw+MxjFZseHFi0uEgJubG5IkwfM8hsMJRVGQZQXz+SNJknDz6YY8K5iMZ9zdPTCdzgBLWRYIgeO5S0O/3ycIArTWJEnCaDRmt0spdinGGEajEcYYiiJvficmy8JmksWdl7Y4NE1Tvv7666Zs1aKVK4bN85Ivv/yKT58+8T//z3/Nn/3qz/j229+S5wXj8QylHCd9MhmRZY98+PCRJOk1ZaIKT/rstilBEPD09MTbt29JkmRv2u92O4xxpaOLxYKqrAiDmKfHNVEUcXp6Sp4XbDaOpx7HIZ4nyPOU3W5NL+nRH0QgDEkv4t27D4zHY+q6xvd9Li+uSLOUjx+v/+g93JnonTp16tSp089UrZmndYPCaBKwpjHDDzpOZzfJ6ibB7XmyMf4ax1K4r9Y0xqHwEMLxoJ1p7iMb7rnvBXjSdwldF43em6YeGiVUw6BW6AYDYmyD5hBmv6BOStmwu6VjlXtNqlgarKwxwiAdgMVRPQzOQG/22wrACESTZtiXpLbAkyPgemuytpMH7v2PJxUcqAR4bghzYI2Lduyb77pjVhhTY1WNthpjaZjtOLY85oCFEQJhTRtwdwgW/XzigwZD0q4saJMkbcHmwbC2++LN1sQ1jVm8Xx2wX5lwwJQ0V0VDSbGAM49dultiTGt4e/i+SyRJTzp8yxH7uzWin5eg2gbAI5/93k+T4sc6sMQPpalugsDtqRSeY7A7ZhDWip9MghwmBlxyvJ0AcCgWfpKkl9JdE8a2CR3xzPD/pyYGjr/3vBS25ca7FR+tqd+pU6dOnTp1+tPRarVkMBhQFAVRFBFGPqcnp3z8eM1utyUIAobDIWmaoWvDZDxB4nMyOeH60yd6SQ9VKaIgJPB8ZpMp19efSOIhWnuku5LB0CfdbQFNWRWukDOM2GxysjQniROMMQ2yxHJ6NkN60O8nvH79qnnudCvo0mxNv99ntVoyHo+I44heL6EoHB5kOp1wf3/PYDDg/fv3XF1dYYzh48drgiAkzwsuL6/Y7TYURYlFsdvtiJMYIcT+ma4sSyaTCWdnZyyXS3zfd8WdnsfLly959+69Y6BXFWXlkutVlVMUJWmaMx6NiEJL1O+xXM0JQx9jFP1+Qp5nLm1cpAyHI1StKMuSq6sX3Hy6Q8qYsiyJ44j1eoEfCIyp0LqiVh55vnOFl1nG7d01k8mI+4cbPH9Emu6IIpeQVlrvcS7DYZ/r9aIpP5WMJyN2ux1B6PP1N1/h+5LFYsV6vSTpB0ynI3bpmrKskNLj/PwcYzVZllIUJVdXL3l8fOTpaY6UkiTpURRLAKqqYjQakqYp1kJRFM3fXHJ/f8PJyUmDoInxfZ/NZsPNzS2j4Zgoilgu3TVZlq6I9s2bN0gJYRjyNH8iTVOurq4IwxBrBb3egLoy/O67v2fZ61OWJX/2Z/+cxWLBcDgkz1Pu7h5I05y3b99ijQvcqFrhyYAwCPEDn6KoiKKQOO5R13q/Wvjk5ISHhwcGgwFCCHq9HmdnZ3tDXQjBy5cvieOE8XiC5/mMx1MeHx/p93v0egmbzYqqLvAK9zkuzTb0+n2iOMCTkul0SBiGTCZjPn26pSrds3qvN/yj93Bnonfq1KlTp04/U7WJ4Bbdskdu4IoYWyMQaHAUrZFu96lz9+8DDsSi92lll/j1kELiCQ9PBEh8PEICGRF6zkgX0pnoDgXSlFyiEUKBrUGLo5LFpmTTWpc8bsAxrovSGeLSE3hSYIXCCuFQG4CwsoF4APvUfIN2sdIl5XluGkspn5mg7fEfm64uxd9gt5t9dwbrPgbt/haHQJFtAhuBlY5b3hrAVimU1a50FI2LTnsI4eF5IITGYN2YiIPJ2pZU/rS0ssW6tCa6UiCkPTKtNcY0paRtElo2KXIhfs8Mts1YtIls2tUGpl05AFJaPM/H9wOCIDzgV2R7bfHMbD7+kOSwOe1ByaPEvN0n6o8T2+7alM2SZ01Vtaa0brAyEk/IPcLQYYgO5vzx+7VjJKXE9x3D3G+uX/Ysc4vAIJDNigO3xFUbg1aqKSg9GP2H++uApNmb6JVCKYPROC47nhuf5t9CdCZ6p06dOnXq9KckISxFkRPH8f459ObmEy9eOpPy4eGh4XWXTEZjgiBgu90SRRGqVsyzBaPRCGsNabpzKEYsYRQCEAYBs9mM+4cPIDS+76FUjVIZvaRPPxkjPUFdV4SRS/8CzGYznh7nDAYDkqS3555LIdmlLh1elgVK1wQyoD/oYbShrEpmJ1O00sxmU4bDAXleMBqPkMJvEuougT+bTQjChH6/D9ay2biCyV/+8hvu7u7I84woDonjCGNcgjvdpSjl0sJaK9JsCyguLs4ZDBI+ffpEEHhEcUjSi1ksliS9kMXiiX5fNM+QmizLWK/bRPUFWhtWqyVKV6TZltVyBRiMVeR5ShS5JLZFu+LTMmM6GzOfz0Foqqpgu8UhGnVNkvTdM2Hz+WC725DnGVobBoMRRttmtaQBNLd3127VgSpI0w2eJwgjj6p2OMSHh3vCMEJ6HsYY0tSlxn0vRAjDcrmil/Sxxo1tnpdcXFyQZTkAvV6Pd+/eMZ2OuLu7IwhCRiPXOTQajSmKkqqs2Ww29Ho9zs/Pub27YTAYgDDEScRqtWK32/Lq1Wv6/T6TyYSnpzm/+c1vWC3XXF1dIYQkTXfUtUvfF0WOEDCfz3nx4iX39w+czM6RUnFxccEXX37OZrvi5OSE7XZDXVd89923TKdThsMhdV2zWq0AuLu7YzabMR6PkVIyHo8Zj8dYa3l4eCDPS6bTUz777DM+fvxAksR4nqSqCoLQp1jkeJ4kCBz7fD5/pNdL2KUZnuex2bhJi6oq6fcTHh6emp6A/7S6p/dOnTp16tTpZ6rjRHBrorYM8APr2xmYLtXcJNFxyBTHc1Z7M9ZyZBg27yGaFLBLVQs8PHzh44sAXwYEMiCUAaEfutSMHxLIgMBzCWDHO5cN8xy0tQ1upGGMN6a7K/s0GJrCUGkRngVpsFbtk97G1AfTX2usaQ1rfu+Y2+/BERt+b06bZz9r0R2e5x3wMs3PRJuglhL5k99znOzD+Wj3QTVs96qqUHW9x4RordFKNwz0am8ou3MEpmF0K9Uyuw9GsXvt5wWW7WvuU+jta+1XKehn3xNHqJs26W72kwCmGQv/mXnuew7b047TMR/92Dw/RpvUtTu+uq6f7X/7e+33tdZUVUVZFvvNGenO0HcTHC4N78q24iYZHzT8Td8Z4lYgkEjpzH+/WVpsm+tsfw0I93s0f2ONxehmMz+dXOHZ/dWOoyslbdA7xu6N+99L2R8vBunUqVOnTp06/eyltWqKMjcURY5SNVVdIYTr6QmCgM1mQxRF+IFPnufsdo5RHscxCIPnC8LIZzIZIoWh14tYLZ/YpRuCKEAbxS7N0AYEPiDZbjZ4viSKfaLIpyhSQHNyOiEIJGm2RRvF09Mju92O3W5HluXs0i29JCLNtsRJRBB4hKFji9eqJMu2vH//I8YqhLSsN2uGwwF/9Vd/SVWX1HXJYNBnOp0QJzFgGY2GCGmZnUzI8pTVeslg2AdhWS4XxEmIsS58MpmOCEKfKAp5enpkuVywWq3J84LlaslwOGA8HlNVFeDCFGnqJgaGwyFlWRKGIXEckyQJYRiSpilSSpbLJUopnp6eEBKqqmQ4HLBYzh2nPvD26egkiTFWEychVZUzHA6oVUkUhYRhQBgEgCHLcqqqYr1eIYTFGMVut2a32zAY9CjLHCFdQKmqC4ajPsYoEIaqKpASprOxWxkw6GGM46oXZU6SxDw9PVDXil7SZzKZUteKwWCI77trxfc9qqoiigKCwLHaB4MBeZ5TFAX//t//e56e3GTJaDTi8vKS4XDI0/yR5XLOzc01JycOCzOZjPnqq6+w1rLb7bi+vub+/p4kSTg9PaOqKq6vP+D5Em1qlHLn++Hxbv/evV6Px6cHoshNcnz33bcURYG1BqVqvvvuO/r9Ho9P9/T7PcbjEUEQkCRusmWz2aC1Zrlc8vj4yGaz4frarXKQ0nH+q6pyr/H4BG69LXVdI6XH69dvuLi4YrVao7VD1pyczAgCn8Ggj5Tg+x5hFPDnf/5nvHr14o/ew10SvVOnTp06dfqZ6nkKtzHzjMOhwPOkrxUuJ0vDR2/TzQ6Z4bLebVHlcyCKbFLrIBF4QuC1yXTalLorXHSIFQvSYrRjUTuj3JnKymiX0hYahCsDFdaVg3riUApqsRhhXXIbh6ixQmMQCCsdvsYe0CqiQdMcY09aJvWBe/3cGH1usrdba4DaveHKT9Lo0KaX/yngtcPMGK1dCalwqXqEB1ofEC8uWHS0WkD8obdqj3CPdbHtZENz7n5q8oLje0tpMLY1y48xMQczu/n05QpacasVgiBwJVXSmecu0e0fzOG2UJXfR5scTwiYprSzPRdek7Q55pf7vr8/H2o/sVBT18dJdenOimyNb9kkyRuuPh48n/JBNNgfLI5NjwEpHcKlwcNYC9YYN4mjf79DoN2v3zsbTeq+LaVquwh+WmzqkDIdFL1Tp06dOnX6U1K76m65XNLr9fA8QRD4XF9fs9vtmE1PmiJyzWK+YDoZUZUFSRwgSMDWbFYLzs5PAUVVZfSSgOVyTtL3yfOCi4sxgRehSkDDyWxGunNGfZptKMuCXr9HVRVkWcp4PEVrxXK5YDKZIaXg5GRGmqbkRY70AAzDYY88z4AqcvIAAEEJSURBVPA8ydPTA+PxmMGgz2QyYjab4Ps+UZTgexHb7Zq3b1/heR5VlRPFAXm+w1rjSkDrkunJhMGgR5ZlBEHAu3fviKKIweDkCIen2Gw2qNogpUBrw4sXl2y3Owb9IVprTk4mXF9/YjQcEwQhdV5ydnbBzc0dFxeXxHHMzc0dYRBhNKRFxmg0ZjgcU+Ql3jAgDEOEEKxXW0bDCWXpyiqx0O8N2Ky3iAB6/YR0l/Ly1SVPTw+MRiM+fbolzwviqAdYZ1oXGb1ehEWT7jKGwwkW5dApwwFhGFBVOev1ynVC2cCtABWW1WrJ09MTRVHgewFxnNDvD1guV1xcnCOEx+vXb5nP50ynJ9zc3BCG4X7MHFIlaxL9hs1ms39+v7q6oixLpJCkabbnpau65uLijPn8idVqwWg0YDgc8eOP77i7uwUk/X6fMHRsdc8TGCMYDHvUdUGW7RCSxvSuefX6BfP5krIsSJKQ5eqJWuXM53NXnvrxRwBevroiy3acnZ3w8fo9vV6f6XS6T6MrpVitVgwGA7TWbLdbTk5OiOOY+/snXr58SZaneJ5kOhsiJBRF1nx2ksznK/q9PrPZGcvlgiCIuL29oyxKLi4uUUqRpjtWqwVFkdPrJX/0Hu6S6J06derUqdPPVIcUsNqnk2ulnqV893zpBguim8StSyEfb6ZBgzg8iLEag8bYhk1t9B5l4gmB19iVosGoCwvSthbmwaxtDds2nVzXmlppVGv8Wtsk0TVW2v2yVVeP6spDkbg3OEqaG22w2oCx+/LJn6awj9nhreH80wT684T6gWt9wMQcXsceJb9ty3ZvilhbtYzxdvJAt7zso3Ox58IjjpAt7X6JBv3iyjxbvvd+1cFRCvo47X1cbnnMFTdtyr/dyZ+sUnCTLx6+HxJFEVEUEQQRYdgm0YN9ueihgPYwAeEKNg/J8jZhX5TF/pr8aVq9LMuG51g0KfRyv7XpfKUcT18ri9au2Kj992FFhTz66vjtINzvN4n+dpUCSKxt2efN+bZgdHsccDzRcnxNHK/4OHDQteOtez6eF+w3KX1k0xPwT0+0dOrUqVOnTp1+joqTCKXVvhclTTN2ux1gmEwmLFdLVqsFabqlqgqKIicMAzbbNQ+Pd1R1ThBIyjIjTTeAQduK2WmfrFgynQ3Z7VK++vJXjIZn1JWkrkCKAK0VYeiBMGhT8/B4j0WzWi0wxqFfwLJczlmv11hrGY1HpGnGZDJu8Bs1u90W3/coihxjNK9fv2IyGTdFmRnrzRNZvsUY5czJMsNahe8LQCGlYTIZsVqtCIKAuq55enoCoN/vk+f5Po3sykstWZYzny+5unxBEIRcX39Ca0tVKZTSjEcTwtAVqfb7Q6yVZFnJcrnm48dP5FnepLEryqImS3NU7bp1RqMxZVkxHk+wFsqyIk0ziqJEKcPT04Jev48UgjRdU1Y5abpFKTeGWtcURY61hiAIwFpWqxVFmVGWJbWqMFaxS7f0hz2EtGT5jqoqAcN0MkEKzwWYmlWfZ2dnTKdjEIayylG6AjR5kRGGIdvtljAMOT8/58///M+bQJPh5GRGlu0Ay2w224c7jDHkec4XX3wBQBRFvH37ds/nd4GmijAMkJ5gtVqwWMzJ85x+f4BSCt/3CYPQBVqUwmI5PT1BqYos21BVGX4giOIA2ZAS8yJjOOqz3a5YrhacnZ26Y69LBkP3/X4/wVrFYJDsr6sgCDDGUFXVfmVBeyxVVVFVFZPJCCGNe/98R5JEzaRUwGKxJAwit0bai8F6BH5IkVdgBefnF6RpSlmVGKvxPHjx8oJduv6j93Bnonfq1KlTp04/Ux3jPFoDs91+WobYGrUtIuS54Wz2PHNnoDt8ijZq/z3rwM/OhDXOULbGuDSzNojm4U40BriUAildshzhzHCLS2C3iI3WuGyNdFfq2SStrXs/92quaNNxs02T4HH4E4wAK2lbOo/xJuoZXqR+Zjb/U2Y6HHLNthk7ezTGP0V7/H5yvK0btfu0/HOEzMFAb9+3RbkcG7nOjD1Gp7CfPDhMStT77dhMr2tFrQ4TK/bICDZHx9ymwYMgaLawMYBbhE/73g26xRwZ80fX3vF+tPullHo2vj81oquqemakF0WxN9F/fxLomA9/NCHwT9wT6mgiCcH+mNzqgvZ8e035qRvzNkl+fM/89OvxuW/H76cp9PY1flpI2qlTp06dOnX6+UsIkELQ7/eaZyVNXZdYC77vU1UlQRjQ7/eIo4D7+1suLk4JfIknLWenJ0SRj+fRsLQNgQ+jcYQ2KY9PN1gLVWXAhngyQYqIXtInyzJ26Zo03RAEHnEcOixLXaFUifQEg8GA4WiIMZo0S/chmigK0Vq5/Yoj4jiiLAuCMGC73bDdbRHS9fKUZYHWFdvdml4voSxzxzIXhrLKQBj8QOJ5kjzP9ynwi4sL8jxnMBgQBAFZ5gpEBwNXZCnwCcOI29t7zs8vmc8XCCH59rffobVlt01RtSGJ++y2KWEQYrRhOjlBK8P93RPpLicMY/r9EUoZqkqR5wWvXr1huXSMbM8LWCxWPDw8opRmt8so8pLdLqWqKrRWfPz4gV4vIYljsixlMhm7MUt3SCnp9RKUqhDC0uslWKub4tKlS20LsE0YqS26z9IMcMifLMvIsgylaooiw5iaNNsym83Isow8L/j06RO//vWvmc/n9Pt91us1g4GbhBiPR7x79yNCCJIkYTKZMJlM+PHHH1GqJssz7u/vWS6XLJdLXIK+T1WXrFYL8sJhaabTCVorXr9+zXQ6paprLs4vmM+fGI0GZNmWs/MZYezhB+D58OWXn+E13P1Xr67Y7bYMhgN8X7puLWGxVuF5MJkMqVWBH3isVgs2mw1lWbHdbpHSoWg8zwV1wjBkNpvR7/dRyhXU/vDD7xgMEl68uKDXi7m6umSxWBCFEXHUYziYcPPpjjyrmUxOOD09w/cDFosFQrgVF+fn53i+5PHxnrou/ug93OFcOnXq1KlTp5+pnhceHiE7+ClupDVhm98TjZkr2nJO0zDHWy56g28RHgaJEQqDj9E1xkq0cAa1tALhW4R1yRYrDsb+nqouDia18ATSyoNhbkG2JnXD+qYxbqUn0MaZ8cK2afMGTWJdHamQEislwgiMMHtD29qGwd2UpgL7DzIOgWP2iBHf9xvjU+zZ5kL8BFdiQViLsQahNQiB54nm9X9q6DYmuJBYeTj29nf3aXPpjOr2fdzfPc8+HFLyojFkGzPftAnv51/bok+swVoJ4mAMt0gXmqR8u9z4Ob7E7t/DIVFa05r9hywrzR5Tsufw/4FS0WPj/Ph6PTbg231rE/GmQatYKw5Gtz0uXbVIYRHeH054H7/fHq3CAbEihIR2IuPo/B7LJfMPY/9TBNChqFc2t1x774kmfW4c91SbP3BtdOrUqVOnTp1+ziqKnKQXUZUuIDCZjNntdqxWS6Io5uzsjDRNqWuFrguKIqXIM7TRnJ2dOtNdVQSBpChSwmiE9CxZviVJQorCoTbW6xWj0ZSqqgiCgLxwKRXf90iSmOVywenpiVt9ZwSeDABLnjsjtygzPM8Ha5lMJ+zSlOFoyHA4IK4jpBSk2Y66LvF8yXa7ZjqdApZ+PyFJElcIX5ekmTOWlZIUZYExhnSXMhiOGjKiOcKEeKRpym63w/M8xzEPEmazGXVds1yu0cpgtCWOE3a7lNlshjGGu7u7BmXSIgHd55tPn254cfUSz3tiOp2x26VNAadoUt/nfP/991hriCJX+DocjiiKgg8frvE8n/V6i7IFF9MTVqsFZVmx2e5IkoQoSsjzkjCIUEqTpTtGo4S63tHrD8BKHh/njMfuGlBKkyQxVVkQxwlpmmGMCwO53p/aTUiIiOGwz2q1Io5DLi8vKArHQC/LksvLS4IgwFqHgAnDkNvbW4IgYLlcIqVAaYXnSfr9HvP5nCLPOT+/4P2795yfX/DF559zf3/Hh4/vybMNu82ObJgReAFxHLNabVzp6O29Y8/XFbf3N0wmQ5QqQFiqukRKKMuCFy9eslnvqKnxA8FytaCu6oalH6J1zXg85PGxYLvboFTNdDolz3POzs8QIqDIKhaLOV9++SXzecZoPMKTHmma8dVXX3F3d4fW2k0YFBmPT4/EcQDCkuXu3P7yl7/kP/7Nb9Dapc6rukQIwY8//shwOKTX6zMcDri+vmY8HhHHCdvtmvPzsz96D3dJ9E6dOnXq1Olnr0MRaJs2N7+Ha6HBWLAvlHR8bbvPe+9Z26bloxwS59aag3lrFMoolKkd69xqlDXoZjO0qWezf492P8FgrUtmtGl3V3LaGrKNmd+YvdbgCjf3afAmkWxBG+FoLlZgtEUp06Sc632y+Xliu0XKOF63Q4F4TeLbx5MBngwcI7FBhVjboGna/TieuNib9gekiJsE8BDSQ/q+S5+IA23dzSrgSjCFh8RH4LdwnCNGOntAesutN1bvz1FbBOpWCrTj3PDtjcYa1UyI6KM0NXuci9aaSqsG/6OpK42qVfPBxTEp3Xi22Jiaqkl3uyLOZlWDaiYv2uvKgBAS7yj9/YdKSJVSzzAuqsXe7K9Rxz9vz4MzxA/X+8G8Pnw9Nuj3x4wrs90jgEyLs1H7a9AY3dwczbhZvX/NlqvfjrlpJnHa1Q92vxKivV7a1RLuuuvUqVOnTp06/enI6BJrKqo6o9+PWK8WRIFPEsdURUmZlawXG3zpITxD1Pf5cPuexXrB9e0t89UKIyRpmTOaTahNRVamhJHD7F1enpOXG9a7O7bpPXk5Z7G8wfMNYeTvcSXbbcrT0xwhYDDoUZQ7gtDgBRWeX3J+MSTpWebLR7bbgjgZ4Achta6pVMnT4pGkHxPGEcpY+oMxWVaz3ZXUGtbbLUEUcnZxSpREeIEgK1LOLy9Iej2EJyirjKrO2aVrduma+fyRk5MZWmsuLy+ZzU5QypBlRROg8KkrTRz36PX6bLdb6rpiMOxhqZC+Qni6SdvvqOuqSdDXDIb9ZnJhzXg8Zr1eI6Xk5GTCP/7m7xBSAZrNZsn5+RlJktBL+tSVwmjLer1jt86pMoOtAwbJjDoXpKua6fAcaXzyXYEqapIwItvlqFqiaijLmt02w2iYjKdNX49ksdhQlZbR6IRBf0S/P6QoCkajIVEUkhcpStcIISiKmrq2rFdrtrslYSjZpRu0Vjw8PLDbZURRTFEUpOkWi2YyGaGqkizdgdX0exGvX15hVMXpbEKebrGqJPY9ziZT/vJXf8n/6b/6P1NuaoQSLJ9WFFmOFIIsT7HSooUi7Pmk5QrpK84vpm5iJR5gjc9uW2CMw9lEscfsZIgfCPJih/QsxtSUVc6r1y/BCgb9EXlWIvAQwsf3JKvNA16gyYoNfiAIAo+izDk9O9kX35ZlBQLyfAdoiiJDSsF2u8HiPj8GoaU/CFlvnghDS6VSoljSH0TUqkTpGk/6SOEzny8wRhBFvT96D3dJ9E6dOnXq1OlnqlqZpnSxRVUIV7ooXKGiMCCkQGmD0Mb9XLRJdAXCbdYqjK0wpsZai8RDWM8FbYUrBbVehTUSLQVaSBSW2iqkrpHaI/B9pCcbs9Q1stfKGe3aaIxVaFuibQZSIxqEiypd2jywIarBixitqCqL7wnw/b2J2RraWIHveS75bi3CqIb5bvZpeiGcyV03Jq+1oLXAGA/hCYT1USZA2gBtnGnuDFIJ0kN4HkL7CFtiVOX48MKlsI2UGNMm1y3GCozxHQudholuW3SK3RemSq9BfyARRiKEOzZxxClvDWJhQTeIHE8KrDCNE2+PtpZhr5uHcJolkRqswWgPiw80aBY8WkS4W24Mykh05Xji7m99lHaIHRcUt0f4G42oDWHgluU6Lj048o7EE75LjDcGtvkDSf02uX6cUHcMRAMtUf8IJSM4lMRKD6Q0zUSCM/TbkiY3idIm2XHpqmaSxFiJNBYhWjZ+M2GjFNbUYN0HK4cholkl0BTWWgtWOuZ/a+gjEVZQK4sxhwkUo9uJApcyMkb9f+5m79SpU6dOnTr9/73a/ph+r09RVABEccRumyLihMGwz2a7QUiwwiA82GzWzXOXT9LvUVUlUeSTFznDUY/lcg7CIIRHUZR4MiCK/D1eo1YFfT8iThLm8zlZVjIajtBaM39awIlDLHq+MzuNrRskTE1VaaRnkbkgitxzaZZlaK0YjycEQUC/1ydNc8IwwvdD6rri/v6e6XSKMYayLHEl8xXbTUoc96l3a7TW7HYbttsd0+mU/vmg4WG7VaBPTwv6/T6PD0ukiEnihNnJlIeHO+bzBdPpjLLKWK+XVFVFHMdsNktGo6krKn3/jqurKz7//HM+fvyARZPEEWm6JY4Dkl7IDz9+T7+fMBj08f2Aqqz54Yfv6fUGDIdDRqMRg8GQ5XKFsR5RmLDbZpyenrFer4mjGKMN86clp6en7HYp5+fnTGdT5vMn4qhHUZScnp5yfX3Ly5dXaK2JwsSl/617/i7LDCkk/f4AAM+TgBuHqqpYLJbNZwvJdDrEGIsUgiSJKYpkn0jfbDb0Bwm9Xo+qLMmyjKqqub295bO3n1GVJel2zfn5Bbc3t/zt//Y3/NmvfsX52VcUu4pPHz6hakWeFczOTqhVzf3DPUk/5uOn93zzy1+S5ynaVAhpKYqC3S5DK0sYRoRhxNPTE0HorOYs2zIaDbi5ueXrr3/B3d09xhiiKEJKSRzHVFVNmjpETRQHXFyc8+nTJwaDHqPRmNXSYYHu7+/p9fq8ePGCDx8+cnNzw9nZGUo5Q72q3PF6nmS5WDCZjNBa4XmCzXZFVWUMBkM+ffrI27efk2W5W10rBOvVpuG+B3/0Hu5M9E6dOnXq1OlnqpbNLIwAeeBPH3/dc7uFQVjZFHTS/MwVXjpmn0vguiLG1hgEg0GhHdbFVijp7EZlNZ5WeJ6PbwM0Gmmav9GGWtVopalU5RLrxpn1bVq65XO7ZLdA1AohPZSnEcJzh9Psp5SywXkc8BxHkBCcjyr26fUWfXIoE6VhpkuH3DBtsrzlfbdpabBWNonqNoUusAZ0w4T3/BbBAp7XJqQ9V7BjDUoJrIPN0CI+HBteIvfnxOX+nXH8HMvjUCOt020RxgISi2mWfor97x3S1oekufu+dka6ASncMbthapE3dj9enmhfX2AFKG2Q1uJ5cr9CoK5dUlwISxj4aKn3xrebeHBnxtgD9gTACvs8WX90XR4ft5Pn5ghEO1atkS73qybcCoTDCgatHEbFlbjao3S+u+al8BA4l19KczR2+pDcbzbadlph3bUiRHMuxJ5X385xCGGwQjRc/naffsrNV52J3qlTp06dOv2JSWtBuktJegm7bUrUmLBCuLDGdrshSSLAEoahS9XqNUIIRg1OZT7PCUKf1WpDEEqSJCHPcsIgQmtNEg9wVp/A933SNKXX65GmOzabDb1kgJQ+da2IooSiqCiKnF4/Rqka3/cpy4IkifB9i7WSKAowVlGkBWHojPLdLgVbkCQD+v0RYFkun5hOp5ydnfHw8EgUxTw9LRkOhqRphSBkOIxYLbcMRwnT6RTfDyiKkrpqS+YVSdJjtVqS5zknp2eUOSzm84atHTAcjBDCQ9WGk9Mpu92WzWaDEGIfyDg/P0NKyWazIsu2zE5OybO8MV4VabomSaI9zvHx8YkgcIWb7959BCvo9/vUdY3nSTwRcHd3R1mWvHnzhjx3ZaWOgd5DKcVoNNob2p7nEUUJnhew3e6aSYMdn332GekuJc9LxuMJWeYQOrWqGQwGbLcrxpMReZ4DMJlM0FqwWKwQSILApyhKekkfrQ1x7BLoqin71FqT57nDYOIhRYAx8Pg0Z9jv0x8MCMOAKA7Js5T37390/UDW4+TkjG8uTlislyyWc8I45Gn+wJvP31LWBdZqNpsVgR8yf1piDAwHY+q6JAgckqfX6zXPuYbpdEqa5kwmE8bjMf/4j7/hm2++4fHxCWttUxzrPh+4DqSUFy+v9hMwYRDieR7LpSu6jaOI9+/fI6WPajqUwBDFAbvdjiBwJrxWlulkgtYwGPSRHrx/vyQIQpRSLJdL93pxxGKxYDBw5alPj09/9B7uTPROnTp16tTpZ6q9iS5F66FzzPY+qDErRYNqaVnotBxsi0U6A7mp9tRGO3MZ49jl0qLQeJ7GWItvDJ5X43uhMzSNt2d6t+ar1hplFNrUrqzUmoYNfsDKOOSLS8gL5R6spfRAWDQCUHsj1WFCxDPExp4nbo6458algVuj2fm00uFaPA+kfca6dtMCjaH9BzA47e853nVbKOlM8mNQizv2xoQ9Kg5tTWHa8TEWI9r3PTaTW3O8RZI4I3e/z9YliX6a7j7mmu9/ZNt9dOlukPtxb9/T+cUGYdWekd6+hjFu31oTXWuNlALfk2jtVju06KD2+jLG7hEz1lqk7yGERcrnJrqUcn/+2uuvXUkhBc34es0EgGgKilzS3HIov3Vo86PS1T3KqLmmcB8ujHEM+0Ppp352/RzOs/u32Bvocn8+244Biz1CwrBfCdCm65XSKNVgb8yBEd+pU6dOnTp1+vlL4KMVbNYpdaUYDhKMMfR6/j4VG4YBs5Mpq80jWe6eP5WqAUNdl4RRyHq9IgiccWiMxpM+YRBjPIsQnkNg5GVj5MZsNhvqumY4GpGlJWGQYLQLFFRVzd3dPa9ev6DfT9jtXCrX8yRBGFKViu1uTRgG5HnOcDji4uKKzSYlz0qGg5CycAxsKQWr1Qqg4YX3sEaQ5xWnpxeURcnjw4o3r7/g4ekDQgjq2pVEOqRJnyRxpathGBBFIfd3dwT+gNPTM25ubhgM+qzXG5IkRinDYr5is3V87dlshud55HnO6ekpSlf4gWQyHSOlZjCM8HzN0+KRfr9PfxCzXm15eHggSXoEvs/Hjx+5vb3h/PyS05NThJDU9YDtboUxisFgwO3t7b7M0xjDfD5nPB7vP9/0ej2WizWeDBBCEEcJL1+8xPM8srSgrjVVWfPp+pbPv/gMhAtYPD4+cno6Y73ecHZ2xmq5JstyjHHjiXUFtP2+jyd9FosFSdJDNqt1fc+nrmvm8zknJ6fY0McaSRInaGUYDkdUVcmnmxsElquXl1xfX3NyckJVupW123TNerNkNB6RFRnT2YQkiQgjn3fvfmA4HDAcTsjSlLIsCIOK7WbrVqBaS5LELuBkFYvFHCF8+v0+f/3Xf421lsfHR0AQRRFlWVJVNUEQMhwOKauMNHVp/vv7h2acBw1vX1AWFUopqqogjiM+ffrExcUZJtcsFnMGgwFhGDF/WnJ+fonAd/eIVcxmJ829lFIUbgLkzZvPGgO/Jo5jxuPpH72HOyZ6p06dOnXq9DPVnv98ZAYelyLCccq5NTePTFrTmpdyz6qTMgC8/d9o3fLK66bAst0qlKqoVUGtCsoqp6rbrWhYdA0iBgVCI/Ym8SGB/NwAbtjTTcFkmyxu/+3KJw/ma8vWdsZlu18H5nbdMrzbFLuQLmniOQ76sZG+x4s8S/IfCi5bY/dgjB9PGoi9Eeu+d+CpW8vhq6ExXtvtmN9tn50zrTVKH8zZg+Hb7maLPXEFpS2DvE1OGwNGH94fK5v9kC5lb9wEilZmX4J5bIS7Jb51M4YurS6a1LcxtuF+t4WgHFj1xqK1PTpnh2R4ew6eb4fJCHE08SCPeOrH1/H+Gj4sRXiWzLfWoV10wyU/3tr9OTbwD1gZ+5PX+2mZaJOu31937n44XGfNtacUSrX3zNFOdurUqVOnTp1+9lI1nJycY42HbbpdXNFkQlmWRFGIUhVFnuGefTxOZif0es4oTbMUrWvu7++o6xrfC7BGEkc98ryiLBXbTUoUxgz6Q3pJnyiMSJIeURQxHAyIowStYTo95eLiBdZYLi8vAYGUHsZYer0eLjygKcoUpSq0VqRpShzHaA2qNoRBQuBHFHnZMMprqqqg1+sRBhHpLkXVCt8L8b2IT5/umU3P+PGH98yfFkRhzGazwxoYDocA+9V6p6cziiLH932sUWRZul8ZmuclSll8PyIKe1ycXyKFRxAEGGNI05Sqqvh/t3emO24cbZo9EZF7cmctksqy3Z+7298Vzi3MtQ1mLqDRQH+LLalUqo0sMveMiPkRmUmWbMMYzI/B2HEAQlKpikwmSSDriSfOG0UBWvc0TQkYEJpeN4ShwBin15NSEIQKKQVt13A4vLBYzF2L/fDC58+3IFyIbYzl6emJ3W7Hy8sLx+ORsiwRQjCfz6evff58x3y+REpF3xuyLKfrNFGUMJvNub29w1q3gHD76RNpkjCfz5nP5xwOBW3Tc/f5nru7L7x9+47lcjHsYHTXpk9Pj9R1TVWVPD4+UlXVMEA0H4a69mRpjhCKKE4JoxipIp6edzRti1SKxWpFWRVU9ZHn5yfavkWGgu3lBVEc0uuOrmt5//6G21sXuoOhrkvq2nnMkzgjimLm8yVx7N7DURzT9R1JErPdblmtVsRxzNu3b9luN1xfXxMEiiRJprlUeZ5PxacoiqYgfTab0bYtxhjquj5rr1sQ7ntnsxnGuEa+Uoquc8NKy7IaXhvJDz/8K1EU07Ytee6a8kkSUdclCEs+ywHBly/3v/sZ9iG6x+PxeDx/UE6DFE+htLvmOIWP7mvnmpDTIMZRXzKpTlBg1SDMPik+tHVDQvXQ8u5tNwwVbel0Q9NV1G1F1ZTUbUnb1bR9Q9c3dLpxXnTboW3/Kmx2Az0VSqqhbT761PVwG4LQbggm+1MoOwbs4zDRrutcUGpeB89au9axgKE9fQpFx/Ok1OgDd+Ht2FiWcgxyOQvNTxoTM6hWpqBdjPdzGjppNOjenoXMZwNerZ1eka9vBueMH7U3DGqZqdkuxwWBYFoUkDIYWu/jkNQIJSICGROomCTMiMOEKEyIwpQoSAlVjFIBQqjhuWm69rQY8SoMt3YYMtq+OudjQP068P9qceIrTo3+cVFDDu7z1+/d8+GkTP93ulnrtCouuB7eP2eLK+NCwOk4z8P0k05n/Oy83oVwGijqfOunxZmuGxaRuoaua2nb5mzR5vVn0uPxeDwez58DawTXV+94++YbtptLBJKybAjDiP3+haKoMMbyt7/9nafHHX1vmc0WpGmK1oa6qonCePBotxRFTZbOSOIcayVtowlUxGZ9yXZ7iZQBUipubm6I40H3kqYAFEXB8/MOkFRVRZ7ldK0my3Lm8wV9b5yaUArK8kiWpaRJBkiaqqOqOpqm43AoWCyWlEXJer3i3bt3lGXF5dUV8/mcKIoRQtA0LT/++195fHymbTXffPMddd3Q1A3ffPN+0uodDgc+fPhAnudEUUDdFMzmGWEkubv7zNPTE1eXV3Rdx9u374jimCzL+PHHH+k6F5K+eXNNFDm/9X7/TJJGSAVNU6N1R92UvLw8s9s9I6T7ets2vHlzzWa74fLygqapiOOIKI4oiuMwfFKzWq1YLBZ8/PiRMAyp65rLy0uMMYRhyHa7JYoivvvuOw4HF/i2bcfV1RVN07DfvxCGbvfrYrHk8uqSz3ef6bqW5XJJXdcYA3GckuczqqrieDwO3nrFbrcjz2dorTkcDiwWi+nacj6fI4QYXmMBQhEnGRbJy8uBj58+03Waru+om5rb288sVyuCKGS2mPGv//6vfP7yGSssURyhrabXPUIIttsNZVmgpKI8Vjw+7twCQTojz2Y0dcvuecfxcBic7vD4+IjWmru7O+eQT1I+fPhAGEbUdc1sNqNp3DVyURQcDgc2mw0AdV3z+PjI8/Mz4OYZjYNh37y5pu97bm5uyPMcpdS0EBUEAflsxuXFBXd3zsH+z3/8xNOTu58gCBAC7u/vaZpmUFBq5vMFi8Xidz/DXufi8Xg8Hs8flFHngsB50WFQnYhXgaALHq0TuJw106dgfWhUM6gzMAIwzv08NIjtoFIX1qB1DwikkEirzwLOsYntIk5rhwGbQ7vc6Vu+UnucDZB0fmsXnFtrQbnWtJDWudCH/z8PURl+cgxFne7DvnruTnkztsPdMEwxaFHGEFcIObgFOVN6DC1uRgXJcB6NRfcagjHklcMQTjDWoKShFxoh+ql1MTbRnVlm8GxLYHp9BAzaEju1ooeQ3bqw3h0bzq0+POb4Ogth3XcIiVKhe21kgJIxgYpQKiRQ0eCMP+1I6LVFT9aRsWXeu1YQp+a2e78B9hSMn4fEr7Uo7jyNCwwnxib6uGOC6Tm4wae/occZlxbOzuXXnwM9tuKHnQvufS6mBQsYXfaSSd1jXy8+TdojRrf8qLaxGO3+3vd6eoxR36J1PzXhX+8Y+IVXyePxeDwezx8YISSHw5HFwgXjj48PdG1P1xkCFZEkGW1b44a+h9RVSRhGLBZz+r4jji3HY8l8vuLq8pq6rsjzBcWxQKBcY9xAUVSAoK4bFosVYRgxDp6UMnb6lSAgTVfc3Nzw9PTA8/OOOI7IspS6aonjlEAFpInztnddT9v1NHXHl/snsjTnw88f+OabgDzPCMOQtu2w1vL4+MByuWa/e2E2n9G1bpjk49Mj+SwjCLekaToNFXU+7Jr1ej21yaMoIk3d4Mm6LZllc5Ik4v7+iVk+Z7vZEoYhF/kFTVthjGuVuxa6U9l8++17Pn78efCEl0Mge0kYKJ4PR5R0Co+2bfn46QNBECKQPD3tWC7X3N7est1eIKWgaWqkjElTFwSPZZAoilgul2y3W+7v71ksFiyXa56engjDkCAIJsWMc4QXhGHIYrFAKUnbNiRJTF3XtG3L+/ff8uXLPU9Pz4PGxLX0m6Yhz2YUR7cbQOAGkQohyLKMIFAYY5z3PsvY7XYYIsqXA4tFxs3NDS+HHYFSKGnJsoTVZkOeZ0RxTJLk/I//9T/Z7/dIKVit16jA+dyFEDw8PPLXH3+kKmsCEbFaQaBCQNJ1bijnfL6Yfkfoug5jDGVRkmUZ+/2eOE6m8Lzve66vrykK54S/vr5mt3/i6emJrutYrdY8P+9IkpSqqmjbfnDGnzzqnz59YrNZYdG0bUuapiRJwmKxoGs1f/nLX5DS7Uaoqpo0XWGM87pvLzYgNEqFXF5e8XD/TBhEv/sZ9iG6x+PxeDx/UEa3omtVnzafudxSTo2P8yDShbJmCv3G4Pv08wYr9NR6lmIMGOVZsGzdlkMRuGD9TCHz9Z9jgP5aXTIep5yGZQopJ5WKMcaF8rjQU+Fy/tGL7QLrccioHALu02NZ64aAnlrOY9PZhbijQH4MPUEjhGu3W8PUvh7Pj5SKUdlyavFbpxRRw7EICViUDNHSIEXvfONDjjp6yc9D4Mlpf9a+/vrv43HaIRR24fvgvRcSIQwn7YxECkmgAtSg5gmCmChMCIIIJaNJiTLqSJQCYxmerzl7bHfuBWbSrgiEC9HHBQrcAs74PFxA7YJuhqa5a7i7581wX2Nw7s7j6x0TMIT0UrpxqsY4l791w2Inn789vYYnjz1TkO7uVzNuXXbHdgrlXw815dXXxib9GIQbLU4t9+60sDDufDBau4UAydkCiA/QPR6Px+P5szGfzymKgqZpBkVFgxCSqqx5+/YGgKrsuHn3LUEUcTi+EChJUTRIaVmvtny5/8J2e0kQxGy3C4pjRd+665P1asvt7R2SkHc37xBCURQlm82KJEnZ7SqiyLBYztjvdshhg2mSpDw9PRNHKUIEVFXlQts8Iwhj3lyv2O+PXF68IUsXlMXPfHPzHq01QShp2ope90MTOBx2cUqiOGI2i6mrlqo60nYlaRqxXM0JQ4HWMVnmVB5dd6AoClarJVLB0/MDcRKRtCFxFHJ/f0sYSd6/v+b9t9+w3++5u/vM9ZtLdN/x8PiF29uPrNfbofAh+fnnfwKWPHP+cqUC+n4c5ApZGlMUR5Ik4ebmhrIsmM0WZFlC37dTyL3ZbMjznCSJyPOcOI4JgoAffviBqnLakLZt2W63/Md//Afz+ZzZbEYQSNfmF5YwVEO7PWQ2y3jz5g0fP35EhRatO6Io5HBwrvzZbMF2cwnC0DQ1ZVny+HjPu3c3XFxcYK1gMV+wXm/45z9/HtruDeu1a8l3XcuxOGKJSJKUKI4p6oqmadlVJWka89PPP7NYzAijlP3+hc/3TzRtx3q1QkjB425H2/XY1pVCDocD89nc7eTsK95cvxvC/heCQLFer6iqgr7vWa0X1HWBlAKpJOv1ehrmKaVbSMIK7u/vmc/n3N5+5s2ba8IgmLRAYRhRVzWzfE5RVNR1hR3a9ofDC9fX19zdfeLi4oLd/pmiKLi5ueHTp1s+ffyI1rBcrFEqoG1bkjhFSuV2Y2QpQSB5fn7k+uodP/30E9iAJM5/9zPsQ3SPx+PxeP6g9H0/DWm0mEmJ4YLa0yWAnRQuv2wQuxRUDGX08wGKYrjBKQmWbgApEmPlUJ62U5taiClxPz32oAHRg4PdtbEtUkgmx8o4/HIM8gd1yejqdsc8qlg00A+DfE6hrxJuyKMeWtZSno5JypNjW0qnqBnPQ9/3U+N+9Gn3vT4LUk8B7ajKcaG1mY5VSHeHUkqMGdvUo4PcnP0MU6PfWjDCuK2bnDQywtX9XZMa6zYFYBGGs1DaDg1rc3a6T5qXUEXEQUI4qFsCFSNFiAoU1gg3CFZ1dH2Pxu0WOA3GdFqbruvcJgWhGOLsyb9/9uoO4bmY3iPWOP+6HBZnxlB6XND4eiFlfB1cKC+H5+QWM8wQ3lvcooi1znU5PqZzkI8qFzPofZzGxYXnASfv+tnOjfFzwC+PY1w8Ob3fcEqeURnT6+l92eue8/Dd9l83832Q7vF4PB7Pn4kxbA3DYNBY9GRZPszVUTw/7dA9CBFxPNSsV5cEoaRuSqR04W+gYuqqQxASBTmHl4r1akPfd0RhTNtqgiAijhLevb2h7Sq6rieOYmazGUpJ5vMZUrrSy9/+/l+sVmvyPMcYaBvDcrHly5d7Pn66c4MbjeB4LJnlIdYK3rx5R1035HlO2zYgDEJYkiSZmtd935LnKT/99DOXF9dcXK6ZL1wjuSxrMpFS1w2zWUiaJgTBJVEUobVmsVhQliV1XdF1LfPZjG+/e8fz85H5bEVVu7Z5WRZgt8RJBGhW6yVxHBDHIW3bDgsBM4RQBCokUM6hHsUR1gYIoKpqwDnPu84Nfl9v1rRNx7fffsff//4PgiBw4b50u14vLi6G63rDDz/8MDXTrbWEYcj799/wvHsmjAKyLOXp8ZGLiwvariNNYj7f3bIo5whhiaIIhKAcdg+0bUvfG+azOV++3JGkEVEUoQLJcrmkqTWPj8+EYcS7NJ00JsdjSdd1w2usaNqOv/39A0ma8Lx7RApJnmc0bcPFxRVN1ZDEOXXVute3qIiTlKKqWa2WOP94iVKu6R5FMUVROXVPW1NVLfv9gTTNSJNs8NT30GiqqgLc+6Esa8LAtc2llJRlSRj2rFcbqqpCCMlsNmO32/P23RVCWB4eHuj7juVqyWq1om17hKgpq5Kmabm+fsPhsCfLslfnvW1bsizj5aVAa81ut2OxWA2PrYiiCClDLtIViOH3PAl1XXF5cYPuf/8z7EN0j8fj8Xj+oLwaiqjdRZ9z1Mmp6QuuvGvpGR3QY2v51Hoew/NBVzKGx2ZsEI8EnHJy5/t2ubfTiYz3ff7nSUfiQkvnEQc7tMG/jhlP6hCLwExBpDF2CmWtFRgFSo6XORakC9GNndwkZyqRU8jrvi6nf7rQVJ/a4UPjeFSoCHFSgtihZe1K825A0fjwYhqMKQc/pUQahbE4b8sYNo+2HCxm0JqclDsnV7g7F2etd86b0y6c19oOCpjTzykVEIQRUZgSRwlRlCKFGxYrhcIOrz1WYJUAq8GA4RSOjwHyq9fFDAogY6bjfa1qOb12486CU3t92DWAdK+3NYAc/DbunIixxD7cjx1Ccynl9PdxF4B7P5wGz47Hej7A9LT74NRAd2G+OD3Q+FhftdJfDxbFKW96t7jitDGjq16/ep+9vo9XD+PxeDwej+dPQJpFtG3tWrUWlHI6OYHCaMF6fUGSpKRpRtc9YIxEitDN0cHtfBsd5Hm+BAKkjCiLGoQlUDFvrt+SZRlPT88kSQQIXl5eaLuGNE2p6pL9yzNRFFEWR3a7J95cv2U+W6K1pWt7bm/v6dqO7eaK9XpL1/VcbDOEUKRJxpvrmCBU/OMff+P9t+9o24rNZkWWzTBGkyQJbduilKbvGxAdh+MzSZKQZsHQVg9YLN5RFG6wZV1XxHFMlqfsb5/p+57tdkPTVJT1C2VRc331DX2nCcMIISxxEtF2NeXLAUvPmzeXVFVNFCZobYZgvKcoSsIwoapqjFEEMmExj9jvn4ahra6Est2uaZt+0oG4ZvyKOI6QMicIXKu5bVviOCbPcx4eHiiKgizL0Frz/fffs9vvqOuSy8sr0jSlrjN2+2e+//5f+Pz5M2EY0OuOOIkwpsfYnsvLC7QWfPzwic3mcvDCZ2RpyuNTwWazpixLIGS1WlEWJf/5n/+JMU6FYoyhadyCRpIkaKORSqBtj+5cOlw3NavFgsPhiFIhSkU0Tc3D4w4VR+xfjmy3G9J0xv5lj1IhL/sdcZzStr17vvePpFFG0zQslyuMNtRNA5jh/FRIKYmigIeHB6qqY5YvWK1WvLy8cDgc2G62FEVBVVXs9y9cXFxijCYM3HkZPfJu0K1ht9sRRQlKBWw3FyjlhsgmaTQcx4IoCoZhshFZZnnpjoMapme93rpBo0XJaj2nbmqqumS5XBJHMe/evkP3gp8/ffzdz7AP0T0ej8fj+YNy0pG4xrgLDO2rmzFmCLo1YKZtdm4gZeAC2DMXtR0DeDs4pBkC4sl3fgqDrbBgDALDYPYY3OjCNdTHUH64n69DxfMg1rjUknEau8vmx9axnbQl1g6NbgNGju1s6I0L8sdBoL9QfrgnOjw34/LbyWNth5De3b+dAmRXG59ayViUGkP4U3Da9y6AdsGuGRrPCiHcOR/d7Xb467TAoSyW0/N3mwGG4+Cs9c74c3ZaOJmC5uG4pVSEYUAUuhZ6FMSEQUwgI4QMEFadNaMV4F43icUKt0AhOPm8R+XP5Jq3Q2BvXqfDp7a/fKWncZ55pmMDMZ1nY4Y6/uRIZ1C2nHQr2pqTA/9M5+Ja6UxDPo12rnatnRd9/IV1XMg4DQc9ee7H7P/Xmugj5wG97l0T3ZwNWR19/782PHRS/Xg8Ho/H4/lTIYQhyxN2uxeaumW1WpNlOVGUsN8dWCxymqYljlI2mwuCQHB3d0vTFiyWMw6HA0oFvP/mO8qy4u7zF4qiJFAQxyFRFHN1dUldN4NmxCIkxHHg1CuBJIpccBpFIU3TcHl5RZZlzssehMNw0pAgjYjCiKZxYeZ8PqNpWrq+Q/c9UkVsNmuUEgSBYrVeEoUJnz59ou86pBAoKbl593bwvz+SpwlKQqAkaZqDgCSJsVbTdT0vhwNhGFCV9XDNy9BSVyyXS3rd8fR8QIqIQIX8y798R54nfPzUkOUXGKM5Ho4kqeHq8mpadLAWNpsLPn74BFbQ9+65Xl06D3fbtoBks9nShOPQeXfdmmWZU5nMc6IoZL1ekWYJRVEglWS3e2Z7sWG/e5ka0bPZDCmh7zvu7nbEcUySxtS1U6n0eobWLpR+OTyjtR588pKLy0t3/gO3YzJOUhCS1do5wiOVDs77kwpSCDldm3Z9S9e1GAubzZqua0miGICiKN2uVWNQQYiQanLoJ1Ky2TpXvbsWF4RBRBBEVGWDUpKyqEjTjDzJuf9yz3Z7QatbqrIiikO0tiRJShCEGNPTdf0wXDaibVuMMXzzzQ1t2zmHuzXkeT79ntB2PXmeYS1IKVgs5jw83DObZW5+UxC54aYqQAhBkiQsV3Nubz/Rti1lWfDmzTvSRFGXLX/96185HI5cXGzJsoTPd58wTy2r9ZxZPkMqp5vBCo6Fa6//HvJ3v8Pj8Xg8Hs//p4wtclyz+FVT2U5DOruucwoP079yk09DL6cmurvIMFagrRwiVokVEoPCnutckGgr0Ea4P4efGb/WG+iMoTOW3li0de7t0d998nifvO1Tw1c7z/QYWDpVh50Cat1bus5dALvvOXmspVBDgHu635MDfhimOZyT062l6zrapqXv+rMmshsiCad2MnZUtbjj6bqermtp24a2cxe1fd9Px/V64CangJohzRVjGdtisJjhPE27BaQc3Ofuks6do5P+RIqAQIVEYUwSZ27LZZwSBhFKBUghUYMr3bXCnS1mvAHDsNFTw3pUmoyKlElxYn4ZPMO4kCFe/amkmgL0UTFkjH0VQp8H8JNOZ/ge3buQvOu7V69V3zkfZ9O0dK0e3hd6OufWnoL98XU7P2dwavqPLfjz12h8vuOfbnDocDNnf9f9cBt/Eeun+zmdIq9z8Xg8Ho/nz0RZv9D1NXEc85e//IV3794ihOZYPGIoQDTMZjGX1xviLODD55/4/HCLDCX7w54vXz5zdXVJGid8+vCB/W6H7jus0ERpQNOVtLqm0xVGdByrA73psMJdP5ZVzX5fYExA27nddHGa8PzyzP7lmcenL7R9w2K14Me//sjN+/dYIWi6lv1xhwwNlpq6fUHrmu1mSV02PN7vMJ1B9x26a4mCgCgIEL0htBKlBRezFZFVxCbENoYsTgmVoK4K6rpAWMtytkF3CklMIEOM7lmulgRhTt1aul5w/faGeAiFu7bn8FKQxjl5ukB3oFSCIGI+W1MUNc/Pew4vBXXVopRbTEgSF4KXZUNTG7SWpIm7Tm7aZriG06xWK4yx7nq+b5GB4eX4SBBawtByLJ4oqj1932CtoWlasmxG32uWywXadLRdhQrAmJb9yyOWnihShFFAHEdYI1AyIYoyvny5J4pCjDCstmuqtqFsO4Iop6zg6bnAWIjCiK7tkTLg4uKKi+0VfWdpGk2gYroO6qojjWf0jaXvoO8Ey/mWomjotEBFEVZAb3uyWUIgBXmSUBclVVGRBBFplLFerDGdpi5L6rIgDgOKqiRMQh53TxyrEqTCIJgvVwgVcTjUdL0gSnKkUDw83BOEitksQ0hLEAi64Zx9++23Q7M8Zr8rOR47pAyJ45QsTdC6oWmORCFkSYAUhiyOB+unoOta0izicNwzny/QvaCpe5rG/U6wWGT89PG/6G1NOoupu4rn/Z6qrjkeSnSnUUKhpODyYvu7n2HfRPd4PB6P5w+KNWpoEYO1AmlHi7nGzWbUg8nEDZMU0rqGNLjBoIjJsT0G73CmXxkDyWGAp0BO+eCpNewQgmnIqB3q1la4QHFq7A5qE4EbWKoNmKHF7Rzmp/vU6nQsUg4aDyumY3K+azEoOxSBdKoZKZ0GRqrBfS3GEFygzXhuhkUEBrPIcFzGGHe80jWzpTgF/hIxhM0W0eshgdbuuRnt/s+6wab94GW30g3bNIbBJe6OTUjlBoAiEVa8CrQBpJDIrwbDmkErgx2Hc0qUlARKEQQBgVIooRA2xKAwSGfBF25wKhjMEOBr3Q87CKzTrxuD7nv6vqPvOrQeFiesU5bYsUKPnZzoUkqUEFgh0NZOexlUGKKkRA5N9HOdyvDszsw6xi1UGIvW42LO6BLXU8BtGBdA9KSTGYeJGnN6PV3j/TS8dbTGSCkmF+m022Bc1BhWMYwR0y4IbcBYN1dgbJy713Zon2OwwkwLLFOz3Zy78xWnYb0ej8fj8Xj+DDRNSRzlLOYLoih2zeVIoS2EUcB8kdC1cPfllofnR5CW99/eUFUFXdXw3XffsVotub+/R0rJjz/+G0VR8eHT30nTLcYYHh7uaNuW2WxGURzI85T9fs9isaDve6pKs91eU1UFxgguLtbc3t66RjBgbE8QKFQQ0LYd+XxBVRzodUffN6RJRNMWxJGiDxVd11OWDXne0jQ1UkjeXr9Bdz1BlGC1IZIRq4tLrDaENuLpZc/D/QMopyARQJblKBlTVS19Z9FNQ9u4QL/TAS/7ln11xBjJ1dU1dV2jpCIMFMXRff1ie00Q5IRh6OYQWYmSIQBhGLLZbNjv9+x2O6qqYjFfUxQVh8ML9vKCN29PRaMoignDiPl8QZ4ZZKDp+5a6LoiiCGP7oVzSYYwhTmKslcRxwsvLMyYOCAJFksTOGS8NQRgQRe53q7JsCMMV6/WW+/t7ulajpBqKNAak+11Ja0NvBVGYcXX5ligQ1H2LEIo8n5EmGX1vCMOINEgREpqmprXaNc+RKBmhtSGOM4riCa2Ne9/pnl53rDcrIhXSlBVZnNBWNSJOWS1X3H1xiz5dJxDSOBd91bJabTgeC9Ikoyydz72qGsywmCAUxHHK7umZpq5QgSTLEsDQdjVZNqPrOoSAIAhIkxl3Xx55fNzx7u0NaRrw8eM/CQJJ0xx5euzYbK5IkxgpBGmSorXhcNyjdUeWJsxmOV0ryLI5799/z/7lmV5XRElA1ZZUdUEQBrRdi+k1YRgCcpgjJlit5r/7GfYhusfj8Xg8f1jkSb48BOknFQeA2+JprUFIi7QMWhQ1uMpd29at9J9SXHPeN7YSK0dftwtAT+5phkGSgBxD69GrPoTgnFrvdkqtwRg3kFMYOzWVz0N0iXOHu+1/7p6wr0PJsXE8Bf3D0Q8//UrbYcyos3G+yen4DLjQVQ73bxDWPa6VYjLjuFB9eEbjfQ2BvDH9EPZajHUDi0Y39xi8n8JVgZqe51ch63kDfQjKp3M3tNVPqxhiWFQ4H2I6+NuFWwgw1mlRLCCmgbEujB4b573R6P7Uxu96dzPGhcXjAsj4asrXh/v69Rgc5JMPXZw0QafvHpzsjGH5GF7LITQ/V8iY03GY09eHSry710G1Y4wT34y6FjHsdJBCIsXJjy7E6EQXZ63xsZE+3IbjZPres2d6Vi4/LQaIaZFpREr16t8ej8fj8Xj++LiQOWKeB7Rtg9baaTBMSNNU9H2HtQFN4zze24s1WrsdklmeA7Db7WiqniiKWSwWWAvL1ZKu6yY3t5RyCtKllGRZhrXWBceLOUEQEoYh89mMtm2nHX0CRVlWpIlTxxzKgwux+84Frl2Dkk5rV9U1SobuetQYp30RkGcz3r59x5e7O9I4w/ZOwbderTkeCvpBr6e1RilJHEW0XcNiMWe3K6bBndpU9Lpyru+2QSpJolK6rqNu6ukYjseWojgSJyFhqAiUIkkSDocDSZJwfe0C967raJqGsixJ09Sdq3Q+eMsT2rah61rCMJyGo2qtSdPUDcMMAuq2HILfcbesu/5s25YoTMjzjCAI6bWmKI6owBU16rpGCMvhcCAKY+I4Zr8/Dq/XGPoboiii73vaviUIY3ddat11axxHRMGc/dM9y8Wa47HkeDxyeXFF27YIIQnDEG2c/1wIQZ7PwFbD7lynTrHWst8/8/JSsFothoC8QSmFUpq2a+k7TV23Z3qcCKUUxmoOhwNChQgh3LlM3HDPLMspyyNpkrLbP1PXDVmWMpvNwBru7u7I85Srq6vBh7/EaCir0g0YJaDXPVdXlxTFkYeHA2EUUJYHgiDgeCxYrTSz5QxhFLvDASMMxrbsX575/rvv+Pz5gSxds1qtEEJSFgVN59zreZ7TdR2LxYbHh0dkCEmyYLlcUhY1YRCSptnvfoZ9iO7xeDwezx+U14MdT83t0U1trEUYF5D/mlji/HtPDvPXIbocB2L+6s87vYgYQs3zIF4Moe/o0x4DS2tdgCqEi72tOA9XX7fbrRTDMMhRv2E46WD+z86RGTzoxnAWao/u6sElb88HZpqTj53x63IKc5kC5pMexwxfG9UyDE7v84GX43n/tdu53mS8TU30MUD/6qm752Cwr7Q4rvE+nrvpnJ6pS4wx9Lp3t66n71/fJuf6uAgyueZP5/R8EeP8mE9h9a+/Hr/mEZ+a7pN+RZ9a5q9CdD0l3e4xxiG65++dXx7T6VjdQskvzuHZbXpuv/muGr9nfKxBt2PHX7iGxxbqN+/B4/F4PB7PH4///t/+9v/6EDwez/8Fwv76byoej8fj8Xg8Ho/H4/F4PB6Px+Px/Onx+0g9Ho/H4/F4PB6Px+PxeDwej8fj+Q18iO7xeDwej8fj8Xg8Ho/H4/F4PB7Pb+BDdI/H4/F4PB6Px+PxeDwej8fj8Xh+Ax+iezwej8fj8Xg8Ho/H4/F4PB6Px/Mb+BDd4/F4PB6Px+PxeDwej8fj8Xg8nt/Ah+gej8fj8Xg8Ho/H4/F4PB6Px+Px/AY+RPd4PB6Px+PxeDwej8fj8Xg8Ho/nN/Ahusfj8Xg8Ho/H4/F4PB6Px+PxeDy/gQ/RPR6Px+PxeDwej8fj8Xg8Ho/H4/kN/jeruqJw5HRd5QAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAABdEAAAGHCAYAAABfzRvzAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsvXmgZUV1Nb521bnvdTeT2KCiGMBmUFHUD000yiDOIuKHQlBJQwTUnxIHNIJDggQEjQMYFRU1KgIh4oga0E9posaoETUkJioOEKNGZB56ePdU7d8fe++qOuee+4YGAbtrwe17X506darqnHffrlWrVhEzMyoqKioqKioqKioqKioqKioqKioqKioqJuDu6gpUVFRUVFRUVFRUVFRUVFRUVFRUVFRU3F1RSfSKioqKioqKioqKioqKioqKioqKioqKKagkekVFRUVFRUVFRUVFRUVFRUVFRUVFRcUUVBK9oqKioqKioqKioqKioqKioqKioqKiYgoqiV5RUVFRUVFRUVFRUVFRUVFRUVFRUVExBZVEr6ioqKioqKioqKioqKioqKioqKioqJiCSqJXVFRUVFRUVFRUVFRUVFRUVFRUVFRUTEEl0SsqKioqKioqKioqKioqKioqKioqKiqmoJLoFRUVFRUVFRUVFRUVFRUVFRUVFRUVFVNQSfSKioqKioqKuyWOOuoobLnllre7nKuuugpEhLe97W0L5n3jG98IIrrd1/x9wV3R3pe85CV40pOedKdec6m45JJLsOWWW+K3v/3tXV2VioqKioqKioqKioq7ASqJXlFRUVFRUVFxF+Oss87CRz7ykbu6Gr9z/PznP8cHP/hBvO51r5s49qEPfQgPetCDsGzZMuy2225417vetehyr7zyShx++OHYcccdsWLFCjzwgQ/EX//1X2Pt2rWdfF/60pdw9NFH4yEPeQi899h5550Hy3vqU5+KXXfdFaeffvqS2ldRUVFRUVFR8fsCE1Nce+21d1kdLrvsMhARLrvssrusDncEdt55Zxx11FHp56F2HXXUUVNjz9uDKlC581BJ9IqKioqKiooKxRve8AasW7fuTr/uXUWi39ntfec734lddtkFj3/84zvp73//+3HMMcdgzz33xLve9S485jGPwcte9jK85S1vWbDMX/ziF/jDP/xDfPOb38Rxxx2HM888E495zGNw0kkn4bnPfW4n7/nnn4/zzz8f22yzDe573/vOW+6LXvQivP/978ctt9yy9IZWVFRUVFRUVFRU/I5RBSp3LiqJXlFRUVFRUXG3wm233XaXXbtpGixbtmzePDFGrF+//k6q0cajbVvMzc0NHrM+Xkx7l4J+YF1iPB7jvPPOw2GHHdZJX7duHV7/+tfjwAMPxCc+8Qkce+yxOOecc/D85z8fp5xyCm644YZ5r/mxj30MN954I77whS/gxBNPxAtf+EJ8+MMfxurVq3HRRRd1zj/ttNNw880345//+Z/xsIc9bN5yn/3sZ2PDhg248MILF9HyioqKioqKioqKiun4wAc+gB/96Ed3aJlVoHLnopLoFRUVFRUVFYvCmjVrQET49Kc/PXHs/PPPBxHhX/7lX1LapZdein322QdbbLEF7nGPe+Dggw/Gf/3Xf3XOs2Wk//mf/4nnPe952HbbbfG4xz1uah2+//3vY/vtt8f++++PW2+9FQDwne98B095ylOw3XbbYfny5dhll13wghe8YPD8s88+G6tWrcLs7Cwe9ahH4V//9V8H61OCiHDcccfhvPPOw5577onZ2VlccsklAIBf/vKXeMELXoB73/vemJ2dxZ577om/+7u/m6cXJ7HzzjvjBz/4Af7pn/4JRAQiwv7775+O33jjjXjFK16B+9///pidncWuu+6Kt7zlLYgxpjyl7/uZZ56Z2vif//mf8/bxNE/0c889F3vvvTeWL1+Oe97znjj88MPxi1/8opNn//33x0Me8hBcfvnl2HfffbFixYpBFYzh61//Oq699lo88YlP7KSvWbMG1113HV7ykpd00l/60pfitttuwxe+8IV5++/mm28GANz73vfupO+www5wzmFmZial3fe+98VoNJq3PMO97nUv7LXXXvjsZz+7qPwVFRUVFRUVFRVdMPOduurxrhTjLITRaITZ2dk7rLwqULnzUUn0ioqKioqKikVh//33x/3vf3+cd955E8fOO+88rFq1Co95zGMAAF/+8pfxlKc8Bddccw3e+MY34vjjj8c3vvENPPaxj8VVV101cf6hhx6KtWvX4rTTTsOxxx47eP1//dd/xQEHHIBHPOIRuPjii7HlllvimmuuwZOf/GRcddVVOPHEE/Gud70Lz3/+8/HNb35z4vzzzz8fb33rW/GiF70Ip556Kq666ioccsghGI/HC7b90ksvxStf+Ur8yZ/8Cd75zndi5513xm9+8xs8+tGPxpe//GUcd9xxeOc734ldd90VRx99NM4888wFyzSceeaZ2HHHHfHABz4QH/vYx/Cxj30Mr3/96wGIsnu//fbDueeei9WrV+Nv//Zv8djHPhavfe1rcfzxx0+U9eEPfxjvete78MIXvhBvf/vbcc973jMdW0wfA8Cb3vQmrF69Grvtthve8Y534BWveAW+8pWvYN9998WNN97YyXvdddfhaU97Gh7+8IfjzDPPnFDBlPjGN74BIsIjHvGITvr3vvc9AMAjH/nITvree+8N51w6Pg024XD00Ufj+9//Pn7xi1/gH/7hH/De974XL3vZy7DFFlvMe/582HvvvfGNb3xjo8+vqKioqKioqLi749prr8Vhhx2GrbfeGitXrsTLX/7yiVWXbdvilFNOSUKNnXfeGa973euwYcOGTr6dd94Zz3jGM/DFL34Rj3zkI7F8+XK8//3vBwD8z//8D571rGdhiy22wL3udS+88pWvnDjf8K1vfQtPfepTsc0222DFihXYb7/98M///M+dPEsV4/Sxfv16vPGNb8Tuu++OZcuWYYcddsAhhxyCn/70pynPbbfdhle96lVJzLLHHnvgbW97G5h50dcxDHmixxhx5plnYs8998SyZctw73vfGy960YsWJLqBKlC5S8AVFRUVFRUVFYvEa1/7Wp6dneUbb7wxpV1zzTXcNA2fdNJJKe3hD3843+te9+Lrrrsupf3bv/0bO+d49erVKe2kk05iAPzc5z534lpHHnkkb7HFFszM/PWvf5233nprPvDAA3n9+vUpz6c//WkGwP/6r/86tc4///nPGQCvXLmSr7/++pT+2c9+lgHw5z73uYn6lADAzjn+wQ9+0Ek/+uijeYcdduBrr722k3744YfzNttsw2vXrp1apz723HNP3m+//SbSTznlFN5iiy34xz/+cSf9xBNPZO89//d//3enjVtvvTVfc801nbzz9XG/vVdddRV77/lNb3pTJ9+///u/c9M0nfT99tuPAfD73ve+RbXxiCOO4JUrV06kv/SlL2Xv/eA522+/PR9++OELln3KKafw8uXLGUB6vf71r5/3nAMPPJB32mmnefOcdtppDIB/85vfLFiHioqKioqKiorfJ1gc+NCHPpQPOuggfve7381HHHEEA+A//dM/7eQ98sgjGQA/5znP4fe85z28evVqBsDPetazOvl22mkn3nXXXXnbbbflE088kd/3vvfxmjVreO3atbz77rvzsmXL+DWveQ2feeaZvPfee/Nee+3FAHjNmjWpjK985Ss8MzPDj3nMY/jtb387n3HGGbzXXnvxzMwMf+tb35qo/4Mf/GA++OCD+ayzzuL3vOc9i2p727b8hCc8gQHw4Ycfzu9+97v59NNP5wMOOIA/85nPMDNzjJEPOOAAJiI+5phj+N3vfjcfdNBBDIBf8YpXTLT7yCOPTD+vWbNmol1HHnnkROx5zDHHcNM0fOyxx/L73vc+PuGEE3iLLbbgRz3qUTw3NzdvG0499VQmIr7pppsm0ofi1w0bNrBzjo8//vh5y7344osZAD/zmc/k733ve/zf//3ffMEFF/DWW2890e4Si4mtjznmGN5uu+3mzXN3RnOnsfUVFRUVFRUVv/dYvXo1Tj/9dHziE5/A0UcfDQD4h3/4B7RtiyOOOAIA8Otf/xrf//738ZrXvKajhN5rr73wpCc9Cf/4j/84Ue6LX/ziqddcs2YNDjroIDz5yU/GBRdc0FE/3OMe9wAAfP7zn8fDHvaweZUQf/Inf4Jtt902/bzPPvsAAH72s58t2O799tsPD37wg9PPzIxPfvKTOOyww8DMuPbaa9OxpzzlKbjgggvw3e9+F4997GMXLHs+XHjhhdhnn32w7bbbdq7xxCc+EW9+85vx1a9+Fc9//vNT+rOf/Wxsv/32g2XN18eGT33qU4gx4rDDDutc7z73uQ922203rFmzpmPZMjs7iz/7sz9bVFuuu+66Tv8b1q1b17mnJZYtW7aoJcA777wz9t13Xzz72c/GypUr8YUvfAGnnXYa7nOf++C4445bVP2GYPW99tprca973Wujy6moqKioqKiouLtil112Sergl770pdh6661x1lln4dWvfjX22msv/Nu//Rs++tGP4phjjsEHPvABAMBLXvIS3Ote98Lb3vY2rFmzprMa8Sc/+QkuueQSPOUpT0lp73znO/HjH/8YH//4x3HooYcCAI499tgJCxBmxotf/GI8/vGPx8UXX5xsB1/0ohdhzz33xBve8AZ86Utf6pzzsIc9DOeff/6S2nzOOefgK1/5Ct7xjnfgla98ZUo/8cQTk8r8oosuwqWXXopTTz01rRJ96UtfikMPPRTvfOc7cdxxx2HVqlVLum6Jr3/96/jgBz+I8847D8973vNS+uMf/3g89alPxYUXXthJ7+OHP/wh7nnPe2LrrbfupP/617+G934idp2ZmcHKlSvxq1/9at56PfWpT8Upp5yC0047DRdddFFKf/3rX49TTz11KU2cwAMe8ABce+21uOaaa34vY+tq51JRUVFRUVGxaDzwgQ/Eox71qI6ly3nnnYdHP/rR2HXXXQEAV199NQBgjz32mDj/QQ96EK699toJv8Jddtll8Hrr16/HgQceiEc84hH4+Mc/PkG27rfffnj2s5+Nk08+Gdtttx0OPvhgfPjDHx5cGvoHf/AHnZ+NIF3Mcsl+/X7729/ixhtvxNlnn43tt9++8zJS+Zprrlmw3IVw5ZVX4pJLLpm4hi3b7F9jWj8udKy8HjNjt912m7jmf/3Xf01c7373u99UAnwIPLD0dfny5VM3QF2/fj2WL18+b5kXXHABXvjCF+KDH/wgjj32WBxyyCH40Ic+hCOPPBInnHACrrvuukXXb1p9h3zjKyoqKioqKio2Bbz0pS/t/Pznf/7nAJCEL/betxJ81ateBQAT9iC77LJLh0C3MnbYYQc85znPSWkrVqzAC1/4wk6+73//+7jyyivxvOc9D9dddx2uvfbaNHZ4whOegK9+9audfYGAxQlF+vjkJz+J7bbbLrW1hMV9//iP/wjvPV72spd1jr/qVa8CM+Piiy9e8nVLXHjhhdhmm23wpCc9KbXz2muvxd57740tt9wSa9asmff8O0OgcvbZZ+OTn/wkXvCCF+C0007Du9/97sU1bgpKgcrvI6oSvaKioqKiomJJWL16NV7+8pfjf/7nf7BhwwZ885vfvN0B1TSidHZ2Fk9/+tPx2c9+Fpdccgme8YxndI4TET7xiU/gm9/8Jj73uc/hi1/8Il7wghfg7W9/O775zW9iyy23THm994PXGCJ2F6qfBe9HHHEEjjzyyMFz9tprrwXLXQgxRjzpSU/Ca17zmsHju++++7z1XOyx8npEhIsvvniwv8r+XGyZhpUrVw5OWOywww4IIUwoUubm5nDdddfhvve977zlnnXWWXjEIx6BHXfcsZP+zGc+Ex/5yEfwve99b8IrcrGw+m633XYbdX5FRUVFRUVFxd0du+22W+fnVatWwTmX9jG6+uqr4ZxLghnDfe5zH9zjHvdIAhrDkHDj6quvxq677johTOiLbq688koAmBpfA8BNN93UIY8XIxTp46c//Sn22GMPNM10WvTqq6/Gfe97X2y11Vad9Ac96EHp+O3BlVdeiZtuummqInsxgpzfpUDlxz/+cYqvDznkEMQYccIJJ+C5z30uVq5cuWDd5qvv76tApZLoFRUVFRUVFUvC4YcfjuOPPx5///d/j3Xr1mE0GuFP/uRP0vGddtoJAPCjH/1o4twf/vCH2G677Ra92SMR4bzzzsPBBx+MQw89FBdffHHaSLLEox/9aDz60Y/Gm970Jpx//vl4/vOfjwsuuADHHHPMxjVyAWy//fbYaqutEELYaIK2xLRActWqVbj11lvvkGssBqtWrQIzY5dddpkg6G8vHvjAB+K8887DTTfdhG222SalP/zhDwcAfOc738HTn/70lP6d73wHMcZ0fBp+85vfDKpwbMPYtm03us4///nPsd122021yKmoqKioqKio2NQwLS5dLPG5FJFFHyZUeetb3zo1Brw9oo67E2KMuNe97tVZ4VtiofizClTufFQ7l4qKioqKioolYbvttsPTnvY0nHvuuTjvvPPw1Kc+tRMI7bDDDnj4wx+Oj370o7jxxhtT+n/8x3/gS1/6UocoXQxmZmbwqU99Co961KNw0EEH4dvf/nY6dsMNN0woMCzgHrJ0uaPgvcezn/1sfPKTn8R//Md/TBz/7W9/u6Tytthii05fGQ477DD8y7/8C774xS9OHLvxxhtvF0E8hEMOOQTee5x88skT/crMt8sa5TGPeQyYGZdffnkn/YADDsA973lPvPe97+2kv/e978WKFStw4IEHprRrr70WP/zhD7F27dqUtvvuu+N73/sefvzjH3fO//u//3s4527XioDLL78cj3nMYzb6/IqKioqKioqKuztM/W34yU9+ghgjdt55ZwAikIkxTuT7zW9+gxtvvDEJaObDTjvthJ/+9KcT8WVfdGMe41tvvTWe+MQnDr7m2wNpsVi1ahV+9KMfJdHFtDr/6le/wi233NJJ/+EPf5iO3946XHfddXjsYx872M6+X3wfD3zgA3HDDTfgpptu6qSXApUSSxGohBAm0qtApZLoFRUVFRUVFRuB1atX44orrsCPf/zjtKFoibe+9a247rrr8JjHPAZve9vbcMopp+CAAw7ANttsgze+8Y1Lvt7y5cvx+c9/HnvssQee9rSnJeL6ox/9KPbYYw+ccMIJOPvss/H2t78dhxxyCLbeeuslk/VLxZvf/GbssMMO+KM/+iO84hWvwNlnn403v/nNOOywwwb94OfD3nvvjSuuuAKnnnoqLrjgAlx66aUAgL/4i7/A//k//wfPeMYzcOyxx+J973sf3v72t+Ooo47CjjvuOEi83x6sWrUKp556Ks4//3w87nGPw1vf+la8733vwwknnIA99tgDH/7whze67Mc97nFYuXIlvvzlL3fSly9fjlNOOQWf//znceihh+KDH/wgjjzySJx77rl4/etf39mc9t3vfjce9KAHdSZS/uIv/gIhBOyzzz445ZRTcNZZZ+HpT386PvOZz+AFL3hBR21jfXzqqafiJz/5CW666ab08+c+97lOva655hpcccUVOPjggze6zRUVFRUVFRUVd3e85z3v6fz8rne9CwDwtKc9DQBSTH3mmWd28r3jHe8AgI7gYRqe/vSn41e/+hU+8YlPpLS1a9fi7LPP7uTbe++9sWrVKrztbW/DrbfeOlHOUoUq0/DsZz8b11577aAlpRH9T3/60xFCmMhzxhlngIhS/2wsDjvsMIQQcMopp0wca9t2wTi/ClTufFQ7l4qKioqKiool46CDDsK2226LGCOe+cxnThx/4hOfiEsuuQQnnXQS/uqv/gqj0Qj77bcf3vKWt2yUbyEgipQvfvGL2HffffGkJz0JX/va17Dffvvh29/+Ni644AL85je/wTbbbIM//MM/xHnnnbfR11ks7n3ve+Pb3/42/vqv/xqf+tSncNZZZ2HlypXYc8898Za3vGVJZf3VX/0Vrr76avzN3/wNbrnlFuy333444IADsGLFCvzTP/0TTjvtNFx44YU455xzsPXWW2P33XfHySef3LFFuaNw4oknYvfdd8cZZ5yBk08+GQBw//vfH09+8pMH7/ViMTMzg+c///m48MILcdppp3WOveQlL8FoNMLb3/52XHTRRbj//e+PM844Ay9/+csXLHfffffFN77xDbzxjW/EWWedheuuuw677LIL3vSmN014yX/3u9/FX/7lX3bS7OcjjzwSBx10UEr/1Kc+hdnZWRx22GEb2+SKioqKioqKirs9fv7zn+OZz3wmnvrUp+Jf/uVfcO655+J5z3teUkI/7GEPw5FHHomzzz4bN954Y4q/P/rRj+JZz3oWHv/4xy94jWOPPRbvfve7sXr1alx++eXYYYcd8LGPfQwrVqzo5HPO4YMf/CCe9rSnYc8998Sf/dmf4X73ux9++ctfYs2aNdh6660nhA8bg9WrV+Occ87B8ccfj29/+9vYZ599cNttt+HLX/4yXvKSl+Dggw/GQQcdhMc//vF4/etfj6uuugoPe9jD8KUvfQmf/exn8YpXvCKp5jcW++23H170ohfh9NNPx/e//308+clPxmg0wpVXXokLL7wQ73znOzsbsfZRClQOOOCAlG4ClZe+9KU49NBD8ZSnPAVf+9rXcO655+JNb3rThEDl5JNPxpo1a5Jl5l/8xV/g4osvxj777IPjjjsOK1euxOc//3lcfPHFOOaYYyYEKhdddBEAdAQqgDw3ZWxtApX+Rra/V+CKioqKioqKiiViPB7z9ttvzy94wQvu6qpU/B7hpz/9KY9GI/7yl798V1dlQTz84Q/nV7ziFXd1NSoqKioqKioqfic46aSTGAD/53/+Jz/nOc/hrbbairfddls+7rjjeN26dZ284/GYTz75ZN5ll114NBrx/e9/f37ta1/L69ev7+Tbaaed+MADDxy83tVXX83PfOYzecWKFbzddtvxy1/+cr7kkksYAK9Zs6aT93vf+x4fcsghvHLlSp6dneWddtqJDzvsMP7KV74yUf/f/va3G9X+tWvX8utf//rUpvvc5z78nOc8h3/605+mPLfccgu/8pWv5Pve9748Go14t91247e+9a0cY5xo95FHHpl+XrNmzUS7jjzySN5pp50m6nH22Wfz3nvvzcuXL+etttqKH/rQh/JrXvMa/tWvfrVgG172spfxrrvuOnjs7LPP5j322INnZmZ41apVfMYZZ0zU2/qw3//f+ta3+GlPexrf5z734dFoxLvvvju/6U1v4vF43Mn34Q9/mAEMvsr+YGZ+73vfyytWrOCbb755wXbdXUHMA1u5VlRUVFRUVFTMg0984hM49NBDcdlll2G//fa7q6tT8XuE/+//+//wk5/8BP/v//2/u7oqU3HJJZfgOc95Dn72s591NmSqqKioqKioqKiouLvgZz/7GR74wAfi4osvxhOe8IS7ujrz4hGPeAT2339/nHHGGXd1VTYalUSvqKioqKioWDS+9a1v4YorrsApp5yC7bbbDt/97nfv6irdrfHb3/52cGMew8zMTGdJZUVFRUVFRUVFRUVFxWJRBSp3HiqJXlFRUVFRUbFoHHXUUTj33HPx8Ic/HB/5yEfwkIc85K6u0t0aO++8M66++uqpx/fbbz9cdtlld16FKioqKioqKioqKjZRzM3N4frrr583zzbbbIPly5ffSTWq2JRQSfSKioqKioqKit8R/vmf/xnr1q2benzbbbfF3nvvfSfWqKKioqKioqKiomLTxGWXXbbgRqcf/vCHcdRRR905FarYpFBJ9IqKioqKioqKioqKioqKioqKiorfa9xwww24/PLL582z5557YocddriTalSxKaGS6BUVFRUVFRUVFRUVFRUVFRUVFRUVFRVT4O7qClRUVFRUVFRUVFRUVFRUVFRUVFRUVFTcXdHc1RWoqKioqKio+N3gsUecChBARHDk9LMDAQARHBEovRoArvg5vwDAOZc+E0j+HcgnRROcc+nztPehtGnn9c9Z6NjQtYbKHI/HWLduHbz3WLFiBbz3YDB6RQ2WKX0BYGBNX78uKfO0goeutdhyASxlYeGSFiFSbp6dx8wgos7PQ+UPpW90Xh6+Xplv6L3MG2OcN0//+MQLDOaYfo4xTuS1MpgDOM7pZ0sDmCNizGnf+odThnq9oqKioqKioqKiouJuhkqiV1RUVFRUbKJwzuu7myCRJ18eRoyXRPYgqc7AfCR6/3pWVv99iNheiERfzPtCecp8MzMzaNsWMUa0bYumabQO04nmThk8zItPI7vhbh+JPl+dFoslEe7I7TOiuGz/NGJ76DpLSZ9I4+HrLUSiL3S9/sTAEDHeJdG7/dDP0zkXXsnyMh8ByGkVFRUVFRUVmw/eddYhKY5iBkKICCEghhwbxBgRIzCOjBi7cYzFX957eC9xPnMEiOG9pFssDQCzs7OIUa4RQkDbtgiBEUJIE/3eOymvyeOAHPtFxNiCWYQ4znk458FM8G6EmZkZNE0D7xs4jODdLBxJnjS2QAOCA0DaHhEuBG4RYwCjlWtgLIITFRxInRkcI9owhhx0ADsQOcQYMB7PYW68ASGMQU7GEctnt4FzDbz3aLwHOQcCgQjwTYNR02D58uVYvnwLkFsBggPHMQgRHFtsWLcWt916K66/7gasX7sec+vnENoIbgijFTNYscUyrFjm0TSA8wxwQIzy4hgRYwAhwkHiPqIGM7PLMLtsCyxfsSW8n0HjZ+GaERwRIq/Fhg03Yv2GtZibm0OMMfU1R4e2ZYQWCJERYouW12NuPIe2HcN7h+XLl8O7GRB5EEYARgB7MALaeCvWb9iAdhwRAmM816ZnKoQWIYzRhrE8CxFgDogcwIgwuVAMOiqKHiDp+0BR42JOz2VXTFLE9JHlfJb3UnjiiOCbBjOjGYxmRqAAcATIz2BmdgswNxiP9fcjjhHadVgx22Cn+90X99r2XlgxsyXIA4yIMQesXbcO195wPX71v9fgxptuwVwb0bYREYSgYz0gInKUZ0YHW+QA7wneO3zmgu/O+ztcSfSKioqKiopNFCWZPY1El0CbRKFOrnNeP28m0btllcH6NGLdjvXPmY9EHyKGpynZ57tW+d6vIyBE+ng8TgGfc6K0n4aJspegGL8jSPQh8vV3RaIDwl/31efzkegLpW9U3nlI9P77NKX6tOuVJHhJjNsxI9Yjxw6JPvTqlJcGjHFqnoqKioqKiorNB+Q8vPNwjmRlWmwROIpowTmAWeI/AgicSL4yfnDOdWL1EDmFl7byjagbGxpxba8y7nHOgVyOw72XWFzy9gUE+o5MtKfrUgRzCyaLcSIAp4SsA+AAIiVVGcwtgADmMRgtmFspk+R45KCxV7SrJKLa2tO28rMdBwCiCKKgZUlfmECmbTcghDkwAkCMZhSVRG8BDuAQsG79WqzfsA5tO0YMATFyUb6Q5G0bASJ4yD1klokK1v51YHDS5LSYm1sPBsOPPEbcglxEDBt0IuVWzI1vw7hdhza0ABgED4cRmB1YFS3ee5B3IPZwfgZzc1Iv7730N9sYRiYg2jDGOMikCSArkWPUiQkGYgRCAGIgrX8EKOZOI4Dg4Bud/NB7yJAJjBBbhCBku3Oy4jmt6tQ6p3uATKTbMy2Pu9zTyAFAA+dJDniCc4DzHs3Iox0HhDYCTPANoWkcnNffA+cAYkQmnViidLeYgcik1ZFnnRkgfaaEVJfmxgjEuLDjeSXRKyoqKioqNlGIEp3gnBHGJcGt1i6JTM6f7yglupVREq/TSHB7n0auW6Bfps2nWp+WPnT92dlZAEhKHd94DUK76BPIRDSVah8ktTUgXSyWVPYSUN6Tja2EBKGZdO4O1CbT+0qV+fL20yWBJvLO996/ng06FzpH1D/dMuw8B5cmFBYk0SODnS+umwd8Vl4l0SsqKioqKjYvhBiFIiUvJJ9zgBJ/zEI1RiUcfeNBSsqGEDrllHGsdw6+EcZWYg1WRbpHCIwY7WVEYRYIWOhlYwNAiEegjG+kbnaMmeF9X8AiVnaBGYAHnAexEucwcYp8ZijBSa22tgXQCpEeS6FBlJzEcA6Fgjqkz5JPy1fWNMSxKPMLMhhK7I/bOTmfx/AeCBzQ+BEQAzgGhPEY69bfhrVr10reGDRWdHCNKJZBAQxSghbgGDTOC6l/ohLJ0kWi+kYLrFt/M+LsLOBawJGuhl0H5vUIPKeTCQCxA3sCSMhiUZk7RIg6nSnC+Ryvkt1LEuV35IgYWrRtQDsOADuEALQtox3nVQghWr/q/SQPRwyQk4kG1nuuzwxHRkREm1YL2HM0KRJJ40rP6XmyPDbmk+dQVjuMx4BvZoTQ5hYhjuF8g6ZppP+JwIHgvUyUOCdlOO/BxPAsvzP27Mv1HYh08oiLZ74zPsnP0GJi80qiV1RUVFRUbKIoSfJpSvT8mu6HPpk2LX3+NPsMDFvMLPQ+pCYfyruUNKvLzMxMWkIpxKdPpGoiUXuKe2hPLJrUnkKiTz0/TgZy0/L+zgjZYnDVn8jok9BlHfsEep8g73+ej1i3AdJiCfSyPn2SvK8479fd7nXZjvyKiDzc/0TUJciJwcGrKksGrgB3CPRKoldUVFRUVGxeaFsGkaiBiQCwg6MG5FlsJ6IQxGBbAccTcYopjzvWGSHCwlSLX7JlCyciPibVds7rnIN3XlXIolbO11TluCqLOZIQyHqtMkaSc1oAHpEdHOXJgTx+IL2GThYku5hWSE4uLD+Qydk2BJgqW+oMxGiiIA+HmJTHc3MbMBpFIfojJ+W1KbRlQkLI25k4h9mZGXgCOESs37Aea9fegvXr16MdM2LUiQrvARdBqraPkaD8MUI7BkN9alRoJJMlUCGT9k8IwIaANqzHhvFtIAdV6Y8BjBGC9AGRA8iDMQa5Bo3zADzAQGAhrtuxKO1tvGJq7xgjYjDSWOwqQ2sTMUA7DpibGxf3S21ZYP2NZNOSVgxEyGRMyIR7TJMsUKW+PafW18giLpK+tnsAIOVhnWhoW1Gkow1omhFcM4MQ5hA5wlGjdjItQJwmCwhmPeRAjuEBeNeIdY/+Z9cD5zEBkVcOvdF6sT5H3fHKNFQSvaKioqKiYhPFEOFdfp72WkiJbqsq5yPdh2xZhq7fT+/WfeH3aWnTzumfZ8F/0zRpcMGqRu7n658P7V1HCy/9S5hi5zIRtDEvPi+WRqL3Se158xbe8P1zpqnI+3mH6tYnz4dI707ZA0p0y79UhXp/QFoeG1Ki54wOboqdixHoSdkFD3bSe+Wxfj0qKioqKioqNh+0Sbkb4MjBO7E4yWpx2z8FaNvxRLxSxqJdIh1gn1ehmkLYzs/Kc1kdZ2IW80N3zmXylO1caH10JR8yYSrvAVCiUmw0WGw7uIXZRJIq0HO9nU4QiFpbVNvqid7ZiJ2U+FTFd2y1HA84pyp3TtePsQHQgpnVosMU7F6IeRNIOBG0hNBi/fqA9eO1GI0aeAJiGzG3fg7r121AaFmtThwIqmp2ct8kOlZP++TXrvEoSxtjNAtAcScBAcQk7W7Xow0AeYLzACGAeQ6mhHbkZPKAPIgcvFOyOwLMEQ0RGA7ezwjhzl4J7ih9ANLJiZDU4uKDL33Ttm0ivO3FYLRqp0Mk6yFkMsWU6EAI6o9uEzz6XNo9S4S1Cl9iVP1Q5HSsHzvbmNFsgja0ETEwGnJwIHBo87iToxDps0KUyyWVLCcCks2P7DUg5xFiYIQYOkImefY9iEaZ/EdIkzbzoZLoFRUVFRUVmyickz/z08ju0nbFSO9FbULqAAuQhgjxaWn2uZ+vn26Y7/gQabsQiT6tHAvgmBlzc3MIZuvivS4hHPZ4115YNCGtF5ySTJP5BpTog3nvIvQJ6ttz7nzkuCR01eX2XpLS5blDZQwF7mV+IP8eDBHlzKKasmtaGf3VCqwjjggHRIIjp0oalwL1/KqoqKioqKjYfGCxBxAYaGMrFhmmHi6E4pn47hLnIYRObJpjG1XecgSoa60hJzs4Akj2I1VVOxXjAUrkL5KqGWA2RbnZqggxm4l/AnMjIS7HibGAS+pgI9WlH0xpLwR6AHNAiKzkp1PSWMhqotJ6I6Yyyv7h6ABEDaED5ub6dRFiNvl3swhneG4scVsbEcdRaXlGiBExBHgSwp4c9MVJYW6KdmYhf2UPH077Gpmi3hGpUnwOoChjqaTwjyBYXGqEtBDiRB6gVoniqJMvwIxvQPAAHEILzOmz4hwlhTbDYRzENzyEVj3kx7pCIfvIM4LQ6GbGj+SMo5veMmJ0YgWk9kBhyoa35OQOk8vPbGgBUH4W+2ISiaMpTZy0COB2jIYJcD7liW0rHvQMqK9OerE+i6ENOo7TlQDkwJB2CNeu94f0XlK2MZJfjO5KjSFUEr2ioqKiomITRV8NvqDX+ZRjVlaXgB7OZ8c3Vom+EGm+GGJ8senlz0SEpmnEm5BFkT4ajToK4mESHcWAYPo1gLTCcxAT+Rlpk6fFYCnE+pJU6x0t+mQ5ffJ7KdcbUpH3z0nnTiHRl/rev6ahsyS6l7dz3kC9h/KLbMcBPg+Ch8qrqKioqKio2HzAymCLYjroxvaiEJfVkDkvKZmdVNRF3GGbi1pZKT9HmEZ4Qu2rxKYjn/2kHaVXZCPlsw97CKTEMCWik8glIl/iJ0pEqCNbuZlFOSYyEGJXQyQmJd9NMR2EzE3EedQNL6UPcj+omh1lXIX80vJJiff80g0tWQhXCTkZkaIcYwYHsUIRH/modiYEI8mBCFAAKakOs2nh0IvpdKNYmN2MTYBI+eTM+iUiku6jaXJ1baso0RlMbWoEQfrXkZzkaAYcHVrSsmIh9GCxnpGFDkryxxZtO0bbBmQhh3WWquuJk3I/xogQZcWAPAuMtuXiHg2skoh5wkOeFWmWTLbIygXLK/df+svI7IYcmMXzHaMWrinj7AjYCgdqQB2//ez5345b2Vi1DWiD/Ny2IZHmMgGC9FymvqCFCXSgkugVFRUVFRWbLJxzSSlNzpa79Ylyl2bgF02uG3XcI+cN08oYOmZp5bH+56Fj/ev106flnZbGzEl53oY8WDESvZxE6J44zIsPke7zEdITZRPgBkqej3hdLJF+R5LoiylvIRJ9KM8Qid5PL88ZKms+Er1/fqnk6pdVEuw8UE5Zh0SYxyiWLsXAtyynVMJXVFRUVFRUbB4IIRPU5oHOql5mJaRh5CsYVChjSxK9LwbIccnkir4kiCGnZLoSt96jDOEz2ZtjlRhE4U3K75IYfcDsN8pzYpTjQlK6InY2EjyiVOIzN934j0ntY7IPt5G15EJnAZ/YudjKwGwBI0Q6p2uQqquNTJc6m9AHCIjazwwOAAezP8l2LAxG5ACKLSgGhOhEnW5K9IJ4zmbxonQnJWk5AuxCqgNg5wBgBwenVjPWR5l4Zw4AxjCLHKgi3hEQxScGTdOkDWNjDAhhjHE7hza0aKMIhISElhezrVhg6ygAHuWoJk9OsKrYWf3VASKfjtuEhNxz62fLB5CzVQtSrveqHk/lx+5zElsEliff6RgMABAZkeQ+28ROevaYtd0BaegQGeOxTFSFGOEYcN4BxdhYijYSHYtCJdErKioqKio2UVDhQUjoEdcg9dwzshkDBPt0Er2/EWnnureTRJ+PXC/T+3n6dZhMLwNpabD3TgPOqAFoBGOS4OxPFHT7eSBtoF+mM+jDavahds2H3wkpOw8/LkEysnBmyQWTnqsRuJWFPDjMg0QaTB8iwqe9l3n6G2vNh7wZaN5Y1Ejx8njns3MwOVk5EVMuza6oqKioqKjYvLB+/XpV9QpJS+rtHNPml0CKVbmFbXY4tLm9oR/XlD+PRqPuOdS3cTGiVtTLEhOXAgIjO7N4wDlSMtI2IjWyXohdIVMZzsWk9C0Jd4F5sJc+6HKdEKAkuhChkQPIhcIIhgC2jUa756bPSuxqj8i5pIp4SswvEmVNAEIEm11JCKqUJlXIi/2M4yB1cjHZLtp17Vr2VqqeQQAje53nK4vnvKjuJS723rznA5yTvvMuqk2nSxvPMrWqCgeC+p9DbWecl3K8J4xGIqryugkp0RxCaHWSIoKc1kltbqRN9hwa4V3G0g6yGsAmR8oxnEvni/KcQaSbl+qkRtRNdVknQcAyjWHjihiB2AaxY2mKyZjAIOdSm1knXYgiAovX+7gdo21b9YCPokZvAwKrCt7lON57vR8dEn3h+LyS6BUVFRUVFZsoyGXlRfdFKdgt34F8bF7rlwlFuwThyfZEgxT77CiratLAwDkNWhNrCpTEMymtXAbcRZqW2CmjVMNMkuicYiNKdWEQR3EUdBKQzzQOzF4HEi2cIzSNR95opk/WTyQV6DLLVLwmyuinTSlj/thukUz2EvhbNqJ8SkFMnCYAbAAzdD1O1eMUmMvYgfI1ys18mHNwDRLlSYdwZ30OCibfLpJ+lnexYaG04ZAMMLRAMouWyZeoiOwgyaQUdz3Y+0r1cvPQiAak6hgbBJEut4WeU1FRUVFRUbH5YMOGsZKGQhALmZ6V0wYiwEGUy967gqgELJoUwrFrR2eEtaWbQKQksEUt3CCrf4147xL53Rfr9fLKPLE54Uz0E4EdgSgTxNIWQrbsKNXiAWYdwwWBn69D4pHOEeCg8buSoYk0z7F/UiAXwWsZa4llDasbjLbA7FFU3cwaT6a+UGI7qcdBQBISKYmsYwq9IgBKEw3OO7l/JJuOOs9gnXxg9T7hAASO4hhPLHVyYmkTIqtFjt2VBkwEjq2GvQTApRUFKYbVh2g0GkFsVcTeZdQ2CCHC/NxDFA95dgxHAMiB9blL40R7Hh0BXBD56TKUniupt66qiNLWRKbrquhIpaAkTzo4RzI+1ItLOazmOAwKYpTDOsnCUcZqImwRFXpoAzZsWI+5uQ0Yt2PdYFee2TZEnUQCzDLI/PFp2tLiAVQSvaKioqKiYhNFlzCftF/p+5xPe9nxIZ/zoc1JO9cyUr1HarvedbuVRiLqNbFDgNu7S+XZoMK41S5hD4jeJfdJ2UuMfCn5PPIe48CIIQC+EbW6EbG9AGuC6J8GtgHRJO7WqmReqHkl3T/N+iVF/sXAiQF2BdedB0J2UePCo9XBlEP9iwgXrj6O3TIsJmawyKNMnhShBHp+DkuVeFJbMTpKp6G8QPY9H1Klu4JcjxFwjjvnVFRUVFRUVGweaNwKUceO5xBiUKLRlOOsKnEPIocYWo2tZQNJCRsY3luaIEbZkJM0Dg9BNor03qvnNiEqaeqcRwRjHAO8xkqhDekcjoSgViqiVJbzkhhCCXSvGnqNrkFE8I7QkKx0BTkhRJ0XujzqWIJM7ezARIjsEDgihLxhqQWAkRgBsiGmWRySku+JqGfpl1RPBgKMSFbyVccFAaJmD2lVoUxkMMmWpxwYiEAMHtC6ODiEAHhH4OgRAyGQx1gV36CYPNbZscaoESAPP/JoRrLhKlNUEREnUt4U5CCxKYnKgBOANkaMRo0KjBgNCA05wLUwyx+wKfkdvB+BHBDGAW2IQsq7BiECcA6Rx2ijbHrvRg3sEXIRaQWuJ0A2d1US3ZNsOepk3yhA+tZiWZt4EaGSxtiR4Z0DeSBC6hKJEFjU5QDDeZ1kMEtRG15RFI965+HcCM57AF5Jc/GsdyHCwcuETRyDo1chV0AI6zAOazE3XoeIMchH0CjK8GEsT05S1kM83okcnFeC35UzEPP8Di/qN72ioqKioqLi9w5GcE973Z6NRofSBvMXavGhfPZZP6BUuJeYVJZjkJwnykr0bt6u9/t8GI1GiMxo21a9rZuCsO9i0QQ4ZS3PRpdxF8AI5cXmXQwxnMss7xmQ2fGCCFdVjqO0lkCGbJ16mUqoPIZeXhtKyQWjDrTK1QvZtiUrp4Z81Pt57Xepn6dPtNt5Zu1SUVFRUVFRsXlByGKZoI9KHDM4CV8MRATnTS2evcIFQhhL3CVkbwTEjoRZ1cXi95xXyOnKPzAiAxyEOAeymt3ezRM8BiOhs6Lb3u3cHHNDCWCx+QiBO21J7UgjA6iPdfYeNzKeTf4QzUovqi+7Kqo70XRAWlI4BcwqmCEunQNhMSal1Yx2bwDAmzIHttIytEgqdiLAMYOpq8Y3JbUjUaELOatjFt3AVRZHmmd41LqVG9FDPdiz6AkUwAFw8EKsw0sfoQXDoQ3iOR50M1Qi81iXdjsnq2qdD/DwIMcgp5uo8hgAwyOCKSu1AzvdEDZibpzFJJ5Jvf1TD8Pp8+Z0IkUmQaSOkfMEEBCT4MWeeVGgk1gBMYNJfPeRvPidrkYQCZUQ8ATiCII8h8QthG1vAWrlOfGAZ4Aalikn1tUDOgGjcz1630yws/CYrJLoFRUVFRUVmyi8F5WKkXzzEeoLKdKBrurcUJZbko8pHeioffvX6KfDDRPtZVq6Bibz9j2y5V00BzRQxhDIOYRoy0rj/BuLzlPOYN5p6XdTIn3Sw3J+2DM3VE75Lj+44XSYQsf0TSYenz9vSVYPlVsS3P3ntSS9pxHnZZlDZfWXVJckfP9z3Vi0oqKioqJi80OeYC/ihN7moZbP9+KT8nh/wh5KnsvGipz2/GlbIJHTSgYjEgKHZKNi0WkIbSozqh2HXFMsVLq2MejEO05tGiMRCqdxMA+LegBCG7skPpT8BThNGrAqIKLajpjHuvVhJ6yciA+R8soulZkjHRoL5NiuiBGN8FcrPiTS3/JkQUcpUnLOw3mfNnJN6R6IaAFyEJsVSsQ+ERUbY3b71u6LjwzfjHQNgNYliqd+1BUEMUj9YqQ0aRM5wux3TAXuXAOiiDaotQnJqs20ooGBGAhtK0S7PGu2FtjGXlLXTKI7+NSnjMY36i3PADlV2wOIqi5yAJjg4HSVckibhEYQvHNZgKIqfSJKN9JugUboOiHFIMfwXsYPIxCiF9sc1g1Wmb1e3ERfqcoLopLoFRUVFRUVmyimqc43xs5liESej4RP1yv0Ih0CvLdBUkl4w85My/uKa+o/6VhxjsVUFkOX6U5VLTnfdAKTwBg1XpZtxgCOLZyfGRQnLIUH/X0j0IHJAcrGl2ObfBbl5VsyGbQyxKJFs+nYRwJl6ubjpFLnVFRSMfXyiQrKjlBKKwel8ynQ+6R5ScZ77ztEejmxUw527dmvdi4VFRUVFRWbJ8p4OXK2HSnj4xhDih+cc+nVV6wTkW7EKQpwKYIgHLXYfUgMArStWHVEJdzLVXNlDJPV4QzmrDovifOyDgDEqiWdO7mJe3/MEDobc06S83ZO7pPsRU4ohRi28hC6p3smcbvijW595X1SfGN1KGPDyBGkBHBaTUlQBXXeOynHu0pGu0wWU0MAMYhVoe4doGR8iKwbukJJXnknInhP+pkRItAwQOQhkxSk/uMu3f92rN7y8ACN4BzJZqW6yWhM9yYCZJuXQjYg9ZQmKlp9pkCMJspEQquMtWzKiTQJ4IiUSDfLHk4WQI6cWhs6jcLlnoUQRRQugnX101cqnGWSQJ7JkCYX0u3UiQxYmt0X3SjV+Qg/iiJ8d/K7ENuIttV4nAHHAJF4vDMDxOqZvwAqiV5RUVFRUbGJok+WL1aJvhibFwvo5yPhgUyiDyk+JtXoPEiMl/m77/30tPJyMm+h6OiXOwmGd4RR4zE3F8QbvREfySHcfg58aWrvOxPWn7e/HNsUaXLA0leG53TNbcoTdd8kFHkpp6NIz8+CqV9yPhkaaFlEqgaatHOxegx5pQOF53lxnqllyt+TEuXxaulSUVFRUVGx+cEsU1I8QB7OCTFZriIlpkR0D03OW1lG8uYYyAhWhqltOwpzU/qmn4EsW0CvbIBZyXFJ7FiipHiGxYIjEODgUKqk2et4wEQwMjAo6oWCNDahAVSp7cWWw6crZiJViV3t1Uxup3YUcZjGgYZ+nEZcjBuoR6Qyi9o7RlBkxGgrCkX5bHXrjIMcdeJnU23bJptZ8OPAoU33OavyBaUK3dokfaT1j9of7FUoHwuneobzTsQm7MAjBsijbcdo2xZtGCOEFkCE90AzIrjG581BQ0jqpcA+EdshsG5YJC+7nc4RfJOV6ASHAMAHQnTi+27PWrbQEYW4KdTF3iYClAUnzIzQtmjgwI1N9EQw8rOY9lsyJTqx7EHkhERnD0RV2ZeTO/bRkaxZXoy+pZLoFRUVFRUVmyjmU5/PpyJfLLlun/vvfaI8bQY0zzn5XHTSFnO9ofdumg4haJK0HCKvS2Wx9x4hBIzHY8zMzAySrHdXAvzORp8wLtG/Lzlrt/+GyjASvW/RMmSd0h8UzZcX0BWlnO/jkPJ8Ia90oKsq75Pr5YCo/D2ar78qKioqKioqNj1YXGC2K0SEpvHJCqNpmhxHQEj1tm07ccagclt/NLV6TjeyWhXPIegmjEamM6LWRby3S7GJK5TXmCB4JZ/ZxJhwhsGu9yqENyluJgKTKIC7q/i61jV2ni8IfrteYFOfZ1KaC6VzZ5kjT6rLU+yoqvLUnvJ+FVMGQqJHtU2RczxxEpwkGxetM7OQw6TFi62KEL3MDPsv6qQKx4igG8yS9g1HRqSonuZyH4XbjnpfI2LUiQO2/tIJhEiI3BbqalF1M7eI3CKEMWJshTz3BOcZjRf+mhkIhQBpxE5X6IoFUCSLnYsJAScTCl4nSEzUFCAKeuc4eaOLat3ESUKoy2rNUlhE+RmNslkq84zdTnm2WfrA7m9Ucp1cBLkITx6RGEziIy/iGO1blmkVjlCCP7dlPlQSvaKioqKiYhOFBSELKdEt73zHh1Tn5TWmpZd+6AsT44DTzXpgG/0AkE1gtCxLTwGRRK72buqOTMZbLboK6HxsqN/kGuQIjXcScMaA0M7J7vSOEHQ5rBsg5ufD7xttWgbHC+ddPDEseYfuyeT17LlIyh3LQ/lsU92UiqMiVclyHWwVqYHd1OdgsbBBXqkU6ivYh4j4ioqKioqKis0LeTUbgVlIc984VWGLgKNpGoQQQJxJcdvIs08Cl2V636C72k9jVFZCVy1DQjCCVby0xTc7gpzEtWLVIS9G3lh0mkWLIUDY21JNb+pqUtLUe2cngokTsV+u8rMxQrm/kktxXt78kyzWK4QNMUawkriiUudMjPdI0tKWpUwT/3CJEzkKESubgjoVSJuYAqo4d/Ce0DSjHO8BCIggBjwIbWTZCFPV29HIcxZrEZnscGLNwkDTOLHoMYW9Z51s8UL6JtW+tpMDnGsKy0BWK0SdKHAMooDIY51oiAACnAecN8uYFuS8PgcMD/NKl+NpjoVY8qRVoNou5DGbU2GUc4RAQvlHTuuC1WLGyGxR+nesGVlshGIgHS+g8zuQFjNACfYoli9tK6r+EGQCKT2fpoB3UreYNr4lLYPTxMNCqCR6RUVFRUXFJor5yPPyODBMolseO15+Ls8xTCPRabF5iVUxjl768PW4twkp7IpDavaB/plGZBKsD8T4gzliPB5jPB7LYMc5RAodcnTTxSTRfUeVa0qZxVxPp0nSOTagywNIObVbHqUBUl7im9NNxRILdZOpkkq/8/4LQIcgt3PL80sFvA0iyzJKkr2ioqKioqJi80Db2uadZj+BpNRuGp/ig6ZphOhjnoifJ4hsOCEvmZIS2+IkUU0HIIiVSOSg1iSmTGclKT0cyQrM0WiULD2EBLe62qaepVCgiI9U1S2EKsM8vSVepk4aQzZCFSV1Dt6ahtRnXMltEuFM5HIfGukTI+SZW3DU+N05EJwQyC4TsHaexYKdMYaqoGOIskFnkF6Vtkh5BJkcIO/gPVQxnSdEMhlbWoXEJOZxHhCv8wAxAo/JVlBeXvvG6zl5A1VZRSBWI0aNSyzJMAI4xggXI7xvdHwnxD6cEyI7iDe+b4V0N9sU5hYxOgQiUDOSsZhODDQjmXwJkdRn3IhvnZRJtZT75fUek94zIgAR8J4xsomezliSkLzNdW4lcEztNmI7Ru6O42xCiGN+hagkekA7jggtEGGe83mcGUMQL36WNkaOsM2WbOPUhVBJ9IqKioqKik0UG2vdspgNR+3YQu9Gok+kD+aNMF58obyAqDOGFPGTQbIFX73AyNTtmEhORLz3Ho1tGBkjYgiy87zP/uhVVbxxEK/Lxead9I23gUlnSW45yTKYnlcqMJMW4qaWUQ6GSoK874M+jWwf+txfhl1RUVFRUVGxeSD//Rficzwew0XCaNSAqElqW+dMldy1hcve2IVHOtmqTALYgWE+57aBqBHqYu0hJPqkPYvER0ocp5gIuvGkxUfdWIxV4WvKbKlel4ws7WVK0UtgDMZFTm1ghHwX8tOr6pgcgRIxagS5h20zwxFoo4NL7Tdim6fyo6KEVqsbJdFJjNwLYQ0JAa5WK84RmkZJf8qkrzXPVk4CnNTozsXUV2KtEhG5u8KgFCzFmG1w0iqAos/K64YQQGQq9kY2+HQOTGJt4kBwkeAbQlDLcedkhCQWOqa0V/scLx7o0eskj/aBTFwQYogAR7HUYR1nEcAwmyKZeGCKYnnTxKRQ17us90+JdL05jmWagNmBQMk2JrRtuhfSv1EmhUKLSIwQhUAfz40xN9ciBoDJA2TjNQbHVjdfzRNAsodq+WAsLHCpJHpFRUVFRcUmiiES3TCNHF804a7LHyVoMrUGgJSefzYlOnrvw0S6BaAlQW6fbamgtg+WL13WVuuhtH0pr18i1bdMs/cgga4jAjUNoP6VsW3BRGiaRoLaJdyPWNbx9wbcCy7vqLyYEqcO944pqoby2DHqKNG5l54HNul8AohNX1RsNCrrczNBHmM6Ply36Yr1aSS6+D7WjUUrKioqKio2R5iNSQgBDYvNSZ8sj0pQljYqQ7GDxc9RsosBXk/RG9XuQn7O5LIR29mWzmKVclWm1MeI1bINUGuOfpw9LRZKx1XZzlx6omdVvhG5QiQzSDhttVGU65l9h4kyrO0mppG+cul6Np7orjiM4MBJycy6oSlYPOnzeEXsVyiaGEPK987Kjqo0l/xRiXmQkOjMgG8AswFnFpvBqH3YHx+V/VCmcYx6n41Ep/QcOed1VYLmBwHk0kRI0ziE4CE08AjOq6WLkxjZe5/IcsDGck4mMpR0bxpVtrfirR5CK+0mIb4BuR4p+e1ZxmKeoWXZcyAKf1O1cxR/eGIHYkJUv3OwjJ+ccyDrE1X+xxgRKIBVYT4et5gbt5jb0IoS3Ut9ZFVC2c8Wt2ssTlBl+uJGaJVEr6ioqKio2ERB5IqlkBpIAUIeTiXGCwLdlNo5GpI3XapZqswT4azlA+ioyo1sTz8Xafnd/KmnKcy7P5d6GDJ1cUHuk9XZ8g3wuxPkeortsmcjEaGZmcF43ToEZlCMGOmGozKgWJwtR6rrYnhmxhTK9s7GgFqcMYVQHlaWT1NdD+aFGbf0y0CatEhEtxXCEP9LZg3ipS5ZXW5qE1PuaHl6tZiubcoh6KZKGtg7C7Ypld99SW/0X+SyBynISHjdCopoEVqXioqKioqKik0JbdsmQlQghGvbBhDNwVTO43GLxlEnZgfQOzfD4pFMgguB61IMg3Q9ZtlYtE/Ml6rn8me5bkyENoCOBUvRlF6dJlcRWro2DNlyJpOaoj53+rOWz0HaEq0/OPlfywrRkZKxQaxflEwWqxVR59umm1aHUtgQA2cCXUcOaUUtlfchizdks0tC4wmFO0+KC2NkMIKeIx7nEeJPnvpXVdicNhTNpL3lyT7nUSdfpE+c+atHVoV6RIwtmHWyQ2Ne8tYXDKBB0xCahoRQJ+ljAtDOiSe63ZO8MgGpL1kV7jMzABARQrZMFJ9zD++8bFRKANiBYisrAzyKZ1FtdyKL+h9AYMD5BiCHyITYAsGJ5VAbgxL5Lm2AmyaXHCOw2rmMAzZs2IC5loHowCS2LqPGgagRixsEhNAWnvm2SmFxQqBKoldUVFRUVGyqSIGPS0FQCgr15Upy3ILFFGT2SO9ErDM8MgmalOEl8W3kOLIaIR0r0rrp3LmmvVswP1GGpds586jep3ZRP2AaYLqJZKd5571sAhQjQowgp9scLYoVX2xoVmS+e7Dok7ij6jbUIczz9mc6UuSxp8be40Qlcw4b+Ng7IEtsbY9TI7oZSGkx6s9gRDtWvKKR9b30NJ4iUlJeVGYOqqpZfE9VVFRUVFRUbAJo21jEpaLEDi0jhhbjuQByBO88vA/wXib1zQMaJHYXgIRLgWNSZluMIeS4WYRA83bTZIJfrC28F4tC2UwUkMjYiT+6E/l3jKETiw+R48wsXtNFnOScRObeNYCjFDMJWS0Es6jJoyqRxaaFY7ZxISWJSQn0QDGNVyIY8E6IXxJVNzvpI+agMZgSpEbcl/If1s1VGUJwBwICgaKq10nsRGQfJE7OIJH1JM3PIJBXYlcVEhxFqS2KcvMh1xqwjZU8iBzgIshHeI1BCdnnPDBLvfTeMEQWziC0umlrjFH6W61QKC1JiJjBHGDjQFXzc+PQNl4338zPU7OiQeSItg1JIa+tgSNG0+j9CAGjkQNHB44NuJFVneQIjSM0Xjen1bg6JoU8iudS7oVsbMugkMd1MJ99AmIjz0tLQGyBZkTwjRDtgRmBA7iNGLct2tCiDbKHVYQHWJ6rVhX8M43a1MSAiIjGA01jm6LapMnC0Xkl0Stw1FFH4bLLLsNVV111V1eloqKiouIOhOupJroENjrLL7uq9C7xnM9XJQIwlaweIq6H0svlqNPO6actlHfo+HwE+jQMW78QZmdnsX79ehAR2rbF8uXL0bbjastxB2KawmopcMiK9dLrfPh6MohKJHw5aQMbBOoSZ+Y8cUPUUXDNV2fnXCevWbnUjUUrKioqKio2L4Q2qHhFYs1s0aHKYyI4r5s7ulY2ayxiWu99jnFBYCfbO0Y2mxbz5lZbkVBsdmmiF0diy+EZjR+pH3tWowOFXzoZCd+NecoN2K0dMHW5rSkk+xmFSttEBqxe3lH8q5m0HVllQAEgJc15rMr6JP4REtaEO2ZxAojPtdTfGp03d5cGSUdEBkIEYnDgwODg4QIV9jAEdhb3iQiinNAQ5t0hguHhpR56XUrKcmu3KMiFQLcNVxnkdEzmSWdM9N5GQpgzMt7awQCPwNSAOaJVGxfbiNbI8rzs2DZyjYnEdo2Dcx5NANrW9vRRex+IJzxI7FWsf2UbKJloCSEgtKLcJ4pwTiwPxTiF4R3Be0KjSwgiAzNuBE6bjuY+iZERnN4vT9q3uvktGI5U3MIAO4cYPLZcMYtly0ZwXuTzDNlYlDkm1f9o1MCBEB0hspN9XHXyRsRgEY4ifEOYGUm3OxIRmaO859U0VBJ9CfjBD36A008/HWvWrMG1116LlStX4oADDsDrXvc6PPjBD76rq1dRUVFRUdFBf3lmn8zuvyy93NRmKK+H6Uy6BPm08wzzkezyjom06XnnPz4tbx/Tjg2lz8zMYDweA0DylVyo/IqlYdrS3yVBFUr9zUL73pw2ALMl0P289jkNHFW+ZceGNhS1vGUZJWlubaue6BUVFRUVFZsfshJdSEU24jDmdCEnCYwxvKcUxzvnEBvAu0ykI3JSLptFi8UoEuPEFH8LYSs+116ZQO9H8IWFB6Ce28UGpv3YzMrrx07CZ5sndh43WNv68V2MhSpdmfZEdBtBHkjt+KIQ207sbsxeBXCJFGcuxxPFJAVn+xUq+52BEIU45gBZJhidiIXIdipFUkbHqCpv5eE5xLRSQC5qdjqiqiZmtQ8hTY9w3oRMMlECNouSptiAVW1fKG9Cmmx6YovA+We5h9mHvFgnqW1keN0k1TmPvGms+JubBZA8Q1EbJs+NzXyYdY/U16mli5DQrKsprL3OQTc11TFllHtmKwLyM8BgcvBgBIjqnxmIpIaRzIDXsSYTaERwPMKKFcswM/LwFOUOM4viP46FHHeM2ZkGkQgRHpF01MoyN+McI4QGsWE4L3nJAc47+OIezIdKoi8Sn/rUp/Dc5z4X97znPXH00Udjl112wVVXXYUPfehD+MQnPoF/+Id/wMEHH3xXV3Oj8IEPfKCqoSoqKio2QTgNaEFIG8XIz9njj1zxWYNOl4KcYZV68j5HzicB4ZAKPPtkL05xnpUPlj7NzmVaGWWwv7EE+jQ1ekmkj8djNI2f9xoVdx7KgVwmx/PkTJcstzRTG+U8OS/SMlxTTkHTda+o9NkM1U0Fj3R+lzTPA9tsJ1NRUVFRUVGxeaBtVaddhL2JEFUiNu28TgExOiUmXdrbJboI2/ARAJhDb6PQPNnvnFfPcKee0oTIrZCKzsH7BswQj+g2aAyc1bimrC69uYFpRHr3mGFiU1HmRCpHzsQztAtMlU+Ug7g8LrHNK3OsF0K2JMmEOSdFPqtCn6zDIer6wLK5awyi/CYWFXqp6mEN6JgZLrIo1gEQRzlGjKZpRO1MBWkfWZzYI0FMVqJOBDhV2stkBiHvN5VEIKKv1lUHDNmwU4lzkFq45HvcWdWgkwUycQFEyMSB+cF3hU4WO0e5op4XQlSP+hZ5LOhgfv3OBYxGXsZ+Hkl5zzHCEcRz3Ts4IrXpsXkOvcksm4iymWoSxO6HhUyXPbw41c9BLH48NZidITiMwWEDmDwiAaEdI8YNALfwjjE7gijfyYF8A+dGepGIGMcIwSOyEO4zMx7kbJNYD+cWpsgrib4I/PSnP8Wf/umf4gEPeAC++tWvYvvtt0/HXv7yl2OfffbBEUccgSuuuAK77LLLnVq3tWvXYsWKFberjNFodAfVpqKioqLi7oS0sSdEOa6Utlqx6AsAymNk+csYUpdPpnCnDJT1fDs+8d4lyDNZj05aLnM+gr1IyxdfkFRfKpE+RKLbAGBmZiZtCjUejzEzM4OsnBkm/TcGJfl7d8Mdohb/HaCsF6fniYtjKWd5Fkq10vzNkuCfCHBsy6VVlWRluvzRCPqh5+nu2H8VFRUVFRUVv1uIDUbXzkX0jK6IH4CoG2lSymObbIrYxfssFuAYlZA2yxUpR9TrYgEzGjVwXq4bNG7xvoH3jXhgxxyfmwe6EbWmXjbMFwfKeaVAIYt0jEs30lyU6Pkz2FbpUX6pMt05n9okZZQiiWzXImXFpKpO9WBScj6rq2M0KxASAl99vcswMU8A2AaenKxaOAIIYmmCSKlcmyNhOPH7jhDlBYTwdyk+tPto9ZJEu69SIyHQxcaGQY3s49O2eWPMybjWlOjSZsdGqlMxEdKf2ECaeBAiPaT7BRC8IzjdEEtWMoiNC0hXZQYRkVg+rxuLslmHchQ/dw4ApF4xREQj8m0GBaJaJwK8czqWJXCQ1RkOARznEIKHdyMQGG07hxDmAB7DocWyWY9IOtnikJ77GOX3j52Q60SMUdOIoIxkksjRwkr0uo50EXjrW9+KtWvX4uyzz+4Q6ACw3Xbb4f3vfz9uvfVWvPWtb03pRx11FHbeeeeJst74xjcOfuGce+652HvvvbF8+XLc8573xOGHH45f/OIXnTz7778/HvKQh+Dyyy/HvvvuixUrVuB1r3sdjjzySGy33XZJGVfiyU9+MvbYY49529ev61VXXQUiwtve9ja85z3vwQMe8ACsWLECT37yk/GLX/wCzIxTTjkFO+64I5YvX46DDz4Y119/fafMz372szjwwANx3/veF7Ozs1i1ahVOOeWUzvJ3g11j+fLl+MM//EN87Wtfw/7774/999+/k2/Dhg046aSTsOuuu2J2dhb3v//98ZrXvAYbNmyYt30VFRUVmyu892iaJr2XL9lEyNQpPilM+q8yn3OmQKCJfF21+rBNTJ9MnMw3P9ndec1zbctfvk8ra9q1+ijbOjMzA1b1yng8TgrjhcpdChbqz98nLKXO09pc3udp5adjsufS4Mt5GSDZOxWrJ0o1z3z1KJdVl78j5cv5yXxD51ZUVFRUVFRsPshq6cmXEc0xsvhOq+I4BnmFNsompEZ2BiCqajgUm0z24ZzrxM02LhiNRmgaj6bJanWn3tUhtGjbsZKpXdu60rpuUnRSqLGVhKW0+Slp3bPCnXly35rumKAbK4ltDee+iVyk6ec2FP1k1wCYXSaLA4lKnMv6m9CIeimZmjZVO0clgWNE27b6Eo/yEKLY1EQhzdtW7hUYCCGibeUVizrbxILUT8neSKmPAKcTCVl4JBMkTu9fJuat+mI10713NnaZm5vDeCybcI7nxmjHY1Wgx6JPI8bjVvMFbV+2DIIq7O0lsTYXL1GqO47JM122RJWfvQMaJwR9oy/nhIhvvEPjSVTtDeSzZzgXwHGMGOcQwxzaMEaIY0R9EQUsW9ZgdsZh1ADeRTiM4anFyMvmosuWeWyxfBZbbbEMy5aNsGymweyMx0zjMfLVzuUOwec+9znsvPPO2GeffQaP77vvvth5553xuc99DmedddaSy3/Tm96Ev/zLv8Rhhx2GY445Br/97W/xrne9C/vuuy++973v4R73uEfKe9111+FpT3saDj/8cBxxxBG4973vjS222ALnnHMOvvjFL+IZz3hGyvu///u/uPTSS3HSSSctuU4AcN5552Fubg5//ud/juuvvx5/8zd/g8MOOwwHHHAALrvsMpxwwgn4yU9+gne961149atfjb/7u79L537kIx/BlltuieOPPx5bbrklLr30UvzVX/0Vbr755s5kw3vf+14cd9xx2GefffDKV74SV111FZ71rGdh2223xY477pjyxRjxzGc+E1//+tfxwhe+EA960IPw7//+7zjjjDPw4x//GJ/5zGc2qo0VFRUVmzIa75Jie5jEzu/lZ6exazfdbFYsuDTSOCt5++V0z+u+C/rKdVNgTyrPy8/Tjsv5XYVPqfSZVsYQho5b2UaUymY+4mPpZdedjVahT88/3L7bg+mbbA63eTH5FrreYsuerx6m8l+U2p8mzxmoWcqbrzH5OeVVkYwsn9VBFGd1er8CznU3Hy09QauNXkVFRUVFxeYHExUaoZ2tN3LsmojqkP3Im6YpJuLF2zrblpgdR+hM1kt8omIZjV+ZGU5tOaQM9Yt2HsyhILYpeWHbRpjyeTIGa9sWzJxEN5kEl2tYOd24jxCDkfHqa05qcVKQ6KJiJoznQuqjPBYpfOR1LxxR5dvmqJzaJNvOoyCrudBw55W2KW7TDVzJETj9FxFU9e8gavQIRowOaAPIOZBazbBUAAzx7jYrHSACFOFaII4YMzyCZyWclfQOodwUtXwHwCH1td2LbOND6dlxzgmRDxYFvm5CyiyEv/StV1FQRGgjxqFN95LIIYQ5hBDhyIMgExXeO4xGI6xbt1YnWiS/kOJeV0sEhMgg5/OzBuhqAJIVE8wgll1VvXMywaFtJTLxi4wbiQA0ohInB0RuMW7XI2IMx+KqHmJEiEFEMiB40s1mk2e+rHRgYjjHaDzBNwTvbPymz8gilOiVRF8AN910E371q18t6He+11574aKLLsItt9yCrbbaatHlX3311TjppJNw6qmn4nWve11KP+SQQ/CIRzwCZ511Vif9f//3f/G+970PL3rRi1JajBE77rgjzj333A6J/vd///eIMeKII45YdH1K/PKXv8SVV16JbbbZBoB84Z9++ulYt24dvvOd78gOwAB++9vf4rzzzsN73/tezM7OAgDOP/98LF++PJX14he/GC9+8Ytx1lln4dRTT8Xs7Czm5ubwl3/5l3jUox6FSy+9NJW311574aijjuqQ6Oeffz6+/OUv45/+6Z/wuMc9LqU/5CEPwYtf/GJ84xvfwB//8R9vVDsrKioqNlWYt/l8Sm37XKaXG4QOK7uHieq+ircsf77P3TS3+PMG2jzt2vMR70NlzAf7ezU3N4cQAtq2xezsbFJ63HFq8emq69tVKk2qfubLuxTcEaT7fPWYnxTX42TEdkm6pxoiq3jEJ5OTpygm8ptiicjJkljKijEZKKCTN78YrAOpckADdP3RKyoqKioqKjYfyCS82Y+oeMWXemdT9mbi2VaCeu/SJpLZwiRbuZSWJiXKeENiH7XgiGKrEVp5mSo8KaMjTYSiRtoaIT8kakibeBIhhNCJO9N7tHisjNNJ+4bU8qT07S6FMlmsY0KGrgKe0ucYKTuXJA0PmTZieDDRafBQnlxvqH0MiGRzTbKxjGy0mTYUdYToACH+ATSUFPREotIO2iemPLc2JcW6epcTKX3PKlByQON8wblHhNiKvYr2TSRbqWD96FIfxlCURR6mu/euASGCyKd7YGp1U9SPxyomIgI8gRop1OVuRuTkfi5e5ewA80q3GLr4HImhoTxYVetQP/zAQbzrA8u8CwDmFmPdcBVE8J4gFvWUJolg9jtgMInnPDGpy46S58XkzXyo60gXwC233AIACxLjdtzyLxaf+tSnEGPEYYcdhmuvvTa97nOf+2C33XbDmjVrOvlnZ2fxZ3/2Z5005xye//znJxLfcN555+GP//iPN9qn/dBDD00EOgD80R/9EQDgiCOOSASCpc/NzeGXv/xlSisJ9FtuuQXXXnst9tlnH6xduxY//OEPAQDf+c53cN111+HYY4/tlPf85z8f2267bacuF154IR70oAfhgQ98YKefDjjgAACY6KeKioqKimECvW8vUVq1TPvcT/M9q4rFWFZMq8s0gv/2vIbKuCPKtXKcc8kSB4Aq0rPi+I64zu/ydUf1xV39mnqf04qJ8sUwNUtStQBpMyGi6RZF047PZ2eUrY+6li9D9a2oqKioqKjYPGCbOJJjNCPCsmUzWL58BrPLRpiZbTCaaTAz22BmdoSZmRFmZhqMRl5ib2+WirIZZFoJV5DqUZXSmVCWtDa0KV6NEQgtMB5HjOeCWHWMNT0AMRjRLj7tfbK8r4KWdmmeZJ0ihGssLUJCYU8TjWymRLACtgloTLYhQtiGpI6Xa2WRg22CWZK7Vn6MZr/Cnesmi5R5tAyJ7EeemGAlcqXDjXB1qR1c2PFYfe1aVk8jsTlSqlsbAubmxtiwYQ4bNsxhPG57G3xa7zhRhROle5xEIUVj5L4EtO0YY7VtEQ/1PNGSNxENxbmkYz1RkIvtzyj51IcQUz3LyZZk/5Ksh/SetBFtiJIeGdH6w/pKX9E80pkQQAiqWA8MBAbayAgcETiijQFzocU4tJgL8nlDGzAOAQEMJtLNSZFWD8iLAWLYSmuGWCC1UcqNOlvCiyDRqxJ9ASyWHL/llltARNhuu+2WVP6VV14JZsZuu+02eLy/6ef97nc/3cSsi9WrV+Mtb3kLPv3pT2P16tX40Y9+hMsvvxzve9/7llSfEn/wB3/Q+dkI9fvf//6D6TfccENK+8EPfoA3vOENuPTSS3HzzTd38t90000ARIUPALvuumvneNM0E37yV155Jf7rv/5rwpPecM011yymSRUVFRWbFby3pZTlK6s3ht6FACx/BmxpXc5L9r8GI1Tk6b8DtrFPeczQrwNilyAtMZQ+LdRZShnD50+kpAEDkIl0C2LH4zFGo1FS5iwd/XOmtmwjyu6VULRj/jrcMde7I1FatEz1FHcAdCBDhPQ+FYRiqae9I0todFBIgAbZ3ZPtGiwrRdPyYHKuUKnbpmAu1b8q0SsqKioqKjYvkGN4EEajBrOzMxiNRrB9dmxTTtlsMsI3WYhi8UMScESLR4WALGO70uIjjQOCEO1D+cQ/PCoBDDCLSlmuJWbepZjAzjVSvlSXBzaLFqkbu0ySlzFR6g/lo4mVUCfboV0IWfvsdIPKckPRvk1Mh6BOaTFdl4qYVrqGpijNe/UkYF7G3cxgOA+y2JTUTKnfU3ipSvkQIhhjoI0AxjpOcyr2IFkJABOMeK1u6MTC5X3sizSYI2II8L4rOskrGKyhWSQkeSLamFcdW7/GGNI9t7GfWQs5FpJfbIAAcAQHwDm1D1IrFjmPlUBHmnhgiP1KhGycygxQjKJKpwhiwCHAs6jZ2TkQh3RrImVSvo1BFPC2aS1najwJagAQy8a2YqMjCnzmyT0c+6gk+gLYZpttcN/73hdXXHHFvPmuuOIK7LjjjongnjY472+saQ/gxRdfnL4QS2y55Zadn0uFd4kHP/jB2HvvvXHuuedi9erVOPfcczEzM4PDDjts3nrPh6H6zJduv8Q33ngj9ttvP2y99db467/+a6xatQrLli3Dd7/7XZxwwgkb5QMaY8RDH/pQvOMd7xg83if2KyoqKipkV3OYMrZUvpKGe+U7DSvGLa37DkD9AykXiNLuXIITgBMpX55r6KdRUtR0Qt3yZyrPHEa3nt20+dK7teoHywxHlL0TidB4D44eY1V3WHnDft0pVJy4Hg9dLkX1/fpN8xcvC6EF8g55eE8u/90YAn3aEKN/PSG2h8ofqke3nKFzJ8lyV6TZXbMLc3fQpHmYKb3LI61BPjlddixPIff/s0ER+ry7Bu1OgnQq7F+SCqqioqKioqJis8HMbAPnCKPRDGZmmkSuuiTCMIGAqc0jkq+5ktiyeaXmItuDJccUpjA23/O2bRE5IgSXiGOzqeNiQ0uoEhwMEBqxjnEB5GJahQp0BQ2ZUDVSO6Y69dEVcRih29UyuLQxk6iHQ1Dv8RhgfuPdMQpy8JeU8Fy8StLegr+CfCeALB7j/JbamCK8IoPW3wJ4iwOtnoD0LXRVABB1BUGOX5lJN4MFGC1AQX3vpdyoZeeJA7nhUX29yTmxGITGmErA2/mWDma993af8oRJGq9AvcmbBo4cWg5CXJOJQfLqBtkLitPz4l0jbY9R+1Y3b2Uh8UOQDUTJMRwTnOP8DOo4h0mV6kaka1zuSq+XwCAXMQKhcXJuiKJjF4sjD4q20kLq17J+tj2MSDzdHROcTqJIXC91DjHKBrALoJLoi8BBBx2E97///fj617/e8eM2fO1rX8NVV12F448/PqVtu+22uPHGGyfymvrasGrVKjAzdtllF+y+++63q56rV6/G8ccfj1//+tc4//zzceCBB07YotwZuOyyy3DdddfhU5/6FPbdd9+U/vOf/7yTb6eddgIA/OQnP8HjH//4lN62La666irstddeKW3VqlX4t3/7NzzhCU9YUD1YUVFRUSGwANbUIPJ50kqCiIC0kU+XNB9MU287Kkj4FJSmoMjKJdA0xfAAaCDotjZkDJHU8+edzD+9DAmnhiCe2OmzJ4Ab3WRIFDm2xHWgVgAinJn8DR2egJ/CSg8R8Tx5nIaLZSORF6xDDpIXA8YQOT+M6flo4lhftZ3HSt1GUDnL0kuTcRLr1A8XHLoMoPqqdRsQ2Wc4Smk2oIqJRI9KoFPxWesN8blk4nRu+V9FRUVFRUXF5oPRSMjo0YjgvKhsAcADGnRktUgIEkXI5owOznkAYqvh3EhJUsBRo2pjIAsoCMwOIdjEvcM4WKxGAEIiU4lIiVSLDwsLO0/wPosyup7rTsUhEiyZ8l2unVXjWVzSHVsIsQuQV7KWi+iMKfumEzS+MgEPiTOHszGNAyDkspDWaq/SiR85lZGFOZTt59nBsYf0BIG4WE0oTRW3eooACeGr3Q0OWhfrnxjRBiGbCUogE0DMwFisfMgRnNeaEuBcgxgb2CRKTIEogKQaV9LfOe150napx3m6D4TIAQyC8w1ADhEOxNDJF7GgSTY02vdEgPNAQw0YEeO5MbJlTFRRim56y1n5bhZDUloERye+5fostW2AdwRqPJhIJy30OSCoL3kExzECAzGo8IRM5MKgCFG7+waeRpAVovL7wiGCPODICRFvCn4WEYzZzYCc/qZln3QHks1jnd27hWPz6om+CLz61a/GihUr8KIXvQjXXXdd59j111+PF7/4xdh6661x3HHHpfRVq1bhpptu6ijYf/3rX+PTn/505/xDDjkE3nucfPLJE4NBZp643nx47nOfCyLCy1/+cvzsZz/b6A1Fby9MqV62Z25uDmeddVYn3yMf+UisXLkSH/jAB9IOwYB4uZfWMABw2GGH4Ze//CU+8IEPTFxv3bp1uO222+7IJlRUVFRsEiiXXpafs7e5T68hj/O+Kj0FS+mlecnBEXXKS6+7iUf4tPYM9pv903vRQJpvfLJea9tWlT/T2rzUvnASlG/si5aYPl85i63vEtq3lOd4aeVOvrr1G1IyLfI50pc9IrbCwzn5HUi/P+mcsuzJ61RUVFRUVFRsPhjNOIxmHJqRQ9MQRiOH0chjNOMxGhGaEeAbIS1nZ0dYpl7pvpE4wigWRw0AjxgcwB6OslIdUCuLyAgt0I4ZMRDaltCOGa15oRd+6OYlnhxMirhFCHElIqHxPxy8E/9sgnmBiwVJjIQYoB7rsgFl9mPviiJkw1SC94DzDOfZWNUU0xFsEiETtuRUmsBiwSHkuVjPTEXardLI9AhEAqKDi0LIEjsQ+7S0sEOkk/lr21Wk762OUHW6+IMDIYqfdxuANrBsxBlajENEMNU0MZgcIntE9gjRIUSHGEnOaxnjltEGfW/Lewq1fHHa1yye9jGq+ppAzkNIcw+Gl3oFQowODA9yM3B+BHJNUqc7J7bSo5kRnG5kK8+D2Oo0DWFm5DBqCKMRMDMizM46zMw6fZ7lntr4kHVlRGgDYivKcHtm8vLNiBhbxDCHtp1DO55DCC3aNiCMg7QnMDhAV0uQTKRwTIp7IkpCLsmvvvSREKNHCA5t6zAeE9qxQxg3iK0X///oQZHgFkGRVyX6IrDrrrvinHPOwXOf+1w89KEPxdFHH41ddtkFV111FT70oQ/hhhtuwAUXXNDZwPPwww/HCSecgP/7f/8vXvayl2Ht2rV473vfi9133x3f/e53U75Vq1bh1FNPxWtf+1pcddVVeNaznoWtttoKP//5z/HpT38aL3zhC/HqV796UfXcfvvt8dSnPhUXXngh7nGPe+DAAw+8w/tiMfjjP/5jbLvttjjyyCPxspe9DESEj33sYxOTBDMzM3jjG9+IP//zP8cBBxyAww47DFdddRU+8pGPYNWqVZ3B5Z/+6Z/i4x//OF784hdjzZo1eOxjH4sQAn74wx/i4x//OL74xS/ikY985J3d1IqKioq7NcpNPe1zf7NPey83UinT+2mAKSBUDZEIRSUFC3U6QdS5QwFtJjuH0vuYot7GkKJZgrzbn3eayr2b7kh8LYNu2BRCi9Go0WsNtVuuu3gsLu/vjpCd3veD9Vhk3tJmZfH1WEw69d7Law7b2yz2eJnHOdfZTNbSzfccADhm79Fpr4qKioqKiorNB7Z/jrxyTA6giA8ivAe8F7ou+6V3X2LFonGaigRMTSt0r1hsEJlC2VbJCdls+8uw66/4KwUGqgJXr3Ore9/KxRDV1kP2JBVCPXl8M+CcbBLaF7hwsmEBHAm3bbEVUVax5/qJh3VeNSmkce7HoRWLlCxcksy7JMntPqREXTWo6nImHdewbaZq8SwhcFZplxt4sqaHKBMjjKA2lw7kRB0exVCk059dX/tyVaa8RAzFabNZs2rM/ZQJ5X6ZMTJcZIj2tVjlwC1C0Hrqs+GdQ6Rcj6YRC6LGSRxPzlTwgJMOQmghm3xGqL2NPbPSN03TTNTL2mbPf9RnW4T48nxHYjgaywSOH8GRs0e/Y2sDyD4BIbDuHaCrI2zzU5J9w7xZB8GDyOtEyMKoJPoi8exnPxvf/e53cfrpp+ODH/wgrrnmGsQYsWzZMlx++eV48IMf3Mm/cuVKfPrTn8bxxx+P17zmNdhll11w+umn48orr+yQ6ABw4oknYvfdd8cZZ5yBk08+GYB4fD/5yU/GM5/5zCXVc/Xq1fj85z+Pww47DLOzs7ev0RuJlStX4vOf/zxe9apX4Q1veAO23XZbHHHEEXjCE56ApzzlKZ28xx13HJgZb3/72/HqV78aD3vYw3DRRRfhZS97GZYtW5byOefwmc98BmeccQbOOeccfPrTn8aKFSvwgAc8AC9/+ctvtxVORUVFxaaIkiwfUsB2guTClsWIXl0k2E0jCW7dQBkZXKTLeRN1s0ivhyEScxrXuFiiO9VgWvrtKNcRwI7QNF6DPiHSm7yutriWBJqLJ6WXtmCwX787jKRdLD+vNj+LzDz1vg5WYerkyvD1hsp2jgbTbVCVBldTjvc/l+S5DQAsrdz4tO+XP23gWVFRUVFRUbFpYzQaqZpaSHTAyE2LM2zFqKzulxiCEUOLEKKqjaOovU1RnSxgHNLOLdEI4L4FCwDiRGyXL4tR0mdkkpjZxhRCOJpXe7LbIChJrNcLSPvBiDe7iSecKprz+CQR/QZ1lrGYSvpocl89IfgtListCGUE0xcDDTk/yIc8yZBCdEIi0TlGIZLJJiYko9jV2Bgpk8SRW/H2jqQq+QCQ+durHSRFmL2I7bmUJw2k3jZ5YnFj7gdWMYeHj06V+bYxqEubsFo/mHVJKiuY9Y30n3ix2wRLbxIgiqe6bI4q3v1N4zHycg+TXahNguhmsDHIyoByP0SLfUvBSe+OyrxG1JURUU1zoqw8cE6sjFpq5SFxMY1HY9BVApwnmEKUewTd8JRZf1ccwFGsZeQhciD4NAmxEIirDGajcc455+Coo47CEUccgXPOOeeurg4A4LOf/Sye9axn4atf/Sr22Wefu7o6G4UYI7bffnsccsghg/YtFRUVFRWLw1+d/U8AFkmiD6T3YeneZRJ9Wv6FyptmZzFMok8PVRZbxnzpS8k7rX4hBIzHY8QYMRqNMDs7OzGxIAH9UsIuWSK6sXWe1m9LCf2WmnexW4ffEUrsaecvZv/y8txS5TP/ezctD2i6KqE0SJmSt/z59JccsMjWVlRUVFRUVPy+49WvF1FhGbOZ6lZU2A5N08A3ssdODBHjscSYbRsQWiHSYyxjKeoQgMzIGyxGJBWwxZQuEaCuowa3emX7R/FEN6IcQMpvsW8Z34QgftYyIWAKeejGj1Z2fpmquVSii2IYvfYBjDAwrlDiNsaC+xYiuTzX8vdV4gBAUYREjh08ezh2yRedKcL+YwoIjhHIrGBK5beVHQoiXeJAUfEHnbiIaEYOzgFN48TyxJPY4pBPdbWX1bWst7H8dn9kNQPBeftZSG4hqRnMbbLhAZyQ0ONWY2WCc43u52RKeV15ELX+MSqJLu1tmkZJ9Chq7qaR/aJYiOzYRoznIuY2BMytFzsWsNONRoX8HzVyTVu9zICu5g0Yx4AN+qzDbHIiwZMTi5mmgXf6O+KdeK0Tow0tQpRzQpQJjBCQrHFkA1dOkwxiwQjM+AajmQajpklpJ/31p+f9Ha5K9NuB1atX49e//jVOPPFE7LjjjjjttNPu6irhAx/4AB7wgAcMboB6d8T69esniIZzzjkH119/Pfbff/+7rmIVFRUVmwly4GyBWfZANAVDX4kODQqz+qIsL6syUrFDtiaabgrxznsPnMqZvNYwppCzppROovsp11u0KQkAjuKC6BzYEVoGOLTg4OFUjS5KCkzdNPXuj6HeGLoh85w/4Lqy+DmNYdU60bT0YSV6uWy0PLdUoqcrFkqs/C4VLy1dLK+9m8rGBnXTyxpWvVdUVFRUVFRs+iitNQAoYQ4wiYJXFOBiHde2Ib1CYPW7NiVxKlHjbyFPHXmAGAFBryd5UFqYFBP/RJT2tsv102Mur6hr2zaRoUbuyjVtDavFYVnZzS6PK/L4ooy5yp6hFCOm2IxspWNX9MOM5P2d83YnKEqV/eB9KJT8RVeanl+OUxZAxLR5qVaVsp2fTSowM0JBPgMRIFFy+6YUV6idZrmYgLLVzTCJrs9K1HKtH2zOgsTvPImsIUpydCx0nCrkWXzIo27wSrq5qHN6NyOCTpbYBIjlA1jsPU2NzoCDk41BG0bTEoKXFQwxkpjvMJJqHmRxt4wFneNOG6SOTjzf1bqe1NtdYnHrDwcGI8SQ9lmMXNqzyKRMCAGRs4UNkzzfI6f+95SvuxAqiX47ccIJJ+CEE064q6uBCy64AFdccQW+8IUv4J3vfOeS1HZ3Jb75zW/ila98JQ499FCsXLkS3/3ud/GhD30ID3nIQ3DooYfe1dWrqKio+L3HkALd0k1NAmg4S0V+52wlY8qf3y2IQ+d9+HoDBGrn3L4qfqARU1w7NuZv3bTrTl5uaWV778AsYZWpc9Jmk0UfxylB/B2hkr89508bXAxdav6VAUP5MXH/ltICGUguIZ2mlZ+f23xufyC3sC+6YWhQVpLkVk7pn76U8isqKioqKio2LRjhGmOplHZKEEqc0LaMGIP8HMXbuW1lQ1Ah0gOc84Pq8HwhAJTVyy4JOQi+cWiayf2R7L1jPecgTiQoxw4O43GrsY5PhHmIUTYU5dBpczsOaBoP34jiXNTAWYjQVV+LnUfbxnTM+SyAEGsO6IaamUC2zU8BIUpLlb3VnYiS4tmubWOUMhrM6ndOJHTggOhIzFzYVM3WpbFzT1N9tC/MwsU5QjsOIIpoHeuGqh5R/c1tEqO0cek/O3Z/hFS3ttnKVSGMjUz2XsjtxnswU9rktQ2h2Cg2IoRMkFvMKm2TzpUJFkoqdOe7ginJG8EpzYEoav9H2CrcnNcmALK4RUh5houAbzycG4HZIYwZgQMcSRuc85gZeX3GYnpG7V63bdCLZDI8PV9yN3ViSMZnRLZHgK4MWISTZiXRNxE897nPxZZbbomjjz4aL3nJS+7q6iwaO++8M+5///vjb//2b3H99dfjnve8J1avXo03v/nNmJmZuaurV1FRUfF7jTIoXcgf3Tvq+JwnlUdRlrzLP8x9pQdQkpFZpT7sAV6S9xi4bj/vEHG78ST6dAIdgHoTLq5sUZBIgGkb5ZgSJRHpYjSIODSbMG877jyydSnK6Kn3A8OadSOt7+i6TUs3b8lp6CvCp1LuE/mGj5ef+xNVqU6FV3r5qqioqKioqNh8YCpmi0dNTR1ZfM4llRCZEDkghogQWrTjgLY1r2hREJsi3Ow8XBFyRwYcCM3IJWsOaOm2oWlX1d1dJZfegym8s60LkditOIJuDoT0YojqmZlVbVw2vitakE0sARR2h8zUEZyUggRrXCLt1Tfb1OjQFbPs0PHdXsx4IZPf2t6iDvaJI4MpW81ICClH04avqvjPqnb1O48AO86lKfkfIyfi1hTf89VZNpM1gQojkvUjwzlOPuKhlRUCzoslihHvIQSENqTnyMh381GXcmwMFwFCfsaKl3fUm6goRVYixnKe0fBIrVny/bRrdT3vNS42ol3HkM6xkvPdeN2p2EuuD7AjeG+TOfpMmu+/A4gZUNW9eKFL3RlWFw/ne8/rFFQSfRPB7+tAbOedd8ZFF110V1ejoqKiYpOEKRqmEeclue4d0PXrVlUAZfV5DugobUBTwvLKO+dgbICkLAnIHLBMD16GrFCW+rdv6HoTZQJgOCwm4AZkoCOUexTPRzhZ/hgiCPJqfCPBbmTEgb74XZHod5ziechkfLjvhi6ZrVBuD6bd68XXQ+oCmJULERfLRidtVvoE+lB6P21oY9EJm5fi54qKioqKiorNB1F9m6HKc0AU1G2rylwmmEo3tAEhRoRW/J5tVZvF9xZrzMzMwHnb8FLT1SKkafyEf7orCFDLD2Ai3slWL4WIpogJGYCyxmkzR44W60gGJlPBlySoKYVb5aqzxZ4Qy5KnFNxYGab2Np/3qNfMJKsQqFRsXNpvV7dtVBDTU5a+FueEiHR9G7cwR50EKSxXnLUxwsZQBEIzagBEkIs6mQH43kb0srGs69Q7WbswI4ZyRbCd4zrtF1tBhmtcaluyhtGNQo3El/iU4H0zITbi9ExFVfvLhZ1v4JyHqM4J4ABOYz+kjXNJ1/fmjWGNtM5jMrmnMdkg2hoAAsFRA3jAsSncjbhniI3P0LhKZ6ZA0s/kRHkegVZXIZDzcA0DrgWcrsBepO1mJdErKioqKio2UQyR5cOKdEDlI7awTgsoFePFO3V9NMrge+K1gDHK0DLSaXnuCPSv06dgk5Zikdd05GSSgPLSUedcWkpp1yISZY2fqpIeqkn5rrk2cuJg/msZuTxVSz6Qd76LLqWGC2MxSvDFVMAGgrn+pP9P+pXPd63FWrK4gmQ3cl2WKseO+qaioqKioqJi80DbthorAoBTwjOqtNgp4R3Vx1mVwyEmi5TuKjcqNv+EKG/1mIQYahcSs785NF3yGAEeJ8o2lCtaSyuRMiZKKudoxGa52o67RCe7pEg3At1UzFKOJqFQyhcinzJWJSI4kj2JzCZGCNzs7V3We3r8JtclxKRMHgIX9S4nGsrNSgsNeo4ByTZqpUR2S105k9CYHEvZCleziRSbl2xzYxMOjIAYG033cC7fyxi7QhjrMzghtrNYiuDIw5FL5L4Q1QGMCOdkAiUygwMhegcK9lzI/UrXgAqrEJOoytphEzLdiRrZlFWsVrRNaZJC72HMKz7T749uAiuPTwvnApyLiCHoM8PJssUTwcPDh6jtARoHOBd0jCZ2RIsZ/lUSvaKioqKiYhNFSZInFQkiyHzvjDwnArEHMWRDI9tFHqa0prSszspjGBPJMIrdAo8y/iBMZ1wLHrPgjHN5KbAuBgUp0F6Qnp9yxXQdKoL3iVppvfP1hrMRvJOd4aFqIHJOd7knxBAkmwa73slGNt0FohrYd6rCvfc+pqUvjjAfSisDWmidOwd7V+EB85Y0cJuoVVayoHPe4kjk7vmLSJ9234rnFdrntopAsnP3vagjFYNAsokQZeM7PWH9RsVH6t1aPaEK0SsqKioqKjYvtG2A9xbr2mS7KGHF51uUxkaaiu1GUAJS40iOqkZXwpcB12SiWzZH1CCDhUQEjPyVtBg5eW8nsccA4Sp2GATvsvrdjtnPtvlpjAyCT0R6DnqmEdcMKHEq8bCRp65DBJfXNdGHEdASb1E631TYQ2OEfnnMDGITt2Q7EaAXoXJxPuc2ZXs+2+hS2gHK8adzMskxGjVoGoLzBCCCKAId65RszWNK9KZp0n1J74XIxprDzMXkTASRKMSZGeN2LJvMgpTIt/g0gtlpu3sTKMWqgPw86YapQWY5xmQTFwHwPo2eoBvbWpwde2F6f/KhvBddxbyMUxMhr/YtpGNR5wGCh4MX3Xoo7nka04g/utwDAM4hcpMU70SMxgOgMUIMgPbhQqgkekVFRUVFxSYKIob3LntzewnAmqZR9YEtryMAI5BrIEsrIxwxfEOyHI5l4yDAiEuAKGpgUxCLbKoByDuMvR+qXVogmMhNIxtFdTLNUqWI7OBQKh9yuweUxEM/TyExCdmbEbmWRflFu8X8RQJMBigyPAhMDlEtbzhEIdcbAnfUF9Z+HQBQXs46WGnkXhus96JI2e496l+KlXjmNAAQi5pOZw1cp3P+RMUG8ltwuxh0ZlnKa/Jger7G9Pp0Wx7TwKkcQNlYKR8DOBb+jVacWbhoi+w9cugMFnKdWe/14iYRKioqKioqKjYNcJQYkYUlhs2sOwcEjpCtD1tEBAQOCKwqdCakUwAhtr2H80IkS+isaeQ1lia0Y1W5m0iFADCBQ7YfKa1D+iAHISApAuzStW1j0xiiqucJgHhMh5ht80SFne0yjN+PyUJFNxNNBHgW0hDZCkKJ1QAjWYs4Xl/pfAAx5o6aZstXWsUgauybo7R0frQXO0SoWjr9F8EksV5EBHNQvQ4JUUviS+69qNBJVw7oMKdQZQdwZFWAE7xHoaiXBsZo46Mcm0qe0vM8IIRihTERODqQz6S/+Jk7xBBTX1pgHGIQmZJjsUMvxnYpntb2tRQByAa3gK2+lFW3kSMiSZ+0MQAcchmc72G5ua7E2KQhsgpUOCJwK2Q5efGjZ9sI1YPACAjaJxGBIa9W420nTyTIVg57wEWwC2Ae61hYN/DlqHtaLTyYWjSJ/sBzbu5oiOQ9z37YwK+7gCHnL+e0sjdqOUO08NLYzlCpuEjSFekgVFZ85JvsOGuVqFdeLoAK1Vl5sXIOKw/002+tPaB6yHEezicigC2rDaCo13bpt/5sWTlmtF5ttNyol/aQuucBoWrziNFtda98KprTuaZ+iaf8mfywL7BEG+gAWygMexYG7q223Qk/gGJjaKSK9wbzVCi7OlVkSv2bCQ5K504+Q6kVsJ4n6pabl89oP7n+2eUzPzxc746DufP8AEP9LM9KJ42Q/rhNMielypO79dHC3/+05QM1q6io2Lwh5LcEbAyCgyOH0OpGNh5JrRLCHOCjLAl1ko/sO4kkMLbv5hyI23fc8Lt9Xoz1RT7XzpH0vuIl5+9/9y7YFUtET42TFDGUvt6JWMQmMXb/aEN2sZegN2qQi/zHYLAueXBj7VrIm/H2gbtFK1Esf4EKnXlnlDIddt5i62tehoMzBXcg+r7lQB5Q5fTJ83Lfp+EUrIP69i/lKxegAwoatorpq70qKioqKioqNn04NwMiFShwjn2NKGYOAFqJwYltwC9xqMaijoyQVgU7hMBMXJwz9TAkDzuVBBuJqeUU5HmMseO1LrA6kJ6uwhtyIhiIsqlliEIym9I9KCsuBL/4Z5Ou1pRqZX/2GBihNTK33AiUVa1NcOVeNp24DLDNS6lY6RnZA3FYUVzGhKwVdkCO7yCu70RONnZl6ToTSQiJri+tD5Pct4io8Z/Vi+E8oWl0XKBp5IqNUnW1rW8IvvHw3otHOkHV7TFtCBqjkO0y+QGdVNFzfL7nMQalLT0II73nUfsQMtZrjecyQVK2PpFrWn8Vnu5OLV9c9pEXT3XbUJUQIyEwZKrBMZhCseEtdWPl4mdmJ9diZ04sidQHAkZ+BGJC20Y4RLF3cQ528wJHhMCIgTAOeUwDpy7+7NSuhgFy0oeI+kxrW3hxY8tFk+j9RdNpTKUPGukGCNOGWd7K6Sx7pfz8k5nOp2FkGsRZWvlrwEYicz5ONtorx4Wcf80m6NWSqE0j4m7mTNh2Wq4PZTHILIjmgjpAyVTn3EYw50FZZ0KirGPRVwRGJGlzudDdCGhb4FDw3nJfSjK3qGv/nbhsb76CiuO6pDtbf/f1abm28kVmGwJo2S7f3yJ7uqaN2Ik59Ync29xn3X7qEzbdHtNLFtfIuRJBUxD++ZlQwr0Y8Haef+sT6C9aMWnTedi47LSy1kX/ondfphDo8uvC+oVqZaTfpIqKiooJjMdzaLxT0g+6RBIYj1vI3yIJajdsWI8Y1mNmdoRZWgbnRjKdGvMmK0TqE2cK9E6gW34PT/88DSWBPnTONN9r8wPs5h2+hlucTBvSL1MiKObO3xH5W6h/NdgGQRicCBAfbFXgoEuoDn22Ac5Q+4YCvCX5xtOU9hVlpUnzJZS7VEp8yRMhE+f3g/Hh40PPUz+9TJvPMzNvjARk70uL9/LzIb82Tga06RmhzjNelegVFRUVFRWbF8wuwjamBNCLBzLJ68gB3mLK7AdttoEm7gCAyAwEiWEJ0ZjcFNM455I9DAchw62ctOFkMcGfyXXldIgS6Q8wQmjFs71V8tKU5brxpqnVvTcP61JUo9RoCGi1DLNEsbqCAIqi2o5gNI1LcbF0V9T2AQGhiMezTU3fB70vfsiNNR6oiP3ynEMiGydZGqQ4uaNu19tibXbOwTcNnAOYQ+K5bLKAIWMs73yyzQGLf77VtW2D9HlkbXcAs/SJ2L/Y5qsA4FBaEJLGpGRcJIkinpl0haX0vXMOjJhsdjodIV0As69xLsexMYqHvwiuZOzoyaeOoBgAVZ1zLJ6VtCo6P28hBIQIhEg6OaOxdgTgGzTOyzMWZZNVI/Pb0GJurpU+08kEjgAFeX6iz1wakWzeGpnBrGPiNCZZODbfSDuXzFznPQKUGqXOj0XWYoCoWQhA0Dy+OC3l79HRuTiZLXIFWZ4e4M4vw+QgsF89kBKRnM/PhKbOy+gvTFrqYAQr5c/dkqFEdncSIDHeVFDamcEvWmftLUlSaaw8+uZTm/u8q+nXnxOBnpfI974rQLA9g3MrJsaN3KumNsAWeJflpbpT/vKn3vHU7qLbOhy4Hjfy3NmTDvRaSd06FdMUE6r+RGSg+xzal2q6RP5yZEBn4yZH+bZcKP/Q6whAfzl6z4X1CZcpnCaFhpCfx2577BnBwPGKiooKAJhbfwtmGpmpj4ERNZi57dbbQEQYjUbwrsHNt9yAcbgFW221DKB7YNnM1iDMgskD7CFKEICpBShqoAos9OWzELE7TMAX39PME3mLswtVSr/MyQBoKV+TQz6KVp8+wTq0kqysi3kYxhjBATk4Lsochv0dHTo+0L6l/h2Y+LtStMnI5CKOWSw9PnUCYignL/yM3B5MKs5pMH0JJRafNAi1iRV77JzGb2y57Jrd68pA6nY2sKKioqKiouL3Cm0bAFAiWG1jSVMXmyoYUCKau/FySXqX6Z4D2APG8JhdY4wMjmIHYq8k5+Pshy4bkE6ukusIBiEMNjMhtJw2LI0lMVpsANkl+qlXbibuRWkN7RcV6URorCQMWC7POB2X7EBIGa1cdlZSW//Y+yCJDqhin5TD4SyU4JILzOUZhwTIxEiMfXEGUp1M8d5jL5H4LN0sVMqQc7t2N5z6yzjJfh9n/wWnCnGxPRH/fVkBQFFib+ccmpHT/vXqwd8ik1STcTMg9zZwEL9yn21C82RMzkvkZOIDALcAyDafjRPlZ7pN26HtTkQ5ItpI8I7hXYNm1KDxHsHsY9TGJgQgiPMRuOiXSIRIQCT7nSKQeGkgcrcei8ESSPR808vHLaUQ0moTIJPrXZKzS7TqWENL4CLXZAvsJwdkjxyaLA8FaSvEb1bxSqLVNT/0adMzbYC3JlG5bRnZZN6UQd/ASIh1KS/0edTGZvJB2pBWN+ivvz23Vi+yAXpm/lOrS5UaWztSAcUN6fUjFV3MOhmSNpgbaNGEigvdohPFTXkATeX9sYEmu4lyhrgHm3RJ6vri+dMrWaHpeKkmFxLe/Hhzayb2CXBWhyESxiqTJzcGMcGTa716/dn/HekQ6/Zj5/7aSb3rQJ4JcQbofgFVVFRUlLj11v+Fc1vDe4+2zRvN3Hzz9Zjb0MJ7j5mZWdxyy424+bZf4x732BLbrbwftt4amBltjcYTGu8R09ctgzjAsYP8tRz+7pyu5O3m7ZPo853SLy8v9Jk8aejai/6etL9XC9TByP40NJiYEGC1xvGIUYl0jqDoUszTD+D7/tnyd3Ry0DGtLYtt49CmoP1yOsqd4t/5y50+oTCRdxHPyMaW0V+qO58SvT+Ymm/ixgaZZb4yf/+zbVzE+rAWuwCIYqzOgFdUVFRUVGxWaMfqm01OVeUSC3TJaFaXCerEmIDEvyEEcIzKmTg4JeFdQZSLV7US7sEIa10VV9THlNJlXF5u9GjXtA8x2vWhn0WJnvJzjvEz32SEMGAEtU0C5IiUU33FeqSspUv1KMcMSQSp9TOSubzORGw2EPPZZpW2crdUVXLqAO5wRiWJ3p0wUC9zpxY7xWQByKW6ElGx2rcBYH7cXGyqSql93TrbJqR5AobIi98+zI6GIBuYUloNkDkxL33KDhw9QggYj0nsU5SAT+McIlWmo2hvjqVlpa0Tj3ImtG2A+exbW33TAEFjYpfbYP3A3O1D48aYKd0LcjImHc2MMDMaoXEEig5tBGJohXAPHjGIcz3bOIw8ODYQR6MIFwlRInH51yYuXPeZnQ9LINFtlkQGdA7C5DNDb76wgwX111H0ovfZ3h2UcCfSRa+TZC2j8I/WGxZN+ZMKV1K5d275oRxrm++1g9mg9GqtmTMh3P2F0cc5XbE/MWC/80bkg3RRr5LzDigmo0rCVssxAr9cNpJ+4bRug6RDwdIW6QSdHCjrznlA54u6lAtVqDinRG+LMeV6pUdsc7T+EL2j2e/XuUOQqwI8lcBFCSV5PjDQZZsnsY3ekO5hp0+4qFH5fJSkdcrXRa4tDyVKuUMEDmSiIi0j6TxrQ7mLe2tkudWPSZ+RSqBXVFRMx/XXXY223RoEQggBo9EsmlGDDetvxS233ApRo89gw4b1uPGGX4B5K3BsEaPDVisitthyO8CPUvDInP/WU++7G5gkLw1d8nmSmOy+d49NI0qL2HISA+rtyb89lj6YOJieJ+aLIK/XjqGfSSN9jnkDy1KlDkwjb7t/63JDpmGgM4b+Rkznzyfr0A9C5jsH6MQQ85dfDMo2En31/EKYRnzPR6CXqq88oCkHaEOKdv09sGDPnlOSnyNHxNgusqcqKioqKioqNh04VdjK5otcbOAoSm4gBv0ZABUbtlnMEkMm251jwAMcSos5kk0alZQOSqIDppoWHwLvPZqmSeXa57SCsiCZwaQEaxB/7mCEelahE3pEKDJhauQ2c0zXMyW+Iwb34n+TqeQ4rIwdC+5H7WOycnt4/NGP9fI7clzeYXk4XyP/NFl2EdOXY5lsrdK9d3lCQXkzUJqcYA4girIRaePShqtGbNskAel9NNV+CLEYV1jMKvkCxaQIt/yiHPcgyIrj8Vg2JvXs0AagnIAo75VZCTlHGCWLSu1LJcLtGbLnNoDgwYCuuJBNRqXO4jRf9Cnyc2/30rzWm6YRz3inFj8xKuPnIdY2NhGlnvx2f9LqDE79bHImZvHzN2t18ovj1hZNovtiQGlUt2OSX/zU6DQ+KH4JujRzR6XUIyu5V2FXlFNukupZN740n59ESlpZmW1Owx0yIjnPwOSvmKISRV7q/vp0SPtEZMPcmPpjeSrK0Zc+AEz6oDnqXL8Y0ukDln9x+0Svzbp1py0mP9qP+hXUqx8X/aN3le3BmSRHrN1WVqrCUB6arE/xSPTIDOocH/op1WeiSuXKA/uyUPKcOT2Pdr86fHqfjF/gFybdX/un8+DnL8z5zi/nWidufPFM5eZnQsoy2+VSvgW8bSsqKjZf/O8v/ws33bACRMDMaAbb3OMemJ2ZxYa1t2DdbTfBOYdWgx3Em7Bh3RjXzs3BscdoO4+ttthCgnCiFFgkVcBGTOBNqrVz+jQ1+oLBzOB34NA5wyrpyeJtFdoAEZ8UCotTK5D9vYeuTHOEqMtrTT0yZBFT1qXflqX2+sQKLECXuC79/i2ESSnEPHmntnmYEF9M3mmf50sry57vWesPxCxMG5yjYKQYa6gObTvG3Nzc1GtVVFRUVFRUbJoQ0k82jAQT5toWDFHuEiip0QFCMzPqktmF+KJUAsfIcEoWkjLLAZlOEksXITaNTPU9b/XRaARAVObQPCJeNfsN8c2WzRtN8c6pbONqyvg2Ri7I5F5My5yEsc5Ln8QIJWkdnM+e4ka2W/wNQCcJzEPdJ7F4GqIMxmcDNi5Kz4QYQRyUeHRwunlqyRcl4t+p/U4iZvMERVbSS9tNVS6K/YjRTIMYWfu51b4K4NimtsvER0DTeIDzCgLAqKdMaFu9Sh9zqafXmre66aiQ3t67pBwHO8QA3Zh0Odowho8OIXjEGLRdUgoRwTemfhd+eFC0RHnVplncOHKIsU3HpY6xuB9Gnst/MTLMwsEYU+esnkbGy6agzNIuc9WIUVZ6RI7gwGgDST8WRtRGtEsfN9pGu4cLj2QWT6KDECDqc3v+TY3Mvd+HrJ7NiTYcpeJTyR3ao1lWmcj8ust0RtmudE5JOhuRilzPXA+tharoQfaLYwMo/YJi6ijKrZ6lMMsmBDzyzIuld86xtEQms3j0ZFkSumQwcltSe6yuxRKTYvarvN7AozzxYyJhLbMNBNO1KH8J6TkeBIfupAmK/uimSYHD9ekS/+X983pGhxhPN6loQmcwK/9Qysadi2bCo/y5PJg9tkxZJndH7hmX/WAzlp1y5iFSCLlvkevZ6QP9eWLwTgNL0PUelH1fUVFRMQ3X/u9/YzTTYNnsLLbaeivM+jmEmRHW3nYL1t16E7wjjEYNnPdosB5z61ps2HATRmiw5fLlGG+zFUJYD+YGrhlhNDtKwTEwTD4uhpD8XWPo0m7Kl+ZkXrNRG4JFJJPT8JPIy0tto20wI4QI5/OmrHdFPy2FvLZ4YJElY7GZN6bd/XOmqcinEejDKxqG78GQL6iUkweu872LDV8swiwLnj1aEFiXu1ZUVFRUVFRsPijV2xIrqrqcYockdy4T6maPAQDeN/BeNp2U40J08pTY1BS6HKGbQQIEh9IOJOftxlU5TiH12I6IISRSV4Q1DKBrB2NCkzItq8TzhIBzDoEZjhhQhbOkZ9EKazxtthtSGCUSVggbaZMeFP/vgfHIUMwI7T3Xie+RSN0ORKmp98epNR9UfZ1JYSm/nHwQUpyIMDc3BlRtbe8cxZ7HN0DjGb4xsUZmF+V+zYAbApj0Ocj7LIWgG4Ky+Hw7UluYhnSCIshGtUkIpBMT7EBkm40SQnRoW0IIlJ6tdC9dvrfe5b4o+9q5ru2h002AxKLIVmPa5qVl1wpH6x2BRwCCrk6I9sQEMFow1A7JARx0IgMBoAjnZDKJOcI7gvME7wGQWR+V/LSo2OV3wJ6jPHabD4sm0WX7A4bj/FkaWwxMjQ+cgsStF+QpIfW9SPXJNpLMx11SjxWNshuV1NrCVJpLa4+S7lVMH0RSupS1bcoilwSlqZrRPR2A+QShZFjz8LFPgqfJOft60/4rv+wYOeNAv6ViKfc9W330Go5sk9D8hWfnR20zWQ9MPCDD1y5V0LlUnuiX8v72l7l3+mjiRJrIyHrdQdVgJ2ve0MtpWmmlA+oSJ5OzksWGt8U9sy/tVFI6ZBYxC4DyPbD8BJ2EAoovae4Wlu5P8WXUmYDprq4AMNmfFRUVFYr73Ws73HTjjdhw6y1o0KKJc5hdNsLc3Hq0a29G9ARuCU0zgosOIcxhvG4D1t+2ArfetBVuWr4McMvRjhvMzG6Jrbe5J7bYchv53qGshCnf5/t8e7CUcgbJUmD4b8pQ3r46oAR3Yuhc+AL1ERu8/PcmeUHeJSQ6sIi/ZAWWQvYuJe/QYG/4/CH1EDBJaFvaUL6lYCi/XCdb89l79zwLAbk4lvM4R/DeZRVaRUVFRUVFxWaDECLaVkhNwGLCzK3EyKq2dQhtRIghEZdGpHc9xi2WLP0RHEyyGQu7FSNEs8VFxpAgQI8oC6REaD5DuSIH2xxVSNRMuIoS2yXRo7Qzx2pGnIJMwe5hbgGSL1vcMFj3RBQCP3nHs6m0J1e1Tovlyncjj9haZGJSU0Z3ab0OdyWkNCeLnBhDivGtv7K9SndD2O5KAmHwYgS4kSv5xqlNi9qN6CQDdPIi27lIOW0bhaTWiRXvZfNPmYDQjVsJMImxKcRlZYTxZA4U+pMdWZ1tzwHA8L4BqIzNc0dl/33WdunmsTEghryZ7NBYxDcERHXvCBEgUZXDtRAiPSTOzHl5xkNsAYwBGoOohfcAOQffkH4OIFlSXdxJl1cwgEHIbV4IiybRHRedCC72vs0kc/+S/aFRf74n/+LpNfQsp8eikpd50OuUUCyUypTckjpkJUF59cTs959+qOoZ+V/qHE71pOI8NrKeKBHYpYI5nd9/HmxQ1btGx1W94GwTkVpe1/qqIHEJsA2E06CYSKx2yseAkY+nNnVmYrp5HTLhDphbESVCd6B5KO9np8upfF77xIoR1KrcKhvdeWLKyQzutKnzjJUdk2pU1ME6hfI1h8C9D+lZXmgMnuoz+bSDbNImk+cT3ULIX2LFNcs/YWWVK4FeUVExH/Z+6CPw7//+7/if//lvrI+3oeGIONdgw4Z1CON1ok6ITnzQeQSOQBivxfp1N+CGG36hQckKxLgM29zjfli+bGvQCg/yas82EKTeHQj0ofxukEKfQrgW1m8lcsCN9N28ULhVli9LLzktIbS0O/+7fPr1htXoiy918RQ6yaZZ/b+XS1RnlwORyYHfwliKaj0fy3+rTZmeFUM5gjNBQU6VEW8MLWJol9TOioqKioqKit9/5M04s03L5Go6dU6gkGISI9CjxpBlHOm9gyhxRaHslIxmZgQE6K59iWwWojVMiCJLMYKdL/+Zgl33rvNK+MJUyqboBYxULgWaokKOIBVAlpMBmechdFe7GhEdEZQoT7wjZ5JWElxRViZ9+v065Iseooh4nZZjIt2Y2j5ZRhY75v0EpR+yHzn3rpNWHMTu2CCq8bk5dkSn9ykReKx9Q4k0N9V9tnEhtdgRxbttLkvE8E0xLiKgbaGTHh6u4wFuvNx86LbH+DSrh3mTh7RaQWd6QkBsgz63RqCXYzZKzwGDQS7Cse4jqXtFOscgCmCMRYgC2SCVAVBowTQGYw6gFr4ZwXmG81EndUSJztEDZMQ5ADiEVlXsPgIuW4bPh8VvLFpurAn55YnFz5nE7XCnMvOgQwcbwDJsx1gjyYtyCkzYeNpmm0U5BCHDE2lLSEfzs6KPdiLccxu6yIxpIpm1fh66IKNoY0mqlqquju97kTcNtooeSrk6xGiR1PliKYhuznmp+JDuRVlkWXb6GixIDv3HFNsEs+rJX1aeemWmnN3JCYJuAltkpvLh6FQpsxCl/U63pSjaL09T/iIp+qJgo9OlObeN8lKA1G9LIhEInWe1j27zygqxqg+1HWDdNLXIq8fK50R+N6isstwTaxyj82U45N1bUVFR0a5nzK0do90QQDFiPa3F3HpGG+YAbuFpBvAOxKQ7swe0G+awvrkVN7sR5loCeAVGM/eQTUYBANT5m/+7JtAzJssbukL3bxVNSc+pHWp98G9QTutPaPM8uftXt+Dae68bMpmPo0ZH/fgCAPp/ixYNSvV0NBFNycBgomjO4oPhRiwIxnDwudRWLNYTfdozJislbGBVfp52vf75QPl3vDiC1E8DhP0Emd85S6MY8yUFdZbhVlRUVFRUVGweMHuP8ueszBYbjbZt4bwTaw5xxCg2zxSyHECOJSmK57VTxS1TileIXCdmiSzKZGbu+Grbtbt5zac6W2Kklfwa4Djn4VwDU48TESIHWMwkUZBsimkcTuI4nFNeTojdopeQleCyiWRW4MdEHpdyBSPx5f9JwnwwPUbhKxkgRCmPpXEpArQYvackNyV227awcYKp9UuRbUmgd218Mv8mMWIL5qi8ocdo5GACarvPzgn7GQpFt5Qbk+VOCAzvgRCA8Vi88MUeCGhb6ysPRxGECNuHk2ETO+VkSvnkUrq/bdt2bHPEo97uD6dnPIYIhBaxVY/1BUhqaatwX40jOBZfEkcAOYb5oGsv6vPZAhTgvFpQq4WLtMPBmOXIpNbnk4IrjozQhkUNvRZNonfcrfuELdsMgT0A2rl6hv5aap+r3w5ycZ0e61+1GKSaUtkXx+U62dKjJHatNA/zt1byVcsCG9FeDHMKstg80Y3wlXJIqVz9hSjGZmW9rXlmF2PlyGW7AzOb1euDoZMVOgvVGdtnyyd5yNg+5y+PSRIh/zJ0Wk2Fz3ZnCkQe1EHioZh9A/T6KG5BzjIBuY+cnpD8zEy0vviRi442xVf+JaOyTgOD4VKRPlgn5FnEMhsNlNcvO3fvxOwBAOhXcfGcDg3I0VO9FQS6rdBIX+MlgQ5UAr2iomIq/uVfvodbbr0R5BqwC1g3Xgeem0NkXerWElwzCwoec+sj1m0IGI+B9ub1iLweERvgfCPL4ByJcp2iKCGoa58xn4L49hLq5QqdTrno/i1LEUARrgAWPnHHpoXtbyX7XBjkjwoTg2nSasN5PZezx7msiBv+K570K2lSXvZRmXEO49gizI1BDIxGIxAcTJjNzizKQvrrRJ2Se3/OtNFmgyYb7FCKHaxPzMqNBv9yEEjXnZXlWy8s5i8NTcof5NwYJwroD0zSjzQZ2ALTPe01ktK4DrqzkHkkIv0RL+OEdO9gsdlk2Zw6rkvq50GTnGyDuayGKgn7qI+U3Ie2DWhDC1DEsuUzU56bioqKioqKik0XakDMmbCzuNIpScYAOAZQsusQAtRUzmILJwSQ+IebD7jwN4nIVfJa4iSzXHEAI5HBIRRELyI4CtOVLW91k0b1VU+tSCGfiBsdRZDZfnAExQjbLLS7WSOlGIqc7Hco4wiJ47JqGxo/OzA7IUDhRO2shDqBBjhZB6moy6IRzjJS1uCNNdCLTHBMACd9N4hz/zLleF4U+eYrLv2IGNTGBUqMJoYSDKdtEpJZJkJC8sUHOFnCEEWxm07kIMM3I4ABpx723stL7xQQlHMsJjpCjIgcQcwI0cMF8Tf3vuCPSAhjRy2cy6R/npworKvBqW2SJh7iMQQgZots6ORAQaBa5fTZYd2A1uqgfZ34Md2Elhw8Ad7rignbiZPleQkU0viP1bddLI8YTePgvQhYWWNwRkRiOznIs2GTNmA0I3n2gk5EhCl7IpVYghI9D7qsnWnwSsWAy8hezudYZ8qXRHl8kuZNPVpIrckGfWyK6JLslMGMjXVKEtche093bD1g5Gv2d5JLKdmpbXD5Molitu+K0gqm/MVNDz2XA8TS/iY3O43PtB5C2JLedP1S6BD9AxtKUlYod78/yie3vPDQsDlRCkVacV1C735MDuCt79KD0atkeWVmTqsAHPr17hL9aQZviDzR58s2nwVgm/h2BsZJfU9DPSRkiS2hsT5OPEtxH9PZVNAPNkOCiRuQmmP3hvRLkQbydiY27Pkga0/xwBSTKZOTJRUVFRVdXHPNNQAFLFsuSywjy5K1GFsgOpDzaJoGbYjYsGGMDRtahCi7n7dtxPoNG7Bs2XI47+BnPHwDkI8ddQUwnfgsjy/+2OK/1fLfr4IMLQnP8lOH5M+By8R3vFQK/e/1fLp8sadg2QKUiRoBEsjb3wk57pwTkjwtUzVls3zfE3KQ6YpS++3uJzjNbUtk87o9+3PGVroMtgb++pZ9Wab1QjbtjyFafYLe1/J4YpDTj586bRt6Xnjy753cgjLOma8W5d/h9MOUxy0f66rOU1U6k/kpiipV6Jo+sXmVKnOYGaOZ0dDFKyoqKioqKjZReD+DpvFgbtQjXG1TOIhlhbO4MsJRA4AmuB5RNQeYD7r3HrY5ZFd5nb2nzUqFmUGqVo8FcS3nQFS+nfiGlTgGsk+5EaoiMHHEgJKbIOPEzHJFVOQEU9onzwM4b6puSD2ikC8xSCRHyLY0UUnoRMCnTsl8noFcI3VQb/KohHjU8YsR6GACbJNUCJlK2laOEZEiIgUwouxt5yy2JhAayCaeTt6jrSwQEttI99yvep3oVC2ux5xOQqiHt02AuMbLJIpakUTEJM4URTrBNR5e+68NcwgcETkoxS7t5dggtEBwBGq8ikQcYmS07RjOq2CHkSyGTNXObGIQuTdESNdmynVlzqLewOLhj8igGOEYYDjh3JToz/Fy19ZF7qlL8Xa0NrCMY0BARMSYWVhAfWZMOCMkfI7J08oJp0IhVhtN5+Cc2Bk1jUeMBAqENpAq6ufHEpToEAIvqc76JCqQZDudZzix6d3BWWKS5RwHJMV1B/obTJ3zy6UVmZTMQ1dST28uc2ZyEvkm2DlWhimdSv8daTdSut6WtHVDNhjpXqQcvJfDanvYpB65BjDimmXzsbKfxB++HOwp0Q/Mq5ROw1/KA2FLFQU6JvYyLfvalmYPEtCJ0C36uRgRU++Cnf5J5U9am1hvya1ndE5LxI3cF1fcl7J7SqseLgooq2Qb2OYtDXrEC/fa0Gk77Le5+PLOLUqTIqRTC5T7vOy/zpNTqOWovE754JaVK34c1v5VVFRs7lixfBajWYeZWcK4vRXcAn7UYNmyGTSjBsuWLQc5IIxbuMZjuWswbgPaKDPy5ByWrViOrbbeEitWzIIoYBzWwVGTdoVfSGW+GAL9d+EJ3i+bjRhnAPZXqxPI9P9CUfdvGmxJqf31tmMx/wHq/L2wiIGKv4USyLmG4JjBbUhKF9OsyAo3ndx1i+8XR26wP4f8wtP8b7e1/Q4oQ44J9OZRctrQZL2bptUfLnuxrR74E63p8z9z3RUU8+frv087NpQ+dE1bog3ICoSKioqKioqKzQfezwCIcG6kolgLQKOS6GLLAgAcSskhp3zMAey8cA4O8GrnYlwKA6ow9nAupviVo1yrjS2E1LaSCysSfcnKxoLcBNIxQ8deloBsG2LkuFhv2Map2ZLEZccDEouSGNWawzb6tDqzWKDEEFMdpH+UxUsKaauPnV9ySkiEfurLBCOCta46h2H6c2ZGpKhq8ogAhiOPiJDrljbKjAACIgcYjW6rFUv7FpBOlug98N6pitrpZAol0QWxso1jU/brhqLs0vXbVjarDYFTHGobvgKuE3s2JCS+ENoEV9qwqEJelPUxaYrknhc2lKmfKfV9UvirEIi0v/rPTOr1In62MokIzufnLaZ6McZR7pPUneT3BS7ztM5WCTjdS8DDJliE82TYpqTOidKdyOm9EL17Q0KoL4TFK9GBPLgzdpG7R+Xf7pKKiSqkgZ2Sh/pUmzLI68BuiLSlXjH2S1dWxL5mnI0GC8LRiszDWr1pBWPavYYu7UhFddvdd7IcHjwWpGmR1vmZY1FXuU5TlqRkMVN2NC+vycVERB7YD48KTWWfiNfMRxc1y8z30KAalD3TJy/QI38TMSyfhWeQi2biNw9Ki05Jv7T9ewJtM5UcNvIzkZvAuW3G8cMmSLKPkqnQS/V5uqbxIqVqzy6c2lf0A5f10MmXwuqo7AtKFyr6KLXD7jJ1+4C40x/mrz7/REpFRcXmCuJWlrq1QpQDAFg2x162bBmapkGIjGY0AqMB8wgRG7DutrVgahEDy+aiLqING7B2/c0YhzEIM1i+fEssX75clNU89D2OlHZnEul9Ajm/DxDnBdLXug5AqPdXXrjubjyREXo/29LD/M0vgbzavzgP50Vd4Vh2/2HIMkYHB+PdafCP8PR2D9qSFMFrDmKHy1zKPaCBgDjHZYvDNFvEQZX7kh6PnjqgV3aXEB8ufIhALweR0451qkyUNk8qyzUPUufqFHhFRUVFRcXmBPM1FwKyfHlRJHsl0onAzkMsTIwIjwBFOBolktc4BWd2LkSJrI2RQWiTFQyrglf8yqUOAGAS70xUayTFatuinEUitsEptgab57TEXqwKwpIEd0aQqne6dy4R4VKOWqMgGu+ZRA62aWVkURIzAIpmJ0OpHiUBy1FjbCPBIytpbzwO5/qauh5ObFAK3kesHdUikLKpizqeKIktyvMYxa5PPMYjxE6kOzEBoHfPZcNQ70hJXYtLORHdRLKRrA8OoXWaT/qMI6FtGeNxi/FcQFBbGctjr9SHUSYKZEWkXDOr8+W+hWB+/QzvTf0fi3rZhrYdxhLErNRYVwDNxusVYxxTo9tz1GU489iNzDtb+yW3QZ4L77x6puuEhJML+0QAWt/LfSGIFQ0oAqQTOcVmtzLhs/CAYwl2LlIRcReaJFfLn9OApD8IIvTSpNOcnmTELhdZ8+AmM5uEPMi1FCM6XXG8W7niXAhx3s2XS+ts/KgVYTA8UXLuKJtBOSOSJyd0kJWId2Or869scVbqt0TkoiBGbfCbiPLi2vaAcPkIGylrDeTUPiNnqdOn5YLvVHAnhdKrrPFE9pSZtL6pRiyzI7afsH5lpZONWC6KSC1Jn5WUJlDZ2NwHdqZ+ERQTkZ2mlctqqDhn6NclTS70J43IapcrmmdAzQcdgP4xy9ZARV+XPq0FWZ5eZadSzldaDqX+WRK5UFFRsblgZiSBcjtuMZ6bwzhswCw8ZmdXgMh3J2UZmBsHjNsIcg1CYNx6221wo+WYXXEz3HW/wU233gznZjEz2gr3vMf2cG47LF++vHPNaWTifLijlehDRDpPXIP1u7/cJp2R/tDnL96c3z7xIuqcGHuLbiQxcoR3Dt57UcREIIaggaJHo38wIg3VeYFLTpFV91Ug3PkLnUGLnJFl6CZX/fNTexeJwaxDBPj0dg130XAdppPeNG+++Qj0hcq3SaZy8DQajTAajerGohUVFRUVFZsZZGNIcU1g9fk2S5P/n703D7TlKMu9f1XVw1prT2dMchISEgKiokwBxE8DCUMIMxcMhkHGQJBZBmU0cBFyEWSQiAreS4AEkEkBMYBAvAphkAhXEIUASSDjOWefPa6hh6r6/qihu9feZ0hAmfabrLP26qG6urq6+63nfep5lRIoFSRdwAoVGdAG48HpKKIbcSprDajAynYMWz/RESV9hkDrAU3cbEig49uAJQ1yIrbRv3Ya5EG/3WcG9DIfUiqUEi0msfd3hI15iIRo408qnoOwAqkyQnFY68cmbeDZgerWTLOZG/ws+ogdt66Z3QmuPaUBIb2cc0gaar3iS1sbuI3HeV+8wV0Cut/SSQ8UbiygfW0MFh3BPSnb4wbh9euV17MXSBGSqwbpHMccN8YD0tIxxJV0uvMh75HRAq1B15qqqmn01EEI4/uZiNcmsNaVwvUjD3g7LfzAQnfn50BrYh0RouXPWhc0gCinIozrW8rPNHDf/nwjRqfQWruEo6aVT6jdlsbNrrUiSLn4vhA0y3XYT4B1MwMCdhbONVWySXoqXAdzIL+fYRFxOx+okI0cZpAuOpTdNDkXxEZNchE6YBdQRECcMkwLZBXdzXxahU2ORZzGMi3JEoZoyiOfDdzbAJFtkDPctKIFsk/jxW2N8+6IzE1n0EZjhCVNUkw8Jz9Awms6tZFxpqBm307Chnu0gZlbgZLm+JuMIdvM8dgk4UZpMFkgJE8N+8l4k4dr0sUGmgUNwN20fSeo0L72039MXdtY7saugWw97ERrjxBIiE0Zn1gdyLgVXOlWQmw4uaaxAhDeBLRagEJov7h7q5EPPs6OJxXayHr9eymaYElIULspez8GK0KgIwDkGw/U5BVuB7HCy2mTem3Zlm3ZlmkNQiMsJDJBa8FkUmEYow1kvRypEixOF7HSBmMladrHCEVVGxaXVqi0ZHVtQpb3yHsDBr0dKJEzNzdPr9drTe3baP9VAHpwmGKguTUgmGbGmzDrDbxT76de6gqjnRMopSBRCikUSmXO0Y5OPEB7plcTEBXBu/W/rX/vGc/qaZggeCKCK0sKS5oodGXROKdV2qbM4LscJol9tHayns7yNnhuN3mvTW3bZq0EE20/IDR5dPbDNr6+R4bDN+7MdF2siGD+tGs5bdP1nF632bm1fx/MDiflMl3GwfrwNJgeLE1T8jzfYqJv2ZZt2ZZt2Zb9nFmSJB4Y9zrfxnl8MmpjC8+QdX6rMQIrjUtg6RUMOv6RB5t13TBqnbX9NA+MEtjPfovIOrQtfydsH9a3weEGRLXWIKUkSRQqkXHbNgAeJFRCUlMHyEr/7QgZ2miMDrrpnimM03u3nkEfZ/WJpt7Trpc1Tgl8Wt0gtBGiDZQ2y612lHIBjsnsDyWFy1UYsBnhgSRB2z+1EYRz18u3V9jIs5yVkhGMDu2slHLgbUj4aptxh9My1w2gbcAIiZEB77IxOamf0BrHK2G91kTwOUmcznhdu+VJIp2WvQyBj8Dudkida6dwHB9QAQ+suzrquuq0vxQxfadvMxvHh7KlbR+WNbrr3XEcLSwQf05BEsjohk0vpUCbZqylJD4oIFBCkqiENE1IUoUUAmtqtK58QMbfSz4IJVszAY4gr+hNANFFM9iJ3dL6wahoQYOtgZaiJavShVXj392DdLHSFj4cwcgASLZ2ifWbZjKHAagMEhi+8EbGpQXI2jZ23tx8xmokBYkokLZGaUUtUiqrsDJDCt8h/M3SCv7E8kLigqBB6tR5bOwgYSzabhcjnEZ8aL4pAn8L6G1MTrVLAKfjtQvnHh8+8Ww7IHkEJgjAfQt8mD5uBHL93xsyqLrjBR2iBn8QLaS8KWJ6MB77nG0els21bB3bgxZBkXxDY7XKbjdm5/nHFAjf7qxTVW4utGjaPT5AXYFxtkOs4XS5LS15EaCXqavakoKJsgQx2tIca8u2bMu2bDNLE4WUKUJZaqMcK70cUZeGOrOYcQWiQiUZk8KgTeocXZnQywbk/VmSXp9aQ1VrVGZdkhsgvCunP5vZzQXYD7WPe/zZDccNTto0aCpwLPBa1xSTCeurKwzXV6nKgkQq8l7GTH/AoD8g682QJDlSKaSQkUhudAhI267fEI4VyQNuaqYb6FiE0H7A5AYFISW2ksolDsJGZkYtJYlPJHWkAHqwQwHD7baL/lZn3+7+cVvb9QuEEC5hUjzndvlhSuWGI296PMPGmW3h3RoGApsQjzY/p6myD7bd5tIr3d+Hk2s5mHzLZvuGgUJYprUmSY4sn8CWbdmWbdmWbdmW/WyZUgJjwHgg0CJbvmwAU2QDpraARId5ND6VtR4o8uxla60DFiPoELANx3aO/k4D3jU/bXtd27cHCD6LByCNxQon8WGNwGgHFDtX2ONiWAdsQgTQpSedCOlkSQCErn1CTC9ZYx2OgpAYEeQ2lMdiPDMdB9A3sjCuzjawMTuSwg143viCEPTRpXSUXmEFwjRtbMO18Bhe208WIgDUDqTWLSKLY3VbrNcQV0qhlIztE4DkJPFs6ajV0NRfa01l3XjBGONkWoyOMBU0skCOOR307p0cTWDzKylBOna7xfp6CGptUUi/v275rImvp8cvw3lag9XWt7s7v7KqER5wl551bnEEZPd3w/iWnmEagwcCqqpCCOkIy7oFpvu+62SHnAa60QZtG838AMAb68YeLj+iJI3BGoWSijRJG5/baoxRVHVBXRs3AxgX1An3ihAClRzeNz9iEL0Br4Ou9dSkXSE2G9vErhQ3i3+0NJ/b61rAZXtskcTFomExdwDMdnLFFnDrf8eyRCMb0963SVDasHy11khT0rND5uwqPTnBGk0t+yzabRRWIaRy021a4G8D5nrwP4CvEcSNY8SI+0+DubK9H2A8i97dwG2wwC8M0cLW8g6nXPgHQxsI9qsahnN7cWsEGgD1UGmPHLiftn2UptHj86sBzqVooqaxqNZBxdQK4QEKx+JuXgCxFh1QvltdO1VuwyJsji86AY/pEXfrwnQxgub847HDi4LYJuEaTp9mAN+nz18gOtegXXchnDJvZ59Ypy0Afcu2bMsObrY2aGGgtlRWo2swlUAbgxWlY1xIgUoMaTbPoDdAqhxtJVk2Q39mgbQ3YFxUpL0eO3fuYvvOneRqlvnZnfR6vSbo2tJGPxL7oQD04AR7cLJtm2lQ67pmXI6YTMasr6+zfGCR5aUDrK+uYE1Nv5fTzzP6vZzZuXlmZrfT68/R6/XJ85wkUd6xSlCeQdO8gCAMLgJz3GnvNcd3AyGDxGWCz5PMM0Q0SeKT/ljnDNba+w5SbI4GH8yOkJENEDQdu9tBliWxvmHfZtBg42GEP0cpVQSJwzaba6JvrNuRBYGDK22nX8SH2e3gG2/GLt9sm02L7bTL5mVMg+6hPwYAPYDozsnfYqJv2ZZt2ZZt2Zb9PFlg/wIxNZ5Dbkxk/TpYz7T8GedjBZ9M6zZR0G4AjyH4WYKQENJ4KGUz8ss0U9vVs5llalp+rUG4pKVRwsQBIdZIr0ndwqVEi/3uAU4iltjMsReiRljpiBvCegKLpa3hIJ0GiQdSDYEJL5UnUYr2TEan6d0CH2kwLPe38SxtY7U7jBVecl54drd21wSL0Y4bbz0eZW3wj4Nf7NAfY3wOQ6+tLXBAO6oJlCglUImM7PColmBEZHkLz6g2OkiwdP3JwMZ2cQDjgerElScbiWyNiQGWFiCKEMop68TgjIU4E0B6X9bE+lmLV+cwGO2DKE5W3JcXyK7+bxnw10DobfTPw0wIlwiUOG5wyUy7eZxCoELHpLPWS7por5MvkApS5ZOshvuggzELr+0vkSqN94PWNdaaqP/uysb14cPYEYPoiQgdx02xkDFC0NhBhiKtrzaYHiDAFvu21ckDuSkAqQ1gaON6aK9zlx3RlE0EkkPHdkB0k4SAuE60InrhbLStkHrMwI6Y0QfoyWWEMlTiKNaZx3ik1IoWON2motsAcvs6+WeKD7C5Y/vp4i56YuNzMkjENLJMstVizbmHc47Ac2zBFtAd26MBcIUvoA32+jBhcxnayH88kRZvPZYnmvO2zbO7A1KHmrTbuA0it1ZEHF6GB6xt7ei55rapkwgnJLrtAwJhTadf2vamMXDSWrFpJ26V2QHVRYy0Cv/3psD51ImL1j8+9kwA3u30vgJE0wla7RTaZPre2bIt27Ita2x1edU5HRi0sNRWYwRIoahLXDZvaalNjUxhJu+R9WaoKomxirKyjIoRtYG5dIBSPdJ0wNzMAnOzcxEMnAa3p+2/iokulXI6kR5MD5/ggNV1zXg8ZjQasrJ2gOXlAyzu38/ayjKmrslSxc4dCwx6PSSGuhyztlQwHE5I8xXyvMfM7CwzgwFplpKlGUop0tQxG6RSiPjWCgwSA9ZEJ0zrivFkRFVVYC25SjBl6nTR0zzKvkipnKNsLEZAIptpkT+Mbd7GFjZZbjc4qAZrDFVdY4xxwXDcVOQ0MDvcju56CNkJ9B/abPRsNltn41/Q9SWPwA7xYtycXb55Wxwp2/xQy8J3VVXUdd1i7MgtEH3LtmzLtmzLtuznzIytPZMWjPba5R72jJIcUkSQuC1p0gC3DXDhVSmaRJQxwWhw9QIgKjiU/CJ0yQLNt5PHCGQJiQBpI4EizuRHYK1swFNXQvxXtNiW1oiY98da5QGywLR2gJKkPftVoKSTwZGece/aRxAI9sH1E76NXf2J+7cDCKFNQFDL2oHorY81Fq0V2NqB2xKEL7OFJkW/30nJeGDXa4QjnYyIY0UnUdIlSIeE3w5Bt0Ft3gPWrr3quqKuNXWtMdqv99tYYzp1UUo739JfN2MclihkI4XjrpU7rjGixTr3ZCAhG9wxYE2iNT7QxtXFWJ9Y1IHuUiqkCkRgd34yaOXLpj+0gfUmaamrj5Oaafdxg/EyLvj7wli3ja4dO9+EnItCkEp33UxLM92lMg2a8y6QoFRKGLPpOoD3gYDlpHMOZ0cMorduDTZHHRtYvH1fhkyr03uF8YqkmcAQIXLf+WXYTjRHcOtdmUqES9wClgmdI/ztgdZQbigrJF3w5ySFwAobL5i2GmNKjK6o9YSiWMfIJdJcUst5hCyRytL0Bl8HISNCHv9tNUjQJjdToLAQ+AhY01BBJsa1hY1NrURzRVxCtBZAG/DsDljdQLPT48UQmQkBG0Fr/9aFEoKulE+spojnK/0F7G4jujs0DdG6OZt6ti53/A767g37LQDnrfPugOetg2wyMA5qVc2qdqUFUXdnetd2kME/4Dcbf4sNf7R+RuBfNOfWgf2bPitbZTSR5G6UYmo+yJZt2ZZtWcfqsnbcAuU0C4VMqEyN0ZCkKWmWojKJUIq81yPNM9I0ozZQjjX1ZIRKB2T9GWYG25xWunaOTl3XHVbtwZi9/yUAeth3CjyX0k17rKqK8dixzhcXF1leXmL/4vWUxQhrNTO9HjuOOopdO7ezc9s8eZqC1RSTMWVZMixgXFesDwvWh6tO71ElDAYzJImi18sZDAb0egOyNEep3A9M2gk3NS4ZU0WawczsgPmZAdsGM9SlZlIZtBGsjwtqbdzMNpy+oTZuLJErzx65CW1zZMudzzNtITlVURSsra2xurrCZDxhPB5RlpV3tiWzszPs2LGTbdsWGAxm4iyEwMbfUO6mjO9p+bXuuq5D5P482NabnfPBSPwbWeSbt1t7ZkW7bx9KK30zKRdr3b0ymUyo65osy8iy7LAD2S3bsi3bsi3bsi372TNjKk9WCLrVAqzzX50+s5PGMEZ7P0I0ILpxLGfHFnZJOgN7OTCVHRPYRuwryKY0eFXDFG7z84AWmOhXikBgVJ6IGljgASRvmMMuCaYD2R2O1Wa6O7BU4AFi4UDcIElijDtGBN9jsrqAt0hCIkxiDX0AwpOsA45ohUWgIraF900DQ1oKx5gXoRShHaNbA9pGED1RjnykTY20tcMHrY4McAd0e+a81Bgj0YRZhhaLBmGw1tfdSJDCX1/hpWg8jmhqLHUE4etWolAHmIuo1e2udWvWqD83t9pEhne4qE5XXSK0QftrJmWYaWBoJ6MNPnEI1DjJniBH6OqltSaw76VQkRSipAPllQCpPJDuk3sKQwyIBB85SZIYiDHGoJRLolsbi/VMe210UxdrYv7W0C4GC8ZSa+3bQ0X1C+thM2klysviGKwvS7qkrCbgcir2/YONH9p2kxKLtqFgt7AN/nVZQ23Mj9bvtuRK+3a2ISKBiAkBuqC7jXi0O80wAaQNmHcwbdrTChrQmOaZ4BfHJJxWUFnNuBxiyyGZKbHVmLJaIrcjhJwAPWxusVRUdYEUCVniwn1R51y0DtZg33GdnV7uB4gtOR7HcLdN3YVty6a0r6zY8NUAu83gs2HktzYOR/e4cfvh07l+/keIHLmd/LnGAMUmciTtsek0yDw1TaINEof2CQy1JvDR/Z5qwU2ts9y3YzzPqYhO+DMEFAJqH/tPq8qyVZcAZW/GJ+vcM6J5yLl1dsPGgdEO3XslnnfnWgQzWzj6lm3Zlm1qx59wPFImqERR64rV8YjV0Tql0aR5n94gJ8sUVgiyvE+iMpIkZy7tkfckRSUxJOw+6hiO2bOHmfk5siQlTdKo7xfAY61rL+8xXYuDM3pvFogYXlLWPX+ldM/puqoYFkMm4zFra2ssLy2xvHyA5aVlhsM1hKzZtm2WXTt3smvHNubmZpkZ9MkT6ZkTFuYG1NowKi2jylIWJZPJhKIoKMsRWhdgQSlFnuf0+336/Tl6+RxpmpCmyiWwkW6QpE2FtTWzczPc4hbHctzu7QwSQVFY9h8Ys7SyTmUslS7QVjgH0zvHRhhI5KagdGwK376bBn4P0cYH8ydUmlAUBatry3z/6qu48rvfZXl5ifW1NSbjMSpJSNOU7dt28Mu/8qvAifR6PbIsw2jtmU5RgOyQFvO0bLZuGqiO53LYYv3+B+9bGxjjBwPb6YLnYTS2GWgefDo/cdedl3c0AhhfFK7v5FlOlmZ++ZEFSLZsy7Zsy7Zsy7bsZ8Okd5OMdSREp49uIjhtCKC4jmChA5pNZJ23UKy4rCPRYg2NtIlzmoUUKIKECLTRhgYIb8gpTT4Xj1G0/H1j6gjuW+tpokLEcoVo8IrgLxkPdodkmQ6IbmQ6wrYOmKfrf+F0130LNrhUaB9/TrblMHp4jyBP4hF2hFCOn+ylGROlHJk2tIUnLTqWuUIohbAKhXFymLZJ5hm0vKUUaF1hbO2AaQlI4/924HKaKs+c180YSrkj6toz4z1KLISJbdEAUa4NCVrewfMUCiEkTsLGr4vt3/iZTT9ygLiUGkRgx4ekpN2+EGjZxhjquvbXTnRIMwLV0n13CT5DmU1fawdUbIdIEoD0SJS1Ivb3OPuidU6B+e7Iy25d0E0XFpSQbuanENQqIZHaBW+EI0q7hLdBf1+AlYTEqh0A+xB2xCB6A5i3AMdwQeLQodkkXFrdYRc323QBR9Eq3xKDZbEc2/otUCKUtJEJHMHMMI3FlxnXCw/4tuoIQX/cUtUlRTFkbnw9CwzJRI00Kwg9IskzlBpQGgtCI7Ekvr5CBCK5awtjW8ecgkvDOZkQIQroqHDRFNvZL6DtDbgap+9MjQ/blzwEIZpKeEZ7yCgMTh4JEQH01taNyXZoJAD7gR3v265Z3QXOp+p2sNFvBNDbkiZCbFbMhh0d1j3d0TdWIjxoA7sfmod0+0i29Y+IHchO1b8LP8QsxPFgU1dbtH9Nb9u6o4Tv651+373W7QBCDBodtIG2bMu27OfdjjnuWLAajKYoxxhRMzEpxiaIPEOkGaWeMNvvU1eGbG4GYxJ2HXMcaX8bw8IwLirynmOj796xmzxL0bYkSRQyNVhRYZEIJbC2dolMhcsr4qYRio3PxUPYhuSWIZu9cQMOJVV8R2gERVlSTkaM19dZOrCf1aX9rK0uM1xZoS4n9LKUY7cN2HnMdrbv3MbCwgJpkmCMxlQF65MKYzSJdDItKkno5Ql5LjGDFF33HOveOFZIVZUURYkth0z0hHoyYpKtkQZwVCWAReuKqpqQZpKZfk6WKLLEuqC4qSgma1R16dgTMnGJyi0IDJoKozVlLVFSkiiX9Mhqp0npSS6tN5H3fqbes21mUsdCYicMjrogMUJRaUmpBetr69zw/Sv5xr/8E5iasi6QVpNIgTGSpWv77Ng2xzHHHEutBblIkd75NQKaoUi44htniwnc9dzgqgoQU85dkLzbtL9s4uyGd37YpfGlWn2rta2w7SXB32ntY8M5uBdvU5WWh+uTNgkbZlb6e8PrLQoDuUrJRUomnPtd1WFq8JZt2ZZt2ZZt2Zb9PFjAo9y4v6VVLRpZguBnOJCZKOkSvh2Q7BQJLA4fEtJGH8XtHzStfdDeuJUOINe+LpvhII4tHZjBzu+UUQHCWI3WVcRJYkJQfD6feI7S678H7esGkA0cBidBQwRQbduPjQSRtk/YALHt8YTDhAKsJjHaeH+SCKA7XWwnI2MRToNdOLkVQ0SYCQB6SAwqkAiZIK0GvD8u3cxRIUw8HxBIqxASVOKY2EI5QF5KQZIE4pFFKkGiJEK5iidSk6gaXWvq2qKkJpE1VWXQtaVSpiV1YiJDu2HYS3RdIaSJkjEIg/C5mEIbhL5irE+GalotKCzExK3Bd3as8Fo7PXTXPSRS+PKlQCrt5FKUbWRqhB9qeCK0lcZjj2DQSCEQ0s08sB0tdDdT1ljbJJttX2cRmP7C5V60wslg+im8RoBRUFUaCVTSMeOtSUAaNKYDoofEptYHc44EQIebxERvboju8gaGlIjOpGNLVyNaxC0boLBzU/gbEcLjIGh+C4JyiruXupGtiMGG43UiXyFa4hvd10zQPDSE307gIoPG1CR6Qp8bGaghaT5GmwFCzgA5QkgUBqtLbNKP7aJ85LCOic4azSV83S3EbL2SJuqGj6Yo4yN64Bn5EfJ1H9ucU9OyobXDc6Y1rJYNaGsRIXAFllZbtACO1hhStMpx16K51gHAndYAD8kFmkKmfm/6d6gdHtAXG7agdb2IJVg2H1N3UGvaIQzXxWLnoxvg8Eds9alOoKZTso17xys8HWia2m8D6B0DBUFLrLXNhmO6t4JLLmsIsT8hwsvwyG74LduyLfv5MqkEaIsVNYIacE6vkClWZgiVYY1Ga8jSHlVRkw0GZL0ZVK/PjvkZDiyvkGd9ZmYGZGlCL09BJiAdwGtNeJdKtDaO0e2d1chTaD+jImDZfWYGEyokyvFOsvHcXuES8QgsZVlQTkpG4wnra2usLh1gfXWZ1aVFJqM1EgyDPGVh506O3r2D+YV5enN9ZKqQQlKWJaPxiNFwxHA4xGLJsox+r0+WZ6QqJfO654mSZEmKUr2OXExZlpRlybioGY9WGI+kT5qUIqVgMhmzsnKAqp6wtHgjVTGkPP4Y5voJiwdW+f61+ynqBNJZZDZAJT0EgkS6gLcROAcSgbbGJ1lqBg+NHJhtmvUILPgU4X0U38oWtPWpxi1gKurxGqnUJLZG2RplLEUFVVEyGa27oIIOU4tV9AKnfcXN4ut288tPu4CmbgefzbDpiYuplb4CLX5D4wfYxsdtr4gzAFtAvxtQhTJFw3oKSHzo234OqbEaMAhbgSndwIEe0rqZEweXs9myLduyLduyLduyn0UTKIdbycDKDuB5AJmDb6AwRmIi47tLSbA++SQCRNLgWY1SQAvvimxsp7FuhJMaaaRN3H8BqBUkIBICK9tYTVlXCGHRpqYyBdpUBNawk6TxvroHcQPbeJp13JyA9IlUReuzsbUCvh100OMa0ZXUi/gbFodOh31cywrpZZGtx+OkQSCRInW+nTAgHRZo0FEZIhBSjG/zUC8hrQeDPTFFWoRIY/AAGQB0N4NVSOmhJt8WViA8u96GhJ3afWMtSeJcSy0sSthGzkVogjBLuLZuVq4FoUFqpDAIZd2YzE9V0KbGoj2Yj6+fH1tI6dj4wiUjDbirNQ7bdDrvykvlOKBbSUOqBGliSRJDkkjSVLZ01i3WunPSoR4eW3P5UDUCl7rVscS9ZIw11LiEryYA661Ri+tTrhCDI0e7hnTrtTZoWVMLSyEccSlLEkQiXUDKakwE0o2fVWGwXhD/SGaJHnliUXeFNl3XHhzJGAIKsGgLbBR4wK9VVmv7KF0CtHW8o9wKQealy8INRcnwEAkDIxFKbTHPw7IpkFIiEBJmkowyzUFmJNaQy3WErUlUz+mcJqBFEOw3Ll+vNcSolC5RdoLCokWCEDlGJA5owIvhmxJpaqQtkQg0klr0gIzpacvtAV8MuE2Pufw5BTWYCACHdvHDWhUKNA34GpHwMBgMhYf2xt1fQfIlrArHCUGLTe1gyyE+5MKDGcJ1PjyAfjA22saDuJLayuHTAHW7SeMgOn5ZphbFl1JY2Aa4m7o2+8TLIJjaL7zQGvAcMcVq77SfaAI9HUZ8+yBbtmVbtmVdW19bQZgaJYzLQm4sWIlMMvLeHLNz81hdgC6pSsPy6j7ymZJsbgc7sj7b52YwWqOkYqafu6QspkKKDGyKsI49bf27PFGJf0Y5UNV9B8cE4vPyYMkngxOt4iO82c9aTOWSdC6vrLC2usKBxX0sLx9gfXUFgaWfpezcOcPC3Azb5+dYmJ9jZtBDKYVBMBpPGI1GjMfjKNMipZv2V0xKxqOJY9sIQS/P6fV6pGkadayTJCFJEnq9HnmeU9c1A22ptZPrmExKimJMXRjKogDh/JW9e2/gwNJeLv+SIbEVVWWpydmx+ziOOu4kZrJBnIVnwAOyFuETOWknvNgk8vTBeBCIOH1tY5tuplMv2i8lK8Jwww9AghaniMeqdQ1eE9JY6xL/yMQlHrUu5XzHfzhCaw/uDr/tzXjJWbtpm2xWttikP9pA5WqQchyLyg92I7PK+zNWtHwCi5AWpRTFuEDXI7KsZteOOXbtnENKyXgsqOotOZct27It27It27KfJwsgufO3DFpDwLkCyxycfyGV81GibIpPWunA54BXeMyspfMdt+/IpHjwOIIUHoexAX+yHmzHS7c0rN/m04D+JuhVGwcuO1dVYJXy+xpfnok64U2AAEJuPxHrLzt+60bMR3aWTedJagPxU8hYbDuN9tuoKN8RqajCa9ELJ5BurWMnmw31EBv80nZuoHgeAvDM8Mho923YYEGBkmqQQmCEazOsY0cHDMhJo4RAiMWIkDjVy9P4ZKVCeo1zWXsyrUUErW+cz54kwiefdddFqgaTsjb0CR84kBZlfX6m0A6e2JkmFqnACoVFIaQjWZmW+22tQIs6BmcQ1uHFEc+VWK/Fboylqg21H1cZbWh1F0RsdxmDIw6oN1i0H286zXqtQUuF1hJHWVFIm2Bs0GU3sS/VtQsoBbz5SIC1I2eiTw9EWowb0d0w/ulE3X1HichiGzQHuwnrtwEhA9PKH6cFtssgEyMaPUohbGRphVoIgZd/acok3lyNSYjxnDxJEEJSVobSFuTpAEWCIEGjMFZhZIKUqZtm7aeql1WFrdbYZpfoJ4LS9hgyy0QOUDJH4CJbol5nhiELcoLAMqHHKtupkGirfMf0fK7Anmqdk20t6ALAIZIXHoOBZd5cLpdNmfCj0yaxnDbTvfXQiH/5ge9NGtK2gOO4wP8bqtM63e7frcBCh3oeE8iKqVWNbEtb79TSBAPs1A3SANyhHQ8OBtgAZvv92jdcDOjEdVPHCYGeFgCuYgVaSWZxDx1rrH8QukBQR4vdT224qcDFlm3Zlv38WFVOkBikdF6uQCBVwszMNrbvvgUL27dTTMaM1pZRqsTKCqtgNFxldjJibWU/C3MLYA29XKKUoSpLjKmRqk+v10PJpPOesjo837yzaLz8SHxjHdxcsDm6vC6xjHHTRsfr6ywvH2D//v0cWFxkuL5CWa8jrKE/6LFj+zZ2bt/B/NwMg36PXp4jhaAqJ6yuFayvjxmNHHAOjbMtpST1rPM0TSN4rDzrfDKZMB6PCUlwgh56BNaVQChBmg0YzPSpq5qqrinLkqLIXVLOakJVFYzLCbacUFYaKyvS3iqDtVWybOCvjUJJiUolxmaUxkStSCMsKrSgFAirOq15pECzd926ZAS/zGKRSqJUQpY63ftqorGmQpoaJUEJhVSKPE0cQz8kCsJ6kHljhpCD1U0eLJhyhHbI3hQGjDYA3pu/Ld0svU1A9FBGBOOb97bw5YZv75n6WE+gkBh0WVIXYzAT5mYkJ5ywnWOO2YXVsLY+YWlpdPNPfsu2bMu2bMu2bMt+6qxJPi59LqEGTA7a1ME7UzJIcPiPFJ7N3cyWD3iD2kRnOjC/pzXHnQSI8jMo5dTsfpd8s2GQd8GoUBclE4KGuzEep7CNrrq1wvv/zfFDvRpcr1vntm52+9ut7/5uM9gb+ZkgC9NlrXc0txEOFEoUUliMc+bcDETvxwXd9o6n6RO6Nu3YvaadFhQ4OcvWNQkSO1prx9RXstGg96SMkFRWShO3BxzojUvkKaTDLZ1EtdMjl1K6baQD3BE1oOPFcjI7TqomUZIgHSRVYHK7+iepAuEDDka766VsxJyifIywKGFI0pAfS6Bk6mbNEjBHxzKXxjjQXxiXn1A6QN43uy/TdTCrwdRBg70700BJhbUy4pAunuBY98ggXeNzbUqJUBqUY9hrK6jrFGNSwni0uWamcfFdQx/WjlwTnXACrdiJaAG1llbIgQ4oDrQq00CpDZDdRDWaQV2TqDJ09jCGaR4YrQ4cOioB4LWd/cJxRKyXiFEQgNoYJmWBrieoekxPGtIkR6lZkjRHIlGJwEjQWlIbEIkkVS7yU2jDpCrIyjVy9jMvDaWco7SSih4qQp0GZUsGepmBWiVXllJuo7IpQiRMcMw+R3tvkP44fAvggmgWht/xHP22MoLPQQaEeIN0QO3uF22JEz8zBxfQCFemuWQbLNwNrUJjfTuRJNs5D2mb43c2Dxd9k+M1PakNYHcfrDHJRaiRJU5Dob15q4GbqdrNuhCWEGKzvkk8RgOGh/qJzvqm/zVxjNg/RRMoCtHSuqqRUpKkoc38efqosZ/t45NSbNmWbdmWda2uCxSWWlq0NpRFSV0KBmmfXbuPY8fuo5iMx6zk+1lfugFth2S9AVIY6mKdxb1j1lYOsHP7DopRilaCYjxB65R+fxuJsMgsc1PxTNBZBKkcIO2YJyHR5MFeHDb+G5xOU9eURcFkPGQyHjFeX2Vl6QDLS4sM19fQdQXU7JwfMDc/w47t25mdnSVNU9LEvZeHY8c4Hw7XKMYlVguXNHVugTzPSZIkDhDSNGEwmGF2dsYnyawpiwllWXrploLJpEBrTVWVlGWBUoosS0myjCRNUYkiUYpEpeQmoc4TzEyGNrNUVcVkMqIuxghdMRqXlJVjrpTFhJWlRdJsSJb3yHs5aZqA1xQMbSesjclbpRANCSGYf1dPt+xGyRAR380tFyo6sFJKkkSS5RlJmmBKAUKihCRREiMSyFwQQargiBpfhjzoZd444Gi9Qw/SJzbUe5MlhwLS42DK2kMcS2yoiIVAxJk6iGqV13YgACtjkD2Me8bDMeP1NSbjFdCCcrKCrQdYFLoaY6rJIWq/ZVu2ZVu2ZVu2ZT/bFsD0gHdOSZp4Xy8wuDfwWqdwMSkbkHc6J06YaegAdOEkPyIoTCS/0sYwfB0bgoTDIBomcNtJErEeooVrQANitz8OczIbtt18H9cGDRguOp/ALJY++akVDuebPmbU08YFD9xpaQeQYxHaUFW1k9wTTrZE49j7xoaEn3JDuc31CHVyeXKaYAA0sj3hWlqfT6rBnhxYLDGeyEOLIOva1h1f+mSiSiqSJPWMcomUTj4Gn5PHWg3GRK168Os9HiWFk/5sgho2OsGSJrGokM31CYC9EposTUgS1YwrrOvLzfaO5Z0kHrTCurJ9Il1dByKKRcqENEkx1snOWFVHeU93UaW/HwJQ74Ml0s3WEFYiAlgvcfI50mBwsjACibXJVIDGds7NzUI96IAh2k1gortmjf+Ghm7rjbTZu6KLp4rW8tZXRFglDvTtgN1h3zZA6cc6qnVjNzdrKNLGstqgZQCHQ5TMGo2gIhUGqoJ8skJSrZFRk4ghidBOR1YNUEmCtSUAyuYoergYkBP2L6qSslgjLdYQagyqQKgEhPb18A9Abairmkmt6csSkYxAKWaTAZiUulYMyVFp3998xIYIwGuDYotWe1pU0+dbwC/xxpvWL4/XqH2dW8sDe913L7d8arTaBo6DBdDdFS6ahaEOofwQWLDNo7pdfDeRWHftNGAeoyu2ga7jlJput9zkBFpltsVPW+vbjPI4OyJUSYpOO7TukhbI3poVEMrxU6iED5aEc4ovJKMxdY1QEqFSkE1QBHxm5Vq7QJbYQtG3bMu2bKMZ4/TwJM6hSZUiVZI86zE7s4BKZ9BjzbiA/swcZTVGYKiLMUv7b0Bbwd59e7nNrW+DKY4iUZLJZIySPRa2jUjkmLrOMaam1oairABFmvWZmZ0nTXMSlTlmQnj30no2WuvzPIA1hkrXjCYTxqMRy8tLrCwtsr6yQjEaUldjwDDb77Hj6J3MzPSYnc3JspQ0y0EItNGMRwXjyYTxpKAoS8fyQJAlGdu3b2fbtm3keU6YMhoGHVmWeSa6JJGAdQlSsywhTR0DvSxLas8yr+uSqiqRk7F3XCVJkpBlOUmakKUZkLpPv4+ZncXoCqMLqtIyKWomk5rR2gqT0RiV5mR57kD0LEMmiv7MvHP4hYjvOBcQT+L7KgbL21MxW9ZNpu2WNGusj8mGwYaJ7JgkUe4ckhQlJMoqlAfRbZo5h9yGhFJh+qgEPeUoHMzEJu/k0Cc223yzjds+6M02RwfZcEw/cAmHcP6VANLGf/H7hzo3sjDWJayqBWvL6+zbew15WtNPavRkHYFifb2mmGy9u7dsy7Zsy7Zsy36eTHoGdwR2hSNQOjKKaQHpApclz0nEEQDkkPNObOLfiRZQ7DHQhs3rxgJCCrDGa2zrFigMws95tC1GfASqI4LjlgdGuhdGdrnxorQdBL10IYRLSmkbOZpADJSy8cVDPUMgIBZjg1qFR6Va+F447yAn0zC3nacWmO+hHGtBSusBeR+48OA81kJVUxbev/c64UZYfxUgSqp4C4GKzQIBMR+ibcDoSKb0/mKHWe23ENLNUFBe59GaqSCKACUVQjjiTpKkXofcESuldFRNY2uM9nItHgAN7HyBw16VdAxvqaRfr339cExy02bwuz2VCjr6xgdrBCF5KsIlLRU+KGFJsFZHtrsQXhZGuPEHUnv5F0OiLDIHqTRVrTGBCW/DMEjGAIXrL6BthbE1whosjmAlhCaRhjRxRFRrneY5JkHIxM9ocAEGX5XQi/y46keoiX6wqbhtOZYIc7YaOcj8RGBT0BqqhAeA8KC4bUBGv4MM4KJHW7uaR11QVbbKC8tCB4mgbRy0uU6S2wmzYkRPrGLEPkqzCCZ1wv7VOkrWYIbACCF3UOsZCjVHpWZJk4xMKSqjKasJspogrEZbSa1xwICoMbpCJ1mMGNUkjMqaATUiNSTJmNyuYuwEbTLW2IYViiztIVtgeIe1ZVsNLjyT21+IcH4N0NwCb6eZU61f7WvcPLrcA0OH6ymaAfmGAtp/Ro3WMM5vBrqdh0SriOlSp2GAWH7TIcKCBu3v1EO0/j7EAVsAdwPwtPuj66mhP4mIAG2SdNT3NUlTRxFPuKlzDAJFAN0nC7E4LQMMxlSksiSxLmmt0AqkwGBR1pLoGqlLrFRI0WPLtmzLtmzaJqVmtp+5l4R2jpHWtWNVW0tZ1EiVYZDccONehBkznqywfedO9u69niRJKMdjfnDVt9CTFWZnZxiPRwipqKoVqnKBJE3clD8LS0urpFmfwcw29qgTsHaWbLANjHOujGcsO+cP6rrCaid/sr6+xvLaKqtr66yuLLO0uJ/JZIwwNb0sYWFuwLb5eXZsm2N2dpYsTRACqqqimlRUdcVoUjAeTxyLxLh3j5SKRCUo6fTMQ1JQIQRVVXUYLUoper0eWSrp5Sm9Xo61ljzPmZ+fo66dBqGbZljH5KJVVVHrEl2XFKamrhQ6cY6tc24dCC+SBCF7aG3plYaqrClLTVFpSl0xHheMxt7HkYokWyTPnP56kibkWY80z1FpTpKk/tXX+EZtRkx7ENJhAxmvUS/wyTPdNGLXDjLi9UpJZmb7yHqEMDWZTMjTDI2klM5hxw+AELjpqSr4GYdHtoPfsdmU2CPPfRI8/WaQ2DCRwhTotqSg6G7riwmJ3p3ki/MJZGAR+f/cSMvPErTNOz6wRawfSLkBodN2NGVNOSpYW15lzYxYX9rHvyWCPB+QpfP0+9uB+xzZuW7Zlm3Zlm3Zlm3ZT71p7ZMgWmhY3TYChlEm7mC4Cw0m1JE0wYkZhHWOfSyCmxTxs8DaDv6jcxddPqDpRJ9O6zuUEQRxHYvbsecDsC8jk13glhvbAMzTrO2o0RClX2xnu82wx+DXtv09B8TLTp3dQR32F9zEkJg1IFFKJkCCEC5ZpjUGYV1Sylq72bXGaKwwGH8drAjylMTjb1bfKIPj/eMmIarHlrzvnCTKz4oVSOE+jUx18MUVVlqECawZHxywBiUduSVJlJul6q+90zg3aOPPVzdkD6zXbpcgPDYpRBKYIl4aRsT+GMkknXaWjjhjLQgTCeZ4QF1gPYPcg+VIp+8ujAsaBOTagksAa/25etxWGITUUXM/jBccm7w7drDkGKtxQtjafURNqnCgvArHtVijwCY+KGEwVvsxUThfi9GaH2li0U48qNWnG6KwB64bzlMH5JY02tcibh72gS5T18adImE/AMQtkL0D/AofAwvrQySOdmLScNu4Wmpbk5gJc3KNhWQJ8utY1XuZlH0qJEKMqMsxiYIsGaClxdoELXNkOksv75MkCdSaVAC2IhO10ykVA7RN/BkaMAbhpz3XFpRKEFqjKMmFcTIyokelt5PKAUYQp3GHPhhJ3eHkw3kHAH0qStRGlRsN8zZQHtqpDTfbCBLbgzDYN4Dkcc/WwLm13F2bbrU2G2JPQePddSIKB7W2aoHk/j0U9GJF8098R8WtQwAi1NE/3JrNm8G29GclW/071rQNoLfKC+0qWg0W9haecRn3CedmBaZ2D2v0BEVNamp6FPSUwFrNRCdok6BsRZ8JPTNECkOh5qjVTVBm2rIt27KfG1tcXUOIOcgkupxQlI6VgKmpijFZf47xaEhZFhRFgS7XGU3G1LpAJQnFZEyeKjAT9u+9hr03aHRdkWSK4fAGVlbmUElCpR0TfTSqSbNZdu48lvm5eZIkxdiKNMnBBj07Ta1LjK6YFCNGoyGrq6ssLR1geWmJ9fV1imIC1jI/M2DH9qPYtuAShPbzjCxRGKO9xEpJMSmYTCbUtXYZ5K0FIUlVQpIm9HoDsjRlfm6GmZkZ1tbWuOaaH7C2tk5ZFtFx0romTTN27NjOnmN2seeYo5iZmXHveWhpLQrCNNn2pygLyrKgqpxmfF1rtE84KkRNohKEklihSWXidNWTjF5uqI2l0oaqqhiVjkFfFgVlMWLsdSnTNKPXG5D3B+S9PoP+DEmaOXBehIFJGAwJAhtHTk9TjO+k4MQ278pQVnDusyzD9nokVqOsJlWK0rjZfjG5qbBuACKCF33kJgM43TJxMCb6JsB8GEi4/fx23o9stMm7IDtT38KzNay1njXkZ5RZn7Qp7C98mwViQAt4l1I6fN0zmYzVCFORJZZMWaQ2lJOCUVGTJpLRasFgYOmlszepvbZsy7Zsy7Zsy7bsp9uilInFA9QewDO246II4fzZtmbzNGISmN5OHzuAnM2xHCC6oQaupEgsCH83YLbDLTzAEVnTbT+stZ4pTNA2LO9wvMj6bgH6oS2gAdM3k6Fp430NFBMwrIbR3tGzFiZgzgTWcgO0K4RM3L5IpNfRDucR2PUORAcjvNcnfK460YDM01rv7XECng0fgweCiI9KKdBaobVxmuIeRHezPD0Rw5gWYz00psVYnzlSGFTiJCkDCcaRW1x+oiCx6HT2vQyhwAdBwvmGGQBBslnEstoE3nZQpQl6OHJJ8IlNjBOI6JQH1rixxgHrHnxvcDni9bO4NhPGeE34IBXqNtW67vYJfy5NjzQ43XdNmrh2EDKMTYJov5vxYY3xM2lB+iSzxji9+un+t5kdOYjeBgzbK9ot4E37gUbo5w147oXrp27cBncNgG/Q326DvwHwtCGo47aJK23rd5OQVLTA/HjBY5UNZTkGuULeuwElf8CEa5mYAUpuQ2UZ1i4g1HZq+lg0SaJJhUBL6RNRGbSpoRqTmJLUJ04ttURbg5EVghpLTW0M6JK+GTNjx2SyArNOVQ4RskfFdgo7i/AJu4SPAkmfDdOGABRNjw7pvAJxOg7+3Al3LlNoExPbr4XMx9CH28/6NgxXVUZQun0xulE3YcVU52gsPjBp+sSGbTbbsd1fwhYBMRfd14hoVVG0/t1QnNx4tHZQp0lYGoNqrUCCuxZx+wjEN+WJWF5TqRCZlf4eEEE3VQQMw0+fqUZk9TIDsU6Goa80fQna9lnTOZWBXGnmkzEDuRekZD25BSOyzVpvy7Zsy37OrSZhdVRgaoEwFZNxQaUThLCMx0PywRBjSvr9lMmgx7BaZWbQZ3VlmcFMn16WIqXGViNGk9rpgVclKrFMJilrKz1UloJUCJGQZfMYk6Ok9BIpfcfy8ElNwVKWY9bX11hdW2ZlZYnl5UVWVlcYj0eYqmam12fbzgV2bN/G3Ows/X6PLHF11nXN+nBCUUwoy4rxqPB5IxJm+wNSr0+eKCfBkmU9t79fbq1lZWWFG264kb1792KtjclCpZQMBgPquna+g5/6qHXtknvSZUDE94N3KHt5jyxNqOsMrWuKoqIsa0pTURSaia4xaFCWRErSJCVTKUmi6GcJAyUx9Jm3hklVU+uKcjRGa01d1ei6YDysKMZDkjRjPeuR93rkvQFpmpHlvZhYyTm/m7PSpXDwurDBR/KBX8BiWoxqlyxISYEyzYy+8B4ztvYzp1rv0zBiOSJrpFI6Sw/GQt+sWD/aCc40NL6lgOjztX2/Vijcuxh+cGqbREdhYNYcuz14qd0AERewEVb76dIGrHHJmHRFVRaYeg0lJqSJoRYGPLtJmwrRq8i24t9b1rITTzyR0047jQsvvBCAf/zHf+T000/n0ksv5bTTTvuRHefLX/4yv/mbv8kVV1zBLW95yx9ZuT9Kq6qKW93qVrz4xS/m6U9/+o+7Olu2ZVu2ZT86s5I2ctAF0YNv5MHhxCVTtB7wc9tMzeKzAmsEqO7yaaC6bW2/yRowVrfqE8DVruPVkUUxNvqPtHTcG7C8CQiEc+w6cgHcbxKfTs9O7Oqe+4URWBQdn1zIVh3CfzYQHrrnIJAIn68psOzdph5va2mv2xAAEHQA8Wlmvda6086OUd7MiGzjn0kqvT59QmB2AxgdsLmmbGM0bbPWabMbQBmBMTXGVkgSEE5Y3/gZp7quqXXpMnUSAHznzSslsFa6MY5RDQMdC7qRrAmzGcBGPf1wHgGbDLKO4RhBlz4cy7WvpzbbIIzjPhLpk69KN5vBCBLlfWxPUrK+/EAKagB0iyBpAekWJ5tjENI4Uo30kqG2mYErpFcnCVI1SM+x9dvKjffLtG2GZW6+oWjAwci6bWPY4VtAgiDBSVYkOKkWFTp/iID438Ld7yjhdFuVtCgfhelopBO0kIJ+enP8Bix3J5T4EwtgpV+LsYbaVGhToXWJ0AVYzaSqmBTLGDMilSl9VZKxgpJQi3nGZjuFXcCKnVgx63SDRA3UaF2iqyEzdkyfElGXGGOZaMXYzlAls8gkQ0mB0SVpvcIuscS2ZD89tY9+soyxNUUtGVeCkhQtU2hNqXafJkKlhLvtVbweIg7448NnA4AuvFqVuy7tgEVUtoq7BKi+9bBr3Qjx2rUfhFN9rQGSu5093GyH7ZqiqVsbKG/CYc1hm1vbf1oLRDhHEaJvtrVXu7b4vmljeeFw4bdsldWsC2U6trqM5RAH4QI/Ng+aZK2bJsQDMBpFQU9MmJNDFlhinv307Q30zA3MiEWOVtexR36XHfb7zNjrUfoaVPld8novebXvcC36U2WveMUrEEKwf//+H1sd/vEf/xEhBP/4j//4Y6vDj8JOPPFEnvCEJ8Tfm53XE57wBE488cQf+bGf/vSnc9/73vdHXu6P0j7xiU8wOzvLvn0/W/dQsJn5HUwqy3BSY0WCTBRYTTEZMVpfZmVlkfFohdFwmdnZPsPhOsvLB6irAmE1WSLIEgG2QklDlimU0Oh6zGS8ytrqIqP1ZXQ1QXjAWQlFnvXBJiSq56ZH1hXD4Sp7b7yeq678Lt/5zn/yn//xDb77nf/k+uuvYTJeZ9DPOW7PUdz6pOO51QnHsueonSzMDciUoConrK2usri4yL79+1laXmE4HpPmCXMLcxx1zC723OJojjv+GI499iiOPmo7u3dtZ+eOWRbme8zOpGRZEnXLZwYz7Nixg2OOOYZdu3Zx1FFHcdxxx3Hsscexc+dOkjShLCesra+ytr7KcLjG+voqo/GQ9eEaq2srrK4us7K6zOrqGsO1EePhiMm4cFqKZQXWoqQiS3pkSY6SKVhJVRuG4wlra2usra8yHo2YjEeUkwnW1CRK0u9nzM302b1rG0fv3M5Ru7azc9scgzwllWDritHaMisH9rN8YB9Li/tY3L+PAwcOsLq6ymQyQWvdYY004L9/7wcnNLyvhIiOqvDOsrWGuqqoy4K6LKmrklrXjsHhHdtWXN+VHX287meDxQHLxsHIEe3vt2ufHxwChCf4Nc03eDaKnxkRPlZXflmBrgvqqqAqJ1TFOmWxTFksU0yWKCYHKCYHGI8XmYz3MxkvMpnsZzTcy2h4I5PJfpBj+j1Bv5eSpwmJkiigmAxZWb7xZtzVP7122WWX8YpXvILl5eUfd1V+ru2lL30pj3rUozYA6P/xH//BmWeeyezsLDt27OB3fud3jvjdOJlMOP/88/nlX/5lBoMBxx13HGeddRb//u//3tnu+uuv50UvehGnn346c3NzB/Wz0jTlec97Hq9+9auZTLYS8P602ROe8ARmZ3/4mTZXXXUVQghe//rXH3bbMHb4ebEfx/lu+fU/GguyLcbgma82Mo+1NhjtGLF1rf2sRoPWFq3bWt4B5A7fQdPas3FpwMc2FtJ83D7WiMj41dpEQD/6ZaYNjBPrHbZvA//WSkea38S3azO3m79NZ3kATQMbuL19A2h3/cS2NXW2UU871C8A0/i2cviZk+iL+JRnI8eABg0RNCDpDUO8qe+RMJcbDE1gNPEau+vasJ/bQQOXoNPjU9JJsEgFSSrIMkmaSZLUAeLCJxQNuvHWuLyNdV1R1ROqekxZTSirMWU5pigLinJMUTpiUlFM3LJi7D7lmKqaUNclxjjdcWNqtHY4qtYVde3yQ7ll2gH6QTKS0Ob+OhIQXenAfiRYiQnSQCpBqgSVpu5bpiQqi0QnpRRpmpJmKWmWuHxVWUqWKfK09UkS8jQlUSlKpSiZIf1HiARkCjJDyAyhcqTsYUgxNsWQY8kw9vDk1JvAgWmiOxEgbbGB41YeKG10aprlDaPYsZg67PbQYWKBMRUr0Ei1hLJa+KwH2B2wHEDQps7ucllgUtWUxRBpCnrC0BcVM2KCxDCsDNquUWtNSp8sgVIMXRREuIQPRvQRMiFTEqkMwo7Q9YRcr2PtOqWo0emAWlQkQmJQCJGRJblrCz1m1uxnu7qOhfxaZL1KbRVabKfQR7GudzKW20GmEbQ18eHQDAqtDefZBpStj2J5QLe1vQPPW0zzwCBvUbhjm3UR6047TlVjAw4tNvkhpjZsg92bvvbF9HK3oC0F1N63/R1kezrneFDfoh0p3HhOId6DDQGkNvjdkpXpgOytaTKd4zbTxGNSW38NpfSSAHVBT68yx5DcFmBqrNAIXWDtBIl2mlf1NdT1GJ1sQ8gSa9ZRXEUvOXzEbMu27L/TrrzySv7qr/6KT37ykxvW/e///b95/etfz5VXXsnxxx/Ps5/9bJ71rGcdUblXXHEFL3/5y/nc5z7HgQMHOOGEE3j0ox/NC17wAgaDAQCj0Yh3vOMdfOQjH+HrX/866+vr3PrWt+apT30qT33qU1GqSeZ35plncutb35rzzz+fN7zhDT+ak/8JsqOPPYHra40pVkC45Jm51YxHq+zdey1iaZH5he3UVUFtxmS9nNHSkLoYU+QJvUyRJgqrK4SUZIlEzvRZXy+YjMckicIKgVQZ2Io0B4GkKg379i5hdR8lBVCyvHyAxcX9LK8uUZUFQlh6/YxduxZY2DbP7OwsM1lOJsDWhrIqGA+drMmkqKjq2jHH05TZ+TnyPGduboZBv09/0CNRTltQCQEeCHbvLgeMKjVLoQ1SKGZn58iynLm5Oc/cFuR5zmAwIMtSpKyj5nmWOWfK6ag7LUGjXaZ3ay0Y6QcZ2jG5bZjeKRzbyLh5YEomGCxVXWO0oapKCmsZiXWyNEWlKUmWIrMMkSqkEPSVRBhLnip6eUa/n1PVhqrSjMYT6rpmMlxDW4sVKVKl5HlGv98nz3uOme/bJUnTqJ9ofFRbBOZ4YJb4v1WionyNrmuErkFXWKWohaE2iWOsh3amCXpvMLEJsG39ezU62p1VTjuR6V3sxoC9dwbbgxkbX7JEVlGAzUMybmzQhgRra6yuYkDABp9SuvM31viBpcGaCmsKrNUYa9xUX6NBeGkbHNulqgrKYkw1HqLLEphgzYSymFAVJVprhiNYW1883C38M2WXXXYZr3zlK3nCE57Atm3bftzV+bm0r33ta3z605/msssu6yy/5ppruMc97sHCwgKvec1rWF9f5/Wvfz1f//rX+fKXvxyfgwezxzzmMXz0ox/lKU95Cne+85257rrr+LM/+zN+/dd/na9//esRsP/Wt77Fa1/7Wm5zm9vwq7/6q3zhC184aJlPfOITedGLXsR73vMenvSkJ/3wJ79lW/ZfYG9961sZDAYdwsrPov04/XqAT33qU/z1X/81X/rSl/iP//gPjj/+eK666qoN5f00+PUhgagDojV13QDH7SSSANpABBwi9hZmSzpgMkjDGGO9dEdzrIPpi2Ob9Q1zvOUzGRvlN1wNRKx7Axy3wCRkZEu2we92PabJEm3gaQOLO7ZVkHkh5lpsjik89tiWwXHArba1Z5d3gw1Bd9xJnbTY9h2w3zZ4pvAeZCSENoD8tCZ6UL1ot3GDwDUIlvMtQfhzA+EULlosb3D+Z5wt2sJBEXhpGeOxJwNWkCQCUG6SqNYI7RN5ShPPL/QtXVWRiGKtxFiF0CCkRupA5nXEIyWV01CPGhiufYwPLFiaaySlwGqBtLLRVrcWKbPYKLIT1DCelS99wl0BxkkgIyzSJhgf6AmE7IArC+HVMmyXrR/7RxwjTPUZjw5L4aQYJdbPxsD78j9SORcbodjYBWRrPGObwZNo/dEBO/0KKdo65SLuE5m9rX3Dv+GShbFeA+KHBFAbZUyaHMLuU9UlZrRIXi6iREWWSzJlEXZMbdYY1SMSmzKbpWRZjpVzrNQVtdAkymKoQSlSHFsstRYlC3QyYpzUrNTCJRRVEiszZJqjlJd10TWqHtJnDSVuxNbfx9qUNJmjMjm13c5E7aJKF8jzHon0gQaLzwgcbnDafaJp23BzirBRa02LJRabJzCi289U21K03/Cs7RxscwvrWhG7KR457dPYCMi3AgOt5dOAuWyh6LKz6TQw3n2IbSg6FnwQADpWvdWAUwC5z33tHmJ+pYhTQLxmloYwzRul3IBblwhbI1SKMJakGtJnRC7HKDt0yfZ8ZmFh15BJgq7BFKuMR1eS5sdjsm1INYtUCb2k2vwctmzLjsDe/va3H2EU/cjtzW9+MyeddBKnn356Z/lf/uVf8rSnPY1HPOIRPO95z+Of//mfefazn81oNOIP/uAPDlnmD37wA+52t7uxsLDAM5/5THbs2MEXvvAFzjvvPC6//HI+8pGPAPC9732PZz3rWdz73vfmec97HvPz83zyk5/k6U9/Ol/84hd55zvf2Sn33HPP5QUveAGvfOUrmZub+5G2w4/bjt59DOVwjcUbxowmE/JEQl2ga83KXoPIZ6irku07d6IngjTJkTIFUVFWhuGoQClBMRkzmOljjPSBRQeUa+0YEBNroA+DXo4wktWVZZYX1zmwbz9KgbAV48mIqipJkoS5hVkWti0wOztgbnaGLM+c413XjCcF1aRgOB5RloUbFChFnuf0+rkD22dmyLKMmX7PSY4ol65ZStm8ipyH13KKR1RVgaWkP5PRn8kZzMxhajdwEdJNi9QVpLkkUQmJTEhUClgqU/vnOY5FYvzQRQSH0gexPfgqpWQ8mrC+PnLMdOESlya9hKzfQ830scZSlSVVWTOpJthRgVAK6RnLZSpJlXJguGdjJD1FnivyHlRVzaQoKMuaoqgxdcG4FNTjIZMsI8/65L0eUgbGhgPWVSqRSiKkCygZ65QElRTYqiYRzqlMVArWkEjn8FshMFY43fdygjIFiS2RNnPa5FZjjXsf2cCIMn5qJy3mkQGBa/OoOe6lULoejJ8maowLUrSmxLadjjAADOX7SbqEl38IbNR17RlKAWwXQI0xlQPuPdMJBGmSuCmzxrHCHJumctv68gLrPsyedEmQDLKukGWBKSZUkzG2LqnLCUUxaupgLbU+lFO1ZVv2o7d3vOMdnHDCCdz97nfvLH/Na17DcDjk8ssv54QTTgDgbne7G/e973258MILeepTn3rQMq+99lo+/OEP84IXvIDXve51cfmpp57Kve51Lz784Q/ze7/3ewCccsopLC4usmPHDj74wQ9y1llnHbTcbdu2ccYZZ3DhhRdugehbdlh72ctexote9KL/9uO+9a1vZdeuXf/tIPp/9/n+OP16gPe85z389V//NXe+85059thjD1nuT7pfHwDvwDzXWmO0bbG8WyCy93vas/wcmNtgaN1yGxA6al130ee47uDAtic0NLBcBE6bj/OtGni9BYzS1K/JR9PGhAKobds4eqgZ0wsjSB3B5AAsB4BUeNmTUCfnp3frEE7GxsNYL0MjnFZLJF4Enz62hwhjZOv/DwlQaZ1vIGfYVmuEutKpbySzxmT1Qcu9pUeOG2u58wqgvo1oWZCwcYlFU6RPEtquj/NJm/xA7VkA7rpKL/cYzscgWv4taB+kMC5fZ4w3eCA+9FPhxl9S+cS01itfhKS5xpJEKRiL9gQa4fuFkRIVlgmfl1HgxwgCYYKcUVsRwvv3aAS60wujhb7iE8f66EMTFGlJUVvcvaa1u4cOZ0cMoqvmasYOEfMz2gZkjEOWNlhKYBL7CIUIA8/mNIPcTluuOgxOifGC8PBogF5JYJv77QJ92NctKuQYg6xK7OgA9eoVVHaN/vw8cpCTJOvYah+TckxPThB5jzSbQag5ZlTKWBuEqOmnCVa6rK+WFIVGsY4W61g0QyNQeoBGgbRYnbm6WAO6xmrNpFhHmO9juYY8mUfJAUbMIJIcIQfM9PokSYoQ0kWoPJAemNA2prrsDrzCFG23WQg9+HWy9WBsP8c2WGz4m2iiwe6nbq52V55+dLYB8s5GU+VOY9nuNFyfCH28nTz2IFXs/pwOIBDauVke69iegtE6ySgZFCtG7JsyPugtRhiULcDWpDpDYEjLZfpihDSKykgENZlyCWirahVdraAQVAzJ5LrrRjJjMl6jnlRYuxdjNCLdRi+FrL81EL+pZq1lMpnQ7/f/W443HA6ZmZn5bznWTbU0TX+k5VVVxcUXX8zTnva0zvLxeMxLX/pSHvjAB/LBD34QgKc85SkYY3jVq17FU5/6VLZv337Qct/97nezvLzM5z73OW53u9sB8NSnPhVjDO9617tYWlpi+/btHHPMMXz961+P24BzqJ/0pCfxjne8g5e//OXc+ta3juse8YhH8KxnPYsPfOADP3MD9R07dmLKMcVwjaW9P8BU7i0i6hLLEIRitL5Glg+Y7edkaZ8861OWNVVl0ZlEG8twWFDXhryXkSSK4ahkPKrIcoXtpUhyUjlAiQxdaVYnByhLjVQpWarIEsHMoM+OXdvZtm2BwWBAr9eLswKqSclkMqGYFJRF5YDlukIpQb8/YG5ulsGgR3/Q8yzrnEQqpHcOg39rrYkOpmlNQQWLFDVSaJfx3dZUtcWOh1RVjfKgqTCapJdjbUaqUgaDGfI8dwB/7t/DkYVjo09iAvvcWqqqoigmrK6tsW/fXvbt3cdoPERKFwiYmZ1lYWEb8/MLZHlGluckyrEv6rqmrj1oSwB9DZWeIIRwQHrSsD16mSRPe1jbo5gY6tq6qb91jS5rRsWE8bpEqpQ065FmOVmWkuUJSaaQSQ+RZqCc3yGsBlMjbY2ta9IkJU0SEiy1sFiZoERCYiR1MWK0ush4dYFEGMrKUIS8NdZ4fUjjpfUEGBOneoYLI2TLuY/Tbo1/07r3mjYaqzU2apEHH9J5lsbr7bvxjwucoCskBiFjCnXiDILW1Fv3ftf4+cdxgCqEwNTe37IW4QF26Shi8RyElE7/33r9SK3j9lobEgSpUszPzpBKxcygjzH+um7uiP3M2ite8Qpe+cpXAnDSSSfF5VdeeSUnnngidV1z/vnnc+GFF3LNNdewZ88eHv3oR3Peeee5exB4/OMfz8c//nGuv/76De+tM844g6uvvppvfetbR1SfJzzhCXzwgx/k3/7t3/jd3/1dPve5z7GwsMDTnvY0Xv7yl3eYcMPhkD/8wz/k/e9/P3v37uXEE0/kKU95Cs9//vM72x3JOYDri69+9av5i7/4Cw4cOMCv/dqvccEFFxxxW37pS1/ivPPO4wtf+AJVVXHXu96V17zmNfzGb/zGYff927/9W+51r3ttYCZ+6EMf4kEPelAE0AHuc5/78Au/8Au8//3vPySIvra2BsDRRx/dWb5nzx6Ajp91UwGt+973vjz3uc/lwIED7Nix4ybtu2X//fbj9HWTJImJwA9mxhjKsqTX6/031ermWV27d+VmM0BCGx/J+d4UG41GHeZ3237cfj24QN/b3/520jTlQQ96EN/4xjcOWu5Pul/vwHMf0DcWo6VnojttczcxzjObRdCbdjPYlQrSfNoF8GUjkezkWRrAuwFmWskXPcDhZk6GXD+yWWkF1liUlxIm4BnGRImUurLUlaVLZwyJJm3DAHeLMbTRE7+1DThyC2oOmLr3x6Rf4QIHFhNQyADNxMBC4+uHIynRw+JkJvEkDGs1Qlms0ei6QqkMicIaQ0pCZUpXrlRo7+MZDGFigHcBiWRUjzYGgNoIT3v19W08Wbetcz1tzFMUAh4O/HbeqNbej/asUYOMJJKGkW8dqTR6y45NjlV+lmSj824sKFI/s8DEWaQhYWdlDFJ6EpIHYkMfExJ0bUFZmhMxsc6BqOSCOq4/SE+OEdY6PMwrtwhK14BY1za+j4gg62Jd2dZ6fXIlEBh0XWBMhRCGRIbZDg273LHgTezDTWDDdhB1KVxeSBN1nwXWeBa+AG0ttTHo9n1zCDtiTfTQkA4Ed5+gyx2zyQqn150Ir4OO1+4WbptENn8HLSbhB1yRnQ6tZUELPWg5Oe30REAiLCqA+LR1umm+Q4e1lrKakOgRsxTYapnh6rUUwwNYs04ixuRKMZcP6GcZQhiMLigql4DNsapqpJDkSpKqBCNKjK2oyiFVNaIsFpH1tSRmL6ZaBDtxndg6XaDa1NS6ZlykHBj32T9aZbVaYWJLJ9kipO9QFmFFbIe2Fnxg3Pu+2Hw8kBvaWvltRAvUbS4k/mE09QG6UjgtEJ6N2za/G5kdCFG4LnAerh1eg0sKi4qRxO5BOsfcuLq1rgklRBmVzXtu0xNa57ABVA/n36pAZzvfxsLX0emfi7hfOLewn/L9UwpQwtCz68yZFfJ6iYFeZ0EM2SEOsFNcz1HiRraL/WT1CmVVUNkcyCjLddaG17G4ciPLB65jefFK1tcX3ZSvaoWy/Dbj8dcoiqtAX3+Q8//ptv379/PIRz6S+fl5du7cyXOe85wN2ph1XfOqV72Kk08+mTzPOfHEE3nJS15CURSd7U488UQe9KAH8clPfpK73OUu9Pt9/vIv/xJw05gf9rCHMTMzw1FHHcXv/d7vbdg/2Je+9CXOPPNMFhYcEHjPe96Tz3/+851tgk7hN7/5TR796Eezfft2fvM3f/OIz3symfCKV7yCX/iFX6DX67Fnzx4e/vCH893vfjduMxwOef7zn8/xxx9Pnufc9ra35fWvf30rgHXktpkmujGGN73pTdzudrej1+tx9NFHc+6557K0tHTY8j73uc+xf/9+7nOf+3SWX3rppSwuLm5IEvaMZzyD4XDIxz/+8UOWu7q6Cmw+UHdJLN1gY9euXR0APdj/+B//A3C6r2076qijuP3tb99hvPysWJInbNu1g4Ud20EqllZXQSQIlQIKYwRZNqCqDeujMSvrawwnBXWtqbRFW8jyAXMLO+n150mSPuNxxdrahLqWSHKEyLA2oa5tdNyUUgwGfbZvm+eoXTs5ds8eTjjhBG5xi+PYsWMHeZ5jjGE8HrO8vMzi4iJLS0usrq1RFAVCwszMgB07dnLUUbvYvn0bM7OzDjz3g8b2VNLIognBVyFb0wMlUiqkUiiVIKygmBSsra6ytLTMgQNLrKysMh6OqCo3cJVCkuc9Bn0Homd5Rq/fANC9fsZgpsdgJqffzxn0c3qZQqAZDVc5sLiXq6/8Lldd+R2uvuq7/OD7V3LND67iB9+/kquv+i7X/OAqrrv2++zfdyPFZEhdFwhhGPQzti/MsXPnAju3LzAzmCPLelgrKYqa0XDCcL1gtF4xGpaMhpqqFBidMOj3mJ8fsG3bDPMLfQaDhCQ1WAosBVqvU0yWWVvdz9LiDey/8Qb23Xgd+/fewNL+vU5ffXE/w7VlRmurTIZDMIYkTZFJhpApwjNdBDBcX2Xv9ddxw7U/YP+N13Fg77Us3ngtKwdu8Fr5S4zWDjBeW2a4ssj6yj6Gq4uMVg8wXDvAaP0Aw7X9jMJnfZHx+iKj8Fnbz3B1H6O1/YzXlxiuLjNcXWG0tsp4fY3JcI1iPMSUBaYqsFWBrQtsXWJ06TQbvVaj0zcvqaoCrUusrbFe39EFWIKPYePHGI3R2rNXWpyiwATDETS01lRVRV3XLnAwRWHp5Tk7dmxnz549HH/88Zx44i251a1uxUknncStbnWr/9L7/yfJHv7wh/OoRz0KgDe+8Y28+93v5t3vfje7d+8G4JxzzuEP//APufOd78wb3/hG7nnPe3L++edz9tlnxzJ+53d+h8XFxQ1yAjfccAOf/exneexjH3uT6qS15swzz+Too4/mj//4jznllFM477zzOO+88+I21loe8pCH8MY3vpEzzzyTN7zhDdz2trflhS98Ic973vM65R3JOQD84R/+IS9/+cu5wx3uwOte9zpudatbccYZZzAcDg9b589+9rPc4x73YHV1lfPOO4/XvOY1LC8vc6973Ysvf/nLh9z32muv5fvf/z53vvOdNyzfu3cvd7nLXTbsc7e73Y2vfvWrhyz35JNP5ha3uAV/8id/wsc+9jGuueYavvzlL/O0pz2Nk046acP53xQ75ZRTsNZukJ/Zsptml156KUII/uZv/mbDuve85z0IITrSOp/97Gc59dRTmZmZYdu2bTz0oQ/d4DvdVF/3a1/7Grt37+a0005jfX0dgK985Svc7373Y9euXfT7fU466aSDgp5ve9vboq9/17velX/5l3/ZtD5tE0LwzGc+k4svvpjb3e525HnOJz7xCcD1+yc96UkcffTR5HnO7W53O/7P//k/h2jFjXbiiSfy7//+7/zf//t/47vhtNNOi+uXl5d57nOfG331W9/61rz2ta/tzP5s676/6U1viuf4zW9+85BtfDBN9IsuuohTTjmFfr/Pjh07OPvss/nBD37Q2ea0007jV37lV7j88su5xz3uwWAw4CUveclBz/PH7dcDHHvssUdM+vlJ9+sDIzfIa7hPAGmtC7K3gMKNBTQYjGj7LzaAiB4tib8DyNyWL9mkUNsCPtpgySas9YZN33ycxrfBBJ1v3Wi8N1rurQ/Ec6UFbsZjRmkNr0Eey/a68Z50YrSd+vi66UbjvKl76+MrEgHR6USqrWaK+0eQtdtGQghkAEvd1Z1qZNH5brefDu1k3DnWXuLHfdzfbRmdRi6nKScA/A4st35mg8WG9uroy4c+2PSTRhO/1VeMaBj28Txla99WQ21osfY+Dq+LmK8QKB8QUolEqRAM8nhzaEYRdPA11mqnvW4qtK0cKcc6jfZa11S6oj6CT6WLzt9VXWBwxCXj+5z5Ucq5NM/oKWizxUJvmMwCx+oJw9kAMnrIO7J1w4A3LHPbBlmM5mhufwUNcBnKlU2N1CbAqMVpWCZ6QiZGSFFQC0UpK6QcIUXqxOeTnWitSChAQFkNqbkBLWoSuY1e2iNPQciKGgl6iBVDar3EaLTGaLROIhXbexkTW1IYkHKBShgnuF9NsMUaph6SCoWSPVCGRCm0nMfKGZRIqKzGYEhQKDq3ebyv3XPXtZftXJsmcMAm16vTbq0rJuK3AA/Ah9+2fV3blz0+4MJDoluH9uFb8wjcdW/+3Aikd2pMJ3oZgizW9xW1Wd2marJZ+e2fMnLa/Bm1gwShjBAWJQDoAVBvTjb8HdowlGUt6LqmVw+ZN4sgJZlM6MsJGWtYM3SJ5KymNBmV7lEyQ2UXMHoVPRqyVg4RpmAy2YtknZm8x9xcSsoKtV1nZA+g1Moh2uGn1x75yEdy4okncv755/PFL36RP/3TP2VpaYl3vetdcZtzzjmHd77znfzWb/0Wz3/+8/nSl77E+eefz3/8x39sGCx861vf4lGPehTnnnsuT3nKU7jtbW/LeDzm3ve+N9///vd59rOfzbHHHsu73/1uPvvZz26oz2c/+1nuf//7x4G2lJJ3vOMd3Ote9+Kf//mfudvd7tbZ/qyzzuI2t7kNr3nNa44Y3NZa86AHPYjPfOYznH322TznOc9hbW2Nf/iHf+Ab3/gGJ598chzUX3rppTz5yU/mjne8I5/85Cd54QtfyLXXXssb3/jGm9HaXTv33HO58MILeeITn8izn/1srrzySi644AK++tWv8vnPf/6Qjuxll12GEII73elOneVhID49UD/llFOQUvLVr371kCDIaaedxmtf+1qe/OQn88pXvpKdO3dy2WWX8ed//uc8+9nPPiz76YYbbgAcyD5tp5xyCn/7t397yP1/Gi1JEwazs+w66ihWDuxldW2FSWVJen2SfI5sdhs7jrkFIstYX93nnqhSYqVCphl5bwZjYTC74GRXstQnrnRMhPlt25ibmydNe6gkwxhLmqYsLOxgdm6e2dk5elmGkkTW+Wg0oixL6rqmqqoIQAIkKmFudsDMYIaZmRl6vZxeLyNNnZSJ8FJnYeCopPSSZSFY2jCJp7UYhZJo47SuwxsgSzL62YA0UQx6PWYGfXq9nH5/hn7eJ1VJZMAkyk2r1J6FIoTAWE1VlhSTEctLLhhww403sHRgiaWlJYajIZPJ2L2yjWZcl4zHQ4brq9xw/bXkec7C/HbmfVBuZjDLYDAgTVOESkhVD5HkpElGpSqfYKqmCoxnaUkUqMRiMkuSOP1CkScu+VACZY2TbhE4h7M21JVGa6gt1MZJtBghUBgSYZisHWD/4l6qqoySLy6BpwIrkNQsL+3nmqu/x9raCjMz82hrmZ2bYfu2Webm5+llmWP7+EGLDdNKo66hQCXOhxR+8CdaXk9g9gsb3qmNb9h2xGUcSEaPERumN8ZBqvZSMXrKFQh6jc0AKfSfwBTqbN0C0IUQEQgJ323tUcfwclNt87yHYwpZoq5+a8r1z4Pd/va35853vjPvfe97edjDHtYJ3P6///f/eOc738k555zD29/+dsAlsDvqqKN4/etfz6WXXsrpp5/Ove51L25xi1tw0UUX8aAHPSju/8nSRQcAAQAASURBVN73vhdjzE0G0SeTCWeeeSZ/+qd/Go/54Ac/mNe+9rU8+9nPZteuXXz0ox/ls5/9LH/0R3/ES1/6UsABRGeddRZvfvObeeYzn8nJJ598xOewb98+/viP/5gHPvCBfOxjH4t94KUvfSmvec1rDllfay1Pe9rTOP3007nkkkvivueeey63u93teNnLXsanPvWpg+7/n//5n0B3JgC4ZJ/QMMfbtmfPHg4cOEBRFB02fdvSNOVDH/oQj370o3nIQx4Sl59yyilcdtllP5T+fQg0ffOb3+xc8y27aXbaaadx/PHHc/HFF0dCQbCLL76Yk08+mV//9V8H4NOf/jT3v//9udWtbsUrXvEKxuMxb3nLW/iN3/gN/vVf/3UD6eJIfN1/+Zd/4X73ux93uctd+MhHPkK/32fv3r2cccYZ7N69mxe96EVs27aNq666ig9/+MMb9n/Pe97D2toa5557LkII/viP/5iHP/zhfO973zsssPrZz36W97///Tzzmc9k165dnHjiidx4443c/e53jyD77t27ueSSS3jyk5/M6uoqz33uc4+oXd/0pjfxrGc9i9nZ2fh8CIDwaDTinve8J9deey3nnnsuJ5xwApdddhkvfvGLuf7663nTm97UKesd73gHk8mEpz71qeR53pl5caTjiVe/+tW8/OUv55GPfCTnnHMO+/bt4y1veQv3uMc9+OpXv9q5FxcXF7n//e/P2WefzWMf+9gNQHbbflL9+kPZT7Jf304k2v62JgDnFqyMxEFHspU+0SSOlChbwOMUFLcR8LZT/jHdHeJ3i+FrG8K1nSpzWhYkHCMWNXVsoONTRd/HUa6JyJPHWKx1eth+Y8Ksf9FitweMylVhWhO7kSgJMn8N6SaQPh2GddjR+RGM3+MMxjb42sI5p+/Z9mxIa63LVWTcDFO3orkuQZ4GQuJQgRIu+BCkUpx/awGfpNbn8nFjBk1ggLc17xswPOikt6VSmvoZA9JVztevKaPRjncgvpXt5moQRhsvpeh8QCClQsnEkXS8GKPx2K9rNy+/aIxn+ouI3AnrJG0aIqyIHVfrlgSPr48bvgXZHeGDDm5cpWtLVQVZpcP75kcOosvNomHWd5AAMtI0pBUt8X8PpgfwsbXMRSZ8OweWD64M0TmS8LIx/ldo+FiHBiN1Ivetm6Ua0q+WyPUqRowp1YhU7EXZkUvgaOZBKJTqY/SQ2lgSlWAsSDmgl++mn/fp56DrCswS8/kSghGaMbYcUckR/XyerDdmTs0wqQVDUTPGUNYT7GgfyfhqFtQ1zPaHSDGDTEunmS4E0gpyabDCRVkSqQjM7Pjwwt3swgZA3SID0C1w7dO+ZqE9Qvu09ovXKmwZH76iBXd3IfcItHeO0L5OXV16EQIi03USrXLjHwd5QIUHZwvo9rcvhoNNpWjJ+kw3RstkDNjEHkoDvLf6Nk3/CmB+WN8A6DGth09wSxPsMBahDapaYTZZIVMVQhswBcYWaDP29UlI2U1NHysT6rpCqT5pPsfq2oRJNY8p10ikIiv7GFuAtMjeyVT2Npu330+5nXTSSZFF8IxnPIP5+Xne+ta38oIXvIDb3/72RzxoDfad73yHT3ziE9zvfveLy9785jfz7W9/m/e///1Rm/MpT3kKd7jDHTp1uTmD1zvc4Q685z3vuUnn/K53vYvPfOYzvOENb4gaogAvetGL4kv4SAf1N9c+97nP8Vd/9VdcfPHFPPrRj47LTz/9dM4880w+8IEPdJZP23/+53+yY8cO5ufnO8uvv/56lFIcddRRneVZlrFz506uu+66Q9brzDPP5FWvehWvec1r+OhHPxqXv/SlL+WP/uiPDrlvWZa86U1v4qSTTuKud73rhvW3utWt2L9/P3v37t1Qv59m01hEopjfsZ1dxxzH6toaK8urzPRSdmzfxfajjmN2x1HUVoIpsLYi7/e58YYbQQj6s/Mu8YoxpPksvbxHms6iZN8BhEmCSlLP1s7ZtXMXs3NzzM9vc3ItidPUrsqS4foak0lBVVVY61gqgWm0MD9Pv9+nn/cY9Pr0+z3yPEcq2ZmuOv2uSLyj0QY2N0tmhN9VWIEQiizNSeb7LCxsQ/kZZmmiSKSbUVdXJcPhCGM0KpH+2AZtnFRLVRfUldN5X1tdYWVpiQOLixxYcuD5aDTyCUslaSIdm1v4lN9CoE3FcDhmuC5YW11GXqdIk5Rev89gMGDQ75NkOTOzOxj0BwwGM6RpTpomZFkPY6wPQGgmkwnGGPIeJIlAKoWQ0rEpPGCdKBdQUcoNLKpJTVVqikozLr18TlUzHq5jdcVkuMz62hpgEUlKraE2OJaNEiijKIuCG6+/lv17byDPeyiVcPSe3eTyOGZ7ApHOIKyb+utmiDonX4aZiGH6qGicgjCbzRgb668AYSV+FqwPfZuYk0R59k8IZlusS7TqZWGcjqJLBBumqIZ+EXXYfSBGKZdQVUrZAcanB4OhjLBde3kA14MufuiTUZtShMHnzxeIfij7+7//e4ANzO7nP//5vP71r+fjH/84p59+OlJKHvOYx/Cnf/qnrK2tRWmQiy++mP/v//v/NoDDR2LPfOYz498BUPv4xz/Opz/9ac4++2z+/u//HqUUz372szfU7YMf/CCXXHIJz3zmM4/4HD796U9TliXPetazOtf/uc997mFB9K997WtcccUVvOxlL2NxsZuU9t73vjfvfve73f22SQAIiPtMyyuMx84P3QwkD7IX4/H4oCB6KPOOd7wjZ511Fne/+935zne+w/nnn89ZZ53FP/zDP9xs+YxQ1/3799+s/bfMmRCCxz72sbzhDW9gZWWFhYUFAPbt28enPvWp6EsCvPCFL4za1AHIfdjDHsad7nQnzjvvvA15ZQ7n637+85/nAQ94AKeeeiof+tCHYj+67LLLWFpa4lOf+lQHhN3Mn/v+97/PFVdcEfvDbW97Wx760IfyyU9+8rDBlW9961t8/etf55d/+ZfjsnPOOQetNV//+tfZuXMnAE972tN41KMexSte8QrOPffcI5J7fNjDHsbLXvYydu3atQEsfsMb3sB3v/tdvvrVr3Kb27hx2rnnnsuxxx7L6173ujibNNg111zDd77znTg7p21HMp64+uqrOe+88/ijP/qjDqv84Q9/OHe6051461vf2ll+ww038Bd/8Rece+65hz3Pn0S//nD2k+zXB2ZwYEh3aZNeBFl6NQIPnEoPYja/26BkF+udBrvbwGXbL9lobQDR+sSiYkOZbb9nel0A0dvLw9/TRIQABnVVJQIgauIswQZzlw0udQhwO4KvZjMMvA32t35vslUE2ztBiUP4baLBypyetyRmdpy6JtAkTQ1JM0NQxLWPjy8gCIkurXXjIXyAw6UdsmgTsE9XljbuY4xx61rBjoChdsHuLpHECNd+QbbFGC8oLbrjrAaUb4Iuzp8Of7f6kohzhf25ynh8IUQE0t3MBAMiyOiE43hlEmu8z+/ptV4RImLE/pRiisLYz+LRmnr6yhpbeRa646EfRPe6YzchsSjYg95w3e0INzMBWKSJbLSWRaCRZh1CRMYxNGB6bQzaaASWLEmQfufO+MMC1iCMS9qorY+sVCOyaoW03EddXY/Q1zGXFswlglyso+yAVGZYayiFwjBPSYqRcwhxNFrMI1QPzBAl9pGY67DcQKIMIstIZiFDo9SILHfgd5rOUEwqTDWmLGr0eI16fB1y8H36ScVg0MMIi2UIakxuKowdgbWMrAQyd5FbAakGoPana5v2s4S/PcRsm9+RsS3ivIB2Mf46tUBtWuUEpNi29OXjZpa2BIyJe7ZnCzTL4pLYF7oVCTdceKi6o4cIYTMpJuy3YYgQygk3uW0vc2W0jxl+xVv4IIPZML7vBIDCti29qxAQCg/7qJUO1CKhEnNgJ5jJDdT19VTVGCsytKkg6ZP29pAlCYmqKcdj0n6OzY6mnFQIRqg0IxWzqGQB0j1kcyehVInI9jDWP1nOwY/KnvGMZ3R+P+tZz+Ktb30rf//3f8/tb3/7Ix60BjvppJM6ADq4wfuePXv4rd/6rbhsMBjw1Kc+ld///d+Py27O4HVaO/BI7EMf+hC7du3aNKt96HdHOqi/ufaBD3yAhYUF7nvf+3YGrqeccgqzs7NceumlhwTRFxcXN9VAHI/Hm+o7ghuoh4H8oezEE0/kHve4B494xCPYuXMnH//4x3nNa17DMcccc8hzfuYzn8k3v/lNPv7xj2+qIdkeqP+kOds/jBV1TZYoeoNZtu/exS2qkxn957cpakNvMMPCth2gUibjAqVSyqoGIdmxaxe61hgryPKcqqxJ0j4yyUgSwc6dOS7hjUGlCTt37GDb9gVmZ2dJ09SDi5rxcEJVVpRFQVmWGGNIU5e8e9Dvk2aZSxia56RZRpqk5GnuwU6IQV/jJ9tNgZWV0fF3+G47gsGstSiZYLRLdJkmKVneZ8eO7UgpSJQigaiTaLSmLCYIYVGJpKoKJpMhQz/zbG19ldF4jZXlFdZWlhiPRhSTidcYlPR7mX/nSpJEkaaZqxtN4lFjnUZ3VdVgoSw1k8mIlaVFB8AnCXlvwGDgNNRnZ+eYnZkjzXLiNF/PRBECinHFBMe4V9IlDnV6kIYq1/R6Tl81SRL6vZxeBgNjmKk046J0oHw/pSxGjJQmlzCTp0zGQ8bjEevDIXVdIwSkMiHr5UzGE3RV05ubYTAYMDfIUaJGUIMpsUYjcXl1guSZe09al8jVOyttRTaBREonJdQZjFiv52ibPuFC1wYTtDtFiF1X0XsI7+TA8Il+kw3sJMeECRbA80azsnVMP3jsAvHdgWHT51ozAIzxU23NFoi+iV199dVIKTu5KgCOOeYYtm3bxtVXXx2XPe5xj+O1r30tf/M3f8PjHvc4vvWtb3H55ZfzF3/xFzf5uFLKDZI6v/ALvwA4iYVQt2OPPXaDlvcv/dIvxfU35RzCdwDVgu3evfuQ2sEAV1xxBeC04Q9mKysrhy1neiwXwMLNZOyChN6hAMWVlRVOPfVUXvjCF/L85z8/Lr/LXe7Caaedxjve8Q5+93d/95B1Olxdt+6VH94e97jHcf755/PBD36QJz/5yQD89V//NXVdRwD4+uuv52tf+xq///u/32FC3/72t+e+971v9Lvbdihf99JLL+XBD34wZ5xxBu973/s6PmBgRf/d3/0dd7jDHQ7JKP/t3/7tTr8+9dRTAZdI/nB2z3veswOgW2v50Ic+xCMf+UistR0/9373ux/ve9/7+Nd//dcjyjFwKPvABz7Aqaeeyvbt2zvHuM997sP/+l//i3/6p3/iMY95TFz+iEc8YlMAHY5sPPHhD38YYwyPfOQjO8c75phjuM1tbsOll17aAdHzPOeJT3ziEZ3LT6Jffzj7Sfbrg+yJNQ0qIwRY4UgCkXUuQCjhNavbhJJmBl7j/zaALzT+yvQMu80DraL1wbOcuyDbZuz2zW0jgA4NYNyZ6dcCncOxw3mBjCC6c9+k17MOwHj7+JvUxS9q40sd8JfD8NBtG306tLXPU4TZAUogGx0F55t60kZdB412vI/orq379sESEagi1qPpIXAiox8ZgHTr/WOnLW68LnqY5eBndLbeoQ25w0y9W51fLmwAmh3xQ3hQHZxuekNACYlMg7RMA1BPN5s1FiPAXXqnLxLaLgQnhJCOrGNaGhbh/KwB7TTnre8TwpNtW/lBCVcN2TqvNh4Y29z6e85pwltlUeFcjkDw/MhB9Ahl+gNbIoAaoMV4sgEkte6GDpNRIsYbt2t9+xZxEQaHgLZBW2trElMABlEn2CRz8HBgH+GierYuSOpllB0DCVVtSaohdrRIVVxPObqKWTUiy/vM9iR9WZKLMbmaYO0SEolVM1i1C80Ck3oGU0NSDlHpXmR9FcLcCGI/WIUQO0hkRpYUDIsbEWoelW4DShISzFBjJ4bxaC/L6/tY6K2DGKOUIlX4m2KFGRbJbYaq+26qNgKpHLMrsrRa7dG0pfXXJz6yum3bxOvi2jAVJt4zwUFtt3e87tBZGYuysTMEprhsbdouqwuWd++oZvq1bVW1BZL4foa78i5JAYFp3ip3KjLQ7msSvARMWw4oVF/QJMadKpOmjQShH/tlwtWrHS1UIiTnaF5waIMQGqsSKjPD2IxI9AzDlSXKcpF85tZYkTMqFT21i362nVRlJJkGcTT1ZEJZf5fx+HtU1cQltxUZKp0lzefo9yxWzTJJDj1g+mm16UHmySefjJSyM7g90oE3bJzCHMq49a1vvWGAdtvb3rbz++YMXm8OK+673/0ut73tbQ+ZLOhIB/U316644gpWVlYO6nTu3bv3sGVs5lz1+33Kstx0+yNJ8vq+972Ppz71qXz729/mFre4BeAYNsYY/uAP/oBHPepRkVHUtte97nW8/e1v51WvehUPeMADDlnfn7WBellV6Kpi0Es56pg9WCuYFBV79y+xvLxGb3YZkUxYWlllNF5CKkUv72MN5GmfudkF6romTxOKokKiyNOMmRknPzI3N8fs7CxZlkVgtK5LJuMxRTGmqip0rZFI8l6PvN+n3+/T67kEoUmSkKYpaZpGBrCwBuunUDnnV0RnV7Sf1VYgpbtPNpPYCPIxwVGU/r2YJAm9PAMJdVWQphnaGCpdkkjAT/0sJgXFZJ26rlheOcCBA/s4sLSftbUV1oer1HXZTMn0TrNMJGGKpJIpUiUkKkUlyjMnBFY4cDswkJLcT7P0gxSj3fRLbUqGwwnr60ssLl5Hnvfo9frkeY8865GmOUq69svyPkpkOM1y45xccFM6TUVW5OjaUueQZZZE1CSJIlEJSaLo913gfjKZoOtZ6u0L1MWEYjJmNB6xvr7Oysoqw/U1ymLi22fs5IJ6PfYccwy7du6kP9uj109REnRVuPOTPkmQJE69DD5fonKndtlhZPnL653zuMwntdLaBU6k9ImYtEu4KpQE6wcAAmLSLOE0F9sMH3c8nwdU453qZmDXZlu19SfbfW2zAWLY39XR+TVplqKkQtNmpjtiUHsq9JYd2fP3l3/5lznllFO46KKLeNzjHsdFF11ElmU88pGP/G+o4eHtv/IdEvrL6173Ou54xztuus3s7OxB9w/vx+ncJkHGJci6tO3666+PeSwOZh/60Ie48cYbO1Iu4MDL+fl5Pv/5z99sED3UdTMZti27afaLv/iL3PWud+Xiiy+OIPrFF1/M3e9+9+hHB/9x2gcG52N+8pOf3JA89GC+7mQy4YEPfCCnnHIK73//+zf4tfe85z15xCMewStf+Ure+MY3ctppp/Gwhz2MRz/60Rv6WzvhLTQA6ZHk6Zmu3759+1heXuZtb3sbb3vb2zbd50j83MPZFVdcwb/9278dFBifPsahxgxHMp644oorsNZuGDsFmw5SHHfccQcFwDeznyS//qbU9yfRrw8AetOmPvefCqCkiMQApYjgeQDSG8A5yLN4RrBosIuwvv0djqe17hJRp8EU60BPKwPgvPn1b8+8i/Iu1pc2tf00scWfdiQdBrDcnZ90eQClJE0VdVl5387lODI+H0171t9UzXwQoN0Huoz8jlpC7Cuxski5kTwcmfYBmY87+WsZ8SMPhrfQseCPtttKa91ibm8kYzRYm/Od3fWXINwMS9cn2ggcDbAeWPgWhAgi0e4rAt3hEB44bPUsBDKObcL2xhiPIzbs8LCXtWC0bUr0hQUiS6NP0e27RoMQScxhhZUoL3to8TivtS7BqTARBFf+tKWyKNm0r2svfH8RnTZvAkBNkwVijSP6GDaoWRzEbrImeviW8WJ38dUgfxFWtIHcNrjr+oT1v1vZZm0zcPaxBaejqStsuUpOgRCg9QCTzKKSfpyerG2Nrkak1RqyWsXWBowEUzIpJoi6xpqCmaRidgAzPUuWC5JsFpI5MAOEXsLYhNrkWJEjzARhRhTlGolZJLX7kOIARbVKVSuknaPUJePJKpPJXtYne8kHJzDRQ8ZGI8sD9CtDVR+gZpmiOkCpl8HuAJGBFAhbocQyuoa0SpixFetCYYVESKcBu+k1mfrdKEq12ri9tb802rqOpUTTgRsL1yo8JMJd5njYzfOhPQugc5RYihCt/tB6OIfO3ImI+Y4hbTNTIcwgClHCMKOlm0Q0ROimWycscytU3LaZJdGVZWnKDBhNbHZBA6Aj3A0cH/audlHfPz4svLa8sgjpIlwkKZOxwk5q1kYV0s4gTEbWP45MzKPl0VRqOybJMMpQMcbWyyRpn/m5nYzW91MWIyozoLSzTOoeSW2Rss8mDfAzaQefLXBk538kUzMPZjdn8PrDHO/HacYYjjrqKC6++OJN1x9sQBBs586dmw5s9uzZg9Z6w9TKsixZXFzk2GOPPWS5b33rW7nTne4UHe1gD3nIQ7jwwgv56le/uiHp0YUXXsgf/MEf8LSnPY2XvexlBy37Z3WgbrXxICVkSc7CwnZ2H10wKTWr6yOWD+ynNzvP2vIilSkRwqASRZY6VrgUgl6WI4Ugz3LmZmeZm5tndnaB3mBAlmVIKdF1RV2VjMcjyrKgKh17MU0SZgcD+v0B/X6fPM+d9IvfTynnEDvpE9H1B8K3HzDI6Og0btj0AAECuLqRNeNcOIsxNVU1QcgEY2q0BzO1LrECTF0yGo6YjEdMignj8YiVlSVWVpcYjdapqhJtKhcMTxKQEqEce8I5nAlSSJRMkCpBKQeiCr8eMV2/AKIDnj2idY0xGmyN1j4x+XiN8Xjdn48kSzOSNCPLcvKsR6bmkMIxzZPMgetSKRAWayqslegKqqwmTQRZqkiSELxQCKmYGfSBHKMH7pp63frxZMK2tXWG62uMR0N0XbG6vMR4MkIJgUpTkix3si6JQCUZiVI+yaaTVdEWFyDBa9lLRVV7MDuwpQCplBs8Bra4cQnardYukVAEnm1k7Fhr4mxVYy1CuQFAHBR4VhO+50ghENKzeQQERn970N0eGIZlYWDUBtiB2H8BqqqiKArKsnADHiWQWSMTA46F1O7LPy92sPO95S1viTGGK664IgaDAW688UaWl5e55S1v2dn+cY97HM973vO4/vrrec973sMDH/jAw7KvNzNjDN/73vci+xzg29/+NkDUfb7lLW/Jpz/96Y58DDT64qFuR3oO4fuKK67osOD37dt3WEAwyLTNz89veNcdif3iL/4iAFdeeWVn+XHHHcfu3bv5yle+smGfL3/5ywf1eYLdeOONADHAFcxaJ9sVcl7cHAt1bbfplt18e9zjHsdznvMcrrnmGoqi4Itf/CIXXHDBD1XmwXzdPM95wAMewEc+8hE+8YlPbJBdEULwwQ9+kC9+8Yt87GMf45Of/CRPetKT+JM/+RO++MUvdnzqEBSftsPNkN+sfuG5/djHPvagxJjb3/72hy33cGaM4b73vW9nVmvb2s+dzep5pOvaxxNCcMkll2zaXj/MGOUnya8/UvuJ9uttg4cEJm+bmOf8GwcaKxVA2UaKpcFvGnA8MpPlRobxdPA/AM/T4Hqom2NFG6+V7T1U0S2rvV+b9R7R/E2sjTtZa1uIT5Dkc36/MZYky6irkjybYX52nvGkYFKUDEdj0jSJx1ZKobVpYZVtJGzjrNTmt23+baOq/l/nF0NtKoxuJeUMSUox0a8LrG0pGiLG9PnSJs8kiQfQm9mMAUQWHr9yRXhAWrbX+3MTLk+RShKClng4l5hI1TQgeRs0b+rXopd6QEsEiWKhomRQCOKEGZw2XuMuPdURgUI7+3PxkuCyhcsFspGUbv8wFnRgv/Qy+QJrLEIkYKUH6X07yyAjY5DWeulK6fNmhesvIqjuAgdNvWPQyTZ1ghCgsvxoNdGnynLtMP3iam5m0f2nBXy2JV7aZdsohdFQ8q0Hcw3CaoQp6dkVMiZYMWBoC2q7jcoOUEq5gWhdUJUF9aRAV2MSW5OICsyQ2lT0s4x+osjkOlIWkOymTHZi7Qy6zhlPxiAqRKaxtsDUJTklGUM3vbiG0hYUtWBUCOpqgjWC4WjMaCLRxqLW16iEptdL6KdLzOU52/QadaYZ9GsSUaPNCF1WCLUfIeZItGVSaYxOScWARE8wqUHJbttPx0Ys1l+GdtDCt6uPFAUpGCEsRa0ZlgUCSS9JSROFFaJVaMMsR8Qj+DVt0X66BwtRR6bB8+4GMQYlRLzGbYA9giRA4stvonjt/tK1DnjfXiJCaonWFq12Eu2dW8uEDAz2dvkWRLO/EC1gP9QrHsa1YyoFRgmE0Nh6hKnXqKtFrE2wIiGVil5vBsECRb6dsr+LJMkx5QTZ76OEJRXHkqQ9VLKXlQNXIntHY5MdlAwY1QolM6y9+QOUn2S74oorOuyL73znOxhjOoPbmzLw3sxuectb8o1vfGNDYOdb3/pWZ7sfdvB6pHbyySfzpS99iaqqDjq19UgH9T9MHT796U/zG7/xGzcrEPCLv/iLXHzxxR3dTSAOxL/yla90GOFf+cpXMMYc0UB9M6CkqiqADQP1j3zkI5xzzjk8/OEP58/+7M8OWfaVV17Jrl27Dhsg+GmzXt5zshfWOVuDmVl27drNeDxhefl73HjDNezcvRslDCpNMcIl90wThUokiZDkec7OnTvYvXMHMzMzpGmGFQm1dW0/mUwoJqMIpAvhkosO+j1mBzPkeY/My7Y0YKN7mzknT6NjAFeQKkWTAcPLeBCcMhrWBpYw564NpreBzg7rxBqSVJLnKXmWgJCoBOp6wnB1lbWVJapqQjkZsbqyzGgYEqBWVHWJriuS1DG3M+m+lVJYpbDSBwOkQiqn6xf0/ZqkOZKOSJn1U0nDt5uziLEGaWpPky698+edYs9c0UZjjaYsRhSTIWsoElbAszmS1AUq0ixDJQlJljome94ny1OUgiyVpGlGkvrZAElGlmUof14yUaQIehZmjGFuoaCYTJhMRlRlQTHZw3BthdFoiBSS9aKkwpLliqzUZFlGlqROWiZeH4m1wiVnNQasdl/W+EFC4ogS1iX3cYlctZ/ybPzstOhFErwNx5BpsflbjrEQIIzrc2GA0gRiBELij9Wd9hz6UwC+p6dDBzPGUNd1HOBE9rtSMSFu2D8yzPx206Djz7oF9ury8nJn+QMe8ABe8pKX8KY3vYm//Mu/jMvf8IY3APDABz6ws/2jHvUonv/85/Oc5zyH733ve7zuda+72XW64IILYmJRay0XXHABaZpy73vfO9btbW97GxdccAEvfvGL435vfOMbEUJw//vf/yadw33ucx/SNOUtb3kLZ5xxRuxr00kGN7NTTjmFk08+mde//vU8+tGP3gCK7du375DvsOOOO47jjz9+U7D8EY94BO985zv5wQ9+EHWaP/OZz/Dtb3+7k5+lqiq++93vsrCwEBnsAQx83/vexyte8Yq47Uc/+lGGw+GGZIQ3xS6//HKEEDHp5Zb9cHb22WfzvOc9j/e+972Mx2PSNOW3f/u34/rgP077wOB8zF27dh1xskchBBdffDEPfehDOeuss7jkkks47bTTNmx397vfnbvf/e68+tWv5j3veQ+PecxjeN/73sc555xz807yMLZ7927m5ubQWv9I/PmDBQdPPvlk1tfX/0vHDNPHs9Zy0kknbQDof1j7SfHrb4r9JPv108zpdg6WAObFTxJY6H5GXUA2NoDoLtk9pvFX2mPb6dlzDTO6AVGtBxsb9rEjbYQkj5uV0ZEyaflW07YZqO/8sZBsHoSUOCkbidGWLO2Rpn2qqkLKlB3b50izIevr6wT6aF2HtiPWO+RndBBTQM+62JTt/nNIC+copUQmCbrWWJ/MVOuaZqbs9EnT8jldwMB6/9RaQZKoLqAt2kGSFhCPRKBiWW7WqUQK5cBmglwMEUBvgHTr8mO1fN+2X9u5JsK2jhFA2Tbo1/QLaw1KBZwijGviafs6eOhMCDdcC+ciWnI1QsZlMhCIrXInk0gyU6NlTS1qN1PXWoSxKOUPZgzSaIS0bgwWAgFCYk0A/wWmlfXUGi8naZt+I0R7zHjYLnFTNNEbMDU2tr9FwvBvGjgNrLLmd7NPaOHOPgE09chmGCwbax2jpxiRiDXm8gIhSzIsazWUUmBkD5d2ykBdYfUIoVeoixUMmkFqyHOJNDvQ1FT1Ook2UA8RasKkThmNJqyu1VR6DaFSUlWRp32E2ka/twtrl1EsMy5HrAxXWFpfYFysA8ZpveoUbSymGCOTMXMzkvlen4wJJWOXoCyzKGmo6nWMECgzi5TraJ0hNFDnWFVi8dlqVYhouZYWQkSwAQtqE/C8bZU2VLpCCacbOimGUA4RMsOIBUh7yBaGHqNxAo8gdztSiJYSQZDutevUI4DLraVtkL316N+88hFEbxSrOqC37f4dQO3u0fx+nYdE9xBt7HtDIAJoZ0gmgP9+mRU+QWl48OCuldYVyhRkoiTRI4xeQugDSHMdVXk12IKiAlPuJeFo6N+SKp9H5jMI4fRsRSmgUqBmkImh14dJcgPWlIhsANl2tPJTw8v1zdvwp9z+7M/+jDPOOCP+fstb3gJwkweth7IHPOABfOpTn+KDH/xgTCw6Go02TPH8YQevR2qPeMQj+PjHP84FF1zQGbhC4/Ac6aD+5tojH/lI3vrWt8ZkP22r65r19fWoZbmZ/fqv/zrWWi6//HLuda97xeX3ute92LFjB3/+53/ecbb//M//nMFg0Lle+/fvZ//+/ZxwwgkMBgPADdQ/9alP8e1vf7szQHjve9+LlLLDHPqnf/onzj77bO5xj3tw8cUXHzTRWrDLL7/8Z3KQnsgU93JxDlWiErYtzFNXR7F/cT/XXX8dq8uChYUdkGYolZEkCoElS1N27drBMcccxdzsDHmaOlZ0NWFSW8ZlzWQypiwKr30tmOn3SDMHoPfzngdnU8dsD/JkeF/AxyWllFHqJK6jcWQEotG+E84JbRxw1fp743dkOgiBtBpKjcRgTE1RVlirmRRj9u+7gcXFfeiqoC7GTMZjqroCa5FKOtmZvmPPJ0kSgwFSOhDdKD8VUSQusadQSBKEVMiQcd6/wOKAQ9jGB7LOGbc40oD0moeJzRzIbgwajbY1UhqU0Gg0BG1CC1qXGF1gKgMTCV4XXciEJMmcDEzPzQZwCUpTB3TnSZwdkKUuoWuSpgh/rlIpUIp8MCDv95mp5zBaMxmPmN+2jWIyoSpLnzjTUOqayXqBEBVZmpD59koSSZamZGnqWCdBI9072tZaDH7ALMBKx0B3717jAupWdt7Rxv9jRJtx5K9/i2UVzLGVujPwjDbuMyUDFFhCwQIwfrApymHAGspJkpQsdzJHbuArWoOlxqf7ebJTTjkFcEnjzj77bNI05cEPfjB3uMMdePzjH8/b3vY2lpeXuec978mXv/xl3vnOd/Kwhz2M008/vVPO7t27Y5Lrbdu2HdG7fjPr9Xp84hOf4PGPfzy/9mu/xiWXXMLHP/5xXvKSl8T3+YMf/GBOP/10XvrSl3LVVVdxhzvcgU996lN85CMf4bnPfW4MsB/pOezevZsXvOAFnH/++TzoQQ/iAQ94AF/96le55JJLDsuYlFLyV3/1V9z//vfndre7HU984hM57rjjuPbaa7n00kuZn5/nYx/72CHLeOhDH8rf/M3fbCAOvOQlL+EDH/gAp59+Os95znNYX1/nda97Hb/6q7/a0U2+9tpr+aVf+iUe//jHc+GFF8Y2ut3tbsf//J//k6uvvjomFr3gggvYs2dPlA4JFhIG/vu//zsA7373u/nc5z4HsGHG2D/8wz/wG7/xGzdb0mHLurZr1y7uf//7c9FFFzGZTDjzzDM7/W7Pnj3c8Y535J3vfCcvfvGLo6/3jW98g0996lMbkmcezrIs48Mf/jD3u9/9ePCDH8xnPvMZ7na3uwGOKbxt27ZOPwyg62b6/D8qU0rxiEc8gve85z184xvf4Fd+5Vc662+qPz8zM7MhMAjOl37FK17BJz/5yQ35mJaXl5mdnT2kdONNtYc//OG8+MUv5pWvfCUXXXRRp12ttRw4cOBm30c/CX79TbWfZL8++BJtoDuykr3utWPnClTiwAnZAvcCwClEyN0SyIgSI0yHAHBw2bgGQHfJHBs2eyOvF9bHP/3vjYB6JAqIDtrXPeIGP90TNIX25xLAboFKEvKsz/LyGtdfdx29fo+ZmQHbdmwnTVPquo6SKNNBA8eIb8SO2yB6xLIsLjjQPbXWORp0bVwMQQi01kgpKYrCkT1T5WZa6Zosz45Ini8QKDb/7XzcRi44AOktv9Yzv6NOufDYAMRr2YDSsrMvcVzl+k1sh4j1CQRNQCLWy4C2genfJBt1OK2JdW4usgu8hD4UpBQt0pNTWz649efdGq9JmaCsY7NKqzBGU6mapNaUtnJsdFwy1QB+S2md6oNydW/nQAp4MkhXX+MCLO3ZG6F93D6bz3qatiN+cieB7ROCOn65zx/ctIP/S9AGK1sqOB2EMjDPGjxUBOC1dTGkkKBSSHIwKcKOyZmQqQR0wrLpg83QuiCtlknsIpXdy7heYVQsk1Iyk8zQy3ZSTOYYlTXaLFHrEbNikUx+h1TvQBfrjCdraHFLUrkdW2bUdkAy2M6AjEzWUENZ1KysLnHN3gmVmfFT3UEJSVlW9HqSnbOC3bND8nyItRWKlLqqMBRgDdamSHEUcCzGKIQZoaRAqYRaSmwckAdZExujZcrdKVPAtQ9nBPwbgbGGsirR1QghKqgnzFSLKDOklvMUMqGSCVmaTEnGhIJt67oKjyE3qPXUOLK178aBofQf00kqGgDp9pbTg1PaW266zoYHcBs0bz80NgsytHT75QbkvBv4aVaL+Fv6zVxfFa2AAWij0ZN1ZLmXXK0ykBWmPkBZ76UYfwej91ObWSo7S10LhB2RKe1Y68IiEoXQAlSNUQPGpcTYHCNz0nwbaT4gzXrk/RmUSkmkhvpncyB+5ZVX8pCHPIQzzzyTL3zhC1x00UU8+tGP5g53uANw5IPWQ9lTnvIULrjgAh73uMdx+eWXs2fPHt797ndHBy/Yj2LweiT2uMc9jne9610873nP48tf/jKnnnoqw+GQT3/60zz96U/noQ996BEP6m+u3fOe9+Tcc8/l/PPP52tf+xpnnHEGaZpyxRVX8IEPfIA3v/nNnUSs0/abv/mb7Ny5k09/+tMdZ7vf7/OqV72KZzzjGZx11lnc737345//+Z+56KKLePWrX91JYnXBBRfwyle+kksvvTQyl174whdyySWXcOqpp/LMZz6TnTt38nd/93dccsklnHPOOXHa6NVXX81DHvIQhBD81m/9Fh/4wAc69bv97W/fccz37t3Lv/3bv21IZPszYT7RpcBgtQHlkmjOzgzYc8xuJpMhKysr/P/s/XmwJflV34t+flNm7r3PVFXd1dXd6larWy00ICSZx/QYBPIE13AffvGen238MGHCdphrPyIcxiF8fYWxsAPMw1wHhIDwADiMDcHk0AUssP1sggBfwAgMFkYSUs9d1TWcOsPeO6ff8P5Yv8yd+9TpVmswkqF+EVXnnL1z58785W9Y67u+67uawrHj5lhlsdqgdGIxm7G7WGCUghhomo66XlE3DU2fpHaH1lSVY17tUZUF82pGWRYsZnNhno8a06L1qLJxrc2GbTGmKA4rbdYFFGYyTPeUKTNDflo2hs82eD41qmMMKJVomzW3j25x69YN1nUNCg4Pb3F6ckRTLymMIoYeSDhnRuDUGDumbdqs366UEckSY+mV1GjROb1SDMycYqlkB5QtXG32W8VgWRGJ8owQdodKERVAh4F9nbAmZVBZtMH7viPm1MoQPM4JMyZGhe+jsLmTIvmevm+pmxXm1GCsMNTln6MqCqqqoipL5vMF5WyGK0q0NrhciFQplZn3wsKx1rJTluxyQPSBrpPCsb1vqeuaZr2m7zvWbWC17iSrwRicMZRlyWxWUTiNNh6lE8oqyM4QZMdi0CfUklGnU05XzUGFOHHghtomWm1Y3ilOjPdJyuwwHmFjQE8dzoEhPh0/w7g6q/05/G6t2GyDRicgBWWtJaYwZiPEmLbA+T9oIPpnfdZn8c53vpPv/d7v5T3veQ8xRp544gkWiwX/5J/8Ex599FF+4Ad+gJ/8yZ/kypUrfOM3fiPf9E3fdO65vvqrv5qf+qmf4k/9qT/1knrdL9WMMbznPe/hr/yVv8I3fMM3sLu7yzd90zfxjne8YzxGa8273/1u3vGOd/AjP/IjfP/3fz+PPPII3/7t375VRBN42ffwLd/yLVRVxfd+7/fyH/7Df+BzPudz+Lmf+7mXFQz44i/+Yv7Tf/pPvPOd7+S7v/u7WS6XXLlyhc/5nM/hL//lv/wRP/8X/sJf4Lu/+7v5xV/8Rb7gC75gfP2hhx7i53/+5/nrf/2v8/a3v52iKPgTf+JP8B3f8R0fsX+LouAXfuEXeOc738lP//RP86/+1b9id3eXr/zKr+Tv//2/f0dw4H/73/63rb//2T/7Z+PvUxD9+PiYn/u5n+Nd73rXR7yvu+3lt6/+6q8ebbh3vvOdd7z/7d/+7XzZl30Zn/d5n8fXfu3XUtc13/Vd38X+/v5WpsHLbbPZjJ/6qZ/ibW97G1/2ZV/Gz//8z/Ppn/7p/OAP/iDvete7+JN/8k/y2GOPcXp6yj/+x/+Yvb29F61f84lq3/qt3zrOvb/4F/8ir3/96zk8POS9730v/+7f/TsODw9f9rk+8zM/k+/5nu/hW77lW3j1q1/N5cuXedvb3sY3fMM38O53v5sv//Iv52u+5mv4zM/8TFarFb/1W7/Fj/3Yj/Hkk09+QqVGHnvsMb7lW76Fb/zGb+TJJ5/kK7/yK9nd3eWJJ57gJ3/yJ/lLf+kv8Tf+xt/4mM79ybbrAX7zN3+Td7/73YBkJh8fH48BuTe96U18xVd8xXjsp7pdLyD6QILYSLoMyMvGNiYTQabg9LbNK79vgO6zNvA0Q24qv7Jlf2TmMgyAvBqLYJIz8c8C52dB9Ok1vRQTfeunCgKg6wTKZh9B7qUqZ1x7/gWuXn2BFOH46JSkIhdu3+bhhx8mhEjT1FhjCDGOGOKoIqBStg0nCGW2+YSEPOBmL0bmFGZ0ZCO7orVmb3eONoaisDRNw8np8Wgz3tkmIP3wkFTWuNcmBy6G61Ib4P8MrjW1VxnfSjmLcgDR0xhUYevjG+R287i2pS630N20GQeDz5WSkFoGeZfhX/AJNQRtzNDnGw31QV9cKUVEZYmi7X4ayFKSgWBxxmGwGVMzxBDoTY/XHq0LCL34ClH02LWKcl4dCF6ITwqVA5Tn2NqTAEGMmwxRYcLHO0jjL9ZeNog+MoGH/j0DpsMGBJffN2Ar6sxUyo7jFJAdAfRhUo+Rg4TVwoT3IeGjRtkCQw/UFGqOiy0hrNG0qHBCqH+X4+MPcrI84ugksF/tsjdbYM0cbGBdN9TdnFkZMeoUk54nhNskb3HucZx7FcosIPQYM6PtNUfrFlt6dN+yrgOHx5abhwkfOg52E4uZRH1S8qTk2V3M2SlrbOFp+lNC6vCxwPsGp3pccRFrXklU9+BDSesP6f0KbfZRqhQmm5GJJGxnPU4olSYLxDb6y7jywCai52t0f0QZTjDpBik1FIUihFP6UGGtxoyq4cNDVkOAbohJbUmbZH9167szjD+cZZx4m7rEU6x6k6YC2+eajqsNyD5lrr9Ym7DmR8A/bbpmcgGKgeGYr3woInD2C9RQqHU4Po2f0+Ng30y3GBOpr7F+he2OSOkGXteksAY0yd1L4BTldtFmj46IdvdQlCUrLX1tjEFpi0+RloLe7BJ8gS0de1d20NUCt3MPys3RtkTRYfTvz+JkP/IjP8I73vEO3v72t2Ot5a/+1b96R9r2R+t4n23z+Zx//+//PX/tr/01vuu7vov5fM5XfdVX8WVf9mV86Zd+6daxH6/z+nKaMYaf+ZmfGdNaf/zHf5xLly7xBV/wBbzxjW8EPjqn/mNt3/u938tnfuZn8n3f9338rb/1t7DW8sgjj/Dn/tyf4/M///Nf8rNFUfBVX/VV/OiP/ugdTPav+7qvwznHd3zHd/Dud7+bhx56iO/8zu/k67/+6z/iNX3RF30Rv/RLv8Tf+Tt/h3e9613cunWLV73qVfy9v/f3tjQnn3jiCY6PjwHONaC/6Zu+aQtE/4mf+AnKsvyUKUz3iWxRDTVMJMDnY8BoTVFVXLlyhd53pOSJoUMlj7VgjAR0tbX0PrBqGparJTF0WKNZzOfce/kAW8yEaewKnLUYa9iZL0S+YwTFN84AbIzuQd9vYJxLGwyZzesqg+hpkOpA9rZhj4jRo1VOy8ufEYA1EjK7uO862rZhvbrN9Reu8fQzz/D81at0fYcxGh88wXcYwBlL0laKQBqRZtHayJ6snRi9xhG1BmVAW7RxWC0Ar9KDdEdmgmDGaOwgRaKNmWzi2cBMcfN3BsaVBufy/cQMzqaccacSRlmRPVPZwA0N4CV1s0iENOh2S7qr954QOryvqWvpZ2PMKLVTFCX7+wfsHRyw2NkFZWi7FkkZVZmBr6WQqSsyu9zgKoerKuYpkUKg7xopLNu0tG1L29b4XrTV1/Uas2qoqpqyLLBOYa3GFRaXi39m1wYtO24Gq4fxwYYtNcivDJkGWsvvTHTVz9ifIYSsZ7nZMzdppdKPMb8/MFW2AvSjQzr9LGf+ViPT31iFilnLMRczHRzUQXP0D1r723/7b59bn8Jayzve8Y4tAPul2lAQ76Nlxp5tjz76KD/7sz/7ksfs7OzwD//hPxyz3F6svdx70Fqfe9xQNH1oX/zFX3wHSAHC1v3xH//xl/yOF2tvectbeNvb3sb3fd/3bYHoAG94wxs+Yl888sgj517ThQsXXlYfAed+/rz2/d///Vy6dIk/+2f/7Ms6/m57ee0rvuIruHDhAjHGO4rBgkgOvec97xkDSs453vrWt/Jt3/ZtL6vA5Xltb2+Pn/3Zn+WLvuiL+KN/9I/yC7/wCyPx5Yd/+Id54YUX2N/f57M/+7P5oR/6oY/5e15uu++++/iVX/kV/u7f/bv8xE/8BO9617u4dOkSb3jDG/i2b/u2j+pc73jHO3jqqaf4B//gH3B6espb3/pW3va2tzGfz/n5n/95/v7f//v86I/+KP/8n/9z9vb2eM1rXsM3f/M3b8mifKLa29/+dl7zmtfwnd/5nXzzN38zIAGyP/bH/ti5z/rltk+2XQ/w3ve+944A3PD3n//zf34LRP9Ut+uFaCGA3VgAcgvTnrKQOQu65GO20BKmgOFZwPpFrmI8zwinpsH+2djswno/e33ba/gWI53tY6d/3nFdagPUbo5UWFtw+/YRzz77HHXdUdiSrmvQNnH1+RfYWeyxu7eL0S3ehxwgIIOHm/MPNW+2b0Bt/p0Fq4drUUBMxBQIUfwJZwv29/axxtF2LdY4dncKvO+pmzVnjMEJWCs0mU2W7eSr5SqzXKOMhW1YXzFkm271cxRcLir5rFZD4CPbmimNRJMUU65MFO8YG9vjIwPmMY726fC2EHYCIWywNYHZzIYIpSf9OgQG8nVJ4U8BygV5Ex9p8PlGRE7JuQxGOPcKjHZSY0ppjHJCOIp57pAV9dUwlsII6m/Oe0amZejCIUiUx8uYeTGC+i/dVHqZlsyf/pmapIaHmiaTfXJBavPYN93E9gBVG4mXUe5lBDPZDKzx2hNdH1itTlHrQw7CIfcWNbtlQFvFMu3Q6At0uiK1S7qjpzm9/n9y6/CD3LxxzLO3A5cuXOB1jz7ClQsPYSkI7YrYP8P+zm129k5xpaZu5xwt76dVr6PXF0m6oog9qe9AK8qq4dLsGiY+zdHR8zxzo+X2SSIRcLqmsEHSOwxcuph45YMzLu7toE2g7Y9Yr1f4viIGT1laZovHsMUbaP0uXRfx/kgcq+JRevs4p+V99HaOwoG2KFMII18hwyVJx55ddiYBJryPnKyOscvrVP4Y549J8ZY8GHtAUzxEN3+AarZHYSepC2kMcGWOnCDmMW1/1x2DCe64nulydfbDm2c8Bd03mvlnmVrnDudxoL/EMFaDLrycQavNtY7XcAe7cdg4pt+fxs9uvnu4bjmFDwHVryi6I8zyKi4cYWmwpiCkSN8v6esbxKAw9h6sstx73/0Ui0ss7UXqYoeqKggB+qalPr6Nb1Z0TcPMagrd4aoZyc6JpsS5AkeL7W7zv/zhz37xPrjb7rbf4/bhD3+Y1772tfybf/NvRm3ZT9X2lre8hS/+4i/mO7/zOz/Zl/IJb//rt3+XLHYxEEMgxR6toCoL+q7j+gvXeO7Zp7lx85Cy2uPiPZeZLRagLLYoWCwW7O7tsDOfsbszZ2desTOfUbiKQUpFZ81rlQHic0x9zlujX9S4H4FodcdPNdodCoiE0KO1IuSiP33fU9c1dd1wujyhbVrqes16teLw1vOcntzm9PSUtmsxWkBgpSQoKqmzci9B2wyQC0iujc1SLTbfdwbQB7kWPdWo3AZmt+4ZYahv9Y06y+ZJ4z+dfAbVB5ZQ2jg8EzYQKaFyICTmIEKKoq8uuuKe4CXtNAR5z4eQGfoWZeQe9g8ucv8DD/LAg69gNptnOb2e0AvYG2McJW2cc/mnlaCL1hiV0ESC9znNVcZd2wpDfb1e0zRNNs4F9C5Kx2zmKAuD1aJzaLRCx0SIIrWiUsKYTb8NRT0HZtC04NJ5KcbDc5l+Dhg/N/Tjecyq8amcZW7dMTYV3nuxBZWiLEVKSQB0cY5i1MQwCZgQ+f98/Scm8PkHrX35l385/+2//Td+93d/9yMABee3r/mar+HHfuzHsrbqH6z2y7/8y3zhF34hH/zgBz/uGir/vVrf9zz22GO8/e1v5+u+7us+2Zfz+6p573nggQf4iq/4Cv7pP/2nn+zLudv+B2l37fpPXPsLf+l/GrGHgbW7TQjeoCgmF4gf8ZIR/Gb80PBrjGkDYI7M5an9yOTn8N0TdvaYxWepqgqTZRiVYrSdQgiSedj3W6x0ubgBJE6ba52CkiMGtAH+TVY7tNaJ5B6Gg72L/NZv/leWJ0vWdUNhC6lLVIj0h7GG1772tVy4eIEbN27gvSepJAx/srQH2981IJBlUbFYLNDaYq1DKYVNct0+dNR1Tdc1pBQJMdF1nqqck1Ac3jpitVrTdi27+wsuX76Hg4t7XL36LKg4ZtlmDcot21RpJXK9eoIdqU3/paSJUYnud4iTvt3YmCL1bEb8qixnzKodseMRwob3PQlP29U0zZoYPSlF8dOMZsou3wq8DMRnLfa50Xp8duJLSLQnZawOJbLSWgshRxvpQ60HWcSM8aokPhaGyjlc4YSA4+xISnG2wtoCa0oKV1EYK98bA023ZlUvqZs1q7qmaVtiUmhjCT4SfEcMvYQqVCZpGbAu97UWf3HA9fQAmE+xbLKufJRwg1aKf/B3f+Yl5/BHVVhURPrz/M6TWU0jNpwJVA2vpQ1QTnb+hGwkiclTsHUE59UQFSBHUBRKW2IqQQcUjTgloUPrJUZ7Qr8i9R2+9dAaVKrYn0nK882jI/aqi+xUBW5+gIoO7HWa/hrr5pSYdqiqA0LX42NNXa/pfYvxgT6d0jWHLOKKylUY80r2Zh5Y04clXQt1s8T3lsXcsSgKfB9Y1j2GGUYf4EykMIG+7ymKK2hzhZAcIVkaf0JMM7R2FLrAqCXKK1pv8arCu33QB5LCkAfuGH9RQwRRjUMAoA+JZb2kO72FWt6gTyuJeMZSolFml97uYG2BG9KEyFEitWG757gQQxRnZM9lZnicPLvJI996LZ15YYsgPmjCpuk42ES4zsYNtxn4g6M7nCyPycn5xpPkeNc4Dic/x2saXs9jM5PYGPTb03g9eZEZPq82ASOTo5AqBNGrjzJm+5gQjdw5RpdUxYxZuY/WDvSMnpKkHUpZYtIjcCBMxxJdGoIxKAfRWPpkSNFitCPFjnh+iOFuu9s+ae3RRx/la7/2a/nWb/3WT2lj+z3veQ8f/OAHPyL77n/Upo1DpShB8AQkMTISwiy65557ISZ6n1jXPV3bsLe/j7aOkBLGWoqy5MKFixzs7eCMxhqROBkWZNHrzt+XmbV3shu2gcmROXwHK0VtFWJ8MXBs1AtMoJIies9ydcqNGze4du0ah4eHHB8fUdc1fS/FQUPf5LQ/PRY5NVoq20+BWK1FQi4pi9GiB661Q2EE4FdGCt8oI8FtFCO7Qp299zuuPDsvkz0sqbyn5NfU2b0t5j1pk3qq8ttjtZoUpYZKClljPLBJE94A6Tbrf5MS3nt6L6B6TBBCygGIGmM0+/v7KKPpO0/f+tFp6rqO9Xo99tUWoG60MMqVRltH5UqUVix2GR2vuq5pmoa+9/Rdl50OCDFBCiQSMYDKYLPRGmNsBt63QWtgy4mbSq0MUjzDsVPw/GwbJWAm/6bnn37H9JzT6wkhjP8GRy6bE6MdohAnSK4l8JJD5W47t/3wD/8wv/mbv8lP//RP84/+0T+6Y74dHx9T1/VLnuPKlSv/PS/xU759zud8Dl3XfbIv4yWbc46nn376k30Zvy/bv/7X/5obN27w1V/91Z/sS7nb/gdqd+36T1wToHJqYwy2wgBEbICJ8fXcsjUyAdE3ds2Z5DkGcvSL/ZRjFANBQyk1yuRuMvqzvf0iBsuWXTUAheO1phGgPJcBPZIchUVN0lhbcHJyynK5ylIlCu8DMURSGylnFaEPXL92nbIsKcuSvu+xztL3Pa5wpFzRciBWyM8porR9LwMxZcxERMgoCtjd2aGpPU8++Qyr5RpQIt0bAikGXGGYz2c0Tb2xDRF18ak9qhGZwlG6Uk2xLnlwmwDENqkjRiHmDKz0kTQSN74dalOXJ6VIDIxgfEpJNMZjmkB5Ui9x00eZkJMECI8Z7yKfPg4ZEwyBEjUGKyT4onKdJrUZW8jfMd+bDynb9opEkACKEtKMiiINE0IkqojJ4LeoqSuMNlRlidKG3gdSUiirh2qq+UdApUhMSqQg82chEoioCDoXYgVGAhjZRx6KjYat7Ijz28sG0Z2KApjqLPMxdOrEmdvCNaeg+ojyD+9tpDkSZHBzE9WQmwKdB4S2llTNaGJP9B3JRKKGmHpS8PjumLq/Tb2sqU9uUdeJanYfO2mJaU7xyZP6I7r+kFQtSFQEtUP0AdX0EDWm3MUVULiG1TLSNwUJQ9u2rP0xrrvG3GiqvX0qt2BnDm04RMUFkQU+3mCn8DxwaY+LuxatPDou2K3uoSw6uvTf8Ok5vLckVaHUjBA93q+JqaePFpLH2CWlaTGpw6UTor5CE15LHUpUroArznEQZzPJ8MBkDVZB1UkhkLoVZnUdtbqOcgYKizYlISUaSpKumNlilCXZLMbTB7r5c/p4YVBv3bx35vBJ2/7kUCSVyaagJp/bDKHNsjJi4WnzztbpEQ2m/Nv4jeNkBwKg0+b1O8H5nJY9vDAC7Wnru7aSUFQScGqUdMlpWcHLAucWAnj0rTyTFDFKomxVNQM1o42a0CW8CgSTsDnimLRD2RkpMsoj6NlcUmJ8xMdhYzJ4vV3k8m771Gtd131EjcX9/X1ms9nv0RX992/f8z3f88m+hI/YvvRLv/T3NxNRyqGjtUI72SuG3NGUYFbNuXjxEqu65vr1W9SNFNXcu1BBiKQYKKylLBzOZUA+gfdB1v4JmDjVWxzaeWmlw88XA8vVEHjPi/5ggE8X/sGJUAiIf/vwkKeeepLnnnuWW7duUTdruq4lRi8AdUo4K2wJY52wa4zo71nnYEghNAKQJwQslwwwgx7AczRJDWzyDJ4rTUp6tHVGga+BMc72Hnf2N3l/ksc3FBJH9ANJOgPoaQTOh+JNG48pA/tJtBuVzoZz3ql1jOjoMQMTOwQ53lhICIgeEyFE1uua09MVe3strijEGYmR6D2khDUGnKNtW3xK9F1HZ4QdIwapFLAtsua6tU56ymhmswWz2UJY6t4TQ8CHnr5fiw2kAin19F1HaDsJaFgnDsk5AZqXAtWngZqzeqDTzw3P/SyIPrThezfOTLxj3IMECfq+H7Ux45i+J1qX4iCKXaO12G5D8aO77eW3P/Nn/gw7Ozt87dd+7bkM5a//+q/nB3/wB1/yHC9XTuRuu9t+P7Vf/uVf5jd/8zd55zvfyVve8hbe+ta3frIv6VO63bhxY6zTcV4rimJL8/sPQrtr139imtbbmYojLjFmMApSNmWKb47dJgRMbaKzIPq0nbWFzrOZBtYzZ2ycgbhy5/elc88xXMTZ7zprv6X8Wa0NIUR0trtvHt4QyRQlJBeSSArGGIghobTh6OiIaj7j/geusFqtNvcTp/01tf02APad7Yz9l8QWd64ghMhTTz3F0dEJfdcTo0IbRR87fGhY7Fbcc+8Bfd8K63nsL5FUEW3xfM9aQUxCehme1+R5b/pZTWzOoY83WalSSFML4BziGPRImUQjBVclAzWmkO9vWsR0jKBs+mXA05TYq6J/znidMRfkHMD/wVG7I4iSBvs3jv0g542EFNEx138aAghKoZQHZUh4FB6rcyBi8GkUo63ukPN3XjI9rTEoLCkHDWJKOft6GLFpfBakQes9j0cjftwwz+T39KJzaNpePhOdBkO+SLVJBZgccDaoM768QdnvfC9mXUqlwWS1kpHcnrVQtVa4qGmz1o5XjlYpfA++v4WvX+DkdMVz1zv65W1mOqCdwc328Smg+waTAs36mLC4jFYpF9tKzM2c2UxTzg8I2rHuIjuLHapqh7ruOF3fomnW9K7lxlFNoS9iqEhqRjm7QqEc2GOUXXBQLLl4MGM+gz4uKc0FSrePMw3aXKIJz2LMjBgMSnUQLCokNDP6vqcPx8R0TAqeFJ4gxecp5m9Gm32836c3JUlZQr/G+CVlqlHKUKuSLswxbo5RmuADbXeK9Uuc8qA8OgWM8sKsK3bBzYShqEVbVjCVdObpDI9t0EffyPlsP9GJJlHafmdoIxAuJxph7q21I02HUEJPvkWiepNzj8h3GifyUApgGuTZTN3p7YyQDOPSMQkSDDVcz1zhBtQZxrOaBJEQkMb7QPI9pQJVLkhJEfoanTQpBEgOm8d6HxK6SITo6WLEp03lba0T1llirLBKg1/jCDijhPFHTdcHUtRE7UgfY1Gtu+33rv3SL/0SX/IlX/KSx3z/938/X/M1X/N7c0F32x+IFqPU1FBaSXah1qQYGHS3QTGbLbhy+Qpt23P95i2Oj29TLebMZnNSDKyWS45LR4oBqzVGIfIdE+B5YOeeyy6ftLNg51mDevvvzTn0qFk9DciKUbVarXj6yaf47fe9j8PDm4QYQEkBHKvF2NVK44zO9UYM1pZo61DGihTLRNJEaYPGZPBaMeqaZ0BdrimD5srI73qQRNuQAaa/snmJM77TOW0MyU4M/s1eNzpQadgGY7b3BUyXgEPe+VLWUdcJlSQIYlJEp4QLovWYIoQY6b2nbmpOTk557rnnALhw4SLOWlKQZ2WNMPOdNcyqCjHMGR0UYbQHfATfdtB0oGRvc67IrKECbazoz4eeUjm8NzTNknq1JoYW3/f0dYPWWgKLyW2nxZ4ZW+eB5UPa8dlxc9YxPM8h3Hoak6DPee9Pv2v4viFzT0yiwTEVm2E4xhj1kgDN3XZ++0gA+N/8m3/zZemk/8AP/AA/8AM/8Am6qrvtbvvUb9/zPd/Dv/gX/4I3v/nNd8f+y2if9VmfxVNPPfWi77/1rW/lP/7H//h7d0F32++btm13KDLvN9tug5Eoe10IG6D6bGbcHXZMxm3Ofs95AHoacRR1x/n0GRtJNLLPB6C37fI04ifnf+c2oWG4/5h1x40Vgsbt20cEHzJpRAAkZy3GGLzvKasKpTSHt29z4eIFXOHovcdaR993k6KSExD/JbHRAQUC0eW2OGeIMfHUk09y8+YttLbj+VJCsjSd4uTklAsX94QYpCc+DgkVN76WsLWHzODtIMMQeEhRMxJM1QCsx0kwJfdfjESV8EZkE7WSYqUDyD5IF0ptqGyf5xqTagDLt3yROGJ0EgdIpBFEH8bdJsiitRpxvc0/6W/JHhjQt0n9IR0xIRJ0APRIOBlIJyEESJoejw0qg/Een4JgfFrkYyIiv2mjEFm10RijwOfxnLzoxQ/BizxsBdcXIqwErRQx+8cybnXGCF9ee9kgehmPaKnwamcTPdtyEDdO3waNnJxgeGAMSRvydwyBmDw6JGxRorWW7lZKHjQQUiIEj+rWqH5FT2DZR4kGdS19fZPV8fPcvHbI87dqLu9fYW9ngWVF7Nc0dcPsYAenFbsFlDOL0iU6VuyVl9ib9xgL6y7idM/CFfjecsuvWdPT9yuihtNOc/204eKiY7Fw7B4s6GNJOmrpg8KWitnM4lwg9ok+nOLDjJgavK8JWoIQWoOKDQW7FGZBFyxgiKmh747xqkcrSxdK6I+pymeo0mX6viSoOb5bU7a3WKhT0B7FnGDvIylHMAYfewgtVgW0c5jdC+iwxhhFNCW+2MGUC5K1UshSASnlaI+AFZvFfZPycHbpmQLeiQFA337s4zA4BySfHjyMiXE+qyl0MD3PNGI2OaHizNgaziqlHAYHVqcBmJjeSBqPTvmcm2ueAPV5DGvUIHXFgKwPgbZEkg5NlhADfe+h6zDdEm0KYTMmRwwS3AgmoIodjC3oYsSmKAx3rSUNxRmsLkhdhws9LrXY6LFxTUqRqB22mpNGAOdu+1Rtb3rTm/i3//bfvuQxb3jDG36PruZu+4PSBuNJ1viYmeQRq6V4KDFgjGV3d4+LFy/QdC2rdc1qeUJRFCQUXdfRdR0KKDIbXQBNs9HCO6NLPQU7B9bC9mtnDfTpvrNt+E/bFuslQQqB9fIG1194gaPbt7OUhuwkWkuqobVZcsQ69KBzbkXzHGVIWeccY8FYlMpFbdJg68gGkNBoY3MAdbjeYUfZ3jfPiR8MTwSUsEKGY2WfmToV0w+bjf00ntSg0kbaRezexFigaTDOSUghoDiGunW2dg2ACZgYswMTMMaTlKZtW27cuJEdCcPezi5lUbBYiP03yKIMz214HgIiJ3yCvvc0bUPbtvRdR0xpu4ipK7BGoTNDpu9Fu7Fer/F9Q9+1dOsa5wQ8JybKsjyXST6Mvw0LZ8McP08ffTpOh+O893ccf3a8vlQbPjeVuNn0zXgUKcFsNmN3d5eycp/ykhr/I7bXv/71vP71r/9kX8bddrd9yrW7gaOPrv3QD/3QS0pDXbhw4ffwau62309NpDk2BISxuChwFkQbiI6DPN/4Oxn70BvQexsMfnEAfXIlZ+yqzTEb21yPpJazQexzmeiT96bfOWTxnQ+sazLWzOnpMjPLNTEIgcH3Hu+zfa+EtQ6Qup5r117g/vuv0PvlaP/FGNHmzDW8BISuNSNADGpkxl+79gI3btxEZdtY7Gkh5MQY6XvPcrkkpYgexN3HnmXLTpbrGvppIxss4yCD/ClO+n+4Ho0wqDfkzWEMDGC50poYt32mQX5lKKCp4wCMyzE6TUD8PO5SkmO0luKyd5q+E+BNUNo73pvyacfgQJJioEoBwj1CJU1IEaON6Jkn8VViEhb94LfG4EkEUMLgDyFQWE1hLSFk38ML6doYDdHkQEVgyCgYWEdp/Ld5HgOeZ0yWotTmnPu+s71sEH0v3uRWvERnRLs7kSbMZOmgcdqeN0Yz4j+wpRSKGCJ9t6aKp8xMAiqCKgjKgi7ACuMrhkjXd6R2TepW9F3Luq9ZrZb47giVDKdrK4Pbw+nyhBk9idto1VMWuxSuxLcBi2ZezCjme0QKFoVnXrVYdUJR9CxmkV7fputqVH2btjjh1PRQ7LCzeJB1tKR6RVGcsLdzDy7NsOaYNtzERwVqN1ePNdTtIfX6Flp3FNVtinlCG0+hFFrvE3VBpRQhJU7rmuhXmDJQWEVKFVX1CEWxw8ycYMwRpF2WIRJ8Td+sWMVbGHcK5gJG7RDYA6XwoYPYY1UEq0AVJG0ICqIu8G4XXS6oygprhZXdxUDfNaQkgIoxJVIATYqsSWBT2OGbZ7wB2KdT6uwCslnMN0NDb04xHjfCwJMsh2EBGlI5hkl7Zv2/I2gzyACgBi7fAI5vywaNY1Zt3t86v06ZfZ6dapWVb9VmAZR7S2gDhbIktSCqSNtBCh3Jt+iiQNk5hU7YCMk3JF3iXEWqFgQ7RwdF8J6UjMi3GIm49TGSCLhQU6gOB6SwImJp1Q7Wiabv3fap3S5cuMAf+SN/5JN9GXfbH7A2GkejESyGU4yRGPq8Jophcs+li7R9S4yR1WqJsY6LF++RIpG9GDTOOewAXBp7B3g+BcS3QXVeFECHbWPfniNxMRS1ScO1Z2Os7htW9Zpbt2/T+Z6yLFEKrJXCQ9ZarB0Y8wKiDzImShuishKEVIakdC5qKoJdegzrboByPbKR5ZwJsmG4YZpMZUcEdA6IUa7FqFMbwFYch8w8UiobeT4fL4beYH9vsZzvaLn4d9r8Pe69Wmd2ehxB9pSi2GJajHOtFc4Y+hgloyol1us1q+WS/d1disJhsoEalSIEn43iDSsmxSHVdMNCkn+aGHq896MmurUWqxVWQVFYUuxo6pobN27Qd2tSjLhMk1mv1+i5ZGENWQ9DOwuGTwuHhhC2HDul5BzD78MxZ+VaXoqhPgXZzxu/09/leSlsLtzug2c+m3FwYZ/FYo5zDu/9Oc/ybrvb7ra77W77ZLfP//zP/2Rfwt32+7apbIsNwOpGukVa2gL6pqDpNmCdIEJUAzbB+N7Wt52xZwbW8Itf3rZNMwC6d2aMnmWV30m8/EjZfiSyJIkihMhqtabvA4Mda7TBFAJs9r3U0em6Hq2hNBW3bh1y8dIlynLGMhOAxP4TecE7Awfn3W8OSCiFNZY+ea4+f43nnruK91OihdjIKIgq0nUtTSPkEj3Y2pub3aq3M9x/CBlzUhO8Kalzny9IgCRCtt8HezdmezcRg7DHxxshZxmkjdZ+ioosWS72uSZnFkyRXGliEqcJ0H/WztUMsi4iGXqWZKtGYumUnJJSpAcG8qnWeiQrDe5LzEGEkKWrE5GQfCb6ehIaUhyZ94qBlc94rVrrkYk+zKvxZ5L7liKp+VrImaZaoZTUyno5KPrLRt6SUiStsSah8YjDl7tq0LgYgH4m0/IM2Kky+K5BHMcUsX7JTqwlnqHnrNQePXvoNJPBn08dY6Cu16xWL3C6vEXTSUdbE+m8Y2dxAasarDIUeo2KJ+zsXqTae4DSGSrVoFLEWSjKkqQLYtI0/Qqje0gdynTMVENlEtV+pAyO2XyX0y5giwofDQaYFZHCLvHtMV17i7o+YY1htZ5h5zNIBh8agl9igZmrqNIOKoFTc9AQVQJOKcxNZmaFssfMiwWVrQh4kiowOqLSCSZdo4pzQvRSzCAGTpsO3dcUs11sEVF42l4RmhrTtRA7YT4HRczAgdeOaEqsLXHGorXCh0jdrKC+QYEn6YLOLEi6wroZzhYjAD4F0PPAyM82DUEeNozzTZR0eigMBUnVOIBzHHY89uzQlYm7SUGZjqnh/am0ylDqLSXp56QHnFzlhWs6Jqc/1VYgwCQ1kScbFpP8k6GUXMoVl+X8URckvUOwDlPMMfGCFOIDnF9Dc0wq90nFDFMk1Fz6OjQepRJWgVOQVEDrgEk1Ia5J3TEOy8womnALy4wYdyH2hPgR9QHutrvtbvsD2MqyhBgQKxsYjHEiwfcQIyl5nDPMZhUXD/bp+o7D26fU6xXNbD4WQlRKU7qCyhWZPL0Nmk8ZF9vFF2FgOGyzJOI5xhnE/k5gcWr0a62xxhBT5GR5wvPXnudkeUoSqwrjbNbitvlfvhbjUEoKgiqtQQvrXBZ5nQUANSppBi35oYLLsOuEILp6SknqrRixPjMeNgyhKetmI99hiMmT6MW4S0M9nEgMGUgna/7lQp3WaMhyMmnbujrbQ/mnOnOMFoYNcv7RkI+iPUlCWOyZKVI4yUiTgkkZkI6R4D1d2kiWDI7B9PnEGPFBdApDiPR9j8/nsVpvUlhjpO9aAqAKS1HMcbYA5RnSOlU+Z9/30nc5Qy6EMLLThz4exs8mLXabhQWcKwWzbdyfcQLOOIrnvX6W5XU2eEQeH3XT4X1HjB6tPXWtUNpTxpKmaV7keb689tgXGmZlgSvhyn2JxWzGQfUGVjce4Td//YMcnjyNKWHnkmZ2ELGziKl6Xv1pF7lwsMvTT7zAetmwu7D0Xc21Z3o+8Fse5TVdG1AojLW0nR9rIgViZg3BrNDsLirQPbqIRB2oZnPuuecidV1z69YJ2ih29wpW64aDi7ucLhvaVWB/4djdMwTV4uaJe++7RHcSeeUj91Lsdnhajm837M8voaOmby3PPHONyCHRR3aLR3jyg8fEULPYb9m5VDCb73H1OU9SDQ89vMPe7j4pFoSuwfc9h6tTdi+VlLOC5ijyod94gU9/7Iv49De8hV/5z/+OmG7SNz2veeRzKXiQZunY21P88S99DWX5PDrdprQ7OH0ga4WJgJHaClZSvdVQOyElJOskjmNwyvCbrhNDG8e7NSSt5WcenyYz36SoM+iotsZejBHvPdZayTydnHcY78YYKXamth378bxM2ZB3BocGW/w8NuL03xRsGTM7jMo1ne4EV4bPhRCw1t5xzMDS1FqPgaetuUy443zjPaNIIW7d4/hZrfAxEHMfTq/l7Dw/b70bfh8d98k64CMopXn8D/0l7ra77W672862hMpY2GC3DTbk9pp6FmBNmfgwnmd8ja2162yGJzDuOWfXsymoHmPE6LNr+Z172HC+oW3vOXcC1+cC2QIFCnhtHXVdc3Jygvce3wdUFHKJswWgKIoCP5EB6doeYw3XX7jOw488PLHDXsxWPn9dFzNY9sg2eo5un/DMM88RQqTr+q09xTmL9x5lh/5MrFYrdvd27/BrzvN7gGzzb/flsAfeAZ5ujYeBABI3/ZcGyZt8+PDWhHSk1SBFs8FthyyICcKXT5lE931MxlXj+abPMqUNnjZcl1IjAifFSMfvGORfNmSXKemF8f7kmnsSWnkgEVNPiP2GkGQkKEBKaKWwVpPQWOyIB6LFf4gZvI8pZjkXUHHTRwNubUwmTw0+4icSRO/VXMT96bMz5jDKbgGLm24604bBlP8fcFBtFChDl0r6fsmuadDGk7CsWGTdHo0yisIo+hRZnh5zfOsFDg+f5WTd4co5B/sls3nFwYXLzC42+PpUHERzgcXiInZmmbkZC1MSVY+Px1jmaGvpfaKtl6R4iCbhHFTWoLynsI6LOwGVOk7rNbiKVbdAOYezBh88fWxJ1LTNimvLiPGKV1w6YDFXzNwCTGTHXuSe3QfY2ZnTxTXKWHxyeCKtPoR0nbQ4ZN3PMO4CpiggRXy4TSLS9zU6znDqAkUqMBFS8ijt0HofqyqcUXi/wjct5vQGNkp6hKaD2JFUQVQlqdhB2xlKaXyU9Ie6WRHrIxbNCxScELWjVyWdPmBVXqacH1DZYgOG5//VMCGGibeJozCVSJn+nIiKswHFM8A9gi2bn6h4JiAz6FoNSP3mG/TIeE9MIeVhHVWoCas8ZQBI3jMjU32yyOfzbM4rWMsGEAIVZOMzWqFN/l6tSLYEVWDSDvgeFQM6tBBrKOeo6gKuWuBUg1aRpAK9zkVjh+Jt/YpZWgM9dTzFpwbjPZgeOJXrDg2pawF3dtbdbXfb3Xa3IRYDsqZYTQqJrutom5q2rrO8S6CqCqxdsLe3g4+Buumom5bT0xNcUdC1LX3b0Xcel6P0SW0M8vPYKRvQEqbG4osBkSoHMc245p4H4EihZQFYO26fHPHM889Q9w2uLNDO4ooKWxRY67DGoq1BozPjXGfGeQbPlc77lsq7SI6yJkVKegz+jiqPMY31N1SMxBgIvsP7hr5tR7YKbJwZMfjBGksiEGJL8AGlNDElfJ/1DqOAr7Nqhl4sRBlMu9zPMAQQhi1wu6ntSPUQqlZZJ5FttvW0GJBKkQEYK5JCa4vvJcDSrGtWyxVaZecl399sNmM2m50DgkWatsX7gPcFiZSLl28MhLHgUQhURcHe3h5FoVAqUFUVMbSkIMVPvfekGDFKbzHRp+NhYABNx9bG2XxpBtIwTs+y0QdD/7zPnqdjnpIwlQb2++ZziaZZs1qd0vc1y5Xl5qGhrtd0bUtRlHzl/+2rX/T6PlIrdwyX752xsyiwJJaHBlft8P/8yj/Nn/vTc554+n286vEH+Xv/4H8l6lP2dktUYbjx/G1u3ziiqyNVUdH3DSEEFgvNZ3/ew7z/vxyyPFljjKULHlsoSaW2Ip0TUsQoqCqDKSTl1lgZpyeHNfOi4f4HL7G7cCzXS7TWtAqaZYOKChUcly5cousO2dsvUUXimScOsUmhdM+jr71ESgZnF7S15uSw5nc/cJWD/QP2L+2QijWr0yWPvfpVPPvU85zcvsnRuqYoe5w94NWvfgTnEr/+3qc5Pmp4w+suURSa27d7bp0EZrM19fGKS/de4rnnP4xNhna94sKlHRaX9njsscepT3e459Nexer0KrdvH3HpUo/TUYJIyYsWJ0nAc5WygydBK5QE21IMKJVomoaiKDLQvlkb+n7jnE8DPaJp6tFG/h7GPCBAudm4b1PApKqqcRyeDSoNALQyRupG5DaM877v83zYaP8Pc2kAn4cMoOF8w70Mx26B5pNAm1KK4MPoGwznnB67lYY/OXeMMVNjNv02fE/MBcogngsYAaSwkVoaAnFjP2cU4EVDk2cCbNM+nQZIp/04rj3KSEr63Xa33W132x1NM2S/j6BjGiQ4YraVNsDiBkQd1txtoFjWvnNqFnIm4DgJGA7nH16ftsTZ9W8DpE//nQ0mjp8+x34/W0x1/CJFzhI0nJ6uOD1dkZKiKiuccRRFycBWTwSs0qQka38EfO+pm4aT4xPKcjbK5H00SrdCZhFN9cPD23zod5+ga3t5JkGAWG001hqssfgQhD1fWBJpC2gHRlZ6yoGRQWlxs3cNxwrBZWDMbwPVegygDGNjKBo69Pf0M9NnNTlLxu/OElC3/55CapvnuMn+Hc6/sbMNgyQR45jMfw/U2DQJEKkkvliWc4gkQooQAypqsR8HokaKUruLnpSkMKpC2OcpBTbymdmeUEJSckqKzw44pFaWECM+9CTvcz2wDdZ31nY5649+pPbyNdFVj9FLRPqjoFYLUAuUtuKHjp0+mWyTBzJ9WHJdGqMUaEUdYR0czicKW6HcHGctSitJQ06SUhBDx+npLW5c+yCnh09SryNuXuLSBXZmD/HAxYtcmllIDXW9xMcGowuwULo1O4XHWPDplHWzgwoFBo33iXq9RIWOebFDozSl0qT+hL5dEv0JKbT4oAh+zYWdy1TVjJBOuHn0HNcPr1OvOpyac3jUcM+O56F7H+Th+17NTllR6j0O9i6zt7vPqj5l3R2x7G5wa/kUugtEHSldQLsKZR2BgO8gxBUhdShdEsOzaPcQNs4pKTNTw+KspnAKnZbYfoUNJ7h4ldAXYA7yJO5RyhKLEjU7oJztorQjxkjTNoT1EW59A9XdxKvbYAyaEusSfXFhfGZ6DHINgHOOfG4e6uQZb8aC3nqd0Ygeh0c2YEegZXwtbb2mJpN+M7Y2urAbPH1SjG0A8xNb3yvXtZF22ZTwSFu3IscMaScSvVMZEBgCrilC9Blwz1FJtBTziz4J87CvUd0JhDXalgQtAQKjoMRjVKCJeYL3EfqOMlxnlq5C6LFqwTpFUtfR4elSINoCpQuUMiNA8rG2X/u1fzx5DpNgSUqoJH0Qk6EPu4R4D4l9lDWSkBLO37CHny/G5tusDUayU1SPMku0PcG4Gqt2cW6Pwu6AqtCqQttCNn0UIa87KUpthbZuODk+HllKstDCwAJLBNl0Y5D7GnMJADZpdKJHn6WMtKEsK/b29plVM2FAms39TTeWmBI+RU5PT7n5wk2eevIZnn/2GjdeuMkTH36KZ599gq5bcnR0RN+1HOztUlUlzlq0Shws5ixXS26fnHC6XtO0PV0IspkMQF4iR5Hl+qy1zOa7dEFTFjOqao7WjhgURju63lOWFfPFDtZYjNEYq0F1xNjR9zWxbfF9R0yBEDxNU9O2tTDiUsBkI21wygdHvSwrtLY4W+DKEpTBh0hEihQW2nLv5fv5gi/6Eua7ezz19NM8+eEPc3L7JiF0LJcn4iyjJFUvS0dpbajme1y8eInL993DlSuXue++e7ly5Qr7+3tUsxLntoGIzZiN430M0hGDY6u1prBuy8n13tP3/egIT53+qUM+RMsHANFaewY8lHEs4GBPCJ6ua1itVxwfH9N09caQGNemQEo9Pcf0XCfEDhUTKoKOGpUsKhoMol/9t//OP33xCfwSrfctRisBSpUlxp6Tk9vcuP4C9WqF0VL0cW9/F6U8O3s7HBzssa47bt66LSCqO6WyhTAtFjtUhUObgW2waWcZiMNrkHHpicFylrG4dR7iCBRv3p8yA6VOysnJMc8+9yzXblzHx4gtKmGX2xKMA2UFhMaQlCFlDXSZ+htgd3BIhkCrUsMapTfbitokaRqybaIgeYixp6tPWC1PBRSTGxeGkWLUcSyKAlSi62q6tsU6R0pKDPQocizOVlRZMkcKoQobROvN+DnfthuKREkPikDahKXNNM00ZcrS0MmDGGRE20RpDM5a2mbN8fGx7H0pcHBwQGEtZVWxt7vLfD7fesZyZkWIkeAjIWZZmuG5aZWN7gw4e2Gel1VJCA3WWcqyJPgqr0lpnMfksTCA1AMb/cXYo8N9T8fm8HNr3Z4w2c9+/iyANj3vFkuMzTid1giAhA8e7zvq+pTbR7fofY1ziq7vaNsGYz6+APjjr71I36/ouw4TH+Tz3vwnuDT/NI5vtPzCL/4bfuW9v8CVBy9yafc+PvT0TW68cJ2oArOFYr7rWMx3iS5SOIMxFbu7jr7p+fwvejPvf99zPP3sUzirKI0hRENd9yQizilmpaGwEW1atI4oI+SYvX3LzZu3OLx1wt4Fg9E9dZdQIWKtpXIFq2YJPjEvK1YnR1Rzx/5sRugTzbolekXTBIgLfud9Rzz75DXuvbTD6XFNTA33Xl5wcN9FPvz+qzz3/G0OLszYnTt86NEoDm8cc3jrkHbVsjs31HVLtdjjLW95PYsDR+9XXHv6Bk+9/yYHM8eTT/wOqJ6HXvEARTHn6vVbvOkNn4FWOySO2ds1xHicHT+PNgHrHNYqhG0OKOi7nrbtmM8XWGvH8TXNnJjupdP3IGu/5n1GmU39geG4QY5o47Sydd7hHNPxPnzfOKajhAHvCGBuzZ3NWN++j83YPy/INJ2H205+9hsmr08DXQOw45zUCZgWHNNao1Bjv0yDaAOTLuZz+LymbDvIbDHQrbVjRo0yOq9L24D42Xk9vbezQMlw3NnXOx8kG/hjbH/4L79KAodaxkTXdxhjZW/UFuOc2ObRUzqLtQrft1jrKN2e1DxBAjKFKzHGyv7kG7pmTdv3GFfhyhlRGZS2dH1P064oygJjNFVVorWi61v6vsMrkT5buBnOGLqmpunWoAPzecnMONq+o25bfPC0vs9jDaxSFM4K0ShJkfJCa3zd0rUtGKiqivlsjgaCD0QvQJXTcxKJ1jcop/Ak6tbTh0SlHS5pklGowqKd3EfX5HoczkrAzzoWrpBCf0TQitBHjHYEBV0KFGWBRZN6T/A9wfQSyELWkBjB2YKUoG5rQvSSLUIGE5Wm6XpSihQ54FgUBYvFgqZpWK1W4zgry5L5fD4CkX3fY4zh4ODi+KzatmG5XI7gZYwRqxTz+Zyu6+j7fgwuA6zrlpgUzmm0Epm5ebkgBsmuK5xkeSyXS1arNTHIdcxmc8rKgZZglNSWMfncVmxpNoEjZxQxiDpA13doK8XmHZbYy1rR+Y4u9rjKoZRjowWtaLuOru9lDegDVlsUAqQuFgtAfI6YEsoUaCMydwnJ/PNe+tg4kZ7dLWfsFBWr01OOlqegNVEbtKnynI344IWoRsx/i99TFAUKTd/2pKSYzxeUZUnX9XR9Czrgg6frAyGC1Q6nRaP7l77/wx/z/FYMGY5KfNOpqZELbW7WyG2b42yAdOjXl/W9Z9YqOLu25fendl1KW3vCsC5PQfQzXzIeP93fzrP3FUrWYaUzC/2UGCNlMUNnENn7QOEKiqIgRp+zHgNSy0fTdZGmbrh9dMyVK5e3rvfltZSDt4aT41MOD29zcrIkJZEuHALZ2kiQ3Hup5SM65ClLOqo72C3DXk3+kSZdFcKAPuf+Z6ORPvhW4vqI3TxcZ4qZzBMTg457SmRZyM3zE39AZ5wrY3ITF1TOvyHVTP2s4fcNOL+58MEvG4By+Q4YGOjbvt9mHKjsw5xnUw99NdxnhuBJKUrAP3nxVY0WBnoU4kYaGOVyUaNtEHN/gkIN64iS86lz5sxoZ6gp+esTCKJfcE+idSJywGl/Ac8MP5G12HzX8KBGwY4p4pkvVp64UsLMopzT9RGvajQFpISOnkDPEMdJ0ZN8R316k1Bfh/Uh1B0kQ2Mb1vMdynteiSv3UapHqdv03TGKI8rihKI4wRUlqEvcOAx0WmMLmBcziIl2bdCpQ8U1Hrh9eszx7edo+wZbFGhToIxBlxXKaE5XK5b1bZ6+esS1mzVhbbjvwj6f9erP4E2Pv4aH77uPB+97BKeRVOiUsK5kd75PUq8gpMjN42d46vqv4Lse7So6pfG0sqmZRFQHdLElobAqoU2HIzJXkd4EEhZrvOABviaGGhOuU5klQV+hTZpoKiIlye5gFpdwsz1cUdDHSO87VGywqYNuRd/UWGcgFWBn8hnjqKymMCKrM7L0hlG+NQM2A1mrDWa9GRvbi9r09QGUV2qbEaKGbSa/v3njbFAmT4Qx+rkBRTQyeTbjcHhv890j4J8XuhggRUXSSQCo8bvUGDQwOhEN+BRJIRJQGGXyZq/GFJYUI6lZYf0p2iRUCthwCs2KRIcqAF/jfML4jgJFaUB11+nDDVSogftQYc66s/RBY2cLIhUomwvmfXyFRUMaAhXbm0BKCZUZSwmdF6bp87lz436xBfLltMHhS8oT1VrALxSKiNIRSyAmAbdSyqnHUdheaXQ2h4V7fGqymKYh6h5RUYAjlbNdRm2vj3R9IN8ft3XLEgK0+uRpmo5bt26zWtXUdcsL117gxo3r9L4jREkparuWdeMoy4KmbfB9R316iissbduJ1ps16ATei2zEplidIYShrkSicI6Lly7SdT1939K1Nc5WdL7FGIvWHb5foigAQ0pgbKJtl6yWpygfsdaQEAfUe89Y2dv3JKNHWYy2bce+iDFhbSHXlZyEm1IkREl9K0qLVYb77r3C5QdeQTXb5cL+RZ59+imeeupDtF0nNQBiRGlh6xql6L2nD8dIAREoS5dB+zIbMWoEOe/c6LaZzmqyqb7Uhvhi4NuLgQxTJ31jMHDHdb34926ChUZrkrLolNBJJBJU0KhkUElj0nba5kfb2ramdA6tRAeu61uOj4+4efM6y9NTrNbCUE+e3Z0Zcz/DuoKLFw+IEW7cuM1yuWRWVrRNQ/QejWgGjtmDk/s8j7EwzMuz/TZlsI8GTDbypszB4TPD38YYuq7j9PSUp555inWzRlsjhrixaOswpsjFnHNhyqFwaAY3B2tVJOYYbZnx+SZJ7ZMXEmNtDeT6lBaNQq1Aq0AMLe36lKZpR00/0XiUoJK1llhVaA1Nu+b05BRjrewzUYJjCc1ivsP+YoFVYJQAaVq0wraM5LNtup0O1zy+NGGaM0DpaVDDT6Ocy/S8xhisdTRNzcnxMbNCnNuqqigzo3bKqh37LRuwlGy9ptQmK2EAu4ZiR4nEcrkm+IBWGucsKiV8loRp2pau7caA11km1HnzbQr0DdcxBRbPOqTTsbvpt3TusYPcxHD+YWz2fT+R0zB5HIjUjw89TbOi6ZZonZjNSu69fEnYuR9HSwTKwlKvoTAPEOtX8NinfT43rj3NY696Ha6Y8Tmf+2Z+/b/8J649c8jlC4/y1LNPcny0ZjVP3NItREfnT9i/qClcyQMP7vO+3/4g63VAu4SxCWMVMSmStqzXHVolCpeoyoS1kaiFDYTVXD64TNd0hOh54xsf4td+9QPo2LGYzyjLktOTNY8/egnnek6Wp8wWJTbP/eWqJjaergksT1ue+N0bhMahgbau2dkzhE7xzIc9hV5y9fkjFruaLrTsmRlKw3weODo64uKFyzz04B6HRzepFh2L/cQTT7+f9Gzk0r2Gyw/sYPWc+nDFo/e/lpPbt7HG0HeR+y7fz/MvXOXoeE19dIPQljz4ipZ7Lsi6EVPAB/CNBzWtDWGpZkUGeuL4elnKhJjqo54NOg7jSWstUkqTeTWMz7ZtJYibFFptxn6MEefcCMZJ4G1bcmk498C+Hs49gnMZLN+202VtkD3ifADnxQJPw7wAAcS02Qb7p+v+ps6F1C0YAMXNHN6s/2fn9fBdZ+f/xkbLElGTfWToG/leMjiyeW04dgrmn7UThu+fftdwL4DU3Igf+/w2epAFEoCyYHMdtiixZQkp4L1GqZwan685mig1rzJjEzWpJYJINBVKoayhmpVEpfAhoXzIPszGXwvBC4DuPUEntEr0tIROEfqOGCTbou97bJTidplgiE4abY2AGCmhtSWpKPJHRhN6n8eGQRlGskIMAZ/rsKQQwVpJx8cz1OuyVrLhrFaSi2sUyuocJ9dYHF5DUBqVoEBjlSaoiPe92NZB4ZwhanluTddhgMI4Zos5UQ9ZHZqYyMXrBOAmBjQiFxQymceHnuhlzAS1yU7w3p9by6PvOhmT2U4tiwJjFBCIsc+ZDImiKKmqSooapjgGkZ1zzGYzufam2bjnSWwHawzVYoHB4jupg9P1HXVT07YtWme7Xgmr2brMfA1ZT1hLwNd7j08hF/kLtDnzI2ZbKmX7ThtD6BMajbUuyyQZjHE0TTNmoAx7ZEoJVUp/A2Om2WhHinclflcaCg5mwFkp2r4jBXBKY7Vm2dT0IWCtQRuDj5LN1vuOEDyusJSFgPK+l3sb+11r2kaYr8NcLgqH0gYTDWiP94nCVTitt4DFj6UN9tCZV2WObww5BtvtvKD+eWvwi7UX++z03IOGtFzD2fUcNnbk9B7OWRvVy/eHBkQmpcRqtaRphMTVtT0KM9Ym0srgrOPChT2M1Zu1LogEYdd6Tk9OuXjxAs66TOJ4+XiEMbLvXr16ldPTU7wPaG0pihmLxSzb7lKPMXjBjrRSMiaHPeNMv444ENt9nEY8YgDEBx8yZcWDob8SpE3GwfDZzbgYQPREHOAsMoA+YnCZia4m+iXyBiP+pvJ3q8G/lu9I4+GKrb7MrHnRKc/+S2TDy1Hn+M9644ts+TBqc29BhVwgdzi/ZEqTg0xxvGFIQSRnhJgua53KGXoiey1rxUA6HIiRQqSM4yXozQ1P7v8TDKIvzG/Q6zex9Du0eoeEE8BQqUywG5ywocflQs4Wi9xclERQKCx9mqF0xIeITYBvsarYGEMorJGbNLbMxbMCzmrKag9rK0KvqLuCS+VFSD192xLUKTMTmZcN5SzhY8eqW3F0UnK0uoVPS+ZlyawosWYXazWKloIl11+4yo0XbtFERXWwAFdRVAW7BPRyTfIN1w5v88K1htNlz37hePzyA3zlF/5x7rlwkfe971f4pf/wHo5eeALfLlns7vLwIw/xysc+gyv3v4b9i/fzwKXH2Z9fxqULXD35bXS8zcrfwESHigrvHcbsolSJsfehMRRmDbZHxZ5IlCidKuh6pIK4P8QqhXIWlIDnyc5Qs12KagdXFDLJfUARMKknxBZ0JJmSRIkp9vDlAbG6l2q2S2El2yBNwOYBg9AAU/abgqSSJCkNz08NqrKMk5WzgARjLGuAPAUcHSf35kunyrAbSDzleNgwYmDQWh/OupmzZ43gjUMBkAUjZSGM+fvT5lMpA/paSSVgUpeDGIrIHJI4VFK0LZHwGAXWluLYpIjuDiEtiakm+IAxe8yiow8dVdC4uELFRjaB2BNVoMXRJpEkcHYHtCWgKawYkx9Pk4VyiNzJC5tFPwMwKCJxDIDlLt5yWrb79aO9pgHolpQqrZLkYgWTgyEa4iCwo7c+JwILcXT45LvvLHaSUhRgCxmj430MAMu517xtwISsrTmyl/PGGWMgAutVw9Xnr7Netdxz6V5uHdzkv77vt1itTgmpR1uLKwvqumY+ryidE821tsH2lhDlvo0pJGiQU33E4ZT0qbJ0BB8pywpI7OxUwJy27VguVxijsMZh7MDuSJSlGR3atl3R1qeoFFA5xdtYYR2VZUGMgd531MtT6npNXddbaXgDi8taKUgZY6DtW0JIpOwwdKnh9uFtVqsVs2rOfL5LWZ1QVgtm8z3qpsb71YZpEaUwiGyQnvX6lKNjw2xeYYxhPp+zt9fRNh2QRs3o4dkOAbjpKJyyxNSZjfI8o/Tspj99/bxNdQrMDWNu+l2bz5w/FwYQQaS3IgNsa6Iwn4hKMkE+6rm0aaenK0JVMisdhTWQEsH39F1H17VEbYhBsghQmhAVOsJsNmd/P3JyuqRet/R9T900rOuanZ0dlBnWQkSLPC+24igOc0te1KNRqHNGj7yvtRuBIAE2cuBDyZothlcG2ZXOoLMYq+u65nS15MaN60CiKAucKyiLGWVZYbQA6CYXP0UpknHjPqaHDWn4P4+fzSsKhqI0autIxqI4adgLpKJ723UcnxzTdrWwrCegR1GUhMUO2sByteTw8BAQw1GMX43WBeFC4ODgIj54TCzQ2WhUsAHVthys3M+5Av2omLY1rjd/JlQ+XA22KJKCKesi2hC8xyiFsgXaBrres1yvqJuGajbD+0BdNxgtDL9NkCRlrXklBcozE28w4mMSebbeC7CcojC7Ygh0TSNOdkqAlYwfLe+v6hbfdWitabtedO9zYViUBOH0xriQ+0ubdXoDjg/PI27N/SGYII7McC9QFMXo7MQs3ZMSqL7P58/MthRRmlFGYwACUh4fPmQ9x+QxBmbzktl8LixP9fEVBVftgtPTFcZfoE271EvHehV41WOP8+ArH+ZNdUtZweOvfj1/+Ev+BPc/cIWbt27xL37oX/Kff/2X+bzP/7/wa7/+q7zw3JL6RJH0Gt9d5/R0ybqJXLhnRow9vQ+EXrS6hzSwvd05s3kgxJqoE42XLTqoiC0sKQQefPiAEB/gff/lKeaV4dKli9xzacFyWYNpuPKKAmsj+3v7XHtuyc6ipNot0UqCUOtVoj5eonFoHTDWo1nw6MOfzod/9xq97+lSw2xu8N4AHacnK/oucaMOLKslrjLcPLzN4t6WBx6eU9dr2vaYG4eHuKrk8kMHHH7oaR591aPcPjomRcfTTz/Do6+6wKtedT8L9xC6f4HdnVNIxygtDnfvO2wh41QbjdEytpRC9vGk0aq4I9Npui9NAzdnQdxRinAC/o7vo+6wv4YsPFmD7wwSbf2cfO6s9FFM/sy+BmmspyABxqGdt4dO98WpdMr0/an0yfQeB9tiANKGvUGCfmwDbON3n79HDiCdTgIEDyDmwGiXPcdkIGDbfp0GO6bgPXBHKv00EDHtM+97wnkszZfZhHUufVRkebKYwYGyqnBlRQoerRTBtxl8jqQAveoBlW0kfWZ/juiipDQaZQ1RDYXcelq/JsZBssrJ3kGQYn5a2M2+99B5dFTE4Il4rLa0dUsXmgx7CjBsMxueFNBJURhLUgpbGKrS0a1rVIiS4ZCB+BCD9J33Yh9oTdICfGub9XBVwma5V4tCJ0AnYuqIXmGUoiwNyWqsrqjXDUYlKudo+0BLwseAKUpwwvTvg8e3gVlVUc4rqqqi7deErpN9U2uMy8WpjSKFDmc0RVHRdB3rOqKNYlaUYj4w+ByBvmuFBKIUPkYpUB4jdd7XZrMZVVUxm81IMeBjS4gepRNF4aiqGVU5p+89MbQig6AVRluKqiSEQB88FTN6H/C+o/dgjJM9TCVsYUgRVvWamCJVlmMToo7YGalPJB0hJHySZ6y0zoxgPxJtet+KXVMWOGexTu7NaEcwQjmwupC9VSOAvnOkFMe9cQyGhUQ5E7mO3nt8zEQehHQU+iaTNaSoZN/3qJyd10UhbOpO0/Y9664hxoD3ORcviXSt7MEek5QEYpTBFVaYvCAZCNbhQyIqAe9EpksRoshcVVVB7yUToTCaIdPvY21itWx0rZmsuVpBUhIcS2nzXWfX8sFvnZiz57az69tUW314f/N3XizSeeeY+MPZ7x7s4elXTIHIs4HirbU672FlWbJarbhx/Sbr1VoAVR83CaNKsjKNURwdH49+4fSc3nuapqFeNxwc7BPbMLGVP3LzPnB0+4ibN28RgvTRQOKSALb4TFprtFN0fSDGJNndSdF1fe47CfqkkAjZlhxIMBtbcpDtETw3hAFrYeIvGYYafDEOGfZpnDtngyADbiP9m/Gv4ctHfyxt/50B680zGuwDwcYUTJ7XWaxH5p7Y2MN+q0mGnJGpx+sfg+UqggokBUGAEwk6agNB5qfWEhjV5KBdxuJSDAQSxKwkkIRlr7Qa/bocziPFHGCJG9mcwY8U5YYJqXfAGbVgdmTs7uW43S/bel+GN1D3r2CpLrBOuyhd5RQ7MliZjSsyGKo2IOfod46IVQYuATD4XvIRQoQUegobIdYEAx5FVI7KORa7F9jfu5eTYodkHAtbMp8dMK8WXJzNmelA17a4wrK3dx/4ikKVmFjjKGianvVKUzeGw+Nj6s5zce9e0s4eIUFhE8Qe2ytWK83JypKKXfzaoQvDqj3B6l2sKajbjpsnNVZpLhQF9+yVvPl1r2ZvZ8Fv/cYv8YPf/90c3j6k0JGLBzMeuHKREFccnR7z7NNP8dinvZEHHn4di/k9PP7w59L/7ik3Vx6fEjFofFqgdEFIBqUWaHNPNhaWGDSWSO8jSh1AMaMNnt4rurXHsKLciejFgujmYGe4aodZNQOtaLqerlsT2zUudlLE0s2IxqFcRW8coboHOzugdGV+zlnWhM2zHJ31M+D08KeepokwffYwAN3TqTmVZBkWG6WQFL3hqHH+y8Q3Eyxh4ArmMwADa3zzHQk2hULHj24fMOpS+QA5eKNyBWCVGFnIaEUKPc7XzNIJMWnqxoN1EtHUku5mfEepFZUpUaEldrcJ/gjSmsKsUH3EKY/RM1qlUH2DTocSSTQCYoTUEp0hJUdRVbhijmQOmix79LGDbCDGHqTMiJyya8WIATEqiMKCGIqo3rHLstk4z/5+dhM/tympzBy9R2lPUgaUw2hHil5Y8VKYGVRgo2ZMXqjPgpYD6CT/CyASBBwzAyiteLFLG4y8IZU2pEhUAsz0XooD6gkAHwMsl2v6PrJc1sQy5WLIJ5wsT+hDDynirKGaz2i7Dh8CXddCCAQUylgKbTHWYoASJTIsRhhJfedJCaqFw7mSoiiJPjCbSbHgMrO42rbDaSMbmY5oejSWlKIwd4zFKs3ytAYS1lXjBhqjRPfLsiQlYbgMzmQIgbZtadtWDJZ6DUrTB4k4FWWF1oa67VFKc3pyjCssBwcHLFdLytmcnd09ei+gexPXgIAQIdcDgB6tNavVisPDQ6pqxnpds1o1lGU5Suqcl3Z1nkMPeVPPFvNZDeUXS986yxQ8V8+PYU2T3U+c6e0xf+dM2TC95LwKlMEohSZiEICWmE3AjwNEP7p5TFOWXLywR6EdOilC2xG7XogJWoGxaFvS9tAHBTFgrKIsKmZllY0cAX+PTk+4cOkiu+UcPRRz0xO2//T3YeYNBkoG0c2g85slfFASjJLfhTFw3vMTIyynNafIarWkrWtAgH9nrTjtKTt+Wst6b0BgoLB5ECOonKteqEQcd6I0SnlNjx2uKYaIijkQEyIxaUKy1F3kZFlTN0s5UwqZeWXpu0DhZmij8R76icZiFleQddxacBavpBi17gM2ZYN0yJhJZ0eUXLfSd47h4QbU1l+QUg50JzH6SWKHxRjQVuWe0ZgiErqGum05Xa2pZnOsLYgBQncsMkFJ0i2VEgc0GSkQZZ3dAgybpmZI2Q2Zadg0LdFHmqbm+PiYEDxCgCtIGnxqaTpYnjZoIkoZZotdFjsJ4yMhiTyX3NPoEUoQfgx8hjt+h6lDooioMUihtYxFpXsGd2SQfooxoLSwY2LqgSBOQZaRikkKtg4SFARJ/+36BnRm4WhhuFlXSJr9x9GOnyvx3Q737D/O//tP/y8c7DyESoarN68yny0oZxXr+giMxRULgncc7D7Il/7h/wdvfuPn8cEP/Ra3ri7Zq+4jJY8rO1bLJa5KPHhvwc5eRd0omiayPE04V9GuwanIYl5h3IrCanyK9F0iRc3tkxMuXbzA1aeOuHrjWXYvWtzMcHraUNfPA5H5bsWssGASdddx32yHGI5yrQDFtWvXiQkeeeQC73vvdZwFpQKzSlMVM+rmhP/7n3ob/+Jf/ghN54WV5nv2LySMg9Nj6OtAXXsuXV7w5jc+xo2TZzg5vU3f5WLKFozzFDM4uOy4efwMbauxakYMPe/7r+/F+44/9MY38+hDB6R0MsqGNE1NUTpS8iI7ESM+O746M4hlkAWUMuPYG8bdMCfOyoaNYDmZ2Z/bNPA7tBfT7B+c6BdriTuzLobrmhKdpucbdFgl2Ke23pt+/uy5puv3wF4b5I6mwP1w3ACS31m8WPqq67rtc+b3hv6Yvj72d9zOSjHGjOuA7zuUNZOiqumOc529x6m90LbtCHJvdIbl+o012w/xo2xlOcss6Ii1BQDeK0D+ttZl+bwgiMywzud+9d6jtRGCmB4yF3TW2Rdwog89fpA2ycXbSENhVwHWB+kM54oNIBk8RhWUzglgRKLpvPgISgBTZ6T4nlZK5GQSzFwFTlNWJdZqTJRlUekkLCglbGab+15IL4CV/rbOgBa/xBgpEK4TECKRRO/luZaukExJNAYBW1P0tPUaryLKKFxRUFYzwND2DSkFjIFqVlDMCpFuCsLqlOKCHmM0TjmMUZSFhSBrSWEtzCoSEkCOKRGV+Al91xODxxhFURhAGLUAPiVslk2ZzUQysu1qfOxQCqqqxNvB9h6uZwCSpL/arhsRTOMMUSPZjUrsyqZpUAhQ6YzNYLDIMaaoaJoGk+fAIFWnUiK0LU2MLOZzrHMQkzA8lVhGIUstGGUwY1FljXOlsMV1RGFIKoIG43QuzC6SCjEEurbD9x6tpeZKiCLzlhAGu9LkjFUJ0He9EEB0ztBLSmGsIZBou5rOd1htcGbyWRXRVgB+YzU+eVKIVNZinCXFhNEaazRFaQkp4aOnMApjNaEXgoAZZIYhBwM+vr07BtHQ1kRyyEXskRAl0yjLzgw4hrB8s+0ykZZFCeEwDNrbUXwKWY82Mn+DuSfrbs6Yz4DuVHhX5L4SyunsIwt5a8BuQuwReVQPyPqEHjCdzHoewZz8TynI9mkKgvUYbUhqAEgTfd3ju0Ds5R52F3tolckJUWoJNV2HdY51W4sM0XzG6XI5ZhrFkGSuaTBGEZIQtKayfwP+KLa5+LTOWprGc/XZa4Q+0bUdhXMolVivRYIpxEBSabNXqIjBZYkWQ9d5QvRokyTrPGrAyrpLFFJUVOTuIoasex/SWPg6KU00gwRLGgHuO+pOJSHVmK0AhWiFD8+YMUAe8/kGSbPsrarhO4aMbTlxSkn2+dE+mQRZhrE6AfPjWIRz8AcH/3fqG8s/o3SWAM/jJKis+iDrzuAb+9hLvS7EPiePVzVc37A/K9DKIrJmmdySwPt873lsxRCIwUMmvciJJ7690mNW8EBImoCbL9peNoh+8/QKnZvTuoqkC+yARApoP4KY2R0TZ2/ou+H9wUBLGx0oHzyd7zExgilkodUaRUDHDhM03kCpPDOjuG+xw32veJx+Yem7nqLa5eLeATtVRaHXGBq0XmDLinLXYbXDrwLBrzB+RWp7iBKJb5sl3TzR9D1dgNPjE/SspsTTrGG1CjiTiL2n72oCDdYFGt+hFSyqGSa0rLuWmdNUpQYVuX37Gn1XQ0ws9mYcXD5gdrCLm81ZtzVVfcQL154HFA8+9Dp2F/s8dPkzWD9zyrrzlLoixhnLJhHNHGNmNL7DhAadvLBPYsTiUJQEPyMr3xGioyj3sG5GdCWqLAnKCBacEgSIvsf4Na47okiy0cVqRofB64pUzLHFHmU5w1rNkEWgJwNKyQuTSXLWYN+wfAX0Tluv6c1ZGNjDMq3SOGlETzahjUVHWRCUzHz5XJITqfyH2tAFJ+fe+nMEW8ej0p2fl7mV0KkTtoUr0IXDZN3Y6HtS6AkpoUONpckRMzDxlCIlZiRSsnRei3yL8djUkbobhNUHabqbaKOZLy5g7O5YLEEnR+jW9KlB6RkhnFC6JYXdY26dyOxoB6ZCpQAhony2Qj+OFtKG2ZN/Gf8epAkSAqLrvAlNMhO32nkA+vD3SwLpShZVKdwnFZhVCpi0CeDI2pKL/yVhfqdsTEoKXmQToINh4R728GFBH1iKL6fFGLO2tjA5ohYwpeu6seidNkaMxuBYLtesVy19F3n+8AWeffY52q4hpZiZip6ujbRNw85iBkDXNOiUKEtFVZZY49BOHCQfgmiG5421cImuk+InVTUT9kQfUVXOStCyaccQgIjL0gPRi55gyJuIUUYMeGszoGPyNfaZDRYJXhgY1tqRgTJldoZQs16vpcBaNqKbXCxvljUiP/CB3+Ztf/yPU80Kyqpg92CPEO7D+562EXmY4Fv6vhHjerJx9307Fojp2p71qmE2KynKbSd8084fX1OwYgqgDe+9+JBUH/GYF/vc9N+LHCTva40yss4alTDaCLMK2dSH5LePtR0fn1A7S+E0u4uZFLeJwloQxqyVauRa/vkgAICLwoTb2dkhJTVKDIxaskpTFGJCKEDnBSEkdcf9Z2x8EozI5vZo7GWsQQEqjhqEQ19O2wAOXL16leeff17GpzYZmJF/ZNZkSFGufQAY1BnQZ3wU287QS/X3WSaNGKQKYxyz2YK9gwPmfYVSCSsLO1pbjLLs7e1n5orIysSUi/8pSVWNUVNUJZIRIFkv1pmRMTkFnF5sLT1/vJ3z2tC/466bgXOd2UgpoYxCB0sPrFc1t2/fxlmHNY7oIrXvWZ4cUa9ORNNfydqdcr0GY3NABWFzt21mm8dE17X0fWC9qum6ToKG2mR91plIugDWGIrCSUAvSmH7AXBvmnoM7A1rwSDZNcTkB2ByWrQwjQ95Y28IWUXA86GYozCfDFqRQZSWrmvFiUwCnistoPrgLaocDBJQN6GTQQrFN1ijsaaALMtBNKT48Tnir3vFF7AzfxidLvHEB27y2Z/3AF2/ZNkume3MuHrjSZarW9x3+QKveMUr0Ch+9Vd+gw9/6BmKwvKmN76J4+NbvO71n8FyveT/+Nkfog3HVEVEu56kaiIdyljuufcenn7iiKaRbICjw1Ne+egOblbQhJ7uxBM7w6r2zP2aVz5esmxvY9wO91ze4crrHuG9v/o+FJqbt2qKRnPvlV3uv3KJ559fQdrnzW96mA89/VsUhRBrlIELF2esj2uST9i0wOiArW6wf/mQL/uf38S//vH3Mq8W7O9bPv2Ne/TxBJVm/O77Vzz4wAPs7Nd0/oijkxoVC4JXzGczSpcBa6Wx84SbV7Q3ehazGV1TszPbx3vPk0/8Lo888IeIIRFUpI89VTHPoIdHLNo4/oyR7NTJPmpMkcfRnRrn5zG5t+Y5Z9jQOWdbnOI77asNQ5pxPN4x9dleJ6YOuug6i+M9HD2w584LJJ+n+T7KxkzudSqBNL22Yf6GEHLmxyZYv81glJ9dlr8wxuQg/wAyTMkfm/40xogFqDZyLVPddOcc6E0Abbims2zzs/eqc4bgEBAYwP/pv0R4kfX45TWtDd7LGGqalrIsBaC2kp0pIEAeVxmoDkrnFHY19tMw9uQaE85IIdzOd/RdJKpE4QpMAmN7Yi8Sf66wGZgRNrQATh1VYZm5GYtihxgiTVvT9rUAsEWJNQarc+ajlmc/r6osWadEU9q4DJYqjHVEvAREs0waSlGUpQDQMeJTQjuHdgYfWnq/Kd4KCoPUDoo5A1lbCfakEGk7kUyM0ROSSNKoSoK8yojTZ61Ba4WzVnTbESDVBwlx997Tdi1lWdD1Ld73EHp802K8RRsrRMDsaDhnM1EHTAY0nRMJm7JwOCvSLNaILFrpCgrrRkC37cUHqiqLUpG+G2QWJdhgcr0EpRRtl+3oGCVbWIF1jsIVWG0lUEQi5eBBOa/wSfT6+z7QB6lvFVWUIrxRyEatF339pFPOLBJd8d57iqLAOCNYk1H4FNFJE434Aj62Aipqn/tY/JGYROdeJQh9T/QSAF3VKyEpRamlEmNisSjEt+pSLmptUb0Q1HxMQvZwGw1kn7NZbeGYz2f0nWRyayVyZAIsk6XkpLChsxZlNL7v0NbhCqlH07c9IfTYogAk41ZIGVoIBDGT7D6OpjLQqVLKNoNI+IkNq0hJY7KOswT5GfeD0aRTEz58Dm6Iasf02gaf587X5PUN4Mi4dm7b39N9Rch2g80VxnMpxQg+3hmMTRswfWJvil+f6OqW05MlvpOi2cpaurYjhMzuVoMMkCHpJKzkEHNQe9jvhKDTNI3MZWeJvd/qC9lf1B1B3JTg6PYJx8fLcc1VWuU93DPABUomSg5iSGZ8DIngoW1FMsgWGp1SZk2rnJGSQeeJdIvYC7kvBieJjSzKFraaIBHHIM5kFG2e0fRO1TZRZNoEt5sGFba1zGGwB7YzyFD5wkZ7Z7iu/HhHf27at5s9VWuV/cBN8HuQsoxB5L9UftaoTCqKmhg8IUQgjO+rCaAe8fl6Ito4tJY1VSeFDpHYSzZQyBiIGgBMNmB/CJtsiiFL4OVs3S8bRF+xT1IzUBarNSbr0pLp/gPPaeOU5oc1wuXTiTg+SrSCwjpCr0kq4IzDhR6bOoxfElA0ao7qI7Y7xbrI7MIedudRgu+IUTbkwhrRUV/eJqTEvDxgYS0kB7NLLNeWvZ0FXVixDi2VbnBaZGJ8aLnxwiGr6x+g29G42LNennB0dMie8hS+pPUrEjVrTigv3YctHDouSd2KGDpu3z5mtWqwtqSaVdz3YMXu2rC/O2P/nn2SUtRdzcIa1s0ppppzfHRM9L/NI696LZcWe7zq3jcSbsxYhwoTC5rYc9p1pLAm+ptobqGjQXNASBafDMYWxBTou4SODYuFMAfdfEaqNL1O9CmgQkPsxHk07QmmPaRIKzE9ErRYoqkEnHWie2rdUOU5o9XZKx0xygH0SMMiME6LrTGA2sQlN+t+2pxjPHYAcPPISVGYmUmcEeVb+Zw1JCnjmx2JYYKzdc6zTanN22OV5PF1lRnqsiOlkAMNsUfTkajw1gih0Xdo30LocGEJqaHRMwJgUkNJzcx5SAabCmIUqQbaG9A8iQk3sTwnG1Z0+DYQVcInTdsF2vUx82qGKRzOGpzpMIWRArN2RqdmRDcjJE/Mzy+cU9zzo2mbKCM5WCGdFfPmGtWwc8o/nY9/MQL8iwGILwWii3xPHFcMxueiJdVHG5Sywp4YQP3ECJhIwdHN92++U41O41kHadDL2gyarW1I+iCmDJgHdEh4WtpOmNh932OyHEpRzujXmtOTJbs7+8yqPZ575hlOl6cCCmqNNoau70iZ1XV03I1afDFE2q7DFCVGabomSJFfbUQWQQkDNoSU2UUOax0hJOazBX0fuHTpXhaLOfP5nPe//3fQRrNenuTsoEjfZ/mWvhdwDzVqE0q/bG+4fd9LcaXs5A4OqDC8BmNOgTIS4FCSOkkSdk1Kkd9+33/h2Wef4v6HXsnO7oLZvOL02IpRokSmZb2OdF2d00ZTBrGg6zpWq1NuXL9O4SqKomJntyKE6vwxpe74ZQuQUEoPFuaZ1+8cq2eDQR89kM5LfC6zNnRm02Sn1uiEUWKb6yRpsFobIh/7/O66juB7yXY4AwaIxvyGOV6UpUimJAENBSyUYk9FUZKA5XLJ6WrJYqfC6CprP+cAeoyiSTcYVtN5Na71k0yVNLD2fE652wYtzjIAlZKUzePjY37nd36HD3/4CVJ2cGSOTKVhDIwsQllPzgJAAxCldXYGRwcACRae0+9nAewBlE0p4ZxlPpsTSisE/2GOoETf3lhQkbKoCHntSimgsgRO30ecrbC2oHAVVTnbMOpfYqxO++jltuy/Qcr0h3E/3M7MUVqAJh+DFBlVmsIV7O3uEH3Hqj7h6tVnuHXrGt63GfhjBOdTEla4z3IKvffZbBCJG7Tj9uERKQX29w/Y3d0j4dGNgD7SX4Y4LzG4HNDrOF0es14bUGmTJpuiyELFjUW6YcwMYPrEqcuj4CwzeOhvn4OI0l8xS3mFzKRJkuFjEsbEPL6kr6yxeb2UQGSMwlh31oCSQlW+61GxhfTx1TOZ7+zy2KOvwTe7LNcr1vUaVwUeeGifgwMHTvPT//w9vOrhV7E8+XR6n/idD7wf6yquPPAgFy9d4I/90T/FzcND6jZAkowHghLyQvBSqFrt8oH/do3QRy5dLHjkwQtcuFRQLiK6gjJq1v2aqy/c5uQ4kLzlVY/NqJs11Qwu3bfghRdeEIanT+zOKkypuXZtyXLZcnHvHh565ApPP33E/uxRbrxwg1e8csbu4iK2v8KzT3+IsqxJyXB66rk4VxwdX+U3fv0Jujbx+Z/3Jv7YH3k1nud5+ur7KaodXvOGT+dnfurneeXC0PVrbFSoaCkKy8WDGUGtCdHThsDhrRWl7glecXJ6lfXx8xhV8spX7fPIg6+iPvkQ83sMRIOyhrpdS+AZT1FYlAoj0zehRMpP5fTi2BMQfVWVs6FEHi9BBiMGUHbMBtTqjGzKJlsnDjZxzuQZ1hiUZPKRA3MxCmPLGitzPM/NTdBse12FYY0UL985N3wxm+L1Z9bQc/bNQUZlAJY365LO9z/4FXIvShsB+3wY13BZQSZM8yT26Hw+Z7lcZjCdsdCqzgWLhwDDGHhAg5HFLiLn0HZgzkobJEIl1TvR99IX1sr9b9YMPf7sey+F7XLg34eWaaaLtSZnD37sBJcQhjVH+svnLI2UEn3XQy7GOGiQa2tQFiQrSPa1siwwxtG2XbYZc8FphWg92xKjNWW1wMRIjBZvalCig991/VhgUmtF4QyFsexWO1zYvUTbdNy61dN1iP9ezeS5ey/EI5UwSTNzFRZo2obkI77z+OBFmlYJENX6Lku4SGB8AFL6PhADzOYz+hDxHiny6BNGQzSaDpHaiilJNpq2hKToek+9bmn7Dg1URZlZ9xGdcsYAUFqLy/3VxyiM9RjpenmmfS8FRLVR+NDTdg0qg9Y+9BhSzsY0FEZjlMFpjXZW6mrlwPIwRmZVKeOoKIhRyC7OWNquw2pL6SpCjGPmXq+8+MTWYgrJSoi5iLnYazLuet9jnEErizaOoqhk3odeMiOUsMmNNThXYEzc2EgpUbfrMSikCwU64FVP0iXOOIoU8TESksdUhUgp6ETovcj8REOhNSHmWlva4gpDn3JGg5esiRgEqaoKB9aJXFj0AtxNgEJXWKIusXkO2t7R9j0hBxlsBBUiwxTVOmsjR1AxoqJkhxWFyM7EKCz3FAO+C6S+xxYlfYzoGClmFbYoZB3vfLYbsvRUCnlOS1DlxTJTX37LdjA6g3sCUGozARlH31yN95hIY+HEhNiqiZxRnshSs/nINM1W2paoGKQ2JPs+jOcfM+8nWN0AmG6hemp7/R+A7OF3rdN49AAAKSXkDPke+bw1DpP0WPMmxojvRf6oKArJKAj9+H0x23RD/Y8pscUaO5IrClNkGZ/tuhQD2CvB3pSfaeL09JSmaQCZT8OeOMarU8auUhr7LqUo9m0fRSauaSmrhfRTLjgqPoIQeqR22wBkD/txtkYHv2hApNnY4ikTbrKjPbFZz2Bb4zkm9ve43zJ+ZgOuq8lb2z7v0FcjRjZ2xDA+z89mi6M4+6BHrkZ5F50LJo6Bm/wld9ZKMXlPh0FKEYTkE0PAykaXFRJSJiSBVXYMkihUVgvQqBxsTXGwfe70pQYAfQTsX4bP/7JB9FRdwhYl2kpkcMC4EgMLdIqs5gc46KCfOZfKV5tIuCwFo1Ri0a9xzRqbPLrvsHFFYcClkp2+J5zeZpZOWezO0LqUwj4+4vtAyClcpq+Jqwh+hdmfYWwgoplTQVox04l9lzhizb07iku7PX1/m2dv/zYnN56B0wrlPV29oio8ujumy5FRVGLta2xcstgrmM0KnAukWHN6avi3v/iLPP7IZ3LfQ4/zyGMPs1qeyEDIzr1G5Arq02N+59f/AxcvXsa6ktImLh4csPAN9+8+zPO1RkXHLj0+HtOFNfSHOBOIoaLzgT4qTut1BhU7knJgSsrZZYqywjqFUg0qePAJrSX1TxPx7REpHeN0Sx8cPlX0EWH/FgWuqMa0JciA8whwbxbQ7Yk3/L0x5oexwRZDPC8MQ+glsVkBhoOS6IaTcrTaR4JvmfXHWHpimtPGGcpKOlre+88M2HPG8PAtw2KYxHgeFiGlIMWIb3t0X6P6GhcbUiqIClKQTRTfonxL6taksCboRLCaqBMqtcS0IqhjktaQ5hBLUr8ktTfp20NCf5uYnieGmnWsadRcmAK6xEeN73qceZSikvGlaXGmx5Qdsa+kwEpZ0uIklRC4Q6Pmo2xeTTTxxn6SKGyfAgnR0FeqwdBglAcc09l93oJzlrU5Tc+VNWJ4GHmDwRIRzUSlhdEgkcUiaxxneYAEMRq5rhypTzmSm0BAgPGp580+yUKqtBQpjUkW3pi0pJIy6JnqjXGgFAFJXyUkohd2RtfVrNcnOfVcU7qKpm65eb3m+Wev07WRK/fez31X7uXppwsuXrzIyfIEH6XoXN/67OzKBlE4h3OOrutou5adwYgjM7+iygUzPClJim4MCd8nnCu4cPFeitLx1i/5Iu67fJn7r9zPB97/fup1zXve829YZY05k1kDMYmDr7WhslLgNJFyCrWl6wzBR1DrreKRg5aokoc5rhHWFqiBBayEsdz5Hm0sR7cPee+v/jL/8yseYl4U7C92WFbHXLp4gRh66npJ31larbOxI8GCmAGruq65cfM6xhgOLuzRtvuyJmTd1IHZPAyzmITFrTAjkKZVdio0H1GjdJtlvAEaXjIANK59k2MG23FwUGQG5L8HZz+iokWnAqU9WsWcYmcl9TGJE6U/dgw9gxgSEGm7DhX6jXE43BsCOFhncYU8y7ZtaeuWumnHAJtofp+wbtaslg9y/+VLVFWFc475fC7z2/d3XIPaMry2r01kMiRItTH2tvt7cDwB2rbl6tWrPPPMM6zWK7Aabe2ofS7AgDDDpgC0sL3PpOZPjKkhaLdxDpjW5pzcx3bTWglb2hUURSnFuiIoFSGE0fiTMSRsH+8jbdOLFi+iEaiVJgQw1tN3nr6PWJukOHm6sz/OG48faZxubgRGGZsh4DExyodU0GH9NsYRTU/XNKyWK5q6ZndnLgCJNSQdOF0eUddLAb7TVFpNvlDndM0YAt4H7rvvMvc/8CA+CjDz/NXn6X3D6TKC8pRFiXNOnKokqcPayvrd+1bWZKZ9MjW286KV72mrX0YnY/JM02Cga0iSAcKw9jJIyymcVUhxZmGzKKMxJo1gmVIKq7Wk9ltHtLKO+d4QUydjQkVUUUhAJWli/PhA9F957y/wn37lV/n8z/0y9nbu59v/v/87BxctN29/CB9FauWe/QdZr9b88x/8l1y6dB+L/V2eeOo3eeWrXsGv/OffwOgZn/O5n03xvBWmdj8j1pFUJoKF0GleuHZKv0488GDFqx87oCg0gZY2JlSnOa1FSnF/v6Jf1ywPPc+bRFUFukVDe1LwO79zi/vuWVBUhpOl5+I9+3RtS33a8kLzPDdvXaU0cy5d+EO88oEHOT78ANeefpJb1zyvfd0DNP4qxILlMuCc5/mrzxNiw6VLFcer3+Ha9Y6L9xbMdyuu3bjK0e1n2bnQEpKhspYdK0yly/ddQhcenyIox63rHfW64soDr+X9T/w27fI5NIpXPHiB+648yr2XHSVHxDgTp5KAKw0is2FJmbHqjOgTpwRd7EjEcU0XQkrI+4+Sei+KnDGGANXjGGUE1sc1ZxjPCqzWJKvpg0jebWoHZWZ3zP6QFt1ln2sRiNNIBt832uJDgcKu68QGSMKMVDkbSRi/WbM8r9HTzK7p/Jpqgw9A9jDPzoIFI3GDIfA2pHNLZs5w3+LIb4qfLhY7W9Ipcjq7dR1Dtw2MwgFYV5kh3WcG7ADAWyNFWSVImNmtMYq2/QhmyTk7349SCTFJ9vStW7e4cOECrijHa0rJC2P5Y2wpgu9F1qYsREJ1BJrQqCwdGbPUhayBkxDJxPbw3tOqNq87YYDfMMaJX68rrFJUM4WeOSm82XfZttKj7rXVjuQDwfdApMgSaoUrcEYJgStE+hhx1uG0yfIUUBYlIUYpFB16CagoAaH63tP2rWR2Kk3UuVhfGsgOFm0L+r5BaamZ4buOiMIUjpifaQiBzncEH7EoYufpYsAnqfMxLywYTUx9znjTUrNKKSprUTGRuo4+BfoQ6XpF7wXAq6oSpXIwQyucKwja0PcdQZHlPyEJQTXPe533Q7BG8rk9Aq6lKEQWyWKN+L7PdDUJamitMoGlzSAaKC2yJDrPy77vR3u57zsiEVcVEEVqSCFrQIZe6Xqpd5Qg760JV0jtmN63mCg2U1WVuGjpuo4+dTShZubmwtQvCrooAKJxhQQlfMCHQAodymop6peSZFVaA76nj1nWJoJKwgKfVxWmmlF33QiIig6+ZAJoL/IPRqmc1SNFmq2R+VhohdVZDkJZfILgA41fi8SfkgKUhVYUxtCFQXPfE5IIUPTeY4qCRM4qsVZ8b6XRVtP7lkggegRzSEImcs59zHMbmX35qUg9pJgNMsVk/VMZL5lsA0Nx1ZiPFV82ZfmLhB5IrtnPGAOLA5KT94FBSjElcjHbTUaSUiJlI+eSlXsE+cdj9GQdzrbWkFmX96NhPR5+DpnFchkqg92K4L2MVyVEhBh8DiAZyepQbiyyi8qyQDHkzMbcb4Pfnr/PuVy8lmndj01x62k2+mq15nS5pOtzjYmtjDHppzFgncRotkNAO0ViSKyXNScnpyx2KwHQg2SlhUzgHgh/wz6ocu09UQ/RIwazsWuHvvtII2nwYbb9kyFYMPhVI5mZ4dCBCb8hVk0zv1IOFshen3eVpCErBQwoMBkjjENAdLBxRht6c97xM5Bt7UHijvFzQmxMkhmT/dcRWNeZ2JlSJmElUZNEAkfiJwZEZlzTd4GE2DHWWrpOdNIHotVZOb1Bm17u5RMIos/mezLJBmdTTYBVhmmfJizlND4sifoliRbGSN97VOjRMaCix9VLZqvr7K2fZ6YCWhmK0GKTMOdi8jm9PWELhTWSSpeUIxUqa2R5UgjZA+wInccftXijSCqgTcJFzYGegQ4wL+hsQJsjTtuWHbfiZmw5vL1Ce89OaahKS2GhV4ll6/ExMKuE+ZliBwSc8TiXSMHw/M1bfPDpD/NHPvdtvPkt/xNPPfV/0jRLurYl9JEQxLFzRYFKgZOja8zKGb/93v8fjzzyMKvDFfc99GnEYp/DtIunpevWFFpTVfdSaEXbRW7cXnK8PGHVRarC4kzClftgZxizhzGSNqf8CTpEVBNRWKxTFDrQpRP6eIOQPCpdJKbLeDMnlfvoYj5qI6Woxsmx/ZyHKaAyHD6ZmAwC/3mBYJMiMSzk0zPl2BYpCQtHpyRGTAzE2ELfij6qr1F+hSsUnVeAIxoBwc8qmQwYFQMIosREH69jcLBjyiwIcjV3Sf2jb6BdYrslKnZoAkkZoilIsSM1J8TmGBUa0AbrpNBaYRocPUkFVqtjSKeENMeYCxSAqFKX9LHENx3R32Dp12h9CWXvoXDg+2dRsSX4PXSc5Sj6LUJfo0yFjQ+gVUnIxVAjGRQM25HWj7ZJuYbEUIw19xKBQFBR0gBV1tVOPeAnR23W7rNAz3lM0g3DNzPAVU4HUxooSMESUFKkS2lQFq0cRhcYtQHRSSkHZYUNEkmyEemsazW6EuIsKJUwA4vdRFTM6Z/k1KAJoK+ypmnKUdMQPCG2tL7LBfZOWK+OIUZ2dy4IQ1KX3HPhHk6Paj507QmeffIprj7/DIeHt1gvV6L7l5ljVpvMVJLx772nnFXsHxygjMW6Qgzr7FYabVHK4pwhZge5ms2pqgUPP/wIj7/2cd7whtfx6Kse5v7772NnseDT3/A462XLzVu3+cVf+iV5zkHYII6CmDx9L+umGKdGmJKpRzTJHcoYrHISOM3HxGx8hJQIxJxWPaMsBEwtyxlGG4ghs+ssv/Fr/5lHH3mMCwf30q9biIndvV0SkRvXPavlieiUemE+hxRz+rEwOut6TdOu8b5lNiszljAYcptNPqWU5X7EEB8Y3FrZ3IeJkMfu2bF59rWz4/kjtZc6dsiYYSgdqgYjREMyqGgzKyPrF8gfMu/GVfdjazFGjBYHpO97dNwUjUspkbI0i1KiJ+i9GBnBJ7qcSj0AKUpp1uuGGzdusjo95fjovlF+48qVK+zsLLBDcVBjxjTmGAf9x5wEmL/fh0SftabH9H0gBikeOwVFBiP45OSEZ555hmvXrhFDxFUlhStxRYlxTthV1oHKGSxq4JNvrz9bz2e0axQMuuNbDMrtZ7zNRB/qJkixTWFJNigiVguIkxKopKWIZFI0Tcvx8UlmXg2BHQ3KsIcwMgcDVLJl7rzeFxtvL3fMij071DtR42qpcrAhjQ6VMJyMLdBW5v26bajbhoO9HVJs2d/fZfdgh0hL6oWhpYdzqkEixbBer5lVFW303HfPRe65cEDdeRT3cnjrOs4ofNdwctRTliVVVZFSoqvXdG1DtCqzWTZyToMjtrl3cZSsFS3PszaMjDE1OnnStVoK+5oBGBRbIWTZB2un+5ZCGym4p3VEEh5Ej3JIcY5RWLjWGZRJGC1FqEKIIzCaosL3Ef9x7t2X7rmP2dzw4Wd+mVvX4M2f8YVEdcK1W79GVIe0vsAVD/D4qx/nngsP8zsfeD8f+MB/5frhc/zTH/jf+cqv/H9x84Wb/B8//WO84dMfZ71uWa57ytKilaVee9om0rU9b/msh9jbA1t4umBAScr1+mRJ0/UsdhUPP1KivOXaU57rVzvuvVJxcX/Oc0/XNMvEsfYc7CpSn7h9/Sax66nrlmrHsLhgMbbl2s0PcOOFmjf/oft57NFXc+PejuOjGyRbUM4CNtQs9g0v3HyWz/6/vhZnZ9T1ba6+8GFWbcXOhTl910GE1zz+ak6Ob5N8zZUr8/8/bf/5a1mapflhv9dtc8x14SNdZdnu6u6Z7p7p6W42ySEhicCQoigMDShB/wT1F/APEKBPAiSAkEDo65ASZSCR4miGJGbaTbUpb7IyMzIiMux1x23zOn1Y7z7nRlZWV04VtBOREffcY/bZ+zVrPetZzyNjx+y4deuEpr3Lq5cDzzeXvHx6zr//b/0eZ9UDfvKjv+Ltd2bcf3ib15cdY/+Ib3zlDjorHv30Y775a7/G2HkxSRwytqnQShNiJvcD2jisqUV2YQIrUPv15SYiMnV33RzHbySc+6GbS9hTDLMQRnXmEGdNbFSlit9EPkRBwsjKJZOaks6p4yKQUqJpGvnsPHUHJXw87JdKa0wBdSYm2c3znYDnm7HeZ+PAm8cb4Ls6dIHAVBBLN5Jc0XSewKub+wJM64Lev+4mmAIC7odw0Fl3TvxeYjG1nArLzrl9cTfdSOD3OueFZe+cIw5xf05HR0fSmWgmA3fR597tul96bpuo0VGRQ8Q1msrWYpaXgagwSWRTIgGljYCDKRKTApUwRMYQcGUNHMKIyqGYVct3wzjhwXYl7o2Jo+UJfd+xixtSAmMlFnPWoU2m9x1dv8Nurlm0C+q2lnFYGXKtiUHWNGsmsDtCNhjtqF2mG0ZylFggZPAhMoZMiIpUckGTEppEXQrkUSuUtVRqzqJt8H5kpVYklXG1SKDltCV6Mf/rQ0KjRRpCa0xdAZEhj6iomGzcVSr+AwhVKCUEIEbuc2AgmyIZRCzjX/aKul2gWi1eH1oTC6CmjBFfkNqK0WsKxJSlOKXFPyj4XgzlrSWSGKJHB1sAVItVIrG47Vbsdh1Gi1nq6DsC0M5mIjsTAoP34qnUD5jaUlWGcQgMQ08MY8khFEaLGqoqBd+YAuMo3kWtlQ4zay0peNGDxhQT04GQFONoCUHiQasygSAdp0oKAIP3qCxybjFFfIzEAKn3aKOIPtJvOxQKZwy6tiSlMAXrnYpq2hgBJlNiGAeUlXUzhogCmqpC1aYA955spMvXTqzUlOmDxylDbQ22qsgkwtgTfSAGiAFICmMtIYmGtUkRlzKMkewjzioq50iuoR8GyAqTHLkUPXBfPD/4IoekoG92HE7A9RRLTfGiQgoguhTRUtGnl0LgFAdN4Omka532Mc+e5KhB2cmr6MCSNkaJZ4E55PdqAuPL/mIK6QBkXMXyriK9JeUBRb4RkwoIs8cLknRcTX5fEwkipcmkXRV5USc+BVUl8ioJMgKwyporUKaxQtKYNQtSjCjlSpf3oUABE5hdijUo/DhycX5O3/fEIh0VQygeQ4cY9lA0FtIfKFSmkLZSkRztpMtiH1vuIXO5B+x3/wLKT3vkAWTfE7zzRDKGCSjf3783fKPevD+fl8ceDH1LvF9iD12K9zkixbYSpuyfW/LqN7TRUeUawKStX6LjfaGibDHlnpZCr5py3LyPfd48T8p+W95r3xkv3XGyX+cSA0nhIk/Xa/IyRJGSSCElJKfdE7oLpqA1aO0k3ij5YU5F0SBPIPrPj1tuHl8YRHdG76Vwyvcr31qmqLQNlJueMjHlsnlEUvRE77FhxPge1++o/Y6WkTpHZuOKeniFi+IKrnKkrhy2mEagisGV1jgjLdoTMJ+BWV2hmnofMKYU6X1kjBHvpSIRSShrabTjbttyao8Y8kCyitdpxeVsztXRkpfn18UQAMAw+sQ2JZKpaZcVt07nLJcV1nR4v5MAzCiMk5Txo0c/ofr7/xbvvPt1QnzB8+efkMIK5SM5BpTJxLEnDiPB92zw3D094/mTQI6WV3/6/+a3/+h/jMHgk2a0kQGFs0eQMzGs6PtrtrsrQrYop7HVfdzsIao9w1atJOYhkNWA92v6bkcYAkezI6gUIV9CeITRGq/PCNUS29zBNUcYJ9ISSk8A+o2bXarp3FyEy8KU1ZSGH4CovTa1OgyX6R9FiOUGOHRwLSYlYhxQ4xoTO/QYIAxgMsnWZAzKOpQMhf14vPleWoHN0AINUu33GXZKzssqUGWx9AomvUUxX+xRaYcJHYYOFTqysRKcjjsYNujYkXNEuQXZ1pic0XFHVVu0bsC8Qx42bLsVWY002lM7jTJndL4j+fvYDFq3KPcO9eLXRLbDz3HxIyp9RRg6YnyFMlsa/SWc3eJjT7SySEkbu7Ra8yu0jAIkAlndbNcqi5wqk/yQit34efpz48b+gmNKmm4u/D//UIcFWlvE3MTeWNgKM2PSjEtxn1DuF+E8jcdDUidmMOaNJOxmW96UsN1sZ0op0g8D237F1fqcXbcmjAOVqej7AVW1XG2usMx4eO8BD++8xWa95sc/+gH/9X/9/6TvenyW9lWtNaaqqCpH29RFG9jTDyN13ZJTZrsVs05jLMZpSALIKQVtOyOEiHOO+XzB2dkZv//7f4+/9VvfZLGsaWoJHDTy/n/7b/8tvvUXf8Fut9u3A0NhJg19AesTbdsSfBCpl1JQiROTNuc9I2jSLU0pEab7X36+c/sOf/fv/j2Oj48J48B//9/9D7w+v+Tx40/4L/6Lf8R/9B/9Lzg5PSIRuVpdE4KApVPiayfDliha7dN5hhDouo4nT55wcnLMl770JRaLWTHO+nlskDeBr32R5OeMyenf/385SvQ0AXiZw/moz8ydMvr2gIm0g//yMPqBoVCuaTiMdzHsyTRFbzsjrAnvxasEpbBVVQqrop/ezmaklBl94PzyStZa6wgxcnpyIu3e1mKMpm1bmtKurISaVcpC8lnGarKyRXuwaLRr0LTcBNGnNSOEwGq1YrfbCWvFTqxzkaURCZcpKJKIbO+yniHm+AaDct+lT35DziVNiebnjIfPAthybvLZKYl27W63gZyIpT09xkRla8jStbHZ9lxfXzMMParUBJWyWFvhqpbRh8IsC7haf+azZCxM5kY3f/cvdWgjoy/vM1jIsqdKcJ/24FFSSlqwnbDN+r7baxPXbcN8MadpWtbaMMQew9RlJAz0kDMEBTni/UBdizRATAFrDeM47FvI+74nBC/svrJeSodKOuzT096v9Bv3o3zi/j5NxZF9iK/eXBP2j0noXd5zup7CwBfW7oGtkqUiwsTaEvBPOmD8KAZO1hYpDXKJZ6QwJSyoTArTmNSFJfjLH3/xL75PyBe8/+V7/N7v/s8Zt5knnz6l61fUsw19Jx4Zf/1XP+DXv/F7kCu+8WvfYP3XL3j56qf8Z//H/w3/3v/sP+bBW6d857t/xvXVOUlFQrSEWGPdjM36inv3l7TzjLIjQ+5JbsE4Jna7gdX1SNcPtG3FbrOmdsc4lxhHzfY6sblWPH3UQdS8frlDjS2nJ0dcXW25feeMu7cVIQ+YKnB6e4HVFReXG169OufDD55T2zt889d/nSHN2Q5PaJaZdpaoZy3LJcxnc66uAlcvR4JXbNYb2qbi8vyKy6uX/K2/9RU2q3NyMa5+650H+OhpqtvcXt7hw+/8BYtmxQc//ecYk3jnvVbILhaa4w3NoiK14E3gqrsixBGXJQGTgjFIUKlISqGUkXUtgdC5SxmnFCInQFrpCbz4OUabb3RxTuNyYlEJ+/JmC/R+L8sFuM/CUDR2kkhQex3eJA0j+zkvYP4IShHGUWRStDBnY4wFRJCVW9qd5bhpyDkZ0U/daqoYbWvFXqLhxkulA24C4zM3WMeHP9Pvp2JDSoc1fCrWTjHEZ8H7A5iuSmePRWtzQ55OrttUfHjTfO5QFJgKHJN8gPeei4tz7ty9+8bnTkWI6Z44W3F09IXT7J85NCILMvqAH7zIpRjpUtPO0bhazrNSYBTd0AmjMwS0FYZ8iImchPWdyAx+KDIX5by1/G5i3KusmFmHMxVWV2AVtjJkZD8KQ2QYJU40/U4K17UUHqJKbH1P5Spm1Uye2w/4fqSyNT4JXcc4h64sQwj0fY9PiZwURtVId5OWAnwW47wUA0Pw1CGwaOYsZyeMvmf0I2MaBfBJWhjy3pcOPkrBTNZw4zTOVWRiKYZYnK2xtmIcBiHjKZECCdN8Uog8Thm0IWbA7PcCKezMqKqWrBQ+imyeFJ2lQ8RHz+BHrK0kVtGWrALe91ROOm1tZclZEZKAwBakG1yJyWXvA03l8MOAHwbmTUtMkn9bY9GuwuPBihlqGAbCOMqfBArplDNaU/gFck0mwColYggYbcWzw4oZa06RHCIJMQ2PyRNywmDIWeTJnNV0vXid9P1Qvkcsa4sYO46jx9nA2A2MfTEGtg6jfImbSveJUNakaOL9oaPDFz3jBM5VaETnv4seH0eydtishEhZwOSMdAYZbamrihBGdn0PaKxygCFrwbWM02QtRS8BX2XZtkrhtKF1DYYKpR0owy700tn8q23d+zx1kujKsAdr9xrSE6pZngdTIUTAQWNt8c+JEodI27ysxfv8YooN854oIdl8lmjJgs76EJ+kjDEKgdsmEFcIAPsYUavCXp+6h2QzKWVVprByL0EzxfJJmMGTx1Iq3NfEwbx9MvNtmoZ0g/QzFSfJWdQBimHkVCTQypaOl4oUIzqz9/OaQlwBmbUUsJR0IW22Gy4vL4XdbDR+HKUrREv86vaSTPLdUhJS0sGoU/bJFIukUZauk2hkv0dN12XqEtI3sAm1B28pci/ScV7u2o2wXsF+DCjeNN0u3+yzg2t/7Q/3oPw6Tz8f9jnZhA/jcrqJag9Cy2tFhmUq1Nw8u8N7pkJQFQmbEg+riRD02Xz7UCw64I1q3wUvn5kLFiESajlEwZgzInunpBiuUtgb/gqXUhf5O7kX1royXuwhTojxYNjMlI+Ve/cLji+8uxutRCR/n28V0LNUA2Lw5OBR0UPw5Diiw4gNIzYM1KFjmXbUcYcaNpiwpTGK2hkMmaRluicDWkmgYAtCKlW2csOYFoAi7J8LKGpk85MBVuFSoh89fUlGIZELi0LZTMQxBNnUbV0ze2/OnfqYZ4vXXF5dMI5bjDGMKTGqivnJEfcfNpwdG3Ie8GPFMIxgDMZphj6xmFVcXD7n4uoVTotz96I9ol+NXF88Z9wNNG3DctFSNxWqrXDF0Xi8fkWI8OEHT7h3u+Wtb/zrxOqMUQ9YW+GVoh83XKzPeXn5CTGBMS1WV9Szu6jFA1I1J6qMjp5QWqP8mBm3gV23JfqKpq2p6prK3EHZCq1PMO4YvTjGVS1qmuyZoudYjjfy8wM4PmGr+6om04Qri7faLxnlJdMCkMuiV94xQw6ZHAPEAe07TBxQMeKTmCAaI0EpzhG1xjnDvuMJMNN2Udyej3ziOAROoseQGIxlW1cMSkHwqHEg5sDoLCPSYaBCR/JXxPGSnAdQG2zW6BFSahjGQA4DaCnqmHpOQpPGHYYkj9uGWrXkrOk2I6+vXjG4zNHylNE7QjxF67ex9gQ01LO3qZoHwh4xCusHyAMhXZH9NZYjIm9j9bvEdJuc3X5rNEXXf6rE/rJHVkUnjkOyd2A4lcX05v2/sRDK078YgHOTufSLT0o2dZhaIm8srlM1mKI7m0Jp409wQ1NTzu0QRKiyacLBBOpm4jo9Z/9ypQqInos+9xVX188Yxh5nLBrFdtOx9hmVW0J3znJ+JK7iaL78/lf4w9//Q/7pP/3HnF+fo4wqAYoummu+aILviCFKK5m1RaMzg857/fnDuUacq7h//z7vvvM+p2dnzNqGtnXUlRVX8lJRNCbz5S+/y8OHD/noo49QqsJ70XSPUaRllJp0/mSDs84S+r60zrGv0NpSmZ4A8ym5tNYwa2fMF0ccL5d842tf4w//8A+5e/c2/9N/59/lP//P/8/88Z/+OS9ePGccB778ta8xm8/IT2G326CUtGdutysqK8zzg1SDgHlaa1arFfP5jFevXnF8fMxs3nLijm+AXp8ZPjfA8b1D+ecAEn8To/fzxunfxKz7ecebwN2bTNjPgiSHx2QOxs/7cv8Sx8QgAFU6CW4yKopMSlnDc2HATyGStrIPa20ETNSWdj6nblqcURyfLDlaLkW70IkW4fVVx7NPn9LUNffu3mE5n9O2DXXdsDw6opk1IoGAmEOFmCR5yXKuRotJ2MRgmca9957tdsv19TXb7RZATKGULq2SsuukrCaiwxvpw01W5BvA6zTZp8ARmf9G6c/emv3xMwyKOCWNwrgQM7SAs+JdoFSmqVuqqha5taoCAz4MAuwUg1frGuaLI06Oj2jaurSj6vLnTWPRm4Dw31QE+tyxqgCEpDBdq1K1Ya+9eAN4nlqcTTB479lse66u11RWc3Q04+TkFvfuPmC327LbbCQGyDfXXzmstft1Y7vdiBa8rogxcufOHepKAI14o7A5GeLl7NA6l2LJDZ3FQqqYvtg01hVm/x1QJfKYrtv+sYlpJMwr54RRHFMuMnIRYY7oG/tKSbT1lBzlktdpUipMsZgYx4lVJS361jlc1RBDJCaPFKXcjXP/5Y6j5Rl92NAPOy4uXtOvFU0zByBljXMN3/oX3+Fr7x3x6tM/4ehsTlNrfu3Xfp1sXmON5f/yf/8/0dbHMm71QMoeV9fEDC8+XaNUzYMHpyjrwUpysR0uWK9GiDUxWU6OGpHqWl8CAesCwUesdvhB5kj0mraek7Pjk49fc+ukYXc1MG+PuFqtqOeWj3664vgkcHb7FB8CJ2eOhw+WYK85Wx6jdxtu31ngx4GL15e8eDzixwvmzX1und4hqytyuKa2jtksYqprLjY/IaXIrDphuTjCx55uN9Bazfe/85SrVwObVeLbf/1tvvrltzia3+bi9Zbbd+/x/PkPeRw2nF88549+7/fxKdD7nqqaCWOyGAFqa4g64/EkLYNKlWSaKd7VhowkdSglib6i7MNSJJwkymIBhG4aCk+/e9Mc91AcmuZXTonkI9qIvq4vzLsUb7SgcwCIpdvowACPsbDciwG1MnLO4hGg9x19E2N7+nytFXrKvUtxKedYAD8hQ0zFOSksHdrJlVLofLOj5NB9JEUHRY5iDKbLnFdaofSkp5r25qQ3CwtQ2vHLPfBeCvfOSczs/Uhd13um+cRAn+6HLtN/HMd9sdhaxf0Hd7lerZnNF3vN8KmbatKCF2Drl5daHJNHWY3TFT5ldsNIXQtbuKpbjHPEFJlVLdqZIkso10s0eDN+HPFZ5F60k3savSeX75qzrHtKSQFzHEYMhtOz0zIupGtyGEeR9ohR3iPKnhyixKxZBXwMhBxw2pb9bWTTrckhs1BzBgJYRe1asoJNt8MPg2iNY7AYkhZtd1c5rNZEPzLETtr4SywYbgBxlPb7EBLj6KXTvcwFWc9tuS8CwA5DxySrZ4wmKsW2SJGoGAgpQE5C4ksyVqwx+5RHwV6yzI8eZyNt20j8YkrBNyaZI2m6xsUw3laIn5HFNLXEXQpsLVKVxEzygRAiNklh31hbYjKEcQ60TQsohq6nUoa6bmialqoeWW1XDLud+CeNHqOtAKIAIRKMmJKSE1YZnDEoramMIWeZDypLzh1CFLkYVcAqEyEFMWpNgcbNGPuB68trfCfFGTKkcWRifUY/EvxIMAJqzppW4qWUBQgbPX3xeRLULRHySC5z0BhDHzz90KOVpWlayMVgOMYSsWWEmS1rZo5B1mStixlnGStB5IdcVZM0eKUIKqIqTSAK+B6TkCJ10eRBY2zF8axFGctmuyMEj+EgGPnLHkZJQT7mhNIUGcCp4FhkXfSbaxgIrKKrQxeSVlJsCkmRSjE2ZckdpyPn0tVagOGcS9c/WT5byWCUgkYu8ahG3+BqCD1F8ihtwCIdAKlo0iuliApSnBjMb3qQwUHSdfIHmObqdrUmZ/HJ6bpeZE+9l7FrD4An0zzMsocYraUrQEuO2rY1u+2Wxbzl+OhIOr+tLZ1IgXEMRTP+YAb9+vVr+l66K9FT8axIQWYhA5vKypzRBh9SKfwKQDzpd5N1UaqApmlJqScb8dLz0e/XHVUkhg6hXwGacyYHuXcHNrsq3VIHua5DZ8EhLpCLog6djsXofCrGpL2h6QH8VkmJDPF+GB+qQjEUrEZPnZZvauvLmCpnr9Xh56SmNKoA6LmYdmZEinbai8vMnQp5hVqrSoeQdFSoco0Fb85p8rKLoiSRy+smYoKa5kPpQk6IfFvBJFMWk9jJkPuQA2q0hhAmLKjcky+Qd39h5M2PAz7K4pRThBhQwaP9gPEjzndU45Y2DbjksalHxx0mjegUsMnTOo3TiqwT0Uil2KrJhEaXySvpuzAaSkviBGYhbf4qFqdkpfZEjZyLGLw5tPK3dU1VV8SUy6KdoLQIeSOGjCobrM5Uc8fROzO+fu8tdtsVu77jYnfNeb/hCkN12nJynJjNBrq+o4sbdN2g04yZXpDSQEiZ8/NXfPDBD/n6e/dp6gpnG8IQ2K07uk1P1/VE33GytFgSQ6lm9VqA/thdsz5/Sb17xjsLza4yXIea6wi7ENjGAW0dKnqaSmHqE1JzG2bHKFeTgkfFSIjQDwk/GnycoY2jHyNJeZROOHuMtsdofQSmKq1ipd3ixjw4HDfhiM8+zv53Ew4x1cTk5wMTcKpSgSptrgW4zQqr5bk+eZFvyZGoHbFpZVF3CessGEsqmu1SmJSEXee8/wzjPfPumgfpmltsqJSizwuu0gk7pQnjFvwVo450HKFiwxAzyu+w4zU+rog5QdYoLDoHNBlV12RrCWMv7Ekc2VaiyW8qgrKonGi1jFljK2x9yphgPTaEkLlan9MYS9O+R7M4wbo5mQptG3R2OGfQ8RJ4DlTEVBFyg9NzlK7QOaJCDwkcAZ2Qa/erHCYL0zAf2NxyAzUqyUKmZOcqY6REkyoXSZyfDSZuPnaTFfQG4KMnSZd8GDH7sVc6WVKUvylaVxTzJgSkCNEXsKK0He/hs+k8ADTiIi7PmLTcJiDxZtuwVMszWU0RTYntchKdyDgiGu6yiIcI1xcrwrijsTXBw5PHz3j65CmfPPqIFy+est5spAXPyPvvdh11VdEXwKqqKnRtREfQSxuk0obKOnGMd7Uw3pBkwXvP8+fPOTm5xf0HD3DW7ivrRisxslVSaLx1dsY777zDy5cv2W7X+6q8BBYK7wU4m5halEBJGNAlyHGWWd2QUqLb7fYA1pTIxxLMbrdb/tE/+kc8efyEP/zDP+Sb3/x1/tP/9D/l0eOndMPIvbsP6L2n62tmsxnWOo6Pj1mvr6iqClIsK01iLK3RlavIwDDuuLq+4PX5a95999291MLh3t2sHB/GmASE5sZ4+PzA9xeB458Pvr7xjDdAzj1g95nnTQDetKbmAl6q8u9DCybTLPiVYLaJ8S7jOu33WGstVa5QOuKKSa0x4uFhkkWbohmYMxR9/gQY69AmYTTMZkvuP3iLWdsQwogfeq6C5/rqmqfra5If2R0t8X7k6HjJw7fe4pSErWtyVuLvEWMxlZJ5a5TCGWFMTsWaiTl4fn7Oq1ev2E1jcM9At0x9p0rQKjTCSj8UEcS09aZxpDAfxSthusbS9wbiyfH5oPRn5QKUEh3Hk5MTjNX4u7dQ5fOUkgC5djVtO0NpjQ+ek7MTxLzNitZ3Ke5pU9M0dZl/ab9WvVnkexM8/+y/bx6fWyhSBTOH0k1WWCj7APugaSzXRNo+tXUo5dntOp77UYJTrZi1Nbdu32e76+g3W7bXl/ua581On+l7xBhZrVYMY4Di19JUwqq0xorBrjHUriqJjhGTOp2KDNbBDPfz53aRArvBSp8e34PsB3xd/rYGW3SH8ZGUA86ITAClPVX+Z7B2KjREQhpFFisnei9suWGc9kVEStAZ6naOtY7tpgOtcVVNVbWH4vAvefzRv/p3+G//ySdcXuxoZpZbJ0c8ffaStm3Z9Vt0tvze7/093rn3TT59ckk/7vjTP/k2n77+Caq6Zr7Q2Gogm9fMjh3NssHYlsUZvHq94vLC8/Y7Z9gSZHQDrLY9QxjQSrPddITRcfetOzx9dEmKmph3uEpDNNy5c0IKmV//ta/z9sN3mM81s2bGow8+4sMf/YToA1eXW2IOnF/1DENku9sSwisq03D/wYz33j9iDFs++dGIa3d88vgxv/Hrf4t7t98mhsT/8E+/y4+//wGLE8Vv/fYxdx8Ybt+puX/3AcnsGPwaYyzbYct8ecJmG4je8md//m1+/N1IP9T83T94n2EXePFiy3gUuF694k/+7AcsjmeQA8Nux5/+6R9DUJzevkUeM9veS+HHj8Q8YNqKbA07vyFuIreOzvDRYZCkLoQRYytQCu891pi9Nuu0p07jVQwzTWFbyTy8CQ5b5/byEdMxjmOZDwrjBDz0KZGM4uL8nJPlEa2e/I4OXU6qrFMxRrbbLfP54rCvTmz3PVB+AEVu7pk5Z9FELuZ7cr4Hjd2chWxlndsz2SX+OIDw5EOxc/qeh/VryhXSHkyd4jKJzcIe1J+uk3zGYe0Mwe/BmxA8m82G5XIpYHA4yNZ0XVfMtCvOz19RNyJZNiXYMSWur6+Zzef74uZ0vKkHb79QIv7zjqSLVAka7yM+JwwZVMJq6UQNwaMte/yjchbnRMIkx0TyAT+O5BywqkYrg08iVxC8J3gBJKypiKMnhshqfY11mpACxinQoo09DD1gMdqhjACNgx9l7w89ylhqLZILwsgOKJ0xtSbqQFeYxEZrhn4g5UDj5BqZlDHZgDHEBAqNNlKYMK6mVbboRme6oUfIM2VsFDk0YxxaOVIOpfMnC9EqidHr9NR9ASgGkjZk5zDWEBWMY4dTokkdw4CyVrTcQ2QcBpQxuKqmbmuRb0qh5KCF1FfokwJCTbJspdifKSC6IymoKoetG8YQqKqGWju6zZYUpCifg5gCGyNsY5SmbhsRH/OROHpwhWyoMtaJRIEAhVrOJ0lntpH0jZCFwZy8h7J1Jh+E1ECm914IjFkT/FhwFQGydAVxHOi9yNcEr9l1I9vVFpVE41whIK1I+Rj6IAQtqzSzeYvRei9vWWlLpUXSJZNIE7uULD43lQMtuE9ICaeLd4+yWDRjyvgwSlcFUqjVgCtFf3Im5ETXDRiVscrSuoZFsyBFGI1m1JE+9JBEUzn7SNIeP3hSUGhdUTWNjI+MjPcYsMbizK+miS6hy7Q+Trrik+zFm3+bG7T3m7SBjHTvoxJGZ9Sku52LhMn0RH0o2moFWSmS0kI6VId3o4CzEwtdtiS1z/tVmjKBhMqxFGTF66eIEspvk2i3p5xLPpr3Rpum6NhPOYfRyDUtevveF4NzpaDItlCKvt6PpCBFlymH18bSNLUUs1JkGEZ5XozM2xlGicwiMTGrG0D8FKrlkvPXF6yvrtltt3gvZF5rjHSFIHI5WoksW4qSwzsrcnwCpk/3SLpeiFIwm7WSL4+jyOuIofaUj0hxZ2KViwBDJEdFVJpIwT3zxOifGNolB7gxht4kaN048rQsTWDzQaYQciF8TFK205vmN98AuZnZCAYk+vg3sZVy/jm/cU5pUmzJEVlkEkqVNc0ciCslAylguzqQoco5TWTCn0lfimcCpSAhnSkSqqOMdNMoMRcWWFD+u5mP5azLNUplXE5d0ggJgTe/0887vjCIrl4/ofI9degwMaDjSOV3zFJHlT06DCjfYYgC4hhVwG+IOpGNwui8Z7RJIhqLXmjeJ4pa2/2XEM1qaa+YDDmAovs06biaPeCXMqRSPTEWjJbEK6okybSaKjKJ3o+kbiCOI1opWuuYuRq1AI6PGP3IEEc2wXMdIr5KUG3YdE+kyps7rHUs6xnjrqLuE2Pfswuen378EV995w5VNWexzBwtN7yuLtjEkTpZhuuRnY+cHBsWtWK7vmbXjxzNa9576w4zBeHVTzlmzVfat/lp5+hosM7Szg1QQ+zQ+hhz9Gswvw1uVhLKjDWS7I6q2rcRkxQ5B/pujbEBXVWkGIAR0iCVVV2hrUCZEz+krJ83AsEy6Jm6AgT8nIhnh9ahAyA/vUZNEyRJ67POuYCK5f1z2rfXqAJIRW3wGIzKDBlsNgSE9YMW8D1EyD7g4oAjQhiZjRtO0nPa+IghvsDnJa59n9uVZ0yazXDJ2r9k1IqoHpLVgjx4VJKOCgGNK5Jy9KpF2QrqEzHVSQm/vRa94qolZgVuTtSZPkeiH0WD2Z3ByQl1K5r0ikzqe+rkqDilmi/QzRxTHxEGj3MV1sywqUKPBkMU1qhxOK1oVA+qIfuE0gMVyBwJFTo64MEXnc4/c8iiIgvl1EYzgdoyz0qylXNp/z/szp8HLv4McPg5gOL+32r/vwMIliHnQMqh6JF7svLFWdmKIWhKxDjiwyi6aSkA5vBWBY0/VDpvnuPhnG4mhHtAZo+5qH23hATDmqZuGcZRzEFHT2UUISgWixNu3TpjvdrgYsI1DVXbcnb7Du2i5fXFS7bdWkwIjSlr2I22Y+MELPYjMeXCAihGnWX6aS3M74kF9PzZc9pmxqsXr+nff4/lvJKbqWVOohXL5ZyHDx/y5MkTfvjDc7quE21sA5DxfrzBhJP2yikxlCKBtDqHoov9pv6wXMcwejartYD+JvCnf/qn3L17l/v3H7DZ9ty9e7eYAhqaVAurZgiMRbOx77YE3zP2HWMS85hxHGmahpim1utcnNNzCZxutNLdAIoPVek3x6CMg58Fym8+928C0f8mkP3A3J3WuoOG3t/8noehJjiBeGcodSgmpOl3v+QxdW2EKK2SOQQx79KaqhIQ3RZjW+cqnK1IMWNMKJ87gQLsAZ9x8OQcaNs1pwVwkCREmMpt27DbrAjBs16vePHiU+rG0fVbzs7OcHUL2jCGRIiZkEpCrJSwmbyHPLEiY9HF73j9+jUvX75kGIY9yGGqCuMqnHWSRJsCsmoLhbHBxOzJh2KZUsJ52K85E7iaRc5F55/PRP9sgXBitdR1TcpzYrSylyG/SzEVY2Nd2H2KqnZybZUUsSYwKJbr6FxFzqa0jn4+K+Kz5/EvNS7IIj2RBJQp0WZp0X3zmTIK5Lu4qqLvIrtdz+XlirZpqOwRlZtxfHybk7NLtutrAT+szNE9SylLvOBDZNf1JDRVbVHqwAIV8FDiwSnJmrpRsspYO3VGUJ53k81yAMnFOPXQlXRzGv7MPqQAnRhDQqnC2NEWhXRMSDNT0d0hgYmIF0cx306evhtYrdf03QhZ41xN29QYF5nPWlo0KhtiNvTDyOBHqlEdDMR+yeP/+v/4zxnHyNe+9j7/33/y/+LunQfcvXfMxcU5L16ucHrgzvFLxu6v+PijJ/z7/8F/wO/8wXv87/4P/1tMozE2szzWnJ3NmM0dVW3QBnKVObu/xO/OaZuKnDLddmTdjWTmLJpbbHdb/O6ak2PN1eU1zlm2Q89soVGp4ej+LbrtwOp6w+bqip/+6BOiihydOBaVpp5rmqrm9N6MJ68vqYwqa0PHvNKcLm5xfDxj26+5fL3m6dNAH7a8+6Wa733vx1TqGOc0f/Sv/G1+97eWfPDoB/TdOd//9jntLPHWu0uSScxP5yjXEcYVxyfv8Ojj1/z4By/56vtv8R/9x/8jmuoMZXaY3JB7xQc/+Cua9pSu2/Dk6Use3nuXHDzRRxLwp//iL7h1ept333ufzTiKeahV7MKKx08fk2JiWHX85td/g7uLh6hUw8Sg0om6bvj+d7/Lb/3mb0qhpazFN1nQss4OzGbz/d4z7YMxRnwXSOpNWaeJhS3rCGhnCXHk6YvnXF1eslguxdTQezHTtGYP3MQosVZdV2hr2e062rZFgZiJTnFKTNSuBsSsGtgz+sTDIRCj3xcrJ6BepEMMIXqJe5Ck2tgDeB7HUIDGN4tuk4atUmCt3ncdpjSx1Uv8CvtreJNpqE3m+vqKxWJBHIf99drurqlqu2cTTuuBKXtIJnN26wwK+31iRuYMpgBoE2D/WTPVyXTwVwHRfQ4426CVo7YNMQnT2489yhlUFnZlUgn6xHp1TcqJummYLY5o2zlpDOzyljF4whjQzhWgkv01BsghUhuHM2Lgt91tMM6I9032+DAy+lEkQYzGVuLpM0YvxenK0DQN2tbsdlu8HxnDiHISyQypx9IgLOLEOHRyXQ0CGpcODY1IGoQgYoFjAXWnbuZQclqFkOmqLAUDpSxtOyvAVY/eF6JENkiKIKC130v55CzdoK6uaJzDjx1xzHu/IucM2TjJR2MkeE8cPZV1zJYzYhAvlJykM0nljM6KHCls+UAM+QZ4LwZ3xhhyjCyWS2bzhcg3ZoVTBu1qTGMZQiKHTG0r2qohxEC7rKmdg5DodzuKTzUhBckbELBOSBFi+NptOpEQDZFWQXSKMAaI0r1tcYQYGHNmiLKmkBV9N+5N/7Q2xJTxKRbQNWKdZvQ7+l0v31spLMKLcs5S1RVkRfIi5TNfztBKlYJO2XKLf45zlpwjSlsigb7I3CkM3oukTGUtJovPgwEaW5HHwLjtME7mawoSnzZVjdOGvhtlD9cKXQkhx2pLUwmhLapI4wxjEP89kWuOoAIqloIDIiPajyK5OQwDMUVMAYV/lUM0yCcgscRhSYEq/kCqkIMoQOokq0EJRdSUcSSiyqgC/hX4DH0TRFeUuOkmWzmTinb5BLCqYsgoXX+TZEsuZ5ApEDl7Rnqe5Duk0yklSBrxAAx6H0+RErnICLVtK1KPjQDau+1ALH4Vky76QZpE2POS90oO40yF0aasAYaj5QlHR0uq2mGt4fh4zmzeMAw9Wgs5o2kahmEohUBZn6+vr+UblQ4KIefpveFqSgmLQRvpkM2Tp4A2RCMSIKREniSnp4JXKLJ+xhK1l0IiQClOTEUT6aI8sLWjllg/qkwIEw5S7kma8m61v/F7supn8tepE+FmB4B85n7k7B+fCDM/F0Qvn5kNGCZCz414u5zLHiGc9jsla+FEfJT7iMxhfRM3mrrX8h5zmOZE2pt8Thr002dnjJqwZJE804WEi9L7jr8J58nkghlNeXbBuvZfvMhJaVvG38Hv5RcdXxhEf/fFX2JCR00Q1mvOkIKA5RRNyEq00KU9o5irZNApFjmWwhHVSpgZSjRJQ9HKMcoUAwxDVrpUk8XV26gD4yiHtG/PsqXVbAo6Y2Gc6wxoc3BlLxolU1uyduI2PvoK78PeNE8BIYmGpA4WaxNHWoPNJD2nU3Ou7F02+YJgR1kotCc4yPmIdnbKk8s16yHSVnPaWebk1h2+/DXFon7MRz/+gAd3TqmV4/lHr2jrkfv3W2ZLi1GBu7cVR7WichbVr7mnnnPl3uPxeALZUduAXQSq6gjl3mOcvQ/NLZJ2xLGDYSv64XmqgDsUHUN/gVY76jrhlEWrWgZs9FjjCUp2tT3DmEPN61D/KpOOfcZ6UDC6CaDfWOhRn0lYtdq338U4gSRBWkZIWK0QPaiATYEYMzaJQZ2PiZ2xgMHp4tadMtkn6NbM0jVN3pB9x9Jf4vInbPMzYl5hzXvUoaPyl+QE0a/owpqdsQxJjFo0AR2n69aSlCFpS7YtullCfURT16QQRD4lBzAOmyEohUojvuuIRqGrUwk8KkOzFHZAjgE39rTHx7iwocKjjSNng65ajAZrwOiAVscYBrKHqn3IcnGHhatR/lraEUPG6Ai5IscTjFt+0an8uUcqFc9UTA8K75wJEJT/BJwl3tyZp2rgYfH+vOMmy2r6+bPHVCE9sI8C5FG6POKA1a6w0UMB9PIeGBTzkSQb3A1QdT+G92D6oRo7bdjT2vGGSdY06pU6GOUq0NphjLAplEvUZkG/kyBiGDwxwzvvvw8JHrz1Fl/+2lf49rf/gp/8+AfYuuLIHQlLp+/w44g1mvl8jrWWYUx0263oB1Y12lgo0gIkLxV8cwCL67pm9CNXl9e8evGa81eXHC1ntK05MGizMM6Pj4+x1tJ1HSEE1us1qERVSedDznHfMjYxdlxlqIIjRtGH6/KkWX24jxNwaIwkojFGbt26w4MHD/jH//if8OrVOf/wH/5DUo5cvL6kaVvmyyXzWcvR8TF93zEMHeevXzKbzVA50/cd3o/FRDUQioa3cxbvB1bX15K8ZwHaq6qYzZa7LRX2zwPFP6+i/SbQ/TcB3p99zc/73f45eioM60PQ+zccwqgqgfSNtTamSU/vlz9Ee7AUI+KheC3nd4iybwZdh/OSOSUSRNI2v1qt2GxXdN2WmCJHizmhaK122zW7zZaj5UKK3wqapma7W/Hkycc8fvyIbvQSPGpDyiIrkpgCIoUrhdlpXk5atH3R3pxACjUBrdZIS3gBXylrSMkHp2VKgiM1tUbmvQ4h0/OQ4CqVoE59zl2bgKybP6c90N+z3qwJYcAaRSpGWDFEJB9VxOhFUi6nEuNEYZRKFzjW1dy6dY+6rqiyQ+epoHX4zM+y4Kff/zzpgJ8ZrwpQRhootSRG0/VSkxv3jc+U8E3Mluuyvg47Rbfz9F1AnVja5oj5bGAxP2E2X3B1dcnU/JNzxlknY0mL/M7gPcp4QuyoXAIlTNopMJ5aZKefJQtkbyA4fa+9TiY356UiI3HkzSSDN+Y6+0Kr0pqkRbs+R0CLxn3OuiRV4gkgZsWBzABKmFjkhM6ZsOvZ9p71Zkv0mapuOQZqFMYnVpuOnAeG3jMWsFCpcCNZ/OWOuw8Nuw14v2V55Li4/hjsnMXSYqtTtpueb/31f08aFTka/vf/2WOWJw3adojsikHbSDWLNAuNrRJV7UTPc4Cv/Pop2TvOz1d8/Mkl2WYUFafzyG7Y8ODBMcujin4YaBqoagvWc+f2KS+fDjz68IqTU+mQao4M9TIxXw48uHOX1rRsN1tmx5qHixOiUawuBjEDVJqLV2uePjnnm791zPPnHduNwraa07MTdF7Qb6HvX/C9H77gZPE1vvSVh8xn7/D0ySeQLnn2+JrX12vsTHPvXcvJWebiqmN1rVkuj/nKl79J3ydePntJZY/45JPvYlTk0Yc/IvaR3/7bX+VL78xIQ8/bD95iNltibI12NRfX1/zw0Qec3T4l7NYMlz2pyagqk3vP0ckCV1tGP5BipqorXp+/4Nade/Srnq4f8DFilXTOfvzJR9y7d58MfPrkQ1zlOD26xxhEklLikim5E6DPFwNQkUIQdrV05kjHYlKZx58+IejEpl+x7la0R7epqxkxB3wQ3eGqdqgMXT/S1A2KyJOnj3j/y18Rqbko3UpNXaO0lRjFVSXpDFxcnHNyeipSICX3m6RFZFsTALzbbqjqBm0NIQbW6xVnZ7eIXjTX+xBuzHcB6Q8GwuJvpZUUrvYs7xvx3MT2VQVdTAS0VZyfv6SuHCDGoDFJUVbpTM4epdzerHSSGpB1Kwq7LR+67RRT93ThtxUD9Uk2IE0BZka8JH4FOZcYRlKIIt2ia3wYxYh7HBitQdtKwIFRPLe6fkfwnnEY8CFT33KiB+1Eh3ooHh2ZAlogmtpNVaOzIgUxOk7AbrfDVgbrNNknfBwkPiiAVkZi08o5jHOgFCFBrQWw6AeJcY1WaKtBR7IKhJQZ+wFywkfR7a6reu+LEvAkBSl48D0pBpSrMM4QYqTre4xz1FPxKIumLcayXCzR2rBaCQBeVUJWaVvpfBTwVjw4lGYvVRFzpLYVOmlUXbGYtSgSrhJJK6UEIB+6Hh880QdSCEWAQLwEtHEYY2maBmtlXvpi2umc/E5kIYWkZa3FZAGyG+vYrLeo5HFoZpXM3yEKQ53FEZvtBmsN1hiGbiSNCaskXnHO4WOkH3pc46hb6UwVM/nIbtOhVaAGfD8w+AGjLFQZN6uo25ZxGISZrzJ+DITRo1C4uhJmZ9YMYyJhsHVNZTW7sSenhDNOSHlaTFVtpbFOPt9VFm0sxkn+0XU7xuglj3YOgsdK0yDLdo6dKdYbw/VuwxA9YxSZoMY11M5SoVGhEO6GwLgdUM4wm83oNj2kxOLWApMMw3YghB7nDOZojqtrAtDFQPCJIYcCyGpM0mQiThtqbXGVw2uRoAoJ1lvx4RnGUQgFSksn3a9waH0zxkqoJFrfIB5C03ojQUo8xKNqkqQr8KQBlbMA6fJue5u0qbN8Ivagbhg4q5JjEEucraYXlY8R4HN6UIhZCb03h5Q/GWT90AVIn+ZM0e2e8oqU0g22uccYid+7vmO92TAWyaiUEnVl9wXSKafXWlM5x6xuqVzDbDZjuTzizp17nBwf07Q1xgrDWtR7AsZp6uYtLs7P+eTxY+bzOYvFEmNsySdGrq+vqZoGNQaGwcuapXVZI4RJbazkybooIBitUSoWzKFcrww5ZdarFScni1K4KLKDhYhoSwfPHkTPAlInI51bIUIIVsZGkWBRKpFTIaQUb52cpHiyR+pu5J43O+xvkr/k76mwnfev3W+iHB46pBsipXIIU1XBQm50zvEm9pMLWSYhOe0ejxEEm5QO+7WAxNM53+w8Yy/fIpJMZWwXNr9GYvRJ/kUo86nsv4X4l6QbLOW4l/zNQq1n0qWX73MA9uWxw3X9RccXBtFvpXMZEHs9SiDb4mRa4LYsLQgTnKFRZKWKHg1lQEmwobTIRKicDknOVIlAtMXAEoMnxYndPgVnBlXclJOK5Kz3hjGQCFPAYwzWCPgygQdS4ZIbb7Smrhy1c/vFKqaE8kEctrWmlpMTnbNsmFUzFu6E6N4hGzFfW9sdL8OKrm4w9ZyqXfLxsxd8/e1W2pSbhradU9UVD+/fZthteXV5je8857sNhp7TY8vJ0Zw8RFLaMHYec/SAyipMdthkWOoaqzLJzaC5R569ja1vEV3LMPT03Ypw/RIV1uRscLrCqIBLG1q7ZV4lmrnFzuYke0RSS4I7ItsZKGmp2a/XMhMOA+BG0abUbd6ocDI9f3rtfpGXJ+lS+FEpow34LPp1Jvao5FHGoKqarKRVSRsw3mOCVKetcqA1IxqjLDZP+lCKHEaacc0ivqLlNTmuqMOKlK4YCWTVUJmaHCK7KHrQK7/hOu7w1Vyq1Ap0EvBeGVf04msBY1yFshXKKIzVWCOVzRx80U+NuJwgCSsn24boWnTdYmLRtEMRxx5bmBdONSjfk6MElUo7Yg6iyRcztTsmZw3ujKq9T92e4CzUw3Pi8IJ+vMBoUPaUatGS0/yLTuXPPaZ5nG9U5w4MvgOQvtdPUtO9nQDnXwxATov7G4v9zz6LqXVJquOemAdi7MU4KBliSbBjghBGxjCQiraj+kyVdALOb37+4XzeZCxPFfDJTO7mbmK02St2ERXZQ7/zbMc1q6uRbqs5PrZsu55liKQQubi85PL6Elc77t67Sz9s2Kw7KSCFWAC9vK+Aah0AhTZWEpnRo9RA3chjoi4km4W1rjDIHderK37w/R/x4ME93np4l7YRw8AyiUkpsN1u6XY70ay+vqLvO+rakZMmxam1S1gEcWpVtI6mbsgx0sVOGEEpkQuYevO61ZWlmtXEkFivr7l164w/+Ht/D2st/90/+af83T/4fZ6/eMn9t94q7bDFf6Gw8Y+Oj3n27And0DP6YoKVYmFzTZ8j32W9XktrnlLUdV10S3UpMBwWsMO9lcLeXuvs5hgo42Jas37ROP68jorD727EITeep6YBuf/cvP8zvU1mCkYL4y6JqWcmEPObS/G/7KH1DT+B/beW8a+NFtm0ElR770EZYpD2TErVPoaIslIUTzHT7TouLi65Xl+z3qxp6hpSZFZXLOZisrXedsznc05mS5qmIueG4AdWmy3rTQelSEnRr1PGlmDTUCmzLzxMTumjL7IZJY5QRe+X0s73prabtPyrIvitlCapjE4CxEgL7BS7KIRnPV2b0lb4c4Koqfg23eepnTzEwGp9zfPnn7LZXMseWQCWEGIh9Gti8qAjhTMkr1eqzD2YzRcs5wusUTS1IythWWlTmNDqhiRJGXgHCaAbEf2N43NZ7HvlI1HfVEWyawLrptdN3R0pKSIJa5zoP6ZMCkFMxYaR5fKMk9NbrNcXvHy5oHvxshi4CcmhinLt6sYWszeFMqJPijJSiEhiCiVfTJOVIU8950rJnqPN4eSVYq9VfiOxPESgb/pjyNMODJVpHkeCFJLLXVFZSxqrDGKUVtptc0ASKiXdQGEsJqsD2+0oEnoe0Z7OmpA0qYs4p3GVwWjLbD7n+LQppA2N1b8aE/2tt2/x9PElXTdCsrz11le5vLgmRA3OUy0yabslKYhj5unLxxwNivtvzamXDluVFuVFxswdCc35ZiRrTUodu7Sj2yguXgaWsyVvvTtD19LN+Pz5lk9fX3Pcn7BctJycHZPVNcF5ibleeH7rN77CW/dPcG3P2j8nqp5Zc8R+3mnPOu84utOiTOb0eMHV68z2asSqgdrAj79/xeVKJPTuLxwZi7InfP9HP+XuPTBEZkeK3ficy+ueptHcvf0+t++s+ejxh5has9uu2TjDx5tPWb92/M43f5tv/feP+N63/xkmHfFrX3qb4B5z6+s9X/v7nqqKmPBTqrxg3i/I3YrNrqeaLbkad/RVoLnbcM5T5o0Y3SeVGFc9bBR37r3L9nVDn7c8vOcY044h73jy8jHt/AwzW/L01QVv3z+h7xOPX39MbBPaVfzl43/Kl7/8FY6a21xcr1nM5lTO7uU016srlidL0QPIYnrf9RsWixmbXWIxv8XQr2RtrjSfXjxmCC94dS3srltHD7HOkHUiEdmNnRj85sCsNozDihDX+LzDLRyriw1xd8nD+/dompbNdcRVLRDQNjPGjsQS61pydiQGtO5lXctFKzaPvDp/wVtvv42rK773w+/xlfe/QgpR4vmQRGe5naOUFfkIbYh+JKdATgPWaTCi8fz69Wtu3b6DcRZDxvvArvPsuoHTsxmm8viw4/mLF1gszs6JyQDSdXd9dcnZ2W0mnHu721A5kXPSRpNVJGaP1pboI04ZWfN84OX5Oce3bmFVojYii5cRIKuypsjKJIKt+MvvfJdf/zu/3NxOwTN0HVY1YKzIs4RI8CNdr3C1SALkGMhhooEmkV8Na2pdcTRfMA7C/tfYAiYkYioyCVpjlCZH2TdzTEQiIXmytiQUIXlCFCkc0sDYI/IQKTNYS9POhIXpE8F7tpsNu24r+682THq3yUMeSlHVGPoszPrGNWSUdKXlRMxSlE7eY3KmsgqLxJ3jsMPFGt3UECNhGAkpYxvDbLZkljV+jNLhgAcSIY4MgzCKRafZonRCqYQjo8OICgYVAnH0eFcVoFGMM40xOGPptzsB0seRvnTgGaUJxZiurm3ZHxXZJpra7Ik6MebSVTdgtMYpRegHBgrhcAwEL4xWHyWmUTnhas3RbEn0Ir/jxxE/jKLWnUvnuNbM5wuyMSgHtmYvnWErh6kDxlYkBX4YGYce52pGFFuku9bnuC/s5yjs0YmsUFU1WSl6n1C6AHxFdcA5g6EiRoWEcxpTGWxtUFGBEem+zg/shl6UBaqKPkZ8Ll0FuUgPBc/R8QltVeGj59Xmmm4cICvmtqFWBpdFizwPARMSta4YE+SoyUFRuYbatiUCsATfEYOnqoRoEVQmdDtCEF+ESskcMkryLJulO0q7ik0IDMNIGkb6fsBHTyzGxlnlIvnxyx+m+GXItBWmsypazxKLTZIuIn0rKrEHYhdKJFMyksObwv5VpYvyEL8h4PmNnFdP8X/xFiz9NPsYM+1JJOUxlJyPlA5ROqNv5CZCwoOIQifQVmGL7vTBf0w6B2tn8F5ePA49l+eXbLfb/VyR3F06Taa4LqVEXdfMZi3zesbx0Ql37tzl6OiIyjXCNk6RfjswDJ2Y4EbP8niJP79gvljwn/wn/2vu33/In/7xn/DRxx9DhuvrlcytmHjy/BUvX5+L1v6NuFEwAI0yoq89AfoieVPKFmVNTdGw23bklJkv5yIRkwYmTXEhopUcnYl6Kt03KWV8yngfi1QUZd3IyJQ8GL+mojm/z18p8W2579ONmf49mZjnjGApTPdSlXs+3cj90JJ/FgA7x4LLFGma6VMn5Yibee3+2I+NSXZLYqR9/qKkWLQv6ua0B9LldQf50RuVHQ5dGuWz0yTYIu+TCrYQU2YYvMT1JR+c/AYmAH3CuA7EnAxTAesgQ/Bzjy8MordNLeZfQhja6+9OOnhiBKUwyhadn/Lhpd3FGL1vs0ZNGsQGPT1fiV5TSolIQmuLKRMwZGFh7qVdbAFVRi+fG8OenWSMbJIpBWKKqKhxVgvYWUygVGEQkLMYQ+wrcwqHaL46JwYy0znlEqgGFJqaJCUnFlZxskzcMh2bkKFpWR4tsbYGVVO5gHMN1jXcvnsP6weevDqn1oblcs6F7wkxMl8ssG6GcwtUc0z79X+T+fu/AeOOt7PjgWpJODb9+7y4fMkHF5Fzak50xjFwNV7hr59w/uIjrtdr7h6d8dX7b7NsFMxaVLxFVVmCyYzWMuoFO31CZ49I1RxV2t7ylIiXwTrNg88dSj+Ly9wA0fP+LRQFRAcK0ZmclLBEhhXKdySj0foU08zJtiZHz6hGqC1Ga9rKkbXBVy2YksAn0Dlhs2euE7qXdnByBeoEpSH4vuh2dUR9zUjgavSsw5Yhj1T2DpXRkhznnkgmKU3SDq1TMTqSYNqaGbbIApEdQwiyaKZMZS1Z1aCOMU2LqWeYypFDJmbEaTwE7Dhg0kjpfyRhyFb0fL3viD4CLVkZrJ6zqOF4fsTRbIHD06uKGDxj7KncO2BuY/WcmH81Y9GQvVQby02aKn2CwypRCVayceaUxGimmK7cBCM+C9Z8HuiYbyzub3QpIEaiKkvbe86RrEdS7ghpgyJBrkhZmA0hy3mPcSB4j7M1dpJq4Obn7beD8jkCvAhz0xQDzSzSMUljTIPWrrSgATmL4UsCm0HHSOw8/eXAbjuw2Sa6AVzj+fTZK/wQqZzmgx//gBfPnzL0Gy7PzzEklrM5Q9+TbSyF0yxa5EaAbG0cphimKNnt0DmJXwGSJMm9SZBHtpseRWS92/D48WMur77G8uh+0dqTQDjHTPIj69UVm9U1/W5NJhF8wOiIMS11SRrk+kDwI5qMoYCatmKInRhLFebyZHg23UtjNYvlkqZpuDh/yQ/+ouP05IxsDN/+6+/wu7//B7z1zvv0Y2bdXdLONe2iYv3hBm0dy5NTrtYropKW11gq9NIiLnr0OWd2XccwDEXahRussbIBTiCaRvQwswT8uQBzElyXsaFKSfULsMU+j7X+psxVKli5oM+qgHATi1UC4huBxgQUokqlPaLwoCDnMj6moOLnALpf5JgMVZWClCPOHmRwtLYYmzGmQmGoTIVKos1PlIKR1aUzK0uBa1KaTmTRLk6Jo+WStq5RIdNisc2CMUb6GElaM1/MybFns/HSgh6RNmhd5EwpWqFlH+9LgUQrKWyHKI7sSUmSpqyVth1t0DgMFQYn7IRJT3AqoKBEhgqKLmZJahWiHwkoZQ6JSd7XDkoAf/jFdLelMCHBrFaZrCGkkd2w4fziBavVpeg8Bi8jLkvboryljMeUD0A8SHAcU0alQFtpljPLclGxHTUoK9wIrQoDRpdvVhIU5PFctEH33UNTfCwDgalmoKfzSBJckg9spZvAd56EvQGlLMYk6QrUYHTNxcWG602kvdI0c0PdVtx9+222w8Cz8zUvnj8XeTmVCVmxmM9QupKVRRsyFbpqsc7JtVKaqMVkNaNJpY08Z2RttAacYQzC5J4M3ihFAMptPz09o60bLi4upP3XOUnQtGb0nqqqqZua2WyOMZrtriMDrhKjT61FGqOqRN7IWldaoQ3OCCPXOollUbL2SceJ3mu5a62JIYseqxVG6KTt6YpeK0WP8Vc5zm43dN2M81cDv/XN3+D3f+/fgGz51re+xf/nn/7XdH6NVjD6jNUWV2WyCsyPGppj2dP7MbNaB9abNbPZDO8hkHl9vqXb9WwvYdhlTk40y6MFtlW8uloTk2exqHn2yRUvcGyvA1/62hk5Jj56esU3v/5NvvMvvsvz5z/lG795j2jg5NZtrPX0444QBxZnBtcsBaTwnuQ982OH0Q0pZF6+6Dg+ntFWiqpRLBYwDJGPP32E956nT7acHVuePDnnnS/fxtma7dWO7/z1hzg748FbX2fdrRh3ic3Fmtxqlsdz/tt//Ge8ehyosuPunZarF89Y3tfcO73LEJ4SUmZzMfDiIvG33rqL05ZsNakKHN9asqwVqtF0uy1h5xnwdHFABbh3dp9Nv+aDn37AV778Fqcc8+plTzP7ChfXW149XxPzyJyO3bjEuhqvAi+vX3BydpvrbstmDOKTojq2YyLkmk8+/oj33nuXy+0lqjX0HrRuiSkyhA43a/jxow959x3H9upTtFWc3b3F9z55Bbrn+fljmvoO1bjFBE3fbzi7dcTVxWuOjo44X63xGeoqcNWtuNxccfvOffpBdKKNUVxcv6Kd3QKX+fTpp9y+c4SdO15eveb22VuiuxwDKsi81Mqw3l7hak+Xrug55oOffsh1v2UbPWEIED277YbOR17+5IJ7d+7z4P5D/DhQVYYnnzzi3Xfe4fXFJScnJ+QMm77jtjP89NFHzBctiZEf/fARX37/a3z49ClVNXJ59QqSpm3mKAub6y337z1kGAK7ceRe20ihLAijOqZIIGKcwueRlD14JVIqIWGsJY2RMUR+9MGPefjwbRpXc+vWbYIfGbxHl67m1XbD+e4x2mx+6bmtUyT0HQMV0cieEbN0D/i+w8dE7Spy8Q0C6c5WWuNTYr1ZoXIihkBVVRL3RS/mqjlCzmJ0HAK+9yQfUUWKw2JISNwViyTPXq7HH4y0+0GKjE0zw1pF8J7V6pp+6JjPxXAzeY9SmZgjKWYWi0XZ7x2qEi3zrDTaKUyKRJ8KgBdL7TSSomevfZ9l785hYNhu8CHRJoO1QuSwzoEKjMXgexgHBhtL54QAKQJFRJwRLMMPHX0/0u12xJhoZzPBOYpfiTFC1FMlHu2GgYSiqWtEFiZgTcUkh6GUwmgLWgrsVkvXPVn0wXWKjLuBPCQq51CjSIgkEttxg4+QnSaphGsrdIax74UMGKLow6tECJqUI8oaqsqiLCidGIJnDB5ljZjBu0oA+DGQQwYjjPO+X7ExO1LOxDDirBKpDC2YjdJqD9QGn6lqTV21kAeU9mgt3Vj1rEY7iTuqpsY6h4oZZRP9MNDtekY/slgsSDYSUiErKtBG9sR+6FlvNswXc46PjlgNHTFndNLCRLcVYfR4n6BIvJwcn7ANHpKYUs7bVq5NjNI5Y3WR+0yELHrcMXsxbVWKWKTimrrB9yM+iC+YqRwpBVbrDTGK50mIUQorVSX+fPZXy7sn6TytENwhSZ639xlT7GWRRepW/n0g/oDOhpQThkLECJGsBOwUsFsVxm6Rz5hAw/Jn0jcXX7JJPjKjUynOMckwT+SKtM9jcgHcJ8BWCpL5IAdbwNqsEtmInKKxIp20nC2YzY9YbXaM4zM2q07INErOVVjfFa4ye7lRULR1y9HRkuVihrOGcegZBpHo8t7TzmdkZVgcLXn48CF/5/f+Lnfv3UNrzZe+9CW22y3nlyuuLtfEFPGj55133+Pq4oq+H4njyGq9FoUEpOislSFHqF2DJkrurIqxq5j7iWw04KNCq4Z5e0KtDVqPZHUwmbbGMEmHaaX3OQVZ8i4TMwaD14K3RpP3HSViKF405nVGZYW1GgNF5qjE/VpLl28MGMTcXMg6UlDJesJF5D7FnEhxynGnzk5JhPLkaQhCfNZiAK4x5e9C6tnnvlMLRC5dwHn/dhMeqAp+mvfgvZm4T0Jg2jPfBZfJOaKQ7gKlJwm3iNYITlW6aich6hiLJEtO+DBOlKBC5FWQRY5xKgDslQhKwUeRi4zvL47Nv/AKIECFLm3gJfkrbepmf53yvsI1AQxx0k3S6pBoTBc8yQ3UxpRKmzhxUwaC1hrrKEneIeGcwJ6Us5id7tnwxR1eZ1IyxCCLgiFDWUim14lZ4VRRUnuTFNlgJViaHH0hE0JkCJEheLIPqJgL0C+LhJs55oBtGuZHx5wdn2IMpDDgrCN4T/aRedVw6+iI3WbDZjvyyfMt7UnLfHnKrG4xbsbRW19nfnaE1SPKZGaxIzkN7SmLo1Pu3n6H268vuN4l7tWepvuYc3/Fd+w1I4mgG+61NV871dw/XlAHA2nJEDPnHtY+0uUskjGmJmkxmkkqk/UhEZ0W6cNx84cJDDo8/+avb/5syj8nkGKSdJo+IA8dJnUYPDqfYXTFqC3e1qA1M2dprABQozUMiIae0YpGRY50wiGSPJsdOG2pnCWyQKkliTU+jdRVolIZ0xpsB72uqdoGYxXD2KFUR7YnjLrmehxIeYtVDpUD2jh08jhVYY2CEBkRxoeYmDhQNbpq0K6SliWjMRq224EUx+LEXTaXAghpV0mlXstVClmjk5eFQ4Ft5ri6oakEvK2tonUBtMbOj/D5iDFkfPrlA3WQosZeGksVo4csq20Gkk6ycBfQJU3V6c9gj2+A4p/DiPzZx/c89sMg+QxYmct8FRqLBI4qJXzOhOj3Wp5aWdmop7YHDhv7G9i+uvGpN6rekz6i1YGshQEtUi9q7wYtgJ8A/dY4dtsVRs9xTlPVNV9+/8tU1nBx/pKLywvGYeDq4pLoBypr8Vlaheu65k0d5UzXd3uDOmHfgh9HnDXSMsYkvyEXPQQJHF6/fsX19Z9xdrLk448fcffuGbNZtf+O2sj6/eLF86IRJ2uWtaLNOElRmIlRVcDBiZkfQiCEIM/NwjRKOeGsQ1vNZOa1Xq/p+56qqmiqmt6sOT9/TTaGqp3zV9/6c7SCf+Vf/ddpa8fl9SVNXXN8fMSTJ09ISSRq2ralW6327dVT8DEB1hN4Pulk1/XRfl/JNza9wzDL5X2QIOJGdXzaT/4mGaL9u3wem/eN4tGbv3/DwHLvKwA/U61HwPx9LT1N1fgDiP6rmA8qLdunGPEmKRSm/UWALB1mzlXFkMyWQCWCz/uAJaupDVB0fTXyxyqFM5rKWmrnqKxFK2GaGS2Aa1VVxLpmu+1Eb107MeFDE0IWM7ggRWutRfoAJUlcYnJiz3tgG2NQ1qKsIytLVLZoS0+FvWLUeRMEn+6VIMjFONiIDvueDXzYuCaJoDeLfux/f7MlUgpeihwzfoz4biBqSUJCSTQk0JNiZC6tsGkKMIGqbknBS7BeNcyaGVZbqRnvNX41sRRvtTGlo3YKOIHCsE832CnTud9sk5wk+FQ5j5yldTQjHXohFK1jbZiiX6tlL9Bq+p9lsZyx22548eolVeu4dXaGcxV37t7nzt0HvHx5zsXlNa4ytD7J/ckK50BHKRvZ2hOyjIExBBTC6HPOyD5nLc5V1G3D0fFSDL5ixFppnZ/m2aSRf3R0xLvvvoszhvPX53sQXSm1j+mMMW+83gcZ38aIzKA2EhemYnarmPTs5R6TooyfLMz1XAyiY0qSj6IIPtAPI+LTIXtJKGttikFeG7y09/8Kx6fPX3K17kkY/vq73+J73/8xZ2d3+epXv07OCastwzBSuQaSJemAdYre97Ruzm43MPY1s/aIl5+ueR52OOfJNtEuHZWpmdeWoR8hD2z7a5a1IydPVWl265GqUcTR8PzZmpcvr1BaU1nHEz7hf/m/+p/wx3/25wxxx6NHW9xjzze+uWS2WNLYGltNZm0JV80wbcRgaNuKcbfim998yF/95RN+47fep/dblNrykx894cUL6NaKqsqczOd8968/ZdetySlw//Zdfu93/x7f/8GP+eN//l3e+/J73Lv3Ls9f/YgYA5uwYdMHtGmY2ZaTo4qjxnF2XxM3PaadU7mKZ4+v6M9HwklFb0bSItDnjgd33uVqveXRDx6zqOY0MfFi/ZqrfoNVmm/8q1/j4w+ewqLn1fAxmw+f0TYPeWt5i+9/9zvU88wQX7MzS/r+63zwwQdQn9M9X/P+V75BtpakKx4//5BaV4RdZuwD27jh8cuPOL215MmrJ7y+8sSouH3nFn/x7T/h/oPbbHYD1dGSy4unfOn9t7gYXnHdX2Bs5MXzR1ytAt/4SuD0+A6fvnzKR896Qhw5Pj7hO9/+Hn/wB38A45ZnVy842d3Bv0xcXL7m3q0zRt/z9OUnNMtL7tx6m+//9Nu8Gx5wdX3O7Vv3yatzKmNpW8du13Fxsebe3bskE/j2j/6Md96/x+OLD/nOBx+COiJ8/AEP7h7x8vnHJN+x2QS+8uVv8Gr1jHppsFrTdZHVcM7FboFqLE8vnnP7zi3Wcc2ffvdPePX6JZt+TVQ9v/Ubf4cPXvyA9966z08++BFvPXzAsycv0bsVp/dPubxcMfMbXp1fsLhzyqOXz5hXM+4dnTGMI8oa1rsN2UU+ffmYrCLv3PsGzrV0w46u29K2FW6u6C7XfPLsQ4Yx8ZX8VW7fvs2q2xJjZLlcsk1bPn7yHZSOv3gS/5yj1tD3HV2EZiYEnwhgGrIf8L4nG7+P5cCiDCJxGkbCuGFrgsSYVhG9Zxg2hKEv7NsDYzUUuRqtQVWyhuVi4Ge0I5Ix1hLSmmHoyEnY1aUuSF05xjERYkc/dsQ4AiJLorVC54iPIlQx6fpaU4nZWzIkwNU1Oou8B1ozxkhlFZVTDKEnZ4UVnTei94Q4sh07VLLY3rPimpBikWsVOYboM10XadoMeiTEkboV0D6lQMiGnBROOXCWqDQ+gfYShxuTSdnQNhXK1TSL4gGgNYPPEp/kIAQAIzFMGJPoIquIUpGcPIosdYuosPUMMKTRE/IARhemq2NMiSFFNt0OPWrMaDjKR6Tk6fut3NucReanbUGJjEvoIs2sZaYbTJROXSJY66grg8qKvutEskM5Rp/ROu1l6MS7ZcS2TuT18qRjNOLHDlcJ+KdzhdUNMViMqtB6JEZP0GBtRaUt7WwGKTOGgSFFPECCyjoq6whkjBX/AltVKCvdbr0fCf2aXiWMcRzPj5i5SEqGum4JJLbdiug9y1lLO2u535yRgPPLC4ZB5G7GsMFkRV0pjJ0Vg8tAVJmYx6mJmhg9eRSWcWMqgkp0YcdOJZIJJBfIbiSmQZjGMWFUg0HMTicD5V/2MIWQlNXEOi/yfDc0ow9/4JDTTn8fOiElDBS211RggVwIEcL4nZjIIPdeq4kOc3jvSVZyImTsjwLMi/Rf+pmY8vC0yQ+voIMpokyRzKB07mu5969enXO12jL0gdFLXCSEH4nHrTPie2RE87ptWxaLBYvFnKoUOWLypCw57mwxox9GvvyVr/AP/sG/zXtffh9rq71/lvcjr15d8v3v/5hhHNgWsPxoKXK4x4sZyR9DDGx2wiY3xjJrKuq6prYalSJOBSprGdkyhB4fBbdM2tCimbVHbNcD1cmc1lYM47j3QjB7NQ8lHU/TAorktAWTx2gxHxUDZJGqjTERdSImVeJ1RWWNAOlGYnQ5im9g0gRrihRMMQffj5sDwTBGMZmdCIZk6ZL97KEUov2uDKYQlgoFiEM9pWAvSnKMSWJT66loIwXAvGeDT2NlAoYVWemC4UtvqBTb5DoZJvkVg9ZOSGpZFaL15JMSCSHiJwnT/RfggC8RSt6nmDT+BUAXjqv4aP3i4wuD6DFNALYi50nnKMKUuJbKwsTi2mvLFAQr51QE+BEGa55YTqUlgYPmUYqysGMVaI22Vkw9coLgRY9aKXFvVqGcXyITSlu6whgtm1mU308DVxth0sUsg1c0QUv1RWmUTpKG76t/Qp02xqJtps61GJuGQChJf85gMNRWKuuVVuiUCH6g765IBKrKcXF9hRmklev1+Zpnz7fM5g23jk8F6TIG28w4un+fYfOC7eULctLEZJnffkAeDO74AVppHp6dcG/WoTfPsOEZJ82Wtx9Gvnpyh++uG47rzO++O2epO9SmY9y8ZtetaHnAbnNNWp6y0WsuouK5MlxmTZczyYhsyp5uqG5yIEub0f6BGwD6/hHhJ78JVMrwSBRjsUxpA6uIrhZd22HFbLfDGU+sjkljlOqvrkFldNhRa4i+QmtIOmAyzI2mTQM2dFx3F1ytH9M4hbJV0fY0GLVEK0NjahyGuVM0ecngigFG3mLSGu0syVXSFuY3JH+OdTOMPiKOoCqH6pHEPmdcGhj9jqgNWWdc3UjyAwI2JzHKMTlgsji4a1tJFVABwZNKy6RSFmwli+C4weSRymRQYqQ65g4dP6Ubn5Dya9o6Ubue9RjY9B1ZV190Kn/uIU7FU4dGKfcpaTlKJHQpsqii25b11Dzz+cffpBv98w+1B71kkTP74pxUCRM5CchCTgV48cLa3b/Dmyz0n/s5HFjwovWsCTFIu2WKmKK5KddGgLGURMoiJ433kXGI5Kz45JPHuHrB2d37jH3HkBPbzYbKWfRizrhrGJQwgEIIqMKyndieQNGCE7296frJxsOeSTxtRtNGPJlmdbuBdX/JP/kn/y337t3i3XdDsYdRAAEAAElEQVTe5p13H1C5shkWl/PV9aqAR1qM40qb1OiD6KyhRN/SWrTJDMMgbOECZBujqFwFJMZxFMPRNJYJfmiJmoxLg7VUzqG1YwwjKQX+8s8Tly9f8u/9h/8BWil22x1t3bKcL3j5/Bm77RattOhKGsMwDHL9i4HZpI19fX29DwZinEx4pjv7s/dbPBhEysPsNfMP43QC0v9lOKGflSf6rGb1zfeWfVPO7rNY/L5pcvqf+szj8WcD1X+Z48ASKMFFlrVV3ODlc0TP1RTJKkArTJL1OoXS0UAS4FBHtBHGjI4BnTw2JyoNVenm0irTWE2lFY2ztFWD07DrAtsuFsZULR4MJpHwhDyKtmnKKBMxQUDXmDIhFuMjpQpDymJshXE1yrUoV4Mte5eSBHfqkiGXPT6LjmU2ElfkGItXpASQcs1BdjFpXZ3WH6A4vx9AdhkvhXmRMkY5nGkxukJRieyMrfYFe7IRwCFHsik6foWZppQidAGlDE19RO0WWNNSVS1pt5KCVoCkpy4hKFWhGxVC0b3fl7ZvrMETq/ymyVCKSRiM5boYI+3Ozjr6vtt3F5KlYOdTwloNRsAJY/S+NTXGxG7bMW9G2lnD0dERv/M7v8Pbb73FbrejqizL+ZzlfMFiPmM2m1HXNa6qODo5xdXV/lydrajr9gB4a4u1Dls5KZqWc96b/5X1aWqVBVknrLHUdUMMEVc5Jib+pCk9FTDEeFDYdqF0JWU/aU3bEo9KoTmERAwj5J4QRmkZjQfDsWEYCV6kkPphZLPZcH75WkzyUllHtRbN75wZ+57Ndst/+A//nV96fj99vCuMGkPbGob+mqv1jn/2Jz+kbgO7c8+tW6cEr9isxBTNp4SPkWcv1hBbjmZnvPp0x8Wrke1qS1Ubbj90jCbRNI6T2zMyjq7b4mNPwuEqhbWKd9895tG4Zhui1LmzYuwzQQV+sPqEL33piH/w7/4m3//hRzz6pOfH3zvn4tWaxbHl/tst77x/ho+KumrQ1hJS4Pz1NedPzwnbxNFs4OE7ltfnj2jmC0xUzBct4XGHMYa2gdW15ze++Ws8f/4JlxdXXL8I/PSHF/zR3/8trraPCfkxz14kmsYyhJ6qziwWc9hWnJ1UjOEl994+I5sV280alxW2svzG17/Mq0/WoLLIAJjE7ffu8ezqOc+fnfP4o2fcXdzi9/72N7FHDf2zT9iuNnzvx9/j3r0HVHdmrHcv2OpLdumcD7/3x+hbBmpDt7ri4nlit4zcem9OMg3z4Pnhx3/CbH5KtoHX60csmzlPPnnJrbMHqMby/Y++xx/90e/zbPWET1+vSMnwbP0x3q151W3px8Sff/+K05MF6rXn6acfo6qAdoZoMp+8+oCN3/Kld77Ky1fPef3qOb/zu7/NX/7wj1n3O/75X/13LI9rzMLygw9/wG/+2m/x+uo587llfLbmRx99j13y3Dp7i10c+PEnV6zXVzx9/YJ7d94jeo/RgUcfP+Xdd7/KrQd38GrgenjN5uMXPP70NeeXHm2O6On453/xMQ/u18xaw/racy/c5fXmnJMwF3b6bk0/7Lj48JzZYsnoB86HJ/z00QfEHHh98YpsEkEl9CNDGjMvrn6C04nzlWbVr5gt5nR5R64Tf/Wjv+Joccrzy9cM/cg33v8aSUUuVq95efEa02ia44bnV09YHM9w88h2OOcvvvfnnF++4utf/zKohFcXPH1+zv2H7/Hp1Qc8evUDnHVsNhu+8tWvcN1d8cmrD2naXyU2twzjAHoE3aGsxTaV5Lg6ixG7HzBGFcAlCAsRLcXGFOi6HdYaGtOgVcWuMFrbppaCYCnQV5VFo0la9M99lFi1aRuMcwyjSBJYpUnKMERPDlE6y4aBfrtBawipx2pN3c7EWC5kgQnIZbsSaTaXxStK9Nc7EuIfYY2iqmtyMric0UTpfItCKJC4txDktEZp8RBLaWSz6QTsd5bFYo5RlpwPZrW5+KEYJaaESsma3veBWWtompamnQDHimHoMSYXHf2EMRZXbqdCcBG/G7AWjJ1IBpCy+DTZIguUSzfp6EeGTiRCqqoSnCEETIC2ngGKMUCIiWHo6Iee5dESjhZYrWmqCmMUMQWohPQQUmQMXqKUnEkxSZE3ZSaZoX3xvUgf6pI7TMX5qpIvJb32wkT3MeGDJ6uIIWPR5OQZh4BWiZjCPn7IOZAi+DEyX5wwb8XrYrPakLSwiKtKOibEMFauvXMOax0oTTaWZDw+J7ZjT1trlDU4JRI5GM2u6wk5cHSy4L233uJ4sSAqI02TJrHbdcQQWV1cUikpXmejsLbCeyQPyVHuVYr40OHDyLxphfxhFcYZhjgSugwGjDWMQ8ZY6cjVqipynAGt3K8wt0GbCbgGpbMw90vtQqCnmwD6xHKDgxCqgJExhYKnCRM8ZbXXn546ZIEiZcH+PbWS+Z/yRPSZANED0SYV8F44KVKZ+TwAXdLAQ8ypp3PPYvo7gcOpjMcQoNvtuLi8YLfbUVe1rAOF3CqxXN6TuJqmomkaZrO5ELgmklKJAScfsXfeucc/+Lf/bb72ta9Lt0GRRzWFKPTy5QuJc7Mjxsg4DpyfjyJ9ZBTtbMZyOTL4wNCPQnoo7bIhBHIIco2yzCHJpUAbS+Uqmrbl9fk5s9Zy984xziRClLxFYmbJB0HIxVnpw/XTBzlaq0VSJyaN1QKuh0kmLAQhdCBySs6aElcesI+kIFuNs0awFNirfkheJOMuRunOEGxPS5yapKglx4EsO52zfI+bY5M9lrsfmUo6TLRSe1LKvhMCGdsJKZhIV0TBgA+ftM/HoLDfp5xayZy2xsn4jYfnKVIpeAn52fuwB89lvNzIzcsYlXyMAvKD2d+TX4xhfWEQ3YfJPKUwqYqMBUr49weGllQyMkqw9XRITqUtR0xEyQIRSDestJQqFLlMnOlGiMO0IUXR7pNAoTAyS1UzFzH5jBiUGaOwrsIUJuVnxoB8Bw150jmeLlQpAky/34MjKRd9KKiMEVf0piZlhR89YRwFxC8TIw4D5y9eoE4z3fYFsY/44Nmstzz5wQdsNzvW1x3jmHj7rTknt46xdYNrF7im5eL8GemiBjdDMcOZGn0cMI2GUVrI1tdXxP4Kd/UR5vojdPTMm4pfb1d8/eSEXJ0xdK9Yr1/RXV3Qb65IGT74/p+gxw33vvx1Fif3OXYnPHDHXNTv8HH1Hs/TCcE1cGMyf/a4gSMcHiMfFs3Dg/tDWvqkrVCRcMrgcsQaUCZg6HFhy2zsIW0x0bKlIZgGFSNjf03jAg2GlDsSGmekkse4odu84vrqp2xWHxIbyNag2xm36tvU1RG1qTHGYWImx0htGgmEw042wzRimzNUbdgNEHIH8RLMgI4dKiTUUGPyWEw1LC5knN+JXjqTjmsie8+u26CsxViL8iOMHRlNUor5bI5W0F9dkLprsq0x7RHiR+1p6Kj0a9pKgpVdbNH+NWn4Fhf9p/Rh5KiZg39F6muir6hnx190Kn/ucXBFlkVQF5mF6f6LYIAA6Lq0FO1Xu7Ig3QRoftFnlZdxc67v12KmlqIblcnp91kqk6oU5jJJAB1bgEr29hNvHKWWVzYPqQTdZCJrLR4NAsgEcGn/msmtuesGxiGSokbh6Pst568v8aOnnVsuL6+wSnF6clQSlCCAWS46zIWFrrVmHMf9dQghsNtthS2Romy2BXhVWhFCoEsJa6QqP13fqqoE9G8SIY2sNyv+m//mv+G9d9/j9PSU09MZCumGWK1W+OA5Pj6m73c0bcV2u5H1K0M3DMyPjsRgaBylgrsHnCjJRpDiaUkUJmbwZLQ4DANN07BYLGhaMSfb9Bv8GHC2IYbAvG344Ec9f/bH/5yzhw8Yx8TV5eVeOsEYQ04B1zSFERv2QcwUWOWcefLkCa/Pzzk+Ocb7GwGVKpJA6s37fgj4hGX6RrB3g4n+i6Dqz8q53BzPN59z87lTd1ZSaT8Gb57jYfRLIsx0volDUeuXx9D357hvWSudAyF4jERu+1NIKewlPlIOJArjtsyrrORnpUXGRKVIGEf8MBCrGlwqDBZhuRhtcLZiuVzQVBXrzcBq5/EEAhaywViFVTVZi6lQDANDiCjvCUVnMMZY9iQpuKOk6KyMFZ8ArYVZNq0ZaWKKT4F8KWAoKQCqJKbUsVzXVIy4Yc8tZ9Kn3wee5VqmiaFz85qGSM4Ga+rimSEgqXTgFbZDVCJFpkROAySAJWesq0SX1HtyViUATPgxcHq0oGkaYops1htpXZ2SGKZOAcjTmjERA/LNjj6LVnYPlFtjhHVN3rcyz8u8beuGYRzJZKpJG9Yq6tpR1RXOuL18nlaTubvFGUNV1ThnSclLop+nwD0x2f9MibeSLI6U5e8YJzm2qZNxmlsS2PvRowz716aUGIZBxgaUaydA9fX1NS+evaBtZ5yenpJzpu/7/d8xRrz3DMPA6EdQScBlL1IH3o9l7RFQIoSMH1ORMxQl2eB7vB8Zhp7drmO73rDZdnifMboiRtgNPVmP5Jxo6pqmbjg6Oub0+JT5bE6/67i6uv6V5vbVa0dViyxT119QN4m6MsxmcHy2xBrF7dtnbNeRl88/lnthDSlprs9HXj3dEv1rqtqKOViyDJ3i2ZOBOw9qZs2MEBLGBdBCnLm87Dk+nXHr9JjtamA2V1ydDxglPkOzqqLfRZTO/PBHH/ON36q4f3/OvTtnHDfv8uOffEgYMpvrFasLTbM0PHzrlF234cNHjyHD3DmWx47Kad6525ByIATN1fVAM3M8fHiHzSpTVwOLmeXRx4+5en1FW2t2q56QtvzgB3/J6f0EzUDKED3cWRyThprlO7d48NsPuXXqWMwGcrjkfLVi2AVWq4HzYc3908Bb795HpRGcYus74sU5F9s1X/v61zmb3eev/+Sv+M73v8/83jFvvfcO87ZB+cjl5pJvffcHhDSiiNx/a8nxWWY79Nhhznx5ypfefZdGzXh9/oix7+j6S3A9tjrGVAOr1aestvDoxQuO751Csgxqx//tH/9XnJwdEVzHOGQChvoksd5dkLWF3NNfblj71zx/8RF3bp2ydMdkndBN4HzzCRff/5Sq0pzeP+KjZ9/h+eUzZrMlj1++4K65BwmO5yecXz/j05ePUHYQDxrb4SrNTx59l+PjW1y9uuTuvVucXz2njyPnr16w67d842vf4M47R/zZd/4Z2vY8u3zG6dmCbb/j7sP7rDYjj19+wMm9BRt/xeW2Y16f8qNPvo/G8Bffv2S5XLBdX6N05sWrZ3RDx717d8kqSWyRPPNbLbt+ZN7MGfOWpDKVzfjs+fDpFe++9VXOL694+uoJjx59QoqKb54tePbJI4y1fOt7r9m98zVW6zUvL85RteL1T1/i6VkMLZZEGALNKYyrSz58/l1225WYaes1j16uODo6JoTAs2fPePjwIY/++beZL5bkRtGc/vKx+ehh8AmlA0l11LrBaQH7XF1DjsSuQ2nFOA50fYe1lpwjde2om1okkkr86Gq37250RiSlcso0tSMGU4zDhdwTw1i6sy0qIgQWpXDG0iwc4yhyKcEHrFH4oRfCk860dU3lrBDnUiravonkJGb0MYoxrRXTUmMtOQSSH1E42UOyAFPRC3EixQhK46wrnjxiirqYz6msk87zSJHZDHvygdYWUwWUkcLxtF81tSOlyDYMaBQxKHLSOCt+FW07wxiNcwat1b54K7rEibpu0M5xfXVdCrHCbHbWFQ5SKiCsLUX0jHEOlwW8yQ0kA74Tvf26NhgcVmtMhnHY0Xc7mqrCGUPKkbauUAqGYkLuU5IC7uiZLxbU7QylFH7s8VF08AVLzqTkUTpRNzVaG0KQ/e8Q/ypsVVPXTgxV+54Qw16GV6HF7LTvCaOA0daJpAUKVI5YFFZB9CP9riOOEW0t1hkxnUTuSYhS2HCuIiUYxoCeVWRjix9PximRc8tZ2OM5gqngpD3mS++8w9v375N84PnFK6LWNIuKbMEPEdv1pJDRRlO3Ylo7BikMxRQxWfANX8g91mictZI7msx2tyb3as+oN6XTzWpHCtKhdjMX+2UPY8UHS4g+mWQEvBaVBAHAD92swCQHeQOpzAXnSjrtwXeiQueDNMUEpk/yFXtsR0nErNF7QoGANZSOulyeM2lEC3FEOi9LTpUnQ/vpfdX+M3ISM1SjskgAKoNxFSlpiJFu2LFerRj9UL6O2sdxWlfkZMFIjta2LUdHR/L3cgFI558P4iPnqoZhGPjt3/kdvvLVr0ncqfU+Xsw58+mnn/L9H/wAay2r62sm481h6FEohixs7XY2Yz6MxJD2Zqe5bTBWurFJsN3t9qbb2pSCYEqsVyuaWnNxaTk/n/H2wzs0bUUIXbHzmTTqp+tf5o8SckzWYBEjdimQSAFfKzBJYQCDqGZkpUunjkgTGQuUe5lLwSRGVTpKJz/KG0SgJNIncd/JoEthIO2B/pvHRKQWFngpBij2RJ59N3WW8aunjgpd+E0KUEUqEkQmd8rd2SuRl27xXMRDpy4GIcAZLfx3owxGEgL2fn6lUzSGRPBRjLJD2hPkhEQI2qn9+RhTMCY1FX6me/HFiKBfGESftJBEG0kTFbKpZQpbqSzEk2xKSTplUqqiyyQgfBGIOFQklCD/Srj/kgNHcV+eNI6zzYQkhmcpy+BRxmKMLtUU0deaWsC1MRjYJ7mTu/0ebNE3QL/CAIyT3lBhe0qilokEwn6QCfNKa2GROWswiCFmJIPVqMqQsufq/FMaHVidr9itO4w1rLqR8/MdafTcPV3y8HbL8XJBO5uDyUSj8eaYnC1+SMS4IaYN66g4e/8Wte5YrTbsdjv89hX+2Sfo7RWzusbuAou55axZgr/m4uUFzz99zth3pBjZbtb84NEV/+K7HzJrP+Tdr7zHg7e/xO2H77A4vubh/Jxh9g1ez98iqbaAVnm/ICZyKZe9WZVQZfAJMHSjMskEQkIOiTgGGDe0caAxhsoobFqTWLOKV6Q8kseMyyM1c3J1m6Cleq/HFZXpaZLG5RofAkZByo7NOOC7K6JfY1MkR82oApux4zRb0KK7uvFBTGO9RlUNCUXXv6bzW1y73GvGR01pX0wM258Se4WdfQ2tjqVFzmTQHp0UTWVIRoKs5Adxsd9eMewuUW5GMzuWqlbw+BCp5gtcU6MzbFKiv3yJrWspBJma0F/hqjXNLbDunJx+yujn7Lggjh+xHXv6oLC9xagVr8+f4vVDjPvVNvNY5rLsiVJlhUk6qcgWZGEu7h2Qywzeg9/q5ph4E0iEA9h4AB0Ve77tZ55vsohFKG0o+gt75ufBfEKq4pNGF0iBLX9uh92N8yifO33eNN9D1HtmYcqxNMJo6rpmGDrGwXN9veF6dUUYMtdXAhDXVct8vqStG85OT3FW8+EHH5V4J9M0NSmMYvpzI0iQZCez3W4FKO87MhmjjBBUUyL5CCS0E2BOF3mDCURKKTFrWwHNkubp00/5L//L/4rl4oh/7V/7u9QFaH/16hV9P3B2dswwdBhtmM1adr1CJ5HUsq6StjFj0SYSUkeYgK1yZN5kH2gjFW60aGf348D25Q5XOY4WLU1Toayh9yN5fY3Shrae8e1vf5v3x5EQMmC4vryClJk1Lb3KOK2KNmXcf+fpflljGceRy8tLuq6jbRvaVu/H1lRlf7MN8gAkl0hjfw0P3+3GaPmCwfHPe97PsjQ+Y2pbAhzRLSzrpCps9DI8UylsGTUVlX65IxUjMZEIKh0dpRCiS9vVpHEYUoIo64GPUjpD6wMrBUUWigHGVKAGfEj0Y6DxkZAUIStMMkQMPmkiBuMajJbitmhrK9AVKSmscqUjbCAkTUgR8IQ4JV1Ip1aJ/EVERgIfckalBCmUWoCsC5O8j8JIgDqtUznsX0eWhEEpjVGT7vxUsEuQY2FbyBoYgrQGTsaXB9kjudfOKtrW0baVmJrlWJIvkVbKY5AW+tphajGCnphh8/mCdrYAMm075+zshOVyztnZKfcf3GG5XGCtpe/FGHuSLzHG7Fnk1jnqui4AufxsS4xkimSeaKqr4plQwP3CQDFG4rRxCCLbV2IhawyRSFYyF2PMRB9K7FYC9KwIwZNiYogDiUAI455hEuJIDrEAKgcmU0yJwXtilm4TKaIIED4UdncMCe8Doxcd7V3XlXGd6Pt+X9wTAFwAo/V6zW4j2rbWWhaLBUop+r5/Q9JF1uKEtodAP03sfG2wVjTRc9b4UVp761quYeUsfiydi2jGXSKFHj9EcBmUK/NLY13FfDanrmqqqhG2aFZ4H9lstr/03JYxVNPFSMoe10SUS+gh0lqF0mu+/msnaB24c/eITx6LtIvWjleveubzOV/9+gm77Y5dN6C1YVaf4EfNavWKq/MRZ0eOtcEH8YxZrRPdBl48W1NVlnffOeFombieBYYuUjcVdSWGs2iFc5mXr54zm2n+zX/9N/n0ceTTZz+hqmo2m8Dliy36MvDy6XMSAdcWs2sqtkNkmweqmWaMA69frnDa0g+RumlIQRjx292a1apn1tToJB463bglJcdi2RBdIJiRcaxw+YTVy56LFz9iffwh73z1AUdnR+xWL+gG0cZfLu9y570zkveMqmMX1lRmhlKaJx8/43q75fbRAyIjZpbp08C4WfHe23cZR0+33rKYz3jr7ft8//ufcOfWbe7dfsDTZz/l0ScdyQ/87m+/yypHfvLyX7A8Vlxd9Ix+IKSO02PHxflzuv45bTOnT9e8uHwKyvH49SdomxivN4SYsLahj5H1qgMdWa8885ni5OSYIW1RNnFxeYEfEn034FqoZ5lxCGLsp2rOzy+5fX+Jyo4xGl5cPKatGuat5Vt//c/40pe+TFADq/GaqlasNmtObp3w6NFPuf/gFhfXn1JVc37y0fc4OVtwfOQ43z3mn/3Fp1xdbQlhZL7MLINmu4qE+JIhBqrGUNU1tp3T7RTrbkXQkTCIXNyHTz7g/v27fPzRx5zdatGV52L9gqPjY3bjGutqrjbX/z/a/utXty1N78N+I830hZV3OrH61KlIdmIzNZsyKUGUZcG6sW5sA7avDNt/hAFdOACGbnVlA7wzDBgyBNgmIYs01aLYTXY1u7uquuI5deLOe4UvzjCSL945v7X2qerqYhU9C6fWXmt96wszjDnG8z7v78FYhw6eLu2pq4rt/gZjAtuu5/MXnxNC5OoHL+l7j9GGP/zWP6NtW5bLJcvZkt5t2OZLbvrn+C4wO6rJXeLV1VNskr/Z+4593rFbS67I4vQ+N/sbqtpxtXrGYjFntjS0wzWb/Q2BjvPztxj6/S99bSdlGULCOsGaaKsgJ7SyFIUjBOnO0UZJBI4e5xRKzCZmct7pjNHi0rNaYat6dHPLGF6UjkGNwYk5SpdWiqQUadtxfBpNDKUtaKoaqyXcMlm530yFeWsLCmux2kiAdB656ymOxd7RwGK0zDVSoqpFAEtRHM0xaFQGZyxp8KPQLl3pZWHwITMMA2RLXZWSSZUVLiVoO4YQheuupJNIOXG/ZjR1UTCrawrn6Lo9TpcUi3KcCxqcu0V9xegJoScm6RAd/ABZ0XU9oFnMZ2hr8b7FeCmyukJCwHVW+BCk+03J2sIWBdpamd8VmWTHoECloLao7LDRikgWZ8QkeW9d3xLHnLeUE20/SGdYtKScsWVBs5gxWy5QOdNuEj4nbGFkf6eI0lDXBWpEVQ5DHlGYcpy1NrcFf5UFEWEdVVViRqTkEAaiD2AMceoA1si90ihKK0WaFy9e0HUekMBOqzRxDNEUTJqw8fvoxSWvQBcJPzpIa+skCDVrkvLk2JFIFCPusWoqYk7crG548fI5i7Njjk+W5PWWkDuaowWxi1IEqQw+yLwgjXOdlEVzkuJAJkVhY6MUMQS6wYt7WWnKqiCOc4WoMmSDKyqKwh46ZH/ZTdvI5L3NGen4Hv87ONEPXycx/M4aOkNSCUNCT2uOOGproxCv9IjRmwwmk9Y2ueAEDzEaZ6Y5mnR6TIzpOwxMlJq60G9DRjnMnqXLZTJTCrBvLAiMRQBtHNZZQta0oxDtnJNi3Dg3yzlTFBZXFDRNTVEUnJ4eM5vNpNNgs6EsSiTbDMHyWctX33+f3/jN38IVxWh+vXUCfv755/yjf/SP+N6ff4fNZi0kCi1/n7KYK30YBI1lDc1shg+RECP9MDAET13WlK6GlNiFCT0j+3FaHwwqk5Nhv3djkSkSQo82UmDUk4gOKCXzbFmnCmteG2Gwy0JEEZJCZUGgGK2wKIIxI85FYwtDWYgGJezzSW/R4zzTEK1oOGoUXEWKz4dlkOQ/TmtWc0B0fXGbXOB3UaXAKMzfGnum80QoH9NanNHklg/7PStAW4yxY5bHyMM/vLh0WGZkbm7MmJvF6FyfDM5ZS/bHGMrqfRzR4yMmfKKk5PF8HzVNPWZMjR9O7peM66E7n+/nbb+wiO6mRZgZRe1xqamVOVRgpmqsMnKBpvGiViNGxToR4FW+TU6VnWNQo7PpsDA+lMo4VEQymmwYu1oOucGHsUGP7r6pJqLUrRg+VcemxVMGwiQmjMK7marN4w6d0my1UugcDszQnGXI0GZkCOlAijJh0YXDlhZlDO3lnnbYsdv2fPbR58yspqocxoir6823jnn4pfs0J3O0M+QwoGcnpOaUFDOEDF7CA4biiNWuhX3Pvu1QCobtmq4bUGaBOXlE0rDNHXYomNcN6/UTPv/sKeiMH3qGoWflMz9ZKTbP9vzxkx9xtPiMt+8d87Wvvc/xg7fxR59RnX2d9uFvw2xJ1rfsVDPt659xdUmV8u7vJLmZCMrLwJzigO13nIRr5joxLwuK1NGzRccNJilsLIhpjSsNxkSC6inyjkLtsGnAxDUqWwh7YlAMaY7OGqs1jWsw1YKoeiKKkAs2+wFnIyF3KKXoBw+hYmYjOnuC7whRYfUCperx7xI5avpuwG+eoPEs8hGVewflShTSwVAY0NYRixmYkhQ8YX2Fv/6UdvMSV59TKEVZz9ClxZQVZT3DaCtOgTAwdBti2uGWF6AdPiq6IeLzjEIv8eFHDO0V224DA2y3mfVqR/ADYdjw2eePScVAP/xqAScxymRAjfiizDQ43SJHJlbar+qK/SmBfRpgUYfi2zi6SNVSJLbRBS2hKzlDHMP5DtVKedaf85qT+5yDI3gaPA684RhIKeB9j7Mlgq4S1p/cLA2bdcuLpy/p9gHvE8pC3/X0fUfftWRnefXyBfvtCmdgt9mwWd9Isrexh/bJYRjY7/eCRgmC19Bjq5fsoqnCKk7cZNJBUK7rGoDtdssw9CiT6fqIyhXf+fb3+L/8n/8hQ9/xD/6Df4fdbssHH3zwWmW5H3qWyzlJKWxRc3R0TIxxDO0St+ZiueT61Uu2a2jbVopkd479JJ5rYxi85+ioQXlNiC37tmW/30obcdVwcnzGl7/yNawpcEXFZt9yfHTEbttxc7PBWUtZFtRVRQw9OcYRJxMON+vbLIxE27bsdruRLa8OTnV1EDcncfzW0Tr9l1I6ONr//7XddaH/5TdimTSkrKQQm5JkFEwc9DHw5JfdJgazUhVKZeExIy6dW+TM9NWSsvD2VJYqv0Kj1eRqkYW4wqKME+dQgiHCkDQBQ1SWpDJBOwYsXdIMSRjYIUkBTmkRecmGrAwxQXQRYy0mGqzOh9ZHRtfCYd4xTXyUOF2s7pk5izFScCWJ+0sKcAMpyKJCa0WOA0M3hYVrjJOQ4okxbq0I00prckgYA4UTFqgwQh1lUcpk3+gDYslZRwaG/n3+5t/8dfp+j9VwfHzEfL6gqWqcK6irmtl8RlEWaCuL9bqqmc+XY1ZCBahDB0bdlJTOHRYmsk3C/XRe3xY/pQXydoE1jR+gRoeJLJijh5wnF5SYELwP4xxOE4Og8CZROhMY4iDu/eDJKRO84GO8DwzdQNd3RB8JUQL5BK2Y2O92vLp8yW6zFcxZDNLt4iXsSxlLIh+c4THI/C+OWD9rHEVRgoKYpavPOUdVVYdCWAiBuq4P872qqg5ih7WOcmSfGyMLpMJJIWQaf8uqYDarMUbRdRJcbIyW1mFX4YdEjC2QsdbQ1DU5aXLuyTvP0O5IsUMRMFpEiANmTFnKcsZ8fkJdVjJedhHf7Wj3PfG2RvlLbT6OTvoQefDmgqz2wnqNDldnst5QVJmymPPgjRk3N5Y333rEy1cv2a73HC8VJ285IoqyXHJznfn807VgEmzNsycbhuC4/2aBVRlnKx6/XDFvSsojzfe+85S6cjR1he92HC3nHJ0nIpH1daRu5iMOzFO6ay7uRf7n/8v3OFm8yw++t+Gf/dM/liBWB8omqoUl+Mj6emDRNCyOZ1xe7fE54aPBKkgxcHV9Sbst+MpXjjh+65TViefphxsKV6KzRkXD0896vv7r79Mph7fXGF3x4seR8BSObMXpceJy9Yrn+wHtIw/uv83NzZr1Ew8kcuzoux31QtHtnqNSyacfr3j46ILv/umf89t/6ze5ePucmS7oNdysN/zgz7+PHiJDN3D/rTe5f2/Om2+csr7eEduSr77zPtttx9WLp7xMn2GrjNYV+y288cY7PH/xEbG3rIYVrhq42exxjeXxi0/RpqaLLc4pTBbHYDIF680GYxPVrODRowturj0ff/Y59+/NxzE/0re9FPG0XM/OlYQgY0Nd1VIIi4mmmWFzpnQGZTwYzyeff0C20MWB46MZRiv27ZoHD09Zr1/RdpHlItPM4fTeks3uFZeb5zR1TTVv2O3Alg0pOv7Wb/81fvL5jxjyDp97bi6fcrQ4w+oSXUS60KO0pusGdOnYdlvOH5yQY4/G0nYDPq5QuuLTT7d85atvUlcFRsN+25LCwOnpkt3+mrJx3Oyu6LY9F+enNM2MYeg4Oj6mbaWjZrN7zp/+YM2jR29QLBWpS/i4BzxGJzY7Kf730fPyasXFxRlKG27WLXW9YNftGAYxmM1mC4wxLBZSsHj84Y9F7Pklt2o+l+5clambSrBUgFEWM4pRRV3irKOoK1zpxnm0eAWsteROMA8hyPidgseaUgp84xzT+0E6ceJwyCBxWgTabt8RktyblVLkUgKKc07S1aStGOq0IC+0HUOvteQGWWMY+oFh34r5RmtCTgzek0ImeJnjxRTlXM1y7ymsRYHcc5KYS5wzGCNjsNArIoIDkC50xi5156QjvZmwgFGCpXNK1GVJaQXvNURFUzW4eiYifho7UZVoGVrD4DvBucQ0doHpMbuoo64DZVmgjKesLZpIUThBklWZm/UNve8ByZzSbizIR4/3EaOtuKdJ9AxIP1xBqR2uLilDJcJ1F8VApMawPDxhdPm6sqQoC4bQsWkVx7MlVVUxJMnViing/YDRmWJWQZZifM4J54x0YmvBQhRuDDAdu76MAOvJWdEPkdhnbHY442SOl+Teb0uHiYYwJLrY0vsgTHjG8EUl50okEaMaxTR9KGorZ+l7T/AiKktWjhQ1lIokBa4oKMoSYx3XNyu6Xcd+vaVre+ZRim5F4dCmxxZWwlf1CIWIAQ2U1h5ET60Uyjqc1hSmwGQZc0gZm7WE+PqAndVEY8cspog1NYZMOJjJfvlN6wjjUVcIc3wEITBxzCcWtNKjkQ0ODvAM6BGLDJMBIBKT6DZ5NKiN4Aw5FphxXa1GjU7IEXnsmk55DL0ftR6Zd45iv0qj8DutrxKTOieaWb5FkkyibRJxWKavBu87UooYW3N6dsrLyxXX1zum7LGyLEejmIjz03rZOccwDKxXa1IInBwfU9UNRVmxWCw4OTnl9/7uv8PFvXuCKLEWnTSDH1BK8d3vfpenT59yfX0tzxWkG3wyiU0a5uA9ymRZDxSj4J8E34yS0FyjFaXvMTuNCpGYEimNBmIja9WLizOKwhDjgHUaiGiTDqIyjBJn5tYhrkTvU3oy+WkhHyTZf1qWP+ikJxED44T9L+jDsZyRJ/SymI1i1Adp5K7uIaevwgbwfqJ23Golry34x23S+mQJJr/XZkTHTurNqLMIOuVWNlSH8NzbwpEaxWtrjBR4lL71OeV4EL0lw0Nh1RjYLKefCOpGkUaHe4wJExPY8eN4P+Knxk4MfatpaSZOu3Ts3OJipk7bf4siuhlFZqVvW6aVEpamMlpsdFGO8mTfV/n2QBkNblysA8RpRzGlB0+sT6n0jYUDqYXl8dDk6aJSh8pHjBPaId9KaaNIr5RUofL499qoA8NzOjVCAqVud6oaDyzKjMGpUvkh32ILgIOoc8gzztMxVWRjKSpHcfIAv7uiMAkVMh9/9/ssK82jX3/EvC6xFlQFUSW0dmRnMbMTdD1Da8vQRRhbm4pmiY+JzWaFtpbCjunfi3NM2RDm5/jQ0fZbSttQuhI7P+HyeieteTGx9Z4XO8FRHDcVbz5a8ubDc85OTygKxe7mKabfEl88o331DPX1f4/m5AKt8uutHer2H9Nlc/fHmYxVUAFVSpRDT/aeloDXkTLtaMKaJldUZEy34j6aopiRTcGrrBlUJRy23HFaBko3kxCusEdlS5l69rkkuxpjKpyO2P6UHDa0ocMahzYFu27Ah5cywbEOxbHwr9uego0E3bhTBnvKJs/Y95rVZs961+HbPb7dUFjo9mu0eyZcrfkx1lmssWhV4E0pwtEwEPuOOOxkIta3xGGPakpcUaF1jbaGEIOIgNsV6/VLmuURpR/QtkGpkravWd840vyM6L9K1DsSnri7Zrt+yma1Yb9d0+89z59FbraveHL0E+B/8otezj9ji6OgPeKMxqtvugamKl7Oaewk+eUmD3+hW306d3IeOWBRAkS1tLZpdVcEHZ02anS2HP5+ek8/o8ijpjFLbiqHcUEBSGFNR3FhxCSOmixX8zgwCxO9a3uKogJl2KxvCEGxnB1R1cLw3e12PF9fM583vPHggk8//lDc4k0jLggjoXibzYbNZjOKC2ZkpY9C4Xhjn8YY771MwJX8WxypHW3bstlsUBqqxtK2AVKidIof/vAD/sv/8v/Jb/7mN1FGHZABNzcrjFFstxsZI41hv29HLnoauY9SBa/rBnfvPkYJOma9vmEYXg/JGrxnWdeEFLm5ueH45BiUkpbP4Ikps921ZFYcnZ3z27/9N0BpPvzxB8Qg55w1hlnT0O4r2r3FWcmP2O/3h8CsiXksbb2J3W4HOXN5eUnTVK8JioyCb2YqmNy6wu+K6X8RjuXf1vaLi+iMoZ3S7SOLvbFir8ZGyl/BiT6bNbT7HUqN97AoxRqTtHDQ9YjjsA5nChIanQ1YDXlE6oQpW8RhdcYahzIWZawgOZRiSIkhZQlX1gqSBauJ2pKVIWQ5v5SWCaA1UjQffCAMntD1RN+Rw4DHk7QiWbk3hxjxIYBW1FWFqgqs0VTOcHo04/z0FOcc+7aTVmAngTPi1rZUVU1dVzirxO0GI8JkQd004tp2Ba6wBweCVgpnpX19wpdIoI26DQfSEmKKHgPflEKw4ZmUvYQG5XGeNLZgxpSIOR6wdFLQGY8NMlmPWTh+fR/RWQSuqZgWD2HqkyieDv9Novf074knOYnW3nt8CPjB0/cDwxAOj5OfC8f77rjkgywqfPLCkjRy79tt9zKvUho/eDabLft9i9Jwcf+M5XJO8J5XL1/y6WefsFmtqIqxzR9ECG9qyqpBj2Gh4kAqhGMZZExytqCq5N7ZDy0uOpxz1HXNhMby3jOFNOecWSwW9LXn6upGim9K2Pq2KKRQMHYBCVtfLDEpTfOc0VVDPnQmKaUpS3E7phxo+x6DhC3P5zPC0LPdQuHkOoo4dq3Hh4RTJYVdUFdHzOpGsi9Ch9EZawqaZvFLX9sgwXkxeYoKlApYC/WswhjwBNZXLfdPM2274/5bJYuLgs32JejEMAR2+8TJ+QX7dsePf/ic61eyWLTIwsQqw6vnHcNQcHZxzHJ+zqJK/N7f+i3uPUp857vf4fHjNcZkyqqkD57SFZyd1rS7DU+fXfH2l885Pp/zg09eUVYBQst/+y8+4+q5wdUDy5NyXHAaqgqiNZS5YLfT7LeRkJ0sL3QixEy7ibTbyIOLGQTNj753w69/811efK55+vQ5904W5KjZvGz5099/yv03F3Qp8uzJc8L2mlllee83HjK7F7l+8pIh30COXK4aHt1/m3a3Z729ZHnkqOsTnr/6HGMjvm95+MY54OjawA/+/CPuPTin3ayYNRWffvQBVokzdLeNPH96SVm3oLc8f/mK0+MLHj64x831ir5v2e23vHh+zXJhuHe/oetWKAr6YU9Me3RpMGXB6b0ZMdS8fLXi6PSU3u9pW8/J4oTtdofVUgBsqhm7zUoQWnPYtyvqqmLvO7KJ1I3DFhkfFUVVkXMDFPjQoZSmbmpSSuyuN1RVzWq/pzqaURQlV9c3VNWC61drmsJhbEUAhhR59723+Pzjx2htubp5hdIeYsJh0Qrmo8j29/7GPyDseq5fPmfVbmkKBVWDNYa2jWgfsYWl66WQ9fDhA3SGl89fEEPm+HhBUTYcHy9Zrzd85b2G0+URMSZWNy959PAR+3bH6uaa2WyONfCqveHodI4pNJ3fM8SeshcHaux7Sq0xOvLZpx+ijRig1qs9i6VgF1N2nCzOefbiMfdPTzheHvHxp4+JWQKYF03Dg8WRCI9Js9/seHh2QWd2DLWnqH55Eb2e1yzzkhCHUSDX4oQ1zWEd7JwUBsuiwFoxr8UYKCs74vEiKmeCl2MiWWYiOvZDz67dk4i4woJOGKXRWfB6Gdi34uAvSyloD9ozBD8iS2QN6LMnZC8d3tpJflkSzIEzjpADfUikCCoAObHb7yBJ2GA/DHR9i7WKjMWqJBlVUeao2jmUzpSFIyUvc1eiYFuVQ2sLymIZ0F46SVPyZAaMraiLin7oUVn4u8F7nJZovCm8T3SLNBYZDCkNdP2OlIJwyKf7k3MURSVuUZ3QLlEXBfN5jSJR2ILCVWQyplvTtT0kTfQinjurJQwxpkNXfCIyZHGX6+jJWVA9ymmIGp8jtjDjsdMU2qE8WKswRSYysO9adn6LMxob5H7qQyASyDlgrRFBTjPOkcWpbZ3kiBSFQ2c9dpbJ/GUYvIhfOtPtesJ2dAsr0E6KncMg8xmvpAjS+oGirqQjOA3olLEJVC7HOUgci9/Qd55BRZqlJYVE9LK+DCqwHbYURS3Sb06C5bGOEDOr9Z5d7nDKoLNhe70hRoXPkRw8YeikAxNN9orsPZUxY0evxhiHVjIXtNpgsxgHbIyjaUSj9EAYPL4f5D0kccpnFeQ1Rnb4r7JZNz7DtHYxmbGdhGkeMq1rjZmMZK8b2XKefDajWD0aTLQCK0/OZDLLSKA5WbChKoPSBoMiG+k+yTqOnxeMjmPhVc4BrUXknzB3iVvcp1EGo4XJnhH8s9ApIlVRoJDin1IFPmhiziyXC87Oz1ivO2KQud9k9rBWj9ihySBhWa/XPH/xAqMMRVlT1jMuLu5xfHLCX//rf50vf/nLo5t5XNeNYuirV6+4urriyZMn3NysGIaeGANGyfiptaYf55s+BJSNFEVNVVf4wdMlmQebsYskBH9YA0CQeWESVOjR8RJNz263AZVI2WOsdJQrPYrio3qeJGxSOpDjrVdKYce5qPyd0UnGiXQrYstaF4yR3zsnOVVyfMS4ikngINnbdag6COh5fD5FMgZ3Z4wDDqY1JhH8ziYvLUZlpURblc7W6bcciiCHv1fTxx5d6JM+rBRaZ4xKo3N91GeYcj7EOa+VBH5qEkYL6kUbMxaFFDHI57Za0TvDMPSEGNHaiUkoiG6kVZ6GQDEY5XEdrBRW35quJwrKX3oN/6WPGLdDuqoe3Yh3MA8pS4VUWX2oZMjiSqHtoWBCVmMrOKPLe/Kzj3t4utTvBu6Z8Wsag0lVTjIIToLTWF07cJPzhB+5bYGXgee2qJLJt7wkOFQT1ViVkEpfOpxgCjB3LkxhrI4HAYRhPO6NNN4cIlrc+/Tkfs+wXXF1veL+ccGDi4b5vGCz2xN8jzEnRN8yW5zA6LZSlChrCH6Pto4hdOz30rrcuEIqNBpcc4ypl/QxAwV9D+2LNcPZnMttywdP1/Rdx64bWO862iESkmfmHMd1wbIpuThf0vpMfTSjrAv2PsDqU57+yT8m/9Z/wNHZxW3W5GFgl7NQ3zly08/FrwdVysxD4KTfU8WBtYFLoxlMzcuhYzcMLHVEDy0n2tGYhs5UXOsFvT2WynGM1AXMiooc56x2W5LfQOhJPMRXDbY+ImiIu+fsvKfr9yxLca/lqLjerik9nB7fxxUF3mtW3R49rEhFAXWFsXOSPmHvA20Q/E3b7sjJYDPE7PHDBtuvcU0DZoYpSnw0+DSGOAAxQ+8VIWi0EZaaHQMv8qiFZWQxb5sFbn4GJML+GpMSOmpiVHT7gBoUytyHysli3+ypZ5cM/jE+rtiuP6PtXkDsuHrx5Be9lH/mdkjvHjlW03WgSRg1XUPTdS3BUZlIHtu9fp4Q+Rf/bmpjGq/lhFTFkyPHkqy9dDLkSKQfq+QyaY85S9spTKMICofU0IRZfyvMvy6kTqfqxKaWz2+wpkArT0jCVAxeHI8hRIqiZDabUdiCmytZpDbzOcHD+uaGvg88UA+5SYHlfEFTOdZX4kZvqgofAjebNbv9nt2IKSmLgrKu2W42hBAwbmyFY7r5m4Oo3vceMDRzhbIG7Qw2FYR1RmeFsQVl41hdb0kKfPZ8+3vf59MnL/nKV97BuYLC1Rib0cZzvdoS1h3GVngfWCxn+NDxorvm7PwcYypW13v0mFhd1xW7vUEHNbZEjmNkkMVZU9a0ac/6Zs1s1rCYz9lutqOQnhj6nv/vP/knfPLRJ/zt3/07PHr0gO1mRzObk4hoq2iWM3bDnuevXkiHDfrQ2qcUDIOMf8ZYum7Pft/ih8TqZgcYqsqNgpmcW+kQpHjrQGf8fro3TmGLSo3jubo9N0fZ/e5V8jPP7ek17j5Wjde6mgrFanIEw+19eSroaFRWMilAKvoTG10riNziV36ZbbfbHj639wMqiUAag4iP6Eg1MqwPLY5KvAJqZKhPqe8KCRqNPhBDj1aykLUGcvLE2BOSp7COoiiEr1kV4sBKlvlixiNXYKs5ZT1HGWktNsaMATgKrTJlaajLitliTtM0FEWBdQVlXYpTxYmQWhUlTWGpy1KwIUmcC5NQrw+BOTJC5CzJE4wTpbFUiASJ3insTcc+3c4PpnMoeD+K57ddDkMYGLxwQsV1k8g5kCfRO2VyzPghMERP1oq2GwN4Q6Lv+gODe3Jkex9QaHLM46J+dAyOPO9JZBYHXcI5dxjj4tjJASJWxxjpOunGstaOE/6Cru1puw7nxOHYti3DMIzi8BxrDCHK8XaF4f75fZbLJSDCVt921E2NUuJcvSLiY2TWNJyfn8sCpO2Yz+YMbfcaKk+4tBXNbDY6T267RKQraJz7jYsqNf5tTNM1qw5My+k5J/yTMDQbUrqi6zqapsGMQcVFURzasKcinVIQfMQPA5mEcwXGakIYCMHjXEndVMSQ6HrptLEmUxeGsqqoqgrnHM1Mo3XNfsis28Dg41iAcZRFjXMVXTew37VjIUmR0q9mRR+GgNIS+tq1A6LJC+N9PltgXUtViOB3dKqZ6Ya33TnD1vKtP/o+z55v2e7EaOH9uPjJMq+VEDjp8tzeJHy3Q+k9YchYnfjqV+5TL5+zun7At//kGX3Y8epmw6NHp8znmq9/45zjs4bVdk2zLfnJk2d886+8y7/6o0+ozIlAMbVhdT1QGItONS55itJQL0749KNL3v/GKavdJWURwSliDwrD6ZHi3umC/Sby+KMdcfcZv/nXfp3/5vKaF5drSmVQCT787nM++O5TZpWjLOD+lxrOf83yxm/cYx82fPmkIWdL9hG/aXj82QuePn7BV75xiqvmhGzQ5hitemLasO525NiSs+XPv/MMeJMQB+LVM6zKZC0L0fN7M6wpOTk/RevMu196g6FPfPb4E25u9lRlw/HRCV/60jlDWJHzQN/BO+88RDp5Zzx7+YK28wzDHucajpYnvLx6SVXJPbBzgf1uYLGoODs7oe/3kAI+7NHGEmKgPj4jJcEXWKPp9y1ZK6JJrFZbqmqB1gUZRT90HB0v2H4WaeYOV2qeP39JWVhurnbcv6jodonYdWgXwGq6IbHedKNZKXN9ecXx0ZKj6py/+uW/wvf//EeU2XJx+oDLJ1u+9u7XuJh9SONKqoXm4ZuP+OCDT8jdmtmyZr2/wWnF+elC5pkBFk3N8ugUYw1dv2O9uubs7JS+D+w2K7n7qsSzl09YLJdo49jtpMPk/v1zrq6vCGg8kawVu65n2cw4WpxAkmulbVtu1hvKuiQGj7WOullwc91TdC33752TYk+32/Dw4QVDzqQ4YLShG8eVbrfHGc28qlDDgA+BNva//LWdepRFXH4qopXFqhJrK4zJFKUjJREvtDWURkR07yUYVwFF6TAoQZbGQIoRbSRwL6YWHzw+DSTlKOsSHyI56xHXptBYUgBTS/dUBoapGK9HAUWDsqMbsnTSkTT0aG2xCLsbY0eBR5BqMShUVhRWcKjKaCIRowXJmJSYTlwtGV3aRFl/aEUKQUTyJJ3fzlYoXRCVIm63Mn+PA12fSWlAFyX9MKCzFFJ0gnnd4AdPGzrwA9poMhIYmbJl8BHvW8qqGLs22rFbrMTZEq0t2iaSElSOsQpnKzRWXNZeuo21NahsiWEgpYyxUnxPXUJHhTJyXsYcsdqQdaCPmezHNY4RbSGQDi5XpxLaylw3pA4/dgJoC223xYZSCuwqijYjlmBCHI1Ho2NZG4PNt1izlMZ7opZsIpVFe4nDQNcNmGhR2tLtPbpIhJzJ2TIMY4YUif3gcbOGjGTrRLwE0Q6w2+0RM5MeTTRiTLDKUJc1Fk+77+hTj9OORVOgTWLftYQgRTbvI6GPOG0JKhF9pg0tGUN2iqGT7qFRuYFsUCljlcJZh9GWsqrRaLp9Ky7fBDZbSg3btsVnTyQRrGez2WCsUBRS9AQdRmEu/awlwb/R5orbjthpeTp1WjO5vac16uhE14f1xi2qUpY2auwiFQ0m6bFxPAF3VzN54lNn8b7mQBrFSZ0DhogauZJaK6wdr3OVR6qiZBhGndBpEkxBmySPG9cKetKLMpSlFCZSlA9YuIJdF1mtbmjbW5zLNL8FKEuHtYYHDx6wWCzouj0hBLqupXQV3nv2+z0o+Kt/9a/y1ttvi1FSW+nqVsL9f/nyJd/61rf4/d//fT766CN22w37/Q5rNE1dU5Yy940xElKUQOWcQRnpOFHgyhKdIv3QU1o7mro4iK+gsFayHLquJac9MVUiDo9ZCcbe4lTGA0ueDLjTepRxnylGbXO0c6g4mn1lLWmmP5tEdCPzXnl6cVVnPa07X9dZD1seRXnFQZy/XS9pJg39i5vorRwKNepguuT1r6+J59P5OtI80vRQdSjSaT1x1PNhDR/z2EmSZU2lR3OTVhmjGINU5ZwX9IUCpMuFbNAGQhjfo6RbS/FDIrVk3Naj7jIq60qNPzdqNJT+/O0Xd6I7K070UWzQCaIWGTzfOQmAMWV1khU0t8fudq+qsaow9mQJ1F3dhniZbOTIHjbh/cTxR0ZPrvbRwafzbWuNOsRcMhZ4OHB6Ri4bU7VuXFjHSXw/fBap1B64UYcyijx/ylluRDBWbcVFmGOUFPFkcWWJKh1xprh33/Hbv32P7CPliGTQGlxR4pxGqxKlKpIP+KEnZnG4ee9xGWLaknJFUkqSpUMvLLKyJqCkxSskLnc9cdjw5Mkn/Pkf/BEffvocbY0slofIclFxcjTn6nrDjz69ZtMFfPSc3ztnaA3OOha1Jqdr9jee7Z/+13R//b9Pc3Jy5zhONPvbC1NnqUMlJcXUKmTKfU+13zMbdtTKo2wBZU1rTthi6dsVng1zpymSprIOZec0eklXLOk07Lo1VQqUVSD5ntTtaXcv2HUtfXVCasbWMOXog+Fq09JurjFGU1XzUXTVxCTKaY47rJqx95FuUChnmFenhOqYwTYMKWKagaKrccUJff8MbRe48hwfAnYI+CgXTo5eOK0powiYlEBbXLXEFjVuNsc0C3AVUVtUguSDtL5ZS1PPCc0M+me4vEV7RWWPSG5GUxpU6pAADks5O0XbE4w6heINhn5NP5zz6ul/w3w+p67u/6KX8s++viecg5pYUbfXqVJ3ujyUEiKxiuN5cFtM++L2F1XxfvbPJyFTQjvJBTnJxA+ViFkEKpkoQMiZkIW5pxHnnNZ3uF9M7//1As/08y/+Wz6ztMjGJIJWPwwUbiYCWIajoyNOTk5p6md8+MGf0+4HrCmJEZwt+OgnH3J8cU5OJ+ShZbu+IXjPan3D5fWNOH5Gp6VzDqUVMQZxyGg9Crpp3KcyXqUkCdPWlZRVjStKjkf0yuXlFYWTwkFTz+lUP7J9M1klQkp89PFn/M7vfJP33/8yP/7xT7AmEdKO2bxivdrhfWZW1czLmuX9+zRNQ9t2DENEJWEObjbXzGYzrNX04zCoVEZlaesd2g43MxRW2u36tiNacfRiLFol5k3Dr737DvfOT/n+d77Ne+9/jbOz+5KsLtNt+qFnCF4mMVqqzJMjVjA7cTwvRcj60Y9/zP37b1BVzcg3lkmXsW7EbcBrMxSm9347iXnt653Wtb+4JHTnjH3N4f4zTq3x+3GOxeSK51CalYeLcCwZABpzKFbmsYUvqTvP/0tsdV2TRseyc5avfvmrxK98mU8++Zj1ZosyluOTU87PTjk9O6Yoq0PI61iZpihKrBFOqLEGP3hiatEq41yFsw5rJXhysThBG3F4FIWlcJbSgc3C2kY7tCuFjWjs7WRQyTxIQsLzLT5tdGwkNYXaaCbsFFkWR4yLPaPcnUL4VBznThFd5iAxjYX5LNTGYRhGB18aAy6FjxhG57akvHtC8HTtHh9umd0+yAJ06Huuri/ZbdeE2EMO9F0nXRk+SggOGp8CnR/wwWONxVqH91GcjEoxm81o6hlFWTJrZjy494DlYiHcRy18z6puKIuSjKCXyJmmmTGbzcg50XYdfddTFI6maQhj94tCUTc1Whv6PnCzWkswclOTU2a1XnN1dYVWimbWACKsT2iZe+N72e93XOpX9EOHc1IAkRlQIKSB/X4n4nXdcO/ePXzoUcB2vSalRFEUhy4b773MLw85NNLebEa2O2hx4KeEqwRldTdbwhgzFjvDWGQTZ38m40Oi7Xp2+1b262xOSplhkMKDtVaK7SnjfScBmgZcIfOzfgoZjf6Aq4s5EbIjx8iUCzb4IMgBY9HOCdotJyIZpQwpJryPFC4QoycmT9+1aBKu/NVQbJmAsYpmviBlxWbTsd4MnJ47tpuESSUeR841q01iOyRePv2E/U2HMZmm0qTYYbQi6Yyzo1kkJYZhLPyNi2dthH1f1prf/+/+BZ+/fMi//x9+jXv3HScXF/zBH3ybGGoia+aNxRUGZTqKIrPbrbg4K3n17JpFsWC/iuxWO0pnuXzZs71uKU3D3/l7b9CFK8rKYIvA1fULyrkw+ZXRFE3Jbh04WjR0u8TTz1YsypLVZeLpkyu+9N4jfvzdjxkCjA0doBWr1vPgvKI6K1m1a773w59wen4uWCKf+fTjDT/49k947+0HXF3t+ODHge13PqKZNXRd4I1HR7z/1a+AloX+ft/i6gIf17z15hvUdUUm8uLFJX0XsKak3Q+sVju6fs/5+TkfffSYsrCs1y3luWGz89xc71keGwbfcnJ8yovnL8lZUdWWMGhKV2EwXF51FK5DjQKnsyUvnl9jrbD+u7ZnvdmwWDSkXcLHkXveB87OztncrKjrGSTNzepGeMMpYYwWfJ2rDgX7ZlaxWMz5/LOPKYqx2FV7hqHD+555IwjKdsxn2G43DMNAtag4qo+5v3zIOw/f4Te+8ltcfTLwzW/+Fl//5q/DYGncnH/39/4Tnr78lIv7R8zmJab7Ft9Z/RlFLSHCatiwPF7InE8l7HzGyfERl9eXxCj4ia7rSEkBlrJ27MOafghY74gqUi9q9vs93bpl3jSjQBMIKWNsQZcSdB05JsrCYaoGM3j6mKhmC4YIfYic3juRNSKJ2aJhCANVZWhvdtxcXmHH+UrMFUVT8Or5inM8dtnQZMNnjz/7pa/tthP0QUZjTInVjsI4wKOjwiSFVhYfI34IqKwwKGzQ5C6SjcI6h3GW/W5PH6Wwq9MgxVsGos0oZUnOkJ0V3vYQyNqOruuSOmWa+RxljXTyaPB+gJxw1qBLhQ8KV1RUtsanQfjCY/6GdpnCWLJODD6MWR0WsiBhS+conYLs5XodC0HGGDwDXnnyMNC2nqp0ModLhhAVAUtlS3TK6JghRoIfiEbwRKnzmEqNYZYZcqS0Bh9bYpLcDHxEZTHMKKXIUd6nRlEVFaCFz64zVmuqssQYRzYDRIskogmmDgwhgvcZFQvqoqHUFa3fYq2iLg1tvyXnjE2aylT4GLGqQGtLMStJQ2Tb70SXcI6cA332OKMwJmGTdOAPMTOEhDKGohCczDB0Im6rjLMWZRAUm4ZEQvVjIUUmpCg4dMIaYxnSiJzQmsLWhJDo2g4/DNT1jLKoGPZbwewoBcaRSZS1IxOJKlOMnXdeJ1xdoFxB2o/ud2tQRIzNHB3PiQGUiig8s8ahcsIPUTqqags60yZNtkhHw3TfV4Z96Mk5YLRgf3IPySucrg7ZK37My8s5k5MHDSn0ZCVIrBjCKPZnejx97tm1nqzBaY3KGvEsKnRMaDVQlI6QMt0w/Er37rougFujF4yIjYOb5+7a5XVj0LQ+fx1VKQJnCkiuSMoHo8FtYLs66GwpCWIxpoRNmRgVMRhZi6Y8GgFBmym7bMxSivLzdCu5oUf+ujqsERFncYYQJbhTI4iovo+k6FgsZhwfDex2nuAFhzLdk7TWzOcLyrKULufjI7z3YqixVkxmWvPq1SU3q9Vtl+u432KMfP8HP+T/8V/83/nBD35A27aklFivV4eikQK5btSEjRx1xJTE1KgleyhpiF3HZrOhKcVQ16WEMWM+WUxoI+sUnyOLueBW1+trzk7vE6IiJj+a9m6Nhwe87FhsOGipOUsXBZmkxKQ87fuD2ItGa7nGrUloNeY+KjGe5NG4ZWweuxTktScj7N0MQ1kST8ZhPdJBGM/NO+viqWPicE4yCvXwxVWzmop3B9k33762ev15tVEHt/0UfDphaRJSCVKTi3zUoAT3Ob5vOb257f42KOMwUfa10mYU56c1pBo7ivWho0uP+046L8acKW34y7ZfePZ+sihe20V3cSvp7kUNr1VV7po/p50OE0knMwUeHGoISlxkYuXPk2Z7kLKnxRFM6bW3Pw9336G6teLbsb35ViQcnfCj205HWRgppdBZSWiputM6wyTD6btn+Sgly8mIPnhyIUXoOvq8Y9i9hOipCrg4chAVw9CxXm24XnWcP3qEsQqjZqQ8J/bQ5Z2w31HiVCeSImSTceVcqkVaoazFuhKfNVVZoZTBqMzjj6741u//Ic8+e8IuBAlByVAaQ20se9sTc+bJqy3ZKN566xxrYHV5Sd/1zE8WNHXJibnh8qPv8ioOnP/N/4jZ2fm4mHpdKVJIjladJGTLZDgaPEfdGoY1MbasrCHoglpplrrgzDn6oHAEaqvQFFDUOOO40DXZNay1YhMT15sNcbfBxkv2+xteXT9nvR9Iyy1VtUW7CmIk6YaY5mw2kbLY0lQ70JYYIrpo6PtWJnF9hx8SGEc9e4hqztDNEUUxRw+RfuigPMKW9+j2z6nnb6PNXG4m1hOiJ8VADgMxGFJSFLW4JfN8cWDp1bM5RT3HWIcyhqHviX3HftURUyS2a4x/gs2fsqiWuOoUbQsihTg2+o6+22BUZvAB52pU5ahsg23mLH3AFjOacs7Rya8ooquJOy1FCcU0mEnHyIF3n8ZmoLs3yp8hit9FtfzcTb3+zeQYnZzOOXty1sKLToGUkH9nYa6SxYGk0KjkMMqLK+XOtXn7HqaB6aeb8KT6Oro0c4EfBG3Qu04u574TTvhiwXK54Gi5JPgbuq4nRdBqz/0HD1mcHNN3Lb7dcHV9xfX6mqvLK/ZtS8yZoiwPCAalHDmnwwT2MLYodVuIzBKacnxywenpOWfnp1irefr0MZvN+nAz2O22dN1AU9csFyfs9z3WaJ48eUrO8P777+GcZd+uMTYzny8gG3brXo59cmjfcPm0Q5vMkyefEXOQZHulWK/XpCws7em4ai2TB0bH+CSOTc7R6XHGGIZhOAR5PXzwiKwsj954g/V2g3OOzXYDyM2/LEtS19N6KTA4Z5kmHZOQrrVjs9nw6aefApl79845PllS1yXzxZHgSoxMGAXtNY77TCLszzgvD/eWnz6Pf9b2GiKG2/vE3d9NSJ67WJcvIl6mm75WRiYO47WYRtfB7f3yl9v+0//0f8tysWBWO1kAOifc9RzpvccVxWFCqrTG6rvv7za3BDhw4hRgCJC9lNASeC/cxaIoSTmhtHSShRzIKYqzLWtCysSsiDkQQ4QkgTwqBVkAp0RM8rgpDPWALWGUakPARxHl9/vuNoQyynsIPowhnJ4YgripQ6Tr92w3G1brNV0/CMbFOmLKoOV8iaNgHrMsTgERe4eBDAcBOI9O91kzY17XxKHj1avnDH3HfF6zWMxwtiaojDKJpp5R1eK63vcdKJjP51RVjR8CfT+QM8xmM5wrCCGIu7mZcXRyQlVWtG1LP/QURUlVVeQk+QApJ6qq4mh5JDkCRkKZiqKkKqWNeioQCBfckjDM58uDi13yYBRVXTOF/4QQqKoG6yxl4Zg1C5TS4/vtads9Q9+iyXR9KyFsMXBzc4N1locPHnJ2doYrDM4YPv34E9r97hCKKugqjy0kFDUlWUDnLEgecc1zCKZT4/7POYsLCQ5hVJMTfQoZTVnjQ2Dw8l8ZE8bKfDVlWXQTEybl0bJlxoVDlHbl0SGuFHjfs9tJ+FrGYdwMNQpFEjw1LtKVIqRE13t6H8hK4cNA2+1puw1VpXCFAjWw3V2iVeK0Pv4Vrm44fqjJUdENnv31gG8D999akLNjv8mkLvNqveXi/Iw//f4TfASlHJWtmR1rrIso7cXNbjLBR3KS1kOtIcSEAXxI9EOHUpnewxtvNmx2W/7pP/0z/sbf+QoDl7zxJQPJcfNcSVeGFqRA5weUKrk4f5s/+OefofyC/X6DztDUGX0mwaylVbRdx+VqS916fu29Y0LuaIctVVOxmNfMyhPW1YoPfnDNstEkn+l20tn14x//hLOTJVVdsbkWgTRnRVlZ3nhvyZtfcQS9JdvA88tLrq53xD7z4Q8HTpZznNI8eXJNVS/oe4/GYpShchVfevcrZK4pSzk3F8uKd959yKtXl3z++SVf+eq7VGVDypf85KNnPLh/znazpygrjo8vuHx1Q+EM6/WOxaKhrK3wO51j8ANVXXN53eLsnNIaYhZn3szNaYdAWRg+/+wpp2dH+CFgjKMq4fTsiNXqivV6TVmV8pxGAhWX8yNevbzi+tUNFxcXfPzJE+nYMo627cfgcRHHJ4Zu23UsFjXtfsswBE5PTkhjkHBZFsxmDdYa6XgLYSzABZRWzJsZsS24v3yTs+YN7s3f5e//zQVNc4zr51hXkLNiXt/nm++/Qww92/VLTqs3eXS6otNbLs7v8/zmCZ89+4iiEiHXuYJXly/o/QDkEYm35fj4hNnxgldXlzTzGevNWtrunWG72+CKgm49UAYZx6uqAG3xIbM8O8V3nnazZ7vZcXR0xM02sdldc//BBaUrWc4W7PYbcpaxcVnOiAScNTiTeHB+JPcuoxhSYLXacvbGPV61W+IwYAqFqYpf+truu5ayKCmrGU1Vk0JCqyjdUAli8mAN/TDI3MnKvlJZkfpAspCMdMj6EPAxCppN9VjiGCyrMNZiyxJlC7Lu6WOH6iVoeLFYUM9nlLNa3OFAjoHCZgqLBBcayVOpqwWlnjGkjgJLYQq0UhKwNwbYaQVFYZjVDQpFioJIcaYg+p59u8eHAYVDKcvg9xIM6SPZe3QWPFqOipAzyiecGjAxkMKA1ZbKKZwuJZQ9K0zUBC/mEmM1mUQ39KMIJWtprS1dL/MhoiaGjMYQQzp0sDkrrlVyEv6w0jglPPehGyidYB19zOSkcLam0CWlKVCuFvyKypBuC/2lrZhVBc4VtG0HQ0YlRRjCuGYQdF4KCauhtBoVBCsWvPCvjREMi+9b/NAxb0rms7mYMLIgcQKJwffEYSD2YbzHIya0lBk6jxud9VobyqKkKho27ZbcRypXUc0KCV6NmkCmKgt04ej9QNWUQKJ0hqYs5RxV4FyJMhZbQD1rSEk48gBFUdO1gbZbE3NPU1ZUuSTTU9Yl6EQfOrBIR6MyFBn6/YBPgUzGlRatxjnhkEhRnM6t3+PjcJANY5JiqQ4e2KOVpsgF0Y/5LRaiiUSb8SqglaB/y6JAKUUIA1UxBrUTQNmDE/uX3ao7qKdJRL/1jqoveL8yU+j54UfjWuKukA5qFP4F13EQ0ScOdeJgQkkpEaJ0XqexEyGNgYwxREG9GMkEmlSvnDIp6RErc0dFv/OeJiOLGlnX0QeM0hSuJnjR/MTg0BJCGLtWFW3bHboRi6KgqkpevHjBixcvWK3WPHx4T8ZbxHzjXCH5N9pQNzPpoBj3w+XlFX/8r/81P/nJR4efTWuEqbO3bVuKQrpLiqIg9RGlwgEpq8bMxxDymJsIVV1zdn7GptsLDsk6zLivUJLjFIInxAGlGMVzwbKIhhjvmI/z7b5TExpZ8E7SySEi8pj9Skbdkn4QU4O1gtDR6nbNDSOyh4BCuPt6YsAofesbu2My01PRQ6k7HQVfyF6a/pdf14AOTrGfIaSLiH779adsjWo6Z25zzcR/rhAsUGKyQt9az8bPcChKTLtRjtFYwUErCFGhkh7n7nLuT2asQ0Dq4SJ87XC8dq39RdsvLKIvi8C0C2SHjq3QYw/8Fy/suw5AKVLIu7vdCXzh8eOPpaQ1HrC7P1aHik26U4WTTd8R7G9fd3o9CQfMh8e+xkcmiyODxMQKQt2K/AehZPy/KYN42qaAh3RHgFFaWrSsfoBNG7rVMwrnSU6x26959eqSH34a2AfHb/y1itI0hKC4uV6R6kSu99hig1IOQkDlmuQ1yRlcc4TWRvirOUPoyTGx9xFnDPR7zuczfuc3vsEf9APPt4+JKHof2UXPceP4tQdLysLw4rrl7Ljhy++/yXIxZ9c+5vLVJeWsZrZoOKv3PMkt3//jP+Dl06e8/x/+zzh69AgzhQXcOWw2Jqo+UMdAo+Es7DlXa1q1YUVHZ4T7eeQDJ8kTdWbtDH0ayHQMRqpppYUGj1MR5xpcrAm6ot1f0l1+znrzkv3Qcr1uKdWG5mxADS3Z91htmS0fcXPzgheXj0e3TInSGRsSNkR6PLttxNgF5fw+5eICWx+Rq4bgHCEmdFli58dU/iHWaYxzKNVTlQnrAs5ElO/RyqC8tIVorbCFozg6JsxnKJUptCQoEz2+25FCj9+1bF5+xtC+RPs1uf2QZhGozQ11tUSpgpCgiyMruA+E3OOaRDISqJXCQAyelKCoTlFuRszVL3IZ/5xNxNDp31ORaSwdHW7Uk0laTTyjO6LqF7efxT//udtUgRu/SsCJh5FbGJO098QECRFVQKERsSNjMKokuRqli6mkB0wtStN4NErsd4TOCQ9gjIVBjYKitGHFmNntdux2O7bbDdYaFss5l1fXDH2HMRJ613Ut6QZy8uzXV3QjGmEYu0biEOj7NAplDSlNAYcS5PJ6VVjed1E4ZrM5TV0x9D1PHz9FkfGhp3IFXb8ja+j2O7q2R5sCXzZ0+z2LZklTibvsb/yN3+If/sOGtt0SYx7bpwx1XaDxWA3b1Y43Hr3N/+p//b/gP//P/zM++uQHbEa0Q9d146Tgdp9OBZQv4hQO+IXxscZYUkq8ePGC7XbLkydPuLj/UJjGIy7COUGxzGYz2v2M2IqDV7jMgsgQETBgjAS9DsPAer3m+voaYxSusDRNdci9kL/Jh/epxqLpL1zg4fWbqOKnH/8azkXx2nl2l4n+8wV0PQZjiRtdQmU0I8toZKL/AtfPX7D9lW98mRizVNkzWJXxw4C1NQtZRUk2iJ7upHFkA/rRATwWq7IsVIIX1JGKHX27ZhgCMY58yyGghP9EP7T0fYexGleUhAQ3qw2vrq7oOk9ZVNKJ0bb4viMOPcEPDD7S9pF+8Hg/4MeAy2HwKKvRyhz46GVV48oZIeWRaW0oRg56TBJaZoyEmGolY8pqvWa9WaOUYrk8Yj5fopTGlQXWOJJSRCXBUVXZiCjR9ygn10JV1Thnx0KR4vT0hPPlkmG/hazY76XT43i5EEFbVeQkhauqqkWsVjvQmrpeUBQFIewOraHO1ShlaNuOrpMFRlFVdL04vFPKFN6zb/fkzMGR3Q8D+7Y9YF5AnIybscA1XZsiRCnIY6BrjAcRBqAsqwMOpu+HEbtSooDddosfem6ur7i5umK9XtH3e4IfsEajrbAJb9prck44a3HWUJYFp6enrFerw8Td+4At3CGAfsK5yNiRDteJkPzk2MaYhdM6XpHAGIg6tazLWN91vdw/vR/dlMXoHt4fClvOSdhc13VoZLHnjBFxIrSk7ClLR103hCCIHaUiRV1RVUvS0DG0N4Lo8cJeTMrgY6DtW3rfj5iMRIyelAfAo9RAzj2ZDldanHs9Z+LfdDNaZq/Je/w+0swaFrNjnj++YrVqsTFz/+iIbptYNAX7oBi6LMFv2VI4TVEqYh5QpcH3nqGXwrVWiqqyZBJZerGxTnKAXlzuOD6es1pn/vAP/4R3vtSw227IaYePUJia9bYlpchsXvHg/H2+9a9+zLNPtwztmrferfmdv/FlYmp58ewabQZ268DL6xeEDMdNyYsXV1T1nJxnXK4C6rzisr8m+4GmaRgGePDolBRvuLmW1u4Xr66YlTVVY+laQS9Fq6iOwdsrogpknaisReVEDpq3Hy15+bzn7HyG1hpbaAmurhKuCMxmFc9ffMxbX5qjdESpgsIuaKoTzk/mrFYf8dnnN7zz7ptstgrrSnqv2O4zM/J4H0vc3Eg2hTGG6+tr7l3cY7PtJU9qCGx3nvPTE15d3VBWmvOzCxQWZxOb7cDx0ZysDN73+Jg4Pj6WLp2k6PsBV0hh3hjH9fU1s1lDjlDUJX0/cHx6zHa75/howermhr6XLouu71Bj3sRsvmDfd1inqMuCs+MTdtvt2GkmHTbO1ZiyYN/31HVFt99zcnLEfrdjoef8+ld+k0ovqVjy/lsPaFtPYSvQgaQixpaQLHnw3Lzacnp0wd/7u/8eP/jo+xQzxWa3oi5K6qaUTp40HPIJAOGdj231w9ARk8dScHZ2xm635+XLa2LwVLVc92SZ8+13LUXT8OrViqPlMa9evCL4yPHxKTe7PT734zWcadsBsqLttvR9izWWq5s1Q9sSQsb3HVVR8sYbb/Lx48/ZblpmyyWbth15wprt9TWzWf1LX9ultcybeuR3J8gelQMpeFofiQqMcsTkxy4cSNqiTEbFjAoQfaQPLd4PpJgIgxdRa7DELPMWYxwGS/KZwhTkSkuPaYoU8xnzqkBpRR9knPe9H3EblTi80ZSloXA1qYcUMqWrcNqis8ZqB9oSSJJvpCTwcsJs1M7ibEEbNN2+ox860qzG6BkxiM3Q2VFYUYpuGOiHiLKgzMB28JRKkRRjeLPDuQqjtDjzQeaUcSA5jVcZq0Ebx7xqKKo5+/2A91usU6AsGY+x7rA2UErm42VRj/fsCEh2EUFwg/usqaoMyNiirSaN6AtVGgkzDIkB8Eq64ZJWlHWFUYbdest2uyMHGNqBpDU6QllUMHhy7+kHcesnZYWJbyVcMCTPvm8JXWQ5hzcePSQMA8+fP0fFjLUW4oBPEgtptITsWaWYVTNSCMIBzkkQcklT4oi6QhcKbQz1rEBbTZUsfQjoIlNUmqz1iCbTuEo6GZVWlK7BFIbBB7KZ/PqJopb9l0fXnasKXFniqoqQPCZETGnZ+Zau34ORMdQaB1kzdFNw6a3hykc/Yi/1KNKPc3ODaEFKMQQvawhrKawjedFPCucoqhKVEj4rKTQbhXaQtbDPI556Jti2tt1jXEnTNL/CnVsMUnfNONP8RX50d84vwuhkKjn8dJzLfNEtrLI6dFBPRlCNFMblkpd1dooJnzwph/Hxt6H1gsYz43+aO+qtrFuTCKrTvGva39OaJiOvrTKousAoK2sEP9B10LaJoRdcnnR/SidhzknC2xu5zj744Mf8zu/8Dt/8xjf40Y9+yJ/92bdRDpbLJc45vvq1r/H2229zdn7BKBKy3e35F//iD/joJz8ZQ4AzV1dX7Pd75os5vu+JKXJydETT1MQQpNPTKrLRDINnsi9N6EYxBlmur695/uwpu74ljgYy6wTZl7Ognfp+g9YzTk6Ox6JlPOhmrwvT0/r0DtFj1BIPAjqAHlefWY3UDikNGa0PnHCVx+DYLN/nUcA2k5A9CvGTzpAVB6T2lCnJqLELdlGKjTmPc/Ccx6/jA6fzT91+f3ebtOJbxWA6Z+WxKTOipqY40un/7eE7ndPYfZFgysDLCglLHfOmRnxL4vY6UIw5eiZTWEXKmpinz684IJKm9z4JyEodDOIyK/+36ER3YX3YUbIP1R3n3q1oftuFMsnl6nbvHaoft4KCvPfxMeNFqtItWx2lboMDx5+l18YWPb6UOjDKR9n98Nq344vsvNdE+zz9eEwZVpJjPL3nfNi5BznxsM/l93f2idLjoAIYh3INpXoTkwPbyy2DH1itN9yser7/medr3zijKhxXVyu+/5NLPvzsBbZp+O/9/d/m1C2JPWQ/oPUp5AqFxmhHzBk/dNDvyaFn3/esr1cMnYgRR4XlzTcv+Hvl79CHyEePX7HrAn0fGGLPg4s5D++d8fKqpa5LwtADDRcXJ7T7Z1w+fc69e8csz5a8t7viyWXHh9/+M3z+v/HX/qf/G5pZeUD7TB9e5YwdBo58y5kKnJsWxZYtW7IJFLbGqUjdb5mxp0NaCmOOhLwD1aNUZvAdvU4MdgZGWtaLckbYGlY3ez748ENCDijtWJ7uqMMOPRTEKOKNtiXZFNxsWl6+fMXFwzcpXUlWifXmCl08QBVzdDWjPr7HbH5K4UqS0rQ5E4wm1zN8CizsO8TulNBdM+w+5qS6x+zoHuiCGDtQJSoqkt8SVCbpxeiwU2MwZSZHT/QeHQeMygTfov0Gf/VtVNpSuURTHsPwgtwG6upNlL6PywVFYdD1gn3ItKsVoR+wyZPiAFgWizPeff9v4vuBIblf7EL+i7Y88qgUo6uPcXAbRcA8sghjRnirAacnEf3nC+g/b5OK+FRjham4lcdpS4iDjAFZJmAxSyL83fRwYTln0JYscKNDNRymSUG6vWnJC48f+/UigHOWEApi7EcB0Uu4jVKsbjbcrK548eI52+0GRR5d0uCHTsIrhoFiZFiGMLDdbfDBE1IipAAxU1U1fd9jrR0nDbfBybdC0BigrAQ1cXNzibMFy+UpoClcxa/92nvc3Lzi2fPHhDCMbc6Wm+srtBaH2XI5w1jF/Yf3eO+991jdXNMOIqK3bYcmM58X+LjnrUcPub56zH/2f/rfsW9f8PabJ0R3zONnr2jbFp31KBzmw7GbDvPkZp04xXeZw5APTuYPfvxj3n//q7x8Jc95dHIsXQVJkDy9F9FO6QkrpMf09BF1M06Ghd/rD87aqqqYz+eji9cd3qM45G+FbGG+5dui0M+9JL5YIPrpostB/Mu3qes/SzT/edeD7EcjOCEsHArTjKYBRf4Z19gvuv3X/59/Qk6RFDykyPXVKz79+GPqpqaqG/bdgDKWqm4Y/HAIp2zbdnQ5C4s754wfBL/Rdi3dfsd+u5XrKwmb2VnHyfERdV0x+BZnDW+88ZBHDx7iB88nn37Kj374AV0/cP/hI+pmRojiEDGj62W32xNzwjpHzuK6yyDtxWWBdQ7M6LJu5pSzOSFMuJqCpmmw1ooI3PUYY7FOCjmdbynmDcb3pCCcVoxGGzc6Se5ObjPRDxKsawxOi2vIGenOqeqK+XzB8dExzip2u8hAJhpDF+HFakfhLH0UB96w26P2LWF025Nh04kwn0bOudIau9vK44cBDezbLc+fvcCOAUhaa4zW0oIcBUnixiJBiIK4Kg7ObnFlK6WoqnK8nmRfWeOoq5rCarpuII3O96PlghgDbWsIVUFZlsxnDQoYBgnU7Ic9bb8TF6uXBYkylozCOIPLim6/4vJlxg8bFvM5bdthraaoKrabPSllXOlEfI7CyvU+HhaQfgiEIO9TxgQt4+gwcs9Hl2Mm0w2CfTK2xFpLZgwKM5rCOarRvR6jeHyqqkbpsTjoPcYkCldSuIJ+6PB9T0p6LJgUoDz9sCUDRhc4bfBj11ZIwhbtYiARiFlTFFBVir7zYI4oyyXWFCiVyHGPYsdipigKI/eLX2E7mle0e49ylrooqco5Tz+7ZHOzx5QGlTVf/fLX+f73v09ZaILxKJVwwPV1Tx/h7Xct6IjKmhRLrl4Gtltx6C2bhrKyhOwxxdgdkgPozHq9oWsV7T5jiVzcbxj8luWDhzx/umO5PKKqDHU149t/+pjHPwl86e2v8O77jgdvWE4vCvxwirUn+L5nP9+zaT+nahTGwmxe8eTJitPjBTrNefzZFp88pcvcOz/l3pk4hm1Z0v/wMckoZnVJ6gOzZYGtM+f3TuhS4OR+QcBLa7pRaAy+tTz7vOXdNy64+NqCq9VTfN5hnGZmDX3byoIqAXrO8+dPefjogqZcUJVnpOCI0dPUJ/z4Rx/y0U8+k7HMWHY7z2JxgjWJjz/+BKVgNpO5RF03dNcdV1c3LBYL0B37/YajoxO22z15xNo5W0IUN3FTW3yI3Ls440cffEhZVCMyq+Dxk0vOzxsKV3B2fspnn30q06qkODk+lW6AIXJ+cQZKxvoYEikG2nbP+fkJq9WGlEpWN2vqRUnX7zlaLnj18hXWGKqyIIZOzh1nCSlQ1dJZVxQFZVHQ+4Hf/o3f5vz0PkVq6Lc9OBGsSIGYBlxlIUsnyxAGzh9ckOlouw33j9+ky2tWl2t0NqxXN8QYOD4+QxlFSIHZbMnR0QkvX76gqkrW6xtmTY0uCq5vVoIA6eV+nILh5OgUNYY4H5/MuV7doFViff0KnTzOiIlq3+0pKukCXSzm7Hc7nFGU1kCyGFtS1XOeP3vO/OiU65cvKOuafdux3/cMIRDXW46OToTH23tigK7zP/8C/jnbrC5ZNDU5RsF8+h5dWDQBnyNDFNNGCIK48T7T6iSd/90wBihq4t4zdD3WaXIIBBRdHwhJjp3RjgFBmlnrWM7qcR4ic5uYAtYYrAGjEkP0WKOoXIFXlpgUylUUrsL7ni5klJGMF5XEDV46C4UIM0O7Z7/dYMg0hSMOGXzAd4mhHejGa6ipQWdBqVhrUaXMqze7PX0fWRRzCuNIw0Ay0jFXWXHuF2WDcyWlLWC/YX+zYt96wgDGGZQ1lNaxmB2BcuyiGKO0GhFrKaBGF2RRSO6LGt3JMkdIaKVxpsSgCP1A3/bkqLGmEN3AjB13GZRVkg8UPV3MDGRMjnRhwHQdKgpmTKUs3eNKU5iCQjlK5eiiIgRZRwwkmtmMsnLYwoJKbDY3+F7mbjc3z9nvzrh3do+hnfPk2XOS1xRKk8uCaAMGTWkdhTYsqoqhkzXPxLStyorZohH8ilXies9JkCFaHbLujdY0VYnG0ntBGxlbHe7F6IxVSfII2OJTZN+3Mp0OErAucxMJCDSFxQRLIDL0EhDquwFrJdMiDGnEH8rf+sEDAa2cCIBqFNFVTc7FiHPJWK3QweCDoSjLcX2YUFajCis8/VFfUlbhCo1SYQyXFXRvkZ10oueMMwZt/nKR7edt1onYf7vd5uxM38sma2RZx3LnbyasMV94jnF9c1iiq8PT3F325JSxWZOTHX93m8c0re3FkXxXClWQblnbIC7m6eUUjKKzHqkOEWcNV5fXPHvxkqo8oR8gRk1VzZjPNV2X6WymLDMpeYpCofH4bsXbb5zztffe5M++9Ud88xvv8+kHJclYjo6WzOZz7j98wNvvvCXGliQfsN21fPCjH7O6XqO1Zr3pGAbpTk0hcLSYizIcQEfQyrB6dY2bVcxnCzZpJ7tuRMREZYl4MI51O3B9dYO2BluW4yItSDcWEuZQV3NUNoDCVU4wQkpKSJOeqOCgUQC3+mWeCiGg8+v4FJTgusRkEjHKkjVkLboMKJIKZD0eG5VQY+lKRAY9CvHj8+rpvMpw0Hs0E1LiEL6psnDyR4zr4U2pNOZcCkr3wEu/FXvJd75K3o6WHKrxAwu4ZqoeJHIKd863iKCDp6AAeSLp5Jf3kZV0ANylmoyiJJpbbnoc19SHcz9PAn6aejTGv89kNBlDzH/5uvsXnr5vXj55XUzI+vBm78rooiS8LpQz/Xzaq9z+PskVd7jwppZ7AKMMyujXn2MS1dXtc0+hYPm119KH96PyBL7PjDySUUi7ZXtPZ/VU2Xxd+H99kOOLv5rGJ3X7mjHXeLvEUzJ4Q4ywXu+5We1ZdYrrLvDVt87o9nv+i3/yff6rb33G7OSU7c0Tupj5H/3Hv4vxCT0MUAWULtG6RCnHMASGfUe4eUbpJADlxZPHfPDBRxTa4RRcnDZcHB/x1TfPWW82vFzvaH3gcrUn9AvOz2dURc2+ywztnlcvBEtR6szVq5esnh1x/pX3eOfNTEiK5dERP/78A1bPP2f2a18WN42+3RHKSVtJPQw0cUVWO/okSBu0weaWIoKJns6v2OXApu/Y5iuyXpN1YJ2eY8yCnXubXh1jigWlsuiyobUzdn3B42cbjAnMmxral5juJdopEpbY7VEp4vdbht2OZ7GlrBvu3XsIyTH4jpkLzOtMtTxmvjxlXjXMDejQsqWisJZVWaJzQ9YSOZf9lpw92iCTI2VxBoaQMCkQb7YQl2Rn0KamUIGsI94nqbJ2LTkO4tBMA6UbqE8sdX2fQid0HmDwxHSDMguqukElR4w9JnlSPzDsn5OckecwFbo6pWjmqLNHgvWg5Ffbbnla+lD3GqvKX7iBpyjhI3kUsr+oD/6iAvrtWPAztiwBuyGNIjpSmY1ZEsETEuyhxi6TGBUqdyKim3zLlZ5G8LuFuzsC6N2v03tyrjygD3KOdN1+DCCREM7BD6xWK+GQ9gN+iDTNgr5vcYXDGYcferbbDV3XEvPYzo4Ee6HyyNgVV4bSk7gvArQeQ0UPQYZBsBnzWcWzp49pmiV/9/f+LtbCD37YM28WeD9QFm6sGovIUZUlbzx6iNaZ2azmd3/3d/nOt7/Dq6s9zcxyenomwlkBVlne/rVH/CTsuHr1lPnM4KxlGAMJUZMgLq5QQHhiyoxusOE1h8SEd7HWjsWCxHw+5ytf/SrLxRHalnz00Uf87Ue/K23GwcvCSXrKMdpgraXr29HlLoK9uPdv2xonF0pZVhSFOzjatVZkJka++ql7yy+6veZa/4LmPrVJytck4Vjj4+8iXNSd+9sXn1sKRiPORRsyZpz4wFQgTrw+8f033f6P/4f/vZxbcWBRVxwt5lxfX7HdbtGuxLqCxfEJZxf3WCyPxRHkLPt9y2YjmJ2qEoc/RhGGSO8zmzaw2vb4QdoerTYU9YxmeSYtjUpRVhVHZ/e5uPeQfrfj+uqGv/LNb+KKmqQUm11LVc85Oj7l9Oyc/a7lRz/+IS9ePCYrueZDjoQYMM5iVSHzwAQhK0JW2LG1NKZM9pG870YMSMLHxBB6cicIrWwSjHOKfdthi4KmbkhxIMWARh/OL2000Q90fsCNIpE2Zgxg0hRWEu37bsfW96x3G3o/0PYdm912xDQpcbJrRdzsiClitKYoRNDetRKWJGzughxg6wdxaVtDVZTEODoqXCn5F7U4G4dBOkPKUtAu1lp86A/hZ2bMFJjCNoWprg+OdKvNIY3+7rU7oVzS0fJwDU/4n9IZWg1nJyc8f/GMfuioVYM2Y3ErRYzS1IW0jg5Dx+PPb8SZpjR9PzD0wuDUSri4zhUohYj7MR3cTzEkwf1k6RBy1gqr9hA8Jfc8KfrEkXEoeB5jDKb3o7vmthicx2CjEDNmLO4Zmw8cycFL+Ju1JTk7YlQMQwYMZTmT6z7CdrvBjMFs2lp8gs5HlIailAWetgWbTSB2BmMKiqLCOUMaMtZkGPMCYvjlQ4MBSt2QzJYQ+tFVJRxolTSxFZ7p6mbFgwdHvNjtyEoKDu+9c07VWIrKU9QdWmf2W3imYHXTc3RSEIImRcV+N6BcRlmoascwKMqyoutXkBIqWJ59HEl9y5e/PuPhG/e5vPqAx89f8ujBEdtNi7MN/+Af/Ptcv9rxve/9Cf/yj2+omswbDy94eO/L/Pbv/Dp//K//NZ8995yXDm0ztkw0M40rO+m+ajR1KS7Pdbvmw3/1ikf37/PNr38FbRZ8+PEHdP0OFQW5cnpSYaoBHT0hinNRK0tdzinMnM+urmnKJXVxxur6GdtuRTXP7LpE5RqKwqFSwhSR9brj1y7OKe0JhT0hR8Orq2d8/OmP6Tz4mFBG4UpD13tsMqw3PaubLW+/tcRax+XlGoDjY41zFUo5fEQKObagHwLbNlBXgn16+vQFRwu55jf9brxXLlguKqytWK83vPXWm9y7WNJ2W1IaxGnnA3VZU5iSqqiZzSyb3Y7tth1RUWkcgwpylo4fazXr9XoUgSW8se1aSluhHCQTOD05Zr/d0g8dZdMwcw4fPKfHR6iYhE2tFUYpZk1D7g1kcX+G0IuYyAzjNNplysIxBOiHQNmcUxVLXt54rHKcn5xwuenZd57gB5n3JXEq9n3PbtdS1xX90NENHYujCwpTUZYlD+9ZgpdQ86ac8+TzJ5RVhfOJwhbMy0DsWjaXex6+ecHed2y3a5aLJaYqSGFgVpY0dUm3b9n3Aa0r2jZQ1Ufsh4hr5vikuH7+ihBguThhs2s5WZ7z4ukL6rKiVwX7zf6XvralrX7qZECCKrWM0SEmuiHIvh16itKSS0fOHoMmD4HoO4wuGDpP1/W4UgTRGBNhwmSpwKB6wpjJ4eYWhdy3tMl43zP4jq4PGKMorCXHgFJGHMtjZ5gfOuy8oC4KfOHQ5DEEMZFDRDvp9EEZojZ0YS8miBRJQ0+ORgpyOwmp9a2nLwcJv9SSseNDJKHQxmKcpq4aalfjU6YsNDkryrJGG4vOULqSwjl0rFgulrR+oIs9OSrQGqtLcoTdfiNoB1fgikICTYtCTDyIQQSryFkzDIHspQutamqaqiSm/tY8kkUMb2ZzKWyP3ahpNHxoZYljDhIKCTzFkPrA0PeorARfozRNOaNyDovGYaQAtmjY+gG0oXSWqnCE4Bl0QSoWDL6n3a35+MMfUyrDyeKI/brjer3GuQIvFlCctpS6YDmb0xQlly9fsQs7gso4V5CNI2QNtgAzMHQRv88S+h0cwWfKXKKGYpRBLDoI8zoai7HSfZNTwlgHEYwqCMnTbmV/lUVBRuPJRKWxppMCosrsui390JLJGMQdHAP4PlCXM5ljpkDXdxidsFbWeSkHYhKUjLEWn4Jwjp0lKzCFpWoaCVFXXswGRhOBdhiIOWGsAp3IRAY/HNZj252Iq846KjSD/+ULZCDvb9ruolnuzsdut1vB867B54udtTKf+8sDEactRSdV4kkU/ynz0E+9hcPjftpodLspU+BDBiUhv89ePOGr3/grrG8kPDJlz+blmpvrNaubFW2byTkSU8+9xRHz2vFrX3qHJ599ytuP7tHeXLO+fEFpFJQldV3xa19+j9PzU6qmJkQRX7t24Lvf/jbb1YrL5y+IJmOt4fLyFSkGLs7OiENP33UkM/Dkk494cO8e9+4/4OVmjRnNFDHKWDMdC6UtIWsihnJ+xBSoGZWXDChjqIqZZMvklpyEJqB0gXG3+JtD0CdjkOcX9uWoZ7/2/SQ0ChbldeytmO1uNYzDUjHnMTF0smBJh8IkSKvDk2fR0qeiiZIQTyajWZ600/E9pnQrlh/03nz7yneKK188L/JoEBzP5NeKRXpE3kzubzGGR9RtCqm8zkFEV2SVSCqN+oCspQ8i+rjHjBk/V0YM29w5h+UIMjHXlVJjppxI6/oXWHf/wiL697/1LXmjMB4UqbIIB3LambeKmuzbMY5wFLhvQ0YnYrHsyHQ4IdQB8G+MoiodVeGk7Wg6IFozMWNlJ3J4DaWE6YrWB0F9es5bjczcnoBaHV53wrzcvtdpmyoqdwQ/BZPl/vYUuN0H2VWohSOUJT4L3213fcnLy5dcbjq+/zRgSsPFec3NruPPfvyKi4sjemXIRL71p5/y8LTmt7/6kKZekrEkVRK1I2VN23na3Y68WuNTzzpkPvjxx/yzP/whnz17gdaKe8cN33j7gntHDYqEdRafIp9cD3z4dIWzmbJ2NPOS3WrF5WrLm2+8xbvvPOL+g3Pa1nO9TnzpS9+kbD5H/+QlbrEkvHpC/tJ7TBUhyTjI2JQgeXTakNMr9mFDYORIkSF1xNCxHnZs4w2b7orHl8/o0w3HRwtMVbD1l2zDDWF2ii09lR8oDTggWcuitty/dx/vAyfLJY5Iv3qK8h6KitDtuXnxMcPmEh08oVdcvXxJYQxVU6PpmDcbZrbCugCpQ6Uts6RplKVJM7Kq6VJmGPYYv8PlgZI93nQwPCO1BcrNwRQYClzuKdmj9YxCecqsadIeZzWthptui95co3yHLh2F70jdM2aVYVbLol/pihQtvvPs1teEADFFNqtrLq/29H4PeUXSFrt8B2UXpL6FcI/Y97Tra1R98oteyj9zy2FMFFDjdc0YGqoySUn4rnCFFSoFEhHcLSrj7nb3+5/GPL32yL9AQUcGbKKI5YyhvVKaBRIqT3HBegypUOMCS4Tvu6Ll7Xu8I5zzswR02ax1WBuYgj61McQwhsYpjfcDrjAMvqfrOo6PzigKacsbuo795obV6oZhGECJ8B9SwGCEjT/0B9zJ9N90U/jCUWFChRgLV1fPOD5+CMnStQO/8Rvf5Ic/+gGLxTFdt8H7gFKOGDJdG3j3nbf5xtffQyGTiN/7vb/Nd779Xf7RP35Bzj11XRM8NPWMFD3t0IGJLI4W5BR5/mzDJm65vr4GoG7qUVTTtCM2Qm7F5rAfJ6F9arWePpc4YStubm742le/zvVqw4sXz/n44485Oj7GWAloCTEeBPuylAUy5JFrLvvEGE1KMPS9MKJ74a23+5ZuVoMSVMN007876fxpGv5fvH3RTf66A+S2IHM4l3n9XHrNhf6Fie7df2ulydqCtiglzp2ckHwO/Fgo+jcT/+9udTXDGEUMBXVdcnHvPienpzx+/JTtfi9u7qKmLCq0MRSuwDqLMaNLepxEfjGpfCpu5CxMf+vE/WydI8VwCOgMPtD1LdYqjk+WIlIWNdo4vnl+jwcP36Ru5mx3O548ecqryyNevPiE4P3o8I9oLdf1MGRyDsJXR0KQtps0tm9LN45gSPpDMGUIgWGQttGyKWiKkp2x9PsWX5S4C0tZFOJKDgNlUTFfLjFWE3PEGsvx0RFHx8cUhWOz3mCMYb6Yj0GeCMKlrljUM7bbrWQgREHJ1HWNdY4weCkGGENZiog+DMMhLKkspXg3DLLgts5RFgXLZkZT1zRNcwhVgrusSw4O9ZSLOyxxufYm9vh0vk4Mcas0Qy8hiQcRfgz6BEbhW92+R2spnAjXi8WRFO+CFFONkVbpvu/QKMpCnPFtt2cYPPu9oGdCSORs0Fi0EaH97rB32y1026acUh7HkjxOltNrwv8XcVIHzqG6xUsNw3DYL5PIoZQ+FC8YsynSIUxq7DDy/aHIUVYlkOm6ga4fKI241awRVitIp1bOGudKikIWi2EMx7ZWkDHBWJytSGEM7sy/WjjZft1KQd8ltPZUs5Kj44qbKylAu0Lx+MlnvPfefebZ4oxH6Ug2L6nmM6omE9lSlobZYsnFxSlHxwu+9+0VcRfZ7rbUtcUpRd8F2rYjeChs5OjUogjkwWG0o9t0OKXZ7q54850jvv71dyFkPvnJCz786Iof/tn/G2ePaIeWaqHYDgM/Wr3ggx+uMOZPUTYyX1SAdFWcnlWUZWDWlERv+ezJijxiDMra8e77C55/fsU/+q/+W7707lt87Wu/xmeffELwgbbtcaXFp5a2C+KAGgyVW7B+FXn25CX9Dk4XDY8/e8nFgxKvC9A9dVXQ7ROzssKYnqELdMPAxz/RXPz1d1GUZJV5efmC1faGdsg8euMem/UNxmjmzgoCxBou7s0xxrFabVAqcXxyhLWOEGTh5gfpVPODZ7mYc3q8wNoCSFRG0+5bUJm6KXHBMgwdSoug+uZbb4wuSUEGlWUjRf+mpt8lyJr1aktRlSyPliitabue7XaHM5ngB5TOFIUlZ0VdVzRNxWZ7jULh/UBVVNLtpyx918v4nhMX98754CefcnZ+zNAP9LuWh+cPefr0CW+ePWPDljfvvUOKgTAMbNs1StekaHBloKhLspbsK3TJrh3YXF7SzGvefuNdfv+P/jGnD2e0bY82e5ZHR1yv1hTFQLu/ZrPe8eiNB/gQhH2+7Tg5OeWTTz4DFG+++ZBXr15hscxmR7iyYLtbc3I8pwSS93z53fuYouTy5SX3zk9JIXFyfMx2s+Xl1Q1ffu9tdN3w6tUa4zIREY8HD047KYKmTFEK0/r+xZJ213L1asf5iSInTWl/eeRDwjPEDmcLlDWS8ZQl5ylFQ/IB33l836NCpDRWOLnOEmyi3e7JXjBrScEQAs5ZlFWYxFg0Fpa0VgqrDcPe0PWBxfECZ63Mwn3H0O3RCmpXoRLiwk1yT/HdwK71WGUpmopidLwn32OMo3AKciQNiZQjsR+IXs75PEiEXughB4POghnxnWfY9xKEbQ1hGHNQyGhXMCvGTiptKZs5KQ/iQh4RLDkHum5D24HuB4pZw4k6Y9e1JDJ1UXG0WLLfb7m5vqTzkbKZUZSWlKNoOTljtYQY+sGP9wgw2gIa60pcWZGGKNz5FFEpobXc30yWMM8Yh7Ewr9DWYW2BBoZuz6D6AwM9xUCKmX2UAr8uFDF78iBFdWNLFosZtVG0XYtKnnlRQFFyVC/JSvPq8ooXL56xvYl89KPnfOmtOZU6I+8j/ZhdE0IgWY2tC7Ke0bawvzEkPUdbzdBHNr3nmhVKaWLM+MFiyxpVFOQQwQ+oXNH10HUdKvUUhTi/d9d7isJBThiVqauSwURUtJSmIeaeXbvFKocC9ntZMxtlUcaOeToAEWM0tWtQShN8xFo3zmssVjv6vEXhuTs3b9u9dOYaDnOpPIapaisiXUiBqBPJikAYvIzxE2e9H/bkLN29ZVky9FKISilj5pWYG9IvPy+HiTv9BbMOt4alu5usbSdh9fV1zBfXFRLI/jOE7Z8hdqcohY7xif9CARRG3eygld6+l5/12Jw12kJdz2i7PW+//Q7DkNhsOi6vPE8f3/Di5Yrtdk/fB2IWZApkZvMZisRnn3/OxcU9/uRPv00YPOv1lmZ+zLvvf5033niL9977Mm+9+Q7z+RKjDc+eP+M7f/Ydvv0nf8pmuybGAW0Um/Ul1iZOz09pSsfL7RX/8f/wf4DVmmdPH/Pdb3+bp88+oVqest1uWSwWdAecaHitwKHH+fswDPR9J4gzFEYZQhwwWjqhQxho2y2uOEMpjTb2jhnqll//+vdiELpr9n69YHJ7JGS7Qw2QJehhjZuTZEBwZ42q8lSI4c7X23+LTJLHzoKpYnIrpBuymJgOsuj0R6+fl/nO69z+Vt5DJopec1Cxb80tB17w+J7kfIzjC97NwBy12/HvJ92Ywzr99vWkW+RWNZ/W0vKxRgPcrRrNWKdgyjj6y7ZfWET/5//dv6KwiapQWAXDoOiDIBOSgn1I+AQGLQtrhKlIVoQMfYIQoXaKRQUxKXaD7CjnFCEpWi+ViqQU2irO5gWnM0dpNYaMUWC0otASGidtFBObfRTRjRE+rrISKGEUGHUQ1xkXZ+K4HQHzSsSAQ5ifMsKtT9KqcBDqtQY9ivLToc7qVlNXCu0cZnGKe2vOca3I3Ra//oTYPiPnNT5v2PWOs+OCsk7cf+sr/I//o8z/9f/1L2nUwPlFw9Uq8Id/8CHvP6w4PbkHyhFyJGaFj5k+JLbbFr0f0GHH7//R9/j9P/mIT59eS/hajuy2Pat1x9vnM45PGmKI9EMgpcim9YQQaSjRCsqmIg6J2cW77OsLHrzZMD97gxv7kKuTc5bLZ3y1+dfsvveYp9efsF3/VZKVdjWdIjYlln3HvFux3z8hhsdkswFn0XpByjJB9z7Q+Q0hXbO+/pTPP/+YwUfyG1+ims/Yec/eHJNtQ+oDKq1wSqP6HewuUbsbzqsZvYXzo3sUGfz1C/zz52ANMQ906yvCfg0+s98P7NaPef75Sy4ujnjn4TGmsSgb6FAMyZCHGwqbMHWJLo4p8wK967E3L7GxE3F594Qirwk3z1lvnmGrt3DVCco46Dym25MLh6oqYn+NcwOVBRUyfrdif/NY2p82kMKeuHnOzl3T6HPq5T1cucT7zNXmJZ8+/hEX9x9inWX94jN++O0/4fR4ztHCYcuaGDNh0KTs0Yszos9sPr/CzB4A/8kvejn/1BaCLPT1HeFRBmYZ1FIO+BCIQaOzx6p45yZwd3D/y7dDxTQzDqB3gh2YbuZaGOhRHAVppINlZOBLY4K8PE8E5UH1pCztjBi54TONCyojA/QhUeFn3vynnwl+RBGC8Lq7TgLVjJXr3VlHXVX4IdK2e/p+oK4beS9JOMYxCectBglVMWN4YxxFqilAZXKHTg5Q1PQ7PTqvhRFc144HF0tuLrf8+Z/+S148/Zjf+o3f4F/84T+XFkrj0KoghkAzq/h7/+7fZ3l8zNSd9ejRBb/3e7/L4yc/4bMnH9H3PYvFKbP6mBR7Hj95zr7tICV0NrRenLLT+5z2g3MFRVFwdXUlXGpXUJbi4BGhVQSuia1nraMsLZvNBq0NH3/yMd/4xq+zbz0ffvAB5/fuoY2W8XYUE3yQSbG1lmHoD2KanD9yw4sxcH19xYMHD4gxYEfha2L4ZdL4PuLoXE8oZX7qeN+emNy92x/O1bvu9bvCuZwnmqm1F/V6iOgX3SJfnCgf/j1VZkfxj7FYrHMmJkVO/+YO+rtbiOI4M1qPjGZhHIeYCR7AoJTFGGnrjjHT9y195zHaYY28vriKFEVR0dQwDP5wXlhbHITboe/Hj2BIKbJv97TtnqYuiMnz8tUVs/kRb771DucXpyiduFldsdlseXX1isurF8TYkpKIi8YaitIQU8b7lpw88/kxko86MESoKul4OHDbD2FF5R33dURnhcGgESTKvGl4cHGP4+XRAVlTVxXz2ZyyKtBWjdd6jSucHKsH9w9i6+T2bsoCP5+zqRt2s5lw/q0sEOPYCuzGQtHkSJkE3+l9i3itsNaMaBZB2VSlONDn87kcz3FSPznSvfcHLNRsNhOcwfgz56S4MDHOY4wHp7r3Im5PWCTnHMMwsN1uD4iGoijY7/e0u1YCuwqHcZab1RbnKsqiphs6QhzFcV1gFGSlCTlhXMHi6Jj1esV2u6Pv/RiI6ASB4gd0J4sSa+343gM5ybghQcWyr7quxbgS50Qk6boOELHfOYf3wi333lPX9WEiPAzDYUFkncMphd93xBTusNEhDvGwH6bBYCpUaKOZwqiU0uLSJJNjIGeFcxWz2VL47FHRes9+N9B3A6DxoRchMhuMdhhTsN1esd3sxoLQL7/tB89y2ZBIoBVt9CwfOOwyM2scsQ+EnaDFNpeRs0cFxw81p/crrKvIdHJd1xXWaFSOvP+NC37wvf8fa3/6bEt2nndivzXltIcz3blGFIoASIATIJGtlmSp5VFhhz8oFG37a0e3I+z+f/wH2BHub1JHu6PDim673YNoUSJFigRJDIUCqurWnc+0h5zW5A9v5j7nVl2ARcAZVXHu3XefvXPnXpm51vM+7++55OSsZh0Nm+2e1Upaz1OOdF3GD/DqScIaw8lJQbkIfPs3H+GHVyw1PLj3kD/9o8/59KMLxg5870EHsnlBUcuiszAN7X4AG9A5sdsE1meJlxeJs7tHbDc7Ykis7taMo+fO+g4f/agFPfLgkcPVmXfec5yfBza7T3B6TVkkjk+WPH3eM6aEInGyKCkU5FJ4zq+etNw/PiMvM/ttz3Z3zTEV9UIzjpbCOMYsIduuLImDBFq2bccf/8mfcnp2l7bb8vTZC8ncqQ3b3cBul6lKQwwDzhqSh24YyWnNnXsLun6DLQxX2wtcZel2HU1uKIoKcPRtT1Uqrq43HB+f4WPCVSUxQV01aJvQ1mJcoC4cXdiKAGLk2O52G4oi4n3g3r0PGfqI355zfXWJMZGvfe1DPvnZp6joKRrLbtdTVTVaF1RVYrGogJHdtpUA9eMlzirarmOMim4QU4UrDBeXF6is8b2nME74x7pA5ZGUWpQxbHbPMTiU1pSVQ+mMD1dkKkLwKKUZ/QAq0W+uCSmyPr7Dr339Ozw//4xnV5/IfUrD5W4PGLRylFXJ6TFcXO+grGiOT6CVjos7Z6c0Tc3z588AxatX56zPjkWYUZEcYRwC7z56m+gHxnGkTBn6nqqs6Ns9VVXSrEaudlu0ypzcOWWz3VFVBUdHJ3z66UtyHKlqw2JVsWv3oAKrdcU4ehZHhmg8WrvDdfuX2YIK7IeWMmeMdriyQSHOY7LH5iROdZ0p0NhkqG2FKRxxjKA1Q4h03Ui1qEAnlNOy8M/gasm6SDFKyLLSpDCya3uKugCbGcJIil5c8Dlhk6UpGkpXYtD4GCBFFFnymjyQPDkJ/5ioqJYNkUD2nuADYzcQfSIGJAMtgY6KpqjIZMKUKzH2I8ZM900lGLSk5ZwsTImzDpPEPOd9Ly7pyeVtLLRDRz8O+C6yXh/jmoZVVUqh1xQ3OgERTURN3Q7MjuYciFEx9P0U/K0oy4aqrGhb6US1RcEQEp0fCVGQDgZN23fk2JOTx6hMVZcTbx8J9B479kH49F3opPtKQUiRaMCViqgD/diSvaAjXWVph46iKSkM5BApgGWzpigWoCt8W7AvHL71XL+AjzZXaFVycW5ouxGSIiWDNoqdDbzg5S2jieAfwoRZETFQQlWNK6ACY2VuH4Ki1QFyJvhECh7rAkElcX9P7tzCaPoi4IsB7RJVJTkpXeqIQ6ZwmugjKQb6vCMpRbNaUBUOHwQPZyYBUtZnhnHwOFtS1w1a1cQAIdxgJXPOGKshJZxSGDLDMKKcIWfwvieT0E4RkH2PQZCh3o+SOxEDzpips63AOoNLYiqxriKEdFiT/NLbbRHyiz+//GSZ00y/dlvsvjGnMimA6mD0fP0l3iCMT1g6pnc9+E3f9Nz8NxTRlRY8Ywpsti2lK/Fesd+N9EPGjzAOiTFExihrtNVyIZ0LOVFUJSFpwhgwFNSLY373e9/hf/6P/zec3r3LvYf3MdZNuSDSwfujH/2A5y+e8eTzz3DOMgwtfthxtKqJvsMuHL/zO79BDC3/8g/+gL/1vd/h937/t/h//Jf/gnfP7kzZCP5QyBhv5YKllA7B9kqBtdIdwmQZTsljXUlVNxyfGM4vXvCd5i1yjocQ3+novEFE/+JxndeQ898PR/ZGEObGKDg/Pq9Fc5JOXn14YUGl3IyffGsc5cPQu909fduElm/Zsm/Y4zcmwFmEn7Wj1ws9N1UBMUNP+O00r4H1JCXdkEKEGJ5kv9W8fzd473kPYOKX55t18tT7Pb2mPWhBgvNG3g81oUjzgbv+Rff/Vzm9v7KI/t6dSFN6Vo3BYNjvM/vdFJyB4qoPbNsg7dZZ4wMTT0azHRNdlxlC4OQMzmpFPybaLSwqzVFp2Y2JzWWk95k2a7bZ8BOtxe2jMo3OlFpRm8xRCctSvoSQpKW7j0i7plIURmFVptLgLASmoMYoAWpWKYxOlMYQUsZHdRgAWmuSUoSUIWWKQlFWmRg1u15ayEsrAyckuflbpURo0IpyseTu+x/wzUffYZUVJnmMHigqy7d+4xu4H/8xP3vluffQsVqvMcXI7354zNv/9PfZ9Z7sIy/Od1RNyf2zt8EuwDZAjStK0sRd3u87uvMNJg/88JNXfPr0QhbnKmO1Q6G43nU8N6ALqSQLRzjwkxct791dcrTW3L93n6pa8rWvOY7e/Tp794D2co/Ra0x3xZ/98I/50R/+Ab/z7ff429/5kL96cs7Lj/6Qy7wipoyNkTIM5K6FYcu2f4H2z8DsQUPl1ji7BJXZd1fsxmt82tLuNzx7ckHXBsK5ZXm0pPPXeLfH3L2LPq0x2rEfWqr+iu2TH/Hq+U/Zb1tCtvR9JpeWdnvJ9cUr6qYm28xm6Bk3O8bOS6tVzgx0cLHnpI+83CtUBb19QnJ/htEFZ6sFd4+OOTp6h95nuouX7F88RgdHGjVhvMQUG3y6RJOIsaFoZEJWFgviPtKrNfnu+9Q2gmlZnxwRx0jebNHtjr7tUUkR04jqLmjHl+x3I/aOIi0ihET7s0958sO/Qr13ycndu+SXFxQv9qg2wJFlyFfYZoToMLojXn+M8pHys4GUfvhVT+U3boMXMXsOuJwFXRHCpB1rCANki84OUk9jE9ppbuuSX2zvepNw+Pqmp9sPQETpKEEV2ZKSOI6VjrcSrsWVm1KSKFQ1O4BHkspkjpDLqwYMiiRV1dsV3Cyi1utO4ttbnpzUhbiofKQoHSEO5BSJXsIQc0xYLezdqqgwOtOPO2LwhCCTawnu0RiDBJTAa87Q1951EtbkJjaJ/RPvzljLqm6wYcv79wo21xuO6556Cj3zcRBx2y1Jac/b77zN3/rb36WsDeSI0bBaOf69v/O7XFy+5F/9mz/kz/78Txm9Z71IOGPp+hFlhTG6216Ly3sUd2ZZljhbMuAlrE0J9kZcqJqidJRFSc4aawuMsXRdP30udQj+e/DgPk+ePObXf/07PHz4kOV2y/n5OT5LUnv0QYoT7Z4YxtcYznNKPEpCRpQWbI0kpNtpzIr7WOkbl6twoiMpx2mkuTeOyVu17VuPzWPmZv55G9ly00EgQbQ3BaIvnAe3X1PdFHDmCQWz8D5PEtTUiacymin185fcjHGkKRTQ+0jXDfhxYBwCKCXBXcoKP7QshZ/bS0t10zTkLMG6ElxmKIqCcRB8T5wc50oJikNlWC5qyrLAWkMMET8KDsVaQ9u1vHz5gtEHFusVWEPTrNhst3zy6WNevjrn4uJcsgWcwViN0gprLMZkrHEoZbHGSF+b1tRFPS2y8jQu3UGQnb+Hg/tbQfSB5CN1WXF2fMzZyTFNXVOVJ5ST+zqGiLGKqi4oy0qW2VECPMuiIClFmq4FWhmqwqFzpjeGuigw1tLU0rmxb1tiCNR1IyzhGOm7npAzVV1TVRVDP5BDOGBXnLH0Q087uaNjjNJ+PAnCWuvDd9O2rWQWaM1qtQAlE/9xHA+89DhhmWY8i3OOOHqGiYN6u5g3juPh+j+P1RAiKUhXjtLSbl6UNWulGc9f0fYt2hicq7EaYhyJQRArZVnjip6chS1qjD2EqMaJBV8UxWEf4CbUal7IzE50nV+fyN8+327/+fb59aU2aa2F85r04ZoL+dC5IIWxdDhWs8t9xudYW1AUFpUi47DDe7kvaWUJYWS/79l3ozCHI5QOYhwktyJZrHMsmiXr1TGKAmd/tVDw3SZidOLBozO6ccd+t0U7xdl9jVYei2b11gl3j++RVMv7795BNYkwJvYbhLVvHFchAwajNJurC4oqsd+dc//+MWVT0PcDxiaKQuFKxW4Lv/Htb9DuO168/BlvnR3xta8/4Go7oLVjHCLX1y3Xmz0Wx76NrFcVfddT1pBixllNXRegEscnNQ8eaa72V9RNQc6KVy9HHjwo2Wx2wp0ejnnxuCdlGLaQ1cDxiaGq1xgC3X7E+5HhesQ4xXJVEUaLUR6FJkbYbXrIkXa/YbkQB2tZy4KtKkv6TjB+p6dL+r5DG0PXBcqiEgdnDJxfvGK5qnCFoevFSLK9uuLk+JQ7p3fZba+oK0fXDZR3FtSLhhcvHhNT5K23HvL48ROUEmSdc5Z2v+fszpq2lXMckHZzBCcXQmAVIrZw2JR5+eqS9XrBarViv9sToyATykLuG+v1MUVh6fsBpTMnJ2vu3Dnl+bPPUUqEQqMrpENMupDG0ZOzlm6csiElNV07/BQyJpi8ZrkgZTk3VquKnBJlU3G0PuLhnUeoS8WP/uoj3n/rA/JgODu5KwXQEDFaOgKydthC3HnKINfaqkDpgsFv2F+3fPubv8X5v37O8fqEalFxeSWu2M3mmof332K5WrDpttR1Td/t6K9b3n37XdZHaz779HN8GHn3nXc5P78gDAPbGHDWYlBkV9AOPTF4Yggcn5wAYhoYfGS1XnNycsZu3+KHVrJljKMqC169fMHJUUNZnvL02WcoZ3G2nHI3IqMfWK4cFxcb3v/aO+x221/63E45Ef0IeQ49NhS2pGoqSp9JhYfg2W+v0TqRQ8LiMNlilCVNoczGWuqmwTQK24iZhJyFc640KiPZTSiGIaDGAETIkRg85CTdgjGijcWVFc6WGG3RVhOcGD1yHCWsdhggJYIf6VoPVhNUkjDJBM44XFNKBkaEoRvElGWl8J6CmCbGMWJtoCprisISxoj3g9yPk6dLLYUqyToTiFhrGP1IUTiGMbDdb0lTp1/rB5wqJCPDFfTbPYOP2EIKqQIJi/TDntIWhDySkmc/JHo9kEPEaIszlmTt5ODvGUdHivK5iqISJKHS+DAS/Sj4Nzvh+rIW00mzJHlD5TTtvsePkbKsGfxIZqCyNabQDGFA+4yKEDGosqYfRsY0YMkMbYsKoHPN5XkgpZ7tJejhlNT29NuRTd8S/J6u8+SkKK2TeRCZYRLcAAlrD5FC64PZRDEZFTOMKjM6j9Dckfk1XowzWfKqYh6IRCJJUESA04pmUaJXgchAbzOu0IRRoZPGh4QftGQe5EzSibqqqJYlY+xAibHA+yTIvnEkTOx1bWVtlwA0mMKQJlZ9URZE71FJ4VOkGzocDpWDoHyUJhol7+8DOSpilvNXaYexFmtLrLGEIOJ9XbtpTWonROcvb26Zz+9ZHLz9803bF9cWrxuAFLdUzMlo9AYR/E27q9XhufqLBp8v/j5KAiy/sA9veq7k9RiGQTozmnqNVgsuL57y8uWWi/OO7a6n7yPaGjIBpcUwd7nZkmNitQSlCsYh8Pajh/zT//A/5Lu/+z2cq9ETnieTD12yxydHvPveO3zy059wfLLi+uqaFD2rxZLkPcZZSmc5f/WCs9MV/7v//T/BasWf/rs/4f0P3hI9MyV2ux1HR0cAB3PQPD+eH4sx4ApLqaXIZ7WT4NuUiGmkWRyz3z/l/NVzHr19Kp3sEz5xNl7Na5RZtJVjKn8Wx3R+7Vi/7iC/bXjk1nMmNni+MYGIiK1ISTG71W+/voTE3hbRbwvo6VDAyjkdxHnRXaQEqY16w77d/HytI3wynumcbxUB8mT2u5nHS0Eo3uz/9Jjs35vWw4ey0iTKz1J6Ogj1B4ldacjS1RpVQsU4OeTFKJNmd/v/P5noHzxwrJYFdWPJ0bC/9nSLhCstEcXldmS7lfa1nA2Dn74kpbjshRXe+cj7dxJv39XsWg093DtxnB5ZdkPE+MS+z1x6TRg1rzxc9ZGSxB2n0M5QF1AbWBdSxghRMUZNkuw5tM5UJEqdWBaJstAMyRD2iU2bGFLG6sSqFDEmBLhuIyFmSpspjCZmRTsmrM6sq8zdhWHXK15ewK7PHNcabaDzwsVeF4qi1Fhjsb3GDB21U1TWo5xGLx/y8MFv0Lc/5tnn/4r371u++ev3uffoa+xefoRWV9y7ZzijIeWSdz64j7KO6uw+ev0B1fohp6v7NMs1ANvtlk+y589++gNUv+M7797l/MUl2mp8TFxte2JWvPfwDkcLy6vNlt7LgjWTOd8N/PDpJfdOa0IYWT+4z9Hx27jmjCZbPuk3tFeRcHVJ+uEfUj/5E/7y8R9x+dlv841//3/KURH42Z/+AU9/+H3CeIXSgSed56edJ0dPCHtiDiilsdqKAKIii1oStHdjJmZN1w7ElNk+/RijpXU+YSgWT1DVCpMTethh4iDMt6FD54zB8OL559jS4f2A94FNzngUSWm8jyQfMWlKHCYR+sAn7RPOP7miz5pd8qAzOkeWznDaLDg7OaO2BVcX52y31+SgSEGEIuMiRZXQU6Cumtr5QTF2ER8yWpUUVUFZQrVa4LRjHDwhePq2hwylM+z3O1L2dGd7rk8uQGna7Y7LFxf0w8DjZ1uujx+zud7RX3ckOzLWCmszOzqMcoTQUTSelBTtpZfx/ytswyAsWHMrKEVphdFGmItpZBh7YtZYVWCLQEwTrsPorzSVeONNeXLhznOAeYyKWC6ujDy5iNPEzMpJ3JxiZJfLpMpJFhG8afIwu4Dnm/+NqPJFIX1+3iyoGJMPIu4777xDu9+yXq+5vt5wfb3De49Whv1+jysc2gnTW2stjt3Fgl4hbV62YO6am4WxWTSaxdj5Z1XJAne+cceYabuRF8GzMcJBvtzu+Ojjn2FdyfHxKV3nIQur9nvf/S7vvvtoqrrONyHFw0dnfPd7v8vnz55ycXHBkyePxb1RFtORkuPeD4NwGbNMGGbBbk6T996TYsZYM3X2GEBTliUhBKqqPtw8i6I4BIR+9tlnvPvuu/zRH/0R/5N/8D87hIFebja8fPVyEpgWjN2O6+sL+r4jBM8cDmuMIccIRHFPuoK6Fg6qmr63OeA0TCiHGOXmMAt4v2hM/k3H8W03+sGices5s1tZH8S5fHj8Bi10MzFQ2ki2xjyjQR0KKb/sFqIsSpw16JymgJ2RmEU0k3xNS1UvKKua2O6QuY50aEngY5iCa+X64L2fhFrPOHhizFOQnbQKl6UIt8FL19Mw9PTdyNXmis+ffM7jp095fv6KEGH0ke2uZRi9FFyC5+xoxbJZYgtDmMRrbQyuKNHaEUImBQmpdEVFzByc14vF4hDaOwzDwZ2kVSGZGX7CpSiNsxZr9IEtWhSyIPZRxBWjZbLXDwN93wHqwOIehl5EKqBuFmht2W+2hBgoipIeQctIa3Ekeo9zjhAju91OGKrjQL93DONweK1uv5/45rL/Gei67uAwnzEmc7HgJmDXMIzdaziTOVNhHqe3BWGr9MEl6b0cY+ccx8fHkkfQ97RtS1mWFIUIJsM4sm/3HFXynK7v8SEy+ogyWsRwkggvE9tQ2l/r14qHh46L/OXzSYJW84HnDlP34C1ky/z55+v3HNQ8ZyJYK9zZWRQ/hKp6D8ZMSBtzQL1orSicO+CL5mNaVQVVVRFjZN8O8r62Eo57ioyDdNgFL9fo4BN9P9J1Iygz7Y+gBYIfiLGgsBZnCxaLNYVbHI7TL7spZem7zHbbYUvF6dmCooaTU8fmqhO2sB84u1OzaxtGn3n84y2PP99z/UqQY7aA+48aQLPfX4qDr1AoDdebjuVSME2bTSs5BSFSNxHMlv/tP/k7/Bf/xQvuParZD1v6wfPss2dst0/x45533i/IoeTixYhKFUVhafcbXCEItpgGlmuHLQbQkXt3ZUz23Y4HDxpiHLm67lkuCt559DY/+1Hku9/9HbabkT/6o3/L0XKFZwqSRETDMF2Pt7uW3/2tD/n4o5/S9wHlHFWp6AslzFsdOL0nCJTRB/zGM+dTjKME/aYk4lzfJ8rSYE1JyIGuG1Fac3JS4oyi3Q2MXeDF06c0VUW/H1gvG568uCK+fMFiWRGiZ7dtJy65BH/3/cBisZSA+zKx2e5QaLq+I4SBu/dOefnyHIC27akqRdNU9H2gaaTQ2bUtzlRYA33nefutMx5/9oTV6ggxJ2jOL14RQ6AfRvp+ZBXXaOXYbVr0UUFVLKR4WVTU1YRsGxN5OrabzTUP7j+g6z0xweDj1AEnbOLrzTXvPHyXb37r26he4ygJY6bvRsq6ImdB7Jwen1IUJTEHKa5nyUdIIXB5dcliWbCqV0SjOD26S9on1os1u12LrSzjENh1W2ntX9WQI4XRBJcZw47Pn1yyWq8YwwBKUVYlo/eEYcA6y26/Z7VaUjQljx+/hJxZrU9EJFcaFxLbzZ6yWnDv7j32u0tSijRNxTiO1LUUvX7yk48pKkXwmsXRiv1+fxDMT06OaBZn7Npzdt3ulz63k5c1yDB4otHkZHBViXMlyiq0LhkZiGkLWqNyIvspW8dbClPRsadaLanXSxHQdcTHHmMUOWaigsLaiQWtMSpRJg1xJHvACzMfDYUrKNcrbFHR+yAheq6gKRs0Az717HdXDF1PYQpBnITErhvJRhP7ARTossKUBdkkQuclRM8oghIRtvO9nMtIQGFZLlB2JO8Doe3RRaLPPR17CluSnaFcVixqR9LQh0jXd3TDSFHX2KIAI+jHmAP9GGi7DSYllK1kHaEFK2BNhuQxeUBrGFQgxpHaGHQOxDERVMJojSUQ93LPVz6jEhQLySMZx4FgDDEbsisJiJmhMBqCJ6nAYr2kqY/ZXvXCYU5XeDtitcPnQIo9hVJYK0WD2GfGGHBeoVHsNgNbYLu/ottr/ODIQ0HYgx8y45DxYySMnjiOpHhjyJAuTVkzScFbignjNMebTSI5z4IWZK8OQhNAmpysMcWJa46szXKS7gSd8VoRfUaNEW0z2oBxkClIQdaXOgmX3EVIbhR8TIyCONQa5+R6nGICLcXHMfbsBvDBE6ZO3qqq8eNIJFJoiydK2KTOqFKTjawjK9eglJFgXp1IOk4BpB5jZZ6ZMtiiQWfF0O3RaKyTAnzWmawE9ferbAfe9i2z2Wtrktf++OZ1wG3TwPxTTDNf5qrP680v/v7t97h5/ze9n4iRB2TILFy+4anaKGIK1IuGGBRvv3PE9//8Ux4/fsHTpzv6DrwXt6/kXHQ0i5p+GBi9dM6eX15zebnhGx9+g//j/+n/zDe/9RsEn1B6DkKdHNVGzg+jNd/5zrf58z/9E0GraUXbR6xyLJs1ZSmoybKo+eEPfsyTTz+jqkvadsfJ8Snn23hAtcwdrbOBq65rNpsNXddRliU3ErCsr42RQrAfBlxh6Yc93/rWN2gWJctlQ87+MP+eBe75+/h5mJ48O8u/9H1PRz8L7mTurD8I1UodBO+bLpMb1Mltx/WcL/ZF44l0x9844l//s4TXzqJ7mrDlGRFy5p8oNeFp8iGwdu5Qn0X52VcumMb0hfEnOM986EJXN+sHXq835akgJaZzCRpVSkl24/yYujHKyeeVzhsmggAqkWczndJfybz2la8ATe1oGk1ZSPBWrBQmG4rS4TN0/UgqMiFmRi/olYmIQKEFVu+MYVEblo0ljIlSR2qrWRQaUuSkSCw06FFzlTQqyJdaKcVxYblTW47KzN1lZF0lxFxnGJPC6sx+hNoqFqWh0oGmjNSlYQhAUngPi5xpnGJdWwpn2A+J7BNRJ5ZVpHQQomaToHSJ904sj+4XXOwyl5cRxsydBlwB7aAYBsVZkzmuM07LxHyZOpY2cLwsCPqMIX6TZl0Q2s9oqobTxvPht76NKgsKV9CrDlMsscbiFndIuiK5Ffrk16iP3uZofZe6WZJjYuhHTlfHPPi9v8Pu2af8+I/+OxqX+J2vPyRnWCxK6toxhMS2H3l6fo6yhpS84HWQJVs/Bi63HY/SiFscUR/fYbU85fJiz4//7M/40ff/jIXL2PCK40Zutp//u3/Nj/7oL1nc/YBCGerd5xwV1zgXOR/g6jIwBqlI+zS3QkhY5f214WhlUE7R7wMXmzD3axAnD4AGrNL4/eckBaVVLGtNXVvKSrE4cyycRsfE6D1RiXMvJsvza89Pzwe6MaKVxigladzTiZZCpt0O7HYjfdZsosJncCqzMomt3fL0s6cSxpUyFiidXAD6IBc8ZxTrUrOoFV0IfPa0k46FDBlNSDKpmCt5cgm4qSRaheBSkOJv8eSaYgpSTHkOW1OgOvjkQlwk0wTGaCgNQHe4AMh3mbH6TbfLv9nW9j3WB+HUMn1vWpAHEgo10o0dMUOhHaUaBYzycx3mf/0mv3rDVb+5f0z1wyQCW86eTJhEdPlO0ixGTsdZ5YQy4YBt+GIxfxbH5598cbLy+rNfE0eV4uDkrOv6gA7IU6Bk2w1UVSNtj1mq70VRkGOkcCWD6shZ0u3Trerw7UnPzLCzRljgdT1xNHMm6kQ39OwHD9liFzVGFcSgeHF+jS0rjHGUhYXsOD1Z8t3v/i7L5eJwNOEmbPXk9Ii33npL3CWjp2vF6da2HX3f0vcDISRiTKSkqOviIC7Nju+ZNRwGT1kKw3YWqOW4hcMkfJ4oGmPYbDZ8+umn/NZvfZe//Mu/4Ld/+7cx1rBer/BhnHAer+h2O/q+o233h+9P3KEZSKScSCrQtsJcVkrETefsGyrp8vnf3HVwe3y87sSQw//l33nNRT5/XvKXJjvzGJrPo9vO9dfwLkx5IcqgtCUR5PvKgkZT6ufv91fZpKkqM3pPVciiRGlBuKQsjh5Q9ENPzGlCeZSTODlOTGiHUnLv32734iDLTOeAjGkJg3UURYm1jm67o5iE3jztw/X1tbRk5yTOJwxd3zMGj7FWAsNoaBbHNIslxii6oSPEMGGEKpR2zLGxUEjr5yR8+kmonnnjXdehlBLutVIk70khkWMijCPdvmVzdUUcB9qikHDelGjbPTEGqro8CNWzE3lmr8/ngNEGP0oL+Dzmxr5j7EXQtlrjjMFoNbn6MkerJTnPgXGBuiw5Wq0OYyTGyNI02MKhjZ3yDuS950LnHOYr495NLHgJN67r+jUe/HwMZiG+73tyTDRVLWL45HKfHe7zuJ0F6bKspKiZ48QGV4dQ0GaxpGrbaX+me50r0dqCShPKJGGcQxlNIjN4j9aGpmkOn2letCilcIWV8NgghRvnCoyzk6hyM6GehflDWOokpCslKK75GDsnhbyu64T9a/VN+OxU6Cmcfa0ocZu5Djeiw+hHlLHoFKVLYnLVBx+IKQGasihBORnjKmKdJsSJGR8NKM3p6Skpwjj+ahXw4zuWD3/tLqujhmEcePL8hYyjaOnbgqvzyNfealgsBY8SNh2XVwMhj5SNYxj8lDUAdaMxVkmHYJSQpbYb2GwlrPfhwxPadkdmYLHUxHzJq4vPqZeK9z54i1fnL/n002uGXuY1p2cyDkpX07WRJ5/scLaAbPFjZH20oh8S9x7UFOWENDMRZwtWqwJjNYVbopQsyrtxx4ffWvCjj77PZ59coIDd7pqiGrEuk6Lm+YvIo3dqkhYEy/MXr3j//Qd89vhz/N5QVYayNGQS7bhnuVixPlmBHtDGEHzC+8hyuaBtt3JNsZqqKAXHNhUdt9srilpQUouyghCpa0e3H8kZfu3XvsEPf/gDqtpg3BLvwbmCFy/OefTWfdp2R1mUXIdrytIwDCNaW4yW+UOYhK0YE2TD0I9krdhuN9SLhufPLkkps141NM2Cp08ueXjvlLI0/OAHP+XRWw9BZVzlhCkdR8pFiS6k++Dy8prFYsV205Eict+P43S+VVxeXlFVgv6yTtP3A8M4UpQVeYykqXDZtS1nJ6fEGPnopx9xpu5zUt6haUrC2E/mi0hSGa0UfdeSU0IZJ9iOXjjEFxeX5Aybq8D6pOSTT59wdb4lqMxysSaGTwkh8Nbb7/LpJ58TYuDB+h4+DHjfE/LI1fYSV5TsBzHyhByJZBE8tVyf18dryanq9pjS0tQNMcq9zIfAft9y7+59drs9KSaWywWPH3/KYrEUd3HoWa+PeO/9ByyWCx4/eUzOmXfefpvnL56y3+/YbK8lh6EsqapfnomuvBb0SRgpC+kqizZMKBS5j41+xOeIc4V8J0hQXCBjSsfRnWOKqqaoC7TLZK3p2+6WgSPjcyRp8Fnuq2U5hz17UowM44hzlmpVUy8qckh0/Z5ujMTFktI60lQUGQPsO8+gEiiDso7kM0Pbk9OIj5HY95TNQooBCqqqFAOWiiSdCSqA1fg8ooolyik2V9f0bUeOmaGXwO+UpZtL5xKbKwKKXdvjh5H9viUbja2k2zVPaL8xJXzXkcaB0hi6oRckjRIziEaTJ060LaXoNrbSaVUoKcqOoQclXQ1d3xNTxmmDMg5npOhsdYHqBUK5qBbEFAhB1q3j0DOEjuViTaEdVVORYsLHgLaCHQu9ZDzFLC7SGCIWCc9LaILPDJ0lj5bu0hMGTfKB7EeyNwQfCN4z9gN+HAlTodzDa+LaMAyH+6QxdnLMvm74OHSlKXVYv8AsoMnfo1JEZrdqPISoBxLjMGIGjSutmE20HJeYB3wYKJzCFZlqoamPLVdXG6okAa9FISgVYzSqsqL7eC95SToSho4Y/KFg7ibcRsrQD4JUc4XBOUNRWmJMkxHC4Dth1UtHZZjyqsRMULgKZx3Zx8k8J+ebcY4MlFqzrH+1LrKQ5d6vspqEPnF6z5jH+XFAigFf2G4E0teF+J97PflrGO7z+x/e+03//ibFPE8i//RP83perjGCQT5/ecXPfvY5Fxdb2nZkHDQhJkLoiaNHT2qktZaiLDDKstlc8Hf/zt/lP/qP/mPeef/rbLY7rHEUDowyaCMkDGAqXAUePnzIb/3Wb/KX3/8+MSRBVIaENZJZQ4ZxTBTFipAU1xtPCIqYEjkrJKBe1qrz/gAHlMusBYDgXSVDTUSYrutYLWu6/oqvfe2bfOc3v0a1HCkrDdkcvpv5e5vn8j9fRL95/MbBnm+9RryR8m85vyeP+RdeQ671sxN9/t5yns7JW/iYDOQ0u+TTVBib92167lTFmAX1G5Pilz/HfI24vRZPKQuPfxL4Qc87zuEPetKY5rF4GGJfPhdmBI2I9ILmQim0mfP9ZsFeOmdSAq3EJIBKqAQmz0Gvc5hp/NL7fHH7yiK6dUouZNZCjNgiAUoWB2PEGHBOnGvdkKdWdDmRfBAxRpGxWk83qSxMqRhJSaoeRkPlpBqp1XStBSprWBaGRalZlJnSJozoYGityFrhbMaFLPx0BUZLRV1N4B2tFM6AM5pVmakKJfy0qFgVIvyuqkxVGMbRkIZEXSrWjWW5sJLGbsStvqg0i0bTuExycNQkFuX8WQOpvaB78TnH3/t7jNUxHN+F7hxr1yztIwr1hPr4LbKx6GpJtTglhQJTvYVZ3MHnkrB4C9u8jSuPqZqGdr9l3/YTG7GmrFf8/j/6X3P+4jnnf/FvWB4prrd7Xl1fU7Sai/3I86sdrio4OTslpz2vzvfEnClM4u9/8wGkHgbP9cvPefjOr+ND5PriFd/6xteo9td89IM/Y7lMrErISTPazLDd8uqj76ON46wJHL+VaRoIPvEsjDAwBaxAmCqF1ihKo7i7digFry4hDVLhnquJGrDaULqZr504qgwP7tWcnFQsasuq0pQ6k8aRsTdkJex8H6QD4enVyBAzTovgXU4CVMoS7jhmqWAaDSprxiDJxbWF+0vNqgA1ndylURwtLN4rnl8Huj6zKBRv3TfcOau42ibS44GrvfCtuxDZhkQn+hdmco/OApbWUBtFbfSk4UpIg5mClaIXfEcgT/sMQ8pyLpNZl4ql01gN/ZAIEyKjcoqm0PThVxPa9l33cydPKUpoVT/2hJRwumRRSMDKm53fX97ehC+RC6KIDiABENoU4iCOMPpA1/ek7BEEi6Qw56xEvJv4fVopOd5JOMAxxokTOr3N5Hi8LZzfvsZ/yY08uYKddQxaXnO32/HkyVNevnjO5eUlZVlSVhV970lObqSJLMIRir4TV2kIUVqttZVL8oRs+OIkVIRWERyrspbn58wwysJV2JYyCY4odDtSNYqlSzhj0DlTVQ5rah49eosPP/xAUBizC13Jh9ZKcffuMXfunHJ1dY1zBd70FGXJMBZcPHtCjOJAFyelOTAGZ2fvzFeeK9Ix3LDPnXP0Xc8c2jeHFM6fccZPPH36lPsPDJeXF5zeOaXtdszBsC9fviAH6T4J4XV+vLWGOQhEK8XV1RVPnjzh4cMHPHr04CBAGjMHrogwPhe3ftE234QPt+k3jVk4TGZuxsoUcnhrgnR701qYnbcr6Le3m3Y8GXeaKcl9Kia96TX/JpvW4jw2WqHKieuNoHCyyhSFwzpLPwx0fU9dVxhjaduWYRgOn0G6DG7Y2lVZs1guUXtxaFvjpvDLOfhJxk5RFtjC4XuP0hLkmNBobVHK4IoCF0EpIwvEqAhBkXGysNXSPaaNI2ULyaKMxihNziLuZPLBaT0MwwGJUBTSYZGSFCmNUlR1Td3UdG2LsYajoyNW6yWS/C5tyMt1M4X/iQt/sTSUZYHWZnKFCxd7FnG9D+QMVVVhtCHEcAjknJ3xfd/jvbhlXCFt5T4EulaE7+ViQYyRtusmR2hBVddkNMMwHJz2ZVkeJtMz43zunHGFO7hd5n27PVG/XdC6fb2fFwo55wPOZT7nQgik3E0L6YwrJiTN0LPd70g5UVUVKMU4hafWRYGymZTExSgoIIu1hRTfYqDQkkkzi95zLoQxBmedhNKGQIwJ5yYMVsqkWyz5uVgy7ytweB1ZHMfpOEnosp6eE6IUWnOecihQB2QOMBVK7PT9+8M5AFNw7ehZlMJ2resaP3TSRaO1IN6yJmbNOHqKApaLiqaSsdj1HSpHnGkO19ZfZfu9v+9Q+pq+v8CUjrM7JdvdyH6X6AfFvbN3+MlHT7l4/q8pSoWt4PROha01Oi6IMXN11bLdjKQcMdbw3tce8vb7a7rhmqIsefFiS4yJszOLK844f7Hnw/e+zb/6w3/Fp49/xt37a7ph4Ic/es5y1XBnsebq8oLf+s2v8+DRGVDi3E+4vPwEIuisOTpZcf/BGm1L2u5C8h+y8HjHoScEzf0HZ2QMbTuwWq/5y794QdhGzl/2lMYJk3+/QwFNVaBV5O49gzETfkIlXr54SVMqlos1l9d7rHVkRJzSxnK936BbxWrhyGkkJgghc3ZW0PXQLCpiUOx3ntOjUy6vNixWJVobgh/Y+pFSGQrrMAru3bvL+eUlnz7+DFc59t2e49M7bDYtIGL5q5cXLJYFKUWUlq6W4+PT6bor59qjR3cZhp67d+8RQ2Kz26KM4fj4hGEc+ea33qLvWxSJYew5PV2QcxKxzWhevHzOcr1m33VUWRBE/TCQCYxhpCgXKGUmhFOg6zrOzo7YbDfU9Zqmqbm63nDnzhFtt8fZgq71aCf309PTE2KMLBYLUgjSYRkiP336Mbt1S3X5kooapwuOiiPM5AJLMdB1O8FC+UjIUBQ1jx49xJWai4uXvDp/xscf/4x33nmfT19+zMXFFWVZ0Q8D5+dXvLro+NY336Msasa+xxnHyekxfT/Q9R1+bGmaBT4M9ENHXVUEL50FxhlevLqkKB2L1YLzywvu3n/I1eUGjWaxWuBKy+7ZNevVgv1OnORq+m5/9rNPGf1AWRR0/Z6UAs2i5OWr5wxDT9NU09xLgSqkhf6X3LLXmFxQOCtZJeOAxpBiYowRH8TM4QqDriRUPhcabS3kiDUF68UCZYwwyZVc611RoLUiTQgF4wyhC/SjJ4dEaYvXiv3GGFbrNc2iIXrP0HaMQwdexIo4OYCzUiQcIRv27UAcBZVTlgbf7rFOio37biSh0EsLKVIXTjokoiDIVDllY+RIn3t2w5b9fg9TuHQ3jAQyicgYRxgTbmww+5ar7RY/SNdDs1jQtiPFskBh5bj5nrHr0TGSgocgYp41DpUgjZ6sEj6PJKRIrnMm+hG7WEjnRx8Yxsg4dFSuwmRFyoqirDHKYk1JUy6ojXTS+SBM+awS2ETb79l3e0LKNC6xKNdklXFlgS5Lsor0vpe8gBRIWaHQcg/FoY1jHIEAvi2IrWHY9eQk+UspSedn9BLq672XENcpS8qP/tB9Nm/CeuZQsL9tApnnweGWAej247NrNTE50af51iGEUGcYI6MVJ3hSiZg9SmeZS60NzbJhcVSgSy9dFcrQLGqckwBdZ6ULqOtGsnNkbgIxBeNoJwScO8yPvA+CqSJNYqfDWkFYde1A17ZiHAnSNRljYBxGnLVURUXywqzXCKc/Z3HCK6OolaL+FbtEb7pR840uOBmARDHLt5yzf81r3Pr5pu0X/dv0BBFFJ+0tvUmozPCLEBe3hXytDfFgbtE8efqCq6s9o49TN2zGFhLiG9LAcrHADwNlUfHr3/wWP/zBR/wn//F/wj/8B/+Q7XbH//g//vdorfGj5+zolOPTE45OTzg7O0VbfeimyEDTLDk5PePZk2fEJN0sGUvOggWc73nz8k6eIxiPg1limgfPOTk388qbItJ8jTKmkTlwEoxN0zTcv3+Ps7NTQr5E4dHTfPv2eXP7tW7c4D9fVL/9mLjQf554fVCjXxPRvyjCv2lsfHF/bq8bbv/O7dciz456mIX8Lz7v9feezVfza99aF98eX1oE7pvP9OXXv/kM07id8DNS1pFOufn1Z/zMXFDICYyRApXoSvP+/vUFqXn7yiJ6YS3OCQNQx0QMBqXlhpZHEYbKQi5iug3oacIkPNxZSMxTSBmHKro41meBAXJM6HlQ5IzW8rjRIsZaC0bLv82VMqOUcMkPic5yUGYeXEqZmMQtrNWMclBTZU/at0wEqyWV1iiwGgqrcE4qUzElfJQbkNGCfTEOslaUTl4jBGEh5faaqz//73j54Yc8+PXfQ2VN358ztHsMiUVV4Mq1fONlg6uPCLHCLe+TizVZHXP64FsoHE1dkVOk7Xuu9x0+ZsYYqRrH+viYD37n9/nJpz+hclu6GHl6fcn+ytN2njFGTpc1ZeV49PCYZy8uGYfEUQnvn5V8+Ou/zU9/+hgbYLPZcXSyQhcFTz79GZcX0sIohQ1DnHA3TidMDoRhxNTQFAWL0rBwsLAaHTM65sMFSY45NFaxLET4LnVgofVhIKcsl4LaaOoJDZJIHFWaeyc1d++vaCpD7TI6jvg+EpwIBUprBg/rUtNoTbTiOCmM8PSNgZyCBNdGRZzCHbZJLj6VVtxfGr5+pjlZGHFlhEhp4WRd40dFxcDWeo6X8MG7S87urri6GkktvJp0ros+EVoYJy6/1SLchunkLrRibTSr0mK1xqrEUZk5Kg0xJsZwK1hCQcyZPjBVRxP3jhT31w6nDZtdoB/lAryqNEcLw67/1bzo+35E63C4mM6ip1YajbSHjSEwhkBhesE7ZEGv/LzrzJtu2rcduFO9U8SreRKhpFKYMvgY6XoJC805IEGnZqqYzs2ews51WpHVSCg8cXIgzsIk07vMrWoHffLnCKvTpeeAq+m67oBLuLy84MmTJ2y3O8Yh3nIsZqqyIitNjOKyzJN4Lu3hwqVUE9bkNZfy7KCwJWVZye/mGe2iKVxBmgKPQkzsO481lkjPGC8F11FWLJqahw8f8o/+0T/i7Xfemj5/hik6SdA3mcWy4vf/ve+xPlpztbniR3/1Q4ah4/j0jOcvn3O96RmnfVwtFiwWi4OQdNuVKT/nAEDho8eYBEmSPMMw0vfdawsxCQodefz5Y46OT3n8+DPWRysg44NntVqxWq+4etUSgrBYb5wyehLgvIQvTveNvu9FNJ06BkII4laejrG1lhCl6PsL74lzseXn/vPNL992o/+8ienPc4l8+flKJiFTwU14bzdFRqbui192M85M7f4KHw2JksVqyXLoubi8IuWIMre50XOosDoItHOYjjFzyGxB3+/xYUAruW5qJc9TSk8hjhbIU9hjwDpHXTcoYxk7Twji5BgH4bSnLNdzgyVmiChZNLuCunSQwU/tyMIedcSUGGM/idpucqOMB0d6VVWklET4T4miLFisloQp9Hd1fMS9+/dYrhaEsacfJrb4cokxlt2uA9VTVRJiNQ+RnDPGusnNPIVwKSnMa6Nk3qegKAucm0Ruaw5iv3VGMFgRjBPu+xgmVIuSMNWsYPQjxhSH68T8WW4XOudAzTQ5/AHatj3goubnzNeqAyZqlIX17ecBh+vdTdCnp+1aYg64Kdhz9J7dTtAeMSXiJFjPBcwQxL2aDxPWqViu9dSNEwhRzu3SFYfzBZhaaGWCOwdIhxBgkND5+VwahuFwDs7dAbPr3hgj+zYx3tu2Y7GYQkSBEEUcF+xWgVbg/fha8KrWirbdMQwDxhoWi1quN9OxzjmSlbSEagOLRY11BXN4b0xG3HJ4rAWVAj6M4EdxReWEyoIz+VW20/ujcIwt+NGiTEHvPacnJdtdz/PLT3AV+ACP3nrI0/NnKBVZHRsKMzIOgbsPC6wt0Vbz3nv3OTlriGqLKRtyTrz1tSUxekIcgZGHb53iNwYwbLdXfPu33+Fnn3zMnQcNfoSjozX37y7o+y2ZJdYq7r/VUDSG3XVH6qFZdew7z2JlURouzj1KJ9bLgg8+eMQHX/8G3//+J2yuLynrirYN3D074S9+8hyrSppFxeg9dVXQtwPLpWZ1ovBR+Mhzm3CK8PzZJffu3Sf4HUplkd9SxJYFrnBsNj3BR45WDUYrspFg3a5THB3B6AfWRwv2+x3kxOXVNUUp4yzGBDhi8izXx4SE4Cj2kaK0ZBVxhaNpakJwXF1vsBYWywJIHB+vefninMUi0rYDZVlzdXXB+mjJ8fEJH3/8U7bbLWMIZK04PjkhxsR2u8GHDoWEH2s0fbujKkuKssAnKUjsdiProxNiHMkohmHEFZb99Z7yVPjmUhj3KI1gZ0JPsyhJeZCCuC7I1mFNQ1FWbNsd/TCSYqTvWnaba6zRWGt4OTyjvdxhBstpecZxfQIEmkoQCgApS85MUZTEMZBSYLPpcAsYU8fxnSN+79//O2yGc55sPufTx084PluwO98TGfnaB+9Q1ks211vaTcdqXXO9a1mulsQ4cnK6xGgJg3OFxVaOZDJVUQIJwTgncgiUzvL0+ROaRUMKEWXEAZlz4PLyFSklqqqmbVuUUpycHNG2LVpnKV6crVBK5qNNI2HP1gqyCGMY/a/QaRI0hanQRtNtNoSU0HZiulokHNlqdKVJRtZdPkeZP+oplLuWbgKVkLwVramrRnAjwaONxipDJNENAzoLpkRCXDVlWVHWFcvFYipY9oyDMMyVyvgUGf1IDAHrCkDyjEKEbvTcsY6qKNllEdCtdZSTscA6Q7vf44rM4Af2YZB8K60IRFAwpoHr9pqUPCprnCsp0BRG/s04cUBDRlmDKUr2nSekTM6Gbt/jspq+38y+a4l+gBCF8V44ktI4ZaiLCq0zPg74qBjaHqM01kxijJbVitEGPXVEloV06YUERekw1glWJwWWZU0/Ktrtlm5sUaXCKSNZhMxio8I6+V5cYYkE/BhIfhLwtWBiyBaFIwdLjiVp0IQu43eO2CrCfiR6TyaSpvUnE84thkiOkRjCoRt2ngeEydgjnWC38Gm3xL6DQHursHLbEBKjZOKQ8wG1mRC8SwZsYcjZo5XFWIWzElRalI7RD9SrkrIpKRcF1bJCF5GiLGiqhpQjw7if5sWZGIIEnFspunpmIV2TUyZMDnhj7LTmE7axMe7QPafQdN1UpGBaJeZM6YR5XziDIjNOHXsqJVSUuZcPI7ZwNGWJ+RUL4AfxfHKhZ/IBXTE/xmEde2OquXmByVOcZ8fu9PNNC5mv4He7tTQ+rPFef8LPXyPJbqrDz+wDpSspiorr6w1KO0JIPH/+CtKClCOb7Q5XKu7fv0OIgf12z4Ov3+UnH33E9777Xf7W977H97//fbbbDW27P8zrP/IZ4xynd8748Bvf4Jvf+iaLhXReP3/1jI9+/DGb6x05KUY/yGHRk1MZ8HFel8haXU3zv0kWOpixgEOH6yyuz12dElYvnc/7fUeOsF6uqOuSlPe0bcvFxSX3HpYInnZaM76mb96SYPMNl/yw/jsIxrfF69tHXcyEN2L2zZg6CNbz32ed5dZr5jx1X09j5ubx6c9ZjJJfevwLrzHn2n1RZL/955v9Eb1kRrRm8S5N/37bZKZAJbL6ghs8v/7ah6dOjtx8+PcZyyJi+m2jmqxzmdYmErQsmotgYuX70V/FH/rVRXRjNcZZWRjrhCs1SoVpcS87KS5BcYXPbRFjzFLRykrEaSMsr5wUWgkqoKxLhgBGhwkBkkWAVcI6zpNfNeV8WCSCXLjV3M6RRUw3SqGngRInZnpIijgJ6kPMOA8qC2OanLFKkbQgaHLOxCQCnDja1dT6nYlR9lneh+mEEzexMZoUmV4jkoeWcfeS62d/SaE9+89/SPfqiejmtcGngFEVVMdoD04fYaojhlzjlo94erXl4599Ql0X3Dk+4eJ6y8dPXqFtxcM7Z1z3HW/fO+HBu+9Rn9zh8um1tOtpTTd4ej/S1CVNbVkuKgptefv+iqfPA0sHTRH5nb/793nw3jPc0X3qt96nae6gVMEnJ3/Fn7z6Y9LtSnUGlW4YUEpNyB6VUFmTQxBnNQrDlFWRb5ygTgvWJWUxCJfTInecknc18u8aWVyiwFlFVQpCyDqFNgliJEV/OLmZ0F85ZzTS6VBoQae46acymlEJJ1NNk4u5Jby0ipNGc2ddsK4sKWX6weNsYlVbRqNYFh7tFSdLw8lJyeqkIsbMcW0w3k6BKnA9Znov54NRMlYSMrGtjeLEGY4LNxViFEdl5qTWh1Yimdlz+Pw+AcmQU+Rsrbl7XKDRuJTZq0xIcNxojhuHM/6rnspv3LpJkJjPq1lYMdpglBVcQhrwMRB1z+BHYorM6eJfvNb8YtFweg5MVUZz674/XfDQpJgZRk9KgnORlwwwQXo0EbLF6ChVNuNE3Di8+oT1OFTzb4fMqdf2cd7mm4TWEizhJuFvGPxBoJ0xAZmpBbKyN5iTSUAKYYAUCTHhXEGMIirrSeyDG2fjgeWrZDJurT3sk9YZYywhDqgZnaE0YwrE1DGOHqcNTlvOTs/41re+NaFcGubWLLgJyFACRODu3WNC/Bpt22KU4d/+2z/i8uoaV5QsFktOz864vDyf8AQ3E6oZuXA7+O8Ge2Om726+QX6BsX9r4t22ezbbDffu3ePx549ZrJa0rbjRjZGgktFPLagIisYHL9+7VijlDt9lWZb0fc9msxHW6VSUExeziJcp66nw8iu4wm5X3v+G2+3i0fz3eUuKaYEwieWze+e1Nrlferen60nGOCsFvcJx7/49rLOyoEyBTMYV0kkQg4Q9zuL5LKTuJ1a3oAbC5BgdpsmmJsQJr7Na4pwRzIkXpErOmbpesGhWaGUZfTtNdjQhZmnpT0gYb2ExpZWQqBjAKGwprHK59ygRDowiR4XJN4WoLzqw57F5+GkM2liquqacWnHHJGIomgPOagwjeRTWpjbikO9HEY2UNqQYGb2fkDSTIJMS4344OLHmsdJ2+y+Nm3lCPk/Kb/jfHPY7xIgfPWRzuCbcdkbPj82TfD0two3Rh64RY8whx+C2iG6tRWdFnM7nmYc6/5zPaxHvNXVTgs6EGFFKszALTu6cQYauH7i8uiKES/kMWuFjJKSI1gllDMY6QtyJADEVWKy1ZG6cPkVRTEUCEbO1MpOQbxE2f0RN4w9uRPSiKA5olttZE2o6z+bw0pmNr5QixPSakKC1Qscv45bmItzt0NEyiytNGSVhodFPHTsNRVGBsqRsycpSpxqtPeOwp9sNdO0OkyLRKGIcJbDxV6mQAeiBopqcVHhspVnnwGb/hN/9vbe5eLUj94oXn2x5+92HdHGHbmBMO5bLUQRYpXCFY7GoeOdrCzbbV2hTYlSDcRqtIz70xD5J63JQnJyU/C/+V9/l+flHdOM5uvDsW884ZIZtS1MZPvi1ewyTS3+5Mnz7d89gbLizvs+/+aN/i1IQY2a5rFmtDG3r2W07ykKzXtX83b/3W/yzf/4/0I89r162NMUxv/4bX+MHP/wRdx4uefDgIZ989hPu3D3h0aM7PHhwhraWf/n//RN2rQRbV5UU9l48P0cEVEHw9F4wDOujY643L/FjxHthX8cUqeuG+/dXvPveffq+40c/eszJ0RpjHfuLDpRhWTnIGR8z3RCwZcXmakPV1FxtrzkuG5ISPFTTHPHs6TOqsmS5rGlbcTGv6pV0M3hP1w6UpePoaM2rV+c8f35OnDJGwr5luV5DVsSYyHiMyVMRM3F1tcWZAq0CKY2YoqRvO+7dv4v3kb4fQI3s9j3OWFTOlKWhLAqOjxdoExmGDlTm7OyIi4tXrFYzs33FfjfiRyhrR13VFEU5rX4Tm+tLfIxknbiO5wQ90F501Pcdx3ZFP+yIfqAqF4ICIzF6QUKNIeFcQ9YiAnl6Pvn4E3we+eT5TylWJaZIFGWJsZaUFMMYubrcYXJivTqmrkpM0/CzTz+haYTtH1Pi1cU5q9WSNIIrSjwBFaWYalGMXUdRleyjZ7ffUJcV1tYknyZm/4g2BUVRUDc14ziyWDSMY88wdIJ3QXARk8bHMARiBGMKfPZ04/6XPrUlJ0WLoUNnisqiS8EsAZiccU6jdGbwnRQblRQelVIYq1AqknMUlGTOxDFAgmHfMYQebaDv98SQGcaAM9IhNU4c9JPTE4yTgkTbtTjrplBgKyGSGkLOBB/wQ2QcJWRTazFNFXVFWcscuA8jxaJhuWxIiik7I+GDFDqTM/iQGUcvoqfVaDP9X1fkmDGuoClrlM6UyaKtsLObqqRwhvXRUor6aErrhH++3UnQOZkc5VyIfmRRlriyBGOpbMW6WZJToB0SZXPE1fYKmw3r1VrE7RAJ+w6DQyPnjqD6JBgxp4HgPSln+pjIU4dqjhOaxWcpQKCoy5rSOJxRxNATfCIMPd3YE3MmjVA1NcZmUogMIeEHiD0wavq9YrhO9JuB0CXiOOL7gTH2xFlYm+bCejYcahGH57nEPCe46bpUk4lH7su3ERJa68mIczN/ve1QVWRyiiSEUZ11whaC5ihqS3YBW2SaZS3jIXkWy5p9B/WyoVpWrI4qTJFAR1bLFVVRst1dQ04UriD4SNfuqKqKomhECJuyvIIPkJnmrHaaU1R03SgGjqqhKiU/6fp6S9cNEqgbw7SGnYJ2S1mHacDHgMmyeizrCtVnolYUVrra8ldgJv+iLZNvliTq1mPyh4PoKgsAw5sU7PxFAT3n2TL+hfeCL6/Ubz9rXiNO3+3PgcTqn/OZZ4l01gOMlS7G6+2G0Wu2mz3bXUtd17RtJqmEdXB6tsJYQTStlwu63Z6qbPjwax/wP/53/z2fPX5MWZesj5aMY8/J8Ql912O8o32852ef/pTnL5/z+7//+5ydnjEOnpSgKhuuLq4JaSDnhGa6lpLJGpJW+JQnFrbg/1B5ci7fHNtDdsB0Tsic0OFcNc0ro2TuKEvXDRSFI0TP558/4cGjinpZURRihhM8pxhL5fVvjt/NYZU1e54wnr+os+DLwrY8N6X82t9vzFxfFudvh3TeaAbcet15n74otN/8VOr27795n2WcTvpovrluzKL3Qbi++YRkIlF9OTPoS+ZMplBSJvTMrEvP+WLcXLPksTSt2bRozZMTPaUbc++sR/1121cW0bVJh5uFQWELEfxizDfVtGmn7VwpV5J2rCdwsdGCU5E27IizkcWyoG5K9n3AahHE9fS7kCY5LRNJ+CBfougwaXLyiLCuEO66CCxyIZC2TAiRgws1pISPwlA3SQTewsli3OqZTa0OQroGYlD0vWIcM9aIAKrmwacm0ZeZXi9t3rbUvPjkB+hmwenZEj/uiaWIsjvvud6cUyzex5RnxFijzYpkF3hOeHHV8X/5v/7n/Jt/++ek2HHv9BhXrnjuHW5Z8b1vfp3vvPeIv/3Nd/nW+2/x4K2v8erxR9R1wWpRcbVt0bpkdbRgebyiXtasFzV3XjQ8e3HJonbcOV1jTM3Xv/vv43OJPn6LzdWWYbvh2Ucf0263FCZx+CryjRApg1jaIIyeLzQyn1aHE3Aa7NORcUZhlJ7c+upwsc/k+chN1WC5WGk1PZ4TKkWIE7soCihcTUMu50wMmRhFyJ7HYb61Dyqrw37FmAkkYpIZr0E4+oXVGCPjwZip2KNF/JdiAVSVpm4cRWEwSlFoQ2UtPkeMFueT4FnUNB5FUkRlGqNZOENl1BToJ++lsppujXr6Px/2wSgwVjo5mkJTOk0WTUnKSnnmid/gY37ZTcSgPIkQcoExxoi7BQ4V/pgyenJx3cBcbirfXxQK33gxJd+6NN3w41EzG1ozY1tSFMECJQKa6KDSUZIR1wQZVM6Q4+Tcv+GsMwvo6mY83Ij+XxbRZZ+F/Szi7DgFCAoP/Z133mFzfcV+vz9wBUXgkc6arvfE6BnHHlLAuoklOBVuVM4HXvHNDVnc6sFn+r7H2jgxKW8EQWs0RWGE3Zkn94hS1IsGazSLxYpf//Vv8/f+3t/n5OQEN2GRUk4YpnCKicePSmijefjwLs3i97h79wHNouG/+W/+a0KUY6iNZn10xLDfMwz9hMgQLvQs9s1iXFXV1FWDc+WEYIgH5yvwmlg1/45TmuvrK/btjpgFDXF1fUU/9LTtnqEXd/E4pgnFcHt8T25+DNFKaKVzjsvLS95++xFlWYjjNAdSMqQ8u4XjGzl/f5Pty21pf3N1+8uCOhyugFndmlTMYrp+09z5K28xjFRlIeLQODCMMqn0QRzbanI71U1FUy/ou/7gZJ4LPLfdyfN57X1g37Z07TAJKhC9YAGqCV0RpzbpRbM6jPNpBkuMCW0RLJc2JKVAWZQ2lE1BvawZxp4QRkbfi0u9kHPa54HRB1krTIfyNnajqipugrHyQRAOQVzVguQYeXn+io9/+lOO1gspWGmZYHk/ErxgBpx1jH6c3N6ZcgrhnQtKOWdB2WmEPTp1rczHKkWZq9yc60rY3NM+zSHCt8M9Z4a7sobClNjJ8eKcIHNu41dmfIuEMouAYm+JzbeDOOdjEkLAKoO7xXucnycubH1w6lhrqZuCbDJd1+FH6SqomgVaWbbbHSjFbr8nkbHa4IeBcepCKcuClCq8b3BWWnyNtuLY4+ZcEBxNMbXbSheOnVq05Y6XmYNtb7vP54WOHNvJtQ4HwX5mxs9M/0SWsHMjgrpMmOfHZDYiC6dMUUh3kBwzKQpooyitxWno2kgIHqVguVzgigrvMyEqrKswzqJ1pN0rchjxg1z7losKrTLD0Msi/lfY1ieOroVhyBibQAeqRSApOL96RoiRu3fXWHtGsSy53HhoAz5F+r7m0dtrjs4cy0WNHz3PXp2jTYfNA7ZY4L2M3f1+j3MWPd1Pot9gVGC5qNgPF5yclJyerRj7kv3Vlr69ph97truWslrinGa1jGyvX3Dn0V3+7j/8Jn/x/Z8AiqKoUdqDjhyfZMZ0TZ5aeb/566f8xfd/yvvvnaKTo6k8n18qPvjNkvff9zz44JSj1UMMR3z6yQs++uhjsvJoEkVp6XtP4YopoFy6X/ZdwFmLShprEHTLRrIg+sGzWh1j9IKcznn+/BkZWK4quvEKax3vvnePTz99TlOt2LWR3svxfPrsKUdHd3j18pw7x3fo25bFoiFlyZVxzk6dB2oSPCPdsMPogpQ9J6dHbLY7Ysg8ePAen376CVrJOVCWBX7oSI2jMLDb9zK/MFrEw2XDanlC01T87KefcH7ZYqwmRzAmUZYOHzLrxSQUbjs0sF7WBN9jVOLenQdcnO/YbWC/hfWRQ6vM8dEZ7f65BC8nj7HQdwM5ZM5O7vH5x6+4f39J9pmQEufXl5Sm5PnFc+4f3RMcZD9So7EpMYwDPhn0smRUA2apUFbxo09/yDh6np2/oI8t3gzs95G2yzS9p6odu3bP0dGaGD12EkGTUrTtnqPVMX3fEryn3e1YNzVj2wGaqmx49eoF62XDYrVg1Sx4/uw5o0/cu3Of6+tz2v2OuhQkn3Gasfc4Y+hDT7/rKcuKt+68y6vLa1LWrI6PuNpcsN2NxJRYNjUxDRRa5iM+qQNa7JfZyqakH3p89hQLh6stCeHLG2WpJjNBDJ4QEzGEg3GptAWGTBgFiTitStBZo7Km0I6sDSHK/E6bktIVGCX5Ovv9jhikwImG7XbLMA4s6sWE2CrJKhGydH73/cCwH2SOqjKoRFFZgooMYUA7hU52EjpLxiRF6JwzfdeRUuJ4fZ+229O3I4tlw6pZScZQWUAhogdJo5Ks2xtrMXpEWYMtNVolymWNyopCWUrlcEnWBhoxK9V1jbGa6D2ruqZcLfERSix2wigWxlE2JbvdjpPmhLpqaH1HJrDd7VEpsKzXaCWIsUyi7/bE7Ik5ynmXIOoSYyx1XTDsB9qhp80ZjMJgyDngY8cQ95DAdz3j0KNMgVElzpZol0kqMjDgR01oNXFr6LeRcS9onaHtGfuRMA7EHPApkFMSI5AxWGMO4qfWCm51nr2OlxSN54ZhfDPxTElk1S+ajw4YCjJKRZQBUyqKxlHUhmrhWB7XeKNQJrM+rjk+OaIb9pRVgW0jRWFRKmLKSFEatDGsViu8j4xDkLVuzqQQyCkyo4GzMqSioChKhmGc7vcV1k4ZTsZSVQ2r5YKmWUzXL8n1AYXReipKRqEQTEXtEIKE5VqF70Vsb6qasigYUxQDRs4M8VdNI+NQwLgpZPDa8ZW/MKFr3rx+ef01mIT0r/De8/+HNfitf3nD72dmgsObt8NaHiAnYoLVao1W0tm13W6p6oU405XCVYbVUc3m+hKVM86UKOD46Ij/z3/73/IXf/lXjOPA/Qd3Wa0XxOSpq5pVc8QYIvcePGB9dMSf/MkfY63hH/7D/4AwBTEvFiua5ppAi4/SxZ2idEWGkAWzFMdDF2lOmap0kxlY5pSz+WL++2w4yVl43tZYCc/1Aa0MrnLsdjvO7qzou56u7SiKE7RK+FE6Fo1RB5PHTGFg0ihEm0iHQsptHU3+MnVTizsQJtOaaF5zV/9k/p2KKfmwRkoHoX4Wy3O+LcZ/+d9uf683esmt/TqI4fHW7/6CdbGaGepf+HxvFNGR8/srZYKJeqby1I17EMA1c06TrLtnx3zk4ILPM/udyYU+c9VvLS5/wfbVnehmEhO1IitpZdJJk7WwjHMOs3QpIS6zo2sOGc0iTIoAmDEqsqihqg220NONP09Ci+AsxLWcBM8xVwptxjlwWeP9zSCwBrTVFMLvkNa1rAgxo5KIm86CjxyEdlTGTIwclMJZjc968mrKv8siCwYvv1s6UNN+aqUOlVlZ+ImD3hQlJ+98nVQsOX9xyd33f4PTXzuhOH2f7oeWovtImGFKgW7IVQW6wOsj0Kf8i//y/8V/+y//lNxeksLI5rPHk0JtoVnyP3z+mE8f3edY/wPeOlvzzV//Dj/9qz+m3b/k5HRNjImIJFFWiwpXFayWNXVVAJn7q5K3Hj3EFgp75wMMjhAyOXr2L5/x8vGnhOipbBKMj9XkOB2zHKbrc5aCiNGHcEejlLSeRcFthElMVlpc5fOCNURxtM2hmfMlQ09icE7i7tckSJHkAyFLordO4iQSvAWQMyFKm3+cFsgZNXUtTKwrxLUdE0TywTmuEMxKYWQhpY0IvIXTWAvOamZlXmtFWUkwiTGaOV7DGGkHQc1tbHIWxJwOVWSloLRQW0HMDBNLfL6IpgxjEDcHZAoLjok/azQhJ4wW8TpMwYpaZ0yeJkT59QveL7PFLOzZmW0n1VINWk9hHAooyITJ2OxJBCaj7KF6/UV8xVQZmf5LzG0/c+lyduBy++kYFA6nGlQ0EAPoiMoGphAMpSKSEm7wyRCzCHAxiviuZJ44FVMkekT2Lck39IXK6fQ3KQ5pEWfEjetYLFbkbIjxHLRmuV5Q1Q19v0Upi0rQDS0hepSSAEGjQRkpj8zIAwnRAIW00xprD0J8jJE8oTKEIezJU0gpgCscRbnET07UetFIpkMOKGspq2OW62PuP7hL3ZjpegpgX7sPKBRGWbn1GjhZL/jGt95Hm3/Aj378Y168fEVZSniZdaBMpN23DONITlNBIie5njo7BfRkur5lGIfpHE+g5pvUlwemUkqCk0Lg+fPnfP3rXyfFSBhG2s2ONAZiCOSYSCGg0i3X/zT5yNGDceQUprDGyJ3TE7SS89ZoRDxQmqwdGAnxyUQiATUtlshyn7g9QflFwvibkC63j67cidWtSUd6rSp+Mzm5QcvMz83ZkaIlpp5EIKRIihozhWL9sluRNVZbHJouZfa7lseffc7l5RXDMEgQUwz4oWWYC0I5HwTi246lWejt+34qmDAxJ6ceBCsc6dFLC7ExlqpaoJRiHHvGcSBlKIoKa4WV7gpDURcYW6FNiTOa5WLBerlk32o2W884RKxTuKJEIS3yPnqsthSFAwTDlXOaRGBBp4QQZQE2jRvf96jSYkj07Z6xbxm7d/BlIXMWMzmvoowhmQkEnNU4I2L3PDqaqsDYqUA1zX2k8CXXuDi52J0TsVj4tRlrHVVdTyYEcXtpMwvG+nA9AEElaSWs8ZQSzgpjXik1oVhE0HZOunD2+x1zANJisTjgqHLOBzF5zjRQ6eaxcRwPzu6qqg7CfM55Cnqq0NZANijE4V7YEoWmsAVVUdJUFXlq6c6FI6aAIsr9r3CslgtyrIghMnQDOUSMcdM9MEvHX0xkpQSLNS1cjJoW/EqTtWC2lAIlFXxyjoIGKDS1bcgZ+r7D5zi16hqqqhbGvY9EtDhRrQj2IUSyihSFmtzxaUKCKaypsE46L4Z+jw8jRifqUsTwoNVU4M/TgiKSUpiCZgNKVxhVUthKgnJ1JKceJqJvWWvI7pc+twGePz7lwcOHsNyCeklILY0pWNYFpZMx0PZb7ryd6PNPefuDNR/96DkoeHXecfKgoEk9z19tGAdF343EOLJYBd59v8U5w3J1xOp4xfX1tTD+laZSnv2+485yQTrvePFyJIw9z58OWDtw505J3yfadiDGgDEBpTqU2bILn5CsA+P5/MlA12bu3z9CZcV7758xph1DvOBo8RYPH7zLj/7yGda0PHxryXKxQhUrjk5GujFwducef/pHn/OjH/w7ckg0C0tZC+9ZW0vWmTFFrLYoJPzVFQZnGp4+3bJoEifrhsoo2ralqgpigGfPLhl8ABvxY2C9WrLvW2JsSalgvSp49vScxdLiGkvd1Dx/ekHwkYf37vPsyUuWywqnHQoJUFuvFzxc3mWzOacf5JoRYmC5XDKOnuvtJVoVVNWCx49fMg5w795aRLq+pW4KjlZLLi/PWUwdJqvlmhcvXvLw4QMWiyW77Y6maTC25umzS06PM0pn+k6KIAZL3VTYrDg7OeVJ+1wwEDHQ7Xv8ACYZYrBoXZGS5/Hjz9lc73jwoEJpRQgeZ2t2bYtvIvfunbJalnT9jv2mJQe5H9ilFBfj4GlsQ6E1ox+pnKFsarrU8/zyMZ9//IQ27FkuK4YhMhC47q7Y9R2udITR0ndeGLq7QN/vWSyWxJzYdpLjYa0Eqxd2xdB3pBAobYmrG3xS9PuO0lpCGKnrJZv9jjFFdNa8en5F0xSsVyvOL86nxbzjeHWH/XgF2WMKhS01f/Xjv2S3bYkxs+870FA1BafrFS+ePmOxqOUz1hX7yw6ly1/+5J66sGKUbrJsMv3YkXKkVCUWhzWanLQ47IsCrRT73Y5QlCybEmsNbkYzKrmGp6xxyxX7LtL7OHHVK7SuCWM8ICWqqqQoLIMXXF4Ika4bSRmqRpEwOGPJSclaK8p1HC2YskXTsG03hDhQL2vKxsoaJivKomQYe3ofCEOPsxWLZsluuydHJWI5kk1kjcXnAMaSR4WxgrxzWuOMlgK274kxUhpDXTnyWFLrkqNmQSbT+5FCK5r1FPI5DjRlgWsWtEOgTJpSO7ICYxu6sSP6SHNcE32CrAQlUyR8J677kEa8lSDsrtsz+D1MhdbCFfRIceP07A7WasLOkwYPWmG1RrmCpDTWGJlCxEBpHFlbsqpwtiBrjzIGlRN+UAz7RN4o+o1kR/T7a/q+ZRgCYQp810a6PMz02sYYwTZkWSPPHVy35xqzkHS7oD4bYECMVFl/QUAny7jSk/HLOcrGUa0ctoJ67VisG+p1Sa4aEdid5c7DMy6vZU5euoLCaHKMJDVgXH0okHddD0mkwna/IwQZN1VZojLCiG9ERF8u1WvGiXHs0EYyb5bL+jA97/th4r7ryRApGQNWG5bNAqU0m+0WpbKEEU80ghQ9KFnbkRO7bpgwkb/8djsU8pYVbfpHdetBxfTNTQ+oW6+gpkXExNP+Kur54bfnP0yvkdUtJPUXX0e9tqa7LazOrzSvpWah10+GrHHc8/mTn/J7v/8d/vzffcb5ecu9e29RLRqM1Tx78ow7x/fo2p7tdseHv7bmk08eE1Ngu99w/dElDx7cx489KMV2syeExKO33+Ltt9/m+PiEvt3xwXvvQFZ03RWb6xek1FMUhtSriX2fbjIjyWgRJEWrmfWsaQzdRije5qGLGAuu0JK/0XnBS1oJYV+tKrSWztmf/OgxpydrTu+uqBYFSkkOZCSQsgjvchS13HPybNANX1qHii4zHWNg7sifTYwqz7ZG0SjVJLTPjnb5nMLpULOIPb1wvr0efX1kvPb9vzY0MqBmAXpe2/58Ef2meyXejJf5ffKbu62jSqIDvjYWvyxuiwwlyu1875qP02v1pMN63RzG8mxWO+gBE358/vtft311EV2LIKCUliqrmnZWiyCi9HAIMrVO2iVTTAeHrFKZYmJUKw1VYVgvNWVt0Xbm1SQURggd887njAWsyhilKK2iLjU2Z4IHHxUYTWUUxhqcNoRRdDdmJ7CBAoWNmawU1iqsnfjozDUcmaTceJbleBszNQnMzuXJrSy/d+OizSkTI9IS13u2F5f82re/R3X3LfzYYpMICra+y523CkxR4fOIpJdXsgQ3FX038Cd/+AeYi49IQw9o4fQpQ0476F/Rbp/x6fk9fvj+Gb/3O7/G/QcPOHnrfXY/2VAtHHddiY8RXRW4yrJcVjS1TE6tNqyWltXpHYqmRivHmDWJgK0X7IaRarFg2ZRUJk7fl5oqWCJU58nBb63w6mc8hp6EuzRHUmQOiCOtRV5OQmRh7paQKqhCZ3BaeOYSoCxOdGIi+kDyUgwp7eRa1JkcBCcU44TQiNPJMH03Il7KaTrGjE+SIJ7mEzZnCq0pjDjRrdGT6TPjLGgLZpz2zQhWpiittMQpJcUUxfQ+EqQap5NRM3/GJAHDWWH0zQVYTUWhnEXUH6Iw0LUCqxTKinvfGjmrbypoCa2gMHKBNEpG7xj/+hThX7TlPDvbZWxbayldKXgDxVTBlWMkxfupM+AL25cY47fdDNN4+KKT4XVn7txebwRvMqWDakBpi8oWmVVlEVWyBTUXvuRcNVodjvPti+3NW3xxmjFX8eed5CCuyJ/le1uv15ydnfHs+WNcWaLVnsFHEXlEpQclTn47BUuN4yDMwwmPMDOjvRfu7+2btIxjPzmv5TvJBBF2VIErKmyROS5LjBE3a9M0kA3Glrz3/vs0ixpj55uTOhRybgod0zFnmjTrxGpZ8eGHX+Of/pN/Qk6Z7//5v6Mo4HrzApIwFq1V5CT7rfUUTKjVQZCbcRI3HMV8OHaz2DwLsEopXCECQNd1vHjxgqZp2G42BO/RSnG8PsIZg+J1x684byfMhFZUZUmKiaP1mpPjYxZNzXxl0WoaS0kWj+rAj7sl7t+apN4ew3+dw/yL4/wwYTkM9JuOqNt4iPkNb/MklZrv8loWoySSEs52mirzv8D88ddupZVArRQzhRHxsO88fT8eRFKjxZm1GTYofdM+1/f94TkzFmQW1Y2xlGU9uYZl8uwKR900FGVB0Eyir3wnKQ7klChdibUN9WIp57HOGAu2cBhTYpV0LRWuIJWJdt/hU0Zlh0riVDdKWti1nR1VGWvlO5lRKPMxT9PkWWvFwwd3eev+fRlPfmQYR957510e3L//WuGgLAuUzoyjOIXFkS9hbHOw6u1jMj9WliXW2tdCKmfn+HyeKKWp6+V0fRgPeQtybVAHFMvsOocbTnlRFIcwypSiFO2NFodrjFRVdcCj2KmgMZ+XtzFRSinU5FaZETGzoD+L6cIml4ClcC0LuBgF46NUYOg9OUuxUWeoXEmfW/pxRBkjgleYrmlenL9VVaPRdLal23fELJ2HagqEknumLGgk3Go6R4w+CPtMRVClmIoYsvhxhca5CeGjAipHtJbus3n+4qMHZQ/XIcnckTbQnKew3SnEOiWD91MBNkkRLMXI2G0Yd4HCWVJOpODJSq5lwoZPxBTAB0IYCG6N1kwuOCes5DjQ1IXw4H+F4EGApy97vv/9H2DVgr/1t79BUXiePv2c4zsaP9REr1k2CfQWY/fce7SkKO7x4x9e8t2//R6Xm5d89rMrXr1MxJBJUfHwrQW/8dt3qJcSKq4LQ9vtsBUsjyuiFwFrVaww2rI+OuYbX6/w/ZL/97/4MR9//IR3Hr6DH7YEH+j6KxKJfbvn7v0TjMls+ivO7lbsriMvX+14/qzjg6/d4+JC8+jdJVUtPaiLZoUyhm/8xinRZ370V69wbsHYNmz7xGfbSx5/+orCaboxipBExJRz55MU3cbec3a6YBwDZVmz2wYePriDytC2HcPgcaWhaioZt34k+ESKiuvrPVkpHr19j/Pzl4SYODo+xseXdEPAVZG6Wch1LiWur3acnKx59eolWddgIMWWO3dP2O02DGNPTImmKbCTsDyOEgSpVKKsLE+fbViv1uz2e6rKURQWZy3n56+QtYhmHANNU3L37h32+z0hBC4vt9y9c0bXe6xTVJXl4vKC5bKi6wJkT87SmTR3fFlXkolcX11gzJJh3NEsLEWhOT/fs1o1lJWjrAwxJ7p9x6Ovv43vR7bdNYGR/RAJ0ZNioN+N1LVjVa8oTYWeipCbrQShNqsF4zDS546Xz5+w6V5Sn9QMuuXjJ4958PAtkg5845vv8sknT3n70QOG2NJ212hlp8JYw/n5pVwOlGIYO4yB05NjXr4cpXPJmkNrdt/1LJYVXb9jt9vJtdQ6FI6rV1fcOX5I1ci1Ybfv8V5xdHbG5vNzdvs9R+s1xgYWS0dViVnp6mpDDILRats969WK7fQZtSp4570zrq43v/S53fmeorSydgqJ3EMaxBw06ABTp6PK4sx1xpJzYkiJPkR0AKsqlqsl1mr8OOLHkaEf5D4JoEvqphIhSDl6BpKPlHVBszRE9mQVaZYN7RAZ83TfLaxcw2yJzY6sS3K5Zxg6Cbi0GVca/DjQjyPLxQqrK3zn6bsgobMsSGEkBLBFRd+2jP0AGVLMErKdOpamwekS7yOmqDHOkXwPKqJJjH5kH0HpSowsRmGLAYyitCsqbdlsM66qqaoFiUR0A1VhsM0C1MCyqKit4frqXO7BfsRWFsxIN/R4DShLWZW4bAmtZwgDY1RUWUQzlWQs+mEkx0xEHKvDfo9DUSkrhq8o89OUElEnysYRBXzOolkw+Mx+7OlbDylgUoHzDeNe0195/PWebtcKhiclrC0FQxTzZIa6lX0yG02Y1qZq7gqVueVsbhP35YQ6ndYzel6HTHiIkKbu2Imjqg3YQgK2F5WmbhTFymEXluQiR3dq6mWBqzRu4SjKCmUMp3ePacOWNEqmRepGdIpokxizdJ/660i7CYQxoJVn33dkDK6s0LZg7CNNWRMz+NlwpCPaTCghV5FCnkJME1o5hhH2bcLaJShF30a0cizLteB5TElSCZU79ruWRVFTLSopHsXJLDLlOu27PaP/1TCqt1eit5affPnB6X/15jW3Qk06+I0h6E0C4JtnGtNYmNdB0+/fLIFuvVD+ooh/++fN83KCkJOYlZTi2YunPHrrjH4/UFUt/+l/+n/Ax5L/7P/+n9OPmcXiFFfWXFztcE7xv/zH/5h/9s/+OZ88/hQfI1VV0PXDQcwevaw/zl+dMww99+/eJY4dH//4r8hkrs6fYMyItRHfZbLPEEDFaY2QFSoFSQubBP98C684z32/qFXAjUA8jnuUTmDyFNitMVozhh6tAt/59vucnVYsy3vstzuCHrBOkWIvGTk6kVVg7szN2aIyaBVQakSEccUcHnr4agGynvSXWxrkYQ8Try9fZ1F5RvR8Uf2Yx9eXNRPe8Mx5U/pmvXUjWM9dKT9/U2/482tC92vbQU18fU/eJNIjnVboWW+Sn/qLmtRhM4f9vc3/z1kMqjcs+F+8fXWcyySuoWQxoZUIyhmpet7wtDTGZJTJ08p/DikDZ5DFslWUpUY7mXzMHzgn4Vb7KJ7V+X0l6FNhVL4JF03yviKSiyheFhqjDAMJjzjIpeUHnFLoUW4KwvMW79wcJEkOuIywTNPMURVhXhtNlumGVCGVvHaICZPjhKqahGQ01lYcvfUhPhkW2hH2G3wOdL5jcfaAQt3DNo4YBrTyFHYBWJSp0Dpi04bYXkhQB3YSvxJmGk7Jd3TXL7nYXrDre5aLmuXxGXZ9h5Q8JYmzpiTjubq6wrmCwlpCSBTW4HOkOr1DtT4m4gl9kjCk5+f8xu/8Lu+//z7/z//s/8Ynf/GHgDjgfEj4JE7ozIw7kRuvzxEfbxhs8/iVkztL4KuWB0OU7zeTbyXhToIoU0hZFsyKMrLAluq6LNpQ030kQY6JPPGMQhDBPE0CtlVMyI/ZkT4J6hN+Rirowk13WooltrCkKK051koFP5BEs1WZsnS4ib+qtZlOWhHBQ5rfHyBPQu5sZJ9PZg6u/YOcdmvffMgYLRNkyRiQcNus1KHCJmnC0rY4X15Dioy/YkVcT6KKmc5ja+xBdNFKdtykfKhqKkl9mxYwc0vRl7c50POLDvXbwvkXfx5E9Ol/i5UQScRFw6GgAFlrmEIjRJecJxD5IBxLG3/itpCZD27hm+1wo5w+irQFzhNSg/fSkn1ycszDBwPbqx3j0ImQkiXw0uAmAU7ai2W/9IHZOyd+z4JQURQHQTXnRFGUdF1gHEe5vhnh8VZlTVnV1HXNN77xDU5PTvjoo4+4vLxkuTzmgw/e5+23HlE3JVpPwj8aY+ZmrzdvCjk3m7rid3/3t7HW8M9Ly0c/+Sv6fstu001t8PO5Op1zk/hGvGG6w+volqJwjOON23ZGuRhjqOuKoqyIMXB+/pKrK3Ngbo/jgCYdRMpZ5JtFeHHPJLSW112tlrRtK6gJ73HhdTzCLOxrrSfBar4v/ZyD8sXxcHs8f0WX+vzc+X1vhzR9Eeeip8ErxWMZS1HNQr+eJo6//PltCsvQdegkbb2uKMhkjBNhsnCOYkJ9hChcyDmkE+aODHMYw4IcstR1xdHRmqJwh7yAuioPHQPZGHKO4j5PCWMt1km4U4yZFOMkYipQM08zEJUmTt+htZayKKSgqaXNHoR7qfQkkEd/GG83QZuC+CnLEnmpSOEs69WKs7MzlFJcXFzQ9z13zs5Yr9cSoNm25Jypm4ayNHSdoeu6Q0jp7aDO+bHbE7GZIX/7HJ952rO4nvPrY3JuGZ0RLXMw6nwu5ZyFNT+Oh8900w0QXsPtdFNb/O1rzu0FwY0bJIlIHeKX/k2E/ptxm5E5Usq8Js73fc+MVVFKClpaa4ZxxBVOCrBa48fIOHq0yhTaUhSWum6EAdsPco+esiwOzjglzvOUJ+dQUIdjOrcs33SCSGg93KBcFAiX30nYe4wtztW4Yvp7ShL6qRRG28l9FAmDF5ybdRhtGQfPyDRndBmMIoyerrtmGNX0nc1OPlm8WFNgJpf70AdgkHZ1IIZIu+8IvsOPI4UtIP1qXNWjB1uqo4TvBn76SaJ2Z9w9+ToV8P0/+zHaGspG8bUPz8hpj1ID771/h6P1gn/5B59yvd3xG795zDc+1IxhoF5U/Ht/59u8uPgJg5eOE3+xQSuLcw3X17BeHzHEyHY70F6P6GT5qz//mM0rw9n6bdrrp/zoB8/55rcbLq8u0K6jrpfkqFgvHrJYHrPffsJ771f8re99k+urHcere6SoePHyJ3StoqnOePLsU9ruit/7u2uSj/zxv37Cq+eab/zauzz9zPPjHz1l0dRYU9L3HYUr5bvSGrJn6EeMdTBGrJXcorouKFzNZ58+oXAlu92e49MSqkRROvb7Lavliq5LLBYLtptXuKKg7Ubatse6ghQMtihYLle4QnN+sWHoR1arBScnJ7x4egEZCbFTwoY9OzthHAeUElzUen3E5nrH6UlFWVSMwwaloFlU9P2e45OKRV0QI3gvRbWj4yPabkdKkdPTEy4vrzg+OWK327Pfb6c29pqLiwtcWbHb7VmuToVjrxL3758yDJ7t9hpjNBfX1zSLJUoFLq5fce/uI3J0XGxeUdeOsztHXF2/QJvM2+/c5fp6C8i6a3N9Sdtvqeual69eslw21FXByeqUTXdNSUHsE85WoJWIVeuSlAJNsyQqz9iNfOPrX8c89dAk9nakPHK0457l0ZJPP/uUEDJuqbh4uaP3A30/8PUP7vL02bODJSAmCW8d+o7LqyuMMSxXS4ahl3vWkChrYZobY9jtWsbxCmsLVnXBe2+9jVGak6M7XG1bXOE4uXPE58+f44qS9WpFjIndtps4yz0g3UXGWXzoqYqa/abl/v27pJD5wQ8+ovCZ07vHv/S5rYyW9UFIgtGyklGUs3R6qXJCH04r5pQEpVeWJck5Ypbrtp0MMSZLlw/By7EcR9CaypUipOmMtgqrHYXOKD2yb6/QtmK5OmX0ll27pagEY2CdmOoKW6C0RZclSxJttyOGjqYpafeCX0lZnOw+Bpl/Y4hBGOo+KIqk2Fxv6PsB5wqapqGsLVkNjGNPWUkIMFbjakfQAz54bA6ENNIPEWsMA/00N0woW9KPLa6sqGqLraSryFgj84mUSaPHKcPReklhFNebV+zaHclklosGQbhckYuCEUVhSkpTELMnRY82hso1pFGRsjj+g8rEJN1lgr4VI8eqXpESjD5gbYGgQwZZU7uCssxUdYktNf56z+5yz/6qRfkSl1b4fUF3PTDsRvq2o+/7w1iZ74cz0kRrNZlYDDEGUoqSC6lmY+CUl5PV5Ly8EU4P8FatUSmBTodxplVE6SxF61Iwk2VlWdSKxcpQHdeoRmMaS7aQradaljTLgrpuyBjqssBZA7rEFoZNN1DVJaZWjGkkxJG2HVC+IA6RPo34mNFOus6HwWOzmE6Cj8TBE9OIc5qiNBRFNa2hLWMa6PpOOtfRpKTk2APDKJrInbsPWFYL2t2ett9hkyEmj6k1RVGibWK7b4kpohFTw77d/bVmm69whn/pkRuMzu3XnkX0N4iHb1pH/xxs5ZvWNPLaX319cYOOuXm9Nx4HBajEs+dPqJuCb3z4IX/4B3/MP/kn/5gXL1r+q//qv+bVy2sW6xOWizUxRY5Pjjg7u8sHX/+A3/yt/x9z//VkWZal+WG/rY640kWIjIisVFXVpbpEq+npEcAMYKDRjAAf+Mxn/nEEzcgHGEgaYQAaPRgMpnt6uru6RJbKShnS5RVHbMWHtc91j8ioqqwq0gwnLc3D3a+fe+45W6z1rW993x9yeXVBCIFnz57RdbsSRzucq+j6PYulkEy2m2u+9Z/9Kzaba54+fcKzZ0/xXuJoP44EHwpZLZf4WGJydQs3UYWBDrzkFTDJQU7a6FCA43LvrDH0o/gsWGtoas3V1RXvv/8T/pN/+WdUrqWdOZLpRG3DVoTYYTUiBZgVIJ0mwuyOB5IuB2RFPB5uUOeJf15yWV4eLbe/P6gEIDnWr3qeN395eHiff6SvI4+95jW/Pk/Wn/u91upz55pAbM0XmWMi8TwRU+FmTL5+zHN4zW0ppNvkttvn+HXHFwbR5TILYzsrSCLfIgS8ScdagCljbl9ckXNBAEFrFcZkXGXQaLQWaY2UEilHQjSMURFuVcFMabuZGEMixyAJuFGSgGstbCMJlC05BUKcmNGSk6UsbPFsFZOGeUwiC6IREYkbcFfkYawzpeVD0E+jJhkOxRgClRLguDwuUBpTN8yOT9DWEXxHUpr5+oRkPNEkdPTiLE0ix55kMugZKhusDnzzG1/jP/z7v2Xo+5tplD+/oK7WK3a7Hbt5TTIau1ix73bUlWGxmrHbnqG1Zl7XVErT1gatYLvZ8emzZzwae8zZv2GlW3ZDZp0y3dmeJs/41h9/m8tPf0QOz/FDkIQ3CGhqlGiWi161gLvepwOIPF2pLvfMKjBCxxYJgJgPxp6xsMWTEiOWkBRjijSlqmoK+1uRcS6jcjE29IEUZNGJUdyWybkYekqVbGpfyxMAnRMpS3OI8OqkmGIlX5cEviyUWokkRkwZH4X9bZxGWSNVS2DSmI6pyMWUJc0qhaEYfeV8MNStnMEaLRrv08KXb1e7yvUeFm4xU9Ux35ILmpgDk/mBjGkffj8Q3ZR2eaUmk76io6e1FPayxmQIyR80pnLpi5JC7q8AEJVojHPrNS8zc3nt74RpbLG2guTEVDZbcrKHanzKoDGyvZgiNZUjOYdDzXQCzV/a9Eub3O0g4KVr1kbY9rloOltL3wuLVtibSdYFLS2lsuHGQ8VTdKYLI7ZITBx0iItW8W1QVUAsfSgCHO4P6sBS937EGMvDL3+Zf/HP/wVvvvkl3nrrx/z0pz/j/v0HPHzwDqd3jpnauG40xgpA+Wv2Ag1YA+vVjG9/6xv8w9+9zdMnH7OcrRi6DeMgz9kYK9XsUvwKRdP9dnAB8rknDeIQ4ksg+gQoOudYLpd0XYcxhs3mmt1uf5CzGXv59ziOB2DspaDGTA7bN8yBCVTMt8bTbWPJiX07Xf+hTebXHK8bH78uSLgdVN4e5wcQ/9bvXnoAspGSUyblSCLKzMpikP17mQ9ahNmuxVskplC6i4QBrAvAW1cV1lm0udE9b5rmoJE9FYAmEHcCZ2OM+OCJIRCjzM+MPKMUpCAUYqCpHfP5HKUV3WYvhlOmgpyKmekeRRQ2XUkYdDEB1Epau0OQplbRFTSEBDkbtFaf0/yextt0v9u2ZX20ZjZvGceBTMKHgb7f48NCGNPJl/1sIEZFV/TTvfeHc07s8WEYDkz9vu8P7PTp3kzjdrPZCCPsYHCZubzcoNSNoeg0Rm8SqJfH3DS/bo+9CaTf7XaHOTYVqYZhYLPZiCFe2x7uybQG1XWNzgqvxgP4PzHhJwb99LOcM4MXrVdni/lsiuUzJ5yzpaBQMfqB/bBn9BF7IEJolDKk7Bm8P6zrs8USZR17PxzWwGleiF606D2DOhRpmkbMYIU5N62dlILPTcFJAdpklPLENJByj6sbZq4iUUxb/SCfs65RuWboETbhYe0Q0kfOkZwC1kZcBanJhHEQeS4qbGVR2mGsoXI1xgiglFJGMRSJmMgwDgzDQPACHA29J44b/Pj7dZGtjmrcHYtNc2oe8fSXhr/+tx9yeX5NiCNVXeFczdXzzBuPZjw9e8Fq9QlfevQ2rV7wlT/8Em++a0j2GUPaYyv46JOPiPSEtMePsuaPvaLbXXNx/gw/VujU8OLJJd1m4HhV8+DeXSobGYdLvvK1+/yTf/EOQ/yUXXfNbtej8ozoWz768IL5cscbD4+pa09Sl0SueHZ2SQyO5fyIREKbxG74BF2NdNcVP/q7DW11l7ffCnzwi+d89skF7733iKurK1CjaMsHS0yBRKQyEnsEn/FeuC4xJsbBM5utyMliraGuXYkNI8vlAu971kctZy+eUlVLGe85YpyRDhcJNri+3kiXgwmAIkXYXHvWq0DOARh58OAOMUcurs/ZdztiHFBaiumuspycHDF0A85orKmo64YXL84kbzKGy8szYozcu3dK1+9IKVLXjv1+5OLijGEYefzZZ2Qis1lL3w9cX/c0TYMfPXfuHKO14s7dI66vrzg7f8G9u/ehyNqEFLi8viLGDu0U17srVK6JqWcYRzabC5pSGD0/P6PvR4xxLOYrri4vCOOe892GWd3w3pe+zPZ6h+4Ma3eHf/UX/4q7i7u4WKNri8uOcdzTtEv2+46qqWirBcZkjucrnu4ecxEvJKdJHkWiacQE2oc9y+WCeO258+Zdnj1/xvXVJad3TmRORYdRmn4c6IdS+I9yT7TWzBct1lVUteZ6c0FdVxij6XsvcUdjeX62ZX18hNGO1bJmuV7x/k8/5I17S5xZ0+93BK9wtiGMsrYfre/x7Ow5Ih8xMJu17Pd7dts9q/Uc01ia6rdKs186tLH0fY8PkrGknEkklMkYZaib5gC2AAz7Dj96lusWVVjqKY7s91vEr8uRsSjnCL4nKbBGTFhDEPkjbRTGFo+RFIgpYhVUtuH0aI4iYVwm+p6978nViJ2JrGvTWhbLFdttxW53SVWV/Tdrri6vUUaS8pwy+/2W/V7MSlPK1NER/Igfg8iMWUfT1Cgj42Y/7rHVDIUiZjGu9DlAGun7nhTE82TXR6wVwl0edhgaLvyO+XKFNoEhBOjBZMnZnDABIQoZbBwGjNY0lfgBKC1yajFldqOnqgNmbklOkXpPHj0mz1k0M1KQgnOImqppsE0LQB8hjgNGGebtjHljcPWMYRy4upZ55UrMk4scLQPooUX1Fd1VoveQQ8D3Q5Fyi4fYe+okmwrodV0XEpArhd5C/PNeCi5Kl0I03DBVS0xajGbkuUmPb1H2pEakJrRBgPOlZb6sqNuKZpZZnDQsTtb0BJLN7McerME46fae1RVd7yEGKmNBRXJMmMqCNWQFla0IY6TSisVyzjbvye4OaMW+7+n6ns73tLZiNCMhT152iTR6Ytb4kDCmEiPiMLLfDYQAdSWSfCknMSw3mrqpWa1WLKoZKsq4XjUzhiD5REyJqm6ErLDd4vfdTbH+NzFxfuPxxf5eSEyviRNuoKmX1F9eza9vzvG6vOW3KwTIuFGHr9M5PndqJfHUYtEwOz1hv91yenqCUopnz54xDIFHjx7SDdDOFgx9h/eB4+Nj3nvvXb773e/wl3/5l5yenrDZXJeu0MRi0fDee18u+Uji2bNnfPe73+Xr3/gGlTN88olIKPkCnE/5P4Wg4b3HOhkbU74wxdWv8wLgNfdT6yl3L3mkCng/kpJ0PmkMQ6+5utrw8UefcvdLazx7XKWYzS0KI6ShlAp+AlkVY0x1uH1wQDIKSWO6nun6Dq+doPbboDnlu4kcOMm9fP55HwBnbr/55/NjNZ1vOiWTeseNlEz+DeMpc0PG/LWvmwibnzufOrDgb/+ssHCKrM3NPfqV7zQxfSnP96XT3d5Vf/3xhXf3lD0pGlIxQToANEzXLt9ro9E544O41iomp3CobcZZVQCvmw+Sihaz0pCjIoJoHOcCmJeFHFVkQZRoZFkDlaXodMnyYYzG1aCyRY3x4IKrUqmtZIXOE8A63S+FLex6eRhK2tGMxRRZBhk3hcmOumEO2wQFLFQIGFk1LfXsGNcuSMPA7P5bDGkgKQ3aYEhU2hAY8HGHdJhYYvSEsOef/vmf8G/+zb/nlx98JBPTjwfGhRyaxWrJcjlns91wtBKjnAD4DEezlqZpef6sx1mYN444SlC6bB33jhd05884//RTVusjhvMnDEmzY8XZ5YbtsGV3dcms0qJpkkPps844o/BGgHRKZSuEYjCbUnmZbNBWIfI5WgxZRTdczF1BgGyfRMpEqYwNIv8zxIRziqrWZVVb3AABAABJREFUtI1jtmgwFlwO5DEydoHgxShCW1101uW56AKi5xIgqSxs8JiFqxER46OUpADjDBgj1x48UtUuYCmqtLnFjK6ysGm0EVf50jIXU2IMsWiyT09nGtqFmV7GP0ra+yJ5snq8eQ2qBGy50NVTMa0pzNSUJZCe9Lwyt9juRR/+9zicc6TIAbzWWpj42hhslnvrUyYFuW9Wi2wKqYAWv8Kx+3Vg+a9ioU8bmVJi+KKNBP3Ji+mMUpaUpQCmtMzlnCe258Qsl2T4Nhg9geWH78scl331ZsO8DV5Pzw84yDkorZi1M5wTtrQxReZI5cL4NgVkTkXuyJbNVjbmyaRvMhKcgLVJf1DkXESbXr4vuu9Zun7aWcU3v/l1vvHNr2FNxR/90R/zxhuPmM8XPHjwJtZpkYIphoJT4KxugfPT57ph3cuGZiZzybbie9/7Lj/4wT/wxDrqqiEVkM/a6gAk3m51A14CLSdwT2th2t8GW6ffDUMv9yxH0oEdng6SNsMwfA6MnJ7HxLRPKeGcsBKapmE+nx8Y1BPr9zajVmthrvymoPE2YPnqWL593L6PrwLrry/O6F8BxJeiYvlZjImkpra7afz87hN8jIFslGjLhsAQRqyxh+sMXgxjMzCfzQilY0Kel8YYy1AMX5WSBE0pSVzGcRD2cxZdVe8NwzhSVQ5jdGHGDcX00xag+8a4SivRBjVakUNpMSKJKVZ5ZmLCe9jAIZfxpsWJPQQvRZU+lcBdWntd5eT9gz+wNjabDU9I7Pd7PvvsU66urjg+Pubq+uqldSm/KFrbt4K9aa2YmNe356610rmz3+8Pc2Hq5LlteCpdGyITMo3ftpVEW4xLBdCeGO5TYWg+n9O2LTFGdrsdKSUWiwXOuWL6O5lq3ZiCTgz6xWJR5txwkHdxzqFRByb6NG9zzsxms0MyrrWwwaumfonc5HCHdWwqAgAcHa3Z7XecX14RQ0Br+Yw5RYYh0PU9fow0dYtBo7Q53HNjpctICBWBjCoFXXtTMKMUf1R5omVeTKBLSpqYouhOFlCBItVnDFSVwdYVKctYnToANDVt44ipJWdfEkKFUuZQDI15JPmOmHvQPSl6QgxY1dC4Gm0cWpcEKWucrbBzMYwchhE/dqQYca4hJ9jv9uw2nr773Y0HARb62xBhf6X47/+7X/DxBxtmrWJ1ktldBrouE0e4eGHp9pF7j+5yfJQZ/Ya3v9Lx5oN3WS3eJOT32PiPCbzg/OIZygT8GLm6zPixI0dLio4wQI4DL863xODIybDfJsZ1xveBp2ef8Bf/+Vdo5z0Vlnp2zEcfPWU5Mzx69IAPP/oQt6hIakCZhk8/fc7FWc/5c81+A+vFnm9994Sf/OL7nF2c49Qxf/XfPWdmlxwtFffeuMvXv/aIzfXA+fkVi6Xm8ZOO5aoiDA7hLgXGIWBcwliDwmC0odsPWBfY7j7h4cMlzimRgtgPVFXF1dWG1bpm9Hvu3V9xdn7FYmlwtaMb9jx7nlgsDMEbNDXGKoYx0rbSJba93tLULfN5w3ze0PU72nlLXVu6bkvdGE5PT7i66hj7nuV8znYIpNYQgmKz6fBes1pVpBQ4unfCfrdju9vQNBVKZU5OTtnvt9IF6j27/eZgmrffdygla+LxyRrvPdfXlxwdLZjNWq6vtwcfh+vrPV/9yts8e/qMmIIQNox0/UaESHB5eUUIkbZ1DINnsZgzjBFXV/htR+0M63bOWw+/TOwtq9U97q3u8u5b73JnfUqFQ41F2zRrVJH4WNqai4sLZqsV/W7Lolpzvj0j9BEdDbWb4eOI0ZpPP33OvbuJfoy8+967PH36mL7vWB8taFoBWFPyaISp3O33zGYtVilCGHn33Xf55JPndH3PvhuxTuP9yHw+p6oivve41mIHeP7iGZDxfuTZ08ecrGucrnjx/PrQgaUQDdzNZsdyteCdt7+E957FcknykfOzCyrn2O89y3aF+YJA2euOcRyLKbVlMjsexwHnFK5q0c4RxpGspBPIOpFuK85MUpwe+iK3Z1mtj5GiZCLkG91fZy39MEhcq1WRkxtRSsg7NmmckYJhW9ekPJTCd0+Oo/iAZIPWFcoqqqbCh6p4dlSorFHaMV/MCSHQdX0Bm0eaViQO27ZhUIo6SCfw6HtcEOA1RkvX76m1oq40+25EE6mdguwInXTP+iHi+4G21jSLGWkUM+ghD7QE9v2WjBZWf1I4Y2laRUyey6szgg+MwZc9UzpAdl0nBYwkeeI4jvg24GY1sbsmdB1XF5nV8ojFYsmLyyv2Oy+d5S6U3Dgy7AeEbaVYzRdY7aDWNM2Mftwx9CPalE66XeL62Y5xV8G4wHhL7BMpDKW9Ob0UU5pCeJpk3vphzzB0B/NsayuMsWhlhEGfhPKVUhIC1S2EQQGSaCZhzqqCqxionTCzq9ZQLwzzdc18VZNtJNtAe6Rpjg3dvhMvr0b2XB87ut1I4yraZkaKQcZR79nudgKQ+54qF437lMV3ozKY9QIzm7PvOq6u9oTeE1LCtOLfUjUVm+0WY0WbPamEmWK00qXufQQ8ucjRTJ2zdSvyMteba6gSYRhJY6CxFZnILvTkDLZusJUTX5WUsNphsxRtfr/ji60Nki8IaeulI09fXgY3bz3JW+d4hYt5OA4i4V/g+nIB0G++wsvFmOnQSknlWokXlbWa1WpJVVVYXYk8Ig2r1QnXmy19t6WqDd/5zh/Stg1f/vK7/Omf/jH/9X/9f8W5ir4XY+erq2uapsU5y+PHj/nmN7/Bf/Ff/BccrdfcuXPKL3/5IYB0Yc2XjCqUQlOQOMxLQcwVUH2KLad/G2Nekg+d4uMp55V/S9Eg+CBEJK1RKhKjGPp6m7i46Hnx/AVfee8twgjJGLSqhCiVNCGMKDMZgkYgQpacX71mXNyUUG7nmTf5/c2vbzDL8iTKk8y8ruMgc8PG5jbR8HXdCYffv/z1No7ym9jbv/L3r/vxq59teuFrOi1uyjmvfP1NbPJbuMjtc90m4/264wuD6CEOhKjRscIUMfnp7aY2ielRyeOWqqrSRc4FqI2AbyJOkcjoMrgj0cvrtCnsVVnJxRhDieSAUUUSo9weazNNa8jKMHqRClEqiyFeYeqasiCbIu2StAD5U2un+DRmtFMYm4VBXG6sAPaJlBUxFjSUAshGGAI0VsBEMbQzJOuYnZ6Qk2d+tMYYR9aa2jQYfQTZ44dzgt+SDaTsSEWbJ4SBobtiva75T/7TP+OjDz4k+YlZdfMw3XzOg6+8y4uLM3bdnqfn54QM6/WabtjjrEGpgNEj81kFcaDbbrizbKjur/nuN97lwTt/gmqOiPUJ6o27DC/O6MaR9uQO4eKaFDPtrCHsMlYJwGu0VPddCbTbCmqriWECncoF5lwKCvIcTXEkBpE98VHY5zFnfII+ykbtnUzbIYl4jrUK64xo7DuFRROjKmOutOXI4xM9+lSKM9OCkCCqG4mYlBMxKwKFIQ5YLdIrmcnkcwpOQKkCXqksZppIBwJJEYKM2zEkYS+km0k71WdENU7GTcqZbpQWwz5EWgvTSM5ZZFycKTW8zIHpnhAQPWdVguibNjxjwFbCyvh8Ze63O2pXgS2qUrekVOBGgkYjc9BYYakbLRpgL4FPr6ngvvr9FwHWtTFYIy31k3xUzkpACpUK2CJMdPStzfylSusNiC5gdgExDyA5n3tfpZQYS0wdCQX4qqqGoR+4uJQxKeubgGdhFHA8JtGxvs0IFYztpk1sAqCdcwd267Q5p0TRRnalMDUZOkZiHHEVrI9afOip65p79+9wVNaYdrYozElZ23LO5CiFPq2l8+fVZ3IAdMmgxLzWWcV3v/Mt/vrf/wGPP3vM5vrF4bOQNbPZjBBE0kO0s6eW0XyQ/5jOnTMHYG9iykxMWJGuqViv12x328PnPAQ5BQCb9LdvA9EyJmXj7gdh3a1W65fYtLdbs26D3RMbY+qSet3xq5jmt4s8r47f3yTz8jpmyCH4KGOVnA+GJiI9KGM5IXaov+sx+AGlFVUBD0U64+aeDMNIt+/wS49znjEE9vvuUPSZjCdlPkBKU4ApWqRdvy+FLzGc8+OA9w6tKlKOxPRyN4ZIpiXGsceOAnRIZ5tBErdpfZNiiLGaMAxlPzZQiisF55Q9XkMs1yXz1aFLu2RKoYwNJzIzBXidCi5TcadpmoMG+W63JUbPbNZS1/WBeR5jZLFYYK3Fe39op27bFmstXdcdZFdms9mBwX+7gGaMZb8fSlJQH5jiU+Fiet2rMkjT1+Vy+VJA3zTNQQrmdlI9n88PTJvpZze64lKEnAoCcLN2T+c6JBVa/A+UVgfGIghTffrMk6zMarVktz9iu+vowiBrgDZoa0g9DGNgSAHvpUjsQyTrVDqN8itzZWLgx0P7bYwJg8jsKc2hAJezFCFzlqL3OHrCOBIjaG3RyhEDDINnjKk8e5Fi8T5gVKSyDc7VhAgheshKGH1VzegN292eoevwYU9V51Ig6qRbybUsZzV11eI9DKMkRNZqVBa5QLLEs03TCiClGxrrib9nHv4Pf5X58KMfUbVwdb1DWc3i+Ig796FuNDau6PYZ40be+/JdHr2d6XaZnCwP7tXkccn+7CH7YSBUPYN7QrWI+A7y2LKoEwMeMlSVJSUhTdw9XZNjS7+JjENH4IJ2bbj/5hF9eEYze5erazFwOzpac729wNmKu29UvPXeQ54+vmT2xppf/vxjnLW8986bGG3Jec8Qzlm3p/S7R/z931zw5KPAau7ZXJzzox9+wumdD7DW8e67b/Otb/0hWmueP39K8onT02N23TU+ZMYxUmVFimJyk3MgRQUqcn55zYM3DKujms22o2kcV9c7VuuWzfU189maN9+8iw87zq+v0VZTN5ZhHNHKsu8HrNOcnKxR2fDee+/xH//D3/LixQWrpUib7Pc7QujRNpO9aLF3Xcf52ZbT0yNi0MxnKy7PN1jn8GPHG2/coe93MkfJNE3Nbi+SVZuNmDROhX1dPGYWiwX7fUfb1nRdL3mQNVxenhOiZ7vbsl6vAMVuu2EYB7JKPH76BGcrYkxUlWPX7YihYzG3oks8Zpxt6PYDMYopX0b29P02MasqZqbBX3nm7pj/7F/8H5jVc4zSEDMqiORHTF7wFGdRJtPvPPPZmsY59PoBY+pYuDNaLmmV5nR9n0+efAw1nJw27Por6nrJbndNCJ6mdUUnd8Ns7lAarq6vqOuKru/RWjH0HUfrFefnFyhFkfgSVrHIaEgHweAHurDn+M4R+64TdneMhKHDEAjBc3p6ytlZ4PTkhE8/+4zj4zXvvfeIx0+foK0TslNI3L97l3v37vD82VO0Tjx/csaDB3d+57kdcl8k/SpSzrRtTdU6jE1FXxpSkC5s7TTzxZKYElcX51AKqyln+v0eZSxVXQnq4iTPnvy22rYmjJ7tfk8iMnMim6ozKCXeFs5otvueHCbz5JGYPG1blzxZ4s3LqwsgEVIkeM9ysSIGuHe/pWosZ+dn1K0UHOu2Fk1v46hcxXzWUreGTMDYhFIBraty3RITaBIpBVIK9CFQaYjK4LSVHC1ltHG0dYupaqyZM+YKomYcOrSzVM6RxoQfPbvthnbeMPiBfujFvLqwwVOCzWZHjJnKVfjSRbK7vuLk6BhjNT2RzX7PfHnMarkmZMOuj2hl8WFkDBmo0FXDGPaMQUDFEEaSUiwWc/rzHV3XFxkWyNFQ6yXjmNGhhaDxQ1dkKSgkIPWSjOIUE1R1RUqqsKglpuj6HUZrrKtoa/HA0drggxTcFAjmIV8OObtWmbqyVM7gDDQVuFrTLiuqpWFx3LI4bvHK08ee5OBqd0nnB4YUQEHdmGIu2bHZXnJ3NseH8RBPTB0xkCFl6SjyCZUVIfTUswVuPuP58xdsLq/wYy/x3FqKZj5NprHS9RFCwFSWlBRx6vgKmZRGhmE8xEzOOdr5DJVgt98T9iPdZsc47NEGhjAwpJEQI7ZuUFrj6go1dcHGkVspz/9Pj9cTdm6KHa977ety6s+f4/N/nyed89dACK++fALLMzc4h7zw83/roy+kjEY8cQo55NmzF/zig19ijSUnw/X1ns12j9GJxeKI+/fvEePI9eaKP/z2N/nrv3m7aJ9L52BKic8++wTnHN/5znf4i7/4CxbLOdaZm1wscSADvsoynwgYEwFmOudEfnkVFL79mqkAM+UQISS0CbI+G8g5YY0hBs/q5IiPP/qM0+M1R3cXvPGlO6wWDUO3R7T1nZgwq0gsOgY5lzxIic/TbUh48n7itq65JLS3XidxszXmFmZYwHCVX5KvefW4LWsi9+jz4+1V0uH09XUdtL/t8dqcGz6Hbb3u2Uyv1a/kMr/LdRzO9wUAdPgtQPRcBiBThWNizhWwOsaIURZVWs+lHVH+L5g+zijR4Sys5azEiMb7SIwZZwypAJhMYKZKBaxTGD1VhjI5BZSysogpCKVyKTrtilz0140DkpiKipyapqk0lTP4AkwkEsZGKqeLjEzR4tHSWh+CYgypFAUEZI8x4cMEega0kuRYO4upDGa1YDv2HB8viN6DMTjbEJ2TxT2KXoRcvsWnkdHvUTmx21xz/3TB9/70a/zN377PuIuyaRqDmzWsTk/wfQdR2rh8CPT9AChWywWreU0erlBxwGWN320YtluOFjPmgG0M23HHCZHk96RgMDqjho7RZ1xbszhasr53F3+5Ie63DL0ve1xh41mYt5amsYwhoCjyOkkA8ZuqaC7VbBkrY5BixyT9kpDCuhRHxFBOFfATJcBIiFpcvVViainQGpwyJZlOBZBPopGvtMjHZA466bemxi2WWQGI9TQBCwhxkCuhsEAoIG5ZgLIihMw4JrxP+AA+cpAgmkb81HyjEdf6YSzmbDmhrCyI0vJNYXdLIUDY7yUYLOVjXcDOGHMpJEDloG01ISqceVkH+rc9nBUGnWwiugAM0nYpIIBMfWet6MUaJTae6vOb+MsVPfXSOvzrWOi3fzdpn2k9sZoNCgdYhL3sRZZjeo9DW4kAwi+/jy7XKaA4ORXwhZc2hJtF87aOtbDLz87OAZgv5tR1JeBa5eiMoqodWimGURZ8Z4UxJkmasJomuYe6rmVMaH0wELwx/xPmrKw9mpy1lCRjwseB1XqGsTAMO05PT2gay2zWiumdVqIBbBQpx1IMERDKB482HADDAwtY6wODX+Q3wFWGpVnwX/1X/yX7fc/2L8/Qm2Lu6WX8T8aD8Zacy3RMwchkoHgbzHbOUVWVSHookbNq2poQPbvdVoBdLV4XzrkDG+A221Wel8JYewiSxlF06k9OTmjaVsD8W8HQbekcyLfO83qg/HX//lXj99cVkKbXHKRkXgpQPr/R51z2xSwg86EeXtal3/WwRebMD92hMyvFwGQM7FxF285RSkAatEgLGaMkWfbCSqoqkfORToEbw1EBZjOVc7jKiG6r0VAkCeq6FtaJsRjjqJuWBw9n3L13SruoSETmywWz9oi+Twz9QNs4lqsFoLi4FIkfazNVJd0xQUVCkA4YVzmU4lBwAQ7zdxqbbdtycnLCetmynDdUleP4eE1Kgbp2zGY3nQwCTitSijhnDyyUCcieWF+3zTqn59O27YGhPT3/uq6pDt0hHNaBaS2YxkTTNIdkeALQ27Y9jNfbnRUTW6brusP3L889mUMpCeteKXUoME3+AsREXdUH+ZfJHHh6ZuM4st/vS5t+DQr2XVe6SAxt26KUOkguOSfa+FpBXTv6XorToZhthpgIMRJDZt95QOFcBUoKaEopMaKaEoNbyY/IXYne7HQvDPpw34dhPDz3EKIwx4NG0RBDT/AaP2q22x5tNe0M6qYmZyU6vcmT3UDTylhOKR6KNWLmakuhRggiwmovZrEB6iYibb1i8mqs7KMhDKTocUaxmLUMRtPtO4xz1NWcNAf926kqfu744Y/+hre+3IgxpjPUM8um23CqDQ/eqtmeBdpl4PgUvvMnLVWd+PCXHf/h373g6cd7To4s3/r2KXceaR5f/hKlO/YbhfI1ylv8sCVnTzszuKor4166iOIAu03HbNGw30Wa5oS2Pqbbn/EPf/8jMiMP3zzm5HjJB788I7eR1fKUv/mrp7x4scP9+X1215l25sF0mDoyjmf0vSV2X+UXP37GT3+0YdYYuo0nWMtiMaPbeeo28+/+17/n7/7+RyQPd+4uMG7k8dPnfPWrD8kq8tHHj6nrluAtX//6e/zk/e/jfWCxqjHWc3F1wRsPFty7v2LsLcYo+n5ksVzw9MlzjHE8eHiEtQLOnpwesdtFxkFkibRSnJ+d8+ajR/zoBz8AwIeBul6x220BySGaRqO0ZTarSEnz3ntvorBcXWypnGU+n9P3e2azGT6M4nPjE0tr6fd7qbEWQsgwjgw+0fUdSmd81zH6Ea0V77z7Nr/84JcsVytZK3VGZ5Ggu7q6Yr0+5vziku0ucHJ6hB8io+85Pj4hpQFNxerOMcHvhCFKYrk64vLikqZusEaxXh8ThoF7R3dJvef++iELc8qffe9fcWf5BtY0Ip8YAspm4jCQTaaqpPsAHWnairoyhOixKTCrTlC+5asP/oCn7hJbz4ind3l88RG6ViQi+2HHknXpDk7sdlv2+x11U+NDpG3XPP7sBcaI5Ng4jIzjeIgFvPcYa5gvapxz9F0gRsWdu/dxtePs7ILFfIZSiu1mS84JozTOJV6cfSbSZq2mrhWrVcMwXjH0G+7ee5PNxjNrG/a7ns8+e86dOzOyCrRtRf492KpZj9jKSaEjK9r5EtQMpQLVzOKTB21LfJHAZVSIpByosKwXK7phpBtHYk4Mw552PpfOrRBRMUl3eEgQxHdq23ck15KUIoaAM6Kpvd9tubq8Zru5RqmMjwNVY2jqRghIKTGMgV0/ityhgtGPnDYN3W5guVwxxA5TaZR1WFOhrRicK4N0kKrEajUj5gF0YDZ3NK0leLDMhGmfE03dknJkc33FJnj6cWAxd7hmRk6m6NQHXDXn7p37WKU5v3pBQgB7V1dQKc5fnNP7HVQrbOXIDvrg2W52OG3IMdGNAR8iNYpKKfzoGbqBQZBm/FS0r2swhvl8wZ2ThLKZMUdcZdFuARiWqwUmjuhaMYQ9w+CpGgF+x8GjqICEZUHranaIcWiMgZB8IYSB0hqIh26420zaGCnxvkZpuae25EUpjQy9YjGrqV1FVchc3nvBI4wWOVKjMCZROU1TQ9MYlvOWtrXYxlAvamgg2IA3I55AiBm86JX30dOHnsH31M6ybI8w1rEf9mx2G5SpSFqJgbg1qEE8U0xW1MbR9XuJ31Qkm0jIAz52GJPQlWO1WuGc4/zikqA0UWU2+x3YGSlFAhlbOcZ+FAnANBWM8iHfstbQNDWVrUh9pN/2XO93hDCgDIJPpQhKHbp3tIaqaVAKoleY8PuR1w7sPCY5lBuy5mtfy+TlpA5f4RYBrOQ1N4D7q+dVt3Kf6RzTYvPyu90wkuVP1auv5zWvp+QxhaSjDlK5wnav6wprI5vNBqVqNts9MTUEHzm+u2YYes7Pz3n85DHD0PP8+RO+/e1v8eTJM37+sw/oup7T0wVV5Tg9PeboaE0II9fXVzR1xc9+9nOePz9j9IFh8BwfN3TdcCguDV7iVOscKDGvn8gr0/Xflkm6fVRVdeiQniRvKV090imfp7CV+WLBl7/8VR49uMv9e0col1Cp5vqiZ9ddYyswNnN0smS3v6aZzdhuNxirEUlaj3WWcRhomhpXWXkfLTLBMXhUkbosI0DyxkI4yapIVeebjlKNJpFfwnJ18SI8kAsnPCcLCXbKl8sIvSEgyoM+5OMvDZ7fhFur1/HseQ25XM6dXyuz8PmxmBGCM4WY9nsB6Tn/5s9Rji8cvRsjG7nRohyeCl02F/YuCIA0afIqhLU8tRoYLTIuwi6KxDCijLAdYsiQE85qkcTQk9FFAa4pWtU5Q47SDhFiAcRE89oo0ax2lZMWGwXaaKzR5FLlMRqcVtTOUDlpjTEGcsw4l6gqzZimASUSMgJ+wjgKb9gZ0feOh4WptMJkKTSk4Omff8iTv/2f4EtfJ7wdmS1WzBcV47glhhGDJScBU129IqkKimFFCImh7/nB3/2IzVXHYrniKoJF0ywaKO1etbHEEHj+/Amb7QXtrEWRmTctbdUwjhtMzKgxMHQDOQSaxvLo9A2a1rDbPGfs9zTrh2Sn6a42DCFi2wXN7ITzF9e49T1cuGDTD8Q4EqKA4D7KM6mcwdpiDIfCmkzICh1BJ7k/aapuFwA4hMIC1zJRdcxUSlEpqBU4BVhFa2WBiyEyDpIgZCsgm7FSDTdaNmC99ySkkGLKY3F66nrLt9iesojIQiGgudEF9NcyXhwgHRHlRKkw0cmlzVweeQqU4KbooU/mLIhkDepmoZk0nYR9LyCh1cKuTkGkaBKTO3omZiXyMEH025US8xZrNOMgkhdaZWatZdYaRp9wv2dFvJ6MhQ79JGXD1fJ+mYTS0hJtlGYyozncTz0t5bc20XLtr1YSf1PlXFitYvxjCoiu9FRIKK31KYk0h0IKTGUFFk3+6Twvv9dBToPCwpnm7a2viqkLYdI2TgdG+M9/8QuuN2dsdxvaZsbdu/fYbXsJqL1HKw1KQGxzYMGrQ5Fx2pjHcTyYFN5e4GNh+GQCxihcJbqdMYq+p9WiAH9x/oyUPI8evc1ycYS1DZPsgGw6is12x9gHAS+1jN2mbtjvR3a7LdY65vO2tNImcooyHhWEGHjzzbf47ne/x9/9/b+jbWvOzy+46C7pe2GByNSQsQ83RokTQC9BiYwla6X9bTIkTCkCim7X0SnRXx76nhQjKQaM1sQiAzGZI3Zddwu0RtYOLQlcXVfCgPbS/m2tKabM0xjgEJdO8aPiJkA8jMLXAN2vO15XCJqO13ViyH54q1A0XVguQdc0Z6a1KkkHlywjCaMU6Qvot/+q4/7pCcK8SOz3YohmrbQVKmA+n7FcLUUj1ges1gdW/22pkIkxPRlKGqNZLhfC+ht6CR5nLfP5jKYRgFjkK8QYWPwB4PjoiPXRCW88vMf6eIFxMJvNcW6O99L1lbKnHwaeP3+OKsG3UqbM1SyFPDuxEiamtZLOr5wZR/mcAh6LtqpzYjbXVKIJPpvNODs7Y7lcsl6vD+zyCbxO6cbQdWLFA4fCDfCSPvwUiE+Aedd1B0mVCfSWwhBUlWj19n1/uJ8ikyOg9DiOTNqm05oxXdsk9TKZjU4+C0op0YZNN9cMHHQg67o+FNJijKgkRdFJxmX6u+m5H8Z3zvK9Aucsdb06jHWtNW0pXFVVRdd1bIvcjKvEOG0ce3z53OK7IeM7BjFnd/ZG3/2QvExzrKzhh8KX4pAgpiT6hyLvUmTdwljWupbF0V0WzYqzszNiUAx9kg4fL74ZIUyMMmEAeePpB1B6AtENxmSCjxib0UoM2zwOsnimWFMRkyX4TN8NWDPHOZFES6kUe+NI5SxtU+OsxQ8en6IYkFYOq28KLL/L8fAdQ8iR/ZjxKuFaw7Dz/OwfPX/2z1Z87Y9rKq3IceDv/vanLI5q3vvqH/B//r/8Cf/wH5/xP/z3/47vf/ApX+II7TqOFy1ZOTabjsqCzpmqdsznDQ8ePcS6OVfnezbnHT7UbHdPOe86Gr3krbf/gi/d+RZn/u+4GH6CtiM5GEgVJ0fvslqv+dE/fMTFZ3tyivz4H3+Gq+DOvRN0lZmtW9zmHv1e85f/49/xnT/8E2Zmxvv/+BEDIzlrtlc72qVF2wpsRpmMHyLPnl+zPp3jc+CXn3zK/XtzKZa0sNlt+OzJh3z5q+/w/OwJY+xxlaGq4fnZNQ8e3GG337KYVzhrqWspAg7DQE6G5XyNH+D63JOzJabEYmXIeMYxULuIsyP7uMeahtH3GCfzu3YJcmQxq9jvrqmrOad351xd7SH3zJs56IyrHa6pOL+6wNYVYxz48LNPOTk6Yr/dEggkAkftCfuhp2kqYopUjWOz63BW8+TpY5SG8/MXHB+vmM+dmGo2LVlpQkrMlkuOTiuGsSdlT07Qdz05JCrboKPBqhmLWc1Qe4wTpnL2ERNh/+KKo/Up79z7A77x9ve4d/QWi2aNso7iNEQqBVZyQmkr2tA5CcsuR9xMi+xINOA1b9xbcbR+xPnVL3H9B/zsk5+zXFUMi2MG3ePziPeZcczMZktySpzvzjB2hnVzUvaMO5GCcVbR73bkGIjB8PTxM2JWdMOeboxUsyW1NaK7rxs++vA5s2WNHz1H67sincKGy8sr7j84YRguRQc9ZIYhs1qdkkNi7jR3jxaE4ZJaj1w9f8KDB++RYqIPGTOzMCYSvzvBZb6scMZhtMN7xGxRJ5yDWitS8mAKKzIHQpacu2osR2bNcrHCh3OMCcX3KKBt6QobRpzSLJuW2jhS01K3DVVTUzcOaw377bXsOzlxefmC3XbH0HmRtQgBVxmUMvT9iDaWmEXe0rpaiDhqioFEwzXHRNNIZ2KKAaWkWF/VDp0V+36LtoZ2ZvBxQNtISkIQs8ZhsiEHjzMGZSy+mrGPHaPf4nNmdbQmxitijvSjJ4U9D5TmjdUD1AjD/lPaRYOrpYtqu90yhswYB/bdQAC6sYeUMMax3e8wdY3LirEbUDFyMlvi+4HcDfjGkqoGZStwljEKqa2qLOiAyjBbrzCzY662HTontE90cY8KmovLC4w1xYtLk5Oh7xIuZsKoC5YzEhmJuSeqjBiF+kOcMDFopz1Z5Eg1TVPTtBalEz50KBWpK4uNGVKkqYWc4oOncUKcctZSGY14gpdxVhlmrWG5dMyWM2xboWpNr0ZCHtn7QBd6NtcDTb3EzSoUGZUN426gb3cMs4b5siV7z+XuElSFrRq0dbiqYuj2NNZQa8eimaOTeAENaSBHjQmZ+cJBXGKyBQxnZ2dc73Ysjk9YHi15/vwJplfE6HH1jvnc0PVDIZ5Jfl1VphTs5V7Zwized54hBpLVdKMnpsh80dBqkXBpZzNCiPR+T/RBCErR81qd8t/imIDLCXydCH6vIngTgK1ULiDkDTntdk6RC/h3Kxsv58+33u8GNJ3Slxtt85v3vi3ROnUGTkDx5z7Hq2h8Fn+6rCYw+qbL8Ozs7ED4MLqlL90BwzhyfX3Oj9//Ef/s7M+5e/eY1WqJc4aTkyOqb3yN87NrjDV4LzKHu92WFy9ecP/ePZaLJfv9ng8++IBx9NR1w/X15hDbaq0PBJeUixpCuClETUSQ132222Ss6f+JxOWLJKLSGmuks7ZtjpjP5qxXR/z5n/0zrvaXZBX47PHHjHuDHxMnp8fstxEfWvbdyG6fZD0vuGRVSeet1Q4/iuz15N2XM7jKospaII+qgOXW0I0ixWWtEdKnDyJzWeSdpgcfSi6u1MsgOUg3zuGJ5luSL7d+Jl9/O6nR10Porz9+lanozVx5+ZD0fgLAc1k/fwcQ/bc4vjCI7oxojNrKEhPC1lWFbVturjYKZZRIUAqqjLwio4lF81KY3CqnScwCyGQV0Dajk+i2TCCcViIhQRJpFZGDAYozc05KhPiVwliNdZqcbtr45asixARZpA6MURhHYU1HtA5UTlFVGu0np3ORp8lJHJwHf6PDbrWAY84oNEmg2RQZ/SiV+20iDVt2zy948NZkgCZs6UwmxBF0RVUfM5o1nQ9opYkxs7ne8JOffMDPPnzM0+fn+MHjakfykWH0oBVNW5Ny5sWLM9rWcufOMbNFSwye4+VcmE+upjWGOiv6DO2sYT2fMzOJ+brFVQuGrmfo9rh6TrU8xWUrDMUYUbMlqpkTUst2B5t9YtdnBi8gurDyk8iNxHQDXBuRwTGlAhkpxp1l8Q1p4g6LXrlRUKtMpRVOQaMzlVGsak2lDTlBHCM4jXEa7QypscVoVkodYjKoDpWoabuYuhZkBArvMiZ1aIapNDijcJURZqvRoAwZ0MagdKnS3a7G5SLlMorEilIgpLVbv0cGzsREV5MBaZbrc0pRaTBKE0rQE5Mw2YWFLH83sTtsBU1tcUqzK23VSmUWM8t85lB9KEWh3/0wumICnaa5M32mRCCryYxmAtCLkepUkNDqUN0sZUDZ7PXnWea3j9vyGLc1o3VpNxSd2aki6pkkWUqVDJWVaOumXGSXIKDJr5yTMiaUUtKtoLW0a6ooy5TKqEknHfWSDIJzFffv3yfGyN/8hzM++fgp282O3WaPmG0qxPHdkdIIZLR25X0NoiMtGm5V5Q4A2WSEOIFcRkmrqnFQNQZrNdY6qnpBjhD6wPb6kmG4pu/uMmslUTk6miFSN1rWq+B58fySTz56zBv3H3J0sqA739P3gcuLa54/f86du3d4++1HrFYwa0xhuQlDQ+mENpo3Ht1ntVgTfcDaGls3mCjdHnH0EpwepIVMYWy6g8zFDXta7r/34o8xDAPWOq4vr8kIU5+UyTFgCjjrTHMIbHLOB3kLGR9T0DcxK0QfWuQVBLDSZe5qY/BZ3rcQOcTQempVOxScy57BTUubzN1fveG/LrB43SEBlxT9pp1fK0VSmqwiOstcF7KQRiV9kFIS4NgW3crf7Tidz8haCzNv2+FjQBknnhDOUdcNdZEESYXxW1V10UMv97F8xknnXpK10n1kdInJptfdBC0pxUOiN29bTk9PWCyOWR8dc+fuKSd3llS1oaprUjKQHNbVdOOeJ0+f8vjx4wPADCLbFYrmudHCZAzBoxQvgdXe32h/i7SLYhgHrq+vae+csN/vDoacV1dXXF9fobVmt9sjLagyXydt8duyKZvNhnEcXwLMJ6mXSRs9xviSEasY5vpDcclaAcyn4toEgE8/mwD5yeTz9s+GYTjMrQkAnwodhyShMNO0Fgmm25qpE5veoIllfk0Fq+lvp4RiNpuRc2bwAzEF6kaKVjFKYSvnfJj3Mm8SzjrqqmIYu5s9JE8twFriRG1AS2lI6Ynlrw/+C1PRepL1E23eSDaTATOIZr06dLgYY9jvd4zjgDW2FC8T4xgJKQgRxDWonEnR0HeT9JbDGmn33u87jFVUVS3zIcJ+6LFOUTeWuppD3OKHPcFnGa+mwegasj2sw9qCSkqkA7XIBxgtzKK+l//bBppGE9Xv10VmXaYbRmLKGGsZO89i0bC/7vnh919wcXHCH3zlS/h+QxgND+7+AT/+waesjjWb/YavfGNFTB1Hd+CP/+RPePb8Kf/Lv/0+uTPobLj/8C5f/9o7rI/XoBw//fln/Lv/+QNmtSbrkcVa4auKb777bb723rcw8ZhnzxouzgJ1k2jblp//fMNnnz3m4YMLjMm889UZp3dbRj+Q1YoYLD/90Rn3H8x4/qRjtwkE7/jv/7t/5F/+sz/nP/3Xd/n+3/89z55ckLtcinkDy2WDHwMpBwgVKSUWyxbv9+x3Xdlve05O5/i4ZYyWh2+uefJkR9+PzOfHPHsSWC0UTVNJch4TKUVWq4q+T2ijePLxBQ8enrDZXOEqy9GixfuOnBEJo11X1h5oGungEekmTWZAayPmhFE6GWLy9P2OlEe6IdH1PVXbsDxeMjwbwBhmszlj9uz3Ihk0my8Zho6hH1guW8jw/MUZR0cLXHalm1fz9a9/jQ8++ECMheuKpm7wMVLVDu8Drq4JUeQw2qbi+OiETz/+DBIM/YCzhpAyYxjYdTua2qBS4v7JQ8Iu8cb9N2nrNX/41T/m0emXsXkmxaScpBmDXDqIFeSIwkFOBO/xo8zDrDS60iKpWEt3aVUvOT5+yHtf+Sd8d/uYD5/+lB/84m+5Hl8w7HdiphgGnKloZhVmY9juOpxrIGmePHnCgwenzGaOfbdhsZgTilyJ1pqVPWaz2ZJ8w9Bphj7y/NkTkYK5HFktF5yfXbJaH/HlL3+Vn//8p3Rdj9VWCA5ZYUyFNQHvO3SOaJ2Jgy4eD5mmnlO7lt22p11Cvx1o69+d4bJYtKSgySmidGRzvSPGwGI1R+0iPg1olaisSIf0OYlpXuUkV1KRkCO5kMqSEs+ZGAKUTqumEVJE5Qy1k4LzPoxi+KhrMdNVWbrGY4+pW+7cfYMQR2IeQdVkEtY2xDQy9ANh1CzaOWpmCBG0E0PJrBRV1WJMTT/2JCX6021j0QlqMxfAOHSgIimM9MGjlWOma4Z+ZBwHxq5HW0PwAZstddVKN7oxtE2DCqK5HX3m+fkLdCcA2NH8SPLGgMi6tHPimNn2O4YxoCuHdIBqtKlAW5rlAmM7Lp89o8qwWh0x2orNbks/eEzbYJwl6kwymT6O+BRQOYFT+DySwo6hv0ZlaIxhu+9o3ZyQE/3YY5zBVBpwbM4Sw/WOJmiSL0Xf6KXw6wPeR8I44kt88aqGsyJjraJeGNZ3W6pWMZsdE8eeOAZWzmFQAiCOkRgC/b4nJ6gMNE5TVZa6NhiXqWuDqzVVLaQqxMO7EAe1mITXltxadtsO4yyr1RF2dPS7HSlGum5PYkRZiw87QuzJ/Z62maOVE188U0wgtaeqNV2I7MLAOIDu9hhtcM7S7wb2u4HdrsP3I3nhaasaUHifGH1gs9lhjGUYO/pOOtMkTpWuxRgiu90GdCKMnv22k85vZzBVxTDuSAkWC+mG9CVnyykWbyCo7Y3k6e96TN26nyfvvI5o9psJaPBy7l5+cCgaTKSI6eUCpGvywYOJl89bQHc1MQ8pQP5rgFA1xf4ln4EgFNNMId8a+l6IJXdO7/DRx5ci36wVddPSdxuapuHq8pKu22FdJhMIYaTvhURV1ZKXWAtNK8S/cRzRSvHixRk//9nPyTmz2+05ORbT6Sk+zDkTjMQO3ntGH/Elpr7dpf3qPZy6M6fXSYyeDrGnKtK30++z0ZycnnJ6epe6nrGYr3l29oJvfOdb/PAf38dH6Iaejz/+OUcna0IxQt93HV3fMWsrUW/QA65yHB9VwjjXTrpatBgED/uIMeCqViSctczPlCF6KYJlDFkXdQFdCiCHwomoQchjvT3ubjCgz42n1yDXN6D1Fzxeh37/qkO9PNZ/7Wk58OIOwPlt7O7/X8cXBtG1UWhbWKWlTYAsbD5JjtRBWiUXo0M1sUQRI8rKSBuswBeyqenyd9oIiBUThKSKgWDCqIwzqWhGQ11prE1En0kJRp9EMF0rnC6yDfpWEh/FWCJE8EkcuWViZyjyIFpLS5uAHMWMT+nCSMp0Y6L3ReIFdaN1n4tUyaTznpKYVo6BzdMXhMV7RB+5utpwVD1gSAPKtNSzBuVmjFkRk+UXv/iA06M7WJV48uxTfvKznxKUR9lIazUua/peF4BVQI7tdktdO0I8JinRM1zMF9TOoKMXvVmlaJuKRV3hnOJ4uWY2r5nP5rTHDzGLe1Afw3xNO1PYo4ecPX3G5cU5Y4h8+uScL739DZpeEy9+QYhX5BxunqVKqBRJsVQeNdgsRYbKcFhArRFj0ZwTPkRimp7PDaBktcKoROvE0GheF3kKLRIVWkuRxmSNceGm0lQ0wkMQFrpBnLdDnLxQk7DVEOasB4r6GtYoWmeoilastYYwFnaEM1gnbIzJt1Yr0YVOMTH6dBjDKb9cnc15Ekq4vTQJMKDyDXgvGycCnuWbwtRUtc2Iq/tcCTPU5gxZjIasgaqStj1UxtnfryVcAPKbzfimlUlKXSlnNNJ1YoqZ5vTfzTnUAYS5/f2vY+6++rcv/Z3WZV7lW1JOZaPLGTFcLO7aOkrhIscDO+BlED2T82S8o+V5KIo8U3r5OkrFWfTPFCkJu/H4+IRvffObdN2Wn//s54UhvZeWqQJ+ZZIUi4pu8GQaKPc0vgSATZv2BNinmDA2s1wveOONE05OVzhnmc3ntNWsGDduiFnx+HEPyrJYnBDnAeMU1glA5UNmHEaeP3vB8dE9rq5H3v/JL3n//Z+y3e4IfkQbOD1d8+V33+JP/+g7zBYNdWNJ5AP7/Y0Hd3n7nbcYx55q1tCPgyTCg4CiIWYwtsxRc7jft/Xep2PSY54kMARoFDBUnndhwjjLZAIzMQheba+bAk5hEYQCtntW65V0pyDFG+nqSIexrQtAX/pVmAptk3zEq2P0d9l8PxfAHsYVZQymmzWjfJ9yIkfIyaByjWGBURPTxKBp0fl3n9+N0hhbkW3F/nrL9WZLHr0YeWbo+o7Nfsdivip6mmL4FZP0nFWVgML7/e4g9aF0xRjGg1Z4zvIsRf4sCMCuKM8vEGJEG8vR6Qk5WSpXF8NJ6eYZhoQfABzNLEsxt9yrnJN0KWSKfFkq71cRU5IiTLn3h7lUxspUiBnHkb7TbFWEKNIlXbfj8vKc8/MlR0crmqYRgCFGYvQvMc8n9vnUidE0DXCjk3jbLHjyOViv14e5Po0JmQeWGDlIxFRVdWCbTz9r25YQAtvt9gByT5Itk/SKFEDqQ9HKGHMAzG+D+FNXwcSEn8B/Z+xhLk/XMZmS3shZqXLdGm1sKYYnjFHUtSvseFmLpVhYcXS0YrvdsLm+JIV4kPxJOeNjglyAdCwTQD7NkZu5h+y5+qY7qsyww/4gPjXFd6FqMFaXe7OXAsno2W12dN0GtDkADtbYou1ezF5NhatKQWOQ8abLPQq+AFiUDhEczs7RacT3W5HAQe6FyHFlSfhMxlgNWkxO+16Kjrtdx36/o+uKXrxzL3+83+E4OV3yyWei+xxToKotbe1ERiNnHj8+4/zFFUdHDet1w/nVGU+ePeFy94I33zyhD1tC9HztG3+ANgPeb2lbuPtwwdHqmD/6k2+x2V7R9ef88Ecf8emnZ7z35YbFbEbdejbbgN+cMnQNf/lX/5ZPPn2f1X1Y32mwVUPOjudPL7h7ukRRA4mUBs7ONty5t2QYEyEMvPX2XWBgtTacns7IacEnH+35b/6f/wN//k/f495bjqcvFHfuzPns2ZZ1pen8QO0clTOQEv2wo2ksMSj23cBMVXR9xG89q5XlFx98wptvrrl375Srqy3BZ05P1rx4vuONB64ADZq+H2ialhC2hLDn5MRxeXlBStIuPfQD66M1m80ls7al2+25d+8uxtwUonLxshE9a48is1yIlNn19Zamabi4uJCCvimkpCSF0xQktwpjJPiRo6M5fhwJPrBaHnF2dsFsNmO5nB/2TnLi6uqa7XZ72F/3u47ZbIYCun3PMHruzqWFvNvtmJ2ccHV5wb17d3jy+DGr9ZKsA0+ePefk9Ii21mQ/4rLBjjX//M/+Ne+8/U3QLTO3wjFDJ+mC0UnAVq0D2OJtEUJZL0qRrMRcxtoDW1IVvwxNxuo5PnYcL1va+THnZ1dUVw27NJD1ntD1PHznEWcXZ6ACq9UMheJ6u+FovaauLSGMzGczLi43AhLXDZfXG+bzJU7PuXwROTmpWLSW+buO+XzBZ59ecnp6h/OzS8Z+4P0f/Yyj4xnjOKAKwJSyYrvdsFq0kALLWYO/2PDg4SOi95w/v+CH//hz3v3KVzm7/ox2rrmzWnB1sfmd57bOTjoGksfVlnauuLzqUWZOCAqra3IcRZYyQciRpBIJzS5tGcaefexIaCqtxMxzt0PnRN1It+W13xF9QCVY6AXJBy4uLwBF7WqcW5DjKIxDBcpZTh6+gTWGq8srck5UtRMJvjET+wEfE3ptmDUtOWvQhn3fUZmyvmtLUg7bGCqbUGmQ3KhSgGHwGuvasg8I8cXkRPICHgcycRckhokZUymU0yTvWVStkHcyJDXy4vIZTz79jPV8yenJMSu7wpMJAygsppLrBg3BYChzOCsWR8fkqqIfPbaqcSmz63p8htGI78qyqWnaCkygi55cK7StRBPaOZHEu3zK7nqDVY7sZuSoSXWDaef0fUdSgZQdYRvx1w1sa2JWWFXT7Tf4fSD0nn6/l1hq2p9KbHOj9yzktapVzE4184fQrjWVy8zdMXEXoO+EvGYaCJk8JsZdy7gfwGdqLbJ8VWtxrcXUBmUEYI7GMmbQSfKMbtfTh1HAvvUc4ki/39LOF1jdUFWtDEwCw+ix2pG1ofceP4DfDxwt7gjgnTsqMmNI9ONIthVJaVAWv9sStXRmX+039J2ndjWxG5lpzdFsxvXyiG3XM4YR4z27bsPoe0avUBiqakFd1SJbZSMhdOw3V/Ta0u89dT0jkwTPiUDM1LaiGzpU1rSzGcPQc3UpZAftlpjfG0R/ufv1du7wuXz4lfzkdUD6banKXyVn8br34FZOdMNEn9jn3Pr6Kz9J+ZvCnysYiBAlDQpNGhO7ncj0vPHgDZ6/GOh7kYZrmpqh31DXDb/88Jf8zd/8NX/47W/wt3/7HwghMgw97//4F6SoeevtL7FezwlxlC5IKx1fl90l//AP3+f5sxeMvRQ/tdEMg+SSIUT2+z0AIYp/2av392XZ0JeP2/nr9PrK1VT1JN0XsMawPFrz+LPHDF//Gm8++hJ/9Vf/lu9879vsr/bMmiOePn/O/+v//Vc8ePiQ2XLL8cmJ3CvdcHU98rTbcLRcFt+CxOXTjRDqnGBUOQfx4MiRyolZc9tWuMqIObjOxT/DoedVebQFI7E3HmwgHnNlVHAbQL/hdBeM51AfuTV2SjdEoTh+YSD9t2GGZ1QhY796vIaQqYTc/Oq4/98MiC4yLfnmPitFRi560t81RouZYgE+jKZIvSiMhsoqrFGQZNBJ9UYGitGKqMDHzBCK7peSv6msMJatER1o6xRjrwUc9xmtoKoNrhY2cVRAzqLXrkRfOsaMDwIJjjYxYY5agTIaXYB8KDc9gy7mhXGSMClJXkzCyB5DYdkr0S7WWmO1sOZcuyAnOPv0GfP797nabKkrg65WKG2IKQARqzKbsxf84O//jj/6k29wdv1Lzq6e8PTpC7bbnhCiGMcUZq+palQW1qh1CmPfYrmaMYwjH/zkl3zv219lvphxfvEZtdVYlbhzNKd2lvXREbPj+yxXd5mtjpkdvUGzfoi2jVRaVeDk/tvgZuzHAGrGf/zr93nn7UdUy2vqq5HR78g5UVvRqFeFkQiioeSAqsjdJDIkAdStLXpOB3mVInJyWPBlTBkNtYPaiqSOSKhoVGFyqixGZ8lACkVrNQgQVRmotEarwkjPUuCwaHxhhvuUieU5ugL2ayXPO0aRLolFR6oqFUxrDVmHIksihZ4Yi4wPMh9E7VzG06Fod5g7wnhVSMGosdIuRwHvrBY9dKIiFmkiRWbwke2QWCwkScw+lnudcJXG1YCRz+Xs7+twcpvZm6dLk/8LZp2KQTBTNwjpJcB6+vrrNv1XtZ1v647d/jvRvy267IciaAHRs4DVKSbpNEgBY5IknzmIJrh6GUSfAgsBUycW56TZ/UrQoO3Bcbuq1MHVe7vZsFyu+doffIOh92yutqJX7j055VJw0YdNapJTmNZD2aA5sNKm+31gQ9aG1brl4aMTHjw8YbWucU40dmtbFSAtkJVi3/Vsri/YbTbU1RXHpyeHZC9nuSbrKj777An/9n/9j2x2Pc+ePaPvB7RJDOOe3e4SnQP/4p/8U/7L/+P/nj/42ntUjRgvhthjTGH8WstqfcL55RWbnWg0UubidN9um65MAf1taZdJlmJirDdNS101uMqS0oxhGEhJ9KRfvHhB348HANR7f7iP0zOSt5aAZ9JHrOuqsI5v9LBTEh8LYdjrg166sHpvxhVlzZ/A9y9a/X71mMba7Y17Ym2P443u9G22sDACoux5qcEqi8FSXCLQ0aLy74G0xaIB6SoqrQmjJ2bRmksFBN9ud4AR5lsBaifAdZLUuV30gSkMEDNwY6rCXBegN4ZAyOlwn43RjN4z9ANkx1517Pdb9l2FsTKGhj6SkqVqWnz0dP1wC8Q1VNaK3h+jSHdETwaqupq4E4CMWZBCkDGK5WLOGw8ecHpyzLJ1hF4Y0pNUzdGR6OlPki7AQT5lAqoB+r5nHEeqqnpJxgUoJn/6AF6DSL1Ya+n7/nAuYa4r+t4fAHMxKropADjnDp0XwIHhPoHjusjtTBIzzrnDOSZZl9vyKNN4m4D424yk6bqnBCKEcCh4AQxjMXPKkZgDox/Z7XYvreXTfZmKY9aag768L66ZMudjITRkJuNqlYt5e5Li45SoxFR8HZQ+yLXkQm6YgEoo3ijOHebydF0pJTabK3bba0bfS1LvRKIjpQCIf84N8166VeqmJiVp5xb2WUNdN7Imhh6UwtoanVqsiVLYDJmcxD/jIClDwDqNIkAIhNGL9rofyFmKNCGMpBwEZPo9jrp25Kyom6KFqzStywx7kTJ58PYx3/nemzRtYLGyXFxc8CAOtLWl615QV4HGOGazxHZzxrMnn3G8rvmjP/sKda04v/6Y5y8uOHuxoWkqvvqVY7lfeM4vLrk+ywxXCzbmguvNxxy/ec7suKJerDFG8fzsktVSSXwzWKyaUduGpK4hNbSVoraZy+szTk5mvHFvSYiRJ88uefBO5vjein/84Uf82Z9+iZguMMCdOwuR/NJJJBzHEQqhpesGTo7nqDxQNxVVq7m42rPfjzSV5vxsQ9cNfOUrb/Ps2Tn90LE6spydXdG2mmdPL1kuHYulYjZrpHtltqIfPdvrHle1xLLfv/HGG8QYqCrpzDg6WvL02TlKRdbrNX0/YqxDJfHFUUrh/UBVNcQwsli2JOWpGkfXd8ToOTk5ZRwi9994wIunz7i8OCN4TztrqOaOcfAs5yuG0XPv3n0uLy4JIeKM5FTb7R7nKqqqZrvdMgxepA21pm4aSfa14Y27p8QgczH6yJ3TE8ZxYN9d4xyonKi0Y9Wsmbcn/LPv/e+4v/4KC3eXqCxWN2hVoZWY/KoI0p+oy5oBCX3oYgIrncKFGSexXDGL1mArBWisaYhZMXeOf/0X/ye0jpxfveDf/+Av6fQFV90FOSdiFF16BZwcrXnnnUe8ePGM3d4zjAPGKI7Wa3a7nqoSDfTtZmC78Vxfet56d0liw2ZzRdscs7nekFLi0cMHdN2O1XLJ+fmIQtiGMYjEWOVgvahpZzUf/f2HPMhHpDCyXt3hyaef0W0DRlniODLqiFKflwj4okcKGpKYxBltqKuKtqmxRlj8M9tgdEUII0YWSMmHUmJInr6XNVwbhzYOm5V0QivpENlt91xc9YQx0tYzqjrjQxSyRUgsZgvmsxkZiRtSUjLX4AAg7vfdoXM0xHhIJWL0aGuF2DGOpOxFrtQPJMA2jrq2KBLjOKCTxmRhWjZ1y2xWM3QdWjtiSIzjHj92pZuilTV6iCSfaFxN6xpCHxiHEZstWmUx9vQjV5stV9sr9uOeRzozXy3xYWTo99iZobIOn0R6CGXEFDUMnJye0g0Sc8/Xa+K+Yz968b9yFVXrsHWLqSxDHOWjWwO6xAZB8phx6ABLyoZ952nrBcpmtEk42wg1L9ekVJHHEZUV4yB7dd/1ZR4PB8D89h4/7XdTvGAtRD3SLtcsj2cc3ZuRkqeiws5mpF4Thg5ywmFR0WJbTT041JBQXog0pjLoyoLRRCX5laotMWcSkT6MbPY7dt1eJGmUp1nMIWQ23YZu6Oh2OzEYXtS42kFS0gXjR0IfiCnQaENVK0KWbvzttmMcPXZmhaSZM9qKZJCtWpr5DOsiKiiGfcfqaMViPme9XNL7AeOFEOm9yOlY44q5aBRJoNpSN5phdPhxS/CBSfpC4iCRalMGkolknXGVxdYVWWmyLnkpmnH8PeVcfk2e8flc+oaS9yqwffOazBQHvwrK/7r3eH0l//Xv85vOd5BPwUrsr+0hF9RacXS05ionjBWANKZATpGmqZm1DcOY+PH7P+LuvWOur6/48Y/f57NPn0J2ODfj4uKco5M56/WSP/zDP2S32/HZ489wxuGcLWQJx9XVhrffeYtPPvmIECRXuCGtiQygVapgSOGQi8EN8DrFh7eNe6fPF0IQaeLS6TR1va5WK1Lb8MMf/og4et59+y2ePnnOj9//SZET1Dx68C6//OhTHr3Z4hxYV9HMWhbzOVfPP2YbTInZxV9H64x1YMxEDtJoY+iA54/PsU5jrMI5qBrNctkQ14YUMq6SjnZTKVISM3St5dlMcfeNPNA0zmQMyT2YSMe8DJTfGhOH33+R47fAtH8bqZicC6/7f6sgelFtFtDbaHQVISmSl4DIWoPWqgCRQRhmhb0D0tYqckOptOwiLb1MbGEFCXyAsbCIjdbMHcycmKORRZvSuQkkgRDBKUU9czQzh1GacSj6wrGEc0m0zVMWAFw8DiRBM8qQFIVdO4GGoovsjAAZsbDgJvatD9CNmT4k2RCyQZPRWljV2mqybVgdP0A7R320JuQRgwOMSGwrUybmwLe+8S4vnnzA/vyXfP1r79L3PctVwy9/8Yxnz/Zsr0dk0RbgNgSPrWG1UqyONFWd+fEPf8z3//Yf+O433yGOI03bYuOaRolh6vHJjMVqxuL4DovTN5mt36BZ3EWZBtC42pFTT7e7wlYNq9U9Hn71m1w+ec73/8P30YMwbp0Rtr6zomuei4SGtIWoIu8hi6XOiqQyrZONXYxNSvKLVI6MVoQ4GVGANonKSvHEOS1aTyofFi6jikFiSPiU8T4TvCzQrVNUWqRiBJNX6LJwa6HAEIBQKvZWQW0Vzkg1VCtN0gqLgBPG2sPmJUCXtPmmCdwLUWR6kvxOOBSqtJQI2K61dC5MsJMUggRMT0EqhrqATFEliFJIgKnwI/rsRkMoC5s2iabV2FqJL0lWhzaW3/0oGrMHYHEqFFACGWFG5SKjI6wpaU1+lY1+W0LlVXD99ut+3e/1xFxWtnSHTAuiMLxS8pKkTfq5Ze7FKFqLN4Dr7Y3/ZYb76yryOWeRiTmARJqjozWbzRaU4oNf/IK6rnnn7Xf57JPPuL66xlnL5vpaxoC6YeJP4ON0LXI+U9hmEzhoqWuRiWhmFScnS956+z7rIwdqwOqENaFsJlEYxRm0Spy9eM7lxTmzdk0+OWIcB4JShDHxjz/4R/Zbz7/7X/5bfvrBE958692iP5hBRSpVEXPD448+5v/xf/9vODo+4p1338I4xxgGQpBulovLS7a7PR7wMTKbL7Da4vuBoR9lDhdgbgL4bhsv3gbSJ5BNzBdrKlcTgufq+gqRA4oHJu/EZPfeH57TdM7bTAmlhH1aVXXxCtCHtWJ6TU4vO7OrW9d1e5OeSkO/Lvi8HdT+qte8unlLwCVmnMMwvKRB/eo5lGpKgiwmumRhYPH7BALOkqwmGUW2stndMAyKtEdpVR2GvlyzSOPIeBbWxgSQ+9FLAOoMq9W8gHhSQLTW0NYNxhoBVo1hsVzRtDNCirw4PyP4hNYW6wzpiWf0fQHHNd5LcDtfzJkv5lircc6U+TEnhMR217Hfi1GvFMCnFl1PjInaKConrOq2aThar7hzfMzpyQmaQHSaruup6pr5ciGyWVlGgJgmG2LMaG2p6gprHTEFXFVjZCNDF5A5lwKwGN2KCbFz1a2xUmSptBS4QpgYMOkgleMnTdNCTPAxEXb70vWjUTEQgyeE4TAMJtPXbj8UgF0TQ+mMubWOC3B7w0ifgP4YpY3bTV0hQeToUkpYIz8LcUoswFgx15qkp4CXpGu89zSNmLNaa7GuwlU1ahhK3KJRpSiaYiDkgFKxxBCJlNWBcDGlgWTxEVFZOhgzIt8XvEhsyD2WNSGXuGgq1GmlcUaR4kiCojev6cfxMPastWSUGJ8GT1VnKqdkHIUtMezQdo61C6Dcj5QgQYyKhAPtSApijiSSdEGmUhSLCUWksQ0qK1LqUSphXKJuM7OFoWnA/J717+12j6tEWktbqGxif73j698+5U//7CuoylPPNGMMjLnj6HTB0fGaYZ8gV2w2Z4zhms8++QjnGrpdZL2e0fUbdv3I1WXPcn6H0y8/QmXLOMqzVjrw+Mkv2J+MbF7MOHu25yuPHoBThOoc21xBYdJurhPdtme/ucK6irrVZNXh30+MveXiRcTZxJ27jj/640e88eaCe/dHtt013a7nxZPM8yeOYWfJdmTbKR6+vWCz9/hxwIdE40SeSBnFdrtn3lrGMTFbNli7ww+abmcwNjGfWz788EPefucRtoqcn29FH9e2BDuIXr9X3H/jDvv9lsvrXhLTlHjytONf/sv32O0uiBGGoeii12KIZ8zIOAbGcRBZj1lFOxNN+afPnuKcoWmidGWGDCazXC7Y7C4LaSZQVxXb62vaqsY3M/phw9Z76tqwWKyIEaIfuTi/4N69e5yeHPPxRx/hXEvfDThniVHhXF3WA81stiBnePbsjIdv3GXZVKAsL15coCuFMpbd/gJlRDt50cyYmyVvHn+FP/nWv+SNo6/g1BqlKkzxCVI6ifxmSmQdUYUgo5VIQ8aYUUa8LJQuc7LsLymJ15Y6EE8SKXeSjCfwo0LrFZWqaHKDC0d88NnPCK5nyLLnj+PIyWrF+fNLLi/PmM9rUq7p+8xiseDu3Xv8+Ec/x4+Z9WpOpWc8ur/kwcO7/Oj9v6ZdDFjXEHxAa8d6teL87JzT42OsMRyt10JmilLcTEyFZEtTz6nqOV0/klOEsOPu3RMU0p1iteHy6gJ+jy6yoc8EL511MQTpsAkeaxQVGhuKVjhCwBEJKo+PAa9i8VeRmL1ta4yp6LotEKhbQ9dDv+mJAapa0Y0CfM8XIm1zdnHBMHia2qFVoq5ndNuOF0+fUTc1292O3XaLj5GTkxN8CmRpCcanEZUMRErHJgxxwPuRqBQmS5w/9j0xBvkjJRJqlsyibSHI2DJJcb25ZtvtycqitEVjcMXEtjENeYhcXV8RuoCzjqYSdqZ1lqgTu35PuIqYmeNUR9Ba9gdvhImPQVuNtpb9vsdoxZBGNvs9ilwY62PpojBkbck6sxtGEa5QHusMfhxE8tMHWfejAHtVXaNxbLc9DtnjnGkZvObF42sun1yTOkvuFTpVRC/dJH3fH+RabpOCbgPq6lYcrkygajXNvObOvVOqhWL0nRQxQ0TNMs2yZXe1wUdPWzckB7lWmEGj+4guhp/ZijyFjzCEiEkBXVnGMOJzpJk3LE9XjH5k02/Zq0A9m4svQL+TrrYk/ZQ6wBBGsJrYjxATbV2TwhalK5arJTolNhdXZCx9F8A5hn4kjZ5uGLl/coeqbfF9YHt5jakdi8WcnGPJrS0zhEyjVMK1NanPQv7bbagby+npMfN5Q9PUaD2j2+/puhGjHcMgflV9vyu+RBFXW5pZS9YKjGW2WBV5GIfmi4N8rzv0RP+YAEp1E/vc4quWJbHkPeWFqrz4dn57+/fTSV7NdSmkvul3t0Hy20jpTf58+9y/6rh1jlsYgFLm0MGutGE+X7Jerfm78w8Yes++j6VDsaedt1xvrskEfvqTn7NcLslZ0e17coTFvGXoPVkbtps9H3/0mPOzKx4+fMj2coO1jo8++ohhGIkxUVnHs+dPCUEkH7WxWCdm4PuuA6UxBXTn0IH9ck72aiFjkoLx3pNSFESmyJna0l31+MljTlYr1vM5jx8/4dnjJ3z5q+/x5a98BbLhzUcztpueYUiMXeD9H/6EL731DkMXWC7XvPnwXYiJ8/Nzhn4UCWqr0SrRtBVNPcOoRCoxu9ELxt5L96zTtKPl4sU1Kb8gxAGtE8fHa+7cX7M+dsyXLbN5Q1VZUuyF6KECwlYvXpBZPIDU7dGUJ7e5DLdI1TI8DDe6+q/mqq/7/kAxfekLL/20vPtrc9/Pj8TMRGKT717++sWOG7326VP/5uOLM9FvsW6tMTjjhPEdRONTl5b5nJMEVCTIIreglaJyoo0dQySM4QA4SeyUy9+oYr4ok9xoYe22lRKQ2kSMKWDrAdyUp1jXFlc71C099BSzgKYREhPLWQmIaSHHCZiOBfiRJFaS15K4ZQEyYxKQIMREVpm9z4wpErMYFFhVPl+O5DBIS5BzNEdHdLse3VhMzhAleUarw0OfNZl/+c+/zeAviKbin/yzb/Ct79zl2dMzfv6zp/zsw0upzkYRnjg+XrFctrzz5hts9+f8L//zLzj/tOdP/+xrLJawPbvCaEO7voPze6xNzNoFzWxG1TQYV4HWxCQGlSjE7bu0S233e55fXbIfesZx4MWLM2zoqZB2TWM0dYWwSpAW1IlRnormuC7P1hhF7aCymhAVIQVAHSRSYgEUrdYFPJdug9nMMls65osW7bI8rxTEsVTJohx8YBwyo4+QM41VWJ0JabJIyIfpkBEj0AlwloBE3qttLHXtbsYNYK1sWzEJIIspC0mMRO/xXsawjxGfRL5IAmeIIMCdEuEIhRQMQDSDTTHaTEXKIJVnEFMmq6mzQQAuZ2TMGlNAdKkF0LaOqqkL8zkQ0u+3mU+AuS0u5Ler2koblFCZyjyX68s5lsX0ZSD81Wr1F2Gm3zbtAFljtLZobYUFH1PZ+wU8Esb5tNRNHQ7CTkpZBHtuJGp0GZuTdMfnrS2mS54kmeTfwlr0PnBycsJ2u2M2m/P++z8heM+DNx7w7OlTtptrUBk/DpgCwsGNpMv0+SSZvdmMhZVdH0BmVze89c47PHzzFPKeFA05DaKtmBPBD0UDrWHWWM4vtnz04S95+OgdxiFgjXgFdP2en/3sff7h73/As6fn7IdMCHd44/Qhy+U9zi+eUzcrnr+IPPkoMYyes7MLCfp7mV+g2O06Rh84Oj4l5MSXvvQ2i+WCZ4+f8uHPf0HOG0bvD2zWiXE7n89f+1wnsE2SUFM+vwAPFxfnaA273Q64aaubihAHh/Ei2ZUzzNq2GE6qAzg4jMOBGQASePoggHwuMkCHbfJWx4WM9vxKUPrrj18t3fJye6Z0Box03Z79fv+SxIdS0pkgmnZGAAocOjuUsmV/GiD/7rrJwRmigs737ILHlzkbkAR71s44Wq2wrqIbdmgt8hSTfr/of0uXgUKXeRGpq4rKrfE+EGM4JHITczuX5z9bLFDK4sdAP3R4H6lcTVJGtLX7gDaZqmoIeSQMPbbSrO0Sayhrs2bWNoxemN6dulk/JhBdbOskcVVZ0dYNp0fHHK1WNK4ijQHtpHPHOINPkd1+z2w+F6AnQ9f1GGPxXsyL+3Ey7UqliCprw77vSjGvFGeyKoSBm7Fxu3ADNx0pt4tsMQnzb+pGkYaILEWkGGnrCqszfhwPRQ1XCgRiphvEQ8A5RB/SCzumBPUTY0bmoLznVBBGZbRVWFeYpLY6xHHWWVJyhyKU6N/rw/i+vYZP6930te8HMkoYuMocuvnIU+KXSdGjtSIpRc4Biy7STImQJl8MU8D1IqUGqEMRspiBF5m3UKSFvB+LRIsl+mmPUKAtWRliEnAblVGmsPiiL3t5QKmeFLeQ94h7QiAlj9It1ihhRPpI8CJNYypL1oqkEqPv0CYfxoVUmjNDjJBV0YyO1A0oo6jqhHURa36/vfvqUuasqzWZSNNG/uiPHvLNr79F3w1EFN1wTTIesOyvEx/+ZMdPfnCGq+DLX12wWMF63aJU4sGDE54+3eJjz/X+nBAcF+fX7HRitVgzny9wtgFq/uC9E6LfoLLlJz95xvf/8WPeeveUo+MZ1mUau+Sjn/+QzdXAsEt028zyKJM1hDgKc8w7WtfSuAW7s4F/8/95zp/+U8fD9+6yapZoH1ktBv7ub34KPnL0RotPHecXW+ZLx5A9VaXpOk891zy4e8rzpy8YTWS+bLm83PH22w84e9Gzmh9xcfGC0UeOlw3b7ZbZbEaMlotne1bLlrpq6botWmeeP7uibmC9rthst9y737JYGj57/DHLZU1Vt/jgqau6JGDCWnSuIoTIO++8y8XlU7puS4iG+VxIK8EnLs73PH12zYM3W0zvadqGYRAZmeura5arNUppTo6PePJ0z3LZst1tmLUtZ2eX1FXFWLqGXrw4k6Jh2/DwwQN+8YtfcHGx5fh4BihGL8a+u/2e+XxOCglnKn76k18y+kRVD8yWNfP5jK6/Zr1oYEi89+Vv8uU3vsf91deweglKOmAUWWJarQX4yQqJgFUp8ElhLGcteV0WaReLAZXIomeFKRloLB2GSQmo4lwNypIj+KSo6jnf+9afc7l7xs+f/JDORDFFB8IwUFeGYehAVWgNOUdOT09IUfaVymTWqwV6LQzFpkmsli1ZZY5XR4yjK0CaFgJYijS14+riisVyxfV2S+UqIjAMI1eXWx7df8DRegF1TRhHnj09597JPdq2Iew3nL24xLiKBw8f/M5ze+iC+MsY8X7abfbkolE8sy1xMxB1pmorXBIzzDGJTFvI4GxFpTLOOCpnsbYYVqpMwhcfMYd1Dl1ZPBGVE+vTFYP3XJ5f0Y8DkKgry53jE3w8Y7+9put08foKUkAsPhBV2+L9yGa3ZWUMIQb6QbxGdA6y3wApe6CW8ZMiSQu7ctcPZD/gjAMfyWNm1szKmMrUbU1GFz8si1aiSX9++YzLy2tq13J81KKsEW8rBbP1AjOrqZwjODjbXUmuoyhxYckJAVQmhAFdOUIaqCygDVZrtJ7Tdz2Va9CmZowjiZGu76gahc6w73dlPmSszsQ4Sqd9SCRVJGGzYtjC41/u+PgXL/AbjQoNOmpMzlhnUdYxDLtDLDyRUqbj1ZzpQF5SsoeSFSpqckiM3sv6pAE10iwXVLoh+UxVzzDacXV+SdCjdMxrizKWMWVijgRgDAETM61WXOyvyCTaZctqLSC66RxnVxf022tWqzXzRYs1CauEAZ5jgqQwOLJXqATzpmbsB4knj085Pj0hB0M/jFxvd+go++Z+u2UYPfcfKZp6Ru3ET8UPo7CZ/UjlNPNFDXvBc8gGayq2eVvyec9+v2G1mgEtxjgWi4r5fMZ+N9DtpftQfKhEqieXGLluG+pmTj9G5qs1l+dXVICdGG+/46EPMioZtOTb0qmfbsDyAk7K2JzgRY1EvPJvddh9BMOiyO4x5Si3sMgDcD7B9Kq8P7e/FvD+kAqVn2eRpD3Id5Q4/ObPitxxOecEgja1Y1SwOjpmv3V8/MkztG3QWvbcmAIhSozmnOXF2QU//ckHvPXmI3QyVNqhUkbFiFYa3wVa13B9sSWNn2DfdULOSIGu30muT0UdHTlHUpLC0NQRqnSWwt0k3XyLcX77mObd1Ak6/UwpxTgKqfWgYFAAeGs0bTsrf+e4d3pKLvr68/mCn/zkp9y7d8rjJ5/xwS8+ZLleY3OgNQoKdqgrzdHJnKurK7zvSUHyXr8dGMaBuq6Zz+eEmPAh4KqmkFQgBEsKkHJDGAeGfmB75XnxbEPTGKp6y+mdJcenM+7cnbFYWZTuSapD65GcQ3nOFWRdiGk3BZVJvnEC3jOgdIV0VKZDQTxPg07lV+5p5IDcqFe7HF7pfkj5AOD/RsmYfFMAyTfD9dccr8nzlb415r/Y8cVL5EUIXEHRtDPilEgoWpGUCTtdnEzeWG6imHCCHyLjMEobn75JNlOK5KQJKeMTh/peTBMwO5mzFRNDxDBGWjiFRi6bhMz8XDbFGEWKJZYW4sqCcwKoxzRpiJX6SipM+pSxWh7I6CP9KAm10eqgXzsGYa+lLJMoGanMWK3EJOTkDXa7HrsZaJslylSiCWZrSRxTICQZsCkH2nmD9ouStOyZzVvuP1zTruHBH8zo44AzLTkZ5u0RVd3w9NlnXA3P+Oo3Tjn93tc4OV2y2X9KUpnazqispZ7VKD2g3YKqWiDMwwbn5hz0kHIqxYNMHwVo7rZbfvTX/ytnz56V55DIWgD0eVuzvrNifmdG9DuyPkNbTwyKkIXxP/GTFeCcFETGok2fyOjDgi6Bxtwp5pWisdKy0rSGdlkzW7dyX1VEpUgxpwbAB2ml9kWv3mqR/Jm2k5SE6Z2QamG6tTVlMpXRtLWVVi1nIWWCKWBkkfqIUc5hNYUNFwVYL3IuIRU3dKVwujAAsyIo0RB3SlEhwZUsBGVjRpXrkYUmBZGamXwFQs5UGml1Lm3aORfDWxtp5lI0SkEkK7z//drKYpza/vUrwZoSAMMAGGEoqKksURbTieH7Clj+qwDG6eurYPptDV5tDFpJEBxjghwwetISDhK4T6AMmpQjJH0A2KF0hWi5Xtn40k2F/DUsdPk3kIVxOAHooiWcWC6XbLc7Tk/v8JP33+ejD395AKS8Hw6bxQSKT+2WVVUV7WCFQpOTrEXkEljnzGI54/jkDn/wtW+i6Bm6hNMKo1rRh04jmUAMCaPl3EerBZvNZTGrCwz9NbN5TQyBt95+xA9+8H12+3Outlu+/w/P+Hm7pKoarq+vxABq3DN2nooFfb9nGEay0sQk7PBnz84BzYOHD3nr3XdYHR+x3+346Y/fZ3ct5oquGCtOEhfb7fbARh2G4fBsJ13m6edD30MWM+j1es0w9FxfX5HSBCha9vv9DXu8PKPJmK2ua2azGW074+joCO99eX19ANEnY2ABJOWZqny7zfGm6pxz5uA4PP30cyzxG83B2/9/EdBdpD+EjX7DqKeMEchJgMJJP1DGriIXWtdvoyP36mGsI4RIN3j23UCIkdrVJdHQLBZLjo5OiDHg44gx4mKvtTC2Y5EhEGf3SbJECt+mSKx4r0sngrkJ1ifJNl0Kcyjm7Qy7dlRVXQxC3aEYFoIwpK2T4oq1hq4bSDFSNzV15QrDc0SRRMLKGNGthVJclzHSp8israkq0V2/OHvBeUoYl7FW1pnrqyuurq4wxnBxccF+v6fbi2SH0YaqdoWRL90Qk1SJ95JsTVroAMEHUriRNAIxG4Ubxjbc+CBMQbl0900mRfYwVkRfPuOMoa1tYaLfyOvc7vY5tHdbKx1hZd2Z2GtwsydOMlXGGGFMx0BKGeuqA6Pd+1CuEapKCiJdtyfGQF3X1HVdNPL3jONYOksajDEyt0dhG9a16N6Pu/Ege9X3fdHQL3OnGImbsvZ7H+hHD0pTuRplrCSAIZRilyr66gnf92Qcba4IIR/kaKbXlUEoRdhbHUEpQUgRFUKZi1KZDjET/YhWHldy2xA9yXdCHnA11ihSlKIuSokpb21wthFZjCiLiDWl0JGGw3iV7krpPBLT01zW8995agPgPTTzBvSIrTVvvbPk23/0AM01Q+jYXG/ovDC4njze4feG7qJiMa+pnOKDn57xla8v+OSj58zmC+7fO+byfMf1JXzy2PD8U6jUwMWzHSq94PTumgePGo7uKr78zrss20ek2PPHf3xMNWv5H/+nf+DewxPefPMhm11gt9EMvXR22gUED34LbduURBScC3TdnvnMcv0c/tv/209488sLTt+4w+q45eLZBbWL2Ep8F9q5pg8jTbtitxuo64raOsYcqWtXNO4T1lRUrqcfdswXjqoeOb5T8cEHz1mtjtE0hMGwu9rz6OE9zs/PePDgIdfXNTHtsMbx5PFTVseWqhEpKWsDVaWIyaN0IviROGpyDvTjjtW64eqy5+6dI95//+e0s4StIo6GruvYbHrunN5jPltQVTt8yBwdnRRZuEQM0k2Xo2e/97RNi0gAiTTIdrtFa8WL80vWJ8dIZ6hjH7bYquLx4ydApm3EzLipZzin2W73tG2LqyvpBo6a6DWL2ZJ2PmPXXzNvRBqo0oYvPXibe0dv8+juN6j0XRT2AHJbXCEBaVDmBotBkbIno7C2QrtK7k8IGFuKwjGQo4BgKfgSLxX2W92QY2AcxbQ66UiMHcrBarnmT7/7z7kerziPj9kM5yJbEntyznTdHpRIJtVNzdXlFSDM008/fsz19TVHRyvQHVl1nJyu2Gzh+bNLHr35LpUL7PadyA/kSF0dlTZ9gxBqPLayfPbkkr6pqas53mewkXt37zCzDf3Wc3l5jqkS3u/BJHzqf+e5nYPk11lp9l1PDMJo7neeVZ0wg8LVDhMcfr9DOVu6ixwpT/4vUkgYe08XOjIBZURCJWZwbYtSToqBRvIbFTV37x/L/hmkK9xaxd27d1nMF4QQudpsUFrTzlqqtqGZtXSdIXoLXSfxlzIHaTSjRTY15YCxImu23WxRBPZdD3jaRUUfPHEYefbinFWzYHfdsdmO7NNIQDFvZqSs2F93DPs9BNBOOqJDBFtpTNugnSEkT84JY8XotFnM0FUlptiDGJG6LObPStuyhkdhwtcVi7bCq8B+v5NxX1ta25KS7LGrWcsYYBy93OkYhExlHTEGnJV9pzY1IWbCaJjpOd2Z56NPr3jxyQ7GGXOzghJ798OeGCI+9Ic44XZuNMUitwkmE0lFiImG7WXPs0+veOPhPVyMeBLaZozTDOOIv75gtVjTLGtmzYLatexjgHpkVc9xVc123zP0IyGKvK7GEPEEmwgmMZvJM7/aXROCp2lbvrT8EhebK1CRurW4akZlNT4J+UxZCxiMbRj3e/GlGAdiNsALTo/f4M033+PZ8xdcXnXsrrbSVeMjwXuuL6+o6xanDbOmYV9X9P2eqnGk5LEaZm2NNpah8xKTWSUG1OH/S9yf/Via5vl92OfZ3u0ssWXkXltXd/U+Pb0MZ+FQAiWOKBrWAnAkWTBsQAIIyxc2YF8K/icsAzJgSb4RJEiUfWORBmxKQ1LkDGc47OFMT+/VVV1VmZVbZCxne9dn8cXvfU9EVveYnGoIegtZGRnnxDknznmW3/P9fZeOIexYrV9SVqIwrOuOssw5PDzA+wuqqkRrUa5pnchsJg3IIOG9i9kMYx3JB7rdhrz49FZNct0Abfe1Gq98bzK/3bPVmb64iXKrPf4mNV8QEH1/Tp/OudM4ugbRf1744kSI/RmgMcV97SaFmRqBVXX92kYMRCuFimLbJblGGmczfvjDH3N+LmuzdbnUo87go2Kz3eCcgNb1rua3f/vfZnO15gff/T6hDxQzR4zQeo91jq5r6a3h+ZOPRVmQEkcHB1xcnEMMNLst1tj9a5ZA8nY8g0+Yg0HHSeVxTY7SN2rl6c/NhtU0/5LixhnX0rUNz549Q52eQoh866/+Cm++9TpouHPnLlmW86Mfv8tuu+X41jGb9ZbHjx6xODzg6PCYalaNqs9hPKu4kf3u98/tvTRX97aSI04BkgNprcMohTU5eTbmVA2eNimCh0e7l7z//pblgePBa8fcvX/A8tCN401hnWATqADTeVpprnFsDWRMmHAaQe+U1Ii3jGP0k02Y/di6ibtc//3qWfoTZEwmj4mffwm+8vPP4j+LRamfefzpMf68158DRJeDcEoBhRy2GD2rk48kH0haE+N1dyElJfJXJMSRBN6r0U9RNvc0+qaGGAhJM8REH4TdPcRAOyi2XaTaMxuBPXtcCzM5JfwQRJJlJkYxoORtlwNLHA/HAk6Iz2a87tJ5ATIn/3GtBSDv+kjdyu9itHQeg4okNEZFtJoYsoivnzMUVcVqteLwVkk2m+MHT33ZcHD7WAqVFKUwIhFiRwgdMUpHNTc5KUhAkoqJzCqOFgW7xjO0DXk2o1l/zKrf4FzOZ948JTczYnPFdr2FCE6XGFNQVhmZMjhdgSkxxSHl/BitDHEYQE0+kg5lLcZYtHZUx6e88ZUFdx6+zvrlC773B7/Ly3d/iB5asqLk1r3bzBc584dHxPqKqv4Tyvoxu5145cWUCCNQPDUWlBLW+TBan1yT9iJWK0qnKJzCGrGJcU7AbWNlYWP03hZLndFfSxkSQdhqknkjlgJGANR2BKZvNFHlUK2kEZNbTe5E7odi9BtP13YlY/NF/NIU2kRQEWJAJ/GARyFhSSSMmhj4ick1LVPgVNovTjBOXdnzSEqk4UMUH/fSyO8bYiIZcC5hbQJ9vSk6qynKDOcMnZ/sMn4xNpuZ0qXHnAKjE5jxrYth3Iuvwb/pkjEcMVMx94mF8M8C0q/vM4GRn9ysJ/9yI08/2pnEOIV23fQ5l/yCie2fYhjH2HWE5DQI9uDNjcU4jY2ZfeMtxhE8FDnyZMHgB5Fb3bt3n7Zp2G5WnL88I/jAJHWautYTgHWTleysBSXBtLKhy0adO8d8Puftz3yOqpwTvGJeZTgd6ZotXdMQfE1MPbLmSohvnme0ref5s6eUs1OMUaxXa/zQcufuKffu3+bb3/5Det9ydbkb2YPyjkyevESDLkVK2TQ13ZComx2Xl1d8/3s/4OTWKW995jM8fP11MEpSzduOF89fUNcN682asihkXsXIrq7F9/tGwTEBi2VZSsjZ+J4MQ08/CGCY5wXO1cQojPbrn5UG6cSsnZjdISSq2Yyjo0PKstzvI0M/EHQUJYMRpcwEroUYMYA2E6P2emiksZGoPjEO9+N3X8ByY+z92WP85435T349sW2tMdgxPHnaO2VMy8A0tvjneo4/6+o7CWDtO4/3QeZamoKqzV4ynGIiDB6NxQ9eLD76QeZWTPR+uGYqawH2w9i8y2wGlv1noJVm+s/3nq4TD+bbt29LkGZKdL2nKEfwMSa2ux0xRqqqIs9EyrzbNKOPfM/QDwLIjKGjkyohBmlQiS2XGROlx305Boa+G0GjgHGJPHNoI3tgGAbapsX7QRj0I/tI1mepBayR4lErhUpQ5gVFNtkqSTFmsgyd6z1bBRJFke3X5VcahVqP4G6isE4Y13FaQyRbRqxGRLnXdw1ZnjOfzwU06ntpLOQ5bmzQTR72NstxWQ7pOrfDOYfRhsF7YggoNEZbbGbpu57Q9xhjcVmONpqYOkIv3v3aOhRgnQjQpwP7tM5dA9OTeifuG4sh+rG+kMZn2zXsdlu22w2Dl/GlNRwuqtGPVFjsddOC0lApslxYMCGF/XoJAsQIQ1PWl3Jk8dZ1Mwa7JnIrk7YfBpRqKYoZxkozRjxvJ/93KyqIkBgGpG60GTZzEA1JlP/kmUUZhbM5xEx+zwAgYaXWOGHwwb4hwdhkT9FLPREl2LrvAyl6tCog5Z96bgOo5ASId5G3P3/C0bFjXV8y1DuuLnfMljMyHdnUHVWVMzs85d6X3sBS8fLyJR8/fUrnn5JZz6OPr1Aaiiqj2TliX3LxcseH713gTEaMih+9d8bySGGLwOv3n/PwzptUpeJzXzzl85/9Cturgt/5e7/Lh+++ENZhgK5OeANOJ1ZXQVixy5LlsuL9xy94/bU5z55c8fR55O7JLSKBzSpy9vJDbJ5YHhzQ1LBYFrReLBVi6FER8ixDo+j6gZPbEedaDg9z1lf9qDbMUCi08Tx/8ZS8ULz51oEoDEzP8qDi7HzLy0tRUj1+8hEPHtyjmi0ZfMNrs4dcXb2ga3qKSrNYZOy2PYfLA4Z2kEZ4FJ/S5eIYaxLLhSWlyBe++BZPnjym71tWV5csFnMyJ8qkPNM8fHDCndt3RR2Wa/EzjpHlbMGyWnLZb0jIXj/0LUM/sN1FrDPkuSL6jjAYVlcdKhn6VvbosqrwPuBDovearteE0LOcFaje8+LJS6gDr7/+GpebK27dOaL7eEdGwXF5j6+8/uv8xrf+Cgezh6S+EKBc632zKyU9Whhq+aMUcWTGaGWFUBAD/dCSghdbRkS9GKOobKRRKHZhflShpL7DWCPKZCXKTKPH/BocB7NTfuNbv8nf/gf/DZmRda1rxWorpp5nzy95443XiTFRlRXPnj3j7t27zGY5i2UhzZXY0baN5MMoy+3TO2yu1iyPDvEjgy7TjrIqOD0+4tmLF1hnOFyWdH3Dm68fc3F2RTd0nN455OVqTdu0LA6P8eElRWmIBE7siax19fpTz+2Uwsi5NjR1Q9cNZK5it+5Yqw233Jyj8oir3YbdVU2wCaqplrX4IMF+xhSoJM2JoW+oFlbsVpMmywq0zQjJE1LLEBr6xnPr5A537p2wWe3QSmy4qnnJcjFnvZLQ0gAUs4pyMSPESF6WUJSEKPtDWc2IYYtKLUSEKdq3FDOFzQw+RmLo2W1bYgJXLuVsqDWreof34PuI71tSnsirEo8mkGjDwK6vCZ2Hq0huLfODQ7RzRKsZiEK2S0GUSr6FTONmOc6WdHVDUAkdEkZifoRtnhInizmL2YzQ9/jQE4ca52T9z6zDmgI/gFaewMC8FF96rS3R5gQPTmWgPM5YSregbxN0hqcfXPDkowuG2uJbmGcWNQFgccCVQjQRIFdqhj0JZ2yOT0CajJG0DzMHxBI0m/Pi0Y7vm5/yzb/4BVQGyQ6YXNOHjEiiD5InEpTFVTMObt0h9R25tRRlhR4G2Gy53GxGi7qE7xUYTXkwo6pKYghcba8kPyxmLIolaFg3a3of0E5LNo5W+BRpNjtsnlAqxxipAX0nzP8UL/ng/Q95+7Nf4P691zl7ecmLs5cs5xU6agqbU69rWtMSh4HbRycUzjB0HS9fPEflhi70mCxDJQg+YrHM5hnGKLbbgYTFZqBUHM9ANd63HB1KWHxZzBh8x3yYYY3iYHHE4D3WZCQfJGfOaQ6XM658TVF8equmcYZ/4u9PnidG7GCsRyeVMHsS0HQw0a8+UoqQwvj1DcLQGHY9AekTyeBnsPI/67yRkjzuK2eg0eKVm2xiOWMwksaGEHCu4r13P+D73/8hm/WWEHOa3SB2U9bQdHJuXCzmVLmDECiyjP/Fb//b/Oe7/5zV+RX1rkZpTdMLWeP44JQnHz/j//i//9/x/vvv8Tu/89+z2zUcHi6pm51YMBrDoOXcPZ3Ntdb4EOGGHZIQXcI+W+CT1qQ3lR9aaxYLyW7x4XruhVFVmjnHs+fPmOUF/9l/9p/yH/6f/kPe+tzbEBNf/spXePL0GUdHR7T9wIsXL1keHND3PU+fPuX2nTsslvP9a7LWkmUZXdcRQqAsS7quo23bfVDqRNZzTgD3idwyrR3SeMuom5qUMmbzGbELnL24ZLPZstmc8OZn7nF8MsdaSH7A2o5EAPz+957OjiSD2JTZcQT0IyFoavrISNw3W25c+7HHjTM46hPfv0G23A/Q9HN+7sbQfGUYp/39JsX3q9f/BCC6VobJLkH8b0Z5sh/ww0AKoxRisvYYgYkQBTB1qNGL0xCjHFxu+tfGKOCrD4luZIOTAu0Aba+xdrJnAJKwFIxRYCCGQLMVX1VXmD3Ld6KnxajwUY0G/QKEW6NJkjBJ8MhByyhitCOIOAiI1iWaVgBRZxTOCsMOBEDPjMaOAGRCg8ux8wXBy+Db7NbYckZ165ikM0IYxGuLCQAUaXFSSTo+ymC1JVclShn6NDBstwx1Td8P9E3Lrt4g/vAtfdPgsxaDJbNLdMrIlBZvThJVNYOoYXablC1Bldhshs4LlBbWSYwRFSPaaGZVwUdPX7DeymL19td/hbtvvc3V5QX1boeyBUVR8NH772PuHXHoAsHeol3/bdr2ORCuAcokoXNuAoZHVcBNREoBTivxfp5SORUiX9SGhBRvKvqRQSGM0eDFNkglYYMPQSR5NoKzCqMEmDaThbKSzy3Fa/jUqIQbgwRFXiqNGOlIRmIYbTe02KloLRM6+mvZj5qOESphlXhzE5NIE1Mi06PH+SiBEX/cm0uCwkdoBlFmFFMneWwAaQXOjh64SfyqrJUgXa1HoD+KzdAvcmXGypxCrHaMtUQ9hq4htgNKjw2RZCEKyBXjwMidRU+ysiRyRRTjbaMdyxgoJRYw4nfv8Te6h9IMkc8dUHGfWi0LuVgIWSc+iKQkjD8j4VZaOawpyTMBfsQFJRHHANQQxGNbLF+u/bv3DI8YRw/hidUoBccUnKUzizaB27dPaeotm/UVZy9eEEYJ1p6JGwQMQz5yGUPKoLRYJ3SdHzvh4DKHc5mwp5xFJThYHLCY5+x2l/R9h80D2gSGQdjx0Qso45xYFzx79hOOb52wWCwwxrDZbcBabj94DW8s9UY8W2PoRim2fCa5ccyPj1A258Fn7nNZr/C+Y7O64Pnjx/TbC774zpvce3CP0ztHGJfhfcQpuLw858XLM7ZNS9KWLBO2czVf0nYtWV6g6w0g6oosN+S5JcsMWZ7hcoPRZvQ977HWcHh4RF3vaNueGAcBkZHgSGsMau/nDMo4ylnJG2+9zmxRUs5LsqK4VkkpsZUKUYKCY5QsjpsqC9lcE2lqpoIw635O/Si+gq8C6NPfN30qJ0DxJkN9UndoLY1KaVZKQWaMRmkjgUXj/Yw2+/k0hW//3Krhn/Nq2w6tDUaLRYzCj7+TJkTYbDacn5/jXIYaQ5a7pqNpGkCJ3zwB34sXdp7nOOtG5pc0i/KxWO26jjiu8SkmQoq0Tctuu8GoiiIv5D0KYsE1qTZSErbbvviKcbRHka/7vkdpxdHRIcY6zi8uaJsWRcSoSQEkcmFnLWUx42A5J3MGazRZmWO1IoVBZLoKFtVstLFxpBjIMkteHMB4QA1Dj9WGLCvRWu1VKUWeSyNgtFKYLFamUKOJaV1V1b6InQ7BeZ4Lg2YQENhpRVlJcGXfdrIuII1kpRS+b8lzKY6dtfgQ0FMIEtcWKsF7uW1cd2KIsk6kRCAQlcg+J9sdP0go0lSYG2MwY0E+AfLTAYKxUeH7/pVhKMzx68N9Pr4vWmmyTJh4MQaMNRgrMt2YAuvNms1GlCyZMVh1SlVVOOsk2HC3RWtHkZcYrRmDUdDT56zEwzwRmQLt9bi3TAccM+6zerRnmGykMi3WNwQl79f+gGTIdIFRc0hgrFgj+Cj7ljTZh3FPDpTl6LFe98Kmtox7kBEbwT6SjFjdxTEnaOjFNiYMQJLD0WJ+yPHx7U89twGsMgx9y8lpzmxuSQy8fHlJbCPWVMRYIEIpjdWBFHtW60uMesq6ec6b78ypZg+5uloxW0jo8IsXPRdXj7n/oOAzn12w3W7ZbXsIiuRA55rFouLyquXZk+8JaaH/Mh++9x0sSzYvI8sTQySyXjWcHEuegXWe09uaoc/AW14+63jj/kP67oI7dytef10CdXedNPZvLY/Y1g1xcJwcn3BxdYnLA0dHM0iipJjPNLdOjzg7e8F8pilyRddoXn/jLlXlOL/YkhkjdgTGsNt1HBwIkabt18y14vbdGRfnO5bznBfPtzx5/pS7dw+xY56PMY4QEl07UOSB26fHVOWCl+fnwMByeYz30tgyRnz7j0+O2WwucTbn3t03+OlP38Mah505mmbLdtdwcrwkDJ7cWtp6zeJgRt215NmSeuep6w6lIovFAbvtFVoHyrJgtdoIq9walouCjz++YDk7wuiCw1tHvDj7CGsdPkR2tWc2OyDFnOQTzXZHkRXEkChnDl0uefTx+8QeysVdvvilr/HVN/9lKnuPNGTCIE9xPPeMknYtsn5Zc9PYGPT4kDDaYJDznEJJboVKIymhJXjJaIhjtoIxBmM1fd+iYgIERJdaWYviOYzEks6jPLxx/zX+9P1/Sj7P0bPZmH8Vmc0rtJam1OChrOasNldUi5z15pLDo9c5OTnh7OxsBFRGZZBRfPTRhwQtJ7MH9+7TtjX1bksIntffvEvbbTF4qqri1jtv8+jxI0Qd2LBeGzI7Y3m8oOkuWa0u6LtAppdk+vhTz205KwpxxzoLnSd4sbhYdztunyypihlnF1dYk+FsTheFmd/3EY0WaxBliN6jkt6HICsmkMhhcvFVr9uabmjJsxw/dLjCUc0KSIm29kQkHLrvO8qiZEAaOjEEQgrjWm0pqjkaabYYbcmzAj/0eB8pioqqmhNRKCzbTQuI1VZT16QE8+Uhja5JUZNsHBnmlurgCE8iKLGjK+clbmGoXI5VFq+gTwmTu9EyzNDudkTv6cLAoCLZoqSaz6QxO3iU0nTblhwDqcMaJU38uhFWuApkOlEWjqTEplGriNUw9A2+q4kjqfDo6ASWlrOzSxLg1YDTFUOT8/H7Z3z0wxf0G4NNM9SQWGQZyfegvTS1Y0vnPUNKqEEagCmJPduUEXSzxpxyaqazjFjGKWJvMSiefXTJn+bv8eW/8DrzgxI3k3NI13XEaIgYms5TDAltM4YQ2cVAIuIWFZlJLAojny3AxuP9gCssfejJMkc1r3DW4myGUVoCjlXCx0Gs5jKHyTNIgeZqgx4USXmqakZuLdt0RdNcEWPD40cfE6LhC1/4Eq89eJ31akXb1GTOMqtmKDuycOsafXTC4WLJdrfjfHWFzg06F3ViVkj2XEiBsswAj82gLCvKcoZxiqGTeqhtOhQWrTPms5Jh2+KcZPsUNsdqJ3Vc1xHGtSJ3EjqdZb8YiH4NRH8CON87oI8gZJJzyv5uezBd88kDixDOxAJ0v45Mz5Wmf02gux7xsp+9rhW7N15XCmMjdKrPx4djrNEmV4ixFvN9kPNOUqxXW1ZXG4zOmM0OWK06IT8mMNaQ54a+a9lttnS7hNOa7dWKz3/2c/zl3/xL/OgHP2LoBq5Way43V8wXczabNd/4pa/y1/5nf43/9D/+j3l49w67Xc1f+s3f4Ac//AEPHr7Os5fnPHv+gg8//AiTSbPEWgf9IHbMfmDKXNvbHd6wEH31c+IVy5c0fr1XFI/vW1M3qBg5vHtEHAZ+//d/n6987asc377D4w8+5Dvf+Y6sy4NnvpjT9QOFNljr2G43aKOYzWZ7UHwYbVNDCHs1Z5Zl7HY7mkaIRkdHRxRFsbeNbJpmrxDvuk5yoFxB23RYY0BJ7sIuDDx9vGboFffv3+bW6RHV3OJUQOk0lt+jRbdiJEDrcfyN1glqYPKfUCmNlmOvjMobX+tp0NwYz9NYZMS+5Myt9nf6JG74s+fhyUZoPH3uG0jqE8TP6yf89Gfqm9c/vyf6SOUPwRNTsw948n23Z12mOPl+SocnpsD1WxmZPKAZ2b4xBUIfxSfVBxKOkNTog+nR46E4RunERSIhMgZQyX4dYiINkTr2GJdhs0IYXKMvexwBRh8jZkoLNorMaYgWq/XeYy74OMoKpaMidiDQeXm7jZU/kxQhM4rcOfLMELywYa3LKO58hjuvfZ3i3pvYosJVFdEoKUCmzhejJ6m28nsqmJheWonsRXgeGZktCPmcPE/46MmyTNivRBilaGU+JyWLwWFxIk0zmTCvijlBF8wPX0drSx8iZhgw2ghpL6k9iNrUNX1bc3X5kouLFavLC7702be59/AN/ul3v4vyHX/0J9/lp++9x2//9X+N6vZdHs5O2J2v2Oz+Puv2HDV49AiGm5FVGRMMXgBvxQgwjUCTMLgZQZURmJ4O7zESBg9hwJpAGgGYycbE+0A/eAYfMUqRzHTYlmlmlSamV6bVeI0WQ2IFL4fbSao0Mt1TFIAHLZ7sRgvb1g/96Ik+yn7G39OPC4hBY4hYBZmW22RxZc84nfyilJKGzwCYxLiYa4xRiMw/ThjC2OWLe1sBpcYMgSDNp19oIdDVGEJkMMqO4XoBj0engqg8THYo0aJSQYp63yxL0ROiIo0Mv74XK4MYJGvgutAT1r9WAnwIiK7Repx74/40DANhUMyyE+6efBZUi3XSWFE4iBKuKRYTlinNSCvLIj/CaIsPAxAIyeOHnq5v6dqWfgzac86N1gjXkn+xW1B7dpUUrML077qeumnYbDZsNhtCCNy9e4em2e7BtBiksM2M2wOpWl9LL68DLxkZtpq2bQSAbBuW8zll5VguK1ISwK9tNvheoVWkH8bw1Bhomi15NmO3veSjD37MYnlIXhYMPnC5WnNwNOfzX/gsf/j7/4i8yMbAPfGKDl6S61Ps+fovf4PDwxlPPv6QFAcuzl7w/o/f5Z3Pvs2dO7c4OlqO1hgFPkbc6w/RVoIM/87qd+i7nhQTmcvwudhthH1RYijKnKLIMVZLQKOrWC7mxMhojSHjpWlqdrstMfo9wHqT4S9roySVZ0XF7du3uXf/Hq+99toYHCgNsn1QI9fFz77D/Mq+OYK0o8IjpVfqzld+fi9n+8T3b143u+c3b1djA08OrK8yG6aic9JGTA00uabC1F6//k9xxcgYKpmRuZy26cU6xiS6rqXtur31SFG4a+b+KA+emBkxBVK8ttSJKTAxCKaGk1IGYyZGsjApirwU7/8Q6JoaUiHhXJmFKOCusZZZVTHJKAGyTDMrK+qipg8eawREr6qKvmto6i0kQ5GJtYD3A8EHcqeZlRnzqpCAamuZz0py69DRM/QtvR+YF5ZFleMTpOhRRLEcchl1E0geOUxVYlXSti2pjfvxMAV5pSSMLufsfk5PzYFpLEzzfhgGGaPDOE+CJ6WA04a+6wTsDX4fvuvH8MCyrCirkpQSbdMyDD210axH+Xk7At9Zlu+Dc6WRz/4zjOl6z1EjOJ32DJBrFvlk1zL9DjFF8tFiR8AZWT+LQhrq03ix1lJVFSjFEOP+UK+UoizLPQNosnWJUUgIUj+OczBKCK9ScQTbJFwVIGnJsPHe40MPY0aLZA4M+2BR+Xyk4NYjaD7NocneiVF+u7eJGiSAvZodEKPF+62wOE2xzwkZho6YBmncGEOR54Qh0bYN3geKQmOMI0VhAklgr7CSBhXouqkRp4lBGs/GZJTF7FPPbZD6whrFrLDoGGg2DWnQ5CYnKxb0HaSYE4YdQ9/R9DXr8JI870nGY3NFli85PjnE9ztWl5HXX3uDL37J8PzFY3bbM776tWMuznvOXm7RKiOGhDOQ7JpyrjFKsWuf0Qwvubp4xPLYkWLC2IzTW3OK3LBe7eg7z90HS642gfV5gL7g0QeXVLOeX/r6IdU8knTBZz77Gf7o91/w4umKoxMJnLt7b87V5XOWM0vf1pSFoutrFgczutATVcRHx64ZODo54enTM3ZNxuIgR1lP6D3WJnbSK6LvAy7T7HY983k+HjoVd+/N8D6x3qwoCktRZnIGiXB4tKAsHNttzcHBKU2zJssMSksAvPcdxuSy5jo3NqNyPvjpx2zWA3XTcHp6gDGGe/dPhHFaaZR11EMkL0q6TnFweMJ3/vjHzOcz5osZQ+/xUTFfLIgRrM5G/DqJzYgLvDw/44tf+Co/fvfHaNOSU4riNUoY/f2H91m9fCm8RaMJ2rDabqhmBQ/u3OPqRcMbdz7P5x58ncPlLTInnqg3D6xq3C/TCKpNa9tNYI/oBJwIEoBpjSGGgRQ9KQR8L3aeU93VhZYp3NoYJXWZcwQva2TXdeM8lj3+xYsXPH788RjkLPVZUcy4utoRgijbjo5O+PjjR/R9z8mtQ+rNRggGmw3z+Xy0xNGjr6ynKAre/uybfPjoEevtjhB6NltPXmUsdMHF1UtClCbM4MGHjvWq5nJ1CSZwdHCMDwMvX7zg1u2CmLxYfFjLdvvpa3NVlkI+IWHRLF1F5hx96/Eh0hSJJoedS0STk0YrI5Khq9f4mCidI6lEn3rq0JJVlhgyUpDG9eA9PiWUcfRdie8CJ4sD/DBwvr7g9NYpRZZT2AyrjKh9+oB12Wh7kzMwoG1gaHvikGNdRt+3xM7Ttw3K9+QpkXRGNZtzsFiyqzd48fcky5fovqe+3EjjepZRHpTEEFldrUlKofOMrCzxMZFbR1cPzF3J6cExi6rg/PIFje9RQwepJwwdikS9rnG2wLmKXdvx8sUFt7zBKctCz4mhp4stwUe6tiN3mqGVtUHrhFEak+UU2Yy67eibmnbXAKBjwrc9dd+CVRzdusviYEG2rbncXDCYGkPG2U+3fPjdc/zacXJwS8KsM8EYymop/swK8IHMWsrM4vu4rzEmprlzbjxHJIwRIpO1en8fIWXI+SoOERsLXvxkxW7zA37jX/0W84MlxcxQ989QuqfzNTE4Zn5gSD0602w3vXijp0QfBsoqYwi9rCOlg6QJvsEPPfNZyay4S99CiortrsUWCa09eZ5QVjGbz1HK4LuWIstx0bJrOwZrKWcFeVXS1K0QAYaW9dUL3n9Pcfv2A06O7vDx9jGzqmS5XECmqLsagqd0OYfFAUMwsGsYwkCWDCpCVRWQNM2uJXeGvm8oMlB0+EHRKotKjmGI1LuOEDYUuSjiYuyJdCQDzai6CY2QADvf0zNgnCYvLeYXxuE+uTbEV76+ZvLGV5no+wViAtOvzx3yM5Nx7Y27KsYm157Pu2fq6jErbn9fkIPDK+eOEcwf1/+9DUxCmnJjfaXUmJU3ft9ax3bX4oeEQoKx+zawXl9SFII1KKMYvAcrAcYpDJASf/xHf8w3f/kbVGXF/Tt3yV3BxeUFgUCWOW6dnvKXfvMvEuqaoW24dXjAV77wee7fOaVwlm3TcHh4yK5u2O62ZKEcg+81eZ7T9N3YIFZ7xfS0j90E0CdsYPr3zfdaWzOer9RoiZmYzWekYeC3fuuv8Dt/57/j7OUZ/8P/8Pf42te+Ttt2fOOb3+Bv/e3/NyFGsrxgCDVd3+ONEI+00XvWuTTmR2u2otjX325UsPd9z8XFBUVRcHx8vD+bTJaLk41qSprdVhQuV1c1h0dzDg/ucnn1kvOXDVeXZ5yfddy/P3D7zpzb9y0u17g8oY0nJbGvk4i5sUkz1gEmxT2+lm5+pX7O3pdGUDBd31elqZ0zjb9XJ9ZNRf/0709eGi3gLNezKo044M/i5f8TgOioMXJ7DP2MUdiUMQizUyaRlsk1BYUm8TnXygJptG1JpDRAVCgfGcZDWUzjoTw5Jr9ajYCrAlKmvXfkxO7XSok9SIiEfbCXAI0xBUIcSDHRDZHeK6wVYFSpKBJeo8iNeKhZI0xyH8CP4Pkwhtr0/rqBp8ZmSEoCrOaZeOalqAgpoXVEaS8d8TFsoGsaTG7RKqCcJirxBI8pEnxHiN0YWub3n+3URTHaYG1GEUsiERsDuOuCVSdNiLIIamsxymFVjjU5ymXorGCxOJRQTzTL2QlM7MYERimMsSN4q1BasWtqbp0csV6t+cEPfsR21/PO57/I8tYDtldnfPkrX+DX/sIvU5QFTd9xcvcBv/m/+j/w5q/+Vb73D3+HH/zB73J5cU7XNajR2MUHaAYB0uH6fYQxAjQlQhR2mI16lMdGfD+QBo9RAVIgBAllCoN0Q0OM+CBqB5Vko+uVAOkxqnE8iMe4T2K9kpSSQ2duKEuLMWLDE2OQ0LmRgTfEiA9BwGMj/s1KCdAt4Kga2dlgtaKPIxCHwF9Wa3KryZQixGl+C/tV7Tc7te8SpxHIUynubYeMEuuEPaymtfiSG5HW+CHivdq/r5/2WpT3iEEW7wlsVfkEXEohF5MfmaoGUobVFb4PtKpmGMzYUFCjZ3IPSmGtAJ9xZDLdDJzUxsCePShhNsJGTwQfMLHkIL9HaRagBqzRaO0Ai0oKYyZ2lLBrUpTJk5kMkmIYavo+0Ywp8V3XjFYIwnLK81y8w7QmxAk4jGjlKIoC50SpkZKEAWdZRhqBmTxzLOYzijzn2dOPaYqCuhb1htLSBOqHfl/cDEOPHlO0JknmNaAuHoqXl5dkWUZZFhR5QVwsKcuctYJWJfGqjJEUhz1TKyaPCj2PH33A0fEJKAmK2u52bHYbvvqlN3nx9Kc8efwxeeEo80I25VHedufOXb7xtc/z8skHhH5gvbrkO3/yHY6ODjk5OWY+qyiKfDwIj/7JxnDvzh1+/dd+lR/98Mc8fvSYEIWhYLRYZvX9gOK6MbFcLinLct81H/zA0A/UdcNutxsVAmH/vvRjU2Ji9t60wxB2dMbDhw9xzu1ZwX0vzHxjb/g+I8Dl/vpEDToVQjL/2G//020TQDDZrPxZ181N/uaf8cY9iG6tfeW26feeCjhj7M/Z2qeskU935VkJSQJ5QT6TfujlM8NTljmLhfjYxyDSP2cNeTYTZUYvVjjOKGye7SWRRmnc6MknzU4JvrRGg9Gk4FnMKu7cuc2tkyXRNzTNDuesHKyMod41hBjIrKMYQeIUZV4bbUjzyGa7QbUCRpZFhrUC6KqRGaOVlTUxSRNPq4Q1ShjqJFSK9E2NKyu6esPFyxeEFDk6PuKbv/QVhhhRWtPVG1Ty+E7TNh191+Gdpe86SGn09fa0tmW33Qo4Pa672ogKKSbxvicxhnNOAaNxP/aUMoQkoL/Rmma7lf1lVECl4PFDhzWH3L93h7beQhRSwZtvvsn5y3MSifOLc1KUYnq5mKEQQGRqShS52LpIsFuLUYaiLMZcgp7BB/KioKxE/980DX3f733P42hrBYmiEH/5yVMyxihMWHf9PZBmTYgR58zIEJ+87v0IckvTQALKFRGxtui7ljyTECgRFQhYHryHpNBj8OsweigrJQ06YzQ+DIQY2G539H2PNRJEq3Xa7zN5lmGdyORJAy7PZZ8Y5wPjYRkU2mQQMmnCak2R50IwaFqC77BalAc2KyBq+sajYsJqTeEMje+JvkdhyUZPW2MLsbRpoe88k9Q5+Cjek7/A5awhc5ahrtlctCyWFbfv3YWYgXWszp7hfcC5nD41zCrH7VsVxgaKecasWhBDxnZds15Fzs8DmduxWBxy++SEqsg5v9iQFQ0nJ5aLlz1EYZPcvptjsh1+UOzaM+aHYIqxPvOGrm64vKgpipz5HH79177O1771DkNs+Tt/+8f85HtnZF5TZI7Ls5Z2Zzi8rfjww/fRNiPPJTOn72rWV4ovf/Eh290ZUUXaXuxxmq6liwPKZThXUdc7ZjNwmeL0zhJUS9c35IXYTR4fF/gw4Jzsu9LY8RweVey2w1izaLwXFr1SDoU0XHa7jvv373BxseLJx0+E5GAUm80V1mZk2Yxh6Dk8XHB5ecXZi3MOFvfJsyVaTSodwzBAXTdonbhavWQ2LzlYLgHL2dmWvrOkpBi6wFW/Znk0A5WRVMVbb9znow8+hBRYba/I8shuu6MoDvngw58CA7tdTZ5XlNUMn3ZYpxmGnqIoyM0t1psalTm6IdKeXfGZh29xcu8NfvWr/xLL8j6WAkbyaVJx5C7GkfwiYbshSsibNWbMCfKi2lPy775rUJnF9wGm2nFUBaYU6buAyxyzeSXNuiBs68uLc5bLA6y1Y6h0emX/VyNCY7RBjUxnomK9qlkuF/zkJx/w5puicKtmBYvFAmIiz0sgcXW15uj4BGsdL168kLndNRTzgrt3T0nP4qhUDpTzBQMtTbtFKStq2kFq+LpuqaqK1fYCSKzXK5SK1LUE/c7mJVcXLwl8ek90l2W4zBFDT993zOY5R4e3WF1tqcMGVTo2vdi4xJBIWlSkKQam4EEJGRTL1KQSJssgSlM4yzOatiYkBSajbjzOznBmjmbg4mxNPBKChDOG6AO7dsuQAsMQaHtPVuZkeUaIHbu2hpDoBsV6s0YBTil826NCws1KYoTtdkMa1/5h8BR5RRwSSg1CXOsGqpmTurvKCE0rmSGK0Qd4JK0Zy3y+4NbxId3Q0K4uUSiaumXoWgqbUZZzUtBkZUHbRs5fvIQAt5cnHJye0g8d22bLrm2kOWQM0ffMbEnhxGtbiVQeozQxyGfvg8ckTb3d0fgON8u5WK1JJpNcMK+IreHp2Tkffe8S3S05mC3Js4Ljg5Km6dlsdqy32z1glmXFeL4oqOtutGaQc0mWiY+8gOVKzlhc16kT0JbUqAB3juQ9vhm4fF7zP/x/v81v/qvfxBwGuqEVtrm1aCQoWBNxLpN6QCX8mAeT5ZZMO4ZeQo+VCnTtgNIyxqqqIM8s61XD1WZDhfhHz8qKkISl2/WehKYqK5Z2QUprdn0vYyQE8rykix2LxZz5rOT84gX9MNA0XnyhDUQdcbkjDYGoPElDVJGYPMdHBzRdTYhemjDGUR0vOPfnxKFn6P0InhpCgK7t6NuWlDrJC2pqqZXOG2lCEWi6GmUNJjmatgM0m90WMgmSdwqcNn/GzP3zXhNY/urX12eOCRIfc/z2Z4zR4zzdOG+MD3GNX37ivhNBjNEDfFLjk65fwkh0k4dK1yTZPbFJjzjo9DpHjCIpJiv1RCJFRdf1I2u6ZbPZ0vfDSOzJ8WNeWfQBozV9CAy+Z7momOUZz589Y+h7vvCFL7C6XPGD732fxXzB3ft3eeedz/GZz7zFw9ce8A9/7/f44Kc/5e7dOzx7+oSubfjsZz9LRNH4xHw+59bJLT7/pS/x+PFjmqaVYHhj6LsBY+xemTTZqEzzaiJETSD6dI6b1CHXALuM+2KxIA2Bum353ne/x1d/6atsdhsWiwUvzl5wfHyLX/r610nG8sMf/IgPP/qIEBIffvgRl5dXLJeL0cYU2lb2mImwMimYJ9Vnnud7Qp9SagwmL/Z5RVrrkawT0NoJyYVEnpWcv1wxmxUcLE55efGCzWrDdvMSo2asVzV1M+fegyPu3F0y+A1KdxA70FOWZBizAcX/niRuJfKd0R5V0L2RXS6KzhSFyX7TNhMY3Qumn7nG0abx92oz/2dPySGFETWe8MWbZ/nxf3sQ/gYpLl2f+idlwfX9/tlA+59Di6KkG2wM2lpQkw+mIsYejfjiCjNCEr65+QYoxuJpZGXFnmAMkVFapj34RO8DIRlSksOwGQFRa6HIJiavsIenEKek4j60ROvEJElnBKpTUuKzNgH1OmEzjUoJ5xgfX+NH9nQiEZUw2Nsh0vvJpWayhhFGtdEKaxBwtRepeAot/fmHPLlQHHzeUN29R7ZcQgRlRQ4W0+jZHKNY2iCSkhCFTeWD+GfGKN0+bSzKOXTwaKUllGVkLac9e9lgTIbVGZZs7LwXOFeS5zNymzH4Hp88Wjnp+txgVmMs7cjSffcHP+D23bv88ld+iYuLf8if/Mm3CTFx/7XXWK3W3Ds9xiixgNntOinyqgXDndf5+r/1v+bBX/iLnP3w+3z0/X/K9uqMIjsHNzCElhj9nnme1AgaS4uUmMTuRYeIjzJBY5DPRI3hX2EY6FpheaTx9/fj4q6VKD+HcM1+T4zfi5E+qamslANvpihyI00QJgYbEsxjLH3qJLBseiClJrtdKeImv2aVMONmENK137bTicKCitAnGPbzcwrinf4nDaK9wEpDNjLqjZ5AdGlg2ZHNKiz0yDBE+iHsGdyf9lrO7u835mlRs8ZgncPaAoUmxgHxyFJMnlgpaoahI+zXA/YsVmFRmD1L6WY4jhrXEqVFairzNk7NblICqxyVqyhthjbjwodDIUCksTc7wuzZk/K8nqvNir6Xw0ff99IQSQmjHU5bae4lNR6MAl3X473HuWuWeoxJPudxnOa5SMG0gszdpd5t+dzn3uG9936yDyf0vidGke9rI82ZYRhwQIx2/Iz1nrEKkFKk7USmdXg4Jy8KjIVhaOjbVlhdw4BzcTyA1qQg3qJZpui7gadPdhweHJFQtF3LdnVBX2/4ypc+x+nxgTDtXLZnoy+XS+6cnhKaNR+9eEq3rXn04SPe//AD/vJf/S2yXABqDfi+R1ux4IopYIzi5OSY3/i1X+O/Pftv9w0ADeTOQYoMzhGTdMONMXzpS1/i6dOnPH/+nKurS4ZBDvtt2+yBbFExDGOjY/Q/vmGbYa2lKGdorbm8vOT09FS+NxYZ1o4hZzeY6K8C2j/LFJ/uN/Lsfua2fcEq//hZkPwT97v59bT5vwICqGtvy7quaZoGpRTz+Rxr3aiu2D/q/vE+7VWWhUhgu2ZkzfYYq0fQXlOWGcvlTKwOthtilHC+siwZhoH1ekO3a3EuJ8sEuAxBDuZWDCapdx390GGtZVbNBQzOLQeHS27dOhY/2UaAfGnMSRPcWoMLZiTIyMKqtcZqSz9ImOZk57JercZGfKTrGoyR99cPLckDUeoAlSLrqwtC3zCvZmgF86rk4uwZ9eol7XZLlmcczAtO794lL0rKWUmI8MGjj1ivNvvmWTIF3kf6TgIrsywffdDFbiKvROGRVKAfepxWLJfzfbDYMPgR7BUigTCmI9q6fehpDB5rDLOqkGDPrqOud+RWcTAr+LVvfpPz83N2uy1VZjh88yG7usakMXdBazLnRvWdou17AaCyTObTMDArMwmCHW2RdDJUVY42DjXKNMvckWdjCGoSZu1iXk6Dez9HpwPGtN5Ono1Tbde27Qiqy9wLIYxNzHbvzyjscC0zLnoBfVLag+eTdZ9CwHYZcxIQJ4SMOFo3yefU9x1t20nAdtIYIyol5zLJokDtmyAKjRmtd6a5aJQabZxFIaiUG9c7K18rJIxXe7F9CxGVRM1jx9cWfMegxfpGKwmAlvXmugkrdjrtXt3U1B0XF5efem4DrDaBu8uMw4OKg1ni6GAh4887mn7LEFcMg8bqA3QqeePhW9y+sySklqurC378g3NePNsy9LBa9QxDIs8GikxRFTm5y7h1tGAxm2P0Ed/902d8+P6a9Vni9OSQapbTRcWzp54sNyyWB6xXVyQCJtPcvmfJnOJgueTXfv2X2TSX/P4ffIePHj1mCB1Ht0quzmvOvxMo8woz23BwtKTeXlBUCW09NmpWVy0pJOaHGcZqMp/z08cr7s4MZZVxcVnjtKhCd3WNNprN9orZwjEEaVwsl0vqusFaw3Zb03Y1RaE4vbWk7zXOVfStp23g8HBO37d0rTRzmqbmtdcPubzcUpUL1qtn4+ebcJnjwYP7XFyc44frxmuWZTRNj1YVr73+kA8/fBetNXXtUXrHrdOKmDryfEHbNFxeXPLg/m3OXmzkcN6Lr7OxGfV2jcvh0aOn3LvzgGdPH3NwUND3LVmWk+eOoRO1r9KG9Xrg6ChRVSV57thsVrSbmkxbjo6O6FKk3W44nh9h/IK/8Et/mcPiPoU9Bs1IahI2mUi5g/iTqjhaNUXS+PW0DoAiJT/aEMr8HnxH9J0wZmPAKEWRFahC0TQN66ur0a5NYUfLtqapyXNRAAzDgHMOP3iMUhweHkgGQbJsdx3VrOT8fEWKUq8dHs7o+ob5QvavR48+4tbJbdKYo5AVJU03UJkMpS1JJVFDv3jJvTt3+cwbb9HUNdFqtnVN0zbsmgarc6zNqevdvmaMQ08IYIylHhqyPCMlsU/zIfL5L3xmr/L6VFcU73hrC/QykruCg8UCnQzaR2mOxJY0KkX7ocNZMxK6xvytkYggRBazb+aGGGm7Dq00QwgMQ0sMitliTpkfoHUinELwirbryZxhs12zbdZENMMQCR6Gfs68OmBVd9AnysKy2qzZbdakpFguFuAydt2OWYwwiAVWnrvRMnNkGGpNVs2FtGE1XkmuyLJY4s89Ycy9yXMjKqsUQYttyJACh8e3eHG5Yr1uaLsWjeLu6ZxqNicMntlizm6opUkfJRy4rByxDRSLOe2YE5TPFzR1IGjNkBKbzYqyLMFomranqTtSNEQP/RDpvMYVc1zu2NUtWm2o8gWFDpx/1PHhD57hrzLuH51wuDgiMnC5umCzrhn6uCdWTMSPoigw2pFlElgaQsQ5IfxISKuA1yGEUQHn9m4AWktm2xCDNOlRWOUIneflozW/+//5Nl/+Sw9xM8WsyMkzS7/zpK4l9g2d7Vke3wHEp3wYGpq6oSgyrLEMfZBA2iA6+bqpCQFm5ZKikpyVummIqiO2AW0t3tfEpDiaH6BtICcnXqykXuk6MhRVUTEvlxwczkFFLlcrfNnS+5bDWzNgYIgtVmuiCng14HVPS0PdrJgvFpAsbedJg8diSUNiu9rSNTu0iVSLglm1oGsDq3XLxcUGrTxVNZJwhpZ+iOTRUJSWXV1T7yKzYoY1lq7vqbstrtNYDJ0P16Dzp74++QAT+n0zt0t94v83ztAjMK5GMD+NBE8tJupMk12PmYZ7AH0PoifSyCAXMsH1K9mfZZB1RDBQPdZEkyp3wveu/z09z/TqQwgobVguF7zzzjssFy0//tFzrq5eELXsG/3Q03YNeZaz61v6tgU/8PCN13j33Xc5OTrhr/8v/13a1Yau7Tk5PSYEj84yvvv9H/Dee+9x/8FDVpsV2lq2dc3f/ft/H20dF9uai8srsjxju90yn88JIdIPnpF3tz9rTjUtXCsz94rMkVClxz1Kzk8G9IR9SMbE0HcU1vFbv/VbZMbw7/2Nv8Ef/MHv8uWvfpXZYkk1X5C5nLc/9w6PPnrMf/1f/01+9/d+j5OTE4bRKniyVJxwCO/9PkS0KIq9mjPLMuq6xjlHXdcURbEH2YE9gC6XZPeEkDBGU1Vz+r4jpYHD5S20clycX/C97/6Et99+k9XVjiePVrz19j1ef/MWNsvIywplBmIULC/LZe1RyY0gehDLsCROIAKcj+N5OhMjhJg9gK6u8e1XztA3/og7xM2m0s/OIvF/uAGEqRu3TnR3pucQC8fr5x8/8z3v7X8UED2gtEMb6f4qIPqwZ5Kq0fdZpB5RmKxq9AlK1wzxECN+GAM5nQDzKI3RiqDE29pHYfhOXtTGJJzx5G70HtUGrXtS8sQobJIsN2SZFjBSiY+P0oiHI3rPfNJaGMXWaVQMOCu9C2MheEAJSC8yN0U7SBhmGj/piVXdB5FdKKbk2oQioGnxm5c0mx3PPzyjuv8ay3e+wJ23P8v86JBBaxIWY0pIA9okdOoxNhIHL8FmCobQ03sBz0DCYTCalAacsThbjoVsEnAQg9YZWlmstjhXkGclWTHD5hVFMScpS9P2lGWGzUphCCMMEoWiHwZWqytOjg94/OFPOTxY8vVvfA3+JPLtf/K7rLdf4bW7d/jRD37Cr/7KL9P1AhpsL1c8fX5OiJHzrsbkGW/9i3+Fb/7P/w0skRcf/JCL9/6AZ//g90h8hFYyPsLYmRox9Osup9ISLOoc1ok8y+qAjop+8PRtJ7ZAWhNJiGpd7SdNHOfLaF1PSBIw6hN7JYlS0pAx096SxgAw0t6724dEP0QGH/FRulRh8HTtwODDfmORps64EY0TPQGZVmRq9O+LafQIFLB/RGXl+camwuTptG8GIb7ZaWTYh5HObo10fUOIdL2nH8Qm6Be5jhcPmNK4pzmr9dRxzQTEJ46NnbAHuknyOvZWOONi9GqivKwFIED49HMTOVNrpIM9PSYTSJWwevLfGhc/JtBd79cbtNqvORK8t6Put3R+BM9HW4Miz7E2I3c5mXFjYZH2Rbws0jL2JhsXNVpqxPH3aluxc7m6vOT85RnPnz1lt9uKzy6TPcmwX6Cjj2MXWOGDx93wWvOj7UgcFQ8xSoFaliWZywWc8QGlJOU+D4GYPCH0OGfFSmvo8Ur80x599Iwf//AHHCyPqcqSlAZC25IpeOu116Q73XWsr1akoWeod3z8wY7HXY8eAjNXkPWeEs28KMhyi7UKlcRiQceIdW5kmnuc1Tx8eJ8sc9S7LUrN0UpCgY1RVFXG1dUlXddxdXXF0dERwB407vtuZGAbuk6aHVoLsDqE0epptIS4WczcvXuX1998i+VyyenpKQcHB6Mtz+TH97PjewLMzI3mxU1Z3nSfvUWTvjHmJgB8HLuvfO+G39onwfWbDZKbtjDTbZNiY71e75lEs9mcmCX0/jFfaaB/qqsfxKM7RE8i4DLD6ekxt+/cYbPZElOkHxpCG2jbBvFj9KDGZm7y4zo04IMhMXYGiYQ4jHtQT4oSWhricANYkTByAU+jMLlilDGAEv9K50BL8UlSlOWMMi/xPtC1nQSq9S2zNJPsCgXLxRyjFbvdlm7oReqcOWZVxawsKPKM5XzO6ckJeeZYr1Z0zY5FlXPvZMlisaCaVQzNhvXFGeWs4vT2HT7/5mu8PD8nJYXKC9oQGAZPXbcELxkjQlxWOJdT5MLG9lEOASnFPWOkbdsRALF7xrYxhm7wKGUkd8IPxCBgcQgejSZzBnIBt9tmy8Gs4nBREbqa9eVL7t+/z8vNCsJAngk4TuixSmGzHI0wQpLvMMCyEhuEtm3pu46kFLk1JJ2EZcdIjtCjBU1IoyWSgEMpJVFleQlrds4xsV6mQh/k0DEMA83o2TjNiWubrLg/iFx7uQ8MPaQQxqaKxY6sc4lIibRNI3tMTDDWdSkFsYcK4cb8upbdhhDwQe+/lsiTsbE7qty67nr9UdaMeSeBqOTQZ61FJUPXBRR+VJyNQGEU5aM1hrIoRvBeQvLA4zIkEDz1oME4i0sGH+yNNc2PygX/6Sc38ORFQz0YHrx+xMnJgkhP27eEqLjYXjFblFy1A/Uazp86Ll/8lHe+eJdtvWKz3lBvEyksCT0UricMO16+3HHrcIZVUULVlWY5ryA53vncbZ5+uOFL33iHxUEgxB2ryxckE/nWr3+DaqZ4/ycf8tnPfIZ3f/CYxx+dc+f0PmWZ8/77P6VpPP/47/+Utz9zj7VZ0Xc7yhngM2xaUF9smGc5hTlgvXtJNVNEZbm46ohA7Wvmi5zlUcVyMaNrIssDQ2YSbdOM1ksNh8cLVuszsrLa1xxd56mqAqUjzmmOjo7Ybrc4lzObVZy9uMS5nPlcPG+1jqw3DZO1mlKa9WqDMxXz+SGr9Qthpml4+vQJZZmT5dI8bnYtm03Nr3zrl/n93/seJ6cPKEpLInByUmKM1DdD31Ova+IQiH3i4+dPWMxvcfc0J3Tw7OyMLMsoqhIfenabhttHt7h39y4fv/gpXeeZzRZs1z3DoFgsZmzrntOTI7HYSwPOGXJnWZYzNhdrur6jj5Gj+Sl3Dh7y9S/8BnePP4czS4zO8HHYgyyMgCwpjGrZa5LSNI/NWKuIas8yDEIkiN5jdKT3HbttS1lILpT3PX0/MIzqpS4lZtWMYegoCgkxvLg4H+cJogy0GSEN8hlrg46Ge3fucXFVc/vWPWEXDoGizFitLwlhxmw2o64jfT/wk5/8lIcPH9B2PVopnj59SlVVxBQpZ3O26w27bUPXtqAUeVWO5z1RJ1qbY0zOxeYcpTUnx0dsdyvy8hg/RK6uel577Rabeiv7XdRkZk5WHn7quZ1iIAyB2XLOfHbC0IcxlNZxfHLEkDwqBJSVAOwYvZxjNQJEI2Sstg3jPDZ4n7BOU81m1M2OiChIvfeiYE6KFA0uyzg5Nux2V/jBk41qhm5o6UOkbQK5LRk6CSNvNz0qGhZlTtdabIK6bWlcJv7XC4PNJER9yshQytH3st4a58RTP8vJS0eIAxg535eLkr5TFEXBvBTbt3q1pqxy+thxdnnOcn7A6ekdNtuGp4/P0Npw/07B4ckpvm+xueLo8JCQOmazHGth267oI+SLCt1ssFlGdTDDlRaVNJv1ml3XU1SVBGO2HV3Xk1KGSpKD4pPF5ZJ3ZFSGJeewusWzyyue/7DG1Ufcu3WP4+Ut2q7lanvJxeqMFA2ZK/cEkGlfBbAmA6QuNqYfG7Bh9DKPeC/fm1ixN2vWzvcoJ2B67ML4eRcQOlbPdrz3vQ9460vHFEczjII2dATfokJP3dYcHN8CBV1bs75a4f3Anbu30UrTNGKLsVzIa+6ahsEPdL4h4cjLnH7XkJIE1q/WG+YLzWK+ZFkdcHS84OzJGWkEZmMfOFwuWeQLDg5OcLnhcn0x5iZpjPdkZcKHyJBasuTIcofNLU2oKaoMGHAGXJnTbDdobZkVJS/PrtiutmgVyPMCp3MyWzLojjDsSCEwpIEsm49K1oZqVqKNZL0QNbtNQ+gii4MFUcn5q+9aXD7Dx8iu3n3quT3O8E98fZOBrm7cPinWJ+a7hjTZ8ckZeVLLTg4F1yS9kYWOuv56JCnIsSQSuM52kud/9cAxZUlpZUd7KXUNPo5M3lfJRxIoqpUoU4cgNdowYkyZcxwdH3P+8lIysYzBoGjrGqsNVTUjhYGPPvqIx48e0zUds3LG5z73Dq+/9gb/xX/zX3Hr1jEPHjzgO9/5Dk+ePAESz8/OyLOMO3fusDg+xrmc5S3Nya0dl1dXLA8Px9ewBiUK8cnbfWpETXXqTfuQmzla+/rSe/KiIMuz8Tb53E5OTghdz2sPX+OrX/4S77/7Lt/8C7/Kj378Y77xrV8RP3Iixhre+Mxb/Bv/5r/JD370I3787k84ODjk+bPnHOcHFIWQn3a73T4kVClpPE9rxfn5ueQ0jfeVc4k026Yz8fR7dV2LN56yrIhy6GAY5Dye5wfMZ0uWi0OePn3Ce+894vbpCaurhnrnubyo+ew79zg5XYBqSCqRFRkxdoJ/JSPjMU32yQn2wPf1uJYhOIWTp/14nYbbTaLl5GxASiOp6Saz/BNEt2nOqJ+vDLmJTekRc36lYzQ9jrrORNsT5/4Z15/DzmWccdPE3HcNDNZkKDRKTwC6SGnTCK6LO9PooRTFV1VZNQJqCZUCViW8El/0iPxc6TSzLOFMwijxDc2sxU4e5iEQoyWzmsxprIkoBlKSP6Qksqo0ec7K63dWNnMVPVZF8T0zEojIHtBUDMGIB3EcvX0U0g1PiDd7HP18lMgZFB6lBrp+i+49u7MeazWXzrI4PsG5gjgr0WVG8hGjZ4SUQDuMlqUsJCkqMZo4jGGaaixiUcSoMTYny2Z43+ODBzWC6Di0dtJUyHJhpxQz5otTrKvGrqMembKJwolMOY4D2mjN0A+sV2v+0e/+Q7b1in/1X/mr/Ivf+At86bNf5L/6f/zXPLl7C+3hxYN7vP72m4Sh4cmzZ/zJ938MMbGYV7x4+YzXX3+DB/fuc+/2KbM3voq681m+cv9XuIj/JS/ff5fderU/lPch0nkorMKZRJkbyllOPhMJto4eGz304wI2iBe8UiN7XZR2I/N82iREfqRHEDe90h1NGAW5k4DZSeJkrCZGg3UCgA59oO2FHa+SgEFh6PH9QBw7llpwIHk+ZKyLv1MkN5rcKJokjP+gJulLGkNPRNEQROawR8p0EpsdAemlCA5DwI8MBGPl9+6HQNf50Y7gF0DZgCKbvbKBTi9HwEPLJAWLKo5LQSREAUaM2qPfN9aFV6Vm0+PeBC211ihrroHzNKV8T3Izz75luX9waVwERg9mGMF/T0gDdbujrrd0fqAfRvaoyymrijwrxMbJujED4Nq7WICMKchxAkKvJW43gdOiKMnzmsV8Tjw9Zb26GllYslgbo/fNRPm5m4WRyFEnD+Fp0zs+PubevXujJ18cwas4rq85wQaUaQFLAtqmJfgWgthAJRWZVY7Ll2c8eu8RhweHOKdJyROSpq5b8RdtpcuvldgUOGMpkuGkWrB0GbrK2c7mJD+wq7ccp1v0XYfLMhgBUGMNRe6IIXJyfIhCXoMZ8y+d0mRZQV5mZJljs9nQti3n5+eUZclsNqPrWrquHTvhUwZFGMHXVz3qpvHivUdpzeuvv8a3vvUtXnvtNY6PjynL8pWApZtA+s0xPTV0/qwrjQzXaWze/LmbReUrFjA3Cq2bG/XN20Ux9GoBcM0OYP98NwuIm9cvAqADNM1OmmFaieomSeiiDz1Nu6XrW1wu1hvEiLMOP7Rc1Vu0sRIaWuU0dU296/cee1pr+k4Cr5VSFIUwM/wg8nWxhRKmeFP3iJe0Ro1gqw8i45ZAWMOuqRG1gcxBayxxBDsz65jPZiwPlmit6YYerRPb3fragsblLOYly8WCwmXMq5Iyz8isRS3mVJkh1jkmDSjlmZfiqai15fnzFzz94D0OD4+5tVwQIpj5nItdwy5Gyjxj5xuCH8iyEmskSMtqM3mEjd7ZZs9mmUJ+JoWFAMiQ59leLaJSInNO7MdCoB1aNIncGmazklmeEfuW06MlKvai6iBgCRwuylc8EouiICaFs5JtUtc1eV6wOJjLGPQ9OlnyPGM+X9C0PRdXKy5XK7q2k8Z1lrFer3FWCBNpZLf3MdH0HlB7j8abfsUT63a6bbJa0aMt2WTBk2WisujHLJ3oBxbFoQSQWUMIBmc1bTeqEEJktxO/SJtl5MVkH+RJXo0WMWnMCXH0nd8fcrQebaiiMNCrKpDlQjSI3GTSj3tUitJoUmLvYzDEqCV8dWgo8kCZA0rvZb9aj6HTWuEyg3MabdXI9pGms/cDYBl6sZgI4Ro0D8H/YkxVIK8s2EDd9mx2W4qZIS8z+jrSdANaV+R2wd/9O4958+ERJ7cWPP74KSH1+DaxnN/jow+2PP7wkqIsOD65hR/WpOgZ+iuMrnBGGj7WGY6ONH/lr72GM/DeT57z8OFrXKxe8pk3LE8vvs2sW3BwNOd73/s+L5+11DvFWez56lfv8/0/fc4//SfvsT1XfJxqvvilz/Lk2U+pCskyOHu6IrPw7MMX3L1/j04v2G06bOWojgLFQYYfEueXDZ0f+Ozbd0k68vL8JUeLGS8vdqADyjh29RqlFXUt1nJlkbPb7KhmB2If0HW0bcPR0WIMFL/i8Kji0aPnLOYFQ0g0/QaXBzJnaWpphi/mC1armpQieV4hOdaK9XpNCBXWWpaLA7abHfN5wbvv/hCb9bx48ZS8yEaLEosPYhFg0aTBU9oSjGdZacLQ8Mu//MusLmti6tls1nS+JTc5d+/epdk2VJXl8OCY/mXPbteiVEmRl7RNz/HRCX4IKCN5LFnuaHY1fSe1xfnLl5ycnnC0uM03vvwv8NrpFyjMIVpbfJLxqZUZmeZhBFCuaxmjLWZUA8cYJFA3jiSJKHZNQ9eiVMQaIaCIqkTqybZtiUGsm6bmVts25MiaudvtXllHs8yRgqhcyqLk/t27PLro+fjxUw6P73L2/Jy3336b9eaCzfYl83nJy/Nz+qEjczlN13F0cgRaMfSytkTg5cUlp6e3UErz2sMT6vUGnQyrzQablWRVhapHBUkuuSJXq4ZbpwuS9tjMkLmKqlqi1Y4vf+kr/P4fntG1gaosqXdX7CYT/k9x+SQe2UklhjDQdh3Rtxgte/AwdBJer3oSEZflwhgOw6hACwSf6LqeEDRllVOWYhvXxo7ee7q+w9pMGplGEbzn6uqSg4M5IUiD2uRiAar1tUWeUoYQErtdzW7b0vUeCGw2W3w3sJjN6HsJgUwo8qLAZgbNCOYP0HeiHFKIPRdayExFUdA0nhAD7a7DWsesWnK4POZwsWToW9ZVznwxw0fPpt3SDYEyLzk6POZ8fklRVRzfus3pnXt09ZptfcXx8RE2i2gd0Saw2rzElkuiyTBOU2QlUUXmywO6xoNuyfIZQ1CkfkAZAS6HzuNsTj8E2asS5NFgvEVZy8XHG977zseobcHpwW0OFgfsdjt2zZZ+6MmyAoUEoB8dHaG1pixlTRLmaUQpO+6Z8vdqVcM4h5y7tlKbasnpb62NKK1MAitnVz+I4iAFz8tHNad35rTHA8XcoSy0ww4VPUPwdH2NQnF5ecF6dYWxms1mLdayzlBVJdZZ5vM5y+UBL89ejlZBWtino7Xv8cExWq+IUTEr5pSuEqWRcSwPj+k257TtDpUiQ99yuJxjM8t6d8Xp6Qm20CQnuVaKACERB48zltmswqcADpZHSzJjyXLH1cqQ547cyXnMOcnMWcznuDwjDFPgeqQsMrTJODg44OLyDKUjWaYpypy2bQg+0DVyPikXBQGPMkIeyPuCqpzTtsPPTto/x3Vd76fxaCJnl1e+z/h9ro/Zcka+ac9ixrP2eG5HUI/9uSNdW7jI/83+0ZMKmBTEbYFXCbw3Dx9KadJEHFJ6fD1yRo5hVCsxuguMDGRjhXgkih3JOiyKnLa9oN5tx7OyJgap9WZlRV1veP78Jf/Kv/KXyWcVf/TtbyO5Yok/+v53+aUvf5W7d++iXMa3//g7PH/+DLSmaWp0lvPB48dcbbdUsxnHR8fEBHVdU+92ZEXJZrOhadvRxlgsbCdlsOxn8ZWz3SfPgNO+FKPkJ8YUqaoKY/Reifnlz3+R5y+e89d/+7dJQ8+zl8/5yte+RgKGvqcfvOQnljM++7nP8h/8B/9b/s//0X/E06fPWS4XNE1DCIH5fL4PF9Vaj1aM3T5fyBhDVVXUdb1vdExnYWPEsrYsSyY10mazQWsoq5Khl8ZqCImrqwuqqmIYAg8e3Keua65WV2Qu4+VZzWrV8PzpFfceHPLW23d48PpdUtqOoHXAaIdKZv/5sx8Lk+v5NZA+ZeGmcUCnkVQ6KbaVmho0o51LHK2OR///adzdvCZcT/9cD/YRyfkEyzzFtH+sqV31igXSK5Phz77+uUF0rQziXR1HG40RnEAAREUUafIUkjn6e8YRcE7TzzKC1Bq0seggxdXkfz6tFFZpDkvLcRmp7EBmhDk8gYUJTQiamEawziBJ5kkYnQJcqP2kEE/YOJr2y6EumZHxPtpm6HERMmMHYvCJ1jOJEgCE0RQFhDVKjx6ACmHvBdLYCPBDR9t4hg9/yjJELg8PKcs5ZVXifcSYXBoBSTwXY+iFla/FnibGHvYgHONiIzJjYzLxzlUW6FAqkKLCaIc1ObkryXPxV7PGEaJiVi5QSjH0HTFEMucwVj5TFQI+Btq+JyrFo8cfc7w8YHd5wZ/+4e/y1W/9ZbLZgn/n3/l3+U/+r/8XnHG4bM7i+IDjoyWL0vHs3R/z4eOP+cKXv8SP332X5nLNj7/7fW6dHnJ65wHl4hbL4zv81t/433D5p3/Mf/c3/yb+7EwOwRMDFGFg57kmL+Qgrs04nBArlsnCxhjQVjqiPggYHRlVCASs1vvulIL9+zh9jrlRVM7gMvFJU6Ptz7TZCEt0YBhGZqrRo1WAyM+dEXDVBrAxCU87JWKSDq9ViSpTlLmm2+dzTKGqQBSbmN5HhtEexowpJUOA3guwWFgBt6alSBuFdTJG/OAJ48Hl5y4ef45rYhzCNYC430zUZIIz5R6kEdzVKKsgXgfYTde1JEePi+U1IDnljqOuLWC4sXjpUQkggPEnFksVx+IAEnaUrApg3nU1dbOjbRv60eqpLCvmsyVZJlJcrTVOmzHwVo3AVmLya5ff75otHkLCDwIeTV38bEz43qwVu92Oq6srLi8vJXjwBiN/kn2JBDVH6TFEZWwkTMF2JycnfPWrX+X1N1+jHwbWmw0nx8cM/UCKkGUFQ5TAU7SmqRtC9BJYFxpIiWHYUVUzHty7x4+vPqSvPduuoetqXJaxWq9J41potcZlBhOgdDlzO6MyBQf5gvmsxBMpMstqdU7+oqTISjmcuWzc+Cvs6EUuvvJ27zs+BSEdHR9RlAX37t3j/fffJ8bI0dERZVlyfn7Oer1it9uy2+1GJus09mS2Z6WscdN7OBU0ZVHw1luf4e7du9y+fZuyLPeA+/UIur5+pgj6Z2yKN6V8N3/+Ghy/vt/Eqp3ud7P4mm6Xr28UFOMDTK+3KAoODiRobiqYtPrki/xZ65g/zyXApSJEJT7U0dN2Nat1YrW6IKnEZuPQWlEVBdaoUeUih1fnRlWJTuix4TEVSdLgTWNBJyDLpJ5KyUsOQdeAN5CkCRKCKC8SwkwJMaKdlWInSREazGgdYqzYcYyfQ5HnGGsIyY+KIUYPY0OZZRR5TlXkFC6jKLIRxEycnBzR7BwfPnufwsLhwQHzqiB0LWcXYi2UG8Xli6f0uzX3HjykWsxpx4NErVqurtbS+A2KohClR9N0xCRKNm0hRGGLTXN+8imdgG4j0jqapmNX12TWMq8KsazpapLvsVpTFRnO3iHPHVoFMqcoc4szFaFvWcyLvYVKmVlRf1gn7HbYyz2tMZRVRVM3ZAaUFhDc2YzlcsGtWydst1tePH/B+cU57W5D6QRol33Qg9IUNmem3X7sT/ZD0/ycWN2zmVgtXayu58nUDJvWi4nFHkLAGSV+5WNTZipmSYng/QjWQFnk5GVFXmZE5PnSZBmYIpPH4hS6Zu31/PMhsNvVAkrknbD8tNqfEbtWM2SGPAPtZI9JnZZsmx6GriP5lqEb6LOIGRk8xjiU0nRdT4wBpSMCjvZ0fS01ipHfScCWjq4fxvwIaax0XbeX3n7a6/TeHO9XJANZmYMREslm21C6E0xc8OjRFe0ucOfO68TY0PU7LlYds9xhljkfP3qMUgVXl56LizOsizy4FzlYDmS5JgyGhCEyEGNHPzT86XcGnn/c8/Du17h4lnF0K9F1gWa7Q4Wedpc4PLjDrcNjltUhu7XivXfX9I2hyuZcvtzxj3//XQ6Ocq42G77+rTdIpoOdRyXLRx89ZyAyOzYs55FiGenCJUOnpeEaNav1mjffvE+ZK3a7DdXM0jQ9TVszMwatoG168sKyumpRXAefW2vo+4HVastslhPp8M3AfK6JDPR+x8nJgrPzM7LcEgNkuSZzAvQuFnPOL55IDTIkSJGurbHzBfUYaLZeb/C+YbbQGO3xw4BKOdv1mrIsuDxbc3txQNhCVpZ88a238IMolF48ekqZzzlZLun1ADqw29bsdMQGKIpbPH3ykmpeiAd6Noa4bQYW8yOcyxlC4OJqoN7WwgiMgdPjU1YXV8Q68LmvfZ437n+Oub1FisKSTzpg0pTRIPk7zklTX4CDNB5sGc9NouLQ41z0fkAbxWxWkKKna2uuLi+QjKyBIh/9lcuClES9F2JEo0drN1k3nJWsiylnhwgxhjEDRdF3njwriD6yXMxRSUk+lk8MPkhQnVJEBaurC44Oj8jzjLre7VWuMQkBxRpHlpfU7Lh79wG7D37K+fkl8SJRzBxNF0j0BN+KrZQxrNeXLJZLfD/QDz23bx9yfnE2AgdaguxcJ7ZOn/KKKqEyw4AEsnZDT99F4rClKkuUTqASbb/Dh4B2GqsczjjKWUbTdGy39ahaVpzcOiLLJBPDjz7pMUasMVSFg6ixWtH1GzbbgWFoScmTZUtA6ghnMny7lZreCDt7tV2TjMb3Ay8uruiblsViybycsWl2dF2NcZq+C6iksDYbLdLi3mYLFUfbng5SpO9EQVHvxK6oyA3OWKw2rLc7cuc4OT3i7PKcru5p257tZkOelzx87SGnt+9yevsueTkjhh7dWaxRLKqSIdSE1DP4Dp3Eyq6sCpyDrm1wJsOaDGdL3DLDD2IRk2cZs5nhbLPCGQHE1ldXVLOM3CwYfODF5QUvPlyzftpwlN9ju9nQdQ1Z5rCZYlktMOaQvhcbzq7rRjZovrdvUEpjneHgcMbn3nmLpmm4vJzJXtt2eB/ZbLYSjDsyUPdkoJSwWFCJLnViD5vkcyYm0gbWz3v864YuDwSraOno2g3BJ3abFc5l1NsNMQzkWcnl+QWzquL49AhbFPiuZwiR0uagNHXTkQhs6x3DMAbw9hGrczbrNXGIbNdb+quWrvMcHh3R0kPy6AShl1BnbcAPA1oZtJXapet20EdUApMUuSswaJRJDNHjspyqWuIyw+JgSVnNiXjKmePBa3eIfSu1mHXs6gatoCwynDW4bLTkIOEyPYbJTg0JqZcTAR8HoopoY7A6I3SQLUpOT4pfaO8WQBo+yUJX++9dE8jSBI2nxE1bFq1Gi74b7FqdQMWbZw51/SdNcj5GTG+0MLnBKp8Qrwk7EdxOkdSYcXjjXJRSAh2ZMujYv+KRVKsVVVUyDIaDg4zdxuKyNYlI17U0TTNahg54EovFgi9/5Qv8e//+v8+P3/0Rj58+4YMPPwIUq9Wa4U//hPhHf4xWirzIOL11SlWWvHn/PkPouXv/PmdnZ7w4e8F6ux1VrZ6yqri6utrboAyDNIvtiH0IucnfaFT8LDHq5vf2KvIb9rVTHfvkycf0dc3//T/5v/Frv/EbZIXlvR//mLv37pPdviPj0Unj0hjDV776FX773/pt/pu/+f9kfXXF+U6IT23b4r2nqqr980wWLkoplsslwL7RXdc1V1dXGGP2NqrTulIUBUoL8zxGR1Fm+CC5IZvNWrKFQqBuIlmecffuKbut2DVF76i3miePd3zw0z/itTeO+OavvMPy6ABtPDqm/TieLFIYP/9Xz76JvYLhE+/pzxDW1Ahqm+vz+M3P5ZNX2qNkn7jUq/efSKJJ/+xrmJpG/6PYuey9XBMCxqQ4ekOPHaiU8FGY0RNwIKAihOhHqwzxbxD7CwFwBXQ04wHkurNljeKoMhzPEnlSZJONjJpCKKVzFtGgldjM2AmgiuPzi6REANrJToIRcFd7sGIKiAQ1+l9HQtSEIdH6qbOWiBG8B5LCaSicFjuQPWAikuQUAykokcKHlrP33yNYx+d+5S/S7Hb4bSKfz3BGzLsTHlKPitdgkdJjGyayXxwjHq0tZvS3ssaOQamBpBNGWzJXkmcLiryiyCvyosRYK5PcWqyRpHat9BgUIX6zUSm6dkeKnbB2naE7f84ffvQR88Uhr3/5mzz56Yf8C3/pN/nP/4v/kvW25s3P3KUsP8/qbM3RYsljHkHfcjSf8bt//x+yWCypDubcvf+U3/wr/zIHyyMyc8jJX7rFerXmD/7W/4vddkOKnswIY8Va2UgT0gwhKEyaeqTC6kQpXKYl5zYlfLiWE/UhEpMiJ5LtGy7XXa5pe8g1lBax8xEEnMnGQAo7KdKFcQb56J3O2GBxTuxEnE/kVpFZmXAhSWisy2BeQJUpVq0sHBIWyvg88v/pdSukeUOCuo9oDUWWyJwhy+Qz37PNJ3nMTZbrz1lU/lzXyAZ/Ze8eHzbGKDL68bY4dganwAjNjcVmfH/S/mGl5ShdvmmxitcbOJMPm/THQXILJMTq2g5mAhKvmcWahBbbo2FH229omt0+pLLIZ8xnS6pqNvriTqGNU1jqtYWLyLSFQaF1JEZGVrMZ/YHV6FcuIUjeexRiD3N0eMTDBw/pu57tZkdDvWedmtGWYOp4qzFUy/ueKYX96PiIz73zWRYHcz766BFHBwccLBcorQgpIqtmGpnrGSqJbdOsOmDoa7R1+DDgfYtVioOjglllwAecsvjgCG3AKmGmKJPIXYYKARMSLikKY1lmM5b5ksXBEnOy4PjBPbxv+eiDd5nPD8TGxVrKasat03uU1QKtrEggq5zMGYrCCcszRk6OjqlmJce3Tuha8eOczeZstlsGH0hKMZsvMNYyjExmowVwbJpmfJ8ieV6QOUfbtThrOVoecHx0LGFh47j7pK3KNN9lrHFdnyYFGPaqJBlt48YpTTE9su1TGK2dxjmp1WQDc11EfbLAmkaq3GkEmfcvIiJJbTcbVYayrMjzAq3Fb9+YcY1RMiZh8g+//v3+vFdVVvgw0Pc1IGGiu92G1XpF23YcHh6Q5xnWCNMhMXlzXu+n3kdRcI1WJXH0xHVOCXPdB7xv0CMAHKJ4hA9DTQwt2AJNGj1aA9ZoXJZJoGffY7ViPqtG26qBqIXxlReWPLdcrdasrs4pS4dzlrre4ftWciMMWAPaiC+19xnRKVarRsCBquLqg3M+fvwRdBcsS4e1oLjNYrFg6DI+fnzG+cuXkBLbS0VmEuVyzmHpML6XQ1S3ZRgiXb1jqGb4KDZbMSW002DGNXlkhLWdNAC899KIS+JfjhJ7L6PCaM0V0QQKoynLBZmzqBhQo+WSsorQ58zLjLI6Zn21Qo8AfdPuKLKMxaJEKS2qKTTHC/HFjjEQU6KxhmFwGAUhDtS7C4YIxhaUBg7KHL2UOWUcpBSEMdMHadhllmxWkZJi13WECFmRobQbfVvHcFzrsM6Olgxhz7rWSu/nkFZycNNaGokRYR+GKOud1nK4RUnQuHNuZDHmEjC2ZwKJ2iwGOcgMvdispahG9dYgKkciKYhVUdeLhUqeF6OdAAwp4ttAkct4MxaG4BmGgEqSA2F1IoVA1wggEbyEsGmga1sB93QktxWEgeSFFGGcw2olrNs0oJMns5Blhq7zhOB/RpXy572KqibEhCsybF7gfUu7W6EHGHaH/L1/+CEXLzcEn/MP/v57KOdph5rZTPONr82JQ0SbgSEkbDnthREfpPEWwiB2ddFSZg50YDafk3TORx98wNnZx6TYMNR2nNMVJI0uNS+fXxLaNd971tJuNaFXGJXxzV/7JY7u9AR2fPjhOatuyw9/8pyHr51w9uQZnQGzjBzfStx7fcmTp5eiTKHHFgKOD31PGBxXVxcoHShnjqJvyJxjNs9p2gatNLmT2mPoAlrDdttQFJblsmI2L/AeqrLg8rLl5NYxT7cvWS4r1qsNRwdzVGp5/mzH8iARPBTzGR9+8FPyQsDGI7skDQk1BPIixw8DMWoOlkesLtey5xaaRVFQrz1znXHZGoxyzLJDfuMrv8XB4jbnZyt+9Vd/k+cvnrI8qvi93/27vDx7xsODe5hcMdCxqi/pY8OL8xcM4QCL4sHte/z0w58AO/qh46233+bycovLOvKy4vbtOzx79pg09Nw5vkVhM+4s7nP/+AFffOPLZDpDrLLjfp8MUx7TeO6LSVRawuBOoNVYzwm048f9IIwByNtdLc1XldjVG4rCiZ9yt8N3Yge4C93YjC9QcQw5HC0jMjfZB2iKIhOCQkxkmcPFgsErmi7RdAPl3ECAi4sLloeHbOodLpvRhxVXu5ZbpxUmt7gyIyZP2+6wdkZIEZsrVrsV82JJN7QMKtBpTxs7sjzjanWFyg/Z1D0JCdd8++2H3L59xLNnj1lfdhSznIvVc7LcsKo1NtdQJ7Q1sgZkv0CTzFnQikBEZxYVInHo6AmkrhELUwshDSTE5qMsKo5ObpEXke32MdLsHpV1aDabLXXTCKHMWuZZRpYVEC1DJyQhpQeyvBBbLyJt2xCDeNJnKiN28rm6PGCzQFAJ4yZfYeh8JK3WGKOxWpSjCs96s8Eqh1YZxkjuivKKmHqC78FLM7bd1TS7HXo2Q0WN7yL1tmZnc/CBzWqNsZqyLEhXkSH0Ek7aBSDw4P49FofHuDzjartBebH0aLtLgh+QoNWBIXTQ1RxWBxKGrQK7piF5xcH8NkU+GzGNsWZUirJ0lFWOVnB8dECWa2aznOV8QbvyPPnoiifvvcCGGa1qcJmjmLsRl1CiKA6K4CP9MDAMwxj4J3XkW2+9xcXFOYeHS/61f/1fp+86nj17xnw+59mzZ/zjf/yPOXtxRYyJO3fusNvtuLy8xBgJNhSmvybEAZU7hpAIIY37TSDtNE/ev+LhO6fM5xkpA1Sk1b3soUNH8ANGC2nFaEPbdFBC0wqTPnhRCM6qOd7DbteS5aU0xhD1hzE5RkmIobM5F2eXxHYAZVgWJxwcHuAM6LbBWthsLmi7gbapUdGRzZ00uJzDNxKSaJUjdgmNJi8zur4lpyIvZnR9TV5WlLOSutsyWxRYB6E1+MHjfTfaUUkOUNd1+CgNS+csQ6glTFxJY0MlQ54ZhjQwxB5tDcZmZKbEqIzYI5lAv8Al9nDXAOT1yXlM57x5ab0/S1+D6BKsLGcgsZoT/DKAiq+qYJOsqWKHNx3yR2wlvYojCKHxZ4F0JuY7av+yJx6vUhqxQJ3OOAY9Pp61lhQ081lBSles1y9Zry/JshlLt6BrWtIQqMqCSODRo0ccHR9y9+4pv/4bv8q9+3d4ef6SJ0+ecnBwTF/3YyN7RlZoZgc5SQ/0bcsvfeNr/OE//ic8+vgpHZIZZ4wmkGh7Ua1uNptROQFpD4YHsQneM6InAtQ1yVBwVjOS9wIJAbBlf4pUZc7RcsEbr73GyeEhp6e3cEbxmbfe4PjObVIUJaq2FmXkvdRKs1jO+I1f/3UeffSIv/s7v8Pt27dpu3bEW+Tx5/M5KOiHXjIagNVqRVFIvlnbtkx+6tmYh7S3eBrVjovljLLM2W63gOBMTbNjPp+x29Vjs1wwCqUQeyMlzdjNuqZtNccnR3zw/jmXl9/mr/61f4HFco5Sa9CjqkhNQLVgw2l0VBgNj2WMqQkUn0aQENyuMZ90fWv6WauhG3i3jNf0/wdET4z5i9fPI/Z0Ez41/uyNx7/59z/r+nPYuQiYo5MEYaWYiB7CKLkV39wxbCVKV/mmhYbR3AiUFLZPDOObPvLsfYI+Sjij04qDUrEoDWoIGEZfpiQdq35kbhutMNZhrcK40YrhBqAYE5I8DhNkOXqjAyMMr7QA0QKSj68lgveJfhBGuwQQKsZsQqxROCM+mUoJQ0shHj4SrppGSbGWIJ08J25XzI5PKWeHdMlTDwPWzTDWodWcMLwgDDtilM6yNgYdr/10DbmEto7GoFpPgWWTrU6BywryTAD0qjygKA7IshnW5WIzYiwBKQr0SNW21nG1WfP8yTO2l5e8+70/ptSer755ix+/+5SPfvQ9Xnvrbcoq4/n3nvLNr3+dP/wn3+bRx0+4c/8ul6stTV0zKwq2mzVlUeAyx4cffYR1jtxl/OC7f8r9O3c4vH2b1bMVy898gW/8Sxt++A/+HqurCzKj9uGc02KV4iTBGEHnKIu8MlKwKaXHLpOwl4WtPoqKkjRb4wR6TcN4/FurhJ1sfPQYKDYmhiYg+kQYWeiZhSLXGGcYQhw3CpnEzohkqYngdoEaiVNwRlHlitwibE2VyLR49pOmxo7Ywey9zJCmU+8jziicG/MAtMyRia2fRkZ+jFHkdz7g4y8GosuidN31lnpRy+xVN8BvJHDqlQVNiSJAqxtMwutH3j9u2j/0uPu+ApLL/FE6TfDnnjks7/XkpT4BmeBjyxB2tP2KXS2yWT9AWSw4mB1SVTMyl+39wxSj//oN1v7EGL7JdpbnkoU6hAAqkecZxojPcRh6uq7FjN38ofekkFjOlzRNLeF14+uN48FP7JIUQ9/SdwKmHR0fcf/hPS6uznlx/oLj5Qm3b52y2W7Z1TuUUUQCPkmomVaKzBXMZgfE0BKCsK262DGEjtzAbO64e2fO+mVD7w1BZXShR2EIKYg81GaE0OEw2JionGVZlMzyOeXskOp2wcn9W9Rxw8XFFZv1GS7P0MawrVegFA8eFCKV360pR0DiYDkfvYHh1skxeZFzenJC89pr7JpamglSnWGzHLoOY3O8l3VziF5C2dD40I9rW2DoJSDJKMNitsBqs2fA3/wcpdB7NcV7arxJ+LKsk9N2ntRoNab2Lbp9bsE0LgAwiaSnkI2fBdGvVVkyn+NYlO4zDpQSQFwlUAGFsPUn9jYjaC35A3Zs6k5Fhbw35hdgopdFhQ8t63WH73cYJZYtCgijNHg5X2AstI3He2GvZ07jh4HgZa0yQJE5GfMxEEJP8CIz9IOEYk8WJqSEM5EyV8xKS1k6kh9tTaylKCuMsTSNqDeUTuIpahN9n9BW7puUxzgYfM12l3h5nsbA30iKHmsj3dCBNgQM7bBF7wK9zwmDpyoKVptLgvc8ff4E3V1hTw8xKvHRh++zWV/g+4EyV9w6mrHbbFivVjz78F26bsdrn3mHtNlybznnrV/5GmeXK5KSUKn3P3zE4fEtCXUyij4OgBob6QLcRh8pC0e+nGO1oWkaNrs1mVaU85LMZVilyLSjKg4pnMEpYcstZzNIkbrbUO/g6PCY3Egg2ATczjIn71m9Is9ynMro+h6vtVhWKSfjV4FRAT/UbC9e0DRbMI5N4zGqxJmKgzynmhVoO4AaqJuBzXqg7z30DYQrlHG4pHFZRULsVMIUQpKEPR7jQPSNhK6rRDcM4xoZUVH23iKz43Ty4z4W6X0gorHOobTFOocrsjFULcM6NwbJCVCuptoxKEIfGTpJgI9pzIkJHkvCpFFBkSLD0NIHsUUjOZw15M5QlpbFIufkeEFVOZp6xfnLM7xvcVlOns9IIaNpe3zfEeJA6hsyY1HRE9uWLg1YBpm7MaKxKCKEAULEKI9WAyl5YadGT9e11I371HMbpP70g2foHSEVhLig3Sm63Zzv/ukLzi+3Y+0aabst1TxwemJ4/fUjHty/z4/efYzNPEMfJRQqRuIAvhfLANmAJSumbZKE0drE0XHGb/7m2xwujlkcBJbHCh97Xl60nD3Z8s7n7rB6WaNMJAwtwwCZyYix5/f/0T9geVzxpa+d8q1fO+GdLx3yx99+Rr2JaD3j+LRks9uRVwP3Hsw5P9/QNAK0kgLHJ0dstxs2257ZIpLlmrOzFcvDgsH3+CFycnyLly/PyXNL8IHZLKPvB5y1bLcdB4dLNusNzsmBcxgi69WargujBVnHo4+ecOfOHdpmK83cxrNzLUolHn30McXMkpLG5YH5MicqR9NGHjw85eMnz0hh4Oh4QZ7lzMyCt976CgezE37p7TnGWYyCL3zmyxAdbz6YQyi4c+sNBr/jnbd/idcffpHPfe5zVPOCF+dP+Sd/9I/4vT/4XQxLTop7FLdL2quaXOVSvxrN+YtnPH9+ycM3bpOSpR88y4OK3aolhJbNZs3d2w+4e/wWVX4IyRC81LVTWPPE3gohvKKy6vseY0UlSoIYEtpq+k6ylZqmlpyH6On7hNHQ1jtIAsSSPMeHC9brDRFNv1lzfHIqQERT07UdkCjLavR2ben7dgTbHcoqmkFhjaLIM9qhoWm3eJ9ouobF4Zz5bM7q6mpfW2cu47JZc/b0JQ/u3cMky73b9/nBj35MVhTkZcHZ+QU+JsmMuLxEWYuyhs4P2L4nJcV6vaEqStZXK4ZBgpPrumO1uaCqDF2XRuZ24vj4mFm1oMyTgMOf8nJ6zGlAQpITHmVFSTt0PcpUkrthxMc+dIGQBWLvWbcd9bbDGE2RS9ZE5ix+MLRNQyRQZgVlXuJcTtcFBh0IKpJljqoqaNodRmcoY2janjzL0MkwyyqaTc/2akNeBrpuIGkoqlIOXVrT+YHUS9PeOINvG2ILJi+wLmNWzSmqjH7QtH0gxcgwBr+GtiX2Az7zZHlJ03VoY/FBrH6yPBPAUw6GZJmVzKPcobzh1skdiqokWbhYXaGikfpM5/RhQ1FW+NDQ7jZgBvquhRhohx273QoKaO2OIlvgQ0nwEa0E28iygsUC6k3H/LBifpJzMF+wLA54GTZ8/8VjfK0o8gIQpV69WwsegORsmMJxtKw4Pjnmcr3F2JyHD19juZhx9+4pX/riv8bB4RGvPXzI+fk5X/nyl3j69Cl57ijyjKurLR9+9BFvvvkmf+u//VskgqgwRPZP5kRh3YdIPwggprTCKovvoL5qefLBBe/8/1j7ryfZsvvOF/sst13a8sf0Me3hQYMhCRIcknODEq/uxNWENAopdEPP+pP0IL3O8zzoToRujCIQM3eGBMkhCIJAA4325rjy6bZbTg9rZ1Z1o5uDAbQ7Tp86VZlZmXuvvdZvfX9fc3wPlYOLFqEMHkvEUrcdJtmNJyVWiIgoqZctXd/hbUCKntV1Q1Hk+CDonMMUOaXSlPkYicRaz8HhIUfHR7imJ2hL17Qsr6/IRjlVVtD1FqU0l1dLVpuGxlmqMoPBz1lGzTw7IsaItY6gHFGmxoCWBaNiD+sEz08vyEtJ73o2dc3xcUWzbAj0RClo2h6pFDEklUvT1NSrNYoxs+mUttdolRNDxLaJ+NA2HdlYIlUidWlpEBiChb5O5+o3OYy4tfbv9qUDGPgZS1CxIw/AgBkgkUhUVMioky/1lhwZUj6FiLcAcgJKCsDf2rdsbTIG/IPdWyDloG3B9mEfQ8rI2nljDA/2g0JJbHEIkXBSiUz4HIa8zIhlxje+8SqTyZwf/t3b/OIXT2nqnq7pyDNFYxu0FnzlzTf5f/0//x9881tfo95ccvdkyuF+zvFhwXg8Y9M49vbmWNeiDVRVzmQyYzLeYzo+4m//yw9B5kOuUlqvTZbjnCUGgSORtoxOVimdtcNn0niXcEFttor4AdNM6YlIMRCdpCQIh1SS8ahkVBSUmaZfL7HthsePvsabb7zOo5cfM5vN0QhkrlOumQuIkAg1MSbL2ZdeusP/9H/9P/PRB+/y13/9t5ycnNA0zW5PtVovU5ZTVe6Y5ycnxywWy11Wwm1v9K3aectgD8HjriyTyYTRKFnGSKnJh/wprc2Ona9UaiAUhURrh9apYXR9fYXzHbPZhPPnDf/L//vH/PGf/g73Hmik7HEuoqQZ8LtEkE57g4FEFhNBRwytli3y8xkXgwEoirufDd+/NeS2QP1uBA6Y4S9h6MO89/lD7P53s5//vA/+/99B9PTBB2Y3Hh8YgOLIlvntnKMfwmWkiAM4IREikGlxE/Y5nATvHS7IXffA+kifrK8SU9hEikxg3QDSSTVsUFOoqSDJ4rWKQwCp2DFn43CjD6LDgVGYmN9SDo8lEvEDMz69KT/IFF2A1rN7P5H0d+9EAmmDQKmA0QGjsxQQ6iNCJgaHHxhhCoFQEiMU68WSu298Y/Bp93ivQASkLDF6DLFG+5roLSF4VPD4mCY3gUhstq2kZpASJ5YraJVCM7QyZEaTZQVFOScvphidpclNbu0r0gQgBtBTypTaK4xBmoy/+Bd/Am7JsWnx65p33/+Id99+n6OH9zm9OOWjj59z5+5d3nvvPV5+cJfN4oyz5894+vQ5H3/yjGo6Ji9LUIpN2/Kzn7/Ng0cv8Tc/+AEvvfEGR9Mxo8MZ+Te+xac/+XuWiwukTFdj2zn1wRO8gwheWKSwICI612QxFfvOp8AR5wdrBZm6YEqkpg0IAmHwSx8k3+Lmhksg7Xb8xqH2k8ONnl5bkF5LDIoDb0PaMAyy8SwT5LnGESivJUsbBoAM8kwM7yMBYWIIrN3euGHY8AuRNt0pKGR7dwtuvKID0XqC87t7JQG/CXjonKf7zchsn5k4tl3EG6/n3YNudbB3zxzWWfHLExgMlgufnew+Y7HB7YCHIVvgc2D+FkxMm7vkne2Co3c9bb9mvVlS1xtilFTlhMl4jyovKQZm722rj+0HSrV3As/twAb5InB0uxjdTu2uRiWZ0SwXC/I846WX7nP//j0++vBD1vUKN7ALd3I7KZMHtHOoNCAoq5I8z/nZW2+hjObevfvMZjNevHhBxHPnzhH37t+haTYIkZjwSkBmMvK8wHtwvqO3HUpo+j4FRyqhOD46QNklS9sSTMBFS64MJuokXxQSoSO5yZNaZdjUlKMqBeiMM6JI840UApWbXeCmEil0p+s6ilzTdS0m0xwe7nP37h1OT8+oqhHT2QRlFJPJiPnejLzMsbZjvV7vGG3eh50VRN91tHWzC2FN4bpDET2EhWxB881mvWO+bq/tjX2Q+PylZtejjlt5mdjd87fH8ZZVcBs4gM92wT9/3LA8vgRYvz3Wxc37kfJmbAihdkG8O8XX9j7Y3Qu/fpMsWay4nQIibTgz6uAhpoZPYix4rLPYPrFTcpPhbAp/M0rhGUKQiThv6doNAo82JjWW2pZgO0ajUdpwaIVRghgsrk9rlTEqNVF0CkETisRy1oP9W0hBhlmucC55tjftBqUFs/mUyaTaNSCMMYzGBWdX52it2d/bZ1SWBO/xzmO0YjIek+c57777LsvlgrFyPH/xgsVqgdaKUVVSVSWjfM5kXLG6XvDxRx9xfXXN6bMnOB8YTWYoKZnP5zy4ewedl6yahrPzCxSJKe2J9D4Vh0ZpgoCm7fA+eZuCSjW6DIOVXQowrsocozUGSZVnzMZjxkVOmRnKLIMYmKltMGKyi/DWplBh5zAmMbUQEdttgJ7ooLUdJgZMnuNtj4o9Xb/hxdOPWC7PESIQhGK1tsRgGFf7SQZtA8FZhHREZxEk8DndFwnAkaagtT2Xlws2rQMMMYCWio0xQMDFdmgSabyvuR3WpJSmKAqUSg3Zvu/x3qcmkhZDLoHEZPnunhQisX4a2+NcUgRpmcI949C0iXHbqE1rio8RMYwpRBxs2wSuS/WFc2C0TJZuQ+i8loOHurMpl6dK4JXWAm9F8m0PmkgKdg5CoLRG5AahkqReCpXs9KocIVN4mo8BpTV5qXGho24snpAMyz4nVf1vPR6+MuX+/a/iXc3F5YJcjfn+v/+Upx8+Q2iFNIqoUrbQaAzTqWBvT3B4kDGf77Nc/RQXISTDPIwqkcLw/rsL5vM9kD1S9RRZnsazNIRgafprLq563n/nnNffnPPpp9dUo4rN2tM2HadPV2gxIi8DJ8eCJ92K3Aik1LTtBikEP/zBR1ycnvDmN2f83nf3ef/dFZ981HFyNMXkkelcIFWPFIEiq4ihx4WWrvPs7R3xwQef4uwFJ3en3L1ziMoUl1dXxAhKGu6c3GWxuOL5WcejR3v0/YKu75lMMpbLK2azOc4GNptm52k6GglWqw5jcpTSfPrpaWry9vD4tQcsFiuyzHBx2RCiIx7NcNR0PrB/cEBYWpSqqOueTDuwljtHd9krH/Hd3/nv2Rvfw9tIOcqxrk0EEpkRQwpx977H+ZI3XzvEeU2WVyyvL7mz9xUen2ywX8v56tfeIM8E1i356dv/QBYLrtdXPHt6ycHJNIGX3uG6Gh8jZaVZxQappswmc958/C1eufNNMj2FmGF7PzC0EtEjDs2UtH7c1KPGGJROikHnPSJGvPUQks1RkRlkYTg/P8VoyXq5om8bhEhezgRompo8z7herFjXDQjBZDKj7zus6+i6jrpZM5/vMZmMub5eYF2HtRJPz7K/SoSsEBhVBZu2RRmD0oLV6pq62eC9xSiD8IKr80tW1yvGxYhPP/yUcTlms+iozJjr64br6557D+/Stg06Lzi9vGIyGWH7HhsS2aswGeu+TrYzylA3HbNZxZ27U54/W3FydMjV1TmjouLi8pKjvSOUkFycX++C536twzZYJ2mcIDNlCvmOHmRPjJZIiZCJ5Rl6Qd+3qLDGNZbVpkmEoyJQZCljIzeS6dExy9VVAr6txUeV1AgChCappUj7e0ghoEprlM7Ym+1z9fwFAsGoqoCO2XhGXo64uLiitx1Be7JcozIDSGzvKMsCbx2KglxUjIoR41GFix3gkTIMoe85hcwRDgwa13r27x4S1muikAlzcD2ZknRdi7eW0FkEQxMlL7l//xF3ju4idWBRnxP8Btsn9XZeGJoukJU51bjgarlK67IIdH3LepXsHqJTGDVCy4qqmuNDoK4DmZEcHR6yN3J88N6HeGFRmUz2kSFjcdFyfVFjdEVuSo6O58z3JoTQ42zPdDylbx1KZvzxP/9nLDdLrlcd88O7PHj4iOB6xqXi4Ut3uHvvMX1vmU3nfP/73+fdd9/ln//xH/PwpUe40NO0LT/4wQ84PNqD88jefJ/Dw2PqpubpsyfMpjOkklxeXXJ+dk7briEKlEqBgC8+WfDoq0fkGq5Xl0ilUTonqkjdbwZltMC6HpEgQLJYEbzE9g0qSwr3pumw3iUiZYysVzVr1SKR1Js1s2nJpp3hjKAcTemi5dmTJ1RVyd7+HlZKMlWyWlouVxuK6QiTlYOSP6KDIrMjNt2KXvboUQJmnYWD/WPm1RFX5xfUdYvOctabFU1vWa43LNYrqnGBC6CKAoWhrVsEkra1uK6jU5LxZMyo2gNBapK1ASHytN+WniwXiUwQM/JslEJvh7Dy3+T4zH4l3tpGD0SxRA0cfrZ90PZZu63H9ostG+jL9iphIBPdfl7cWdt95n3dUn1/9v0O5KDPPSMEkDF+5vW3NsfBWXqfVP4hWu6cvMRkcsTVZc2L5wuUSspu36d14tVXX6cocu7du8fBwQHTacWz5094991nCODF6TOkzHG+xmSSybRMDb5cU5QnfPLkI9abFO5c1w3TeUlvO+omqcSV1Lu919ZOKvgBmI03pL4btrO8dbKH3WQUOOHRJqlhEok04xtf/xonhwfMZhOOjg6YziZkWVIbKKXQWQYC3GAt3bUWrQ1eODyBkzvH/Ot//a955533WCwWjEajHQ4RQmC9XlOWJZPJBKUULz14wP37kXfeeYe6run7G5a61npn+bitvbe2i9sg461dzFYNubVj3f67rhtGo0RWDcEz35tzenpKCJH5fM7Z2YJ/+NHPObzzmMrkEHuUMjuygg9haKxsbYLkjoz5hRh1/NzI2gHon9/X3yJxDl9LIb/kRX/5SE3Fm9f6vL3MF/3OLzt+dSY6iSEa4tbuIEmSE0U+JjaWYAeCJvZvYmQlv/OBxh+GT4AYFkSf0nFvbYCEACPBqICSESdSca8NKCOQVhFlCjKVKhKjRUozeFklWfDtk+MHAHUr0dh6uCeLli3YpXbM0Qh0DhofScTKmwDLzkV8BOcDSgayDLQOCOGSZ6wUw2Q1/H5SV76YzVgvlpx/9DFUI6yMdNEhlWAyLpMUXGiUKvHaIX1ASg/CQpSJ4RU9ShZIpYdCJ/m9KKUxuiQzo6HwLzBmhMkrtBn8P2MAn0AZMVgWMIR+bQPE3v7JTzGm4/GdGbkek9VXPLo74smy5Yf/5W/5v3z9m+T5mKurK0bViLZu+Lu//gGq91wsr5FFxvWLc66HCawsqhSwVC/54V/9DX/wJyM+ePdd/ubsnKOpZOIcJ6++zvLsOSo0CLYgcdx5j4cYEoAuExNKKpn8wL3D2uRb37ltF9RjlETLNOmlyY7dazJcDyGSx6MxBim3oZnJakEMq1MYGkQMdgFykEBZ6+h7nyxkYiTLJdOxIQhPlTlEs1VSSDItkSiSZUQcFqDUXAkBnB98rmGLm6cGj0zBvNuJyzmH75LcCHFjMeG8o7eWzkU6/+vbPWyP235Ut4H029+PpOvwRdKX25YaN6/55b8rgaPqM9/bpoZvJT3J+9QP6p/UIXa+TwC67ambNX3fI6Uhz0ZU5ZwiHyfZ75aB/k9Mhrcn0NtAz/ZncJPanb4XcL3FO4vWir39OVpJ3n//fT795BOEgCzL6Pt+B8xvF7J0PklhmSEOgDnMy31ijLz11luDb3jG8+fP2d+f7RY8pRTOdWmaUColzeclfd8CITFLhcDkmnJWElpBqCOx8/igsMMGRCGRaIxWVOWIcTUmVxlKSkaTEeP5hHXm6FxP7zzL5YqyrJBaoY3eMaZjhL5vkSoynY65e+8O9+7d4fLygpdeuocxiqzIkEownY65c/cOWueYvEwezW3DcrHEOY/RWWK6lHK4x3q8724SxoeFXWuNs5a+tyyXS+bz+Q58uz0WP7sMDxKyG2MctvPmbl7YPVTsxsC2aPmnutKfAczjTUjJ7TEl5e3xnMb07XsnAedqB55/vhO+nY9+ExB9U6+xriZEj9YSIROTUGuIBGzf07QNMQ6M4SjwTtAGi3dDk1lFbG9p6hSWPdwNWJuAjxjBux5rW7ROIVlFUaSQyDaF+mhtBnslQ9c6NnVD1w/sdS25vDxLzZmyYMqUtt1Q1yusbcmMIs80WgtCSMVeZhTOGTJthuZ48vH11hFDYDxY/oQQyLOcw6NDYnPN+uo5bdeQZznLxTUSmE2mzCZjLl6c8t6772J7y97RCdfLDQ8fv8xoNCbTSRKqVZq1MiXouwZiwIuIJCnPvHM74JeY7J/aLo3TQEAZgbWOtmmRCFRVoXNNnmVMRhXjqkDGrZ+2QGeavu/p+z4xjrue4FOTedO22C5ZKQkibbMiRkHf92A3jEYjms0abWC9uODZx7+gtw3T6Qihc+r1kqqYQ+w5P33KfG8G0uJDDwKU0OmcSg9K4lC0naX2ia3W2VSbeRuG0HeQOjWGYhjkxmrLdE3NmWSTlYDpED3Cx8F2xqN1lixebm8oRbJn6fqePgxNcVJgvAsBG0KS5BKJIo3vKBWB5J0fESATCKSlxA4qwc5ZtFP4kAgcXetYrerkhe57jMkGL+HkRxxCCiwPvqd3Nl0frRARMpOUFcpITG6Y7e1RTSqs6zl9/gm17YiDF7DOBdL0KO3QavDs+w2Oxw9fxZiB2VM3SBX52rfucXH+MW3bD7Y0gcm8YFQKRqXHGEdZRs7Oz+mdAwkmE4igWV16hEvNsicfB77520d4rgj0mFwg8EgVkNFz76UZi6tzDg7HHCjDYtXAdWA+HrG6rLm67BBCURYjlIHOdYm9LJNqtSwqPnjnkk8/Pee7f3rI175dIOUx77y1phxLqpFFCE9T9ywvI/uHGVmmOX1xwXrlODzYw/o1fZfUM588eY5UEucso2qElIJRNSUzluvrlqLMkMqluasPnJ1eJ8ZVXnJycp/F4hqlJF3nGI0KFterxE6MCmczrq42ZJliPp+iteDqesFmY5nPFW3XEBBok3F9taLISqYThe9WjLTmzUevc3fvATruIUyyDQq6Q0pSkKDSCBTK5CiVEQkoJDEaxqMDZAzcO3qF4/ld9ufTlBngOs5GLdP8HpfrC779lQqVKT45+JhV95S+b7C2pVlvUlMuSE4OH/DKw68xz+4QnSJGQWY0IXqIfiAk3m403zC1jDEDScgTvaeuazKTQkat7QfCg0gNyeWCGDzeW/JMY/uGerNiPErWFX2fPntZFpyevWA8HmGMJM9HlGVJ27ZcX1/ugtVCCHSuQRA5nB9wtb7gdHmGdx1RRcaTkovLC46Pj3hx2hK9Zzqfs16vmI4nRJfIT/t7B4n4hOHkaMwnz0/5xTtPuHf/gHbT0raWvEh2ZCEGvPUooch1zuHBMevVkr5vcF7SLFeUeYGzA6mls8wmM6x1bDYdUhacnl3/2vd2EBYfI3VticaTZxlGadqmQQZB3/UENxDY+nSvVyGwXq9oupb5/jSt69YTQk7btsznc8ajKjW3u57GB7IyqYfREhdS87HvOlL+iUcIw8HBPgf7x/Rdy7PzM2RmmOcF947vcHx4zPnkgqdnz9nEmvnejKLMKauCi4tzitxQ1w1IiY8dLiqkLunWNet6QVlpVFmy7nrqLqIlRG+Q0qBCzqzK2DQrVnZFIZMq2CiNEpLok4KwblpGmWc8rRhPSxbLS5p6w6gqOW8WNE1D027zrhxllWNMlerP6IeMnoYQBEIYtCqRyuCkw1QZWUzNbaMzinGqY00sUNbgG8nVasN7b39E3zqOxmO+/Y1v8n/4P/6PHB/tIUVASfjwgw9oNjWL5ZpXXnnE9//X/5Vv/9bv8fDlN2i6nqrKOdyfcn15iRSSF89fUNc1jx895g9+/w94+vQpmck42NsfCECe1159g+PjOxwdHfPRhx/x87ffRqqU3fT973+ftmkT4aAasV5vCAjyfExb13S1x4wVbWNB9BSmQKvUVPMiUuYFeWEwqgDh0EojdbLhzIuMLM9pu5b6uqFZtkxncy5eXNB3NqkW8QRX4H3H5aZjXI1QMdI6h29rZmofIRMRsGt7BJK92R6z6RTrGwya3GQE6+hjR5QJV/IelFDoaPA2YTjj8QgpI22X7M0uLy9x1hNsTE2iLMN2gSgVMsswRZXW9SjwAaqiGjIHLFpnjEZTsr5DFR1SqmRh6ALZWCO1oes7fsP+92f2xzdQ4y9bRSZrlZt/bxnqAyqX8B1x82Z2xMDPYWFfTAa6tb8Y6q4bGtsNOSkJ0raP/ex+KO19folet8t5i4MFrYyR58+fMJ3e5bVXH/PhB8/58MNnjCqDmmRMJ2Puv3SXl166y+uvv0ZR5Ny/f5dXXnmFo6MjPvroI548fYqQisl8zGxWIVVkvVlydX3Bj3/8U/7ub39C30nyYox1ktV6eeMKoXQKxVZpP+acHz7zwDofCJA3e8OtjeeAEcQti9/v7EtmsymH8zmFVkQfODk54du/9U2Ojw/Ji5zReITROdqYtJcVAi2TSmBTt6xWG7q2ozA5hwcHfPe7f8C//Jf/A//23/7boeGcJ1vTIdtqvV5RFAV5nrFcLvijP/weAD/60Y/YBpsqpciy1Lx1t7KFrLODlQvkeb7DbLb73i2Inuxs02dv2zR/bPN75vM5dV1zdnbGbDbjww8+4af/KPit33o12bn6ZAeI6EkWptvmxJYwOdia/kpb2u1Y27Ip+cyITIzyGwLnF+3Tvwj9EXIgdTOQqePgIhJ2iX1f+LwvOn5lEF2EdGIiciePD0OX/maAbTsCw+Y/MqSyD8GHWzscGYcOkMT5uNtDbGXriaE6BJapiJBD+KhKMm8pIkaqxLKRCiECSkWkGkCwECFsT7HcwrKJQe8jtu3pMklbN3TWYbLEwfF+APajoPWRNoDdWuaK9FlckLiYmBpq6y0e3QCIeoRIHVslFaPSkBU52fwAVY7JTMmnn34KWYkcF+TjkqOjfcosx/m0+RUiR2iLdD3CG6R0ON/hg0fLHKkytr5Byf7DI0WGFJo8qxiPjsizCcYkz+Ku7xB5kqCk7lIcbGBUYqanE09ZVTy6/5AnH7/D/tEh08IQmpzN4hmLv36PlbQsrs74vX/2HZ5dLPnhX/0lCvCbFUbAcrlitUkJwU/OTlMgnWop8xyU5HK95sP3fs7+5iQpDrIDzrqee/fuc/+Vh1x+9C74QIzJcUsOLGQht9P4li2eutPeWZxXOLdlkAskyTLDhZgCedhC6emWSE3aiFSKwiRWpFIyLVAyJYtvu2TO+bQAx+QhZrQGqYi+x/qtNzvJC7o0jGLyPzciedeXJoE81gl6zwCyDSoKH/EefNgCazfvVClJJiHX28APhqLPYwd/WGJqPvW9TZ91CFv9TY4vnHx2oJ78zM+3wN5nrFjSbLYDDG8YwEMoqWAHEO6KgzRBsPVrvAmhSL8nybJ7fHAwNNEgsWp729PbQbGhUsc1NyOMLtA6Rw9hiF/22bbf3yZZO+eG9xB/qRlwG9hUSoH2SJEsWlarJc+fP+P999/jenG1e74YGNsJcI67hXd7fvq+p65rZvM5UgiePXsGXjCfz+n6JP998eKUw6P9IQRLE3C4was9MXbTfKBiJFM5QkR67whRsHc4x24sMiTgvvep8SOUosgKZID9yZzpaEwlk0VH27fY6CgnFatY43wkL4oUGho8QkqqcQK5lJLDxtZzdHRAWeZMphVllfPg4X28d2S5QcoEso/GU7QpcCFyvViidWLUxxDxwqGkYuPWgx962BU8kPzcboe0aq1TwnrTMJlMPnOdE6C9ZXBHtl7kMTp8sEifCqNtcRluh+KKm0bS5xspXzaGbnf5JXFXjNwOJr0pem/un+3z0tgPX8782DaT4q/PeNlsliA8SglMpnE+qRa2TJHedkNQjUcpk/IXiDRtTXSOqipTo2Jo+BVFNgRHRTabdpAPSqQCb5PXqNIVSkus61itFvR9T6YzrOvITEaIkfWmTn7a0yk+pEDdTd1QVim0s+t6nG0T+9R5rK0RJP/9NA9YvO/JdGKZhd7RxAYhGBhSDTKmArGsSo5PTnjnHz/CZBlZZjg9PyNYy3Q04vDggMvLC370j//AwXyfP/mTP+XeS49ougQWLy4vkVJRjMe46ysW6xrbNaw2m9RwFTGFYEtB07Z0fZ+aoeYmsFlpPSgONM4Klv2SUVlQ3r3L3skh+9MZe5MxmVbYvodBktu2A2jutuosj5aQ5QorVArvazr6riXYHikkfdexcRtCm9N3NUoLrs5esLp6jpARl0U6t0S4CKHD9xuKTKN1pO07lqtrVusa78BHQ5QaWWREqekC6GrM0Z079DbQdZZ2XWOUZDquMEVO3XacnV3umoghBLTVKcxcpGtiLcToKUwKtZUyBUUZk9HbBH7rbTi6D0Q6XFCJIes9LobE6BkyOnxI+RpCymQxphQIRRQSFzxd37Fda723xBjopKBuFH0bqNeO1XUCi6VyFIVmPC6JIW3M+y75pK/rhqAV49EIYTTBJzWGGjZJuswoxmPK8QTZtQhTYmNL13UYL/Au7sgaW5XVb3IsLj5mMp2l5lU+wnl49bVjFheRH/z1OwgRmc8NZekpSyhHEqkDHs8nn3wEEvJcQ8w4fdqxvvaMTEVRTXjvnSXXq5rf/+5dRnONEI62W6Ua3CT7GyFfcHn1jAcvH6BMTpaN+OQXFxwd7lFWG548PWfTeVQB0Uqkz8BpglijqjUyBISuePG8RVWXvPqVr/PWT35Ct9xwHE0CeKLi619/zAcffshkKsgLxXK5wPuSgOPBw4qnz54RYmS1aNjfn3B5eYUxmtlswr37E5zvcL6nbT15kewpqrJAyZyL8xWbdeTBgxOsr1GqZrFYkRc5m7XlYG86hOJC223QJq3v49GEZ09XyZs1U9h+g5YV6+USXCAXU0ajnAfHL3P38C5Yj9JhZycphYQgUCI1prfrj5Y5kYDWcvDATGkex8fHBNsNtoCK6HP+5I8eErC42HB+dcrz02f81le+y09+8Z947+Of4Yue6/VzovUcHtzj3sFrFOoAEQ1SJhJDiC5VyEONnTbQYjtAUz0uGJpKAWs7tNZJyWEU0btU30To2x4RB+KTEmgtiNFxdXmOUpK62RB8QGpDWRU4Z5nNpxityYwiEunahmfPnuG8YzqZMJvNEhloUA9W2YhxMeFidYXSmt71FDLn6PiQ4B35ENisJcwnExarGiElRV4ync558uQZRVnxyZNnHB4d8fziirPTFdNJhkDS1C1hsJcz49TIXy0aXA8PHj7m9OwZUUi0yXFd4PLiCqXE4BFtqTcNWues6o7luvu17+1Ij4+BzCiUBI0gFzldX+Nsh4w9rW0gQu8sRmfkRYF0PZYOrVODJgW2JpuWPM/Is4xGKVQhIUoCAS0jmVYUSg3NG4sQjr5vyDKVbMlyjRlVzE+OCJ0jF4a9yYSRNjz+6td5crzPu5cfUo5GSKXYO9inqAqapsFFaNoV0XfUbY9cenrX0TRrRpM5bddT245MGkCglKTMc/r1hmoypo4pB0GPxxR5TlFkQx5Lqj+Cj9i+5emLT3Cupak3XC0uODiac3l1QV1f44OjyDOCz/E9jIoZOtMpJ6RuaGrLZDRjb36CIEPrgsa35JlmpmeI3tOuWlznEE5gbMGsOMBkOcIqvv3V3+VPfueIr7z6Vd547SuMSgMxMB7lhNDzx9/7fbq2oW0a3vnwU/74j/+Ee/cfg8yoqhEQ8D5yeHRCQNL2lt557t5/icPDQ8bTGavVkiLPqZuaIi84eeMOTdOy2ax5+ZXHTKZjxpMpb731Fq+//jrPnj3DmOQ/3zQtbeMQecDMOzIxQkeYFDO8tBRZgdYZ3gdMnsg2SgomoxFdm0ht2iimswlFWaa5wSqKsmKzaekbS24yZAxMxgVSeLSG4Dpc27HqHHvTCXdOjnG2J9eGXGbENlKWBbODPfb29pFKkCnYuI4YPOSKKqvoQqpViYZxWXL14oowhzw3TCZjmm41eKdLlMoYV1PWiwWqzFLTW0jKKkNnmlmMlEWOIJIVJVJnmCiYzfdp1zVVOWIym+DlEhf6pGLrHavlkqooBwXDrwq1/Vfu8xgH/CGhVSH43depXroB+2JIdfsgFPgl8lDCDNwOSP+8svozh2Bn+XIbON/99xkSUVoL4pdYx94o17fkuzgAfmKwtkyezmWZ1so7d4/48z//Uz7++AUfvPcRSkX29vZ48803mEwnTKaTZMXmA1U15rd/65/xO7/9zzg9O+VqecHZ+XPOL19weXWJcx1FUTKZlvzZv/geP/r7n/HhB88wpqBvbNqfSIHWasdGT43i7f4ujY042Dunixt3+/UbH+4bMqZ3ltzkeGd547XX+MPf/2ecHB0xnYyYTsdkeT7sZW6aHoSUvYZUaKmYzwx9e0pm0jlbLBb4EPlX/+p/z9tv/5y33357qI1hy5ovipy+bymKfV577TWur6/58z//cy4vL/n44493++Su63b7zASipzHUdd1nCFtKqV2Teksc27LWi6Kg7/vdtW3bpDDN8xznHKvVCq0jf/e3P+dg/5AHDw+xdpX2giGpjBKBWSPCsE/jBvTejf8vRdRvAPS0Rb8Z66mps0vW2z36y8bl519V3oLjI8CQ03l7H/5lCvTbx3+DnUvYvdXkF7wFC1wqiCRDaBhDSCeD5cvNa4SYZB9iwDi8jziXWOIqJu/ydIKTDYbRKWxSyQSGJLbwDdCZwg0iUml0ZjCZJrh+B2qkroJIzPEQsAOzvOstfSsTE65zFLlM9ggu4n0CYRsX6WICaJVME5ALgegSyC62IExM52AbqCoALTRGafLcMJ6OOX7zDQ5efxUnMjyCbF5RjseUkxJEoLNrhPBoVRHxqVmhOoToEpM5SJQsd6xp7y0+pEKjdw6jcqLUKF1SFnP2pvfYTni27+jbBqU0eT4mRDsAziXIbLiecHF1wUsv3+POyR7jomeUS6JoOXnpEUf77/Gztz7m9MWn/P7vfo/zyxU/+fsf8vz0lHZTkivBpm5QUrG8XqBEsmFo2xYRobMdbjzih//wE6p33ufV11/j8vKCflPz477mO/enlGWZJLQyFU87BvEulGAYPz6Fuwohh9DaiIhgRETLNB5sSJODFltm+fZmGG6WGFFEMhkHhoAi7kD3Gy/sbmASayVQWgydKnB2kC/piNaQ5YLCSyojKCQoIZmXUOTQOxJ4uWvvJhsXPyg2QozDuE/XQUkockFhBDaIgfkWcWGwl5EB7yK2c/Sdx7uIECqB/L/B8XnLlV/+/m2wfLj7vpTlfXuiu+mgb1/7ptOZrsg2OT41wG5ASB96uq7GOouSSXZvjEqLSUhsMiEEShoyU5DlBUYnQCZ4jyN1Vr+ITXwbFL8BNAfvuXBTfNwGVbcyJ2v7wULAD+qXJFfVWkF3YwOyZcJvAz6c87vf5WOgrCqElGw2G5q2pdAlMQbqzQZjDM+ePyMSODiYM51OeHG6wRhNiHIAjBRaFwgEZTWh6xuEVlwvlhxNDjm5e0QuNIXMqDcdbW/RWUFVjshUxt50j+lojBHgNjWd67DRoUUKYHF+C/KmkEGlVCq2hCT4MCzqHeNJAs+71vLw4Uvs7Q0hJwSKUmN0QVlNCEEwGo3Zm+9xsH9AU9fY3kIgsVpMznjUs6nXtO1qZ6EjhdpdByEEFxfn7B0fcXFxgTGGLMt2123bTU+N7m3xGFKIlG3xPnnfhZi827bjOA6d8tuKitvjYndf3Pr6M4nhMfHcb4PqN0D9TaPrpkgOA0jhP9MA+/wYTUXOb6YyiQSkjCgtUAp6m7yYvd/anKXH5UWGHkAVKWXyWbUCrUnhnaVGALlROGchuuRdKXJsb/G+GxrIia3tbMty4TDGEEKkk5r1ZpmsWDKTQjmBq6uOPMtTuKSzrFcNXbdCaQMioDTUTc2L06e0XT0UcKmpkDz8ktVbW292rInxeExdr4nO07TJV1ALkmenSfesDZ7xeERRlvzdD3/IL372Fi/du8dLDx/wDz/+B376s7f51m/9Did37iJMRu889XrNclNzuViipcKQGld6CKeWQlJqiY4KH0NiiEuZBHgyWX8okdErgcgL5uOK+XRMlRtk6LHdmmglrreDtU6HwCHiMIZ8sviSWlHkGZlOIGPX9biuIdoGKUF7h9ssqZsEYK27hvPnzwj9mrIq0dJztV5QVHOMihB7BIEP3vs50shhLEhaazGmwFRj9HiMKQtaH3EiqQFH4xKlGlzfIAFtBLlR9FYNdV5i9hij8EM4t9aKPM8wZgjVCg6tE3Mmz3Os9VjnB8ue9DrW+bTJiXGoG1Mzwdoe53uIAuctIUqUHNYQRGoKGT3UFAlod12Hsz0CgRPQiFQPqOippUdIT1FIpFbUbbIn6Op2UDhAcJYoBb21CJXkqdE5UIqgUjPnarli3XS0fUvnAjYmH062odxaorL0nlz49T2TAT79eaTtLpjMKk7uziinkaCvOToJ3L1T4GPPqIzkucSYiC40UjhWm5Z1s0l+03LC5ZlldZ3Yf11jsZ2nGEG9Cvz7/88Tvvq1uzhnefh4Sl4FTJ7RNXDnpECqwHpzjjI5SM2mdTi75OikgnxC0zq805xfNOhY8Pjeq5B/iCk7nDVcXm3I5xvqVnF1+T6jGWzWEusSc2xvXnJx9oTj44xN2zDfq4ihwTnHSw/voY1gNp+w3nQURWradH2PyQTL1ZIs00gVyDLJeh1pGs+9u8f0vYVoWK02XJxvODxQ5GXObDZByEieFTwPl2ijefDgDr1b0duQLA6kp08W3oiQsby2SK6pSs+kLDh/2uAyQTm/S6nuMh2doJVCSpfGzZCHQRgqUwGQLKGElCA1UYhBkaLwrkdIPZTtEaVyok+beBElKgr2RydMH87xoee1e99gnE356dt/TxEmjEYVv//NP+XRyW9jmKJ1Cvl1fsgSYEsAIKn/doQmdk1pOyjwEuBtyI2ia2u6rkGKlJEFjouLM1bLJScnBxANm/WSUVWy2azwCjabmoePHtN0HW1bU1YjmrZms0nZGnmeU5QZXZfO9XK1QBsNIiJjYrV3taMwFdq1CJUC0TKTI3NDXa85Ojjgw/c+YDyekBc5bddzvbrm2elzNm2DUAaVZVxeL7CtZX9/zqjIWVxe0YiAUoLSGKajMUoW9J1nvVzx6OWHFOWYLDcU84x+03F5ecFkWu1qN2cdRTbi+fNTynH5a9/by9UKgiITFc71dA5MYTCiJLoeQlLthsjAEtxnNB4RKYmLnr7vcM7Sd54YkmLs8vKSzBgUGpUbpDYp4LJtmU6nGJPY7tYFFssa5zpizFmvF8xme0itePnVV3FND61lmo0wMWpyeWkAAQAASURBVHLnYI9ipDhzp8neA+ibBoWCIIgebGwTViDg/PIF1lkQgSIvqNtrMJa8zDCDDUCIDW1bU41So0YKSVkUlGXOuKoSocg6HAItJC70XFw9p2muqYoS27eUWcF0MqJpVqkGC1BkI7QsmY5H2NjR9GukSGpWrSoEGU3TU40DOpOIGJmYZPcWrEBHxaOjh+xN73K4d5foBFrkfOfNKUpkFLqgKJJt2VYtXlUFUsJkMqYscx54STGa07WePC8JUeCCSzZwESprycsSoRRn5+ccnZwglMIP9aRA8uqrr9H3PfP5HCkFbdvy+PEjnjx9jtaax48f88orr9D3PS9evKCuG9arlqdnn3C+/JjmypEVhlE+QZaR0ahKNVSmKYockykyUzIZjQhhnTKPZPIUDwTW6xqpNOPZFKNzRBTMZxM2mwV7szF5rtmsl0ipKbI5MkomVQkhsFxcYXyqc50MqJFhPJ8QRaCuW/b2JjS1oOtairFBK8N6WVO3PVoI1naN71aMxxOKKilk2rbBR4dQ+S43hZgyzwRp7yyNTEQH5yjKKtUkRu/YkmVeYjctzWbDbH+GNjm2TViMiIJms8F1LdV4MlzfX//YYmphB3iHz/0Z8DJ5i4UeIjEmVm/K8vMomdaV9JhETrjZ/2z/3Ozft6D4P9UC+DI7l+07v30M4v/hd29/R3pcIgPKAZxWSVmaOaaTGUU+5f79h3z3u79P37UcHOxjrUUpRd9bmqZlMpkxHs/w3iGlYn//kN7VqJMjtPJ0zQqZaxDw8KV72D7yvT/6Dl3zn3j+4gyIOBvo2qQ4iwikESnU3SXSrSAS/E2mUcK70znbhWuHhEMG0tw0Kgsm04o/+L3v8N//xZ8zKtJ9MpuMyKucsiyTrbOK6Bjw3qJVDnIL2KfXPTg44qMPPyR6x2w2QWvDhx9+xHe+8x3Ozs5YLAYS0sAsL4qC8XiMEII8M7z00ku88cYb/MVf/AX/5t/8G5bL5c6mdsso32IXWyZ90yTr1K3f+ufB9C1usV6v0VpT1/VQm1uyLNvlUiWmes3TT9b84C/f4ujwT9P7lGl/CB5JatSmJr1EfgEJXXwJMV1sMRhu6KbbkXgzeoef3/ILF9s+yGee87nXDrdxudRgkELu7hu4eY1/6viVkbcQLYJsx1AIW6ZpvOngbD2X5PAJtiDnjaQkAaDRp0/pwxZYD8lv3JMsYgCtJUUmyA30ShB8RAyBCCGmweDDIOkxaVE1JqMX7sZKZbgwbrCe8XHwIh3AhC2YD8kCxns1+Genx/qQrskWRLMBOhvoA5Qy7Jj36T0NMoBI2qg5h7eSq8trPvn+94n/+a+5c+8e2b3HjF9/lf2794hiRl7kKBMQWPJsy441SJmhZYXFIWXyo0+skcTE3QYL+OBTk0FIpMnRWZX8AZVhtbpOk5HrWG+uGFUpoCeGgCxITK3tEPOWD97+MY8fPEQOgRBZOYX9O7z5+iN+9MkZq6sLemf5wz/6Lj/6+x/yH77/faK13D3cIy9yFosl2ybT1gJj09REBJdXV4nhIiTnl5c8ePCQxfUVnbVcvXjBf3dSAgu2wGgInuDSZ4/SE2WSMSipybQZmJmJXRtiYlEIBL2PdB6UDBRKJuOGuJV/QxAJibe2JQQNIkPInG1uoIhi2EQMG3W2rFgxdB8jzg5WLzKgFGSZwDtJaSSZTBvyWSXITSSkHRJaMvj2g5RxGDuphxaiGG7WiJKeqtQUGmLPYOkzjM+QAtKcc3RtkiN7P0hrw29u53L7+LytxC9PRV/0vdvfT4zfrZxn99NbgHSahG+CPb33O38u5x3O1UNolENrQxlLQkgSpDT/SIhqsDPKMTob/Dr9kLSdgOxk23PDVr795zZz/PZn/zyAvvUEtrbHOot3KXimrmvarqUo8uT7GGNKrx/kY1twd+uzvXv9IVtiy7K21lJlI5bLJcvVNcvlkqLMeOutn/Ktb32T6XzOpt3QD51gIRSBQBQK6wImL7AiEBXoKqdxHcd7+7imIyw6inxETU9WjtjbOyBTOdNqymQyocgVTbHE4XG4FNhoNCbkKNIc2dU3nWopRcosCB4hBUWRIaWiac54/PhhYmSNSqIIKCVRWpLlhhgkoxEcHh7Sdj2ZMbyoXtDWNd4loP787IKu63ce7FKk9HJimlMQguvrBW3b7tj822u8Pb/bYificd5iXU/fN9RNPRR6Yuiml4M/XpK0x8FDYitj+yUQPSHzNwvs7b8HcPy22sI5xzafY8ssicN6l56X3quUny12t42f242b34TvkmUp3KXt0jWTQ9CKD9twU5XGb5GCXL21iUGYZfRdYq17F9E6va/eWup6g3eW0bigLApaDdYmUDLLFT70eA+9hcxlA5Bu6bsepTUVI7TJGKJICLGnKEtMJlksrun6NVmWpwBZFRDRYXuH63O0ghhSOGpe5QiRAlm7roXoEyNlPEqbkBAxeQpHVFJweHKEjJbl1RU+Jta163uePX1KUZbM53v85V/+JSdHJ/xv/uJ/x6PHr1KOxkid2NEBwaSumU5noBTOJ3Vb73qavk6AlEhNmuVqybregJKYLENnKczVeY+SGVWWkxuF7Wo2iwvGB/uIIFhvNtgusWf6riFTIm0ABwsFpSURTVARby3NZsN6vSYzCm97BA7vexQBHy3eW85OT7m+vqSzltl8hLU2sV5ihpAFbdPy4cefUjc1KhMcHR8z3ztibz5DipKYVwSt6K3DBuhjAo1TXkRioYYYsX0Ksuttl3z4fT/8sSQ1RKQa5RgjyQYgva03yUpg2HA522NtB0KlPJpIUl0NtmXWBby1xGjpjSCEIt13MuK6HqSCAD5kKJVCnI1WKEEiPPQN3rpUxyrwLq3DYgAsjVFpviIm67auTzWRSMqVTCuiSkCiIF3rzvZ4IhkG4XrW9Tqxm6TADkrCpCDyKCkRkmTlEpKi7Dc5vv8/f4LUiulsTF6do8qWV746495LOV//xoj1WhI8tF2HNokF5SIs1hapBb4NPP1kydWpR8uS6FJN6YPFecn1ZU+eZ/zj350RouLjDzZ857uPmO4F2rpnvW6YzJI/bjXJ0Cbnza+XvPeLj4g6MN7zyDapSe5Nc4L1BPMxm7pmXmVUewI1CgSxQRUV64sa6w1llTMqx1ycObTKcbonRMt4ktQIs3lF2zrW6ytUJrlzd041HnPnzh2ePn3GxcUVXbei7wMPH97henlOlikePT7k6uqaLDPUdcf52SWTcUE236OpPXfuHfL+B6dUI43JBEcncxQZH3z4EfO5JisiTbNhNM45Otzj04+f0m08+9McgWdcKgpdcVge8e2v/gG/99t/wiSboqUmuIgVjuQeqIkygX+JcJDyOlINHQeWlYEgk62IT/c9QhKjByEROll0BSeQQRKDIjMFwUkeHL3M+bNLvvbqb/PeR//IbD7n7v4rmDgjVxOCX+Jwqa7WQ8aSTGBSa7udSit+fm0LiQXZdQ1xyC4SgLeOtm0Q0jGbTtibTzk7fUa9WTEZV3RtjRDJE30yGfPkyRPKUYXJkiXEZr0iz9UAoLQIEZhOJ7v6KK0ToKVBBsXFi0vK44qrJ0vM2FBN5nz8yTPeeO0hs+mUJ588I2UIR4SEPvR4PO988AGT0QQXIpvWMpnMsI2kymaURrA/2+Pi4pS9vT2sbVlcLTm7eM6jR/e5XqxYLFasVhvKUPD8yTmP793l5ccPuV5ccXG+pChTXVE3LSZTw9z26x3NxhNdwOmOXCuUSsrWIAxFUaEzSYdFSMlkMmW+fzAQBGCxBGsTgDUezzDaEPEJUPcCMDiXWLzRhaHGCnjb0BOQylGVerDcg3qzZL1cEl3ajx3O9/BNR4HiZH6AMhK76ejsBiGT+rmvazbrhtVizXq1TvtOI4gyEvrIarVGSUGwkbIo6FxHngvoU60tRUAZDdJRb9ZkJif6QPABrRTr5ZI8yxAuIjOVSEaipQ8B7dN83zU9uSko8xFa6WR5k4+pijEgEEFhvefw4A6bZU3bOOpNgxQZ9XrN3mSKW3c07Zp47TDk5DpjWkwxtWDTLshMhdCautng/Bpre66uLgYAcMyoHJF1mtyotAaWOdP5IWU55covUj1jMnKdMyv2WSyXKGXI85L1uma13nB6es5oVCGVGQg0GUoZikIyGlWsVkv29mb0NqkMj4+PGY/HPHv2jK7ryLKMuq4ZjzMeqBPunIwha9He0bWesjDgAlfX52gtBus/MXgr58S4ou1qQBKdwPqINhnjyQSTFYyrMDTQG8Yzzd5sQlkWPH8aWS1q9qYzyqxkPhlz9uw5mVBJfU5MAbPRo0tJ3W3ofY+LGS706ExRjQpcBKUNRTnC25j2CdZjXYdzitVqyXJ5zXg6HbIu0l6h62qKwTLPhlRbtF2LFFBMSoSELE+h08mtwNL3HX3b4WPHZL9ASY2zPUYX2NbSNl0iNf4qKNs/cWyJQts5djvf7kB0BmW2j5/ZExCTrY2IN/uL3R423jB7P28LA5/dA0NygLjJXhSD9cqw9xXJ5mILbqXXuMEabr/OZ0HQuH219H5EIAQBwWKdxzuN1g7rMqaTQ8bjCcQRIYS0t/AeEDvLznS/Z2w2G2xvkQJs15JlhoODPdp2nZwDBhu+e/eO+Vf/6i+4vl5zfrni4uKK9WrNxcUV5+dXQAKZYxBDXZbO/w2x6QYX2O4Hw069myyhM6Mp84yX7t3llZcfU2YZwXmIN3iGHLKuQoh45xAqG1RoA/gYBUZn3L9/n9Pnz3jv3Xd5+OgxH370Adb2vPLKy/z0pz8hhEjT1CilmEzGfOMbX+fVV19FCMWdOyfEGPnWt77Fz372M/7Df/gPu+u+bUh83spnq9TdXsPt31vF91aZv93zb4mAW1uXLb6XvpfjXeCdt5/y3jtP+PZvPaB3GySJpCoGrvfWb14hCUPIaPq9ELfv4dY4+jxgfoMs3XzNrfeeRtu26c9uzIsdSH6L3jm4Htz+ptien93f/7U2Uzr+G0B0ktWK0ogYkVGw9WdJHuMiMcuG3x8AGwI+yhTwNJyc3oNL90eyaiF1hhPYLXFxSKtVgirXFCZQCxBKIIZJaxteEGJEqZAYNoVCakXcnbg4bHq2waCDTFJLyspQFiZJ5JRIXusq2b6Ewb/ax8SM17dOYggCC7TBI7fJyXEbZJoukw8hgdZNZLnucEGw8YKoeySBqRBUD07QRtD3G6zdYG2DKeTgnZZuamMKooe2a8i0SKGBQ+gQDAF4IezcfqMYQhqjIMRA39YoqVhdX2BJAWF1s0D3JiWx9w1CJ5m0UprDoxPqOw8ZVSXTSYFyDcJHTBY4Oqy4OLvkg/d/wYM3v0UxmvNHf/xd/svf/DX1uub52Tlllg1esAkg3PrS994hhcbFVGjPphNCDDx7/oQiL9g0De9vAg/zCW9Mp4R+jQ2Bvgu0dYNUjtIE/O6OCygjic4TBnZ2GBYgH6HzKeAtH7YkIYCLER89Phm+JLsYEZDSoeQg39l69guxY7y5AehMbLq0kDjv6ewQPqi2IbWgtEAr0EKQ6ci4BKkS+C0hebqLtL8nYYLEYYy5ONwxInmIlTloGZHeo/QNqB9CCntsWksk0PUBFxLb/TdC2bhZdD8/adxMTp89tt+Jv/SML1pK06N2i/YAUAox+FUPOQWItNmRERQC5z1ab5tKZrfYCUBKjY/bid/svHTVrus6dOiGz7AF0m8W/c9adiTA1A+A0PD9IAjDwqKkwiKQQ5Cl957lYsHl5SVJhrVEa4OUyT8v+f7qtHj64QLFG2mV8w5tTALhBh+zaTXh+fNnIAJvvfVTvv71r/HRRx+zXC7453/6PabTCZd9B1LhvSREmTzWGCyMhnNcjsfExuGEZ7I3wb9omFQTGueQJmdv7xARFZnKmRQVZZExznPWfpPGtJLkpqQoSrp6hRBQt21ioMebJleSdg8ybqkYjUfsH+xjbbIA8cGRZRnGJBk8MnW75WChs1yu2KxrurZhtVwlVpF3CJKVSxTpXCkJRpnduWu7lvU6sY6bpiHLM5x36bXF0GwUkRDdEELYD4/dDK1lmeymTDb48m9VIT5JR9naEn3ODiju/rcrXHcLbUyrx3Y8pfea5moh09q4XZsTiC4QIi3k2xyR4clDkyUQlCIoiRfyV1rMv+zIcjk0rAdLG5VA54hIrEgRE6M/zwi2pvctMaaQL3B412GDxwzAYYievtskIIfE8FQqYjKJkBLnOsDtzp1SkWxQLyDcAIZ0OOfJi8TYiIhUDwhQKtC7ZijcHCH0KXvEZGSZROBRUpBnOoXi6sT88y5SZIo8ExCTekUZRVYomralbdYUo4oqV1hnOb+8oO8DpckwRc7+3pxPnz7BKMW3v/1tZrM9Lq+XjD0cHZ1gTE7dtiipONg/AClpu45NXSOkYDSqMHlqGJgsx3rH6dkpbd8lSy5n2Wxq+jqB/VEKbNPR02OmOcI1dHaNJGXCONeTq4jGJQmWHzIWpCHPDEWuiUYSwojrxZJ6s0HYFZn0aBGI0iOi4/LiOZ9++glN2zPdPyBEePbkGacX1ywbWL7zMc+eXdBbx2hckuWa3nZEBNOJ4OBgTj6dcNlsuDi/4GKxwmtNMR4TSey3aHtyYwiuwzQ5NqRwYK0FTdvjXIu1Fh97ytIwGhUURUVZ5PRtYtVEwhAq2LCpGyICpTNiTJZOXecJoiBl7VhC6PCFRMl0z2oZWHZ1WjdUjnc5UapUn0YPPuD7FtduIEbklo2JH9aR1EhPtWgKsIveYgRUmYEQCc6hRLI2MgjUULhb52lsh7V6ABjbBJhrvVNpQgoi00on4HHbQP4NmejTY5OYR7RkogRf8P7bF8z3c8zIsjfSg4IuKUu6NiA8lHmFrXvWlw2Lsx7hDMpI8kolNmKR7Taz3gqMypBIVtc9/8v//FOm04yyDNx9KTHR09gvIWaMpwphFD6ksKfMCIQMZMLw9NMNNQ31xrJad3zlaw+5d3KXrm84PXtOVQheec1glARaeud4cdawPz/EEyiqgra2CK24Wmx446tz6n7JstWsrxuW1xsyU1DoOa88epnLqxc421EWM/JCsF6tOTk+JM9z9vczNmuPs4IYVty7v8d77/6c45M9+n5DWSgyE1kuFpRl2kg7p7m+bnnp3ozLswXHB3tUYwGxJHYZ03iPe/PX+Jf/w//EfHJC7MNgawlRplpVDIzvXb0pblh8A4aBtx6j0ybGhxTemfIoJH1vh8A/gd4GbIvAxnVI4TFGooPhO7/zB/Su5Y1Xvkmep3Mi8fT2Gmk8W1WvGnaTIYa0Rm6VkoOyKwxZU0pKXDIXx1sLeDKTgGJLxMp0L/Qh3fPTWYl1Szb1FXW9HJpUgwWmbQk+o90kO47NesX+/D5d32Ftnz7XEDLed47gPHEIll+vrzk42GfpGpSqkCg+fv8F48mI8+sN4/EIKy/BKBpr6dom2XTWjhgETd2RVyVXF9cplPpgwj/8+Me88cZDIHlGP3t6yWxeoqXi/r1jlI5IZfnk049ABLIs5/Jiw8n+inXdsd6sEpvYg1CB3q0xlcaUvz6I3m0E1jmsccTCIwtBbjIiinGeEUNaR7M8sRNHoxHreoMLyfe7bXsEisyMmU73iNFxfnHG6jrVqyjBZH/EaDpncXXN1cU1hASaV+Nk3xeRAxgSaeoNrvY0V2uyg4iOAoqCalrx7Ow5Hz7/kOX6msl4ggwGg6KSht5Lmi7g84DOM3Jd4D1IrfHWsV636EwhoyK4SN/0uD4pPafVhE3dsVgsKbKc/cmYaVUgpeTq6oqqLMmjYrluUSqiMoELHU0XMbLEu0h0EiNLiALb1dg+YKYFm82GICXBS8ajfTI15tJfEaOkrAqkiCgXCW2g9CW/9cY3yEOWlEkhMZvLYgIiw9pIUzcIk/LhPn3yhChf8NK9h4xHKVTU2paqzJjOkp2nzlIzQEQo8xyRZaybhqwoWW3WVOMRy82a0o1YrteMphMCkSIvuW6uqSrNFtTMi4yqqli9OB2aVQ2LxSIxVvOc6XTKyfExtlvTtTXL5RUXV89plz3FdEQWFG1zTddvGI0KslwjETRNR6Z7jM7xYUVvW0Aidc5sPsdkGVJlyDzifcvV5TnGCPqQo7zGRZBZDmVGFyNeS2KuCZnGKihHJbpMcxpZqqmN1jgsxShPbMaQSC5aZuSFpI09tm9RuaR3PXUTUUoM9aBmMp4CGW3T0YcNjWswRYk2Eh0FhUnWOUVpWDcbjDBkhaaxKdS462rGozEuJmVMVmaUuWJ9bYkhoCW0mw22/c3W7psMrmFdEIMNyucek45btq0iDqq29Pgta3344bCW/DKADl9EJBM7EPNmx37DwN4illurzC2ZiNuvs33m54BI8Zn/pf2+0SbZ1mZp7OZZRmZMsmntU/ClMSYRaLShqiryPDH+ldJIqdFbK2disk0UCmMSYU3JtIe4d++IBw/u0/XQtB29dbz/3kf87d/8HU+ePEdrhVIZXddDDEnlPZCdvPeUZT6sf8lSRms9kOE0JtOMKsOdowP2ZtOELAlBURZpTVU3djhiZykdBwU7AxCkBkB9sMY7OCQGz9nZGV/9ylf4j//xP1JVJffu3ePZs2eMx6PB4aHh+fNn/NEf/SHf/NZv09bJWk1KyZ//+Z/zzjvvcHV1xcXFxWAfmZrQZVkMzXi/A9i3yt3tsQXGdxal8sb6tGmaXT6bEOImjFRqqmqPvlvxV3/1Q2Yzyf0HFRIz2NeIwQ093hoX4jOY05e5GtyM2y1wfgtajzcAutg2JbjdaErndgvA3wbnb+6GmGDsW2M/7el/dUjtVwbR5VbqLlRixyqxuy28dwkM9IkxEYYbOLG6t8yKgb042HEgIB9k4tviMUDyG0dgNOQGsiyxipMPk9+BNwxMXqkE2iSZsJIigaFsafoSxOA1LQVKRowGM3RZIYGWxoAQKRDSx4gnsclDDLsLnTw3I46ACxHUllU7YHURhErAvnWB3iamfAgyhVHaQPTQ9B/RusDi4oLp3hSlJeV8yvRojncOqVKxrZRBKUeZj6nbJUpmqQBwDq0z3JYlQgpejCGFlW2aGiEN3vUslxdIAV2/RgpFUY7Se+4TiKOzEX1vadsFzvUcvPSITHjySiG6QFytKU3BneN93njtFT54es69t3/KVx+9gvA9Dx4+5K2f/JT1useZDikHGerAzE8D9mbSjUSul9cJNBGw3mwYj2ZIKXgqMv7v/7d/zbt/9X18e0nUGb31KNeTyYh3DCzxFAiAGLzCXRxS5NO5D3HwRR2ue+L3DDEROywsokRihIvBchLSYiVkkphvJ9ME2CZgESEGe6LtDZ+sLZSSO1BZych+BZNRQMqAG0JPlRQ7u4dtxyx52sfB53qYW8XWAjNCDEiRrJO884TA0JH0pPC4SO8C1ituh4r82scwc4h4E6oYRUyAz+cW4TjIOG53A3/ptUjdvXjrebf/TkVB2kxKbRL7TQ1dT+cRuU/SoJ3VhmaXcyDU8Fp9aoQpMEqgZWI8tyIxaAlpfAjvEcOGUCmdbJIG3850vR0xWkLswCULEQbmcBzuayllWqyKiiIr6VtLnq256K+RUiNlRtc5MlNAlCmToGsRqGHeSmne2wneO7dj3gfnWC0vU3iIlLz99s8HgKnnr3/wt8xnE77xza9QFjlNU+9AJyEFJs9oNi14QbDpnKlCY6XHjDTz2ZwDtUcTAr2A0XicPN19JJeGuSxwRUYXAyLXhJg8WUEynmiEVtRd8lQjgiQQXEPfN4SBta60Yr43xwWP0Bq0ARswJqeqRkiVo2RGlidLpMlknBhaQ1DuZDJGCUlflmij2awWWN8zGU0HKxyHVJLOtVjr8H1i8S+X10ThqMbj1GwRktzotK+Q4IKj9x4fI86l+zDVhYrUIh3sGPADe3popMUhBCYGoohDQ3UIMA0W3zf0TYtAg0nzngueZJ0icMFifZfkyqQQPUL6neB2bPl0DMol59BD4IkeLFwSsJfxK+nKvuQYj0uW6w7ve5zrCNGSZRohcmIMuIF5k+cS2zfpcT4SfMfWss1bS187lE5KtG4IhetahRTpPtFKIIWk6zqIaihSDUoyMCcDxiQVmHc9nU9eq6n4jWyaVbIRkGkTH7F0bU/XWSApyGKw9K4ny5In6qgq8c7S1Q3BtSkwKFqcSww2owu0MclftPUEEVB5wWg6pqgq+s0GIURiU6nEwnjztdc5ODjg7PyK2f4RWucslis6a1HasLd/iPWO1XqdwrCEpCwqiqpiNBohhnkiyzMe3H+0a/4sl0tenJ7y7PSU8/MLNosrzChDFRWLs6e0l4I818xnM6TO6JqWXCeQ19mOYHuQAmEq8Iqu3bBY1VxdL3l2eobrekyoyaWnyAXB1oxKw8XFBRcX57gAeyd3ubxe8e6Hn7DpBRfvveD6ak3TWmbTKet6zWRWIjV0Xc2KS5yNHAqYzyeEMMMGR9QGlWfUXYdSYEzOpKowUtJZR9f7ZNtSaOomEIXF+gbnGyI2eYdrgckSWBoGFYhzNql9+p5tFDdA3/c0TUcgpGB1eggdBIWSyZLNqEjwyaM/mjSnpwokEMMwz3ZrcDVaaXKtyXRSiW0b3CGGFDTqhhrQObTJUkPVRvAiefBHiegDSkChDCILNF2H67ph7fbJt921SL1tGm+Di1IIrdjVpr/2rQ2AKiRGe5zr8VJgO8Vrj2YcHEpkaXEhBcWtly1dF5PvtMpZXDrOn3muXsBkpPnGNx8hZKCqcparFU+f1CyuVpRZydH+fcp8j+V6Se3OiU2DED1aG5qmI4qKGCVaZug85/mTBZfnlvNnkTdfP0bLFkHAOkOhBbbpybQk1xlv/fgpR4dzEAHnFEUpOTgINE2LkpHJTPC1bx2TqX2c6Pj46QtW1y13TjSHJznTvRHNWU3nLNNpgUJSFSXPl9ecvjjn+M4Jdf0CF5MyoigMPrQgFIvFFULC0XHO3l7F3r7hZz9vefDgiKZe0zYdT55ecP/eLGUZeM/V9YqyHHN0eId+84zNasmoHLE3OuJf/o//Jx4cfoVJeReoELYgxjaFcN8Kj2bYlGt5i4w0IOk79tRQnwgRhqZRAJeUWXpHCEhBxsEnUlJeFGmOFyCVQIlEyCnzOSEAUSR1J+CCHzbESVGTZWannAK2uE5qTGsJg7JKRBKgpVMz1tsOZTQhOJp2gxQB51o26yui1zjbslmtUFLQdgl8rMpIlmmKLEtkGOs5mO/hXEArnc4DqRFdqDzV4GKwdhiVCAHOO84uLlAy587JMWcvVvS2YdV0nF5eM59NqFcLpFLUFxvyvOLo8DitY0pzvVqyt5ehlcfFmqPjgovLK8oio+s91kXOzjbs76Va3QZPUUHd1mxqx978iN/9na/x/Om7HJ/ssdpcI6TCBcFqsWK2l9Oua8RgkfbrHO16IEsZS/QdnQWPIcsqZtUIbzcIZYgx0Pc9y9US6wNlkQ/kk0BwgnrTMZ2C1oa+96xXDVL1SC059GBkhgySZtXg+pbpPIOYJ8BQC9q2o28dXQ25HKGQNKsVmVJoGfn0xVM++eQT1v1qcChKWStaKeaTKbNswhuPX+P99fus7IYiL3EWiqLEa4dzIfVurKC1nnYTUsCtjJyM9nh2+iLtv71DaYUfgMeua6kmFaMi1X+N2yBlAKWINilRgw8EL5mMZqnBKjNmkz2KrKJvPWAJvsE5wWQ85+jgDt66m7pi7bk3v8uRPuCg2GeiKxpRUzc1shA4nz4rQTGqRkmhhuf4zgmruuNnb/+Cr7z5DYpyhtIjOhe4uGqSzVjegPdE5zk8OmbddWiTLIa8t+zN9xiPx4QQuLy65M7dOztFU55ng8dzYtVOJpOd9UPf9/Rdh9GK+d07ieDn0hpxdbahXTQoHxibioPpjE1YYJsWjESSrCqt66mKUarN6gbvBUEEgkhqKqkEQt7s5VrbcL045WJxQZFp2q6hzMfYPnJ87z6tj/Rtz9K3qNmIQqf1VowKMAEtDV3oWdULqrKkaS3zyYjLswsuzpaYvEKaPGFI3uOCpSgrrGupLxeMpwVlkdO1LdfXV5TVHqbMKWSZVIzeopUmUxofEhO3rh11VxPwg4VRsj9CwHg0pnU14CHAqBzhuw4lFFWuaWub7Ch/g0OILYC+xRe3WUrpfzHcWNKKQbGUHhsH54rIrR328PMBGN/h2Z8jtn3+31/wve33EYlItlu3EIh4yzd6iwPI5DC9xUwg4QhysMqFwapZSMLgY02MGKXJshT0mYTGagfUbolxZVkmUpQfMkLa1GhRSqGkSdlaOxKcRQg1qP17GFjUeWaoyoKvfeVVtIj86Ef/yPn5FbaPZLrAWU/bdoP6IlmVZJmhG0JmnWvJ84y2bRhPRkwmI0S07O/N+PTjDzk/P+XhSy+x9aofNpm7c5t8zeWOFZ2A9QGgjimnUAiYTCdMZ1OePn1OlmXEGJlOpzsv8pdffpmjo6Mbu2QBs/09REwg96uvvca3v/1t/t2/+3e7a2RtGqNSKry7YdlnWbZztNgCz9saJSlM/U5Br7WmaRqstczn8x0zPT3Xk0tFVU05e/GUH/39W8zn32YyMcjBJllw45EvBcPnvyE5fimIzue8/AeCwQ5Ojzc9GokYiH7JFSQRAmLC7rZP290p4jP3Qhq7/NLjfpXjv8lIOcYtLJl8NlNXI2JdAvxCGPy6YjJxSCxbwdbuZOuJ7gM70FAJgdIpaBEEfgiFMyqiVUQpj1QxwR3DJ916SW9v6u0ADdEPVic9Sg4A/lAJSiUGO404WLlsAVIwWiagjDCApKmTHPwNhA6DQcDQHNhCtM5FJMm6RsDARE8s5oRIpuc752hjT4hr/CdPaDvLVamZzAqmrzxiMq9YL1fM5goRkoRGK0NVjrGux7qOPjicj2RFhrM9W4QvhEDTLpEiI0ZF2zcoBBfXT5AyYKTGmBznDCIfDd08RYzQ1jW971kszwkcMCtHQ9dqQmiXKC85OkhhJlYpPvr4I1SAn779AWcvzuh7mxoUPrA3nrBYrhOb/3bzIfY35zAENps1WiqUTp0/hKYOmqNv/R6Hd0+4OP2I2CxYP32b2J+B6BPkPFxvYkiFgwt0fQKX5U56lKJkpZQDG3U7AuJuHMqh26vE4Jm261UNr0NiMoeYFApGJ1Bk62074GwDey39EcPYLHTk3p5kNpZYn17ntpwkAeeD5US8deMKsVssd57K0Q2FsBjY9mpoUImBhQ+tTTZD4jd1cxnW8qHnhRpA6mF/98WPF7sT8UsTj7j1xT85Sd7C/oUGNdjneGXRoSLEwZ4ppM/NdqJEgrdsJUhKJ7a4FJI4yK+27N9kp7JtekViGAA/uZ1Iw8DSdThnES4SpENFDdGjokHoxPaKmGT35Hxish3sc3F+QVlWtE3LqBoNVjR2mPyTXYZzN3KBnUVHEEOBn8ZkXa9BVMQYefb0Kc5avva1r3F2esZ//s//mel0xNHR4U5KtfPM1ooiL3B9j/eBruspckOQAplrRg8O2FxYqqykCpLMOVSM+M4jpKZV4Iygw1FkidmRx8Tcdz5QFCWz2R5dl1gCzltC36WAsOjRSg5MuCQbkzotxGVRkuc5WZ6jVI73Att2tF3HarWi3mzouo62aRLzejgPSimm0+kNS25gJmxlZXWd7CuaegLCUbdLZt0eo8kUrTOkKNhaM0ihBy9DhUClMYJISio/hPUMwaPBOpxNFjJBKpRQqOHGCt7TR4d3PX3fsqk3bDYbjC6pymnanLqeGFOhF6Kn71vaph5WaImzTWJBWZfsqkIKigkhghRolWwnkoVYSGxVIUBpvugW/FWPLNfYy5YYHVlm6O3Aoh+AlhjjUJA5JN0Q5uNo6g0QyY1GZZLGDeCokgxZzMluyfeMR2OK0iT7F5/sBrQxlGVGjJ6uWw++jRIfPYGkIuq6mq5LbON6s8L2PWLLJrMusfBcjxSaGA3WRbq2GxpJCegVwbNZr+ns4NsvUsNSKkVZlsmWxiaLjehdsvKyFpUZ+qXDdh0GiZfw8OFDHjx8gJSS0WTKeDIDqRHKMKsm6Czn6vqa1XpF2/eU1YjDvUNG4wlSZ7dsgDwySnCpsVeUJdNqxv70gJfuPWCxvObTj94j9isKFYl9Tet7MjGiW0dUlqTdsW+omyVdvWHTNvgYkCYDpUFnXFytee/DT3n7F+/jreP+wYTDWcHh/gTfLRmPDJeXlyyXK9A5F1dLzq7WXC5arjee2gqCHvHJsw94fnbN0eGYdbvAh46yLLl3x9DUC95/Z8Hk6ABhclS0qCxHaoELkhgSqHeTZRJRKhJiR4wWpQNCOJxv6PsNXd8RYyDPDCFkuN7tbIVu12tSGbI8J8bkfW+0p256ovdI6ZBYRHDge2KUGBUojMR6kHiIfrj3PdG2BNfj+ybZrmWaPNcp60TGxBBWEoYCPAqNkAptcrK8oMxyRmPDrKzItGG9aei9JUjwMiCQqDyn7T1d3yKiQw/kGBG39aHAxTTve9cPm5Tbm+Bf78jNCKkbMtPTNj3Hd0746jfeZDJeYcWay9MFlxcr6k2HtxHhDUpolheCzXXOH3z3dR68YihGHZGO9XrDYcj45u/e4Sf/8D5XLwICxXJZk5WaYp7TnvWQgZeCTR148smGk+N7KBRKQalHCGuoL+Fv/uMpjx4fUI4lT56fcnllOTrUjEcV7aanrh0r3XJxsSbL4f5Lmuglo1HGfD6hax1KTXjt1cfILOfJ8+f8+Ie/wKjIdFZSNyuePq15ZZThtELIyNWi4fBwhtKCTz75iKOTks31EqFhb15ydb1gvVmRZTn3XypTwHGo0Waf1984IcbA4eEh14tLDg8mSJVhvWO5anjzq29CEFyfXVBmBu1HhHXGt3//95gVhwinca3FaIuLNQwe/dtN8e0Q88TyGqS5w5Est5Jffvqa4Y8anpfC7mNI7Oy+azGZIfpUv+amBAF9twG5zZ5JY1sNygznA1qk7BipBJnWuCF4VytNFClscss425ZuUqa5nzAQVQaFadvWw70bkMJSry+p19csr1qUFCwuL8iyAqkNIGnbZIcQQlKvllW5yzoYjyuur68J0VEUOTFmA/HAMB4dkleGN17fQ40rahkofMN6veblVx/w9PkzXN+TKc1qsSCGltyUTKYTRMwgJhZj3zs2m4753gTnLMF7TKZSeLCTKczYxeQbL1NtlpeS0uRD/S24ur6gWS/AJ1u5etPR1IaiAttDcArpwDftr31vi96gSfZU1ltcC4V2FMbQ2ZyIAlnj3Yb1ZsXF9TU6ryju3icrZni/BuPp2obrxRlFNcZUE5QLuK7Bdx3L5y8QowlsGrJW0G4a3LghxCldp/A+Y7V2LK5rchM5mCtGZUbTLbBK0MU17z37kM4FslFBns/xUiblrO/p2hXTbEw5Lcm7AosDEVE6Mtsfs1qvaWLLpKiQwaFFRhYrlosNSgrMJMc0hmyl0/4sFzjtaGWPmRiCdLhYU5aK2Cj6LqJMgdIZQRia4GlsTwwO5xpGoxHT2YjoBZmeIIJnlEtcSKoyrSSFyZEiZ2zG3CvucXfvLnujKSbA6vSU82dP6eo1fZ3qOTOaIKsJyzNPEwQ2wovzS4wu+aM//B77BzPOzp9QNy3T2ZyXHjwgKysaH5Ax4vqWxfqaYjSma5I386bZUI0rQoyMJxPme/tU5QjXe2LsqSqJdTVSSEajMdY61quai/NLnj19igJODvYRBPq24d3338a2DaNccWc+IkxH2HCQ1I7ZQ15cXdCKESq3rN2aurV43eECVCZ5J8dSIDOD7SNCRzwOnCf2a6xraZsV0XvwGX0d6JuG0XhCNRrRrRaUVZZqmEwzyVMgK8BqtUTIjM1qSXSB6CxBRNbryMXFNaEz5NahM4WQErvusH1PPpth/Zp6c41RE1zXEXxks96wbFKuk7UBIyTCB4pC4YPl7OKUzvUoE1FG0mxqvMqIXoJQ6HzCVd0glGekCoIXTMYjjKyoNz29i6iMpDr4DY7tHi6R1rag4g1WKGWaa24//ua48Trf/jsp6geW4Wc4t7drjC+CCb+YBbwFVm+UsumvLTv5y/b1AEJu9/1b/3YBQpNnFZAnHMpkFHmO0RprA1ImQH372lvWs3M9zjmc98kq0DlEBC01VT4i0zrZ9LQy7a28xXaBKATalPR9Q4wak2W88uoDJtMxH7z/SWKkq4zr69Uu1NOHuMuLSmC+3nmAQ+T+/buMqpLl5Slf+8qbECOffPgBrm25c3yHclQhlERnJlnhSIViu95H8IPl9fbciLRumzyjrlfUdc1sPuHgcJ8Qk9LbGMV8b483v/IGv/Pbv43Jsp2NKcPr5mXJ+uyM73znO/zgBz/g9PT0xlZGSuqmHtj2CV9Sg4Lt89fy9nPcQPTbnoemaei6RK7aWqgKkm2WDylj4MWLSz795JRvfOMRBIEU/oaJLvxgWnfjbrD9vVvL3S8ag58fwbdH7a5ps6NYJrws3U5bbEr80vO2WNe20fFl4/i/dvzqIHqMQ/EQdr0vH5L3sHNb65QUiuODQPpkfxKSihbnwSWSMoOanRgHMFuA2rYTYupoaZE8TNN59Wgpk+exGuwaYgoJLbahcxGiD3jrCN4nD+soBvYFSZYlxQ48S5sbjxYRLcGRPKfjrkOSBrhkAApJwarbQbPtsKTPFlIjQaRwUuu2YDzEwXAl3ZwtXYi01LThnPE0Y292F+0tH7/1NsevPKIaHaIKTYgWRAplG49m1M0S2ywZjSdY51NKfQgoobChJ9g1IQassxRmStNsmIxG1M01IqvQOgW3dW2LGc8xKofo6XzLcr0kyoxPnn4Kdx6QT0qiyBDjPVTdUxWGo1nJW59ccHVxzd88PefJkxesVstkp+KTV3gIgw2B3yLD8dbIT19sIesYAlle8Iff+yM+/PgJ69UVP3//nD/7ne8wPTpms1whfE9/UaMHFYKUAu+3/vUOayOdS4CVYOh8CsiExMhbgpG4bXkkL20pIpkWic2TXjpZDWyp4Ai8E6mxE2PydTYqNVH8IL+SKShDbhcKIpn0HIwDx3uGySjjetVjbUAJmZjvJFsHL+PO+38LnguRJL5RJDAxBFBSUuRFmiiFJZI8qFODAGxwWH8TUvobHXEA+W9ZoAx35LZfeOvBN02H7T375e27sGs4fNHvvPFXC0PAqB0WDjfYXOh0juS20EgZDMStDD4AcvC2HnyvxJblO3QGxE0DJ4RAEEnZkkBxDwzBtYPKRQweUH3siIA2hlyMBhsMgcQQQ6QoChaLBV3fJlalloxGJecXF8OC6dL8NfhvKS2HjWd6P1tZ1a6AEolxuV04Pv30U7IshaR9+OFH/Pt////lz/7sT5nOxljb7RhrMUa00ZRVyXptk0dv3yNljlAScVIRs45m0TLxGtlbhI1E6/FoFiLQOc/0wSE2kwQZWTUrptP5wCxIadwAZVngnE3M6eDQUiL1VlWRwv+0VIkIvqVYxtR4NdqQZzCqKqbT5Me+XC7pmsROCyFgnRs2tYnBrbXenaNtMbdaLVmtrlktc3wo0JnGB0vbNUyne2RCk2c5asjr0ChElMRBEhpihOjxzpECejxx24B1kSDkDet++J2pmKhp2zV1vWa9WdF3lqIY4VyHUoZkjyIJMTUHk7dzPzRNLJt6xWJ5TdM0yX7Eu8E3Pd37SeKokrIqk6kgFiDUL9sp/bccz1885/ziDOc6hASlE0gghMFmaQ0Pw5yqM0GRG6xLHucMDTotFbHMcNaytRCQMuKcTWurIjGKlcCroYEdA9G7Qfacwr8AkMkWSWmDtS3X15eEEKmbtCFVGvQwr0thdqqxptkAKfzGuXRNqiKnyNO9n5kUkhd8AJVYH33f0HRJ3r5trDRdm6wRTGqMWtcyHk+psozReEQ1GpEXRWIQSp080aVmXdcsTs9oux6pJNP5nOlsTlmUCGV2GS9GZ2k+UDqFuG5qVv164J8IZIBMS159/JB2ecby/Cmr63PGpSF0gtr15FWFMpbV1TWsr/F9h42eqBW4Fovi/GrFD/7Lj/nHt97hkydnaGV446UT7h3PeO3le0xHkuXyirPLK5arDTIX9M9OuV73LGvP6VXD1cpyfb3i9HRJrsFkkbk0nJ33NM0G1znunTyg95Hl5hqHwEnN9PgO08OjIWRW4Kyl6yVlkVMUOZ52uCdruq4mxB6twWSSrov0tmWx9HRdQ5mNhoI9Ne1SgJTA6HQvCATOZSlQu0/gM9GidMrJcbZJNl5BkGuRLIeG/JDEfwqI4JDCU+i4CzA1eYaQSfkTpcAS8C7gYkQED0KQqWQVJpVmVI6ZT2eUJmc+9jR9x7pds2yWNF1HwBJdnwD70GOUxJiBretTzYpzqQlrO3rrcT7swNFf9/gX3/tXnF99xC/e/REre8XZszV/93fvYMOSqC3TeZEaHbri8rLlYDahXkukyHj19RNCsFxfdRwXGVJJxqMc6zyb9hnHdwN3jqesLgV/9Z/e4fCoZFoEJjNJ00WaLiCDom8FmkOUlxQ6p9+sOX/mMJSEJvLB26fk48BkryDXlmADwaaGgoo51+cNhclZXG44046m6bhzZ4qkpygNZ2enrNfXPHr1Ze7cPeDkf/sdfvhffszV1TVFkXFyN4UQ+xDoGstkMiIrUrPw6Djn8vKcSGQ+TnaCk/F4yCNRRCJlmSFwfPzxh4Bmsw5IoVita/bmU9rOY71lPJ0ghKSuW7raMTYld47u8dpLb/L1V36bveoumpJcFwjAdTUyV6mxOozrPNMky8CAd32qIdnWSEltg0hraAw+jeBBOt33yT4gsfVgXa85Ozvl4GAfpTSj0QRnAzFKvA3I7IZFqKTE2n4AacRuE++dI4hAygYJRFJtG50fnrsNuLup4dTweZIqONV4UUSc7ai7KzbLS6xtCM7igeACUUbyskAOQeBlWdL3/S6s7OrqiqKqiDGx4v0AHm02G/qu5/hwn6IYU3cb6mZDDBJjMtbLcxb1mr35lOlkzHq9QgZJVQ3zu+0YTw5o1oH9/UNkUHgfuFrUrDeeo+MJl1dnFEWJMQWPHj3k448/YjqZUTcbyrIkhKRwOT+zVJMUENl3LTIYgrXMENS1ZTw6SnWs17R1YJSX1OvfAESPya87y3VqQms1NB8tz549RYiAtyuMDinEsW7Jy1RvVdV48BHXEDO8s4Ma0jCeTKEqqBfXuK5jZT22tSyvF2DS3qdeJ6Z6nguUKgBH3TaMXUcWFcH3bLoW7TSrpkFkOUaaQU0naXtL10fWXcMqbLi6XrIMGyb70yGIPCkpx5PRYCuqKMqcTBfMJmPyvGKyPyMSMZmiHBlEAJ1JTJENORBzri4vcP1AOIgCJTUxgNCK3lmk7XAhsNms6N0SWTjOV56u8QQnkeQU5Rjjc84vN9SnV0zzCcfzY16//zJlr1g+P+eifUKlFaNMczCf4KuM1dLQWMuy6XHrJU5m+KjobAp7/OqbX0EKwccff0hvHffu3efho8cgBU3bopRkXOaMy4K276n7a6ROuU913XF+fsXDBy8xGY/xztF3DUWe1CPe94OyoGe9WbNcrvj440+4vLykbRpef/VVLs5f8Dc/+Cs+/uB9jJa8/PgBmZywWlynRlGEuu85vb5i1Xe0vaOaPqacedpwjc4DrrtiPFJsuoZKjXCAVR6FRqOwtqHIdcoP6S2E5F1udMZm0wCei4sznAvM53O6riP61LjOdZHsGPsKKSSb1YrZeA/oicHT1D1SGiazOc4GNqsNLgzOBCESXEAOAeN1XQMCaz0ZELyjaVrq9ZrpqKLXHVWes1wtuby6pAs95ShjokfYtqeXPfhklZkVJVfXV5QjjbaBGDyTqSJKR2d7Npuk3JxMRr/2vQ3s2LxCbMlqCW+Sw/qwfcyWkJWAc249HpKt4vDc4Y+Qw57nc4+Fz5LattjZF20wvhhYjDfvky3Iuf33lgi4JRUm4tvWakOiiNsgbSXJs5wsG6WsBk9qsg7kvd72KVy6SGrZLcgaQkry1lKjVYYVPVqZtNdQkTxjt+eKMWEpi7omy4pEUBIOk0nu3Dnk8PCAw/c/Yb1uME+eI2Wk94HNukFrjbNxFxyrlBxA4pzXX389uVk8vsf3vvdHECKz8YTMZOR5lqzbpNzhO9uG+O58hUTkSzmYW7uUpCKbz/eYTKd8+umnTKcTfvKTf+T45JiiLKiqioODfYqy4M7du+n6yLSG277D5Dmj8Zgsy/jd3/1d/vIv/5LFYkGWZaxWq4F9bj4TIHr7Ot/2Tt/+TEq5U7ZsrWH6vicFiraJGBAceTGiaXoCnovzJe+9+zFvvP6IXOeIaBHCD5hrmvPDrXrinyJZbn++VTh8Gc6VuMo7mvruvMf4y4bDN7+THT74mxy/Moi+ZVeJkKwktic6binYwxEGBrYS7DaVg3oxBXWGncVVknfIRFu/ARIHLFNuQXQFIrEBlJIDi1oma4tbGG0MkUhiZkBicguZGO4pkDEOryt3wKeUgszIJInadqAiuBAGRnzCVJUYuhYxdVOUFBglMDo1D3qffo9UpN/n2cmTGbohKTw1ELylCQ2dC0ifcfpex/XTTxnfucvhoxOuLq+5f/8EFy0Rh5A6gSo+ZxRnCAS2vcZ1LSDApEaB0I7ebxC9TotPU+PsBikh08UOePZmYFjrjOX6iqfPP6S2STY/yUs++Ogt1L2HCOaMqpKgDEU54e7dA/793/wcLyTewvn5guvlitQmSGxK6/ygMLjpWqaWRAKBtv3R4UcI4MGjx/zJn/0pP/7Hf+DDT54j/vmfUkWHzzZM7rzKun2GbHpE6AYZkdtN3D5A7+Ku8A+D/7iSAjPIP2/kIDd+2FqJBBgMHc8UEmmx1iGVJhLoe0/TBZSIu05V8B7bp42vSpc1qRpkGmvTEeSFZm9myAuNWPkEcAdADYGzXtAPY8Z9juUd4pY5TSoqM0meJ78wQQJdlWY3IVsXkuzcD83n3+TYenbG8JkJNJ23W48bLtzWL3oLgsd4w6T/TGf8s9/47DHcG0nhMthG+BQelpiyyXV26xkdB2WJEFt/53YA4hOTNdUcgYgYwHg7sF+3E3H6fMn+KOK9GHxp/Y6Fbm0PPt2nve3xwZMXJUgwWUnwEJyna1u6rmO9XiIImEyR5xoXHFolQC94lUBxXAr/ZRtQ6odT7gf2ZWIkm1uBktvN8qeffpqYwk3PB+9/DPwH/uRPvkde6F24Z2oOeLKiYCwi9WZDjAHrHU3fIsuS0UmFx7G53GCFgEyypmcRocim7N89oTyYkmUSFRzrpqbtawqdJduXLDVzlNL0fZsKvJi8XpVMxVH0gxXV4C1sdAI3rXM43yFEsrjaMoS3aeNKKTJj6LuOZdNQNxsYAjCbpmEbaLJLBfeOFy+eoLTj6GSfosxxPt+x4gtdkClFDBLvLc1qRfP/Y+2/mm3LzvRM7BluuuW2Of6kRQJIFFCFYtFUNU2zya6IjmZIEbrUjX6dQrqVrqVoSaFmN0l1s1ioAgrepTt2u2WmHU4X31xrHxTJVnVCC5GRyLP3WXvtOccc5v3e73kPB9rDHmtl0ygbL2n3lSR7CTYNYQ4F9RNx/nlyP0a67ob94YbDYUfb7gkx0dQrgh8xRtqnpeghTm4/DbNQn+kHcctcvX3B9fW18KHjzKcjz8UxRVFbFquGzdmSorQoPYOffo+Ffr/bEmPAGCn4GmPmVr458CaKa9dajUKEnRQCxdwp5McBDxK46Kx0DGhZCwtncNbMf2ciZWESWi3jepp6nCuwhZkDd2Xxt66iqRxZZbrDVoTxEx/yWAyDonJYLYGWxjiquiKEkq7tSdnjPViTJK1eK4ahn1mj4jqZ/MTYtcScpDOiqAjeE4M8s4vlgugc1hSElCmqBeuzS4qq4cHT5yzPHhNT4vr6hraTkGNXFBgnTPl+HNHGYbLBR2nLDDGjcqYuK1xREF0kj56u7xn6gUN3R9ffsijBpB6nEqo0xGng9f6O5WpFWZUc7m7Z3lxxWVmySagQCJOnmwJ33ciPf/4b/vW//ne8uj6QlUHpyNXugJ86rMl88tFj4jQABaZcgSmJ0eKcoz285asX19zsPfvDgaq0oDy7vqeoE9aVeD/x1VefoxK4umHXdqiipFitOexuiSpTNEv6rqdtW4wyrNcrFoslOUkCiZ9G2u5ATlLoLgrHer1A0FwGoy0pTPgpk+Kxc+y411NYURcprCUWkcl5xuRRBAkNRQo1KDuHOgWcsYQkLemQhFdpoNAWZSuMhtIpCoc0HVpNVBAD5CDFNB2ArPDIP3HeDCalGOOEUpmiMrisyd3ENB1IBBQR5xQ5KpxVFIXgw3yM4OPcTZnmQ8tc1NfqP/Pk/t1eF6uPITreru64u/Es1g23Ny0heRZnhu4wUdcVQzewXm9wds1+e8PN6y2/DV9hC7DGslkLxucP/vAJj56usXbg4rKE6KnqLf/sv37AL39xizaGxi4wBtJUs73p2Q0jf/HvXvEv/8X3mVLmB3/5l6TkGb2mrlbYApTriXHk4kJYtaCo6wqFZxjnjiNbsN95nK358vPA2Xki5jsuLhZ03chf/MXPeO+jB3z0/jP+8I++yf/8P/0lIQ1UdUAbR1VobGNZLJY0dcF2u6UqKpzTDKNnt524uNjQ9SObzZJx6unaCW0yVQE+ZG6uDzx8cME3P/mUH/3otwzjKGiyypJC4ObtFburA+GQ+Ad/9id89xvfo2KDS0umLmMqYYjn5Dl2KMqZRVrN5YwU5kNenFsbZf+nUKgUUUgQbYzxlEMCUFqDD4H99k5yEQrH40cPaNsdy+WKGCbIDo2hrhcM/V64/XMhWM3u85gyyjjKojrlvKSUpAsrCvbqmFFz3L+dxmk6Fv7lP3OOTNNI4Qx1U9PUiTi1dF3kZnfH9m6LsyWLZiX4DhSr9YqicBwOh3kt9JydrdkdOmIMlGXB48eP6boO70e8D3NI5oi2BhU14+hxrsQay4MHlwxjz6JZkELi0aNLMhO7/RVRS+F3uVrx4sULxnZiuVzx/vvPmcKEKxVa3xEDnJ8vaJqCGGePW5YxqbVj9JEUI7c3e1I21JV0qwyDhN5VVc1yWXE4eIY+YC2cryou39t87WdbaUMOkJWiqkusszSNw/uRqzdfYq3Cjy2lg81mhcKgsyJOnqqqpJgf5N6WZQkxzkHLlqygrmpSjPRDT3cYGMNEVZZYXVPYWroRKGjqBenMsd+3TDGgnMEVJf32QPCBZl0yxUDwe1SKlPWCNHkpsIYs4e4hEW2mcDXTGAFN9IHVSnAlpbOEqaMpa6qiYbU8Y7Fecr29wvtAVVuc1lSV7EOKosBqx+5O9otKzbt/LeJhyoLw6IZIypqoA7rQFI3levuKcfDU5YJhl3imPuZs+RTKlZwj2sDTR0958/OvqHTi+9/9Lk8fXrIoHGkcOWzv2N7d4VGURcXTZk3vI+2UuD30vLm+5dufvMd6tWS3u+PywSM++vhjHj1+wt12y5u3V+z2B+7u7jjs73j+9Cmffvopq+Wam7sdMSaqcsE0BqpS7lEMgk6TFVJE9r7fcTgcUMrQ1A3ee+7u7nj69Cn/47/9N/zgL/+CZV3xwUcf8cHzp5SlZRxaSqNwSrSU4fqah+dn1MNA8CVKLen2t+R94Pb1Na5a0LWWFJfYyuKsocqZpl5wudnQj3u6YcvQeupigbP1CcPRz12m7W5PWS9QKG5vbgjBU1UlSnnSLlEWG4pCAnlzFsF1GEZIUFXyTPvgyTnS7vasVkuqxZowespliTM1Csm62e8PlIsN6AJCZhp6DilikXPJdr8l5MA4TdTLirJu2O0OpJSxqqAulriiZL05w8eOYRrx3rMJE0lp6s0KVZSMXcvw++JcmJ2zp//Ks45+j245fuVoazuaPOVE+a4T/V6oOgrXJwFS3SNY3nX96mMX+TsC5Lu2OZHAf/dzzH/hd84kxyLrfbH1WHy9f6+sxPCWUoA0MaoBqxvB12rRsayxM4YDCufEoR481ji0mQvRilkT1KdCsNJzp3h28+fJp88QYqSpCsZhoKwa6tJxd7dnsdjw9MkjQshsViv+zb/5dzy8PBe9JRuu3t5SlwVdNwKJwlqWy5r3nj+FHFg1JavlgpxgtVqJ6F9WhDAHilp7n9NlBIumtJ47jY826Puwbm3EjR77nsvLS66vr/n+97/P27dv+fa3v81+v2e1Wsnvl2YVPitiiLiiJIdAXdc0iwXGGB49fgTAMAw0dc3k/Slg+Cgw/+1/jkK5tZZpmubzcT79f5BCflmWjON4cujvDltWy4Zp9JAdX315zeefveZbnzxFOu3ETCwi6jHj63fPtPfj6Z3xdvq23/3+fJSdT+N3Hv6n95jHxSkk990fJGNakU5a9vGfr+NG/18loueU0O/oYoJokZAvxb1jQVAU80KGbABCzAQN5ih2HRcDBUcXv1b3qBajDdrcXwRUOrGw4V6cP/5Bzml29k0zpFyY5iEpUjbiXM3yKx+d6EYrnNOS6B0S4eiez0fh997JrJAwiaNaWWpF7YTdPg75hHM5iqAKhVGaNDvtM5mUYMqBkETYd3hc7jlXK1xp+PI3X6DfOh48XGPN8XNmrNYUtkBjBDcxO7yLoiCjKK2FdMTUBMbpcBIdM+JksDmi5wDAql5wt7/l6uY1d9s7ej9hlOb1fo9Rmd/ESGk/ol49n5lTlvefnBNCoB869ruJ2+2OnGf+bzhWP0UoznlmMHHkIYmw+e4UnYHgI1999Rn/7f/mv+H7f/RHGDLZOtzqAQ8uHlPpyG9vfk6arlHJSyp7yjMnzMz3V95RxuPxQTCnMZXzqUZ6Kpw4A4tSWPjHYIcwCxRFWRJjZpwiY8g4NbuFs3DrpjEyTglnFGWWQFFjDGVpeHBuQRk2m4qyLNA6oHQkkQhJxm9K4KMkm4ckjs08PzuCx8qz2B6oFDM6IZ84T1JsSnMRIUs7fVYzPujrv2KakFaecDpsGWNPjcbqiCmYGVnHyVYmcvmud5mfp+q3SqikfmcR1+/cHKXSLGJPxHh0oUvg1b076zjJpdmxkvFxOB1KBSOg5WtaE5MnhJFx6rHG4ZyVg5dSMxomEWeBJWfhBI7TwDD0jOMws2xHum6Pj546COe4nucPZoeUtUachlYR+4lHjx+w3x8YhpnLOAsqKU20U39ywx6fgZTF8c58X08c+nkcxxjnDbJsTkJo+cXPf421hn/6T/+MyXcYOy8uShin2lrqRYOfRmIMDH4i7m8p1tBc1phFSfSR7AoqZ6mXSzaLFfVyiakrjHWiJpljAUDNIrad0TrhdG9TCuQU7rsj0LjCCt7pnRa+44IZY5x5nHsR3azl/Pxc+LUvX/Lq1WtZY/TsXJgRDydszSnQZGIcDtzeKIwLLFcLqlocbFZrrLbEscWiyCmw3d5ytb3l7rCdQ2gUZVETvKeq67kwJ+gQiduQxVSKBjLOhb/+lv3hhn4QDE1OeZ7jMsaUaG2p6oYylaSYZtdsQKGYxp7t9pZXr15wfX2D9xMhxHkOnVv6LCxXDa7OKFvhaju7/AX/8rWf7RyY4sDU7ykri7NLcvYYVWKVIeQe8oDVEU1iaCcpSFiLzhmVhU2bgoRtSSEjkqPHWrBaiet+/l1t4TDOyfwcJBSzKgpczjNzPeLc3HYaIUztvLc4ijMylpQ1JN/LZohM4SxGBSIBxShIKwLdMOLTRFWXGGvIITCMAyHNobGpJ6e5eyt2xBRQ2uAKwCQiimcffMR3P/0um/U5i6pmtVwzYbnd7YkpcOgPTGHAlgUYCUs3aLIq6YdMSJ24TOuKMI0cdnuUUizXG8HS+InRe7Zty357h5q2pHagUgM6HPDDnnGaMK6ibpYQErFrWVjF9nDD0Lccuo7RJw695+XVll/88nP6KZBtQdIFdbOk3jzg+eMzHj25ZJo8TX1OtXzIxfOKmC3jGNhvd3zRvcAETfCCTfE54Cx4FTlME6qX4nIKiaqwbC4fkI24CwkOF0a6w57Be0YfGMYeBdhC9lTeR6axh5zQWdF2g3xdG2whxZuUgmD3VMCHEe8jCjPP6ZIDkKKiKGqUM4QpUxSeshRxURsJDM8zg1ephNWJFDqIEFNDynL4KgtLqRMmR5IfiNNANAVV1VA2DdE1TB3EYcJpxcJpTFZYpSkMVJXFlIZRBwY1EnKLs4apHJjsHl1MWCuZQTE6plGQBcqAKS3RK1IIuLKUbAYjoqTR8VS0/rovq2suzp7z6TfFAfWTX/4NGI8ymc2Dhr5rGYfdkVbDLz67wSgp9rgCjHKEseLVF5l66fjX/48vePSs5s/+2adk9mw2JXYZ2CzXHPaZu/2AsYbdzZb+MDG08ux+9tvX/F//L/+G4KOEz2bZq2BHxnjAmITKMj9jNIe+ZbPZcLM7UFcaWxY8+/Bsnm96/DQyDQWudLx9c2CxdJydXfDiqxsOdz3f+uYHfOc73+KvfvhDytKC1vSDZ72wvHjxlgcPVhhj2O62cjA0cqgNPvLiqz0xbqgqcaHf3O5o95HLizWPH9dMQ+S3v/2Mf/APvse///f/gZgHqnoprN/DgYW1/Mmf/X0+uHiKmQzvf/gRhavRtsS6SpjhJhP9JMI0YGes27GrShuNSokYpWDedR1N00hhdW61DlHCaeMscEtORyAnT8agdUFZlSjVzAJZT1mIO7UsNIWTTrZjUSrECaUVBi37zDDNoW1zh61WwlA14lo7Os8Ajq3eKUaMUrOrPdEPPSl6YpjX+METkiErS7PacPnwEd2hg/lnLmc0XM55do8mxrHHh4gx923qwixucc6yWCzQOmFsJqnE26s3/MVf/numInL+7JL9eKCuSqx2OFVQuxVlabi+umHykPPAetnMPyfgQ0l/19MOHc+eP2O/TTx4UHNzPVBVdzx//oBf/uILqqrmzZtrQhzRRnPYg3ayzxnHCHFkHBLbu5Ewufl8KLzZ7fbAcuEwxf8qaurvvKqmAQO2gPVmhVKZnMSla23k4nyDUTXdYUeOGW3FuBW8p1xUpJyZxolpGKiKijB56fAqHH70GCPhbz5GbGk4r84luLJacXH+UEQPDWiFO6/RRjOlSCBTGE21aPBe8lZkL5+I3UTpKhSaqqq57fYs6oaz9YZubFE4ctYUriL5lsqVVGWJKQTP9vjxE2H/awfGMg4jOUQKp6hKiyulCy5nCZw1upBOuMKSyLRdS1aZKQyEnCB6bFFSNhajVzi7YGz3RK9BLzFj5ubzgS93vyH0hnWx5I+/9Q2+8fAx3/j7f8T5egHBk/3A668+5/r1G3a7HeeXD7l8/IRDyHRj4NCPvHx9xRcvXpPRfOfT77JYLbi4uOD84oLLiwtev3zJL3/1K8ZpYr1e8/3vfYftbseLFy/5qx/+DX/w3e9xfnHOy5dviCHy/NkzyqKkb7eCObp9i7WGi4szJEsnslqt5/145hvf+AabszPevn3Lx9/4Bn/yx3/Ectmwu71he3eDnwamODF2Hdlknj56zLe++QnWFUQyfd+z328Z0wOG9DFvbrb86Ic/5bc/OwCK4O4IeLTTPPvAclbWxBC4fXtF2yZWmwU5jxwOnYRde0+cPN3Qo7FUjwr8MLLf33EwijC1oODBZQVZAlbHbuRss+SwPQgrP0HfHZhGj1EFlxeX1FVFVZWMQ0d36Fk0a1CZftiCMvSHHh97chapKafE4CdyD1FnTGGxyVBXC6wuSDFLMG+1IoyZOCMsYrIcDnsR59uOplnjcyCQuNsdaKr66y/cgM73GVnymgupWaGUYByPeFw1O5bvNTERphN2fo+Tskg2em4Qvz9Pppxmcf1eDFeAPQoL4hY8mSDNidt1VGzUHIipJCMm3Qv4R6LEvQgpGQdBSTe2CMcGreUMX1clSo2EqWXsdxTFCj/GGdcpqMZQGfZDSyZTbM7ouhaNlmsWE370J5TyjPFHZY3GzgZHK6KxKXDaMcaJ5CXDa9kscNZyvlnjior1es1PfvpjlsuGzXpBux/p2w5nHaqWYPu6dLz/7DHf+843efvmBRebM1TKEoqaAik59oc9TbOUcyMa4lyEUAbsXKyO06ypFEjWG6gjwkY76kbjg+fjjz+iKBxPnjymrhusLairmrpaorRDG0cOoI2TsaBFiP/gg/fYrFeQAuvVgjBNlEvJOxLkaKYsS/b7PWVZns63x/P1OI6UZckRM1QUxYz7lHN4COHkTD8cDtR1SfAjw6QwqiIGTXtI/PSnX/Hee4+o60RhFTlByharzLGPQcbtnGN5ymI56j1EeQ7yLIhzPwyNMmKqzGqmCh+Z+6I4SsFfz+N/HuwnBVJhiKh8zJWcjb/qaPX929//v/z6O6/u4kKHHCMZJenfPszVQfn5KUkVTCpFs3A6P2UxQwozLUMz88nFjaNylEVAZXzmXpxIs/s0z20siEM0xCh4mKiYp0hpe4yBGBJ+djSFlIlIkAFJkCpz8QqlM9ZCdBmlC3HApqPTVd3LvykTyTO0XlzpMSZKm6hLcKUid/I5lc5INpCCpHGa+Ybei7jkTIyZKXsmEwneMvYDbz//ith69PmS3R9d8+DRxak6qI2itAVWJ8YcMUhCuzWOPE8WMQQJFNQFh3EgpojOGaMKEkquf9Jo4+j7jt9+9Vtev/mKmDJ9L62KhStJYaK0hm4cieMgPysHvv/tD7k8W0BSdF3LNPUYK5xi5uJHzPnk4je8ezBUpyH5bq0pZbi7ueHnP/oh/+xP/wvq1ZpMplqeYazCPvuE8uGHDPuvyEGYqsHHmQl97ESQB2ceHSTAzVUaWVfUO9dePkdpoXGJwulTO+9c/oUsYzt4CWvFSIAjGZKPeJ/wESCRtcI4CV1yhWG1LoFM2ZTowqGthAjGHIkZirnKEpMiJqnKSqFJFtE0d0OEmJhCpFFWnP0pk+PxUVcnoT0dx4fWM9v967/6bsd9OEeaWe/HNvP7Snacu0+MNVgjThxtHZyEdfM7f/c4L/7tyudRSA+hJ8VETNIanHOcW5Tk/h1RKCBitw9y+BpDT2CSg6ueiVspk5BwER8kzCvbRIy1uBaPLWEgbcbRM44949QzDIIdGKcRQiSEnkO3laCZNKGNRRuHsw2lrSmdY5oGmqZmvV4R04RWmXHsMBqaqmC9bNjudoxDx6Dn8foOzuUYLIdKc6eNOfHK3kW+AMSQQCmmaeSnP/0ZVe349NNv0G1lEdNmFuERBIpzFj/OC18I3F6/Ia82VIsFGIcua3EI1zXJOlLlwM6HG1OhS0NhxIV+bC3UaIwTdt3kB2L0IpJpCQgcBo/Kcxyg4m9tquYgmDmk9ebmhtevX/PlF1+QQmTRNHz67W+z2+24vr6an1tOwSbHtscjNzn4gWk0bG8zIQ5UQ0ffd+z3W9r2jlXdsHQlViEt5t0VL15+wTiMlK6UAyKJcSrFGaFmR3+UjQMZrLMnB/w4DkxjR45+TmLPhCzIlkn1KO3RxuKcJllNCIKLIQmXLwbP9u6W7d0t49CdWHMgBaqYArbQ1KtCOM0OsAk1hxIekURf5xWjn0WXY8CMuHPzPF60kvBuYzJhGOg7Ca51i8Xc9SEhU+LKP8U0o43sB8YYiKdtEaQIE+J4k/ApCXiN0UuQM9Jiv93GecwLnz1FcQFnDFbia5lSRCmDdQUQ53U6khHeZZwCwSfMoGlCRVmVhBDwXoKui9KhjVy/EAamyc/BdQY/QQiR/aHlsy9esGjO+bN/9AHNYsEwRZwTTnvb7clKgvAAirLEuRKFw/tI1IJrKEt32ngmMq9fvaK4ucE6Jz9nv6freqJvyd0dtR5wjWYaO7SCs/NzbFFTNwspQuUkrNS7t0zjgPeR27sdd7uOccwsS8vzJw9YDQlbr7l8+Jg6e54/fcw3PnwCsWfRVCzPLnj87CMwJddvb/jp3/yIH+fEotCsa0M7ZKLKNI1jtWxQyhOSYOn6oWd72JKtpVqvsbZk6FpsWbOqavIshC+aai62aXyYxO0fPdZoyqJgGPSMqpL53HtpERWhzpDyNBsEDNbI4YJs0MqhEWesVomcJ1xRUJQFiiRudWspnEUrCeeNQfBYem6X1Ua6D8XkkeiGDmsVpoSEIylPwoOKGANWaUqrsQjurbBa5vPNkqIsCVkxxIG23dMe9vRTR34Hh6HIFGWBRgK4u6HHGIez5Wk/kqNm8h4/pd87z+TivKRqLvjw46cszwwvb36OcglbarSJpFHWvqJwrFfnDPvM7VVEoRlbOSjHdKBpKrRRpJy4et3y3/3f/opxnPjgvQ2lXfDptz5iURp+8Yu/RrtIGBJOFdhGkwhknejGkeAV4ySHu6Ko0KU43rAQkgJd4orMOHkO3Zb1uaUsNdok7vaCNTIWEpGUA0qJSeHqqkfZjpgMwWf+w1/8DY8fX0heipIu1aqC5bLh7Gwzd/okgpdOROsyq02B9xMffrRgGEbWm4oQe56/t6ZyS4Y+MPSRYejIecT7nj/8w2/zk5/+hMoUOMAqy4fvfcDj9TkuGJ6sHmKyonAFuihJKEY/YnNit72hSSvq1YoY77MwFEb26DqjZhyh1ZDjJB0VSUMyhJjo2gNVXaOsxU+BFDxGSyjr1E+gpLjdTgOFa5jGjrra4Kc9oIg+4KeIc4JuIWdxriVIIZBMEIF+Fm9SlBDo6MN8sL5v8ZbnN2GMk/NSVnMxyBL8OJs7LM3qgqwUq/WaaRpwIfHF55/z6ImlqJagFHd3W8qyQCnBOxmjuHzwUAwlORGCZxgGhiHz8OFDUvB0/cgUPQ8fnVOWjl3fcn19w+bRGTEP+D6wvT2wu/4VH3z8Pk8efciLN1+w3d2yWmzQJrPeLNnvDtjSEoKX4kW1IifDYqE4HPakHPj4k+fc3W7Z3u1RCup6wflZwxQ8WRmGYaC0js2qJIbM2Zk4wUMY2GzWFGXkMLQ0sfnaz/az95/SdjuUCpydrRiGjqETcbEqNZuLimXVcHut8WNEGUEyTX6ipJIAuBnj2R8OKO2wrsKPHu8jy1VDvVnR7Xdoo4lJMQXP2eYcrS1d3xNTpKgLqrqgrivSNEjxMjs260umqWP0He14QGfp/TkceroxU86oNVdaNudLNvqMcZwgG6wuKG3EactmtRIm/TRweX5OUy7Ybg+MIfL40WOu765IOVGWGhAU2zSO3N7e4afAerVGGU0x49p8mOjHHsgYZ1gsalKoWLgFNtSclyU5O6bbyPD6wDhmdC54cvaAj5495vFZzaaMhO4tX7x9xasvP+dw+5bSaKq65vz8HLTh9m5H0I5h8nz++Rf84le/YQqZf/bP/yvqytFUFd///h/T9T2//uUveP32iq7rabuOz3/zK37+k7/hT/7hP+LP/uwf89Wr1/zFX/6A//Kf/jOUVpSF49HDS4yC9XrFOLTsdzsuLy+IMbLZbDDG8uWXX7Hd7kkpc7Y559Gjh6Dht7/9DX/9ox/xxWe/4e72mhwCRaFZr5dslkueXD5gv91y8/YN2hqKqsSVsFqWlBjev3jOulzwZPkQrWqWqw3b/o4f/fRv+NFPfsxf/Y8/59c//pLVZYMpEq4uiWXB3faWLhyI59C3I42JhMHzpn/N+dkZdVnRt5aqLDhsD1S1BILe3m7JMUgHpprhdxlyFpxRVVYUtuLB+SPC5CUbpagJPhA82MJgi5qlli7Tcd+jjeAJi6oCq1BOXMuVqVFGURcNKmgW5ZIH60vIirv9nvZwYBgmNudnpEkMfykk2ral66fZDNRS2+r3Wrv1SdS+3wToOdPr+DoayFBHR++94zZn0Zd+x0mrFFmrWWy7fw/pdOIkpKMQBG7mpNPJ++bTPk3+kJOeOEu9okTc24TlT38HyaFmr1mWfKL5jdJsdhzHlhgmnJXMraKsOX5kPYv249ADsFgsxAQbRJto2wPTNM0G3Bm36SUMVrLQIsY4wRWZRJhGckw469je3XFx8RCyYEVTlCLtarVgtVpxvlkDitUi0rY9Ckff9YKqTCP/4E++T+MMzx4+ZNUsMdpgtCaFiNcTxjrGaSTmhKOYTZZuDpfW9y59IipOYCzHroGjTpayoiwrFoslR+fwer0Gjmdgh1IW5vzIk6sdcaVXdc0n3/wGP/7xj9hut7hZVyiKgpSjrFtlOXcq3et0IQTJMCuKEwddkHdyjY/i+jGTTRCgcS66JzlbFwWFW5LzwJdfvubt9R2PHjqsk6wqrRQxH13gszk0H5EMR/F8DsETORl1+p+ZR9bR4Hw/snQG9NzJz71+Il2t87XiPmBUzZ/h3bF7JB28K7b/XV5/ZxE9poSOQfh8KTHFLG4jD8mL6D0HzZ6E6mOBLWWIcb54c1VKMBjALFDmDD5JGGOa0fMgbqWUg1RaUpzZ4hImmdIR0SATREyS4HtEuPggAzIdRXAS2sjGVatMUci90lqE8RCNvE+cq2zMHGXFqX1dIRU7oxNVmaW4NAug2hwDU0VwtDqfMBvH+t6xMpJmXtM0JfZ7xTTecP2qxV00/MN//ISzy9U8oWi0En7skbWpFTPaoCDEQGEdQSmMdiJ0kIl5Ro4c2UxoiqImxMAXL37D/nDL7eEG79O8ETbkMGKVpu069rst6dkzsmug17z/5JKP33vEzz97y+5wIKRIjmD0/cAOWZA1x2n1bw/B/Lf+7UPicLPl81//is8fX3Dx4BG+P6N5/z1SDrjmgk/+6f+BN2dPuPnJ/4C/ekEYr4jJE4Of3dz5NOyPFd18/P/MY3Aush4/kJ7HH7OQdKy8aiOPaowiWhstHMnjIJYK3OyG13OSsT6Gi4JxIoQaa04PZUyzCD7Pk1pWPDnQKzUHlSYR2+bvD2lu4Z2xQzLW0zxP5pmlLeP8dEX/bs/7f/Z1ONzJpJjEeX9sQ0rv4F3eZWUdQymU0ihTCNPZ2tmd7eZOitlRquzvbAru3ysS4jBzqmc0FNIKL4tNOpYNkM6SIKL32NONe0xlKItqboeSKZUkuBQfPN4LPy1GT4wWrRynKVkhTu2h59Du2O23cggkoWIixIFx7PBRONLjtKTvW7wBVRlxdQXPMfMg54grChSwXq2oqnJO+zYMfU8/IyUSeZ6zfncCt3Zu+/f+VGRwznJcGHOWSm5Oka4b+OEPf4SxsFpVXF0PLFYNTV2jgeQsRjN3VQQK5O+9vb3m/KGhsA1FjJQqYWIiVYqkNVPIJ9Go0LIB8ElwI845tLaMY8cw9PR9RwqepqqY7zLHBkOt1SmM5Vjdzsi4PzoNbm5u+Oqrr8SZFyKvX79mGqQafn5+Toqe3U4wH0c3eAhB3NFafh8fJvohgU70XYsrSowxdN01q7LmYrFkVdcYZ7DGc7d9yfXVLct6hUIOeDE0hCDInUwiJ8OxFbCIBTkVKErBUSgpKBgtVXE9L3J5Ls4yF3ljGOc9wDwHRcEHpSBuQmfd/D6zy94I/9vMPGfrDMooCeBMnpiCtO99zVfw0yykC/PdM6BsgzPS6ZTixDhmunbEaUFQoBQ5x9kBn3BOy9yTJlIWJnrlSmE7j+MpeDfN7ZMxeIy1J2TPOI7E+V4aK/dymN0PMrY0IQa896jgydg5t+M4l1sykXGaaLuevuvISuOKkmGQ0KGuF8eiMP9GYvKUUzlzADUhyPzVh4nKNaQoIu3zp+/zX/6Tf8F3v/M9op8RO1lxfX2NNo6yqglJfp/lckmzWKGUZRgCY98zjofZ7ewwTop/Xdey2+3wXtrqUYrDoRU8UfKsy5KLzRIdD+RQ8ujBGavNhimJYydMo+z7VOJ8tWQbJl5/+SWurHl0sabtI+dnZyyXW673I6ZcslrXPFxe8MGTM5492rAoL1hv1tSrNZsnz8nJoFPk6nzFNz9+Trm+Y3F9y7aNsgYZDVHczGnKeJMgeVw/YKserGMMLclYlptzcpqEf6rMbIgATZCMlLnQYR2UlcF20HcDGXlWQhyIcRKnmNVzkTMigXMFzql5jyjZFz54QhzwfkCbo5M7o5KCHCjLCkXGe3167lIK+CiHq6wUYXYAYw1JZwIQAJtF1PfjSE4TZVmzWTUsqgYzP+OrVc161VBVJV4Z6qQZhp6UxMRgtAjo3nvImaK0aMS1ozL4YQIMdbVk0ZyhMEw+ziicw9d+tgF++Zu/4aOPP+bf/tt/yy9+/TeEAO+/dwZFS0iZwYuhxRUFn392zX4bsKYkJkUKjsyAdhFT9CSlMFaTs2XsJ6zJfPbrW1Ts+MWPpBujWiWapaK0hmn0855P9s0jgZwsRhc0zZJpGHFVQdUsabs9MWnGcWK1drRDZFkZrFWMQ8SpRNYRZS3GanZdRLlEUZfC0gSi97x8tWe/MCwaw5dfvsEVTjo7jeJwiCxqmatfvNxytqkBOVivzxusS5SlsHqrOhDzxGpTM40Tr17e0h5Ggk+sNiv2Xcd/+Mu/5s///J/w8O05QzuwKpc8fvKMP/3Df8DKbChTNedLyIFGoUgp0LcHqkKei5wi08zrVUrPxqsZr6ik+0cB1hnhbGtBqpADxghSLARPWTr62RGWc2C/u6OwJTFODL4jBo/dlKgcGYYDVV3S7m4Zx479YUtZFaxWS9ln5fnMNj8Xxzk4pYwyspcXX4k+CS/Hgr6eUQNaK9pDJ4V0q9BR4ZylLDfc3LwFFG/evsUVltvtjqg0ISs+/+orLi8vcEUpIeQaztdLpmliuVxKeNw0kmKiqpbzXk1RlZUggtoOrTVPnj1guPK83N7wenvDg8slTju++wff4Te/+ozf/Por/sGf/iHPn1nSV4Grqzcsl0usLrl88ICu76hqy7c++Qb/7s1vAEXf7YnJc36x5NXLL/Hec35xxutXd1gLkxcz1qFrubjYoCkgl+x3e1ZrK+NsvUGZQKKlHUZ+nyZRXSTO6iU5TlR1gZ96Uswy5zrFGFtsQFjppkCbkilmhmligQgjOgYKV8gZxk84W8k6FyLWOsqyoKkLmuWCQ9ezP+ypakvbHnjz9hqtDY+ePQKEY28qI0hQbSlNBTqJWKQkm6YLA9vdwL4LWBuxxjCNB26uI0+ef4uUDdaUoBNlsxRcoCmo65IP3/+QVbNAupCk+/j87IxuEFOL1uK6LQrHYd8RkxdsawqEiBSyjQSxxjThCjt3RiVUdASvGbYD+WAYbnpMdJzlNWHusrx9/VuGm9/w6b/6lwztll/+za8Z+8R7Tx7xjQ8+RKVAUop2yrx+fcXq8hE3V7f8/Be/5LdffEk3eT751qdolUhx4lvf+oScI8ZoHj16wNn5GcvlGlcUECTI9Mc/+xU/Gn7CH/29PwEtKLLlsqEuHM7K+pZTJviAUZbClkzjxOFwYL0+Y7PZcHu7JcbI3d0d5VCxXK+52275d//T/4exO0genFYU5RKjNH4YefniJZdnG87O1lSLmnboyJPBmDWLuuTVVy+4un5LUTqKpmEYr+nbyLKwvPfgAaXRbNstN59fY0tNc16yWG64OHuG87c4p6BWrJfnJBd4e/eGt69ec3a2wTeCfet7cazHyRPDQNMUGJOZxp5MpGlq2rbFOsPjh08pTYUfIrfXN3SHA2HyVIuGulyQc55znBKHbYtlIPkoGWRmIqHQ2YARMa+qpRMzDpFl0dAUNTorcpOIXaAbA+3dgXWzZIoFKikO7Z5D18u8jOR//D4vM8+litnJjTrlIwGznpHJSZOVEnPq8bx8FM3fsSce/764Ve6znfLsMD8ZCuWbRcR/J1PteC7NpHfOqO+eVeW9JRvu/s9zPpqm3sG/ZiWd9klO8hKmKYZXH0diiIDF+5Yc12hV4pylae6NcdvtHdbKvNr3Aykluq7FOkGOEI95HRpj1cmYZLSZu6/E2JeSFGanaSIl6eBYLBqurm4oqwadNavFgg/ff47Smldvrnn87AHTlLj55TWFszxcnfGH3/om66LA1SXKWMIwYgBbFAQfUErjygJtDHHWBYgBpzVWyTk2HTXLuXP83ULE8cZoYzG25Oz8ElAURYlS+ndIDCjJijuVNpQSwb+wPH/vOTlnVqsVb9u35CxneGMNuT2cOq27rjsRBI7aw9GFflzvvfcnh/oxTPRk7ouRccyYQmgfhoh1Fq0zt7dbPvvsS548/ZS2k+78uqxhFs6FvnDUlGbtZy6gkwXJfRTSAZQWQ5VGsg9FLA/HUY8+ZeYdSz1GzFnpqBDei+MKyVK4ryadHrfTZ/mP78t/+vV3FtFDmDCqJsWAD4kpKaZJhPRZ42Y2OcwiJsSk8VHPCckQDBjHSfM7dqAA+ADDpBjizC5XzC0gxzYR+b1k8B0/1RE2n2ZhktlJqPAhEcLc6jEL+sLLFp6r1ZmyNGgcyorqf2RUh5OLPYrYkWcO1fGGKoU14OwcDDE7dI/jWVzwEadAwizvCwyyoboXeFOMtN0gQV/J48bIeLfn7uoN67Pz2WFlZ6C4iHt1U5OzJWctIXjGSms5x+k4zWgTJRsunQkhUriau+0dv/3i12gtSfRKG6yzwP3k6JxFxUCaeig02ZUsm5L3nzziP/z4c2KI82HhWM2ZRWilCCQ0/3nB5x5KINXWN7stw9Ay7m6oz5ZMPUzTJWVVkZWiXj/m2ff/tzTr93n51/9PQvghUz/iU2DwmSnMjux878w+FjqO4vmxyHK8DxJKpwkhCQd9FqYVwtr3U5wZ1RIWqdVRAEyMY5x/jojxRxwMiDBvrbjeyMfQ2TnIcK7CaQ0kdSoupSwLjJGTPhKumAgpY51coxji7Lo/suDjXPQ5Vs8U5K8vsgHstnfiksp+PtA5tJLNcgj3E1ueK4Qy8SLX20pQlrUOOzt3nZOFxCiL1Q4zfw8o4Twii0hO42kOUHIVTvdMzwe4PG8w9Pwg933PoTtQ5wKWGZU1KTBXKeRwOk0D3k8Y7Ugxk0MmqTgLn5qcCnKeGMaeu90VNzevmXwvol+yZDzdsCMGj9KGceixekvhIqW2BJWJYaKwhrJ0FEZTOMNqVeNDYLVcEibPzXXEqci6LtEKhhCJWZ4S5lZ+eV4hjtKOJYK6ElxTBu8DjIFp9Fit8P3EIXp+9IMf8fEnHxFz5PPPXnD54JL1asFy2VA6TbCBqixQswtzCiO3t295cPlIBO4cGPZ76rJE1xIMmYOMr8IVMicaCY50zjCOLYf9jrFvSWEgxYyzJc6K2OlcIfffFlhj5vwKYR+nrPAx0rYHXr9+xXa75cHFJVdv33K3288YIc9+v2V7d8Nq1WCtFefTvOCq2dmQSYTk8cHgkhEOdvQ4V2CsYRgtvmkI3YHyyRNKUwIB0izEJYdPA+PUY7QihIEY+/n50hjtMMYRfcbnhJ4LCSqDtRU5KbTOhDhKh4+Sub+w4qCNs3v/iCMLWdrpq1qYecfCgBQvZe2Js4heuBKykuBRlckqzmigr38Sn6YBraF0jrKUwCeVlHRZhImYJpkLg8E4R1WXc1t/IKtIVTtS0jN+RrqA1Nx9IyG7CWUENZBSoh+HUzH1GFbjvSfMXD0zM3jfRTwdXQ3yb4/SCVcVWCfdOiFOxCHMbkiPNhIiOgxesC7BE6LkotR1zbEAPwyRlNzMATTkmEle1iE/BB5ePOKf/Bf/nG9/8w/wQ5A5fhBUT9MsCCnSDQNKw2q9ZnN2gTEOHzK2MDSNwoc9Xd+xP/hT8ej29vZ3XNfC/TeESRFCpKqECVxrw8PnTzjbLJhChFEEY7ShaCoMNXfb1+xub6mco2ka+sETxh6fFI2F1ZMLnrz3IWVVYeNIYz0FI+fLM+rKMAwt3fYGZSrKQvPh+09ZuD/j9rDl5c0VN9sD/Ri5Owy8fn3N9e0dkx+J5Hn8ZnwITNNITIq+H2j3O5rVGmNkTHgf8WoO7SxKSIEcR3IcIU0YnSCLOz1rjcoT1iQKB3VpIDkJVMyKqjJUpZv5mBJWKC2wfnY2BtmDuGOxes6PUUF+jp3L4nObqC0srnIwO/FcLcgv7aTDQVsJrHMa3KLiweaMD54/5XJzjoJTK2vh5Lk3CmkdRp8Oe5J14fEhYGZDQ+kKko20M9e5qioenD/gg/e/wXp1jnUlXTvw6tXLr/1sA/yf/s//R4yDhxcX/L3v/z0+/6omtbeklDiEjsVZgbOQvadrPf0BVExEn+bDQyJFiJPBOkWcURHyuyWKMpHThO8zfgqYgyOgMS5SWAvBy7527visKk1OmcUiM009t9sDTTAUhaBItAVXFuQUKZ3DTyNNXdP1gXFUvGknnj47AzMypYzPmjwFKicFyWqhURayjpSlYHIWVck4jIRB0TQruv4tz99bs9v20nVqBDmF1kwps99PrNfCZn592FKVllW9ZOw8QSd2h5az8yU5BX79m59Rl5Zpp/n2e9/hux/+AeEQqM42qJwZY4QpYquMKzI2B2oD/X4rqMvkGXtPSoqqlBBdlTIaCcTzzKYFuYgEP6LyjEZUCWMLQvZ0upt5smkWXxLdYYc2itVqSdd1BO+pq5qu63AWqqpgHLasloVkeeQJ5yrCJAx5UzZkrQl6Rt0kT7b2hEAgi2ieU0InMR+klAgpkFOkLgvGIdAfDvJ5uj1FZfDjlqnbUhjF9dsrbNlQLs4Yc0aVhrt2z/lmLU5IpblrhQPbj5M4S0PEB0VZrihL6XIpnCGlkaLoePnmc4aw5Xr/Bq8y2ha0u4N0qvrE5uGSevTc3t5gnWWzWLDLnrdXO5bLJQ8fP0cZy2LRcHP1miePLdYp9gfLzfWOuCpxFuqqIuZAP0qRo+smxmlkubI4Y7lcP+dHP/gVH39rjXOGnC3DlFmuaqwpKI0lDtPXfrZvt2+pqhKjMnVdMo4TwUfWqxV1bdkOb+n7ntI01FWFUhbvB5Sf0NZgrGXoOy7XKypXcnuz5bDbE41FGU3XdZQW1pdnXDy8hJtblAVjE0onqqqgqhacn13SLEuUzrTTnu12hzOWdtfSHloWi5rNoiHkQKalGw1uGokpS0dd9uzuDpTNJWdnj4UzP4+fOPWUznG23mBKS991DN3Aze0NMSkKH+ZuSCBr6qqWQHMN1krn2ziNhKjmfCqFtZrFosEW0oXd7iOLYsn1my15F3i+esKTywWpC7SHkYnI/nDHB88v+bN/9PfZLGsqW/Fn/8V/zeXFOTZ70rDn+vVLXr655sXbG7wqefXTX/D27VvatuWj956xH0c++ug93nv/Cd/69h9QuoLDfk9Z1ZxtNhKmHiJ939PeXdMsFvzzf/7P+Ysf/oSf/fyXPHjwAK0NSkVsZZimHnJmHAasNoKZmwL9KIHC3/mDhsViwXa75e5ui9GW8wcX6IPl+Xvv8d/+q3/FzdUbdne3lM7ih47Y91gUi6rGGUM/9Ex54vmH71NVFYfdgc+/eMUPf/TXXL19Q9nIPkxbRVkvUcpxeWFZrR6yPdS8ev2KfugY9yPt1nP+3oZgIkWt2TRnnDVnhH4kmyDoKRTnmwtSyjy6fEbKmWEY0EY6y9p2J2pGipiT+UyhlZiLunaHnwJxEnPKOHhcnfBMrMoCYy1aaTbLFfv9Dm0Vo5+YYobSUDcVfR+wSjO0HQXSaerrkf3dDq0NJkNjC/yYMLXmwfkDbGk5HFrJTUmBi9UZm8X691q7DdKtpqViKV2373TNi1w0C+RqBuSqjJ7T5t593Qc0imYhQisnUfHePX7PiFZosk7zOVR+1qkz//T+72gLSjQmQcrm39m/vxsQmbMYV+Xf0lV59GDrmVVtC4VWkqMw+RUoQ87HEFVF2/Z03cByKca4Y7ZZ09QYqxmGRIgTZkaPSbepOKi1mQvCSp243UM/UNe1GMtmV3bOmaoqudvuePjwkk+++TFffPGF5PZYx69+8xnNqiD5wB9+91Nqo3ExUmZFUTV0ITB0LWXKuLIEJZ3gypqTmzjpTPqdTuKj4CFGwftWxNnuqg3OlZA1ZbmQqzajc5Qy6ONDMSu/eda/lFJMfmK3vaVwhsdPHvHFF19wcXHB1dW1GDVJp5wvuab6dO9AkHIpJeq6lmdyxrkdO2yPzvRjxzyIPpGUwijpeutDoqogofjhX/+Ef/iPvoPWlpRHpjDKvjwf0czpVFSQkNsjIFWM0HJSE7Fb5dl4psUMRD7aZeSZSTnNaJdj/mFCZSk83XtMZ1PsbPhVpz9Tp/d5xy59usb/S6+/s4i+byfqRmJGpikSEvgJYrSk2VE0SXYSNsmvPcnekhQVXiuiyjMvnNldmwlSqKHzcBih9xI4Yo3GGknFzadLFUW0PRYKQFx6MaBnNk5K6oR6iekdrEvOEvjFUcxU2EIDbhbYjaA0siLOwznNFzNmaRcw7zwHRt+L4MHPjmNEOPVxFs3Nu2Lp0YEq/61PYjfErOh9YIgZrxVpHBkPd4TlGmuEcycBCQOuEEcoyTKGgAlWPr8xcsAxeW57CZAVkx/R2pGtYre/5frmmt5Ly2jKGTvzZ5lFTZ1h8BHJ1FanB31RKJ4+vGD04Z3q5z1vSOu5xWe+Pv85F/rvfCVnLpoG0x+I7R0+PMYlxe7uhofPnpNTAK3R5QKzeYYqz8jZzkE1MAaY5kpk5hhqAVklshLxJnEEvShxVWbzu0WXzMzKjcSY6TtP1wbaIaBUBp3QViaM4AP9HCzqDQiTUs/VWQRxYqXyGSOECFPKpLnQdaw0H7EtUh1VM85FrkvMMESJGimsuO18FLawjwk1h9eGnAnJzGM8vXPlv97Lj8O8qHggE8OIIHFkUhMHtWQgHB3xitlZ72WCVdqIkG4LESdmPnVZFFJJ1ZqspK0pIXx1nY+YFVk81Iy4yQARcoqE6EGLoOj9SNcd2O9u8cGybDasmnF+35GUA5Nv2bdbxmEgpUzpKlJIc0UzkXH4YOjHHXf7a65uvuLq+iuGsaWqajIrtAmEsMdqS4OZmWsTSXUMg50rgOIiK62IPnVVUdWOKjuWi4rJKN76kbP1EqflJ6dhIpBRxuDDxDAGMhHvI1pJiKI2ieVqwWLRiDjlI+w6ovdowKJQUXHY9vzwBz/h+QfvEULkRf+Gt4XmbLPi/GxJXRcStOkM52dLNDD1LUO7oylLsnVkFN3+wKJaUtoCXTrGMRDy3BJXQIqBEAN919G3LSlOIrgjYpGz5X0eAUentkIri9ZSjNHouSAqoVbjOPLZbz+jqSvOz865unrD0LcosvBWwyTPbXrHcaHcHCh7P1/nuU1La8vkR/AwjoY4jFSPHtGPE8oZcUEzd0A5QGdpuXeOHL0cVkgoPXPQlbS255DweYIcyVi0LqRjxSTi6ElI1gQ5UmDJc8FA3CAJbTVGSUtxUTpsoTH2OG+Y2dU3FyRNFjRXjJKAjghFWt+3cX6dV1NLIcUWwg7NyZKjcHGdhUmBypEUE2me11AaNQuGSuW5jRBJVk8ZpWVOyrMj4RhcJy7uo6tAxHOtNTFFQpbWSq0kzNs4K2gVrYQfj3TshBxxSYuzcm718kdMx9zxUhlH1weGvkcpg5kfScG4ZOnimBmt4yjdbIWtMJTUrmGzPufjP/wWn37rezx/+iHX13dM/cQ4TJSFFGPudltuZ57y4yePqaqGvh+ZQkfOGmdLjLPzoReOqffjOJ4c/FVVyIYOmftLZ3CqolmsWCxLHp895HJTk+PIlPqZaavIOhBSpu1H7vYjU7Q0q0v2bcf1zR22qrk4P+dps+L84gHPP/iIGAOH66+Y2lv67Rtu04GcnzDh+O2LH1Etzlg2S5arBZebb/Dq9WcsV4GbW8ft3rOqFjKGQ+Zmt2WKA6OP9FNgMYcX56QZh4EvvviMbpq4fPwIpTXDKHkSTVOz3myIKMappesPTNOIMYmyNjB4IOAKYX6WlZ3FVUtKjhilQ3C5KiiLmslHxkHC4oqypKwN4zRIqG9VoFQCFUhxJOeIUhFnRJgzTuEKjc6OqXL4ccSUhhg1BrnHyhp8krFVWGjqBQ8uNlyerThbN2il6TvNOE0M44HYJzyBKU0zamoW0rUE7Fpj5GCQ5ucjJQprWF1cslqds1yuaOqGB5eXrJZntO2AVV+fmQxQFRVlafjmJ+/x9uaXeP0lxmm6LpNwVBZcGfFMPH+/4Uvv2d+NZKMEmZhkDzVNEW1mZ5ySHBhjpHMg+kRK4lQahwHBF0KzsLiqRGuYwsg4QVVnjFF0XcdiWdG2HccuJOeOe808twnPnX9TwFpHjCOr2tG4gpuQcYViGEU4fnwpmCWlIiFkXFFhrWGKiRgVxpScnRUcDgdGH3ClFHb7riWjmMLIWbGaW74D19c7ylKzWFqaZkXowYeIKxzXNz0PHloKa7m+uua9Rx/y5//iH/NgccH2ZsfHTz9iUS64u76RgM7SknOkbffYubNot9uyXi0ZR1Da4lwpWC2lZjRdL4fZpNCFk/1BiKToSVE6GLROqBwoF7PjvD8w9j2VlSLTYX/g4aMHdPutnFPmI11RFIx+pKoc9bJhv7uWcPPkUVHOKdFU5JkVL9kwYhSy83wWAvPYmNdg8hyCJ+NDAsc80zTRtq1g5bTm5uaaqe9IMWKtYbfdEnXL5cMnQKZvJS8i+pHHjx5yfX0roWd9T+lqVqsNh8MepeauxrlL7+5ui1JBCptdz6tXrwUJUJbowpAnz9n5mjCjvI4Ip1cvX6JMNxflE2VV0Hay13jy5Cn/w3//VyxWCoY4O0AT0+hZLZczTqbj4aMl3SHSLEpcIWJEWRkuH654/8Mly5UjhoC1xYytirTdAWVkvvi6L6sMQ9sRo2ezXlE3NV07MsXA5eqcXb9j6AawAUMkpYHx0GOUJaPZnF2wdJb3nzwmx8gwjtzs9jhTs6gWknFkCkzRUNZrysZjqhLvW1amRrkFWluMKyjrJdPUQTToCJiJ7e5OsipcwWJhKIuCmop1qAjxjr5vKUpDyJ56UVJVBq0CWkfG5IlZEBvJKjCwO9zx9uoKrQv240BKGjVGVosFZ+UlpdMsioa+7yBHRj/gXCFzroJDe0cMPavVgqppmGKib0d0KhmvI+OriWfrC84Kh29vaXe33N7tudnu2KwXLMoN7e4NDzc1zbJkDD1ffHlN9hO3b99wc/WWL794QdYFmJIXb+5Ybs55tN6w7w9889Nv8fEn3+Dx0+fU9ZK3b97w8NEjrm9vub3dMU6ebhipqpqHF5f84tdf8X6u+P4f/j3+3//6X9OUFVPbCb5kUbJtexHpjGW9WlEUii+/+IKsEi++/JwP339CxlCVFf14TcoTuy87FsslZ5szzh88kr2bgv3dNZvVgsXDSwpruXn9hnEaOVusmMLEy6++wBUFKUR+8Fd/xfXttQSoG41raqJKtGk/axrSzu02inOzQO0C0xAZbzrulCU1lqooqWqDcwWExLLYAJkcE+vlmv3+gFLw4MElNzdvGMaBtotyxrGOkCAMiUKVZDL7mz29HomT4B9LV2CsYxgj05SYUmKRFN2+R1nLo4fPqPdbohq5229R0bMsaoyyeMxcBDoQTU3bdjTLNb/58iVWW+q6QTlFUai5kGkoXMlmuaHdSgZEc3FJUf2+TvRZCH1XQD85aU/2RBKZmMO9oM69OM78ffr0HsIFF0DXMadMvutdJ/pRx4kI/u74v/uzxt8S0xUnw6LKswD2joHy+N4KRVbvfs7730YdhXwFZu6CncYd07SiajbEIHPr0RkNR4NNwFphhRelOHFTegdhg8z1xhiKokBhmCY/n69E6A0xUJYSdvvxN57x+vVrLi421E3J26uRj7/xAXVVEvzExeUZrh746pXj259+wu7qmj/+wz/AEtm/eY1ZrMhZoSsJEp3GgeqUFxUwzqBnc+49YiSK83oWyt/F5c53UJ6POcswJsl4U2o+T2b1juD+zm2Zr29MYkTb7e7QCi4vL+bzslyzKYTZLa84UgWO1/ndPLau604dxEcXuqxpeUblmFMB5ZTXNiYpFjgA0YeNcXz2+Vf87Ge/5dPvPAelmKIHFWb97ViEOYrWWjj8Wpj3aSYu5Bzm8a1JKmEQLVcoB0KDQOvZZCmaGrMwL1qy4W8rknnGEh9ROgq5V2bWp44v9bf+3n/q9Xfevf/4lefpWUvthJM7RUXbBYZJnOghQj8hFy/I5xi9hCjmpIRfKloUKs2buCADwHtxok/h+GDIxxc21BGpkQTnwjGAcfYkJk+mIOYwMxCz4FxinnEvciiNs2gvD7Uceqx1ItIFeU8/Y2DE6X6cwtQsUmZpp3nnz2OSz+1DJkUxA6corYc5H79zVgTn6uLpJp4mw/kgkRIpKQqjefTgjEVh5/YEJKyKjLWOlAu0shjrCJkZa+Dnh9LSdS0heUIK+CgM2bvtjs3S83J6RQoJqwvauMM5i7VubpGTB9wo4S9dXD5CGSNCm3Zoq/mjT5/OopYUN04DdcZRgLSMTDH8J32T7zQFAeJ4PbQHHj065+GzJyzOLimbM0G0hACFgyyud1xFiIGu3dIPgpORYoVGMQs36riQ6NNdmuuvp5bKjAhDbkawCEtf3CvDGDm0ge02cbWTg5u4QJmdvJ7RewIZmxI5Hx+fueLp3BxyOSOPQmIMx9KCCIh5LpoIFx0Zn/NKIyKQdA84IxkeSgmPLcRZeM8yvn1S+CCtuFnpkzv+676mKCgTEC55CveuXH2sXKYgm24tVT4QnhpZeKQpeoZpQqmeY5FFa+HD1nUtDgFjCUk6I1JK6KxnRIee29TuywFxZhv3Y4e0+St88mzbW7aHK3aDBMQkwNlmFu1Gxqll112LQ6dYsG1vqVwtxbso13LymWHcc3P3klevP+f27jXejzhX4twG6zLWJs6W51hTYJQjRcUUJ8K05ZiaEIMEylVlgUYc2+RM4Qz7XYfSiYsHZ4Qw4nrDQldkxDkTozxHGjW75AEVqeqKBw/PqcqScfKMY2AcPJ3qTmKdsxofI2T41a9+w7NnT3HOMo4TXd9DDvS9ZVFXvDnsMOaDE7vUx8jkI8ZEqnqFR9zcoFk0TvATMx9MG3FW+3FkHAZiDDOSRONDwhpB+ChlMCmhrZuZ7I6yKNFGOmZ8iHR9PyMtJC18s1nz9s0b6qpks15TOMPN9bVMpPldDms+PVPpOD9nS4qKGMWlUpY1KTlSlgLiOIzsd3uunaUbWrqhpbAFi2YhRS4vApcGEXeDjHvrjFT9tczbcWbY5iwCkzbSJuiMOCZ88FI803ZmOgO4k8ggAvnRSa9O+JKjY0SsHSJYC5c9431gmrw4Wp1IF+n36AlPyVOVhqLQGAtkx9TLuqaVjMG6qVitzBx0F07dXpKRIDxzbTSlldyHGCPd1KITp2fXB8GxGGfn4vf9JswYec7HccTPa7Y20gUVY5wzIhCRfN5c+uAxwcxOkkiIAZPNO8UV4VjHFChmrFMMUXITTqXw2fE+ZXKEUjm8H7lYF3zn29/j2bMPePPqinEI7Ld76rLGGifOrt2OlCJFWcoGcJIxElLC6AJtBIVmncEVjmN7q/cTZVnIZnoWc6IP+F7yEs4vz3n4+DHvPdxwViniIAHG2tRYnQgYCdWbAn3IVJvHNJvHjNNEqnpWj96nbhZU9YLVei3C7GLBYb9lN7bgO0LSvH1zIOLocsEPf/5btGt48vgRjy42PNw0tN0tvt8Sx4F+19HvAxZYL5eyzvWRYZw4tAOrVaSK0I+DtNEqJWitoccWluAH/NQzmUh7yCSl6boD49DNzmY9d5eIM19rQaBYq8V1Z5FgTp1FFE8TSlmUisTUiyvSCXrADLIGrpbV3GURyUyC/SNgrHTmKSsu9ZQTUUUwmaRlv6K1oqgKXOEYJw9EVsuas/WS5bKgKEDhmaZASCPdcGCcRoZxJJAIKjP2Pd5LRofSkh2AEkyY1gqfE2XhqKsFT58858HlY4qiYbnY4JwcBPPcgfL7vC7PC/78X/zvWC7OuR1+xYu7kd14RX9IjEPi2ZMlVh9YnTnKB2dkn/jJ9uX9dlTdG1tgDku3sp9SQnGSgrgzuCIRfZy9FYZxgHHyFJW4v2KQ+UspzdVVz8NHDYuliPyHQ0sImapw5KwoSwg+YI1l345UNVK0nzngm2VJN070aaKuNUFpooqCUqysrEde5tMYNdvtgbONo+0HNuc1PkxMXgwVRWnmgnRCEdjvEg8engEH2nZAoThbXnB+cUY3jnzwwQpnHTGMWAzPHj+l2x34wc++4I+//X0JMJ88zhpx0/pBOjeMxlUNMcieevITlbFzoHUmpQlUxvuelCe0MozdSBgNhbXoOaRZMlsifhrxXopVRVWh4sTQ7miniQcXlygy4zjgnGVoOxTiPI4RlNPEIUhh2hrilLm9fgUpsjm7wNWanBwk6dICmWvVvMcnR2HlKiUBwHMreU734zzniHOO9WqNdSJgrDcLfv3Ln7JYLrm6esPbqzfc7Vuub29ZrVdMQ4/3gWaxIEbPcrHg6ePHs3Mwc3X1hrPNGdY6+qGX+YaRppHwcGLkbHNBDInSFlwdDjTrzNl6wdB3nJ1dslmfc3uzw7mCDz/8mJu7L2mHPUp5VquGt1dvSCHw6sUrum6LMgUpeZ49e0KM53jvWS7XKKXZti1Kez786AO22x1XNweePpNA9LMLx9kDmQtTknXq/GzJoX3LkycPaIctPnx9J/rjx8+4uXnLdncteLQKykWJ955piNR2ic+ZFA19N0IEnTKVKchJxO3m4pL333su81NhMFdXVK7CoBiGUc5vrpzxAQ+43d7gc6BpKqLu6Nqew9CTdhIM3u52RD+RQkc37nB2xRgD/dRjVUnfa4ytWK9XZCYpxBWWsllgtSLHgarQaFXIvsgoogpc3V5zu70moggp0AaPSoZSaZqqodZSmN3d7Ml4+Xs+cn62Bl3QDyPbux3d/o5l7UAZ+r2n3UecCvhXPZuwYKUabt6+JYQ9Te1YeSdh6zHy6vPPuX35ktuXL1mv1vM5NxAmz2675+btDavFhmcPH3J9dUNZNvRRMez2XDy6YN/1/OgnP+N7f+/PSFnz9OlTvvjiC/aHlqQ0tnBcLpZkpcim4Dvf+2N+8tOfURQN/+RP/5S//MFfEsNEXdesL89o6oa72xu+8eFHIjLv9tze3TGNI9YY2sMdZbng6dNnLM8ek02m61t2d1tu7racn59zdvmQH//or/mbv/z3VFaxXC748P33eXx5CSmyvbujqBwhel6/eMEXn38BKrNeLygXFdoZYk5oY1HGMkwDd7st3TSwXC/xVaQsCuw+kAZPbANkgz0v8GMPy4ZiWaGVZWhbFCI65hwYx5Z2kC60xhUEFanrBcEnWn+HsZrNeoPSiqvXbxi7W4wyrDfnoBzZGJblEls4xtBRVSU3t3dcXD5geXGJqWvawzVqv8dkTUkhgbrZEKZAO/YMBGKEMSmGpCAGqjPHoqkYugNjHDn0HWhNUy8lB8Rq1nWN/T07wI+CJPyuKH6PpLiXV+PckHr8yolt/s7fOYrYGkHXzHGM95ra/PWTPHh0vStOIvrJZMh9/pFS9z9b5KzZEZrTSSi+d7fffyirjvqGzO3Hr8h7iEAaQsRPHdr2pGRIEXCgtaGq6rnozSknwzlLCJFxHFEz5jNGCXLXWkKd/RTm30k+Ztu2lGXFMAzUdUNKif1+x/vvv884CRLq6ePH9IeWR48eUi8W8OaK73/3U5yx6E8+5hvvPyNs93zxm9/Srza483NWjx9RLRoRcZU6/cxj1/QxM9Ao/Tv3Wen5usyEBDVjfU738mhGe9ewehwPR+LC8WbkTFaiffgwcXd3y9CLqeW9997j5z/5Oev1GSoe8SWZIzv++E8I4TR+xnGk73uMMad/Hznox3F5xKsef58cRacyOmCKeUwri9Y1f/WDn/Le+4/QLlBVzGjSdPpN83G8Kw1G8NWir0rGYpoDQMXQJeY0uVLyHjqLdma0YFok2yydEMHGzDqdyveX+Gi+OxaVZvd6zO8K7up3nsn/3OvvLKL/919qvrkfeLLwLLSEge0Okb6VwNEpKA7T/HhrkeNGr5iiQmVhV8ckLto5M0tERA/TLKLnOdn83VaBU8iCSqRsIMr3hnh8frM4A+cqSgjp9HPiHDB6CjNUamY1G2EOOUtKmilEQsqMQVzrolf8rbSnLAM/Jk6CbIry2SX9ff59ghQFpLJj7vPg8jv2eY6JySLaCytZkXKmbio2F89oFgXJ6tPDqNAYVWG1vK8cQgNBiRiUED5RN3T0wwDKMfqBcRy5vr7l+nZLylDXNYtqjTHC4tVapjxjLFElTJaW07JYkijIxpFthbGG9x+sqEvHbi9i8JHdaI1Ug8ZpnBnjx5rafyycvzvr55woC83L1y/56stHVIsHFMmyWDX4cTwFsk3jiN+9ZNq/Ivk91uSZIiluVGFiz++t7ge+cMDma81c7UVh9RzilBLBe4L3+CnQd4Gb28ir60g3eKxOaOUJQYlAFAIxzo/vcVzNgnAMYa4cSvheCJFxEqH/eOvn5UjCVxP4uTUhn6rASjjvIaMrsHNis3Rs5FM3Q0wwTjD4TEiKU0Hz93j1vgOVOHJpU0oSFIm0hMrvHGcxMJ+47xKCJddVCkgygQnHO6N0xBSauq6p6wUxJxGGvSdFsLj5/SURPR3bvrQgpPqh5dBKsF/VFKCh7Q7c7a5IBIapZbu/xZhyRvGMTH5g198wjiPWVFRuIa5oJbzSGCQ05HC4ox929MMB78d5ETG4sqWsDGebJa4oKW2FMzWaRAwjPg4wO6ZTimiVqKuC/W5LXVp2ux29gRBGHj1+wM3NNZhM3RRsKmH3Xd/c4ArB5Uh4k2wQFouKR48vOb9YkVKakT4guAJOLuljFT7ERFKZ7XbH2dmG1arB2UIqyVk6XfRcXbXOCh9aO0m9zxqtHU3VyJyUMvv9nsViKe22Ws8BgbJJSTlRFCVaJbTK2Jl3LigfKw6SojxhfYyxWCcsN20SbTtyd3fHL37xS371q18TponlUsIr28Oerj3IXJ7FbXwU0f82T5+kUdmSoyYFRVKKZGXeQs3iagoM08jdbss4FXTdQTiIrpAOmyR4FpXFXRHm3zFjMCagVZRw5Cx9GLIXnMtyKc6hnIngJ7yR4oTKRwdyRjqcjm69iWEYT84Kc6qMH/+ZRSw1uy6C/IPVaDc/e/Hri+hG63mz7gkhSoK9Lqnrmr6LqEFhtNzDNAc+ppTm9nQpJIQY0FlTlkeHwkQcpSW2KAqyVqcNWcoZZeUeHu/du62eKaXTtTh+/ehyyDnP84GEAo7j+M5GRm5YCJ5pGvEx4P2EUpoYJ44Fi3fbQiVssJR7EWCxXPLexx/z3T/4Q548fso0THRdT7fvmUbP+eacTJaAYZUxzuIKyVIYR09IsrlXhYSShhglINAYUoKicCgF09RTlg5NxhpNtobSaZbLBU+ePGW1WrEoDWFq5bNiyUrGui0N1UJQTMV771O6BsX9pvd4YJmmkaYqKZxlHIf5QG3p7wa6yaNcRTUlrg4d2zZwvXvFYYxc3275q/0Vw+EFKR1AlXgvz+v52YpquWbKEQqN6gzj3DFztra4SoLa4+yYijHgkMDgsjIolWjbLVlZ/CTucGuk0GCzJsUjxkha7mU9SThrqeZg1hASMfRM07z2+e40xyhtUNqSUdS1w5gg728TOUSSjhRlgTMOnEEZOYxMgyFaYTWGIHgmbQqKyqIMDHjq2rJclCgVGMYDwffSIZEz/diy3e/xwZO1ws97h5TmNmPAezF6iPtHOsisgXq9pqoKjBUhevIdL18dMNrR9yPX11df+9kG+Mf/8E/5x//wv8G6Zwz5Nf/3f7vj9jCyu3lDHjPDbcXT9xcoPZG9Y+w6jLaMY2RuED92HiOImnliVHN+jFJgCsIgzOFsR1xhCJNhGicwwsBfrh3WSiGtcDU5ackEsGIUCSHTddLGfXOzJ/komIrKUjiF95G6tlgl8+2iKliuaq7v7qibitv9wMV5w/n5krIqePXqlaDtBiQ8KwuS0DkR6bVxLJYNi4Wi7e8wpsZPsFzWPH4MKM9yueH6+oZhiMRFYrluGG892spcFX2kqgruru8o1zX/9E//CZtyTalLLJrgpftwGlrIER8CRWG4uX7LarWiLEv2h5YiBqwxc2fGgNIJaw3toSV7xfNnTwGEA508RWlJSTF0d8QwcXdzJU5zpVg2Dfuxp28PaFvz4ssvefjoAWM/ME3Cu3dFgyqNIMe8hLepOHK2WXF3/QY/7CnqRtB1cSRHAxhOoVxKSXCfkvObYIrEfZiix1qH0uJ4D36iaBbEGOi7nrvta7SS+epw2LE/7Igpcei25DSilWKYRtbnS0KaqBeXvHr5gtKVoBWuKClKy3K54u3VW5w1FGWJdJrANHqur+4obYWtanRdMiRPSp6ikJC4z794yfvvPef66pYvv3zBcuUIPnJ+vubN25fkXLBaNHT7iYvzJaaAslxS1/Xszpt4+eKVCDZeUVSacep4+uyCyC1lmagax3b3GlTLfj/x6OGHPHn8AV+++kKKyCmysBX99PWd6KvVgra9OxkTUkqSIWENbbsnh0xVlFhbyJ7QKErrKJrixAS2VjHFSFWVrM82FMsFdVlx/ebt3N0eKCtH1ZTo8RhkHqRTAo8rIOO52+6JYaLfd8TQg/bowmJLhyex8z0qjmy3ovZpnVksaoaxoyprlDK8fX1FVTRUVcHl2QX7w5amKamcZbvbMgyexWpDt9ujyBSFRudMIuBz4O6wx9+ObDYrLi4uMLkn9ArlMrv9Xsa+rdjetSgT2R88wwjB7zGjoS5LdocDUxq4fHBG6ieKqkYlxWLdoFE8vLhg06zpti1Ga3768x/Tdi1V2fDRR9/i4YMnbPctb7cD173nEHZ8/M2PuLrZ0qyW/Pk/+5cY6zDast3eUpYFT54+RVtL3w9MM4t+sdrQtj1/9N1v8ur1S6rHj/n0k/f52c9/xvXVC/rumsViwdnZOddvXzGMI599/iVkWC8bHl4+plks6PuJVy9fcLsf2XY7Hjy85Pnz57ii5De/+S0+Zr73/T9he3vD9vot2lpevX5DmCYenp/hrKLb9mDAOs13v/cJRVVirOPm9o6b7ZapH8VipQxFWXGxspShgyLRDS0Hf2B9tsH4StbMIcGo0S6xbXco66jLAqUtQxjxs/lP5Uh//RaTPYuywGhDJNAOPcpAtayoFzVKJaqmpLCG3e0OH0amGFHZ0pwt0AbOHz3COE0/9FycXYh5MiZ8CGLiTJm77VbQu1YLni4kQppwthZ8Vka6942cWVP0tF1H17UUriDkRGEdq6rmsqxYVfXvtXa/iz+Be3Plf2q3b9Q78/M7f/fotv6d983355WTkgwItve41s+C6Cn2c+7aP1bX81Fwn5ULrU+nlYwYjcQp/LsFAPmO4+c7drYd9xjzHh3BumDkayl6/DRiXYPSomVVVT0L5GLsCTESguy5pklCp+umkJwhDa6wp2vpZ8f1MAwcDq3sm7W4rp8/v+DVyxdcXJxhjKYfOtbrlRh6cuTs/AytDau65oMnzxi6nkpbNnXD6zdXlNYxTROLQgpPSouWlsknEyX5WGSWZ0odCwla5u5717noOsbc2z6ZKQpHHM/xnHh0nJ+udWbWauJs2gI/ea6urzjsttJtd3XF2dkZ11c31MulmDumeHKUH4s4YTZAHYXyw+HAcrnk6FQ/GqCOf+/dMWu0IaHlWnjJNLJYpilhTM3V1YG2jdTLjJ+NujLmji5w0V1E7J5NsEqJgfWYe4not0khmS6z6VSwLKBSwmjmbM44G+7SbNQM8M44Po7QudTEfXafFJTe/Y7/v4ro//Ot4ssWvtF4HjtPozLDkJkGhc6KkDL9KNUAFMSsab3gXAzi8g6zcK7FfEdWMwJmFsVjFsdwjjMvmaOYnoT5HTU+ZKYRpkmQMVojgo+ZXb7vCPApI5NnlgFrrcI4B9ainEV7N7dUBFJSc7VLzYiVWQSeAzrFET63xszvPXh5FKYoYnGI938eZkY5ORNnX1zKmZhnYniWooJPwmCPSZzuDx9f0Fw8pyg1SWV89GQVQBVkXaBMhFnEyel4neT9xnEUUcd7bOHYH+7ou5Hr67d4n2iWSzaX3yamgNJQWou10vqi5hBJ0jyppSyM+JjIuiBmTb1ouLzY8OZ6zxGVgpGgrpil3VvluWrKuw//XKHL910GAKUzNCby8UVBE3d0t78VUSE9pa9LrJUKUb99xc3f/L/oXvyUPB0oC0uYIokgLbDH0l+6D6EELUz0uSiSjjK1mhOCUyKGTJgSQz/RDzPr34t7SEIEZTyc2pyiBmWxOs6LgjjAwzTRd+Ps0q3QVpFiZJzSzGxPEjKWIz6Cj/LgxyyFBD0L8tLAoXBGBAZj5Lr56E/FnTR3VUxBMc64mELD74tzuT1czTxuRVFKwvTucGAcPFVVz1VeyDlSViUOcZuO00S3H+m7AT96cVxOnuRlAot5YLmsWJ+d4YqSyYsjuW27GcshztZpmsRVaJidp4GkAj5OhOjFrTdjxlCZ0Q+kHNi3O6qXX+JsiZ/CSfxSTsT6rp2Y+ojBvBP+NtF3PePUYZ2iqhxFUc0MeM+UIspWaL2iLCpxomuDypnJj4TUy+SspP1HAVVp6Ky0aRstOJzlUrhi6MRiVVMvGoqi4m67Y7GoZs69msMUI+XC8vDRA87PViwWtSxaJjEMAyFMuMLMzrCMMQ4w+GlAGyuYrJmDN00TZWGlCGA168cryrqZO0+kA0cbS0KhjBUON4qikNb7tj2wXm8oXUE/hjmgNc0utYCzTgR6Lwtvs1jM1ehjGNmxi0h+lsIQU2C5XPH0yTM2mw3ee3bbW4zWnG3WrFYLcg4c9hMpBnISIf4orB4XegmvPHL1DByLBHM12hqLVjBNMjcM00iIEzkFSufIuSIhB02VRLj13jN5T0oBpSynxPLjnI+Me3ImRz875aWKq/IspGuDdg6tpZsEZubbHGbj/XSq3N8XBY6ZBjJ/6dm9HmNimiJVYaXDKSZB+nzdV5Z7k5UUwIB5c5JnUUQCDrUZIUtB8F70nufQJHkPwh8P4ow/Fa6DLOoIViymiB88Va1YLBaCp/GTzKIK0IqQImGMhBRn9n/GR3E7iChryEkz9JOIa03DaiWulLZr6fr2ndZC5kLP3B0zB+QFH0hGURbiOggh8t6z9/iH3//7fPDBxwB0hwN+HOm6jtVyRVkV7A97jDOUqjy1OU4hkPCQDZ5MUTZoY4nTJPiWmX+ec0RppNuALO12M9uzqguWTYWzgvyYpoiaJgwKY0sSMrejLIu1o7QJpzXGLFAYQgzzocKjiVQpoFJg6A/07YHDdkvbdmhbMg0BS8lXb/a83HqWl+9zG655vQv8+vNf84u/+QsuVrCooCobObyYkrrJfPCNb7G82PDXP/kptiqJGcaxp58Ctiwxxp4Kjj4EdDASNJ6EMzh5jzaFBNdmEdNy8BirKayGZOeCoMJax7On7+N95Pb2lmmaOBw6YpgYZu6hRjomcpzm1mrhWE5jjzHS/ptjQuuItQmjI9Yl0JGqsdRVQQwlfT+SsqcqpcCljAQTOSdBmuJ6HxmnkbswQJZA3LqpqRc1b7c9PgW6biTEJCKj1pSldN+knBnHHlc4Uowzw9kz+p4xDPRTSzd0TFMUJMPkySkLmuD3eP35f/m/Z+otbX/L691vefHill/85Ib9nYWc+Mv/6RU//7Hl0+8+4OJizctfvyL2BqM02kgGSkrSUdmR2JSWnD1FoedOFAlAkzAxxTDKGNfGzmiQhNNWMHXxmJ/hqWtz6h6y1lLVhhAiwxCEvR4kn6YsK7p+knBbDVlr+tFTLyqu3txiS0VTNdy9uaUtBqZ+oCzFOZCT/LwQE8oYEokpBBgyOSnpctps6McdIQQKW+N9oK4Lrq5vpNPVWKZRgqrrsmC9WRODzItWO4Z2JE+Z54+eo5OGkCUHY3ZeT+NAjOKCKquKsd0zjT1eGTk0K0V/2DP5kaapsFoxDJ67dkv0gfefPGZ3c01ZFZKHoACkq6quC25urrj96jXDGHjv+fsYZVmtzmgPBxauEpxcJbi8vve07YGHzYpEwhqN94rgM1W1xA89dbPgdnuLKSualSHFER3dvEeXQ3FIR4SYZuh6urZltVpJVzCZECcOh708lylh9bGjUNbnDOx2W0IILJc1u66XtUdF6qYiKM/V9pouTGhnGA8DsQxUyxpjYbu7paoKzs5WWGcIYWLfbnnw4JLFsuGTjz9hYOLHv/0ZaE1hHYtFSdt2fPbZb9jvRqpSkFDDEGm7kYuHNSFOhDigtScEx9nmnNubW7Se6LpWuoZi4uxsw5MnT9nudmTjOLRzLkJpqGrDm6srnuhSjDZa0/cTd7dbXrz895xfLtHOsb3teP/Rkmnbfu1neww9zarBFI8oqhIfI1VTURQlh7sdoKirmpgTzbJBGUtZ15h5j7ZaLtg0Nfuup59GtJN94aKuGcaWeiUmh9VZQ1Gaee2X4OcUM+MoRWqDQ6nAOLWnoPKiMSzWa6xbEpVmCi1D3zFNmbIqyEgBapwEs7rbdvR3E6tmYrGseHB5zl0IOCfmB2MNm7MHKG0wpsfqSDl3V17vrpimUfJWfOTCNqyWD7hzI+12ZEx7dt2O0mqmIUrnRB4JQaExqBCJY2Q73GJMprmwvLi9gilxUa1ZP3zIsZPtECJvfvNbCmMZ25Z23/LHf/zHfPTxtxg83G07DpMiVysuzioe1RVt3/GNb36bJ8+ecfnw8bz3Czx4eElVluz3B+m6aKq5Q28ijHvOVjXtYcdHz85IeWBSPZ9+8z0g0+/v+OrFSzoVUbbk1etrfvyTn/HJN78FSvPo0UM26zXd4TWPLi9YbxStP2d/2PKbX/+S8/NLvvOdT/nlL37FNA58+M1PedMsTlkFw+TZty1PnjzkbHkBGg7tllevvuLs/BylLGTH5dljLs8kELEdBnb9jpADTVPS5gN7vyVYT24WFNoQbiYO1zfUNwMPFudgNFNK5G7EjyO7/YGmqSirkkjkbr9jUWj81OGnQFU1HLqBRbMkkTh0ewS1CM+eP0MrzaFrsWVBWWuwnsH3LKkgGR4/eADanYxvWluev/cBiczt7Y103cbE0A3kLJqNdQ7vA4vVkqVaoAy0fStCv5bsgGn0OF3itCX6QLe7Y23/YwH7f80r3Af8nc45R/wfcLQ0z0rKvSAtX3pX5DuK8Pfvhcqn/z4KtrLln8X3OcQvJvWO2TS+8zny6X0UzEztdwyL4kyUd0+/a5R51xGfETa1movAdkZWqqxm/S0Tg8fnA+uimTFcmaqqZ0OPnAsmPxLCiHWWaRIW+pHhfTSmaq0Zx/FkypHO556mqenaXora+x1lVbI5W9H37WwAqZj8RL1YUBUVaQp8/Pw9auto0fS7Az/9wV+xrBvOLy/ph4Fy1UBVoN07GW9ZdEalRWfK6qhJ5fnmyGc9ieP5eO+OaNLTLZfr9ju6zn9a4xHUmnRkd13Lzc0129tb0eO0IC4Xy4a2b3FFSdM07Pd7+r4/8dGPKMpjAGnf9ycz1VEwP17TEALHMNJjweCITst4MQXNRqahzxyKzGdfvOV7339Gpp/xrr/7ewnH33LPM5fraYwhKzt3Ymtyuhe578f2rM9mSErNpAcxdoh5ai5iq6OIfiwOzQiYJGcZJe2kJ7PrkWrx/+v1dxbRDx4+84lDb3jPwkMTSUHhvXAVFRCSpjDiZIkZpiTCuEJE9ZjFgZuQQ3nKnDAqo1fCj87H5zKSUiDMiBeMfO80pZNzPc2bvvsEYnHspZmVGxPEJF9TWsJAjRUXutJWKmtGLlTCSIBWjoxJ4fOsC+TTJeUoCOdZuB29OM/HSb7NJ+ij4pAUQ85MgEeRkhZRcBZzRX8QXEZKipCzXCdjeP7RE8rFOdZEptBJ2FDOYs3FgRKGck6BlL18CRERUhQ0jVKGvm9pDx3buwP7XUvTNBTWMLYHyqqiLmoKK2gCqy1ZaTkMGctqvcSWBc5VZKOJboPS11yeLWkKS1U7xsGL41xL6JDK4sJWCmKIaC0Mt4cX5zhnWSwqtDX85Q9+RIiRylmePWh4dl7w9vqaZr1i/TyiVaYuG6yWqlCYel7/8L/j6qf/I8Pbr4jTgC2lmimIExEv0jviPBxFc+4DRefF437OUvgp06vAvhUnutaKpk4sq0w/5jns41hlNXN7iIwLcXfKz/M+0ncTPohDy5YWHwLjJK3/MUckzFLCTEM88tjnRS5LwULFTGUNdaGpyyTJ7NyPaaVAG5kEfNSM89hRaS5o/B6vz178lrJy1HXBer0ixsjt4ZrrqzuausbamfGsNavVksWixhhD1/dsDz2313fc3txwuDsQRi9O9JhYNAXn5xtev36DcQXj5LndbjnsW1CGFI2EAgZPmgWNxaJBGxjThLKwWDQsVw05JOHCqTwjZWC733Fzt0VrOaCToWkaSlXIoja3RPc+YmfLfoyzs1YbnJOujpSQQzwKVWS0NbiiwrmKwlXCDY2enEdi7EXAUzKXhCDjonCCoohB3yOWbKasHWV0GAyLeknX7akrh7EF+31LVRbkQtGsGlbLFTlDUZTkWSB+8/pa2kqdQidOrWGySGickSDXEAKHQ0tdCS+1KErqqqCuK6wtJUzUHANgK6yVP3euPC2kVVXOYq9sWsapR6lEURaEaSBrI8GK0yihxcbOP1vEaIx0LsQYcRzFdItJ0Hc7Xrx8zc31HVVdoVgTvKcfOmIIBD9hrTk9s0eRVERbQa5Ya4kCa5X2ruMiEhUk4d5LgTGRSPRTpC4sVsuGri7KmR0LwXumcWSaJMxMggjl9bdDe6TrRAq7R8RJCmFOmIekA0EJ/1tpNWPf8jxPHQMxj/x/PW9KZp/H7Aw5/sQ4d6PEmFFBZrMQvr6IftyAaC3idPASbmn0zIRLinEYMVY2L3VV4Zw7Mc5BYY0EkmplZkegPDtqbstKyDxpZlvusTZw3IjFGGdnqz7d0+NmbBzH+XPKhi0mKTqqDN5nfJhwRcnSFvPGnxPPThv5vcStckRvyaY+Ril+eB8xxmK05ec//TnLYoPCsFqe03Uj3aGdHcSJ27tbQX2UjnY8kLKimNnN3svnK4oarS2uKMiI+N933YzxmAOnjcImDcqgUOTgaQ+dMOG14vLhI7AGWzgKJbkm0csYscZRlopCRxlzypHm4nskSXdTDBgCeep4+dVnXL38kqs3L0hTz3q1IOmSq13Pr796xc9fbFGLCw5TpOsO3Lz5isPNlueXBefrguUiYm0HdJh9iyoLlucXPH36mC9fvsEnWCyWoIWFGmPClSWuKklZMU4Tkx9QOeCsrM3aCH4uzy24OYtgDsV86JGiXFlWNM2Cvh9Oc1BZFnOIrYytUtt5wxxIMc17uZIujBgjuCKtDIUrZH6IHj/NCIvxwIRFMVHMfO08owFD8GiVJBDZZJLyDONBujCMwVknmLMxUq1KtJPxNIVudgkljHU4pyjcPC96zaKR4ktVlrMOmdm1O6YkXVt+igzDSNu1aKUoyt+PiX513fHi6he8uPo1b/efcXd7zdRlalvLHkMlhiHwg3//Cp3fCsrJJ9lHFXNnzLwPmcZM1yYuHzTEPJzmjOgNQ+8pC8EJWqvF4epl7TP26Iq7P6ijJCiuLK0UlqymqU9NuGgNhbOnw2OMkiEQc+DsYkUiUFeWunEs64ba3ZIDKAfTGPBjEuEvRQ69BJQPg2a1lpCxomzIaPphJMQgWU55QqmAsTLWUspYU0i3Y4pM+y3ONbii5HA3MO17SqtZ1UtsttS2InnBA93uD9Rzd1wKgWa9pO9bCenUin17wDhLGEU8coVhv7vD2oK7mx2fffaC7e0Nry8/w1rNg0cPWCwXDNPA5YNLDt2Bm6srrq7vWG8e8avf/Iyuy3z3D76Nzp7rq9d88kmDUoqb6ysWyxWKjDPS6hv9yPD/Ze3PeizL0jQ97FnTHs9gg5ubu0dkRmTkXJXVVV3VYA8iiBYFQpCgC/4s/QrdCJAAXRItSCBFCCAostljVdaUlXOEhw8225n2tCZdfPuYeyabUjGTx+FwdzM3s3P2WXsN7/d+zzv0tE3Ltuso2orl6lwECj+y29xjXYnOBucqFFL0CNPIMHnh/Wop+K1WSwon1ykEKQgLu1aYgzFlwjDhfWLRLtk83HJ1dUXXbWZkmPDdXWloljX3/ZbNfkt0Gu5v0ZMgzxYnC/b7HTFG/vYntyhlODs9p2laXKnYHx5wpbh8f/C9P+Ch79j5nu34yHb3QAyRsqy5DxK8+OLyU777nS949/5rvn79Nc1Ko3SkLEtyTiyXa85OzvnLn/xrTk9XeD/RNAv2+wNv397w4uULnj9/ye6XvyZnuL+/xXtP1wVubzccth3LtuFbn11yc/tA2xq6YcsPvvV9TpY/4uc/+bccBv8739s+DjRtKW7G2cWorYj5bVtTmAKlDZvdlqIqaRYttnAoralLhy1L2qZiu3nkcb+jWVYQNCaALmQvWZaFZN+kAEoycMaxZxwnOedlRVWXNE1J2klOVVW0NMsSVzYoU0hBlYmkBpbrBXXl5vOpdK/2/YifonTCKXh4fOR8e8I0TcSY6LpezmhaiZCP7O3EgCAmhxA8U4gYbVksz0AVlGVLt79n8AN+GjC5IPiMtpqcNSRoqoZ+v5NQPCUFH1SJbR1YuB0OTKViuV6TQ2QTI/WLNTZrujjwT/7jf8YnLz/Fli3TYUKXiewmKDOmKri6u+Kb3/yMjKLrBrTSlKWcf9PkScHTVG7ewyXplCkr+qGne3xHHDtu9xsRakPGJ812t+Ns1bJqLEZHUIl/+Mc/4p/8039K2y4pjaKpMne3X3PY3FHaJUZplosFL86XeJ942O7567/8MT/44R8Sw8Tu4rmMg/trhhRpq5LdoaP/8iuqpuT88hmLdsXZM8Vms6VdtphCzin9IAWMKU6YyrB/3HN1847l5YLLV8+Y8iDzw74nmoirlCB2syCXjC3p7zb0/YDSVrpBraF0BaUfUCqx3W7lrIvh0HegLJvdQQqwpSOMntu7B5rFim13IBPpxgPeSLHv7bvXlKaiKmumECnaBh8mWRsOkbNn51w8u2QYe+qy4M2bN2z2OwCUMmQUi6VkSSkdOex2hMmjjWOcJvZjR1s0nK/OiT5wv98SPzoz/C6POONgjthfEUQ/iNAfxPSjUPgBf/HBnAjzl6Py0fI3O3TnLz5Kg08BoChSmrWvNKPcZlOfGELSTPTMs61dnkD6yPhzzKM6fl3Os8ntycE7v4I8d6Mm+Tpj3YyfVWJsm9WLnBMxeZyt0cphZm54SlG6U4NkIKrIjPiCw+Nu/rnq6bwIgqEcRzl7NHOX1DCOLIuCcZw4OTmdheMJYalLzpwrS7SylKXlrFnS3T9SBcU0BJZFzX//3/1LvvuD7/Pt732XZr0iGIUtnJgy572uSQ6LGAdilnNNymLoSVGDVjPa/BiWfDxtfsSiV/OfTw85w33spD6++dpoUvSSD5kjd3e3gnMZBtq25fHukb4fKOuG7XbL+bNnTx1Xx/OXc27umB5wzj2d/46FiWOx5uksB0/n2ZSTaIiAVknQpOQZOy0M+l//6h1/9KffxlgxJevfkquUOvLI5w4JBVFpCQZNGpVFg9VK3Oo5fXS/qCPTfL4soqbP1Ygnyf3D31Wa9ec0m+TEAEH+cC6fr7Dsb/7/PP7eu/ccFSEpupx5NAgDO4qoLW4hud+KOX1TKzW7rOXJ+6QICSZZg/EBlBUcXwgwTPI75PkA6uCIlcjzS0ox431inMAHKT3k/JFYEfzMNxfcQUpa3NQo4WAbsEqmFEkLnicwpQjJ0PuAT/rJRa2VOhrsOHKkpOoiB4AQxFnfzxibrYf7ZLiOmgFFmTIqgj8myc6O/RizIG9SJmk1H/alLf7Z+Yt5YAZCnIU4JQJ/yhZUCdqTwjRPbsJcCmEUHvCMq9jtdtzfbTnshWFalAVlUVAWdg7AmhONs1S8AJyzLOsFzkprtzJmri5ZsDXrRcvnry75y599JcKPzIME7yFHOSgbjbKGf/BHf8j/7j/9j/nDH/wBV3fX/O1f/A3vbm948fyMh5tb6kJxUms+vWjJGSYKinbNyfqEw+aG149v+PSzz8AYuqvXjPc3jP2epAImF+LG9nOBZXaY/7bj/YOAfuSiH6+XCDTRJ8aUGfsoAltjyS3s94l+8ELVzVIBlgLNUZCSG1CCEkVUPwon0m4lQRbTzNpXxzH3VA2eL9x8c+cs02dCxlthNNXMJZWFZQ7iUgprZByNEaaY8XL7kX6P4EGAr999yXIl4jgmorWhGw/cPd6wO9i59T5inSWpc7Rdo42mmwb2U0/nO7qpYzfumLqRYS+Ht5NiyTB4jM9M04HH7Y7HzUYEqKxJ2TBN4nR2hbRGKy1jMucswZVFiStLQvT4uapYlRXGaCI9PsrXF1WBNYKHscZhVUF2hmgVOoV5c5IxqsDYBogYKxV3aRWTME2jNNYVFK4SJIm1WDMHKqpITMMchGjItpjnP01ZWAk/bCpGP5FyYgxycCwmQ2FLFAnnNI0qCfNm4GS9JqNQ1rDb7Xl2cYrWlqJQHDMT2sWCsR8Jw0RM8ck9qJW4bmXDEKhKx3LRUJYVaeaBCe9ScAKCxtIYW+CKAmvFhX90X1dVNQeNxLnzQD4umQfiNFZKPbl+pX2NGWmhP3KpHp3OCjdvos/ONJ+8+oRnF8/4+s2X0k6mFX4apegy3yPazEWynD/w8vOHeyenSAoeCjuvEYKc8V6CD1OY8HEi6UwmolWBLQspoigIo3Q95CgBmpMfxbFiedosyBIxo5Zymu/DKGJnCEQvHRIpBWk3zlHayHIUV+K8gKdZsJOWu2OgTOSYu5HJT3tU8pEDn4XpNiPJBMPyux/Ej8VlYb2bWWA+7pw1Spu5g0kOmFpJOHDOc46IOm7AP+CackokrecuLZm3svrgSrFazUio4QOyhfwbrYAxxhkb454Ye4LvMeSkntzrwj1MjKMHJQUFax0ZpECJerq2x58fjgUxpUlRip5FUfLNF5/x6atPMFpzf3cnRflpZBwEUeD9yMnJmn4YOBz2FHWDftqYS9vlYrGkKEuKwqKMZgrSaSDhU4acIuPQyxMh4bSBQvITMpGYPD6OxFxQGSUotZBRBqyeQ5dJDD4QRy+OliRGAK01ZVEIQq8bOGwf6DZ3pHHH5ekKUzzjfrNjypn9OPH+/sCf/+RLrrY/YwhiRiBMNCaTVSCqCq8UVZGwNkM4cHv/nqASq/UFl1FxdfsoA0lrXFWjyejCSeFfzQcWL/gzg50PMCNqdi9DpijKGUcQ5/c+PN3bd3e39P3IdrulaRqaphZUTvhwKDoGG2ljxd2jEk1bEsM4u4YDrqlR6Jm9PVFkS46JvouMU4crrRRB5+B1lx3ZaIJPhORJ2VIWFmO1hJPrRLMUB8/t4y1TmsBAs6wlqDrJeNYGMhJ8VTcSslkUDuekW6asGnwIbLYPhCBzmrEaV0EIA+G30YH/Mx//p//r/5E//kc/4hC3vPn6NW/fvJd9Anus1cRsySkTPfhBobE47XEGxjij8bKMW2MzwxA4HDTtokDpRI6Ku9sDflKoHJ7Y9cYkQsoUpYikIN1Q4xBn55eXbICc6LpJ9oYK/BQpqxofwnyAFzHW2RkDpxOPux2ni5qL5xccdh3d/oBVhrGbKFpHUVmoBIUYYkSbuSNJG9rFAh96+n5k6A+cnZ/jypLusMepTFEV+DAwjhNl6ShcyatX5+yHG1LODGNE02Oy4He+88UXFK6ksiUazbJtiUNkcXbK/rClcI7BD/SHLdvdnrJuCDFzdnoijqgoIfXTNJBS4uuv3/L1V+/56U+/pN/tWP/Db1PXBU5LmPJ+t6NpKupaOvj2feT11+/xk+Kv/+bnvLh8xV/8+3/FelnRNjXr0xWh93Mxo2QaBqKfQAUqa+m2e3TWHHYD1ilcvWKtM5v7d3TdjhJHaqQPcsadP2G7YvBz2CxPrfTOWnJKmLKYz/sWqy273e5p/71aLvk6Rt6+e0dZC/9fcjIS/dTRTT132wd0Vcv+xlSg4PHxgZQyZVkwTSPOFQxDR7to6PsdIUzUTYurKsbRc3F6yZd/82N205aDv6Nta/res1jU1OWSnDNv3rxju9vPWSRibJmmEasK3l/d8O1vfZsf/eiH3N7e0sdRuthSpG0bcoKrd7fECIf9Ae08TVtTVyM5e3IuefdmS1uLyapqNOfPL/i7v/sZ3//OivOXn/Krt29/53vbuEzEk1WWrq2qwIwSPK2sRruCsiplr6MyRWGJyRMDRFtgC+l4DBmUtQx+YtgN7LpMDANKZ7L2dP1GDhLRCeZyziMKIc1dj7IPss4ydZH1yQXaRXb7HuscVVOxak6otKNyNSGOdP1hHjOBcYiUxYJyVaJLRd/1vLl+j7WGbvSkHBn9KKzwmPBjQKMZ+1G6JYsSZx1VASoZxjGw2e3xBPrQYwvF89UZy3rBL376S3wKlFXFoetYLDSLkwYbFMtyQTKRyXpUUVKVNWZlOHl5QbNsJZD30GOmiJ0i33r+fZ4vL/BZ8fj4yGbX8/76gcMY6MaJd796w6tPP+H0ZIXRjhfPL7g4O+F02dAddnNwuianIKHuMfB4d8/Xr1/T9Tt0mjg83EIYsM6xXJ/TT4k0jvy7f/OXnF9cUpQtLz/5DOKA0w05DJJtNPQsa8vyW99gGjUhKXyapNspZcrTFc+fXXB9f82nn77i8eGO588vOeweuL96x9B3rBctZWmJOhNubtjuelQ2DFFzd3XNGAeuH244jAeKqgDtaVcVzbrm2eqMLu4pyorz00sK53i43jH1B5qzBcWiBOPoDpNoDlFTYElGAgG32y3LZUtdlBz2G1LSxJQZfSLNbtaYEv2+5+HhnrZcMOwfqcoa5ypSnth3e3T2NPWCYduxqjIqwWHoqNTErusZxsTt7Q27Q8dquaAsClbPTqi/aHh7c83gA2EKJJUZpoGysVw8O+f65or+0HN6eo4tHM440iTuz8IVDKHD/B6hwSAiuhj7PmBaQKHibzm7UZiPdUB1VKTknHFUFI4mQQEEfwjxzCl/MAblucNWK1RWUmx6EhBnAT3P5k0ST9987j5Ux/3K7KaWbtXj3+dnlJ+AJSJ4IigOlSGSIZsP+1plsLZAW4cxiqapsbaB2XQ4+G7WN46dvSNGV09c76Ou4if/5Kw+dsgegz7HcSTFSNM0LJdLUML+1rPpR4R6hy0KdDaoKeGSIm47br9+S7fbY4uSH/7gh1AVjGRO2loQNIXFzsGUkgsTSNairUMpTUiRRCQqhU4alY6G3NlBf9SGyB+9n78toH8sCPP07yNfPOeE1mrO7DI83N8/CcxiTCl5fHykrGru7+9nw1z15Dg/ntFSks7poiienOlHEd0Y8xsO9ONZTjq45ec7pzDzOTolTV2tGH3m7ft7+iFwWteSnZY+6qBQMs6O8QJK59kIF+cOhtkIzYexd6ztHAs0cl5XZH0kU+inAv8H3FF6+jOp47hECEcfnftFVJ6v8W+r/f+Bx99bRFchY7N8QUIxKhHWx6Ag5SehY5oP5k6Dz+CjoEx0zgwBlFWopNBjJugMUTAoQ1AMQRGzxmqoHDitOcI0mB1qg08MQTb/WkFUmZQlSHMKmSkYpjiRsiIkjY+CWxH9e+btzPNDzkehIzN64aJn5JyZlLzWY0TlXHMTZqnNwqGehfQuavqcmQ6am6R528sNWET57bOWQFTmQNWUsVpwLyEmfFIkrSVBN3qmGDDay40xoyRiyBJumA05jwTVCU7CSHEhz5NMzIEpeva7jv2uRylNVVWURU1ZVRijKAqLdeIaPzr7jLYYV80MQAm2U8qgbEsyjtBpirLiH3zvM/6L//e/mrESM/4liXMgzgNysXB88e1z/vhH38YfNvzqJ3/Dv/2LvyDEwNhtOF05FoXjpJHR9PLlc370h9/l1cUrmsWa8PYNh/tr/OUZq8UZq9NXPJDISm5WciL4JEG0+ejMPTrSPwjZx0YaEc2Pld2M0yCFzEyOGasU2mbqWqOMxTkvDu/8Ebs+ZEEORZnIExGlM9oqXGmoFwVlspS1uCUFeyDToVIao+fgA33E+sjEeJws4+x+yzlJ4ScncX0mcbrHrDDHoNGkGDyMUTYbMQky6Pd5bA57soGQPUXjxEVnZI4apgHlZaK2wVAdCmyh0UYxjYExeKLKVM28sK0COkNVlJSmlDT0KbA/bNnve5ytRFCeq/9Wy+G6rBx1U0j7bpyIWuOKAmMLQBNSxoc04xJKbCEdFLYonvAh0hIkRSIArYQBbbUTpMXckqa0w1qHsZCyx09RwkhmrpqIwXOVPouDROs8V9ojPgyQMimV1FVL6cSNEzwYK9Xm0U8orXDOYQt7zEqnqSv2hx6FFhc6Iu6OQVyRzpUc9j1t284hnA1GWw5KsRtFTH1yLMyct2kSx9KirWnbRtArStrUntzRc1VVazf/nIqiqHCuIMbIOA5YK87Ecehxi5ayKBj6fnaTCq8+hkCOCauNJF+nYxUoE7zHWUthxR0v11A4e35mWYc5eC6EgMoJawy6quYU7ZKh6xjGgNL6aXMggdGCsshGz503Ho2bO4ZmRnGA6EdxzBQG6+bA2hDxUnVgHEemMVHVBvMRMuY3czCOvDpQ8yblKCrHOIvacUa8zOuTmuf0j4NEZfFkvnZZMiiUdGsdOzllJpir6Tk97ZWOm6v4UfX/d3koJNMhZi8ib5557aS5MGfISgneQSvGYSClMBePRBQ9btjyzNfPSVY0refNlTXzGJPuLnEnTx+cAvMYlHGpnnh8IQTW6zVVVbHZbJimiaauIcJmu31yJpfzpk4KF5lqHtfdMMzuNk1RmNkpL6/aGkddNU+oprqo+fTFK9aLlSC0QiB4zzAciDGx20fadsHkJ8Zp/LC513pGL8m9UdU1VVWRidIanI4dBoq6riSXYZCPTYMEUcp0JKGM3WHHYrVk1FkKh3OmyJyZQ46J0U8QAyplcSrHJMGuzmKMI04wPA6YFLg4WVGricIabrYd2/2ILVo8GdecUNQrhvtbAlaERhWJWkvoqNfoPjKGQFXORcL7G5RzmLKhrgrapmbbdYRpwlU1yhqauqFqa4apIyvp8DPGSNjbXOgKIcxOw0hdKz7m4Xs/PbmJvA9PuQtHx4i4g8QlKUUX2cS5siRGT8peeMH7iFJS2DFW2jO1ziingYA2EtYYs2K5rGgXlruHe0BRNwXLpnnaV2oNIXnyjIcoa+nombLDR48tNMM0sVi25CgiuppPkDHIz6pqmS8LUzNOPc4W+OhoF0tSytzc3HLoJ5bLBct1wzjCdrf5ne9tgNe3v+DL/8cvQWmCF0OGNRmlA07pp0NE1AldKAqTiH52qmkxmGQS6LmglzSHx0SaFG1TMAwjJ4sFuzSQ84hzSlxqRSbPY9d7cA5UVkQPygdqJ7i8srbEHNBGUVg9zxdgMng8mJKiKhnGAWsz/QCezGEYOH/WctpUbLd7qtoRtomhz7QrTQqRMGpSKBj6yGLhaNuC9+/vWS5FBB6nyPt317z8ZA2xZAwGZSLZJMrGYE3By2dfsHno0KkjpkGY00phUgbtqF3BarEi5cTkJ4IpUUqx3e1IRHw3MowHQcxhKChJ0ROGTJjbVP00sd3t+PLLr/j//Lf/kqo64er9I21dsDo74+RkxfnzFxhraBZrlsslGSjcgmfPPuWv8t/yd3/5d7z+6g3/r8OOT1+uKJTh/u4dTWOpqprbq/e8fPUpKYx0+weqVUtMkXHs0AqssxhTomyBsY5qnEjZoLSsgdrImDDWUTgnuRPez+cEK3tWI/tSm9wsrEjodsoBYyRbJEUJ28YYaetGY5SmsIblckFInsIYFk3D426D0k4E3H6DMon1es3D45aqknDmzeMd+80D1iaKuibR0DZLyhB53X9Nt9tz8uIMvQn0hwlMgTElL15+g1//8hcs6pZut+Ebn79kfW65ub3BR+lASDHw47/5C37ww8/54ovv85d/+deEMbBYNPgx8PrLt3z2xef0wx7o0Mow9JqiaJl8pF0u2D0+stlv6HtPszjhq19f8e5NYH/3V/zjf/pDXly8+J3vbevE6blYtIB0Di+WDSkFvMkyjytFUZfEGKgKi9biKswx0HUHlLHYsqJa1kyhoxu27LaPTL7n2bMTunHH3aNmLCNteYZzcub0PhKj7HlTzEyTpywrJhepmzWH7oFuN2KM5aR9xqIueAzSQT4Me/rhQE4z4mgYpWunzCQdsJWl8wOhC9TLJc4V9NOBmHpMNjhd0NYL8qEnYbCuIo0DCiiqmhgTt3e3PGxuOQxbLlennJ4saMuWpq3YPXZcXn6CsY7tboM2sLpccXn2HF1p+tQxdAOXq0uccWQnXTnFquL02QkPb94z9gf2k+cul1hd8LDZcnv3wN3DlikkNrs9n33+Cd/+zhcs24rPvvltPv/8W5ytFozdltI5EXEQB23fHbi7veHd269JIZJDz/b2Pf7wiO93OGc53N9Sn1wwdoIj2jw+8OxZgVaJYfcIKbBanZB1jdEjGsmFsdGikiKGnmWp8dYw+Mxh6vjWNz7h+uaes/WKuxvL+uSEu5srcckqhbYWM+eN9JOEfg95ZB83RDdx/u2WKiTW52uurq/YDI/s+kdC8izahugDm9sNlxdnkjdWF9iixdaOMYPfjWStOW9aaldwu7tjt98Q8Yx9J1lTKbJen3LYdxhXsnANKSm6oafve/rDlqFIPD9/yTgl2kUNzrIbd8K49wmGRJFHyUZRHu8z2/0jZf2M0/NnOK3p9h0Pwx3RB169fMnLl5/QjxObzY79fs8wDrCJXL54hrGCq4szVsJZK2vAMJCV4ZAnStX+Xmt3/Eg8f/rz6Bb/6Ld+0jBk2hWk5SysHvW//MFvm+ePfwgPF51L1EdBrOj8W7gKdbSjw3EP9kF4FAVcZT3/3Pmj89+V/Q85dvNTRt7x26osoqvWmoyczY7dymEWbJ0tZgyoo6oSh8NWCtGouRM5Mk0TzklxLUY1O+jzbwi/xliYOyUf7x/4/Fufs1wuZ/Fdum601pLthUJnMEnhMOzvH/Brjctw/fYdKoNrItXpmma15vknn4j5QCuMs7OxLD+97ugDXnmMc1hjZr0Hjlkz6ujJTR+MwPMVm1WhNKtV87WfP/vh77NJSM97rixny81mQ1mWs7u84P7+ntXJml/+4pc0zYJh8k/n3ePePOfM4XCYtU37hMM58tKP1/P4+Y/FdBmvs1hvwbk5nFxpclAoLFW9Ypq2PD4eOD094WhQfnpNaiaHxDCP7Vl3iwmlhJSRUkZnJQxkgMjTdVNK9qMxJNDpwxl61vy0XCQ+OPm1/L/ZBXw8k8P8c5/8qEde+v/vx99fREdjmF9IhilJUJCPWUT0I5NWa/pZOD8iXUgiSjuln4TrbDSljuLii5CRdmVlFNbOjJ2kCPFDJSymxHDEuUQwBQxAHw4UvmAYA4cxsZ8UKSsGrxkCDFEWixD5SMyeW/JTJEVxs6SUxQ2ipZopN3qaHcz6o8GvxAVgIFtFUJqboLjpM3des/eZSmWqoKmSYkozpydL68NxLks543MmZENWmrqpaRdLctazU0sc4eLsM+Q0CwS6xOiabDImRrxSaG1ROsgEObdpGC3MxqqqWCwkLFCY5QmjEsYys54tyjjcHF5jnWXyI9M4QoayzORiCbHnP/nHP6L9P/8Ltl0nbkkfPhojmbrSvLywDPe/4Ge/+DFnp5/Q7R6Y+g390HG2MFRGY+cq5sPG81kWZMfJ2TlFuSSeDSQfKIoK8ohyBqybK6GCsfA+EcKHSf+JWz7LUagj435+Zk/rR6a0mtIqlJ4d+FZhnMI6RYGmLI7hjR/uRwkoSbPwpVFKAtOcNeJw0xU6Q1G6ecIZSVnPBVw1O9aPopp6WppAxoRRGa3m8ASdZ8TDfA/MP9cYmbD6KbKfpED0IZT09xPRExU+Gbpx5H77wMl6TVIaXVR0+x6tFCFGCpXYjwPsLQpFdxjIk0FFabE3tUW3mbIy1I2jMAuSV1y/vyEQqZsap+aqfhToTmVqlFaUTSmYDwPDKCF1RemwxkmhKGRJgDYKUqJymqqoiakWJqoWDpdSUg2e4iQb9ewhS5to8oGkDFFFogVXKMCT4gQpYFTC6YxVmhQgxcjkOwmPVAZ0IYJwGsl5xJiIdg3GVWgqAsKGzLogqYh1FTH1FLYgBhHwtXFUFYxjoCzkgDpOHq0TZVWz33UYaynLBq3h7NmKYd9RkOk3e6YpkrImkAgKYg4UWrNoFqyWS+qmIhEp6gpl53BXPTvBskbhxHk4u8RFXE9MoydMHmuEl0vMDIdBhMWhx6DZbB/pdzuaqmK9XlFmxf7ugf2hQ9mCdrlGF+28Tohwb4zFakNdlTRtTVk5fJjQWmG1JY4Dycv1VxrQcWaPizArrk0nuAwCZDO3lM/YGDcztUcvnQZaClJtWUtGR46EcSRpxURg8JM4L5In+VECUZUUHRIJssz/OUW0VWQCMcvhIqtEyhMpTqQ8gY5gDGlelNMsdus5EFVEdbm31Zx/YJUlIkzmREJljUoGPbPjVZh/PoFAYJwmwu/DRE/TjF1J+Bzl9eoC8oRWibIwoCpWi4qcR/a7DeQCZxoJHC0KyTlIgZPVCYfDgWQNppBQRuZ2xJQiVV3iCktrWnbb7czx9lhraNslx1Cg1WpB0zTs93u01jRNNbfS1yglYrXPspk7WUvozziOMgfNAblHEZasGcYR67TggFKkqhqiTzw7OSX5zMWzl3z38+/x7ZffYTyMhBBwKvF4eCBMewl0zqCdppwaxglC1BSuQWEYhpG2XXF6ssYayGnegMZItz+w3+8oC8uoFIpIjhMqe5QKxDASvJ9byZUwwVOUYOKcGUPPNPaMg8fHTEqKyQeMtpydnBH9iMqZwloMEd919PuNfK+yYjgYsip57Dq+enfF46Gnuzvw1dWGh03g4vyM93cb+ikSYk8mYpwj28yQBnSQ8RhSwmhhSk9vvmYMkeX6lLrWRDR3Dwem0WCyJXpNYUqKRclmN1AWThjjZJS1hCH8Bk/xuGEHNXdlQAiBYeioKsGUnZ6t5uJOEDcLGWNl3Goj30ubQFaJcRhJqaCqG/oeXCGBZ9M0UDaFBEsRKBolORCp5uRsTYgTh35Hypr1uqUoJawvd4ITy2piDBEfFbZSjCGJgO4cy/WKoijYPGypypZxGCS0dJqYlLiho4+MfpjDpBS9H8jWsjBLmrZiOTW8efOAHUGZ2QH/e3aRYeY276yBRIqRYUxUrZvxN3LtXKEIKpHwFLXkuaQoG3Rj5GutMYQoTrUwJrZjJyYC21NWCLoMMb5ooGwUOYA24ErLNHmMMjht0IXG2IgrLLZw9PvIYR9YrYXhnFSmaR1+Dg1rGsenn17ws5/dEXxiu5swheXh/p5hDCybUvjsWrpU6qYgRcc4wMX5OaM/SDdDXcGcq1DXAIn9rme5POX2euCzF895/f4XNHVBSvDn//5v+fbn32e1PMGWIrxMw0Bd1JSFoW0lCPH29pbVN5ZyyAxy0HXOME4j1loWqzX96MlkCmcYui3d5pGybuXcEz05ePpDz/X7PcFHvvPtP+HVF9/lxcuXpDmUuXRLjDxxVByJIfAHf/CHrJol//V/9V8ydlucWbBaVJRO83h/S9W0BJ/w08hqdUqMIzm3pJxROlEXhRQp2oaEIYZEUS7ph3EuaEcMEVuUhJQ4HA7z/TqHaqvEMPYopShshVUioo+jcGpRiUycA55FgBknT1nVKJXROmCVoSkrQjJ888Ur1M0tf/3l12Qc9WlDUTVYq1ienIgTPkUJEzQWlRPLRckQAyhN34+UtuDls+dcnJ+ynXrWy2fs9nfEIEWhKSi6fuSTy0suL87RGbpdz/axo25XoA1VrdHO8Jd/9Xd88kpY1CiZn4ah4+xsxcP9IxfPzpnCFmU0d7cHrKtoWymkrk5a9tsDZ+dLUIamXvLP/5Mf8njv+df/w9/wvR9c/M63trUKpSQc209S6DNGY50jRDn4T9GjYT6nZk5OVnjvaetTtruBjCAYy9owekdKA3e3HfvDlqYu0Cax2d4xWo9aFxSuxBgj+8EAZDOjjxLL1YLFSpOVou9HnK1o6iVWaYZDx8PtLT4HrJPO3XHyLNoWbTwhTCQ1AQHtNInI7f0dly8/YVkv8NEzThGdZa+dk+yNvY8En4iDBNc2C1ljHrYP3G/u0SbR9TsqGxn2He2iRmNYrhecnJ3yi1/9ki4cUGXCLBzPXz3n+vEKHwPXD9e8fH5JTEkcokbRTR2BCV0oSl2y2Xe8f/NL7u8f2O8PTFMghMRn3/qc733nW7x49YJPX33Gi8tPWS8bxn5PUYkDVruCHAOH/Zbrd2+IfmJZl/RdR/aRs7amDwf2Q+L9m9e8+qbj/ZuBTRdYnD/nD3/0B8QQeP/mS7Q2nJycEs/O8e2Ck1VFGDfCoe8zrmpwpUVljaPAFAXaWKZ+z+WzU4a+4+3br1mdrHj2/IL3X3/N/f0DQ19ycrbGxPhUEO/jnlSMbMI1J82Cye8YS01v93SqQ2XRDmy1IvjI0A2YGIkKmuUKXTRk4PH+ARsMpyenGK2Fvz2bQrRRdPs9YRpxZcXJ2rJo11RFjfeRu7tHhm7CGMfFxXN8H6mKitJVmEKR9Ih1FbXKDMNEgeXh8QGnFSfP1gQ/cjjsSLmmLCTLJUwTwzByfXVNW7e0Z2sOXYePgaIqOTs/Y5w6NtsNVV2zWonxRhtN4RwnJyvykIij5+rmho/05N/jcQxpnAXpYxfrR132SSmYz7Z6NqUcccZHJrYYRWezKJLV9UEWmB3psylHzEkaaxQmmw9fPEMDj67y43NLcwaf/A81G2Y+aC3HjuQn4feJBiBn6JTkfGS0no2sGhUFoWGVZgoB24jhUxuLMcV8/yvZV03SKQUFOXmckfN8WVaMk6A7x4MnJWjqhnHwLOol24cNj29vuWjWPFvK2QVrGP3E8xcviT5issJphw2Ks1FB3/P+777i4bLDTyPaGd5cv8c/QtOd8qefvJQ9UUoYVzDM65wrHQnplsRYAjPaUGvpcDdi8mG+RmLSTE+FjKdgy3y0fx5RI8cg0g9vpWheQtMIYYAsIeApBJaLtXQIWU9VL7i6uqZultw/PPD88pLdbjcXx9VTYPU4Y02PonQIQUyUs9BsTYEWYRalDcrI6zCCYJCzrcpoXZCToA3zfC7Ucybg9dsrvv35BSn3cwfGLKbPhuY0Z1cSFRmD0hVaR0hi2kvpgxZ81MfU07XIsz7Nk9Z33BMfDbQzQ0eKAk/7GjFVf7gHP1xfKYD+LymiF6CSEud1gMknycwKc8bpXAIrcpLD4TwI/KzsJxT7KROSxE5GDVkr5rxAjkZA5TSmyEwps+08NnrGoChsYjqiXCJEC6mC0cCYIsMw0Q+K3QCbXhGzoZ8Ue6/oQiSS8dGQIgQ/iaPQB8IUmXzAe3kCWspDaG2ePOhPQaUoIjCR2afEJgJRscmOa5+5ypGHkMW1rODRa5osruQ4uxiP7CnyUeCVaqSYORPX19fsuwOrdv6Zx6rkrOMbpUmpQJsakwNWR6yJRCthm0ZrSueo64a0tljraJqW5fKElORgnqKEKwIY5ZEAEU9MHo18zhjF2q9QCpx1IurYli8+e853P7/g3/3tl1I1Og7iucDQtIpvf17y8mzk7vpfcf9mxbpxPF8n8lIq6jmIM7UpFTmNxOyp65qcI9pavr665ud/+e94+eoM11zQnL2iPP2Ufv9A9A/S9jUzcGWiTnNwq5od8rN4NRcsnlzfSW4cqzJ2Dg7V8NQR8kHwPi4ezIUTETt8SFIIQXjMwgXVGK1xVtzE2mgyam4zluqvoI4+CPkxHVEzPBVpDBIoWpWawiQK+wGJEeJHrtWsmEKmD5JBYGYskDpyh37Hx7HSGPNE3/cURYmiwBWOqsrEMBHSRJrbpATTYIkRpsOIn6SAUxSOwlnqyrFoa5yr8SFzzhl1WzF1nmE/iuDpheeOUpR1Rb1oMHO1XwHJgmsEQ+L98MSVNkacUm3VoLTBB7mHjXHSOplh5w+Mk5dra2cLf0RCi0PGp5EUmfEaRze0HBactuSQCGMQtlgxUBpHYaxUz7WRUNg0jzltsLZEqZIceykgAigt+JtcYGeUg9YOWzrh44ZMYYQ9ZgqF8SIC3t7eCcctJM6fnTNNPVul0dmg9Q2KI9cOYpzIJNrVgsWi5eR0LUFftqSqKtl0Gf3UenUM7kgpMY0DIQZqrfBJWN8hBKacaJua/rCnPxwYh5HT9ZIuRnbbHWEYWS0WRB+5vroGpXBlLcJcziJoFh5nAqhpXkw1h8Oe29tbrq+vGSVtCqK4xPO8QOY5mDDNfPNkzeyYkLBGY2QDl9V832fJHFDIgq+1xroSOx808+yiDjETppE4O8ANUmRMIWKcAStBWCpBCpGkZrfEvNE5diTlY+p0EvH76VeWUJJjaxr6GAqTheeWNUY7jCllc3zcBOWPXPBJCpyCoJF1aJo8fpxmset3e5SVfeI8juNADBnnoC4r3FnNfl/SDztWqxZrarzfk7NHqUjb1hwOe6yzaCUu4MWikYMQiSlKNkZRWKYpYpx0qGidsYWmdg2FKwA1M6MluKyqKtbrNSlF9vs9fd/N86kjRk9ZWbRZSRsmYQ4lzhRz18lxLmjqGoVjsViw3T0KnqGosMbhp4QfByrbsCpb1s0Kk6XgZ9D4lPBjj/cdwxQZfAJdUDdrbFlxVrfUdQvkuQ20xVqDnlvZ8jxGNXIQUOQ5eFfjjIQjFs6QjGEaJ1KYpJjuR/abR0wO9CowdlvGYYcGum5gs+04O7vkm9/8gmnq8P2Opi6ZxsTm8UD2I3XpCHHg169/xW77yHbzwNXVFW+vrtkcRh73E29v9my7zJAKTJ7ktxGjgnNZgptzZoqRjBRJjU5U2RJzz8PjHcoJvqasJAMkRo9TmhQnDodHqqbGFZqyLogpElLARkvywme0VqONIyPM+aObRTbNkRg9h06QGFUlDt9xmvDTRJ7bh7XWVLU4YIwz3D9syUDTVmy3O5SWefbY+VPUjpADZeF4dnlGXVfEKM/n4eoaY+FsfYpSCEc5Bs7OzoAsQdAIuuLx8Y6yLAEY9h0QqaszTk5adpsdm82GtpUxkcaAUQrrDKjMvttTliVVVePjyHb/wOHqgFKKZ8/PJFjLZPbb3e/VZQKS7Z4SBB9mZxeYUswAIQUpCqLFJaQ1+30ALaF/ymSmKczr6rG1WQ4s1opIZ7QU99KcvWLsnA+ThWHedxO2UHg/MfmEzprdYaRqFKRIipmicnQEfEDmWy34Rq1LUNKlY+acEzXvvbTSWFMzDhJW7tYtz1+sWa9bru7esi4bNhvP8+cX/OTv3mJd4PREXG2bx47z81P6bs/z5/LeNlXFNInpw+qaoZ9oSwip43H3S2pjqU1JjAMZT8qGEC3Wad6/f8d3P/k+3k9oFKWraJsF49RT1y3r9jk39/fUdSs+sujZ7R64ev81n7z8lHaxxjuNIvCP/uwfcHX1iLMN/4f//D/nu3/0Q4ZB2M+VLUBJV13hHL6X4hw5c375kjGMTP0WrQPOCT7RlZKfUpQlTdNK95t1cwdlQCstHabBELwnEiBHCmsJNhK8R+kJYwvGcZB9y4z9ska6CFKMs4lOXPUhRzKR3W7LMB5mhJElBM/QH7i+fk9RuLmLLODcjHPykZPlkphhP0VOTw90YaJLIz0Tq6rlbvPIs7MzdErc7+/50R/+AdPQ8/hwgy4KXFXNWCgJrL67v6O+XPL67RtcteDi/Jzz8+fiSsSwPUR8LFkUBWVdUtcHrK4Z+wxECa8PA7/69U/57Jufsnnc4P3I6dkasuLXv77l29/7JsYH3r67pipb7u73fPbFBdM0cHn5kr4bef78nKurd6ScePv1v+bqXeSHPzx9yvz4XR7D0FHXLYdui7MlGcV2t5GMKQUxDGIqmPma+2FPMUrRUquWsmjQytGUDUoFUtqjY8D4iAuWaZ+wdSS5A2E4UNiWprqABHVZ0ydPjND3guurGkPdah4279nuD5yeXLBct+yGO7rhgV33SNfLeq0wTJPGqEhRWkI4QO9xVlNVBSFMqOjJfqB2huBajKokoDoEPBpblkQkbD4pg1fQxYE8KbrpQGFLCmVxUbHdSrGtqhfUZwtMqXh2ekqMn/DweMUUAkMciThSqHCmoeORUR3IyjBNljrB4bABN7EZb3m3DVyoc/bjjhgHnp+tqauasiy5uHzBy+fnXF4+J+eA93seHwc0mmlw7HYHLl5+yubuioerN8RuK3vQGAnDiO56XBwYfI+di7Db/oDXLVfXd/zZn/0zfvW3P+HHf/kX5BwoC8Mffv97hNNTymbJO2cxVjCYZxeXtOuCcQiz8aonqxGDxmZHHBRlUdKszohqACZKZ7i/vuXu5pb9oef8+TllW3OYOjb9I3o5UK2BYmLoJ9yYqV2BXSTKuqAfB2xp2N6N9JvA6LecXLaULnPYPEJyxIMU1vSzyG33yL7fE1TC1RXWKLq+l26YaGDInDQLVvWK+8MDa1Vx8mwJFrKd2OQtikxZOLq+J5AY96ArRannEO80sO1Hzl9dsN1sMMGggme337NYLEFrVuszQX2UDbvNyP6x43G3oWlKXGmxriUrRV01xEEYz6pSIlaaltVpS+gmiq2mH3a/19p9VAGPHOjjIx/t28d/k2fMcxamtvrgXP4fGeiOusZRlD8yzGdB9Oh1nmWoOfxyViZnQfHjUNI0d62S8wyJCaj0AetxXK8/uEzn5zvz1eO8ViiVSUfBOFs0GaNLjHMYZ2d8qCBci6IihoQ2w0xLEPyjxkhY8dA/ifPOlRy6Hc44zk7Oub+54+XFS27e33D79TXx4Hn+8lPeffk1y/MTfAicnJ3S7Q6sFytU0ky7Hk2BiRP3r99z+9XXTIcd2SrGHDl5/ozF6SknF884eXY2F6fFeCWajhRfi7L8IDJr9eRAVznPnYuiByk1Y1WVFv1vFts5FkX0setA9mC/MTj4MDZyEoNc0orN/SPf/e73+Is//wsuL1/y53/+55ydnTFNgbqu0cZwf3/P6enpEwP9KJof96BHDrqMwTx/Hj640EVT1No8OdSJCqsR/c4YFJIzZ43sRa2VjMCHhy3j6HE6kPDz8DfzOVl/ZCaZX68SY8cTpvnJgzrvE5+0OsgpoeeRfbyj0vH65aNY/qEohPrgUlcfjdmP7yHgIxTM//Tj7y2iF0ba5icf8CGhvSJ5EQj00WlLxhuFzwqXNFpJ6A9JnOlk6ELCaEXUkJSh0AmroU+KKWeSzVBlHkLk9VahfSBN8+SRPfsB9ijGKmNrCE5CSYeo6LvMbtBsJkWMis5n9iEzJBFdhgDTlJnmgJEweqKXMKXJG3wQvIpcY3mD4jxx+BQhGbzKjBmuB0feGiYPr8fEdchskry+PEPtH6fEkDM2zE79mAlJOFAxyd+1ggCEGNjvet5+/Zah71k0+kMbgtIo48QdmxIJQ8KSk0Mbj8kRG8Xt4bQlmcyyWWKRylxdNxIYOLPbjRZulJ8mghJnR8o9OSv22tFWa8qyZAqeoigAQ1QVQUk40X/6v/ljXt9subq6e5qsi0rTLh3nZ5pXn1acrw3OTOyu3nHYJb71SUGcNN0wMU6JfeexruD87IST1ZL99sD5xQRa89X1Df/Nf//f8s/+V39GszhhmnnJT0M9R8H1ZC3usZwJWeGT/K8yQZ6d6DFlcpbbS7gPCZ0lgFLNzv6cJT06J0ghEXzGx2OIqBwijwWkyUdiEoyLnquqaBnr5Cw/NySmWZSJ6egqF4dhZNYOUxbW+fyaUAqrM4VNUhixYNTMBwyJlAUl4yOM4Vh4iXN7VX5avH7Xh3OC1gk+oJXwxepa8CCFcYyDQulATH5evGaEAVLJS35mJfuETpm2KiEoogkoq1ifLVgsa7r9wO5xR7frmPqEmiRwrl3WFMXMOU6KpME4S1k4YWRng8ma2lQsly1t3VKaahaqM8l35JhxRgIunHEiSmqNLgwqaXHhZQVxIngvr1VbytrNKBoJrXC2EB51SEzjxDhMRBdIhWwylLZoK664hAXlwIgwFJOElsYo+JPjWlCUBZJcncGCdtI2JpzljNMWGwq8T/S9oFyUUiwWK4ZhvsYhC7LAzAnpSZT8qipYtA1VLRxR60qqqqQoCpyzOGtlDB8rxzkRgyfHyH778BRIlmNgu3mgKguiH4jRUzcLQojc39xSlo62bgnGzW6wAdBUdUNVNdSLFXW7oG4XFHWLqxYioM2FIJRisVhwfn5GWZT4KSO2sY8r8ILhiDpLoSyEeREXXr61UuW2zkoLVwr4IHy5mARTgbKgDYW2JDJh8kyjCAGmNFRFiVPCoLXaYKxDY0gqQhLOXjAepcWxbZQlJykMy3IPOmvszPQ32WCyQkU5lGedpTr/JKwCGJQucW4WE5mLAEmCDoXkMqeJz52R0QdCnshB3Fm/68M6R1bQtDWrkyXj4JkmRdU42qbBFBm2PabIOCvt4iEEjNMsVwtCnNEbSlBgVVXx7t17lDGUdSVrifdz0SERI1hXcBQll8sl1lqmcRKhJWrh3uVAVZd0/R4fRpq6AZjxLMeQyGPivWQTyHrEUxgO83jRRrHvFFVVUdcFwUeCygx9T7Vo6IeB+/t7TqrzmfGbCH2QAOhxYrs7ELPFuYFxHHm2OqFuFk+MwKZpnsJ4gKdNnLWWtm0wVgGyoXVzEXsae7ROGC2uyiFIR8GQB1K6IfgDKQxcv/+aqd+xaBu6bqBpVzy/OEHpwPbhAZMH9iHRdXv2uy3WKKrCsnm4Z9ffE5XnfnfD63dfcvOwZbMfud8M3G48w6TJrmG5qjCDOIazSuIYlRPUh+yQueATsxQHpjDRDwN102BdSVEXgmvIGbQ41vbdDlcYnNNyLycpfian0Egrf1mWTNNI33fEJC5zY6zgnLTgkTJgbDGvAYLs0Vqh5lA1pcS55gpH05Tstnu6fkfOgZOTlWCIRnHEApSlhGQ7pxmnXpAESQwCdVOhjSKEiapyHA6ThOvNnQ0hBLquYxgGvJcCf4xSYHx8fKRt2pn1O6F1AyS6boe1EsgU48Q0eWIUUV8bx8PDPVorhmGgqipkM19QFJa+//24qgpNDAnvZYuTMxSFImUvoj5yLfXcSuwchFH2IGXpGIaAMeImk3ZouQbWOYqioOsG6lpc5nrGkw1+mvMionQLubk7z4EzBUTmoHUJ26ubJXVT0Pc9ox8xhWKxcGhdUFjF9mFDUcgepF3UDPcb2qpk6MWss1rV7HZ7Ts9XuBpQFh80xmZSDrx8scRPI0YLZ13eh8QwKG5vH7i4eEaIPe1CjA6np5c8zCGR3/yspWlgP3b4kPBhom1q0hQYR0ESna+fkXPET5PsCTEcDtIBsVqdMOz3KCx13dIPe2Lw3N/fcnX1lpPFEqLkFJ2frnlx+ZJvfQueP/8GP/jD7zN6z+P9vYwzDPVc/FbZom1DVgpXWcJhww/+wR/Q7R/ZbG6kezFLRkTfTyxXC7TRbPd7FssVlTGCvQS6gxRwxqEjKYMmItS5POc5FHIgVzLPk+XjStmn4PeyLJ/yDtp6weQ9RWE5e/aK7eaBQ7cDEn1/IGcpyosJSIwNfT/SViWqaShtCQHeX90xmcxiuZSg39JIO313oFCa1XLB+6srgp9YLxs2+z1ZFzT1Em00+/2G5aphzJ5nl2cobVmftNze3vPu3Y0U08uasgm8e39N1SiWyxOq8oQf//jnfPGdF8DEclmy3w8oBZeXl9xcX0EOnJ6ICHH1/pqT85LdNrJ85VgsGsa+RwHTONE0K8qyoGoK3ry55rtf/IDC7rm932HL3/3+7vo9zhXsdz3r9RnWWm6ub9nVhRR4s8Vo2a+2i4aytHRDR3c4sL/PXJx/wuWLV3PHtafvDgz9gapyEqaeJVMnhYwhse/uSDETo3BxY1QcuhFiYrlaMk0TxmgeHm/JSQqf4ziw3W3Yd4+SzRUT+82I1g6FJtdQmJIweeIETVnjTEEKA3VVMvYjJDV3Rs5ZR1H2fSklttsdfd+zaIRxf+yQWLYLvJ5YVAtOVgUPW8nGGXykbBy7w4GLsxekmBmHgLIFaM37q/eCg5hzp2KWcT6MHfvDTvY7SjrHrS05Xa64XK05aZaM+47lYklKicV6xcXzC4KfWK4bpn4PrqSpWipn2ZPoug19t+XibMk+7Pjqq19zd3VF8IFCF+KojT3DsMNqxdAP5KrGFSVffvUl1zc3PD7e46zi8WZP2D3iFHz66TdYna5Znaz4h//oTymbmuu37/BRZKQ0XydbVKiiRpeRi2en2J9nCudYr06Iw0RlLYrE1dU73rx9zenFBfeHDcVCU6SIMxk/9Yz9AU1F4wqMyjhnxQCWPGWhsMuKbCQHZvtwx34TOV0+p7aOYRq5uX7PttuRdUaXFm0NOIvWBYmMVQVpguKkISVNN4wUdUWzXoOFbtyCOTCGkTKMjGOHTxPkQPABV1q6Q48pLCYmMobNpsNoJwfmKbC5e6BZLHj+4iXKGKYQOOw7yIrKFUzTRCKQcpB9ufDO6Mee1XrBmAYedo9U5zWLkyXrQ8Xoh99r7U6kJ8HuGPT4lJv2hHuUdTsRP7iW+fC5o/iX85OX/ems8iTOH63qc75Rzpmk4uxwn+0+SjrDJQfvt34/sdvTrMUcXdLzr3wU4p+e2JzRF+fvIXlSec48gkCIc2acVmTjyCbiZ7KBVoJAVkpRFAWFL+S8qUQv6NIeZTQE5uDzSFu22KhZFQtSH3jz89dkH0hj4P3bK1ShaVeyvzBJU5UVoZvwg6etFsRhYjN47rYPeBKbbs/Z83O++ckLPInFyQnVcoGrCgmEnc0nx9yeGOOshX5gwx91maOLWsmkJphZheQOBjFSfkCkPIGIOV7Q35Z3jnpY1oowBoyVTJLbu3uhTsyY0u12S0qJruueHOd3d3dUVUVRFBhjGIZB9mUfFXvVnGk1/+sJ4SKmww9P5oixPBZfxODEnKsmpgxjDFOA3XYnJirjUYwyOtXxRP3BvCodF/IqlUpP4/HjotKRZf/0TPSxkHMsFM0O2Keb6yjCz4UBBEPIb+FanrLc5r//fXS1v7eIrg/yxsWgiQHwEppptMaa2dUHdClx8BkVBQmg0JgcKHTGAwapyPgJpgyF1tgCJuWYtCYS8DbzLkJ/n7CTpo6Zb7SwcpnrQfMuJ4KFlYZpgvEgFaHtAXYjDN7hY6KPwk6f0JgkfPZpioxjxKhIGAMpQj8kukFCT0M6zl1ir/YzsyjMFaIhC+P8pnNc+UwXEg9jopuLAHEWtwZ5uwgoXFK4mCF+MCuGJMGQRkvQQiQxHgbefHnN+zdvOT/7dHaNGrSuyNk8iUxaacEP6IJkAjrGmZc9YozG6MRy2aLVyNCP1K6gMBpT2Cc3tDUirIUQ8MHPVTVNypEpDHTDjt3hgboqUcrio4QiFOWK733vR6xW/46r6xu0VdRtyfpE8+k3CpYlnD2rWa2cNEIEYZYuihO63YTdZYbBM06ZQ39gjC2Ls1MeH9/zTb6HVvCsWfHZqwuMLUjaEMlM3ZZxHMhJoWZGkjg65WaKKc9tyJmQpHrmk7Tox9+4/Y7W+UwIEZ1E1JLvmRinwH5IDF4wK8vM7NCW7ospiMhQwMxSUvNNrWa+lQTcjmOg94mYEsxVuyO5/uhEP7rlj/eqmrllKUW0mkMbYiLGzBRmCpaWMVDo2Vw9dzS437evLHmMKkAZwuQZup5Fs2DR1kzGY3VGq0g/iBiZfGLWjSSAC03W4qYNY6TbDYKNwGBqR1EYlNYUyVClgqQ9pkgwIK7yhYR4jqMXQQ5hjTlnxdlqKwnAItI0wu63SkJE+3EgTEEEi8KhZheX0YZkhJVrsCiVJNgtZsromeaQ0rIsWK/XuKIgpnlxmlsrYkpMfmD0JUbPoqcG4wqp1mOJaFKe2xTz7GyaRXQxwGcpCBgtrOMpY0qFckYSxbMjhERbN9zePNC2zdOC1bZyuCzrRFl3LE9W+PERlRQpJCrjWK2WVLXDzJXf9XqJK9zMl3M4Yz5a3PMsoo9kbXjcPNC0yznMRTFNPSRxgtZ1xeP9HcUsDu0eNyiYxURNWVcsFisWyzVFVWPLirpZUtYNrqzJ2oLSc+U3z8FeH3IKUhK8ydF556zFFoK2yhHGYXoqcuW5XcQ5CZplXkRDChBEJMrMrXHWEMiMM591GAemccAYRYXDOkdT1TjtpF3LAMrMq2wmZs+UE+hIYTWFq8jeELIkoGP4jcBpgxaRO0WYC1+SKD6XGGfImjZGwnPmEGZyeHLf55Tng2QWEX7GMqQchTn3e4joL16+JKVR8DMqUTcFi0WF0RZlAlWtSEhb99DvqZuSlETAHsYD2ojLoiyrmWVdc3t3zbe+/W18ECbnYrGYW80N2+0j42Rwzs3jyrDf78g5s1qtULqSNsJCCtfOySa0aSvpUJh4EjCnaeLi4gLvPW7udEgpyca6KOgOHQolGQPzGB/H47jJWOsoi4qT9SkXF88BCCky+ZFDd5CxMUmxOQSPnzzkzNnpKUobNtvd7MaQ90Qb++QaOQqKi8WCsnLkHIh+xChFVddE33I4bOW6WItzJVMamKaR6D1j90iYOg7be0pnuH7/hsIVvLy8oNvf83B/wzj23Lz/GTkJJiKliPcjfdehyKxWS7a7R37x+hdcP97wePBs956HXaSf5vXBjyhbUDhwpYEswexPhdwUZRmdhdSUI8z7uXEa2O33YEUstcbOWRRyTZyDunFYK9kZKIV1mbqUThWlwdhE6Ad87DFakfFUVYsxJT54KlPKGpsmyTixzPOZ4RhCqrQE0KY84grIamLyHefnzwDNer3i6uodSkuhbrN5xBiIaeLm5obVakFVVxSloSwdIYwMY0fbCKqj6/bknOfwU8HMiKBs8V7LvOMM3o9MkybEkRBHxqkTh7yFojQyx9ROxraf2O3uWa7PMFbT1DVaQz8MrNcryJmmrXHF33sb/h985Cgc8hykM1BpIM+YKNTcpTbPOWSKUotZIASUgaZx9L10H2otXSRP3xtp/w4pCaIgibBQ1Y6oJrQRtvzkJ9brEpSi203klKkq6TrV87rjo+fy1ZLHzZ44aZZrQenE4NhuPYuFHLzrxtK9CVxcrthuPIdDZLEoKUvwoUeZitWq4nDoeP78gs3DHmtkv+eKAq0d+/2EMSNVFXGufCqGpeiBE8KUaOoFj3dXGBVZLBdzYU8ccxKIamjrhrKo2O/2nFXnKK1mFIGZg7ElTHyz2bI8WbHf72eH9g5XFEzjxN3dDfv9npPTU9pliSkaymrNs4tP2O82jL5nXZcUhXSl7e/v0MpS1a2Yj+oC5aXT7+T8Gc2ywtUFfjwQuj2TP/Du/TtOzy/wMWLcXDwOAaM004w20kYTUkJZKyJAli4Fay1Hrn9ZFcLCDpEcA2PvxU1mJAcjeDlo7/Y7KWhbw+GwxzmLMYI2MlZjrAQGK6WlgDb0tE3DMI4Mfc9yWVG5mtVyga80V4+3nLYL7KPl+ck57XJJYwzdfi9YSS35NiFEYteh1Zaz0xP6/sDbt1esvrnk5PSEqmy4vnpPd0hs7vdcPK/4+S9+zve//wWHwbDvOsqiIYye73z7Mx4ebilqKeZdPD8jxsQ0Rr7zne/xcH9FSoEf/eh73N5dsdtuqEpLipFlW9I0pYQJVxXbbcfNzTUXz0/YbG65un7Pd7/3I3715U8Iv0enSVmWdN2Bm5s7qmqBcxWgGfqAn7Y4V1AWCw77Xua4quLQHzAzzm61rrA28fj4SEwjj5s77u5ucbLVx4dARU2KmcRIUnuG8VhIO/Jz04y2WDF5CdyOMdLULePUkzNUdcV2hxQSNXRpxNmSoqgpTYkCKtMymQKrG8qixk9Q12tS0uz3Pd4HyrqkKErJzZhGHh4fuLm5JkTJ2glxwlFiUExzl8VqtaJuHGM4RdsFIStyNhz2Iz5kNps905g4W63w08T17S1tW1MtK5rWEUJHzIZhsjw8ahSeQoHCUhYll+fnfPuTb5DGCeXlfLfb7bAms9/ek7RlURX4kKhPzygNxLGjKTROTdQ281f//t/yy7/5a27fvCb08x6hLDk/W1OVmseHB6J1RBXwTNSLJUOYeP32DUVVolKgKgrS5Cmqkml3z2QCi8szfv6zn3H3eKBqTzg7f8nkPTGNpBQZpolmuWZ5/gKs4uKs4fGuxwCLtiH0ey6fn/CtLz6hWba4dsFf/fxnfPXul5yvV0R9BzmybGpiiGhXcL/Z4wYxWzijUC6Js9MW6BxRKVNZi1VgC8dhM3F3s6efJupFS5xGxjThioKmKqnbhmW1xGDZ70eMi1AUeANdGjBYojZUixanDDGNxNTjw0BdGQY8WQVZY6uWRXsijHTlyBFKo3FFxcN2R0/P2A8s1ifc3T1w2O8JUcKCC1sRVWTo+qdOa2scORi0cSjj6aeRMQfO1y3j65Gsf78CuI9HR67oOk8sczJJpQ+IiVmkfJLHjx8+UhJm9It4BfWTrnAUJ4/iugIJVZx9uiknwW3kDxrFk4ieZ8H9+Ct/YHV//IsMKR6d6fNTyyIcxyTr6czMmF3romFkJGuICGGy5HCgKDxaCwI0RTGpVFVJytL9Oo4yJyUy1jlMjKQYKUzJompJQ6AIlvdfvuEXf/5Tbq+v+bP/6I+4v7njW9/9gm6z5+TkDIdld7/FZMNpe8Kw6VERbjcHrrb31Gcrnr+6pF42JCuicB8nCg2uLrFFxb4/oBCd5oi78t7jCjMb6MzT+3PEK3/MEc/EmVKQPzLnzM5+ju/nb3UZfBgMT39K7o+mXbT8i3/xf8c5x9XVFc+fP+err75Ca0vXdbPhp50RbIJbLMtyxs0cQ0I/BHV97Ej/OEg0f/Q5EdGN5IMh53M/CbM+J0Plqicn+3Z7YL/rWC4nlJqevv6IsThqXupjUT1LKGsizWcOUc4jkeOoZh6l+lh1mgNbpTCUOfLOP4j08+v6yMk+P5knbeL47/9FRXS/FZFKOIrirlNzEJk2lrIoyCj2PrEbIiEoqVimQO00ZelY1pbWKmnjUAmrMkaJoJSMRumEDpmgPHc+827MpAGWwD4YGguPE9w6QdF+MsJDUrRDQakU17vEplf0IRGzZkqZmNU8HIWPk2ZHcVaCook+Mw6K/QhjiPI1UvYTJEeacRzKEFFMCcaUOWTYhcw+wpiOk9jxTRLn8IAiqkyZMmWag15zFjRAmkXQlIhIcnHwkYebDT/5yV/xgx+9wKqarCtisoI8OFb7lLCEMY4i1wQmskpywLWWEMPsqhIUQVk5YU5bEUGqquLICHLaQpgEkROkRRIU/eB42N5QFo6mXqHLlml7S6EVXXdg3+9ZP6uwheLlJxWXzyyffOoocqI9cZS1iHIaxWJRYkOFsQPKWDb3e06WGmNK6sqyebxG68AUB8jwzc9ecffFdxjGgapZsD67wFRLqbIHceJ6L8Gd8tDzjYawoufk3zBjdCKJmLQkQeujSJpnd9lIzgnjjmGgmcOQGDwkMj4qaWHKislLSK3QGESwVPOC93RjJjm8jFNCMiCPIrsAIY6hokeci0wAcrMnYApzSEIWJ+4YI8MkYboqy8a3srAoYR9mV3r6zUjE3+VRlIZjooI6TkBzO35hDRSO6C3JFfKaY5KAYK3BZFwpItEx8MRYCQ0NIeIPHt8PaGMIPhDSgKs1Vb1ATRmjDWVVoJUh5IAO4rw2VkQdV1pQVt4jldBGuJBonpjwttDCX06eqjYU2Uk7ulKUzgJGwoKjRStNYRQ+GsrGsFwuWZ+uqeqarDRddyBH0KUhEkjKM8YOFTwBzxS9tDMrGRtTCIxhQtKgRQgYpxGtleAllJPWtiRt4q5VEMxcjFSkDHqKpBBmwXh2kVgJRkkJXFlQLxqeXz5n/yiHAqWgrWuWbYMrxN1bVsKKPoawaC0YhBQS1hpJR48iFCqbiT5zdfWOpqk4PztDZWGjez+SoqepGraPj6wWS+LkiSFKW2jTsF6dUJYNzhVoW2BtRVHWlFWLdY6kzFwoOvJjHW3bcjIHSYZxxMwtfHEWy7KfETtzCOpxI3jEDUlIaZq5gMw3xYeCVMwRnwMGhUpz14rOBBXFzV45bFWgC421suOLec6yzzKufOoZUyBS4DRU1mFNIe664InKE5TgjWLKWCwpeySo0zwV3dSMHIl4UAFlM7ZQZGWkHTBJCGdKkrmQ5sDWrIR1l1Imx4jBzO6N3+1xd3/L6emCdtEw+Z5pnMT5nzI+Dkg1fCBmjy1kUxSDYt2u2O8PoDM+TuioOXR7YfCXFuuEWd+2LZ9//hldd+Ddu7d8/vnnvP76NcPQU9cimO/3O0L0lKWbXeuZGA3TNKB1ntsjZUddlm4OuxU272effZOf//znVJWI7DHGWezT0ukWjhxi9xQ+WzjpxFivz1iUK05PTynLiuA94zjQDQfGaSJnhbGOprHsDwOr1ZLnF89YtC37vp8dcceDyseuiONj3uxmKSAYVRDmcCMRXgtiGAEt7P6YGENHzp6ichS2IIyO2+t3dN2es7NzHh9FPL972PDTv/sblm1m0Yq4MAw9m+0jwzhR1w0Puwfevn3L1+/fMU6BIWjGqPAJlNUYVRCDIs+b3BylvTqFfIx5eXr+8+T/hEpKOeJjZN93mAJstcBaw/psRVFa6qYgJIsymWE80M3haDklrANtZU+RMrgCXJjdziFSVobFcsnD/R1lZSXro+/ISCaDtQpt8szBD4LZKjQhTYTQsz5ZUFct3/rim3z55Vf4MMyu60Rdl/T9MRhxwvuRsjyjLAUHVZSWotCgPCjh/I7jRIiRqqqeBFdjzJObp65rvJfOvKauqaqCh4dI1+1RStH3Pc4ZGcvpOD4VIYqbpixLdvvNzIOdZhZ8iffD73VvA8SgCBNkP3sKtHQDYI+HtYy1kllAFgxd4RSTz6ggWJflspzHt3k6kMv7JDg1ed5qfq0jzcpiC+m6M9rhh8QUPMZq2mXJOHowEes0bWsxJoHyDAF0IV2sOSWOeWQi2gkI7fS8YH1rOT9f8P7NTnjIMeMqI/kkKWBsxdAfaOqSw/4RhSVMwtj87nc/56/+6leE0D+9596PnKxb7u5uyTkxjB1Gi2jRHSLjMj4J4inCzfUDL88vqcsF9/db6lxgrRLkYdZPBTpjLMMw8ri5p2gKun6gaRtAOtradomxheS1KM3q7AxXL9C2QdeVrB9GE4PHNktu3l/x+LjjsB9oFyuqtmF9uqKsNUoHbAZtSk7OLnm4vSJwIAPL9QpbOEY/UVZSuBy6gbauGQ87YvCsThY8bjdoJFAzJE/OcmiVvB4p3BqjKG1JXTiMNQxDz8PDvXS1OUtRFoQ5zG2xWEhIYArzATvhvZh5QgjEGIkxQZa9oHFO9rk+UriKu5sBfarofC85TcqxbJe8v76iiJll0zCkSNVWbHc7nCtp2pKLi2dsHu+Z/MhqJV1Pi3bBm9evqQvH6llFoT2urFh/8Yx3V1f4qBkGTz8OZF9yftbSVIbDsMM1Em469OI6Xy6WXD5/yfX1W6wJGJ3wk+dk7ShcQc6Ow7bn4uIMpRTeDzy7OGWxqLm8POf05BXX795z+fKSrt//zve2MYb7uzs2my0vXwT8JNhCMQ0EhrEjxkzXj5RVATnTHQZOT9e8fHZJ4RxXN2+4vblmtV7w8HDP4+OGsjaCEM2JrC1N09CUFf3hhuQGlF2RUiAzo9dWS+maGcI810km2DgeqJt2bvd3TFOPSgGjDXVVU5UtShnKouDs9BkoKT5VlZ156sK8HnqPKUpygr6beHh4oO879ofNnF0mXbDLRQM50h92jP3IyeqZhDaHgaJoGUNPCpH9fqSpWnxIlFWLUpq2XfLu+gptFP2wReU9hUm4oma3PYBp2e0eZN5QhpaWwtWcLFcs65qgNIPf4VTm5fNzru9u2N0PPLt8ybB7ZLlYMuy31EYzDCOn56f8/O/+Pb/66c+5vdnwi69v+eqrDff3O3KCT88KvnE40FYGrQVRO4QOvV5z8vyMN9e3nD6/wJBJY4c/7FkVBZVWmNyxbBt+/vNfcLXZUzSnnD5LfPn6jq7bY62nbRpQmcEHTi5ecPmNz6mt4xuvXvKLn/0UPxyoCo1zFS8/ecHLb3yD7RD48c9+jg+JzWbH5WcNXRzJIWBUKSYrL0HplS1x2hIZ2PcPJBVYrVtAsV6taYqKwpYM3Y7DTpy91hQEnbGloyobccyGgMaglWV36KgWFcWiEfSLH6kLKf65ssQA49DTh06K/VFJN30QTjhK0bYL+k54/haF05YQE8t6QdCK7XbPGDOb3Za2rghhxI8jSjkymbps5sy8zOgTxjjKqqYfekKM3G0fKaqK7X7A2d/v5O3DNAuIkjpoZiFRTkwfHOBqXiM/6HrqSfQGnvLFjm52yXb6IMoqZn700Wk7n5nUjMDV87nzKKIfBXnyUUZPT8Lu0eH+kYwu553fEvePoZnydI8iOuQkHeBiYI6ErDA50lQVi3YhSDKlZa1VxzPscS8t31MbK2OiKJiGgWW9wGEIMfLrX3zJv/3v/g337+8hRm7f3bA4W7F92LE4XVK7kq+/fsPNzQ1/9g/+jIebO56fXPDrX/yKwjlOXz5nuV6gC0NQCVtVWGdwZUW1aNCFZYqBBNRzZ6xS6gmRoo0TbZR5TP5PPI6itZm751NKKD07pP9H2vlxFHz87w8ZXN5PGG3YbDb8+te/xhg5ax+DMY8u9/1+PxcmKkIIT/gWWadFYzyK5T75p/fy+HyPe9UYf/NrSiv3GMwdCMdsMKUJc0fbYd+x2exoWoNSk7jBFcxCjgTbMyNn54OIdLvEJ3znk/P9OApnTTxnuV/0sePi6Y+ZP3+8cfJvXtr0UQHj6eP5w1j9+1jX/v4iei7IPorQltXRCEhQCqM0YV4cSRkKCSAa/YRFUBe6bChbR2MTxcywVfMkgZFnYhuF855BZwYf2Q2KaYB7FJsAlc70JEKG1QQPI7xOhs5kDJHNIXPTZ7okb4oEIIBVCWsUUzRsD9BsPIVVJA8xWLZDZjdGhhjEiT63H8YkfHNrNXEG1I8ZOmAfE7sUGWa+7YcxPr8VUSYjr5Q4j5XCIuB7Nb+TKorInLR+Ynr7kPnZT97RD0lcB8mJgB5FKFCIa1MpjTGVMJ+NYxh64UhlTcaLoFcqDoeInw7E2LNYSCt7zomyrOeiQiSDYDqQooHynt1hj0HT2ILT9TNW9YLRNCgTeHZ6yTe/taAfPZD55JuGl88aLi9qnBNHoSsKnBWutlk2pNGxdBXW9lgMTZ24fdgxHA7cfv0VTVni9x35PKBU5OSkIacogrNuMOUCoyt88vgMY1BMMUkq8XzZjyL1FGURCFlQOcKb54lPfhS+Q4zEyaMwxCjO9hDl8DglSQz2WURq7xODT0wxziK6uKtyCkSfnhhWOQubtB8TY+SpOiuV2fnzKSGsZUVSM6FsFkun2U2WcmL0IzpkJp/xWYo+Vc6c1pkzpXgcDb3PkmqdPxZ2/uc/qtLOrbeCy3BWeNIpqlmgsuSmwlqEK5qFe5VVRNlMXRfU9YI0c5tcoTEGMOKcjASSTnOFMlFVwsgUxJvB2YIcwQRL40Q8VqV0T0glOpJVEmHdGpwR3qtKEV0oCmWJgxfHpE2C2SwCAAEAAElEQVTYpFksKqYgxY4YAtpCuSjBQeqgRNOsKparBU1TU9S1NA1ZyXHIOjEy0sWePHnGZMSlnEZi8vMGIjH4AaX2FK4kzo5RbeR52sLOjNk4d0h4jBYBdZpAOYufEqUt8btAzgEQl+A0efp+oO96ilqzWLa8eHnJ+9fvidNeeKjWyG8nv8uymEPiZEJSWmOUISbB1xSFm53DBbvthsVyTYwTtzdbpqFjtWy5u72mLBxjP5CMQ8XIOPSM48RyuaIqa6qqmbtXDEpbnKsoyxpjxeKUlRE+mpLilFaZxWLBq1ev+JM/+YcMw8Tf/vVfs99u8HN7TIiR4AM+RY6ojI+r4FIUmJEh0pYxz5lp5r4rCQI1EZwBJ3iK0llMWaMV2MpgakPEE0yS0NqQmWKmrCSItB8n+rRlihYLVLqArJnSyKQHpjwRlCeZSFYS/2kApcTBq4wlqkgIgwjhORFVQJmIKxLaibhmMmiryUnhJ4fMpmkOLxUOcEZwIM643/nenqaR118/ULeG1aqdPzbhXC1CYZjwoSPlibZesd/1pJS4fHnJ5Cdi1Jhk2B92uMJxdf2euqkYx4Hz8wt2ux2vX3/F5YvnaKO4f7gHpD1xGHog0/cHtIFh7BjHER8ck5fg0KKwTN4zTQPj1FMWJWVZzBu2zHq9fnK7xxgAQYFozSxcatrFag6rhWEIWFfgTPnk/liu1nMI0UhI0jaqjcG6gtZYtoeOqmp49fIll8+fkfxE9P43ijhwFEjFrZ3mgqn3nnHqsNYIAz0lgvekDFVZM+aM9wmFOGyMMRA9wzAxDQf6vqM77MXBP3RsN4/s9ntu7u4pnOZ0cYYzlv1ux9v37+i6XjI64h2HvmO72xGiZGRMQUuhl0RWJVW9on/c03W9FBVLTVFY2kqLm0NLsLZWwv4Unr0UJzFKijwqAQGnE93Ys2bBcr1ifbJgt3/Ex5F+Gkh5oigLnC4on4ogAzEmtE7UtUVpJVxsKw7ox8cIBDKJmEay94DMYTlqQpCOhBAUZWnwaaQf96xXp0xTT1ka2rbi/fv3fPLJK+7v7yQwr62xNkoRtrAsVwuqqmS3a/BhIiX5XFUJQqbvO3E/zcgX2R+IGcGPnjKVHA4HtNaMfiBOozh0nSEncSd23VEYH9FaHO+LRcv2cHgS4KcpUpaWzfaBqnou2J9p+p3vbYCpT2TPE75MAdMgmERnDVlJ5kRRSIEjJ0VROoQpHVBqoq7L+UAn7bjaSLOPj1K918ZKoToIzrEfPXV97ACKlDX4EKlLzTBMlE1FjB3Gynsf4kCz0Oz6CT0Xc0PMbB4Gvv2tM8g3kh2UPEpNfP7thsXSkb7u5+wbRV03XN93EsJtlyhdcX39QFkZptSxWFT0kwiL3/riGW++fgNKsIVFaQhBemF32w1Fabm7faCpa2q7JEcjeTBDR9PWbLstm82e5bM1+23P6mwpnU/eUzoxoRhjKcuKt2+vaZqCk5MF94/3LJYtKWUunr3AKcOb119T1SU+QT95vJ4oTENrLTobcCXBR/7r/+Z/4K/+8ifsdz1aO5wtSaHjj//4h/zjf/KnFJUiRwCHVgWnp8+5m7YUfqJpJU8mk1FaHNcqWRZVRds0vHt7w/nzU5yVwu3gPYRARhF9xNpSsjeSIsTEoqrJKbPdbdBaS9CnytKJFzQ+ZIahI8SBlKRYIqaBgof7ievr9+QcnwrfWkuOkTaGkBPGWlarhuWiYrKBZVUxxQmsZnvY0yhLGCYWdU1VVyzXK5hGpslTFgWH7oBzwuofp4HD/QGNod/tuPjkksJpWDl2XcfYPzBOe+lUtEclquP160f+t//Zf8TPv/wpm+EA2TH6SE4arRyHQy9mBhPJOXB2umTfj1w8e86P/+INL161XF/fM02B5XKBtZbDYUe7qMk5SGE9fxSA/Ts8usPENMkZL4TE5CW4dRwHTk4X7PcTh24nqMgYGMbA4XDg+fNn5Jy5u7/lzddv2R+2nD87wU+BsqhlbR47fOiZQoZ0yuXZS+JwoOsfqZYr3Lx3jVmEi+O8aMyMUNNpZp0L51spwzBI16rKmv2+47CfKIqSk5MTbIozIiaw3faEkNDKsmhPiEHhTMnj5oGc4fb2kcN+hy2kW+dwmOj6HXV9ytSPGKVZ1g3LtpXCwTAxTp6un+jHiapusM6x2W5wRUEx741DkBBgP3UM3UBpNU1TM44T2hX0fmIadiRboJWidRXLxYqiqFiUNb4oMFryPAbf4yZPmg7E6PAaTFmzexAU1pc//Ru++vG/4XDw3N71/PufvuFur/nsu/+E67sH/uL1jxlD5LR1PDtf0w+e1MD+dkOoTzk7fy5ZWzkQh4489BQ5cfv2LavWcvO4p1yfsy7OeDx4/uW//Rsebh9ZtpamimiV+cY3PuHFq1eE4cD2/opmuabbPXB+2vJ6+46URnzwLFftvE+3kCz3d1tcW6FNS2EbDrcbirLCxURdN6gUadwCoxW76SDjRE1ou5Diexh5HO5xpuT07ATnDEo5fM6Y2lGtWrISETll0VicNahUcBg6wiBIlbIqxbwVxWRTliW6LmnUSrKTlCIGTYgTRVXMZxCHNpGYE1VRYWxBP3Xo+Uy72+3wScLJ27rg/HTNFLwUlNLHYrRBkdA6E6ZIW7UYJMT+3e0Nozek9Pt1kYUUxHms9WwI+8DGPhqG4GPx/IPelI9/V2L4k+wmOR/J/u+oXR+RKx8xpD9SE+UjIk58LMx//H8/eCyEiv4kpM8i/RFX8kHwz/yG3K6OItkclIpoOzlGUpJzgA8N6+VECBFr5Pkc3cMpJYw2OFcSU8K5gn7ssc5ijaV0js31HX/1b/6cKhcM+55Pnl/inGO/37JYL3j7+mv+5MWfsrl74L/6f/6X/K//+T/nr/78x/zgOz/gpz/5O1Q2PP/0JQEvGozJZK1wTmNdQdHUuLoUvRLpEirLkiMC5GOsizGSY/K09qVIinMHKxljhJ6htJ47EGQspJTmgraIRmKcjWQ+Nlp8uJZay/kxfSSAv3v3jh/84Ifc3NySs6zTKeUn8Xy3k+5aMSWNvyGqD8OH7DkMT6+N43s6Y2e8n+ScMzvaXdsQY2YYJKA0Rp7MKMpMWKdmVO2IUgsE8J3mIr4w4J+ywUjyp1LzOWsOFUU+n/KH89iRgD5XZ6S/Yna2H6nywBNP/SOfs+g0or7xwf1/vMKZvy8l+e89A/zD/+x/TxgHpkHS4p8qAVoGmHGlsGKl/MU0jGz3O1CKpnCs64KTukBPHckPVIVFxcRw2OHDSNAJqwNl2DFOj/TjwCT5N4zA1RyYlK1imTNhUDwMhkNv+TpmYCJEhY+C8jAKnNIYpTCzkPnQK355Dds+4uYY1wzc7Q2PYyZp4QEeJx50AqvmgAC58D7BmMGjiPE4oamPxnYGL0zR46eiUvRzS6QjYWZ3+3xOlSrU06SZefPlA7/65RV/9Mcv0bOgEtNETsLEMhhpMyLP3QASgGddgQ2RsmQWhaEfNgz9BklrVpRlw8Hv8LGhrtYYW8i1yiIykzxTDOQhYbNiawv2+wcqp8nKkJnIueNP//ib3G4809jx8uWCxaJgsSpwVhhWxpU4tyTrAa0WUFgK5cgYtHaU+4miLAlhROVAVVRYZyEHfvXTn9A2DXVhULYEXWHdEm3c7A5PjEEE/wzEHOegK3nNfq4+Hd+eNKf05rmAoVEiavpZ3FCKMqWnRUHcxIiIHgXxIu78QExzRW6+u2KMBD8HKihIMTONnn4U8Ttn6VJIWs08buGAPU1K6kO1K2XpTDjy04NPpCkw+WMhQMJQm0oTjcVt5QAr4+D3c7MV1iCrgyBSjBG3eUZcAkUpuBCjFUZ7gkd40oVBm4CzJU1ToZQRhA1RCm1ZYzIYK1W3wlrpvkgSVKSdwTnhTievqJRspjVKegu0BJFZW4BODEEObXF2ogrHTYTJwjiM1SiXsVmxXNYMk2zGotOyudI1uYdkHahAu66pFjL2jNNYbZnUhPeBMY2QM4cpE5KhcJpMEBa9lm6QmDyjH1Gqn7sJkmAeCkdRWOGI5ijPOSq0CkQyOllsNgSUrBfJkJK0WR2F577v2e12dP1ANpaTkyV1kzk5XdPtOkrtqMuKuipxhaKqCoyRjal1WjaEWVwKZMhZeKXOObrDjrat6LsdtbJUZclhv6epS/q+o+8zdVXRd3uZg7Nsj8ZhxNlKGOdotDGUVU1R1RRFibGOD5wzeWg9d89kcTxdvrjkT/7kTzBK86uf/4yH2xsO+3lRVYYcPLj8dAC3bm6dN/M4dE6q1uqYSyBs2rJ0FFWJn3nmrjQYrWc3Q0mKAWU1U5Y5JwRFoQpx7boKXWrmbD2mcWCKkTJZylAREgypowsdIU7CT3egkyLrCFaTNSQt3S5h3nAoLXOET5Pcv04LYz0kdM4YJ62Q4dgOaQThpVxBjg6VMxZD8XuI6F2/px929GNiGHasVku0dqQx0y4WTH3PFHom31O6ahbYHXd3d9R1Tdd1T50A0zTx8uVLdrsdz87PmfxE2zY8bh4YR3F8397esFiI661txRVZ1RUxTrNLUbHdPtL3PS9fvhQ+X+nougNFUZJzfsK/vHr1it1uQ1E4pmlgt9s+BZKmFLFWci28F/5zVZXs93tSTNhCkDTrkxMun19iKdjcbRjGAaU1Tosrfn16xv3jDqUsd7d3WFNwfnE5F2OZRZA0tyUqnDNPbp2cMyEIdmYcAt592DyT1RxuB0ppQsiCDU+I89TIQW0cJ0CcfMMwcv9wz/3DA9o41i8vCVHx8LDn7v6e65tHptETcqafRinSqAIfPIeux0dHyhplDeMePv30BfvuLcPDgdwlQq1Zf3JCWxccxj1KR8wR/6Hm16oSU5hYWIsrHdoaokpUrcWEjE89+8MD/bRhfbJAY9nsJ1RO0r2yXAia6vaemDw+jFRVQVUUHA6HOax2S1lZ2oVs5MdpxBUyZ/gwsD+MnJycsD5Z4L1ns9mwPzxiCo0xmZgGmnbJV69/zYvLV1xdvZPD44x+quuKZ8+WBC/M7BAC45hZLle8ffc1dV3OewIRBkMItG1NyrIuffe73+FXv/qVhIamyG4v4846g58ENSCB13IAbBfiVI/JoI2i6/Zos6BpW8b7B0LwrE9W0g0yTRy6AyEGfBDh9vd5+GEem8bMZwlxr3mfUE4wLJlI3VbkFNjvJpTVFJXBzdP05EeqsgQFy5OSTGLoA9YYPAql01yECCwWJVMaZyON4rDzKBBXXiHBX9vNgdPTAmMnjE2ghIWqjey37JxxkRfiUH/xqmG1tqQExmnaQkuQ3+WCr36dsbYiBCl6jkOitBFrLA8PO/7ojz7h5jrQx0TXjfhJ5v+YAt4nUtbECJvtAa0kTyHliW7fEwZFc3EmSInDgNLQHQJ9DzZ64ims2pbVomUaBnJO1KYW9FD0vHl9zTh1LJbSkSFdB44sFQ3qZsHi5JSiaanblofNnhPXkubOh+VqTabgZz/5Of+X/9t/IahJU+JDZhonVNjy9u2XfPHFJ5ycCjJrsVgJmsUnUpTciQzsD1vqekHwE/t9x6JeEnOirGuKouLh7gHjDMGPlM6wP+xJiMnHWEezWKNSJPnANK+rKQamUa6nOPjHuRhYEkPgzesr2kWFNnmenx8Yhj37/ZayLKWgUlWzsGDYbDf42jFMia9uH1AOtJVuPqcVKSqeX7xgoR2ulK7Y9fqEoihlDxxhGEaqsqUqa56dXVAVDdvxwDj0s6Dd0lQFMXqWp2tu7jdUlWKz3VK4UpCPBEY/8Ysvf4EpNM9XF7x/e4uxissXz7h/vKNpCu7uN1ycG0IcGMaSvpM9/vl5CTHT7TueXz7j7dt7nl+cMHrprHz37h3/X9L+88ey7UzzxH7LbHtsuIy015NFlps2UqNbPYI+aDTSABpBgATo3xQgA2mgTxLU09PqaVOO1SQvL+9NG/64bZfTh3efyGR1dRdxa4NJMk9kRJx99l5rr/W8z/t7TlZP6VzL2dnZjx7bzcFRlyvcoOh7ySap64KHzQ6lK/LCcth3LJen7Pa7KQ/E07UDH9or2rabnlVKCsc6w5p6yqpxJDrp7hgXWDXj+eXXvHn3S/q+o21HjKmwRroRspR/RFhOa8ljYbDrBoRpXjCOsn5uDg0KzXwxZ4wjF+aM0fW8fftWMk1sIZx3PaPvI3VQk4BWTCHsOdYKMmw2LzkcGh4eIoUx5EVFmeUMXcfmYY8yChd6otYoo6fCy4K7mzvWi6WYpXSg6/eMzUjfH8iN5snpKftdi3cJayJNsyfFjpN5xTwr+fzZ5/SDo+kGdn1PrhXWgPcjxlryGPFjD3liaCMmyRo2aMP7777FdyNvvn/P/+tf/oJvr3t8viC721EvzxnWz/jl9TteXazoYks5n7M8P2HoPOuzpyyrmrY7MJuv6LTl5Mkld+/fcX1oWZ694L4fuL39QBM0+9bz5vv3vH/znrNFxs++ecLZ6ZKr9x+4u7vjy2++Ef5x19E1Lc71kAaMjTy7eEJKMA6etgEdc7qtpzsEjC4py4Qtdxy6gXGMlFmGTgYdLUM3Er1G64JkBHVYzkrCmBj6njdv3/L1l1+yWM354uWX7JqWh3ZH60d8lHmlKDJ8HHDtAZVZ9s2OmCJ1VRO6gW70qKxAJzF/GA2L9Sm9jwQ0RWhwQ0uWFagoyEybWZarObOyloDQzOBCJPrAmATlWuYl7eGA1fUUAO5xk4g4OkdW1AQXmRUVyot+o4qS2XxBPzhCUqj4+3hV/9OHmhyAYpwRZ7GKUx7fJIzqSWg9CpqfunKPryktOIyUpkDPCckWEUf48Tg6k+GjS53kP+YxpSPLm0fR/YjcSOnxC7KfVJ90M6qp4/iIn1FMTuIJfflRFwYlKEEQjK42hhQNgi42IjZnFqWCoMVCnD4HK4bWccTajEorkvdIJkwgK3KMtfzwmx/4g5/9lL/4N3/ON199hVUw9gNJJw67HW9+9Z7PP/uC25sbdDL84q//it/+6nv++f/sv8QRsFUhKB2irCN0IisLbGFRRk94Lo3NZE+WZRlqwrmUZfWJIztNxkRFbvOP+tJxb6gmN3oKoDTWHHHLk5CuIEW5nh+v20cB/SgcpyjO70Nz4PT0lMPhwOHQfHKPCFYvJensLMvy0dShlOT0PHYwTO/3eJhPULDH7Kssy2jb9jErSClFc2jQWskeW09uciVrsNwaweT6RNeOGGNxIU3IIZGxjTIirKtjISZN95yYytDTvaVE00iPvPZPjkc0spryJPWkBB4F96PoLi9JLkB81PyOp53SR23497Gi/94i+s//8VdSDfTS8kkEY3KUziWwQ0l7RqkNmbGoJGJOSGLpN0aT0BLmR8QoLRPa2BO8CMSvdGQ/PPDdu7/gl9/+JcPYE5wMcIBkNctVybqeYQbF6B3dOJKCoqhn5LMKYyw6epQfUFEWbiQv1e+g+H6X8XYvvL+jwD4EOLgk7cApfcQQJamXqOmDilFRRqFC2QQZhiYFuuk1EqiYpt4WEeOTAowhKkU0iXHwmBgn93R6nOSCEquySom+dfz223d88wc/ZzGrOHKMYwqQxD02+snFP7GrbGZRWjZwVhlUDPR+wIWGcRg4WZ5ze/+G1fIUaxUuDQQi1pQy+aIf2UghiDTroiMkcWaF0RHcCAWsl4rVsqeYLel7xWKesVyV0iadSZteZivKfIayJSFZEZeSwiqLzhYU9cjSjWxubsUVlmlsLoGd64vnvP7L/4Gf/tGf4LyjbfZ0B2nrdQH6MTKOE79+mrQlAFYmoBQlSPSYvXh0oU+XBa0iKI9TCQ/oJGK3MXL+UgFLIr5HLYGeLjBOgrvSCWuPDzppedHToI4xMQyBdogMLuKjcNpNlPebohRkhOk+PZhM4tjBEyNEuQ2ngNLIGCYSmoLMQpFJIUc93qgKq3+P0f6fOYxKwsk3mTjMk7j0TZ5RlnJdo9VYYyC1hOAwmZ44tVLMMUUks5YYxR0sIRoapUSIyTJDqRKjG6XFxypSrimKkiwWaK0IeUTnmqQCw9ihlKXMa+pyyRgcqd/Sux3KgkTXjUQCWltyMnHGa8iKTK61DWgtfNgUclQqMfMMP+8JaaBYFNhCE9SxEhxI0UMUrismkJQjaTsF1yaSFve8IqGUJaUBnwyZUiQd0Zl0IwQVwUr77zgGGDOsLVBIeK0UdCRANoRIDBptcspS8B5d13Bzc40bBmIsWMxmxBTJylyyTKOCFEghUJUzMmtJweOHHnJDDCM6FRIUOFWtXe/E2a00MUQymzF2HaaS9+EGR1XNub25IThp0zLaSIdB0sQ0YrMRaz2hTBMySeafosoxmZUWNmm5eXQSKMThmmcFy8WS8/MLXr58ycP9Hfvdlmo+wznLMLSEMQh7XCV0pjCZwmYamxtsbjBWU+aVuGV1JCs0JtdkucFYhdWFuEStBOcpLUJZCAalRVRz3jGEyKgiWmfUmQTuGR2wMYFP+OToXEdrGnyMjHFkSNJpYLJMWOde3oOy4oZXWh7Xwu3zgndQDpdGJpIfigyUOJPV1A6qdMAWSjo8jOB7siD4Cu1lXP3Y4+RsTT1k7Js7kgps9w+UxYyUNNoqbGGYLdfcbzz1fE6Wz7i+vsaHQFJQlAU3NzekJNgAYwz1rKaoSjbbaxbLOUVR8OHDB54+fUqWZRSFYJ+yLKOeVah7mM3Erdf3AyFEdrsDp6cjT55ccnt7x2w2JwQJN97vJQjv/Pycb7/9lt1ux3K5pCzLR9SGtZblUjjESivarpnEOvlM8zznbHnJk4snKCX3sA+etpcFYHsYqOs5eVbgnAT9NM2Bvm8Z+o6mH1BZRpZbEZD6DugfnSbWCjLKZpY85sQ4hTynaZ6PkbwoGIc0BeRKy7I1OSlmUigIHqUylM7Isor9vuHhYcswjtgs8bDZ0DctQ9/TdB39EMTRMY74EDHWTHiEhA+y2SmKXJxVoePD1RXGWJarFX3Xkpearhto2wMmSxSFBq3QRmOthC4mPW0OVSQvcxbrNZ5I1ANVnYPytONBHAXGYa0U7JRm4sXuiF6KYiAhdd6rCQ/gUTrD+YEQHHVdEaOnbUdAkWUW70fGsePy8hvyvGC323I47BjHntPVCc5LXkOWWW5vb4R7XxTc3t5O+JaR1WrJZrOjyCrG0fHmzRtx5T19ypu3P9A0LVVVTO6bTub9FBmGkb7vqaqKqqq4u7sjhClMcSbjInjPvJZA3XEcRUSfzeWaT8HyzgmD3Ey8yqqqcaMjzwsUmjwruL25Y32yfnSE/dhjilBAqyScR22I0TP6RGo9RZZhCks3OpbrgtjKZxRcIi8VubW4EAkuUs40ffTUq4L2vYMkIaVZnjDGkReJEAZslkgOydrxgRQyCAanFSaXvKQcgzGRvAKXFFoJbkIbBUgHWZ0FotlRLxU+RdrGUi8rSJG264DIF1/OmdUZV1f3zBaK0hSQAt63OBf4/od7lArUi5rNZoQgxaDcWtpOguyzLFJYS12KiYOkeXFZMQ6CEEra0HUDi8WMZh/A1xT1khQUKY6o4HBNj84Tzjd4r+haj0oDOnma1nN9fc1sVtI0G4xJdGND0IHV2SldM2JsQV3lEBSlzbj98A4dE6SSdrvn5v09u6aXrlebMfqBeeZYr6S7YVadUZWWtu0YXIPRifXylLev39B2BzYPt/zBz/6YGASnMfqeMQyYrKJennLY37PMSnabO+bzGmOgbwUHM3SJIs8wpsSiiWGQEGjfTfOpksyIocdoaaufz2rKyzM2m2uaw4EQR7pug3c9Rke69oDWOUppwQUS0UTyXHM3bHnX3XLx5Tkuwes3r8lNwczULMolNkYI4mhr9i33N3esFtKe7p1HK4PVBafrp5wsnhHKLZ2/J6s0d7sNP7xpUMZQBYPNa9YZeNdO78XiwsjyXPP+4Zr5vGKdG+bLgqquub66pfcDoc2oFgvaruHs5ILnL79md9hzd3/FciXdWb6vMErzxecXLOY1azPjN795jesD9VPL6w8f/l7BoooCN0bKYob3nq5vmc8rnC9xrp/2fobLy0s2D1tCzEEpttstNgm+qSgqfPBsNltigMOhIysNSuWUVWAYRxFEnKEo1lTFKZvdBw6HhvlckGNd3zF6yUBou4bMaDHBTFk0TdOglMW7RDcICsMn6dTFKGbLmuXJgqv373nY3DCfL0hIN9fV1S2bh5b5qubsyRKtDevVGSfrE8pa4cIObWbEdEXftpT1nLEf8H0gRMPdZo/JDOvTOUlrqrpi9COzWcXmXibIEB2HwxbnB7q+oWkO1HlBVc1JyTKrS4rZgvvtA6MbGFxHG/bc3F7xpxdfEzEU1Yw49vRDj58ylWazGp2g2e+xhWFs9tiq5je//p52f6DrYXVyyRdf9NjlyG0zYPyBuB+pl2vsokbn4G3i6r7hj/7Ln6G2e/oxUq7nNN1ASAaVVeSLE5rX7/A256713Gxa7lvH3a7j9m7P1bsrbEj0fYCoMUqC4ze7Db/4y7/k1Rdfsjw9ZYwjTbNDMRK8pyxzYog0zcDD3cDdhwe6Q8+71wde3M4x8xGTFTBMXbbKsF4u8EPi7mZLPmE4Rj25F63CqgxSjikyPtxe8cXLl6QUuLg45f67B5rDlnJeY6ymHzqU7whjRzWfE6O4kclyhnZAaU3KRlCBvLSgwNRzitkCj2HsIAXBDAtCJKFSoKpyiqpgTAlV5Pi2Q2eWebbEOQmVT8HTNs2Ez8yIQbJ/Qkj4wQvTOSpUEGEteEdZ18QEs3n1kSH9ox/e4oAVATv+zlrgWPw6yqYx8vH8juLohOGa1PiPf38UWf9jN7H8syPWhcmIJ4L7R7VbdDthfoMI/enRAX8UK5kE1iOSXX7gBH6J8vWkFOLxkzcUwqR7pQn5bC1FNuN0dcF8viAv8kchN4bwsUNCi0HNmgxUQnlFMIloDNokylnByy9fMa9KdjcbLl5d8v7+A1999ZLBj7S+57ff/xaVaS6ePqHrer788iv+r//n/wv/2//mvwWtUbkh5RPtQU8FCTU5+wHnHRbIcsFFHl3kZtpzCE5S1oWCZtGCWJ7yG6y1j4iXGKUz628eR3FXJ/34Oci9cMSfwCftCCilGcaRzcOGtm159uwZb9685smTS46horPZ7NEVLx2Hhq7rprDwKEYv5x7d9Mf/L/eIerxnlEqPTvYjcjalRNs0iDMebGaZz5bkeSU6VvSkmNBW03bStfRx3Ez3+GRo+VgYkkKFGJWOWiccazTpb80Im9z5CLpZTB5pGhofg0ePqKEYvVyjyWStOOJ3Pu28+Lt1td97Z/6H/3QngqLXBG8IIUM8v5ojazwGBFmiFCoanM8IURGjmdoQDcSS40BNEwuWKG2hmkQ/XFAsc+rlipu7B24ftjxsN7ixZbma8bOvf8bT08+xY0677ei7Hm0yZssTZrM5WWaJ3hH7BhU8KkZUdAx9g9VQZTnROfrDTjihMVImz9J7YZlGhVbg3Sg8WqXFgd+1UiQYOoqxp06JYpq4xin8bbJ7TuJ5kqt4/PM7l0YmLJnq1CTWy1e1mHn44VffcXt/y7x+JZUdjaQBpyDBIUfueUr45Ce2rrS4K60JfqQf77G2J6lIXhu2+3vUIVGVFUVdE1JHpq2Il14Y3N5LeI8xGgpF63p2XcO8WmKjLO5XF+doM5BbhTU1dV2I0GoziqwgMzOqfEk1u2BuV2y7O1oOpFJjdUHKMuyqoL++4smLnM31B1mcHHak88CTJ0+o+SNya8mUo0wj29sbmrZnHD3DmGhHg58+85imAkaSISQO9CgtUknusZCQQodKkiFoIqNJOAPFNElrbdEaYtTCiE+CDpKBJwGlx0BfOzlMxz4Iy1+Hx8E5dIluUPgornljFHZCxaQgqJAwcfQTSa5dmibPNDH5p8HuPLigSRzbsxNGT+/nk+e3/lsDKH7/IzdS6MpsjjEZQ98TfERbUDGSGdA2l2abokAED+HURvIJC+QfH5jHR4PJ7TRHyGSmlWaW1ZAqxuhIuWFRLyhiCU7hcTgz0MUe73tyU1IUJXU1J0+RMfQ0LpBpi0GKeWmaOLUpMSahtCM46TgQp3aGQkPIsabGpIyh7XChQxcKlUnDlAsJ7yMZSh6KKqJxGKMwRqqikUScgiqiStjJxRnUSFQimJhCiytTCcYDLMkYsrzEU2GUBBurFBl9QKeEmkI4jbGEFEg+0ncdu+n+83FktVzI4lIJg17FyeWdIkUuwqUEwniZJ8YeHwqUscQYKLOSvMzxo7BCg/fU9RzvYOh66nomIo+yVFVH3w/k1kzYFkuRl8znKxbzFbPZkqquyaqC2XxOVVfYLJuEZA0qEdUxaEY9PuyPtoU0pbx3fc9mtyUEcZgwYVxiEkyGYeoMmerIMUVUCMTgSSqKA73OMVLHRalEri1lmRFiT1nK2IlkRCxqKrgMXtM76QooMisCovJYA3luqVNJL3tQfPQYHA5P1AllDZkqKHKNd46QgnQtGabFrDgtpTNBHLpymbQUz7wDozDaSodGknGN0eRljjWgdCREQ0olVk0YtR95hDDi44D3I+uTNQ/3G8pyzmK+kIVUaVmtV8zdgsVixWHfAYqz8wvKosB7R5YXeOdZr07Y73eAJgTPxcUZJ6cnXF9fc39/T9e1hODZbEQE74cO22lSTBhtGQdH23TM50uapgNE1O26nvPzC7bbLXleYLOeGCInJ2u2uw3b3Yb5fMZiuRCHfFnR9S1ZnnF2dsb95n5aACZmsxl1NWe/36N9zrPzVxJI54TRq4wUJUc3sl6f0nYDzaHh5OQMow3f/eY7YoRiNn8M3hHHufADj6FTeZ7Lxuux1VCeCRpxBWEM49ATvPCW60WF0RrvWx420sronZx723m8CyRlyIqapA2967nbbFnWEtIpwedKurCCFC4jamIOM/FFpeyf55GyNjxsbsnLGTaHeVYAgd2hoao0mTbTKkRjtJ7CyWXRa6zBZhllXTJbzHBpZNduqGY1Q9+jjISa3tzcCMogsyRyssmd07YtRZFPQnk9YReEUZ5lhpOTNWVZ0E3c+SNf0bnxsUV0uxWGeFEUrFaCF9g3OyAKlscn2rbl6upaWnqLgqY5MLqRly9fMg4NmSnJbM5D9zBtbpx0i7mBLLMMQz8J5HM2mweOQv5m88B6vaJt5X6W9xQe7y+t1PTeKoQ12VDXFXU94+3btzx/9mJy5iS+/uoblNJsNhuyLCfPC05OTrm6+oDWmjwvfvTYBgRz46a21yBChrayCQw+0ewG6rWdEC1TQGoneSfeBZJ35NZiNORlJgXgNLI+K+kOPeiEyYysrafQ0MzIOnkcHGWR4cZIP/TMVjN0LpztEDSFlTyeMEowVIwJHSEvjWQjKIjJcX3bspwbojXEUNM0HX2vuHnf8vRyzmKe8/TJOfNVxfXtNfO5bAqtZeJvBrp+4KuvnrM/bHm6eEJVF+yagT/42SuWixWvv/uOFy8/5+uvvmG/byiKnO32nh9ef0fCy0YziOO3ymf0vWT0rNeneB+IWWDoe1xeEnNP2+4lmFoJBu/+7oHVas0YBMWWkrjeVVL0Q0vta4qqBh1wrqPMC+5v31FkNYu54h/9gz/gz//q12z3PYeuoawqzk9r/vSPf8LQ77j+MFLXOR5FUhl1VQpSxY1kmUVp4QVrlTGfLQhxpO92aAJKe4ahoc8DqMDD5pYqL/ChJ8Q4caL3BN/KPVLkDIM4P8uylOdqnoGqIUqhJjEyjg0JCTI0NlHXJa1WzBZrhtdXZFayUEgBa9W0jra0Q8uhafmwPzAECXZ+crLm9v6awNf4oUc7R2EsXd+KYGtkLTBfzNA6EeLIi+fPWPxyxnc//EDIPFVhOLQdgx/RKSN2HcoWFIVhPi/Z7HrU2BJj5OLyCfv9lnEMXF89UNcl243kOhwOh8fcgswY5vMVv/71b2j7ln5omFUlzWbks1df8OHuA+v1gr7znJysqas5wWnWq1Muzk7pm+ZHj22jC66v3vDi5SWoQMLhfKSul3TdgeAke2c2n3F7f89mvyMvNJ1zVAZIidKAzTWHdkPndgx+jx/kfldaxuksr1HRs7lrGLuIH4XVK6KXw/meYRRmbwKUseRZiXNJMjeiFIfv7x/IspLMJFSpiHjqWc6zZ0+JPnF7syMvFpydPmX7sGV7/0DfRoYugBqZrzN5hkbLerVmdTKjG3J87Dk7dVw5R0AzDJ6x7ajLGaU1gsW0esJ0Kfq+lc7K2YKQNJicfXtAG81sPnWSRMmcKq1GR8uynPPy4pLvP+y5b7boRcFVf8t/92f/H/70q5/zzdNXzPMlaqwZhw6lJTiY6FnWC1x7IHQt7998x7DZ0R5aDl0iqoyfffGS+4f/wDrXDGFkjHB5WrEoSxbzimHoyMsCr2A1X6CTo+l2nJ6d0nQ9mJpISecM13cNYx/phpHtpuH2ZsvgAy+enLKcz1nmAZtG0tBhq4JZWfP2/Rv+avPAz/7oZzS6RWuL63vyLGe/eSClHBcT3//2ew77B4oiYxh7rt/foWcj+aziyfqc4bBFx0hmS8BwaB21yVnOSywFbujRSvBpm9Bwcrlmt9vgssC77RtWcc16XZPncGhbcmNwyRMIRBUYxp7Dbs/5yRmzoqQZRsGyFCUhRWazmqhEL4puYOh6+u6AiqLBRCXr6L7p2F9v+PLzL3j/cMc4DrRNS5HlWJWRvKLreoqZxBiO/WFawWlU0BS6YNvsqecGHyO7dpT9q1YYM6DMFNI49n+vZ/dR+xZT3bG7cdIt1ZFjzqRNq8fXlJ741ApSmMTqjz8VraS4dcRUPOIq0kdh8FEg/ETAl+2mPL/VZNSTLdsx8FIwtGIoFxxxUoKQmZRIUTqPBtKkSUpy3pjQtXLCUvgI017J5gVlWaK1kXWxFwe6UmLUUyo+PufFfJjQmSFOeOLoR9qmZ315wmxeUS1rqnXNv/x//3/phhaTGd6/f4udFfzxP/oHqEzz9Pw5f/HLv+Szn36F05G8rqAwYA3aaqxRuDDgfCRXkBUWbUSEHqb7XE3mU9THzC41dQ3oidEfgqh7j8GYWmHsR77433YcHdhHPnhMn3x2xwLJJHo5N+LGkaZt+cUvfsF8Pufu7kGCxSeH/H6/RylFURSEECgKCW8+iuvHvc2x0+FogDsK28fXj851a/PHv4sIL2aZmAJ6kIDpqppjjGUYpaiafGK/bR4JEsdC0OQBmc77WBiYEEAqPhaZ/sYn9B9/ZhyLPHEKbJ1+zqQPpsc/YkhOSbo+9CfFIBWOY2D6+b8Hz+X3FtFffbURMSImYlDEqEWYQjZyMVoZXAmIZnLwKoJXxKCJXpOShWiIyRC9IQQR5WOUr6WosWPP5SJSXJzwrKt52C152M5omj2Les2ry884n73EppoYZHOudUaeC4dbazuV7QIGRNBHVEs1TSYxRYLzUmXyQW7OGImT2ApJBOsYiD7g+57Q9yTvGA87mt093o9s+pbvbq443F5BmG7CyRmOlclF1BO5ICYmsqgwUivhCN0JKDyiwZRWc1Ia1plibEecD1itZMAaR3Qj3o84L+4upoHroyPGqa0nabrhgUP/Aewgi18zMF+VxORBe2J0pBSFI5sJvzgvNFrnDNo/uulG77m5v0G5xJOsIy5ecHr6nMW8pvMHUq6pSmFoZ0VFbudUdkFdnrJafEkxf4bdvGYcf0HKNRhL0BnBGWYXz2jefU9Z1riuJXqP8gfOT1es6y8pbMIkYWfpcklIBufF4T14LYPxWJGaHOIywafHezEl+NiKNIWFkuijFDtGNTlH01REcInOJYYwTSgoUJqIJkR5SAh3TNjgXZvECa0+csz2+0jTBVwQLpk45NWjaz5O7+vYZRWjIGfi1PlgFSKkKXndR8HVWKOQrEw1BeaK2P6xVv3jj2U9Q9oxS8FyJOiiwyqNRSYXpROlzTG1ITO5VGCtYQwSQHdkch2FlCO32Bhpt5JNb8a8nqOUYvSCxZhlBbkqINe4iXfcNw7nRwyalAZQDkUkMgIBrSDTGXkmU7AxNZmpUDrifIv3gtDRRpMZi81yUsyxRsT6rJ4TUo6yiaTFwRdSoh8dQUOKdsoPkPb+eLx3fAAl7G2lNcYaWS2ogAuDVMmNQmfTBiRGonNonUswCgXJRhG7oiNkkDpP37fCXyusBCwqYfF2vdzb3dhxf79BocjynJPTU3hs65N7Ii8K+mnD6YMEQGmtKfKcPMvpx4EiE9yKD56qrKS9q5qjjabtWkKMzGYLTs/O2e12LGYLQkgURcWsnrNankhY43xBUZVkuSUvBeNyFJDVtKD7+Nw7Yo3E7RljwI0j7eFAc9iTEKe1tdKu7ccB7yIk4fOOo8xtWRYoy0r4/CmQkNAfY5S4YZUwffNMY2wiMxkpedCJwkqHhZo6FbSXcJPE0cE5OYKVEbe8XaEHWSRJvsbkSkACWrXRMo/nFh+En5yULG60shijUdowjKBR1NkcrST4clDjxKiTQDcXI6aULpj5vCLLNC4MBC9FB6UD3v14bnLftxzaAxBZrZZ0XYdSTMVSSzEFayl1FOMEGVAWFUVRiJtW7ckyw35/ICXNfL7k+vqGzz57hXOOd+/e8urVK968EVRG3/dTq/eAcyIcSkr8gaIoefXyM96/uxP+ergRJ5yPtE3HEZFycXGB8yP393fCwyxzCcybOOvCyJf5ZblccH3T4n1kVi9EBGlHLk9eMJ8tiCHRdT3Oi8NidCM2y5jN57x7d8XYj1w+uWS333N/d8fFxQV5VT86QUIM0q2kha0ao7Q2OncMA5Yw2OAlfFcjeIvZbAapInhP9A43OkJIVPWSlJSEgY6RpARXZbOask6ETkwF1hh8BLQhIq7BYRwAabV0YZw2F1EQXMkRwoi1hvnM0LsRH1u6weM8VBUUc02eG1BT9sO0a0tMhWHvqOYV1XxGPZsJazS39HFH1wumoawk/2EYOuZzCRcGEfK9cwx9L23aXjimh2bkGKLknKNppN10sVgwjj1ZlnFsNzXGsF6vubu7w3svPN2J23i3uaMoRHTWSnF2dsZ8vuSwb/Dec35+wfX1Fff3D3inuDifsVyu2Gw2eBfpOuH4N82erm/Z7w+UpbAtN5sNdV2zWCwoipzNZsNyOSfPc5bLxdTKamXe7AbatuHs7Oyxu2I2WzCfz1ksDuR5wTAMjGNHUnZy/kih8vT0BOccXTdMm5q/30Y8K3J8HCQARiqORJ/AR6zOGUMk7QJ5TGgsfRfkukcpbAv7eiQvFOXCMptlBO3QpUZllsN+pKxzQBE8VMUU2p1k3pYlmAadoYxCW4+PgaYdWT2pkG5JJ0WZMlAUgsgpS8M4ek5PlzSHkbZxLOYl1uTsd46u1XzxxXPGfqBtFL/+5R3/03/2GW3XU5S5rFcyK2GE/cg6L5jVJdvthtH1zOc1bd9T14YfXn+Lc55f/+Zbvvv+A2/ebPlf/9f/iO3uXt6zsszqisNh4Nmzp9xePbA8P6FtOzabHaaYMcZRnGcp0PcNRhkJD1OG9fqczXYrn6mSItpqveLu7o75fMa33/2admh49dnnks3QjwRvxWk3bml2A9989YQ3b15ze3OHVRqdAst6wbOLE7r9PWerM9ruQDIanxQp1exvbvjuu29Zn6x59eoVwcuYDDGQZRIy2bcFeZaRGBhGh800TdORwkBC9gAmaDKfE6OVOTvmFEVBInB1/YEYI8+fPyPPLdvtVgpQznN7+wFj4iSyDvTjwKFNlLOlJAtP4VkxCEoEk7HtHN+/uaJpR7J5yRgj61VBjA6P57sffs3ZfMk8K8irnLwoZJ5o99jMEpLHGMU4dATtef7sCf/9X/2P2GXCJ81mf+BkeULTdthq4qbvD1SFoSotZVnx+RfP+PDhmqIsp7E6QtK07YHZrCYET5Zb+qHj/qGj2Xtu7g+UVclyJUx87+G//xe/5B/+k5ekFDkcdgxDi/cjp2drtFacLFe8ebP90WPb+3HCUQ0sVhV5nrPd7livThmHkagDdSUsfJtbHm7vOa9PGb0nDDsJac4Th2bH6Hr2+3uMgdkso6py2v1AHDTLekFm4GGz49A2eB8mp6lkZRgtge8hJRSGmDR97wnek+mCZ09eEINie9tTmBKVRcgM1Tzn9PSU3OTc3Nwxn51QliVFZtnv3kFIWF3i3YDWM4qqwGI53O/YxUjyPe1wYIgdKk+URUmz77Hk9L2jKBKLxQxbTg7SmBhHcRa/fvM9Vbnks1efc3/vaHtHVdfM5iV5nrHfHxhjYGwbCh3Z3t1zsl7SX1zw9uYdTd9hQ0Pfd8yrmkrnvFw84aSaU1dzkoq07YGh2ZNcx9g29NsHmod7Dg8bgo+UqmAYe07yGX/86oS/+M0Ng8k4ubxgnnuKLFLOCl59+Rmz1Zp90/LkyVN2mw390ErYZ3fF6BJt50nJ8r/5b/53zAtFsz/w8LDj9Zv3vHv3ns12w/3tW5599YzcJPw40Lcdxmi++vwLPly95cPb37J8seJw2JOrGYXNyZR0pDzsD2x3D+zaLSqLLJYlRVmwbQ6cnJzx5GTF273Mlw+7e4piwerslMNhz8PDAyrryMtElkNZFRT9wDC2nFysIBfh6nrzgUW54LMXL/jFX/+CZb2kKDVXt1f0o5PuyzGwWq5ZzufUtXRu2WJBUOCiZwgDD/f39F0rXeQTLiNTWq5nCmRZwayoMAFms4rgB6yG6EayPMNmBeN+wPUOHyTrqe8HYgzsNnuKvCSpxC4MLJc1LoxE57EmZ44WxGUv+Tp/r0OMrxxN5ZM5nEfe+NGIpNIjthfUZKw0j27zj2r70cgp+XmfCuiKj6LoUS6YktymTvhJT1GTgxf534+Mbsn2O6Jx4ahtyAkc96FH3EaaCgHpd5zqxzBHjbaSPZHlJVlRSHeAcxN+NIrxxlpQOUqJYzkGRZ6ZKbPNMwYx6rjgCDqQTODZV88pFgX7cc8f/ZM/IdwdyKxhtV7w7v6G69srvvyDnzCkke/e/cD/6n/5X7NvOi7PnpOMmsLZFTa3JB8IURzX0pHYU+QlmcmmvCX9KCSnI9HAGowR1OlxTc2EvvPei1ygzdFb+588QvQSOKs+suwf3aLTobSmaVvu7+9x40jXdRwOB7RW3N7ePor5Xdc9ZnOklMgyyeo4fv1TsfyIXjseZtIYnXOTaegj6x14FNO9D8QkyV6Hw4GymDObzVFKkJsxJNpWzFHpEwyQ3C9JNDQUn6JgH09bf3LvHu+5v3F8+tJUb5r2NSLax3Tc70yoHhXl3pw+Y7lP46NjHRAB/u84fm8RvZ4fvdJpOnOmi+mnExCuZ+IT8XKC6adHsXMSGuERZRFCEpAsgoEIPuBTJEZDDAWDVwxjzjieEIMlI6DTHTHsSMlAsKRQQMogSUJpDIYYNWAkuCdZ6XiJMhlooczKe40gbRJKHBZM1ZGUplYSEdoJwtv048Daj+Adz4eO+dvXxL/8N1x9eI0fR5xKOKvA6I8VPAU6SouRzqdAOcXjIBRWgkbrJN1QQBx6uu2GkEQM0oCKMmn7MOD9IAunrEQCzkZSiiKCu45t946QOmGeGktSPbNFRgzCrUtRJkhjtLg75Spy9MfHKO2b3me0XYNfnFEszjH5nJWtWMxPGXeNYByKnDyvxH2eLanyJfP6CavFK6r5U7wfudv+Gq0dKRoCOb1LjDGRz0/QSZGXC1JwRH8gq2fk1TmMB7Q1nFyc8Pzrn7J7/UvadoMLkdELr1w2Ex/F9KmQ9bF1g2NFVCaxoBVtjGxHcAaGKPSdszgFOfaRdggiXEyDOSSZvEeXJjetlFm9jwyjbBAzk5GiuJnbLtAN4hQ/ivkR4bVzbHVKxyCDo0gDY5THnTUJa/TkdpPvO7KcjpNKSCKwH8/y9yiY/WeP9XItqCEtgpds4iIxOVRSJBdJRgTcGCJVnpGmqqqL42MVEz4mQYcgaBRlFcbIYzzPMgk2Uprc5vjQo2NCm4QpJASQBoZxnMZPxMeB0bVSTNFRuOcKaVO3sqnNbU1hhRk6DIjL3QQCEvyqlYQ4BgJZllNaS4iGkPzEXZc2q9z0jES8k6AatMIFz+gdIUpgcm4tRh1xDlPLU5T3GZLgfZRV5LaQuSXpKS5DCjJo2V/aPFFEjRuTsKK1nVqM5Ho650gq4UaHMZqhd1hrOD+/YD6bsd9vxdlnFC4KLCQrclAwOCeifwxUeU5ZFFRVSYiOwgpqY3QDRVk9nkOInkNzYBhG8rykLGqGMTGrF9SzORdPnnJ2es5yuaIspBUzKi/IpOnRoI7sMo6wmqkLJ0ViCHRdy/39HR8+vOP16x/E/RkFGxKDLL6UEhTHx9ZDsPYjM0/Y78IR1AYpdGhBZtnMYDNFxBGiR6mISRJalJtcHGwEfAhk1hBSnK5NEEajl/ukzCpGH1BoMluA51EsTSkICstKuG4IHsaJPaiks0RPoSxGJcBQ5nOKvCJER9c3gkMyVhYPEcoso6pKiiJDwjEd4zGdXAnm6sceQUUG12OsbDhMntENPUllgiUpCvaHBq1k3hn6njwTRvmx4+nkdI13kg2x2+24uLhgfbLi7OyMP//zPwfEdemcJByWVUHbNlRVxWKx4LBvqOsZznlubm6o6hkxJW5ubvnii8/J85JhcOx2B+bzWjbaRcH79x/YbLZUVUXXdY8LQGGiyzzz7Xff8gd/8FO4SbRdS5ZlNO2B9eKMvCjYHw64/r0EF1txX7veU5YlfT9wd3fP6dkpJ6enDKM4obuuo+g6kplQJUoRQ5zaMzOU0o/jn8mdg9IED+MwyPyUz7E2YzarKHPpAAneEaNje3ggkejbnjyrKcqevkvoKXzWek+OuD/9OOJHxxAGAgHMxOckoULASp1HQueUcAW18ShjWK0tPlrqlOFDQOkgXRphJLqIngKuPy101bOa5XrJcimFMhA0XzLCqpyvxI06jiNZbicXi9xr4zDKIl5GLnkujNwY0uTczskyS9Mc2O/3VFX5iJE7InryPGe1WnF7e8vFxcXEeDyw2TywWC/YH3ZonZitFqQknStlVbHb7nny5JSUEvf3DxT5jL4fqGtNURTM53NAii5nZ2dcXUsnZZ5LnsNyueT09PRxI7LfH4SpvzIUhZzHarWiyAuenF9yfX3Ncrnk9vaWly9fTUgDzcXFE66vr6dA3B5QLBYrnAs0TUtRlJRlyZMnl9Nc++PzDoDJxZemTJup8zHKHCybPIMfB0IbcUMgBbBGAruPyJSytqxPFuyaLZfLGcENRAZmq1xEy1wxth5tEtYo8jyTJbf3dMPIk/M5X33zindX7/BJsVhntE2HMhmj71FYYoCqFCOHBAdLrkE/9Dx9Nqc5eMlMCVKwOjmdE2Pg+fMnaJXzQ/6Wzf1efndKZFkxFWahrue0zcC791dkmabrWvLCcHo2YxgbQnRcPj3j5mrLq1cnwMhmd8tut3kMqTImY7EQLFM/yDxfFILhSZm0aMvvDoyuhxCxWjimZ+oClaSjq2kaFosFVmuKTDjSoxu4u7vh/OIUoxP90ElQmjWMfuTtmyuiG/jJ15/R9567hwOzxZr1cs7Nh/ecn+Tc3znyShE1RK1x/Y7msKMqc9pGwv3yzGJNYhwa3DCiVWLXbKhnNSEODKOiqitMJuai5B2yh/NEPCglGUxDmMIAkwThbre8fv09dsrLub5+YLao0SbQ9geMkQDrEDxlueCHH96idYY2FhckQ0DZRFIZh9YRlcFMnat5ldP1DXsfKG3B4HrGUDIqWXudzk44tA2ZNqxOTuidpx96cJ7b7TXB9RRFzq5rKSvI8xqUQZmMBDRdx3pZsd/eiwlneOD6ds+szrEpo+97VstTNpsH7h8eyDJ53vV9Q1lUWG9ZLte8/bBjsQIfHG4IfP7ZZ3z9ZU6we3FiakfbH+gHjzGR65uO1WrF2cnpjx7bWZ5YrmpGN6CYkWcl47DBjQmwKJNT5pUIPUbjxhE3jJPhTUKau17GWfIRN0jXYm4K6mrG9qFBAn0jbd/howclCLeu70UUMpJztFgsGaYCqR8Dfd9RFXNOTs6o8hmHfcvzZ09xo2PbPFAVOU8uLqiKiqv3V4yt5+nFU65vbnD9OBW8ZxgyilKerWVZU2UFsXd4N/Cwabh5uKIfG2aLnHm9YNvvSUYz+kA/Oi6fXRKNx9FjjOJwOAAibHUPAycna2KQuXc2r1jM1/T9yNAHEXa6LcXK0HQBHxrpblSWOhXMU8mLy6d8fvGc52eXnFQr1tUcbRQxSg4LswrfHcgImBgZ+5FyCITRU2hNyMXk9vPnJ3x++YT7xrPpPDfbHZ3NePL8Fbvtnu2h4+T0HKsylMqoF2uUySREsRv44fVrlDHkVTWh13pOTk+IIbCaFXz767/m6ekznpwu8d1uMgs4cAmbjDyrtjfwoHFeUSyWhJjwMZB8zzh4UBHHyPmzFetnBS8/P6H51T2FNfTNDl0kstxw2LfcX2/5+c//CHOjePP2DYt1DgaquiSGhNGacey4WJ+hidgiQ6O4u7/j9OQUmxuyPOPi9JxD2zJ6GTur03OK2RyvIGlNWU5c/tGx2z7QDR1KK4a2IzMGW+ZYo1mUNb33rKqKoR9ZmoK6riiKGSfrGc1uT7dvMcnS7HuqOsOZyJgcMQWquiJGMRR472maA6fnS7SB9XxO33eEkGQPETyEQGH/fs/upCQXLR0d4kexgqN6AZE4OWYtTF3dKTLthyfkCkcXuHzHR6TL8YerR+3h+Pff0d8/vqNPBNsp4PGTf6PR4jyfclfStO8zRk9GRv0owigFRy66mAbDpKuJ+UmpREqSC+NGRx97hgmVRNKE4+84/klJOiSnDjoXPcFP5r3MkqeC9XrJYb/Hm8iLrz9nsZiTXXiGtqM6nXPzZ1v23YHeDfzw/Tv+5B/8KdEqnr56ThtG6rKW3AedMHkme68Q0MYKqnly7LthnLr6imlvILl41tpJOI8k5DVjJEj9aDI8iu7ee/K/BeeilSYyOe+Nnhzvio9+7d+5XIQQuL6+5he/+GvKsuTq6gprxQRibcYwDFRVhfd+MjZKQHxd149s9L7vH792FNOPovVRZB8GyWBSikeUi9wbx/Nx2EzWnCHI+AFx3YcguThi3MpQJptytoSBf0QCPXZJTDecVnoaH0dRWW7GGP6W4pX6+L1H3r9CCiITDUcE9SiGOzG8HcV2Oa9j4OhRm/u0v+M/dfzeInrEfRyM6iO7RvT0+Fgc+ZiIOg3aY7owShY2Shas4mJXIrJNE0BKihSjBApON2uMiZgyYrSCv4h7YtyT4rHiZSSdNh5/p55EdEOImhi0oGeihBfEoEjJCmImZfJ9SaMoSCEj+CnkIAnTNARx3hPFUa9DIgQNsUAPsGLG8+GMUDxIGnnrUVGjdI7RhjITB2eZzch0jkoKjcEoRZFlWK0JwQljbWhJfYNa5aT5E3FsBwnKSpODPk1izxhaFBZjC1LyhDjg/IjSA5vmA2M44JOb2i4dw7hnVq2xVtp5wUKKaBWxRn4+CmwyQu4tChblHIWhyEtMZinqEhU9Rb7m4slX7IZb8kyLS7U4YVauqco5hZlRV6dU1SlFVqITzIqaxje4MaBtwuYZGCsVSZXx5POfUs5PEF4waJOjippkQGczmu320c3sg2LwCRdg/LTEOVVc5T5RoKVyCghmZApsaALcdoo6JUYDLXAaR+ahoHeKIUgIqHqcICR8yQWpmEVERE/TPSdMW3ssqeGjCPzHZO2UJEjDa4WKxwLSpPZrNbmgYfQQdZoEQnnPLiTchH8JUdrDvZdA1DFMZY/0kcn1Y48iLwS1kxTFxPsKweNcL4K+NmilH8NOlFIYq/FT2K24Qn93AgbZ2OskwVUhBYyW9qTMZoxDQIeECz1BByqrMJnhyLs5tqONY0+Ttpgsm+hIx9CVMFXQFVYpMi3czVxnmBxCMo9c+hiYhM2PSJ68qgkx4LyDFKmKDJJH+ZzMSHdGVMj4swZjEpkJGCUOOHHpygNDHuieqCJGGapiRpYXEDXRS/gHRoTl4++LyMO/mlUSmjsm+qFB+UhWZFQpCct9qtQeDsLxnc9htpjjoiTK57m04ocYpgq4OBaiVClpuxbvHU3bUBYFeSZYHgmSE/Evz0qqWU1z6NgfGsoSsiwnYcjKOecXzzi/uGSxWFAWlYheVktORvAkpMVSIVkQRzyRViIsO+foegmGvL75wG9/+x3v373DjwPWmAlHJJs9rXhsQzNGk2WWLJ+wU8GT5RV5Ln0JPrnpd8n1yqyVMGbSFDCkyU2GVYbK5PJeoiM3OUl9LCDGIK2kyoqLUGlLlqTbQiNBjXFsSeEY4qOkzd0mXNODkWA/rRQk6VYw2mCUxRhLlc8oi5LRt0SvwVpko6clRMvmzGY1xqgJVTESY8AHR0hOxsWPHdtTcG9dFuy2jbynHGAkzwwPDx+Yz0v2+x03tx8Yx0Be5OSFIcSOdt+T5wX9OLJcnNA0B8qy5MWLl7TtgbKsaNtOAu6mjp7gxfV7XLyNzjF4h8k11Tzj9dvfgB7px47RC7aq6w+cnp+RZRprDS9ffsFut4dkGAbHw+aBxaJCmYjRdmp8kGyUu80Dy9M17dgyuB5tYX265NDu2ezuuDgpCEEwFM73KKsZ/IAbEvu+5euvviGqxGK5lCJ6iLSHPYlEmWWgJQ8iJE823UfWJLSKjN6JkBwF1yXuxpzlasVquaQsMhSRTjXS9eICVhnqoqIvK0EgNIa+a9ntdpTzkiyrQRfkuYa0ZxzFJaWNpjA5Ej595CZKV4FsOtTHOU4FygyS8vgpEDuhMMaTrBR8dUqokNDkaC0Il6quKUpxZOe5RdtEP4yoBC+fPqeqKq6vb7Em42S9njoYHEWRMwwjITqMzRnGgUik7VpiCvjgOVvK/WNzTTccuL6NrFYnaGsJw0AE1qenHNqGvCzIiox1vub9+w8ch1ZdzPEh4P1I34/M5jN86FgsKw7NltOLcz5cfeDy8ikxDfRDYLGs8LHDKMWh2fL08hkKzeXlU0Azjo4QAovFckLaJU5Pz7m7vSV4KQ42jfAmAWnJzSy7/Q4XHMoo9s2eLM8YxoGiKrl89ozNw1aKs7kEcwNc37zjxYvnGBsp0Fw8+fHBgwABL8xRCdqBx3bWRAoBm2UkM21KksZqxdA6VFC8enHG/XZH03lsqVBDoDm0zJcVwYwMY8fpWU1yin705IUlmUhInrEPLGY5Rmvy0nNo7yhKRbtzRG2plzPG6PFe5thi2vyOLrBaznnY3jOrc3a7llltWa/n+MHRdi1FoTk9q9k/HAhh5F/969+wXs5omobVmXQ7XF6ec3NzL5tNpcgyGXf7fcNsUQNgraYbeiCyP+zJchkPRRX59rtvOTs7ocgz+n5kHEcWyxWLxYqqyNnd7ri96zj5/AtC9NiypKpKEhJ4Fp1DlwUxKsaxw7mBqiy5bQ9UZU7wJU2zZ7a4ZLFY4tzI3c0N97c3lHk2bVoVbdvxq1//wHYb+eHtlr6XVvZhGLn6cANugxu3fP7ZCU+fnTL4jnwSiq7fXgMGpQXBlxYLvB8Z+pblvKTrDsTgaFvpukLl0imjDcQRsXcExnFAGwl9HkcpyOfeYrRmsayJybGYLzDWcHt7zZPLc3wYqaoZbbej70e6vkf4rC1EKcq1bS8YF21ppjDV+3Zg33WEpNhvHPUqo7AlSkFzaPjyxSuenF9gfeJktWZez/FDYD6rKYoanSVyY0lKU4ac3/7Fdzy/PKGOwty3xkLSzKs5LkVO1ivurq+JLqAymM9LaqDrOw7bHXU1Z7fbUJYFVVVxODR47yTLg0RRSuHk7Kzm+YsnaBN5//ZaQtNyiyngYfNASoHzi1OqyrDdbpnPV+wOu+PA/FFHWWu0rXGjJ4RIc+jRKqdpemISRFNE0bYt4yhdx8cOGm3Bh47YjczrGTopFvUSEvTdQFlFTtanxHlks9vQtweMSezbPXZ2xChIODZT8TrERMoL3Ohxo+d0VVHkuXRA9C3zRU30gcHvMQqqvGBz+8Cb334gec3ZyTNyk3H/cCcFvmnfslqvWJ2uKMuC3BgROEd5dgxDw+h69H4kVxluGPEaXAgcupbBe6rK4EbpLi9Ki/eBPM8Z8Pzww2/p24F+EJF9v28IPj2GmPdjQ9MZZnrGvnGoMVKQMdMlSz3jJ08/46cvv6RMlna3h34gxYCPo6wFSZRGci/MyYlkS2Ul27t7fNsy9gOHQ0NIGlsteDJbErqGtFyQLU+4OL/gj1++pFqs6HvHw8OW9fqUMYDNpOh8f39DlRd88dNvqMuCYb/l5v6ed2/ecvPhmspGxmbLk/Ml89KQzc9FI0FhrOXQylrmyfkF0RrpMNY5m82esLljtjxlc9fzw/dvaPo9T7664PRZhc4nhGIKbO6uoYaoPfN1OSFFEuuTJU2zRmcJY2W/27UNVieMSvhhYFbOub9/kO7GWcZvXv+azObSyWAyVssT5sszrq9vqas5cdr3hTBS2ZrMaLa7htVqziyWgpSYopf65Oj7llVe8/zsKfXJml/95te0Q0eRW4qy5HS9ZFWXxIuEHwJ//Ve/FqORSo/sa02iKAvKqpLxFkeKIqeuS+kO9QrnRrq+weiMKi+o6/pHj20AkvnovZ3c4PKf8CgoxqMWpiJq6naNQIpB9CSlZSxNLvV0NJqnj0K8UBKOyJiPQuNRjeNRn/to7EtMpqgJq/H4WpqUlulnpvTR9AdxCn1ME1N9Ev1TQKUAMZBpLabbJPjSFHp2uxv2ybFYnOBiKz9fKeTZlqNiQhlBSxH9ZCYV7F9mM6qswCzWaA17f6AqF0Svma9OUXVEdz06rHh+eKCqa1brM/6LsycobZjN5hitJl65LJ6zPCcvapyPGFsQksJmGcZkjxlC/TiQ5SUJyWCJMZJlOc55VAI7sXo06rHIoCc4hTpSPaJBTTlYCcnBs5mEkBpxiBGCm7SVScD4eHEBCfDtuo5/++/+Hbe3tyLO5yVt2z6aUrquoyxLjDGP6Jaj4/xoYjmy0uX+YLLAifHR+0D0nqQNRmlCkFzLIs/ZtB1d18p+1YMxsq8dhh5rM07qFd4nBjeACmRWYR+LT1qMSumIAD66xCdDGtPCdtKame63o9z1O3z+ybD6sYNDNGaNnpzsUzFGpUe9U/TH489mGjtx6paYAkr/juP3FtF/BzD/yX9phEMajyfOkXcbH0+aJLZ5uf5T4JSS70PL35XypKRROqGTsCeVMliOVagjFzZNlTcz3Udxcvv6acKQS3/EZcR4rJaZSXgXcVPc8cf2mKMYL6gZkv5EpBfeW5i+Hjz4KWA1uMTiiz0Xfxj5k2ZN3xU0h4a+T/gxw48lKq7Q+oRMn2LUAmKJVgVaGcqsxGo5wxgDfhwJbqAsNOeXK/RsyeAURS43SEwRH3pG14vgQ4Y2Gc63jK6n6/egAuN4j0YcalISFAExKSfCk2op8zUqKWIaQBtJVR9ECCuLgtxWWG2pyjl1PWcxX2HLJfgtqjCsZpeUtiAvMupiyaw8ZV6fUpdrjM4p8jVZPsNoacealzWpH2ijYwweWxj8ENFZRr0+JZ9lzGYLEcSO4lYmvNHqpOL8i5/xw7//l/ho6b1nCOpRXFbTQErq2N0wPTQUJKNRxkowSBABdEya6z6R+QhFos5gT6TxgcOg6X18rH6NQdH00jExBsG+fCyIqUcB2WgtrmbkvnMpTucij0IXpU/DTIJujIqogCihlyGCmwpJISlpfcXivGMMiTFC7xNjgM5D5yJDSHgNQWtS8eODBwGG0ZFlpdSutISA5qWENxZl/tjS44OXbVcM2CjssFxl02SmHidhafeRkOEsKymynGCEWxf8KOJ0FOfTOI640JFMojIzMqWpbQHGCofVRXFNG6l0awzOBw79jiJKAcGogkxLjkEIxxBHqd5bLS6zMhMMQwgDoVQUhSU5CE422U13EMePZmKDa0JMGJVRTOFaIY6kNH5cgKTjNdP4CIEBXZTiVM4yMlPgBnEHjSHifWR0XoqHk4syLzJSgjE5jIWitMQghT2jM/p+yziMBCdMVhQ8e37J+uwEYzVd21BMbsbR9dJhYo0sDI3BjT2uH6mKgqZzeF9Q5Dn9tmW332NtwenJGXW9nDY20mafkqaqF5yeX3Dx9CnzxUIq64XgCRTAVEg63hthDI/tekplE5tPnHe73Y7379/y3Xff8t1337J5uJdWNa2kkCctGwQlHQc2kzR2QQSJsJrlsnEsS0tRFQzTtXbBUVUTF96PxKTQRlwBmS2osprK1kCSjhhlITiikmeKGwdGN0q7cgS8wlKQEK5wjCM6aQpTkSvLvJpTVzWYRDs04N0UWCcZCMenY2ZKiqykrmYiJPqRmNzU5W4m5raEgVkzFaBClPvPifs5qUimf7zjpe8G5rMFkLi5uueLLz7D+Z6m2aMqy93dW7qu5MmTJxiEUexcoO93cm2sEQEtL9jtNjx9esnDw4YnTy5JCV48f8l8Nsf5kefPn/Pb337Hu3dvOTs/5fz8nBiFl3x6dsIvf/1X9K7l5m7A5FLNiij2h46sMFRFRt/v+cln31DPa/78L/6Kspqz221ZLheE2DOMHavVKXlWEZGC0ma/5fLpOUrD2ZNT/t2/+TNO12dcrJ4Rk8dkGq2g2W/ouw6UoWlHKeRmimefvSDTGV3TUVU1l08uedht6Jotq0XNbDZjNp/hvJ8E45KEhP+OTj6vth0o8pyLsyeCGZnNSCnSdgPj0DKOPVonrIGizHFjRpFb6qqcOhZgNpvR9B2jCyxWK4a+xWgzsTg1Slm0kWLS6JwEYIcoi1priX7K2oiRMhMeJUqchn4K0dBK3DEYWcynmCAFtJE254SmKCuKusRYcaZmxrCcrdhuDigsWplHvuJ+v2c2mwnKxXuqquL09FQyDawsgkfnqWYFWZ6R2ogy4iTrx571yQUJzWotmBObWUoqNpvNI75mtphRVzWZzoVrP60t39y9IRKYzRdcXl7y4cMNTddiipyQHF3v2G43ghEqc85Ozzgctjz50z/mt9//Ftd6ZrMFbXPAWsEVkTTr1ZquGyjLGV3XA4nVas1sNuPh4UGExhRBK07Pzmi7hrIqpnmo5vb2jnF05BN6Zj6vuboOIjLtBt6//4GiEDzD+3dvfvTYBkg6kVdKujqlRW0yAACI+2ixziVI3fsJsZhwYaTvD1w+XfH963v5rAeFwmO0Z3VREKPHj4HcFigN5czixpFintFuEtZGjCnQNqG0dDIpZeiGlnqek4wgKTKb4Zxnu+l59nxJjIqhD9TlMYcGrq7uOFnOub3bcn6+YBi3FFWiH3fMFyMoWJ+ckNSe9XrNdnugOXQi7CfFfD2Xwrw2VGXNYlXx4fo9WhtmsxqVDGVuuLq5IYSR+SKnnuWynlOKEAMoz939e/abluV8ydmiRhvpFNP1Aq0/BmwW1uKDhOA9bG5xzrNezSgKTT8cYOtRSor06/Up79+94+F+Q10V+GFgGHrGceT9hzsObeD1mztOzz/HlgrPHS8/+4JVFfnNX/97bq6vif45fmywVnY6eV4SgyNER5ZpDvs9ZnpGBj8ymEjftqAiVVYwOjeFzgobP0U/OcvktRCFOZtlGSY3HA47tBaTQFXV9END7CSY+/ZOkC4QKfKSvu8oy5r9biebVj+y3dxTlDXbQ4cpDLNlxX2zZSBickN7GBicQ5mKfdOyXJTooLi/vuNydoK1Bd2+58HvJMekF0RQ0gZlE5XOSFGR6YztwweGbMSajMPgefrVc2yWcX17xd32ji8+/5rvv/ueGDv6vqOeV9gsMrOSk/KwuePs7IyLixNCiLx/f02eS9eGbwdmTxY8fXrGYllyODywWlU8OTvj5OSc//HP/gXdMHB5eUrbOMbBURQzSAaUYd92P3psF4Umy2o6NTKOgRQHUoShH0k6gpaORaVbwU1ZS9P0qKQYRieB1lGznq8xeYadZ8Im329wcy9Bfjbn/bu39P2B2SwnqUTTtszn88nAmsinADrvPXmW43pHkeeoFDjsNgQXCN7TDp3gAE+WtMOBh9s7bj88sL8/YFLG5u6B58+ecX3zAe9HvLeUZc16veDJswtQAedHikozDB1Ns6Usc8o6p7BQ1TlVVTCMkdVqTlEVOD+ges0YRsqqmDpREtYaTJ3RND0uDKQkAdGbjccYJuzhQF5o2n4v/PayYnAjKMVqseKbyy9ZlxW//dUvmec1lS14iAljoCgsISZUjDB0mDDi2haVxMhl8oosmxHMAZMMVVHRDtIJq63hfHXG8uIZr1+/YXM4sD6/YLU+5fzslEPfs3vY4MYBP/ZE1/PVH3yN9563765J40BeFDx99pRXz56yf7hmd/+BZr/nZFlhTYVSYnIZB8n/ODQHzk/PcSRMVgtSzii2zT37/YHoFLdX97SxQ2WO3juauyAGr9FhNHS9QytLXhn0vEKbSIiOiyfnNENDRHAcfd9QFQWFtSTnafct/dhhcgMW9tsti3qFUknE3yhC5MXZOUVRgRJNSWcanYHSgcDIajZjefKUX/36V2S5pcxLms0Nvnc4O/Dsm0tCZqnLmrFoGaLj4XqDtZrnz55ydn6K1QUPu5a3b96TFaAsVHXFEZ1ijexvzs7P2B8eCGFgNq/EMe0dXdeRZ2An087f59BIUKqEGk7O8xila51PDajy92NH+tTsK+7do4X2E8Hvo5v2KKSLoU2M7p/0wCs1Icn+E+ehpjy+o1ExTe/oaFY/vjyFoB6FXa1EgD0K+ZEwoXfSZN6c3kcQRKpSmnmd4+IoeSljBGNFtPYanUClgEqeqBJZbkXYVwUF+ST0Orquwaop9DNlzKqFoE3bDjMM/JN/9j+XzIUpgNhaOxlLLdpYtLXYTIRnyYTQGJMLVtSJ6S8BdV0TXXwscOhpbvTekdlcushTEhxtjLhw1DQ1Ns+mQFKNrqpH42EMgXEYRdg1ZjJGfvpBP159PiUvrNcn/OpXv3rMFhrHEe/l2T2O46Oo7L1/dI3HGKWTdMK6SIeZeTRCqglPrCchOUUR3sdxpMgLHIHmcCBbrwHpFEdZQhgeNRGt3PSeJm1WCZL42J8uGP2jBivmtHQsIGnpSBfksfnkU0iTwK44CupKCf3gY/aa6KVKHfthJ/O2UlMuz8fg3Ucz+HQEAimpqVD0u1/7Tx2/9wxgdCau3hQeLe5aWTItLEyfxNWgpqp7VFNQgTpGaCqOda8wXZDjRkhPwQTp44gU0Vw/jtmpKhFRSVpVlA4fB3RKxMn9N9UYJMVXyYZSCeRIftAnicdq4lmlx//SIqBzbB34iAqJx69HNVUBtYihKRBjSQhPceFYHfPgFTEYnMsIvhBH8TjiR/A+EYIsAImR5KVyE3xB9MKgXSwqzDwnTK10aXIljn7E+UEW8q4TfIVr6YedVPu0iEYpifjng+eIQVBJ+OoZHkJLli1kMmMgJkkclg2WhLOUVcV6ccZ6dc5qvsbYnNR/ADzn519Rvp2R5xlVtWBWL6mrE+b1JQBZNhfUQUxolVFYS7DCMXfeoaNgEzJTsSyXLFZPKKoF+/t3mHbL+uIzMAUqOLSd8+wnf8Lqs59zvx0Zt/eEJFiPFKTbQYIxPtLBE1IAiQni1DKlADNVnjZjRDmNjoEeuB0TZXTcNZbey10Ugc7BZi8CxOCiJIDDI99chGOL0YoUpbKXtNSRA2rijspDJ0xFHRcTPskjUaspzECnx/tSJu4ChWEMHUNABNqYGDz0LtEHGHODMwpnFdvi7x7s/7ljCIFk5PwyraTwYA3Ja/SEL0jKSzhZEGZxUOIwc1NlU9pz9KOYXhQFhS2o8orMGpwaptb+kaCnSqMxYDRu7Gj7g7T4Kc0irxiVn8ZggCRBHDbPaYYOPx44xB4P1GVFSCOD64neE0OPQ4JOYhQncY4mVwYXHc4PjBFsyhi9o3c9xgLB4+OANSXKyJipslIcLmFa4OiIiz2RhPdMIbYapS22kNDAgCekILOe1uLoDoF+HAhRFtjHVUyMARcSIQqrtFpUEuYGgpOJmizPBL2AhOGMNyOn5ydcXJ7Tu04WKDGRZRY7tbIf8QgfW7IMo/eUeU5M4vDJswznR6JSvLt6T1nsODt7SlYUDL2nygtevnrFy1evWC4l2E8CBAWJkuJUqEwJFwLOB8QJawSbEaKE83pP03U8PDxwdXXFmzc/cH93yzj0UydSIrPCmMt0QfAjKXiUVmS5JbOGqi6YzWrGccRaRVlllHVBoXK6cSCN8REBI1V8S24EnZPnNWU2o9DinlHRgcpAO7wStroUZi0GK3Ml+rF7wk/XssxmWG3JVUad1WQqw8eRFESY1KZgSk8ghTCFVGdkuiLFRNcfGIZGvseJ+GStETyIlkWO4FAgt7kUKDJNIsP+LW1/v/fYnvAiTdPy4sUlv/jFL/n88+dUVU0IgeVyxThKmGhMiYuLJ2y3D2w2D5yenbBYLKnKiu1mz3w+oygqfvXL3/DTn/6Eb3/zax4eHvjDP/w5b9685v7+jtvbWxbLOefnZ3gvi7i6rtjvd3RdS1VJ++DXX3/FL/pvORz2vHr5GeuTOUoFtAmUZc7bt6/JMlnYjU7x9Nkz+n7PZntPioqiqNg8bKkXNbv9lu3DBmuFuX12dkbbNmzVFt/D2eoClQzNYY93jn70xGQYfcd6vSazFjeMk4Ow4uXLZ6S3gcH1WJOIUZx2Y5D8i7ZtGZ2jbVs2my3bzY5hcJyenlIWJfWsIvggm0SSLEJjxAcJOGXqGlEKnBuAhM00/dDy8HBPXpTsdhuMnjJd0FPw9YRgmxAzEsSUJhe5wYVe1iQk8rxEGzOFEUcJLI6glJgQjJU5UiFh0NoqstIyX9aUdc7yZIELPftDg7Gak5Mzvv/+e2KMrFYrvPfsdrtHhvk4jmitGYaBi4tLttsNKUVCkMyHzcRqtFmGtR6t7eN5kBQn6zN2+w0xQlXVE4t8ycPmgflcOjnaw0C/2+PcwJMnFzy9fMboRrbbDdYUzOdL3n+4oigKLi+f8HB/yzC2GCvONlRku9twdfWBlCLL5Uo2cPMZ9axkGLyIiNN42+/3nJ2dEqN8hkecUEqywD4WC5xznJ6eo7XGjZ7VSjjsRVGy3+9Zr1cUhaBxxJXs5P7dbOm74UePbRC3WF5YCdO9b3Gj59HDQiIvDGVpIHisldDsIssFr9X3rPSM569WPGz3mFwTE9zfOk7OC+oiZ78fmVcLTi80N7db8hySCdQzTVlbytKQ4ojNFH0fcEEzX+XkpWDaqjojjFIM1cpMzyDLOGQoZcmyhHOBPLccmo66zjk9W3B3d8uslJDC07OS4Ay3d/ecXojbtmkGJItAUVU1MSbqekbXNWw2e6q6JHjF1e2thL6WEqLV9x3jOLA+KTk0O+azpayt3UDXHTg9O0GrxMmsRntxPS3nwsmPwdMOjtxoYpJCc4yeoT+QZQUpiaCdUuD9+9eUVUXezLC2YBgdfdtASjy7vJgwTYGkckIYaftA9/6G0ycv+Mkf/JT7+wdef/c961nBxemKupoztCOq0Fij8WkAgrjXbCR4jxtGCU40Ee880Xl8dCyXM/phJM+KqTPAiKs4IZkXWnCERVWS2ZJukMDfPC9pmv5xHTHdcWIwCEHOfejJsoLN5oGH+w1+CNzebrBa07Ydt7s9F8/O2fWOlFkR1/Oan/70M16/+YALgTzLOT97Qr/b8JOvv+Gz558z7loyMqzKqesZPnqyrManhDE5WiWWixVffP4FN24LaYcfIrOyJgyB9fKEQ7Xn8ukFma3JbEm9mHNz/w50z+XTU95f3eJGjzaRQ7MjpZlkVlQVz59/xg/f/8BqMWO5lOKmuKJFFL+7v+HP/uw7Ll+csVycktuM6+srZrMZ2/uOp8/O6V1PPz3Tf9zYVlhtMUba6Yfe40aH856ooJ7PcW4kLyVvJXqHStAeelyc5oEA+7zjZLGmKCvappVOvJjY7fYUeY7NpRMyGSmwxUmMKasaHySLpus62rZlXs8gSX5Bio67uy0ahTUZXSec/by0aAW7zRaTNMtqThoNfdsSw8j56RJrPUpr5quSZy8vyOuch/0t+JHKGrp+z/6wwdiC1fKE5VrElFf1MzYPDSTFfD6XTtWDgyxijWA3jqKR0RIMvl7NedhsJif6jjy3VLXFh4FZXdE1PU3bgVZEK92QTXfgh9e/Zbi5ps5KNh5s0iyqkvWyZuwTKUDygbE5sL+9od1tCS5grMVmNS4VGG0pVqcYm+PDnoTm/MVLnrz4nNZ5/osnf8h+HLi+v+fm9prPXnzO+bMXdCFy2G/pDxu+/uwZ1+9/IMRIlueYXPCTRZ7j+57l+oQvv/4ardyEgxTxK8/zKbdqYLO7Exc/gSKvGceBLM+Y6zV9F7CzgtPFCbvNnqHfY4Pi6n5HUcwJyVDmBWkUbEb7cGB0DqtzwDA66U5qhwNlzMitJnknIevR0GxbTKkJOJRWVIsC70YOzZ6x66lswa5r0EqjESTdbFZwaDY07ZZBWRyOw9BQhJqsyslNRvSBfuyps5yXz56jxsjm4Z7zs3OKOmO7uaffj2xvH3AxEtAonaGKjOpkRVUbkhLec1GWWJvhnew3Z/mc/WFD2/ZkeUbX9RhjybJCQi+BrvvxBTLgkSutlHq8biD7Qvnfv4nw+JtcaPU35ubJQPiJ0U++Cz7FYajf+SOayN/kTP9NgfFvfPlv+Z2fECo4BpFO71kUS47o5CMn2zvZC1trKMqKszNZQ8UU0Vr0kxiFSx5DmJjVsqfUxlDVtXSjTzgRow2z2Uyc1lkQI5fzknkyX4gjuyioZzP6rv8EJSgYPKUlfPOY63bUNIZhePx8jtx3EdrN4+uy9nPYKpc1NzLvy1pSsLzGyO+zJhONKQS0Ub+zT5d/Gx9/9yM4/285lNbM5jNOTk74/vvvCV6CvLvp3I4FmSPT/SjuH53ox/P5VED/KKTzyf0YJ+57FGqBtfI8Gga01pJ55cFr+ZnWyho6TBitpKRwYbNMdIP4Ueg+0knkiBw96R814yOD/3ifiwAutvM46chHfPZ/fG8+3o/Ta4+CO/p3zvEYnvopmvhvY6//zeP3x7mo45A4TtD2sY0EBSYVxBRQU7VJcWzVmm48pCIQJye5ONIl1E3YSMeqFWK9n5iOxw/FKPlAkwo8KlAqQgqPv+Ex2JOjG+cYKOmnCsWxeWVyxWv3O69/Wkk7FgOOH6SeXhPWzpFNpR5/X4qRdOSvT59TSuKUjtERY0MI0kYWogSz+IAkK0crYatRHKhQkOuS3JQ0/ZzafoFK0yem5PMdnSxoxNUonPOQRlLyGGOZ2RXOO5wbMcbi/YjRIhBl2mA1FEZNIT3iKAmxJ2DIbDWJF4Yir1kuzqjLGjJLKs7AdZydfs5q8ZyYWqpiRVWcMK/OqaoLEQ5UidEZbdeh1IhOiRyN1YYs84y+Iy9ztBcHtvMROzul/fAD4/aevJgxt+cko9Fq5PzFZ/zh/+L/QMov6f7dvyXODix6x+7QCKNvSkD2QQS9ECUk1k2ugOClimWUplCKjkjrHAoIBq56IHq2eyWcdSP3RVCKZkiMTsROJrE+xIgLkRCmZOokDl5t4xTAIy6fNFXZlVFEnVBHYZ9EPKZcR/mZ+rGodEzX1rioON7xj2MxKEYUY6FpMvAZDPnfPdj/c4c2VsKIQqQ2mpRpglSjUJMLNuGkjWoqcBmbTw/ghJ+6O2LwGCOb9KIoKbOazOTSASE/gRgiUbmp+p6jrCbqRDe2mE5jlIbkicnjUmIcnIiXlYyJKq8Yx47RNaSUU9gCoxPB99IhkiIjTsZ1VIJfmZIQUOC0xyfP4MUJlVQCI47NcRB+qy1mwgXPazLt6RpBong1MIR2qp5afNTYrCTPS8HbDAdpJVVSMcmUBiMsfoWIrZktUClKqrZ3xPCxA0RrzTjIWFUpI3pFUZaMvadrRSBrdg13D/ecPjmbFusZyXmU0tTVnLIs6bpGWtCNAcyxcPuxwKQUfnJSpuBQGNq+I9vtWS5Pmc0rXr36nM+//ILTs1NxXeQ5PkjIZ3SSIRDD9DwwlsLkaJOh9DF41+FGzzAOtF3Dbr/j9vaG9+/fc3d7w9gNaAVFbkV4IFLkluVqie4PkxCoKcsMmyny3FBWNShHMjIP+iBOyxDECa/0xKS2guQoyhJjcpS2Ageauo/S5JBIMRKjRyOtxybK5gRlSNNnYxFnQGEy4avrDBJ452mHHj9ErC2pshprFMEJ1y94aRGNCXHLR49P4kyMMVKYgkBi8IGPvHUpKmtjyUuLD5EUghSCf+SR2YLdbsd3393z6tVzvvj8S+7urzk7W9P3A0UhDMumGbg4OyUG2GykDX272bNendL3I0ybsh9++IHT0xO2uw1aQ9PsePv2Dd988xX/+l///3jx8il3d3ccDjtSSqzXa05OT3jYPLDdbinLUxbLBZvNA/3Q8fnnX7NYzijLnP/wy7/kn//zf8J3v/mWFA3L5YJXn72gaVtZUmnDanmCcwmtLOPgObuoGMdeOglCYLVasZit+PY//IZCtzw7e44isr3f0Bx2KKXYbg9U9ZKxd5ydXxK957Df4f3AcjEj+JGr69fsDlu6YU9mc4qyIi8q0HZaJE+YLucnV4fl4uKc87MzyqLEjSN926FUwmgxBjg34F1LCgOua+n7hn5oado9h8OWq+trmr4nKbi+veHs/ASjHD4klJ7GsQKme0RrjfLCwj3yPPUkhCijMdYSImRZQZZP36OmLpWswRiwmSDUyrpivpoxW9bks4wxdkQ8ygQ6NzDeBsZRXKlN0zGb1ZMwLEiKw6HB2pz5bMlut8c5z+HQMJtVrFbrR3Z6XQmLsSzLyU2bmM0WEs5ZLyirku32gdVqxdu3bwHYbra8fPWK4D3DIKGeZVlS1zV+H7i9vuHi/JkEU8XEq1evcE5yYsqyIARH349IfgJstvf0fcvLFy9pmg5tMj58+MDJyTnzxYL7uwecc7x8+RJI3Nxcy/PInbLfb7m4uOTt29d88cXn7PcH3r17J/kC1tK1/VSI2j66d66uPuC9p22dtA0DMYOnly94cvHsR49tAO8MqY4UMzg9fcl+e2DzcIcfRORxPpIVBV9/+QXXN+/INCiryeclqXTs+obZsiAbBpa5QZNz2I74mMjKgC0iu/2Oy6fnjO5AZjP6tmCxtJB1BDuitYJc0wwJm0Ws8SwXJSYZMlPx+Tdf0bX3nM0fyApP4z2ehClLSNA3e7764pLD9gFlDEpZvFOkKuf12wcyXXI49FxenkMMVJXi0HRkuXRozhcZMQ10zqEyy37X8OT8ORkzlL9DKcViWdNuW2ZVSQojYYz4sWNW1pOzPMP5wPXDgwgNXcM8lcxPVlidMbie5D1n6zXJe/rmgIMpFMtirKYfW0wmhYj5aikbtOgxSlAih9Bxf7uFQlHbDFtYlicV8/UKkxf85jfv+fD6rxlGwSNdrjRfffaEz15dkGcexUhZZXjnObQd26ZlNpuDjrjQMThN34l7fFmtULpgUS/I7ZxZYTFRozT4GPEhyd4geuFnh8jQ7mnDgYhHE+makfVqzuHQ4oMEA8/qihQtt/c3PGzuyDJLmRccdnvaQ0c/JDAFpbVcX12x6wfiYcMg1gaUznERCWotIjlgkiGojmplePvhNcus5OniCfN8RkwRN/bYosaokjLT4EZC6Bljx77ZMXrP0Iu4vlqdEdUUQz8OfHi/4+uf/ISzJ2tsptm3Ow6HBnM3cnn5GV3fQYpsHwaqUgJiv/zyS968uWK/9zw7fULfjVR1xfvrDyit2dwfmFcF//Af/Zx379+yWr/g7ZtriGuurh6I9Ay+pela6Xj6kYdzjnEY6FpP2zhpq4+yNh6dQg/+UaqK0dO2DfN6zn7fEFWG1ZYwOm5vNsyzBatZTR87rDJ0TUfbDWSF5eLJGT7VDL4n4LFWcgcOhwNKC79+HEfpJtneM6tK8kw6CLtGiv9ayTq8bXckVZIZjZ+C955dXLC/bwlu5P371xiTWC1rylnF6fkF1SLj0G0Y3R6TjplSA9ZYdvuWvJyzPBfRW5ucZ8/PuX5/jVWBtm8IWq69c4KPkrwZwZm0bc9ymVNWlrww7A/yXKpma7QRsc5oS7KJbdswqp55WfHu7j06D6wvnjH0LSpAZTOqIjGOETXNIWkM3F99YFHmXJws2e8bmqajrGbMFguKPGe9WnFoOi5fvkIXc4LKqOdLsBm/ffua9+/fMvjIoe3o2oYxBVbnlzTbB0qdOFtW/OkffcOh7Xj7/gOHXYvOM1RINIcDm90BXVTM6iWFgTT27LYHhnGLD56iytFKs91uKKqMzcMt64sXE85iwXJWYuOKf/5Pztn8i/8bZaGoZ5bxVoLE5+szTDhgxsCz9VO+f/c9Xd/ym199z2dfvsJYS9v1outoQUP4FCjyirHxdL0jMtC6htV6RVYaDk3LboAbfUNRFmRAioFuv2UMA9W8IMaetusYh0jvPW0c6BF8Zj2fs7m7J6TA+dkZ5+tT3nz3mg+7DS//8CuqWcXoZ2RNQ+4cV3d3dM4LMiTA+vwCa6QbK0bp9lss5vTdSN/3GG2m7lzJRiryeqIaiBB6aPbov1/9e+qqOzKc02M+hwh64XdEdK0/FbV/1zR3xKcej6NQ/ako/wlM4pN/MxnfmDaIHAX49OjkPf58eS+fCvgff85/LDx+RGZM/wiShPcoLQhf0GQ2oygqlosnnJy8YLPZ0bWJ1eICrdWEsQqy945ODKqkCZ8ibvuQEn4yMvgQHjsi8zyfgu6lU7NtW16++IyiKNjv9ywWxe+eAwmf/O8UD45FDu/9Y6D9kf9dVcUj4zwEeU+CApQcqZQSXdeRZdkkaAsKVxvB2GglbHLB0Ap2qVCGNL0eQiDLlBQTHk3Hv3vE4Bn6jn/6T/8Z/8//x3/Ht9/+RsR85x+vSYyfFmc+BoIeRXTn3Ec3/CQiy7mbx/M7HtaYyciWPzLVw+SyRx1F8Y+fW4xRiuyFmEnzXLCaR71VHbWuBGmiN8g9NAnc2kz3zUc9+OMQOOrGx699eg9GjmgXIZ3Ap1i1I/v8aOQ+flbHz+tTk/ffdfzeIrpVlkQgTq6olBIBR4phCjyQcIM4DcLjRdEaVJIJYgIsoHUBBFLyMLFvmGoD8s1TOOHxr0rCQLU+7iABJPhH7PkBQWAfXcmTk1wd62BeNi0To9qoibs80eYfOe7qkxaY5Ke3YvgoqkcSUrUTZlN8vABRBxJenDHq+G8TegpU1cpCMhxba9Lknj9WE+NUPUzx6OcvsGmGHl9A34PL0FqwIWr6t8YanB/Is5LMVphk6YYtoMiKGUYHMuP5WDhQZKYgtyWzckVmSlxU9G6U4D5jpRqaJBSkzCsSMshCFogxQ9WfwXjHvFrzJz/7P/Kr7/7v5KamzJcU1TnWzkREQ/48bK9JoSHFQAqeTCdyC8F4gt+QZTWn66fk5QznE3/5i1/w3V/+G/7b//3/ifn6AmUVKrSYLOfk1Wf8VP9XVE8/Q+WGhObq9W/59s//LTdvXjN0vYSNRiPu2Jg4uMR172nd5OZDUxq5DrsUGB2EMXLVMQXsKCgsauKSm9IStREBKZPOAa2ZJk6D82BdBCP3V4zgJ0b/42BEicMD5NrJHfCJqDldH+Q2jCSGcaAxiX5MhPjJWFaKAeiMobPQEPExof3vO5L/9qPKC4YgwnLrGzRKHlohEFxkPptjUey7ATf06CS8f2NKKBxwIIaO5EdSFLdbFhNWK3wKDL6ncS2D79A6UETBsmQx4VNijA6VKw7xQAoeF3r6lPBxKrCQUfmaWaqpdMaATNCFL6jTnByDSw6bFwRmuHiANJIpRaENOiZwPG7CutgSx/CRJRwiOvcEPWJ1xKKodEVGhc4CjWk5jAeGdMDFDqUMuT0WmgrKvBJXJSVWBUwE+h5jBQujYsREqHUBpsB5CbEMeloYGHHtaQtlnROcJs8qhtajO01WFWAMTbvH6oK7+w3Vm3c8e3ZJZoqJERfIkmZWr0gh4Z3wSLNM44KT4E5tiVP4SwCUSoLnjooUNE0/UlSRn//8G376kz/g7ORMGOpaOGhhDHStONNsnpG0EkFPCy/OTAUXcUoNjGnEJ8cwdnRdg+tHMmWpy5Iw9ATvp+AU2cz44DFWY7SSMDIs1ihIHuc66qzCzgr0LCcqjUoZ1kRMCiirGE2gNX4KTgYTE2OMHOJAx0ihxa3ukEV9YkQlCR8tdYEmI2lLsBlDCoQUCFECl2IaMEYzRodRGclrQtKU+ZLSltS6wBiFI6OLHWMc5UFsA10cGRIkZVG6wOhIMganIiGMEDTGSiCpT4HcGpyTHIUYmDqwftwxX6xIgLE3bLYHlss5RVnjfKKqF3Rdg9I5FxfPKGzJhw/X1JUEOI4ucPn0Bf/qf/hX/ON//D/h7Zv3rNZrgg807YFf/PVfkucZP/zwHZ9/8ZIXL5+jjeL+4Y7ZfMbNzS0z72FyfFRVJZuyxYzDvsf7kZubK/7RP/qHfLh6yzc/+Yqm2XFzc8WTi+fEKIviECzPnj9jv3/g7du3eC/P5p//7Oe8u/6BPLfUswW//vZXnJ6e0jfDVIyEssiYlRlpVtLu5RmvlQgzMQom5e7uhubQUJWWk9WSfthzdf2G69srum7PYrlifXLGDOkgqGcLvIv0h4aiKPjii89YLOacnV1QlvnUmtnL2iR4HMf8BpnID/sdu83d5DKP7Hbi/D+iyQ57eY5/uL6iyrUUyjM7OWQSwmIxsohU8nzyPkjLvsrpB8fRupCm3BFjM7K8nFwwCRsMtoDZfEZInmpWslzNsYVhfbqkGxvQgWpRcrh+oJzPefHiBdvtjuVySVlWNE2L99Ix5L2XkL7VGjc6ZvWc3/72t8xm0rFW1/NJ/K7IbM7Z6QWz2Zz9buCbbz7nzZs3PDxsKbseYwzX1zfs940EYLYd4+AmB2zk+fMXdF3D0IyPzqP7+3sWyxOqqmI2m7N9uCbLJYhqvlhykZ8xDD1KJZwbqWcld/c3VNWcIi9kn5cSdVXzoDY456ZAW0EbGKNYLBa8e/+GEEZevXoxuXAlvGm5WHJ1dU1KiaZp0Fpzf38/oUc2zBdzci3upc1my/39nvXaU5cnP3psAyibE1PH/cOOO9fz/PKMs4sLPrzbMXZgc8W+3fL+2jOf5xz20HYj3eDJjSa5xEmhp81awNrE+ZM1IQ7UhSYboOn95NK2XH/oGJoM73uef2aISTBGCc3JyUzcWLmhzAzR53T7kX//b/6CP/3TLyjyOa3bkmnHs+dGMklGafcehpbZ3BJ8wfu3dxRlSdsGuj6wfibdYa/ffODyyZLcSddfnlmcS9zd3VLPDT5ZCJ7gLBcnT3i4e0BhKfKc5tAw9gOLxXzqgkq0rSeup01xNicqMQxVdUm36VgWM3RQYBKH7sBqsUAbzdWHGwpr8CFibUbXtZKbkFn2hz3p/0/afzVbkqVpetizlKstjz6hUouuqhacnh5wBAwcXIBDwsgbGnlN/gH+JP4CmnEMRhiMIAEQRjNy0N3TM13VVdVVlSIyQx59tnK5FC+W7xOR3TPTPVVulhkRJ2Lv7dt9ufta7/d+z4ugqCa8ef2Gk0PBdrulqDJmYsF6s+b7Ny95dHDEpCyZlyBExsHBIz7+6JSbmxXOOqSEs4MZmQwo7ckLjXWOwXu6fqC3LrnZYqSaVAknZHu22w1FXqCMRHSCLM8TB11IiAE/mmu0Ti3xfduhpEwihIsUeYWIgb7rRja7JZLQdkVRIaTEuQGtJVoLhIistxu6fmCzbcjLBdv1mnKaU1RT3HbD7WaDKCVKakplWCzmvL18g3WO3g2pC6GJyBB5cvSU+WzJ0FkGLAeHS+qhYTqZIIgomdZ0cszOur27wwXPcpkQMDrL2O52SA2TSYF1InUvRc9221CUUwYX6Xuodw6jNJn0VAenKDnlenvL/cU1B1XJH3z2T6hKw1ff/Bq0om0HkIYQMp5/94ayKDk+POLP/+zfEIJhfpDx9uKWTz8/Y7COwXk+/OjJb31tN3XLel1j+0gIEilkMjgpBX3ADT3VxCR2MQGTGfKspJEWSJ0fITi2mx3ryZoqy+m6ltVmjSgMILB1z3qlqWYFdtemHBoVUVLT9za5l3WbRDQR2G53TMoSJTR901OMzsQQI1oqQh/BJ/NAsJG66QhqQ911KQ+LSF4oZosJk2mOEJ4XL7+nG2qm84wsz6nXO+qmJ6CoyjlCaKwf6IeGdrfiaHFEiJbt9p6m3hC1IjcKgiLXFYv5QTIbDY5C1wjh0DqitODk9DB1oExLdk1yzg/WIoSmaXtcHIhdJM9Ltrblqr5GBsHx4jChLJWnty3S9mQq4/r+Ci0DCujbFqM0J8cnZNUMVRYslwfECE+ePOZuU6c1LZHV/TXbuuHk9ITT02O++vY5r972WNexvb/GO493LSfHc87OjvjqV39N13cUZc754Yy7uzVt19J3O7JM4kXG1fUNKloyJYg+YSRu7+5QWnJ0vMTkCpNLOuupmzUiK3A9HM5m+N5yOl/wk88/Y/KBo5lsUfKGD588Zl7NsF1EKkdpZpwcnoGU3K0v2W52fPz5Z6y/W5EZjRBpLe0Gx3xxgG1aJOOafEjdfrnOcdFTKEHTdXRDjw0WHyKb3Y4oHDf3A9XMUJQ52WROs0u4oN6ncMKy6HExMp3OiEiev3rJZr1jW2/p24Y21AxDP3ZaOoYh0HaexdIQR/OUVJq2Td2SIQSC95ycHvHqxWtsnwLRF8slTbPl4OAYiKzWa5RU2MGS5dnv9OzeC5gpzHJ0ycbk+vXe/8AtnHSlHwZ9Av9OJ3oSx9+JmennPxS/HxzHiL/1b37AnR7F1SSw7k2nf3t7X4iMo5lWjsY1HgRN+fCfUhlFPqXIK4IXPP/2OVk25dF5yW63oSyTMTHEtEaOo8CbjCT77Cib5tU+JHE6KpTW9MOQBOOQOBg+RI5PTjHGPHTY7AMz98c6EFAP+8mDK/shA250nT+8xjmCTs7vruvGuXkqKCZjahLfQwgPYeghWISSaG2QWj18hveJO/2gmQrxcP6TO/3ftyV2+maz5vGTJ3z11dd043P7/YLHPtT0fR76/nul/VV/g5efHPQJ8/LufCulsNYDqWNzf+z276UfvpN4+My0dlFEoSiKAu9jIjSIhP0Rar+fe3131GeFGA3XqbMPkpa8H/w/oE9EEPKd7vz+mEzZM3veP7wTzRO2+f1jtNfs3i8K/V3bfwTORROjTKFrYsSYxJELuXd2x307fxJnUvhiBPwY0JdcUOma2ju69wcjjKc8pIWlUKmJQbwTgIXQD6J2JC0SRYSoFCKMKJkHITyMTnUAwwO2fsTRpEOfCgO8dyDTnqn0MpHCC9/xovYnb3yN2AvspM+O+8RiT5L09wz3NMFhDFoQjG0IJIGbBwFVjmJ/hOjAQxRrYmjAz8bcqDRI0uDWDzfMEGIKw9NlEuiQGJ2RGYHzA/uqT6aSO7jMp5TZlMF7Ilu6viaOrU5K52R5SVXNOFieMKkmyT073sRFdgy+4/zkJ+zqt3S7l0iVoXWJySb4kI5Z3bZY74lDh3AW4XqMjHilcEYSgsVzjywFy8MnbOuWN6/v+dlPf8OzJ/8/Th9/SHF4hooDhJZ6d8muq/nyP/lH5NMJh4cJofPrn/4Zf/Hf/td881c/4+LqGlxEC4GKMBDRFpAOHwJCJ7ZyLgXKJYed7SQXIhCVJDeKAUEtLEpIOinpUCgZKQuVMFwygoTdEAk7yHvIVBq/PkJnU5DYYvouzEAKUGOnhJAxnduHv9tzx9LN3nrY1gPBWratS4x30r3DRqi95soLNi7SxlTMUb+jiO5FCrkUVuCdJTqXQudiQo0EldoIpRP02w4RFEWwVFlJrnKUigwW3BDG4M7E4/IkPm2ILrVkucSzTS09ASuSs33oLWVWIqJg6B19Z4kijeHkrIR+cHTDkMaXMhhdEoLBe4UwmiwDU1QgSpTXDEOL9A4ZBTZ4BHZ0YScOn7UOZxMiCTugRkTSoCoGeoRt0CmRb7zKFd4KvBMYY8jyAi0yZCA9fGPqmAmjIIwA5x0xyBEJE1KQTYQyzymzDG9tYu1ZCyKihKcsDCETGKWwfUc11RA9wyDQ2QSAYRh48+YNQggePzpDjIvgQERpQzWZ0LcJvJLalxLvS4ydP2IUTJwPxKDTolvmzCYH/MEf/DE/+cnvM6km5CaF/8aQmHopZVuMoaOMRcMRtTOyBGPcV7wT4qiud9zc3HB9fc1mvSb4QFmU+GGg6xoikSw3KC3ouzbhpIxBx+SWG5xLD3UXaLqew+USqQyZKVCFQrSC6FLQrdCRIBzExIq2VuFcgVUSj0oFVCTWDaBLMlOlMd47pM7RQhKUTKxsH0Ym8w4lA8EohpAEyTLLEVFhpCTPDZnSqJgKsSJqsrzE5DnWOzyRpuvohx4lUlhUCD65EZRKWQ7I5KwPyYGQTMUaZQw2JnTSb7s9evyE2W7Oz37+Dav1jsF5ZtOKm9s7zh+dE6NmsVjStJb5yWEK0w4BqTI+/fQjVvcbqsmMrh+4uLqkyAvmi4RA+PyLz7i5uUEIwYuXL5BSstmuOD45Zr3Z0A89xyenXFxeYa3j5OSUvFDsdjVPnnzAz376fbpOnKWutzx5esZqdQkErq4vWc6PCUEhhKWp69EdoVFSUxQlh4fH/Nuf/k98+tlH1HXNcrnkz//8X/PFp19QTSqOD5fgLO12Q71e0zZbhFIoKaibGpMlAfby8i1usMymJf2gublpubu94PLiNV1Xc/4oFQeGYeDo6Axv++R6nE/ROiPPkzjatQ3ESFVOUTKF5mhlkEITY6DrLPVux3a7SVgWH8YAHs10OmG98Wif7p1xFGGNTl0q09mUOM4RkAqhkiMVFNIUKGFRSqCMwdPhfCCMLEAhNSk4yqBNgdaaKi+oForFckGIjrwwzJdT3l68pul3zJcT7u5uEiqkKmmami+//D1ub25pmoblMgnWs9mcrmsfXCtFXnB3d89yeUBmSrKsZL2+x7mEC7PWU5YFk0lFWU7YbW/5xS9+ydnZGUqpMRDJEHzk5PiUu7tbjo9PyLOCYANN3TKZ9Oy7Dy8vr9JCOgpWqzV5URKCp97VnJ4uOT074e7ultXqlk8++YTb2zs2m1VCbPQ9i8Uhq9Ud5+ePeP78BWVRMpvNErtWSm5uavo+oWDm2xkxeHb1mmfPPuLnf/VL/sE/+BMm1ZT5fEnfW169esXjx09omobJZIJzjskkictN03B+fo4xBUTPcnHE3d36d3p2G52yeoyGvnfc3t3RWcnTZ2fEqHDs2PUeXUZa31LOE6IjMb0dy8UCYwyzecVu1zL0A+td4KPPFkgi06ni/iottIQMTKcFu1vH0fGEMDQsj0qC99xe7Ch0xcHU0Npb+m4g2JBCk6eaF69ec3ikUEaAD2QmrSWigqAUWZ6wZcPgubvrePJ0wc1NDaTW6MHtyHKom4YoYTYr8E5Sljm73R1t7TF5Tm4Kzh59QDu0PHn2iNZtyDPN3e0aLVMGx2Q6p+9bPvz4jKbZEaNEGUPEUxUFuc7Ybm8ZfIEIHmctddPw6Px85N0PVPmUpmkpkjWIYehHAT2ZXOp6lzJtXMrtAFJWQZYjpKEbLJnRTAqJ1lDvthSF5snjhDKSEmzTEb1Hagikjpe6bnEuiUdSWXKTMx3xR7vdBi8FxXyWsl9ihw+ark8heQLxwPaVmDHwNj2z93M2JROyIAZBvWuZLRZoZSizjPv7e549e8RsOsH7gTrLk4OumFK3nu9f37I4iiA1u65jva1HNxkEL1lMJ+Acdzc35HlB3+2wjaAwGkWGVjlFvqRpA08Pz9jc3hPEmmpakBmNkonVbmRkmuXUTUvwgsPlMU458jwFqs3nU+pmRzVJeEw7Bgdvt1vyImXCWOcpioJ6fc98WrC6X0Fs+C//83/Gi9ff4l2gW28ozQFSlFy8TciezW6Hc5o8n1EWFW/eXPD48Snnj8+4uX/N8elHHByecnW1w0fH/bb+ra/terfDW0cMCesipWJSlqmw7wZ6OxBdYBgiznvmi0MqU9F3jrZvKXODE4qYCaQObHZrble37OyWTBfImEImVzfXaH00Br55KuPQKBQJb9J3O6JM88CIQqoK30m6lU8CupJYFcgzg7QDQ9PTdQ4XIllZsm22MBGoKqAKEupJNHS1pd7uWG16grBMpoIQMna15+qqQWJ48vgxZ+fHtDxHSY+Ujs32HiElTd2OnGeBCpJCVhTZhGV5gAB2uzWhyunZYlQK266mU4QK9LZHKEnvB+qhQcWc6ASiy/BR0U8j7kjym91rjucHXL+959PTZxzpOX3bU0nN1d0lna85mSxpVy39buButcULTT6d8fEXXxBlRjWZ0VvLarNltlBcX9+QlyUZkbe/+RVFVfHZySlTo3l+cYmIjvbummKu2bWRb170HB4cY6IlDGtUbFh3t8S+4WCm6NrIQKQ6PuTq+pLd0FMWJQiYHy6pm4ZXby758KMPaa1lUlX4YBFkZPkEvKAMIPsdnx8uuJIdu3bHLMv40fkRmSr5ar3DScnVZk3dWqSZcnaWQhabTcO0mFK3GzyRwaYg8t67FCYfDU4ViAiu97TDQFFMULqgtpZ63ZDELcHgPNUkT10YzhKsICsKfN0yX8xwQ2JH39/eIQLokHG/qXm9veF4eZwCU++vU0bHdsPs4Jyzw3MqkzrktrfblL3nLfPqnPPzkpubC+7v7iB4jg4OeHL+iBcvX6GNYrFYkuc596s7ZrOCQMswWIYuEOy/wx78H7HtOdXvi+jwQ+EaGEX+iByRL3t9b//pP3Sij+HyMgmskXcC4V6j2mvuyYGdXvpDEfyd4B5GBOk7Ef2d0Lzf3hcv98I/IgXP/m0xUo6GUYFznu3QEEJPls3JMkU/tOR5xWAbYhB450eE8QAi6QeDS3lfkyLHKEnfppxA530ymooUDqp1QpZOZ3OU1gzWoU32gH303uNjTJjVGNI6dJzHvo/52Ics/83vuxeK36FeJF3XIWUSjPdC8zuxPnV/p7mlTkXw8Ty/c/qLBxTKw3l+KFz88FgKkcy0fd/TtS1HR0dcXl4yDKM++p4DfY8ofF9E3+/7vqDy/u//XZiXEBNO0nv/EFYaQqCuG6SMaJPQMfvXp2BTTZ4JXOzITEbwIoWzyndd4bAvIO0d4uM54B16JRmnk5Aex9+/G1pJ03kQ0oN4OFbeJ3OrlO8fu8D7Q3j/Pvv3fB+V9Hdtf3+cy+iQUkKPztl9i/ke6D6mqpJaGyUyISv2LElIorZI1QMiSJkngTnaJLzEIQ2hsQUsCRL7wTkeqLGfOYWC2rGSEBKIfqyS7bExcTxQchTsY4QoHJHknEr3pwSdT98pJeAmLEwCzO+DIcdjOjo4h3TCxBhoGd+r+uwxNSQGe/LfG4ipepOKDDYdt71dctySuJ5E91ScGIhyTSi/RYnHSJuR65Id6aLOsgpvwYcE8HdDj1Q5eixcCSHJTD6+d2qPyHVBnk3IsoosKwl2z4XS9H0DwlLoCmcbMqMospI8y1N4WWcxeZ5Sg6VBxECRLQnZmiJfIkWWBr+UeAQXty9YNxdov2YmU/FCB4eRoDJNdCko9eL21ywnx8ymJ/g+YJ3lV7/+FT/+/jd8JDzy4IioBHk+MJkZcqNR0QEti+UJf/yf/gseffwj/of/+v/G83/5L3nz5hJCqvAOPlKPbmOZaYLWDFIxEAhC4IPADgGPQGaQBYcFaucwCHKhmUrJRAryUqVSTIx4Cfdt4K4NaBnJJOiR4b/uPLnWnMx1uuBj4mER0wPGh5T+u3ejCyKZkqNr25HrgHWBOkai0GgDCo9Rgh7JrVe8GSKbISTmvIxI8Xdf7P+hbQiesqrIfY6MEK0jVwapAs717LoNRkgGHE5EnOsxfQ1CYUxalCgZ8EqiVCpuJWamRUSBDw6FoNAlhTTkShNtSKiaECl0ySSbYTIFQ0QqgVQGpQuCSsGrUmi6PjE+kYbcLBDeMFjQOiCERwZHlglKSpARH3vark0V3eiIjEWUPHV1aJVCE61tUcEShWdna7yQmGipzJw8LzFZhrEFxlm0NhRZQamTeBB8xPd9ikxwfQryySS9SM5shSZlLTAW1lJ4jZYKh0pcQd9gfQcehI/kOmOwA1IGlApUU00RNFKWI25jTYxwe3fL4dESrSVCS3wYq9dKIbUaEQ+Mk5kRGcK7+5UUmuBzvDUsF0f8oz/5Z/zoRz9hNpujtUaOLVI++sRWM4YsL4kBBudQ40NXKolQ6SEW92123tE2DddXlzz/9hu+e/4db16/5u72juAGQvBkRZ5E6DJjNp+wWiVeu9GGaT4lhn1nTxLfEJK2HZgsZxTjImFqJMZntLZBxEBmSOckeEK0qZCgkoAelEKKDPyAMhJtZEodVxYnFEqPbLyx24jgUlBtdDhpUBiUEGAtRgpkTJ1J1nuGYMeiMSitUUbgvaDtWnbtDtsPTMsc7wGhcD6l1QuZpf31DucG+j6xF4tiQowpgPh3ubrrtsNHwZOnp4QYODo65X51y2J5Ql0PPHr0iPv7O/I8Rz/KOTw8YrVak3jFSXCdz5ZkJufxo8f89Kd/yY9nP+Kbb77miy8+4/vvv2c2mz04d6XQnJ6c8/bNT1Mg81gUOzt7zKtX39F1DY8enaYwqzCQZRmvXr3COcvLl99zffOWdrfj4OCMCFRVycnJnKZpsK6nrmvybMJqteJ/9kdLqiJns17x8aefcHF1RZN1WGvJtKYwGSfHB2yv12xXN+w2d5hywt39lqJast2uWcyXbDcbjJbc3GzITaAXkbpe03Y71i/ukSqhbASCm+u3lNWU6WyJMQUhuBS6FzyL+ZKqLGjrHUVWUFQThr5nGFq6rqFpdoQQOT09Zbu+4+4m4WXm8/mDmO4bz2Qy4fLmBoBucEidEZCsNjvyPCfLDHW7w9rhwZ3e9Y4oQbr9pNGwD0SKMZAXGqTGR0lZlGSznMlCsjiac3i05NXrlxwcLckqwdX1JdPplN1uw+39PU/On/GX/+ZX/NEfJjTLt99+i0DinCUzKTx8Uk3HYNEJQmwZBsfh4RHXVzcUhUEpzdHRMUVZjGx8OD46ZXXfkmclZVGxWW/5+ONP+M1vfoOUMom9eUXf2eSALKapY2wMFw0h0nWpUNM0PR9++DHPv3/Jdlsz2IGu6+n7ljzPKYrjBwamc54YApNpydXVJUop1us11lpevHjJkydPOTw84s2bNxhj2GzWPPvgSQoVKzKKMqftdlSTkq7r2Gy2xChwNqB1xt3dPdb2NE3D8fEJWmuurq4IAW5u7nj69Bk3N/cM/cD19f3vcHUDzmLKtGAZROTLHz+h7u75/sUVH3/8CB8GCiMJskMbQ14WPM5nfPvrt6gyUtcNPmpm8yIx61XO6xc77u52nJ0bJpOK2WJLnmd0dsPxyZxoJSIOGCWZVSWbdUdX96zurzk9nfL0w1Nu724xVcHx0TmvX79JZg6hGezAbteiM40Pll0Nn392jlKRq7cr2iYtoPLccHBgWK0sEUtRKJyFyXTOYhEQUrC6azg/X7JeCbTReNuDydjVNziX9jnPDX3Xs1xO8DbQdO3DOqNtepTMePzkkJvrJNYWpiQMjkwZ3JDu/+t6y3KxoG4aNrstR8slTb1FGU1VVXigLHNu7+7Ii4J+cMzmixSqJQJN16TwMkBGMFmG7x1dN2CHgaJIuRlSOqrJBKVCCgO0gXboUrjfrqfvB6yNaJWT5xMEyVm/3WyZzib0fUdRZDjXs2rXCCFoW0/XtUg5ox9S50sIESNTu7u1GV3Xk2U5wduRc5qRZSWTqUj8X5nmDkWR8fXXX6HHeaG1lvvVhiA19eCIheG7i0sODg6ZTOYIk6O8x4tkTgouBXdrZXA2UmYTjLYIkdBDd+sW9XsVh6fn3N1v8N5xNJlQVsXYR5qyR/ads4+eHvMnJuP//Rf/H4bQEERkebjg6uqKpm0x2SHOW+bzJbvtPV3TcXZ2zus3l0ymE/qmZ+gjt8OOw+UJ6/WOv/7qpxRlzsnZGc0u4lxgu1vz0UfnbJoN13f3nB4tiEvJerPh9OScclLw/ctvOH+y5O3bS+qd55uvVpw8M5jitw8f7HtLnlX0Y55FURQUZcHd3W1qpvaCvrfo3KBkzsHymEIV7DYtdVtTlAV9hMPjAzKVcX9zR9u3FFVO3TVkwiBNRts0cB04Oj9MOBQkMgpynRGFp613CCPSudOKqizZ3azp+5Rz5FWEXKBlaq/vXc+m2YEQPDs7QOYhhU+rhF2clYbcKKwNrOqGLEh0lRGGnm6IuL4H5yjKCQezKaeHB7xdvcRFSZUXDI1j6C3BeZQ0SKMfUBiQ3JxKStqmTh0Wwo88a0Nd1/jgiSLh/q5v1oSgIAiMyRO6UMgU1ttt0HNN0ILpZMLcFNBa4uBpfI+zjklRsdvWrG9WDJ1jsJGDs2OefPghy4Mlk+k0hb4aw2eff8Z6s+XJk8fU9ZbdZkumI5v7W9Z1SygmHJ0/IcqS16+ucXWdVJog+PgJxHbD6uo5rt1xcnpKVRToLKPIc67bK7RSLOZzDkxykG53O66bBhc8xWTCxdUV50+OaHYNy4ODEaU6EFXKhJg08EV+zPbmOd5ueLo4odk23Lod19s1wkLvLHWd8qsW0wnOD2zvr1FZxh5zK5VKOUB9j23siKrUaJPjbMdgHXlOChYOms12TZalzohqWjGdVYi6x2QRk+dEnzjYMaSg1K5pETFhL2azaRLbpaP1Ddoo1rsNISY38q7eMp0uKCcl17fXuKAwRuDjwGa7ZjafcXxygtbgneXq8oqPPvyE+WaD0AKdGSZGc3H1FppkhlRaU011Ynr/DtvewRtCII5C5/7PsBcW92L2O15zctaOrtwHAXxvaBXJ1Tvqc3t/rxSpHM0oku/NYXvscgh7MVGM+7Dfy4d3YI/q+JvM6PcF2ncc9ojbZ96QzKHJgfxeHpAPo9Bq0CqCsKxWVzib8i+MziGOppLo0UaO69L9Pnq64Gi6hEGaTqrETSeSZSngs6zK0VabMj/SejaJ2cElLQaZUJ9Kv3N97/OpkpM8BdrvOejvu9SB8TnbkmUZeZ7T98PDudqLyem8pXNorUWr1L1trUWOCCQp9Xj/kuO8Xr53TPci5PvnHYL3nJ6eUjepK/To6IhXr96k9XWWAf4hr2g/tvbz4H3I6h5BsxfX93+fcsZ4+P5+r6U9zKOTs72qSry3KD2+VqpxbRTSmlBZYkzYGuc8s1mFVnLUJgTe73NDxMO5FWmwJSY2PyzyCPnuz/t7/t58ppQaXezioXiQhPTwXvHDP4joUqYOL4AoEhY6jsd9n4X3H9r+3k/3wbcokSpb6Qvt8RhZulyiG79KckZ7IERHArykC0iQRKw4iuF7MTuF8A0EMaQBhEku9bj/nAjRJ3+4SA7PGF3CoST/b7pEohsF+HI8gJ6IRwkzWvn3TvMkYL/fJpHc7gopclI7zZBCUeP+W43iU7K+PxyDOLZ/JIf8u1/3HGLB6ACLe2H8HTImCfCwx8Qk7IxIRQYEyCE50dUAeHTzSQopUSVaFuRZSdCJC9x229QJEMJDFSdGh5Qp1EgpjUBgdJEcnDpDSDFWzMrUfhID1g143zJ0a7bbLU/PNG4YWK/vaNua45MzJpN54uFLxZMn/5Bh+DKhPELitgbfsx2uacyvqKtfcywfQQfaGXRIzOAszxkySRwE3rWsd9f0TjKZLZBK8ItffsX0v/mv+D/+n/7P6OmAyHImkzk/++lf4oHlYk5uFFk2QRdznnzwMf/kf/m/4VcX16z/7Z+zazbU9Y5mt6P3KchTZ4ImBK6HiIuWLqayT/CCboDrENA4vIPeQy4DJnqEdcxEavlU+1afZMN9cJJnUqCVIMaEkzFKMM1lGi4xMubhjuz7MQdg5DLFmJDcxIgLEiE9GyuQLtIRQINC4jVsEdw5wc6l6mkcb7jvM/x/my3q1IpslEb4iAxpNEuT4UNLO2zYDp52aPHjFedjIp17JxGEFOZYGGIMqSocQIY0qQRJbgqiiEyyimlWQQh0rsd4SxZyymqKVAKfezKRo0USn4OUBKGQmUFmit5JvJPkRhODIQo1CpgpJFPJgug0OIG1nrprcdGSyTDeXSSZmZBnxVhAC7S9p3cD1lvIGlyEQka0ztDRpBCTSZUqt95TFgVaKWyf2i+DdSgjUFEkXnjsEV5hVJEm+FqjhU73rDDeO4IkeoHGkIkSgkbpQDEBqXXiKpoZTdsSsDg/YHSe2L5yStOkSfz17TVPHp+Mk6KEFYrBP9wrhVRoAftgvOAD3oMQCi0Lgi+YTo75x//z/wV/+Af/gDwv3+uMSA8f69w42VD4sSiktEG+1/oWYwoN9j6kkLamZrtdc/H2Dd89/5ZXL16y3W4Z+lTUUAoyk6FzjSkMs8MZqoBw4wjWU4wTHjFWrdu2xZDhfQpN1JlhcAMyk+igEC51AuRG41Wgdw4IqV1Mjl0BskUpR9SOgMC6FNhl7UAvJJiKXJfjvd1T5BlBZNgQsXEMSm1bOhmYFVM0EaEVPjh8HJAxOY+9dYgIQ7Q0NoV8lVVJWVUooG1TMGVQqW0/+JSd4bwlRI8b0rPRmBL9g4r7f/y2Xm+5vr7i6vKGg8MleV5yeHDM9fUVWWZYrXZ4LwlB0ltHNzhOzx+z2+34y5/+nH/4J3/CyekjvvrmG5SSCRVxfUUIltdv36JMltjgId1Ps7xi1/Qcn5ynTAxpWK22tK3lbnXPweEU76FpOmazCX038ObNW6qJITLgfUApQ71rqIoFR0fHfP/999zf3+N8z26349M/+oL72x2vX7/g8eNHnJ8f8+rtG6RSnBwf0+wauq7Fu54X337DIp/RbtcENzAMKmUfyA4p00Qvzw2b1T1ds2a5yFnf3dC2W7QS7PqOzfqe3XKJFJLJxGMyQwwDXWfZ7erk/IhwfXXFfLrg/PQx5ekjgvcYrRO32gdmkxmTySHr9RXDYMnzHDeU7JTCmOScybIcpQxG5yhpsT7lb3SDp+8tPkJvHZtdQ/AObRLnexhsutfEPQeR8Z4gkwjnPFFYimpKNZsxPZLocqB3PYFAlqeF4j/44z+iHzrs4JjNloSQuoNigIuLxPY+PT2lqip2ux0XFxcsl0uKIhVcPvroI7xPgs96/Rxjsge3+mA7pByYVFOs9fz857/A6CTMC5FYjpcXl2Qmp6wKrq6uODhY0jQ1y+Uh88mc16/fcHJ8StPuaO5aiqKk73syk1GUFYvFgvu7e05Pznj9+jlKSc7PT/Ej1iDLcvJMMJsu8B4ePTrCe8fFxRseP348LhpSi27irpdst2tub2+Zz2cMfU9ZHnN7e8NyMWe1WpFlOdvNjpOTM7qup2lalsvZGPKsx0VS5Pb2lsePHxNCYLvZEoPk4GDx21/cpK4kocBIjRSewdUcHCu+/arjq6+/59lHU4qJprEDHk/b7JhVh5w+nbPebJgvKvqhZbXynJ5WOCd59vGMm+s1kFPvWswoysRBcHO74/T0Q5rdLVWVwq5DtEwmAuc0h0dTqjJjrTRnp0f4YPniyw8ZbMvt3TXVJENtBVWmudt0LBYlzlsWsznTSUYIgrzUFEWWcCHS8fjxY+7u7ilyy+FyRm8b+r6j6waur29Ty38e2W0HHp1NefHqLUWZs1xMyTLNZJoz2IHL+5tU5BeC+XxJAHa7gSyPzBYT6iahA++uNizyAlygD+kZr7Mj3rx9w8F8wc3dDYVOrOgoUz5GiCm0VI7uLecG6nrL4dGPuLq9YTFZsN5uaJ3DyMSCzbMM7IDrA1KJlNESWiLQ1DWCwL4Nue8HjMkoTUbwkqaxGCUJLrWy77ZbhIg4O+BsT9s1GCNxdYcQjCGN7zpZQkzM3cS/Trztwaax2zY9J2fnLJYnrNYr+mGgygpW6y3r7R2TypAZw26zoekdA5qL1Zb7ZsAqwdV6g951mKxCmRKFp7cRbQrevr5ksajI8pLB9hwcHrBrtzRtTzWZMThLPsnJzYKrlzXltOTu9p7PPvicGBXz5YzcaNzQ8/XzV/xf/5v/O33h0LMWbRSTeWJ7t33D3Voym81Zr2vWqy1GG4ZuQAq4vLiirEqKPGeQkvu+wWvPLjRc3Vzx/O1bPv34D3n76jX1LhU/B+sxJolOR0cH/PqX37Bb1xweHzKZlNze3eG95/vvX+Md3N2uWRxMfutrW8TUFWBtP3YHT4ghMYIJEqVMyjcwE6aTLHWeuNG0Qyru9zY5wruuZrXbglaUVUm/HcaOPY+QqQvQ9j2IQAiKMHgICcdglEkg1xCYVRXODWx3W5x1ZMawz0Yb7ECwHqEVWTViMXWHrgKmisyzGSpMKXOJ0YLb27uEWvQO4SW26Ym+p1QZJwcLMl0Shob17RVN3RAJKCHx1tHsGrwVSAmFUGn8o5hNLNF78ix7KLRa50El9Gw34kNm82RC2607jJ4hUAk7luVMy4wYbFo3zKfcXq+YqAOKqLDrGiEEzltKU4JzbDY1QmtULkE4Tp6cc3R2wnq14f7+nulsQTmZEJsmZQQ1HUZGSiORQeFDhpzM2ZETVMXNxlKLGberhk29Ahd59XJFHmqeHBdMTUGmTeoA1ZphcGhjWN/dISOsVylb4tHTx0glubm/w45hlXZw5ErRtz2qkngGOtdS6pT1Y+8GjmczXtSKkycndJ3lNy9fcNd3FCLjtDhGaZWylIJhVuas7ndsmx0+M2iVoY1JAbjeY2ME68gqg4gR1ws6a4l1DbZlVs5xwaHCiNyNBoIlM4oyNyAkrh8oTMb67p6u7Yk+pNDjXY1O2jtZZQjSEhQoo7l4fUE1qVBlwklW1YzprGK33ZAXOT44mr4mrCNFaSiKirre8frtW+aLBdpIqklF17f0fc/p2Rnb3RrnwWQ5JydnePvbd4jC+8GhyUC5F0yT4Kne+3veCefvbXHfzv43fpbE0h+6qd8hWd4Jv+/e4z2BUvztsNIfBpH+8H3f//WHYaRJyH9wvT/8L4wa29gZPhYArO3Y+Q1GVWitRu3PI0i4KilGlrjUCZPiU/t9XW8JPnB4dEj0qfNZSgkyccaTCK4xRo/HwT98ZghuFFcZ7xUJvbJfbyaB+113gBCCLMveY5+nzJuyLNJzs+0gioRrGYNB93zzFGCqxmJCCrOVRv+g+LHH9sT3zs97Z/adMx3xcNqbtmUymfCTH/+Y66trmqbhxz/+MRcXF1xfX6OUoaoqmqbBGPMwFvaft+fyvyvQ7AX0hOmS4/p7z6EPI299/zOtk/AfHzrd08HM84yynOB9MvCImI2ImCyZpwFiGAVuh5B7tNDe+a7TNfEDwZwH8+kDkmX8DxH38aI/EOL3haH9OdzrtntT9vtj929iXP72Ofjb29/fiR5SqjLBv2eeFoQ48MAnjyI5zUNyg0d4EKoFe9wKqNEtvhezhZTImG7gCdMyctX3BwzxnrN7DNDao1LeP5h4YlAE0RNl2mdIoX7EfZCpHCt0hsTTse8u8rG6w8gZ3bu308Hf885HtENMTHOSvR1GoH5qP0iCtRh5UOl7JxxNeo90RNLx8uNFxIN4LsbXRxmQ0aYAyuiQsgAFmakAhdFpYsPQoXQ/3hz8A5xfjDcsJSUxpgvaKPOQFuyCQyrDQXVI06ySa4aIEClZ3dqe3nVE57m6eUvT7oBAUVR4PFrlKFVQFkV6Pz+kwNPujp9f/ve83vyPFJniYHqCALQpUDHQ79YUKHqtkkAwbPnmu79AyiUxsxwezXnz+i23t3esN2uqoyVSCM6PD/mjn/weF5sNu/ue0HVEIouTZ6h8xvmjM/7F/+4/4+QPajr3hma4Z7W5Zr3asN4MrFeWeh1p60iwAekj2gqCS4xoF9NkK4yVvl5IVsEx9AHtFWasksnxLMmxy0GKiBECNbowApFcK3L1rhFln1UtYhxfx0OAKCKOQaKRoCQCzWUf8SHQu5ACEUlVsWEI3Mv0aNF6XxnmwQH7227eOWw/YERq2dZCgDEEFciCZXAdu90OGwJKG/IsJy9KiqKkUCYVrUb3ufWpFIaUo/M+OUpSB4MgK0ryvCI4j1MptCUMCRciSUnPLgoyYVDCYF3AxkieF+STnN5Jhl5R6BwpilHgDQQHPjhi8GSqJKhAbxMTtMpLpEoTOzcMWNmR6TwVlFSG0ZpdC2JI7Z02DpRlwPqGEDMyM0lcYamw3YAdHDJL7U7DOAlxNiCDpDAlRVEhhSLKiMl0cul7SdtavA1o0uS3yiZMixmuCtghIXMCkaZtOZhHdk1NltUMoWWzu0MaMCjkEMlLiTIKFzp6O2CyjBhScSO1+qT72TvGG8SYJlfeSzI9QcSSjz78lP/8n/8XPHv6Ecbk48Mn3U998CMrNYznOBUhtUmMZR9TaI0fW96tTWiapqm5v7/hzatXfP31V7x88YLtZkPXtrjBjm6BFBBmCknT14jsiFk5pYsN9WqDzt61sSmlQeRAxLtA3zmqicSLVCi1Pr2nUYoqmyWWf58KTc57BjekVnVfo4PG9W6cfKb7YG+7dMcWkagStsZZi/MuXcFC4axFeEfb9ohMEXKB1ApPpBsGohjIdYF3nujTPcbKgSA8WW6YTmbMqimuH7DO48YFXj+0aTIuI8ZIClPQtj3d0CGVZmIy5O+gogcf+OijT9isO+7vd7x+fcH5ozOsTW7H59++5B/+wz9mt9vy5vUlJtP8+tdf8cknn9L1ln/zF3/JP/yTP+arr77l9370BdPZlCdPHnN1fckwDCyWB+MzWNI0PUIIVqsNeT7h7dsbxMeGg8MTqrIiLzKyXFCWU6wFYwp2u56f/OQDrq/fMplOcG7gbnvLs6cnSKn5+utvkht+vuDV6+/4ye//iO12w8HhQQrMFrDdrNnttriYAo8vLi4wUfL08SMuvn3J4CPtbk3dtlRSYe2ADzUnp0umkwoBXPcNITpevXjO29cv8N7irYUYaJodb16/ZDgeaLuWzXbF9c0FQijatqPvHN6nNsQPnn3ERx9+jDFj90MIZMaQLQ8QAgbboJTh8PCYN69fkgJbyzTWI2SmIMRU4EbIhGIh8Ze73uIDKOXohzSniaQJOlITo8Z78eA0Sq8X9NaBMuRaJpZk8Cht0EZzcnJIP7T0Q8tkUrJerzg+OqHvLav7LZ9+9CWr1Rop08T/fbfSwcEB19fXFEVJnmcMQyoqPH3yjD/90z9LHThBMJ0ugcSxnM8PWK9XZJkGBIvFAdY6siznk08+5Ve/+hXn56dorXny5CkxBhaLJXtm4uPHT7i9vaeuNzRdw/HpKadnZyzmB7x+e0nX9tR1y3I+xdmAs+manU4nfP/mexaLJVdX18xmh1RVxcHykMF2fPXVr7E2oJVmOplxf3/NwcEBxmgePX5M1zX0fU/b1VzfXFCVc6yzKKmZz1Nx6vT0nG+//ZbjoxNOz45o247r61u8T90Fdb0DUlCpNpLlwZx61/zW1zZAVRiMAa0yEC2BnnKaUU4UfR14+33NZ79/QMgcnQvklad3d5w9mXFwskTpnuUk5evMFwWbzY4iL7m/Sw4wbQRKGW5vdyyPNNt1z1psOD09ZHA3KJOTSZBKEmXH+dMFtmu5fFPjhks++fQJzne8uXiDjwPVZEqRGbQUHCxLFssDyqrCDi4JTaJjMqno+h1lmbNYTFmv78nzgjKfMQyW65sNz54dIJDjIqwnBIdWGdvtjqxQuNCDmgKRSVnRti1xDGium56iSEVS7x3WebRODqoYYLNuKQ6KNAdTjuXxEuscLnjqtsYNljIzIATWWdqmIcszHjpmJazXKz7//DPu13cUZZ7QSDp16XZtT2YgMyXOpbZlEVMHolCCtuloGk+mxNiGHBBCs9k0eNcSg8JozXJWMfQ9H3/yAavVDWWZQsWl9GijUkdOlha/w2AxmLHNPwnBKTCtou88w2Dpux7vLevVFhc9y+UcPwYi3903rNY35Hkat7ZLyAhpBNe7Dic0vZQMwTH0DttaQmixPqIyw/J4wuAgSoULir4eyMuCbdty/uSUy8tbog3cby55cfENi7LCVIJuaHj06BxtDN3geX15yZMPHvPzr37J/+O//+/Y0rI8X2Bp6YaG9fqWEC3ISFGUeA+TsmAxm/PzX75AiozJbML9/Za7mx0ff7pkcJ5+zIdqtveUlUGqjH/1r7/iyw9PqHeWr37znNlhxenJCdNqSgye09NDbO+4vt5wer6gHxy7XRJThNB0TUPX/vY4F6UMQ+8gyrE7UOJcSIHKQ0CpAp1nyeyhc9qmpXN9wp5pk5BKMbLe7fDW44DpdIaQkfl0Rld3BB+QpOyIOHbwWSkZmtHprSJZleFlQilN5yVd0zJYS9f2hAi5ztBR4gZPDJKqynD0mFxQdxukVjT9DuE1VV6iomS3bdh0PX10eCNxQ0MRAs5CYTKqLENEaDYbhm6LzxymMHiXeLx5XlDbgc22JUqd5rvskCiGrmfoLedn55TFlDhEuqFh6C3eOvq2YzqZ0A8WNwi00kQkLoSEY8xSB8Z0UtFL2Oxq+ryCkEToGALTWYUUESEMQcgUdh8Fj88fsd5tufzZX/Ls8TPmizlVVaGzDKUNQiY35u7+mmazTviSxRHruuZn3z3Hl4fEbMndxrKqB4QsuXz7ls31JYdl4PjgR+TR8d3z50yXS04fPWI2mzOtppRFzuXrN+RGEwS8ffOKrCw4Oz+hswOb3RYXoDCa6MG5AMbgBJh5RW8Db+83fLe9whaGb56/5vTJUTJIDI6OJMxkeYa3gbZtOT5c0u421MOAFoqmbsnLnMFaFILJdILt3ajbKCaTKUpr7ODph4E6Ngip8DGglSTLTdIinEeEjL4f0FXBvKi42tTIkKyQe0Nnwr8JQugQQlGVE2KUmKLAJ/8WXd+ijeHJ08f8/Oe3NF3g4GBOPziCSOutpt4hiUxmU6yzdENHURZ03YDzacxpY+j6lPmktCAvyt/l0f1DQZr4g5/9bSF7b5T7dwjpD7/uX6fSGi7u/00YO/jCe++X1m7vv9vD57KnT7wTxPeEiD3v+m/uw/thnGNMKjGOa8n9v5cjnSJAwBOjRQqNjzrtS9RoLWi7exCRoswSShjDXjOLEUxmECLDO0dZlojxObxZryjynNl0Oj7vI0rrdM15NwbPp2KCcykrcV/0jjEwDMnICzy4t/M8fwgI3R8jAOc9IXjaNnVfJr59QrdWlXpwd6fjw+hgl/gYHlzSwzCkLDGlH9zmMTlz2VM39ucvHcu9CX0cK2OI6H5+mec5xhiur6+Yz+dImdBRAHmej/x1/4NxtxfU99veBb8vJuw54XtxPQnv4QHbsxeetdZjx2nqgDAmoyxL+r4dj2fPdrslhDP6vgcRESIgFQgRRrrHWBgemf5hLC69n1cpHjrp44OZOt1b4kPhSI5Fl3cc/veLRu8f13fXQvwbZrX3sUn/oe3vHywqcwSjEXt0fUcCLg6jGCiT2BcESIeSAhlHtvgYRir2wjQRlWTBJALH5CRPSIo9TD4Jx2PDCUqME02hgMQxGrXOURRXqId9G72yY3qrJ8LY4qFjPgahjqch+vFmMQ7u2I4HWab3iGmgCiFH17lJSJsYkfG9mwOBEAdk8iaOYXmSMb5vbNPYi+x7N/y7m18c3fR7ET2Fzia8BFGmaoyJCJ2RZwVEiclyeueQ0pLpAjv0QESrNCGXUqQgF62xg0ttbzIVOkIcEDGjyOeU+ZK+3yCkTME8IsO7SJbl3N9f0A89t+sbFtMlZTVDCUlb3xHzApPP2PPmldTkWQp/nM4dJ/kUFQxB7vBZQa4mZH6glArjLIMWOCUJMtL0O27ffEu/WzOZG2Yrw8uXL/jTf/Xf8V+e/B+YFAWIyJPHS2wmePHrX3HnPH3sKWYLSm3Iq4yTp5JPS4mNFYiCGA5T+3Dfs9413N7fs15tGLqIDYKUeScojCF4uF8NXF471nepQmaiIjaSvoa+E+xt2GIUrSUeYkB6EFEQfBK7da+QMQXLKiF491hJI0vJ9HOVzMLs07Hj6EiP8R2nK4SII2JlwGYKr9NuJJzSWLyKv5sT3SDxvaXUhkhyCEcp6IYejcFEjVEZSkmKYkJVTChNQaEN08KgdUbTWlbrDf1gUUZRTgq8tUQimSlS6nT0YAxeSwbb0fseoRVoQe97VFahsozBOhIioGJwnsF7jFZkEoKIeOFRKiQch4TgEpN+HyZotMB5SVnkSeRPAQsEMjbdQN81KGGYzSdUkwVFPhmLRwUqbHF2SA5gN7Dd3ad9MxmZzlE60gwd0icRMisKYggpbCtmFNWEoirpbY8nEGVA6UhwQ+K6xdQ+NKlmHM6OKPWEpm4x8/SwX2/XxEKzqddoE5hqw+XtHZGIdT29HdAZKX3eJkG/6zsW8wrbWQZrAYcScX+DfHe/C4npLkXFtDzjn/3Tf84f/4M/Tq3X4+gUIiQ3tB9wLmBtCjeRUqWGajVy0hGEGOhtagF3bkiO7r5ls91w8eYVX3/1G7795muuLi+IPgX37JFGIgpc6CmmmiAFm92ao9MDJouKYBsy/S7IRIhAUWpiFPT9QFMPVAtH3bUJ0bVPPheKwkwwlSFTE+q2BiTOW3rbEe2AHiQySCQaJRRGS7ySDM7Sul1yTzkYeks39PShByPQUeOdR4XU5pybjCwrCC7VP0PwKAS45NyKOoLw1NbigsB5nybtHoTKMDJiQ7pWnA80Tc1kmni81ln63ibklkrj4rfdXr55i7m+4bMvP0VKye3tDferDSdnjzFGc3VzS9s7usHz7Tdfc3p6xPX1LacnjyiLCWdnZ3z7zXc8++BDvPNU1YSu77i8uERKzePHj9ludwg8KrXUYHSOd5HPPvuSvnO8ennB+fkjPv/8S7phyy9+8Ut+9KMvaZu/5h/9yT/mq998wx/80Y/Y7e4ZhmtOjs9YLA65vV4RQlr4X11foZRkt9tQ1z0HyyOs61nd3xJXkfvNmoDg9OxR4vVVc9brFUoKvO3xQ4dzltu7W4be8/jpY+bzKdvtNrV4e8vN9SXBt2zWt1SzOSEmR3WmDfV2S1lW7HZbqsmE2XyBlIq267BD4PDghA8//JSnjz/gYLlI93gpMVpD8DgX0VphzARiz+WbN3gb6dqBt28u6QeL1jmVztnt6nECp9BGIpQmkjIzcJ4Q5cjIFOzD7rXOiKg0i9jzBSNIpYluwLqUC9J2DVIJJpOSV5cvmFQZ02nJpCoxWmGURpVTri9ekKuS9e2Oq+tbjo5OybICaBJKCUnTdDgXuLy8oigKptMp19c3nJ0+Issyuq7l5uaW45MjvPccHBxSlhU3Nzes12uEUHz+2SExwna7pe9TS+rFxRVffPEFv/71X3NyckJVatabNUYlF1JimS9ZHCxZrddMP53xy1/+NT4Kzs7Pef3qDev1ipOT04QqygoWiwO8/w6lNFpl2MGjZcA5z/XVNXW947PPvuRq/L1Siu+++24MjD1ECLi9vcFog/eeR4/OuLq8x2jNRx99zONHT3n16g3Oee7u7jg8Wjwsqo6Pj7lfJZY8ozFksZgzDC23t1e/9bUNMKlyVusdoonMDkqKKiKV5eCk4M26ZrCezd3A6bM5PS1SCbp2IC8tJg8MQ5/amWVa1A62R4hIWSnqpufwqKIodEKqSc9iWfH25S1C92RFSxcGisLgo0BXgZcX3/Hk+JRHjzOKQnFx+ZpAZLPbIrWg7TrOjxdEJWjswP1q4PsX9xwfZtzf7Dg4KFlt7sgyRdcFIBWviYKLiy0Cx9nZjFevX3N+fkoMgru7DcVEUWaKvDDYqGgHT2qpk7x985bBD7iY3EpFmVNOiiR49gO3d/cYJTg7O8dHwQcfneOGjr7e8M3FC/7o4x8lR1Wes93uOFouQaR7+na3Y1pN6LoWrVOBzrbJ/fvmzWt2Xcd0NqeqSu7v1mTa4EUqDK+3W+rtiuUyFRaaukZoyRACd+sts7IiuIGjowMg0PU1Ej0KGpq+61ESbD+QaYPte4rM0LUtOXma7+/XMmLf3pzcY9YN+ADOCapqAjRUkwpjJQcHR7x6/ZrB1phcE2Kg6Wpm8yl937Na3eE6m9rS8xkvXn3L97cbrlYD8wPF4njG6maLRnNweMLzFxdcXmyYLh5h8or11lJMJrTbmulCUQ87Hn9wwJvn12yba759YXlyes5huaBzLXlxhs4FWkeuv31D8/Iel+2YnsFm10PekYuMftswOMfJ2TEvX75BasPl23tOjwJaGYzWNE3L8ckhVZljtEfLjIvLO05Olywmc85Opmx3a2SsOPviKTp6qklJ33cMt45PP/+I4Cxd33J2fszqbsOnnz3j8uqO9X3L0ckRZeFx1rDe+r8lOP3HbE3Tk2cyBbqOTkznUmB5YtM6ut2A6AxZNqSx0LYQU4dBCAGhDQ6QxmBKyKcTvOuZT+dcNVcoLTk8OMBLz2RWJJyaDPigcTbg+54iSuZHM7S2uCFiB8/y4Jir/obeeuyuw1hF8GDblHGGEsntnCmkh3VzT50L8qznYD7FDgNeKtREpdylLqKloZpWKArqocdoQ1mUCY8ooXNN4sLPS+bLY66uNvR+Q9sF8jwVm72PBDsAkrubDVIp8qJit6spjSaGNukySNyQ8kukTPeYpt4hZEA1nqHfcjY9x3WWtumwBQxSsN1teLw8REpNVRZsNxtmB4cM3cBgPdf3d+RFxccff4yRKWjTZCoV27oOIRXGpA5wvENZy+JgwavvLrm8uqY4mnC1fslq17HttmP+WcZkumS9esGmrvnkw3PWd9dURUnwgdVqhRQSqTUn56es7q65ub1lMp+y3W2pb644f/KYw4MDNpsdkYTGzMuK+eEJu2bgum55c7Xip8+/pjcN2bGgrTfc91sQnuW0YrvrkvO2yJGyZOgablc76i5yfHSOJ3Jxe0ltdyAkKEnjO2w3IPuYEIqFSnx/oxCZIDpHVZb4YDlYzDk/O6Wrd3QIwiDodhYTGmKeU5iMaDtciBACTdMhGs98VuG8p+t7+m4gy5I5KoRI03apuLprOD6Zcnh8zPX1Bd1gyYspTVszn81Tdz6RxXJG07c4P7DadBiTU00m+BBYLA9QRnJ/f0s/tBS/Y7DoHrGRROq9gP5D5y28E7fT7+F9If1vhikCD6KiEO9QGHu2+Q8duqMAvhcVeadJ7YNJ3xfSIyNbnR+K9++L/vvu4dFOzT6BMI7GUzFqUPsJahSJ9CBl4m07X6c5edMSA1TlAXk+QauU0+G8T0NLKQY7oHSaE+7qGpNl5EWeEFN+7LTKMpy3oxnMPXwfH91o7IuE6FJBOcsenNrAAxMcwBjzgC+Ro2Gy6/rxXIkR96IeOON71//ezf3AGN9TK/YIF/WueJKyDd+nSYj3jvPDGedhCERYLBf8m7/4irzIx1yUHYvFgt2ufpjHpnBN+TDm9oK43GepvYcP2jvm342bdyG3AkY0TRLQ9z8PIXWyGmMwRo9ooGS6SaHTlhjj6O4fCKFPDHWdTKRKjQa9OCJnpEQETyAVKpJBeN+VNzrG93QQEtlhL/ZD0jmi/CHf/V2Hh8D7d8d8f1z22W4/zAb4u5/df28RPVNp0SSQhNgT6AgxoKIYw0YVMorExR5t/aAIjMEJMFYTLDYEUk2cvYw+jhCP2Lsox78LMYzUdTe+9ziBZITXR//wXkiJGj9zH+QpQ3JzBREQId1UhGBMDZaIkVUciBCHsSVBjydKJmd8lA/7JEhM3TAmw79rAUhp6JAKClJkY5hYaj2QqFHgjyMCYaywpC+fOPJSjO0lo6guUsVKygimRRqL0nOcqFFSo2SGFMnJK6PGyAwfepRQSBEgd5iFIyt7TMgY1oGIwpMwO1JAlk9p7BWOHVJ5sqgRI8P58u4VTd9j+8Qrms7mLBZHeNcRgx8rkv6hQpqEtxZ0x8nxMYU7gdhi/AbVC/LuHJoNk9kJbXdHQaBVkaAE6ICZ5AxtCkDMMsV21/P/+m//n3z47EP+8I//hOLoFJM7pnlBOV0wrG/ZrK5xtsH7KdpMyDmHMKePL1FmIM+TuD6JBUeHcx49WtLbhJ4JCEJMiJZcJ3b8tum5vLvn5r6maxz9EKm3kc0d1DtSpcsJvEsPAa1kEh1GPvLQwW7j6ZqItREG8BZwEUJKTI9+PK/jXFPsW0tiEueTlCnZtxSkUMcU/hlDII54CogIJYhSvBtKv+VWZTlKSLLxoaK0Aq2SMzcEZC+ZT+aYYkJmSgpTYqSmb3a0bcNinqOUoqqmICU6V0ynJd4lZrjODL23DM5T2y4VtrCEMbw3isjgeoSVaKkIIuCCI8rIbD4lRBj8QL1d07RrovC4oUVnDpQiCJ+OSRR0XY8NkSAiOpOo3ND3A8JDFjXTvKIP7XjpjQWIoJlNjjG6ZL11NN4SrKescnwMdP0O18OkmJJJjQ0dtumYTWcp4JfkgDUmR5rU6dENAyJ6lPAUOi1+wpgo2/c9rW7pdEqYb3Y1WjsG29GNxYVdt8WLQNu1tG2D86l1VQrISkNZFnR9YOuHsdgSyIo8OeZsR1mkCbzzLqVgS5U6HqLi7PQD/tf/4n/PZ59+idYjN8ztH2ZhnHT0WJvYynskwZ6VF3zAxcDgPG3XJYe5H/A+LS7rZst2u6Gua+wwjN02ISEBYsCHgBvd3rM4xeSGbbMlaxISaLacYIcBpUZYl3eYLGGpXG0ZhrQg6e2A0AGjJEYYpJAYVVGYHFFpokgCgnUDzg8IAjFKtMwwMkNJDRKsCMRg6X1LQZbGRBwDtb0j1wWlUbR1jVSSKi+pigopDa0bRoEiOZQKXZLpDCctvY8MXYdzkt5YZOyJPj4UkIWALM8JPk08Q0wMx21dQ0wZCds+FWt+2y0zyUlR7xrquh5b+iO7XZOS6osJ93drvn/xHc42fPDhB5RljfWepu14c3HBdFrxh3/0+/zyl38FMtANDUU1wVrH9c0th4dHOOeZTOdcXlyxWJRcX17z+MljBhv48ssf8fVXX/Psg8dYm7oWvvrNN2Qmp+t67u7WHBwc8fLlc370ez9BxIRmKMsJb9684fTsmBAt54/O+P7753z++ZfUzZZ6u6PrGqy3zCZT6rbn6vKKw4NDThaHvPz+BUtTIqNKfP/gWG03nJ4+5fGjR6y2HfVuTSTSNDUXb9+Q5xCDwzqHFAobUgBdWZYoKSjKgrIqcK4H4Oz0mPOzpyzmJ0wn85G7ukNUEl0k95J3qcNOa0nTNFxeXhMj5EXJm9evOD4+BSJ39/cIrej71wihUFLjgieG+FDQShO7MeMliuRBiOm993M0pUy6D3iH3HelIOjtkEIuXZ+yVJzl1auXfPjRM4QA7zxlXvHrX/2Gk6MzLt5e8eryLQfHS54+e8Ld7S1d1/Hs2TM2mw1nZ2dIKdlsNiilyPM8ieMoPvroI37607/kww8/5ObmlhA8jx6dMpvOaJqWqpowny/Ybnb0/cDt3R3Pnj0ZXTtJ7FqtNnzwwQccHBzSdT1t04JIrpqqqtjWOyaTKddX6XgabYgB5vM5y1nF2dkxNze3WOsZBkfwqWOo6wbapuP87CnffPOco6Mln3/+Bbe3t5RFyXy+4MWLlyPfsufi4iLNx4RguVywOCi5vb3B2sCrl9/xn/yjf0bf96zXK6bTKW/evOL16yy10dZDCkYtDHWdCjZffvkFm82avm9YLGe/9bUNsLrfjnitSL3bItUcJByfTbl6VZMpCSIhiA4PK0wm2G5aXjxfcXJaEJGEmPJJdm2NkukYHhxPWW88g08832YzkDtBsxvQRrKpd0yER7hAXgjQ4K2jygxSCw4P5+zqLdoomrZnPsvJ8oyb65pclUwXEpML+vsVWkncAGU1pe9bHj0+4e7unl3bcX6+IMsKXr+6ZDozZCajbVsgBcgqJckLRQgeqSMffviMr7/7hvnBjKqqWF2viEFCMCxmqUCZCu0KCUzLCZnR9E3NzeUVOis5Ozul6zR2qOlj4PXVBUtVsZjOyX1JIAVE275Dj+6uwXa0fUcUKWA+jCgOh2DTtix14osqRFoqyIgLPafnhyyXc2IM5GWBG3OOnLunmhzghp667pEILl9v+dHvfUnbdQxDw7SoODhcsFqvmM8nSJE6Qk1msNZhjMb7tO7IMkNRZCMDVGCKHK0KhgHu7u+pqor5fE69Sy47pRSbzToh+khzgjvX420KSZ9OlkQhePn6iu9erGg1HJ0s6PoaHxyLgwrbO1QW+eTTc169vqbveg4O5vzizVsq6zg8WfLmYoWTDT5MefxkSaYV237Fv/7ZK/75P/nP0KXmbn2LG12jT5+e8q/+zf+XzjeYmeSLjz7ku4vXaCRlOaWuW5SOTGcl682G5eEBJgsQBs7PpwyD5fToKbbNePXqLb4VlIXk7etXHJ/OePN6x3xhsH1DGHb4oSUvM4QP2OB5/vwFWgmMkqxWWw6Wh9yvb+htncLuJnNOTitev75gSYVW+W99bXetRauCspiMuC9F03rquib69BwYXECYnBAiRa5JXcsJZRC9x7tAdD4FSQqJNKmt3g6B6BVIxXx2wBAsuTE4L9kNjkwWRA3BWla7jvnpAmUMfZeE9OMnJwSfcXV1RT8MKYNi1zO0gaFvOTpJ2LbD5SHbbUvsExu/HVoqnWH7HiE8qhBIkthVFVMOlqf0XWDo14iYnKCz+ZSLXcu2blkezvFCsDw84ODsA46va96+umC7XmMHCPm4foqK6+tV6hQ6zIkeptUM5/qk70VB16ZMjxgVZVmkeaaRdMOQXOk+EJoeLRTOaC6HmtnBnGI+Q1iHznJknmPbjmgMEsmiKDk+PEaJlNNQFDnb7TbpEdqw3WzZ7nbk2oDOKEZ282I+Z+h7/qf/4X/kepOCrGWm8FEQnaDUmolpQUa6tmFaTbi5vqH3nigkWht22y2SiClzTs5PqOuaalKijGK7WnF0fMRssWC33hKsR0bJ68s7Xl+twEruNpZhNqXtLe22RlSKZrPl6HDCpCwIITl+Y5r6oLKKTevpo2EhC8JQM8lKbHT0Y3DjdneHHzzOO84fHeFxNG1NmRcI78lUTjVdsN1tkytXKLyNTIsZQ+eItmV1c4fSitligXd+vP8Lgh0NLyKkhlEPXd2ghUbEdOwGO/KQJex2LdVkztwOxAh1s2OwgbKckOVlMngqzc3tJfPFFDtY6nZHkU+YTudY26eu6s2aoWmw2e8mou+xyElf2geAvkNU7IXp/borCe3yHeFhxH/sRdD3hfH3UTD7n73/fjDiZPb6gxAPr9uzpN8Pmny3v++LuvE9l3XSKqJ4T3wU+38THtaoqXt+fA9IplGSGBtxBNcjZIZUju3uDuciCyHRZf7wfSOBvu/x3tN1A0PfJT0KRkd4QkuVVUXb1qOo7kYzmBy/U3rOButHc+kozA/DiDnT4/t3D8fCez/ik1JH5mazIc9zQD6ElSqZkLZKqYfjKIRM4r+QCJU63pTSuMR2xbmkYWpt3iukvO+Q3h/TtHbjwXCcTsSzDz7gX/7L/+rh3FnryPOc6XRK398/nPM9B/1vutD3328vKu9F9PfHTgghra+NeRD59yJ6HL9D4sinTghiOkd5bhi6DqkUdd2wXm8pCsdkkiOVIoSkXyiVDMb4VPiPEjyWEPYi/+ggfx/NLd45xpPR+R0n/d3vU1HiAafzcK28Q9okJPcoBYV9bsz7Jul///b3XpkrBe+BLNCxAJECA0NMgmRy1o5McZJdX5JY40ooUgNHYgIHsafXxMQsl3KsXniEMEih0oCJyaEuRqFcyuRGlEISok0VjuAJQhLwqBSJgiBLQrHcu+FHpy9jO7LYJ+qKh89Josbodh+F8+TyfefojNHjomPfLi32PuJokt9DuCTciIRvESImvI2Qo+M+iZLIkfck1HslpvReITqi8OnYSZLzRApQPlWxdYFR2fh9JTKkgIRM5XTOpotUCOSsIzscCMqitUKVBXZbEIY8BRrqCVF5GvsWX9RI75HdghgUQin6puFi9y2lnjApD5hPF2RZzvruNUbF5Ox1DqkEEc/gtwzxhiglORUbv0NI8HqLI6Dkh+AMmTlk8FvyMIypvSCyiCo12SQnqxRZpaALrLZrfvZXP+WLz35CueiJSmBwODtwu7rlqJwQnEsteEJwNHtKtZ3TWZDKIWTi4CmpyHPDNBaEqHHRJxeOd4QIWmbEEPEEprbES0U/s3S9Q5YdMe8xfXKziCjpekffW2IwaGkSokVANgjKhUB04GwgOrAOopepS8PLxDIPKWApeLB7+kZMFbLoQHqJCgn9QgBsGjcBkbowYkTLFCrKyFT+XTYvOwISpBnHbmKcK5MmuzaEFKY3maZqYUyhFV3r2PUd7BSajGmRoaVniD3KeJTOkoOSFqM9SEHfdex8y2I2AVliXYtSHhMcdtgSZA5CMnhHZwfyMjkkB+9oug2bdk1ZTchChht6ylmZrrVgyKmIvcB3Ni1UlcDa1DalvaDKppj8AB2noyDd0XRbdm2DLjJs9PTREaRKXR/FHKV0cpG0LXmUZEgGBapUeFpsD4aMykwQOsOrQONrOjcQ3JA6FYopkoASHi8jq2bF5d0tE/OGQmX43iYHne8ROmJyiZfJCb2ptwRvEF5RVRnagAs9QrjR5aJBWkLsyZVBOEnXB3oRyHKF9wO5yYguh2B49vgJ/6t/8b/l448+QemIzhJ/zuTJse28HUUBT4yJcZjCOuRDJT74JIJ758Gna08JmbpZhCDXhoODwxRaeXdH23XUu10qIErJEDx2cDhc6irINMpEbDcwLZeo2TF2c4cXDlNJgg9kWqOEIfeB3m6wXRLCjdTM5xNitDRNQ+tavEjJ42UxTwzRocMUGQRP8InvrHOD0hofPVnMcT6kNmE9T63vCorckPmBrNRoHRi6DqUieV5hsgLrUniO84FgI155YhEYZE/jGnZ2S+87qjhjqkpMntPZAecDdkjnbzGdMM0maBUYXIP1A8pAjAZlSnQIyL/7Wf7v3YrcYIcOomc6KZlOJnz9zTcopZjP5sgMhn7g8dkjbm7est3u2Gy33N5eI2SgKA3TWcWLF98jpcK6SFmWIw+5JYXfKJyNaFkxnx3xzTfPefr0McMwkGWGpu3xIXB5eYnzLV/83hcMnaWqFlxcXvLoUWJRP3r8lGFwHC0Pub99ya7eAgNKD4jgcbbl/PwYcBwcziH2vHm95uT0lBAlHzz5gIs3VxRZxaI6Yjo9p71bowLofMbtyyuawVE8K6h3a75//h2Hx8cgJHV9DzJgXSTPcgbnsQEyrfno0RN+/OMfM9gBHwMv3rwmxMDx8TFKSrb3N6xu7jk6OuP80QcMuidOUoAgQF5kzLOSrqm52q4oy4rT03Nurm/4VBZMZxWvX39HMBpjFPfbDeL6BqkNYWwfjzZNcl3v0EKNcyQYqz0pKGksUBmdJaQQPLRdmkKjNGQ5tN2KFy9XTBdJiHzz5i2np2ecnZ1xdX1Pnk/59vl3SKHIyoybmyuyDIIfeHR2gh1a+rZmdSfQCmbTirvrK7QQ+OARwvKLX/wKa1vW68B0OqWua4q84vvnL1ndr8mzjJvra2IwLOZLyqLk6uqGp0+eIaTk22+/JcsyqmrK69dvgRQyq5zm6uaak7MTblb3PHv6IU3bsFwe8Pr1Ww4Ojvng6VNevXyBEoKh9xwsjrC9Zz494NH5E2JQxCBGN3rOYnaAVor1/cu0qC8qvHWoomS73RFjYDKdsJgvaJsVeSEY+oBAM5sW9P2G6+tXVBMoyhwhjzFaMpvNaBpHXlSEENjt6sT0tAPb7SZ1KfyN9uz/2C1Tiqys6F1PMYWmaTnSR1TzwNknitwUbLaO65sbsmoxHkfDclFyd92xPMrx0mFUYLXZUOUVvhMsZoZiJgjOI1TLdCEYOsf52Sl3ZoWNA8PgyTJNvXZMZxlDGxFV4g5npWF701OWae6cac1u1VBkBV9/t+XzH5VUUzg8qLh6O7AZPIdHUzarjnbXUuUl26Fjt2n4yU+eMa1Kvv32Obu6Zzqr+OLRGU1TYzLFah2YVSXODlzevObwoGK92mKjpMxnrO9bNjvHYiHJtCbPFa5vIHiWswXBRapJwp8M9Y6X32z4/T/4MYaAtQPbboepBP3GsayW6Gqa2KZSkRU5SmTs1muEDuy6HdbD8ekz6tZz26X7eTERRCFQRhG8IzCgCzBaU29bymJKux24v98h0AydYAgSUxY8OX6KiZrf/PQtX//iNR9++pjZQfoeg+vph5Z8YjBGIY1mcHYUEiVKpk48YzKkUKPJR2KHgR5H26bOlrqt8dYiYqCazDk/0/RDx3Q24ermks12jUDSdw0Cxf2mZdvCxcqjq4p6vWaoA3kWiYVnsshSt2y3JlqNEA4/tGTKcHZiKCY5RQmnP37Cm4s3bFYN2goGF1CFoZjnfPX6WxbTJRaLChlKGozOOD95xJ/+4i+IRU7sNd1OMvQdi0VFALa7BinSXLl3Fm1AC89sGTHykG5rOZx8yK6UnB4tKJY91TZQFhPaHpxL4Ziri685PT9HKsW8mPHddzc8Oj/l9uYWkznqXc+by5rT84IQOpyATb2jnJUsDnO2IYxpfr/d5p1PBqwQGTpL17ap60sI8jJPSJPOgTRj7kLqBtNKJKfwYIHEu2+7Hq2zxEnfNbTrHfjEj95tO4LwNG3D0I/uP+UQQWLyDIyl6VIXZN9IhjYgomY2W3J7s0oYlxjxNqKjwvcBERR4ifCaMpuTL5b0bUe922BlRnA9SIcRBnwPSlF3lqKzCAzT2SLhFwO0/UDbW+q2Q7cZWWG4uL7m8DhncXzMdHLIm+9ecnN7xWbTMJtMuLu5o6tT2HleSWazBVqPYjlp2W0HTzUxZHky/BhjMFqgtMKM3eL9pmY6nWImJetgOT15hM4qpg7ysqK5uUNoQwxp3S5i5HA+TzkgbqAfuhFHKNN8RgiyzHB7s2I6qSAG7NCjlObk6Ijloqa3kcFuU95SEOA1ypRMpyWLRQpxv727RZcltzc3LA+PuLi7o62bZOaIPTFE5rNZQpSUBdY6ZIDJbEqM8OrFa+xmwkXT8c2LS4ya4KxC6DwZSYTHiwB5wfzogGJSYUPqULfep+KNk0ymcyZVGnvReiqTM6Ap8pK+c/QMbHc1bojsyobZYYUMgugC0aV1BxFiqizS95a2aRmw4BVaGEqT46LHdkOiIYTI0A8oJclURaFLQuzT3FknA5oaxbXMFMSY9KWm7TEmFY26vqbdbdLPm4b5fIGWkvv1mu12Q1YoTJ7hg2W9WaOMYTadslrdESJ0bctO/G7Pbj+6ax/EQJHyA8NeAIzjQBUxoZQFo1sW9qDYvZC918z3psZ/H3LlfZ65lDJpEfu2xRjfdXTDD36e5EjPHrH8Lrh0NJw+iL57qoJDqGRSjSF1N+9DrV0Yja8RIHHHJXtMiEBKjwwCpCVKhVAzpILoU7eAFAM2dHhvGYaOvm+JMZDlhkCWTJ46sqtXEON4H0zdOzINIFLAp0uObJMMWIMdxmBt+SC4hpDwWXvHdTIeFQy9IwSHc6lDKBnEIs7Zh6KCc+n9hRQMzoKQZDqhi5SUZErjXHJ4p3DZyBgHiRBjvhl7FMkoDu/d18ixuO0p8oJ/+k/+Cf+XX/41q9UaECkoPESKonhYA+xxLnuB+d/lQk9u8oSu8T7iXdKrvE+oq/RdR9f8WCRJAvVewx2RP3icG4g4hqFDqsj9nePNmw3Pns0JMcOFSGayRMYYu2fTIEqfHfDJaLkvGsiEBA9+FMAf0EHvFRtCII5aahz1Z/YcHEQyT49YbyV06qwfPzv4HmLqyExG8L878+DvLaL7UKOETlVRRnEcECKOgzI5aIXo044+zBwSY1vLMonwYSCK/UUTCNEihUkOcWnGL/juC2uK5AyP+88MY1QpSVgXAbmvmsTxZjTiRaQAhUpIhWCJY6XNx8TSFkIhYjaeszQo05/zUWxPLvQoRia7gIgnxg4YHdsihUcmbqmEaNJCNkbEQ7XNIYMEsb8gxv2VCUGBVMSxZSIx4f3DVRRjIIqQjmt2jZBHVGUShnzUKdBzPOJRqCT6B4EwEmUcMtuSWCMSOWkxuiB0BWEwaHlKHZ/jqkuErlF+AsMcJQ3Wa6T0ENJF6tzoYhACH1LycCEEzvZINxBlzZ3/Gbfuz+jjlon+gFn2Yxr/DZ4dUvV4v6bK50gsmSuxu45K5YQMhgyaIiBLQXaQUbqMGRkQ+Nkv/5Q/+MMv+cfH/ylUB4jY0w+39ENN260JIozFmsikKHh0ssAMGaj9haDHh4MnjbokXPsQ8V4QYhK4vY+ULjLJJSEKJnnGUAgqE5gVHucExIzeerphSPiimCbvu62lbmCwAnKVeNneI6LA8C64gijxLo1TObY6KQ25UWgd8XacTwpDpgSg8A7iEIlBMDjoWo/t0/WtY3oP637HYFEcTdshhaEqpwjnED5VRgdr8dHhgiXPFN4FurZFyYBQES0Ug+1Bpf0JwbLdrRCZoygP8D4y2J6iLMmzCm9r+n6HDylAsB8atIY802x3yQGb6Skmz1Ba0nctITO4YAkyYKVnnudUakY31Eh8uqlGQ1FOEEpjaYgyYpWjbnd0XcfElGMblmKSFzjfUDf3DK6l8wlv1DlPN7QjWzwmF14ELTVVUVGaHBlFap0OA13fkaucPKsosgwvDVE4jMqYzKa0zQ6jEgZGx8AQOgYvyHKB9xCEIwhQucALidAGVCo2BJ9mJVIlvmVuMqoyB+kQ3uFHF15Z5Yl3Fj12aBPOROvE8guKIpPYIaJEzunxM/7ZP/3nPH3yIVKCNmkitufc7UNEfAhAStgWQuHHAmearMWEhhjHd2YMmdE4Z7GuQwrIs4xJmcJb87xkOl3w6tUr6qZOd/GbW3q7xrlAFDBbzMbxn0KDqskC5x3eN8xmBX3XpeBVkaN9cgl3Tct8uWQ5X1DNcoR0hOiwPk06Y4woocmLgmw6x/uBpt5BDGilqGYTBmfTa2rLtJpQljMymROHhqzIkEri6wHvB7bNhqvVNYfLM4qqxMfUMeT8QNfVyU8hYnLhdB0Oy+CT6KqURGlBEA50xLvEW6/yAiUkgdQtEENMQoERCFkQx7DnqH7763u7vk8usXZH27QEd8ByMU8TYx9o64bddsfpyTEnp6es1yvqpqaaFHR9zXI5pSwN33zzHOcCn376Gc2uoawKzMzQ9wE7RD788DOcDcxmGT/58Yx+qPnlX/+MH//kC+q6YbGcs9mumS9SQK/JBE13NzrYpyhlEks2Bt7UFwxDz6efPOPP/vwleX7Mr371FZ9++jFnZ+d89913rGb3GGNYHBywWu84PX7Eo6PHnM0fc/XqBjUY2sHTtTCZzzk4kpyuB379zTd89/x7Budo2nvU1nO/WmGDRSgYBjeiKQyqkITG8sGjRzxaHoAQ9NGx2224ub/jxfNvwAVClDx+8hHDZMZ3333NJ5/+iCzPiBIyo9EovG3pmg1KwaOnT6h3PSfnzzg+e0bXbvnLn/8UXeRM5xUHx4dUF9cMISLjgIiBobdIRr65yZFC4XxaqAhlcAgcligFQ7DY4EaHWprgCxkpSkU/bJguFiDgfrVNPHJT0DSOw4NzrIO79Zbl0RFvXr9isZyPrlpBZgTXl29RWlPXNUSfsipM9hBoqIzgu+9+xeXlW44OT/jogyfM54f86le/JtMZ69UaJSSb1Zr5fEHTNGSm4NWr13zwwQccHZ3w9ddfE4Lg0aMnvH17QQiBxWLB3eqe9Rh06EJkNluwq5txMSaYVBO6puH06IjT4xP6tgUPx0cnfP/9d5yfnbNd77i7vuOTTz6nqztefPcCJQVVVXC4PEJKyddffU2eZXz84Yf81V/9jMlkwt3tHT/+8Y+4vnFkSrNp7jk8OGVaaW5vXlM3a5YHc25ub3CDYHb0hPu7e/7kT/4R//bf/pRH5yc8e/Yh3g+s12smZcV0MuHVy9e/07P7o0/OWa8GluWCgTXD0CKQ5LnEdhG3ExRZRbvpuHm75uBYkReaDz5Y8otftHS9ZXaYJfFSTanXLZXJ2Ww2tK7n6HCGFOCGQFkUbNYrEIkdL1VGCElMX923zGY5RmdIVfHVt6/Ic4HzCTG0qwOBHGs1x6cFuzqFeM5mS+5vVxwsD0A45rMS5yLD0LFczui6nj/9sz/n/PyI80cHdH2L0RldtyPPDZEk5G+3DoljyC0HB4fc365omobZ9JDBWs5P5+QZvL24Zvb4ID135iWbzYZPP/6U+9sLhrolzzLqpuaXv/prlNL0veNoPmXV9lS5pr27Q0rD0XSGDIFN06MVrDufeLpC0cfAzXZAGY0XAu8d96sVJZJca6KIeCLaJGfcdrNjvdpxevKYZ88+4fk3L7i6/DmXtzf83pef8OXHH/H8199zcbXhg2cZSiVn2GxxwGxWIUTk7u6ark/mnKLMQShW6y19PzCdTR4WkpmRaQUnBDEIyukEqTK2qzW73Y7oHbu6w4dIlhuur2+pmxaTFfjguLvfcHb2BGmm/Plf/RVWlAQBeZFTGEPf7mjblvki4SNNljp0z84XtG1LP1g++OAxZ4+epJDFYcdnn3zC99+95K5tmR1kbLdrzOGcVb3hT3/653x4+oyffD7j/v6Ku90tr65fMztZcLfbkRWaDz/6kM19jZCBwe6QMrV4vvjmjtnMIc40SnUIIWm6W3Y7x4+++GPk1Zaf/fX3LM48y+URb16vOFgusDaFtR4dLZHa0/U1EcP52ZTNqsWoBZdvLghRMl8YtpuWsoocHZ4gMLx5dYWUkfl8yvp+/Vtf20tdYizYraV2FrTE5AdMJsfEmLoFZUjdrlGkgLYoA4N3CCnHINSM4AdiF3CiozU7dvcNthuS4G4UN5srtFGEkb0vnaAfOqRIc88sm5CoTxlt2xMGQb3eUW+22HXD0A1kRUmZ57R+y2AFu9rS24Gs2pEVOZPFDOtbQrS0/RYtI0pKpMzAR5rGIhmw9p4in6S5fFkQtOLedliXeOu7bc9UFlRlztXbC4K75HB5xsmjc/KiZHN/T7PtaNc72rolw2FbaGWHNi3SeKLoafqAyStmi8OEtnn5hsIrjhYLLAOr2rJtOja259HpEcVihrAGPVkyWz6mGAQ//6u/QJtIsJ5lVTHJM6qiwvqBb777mjdv3vCP/uSPcXYYA00TfvP66o6DyZy3379CZCVN22GqKR8/OubF1QqvJNdXlr5LnYOLoynHs5LPnx6zub3l+vaGSV6wfX1JWZUpLDfT9K6nbTYYKQhWst3c8PTpGVJFZFD0fWA+l0QpyA6OuNh2rBqL7SK93REDLI8O8blCTkqYZjidURQLBm/TmgXwQhK0RARwXc/hZIkIPV1QRD1iY5VE2Ja8jISY5sfXb3cEoJpV5HlJmUl877CDRSmBC45Ns8MR6dsG4VP2W9dJyqwkj4Y4BGzMaAeHKgxhAOs9IYAdBAZF1zi0jLjB4nVPMSnxscOFAd8BLmB3DdVYoOnclulklsxzg0NGRbftCV5idM4gPNcXlyw/nzGdVKzuBGHwI1Xgt9+i2IdKyhFz8r6IPv6bmERfOaKBGcXT1EUdRgHxPeTKyCF/3zG+d+e+E9H3f07sacLeZDo6fWMSVRld0anIMbp7RUgam5DsEaB7I0CiLSS05f677Xnse2e93Iul++9G+nw9FngRkRAtPtZE39NZGNyMSpygVJ5MuAwE7xLCOCYMTIgB7y394BhsT13XCCCT5sER7uxox408uKkFghjSvnuXdMvk/A8PgvP7QZy7XQqnBx7yBWMMOJfQJUKo1MnP3iWdug18CFjvMGTjsYqjAzx1XiYBOAU9I5OJUYpkjEnvNe74nuMyCrxapY7Vs7NzfvKT3+dXv/o1V1dX3NxcM58vmc/nrFarB0F/v1/7DoO9kC6EQGtNlmUpgDSAD/vjkQpdPoQU9i4Ezg0p4DX4sYCjGEj4GmNSd4R1lhAVSqXvvNsELi62zJcTdC6oBCmLQyikSh0CBEVwoyNcxaR77wXwdNSTLiIlUpl3RyOEZKh+IDMkwXw//r0fDdwxEH0yYgsMSuapmyMm02oY11s8OP7/w9vfX0T3EKUnRIESBciIjwncL8We8rOvlCR+YSSJGoHEw9FyQoyKGIcU/IIj4PDBjdUzB9jElh6d4EI4ZMxHlEuqbjAiVyQaVHLG+2iTR15IiCOzfLxpEMPo2tKE/z9r/9UsSZKmaWKPEuPO/fCgyTO7qrtqerrRM7INCFb2Ehf4G/hzuIHIikAgAllgL4b0TDUtnpmRGezw49S4mariQs1PZPUOqc2G30TGiZPu5maqZqrv937P6xwI68V3ofE06Ag3TFyJDzO0QmPQHn0iHUK0vpHC+UHtQxEAa9EEvjrn1XQ/QWxLZw82Yl91GyRTFCFCDBccf2wOi7ENjtZzoxAYN4QNiB5HAeKaUH7uN8wDXzjQ8dBuEgwsSy/mSxWiA00QOJz0rnElA5yWdLqGztH314jwARXkOFEjI40RvvqodYQQDc61tF1HqDxCwVpDHGnieIbWGUr0XsiXjsbcU3FFy4qy/ZZQT4Z7tUDrMU6DiGM0krCL6PqY1AmstrSpIG8sXSqJjuZYZbzYZA2b7Yr/9X/5v/PFpy+YJwsa47i+fe0drYGjHQor0hnAC72x1FjRDzfJfnjIDDcEZ5Ge7owbbvJiqJBmWcx0ktIbBoaYv5F2vaHrHF1vqZqGtuuQSqB1SN/3bHZbVpuCsuxxQNMa6s55l14SkyUxQjjapqGseno7DBVlEEoQ6tDfmEzlnbXjCUkUIVC+pbAz9MbQ9Zai6ijymq7zbOnedpTlf79i9t96zSdzkrClLFtCHRPHKX3nSKIRdbsf2oIbrPMhK11fI/DnxrsdJPvtHqUERZWzK7YEI4VUMba39L1DhilaTpmkGS4bE4a+3ae3Btt6zplAwfAQNliMbbG2RwYCqX2Qo8VRdw2ZBh34h5Z13mEsBmG4EQahBpd267+jcz7pWSqHEQW9rajrkr43oDRdA33nkA76vqWpcwphB0fHlDROiXSEbXu0DqjqkrZtGE/HZOOUSERUfY8xvQ+1CkJ0mpHIgDiKaLuWoNVIpUhHMafzFGnAtB1aanQwJi/3NF2B1o6u7zC9DzTJ4hSlQ3ToN8IaARLyvGCSZOCgq/rhXtIQxYomN5RlRSAzMKAjxdOnT3n69ClpmnjWsuBxcfCHLYCCQPtj9YsiOzwHPlS0PzzU/H1fKYFUoLVCIJiMpkwnC06OLzg9ecKrV6+4vHxPXTdMJgu+f/09m8Ixm81ZLGb0fcV+V2JtwyiOScJjmi4nTjWlDqiqDucEKvDHhXCkiQ+qtLYmL9a0bUmgxwQSbN+TJiPGowwwtK2jb2OUDJjOp/SmoTOG7XZH1zdEUURnOrCOKPbOorwqyOstnWvY1TlWC2SsqfoKLTqEEATSEEXK514oSdN3nm8dQBokkHqn/rZbYTtBEMY4aQnCAKk8m7ZutrRtjxQJoZI43Q6BKy2lKQnkj28b3W53PHv2DKUUq2pNVV1xfHxCVdUcHx9zd3eHEN5V7oRvOawqzwDUWvH27Rs++eQTzs7O+PWvf0Nd1dR1y36/wQeoBxwdTfjo5cf87d/+vT+PXe8D/oKIy/dXHB+dkSQTymqLMYb7+zWj0ZivvvgJf/u3/8jz5y+5u3tgu80JtaCtKp4+veD29obj4xOapmU+X7JabfEB3T60aPWwBakxRtI2ht224NMXn/Hxk884nizYrzaMkwTl/PdZ7f4f/Pqbb3n1/Xdsq5zF8Zx3V+8HlrvvkBhPJ15s6RuenFywSMYcHS2QAtIs5e2333Jze8f17S1d03B6fEooA7a7HQ/rnJcff87R0RF6wDo5a8jLgqbMUVJwcrxE6oDC5BwdHdG2HeuHK6aTEUL1aCmwfcfxck5V1oTah/X0bU/bGfyC/LDWUDgnfTEYx2Q+I81Sri6vB/eVwRiI4pjFckxV75lOR6RZhAgMu1XNblvwl3/5Kbe393z36hXPnj3HdD0319f85V/+JavVPd9tV+T7YggO7jk7Ox8Cfn0mw6FlVUpFlqa8fvMd4/GINE0fF+tffPE5zjl2ux2j0eixcBdFIXd3Nzx9+gQhYJ/v+PiTl1xdXSGEIE1T1us1QaBRjSBLY4p8RxgEqNEIpRSb9Zb1est8PidNE169+obpeMrDwwPj8Yjb21vu7m45OjqiKEq/2Woq2rbnz372E7TWFEXOaDRmsViwXm9YLuf84z/9A1988Tm3t7ccHS0fNxvb7Z4kyQZGsWE8GTGZZZTljq++/IrNuuT46AnWhdzf++6Lui2H0Ed4+fIlm/WaLE2ZTiY/em4DfPfmmvliyWia8fWrK55OA++qjBS2Dbi/rIiiltE0oAwNQlQcHY/oVMuLj2a8en2HzC1CGp49PeF1e8NHL5+yK/e8vbzGWQgiSagDnFXMgpD9tmC7b5hMM5yFsiiIIk1RdKRJTG8tTgqSbAwC9vuSurGkaYRFcno65f3lDgHEx4KTkyn7fUlRVHz+yVO22w15XlHXNW3bkKYhVVWwXt8yW4wJ45CqblhMp7x7+84bE6RDWEHfGbJkRJrGSOG7G8bjiPX9PZ9/9pIbd0/TdKSjjPlySWcNX79+S1Pek8SK9X3DbDrC4mhrj5SqrUTECaui5ebtA3f3O/78p18RIqg6b7gppOThPmd5fMo2v0O5ipu791RNQ6xjxs9eIrRnfwZS+cKz9S7h+WxGWbZstxvW6y0Pqy1n50dkkzFPnpzx9TffcXf7wFd/+glZGjOeT7h4Oqdta5I0w1rDfHmMUmII8vJerNHI34O2u3xAIFkgJlLaZ+oIhRABcZySyz37IieQEmM7dvs9SZogpKBqKpq25dPPPkXrlPvVnukopWhaLlcPWCUJswSkRWpJHCu6viULxmy3a+J46ounASSjmKat+Id//Du0jjm7OOZkeczFX13w/vuvedjeMp6P2eUVwTzEJYJvbr7lJn/g+mbN84/PuClvmCznZEHC9+8u+fSTj2jKltV6hVIOHYcI4Tg+HiEE1I3PI0nSAGyDabf8zT/8e54/PyWepdR9ybff3DOdhMRxwps3bzk/n1I3NTQdYSRYzmd8t7rh6n1J3yrmiwwdKI5PEoSq2e72NFVNmozZrQvCUPPlJ09o6+pHz+3j+RFl3bG5XzNZHDGeTHHKUdYF69U9Zb5HDcF7XduRpCFSWfZ5jqTGOct4PEY4ixbQm458u8H0giiKUIHDis5zza0gSlJm0xm7zY6qyAnCgH3fEYYBaZYQRQlh6EhGGbfXN5TbPa539K1lOo3YlwVCC5I0Hrqoapq+o606oixkPBvRdzVS+IDBUAc0fU/VGHAhQTihrjvKakvadwgNQaQYTzKy7Ig4CdkVOaFOGGUTRscZ1++vePvmFYiIo/kRF0+fsLm/p2tKwtAxGmuc6uisoWgglHIQ8wVRPCKOItI4xhlLiCTREW1V0bUtMoqYzmeMpxOcw2PAmh6jYn756huiyRxhCubTOROtUdawWj2wK3xY5k++/AmruzXb/ZY4SYnClHxfM0nn/P7Xv+PqdkVtlOfrn58wu3jKfJLwn/7xV1RNB1J6g9N2zfLJkpPphPt3b9itNggnmY1GtHXF7e2GKEtQSUiSZvR1hTFeGNpsd5w9OcK4Bh1EdG1H3fZsypqicfTGu7XbtsQNex+r/d5SWUFbGdrGI3r6qsfQ0ThLawzTdOw7lWWBDhT7oiGbTjDOUeY1VVEjhWAyGVNuChCOpjVkKkQoj7+xRnDIsuv6Dtk7eucDluPQY3jWqx3z0YQgVcxHMxLX83Z1BwKSJMFUPdttDs6Rxonf3dsA13XUu9znQwUC2zu6tkej6JqWyEEsFFZrmqYmC0O/jneSturoe0hTENayWz/wze9azi9OCRTU1iD+BeaWw+ufu8MP7ud/9lsI1KCTDaQE8b8Vx38oVh9E1g8s9YPceHgNvzN0JvmfuMPHPb6n16AGV7rwwr9z/1xA9ybGR+qDGEgOFqzzx2XNIMKr4VgGEsQfysLezOqF9g/idN1UGNMSBOBaizE9fW8wgyM5CHweUNc1VFVNVZdDRomm65tB3A4xpnvcu4rBoOacR/357/qBBd4PRrED4sWHGUfUdT10EfqgTs88/4ASiaIQhuuolBq0QYGQ4lGUP7jc/f8vHo9HILDG4qxABPYPxsUPx8nhvAsMUkk605Lne46Pj7i/v3s0vZZl+RgUesDT/HDc/fOfh2H4IXwUORQbPO4lDMNHtvrhWB6581oTBHo4z4ew034Q2S1JkhCGIdZa1qsdl+99kLVWCYgeHVjEcO6EVI+hts5Jfy7wRQ8lPTpc68MxfEANWcyQy2mHuTGI/QcEpsV37PpKFcYZb7wT2pNWnEMJiZMSjFdrrf3vF8n+aBFdyMUg5Ja44aCNs1gHcgix8804gbfeO4kUEQ7nxWFb+ZMiPJNb2h5k76tUTiAf03d9JUr40heGks5VPgRO4lsYnOf0ukOFxp86DqFazhkvtguJcP4rKpngAzu9U8OJ9nHiukEQ98gZh6PHiQAnQwQ1iBohzDCQw0HoG4QjAUr61ofe+YmonHcQe6G9x+GrZcJ5JrkVBmkPTHXPa0c4hOwBi7B2KCJYpFMDf71Fum74XiHGOpq6GVxmgXdt9hZBQKB9SIrUBju8J8Jg6ACN1N7x78I70FukrP3NL/CuUIX0k9A5pFQ03cH977C2papWSKWRwSG8o0G5Dk2HlTusLf1mQe5RbkzfV1RGMJMaEYQYVyPCEGVidNsRW0saWrJU0naAipmEZ+jhBofruV3d8u7V3zN98meki6ek2RSrDXqS4AaOs3UdDou1LULaxxYkfyfwD5ZDK4yS2p8X59nz4IM0kiAkjgB34OIP18KFWOcDxXrjRXDwbSDCQt+f+oVo19INN/je9GgVkIRjAhVgbEfbNlRNR90MAYvUOOnQUg8tLL6lMM081/nQjSCcr0L3vSOvGvZ5Rdf1dBiMNTRd+8dO5f/iK4kSTOtQ9ExGYyaTBU3VeUZpv6HsJG1bs97eE0jvQqyFR310oX9o1G2JCgUyAqEEbdeRUiM1RAOPSglf+JJBjwo62t4HbNRNhVaKUTpBCk1TG3AdxjS+cFUaRPDhIbTerAhtzHgyQgc+YMQaS9s1tH1LbQrP6FWJF3WlZ4CGaYTUlm25oe1qBIJAx0gd0liLjiGNMnb7NQiHVMa3YR1CE42lq3uE8tVj6zyLz+EwrseIHqEsWiokAdJY7xTqfREujRNML1AyIgxiqqbCtB3jyRgVZAPbViCGLgprLbJtGY8y30LYVoSRJowzmrYhiiNfNKgbglASK4VtDaFQJElEs23oO4kiIIlHfPHFl8znsx+I575w8Rhsay19N2CLhHxcAB0e4oe08UO6ueerfRhHPrwrHFq+NGNgNu9ZHp3w9NkL3rx5y7u3b3nz+h3X1/foes8om5CmGUXZEkWhxy4lEU6EtL3Eip4osnQGqqYnjGLiMCFOQrJxAqJjvVlR1luklIQiRkqNsRbnOuq6GMJaGDo/4qE7wrDbD3zKVOOkIy93hCpiPplS7Gt2xYaq2/uCnrQ4qeixNH2DjgRYSxSBDmPqHkznaMqaQEnCICQZTZi4MbkpuXu4RwrFNAl8u2mgGY3HVOWGtqsIw4QkmRPqhKopyYt7pHIIDa3rfvTc7tqeh/sVf/7nf05Tt+ggIE1Sz/vuOl68eMHf/M3fE4SaFy+f4fDFqd1uz3a756/+6i948+YNbduDg3fv3jGfLyjKkp///Ge8ef0enODf//v/wNXlDX/xl3/B3d0tAokOQq6vb1kuz7m9vWOxHPPxx8/5zW9/TZFXBKri6OiEpun59tUr2q4gS0LW93csl3OEUBwfn/DwcEegPe8/CkcokYITVNWWjz/5iK+//oaTkwtsB3EQs5zO0ULx0fPnaCf41T/9A3lecPH0KQZJ3jSMbU+YJIxDSVWVaB0wno1IopQsG/FknLCYTTgezXEY3l++46OPPuH27pbf/Pa3VG3P0fKI69sVpjcY4OTkKVIp6rpmqbxQdshO0YFnMHdtTb5eMZ8uUMIyzmKyOADT8f79az777GP6tuZkuSDf5+xv9kiED7SUFhXoYX0gOISgO+fxJHoIHi/LhtFoglO+6ylJIsIwoG4taRbw5MkpVzdvOT87Zb36mvv71bAJgMlkzN1dQCoS/uN//A989tknTMYT0iRlfHJOc1Tz7t17hBCcnJySpil3d7cEQch2u+Xh4W5gI/bDAl6z229Ik4yyLAjD4HEjEUUR4/GIvu+HrhhHnu8YjVKkFCRJzLt3bzk5OeHk5IT7h2vAcX5+TlHkvHjxAiEkl5dXtG2NEP74pQQlNGmaYG3vudlRiMcFGs4vTjHWF6rarmK/b8mykS8kOYcxPXEcD5u4nrOzMy4v39P3PVk2ZrtrwEnquqOuG9qu4eRkwXq1oSo7yqLj7OQlV++uGc8mNHXObis4PVvycHfHyfGSsshRUg2hjj/+ZYTl85+ck2YJtRljbEXbGdJxwOl5TBbD/W3JeJQSSCh2G5qy5fgkYXYacX6esd0VHB1NsH1Llirevrvk6fMLLs4M2+2e+XxGVdZsdx2TSUaYBMy1X58UtV97NI3hxYslfdewy7deGNtXfPH5M9ruGqQjCC3jqUJISxBE3N/VCLbMZhOSVFJWjvV654N5VUIQdLRtwGI5o+tatrs9m82OPK/413/+M169ek8YevfU1eWaUaL9fsA5tJRefC+1D4FeTpH4wuvx8pj79ZrxzDBbzLi6/T3HyxSlQeY1QglsD30Pbd1jwoDe9bz9/hJT91jT8Pb+Ftu0dF2NUQ1VDVEwp92taKTh6v0bgsgSZjGJTpnOZ4iqYRIn1HWFFb6VGQFRFJNlE1arLVXZcHw8ZzIZIaT0YmYccHx+Rr5tWCzm6Ag2uy2jLCEvSrSWvqijNTrwgaKBlhR5TpalNK1vfQ+jyGeL6AgdxDjhsXV54cdMU7cQ+aD48WTsg9tkMISlZwRRgtQd96tLrjYNRoXIKCOvClxXc3p6BGKMo6NqfEdTnIbc3uw5P4uZLmLqfs/Tj54x3Y343W+/45tvVlhXUZUtR4uAtE8IVUCnevZlTpbFxJOQ2/yW2fM5N/k9LlZ09EwXE+43K+p2R9NtyUaS/b6AtmWUpUxn3oW42wNiznZXEAUMa7GIV9+94bMvntJuarTuaFrPWx6NMiaTCVdXDxwfnaC14PY6pywti3mKEL6DMklCbm9uuXg6ZTFPyfcd6/tLpIhZP2xpvhBsN/2PnttCak5Pl8hgxXiaYmnZbdZsdivauvQh09aBMKw3BUJP0aGm7WE2DomjEB0oqqolijWTKGW333nsXujXmjpUhFFM7wxRHBJEEVVjyMuOyPj9NCNF35ekKRwfzejqjpv1BuUU09mUoK0ZzRSbsmQynXJydkxnGtJx6DszI43pfDZOEAa0dYkegu+Lqmaz3SFFissSL6iKnrLaokIwTnB+PkF0Gimm9NYShT4XQSLIshHGweX1La9eP/Dk/IJ0mvJkdEFd7UnSkF35QBgrLA1V4/cQWknfiVfumSQJ0+mIRIXs8i03D9eICEbJnFhH0BpEBEoHbHZ7/uPVL5hGY8bpiFE4YiSh3q6p91tef/+aIIoZT+a8f3vN/f0d09mU9f0dUoZE4ZhX337L92/fsi1q5sfnGKN4//aKu80eG2T8yWcvkEpxc3fPk6dPOV3OmUYh929fU6z2bHc1+7xmk/acn8yBEEzI6moHqufoaIrSMfl+hwoteVESJylxnNBZGE3nlG9uuV/nZOM543HG3hjqqvHClzCYusPRs803zGceAdM2FicMKg7BeP1GSME232GNoWoaevak4xFuwJPWbYOzIMOA8/MTGttQ1DVFXRHqCOEkVVUQJyFVVdM0vkMylAHS9BR56TtjWst6tSFNEo7Oj0mDmFW+J5xqWmHJ4tRzw42mqQz0hqY1CNEgcMhIgRK+Ax2LkIqu6bwDXWuyJAVrSOOYzV0HfY9UDV1doRTYruXy3RVts/dh9c4jn/4lLzUwug9YCs9h5g+1bn8nQEiNOLjWB4HTu2wtQzbiD4xQH7pxD45zL9TiLc2PHzKItMPPDwGjHz72gL/wf5XOcyi8SvmHLvThKAfdzWdvIQ7m0Q8O+EdxlsNnDv764fOlGPaj7gPxomsb2rYiUn4f5LuwfTivlGBM7zt2h4JBHMUA9H1HEGq0VkglhkBS+QPTmNdqrPV5Oh5J8sE8KYR4FH+ttYRhSBiGtG2LlMEjX/1w7mHYG3f+OiZJ8gfX+vDeSvEouh/wO15El15rkx/GghTy0bH+KJ4fxHW8uOxzjjw6CuDly4/IsnvW6xVd57nude3/zRtexOPnA4/FgCRJEELQNi0HLvhBKD8I7Yc/f/izg7iu9B+yxw96Tds23oCbBNS1YbPJmUwyAm2JYo/Sk8nAWmcIQmegaghfPFJSo4Uezrc8DKAhjNSfJyvF0JH1WA5CuAMK+YOubI2nFyDdkCIV+ffAIZ31ZmsGafC/8/rfhXNxosW5AkeJYXCDIRHCEQgvwEGNhxFrf2AotEg9y4YKaAarvADn24yldTjR+AFkfcjm44Szaqis9X6jiKAzPZhm+KJ2wKIIHgMahlYUKwaEDD2OHlCPrRE4493HztA74YM9CRDOJ70iE+/wchXQY5zB2gZcDULhBi8zUvsBz8CzHvjA4oCAYQgG4XBhzBDKergVKR846esefrAohbQCMcCRDN5l72820gvyQoIM2e43SCHoGgsuQEu/2ZbKPKYOO3okEmN9K5lAI3U9CPiNr5Q5wGq002gp6ZWfpEJK300w3JibtqKuC8KoQnclVnS09YqUDVkQg/Xf27qO3g4dAxKc7OhlTycNZXeNUz0i0AhnCDoIMSQJ1L2mlBahUjJ3TNc2bDc5+7LlZr3iK9eRpRlPXzzhvr0mGUeoQPgqOwrEkEJsQRxaOQaOlxBucN/4M/khqEEMs8UCLUiBwE9ohiKRH69DGrBzOCIEIQzBub7DQmJd8CGQwCkfyigipJAY22BsgDWeYe6LUO0g7PsHhxuQRI+tLUNoiBhuKs4qmjalbseeGWV9YcC4H79QB9jvd4DE2t7LqdYMTntJFMWAd1AVRc5sMkEqX9UP49ALy31LECmctkgFYRrQmY62K/GGTIVze5rWh+OZck8QQ2cVCEXfQ9/1ZHFIEqckEfR9iek7XOcDy5qiIQw0kQso2pLetHSmRboYqTS+78NfIxcYWttC7xcOaZoQJiG1aem7kq5v/M0+jEiiMU5J6r5GBRLlOjodYbUhTLy4irBUZQlOoqX22QfO0g08/rZvsMJilaDvWpRQaCFpqo62qdFZhgwh0P58SacQvcP1hxYjiLQiSxNGOqDtC1brirYZwg3blqqs6K1BixFoSaRirHJUReFxOTpCaIPS2l+bKCIME6wJ0Crmyy/+lCdPnvqHtRRDYvjBUTDc540dqurexWqNv1f7Yp16rND3fe+FAGuG6rN4FNnDMELr4DFw1RpDHCeMxhNOz8759LPPuL954OOPP+Xvf/2fiCeCNJ0APUkUEioNQ9dG7wwGQxQl6CBG7HOkCxknY1SADzKt99RNSaB9toYQPghWipDtfk3XtczmE5T2vFVEQLHaI5UjL3NkIMiy1AcPVQUOQ9UF9HTIUBCHCVVbUpUtSTom0iGB0pTFniTQBJohzMWghV9YxXFKEkf0fYcVsN5tqbsGrGAyMkihfHFHQ2O84yhOUrQMGWdjoiikrff0pvb3gEPb4494zecLLi8v+bu/+weePLmgrivevH3LeDRiu+342c/+jMXyG6Io4urqkqPjJYv5lLfv3iKEx2gc3AzPnj+jrlqeP3/Ow0PC3/3dP/LF51/54DAr+PKrL3n39h1VVZOmEftdwXx+xMX5U25vHlgspgAeVeQcl5dXpMmUpu4p8oqPPn6B7WsirXj16pV3WM6ndH1H1/aUVUWalAgkgQ45PXlGWRnq2hLHGftix9/97d/z13/5b8mmM7qq5le/+Q3lfk/b9fzy17+mNg1NL9BRyPOPXzCZjdntdtzf33Px5InHgDQtQRIwn455cXzB1as3SCP593/zH3h7dc13390SZsHQ3eCdek+ePuflxx/x+eef8+zZUz8eBbRVQ10WmL7CmY66zkmTiK4pyeKUqtiisJT5Di0c01GKFo66yHlydsZqv2Wz3XknhvKBosYMGDvB4CYN8Zsl9YO5zeAKaTCmIwgl02nG0fEMRE/T1lgcP//5n/Jwv6LrevpJx2q1IkkizqZL7u+u+e67b5iOJ9zd3eOc4OTEhz5ut1uaxrfN9r1B64A0zRAyoXcNUiriOOTXv/kVFxdPWS7nfP/6O785kUMRcmgPffHiCfv9nqquOT07Ybtd0/cdm01NFAVMp2PyfIdWfrF/8eQp280WKX07Ks4yn8/57LNP+P7717RNQxx5QSaMQlbrB15+9Jyua1lv7n0YlLM8ffKM+/sHnFVUVcl8Pufzzz/j5OSI3/7uN6Rpyu9//zuOj4/JsgwhBNPpjMvLNxwdTTC9X8PNZ0tub+5IkjF9Z4jjjKpqGY3GSOVI0pDLqze8ePEMIQy73Zo0jQFHHP/44EGAz346IZv1rDfvmCxS3rzZ0xsoyh0ffZZR7SUn5yOur9Zc3zb8n//HP+Pm+p7NtsIlOdNZRqAd80lKbwRKaXQW8ub1ez755AWL2ZTVZsV+XyGVojMtq01JqCOOjsaUBR7rYnyg6/XVezpr+PijGcY4bq6vkcIwHcdD0ccyGoV8/13PfDpH4OhNg5SG8VijtCGMFBPlu4+qqqdre05Ozri8vEXSc3y0pG1buq5ESkEUJURRTZYp5tMM07Us5jM2qwZFxHQyZbNa0fYto/GIh9WOZ8+fUw2BZFoJhPLPwzAcOml7wfF8wWZdIKRin+9ZLCccLWYkWvJQbNiutoxHKXWXA4rr2z06yPjiy4/o7wXL4zmb2z2zzHeixQLKovTuuq7xzi+naGQ7BIEqTBQwGU/pe98lFccxcZLy/fc3jBdLRrMxp2cJ6/V7zCAGGgezxZK6rgnCiDiOEM4gpfLP5Cim7ZoBKxbgEYcBTviMpSwLWa/XJFnGdrMiSb0g35neZxwlKdY53rx5jyPg6m7L1bolXQb89LPPqNuK65sb1tt7JtMZYRQQuMAjNK3h9DymbnfEk5Qsjbi8+Y6Ls1O++umStnZMJxpBwWQ6ZV/mPDyUnF/M6V2DUJrGtOgo4urmDh2EJHHCdlswmy+YzyfUdY5xBfP5EkdL17bkxY40TWi7jigc07SWi9MlV5fvccZxfnzMu7cFfSc5OTlhPrc83G8Zj7OBI9uzWIx4+3rFixdPmE1ibB+ymC/J8xqpK7795hrnHA8PO46OY54/O+f1dxs2q5onF2fcXj+QpT8+OPh+vWF2dMxsPqFuStabBzrTksWaSEQ0dYtx+Bws7VFfIoiYHx3z/GwJWMqy8GaCpvfzSqTUZY8UFkvv8RVh2Sl3yAABAABJREFUymQyRyrNdrOnN4IozkjjmCSJSdOYuqyo8orFdEzdNZRVSxRGWOHIpiFhapkvA0bjEUo7eueYjEcY03kEgISqamm7GgRUdU2cDGi3KAEnsFTEaQhSEsX++TAejQlEwmb3QJ7vWW/XTBFIpSnzgjTJENIwX8TsdyWr7Q27QjOfTzl/eYaxLev3l3QDrtBYR6A8dmqXVxT7AvqOUZSxWC747W+uaJqGJ+cXjNOMvmipyoKjixOaquPy5obUJZwdXZBXFdLIQaAXvH37zq8dwoTvv3tNuc+Zz+dsNiuOFkc83G/5x7/7LWVR0ZiQVW55yK9IA8U0VZRlTXoR8fx4wfHJkvXZEfPZgnyzY/P+GqqWLEhox4IwSynzls4ZkkhTlTmhVhRVybv31xgbkWQhIlAUbU00GlF1PdnsCC002WRGWve0Te01ASVRWoMQmN7Qud6bO4KQMq+R0tF3Eosl0RLbOeqyJtGauqtpqxZjHL0AFUaAP9dFXmI6Q5pNmB8dkZc7bu49Ek5IgzOGPC+xpvdajhJopehMw7rYYzu/JynrjiwKKMqaYL1HByGRiHi4fSCNEuIgpmsd5a6lN9CWNWWVk41AqQbRCIIkQEhF1TakSQY4iqImSBOSIPSFySH75OHulgCBkwbjepJYYo1flyVphFOSnh9vbgFQ6oMQezAxcNDHDqL34b/RCA5Zgh+8216Y9fv/D4Ge1guH9hAWyaOQ/gFTMfzpPsjp7vEzDyYrOawlh08dcM4A7tGlKx4/1yGGAoDyOtpw5EI4lDrQ1Adnu/+QQTj3WGifd+i5Fsb0vjPYtTiTU0Z7xokv9BmrEIf8RdsPawHfqax1AFiM9YGhWRQOGiII4ej61hce3UFvGTQ35Tvg+94+XpODIHz474Mz/RBEeThPPzxfh7BNwHf7/MB1fji3XsA33jyig8EYLAfdUIAc9nzOfhCLxQHVcxgPYkCpGNq25fj4+LFz79tvX3F2djaYTUrWa49BDMPo8XP+QPQfRPCDq77v+scO9MP38caYD79/+B5CiMc1bNd2NEoSxwliCDuOohAp/XnveosxIYGOUSp6XHuMRpowFGilcVYM9xqviio0SmoCqdEqRGrtpbqBCsJBa1UtdjhmhxvOn0AG0pucrMP1g9jivOlbOIszvUfADN/HGQN2oKr/EdvuP1pEr80dSmYIF2OcpXO+hVFK9TgZvPzWI0U9XOQMIbzrSBMPzG4vZlrXYWw/ONDt0DbgnezGdFhqfHtIj3UBSsS4oe1Ec3BnHwTxg4j4oaJjD2Izw3s572jtrQU3cIuG3zHDjcMOojM4hGkRzmFsx2Hq2yEe+cAm8iEiA4x+YBf1pvducXxghEeGeGe8EAciN/TOh0QecplxAonCN2Wb4ZwObSCHc6OHqtEQrKpViLGSqqmwRqBkjA48+N/JgV9sBEoJBJ3/NOmAxjug/SxBSIWwEaJNQcgP1VHh3dlSiscog7opGY/PiKIxTZMDGuUkTb5jOX/CTLzglr8HBFoGODqMqNECjKhwWlCxg9YideC5wloiTUuiJX0aoaSjrSW9U4SzI5LFBqcKnBxuRkqSzkJkDtFY4lSHoR9Y9w2Gchi1EueGIFksSDkwzMTjjRzkY0CboxmKLW4YOwxdERp8GcSL2cIO1VIvrvrHihvG+Ye2KF9a8Z0STjik7dAYvPNXD3Ux9dgaJcTQ+sOh68G7QKzthlAOgXARaSJxLhyKPICVIP5lG/HV9oEoiqm7hvv1HWVTo3VE0/kk+yQe+6ptW6O1Zj5bUBWG4+UpyrZsiw3GdRTNHiMcZVthOotGEocSR0FdVyiRedGsKWmNQ8Vj4iTxjP/OY3e6piMMQnSSstvuCFVIEsdUriQIJSqBkU6Isxg7BGtESUJXdyAcnWmoOt862FeWMEhJswytNbvthrreEYcKYTVJOGYymXu3TGdA9nQ7S6QCbOAflkmaIKyk2FVIHZJkKWESUJcVSnuWlw59lbSnoS1LVKcJVEBTVLiuwcUxbWfpW0OsM2IdE8gIOdYUdUUQRyBaoCOJAwIbs1o5j3DoDXfXd5iuJ0vHlL0iEBGz+ZRQhtihXSkKI8IwxPWCqrNoFREGGbYNWM7P+eqrn5KlI49FOLRkPRYf4cBu8+Ibg8Ppg1vi8PCsqoqmaWiaGmN9m1sSp0RRTBBEhGH0GNbh8VnCtz1Kz/2TShINm7azZyfcbd+jY8M+DAFDW5Vs9nvKsiAbe1fwdDbxnLnLS5rSYoznpXVlw3a/JkgC0nRClqWYtqGsaqIoZLvdsy23tKLG0pPp6QcUk/Abz0B7nJcOA5JRRJlXVG1I23Yk2RihYXdd4lrHfDljFI7ou466qMjmE5wzNHVNqBIiHRG4gCwdYbE0Xc16v2FfFGitfZHVCuIopigqmn7leZBWeiEy7JDKQusDt7q2IY0zzB8RcPJfexVFgZSSp0+fUNc1ZVXQ9y37fEdR5NzeXbNcTlmtHvjqT76gaRuSdMRyuQAsf//3f09ZloOzYsTNzYpPP/UF3zCI+PzzL/if/+f/J9PJHOcsm+2a+XxGGGn2+z2TiecaSylZrzbc318TxzFl2bJYHLNZ5/67q5BRNuHy/QNNXfCwWvGnf/qnXF9fMRqNieMxvVmjdYzWmul0TlGWGGNJ0jGvX7/jydEpv/+nX/PR2RNMUfJwc8tvfvlLPn7xkv/3//L/ITu/8M8cKVicHFP1LRQ5IpAsTha+U2Ax4+7+jl2+QdKhL57w8ScvafOWf/inX3LzsGZ+NOby7p77hz3SCf7Nv/1Lfv6v/5yf/+xfc3Z26oUD5d13dZWzzzd0TeUL931FW+0IA5+x0FQNbV3w0bMnSHVGIAUfv3jBf/h3/5nxbMGz8wvyXY6VYJR3s5hhoYhQSOWDs5FuEBQj5vMZbdsNGwSBpadtS0aTiOOTOfv9hrOzU96/vyKJEuIwYjFbECjF9eUlT5+e8/d/+wvOz0/Y7/esVyvqqiPLxlxdXfPkyRMmkwlFURBFIUkSsdluaZuG8XRElmXkeUFZlVxcnGOtIc/3jEYJMGOz2eCc46OPXqJCTV0XvPruG6bTKd+92vDJp5/6RXfbAhprDS8/ekHb+gLEfrtGAGW+I98XHC3mXF1d87f/+T9zcXHB9eV7RqMRXdcRhhPOz88IAknXVYShDxmPooTdfkPTVszmS4q8QgjIsnRoAVYcHS3ZbjdMp1Os7dls1nR9wnS6oO8sOMUom9C1hq7zjqYkSfno5Zf83S9+hbGOOPM5ITjDOEu4xfLrX/+KF89eIpVkMvqX4VySaQ/hnl5s0ZEPTq+KhjBscZkgnY64W+158ckxm82a0TTkfmX467/4V/ztb/8D1jm0CojDlO2uZLvpePHROfut4fWrK548u2A2nbNatXR9z3w5wrHzAojrOD6ZgTMEgcC6muk04fziHCE0tzdb2qIiVI6mKP28r2ocPUpFPNxXHJ3E3N1tmS8ChBT0fU3XlSRJirUhH3+85P5+RRyPcTbAOcVifsJmk7PblcRxgLUVT54uqPcP7PMtUlrm8yVB0PLlF5+w31WMJhOubm44PT1mvd5y+f6a6XxMVZaMk5iHmz2z2ZhIpXSV4WRxSle2TNOE1a7EmR6jBWWRkx6PcS2cf/yM27t7EL4zzAaOvNny/fu3SC25uclRTiC0Ik4Sytt7YqlJwpDxeISxFtP7wnSSpIBlNksGUcWwmE/pjDekzE6OsW3gLUHOECcxaZKx3+3pupaybOj6HmcdSTJmtbqh7xpG4xHW9pRlzWIxR2qFlgFC+m6xKE4IQp8N1dUlQlh2+z3KhSRpyus375jNl0RxglSaq+tb0Iq82VHuQsrrN8wXY548m/OrX24RElabLSenE8pq701WkSSIvAEqjmKWRydYLEEgmU8zbOeYjGM6VzKeBrx7nzOddCyXYyZJxnrzQG8UysZEMuHyzYqPPrmgKTvy3Z6mLYhCzXq1YTKe4VJ4WN0zGk2QSvL+TYMzPYHWxGFClkxpKsOf/fQLvn/1NU8/mhLHMdPplNub+yFgTRPHXrAPtKDIt0jh+Obr92glmC8iPn75kt5UGLfB2obN5pY0iRk9O2K/LYhORlTVj3erOim4vrvGWW/GCLX0awytcUFMLkqKugbpGI8j34ouBPPZkjiJ2awfKMscIb3gUtelx/ocjeg7n3PU9UMBJ4yoG8Mur3BSMZlOuTg75WgxQyvJbrtlvXpgu9r5Ndk88wHcriIKAoJIkU1isnFCXuQ+c0cnBEFIke8BS28c9pCjJn3eWDaakIwXRLEmCDzitakbL74QkYQTyp2j3pcU+z1NUdGmNePZDKd8x6FWgkg79DQeuOk5+dZwNM/QoUCY3otjKkDYEGv8GqwULb0xbFYrClUwny6QQcTLjz7h6HRB1zXUTUHbdHSd7+jb3e55dvScqm0JnOLN+2uqLGB3/ZoDIqEsavZ5zmQ2onMNkZZc3r5GWMWf/OQJy8UJrUj5d7/4NW/f3PHzr36C7CvuHm7praTcl5BEVHd3dA9bxuMJ2kFe1SRxwGyWkM4yotB3Gwjn0Di2mw3d7Y71vqEoSno3giCmrHtU2RDokJiQ1S4nihLm8zl5XtG21uchaYGQdkBjOgKtUAHYtse4Hoei7Vu63tD2DW1ZU2lFGGqEVEjrsAaqskZHIX078JKFJE5ieuNVjziIiZLYu5ojzVxoMB15tfeRcIEf58Z1dL0XTtNpQhqnZElGXTVUZQOVw3SWig4dSJQKafoKKQPyssZ7SwV9a6G31FXN8vgEQ0+oIsqmY7/bcv4soS0rNg8PBEpwvDyirRuqfE8QRAjpUDpAawiCCCkVla1w/0LzmpSDkXTYX3nR+xDS+QPxWshBN/ghPsY9OtF9t/APUZsHp/MHQf6Dvf0PkS6PArr7YFJ/FGr/GTJDHhzMg5X84HQ/vOcHId670aX6gAQ5vB6d8o/v70V2j3jypgSP2fWdzlL2OFNTlnv6viaOQsIwom3xnedtN3QQWoIgeLzPWeMDQbu6xAcu++fdAQXjaQSHAgO0bYtAY4x7ZIMfWNoHnMl+vx/QJX6t4vqDm/yDGO/fUz6aKR3eCa2CYLi27rFgYqxFOW+G/OAQ99dbS4EVdrjGDOPAn+XDOPD/T0+SxN5tLwWT6Zj+u47f//73ZNkIrTV937NcLtjt8kdUkJSSMAwfx8pBKNdaY6zB9B+wQodCwmH/f8DbPI4L5U1WSikC7c/bIfzT2h4hvSEzjgOcM4RhSBR5rSCJY8JQEwQgpfMagZAIYbAdgxVZoUSAloG/bsN5cnbotJA+c9JaO6ivg7H60G0vhaeGKDWYYz3+1w3Bvs50j0Y1Z81jSq9wH8btf+31x+NchEMI394gLATCC4DgBkHRfzGFxjmfWmtEhXOlb8sQDkk0YDDACY2xNdgOrRSSEON6FNC7FmOLRx6YsL4NB6eQrvMnyxkc1h8TPQLPa/eJtXKoaltw3XBRNHZgYjuMdxMLhZIxHtIkB/zKoUrjW/UFCue8OCdViBQBWsYY22JsQ2cccnCQi0E4hmgQzT0v3rgO61ovzMoY4WKka/Ahona4aw1OZOt57Y9Bp2gsPuHW0ngBdxCDBaBVSE+LdY4oTJC6B3qQHUprJDHCtX4g0Xuuu9AY1+Jc74VbZ8AJhEkHcc3f1AVyCGcVSKkp65IozAjDmLLe01Ybv4BzlnkA0b7ldPIl6/5bau6RzqBEhGe6CzpnaERD6SqkVCAHREagkJ0hsQ4THR4SDuME4SRjerxApSnp9Aw5jLl0HCJdgxyX9LLEyhEGSW8r/MjQOKcRYmBAuUMFdyhbDOEYvo1D4mjxIJtgKIJ8eFgJcegUOLwkYkDkOIcvQriDKG+GOSGHB6wZcDDiB0+qfihxeWTJQcw/PHw+iPgOJzzWQzo7XI/+8RicE74mYiXCfWgb+jGvvNrRuw6hJHVX0O475vMjOiMxWJJk5AtKQcB4NGE8WrC6LZhOFsi+oKh37JuWos2xUlC3NYGMiMMlWZRg3IrN7p5AwWS0INApdVsTjTRxnBAFIabtqYuSrisQmeepGeNQoWYUj0mIEM56V9/qkqouObk4R+qQvu09q134roLW1PSmI9COIEy8GKwlYRISxDM0irbw3Rtt3RPEBiU78nKF6lMiFeEiMMpirF8USkKmoylZnNLqhjiMCIOAsiqJVOyLG2JwUghNomP0bEFT5X5hoyUGSRhGBCpAiwAnJKmWCC3ANbTNntb4KjX4B1G+ydk+PDBOR4znY6yRVJuaNMyI4pgnJ08oXYUCYhVhWpDCUJaOQEu6TvHTn/6cly8+JQw1Sg+hIo+LK//64cP18KcdHt5d19F13aML3TvRS6zrUIM7NI79hklK9ThnENYHLDuBCiQOReQi76pwigueIiNH0W7BCeIopKlLLu9fEYqY5dExYRgynYypqxLXdlRFzWSSsJhPqZodZZ0ThwlpPCPQIUJaYgk6CIhtQru9o9wVqEAwWk4InOfn9q4jiCS99S3cne3obI0M/EM4imOCKKHuWhwRWQjzeEZgNcI5AqExfU/nGpQQJHFCGo+wsaDteu8uz0aY/ZY0mjJJU0Kt6dsOoR298fgpoQLCICUKA4Ts2O5uaKsOJTRnx89pZc+u3P3ouW2tYTqb0puW27trnj17hnewGMbjY+4fbmi7mjQL2Od7vvjic375q38cFoviceGUpiOqqmQ6HdE0Dff3K/7qr/4Nf/u3f0uaJFxcnNE0JV3XEASeIziZTJhMJvzqV7/iycUTkjDAOsHt3S3WCBazM4SQ3N7c8fLlJ5jecXl1zcvnpwTBc5yD8/OnXF/f8sUXL1mvduAUb9685+y0xwCnZ0+Io4xvf/M7dG/5sz/9Cbvtmnq7pcpzvv72a54/e8b/4d/+G375+g2b3Z448QHg79694+UnL8iSBGUV69WKt2/f8sUXn7Pa3lIoy7t3b/nzr35GlxiWJ8f8p1/+mk4ojHMEccj5yTmdcRR1jVCHdURPWWwwbUddlTR1jukamqZCuJ5pFrG6v6EpKibTOfvNmqrIuXhyTFHkTLKM/9Nf/zX3qy2//vZrQiXJq8o7eCSoQIGUmL7HWr8Z612PUtA0FVKB0lBVFdPJiMk4RSrB2fkxu/124HQ7nj17yma9oyxL2rbj9PSMJInY7bYoPbRk2o7ReEQSO8qyAJJhwd1ycXHO1fV79nufh/H8xRNev3nN8uiIvjcsFnO+/vprPv/8S7SWTKcTTk5O+MUvfsHR0dKL06Jjs7ljv98QhorJZMr792+5vr5muTziiy+/pK5rvvvuFWGo6bqGPPdCzmp1j7WOh4c1YaS5urxGa/kY8nRwEt3eXpMkEVVVeIdt1xIEIW/evCGKUibjBfP5nLv7W27vbnh4uGc+nz/eh2ezyeNmSgeS9XrFxx+/5O2b98znC9quQgjJ0dECax1VVWGdX7TXVclyOSPUfpMVBiHFLsdZy/x4zutXb3703AZ49nJBVe9AVXQdXDwdoZXDuZama8kygZOO3397yyefTQlSgwthXz8AjjyvmKQJwgmKvEIKx/evLxFGs9nlbHZf8/zFMScnU16/uaNte+I4QNIzGqfk+w0I44PRXMDp6TFtJbm5ugYUk3TKm9d3fps96QlCn5uRxD6k25qeOArpupokHrPfbnmanYLzLtquM4RBwj/942/o2p7F/AhcyLffXJGkmqZpGI9Sdtsc6TqsrdlsLScnxzw87Hi4f81Pf/IZPT2TxYTLm1tM73BFRRRFWNsTqYBxNqXMG0ZJxn6fk0YJD5uKv/jzn3P3sOF+/cDd+oaLiyNW23sa00Kn2OQtf/LFz/lPf/MLlkcZvS3ZbnNmk2NurlYoWp4dOzbbLWfLI6ZxQhxougH5F4Ypm82G8dg7lieTKXmes1qticKIaTrlbrPl+PSch7sNddswX5zTt5LdakffG6I4oSgqxuOxD+u2gjjJsFFIEIY0jSNJR9w/rFkuFwSxJghinFTUVUtRVkzGY9IsQ2Bouw4xdJKlWcrDesXJ6Tnz6ZijE8G71YazJzPuq5x9lbN9u2U5S3nxYkzTC9aXLc9fJkSJpqoKtvsd4/GIYl8jRMJiNiXPV5jeUeU7To9PyHTAurwFB598NCIKNNI4AjSjKKVtAy6v3/Piozl9LVnf5R77g8K0lqLpaZqWLFkQBgmBbHi4Kzk+XjDKFGkS4EzNl188Zbtu+fbrK9Ikwoqe71695uTkiPn0iI8/esH7y2uEEIwnE9K4Z72+IQw1Ly9e0tX3vHm9ZTpJ2BU7spFCSUkUR9RVz3wxpq4cyekEIVrKav/jJ7cSrLdrtIRRGmO7nqZqmSxnpNMxOt4jdluKqiDQEqE1fdcSak1VVtzd3dN1LWEUPIoxURQzGo1xDrb7DSrQ1E1PbwS7fUXfC3TgcT7LoyWTcUa539N3HaYzvH31wEefZLx48ZSqKyjrHWGq0WFEivT7d7pBTDG0TU1dlXRdS5RlJGlMtd8TJSEOkFIhdUiYhMRx7/F7ylDXO2yvGR9PqXJfjMJYZpMpSRQ/CnCBDlECIh37Dt/WMdJzlJD0u5JknJCKZHAiauraYoyjpEEimI4nbLY5WMO+qPjoo0/JRilFtaepStLRhDQV5PscYy3WOIQTrO5XXL254mQ2ptk+EJqOZrtBGKirHikC1nuP/JmEIYtxShrHKKFwpkbieHExY3e/YjHOUF1Ek1fc7Bs6a1k3d6jO0vQFrurJyxwV+lwzHXT07R5jNUXtEaCTNCGdhLwYXZA+lLx7d08QRR5B6BSGkK6Bh+/ek9clu/0WoRRC4POMpHeAIryRQ4cSKyVKOxSOtmtxA7q2rQ1lXRFFIb1xGCeJVUDf1TRNS4RgMZrg8MGcDK7Q3WZHmW+9WS2wxGlCYyxpkuFMT1lXNH3rBfEgREeGuvZu3el8hkRxtDxBOMl3336P6hscGhkldMa7T6uuQWtHNI4ZjRLadk/VVmRRQBLGjMOUSBginVLTUBYNWTJiv9ni+p7tesP8o4+YzRfs9xussIShpm9aVJCwWJ5RVQ190+Hsv0xE93PysA87OI3VYwDk4U9rfbHA/0w8dgV7t7nfJx/czCC82C0O7nP/GV6g94LpoQvZuYNz/L90bO5wgP7oBmH9gCT54K72HciPAZyDsC+QSKcef+fwsgyd+gOCw89ji7MdWnhToXOOUGmc9MJwoBRRILG2RpANHZkNpvdFggN6zzmDMf2wB5XkeUGsPf6j63q6zgy5id60JIf3F8Lz1KMwIE2TR4H4h9/TGO8cP3DE26bjh4UG36XdIaUmSVKKwThljMEJvNGv++D4Nsab0rJ0RF3XiKEbQQjhw0WF190OrHt14Hk/7tkHAVlItFLMZjOyLGW3C/mzn/mAUdP7sRtFIWVZcXR0xHa7fVzLHkTzJEkev2vf95gBExYEwbDXrz8QFoaxcHCjh2EwnHNPwUgSvz+om5q2bf31BBA9k9mUo6MRebFFqRPi2DvjgzAg0F5nxjnflYI/70p6R/tBpPdhq/4sWA5jzg0ZYuIHY5thvgTgb1uDWdVff2t8t8OhK8Bjaz58P/9d//8oojs6OjO4mREIESCc9OKwClEyQorhJFg5yJC+ncIzx3N6Knp8dVAIBcp5q74w9LahtYJQAsKn01o6nDBIYnrbeG+v6xBSD60YBkSLsP3gBfaTRYqQQOjB7atRKAQNgmo4fjUIqAcukOHAURdCI4X/Nyt9MwH24MNWKBGihR7c9HhxVsqBueOxHIoAJcMhbVmA6331mx6lxmgXIhE+DFUqz4ITBmd9675PanYIFyNQWHf47g6kDzoU+BtQoEMaITHOIpTnPlkMQrUEkRt4WyCEwtoGIQKUzMAGIGsEYGzjvdm6H1zX6lEIw3nXmxCG7e6W89OfUNZr8vKW9cMr+s4RRjG7pqIdKxbjp8zkJ9zbkt7uccJ6fpjZU6h35M13NH1DIOcQWLQQyD7CxQLVQWQtRPgWKiHoXMjInRJ2Gjd7QeUiYiFZzp5zJX6JDS9x6mNq29KZFCcarOtRLvbBt6J7fKD4axjAUKuCHkE8PEQGhv9QBfRzZyigwFCEsUMB1WNgPO/ePrbYeF6/GVzphwCHIThDHMam9EWFIfzDDhx9j3TxYac+lGMYc853BHBgmqGGCSm8wO7s4Hl/+GOn8n/x1dBBJ5iOpoQ6oCwG7JJw2A7SdETfl57Xpb1IXnclbV8hTIfUGlGD7TqcdKRxRBLNOV48IQ4iqkb6BGznRWQhIxrrEGrg9wcBgZSUux0I/IZEGBpn6PKth0M5hW1aXNcSSsVDtWHcFERY8qIidAGTeITEUts9eWMIY02apYQ6xfPPFUpLTOcQwiKlI682jIKIMI6RRUAY+o4MImhFTVGVWCDNJmTzKZiWsvRjWwcBu/0OJTRh7FngpuupjSFKNUkSg/WhsGGQEEcRodAIK5FKEOqAqqyp2wJsiQoCeuMoih1CCYIwYDpbkiUj6L2DOksmSBkhXOTDU6SldiVaSOIwIslS4iTB3BY0Xcvz85f85V/+1dBaJlByeBALP46AIQ3eWxEObgO/8PELh6bx7XLGGKqqpGnaR46xd5+HPxDfhwWhsD9Ynw2LMSlQgSIwgjSLsWqGVYaiHlHXBc5ZlNyTJiuiqEEQEAaxn3/GoaQaWs8EYRAjJDy7yAiigH2es622BGlH07ds8j15XiC0795RWlA2W0bRCIW/Z3dty8n5CVJL7lY3Q8C0o+8No8kcrQKQfqGgQh+W2g7noek6TNGB6NA6oC93NE2L1jEOxXKxZLVbE+qQi6MT4iGM9vLhHZvtCisFk9kSLWNGYUaSBHR9Qdc2BFGIiALiLKXc3iN+EAjzv/eltOD09AhrO7quZrtb4fCtukfHF6zXK+JYcXJyznZbcHt7SxiGzOczuq7h4eGeNE3J85yj5TFKBaxWK05OTvjNb35DUeQEQYTS8M0333K0PGazfWA+n/P06QWLxYLFYsnR0ZK3777n4eEa6wynJ0+wrufh/p6f/fxfcX9/zWp9y9n5GVVd0bYdYZRQ1y1N23F1fUtR1dzd3+OcY7PbEMYxZZmzelix2Tzw5GjJZv3A0WTK1d2K6/dX7MqCynR8/eZ77tZb6gbmy4zZfEbVlrx994bFYoa1jsV8zm634/e//x1luWfx6Sd0puf7t2+QvaSqa98KPF/S3O+QwlK2DXebFa/efM/zl885OV3SNrkvwjvHbncPWG5v39P1LZ998pLrd2/45nff8dd//T+S77ckScxXX37Jdn/PeDzGGsfF2QmnZw3vb2+Iw4DV+gEZBKAUzho6Y4mTZOh42JNmKVEcYF2LUrDbFRhjiZKQiydnnJ4tyIsHbu/fY0zrcSPZlPlsxv39iovzC+5ub1guFzx5esZkPOL29oqzsxO2mz3WSkbZiJubG6SEzWbtcV2hxrqO2XRG19Xc3d0wmU6YD+eyLAsmk4ymrUjilDiOePbsGbPZjKurK0azEKUF40nCdObb/4uiZDzOmM+n1HXF7e0tT58+pchzBJblcsHvfvc74jghTVJwlq7tOD8/palrX6wQvjNws/XMaKnGKK2pNhuybEwcp4Rhwmaz49NPUvK8HO5vBev1isvLS54/f8aTJxes1158TJKYOA65uDjn7OwM28Pnn3/Gf/7F3/Dxxx/x9u33nJyc8f79e/L9njRNPVavt4RBxMnxKfvNjvOTc8Iwpqkbzs7OfvTcBkhCx3ZbE8YRbd/zr//VT7i9eku+zykKR5oJnj2b0tcrurKkqQqm04B3V98RaEdVVkzHGTpR3G9zTk7GxPGMV7+9RDhHXhh0pHGi4eQ0pso3xErS144mr4kCH/LZVAFnR0d8+7u31G3H8mhEkZcgFR998pQkk9w/3DCaxGw2DVmakW+2HB9Lzp4s2ex37PMO4xR1Z5CiZzRNaeuW+9UDWirSFDpT8k//+FsgoMg70iwg1CPeXN1xfDxFxXtU5IPp93lH321pbU1e7zl/8oTRZMTq4Y62bbm5veVoueTp0yc8bG696GgN2WLEptzxxc++5Hev3/Lk2ZKJjFDxjFEccvUuxwqJiCR1UfPrX/yKRGpMLTmenfKwume/u2O58B2DD9s7urzm+IsF97sVs1HKZDwn0wk3l3csl0cg7MB29cGgi/kRbddjEBwtj/j//q//jvl8wvn5iS/SuogoHROlE0ajjCROqSpvImmtI4onLJcL8nxHFBvGQyFaaYU1gjAaU1UVURJibEdVVIxHPsz4/OSc9X7H+9tbcmO4KXLK9T23TcO3312z3ecYAaNRRm1qWiuwUlGajulI8cXnM7q25NmzM/Z5SBpH9J1BxIqj2YSby0u+/MnH5MWOoir55bffMJlmCFcSao2xjhDJaJJRVg8Y23F08oTR5BOKouL8IuF+tWa9yvj+uwdOzyYUlaHr96zXt1ycv4Q+IUsiyo1lMtJI7dc5QkHvCibLDic6xklEHGUkQYw2hv39PYtxhgkCSmvZrO5oy4ZPTz/i3fsdcRrzsz+f4WiIOkOgOupG0RSQJjPW9znHx0t+9ctv+OlXT0mCf8HklpIwCkmigKaqWN+umY/nLCenFLajs55pr4KW3tYkkaboGup8Ry18d7ZSmrbpEQKm0ymLxTG7/Y4ojmm7nrbrkTKk6xxN4xDD3tW4nrzYUxV7yv2e68tLmqpiu2q4ijb8yZ9/hOtqeiUwdFRVQNN0VCb3gmxniUON6TsO6z+GHvnOePNcWZaUjUGoiF0REoYSpTpM32C6mrbpSGNFHGSowAveViuMkFjj+epRFBFq39NeFy111TBJxr79XwoynVDFS6xTRPGU91d3yFhRNzVN2/pQzaYnm884PjlnPltw/3BHW1vG2Yx0pCn3NZdvrqFXzGcLtNTc39xxfv6Uo+mIdnuFrAwuDOlrL4yVLbx6855PP3tJNjkh0I7tdsViOiHLErqm5uI45d0oZJrE1E3OJJmwqXPatuFoNqNpO8qmpe476qbl/PwcpwwhLWGk2VY5RV1jnC8OX5yccPX+kiQN6fqeWAjS0RgdJDhC7tY599uSKIkYTaa0bf0Y/med9ex60Qwd83bIO/O5b6ZvEaEiy1KqukE4QdsalJN+G9F1dHWLk5K+szRNR5aO2HcO0/ug+XK/w/UNk0lKW9QYA62x9DKgbbw2lMQpSmvyosL2Xmi31mfcFUXFSm746NnHPD17yq24Y5xqsuMF6/UaIUBHIWVZorVmshjTm4gm30PrC6SzdMx2W9AWDVk6YbHw4u2762tQDiHh4eGByWzGeDfBiQ4rIR6NcFaTl5YiNyRJSKz/+yLbf+vlXbHCF3Ud3sz4B27zoTMYN5j73PCzg0tdD3s2jXVmELvl4K4enLWDuO01B4/y9OL5h32gGzAxB6vgo0xrvbImBtwFHl7wmJNlrS+ee3b0QVBWj4SKH4rnh2P58JehY18MeJdHSoB81Fm0lh55IyS9aXw3ZTr1yLnBJGGdQYgDakUTx3LgjhvCMKCrKnxhwac2+tuq8GgzpQd5TxBHPjj5gG45OK990UHStu3jZ3okoac2+Jww+TiPoshn5x0c4EJ6NKrHy8hHcR7wJqy+IwhD+q7zXQ9K4YzX3cAg5YED/gHqzA+v0lA8iOOI2WzG9fU1VVVxfnZG07aUhXeOHwJSD8fwQzY6fBCO+8Mx6fAR4RKG4dB93jwK8MAj6qXvPxRS2taHt0ZRBFiUcihlUVqTZRFffvURRfmA1J6RHkYHBrof6368eQqGF98dFoNx3Yfx6geX12/FUBw6mKsPDJZBh9BDd4CxHnHkrDf3PfppD10feKzQUDLyxlj3w7n4X3790SK6dR61oIZqiTEevSKIvFsUh1SK3rZgm0E4V579LaKhGuGQMn4UEIXwFn0pJEo6pAt9JUE4cBG4zvuxRYgSAQ7obD+EhfrJ6tCD8OJZ6sY6QCMIvbvcOnrbIzjY9X2rgJTau2Ccb0UQVIAasCAtCIcZvoMgGBxgAzLE+PYQxxghEpwbvi8+IfYwADybB5yQnv/jgsHR36OQAxfWfz/rWu9YFwLpJMIaoAKnPoiw0gv1QigCBa3oCAOPTpDKh6oqOUy0sEdoH7DpqH210kl624PLwXkXvnMdAh8QehCCBWK4kfjKsrGWus6RwreK163DCkE2PaVvO1yX8/r931GOJ5BMODr7C1b97+idw8kOXISSgsp8DTJAiBGGFqE7etEh+wkW734PnQ/wiJUB6QMksDHKTdg7xf1mzblISUJFkkraoKQXO/bmO3ASKR2NucWjWezwELBDccI/bED7cFMBhuJDqwwK4YZ2nIHLZWGYsR794w6LwQOiBzVcIy/KHwT0g9jta4U9gtAH1wo3tMUZ3MDq92tL5Ts6RDC87/CQFF6Ef7zRMbRfiaHFRfpCh/sXBA8CRKMRovNYinQUc75YEkhFFMa0QFuXKG2JUk1lSu43a5QU9GqPlAGokDQcEUhL1RT0QjNK5ySjkEBCVWlmo3N627MvcqyWtHQcxSOwHcV+g3Q9iIYkyyi7mlpBowHTcVc9EBrQnUMah1OCIIrY5CvSPmEUzRmpCboHaXuECQjDBB0KkjQjVgtfPnM1VbemqnqaskWlPcSQ25qsnzCKnxKHCV3fYWTPfr+laQp6G9I4wbrfYfp7ejUEw9qOvNwznU+QkSQ0IeMo9WnPoqfqWoq+IU3GSDHCtuACgdAWI0ussHQq52Z7jQx6RvEJyIRea+IYpkwQZowUmt32lny/YRQfsVw+p+l7NtsHuuKBTbNCOU07uWA+OifWY8I4I3MZ/8Nf/w+cnh79wBHwgXvnhvY9Y40vxEnxg6Kgbwdsu5q2awa3qm/9DcOA0XhGmmSP+JZDQdKYYeE2fMiH9jV/vxNOIDXoWJCImKVckrUjOtPTtA1huEUISdMUvo1SQFV0tK1kPDmhMwHGWNar3IdBjqc42yGdA9tTty1lWfCwWoNzKO3TKUzX0ihBHAfE8YQY3x5WVyVlu6OzDQ6Js84H1jgJtkfYBuFKnNLIxHcGGWNpRUdRNkgFTb9DCMFssiSTlsX0iKLc8/7778iyFG2889s5gYokeb5Hi5hxPGEUzqDz/P5QjbBasd5vMFj2tqDJc5z5r9hF/ohX1zUkiS+gLI9mFMWWLEuQKqTtCtIsoCgL6rbAOMNutwMHdV2zWMwoy4K2bR/5frvdjjD0POfr6ytOTk6Yzaa8fv2dF1d3D0RhRFHuyPM9X331Jd9++wpjGlbrO5q2ZLFYoLXi/v6OZ8+fYq0vLDZNzaeffcY//sN/pmlbRuMZ2+2WJB3x+s1b2rYhG6UEsWY6H1FWFY6e8SRlvhgzX4yYpWNM17Pdbyjakm1V8vvX3/PtmzcUrSVQjjj2bhJb+wXoZrPh008/4f7unvEoYz6fcXd7i1IBbduzWq0otiX3D/cYC3XdMZlPubp8IMq3hFnA3eqW3/7+12RJyIsnTwmkYrda0XcdTV2z2tzw8sVz7u6u+NWvfslkNOfp0yeUZU0Yab7++jesV2s+/+JzlPRdb0fLCR+9eEFZFXSuY7XZ0NkeHYVopTm7OGKz2VCVJdNZiDElKtBI7ZF5y6MlpydHSCV59+4t1tUIIdhu91jjA4SSOKHMc6wxjEcj8v2O3/9uy8cfvyCOJevNisvLaz79+Avi2COXXn33Dbvdjj/JvvAGi75ms33go49f8uzZBcfHxyilef/+HZPJiL5vubu75fj4FOccZ2endJ1hvV6RN35upFlEGGnKsiIIfJDS3d0tIFguF5RlQVnkxFFIvt8zHmWUZUWnNWWRc35+zjffvCbLUo6WC5wzvHjxhFevviUIQrI0HdpxQzabHWenzzg+OuPd21vKsiaKIpIkIc99y+v19TVPnpzz1VdfcXV1yf3DnQ+KEhFRFHJ7c83d/S0XTy44OTnGGI9tUENr65MnF+x2BWmWEIUp0+mY2XTJ6UkOCNqmYzqeku+LHz23AXrTeYdVEFI2JQ/bW47P5ozGgtv7exw902nG02cJcZTQta0PKC8KFrOUKPBBn0I7JjPBYpmy3+Zo1RMlIdOTmN62GOvzgYp9xdnRCfu+4+52zemTKU5anl6ccfn+mrpuePbymPFsxLu3N8yXY+xQdPz4k4/4/vV3NL1lFI3YbmtevBxxd3vP8dmSpt3TBl7YA8+vbNuGvm8ZTyes1jkxEWmaMp1O6E1Obz1WTcgIZ0OEbDk6GdH2HUXVMZ+e0PUt+3zLm3cPLI8nnJwu2O22NKHl7u6BsuwQumU0TsFJqqoiGAX8069/jZaKI5NS9yXrzT190/h1Ud8TBiFxFGD2FUEYsHrYECUxSiv6tkEkfsN+c7cmXqasNlvmaeDxKShurx8oy5qT0yPqpgAcTdP4vZO1xElM23WEWvLzP/0YOfBGd5sNUgrSJCZJkoErHdEZg3h0MQYEKkSiieIY01uqqiHQiiBMCYKAru99J7Bw9BgWi2Pev7/EtA11Y5BhzH6f0wrI+5a3b77l/fWGKEywzpFMQubphNV+TxiE9D00Tc3x0YLLyxvu7+5YLJa8fP6C9+/eU9aGtiv4/MuX9H1Nb1qiSHN0PGYyHXN7tfXsaqEp64qu76mqkratKcrXFGVLlk0IAsnzF2cEgebJxRHbXYFUCYFuWW+2TMYbZrMZ//CLS/q+4f/4P33MZv+Ac4737+958uwUGQjf6dB0mMZx/OSUSAiEKgnDhNz0tMZwfLLg/uae12+/p2kESRyDLBDK0vcVpvNY07KqUaLl009f0PeGn/7pp0j8HPqxr6ZviZSibwyJTAjmIcv5Euscl1e33G5WWHq08nvqLM4wraPY7fx6ywratqftGpbzOeNkgm0cq7s9k5mgbbwBJUkUZV7R1R4H1Pcte2Np8pK2rhnFKfv1nrbukE6Q70s26zV67B2ATdPQ1o58XyOU9ffosiIJQgIVIGLFpt7heofQHgNX1x3OKXpbITW4oiOKJKOJAGeGZzRs81tcNiINFtgOkFDXBUeTU4IoRusA09cIfCdrRMQsnKJwxEmIFZZAZdzcbTk50SSBZ59vdw/c1SWBCKmrjug0ZTqZcXV9yXevvmUyHpMcn1BXHXf396w3D6RiRKJj2rrjZHbM0fEJUjhUOiaMJU1TE4WGp+c+q+V3r+8QaGaTBW+//5oXzy6IQk3b1oDFdBXH0zEBio6IOJRMYh+KPYszWm3QsqMvcxyCqiqYTGNMa9jvGybzOXVnEAK2+5qTU814eUxdN6STBKUlk/GMLJ3S9tC0Pfluj5Awn2bo0GF7Q1XvfbYYDiWUz5vqG9AWJRVlUfi19MDQ7pp+2M96EbhrfW5bpCKshKqoMMawXC79+lso2qrCYsmSlCRKKZuKYl/SO0u8WNBXBqECpNLUlTfpzEZTZpMJV69fY3tHJEOqfcnD3QPxaMSnywVb17Opc5Is9ajKwHe5N3XNfr9hNp5xdP6Eqzff0/decHUW+rpFBnB2euTdtK2htYa6y9FhTDoZszw9ZrdbIZwBC9vNHucaXzzQhjD68XMbQIpD5t+HkMmDWxwOsql3ah9yAhGDOVJIlHJoF2KM75A44FW09pgS+MCwNsZnuzknP3TOOx/7yWBIfxTPHY/vJZxDDqYq4cTwOd5tDgzOc0+k8LqRHkT6Abl82P/xwd1+2M2I4WfCuUeBWAiBk95JbE2Psz1CNJhuS6BWjLI5So8GIT0YjsMRBB7V0nUNaZoRRT58OMvGg3veDFqPGBCl/hmrpBgyvNSjwx8YxPHhOg1O9CAIPvDOw+iR2mCGc39wmdd1SxiG1HVNmmWPbHE9MPAPgr+UkrqqGE2mg/tcDpTqAz992LMfih7/vCjBhyJJFEVMp1Pquh7wg/66v393RRgG3N7ee91WfhCGf4gM+mEQ6AHv4osR/rwcMDYHof1wjsIoHPSOw/v4cRPokEMhDuEYjROOTmYEYc8nTy6Q0hFGPhdQyghrOz8unFdSpXQwBLAaOhgEcDvMXyEExnnWuZQCTI91zaMxkKFDwGNlho4M4fw/DwUgX5wDhPHonENjhhuQRFL9d+fwHy2iJ3pMa73wjRVYYgyghfIp29bSA43tfaCN0N6JTY91niHtBcnA850HxIsTCov2bRXC0dsGKQ7BjNpvJmWIEvEghGoCFeHwYQrWalARSvpAzw7PW5TK4agxtseHlRrPO5ehd1oiMKJDygRQOEqc8454nKUXeDexCwhlAviB7y+vRjjQKITUdLb3VRJhURKEaOnswGl3hwjSkRel+sI79pH+5ikcrb3H0dK71k8aQs+GEgIfNhr6qor1CbJCaByWvvcDXinfzqGk5yNZ16HiGqcMUsR0tsRhEYwQAnqXAy2HNiIp9JBfMLjgh7BBPYSM+MkhOTk6R+CwrqS3OY3pafuGJBqxOPqEmzf/iWp/z0//6v/K0fTfctP/v1DCO6wRHYgaKf338a77HBO0mF4iw4i+tgjr2zZs6xE6zkmc7airivfr7wmcpLhp+fLP/oyj6Alv7D/S2RWV+4be1R4HxA5HNRQsFB6b0z/+t3is7urHFiCPZvE3BkWIG7oUvMJthwn+ATXjb24Bj6UsDg8KXy0WKH/uOPCZHE7Yx/YoHs+9D2DxL+9kP4SNSqeHdp7DA1UOIr45fAIIX2FG/HinKsAiyYjnCbvdjpqOk8UYUzlGUcKu8w7fTu4IswzTGbTSjNKMqikw1tJZMCgcEYgGoQRSGzpXYq2ip/dzNwi5ub9HaE2cJWgtyDd72qYC6xf72kQedRGMmMRHCG0QnUG4niDR0BlaW5OEAisairwim81xUlM3NWEg6XpDbTqwkjrtELohTWLKsqJuHaPogvHU0XUFVtZc3r0hDROOR6c0RU7TWLLxhLoGaxWj0Yg4CVit7xDkvkoeBIRhSBAG1E3NdrehKxqs6ZhMR5i+p216WmPRaEIdUpQ5u7IijiRCGV8s6FucbHEour6n73ygamA1o8kYTErfdmSTmKIElCMbJYiuQlWg4piRHrG521GoilQZhDZoGfMnX37Gl19+9RiU8sEJcFDRP/D84X/rHDDWJ3vvdju22+3jwzpNE+I4IUlGA1ftkDLuhvsS2B84IQ7O9h+mgfsFhyJyIU4IRNcOP1doqQY0SE3dlhjbEGqFzjKUSsnLAhB0fU9RlUjpu4BAoqwkEAFxEFG3DdaCDiLa2lA3LUHoA2L72h9TXVSUzRahfXjPxckL5qNjyl1FXTYYHJNsThMatuU9gfbV8zBQBCoiDEIMGa3xi4DeGIztaauK0ShhPEq4vH7HaDwjzBLCLCYRCX1rUYFEal90DULNvsi5ub/FCsjGI+/66DRJ9ONxTXVTcXt3RZomTCZjdvsHeiMG4XLiMy9cTV0X3N1uOT6+wFqfWbJczgmCgN1ux2Jx9JjwniQxm82a+WJGbxp0oGjaks8++4zr62ufa2I1X3z5OZdX78mLLXWjmM1GfPfdFUl6xv39DXE0paoKqqoCvAv49evXLJdHPvDRQZ6XhGHPy5cvuLu7YTwZcX9/RdtFtF3FZntPU1XMFyN6UxPGU8q6YLW753Z1T0PHrmvY1g29AaUyppMjqqLk5YsX3N5fYZ2hqUuiUNG1Ha9fvWKXF1R5xdPlGS4e8fbtO4Iw5PnLp7y7X3F0smC9WxMlAUW5pyg3ONtSF1veXTxlPh6RRCF1WbLdbAjCgLLa8/Xvfs9kOvFOU2NYHB2x3jxwc3vH8ekxddsSxgld54O5jpYLxqOUZ0/OUaHg+uEGFSgWx1OysebqZsfxyYzz8wXXt9eMpgt2O0OSBCyXUxCGJImZzVIur1+jVcDR0RFff/2eZwj6zpKmGWXp+eW/+vUvca5HKsN0NsJaw89/9mfUdcd68wDCkqYh48kxjo6m6ej6itE45De/+SdevvyUIIi4v79jOp2wXj9QlDnL5YIoCgb8U8B+XzKdTmjslt60hKFitbqj6wxV1TCbzkiHjbG1ltEoxfYxRb4jzwuqqmY6nTGdTri/vyPNEp4+O0EgieOY91fvmS3GSCWIk9ALIasNfi0niMKMn/3ZZ2w3NV///hV/8Rc/p21r7u/vybKUNI2x1jKdTXj3/g1lWVJVFbvdhq5rODo6Yzqd8O233zCZZKw390xnE25ur5mOT1guZsznntertQJh+dUvf8tyPufoaMn97S3r9frDov9HvvK6xmm/NkjSgO/fvKU5W/DJyyc0rmW733K0mJJNIiRyCJiELBmRpglNWxElPlgrDKBtGpQWXDw5RqCYLFMqs6eqHUoJzs7nfP7xC+5ubrm+cxjXcnq25PL6mq4pkcoynqSU5Z4kDVmt7mgbwXbbPHYhhLHm4WbLv/pXz1ivr5kuJPtdwfFyie3vKHLflZRECVXVoHUw4EomTMYz6spy/7AhSQ1KW/JixelZSBxExJljv2so9ltevnjK2ekFTbtlPpvTdlvubu5xJmU8ShHWcVNt2O87Ts4Sj0GJR7y5uiRA0VQ1Lgz51a9/T5wIzp+ek++2REGA6BRSw/J4gpwGPKx2IB3Xd3ccL6e0dUdTG+qqwfaS27s1IxUTnC5ZTqa0ZUcgAmazGXGcYq2h63x2hMULU3EcEipwfct8MiLN0mEM7vwGXkCS+PbzQ/v4Iavg5uo92J7JeEzXt2gkVZ5TS8F0rrC0KGWpm4Kua9FaUdc1WoesVr6T72FfsStq78JUjjgNmUw1AkMQBsSxRAWarnLYRNE2lnAETdswm40GHE9LVZV88snHfPf2PbcPN2x+v+HLz5+TxAFFXhBrRVf6ThAtNVVpEE5zd7thPptSNN417rtDoSxLFsmIvnNYq7m9aXj2bEGWSfK8Zb/fIUcxyxPFaLSkbkvmiwXX19d0puG3v33D+fmEqmwQNiLWYwKR0VYF03ROVVc0RUkyHtNWKxaLiN22RqkAIRqubq45Pp4ym05Z3ZXgeiaTCVoFnqNdN9RNhROWXv34+d01LU3VMpIxz158QnqWsc43XK4u0YGiymvqukIqwXicIpwvznR1i6k7rLA+MwuYjOb0peX+4YbtvqKpYTTOKE1NV9a4FkztkS0KH7iYO0HTdExfLuhqiehDFIZAwfX7KxYXCVYPnc5YtFR0paM3PVLhaaXCj4umEkSxwrWOOMzompy2s0gZ4gzYviEZRSg8f7trNdZENE2F6QvaUQpK4FzrjW4YkjAAJFXfcre7xe0MC6kp+h11UxGMA0wIrUopTM+myGmamnKz4+nyGLfPaRrDOJswGU15//YNv/vd31O3BUo8pR7NyIuc3XaH1oZQOZyxTOZz5rMjlNZc3VyCa0mjEDs7JpaOcDpifhrzk6+ecPX2NW9SeHZ2iskLxCTD9T3CarQNeXn2kkgk9GFEaypGcUegDcvFmKowONdSCkG3vUIKi2gLojjAtJab21ukjrwIKRW36w2jyYT3l3dU1qAEdI1F2JA4igiCgskowzYd5Wbng0edoWkKzNA17TnWg8vTGtrW0tbtgFFoCXWCwnfcIo1H8moBVqLDEUEYIIG6qnGmo+vKwZXsccDxKMUoRdF0tF3nWelYgklK1xh6I3E2RKuA+fKYxWTMu9+/plwVLEYzkiTh9vaG5GjOyfEF5Sbnm29+x3QyZTxkIgnnu9zbvKIxAfMkY7o4oiwL9k1L01t0ECKDniLfIUXP2fEFeV3x/v6Gqqop6pI0S0jaEeVmi+0N9b70e8lJRuccefMvw7n40MmDM/mHRpnDfuzgOmfIIPH/dkCy8Iin8P7Zg+Pckwo+vKPnqR/45/JRnH+05v1gf3bYAz4an4b3lVIhnHwkEPtPNI/RhEIdUMAKZ+VgJvygjTDgZ/7A+SwOJsAPJsE/dFx7xLFSIJyl71q6zs99EMP500gl6HtvJAyC0GdeaS/qm7rDGu8Sl8KzzT3v3GNKg8AbUsxw7uDD/vQgqksph7BpH+J52PcfvvvhWh7c2Dj/9/1+z3g8fnSfO/nD8+lxKXXdYvr+UbS31g7G24HfjRx+//BJP8C5DIUO7/wOOTpakKYp33///VBgcEymI+qqYTweU5a1N80Z83itD9f7IJofCgF9Zx7Z7m3bPgrnUkqSJCGKIi/gO0+qMMZ33EtpPHNeCaIwphueBZPJmOPjOWHkSDOQSiC1R1r3xhcVhfnQ8fAhDNKbKQzeSS5RWDM4+e3AXn/UmjsOeYKHAoRWgR9TQg6FgkGMNw4OTPThvd3BMIvPmfLc/P/2648W0aVICYQaGM0gZUrragIdEcoI52rvNHc9iBBnW5ypEAqMPVCkey/Y4jlcxpUY0wE+kE6IHuMcQmjAoobWL2cdnW29Td95+dAh6E2Ldcb7x12PpUPicQSGDusKnPBhGNb1OCTKKZxr6J0XXnqbY516PMHW2aF9JPDoGMC4BsUIaxMMLUpEBGIErkOI1nPQjQWRg6gwTiBdgnQG6zr/2YP4bVxD73xrXecqfKpy6dtRnEAJiRVigHt4N6sQHXJgvRvqoYriq3cwLFi0Gq6L9NWZcEdlHujpcFS+eGEEUvoQPURHZ2rv6he+Dh7SIYYKp0PgrO8usK5lMl4ymywRQhPFGXGfIKWvzt4+vCNw3iV/9fYf6Jsd/+b/8n9jx6/Z9b9CqwStBFIGA9MqQmovmHWmgXBHLyJ/zjoNskMnDRQgAkWUpFxe5mzz17yqGr6+/j3Pnp9ysfzX7M2Kmu/o5S292+MIUHrghQ3C3sFt65wPrfEdELEvKJghgOBxKgxCuBsQK0N7iXusCHvx3Dn7+Eh6vK0J4bsWhkksB37c45i1vpAjBiewIPBiOdFQffZiuL+JDA/H4dgPGKVDvdg584MxIoY58+NfEx0RJTEdLdu84M3dO1KTkMmE/eYBI0rqZo95qOmNIlAjnINtvsXQosWYMPIhFmVbEqeadKwxwnPkjHTUZUGgNPP5nCTJ6G3Hdn3H3c0V4zQmCgPfYmo6OtMzyRKiYOTbjGcjQqWJtMb2PVVXsO3uaG2JDS1Nb8DVBNLjK4TWRDIjCTPoFPtyTxB4JrkUMYvJGX1RUwi4XN+w3m/o0o5AhnRVS1NLToMY+f8j7c+eJcvy60zs28OZffY7xpCRUw2oKkwECJJAU+wmm01106hHPegvlJm6ZW0SH2QixQc2G02QGIkqVGZVZWbMd/T5zGcPeth+byQoiEAXjllYhIW5+/XrfoZ91m+tb8mUIovQkQTR0fUVseYY0SKgaRDHso1r2qZFCkE1NCRxjI5inBasyy1GOLwfQreBA+cG2qGj7jvKuiUWGSY6gNf0XUMUjZhMEqpDy75ekyaas4szsixj8BW9LbGiZ1zkZKMcOaTMs3Nm4zO6SnFx+pTf/Xu/R57lj5fdv2p7YKHBBxH9oWSkaRru7++5v78HQqzLGENd11jrMcaT5wVxHKbRD5y+B2f7txdi316kheKu6DE21x+n+/Y4DZcyCsNDLFJECHWMZsmYbhiQOjji27bl0JRkWYxSnt22Ih9B33ZooRjnI+quwxgYj5fIaMC5js6EDoQiK2jLht4YlNQoB6Y3bFcb2moIqAgESSRJCoOlYegrvPFkaUweZ8wmM3aHPYc+8HvbqsKPJggcSSSQGAbb0tgW7zQy0qAkg+0pqwMiExSZZFeXHKqaqmnIRyOUSjGD5WQ+Zzld/tLHdp7HWDvQdg5ZBUZecKUvORx2x7Z6Q5aOGE/Cgs9aS9e3vHv3Dq0VZ2dnXFxc8otffE1RFNzf33N6doIQjixLca5nOhtxf39NWe44PT0lTRPatuRw2GHMQN/XCJlwenbC27dvWC4ugktJhgTUq1ev+NVf/1U+++w596sb1tst+8OesqqYas0nn36CdQNdXxElmsG0IAeM7zg7X/Lu5Uv29zf4rmW/2VHWJb0z9M5zv9/SGIOwkGcCYxrKg+bu9pZDdWA6G/HjP/8xT59ccP3+ijTOyNKMzXrLp5cv2Kx3fPnFlwwIKucoRjmTacF8OaXvB3RckCSS+9UtwvQ0+w2fvXhBohWxjlDK0bQl796/5uLyjCzKSZMR2+2WSKeh5C+K6PuB6+tbzs+fkecZL1+9omvbEKXGMZuN6XxD53uSTHG3uqVuGqbzJ0htMLbFe0MUSZYnC6QKsdX7+zvu7q+IY8lnn3/Em7ev+fjFJbPZjDev33JxcUFRFAxDz3w+wzNwe3sLhMGmkIG13rbBjfvkyQVKCW7vrsnzgNSZzgrSLOL161d897vzD/goJWjbmslkTNe1VCYUoVtDKJ+t2iPv0dL3LUJoTk6WDL1hs9mQZ8XxnBAW/HVTc3Ky5IsvvuTi4pyu64MY3LXM5zOkVOx3exaL2VGUDOelqix58eJjvvrqFc4KhsFxc73mv/9n/4L/+f/xf+Pm5hrr7DHC60jTlPl8fnRmhmKm6+srxuMlUaQ4HPYMvadtQzx9Op2w2a7Ybjd4F9KQJ8tL4iihyHOq+kASJ7RNxfnZOev7e+7u7vjhD370Sx/bAK3tw+jeBqSfEo7r23sGW/P08hRrWm7urkmTjLapwhpJReRZjpQRcTwQx+F7Go0mVHVNmiZ4r5nNxuyrNVa0pFnEptlz+eSCd+9vSRPP5dMx508vjoV4ZSiLTCVSeQbTAw5jWkbjCbNZzqtXLzk7P2EyTXj+7Cl//ic/wQ4OZyKaumM6C4WOVdXhnCFSYVD99s2O+SzmxcfPub6+YRgMSgUWqrEO61oun55w2Pbc3W5ZnGRkaUpZVtzKK+aLnN32wHSSE2lLnqYIAnv0ZDnl3bstWmu0UmgPsYzoyoYsTTlUA8vzAkuPcYbW9AgkSkeBx3+/42S6JMsTemuomh6JIk8TDts9Eo30irq0eCTz+QJnHW3XE+uCyWKEtZY4Trm7WzEajYP7fjDIMggRfd+BEORpBM5wupjRG4vzoTi6bdvHpNkwDI84g/JQcrJcEseaug7dLFGk6fqW/dVrkiRhMpmCjMizgs12j7GwKXt+/OXXrMp7dn3P+YsTTi9GxLnm/MmMSEVEScrtzT1CSp6cLxmsIMonZCPPerslTWMG59ASOtNwffee04s5k2XGZrM6roUtWRLTleH+cVRkDL1jPM4ZOk1dOsaF4LDvQwnleM752RPevn+LlJKbmzU6yjhZzHn19S0XTwvmyxl4wXgy4tWrtyzSlM70SCOQSjObTkjPM/CgJynVXlIkc4bOk6d5QGfdr/itH/4aP/7yZ1R9g6FjOp3z5PIZd3crmmaLd7Dd7JiMpxwOZbindQFxR+uomwanPFme/tLHdtJD1ww8++QJpycjNocD6/2GajCcXT7lo+eOV69eBWaujNisdxwOh1BgbkFo8E6Gnh6VsLpZcf3+FpnE2LJkNMqC8G8GpHTEWuOkwnQtfdfT9jYYHdIY60JnmY4EkZYB02KPpivnwThiHeG1YBhaYvlQfiePBc8xCo0WijRL8Q66ZouWkjzJKU2NFClu6HDOsds2pMmYuh5IMkWaBnTDfr/F2h6pJJ5w3BjT0bUdMRG96Vm1G6q2RhhFNhvhJIwnARXWti3Ces5OFjz/6GOu17egYKh3/MkXP8ZRsjgdEyeeYQjmujiOGKVTlEnIXIYUgq6r0BJs3+C9ZVXWZLEizVM6EZOqmN/9vd/iz/9IMXQBo3d+esJutyaKNHqQLJIxqAmmi7CRYHCQ2A7lFCoeoYaBYlpwkS7oRo7erolGkq6tSbOE3liaNjjSy6ZkvVoRZwXWK2bLU7yBXdVAfGA01yxPzxhN5gx9ix0aIq0o8gx1L1Ba01sbyolFKBr1MiDsIqXBeZwRlLuavBgzaEvTVqBcWL+IwFi2zlJMxiQu4MqsDSnQ5IiPAEHXhkLIru+ZLxZHl6uibw9hqBZn4DxSCNqmZZSmFFkehohtRdntSVTBy7dfYXrLOE3oqpLZqGDoBszgjzqTpzcD99s1pxcnJEVC3Xf03pAlOcUopW5r1usNz549R6cJPhbkk5y2adGRIs/HVLsKay1JUqCTiDRJsb7iA7b1l92O/XPCf8uRDgHnKj5gPARHymt4nPMOrHv8PB8wJd9GsjwwrJ3l+Bo6PFLKR+Rm0OHCvd9/bnT6gIH5IKzLR0d0UPQ+ONKPsrd40C2OTvej0BvKT923sDLhzcpHnviRZf1oVfRHo2xwDEsZ1K+mrSjLHScnMxASY46OdedQShNF+vE8PAw+mBCjQJ6I/dE5HYcSzmEwJHFOFEc4648icI+UAh0FPeXhXlXrgI9pmuZRXA9Crnws9AzCNI/oFAgJHakk8ijGP7jZv/1vIQR1VVEUBd70SB0fO/U+GM//srnt20Y4iVASM7T0fcdiseD73/8+WZbxp3/6ZwS8Ss98PqdtO/rePKJYAz88mIMfvnfn3OPPetATgYBh/ZZLfzQacXJyghCCsirZbO4e1xYP+473weQmlTwiXcDagdFkRG8qIqEwVh57FbpAVXhMyj+QGFwwn4pjR563WCMeS0CdC0Y/6cURdT18QNMehzDOuaBrInHePaY2nAGPRYqAcHLfIkhIAUIpdPzXs9j+xspbPdyhZIQgQmPRqsUYy2BTFHNCMePDB+BBdoBDkCJFGpR/D8a3aClBRTiXIOlA9HRmHRAo3objzkEkwpROeECEqY/zLYOpcB4cLUpqHFE4sOWAseFmUziL9RUgsV7hXQUolMzRIkIIh/cRvdnhEGiZYH2LEiOUDJKlFxIp9BFub/EuRFk4iqvW9+AMUkboqMfZAWMNCFAiRStNbweMsxhRokWC8ODEgPU9xjmsH1BCokWMFMOxRNThvcb5ColCq6Nb2Qc3k7U+TFPs8T35MAV0PpwEiFtk0YDqsaIEWryXIC2WsNh+LEgBlLJ4ND4+HHc2Fb4DQvRBRxGTyfzISYcsnmFHjrI8UO12pNEUax2T6Ufs3v0Fu/ufYvf3TBafUfe3CBlYZ1IIjG+xbkuqUrQeoaI53kLvDEr12EbhpQFliYRAGkOWzTHecvWqpNxeEQnB9u4tk/knnMvf4mfD/0jrV2ihkCoUj4adJuBVnA9O8g+sMYGjJ0DtxfHgMo+O+w/zUHl06T4I5gYeP5tQCujxofhWHKd66CMLegB/LIvFH6f47nG6FvBEDu8V8jgxc0ch/UEQF+Khjfs4zcWBl49JitACEGL/TpR/00P5r9xSmVKVNYf2wIBlX+1RWtN1w3GQYHDW0pYGpQqWl0tUJLm+3VObA6MEPn7ylCLNGFxDZw54b9hu18RJgVAgkzDZC+MIR982dK4miRRaaZqmZzyZMM5zzH5PU+0QsaWpWvI0xkUCI0GkDqkEYpA4YxnPp5QHQ9U1ZHFEtT9gvGE6XaJsjDcC7wz9UGOFJ8pipGoQsmc0SXF7h3UK51OaTtK2FV3rWbiBOElwomG1vg/fh5eYzmN9cHfGWUBcjMdjyqpE6RStY1SU46UmzScoqTmUJXU3gDcoQSi0EI528AgVHMLdYcB7yPII6zrqtkfsNlR1RdMdEHpMkS9QWuNEz76+p7cNUTJBkbCcjBjHJ9hWczK74B/8zn/F+cnpcb+E/9xlAIER+u0J9MOFtOs6DocDNzc3XF1dEUURo9Ho8aLvvaduatrBkDU1eZ6HWJdUSCWPxSTurxTo4cOkX0qJ0prRqAgLl7qi6zqMaambPX0fcB3O9XR9c7wQG+IoOZa8KNq2Czd6SUQUJSxmI6o6wkvHoapCeZJKyLIR6Iq29UgnUXGMlJqm7jGDZz6fkqiCvu5ROiKNYvIkIxtlNF3Nu83XGJpwTiZCesXQdbRVjTeOi9MzVqs1N6s7ymyEFgLXG+qhx0vP7eqOoh9IRpp+sLRtx26/xfWWoZfUTYt1AiU15b5ByZyLs0tE5Sm3v/zxHUUxeZEBhrv7a5IkIpUR2+2GoshJs4S2rVBasN2uYSr5znc/5/37d2y3a54/f879/SqUxbQdi8UJw9Dj3EAUKU7PlnzxxU+PWAFJFEus69BRRtPUNO0BITxxHNG2HU+eXtL3hqIoKA8NZmgoy4bTsxnr1S2ffvqMl9+8pmt6pNDEUUSSRvzpn/0JZ2dLtrt7vA/pgizLWa23jLOCvrfEccLgPCKKWe121O1AVqREsWIyTXn76ob5Yk4+yri6ucGpAesGxpP80RUURQW3dxuePn/Ou6tXaL6gLxt6KUJ5mpL82U9fc9bUfPzRBTfXt3h6rPFMRwXjac7Hnz5nNCpQTjAZj+m6jqmS5EXGcr4kjTKqqqc3DVIbbt69JcsTBjNgrMV5S5xEaC1pu4bFYk7Z7hHDwNnJhNo0SDlQlTvG45go1tze3qIjTxw7Li7mzKZL6rpnOg2cxOl4wtX1Ww77JSeLU968vWJ/qDDW8/U3b3jy5Iz5YopUgqurO7wIQtBkOubq6poiH7FcLo8LcMV6vaXvTXDh5TM+/+wH/MEf/ClR5EkzwR//8U9J0oyL8wtmswlJklDXmzBoVBGXF88Yhp5Dq5nNJ7RNh0Az9BbvwnBmuSzo+wZoKA9r0jhm6C2bzZ6nT5/hPRgzkOcZTRNusIZh4Oz8HOcSdBSRZTnDYOm6kEp89uwZQx8W1q/evKIfej7+5CNev/4F0+mEPE8xZnh02zRNS5qkjzeUFxcXVFXNzc0tSRKTZQmb7Yq+b8P3FmW0TY9aBn7m2dkpr16+JMsTVvcr5osZP/3iZ0id8OzZC/r+b4di00lEiqdvBYvJhNV6gxk896uKtuk5X04ZTMe+PmBNyF3GUcTF+VO+/voVpxcx+lggnCbjo4GkYTB78uKUQ++wrsN7wenpnO12hRYJxgYX9HiacqgVi5OC9e2BOIoQR4ZocAmF7qI4SZlMcjabPV0Po2TOxx99ihk6ttUVo5Hk3Zt3ODyjUcZqtUEpyXg05f5+h/dhv/js0+/w7v1bVusVMzkmEoK6bmjaHWlaMJ1Oub/dMxkn1HWDEJYodmzWO84uw4AlUmCk5v52y2Q05ld+5RlOgnDg254IgSZ0l5yfzdmXa8aTnMPuQJZlrFZb8kyihOWwa5HDiulsRpFnzKZzpqMx5QZ6WmKlSaOCH/3mj3h6viBPPalSqDhhXCwopilN29K1PXGUcXpyHtZ6OGzf8/bNK4SURFpTNw1911EeSuI05fT8kq7rHvmmSZIEYXw8xvaWm6trbm7vQ2loPiEvxmRpSjO0bPZbyrKk63oiHdN2PePxlJev3nK32bMua1oviUcphpbBpaAcveuI04hXb6+wxjBSmmKUcbjf8+KTZyQjuL3bkOaCs4slRZ7z7v013e01H2UROtJMxxOUAjMMLKZzXOe5vbuDWLJcLugaSyQluig4OVmw2x0YjKRrE6Ko4Dvf+QwvBdPplN2hwRqL8J5f+9VPuL17yW5fcnN7A1LQDRaFpG4ahNJMRyNur7ds1obTE43pJW/u38CzU/rUsz0MDPT8hz/6j9zelzz/zhIZQddbvFWY3pNEIxazJfd394xORwAYYzg7C2Vph8OeJ5en3G43PHn69Jc+ti+TMcXyhF/5/mdUXcmbq29YVQajUvztHU/Pz2nqKqBRnOfm/Yqq7NBaMB0VYT0nFd4KmnZgvdkzDB6tLcJLdtsd8/mMvu+w1uOkRwpJF2CmtMZxerYAYclHCV1ZI5XHuYFpURCpBKFinDCsyg3e9SRJQRYpdPTgtHVUVc3JMuA/tUyYjqYoJJvbDV3VoFyCaQfurjcUE0FRpLihZ3Z6yXKmaPsS27vArBZBDGnbitu7gTRJkcqD9ahI0XeWoshYns243d9jtKTabxnlU/ouJOycT3h/dcfJcs7l2Tlt1/Dlz76m3u2Ic0GsJMJb3r9/C17SD3tm0zFCQDt0zJ+OWWRjfG8oYsW+qkOJ9GCZnz/Beih7g6oPIB3ZKAMlKNuAKOTYnzYYR19XpHHBYjYlaVucHehtTKwLfBwwKSLzbF5tSEaOXV0xH48YBkcUafZlE9y3fcRonHM41ETJiME2SK1wKqI2lkJqpFZkWpEkEmcV3vZI45lNxlgv6LyjHGqapmRgQEae+WLB0Hbh/KRDR02vekZZBn6g6xqGpkdqhfWGpmtoh57Ti1OEBuMcuyNCzJgBe78Jrl8RsKnOEY5f4UmShL4xRw3IcXdzQx6nTIspWRKTjROut++ZnI0ZaLm9uyUhYZKNsEnYB/p2oD40uCN+OFaOwRkO9YHxdIzsBlbbHXXX0dlgaMlGKU0XtJGz01Na07HarDHG8OTiksXilPX9mkxE9EPYTycLidTurzuE/5rNH4XKICQ/3IsBfNslLB71RcdDH5sjFNwKEXSB8KCwb4WX+IDRDIKs+Nbrfij79M7/JSH94e8PRZKCB8SMwwYn+6NWErC3oHAEcdPbB8f88TUBLwRWHP/2D1K/eDQnPmBJHhP7HLVA4UL6RFq86Bn8lm0lIRLk2RJPTKRHgfPuBoyrEcKRJglaRjS2RWcB8+K8Q0n96NDWURiGgKfrDUkskb0EeXxXUuKdQSpFfFz/IUKxvJAa64I2JnV01IQsg7VI4YjjOPxbKwYT+On4ByPZh8/XWodSD7z1KKSwlXpE5ASsyQPV49tb+Aw9QVyO4hRjwvtr2xbvPL/+a79G3TQc9gfKYzF03w8hIeJ9EMVlcHI/qvV84LU/aF9xHCElKK3J8xzvPePRmLPzMwSw2W6pqj3DMOCcQUqPdR6lIrSSeOEZTWJOzgqipMMYSRxLYpWEhIB1xwHMsdxYfHt/O7rK/ZHp4D7sz1LIMHQ5IqbCcEcFUsPD5+ehd8PjIOQRZeMNXlm8eziOHn79oAdKxSMz/6/b/ub2Ve+x1qClBZkfL7EWKZrAvvZg3cDgOpS0aNlhXQzCIqUBDEpanDNYIiIC6yt8X4HxraRFEOHo8KQomWBMEBDxDiHBW4EQGYnMcb7GYY+/eMBFIMMXr0QKfoSXBuG7IxYjwjqHlvp4oyCI5ISBBi+Ci1WKIYjRHoSP8SLFe8ngagYfBCThDZYWK2o8kkjG4AeMrzHegpdYv0FpjRcWISSGYwTNe6zrEcKgdU7MCH9keeMGrIvQIsVK8XjiC67qCEGMUTusO8cOhmEwIAJ/augGZOoRboD4gI5qnLBhGkuKlBFKxI8nZkuE0O4oxAaci6HFCReYxfIodCFJo5g0TtA6Pi5AHEV8ynjxjFF8xpt3P+b99htGkzNefPJ7tIefE6cJnbvFywbjGqDAKYEj8E971gxuS6QzhI8x9oAXEU6GBm6tA8pHWZCxZXmZY7uI7eBw/oS324YLYmb2U57G/wMvu/8JJ1YBq3McNnjRIYiPeJThyDl3QUp39jixVVhvj8gfHag5x2loEODt4wnL4cJnhf2QqCGcPP3RYR4OaBH24eNzPzw+uNzFUX4PfPawIPVePDrgw4UpMG3D9DgkA4Q4xntEOOmJ47TPeYel+xsfyn/VVtcdlW3oupbOGYp4zHy+ZDResK73NE2PFMFlfzo/QwvNdrcFIXBOsttV2DPHYb8DaxnlYxKZsq8r1vc3zOZz2q4B5zldLDCu5d37VyxOZzy9fIq3gtubO6wRdKbDo4Ae4yoG0+D8mLrraHYVSSrDot85yqaiti1dJxhaz5OzU5Ii4u79DcJpptkZ1huSFN5fv6T1A3keUx9uWeRLhiYs+tJkxCg/QduUbrglH41I84i2q9kf1nRDw3i0QDhFtS+ROrAmdRyR5Vk450iFiDIinZIkY4bBUR4sUSQ5HFrwjiJLsGHFAEIQqRidZugoJXEN1nWU5TZc1JOc7WGPcTVEAyrRqCilbgbQJZvDHZ3vWK0OFCpmnJzy2fMf8snT73F+9ozJZMqDTwHxoTD0wyn9L8frHphoDwL69fV1cMpGoXH7wen28HfbNti2YjAt3RDwKkmSkKYpURQRGrA/iPTfnnx/m6kmlULqwF0bFQVKSeLYI5VlvWnDcKfaIkRwBJlhwBgL1uBthzMNbeWI9ZRIaa5ur9ExRLHEmJYkSrHOU20PWHkAORARXAvSa6SPUMREMiOWGe3QkBUZ0oOWnkjCrm1oygP5JMUbR9vUjPMpkqNTJ47IkohUxyQqpikbiiRH+IjV/R029qHNXbZUjcWYjiTKqPcVsYs57Cu8gzyfUKRjtpsDmc6IRYzTA/ty+0sf2zpOwuBOS7xwRElIK0U6waNROibNwqJ1Ns8Z+pbFYsLd3XuMHXh/9Y48z+n6ljSNubg84fx8zp/9pz89tr1vaJqKZ8+eUJYH5vNTjB148/Yls9mE0Tim7yxaRlxfb7hTQYR79+Y9o9GIw2HPqBizWIzY73f80X/8Q8bjOUV+YBhaiiLjZDnj9PSEf/pP/1v+5b/8l3z55c9YTM+w1nLb7Wkrx3hyQqIijEzwWhAXY6LUc3lxGRIyhx1Ca86eXzIap6zLe/JRymi8YLW6BxRlNdB2gtWmJ0lLpE4xMmL50QlLPF998zVRLJgtEqT0PHmypKzuiVTKYXtg6FuENKELIotJdcFkumCUFQxDw93dFUoLPIZhKBmGmPVGUFUrzs8X/OTHPwlumUgxDBXPnp2xPaxw0mBli14N7Os1o1mBl5KuLVBRTlUZsnzKbC5ZLkbUdU9V3odytCfnvO0a8jSjSAvevnzP2fkp5aHhm5dvGY0Kzi/PWW3WTJdTRtMJ6X7Dbn/ACcGhqhkXE6bzJfvdgY8/+Q6LxYI//MM/PIqUCmMEX391hxkEi+WU3u7o7YHk2JljjcBZWCym3NzWeN9R1fdsNttQHoXCDQ6soCkbdrs9WisuLoPjve/bEI0t5jR1z3q9JU0TwnXSkOcJk8mI69u3ZGmCdTnlwZAlEwSSNElI0xylYqqmZLaYc7++wYmOu817rq9eMypiDuUWpSLOTi8wxhFFCW/fvuVwqIgTjVIRSZyxut8jUJxfnCOEp6pKbm5aLi8+YrM5sLrf0jYDp6cnfPXVF+z2W87jc1QksdYxDAPz2RytE87/lsWiQ9+TZxMSIZiO50RC8/72jii2JCrh6m3Fs08WNH3F0DcYD9YI+q7j/Ow5bfeG6SzDHr/DOIYnzzKiGL55+Z7v/eiCd3cd+0ODdw2zUY7tW3QywgvP1998hVCgY4mKPGcXS7puwNkIHVnKqkR0niSJ8PQMpmIyWXD1/j2j5BwpJBdnl+zqO9qmA20pRilJKhhMQxwXeK+wzlLXFZtNxd1qg3MSJRKubrYI6bm7v6dIBhazc4r8hJvrW2azgq7vqaqGokjBhz4PrWKUkDx78iQ4weoaIo10kCjFJ8/OEEJSdT1JHJEPOSfjJdf3N0ync8aFI1EpfW0oIkhjSbnfMRrNaaqOzmkmyRgSw8dPXvDrP/o1Pv74BT/94k94/e4tf/c3fgPRxTRNhz30JHHKeDRBCo3WMX3fkBcZJtLEWUh6NXVD3fSsN1v6vufFixdUVcXhcGA2mzGZTOj7PjjLihFxnDFdnJHkGdl4ij4abbZlQ5pnxIkj8xrrJM44vDfkRc73f/gDurjgy+s3FMmY+UXB7faK3ve4wVAPBjqDFZreG9JRxrvra9rW86NZSt3uOTkdESea0XhEU5dkuaZuasrDjjwtSGLN0Pc4K7hbbfnq63s+++wp68M1q/s1y8U5y/MT7u7uef36G4pRwv19w6/9xufkuaKsS6qu5tWr91hnmU9OWc4uiLWgqodw7ri4xLoIZHDsd22L9CFB+OTJBb/1G08o9weu39/wZntF2W2ZnMxo2o5klNHtOuKRQsiQhijLLXXZ0FQtF2dP2B9Kzk7PEYiju9DRNCXv379hsVhy+eScYjJCir++nOz/3/YrH3/M8nyGU4aff/OOzW7PtvF0bmC7PqAsXF485eU3r4Nw5SSRjlFKULcDDkMySlBasS9rdvuGRMeMxzkqDggErSKySc7t7S3eu4BpjTSzPCUfDzz/+CnSw/J0yr1tyfKYrMgYTSdY5yiiHK0EG78nH+UgFONxymAb2rZDCEnfG+azBV9++RXeCyYTi7MObx31vmN715EVCTK1yAbG4yBu1VXJ82fPidMFVVsHtJ8diCIZzCZ2oG4qlPQ0dU82yiESlKZhnoxovUV7j1QC70PRdNv39K1lt9tTViWffvSU6WjBYrynLBuc7sFF3FxvuL3doyWo2BLFinlRMJ6OKKsdyzxhUiTcvi3xQ8d4PqMYjSmbnu1qRSo83eoNdhj45DvfoTxsKduGLIkRSiGTjIM1/MU3X+Dbb5gmSzKdkaiIxXSKH46WRTvQ7w/0+wNpEmO9YWv3zBZLMpkwIaLtDTrO2JctUZKT5xOSZETVtEEEzHJ6Y0i0IksV+BjTWYaup+96ijzDC4WyA3c39wymxwiDcxbdtIzzEc4Kut4i5INQp0miGG8tTd1iE4+zwVS4ut9hvWMyn6B0MF82TYMAjG2DIFcUGFPT1B2968nGGTjI0oyhHei6jqYqWU6mPL34mPXmnkNXQyqxkefm5io4kKMU1w/MZkviOKMvDF05IFRAoQ6mo4hy2r5l2NmAXUojDtsDnbW4wXC2PCcrCu7XK6b5FGsNSgR2eN8N5MWYvrdcvbvCWU9fDzAPGMu/1eaCIVNKebTKBfyrB5Q8Ilm8P6JZfEjkHsXVkFwPLnaEDEKg9Ajnj1zqDwWk4uFp39oecR7HYtCHe7QHgffBWCXlB0f8h4S/OGoW8qhJBMzWYB3i6KJ/+BnhHwIv1FHaeOBQi8c/jgdHu0eKcG9onQkyu5R41yFkMELumw5izWgyBVXQNwrXhwLONMnxosP2A3bwJFFOnCV0XYtzLgybhDxiRxQyigNCLAoJOKFk0CitJUoSkKEkM0qygM3SAQMjlEJFEUpGwYhmQsrMCw9SgRT0gyFOU8qqIs0ywBNF4lEIFoJghj0OOqxxqERjhj4Yv6Q8iun2W0OPYyksD+lucF4gRYTSCTr6gKmJo5hIafabLXVdMwzDsVtLPJrhnDvSGh6GJRwJFFIQx9HxnDmgtWY2mzCfhy6ryWTCbDbj3bv3vH79kuVyyW63BRzWhtSRlII4CUSPJBE8f7Hg/HJEFgvyNPQkojTy+HuFpOmxo1J+4K4fo6TH6tswiMFxdMaDM0HHQAq8kDihcMIdjwG+dQzIRxHdq2MKw3LkonukVEeKxFG4P6ZD/rrtby6iixwhHMYPGKeC2Cg7pAAtZRC+ZHWcKWkcOcZH9EOPFD1KSJR0aCkIaBGIdYxzXcCKiARJgSBFiAEvLMaC84pIZngxYOkQ0qIxaNnhnMcSI4jx7MIJX2bHItIUI2qcaFFMkOLI0PXxMTISMCtKZAiRg7MIYcAfHcveYalxvsU7gXUDQoxRQuEwwSqPxAuFPcbhrbcYb7HeoElhMCEiJ3M8HuNLJAohA6tcSon04K3Ce4sTBiXTo+P1KHpjggjsPTiLlS+JeYrzkq7r0BpM32N9aMDWkUPmFUJ7hIgCU9f3aDkJB7lvwkDCjxBeYO3qKOZKSHbIyEHPhyiNDScndWRdBcHXY6lRBGf1fHlB1695++7fM59M+eTp/0A8eYHpBMg93rYYEYISUrZYYcEnWDFg+i2RnAGh1M+JHucTlJA0bnNE7FSIRLE4P8ceFI2Fr65es3z5xzw7+S6n6a9j0oo3/f8V7wc8JpyovcJjjieg+JiWMDifHgVvhUAdL1nu0X0vUGGyyX9WduEf2p398aIQePdhcPMw4TqWj6KP+KIwGXNH3r1nwPmjWH9Etzg+sM5CaW4Qya0PqQopjkwn5NG9bo7TW4P1gXfvXfs3PpT/qi2dLBgaSeF6oqFlPpqTJAWVgSibUMie6nDHdDIhUTnlvmZf7pgtFrBPMVJyc32FYsCaiixZMs8uMPmOm3c/o+8MMhKhdHfvEM6Atlhj2e8r0qhAyZRhEDilaPuOIlUhwiYF4+kMrTXXNw1dFwpqBhyHtkQ5hR0kcZRxffuGyTjG2Ia6rhhHjjSJaLsNceoxVtKaDtceKNKC/eEQzj9KMC4yaCOSdEw2LiibLbvDfSiQiQN/cjJakueA8OzLLcYZsiJBRwEHVdU9cZSEzzRJ2Wy3RFGIobZ1hcaSxoo40ggJg3PUZY2Qhmk6pW5aymqHjjJ0lFA1ewZXh7SIgEPZ0pQ9XicY17MvDxz2hvPRhB/+9q/wu7/zTyjiyfECExIRD4OYvzT54cM0HMKFxpiARynLkjdv3lDXNaPRKGCXhuGxqTvLMpxzNG1NM9Q4YXAiZRgGzC6wQSfj6WOXxMPPCpNq92GQd+TLCSmRWj/G3vAQ6ZQsnXBxnnLIR8S7lLYt0ZHEu4qh3VMkGtIU6Ru0hjzWiHHO1zdvaA8lWaoZ5TmR9IjBYweIspR2GFBKkqmcSCSMkylmsDBotEoY2gPCOSItEQwcDiuE78mTjFQlOO/obEkcxQivKMsaZTrMTUt1aJlN5mAkicqRagzTmJIDTqX0gySSCcvllCzS1NuOJ+dPub69Js8KkjQjUgnz4oTl4pTFckHdbNlXd7/0sT3KM4a+IYo1s2lOksQ4F/ANsdaYvidONDrSTIoZ69WBl998xYuPnuOseYwfdl2HjiLKw4bl8gQlBWmahGiy1mx3G6w1RJ06Oh0EZRkEnlcvXxOrCNP3SDGlrUuElCRx6EL5+ONnbNdb8jTliy9+xo9+9GvkWcyq3vLk8hlnJ0sm4xH/4d//eyKlGecjbq5uOT+7YDaZctjvGRUjzs4vWN/d0TUt5xfn3N/e8sWXP+VkseDm+oY0z5nOx7x8+XMuLpcsljPW6zu0VlRVyWq1I45yjBlo2prbux2z2YSpmIQhQh7zzTevkVLy0Uen/M7v/DYIx+31Hc4anlyekmWKzX7Nfn+grQyTYs6v/ehHJImi6RvKpj6ym1PiPOZ+fcfusKEfWjbbFePRBCUgTTT7qmI6GzGZ50xPUqbrhNfvBAMDvbP8yg++y2AV1iqmkylKl1jXsj/sefvmhs8/+z7WWaqm4fr2ju22JI4FQimWy5NQJKZVSEnJUCYbRQodxcznAVOwXC4RUtPU4UZkOp1yfn7OeDzi3bu3VFV1TAZo4jjh9HTJZn1HlsXEscKYHmN7DoeO7e6OwbQUxYj15p6+GyiijP1uFxxufUjzee8YhrCIHY9n3N3VlGXJ+dkZ+8MKHTmadou1hrZrOJQSpS9A9AzWgjBoLanritF4RJqkZGlwmJdl+fizLp88QUrJaDRiMs64vb0jipLAydQ9V1dX7Pc79LG0KjDBz7m6uuXs7AwhAlLk9etXnJ6conXM2el5KNg7nudW6ztGoxwpYTqdsNseaJqGPM8ZjSaU5d+uWHToDEkRcbOpmOWKT54/ZTpN2NU7bKd4u6pJRgOL84LBObq6Q4qIn/zkZ7x4/kOWZ+cIUdL38OTigqubFavVniLOSBPFYAIS5/x8zPX1irYRzKcLpFSoSLNarVksQ1w4TiKGYSAfjxn6FbPZhKFv2e5qYIsUgidPJqHjI5+zvtkxGScMNiCKtBbIWGNdT5wo+r5hu73n5DSmqWr2+x1KjRgVEZPpmKq0VHtPMQls0Tzy/PSnr0jjOc+fPyXLBV99/YrtdsvZxYJDuSKOwnXBWc/bN7d88vFTvE95c3WNcp55OuI3fvCrTCZTbrdbfvKzL5kUE0bJmKG5QjrFYdPQa492MaYO6bgiyfCDwfcOnWj+D3/vH/Li6RMW2Yi2boniIF6+ebejMTWz0SjEvn2PlIooTkkM3N/fB6dWEtG0AyrOGE0moGIOhwOLkzMOZYlxnru7O7z3LBYL1us18/mc7XZLmmaQaBZnF+RFTt91OA9pWpDpGGchzxLms4ujUWDLfr9hGAy397f82U//DKsMcZYwYMjHBZv9juVijG8SDq3g488/4vr6mvX+wPmzCaNRxv3uPWXV0nU9SBn6SrTAC8vTZ0/AOO6vbjk9PQ09AVKSJBnnl4rruxXFJCGJM6QSxInm4uKU1f0t96t7BBNev3zPRx+fIBU4F/A/xkmk7JnNMmazgs8+veD1m2tevnpLFOdMZhOsrOmbkjSKQEjubtesrtfEKmc0yliepci0x0QNu2rHNIqQ44RcRiR5zLPLM1Z3V+x2d2zWW+JI8MmLj3n9+jX7oQ2pxDgKYoUZaLuK3W6N9Jr7u1/+2j09yRFa8IuvX/PlV+/onKTvDE3f4gfHq+Y9F6dPSXTK9fU1WiiKSRHWKEPLgCeKI4pRRllVGGcpooxRkaOTMFDo2h4lNdZYILgmJ7M5SMEiixEqsLEXJ2MkPVEcI3WMSjKM90RJjpKe0/MzTk5OjulVg3URh8MWay1FnhLFMVGcE2djuiFck5TWGNvTVJY0C/ukkB3eGeJYUpVbbu8Ui8UUFeXM56cI4blfdeBVSCAqGfCHOgr3bR66oaPpA9/fGk/fGfJIsN8HPFwkY7xzLE5m3G/3bDY188UF+7qh9wckCdW+pK8tyUSF85gZaPuet7dv2dpbPlpOMN2A6SrOFguWF+dsDyXX797SVA3zccG2Hri8fILNJ4zHE3are9q+I84LbKyRkec3/8nf5w/+7Z/ydn3DPJ6QiZhNc8DEMZ8+f0FsIDGeZ+enVGZDEidYLbhZrZEqI87GrLcHstEIlcZBTJMxURREwt4YhIQkjkgjjWJA+AFjOqryQNN1eKlBS6qypukq0KGwWuiAT7ODYzKa0A0VbVOjNPjBoKwiTQr6zjN0nq7vidMIaz1l2VC1DYuTBZPJlP1gj1i2MOiRUhNFSUiDNiVRolEiwlmLsxYlFd1QUx1K1LOY1g509UA8yVjv9vStYZyPGefTUAraDaRRwdl8yWG1R3hJO7R0tqFvYvpmIB3lRHHMaFKw2+6weISSbPY7JtMZs9mc/WGPExatFHEk2WxXLE9Omc7nvH1zhRaKyXiKkB1C/u2c6AIXwulBRUZ4hzwKpNKHQk//8EhxvIcTH5794GAPRuIPrvXw94Mw+eG+74NAfuRbK/n4uA86Bx/0CMFRRP/QjfcgpjtreUDmPvQvBtTMBzf7X+7aOvKuj1xwcRRNH96eNx+e9/B45z1YG4gU3ypX7fqawTREagRSICTH7pyeuqkQHpJ4RKQTetsdGekRSmmsDYJpFMUIIYkihYjDoEBH6hFlGscJ4oFEoSKiyBM6Ofzj/xkTOvC8CJ+r1uH5zis8DqkkjjC8E4jHe2DvQWuBtaGUGkS4FxYCqYOJUhJIBh+GIAFPFAT0oxPdB763l444Tnjy5Ak//OEP2W22/OIXvwhrc2NJk4R+sMDwKCQ/FI8+JNAfPnfnHP1gSJKYJAn8+CRJWCwCb302mz3iat6/f8cwDOz3wYk+GhXc3t4yGimKPGDpBttyoqeUZcU5BVmWo9QDPgUeEEAPgrVzxw5BPMYGAH/Avh7NrDbcs4S1U9DhfGAePyYa3MPn8i2DIO64PwsBUh8HNw8HTOjU5BGPFOgVxvznCYD/3+1vLKJ3pkZpG1hrSuB9gidBqRlSGNSx2VRJebTjxwjbBB6oPsWbmlhqtJZHV8kYvEWKe6SocH7AuCSw1PEIn/CAhulNB2IIkHcPCEVvDMb3JDpCkuFchfUtCHDCIChxVHR2j7U5keyJpESisV4CEVp6hM8QMmdwLc4IIg1COpwJKBWQDM7iXIISybGMdAgnFR/R2/4I1g+lp+EkKB9Zvzww0QGJRGuLd3FA25gW7ywQhVJVNghn8OIcIadHsV+hVBGKGpSAuA9uQp0hKCnLDV1ToeMEMwykI4XMepTs8XbAeYkWRcC3OHmM+ygk8XFqlmIcKAVEVUDNHFEvxj6UZAqSNEYpSdsewglJBleyVJrOOMp+Q+t37On4/Pw3uTF/QuPeYkWFpEDLiMFtEVikyJEqQZKEKIV0RKIJ8Qyf0A8weEAGTpJjgxMdclySLT+lr3O2m5Lf/8N/yz/+vQVzccEk+4RITI+lqeHzxkucs0G4FRpEjnf2eMAGrIaUGkmElxbcMV7kFQ9IJUEQQIUMzEHvJI5QzhCmpA8ntAeB3SG8PD4vTGSVyI/RGMNDEYj3YTgSzo5BzA+ip8ELiyQOlxxvj48J7tHgZH8AonTgu8dgz99mu9seSCLJPJ/TNhV0juurO1qzQ+LIU810fIoWAuE08lhkOvQDdtCkUUpb1WSxI4tjGATKp5zMEj561tGYluvNNdb3KO0Yuhq8w/QOmSj2mxJjCTe4o5x66NkfKmKdYKyg7z3DYIh0AYNnf1hjIkPZlczyKbPJGGkV91crhiShGKWY2qFVSB8MQ83BbKiMR6gIes/gwiRURxLrWjbrd8guxljPblcSRSW92dGVJUk0xnUC6RsmkxyhggOyag6kWUxTNZRVKLtzpkdKj7eS+Wwc9gkauqqnqXu0TIl0xHg0Yb3ZctgfyLIpnSwoqxohBdY5VusNXgykI4mWgqpuMLUm1UVgwmrNeDzFdCkvnn6Pv/d3/iFKZCHaF5JdxwWKf9xfH7awmHpAHYWL18PF8P3796zXa87Pzzk5OQnYkd2OQ3nA2FAoZoyhrPaU7Y44keSjBOEcq7tbtrsV5xcXzPJT4ij7gImxjqZtwuRfq+PiAby12L4HATo+ctLJiJTADjVK5BS5J4kKuqEB25DFMcvlmEO5po8VxgxsVrccmpKqq9ARxEmEd5a6KsnjGSqKkFpTHXa4GPI4J9IZ48xwKA+0h46h3GOMo64rWgyz2ZhRnuFExGon6SuDUhHjYkbThIIZ6x112eBcxHSyQPmURI+IZUqsU+bTU27bK7q7W7SIOT1dAi1dVfLpi09pyp5Y5ozzOWmahsihioilpi0r3t18QzvsfuljezANXV8x0dPjYlAzDBwLeyR1VaOjBO8FbV8zmRYYY7CuZ74IIp/3Fh0JZrMxWguGoWE8zlmv7/DOkKYxeEccaZ4/f8a7t29YzGdcX1/x/n1FnmnwljzP6IcKIQdOlqcI4Xjx0VPmswnOWEbjMa9evURKi7Utn332grI6kGUxX331cz799DO6tuVw2AOSy4snfOc7n1PXFbc3t4zHBa9ffoPpej779FOsHZjNpnRtRzFOOXnyjCRR/Nqv/ZCffvnnfPXVnu985zO++uob0jTh5TfvGY16pLLc3d+g9EDT7nF+TlXvabsaHQnu7lqyLOfNmzfc3d5xv7qn7WouxJSr+w039++IRMz69sCnL76LihXL+ZhdfeDNH79BeslyseQHP/gBUiiSPGGz3aC0pCgy7u5uMMbQdB0iichGCaPlBZvyivnJjFdvX2LwOCFQUc7n3/kBw2AoDzV3d2v2+5okzbl88pSvv3rF/WpNkRecXyz55JNPePnyG0bjgqqtadoST4K1gUE+m00Cj1lJDoeaopjwyYsL9rstxhhWq3uiSJEkMWdnp2y2K4QQFEVwn+wPW66uXzGZzPE2dNdstvccDhv6oUIqEcosk4JRMaKqDlR1DT7g0B6Kjfb7A4dDxcnJKYvFCUor0kxg/YHRaIwxEohJe8/+sGewFdNZQde13N5d402KICNOYqSoybKAWdhu1ugoZjyeUlUlURTz4sXH3N1cs1yckecFh0PJ4VDinGO/3yGkYzIdESc5bVvT9y37w448z3j69AknJ6doFR9dzimz2QzvHFVVcnZ2ys9//jNmsxlSSqqqRAjJF198wXe+813+8+Hm/95NoGjbBh1F/Omf/gVPLs8pJmlIldWe588m1G0NNn/sDDo9nWP7PXme8ub1FbOl56c/WfHJx58yKjRZWtC3ljwNKdAsTSmbNpy3j9ePu/sNF8/Omc1mgUccx5jB8ubqhhfPP+eLL97TNj3T6QgdBSdQEmeBHeoc6+09v/07v8e/+1/+V04vxsRxhGolOIfpB2IdUdUVh6EiS3Pms1O61lIUCevNjqE3/OQvrvjudz7j4jLi5eufopRmsdC8ebNCxYbTszHf/f4LvvzyF6Rpxtu3PhgO7Jo8z0myGOMdcZpwujyl2u9pzcD7+2v2Xcm23ONkz3w+5uuvf8HHHz2lHQZOFwuu3t8xH0dcXJxQVzVmMIyLjF/59BP+8e/+15zNTx6LsLIYmrZkPMpxAloz0PUNMTmOiJ9+84Z/9/t/zv/5//TfMdQ9GMleVxjv6a3l62/e0LYtfd8znhQUk1HgfEcxaZJS1i1nZ2dHDnxNWTVkxQlZPibSmjQNQ5y2a4miFGJJVTdUdUPbNaRpxqHcstmuuV+v+MXLN8hCIZVnPpviyLlbed693vP555+y2TUk6ZhPvzPmyy+/IB0piHr6uiPLMwbbo2PFeDLj+uqKp8+ecvX+DalKMb2gbRyTacput8W6lpPTBd2wO3auOJI0xyGZzKZ0PcRJwS9+ccdmd8vwzR0vPj2lKBRdrzHVQJpFKN2D6ME7los5XbshjT1JJKk7jzDgRTAbjYoU13u0ECSx4ulH57x6+zVtlzAYy9XtFZPpBcV0Qp5FVOWWpm7oup50JBl8xzfvvuLufsf5yRKlNefnp9ze3eKAOMnZbA7IXrK9/+VToqXfc/Vywxe/eMeuFHgVM82nSFGxr0t6b/jFz1+Ga3ofhGfnDB5HXqSIWDK4nul8yu3NPSoSID3GBOOVkjFdO9CUDWmc0PcNQnhOThZYHL3tGUyLEuHeeLYcY3A4obl8+gzjFEmU0DV79rs1cZxQ1y1NeyBOgtFrNJqgZEpRFJw/ec58MaXtNpxenAGC7fotKpbEcUrX9GRFhBIJk1GK8AltVbPBouIBszLMl2POz5/SdhVmGIjTnEil+LRFyzDgirSkrWvyKKGpOxQx3kk2mwPOwvJ8Qp6ldLZhd7ehLju+8/n3MRgG29O2jixR6OWU8STi9MmS9W5DN3QoESGlI0s13fqANz1npwuc7an3G4TteXK25Pbujmh6yt5pvrrbMhplRMkIIRN2gyNloBt6SGP+zn/92/w//+d/xc3+wPnijLpq2F0f+Omrn3J5fsb4fEYxyzmst7TGYFBMTy/J8hm7fYPKBF6nZEWG8p5EaGQSEYsE64+iqTdEIiIiJKe6psb0LU3bYJRmv91xvbqhNW1gBkuPlhKpNHXT0bYrnr14xtX1VbiXPRYgd70jiSdYEwqevQwJU+8EfWdYrw+8+OjJEVUR0Q09QiiqqibPC7yDoTc0dc3p8hxvOrwOaI+A8kuJ8pTWGYZuYDoaM1jPpFgQuQhJwr7esV4dcEuYFWPmWU6epJRtzftNifAOawaaOpQvp3HGdDqhH2q0UhTJiPVmw+eff0Y91Gy2O2QEQy/YlxUWx0fPP+biyTnrmxV9N6DNgPzrkcn/5Wu39wjnHvv+gsHxSDX3D8LfEd0qj6jaD8/mkan+6PzmW4Ioj05zjsSLhx/zoE08ljCKIwL3eL8YxHOCK1fCB8x5uI5b+18eHnzAyDwI4v7xLX9wBn9INQsCQAB41M6klBgzPDq3PxiwHHVVcnN3xTgTxNEciaJpevqhxDpDniRoHYRs68yxnydw0IUUaKmP78kfeefhi3TuWIzqOQruQdtTSqN10HoecSc+INc+iNwWa/1RDPdHVIvCORt0KCExx4LThz95XvCgc1pn6AfQxCAF0vtHWk4wJcmjCx0ehHRxLCDlaHrlKPD/6Ec/YrEIab/V3T11XbMvax5Y7MaYRwTctzn4jwghz5GlPmO5XDIajciyDIDJZMJms+H8/Pwx+bbdbtFaIWRBHCf0fRDnl8slVQN5XvDs2XOiKJSOE4VhkFJBo3M+4JwezFdC+COe2odIBkFsfxhAfHivH0x67uH4ETIw5YU7/r5Hg6x3ISMhZHC1CwVKIP2DE/8BewQIF0pK/5r9HP53iOjT6JLG7UBaIpGgdELvgqpvfYuXA5nOcM7TGxcigkRoWSLFCC/tkXfjqbsNXipilRGpCOPSI3rDh/ZpJJFOkV7hRE81tDg6xFFARCi0sCjl0UohRY/3MbZPESisq/ByRaQXeJNRmhUKg6dASAOMsD5FqwThHbDHK4eXBYYYYSTOCpzXWN9iSYjkklh5en+HtRJJihQxWjicOGIRcHh6EpkEYdZ5OJakemdwPkRVvIuxrgviulfhgBYK5RcgeqRKAXN0fNsjhkSCMHgdSuHs4HHGMbQD0gqc6amGkniak+hQdJLpBGmP8RPvkSKmNQcQHscKwhz2yEVu0CSPzmtrQ5QLZ1Ea8nwUoh2uQgiB1jMkEfgOrQRxmiMizeBbonRExTuMOIAbofUoiNV+ehSFI5wzCCK0zHAIEBpja8Aj5IBWGUkyxtme3rZ4l6AST3FyoN9O6buC9f0dv/+H/4Z/8t/8X0jFAulnDHYd3OHHs48SOiysAYEOVwP/0ICsUMepk0AhH5p4jwMpT7ioSMLA5MGFLtCEdugwYAhFBg9RKHs8wYWyWo4XKPlY/JEG1y8GjwjTRhGFk6D3IbqDOHLBjpGpY3zrIW4jCe9F4MJ0Fo/38d/0UP4rN2MMDILJeEGeLimrFi8Es3EGzlAkMc4ObHcbTk8nCFPTiwlOW7SqGOqGmAzfWaIsQ+iIq9V7VKZpqNiUB5wRjLI5elAUOuPkZAk+pRhN2Lgdh3pPmmqkHBC+Zn+4YTyeEMU5t5s3R/E5JYrnIBq0rhgVBW1zoMg0xWzCoYJWNGRxztCVHNw1onEYW9J3LVIkJGkMOkblU7SPKXyP7g7keoqSUyaxou0PWCqieMKhami6njxX1K5BG0uqJHkh8U5h64GIjPN8ycbfsj8cED7i7PwJi+mCoenJyDjLT+iHipvVWywCowxV1+OtIk9H1H7HtqtxRqOEpjpUCNEzziacTBdstx2oju2h5FKdcKIuiUY5603Ld5/9EOUSlNBI/eBWkAii4/7zgL0KPgfr/JG9HPZ1Y+zRgf6W66sbkjRhMp6RpAXImKrp6cwOhaVxNVW7o3VrOrNjXwqms8mxzVry/uoVgy0R54I8mxJHGZGOEcdFoLEdTdMFxJRSAXslJc55mq4FL1AyIo4iojgm9TnOW7TSCKmQJ5q2r7hZrRk6RxxPac2KdzcvcdqRJAlFkqJ6QV13aJWyuHxKO1iG3pFqR1mtOIy2zCcpeV4wKTLs0BKpKa1yvN/dcDKe421CLCcc6h1tF+J83d6ymJ9xcX6CsT2x6jHDgBQ909EpZgCFxntHb2tmiylXr2FWzIhVzCzLWK8OFElGkkbc3N5zenqGjgPXve1bdKqoTMX9+zvKzTXqb+F4ETLcXHtvqJsDuo9IkzFeCqq6BO+JE0XTlOy2LaPRlNOTU96/f0NgB2qSNGG/2yGkZbW+5fbWMgw9k/EEYzq8s3z+2SfsdlvevH5J3/cURc7Z2Qnv37/nUG5ZLiYoZUniGCkjHAPnp5e0TcfXX/+Cs7NL3rx5TRwrur5EKcdgWrwf+PnPv2S32zEZj1lv1vz9v/8P+IM/+A/kec4w9NR1STHK2e23TGcTXn79DT/94i+YT6bIOAqc3b7D+Z7rmzf8vb//d3n1JiGKFVGsKEYZddXw8ScXmMGTnp9yv7qj6xSr9Y6PPzGMRimbzQDecXk55uR0wXg8CvvbOEVEPZ0tqZo93lmytCAdJ4gYOtdyv+s4HNaU1YGT5Tn7tuLN1VsWs0Xgeo9zniVPSVSEoydNY6SGFouMPK/ffkPbN+yrHVVzAC1wUnA2n2Nsi7GGNMsp8glurtEqZrFYcHuz4vLynLYZ2B/2RFHE/nDg7MkpcSI5lC2z2QghoK5rkiRwxJMkwxpPmhRcXD7h/rgYj+OYtmuIE818MaWs1sG84Dv6bsDvKvK8IEszqnJAa3F0o7ekmabtGu7uduT5lK5tQViMGcCHtU2WZeRZ/shvvL+/J4pCqVMUdRhbYZ0MeCIPk2mGkME5473ncKhQqkXYjFGhkRLqpsJ5z7t3V5RlzXQ2D273vme/3ZNGEdPpnDwv+PLLL6nrhk8++YTT0yVCWG7vrug6hZSeL3/2U7I8o+1CmdNut+Py8gm77Z6yrLm8mPPxxx/zn/7TT7i9u6Wu9kyn08DJHCrmixlRFHN/f88w9ME1/Le5dncKm/dMTjS/+Nrx5n3FqVWoxDKaCNq65buffcZmc09zaIlTQd/XnJ5Nef3u5+z3FUKmnJ0XlNUV/dAx7CSmaRhdSrbrFekkoalDj0ycxKzuduyrHmNvef7RBdvDgfGsQLQNSaFZb7ecnxRHdN5AlmriKKYuoWk9UdwzniW8uf0pp88KNps7prMJWie0bYdCUeQFMpeYHnZ3A4fdlo8+PceLHmthu6kZjwSTqSIrJFoLuq5FRynf+8EZF0+f8rOf/YztYcWhLFlvC8pasl31SGHQEbSD5e3NPSqSzLIpyIjRImU1bCnritXmHodlfbilbCpuru+IYsV0ugR7GzirkWIxnvH9H/yQf/A7v0fhImbRCNN7bKz5f/+rf80//oe/h3egdYQTmrLuWUQe29To0Qm//5Ofcu93DFIyTUcIZflPX/+MahiYjxZEUcYyXxLHEZNZxqvrr9jttkyLJWkRsa8q9HZPelHw7PnHoQcjGYdkpLGPN8Z+MDgpMN6iIhicYVQkNE2FkjC4gcX5CadPTrha3eIOPW1Z8PHHH7G6WjHLZ/z7f/uS/+af/iblbs/3fvgxxjzhUK0YjMXSUxQF0+UpRVFQlx1NM5AlY6QMJWRK5qw2wXU+m434xVev+OSTM27uBoROKIoRZvC8v7lH3+05XV6yWh341d/8DtvdDYiG1fotJyenvHj6gjw9pcgLfvzl/8Zwu0fJiPPTJaM8Z73ZkEU9yiZUvaapg5krSxKevbjkj/79nyGenjI9L7BDQl9KUp+BaxB9Q5KMGWURbhjwXgbU13ZPOq44f7rk5MmYSMA3r15Sdy0guLmuMWbDfD4l9QmjfP5LH9tXu1vuDjW1ARXPKcsWrSWXJ6f0ZYfWCW9eXVOMEpwNMfy6rhESsihjmhdsDg3GDMxnYw6bkNiu6wqPRREFdKIQRHFGFEcMtqMbOnSs6fuONI3RSjC0bbjXkBBnCeP5jL6XTCczNveeXVmSrNfc3dzSDzWTSUrbNoxHE4QQjMdj0p1lMl+QdhLTH0iLnHyckyUJ08mS7f6eLC7wJiJSMW1tGI+mzE8mzE4uefnqNbvNntl8RD7NaNouDF8jTdd14d408phhQOsYbyz1oWQ+PaEqG4bBMxlPGE8mICz7dYlxPTITDLJG556IUHSXZWGgV6QpWZpwlp6zuitBClb39wxDi3cDn336Ccv5lJ988QWXZ2eMPhlR7Uuq8kDpNNvWsDclbrUhjyO0dyhreTaJaKo97cjhkxG//o//Lv/qX/87vnz1E8xgiL1iHseoIWJ/6PG5ovKhiPfJi49I8wnjyQnzQVK3A++vb0ApnOkRBOd5Nh4FprQzuN4QK9DOw9AjvUOpgAO9W92z7w2tGfCE1KhKFFIohsHSNAFrdb9Zc/n8gvfv3iGUxFvF4VAjfIwUIJWmKEboWDPYHmnCPtN1/VHHgKbpQIT7kKKYcHt3CxLKfcViEtzq08mE6nAgzxKyJEPGET5SqFjQDANJklPICbvrLYfdPe3QoIVgt93hmi6k9ruGcZqQJ2lYa2cJm3Ib+O5Vx3K+pPdZQE9YSVM2vHr9hpOzE7a7DWboaGqD94aqPnB7d0OWZ2R5gunbDwiIv8UmvMN/sNAGAf2hg806nBRHzUugjriKbz37g4jqP2BYgoBhPzi8/UMB6AfB/IM7Pfz88JgPwrrHBqe4PD7miI6V4oPDXamHUspQ/vlh+/D63zLB820EycNjHhzuUoQUcijy1I+O6Af8ZyjtDMJxcFJbmnZHnsyCCbb3tG0L3pKkyeOaqu8bnLd4/9AlaBAohAr6kNZHdLCSR4d5dBRSg5HjQYAVQqFU4I97Hzjexg5Bi7MBrSwE9H0wWPZ9R9/3JEkSiiydQ+sIIcCJ0BkW0r366ErXR0E4pH6sA/lYNBs0URFz1JQeBg/iSIwIZighYDKd8qMf/Yj/+//4P/Gv//W/xhiDloq6qhBSE8UR1ln6YXgcYDx8d957rAv6UxRFjCdFKG43htPTU4Zh4HA40HUdNzc3nJ2dUVXVY0JdSsFhfwj39DKiqmrAEWfJkWLhiKMcKSp8UP0AMMYxDKGANeho4bU8LtQW+iCeWz6kJB5c5koqcB/QQeH/wj7lkEdj6uPhdWSfB9FDqTjov94dsc3uw/FzxE5b99ebU//GIrqQObFUOL8hURJLj/AOZx1KhiJN5SWSCEcFyqCUwzgDbo1Wnta1aK9ADQzOocQErRqkTTHOM1AjvUTLAi/3WGeQIkIKhZJpmB74INrHOsLikS4OLmXh0WREImcQDi9ivNVEYsooyvBiDT6mdx2GGoFD+gxnBqxrsD5UCUVSBXFbaDxJcM9oTypShKgZTI+XEuO7I1CiA58AMaErfMD6ASEEg7EoJVDKh2ZrKzn2m2NdjxImRBHQeCRSFDivgQjr9jg3gEgRXofplJOBCU+MdQ4zGJTUyDg4RopMY/yKTGmMS4ADnhLrQzmpwBEhwnsVGV6UDE5gvUWrAelT8A9tx2GH1DqmyEaM8lnA30Qxu+E1bXPHOH2K1pJpvuDap0BGGqfIGEr7dXjvJDgXpk29s2ihwHckannEq3ikjxlsSCQoPSWSMUpaBlshCG330p4ivSbKHdnY0bcZXT1mXe756puf8MPit5mqH9D59whhET58d16EJmvnzZEhHk76YYqVAQnOtaHYVaZh+ovD+RrruzARFUkox6XFiw4lEgIaJ3yXwpujmG1xPLRNOxAaceSbe3qkdChCezQ+OZ4QPhSahuLrIK0/TtiO5Qnhgnk8FoV+xL4oUjwGx9+unOzFJx9jG4tvLHYwRFFCkip60ZFmEVIAUpNPxtRDx77egYSubTC+JctnxCZlkhXkY8W+3VM3W2IR0XUHunZPGueM84TldEquJUU+IlIzvBAIJam6A2/fvyHKJG1/QGiPV4bFyZi+NRz2a8b5Ap1qkihldbhnXMwoq+AcjhPJeJ5zd3fPYV0RqxGr3S1+GFjMxsymU5xPybIxeEG9rhkVY5LM0YmE0/lHSDeh9w3p+JTb1Rv27Zrzkye0g2e3azl9ckEWew77LUWas5yeUB06Uj0mz0IHQxQlbA8lZhgo93sSFWPanixJwCn6bmA8nyAQTEZTIueI0GybQ2B7ecXpyRmpWtPUG2IZ43vAecpqTVv1DEnBqDjF1DFn0zM+evqCUVEQJwGBIx4vUYJQiPvBw/AwnbUujP0UYQHy+vVr3rx5jRCKs8nZYymUP45n4yTG0nG3usX5jrar2B+2eKdYzCsm4wVFMSGUDq7JoxvsxKF1w7iYkyQJWZoHzAICa48N3sOAE57BmMC/8+BsRUWYVGstieOI/b5ms97S2hK0QScaJUc0zZ4oGaOiHCENZ4s5Q9MR6Rjfa6azBc4Lmm5Ay4w4HqFig3WG+9U949GEIkvYdwcGU9PLiro/4OQURMDy7OsdKI8VhrqzzIVAx5rDeksc5UyKKYfdCjtA23QUucYLj44kb29esdtvefH8I9IoxpmByXhM1dZ0Q8/pxQlt07Pb7Lm4PCfOBEJZvvjJT+i6nukkpmp/+WLR0+WE9foe04cSX4HAmj6w++OIfjAIbzldzsHueHpEXAyjDHxAfORJxEc/+H5YnHjHq1evWc6nR+e8YnV3S1uXeDvw+Scv+Oqrr3j3+iWLxYKz5RwleibjUHrX1D1KWvqupGtLtpsdVdXx5PKSq/fv+PTTjxhMw253h1KWuqn57d/6Le7u7qiqHW1dsrq74f/43/0zfvGLn5PmMevV/ePC/enlE54+u0SJsADfrNfoSBLnMVGW0pmOH//4T5hMc/I84+uvf86LF5/w9ddfs5hM+Oqr12RZzHe/94Lbu1uGwTKd5bx/f8WTp2dICePxlKJIMbbHY8nymOnylPv1Lc3QUGQRKItSCV+/+YptuWEyLojjEI9urEF6wZur13zvO9/j8jT0nGAcMsuYTpZMx2Nu7+5wpuOP/9OfEWeazja0XUmUKuI0JS1yTs/nWNtS1iVDN5CkBSenl0ync9qmo+064jTmzdv3tO3A23dXDIMlTWOUc0Sx5PwiuN++/vpr3r9v+OijFygVMZ+f8uTJJXGUUpYlu92WJI3pupbJdITzPVEckaaaYpQw9BCnEaPRjJffvCHSGUUxIkkVE5ljXQsiYjweI3xgQsdJwKl0rcHZ4AxKk5w0TVkul8xnC87OT7m/L3j99i8wzlLWFUUxxjnHOMvZ7ivKqiVNM/pBYJuOSVHgsURaMPRD+FlRiMS6YxFhFEXocXR0Iglub+84OTnl+vqa9+/fE8ea8WREMfqYrq/Y77es1yvOz54db8wCz1SrmLOzS969e388zwZHzXa7xruejz56TpIkR/wGjEcjzs5PHlM5f5stTTPyXNP3Nc9fFHz98wM6sXz2vRwhDBLJ/d0VbdfRtC3FaMJu2zKdZCxOFNN5isfi+iCcDcahVBoK/5RmsGEdztFV3bWGJxfPke8t1a6k3iqUyzG1QyFZznI2m/d4J9jvW84vTtju94zPpywXC7744jVpmoXzfhJuYB5StZPxhK7donWCICJJJOV+Cyjmszk4TduUeA9XVzum0whEC4zIi4JxPuf+fsV4NmK7vWXoKlzv0FYgjCJNEv7Fv/gtdtsNL1++4pPPnrLdb8mKhMNmTZrpcEMnBUNvEGiqssbVa9J4jBQp2/uSxSjhk6efgRNcvrjgd3797zKZzsNN+eAQzqFjz/vVW75++Yr//r/95/R9R5ZMEaRUpaFSA7qHQ/WGd+9ekk2L4MYWls3ujnc3r7g7GDZ3X/KPfvcfMV3MaeqKf/P/+v8Ed/xsTN+s+fTTz/ned74Xbl55iOeHdUCkNV3XYm13RI6ExKNW4R7C+Y6ubui6ju1+x1/87Gdcr++Q2pNlEU+fnWH9wOGwI81S9ruKLBPEUcJ3v/M5fd0inEaLDGN7kkjT9RWn59Ojmy2g6Pb7NePRmF2/ZZzHFEVOVW85PZ3x8ScLVpsrskySpIqua2iqDmdiRNRT1u+ZzASLRY6QI8qyp68Gbq+uaEeWrmrDcV5EbNYli8WCly9fk+cZ89mUk+UJf/Qfv6RrDTIOBXTjSej8+M2/813yIuN6c8t4MuZQWZZnI/Z3JaIbmM9iNps9bV3z7OlzvEyIb24xvkM4xe39mmeXl6g4YrtvOTuZkmcx798dmBSS2fmM199c/9LH9s9uW84WT4i2V8zTKduffsV20zCdvmA6XWJ9YJ8fqpYoirBO4XBgPaY3SCtZ5ie4gyeJEp7MLygPJabuMH2P8gYZx6hJTuM9kY6wxrBZH0gShVChyySJIspdF1IMMkPqMW07EEcpzrZst/c4YLsvKcuak2XKuBDkSUwRO3oRobyAfkOzUYzGS97t91TNQDbKUHZEno9oTY31iqvrLXke03Y1+fQS63rGI80Pv/c5+82eoa9xrkEl4XeuO0dZG7LE4IXBSwUqou07enrSPOb+/pY8j3n27CnL5ZKr63dEcc5sPkUpz8nZhPFU45xlvVoxKEueKUY64u76jtNPXxBNPc733O92VIeGeRJxuljw/uo9Hz37iPF0jnGW7X7HIAYGO6CkxLQtGIMxPU3bIr3nJ/cdcRIzET2i2xMlGf/V7/0ar16/w+HYVx3nJ0tiGdN2A50xnH30MePJhGw8CmkNnYKSaKFROiLRmm4w2EjhhEIKzbjIj0WpBju01IcKpT3KQyZThu2G1X7NoCRIibQK5R2JSBC9CAlDpdBS0ewODGnOyWjBfn/ADj1KOoxpQEX4GETiQTqUi4kQCD9Q73YURcF2vyNOFG1bB0SVFzSVCWY9YfG1hbbHqY7l8oR8PKGsarZtA0KiBfhuoC0b0lHKIC277Q6tJaPJGLSndg1936IlTNIZo/EEJRKccURDRrdu0bFiO9zx/DufYbynqVtEEnN/fUeWFXz25HNevv2Kti0RWhILwWG3IlmeUswL6qECpR8a1n75zbvHG7MP/HB3NGWCcEe3thRHQ+a3tw+Obv+t1wjInIeXdY/i+gOGBILL/OHxDwL6XxLRvfvAj/Yf3L9e+Ef8xhFVTXBBP4j2H3SK4GT/gAr51i99fN+BPx3MtaCOKA2lwvXL2mAiFMIC5i/1aqlI4FxL3W4AQSRy4lgjiIi0xDowQ03XtSR5Rte1x+cG/cU5R5JEJEmClPLoDA9DESkkSuoj+UIc8SLBgAYC73gU9vXx+tr3PVmWHZ3zwfAUCr/lUSz3x9eXoS/s6Lq2dmAYLJCgVEDhueP35H14T/aYRjCPzv3w/oJ26I8ueI7fmaIoCj797DMuLi64vb1FHUXxQ92ErhofRHcdhXWncKCPQ4sgyCvSNOXJk9DVUxQF6/Wam5sbnj59ym63Y7/fs9vt8N7TNA/p8hF9H7Ay88Wc8lAHsV6D9zG73Y7xVBKp4UiEAIiOvlZ/dOy7R7TQ0RuOO37W1of9Cz4gXJx8KNcNiocUIWkppUQ4h3uIVBz3v4dUvCV0DKoHrc6Joxb3ASnkvH3EHv2Xtr+xiN70HTryATUgOqxLSI5ObuNanBvT6jZEMb1AUGPsHoUlUgmR0jRmHUQseYkTmt4dcAgGWxKrOYg6TONkEqL/pkKLOoiYXhApResUOIkRDuEXKDnF+D0aRyolg5MYn+Hdk+N0JkzRe6OJZY6QFd2wx8kEa1xAZHh9nMYDcoMzGiEKIpEgRIwV97T2JZFMEcLg7cBgJEkyRrgIZyVKZSipcNZi3B5HGUpTZYIxPYmagW8Dgx0ZOOOyRhBhXNhhrDdI4Wj7Lc6WaO1AOKSYhs/YKrAxwsWoSBDH8XFaEk6aSgtc3hDFMU6Gi7kgRyCxbofzPYocJwaENDhvwDdEESjZ4kyB8OoR5+II0Zo4joijsKvEKkc4xyBuqQaH71P6RrIYP6ftfhXvG3wWc+CWwZZ4L8PJR4AWacD+eEs7rI5Cs0H4HK3nIDOsL/G+w/iCYajQguCU92/I1AVeatSoIW1i8npCV7f8/OVPKArNyXe/z2r4fQxrIpUgRYb1A1oEh40UOcprvOxDegIAd7yiHKdPmODe8QAS58JU1/sWLxo8A4+t0z7EQjwf4jwQhT8hOwrHkxMifuQ1hQLd40XTB2SLfyg98OEkEYxDAiWi8NaOxaPe25Cm8BasAFmAN3jxy4tsAFXTkYiIKA/laoMdiLxms9/w/NkzmqbEWEc2yrnb3mFcS5IpBjMw2I4ksqSjOPDFo5xMJZRDC4MjcZJZVjBdzAPDtt+CV7S7A9gd96st08WC5cmc1tW8ef+Kze6e+WlBWqR0Q0esM4S1uKElSguW4ynfvP052IxI5RRphOstWiUor+mtJc0zhsrhOkMSJUzHM/pOkMUFq/s1QwsGxclyjokzYuJwI9IfqMsW2w/43pHEKUmmsB1oZ+n2FWKQTGYnjEZT7LDCmIF62BElmkk8ZXco6aqSWZGz3dxgmo4+DlHupu4ptzWT6ZihbYhdimh7hkNDggxuazswSmPG+YIii+m7HmsNdbNnOpmhNOx3Fflozt/5jX/Ii48/Jo6+lS38lgNAHItIvH+4WLnHRVVgkzvu7+958+YNu92Ok5OTgKhpK+q2Cucm0yGFY7fd0duabqi5vr3F2QFvDuy2B2bTU4pixHg8o+saquqAGQxJXFBXNdPpgiIPhWfeKfp+wLqApTEYur7h7u4OpQR5FoZMxoTFUxwnpFnCcrlgUw5sdmvKQ4lCkqYJcRTz0dPPWe9uydMc4hQlghs3SiLWm1UotpGC6XRMXffsdzvm04w0iYCcuoyRHgY8eEvXNxwOe5wxOG3JipTr+3vmJ0/IxhGHZg3KcGh21FXDJ0+eczgckAiSNKHtGxrbsj7cc6jWbHYZTy8uUToisjHtbsvYCXrb05qSQ7PhWXxGnhXstyWLyRTvJXu74912+8sf3MKQFzFN0zAap0ihGY0WdE2I9EkXWKFnZ3OipzlFkfH69Wucc3z26ae8e/ce5w1luaOqKp48fYpWx8+xqhnPCn70g+9zerLg1atv6JoGM7S0bUnfZcxmM2azMWdnc9brNW1XMp0sEELxzctXfPzRR+z371mv7ymKOOAyTMVyOWU8zsmLmHfvX/H06XPSJEepN9zcXqN1xMefvOD91RtefPyC1f0qLPZub0jiGKUUi9GItm14exVKTO/XN+RFjpASLUIxWZolXFye0rYN3sPHnzyna1uc7zk5mSClJM2iMEyUmovLE4SQ5EWCdT3Pnl+y2t4RZbA93JEiKCYpSZzjBk0xmtB3PZ1rUTKidx1lWwVXbxSxrbb0Q0tzKFnO5tyvDF9//TP6puPs8oxsOeLVu6+YLcfc3L0HaZmfzBBK4bzgUO4xBvb7EuEjnj455+zsjKIo+OM//jN0pFguF7x69QrRhxuk+WLGZDrmzbt75vMZRVHQNC1xHFMUY7IsI4lTum5gNl3QNC3j8fiYIIPdbsPyZEFR5Pzwh99HiCCGSglRrDjsa7puIE1CwWA/9GgNdrAkiWY6ndG3cDjUIBx9PxwNE/rxBgU43jRwxAKMEULjrKK3kCaSvreUB8Nm1ZMmMJ1mJNGY9X6NpkULzW67BiHZ77YoFVPkGQhPWZZIqXn27BnOOtabDU3T8Ju/+Zvkec4f/dEfMpmOmU5z0ixiMApjDdPJBPBMpxO0jpjPFzjn6boageSwPyClZjqdkqYxd7d7VqsVQChjto6iyMmyFGOHY0HqL799+ukL3l39Ah0pxhPNk2cQxwIpbWBPak9bt3Rd6FyxA3gnefnNPT/80SX9IKnrEucMh7pDirD+jpOM8WTKerdlv2loK0MSJ0gZcX2z4dd/+I/4/f/1f+NP/+A1v/ePPufm/hWjSUZT7kljTdM5tM6oK0ukCozxWNeT54LpZE4UJXStIc9zbm7u4Jjqe/7RM/b7EhUp4iTi+UfPuH6347/63d/jx3/x59RDyXg85vvfn2OtI8tSVqs1Umhub9cUo4woFlTlnsOmYjGa8eLZZ7xb3QE979695ruff4ZzPdvtlvFsRJLG3F9dh4LxbAxO0jYdi9EF+/uO1khkFEqrLk+ecbiv+ef/7J/TVA1PTi9YZPPgmrIOLcDajtZ3/Phnf4aIBWVb0/Q9tWnZbvcUeQGlYRoX/Mef/IS+rxjLGZ2p6auKqq/IxzlDtWZIHP/mP/wvnC8v8daxL0us70jznL/3mz/i7PScOAqprujojJRSYt3A0dKGMT03dxvyPCOOI4auCTlIIbjf7SjbPTLRvLu/YdfU5CcTch8hdRBW9+WO5cmErm+IYsl//IM/Iy/AOEfdDjz7aMZ6WzFbxDz/eI4UmvWqpKkGJuOCzWrD0BucFQglGOzAeFJwqMrgfO07EJ4szem7hjiCr1/e8b3vnbPd3jKdZlxdfcNstuDt6xVZFiFRnC9OOCjLy1cbJlGMNRJjLEWRM5mMUUrRdQO/+qs/4v5uw/ubV1xcnAaXdLnl3dufc/nkkjjW5KOUst3jZISKJfPFnMl0RrOzbPf3tG3J23cty5OUqq1Y3dV8+vkpq92OfFwQSc3FxTk/+N73ef/mHe/evmO9vyEa/fIGl7L0vHg+J0m3XFyc8fMvv6Lre67v7lgsTjCDpW7CfWCWprRdA4QiTRDs9yXz2Zz9ds/W7FjMF5ydjHnz7iW29wHP6j3egHOCSGjwHV3X4ZxkcTJh6AcirfFe0PeGSIMSEtf3eOG5eX/F+zcvSSJFohSJVCzGc5bLlDiS9IPn6mZP2zUkkWN1d4U6ptD7vsPjOJQ7BJLRJKHra7qhIVeK8TRhsBX7suXVa1hMz8nSlMvTBevdNYMzdIOgbhsQDuMG1pstWTJCqgEHZKMcpSTTacHF+QU/+uGvorVmMhnx9t1r5osJSoVeASVOWN2tGDKHURbhHOM8odoOrK/uOb245NBsWJ5NuNmuSWYz6r4GGcQmPGzXG4a+Z5TnVEMVXNF9S6I1kZDoOKapa3onkSplW3VAh5A1Xd+S5RFSaVSWYLBU5TaUKEcJ0/kilPHpCKEjhNaB7yw0pycnCOtYdQ296VGDxNskdP3ICOksCkhmc/j/svZfz5Jl+ZUm9m1xtMurQ6cshYKoBrvRgzE0u2doY6TR+MR/lOQjaTZs0lpYdwNoAIWqVJWZoa92efRWfNh+IwvkEA2rmmMWERkZEdf9up/je5/1W+tbBLb7mnq7IyIiJN0QU1GJTnCjiz+Ci6XRSmGtJdGK7XobUwVpTu97kiqmxFQa33/ve8Y+0DeQpwVCBnxikUowm1cYo1mt98zns4ixsC66rkOgqVvySc7VzS1/+OgJOstQvaXtOpSO/WJ90zIOhvQoQaea2WIa+e2JRGcaZ0dkrsjLgmxa0uxqdkNH38e+FCEjIWLb7OD1K47PznDB4nAkRcb337/kT//oTzg+OuVqM6KUiKXESrJe3XN69ojZckE3xKHI73MYM3xAtUS0RYguXfkDWiMOPZNIPAiHzrWDeP3gzgZwBy0o6owC780BRwIc3ObWxqF9dHtzSCW7SE3gB0f5DxjQ3xLQAygVdQgOfXgI/0Mhafwn+BDNT+JBCD44qgURLeN95E8/ON2FiM535w/3qMGhlD48Z3dwJ4cPzunoTh8RSjCMG0LwzMpTtJrgrMA5sONA2+2iYaLj4DZXgAUJSiqEjCK2tZAksUPAufh8lY79c1JFDUdIgR1jt4AP/kM/Tj90dH2LEND1DTFU4Oi7Hh88252hKkuSVGHMQHEoOY3vd0Soaq0P740hyzKsGUjTDOcheH8QhxU2HIgKUpFoF3EsKqaIhODwmkVjyKeffsp0OuXy8vLDfXTkuAvswdUvJXHYLg6YGxE4OloQQuDjjz8mz3NWqxXWWl69esX9/T2PHz/m/fv3ZFnG5eUlD91mRVHGAYSM52LX9eRFRpppnK/JixQfDJvtLfOpRquYGI9dnuGg1T4MVh74+w/I2YcTEB5E9PiaxWRBxG4fFLxDQa94wLMQsP4Hk4b7LTH94Zx++PwTQkdXuvcHEoc5ONT/8eOfLKJLvcO5gPUdTkxw3tK4HV4oknCEFNCZNUqkpLL88OBap9iQ4PyADhotn6FIKJVhtCDJEYkFVnivEV5jRU+iFIv0M3p7yxg8aTqNG0etCVLgvCV4DqUBE1IZUPIe44cYUxJznG8QukBSkIkZ0gvy9Aj8HY3dEYJCyQmIHBt6BJbgFZmuCGFgtANCTtEyx1ITpAUX24jLbE6qZyAlTo/wgFxxgkQWSJkziCZyjcQU5wVK5iQyOjWVK1FihsOg5YDUAud6QkiQMkGKkiBBiBxjQxQHE4cPBaBIlKLMC6zpP1xEVvbI+Z4gPYEdOpkh/ATvt4DDmwQXHCpxCDHgrUegkcLED7RQoEICIjKZnXUIJDrN0TrB+Q7jWwpRce3+HXXYkAz/hmbXgc/QSlPmj/Fpjet3CNGTqRlKVHjRHJAtAa0zEjECySGmA0FsCEGihAQZGdZSCmwY0WoJIW6IkBZdbcjanGw2wZOy26149fJLPvr8/0yhnrLx73B+QNIepk3qg6boROSVCyLbU4qAFkmc7B1iIg/FnhKNEDHSFoty4gXoQnf4/wWI6ET0XsW4E5rgNaDwDzhz4RFK4XzcBOhgIrYoRN66x8RkBDriNdBI0riIEH7IoRwmxfHitoCCoAhhjB8Av8fRNj35PItRmhyU9/Rmh0oshhaVK+7v1gyMdEOLknBczrCmox0FTbfnYvmIq7dXtKNgupyTZSmlmDCfLxj8iMoV/diS54pxaDF2YOw6RtOy3XqMnxE5bgqtsw8Dhf1uT6VjQah0lm67QSYpR9NTlAo07T3l0ZJxjEO4IqkwzpNnJaeTGVWSkghPu63ZbgeOTzUqhazQCDFQ95a+GaiygFIDlj7yj9uOYGNJ0Wy+5NFxQV+vyXVC8CB9RqYmLGZQd/fUzYbeBMpyQrvfkQpBk2pSAS7ExdSjODu+IIgBPwTGeiAVkkSlzNMSH6DrRoZ6j5CeooyR2mZnGFzAGktyKPCTYcbp8XP+6Oe/OLDf44bqH079OeyxHsbVh791iNB5YLvd8ebNG+7uboHAZFKS5wl1vYmssyQBKem66PAIOJp6Dz6QJBnrzZazs5amacmzgtn0iI1bxRsk79jvd2hV0HUds9mSyWRGluYIoWMkVji0VORFitbw/vINWXYorTycB8Oo8B7KYsJsMkdJaPev2e42cfPhJdN5BuuR+5susnuFIs8LlFI4a5GKiJ4IE/p+5OLsEd56hr7BO0+ZT8l1BU7QO8OzJ5/i9oFxtGRZjrFxo7+rt1RVyb7p4mB3CCzLE9xomVRT9k1L27Vs2w1oR2MbumHP7d0l1g7MZnOs8XgCl+8vaceaoEaGvuPq+i3TcsbQWmZFyWx+xK9e7+Iw73c8ppOSotDsdlu6Lpa2lkWFGWq6buD07IR9fcvV1RWffvJj3r275u2bdzx+fE5RFkymFX3XfWh47/uOJFG09Z4iy8nThFW9pe9qlIC+29O1NcfLJXmmaZsdWa7wPiLCjpZLxtEgRODFi0cYM7JYTOm6GikgTSVSJdTNnt3ekqYpSsHt7RWJzijLhO3G0/Y73rz5nsViTp4XVGVBlqd8/913/Ot//a95+fIldddwcnHG6C2PHz8i3+QgPMZYvLfc3W1YLOc45+j7js0mDpEePz4jyxNub69RSrHbrzg6mrHf1whpWS6PSVJo6oai0FwUp/RjTZLEWHyapUwmFWPnwcPR8SnL5ZT71TVpHgvRbq/vmM9mjG6k2zWMTcd8UbGrNxwtFkyXRzTjnq+//JqkFNxvb+jGBqGg3wwslkcf3NjOBRaLOVqWKKVYbzaM40hepHx+9jGvXr5Ba8HJyZy8SKimJ5RVGbFNsznb7RaA4+MTJpMpIOj7Hik13nu22z1nZyecX5xyd3fNmzdvsc5wfn6GUgXrzX10wirNerXFOU9ZzMiykklVYbzg7m6DMQPWOoRIEZT03Yg0gTzLWB4vGXpD34+07cBqtSJNE4ax5/LqHbPZhKa2BF8wm89JVMbgGqrihDIfuL9foaTj+Ogxdkww4y6WvjuDUAprY8Q2TWPUd7/boqRiHAb2+5q2bbm4uGC73fKb3/wGEEyqCVpLVvdr2m6Ps3HzPww9ZVmhZMLd7Yo8L5BC07YdxuyZTOK/a5qaLIvD2bquD7Fdz3Yb+c95Xn6Iw/6ux+XlO9I0p5oU3N1sOD0vEMKR5Sl5orjfNTgTmBSKfQ1FluGdQi9T7q53IATz+YKuNnjj2e17lkuNVtAPDmcE292OLImlx9c3DakSvHz51/zr/+Hn/Lt//7fcXL/j5OSIwQzkeo5zmzgUbxKG3jGagWFs2G5bZtMyXkdi5OLsAh/g6Gj7gWF6c3tF24+sNoaf/fRTrLE8//iEv/rb/8QwWNJCU9d7tErJiwk3N9fMFxOUSgk+DgabpmG3bajKjEQUTMsjjn2A/oa3715TlSnPnz+lbufcru4YhpHT8zN0UAiboULOzz/7Y16/fYftVjy7uED7hPmkYD6ZsZwuqGTJj3/8OWF05CHFqYAJETtpcBg/cnX/jp3Z8V/+/j/w0dNTkhJQI7959TXPlo/4j19esjMjWVZxcX7M+9s7lDXc3d+y7mryacaikKSq4PWrV9hhBCf40Wcv+Bd/9r/h2bPnSKVI0zwu8wcXW5qm0VTjPRB54NNUxuu8D0yKEoGnbnaoDJpdw6rd0oWe27rheApSG5puR5qmHB8f8+7de6oq5ews4G0BMiB9z/2qo62nLOfHSOEoszm7leXtqw2TMqXu9hE1IiqOj08ZTc+u7sjLOUprklQDDSoE3KjQFOR5wn/3L5/xy7/7JWnq6Lua45MjklQxnU0psgThJe2+5v56TaZLnjx6xnevvmEYBtJMkaSaoTf85V/+irPljHEQnJ6eUJU5t1fXFLnm4ug5bddxd39P03vmywlJmqDTFOcC335zzaOzjKwomc5K5nOLc4GiSSiKnEynuNLz8YsXmG7g5PiIy7fvkMLz9Mk5e7PF/Lfvw///HqeLM7CSIivJspxnLx5xeXNDbztkGiiyjKzRDIMlKxU6y/HeUzcNphvphoE8L2m7nv2uoWlGnj19ymx2zN31TexvEildaymSjKTIcKJHiYgDkUGw2dZomTCbzOn7gUxLFAbTb+nqkZurK3ADCjhZzjjKS87PTvjR55+BCHzx9dfc3b/l5GLLYEakltzcvMZ6w3RS4sae3mwpQkqVT3AI5suCsko4PTsiSSV1vadudvSt43RxTlWmUaTzsN3tI95We9JUoBOw1kT3qdKcnZ8zdpZqOmE+P2K928SyvDJjMp+i0wxBwFlJvevY3rW0W0NXN5RlxvvdlrTIEKMjbBrmWQHLOS9v3yHDGAX0cs6+2WF6w2a9oSxLTpbH1CMHhFrKpKpIpGTse5LUErSIhZ5S0w09bdejE0UQCUrEnp8s0UymeSy7zgqyLMfLyG0WSIKPqSqJ5Gi5pN5uqKqCrttjh4Z6PWB2grV3pAecohOeIAV5PqU04MIK5wQEhSQKYd57jLXR6aqjcNn3lnSaMI6G7XZLlmVMp9MoVYVAO/RMipKiLNitG4xw2NFSVgkSR29bJpMSUw9keYISAUlgWuW0jcM6z65rmJ0tMXcw+pGhdxjXM/TR3CYQh3R8gpaKIi8QSqG8ZgwO27dxH6hy8qpg8AavPVpqgh3w3pFmGpVpcpWxWt9ivCGvKpIso5wVNPWOr77/ij/64z9kP2zph5bgIvhkt6tResVyecxot9jx91u7hRLxPQwuqs8BhBQfRD44oCuIruRYhvjDfZwP4uCW/cFJHN3f4Lz94CzWWoGQ+IOoHh7wsILYcYiMiX134JhDpMIAD1bnKKzH9eTwzA736+C9+2DKeigFfRDlPzjRhYAgDy74H5zHHDzHSnB4fvKAyYkGzAME5oD5kAdB3eMZGEzEKFtXIVWGd0m8t7LmwP2OxoU0jXuwB/E6YmPiNCUEkNLiveBgIueBPe69Oxi5EsCz39ckiY5iureMY88wdGRZ9sF13/cdXd+RJAlCRGTmVCpAHb7WQyIgMB5EdGvNgcEeMc7GxLJxlDoMIvjw+srDe/2hEPbwuPFtikmE4+Nj/uIv/oI3b94QXNyzvr+O9zLJwWyXJElMYmpN0zRImeKc4+jo6AMDPYRoNhnHkTzPGYaBvu9pmoamaajr+jCojgJ/nqdAoO9rjNFYp6mmsFxWLJYli6VEhCGe8/4h/cAHrrsQP3Da/QcUywFTdNAslNQ/8NyRB5Pgg3s8Dn3kgVMbQjgMcx5wyYeBTgixt+5BXjvw1QUaIczh7P7h53/s+Ccrb+rQqDtai0WjZE6mMwQZg3MIJaiSR+B7JALLBELAifiiSpGRqFMGL+iDRYk6OnytQIQc6z2JOMaL90jVM7gSy7ckqkD7BOv7yFWnwoyCIBsKXTGajhQJfsYgIk9X+Oh6YvR4N2JwaDVnMCv6ITB4g7OCKtU4b2NwICiktAiR0ps+llwqTQiG4BPsqEB7REhRlIwu4OlinaSymNDh3UCuTxEsMW5HohRSemywWL9GS3B+iG9WyAmkGB9wvkMFTaYynJdYDEI7ZMgRLEFYtHJIURFU+SEu4507fMwIkOCrlmrRI6TD+zuG3uOcRogQm4iVwZkB6VV0uMuRQIrwmtEb3JgiRRbLFnVkU3sfp6BSRoG3MVeUacVC/hnXzf9M391HF12/Z1Nfslz8OYYW7yVaL9A6xYcW71uUTOP7IxVKplGwVhk+BBKVHdBAGVrFYgDrBT4MSDk7fMCOCATJZIS6p6xmWFNgR0nTOPxQMEv/mHX/FSZsSVTkFAenUKI88Ptbgo+uF0GGEnlcLEQsNVMqxdPjgyCEFEm8OfahjamE0GF9A/LAbTqI3kLkcQiDA1QcSKgsOvToCMJCiGWjLhhii4bH40BIlEgIWISIUZ84rU2QJBhfH9z8Eh+6QzRFAOOhbDSWVf4+R5WXEDwyVYgkUCQJtm5JkIxDQ5IXtGMDiUIkEulBO0lKwtANVGmBErCYT7hbXTKZT1nMj7D3knl1TFolOG35zfdfcTI/xoiM66tLpAg8ffwIoXJW6y0axcXxGZ989IK0jNPddteRLjJSnXL7/o5EZkzmC37y6Y9Zb28Z2i0JKZNZwXa74Wx5zng90m5r8ipnMjvGjS1npzPKwnO7W9OGDVkqydOEzd4yKZbcNNdYYzg5PgLhmU7nmEOj/NnxKZdX7zG9YbFcMClKJsWMZtfg/EiqNHmW0ZuBcRhYTGacLY8xTUczDlhjmU09w+go8wn7Np4Du21NoR3TYsJET+jtyOAH4ngc+mFgdANdA9e3W1Se0DWC1gfmacmf/rN/TlkWsR09fKB//QMhXRwWqnBwC4QPBS0BMxpub25Zr9ckScLp6TEXF6c07Y7rm2u2m3Us9MtLXAg4a+jHGhGii7bt2ujUGDr2+yhKlcWEpm7YrlckSRSzt9vtYeGtWcxPKIqKvIg3/tZZjB+4W91QTSryPGW/v2G395RFLCrJswLvoWkN3mm0nvLiyefcFlesNu9x7Z68yNnvblguz5FK07Qtk9mCu9U9IQTKrCBNFMv5HHVywtX79ySJwLuBVGdMygVDPVLXPYvFOV0rqNQULxyr9R2z6QnWCewoIv89ib0XQ9dhEstsEaOnbdeSTXIWRwv27Tpu4pLI3v7Nd1/z7NkL5vMl54/PuHx/Bd6RpIreOZw13N/dkaics+Mldhg5m54gzn/3a3sYWs7Ozljd35HnOYv5KXla0mlLUuXsNnuyPCXLFF9/9Q0CTZoosjRjt9myXW/IsuyAGgrUuz1KKObTKfvdFm8LUi2RwWOGjovzUyZl8YGZWJYFWaHZrDckOhbhZanAWs/jiwt++ctfURQVjx+d88u7W9693fPRx89Y3d9RlXMW8ymr1T1m7Hj65Dlt1/PZZy+4ubmnbbZ0Xc1kOj1w+wbmixlffPlrTk9Pef36NdPplOcfPYMQYh8CntevY9zfOcc4WN69fc9iscB7T55nnJ4eIxW8ffuKoijYbDYcHR0xnUVXyXI5YbdbcXV9hRCSp0+fM4yRzzedVizmc6pqhi1gMllwcfGIut7w9v2ONM+pipKuLambHTd319hxoMhyrtc32HHk+dEzjpdLXr75jjdX31NOcnSqUalCKonSmn4YMWbDdLbk5OQiri9e07RdHLRlsTTZ2IGiSjk9P0aphLzMODk5JUkSzs8vmM3mvH9/SZZlTKoJVTVlt9vH4q88ZxgHurYmSSBYT13vY+zU+cg9b2u6dmA6nWJNz3bTsFgccXK0pKwKJpMJdTsymoG+7zDGQsiZlCXL5Qnv3r/GW8HxkcI5Q9d2SKkoihylJX0f+b1llWK9Is8XnJ+/YBwt+9pRlscslo7VugNyVFIxXwTW9y3WDSSJxBiHkoK6bjg5OcPawDBYnDVcXl4CHPjkKfv9jq7rODo6omnizcAwjGy3e/q+YT4/YjKZRT58krPbNTRNx+nJeXTyJR6tFYvFGd9+900U8oNlv2spiwopPU3T0jQN06n/BzfMv8vRdnsqWfDq5TVSpUxnOev1mqvLkc8+esSjiylt49jva/KTOMg3xpImEz777Cf87d/8LV29pigLUgWLGSih2O5brq/XfPrxpyhxy9u39yxPYDLJuH7fcHYUuFsL/vu/+IivvnkV196LJ/zdL7/jyTPNarVBMSFJMsIBb2hMix1B64zlsmS12iBlgnOBsiwZxo7RGLIs7o9++fdf8fmnz1kujmgax939LY/mS6oK7u93PHp8zuvXK84vTnj37gYlEqyxNIfXOlUTwiD5r3/31zz97BHnF6eM/YC1lu+/f8nzj55RVRO6viMA1gh29zX/23/5p/z5v/xX/M//9t/yo0/+gH/1L/8cbfyH/bYbR4L1VEnK6AbevX7J7XbDj3/+I9qmRojAYCz32z2j6nl5/yXfvvlLZosSqwwuCfzm/Rvy+ZT7VwalHUla8M2b1+w3G3SiUYnG+Z6gBqywvPjREZvbHTpoPv3kOc+fPCVNUpIkQ+uEROt472UMwYNSCUqDtSPeQ5aV5IXh/ft3bOSaICyv374lCEs6zbhrbhGF4M/+4ifcb1dMqgvevbtEKMV2X6N0itCCqqo4Obqg7UZc6Hj67JR+bDl/tIhlfRZWt7c8u/gc6LBVRt+NfPvVCikyfvzTF9yvVtR7Q8Ah5cD9/V1MAb044i//89/xo8+e89GzH/F/+j++YLd7z2Z3xeAs22YLSiG1xrSWfJbz5MkFd+s9HsvjJ2esNzeUZc67d69xTvKLP/kY0zn+9m+/5OTiBcb0KKUp0pK29pycnIPWXK92ZHmKkJClOaubDZNU8OrVhtOzCiUlZ6ePsKPh9lpQlQXvbq7IFhlfffkdn3/yjMvr97y/fBtRKVpRLQqq6nffmx8vTvCDY2gH3r97y0cff4xIFc3QUc0KVFBkW0XbjTTtjtl8ymQyYzIUtF0s+9WZpppN2DcDddvz8vVbnj9/hmATkXojmOCYqoQiKwlDd7gXhrbtsKOja6OIEjx4O9I3A7iEttnTdxuqosQaz+nxEdpKsiRnNjlhs93TdoHF8SnVfIKQjr7tuLy6pq73nJwec3S0RBKRXsujU4QIpJkm4JjPJtHZ2Vv6oSfPFZPZhFevX2F9z2a/YxhhdlzG+95gmExSxk7TdT3ldAZBMpqese8pyznXuyuEhPl8RpYn9H1LolK++PU3zIo5717dUK/3LCYTRKboRwNCcjSrGDd7kiTDecfr2xtM2zCbzqjbngTNJJswKXKqssILxXxiEEGQpQlZmqCEpEFgrcM7wWS2JMkyCmNI2paAIM1yXPCIZE+ZFUyrKUppVJrRG4vUBwa2iyW5owkIH/BCAh6tBFoGXN/TdwPGWYokRaUZs/mcUUgGZ2MJ88kZ4vV7utaAjsavfhzwPlIHBA/OYkWaBlQSC0G1VxyfnhyKshuq6QTjLePQUWSKsshotg1t3SBFQl7mpJnCOkNdt0wnE8w4kqUFz55dsFlv2Ww3jNYiU8X8dEJrOtarDVVWMjQNWRadtgKJs5bV/Rols1jWKELE1iUakSik0gzWcHt7i1SSsqjItKTKp6SJjIiQJEcBzo7UO0s+mVBVFVmVErRgvd0yn5+we/N9FOqFJEjP6m5NlpYEL+Lw4fc4pJJRBbDR9S+FREsd+dAEgo+cau8iJz0WkB562PwD5lWiDhjMKFZLvOfgpo2uaqUjO/0Hvrg7iMfy8FgC4SMS5CEFqFAfzFY/JJv9IXH/UPZ4cPgeHuuhFD6WRD4I5A+C7w+aSUTMxP44fzAuBhliN1vgIKZKlIza0wPiNjrxBcE7AobgDdIpjG3JkgVKa8Z2ZOh7rI1amNDJByFf6+TDsCGieiMb3VpzEKIfHNExoW1tLM5smj193+OcYxgatE7o+ppx7BnH/iBg+wO6pafrGsKBt9/3A3keOxgfehvUQRx3zoI4dNLBgRueRtNXInjo4ZPqgJM5vG72gTLyMOD4wDf3WGvZbDZ8/vnn/Ok/+2f81X/5S6Q8DNmaHt+2WGtxzjGZTCjLEiEE+/2esiw5OzsjTVNWqxVSyg8pyvl8zs3NDW3bcn9/z3Q6PRhtFM7F1y5JUtI0IrGsG2m7luXxjCQLKG1RiaJI84hT/C3Uzw9IwwcSw0Egf+ibfHCQK4nSEbkTC3WjiB6Z6R73IJAjD+dRHL5EmkVMGYgAktgpGV/CaIzmcI6JoJDCEQ5c9f/W8U8W0X3wJColIyVTksiycRi/I1cOwQItKqQSaHEMPsGxwYaRwQY8O/AZQQYSnSKZEFnggkyWEDqCFwhV4UlJdYamwRMjI8Z14FI6b5BSE3xgY29QZPTjikw/ImCwwQIDwg5oNcMLgxaGhEtILcFPkH4BEorMMIw2RtNEhnUBgUGLgFAbbNBIUSJFjlYVxu1RxElIohySHhda9v0NqTqiSBQidAQcSeIo1BGjuSX4gEo03nVIGXBeIEgYTI9SFsUx3jd0fozFEjJ+SFq/QYoEJRYY05IpRXJ4Ux8ulngiWqQMZJMGqfc4N6JQpEnGKBWxZCIK5TZEB39E8tzj/ZKcRxgfSEKJForxcHJLdWhHNgPeGaRKIATumi+ZVj9hkfxP7Nw9IXT4MOJp8XKgCa9JdYnWCcat8GFPqmJMRAlzKAJQOF8fuPcZxlsIe7JkiiKW+UEsoBMYRFBomRDkDpXtYV6RtpY8r/DhFK0SusFyXP4x1/xnpHiPROL8Duc6bIBUJ4CL4riKxXYWj/QCS413DiVT/AGbkqgAZBBKrI84CyHKyFWiwYUxDgWERor4PQkknuj0j9NSA2FACBf59dITguKBYyZCQgg21nuI9LBYPUyLRwIGISyEeM2BQIooBMXlJAE0Ify32U3/2PHk7ILXb76nayxm7KOjeDIjSRLW+3uu727pe8NkuqDKJ2ResiyXJF7wXl4zzSs29/dgB86PTsh1SrvryP0MjcL0A+kkIREK047kaYUwcaPc7wZmiwnLyZLRjQxtRyoUuUxoupqMDOEVWqekaYbwmizNGPuGYEdkCIxtDzZhUcXyrV2+QYucXGZoNKMNGAvGBYz1tGNPOSkxvmNXD/QNnJ6cMfY7RteybWpm02OWR6f0fc/9/fqwj0iRYkKeVzg70jVbjB1wticpNIHAer3m6fkjJllBYwRN01KUFc2uYzAey4j1gu/ev6HMc6yA1XaLdx6ZKIIVDN4wdAP5JGXXNHRbBb7i2eMfYTrH8eJT/s2f/+/40U8+/5BS+P9tRI+HJC5wPwxf4qITiLzr3W6P97BcLnn+/BmjGbi+fs/96p6x7+Kk3IwROaTihs17R9+17OsteV6x3d1RFiUAeVYxmy3omg373Zr5Ys5oepqmjQ4XY5hOFxRDTlkWCBkLDRMdCw2rakrX3eNDYL25RSAoq4r1estysWBenSIseOmZzxdUU812+5713Q2LyZKqqMjLiqbr+f7VS4L3nJ+conX8PGuaHVqULBfHKDVyffWW2moyNcGbntPjJbPTM9qtYTKbUjc1z148Z/A77AAhlWQyo+t6zGjIVcl8suDt1Ruub+/wCIZx4HixYL11JEJz8eQ5Qgg2+x1HZ8ex2MoFnj9/hOARbdexy6ecn5zxq7//FSfH5xyfHfPm5TvuL285Wix+52v76uqe2WwWnZ4HTJU7JIKWiyO2u1uSzDGMe26ubzk9fcR0OqeuW9J0w2y2IE0TrHVkWRadbvs9zlnmswmr+1v2+11M2WjJdn2PJGDtgJKx8G91v2K+XKCUYr+vOTk5ZTqZ473n9PQErTXHx0tOTqNw6b0nSeJm7O7ujiyLpTxv371isViSF4qyShj6uPmdTSdst2ueP3/Kq1ev2O9rdrst1hmOjpdsd1uqsqIock5PT/j6629wzvHRRx/Tth3b7Z5nz55wdHSE957vvvsWIR+cL4IsS7i8fBc5nWXJaFr29R4wjKOl71tSlXO8OCXNYD5bYJ0nL3IWizmr1S1X129wYUTKjKoqOD5e8Jv1HavtPV1dc3ZyigsD1hi+ffMtd9sZvelQGTgGZrOKwfVonTKZTLm/3+J9YLfrUHJPonOEkNR1zbNnT1ksZlg3cH9/T5pJXnz89MNNydHyiCA8SZKRpjlZlpPohKOjI3a76MquqoLFcnZgwE9wruP29p7tdntw2Uju7qIDfTpZUNc16/WO+eyIqpwznx8jhKdpelbrFQ9dDMEL+m5E+J7Hj14wjiO3t7esV1uKoiTPC7TWTGdRcN7tG5w3tO0eKTRnZ0+YTpa8fPma7aZhGFxMt+noBhr6kaIocdMKO24PKDLIsgQtxaEMyZOmCVVVYccRnSryPOH+/pbtdsezZ0/p+57NZktVFRwdHzMMA/W+pu8H0mSg60bGcQtBUZaKcbTkeYH3nvX6/hCbVlTljK7r4vloDFonZFksFvXec3S0/J2vbQCpApvtDu8Czhu+/36PkglVWfLVF2+YFhMEiixPCRicNYgAXVNzc3XDL/7kj/n3/+H/RVlO0Ulg6AeUVlgLq/uRn/5oih23ZKnGWMd0lpGVMBrH67eX+KD5/Ecf88VX39L0WxbH8sP6bH1POUlxfSyx7loY+5G8CIyD5fT0OIpYieLdu2vSTLBej7St5ed/+BFS7Hn3bsdmpXny6ClSZDhG0nRxuGk0aB27O8pSo+Vhb2wUq26kyAvwhiefnZBNBSYYnIPLq0sEkiTVJHkUxWQS2K9X/Jt/9T/y048+5esv/o63r3/D/HTO2PwU10KaZhRViQG6saMTcR9+dD7l//L/+L9xtX3LP/vjX4ALFNWcJJ3ifM99t0LZkWbVYKVi28F3X9/y+YuCx58+YrO+Ydtu8NrRJpYQBLl3JCHygo0fsL3j7OSEQpb84g9+wfHsBKwnS7NDGsWjk5SiiAlPpRKMszEFq1K6bmQ6XXB0NPDv/tP/m5vdO0Zj+ekffMZtfcPN7gpSz832CmscoYG2s9zcNHz++Rmfff6M169fUk0y9vstzhfoVHB0VGGDxPs9zkvWa8ft9YZpfsJkntP2t3zy+AzNgqxM+eKrb9hsB37+h0+p6wYpPfOjeWTiipEnjxe0bcsXv/olz19c8PzFCdavuXp9zWR5xO26ZvL8jKbfsdqsmeQzEJKmXaMST9fvOU1nSOVJkhRrHdNZzi9+8TEqjdgbO3oePf+YN9/fcPt+R0gGlpMJjbOIoJjkBUZYPnn6gpdXr9nVNcFZHl+cMjs+4Xg2pdk3lFXG91dvUHhurq85Wi4oJgV9F4Wl7WpgHH531GLXdZR5TgC++eYlj548oSwrxoP4EoSnmlVkZRqj6MEjE8nx/IjTQ3leolOypGA6n3F3s+L6+p6r2zuabgTj0MGzPD2lLHK6tkEER/CWto5Ca5oU4CVt3dP3LbL3VJP0sHZAVaXMJlMkSSz0s7CrG27u72Knhwg8ef6Uk/NztlqTZXuurt8xncYOrWfPnjE+7tjtWz7+5DOAQ2rN4pyJiJNiTlbmnB6do5NYXGx9T9v3SFVhrcd7MAcUaVEUDIPBjIb1+tArFALGGgYz4LxlMi1YHC/4+st3eAO7TUOVTFEqo2lueXR2HoXGJOX49IzFZMbb71+z2+6RSmG94f3NLdvnWyqdMylTzk4WBC9BaDoTOFocRfyikvRdQ9u2BCFQSUqqJVmao3SC0ilJXhGkROpYbpjlJcF6kqRAaU2QEqECSIm3gSCjAOmMRQSo6y19vcWOPcH0hKFF+5FJnlFlCUpH7WBaVShjCWmCT1MeP33CF9+/xATP0PUELFJHhIXUEhfs4fM1RyWachJFt2Ec8QeHp1aK2WLBZnvPvm2ZTRYUladrR6wf0WlFmic0TR97mFR0DJthROjAo7Nz5rMpr67f0Y8950/OSXXK/c1I5x1aRfQJISACDJ0haEWSSJI0pR07kIKiLA97jDGaI70nFYp5WiDTKNwfL+bU+zWj6elNIAhwwlNvd1hrWR4f0+wb3l9es5wu0TJi+/AC4QVaKW6ursknGv97iuheeIIIOCJSRamYnHdEIdvjGd2I8AKhfkCzeP+wlzqgKXTcjz0k1334bRf4g+NX/pZz/AFJS3S1hyiOP6RF4787IEPEg14RIrIM+2FffPgueBDX473nQyo6Ms8fROmIE5E/0AfCD4gYISX4EbCRCY6Hgz4YQojdcEF/eK4OQ5IolE5ADoy2xdoBTXko/wQEOG+QTmIPz/nhe1bK/oB4CXEQ8cDjfkCSjMZgjDkMqkaGfoyv0wG/s92uSJKIFbHWYEx0Y8eiURNRmQ9faxyRWRILvNWDk5pD71hE1jwkEpWMwrgNRLE5DaQqic55iH1m1kTNKJbVIQ/vz+EGj81mzX/49/8BgDw/9BntdgeMT8B7S9cZtJbMZhMuLs7ouoazs1PKMv8wPLi/v6eua/b7PZvN5gMv/UGwfzi/lEzjcMu5qKMKCMGiE8/Pfv45n31+QVbWGLtGqslh6CM+YIRi+uUH7eIhCfC/ePxAMIqPc0C+WO9xwR/eT//hvAM+DE4gctOjidZ9cLE/DIVCODy2iMht8Y89j8PxTy8WpcUHHTnTrkOqLrp53IZMKQJbXPiIIEZGcclEPcbauBFPVQqywJu3IM9QQkGoIGR4d4dLDMFLtIpuXzhmcAYte6xI8CGl0LHIMYRAb29IkxIVcppxiw0jzkexwgeJCilZNsE4h7UB4wKlFmiVIkJcvEZXM9oWJUuM3xJ8gRYVQt8RlEargLc5QeRY15Noj/Y5AocLV8SPqwkR9z1DUtKZFYnSYHuSRGCcRwqNVhZEi6CPZagscb6LSBERUMkkYlqEARzO9HgMg1Xkao5UUewb3Za+c5Tex4tLynjSCAlYJhMB2mOdQKsXyGCQosNRo+SIFilKaJzb44IDJigpkMqSmhNSWX14v2OTsTsgVCJ3ydgepUoMK17u/q+cqH8DPkbAYvtwwXS54Nb8O2y4Q1DiQk0iK0J4iOWYWNjJBEQS49R2TaGnceAyrlCSiDrxJR6D4w4pKpwbECrg2ZMmN6T5IyZWxmIjXWCcoAwViZgSZIINewSCPJljbRRkhAxI5SHYGLu1e6w1ICIvzroh9gZLD2LA+w5BF4tJhUGqFIixYc8DP8wRxIgUEeMiEPF5+zZyKfHxcUUSPwFCLM7wOETQ2BBwh7cxXvMPLcWxJEcg8cLjg0EEHYdI4SGOdVjcfo/IKIAfDYvplMurV8wWJcEFzo7P6buBlb2nb1qEykh0yqOzx5htS5VUTJc5N82K5fSU7fWGPC0ock3fW5q65+LiMUkKIrg4GQ4D9zc1F6fnzPKCooiM7ME4cp2QpxmrcEehM2xrqPSEo5Njxt5Q72LCQQpB3exxXYfS8OTinCxLaNqW/abm+bPnZCFhVk5ikZcXGOPQecAJAaRMkwWpF2idcbNtmM3nnE4esQuCm/0lt+s1QhUgCpq2o+u2TKoEHzRldYpSlvdvv0UJR1fXZEkOTnB7fxv56NUMOs/YGjJVYvvYpZ0mGX3b45VkMHE4eN3cMS2mzPMl80VJmgZu1tc45ZBlikxznA88f/oTfvFHf8Hj04/40eOPeXJxFs8LLMPoP0Tv5Ic4XTxCCB9GLuEhpBQCznt2+z3DMDKdTFgezbi5uebu/prNZkXbNWgJaZLFlu8kJVMZeZ6x3Rlubq9Zb+949uwFm80dRVEyGoOSOdPJnNOTi+ikcHHIJ4Rjtb6JkbZgcT46k+Ik3pAkCXZ0LJcnrDdv2a7WNM2exWLGfr+ibta8v/yWs6MLlvNTBuMPPEPPbHpBriYs5yXv714d2M4V3716iRDw6PyM0+Mj7u4uCd6zWm2YTia0/TVm7FFiwvHREUdLxVV9y2g2BCEJcmB5UjI9yfmbv/mOUi/ozcDt+3um05R2s+fp4+fMZ0d8/+5X1P2W4+MzdvstAYcbHE8unuGM5YuvvqBaTKm7mrzKCNaiEVy/u+Ps+Ckh88zKOWVRMZnN+PWXv6apB2bzBR9//NnvfG0vl1OyLLoVh95gRkuZS8pyQlGUGDvh5v411nb0vePm5obnz59TVRW73Y4sy7m4eMRqtWIYhojlmc8wfYuWCXmeUdeC6XTCer1ivV6xr3e0bc3Z2RmCKMylOkMnKbPHS7Isp97XvHjxAgLs93t2uw2PH12w2265ubnm0cUjmnZPkPCzn/4EYw03N1dMJwXj0FJVKWbsEWj2+x3VZELT1KRpwsnpMUmSkNYJ+3p/YFFb9tstFxfnnJwcs9nsDoJtSl3vubm5I88zlJIYY8mylPPzC1are6SUFEVBUWRIGej6iHUpyoT2fsf7d+9Zzs+ZVnOENFG4yBQnp/Ez9Pr6kmFoUQqcHw+oooSue4T3DmN6etNTzZZs6y3fvfmeFy+es6vXdMOeQmfcr+9o2w5noR8sw2DJs5w0yQ9pqNhnUrc13dBxfdvGIXgwtG2HTlI+evEipq1coG4bpFBs1nGAJxJJVU0OUdpY8Jkk8jDkdSglOT8/Y7vdMo5jPJ+GkbbtqKqK6WRJ31m0SsizBz6lpW5W3G9WBGEODnZJlkrSJCfPK54++YiuHRlHx3SaxjWNQJopNpv7KLKmkn295fz8E372s5+z2+1ZrzeMxjKMPcM4ULc11aQCGciylFAVNKHm9vaG2WzJfH6M1inDGG8QJ5MZWZZhTBtLH8uMzfaevh/56aOf0vcjd3d3VFXJYn5E33e0bcfQjwzZwHw+5/L9LeMwUOST6N4T0T1jjGEYBh49umAxj2zKrosxX60cShkeCr9ub29/n6UbgLIoqOuBLFVY0zKdVtQ7w9Pzx6zu7slSGTncQtL1gcYM9IPhN998T12vY7lWGBCiR6eeonDM5jmX7y313jGblLx/Z9G5IiC4ue6ZpROKXPHFl1f8UT7j+Yun/Obb7/FYknSONQ37fcPRyYxJmjH0Ce1+ZDZbMF/m3N/d4z188cUNn3w6QwjPZt1DyMkzzd//3XseXcTruO/gL//LF/zkpz8mTbbUdcPTp48ZTU+Wa3bbHWmS0tY1x0dzsrRAZzOsM3TdLS9+/Jhvvv2GNC1ZHmWIkLFcLAkE2nZAZ4rbm7dMi4yXr76gUopvv/kOoVratuPrr/6WP37xRyyqnNu7a96+exNRTGbk6PiIu/tLfv6Lz/kP/+kvKcqCZtehqwnv393AItCakVIkBCQWiaFjeX7GL7++5qd/7Dl5NKce16QTRXWaRYHfaRSO0Uish/2uoxKe/8P//n/kyckLcp0iMxexG4d1X4R47Qgh6PuRwYxxGBkcw2hpu57p/IiPf/QZf/1//2vSTPJXv/prBmsgjUVdQQq2u5rZNHbeLI8ykjTnu+9fMpuVvH99xc9/+jO6oaHrHev1NeUspelq+nHPdiN5/uyY95cvuRAp1/fvGMct0/ljkqzk0+UTdvsNxjZstncsj6YgHXfrNXW95ac//5i7mz3zScHV9Stu7r6imggcltV2zWyR0w+Gjz79FNtZJlnFV9+9QnaOp8/Omc8rXr/+niwrWa92nJ08oSwTxlHw/uo9n33+E7Ts+PWvv8P3mkml2TUblmeP2LYts8USJyVhKuJ68fiYm7v3jEPg62++4POPPkM4wXe/+Y7ODjRupKpKyqJkX+/Z7vc450iShOA0J4uL3/m6Xtcrjs8+ZXE05zfffMfN1TXBefbrmq4ZSFMdhb/lgtl8RppohrFHKKjmBXd39zhA6Iwnzx+zPDpCZwk313cE6UA4qirl/PyIwTXc37zjeFaSJrFg2DmPNQPeQlkVaJVgfYdQgrzMyNISqQTeeRbLJUlRUtcNm27L5fo1o+8RyQgyIUkypMyZTiWPH5+TpwXHR485OT9jvbvibnPHtr4nSZIDdit+b9ZHXnVVFUwmFftNS5qn7Fdr0jRjtAAJShYEF2ibjiJzRNavwewNx/Nj0iz74IAdhp5+aNntwTNwv7pHi5L1bsXTF89ouwaRK9qxZsBigJAlUKQxxe2hyjPwMa1d5CllkaEVoBWDDdjgydMcqRTb7SYmmYaORCvSIifYQDAjQkrKakJalqg0xx44vXWd09Vt/MxwUfgKARJ9GASZcMAbSHA+pv+CQWQp7cYhdEomJZmWSCnQeYHXmizLyKqS+7ZBZ5qTsxNevHjK28srmraJ+NHw0BMWH+8BEyGcJM1SIPKF0zRBioztbsvRySnres/gDEFJVK6pFgVKW6p5RmcMQQiKrETJlGEYcdpTt3tMFxO+P/rsUzb9Oqb4CTgzkCYpQgn6YSA4mBQT5rMFfTsgEQzGMKmqiMB10d0rpKKsChKpcL3BmkBwBhkEwcAkn5JM5yix5s31NflsAmnK0Dv0NMXZhv1uR7PumM0rlFD4YLB2RKmIZvOy5J+gsf2jh/WHYZg8oEwQeBEOIh8EEfAiRNzrA9b1oA+EYCPmw0uED0ihDkJhLE38bVOVcz/gVh7ET+CABQkHfro9OMpdRMf4eF1FjIo4OMYf+uJ+KGl8+O8HB/uDcPn/jRV9KEhVB73oAUkS/+xBnI7udSlV1KZkdBILItPde/DBEu9oRTTO2p7eNWSiQ6ZzpNSkSUogYzBjTP9/wLJELnxMVqQ8oG+kEIfBg/2AMxrHEWMiFsYYgzHx9cnznKbZM4w9UuUYOyJkLBs1h8FS5Lx7jI375L7vydKCYRhR6rA2hPAB5/fgfE+SJGJopISDmz++VhF9k6bZB0zMw/5RHIpRH157rTVDP/A3f/M3B6NtTI+NztK2/YfXfBxHxnGkrvdUVcXjx49J04S2bRmGAWOigWkYBpxzsfdAKXa7HQ8s9Ph4KVIkEXNzOL+0luRFymSa8PjJKWkmKCcpiJJgHzSwOH4RQXw4l8SDw1xEvNDD8EWICGGGaOgW/qBxiMg18s7hg8fif8uBHr+WlAIPuPBDuW4iAuqB4/9b1/AHvAsSxA9DkH/s+CeL6M7lCN3jhaZzhlwKpFwS7EgzjGTpMdZf0Zothayw+i5OVFRKPd6ilSQJJ4SwwdsOx5rAhERlhGAZbQlBQ9B4rhH6ls5NEaLEI3FuQs8WxDtGZzHulDxzaGUJzpHpgPBxcidlFft01EgqNQqN1tHNpJMNwceCOecHXOhRMi6uxu0RNsV5jTMXBCEQsiP4gc5cMkk/xjNCWOLFiAs53tdU6SmZXlCbAiE8WvcE9kBGKueMvkfJBULVWDtlCHuKJKF3UFvNVO9RqkBKMNaiRIkJNYVakiQDeaKRo8GHkSJ/AjsIB9XUeR9jV1rgVA3B4oJj6N9HB7fMGK04cEBbQqgRUhN8ymDWlLkCvUNjSNx5LDM4fPZ5H0/IIq+QUqOFRgSokk/ohjvM0OCtwdiezuyYLI/x5WuC2yIB7yGVCwTRRavkgJMDxjZkcsYwWiSehCOsSUAoRmvj5DcMePoYnRs9Sh08+AEkEsNAogJ5Ghi1x7iR1XbNfHEGYqRzN1gXS7J0OgUxIkOBElOMv8H7beSgA0GMCOJUM/j4tVSQWAdBDCix5+FaE4epnySPH3JEB3gU0B8KCg5pA5EQfGxDluEQ3QoO78KBY+Xx3iBEgpD+gHiJ8Z1YajrGElEh8U4QiCUkzguCMPiHRUvGGN/vcyyrCZqBWxxH0wmjcTEWNQzYwTMvj9k3PcoKpnnBdtdyv7nnzZuXhFJw9GyGqxtyrVhvVswXJzz7/FOOFqfc391gbM/tzRWIGLm8X91SJAUKSZJnrPsNzx49ZbfboAi4ccBjOT07wjnP/W5LIgpmsyPmVYXxLT7RDE3NyWxGPw5s+oFhtNxd3zCp5giluLp6R9s2lFWBF4KsWlLlBm8cuZAIGxi2I4vHc4QJtLuWtu8xzpOVFebABhzGEetblvML8irBdB3eDqRaxgZ6G3Cjox4HlJZcrW/J0UgjWU5PMGNApyVplVObhn3fIMqMzWaNAkZjmZ4uABi6gb7racYeL1OKas7Tp4/583/+P/GTH/8hR9NjllmFcOCDYxxHhI8LmlQqOlZ+y4nuguBhtxc3HgdmnotxXR88/dDx6y/esN3e03V72naP84ZJVRCCIU1yEiXAa9zgKbMJzgR0oijKjO2moa43DL1By5J233J0NGN5tDyU1sFue40xgUQPOL/G+IG8SKn3NaOxnJ6dx0Z2L5hMJ2RZQrvvkELShYbjhaap96SFYrO/JStKrN3H69govIG72x1tM2DdimEYOTk6JdGKoR+5vr2lqRsm0wWresX97o5HF3MSX1DXHX3oeXu5ZhAGKxRZMqfp9qy2HV98d0vf7HnxkydsVz2TrGTf3uEQNOPA/cvveH35HUmaMRUnPPv4I85Ojnn9/XekWvPt19+QJxnCOxKlaOsOJQRJkWGdYxwdi9kRl29uOFqc8+TJixgbVXsmkwVN1/7O17aUKbPpEmvfUJZTFosj+m7Ee8N3q3sCYyzdvN8jgCxJaJo989mUxWKBc47V/Ya8KBFCcX1zg1SwmEwgOLKi4OT0hNX6PhaxesdiuSDLMwKBru+QMsFamE4quq7F2bi5/+LXvybLc37yk5/w/Xffx0Enntl0QqIVRZazWC64urzh9OSYet9SFROEhnq3Q+volJnNJiAC/dCz2d6zXC558+4tpydnbDb3sdC4FwThubu/YTqdYoxjtVqjVBSL2nbPbhcOLpKExeIpUioWi+Uh1niHENGJsd/bOKA3Bg5umrLMUSrD2A5nHfP5glQn3NX3TKYVSRZYrVvyrMRax+nJCQjHr3/9a5JMIpSj6XYYNzJbHDFbzNnUK3yw9L3De8kwOLrOYY1GqYzj5ZQQBHd3dxwfn4ISKC0REq5urkDEIZZQgm7oWG1WfPbpj9huD8Oz6ZzNZhMFeBeZj2dnJ2RZyvX19Qee5Gh6pPIcLZcsjpa0TYdSCW3TIkWC95LgFVU5P/Bf5zhvcS52PFgDSquI/+tHJJ5QCt68fk2WR3wcQN+PjOOA1vLAwb8BDB+dP6HISz755HPm8wXvL9+TlwmIkuub95gx3vxIJWjaPUo4hq7HueiYns4WTGcznIdhbJFCoFXCOMQCcKVi90RdN8ymC7zzJEpztFxydnqGSiT7umIymdI0O6Sc0/cd/dDTd4Z0X3N0dIpS0W1VllFQcz6PmDSlWSyPDgWfA5vNJorDfc9ut/udr22Aj86fM4yeYLb0JsaU+y7yY/vO8vTxYzbbO9abG1589DEBTZob9rtbpvk525VFJRVWGNIi4bSs6DYdHz0+JUt3OHHJ5z9/jir28SbRBpafPkXKNJaVbjZ888VLnr14xKyccH23Z2gt4ziCFFzdNDx5fAKu4cXTE/ZNw/vbS1ItaNp7fvbTM/rakLoJd/ctIR1xYeT8dMLjR3O+/uYVf/ZnP2O9gi+/+TV//t/9Aev1/sCbTxjHHq1Bp5pqOkWlCbYfaJpbjBnJMsHd/YaAip0ewVEWFdWkxFjH/dUto+1JyNhtPFXS8vcvv+Tm/pLFYooHvr/6NUUGk+2Cerfj25dfMi0LltNjrq5fR7axD/zLf/Gn/N1v/o5dPyBkRt/sKVMo9JT23lB3PfNnC5ZLxWKuOT8+wTYjWxtIjjT3+zWPz+Z0657tGhbzBKkg0ykhU/yLP/nnvLh4jg6SeTWjtz3xfjugZUybSUCmms44hJFooUHEZM/X33/N4EfuuhtcLgh5QjIpUMGzWt+zPFlweX1NWc0JUnN8dsqTx8/Jk5TdesX1mzdcXEy53nzJRx99wus3AyiN9ZrZPMbA+/qWqgr84k8fk+SBTfOWfRsYzJ4Xy+cIPK7oWO/uOTo+ZbZY4ELDNIz4NuerL3+DD3C3ek9e5BRlQrMOlNmMYRhQStG1HXXRgvE4azk9n7JqG75/dY0gocwndM1AkQpefvcln//4Z4xk3G5qhq++YugMn370Y+5u9lw2O/J5zuXmjuAEqUjxSpAca4yt2dWG1fqesgp0VvCr33zJ8fIRn/7JT/kv//GXLCefQ9KgdMKubkj0Em8s71/fM5n2nD86+p2v7WpekpYJs+U0IsS2O4piwiSbsq43DKOIhY9pgtSK4+NjirJCJYLJLOX6duD66goZNLPpkvn0iE8//5jZfMrd5TWm7vjks2ccPVry/mrHZJKgtEOrlKZucE6Bl0yKGamO7HAjoR9HrLckSGaLGbv1wL7pKKaOvR3xRaAXW1q7ozUDdmh5aj8jL2Yo0XF6eoLtHcJLhs6wr/e0w5ZXb36N1po0TTg6Po6Flr7DuhrkDOMiosMYQ/AB6/zBeZSgZUXX9xAUbTsgRIEPnq5vSVXGRy8+JctyNrst1hs8jq+//hIRRrR2DN0eERTnT8/oxhpjOoyz5GnBpq0ZFTTKk5cZ8yTn08dP6FY1VVGSKoUzA8PQg/S0FlzQOOswXc9qtWa73ZGXGWU1wZgxFgU6Q6IK8lRTFAVJMSFIhbGOEARmsDhjol6kFDqRqCRhmlYINNbDMI60+xrrLMdHS6RxjDpDjA1+3KFVQGYZMs/Jp3P6sWdWzpGDoB876m7P2fkpd6s1AYf1hoBHJzrysmUUYfthINcCFdQHQXQ0htlsRm4KusEwX57QtGt0nmM2LdWipJpKvLLsNg0ERaZLnJUEK3Em4GxgtAMmH3ABZpMpZhi4W60IZow0giAIzlFkVTSV2cB+u0fpDqsCF5MLijxjt91QVSVZkmDtSNf1uKAinm5bk+oE6z3eDZR5ytNHz8mSipvtBkhI8pL9oQNAqzQOQ7VicTTBOHHAbsQU+ThadPr7dZFZH9fpqEMIguCDsPfwq5AHARmLBETkURAc0f19KOX0wvBQBCqlJjz8W6JO5MNDCvlAGRdRo+FgsgIfhc0DXsX7qDWoIAgh6hIBfxA+A/iYdIlf68DpF+KDEPkPRPRDeagQcCCoxEd0D6lpiRQ6pv0Vh9T+QwHqDw5yrCe4+OfB22h69AMuJJTpKZNEkjgFMmE0ARtcNIWEgHVRKxz6kSRJKMbsIPz/gIrx3vCAtBnH8QNmpW07rBuZTEq6fk/XtQdxvGUYekIosNYyDH10t4tweC0NIOmHkWmIQ3ft/Qd8TJZlWOsPeJeDe18qhJLoNJpMo+EyDjSFj/0niYzJ4OB9TJoHDpzviKF+9uwFn332Of/23/4/mU9jCrkoCoRMGcY4GJBKMpmWWBc1vM8+/4TVagMhYKzl8uqa1XrNerUmTVOss0il0El8Xl0X0YxapwcDaMSieGexOKT2TGcl1USitIsMcmRE06gDOsXFwUiQ4oBROZz0UiKDwguJDg9UfA5JBPAiojODgODB+jjUC+KQzhAxRSOEQAlNEAYXIByGN14rglI/YILE4ewOcTgD8b7yf1URfXAr+nEgKwq8qNjVc0JyhQ1fk6sfYb2lt1ekaiAIzW7cUSQpqTwnETlCJGSqQvo8uqqNQSQj1iVkeo6QAhMEzg2RET1YyuyEwSiQhj7UOGEQ9ASRERgZ7RYleoIK+CAiisM3dOENnXUkqiT4klTnGPkW43O8n+LsLYgZLgQmSYULPl6sdsDagJYGKUd6C9L3CBy5OkeJDOcyAvP4d2xDH1zcnLsdqSpQfkTrwOA0KmR0JjroEQUCKPXZoWxTMLg1Xgi60JE6hww1oEnSKabfolXL6AyIHhs2iFARDiciIXpKlYolOKQdRo7IYBEyRUqPVlM8hoQEb1OE6jC+QcsMKWdkeonAI7CReT/mCGKZjBTiEDOBJM2jm1QVSJHQuzvOi/+edqPA39PbGkPP2fkfsrF/xeC2qKTAes/oojCj1T2FPCGgUdrgxTWJmsaJkUtoR0ualSj1mECMbSnRg5ckeoYIAiUkg/MEL8l0hUgsygiyTFPvHe/ev+JoUZBMS4QRZDqPwoPdxaiQ2CL8Hu+bOL0KI1pVaJEdYkrRASBlFp+zjO6+QMCFjiCSw1JjQKgoRwrNA0eLIAh4nDcYN5KonFRNMK47CPDhwKNWhEOJRQgKpUqkCAQRW68DHinA+wzvxSFmBUKq+NoIfRDzk+hwl4Le3P9TL+X/xaOtd2zXN7hhT5W9wAw9Kk3o6zXeSc4Wz/DdJbnXbO9u6W3L7d0Nt+0dvg30fUuuFOfHp6R5TllWEBy36ysa02GNYVt3LBYzusFws1lxtjyma1vcsEZNC27bK24u31JVSRSU6h3vvnhFUhVk+ZKxkwSnqJKBNIPGjezvV5yVU3AeqRRFlXBze83keM5+swZnSTIVo/nfv+Hx85TN9pZJpcjzE968usQLQW93fPvmiumsohAz1tu4oZvPZ+zbmnI+ocglm80dk11BGgR2MCg5ocpiGe9u7LFC09kRh2M7GhKrMV6RipKt2zJLA50bWO03+EwQsgQxBLwJ3NzfMp9XTCclm41msAmVPEK5GWdHn3G6fMoiryjCiOlG/JgR0JEDpgyBgFIaqdQHRhqA4IDx+C2HgPXQD4b71Yovvv4Vb9++AizBDwxDi7U9WkuEMlg3MK8ArZEuxwVBkUyZTY5IfdzgGNPTdw1WespccrNac33zisePH0VxD01ZjaRJRtsOdKbGhJH5fM7q/o6zk3NSndD2I6M1BCRK5iwmM4QVFGpA54E0z9i2LT4RlJMJZqjpxwFnHdZGJlqa5RRpgbNwMlWMdoSgaNoB6yUv373lpl4zm04I6pjBBJ588oJ311fQDlycP0YHRVXMEVIyuoDSBYsTy+Xdt5TJI7RI6LsenSqc8iRpxux4QVFNkHnK5f0tt7d3HFcTwjBysjjh/OyI1u55dHbKd9+/59nTj/BuQOYpo7BInZDqiP+oa4Mzgs+efsr7+yvu95vf+dqWsuDk9BnBf4V1ivvVmqdPn/Ldd99j3YgQMGwck8mcab5Aa8G+3uFtT1FNeff+hrvbPU+ePsNhyYoiYhGkwA6BIlFsmxprR+ZHS7SWTGdT7u/uGEbLbrdnNjtlte45PplwdDJnX+84Pzvn7//+71CpZnQ9s6OKt29f09sOrxzF5Ji71R2ffPZjrq9u2e4d08kjkmTKvt7TdIZxaBAyCghpqnnz5nt8sAyjQknDmzffMJstKMucJEkYxo5dvaaocipT0fYNQjiqSY4yltU6ItS09/RjTB4ZE+i6hqJYcn5+QV3XEfelc5Ik0DTxM/vq9iXL5RxjRmazOVJKrq+vWK/uI2+8WCC8QsqUIp8AYG1PlgekcghhsWFAakdW5Og0pyinJHsNeMbBkagcXaYU2YziULLbNHtW9Zo8l6gkZTaPfNjpdILWmtvbW7IsZzYrEEJwfXOFc4E42Q1MZhXWx/jpertCahFjrMHTdC0q0eRlxe39JdV0zo9+8gdcX91wf3fH7WrLZ598xnw25/LyMnY/zI4wwbPb7XDOkmQTPj29oG13rDcrpNsBkouzc6QSvLt8xXxRoWROvR/Y7/pDGsCgpMYxMpklPLo4JVUp716/5/7mivPTKUrN+NWvv2Lo4w2SMwN5KmnqDVlWsNnUCFmR5hOE1tRtw2gNWhekacHJ8ROEUNzcXeJCy6RKyHTJ6nZNkecoHW8KItrHo5OAziRd17BebfFeMJstYi9MV5PnER8RQmDoR7ROyQqFCwqpcrJCs901uBCYlSVNXf+TNur/2PH61SV//Is/oRkC42bPZj2SZyOLaSyYO/3RU9rhhu124IsvXvInf/IT3l++oigVT47n/PKLlzz5+IJybljdvcYaw8XZE95e7vjpjz9nvb1FpQIvLCEM4C2ZgrptePzkGVL03K+27DYrlscThDjh7rZmMimpDjez201PIjRVWRCCZ9VsGBwE25HIkuVyShglHz37hL/54u84uTghBMv19RVPn854+fJrHj95xKefzaibDXkekyHX1zekSSxo3Ww2PHnylLu7ewSK1f2ayaQkTWa8fnmJTARJGovGqnJCVRVcXl2zPJrw/rJGyYSimtMOjqvbb5lOK4z01NsNfV0zdJY8mXH99oaL8ymr3Ypx9Oy3I5nOmJYF+bSkrxvu6z3NbsQOlk6BSwz3TQujwq42/PhnzzD7lrFUpGdL3t60JOQ8mqVMhENmhjE13K53BB+YFTNePPqExxePSXRCmZWRlRtgtJ5ECoTSODPi8fjB4nxAikBdb7m5u6SYFbSuZt1usYklTTW7fceTp48xJq4DRZ5xujxG5xm7fUtRTnjz7g2JcujUc36xpDd7SByv375CqgwbJLd3O84ujvEYFkczBI6m31ConMl0yfXtCjXu+cu/+S/8+PPHiKQj0CNlyXx6zK++eMfxSY6eaHZbh3c5643j7JGmmqd4GvKs5GxxxuXNFc6O3NzccnF6TJpJns4uELf3rNcbEi25OD9ivbmnazvm8yl/+6tfMZ0v+OSzT+nalqa5px1GZKo5PXlKY1Z4U7Pb1GTqiiTVqFSzPFnQ3d1TlBohR7IyJ9Nzemv56tvfMFlWJMojskCQliACy+WS62bN8XwSC0pt8buv3TrQDTVpoTg+X5Kpgt16x8nylNEOeGEwfc9+vcW2A+39DiUkk3nFvtYUacWi8tS7mrvra4Z2pKpm/PyPfsavhWe/3jA/mSNFoMhzgi0ZuwZhDXlVAilNN5IvKnSuUUJja8vQN4yjpWs7zo8fsZiU7Dd7gukQEk5Ozylzz2ZzF01yQ8fQd0zKI9q2xnpD09d0vWfb7khLiRICJQPj0OHswD716FTgrCfJNL7zGNVzf3PN7fU9QiVILTg+PyYvCnozYq0ihCRey0XJvmnJ04rZ/IisKHA23tPiJXdX96yud8ymKVppEh3IiwTjRxanR4ympvM1WV5Qzqa8vXrP2I/ocopIBC+ePCc7g1JI0uBxXYcdW3btjlFlOJdQ39e0fYcXgTzPyNIM5wJDH4086cFV/sFVKgQqyTB+QCsVDWiEKJzlCUiB0JIgNDrJkEimiwXi5BjXNbTbNff37+hW9yTSkScHQUorJvM5Mi+w0uOEQym4u71m33Z4b5lXBWWm8TKw3mzBy8iQPvCz3WAYw4hwkUscTMTjZDpFCknb9CxPThhNj/cCFyxpkTOblqzW1wfRNYqCy9mcxu1wY8BZh1QepQVt3zCdzmKZpvVkKoliGZIsz6nKOfUuJpzG4BDOU5YVWZYyDB3j2JMlirFrGUfD9e2KanFClubU4xa7j+/F8XLGfjR88933TMqKx+cX3KzvcaMBAm3XAxFfuVtvWUwm0UHrQGQKnSUHtKX6xy7f/+bhbDg4jn9ASTyI2A/YFSEOAjA98oATRiiEFj+IvyLyxRHuoEGMUQQUEif8AekrcTa6zKU8/Jl3wAG1oiU6yRhHEQfgCJQ6uH3FoVg0BGDEB0V0qUctIn4NfTiPA3xI6T+YAv3BPe7+AVrm8NMHdIbWD873eF8pD6/vgyAvZET04CNfH9EjhSLQMJgdSgd8IvEiwXQRbyZFdJwbMzCaHusNMnjafo8QEW8oRUIIgjQRjGNMTIYDEyYK6rEMNwSHMR0IT8DSNDVKaZpmxzgakkTj3AM6JCGEyFX3H/jqYxTXg8ONjmFIyLIc5wzew77uWS6XuOAxLgrdQcJgDWVZErxnNANJkkREtI9a0YEkDiH2CU6mc/75P/8zrq+vef/+HUEEyqrC2BYhIh8/L1KUEod0bSyoTVN9MHms6ceRthuQOiHJcvr9njAYsixD6gShTLyF0Bqt4pOQIsEHSQg9SQKPniwZ7RZkFp32aYYLIwEHwcfBoNaAPKQqIjJIhijIK6EO556E8JCiIJaw2ohAEkIQhI50iRAT+VJoEpmgVGTgyxDXFyBeH05CkGilid6DQPA2ivAf0Mwqft3/xvFPFtFTUWD1nNHck8spVu8Ruif4FwS5JziFZErCLb2TBDwiKNrxe6rkjC442vFrdDimkMckwhDcitEbOMD+hSrQqqMdPYlY0pgGKSxaOvwYo7ijUcyKY5zv2fUbquQZqbCMXiFVjXXDIdosWLeXKJGQJDu0LVCyxflFLJSxHd56BhRBKDq7RgiPlwOJSPGhIyDRoqDQC8yDK1nd4kOOFCfoJMNZhVIN3syRKpYjeE5JRYZE0YeXcWOrMsDR26vY5i0zhLikEDMIBmRKPyiCb7DpNUla4v093lcEHIk6R4k5rgYlMkLiyPMoGmVFjlMOlRZINYloEJVTJFMGe4vWMBqBQ5DoE7w3SBKSRBBCx2j2ZHJGIuZINI74ITC6GCHp+u7Ad5LM0scwtgyNR2FxtseGjrJaUE6hsx0Sg5IJLowkakrwORqP9HO836N5xOhX6OCwIo8YHKnwtkPrHGSD9Q2FekyiUnrvCd6hpUQLATJB4VDVDvrH5Klm0GCcQSUFIuQIH5uYk1Qxun38UBcJPmyAPbCISB5vIpPdrcFKlIhDF4lAIDCuQav0sECk+BBHwFoovB+QHOLuwcf3BUcIEiUmCBJyVTKRij5scGJDmoD3klhkmyBVAUJFNpQY0Yf76YBHSE8IEiFSgvJ4WkJIPiwqPpjogPD9gY3+ux/v3l/S9Ct6Z7m8u2VXjyyPTzDecbQ8IpWaxWxGlsYNkzORJ5XoDCMkd9uG5WTCtu4oi4pt1/H66g6ZCEbTQ4AkTdE6pSimmMGTpAVJBnfDmnZzw7urhpkuUcmCvuuRrmDY36FTx9nTBbvViOtGeqfwNrDb7zGjZbdv6EVg8ILJZMru/Q198CRaHRxJHVIWLI/mSFGQJAWzo4x913C7vkckCb11dMZzXEzAdzhgs92yr/sD62zkfrPHjg5nL1EOMp2xbnpOjk4YhhGda6ZDQzs0HE0nuMGQknAyOyfTUxoz8PK7b5nPJ2gku6Yl+MBiNud8csxqs6cberxXXJxeMJ8a0qwikDBLM1Kg2+3YDy1aKqrJjDSf4LxBiQGkwEsNMhYPcuC8JTpFCB03JodNTQiBfmi5vHrH/er24BY39N2eYaiRIpAXGUoLRAEmHSGPMSkpohvm5PiEm23LuzdvKMoSawfaIZavBCK/8eXrb2n7hqOjU45PZ6zXOxw9Hsdo/WHT5zg+XTKZVVTzCV//5muu3+44PT5CKDB0DKEFmSASTW8G0iSDoKOwKCpc6GjadRQDx5QgJHlessiWeBxt25ClKWMY+erld4gs4fjkFDwsZ0tsb5hPp7y8umS6XCCEZqJneDwqk2RJylev3lCVUz45P2Fa5sz7KdPqjOXyMV9+9w0uDIyjxNsKTUpKxqOTR6yvbzn+6Iz5acl/+pt/z9/+/S/JiyXHp6cMY8vt9p7X16/Y3N1Q5RV6VnJ9d8PJ7IggBHWz5fGzR7/ztV3vO968fkeaFqRpQtd3ICDLE4ZdQ1Hm8abHeT5+8TFpovj+5bd8+/23nJ1dcHR0wuvXV9zc3NC0O87OF4xjy+3tDVUxYbNdc3S85O7umrv7O8oqZ1fvIg7Mx2Iq4yyI2OJeFFPu7u7ZbNfs65qj4wU3N9coLX9ro22xzhwiffCHf/jH/OY335HnKqaidIqzPjo7EkFd73n85ILJtGS9vsfaESEi/9pax/HxGbPpnLrZcXy8pCxmbDYNIXjyImMYushJnVSxcyJo+j4WBnnn2e42tF3KZFpGVqgSDEPkMadpgveWREvGcaBpWrIsp2lqrm+uGPoRCAfHSmAxLwnBH37vyIvoFJUqCgZtO9K2+0NprybLCsZhYD6bQMgoiwUff/Q5d3cruq4jz3OyTOODoUhyzs/PDm6+9PB4Hp0kLBdHh2Il2O9qxmHAmLgpzvMcYwxaa66urlBKkWUZaZrSNA39KLHOs6trsiynqkrWa30Y2sWBXVlU2MRGDrNxNG3khy/mMybVnE8+/oS7u1v+61//NV3X0TQtk2lJWU5YLI7I0gnX4p7tds9+v6frQSjBbDqhKiakaU7X9dze3JPoiC3ROmW5mPL+7S3Bx1K1okhxLt6UCpFSTFKUzqIj3oOUCScnZzx7+jHLxTneKYwfuV+PFHmKN7HoSytF3/Vs1mvKSUbbNDRdQ55nlGVJvW8jEiiryLISY0Z2ux0PpVWT6YTJdBJ5jdYdIsHx5qwsK6rJBGcdwzD8Xmt3Wia8vX7HYDrSTDCfaobOcHY0RwjJ/foK60eWxxXTyTFffvmWzz+/YBw66uEWlOHm9obHZRm7DxwgC/Dw6rs97TDy6Inh/OyCb776O6aFxpia/Vbw69V3/NGfvOD80ZRx7NnVa8pqwo+Olrx7d0VWTGibDbfbjovTY96/X8cI9qB58fwxu80db96s+PwTxUcfvcAMkYl6tJjy/vI9p8fHJKni3bua/WagrFKMq3l/uWWxnPPk6SPWq21c38TA7c0toxnpWkNRZHRdR5rGNd/6Ae8cRZFT5CnD0OODxVuiI2vwPH56xn63RQ8pSEGS5didoh0DdR/REjdbQzEbGbuWv/rb1zx9/IRS7Km3mo+qT3j86Bm//o//lbOzc46fzqjLDetuy9GyIhWabbPl1cu3TPMcPFxf7ZkVx3z66DkhtKx2byjLlMXpnCydcvX2nte/uWea7pHERI9NPHboSfPsYLhwDMailcJ7kIlG25Gh73h39Yr//Ff/kU2348nnT7ne3rA4mXOyPOLxRUa92SBkwA4D2XRKJjMSIQh5Sjvsmc0mPHn6jKbeU+82dHeWKs05Pr7g1as3pHlBWSZoDc4PKO14f3mHtZ4nT56xPDrh8uaOEHqquWLdvmUySclyRZ4kuHFkbCx3w8jzZ084PtLsNgM/+vwZnVljzIBjYNVuMXkCQnNx8Yhh7FhtbtAicH52zsfPL0iVZxh79vWW+9WKalrw+v01gwns64bZbMLR8ojHj58wGE81mXH57obJLGOapxw9nyOCoe8tX/zqnsVyx+KkJE8r2tZS5QXz6RxCzvp+hU5h3d8xKwraIefdu3vWScPHT58wny4pp4/YN7/79W3NwO3dFVVRsDiZokPKu3dXh0Fshg8BVRYEG5hnFZvbFV3TsStLTKlZzKc8Pjvl6eIJN3d3yDTj7dUN548f8fGPP+Obr79iP7asL+84Wi4wvUCqgrQo+OSTz+gGy6++/JLdUFMkOdW0gFrEdIMTjK1jL3oen1yQlI5Ke1rvKPWMZTWlm7Vc3byjKgu8GxHCsd2tuVldg3dkGqq8RHjP0Bh0lSJcQp7n7HYbZGJJkxwlSpSxlLOE49mEm3f3uDFBZzlYCN7SdT1ZVqGDQgRPlqaA5uTsEUVZYIzj6uqatu3ABLq2w+0cvfdkVUw+pZmkbracnJxzddeQloppNeHk9IxxHHnz/SvSUuEGS1M3nM6OKZ0nk9D0Lc1uza42tCHF+wSzqhGJQiQK6zzbXU+iEvq2R6cpaVFhvYdxZNxuMShk5rHOgbckWqFVSl5MyIoC6y3qUMKnkoSyKsmzhGAGetfhFGjXM80N49DhvUaGiJR5SHvpKsUHz3674frqklYIBuNimgUwIeDG+BzcKNCJwlpPMAFCFLP7ukPk0U4sg8T0A2M3sFttqSbTiIoYRpQUtPueYefIygleCoLznMxnFKNjazsaY0iyjHZoKGcleMFus40dfFWCsxayDEVCkpQQBvK85Oj8GJlEk9Aw9Kzub7F2pG1qirykzCsm1YJwSFPMZzPGoed+taPrRj75OKZiXt9c81H2jFlZMFqHNIZBxGB4VqWMtcMNnkSmeOkpZiV9GKIL/PfsIuuHniQ5OPvFA3bFHX4fdQDvbRTSrY2OcCUOzOnfQnpwQIiEKKLHDrWDOBhi0agQv42Ajc5m5w5ObeQHdMYDLuSDCevw68Nzg5i8fGBZS/HAWo/3lQf6b3xs7IfHi31YfMB2xNS9OiBqHnAkD+71B9yM/Qc864fvyTtPEPH1DyJC0owdUFoQnMIYgU6ymJYYLEoJhrFnHAeSJBbU9kN7wIokZFkZ99njcHjcaHbUWmFM3IenaYoxwwfevHMWHxxmGA/fm0WIDGMNaZoyjMNhL2jQKo0pWwEP7HkzWpIkObzmGUJEhGLf9yTpAZnkPQiP1vqASUkOZaiRbKCTiPb9UIxJvMfPy5Ll0RGffvYpTddgzMAwGKQErRXzxeyAnHGUVcZiOUUqUCqie9q2iQiaLKPvo7u+LEvatqXv+4h31Zo8z4FD2vVwHiglkFqxPJry5Ok5k2lOmiIKuWUAAQAASURBVMZuLCHCb7nu+TCAEUKihYwDGBcOyfnwD3+IeG7F894dxPV4LUgtUD6616WM94hKRrRyODhkpRAE+WBejdeNl+4wpHOEiLmI3QfigSrxv6ITPU87Nu2KPJMoMSNNbxEUaPWczr+FpET0E5wbmSYzElli/IrW1tRhh3E5VZmDNSit0Wg8KYne48wcpRR1v6FIHKmYElSP9QOpzuMgTMYLPEtP6O0a46HIjnFmi9YVwgV620QOqrRxkpGCx6LFE4zziDDQ8wZJQj9MSXTNYO/Jc7CjQasT+mGg5Z5MFWgxY7Bb6vESyQKpAloNDOaenpYyC5TpEaPpGMctKl1QJSXd2BF8R65jUWeeL5A6Z+hzlNCkSmFthxs1ebYkhJG+f4eQBUI6XHhDIo+AFBFaPBrnA0pMqcJPSVWJzDzGOhKv0YlkNJK+1qTaUqY5OmSMboMSE7QCIQZGo8k4wsiWwXWkokKLKd5cEEKFdNOIGREeH0FRDMPIMPQYO0SurqjIecLoN1h7SxACYwfOzz5lG/4rTqziMCEMwOHDI1R4ajpXI4SJZaEhRwiDsR4hEpQKKOHx4SYWfclTBpvSmQ4vcjKd0Ns1QWgSOYloFx3dskhJlipkyEjUlCJ8wmj/HUGCDgbtp5HJLBOcS3HhEVlSoOWW0dYoKciTM6xv4gWNJYQU4+I02nuBODDWQcVegNAcLuCM0XXAQ2O2OUw3K3wYaUyMsPng4wUqPd5atAxolRJCh3UdPhikkASZoURxmCoeprTaxwibf4jwtXHREYeJvzssRr/H0Q0D3ThigFVTc3O7Zdv1LI+nkVvfRje9VIr3l5ekWcLZ6TlFOSWdLAjOYtqWsW1ZPjrDGIexAhJLJiRt3SGVwBjLcnlClc8okoym3uBFYNusSKVnNIHr7cjJ8Uf87Gd/xPHijKvt96xuL4GMvJyjMsXV9XtOTxckVuCCojUjrfO4dmDbGmTT80d/+AcMXU/jLef/H9b+68eyLM/SxL4tjr7Krik3F6EzIzOrsrqqpqoVetjoeSAI8Jn/KEE+kJgBp9nsnu5pUSJVRUaEazdtVx69FR/2NY+s4cx0IZMXcCDg5nHtiiP2Xr+1vvX0E87OX7BeN0i1Y7AjH26u2bcN1gh233wTSw+7BkqPSuNUveu6WI6YJSQi8pCl0Bwtj0iFJoweVMJgRiywLObsH9aEQjAtZoyDpelHWmrabkehFSeLiAl5qDc471leLNFBM5lPSbVkMSnZ3K+QTrC/X/PZZz9mkaU0qztcs4mC2mJOVha4YY8dBxTDIcqlkCohzQus8wgZs3NSOlRII1LIx43m/f01db1mNp+w3dyzWu0YxxZnR5QEnUSmpCjFgUkXj5Ukic8/ncz47tWOum6Yz2b4YNjVW+p2z/nZBUjPtn5gs18xvZlxdvaEyawiyJHNds/YBLq+4WF9g+NHMbYbAu/ev2Y5ecokL3j74TcE3bLZP3CiL1jtt2zrDZOyYi8yJskcaxx1XdMNAyoJ7HY7jk7OUFoxjoYiy2i38XtNphkOGA8LhGlSsbm5ZXlyEl+7afi7l7/h/Pw5rR2YzGYUZc7d5R3GenyA3jbkTpOkmsvrK6yTzBYFd6NDa7DjgEYxnVX43pPKDB8ENsDpk3M+rO54dnbMw27LtCq5vrsD4fE6RZcJ9VDTuxEnAvu+YXAdD+vrP+DsDlxfX2NtXESt1ytWq9WBeb4h0QlCRpbe5fUVZ2cnXDy7YLW5p+0bPvv8K+q6Y7PZkmWatt1xvJxhTWRi3tzsKMoscvK8o64bkkRTlhNWqw1KRgZi8IKXL7/nx1//CK01XRtduMMwMIzdwaURF/HZpIgLTTNwefmBzz77ksXRjLdv37Ld9VRVzr7eggjsD87wy0t7wDdk1HVD3w+07UDwGu+g7wxJklPXDW1jkUIzncwJGLrOMgwGQrz+53lB0+wZTX1wsO+pG4NUhsVigU4k1lkQlrJK6VqL9+Fjea7zluubK+7ubimKnEDCaMZDBH0CWEYTXSVSQlYUCBHo+xFEoG0b9rsNSktmk2Ma0XB29pxptWToPeNo2Wy2SBkoywVFGV3mZVWx3zdopaiqKXf3d+R5Sd8PjKMlePmRD9m27WGwqXhkKUb+48gwxAV9WcYi7W44lKKFwLSqSNKYgsqyFHeIsU6mJUpqurGjrfeE4BiGhn3tubkVFGUaS0inM+bzBZvNjrfv3rM8PqZN4j0xTdNDv4Wh6wbmi4Ll0TFKljR7S99sWa3uQI6sN1dcXFwwqSbkxQ7vB8ZxwJoRpYGgSYsJk1mFUCmjsZSTBcELFsszlifnLOYnjIOn3E7Z7O5iGmZsMCYWKxkzUskc5x1t11HXNf3QURUVi8WCrhuie1Bp3KEMV2uN846+bw88zsf4rvtY0qq1PjDnI9LlD3moRB9wDgOphPOzBRjHT3/8BferNVY4nBfc3e3ZrTQvnj9jtx55evGC7e6OZ58UjBaUhCSJotRoDT/++jO+ffmWSVXw3bevYhcSKdZInpxc8N/9i3/By9ffc3JW8vLtS6qyYBw9l+9XfP3TT6lmE2bTJUpoppMjEiXoGsk4Qt0m2AG6duTJRUU5yXjz7juOjpYsjwt2uw1lER28u01DlhyBTymLin17w09/dsqrV1eUhWY2nWGN5pMXX/Hy1a+RUpMXgdl0StP0vH1zySeffILpR0BycnxCW+9J04Q0TflwfcvR8ZzGjrx6+zridwQIrehHy2ih3rccTRVBp0xPpmz7OCweE8l67Jm8OOH25o7LX/4VHZKv//Sn/N1v3vHFl58z7Bvs/Z6vT56gBsuI5OKkYne3Auv5vMp4+fJ7Rg0vVwNPf3rC5eo1ny6mrNYrlscnKFvy8y//lMUsDsKKskRJDvH9eK4KKeBQOGetQ7iBvt/xn/7Lv8NqS3Wc84vf/g1GQjr9ASk2m054+fIlQgiOj46YT6d4Z9g2HXcPLc8+K3j95i113XG0qMjykqYeWakNP/76J6w2K9qu5vbumouLMwbTsjiuGDrBy1fvmS8msYxMOvZ1vOY0tcAbRbXIWN8/8PzJGc56Xv52y+nZEb3ZkftrVGqwVpEXR+RpTtc43l3d8I+OZ9GppxW77ZpxGFguDMHBbLrkYb2hKOb0vWG2OOb+Zk+e5yiRRtyBErx99Y7PPr3g+SdnfPPLW/78z17w7Xd/y2ya8uziE7789Gf85tdveLi7JykseZmT6BylEq4ub+m6jkx4RGrpbWAYHc+eXuB7wYd3t+RfzFk3dyR5/nuf21mW03U1O2OYlhV2sJQTTdOvKMqcEDynZyc44yiSDPyMNEto2gGzk/TB0qaG8nRKllR4IZlO5nz37ff843/yF7x49oL1+oGHhxXeOZSEL776EaSespoh5UgSNMO2Y6w7Kplwtjzm4WGIKIPRMCSG7W7P6dEioitl3LcM/chyeYr1BiHz6Fzv9+ybDbGMLq5Ni7zk7vYKgUNpS6JztBIIExj7EUlCmmvOLs6ZHc/4Sfl13J9sLdNpgfIG33doF409ZVlRFgW7uuP45Dh2Ftze0XcdQ9sydj1usNhmYDmfo3KBlR3Gjmy3DwiZsDw5wrqOokyZThfUdcfy6BjbDWQo5kVJmkoIHo1k7Duc9REDIgPNag8iZVrmOALNONJZwzBEh68ZDJPFnNEO0FsSawhCoXWCcpFzPA4dQnjyvKKsKlSSooOLXHXvkH5kGPbUZkDYkUIqzG7DJM+wsjxgxOThjzoIXzFp3tUt+31D03bc9S2jD6wfHlASnITRjYzGE8xIkmuctygtcMGRFwkeiz1wtr0TiCCxo2NjNixPv+Dh6j4KZV7EdaNOkcYznUWBvcpyRFlineOhA6FkTIhZRyYLNus981lJXuT4JCEtJuRpSb2uybOM6XRCMSnYd7vIVneBqppyeXlFqiVaOWrbUebVIU1oKfOc4+WCSzzbTc3DdsNyecTFJ5+y2Tf0XcvR0RGTsqAfHSLRNKFHZ5rRDAgCOpcsJjOads+u2R3QD7//w3tzYI+Hgxga+9GcfxQPPwaIecTIPHL9vePQw8LHLowopPLxdT2KhfF5BEomh+cJHxnmIfBx8P+4Dvz4G38nvfzD/xOLRIX0h/1wTOBHB7A4OHv977z+EJ3bH/nej8gZGVG2QWKDPQyDHx3v4uPnElnt/qNISzhgS4k4J+c8IhhEGn9XLNkOCBLapj4QFYaDuz4OnMfRfOxyiC5oG7nsQRxMZ49lqIrgOZhPBNZ6Eh1TyD7EnqC2bX5gvBcZfd+RJBpj4trQe4GSGmPHj8WlIoLeMcZEF76QKJUcOnrM4e/VoVvjcehw4OYr+dHUipCHDrTHoYQAIXDG8vTZM6yN3/0wjAeRH/I8RciAMT1FkXN6ekKS6Mj715LR9CRpFPMfDR6Ponl26JV4/ANRm3XGEoKKyB4CSgsmk5zFYopUHucM8lAAKkX4eFzJj8zzSGeIBbWPSOmYrIjHkft4IggRhXGpDlz0EGsbOTjIpYzmHg5DpojIId5vnP+4ZgpYAoeuuPC7g5qY7ojLqf86JvkfLKJ7YcmTjER4hPyA7S1Cl+zGNzh1j/BTJulTtPiMYVxjhaEsjul9S65OMBYULUmS0dp3SBU3ZlKkSDbgDGV+EkXc8AbnJUKe0psNSVIS5C0yGAZ7hJIS73ekaeTWEI7QyYrWKDRzcrWms5DpKZIp3k3I0wrru8gPROPDnsG0FHqGtZZcnTK4jlL/lM7+VRQA9RalMqSssOEepMa6lEQp+vEBP/wIF0qsG0hSjVQf6M2CRMzYm3u0qlBSk4oCb1PyxDLaBokmqHV0K8qSu65mnp2RJRO6cYN1e7QpceRoIXChjmzR8J6pVQTTx4uvcLhgcCE2UqZhRnApw7hGMDA4R6K3ON8cSjAlWpa44EhUgGBQuiD4LSFIpC0RxIZ3T4jT8AMD3AeLICd4SarnSL9DHYpmMz3lfPkFl+E/YHjAOIezNT7s8OoEhMW7Fh9KquwILS2SEu8dWsaSFiVaEjnFhQFkFqeTMroVFEvsKNHyiECLdTWGQJUqpLZRhEgVzirqes/05AuK8Rk9a4RvUbIiiCOCvyFTBQOOwJbRdXhgcHsSXUVUUBAQckLIgBKtAkpoPD3OGYLoyZMCTRljbiJFSE8iLdZ5fHARTWR3COXxIjY6J2qOD5rgfGwRpkFKfbhpJKSqIjD8MLUNsYwUwIdDCZmLzCctFdZZBAnGeqxLGf/AlvAXn3zC332/xhhHOw4sTk8o85LddsVAj/Ianac0Qw86YTJbEIzlaHZMEClSBu6aDRkJE11iCbRu4La+ozzKSPKETBfMqwXeCqwZuN9sSbKU2dE5N80dRZkzU1OW58/xboISFXc3a7bNDpE5AglS1Tx98pTF6QLvXGToh4DTChs0Xd3Rj57Qd6zvtwzjSJoXeCF5++4DRbGgmszZ7l+hkpQkL5jNJ7R1LB58/fYNp5+coHTCOAz85Md/TF3vuHm4xFjLZ88+RQ+KVKV0zYBCMXSGXBfsmj0qEcyKKeu7NWY6QacZaSFIU0UhU4RX7NYbnHUkOmPsOtquJ8sVm3qLswYzzAjGgYdJWfL0/AzpDc3mnjGJLpLl8QIzdFHIcwZsF3E/QlJNZ1gpGW2MS4XgSZIUodXBDdxzffOBX/76r6jbNVmmSLN40w+HpnDnwFoXOxd+p6AjTRLK6ZQky0mL59zdf8pvvvllLGtJFCiP9R3r/R2nJ8dUkyho3tx94GEdWdSLxZJykrCvG65u31G3G0Zbs72+536z4n51yT/+l/+CvJTszQd+/d1vWdf3pIuC9W7D9e0VT8/PUS5jfnrGZy+e07Qn7LsjHjY3LM4KpkdztqsV64d7nl9c8JOf/pTvv/uW1c2WyKLL+ezTz2mvH5hP5uw2W+4f7jBYvLVs6jUneYZxI0fljH7omOkpR2pC4iyEgU1TE4qS/HzJy9/+gtFYjuYlucgQQ6DIMl69fsV8MsWYnqvNJbvxDmMG6rYhTaf8+le/xgdBmqWUkwnWW5rdhqYb2LcVRZIwmB7Vh//6Sfy/8dhuN4xjLIdZHM0oioK2bZnPZ4eEDeRpTlmWbLYr3r5/Q1nljHbENYG3b9/gnKDIU87PI/Nyu3kgSSvOzi746c9+ys3NNeNoeIyGeg9JkjGfLciyiAnL0gkPq9j27r0nyzJG07Neb5jNK9brDV9++SVKw3p1Sz3sKauS3W7Lv/k3/5rZbEFRJIxjw/3DNT6MLI+OyAtLkiisHQ/MXEmSpDgXkzJKpWgVBT4hNcZYghs/Lra6Pg5Lq3JK3IQk5FmFcS0+DIymJTDifM++fmAyzciTjIA5uDUKINB1Pd57lssFRZGx3a6QSpBmmjRTICRd17LbPwACIQN1s8PYkWk6Yxg6rB1BeIaxZV9vmM8XLI/OKfIBM4DNAnleHUTxWL6mtOJ4eczl9RU3t3ecnz+NC3eVUFVzqmpCCFu6bmQ77OMQbRjo+o4szz4WDKVpSts2hwV8PN+TRKOUxDFSlgXpwSEzDANCwPHxET5EwTjPC5SSB0ZnD8Iyjh19H7tTHh7uKIuKLMuoqilN06B1hhk9N9cPdO1AnifMFjlBFLSdpygzsqxgu22xpsHbAinFoeIhHF5fwny+YOgfqOs9SSqoJjlJWlKWM9I8lo4LmTKbT1AyIckKPBKhEpJMUJQlRVGghIQqkGh9cAzF+/l+v2M0I5PplBDihjPLE5IkxViDHPrDNT0WLA1jPBam8xmpknjvkCIOmjhsIh9xL879/uc2QJlVLBdLbm5W9H3D0WzCJy+e8nB/x2gVd+s9de2wo0ZIz8Nty/ffv+Lpi5yLz3IKNHbnuHxXc3oakQHr7ZZ+DByfFfzmm+/JSkFZaNqd509++lPm1TG/+qtv+PSLF2zXD1y+2bM8m6BVxdmpxHjLbtew3/acHR/R1iuOlkdMFyXOKe7vLWenJ+jEIhiQGqpZQTfUEByTqsTawMPDA9Za+l6zmM0JPibauq7myy9PeP/unu12x3R6wvffv2IcPS8+OaeqclbrB4yxzOczFosj/MYxjDW3Nw+cHC8xYzSilEUsXc4ySR862nrAGkueZXGDS4JQnt72TDNFPollyn3f8Olkxrbp+OA3qEWCGATYAZVb/uhPntL1K9Y3NxTjwLlr+dNnx5ylRywE5FWC9Ja1ErwtSv795WuEUfz27cDieMnNuzVCKVbX7/k//R/+z3x18WNO58ckUmHNADpghuEwDMgZxhEpFVpL2q5n7Hastnc8++IFO1Nztb7m6PyMbRMH3z/+8deMpmU2iw6zrm8wxvLh/RUvnj7j/OQJk/kRTgfa3Zbrmy1pWjIplwRb8/bNFT4kVLMEKSMvOMtzshL6bkBJT7oVjP1AqhVFUVFvDUNTIu2EP/njP8GOLfNpTtvsePfuiufPz5EKPv30jNHVrHf7Q9JHYoXk5OyIMSzQaUpOxfvbNVUxQcuM3S6+f2MdQ6fY7wIqKdltei5OviZ4TzHRqMQzmDUvPivRRcv3rz9Q7wLb7YzT82PevXvH/cOep2cvOD1egBZUi4Sbu/dc7ddsS8P52QX73YqykITEMBqHImVaHUOa8ONPvyBNW/a3v2V9f/N7n9uz2YREC9brFcYqELA4Lenb6AbMZIZMBEprmqbGCMPzL59z+e4G/zBi2p62rtmlAmtH6r5nOpmya3fcXF3z4sULtts1i6MFwzBQlhnlZELQgcsPN8igqNIJXVOz2655vjjhyZMT9uubOBj0Hp0m7Jqap09PGVyDG2Pyqm06hDTk2QSpc/pxpGnWhDCSZnEtPKuOUDrBect8XpJlQBDoRFOUZ+SVQqmU9aqlnFXcbe6x1kDikdISbMfx9ISTswVv3rfsm5bJ0RF9b9BJSllVPDzccX97E9cbfU+V5/SmYTKruDg5xStLbTf0g2cYB/qh4WF1zWhiOnOz3iF1gguGk5NTus0apaFpd8hsihSKfRuxEmmaMCkzjl2CUBmmjZx0bwx+HDF9zzgahJB4b3B2ZDCBzu0pywrXZ9hxRCYJ3dAhZIqSAaUgOIMQkAiBrddIYaM7046UWcr+bo0YDItJSdsFgo9u2zTNADDWEKzG2oHNesN2s6UfBjoz0ntPY3vyJIkmmSpFGMehcAERJFLG3932DfOjOTc3d2idRcxVPqEofExuShmxN1JjBkuW5Dw7O8cPI9PlguvbG9xgEAGGoSfJ03jtEin73T5WRnpPcLHcUkpN3wzM8hlVnpNUCVmecbu6Q0nFcnGK0ooiz2ibATP0ZFnJftOw27RMqwlCSaSWaF1yfLIEJWjGnubqiufPXlAdLblf7Vmt3/PlF59RZRWd6xEyIJTABosZBj6/eMH+Yc0kLdjvtwj/h927lYqfqxCHBCaR2xwOpYrRucxBIH90kUeBOSY9H529j+iUKIw/DrL+XslniMk+If5+ESU8Cs/uo3j8WFz5u/8dhf7Y5RZJLFEcR9iDQ/2HvWJ4HAoQCyJDsAfhMpZ8RhFWxeMaGU2iB+H/cf/gnP8o3IcQ3+cBihsdzxIQCudAi4QsT2maPRLFOI64AG1jcL6n61qSRCJkYBx7lI5rx8fCztG0WNdTFQukO6S2ER/FeykV1lgE4iBue6wbybIUY8aPw4oQPF3XxGL7EMjz/PAeEowZKIvHAnpiObC1ZBl0XU+eC9I0jSly/v5gQ0r58f0/dp/xiLr5CNB/ZBKDVJqqnPDVVz/i//Nv/01Et+1i+XqaRYRJlkvOz89YLGYx+dnFAd/19RX7Q3loTE2WH48NpQ6GqEcBXcQ9ghYhviZc7J4MgSzXlJOMJOkR0h+c9orweOwdIDTxPfE7Jb3hh8GR8ARMNPX6j3+JED/w932IhbxRLxOHYz8cjsH4XI8dDo+Fr87GQdDj70fEoVDsgYxpfR9+wAr97z3+wSJ6NzpGu8HJJXWzxRrPNC2QIiX4klkyxbmGNvQkaU1vevomJ0s0nkCQGu9SBh8o02d0tibJS7CgQosWT0CusSYnsKZIX9CNG3S4pB0LsvyCprcUaULv9gSxZd97XMhRYqDQASO2eNehtcSFKc6DFpFpbvyWEDSaBTaMpFrTtk8YpSDBHCgHmt6+QouOwXsSXZGJJUPYYcaK/PA++uGYQqcIekZzKOpki5I9zXhNEAah9jE6LBKEStE+ENR9nBwlCu8GVJYy+j3TSpLpG7zXTPIl3hYM4wM6CSSyxEpJqhTBlThr8b4m+MeJj4pMZKlINPgEJFM0MywPETESUiSeKj2mD5domeLdgAuewRqkAG1O0OYoxluD+9jSLLUmL0qUOPCICBg7kmYTNvU1oxk4Kp5A8hZCTZZMGO2WRB0RxJREZhhvSJQ+TBq32EP5kWdPEA4hM4KwjLwkkTlBjEjxCUrNGO01UqxBlCB7lJzTu7gQM2JLqmuEOo3XE+8ZhoaZfEKeqIi4CD0x4OSRQRGQkQXLHi0iygeR4r1Fkh+iTQatZPycQnSJSVJyfYoTfbzRhTgVVDi0NChpI1dLLBhdiw+xPA22eN/hDm61gCJVmuCIQgMcNuzhAJBRIA7iZTAEBEN7KN6QBi88nZUYG0sk+qGnN7BtGvj0H3o2//8+vvvNt+gioSwrrIg3tSRJ2W32MDoWs+NYIGsNOkuwATKlGbsNBEGRphRZhvRwc3mD1gW29zRtx0N7z6ScsqgUblix37Ysp8cIJZkfH5OepHxYX9PWNU+fnGJHhXPRpXxyeoyRO/qxZTI7YrOrubn7wPJohvQCKxy16VibMTobhkDwUCQFaZJRVLEctB4a7Aijcwht2GwaxsFRlVO+/PwrmrrhzetX7GWcpBdpRi4T2u2evm0w/YD1I7v1AxUZIisZxp6m7nl6/hzXG6QIrDdrkjRhnh0xuuiCTsuE7W7NuN/x9HiJsZ7clSTe0a53fLi5Ib94SlZmbFYN26Zmokt0ksRYl4am2ZEl6eH4VZhhoA5r8JZgR0xfRxa6iiJXYi0uyHhtymMbeIyfCdabFb/4xV/z7/7dvyYrE05PTklSSZIoxiAQXhx4jOJj5C/AwSWckiQJWickWcpPfvxzHh7WmLFnV99zfXdNNZnQjx3OjxErkSW0XUPTtugENm9WpHnOyck5SItQnm++/SU2OF69fc1yecpkKSknKWf9Ef/6v9wxmI7L6w8kaoIQUM0muM6QpGBcR16kfP/mgWo+ISSSXb0lyTVf/9HX7LZbLh8ueXvzjrptePLkCXfrB/7mr/4a6gGza7h4/oznLz7h+jc3WOf45NPPcDZwfXVFnqZMiwpFxc8//xqVOprR0PQjIS95u7pnM3ZIn1LmczIynHdkRUJILKMaMKNkvlxw9+4tWikkgaHr8DZwfHRGmidkQjO2A1eX14gkRRwHXr38nuXpMcjfvzg44On6hjRNcc4dIoMdZVkcnMeO6TSynFfrDZNZwWK54O7hOhbKpgpnLdMyp0g1+JSb/R3BS1arFaPpadsWKRVd06C0xIwWazxpWlAUFVU54e52TZokB6fzwGg62rYjSWJRXNt2jKNhmkV2aZYVDH2LsWNkayeKq6t7ptOSQMJ6M1BW+cfhY4zxRWe7tQ7vBGmaH5wesaRo3+5J0wypkri59R5rPVLIw+JLUhYTnBN0XY+QYyyh1ArnFD449vUW5yvSNGMYRtqmjS5tL+j7Du9ndF0cymmt8N4emIk9xo5xY3jos7B2AMLBfRIHrlpr7Bjd7FmWIWXOfD5nGCxJUkbHSq6oJlOGoaWua3QaNy7DGKOl42hxNrCYHxGCYDaNrukQRrROGMeRPMs/blLGMX4ndV1jjGEymZCmadwQhYAUP5z7/dCxXq2RUnK0OGK32TCaEecM+32MSDpnUBrSXNF3lrLMEEJGl46exoSLEAgiXzyK1HuWx1PSzLFYZhSjpyoLnBu4uVlDUEzKMwSSYewZhpHVasPx8ozTk1M2632Mp5YFeZHhg0RnOT4Imm6MZaXoKNw7WG92BFLyrMQTkErjnGV5fHQ4R8EGg7EDbddg3SHCmkUxvKkbIG5mt9st1lq6PvLcvXckiaKocnSa0bYNeZ5HR5NzsQzJWrLD+fGHPMqk4vu/e8esyvj0xTmbzZaiqqjKKTcPNR+utxzNUqZHBU9OP+Hbv7vjv/3nf8Z3b3/N63c3FHnOvpbkWcTKjdYzdB0P64YgAhfPpnS95/nzc1799g3bzciv/+pv6LY9r75/yYvPXlBvBoqy4PX7Kz75fMZoPMUk4PqR4Edm04Kb63ecXZzhPHzx5YJ+2HB+vuBhdcfl5QOnpxPGcSRJFZvNhof7PScnc6yW/OVf/hxrLfera1bbmqPlnKurGz759MWhpFgwm+UcHb1AShjHBkKg7faU+YTptOL129cIEdNAQz+y2dyhU8lkPsXbwLScsrq5YzLN8VbijSPYQKYyvvj6Oe/fvcF7wYcP1zx9fkTfNyzmZ+SzlD5p6eueqcu5qHJMXcMwsnl3z4+D5Klc8C+PppxLx0RolJAILZAeps7zyUnBWXHGhdX8X7+/wmYJZ8fH1O2Orm64evOWv/jiz0gDSG/AGVQi6Ycdxmq0sPE5AXvAGq36mre3H+i95RffvGV6nGK85+LsKU3d8Itf/ZrpLOP2wXNyfsyH9y1tP3B0fMp6O2LCFiss+6EhK3JOj4+QQlJkJb9+/ZLjkwnffnvJV18vMday3w9YE0votMwIbs+k1Hz2yRd8+3cvubuvmRQzNg+Gy90tD5f/nqL0zBYlz5+fcn5xSpIHbq83aD3hdP4pP/5xwofr39CZezKpcKFnebKgaXqUTjk+vqDZ1ez6gfMnxyRJxjh6fvWL91w8rVguTvinf/7HXF+u+eabXxHuRvKZ5smzCqEt96tblicLUh/ozB4vRkZvOF3OuLm/YTkb2O8cebVkWs1JEsXNzQNrvWG5PGE6S3j7/iXVbIK1gWHomBYF3716CaFBFTB0//WN+P/Wo+vifTt4z2q9YlJNGGxLUikYBIlMomt4HBnGDiUFXhs+/eqCD+IWRKCcCLISJlnFERP6YUSqiv12S/r551FYSVPKssAHx7v3H0BB1/TU24YETTAWZQXTtKLZ1OxWO4wYIvPdR0TSYDss5qNDs2kGunZL2+85PX9K3xusjdfTPM8o0znzyTK6IwuFVClts0WJhKzIOTldohNwHu6va96/f8d2t6JuWoy36Ezx9Okpn3/yhCAtRZ6yXhuMdRjnmc6nh/R6grMG0jRiLwhkacLp0THL+YLL+/dYbz6WCPowsNs9gBpxwbC6veXZJ5/hjD24dGNiSklJUWRoG/fgUqVImVGVCWk+p+tH1v2AVpJUSQbv8eOI60eq6RRrLH3XgbOHdLKklVu8SNB5zoijyBOk8HT7FWa0DENHVeQUjOTC4b0hTQT15gbfdXz67FNGJ9jVNU3bU1UVSZoxehfXnGakqWtW6w2b3Y4gBOW0JBGCPEtQPortvu0QCpx3HxNWSkkSpTDOEIju3JjWHUl1hjeWkyenBB9TBpNqSpkXHE2nTIoSL1vqzRY3GuxoaNqW7X5Hupiw2axI04ShjujPPMmjqiYUQirq9RYxeI6mc6zpWUwnbO5XHB0fM5vO2e63SCX54ovPGboeJRXzyYgZDYmWB2Mgh3uzj/g6a+iGnuvbWy7On/Hplz/i229+y+XtPYtpiQTyTNNbi04Uico5Olpw880leZVSJBmjNX/QvVvIEFnkWmItHxNwAErLaLQNjwK3JwQdnecHBIYQ/odyTH5wrUe389+/7gTiWk8p/p4Q+ujMjSJ4OIjYsewy4lUeBXSPwCNFdOnGSh0fTZZ2RIiAkPKAG4myvxfx2I5C+A+Dgcf3Hl3rUchG/G6a/tGV7A8IkFjY/shcF8IhZBwUeQ9eOsax5fL6DWfHCXWzAwLWepq2ZbfbMJmUjCaaOibTHBm30jhvkYPA2A4tE6z0aJUghMRaE8tZbfyMHt3h8drcHT7LOCAYhoGuS2ia+rDu9ygVGfdS6MOaOHapxWSiwLko5u/3ka1e1w1VVX1MjMfvUn5cp6dJhpQSa+L6Wqok4nseExEHHd05h84y/vwv/5L/9Ff/iV/96hdkRY4IMT2qFJRV7N9yzpKkCu0l1sb+p7u7W0YbizeLovi4b/xdMT1NY7I07s0ii10QYkI11+SFQmuQKuC8ARPd4/pxGOI5OPhjCiD2D/5ugkHEfW/wh2FHOAyLHtMCEf/ivT18TpFjHj+3w3EbDkMIFfn+QoAP9jCoCodjNkD44ViLqQqBEP7/vziXEGYI6VFuxlQn9NLhACPvUEGy6W7JkqeMYUPTX7LIP2M8tMB6taY3gVIKDCVhGBn8jsQLTnJH3c3w+h3N0BN8Sa6+RnnDNEsQ4Yi6g0RMScqOZnhDKqdM1DPafkuiS5A1NhiEO0IlPav9LYk06GTAhud04y1CNWih6P1t5GoHR17OMfYeXEntWhJZIiRk6YSEOc7VtP41qIwQLM6vmWd/RKkHGrvHuxKpHULe4ENB8BdouccZyLTCuB6pGjbjLYkydP1ApZ+ALejHks416HCLTjyN2yKp8KrBhAQjIfEphj3WGYKVZGkPQRJEAKWQylNkBf04opQgiBHjYqy5SDrSZE9wOTo5ph0e6NRvCKQMpiZQYX2LDzWlfkrefYqwETsSnedxIaSCRkuNRGBNh5AZbvTgoMyPuN98y3Q+ZcN/ZAhrtGtQSUAzwQGDXZPpI6wRCCnxwmLcgJISJSCVCk+OtTGyYmUgcMHoViQIUnWKC3sMb5Aiw4c9PpQIaXEMWF2TComXEiui2HjuLtDM6cNvcX4klecI0cbpFYJEGgZXI+QEKQTWjwihCT6WzI5+Q5bMcG5Ay+QQm3l013tSpYAeJVJUZA3hfYkLA6lwVMkRIQWko7cekZ1g/R7jRpScwoGjFhhJdBHTFD4lyEM8LlikzLDe4VygNwngsKbD2Ip+1LSdpxn2rOortvXIrk7hn/5Dz+b/lfPbOLIqobORXzyZpAztQJ5kPHl6xugCRsL97TW+HciqikxpLl684ObhknZsSZG4wXM6mZMWM6RXLMsn3O4+0A0DY3fH8fyE6XTCfD5h/smM6/UV63tD2w8sJkcomaETyXZ3z+q7S4xrcT6QJjMW0wuUXFPN4X59jbWCFxfPedjEUg/syG67pUhTFlnB+dk5L9+/Yj+0lNMJ8+kRu809R0czZtUxIYf1w5rteotWEu9GTo5ntHVDEhJ+9OVX3H24p+0aJrMCrOXdh9dMspy2H1kcn9JZw3F6BgRUSKjXNcLHJuykyLhZ3XK1vqau92QEigSSpMS6WFwYpGRT77nfbVB5gvWO27sH8ouSi4sLXr1+y+ur98zTKUfzBYMZMNZzdxuYVSXB9LixixxzpalmC4TSSGMPLsuCokoJwdENBucUr1+/4m//9q/57Xe/Rih48ewT5rMZgehQTRN1uGn7j23Wgrhojry8eMwokXC8eMLzi8/4/t03dO3IdrtjvlzQDi3dTU9VTphMp0ymE3SqGM1AmqfMF1N+9Xe/4OnZGWWes90/8MWPvuCX3/w1SSu4XP2at794y8t33xEUPDl9zvp+i1eCLMmwxlEWKXerNzzcv2M6mfLs+QveXb5j3dzy/MUzRJLy8t1LBjNw/3DP3faWL198TlVNuX94wA6Gk9mc7WApJhXX2zuc8xR5RfCAC3z1+Vd4YziZH5MdzbhZ7QmyYbXdUO96vnj+Y9Kp5vb6kjSdsF+P7MaWP/rJ11ytrrjdX2G3hj/50T/n+GjG3/xqj/CeIs3QKE6PTvn6Zz/l1Zs3nC2WbFnx5Wc5V/c3SDwnxwtUkrDabX//k5u48MzzjFh0YwnB07Yt1lqstSilaLoW6x9TSJauH3DWsworMl2yrhvyXDOYGhEcu+2GarIkSzPKsqCud0gJXdeSZfG61vcjTd1zfvacy8sbJtMFIThubq4oypxxHNC6xJi4SHr//gPHx3OKojwcbwbn5O84ZAAR3Sx5kXJ7d40QKVVVYczAZFJhTMrr128ZBsNifowgiqNSplhj4zDTRfyKlEAQzGZzAo5xsMznc8wIm/U9AThazigO7zfg6doeZwJPny6Qs4Tb23u8lxR55KCv12v6IWccDWmm8D6W3o4mCuZVlSNFdHNHZ7c8iO4OrRLyPMGpQJoq5vMZ46jYbPaRDauSGFUVMJtN6XqJtSNjMzCfz1ksT6n3HX0/kmcVVQVFXjGdVKRp/zFmOgwdaZZirUEnmjRLqfd7nLc0bR3d584eRI/IptdaoVQsae77jpOTE7SWHzElCEfbNxg3MJiOPMtIksinRMB0OmXo7xmGnqOjJVJKXr16zTA2SGVBBlwQJIkA3zLamgKJt4dy4+kxzjraZk837BgGcygyndMnjul0hpRwfHyMVFC3ARfAfoxwatI0OvSTJLIr15sN02kspUYIur6nyDKsGenahmHsKG2Bx6PTWNqsEk3X9dR1g1KKJNFsNyvu72PKYjabUhQZQiasNyu8j2VVeZahtMYbi3cBrROm1ZSi+P1xDwDDDkwbXVFxMLJms9sxKafs65Z9XXN8MgExIGTPn/7pBb/+1TecXCTc1GCs57NPXrBe9STacXK+5Ne/uuRkecJqvWF1v+H86Tmz2ZSvf/IF71/e8s/++VfkBF6+vOJP/+RPkLrgbvuG5y8S9u01oZzGzy4rUcJjxpZ+6LFuiM5939O0exAFx8cTdKK5f1jTdZZPnz+jazrOTqfc3t6R5ynOdDGKfzWy3YzMZ5r1amAc3/LkyQldF1Mj11crXrz4BJVBCIbT0yOG3jKdlTTNwKSaMfSB1cOaLM+QQvDu7S1BFFSJZVJUJMKj8hw7Bsq05PX1ij/9+ZI3b75BJ5ai1HTdltEOBGVxg8VsVpyonGepRu56Mp+xUBU/+otnfKFT0vUthV1B46FKkYsTfCaxrkftd8j9hh8DOkl4m2uGeYYsAsZL/uSPfsw/+fM/x7YtxiqUjNfuTjuU92y7HmfmGOtj4ZdUIAUyVXTWsGlrfvaPvmawlvfv36PChDevXvHV189p2jUmOF6++R7nBOdnL9AqI9Oaut1y9XBJJj06iWKClhYlA19++YTJJOfoeE9VVazWG46Xx0gyzBjY7W6QcuRolvD+9SuOqiM+v3jG1e23/OO/+Iy3r1r+5q9fMxpHNStYbWpevfyeP/75MYOJ/Rm//fZvWJ5OcaHn/mHH6ZEim5QM/cBiec5q1dDXMK0+4burX9Oa1xwfH3G8POXP/pun5GnJ5bt7/u3/+3/iX/2rf8pi8TNuHi7jsaE817cbbMiZlAVPP0m5vr2hnE1BK9b1jtPljMUi4xe/vOTbbx/48sdTjk8qnj15ijGO7XZFVcyZV3OGsaMf+yis5prTiyVdKxn9nqLIfv9zu2/puoayKvnwYY1KUkxwjP1AkU0PxhuBDJFTniUJnWtRqeTisyXOW4SCoC1ZmXO8PI1dDruefduzWq0i99cayknJbD5lv9vxcH8fO83qlklRoYTk+bOnLJdH/Pa337C7rxG55/jJFGNHlBYMvgPtyPJ4P2nCEPn9NjCO8X77GI8viwnz6og8yTGmRScOMxqkk+R5gR9hc79HpVFQ6eqedt/hpQUC5XSCVfDi+VPm04Jf/faXqCwhaMkQAuVsRpCK3X7L0ycXvFUSlUg8AeMsXd8yHUqevfgjPty9Zxwj0zfNIsKw61rSMuDsSJoVtG1NNYmlyGa0gKAoKzb7PeiCoDQGSSAi1cTgMNYzmVXoTuOtQwfIhcILQSIFZjSMw4gZe8RheD44j8pylHAEJSmFZ+x2jG1PvVkjvWXV7DhdTPns+VOUhr7ZIlzHF58+I0lS7q8eqJueyWTOZFriZRzYegL9MHC32XB9/8DoHPm0RBYaIwJGa6QLeBcRCjpJkCqiPo2Je1N7ENFGYzk5OWa33TOpCrQEM/SkiWa/3SICVEXFJC/wJvD9+5csZyXbrsZ6x76tqdsmNuAYA4fnrOs2ouimFUES75cuMDQDpu5JQ/Sv1mlKKjRucGxWK95eveP4aIEQjqP5gslkhp9GAXIYD2WKWmNcLA4sD+teY0aG0XC/XnF2mvPFT37M1fu3dHbAuY5ympNPK1KZ4qzj8uoKpdXBOZxhzR8mokesm/oB8cEjjxseXbKPomDkmj+iVeLPfyjuFHgfvzcpH7Eo/wvR+uMaOor2QoiPLu9HwfYjluR/4UaHH5AuAhAh8tCVjO1xSsbnctYhiK/DAypRB8RKRLKkafrxNQvxA4v68TVqrT/+3kdhWkr59xAz8c3EnhlrYkJ/NAN7t0b4wOXVS/ouYEaH0or3795RTUrquqXravb1BmMn5HmC0gFjBnQSMYcyrKjKBXleIMUPwwUpf8f9f3DVQ6DvW6SMuBTv7SFZOrDbbw9ptfbwPUrKKuKurTUgDgOAQ3lpkqQHV3qOtQcctZCM43gwtczi73aBNM2ioG0tUumD5vzI8I5YlySJDPY8y/mLv/hLbm6uef36NXaMCdaj5ZSyLHlEcM5mM8ZxpG36w3cd8ZvOe37A/oTDejd2WEZGfDTDjH3EPSutD+hXT5altG2NTBwcrtvWyL+XhohMfn1w2R8wQeJRAo9diBF59Oj0fzzuQQoVX593h8GNjMmI8Ih9OfyRjx0Aj4OiqEVKYpmuOGCEHgdF0eQUGe4RkfNfOYf/q//i4zFbMlVLjtRn9PaBbdjiRIe3M+b6CVpdsPPfoJTH+jMIxxQaOnuFdxpJQyqW8cPVKV2fUmUFxjTM8qd8aFak2hD0nkIVjN7iTIeWOb23iOEexT1KztEhMJgonKZqTfApxkGlA5KCUZ2SJgbrB0IwdF1GVQT6sCGTiixNaceAVjsSWRKCJE8lwXYkqsAYjxF3KC/J1SneOXy4QQZB29+RJkck3uPUFhU0OT8B2TH4kYChyKeMJkUIB8Ei1QLvPBk5bbtnWt0hmVOm4EeBChmBJcZJ+n7A63fMsjndeE0/1lTpOUJUtDvDiZsc4m8eKRRKJ1gfGH0budqc4VnTmvckYSQRXxKCRUiL8QU+dCg9g6AJoSRNcvApsp/jQ0AojXAOBBRJDv7xYDe0/ZYsmaJUwv3dLbV5y2S2IJv1PPCSJHF03T0mzMhkjySA7xnHe5Lk9DDJDAhdkyiJCnOCPyNRnoQl3h8x2A1WTtFySiJaehtwaCQeHzK8HSLvVBYIsYZ0DcQokFOebqhxAxxnf8Gm/x+RYkEIHcav8U4QvCfJDcZblDKMdKTqKaO9IUuOEX5O4CuEaAliZAxgwy6y/GWOxGFdSyDFOk0qe6S8RognBDQ2GGb6AiESBvZo7RGiQ/oZwQQkUWzPEnkYzIy4UCE4oR/fAgnOtxizxVpoxxbvJnR2x77est8VXN3XrHeW3d6z6wLW5QzjP/RM/l9/XFycE1KLqXsmeQ4iQ7rAs4sLNpsNy/MLBhFwdyCTFC8ETsDgRrrQs2nWmHYgISOZL6mbPYPz+ESjZcLxkxOaXQNKYr3h6uaSwdR0fsNfffNr0Cn0loUsyJXm2adL8mnJ3cM9L799jXAJqSyYTwN19wFjO4Is8VowOzlic2sxwXHx5Ay/7fBdx/X1Jdv9BplLkIb17gYRBG0tcKOLDvq8oGlqBI6z0yWn58e8v7lEeY3petYP98znM7TS9Lser6GVA/vQsl0NFNmEm/YeFRSbzRpZaFarDfm8ou5qsjKlaRtsMGzXD5ws5mQoPFC3LTZ4hHckeUZQAZVorPes93v23/6Wtu8Z3jl+8vwrmm7PvhnYbmvKrODzp08xzR4z1PSmYb44RqUZxoPMBmSSUyGxNmezWeO9ZDSB//A//3u+/f7vkBrqZse792+4SZKPBYnDaEmU/hglfIyPJWmCTnTkKqp4w9Yy5ZMXX/Lq3Uv2u5azsycordit9jw7f0FVTggERjMyjCPjOPDVxRO++e479vUWs5zTtAbnR95/eEdV5bx5/z3/j//xjnboMV5QlnMIkqEbeXJxRpbGabzScHI8xY0jR/Mps/kpr969YbW552c//5r71T2beoPONFY6WttjreH0aIkLnt12i/WeixfPubm7o3M9p0dnFEXB1ftLZpM5s3JGkcfSwMYFiukEa3psgP22pXuoOZldsMznaFkSjCFIwfvLS2RqGeXA5f0Hpq+/xw4XpAfcgxKS+7t7lvNjLt9fMquOeHr+gofrO/Ik43R5jBlG8jQlzypS/fuf4FLGFMyj+8Q5R9d1hBAwxh6Ykp6qnFBkJVok9N14WLymHxfY1lqcdyyXx3T9jjQIuq7laHnBcrng1auezWaN1gnWembTBUmS0XVxIfzkyRlv377l6GhBlqWHhamn73uKYsFsumC/39G2e87Plkync4y1kWl+iED2fYcPI33fMI4jygkQLqKGUk1Z5RgjqSYlITRMJiXD4LB2xPucYRwOze6eYTC0bYtzPXkpyfNYIF2WJb1wHJ+cs9utCCH++xAkWicYa3h42KBUQZ4VOCvI85LptIqC9oGFGDcpAefGKKoeNg9FUeCdpuuj+8J7gTWHxVsiD66IQN8PCCmZzRYfmet93+NcLJBSWh6GBprdbkvbtixPYmSxbWtu7wRPdcpyeYqzMVoqhKdpLJPphH29wzpPnmfRsZYohjGWqT4yUKUSCBdItPoYtx2N+bjo7fuYPCiLHAIMQ0c3GnywWBedPGmacPBBkaRxk53nGbPZPDpYlGVapuSlRKiIyOmHFmMG+kGhZMpiEQcBXa0Zeo1WKVlaAglDHwckT86fEXCx1Mm0QPz9P2zMiL0uhzLYiKpzhDCSJOrAmRSMdkBL6IeWzXZFmp+DjFHbJEuj8GEs4xhTBVmmEAKsjeJLXqToJEZEt5s1WT4lSVKGwWDqhnEYEAhOlicYOzCOfxgT/e7+jvPnS9CO9XqNUhohNfd3a7wb+OqLY9abDSbLOPGBN+/f8ezFKfv2ismkYL9zuGCpJoKrD1uSRPDzf/QJmU74459/yr//D3/Ldrvh4eGBv/yzP+KP/uQFy2nBi7MpeSH5/uWvmC4yvvuw5/yTgtC3rB9qqrLEOcvt+paqmlDMEla7dSyFNo5yUtKbluZ2y3QyRyqwVvL02Rn1vuHXv37H8ekRg2npXc2r9/fMl1NGIWnHnsm8YDaZRMccgjRJebJ8wt/+52/54z/5jCo/ItU5dw83PKwuKfPApxdn/OZXl/zkJ2eoRPPy1SVVMUVnms+eveD9u+/w3nFycsTYe6TKcF6w2mxJi4Tr+2tUmtAfhk/fv3zDbFJQhIalcPzLn3zNp/Mjxt2WdLQkdY8eBxAd9lCgJ1QGTuJ7hfUSgiWzBj0YnobAn04L/sq13N1vaFrwRcp2c0dZCTI3Mp/PsIPFjC0hWOaTiBu7Wz9ws77jybNnTOZz7va3rPs1t5t7wv6OgCTPJvzn//jXPKw3vPj8KVIr1vtbpAoIlfCf/8uv+fGPvqCawc3mkt6NtF1Pmmacn52z3+zxfqTMK8ZxwIyWm+s7Vps9X371FVJqqrzk9soiGHFpTPKMtmG/X/Pjrz/l3fvfUncJP/n5U+q25mg5YbdrWcwrlMm4fveBxfGOJ+cTIDDYCYo5N3d7jIM0m+A8TKcLNJZf/u235Llkflyy7zZ0lw3PLz6lbx1plqILw7/9d/89y9Mj5otjunHk8vKa0YMn0Kg9je1RKkeGjFTPKIuUh9UKrRR//pfP+eabV+z2a/LCc1t3vH37wM9+9pT3b9ccnxS8u7xhukjY7B4gQJVXjEPDYlnhd7+/0CaFo206inLGdLrAe0mWV/RjiwuBahJRYmmuGE10CVth2HYbTmZnmMHR9h22cwzGE0xCGCBRCc7sWT+syJKUxSLyzI21nJyfkOWK/bZGBkGZFUjg+PwEtGC/60hVxeK45OLpE+puIC8TvBwP9+KMponItqqaUk1L8qJCp2DcjiwtKYop3gnqoWYYt3TDBmM7JBluVBRVRb3ZEdQY4/fGUmQVlhEvR0Zrqao55aTi7bs3POzWHL14zqigKjIaM2J2O2RwZIlGaslgDaM1WCHIipy8LPBCcHx2RnvVgBRY51BKM5qRKinZ7lb4McFsHMZWLOfzKE5Xc/rOsPOW0bc4B0WeUlZTVJrB2KETRaUrtNL4wTDsG5zW9AGGrqdzMBgbz6eqwCJougYVAjoEUIIiyzF1w/7unmG7QtmeaZnx7Mln6ESgQiDRgeXTc9I85361w4XA0fKE2bQCYentiB0HLIHRGe7Wa243GzweLyx17+nHAWE8qZecnZ1yNJ9xc3fNZrMCmSKRBCEP/TaRve194MmTc/q2RWBZLhdYa2IyTmrKYoLwcH93F3uLxh4rwSG4XT0gTBx8X+5WjM6SJgnOefquZ7FYEBRMJjO6tqdIS5rtOhq70hxc4MXT51ze3LHtrun7lt3eR3fxQfTEyXitdgPWWgSR2z52sVC9LCsmRYpOM0RSsm8GgkxZnC0Zux3dpkdsBi6ePufs7An3DyvevH0TxTUR8KP/A4noHARH8TsiYTig9KKzNw6dfihsFEhiseLHK8RBLAwEH82PcbluPyJAYgmojMbcx/LPw8/EgTDwKNRHs0EcPjwmM9RhryelAC8jf/2QxPePhaAAxAGMDQ6EwwewTh46YSIWJQRi0aMIB5f7o1At0VIcSABxSPAopFvriEJzfA2PaNtYEHkoUCXQ9Xu0FAzOcPl+TdsYsrSg63qm0xmb9Yabm2tUIum6gSxXyEMatChTqmqgbzxhGT/rNI25Fecin5yDSzkOKgRKSYwd4mce4jBgGOIe01rDOEazSZqmlIWjKAq0lvTdeEBOFvF7IRwGKZFX/9gLJWX8foyJTv48zw/p8EdXuEc7f0AuSQgRGfnD4EVQlhOOj085Oztnu93ycHd/GNwkFEVJUZQHwTsHJOPgDj+rQAaGfvz4PcRjITrQvfcfxXREQIv0I6EhzVKSxKOUZBgHMhMQyh6OWUNQHPYPHF6nIwR1GNb4wzEcj0PnetxBRAc+po1BHApe43F/OAx4ZM0T4iDAh4A4/DcHl3wsBPYfhzXwQxrjY+dAcDjnUfK/jmv6B4vo0mcIVXM7/i2FuECJgolWeHnMQv+UPrzkODwFecGqv0eGBCkdKWf0rMBNGL1EpT2KLQq4ra+YZSW1+4+U+oKgJkjXEESGDJDKJb1fI3yCdwLjz1AixqbSJCFxKcZek8hjVFpC+EA3TikSz+AcaSgYwwNZkaClpJQTgtiiEWQ6gLpn6I9wLkP7lFwbhJhGpo/OmaQJwUsGUZMxQcgRISu29p6EAcWURCisW7Mf75EqQYiUwW9Q+ggRMrQ9QcgdTjiqNMcHQz148GumqkImC9pwi9QK6xOCzMj1BhcUgxnwzuKTHe0gyN3PkS4DoQ5A/si5SpMEvKZzFsIWEGTZZ1j7Hi+3jMZjHQiRIsQE60cIW4QscL5DmCOwRRTlhWIM8YYjZSDPYgmWD4Hr1RvydMLJ8adUy4zd9po+ewmZINgbREhI9IKEEpDIoBFiAWLA+SgmZGlC8HNQjjG8RwjNMF6TKkWiL5gkzxjdA0GWDLSAJlUdzicEn6BDLDob/UCiKoTWCAVKKxIhwFn6tua4/Odchj/CJRtG+4ASmqB2FLkAsSOEH5Gqms4avBiiG8PdoXWP95rgJFKWpDJgxQQp9iQqZzA9g9sgmeCdxqIp1RmD6wjkeGHZjWvy9EsMKSLs8L5HsmSSKHxQeL3F+h7jO/reImRN8C2jC+ybNcY2NG2D9yl3+ys2G9gPns3GUdd71vtAQBGCYhgdSgXgD5uIj0nH6fEZTdewmByBktxublh3ex72W5JqygjMiinT+Rw/DrixZ/vQMtU5SXnE2u4pyzntMJAqsKNloSQnx59w1+242e2YzCXDrmcic46TC9pdz5uXD5yeHuFcx99d3TI7mpDuSmYnM3brHUmWUEpNqkbq/RqLYzI5R8kFu0ZwfXeJF57JZIIfB27XN3z26Sm13UEiSLKUttkzzWdsVh16qpCq42G7QaUpWVGRpDneBT5cPTArlwgChpFQOpgJbncrknLCfjPgrScrovPVDHvW60ucCZjecnH+BdP5Mjp5m4aj6ZShrvEiUCynvLp+z+nyGRcXnzNf1/jBYoXjrl5Hh3JSMFss0EnKbr/HBE+/f6AZTvhwPXB7t+Hmds3RbEmmA/12zdA2oARHpxeMdmRf78jKGVk5xwmJTlP6vifNU95fveVf/4f/J7vmgdF3KK142D8wm85Q2TQWvARBnmcYa/EOtMrIiylCpREZE0D6gAJ0knJ++pQXz37E3WrFtr3n8v6GsydLyixl6FvutiuKSQmKKCY6w8lsxvnZGa9fvSIvMv78z/6Mq+srLi+v8MJxtf7AdL7ED4pEFQyt5+LJUxyOdr3iZ+c/4eTJMffbO3bdjj50qEJwefsdooDru0uSoDmaL3nYbtAy5XS6pOtrLm/f8v3L19gQuM3umWQFny4vOEoqsuo587OS377/npvVLU5LqqrCB8t0GrjfrqjynPOnn/PrX75jv72jkM+YJTnXD7c8Pb9gHARHxwu27Zrb1YbRekbXcHnznvfX97jEk81P6HzLQwPD/cAnz1/w2+93bNstWZ6Ta0Xbtdy1W8q2RyW/v5vt6GhO13Vst1uqKjKpowPdxUVRkmFGx3y2IE8LcI79pqbKJtEVoTRlNWUYDKvNFlRJXpZ4HE3TfIzHnp+fsdmsDjz0lMlkxmx2xIcPl1xdXZFlmt1ug1KC+XzO7d0NR0fL6GQYDefn50gpWa3uGAZHllmauuf5s09IkhznQmSYtx1pnmKcZhz7GMeUEqUKQnAIEcgyBSGW2YUw4sPAan1N0+xi/FBXB1SRJc0SpIgoljyPXR37/Q55KPtxB9zLfD6PqIl1RJk0TYs1AWdjbNVaE8XoUX10WQyDwbpHxrg+bAIkWZZzpBRtV1PXLUmiPpZ49l3/cTN6fX3NF1+cMF3MuL6+ou5qqmpCmiZImWCdwY4d1jukTqIzHMtmu8a5AwvzEI11zh7Yxx1aR1Z7P7QHnnmKUoIkUVRVgdYqIlbGgTzPo2vd20PZUA/E8tPgPVmSRAF+6BnGjnHsESqW/2ityLKMyaTEOcMwdgxjx2b7gBCeJIU0jXzGLFNYP9J2cUAiRMBZiyNwdrYkTxOCy+lai7EjVaIp8glS5EynR8xnJ7TtjrbffORTyqAQwSJR0RHdH+LzxqBUZFubITCYnm7ssS5y0MvpBOcG1tvAMPakeYZOE0IQGOOw1hwGQOMhRRHd+H3fUe8lSseuE+MsQhRoFdN2fW/Aw/FywXQ+oWvayMH/Ax7FUnG3u2F2NEWGuIMYB8PN5T3Pn59yNC8ZOsP1XUeWt0yKCfvWsl57Tp9PKI4F1rYkScKXX10wneWsVtc4s4NbDWIkTXPmC8X3r37JxfFzUiTf1O8oyiO6zZrV7Yaf/flTBuu4fb0jGMHYQpqnyFyw6TYcHR/HgR2KD5crnB+YLxKenM+4uXtgNpnw5HxJOzxwc3+LzhzdaNCZY6Sm9yPvXr5GpSnTyQRHz+gkebFku67pGkN1Evjv/ts/5v/2f/93fPnVZzz95JhJ2bLf37CcO+7fv+RsXrCcp4zO88mzH6NSzav3f8Nf/fVfs1xWNM1A/fo9VTWlG1OKWcXdtiafL3jYXRFGGDpLVSpOzxbc3z6Q4Xm2zHkRBubdPWQOU+8R+45kMacVkKsJIijAgqjRTiKHDjfW4F3cgDYdPy1Lvh0Grq4asmLKpl3xi1/9F/jyR4zVnPuVpG0MEoszG07PP0VkBa+uXrEeHrgcPiB0QetaXt18R5JqnPGUxZzXb77l9iGuq3fbmuV5RiUiImRoR87Op2w2a2yiqE2PtZo8n6GEZnXXst8ZbLdifddycjbjZz//E75//S2z2SKyu1c3nB4tmBQzxrFACoulATnw/vY7dkNBWmSs6wbvLX/6pz9HEjg5mlGv70j6nrN5SZIpCnJSueDVzQ0uSGyQ3N3vOTmp0Gqga/ZMqzlFoen7GtyMxWyOHS27/QotE1TaMZ0VmAfPdlcTKDEjPHvyJdv9DcbXBDcynx4xqU4YOsnlekdz3+KFxNuWsxPH8+c53gtevlzx+Ref0rY5V5c7zs/m9N3AV1/8hHeX71ht92hqmu2e48UMRUWifn+ci7EGpRXDMDCdzCPmUQbm8wVlXrKYzRESNtsNd3c3jP2ICwJnPFVpKWcL1nVHV4/0+8D+xoABVWoGP7BYLEiyjLOzM75/+T1SS4oy4+z0lOlkxvp+gzM23hMKjRWOk9NTzDjh/NmC6WKCSmrSXGFsg5YCO0qUGplWc7I0ReiACwFTd5jBokRCcIHb+5uDKLtnGBsUimBTRu+pTud0fc+u3uGxZEnK0eyIzX4NiYpuTuD67pbLd2/pMezalqPjEwKw3+3AOnKt2W93JEm8Fus8QYXAbDIhrVLe37zjxaefYMJA222jOzpEoXA4DDubesNkusB5zXq14mRyTN87Ng8rKEo27cB0dkRVFASlsMEjE4UmFrZ769BaQgjkWUrdHJBzXjMOhiAcSEXddazqBlk0LJZHpEoxtgnSjMxyiVQVykg+eXpOlafx3iI9eVF8LLBOsoyi0qReM5tPqOvdwYWd0I4ju7rmdr1mXe9BQJ5JsiKnrmuSAHVdU+93TGcl5xennJ0tWa/33N09YKwHXcbBvws0Y8vRYs5tvcU7y9n5BU3XkyjNcjnHGUvXjyiVMVlUdHaPfSzvE+BFYLaYc9fvMZ3DmuicVTIOtHWe0A99TGxNZ2zv7tk8rPj0xXOmZcmmbqKpIg2URRbXKYlGKsl2t4lYIApGZ9hutrT7GuEjCsK7QKZTsjQhSxVeAcKxqzeUpSbJFUfHC6Y6YT6dUGQ5k6qKhiAb+d/Sh49C8+//kBFrAQfxWx446fJ3xOJH0Tvicx7pHbG/jY+sZyE83sUizhDcR5H8h6JPYur9wE1/ZG1HPIf8KOI/ionG2MPPNUpFhIeWkuAiU9q7gCMK+M75KKofxMdATHag9KFbT0CQWBNffEyUBiIr3SOIhlD5O6Jl5K9LjDE4F5BSA/IjGsb7RyFdELwDBvphjZIjKumopim3N9cQcu7uVrx585ZoYslYdw2LoynODxjTR/PN6JCiPWCLHdPpAin0x8/B+4BzhkeftNISYx/XfvE7sNZ8TAv0fccw9OR5jhSK3X5DHBAI+j5FioQkyfHBYa1jOp1gRktZFh+F7si77w/p4XAQmMMBDRgTZ4cP8yPeG+IQxPmYGjk5PWU2X7Berxn62GXUdyP39w9UVUmW5TgbkEIdhOsfMDKPx8fjQ4jYffL49/EYdaQ6Gp04oF60Fofj+JAq8BwMXBaSv5+6eExeaK0PqV3zMSFhnfl4XCqpESr2WEmpCZ6IF5KPQvgP7/0Rl2itO7zWw8d0QCMFH68/OP/Rif44yIrzpIO4/w+4d/+DRfRpJjF4TP+eRpoo+OiAZkHjPuB8T5A3ZLwgxeJ5S2IXhPCMQiUcVRJrJFP1lIfwLUXeMrNz1u4KqTLKpKI2l2z3N5xPJIGRXCc0Q8uilHSjosqe0tue7bBlmUxABVobKGxA+A7UMfWYMtMCaVNquyaoDan8hLZtUOWMTOWx0FQ5OvMFOrygSB9wrkOFPZYHyuycsc9xAkywECz7/oGjSYlHYr2iNi066aOob+dICprWgnQE9UCmH0AltD4j9RBsHuMh4hIrRsq0o/MWzR4YaOopy0nCMNxjncXYloBHqZzROBIxgJsh1RS8IITh4wEYZJzsyDASWJOnR3SmR1DR2zuUPEOoFq1ymn5LiUGww3JG2/f4tmPhiBymQ9w/Th4DaRKZoVJJynnG3v6GLAwYNTI/kQQ1sjIv0eoYYycE7lFYrO9wPKBVwNsJmVoQ2BGCwTPg3SlKPMGwxtpAzhldf49IWzL9gUSe0NkWqSdIobE+UGQJXWcRPiB8g3cJztdxMiUV2hPLE30g44SZ/jM29n9AMCXRBm9jnIswoOiQPkcEgxOGRKcQRry/JqgjRFgQxDoycf0dSpzSmhHrIEvOCd4ghSZVZyAciBs0kiTxWP/Arg9IrRByh0Yx+jZeZIPCjPdk4gXOlHTDNe34CmcT6rZls9fs6pbr64bdPuVhO9IZQWcV6y1MJinjaMhSFS8wwDzVbHf1P/RU/l993NX3hAO/a32/Qeca6wz7rmYwAxLB2dGScXSUOsM4T1fvKfOExXTOpKzIywmD8cxnM+zgSKTg/foD03lJvW+QnWW+LGirBOMEa+8YDzdjM1i6/R7Xtlzf3yFnGT9KP8OMPfV9x/PjT7B+iMzxaUI/GCaFQBwKrVwwDL3k5sM1J4slu3bL7e095xefMpoREQJPzi+Ypo7gRq7vb8iKDC8VdddSBsWsmtHXA2mSM5qeq9trynlJa3vacUAHOHtygbUd93c3SGIUyg4jVTlj09V8uLzk7PkzVncbysmE/W6P857W9qRpQlvvybOW5P6ePEmoipx0MqEbaoZdS03NtJhTFBWDGVGpph96NvWO/bhns21ohpaw9bx8p0lCYLteUVQLxgCr+3vW2y3TRc9s7rDu0W08sDie8T/8v/57VptbnBhxxA1aCIG2beNNzFiqosR5H9EE0zkgCeKxET1ea/q2ZzKJjF+dJfz8j/4Rl3eX3H9zT5ZWlPmESTXl6vqKh4cHllrEjdDpCX3X4ZzheH7Kp599wt3dDevNCmNH/uk/+UtevvmevV8h0eRJwcnilHpbY8eBD1fvWUwrlAps92uubj8gtKTtWm5v7/AYdJry9u07nh5fYINE6YxEWT599ineR35omWe0xmIJWCkRieL5+XMaM5BkCSenx1w9rPAh0A0Dfbdnt9+ymB9RNx1j7/jR11+SJ5q81JycLmhMy2h7prMp337/LfkkRycpvhc87FecLE/IpxMMBuMMaa55eLjnq8+/5Gg5Zf2wQSeKrms5OVnStjvSLGUyr7B/QIFRWRW0XYMPlv1+h/cTqqrCWkcIgizLGIaB29t7uqY/8AIFxTxjs9mglaZpW6TS6ERye/9AkngEGUWR45zl/v6e6XTKyckZl5dXCAH39w8MvUXJBK0i988Yw36/5+nTJygt+eKLL+i6jrdv31AUBWVZsd9v0Tpls64RKPb7lhfPTw8OGs0wGoRSaKVRRYGQCQC73YZh7MjzDKkCOiUmw6qESZVxe3NHCJaiyKjKCdtNQ9N0+CBpuxrwTCZVdLSX+UfmoU4kSarI8tjkMY4J1qUY02NbQ/AK7w1gPsZnxSFSa0zcsGiVk2aRRV7vO8oyYkDkgdWulCZN8sM9XRzYvoarq2vSdEZAYezAMAxkecKinCCkpGmim74fxgPf0BBwSBndMuM40NT1IYLds9tt6PqaqirI84zdDu7u7tjtduR5znQ6Jc/zj4z0EMLBpZ4wdAN1s6eta9Ikia6vQ4lP37dsNmt2uzW9GSnKEpT4+N6ia73De0NeJAjp6cfoYp9MKxAReTOMluA0aZLQdS2dc1RVfnCIgECiVEpRTCjLjOlkTpZVSBHdMc7CONi4wA4urrl8XB+IkNB3NeMg40Y6xEjqdDLDA2Yc6boW67rItU0TqqoAfETqhRjRH4cBLSRVNcHaPW3bYW10wI9mjP/WOwIaYwxKOiZVQvAGaz2L6TQ67YKnaWr6oflDbt0MYw0KrB1IdRZTGYmiKKIAbv2IVI5/+k/+GCE13//2t5i+jZu9MUdpxTiMSCTlIuPm/prR9qhgOT5asN2uuLxe8/xZQaoVXT/wmw+v+fpn54yDpXUjJB7jBr777pL93qN8yhc/+Yy7h0sSpUjKiuAss0lJvR9AWKpJxCQKoen7wPFRRZZVvH3zPc5ZJpMZr9/syCuNTjSb63vGPqBGy35oOTtb8O7DB8xomVQVpnF8uL3k+TPNn/3lj3n1coMJPS8+nbPe1nifklSaJxfPuLy7R+qCxazi9Zs3TKoUb2Oxbd8r6togpeV4mXNyfErd7kj0nDwbmM8W/PabVwx9R57lTCcLzGrLhDn5BkJ9gz6b0Ny8J88KZD4ls4rgRhwWbwNyBCULZFBgpoheozBIZfhESP4vn3zKv2le8Z93Aw+D47Z8YP3JUy4/3OFaxydPP2Na5ox7zfVuQ9ANjRvZDnVkLRtBbzuCionBunUMYwMy4Z/9i78k1ZokG3CqRRqNtYKjo2PmxTEf3t7QNZZCL2jaERUKZtMFWiqeniT89jevqXeBP/6jF3x4c8Xnn3yG9QO//M0vydIMP4zgUrQsaZoNy+MlbduSZVHk68cWqeDJ+SnCC24vrzlZzDiuTtCZYX42wWqHaS1d3fDkZInbOqSVjKPl/mqLPE3ouo56s+K/+1d/xjj2NN2a+4d7utaQLzKsG0gLh9Ce2Tzh7m6H94GiyHn/9pauT/j6J1+w29+zq2t0EvgP//OveXJ6wXp1jdKedrsncSmTako/tJweTbFjw/PnM/b7ht7cgoW0VMxmFUE4jo+PuLt5YLFcYMzAZn33e5/bxkvyIroEM52TqJRU52RpgU5BqMj4FkHgjUc7gULR7Doa1XI0PSH04PuAxdEPO9QBa4kOJKkCb1FCEIxjdX0fE+I6JUtyjhYLdpsV01mGpUGWOSefzcDnlJXEm5rjSUWiFM4HyiynDwJnAom0SCvZ7xpccNzd3eK9oZplWLPnbv0tx4sjtNcss2f4UbNxPbPFgn23o+5rEIpcx+JfkhSVFGgB1q8x9YrLYUtWakwvKfyUs1mKHVoKvae3nlRnJKJAJ5Kb9RWpDqSFROcOo1uu6jX99UhRTRmGkSKNLuPRdNhRUWYVMm0pMkPXrVnvNcrMWKSKwddYVZHMTwjVlIGEIsTXOwTwCoSNOILRjchU4UyIhgjboaRkOqvYdVt2XYu0ntV+jdgbnp6XnJczKtMwT1N8XuCTgkQGqiRn3K6QWY6VAusC80lJQop3DSoR5EribBPLUREYAptu4Nu3H7jftay7ESHhaLRUDuZFgXeG0UCS5+wHw+7VNYnSnJ0d8/kXL2jqPbtG0nYDXd+T5SnrZk8xn7Hd7QhSkac5ozGMfccAYAWTcorpBro2DprHoaF6McMXKTLTnByf8HB3SyITApLjkyVKQZop3n54z3J5Qbmccbo/oxgDJ5MjhBJ8WL/H5QPORMHV9gLhNCLPMHZAaYPtoWsHfDegu4Fn52dYAb2AZJGji4xuHKjXK7RQJH5ADorFdIrWM4q8whrD+/cf2O12SCSDcyAFiZdkSfoH3bujMBzd2cFLlEwPXGYI3n4U9w6qeRTNH/vVvPgdwRN+t9jzUXj/e8WiPAqZ7u853uHvC+qPQvojxuMxiaoPa26ILPRotNQIwHmBCAJcOKyVLEKryMkWkZMuhCYEgRQyojpCFDOd99Gd7iwQS6rj65IkOkOgCIS4j9BRQHXOoaSIJJOD6C+lxBhPqj1C5phB8fr1it1asHqo4/ldFNxcr6iqHOdqxrFlNB3TaU7fjaRpwPsRY3qOh56ynPzwmoPAextN3wfx9lGcjfjD+NkliT4kuKOQO44D+3pH0zQ0dcN0OkerWJa9mGeMw8AwOCZV+CgEG2NIkvTwOcRhzTAMcTARxIG3rvlBOX8UpmNawR96jJy3nJ+f8xd/+Zd8//33vH35Gu8dQijOTi+YzY4wJu6JQaBVhhRRwO8OeIPHMtHHYyUWUJcfjULeewYXnelSaKx1jOMPQngIoKQm0T8w7x/xKb87hAohstOtsx+fN7L+1SEVnZCo5PC+Y+GxVgchP8TjKSKNBMGLA67JHtIYMZHxiOF55AfFoMej0SieD0pKQCGFPqQD/vcf/2ARfe3eYW2gdg22X3M2PWPwe4Lf0blfRa5VumZot5wmCePQg9wShGQ3fmCwgolcMvqMfkywIWeRHJHKga7NaXVDkksy/YTRDpCmbMYRwYTRCBKpWOqKMalplSARexyafiwp8gHlNTYIdNLSdBIrPK2xXGRfoMSEqiyRWiPFhjJdgtwx1U8YvEHIKV6cI8ICbwR9aJgkcxwDxr/lJPsxxnnaDkaxpyoVSqd4+ugwIkWKlHmaIbxhdCmpDozBgtQINkgKjGuRIqGua6rqiKAMBEUIGUrkWLvHiQ6tLK1xGBy5lBS5xBrBbHiKDCnhwEYyxkS3VRLFrdEorBV4c4kPHTaMCK9woWOaz1k3NYIT3ty95uFhz6ptePfGcFo6nv8jEDqyYkGgEo22jqbdkiSKJMuYigrlj7lv/y3T7Ees7A3v9++YTj1CxsIVjSLYkVRXuNAgecK6vYdij0Cz79+ScEHdNUBDZ/bs65Gm/xUnS0VZJFS5ZJILBBlCaBBHOD/S9A9YBEFkpEIyWok4DDmFECgtkUHRNlsW5pTz5P9IO3yL5TXd+IZUVYy+RKuM4EfqoUaqBHyMHSuhQByThhlxaZrjfYYMFqEEyiuCdEhAJVOMdQRWeBK8d2RJhbMWITWOS9q+RskKzICTltEZrANvHc7c0NQtJgzsGsd213O32nN177nfwGoXsL6nG6AqNcYThedVdDN4OSBCwBpYDzVa//7FgwD7sSHsFKGD+XTBtm4IWUSOpKmmbfaoJEE6j7CeZ+dPGPsZWsFm/4A6FAgPY0+anXJ7dcV8Omcnem5WK86zBU9mSyqnWJ6d8ct3b7ht1gzra55cHPPVVz+iuVvz7S9/gzEBbTNG49GZYrKYMnjDutkj0gyVFVy+/Y7PP50y9AM+jKSZputqvLPc39+RZXHaenN1i8dycjxn6HvmsxP29Y75yTmzxYK/++Zb0jRn7EYmxxOYBOq2Y7W5Z71f8+TpGd0wEgLs9g1ZniBE5H0JYLfeIfKU3aaj2Q8cnc65vL1EKMl2veGzi2e8+8XfsrVdRHPoHO8Ddb0nAOfnx3y4e4dMBEFAVVUoEYvVrDEcHS8RreB+sycNKc0wYoNn2+2Rd4E8kfHGrCquV1u+e/lbvPd8EiRaZwcBtKEbBjb1A99+9w3jMKIyEAG6piWRBbvtlr7rmFYVWikSrbHBM4yxVG+323ByfMJjy/V6u6aazEmTFBs8Z6dP+cf/zT/j3eVbLm/fIHxKWVSUecn5SfwM/WjYpTnTPGe729KagePjI4Tw/Mf/9D8dUAsRubBaeTrbcDyfY9oBYQN+dKQqZbaYc79/oL1vsFjMYCmLiofNmuPzU/Zjy9dffIUwguuHNVmW4UfL8uiEu/trBIrF/JiTsuBq9UBaFuyHgfXQ0geHlimDd9HwEcCPDpwnIGi7jvOTJ/zspz/jV3/zt9zdXfHu6nts8GSTnETFyP96t0H1isl0wvXDNV0xsB1qzp6e4YLl+uqS2XTBk/NTxrE7LGQi9iRRmlk1Zb9ZRfYfhmef//6twfv9DmsN00lF2/b0fc9kMolxPGcQQmKMjdejwVAcL9FasFrfsVntOFqmeN9RTiqKKuf24QNCWU6XT6gmE5qmAQJZltF18fnTtKDvB8y44fj4mMlkym5/cHC3NW3X8uTJk1jwZCJyYxgGqmrCkyfPmE4qri5vyPOKthkOTvTo2Ag+0DYt1SRDSM1ozMH1HTEgXZcc3CuOMemZTRcgDEiLTiRZlpKmmsm0RGnwITK38zzBeUvXNygt0AnM5gVCepzvUSoOLl3oSFIXf68FITRS6YOjfiRN09gjIhSx0DSlyA/idL/m4X5LV45Mp1OkkhT5BK0VRV7S9z3G7Eh0wqTKUTrl8uaKsqjQiUKqiAmJJh9BURbMZjOc80ipWG8eaLuWLEspioyu79jtt9GJZ6JrP0kVeZHEVMmBixkHBtENXxQFXdcBMJlMmEwiksmMJoq+XYuzCVpPKPIoMu/3Nbv9ButGlI6uKSFkdKcPI94Y0iwly1JC8lj846iqiiJkdF0X3Xmj/pg0bPeeYlqxPDqGEDecWVpwvKwi71NGwfz66g4pNtG5o+MQwloPwR1YnR6pUvI0HmvBCbxwsQiq8fTtDpUWMTVgLaMZuV8JyiKjmpa44PHBMw6GfjAxiSMESZKQ5zl9PyBlLGyK3ix/KGsdGMaAVmBtfE3OeoSUGGto25q62R7KZX//RwgWKRXjOODwyCCotzVaKowbCNLzsz/6MV3refXqO8axRSrPyXKGN4FUa148e8Ll9RW7ZkczRPyY7yznJxlPnz1BJKtDZ0YglJLPf/QZv/n+lsWxopxVjE2LcSNVVfLk5IIXTz7lF7/4FcenzzBuxWBavDGINJAmkmfPl7x6fcd8HrDGsjyqmE0X/ObX70FuKPMCKR1Pny4Q/1/W/rtnsjTN08Ouxxwf/rXps2xXT/e4np4dt1yCJChIEECC/EtfQN9Ln0BYgBBWAKndnSVnqfHtbWVWVprXho/jz2P0xxOZtRSk3UE3o1AopHmjIuKcE+d57vt3X5e2NHVLpBPO5iNSXTDYBuEh0rAvV8gI8olCaM+75VuEi5nMg+i4q1O0Szg9ydjVLV9fv+V+ueGzTz/h6uZrdGTY7i1aQVntiCJNlgXWq/M9/RC+PxNytE6pqpok1tTNQNO0LBZTtE959/rAzauOh32Dv++o6grxyYTYWLS1DCMVjk/fI0sPkwiXaUQfQdTTR5Z+FpPplId1w3/5+JR+3zMWEeu+4sevv8L5CNd2/MGf/wm3b17z1ZvX1I0hm0z56Zc/5OzxmMZ2jKczXl2/YTKZ0O4MDx48wgySpq/5+x/8mEcPTzk/jZmdjRBETEYzJMHlMptNQwjhasPQwYOLGZcnF/z61y8o8gTpM85Pp/Rd2MhHSmGMxw8DTnim50+5vdpijOHs7CHL1TU60oxHJyjtWG1XnJ895OL0lHKzZ3d/IDMJy/0d04nGZxBNBH1lWOQX/Pgnr5CF4MlHn7PeHvj6qxd89nRCIuD07JKTecqXL95xdXXPoTyQ5THeK+q2RirBdr/lZDLj0YOEtusYhoYkgdOTGbZ3ZNkMF8Pd+p7TizGzeUKeXYKDySjn5t0tL69u+N73f4/N9g6pBvIMiiLh5dcrHj99zna/xXlPHGlev7nl7LTAC1it7hmG5je+ts3giEYJaZQiUUQqwRnYrndY0SG0pS4rBtN/SKu+l78NNqyL9+UuINq6jqbuSJOMSEuyKKZpG8bjMdfX1zhrqcqS1fIe4zoiqUmjGKUkm/UGtEVGAiU1J4sxdb1CSckkjemajijWSClp6oq2taSZZXeoadqOtu84VAcmowlxlJDEmnFR4KyhrgbGsxMOhwoda4pxytX9NW1fhrF865BoDqstSRzRNy2yt8xHY87PTknShPW+wYiIRCq0UizGY7rUI1RB2fTMJxN+evNjdAJnl2Oi8Yi26amqhs3tKwpdEEWB52v9wPZwYCQLpIDFyQyPoS4rdndburVFuYY4akjKPWfTKa0xTDJJnmckMniDTO9Ag3GW4cjOdi4IUAdrsAOMRjlOG1a7PaYdsIPhbDHicj7lPC4YxwlDVSGlIooU3lrWmw3jfIIfBgbniKIU72Doj80QN9B4S9PWdKandZ7r7YEfffkVb+9WCKFxR0yF0lEIzfmQBM1HBTpOsKajaQfquuFQvmIxy5jORownBUmaUlZ72q7G+IHReEQxKejNwLgY0dmOqgkibecc9eEATqCl/tDwkUIym8/ojOVktsB2Lba39MciYtu15JOEumyI9IHZ4xnf/6PvMbaKKIn4m1/8kNVmSzFN6YdQSBcoht5RNV24DwuLjsKkpbeek/k8fB/aIRRBI41WEi8FRRqTRDFGK4a2o9qWICRl1LDf7aj2h1BgFpAnGdkoRyhBkWW/1b1bCPWB1xwKr0c8ybGACIr3fHRjwropihLiOMFZj/fD8XlCwCTSniDodP+bwvh/+PiGRf0N0iUUhd8n4B3+yKYWkoAqcQGR451C+ID1+YZlfcRgoDFGQh/S1u8RG0HCG4V1stBIqfDeYl0ffDnG4wjoyCCMDAVhJ8EJjoz390gYfywkh/8K3stXAxI0jhVJ8h6B43j0+IzV/TVlWbHbHYh0ckRDarp2oO1rjG1xzmOsYzQSwPAh+T3pe5Q6Fu+FCphDFVLf790BQn6DqAl/dhSzeglCYa07pqv7D8c0S0dIOdB1LVonOBv2Dkpp+n5gMD1Jkn5oiATJqSHS8XGv06GUQyeCKHrv1Al+DBBYGyTISnm00nzvD/6Qs9Mz/uFv/oF/+S//7wgEaZoRR6FG0LU9dVMz9Iam6UIx/9hoeS8UfR/atdYe3WjhvNJagwsSz5A4B2ePzHoT8EKx1CRJgpQKawaEsIGh/77Qjsc6QxCIfjMRIRBIGdA2Smo8EmvEEcfyDSvfDQbrhm8mFKz48PxCiIDLUQLnBO/FqPj311a4foQK57XWoRngnDxObvzHH//kInohGza2xQwLkigik3dsK8M431PVJ2RZT+RHZFlO60LR2CqD8Dd4D81gGOkIIw0TdUqNobOWwcA8K9Cix5o5RZySZRWNq4jFhF556qFmrM/4ev8VeTKw6zzabnDKoUWONQlCCzJhOdgKJRLmuUNbTeae0/pbGvuaXD0i1QIzdMS+ICNC+D3tsMTrCYk8IxU1ncmJ1D1Yi3IpnW9wjNBqQLqYsr1H+pbeetA7ur7HqAQlYRrPqPoOMzRY70nVE5SMKPt7dHxOEu2YFE8YJ2O8XFJ3JbgZkZLo6EBTQWRmpLHBuxhnNyibs6tjFsMcKUOx8j3k3x0h+oNtOFQ1lWhZ33c0jeXuUFIUCftyQyRmXC1buvo6dC3dFMTAo5NLPn/8+yTxjCDeDF/GWimss5T9gZv7t0wWObf9v2Xwe/L0Oevhr9DJKWNj6dolMrtgcLdotaaqIrZ1T6561rs7/pe/ueFsscKjsKJFiIbVzZ4sU4yKGQ8ezYjjCUU6MC5ShBiFhKyRSAzNUKHUHGHv6G2HFA9o3AYpYoSq8FaC9VjnQHrqdsN6dc3J+UecJX/OlXuBFCPwd8RKI8Upxkuc6NE6w4sd1gx4wmK3H+6IowmeiiTKGdwpg12ihURFHqXWmMEjmaJVKKZHckTTbfE4BixdVyH9gtZbhv6Opt/Sdpp91WAGye39FffLnmFQ3C4dTeepe4+MYlqrGLxHS0cWQ5bmbNcH5hONEJpIa/b7Dpw4nguOwf7mSVUAEUv2zYFHi6dUh5br2zviscIL6PueLI05W8wRLrDVEh2zK1ekeUw7DCymE27fvWM0mrBc3qI1tM0BiaVta2w6I1vMyLIZt/crfG/wdcN+s0QIy+6wZTIZ8+zjzxAqYvrgjF7fY00FEYynU/aHiihJ+dlPf8VkOiWOJL215FmoCEdpRv4gFMSTRBJHOferPZNpwXazg+4VJ5OBdhgQueLqZsXpyQVFWlAkBbiwoa+HlmYYmM5mZMWIzuzoyoa6ari1PVkm6buO6XiMd5I0Lljt9mTphEN1oFfBxt07y1evX5PlOUMvaKqG0ahAq4jBdAxDz8nFhDhWzE/n7GWJbS3O9pwtzliv1+y2WwZvqPc9moQkjhlEYHUN8lj4NI72cOBnL1/w9u0b4kgzO5kz7ia0teZ+t0cnMcO+x9ie0bjgUG5BBHFonKgP45d912KGIIqSWtANHYfyQFU31HXL7//uH+CB7W7D48fPMc6ioxiF5/nTj/lv/s//Pf/+r/+S7X7J3//NP9B1NY3pSUcZ3/ric1Z39+zWG0ajguXmnvOLBcUoIy8z8iILKC2lOJtfUFcd02LCYjqj1S2ji0teDAO3dzeMTjLyUUGejNisNwyDO/L2DTjBdrtjkkxIopjmWKTdbrYcdhWb7YZEx4yO44X3yxXzpxP2fcuyPrB3e25u3zCbTFlM5qyXa04mC4xwlE2DkPCzn/+EZqj5+s1XlPWKp8+f0w6w3q45rHdMJmN627NercF7qraiMwMnl9+iKkvapufiLGW9XhIpRXnYcjI74/T0hNXdit1mQywj6qbk3XbFer2C/9Nvdm1vNjv6PnTk5bHZqLTm/PyCcl8xmFBE1VIT65RHl09YnEz5yc9+wOFQHqWdR5GMDaPH7+XWWiu6tsMaz93dEnjPXITJdIwZYBgsy9WKpm1I0ohRlLPb7ZjP59zf3zEMA0mS0nUdk8mU6WTGeJTjnaQsK6IoJMVHxSiwj+MwnZWmOW3XHAuI/XGjGYeRUmdQOmwI2q6h6zuM6VBKUdfh8y9GU05O5tTNHucVs9kkLMrxjEYFdb0nSWUoRncG6OnqHc63JGnEMHh6/15kc9y8iMBGDEkgjRT2Q6JBqRjvJW0bXuuo8Kg4JopiiiI7LugkRT4GQtFYqoh9VZPnGcYZkkyTpJretAir8F4wnc/J8hHOWfb7uyAoSkOjoOtCgyaKNAgCBkZHxGly5Ly7YyE/C80mFxa/QkjG4zEnJycIAVVd4pyBo8G+6xsyE6Gj4ij+DMia0WiEjGKiKAcfUmDWGeI4JYkjBISNq3c4Z9BKYv0RdSMc1kiG3iO8ZDw65enTx2RZwm6/DkktHbiR/dCFzW7ZUNU9AhWm5jKFE21IoGPCZ+o8kcoo8oyuk7Rti0CSJtGxgTRwOGxpek8aRzjrOJQVHOVI8jiFE8a4PUIqmrqhpQMkeZ7TyeEDM/P9eOkwGMwAZjA0dYN3niRJcTZM+XRthfeOOP4nL8P/fz5G44ztoaWIY5I4oT7UjIqCVCXoROCkpO88b15dgYlJkylCGrxXPHvyhCiO+NFPXpJNwloyzVOqZkBqxbvrO6bzCeeXM5wRnExPuHlzBfKeXrbo7JxdVVLVDUkWsEg31zfYxvPk4SV//Xc/4l/8F9+ibrest6sgrXeGLE949nxKVdZ0/cBiekpd11y9W/LoqSJJUyaTKfvDiijTtF1PUxvKtcF1LZ996yGDL4mjhLbvub/bcnJ6ioqCwHd5e0UeF/w3/+3/kZ/88IfMJycYKdk0t4i44cHjM9reUoxT3r27I4kD43U0KlBSs99XgdkqLIey5Op6x2ef54xGMYNpuHw4Qkdj2rpDKIueTfnrL1/z4q7iu6cRfLlhcXrC9Mrw6WbFyFcsPnlCpAS2qiGKsDEM9Li0JC/CBjNOE8TB4XeGdzfXvGsc23GKmo3YDiWzkwXxOONf/k//A8pYrq/vWCzOWL9c8W5ZE58rvIJCKRZnC5qmo2w6docSiHn0/DFvb35CMUkxNjS4vJfcXK0QwHwUGqjPnz7m7uaOKNOMRhpraj5+do6UcH56yU9+dsDLjLZds1otmc6O17cIhZ3xZMJ62dB3lo+ef8JyeUdVWk7nBfP5hM1mx5MHnzKdRGTPMn72D1/x5GLM6zfvePadh9yv17jWI/uUhxdnvLu74q///U/Q0Ywnjx9z2Ox49GjG+dkJd3dX7HabD16MNMn59a9vePhoTFVWlFWHHEZMx2OqQ82hqkjzmN3hGrfTqCQhmWvWh3uKfMqjp2P8MKKre8pdxYNHM55/9JCy3HF6ekrdHmjqCut7Hj1+yJt3N4zGaZCuaUWeZ/Rdx2Z/IE1jjPnNi+hZHCOsoDM9WkUU0ym960JSN0vpXUPZVnRty2I2I04SdtstDx49pKwqHjy54OSrOfe3G5AKh8B4GKUJs1kQIPd9y3a7xlrDZDIiijXODezLA0OUEmvN4ENYo+1vKMY5xTimGTryNKPHcOgqEuGpTcXVaoUxgtMoYl/vSbKMLMlohprRZMxoNCXPFFplrG7u0KRsfMl4OiEdRVTtDutbymrL0FkikcBYEAlQssB2HdPRlCePHjEZj3HGoU9G3GxWlIeK1f0tSZxivWYwB3oDjy9THlycUtYl7aGnjUPxpt07TO0xriHLInQEVhj2ZUfjPHkRI3qDRqCMoJAKIQz7ZouiY6JmCB+KjUIL2r4l0t/I+Mq6pOs7vAuSPOE9aZLQDgMoSd+HIMm+rRFWEkcZJ+MJz88vmVjBtq/R8xGzOIOy4fb+Ci89QgoO2w1xkhIl2XFinvcVLDprWG7WOCmpjGF92LPebun6Fo9ExwrpYbCGtmyQWpBmKVrFSDRFpomko64rnIXdoWSwAyoeUEozm09ou5aqrdgfdjx5+oirtzchHRprEkLCtt7V1Js2TNyOE8ajMJGvhGboDF4I9tsDOM9+v0fJhKou0WmYiI9lhO16nGnJ0ojvfPQF//CDH1JXLbYH6VMm4wRrLVk+whhP1XTUTUOaJXSNQQtNpCOiNGW121IPHa0A1XVkiUT6UNivqpZYJwytZb89MAyG8WyCM4ZJMUEJSVXXZHHGYnpC27fUh/q3uneHoqHB2FC0FR+4E4EPrjgWriVY2x//KCTRhfBIG1Lc4hgkFGHo7pj2fo9mcbzP2gopwQUkyPtCpZSh6C20PLqT+FBkf/88zrmwphYK4eA9Iz3gOiBNE4SIQ7BRhMK7UBIdJ6RJhvcKZ/lQkHbOYmxPf0QIOwlFOiaOkwAfdC5QZxThz20orApHWMt6EWSSR4528BUplFYMQ4XWGXke8fDxnNWd4e52izEWa3oinbDf18Sxomlbur7FOUnTDFinGaylGwQIHWo7OiZNsyN6yh7T8ArnB5w/SlTdkbMtBZHSAUcoQUuFkMcpAmtpuwpjDFmWorWmbWuSJEWrjGEI+5MgEw0p8MDy92gtj4VzhTE9gwmeIWUlaA0cp4sA8OhjelwIiRcepVM+/vhzHj98Rppn/M//7l/zxRdfsFhMORwOwQN2fUPp6jDlGyeIZkDzHo8SfBnGBv67UvKIWjFEUUIUHRGh1qIjiTyeR1KogGznPdbFoaX+kKY/OmhD3ezIMA/napB/4gUcC9/OgbcW7wzeC5TWQWwrZLiO7HuhrsW5b6S5wLHhEqxOUkpChydw7t/jecTx6zMEn8Lni/tPh1P/yav3Xe+o+xGREKQyo7cpOioxvuN8NKHpLTrOiIQDdYKUE6phx0VsUInj7nDLmrckkUb4AMIvUkPmwbkabAniEtSaQ6/pTIlM39D1E0ojUboOXEsXkakGISXOdYzTKdVwgxgKSCdMoprONWhXoNVAxw6hLJIRrT3QmB24nCzSdOrA0J0xih4j/I6yf0uqMqwvMLZC+nOE27FvNkSyANmRygnSndD6d0TUCCbkukUocLak6w2JnrA81IzSmM5ekUVjVCTIdY9E49yBTVfR91tSPSNScyr3Am08Wke0Q8KhrMiiczKd0bSernYk0SnS68A/PXYdBzPghKHt9vzq5Y6//PKK5bXH64GykRTjir6ByXTH5VlEkWb82V885mSkiOQOMsWiv0AfosCnwh1P/FBMcFgG09MMd+yHH1HZikm6Qsspdf01kTtjtd2zbO449CtuljVffzlwe2dpOk/feJyEv68apgvNfBLx/e/NWTyZMGjD7333CVJUZMVApAyCEqE29EbRDYaMBYoUj8VLRRZJYr9HCYlMx/ha4axC+YxU56RpTp5PsIMH65lFf8Tr5v+GyDKcO0U6R09g6Y/SOdYfAutNVCg1RhIjvCJOcpphRdnsEGJGemTqOVuT+JxuuAtZuqHHeYM1JQmXdMOAp6DqKqR13O1f0PQd693Azf2eutIst4KuBe8kdS+5Lx3zqaTqJSdFTlf3KO3RSpKlKettTRIBMqQou67HK8koV4ExaiR59tspTla7e6bZCfl4hCJhNp9jo543ty8QHpI8wTlLFscUxZiu7RFK0xvHeDYnG42xDgZjmE9npNMZy5s7xk7jCFK16/2Oh8WUQ1PxaDxhnEU8/Pbv8G59T+ccm8OWB48vmc4WxNOCX319S9N0PDh7jBQalXjuV2sgjA3e3tyQFDFKCp48e879zT2b+w2zyZzpJOfX25eYzjAqxpyfTpkWM65erTh/9IiXNy/RUmKaHnoHuSOKYrIiJ5FhUWpMh+0MwgnOT844X0g2uw1NtSeOIrrGMC6mCK9YTBfc3i2x0YAaZ8hIkxYZ26s7JsWI8zynjGoW0xnT0Zib61tmiwnr9R1nZ2dUbY2KIqp9zeOLp0QqIs8yTD/gsOSjBXXV09cl4yJD4ajbBussrbG09T3raoV3AxNZUHc1TV9jneXlq6+YLhZHHET74bzq+oYkSciSo2VbS1rTMxjPZr8ljiLiKOZ+vWQ6mvDm3WsmkykfPfuYqq1oh4aR8kgVximzJOPxg8c0h5b9uiRPIjZVQ921nJ2d4XpLeSxOSa158uQpcZwQxzGfffYZDx8+4H55h3WO58+eU5U1n330LX7x018SxRGjIuNP/+RPeLf9mpfXL8jygovTC0zrSdOMxfSMYlJwfX8DznJxfoq5W7JflZT7HS2C6XzBeDzn7VevqHYVkY5IipTFbI5pBzb7A6gE0w/sDltEF3N5dsH5xQKdx/z8l78kixPKQ8n6fkXbtpycfoJSimq3IYlTiiInSxJcbciihEREYeTQebqqo9xUfPHJF6hI8/Wr14yKnDjWOG/Z7TYIIIkSFpMp11dv2exWJKPJb3xtHyqDQBAJSRxH6DhB6QTrQegIZxyHusUOFmk1b97ec6gb1psDk+kEHYFxfVi4+JginYXUvNNICcvlhvWqZDqbEMUaoSxedkwXObc3e5RXbO5vMcOBUTFiPJ4wGo0CaqQOKc8wfhc2ybPJjLbtKasaFQmsa6nbJVW7QmiLcoI0yoL0x3nSJMdZ6BEUxYhhaFEyOv4/DMvlmihKSJIiTAFqT1muMK5BzmY426GjCK1i8mTE+dkDDoeKvu0RQgXeZhzTdUE4mUeCYeiJhUFIjxscViickyTJkWPoQOkY6zqkkrSDIUo8Ok2J4yQsyI0liWJOT88QEnb7HdZKkmSMOXL7rAGBp662SO1Ikoi6GmibDCVSIh3kprGKMN4fUxSKOE1BhM1Q3TYIHRrjcTYG4Wk7g7EdnW0ZbM8knSF1AkjafkCKiNn8AqE0eI+1W6TsyRKPkoK+NyBbEC3WG7x06DgiJKcimtbQtz1ZPqUoRiFZLDxJ3BNHkmGoqeodXVshdY5ABuyWdEjhEdJwdnbGZDIOCfXB03cNng3DILDmuEDWkiQPss+q29EYyIv46AnpsaYjiUecnJ0wKqY0dY3UCmcNWkekSUFdd1TrJdbUEIVEYtea0DiRgZteN1UQQ2pJ23Q03YAzniiKgfeJL0jiozD3yO1UUuNdQ1OHZFgcR7RthTEKpYOESf12NXQO5YFhgNOzpxwONUJobm/XXCxmxDIjKzJurtYoYg7lQG96vDJkacT2sGO3O3B7v+Tj6Tld25LmAdHUD4bxZERnGookIU8nvH59i+165qeSs2xEEsf85b95xeMniovzmDz3mLbnxVcv+dbnT/jd33/A67evGJzB+YHJfIoyhro9oLTn2fNzbt4tMbblcKj5Z3/6jLK6YzIpWK37IPqLoWpa+j4hilNev3lH1Rz43h9/i9MF3N2twxRIKYgShY49k1kKZuBf/Y//ilgItusDJ+ePKaZjIGCiXn654s/+7HOWG0vfWYbOs993XD6IiHtLHCmaZodzgrKqqOqSJHFsdiu6tiFLI8ZFuJ5ULtleRLzRmjdFxHBvKG5qnjc5u48KnvQN8csVk/mEbhrDvCAdFHK55abfMkoKxial/NU9mpRfv77m73XP3ViTzGdMLhfUqyU//umv+d0vnhMXiqHt8YVhZ0q+ut7z8eePUHFD7waub9dMZxmCAa0l1zdLnj55TJII/uzPv0Xf1JzNzmmHHmsUjx4+Yei7MO1jO9oOvvXFc9bLNXHsqZs1T588Iktj3l7v+fYfnHJ/v2S7uyfKJnzy+WMuVifsDwdub98xnz7k6dMLrm+uKF+tuHhwxi9+8RKdTYjSCGs9//jDH/Hn3/9jSEd875+l3L695uzhQ5b7A2/u1zw+f4BOYkzX8/jRGcmi53a15+OPn1CkEuE7rq+vKduG3sL1/QYpLZvdQNtIinRKteu4mM05PXnCX/3Vj3n0ZMT85Jyq2SEigTMd9/c7kiZhOstwpuLQXNNWIaSSTfJQpI0UlycnJMmI3ZuWwcb0gyEd53z06YiubXHG0bUdVVkzmc1JshGjyFKX69/42j6bn/H112+QIqIoJjx9NMXllslkhM4EN8tr+j4069s+pHrxQb42uIFDveXjz56zWm3JioIkK3BGIGQIYU0mY25uboii6MP3bZRomm6PaRvu725ZzOYUowI9meGlQ2kRvg+doekboiHHKSi7GusGknGGLXu25Y7BGzIlmM2nSA3TyYzReMJqec36bhckomkOxJycnfPu7iWbwwrjelQM1ni0UGRZTqzB4GjsQJ7POAwDt6/fgHGkeUpXl+RFStW0oYA+GF6/WQGCNPbMF1Mm0/GHZvHhUHLY9GxuSyZpTpe2PH72gNvNkqEX9N4jtWCaJtBDFieY9h1OGVorkEjWh5IRO0SvyWJFlo4ZJzHOePp+YL3fYoYuJLCdQx0LYkpJsI7RKEfHitE4p6563GC4mM1YpClmvWMyG7Htau73a/rbDbbtmJwtqMqSar8nmgXRuSFgB7w1DFXF3fKewTnSyYxyt6UtG2IEmVK0Lgghoyj6sD6I0xgVWcqqJoky4jg0nbM0xdgO6zratqarq4BrGFqK0YjxOEypCeE5vzijOlRhv5hoTB+kgFEsaLuaXrX0Q8tsVICT1PuG1gxYPLvdlq5tw/0/EUxHKfvtlkfnD9juKhSOu9sr9GffYb8/oHTCyfwcKSRx7EAFV0BZVbRtj5IK7yTOQVd3qDSi7DoO+y2Dt3TOI8uaLAYlPEmUhWJdqhBoJuM5QkmyUUoaxRRJyqQYheJ8npFkKbvtlq/W+9/q3u1Ex+C6UNMRCuMtwoZ0uJIR+ogu0ZFGWHEs1oZ1r3MC50MSW6kgXfSokGIX+ihXHLAO3ruCZBjpQ4mIOM6O6+4uNDZkEpLgODrXEkURSolQ7O5b7OBwsSecuhZcDypCKnl8/QJP4KArEaFVRKJTEh18EhYfZJleHrF8ChUrFBEOT64y8jhHCknXtygFXkZHNrbBHT8HHUX00h5Ln8d09BEJ470jVhprBpztSOKYk9Oc8/Mpm/WOtgk1s85YoljSdg1t2yB1RlX17A4lZxczJpOI3uwp2548S5mOx+BBqYA6iXVMlASUkjEhJW+sxx6xm0LqIOP1PsgpvSHgS3q6YeD27i3OGc7PJM6PsE6EqcVIoXXM0PcMfU95OHBychrQuvstaaI5HDYYE1jrIz9B48PaXSpC0l0H1I54/2tQ71npyvFf/lf/Nff3NwAUeUp52HHYb9ntd1zf3nO3XNF3BmxA+DgzoJWkx+HtQBJrcAO2b8EN4CQWTxQH1PChrEnynLpuGXqLsxJjHXLwxFESwj9WIIxD+wTrjilyYHAOax1ChlCMFuF8NjYw/wGMdVgniBOJZwB/FMA6ceTAy+M5E1LtwmroDV1vQ2NK6WNaPTQ6ldZh0kJApCXehFQ8/pu/8x97/JOX75F9QqY2WNOiXMdcj7gzp9yXP2OibxBA0+wY5wVtbxkl4UW8rd5BlJJnMdZEVP0Waa+IlaaXmiwaoXzBwafU3T06Ngj/hEzEbOsOHSVEoqFpeoY+higwJyf5CO0V3mbE4gwl92yaa2Ldk6mI2BWMkjG9ecVYT5DKYmyJ8AVOPKTqK5RqEbHhbXlLoccYV0MeH/nuGc4ZOjtghooodSRyjBISuMfILVqkeEbEcc/gBhJ9zqFbM00dT07H1ENC3ff0bqCIYmKR4n1BFt2w6j15coF3OcatkP0JfTtCRzFNV7Ku7xnnPW+rmFevt3w6+pzo4QneB3SLlBKPQsmwWXt7/xVfXd/z+oXEWkOax2grcZ3j5DLm888KPn/ueXZ+RpRqlGvRseVuWzPzcUikHUcXhBAYe+yOErAi/VBzfXPNq5u3dNVrIn/C9nBgeyh5fX1HWXYYJ+mHsPmwTrEvHdnUkY8ET56N+dM/+4SPn47J4zviWDIMc2I9gK9ovcLaFOfGRN5i1UA/bBBiibUZafwxgznFI+n1jkgoqrZl4cYIP+Hxk+8ynZwc0/QiCB5w5OIRj/Q/R3Y/x6BZ6nc0tkMojRmWCHKsT4n08+MFukFJTT+s0fIBVu3wNsG5Ca0pafsBJR1dC0pZmuEaazVlq3Hm57S1Z19C01nq0rA5WJYriTGK27sYEon1jixVNK1Da4XGcNhBGsfUdU/fWoQOUq5D1dH1jqKQCK/YbmuSWBGpiKr0ZHnMYBy7zW9XRC/rHZ88/Ywnjx9z93ZDlsWsy1uyJAviMy24ub8jEjF105JkeUh0I7GtwWxLjBPUdUfX3PH4/BKtNIWMqFvBR0+f8vZ2xbCrqDcbnn48ZTbJ2Rx2ZFnBennLZ59+jDIepxq6wZBGMUOT0NY9k9MFHz3/nL//23+kyHJ2my0fffSYwfZUfcN+e2C/K5lPT3l0+Yjbm7eMRzP+5E//c5brW9pmx3w24/VwQ9e0RFozKQqGpmW/23Jzc8WTZ88QSmIaSxanNM5SpDnSy7D4H03ZHk75wY/+kdnijP1ux5MnT3j66Bn3d2v6zrCq7sPNXoZx/8cPH3E6WXC3XnHyeBHSjx50JJjNJtzcNRzKmveDBN/5vd9lFI1Y36yYTqYsV0uePH3M9PQh19f32K7hbDZBCcHyfsVqvUUnKdgKnSb07UBjSu43d8RpgrPw6u0r3NtXwQzeNeAdozzH9B2R0ggXOq55loWbkHX0fYdxhkhGzGcJSZ6CE7z86iWnJ2fESUzXNzh6OG5ymm3JL3/+C84W53z7i8/5t/+v/wdplHKyOOFkuiCNU2bTGUJJzi8vkSIsnrfbA8+fnxDHGTfXd5yen7HZ7MjynJv7a6p2TxZnqHiBF2Gc1VrHdDyj3rUsRqc0TUM2ymmrljTRmEMLWLR0jMcZSSxInEfoCBWn/M4X3+Xi8oJ//NmPGJxBW8Fuu0ch6KqGaTbi7MEFbWkx3YASEtd3PDw7IxERtRFgBJcnD3l0/pR9vaPIErAgvWe7XLHdbXn89CGu71iutsymOdubLbPpjEzlGGv59JPPqZoDg+l5+dULvvXx7zCZ5fRVi9MxeVpwKBs+/uzkN762u9aT5RGRTj+IXvaHPcYaJpMZydH6PhhDqhO2+z1NX9G2A2IYUJHgcAhyrtn0hFE+hmyCNQ191xFFMd7pgNGwliSNMK5ne9gipKbtQtI8zz1KhEWiUprNZgkiSJC6bsAOEdNxRJKMWK3uqMqaOPMIObA7tOx2Fd4qoijBE9LzZrCk0wLQNKuOrh0Ip7NEypgo0igZ410Y5UuTBKRnOp3Qdx273YosLairiqGzzKcQqQ1t0yM4ipCcIE1S7FAHgY6U9EOLxpOkMUYL2sHSmxbnHEmSoXVKc5RO9kOLUGE8eDAWJwTCe5RUxDpG6yjIoJz45l8rkSoKiQ/XM/Q1pmmoSksUJeBSlMiYTs7xYxDS03U1xhqSJA2pdxfYipvNGqk1s9kiIFykoOsqBuNCguODmV6E12JEGBd2kq41dF3DZrvB+44k1ejIobQlSzVKQX80XSkVYQy0TUfZtGgZk6UjimwSElJ4dKKJNDTNga7twprDBixGVR3w3oWR8STG+YH1ehVGyg8lzkviGBD6OMoZE8cxUiqqqmK7DQ1toVJUFMafnR9QSpIkGWkahOfWOfqhwVofGuhotIoQ3lLXB6IoDqPDnSNJwBgfzk9riOIwiRVSSh3WBYyCFuJ43CVKCJqmpu87hPA419J2AdWESI+iKkGWJejIE+vfjqvaNj1nF5e0tWG12qGF5PTklEcPL/n66i3rw5q2giIZ8Rd/8fv87d//HaP5iI8+eUy5u2VXHhjPgjx2vVkxVSOm+ZSvb5c4U6FTg4obqoPnxYsrnj0+JU01uAbpLc+fXPDRRwtMtycfRehIsi933K6v+PZ3PqW8WrO82TGZal6+/prHjy8wVeBXbrcrlIbBtHTdAVMoTk5mrJY7hIhIE3DegYc4EqT5mNE05vRsxNX1FQ8uHjAtIp48/Iyf/vQr+qohG2mUhMF1ONsxns55PHkAdsQP/+5nfPHdnL4b2Kxa2m4gSiJubw+M0og4EiB68pEjihzVoaZvw3ldljvyfMZoNOL0ZMFuu6OpHVprvKgZj2MenzwhahN2/QrnPF+rhCj1rIxDlR3fySBOI8ymRzSG/t2OtpiwlBnbX16ze1tx33m2Cq4fpAwXKaPTFJlITiZTuLSkSjJ0LSoVTM7H1JXnd/7gASdnU95d73AO8jxnfb9BCkGeZMzGY7abDc+enLFeX9M3DavbK+JswuXDh5THSZOu6xmNCjrb4oVgMpszny1Y3t1xc3/PbrtFZznJ1DE992SzUw67FW+vrynGM1abHSoydMMB7xRZFlOMCw6HFd/57hm97anrlrwIBaEf//JH/NHv/wl39/c8/+Jjohh+ef1zMpPy4PkD4lZzNn/AanOLbQ/MLiSNWWEOFttbJvMFL98uefrJcxrriXSE7R3zqaHeDczyC0xruL274cHTCdtDyXJbc355wma7Zb2rqeue7z19Slkt4T0GpZCM8glfvXyLdBJnJFnS8esvfwLS8L3vf4GOZ+wOGybxhN2uoi1rxqMJy2bPbJGQpCMmBbzzvzlqUXrB3dU9WTZGuJjNZofzAx998pTWNIyLCUU6whSOtu7YLPfHgrNkOi344Q/+kY+efcKDB+esl3tGxeTIUq4Zho7miP766KNnLBYLsiwjTjRfv37BQWpOFgu8DezaYpKRjhJ0oujNniSO6Pqeqi7Jspxy19IPDZEaYbyjOhzI0gIpFUU+om1b9uUOj+P+7h5v4HR+RhqPSZMJZd3y5vqGstkymqQU44xYOxJGjMZjjG2QsaIt1+xtw2ZT0pYNrjcka8nQNJxeXDCdnnCoWgbrESoU1u7W9ygJ69WBZ0+fcnpyRhJnjIsJP69fBtqbkKyWa8aTjEN9QKiUtvYs+wO5zJjnU559/ClX99fEsaZpB9a7HSM/IpV5QGJ2HRjL6fwc37eB694bTN8FAK9zJFGMwJPEIYSgIkGeJ9RVS6okT85P6Q47xrFmud/RuI7tYUdT7TjLpwztQFmVCOFROAY3MNgu8IT3Fbube6RzZEnG9Zsrru6XdF4wy0d0TcugBf1gSeKE3gwIJZE6ou162spQ9lvGU09RSJQO91npBdbDUNXHBG1HNwwsThdhSs5Y5tMp2nuub29AC7I0YTwd4ZJQIB58x2FfU8QJpncoFbG+u2NxtiCNYpy17KoG2XqSzqG84PPPv0uzf4E3jkNz4O3btzx4+Ih3+y0djkO5D5MlowiPoCxLjAWpEqz3AasnFZvNDqk9SE+W52Q6NL+1MOAMpjeAoqxrtIqpytBYqIeW0/kC05fYwTH0PXXTYZ1Fecc4Tv9Tl/B/9OGcPeI+LFL4ozRRYG2QZmoVAUekx/GfgIrrsQaMCQJRJ0My/Bu8yXEowR2ln84eJx2PHqQ4JUtznIO6NqgjczrIPB1axyRJShxr2q6i61oGa4iigPCy1jD0YQIxigOCpWtDzSg0AOSRVe5w1uAsYU2FCDUaSSiKOocSAqVD4885F4IuUqAjjT0mvCOl8f4oXVUK6RzvM8TIkNz3XvGeh/0eCSKF5+JigdaviOMIM3ia1qAjRd8NVFWDEIK66miaFh1Z6qbl8ZNLlI5YL3dUaRmS891wDOkYpuMJBQnOm+Bks/YosjQfEDnvWeJhff0NO905S9NUrDcrRqMp0X6L1gl5NiKKNHV9oCwb2q5GCEFepCyXS4yxZFlCksRUVclms+H5s4ROKGJHQErqGCGP7PwPZ9mR/41HKk2SpvyzP/lnrO6vmc3mGDNwf39H0zas12u8B60jjPsPhar9B6HrNxMTYfLS2IFE6bDGlxItAhZL+LAH67oGoY7yW2xo8noFaIztA79dabwAYT3CCaTQ4BUuUB9B+BBYch5jPA6PGULi3Pvw+57Aqw+TCf5DMt0Kh7XBRxTcU2G93fc9zjmyLEMffW/vBbkwBGz2/55J9FGiSFxMZ1KMMVRyYLCOSTrnVH2L3fBzvM5J9IF1K4lNGBM+GMOJTBEoiD26/y5W74hkg9QNe9OiXYzwCXvTcxLN6XyJ0I5x9JhDPzDLc6phgzc5JzrhdfmaEkumFZnYYaVjsCnYBUIfMH3KICJ6eY81noPbo+KBskkpkg4pd2SUqKhg1b5hnKVEQlDogk1TkoocG1s6SryURxFFg3A9Wnb0GIS/IFGe1qe0bgZW4VRDFFmEUnR2zTAItC+YpTOkv8bbmM4P2C7wt9phYL815PKcbnjH1eoAPsdY+HoNm81bdmtDc4iYfvYY+TDFuaOl19kwiiDg0K75N7/8X/jq9p7IO9IM0tRx+XxGPMuYXyqePYwYyhV//Q8rZKZZX+/IJgX24BHnmstPQtdKCI+Q8gNbKdIR/VBi2if8m/+n51fXK5zbINwV3orAXB4ESZ4xHscUSUTkx+yHlt/5rubyqWNSFHz2aU5R3FJELYMYiKQhygxlWZDGKbEoMLxDiBG2TzmUHYN7zmrbksqUVR9zv+44VDVS1FxtPHFm+d1Zy+/PNGZo8ARmmPfhgpVCIkVBzJ9xW/2vPBwm7KZjlGpIpEc6DbJEdDPqwWGcQagRXXfH6egLqr6h6QeU2iPkDkGBHQp27T3OaZrmgDMpd8s1g1WYQbPd9by9GVhvw01xGMA6yajQzBY5u67GtJJ+UHRdS9PZIAXroB16klQS64gk1gxDT10OaCHQRDRN4N5qHYcL1wdGq9ISI3+7InocSXCO1XJJtW+CCNcLpPO0fcfdZskknXJ+cUFd9dze32MltHXH2cUpWkf0vaFznouTE9arLbGQWDTnpw/YvryhfnvHqnvBg08esalWtLIm9elx5N9yv77D1CWnZ3PSKGOcj8l1weFwwFrLzdVbtIT5dE5f1Xzy8afc3t/AfhsWPToNBTOvOTu7JMtr3r55SzFJUVLy9s1bpBCkScTE52RRzPlsTnU4cHp2StXXeCHQRhFJSV+1VNGBIi+Yj2dMRlPefH3FKJ9TV33gNA+Wtmk4mc8x/YC/63h5+5az7BFZkjAfFZh64Gx2ikg8h3rH1Zt3IZl97JpWZUs+HpGPslBYMmHMTGvN+dkZQgi+/vorkiQjSVRgKjYd6+WG8WSBk4LxLOHs/IRXr76ibTpu17egI5p6YLPfHg3uEKcJpmuPQg9BU1VYPXxgkulIgRKh8BkleOGPRbeO11+/5nRxxs9//jM+++xzvnzxK6yzPH/2CZ6As/r6q5d8/4++R57H/PDv/orHTx5Q1hW7zZbxdMz5xSU6jdBKUu72pOmIvrvl+uqWPBsdi3+C3hhmecqrV19xdnKCEoqf/OyHnJ6ecX1zzdnDMzCQyIzJbMJel2SjjLvtLQJHVe24v79G6ZjXr79CCRj2Bz794g9pW8N3Pv6Us5MFL199xSeffMz9uyvOxnN8XyNlS7tfk8qY+80ti7NTRsWIl69+xenJGbv1louTcy5OL/nyyy+5fbeiNRW6gP16xygZMwiYFQWpDlzRjx8/o0gnfPbJFyzmp/zrf/tvGC0mLB6eMdz3tDZIc++Xd5zPLzmZLXhwdsavq4raC3704uVvfnF7GLrQVEwSjXeKoQ/pqPlsHviGQtB3PYmKcL4nz+dMJo+5vnnL0MPQe6aTMG5ZVSWTySQUhLEoBXXbQuVIc00cZQin6DvDfH7BdjOQpQXW9PR9kM4kSZC+dH1FHKvAr3NhNBM8o3FIaB0Oa5S2aK1o6pooyklkSC17Hzjs52eXHA4VN9c39H1JkoRUf1nW34xeurA4klLj8aEobqBtarL0KGhyPVGk2O42VIc6cPS0P2ICEppGhTFSqUORWQkEge8ojMUawzBYkiQkubRSxHFMVbfUdYPWcRhXdALvgzjJexE2UEoSxwnGeNpuYDAOaQ0+Cg1TMwiGwWJdR9caBJYkCuIgay3ODLRtg9CePM+wFtrGMNiephmYLhZEceAY6kgzmBZ/tNH7I8IFH5ABcRyT5xnD0OO9Y7ffsl5tyDKP1iokoCxw3NAppUNT/IgWiyJJ5iBNC/I8I4oUXdfRDV0oUvr38iELoscDh0NJ07QURc6oGCOVoCprNv2evh/wDnScEscxQobNnhSBXxjHMUophqHD2J48S4gSSd+21Hag720QW4swcdN1LYMJDQ5nPRAd5bERXdvTNM0R95Idn3cIkzqCkPCR/ihVfT+xx1GepCjygjgKrNCyDJs5IV2QbgkYhpZhMMGpId6z4X87n4nUGutC46jvPIMbmBSS/b4MqexRiukOfPr5Y7puw/w0DyzlbstHnz2jd4bNes9mc2CzqhEIzs9THlye0Q0NvQubaC1zPDEeRd2UOFEzuJTf+52PEdJzu1qRFgnG9YznMbtNxWp3SzGJeZrN2Wz3xGnC/lAhGACP0hHz2YSm6ohTweJkRFO3dH2HOt6HskgHua2aUR08l49P6LuKxckZ797e8sWnf8ivfnrLyfgR19sfMZ5MSZKEqtwRa0VnQrH89771e7z9+i15JCh3LX/0vY+5u7/h0ZPTICHtw8bKWocXLV3XYW1EFI3JEoPwEVpl1NWWumwQPiFWGeWuQx5K1ruei29f0u5KWERMJynZfMqm3DB6sODHu4Hlmxv+ZJkz8ZLGtNzsG/7d2wOvv3PB2Uef8jf3P2OXGqbjAkYx1tcM5ZZ4XxG7iFwqfD+QZTHEnk62pAjiKEIogbWCx4+e8+LXL8hTjVKS29s93/veYxAjVqsrBA06gng04t31km5wzGYTiizn9vYd43FC2VTE8YjL80dYB6PJnDevv2Y2G/P66oq574kjR1UeGPqeu+WS05Nz2s4xmQgcLSdnY37xi1+x3LRcXi7QOuAjRvmI3obUa9Ud+J//33/Jtz/9Lj978TWffHyJlRqdFNyutkzIGWRH1R5IckGcwL6+Y6RT1qs9N6sNm91AsalQSUxVtbSHjrP5OQ/mF1y/viMiZj9c0WPIJyNefbVltbsnH0V89vmn7LYlyk9pyzX5OCAT+8Hy6u0NZ5cPGGrDZlUxeMvvf+8TBttzaLcMZcl4Og2+me2O+XiG6QV3Ny2ffDZjv2+4nM6ZTn7zBvjy5h43OKqhpi4HmrZjPM35gz/8Dn3TYVtHve8wtaOIJ6zu1/RlxUcfP+HBowX//t//Fb9uLZ989Dld2X4QbM8XE8aTgu12QxRFTKeTUISTgqEfEGh225Jnj58ghcC6AXB4NyCBoW2Zz2dU5YEeTVHkDENP07aofIzpHWmSkacj7OAwnaHaVawOG9p2ipSCk/kJprW0rkdrw6F2qCjH1YeAQs0SUi2ZpqdMsoL71QEZSZRWdDZ8R0jlcaKlqR2ZyhhlE/LZlNv7Jcv1hrwICERDy/5QI2PNoa5pu7ecnZ1w+fAUbx2+h6Gv6PqaxWJOWXdsyw4Vx3jgZlNx63tQA+cPCiItWd+2TNSIbVkiZUvjKuYqJY8S7r/8NSqOefzsCZurO3aHBq1kEMAesWgqigALQhEnEUmk+ezRYyZJgu071u2ezWbNTVvC5Rz7aI4jpVs2tF1DoiL6vsOZiIPxdMYh6jogWoaBfVny5n7N3XaP0xGTxZzxdIwZWnofcHfOGPJ8hPMe5z1FMaL1PaYbONg9URJYxnGakqkIvAoF1KHD+dBcDhNLYw7bHaZpaMoKtCZL44DjyGIkAuEdhU2wxlIeGk4WJzRlw05tyUcRWepprQUZ3FqZTDDtwOnklN3qltxqfvzjn/Gn//w/I375a0RbMcoyysHQtT35KGMyKRhccIXFSUbf9LRVQ9XW5JOcxemCJI4/TDjgBqyz1M2eKI4wZiAfT6j6Fis9QinqvqPZlVy1VyRxTNf3lFWJd4bJ6Ldjont/lIr6AJQIYk+OPGePEEHOGYp5/sPPGONw1n8o8kkFzqnQtD8iBq0NrHVnQyE3/Kw9FijfS0QdUgi01mit6ftQCH6PhglCRnmsOYXUt5fvC/KOwTpcF4qWxgwIGaachRAYN+AHi/PvJ02/EWcKwZGLHkI0SkbgBf3QI5X4UKx1x/NBiFAEdtYgnGVw37wf7+3x/YmAi/Mc19cByaKUJIr1MY0v0JHCGEvT1Mf1ekLXBfSec56+b7mNtkeqQUR5aHBugzP9B8FnpBU6CZ/B+yZF+LzfS1tDUwCOvHREQIocmSGD6anrktXqjrZpSNKcPCuCI2cIxz4fCoqi4H55xW4XpnjX6yVZlpMkCW3bst1umY6ATBLFAknA6IQa3v8X4vfI3PceLi4uqcs9SkUhPX4oub295+b6lqZtg8NAqCNGJfh93ktA358rspcfnErOWYwZAHmUpkLTDigZkcQKiaFtWnAeJzyRDFMLnvA5BYSYwxmBVCEdroTGWRuCMaiAGTLH4rkQICzS8eFzF9Ii5VEsemwgheaF//C+Qx/z/flqGIYhNDu0xjuLEt98ZuJ4Dv6nHv90nEu7JpdBNNf0gkgLIjWn7RPW+lfshlucOGFwG5Q4YfCefVMyTx5jzZZIJHRGgHiHtYI01jRDzKG1zJIcoUtS5dnzClxO6i7ofBNGDHWKFmfoJKLr18yylMZVDDYjUi1+mLBtrxhnA5HIMLLBe0uRwN4MeNlRdROKXOJ7Sa9AMGD6mpgILwY6u2TdduhIMnCL9HOqXpDpjLNRjCUKeB5r6M2WWJ5S9ZZ5YlkNmqqtWRRTBtHQtQu62pEnc3p1S28GrJ1QNhVldYkQ8PbuDdf3HdfvWvqNJosl61bT+B31wVBXfeDQ5jFxnXD5/U/AB5Nx+BIKnROlIt6tr/jVqzsO+5qh04ymGSpVDN5w82rDy1eWv6092lna1uOVIPICz45+F/Ff/F+eBFwNDuMN+D6wiIQgUiF9uDxsOZl/G/fmBS5uWUw1Wnm81/zB746hlWjR8fhhzlAljM4njE4HigKUcjBUCJcROUHrDYaW1lq6Ad6t9xz2JY2xWLmh21R8/XqgHK6plgrhOuJ4ijctscxQccRiovn0ckw0OlC612zWkERhFB4viZIM50EhOc3+kHv3OV83f4sTAm8NjR0QPkFK8FikEwhv8M4j/Yib+9cImVCWmijvqSvD/X7PRI/YdRUCT3VQ3F3XDH3EoWlYbh11rWkqgVAxJycR+6pFpIJd57BdS1UbkpHGGEeeJay3NV56UpWQZRrlNWUbij+b/cBQeWYnEYfDQL23FLMYEzma2nBxMWK734GXzBe/XZotlmF87hBv0WkKkaBrDMV4RmsNCsHZ4ozN/QYhNA8uzjj0O56enOON5u3btzw8f8BkOma3WXN7d8dsNME5wfJmhTpYHk5O+fLFl5R3G6RP6QfD3e6e0yfnIB3KKdabliTusB4SYhajE56dPcM7yX534I9/749o2pqT+Yjl+gYnDadnCx6qhPXdmpPTC3Sc0W0dHz044/r+mum8QJyN+Prla6YnY8p6Rz/UtNWBu9sbohBvpC5bZicLpHQo4PGTR1gzMNiBzra8ut7Qm4bFaIy1lrPzc5w37MuSJO2p+z1KKsajCZEOC8fDYc80HTE/mRPlnmhreXz5J1xf37PblSRZhhHQ1S0Yi48dte/orQkLBu9ptzsG67B9jwJuTZAcpaOMYpLjBKjU8tWbV+zLPafzEz5+/jHLuzWffvKMpt6x3mxDp99LTs5O6LqWqiyxxpONcry0HJoDHz37iLwo+PUvX6JlRDt0DM6w3qypDiV//qd/xmq7QY4811fvmOzGLPYT4gSaYcujj85xogUPDx5dMhqPefj0EV+/+Roda+IsQcaSru2PosCIKApCs8OhpNy37HbvePKtB0RyQeQHEhUYxZtyS903CKXpWoeS4TNeb1fHxmVP3bYUoykHuedQlnTDQFuXnC5OeP75Yx48fEK56TifzDisVkhn2e823N/e8J//2T/nxLTcLq+oG4fuJbPFAp9F3K6WzKZzhmHg6t1brt6848//4l8gEHz2rS/42S9+zFDuMW3PycMFk+mYNE1Yru45PT8ltgW/+ulLcjmn3LZIFQRWd8slSTKi2TfM56e0duC+WtKLjl7VbPsNh0NF2/3mzoM8DziltjXkeUKWp3RdG6aNzECSZIEHLkALRxQ7ZvMRaZKxWq9wxiFFQqRTiiJDKkecgDxKqZIkommGozxHYI0ninIkMWmSY8wK7xqSRCKNpSz3x4KnOS46A5c6jHgKhqGj7xqiSGFMR1XX5HkRcDJKkaYpQnr6IeCK2magrlrqOgge8ywjSwKWI45j7MjTNE1A2agMpSO8M8RRThKnSBEWVlIGWaUdeqQKiXLrB7p2wHtD1xraxmCsJ4rzsODuerxwxKnG6ZDG6YeW1A1HnIyl7QLcMYoSdBSRpxlDU1M3PbqsiNOU8WRMUYzxKIwlsNW9DKzArmEwFqUilAqYPiWjo4hX4wkLWOcNsYqI44i67ui7Dmfth4RIWEwCBLRQ34fNsrcO0/VYPIlOyNKESCps3+NxNGVF13RhFBNB2xj6fkCpDikjRsWUSAnW7QFrPGmakhU5wgus7el7jzOGrq1oG8OoyEmzgqxvg8zTdvT9QBTFTCYT4jhhGAbatme/P2CtJ9LJUQqbgdBYEzZN9igw0lqTFwV9HzYqxgRWoxIxdd2xWm0xQyiAd33HYI6bAh/EnEMfUvFKRTR1FVAvaUocp7RtFcSm6v3GU+JsmAhI4jDFZ/oerRVpkpKmKSDCsW9rpHTEccQwhASS1jGzeUGaJjRNHSa9fotHIy2u2ZGZwKt0g+fFl9dEFHzx+xdU/Z6kGPH19UuePn3Mt77zmJ/97KfcvN2yW63483/+fX75q5/z6qs1iIxD5fGrDZfnF1R1yWAsQ19xMk/5Z3/2nLYZGBC0/cDV7ZdMR2uePnlM1zcMVjN4T+8GrHD0bcXp2Qi7M8wnY5arikNfc34a0XYtMLDbbxgVU1Q0Y1+1VOUBFRdcvQleoacfSYpEMh5FJNlAWTu2K8vN7ZrZ5AQZOUS6ZHoZUUWC3pYMfqCxkI3iUGzzkh//9CckWUgsjsYwmD0KxdDK0BSsDgihcVYxWIEZDEmSI73j6ZMZsU7pG4PSocl9fjJHuhTbC4qx4ue/esHzj2o6elrdcXay4M27Gy4Lz13ZMX78lK+Npr9qWGwaFPCr2vGTaUrjd/z69TtO/vCC6u5AbQWLaYGTEZurHZ9+dErfNESJRsYwKMOLX98QK814FnFobtnXmraFd28OFOkC2x54+HDMxXzBL376Sy4enXB6MSFVmnLnePGrAxcPxsG9st5ijeTzb52z2rzh0JacnUx48/WO+SxHxwarLL94cUWc5Nx+HZjkF7MHvHv3FtNZ3n79hlFRMDsbc3d3z6vrLyn7LZFS3K92gf28P/Dg4pJICvQoou0O1E3Fcv+KYpFxs90SyYSJTjgsb9GTE+6qIBnUSYGQA5vVEoqU8wef8Pc/+CmnD0+4vv2ap48vKZeG2fgM6xvUuOPs4xn/9n/6KcWi5/LRlNF4wuJkzK9fXPHgUUGc9pxdaky/RsQ9dd+x2wV3gR068D2ffvoZP65/RV6kWFny5PmCtpe8eXvAOU8UeZzo8crx7s2OqizYriAfe9bLgaEe/8bX9tXbd3gbOLFFnlNVDfko4ub2mrbuWC933L9b0jU9i5lCWk1dtfS1ZTpaMMrH3N+seP6o46Nnj6kOFXkq0doTxxqrBMYY7u9vjyn0hP1+z2J+yunJOUU+wjvDbl9ihpbBaZxNkXhM19LWDXGUY01Is3rryJOCvW0ZT2fgJeX+QFt0DM1AEkfkRYppOoRztENHWa2I0xznQsMqzRqUMlRVQxEXFEXO6n7J/n5NMS3IpEIqSTc0pHFENVT4QfLkyRfkxZS6CQG/wfSARWnPanegbgbmkylxnoafa0vufvGW+fiER8+f8NMf/wglLdvNmizKuDkcqOqSxVyT5Qt6rzg0W9K6JFKezEcMnaEqt8QqoRctRtY8Ng/59VdfsdnuOL2c8nuffI6sB5rlHgvUdY1SksEaBBrnQ5rzyZPH+H5AeSjLA01TQtkwiWPupWedOHzVcFFWWGdxStN0LabXvG3XyDhBlhWxtWz6kk3Tc1vtOQjH+YMzyrahk5Z0nFMOLc57kiwlihKargFkcMpEAmOHIAZ0hsEYCimDMF0leCeQMuAVzOBQWmKNYxgMqZTMJzmDcESxBikRRJSHEuVNWLO4iKbpqA4NRVbgjcOaUAidjEf0dIBDAG9eveHs5BGqi6irlnf7a7quZ3GyYFtucV6QpwWdizDDgIoU52enoZnWWfwg6FxIn09PFsR5hhkGvHM4JG3TByfMaIJzjt62dK4jzgOuKYpi6r4LSDznyHRGJBNOiozONyzOFr/VvTv4dCRScOScq2OKXHwouDrvjiz0D2TzUNw7CjittUgTvDzBSeGO0sf3POgQ9oAwbSFl8N11XXdM8HIslroPiWohQnBKCH9sLNtjYdKG9SmgdAgQWBvEuc45YhUdC942eL+cZTBxCK8cC5sCcUQNDgxDONaRSvDm+FlIEWSix/cRFDM+vAZCY8F9kKUe0+5H/jdCHpPu8ggyiUnS4BxyzuCdItIBiQshHAMKM7jjvT8EU5f3e9Ik49GjM0BSHppQoGZPHGv2+zVRTJjs/N+IW8PxCU2IY9EcAV5gnP9QyJcySC0P5Q5rDUnf0jQlcZyEiUYV45ymbT1NU1EUI4ztEVLSNBXvnYjGGNquCRhHJXEIpA0Tn/yHL+s/eH3OOibjKV988QVaC4zp+fLLFx+Oo1I6IA9788Ht8E1zQByFpYFHL6UMotO+Q2BQKiZJQuJ/6B1JMsKaEu8lkR5hrQgxKXVsCAn14bwXwqEUREoF95EH64fj5yhx3mMJ4T+BPJ7TRxGuB+HFcZLim0kMCPxzrYMXQakQJDLGfBDrOudCKt0apPDBgSTlBwTOf+rxTy6it64KN13jKPKYynTkyQ2tcSSiYHAxxhzYlgnzvGKUjEhtjJQHvIhRyqCRWCephobUjzi0lmkksX6JFw0PJh/xuvo5J6nh0LzAobAiYz8Y5uk5np5uqDF+TMKEVAxI1bJqMnYdnIyn3GzXzMYZxvYkQ4cSDdIpnKtouhG+m1AUhnXbMk0tsSzwvsHQkeiYJA7F63V1QIgFVqSUcoeSe4Sdsqs3CKdp9BqtBI0xmGFCnnqk6JhwyW2z5HwUs9oO3NQe4bfUzYSfv96x3a0YyoLb+z7YgbcghaPreoo8p+88Mk9w0mN6T1LETIoTvnj0OwjrjuMtIcEqHKAi2iai7zXFZERfgZcOpOLNlz1OduAi2kPoNHvvSCJBYxy20/zuR5/y+fOP8SKYdd17pbIQRCpGK0VtSv7Hv/4fyBYRf/j9Z6jpNc8vJ0TpiEkeWEJeaIiWKB3Tdx2l2YNfcLuuGIxgmlfc3YQROB85hkPM/tby6vqOq7UhMh2DjPBDzelYcXpeUMSGk2c5RRzx+PQBDx47plPDoVmTTx1+aCjyLVT/itJ8l6+2/5YvLv+v6HiCkwd6e4XwBZ2tqYYNXdKgZI53OU1fowkpse2uD0VcLemrPRhJWXnawVCWG6IkYV9HrA/wZbfFDZKutqzXPV1jyQqoO8Wh8USxY5JCufNomZNPHLvBIBEYJ5GJgMjS1JBninExCiyodsAaAdLhpcD6mlGSchh6ysqihCbREX1piCPPUHnevtswnSbgJX3326XZUj1mNJozXcwo9w2vrr5CJhrjLd/+4jusb27o645JPsEai1Ye2+2QVtOVmtjBPC8wfY81PWkaszlseHL2GF9ZvvP7X6CdoBGWl29e8vHFp7SbGusNtqn47NETRuMz3o0ekhYxs5OUXEpuXt/iXSiifPv5E66XrxmfpIi2odlXDNaBcCReUMQx282KTggejs6ZFCOcXBCNJD/+2Y+434RN99D1RLGm7geEUqRpgZSam+s1zx99wq5ahptV21BVJWeXZ9SmofYNWZFwVpyy2x3wxmF8EMlJBk4uT9hstiymJ2z2FaMkZZGPiKKY3WHLPEl59OCCJDojjue8efeK1+9eIiSMRhMmWcFsfko+mrPbHVivNvihp9229G3P5cUl8/mc1XJJOhrRNA3365sgCezBCElajMjyHO8NoyLi9etf4l3HxdmC9XZD27dsdlsirUnygDeQSkMUbr7r/R0qOqc67JnkC0SiGPDs1muctfzq179AjzT/8Mu/RQrNX/6v/xpnar58KXizvOH88SWVXbN6d+D8/Izbu1vWhzVNXyN6wc3qivPLM5q6Y7/Zk+cJlw9OOTs9C83VNEdoxcNHZ/TNnseLOdV2T1k36Dji2UcfsdxsqZuO/f6A9IKTxZzNfsfd3T2390um+ynf/tYfoLTnF7/8KWfzOZcXl2TJGO8EWaT46stfcnF5yvmDU5z0/MHvfZflu3eUrufRo0dUQ8bpaMbiecavNu/Y1RXnJ3N+8ctf0PYtsYzZbpd8/sXn3K3uSdKMZyeXGDdwdX9L1VacnZ9gpWOz2jBylt//zve5vHjEcnvDs2ePePHuBYqC7bpEDoJPPvs2b67fsO52vF2/ZrJKyeOUJ6eX4KLf+NrO0jgUpIcBrY7qTGvx1mDNgJGSJI44mU9xtkFKE8Rih6OwMUk5O7tASEueF+RFzGp9i3NhDDUvxngfuHdKK5IkIKa0zhBCobVgv9+jtcJ7z+GwZ7sN43njSQKEBGikNEIK+r6hbSvSRDOfz9jvHWmSBMalDRsuSeBOt4Ph6uqaoTcUWUEUK5I4Ic/TD8VjKUTYYMkIrVKybMTQNzjrA3fxuLnru/4DKzDWMX0X9jiHvkXW7riwD1b3wEIcGKzByyDyiWWQSYJFSkGaphS5Yx+36DhhMp6SFyOUgN1qRdc1YTTR+zAGGWniwZDlgmIc3m/T1DTVnqF3RPE3iJc4yknTjCSJjiOXYfMjbUjzGGMZhh7rIImTD5uhJA2ThGV14LDfU1UVxoSRX+9C8lxKGTirJvy6qg54J5EiPiawNZ4gSI1jEziDMkLJBikkUZQSJ5q+CzLXWAvSTGNsEHIaa4jTnCjO6foS7y1pmhFFkjTJj9IfiRA6HHPj6Kwhih1DbwKPExE2/M5S1wNRpI5MTkvT9gjpSSOOqflQaLfWHTc9MAw9fd8RNjhhPNp/kHOF507TjDiKaZvmKNIK52/YbFgkEpEIoih4Jd6PGltrw8L9eKykciANnpDKiuKE0SgnTmKatjq+jt/8UYwzTOdpuhJEuAan0xH/+LdLZhcZo4VnPC3Y71b87Bc/YVyMEFiUlFSHhrdv3vHxR5/StVfcXn3NZt/QDRL8ktEoQxqDw2JcxeAUQmlUnKNlQZxZRpOMfmiQOqS/un5gNl/ghh1t47C9JhIFznum45iqaTGmQRA2NkJo6rpjNjujawyd2eKd5ltffIcf/OBn7HcdSapI05qyKtnvd6RZyrgYcXfzDqUNdbei7jSLRfDRZHnMeJJj7UCaxMQywXQlcWGRUco4zVGqoG0cu/WGzz8958c/OKCUp+065osZfT/Q1g6pPGmsWN23PP3olAejhO3+lrLa0lc53/rkGcvtLY8eJYxHCcu7Pefnwc1QFBA/mPNm/Y6NXbJ4lrM5zVj++J63X27otaURnss0IU8E692Kb3/xGT/+4TWffvoJr9/9ikefPuew3zOdTMiKDCctq92aLMmxbU4kY0hKlqsNziuSpOWzT57z0799gemm1G3Hxdljrq+uiLOE2UnCZJbwJ3/xlH/8wd9RjBSnp49YLfe8fPUGVIOMEu6Xt2TxhLGBF1+94/LRJdv9a8qyYj4ZoUXCuzdLoihhGADn6MyA9/Ojt2CgGM2w/cD1dc1kMkaKiC9/fcXDh084VB0PHp1TNRuUcuRFxmZTc7YYMTFT/GzCze2a0wdnrHb3bFYHThcT4mjCclnx9Mnv8fSZ5er+DY+fjYkjjUCTJDmTWcrL17/gs09/h9/7/sfsDmuU8vRDy+nZgpOLz1hv11jb4pylMwPZKEy4dN3AdrMnz3J22y1fm6+YTAqurm9R2mLdnovLSy4Wj9msS6pyzflFSqQdFw/nvHlzQ9NUnF5E7NodNvrN1+bFbIFMe7SKyYqC1XpJ07Z8+euX1Nua/f2e5d0qTIUZ8MLhpON+d8V4KcnzEVnWslyu+eM//mNu3l0hNSExjkAREYuItqsZXEOeRrT9nvHJIwY/YTwasVmtqfsOsCRpytA1QMdq2dD3PeDYbTdESIpkhDUeZz3lbg9OUpYH1JMH6FQxHs2ZFAX14Kmqiul0SjdsMFg22y1CCUbpDCU9+/WKaAqtqXh1+wJpAjd5aDrSRCKsZ+gtUZRjlWU+W7AqS7Ztw6FucN4TKcE4i9mVCpVpThdzhHcc9geM6Vmu1qzWNY+ffMb3vv+n/NVf/jvS2NN3AwkKg8b1mnSWcjKa8Cw75ew0x7Utm34DBmrTUm5q0uyMqu+52+/YVAfaYeCwPbDb7dFSMDs/pTo0VJsdcZxiO4vwCuklkVTs1ivO0hGz0YSrzQ14RzGfIq1j08OeGmM9sWtIlEZ5GJxho3u21pA1MLGSw2FPaVvuqgqTZzx78pzNdsemqcjyhN4YvPB47fACPJKuMQjhkLEizjPkoOi7Fq00XdOwGwZGkwlCCbwKEj71flJPSbbbHbI3nIzCXkrSo4SgN56+7XG1Z5QXCOGZzxbcXd9R7nfEOmLwA6lOcV2Qg1qREWcaYWG/q8j7HanWtDpnd+h48eVrRmlEMYrZ1gYpNK5uOVQlzjuePfuYph5Yr+4pkoyLB2dY6XExHA4HZtMJVhi6usVJRz/0jCdjjDVoEdP0LQiBwzO0fUDvFDGxVkgtiZygGI+Q6QLxT620/f95GOOBI6bkQ6HZfcCBeB9S5Ep9I450Lgga3yduvfchXGkhSC4NUfReThrWUdJzfE6NUlHw7RzlofiwNlQqpKrfC0fbtsOYgX6ojylkx2DUB9Z0CBnEeGGCvAfBYC2ubcB7jGlxtifSMZFOkEe+uT2KUI3pw7pBhsS7cV1YZx73KAGNIpFH1vUwDB9S3kLFx+L1N69XCHXkuOtwTjgBLmJf1ewP28CV90lA5HYhNQ8SM1iC+FQiRIT3YdLy7naLFIoHD+eEoHuYBvjo42fc397Qti1SqWNaX31IOIeghf9wHKWUOBsmUAUKrSXev8dY1oCj7WqkFIzHE7QG5wYiIxlMF5pcrSCOErz3aB1S6EII0iQk45VscECSSnQcHwvo/th4AY6NCwlEUUzTNMd1O0RRFPwQh/K4D/Dh++GY1g5TA980ZKSUH6YE3k8ndG2H95I0Dc83DI79rkb4mNOTx5TVJkwMWEA6OmMJjTKBMx7jDEoIhNdEUYxWcZiOlRIlQoPE4fFOYi3h94/7xW9e1/H9+m8mHsI7DueGUgqt9YcJjPfHTUpJ3wUfjFRBaBquC/e/L86lsQNDpdFSsi53TKIRO1Mxyk5QOJyZMS8MZyahtjWrqiTVEftecpJEHBqB0GB7wTQFgeWkWAc+ZXeOouemv6ZrJSUxQnmEqVDasasmaF+hhMAMp/S0ZKplPbR40zPSKYvFKbv2NUZPaMySsUhpOmhMxsXkHNv15OI52+gNTWvJ9SXC7+iNoBsKTiYLSrvC2J5IXZDr9DjS4IMosnUU0Z5xFrPZm8BGFwkDKdLAZhfzrl+z3B0QouddOuanL3b85OclpouJsnu2B49zEj00TMYZrhPMLhTltiOTOWawRHFEWfdYBoYeur3m/Pwxi/EcnAFng+nb+yPDXLCvO5a7AeE8+UyglWe5HNjd9ySZR6UDxBJlI2xncL0kKSzFaMR//y/+O7I0+5Bwd95jbRhbVlIxLk745dVP+bsXf8N0ntJ1Oz6aNIik4L6+Y7kMLLb1OgU1gDiQjQ3GC96+ODC0Leu94+zBmENTYtuYJBvxeHzCLEr5z74n2DYtUhnGY8Ekn6GSA6OztxQqprcdSX9CqgRxWtKJHqPD90NnMlJqmuiHDPk1GS0vTMMwOKQYaP3PcH7CoZH0fofOTthUt0QiJI0RcL8ecKLD9hXdYHFN6Dou15K7VQ0SDkPHYRvTN5ayCiz0ItfsNgqhW3aHgd5ZkjjFCeiMIYoFcdZztx0QiUYK0LGlN4LtXU+cRQifoLVks9ozGSdIodlvBqxUFJlCJo64VJjWowvNycOMqq2YjcfY9kDXQ38QjBYxVv2nBQj/scd4dsnJ+WOurlcoJEma0dueYpTRDjUiFVhrmJ8vqMqK1foGvCSNM3pliJOYIsu4u7thHGdMzwuyLGUynfJaWO7re4aq5+LjB7Rxx4PnD7i+u+X73/sjvDSkec5u35HnOW1f0fcwHk+YzGd4oylGBav9PU4P3Kzvsa4Loz3W8vrVW/7wi99nnI8oVxuS6QQ1TvnBy5+SjjW2GlhVB4hjHj59TrndsV3fc3Z6zqgY431ADJyenPHw4SO+emO4evuG6lAyKgq0TDg7vWC4u6OlI4pGnF3kXD465+u3r2j6jlE+Byf4/Nnn9MLzi69fcjqd8+T0ghe//hKnBEkTxs8aFZFnExbzGWn2CV+//po8zVAqpKrnp+eIgwQhuF9t8GbAO8tqu2Z32KGk5PmzZ5h+oC4rojTm3eoOoYJZPC1yDk3NerPCO8/F4wdUhwZ1kIyKHIDpZMyoyMmLEau7JVEmESlYb7i+vQMl0UlENMrYblbkeUYexywWc+73d7x9/ZbPvvUtZFryo5//gLu7G4yCL6LvIr2nWe8ZxzPSJMVLy8NHF/zq5S8ZTI0SC5xtqaotSTonzaZEsabc7Pj+9/+AwVmkGvjyZ79iLFLOzx+QppKnizOGbuDR5SVv3r3h3buvOT85Z7V2tF1PWVZcnj5gMZny6aefMPQNL1/+GhsbykOJljGClvXuDtc3PDl9giktvrdI6bh/fUUtDG/fvOMkGvPsO5fcbrY452jqmnY0IsnDJvE73/o2k+mU7XaLdIZYCbxSZEWBWy5RKmZ1twZjmcRjLueXPH/0nLrq6ZqGcRajrIDOoYxnPpmRiAjtBcvlGikt8XhCJDWfPH+IG+RvfG3PphPSNKGqw8Zx6Fu8MyTxCCmgrkqauj0uMsyRNRgYcnmRf8BmeDz90AE9w9ARxzFdG3hzQtjjYlgSxylltUPKMP4Xiu4NwxAjZYTW4QtcSn9kpSeAII5GAVMgHHEiUY0g0po4isjSlFhnlGXHfr8PSSgpSGKNEoK4yJnNxgjhg6x0MB+SMnC0vYtgZJciJoklVVVR1xVae9q2xgw9w9CS5wWREjRtTxTHWGvAeEbjjCwXODNQN1X4GTeE5/cuCG+sQcnomI4JCag8L5BKk2VBqhrrCI1iv9+SFwlZlqPjKKSElCROYpJkdMTuGLSKEQTxt0ASRRGjomAynjEeT7HW0rYHBtPhOxPSyDZsPJzzqCQ7cjS74+hqy2G/57A/UB5qlIywJjDapQyJpb7vGPoOYwfatg0iHhc2cpFO6GXD0Lf0fc8wDGRpyng8JYn9cfM3YO0QNuLSIWWCkAaEpx8GZDuEwpvXAbGVx+joiIo5skAjnRBFGd4NOCdQKsYYz3AcKxUibFzatglukeiY0MITaY21Hda4oyclJLzMEHBCZVkek1aaSKvjhiqkpLIsZzqdheP2HyzS3yffrbFh5NZ0KCWZzxMmowLvHW3TUrctzobNCjLHyxLnBpQWJGkUGIyR/rBJ6brhN762AU5mM/a7CixsqopEeS4fjPmvT8eIyKASBTiG3uN9kD8VozFYaITl61c3fPTRZ/zOt2d8+as91V3H0GnquiHNwDrD2fkYqTzbzYbxaMa+XBInCXk+pqm7wE0eLPv9gcl8wmHXcHI6Z3u/5+pNKFxI6SnLDffLHbHS5FlMrFO6tqdvHft1zWw6ZzxKkSLn9Vev+NbnH7HcvKFtPE1SkaYxN9eedCaQsuPkNMeYPd47lsstKgYVO1QsmE405b6lLCtSbfjo44/ZHbZ41ZONRrz5+o7DVvGHf/AZ2+0dv/PdU5LEsjssubg4O6bmcq7f7WAQnC5GvH1zy/xUoyWMRjGbZkvTzqiaHfk4ZlduyMYeryzN0DE7GTMuxsTrmKo+4NKOSmnS784onszoNyX1zT0745BqQOqIq5s3PHm6ANEzm05IooSbXcUXn3zO7rCn3DWcLi4RbsfqfkWaKbb3DWk8oreKppT0Zcxf/MX/gdVyg1ndstkv6VsRnCm1CgVlbjl7OGa1XHK/uSPLpww+A6kZj3Pu79YUqcXT8ejxGTqRPH1+wepui7Axb99cMxrnKO0ZbEOaKnBwfX3H3W3Pg/OHxFFO2W14/mwMOJ48fc6//3dvKMsNn352zoMHj3nxVc2hbKjqW4SSVHXL2ekJr17cUVWGcx3z/Mlj5tOWd29vmE0L0iLn9Zs3PHn6CKKWIteUVcnjp5ds1xWjyQm3a8/VzRsePfkMcd2yXN0c+fc9k/mMOI4pywNJkrJaHvj/sPZnzZJmZ5Ye9uzhm30+Y8w5IxOoQgFVqC52Ud3somSSSTTTha70B/QDdSGTRJqRTVHsbnZVoWsAkEhkZmTGeCY/Pn/jnnSxPQKgiVK1Ae1mYZkWedL9uPs37L3etZ51eXmOtYauifeFen/g/HzCyq64va6pioIiK9mvOny75tHlJ/QqQWDp+xuc2TCepPzzv3wAYqDrBkbziuXu6vc+t0fzOenQ0XUNKrV40SNUwX7fsL5ZcvfqFudBpSnWGQw9OoPGwMtXAdDoNMN4cAHmZ3NaeyDIlFRr2kOH9ppS5vSuQSnPo6dnlFPBq7drTss5h6s9vR2YjEaMijHb/Wu6YUc5GZMkOfv9nr7rqPKS6XTO96/fokmod3sSleKdQWjIqpQiK8l1weRsjJ15Xr1+g0o0xhtc6BEOUgpcHejWkJ+MGXzH/GGGGxQ2cRzWBwYfU2GTyYgew+nFAp1m7A83BK0IQXB6csqw15Sp5mw8I81ytE7Z1w373eHoTswwVvI//Ju/5n/9L/6K0eicw27HbrMn0ZIik8ymU7AWU+9ZXm3w9RmjqqSaLiIebwVFsIQgOAw9JtEcXOznkGqEcQGUYLfd0neGXd8zPznH+A6hoK4PFFXJ5m7D2bMP6cxw7EpRHLoBYeBclbxxHR09zDNGoUANHltpdsrQDoZikOxWB0w9kFQJMiu4fPCYtzcr9psNeZbS9h3OHcsoE0GSpTSHAe8laSIIEnQWy/4QYIeBVCekZUqiBSGRoBXJUbQKQPBRDGu6ATtYTi6mHLYNqu3oDobQabTT2NqSKE1/aGMyzVl6a3AiMJtOKIsxk9GM1kRBq+sHVoc188UIupakOGX7esff//0v+Rf/xY+RwaOKgqrMSbSgberId/cJ3sQ+AQKMpiN2zYG+bTDOkKQJpu8wrkdI8MLRm6NwLnz8p4+4FNf1JFpQVRlpoUEcO0wScC5w9fr3P7chYlmAqOWE3/Kd37mbo2kyvBcuISYRI8olHPGucVjvXGSUKyXeC8tRJEyOgm4UDSN0/HcRI3Fv75yHY+EnQAgeY8x7jMc7cTiEcMTbRUFVyuj0jQiaKHbH39Mfk5A2Mq89IFREhzqLsUN0myPwIaBURK0kiY6vKaOJZXDxd7AmonekVEjleVcgqdSRsw7vsTjvHftCkmYZQ2/e/96xv0f9Th8Q8YVC5KoHosO8rluur+4oioTzyynOWrI84+LinOAHtruaSNk5suiJTnrnIqIvOu4F70i7cV0cDSvvWONSeozpCSEcxeeepj2QJiXexwGRMTE1mU6z45pVxu6PdxiTlP8ZOkZIEVEu4h01Xrx3pb/TltMkAy3Yb9e8fXvFt8+/Y3m3ZDARF20jUBwh5RGJIt5/3+++W3NMLMTnDe+HCHEQ4+hawWR8yp/+5E+4X93w4sX31HUN2mBcS/ABZw1KgbAOkWhEkHgXB1jySNsQQkaxPBD3mUIShCDwTjCPw43Yk/RbxGI8Ht/9Tu44qHq3nuf9z70fdPj4zziIUQRn/9PiXLIwx7Ups/GKqiip1ANIrxl8i/GWs9ElSuwZpYFKTbjvBErlCFuxqdeUWYGlZjIe0ZoCO8wYxJKJOscLSEJKHqZ0MhDsgTLJkElB7zsa2XBoMyaze5Q6Yahvkb6gSC/xfkdjWoQemOkFpVjRDxlDMmBUTqIkq8MVkDMbDUijmFQnOHuP9zmpfgxiQ7CGNDxhkhkG21MUBcZec5Z8wu2QkkhD779mqh/QBs3zqwNvvvsezz0+Ebx5MeJ+u8GYHqkk9WqLLBKMK9jc9lw+LjjcDpFfPBU4O6C1IFiFaSQnpxnrTYMLPaVMsSGgq0ApU/7Vz/5lvGBYh/fvLnbRReet4eb+lrQMJCJn6Bv6WtLvDcEFnJfY2iGTwHhS0AeP19A3hv/dX/wlf/LZj46iyfECHiCIyMZL0Dw4e8J/9zf/mqbfs39Vx5PqV5pf//wKg2O3BJ1JmrolVwlmGOLBrwNaOFyn+OCPR/zw48cI3TGb5JBoLhYZVbpByZbBQ5nmGHNglFlqM1ClFat+RVZKgrzDJXdsbUAkJ6SqQemcMn2IEd9A4jjYFTuv2Iq/RolA7FZocGJH7yzafMK6/4794Ci1RImOfrDsa82+btjtLPtDzouXexIp2W4kRqQg7fGmm9F1hsH1NKtAmwRsCKhEsttZJnONbRQud/S9R0vB1dXA/hCYFBp8QGhPv4WqrPDBYHpLkqWkeULTWqaTlNE8QQRPkUnWq5bpqcZZhfUeJ6LrY7PZM3ug2d4HJqOSrm9Q6j/6VP5ffDx49DG7XcPV9ZJRUdL3NU8+eMhmd8+hNSxXt8zH5zx/85zZaE5RVbSrPc2+IwjFp599wm9+/RX1bs+4Kjk5W7A4nfNmc8OBPff7jiopudvc8eyHH3L68JTx2SQyrm9eYa3l+vqa+eKcH378CevdFbc3NwiXoKXkdn1DUaXcbN5C2qGlRIUCNziqrKTZt1wsLhgVht4JHBaZC5yw7Js989mcTJdomTCppnhjcM5xc3PLk8dPqI6FS8+ff8PgWp49e0awnqqsuHzwkN88/5ZMFyx3e+79nmIsCWJgcTrmm+cb8jygSAmd58tf/5Inn3zEH//oRyzf3nC2OKe2A8MQxZS+Hfjii3OaNsMYwbiqOJnNmM1GfP/yDU2zRyeKvCz42T/7Z/zmyy+ZjEsW8xmvX75iNpmQJgnb1T1/+ic/xTjLerel7mqkjCKcTjU3z3/Df/af/XM+++AH/Pt/8+8ZtSPGZcVsPmOz2WCsQEh48OQB55enfP38qxitn05JVMnQWTTxJr9rO86mc25ur7A4EpnijKGxDdv1HWenJxxMA9oTrCVIi9SCP//Zz/jyN7/ksN9ih5YPnj0iTSRb3/PHf/IF6/WKYWi5ub2iLEbcr25ZnJ7SdY7Z6BSzrdnvWz7/8R9xu1qzW23IE02qAu0xMmrMEFM0JpCplM39ktcvXnC/umMxW9BmGcNgEMFRFJIPP3vA3/79z/mH73/JbrPjLJ8z7Buk8QTTURYFudD83V//HP94hj7JQSj6zpBmJUwUSVnw9uYKPwy0dU3Td2STEau6ZhgczgRm5YxCae6vbvj4zz7kZLZgaD1tu6Y3O5SHy5NLlM4YuoZ2f8A2PWbfMpuPkDagEkVeFeT56Pc+t8syIibeFyEKQV5kVKPiWObj4MjEt32DUnC/WjIdnzAeVwhkjGeans36ADKyAYfeAIp+aGPsd/Ag4gJfEBEtzvVY11CUsURGScjznLIcIYQnzRQIiyDGO9uueY9x8cFQN/sjvzriV8ajjP3+QJYpslzE0sikPBYhabI8Ra0DNzdXZHnGeDxmOpmSpTnGhiOfENI0oygKhiE6gUPwDENHkmhCSOk6TxCRz6eTuEFo6o4syxBSxAGB6o88RIs8LlbfbSYOh5osm6BkTp5nDCZ+xmVZkiiNHQaMHVicTCmrDGsN1jusd+gkfR/X9SG634t8hKcFTBSY05Q8j89d1zXOR/xTPxgGM8SFaAjHElf1nlnYtAfaLnYixIb6WCosAoxHY4osh+ARISAIaCWZz6Z0bY8IEms9wUdninWGut5TFBVKxVjsu42XCAbTdQxDR/A9LuQMfU/btpFFHiQ+KFRSImUs/1Qq9oRIKamqiiIP7GV8Dq1ysrSiLCpcCDRNe3wPnr7vGUwgcRKt4wLZOhO7H5xHqRyp9FHw77G+f89LdM6THdMLTddhrWc8nnJ2dkFVjTgcdsRNl0ZJgfP+uFHj6FjXjMYjRqOCEDzuyKCPor9GyJR6iK6jUTUhTQTj8RiBYL+r2W3jd/eHPNxgOJ1P2W62JDqWQHXdltl0QlpWGO958+aaw67DGY3WAiEsn3z4lKZpef7iBa9fv2I2P2V+koCaoBUkSfy8dBJYrzdcXM6ZTHOkMAhjkTKn7Xom5QgpFJ9//gW/+PU/IA+KTz/7kOe/+Z4HD865ebPlh5//EX/35d9SlJCkMHSOVEduv3dQ5hXWBJTM8B7G04LXr98gQkqaT1jerTg5rYDAYjqi7zzlPCX4jr41XJxN0YnmxcsV8zNNmmmst1RVzsG0aK3xeEbzgtVqi+4VT5+e8paaf/z7X/HhR3PGU4u1A86bOOTwktX9nt1+x8XJBeu7mq49sFsXPHxUMNiBooTV7pr5yYh2sAy2oRolTKaxqHW73nL7fMDetoxPIQk9SnlkAtYHnn30CQ8+WHB99y1JGl1x1bhAhsB2f48McHN1zcXZKYmSfPHZD/i//t/+76R5xovvNvzojy5AGapKs15ZXn2/48MnH/Pd8w1KpmybNScXI5a7t5RVSl0b7u4PnF9OSIuBcpTy/DvHsLznZGEZj8eUI43SLVla8l/+F/9Hnn/zD6wO39H0PUlywsX5I7arXRwY6pwkdQxdh5cOqQLtxnJ329LsrvjBJz/i8aNP+f7FlxjbMR45/vTPfsirlys2G8tu19MPgaYbqKoMISIebzA9y/ua4DLurtf0Zs/J2RylA6vNDVVZsa0d4cYym5XsDytCkOzrNcZbmjbj8sEl+90W47cY10VcSa451DveXu1I0vRYBpxwcnKCEIoizzDdwGK+YL81DL0gSwXTaUmixlT5GILj8eUTvvvmls2h5/RJiR00WeGp6xXOZnHQtNmRZgrT/f5JE2NiZ8ihbnA+cHY2YzKeE7zk7OIM3xus8TRdD85RpBmm60irlO1yz3Q2Z1RMSHTCer1iMs7joDgrGZUjDsvDEbtl2bY7itMcoaPAYO2A0oKm2ZGmkKQeoQxCBrKiJE1zEqkjKk2VVPkUJXISoTibn4ATSCHRySkyBE7nJ0iRM5+fMcpzgvd89+IVRVlGZ2VwSCE47De0+56z8xPOzk5Y12/J0wLjXSyi8566bijyEUrlFHnOdLIgSVOSNMUGR55lZGlGMVswNA1SFAx9LJuWIo3Xfu+ZzWasVjvapuG//x/+NaezGa1pab1BaYUoJEM/0LQdZVWAltys7zEyMBpX7OsdVgnGswXD0NHUlr3pkaOSXXtP2G9YDGc8PD0DAmkJ9XbHSEtCobHSsG1rpM4QpaJYVBxci9WC1e0+7leyklRlzLzmamgR0wUP8nM2XU9XSfYvXzAiYXu7wu06tAdCRlCKr37zG4wTJEROcnvoGBUTamNJZBEHSbQkRUaSBHrTI3zHpt7gh9ijk+dpLNcdPMHKmPLPEoSOAluSpKRakqcpITjSKoNaUTcNeRoHPlU+5tDsyJOMzW5FsAHrAipNSfMCEzxFmYGSzCcLbm5uWF7fRTd725FngvvVLUNouLrZs19/hDKa2aji+uoNT58+YDQa0bRtTO2MY3H8yeIEMwxsNxtUlpCleVy3Bki0pu16qtH46KpXVDoBIWJp/eAwYcDans4MeOGYTqcExLG3ZqA3f9i9O7LJoyHgnVAe136/df6+G7ZbO/wWbeE4in5xLau1IuCRwf3WKez9Eb0Rec/BC9yR86xUFBCttb9TLB/eu3N/V8x/V3Yawm+NekJKrHPHfpcMxO9yqI/JSp0Q5DuUoMKFuJNQKvYSCamQxD2D8w68iYnooLDexbaYo6AbCAgtEdaDEscSyMgW10qRJBGJY6zDuYhxEkKhZYJ3UNdtFLyjneboVH4nqEZUSNSa42BBiriH6XvDarVlsRiTpCmjakSSKCaT8RHXF7ud3g0j3onYIfwWRxOEQElxFP1/N2UASr1zdkOW5dEpTXRZmyGmWkFQVZNjUb2i67qIYS2L4xrYIeS754Lf9uyo/6/DLaZvJUonmL7n/n7Fv/23/47/8PP/wGazRSd5/D2Pgvhvndj+d44L/76HyNp3Q5aYSBHvnP045MHxD3//JT/72Z/y2cc/pSpO+cdf/APNsIzvUcf7jlISlDweQyImHQIoqY8DrbhHccfPTica966wNrijYQmE0CiVkWXZ+72BfPddeIdzIjLYhfyfoVwguueFgCSJaQYp5fvX+6ce/9HKm5B7+mDZD4pEB75pf8VpECTuKUMYyLKWSkuCO+BMxjz5CT2/xqg7hM+xco+SB9ZNi2WEHHZsnMIXBVl4RMceESyTfMKhSSllRZEVqKag00uGcE9vDYIt4+SSXAd6NkgFuUjI5BhrmuNCJ2fvWzI9RdiWtEixIaH3e7J0D2JFoiBJnnJwAwSDVD2hl2yHniSM8GFNrnMGtlymD3lT33J3e8rdUPDX//gd398tuX1pkLLAFRrFgUNtyNKM0GjykWZ9OJBmMUqwXVpSVWCoqbKC/bUhaEtaCUbjjLvXDWtTUxaS+ShF2QJnex5Mn/HPPv8LeH8BE3Ez3BxIU42QAdP05Lmm7mqCh2meE1xLcII29CQkjLWk37eExCK8RKI5nc7pmw3Z5CQKJcEhpUbJ2KadJhlVMePQ3scDrFB4qzhsFfutQciUvuvwNkOIJk7xQopQnp/+2TnV1PKDD58ymV0zO90SPGQhZzccmE+g6SxCTwjNLeQ9Quf0vkF6aFtB8BlBBUzoKcSEIiisgSItqAdJZ79DpwlCeDxj8lTStre4sCeVmr7XlNklykpWu9cMTlK7nE4m7LY1251l6DXWwvI+Zb3p2NcSLTOGVpGWKT50rO88VZHSNj1CRfHGCUeZZtgQePJ4Sjt0UegeJIksME0gSzLKicMZR6YyatNFzmaqcD0II+gOHeNFytBb6r4lCQKZBg5ribMS0yuyJPLnVzc9Ohc06x7rPIlWmH4gWIVM/rBY2bdfPafMC9zg6UPLw4cPwAeKImN5d8tkOsO6QJYlLM5O2K5WjMsp3gamJxOub67YrTfkOiXTCfP5lN1uRe9aZKGwracOPaPTKQbL9rBjPpkjpaTtOu5WS/KiJC9S+r4jzTRPPnzC/e2Ww7rl9OKCptvx7KOnfPPqF9geMpuT65LzDx4xzyfUmwNuMOhKc399xXRc0pkac6j505/8GevlDuUFZw8eUe931PsNAsF6taHvBrROefr0MZtmhZaKs/kp7aFlt96RypzBOH760z9DBNjXt9wsrzFuIEkSvA+kSpMVI378xY/wiYoC+mzBxdk53759ydvlPUkSqKoxZZnh3YDpO9xgaA8NkzIjVYrgPUpK2vpAcnnB4ycPOTuZ8/r1S87PT3n04AGb9Zo8z8jLjPPxGYtvZ3z04Qe8vnrDYAZ2zZ4kTZFaobKEyXxCoiTdvqZvavI0RWnJrt4jlGI8rTg/OWM6GpOmOS/6VxR5SdPHuN2f/8XPWN3d0bcd46zEBo+3jqRMOD95hBt6Drs9y9Ut5/MFk0kJzvPizfdc3bwF5ZjNJsymY+439+R5ghCOR48e8PbtNbe3t5yfCSaTKUpDqANPH39A+STh9dUVh0PD0A1ooVjf3pGIyK7+8NlT3ry+4mZ7z9l8wcfPnvH61XNurt5yf7/k448/4my+ABE41DtWyyusdBSznLvNDTSOwSRkVYX3jkwn6DSjrxvq7QE5T1HzjKHrqU5LVJKyCTtevn2Ltz1Pzi9IgMlsSp9ouvrAbDZnMZ5wPl0gOsNf/PBPScc5d3dXbO9rkkTRDo6PPviIy0cf8ObtDVe3d1w+vOCDR0/4wUcf8euvf80PPvmC+/WKLjiy7PcfkmVpQppqxqMSa1uG3iJEwAw9ISsQROeAlgobwBjDdrvCDJbJeM50OsNaT9P21PWOLNfvFyDjcUU1ytluN7TdDmFjsdBsNmKwA/erKwbXoLTFOxPXyXrEgwcPMaan7Q7RWSw4LgA9zhus6UmUoCoy3NDSNS0iT5lOFmRJhtKBJA0xEtofkBSEJI1RRD/EDTmBMsujU0dK2taSZhlpmiJFjBMmiUIIRT8M6ESQ5wk6EXhjjolIibcOY/37mCcSmvbIOheavCzIUs0w9PRdE13dytJ1HVn6Do8WIs7DW1CKNE1IszQy2L1ld9jQDz2gKIoJNsQobdvHiKvWGZ6IHRkGi7PR9dO2LYfDjraNYqx1jr4fsNaTHXmyaRpF+b7v6fruyKKGIssZlyOM8SQ6YVxVpEojpKSzBq0UeVaQJ5qDbrEucuh700eeuh1wzrDbrXEWvEtwJuJzRDgy8m1/dPx0ICTGDXTGEURCnk/IswQ3RC5430XWuhSKUaXie3bgnQSl8C6Wuip1LM2yHu+H+DreYK2ICQYlCRjkEasjRfxjjOVwqEEYptP50bWSkqgoiHbD8L4QShwX2sZE08K7IY2xPXkenU+pVsxmY0bj6IQbjMO6HucGtM5JtGawJhIcZdwcVdVxEGChrjva5ljm/Ac8pFd0dUd76CjzFGs8682eNM8opyXBgVaasiioXU+elZRFDiKeQz7Ab77+FZ99/gkffJyRvJV0TUCJDGO795Hvu9sVDx6e0vcdRSmjsxqH8wNZmnC/XDIZz1jv13z9zZeURU5/2PLw8YgXb75isDuKSvDgQYk0Celx/TKbTXn1ckPXCD548jnffG+5OM+ZLyYsVxuePTthklR0rTmeoxkvXyzZLC2ffvIA5Vvqw56T0xmJ1DRbgw8DxUjjQ2A+n9DsY6LCydgTsNms2IWWs7Nz0gT64cB0KtjumjhwMo6hdzjrYtFmveGnf/YTfvPr7yL2yBoSpRhPStabjsF16LQlIFltduRlSl4m9LbldtdBqrG9IdGWrFLoKiUdS7786jUnjwYmiwpnWrSEru84W0xxfmDo4cmTj1jevGWzXZOXKfPFmPV6ww+/+ATbGgw13SGQKMlnP1gwKlouLwr23df0okXZislCofWC5fLApJxzv7xnfiqoqpyz04cUecpuv2FxMiZJMrpekKaB//q/+a/5X/3lj3n99/+AwzKYFG9akjTnsGyoW8PFwxFZniO1Y73Z4l3Gs2ePEV7zD//4Sx49mHP5YILDs96s0EJSVjkXD+a8frPi9VXLdA6r1Yautzx+MqK3PWeXcxQVt1fXmKGmLBLOzsZIndMPPX07kBUepRxKeq6u14xGC5IspRt67pZrdGoYDWtG4xyTZmz2S6SMJhdje9I0Q6lY7CiImKfRaEKeGvK0J89yhDIMfcvt9ZaVbLk8n/G3P/9H2gMcmoGTh0/JkwU3b18xm42ZjEasVmsECrPztLe/f9KkrteMJyVZDjBwdnZBno0QZBSnGSfTGV07sF6tWd2vsZ3B9g49SzEB2sPAZDLGuoE3r19Rff4hKpEMpkcrxXQ6ZX27YfA9Td+SFjlG9sfhY7zWGtsRnGU6PUErT1EUUdQiQeucoOP1KxMVoQ98+PApo3L0nsPc9S3SBZQUjOcnzE8vsX3Pbn3P/OQUjilLJbKITMgDiZR8/OlTDnUs+VaJBxkTShG95knTkuAkRTFCqgxHoOkaGmOYnczxzqJ0RjscuL9vYnF6rjHecmhqklRzcfGA1WpH3RwoMk0zNBTjnGScglZURYnqFG3TESRYLJvDln51x2WuWW9WeG+xiSDLUlSZ83p1R4dl4wZCJrl3PSMZ6FMZhVct8UXKaFLRuQNFMWLQDj3O2ZkGSWC9W2G8IEOQK4XwgcwBwfDKrPjh9DEmy/jy9hXaaapesast+66nTFKaQ8OL+yWuiMjGEDz14RCFMOvi94HADANZltG2PYN3WOEYuj2D68C7WO5segwWoSQBQZYlOBf3hm3bEVR0g+o8RWiwAspRSV93UWyfj0hkgtAldrCgZEyKdRbnQaqE1hiCAdtCUJLJeMpar1AuMDQD89M5b5fXkBrEIHj+9Us++9EPeHl3S3uoeXt1xaeffkpvBl6/fk1RVaRJymw249e//JK+68m1QvqAM46hNzjj6DtDlkcHtlQylqhmKWWZI0VCV/fsD1v6oQatEEnC/d0KKSPne7b4w5joSZIRU5niPZf8ncNaHDEnMUU4HHto1NEYoY8ObPk7LmGFklHwdi5ec97x1aNDOuI33v2/kYFu488G/z8vwwy/FcOl0oQQS6tjUboiSdMotCIQSkeuvbeYWOhDqhQ6SREiBSRKarxxeKLrXiiBQoDT+GBxITLcg3VHk0NkVod3aUoV0/wc+e4+JMcugSjAu2Bil5n3SOGP2JQMKSP94bBvsMYTjp/DOwwOHF3JIqYwImP93XuPe/H6UNN2HaNRyXwxjwa3PKOqKryPSKvfxbi8+x7f4Vxi8va3rxeHCtG5/s4FraSKvT86izjEJI9JxneMdQRd1x/d0wJZSbSuCMHijt1WachRWh73Lb/rRI+POCB4NxyJqcz1esPd3fL9sWWtozceQrw++BBLU9+VzL5LHWitccdiznfHn1LJcfjq8cGhesF/+9/+j0gp+Kv/8l/x2Wef88lHP+LX3/5PGNeTJAq8O5o6FFLEZIZ1MTalfEDriFVxzkVRW4I+vsem7bDWIJWEAMNgfwfZEtA6pi4GM+CsxfYDAUuaZkcDjXv/fUkpYufDcbv2u2ief+rxH70zf1A+pVctIm3pB4vuPUl4hggJE1FxGH7OzaBQAapUs2ruGVcFwVg2zYZp6kiEwocc57aUo8+p6gnjbMzucM8sm8UFedCgVhx8S+ILQlZw6p+xDc9J5If0bqDMR3TDLRkV94eWLGuwWqOSCYUaU8onqPANTnQMPkOHlEyPcN4wVT8glRNa/w2N2SKV5ySHVZ1hKUh1xdVuT7svwe44mLd0+5e8uBZ8821CXz+nbzytBTuUpCWkwTIYA07RtY5xkRJST+4ybO/jxSJIxnNJ22aRvZNoXBC020CrW8IooewLtJex3GqwlEnB//mv/i+MixnBN8fTKZZC3F+/xps9+WLK89ff4JzB9/FEWa1qnBUkhSBTmsQlaB3LntoVOAyTRcKmvyNJq/dxnzi5iezPRKWMyxlSaJp6i/BQNwNZXrC87SiKEUrCdJzy4NmYKrvkh58+4uXtd4DgX/0VwII87fD2AtwWEWwseZADiV2QaEMiEhJ9Rj2sSFVH7zzC51TFBNNldE1LUSbULkWngd7saI1nIT9hmhmW3RtccNTdgK9ams4S7Jh6WCHUhMN2y9AE2s6wrQf2jaU3Gd9+ZfACdjdtjGRfpKSJRnvLZK5YDdD0FvqCrLBg4o3L+YFqnKBlQpZLDquUoTMUZcLBd2gvEEpQjVLGi4KblQERJ6n9xlMlJd22ZVxUqEzT7z27u47gA6NZgjSSzX1HnkuklmyvDBePRvR9/JlMF6iZwrWKg+lJMui7jmT/++MeALSxXF99z7gacX7+mE8//oi///Ln3K6uKfIMesX52SVKZWw2O3abHZNyhAuG9XbF9dsbnLXorKQoC2QisToQXHRUplnO4/Mn7FZbQOOHWGZ6e3PLbhc34/NJceQU3uDEAe8czcGQ6pJDfWAYWtCCpulIQsJJtWDwgcO2xe8MCRIpPIv5BDfkzKYVL66e84OPP+Z0MePlty+osgm3N9dIIfnwg4949uwZaZpyc3PDfr/n1cvv2fUrLi7OaNqUyWjGbtvwwx98wS9+9WuePHnI66uXyEzgPDRtT9M2XF6MmI9PKMOcuq25Wt4xKnIenp/TmJ400VxeXHB984of/OQztpt7BI4sUTx7/JjLs8csb665nJ9jjMd5T6oE333zFQ8fXtI1O+7evqEsS549fgDCs2+2rPcrVKpQQrC+XzEuKjrTkSD5/NPPOOz27OooqAjvmVdjMp0wmU059C3//m//hizPmYxGDJ3m0cc/ICtyrt9eY71HS8njRw9ZnMxYLW94/OQRiUoZekN7aMlHOS9evuB8scA5x4fPPuDh2Smyd9zdbZnO5/zdr3f8xV/8jK5r+frbr94P/7IsZbfbkyY5aVIAkvl8TvCeREhOFyfs7lZkecr+sGU+n7LuWnbbPeP5iIc/fsSbN6958/aavun54pPPOJ2N+fnfvKXIC/q2pe96BAFje/q2pusbklHObD5n++1zfvrBD0kbwayasFpMWW9WLNcr0i46PPMQ6OqWxWQMpofgefLoAW9vr+msYXc4gPWcnc5YuhZrDUWZUhQFXdPy7OwB0gW++vIX1O3A6fwBCIXoJKvtFqdvWe/2pHnKyXyONQODNXjryYsReTcweEXb/yHlgwFrhihmusg2LIqCREfnb9u2OOspi4JUQNs1hOCxrqPttiCH48IbfHAMQ0zTXFwsmM5GDMOB9XoAIlezrvecnFaUacZmd8Xg9mgVwAWsBS09h8Meay3ORzbhdDqjKkdkR1Hfu4EkkYxHJW1TU9cDeRo5jFrpKJJKG2PdfUcIA9YrjE3xvo8N7youuJxztHWHD5I0TRDS0zR7mnaP9yayQGV47zyXAmQiouAt7HHRC8MQy5h0ktD1lmGAJNWkacloVNJ3Hc3BEZyHEAt8jIkYjzLNSBINwkeXhoy4lf1hR2ol+3pD23VkaUmWF1gb6LsofAcEWqd4LH0fhxxN0yHFjiTpaNoaHzxZnuLaNuJnju81TROUjovcw+FAIKATTV7kYC1ZWhB8j5KSLM3fL5AhLp6VEiidIVVC3XS0/f7IDzTvBwNtV2Mt4DIEGZlOyFJJoiXWwmA6BuuQSmFcwPvInUz0caNgDZ1tj66Q32J3kiSnLCc0dRQ026anaTrSPI0bMa1ARBdJMP7Ia48bhViI5OLn7R1dF51Iw2Ai27+omM/neC9pG8NgDEoqnAxYG4to37HRsywH4RAiQIhlVKPRmDxLyHONEJ6ub+n7lrY7MJguLsgTiY2p2KNzJ6UsRmidUx96hi5A0GiV/0H37suzS968fkOqCtouct6fPf3s6CKtcThOThdsljVlXnF6csLhsOLli1ecnE55+OCCl69ecbe8Is0hzQfMIElVxX53wFrJyUnFZrfi6u2Op89O2Ow2jCea5uAxpmW/twgpmSxmNH3PYb/lsD0wLSds97eMRiOc6+g6TZEXfPLpZ/zil1+iE0+aRGPO7a6hbQbcoDHG82d/fsGvfvkCKS3GQNdCKQJZqvmjLz7CdDnXr3Y8fHjBIAP7TceHTx7z7Yvn4CSmN+RlwaScsbm/psjHdD2YznN5eYYSJZv1Hpl4TOgwThyTOWANNE1MLOz3B6TwLO/fUFUJVT7B+T3b+oAQMJkWtH17ZOpK8kLjgwHhSTJNNhrR1Q43BIIYGPzAfCaZTRTzL6a8XL1GV4pUlRRVyWE7cHO7xgO+d3St5I//6CPWmxu+efEbervnfnvg9GLO0NWcX844e/ApL76/JjjNpBrR1DVO7slGYNyWappi+jgQMAOMyzFtvWFWjZlUHZ98+ozt7o6+t3z3zUuurmp+/ONnbDfXvHz1NUV2yqu3t5ycW4TqmYwLrm8EKhFYF1M8xTjBek8IPY8e5bg+Yzx+xhc/eMabN99yf2soSk9aDuzutozahIePZ0xPnvDq7bfsDh1FWgEZaSaPG9+OYD3BBFwPlJ58lNGZDplodJJwe3Mfu3BsgvMHzs9O+O67V1xczkiygaZbk6nzuNGXIg7LEoVEMAwD19fXXF5eUBYxdbVdbUjTlJP5CVprbm5fIaVischIsxyROLIJjOYF3euezfaW0SgjuARnJcvlfXTDDQFTRzzJ7/uwtqFpBtI0snS79oDtBaMqZ9+26CShSlNmsymPHj3k/u6e6zdXdIcOmWlUEGihCcKxXq1ZLu9x3tM2Pd2oZzKd0uyiMz11mrIq2feWYbAkOqU+tCipGQbDeDKhaw4kuqRrB1wfUCi63YAKkuVmycnihEynpCFFZwlVVcbBtgtH9yAcDjVXb9+wvLlmPh5h+pZJWoBMOfR7yCVqlFH3G+7ubwg4lFAkRcJgjsVyQtMcWh4+vKCqJiQ65X6zZbPd0ltDVmSURcl6s+XqesV23TKfT9nvV9RtS5LHQrvddk9Zjmmanm4YyI1BZ5q+b3DeUqmSyXTC2dk5xajESY/xlqDABcfgDfvDjtV2R1XlKKHJx2OCyjnNctrdhlDlrG1H2zfcXt3gPazqPX4wNP2aTXfP2cMLhJLcr1f4FJwQyPkMoRK6PKXzLZPxiMQmXIWGrzdXuGrEsj7wxeQx7rqhaQz3bUuPRJUFuqrofTRb+T4m30Qa3aOjqiTPc2oTk1zvBKM0TTHOsTiZoo4GAiFjGaBQklS/E6kkwkvs4N8PzEs9wnQObwVlWpEETZmVPLicslqtCSoiIMqqBCcIQVM3LfvtnkKn5LOUtjcI1bK73zAux+yWa7p6YAie00dz3thrvHNcXd/yl//5v+T71285WZzwZvmG0/MzJtMp0/mc65sbTs/OqJuG/W4POnKpTW84uENcZ3c91kQ8UDUeIYLgYHYoLdGpjsXhRU6SLfBhQlFkR4ToGg04G8sp/5BHnpeE4I4O39h3917AOwrrWiu8lxgj37uU86yIDHUvjvgWF/uC/G9xNErJI6qE95i+iCzhdwTC4+u+F+85JureFdILhFQRIePf4V5SkiTFWXd0b2tEcETnsz6KqhopE5QMxJJUeTRqisgq52i2UAofNCCQIor1gzVHLSrqUXFRfsSIcMR6EI/D6Hz3R7RJdIQrzdGB7LHOxmJ7L5FC0w8We8SpxPf4ruDSH3Ew7zjrcT0ZPJhjCWWSJMznM+Tx841/r0kSjolUj5TxvYTwW7d7TC4eeQ+/I54L5HtBPU0zkiRDqwytcrQujgx0cC5ej/t+iN1RSUQC9n0HR5393THzWx0vQm3eMcLj493vFAcERVXx4MEDzk7PY+qy7cEHtNLHXp/ffb54nL9LjIYQMDam+WPaQR6HML/lvw+DQ7aer3/zgiz9GyQFF5enjKsx+/pAohReKeI6PaCSBI/F44/vB4J1KBkHSnh/TE1LkBzNORaJeu9Y77oGISDPM6QSx2tbTHH0fUffC7Isj8kI926wEM8xrRTmKKwDSP4Ti+i1uQe1J5E5mT7n0fhTOv+S3l/RiQxrNGk2pjZrzNAwStb0TYvMesqRYugX9M5QyQV5dkfj3+LJuTm8oUgV7SBxfs1sNiZRoNWIZfMtWr0GL0jTihACqcuRoieVilw4HpxMsOYULUoCtwRtWRtBCJZC5jitaRlIOs9IT9GyxIUtgUcEc48Ugc4vSeVTnr+84na94Rdf35EmY66eOyhrdGq4+i6lSCtGU81m22JCAAuz6Yz7NxsoPNVEE6ykWIAyI7zeUMgcYVr63tM1A0niGboY0x9NckJf8fjR53z84EeY7YH76zuc6Nlud/xX/5v/Ez/++GdY1yGPkyHnIoezmMzYLQ+0xuMErK8HvA/oCowRjJKcchIvxFiwzjI/qRiNI2tdac2Ts88pi3E8YP0xziIkLoS4iFw8JMsKzk9nhLdjnp3n9IPl058+YXEiyNQemRR88ceXbA8HRrNrfvAnjzBmQ6Y2VDpl143RpYFQkMsW7wbmsgI7wkqLc1uk2FMojfE1xmRkSmHdgE4kPmiarSSoIpaYKUkIt1wfvkQTMHqg7Sfstx1NvycJFUOdsNym7NYD3nTsVwWtqbldGwbrEc7RtgKpNKaDyWWCDJLr33iyseL+rWDfDIgg0JklU5Lbtysun5UYp2kai2MAJGmZE0TAmJ6+Czx8VLG8q+lFS2ijkJKqAjMYzs9HOG8ZjUYsFiNu7vaEPJCZDKmhKlNkrml7h840ZVqRph1dV2MNpELiB8uoGLFsNtjGI3tPWmY4/jCuKvWezx4/5NHDx4wmY26v3mKHIV4gfcpZ9QAxaG5Xd5yenPDhhx9z2K7pOkfdHLi/v+Nscs7p2Rmr3T3b73eQBxyBXBfk48j4LbMpjy8f4oYe6SS7uma33TJdTFFacb9asjksGc9VLCMk59OPLumbeJN+8eIVfet49Pgjni0+I6jYVp16R5kmvLm/xrme8/NLmvbAyek5m2bF3/ziP7Dcr5gEy6PzZzybPmOxmLPd7Ui05urtGxbzBUqCTuHi8pTFeMH3X79Ai4Lrt2+Yjse8ePk1e7NmVI7QKidNK/Ks4cGDE0xvqJsel8CTDx5xOT/lsFwxmU352Z/9hG9e/5qAoe87bm6uUMry6Scfk6dj3nx/y0ePPmQ8mfHtq1cYPNXjnPv7G2Tocb3n4dkp85MF+/2W3WHLarfiq++/Jilzzs/OaLuW3W5LcmwoP5vMuF+tOOz3zE7n5OGEZ+cPef3iJXXbMJ9POVksmE4nzKoRBxtYXd/xxQ+/4KMnT3l9fUUrAufnZ/zqy19wfnrCyemcXBdU+YTX61tuuiWjakTXdvRNxygvyGTK67ffQZ6wrpecPjhBJII3379BSI3zgtvra5I0Yzo9pcgyNusaedysCRlo6pqrN2/IdQIisFzeYa1hPp2wfPsGNZtS5hW7fY2UgrOzBVJYtqs7hLfMJmOaQ43tDY0xWNdhbBsXRYPn8mJBemEZbjcUeowbeZ790We4r75mLDX19ZrlmxvKuEIhlZrQ9QQs1hakZXbcTCVMxhXr+zVtYihyjdaSq7evmZdTvq97Pnz0hKrMCMBqvSTNM2ZnJ5hMY5VClwVm2CGVYlJM+J/+/b8n0SkvX75CZRnjfIprf//YqHM2uoiHDp0oyiqnKGLBZNvWMZKZFoxGI5JU4pa30WXqLd73uNBQVSN0kh5dzZKTk0sePXpMYODN2xVt15JmiqrKqfcerRVFlbBvA840cWFlJc5qtDLc3S2jS1gFrGtI01ical04Fn+tSVNQOpZmxo2AjMWSzoFwdH1NN9TR7SQtbddg7JHbncijyzwmt4bBkOUVWaaOYnRDCBadSEKQlDJn6Dqcc1hn0FKiE0maRmSHkPIYqfWRYxk0KinQqUKqHEQS2Y8iCmjOxrIn6R1JEmPmaariXkDE+7L3lvV2Q2401vfYo+t3MD3GOnoTRXRvPU6KqO6HKC4Ng6ERDUpF/ISS4hgj9SRJQpYWTCYT0rQgIBmGnsPhgE40STqOzpdEk6ZxFa518j7OK4QgS1MC7zYmkOcp/RA3W+4YH9UibobA4+yAQKOO701JhZcKgcfYDustnoAQKXlaUlUjJpMpQiga10PzzpWjcS7G9ZVMKcuKohiwpsFZjzGOJD1yEgUoVBSotYjFqzJG/oUQx2Igi3cDzrV4F0UzrY/cdZ0BmnofNykhHBEsQkRWeTeQpjoed3ag61qsG6iqKbPp9Ih36WiaPWao6fuOpt1H5E3w790tmSnIsowsK9E6R5Aw9A3eS7JsRJb+/iIbwP3Nmr41JCpD5TmDs1xdLVGq4PyBxoaOoY+x4/VqYL3akRcwHRV8/dUNH3/6GeOq4+rtPY+fjfHeMAySalwgmHJ/t6XIA1lScji0vHq55ulHc7aHW4pCsakbhsaDiImeJMnpOoPpG1bdjskow5geMwRu3rZ88tGMly9esttumc0z2rbm0aNLvvvmO66vlsymp+y2DWcPMj74aExwBatlB15j+obRLOPb7+5oNnOuXrXcvnnLf/4vP+J+fcW4WPDJB4qbzYuYVHBxKKJkyv3NllGVksqc2WTO3XJD27fMFlP2hz1CZEfEkaSuYzLCOYtOQEnFy9ffcTF/yNWbJf/iX/yMX/6mZrlZY4NAaIV1nrwojptdRz/EDoNpnvHJyQXb62vu9nf0TrLd7jnNCuwwIJGUaYHGEwYQQTIYg0gVeTlid9hzdXvL6XmFkBn7YY9oHavtLQ9OLujaHS/evsEMmkzPsKZlubxm/qAiSxKyUcbtzS2pKjg0W6TIcF2FlBnT8gGldvghkCWSl9+9AA8PHy7YbgbydM5206Azz+lpRdsOnJxM0drx4x9/zHpbc7tc0doOmUmElhRJRjWS9KqmHKd8/e3fMh6N6fqe84sTjD2wOCsxrmEwUI0jbm0yzhFuQlUuWO1fMnQDF7NnZOmEk9Epu92Oalrx4vvXTBZjut7R1J7dusMNktOLy1iYXNdkecF6s0fojosHKcu7e5xRoGx0UYsoCJjB0DQD9/f3qAtJcBLrDd543ry+oSgylDQkumLft9ze7zg5T0EOONnx+Z/M2G3u2O0V08mEptmTpB6t4YOnT6nUOVr+/imyJJGxMA1HmqbcL5cc9ktm4xjt994i8IzKnE8++ogHDx8wnkx4++YtBE+apLijCKR1yma9R2WCxgwsdxsmWaDuW+bllEk54bDd0YfY6aB1znq1xztNWczBZ3TNntBrxADCe1ASszcolVCqFLc39N6ifE/moAsd49GEs9Nz2qbl7777mmy35tWL78E50uARg0EOBpmAFtD5jiyRvLn6jr4dGJcTpqMTGtux28aS7frQs1hcMJ/P8F7QDz31fs9kMmFwA8E7siyjaQfWuxolFdPZGJ1YhA4IHfnBm82Bk7MLVusdTdvT1bfM5mOMM/R2AONRpcaYDUmWsDidY7zFBsfp+Smf/+Dz2BtSH2jqA6vVmru7JSdnl0xmc/SDC0bTMc45Vs2Bu+2GUpS0Tc+wOzAa59i0xBx65qM5lc4YpZJBKpZdT29rXC841Guens0pnMKmkms1YO7vOC2mGBu43e252x9Yti0+L6DpkWmG6xqUtzgzAIGhP97TkgTjLGlWsNvXVOUIKR1Se5Aa6QWJUNjjeisg0GkCwdB1PV03YI1DKokbDFIKlBfcL7dIAhdnJyQiYzqdM13MWW42OAeJTgmDxw0WZyxaCqwz9G1ACkHTdVycnfHymxfMyilCJKw3NXqVsXg4Z3xaUZuavvOstlt0qkh0wsnFGfXQs377hvF0gi4yLp8+4pd//4vozlYaOxiCBO8UiVJoqcjTnL7rY/9YqrDGkKYpth9YrbcU1ZTZbEI1GlGVFdfXd5zNz3AusF2v6br2D7p3axULMH14h82w7529EN6Xf8Z1WuRoaxXLF5VMjmjf+JBSIX1c84ijlhOO5YhKZWgl8CLwvoReivcubP+Obf07DGjxDtHionjpvQOhjuJ4Cir+nbMBULG7Jz1yrInIGfw77vQ7gR6CjyaN46sgjyibJElxzmNdFFXdkcktRCDw7r3EZ8+SHHnc54bgcN7GNITWSCUiKxuHc5b9viMEeRyO2yOe0iOVOJZVxud8986VjqiYdz/z7r+EEHuD1psN++0GaxwQy2DtkQ/+jnX+7j2/c6fz/jt9ZxIRx4FIejwOUqRICF4hRAJB0fcOayKy5F2KIIr1Pfv9HqXisf/O8e7Du+/oyEF//0VG80fgnclEQYi4k7zIGY/HjMdjDk3LYDzGBuq6+R2nvnj/53cHMHGtPOCcI88jhs95TzhWVgcfByjr9YH75ZbV/Z7JZE6ZlRz2Hm99dMaHyC4XWhJkACUJIqImg3NH/EocFMXPOeDwIKKL/d2eCiEQgYi89gprB+o6EjtCsDg/YF10yQs0BHVEFR37BuLBjrWRn/9uYPhPnsP/5E8cH41qcIMgtS0785zBpqR6T5UUSGWpkpKBHcI3MeKR9QQSfBijVEVVemQwtG1DkpVIE2N4OnEYn9EaT1H1bIc1uS7BQ5lWrIctE/EAa3OqvMS7HmcdMm9wEoZ+S9ul6CR+4bn0eH9AiQvqYYfQEinfkCcLQgi0/hbbfUQmPgJn2G0t391/w3Lb8It/NFy/7UimCcNhQIgC14IMCuUMoQ8MjWNxkqOTwK4ZELolz8CTYOuAtZ6t7KEDOwROzjVqOubm7Z77m4bFWYrtJOOppG8Ci/E5P/nwr/jTZz/mNM+OzbDJe9FABIsMsWHZuR6HJyhFNZ0TNLy8f4MJBqRgcZLFi+jIMVYFo4Vkf+joqCmqBJ0ZrNdcXKTItuSTy0/jRGkw7yc5MdEiyNOKPI1R8P/t//4hD//yCVkqMHZgNJIIWkbVjK4fM8oc2eiOXEm64Q7rJUk64/ZwAKkphxpr9ySjgcSf0QcP+g1af4oOGZkqMINlE9Zk+hGD3bBq7jktFgwmkKYly+2ScakINuDlQGsLBq/YtQq8RQRLtzzh6lXDbteSlVPq/YG77weaXSDIFJICz0CSxUW08ClJYsEl7DeO2ekoukOlYjTPmUwybq73+KBIi4BEI1wgEYH7W09eCJRuEEGRjBxFmnJ3tY/iEJq+M9g2YF3P2ZOSw9rTN4HRJOWwt5jekRYepwVFVmJaS/COchz5nSiPxOGlpz/Ei0mqEiwDfpBMnkiUCvTblDT/w5joP/jgGU0TCw+TJC5sjDWkacanTz5D25TBOkZFhdaaalyx2ywRWrC8v2M0qXj27CnnJxdYYXm1fgUmUOYjTian7DZR0JxPFszGM6qsgBC4vV3y6MFjgvR0dcN0OmHfrgjAYV8zH+WkOuH80SVvrq744Mkn/OqrA+cnD3n06Bmvr95QViUno5K+3uPtgBscN6tbdJrQGMO+H2iGnuJkSlaOWB/WTM8m1HXNbDrl5z//OVLGKN8Xn3/O281zDocNyscbkU40Il52caKnadcMXYt3kgeXDwkhsNku0VKy2e74wQ9/RL3b8/bta3a3S/5o8ccc2obl7R2LxQlN25BlKU3dsJjNef71K7w54n3qjtl4hEgV13dvkAykOmc6mnExPeH0/Izr5R37pmY8nXJyeoY42mfLsuTB5SWbzYq7u1uGuuXR+SW9McxmM5RxyEQwmY1xwrM/7PjpT/+E+/slfdfxoy++wBtL1/f8xT/7Z2S/+gdevHpJWWZ8+PEHeGOYzaas77YEK8mShI8vP2R7c8fqdsmPf/hjZqMpd29vyHVCEwb6wTKeVrjgOLu8ZLPZ4qzFH9EYeVaQJCXn5w/44NkT0lTy9W++IjhHPkvp2hYbHONJxeX5ObkXnJ+dM5stOOxrzs8u8C5G2ev9lrevvidNFNPxGPlIsby9Q2mJ8z3jScnp6RlBK+q7LWf5lP5+RaIEr9685vLTj6hOJmR5yTSNnM6HT5/war8kGM/lxQWqTPnly+fcNTtG4wm7+kA5nrM97KhVw2w2p93v0EFxdnLCvJwgE0k39HR9S1EtuF3f021vMEKQVceYpRDUXc+LFy8JQmB9oGk7qiSBduBievp7n9td36GOIvBoNGI6HR8Xl3sQkXmZZSlSRTE1TXPq5kDXt/gwkGQC5yLqoiwr2sbgnaBrO4ytj8716O6dTCYoYUkSgZIBgkEnATcYvNMIoRECiiLHOUfdbNkf1qRpdHCMKkF65Bt655Eiro+yND3yUvdorckLyb5t6YcOVcZFohlagovCcJGn5FkKwZNoTZHnJGks54wMQUdZpSgVIqYrmqAJxL4RnWiU9iAjK975d8VOAmtBJRlpkpPlmjyPRaLG2Mj18x4b4qJeSIHS0eEeOMYRE4kMgjRN2OwGUAadQMDRDS2ijsLF0AvatgYLzoHSBufj5mIYDCF0cUOVytgN4GKZappl5NmILMuOC3yHsfY95zKWnxJd58cBAUSn+ruoLyKy9IXI8MEymBhxdUe3VGyxV0dOZlxUxxLYlEynCI7uleNaXoiAVoosKymKMePJjNF4irXRaSuP6x6tVRQ+jcOr6CxKswydGKzzdF2HSiSCd4VG/shQn5DlGmtb6sYCCnwcOAiRMJ1OCF6xXq8BH2OzRxE9yyy56dE+urUi0i7iitquo+s6jOmom5q8yJhOpmgtsbZnf9hwOKxxNiYAjInilpDhOIQp8cLEjVWQ9L1FpFGgmk31cRP4T5cX/f97rLdbkiQhSTQff/IJb99ccb9cU+82tJOK2VnJ1h4YTxOshc1qR5pl3K8HxtWU169WzBdTOmsxgyBLK07OMubjER999IibmztWq7vIlTeB9apHiA2XD0t631KVoIXisHfsNhtm8xOs8gQh6U3N5uCYTAQqyTg7zVitNizmOTINrFcDWg588EcjtE64vrrhwaNTdqsdT58+QAnY11u2G48gjSkRDaOJ5KNnj/nn//yMF9+/4e3VlsVpxd99+Q8Y50A5qrFGqQSXCD748DHr+zUqyTk/u+T+rkaKjEAdk1rF6IiuSRhXBYRYZCuEIk0su92WcZmzXN0wKib8v//dvyOvNEJq2jqiFvdrWEzHDH4fXefGMSnH7B2oVFPbgcYG2sagdjXpIiAThzEBQoKxHVpqqnHG5CSNgkcvmI1TlvevWa4DJ/MzPvnoC1brf8vV27dsbgbOzif4QeGtpRgb/uVf/Ql//Tf3GGC/7dhYg/cpu6bm7GyMEDmfffoxf/3v/po3r9/w53/+Z6w2r8EFTB9LiR88Tvj2mw3jYsRts+Lhs4QkzhfdAAEAAElEQVSgLC5ImqbnsF8j1Y7Hjz+I5pf7mu36QDUqGI1ykAZLze3tG2ajOQHBbDRlvTY8erZAkJNmJWWZsdneYjpPnib80Q9/QN0uGadj9l1LkRSkaYtQCc6nTKfnbJobhtYwHBKWB83L72vywqLyDdPJGbd3N0ymOcYE2tpSFxnb3R6tMqpxytAOoKKwVNeGNMnouz3rTeTIRrECqklGe+iZL6Zstw3lKOfQ7hgGR1pKhBZsDyvSLDCZjKn3jq6LCbJDveV2/ZqTSc7e/v5CW5FlCOEIwTB0PV3T0u4FttlQlGN0KqibPa/evCErUx5ePiAbZ1SzCuUciDiQSYqM2XRO2zUsJnNE0rFu9mxXB9Y3K+rDjnykGM0zRKFwJpClJZvVHq0qThdn1Hs47KAIKdIqvDNRtE4bSq14dH7O7e2aw7JmvdwxPZmwOJlzf/0KHmm01ATTsV1t0dLRdx19LRmrHNd7zDBAZgjBYGxDP9SkacG4GiF8LAk2A4yqGYm2TKdTDodDFGCQFFUZ+6W6hq7ruV/eU41mNN13zEpN2+2pxhnl5IS7+zXLZY1WlmriSNKK1bZhlGXIoBFmoEpzEo54s75mt99xfnHG0Hd89/xbtssVKtfsDnsSIfnwg8ecf/QpTW9oesfz51/xl3/x5+RFRtfU6CxDJSnCCi5Oz/G6omnWFKQ0m5a21+TFCdQ9tm9ZLtdUp4sopnct8+QJvm/47rDmzaSmOBjCveXt/i2bxmLSFJcoVkOLMIoaiw2WJMkI3pPleUTPCGibAxiYLTIW8znb1ZYsFTjj8cGikDS9xfQmlnMrjUoU1ThBCEnX9lgbRcuyyhiNCoZmoF42VHlKo1rG8zF5VhGEohiNUS6BzNOs46BmWpV4H9judhA8mdboyYgseZf6Dug0x/Udm11DvsiZnE3Z3BwY+o5fffstn/zoMfevtkwnJ6w3a7a7HSLTPPzgCSLVBBUxJ2mS0NqBQMAaG9GFg+N0fsbebqOw3/ZxgKYkVZGRVBqZZth+QFYlbd2wuV+jdU6Zj5DjQCMPf9C92wfzXkAXIhzXXhFt4Y+O8BA8aZowHk3iGhPx3uyA9zhrSVKN0gnSBIbBAtEZ/G5toaQgKHlM6Q1Y8w4NIyLLWoqjKC3es8GB9+5m5z3OhSPa7Xdd1oJh8MfnUijl37uXnfX4o6CrZIjFogiUVlHUdr8V7YOQSBUNU+q47vPeR/924Ch0BxBHUkKSwpF1LaQglfq3YrL3+GNJqhaBwyEiLK09rseRGOuiQB+OTmiio12EgBQJnmOhq4IkVZRlxmhSkZcph8MBqSXlKKdvPNZEBKHSGimODnck4ripCB6C5L0bn2MJaUAcDRegVBTtQ3AoaVDSkqYpQgpMZzkcDqRpynhcsd/vuL29Ic8zyqLCGnMUgY8C9zE5GwcVcRjzW6zLu/RBLO5cnJ7zf/iv/iuCEPzr//7/xdfffBf3i0rijUNyPDaOoAOtVXR4DxHp+A5X8w59KII9vrdoyDHGEDwslyu++uobnjx9TKrTI54lIFTcb3kXEwIICfJdP2MU2RHg8VjvcHgEgcH0SC3el+DG53AgRByOHAdF7zj9QopjciEmLN7x/aVMjgOCiKoSMvLxrbPxXfyndKLv+hUJkCaCWRqwYUs7lNgg6DvPLF2TyIBBUyYSQgIqTh6F3aLlQOccMklgWPAoL7nXvyBXAi0qluoNUqSkMqH3LZoCmd5QyYRZqrE2x4dYcmSNZtUapkmOHx6TqI4uvCT4htSfHj/Ia5zo2PeSeZlxc7hnYOD5awnNlyzyhm3T8/XN13z51YpUFwQxpnUJzToQ+gQnGlwfyAtLOvK4TmIaBaLHKBCDYvokwbQDq1sLInD6sMBZS9dZBhvYbjtGRc4ozzh9kNC2Fp3D0HeU2YI//vgv+fjsQ3IMXXsgTwqQGqU1wVicFwidEPwAvgdixMpbz7bb89/+3X/HbX3L6UnO+ROFHzyCnCRIpqeKSXrCsrHkOTx9cMp27/nkww/5wYNnPDh5hOljIRkyFpFZEaIrMR9RpjlSCHTm+fDpDhs8214hMCi3w1AzHQ/0dsMoGyiTijwF788JoSYfCSwVXdcjRMGmVwhuSPyIce64P/xbZvkZu1oxUjN6u8PZl9hhQZ58wbpdRR6Y8WxXA3UrYtmhFrHITlhul5bb+5qRrJB47m49d0tHWShuXnlUoUnkFO8tZmgwrUCdwOn5mHojSEVECownI2wrWJwW+IxYkoJnNs4YPFRVwXbdMD/XzGYTunYLIYotwUqcsaQZKJmgSwUiYdceqIqSfrB0jaHdBCyGw8HhvEeLnKGucU5QlpEFLBGEQaMyRTMckFLS7OKNIl5mBeUILp6UeOlpNh48R57u7/9oB3j0+Ck60dxs7rjZL0mKgkVacn4ypa73vH3xEp2l3NzeMrh7jOu43614sXzJD579gCRTaAVPHz6gNhtuDnfMTh/T7A/MJiXeWnb1Hc9f1Ty8fMpieoIZAjJIyjJDpnB1d4PwYLqEUXHB0AdGoxlN25OIlE8/+RH1XcvVd7fcXv0/+OJHX/D8+Tfc7zJ00JxcXHJ2es4vr35F7xtWqy2JKDjL50jlWSxm3Fxdc79bMR1PoW1RKkErSaoTVrdLmrpGpym7uyuqcsajp8/45uvnnJ6d0SuBWVt813A6WSBDjWdL09ZMihNOZxcUomS5u2MymlNkFZ0F33lOxid46ajGFV3d8clHP6JMTjgpO0gFXe9I04HRIqX3B5rhDpkY+q7m4snH9Puevm0ZVRXTdsLJ5JyPP/qMH//kJ/w3//r/yX6/ZxAemaZ8/PEnkaOYpHx3+wa3mLDc3qNLGFTPwccS0sXJjLv1LU45dBkRCm+ff4smxyaBT754GgfbXpOM5vQG1oc9ZWGoJgXKdfzxj75g+NAydAO7txvO56e8ffMCpwyr5ZK8qCiSknJRRZ78YcvJbI7rYkmhcQYXDJOyZKQ127RkfVhxt7oim4xo7UCeVGxuNlxUJ/Sd58WbN/zw8895++oNtIbVfo8TAZ1qTs7P2B8OPH38Aa7zPHv2jBdvX9D1DUhFhuBnn/0x3//dl/iQsFzvaDSsdve0+Y67u1cok6CfjLkd9ngr+Df/5m/4RVbwk5/9hJfffoccKSa5RmcJG7MkjDx5kqCEIDGSBycPSEXG3/3qH2OXRGcJQnI4bPBFyup+i/PwaCIQXrK3gZe394xHc7IgkCZiK/COYiSZzf+QSLgEfUTo5CPyoqI3LapPUM4e0ROK3hzwIUGqQJ5ndJ1HyRQRSvpOxiLkJBbiLO9vGcw+YqeQVMWMUTEmTyoGdaBttgSvUEQXvwGETNCqQKsYnQs+MPQWbwVoTaIypBRY6xCkaB2jx0LEaN5ge/CR4amzDKEk1nmapo2LpRALbjCxh8QFD0Khk5ykSCiLAu8N+909NnTkuSBgkdphO4Pz8frt0aBit4cXAWMCw+DxIcYGtVIICXmuKMoEnUShOgiPxxKkI0lTdAY6Fe8/M2dBolEiRaWCrBiRZhVCWLTSSCzGWvq2JXhB11qMaaKT1gmSIEFIhEyxPkYdPZConCQvUC6mPJTSSK0iZ5QAaIROKEoFISDDsfzXe6SWBBERb83Q4HeeoszJ0ozeWcqiQIsE45qIzLEdPni0UAgB3kWcg0xjzFhpFVExHoST6KFF6h4pHEolVOWcyWhBkZVIFFJE91uSabTOydLifXG68y6WpApHkODwNH0HWrxnfyqp0ConzwrSVKLwGJVggsN4iZQZVTnh9OQMgiJ4aNuWKh+T6pzgAuOqQCWBzvTHDY0iDktsdKq2ccNQFBMWiwVFWdEPA/tmz2a7ousPSCyEOJwnHBmxx5IuNSiGoaPvDowqTaLt+00xQtC2zR90764HwzhVOGm4W76hKCRpAtk8JTiHdw5jdiRJymSa4FxG2xr63nFyMuf6ekVRTXEBXr3c8fjZgtV6jRn2XF8PVOWcB48m7PdrsnzEg8sP2K4P3Lx4w/mDEj0KKGHZbx2uH1heXbGYXdAcWtJ0zOawx2LIM4kUPZPFmIM9kBSK9Y3g8nTEfn/HT38256//xw1Pnna4uqY/tJzMZixvrvjJTz5kvWq5vr0mHyVkI8+b+//ApXyKyAfs0NNaCSNPtwtU+ZTNZmBUTWjqlvksR2U9vRd0B40bYjHjzd0amXo2uwMns4rWWqbjh7TtOg4Tj4mIvtFkOooDk0IzHCSb+wMIR1XNuL8eMLWmSqYYYdBBQh/QueLB2YKXr95SnZbc1TuykOC2ljAWOAI6kzih8EIzOIO3PaFvSJOUMpOYoSHJJJfnz6gPnq9/9Zaz2YIVNwjZkldzLvIztps943HCl1/+kv3hgAkercd89at7fvonn9MNGw71Gu/33K9LLh7M2G3vePnmH5hNZ4ySCbP5lLrpGOyK8weetllSZSmHRjAaVxRlh84lQkzpmo63b74BMXBxlnG39CifY62nrEqW6yVOOHZNSyFLMn2GrqA3PaNRoOtXWKPxJjApZxy2Nc+//lsSLVmtauaLU3abho8/e8Y33z6nmk756je/Ian2pKLibP4R/f4cxQ3TmUErRxg08/EFu90b5vMJqxvJ0jmssKhxxHApFQsQzdBHd/5mYDQWdJ3DiZ40S1B5Rr0eKMoRrbWcPTxht9pyeTmPJWZSs7xvePrshGB31N2e1aaj73Pmk8+4XX3FZrfjcHjJoXP/9En8/+ve7d17rEHXxYSDlHnEEViLkIrZbIrSju1hzf2Xd0iRkIgEdSxcRyXgIdUFIMh0yWQ8Z7PdRU1FSra7HV5knD1ccPbwjJDmmAQO+yXeSkbVgu++fY4WgkmRMwwt48mUshwxny/IfOBsesL1izu2q4a73T2T+pRHHz6jd44vf/0rUpUwflRA36OrnG0/UKQpidTUQw+Jp2sakLDd7ciyjFzleBe4ub9l3e6Zn5wwqiaEcMC72I0RjWA9qdKkecZyc8dhX2MsPHj0AdPFlNRb2ubA4uSctm/Ji5yi9Oxrw/XNFbPpKQTJqCopswzTHNBe09c9W7elHJXc3S9Zr9bkWc5ivEDaQL+vOWx2BGuYlgUbsWa7b/AqYVqNSbWi7zqWt7fUuwOzyZTCZ7RtyyTLcSaj7xqaXc0oqWgPB7QI1Nua0AzcXt9gFmVExXvFhBxxcGzLjt1yjf2+Qc0vYDRid3VPPo7DpwSJtbGofN8eKKQ+djcpejMw2CgSXV/d8sGTj2h2OzrhUalEqIRJNcYNhmZ3ACRSJ7jW45GcnZ2yWntUItGZZrqYkucJ27stlydnmG4gkRlaJO+H/sEL3OAxdYuWivQoSo9HI6oy465exf200iBAp5rBWaqsZJGdcXAH9vuGi4cnVIsRh7bldnPNP7/8KeXqFY0baPqWfFxQTCqqyYT1dof3gu32wCKZkiUJSEnfD2iiUJ0qxbiokCri53KRICzooEjSDOvhfn3HuCrou4FMpXTtgJWGk/mc2aj6g+7dSg/H9Z6PZgHhUQG0TqMAGX5rfvAelMpQSh7FYoOxHc4OqCQHHNa1+ODI0opE5+8RvUJ6VBII1iCUw5oehCRYcUSGRCNEUQjSJEWG6KoehlieKqUkSUuUSkiT9L2rXAodjw+hSRKJ85a+bxGCmG50nuA93sYen+ADOB/X8DrB2ndGtRSlMqwZEKQkWvGuH0nIcGR/O3QiUUKiRMA5j5LhiBKRoKJT3TiPtRatYnL2UPf01pMWKUF42jYm7J1zBBd52vJYximFQB9NTFmaIbTh0ZNTihEsznPykSSr5hz2OaaFTIIM8doMYHqLFBIICKmjyUPA0B+d1t6TpQnWxWRR1w8QJMZ1aGUpS4kPDb0ZmEwmMRE59Ay2pzcdaaZougN1XTOeVkyns/e9Pe8GD9aYuHcEfiuge0RQR2RN7CF454bPyxFffvUVf/vzn6OUhiDiMOEdbkaG40DAHf89UNd7tFYMvYNw3LMJEMLjXOxckjIFIRgGy8OHjxiGhm+//YoPPh2Tj6b40GFcHYtgdTQCK6mPyJ8Q3ehSUvvYT6WzLGYIrEGLgPfD0agEZujjEEUloBKEPvYOqgSPRCpFVqRH8dzjfCyfRXi8j91IWV4QwkBvDc4ODMT0wj/1+I8W0UdaMyp6NJZd50hVyi50lKrAaUnIZRTEVEnj7ylywUhNkFLhhaTrLIU6QecNtpO04Vvs4LF54P5wz6yS7JoNPk1QYkYpf0xt/pGJXpCEE3b+BanoSHWLVFM+CB9xCBsKBlS+oG0XZLLH6TWNSWnbA1pMSR3c3q+4vi/5918fePl1Q1cHJK8IXoFL2B0SZmeS3u0pxhn1AbIkxQXBoDoQgSxPEAVgoG0Dvk1IMtgud+y2cRJkamjampySUaVYrmrM4AmF5vSJ5+a1xRnJ/CLh/q5nNkkJzvLq9d+RnDziwcUzlMowpmMwe7bLt8xPP4ixFHvAmw6Oce/ed3x7/7fs7GuKsePyQvPgYsbqemA07XnwaMymaXnwRznl+DNE1/LxxwuUzMiE4oK/QBwyQognMV4gdcJ0espkNEN4T6IzhBDMqgV9OLA1KU23IyNFpQmDNdjkLUpoZJJjQofwcfNaVoHtwVLqwNk0oXdjbg7XlNkILcakwvCgFBzMlrZ3pMWUzdowyufYIWEkFhy8QQQDXUUIl7y83iASx/5OocnZNBvWtwlDL7ijxrQ1Uo04HBqcgcmiZLPrGM8Am5GXCfe3NfSS9hCYniZslj15WlBveoqJQGBJVML6qiVPC8YXGbdXhs2qYz4r2FwFhOiRQVFNYhRLZhBkxXbZMzvPGPB0O8uAYDIPKJWzu62ZnaU0vcANglE6pTE97dahx0Afbyw2GPqtRRuJKhQ0Cq08KgehwBqDa1KG3pOVmulEcehakqz8jz2V/xcfTz76mHa/RVrH/WrD/PQUISS5U6SJwpYaY1sMBusNMp0SBs++2XF6es54PKUfDGYw3K+Wsdm8jSkKbzx92/Pk0WO+/eY5to/sMecDn//wczbrJSIYVAIPLi7YtS2tMczHC96+foP3jn5omc3HtE3NYj7nzfXrWBbYHXDBsm8GPv3gM8IAz795zn2zRKYJ0kqyJONidsHrty84yD3BBdqhp8JTJIInHz5m6DpaU7NrYgGuVikGg3Oe29sbJtOKrt/z/fI72qHlvDxFiMD9/Q37fsfJ9CyWmljH/dU1lc758PEzXr15hZIqFuaRcHt3j/MrvA9UoxHWxZ4AbzzZqEJknl19z7a+x3iL0glJUlCVY+bVgm9efssQHBcXZ6yXW/bbNdevX1FVFc472kNDkWcoqVgsZnz5i19xcnHCOCvYCxGLgJ1jVJUUSYHpBz788CPevHzFz//D3zObz5jMZ4hER+aecKRJQpGOGRUznj1+xul8zs31FeNpyc3qBrsw5FlGKhMeXVziTM+L3jA/n7Bdb8l1xupuzdMPn9LbARsM1WhEoQr+87/8S15dvWXoOn7z1Zf8yaefI73g8dlD7usteZYTlCTXBbKFNEs5OTlh2++ZFGNug0SojKefP2UIjp///X9ACM/l+YPoYE4y1qsNn3/2Bb/59itGecWTy0tW92v2/cDbuzuK2ZTRbMHN7S1i3KJzQd3VqHJG3dbUm4a27li9uaO3hk//+BNmlzPWzT1921M7i1Awrqa8eP6aiZyQPsi5Xd4gE0Ez1DTbljwt8Ykiy8YURcXNzS1mGCIa6Dy69L1UnF48ZHl3RTf0FOWYpm/4/vW3v/e5HUgIQhNCwDpxZIPnlJU7OoUDzg94BFoJskzjfYFzkCUleTo9LrwkSaKPIqMn0OODItEp8+mIPC8RxGMMYZFFTp6m2FqCT8jSijTJ8V7SdQ0gSHSC1TlaJUih8M7Sti2J1CRpEvneMgHvMLYnBE8wA6GNi9YkyXFmiHHO+E6wLjpYlLYEqVCDQakMoYGhp2v3yGRASI0x3ZGzF92HxkaOotCeKolO9qbt6DpPosujuzsu6nnn/vAhYkOCwwsPKqAzgdQBpIt8dhndVSFEsTg23Efmt3M9WgryDJTs8cHTm56m2dN2DVJphEiRqkCpIl47j4genUqUTtBJiiWQJilCSlwg9rQIkIlCSUWaJ8d4ZED4OMhFHuO6xMXwvt4yuIFRNcIFf2SCZwQCgVjgGXw4ols0yARrItoshGNEVWcIIZFek5qBZIjlVUomaJUjRBLRLIMhYElSSZYnEBJCUEgZndz4EDdGwoMClWiEltGZIiJ3UR4Lj4IXCC9IVEIiE9p+j3OSLBkzHs0Yjyd4JxiPenCCNMnRQuGlI0vAkcBR9NZaY4yhrmvapqVtO7RWzGYnTCbxXOiGjt1+Q902DEOLCJZEx99DEs0jWZLhgqdtWvb7PUIkjKpZPNaThCRJjk6z3x/VBJBlGX03IGXKbr9jNp1QjVLulyt0SBjZlCIv4rFtA1mac3fTcno24n655dHDOVILUpnhGRg6ixSC3a6mSBNefH9NUaacnRdMpiPulm/ZbXq0TDnsA9N5hdE9RRGLxaoqxwfLaFyx3m0ZjyvW6y1inJBpEMKR6jGm91w+8oxGE16+eMlPf/YJ3/w69ryM54LBD5TViDQX/OrXX3N58ZRPPvmCQ7NESslme8Vh9zXj0ZgkdXTDwLOn5wRfYLrYHzOfj1guX1DXB7x3tG1Nmc5wTvL3f/ecyYmiKEo2hwNZWtDJAy4Y9oc9JycpWa756KMPuLneUBQj9rsWrTKc7ZhO5hjbgg/cLjc8e3qGEJF/nyQK5yTBa7w1JNoxnZSUuWS76Wn2gbPFgtG44L69x/USqWI5YS8Md+s1J4spd5sV3sCjyy84PX/G9d3/hA0NXSPRek5SSOq+YXW/YzYbI1PJi7evAIMNFiFzfvRHjzHesN1vAc/nP/gh++2Wi5NzRnnD21evSJViMh3x6PKU7X7NYTCs9hsO+4EsiwYLqWq0fjdkSjDBUiQ5g9ljB8ujyzNubwxOAlYQQyqKuu0ppp7JNGe5vmF3xM7NZxOUFPhgyTOFyRMOTRuTfbuWahr45MMHfP31b9i1G04WJ5wv5uzqewIe63pOL3KmJz/l19/8W+y+4dXyax4/PmO+yKgPA3/+sx9xv77i1fWeoYMBS1HlWLunKEpmkxO+/uoF5X3G6WWBLixlNma9PPDJB59ie8fAjrar0Spjva65ud6S5ILWWJIk5XQhOexbQHLYd/zyF98wP41FzW9fbfjww49/73O7pmeUleAkQ9/gLCAcQTUUpaY97BFJxaTQmGZDXdeIMGI6PmNf70l0QlklEDyb5Sq6ketAlZYY1RNUwBQFHQGZpBgLhJRSj5FVyny2483rN4Blt9swKcekpcQEw2hxhp6XaGkJ+56T+SPOxlveXn2LcYFOdqz8luxc4fqG+/WG0F1GbrGXpOjY4TBJ+L5bc5JXJD7h0Bt0NqJMC/p9w6G/Zd91yFQCjs1mS9tYtuuB8SgyeW3X4Lue6myGE4IgBImSvL16wyeff8LNb16wWW04OxPsdwYTBOePJ+T9PftVjTUl41KSJ4ZMW0ajgrZ39H3A24aLB3OefnjGq7dXpHJClaX0ZsXgBqQXjGZTgpQUoxHXN0sOm4bPvvicLFNsVxvu3rzBNoZJOWfb1dS9JcGTIukGT68EQ9lza+/oXY50Odvtim6SIg+GKYr1fs+y3ZF4ie0MdZawzjRnuuD7r9/gCRQqx5ieEAzSC6QVOAMNjlQPhGBjSkpEpIkTHtvvmU1LCNG04IIFP1BWirqxEcsRLGWRk8hY5vvowzm98yRZRJGudiuaviezGSEIbHDU/sBYjvC1QdaGxBva/oAzjvPTczbbNalJefjoITb1bLdbpJYUo5ygDHU7oDON8ALfGOxe0u07Ti7GtPUB0gHTOCbJmPXmJVWZkY4zRpOctq5ZvV2zX3YMFoQKZKkg2IyuDchMc7YYU5SCXBU4G6KBTQiSTHPY1+ADmU6ZVVUsorcebRTCWA7NDm9rEvWHdZEhPEqBI+CcIYRYnhsZ1IFIAhHHpN47JIk5okpikTl4jOkIWIyJeI00KaKoKSGKhjGBGR3FDohpvYjQtPS9QSlJnv9/2PuzncmyPLsT++3pTDZ/s88ec441ZFWSRUHNJhtoCJDQDQl9owtdCNBT6EoPoUfQpV5A0IXYkpoU2cViFSuniMyYfPxmm8+4J11sc8/qbkGVyKTIVndsIBDwcHf7LMzOMfvvtdf6rRxIInSe5wdmeCo+VUol/jqpmDOVXB646yI5h52zOO9QMv3wSEhi6IHxjgShBPGA34jvMCdCEPw7HnxC02idEYIixINLmcT+FjISCOhM40P6mQR3eM0ESIkyGq0N2hiUzrFDMtj4IMiMZhjCb/EqiWMMB5ezIJUgKyUYT8ecniyYL8YUlUEojwCKKsMICDrhXuxgaZseUHTW4YOjLAoGO6CNwnNAzEQIISX8t9uG1WrNyckp2+WKECLz+ZzFfIagSwazwwyfDk0c292a1foeKQW7/RrrBgZr0z6Jw3fzgWsvhPw7PPT/ZsFoYu5IEBFtkunu7OyM+/sVxmiGvoWY3tt0zpqc3FmWIYTA2v59h1EIqYg0BpAyHRiLw+uplaLrOtbrNX/yp/+YopTcLe+JIo331iUUlPUOcdiOvuPJv0NeumATKSBLaSZrA3iFH36bhJUCUCIlh326vpP73LxH6sQYiIL3127ixkuQBq2zhHzyFqklwotkwop/v4r+O4voIsJqGyizSO8rVh2M9JQgA5XS3O86Cq0ZZwFEkaLomUeGa5xN7uYh3HK533JiOnJVMS4Sv1TIGhE1QzQwWEqzI6pf4vsBGwRV8SlvXYtWbxH+GUpAzx0xTgnKIoeSqX/AbnjNW/uSt1cROUy5vLlhPzi2W8ebV5G967HLkuo0cPcqUFQR0VmUEmzvB44fzVjd7ZFS0LgN03xK3TqaxmPKlrzIUDq583Zbz8lpQfAdpSmpMsEwaVFe0vsB12uKaeT4ocQ3jqa1mFylDbFxTBYSrfb8y5//nygKy49O/ikfPfwhk7EmxBZtdwgsbtwg9pEyL7lbXrKYPOHl+td8e/+vadVX/NE/MKzXJ5w/Uzx5UhL2OTLbc3qi+c2VoAQWJzWLsmffvqUJjkX2p4j1gwNfK2M6P2c2PUEeSsW0VOmmE6m4S4rHLIcRKiiOTcUo32NFQT94nK3IxQM27oZS9vS+Ig5b1EihhEKwZN8KnILpyNE2PVo/YN9bdn0g6h2tq9nu/zXRzHEy4Pw99SBoakdZwFdv3nC7EsjMs7rzvHpVc/HA03caQaDIKu5va8rCoLMMGwJ+33F6XHJxnhFp2K8FdBk29EyrMcurgb7rsJ0lGIEyhvV9SyAyPs4oqoJ9t2V/o1DkzMqK+azk8naPCgo1krR7iRKa6C0+d6hcIqIAK/E9mEKwuuwxlWM2KfFtJIhIOc4ItWc2yZhOK/aHYc0Au61F9IryWNPWlvmxYLiXaKtx0eHayKbp6WtPPodqnIpmdTS/6638/3G9ub7hdD7j5etXBAF3t0tOF8cczRbc3dwxOxtjcs16u6NzA8fDMYOzSKl4dHZOJnJOjs5wnedoccredoS7G26v7vB95M/+yX+M7Qd+/IM/5eW3bw7XBmzbDUE5hq5lWo6odEU5mrJa78hKRfHhM7abFW/evOF4cczaLCnHBU+ePuJuc8PQDwgERZ4KB6/vbnj98g1D4Xj24RMutzd8+OFz+n5AeogHPupyvSFK+Orb31DmhjwzLBYzjh4sePHNjpHMePDoHEEaLNtuj/cWHweUUdyvl/RZyWp/AzqSi5z6ukZ3OefnD0HAL+o9WVFwdHTEyI4hV3Q+8RSVVry+fEuz2fLw7Cw5bW1yoe7aFabQHB+ds13XVKM5SM3N7R3aGPK84Ph4xhe/+gV2aDk6mvLk0WPevHmDDoL1/ZLR6RnzeeI3Pjy7oBtavvfBxwyu5/bmhmbboArB9GRKMZnQTGt+/cUXgOTpsw+5X6+YlBPW+xs6Bmp6drrmwekZp0dHNJsNwoMfHG9evuT89AGZytBG09Z72q7n/suv0VnObDxhCNA0Ldvdjn3TMpvPyaPh7evXNPUeJQKd7bHecXxyyriosB6224bRfETbtNSrHbOi4tOPP8aHyGI24615jSoU3//0B2ybmq+/+gatI9/79DM29zvyLGO1WvHBxx8kF6nO2d5t6PoBP66wk5IA/KOf/pS3N6/41Zt/zRADMZd4GRK2ZxgI3uNFZHF6zOn5GQMDo3JK0zXEIJgtFmz3Nbtdz+mDEV+9/IbROGe5vaecZDx4+IAYJJumRgnB6dExbdshlUqDltacnpxgsoIYoZrM2e5WFNWMZnXLzeWb3/veFhEypXF4/GDxNqFMyJNDpOs67HBgQfqEPzE6I8/i+0io1io59JQgklwnRZ7KzhL+Im2e2rbD+8BoXFFVFc53NO2eEHqkEQgJ0Vtc8BiTU1Y5iMS5G2yLdZF+6BBlhUcTZUCog5sipgEwCuj6PjEB85xRWZFKhFJ5VoqeekSItG1HCAOzqUlMczskB0OwpDhqcjuDJsTA0HsGHEInXJFS6sCBBJNJMiPoe4v3PV0XiZgkmEVPmvMVWhmUMofPDo/ta8ajDFkk972UGucGjMkYjSZYm4FIBata68PzjAjUe6ybMYYsL8izAu8Dbdu8526nIVEQYmI+IgQuCAaf+pi00JjcJJyNFQSXDgffbZ5CiBitEKQoalO/G6AFRTEgRCqwEkKTZwUQUeqdOyXhTwQaKRVKGoRIYnSWaUKc4oOn6+oDexC6rkPEBilSyWsqJNJ0rcXZIfF7M0kUjhh9QnS5gcxkZJnGZMlR7p1/P3B77wjeoHRCbAx9KnHSeYHRBWnYT9Fonen3bPfoLSFYiB6jJGWVvke2bsDbgabeE9xAUc0YVyVaSnb1nu16R9+m3hRnPURPcImvOSomVOUIrTOGrqXvLXXdUVWJYWqdoyzyw2YkMWb/kGV7j7Wetu148GDMbr9lOpmwbVSKp4bIyckZv/jFN/hBU+YzTk/nPHlyQaYDTbOlKPKEA5KSoXMEL1jMpmxWHeuVYLWyWG9ZLPYH7qhCUZFnGdt1Kh+dTCdsN3uEDAyhIwrF0dGMzXbH0TzFkEfHY/b1wHJd4/uM45nh5as35EWKjVcjcLQU44y71ZIHZc7seM623nB5c8dqs2MyNUwmJf0QyUeCzW6HUgkzKHcbphPDpt4Si8h8foq1Y5brK5puwFnB8WzM5etXvH2zIaopqogIHNfX9xS54Ztvf8N0kXN0XJFlC/b1HUJYlqua8WiC9YJRNeL09JjtdslqveZHP36OMZrZfExRKrabLXU9EGNB1y3xdgfR8B//R3/Bf/nPfsY3X9/y6ps1f/bnn3J+pOnahsVxzuB78tLw/KNH3C9XdM5SZCPqvmdTbwnKE4h0HTx+/JS6uUYpxeAGUAIXA8IonE9ChI+OEBq6bst2XzOfV7x6c8U4r1jeramqkqePHyEO6RQJNLuafQfEDK2A6NltV6zXGx49nuCtZ1SMqDKFinB2NKUdahA5n33ykL5ZEYbI0Hq0zFm1LQ8uNJiB0/Njrm+uaNuehw8qhr5NokgcGE9zquKYt5c3FONIMc742S//lsCAyDxvb74hLyyTuaAbelTYstt+Q9evePLsgqLsqE9LpIr0botzkd1+x9XVNYv5CUjDMPQEbxHC0HYdUvT8+MfP6fcjXr255vgMhlJwf71jkm25u90yPe2ZTkfs28Dx4inRrdn3G55+sGBwDVJEptMpewG73RoJ7LaevMx49LDivSnw91leMMon4CJ3+01KAGUapQVKRbLMJIFIWKzfo5MihyCSZcm1Ogw9eZax221ZrVb0XYcZZTRNQiz2XU+IjqZruF/d0YeB6eyEk5NTHj98xG61pe1a5vMZMkryaUUfO6ZHM3qfSiU3q3te3V3z9HsfspMD5VoRK8/29jUmB6l6TBW43yxRKsO1A/W2YTKeoGzO8ckxcWgRQjIeT/FS4K1nt62xziZmc4DB9rx5dU+ZTxl6aPYDeVEwKgxD8MihJx5Ek77uuFvXPHr8AafnF9TNnsHBm7dLdK44r+acH51SsMd1PdVIU45KVJYzyyv6+xU+tITBUW83nD86Yrdrubm6JC6OAMdoPEU5z9HZnNGkgigYTSdIYXj6/Cl5UXFz/Tm73Z5CGba7FQLDdJozmc4RQ8ARKfVA4wYePTzH3fZYa9n2PUJmBNczMhmX2yU7X3P+8BEvtlfk04qGDd+8fU039IxmE0yREdvULdP3Djd4yqJAisgwpJK9cjbGo8kyjRCRUVlQFRVDH9ltdzjbI0lYhMXxlLbrAEWeG4oiw1nHbDFD5RlSG/q+Z7NZsa13iGbLKCux9cDx0QRrO3arLqX2pxOcHXDOgYJHjx8RQmQynXLXLAkyElVk32zRuaSMGpSjbhqCt9ghsFpZnjx7SBRQty3fvviW+dGMF2895dikRF+MXL19S73qmY0qbFMQvMcNgHe4YSBaxfnJCTJKdm3DZrujyCtUljEMPVfLOzJlGOcF4/GI3vbsNzWhFijn0ULQuP6Q9Pv9V13X753m4BNXXIskkLskVmt9SBEc/pv3iXqQCklTuaLv+/eGGCFiEgPdcMClRHCp30ZrQwgWrSVKp4NbpcT7OVKIxPfuujYJ4s4lV7fWaK3w3tO2TZqtVHaYvyJKGawN9EPHMLRkmUHK/J2KT4ghzWYiYWMi8YBLke9RMCGmLpnEvZZImRjb1qbeH+ccIVoyI3HCEmRIrwMRdxBjsyzHKIV0SfhPv+cORotD31+e430qDRfvBepUXikIhDBQFIbZbMLF4yPOH5wwnWrKSoNwB6HfoUxK4WYOettydX1P8Ir9rmEynaBOCrrOMdIGkNxcrzDGMJlMcbbnxYtXZFnOerXj6vKGyWTKavmKm+qOBxfnxKDY7bY0zZ68MDx4cIZ1PeCZTKfs9itWqzvGF/PDNZQSA1InJ3xCU6oD0128Z6P/t1eW5ZycnHJycppQbQesY7r+PO/wQVmevcdwdl33vkw1hHg4WBFpXuBdUiIgpEEpyZdf/YYf/OBT/qP/+C+4333Jtu4QMr2W4fAzUqrIE6NMRh2hkCogQiqtVzqglMCHgALQguDTnlIag1QqAe2jPOxZ9WFPk66rpPULYkyHP1JIsqxEieqAdeE97iUlolOK4u9bv7OInmc5Qnpq5xAiZ1FoqrzEO8skW1CYDOe2eJ+iw9EeIbVk29+iGdhZx2wUmJgRg3B8u9kwKQRt5yjynM0QmGVTpBhofEtrXyDlEb5vuCz+z/ShZbkumZeaVdciZM84HxF9YLX6Na9f/5xv3iwRuef6EpZXK3Ybx/wkZ1enSLge5UzPCrJRw2xmIEjy84jrDXEwhEYROokZG4zx7NcNyhsGH+g6jZwJnFaoGBClS9zaK8l4lpPlMKx7KCLRK2bHOVFDv1X4IXGcAg60I/qC2CuKU08kEJTgL7/+v7Nuv+aHPxmlCEsm2a1axPgrstDxdmdojKPJcvwDS99fMz/NOZtN+Hj+59zafwNlTSgdW3dF66ecLVxyXAXJX/5bx+0bw4fPPuPs5D8h2pzZdMJicUZVTRFS4CPpAy544uF0JxJBbJkW++ReM3MiZ6iwwbIjizPabqAqDT5okAWjscB5GMIamGNFAbbFq1RAsa/v0HJC00EdWpQ3eClpG8m37beELmfY9fi4pRwHvvpWcX0pGFUGbxXeZly9EBidygd83BGUwUZF3TaovGJcaLyNZEbT9gODHRB4zh5M2W8bsgyWb3rOH08JTrFbDTz/9JjXr1fUO0c58dRXjmwSaTaB2ZHBjC2mERRSkk803TJt/uuuRWeafd1QOUm7FxQjSW/TB8O4zBl6y7CP9F4yuTBY7ek2AZlLykxhHWzvGsbzEbrSrK63iZVsocoNUrvDQOE5mlZMZ6mMyWSKTBdY/4e52VCaXdMwnS+4ePKAr159y35b08gRm/2Sxu0Yjyf87Be/Iq8K2qZj32wYlRXDrmM8nvDqm9c8PH+Ekal8UCLp6pYf//CPmU1mNLQIJM+fPEOr5PLcN1vu7q6YzSdc3zd0bc8nH30PLRu26zXb3ZrFYkFuFEI4qtGYfb3l5uaGx08eM51MWa1WeBdY3i1RQnF2csrG7Vldr3l09pCj2ZxhGHD+IavtkrbpuL69pZocGJdeMhIlX/7NFxRFSewVgozT04e8ePGC0bhgOq149fqbJLggyPOc6WxOL1rWm3v6tmPCBE1ke3fDycUFvu8pZjMu377Fx8jN8p7clAztntlszH3Xst6vGU9yjuZHrK6WiYUvFMEJ5pMFVTZn6B2v37yl2W3JRzkKyWp5z2effgwONpslxw8fcTw/5ng8p15u6eqOt2+vOD9/yFD3LKYTnjx4zGq74vbVFZNixHa5YZSPGY9nHM2PODk+Iy+K5CZFIT1MR3Nubq54dPGQYAOXb1/z6UefcLw44urmkuPZEWenpzR1S912XL59g/eexdEJZ/kp2/2ODz/4kG3T8ebqEjc4tpstz7/3Q2bZiFcvXuCjRQFHixkhBsrRCNcGclUi+ob1/ZKj4wWu6dg0Kz6dfEomC0IMGJUxWEtVjLi/X/Pg7AHT44Lrq2vqdZOYjUWBVornT5+xvLpmvd0wPz/j1ddf88N/+Od89Ztv2DUNz559wMu7L6hjg6w0/TCgZYapSk7PT8mfPeaT733Mcrti3+353g8+S8Pu4NnVDc3GUhYT3l5e82c/+ROaZoMNlm5dc/rkgovTh2w//5yj2ZysLHE+oLOMerdHaI8QGm3SvWOHwMXFY5zrWa7v6e3vz1UV4lBwKBQ+2oP7Nbk67OAOImBi9xEEREVVVYxG00OEvEcqQyTiQxpOtcnI8oyqLJHS0B6GL+8Dx8dHFGWO0RGjC4q8SgKp64mEVEQUBVkm0VpgMtJGbNuilEwDdRBEJFIGtEkFlkolMVmI1LoeQjhwvfPE33M9vgEfbWJ2C4W1qYxHyDS4W9/jY0+0Azbdyu9FaKUylPJYa+l7R9d5MiMhKrQGY0CZgAoe6y3OkhAmUiUONg6BSp0YQh042ILddk+MGePxcdocCI2SgSzLGY3GtC1YGxM3UsVU6qYMWVZSFMk9oQ8/J8+LxFiMiW3+jsXubHpPhz4dItgA1oPSOaVQyKJACfCCA0ZnwPkUESa+4zSmOUUIgXcJtdPUPSEoCJEir1jMT+ltcl/3/R47DAhhMLogM6NDWac4FNlpRtUICOS5wdpU3hm8T7xDmYpjo1cIIbG2T3x6AUVIhoV4SAlIwChJpiWZVgTniaSeEH3gIiYu5Du2esJK+CCxIR18WGvx0aOUoO1rVO2BtAmNUSJ0howhHTLYHjd0KBHJjabIU2mq7Vua3Y62btFCE6Qh+uS6dWGAIJmMDCYrsC7QNB3OBoRQh2SfOjiSItZamqahaf6wcrLVsmUy0UymGpMr2mbPEDLKSYkbArvdlqPjMeNRwdVmz3z6iDyH66sbTo4qTk6OKauMm7srMp0OykejGVJEHj6YcXqssG6gH1aE0GGtxZicwky4vLpGKs9uNzCfSsrRiG7YEfoeIQ1lMabMDP0wUJUlXZfcu+2+wfaWMIyYTiSuGbi/ivz0p8+4XL5gOp/w8vXVobDY0PeOLE8cdZX3SAPThabZDlT5GCEMSmX0neOmeQtIymrMr774Wfr80tA0ltOTC9q2JzNpTp1Mjshzi2oNRI2UkWqc7nUhoMgLXr3+mvG0omkc5WhC11mGtsXNPVdvb9BGoseRKAZ8HFiul9jeYvvA/f2eZ88WaB1p2jVv3nzNRx8+xPaB27uWUTnl9avXfO/7T5Em8ObqDXmWc3VzjRCS2fyM1apB51esaslkfsyLF2/I85zruyuO5qnIdDpdJBZu68iKAtf1RDyTyZSuCegQcT659YWowQomx8fc3t4xmYyYTkdEHOv1CikVfSto9j3Pnj9Em8jd7R3zWZZMI4fDulE5ZzoacXn1NacXc7Ki4u5+Dd6y21gyPSMqz7OnC4oypxoZvvjVKx49fsjd3SWvXr2hKjPKssRaR9M0dEWkGo958PgxV9fX+Nhz/vCI2/trispjjGEYLNZGplXk6CgHTpjMPKvVhrb1zA4iYZ57/vZvf8VorOn7gdF4wuRoTtMu6VxHnilubi85P3nOxdmYo/OAtSl58tGz54zHZ/Qt1PW3zGYjht7xb/7VzyiKMYtTgzEZ80VFW28oCsPV2zsuzo/pm4zFfI4bluQarq7e/t73trSaZjNQ6JxuP6SysyIghCUGz2J2hBCCfbOkKGcI6elaEDJiMoPWmr4b6Pqe45MjNpsNbdey7+v0/d8P2L4jz9Wh1A3W2xVNmwr45uMZzx4/gwiL+QLvPdOHp+RHFSY33F/dcvP2LUMz8Ou3L3j65AknTxacPhnhXM2+XdHs6zR3xMAQHc46CmMQhWQIA6prOBpNaEPAu5ayHLHa71jd3pNJhZKQVTmrzZ6qTIWWTnr6ziHwxJDwDuUsZ9c2+BjxNtJsarpNw4vfvOTHP/wB23pFNS3pXaRpW7yFH/7gCbMRxFGgrwVKGdrBoVRkPCvphha7A9t3ECwffXzB4rhDiAKjznh48YAoIzITiaNMJCtLMpVx+vABXdeTZRXn5xco7+na9H2QFZCPc2zjoDGMyjn7/o4gJY0d2NR7XCbxB8RbbiK7uxVPPn6KL3JyVyGCQeYKvGSiJ5g8Z9/tiVIwvPvOMYKqLIjRUZSSwSbhMs8yEAn1oY1i6JOA1bXdoRcNuiF9zo7nI6TWqUjQeza7mnKxwFp/wEuWfPD8KaXJuX55y9D3qOjIijltu6fZNmSuYsgFm3rHbDJl2+w5Pz1jv93TDC2X1zc8+vABUUWadk9eaibTiuASm7/rWopKkZUK7y2Pn55z+3LDzz7/Ff+z//Sf0tctZ8dndLZjaDq6/Z5mV3OyOKMsDK7viYPHKEVu4PTkiOloQt10ae+sBENwDG3P0DliplB5xrau0UFDH2mbGnfvKHWGiZo29KjsDzOvhRB5xx0HCCL13Tjn6fuE71VKH4TAhCyMeIhJFE8liglJa0wS2+OhhDEcyjYTWiSlNPvBASF14YhkLEg9N+kwwDn33/j33+1rCcHT9y1t2yQxXidOeuKzJ/ez9wPeD4SYErCpiykVk2qtDkaKQIjpoE8eikODd/T2UPwYHCFIQOGdZRj6A0YkHaS9K7m0djj0OKUSycREf+fGTnN/73ru75cobXA2HVQ465AypWmlTCXzUpKMIgSqUcnFg4ccnyw4fTBlNpdUI6gqg1Qe73us71FIhNJEBT44Nrs927Ulz0aMQsY3377l+GTGCM3N9R2bXcPR0TFd67hfLtnvek5OZnzz9StCiOz3HV3Xc3F+jlFrBJrtrubq6jXjSUU1MsznU6azMUJ49vst2+2acObf7wPe43sOjHdE2kf9d6Rz8VtGutaG/X7ParV6zzEXqMMBAwdRWZKZ9BnQ9z3DYA+l7em6jXgEgiDeFcmmg5lh6CkKTVWVvL18Q13vmM+nvLpsUCq9/11vMSYlF0JI1xkHrKLSEXPgu3vfEbw8sM4jgnflpiK5zo1BGp1Y/EEgD7NKMtAcCl7fsfWjBDRa5WRZQQwC5w94GJHul4QJ+nfIRC9kQaYrXFwSlSczEeEFhA2d3RBijs5ybCtoWs+sckjdo+wRRa4Yu1P27ufEkOKkpYrsmo7oF4gYyfQaq2/Ydx2FmRK8oTA9O3tDtIbWT/nZbxoq5bm7bdBlhx0cu6WlD/D2lWPbDWS25PihwQ2WYAXf/KZm8UhSTCRDHeiV5/6lo8gErokU4xH12jKZRYxJ7OlyLPE1dINnvNCIzpGrglgMCK+JMlJGweAE5VQRReTy2w35OFCaMUFGun2HLALO9th9jq4c5SQJ95Nxxk9+9JBnz8B3nqOjjOX9lo8/UszGW4x8yv1+ie81ZyfXdC5SzM5ZICnyO4Y4Jq/mjCctdoj48i/J5BWjbMrbuwxlzrh9G/nNlzuOLlpevTZcvm04mx2T7T+mGn2AydMHXKYizfYVebUAmSF0gYsB7wbyDBBQ9y8ZbESYBucLMq2JMWNijsnNnFXvib1EmpIgOgYi9XZHYMKys1TlQNdP6QeNFzXYNet1ZLePPHlg2K07LpeO3kasl9y83rLfwGS8wMYVN68CHsXm3rM4icTYU4wqXCO4eumYPwqMq5IQLeNCEmvBvnOMM0u7trR1YH40RsrA8qah2XUMjeH5x6eofMD3hrbrqbt7YoRRlbO9axAC+nVkcaapdwNVM2Kajwje0m9SwafrO/KJQMucDz84ZnO/5fxhhhCe3d4Tq5zgBnQuCGWG2ATOH4755tdLYjAUckzdb5iUc1q3wdGDDGRZRmvTaZ8pJE0XwSuOpgW+iay7mgePx2hV4HxH+MMOxJlPp5RG0Xc198s7ur6l61qssxR5ydXrNxydnCGD4k9+9BNMoXB9j/eBKq9YjCZ03rKYL9C5xr10jMoRs8mUcVlwf3NLYSq++fpbgg98+sknbHZrZtMJbbdHKonOclCKy5tLohfsNtskmoX0gSak4H59z+3tLaPRiLbpkhteJOFtdb+iVBVHi2P8XrJcrzh6tmDoW05OT7m8fcNytWRT7ynznNVyyXQ6xsce6weW2w3TIChVxeAsn3/xOUdHC55/8JR9vcHkmrgPZLqkawZebF4jS9B5SmWcLyY411NWBbvdJgmlduD+7p7F8TEZmofnD5MbJ1MsiTRNzVdff8X2dIcxE46OThDRp0LNxjGfz3GF5/zolK+//A1aG6azGcPQ0/suISxcoGs6Lk7PkS5Q/UlO0za8fPuGwTtkBOUF0QaabY0SkmePn3Etbw5xxTGzyYSubQnwnhVYaMPJxQOazZ5gHZnWeDdwdX3J+cUFUUZevPqG/XbHbDajyApWyw15XnJ8cooyAakTFkYJxR//+I/48uvfcPn2DTIInHW0+y2v377AaM3J8SkfPnmeSsZGIx6fLnhafsC//dVf452jtz2T0Zi721uEFeRlmdhvSrDebdntdxzN51TTjPube46Pj3j88Bm3d7csZnOul9c0bYuNntb2nD16gClzPvr0Y15fvuU3v/6c/bbBF+CzxPdXEow2fPyDDzg6OeIf/MVP+dmvfsaXX/+GGFKcvShLPvnwM37x5W+4fPOG2dmYZ8+e8qvPf8bx0TFZnjbxL168YDoaY7s+MbOFPIihA13Xs17vmS9OefT4GZf1nuPjOVYo6v2e7g/gJr9zkWitGWxiiMaDwCtFKmlMs7YiyytG1YjpdEKW5Ww2a+7vbwkhuWWVEskJHC3WKkJeoFRy1DrnGI1GHB0tcH6grtcHN/MIrdv0Z3qbsB9CEGJCzAzDPjmrD2JrRCGdI3NJ1JVKIKMg+BQBFVIiVXI1K6Xoe4uUjrZtaNoWkxmyoiLEAwNSvitJSoxFpQT+EFlNxUVJcNfK4KXHC5IYLnKUyAlSorWlrDKKIiPSYy1EL1KPi5kgFFjvEaI9IF6SEC1lenxnPXaw78uU8rxA+eQY6fvuUNqkkYSUeHHp/dAqJ4okkCeHjiAvcoADf7LB2rTRQESQiujBW5tEax9x2hDswBBBHDZF3kdCFO9d2qCQQpNn+QFn4hgGT123eC/Ic0WWlUynEutK6mab8EAuCQ9GJ8d3lpUHN/ohCioUWVYcXPZDOlgIKpXZFhqI1I2l75K44r0HERn6AMOQ4uW2TxuoGFAiomXEyYCUAaMFRZGTZzl5buAgThfFmH3TEoLE2UA/pLhyVZX0XUvf1wjRI0REqvQ6SCK273DeYfsOJWA8KlFSMhpVEFMZ3tB1yChAaKJLrGEfJdangiKJYRg8fWjo2oEQFLPpMZPJjPFoQlmOD478nq7rGIbh9763AT75+CHr9YpylKG1pJwkPEmRl0TZkRlJ0+549Pic5Z3l5z//mtOTY4wZuL60BG+4eJgQRVIEqiJDy8h6ueHO7snNEdpoytEEpMbaHucitWt4+PghWQ5v39yyvG/JOigrsNHhe8eoqlKJrVQ4IQ+9AZKHFxe0jePVtw1HiwnN3vLX//WS/+x/+X1eXn3NqBoRfGC9WnN8fEFVZjSNpaoMt/f3uDBiX3vGoxmffPBDvAu8fPU52iS3mtKapt9yc3ePNgXbXYuIge3mmudPHnN2do7J9rx+e82nixPaZiBORigNRSG5eDCjKqfc3qxxQyAA86MFg+0ILlI3Dev1mnFVJfRStyXLDXermt41eO8QKuPmrubBw5IoAoMd2GwbFnPDsw8vODpr+OLLXzKZKZbrWx4/fYCUaRMZrOF+ueL8tODBgzE2rHj15gWffPgDlCgJPhU+52qGUIrF2QSVazb7O4QeoKsZes/a7tA6YzTRLE403jl8tOhcM57OUwmjtYhmx/RoxHKz4+OPP+D8UcbbyzcsFqm0vu/WjMdTVss9s8WUGOH07JSTxSmb3T2v316yOFngosfoSFt7fvX5kp/+ww/QmWFSFQRvuby6oxqN6DrLBx88YbNeEryiLI9YLZes1ktCkChVcHp6xue//gqpV6lgzXmauuHBoyPssON+ecfLl9c8eXrKeDHDmCQ2Ne1ANS744NkHTMdbXr+5YrPZ0/Y3FEVJUabZQ2nBeJyx3l4yWxgePZ2yWXYMXU/0Ha9eXvHFr1/x0Q8F19c37NaSh4/mPHv2CJUJgmgZBs/dTcN0Jnj8+IIvfnGD8xknxyf87S9e8viJ4enzR7/3va3JWV5tOJkfI0I6XA4hIg5YjkggzypmekFRBfb1hr7tk+uYwGDteza2yQ0PHl2wWq2S2BEFIbdkekLAYUqFxWNDoDSKerdj2PV8+PxjijznW/8N0ijkJEcax+16Sb3bMRmNaHVGKBSXm3tGRvLBgwuyMKXeFWzbHVeblKrwItDZDqEl05M5obfslmukg0waQgatdaxWG/a7jsVowng8RmSafujp2wGJIDhP23TJDRsgRMX0bAwHVJ0oAnu/IxcZL758zdMnz3ny/AMCHdV4xPJ6z9njB2zvHOVMUk4Vk9GUzbrj9n5FnhuePn+IzhT72575YoYUksXJDJTEhZyuE/Q+YtuOkS7ZtnsGN+Bd4PufPiUvc169eIPrA8fzE3QMRO8Sn74oyaoSNcrJZIkPLWNTILIcPanQbWA6LvFjxc3uhmW3I1rLR/MZ31y9ZvZgwd3disnRmBpLlU3p2o6+bhP6QmkypdBCc3Y2Q8jIdrdJRYGK9Hle5pRlgRCSzXrPft8iiDRNQ+8gHxtG4wmmNMlbEQRZzNFFgVIGh+P+7p79Jh2WH5/MMVKwvFyhjKB3LfumI6MghJ7b1Y5m6KHZUeYl+67l8vaGdujRRlO3LY+fX7BtBSZLYh0hYqYVMEYagRdpJqjGJaNFz/WrJZvtilE5oW8GdKVpdnsKo1HTinp7T1Eo1itHZnKMihSl4vnzRwjh2O5rhmDRuaHtB1bbLXqckY1yULDvdpzmCzrX4dzAuKgQraPyiqPFBbvmD+szeSegvyt6fFeE+A5rktyxAWvt3xHFOYigyQAQgj+4zN8VhyZB03vLu+J1rfQhAenwHhAB531C+UUObnQOYnRM30FBHATUdwJ6T9s1dH1LpSTeD/S9pSxLQpQHF3ES6ZP90hOiwAVLb3uEEsmAmhTr5IoOEA+l9fLQXilkBBESCoa0X9GHItUQ3HvR1MeAHTzWg/M2mT9iQMmkkXoXcHZgs14jkDjbo7Whay1ZVrw/uBAHs6g4vKaPnz7gpz/9KVmuEHogyi1FCSYPhNjhQovzXWL3ywwhFUVlyPOcut6hxJhvvn6DUnBycsrt7ZavvnpNUY7RusUYR1tbgle8fXPNZrNhMplwe3tLnie00Hq9O4jFnqZpyMtkEuqHhhg9znuct7x+/ZIHJx8wnx/hnMXaAWMswiQwDYRDYlRyOIF4d+XxTllXyvDDH/yYs7MLvvjii/Q6e49W+r34LkQqhIWYDInOE0Pqgnr3vkXi33lNI0IairxI2o6W3NxecXN7xbP5GCECzg1IBSBwPpIZk67v6N8/BgehHBLaJviEjXlnXiKqv5NGFYey0YRD0kqnMlkhCEH+Ft8TUv8jB+Z5Kq1NBw9aq9SNpVIZsZR/f4zsdxbRu3DD3g5UnLHt9ux8ZKw3CKVYB0foNCa7Y5EvaJyndY6RFtSupjRHRNnivKIQU8bilJ5rJkVJpnP23YoQS5a7hkwtoDvHuVt+/tYxhIq63/NX//WWdq+J/hWb28DxE8n2rqHbgtAZIQjG8xHEQLuJafAbPDNyClWyv285Go3Z3zcUY8VopjHnBa+/WtNtHIuzCU0LSkh0VLRDhxw5OmfJM0OUA66PNLuB4AOLBwXbVctiUdHsB04eTAjCoXLP7rKn20M+iTx6NGWpLf3gmB+XfHr+hO//UYYSlk8ej/FskVzyvYc/pgl3LLsVs+JbskIgsxHbXiDJmVYC21V421EVU5biG3YNeKHxm5LLt0ds3D3NfkfTdnzxc8dmFaiKEU7vmZaaujEUD49RSqCMxPotq6sV+SgnKyqUyvChQ4gD91akC7rMTsi6T+jFK4iR0PWsO0+ZGXbdNWUxJdgxzvX0VtD7gWBzVnWgi4ZmF9itNaKwuGFgvVdsNpfoLOfb/5dHdJJ97dhuAC1wbUGMnnZVs7uXkCV3n4sWXyzYvFnBw4AKKSaTZTm7fc3QeMqRQY0EykbWS0G7A60D6zvLeKYZj3N0Bp3WuNhCU2HpmB8XhFZizEBXW84v5uyGFj8oJgtJOa5SLEgG/ADTSU6IGV3n2XUrmtBTjFN0OjnPBsYTxXK1Y3FquL8OxNpw8WRE3zoWxyOOFjPqmw7Xpg+9TGq0Uqnoz0uqcYmTLVUxwTpD1zWIAPOLimzQKfLvPV3n4X1L8e+3XNfS9ZGm2dHQ0/QNNlhuVjecTCc8uXhGlhf84OMfUqqSoeuxjSfLNLat6XTNZLxAiEiRZ4yrknL8gNXdLddXr/jBZ3/E0FmqouDm5obPf/lzmq5mspgwqipEBtv9hlevX1NlFUVWoJTi+ZMP+PzXnx/YvomR/ODxQ1zv6JoOOUistxAdp4tT+m3H6m7FttuSZ4aizGi6PYMfs94tkbngsyefMnSWn//qb5jPLhgVY+5Xa5wVhKD57IefsZjN+eLXX3J0coQPge1uh/Oe2WhGcJqiGtE7Sy/b9AVQTtl3A9NRxqbeYp0n68o0zGYZ++2ek9kMMThC17Fa77Btj1aGtt5hdMF0coQMGiMMWW5omx2TvODk8Qn1vqEbesbzMYujY776zZd0u44ffPpDpMxw1tO3PbZuGJcj8jxn33e8vbmmbTrm5YihG7i/vWM2naG14ej4GPxBOIrpZPvi4UMgsLq/ZVwUmKD46NEH+Diw229SBFALem9R2nB6dMp4XHJ3d4vShocPHiOFSQkNuyUoyeu3l5TFiCqvePrgEaMso61blqt7CmOYj0fkRc52veL2/o6z00cU5YjruytefPEt+2aXhkXrMVnBdrNiVk5TQmCYcHlzzc9++beURcEnn37MF1/9grZpGGdjqqJARLi/v+P+/o7L+ys62/N8VjEblbSbDUVe0bYJN7F6s6V4NKH2LUWmIUYm4xKEYRAd//aXf5VYo7nAB8tf/9XPmRRT/vP//H/Fohwz//hTqirnX/6Lf041KphP5vztz/6W08U5P/7+jwDY7Pbc39wRSG6xo9kEtx24a7a0StPtFsjg2a3W7Pdbyqzk4cnJ731vZ0YnZ0I4MN5iKrvJs5w4mqRrpxvAgCQ5ZrMsieNaC7I8FQyZTFKWGdpIuq5Nn0co8jw5i8uyYD6fUZYF211P1w2EAFq9Q1ck97gx5sArTEOYdT0+eopCojU43yOsZ3AGo3MgIWWsc/jwjmWY2MN5WZCrnKEf2O93WO8pzYjxeJoc2q4mAt0wvN9sZJnGkx0eM+BsRJJigClemAYx7wRBKYqiQmmHySBE+97l40Pa9Eiy5KBXGQL5PjrofRJ0M5OlguimoSj2VFVFURSHGOw7DrdJMVznDhg+SWaSc7m3zcG57ImH4lBjMoQH6+RBzA6HDUzaYAgBSkqMVhghsH2Ptx6jdSrvkgrfp5ikVgopFHleUhTlIcKbnC0xDEiROO5Kp4MGiOSZxRUuoVyiSnFI0mtgTCrwCT7gDxtEEGQmP6RKJUVhUkG48wfHOEilKQp5GJ57+r7HuQEIGGlQRET0SDyCxHZXWpBnmvGoTPgg57DWonVKQkiZekuscynpoRUhV8nc4EJyw6iEJ1JKpo3IMEBMBzpSputsNKrSJs07ovep56LviV5wPDvGup56Xx8isIquHdLhjFQcHZ0wmUwS51MZQvTY4bf80j90mSzy0ccPefX2Bb1ryPIc56G3niwz5HlkX285WhR8+ulTbH9J3WxQ0jPKCnbbDdb1HJ8awNO0NdttdyhWDew2O5xT6Kzm+DRjOhnhIphcs1rfJm6/SMW1idEZkIa0ybU99balLCZst5ZiJClLxb7e8fply2h0zGh8RmXOuXp1hes13T4jVzlnR3NCFNTbPafHE3YGFosR7fWGu9s9ZZFxf19zstjw+effMp4OiN7x0cdPqNs995stTkT6wbOpLbNRjlaB3XZPL9OG7/i44u52ibWe7a5mMi/JCs18Pkbrkp/97S+4eGiom4bp3OBt5O72BhEEd7fXaCJ5lbHebSmDYTGfYkxBvW+wfcf5w5icqqrAhQaZO67vb5lMZsjC4oeG1gu2ywFdwGhUEYJgOrI8efBDvvrqBbOpoqoyxheGvrtDsGcyWvDR8+d8++uvMXrEttvx4MkjXDFl29xRmBEEc8A8BVzoODkruL3ZEeMAUvCv/upnHB+P0FnESHjx9jVDtLy5fosQkn7Y0NsBGyJlJejafRKiMsO+qfny26+p6x5p0q8Lm2ND4Pa+Zjab8Of/4HE6eBtPGZWGf/kv/jnBdWy3Gz744Bl9NyTG/KxiMjrij370mFevv+U3X16zXfdMpzPwkq52FIVCxFQ6uFxamiaQZYHj8ymTuUFIjzaG1eqe6y9e8w/+4jHL+w3f/94f89lnn/HVq19we39NZhLXNvZJVIghOXZ//ZsveXR+SrSBxTTHZCXPP3jIaCpZ7V+y3u75wfd+QGFGXF2lQ3m0ZjKeE/qMb17+gs8+HTOeTlnMZqzXW37yk0/Z1m/45sW3v/e9PSrHLG92hz2FQWmF1JLxvKLKSrb3K/q+pygy/KHrYTor6ZqA0Jq2bXHeojNN09WpXE0kjqxSitxo8kJj3UDUaTyQRpEXBj84XOeJ1jNZjDlaLNj3e3b1lvv7O5rVkmlRcLI4pW463OEwerCetvM46wk2RwfBoqzw+x3r7oYoIl4GnHJU05Ju27C8vWM+mlDOKzbre/o+fZd13ZCwJ9owrqYMreP8ZMZu05FrS1WVQCTLNU3TMD89pt7uofeIIBFBooBff/4VP/mHPyAqyXwxJpMVx4tzbt++RciIiwPjaUkxLhjZirapqbuG6dGMwgwoZdhtBx4/nbGre9b3a9683fLrX36JjIFPf/gx23bD5e0V89mMH/3ox9T1nl//4tfUqyWTQiODw0hJWZToLGcASjNhrMfoQmJFpLYD63ZPm3igB1HTokvFo6fPaYaB5c0aPc7QSjGaVLS7DboQ5EIzcSWDDYzKCe12Ry7h9GzKYAeWuzuETmKSUpLpdEwIgd1uh1RJQI1SoAuN0J7RtOLk/Jjtfsu+3qNVxuLojMl0xtvba5RWzKcLurbl/mZJN7QczaZMZxOQSUR10WJkhsfTDB02RvZ9h85zggRT5egyY5ZPmB5NmMymjBYjhILddovtHEZpQvSYwlB3KaFthWN2MsdHwZvrNzx5fMHV6obgI/2QZsHj6RwpPPu2Q2c5mSlRmWQ8MuSVYXm/Z982tLYj1ZKo5IK2knE1JwwD43GRjBMyozgrybeGqAcKozmtjjjKZn/Qd7dznqEfDsi+tIdXSh2c0jL1Bxw4123bIGRMXGp4L5B77w+zSXgvwns3EE36jFNKHcrt44Gr7hiG4T1H25gCcRAqvfeHQtOUdHzHYk9CaTiI+OHQIyQOwq0+IFPS/OyDx9oBrRXqcF05b7FWYiHNrgfXcUKCpEThaDSCA+Lu3az+bkYWpMOESEpUCRIWyHlLHCwAWWYOpaby4EYOBK/ph3eRU4lSBqV4z3h3LqVWk6NeoKXkww+e8tFHz/GhZ1vf0LlINcqQ0hOiR0iLycDbNqUhdU5RKsaTEu8db968IR4KeF++vMRkmjyfELxku2nJMk/XOW5v7zlaHJOZiru7NdZGZrOKru1Th8W6R2kYbEueL5hOJwfzUo/JUofU7e0d292GpmkYjSdYl/joQh7SAeJwOCHEAeuSOObAAZOeDpTPzx8wnc5ZLhMTPcsNMsr3xp131+YwDId9QTyI1wmPkhzryaCTkr28TwrH6MnzjMlkxMuX3zA9uUDpiOsd3sZDKSrI3CS85pDQPDEGohcgVHrmPh7KYEFJA2iMKhBC46zHuw4rA8bI98J+ehxxSBzEw32VmPBSQDwcUAnhkTImfcPLhKZVGYK/P2nyO4voihFNFxFGsas1z08L1s2OUVWQcw5UDEOHKQeq/IKm36IsaDmwGV7jfeJLGaEZXEOmcmbFltY3tH3HVdvRbWfsrw0v37whN4o3l3vqvcWLmvtXgaOzMcMgULnBW0+Zj3BjS72zFMqgtGDftOzXEYchRvjgk2M2dY3uZCqIQqDKQBCCuzcdEct4lnH32lI9Su3NbbOhtY629hRGcHymub4cCIBrUmP6jJIQA91QMzkasd+2DLWCbsCMBMenI5Z3NdnUcVrlnJ6P+fEncx5Mch4+nrJrFJNsTm8NrTPU7orT8gHOrjFhTGk01/dzetfQ2g19t0UGePH6DlU13DQ7piLn/naJ8IL1WvDybUPwnpOzETosmM0b/uKnn/GLX33Ds4/PyNYlcvYF8TijGRZs91+St0/56ORPGbhHsiI4QxdfonXJsvmv0FJy1f8/KKoTBrejqVsqrclVQ23lIVp0T1fvsV6SFxPqdZnYSB5GSnN/t+Fuc08UFmLg7bWgqy0By+6uQHSRQE63i8w/8IhY4IYeU0qUmtK4NkVNLy1l6MiNormDfOSoqozmHuYnI7be0fkdbg25GRND5INHD3nz5pZYBWZHFaurPQLDZCwJQdAPFlk4jo5n3L45nKQWPZv9Dtdr/D6wD5Z8krFbOmBAFwZlO6IQFKVmtYpUC8MwdMxPZmxXO6pJOjETQ8buLjIr5wyZJ585bjYtjy7O2DYbnIwokyJ2i9kEKSUuOrwJmExS5IbVdc3sfMxkkhNEz8WDittLg3UdIpPUt45i9IeAF2FalfihY2c7blfXqGlJEIHO9Tx//ke8efGSN2/fMpvM+fqLb5gfzzBk3Ly+xEjQwbBd7ri5ueH8wTl93+Lp6bsaN0y4unxNmU+ZTccUmWa7WdI2ARUUo3zEulvz+W9+zWa34cmjR+QmpxIj/s1f/jW9G3jy7AlSGK7vbxlPJ3R9T5lVqczPB2bjGfv9HukUZVVxPDHYMHBzd03TbPny28/ZtntOHz5gNKkYmhUyBmzXMRrN2K73TMfH/MU//MdoOfDzX/6M09NzsiJju99Tty2D9UxHY6JXnDx8wP1+w/X+mvPzE7p1h6REV5rd6g6jckbTUeI1VmP63pIpxfrumv1uw/HZEadnx7RNjRaKi7MHrFY908Wcx2enrG5vOJlM2dVrhkniELddS4iBzWbPMAQePXzOowcfsFpvGVRAIri+vKHMMp4+f85sOud6taLeNejzcwSCy6tr2r5j8pMFu7rmZHGEG1qGkISp8aTg9uaWTz/5gCorefnyLZlQ5KMJRkFeFQgtuVsvWd2umOYZhEimNePpBIhU1YiiHHH35p7OWcosox8G+rZjqDsYHOOypNvXaVDuOx4+OCO4JGxs9jtud2tMrrh4+gjvBm6vbujqDmMKFIoQLNv9mrqvyUY5IXiOz4751a9/yWp1R5lVzGZzVssl1g6srpbcre/Y9jV1v6e6eYtwgqPZMb7tWcynCA+n41OaGLm8es3jRydMqgqUo/WO7bZnW6cylrZv+NnP/oZXX77moycfsVkuORpPEFpSt3uKLKPMcrQwSK958vjpgR2ZnPrRe/KiYFvXZAyIvmZSaPADVa65OD3m2xcvKapUgJjr/Pe+tx8+PGe9XuK8xRiVykF1ijMWIqMayvdFQibLsM6y223JcokPPQiHyQTaQMTSNH0StAfB0HuKosQYw6iaUBQZw9Cx220PeI0UJyyKkqbZYa07uNlz8kJhQsR6g6ztwUntkUIwDJau69Cj/D2z0FpPeFcyk5uDu16gtCAMDmkUpShROolHSmdoY7HOM9gD24/EAIxSMQwtIQSUzJEixztBDC4dKEhBpkYU2Ts0SSrNSRuX+J4hGQIpIjt4evtOqJWU5QgpE5YnijRkDkNPXe8ZVRU+uAOvtqNtW6QUGGMOmyJIaVoNYsAFyzCkqGHXdcnFrjSD7RMbPDja1qdBVypEFOhDgaOIETf0iIRHZPCBmEmKskIqR9cODIPDmByt08zU9wNGZ3TtwG67Pgy1ElGkA5Q8LwCf+IHIhCgJiT049C6hukRyjvRDl94zlRxN5lCoaTJJxNJ1DT5EynKMMQFvLZmBGCV70dF1Dm89QiSHkRagRDx0oUckAYEnyxR5nqOUZjKesds2KJVyus4fOJsyEoNLA7OGbmjJc02VV4nXyW8xQe8cYlVVvS9T6rr0XjnniF4wKitOjhYIBU2zx9mAUh4hJF3XU5Ylx8enZEVBjOEQg+0QNm0y2nZP3zdI+YfFyF6/uuPP/vwTRqOKzvXc3a+p93B0PGJcKYwOdF3D2+YNeXbEn/7ke/zsb3+NVjDOZ+RZZHl/izYF42nBeDzGhzqxKoVgMs44PTnj81//mqEFxiWjytAMeyAyGVd4pxIuKHjywtC5lnc9BXmesd/XfPD8gn3XguiQwnJ6fMHr6xv+2f/tlll+zKSSfPvVDecnU4bOMcoq9m3Dfrvj9PQhfbOlqzO6xhwQVQVSNfzy88/58MMz6nbJdue5ub0lysi26dBGsd73DF6ANIRh4O52D94jtWJ+LKl7i5aJ7wpwdjZHa812XXN70/D46SPabs1ytcTbdA8OLeTHFTZ4xJDEh82mQylYzMcUI43zA6NJlsrTVEZdb6lGOTpL9/V+t2UyK5ASbAxsdkuU0JT5BBEk99cr/viPH7Ld3SJCwdA35LM95xea6zcb1qOOu6sl5+cl49kR6/s1nW8SF11p9vs9RVGA8JgYkSpycjaiqxXL9T3LrUWXghJBt9xhXUde5lgcVWaoRAUyoHS6x87OznA24+3NNVEHTuYnXN5cI0TP4FMKZlzlKJ2EoTLPqMoKozX/+l/9DRpFplOvQvQgMESvWd5tmU4M+13DZ598xNB1nJ0vaPcN+006hJwcHyN1Tj9Ylqsln332GV1XYzISD9zBarXn6dMnzBcTmqZjtbzli8//Sz759BnHpxOCbHFD6p6RhwPLuvF0rUdLTbP3ROeZnMzZbDa4AEXp0J3Bucjd/TWzyZjZwrDb31NUFVdvb8hUiZKaz3/1hk8+/pT1MvBXf/U5P/1HD4jaUk7K3/veLquMLJMMLqRycGlAS2bHJ9i6pmnW+CwAFXWbEDXz+TlVofHCHfSSnOfPn/Pq9QuW6zuqqkoiAhy+ywLW96ByZvNThJKMCk27atFK0+5a9vkWZy3b7Zphu2RwFp0ZBuHZ2yalvoJgOh4jTGDXNri6odvVKGEQMmckp4jhBpmn6HxtA1mmkVliIu93e6KRqeRQaExlCO2AHTxZVrBb9zgbmM+mCAZiCJRlhpSRvDBs11tOLy64ubrFrjvcztN2AVmMuXx1zf0HZ1RHmnJqqKqSy5trVndr5sEQ6pZNPXB0PuP4Ysx6Ham7lvFkSj7TrG5qvv7iEig4ezSHsGa3XdFvPK5zfFu+4MlHj9BK8ejxI8bjit/8/Ddcv7hkMR6Rk7Pf1bgosMoRqFG5YhAdcgS60gx5gypGWAU32xXzyTEienQISKE4Pj/hzbdv0E6yX26ZHM9hYhBnkvnkGI2mbzt6ZylHJcN+z6woefb4Kdt9TT4p6V3AWUdV5Ggt2Ww3dF2HsxFtND6GlGjKFacPTtCZwnuH7SzjownIQO/atO+ykfnsiDfbt4hgcF3HXbdmnI2wsSdXOiVhw4CKgfF8ghWBetvQ9A1ZkZOPCuquwQvLKFRsthswkOUZbevo2wEjI4Mf0N3Adr8lyzU+tMwWR5ixIaqAVIKiKrndLhES1tsNRsB0OqHHk4UMKQq86KlmOb3vWO9r1vs9OjdEGQk2sphMGcJAjkBozXg+w2hBNJqzDy+w157mZkO723FzdcsPPvvhH/Td/R77R+Kie5/my3flh0Ic+l+Cp+87UhFpEta9T45vrTUyJP536qeJODuQGYEPKen6Dt0S46Ei/nCAb0xK770T4v8uWubdAb8xOs2lByNz2jukFUL6PkyYRf0etRKCS10GISbhWaQenxACeZ6Sie9QG1JIAoGmTX2EXTcchPZ36ct4+P84IEO8x8XkytdKEGJySCcnvaXvuwNKUeKDJzM5Sg6URUryGKMP4nxCPGqp8D5ydLygKiJPnj1CG0GwEaFSKluqSNfXaJMwhEI5ordECXlZMppqFidjpvOC3W6FlDnb7Y6uNxyfHOOdpOsHptOC+/tNSn172O32ySxiPVIo2qY7YG5KtCEJ8IWmrDLCAdljrWV5v2W73VFve+7ubnn08AnOv2OYB96x3pM7PBXKRgJEiZApfQypZDOVnG7ZbrY8eviYvu/YbNbkJhXLGpP6sITgfXLS+/j+vUuPE9+b3N8heqwdUGpMWZUMtuPi4izNY5v7hMSx/QFBExmNSgQqYYCiSImJ4JHCgFSpOFSYhPd3DkFAGsXJyQPmsxPapku9N8OKYfAHUVweupnSoc1utwckRZ7hfH9AI6VCX6U12qSZTUmJNholcrz/+82pv7OIPi0yWlshRSRXHSE6qkKw7rbI0DPVp8z1CbW/QwbIFJRaEGPBKFMEPyEiqYeBUknazrGpFda2bNbnvHp9z9VrxTdfbFnfD+R5RBhH0JF2n6J8+21H2zoms5zL657TY0MMgtFEcXomaWuFMgKygvE8Z79y+CCIQVHpivqmIx71xL1k98ojxz1FlnM2ndF3gaYf2K1q4ixnpEbsVmvcFLb3A846tneOYqzwVrK6aVBBsL2ELh9oe0tWOepNer1Go8B8UTKbzjhZdPzwE8UHT5ZE5zCh5MH4KTf9XyKtom/mKULXghw+5MurN1ycPGO1vuTFy1e8vet58zJtGrY7Rz65wdqOfIiIouf+7UBW5VRUeA1HRyPmiwWVmnJ61PNHPzrmkx94pmVPED+nK74GD7HIsPKXfOP/OV4d07cvwJ+wdS9wMqOU9+gg0foZN9tvqTtNNTnjfnuTGIbOk2U9fTcn+I4waKBm6Epul5dMZhlIwX4/QquK+5sdTZfhY83N68B4NKHe71mMptysaqQxvPrCIjGcXYxo2x2+t+xsT6ENjz8dJ0EAh/AFtrVkJfS9w8qIzCR9a9gtB86f9KhRROSe84dH1P2Oq5d75tMynbKKgWADu7UF5VH0qEIzMiWr1pJnOa519GLgaHrE29f3ZCNBZUps7Nnte2RQ2MwiYo6MEd8rdt2Bl2pKvLcsziQxFGzWDSIT3N2lRuDgIuvrgem8QGuJllC3PThBOVfMqwoXOqqsQp1bYGBUZNRh4O5GJgafHxiVIy6eTNJhxh+w/jf/6//9H/T3/we3/hf/oZ/Af3f9F//z3+EP/U//v//2P/2n/7t/J8/ld1n/+N/bT/rv+frf/of98dNphXUNu22bGHlBUO+370/nne2J0SNILhPvLW070A+BEC1SvRvy7fviT2sd3kIMA33fHoR0RdNqlsslu90GbdTBxZIY4UoZQkhOBiECeaGIQTBYg3MS59IAmJAtYJ3HhUCmkwMxhB7nPEoLpEqJCB8du3qfhBEdyLWhKHOkTpiXENOhR2qql0RUYnIHCC6hT4gGUIehyyCFxNpUuJqbAp0ZVAh4l5zVyfWdooHeeep6j/M9Lvbvh7bEdU+x1DzLyUyGc56m2bHeGCZhgjHqvdMsucMypMrIbXJnJ3ajoBtasoPIrVRyFCVnumMYLD5YsiwdjhidIY0gZIGh6w9YO4sNESccSucYk7jlQqjDxik5h/q+QylDjJG2b2mamr4fyLKMoe/T4Goj2iSneyo5PfD05W+d/P0wYIw88M9hsOlwejQqyfMcbTQQGGyLD5ZRNUVKQwwR7yxGeXyQxNimQrE+uYXKA4PfGI2IA+gUwArBMtiOEWOy3BAilKMRrfX4BH3Hx4DwgRgtRh9Y7FEljqJM75VWiiwv6Ps+FTjFiPMB4ZLrfbAu/VoKqqygKosUZSYcnndy42dZRlnmjCcjRpMRoA7JjZa+7w+om0DX1/hgD+6Y33/N9BHCRdwwYJRhUgnyWHFUTcnzHmfXFNrQtzl92LAKgT/5ySf86lffUncl/RB49OADvvjNl3z0SYnODUVVYHG41uG6Fat1zZOLOX6oEP2YZliyqjtE0Iyq5NwXMmM+NwzOEzpFu4OqkGx2PeNRztvr1xiRcX5eMDo6ZqMUQeXMxiXb255XLxx/8zd3/JP/7IzdfknGiNAPjKZThtDzvc+ec/V2hfQeH8GryGRRJad8qJlNC2Lo0ueEVzgncC7St5ZMaXwXcV3qSTDa8fLFHR+JKecXx7Rtj+33aFkyqUaIoNhu9+gCgoK80gTnsN2AGxx5XrJtOmbzCdu+JUaFd2AHQ9sHlps9QijqvcOLhtNqzicf/THrO8cQv8Z5T5kbRNQI4QlYvIhEIbjfNJTmhF/97WvIxzx6smC12pCXJat1x6ya8p/8kz/i+s2K2fEpm2bPsq558vyUTEs0Y3SuaW1DPwxIMrrG411gOh3T0zIeG4qPpux2G3wj0CpSlAXWtwgl2Lc5RXHMeF7SLldMwwTTJPwUvcHHHbv9FbvNiKZbM13k3LztKcvI2cUC4oZRNkG4yODveHB8zt31FuUHQt/z5psX6HjKaFxyc7dhMvF04YZNvWM07zGjPTerJU+eL9huBt5ebvnk46dMRiWzSY6MgfOjM+6X98nxNsrSQbDec3o6ZrW2fPLZc37z7Ut+8eKvWewztIZRMeLi6Ii2XuMtKCkoS4MdFFHA1c2G+6sjPv3e99m1X3Jzt2N1G8mykhgN37y453gxo8oNUzlhbVtOHowR5ik31zcMds1Hnzzg4uFzlqtrUDH13Pye6//yf/zlH/TZ8N36D7j+C+D/8B/6SXy3/vu8kjtcvxewvY841xF8OCQOU9Kw63q89wccoXqfZPQ+kOcZ3h9EzHhwL5J6hATq8HfDIfWYRE/vYpq3o0SQumHeJTxTSWR8z81WWqK0YBgG+r4nxoDRwMHZnmVZcssjDnOswgdL27aEQ8Gj0oq2q5FKMbge55KLvihKkBHrbMIQRclg3WFmFwfGt3jXT8qhxIhAQGqFyQzeW5wbGPyAkilxk2XFAd/oGY1HSNm8d71b6w4O6Yg8CM2z2YR/+A/+AVUZefrkCSYzNO0WIQJVVaC0xUVLwKOEJJJQKzaAyjzVJOfkfMLFkwVvr2+xNiKDpt4MSF0SQqCpe6zdsN0kPFmWZex2dTJWiGTWEVIgZQCR0rnT2RnHpw8pD90hIQSchf2u5+52x2a551e/+oLvf/+HtE3NaJzSJQfjPRJ5uBYOLnH5zo0u37+eMcIwOO7vl3RdEtCzTOPcQFEUjEbl+31H6ovq37u6Y0xu7uTSEQfxPhWAcih8nc2OCXEgyzUPH56x2v2aQIdSksnkmOOjY0ajMZvNhvXmDiHSXlBEiFIn0IqE4N3B3Z7KsrU0VFXFyckpu+2eze4udTkNlog/mGoUxydHBB9pmpY8y5jN5rT9lq7f4W08lIlKpIz4waaC25AO0Yh/v0T+u+NcXE1yHzkqY4ghJ8trRr4iz3Os7elFRIYJUa2YjRx9X2PUjE0rkCHy+Zs127ZmvfFEO8cNHbvNmt1qw92t5PZ2TXMbGE1yyAzNRtKrgVzntPcOFR3ThaFeRurWI+uWfK5xW821G9h3W8aTkpXdELzBTDPubwMYyMeCtu0YlQrf52ReEZ0gCMumaVHWsNs3PH1+xtWrLXIyMDtVNOvIxlnOHo9R44GqEOk5Hhvaux7XSapcUOaSvIqYPGd3A7OF4sMnks9+XNJ7z/HxmhhPsGHg7dUty/WSq5sthfLY4Z7Ozuj3b+gtNF3LKPvXWGtZryxv7re4VnP8wFMWklwp/vgHH7O6amjqgtmHEiEyPvpszvFDCdqxGB9zXDYI2fGn5Ryvr6hdpHO33AePFh60onNT3uxyZNmRhYpMXqFlRtff0oY9QxwxlfeE+jEyv6Pve/wwo+t2DNHR7DWb1ZqTM7hbKkLIqdtL2tpwu+xYLxti6OisoGsifSdoduDRCBtSAUzWEUuHzmAiM3CHRmckXkVKpXC7gB0L9muBs4Ii89QrwdkTjfCGu/s9k8mI4CQnF2Nwgcw4VvUdfe/J1QSjRpRTT1FGLl96mi0cn1f0faTHMfQN0mnuLxsWZ5HFac5mJdjc1ZSjjHIs2L+OlCcZzdDjo2V9p6kmmvVdR64DZaUpdEm30eSzyH7XMBpr9l3H+WLKzZuGo+mc29drIh1eefqNwFQpXjdZFFxdb2jbLafzOcvljm4YmC4MfXT0rWN9f09hKsqRYrfcMx6PefviD2Ozfbe+W9+t/2EuYwSTScl6fUW76xhyTQhpkARo2z7xA92AtW0SFaPDuRakS65mGfEhMQz7vqdpWkTUKKnxPomcTbunH1p2ux1931OpincOm3fRvtR6fhianQWRhPqE/YB3FjkhNM7FA6e9hBgYMoixPbBgYbAdXT/gbJOGUGUwusJk6hBn9KmxXaRorA8RKXQaRL0/4JsUIppDi7whyyJFUdC5nhgdSoPRGqUgRouzDkFOZgxKKoKX9F1L29dE4SnLEilJPMoIWaaYzycU+Zim7uh7R9vtyXJFiGlQDdEionwvqEqZoqXBy/fOniQGS7Q2h3KmiNGGDvk+WgkCYzKUlCkKax1Renz0EAIuDIQgscalAwgX8N4zDD3OJaZ9cgClWakfukMxUcQHh3MCEQIhRqRM7PIsMykh4MLBSRJxNrnRlTIYk+Oco7f2cPiQNi7WDQxDR4zy4KzPkyPKDUALTqCNRMnkNpdaMioLpqNpwklJjfEDUkmESB0u/dBgssSfzvKMsixoh54Q7IG/mB5PGYWWAqPle7SQEhkmL0BqpDJY5+n6njC4VMwkA4NNjE+tNeMqpyjKQ4S4Bx8OLvsMnWWYPCG5Ek80ubC2221KaxoNImJdS0QcNqC//3r17R0XTyrW92BKBzJgO8Hrb6/5i3/0MXU/0LWRu22DqTwuwNvL1/zoj57w61/W7Dd77u4znj+74JtvLvnk+2eYLKOIJba3RBkpCoFrPW9fran3kccfFbAbePNyyXwhmE1KtrsN1Xxg8CC8RsmCCAgpGNxAXkT6bY/tJSfHj7m8esNkktPuG/70z34E3ZSTs4HN7iWb1ZLHR89pmoHRaEI3dPybv/4bqmyMVoLcFDjAR5cOSAT0XcfDhzNcgLqHzsL9XYcAbO9oreXjJx+z3XT8+U//iL/+t39JUUZ2m47RaIpQnsyUlEXFixc3NPWEslI4PzCZaOqV5aOPPuDNyyuKPGe7TyWGUhsI4J1ltdwidUU1KknFaBqERUjF/X3Nswc/4otvvyAQ8S5QlAV5OUaqSNMuKbJUlHx9c8mjx7OU4Bo80/mU9WbFaDSGqNjv7lmurjk6WfDNN1/z6Ok5gY4QPdU4bUin0xH3qy2T0Sx19+TJfX1ydMSbt0v2h3RdWVQ4N6BkhrUpBq90znJds61XHGUFT48fEWPL3X6NFIa+61ASJpMLnj4/583V3+KdRcZUvJo4vIZ/8f/8GT/+8SlukCiZcF9SCC4uLpgVn/L5V/8Vbd2wXN0zOYLl+oZAR92uKSrFsFF0feTDxxfstjUhWGYLg9Hp3qrKgq7v8c6nXgsj6IcWEPzqiy+4WS9ZnE4pRiV2aOn7juXynjwvWC43SJ2K1LreE8jJc8m3n19hshHnzzVnZwui3TN0CX0wm49wricoxfL2nvvVir1f8eDhAz7++BlNveXbF1/gbI/zjjAcOsy+W9+t79Z367+1+n44zHjy4MC19F0yi7xLHP72kF0ihcJaj5JpfpdaEzzYwSMEZCaSmQLv0jwopSbPK0JIhZpCpMcS4uByPxgy3gnoIaSZMIRAURSHxFd2QJ+4ZEY5uJiTGzkJ4amv6N1jSWJIM57zDqWqhNfLDFmWEl/Ot4efkxznMfpU5CjUASfoMDoJ8MlA437L244CF9zBrKIBj3UJyxF1pO9T8bkQyYmeXNfvBN7foj5iTH04IXjOzk75kz/9Y0alYDYfs9/vECKipDz8IxBS48KAtV3i1UtJcCltJ6RmNFGcP5gxmWpurxu0LOg7x93NmvF4nIp4+31KyUpN1/Vo/dvXM3VVeZAwnY2xrsXkgum0oqoS2mm7bVjeb9nvBoJLzO7Ly7fsdjsms+aAMxwOKVyBUBJBOvxO7P1kepHKvH8vQ4CqGvGux8gYg3MWLQVlmZMX2eHwIfVrpASERApJFJEY36F+3hXRvjtc0XR9S1HkjCdTvHdMpmNudz1RDJTFmNPTE85OHxCjYLncMAzucF14lJRprxc9gx1wrkerdGjjvWMY9lzdvKLvO7rGst1uDt1LmrppE0tfJ6xNcuNL5vMFx8cn3C8tg90RVfp/CcHR9e59t1fwAilG/26Z6KvWEwaBUh2D6PFqhgqeT8onrOJb1qGm8RHBEQrN/b4lDAWOll997em3ms9/BQOeut0x7LZkWY6WGVdfR4JxjDLD2fdytp3n9fWW8/kRQx+Zno7JY8Fu2+CtphwrWjcg8nT61oeefmkxpmRzPxCt5O0veqYXjplZQPSsup79rkWrgtA5xjPBxZMx19eButlTmTEPHh1T1zVROlRu0KVmPp2ysrd0rifzOSOZ0fgNozynFyCqyLZuKY1h/xIWH1s++t4p/+SfPmOwXzEqSu4uO/6vfx0p81uabsR6c4UdJC++dJw91OyXhhAabNhSzTJCJ9ntLKdHFUFYJuM5UWrKKvD4o5xR7vmf/GSMNy1DkzNajLDtGC0imVwyENCyZVEKrD9i3Q7oquNqMwAj5gIQDhVzBmuYjSrWzZbWR1RokUiC0NhYEL2hj1fYbkOzvKIbNIYF3uVc3UcIsF7D2zctb1/6xCyipO+hLDW7PdgmOdLuNzsWR6nlvlxItquao/MM35aoIBlcj8KTKUPr95SVojOReuWYzgtEECADF2dTBuvIR4o27hHSMC7GjEYV0be0ocG2mn4D8xPF/DhL2IjgaF2G7WqE9oCi7SU+OrKoiMGzuh8QXnJ0PGFzWaNzQ+40Q3S0m56n3zvl6nLJtByzWncoIlF1FOSI0tI2ga6LMBjMXODaknU9kPsx/UpzupjTdjsWJyXT6TFXlw4RXcIIYAnbDt/3TI9ydpsGnSmEleRFQb+Hbj0ghaFQOSLzECRd33D6qPhdb+Xv1nfru/U/ouVDR1FIpIrUzYbBJuadNvK9eB5jarvv34l8eKzv8UOPlB6lFTH6A6LEHmKHkqqsyIscYyTgaJr68HjJbf5OBNbaUBSjJKgO7cFd0yGkx9oe75MwriSktpkMgUapVGhOFlByQIgBIUKKi9qIDx2IFikS+9eH5IJR0iC1xgSB8wHvw0EY7RBxwMdUuqRUhtY5Sqafl2cJO6KGPZkxyeGdGaRMrmTvFEpqUB4pEoYhxFT6JFTE+YHQW4SI5HlOmeeMxiWjaoQxhv2+TqzYcIi6hvRnEQHrhiTMJ3s8kSRoCSEOzEl3iErK90xKqRQaQ54np4pSihgS6zdGj5aghUYg6Q8bjbZtUToQYjgcRCSszWB78qxA61TQZ4w+OOvTZst5m77vXEAdItTp+SRzRTwUQCupiAjyzBxYigHnE55H7iI+JByM9xalCpQ2SKEPzzvS9y19v6PvWmKwyQEPaCkxKuFgtFT0Vh7Kp5JLZb/bIlRLiBLnE8PfRoG18YClSG59rVMRdfAhOeBRRJFI6y4AUoM0DLYFQKhI8msJhDIUWlGVKQ4fvDsUbTqIEaEEHJxdUkVCdPR9Sitst2tCdJT8dvOXSq3+MBH9r/5Z/Qf9/e/Wd+v/Z+vfX/Dtu/Xd+m59t/7/avX9QJ4nbEY8YFWGwTIcON5FUR76Y34rpifXsibLkjN5GIbkCg4JA1gUJTHIgyNZoOSBk+7B1i1KSfI8pypH+IM4/U5Ifyemp5/DAS/oDoWk/j2vXUlD2/e0bU9mSrIsQ8rEFg/eQpTJUGAUUklCTHsIRCqCty7Ng9b2CJkf/q5HZcllb61nsD066GSE4d3r44g4Qkyu7SwqfAjv+5Kic4hDEarSmrbtWG+2idHteV9KGeNvkSdZZvj+97/PxcU5k0ox2APmUWtklO9ROFILovf0Q4dAHQ5yk7vfR0deCs4fjLl4NGe92mN9jxAZfe+J1CgVDwK1O3C4D2WZIbHmyzKVcEoRODmb0DSp70YbKEc5292G25sN203P0EliyMh0YHm/5JtvvmYynVHPUm9JljtizHhX+fmunHYYekBR5CUmKwCJVoKL84d8/PGnXF1dkuc5EY2RkqJMbu5h6GnbFmvtu6vjcI0cOoxSDOK9o15KdUglCCbTCaenM2azCd5bIHXlZFlBUze8GS6xg0+4vBBRUiGz7EBX8NiDQSaEnpBJcBm97bGdp+323N1eokTOdD5nPDthubyjbTqsGZASvv76a0IAO0SKfIyUa9abNU27AxRKpHJY27sDgtCBE0zHJfPZxd97D//OInqIknElaVrFLDuiaR2titz3P+NoEmn6KUGUOHHHi1eRblvx5s2Wm5Wn3Wdsb19T5BO29cDZRY4wAecl9d6xeGpohgEZPX1QLK9rZIA+7KGMrFZbXBeYnObUbU9e5hQjQzSerg1YB70NlCcRv1PgI8VUoaae7X6LCZJcaEayorlzPPxsRNZX3N7tqPcdvvdE23FzXzPONccPDU0/UN97zk8F02KCt9BISxZLFmeW7a2lXwdk6SjHAuEEfjrgnaCQkW8/X3O/83TtHUJ6dldj7m53KG3prKDMoIgV3/yqRY97slIRvWI6HrPqHPPjnIcPp3z7Zc2f/9mC7/0oo5y1OKXZbzpGpcYrj5IdMfRIWvwQWZzdse0jUWZsBkGle/JS4YXkdCwIQ06pFFEOhFAxK06J9i3eO4LLsbXGeUUfPfthx7gMQEHTblmvA7dXLVLuCd2I16/3VJVmtRsIgyTjiPubPdJElA68+bplcV5SrzwCUFGhMwgbSZYLzNGY3bZDBJewL61gXGVYaxnNshTpwXM0mxDUwObS0wtHHwOjRUW96+k2AVMkJ2W+nxB7iQyGLJMUJxnZKELMkUGiSk+3C2x3ngePSuIQaZqedj+QFZKm69kuB+YPDO3G0gdLu2yYjCqapiZ0AueumE2n1HVLObG4kaXKF8zHBW/u7nHRM51W1PeBrC/JtcaXLWM95na15q7pKTLD7W2L7wT313umJyOcTJwpNzh856h3gd4GRrri+Kzk7naJDmPOHozZrRyPPz7i66/eUk0yhrplfDT+XW/l79Z367v1P6IVXOKNGyUgerz1eBkYiBDT7ytlECIcSjE9CAfR4b1lsAPCQj90B9RJQpoQHVQFWguETGKydckNGUnR0xgDWmcolVi4rugTIkaADzY5pF1iQAo4RDc1iAwhc4TMcFHivMf6gIsRFSM2BIigdIotJjEz4INA6ZJIQsSEg0s7lS0FbGuJ0RGiQyoQuUIIg5IZIDFGUJQZedAE59+zvIWQCDTywH0MIomlQsT3zvF4cNQ4mwTwIs8py4KiyCmKd8zxyHazxboeIQM+uOSQVgLv3xUqHYpLvT0MnZDl+fsBfBjS65qEW1JhpU7icjyU5ogoODyh9/xDKdJByeACWQEQGYaeYegPJVPxwH9PQnPaRKkUx1VJDLfW4sOANolDm0pRFc5xcJun2Ggec7TOECLQ9y1EGGxy3Uc8WZ6Gb23Sc+dQYtQPPbvdhrZdEWPiQ2opid7jrKXvhmS+0CYdvsTEehxsTxh6IpoQBc7H94VCeW4oyjQTSBHelw15n3ZWAoeTmiASPsj7wGAd1qfIcSDhiKQyZFKTm1R0HvyQNrx9lxibOm0YEYk7CgHvHH2fDi7atgHh0CaCSAcpqSj33/tHwnfru/Xd+m59t75b363/gCvL8kMRe3JwJ8d3Qo4k12/CtEqpKMvqYFCJGJMTAuz3zQGVmKUi78FiB3tAxKTkUdN0OG/JMkNypwuqakxVlXR9y36/PZSImkPhpjzMSOk5vnPnJhd3MpFkWY5zgbpusK5P6U+heSfYCiHRRhFlcoN3vUMqdZh3i1TmGTy969GZITMaNwxorZlMZgQPy+X6fQdQURZobei6JjG5Q8LdOBdS5431afYCTJYRkm+Cpulomg6ts2SmIBzSsP7wPOH45Iif/OSPmUwqFElATa74/zd7//VkWZZmd2K/rc85V7kMmbo0pquBRgMYDKYxGJBPNL7wv+QzX0kz0mzeBgYaiIao6ZIpIkN4hLtfecSWfNgnvBpoEF1GgwFGlq+ytMqMyAi/cYXnt7+91m+V6kbXkhAjkkhdOpsZ9xgw2laDUc4UGVmsNJ9+ds3b13fc3gQQBik1wSeijOSsSSnV+XEuahWiXlRYN5tOpGexbMhlxFpdGd1SV/RIEihhq4Fnqn1G+8OR3/7ma776wY8ZhoHz80KI1R0vlELO2MY629e+KPGxYHN+vbpFx5dffcH/8r/83/FhpHWOtmvQWj1c7kzTNLvQK2Yni0zOH5fn8wGO37+PrdWsVi3LZcvTZ1doI7i7v0EqAam+r968fc0wBJxpabva/XY6HVEmY61CiIKQCSkzUouKXElTxTgacI1iuXJo5TBWzGnqaTb4iNnUlXCuhaK5uXnH27fvKKJHm0DOAiULQtRSYSHATxFKTfVeXlz+rZ/hP3iJvrGKpekQ2TEEU9vC5YDpOkp6xmEveN9v+fr1wF/+S89xu0eJwn4vOD8XFKvYh55kJ779NtM0ltMucNwlGgWsC7Io/CEw3iX0uST6Su6xjcTLiWkULM4sEsVxNzGdEu1KElMhJ4ijpHUgzjLjJMkfFBOeeHSEs4n12YKmW7K9P2BlhEVkPGbkpPFGI8t8UGLD3d0HyIpjPvBkueZ4NzIdEqOOhBSZfGTdrWjOIYpINpll2+L3gte84ft3W1KyiFTdTCXVyPjzz8959WtLlju8GunOOmwDdgEvL1f82Z9fc/O95pPPt/zwi4bf/PsFn/0gYTevWZsrXt23XF0oRvWGtdW0dskUDJPpkPrAaYJ1s+D1oUerS7a+R0rNshkY+iWNE/Rxz9gLdHOPzBNLq3DpCSlJhC1MoXDYRaapMO4H9jvQqrA/ZWyj+PbXgtNxR+dW3HxfmNSAPwlk6bl8tuHrX9zw1d/ZcNglXONoXgpu3++53iyYTonlWjLeWwIjWiimNNYIzwhvtgfOzi2iFBbpgul0IoQjffKcrc+I5cTObymT5sO30LiGrDJt09FPR8bo6VyH0RGhwWrH4TgRRoFbwqqVqHzJ/faeJ09XsI0sz1tyypg7DU8H2jOJtqAsLKTF0nDxqeZ3v9hjjOU0JE5DoBWSnDpO0XO475mmwsWTDdM0cfGZZTie6NoF+/uetEqEErFOsb7SNHnN+1d7Ljdn7Pod4ujo1hLbWOQ57G4Hnn/WEbaCaexJSaBt4nib0E7w7bev2R33YFqkd+T8mBl91KMe9Tc1DppcIlqvkXILBFJWZJ9m14JCG8dysQIFIYzEOFHmgppSCikkoq8MRe9rrNJZg7UarRUpeWKqruY0L3pDqMOr0Q4h1OxOqEN4KXVZWUomp0LOHyl9CikNCI2SBlEUKWSmMZBiRswMvxwzWYHWAimoA+lcEhPDBNmQoiTPsVYta3RRNA0palKpBUyVeVcPB/XX1/hqaxuC97jGYqytTiGRkVFVdqOQlTmoEtoVbJGUJBBFkkv9eUWL0xucWWH1AqMhesHpOBD8gCChpEIUhSy6FormTI6QU+VsxxiRwuDaurDOOTFNPVALo3JO9XHHgvf1woKiQIHUqSIvciKnREiJWOqhLMSJXFItfyq57tvn//HRkZTLQ2zVOAOlFpgO44AYE4iOtu0e2J3TVF3ZhULbdjOeprqxi/Ck7ClB4L1G6RajNQJHTrmWYaWeKew4DVvG8YA1EqM0QktSFpymCXU6oFwteo15IpZILh8RM2aOpMo5ulrQoh6qGmvRuhbHphCRqhZ/hxTq0IxHT56SawHs8XCoxVuuQSuJFGCNRgqJIDPGRIrTzOLsESrSNaaWdsVci5DQle1OPbSlHCkEgq8/F2OY3/ePW/RHPepRj3rUo/6YtOhWdXGeEt57VssVShq0MgzjQIyhloDmiNESrRwxRBrbMAwjzexGl1JgdDU61F3yvCjNBWsMUAhTwDlL0zR07QKjaxFtyWV2nFcXfNu2eF+7W+pjsRhdl60xRnKOM8IksVgarCvEdEJIB9SLAQClCkILxjGQYjVpIAQf3r1BG8Oys/R9T8manNzD7y+lYNEt6pJeDSipadyCnApjX+ai4oDIEqKm+IgsGiMqv712k0SydOxvRxSaEhO1v11Wl38BKUAbwZ/8nR/y8uU5SnrIqp4zlKfIA8oO1VCUa78jaJQwSBsZxpHBe8RpqEv65BFK8ckXG8bxc/4f/7df1B/vA2VGpQRiNeHM5x5QcyoyUHJG6cR63dTSZNdxuXnCNEb2Hyb2dxOhh+kYGPqJMNX3RuOW/NVf/Zr/4Z/8j0zTOBtv7NxllShlPncpizEtM2PvoXgUkbFW8uXnn9A2Gq0UbatoWkkpEyHUJKX3Hqg9VvV4UM8ShYSU9RJDUhEwWmu6rsFYxcXlmifPNuwOb/CxvheUdIQQUEqyOdMIEQnx/vcXOkkADqninF5okbKy0mNMONsgREAog2kacsp8+/p3iBlB03QSHyprXqhqvhJFEbOiFIExouJJEQglGMeRnAqr1YZONdzd7ri9u2W5Wv+tn+E/eIl+c7ol2sgkLEVLLlvDMSv6bcNfvfmOf/cLy29+cyIkyekeUpCVbZpASs1EYrsLtKrhdOzR60JzLRlDxjhYb54wnI7YprD4iUS4QgkKi2N72mGt4XSMjCmQe8nQe3QjSDea5kwRTzDuPMFOSARhLPT7zPJqPjwLgV1njsOAyg3HcWI6jPWNtTIsVor3Nz1PLxXZnAg2YJXDx8zr+w8M7yQvnl+y3W0RSuC62tw+Th4pKnvIKsNh8ty8z5y9jIzbgtOaLCaaxYhyG4QV/ODvrJFGc/cm8eOfXdOpJZ20fPLjd3z61Z7tDxyXzS2ukbg/F8QsGIrkgx9ZtgUfEtp8YIgNY0hsbKHoBbvUcDd6rqVDlCX9oEGsUVkigkGkllEe2Q2O7SFzXgzBv+fea46TZumuOewSUzmx23re3sBypbj/MCFFZPSBNzeB472lazpe3XjWXcPF+gnvpwNSCF6/vefqMwtKIFLiuBt5/uwJUzei2sRZ13F341mctdweRoywtMvI+9cTl09bfFDoqNFBMwXB6Rh58qXhQq44fPB8cv6M25sTaRS41vP8y47DISG8QljIg2OxaDmEd+jsKD6RfGG56WhEARFJbqQTZ8To2W09yIRbC9ZfWs70it39yP4uoqUl64SUkWmruHjpmA4JkTXdQpOmTDGJ6DNWWjZXEt0qzs4uePX6PX4IZK8xwnLsR0SWPP90w/u3R7oLCGUEOSKSoojA8SAQB8E4jTz7dEXfj0gnuLsbkVqhEIyhZ7lUnLYTZ+crpCrIlWS7n/7Qj/KjHvWoPyKF0CJVpusky+VIYULKGlt7cCagMLZDmswU+uo+lx8L0WoJpVYWJQ1+OkEGrexDcWhMdaDL2de/j4GUIm1b+X/VvSDQ2s4L1okcCzGlWsBZmF3UBokm57kYR8xEv5wRpRbllBRB1uWjQNZlPPXrKGmqK7wk8sy206o6woPSpJzmAtOZQVhAKlPZfuSHBvtFs0Z0Em0k2mRSHvEpgappISEkyELOEWSiJlUNQhhyLijlUCwQ2UFqKKWyyq1JGG0Y+iMR0M0CsgSh0VJDjuTiIUNO1em+XGwqx1FU53rFh/jKnIz5IR2gxoxqNUJULE6xmaKgxEDME+PsSlBKkdJUn3syQglElmQglUzMCSk1yMrpTLlUtmRJ5DJHZ5PH+gbnQKiKlTGmOnRCDIy+R44ZqQohDyATqAyysumDLyhhiUGS04gUhXHaEdIBhK8HF+kQ0tTnnMRhPJFVpuiAsfLB1Z5KHaqbbom1S1LUTD4R/UApoTqKcqmUlQzRF4wpiCLwc4GsEBrjAzlVd36OEaMUVkvczOXUc5R5nCaSEIQykokkRhARISMxQs4Src5omhVKN8R0QCqFFJJUanGvCPUCqR47/r8vHnzUox71qEc96lH/v6fKJReUXA2BpRSaxlUntGIuPa9dPM5atLIcQ+R0OiGEYr3e1DlR1hTk5EdSrshXITRKy1roHg3DMNR0oRAE74nBz4WjhlzinNqsX9PahlyrdGphvandLzFW08bH9KKxCmMl/Wmsfx40Shm0dggR6xlDObIS5Jip7GyJRGGNQ3QKZ+y8HI3AxOFwIIZaCN84hzGOxaJigI2ZKm6yqIpBSRmJYtktcI0iBM9pGiBbStacDoGxn4hRVtNEmI0LsvYMvXz2lD//B38Pa1UtTx3K3MGZyWUEMSHk3LFaqtMe5hlbK3Kc+4pSqqYjKVhvGr76wUv+5f/6V+zvPNosqgte1HMJIteEqJBIYUmxFpX7ONEt1hhTkZmrxYoYMvvtkWlM9KeRMBXiFMgxIUpGK0nOMI4Tv/vd73j6/CkhTLjSzGjliJ755xXXo/i9Z0P8NV644LNPP2G9XnDYR6yp3UwhRKZpYpp8LfucrU5SzjgX+XEnX+dZZQxN67DG0naOtlN0C4PSgUJf+59khgwh1kSBVJZSEqf+WDE8StcLhxBJU6RQWC5XUDT73RHvI2cbh2taoNAPA1ormtYSoifECalA5VqGm1JGlTLP2amaumQ9jxpjKrpHKa4uL/jxj3/K/d2O0/EX3N1/wNj/gsWilhccxxb0gVD23OwGXr8VvH295d/868D2NuFzwDYaHxPXV+fc7u4wjWT3PpLVBEUxTYHVheF08GSv6N8U1OeJg/9AUYopJsIhUo6wXnWEMLC4NIy7QDa1zOr6kxXHQ8EtwX+QlFaweeLATQQBTy7WfPtXJ7qlZlXW2OeZYk7ElDkeJrp1ZLqPDCfYbBTaSk6nE8jM4TgynDynfaQsC7vvCu1KolXmzdd3cH5CjpoUBV1WdG3LOAaGHFitlgzhQFaC4mur7BP7Aw7yFeulZn2e+cGPLT/5XKKd464feLKWrPJA8hLfTgzxQOMErsscp4Ht1LFwiYV0HPOIjwNogc8nnEqk8Slv73ZcnRmcHMEumEIhpbPq9uPEOBqM2LA73DGoA1kuuT8MhNAxHQW7O8EUI7c33yO1oB8DYVS0TvFv/v0tT66W7G4SwjpyUYw+IpRgfWUYdgM2Ga4+6zhsj2xaixQTIQUunnWYRnA8HTieCjZqxJhYXjT4WLDKkU4RnZZ0CI4fEi8/X7I9DNzfjmzOobtQKCNQI9x+f6QpC/rbSFKBpWkJQXPo97S2QWbB5z++4n53j6ShCIXFcthNfPfulrMrw7MnG0I/ksyA7RRWNBz6A1efLtjvJ/r7nuWi4expQzoWbna3BDxNY7ALQx8m2lXBNoohjKwXS3wz0q00Ijm2u4FkE6bNNAvHyq3Yb3ukh1Lg/bsTi3ZByYWrizWLM0u8DAhTOPUT6VQwssUaxd17hXaCtix49nSFjx4bBIf9kefPzzn0B3JW3H8YuHj+iHN51KMe9Tflw8S67Qgx8uz5U4ZxByRi9AzDNMdDNULKii6RFYOSU0SQHziCbdPi3AIlLX6YZg4683DtZ+d6XbxKoZHCAIZSNKAQKJRSKCkICXKeKBkoH4nTCkr99aXwgApRWqONpGkcKTMXCQaEFAgpKFGSU0HMLnalLCWpOrChsNbRNi1aReIcj62L5DI3zdcypZLFxxJ7BKIWO2kFwhPSiBAKawUlVzdGyqEOlaUeYCCitEKh0UpjrJ6fj0yKGVShxmkluRSmyWNMgzX24QBTB1s5M+QbTC6sVmcgK+5lmvrqQC+JUuowXh306cHFo7WeD1BQpvwwJEolq9d8zukqrRBFVKZ7THPcVWC0w9rqus6lzK6kRMmBaRoYx4GYPNrUg4ezDUrVKLAQEGOs3HVV0EZUjmWpjPXKc6/lTyVPpCRQKpNzoO8PTGGsiBxVy1NTzqSc8CGQU0Cpisx3TWW1C1Vd+zHWixRjLNY4lI5EVRjH+rxEUchJPjDmYX5d5uctJY/3npzLQ3RZKfWQoDCmlhl+dLqnUp+PRERpOxdL1kSkEJVX2rZLtG6IMdI2LdY6fIwwJzykVCjxEZXzqEc96lGPetSj/lgUQi3IrBIcj0farkXIjDFqnhPqos4YzTiOM5vcoJRkmsZa4qh1LXicE56oAqXOfc5ZSrEImRiG8QHZ+JF/LqiowsZVk0z9C6ZU0TEfZ2Tv6yLVGEVMsRY3Up3bNWVaS+5BoqSuhotQqjNdG6Y0AQWjLFIYSlaQBVotUTKR8wkhYBxGptGjtUVKjcwS7weGcSSXkbZpKEUxDJFhPJFSouvMA45GSoMyLSW5WhApJWqeJyuGr5oWlJL8/Oc/57PPPqtufmHqkkZUcw4zzqVeIkh89PhQjRvaUItNDYhSTUJamzk9a7i8WnP95Iq797dYaspTzLiTj5ycOksWQkhYBSF4rNWUEgghVLTh/sQ4Bsi1B0kbRbdwSCUJQZFyouk0KWe+/va3/Nk/+Lv0ff/goC6z+/6vbc7/hj6iGl+8eMGLFy94lQLGyLlMNDEOdREthEIgH84PdSlfLwOc64gx0DQd6/WyolzWLeszy2JpyWUil4mSPUJlrFaAJib/cCFTSsG5tl7qzP1Kox84nfZobZAyMwwDUtby2VY2lJIYxxFrKzaHEuv70WrEXMJrrcXZjlIE3qf6exeB1r83/mitWCwWAEglWS5r0eqTJ9d/62f4D16iH0JPfwh8uE+82W0ZT4XTwfLqd3DYZZZrxRQUMRYuri2H3cCi6fhw27O2orqthszTHy05bSHuC8oYPvmpI7mBZqV589uBEmsxlS2GOFXWET4wjYl4UiAS93rPtM0MIWNdQarCdpdoLgsiS3bvA5cv1hR5ZDwcWHaWoU/0W3AO9u8jYUwoLyk60vvE8kIy7QS7G890LDStIWmF04mrTzV+n1lcWl69OpKPiUW3YNIBayTYgEs1tmGk5vkXSw5vFDoKLr4s/LN/9iX67A3P2icIccAubriwK9bnhhL36HwGzW8RObG09U1plCOrFql6Fgp8LqjJMUxHbNvi1CekcgCxQzaCPjjgnvvbhv2xoMwO8hKpI/fbGz67Oidmz5Qib98FSlnw9vUOGzLv3sAwGI4nwTSdQNcbPWcMxxvJWdewvd1y/kSRg+bsQgCZw84zjoXh/Z7hu0yjE5uLwof3I5eXls0FHLcTsbdoa2spG5n9YcQPYK7qxYg+86zWoP2SEAQf3pw4+6ShXSn0ZOl3kdP3iatnK1RS9H5isbCcTp7p1T2yg37y6JJ4c/8dCkffexZNg1Cw+ESTd4KN2/Dq+y2yKJ59Xl1uL75y3O0yd/c1laCMIhcwnWd/H4i9ZN12KK243+04DvD86VOEmqCrfNNcHG3bslk2+AmU7bnqWn73V0fOfzKiD4JiS11smAk/jQz7ifViST9NFO8oIlKK4/DuRFLw7vvA5mzJePAs14LxfqQ/er786pKvX2VkyqjekVXicrOiXYf/7Of3UY961B+ndrsPrNYvkCqzaBuk8vPSMpJSJKWC0ZZpGjFCPix0K1NazYWMFq0t1lj02hKbiNQJoSLTNDJNHqmq24FSi4ekdEjhEFTeeM4FYqnLb2RdngtTizzVvGSXprpzcn44QEhZ0FpSnKJgaoEQoHSpBwzlKDnMy+1alkquS3UhFc45rHV1gTt5hBA450ipVP61D4SQoPz+n4dhJKSCtRqhUmVkh1ijgNpVx04SM1/dQ0qkGEhZIMgYYzG2Fg6FGJFhIpca2y2lUHIhzoVNWuf5axeUUrRtS9s5YmwIKdO4FqmrIwlRy3QEzKxykGIuzDQVkwO19PPjMj8phVKVYxlCnOd4WQufcnX2J1EoOc2x0oic+eg5ZYL35DSRs8f7kWkaiWlCqVocBAVneVjspxTwXuK94iPwW+vKKi+58hRjqixyKWthVU4B731NMIRQDzolkeNECB8PPWK2BOXqHDcGqUFIgVa6PtYQMMbWSxIp5oOAp5TKdq8XRgb4WNBVl+QfuZ8gZneWmnmVM09eaqSq7HalCoKIVgtSEGgtCFNfD5hCgnRY26BkvTSoj13Vw7IMSBJKCbRSs0vocYn+qEc96lGPetQfk6YxYkxDKXWBPk3jPOfkOuuITCEhZMH7iqdYLBekWPdEH4seazFpwodapgiJggdhKh9czAYZUcs5hRDzAjHhfUJIyWa9JJrIOI0PjmCtFafTiRACUsq/xm7XxOgf5mUpdS3uLNRy0Vyxdv1ppOT670NdtGtVkY7RJ4Y+s+wcq1X7UGKaUl2MKlXRMtMU8L7H+8qsbrsVQ+8RojqLSwkz47sghaJxlhwbfDSkqBCI2RAkcdZAqbPmk6cX/MU/+SesFiukrM+JVHOHkChIUcgiI2TBOUM4jbNZKKBMTSdKoSjzOUKrWjKqpSF5wbNnT/jm1wcIYIyccX6F2bs9d0BJjNHkPNE0qp53VC0TRUSUK2zajpLg4nzNNEX604Q2EKMi5vremPzEbveecTyy291z/eQpRluEckghAfmwvP+PJaQkeI82hs8++4zd9h7vxzqrJ4+f4oyekfXPLBWl1JSsUvWCom1ahqFnuVywOVuhVOH8suP5izPaRcHHPUUMIDxSalzTYKxkHGGaBlKKsxnHUorAzumHmMM8o0uM1rRti9YGa+181uHhoqfiQQXGVBNQfY8HSp7PscqQU6jpXVXPSB+d6qUIPnz4wP39rpp8/IAUmu327m/9DP/BS/T/8//ljuG+0A+Zcdcx+RPGCVJQ6KIRK890yNhlwHVL3n0TKINArSS5g5gKQkE6Ft796sTZS0WzUfS7gf52Qtqe5UYzGWAwfPbjFd98veX55RN2fACZsCvNsPOkHpqlpn8dmFrP5lPNky8b7t5OSA1ee443GdvB8Tbg+8zV5y1CTwz7jBgMP/3TM24+HEmh0K0drdaIJz27HbRP6kFY6ET7rOD9gJCK42ngyeUFvfIEAsUobt8HmoXm6WLNFA9cPl3Q5gZxceLnf3rFn/y9EeHe8cnVgo0cOHjNboq8DweMbMg5MsqE05Ex9IRjw6IL7E4BYRy5eG5Gxfa25+zcYuw5rVgQw5aTl/h+yXG6R+Z79odCu1iyXjp88vSnQh4UV82X3O5ec+qfME5bfv3bkcMpsVomypCZUmL7TmLWEbmJvP9a0bmOjduwuLjl5vYO13TsX0FqMsVGlFQ0Yo09S4y9p1kkLlYrUvKUyVBMy+32iIoKtUispGFKMN6ObNZL9JVmdQm3byZkF/F9pHuSQQcu+wX795Gcd8QhYU1Du5aMh4i5GHjxxRlFTZzuEucXLfvBo7XgeOrpe9isNVYuuHs14b6MTH2iENj5A5GRq+sz9ruJs01m609kWzgdBlybaZYN037E94pubZjGgu0aZBdQW8ezK83b7Q0vPzG8/ubA+qxFo/EHzavjPX5K5GPg8MrQmYYYAjEVpEsc7jyqCGxSXF1qxryjP0kuLzT7+8Td257lWV3Q0Cu6S02SCR8jbdeiD4r7457GwJPzBSp4tuxpm8K6+dvZTY961KP++NSPOya/omk1OUekBB8CaS5Sqe7zQggTU4iMvjrEjbY4Z7HG1CVozOQMzjUsGkURHp9ODENPCAldqrsiRmoBJ5aSDZmPOJdIkQWpBAVVexzmPhpBnpeJmlLqYvZjOanMCiEzShWKqMvhyvKLs4vdAgIpDMa0GLUgBgh+ROvKaGyaFm0Mqe6rkbIuVJXSKFWIsSJLQkgUxnrwUBPOGZQuhNSTZvxN68yDQzmXiA+aIqiosJwRGIpoULpyy0OcavmQoC7OcyFTI7x1ke6hKEoBbTTWWZSShKARPiB1PQDNJpl52Jez66YeSD5ecjA7zUuWM65GPFyGWOtQSs9L6+r0yKWQkyLFRBGgZMWLSFkq17tkYgyklEl5IoSx8u9jwPsR7+tAq5ShlHrgq06nSIwBpSs73jlHTIEUAQq5yPk9MSNxSqwc85QIsf5aIQI5SFICIQ1G2xrbbBxd19A0BoRAykjJgmEciUHgXEZITZjG+bEHYvRoXd/HQI0Cz8+PtTXeGeP0UDgFCqlq0iHFQpR1qV/jyAJtbL2kKUARlCSgRIpQGNPimgVC6gfW6cPyXjhK9vW1U9Utk9N/9W8Jj3rUox71qEc96r+hjHEEn+Z0nsc6W+fjUpe4MYW5+6aQVYO1DcYohv5E2yqUEg8olJgmvPc4pyuvOkPKnpSmWm4ZU50jSTN6sbrLSy41zRcyISam0aPmlCd5TmVaizaK/jQgkBilyDHOQEWJEhIpVDXg5EKKkZLmec57pANRyrw4FkhRTQTWWIw2M5KwovVKyUxTxbZorWanckUJGmOYgsOHASHrcrqUapaocyW1r7HU80dKkuBrSauUajbu1Pn2Jz/5EU+fPQHqz/ngkUKTZuY7os6x9ZLA1Tl1/rEQSkUY2oqChITWup6tvCcENV84FHyckMrWS5FS5sSunvud5iRprvNvPxy5vN6w2lisM/MFQGEaAyJHiBHXFrpFQ841yZCBoRekUtju7rk4v67Ej7VEitr59LHw8/+T1HzAWK1WaKPZ70cKghDSg3mons/m9MLMn++6BucqAz2lyGaz5PJyDSJwebnkybM1UntGv0NIDyJQoCKKYv1LKkGjWxrXkjPkJLC2RSmJ8ZZF12F0TYKulsuaCDauXrjkSE4VG6SURmmDlBU9Wh3yidOxnk+F0GhtgJrycE7Nl1CVVz8MA1qHh3PCOE7cvH/7t36G/+Al+vsbw/G2Hq5vfj1w8ckCP0aePXVstydkXHH/7o4njSWXAbmoYM7j24LQAzECOnH3buLimcNcjBhtGaeRNEicWrBcF/bvRxgKv/l3W+Qycbe9Y6THR/DDiEgSbRsuLhYYTgQF0WfkMtG6BpUN6wvYBc/xlFAGNpct9+8npjJBcCyvNFM6zpEXiZOK+/c9IWdQ4DaCaVv5TrFPhD5xuW64+TDgusLgA1cvBL5PvPjKYRvLy6ea3/3bwip3/A//4Cvs+oYffLEipG9pFoZGbhmS4fboWZpPCfkd511mP2m+vjtwtbpgCJGNW3J70MS8Y60g+iXdouHqfMP96RXWtBz7iRQNjf2Ekz8Q4jkyCJbtBmtbbu9GYkhkenKK3Owz96fE3fstfjyw20lu3kycLmDqBVoalmvFq99NNO2Gi3WLLz3HcotWlqVz5OjQl4mb257hlHn+YoHvfY3YIOnfZ2ys0Y+r5+ds7w4sdKRgWJwbWtvy5uaep59f10iICAx9RHWWMEqQ79kdt1jjWF406E4RJsH63OCWnpgSrlky9CfcRuGz4NMfaVKc2N3B9jTSrRbs3k4snERk0EvQXWK875lOCttMDHeFvTpx+bRhiIXjCaIaOe/W9NMJ1WikFNx8N9B1hm6lOA4Hyl5DVggNw/vMr/sjy41lvepIuXD+tPDqVeTDq4lPvtqg1oaUNKEfEFLC1DL5LcuNZeE0NJLtN7Xp+M6P+CnSrAVFJhppkRuJHyY+vO1xZwWje4bmAJNlGAJP8Hxz85qoBE9ewuj/4I/yox71qD8ilTKxP9yhzGrmaU/s9tuHn/8YwwxxYvQDqUS0VFjb0rUL2sbBHLPMWcwLZEkqzGVAaY771cUuRddUzxzbFJgZDzO7W1RFsiRfndfVzQwCg1KmLi5zmn/vUAtwSiKVACU/uEVyKXV4LhXNUgdGixDzwCoCRSiENAipELmgtCGmTMwQYsLHQkZVDJ0PFJ8IMcHksUbNC/bAFCpnz+gGIXSNDgKUj2U3ASkiUhmkqJcSiVQXq2SKKA+u5jo4Q5GSmDNTrMO7EKoWpIqC1AqrGqSxpFQRI9M0EucyHmsNQoDWHxMCbmYJitl5AjmX2WXx8UJEPpQ3pZQeMDZKS7Sel/hKPWBZhKyHg1wSYi4nzXMKgBkhw+ysEeL3EdWKA8rEFFCxMjON0UgliLPDJ+R6iZBidVzlnEipcjmlVJVfGWuMFWrCIeUJazuWywWLZTcfCIEgGPoJH3NF6SERUpH8CFS2pvcTMUaca+bXQTw4qpxztDgmP7uufEAIqtNd1MKtECJpdloVUSDWMtTKTnc0rSbqVLsDjEOphhAywzQQQqBQL2xS1r+/TCiZRJ75n4961KMe9ahHPeqPRdbWcs2cqxN60S1QmnkpqEjzDJxzTSQOQz/jVcS8bPazISCSS0XFKV37YuI8T41pQM8O37pEhmHs5z6dmlIMPtL3p7qMDwHlXMWqiJpIdM6hjcJPASUlSiuGYU5cajNj9Jp5ThJzh2ihaQxCZJQC4ea5dHbCay3RC4vW4P1IjOXB8GGMYbHoUFrSD8cHw09MmWE44Kf4gNtL2TMMHq0lKUmOx0hr1zi7YtGskeIdTeMoOZNSxdx88skz/sk/+ccYXVEoJReU0IRSne0xjoQ4ksrIMAZSHkmlJl9zEdWZnwtC1tLRmt6t6MRcCghFjH7m2tc0bUwVR11yQVAQsuInc0lYbRBiJKbA8xeXnF3a2nsUPKUkbAuN7VifObRqSBF293uGMZKzZJoSwSd+9atfcnX5lMPhwMVFrgaph4l9dtn/R0oxzmgWQd/3BO/p+x4fEsHXube+JhYpBd6PSFlwTrNe11ncT4FSHFfXZzx9dkYqAxfXLd1SEOKBEI9zSWhGaoH3E7vdjkJivV4jpZrNPQKoZ5Hg0wMyZxwmpEg0TU14liLqEj7UjqhcMkobSmbGZGqUqkhKa+18hqQu0Ut+YLrn+T2hVE0WPKAuVaVtfOzP+s/pD9689X2kXWluvo5oK9ltB1xWfO13tRF3dwCl+PA2ILaZwxuwxuKc4OJZxh8dw3HiNAZUO9JNClLi4mnDu2NALAPDmHn2A9jfJBCJ7sJw+01Pc57pB4WwBbsQSCV49/pAMYEGw7jzvL9LCCXJ6UTUDT6M5CRoFo6Dn1A2QS8oZA77gaITxjjWdsn+w8DVkzU3N0cYBXffDEivSNcjTREQJVEVPvnqjPu3I58+XfHDn13y/c2v+Yt//BXLZcPl+cT/8Z+u2JxJunbHGApN9w4fAzd3kU8XI98fO5wEaTJOSqa84+ATU39Oe36iHyVT2hLSmsEndqeIKCvGPJJiJCXH7TagRMfxIGlsT6KWAIQxsbt7x/XThq+/Fux3Eyn3NJ3m21/dcX614dtfj5TB8OSzjmfXa8bcs7098ewzxWojWa0W5CBpG8F4X1g+c0xHxfWVZXufeP3mgJIG5wTdheT2ZiTqTEqeFz9ecH12xWl3x/27E0vlmEJi2Rjud0dOOmBVS/YTp7Fw6o9IF+lWK1TnaPMKSqYV5+z3R3QXeXZ+TZSB1Znll7+45/wqUYojjhO7/cDaXRPVPZefWxa3G9oOulaRKOzve1QL061GN5r1Zsm0C4y6sDuOTMHjRyhIFpeZwzgx9uDVxBQSTy8dKktCX3DOsrzQxBGO08iXP73m5u6G5Zlh/36keM238cDdzUS3cDSbwuHDyBQ9SoFzBiEC19Mat4SrK83bNyMqtbz9ZsvVjzqyjJwtVgx+Yr/1kBOr68JiqTiNgbevj3QrQ+PWHNKe3/x6D0WSi2Acehr5GAl/1KMe9Z+QDMQ0cDrVQW8YT/jJE1OmcV0tV0nlAWOScsQaS9ctsbbFubYOh0aQYmQcPFPJxDIwTieGwZMSWGOQQlX0ShFIoXC2m+Oq1eke00ShoLWja6nlkLmgZC1kVFJTLdDzwnl2VudSuXU1RjrNY6FkmgJKOkrSFGnJURGBGAqlVHb5OHqsrct4hEbITMqFcQr4EJFSoqWiabv5++lAKmLmfCtSCpQMUqvZdVH5e9MUmXyNByKgiIwUBaklpjE1TmgrZoVSCDmRKMRcHT8x1eZ5ZEWCKCGIOeKDR80HE6Wpr0mqhZ0p10Fc699japTSKKnmA8LsIKKgisK5ZmaRU53Voh4AhuFE8ImSE1oKsJqcC0YbGmfnYiEFuXIiMyCVRcgyY1t0fd5m50dlQiqsrcaEEKpTXUiFVBapTGXbKwi+XqiULCtipyRyynPksvbNfMwnPCzoRcZYyXLV4RqHtRZj1FwM6gH5UEIlhCLEPF8EVMxMXfiL+X0YMKay47U2NM6BDCBi5X6ajLWWpmmg1GV+jJkYCzkVpJEIXZ1VRisaazFKI5vZiSUUIB8QMRUTUxMcKUV8GtFG1gX9g/P9UY961KMe9ahH/bFoGAZCqOWVTeNqH4wUlPhx4SnmksWAnya6dkkIce6d6esSOUWElCgl5lSgIoWKbfmYuBSiLnqHvkdrO6NgEl27AOpsVR26Eq0dOWdCmAjB0zQtwzBgopnTdDNesXxELtaFvnV1Ls65oiCnacI1ld3+e0SKIMtCKZ5h9BXVMSRAzvz2JVJKpmmYDTYSYwxqXv6Xh1kuU0rFn9Q/sySXPLutJUoaunbN2eaS5eotJMFmvUFrzenU88/+2f/ED7/6srLkEYzjUGf0MIFIhDQyTSeEDYjiySXXzpvqoqiIEF3nPTX35+RcyyvHwRN9xtiaZrRW1wQsCWctoXqB5sXuxy6kMjO6Jd3CYGyiCI8yGeccjXUIoQk+E6eR0/ZEyD1CaoY+oo0mFckvf/m/8Wd/7x9yOp2Y/ARzOhfqzF3NP//hIr3iNevjuf1wyzAMtY9oNuBUbKTG2mqe8j7Tdg1nZ0uev7imbRtev/mei8WSTz675urJktFnliuJjzumfot1uV5y5Poc9HHidDqhtaoGmRjxvjre22ZVv070xORxTj1gb5RSD71Vtbuovv412yuQUhDnrqumqYaZpmlqkjZGlDUzkkbOaMqJGDOLxQIhyvy+98i5H+y/6BI9RI+loXGWixeC4/vM6mliOGk26zW7/YHmLCN9x4ebI82F4vy6upVe/y5hXEFpwfIKsjLkBGMfGcbqZnJOYazjuJ9QruB7QTzVg0pOihwTTadYXQjCcSREUFLRLiV2pUleYLSkadeMp8Kwb5hOnqefW3b7I/t3heXG8eSnhre/nSg5IynQHjj/FKwZecY57z9saaNjcWawi8J4SixXgqXI/J/+D5/z698e+Kf/6Ixms2LMcH2Zac2R0e/pbE8IgiKgaxQha4bpjNYdKGZkfSbJWeNkzyEJdiHgGsuLyzPebb9l1Tm835DKAWcF96fM2SLw5p3g6dOIQpLkgDWalBTbXaBbOr5+fc/xvmO/s7z50ENu2O8dxzGwfT9xuk8YC3Lp0CvFMZ14/uyMu63n/LzltE+sLxOXXwXev4EPh558ZxlazXY7MJwi71+fcE7SnjuOh5HDcU9KEtkUNqsVp30g7t9jNZxi5MW55M2/99yZD1w9UZwOcP6s4LoLgt4TokU0UMigRk57DxSaqwNLqznsTpzSnuPWcxwk0sDNt3uevlgjxEg5eLKIDCfJWAb2dwdenBniUbK5aHErSZctiyvL3S8VW99TCKwvG55/seH9qxNyoxFZo6zim9e3/OjnTzncB9jCZrWkzwemm8AwCbb3iRQyzdKw/9Bz5tacjjuatuHuQ8JeJYoJ6DOJHzWmDfi+ftMKI6gWrr9S2KYhhUIcIvvDfIOmMsUbbt7veP6lRp21nI6RbDNJ1SXP8bZn6iWyZKw13N0c2JytOb+WvHt9D6v+D/0oP+pRj/ojkg8D0yQRIs2c68q3K7lG/MRcslhxIHVwl0JhdEPrliwXtbS4lz3jeCJ4X5e6Ic2uE0me/1lrgZIaqTRN07Ba1kV8CJG+z6RUS4q0lmjXkHMtKfJTXSaKmcXtbIPRtnKlpUAWOR8wqgvGzxHNFAWxFErWKCnxggc3TC4KkWVddk8RZTRCKnKsDp5a0lRoGos2FspcDqTjXByaSTERc3WT1+Ii9TD8eu/nIaw6l2vZU11+G1OX7THlWkZUBOQaUx2nqT7GnFCqDmnyYdlbv27FnOT552aHPxWXknNCSoG1hq5r54iiYBoDOddhXEqNs5Bmh3c9UInZXV7wXhHKBDkjBRityKlgtMRoSYqBIjNGK2gsIVYEz0dHh7UWaytGpjLs5cy7NBQjaiQ2RkKobhgX55SBkDOKBmKqbvlc0uz8tkiVKWkiRg0EPpZwaq1pGj0fqtR8MTC/V6XGOUWZETsVB5TJJT28Plr/nokuRL0g6doO51q0VsQ8zsxRNcc5Tb28SdXhk5InhEROBd0YZBaUlEjaYqRDNgrnOoRUFApZiMqrFNXV76wj64rSERS0qYfWwPRf95vBox71qEc96lGP+m+uUmJdtsqatgyx8r29n/A+I2TFzU2T53g8zonCFil1XermVP/dkLG24k9SEg9IOqMdKfKA5ci5FlzWmVVWXMl0ZL1aorXAz0Wn0xjRElzX0TRtXXT6OJsoIkY1LNqGECZShMY15OgRZUbAFIEUINF1yayryc9PHkoiTFPt3SETpoQyLdZ0SFWxINY6vJ8IIdC1C3zwTH6cDTkfC+IBBGbuApqDnlhTzzGbzQU//tHP+NEXf4oocH19xTiODMPIz3/+JyhVF6m5VAPHNFWES0g9MfYIGVAqIfRHvGSu3UkIrGurcSYXhK7zefCeoe8pWbPZrOm6BiEKMXmE1HRdg/cTUkpCLGRRZpe6IKUIorBYtBSRkAaslkghsEagTEbkWmyaRMbaQrOQpGDQZsHYR95/eMfhMPHq1Xc8e/Y5u92WzZme+30EStn/5HvwY0JWKcnzF8/55S9/UQ0mCWKcQGTa1tYLiP6AayTXT8749LMXPHt2ze3de84vGl68fMann1/VCwAV6ZaWkBJ5mgCFkJKSYBw9pUhWqzXjODD0I92iQ2dmV3gtLPV+YBwOpFDPGdMw0m06pDQcDkdyiDSNxdhauJsFaKNIOXI6HQixdgg0rr5/pynMZ0+FVrpeZpTaDaBUdcL7MALQdR1t52o3wd+iP3iJfnlpGW4tmytBsiPLK833X49zWdKR02FisZDYZWIlDEXUN5xdQVc0T15I9rtMGAqffLLir/7Njja1kDLr60QUnkYYcq8YfESIhM3gWkPMnpefr5h8T/aaOEetZRZkEZE6cnbVMh4gBU93Bt3S8P595m43sFpDJy3NmcJ2CWkKce/Qq5Y+jXStwjWO9dUE1tF+ZfmTH5/x+Q8X/OZXPT/9yZpGeZ5sAs8/gXb1DevuijNhOMUdpUxYNwAWjyAVgVUTtgi6tnDTD+yT5W54T/CZpXtGzAUlW/anGqdozYb1IrPrE31fGE8DKRmm3hL6gf0+MIQDw1Hz6m6kpGWN8yTJL3870SjD+tzxm18PLNcTuwOEI8Rtg0AxjhnpEtMxcbFY8urXR7KVpIOlvSzstpEsJo492JVCXnvQBttmhvHA5tqx2Cha1bA7HBj2GtlEPv30nGGv2N7dokrHsx91aJc57g9cv2yQWrJ7FxEyMg6O0t5wuossziIf7jONyqhkcE5xugvIS8H6bIVxgrffHLl/NWDOBMtlgxUWuQiM94mr6xXWOaZe0jYnxBPH4SaQSmA8ONwCNueW20OPbQsEi9eS4AvTPpK9RJrEk+cLdh8kT7/YQKn/4dAysTuMKNcwyZH1qqUJZ7y7uUUmyakfSRs47AtajVxet+w/BEowjLfw/ZsT3VXg8F7SLBNnF5LdhxFhPJvccbwb0U1h81xzdt2yvPaMp8ymdMTYV9ZsDyJAZxvWncRkzcvPF0SfyRTGlabdKM7PWuIpo5T/Qz/Kj3rUo/6IpFRdavowIYWcm88NRrfY2bkrZaJp6o3+FCYkal4oOrp2VR3Nav/7tvdSUFnT6hbggcsIAqHTvExWWKcxZi79mYuJUowPrgIhEin/3hnxsfDI2A6l1LwQrovIj5FXJQ0lT0xTICXIRaFkLdEpGEBVs8JsRY4pM4WAQaB0dVxXN0+qi19j59LNhNGW4iDGusj2YSKXgJA8PBatJSEUQgxz3DIjlULpygYHTUoZqNgUqXq0iUihmfxETJGCmAs+Z5RJrnTJXGZ2fCnz8lzNz6+av66cCzHF7A6xGGPJuRBDjfXW51CjpJlRJIGsYy2fIhGj+muuHo+S9QJCUg8+oj4wRKnIE6urIwgBKdf4LiLX11YbtDIPDEJBRZSFOM6lVYFCxIwK5wRa2dlhVOZDYKaQZ7eLqDx0r2fnVOWQSyGQqg75kGcWvJjd4Wm+cCgIWd8/89lqdlJNDxdEai6+NcZhtJn5onbmu0eCTzVVMF8kpVgPTPVyI9eDDqIiexTEXLE8KRek1Gjj6mtBZgyeFD2ZeuHRNG6+VHBIAUrVx1edU38zYvuoRz3qUY961KP+/1c+9HWW07XI3Ag7F4DWpaeUgoKm6zKnw5HD8QAIFoslIRTGYZxL2+OMrBO4Zk3JFTNoZ2a4n+LMfxZMk0cIyWLRzOWMkZB6Cnqe4QPWNTh3htaG5XIFRXA4nhjHw4wZqTzycfQUClHVThnvE9MY6LoFXbsgBD9PU5WDTkmkENHKoo2m748MQ49rwAtYLFfklDFWYy1M00TbLoFTRXoYTfxrjO6P5afjdCT5iFaStm3JUfLyxUv+5Ccv0aVhuVgxDgPDcGIYBmDmwZcMBaQQs4N/YvJHYh7mdGLFJ5aSf9+hJC1GW6YwUERG5VzNSbnO2m3bzA7rgtLQto5PP/0C7RS//OUvGfppftym4jER5FIxMOv1CqUVQgq0ACESCMipnlXarkWrTI4TTZFkbdlPge12CxSWyyW//tVv+NM//YfEGKqzvm0rRgaJ4G8SCz4u0a1rePLkyWyQsfhQHf9d13B2vsH7iXGC84szfvijL/n8i5cMw4EYe7764UuePL1gsaoc/kYWXAMySSavCDFSRlFNTKl2Ul1cXDNNfb3QoaZ2LQqlJTFNxFQTANM0VXOTVDRtS4p5TiNQDUfTRIgBqRUxBsax7tAQ88VKKUhpZhMWD+dJpTRdV9FBUkpCqPgcpapJxxg9z/z/ef3BS/QU4HQKXF4JXv/ao1YRaQSXXxiQAe814xiRlwNhr1AakshkNMu14Xjq2d1CiZL3d/d0XUGGBCqy2wYuP5Uc+5H9IbI+N8QEx/vMs08b7vYTuxuPXlpWS4NsEk0CQQAMhw+ZfjshDLhlQuIIveLs3DBOBSMV3ng2a8txD1fXmX/+z1/Q7xLv3i5pXMuPftSxvfvAGAWffNbyw58MLO2Bz552uO5AJxuifMeqrFAmU9KClAsLu0GVng/7gfNOESgMo6S0BxoticOEMgv28Z6QV1gjGcKeFCHLe/rhgjKNiBa+f33g4Afu71f0fcCpBcecEDrx4Y1me1yQUmSY6jJg2Efu8sjvftVzcRm42Stct2J3nyk9KNuzPutoLzp2hyMEg/eRd68SMXk2TUtzmTmJgc5a+g+Gs3bBatOyH0f28cTd/YmzM83oPUySUE5szlvCWFhewDBG3n438vyLJSo79seJ4DOf/rQjioHDjcU2EWwhnQwfvhtYLBQ+BlTSGCuQIqOOLc0ykpNgu9txdzPSHyJ6adEqcfWi5XQbeP96YuHOuHqy4HZ3S1lG0uRQzpD3iuW5R6hETpa3u3sO28LxPrLZGM7XDRu34dRvSSjCrjC1C+TiyFlcsrvb0Sw0T750DHvF3W7HctliraFdZbjPNEvFjT+waFrETcP6qcKtQY4N6X1ES0uIRzquCWZCyYldf2J/B9OxcGpPnL8wNEtNMxqylwy7DN3I4T4znSJdI3j27JrdsKVMGqUCOSZuv54Y40DRAbvU7IY7zvUSp+Du+LhEf9SjHvU3JUSu5ZXU5bkUui4+taOOt2oeUBq6hWN/3DP2ldssUDTNAucago9odSLrCCkiVIdxEmMdMVZHTCkZHzwgZgd8T5oZ0KVUvnZMAR8SSltCDDPKQ2KtxhqHa1qMbZCqFuKUkpCiYjI+xh9TKgSfAA1FI5VDqxYlDAiFoKBkmRf1ghgSUiaKkCRAaIN29QLB2AZlLIWEUAZlHTFEcqjFNzXuamhah3U10lpKmMtQa0O9c03Ff6AIITOOgTqda1I+oo1Fm+puNlZTsiPEunIOOZFDRimNyIUiJMp4TNNUBAq1cFRrjbWGXAwVc8JfY4lnUq7LfkRGzpcRH5fwOUtSriWbHwt5UorkFBDUBXXl1Nfh1LmGQsb7CWMsTdOhjSWX+mM5R7SpS+mP7yUpJUqVegAItfw1hkDK9SAihUI6Pa+Mqwsq5VRLTKWYL1nEX3Pgq4fDjVJ1kQ75geWeUq6FWDETfEAZKAhSSgzjwHA6Qqkc/pwTSUis0TMeqF5exDlGOo0B71P9bNiP/Ec9F7PW57aUhNIGrSuCx+jZiS8lUmoEul4kCFExMjkS0vQQH/2ommJITJPHT74WVj3qUY961KMe9ag/HolMTJGUxXzJb35vJDGmLkJ9mlnhcnYVF47HIxSBcy3WGXIOnE5HvJ8Z6VmQooTZYGJMXTgfj0eESCyXS9q2mWfHgZxHlOtQWs98a41rHCVDTfspJPKBvz4MnqZxIMqcHoVCpORICCM5O4x2LNpzVqsVr159x/v7WxaLFilhsWiBzH5/qMXxSAq/79Wp7PaaLJTSoJRD62ZGzVQUX01F1uclxpq0HPoeq1sWjeP6+ppNd0meBErImc/eUkridDpWc4OrZ5dpSggEYRqIqa90ADJFJISEFApKufpY5GxcmfuCUqpnn2kcZ2OExojhI3WErmv5+3/+ZyxWHSlF/rdf/JKc5MPyOpcyY0o810+u2KwvUKInF0HK1IuInFksVzTNkpI8IdYLkxQyKWXatsXYjsPB88tf/Yp3797y2edfkZKnEJFSkZNHPqSN/6batsUYQ9t21TE/VCTKkydP6LqW3X7LuV7zs7/zY/7+3/85iMRvf/eOl58+5bPPnyBVYpx2aJtQOpDKhNZgrK7M9pCr6cY2NE1L2zpi6jged4zjQMoRo908H9ezChSstbRtS9euEcB+v68GK6Uq5z5HKAWjNUKBkBWlY62upbQ5UigMw4SUiuVijTaKmCRaa4ypZiNjdH1PUxftFUn5tydF/+Al+pA8fYF2dJxdLLnZ77h47rg6b7nfRtpOsble82F3wLhEsxCIbJgOguaiMPWGvAXRwpQDoS2QBpSQRFu/SRy+F6S9YegCzkiyV3z36x6QLM4Dqji++8WBT3/uSDmibWLZGY47weasgQQyK2Q/8uSsYywT4RhRR80/+rtP+It//JRv3u357Omazz8JjENG5St007NpLwjT50T1C6xpud1K2kXGLweGvCV6jbUZbT4AkpAGjuGexgXIBd8/5VAONMuMMeCnlvsUeLfdc3bW4sclH241Vh7RYgXZgpCkpPHJs9uOhKPmMK443XW0Z4av34yMpxO72wm3KvRecNrC1bVjGg/cvAlcXVucNAw+kg6Kso7EKBC94OnnEi01x32m7yP9ceKTT884HgfON45hGxA21+i8NGhZaC8yh/2Ry5dL/u2/vaNbCjZXDVMv8cFzSgNPnhn6bcbnSEJy/kxyGCZaIxAh4zZwOGTGoZCmxMkHmEDmyhpNeJKXpCS4fT/x4ssV98ee5bVmTAdOHzQKizRj5YFlTwiB0U8I4bFLyS9/ucU6zX4fCDGwWjYsVgnjNMMtsBT4oSCLZ2EbXjxd8vbNCb3ZU4pi2I8QBeaHkff7LWlvWZiG1mm6DQzHE599dcGH70fW54JGdTz9wQkRNC+endOtNZvNSAG2dwNKSjaXDqTgGCKj2PHJ58/4/s0b0ij44d9dcnyfGE4D/ZjJx4iQga5t2d1POC853I2sVmschnCM9LtAERNpkCATIz2XL5YcDnvGMTB5ydvfBlIsjPtHruqjHvWo/7Ryziip0DNyRKu6QB7HgFZqXlRqjKsXnNHPxZS5kFJFl8SHpvY66Cqp0UpijMU598ArTCkQkySECR8Gck5zEVJ1V4vEjIOpTu9alOlQqpZkKl1LNj/GTUNI5BIfuIcp5Zm7WAdtpR1KVWZgRtQoqawDuHUGqG6OAsRcnctQSzmNruzrivAoM8M6oaQmSQFM5CJxzuCcxVqDmp3Q1Q0ukcwLbtOSs5hjqIk8FxHFBA0CpT9yHy05VdzIx+LL+pjqQjklTYhhLo0Cq+oCVimF0gpbLDnHOcpbI54U8D48sC+1CjjRPZTqhFD5giFU93yKFSvyETcCUHImhPjXXPoKSr2wV6pGg4WsS+0Y/cOvg1q+WYqcL2gKSqp5cfyxFHXAaIsQGlFgmjLe1z+DVJGa9i0PjPDqxJKo+evVItOAD4G+72cniiTETPC+XuLEBONITInJj0Q/ocRHLI5Ca6AIUqqu/0GOKBXnNEBGKcty0VRGpKgczxDC/F4dCSFiSkIFQU5g58OplLpeVggIKYGs7E8hmS8qIiXlmgjImRgSPkx4Hwje87GU9VGPetSjHvWoR/1xyLnarYOo3S8hVqxtmJfCpVSki5zThSnOnTkztk4mP7tqFV23mPngGusUOX9MulVXtDHqgR/eNK4u3EtCKdBKEvxQ503bkJJnHATBJ/rTiJjLHkuus82p76HkGb1R17I5RZSWOGuYppE7v+Pp9YaXz7/C6g4lv8aHAaMLjW3phyMlCVrXYU1DzILJTyhZe3K8D5QCOZUZv7ecH7/FKMU0TfhpRKlqTFBaMY2BmELFe+SKCXSNo6RETNWQU9njmpgqnk9KVYvoc2IKPUImXGsQUVXDjRCkFHDNkmkoTGFisViQiyakzOQ94zBUY4lWFYFIQYn6un3yyQs+/fQlykh+8IMv+d1vv+Z0CqQckdJQYkYCwzChlKJpOrLI5FTn4RihpMJKWFKEYYjkrOo5CU3bGYY+knzFVN5+uOUXv/grfvqzP2GxWhLihLNNLQj6DzQXjub63rPWMo5j5difJKt1S9O0fPrppwxDj7aZy8vP+fmf/pTVpuF++571WcPl1YamK/gw4NOAFppCYPJpNtoUnHOklIkho0xFN6ZUGIaRafIMw1iRlaUy+JvG1fd/NmxWa0oGJSXH45G+79HK4IPHOYtzDTF6tJIop2iTI6VYy0FFeXhd6h+/pnFTEnMyg4fzRtt2lFI4HA5z34Bkmv4LLtG3u4J0ArksWCO4lA2ukdzf3XN9ueJDGRjLSPARtxTYrqCz4cw6ihyxF5nTG4VoYDhIjIa2Pef2ZlvZNMsFUfZcfCk4jpCKIpvC2RrcueTmu4DaRpqFQLWZOGTu30rME8UnLy5JjFytHS+vz/j6+3f8xZ+vGaYByYoiMp++DPz4c8HZVWEK71Gp49Pzpyh7Q9HvCPl7MOccp7c08oIn518ylm9R5UCnVhz7nnXrGf1ELpeU9B1SOmLONPqAsx0hj9x+GBinwto1KD1A6fj6uxPPLhXd7Bgeww4l4OvfZaYSaLTmeFJoGUlJMRzu6e8i969MPZCWlm9/O5BswQg43o8oa9htC5uV49R7wi6Tc2CKGdPUmDW64Rd/dcdnn2yYUuLJ54bjtCUhQSUuXjiOQ3XaTYcRswjcvOu56Dac3meef9oxHj3DqSJw9q8L158KXn97YrNZIqLAjwWlJXhBn08oCWKyvP5dz6ZrePLDDruz3H7v2YcTT643JAnNSuKPnjwVbu96vEo8Oztn/2Hg+CHSbSTozMvPLTevI8GPpJzwOWOOmX6XMVcCv8uITuAaTUoBLRum2IMfOGwj158K3h8Dx21AusDtrmfVnfHpjzYsVguyPbIRK3673ZFi5niA52rN6twQT/Dk/BIRJvbHE8e7jJSel5+s+XDTs9gI7ndHbLygW0uGNIIsOCcxjSCIHmUL/S6xPx24eLnBHI988smKb78b6D/ANO5wjcX3nuWmpeRI8nB7iqjiQMP+w0DShdx65FHS3yfOrhc8/bzhw82J3b5HhD/4o/yoRz3qj0iLxZJSwNmuLqmlxdoGKTQ5n2YERo0UGtPSNhE/JEopDMPA4bDHOUs/nhj9EUEiMzvIY3X5Ki3RpkZS65JSUISgUNNmUmhUqcy6XOqQFePHZnRVF9qmoscogpgSijrgxFhjkq6pLoqcPi7QK7KkcS1Gt/OldK6OFgFKS4w1lCJJpbrQS5TkmChJIIvCakfrugeHjxeekiaEzKAlMisEGm012mmUkZQUZ4eKQAgDWUJSxFCd1CkKgg+zuwg0AuFAz4tlUagOdqCIuvQXCGQRaOEwqkELQ44FH3wtrEy1O0YpQS4zqk9U5njOhVJqfDelhBAaisKaj7zNepGQs6hszFgX/EJptE4zNiaDCGQCMSSm6YSUlq5dIUT9cyHr4D5nJCt2hkJMkRw8SiWcdQ+ucq00UUhShhjqElkKD2T8zKnPuSBFQgqwBZhZl6Ukcp7d/CLWYqYxkXPleTrboHUdxMcxklIhxurir48XRFH4MFbuqHFQJClOGAPMC/JSaqxTaYW2hsZUpFHOiUJBZioHNCdC8BUdIyGJgmzrRY8xBmkNSEkqBVkkSjqk8JTc1+6AGAgxkoBUKgompUwuoNXjf7sf9ahHPepRj/pjkjWKnCQxhlqSaQTGVAxIKRMpB6oJJGCMJBCRsrBcNTOOpBotjGlo2xYpl6SUyHlAqYyQYIxi8gMKaNraSdO0hmlM5CiZJlguW/rxUBncjaLvT5Tc19lXZ3IWpFiwTlaDh/cUMotVV80DOeD7SErgXMc0eFI6YfZ3HMc9Z9drbu4N03bPFCZSP6MzjAWlqvlAZPzUA5UDL6Qkp0SMUzV9iGqkULKQ4gltKsN9fxpQCpQ21XUtdDUGFao7OWaSKAiZESqTieQCucj6teNAUYUoRsZyzxh3SBPRtoGcKxZESiZ/YvanM4WKb/HjxDQOGCWIJeG0pLWKkgLOKH705Zf8z//TP2e1WJJLZLNaYrVglNU8QikU4ZG6MHnw0RPSQMw7hKiIQuua2s2TwIeJ0U+Ypl6OHPcD+/6I6xa1h+f9gJCOf/Ev/iX//H/3v+f6+pocI8KCkrU7qS7PP5ouJUJWE4yUihefvuTsagWmIn9evHhO01q228QGwQ9++JJ2EzmMr8BOnF1LTNszpB3KFgyZMUW0FmTp8LEwxfH3PUpGgYjkEpjGiWEYSbkaW7Rp0MbSNCsQAm0iZi6XHceR7eFQzT1aUEQGCcYZtNHgAZUJaSKmoRa+4jBGM40epTTO1X8+nU4sFsva35QhxFTTxRliLEyTRyvDYrnEufZv/Qz/wdP7qm3Y39bYq9UCnwL+PjMNhfX6RAiB6CXdSnDx1DAOnuM2M+YBJTxNWzh7qdFW0B81u5uMvjyxuCgsNLil56I4Ll60XI6ObX9LexlZtpm7m8xnP1hzOEYQcL20ZN3yV7/pwWX+8V9ccnO84S/+0Qs6XTj7d9f87Mdr2qahMR3HQeG6e1Dfc7409MeA0pL7fmIh6gdCqDs6bUj9NQf1Fh8+4OQKrRP9tCUbwWEqLLvMsR8pcc29PyC1J4yFyb+iJI0xiX7QHA+ZIiIhThiRORwk3746susVd9uRcSi8/lry8hPLwQS+/X4kh0JnDZqWvrf4U0aakcZonnzS8f72xNDXBYAmQ1Ds7yPn5x1v3h/RttreTFMPp6++i4RY2PUnfAzcvasL2OurFt+DMZmxj7SLwuGYsCYz3ml6eoxWjGPET4Xj3vPZDy/4UA68/XqkXVtO9wFpDLu3PU9+lBhCxCbLsl0jQ+HiqiBk5NXrOwSK7szCbcv9dsvzL5aMw4hbFT778Yp//5c7usbS9yf6k+TJV4Lf/eaORmv2hx6VGopfEcMWkRTr1YLxRWS1UkhtiRJkO+HvBSkI1ueOIhKL88hysUY9Fxx3IzEVVusNV5drdv0Hvn974vKThikfOL+WLDvN8X1C6owUmiEkDuEGHSIlGb768Uv+9f/zW86WENPE/daTUqZZSJpG8eGmpz9Erq/PUDbiFoFnzQVl7NEl8/rtjouNIaaJd28mXnzecngX2e8Hrp425KRIaYBFZHPeQobhWFhly+XqnNvwgc62iKbluA8UWYvk1l3LPjxGwh/1qEf9J/SRHWgaFt0a57q6QC8Fbe0DskTp6hw2yqHUyDiOTH6q7D8ZGMOekE/k7EEkJJowRUKZkLI6WpKWNeHjOqxrEFqgHBhlyEGCLAhViLGWP0pZE0k+ULEYouJHhMxkUR3vlR2YiCFjTB2QrSkIFFJorDa0TYNUinHy+BDJJZJKYYozJ1sKCoIcoQQ1874LEoWRBqMrfkTkQlKBKeTqfrcGpSVCq+oZl/mhFDRnQUmOGApkQU7zAjhJSpaQC0JkVMmInKmbYMjJQ/KUEiFDLrWctUiBVRaZLSVIpNJAYRo9uUSkBCMqVz6LjJCJnCClzDQGQqxLWaUdKjjMNNGgAUXMkpgEPhTGKeJDpKQyJxMKRQSUCVgRETITpjwzJyVSyIrIEZIiKspEaEmYXT+VowlWJmTOiJnv7kxLSRkxM/RTzERR/9zV8R9IMSNLJTUaaTBKEmRBkBAiEVOmUBnikw+1zNX3dO2Sxi3IWRJjIfgCRaGNQUtbWemlvkbMbqY4u5A+usuZnfht26DdEqkVRc1piVKq2z2kB6e4RBJ9YpqOJFFQRdO1nsVCoYwiC+plkaoYGJEDJRpCLEyhXjr5EAmloEwtndVS4Zz7r/0d4VGPetSjHvWoR/03lHMWrQVhTvY5V3t3csgoLZBKcjr1pBTRqqFpXC1GNJUvbYxhHCe8HyB4BPXHlsuWk0yzQ9fQNIZcPG1b04TdwqEUHPcTSjXEKMhF03ZtnTsFZDLGSGIK5AwZhVAGiUPqhiJqH1FOET8NWNsw9J5cNFJatDScxgN/+e/+FTF6+mFXV9AlkaiO+eVmRYyZaRpxtvbD1CSkoGnq10ippicrAtAjZYIyIrQmExGi9tIcjgMxFcboyWNfjaRzYlGQUQqkqk7kIkSdaZWizI7/WCb2p/ccxxuaVqENxJSROdUOI+KM35k4HANamNqbkyPSKDS5lrNOA1Z3iAL/6B/8A7787AsQku3+lo9O6GpqCWitaduWKWwB2G73wBU+DEglSDHjXIOQkmGsbHDTOKZpImdP0xmePb9iGg3T1M/PkeDNmzf8v/7Vv+Lli+dcXV6SYkKqMicT/mNqQU2vSin4/IvP+cGPv+T2znJ5dcFy2fHhw1tebtZIFVHuxGnaYh0sO03fn/A5EssBA3if6PseZy0LlnXWjjNP3lR8ip8GTgN470HUdIT3YeaWa2KsxpwYArkxZB8ZhoGSS0XONIbj4Yg0EmU1QkmMcpQycjqd8GFCazsnEWoqeRwnmqZDKgGhEIKncQuGYXjA6ngf62fFLSom6aOp62/RH7xEDyOorLn9fuTLHzvuCTx94uhPieOpuqSEikiluP/gsU5wfztiF/DkurC/VfUG6CQIPYQ+Uy4T0ykTIogwoqzkeCx8+GaLaRTnVy13bw9cfwKH257TbeLP/nzDP/z7iRIvuXQt//1/v+IHn18i3YKJHWfG87OXP2Glf8ou/V/ZuG8ZNx3vjgqUZmUz5+ZLxjLShzWptAzhW7KIFPeWTXPN22ONsuP29JNn2Z7THxP3YyYpQRpGjJH0p54kJW9vPMvOsD0VzpeGw05iTOE0Jfa7wje/CXz11YJ//e8mdFFIlQhRcNhP/Pa3I1lJth8Ezmq6Z0u+e73jydMlTdL0J0N3tubueIcMDYtFQi8DYYg0yRDkHpktaRJkD6NIZOD6BQzbyOefWW5vEnECbeHyaUe3Mhw+BIiFxUbTnxKNg+gly3PD4jzS745IVWiW0C4Vu92BZz9qSEGxXi6JIaONYHU2IjQk39HvJhKZIewJMbG4gk+en0O23N7vye2ISjD5HlMW2Osdh+HE2cqyXFu0TWgRefObiTQVhIX728h1s+Cwm4gl8eIrR8g9JSROx8L+VrA4UygrsY2i3XiGPnJ3E3CN5ObdPRdXS9xoQE4M4Z5Xt1vacgYnw+k9lNziFh4lYdgHvh/f8/KzC158afndNz2iVMfgsd/z+U9WnG0K++8KJqxo1hPH8QPHd4b7NwFrBNM4olNhfw9ZBGLObF9D+1SQg+D9G8/5akG6t8gi+fRLxf2pp98lFkvJ+kKzuzvhrODDLpInaJeakiv/9bMvLvnd7fcc9pGQA1/+YIk2zR/6UX7Uox71RyQ5Y1ysbbDWzSWOBq31jKrwddmrK/NQSkU7Ok6nI+N4YpwWTD4T4ogPnpw9SoGQhpQrriPEOCM2auxUyI/8xFRdHwSEsFirENISQiGlSClxLsVMUDxKzqgVq9Fz4cvHQSjnNJcuQSkOJSuDurUNTesq/qUkQvQPRUAxzixJoSnkmZcOzKxta8xcGFrLg6SUFUWiFKIolBYYazG6kFNmHCdyKMR5MKRAitUhE0OaneIFcuFjZaRCoKVEz6WXuYi61A3V/aFkdWunGClNQUkxF2LWiGURcXZGh/pXqcv6/wDFQl36xpSIOaPmYdBoN8dXDSHU8s+UCiWDFBIlQcpCEfV1V6oWa4qZNQ/gg0f4CW0cWkIdwvNcUuofOJw51/SCkrW8VQiJUhqZYn0fpEwUFSNT/93q67HGYbSujuz5cVQeuiKVTE7VEFQ7fiI5DTOWpUAxpFgLk5SyqBlBo5Uhl1iHYAGgADEz9Ut11lNjnDEmYoqonEDIeihLaf5c1NKp+p6rseEQIiEFUhfmYqpESZFEvXgQ1MsficAoQxES72M9cJWPn0c7P8aMq9b4Rz3qUY961KMe9UeinAPaKBCaGD3TNNEtOpqm4XQ6EmNgGj3GWNqmQ2tDnVt4mAOdM4TQU3LFwRgrH7CK3meEkxij8KH28YzjwP6wI0dFiGXGj2ScczRNO+MvDIuFIyU47Hu6bsmiW6OUYRxPlCzQesbCBE+YetpmTfQ9Ri9wtq1zbsnc3n5g8iNtazBG4ZqWnDPjOJLiWNObKaBVQ53ReEgIplgIvqYWpVSzYSORYo9tWopQWGsYp6kWqdoFIlpkVjjXUVecAvg9ykQg68yLojAHK8n4MBLCQEyV6S5EfV7quSbPaVXmrhxIH3GMIlXTRgoY5dDSkJPk4vyav/unf8ZqcY6Pkf3xDmPUzHKv55ycMtY5hrG+Rm9ev2cav8AYh7GaYziSc+2y6k8DQsTKY08ghKJbLChRMA0Tp+OJYRiJMWKM4S//8t/wT//pP60XIAWU0FDm50JAdaSLeVYXCCFwjWWxaEllTbcopLxjivdcn59jneV4ukepQkYSswI54f2IVCCFgZIfvkbJEHx8QExWDGecMS59Rce4Ft20GKNRqr6nCqmeC0skpkJOtSC15II2Eo2a8Z6KXAJ+TPMLG4khPczW3gestXPvUaiXEQK6ztXPxcw9ryjSMnPRDU3TIISsCNP4X7BYVLY9oReUJPBTxsfEh7uAQjKNhvv3gaaD8S5hW4lbKkRSxDFyuBO8/0VBnhW6KygKrj5z+BAZxkyK0CwLMWT84USMCv99YToGrr8QXJwb/sc//Yp/8b/+jh9+kfjTny0Yesl/98MrTHcL5cRiCSYLWv0emTSNWXOfb3g7JMbQct4ZnPL08cTAN0zJ4tMAaIz17IcRpGSjBhpxztvjLSnVUrXBjxT/DKMTr26+ZpE1jcns7w1mISgp8urNxHKhePMOhj7y7nXkOCmev4x4Mr/8TQ8xcX87EmOmW3eknBlLIY71hiz6+k2wpIxWidEEosy8uzlCk/jk8zXffbfDNpCC4PAhoUymu7JoPKXJrPSKJ88TQo24tuHVvy3Y88LmUtLYyowaRsGYB4wEOTjO9BUH7pBO8P6u56zNqCxJQdA4TRws+7tIWIysVkuUgf22J+aMbSxWGZyTtJ8kDof3WOc4E2uy2vP+ZiDHwHffb7l8Jjm9LrRry8EfaU3BpXPOnw0Y3/Lu+x0hZ9plg94oWq1ZrdccjwMXzwSHrSZ4ePO7iTAkLp9awslTVoE3v008/awji8T2nWK/9ZxfW4zq6NQV8eKe/TvQ0pF9JljP9/cfOJMN66cCqxR370eELJTg6LeJyyvDFEaKEqQw4FMEJfjt13s6vaSIxGZ9xevjG55/qklYrl8syCfHze09xQhO2wmrDKiJ432hvdbkwbB0hv5DQipD8oncK1zvuLsdcG1BFI0UjidnBpU0apPYvj0SS0+wLWPvSamgNdzcD5y26Q/9KD/qUY/6I5IQNbInRC1hyVNByoDWllrcWZBzI7kQzAOHwTlDTIG+ryU8iFjdxDFRTGXMiVTwITH5aeYrGqSsQ/HkqW3sUlYnRK7oEK1FLbpMzMvrahgpORHnYU5qXYsohaCIOuinLOeGdUEtO4qUIrDOYowEIdGhDmExBESSGAtCOqSov3+hxllLyWit0bpiXKob4SPku34NrTXKKpzTCBGJccRP0+xS5mERHOJYB2hdKGYeqebf8+PvWBerYv57gZIGQcJog3Mt3kf85BnGfn489UJAqkIpkZQDuXhyqUVBEOvzP5dZamUoRpJyDyWR4kSMEzkFUAYB1cGvHY1tiVJB9ohSXTGUGg+FgpKQZSEniCmgYyCEEe8NSIMQuT6XOc3L8YifHScpZqy26JkTrrWdOfZ+5u7X8ixTFKVIci4YqbFWz+WmAiE0UlggIGc0DwUoqSJbckKIgKCWouZcF9dSFLIsSErlghpHLtVFJYWgUB6W6HJG1NSIMJRUuZMxRooU9f/zR1RO/oiPnDfpeX49K5Yn54gPA/AxSZHn5X1AazEXINXDoVEWocEaXR1nsV4MPepRj3rUox71qD8epezRwgF1lpwmT4gerevyt2kWdX6RiqapaInKe9ZM08jxtJ9NEgKlNKfTQIg9IUYK0LQOaxQxBnKOdVHvR0SJlGwpRVOoZhJj65JxmjxCKELIjMOE957lUpFLJozjXMBusaZiIUvRKJkp2SClw5qWECorvFu0xFhLG733pCRJ89w8DAMxVKe1tR8LL0VF9oWK/IuxdgZ5PycTY8IYwTQmtBFzx5HgdJrougVts0DlNefdS6xtK24xFYRID4vij0WppQhSyfMMFxiGA8pkdM4onVFKEtPcDZVqH1QM1ZUuZSHmsRapxmoqyrGQdEZrw2E38LOf/X2ePn0GuSOfjrRtw2KxYLNZs9v1dRmfJTlJlGrRWvH29S3ff/+OT75o5vNZ3dl1XS26Px56lIpUs5JBKk3wtWvHh8AwTBhTn8tvvvmGV69e88Mf/gSjW3KicsIfBuqq+lwUfBy5uDjj8uKc3eE7jsMBCCw3haaLNJ0hMXP8fc92e2SxbBGy9idp7bBWPqQ/T8eBEANGWwCm0VMKrFar2uXkx2q6UQKpQBuBNooUEyl5YgwIKVEKrFXzazAyDDWRqrUBkZjGsfZA6dpPtFyuSDEy9B4x/7qcE31/ZBwHum5BzrX3KKaA/XjRImpHV4weH6qZ5w/RH7xEP25hsS4EVfDhgNSFHKDvC4hA21n6Q0R2GRz020igkPeK4buCe1Jo1mBXBas0ShXksSVuJEM/YRYC34PO8OwzAUHw3/38isXZiZ/97IynzXNWNvDznwq06Nk4i3bv2Z1KjfoGQ0wjk8rEvOGb078lyEDOLUXsaM2JXE4cRl+/cWmHVJf08RYnFzTdSH+S9NkzhhFnF5RiSOmeSOTD9j1KJvrJUNSGlAzB7/nVN54UDJunnqHXkBxvXh/4+uuMTIbTcWSMAqHgeJcZb0o9r54lNs9WvH2/R4nCcJAk4VEfEqurhNADuUiiSPhQKL3nV7e3FGAtCl1nGCl0l4Lzy443r0auXjqcUOzvRtzKYGykWRekKthly+k+IZRguerYXDT42DPeJT7cvsGdS3LQnN5l3inB5sqwXjScthkfM8oGnn5miFPkzfeezz67ZDid2E+ZEgTPXnR8/fWe9bplfwpMww6VEtdPEoe7wqITHG7AWl2/AXYKkS3tynP33rNYCM7OFX4w7E6B8ZhYvMi8eT1hz2DYO1TseP3tjuWyYbm5Ynva8uLLNbthy+VLy+k0YkdB1oXlcsHTywu2+y2y8UyHE9oaLs/WvP1wSzpOnF00LFcNaQfuqcYYz0HsuHix4e67gX4orNcL2oXk3ZsDfpxwrcJZx7SNTAxs/73HNJLdfU8Q8M2be5rgMIvq4n/67Am7+yNpErx+f2C/G9lcgZaW9qnk7mZieB2QrcRdwtJc0LnKR4tkRBLoVnJ/2NJsHFGN/Prr9yjVcHkp2B8nwkmQ1WOx6KMe9ai/qWnys2tXEEKmFFlv9efCSKUUbdtgTC3QHIYeRK7cxMkzDD3O2RnELQFNSYKYKqIkhoT31SEiUNVhTqox0CKQqWCkxemIc01lcCvxsED/6IQos0MbMbuqY6qHiSIoojpTPi4clWJ2NuTZTS9mtl+ocdHsIQmkElhr0KoOTZQa/0spo2ub5VweU5erKdeSJ6Uk0tTooNGSlKfayxHmy8pSG9yVhkKcizbFzEkXDwtz4KG4MqU6mINEKYe1AmstxjiEqFiU4/HA8XikaSrjspSA0ppSqls6xLFGWlWZ3Ruz61mClNWdlFMkCEkMIyWHeVEukAisqQz4sdQDSMmRXOp/O5Q0dRlfQMnq4heIygQvgeB7wFBny4QgIykIqvM+50ikMt+103OCoDr7hTAzK71ejlRLu5jZhaXGLCnkVOrCOUtyUoBFzs4ZgUdKP+NhZC0ylRopJCkWoogIUkXwKImSllSah7LSPDviAZjdMQjQsSJ6hJwqz19St+pzyiKGmgSAjxcgEmn+3+z9WY9l2ZadiX2r293prHVzD4/uxu14b2YyyWJXLClZSlIs8EESIOrv6XfoWYCgBxVUVGXHZOZto3EPd3MzO+1uVq+Htc3uJQgwgy9FgGETcIR7uNux0+xjZ64xx/yGLCxGcmn2BfPrWtz0AklMASEiUigqXZFkGdooUVj95TPezWiZ53qu53qu53qu5/relHh0QHsqU1E3FSllmrom5Ux/OhFCYrVczdtzzCHuCW0UYZqwtrhpU/ZkApOdCrYFUcwwVU0/eDKR3e4ByNTVkhADOZUtO63r2b1eBFolNcMwEkKirtuSjzSMgJj52WI2sxQ3dUoZa+1Tz+fcyDCcMLVEaTH32oqq0kx2RIjCyFYyFXNP9NRVVTZhRXFre1/uSzknZJz1hBhpmhWr1RlCSYZppBIGskIpQ0oSYqapF0iKgP47J3r5JUQxFCEEKUYygZgskz0RwogLPWLyZGq8L1utUmmcsyVQvjLF9e8DIU6E6Gh1hZx7++AFSrZ8/tmP6bol3hbDUbsogZkvX77k/fst3heHeYwZrSpSchjd4l2kbRakmDC6xvs4G0uKSakgWzJ2CkgG/BQYR0+MCWcdgqb83kX+5m/+hj/53/+PSKVn2XwW0LN8DFYqOUpSYihombopm70+PyCERyqY7AGhamKyZOrZMZ7KWRGB0S1SVIWpbwTWWk6nE13XYkyDUo9boGl+/sXs7E/zUEiSUiAlz2THghK1jjqqGSkU5/wkUYJj7YCJZj5/xfnsJlCqJkVJSpK6bjCmYr3eYG3Bk0LJW1LScDodi6s/JaIoNBXnbDGEpVSei++AWvzuIvqXsPkDhTCJcUjILPG+PCjvy+uRcsYYwcVlzaB9QZbsA/pFORgFl2gS7PeRtktUUtCuYLnUhBE++0KwrgVZRsacePW64uPrH7Cs33C+Hvmf/sWCKN7ipwtWZ4okLlg3H7Eb7+ntOxrZQk4ktSJhEPlLqmogJ89+qkkiE8nUVXEyuXCAXHHqA1lrECseRovOy3mt48RhEEiZWNYTH7YKIVr2/T1GdGz3E9t+YtlpfvNbzTQ6Gu2YBkVXl8nK7YPGucKCHT8o/C7w4ocrTlvHh18eiCrSNIrsBGcvuiKeJ81hD9M48ekPG27fwuEY0WeCaDPbbUZFgVyBrgW77ZGrTzLNYsS7wHhyxFMu05tK4nLm4VcTN5fnIAMPDwd+/OMXvP32wH4XqJeKi+slHosNmmgzicC7LwfcGPnsR5c83Ef2O08MB5bdOYd+j1QeLTU+ZfphJItMXQvEVnD5UWZ73/L1bycu1wsWywVeTTTL4vh+sYkk33H7bk8SGXdn0UYSvUJXnkUuEyQkfPxyzfahBxNZiCVKSJTOnJ01eCamU+RHPz7jw8OR5GpQnsVCsFo33B8sX//2nv3Jo2Im9Af6Y+azH1zx4XjHeIp8/PEF++EBl45EETn1I1FpZBXRUYEXRJ8LhuViwe62sMiXZ4ZPfnzF8dAT3ETyI5vVOdvhwE29wfaRaE/c3Z5YtIa4Uwwp0a0SjV7w1ZdvWHUNuUqIrIgycH0DKYAba/bjieOwJ+8y3TUkK3G9JLvM5WeGdNKMx4H1Zc2L15ff9a38XM/1XN+jCmF2kPuAlFPBeeSEUhpjDFVVkXKYm6qItSVp/jHIx7mC7yhhkrk4JPKMYEmzkB1iQYKoIirGnFFJlMbNR6SYCCaSUiiIkpzn24vkVNwRxZGeiCnNpnBB17VorRGUkCCl5Py1AiFK0z9MAzpopJRl/S4FRE4kIHhLNArdNtRVDVWLQD+J3gDjOOC9Q6oikha3vEaZqjD3RCRPAZKCLFFSFhd4LYjOMCpBFCBRaGn+o+GE956cIPpEMqkIwlmRU0SrugjKWRShtaoZhoFpGvB+JCWPUuunVPkYA9M04fwwr9I21LWeHfUaIRIL0eGcnVE2xXUkpcEYgxCw6JZUVUV/OuCsxFmBdRMxzEcMJZFSYBpFShnnAj4MiOxBRBIK+bh2mePT0ELK8hikFDNqJc1YlETZXC2HFyXVPAwRJaA2J3KYEKLw3GMsnHcpa+qqDHseD1HkCRHLNkIR5cttIRUpJbyPSFnWbo0xZfDiPZH8xNXMzMxzEeeVWIkUgcSEdY6qVhilips/RmIIOBdIIc4DBYFCIRTIXBgzKbgSXFWkc+LjeyWU5j6nXAYllH5ZSlkwQlIyhoyz7n/TnwfP9VzP9VzP9VzP9V+3vLd4X5y1FxdXrNdrTscTSmnOVqunsMPHnjLnTMpF7BOi4P68d0gpsLbkzRSjQOZ0OhED1FVDyX8BH3zpn7wlRc2iWwHF6DGOJZCxadrZ+OAQIhdsRkoYUyNlpu8HUgTnEyE6TqcTznlCsChlSlAnnpgcMXqqyjw54adpwFTmCbcoVRHxQwh454EirktZBOmCT5HIGQ8ZQyw9oVacTiPH48DFZYsxdTlHZMk0OpyPIPS8Ycn8GEuHVrr8gvMr2BBPTBPWHRmnI+PUl752NkhIqYjW45xHK4MUmkBAm5qcPTE9ImEEKUq0qHn96odcX36EkUuOwWKMwngNIrPZbGiahmmaECRCjCXwNGeur6+5uLikaTp2uy05g/eB/lQC6pWqMKbCzfdnGh12cFibSw8vzczxliip+fM//0vu7x+omyVN84j8FfPzUX5fhiGZnKFrOjabzdwXS3wITP1IzpGl6+j7E4tFy2LR0dQL7u7uCv4kR7wbZoe4IniwU2S1qmdHt8C5MiwyZiohulU1G6ger7+hOOqdLWfI6JgsWFcGNI+YmqqqylBGlDNkORPGsm2aBX0/IKVktVojBNzcvGS7feDDh1u89xxPeyqzwNnEYrGYsaP56Tz4aHp6zEz6u+o7i+jta3j9Q8FXvwVpYdolTAftGai9ZDwEulXm+lOFkp5+ikQnOP9CEKuEDKCk5vAOlIblR3B+LiAIfvSF5mLdUdcJlSVNu2EMB15dGhbqxNnqnKp2CL0nuIDSht5PVOqMVq/R5kSONVM40uTMFP49KXzE17eez146Gl3j/AKbjkQSJz8xjJHTMNBVF7h0JIyOftgRUdTigFblAjrtGpJo+LL3HI+SxdLhnKffb7n/kBi94FA5+oNgGDxXL0pgpzaGMToCkpQTnanIy5HrLwS1Eex/E5FWsHhZk4k015KblzWT95ydt9y+m1AV3H0IbO8S9ToRs0CvJNMpY4RHy4rdQySJgVpJzCqxPwiImt3dxPVLTb+LjIfEYmUY84mrTUNXw/v7t/SnzOJKUbXg4ohUmctXkhzBDoGm1XRnktM4lvCAXXG87fodmyuNzJLkI7e3B65fKrTWbD94ll3DsT+QpQDlwQzoWNN+DNvbES0VIklO9wnZCJzLqEBxAaZEQrA5X5O1R21s4dHfR0xjuXq55O0vT2x+6ji/7PjlXwVMpfjw3nLYBVQD2kiG2HOwO0yjufvGcfPyiuHW45iomkSue85bwf4+8HDYYkTH/sNASIFbN7BoFGbbcTgd+fy/W+J6RdXUtGbJt8MDMSdMpwnsELFDSseyWdBWmoPQ9AfHsA+sz1u6qiWlSKMN64uEnxLvtrfYXcTvBs4/EwhtCZNhfziRLRx2Pc2qRWpHEpmqqajXFc40RD0ShUfmho9urjmNI+o7vNmf67me6/tX02QL/5uy5ifE43qiI8YyiS9O8BLiM012dqWUZjzlSIhu9jKXJvRRyH50cAuZiSEyTXZGWcyYlpTI5VMQly3GFHSHlBKXPI83k2JpWHMsTa/3HjlJtCprjFqrJ447FOG+ID5KA15Y7o8syPTkPREZcoxIMpU2KF0jhSEEPTdmvoSiB4dUxSGhtaKqWipTIZUqTMFYUCU5CbKY3chaUVWBuq4KrkYojK4wVYVWhav+2KimWJw0KWYyxTkdZ/58/r3HoFRBqoTgi7Mol0GBEGJ+zYpjPcZyu8Y0kCVSaKQGqdrSu/hETqEMEXSF0Qqj1Yw50UjAV4pBFUd2ipBTYR1qLWfHSwBcGTikWFBnaDRqDiiiBDZJCSpBlvPxRJBTJovZrRN8EeZNNb+GFTkL5DwEycgi0Cfm60qhlcToutynR15hFjMTvwxyyrWhyEmhlSKkwmMvTbaZw7rKteq8w3tLYVsalNIz85LCs0zF2aVVRZzd9SEERM6E+TlXopoPlyB0GQqklGYnekHIlKFBcVAVEb6I6UqW11WQUVJSaVPeOynz/Mn9XM/1XM/1XM/1/aqqqqnrmmGY5u3LQM55zrDxnJ+fU1UlSNI5j3OWxaIBkRjGI4+5Lo/9eulfzZMwqNRjcKNhHKfZ8Zs4Hk8su4sng4J1lrZZlJ4/C4RQVFWDEJ7VaoUxhtPpxDj1mEqUM4NUZAqnGqHoTz1aa0IyrDcdSmeUFgxDCW8chrFgW2bmuVSScRyeNlJ9cDRNw8XFOduHHcfjrgjtUqO1YblaMAwj292W64szqqrh4qJluVzhvGUaLY2ROB8eqXuQKFlIj72WUDNyr2xuMhuFTv2Bh+0dD/t7dB2fesJpdKzXZwWLmBM5C06nAWst52crkqppFATvUaLG2Yy1gX/6D/+ARXdOf3IoLahqQ+4zWmnu7+/K1iiBlB1SgnUnqiYS4oAxku3DFiEFdd0wTQ5rCz0jzXlGznmUUggUUsB6ueDP776aNzRLLt/f//t/yGq94PbDLdfXN7Qdjxh0fo9PSEpFSBeinLHOzi4pjno4Hi3L5Yq+P/FwPzCOI1o39KcdTVMTvaY7v0ApzX534HDcz9eLRuuKcbCMgyXGQFUX8f8oTqxWS7yPT9sLxmjGccDa6YlPPo4DzlvatuR4hRBmEb4I6dZaxnF8MmUV4V/jvCPLyGKx5HA48PDwwPF45Pr6hoeHu9Kjp3J4bNuGw+GAlJK6rueBVTHqlDDbzN9V31lEFzV8+yGw30OcQNWPsHpFVB7VgDkTvP8QCadI3Qmadfk3TS2oFoLDvaBZSn744xZhej7/gaSj5ec/WbJsGpS2JBH58LDjx5+8wMX3qHzFunuNUPc8DA+EKOlUDQR8usdmh9YXLOozhvHIYcgMk8WOt5w3V4QQ+DAcWVeK0XtCzNgoeNgKLlbX7HeBPiqiqKhNjYgV7+4Th/HIzcUNk7vn263HZYkdT9xtJSIp+i2cpogPEJuKccy0neT2XVkNWCxbspQEG0hZMB3h5Q8EoxVsH44EL+heGs6uW7bbHXqZGE8RvQ58/bDnctPy5lsYDwlNRqqOaR8hRvwocJNExES7UYgkkEvJw12i3yaaOjGMcDhZugtNu9BUOqNrycQBd4oc7iTDFppFpF0Ketvz2U9blJMc7kuY1+Yqc3a25D/8xZFuLVmsNKeT5eZ1RbfUHPeW095zdqUQuea0DQw7z2pjadcVu7eOzSvF/fuAqAPDe1hWG6TwKJ3QTWK5aej3Hp9HtKxYrGumKTHaI5VJZJE59onLmwVCR0KwfPLTGpcdp6NFNgpJoDuHFJaMeJYrxW4b+OrLDyil2axWtEuPFoJ6c8b9fkfVwNv3HhlqlhuFNJ7N1LG780gBMkkOxx1nF4rTaY8yoCrBYecK+8oKxLri/tYyHALGZIyB928+UC9amroitWDDxNXHxfEXQ8JbxXKlMReC24fAZtNQN4I0VPRh4OEXgT/8h+dMIXL5ssLdFyRCzoLdbQlwMLYm6EhzMXDeblgFw29/df+dP7if67me6/tTRhfxWZsiXpMfxcqMEHleqStYliElrPNzAMzsMFYSyLMiXVYgY0yInAsOJAWK2A4pBrRST06GgrRWs6OiOG7qui4MwVQY18x/XwTT0pwL5LyS6DGmomkK9sSY0gCX24sz1y48BT9CYUOWf1Rcv8RMDhktC/tRyOJkCMFj7YydSXF2MBdxVQiIyZPn0EnnXHHeI6jqBqPLvyEJ/CKgZQntrOuayhiYDzaVqQsSJETAF558Ftg5gb5w1euCYZmDOX9X+ekwlPPvC+jldQkh4l1Ey3JYEkIg8hzKSS6scjei5exWV9XM5i7uDWEqgqkx2hOjJD99r/JZlUlICVpDyqE08KqCrEhx3jKgOEAQ8ilQNeVEyBFEfnrsSom5cS0lpSTLx8NYuRZSyvPKpiLlclgownMqLm6pQRSOv5QGITSPgaFCSOrGsFot6bpFcb88MQ491g54P4HINE0u4yApZ0dSOVkoKUuI6IzqkaKwHYUQKKlm4VtCFKR5K8O7gJLu6bUSIiPmr40p4uzMXM+QU0LpmT9qdOH25yKqP9dzPddzPddzPdf3p6RQVHVxUZ9O/ewEb4rAPI0lg2c2uHjvGcfiCm674uIdx/7JoQslYD2mwDA4cs5Mk+V06rGuCLHrTYe1BWtR1TXel35SKdBa03ULxnHC6JqgM+v1GW3TMNkRYzQpK0IsPG9IGKPJuWQpZRJNW83ZS5bFsiFEqOuquK5F4XiH4Kgqg1SiYPe0LIjc0aOUYBhO+GAZxhOLxWI+nzjAEGMxQ5StwQ6JxFnHaTgxjQ4lOwQNXdvNW4AFoZgpuEeZCzdbkDDGMEwB5yZ2uzuOx21hrjeKxWJFXbX0x8A4BJqmIWc5O84z42g5HEa8H1iuKsaxRyFRacn58pqXN59hzAKjYRx6YghoZdhszvjJT37McrXhz//iP3DY97Rdg0kGZSJnFzUhjaXrjaWvN1rjnUUpSdvW9Ke+PH9SEb1guVjzy7/9lt32xKK9Yug92lT86Z/+SyZ74vb2W376k5/g3YSsH93oj734jMtJxU6vVMWL61do1WCP4CYYiMSoORz6MkQ4eCCzfXgoZ7+8p25qptERvcDZsuXpbMT7AaUUWku8C+WaHRzkvgTLpoKoKbz8smFR+mjmzdtx5rgXlrrWmmkq4anH45Gbm5sn13gInpwV2sjiOD8eUUqx3+8RcwZS1y4ZxommaZnG4m7v+x54xCSV5/xx88G5v3tL9DuL6K8/Lo7qm1fgesXhFLEWjg+exRVc/RTu3sI0CJoq86OfZmyvkFmTCDirqC8Ef/Inl4R8y2pjWC/hqmu4PFviRktXRXo3UuuIECdqA10dEObfodUEIeHcAqMGjBmxfk8/3FJXHwgxI5TAOqjNimm8Q5tLtvvIyVaIs0A/GmJqsE4wDYK/3d/ipogSHW1Vk6uO48ny7/56jzSaStZs946vvoz0DroFpCGx2kgcke4cgodh5+mt5DQI+jvBZz9tef9mYrIJs04MtwphPYlMSJLhoTj3PI5DH8pBq4Nv7vcsbCIJyUl4VK346IUgnATdpqZeT+zfBYyWiDaRElzeNLx7P6KWEvuVoNKCahkwR4EwoCuIwlK1DUIE+t6zaM/oFpbx6FheaJIOXL+SfPlry4uzFdIa3r294/ILx9vdCa81vc9oB6d9oKoTJk5YB6aVvP6s5pd/MVLVkvW1JsnI4T5S1QKpBauNwrqAqCKnYSRny9JWTD3E7Gi0Js7uwP3Ok33NYl1h7Zab61cEMWBHx/29R1VwcfOK1N9zeHCEMeJy5M1XA1cvWmqZuLl+gbc79uHIzc0KNyb6wdItwYXMYBPVXrBsVwgD23eR5YXk5eUaYQOrK8X9N47uRWJ1ozjcObQqHw7b2xNf/PgGHyaUqDjZB6TRrC80776eOI2eT/7oimlnefkDxdtbizKaOHiqJtNvPWkdONjE+lpSLxMyN4g6wyQxjaAfPDHCh9sR04IMhv2u5+bmgiw9b//myLrVmI3gNBzp7wRq9cxVfa7neq7/tJqmQRuNmUMvU4ooLUhRF9FSyrnR0XOQoiSEeRIvMtqop+Yxz2JhSpEcIznaEnqZfpfJkHNBmwihUFKRhSgBlDMexFpHVVXFQZN8EW8TxCTIafa7a0Oeh4dSKoyu0cqU2yOjlKGqaoSUEPzv1kOFQslcOI1xvl0fsaODTRFvlS6hllVVYd0E/lGsLmJvObQkfLBPTmNnLSmWRHdZ2mWkEBhd0bWL8vmAoK5ruq5DSom1lmmaOJ1OWOdIs1PIh0jIAubnJedUOIvzcOOJqZ4fmfHzffCl6Uvzv0kxMU0WKSqauoQQFadKhkRZfQ0TIhUETdKhhPEgEDnP2Jqauk6kpPA+lAFJCPjsETIgZMlUyfNWgqBMK3IsAjhpfq2lRCLJqdyvkAI5pydHjxDiyUlSAjgpDv/ZLpRimDFDFGc6Yn5Ny3XHHCj7uKlALlgcIcrQoDIVTd2xWq1pmhJmVRxaAesGrBsKHmcOtS08dYWUZUNSKU1dV1Razbz+MjPy3lObilrPjPWQiXPYaJoRSUEVd1MMYXakF656Ee9TwflRhkr1HKJqtEbkRF2XoNbneq7neq7neq7n+v5UzoppdIRQep1pnJ6CEK2dsNayXC1Kn9k1ODdxPB4YxtIfOe9nBJ5EKYlWkhQKdrCpa1IU9P04m0wkzgYEGmOKi/lxSxWR6fueEBLeBbp2jRCGRdeRUngycFhrGaZTCafXGa0fNyEN5+dXGKNKfhKZ7fZ+5lQXV31VVYXdniJVXc3bn6XX6rqG4dQzTQMPD/cYU7FcdqxWK/b7PcNoqaoaUymadoN3Hl0VFIp1rmwaBsc0Dqy7NS9urpFCkJ566tJ/FQSgKoLxvFVYONwnpMosF0ukTlS6hWyo6wUhRLbbI2dnG1IKWDsihcKOCalqUpIIDF27IfsVn7z+MVcXH+PGPPO/84xkFCwWC/7BP/gH/OznDusc/+uf/RnrM40yC1Qlufmow4cjKXVIqX4PQVg2MKUULBYdQijuHx6o1ZrjYeLXv/warTr6k8O5wM9//lM++/wzpEx8uHvPdnfH+uzqKRNIiN8ZWgSU7CqhAcPF5gXXV6/55s3fEoMmBElKGjulMhAZI3VdcTwWgTzGPTlnKlOjjcK7zND3WDdhjEKqSF2bGTdU0ESn00PZ6JSiIKKXS1IuDnuArmupqvrp2hyGEaXKkGcYxqcQ0UfzldKGcZgQwtO2S5yzvHv3DmMMTd0Agv400rYtcopUpmEaHcfjgRD80+aHMboMJ+acou8CePjOInp0RbTVNcglqEpx+03k9afwsIOFhJ/9s4q2WfJwH/nks4ZX5xu65pxv3rzl7bd7Pv/Rhh/94IJxUqTsuFpd01RHWnXOgzvRqA7dXNDqnvPFNYdxyxTeUCWNjQti6lm1FSKVg57ILSltOfb3RN9RNwaSRlcNVd3w0G8R2rLrISTLOFpSWjFMHu8it/eecYCmGnnYBlI6se0TWUiGo+fdb36Fj4LtFsYJbj6B0/vM1ccW28N0r1l1hu1hpK4E4ylTLwT9LuKHDFXG7wWqirz4keJ4Xw6UZgN1q6jaTPSZ6ZBJJDCJ/iDoVpLbrSdFQbdO3D5IzuUOO0qoIq0y2GNisc6IbqRaSe4/ZNyQ6DYJlxKLFyCs5uLTioddoO9DSUO2ivqjnpTLSvwwjpiFYLWs+CDKWvLp0NOtFN9+CBzuBWcvIv0bwcMQ6NYKJWq+/MWJ5VqDVxi9oD6baOtMKze8+eaO04Pl05+3rK8TKiUOe4X1ibrVvH9r+fBNZrVacP/hwKc/yty/z9Rt4Pyi4/79wLtvAotLxZdf37FagQ+S6ZhYndWA5f7DyMVVTdOtub8fMZXDtJnxlPjyy28YrWG5UNx/O/Lq0xU2RCKB+/sTMtQEmTk7q0nRsn3jGB8S2lvkMrDdTXQLjWg09+8d9UIgYyCnRLsSDG5H9AmJw6XIq880x+1INonVKw1yZPAj9SBZLiRf/tJij566afjk9Ufc3t+VFOF1pmtrXLAcTyPrK4OdFG++PNIsVhjV0nUDp2HAVJJFJ/jq64Gz64YwJsIkePfNljZckNff+a38XM/1XN+jqqrS3BQRPRFCJsY5zNOXgM2mrmmaFu8jzkVy8rPzorgCfl9EL3z0NONGHJlHdmJpisQsUCppkGJepsxF0HTOY617SoSXUhBiwLpADCVoR8ki9GptZgSI5jGAtOBfxBymqdGq3JtykCgit1GSrJhDISM+RobTwHgaULoDoVFaUTcG5wtTUsyu6RgDMRZndsrpCfcSY3gKyfQ+FC6jEKT0GMg5P9empmsXZYW2dkhxZBotU7L4UJznNoSCgqkfA3ciIUCcwyulLM6e4vYReO/wwRPDYwNchOQYE3aykDUxSIRQ5GTJubi/Q4gI/IzlgapyKGWQQs5ierktY2qCF6To5s2CVIJoU0DKSMyeTHpC+ZQvLFx4ObPsRZJkKUlIRC6hszmX1yRT3NjOeXIuIrZW5fVMOSLzHPw5O9HJ4ve4hOV2MmkW0stzIuYDkZxXUJumpa5KkJCUqhwSBITgcG4kRoeUMz6HgJAJpQVGGyrToLWmqcuGQjmsFLe6yFCpqrjek8LikT4QEqRUtiWKYF8GDkKUlV9EwnuHnQLOluAlXasisJOBMsgylZozAZ7ruZ7ruZ7ruZ7r+1JKaiY7EUJCa4O1IyEGHntsiEzTgBWWplpgjME5NZtSiuCttSYnScwgKX2RnSbaVlPVixmfF6irGu8tVWVYdEUcDj7hvUepRFW15Gwhy1nMlPT9gA92xsyU/qnSDVo1eJcIHqYxMme0s2gXSMGc6xMLcTGXv3wMaTwcjkAxkmqtGcfTvPEqSmi9SGQiXVvyfby3GCOp63IOqCpDb93TRqxzxezTdS1SFhey1hrrBkTsUMinTVUpSvi71hqpSji8966w2o0kihptoGlWDL0tRg0EOQX60zg77dXshHacna0JLiBFRQiCziz5g5//Q6pqyXg6EUKcM4Q0TdthvcN5y2J1xs9+/mN+9ev/wGpTsVzX1F3F+kwyTjvqKVNX7WxCAUTGuom+72nbBUJInLNUDdy+33I6WWLQeBcRQvGHf/hHXF5eoHTi1G8ZxgMxWVT2Rdgnw7wpDMxu70xKYKqWjz/6lP/l38lyXQVwzuHsI98+onV4Mvt4H5gmR2WKuF6uZUFlamIqZx5ywSyL+fknC5q2nZ/HE0IotFbEULZerfXzeVXMQbOB4BPO+sKGl4auXRBCKuK91gx5JATHYpGoKo33M/5F1/P1WwR2rWseTTrWepqmYbFY4Jx72uh1bkY86r9bV/vOyttv/wr0CpoFZBHJEyyXsFpCGOFsBT/4YeTzmxXR3hAYub4acNPAzdWn/MOfR9rFASmPpM6R8omL5UumSSDEia5LaFVTsULrwt9cLl6w6wecv2GcNFptmfxXxOCRMhPzBMLjgiBkhx8WOBs4jAeOFrZ7gTCJ7DP3D5HjAMfDCTtC1wne38L2Fl68ELx/KKCm0yARJPwE4wmOX5V13vYHmbs3AruFqDPnZ4Z/9POXPGwfcKPEKM3tO8vLHxju7yIoQbOCw9eZzSewPE8oI/jwbWaxgmglUQR8n6kbwcV1zW4/gVN0S0EaBC4nQgChMv2Yef2xIgX49teRbiXQLdzdJpZrRSRgjCAH6IdM28DpFKm2A8MxEY4eESUff7EkqYlwjCQSUkB08PDeY1B89YueegGqgeQFldM0ZsGBPdJYzq40MVuqFagmc/UxDNMDOWakUvTugXoVydScesf6SiDkgnHoybliHD21yXzyI8HhYUBLiQsTl69Lqq6uAJVYn7VM2XG4dbjRYEdP12jCBPvB04/w2bohJomuNXXbIpLkq9/eE1PixeuWHDMff17T1HB6Nz+fGZRJhOB491XP4oUiT5lYZ6wL1FNh1je1Yuwlx1vH+o8q6rrhm7/tWV3kEqyrIy9fr3nYJYINHPcRkQS+V3x4Y5FKcn/viT6z7yPpVMSSXfqAWmSMgn7y2F3g7LyghPZ3luGDQHcwygPNMhJyYDxEbj6FKA60VcXdNz3Ll1CbS9rOsTYt8vzvThF+rud6ru9jiafAzMeAoRImA0qXEMa6aaiqhhg9KQ5lYK40Qj0GihY8R55duDkX1jmUz2olQGbIuYiNSmmkenTZFud6DMyNf0QpPzflxfKbUiDE8r0em5zfBQqVYJmUStNc1jNB68K2Tk4W0Z9Q2Nu6QokS6mmzwwuPHSf2uwOq6gipNPTFBf2ItSmNrpwRIzFGMnEWYW1pLrXBqOJUIENMZRiRkyjCtCqojoIJKUGobdOwXCyIYR5ix0ROiZQLm77gY4pg/+hCl7IEqCqpn7jb5N9tDDwK5OTydfLh5wABAABJREFUXGAdKRYxWeRITB7v/NMqotbuyQkkZXHz13WDlHpebdUolZGi9APMYnBInoxHKZCqhIYW8sjsSheSPCNPUo4Fv5Ll/BhUEeKFmIOOEt4HQCJlmAV5NeNjoLD1i5BersHyGiASYg6slYL5QCjna0OjlZnZ5uLJZfTYqJdrPJXQXDJCMYvnJZdHa4HWAqULyqWECD3ejxm/IgsfXquKHEsmQIiQXCiYpMfrAQrqKM9huRTXlrWeGA1KelKShCCwVpBzmAdT8Tu5XZ7ruZ7ruZ7ruZ7rv52y1rNcrJnswDSNpQec81QK8kQyDj0gWHYrJhsYx4G2a1kuFzDnFAmhZhNKRdct0FpQV4UdLdAgFC+ubziednjvWCzWnA4lkDTlhFSlZx/HESUr9rsDTdMipJizZCIhJqTQaN0iRIWdegYVOB0nnIusVku2256UHG1bAiWVVpxOfbmf3RLn/IxzVFg30JpqxnN4Vovl3BdlhuFIXWlO/UCMgbZdoVQR9YsjfsLU7WywSVg70bYdIXm0jLRNW3r4YEmpbBfyiNqTqmTmRI+UktEOjNNYcgwHj5AVQlR4PyClwWiFc4HT6YTSxeDRdR1uLA70oR9QWhBt4h//9/+Qzz//CW4KqLnXXi2Lphn8UNB9ORGCpaoEL15uWJ01nF8WU6WpHNNpZBwrgo84H9BKM01uHrK42WACxhh22z3fvr0leAg+A5IX1y/4H/6Hfz6H2AuWywWHw5Zp6qnMmkyaAYal8XwcgBROvSFFz4sXH/P69Sf88ld/zeH+Hu8tIRb0jw+ecTziQ8lsKuaiclaJMYPzaF2jtSHYgqEchp6u60AUznzbdgg0KUZAsd8fWK2WVFVFTHI2c3lOpyNtW89ivcf7QNu0HI9H2rYlzDgiO1lSjjMeSOB9mrMBYLfbUVUNWhucC8SQOB6Pv8djN6QUmaaJ5WpZ3PGpiOgqq7/zPfydRfTmClZrOG3h/Bw+3MHyDGojeflp4uIlvLkLKN6z6U6crSr2+1va6nNa/ZLAQPI1S/OHRPlbbPqSfvwlZEFIE7JSoBpEqKmkgmyoxMdUInLqD4zuQFMdqI1iP0xURjJM5QfQYA94nziOR9pKcvttZnASnxO7U+Z0gGkEhOD2bWa7BVVl3EDhX1eO0cJ+L1ivBMd7CS7TSsNA4PJn5famMUElyF7gUuSr23e4U+DyWrC9jTACSK6/6MrEqF6x+MlAvYDDMXK+EdztMn6CZCNGS9pN4YiOYUKoTKoSFxcNm4vM8ahYbxJMcP8uclrDaZuZjpHVq8J50hrqOjN9qfjJz9bc7Q6Md9BuoFsaZDbUqwHGcoHf3DT8+Z+dWK4EN6/rwmyPkJ0meAdVhk6xudYcTgJXJ5IbuLpe0B89djToNrJ5ocBrfv0XEylnzq4V/SHQLjLdKvPiRhJjyzicmOyelFpOh4mrm46cLW/fJCotWZ9JTneCxIiIkuAtN59WuMliJrj8YYWuFbdvE0J6lmeFi/XytcY6ix0crz7rOB0y79+MNNWC6080zgeaak0Mjr/99ZaYQVWAhJdfZNxeMyZJU0vOPgLvMt++P3BxvuS088jlhA+G5XXDZy+u+M2vjpxftrz6eMVxPyE1bLcTVQNCKs6uKgSKL7884vu2/LCVmd1pRLWK1y9XXFy1fPPmPatFTb+3CDJtp7i46hhPW5SUdNewOVvTD5YgI+NQ2LCnk6CuFJ98seT4YWJ1ZjCisPmTOXG1XP8XfXg/13M91/ejCqpEziJtmptJUKKI61rruQkSGDW7eHNCyEyKDpsCUhakxoy9JubikhEiF/Y5pYHLMy9byTm4p1jIZ860nF3tj47m2cUsCnddhCI+xuioqjwz0z3OFdHRuXJfq8rM4ZAKqSRKFaZ7EgXnolWF1rMAHUpI0zgMDOPAwk0kKZ7ChpxzBVuToaqrOVRG/EeYkfQYtgQoKTG6BEgGFwjOE3xAioxQamZqF3uH1pq6qmjbtjg5fECGiCITUiKmWIKZtEBmMSfVF0e/lqK4oWVZsxRkUvJEFDlB8MVxooQsWJjEzK5X5CifQnvIGa8dIVicq0oAqdSF/WhqjKnR6nfCt5TMjHOBmkO+tanKgB5Is1M951Rev9m1nmL5vQCEMGilIEmyUEVgz/DorBLEebgyP+4s5nBRyLlsAljrCL4EkmotqIRGyMcVzlkwfxLN9RygZdC6CPYxuNnBH8hpvm6MQpvyNWp2tOcUiMGC9Agq0uzIESKXbTMp0YLyfeb10TZLlElUdVUClIx4CiLy8XEg5HHe42Mko3Ezciil+VohoZVGCYUw5n/DnwbP9VzP9VzP9VzP9V+7CvJCF0EwBOq6m3EoZROzMproNc7ZOYgelBZoLYu7uSlCojE1Q7QYU1AtmQpkJGWPEIEcM0oXDUjrhqG35CxQSpFjJIZI0onD4UBlGqqqnQ021cyq9hijcG4iC09MZcOwHwYm69msz8mUwNKYAlXTMIwD3aLCup7gSy+9Wm3Ktp8UxJjwPhax1TkmMaGVIodA9I7gLHYYWK/Paaq2MLdtRtWSui20iZQUy6UgHSdStHgrOL8842zzihAzZEv0CZ3L99Simrc/c7kftiAjl6sVNgb2J4lzRZQNIRYzSCpCcNN0nE49WkvquuLqxYbDbuDy4nPcIHl1/QX/6I//J4zusGkgJ8d6tSLFglXUjGQfEDmxu//AL/7mz7i6atlctgg90Sw0MWWUqvEW3Ghx3rNYLNisz7m/21ICPwvqMSXY7yzeatyUAIUQkZ///MdcXV1QGVPOUyHzzTff8Md//5/Om8VqzqkSc78+X4xi7sNlw+XFa9arSxbdiuNpR86RaRoRooRxDr1HKuY8rERVLdCm9N8xRsbJ0jRm3h4WjJMHYdFzJlDKME4lLLdpFr/H8ZcoreczRjl7WOtZLpczu1xQVQW9gij4HWcdk53oug7vPTkJQiibsyFEnAu0rZxzpQZcmEgxYu1IVZct7RB9GTLUBWu6XLYcDvvvECv6XyCiWwvhDl6+LI6zn/yBYNnmcngFHh6Ke3nReIw6sl42KKk4je+p48/YrM+Y4jcYtcbIT5FCMMa/RaoBOOd4PKDkRKMPaCOx05EUPoAJxDDQaMj+h2yHD9xu35ZAgVCjxOekcODN7Zd8uEt0y0gYFf3B8Js3I72HMZVwrNhnHt6BTVDXEALcXIPzcDxBcJmYI6++UCwWiX7w2DqTNbx6ldltod9mwggxZIadwJhy4HQisriRJJEYnWW9XtIfHe0yM/QVJx+pTbkfVSdYvpDstnD9GnbbxGEL9QIWm0TVZj7cJ7bbTBKZqAXnLwXLtSaOgea1wg+B9YVkOAi+/jrineS4TZyO5bFNPbjJ8vlPPKpScFHz5tcDh+HIz3++4XQKNEtoq0vefvOBq5ctf/PXnhefwXJZ1rLv/sJy+Ymh3WT0JLEnUNIxHBJVq1hWHe9jwLuInSI5KWISHO4hbQLTYLn6RPJiU3HY1gxDQhmFMDW7+4lPPq3xE4VPFEaSEKgqYYzg228j2WX0IKjPPGcvKuxkePFRh9bw219bjMrUHXzzS8vqrCUZj02Jqr5kcVZYSYtlwxQ9fQ9UmRQEh1sYT46PPu9YrBYYCW/f3/LZTy6osiStLDc3K06DJQFv3ux4uLNsLgRaZdplzcPdwN23A1evay7OFW0r2N0l/ujnrzicMvcPO1qj2FrBcl1hNMQ4sVlr+pNlGgVd12BPid39Ced9WUvfKFZnhtF6TC0YdhIhPHfvHbpOVFoRxcTtW4evBVebFRevara743/BR/dzPddzfV8quEBOEXJEG4rImxOF01wCH1PIoARkh2TEVJ6UBqw7IGSc8RgCROEtJiRaZbIAmRVClnDQGAJVpVDKImVZ5RPJI0hFHFamBDhSQk5DiCSfIGaUfAwaSghhC2LFB0LokXNYqBCaqmlZLtYYFDmW4G4o3OmEIOaMyEXURgmiSAgjGP3AdndL1y1p246cRGmWbeHfxRBZdC1t28yudxBJIecBBMmTQsSnwkqPPpJCKML57Bh3wZMnQZUyddMgdIWqaqSpQY5EIQkpk2SeeeglbCdHD8mhRECqiNYKrUEJiVYrFBqRBESIZMIcVqlNw6JeUdeFy65U5nja421ApJ7J9iASwWhi1aJkjZJVeVxLTcgWckCrhqrp0CnPAwZPFjVCeBAeCAgSSYbZnT/h/TRzzAVKVORcAkAro9DK0FYLrC0szRAnQhxIweEYyakgUnJWRKHm19ATRSKkCe8Hgg9UukLKBpNrNBVK1kit5pVcM28PSIwxmKoI6NZa+mGg73sykrbZEOMCrUtIUxnClEGPIBP9QEgeryKVrlC6KiukWaFlixKFKa+kmQcNkSpmtDFILTCVJiTP6CYigoTEJ/AZghA4O6GUJiRJSL6swaKQtUYrjdbVf5WfC8/1XM/1XM/1XM/1X6eaxnDqd1hXFWODrtFq3irPkpwNSrYFI+gCUkLTtBwOB0CyXCw5HI6M41gyjXJgf3hgsahx3j2hGPv+RFU1eB+ojMEGj8gSrRu6xbr06ULSNA111ZBSou9P1HVDVdXEmOjaBcd+i9CeRdfgQ8RbS7uokRqcC2zONvT9aWa7J5QSLBYtwzBh3cQiLzgcDoijwAfLq8UrXt685rjfw7yJKohcnl8QY2S5WFLpCjs4bl68Zlo6vn3/ls1lV3CTGIwum/0hOHKuudi8QooF0xSQ2UGKpKSKzUdWxCTwPiJUCRvdbDa8u9N4Gzk/v8D56QntUVUV4ziU7cIs6U8jn33+CeM4Mgwn2naNkRdcXHzK//FP/y1ny4+JNhB9QQjmVDP2gbEfCdbippFgJ37xt/+eU3/PFz/9mKYTTLPpY3QWIxsWzXkRcVOCJGnrBU3tiCGRYxHBU8jYEXYPI1oVpIlSij/6o5+hNUipSCnT1guG08Rut6dp1xjTzRuiZYDwOxGd+c+apj3D2sw0405CzCBKblFKomwc58wwHFivO9Q8GAkzzz+FOLvWHVpLmqbkROUsqOqmXBtaI2WatzXLdus4FbRQzmVDWKkKoUqu1/F4ZJoszlmMMbOozxwEOm8uC13OHL5stJbhTxHdpcoMY6TvD1RVTYgBFSXT1KOUIuVA3x9QSrFcLZAqEaL9O9/D31lE1zVUDdgIddkOwTlQppzHry4k64Xgxabm+qJBZkOrr5nGPQ/x19QrA/orRrknx47R/YK6Spz6BpEWXFX/Cssv+dD/v7lY/pDej7zf/s90NSzrJadhyzg5XIxIuaEfSljk8fQlQlre3WVuPwAZVuvI3a3jm3fwcA/NBqY94KFuJTkmpj20Z/D+Lbi+5FQhYHUN6yu4ewfaZMwSzi6gbeB+EiQyegNtXWCsQknevk9UQhI1DFPk5qVCq5HhrefsvGV363Eps81Q1YKlEWQyh9sICmIUXL4Q+JBoWri9HTkdBKuloescuoK7+wzbkYtOg6y43wbGUyY5QWdqhkXkmy97Up0IUzlgN+fw9uvM+iLB4Njdwm/lwN/7aeTDKWG3EZN6tIDJJn7yRxe8efOAIGP7THOW+OyLwuwep0i/TQx9wqwUpMiQel590RFHuH8/cP26wqcBIeBwCKzOE0ppjgfPw84ilOLD1yOrF55Xn0uk8uzfwMVNzfq85ng3sb+Hrquo6kjAo1XNYqEZp8BuF5imHR993NK2NW9vT7x8VZESvHtz5Oq1IWNZbjxv74+sLjU7N3J2eYbPPe8eHJvVgpAjy4uWd/cH2jvoFpKrszUPtwPmXLBqGpQGqSLrdc3tW8+nn71gHBxvvu7ZbIpb/eajlroTjDbR1grTBL69f0/btVSVoO8jN+cdqhEo05NSzXiQ1E3L5pMiUNRNg7egTeKj1xv+9m/f8fBwYn1m8JNCKaiU5vAQSVXirduBgpuLNZ+/3nD7YYc0E1fXzziX53qu5/pPK6YAURRRWBbmXM6FPU3WSOFnLrgkuJ4YR1K2xDQV5nmakRNzeKSYk+6Ls5hZFJybMQVKAURinF3LKc4u9CKu/s75LouYqeLMuC6hjFqJwlcXkZhiYZWn0uApbchC0dQRpZlRHhnB7KAnz0iN0pBLBaaSmFAwKNaNQOGlC/SMRgnzemThnUtlUao8TiU1SkZCsMQQ5z+DFgL03Pen8n3L9y4YEu8diVwa2Ux54kUR+H0qKBbvPc5pICJyntcsY/mzEMjZla5UYaRnMiEGtA5UVbntxWLFYrGirroSDJQSWo4IHnndYUa1mIJqobjIU06M40BWCqMbjC7hs8XhEpBKk7IiRkGisCTJnhTCzNEsDnTvQ2medJpd7mUQoqVCqxqBISiPD5nsJjJ25mQKZCHBkLIkxECKlhgsIVoyBdmnZEap4v6vTA26QlemiObGIIR4OuzEGHHOYa3FBTcHmiq0Llx9Pf8yxqBEAhGKuz94cp5IeR405YhSdcHHANpomrZBUGGtAxFmDAxzYFTJlxFSkmOeXVqCjCr/XylSLs9ZihklLFpqjDYIXXAxz/Vcz/Vcz/Vcz/X9Keempz5Za4OQ4LylqduSeWMnYix86Bgd292WRbdA6xqBLPk8M+pvvV4VHIsq/dTx2FNVjrqpZ4E+Y0xFCAGta6bRkbOnaSucs+ScZ1RiwgdX+q0ZN9f3x+Jwz8xnieIsFxKsO9F1DW1rWG+WxFhY1ALF7e0di8WiYAM1eBeIsQTNV1WDFJLdbs9+t+NstQYyISZMVTPsDwghqZsG6wYOxyMhJIypGIYJ5yyL7pxMoGla9vs9znrOzy9JqTwnpBLsmWIuiI4ciUkQo8TomrquSnCnKsGsORbmeqnMNI1st9sZoxhYLJY4G3m431OZllevP8VPHf/6X/+f+Ojlp2hd4acTAklV14yD43g6Mo4Do+3JOTDZgfuHOy4vz1ksOu5234J0hDRhKknbLFguV+x2+4KmVAbvI5WpsCmglSGEkuG03x3IKdO2BXHyD/7BH/NP/sk/oWkaHoHnbdvSNC12csQQ0Tr9XrDo43/F7/1HoJXm/OyS/faIrjOn04m2rXBufGLOK5WJfWGjPwaG+jnotojgkZwy4zjRNDXWlsGEMRV9fyqbunU9n78S02QLWkjK+fblfG40T8G7KZXtBSnLeW7oxyJ4S0WKma5bPGVApVTwLnXdFNe51PP3Ks8dlLyvcRxo24ZMKluwbuTUHwpnfej/zvfwdxbRs4PJw+mb4pj+IDKbFn74U1ARKpV5/UKxbBWV0hBqKl3z4uIV/diT4h0h7XiwW4694GyhSXzOMGVSgBf1ax6GXzCFwMPxDc4rIPCw7xnrE+/vEzFOHKZEVxtiEvzqS8fhAeoK9iO8+Ra0Kn8+7CO2hxxAZLi4AW9BiYS7hU9+WBXn2CKSrjPGFOTLooP3byIP35bhgNkUp3rysNrAlIvjvl5E6lry67+CsU80q0SWgmGE427i8nzJi48lD4eewyFz/hGcncN+BwmNkoLlZeLmRhCC4n7vCSP09wIlBMELzs9ht4W7d5k0QTiH1RmMo2W3E6y1YthlupUjukzWmfUN7L+ExbWg31HQNCnSiEy3BlHB+4cJLwTnV4qHD4HBgtwPrEbD4UFyvEt0F4HFlSCIgdt3iW7puP5xhYuK474cpIPMLFeG7d7RLDM+OR52mZubFbsPEySFpOJw53ATTPtEEgIfMt5mPIKoFbthoPYCVZXBhQ172jWszzPDMXJ9rfjll44kBZOPqErw5hcTVVdYtD/86QW//c2et186DluHUvdsbhR370a0EvR5JCMQFiZGbIysziLBCU5iix8zb/8qsbmqENeGw32kNgURsDSGk/EsF5qv39zio2d9Efjhz1qaqmIIjsvrJXZwvPn6ns16QT8kGq24/lyRGZGy48Odx209Ux/5ox/e8Pb2nqqOtAvD6RBpWvjqN3fIBFcvW2LytI3h5rMVD+8GOhPpLgVhSlQNLLqKJI/4ANMu0bXhu76Vn+u5nut7VRF45DWXFbyYYglAzKH8bJw0PkxEfyTEnpAdKbmCNZlN6BJZkCXSIIWB5BAyPrG8ywpqaXyKkOzntbz8FMpZwkJrjNZz2Gh8Qro84jmUlmT83PCWbbeYCiu9rhrqqn7ikmco4rF+xMOUpr84R4qDvqo0MRmEhZgEznm8L+t+ecaopJQp7VAkJk3XFfePlAJtJD4UQVrKEsIjkU/BOkU4D8WZkyJGV0hVnBmlAQ4454ixsMxzSphalyDQXDA4UpZ1TIAcyxCghDtZ2i4gpEao+fnXikoWhv1itWSxWGJMA1lih3Fmsst5XVMipXgKIy3sw4JMSSTIGoGcD2u5IGKkKi4RoQix4GSkKAGwIYBzieCLIyUGZoxLBsnMFi/IGGOqgi6RkpRt2TyYWedaq1mBnln90ROTJWUHIiJk4cAzh3wqXV4H3VRIXQT0R+dNYf0zP7+lqTda47VG+Dn0U5TmvZ6RL0KWoPCYIiE6QnBAwpgSGFrX9Xyw0piqRpu6BIDOSw+ZWNarkSQvZj5k4d3nlNHKlM0Mn4nM69LzEEYLhVWGumoQjfhO4UXP9VzP9VzP9VzP9d9OpZRYrwuKtaAFy5ZjVRVOsw/FjFKCOAXLbsPmbMPhq684xp7FotCtHw0oUhbchXOuCLo+IIRHCs3xeKIyBRuj1e840+M4cjgcaNuGqtL0/Uhd1ywWLbv9fg5lnJGOWaJU6YWUVmRvETIitcdNkWkqmMUYBVXVMU57lDLklPAxcjoNFOwIGF1xPPX0/ZGu7fAuoHQ5X2y3B6x1aGVIqYRLHo9HQCK1YnQDbhJo5UjZslrVaF3Rh8hysS49rFGQBG4aiSHMG5GZNGMnVdLoWtEPR47HHS5N5FwwJX0/MAwDdV0VE4cxHI8nzs7OSAliEFxefcrV5gf8d//iT/jk9Y8RuWIabdmqVQbvJzKJ07Cj73uOpwfG6cRXX/2Guq3QlebDhw/cbe949fEVWiq8n3Cu0B3GoeB5lPTEcCSlzDg4jAHvC1u87wfquiGExOeff8a/+Tf/hpuXL4gxoLUihPyEVPnqy6/4wQ9+REmUSgVR+YTWLDrpY2ml+MHnP+L8/Ip2ATFNpORn1MmSYRhIKVBVFcMwIGSgrpsnh3gR1SvquuZ4PMyueI8QcDyeOB4Pc0hshRCSqirbmEKW90HOGe/h/GxBjJnTcZjzluTsnJcsumU5t+Xy+kipWS5X5YwbxzngVCMl5JwYh4mqVrRt9ySOG6PnzKuA95Z6uSSTsG6kqtW83fqfr+/cvSsN0wFSD15D08KLG/j4dXGevX5hOGs+JkZHdBJTJQJbJpcIYUTKNdGvEDje3o9MNnG2gMnu8KHn//Pr/zuH6Z6sB7SyDH3Cukx/gG4V+fYdxJixDpqFR2S4vYPf/gZenMHdHk4OTAZ/gGoJm6Vg9OWk3TSFVbrbw8VncHmlicnSnzIPe1AJ/ABiAZUGU8FxX1zqycPHH8NiLXj3LjP1sGqAnHEhs76Gj38Ax2Nmdwf9CYbhyNWNQKiM2cD+CKouPwjb1rO9L6K8qSXORsYBCDB9mbn5aQU4vvxV5NO/B26CHCUffZZpQsfdtif6zGkXytpwFCw2mcV5CTc7LjxeZtqVJC0yWkg+/UnN3/y5gzqjmorUZxrTAXt0rbDBc7jbsly0ZDFiXWB5rnn33nPcizJgOLcELwkxs77oOG4dn34Oky0BZtuDxFnBsZ8QxvPhFs4uKuougSoYARdKWGpycDplXnwhOOwcSUi6BezuBfe/BaMkp50kyzKlSsBHP6jJMSKVxVnP5UtDVUnWF5JuK9DNguuPoFsJxmkiHCW39wG7H7l53aFVzeE+IprMME1UynB2WfP264n1x4LlpubDw5Gb85bXH50zjRPH44HN2ZrffPMNLz5K5GQIMTC5nnffHjm7rNjvIstWc3nV0nUNd7sHxkOkPdMcj5HDw4GJRGME16+W/NVfviFKz4vXmkxDiANNq9CVYnnWst31xCwYHka2Dyc2y4YXn9TcvR8QJrK61nz97QNV1WCMYn8rqEz6rm/l53qu5/oelZSpGKElhWtOKKnpMQOqiKhZ4DzkeCLmYXYru99zLIhZHDVIUYE0lH2qNIdDPoZiytmZUEJAYyx/r2Qs+SNSYbShqupZRM9IWQJ+MgmlFFpJkIGURMGqpExGY7SmaRqapsWY4i5mDoOUooR5psRTuGaKYQ6lhKrSZTUylO9ZVi3F3JQ9irChOG1EIkZTgnnEo1gOMZVAailBS0lOkHMJoEkpE1NhxReRWJMRhDAQ5oFCjLE4tXUJfFIzhzzPgZQhFKdMWbPMxBjLZ6uckMoQUyDmhNCKSlbUTYtpapQpAju5cNiNaTCmLiufifnxZrR+fL0gJgcU1rpAQp5KmKyuaJp6Ru0UbF1hhCdiSHiXiF6QgirDl1xY6TkpkAqBQcmqrCXrCnJChsdBy+Pz5Qq+RcyDkJhIyRKTIydLzq6wG2VGyPT0ezmvdUptnoJgHwVo54rzXKkyiCnBsxVaJQKzW0s8OlwEQiqSCIiQi5geSlaAkgK0msO65LwFoEgx4VzEWY8PHhdHYoxP12Amz9z8PDfuGrLAq0i28xpuykhRBlnel+BRX3cY9fzZ/VzP9VzP9VzP9X2qPG9NClEC7buunYMzR3IuPoJMYBwnum5JXS+IQdK1q3lwL1guV/R9QbosV4uZ5x0K9zpFYsioWVAle5yL5GRZLtd4F2ZzSBHUi6mjBNY7b7m7u2W9XrNeb1BaME2OhVogjCLFyOG05eLiHB96QsqMo6apN0hRkI8uLKlMjVaCygjG0VLOEmJ2LjuOh4Gc4GJzXrZOY9ktraoW5wL9MLFen/Hhwz3eO3RVMCUpl9vLFMOP0RVtY2ialqbpcDaULKjEHFTpmKwnC0lVtRwOO9qlIAbHu3dvyOqENpmua9C6bMlCZtF1LBYrtg97jvuBi4srarPmfPkxf/on/1cuLz4ix+LSl0iSEEilCFNknE74MLDdv2e/3zJMJ7798JaqUfRjj6qKK8NOnixiwe0o2G2PTJPHubIlCxay4HgaUHLC2cA4uNlgFPnZz37Ov/2//Vv+8T/+R6QUkdLMSEjNNFnatuV//bM/43/80z+lCOgF51me6d/50X/3Z8GL61csF+cM9q6EgCbHxcVlCVV1rnyNaIjJAwLnHMvl8slEFWMqZ0+hIKun8944WoTQpATTZKnraj4HlR5dSoExZasV8Xv5TmSiDSwWHTFGjNZA4HgcqOt6Pg+c0LpiHMd5y3jg1atXWDuVv1dyzlESSAkhenywZOKMNAKtJcYUF3rZaP7P13cW0aMrrm5qOLuGqzX89OeSqoLLi4azM89iKZkmxTjdM3iHiXC385y1msE2nCZHlhNtBff7DCGzG7YcDoG2aul7yzDB3UMRlYMtm9JZgmmgNvDVb6FZFEFgGEBomAJMx/Lv3AgIePmJpBaC6T5yegB3zCw2PJry+O3bgR//0CBk5DjC23ewrqE9Lzzx9RWEx38u4c3XcPkScoS0hdsB1CJTLeHVZ+BDWW7vVkWsDx7evc2szmCzgft7uP0SLl8qHm6hWQaChTffBD76SCLew+oSjAKbLVQQT3DaQ7eAuw+JaYLD6YBsIT1A/y3oM7BTxlnoXUKmTLeA7GHRJuoFXL+o+Ju/HBkcyAhvfx1wISN8T4iCF9cK3Sb0FSxMzds3lsszwfnVkl/8cocbBT4m2nURhX/4g47zpuLsrFyA+22mqSTrtSCQUVXG1Bk7CSY7IU2mzomLTxuGwWFnF9sxF7RATJByJg+SyxuN/TohYgk5EDJxPJUQsqvLBdN+ZNkKzl9IjkfLuA/cf3iD1JqzdaDtIsc+sVg2hE1CNzBtKj58O3Dz0waTi3NQG8FwzGx0gz2dYAW73UiOknSe+fM/f8vnXxgm7zlu35NCcdTZCdaXhndfDwiRWG0U09gTqxYpBFUdEVnTLQXKVNzf96wWEkNLSuMsaqTyMywk+qOlrlsm19Nd1VxeVtg4MI0J76GpFH0/8sUPJNtdQC8S9bJiMcK3b0Y+/fiCceGwk/uub+Xneq7n+h6VkI9O8XklLqfCrUt5FofnxPcckQwIFSDHAqT4PU6elAYpK5SoQBiQCURx34ZQVujEozCaZ5d3LMJ2lgklS/CnMTVN080OdDnfr0iIxamAKGJyiLFgQLIqnGqlntzsSmmElDOLvPD0HpEnKXlCsIToUOJ3gaZSCWSSpDQ7L4ScGdcFBVMc+pmUHm9TFxdHKiuGQjKvAYIw5klgR8yhnhJcCEVsj+X5LQeT0scoJcraoICqNoickaq8LilQwnBE/o/cE8EHohhQpjDHYw4obTB1jalqhFYkkREizi4hRV1VVLNbPyUgZKIu+BVkOSSlVBzb5CI4C1nWWJumRSmJ955pmnDeEVMgJ0eOFh/CvFpZgs2FMChAq5rKNDRNQ9cuaOoOLSum7PHBPQ0iIM9bAiVkFDGzLFNAEsliDohVBZX3iOEXSiDUYziumINjxXzoik//fWyQU0xP11quy2aCkLKs+SqBZEb8z0MfqTRCKZSpkaoioQgRQsrEDD4mRmsZpgnnhxJmH9xTwG1VNQX1ozRKmfJ1MZFjRqKQhf2HFmWY45xjVJrKjHPo6nM913M913M913N9X6qq6qdA+dLHlFD1YShucGPM74IUESwWK/q+pyCkBTkJUhIYU7Hb7Yp5I2cWizWnU4+UCe8D02RnTIbB+8Q4jlxcXEIW9MOIMYbT6TRjYdYIUfrB168/QkqJc5ZxlHRdx2JxhvM9UimstYxjz3q9QSrB6XQkeEWllyAEwacimqJRqiLFYoTw3nE6jiyXC87OLrDThA+Zw6HHOsfHH3+MmnOWQohUdYPzhXEdkyLLYlpQUtN2LT5sqeoGb0o/pk1NDKW3bpoOcjHBCBeeBOOYIofjiYftHboSRICcsNNEzhlnJ2LQiEYRfOTy/JrN5pIUBT/6wWf86Z/8n7k4/5QcMlJrgnOYWtH3jugt1o3cfviW0R15/+EbDqc9w9iT8PTTwDieMLGcZ6bRE+JEzImzlcFOAa0ec4Z0weekjFE1bbtECsfxMJFTRinFv/yX/5I/+IOfIwRli1KWHl5KweXlFT/5yd/jq6+/wdoR0YDSJT+o/Cobq3letMxkRBZcnL/g5sVr3r4b6NoFD7sjBXEzsVh0LFctD9tbUgqYqiGEYh5q2xbvPUqpOeyzIHykeAzQ9bRNjZJm7qE1WiumaSoGmKxnEV0y9CPTZMtZKAScs6xW6yfR3ejiwje6JpMZestyaeZNCsPxeCSlkpN0fn7+dFao64acPdM0MPQDTdugVEGDKq3YbDbs91set4P/c/Xdneh5Zp1WcHkB52swJrHsBDdXZ7T1yGm4o2lWVGSibzG8oqseMDrgo+JkR4SE0wijjYj8GwSZlOEXvx0hQT/C7gHsAdbrGl3V/PqrA5uLgpRxEcbbcs/3Q8G6PowQNdADsrjkjU7ECGdriHsYdpBnfnujiyg/9J7sod+BVKBX8NXX0DSwWoCP8OqjwkP/xV/BcUokC905iBqSguUSvv6qfE8QVFrTT56LM1ifweZsPqzlwmdv28CHB6g28LCHroOmyVxfFbf/2SeK9+8j3aoMCU77It4LDdMENsDxCM0lxDWsL2F3OzvwHNR1ZmEKgkZnkB6+/GuHnwSbFwJ/TAz3ic//mFm4yOjWk2Kmd4Hc3LO8FLx8sUEoyc11xc1VQz+cMHVxFXprEUuL7SOHraBrW8AxTSU0q6yjS4zOSJU4DUUkeLkWpAynYybLjE2FSd8tBdbCdEwEm8g+cf25IeKJLrE7ZqqlYLc/UUXBcZfQVcJOEhsEbaeotCYDTVtzvztx2dV4HMf7kSkFvIg07ciHr+H8lURqgRCRu3cHTJepasnhofByv3078eKqY3ewaCPY7hJZCD7cGm4+1vjck2WirgXfvLHg4eF+oqk03/6HIx99tMAFz2434SwsbhT3dwmZNE5O/OBHCyYXaRq4ez/RNhWilpwmz/GXe0QSmDrx8lWNd4HDVmInwc2Ljn4c+fBVeZ6uzw0P9xPv70+8+vTvfrM/13M91/exyspnYcrlMsDLRUguk3ZPFoXnLWQoCA4oYS1CF9E56znVvYjpGQlZlXBImWfRl3nlrnydQBdHe8rF/RyLcG1MTV23CJgbl0TKAWyeE9QTIc0uiyB/L0SyRmlT7odgvu3ymVTSShI5B0IsXO0Y/exU1zx6LQquRCCNQinz1IzFWAR0QX4S9FUoAvuj214KQZq/b86ZPDPYy3Ml53XI/MSXLCKzJOUSQtlWLalMDlDaAJmcylaASPmJVVjoJJkUixgvpCWkwCP6RCr1NERIKRFiKEK+UOSUQYDWhrpuqW1LTJYYElFlpJn7EUTh01cN2tQ0dUvblEbS2pFxnHCucMJj9MTggDhfP4IYy/UjpZ5DfSqMrmiajrZdUJsGKTU5S0J0ZcjgC57lkV1fhHSBlhJpNFFFckwEkUhxxgMpg5o/2zOiBLfOmwPwuAJdsEExxsI7V8WpJKSiqsqwojA6y/CorLHOWwaUbQBjJELqEroqqnlIIMlZlsE3Geci0zQx2oGTPeGdnREx5bGqulxPACnG2fCTUVIRmF+jeYMhpIi1jlGPhTf/XM/1XM/1XM/1XN+bcq6EyxdzQUapYkix1rHZbPjw4QPn5+cYYyALht4y9NO8iVmjteJ4OCIkdN2K7XZH23YslwtiyBjT0p8e0EaxWp2Vv286lCyu58PhSMrF3Xtz85I3b74h58w4jvOGZWIYhtlhLDg/u0CImrrWfPt+R9ss6NolzgWCzyjV8PCwpdKO0uIqKtOhlKFtFpyO7xgHy2KxKBt+SZCTxLrAMExUVUtddTgbSTGyXq+ZxiNCaGLMXF1d8+7dW1ZnS2ws/fU4WoyRHA8nol9zfnYJKdO2C8iGIMf5rKFp2g7nPNZF6krz9sM9PkwIkWhqzek40HXd0+POGepqAVmRoqI2K86vr/nf/bN/xScv/x5kg3c9mkgi4Hwq7nM/cXv/jl99+QsOhwe+fPMb4owMySowTSOo4v7fnG84nvbEJJDScNgPKFnRdWuEEByPe0KINE3L8djT9x4la5bLMx5ub1ktO37y0x9RVWbGowiMMU+bmQLF9dUNP/zih7hppNJloJFTQKjHs1EqG7A80dFZdOf86Ic/5823vyYlQV031HXN+XkJPZVSUldV0eXqlq5rcc6itZ655OX1WS5XCCHY7/cYU2FMjfeWcZy4vLp4CghdLpeE4KgXFZBZLBb0xxOVqUkpcepPLJdr7BSeDE3L5YK2WdA0DdM00bYdIcQnQfzm5kUxFtU11lpSyqxWK1arjv2hbDY8PGxZb9Zz8Cm8e/uOH/7oC66vbubh1X++vrOI/kf/GI5bWC7A1LBewaptWLUdKQZIVxjV0g97rDfk2BGCojEdOSXe3m95vwPri+u7riCtC47kzRu4+xZUBdOMNam6cnBpK8PkYbqFlxdwdVlQILv97FSXQAuoEhQaPKyv4fYWrIXFuuBn1EXhnR8f4LCFyyuesCLdAioBhzuwWxgviuhtKhimwlk3DeDh4y9ACuhdGQZIBRcXReAexsyUPasVbHflYkwC7AR3D+V5TBkWl3A4FJE+Z3i4y4xDwbsgI905nN/A9gNUFZxt4N2bcp/2W8oPpznY9X4HdVMQNIc9DFt4/TGIc/jq1yASRAlmmRn6jHLQXoFWAu8zXSeoq4JOOew9gxKMPtG0CRcmjgfBZ583CDHgU6ARcNoHlq3g229ASvjoYzgeI29/I/Bkzj83CBzRJ/bbjNSQgsKNgqk3+EmSRaBbZF68rmgb2N4nvv1twvaRj3+cWa4sd+8kIgsWl4nVUrE7jqwqRXamBKmtAg93Dtk6qjrx7lv4+huP1ob79w9sXgrefwUhgqzKFNXUJ9ZnipONZJMYTPlh3y4NbaPp+wlCJOI4HgybVVMO6iJxdpZ59zBy1iVWC7i7Bd1l5Jg47SKvv9BEFJOz3H/wKK347POG3X5CGLg4U4xDZr9zoOHy1ZLD0RMZcFazXFeYVhF9JOWG5BbkuGexUty8vMD6gHi35u7dA3/vD1+g5RE31Nhv9hz23/Wd/FzP9Vzfr5pZcsjCbM5qdgWnGa+RZ7c6aGWQsnC7pdIIqUsgYhKkKIr2rpiDSct2jhSg9e/c3ZBRKs8u3URKfnazS6TUaGUwpkJQ1vVC8FS+IqXico4xzpaIwkIsa6kSgS7N7CPDXMxDAFH42TkFUip868KWDGT5yPwrzvjH3xujqOsWECUck0TOGiFL8Kb3tmDgqpacMyF4UvKQMymH8pilnl1EAqkESigief73cXa5S2JKNE1N27YzmkYg5KMjOZOiL68RCiVLoGq53TJgcHYiz+gXoyuC9AUV4gJSlqa1aRoqY6ikmRmOal67nBjH4jx37pENrqmqlm65wdRdEYLrFik04zhhrZ2HCmF+3MW1rqUhiUTIZV2TTEHiqMJPl7KEf9Z1QdUopRGNwDo1myXK66pm0biw8gV1Vc3DgIwnIqMsTVZ+dMorfASTCpouz6L4Y5Doo5Cec35a61QzLz1RWKEhFEE7z/iZcp2qJ3Z7zpKYxdMGRU5F2M9Z4FzZzLC2rANP1uFcwPkAKIwut6eVfuKMkjJGaZxQpBxRQs1bAnUZaMyOmGmyaP0cLPpcz/Vcz/Vcz/V9KmsDy+WSFAOBiNY1p+NAigmlDF23JIRIXTcoaRAzCsUHj1IV4zixXK44HA5450AoHh72TGMRgdfrFSDJSfDh9n4WFBu8HxjHYho0usb7wEcfveb+/h4hJCGEmafOU1Cj957FomGzueEv//L/h3MBKWu0brm/f0CrmqZSdK0hZ8lqtcDHInK/ennF6TTStku8CywWK+IhU9eGcZxYLzd0i27OX9JPiMVx8rTtmu12/4TNWywXZfM1a6qqZrKB02lk6HsWzYazs0ukNBTXQkHrNXVH07SEGDj1PSEOHA5HpnEorG8jETJzdnbOsluw2+1QQnP94gVKVLT1Al8p3Aj/7F/9H/j04x+Tg0YKOPV7vv7bX3J5eYapNNZaJjuxP2x59/4Nd/dvOQ47bHAYrcgyMtmBrmsxVcXFxSUf7j7Mwa5wdAOLrmzHnk7F/Q2y5BCGVM4IqphmlNL85Kc/4fr6CmM0IfjZZZ3R2sy3qWjbtmQjkUseUWneIaff64fzk4BesqYMP/7Rz/h//r/+H5ydXTC9L2K+c44QAuPUM44FpbJaFT75I1bRmIq2aRFCcpp6Li4uEEIRfJgxiRXn50uauqY/DchGk3NAKcUwDDRNxf39HWebi2Lw7fti9GmLWJ5iRCvD8TggJbTtEq0qlNT44NjvD1xeXtK2LbvdDmuL4aVc25FhtBjTYK3jbHNJ8JHVqmBqzs4uOB5ONE07nxH/8/WdRXSp4PJa8Ic/rXh/Z1FScHn+knV7yaJeMbl3TO6e3e6OZfsxIndMaWAM31DXFW/vRm638LAtWBaA376B/lDEeQzEDEMPKRZhGOHYHR8IY/m7sS5f23YgevjscwgJDiM4AR9/Att78A66FfhURGsk1N3spDflGH3Ylwvl6gK6dXGJ9wk4QajhQYDv4fQA8aYI/EIVF7hS5fZWFPE86iKKx1CE97sP5XvGCM4W0ds0hSlPFnRNZrOGFEDr8nVSFFRMjOW5/vYbsCf46DU0NegKrCvXfYqwOp8fl4FhLGK97kCvYdcXZ3qqYLoXJJ1Zb+Dmsvx7O8DDfS6O+40kIXjxakHdjCxXNX/x7w/8h1/ucT28eKWZbI9zng93sGzAjnD3rgwsnE+oWrA2NfU7hzSZY7DYXaZRcNqWQYbSgq++tjzcRS4uBJtLQXIN9hAxOpbHdJl5+YlmOAW0bmi7iabp+OZ+4pttpG3hrEus14LdLpKiQi4yfYCzC8VoLbrWfPppyy9+Hfjmq1TWqrcCuYDbD5GLl4bKQDpIxkNCNZnLy4r1SjAePOcvV7z/dmT7IXBxrvn22wOTz3RXGR8nDmOmkZJWS/YfIquXIEWiWiuQcHFWcXG+QAlH3WamMXFeNVxetuwfdly/WrJ9CLy/O/FwGskV7G/BdInz8w4R4P7dicM+05jA9csbjvs7pmngeJr47a8t0xamU2SxFPTDiesXhtF+13fycz3Xc32fKsWy9lmMv5Kc9fznQE4gZBGWhWAO9hSF1CIVQkpyioScibmEHEk5u4FzQiRmZ3QRjMvqp0OIsqZYBO/iAK/rlsq0xQEuZi6diE9fV34VfkeeXcLkCiUbjOlQsiJnMaNjfHG/i1ycPDkSo8X6Hu+Gp4Amo2rICimKm17J8quqqqcGSSmPtUWMF0IQU3mcRlWzw1ugVHEkh1DCVoPzVKamMvXT/ZZKYmbXu1IRZHneTVXN3D+JVAKhDClJYggwu51zKs2l1golNTFFvLczF31CKk1VNSQiIQfSzKqXUlJVGi2hUgVLYkzBxdR1TdM084pwxrkEBIxuWHQrNpsrTLMowZxZ4L3DuolhPJFzKpzNaaRtG5Zdi3rcNiCSkyclUV5nZdC6oqoqmqaiaSuUkEihsCSkfHS/A3lGt2XmDYMGsiHmEiQrkkaQkAJSFsQkiUlRyZqMmYNg05OQ7pxjmiaGYQDKga+8tjXwiNoBbQQxzOGqoqyvKmWg7jCyoFtcKCK7kKq40FE4H3FuJKUijntftveErCBHUlYgzdN1TRYoUTbtUgzzyEawaMuwAgSeGfGTEtYF5POH93M913M913M91/eqgi896vY4FG6zMoSYaZoFRtfIWfRbdGVAfzoeOZ5OLBYdWhuc81xeXuN9Yr8vOJf+ZFGixU4B1mrmTyemyTGOAyGU7dCuFbNgrTg/2zCNgRQlp9PA5eU1p9OJ0+mEEIJpmpgmy25n2ax7KlPTttc4N5Gjoa3PaerSA9W1IEZYLDtiUtx9uMfa0jstF2tYlA3GlGC3PZCBuu3ouhW73R6tiuFBG41WDTFFrJ1ouwUf7j5wfn5G3/eEmClLmI9YRs2L6xtWqzOYzzxFKC6icYqeGBOIgvS1dipboCLz4sUlNkh0rjgee8ii9PYzxjInybK74F//y/8Lr1/9gBg0UkjGfstyJXl4+C1//Tdv+fGPf4KpFnz11dd8+/4NIU+cph39dEBpSbNcPZ01fHCcnW84nfo5e6ic0a4uzwk+cTr21HVTkIoxzggf9WQmQmjqquGP//jvU1WGGD1SKqqqespl8q5ssJbeE7wrG7oiUbKteGR+i8eF5ac/C2H49OMv+OmPf87f/Or/izEVIQTu7++xdmK9WdI050x2QEpB3w84V7DCm80ZTd2w2x0A+TTIEI1gu92yXC4hS47HEyA4HA4sl92T87vkaUWc8xjVoJUhK03wiWGYMEazXK4JoSBAYyiI6GGwIHIR+ccRpQTjOFBVJUh3HB1N3eGsRykwpqHrFPf3dyhV3k9KCXo/IUWFd9Pf+R7+ziL69ZVh065ZdB3ri695db5C64GH6Q7Hp8R44th/4GQdY/ia6CXHITDYgKoDCTgOsN2XQ03bF9qMs9BbSBaWF7DzheftjvDRNSiVoYFs4e5YBOlhD+/fw8svilmpVjClclsZOOzgxSuo64Jmib7gTQ49xKGI8SKANEUIPrsqv48d8LJcSEqDryGN4D2cnZcg034LSkI1wPgBclcE8MVZ4am7HsY9qGUR29vlLJRHqBQYXYJOjYSLFVR1Ed6lKI/p7BzWGzh9Wx6rD/DwAMHBdoLlhqevV8CyhUVThg7bQ0HETG+LcP3iBvxZZnuElx/B3R2ICj5/pXn/JtJtMtpkhjHyy/c7MoIhOOq6DCf8BNMU+OabE8sFrFeCtstFhBaZ03uwGX71ZqJqEtV5JuwEndZMwvHhXpBsplqXA6PPArOUnGxioyQ3zZLBK472FqGhXmdCkNy+lZydJaKE23eO3UNENKAlbM46Uo54AjZ61mvFuw+J5EeCADdm/vrLA4FEMgLTlM2Jjz/paBvFtx8mmi7jp4KHVVKw33p6m2kbuNqsEbeCwcO5SXiZEbWgHzMbPQfrrhKnXcYsFKarsWHi1SvJ+SIhUodLgm8eetabQNsJVk3Lcdgz2sh4V1avVueCh7vywz9Iz7jL9H1g/2Hg6oVgwrNeW1zy/PDHN/zym68xasPoR6oryVd3D8j3kcsLA1KyuXxeCX+u53qu/7RyUsXUm8TMpn50285okiTKh1oubnMpFYLCGxRSoqQvIYwizYGjhSsoUiJLSYrFxZ5mPAcUVIaSCq002YCSFW2zeGJ15xmp8bgymmdhtJRAyaq4m6lRssWYDikMKULwxSkvlAARi3s8eXwYcX4keFvCYjIoDFFkkBIpDUrVKFVhjEHrcriQsgwBQixYl8eAUyHLppWUZR1QKXAi47zFW0/wnlhHatOgEU9u/jIImN36ZLQpK5YxRqRSiBmrk+LvUDdCKKQUBVujgOBwzhJ9wFuHqQBdAjZziPgUSXkOwkyGutIko8hSlYBMxew4r2maiJS6sCUrQ9MsaNslbbOkapYIKQjB45wlBI/3lmHoORx2ZDJtW6GNxOgSEKRUCdRM8zptEcMLxqcI5AltCiYHkUDEgsSZXfqlQVdzAGmDyIaAIsaEIMyhtBGQ1KalrpbU9bK4i2YRPc5hrb//6zGgqzjcJbqSBXkTE0KU50OrwltUqlzyUigikiwyKqey/SDlfH0qUsyEWEJBUypDCykMEjBG0dR12YpTVUEb5VTCRnMihohWCqNq9BxA6kNASjmzH0uo2HdZGX2u53qu53qu53qu/3aqMh0pScbBkVJmmjxN3RJj4Pb2A0opLs6vZvb4voSRe4sQC4QQbDYbYN6qzEdy0lSmYxgs5+dXCDRN01JVmuPxSF3XOOeett8eHcvj6LA2oFTFNI1sH/ZzyGmirk1Bz40TZ5v1nEsjcTZSVSvWyxXH/RscsFzW+HkTdJokwzAg5xwjspvzbAzWWjabM25vJ1bLDSlFTseBobecny2w3mOMxjmP1mrGqtQsFgsA1usN/bFwsoWUVKZCLCU//4M/fMKS5AwpRLTSxFA2CK2bCCkipaBpavZf73j//h1UW3w8IGJNbVqMrrm8uEYpzX7bs2gv+ef//b/g+vojxiGS08jZqmbs7ziO7zm/0PzP/8tf8tkPXnAcDuwOd3z59W+QulANulWFqSomO7LfHzC1Kq70nDkeT4SQ2Ww2VLqiP020Tcfl5XVByiQPWZJiyafKWeB9Aso1c3F+RoyeplkiZUGpSKFLbtC8hay1Zr1eo40mx4TUBd+p8u+J6OL3hfQyCJHS8E//yT/nt1//FZvNGfvDHdaWMFCtFFnMm7hSoLWirlcMw0AInqMPpJRZLlcMw4S1jsViwfX1DafTcT4rFTNVfyqOcmtHukWDtfb/z96fNVmaZel52LP3/uYz+hjhEZFzdQ0NEESTNIJGgqQocLjRf9IP0gWNkkkmyWimC5EmQmyg0Q2wu6q6KjMrMgYfz3y+cQ+6WPscj0LDWHWhGyLPMvP0DPczfNOJ2N+73vW8IoDXLbrK0DqNgrni4vySJDWkaUoIYhKy1hEC1HXDeFxRVSOsHVgsF4QQmE3nDINlu2nIsoKAo20bCSm1niKv8A5AMxpNaJo6Tu4q/lD90SL6bJTRNBtun7Z0FuxsQ1dvSNOED8vfsNoNXM4CDnCuZ1vDwxKaDrY7cVsfECZlBk2A0USCQDcb8ImI6Odn4iRvGwhaHOhJBlUhaJXeQ6ng+gxGuTiwbSJ4mKEHPLhO3Nl1K+J5kUlwlk/EdT6ewHIpmI/HB0GvTKfyPBxkc0HBqNjwGU9FpNYayAQj4wbhkpdTeW7XyjW4a0S47zoRxL/6UrFaBtZP8piuEETLZinNBOfleCQ9zCby3AND3Sh5n6e9YG6yDH769Yj1pibLNZNSOmrOemlG1ODXMDoTjvu+lvcYn4nLff0O9AXsX1rOLhXOgw4ltt7z+BbSSaDr4eWl5v3bgN8HwiBomaGDV59lNE2PTgL399AbwGvuv3eoEpIcQhv48HGg20NSwvkLaVI8PHiaQZHlinYbeP/oGUYLXt3MOJtd81d/uUSlHbV2vHij6VxPu0v5+KsBLhSXl4HrK1jtahnJUDC7TsiNobMDZZWQjxz7xrPZBHwn9+83X8FmFwimJh8rso2EsAYN9QAzpRkCrB4dWQaLhycmowrrBh43NSoNeKtwFiafBzoHzQDjuWZ6Act1z27lufOB7DPH453lcW0ZLKQVbO5hNK4ZFSmkKUPoqQqN6hXtypFliosXhqH33C6WJBpWW1juQGc1aZJzWz+xbD3nk4TyKpBlltXCk6aw7npmoywG1J3qVKc61e+X1nkUq0NcK6nIN9eoYEGFuIAA6xUKCVlMlYy/Ga1xGoz28g8TInxL6neIbvC4mHIHIRMgisJJSpIUcTGfHQMhtRaUyjAMWOeOIrpSBq1SUClGlSSmJEkKEdaNEee8EhRNCA5Cj/Md/VDT9w3D0OKdRauUYMLRga91SprmpGkWRx1BnBjhKHJbJ4uIJBGneN/3GCNxO8ZEHrmXf3+6tsdaj80sWVqQuByMjmFAgm0JkrJ6dGjoJEHZACrFWsfQC9ZDB0eWC/vdaMHHBA/WOZztMVrjB4sLYgcPQRw4wUBwGjcMDJ3GEAjO0w8O761wIIuSNM3wLpBlGWVZkmUjsmxEno/jYraVwCo70LZ71psl+/2WqirQJmASCZoyJiV4WcQHr+OUARADQ/u+o+sMWskEwDAIXsd7i4/nWGtpsBiTYHSKUeL48S4wKIu3/ZEBORlfkOUj0qRCSO7DkVF/+AKOwvpBSE8SEycVHFp7VKJJE+GXy5UJSkVkTtAY70iTcGwySRhSYOg7Diz6A5IoSXIUGVkGVVlSFMLq19rgvAQqGQXeecjAJLmcS2uPr58YuWFxPggn/1SnOtWpTnWqU/1oSquEzXqHc3B2dsFut2U0kvDFptlzfX0l6wrnj2GRMqE48PS0E0Rh0Ay9oGCKoqAsRuz3NVU1om0lQNTanqoaxXWRpu8sbdszHk+4OL/k/uGRq8tLnh4XTKcz6nrHaDRFaUPXtVFst2ijKauczdpSFCM+vL9l6DWJGbHdbMnzkn29BzxpOj2udx4fFmRZQdcO7F2L0YZyVjEaTeJ6NGW9XlOWJUmSslrtODu74vHxkSxLGIaBNDWMx1M2mxVn8wpXKDabmiSJE6Iefv6TP8V5j1EGZRRGZeAc2kjouw+BruvY7Das1yt2+w2T6Yjbpx9QxoqZt5jx9PQojYLecn31mn/yn//XnJ+94bvffsdk/IKvvvgZi8c77u+/Z7V9x7/4l/8jNuz5q3/5P6P0iL4LdH1Nu9sz2A6PxeFQAabTMUmSsVqu2Ww3bDc7qqoizwryTNbCk8mM29tb6npPVVV0XUPfe7nn0NB1nr6zKBIuri6AQ0aU3MhpLWYScaAHTJpydXmNMZqmbRhVCdZ2BA9Jmkft3BwW8xzQLs4Fbl6+5uuvfsJf/stbhsGSJKkwyLsW53ryIo3Bp56iSHHOsdvtsYNgiIqikCZIXrDd7uJUg4jdg+3J84TziwuUcuR5Kkah4GnbFk1Aq5wsk3DVEDyz2QxjNPt6T5omdF1PlmURM+PihLOhKEr2tTSOqqpitdrQ9z0hQNN0jCcjvA84F7i4uKJpG9rNmr6zTCZzrO1lSvcP1B8torduz8MDfPX5jNVu4P1tIE9Ba8tvfwCroGlgv4WigvsHEczXK3h/L66fvociFYxJ10M7ABZcFJPrndyj5ymUpYjT+y3Yp8gw93D7BGUOLy8hG8FyIa8VJ6MZjyEphGPuLAwNuBSubqQztd5C08pzhhbsWtzkwYEGZucimD8twLVAJvvlHLy8gYd72A7iKn/9hTw2yeDDW8GcaC1BoB9uJRj04SmwWcH6AUIKT2s4aBFZDm+uwKSw2cpxjphWSODVpQjRm4/ys+k57JoGpwJPS0eWwLjK2A8DxgRmM9mutofza3h6kODO8SUUGnQOuYbNArwJXJ1BmjWkBWQTyMZyjJZPgd2DTACklYj/aQbLp0G471bCX+0D0AVUKkz22TW0OSR5oG9BmUAxlwmE8RQuLxWTasSvfr1jtwlUlefDasHI5bikZz6BZm9JK6jGhvSVYrlUTK891VgaH6NRYLVx2E6x3FjmI0tRaBm37yxaKZIkUFWw+G10/SvFu28DT2Vgu5PznM5gPIebl4rxJOOXv6qpG+hcIDUdZIFqHAg+oXoNy1vAW16/SvlwOzCdwdBZJhNot7B5Cvx2D8Z4+YspUzwtA24beLyD2dzz859Peft+yYcPgYtzce4lIfD6yvDdb2Q/slTTu0CaBO6fPIaGbd8y9Jp6v+XivKQYtSQqo+t6zs8ydhuLtaeR8FOd6lR/t4zOIFhssPIXvAatZApG0BguhocGBq9QWpPqRBy32uBNQFsroYkqoDRgRTRVIH+hHisceYYgzt0kSUjTnCRN4hincMWVClhncd5HN7aM/YmkaNCkaJ2hdS6ieuSaqygk++DxDAQGnBP3S993MoIXIE0SCQ31h/BPInNRyVrxYFeBGE6qjk7nru0g0yjlcC6glCTZOydJ80prUJqu7wk+CDdb12R5jtYJRN674GG8PEcpNLLPSjncYKNYHtnyOokhkyHiPmRdkuiERGk0CuVjoGd0gBujSZMEo2R0te9bnHWCHbESWGUis9ypQJJkJElOkgg/UKtEJgy8x9qOtmvY73c424n4rAJahfjvakImlniyDLl5GxwheKwdEMyLxQ49nXYQDP1gGWwfETle3NzGYEyGNilKGxKToZTDJEMMkAJ8SlVMGY1nKDK0yuLkgoMgfH8QZMyBCd/3suhVSlw13nswIbrkU1KTHkOPgvdxykImMJLg0Ib4mZDzKdeSi9MTSQz+MugkwSFOhyyTpkyaZGgVcDiMFmd+YmVbURpPiNe/ZnAOZ+XYikv/JKKf6lSnOtWpTvVjqiQzhMEzGo8iq9yCEgRLWY0YrGW9WVLXNWUx4urqmhACu+0epQxZmtP3lu12T54VVOWYEALz+RznnLiGs5K27bi+uqFuakHquZbdrubFixsxBaiUphWs4XQ6Jp9P2Kw6wNDsHBcX5yifsl23zGcF280Ds1nON9/8CavViqLIKcqU7W5DXe+4ublmvd6gdUJZFmw3O15cX/Dx4x3T6RTvYbdtqPc9203D9YsrimJM0yzxXlGWI6pqRJpuaOoWH2AcCobOURXntF2gafcEOgbnMaYgS2Yk+hrCmKAStPagA0p7MJrClIJZ9g6lHcvNLdY35GXGeHzOy5tL1o973rz6giyb0jYd+33Nf/1f/h8YT6csN4/8D/+f/56f/fRPGfwjRZHx/d3f8OHjd7S+Q6WG++UDdf2R4DUm1yjrZIo1LRiCx1tPVUmTIxAwqWFwA9fn12zWW/bNhvFoRDdsCfRoIzzwfnC0jaVtLEZr1qs9XdujMPzww/e8unlJkkzi+tkTvCWTRbrcr+EYT0vWq0WcdtVoI+x65wLaZKBk+lOWo5LHZAycza74z/+T/4Z9veXb7/+azeYRHwI+QNc58mLMZrMn+MCq24hLPiiKYnTc19FoRFnmKAXL5SMhdOx2PYMdgBEhOEyiubo6Zxh68qykyAeslaaHNooyBpd2fR8nog/Il568yI/Ym/V6S1akKAVt08v+6S2BQJonEe+oyfOUzWZDWYqxJnjPbDojhEBVVux2IuT/wc/wH/thN5lwwAfvcAFu7yXwcreD+3ciXK8f5J40ycRdvl6KyJ1XMBpzdAd3tbDQmy2YHGyQgM+2kucPg2BM9hsR18fjiFDRMBnBdC4M9LaGYOFsKm73xgpCZTYTjvh8Jrz1bi+O7n0T3xdQKRDdd94LLuX6hbjUHx6Ec65KCAm0YkYid3A+lffYruSmvCxgvxdWeruQ9x1aCV4djaCSrATWK2G5v3gB778V3vrsK3HIN7W4j7sN7DaQT+V4973haeOwTvZ/lMLf/qWHcWS8A4tNj1ESSnp1DR8GcdQPVvaLICJ4X8D8BYwruH2ExIOdQ5t6dh1Mz4Acnp6gs4HiUu7/ikK475sNTMeeroPVRpzp6RkUo8DllQj8y5U0G84uhNvet/DwCP0OfvInkBpPkrZcXhref7B0XWDxCDrtqMYwv5DrqR1gv3fc3TvUFFoL9SPso5gfPAQM3ln2Lexqz9B1vDw3tBtLqkEFBUng9haGTYBBBInpGaw/QnEGfQ3vP1hevbTMRgl1Z8nH0HSW+RkkJuHDrePqCsbnmqFPeHGjWK6hHxzNXiYmqrHi8SFgW/jyH0BZQ54ldNZy+WXKx3cOk2qGvsN2gb6R64YCssSjgmM6hmpWcnfXMPjAZBLY/hY6rTGZBOe9epMzKgqWW0tIHJM4tWB7j05PN+KnOtWp/m4FJwJxYhQBJ4slJT5pYzzOiRNdKU3QiXCelYkBmJBlCT4khN6LY90rtDeEIO4K59wzxiRiLJwXBIYxCSbJyLICk2QEwHnP4Cw2cssH67Fe4b2OYnJK8AbrROgHg1IpRhtARTO8FyE1aAbv6O1A23Yxgd2RaBkBDV5Y5UabKJhaETBJY4L7gB06XN+jg8KEBBU0ygYCGq/AZAng6fuGrncMVvZZDoWiHix4K2ufbhBRNZOAS6UUru9jNotCRRHY49FKEZQnKE+S5eRFLuK89yQ6J9EDeVKgrCNVOWnIICSkSYpJM+FAGkWSJZjE4FHYvqfrOvp+wA5OhHh/wOaAsoYsInScs3jbERgY+i3O7iDUEBq0GoSzniRkpiBLRiRpRZoemhmxQZ1ICBUhvp9V2GHAh14aGNYR6PDBo0xCokbC2acEXZLkJShhuashwfsErUeURUVZnqEpSJNMQqW8J+hE3idokjQjoLDeYZKUJASUUSiDsC6tJk9SsjRHxYYG2uOtA63jtQieyOH3nr53EkDqD7gjQfJoDcboOJWgCFpGhlOjSbVGK4V3njTJSFIJtTIJ+DDEqQF/bNTEXg5+EPa+MScU26lOdapTnepUP6babJfMZmc479nudjH8UNYYznmss2K88BJCrkiYTKZ4ZzBGTAxJkkGo2dc14/FEgtbtIMJ2UWCtZ+gFaTeqprRty9nZBO+fMDrh9u4O5wTd++azrxl6x3az5eWL17TNnnrXk6iSz1694K//5m/JkhF/8ie/4OPHD9zdfeTNm1c4b3laPFBVOWUpAfXD4ESsjWaD/X7PMAxMp3MeH0XMHVVzqqriw4f3vHnziu2mpm89b958Tr1vIirSEFxgv++pqpIkzeltR9svqEaK/b4FO+Mf/Dv/mPH4C4yeR5RgI2xo5WQ6sxtw1oOG3u5phiWPq3dkheLli89o6pbJ9JI3b77CDmLSePv2B/6/f/7PGY8r5mczrm8q/od/+n/m8rfnvHj1is1my9u3vxN8ow9stju0kgnNJEnp7cDl5QXb3Q6lEupmG0MtU9I8QxnF5YsL2qGhdy2jUYWnZ7PbYlLN+WSGd4amtjz0a8qyQpGTppCYMV275Te//Vv+g3/v38d5h1aaxIgQbrSRdbcCrRxVmfH9d98xn89RiYRxZiqgVUqWKIKS+xyCilOaIYa5aubTl5T5BUaVvHjxmt7tWK83tF3gLB3Rb3cyqasNs+mYpm4jv90QguReNc2eEBxZrkjSBGsteVkAMsHqWkdVVRijsdaTZSWJga5d03YtRVmQpOlRPC/LnGEYZDpZicgOZfxdKVhID+O8ZLPdkucZk+kY63tQnuVywTBYynLOarU6hrDOZjO6rqfrehKT/sHP8B8tom/28LiEt2937PYiUCsPP3wf8SYrcS3rVFAmeBGHlQHbwDZAs4CmR5xrpQjnIYFJBY0SUfr8QkTf1QJu30E1h7NzcXmjhe1dD4JhGUby3DSV9+o92FIcxuUIKheF10RQL7s7EVJdLoL5bCIBnsFH7MtERPw8ho9SxmsKcdHfPcCkFMyKj+K0PpP3ns6hdzA/F5f5eCROc41hUqYo0zKdK66uUrwdWDwEQiJu9TQRfruNjHa7Ex768tbhFbx4I+8zu4CHW+gGwbTMZ3JcdnsRutVaXP6ukYBVAhQjEbw3W7g4F278bA6pEmF8+xFWqxhemoPqoCjlOD/cS8NgPJd9JIUqyI3jk/acz8VxHXwgL6RRcv8RLq/lvYKHpw3Yvbzv7RN8/qVlQNixvRVmfFPDZKJ5fPS0A1SliNNtJyJx00JqpOEwWEH12M4zOxO34ago2PgabxyDg2mheHkzYbva0tcBtZfrzHvpm1TXcH0ljY0ih+A1Ny9nPG6fIgP30DiwzC7kGk4rx3oDw/fSNNnvoF7F5kMSmH8pnH3bK7I8UFSWkkBVOYL2ZJXHozg/10xGCrzi6T7w6o0mSxMUA7fvaja7QDaGF3O48wqywHQqozrrzY6PtzvyVHP3Q8/8UjMtPGdTQ9+7P/ajfKpTnepHVCFyqgM+ug0GQDgcR3FRCf9OGwkaOgQfeh/HpqI7QUIZFS4ItiQECck5vhfy97SWeEgJJ1XmGFLqkW1xToR2ay3WitjrvAjkISBucwzOERdhDqNTcRrrAGgRQH3AeYd1jsFa4QFqhUkMiTESTqQ1SaIwWrjlAY8PA36wDP3A0ElokrdOEBwojNLYEPC+p+8h4HC+lwaAFSY8Slz7QYH3Du8CLgxY5xmsw5jI7VOKEJEgEoAEyhyCXMX9nGYZaZJKoCYKo1NSk5GnJdrL8dQqIkmKKh5LMKkmTdOIG3FY6+l7S9f1EbFj0Ca6oYPDB0c/tNTNNt6wBZRyeNehsCRGpvjIUoq8YDyaCjs9LUlMIZz8iO8hBEymaVtH3zm8H+jaAe97ipEsPq21DINw8rQ2uKDE/eKlUSP2bzl+1oNJC6bTMUVexSDagjSVZapzjr7txKmOjm78gDEpSRIiykW+Cy4oFZQPKk45yLWu4oimD+EYYORtiOFE/vj8YbCfIIaUYFoMESckEwFJIjezKn7OlDGfTCI4AsKDFxa+4A5VRM7ES+h5gvZUpzrVqU51qlP9KGq12jAez/AuMJlMBfOxWrDd7pnORuR5xna7osgrsqSga3u83zAajTHGsNk88vj4xM3NDXlesFwsybKMoe/57W9+yzfffEORl9T7j1RVhcLw8sUr+r5n8bRkv6/Z1y15XnJ3d8f11QvKomJUVSgFk8mEptmxXK04Pz/j4vKcxXLBz372M1arJft6AwratpGJyEQznZ6xXq+ZTAT/d3//iMKw2+1Ik5ztds1kMqZrJQum63pevXpN3Qg/XWnNx48fmUwmpGnKcrng/PyCqhqhVKAoM3aLLZPJhMFumc8v+Jt/+Y7/6j9/ybic4Z0XnIn3R7538DAMlj7mzxSFHOfl8omLqxnv3r9jGAZe39zw3/+//p+s1xu+/uonrHYL/up/+Rd8/fUXrDdLvvjyDXW/5nHVcb+453BPNJ3OeXp6oixLmlrMQev1mjwv8D7QNC1lVTIqS7bbLZP5nL7vaZqG8XgsmJGqxA4DKkkoihzn4OLinGGAzeYDeZ7QNJbEZJyfj1mtthit6PYtwQdcP4DWFEUm69rBkiTmeHs2Gk3o2oH1eoNJcrTOyfNK8p+U4jDp+1wepRKs9YwnU6aTGX1vyQpF2zbkeUbT9ngvZpAsy7m8vKTrhOEuPHwd3ecLssyQJAbnHGmWEAiMRiN2ux3DMFAUBavViiyTANPpdMrQ2zgwLI505yxlWRBCoO8HnLOkUVhv2xbvPWVZsl6vGI+FEd/3kjMwnozZ77dA4MWLa373/fdkWXbMJVJKsdvtsBHjeQhn/UP1R4vo330P73+A/YNwwtMU7j4CXhAk+62IyutHcZbnObQbIMCwAn8F5VyEc+dFsG6t3H8PVkTduhYBdbsTsdtvYFfL/0+imz0Aj3fiTB59Ln9+eBB3dDkV4ToEeU03iOBdTuDyEjorwm7dyu92W7i4kpv+zQZWa8gbWN7FJpZ8/qifoLqE2UiwJR8+iGBKgLv3sNkJ/3y3l9chiIi830PdeHZPPb6GcaUxKnB2rgGHDdJwWD7J8cpfwfVLWD5KOCoJYCFLIR/B40Ien5Rw80qO/XYd3cgDfFgBTkJKd3KtkOcwGQseZ1TB794KPufmGuoO0FCNxDV+NhV0TecR5nkv4vfQRWG7h88/T6nGKb/6Vc12C9M8MBlLI6DdwmQSGxZb4dTbQY5jN0j4674R93kI4qYvs+iO/8Fz9gKuLhVdE9jv4exMXkdpuDoHdMZ27bg6y5lOSwa7oR80632Lc54kExzP0Afevt9ydgGDh/wLePhbGLawN4Lg2e2F1w8wnQa2uxrlBPMTnIgbNsAXV4IZ2m7EYa8CvL4u2e979hPHeCrHab2G6+sKrzWr7Y7mMZCV0HQOnSr8EKh3nVzXTpA6EpbhSfSY8Tgn6JZ0ZHFKtIWrLzQOT6o1QSnef+wZBrg8F/yO1hJGWlWe3Pyxn+RTnepUP6byXkbfQrASxKkkoFJrCYQBRYgolSRJ4qIqiow2gAoxGV5F1p46OpFB2NzSfNTRjRBDSj1HkT5EZvYwiAAr/GpBjrhPQkUlbNGhMOgY1GiHHp8WEJuvOm6b94JDcaHHDZ08LlhSk5AmRoI1U0NqEozSyK6Ks9hZHx06fRwDtAQnWBmtFUlq0CaJaI8OHwaIx01cLuqZlxfxLkmiY2NCHXndNgruIYb7HBaVRh/42ilaKQnqSRLBzijhzadZhg8VeZqII0ml5EVJmqZY78H74zEX1EkgSeQcHsI2pdEhC0QfAlo2F2tbmnYXWZdgvUVpBNmSFRjjSUzOeDSjKsdkmXDVjRHHtvcDIS4+27ZlGHo48N8Hi+48Siv6vqfvW3yINzYBPM9c/aNoPVi89yRpSpFW5Fl5xNUcjqdSco1ofQjIVQR8bCIo4LkpA2CMOYrkWifRA64iY9RhrYuPtyK2B/13WOs6Xs8HLJExBpOYeB09N6G8DzEg9zARFj754hkzAzE0leM+nOpUpzrVqU51qh9XXV5c07UD2+1WMmAmM+o6J8typpMZAcd8ds5uv2OwA0oRpy1l3ZOmKWdnOSBGhOl0EgPkcy4vr/j48Y79vmYymbLb7RiNxgyDY7erub5+KWuQmE0zDI9sdxuKokBr2O03tHVDlhomlxcsF0+8evWSh6cVfd8wHpecnf+E5XJB0zSUVU7bNtzf39O2LWVZYJJEUH1GY4zgIet6L3iZ61fc3d7T9wMQaJuWm5evsE5ykpq2ZjKZUI0KMc4ox2AHHh52zM8uuH94h9aKvvOMyhlffvG13BfE9bjzXqb8fMBaD0GTZTlD21I3NeOJYESaZk/btaRZxv39RwmppKcbdvjQcvP6EutbRuOMDx9/h/c9u11PUU4hKNI0p2ka9vuasqxiaDzkeU5ZlqxWK9IspW1bRqPRMVQ+xKykxWJBlmXUdc35+TlaKfquJU1zVqsl8/kF43GBHXb89Ke/4N27W7RK+S/+yT/mbDLn6zdfMB3PUEGMNi6akggBnQnWJHhLmlVkecFqtaYoJozGMnGpSRCxUe5NQohISf9sONFKUxQFTdNQjDJGI2nwdL1lv98x2B4zwP39Hcak3Ny8ZrkU444YRQKD7el6ma5IgmY0GlHXNev1mqIQpEqapozHY7bbrTDRlaEsCpy38d7QHzO1rBWjUFEUx7X0wVC03zdIByUwGo1xzrLb7miaJh5rOV8hwGQyxUbTk9aG9XrDaDRiPpcw0j9Uf7SIvrgT1IiXfCw+voUkiCCZFTI1seqgcyLY7lcScokDNYLZmYjB2gi/XGtQLYxKWOxgqOHiQhAtmw1cXsDkjbynSYRpvt8Jq3zoBceSaHhawnonruzxVF53ETnpIAKtEYQq53N5vO8EM3N+Jo/f7OS793G7RiJSd4O8TrMRIdU6mE8ElzKZi4v5219CdS6C8Os3cPtBmgntABdnCauVZrvqyWdQlJreOZrWs20EXRJqIIsieglPd/DwHhjDaC6C62wupq23byEU4oLva2G+L5fiug8d9I9w+Vpz+Tphse4JXiYDxiNByWyWsP8IqoBhDrOxvP5qDe1e8C3nZ4KqaQZwSo7N6xvBoyRJYN8MqGSgKgCnUCGQ5YrVE9ghMBpLU2DzBBcv5JipVARru4U7p7i6yNk2HV0T8BbOx8K5LxJF10LTKNIskGYi/AcFRZlRZGO064Vbqrb0duD2URoi11eQBkU3BGwPOg2sGxH15xdyPe5jOGvXQNbB9aXgZj7eBvzQMJ3IecyMZrvUbGpL30KSQJ4r5tMUb3tGo8D9vbCm6r1cn8MAo2nLYh14fIR+AfMb+MXPCtRFyu27HW4fcFau18lEEDiDhbvHPfPLnC5zZAqaPXz/G9juHRdnmlcvL7hfbshLhxvg9iN89tkUGMgSz37oGI3/2E/yqU51qh9TeS+CtfMW9IDWXuTEuPAQ1rlBKwl7VErEREmDJ7p75UuJteM5SFQnKC1NxzQ1R1eDCItAEPe1cMP7GOTinl3s8mrPomiI7xM8aCfhmZ7oXJfOtQ/iJB/sgLMdg20Y+hpnW1AOo0REzzJDkacYnUR+hjvy133cnqG34j5XAW3iQixJSZOULLpI6ALD4PHI9hojATxHwdV7rBWIufpUXI/H+CAAq6AxiICutcLEGxitNVli0MepAENIFRmKNEvRwcb0eWl4DE6c94Aw2q2Oor8CncZwckXXdUeB++D+HwbFMBiUsvE8OrIsQWkZBU3TjLKs6HsR0YtiTFmOKcuRhKKGQG9bcYXYga7vsENH8FaOXwj44Oh6i9KKobeRzS7HTml1dG0PQ4/uYminJ7LFDYlOYnPA/N71670T1ngS+ejBxSkIEdF9ZM9753CIy3wYJFjUaBAAi9xgiNt8YBji9sUbkBBZ+yhPmpnYVEhIM0OWJWSZNDsO7pgQiGJ8F7FGCuMGFPp5koPfF9EP18VBnD8I9qc61alOdapTnerHURL2mTCdTsnzgsfHB5RSvHz5kqapybKU6XQqIjCBrMwIIa6Hnadpas7OzpnPZzw9LXDOcTY/p2k6Xry44e7ujqbpuL5+Sdv2tG1HlokY+uLFC4ZhYDIeo7Tm888/IwRFXuTRDBAoiozz8zlVVVDXO9q2YTIdsd/vWCyfuLw8J0kMo1FJ2zW0bcNoVNF1LSbRrFdPVNUIay3j8ZQsLdnvGtI04f7u7mjY0VrT9wPTmaGscp6entjvt1xdnTGdjTFac3Ex5fb2lsG2rNcLMcv2sKl3/Ad/9o95c/MVSpko3Ms9BUEc1goR0JWBXeNYrRZ8++3fkuUJeZGyWD5yNX5BbweaumawAx8/vuft27d8+dWX7Osd47HgDDebNd57tC6BwGaz42CoyNJMsDHOkSTS5Njt9sK8H+UMfc8wDMzPzui7nuA98/kcawV5UpYlbhjokQDSd+8+kOc5aaaYnVUMbseLmyn1vqUcBf7hn/0p42TKbrtjMplS5Fryfg7YywBKi2ifIKz5h4cnMQupRDKmVCJO1WgyiTOSHKKtjBE37+eff06eZ+y2WyZnOR8+fKAfHCbJqKqCEDxd33B1OSNNE+p6jzGGs7MZIRQEHJvNmjwvaTtpKDRNcxSskyRhv98fOeR1XZOluZhhvI3ZWgl5ntO2LdYKqsZ7T5LkGJOQJglt2zGdTiMq0bBcrphMxmy3O4oiZ73esFwsGY1GxzV7VVUArNdr+r6nqirquvmjTC5/tIg+nwnypEmF3d1u5Nn7HnQNuRH2OApUJcxvNQJSEYOLCh7vRbS9eCHM8KaXe1vtIWyhLkRYzhKYTMXJ3Lbijr5/ELH+4hzqdXSaR8dwWcF8Lo7gto3bEAT7Yozcvu13sh8BUIn8vijE1d6ugVLE9jwVJ/J6DTqBLz4TkbmPIV9aSUCn0/L78Zmw1jdrcaNrI67ujYIvPs8YjVK2m56ra3D9wGIvTYR6Bf076ZSoa3GTTyoR0D2y74kBn0Caw9OjsNaLEhIlYjsWfAM+l31IUpifa3SisU5c4ImDVS/bt1nL+QiI0/z1a5kgWNzLMbEeNg/SnPjsJ+LWHuT+ksVD4M3nI4Lt2S4H0gI29zCfaNYbz3ojn8FxAtOxfJ3NoNyKg33bQFbCqEp48/qS73/3wHbXMZmKyN072GwDVVkwmyiaoaPZezaLeL29sFTVhvXa03aBl28C1glXvyxgXEBXB8ZjEdLHE0haOR9dA62TczWdyEVQVaADVLm4/rMcrm9kcmG/VejUk6TQN+LUz0yQzhfw179qeVpApqRBYLxMZzw+eJYLwRdlEeuDG9isLNNpYFTKFITJZarg/EyCZvsnx8Ompu7FZX5ApI5GiusXInDstgPrJ5iMDRdnhp/+5HP6vuVp+cjbdx3J4o/9JJ/qVKf6MZUPHuetiNDBcXDHipAnwvjRMe69OIW9xwk6Gq1FULfWo3gWhokBnVqp6Go3sZsfxUGvYyinOqIzhmGIIno4LnQFsSHbehDRgx8Igxdch85xTpwHTimU8vjQY10ngaJ9jbMdIVj59zk6vQXnYjBaRdSKZehFPHbOi6jvRKhX6oDmMCQmwWgto5A6QSWKvtd0Q411A1pr/CduZaU12kA47otCGWGw+09c9lqBNxoTTOTRy/uZKLo77zHaHI9xmmUYnaOVcOa7tqPtBgbrZLKAgOssIXi0yUkzg/dgkoScApOIazrpDF3fYtuO3jp6Kxx2k2QkaYIyGYkCtISVFkWF1oEsLamqCXk+wiQZ2mjCUbwX17ciCG4laFx0pwcC1jn8cMDjHErc2+JIFxHdBy/Od2XEqaQSdEg+uRYO5ytev0phEgkE9YHYjBFEkAjVGllBKYJXOBuw2uMS+ZmPTnEb0T8Hkfsgnh+E74OrpSiKo+NLFvEpSZrI8fdyTYdwuK6f91VrLccg+BgWe9if58d82jg61alOdapTnepUP54ahoEsy49N/aenB8bjMVVV4JzDmJIf3r7n7GxONcpw1tJ1HThk0jLLaNuWH354S5pm7HZ78jznb/76V3zzzU/54osveffuHft9Tdt21HXDZDIjTXOUMtT1hul8Qp7njEdjdrtGsv7KXNy+Q48xirLMeHrco5OUPEt5WjzgveXx6Z6u6xhVFXW9xznLZDJmsXiiLHO8H7Hd7phO57J+d47xZIp3ilW9ZDaTIMlXr15S1xuSRNF1DSE4zi/m7PdbhqGjD4EkvULFbRsGR/BgdEGeVvzpL/5MxBgEayhM8ITgB7z1MqGZapSB0bjEB8tuv2WzX4Iec319xWg8ZrPaoFSCMYq66Tg7v+Tu7pHJZMy33/6Oq+srlMpRWu6PxuOKEKDIq+O9S1nKz7quR+uOqqrou56yKLAImiE4T5okvLh+Qd/3TCcTrq+u2O/3x/un6XTC+fkck0CaKa6urqjrltl0ig8dH26/5Ze/mnAz/5zz+RVn8xngCKEnTcTZ7e0QWwgaaweUMqSJGGUks8lEt7k0GyL/Jk4d63i/IwaVVzev+PnPf84//xf/b0Y+YTabsK875mfnbNb3Eevi2GxWx8ylw9fT4oGzsxnGaNKY4SSoniVffPEFVVVFw89AlmWcnZ1xd3cXeefPhpTDxGdZlrStIF52ux0Bj7OWLM2o9zVfff0Vi8VTxLIMNE3LaDQmBMdu1zKfzQDYbrcopcjz/BjI27ZiiKmq6iiu/6/VHx8sCrx5LW7mu0bEaj0WETLXIkSGPZCKg5tS0Ck4YVp3XkRgrCBJhkoet6/BRt52kkmIaFGKiK1C3MIE7u8E75EAOoPyTBzYXXdgUcN6Idzws3NYPICx8jp+EOG7LCBPYEjkUslyEai7Spz0uhI3NkEwIvNrqKLDV6fgelgtZTt6J+LqT/8e/PBWtq+pxUVclMJ2N1nDctVAKfv69F7wIhcX4vAz54Ekctj7PTwE0FM5pvs2ivbAd7+B3T2EDTQ5UMTrHdnW5kmE2bMLuLu3pIVlNBa0zn4P1VQaFmeXcB/At2BbaQ6sb+X9fSuu6pBIk2MyFWf+Yin7lBWwXPdsdpbdTtzZ/RDAy3FNjKIo4Wxestu0FJVn/RBd8qK3EEYwGVs+PnxE545qJKLyvoHJuTDdcY77x0BVyeh3uwW3BvPaM7/0IvY7OQ92EDE8SWG9hczAbh8Yj+VYbDfy9dSIUzw08MUbTV4G6l1Ao/jyi4x//qHDB5l6sD189eWEu7uG203H7l6aGwaYnMHiCVaRz9/cgTkHt4TqtTQcsgJeT+V6kr9cC4pLw8cfNmS5Jis9+SCfjc0a1nVEGzlpFjkfcTh7uHoVKCaO9+/X3H7ncS3MXqWk2cAv//ZbilxRDx06U6dg0VOd6lT/xvLeypezoP0RiyGO84NzWqOUwbqDezaiU8zz7/2RT/6pKBgF4gP7LOIytDbCAg8SaHlAuBzESxCBXr4LZ/0Z8xJwVuGdJU01KklFLB8UWhM50wPexy/XgR/QIWCik0KhRNyPQj9KmPDO9cJfd7LgIwjWxpgkYmAkwFR424YkyUgyQ5oZTAdd39K0NiJNnp3pCoX3z5iR58aBIzjhYquQHBE64pYBo4Wn7b00KHxkp6sYhJokBq09qAHf9ljnDy0QrBW2vdaBEOIkQZagrSyMlRNBXxuF0h4fLH3fEoJGm0BgwIWBEDQBcX0rnZJlwpAv8jGj0ZQsy5+d1DH06uCklhsOIEhArI9Il8FZCRUNEenDAX8Ch2BSax3OBwgGowJJIsxExfN1YIOPAUEO7y06MaBStE4jp1wTgsbagyit45chRKSQj+O8z5z/EJ3tnzQ4YjjoYReTJKUocvI8P455HlA5xhjwAR8U4GKDx+K9i9d7+D2kT0D9nlgvn71PkDQnEf1UpzrVqU51qh9VZVlGlqf88PYHrq8vmU4nhIiBO2BNyrKiaVqcaynLnM1mxTA4rq9fkvQ9fd8xGk24u7tjMpnw8eMHLi4uMEbx8HDHeFzR9x1KBUajkvv7W5xznJ/PaNuG/qEV1vliyXx+yXKxoCorxuMxCs96vWK9WWCUou0a6q7FugGTCAt9GAb6QZNlKUUxxntPXdey9kUmHcuy4OFhQZYWBA/T6Tl1vWU2n6K04+PtW6pxTtfVXFye43wXGejQNJrtdsNut6EsMwbnyYuEPC+pdwN5VvHzP/mHGFWKIcboKD5KJpBnoO8G0iwH5WVtW6RyLPee/X7P2dkFi8UaFQz73UAIEiafJjkP2ycuzq94dfMlq9WK8XjGaDTi4eGBNOkZj6YURcGHDx+5vLxisViwXK6OTuvpZMYw9Fgrbuq+79ntdlxfX7NYLHj//j1/7+/9vcj8djg3kOUpHz++j8xuy3hcgHKkmWJXL1mubgmcsdk+Mk6npGnK4Fq2izU3N69ZrO64uHjBYDt0KkK0Coqrq0se7u/QSkszRqUU1QgVJCtKEcMHkbWqVknEZjr2dYO1FpMYnHXU9Q6T5CyWT8ynI6zrqaoR+32Ni+J7kmjqek/fd1jb44OjbW2cvMh59Ur4/Ov1mjRNef36Ndvtlv1+z/n5OU9PS4zWTKbjo5C+Wq2Ehz8MlGXJ09MTTVvz+vUbFk9LXr9+zdPjE4Md2O12vHnzmqenJ6bTiTDds4K27dAaptM52+2GoqhompqiKBgGizEy8SoTIP/r9UeL6O++hVffiICZVxL66BsRQdtCfk4OOjq69UyczX4lz9cazq5FfD3wyselOHGVFvHaGxhPYL+RUNHpuQiY653gSpZr4VRfX4swX1WiJesgIul+B2khN0HjUlzrywXcvQOfgkmhTMVBrnUUPF+n/O7dEMMt5fdtAlSClKl38HQPo6kEYa524lb3LdzfRoSMEaH+ICjf3Mhrb7aB+ydB3Nw+yedaeRgmcHEDyUQ48Ms7JPHSwYsosLeN+KkyDdu9uO71BEYzOc5pIuJzdw/sIFyKqzx4OR5WiTPfBRHk8fJ65UgaEA45D97AaCKBpJMzcb+PJ4IoQQmypmvknD7tBnqn+OYnIx4ea+r7wPkbw1efT/j4oWe7bxiXFdt1x9CLO/vdezneF3MJpt03gWZwzMaxkWBg18HLqUwk/O67gTJiZra17Gt+Iwz+oRfsjRngbgEvbuCzL0RQ/3AL8xew89Ko2G7F+dd28PqlTAecXyouLipuH2qWy8DVBdjQk5TSFHr7LZxfw65bs9oFVC/bmI4F47NZwvJB9ifNoL0QtE91CbOpbGOWC7aor+Xa+fC+xvaBxQJGV4b1yrO6g2kp50Pn8To2cqzvbwUfZPegXkHbCTt/ci4M+yTrGGzg7rFmPtH89E/OuJjteXhq/9iP8qlOdaofUQm2osP5jiRR0VkASvlPRM6EEDiKw+LGBRMSgj5wosXl670Iz8I9l0WG99L0lK8oIvqDY1lcwDa6l2WBquNIpBZR+BMWtbiXBQMjv/JReJcbAZRDaYcPPc61aALKqOgoV3yKzrDOYoJCQkotPkgA6TAMBA+JSdBGkRgt6xYNSeSoK20gjpqmaUIIOUorAo6mbbFxFFDF4NCAiPGJMc/uZGvBu8gldHg34KzGDT3BJPijmHpgm4sArBNB6xitCKEHpUiylNwHht7RtA3D0JPn4ow+sNaBo8PdeXEEWDuQpCm5zz8RiZMjq/DQKHHOC/omMSiVUhSVMNt1AnicG/BWOPQKwa+E4JD0dUWaZHilCFYWIFqnHPRhpRK08vgALsg1YVAkqTxG8ChAEA65XDuW4CQMVf61DIJq0UrE9k+Y4nLNabIswzkTcULxurNy7RyaP3LdHRoB0hSQCQAVr3G5uc3zZxH9059JxoAiYOmHjq5vsLbHWhfPo43HLJ6PmBdwEOE/xbz86+70U53qVKc61alO9W9/TWcTCdqcVigdqEYx6NC2NM2Oru04Oztjvd6glGG32wBiPNltN4zHk7g+8YzHEjY6mUzQOmG7Xcf1iuPh4Y40NVTVXELT7YD3Tt57NuHx8YHLy2u+++473rz6jO1mR5olQKBpaqpRDkpMJCZJmM+nrNfL4wTnbDbj7u6WLEsZhoE3n71hvV4TsGRpzsPDPfP5uRgznxaEELh+cQEMrFYPnF+c0fdOUBzzism04v7unizL+clPvuGXv/xlNDaUrDYb2q5l6BWEjJuXX1IUM0BQlAQIKhwzmwKBfmhwfmC5XrDvnvjw8R11s2MynZIXKcPgWCzWfP3ln9C1jtV6Rds2nJ+f8cUXX6O0oSoK+j6gTYbWJdqkpKlMAiRJSpaJm/ns7IyqGvPdt99RliWTVxPqesd2syHLMrabDa9eveLh/p7NZkOeZTR1g3MWBQxDR93suLy8AgJd11CUKU+LO7wLXF5d8NXXn7NaLrl//ECZThmNS9b7RwlpTa55WLzHZIr52TXdUJMVI1SA8XgsYZ77LUmWoVRL17bkeYUy4uJXBIIP0QEeUFpjkoTVakVVVTGUdSmOcqNo+4G6lrXwq5s3koM0dEynYwIyYXF1dYHSgRRZ/y4WC25ubpjNZmw2G5qmYTabHU1WIYSYE1BIWGuzp6oq1usdL15cU9c1bdvS9z1KKZqmpd7XFEVJXTcS5FoWXFxc0jRt3O6Spmm4urri9vYjwTuKouT8/AJrLWVZHk0t6/WG2Wx+NHz9r9UfLaLvtvD+I+weRdDGy7ODEUGxGsnNWRHxOrsYFtpV4q4NDq4uRGj8/i08DSIM51Nhdm93IjoujAjIKolhlnsReEcXEkS6XwvTvG1ESC5zcX53LRRTcVCnkbue5SJydzci/A8N7Fp49ZmIzW0t4w59LeJ2XgriZTySENIDSz1LxaFcjuR3i4WEVDYOJjVsl8Larqr4fSRO4uUaqkTCLttBBP1+AbsVXL4UBIsKkGpx9U8nIpq3nQR/rjci6mcpLJX8virEgd02EGw8g6Xw5+ulQuVBRstLePlGGgddJ+fPGHhxJc+vexHbuxjYmkRRPk0EJfNuLcf0bA4PH+HyWhzfZZVyPslYLfaMS3GWN92GxaNHJ/D+3SOLpUwUpJkcvyFOAlQj2beLiabee8pM+PnLtRz/zRPUG3jzheyWuwObyXNsJyGq44kI6TqV41oUwuV//UYwQM7Dh3v5brScj6yUa6nIAz+83/P+Y2A6UWid8f59Rx8gmcq5SjLo+0CWER128hofPsj+D04c+qGNuK0CZtdyXbz7TUQLnUG3E3xRNwR2WzifGb7/YRDG/gqSl4JHCg5Wj3J8igS2dxByyM9guZJrezqXfXz7Fu7vA1l+cH6C0R2jynB/q//uh/ZUpzrVj76GoWOwPSEIp1q55zBKpdQxcJHAEVNy4G97f/B1C0NPRFeiEG/xIbp3OXCmiQKmFZxGdLb7wPH/iY995haGGBQqzmClNRoD4bCNgBKnhPcWlEXhCQjKJImOaHQqAZ8mw6gUo5LIj/SAZ7A91okILC5uQdGEoLEuoFWKUsJ4P/DRhaQtYqw2moQkIl8MVh/GDB2QCDPcJGijxdEBBDuAllDTQMA7ix2iYxqFKhRZlnFQwI8oGJNgkhSlFbbvUUlKWmgcGusatNGkmQjoEuYqSBydGJyLHHFAGUOSZaAVQSm0kTE8YwxJlhO0xvpAsB7nFegEjULpBLTBBU9vOwKxadHHwFIEUyKBneHItw8+CvNaFuNyDCVwNSTi3rbeYbQXwVlF/I9KIhNeY7TB43Euqurx9UP4/RBRMMeGjYS2ynGz1uFjcGjwktUSvEMpJ9eyDvBJw0VrTZqkR+zKpygXaaCkvxcsesDH9H13HEH14cD5l3+Yn93lGqOT3xP8Pw1U/VRQP9WpTnWqU53qVD+O6vuG3W7Ni5fX9H1Hkip2u4bV/Zqrqxc4Bw8PD4xGI/q+I8sy1us1o9EI6wb2+x0BRZmWVFXJw8NDDErsuLg4YzyZsN/vuX5xKWL3fMJ2s8X5gfuHW9589prbu4/M5nOcs6SpYbNdY62jbWvhcw8d3XIva17v6GuLMpAXOUopdrtdRLAEZrMZHz68Zzafk6Yp1kLbdhRFyXa7RuuUy8tzuq5nVJbc398zuJqz8894Wjzw8uaC29sfmM3mjCclSmk+fvwoaJFoUhCn+56z2Q17r/n5z/5dkqQihOj8PJSS/wTvWG+WbNYbtvs1zbBi8fSI1prZ/IzJdMrbt+9Jk4LNZosPnvOLc+7uPrLerMmLDB8gLwqyvCdLC5Q2vHr1hrbZcn9/T5rmvH79msViyc9//nP+5m9+xcuXN9ze3vLhw0f6rsUYRRKnXbebLcYYirxgPptjh4GiKKjrPWVZstkueXy8ZzabRY73imHoybIsGnREqHZ+YNcsuXtSJEVgfnbOr7/7X7B4frj9lsl8TGsdrrWUVYnSgfnZlPv7R9JcOOLr9ZLZXJEXFUcuugI7WHQ0kFjrefv2LR8/fuTi4oJk67BOAmHzPCMvFLv9Bu9l2nUYHDc3NywWT2L0SQzbnfDGLy4uUEpFrrmlqiqWyyVpmrJer9lsNtzc3LBarVDK0MYwUPnZkmEYBCE0GsUJUM/11TXWOooiZ7Vac3l5SQg+fj48xkhgaNd13N3dMZ/N2W034lR//Yb1Zh0nMWZy7xWzvXa73R/8DP/RIrrPYPsItCJwh0zwJ/OpCKVdLxzsxMSAzk446p2B+6UIn6NKxsf7lWNogBSqKwmPHFVwdyfC+nQKKoufAQ3jOUxKGNbi8N2twVSCT2lrEVYJcD6T9+8GcW/vaug7QavMzoUjvq5FHO2tuMB1fA+cvLd1Iv5qI85nk4sQPb8UkX23lhDUfAYXVyJs9w1MxiLilpW878NC3NSjqQi4SQd1CkPEwhgD7Uq+VwWcv5D3f1oIl3tsoH6EOpHXQMn+7pbCYw87oIf8StxzwxKMCpRngrTRybMzWikRsAcLJhNsyLqBKgrFAXns7gdICgmMXazFyT/UInRfXoMFXOhZbHuyAkwND/fwpByX14pqlPKrX/VkOTFUDm4i5mS9k/eZTxO++fKGv/yrD3S94+IcNlu4f4RJDrMrcZB/vJUph88/k4ZBlkeBu4sNk1F0q6/lZ+VUhPhRlWNtR9OIs7tewm4ML1/CD+9gPwTsFkZF4O27Tl67kvOYZ5rN3mNWMWC2l4ZMoqUR0fcidrOAsI5/SafCN3c30D7C+AsJZlXAdCZM+3wOZ1cimm+2kMxh08r0xflcPju9lWtOBWlCFRU0Vq6H3U6ua+dgs4fLQo7Hvvas15Zd47i7P92In+pUp/q71Q/ChQt4lFZILqb5hCEdxeKDE/wYImpivrkGLZxxY5S42Q1ASuAZY3Fw1Q6DsNcFn0HkkT+jX0T4lG17DhP9NITxEPJoBLGhD2z26FZW8f2VBpOgfIYiirImRZsEo0UQJyhcfG0JIv1URBeMjLiSNV4pfJIQgsN7jfcDSh+W5YfjIjibNMsICFvbKE1QKT5El3EIpMZQFAWJkka998IPT6Kg7uzAgISYhjSVRauzqAAmonWUP4RxGgiCo/HRYKO0IlHJ7znotVLRgR6Z6SFEDElGCIo00WhlCVHc9l7RdxbvAsZkIogrQ5KlKCXCvPMO21m873G+w9uBEIiCsgjLRN69d4dRBEdicrTRx6BalCIEeb3eOqyWbZTrTvbJ+wP+5Bl3gpZ/16W5QmwYCOcf/zy9IAK6JjEBox3DYCO2KHAIkj242XXE0ZjowE/TZ4zPwZ1+4KADpGka3/eAoZFJhraVgFXvHcboyPIP8fNwwLZ4ID9e6wD/uhP9VKc61alOdapT/bhKKU+SatabJV3XHEMUx5OS5eqRspjw9Tdf0zQND/fv8F4zGlVx4s6jjcZaS1EWtE2L1sIUL4qCfb3j8emBPM9QSswA7979Duc8NzcvGYaBqsqZzsZY23P/cMfPfvZTHu4faenouoZ6v2c6GbFcLhidTTkbz7l/fGSxeCTPc5xzTKYT+qGjH9potnC0TUua5CilGQZB8wkyoyFJNT4oitJQlAalUza7J7xv8aHl+sUZxqR0nWU2PeOHHz6QZTnWOlbLNVmRUZYFPnhG1Zy//4s/Q1MQvCKYg8ECDuYNHwZ29Zq//pt/yXg2xoY9AU9VVWw2W84vrnFW8eL6FfcPH5hMKnxwnJ0Lm3673dJ1A9WoxIeOsprgPdihp65b0jRnsVhgrWW93mCM4ePHj4yqEWdnZzw+PvH69U0U0g2jkfz8sAZcLBZsNhtevnwZ1+WON29eMdiB/X7PxcU5Xddyfn72PGFrLRcXl3hr8Wagtms+Pjmsbri9vcdbOD97yWdffMaH93e8evUlaW5Yrxe07V7Ok7f0fSO5SCqI+BR8nAiNyEmdAI40Tbm5ueGf/nmHSgaU1tjOkqSGruuoyoKiyMiLPIZ47o9i//n5nP1+gzEyGdu2LePxmLZtqes6ZgBU0YRl+eqrr9Ba07Ytk8mM3/zmN0ynYx4fH3n9+jXL5eKIDFJK8ebNG+qmZrvZs1wu6TpxqHvvjq8rCE1pCEjWQEKWFez3e9brLYuFuOwJmqG3VOWYLC2oyj/8Gf6jRXRaYJCw0PmFCIKDk/vY9VpE5Itz+f/lRly6Tu5zSAxMRyIa2kG41sVc45QwRlINSQ4Xl/I6doDlFu5bwWhcngk/2geghGwu2BSvYFnDbpCwzXEl773bixMYLwLyaCoO9rYTZ/fqSZzXZSZYmHIsbulmEBe2QwJTUdDtDLvGY8qAiwLn+FxE6iwTod16EZ3rFt6UwtZuNuLizqP4e3cvYrKKWJmhkZvzzkIZw0NdD9PLeJwRPvngwMWw1GEAvwMq5L5MCTZmPofqCwm0DAjSxnlYrKDpxK1/cQmrlWx/ruBqJv9fFvIe5+ciGIf4vKKA84k8ZzKXY7RYSHOiaQTxstmKsP+TP4HJPLB86OkGwfAMvQj2gxfxeVpBNdZsV46mX3Fx5UHBfGYYeo/ZBaaVNDGyLKMqLYPz2EEaE9VYkWea79869p0cfxtEfG86mAwyffB01zHUsdFQQfc7xb0NVD8Xsd0HERzqvZyHqjKsFo6nBpLSo4G7pWBz0ok4zNtWmOjTmZzPD79FgitmUJ1B6KUBsJlHN7yD6zfSzLm7Exf5Dx8dWsObzxX1LtD1UDew2MprWSvHvLwUoWA0ykltz/LB4QaYX8HLKxHVz+aK9TpgtKbe96xWEnJxqlOd6lT/evlDgGJEeYhAGr+UIEuCDxHncnD9PmdZKCXBNAcGtTEHVjqAwQd7DH48oFe8Eze7UubIRAdxQB+CG80nbu9PnbpaHRzoWtjkh23VgkwRV7gIxoQE7VPwHhUFY6UjiiUysZ3zOB+36yj4exF1ncUqRZKkRxal7IMFnkM+Q2wnHLZdcCjyXaz5Ob1VdF0rnHWjSRODCRlWBQlY9R6jhbXuwoGF7gSPogMoI6HSQQjiPgThsUUXTh9d4NY7aSR8ch5NPD7Pzu1nJ7SIwsJW93EaIBDi6KSgc5QKIsiblDTNj+fBe8cw9HTDDmtblA/CiY8L0k+DiEKQMFlFQpomJIkI0Vq6NgQf6K0FNaCVk6kAZyNH3EOIfHdDvD4MQYXYPFGgPNooUOE46XC4TrUW979KVGwQJREl5GLgkY03LV7c+Dyz0OWalmP4Kav8cO0fvh+Y/YPt6fueYRAnjlzTQQJgef78HF7/yEb/NwjmJyb6qU51qlOd6lQ/vnp4vCPPc9pmL0iO9RprBybjGV2vmEzHfP/dt1jnmIzTKETKuiHLUqx1JEnC0+MDJkk5OzuLzf2W0agkScWtO52Mubu/Jy9y2qYBFRgGMRumSYK1LcZonp6eeHx8oMhLrq+vYi5RQlHkMdS0xQdHnqdAiKzvAa01L1++pOu6uC6qOZuf8fjxiSxNo8HFkGUJIXjatubpqUfrwNX1OZPJmA8ft3R9jdIlq9WSF9evGYae8XjEfHbBaDTht7/5jv2uRiWeh+UD/+V/9r9nPr3AORXnZUFY6OG4jn58vGNU5QQcxsDNzWt2zR37fklRJazXW+bzCx4fF6R5wrZeUlYljg6Hphgl7JotTb8jKxK8sgzOkSaGx8dHzs/Pubm5oWka9vua1XJN33XMpnPSNKMoaq6urmj2W5xzdF3HZCyicFEUGKV4eX1NliSs9juSXEw0FxfnDMPA+w/vybOCNM3ouo6+b+j7Aa0TurZnsA2bhwX3T/Dr3/w15+eXPNw/8adFQZIrfvXbX5JXJeNRyWLxyGa7jlOWgdG4ohqNSFPJIiLmWFkX0EGmcIPyGKP45utveP36Nf/j//R/pxwbiiIjhMBge5QqSBKD9yKsl2WBUnB5eUFZFljXkeUm3ocIF15rHd33NdPplN1uF6/rjOVyyX6/pygqXr96xXQ24e3bt7x8+eL4PK2JYrlHK0WeZywWS7788ivqWl5rPB6z2awZhp6z8zOyLKVtfWzqFAzDgFKKoijwXhzrTdOQpilPT0/Udf0HP8N/tIiejkX4TQ4uYy1s8LaFMET0SicidrODbAxPT8KizkciqNa1Z7MBlcP1pTBXjw5cRLDcrAVjYT1cnsNsJhgNH5nexQSuJhKUuRuiiTyiV/AiqB71xE7E/L6HUSGvsW/ARq65CvI6WsHZFIqxiOqbvTQJXA/tKqASccArK9s5KmW7t2vZXmUiXkPu7xH+qYjSeS6vuV5Jc0CXEoy62YoIf3OpGPqAtWAkO4CntYiy1y/hqYZyBiaAq2HZAVtIr8FMBVOjc3nN7ZMcN7cRl3ZnBTEzn8HTg+BkTAGvLmW7NlvZ5MyIWF4UkQm7ETH43UfZn8m5uKXrjTjx9/GYlVnE56TweCci9dW1hJm2DRQZPD6KoJ8Y0NpTVfDtd1u8h6sr2G0dTytpFLy/EzRK11nazrPbyTWVJPCnv0hodprHB4fKhKmfKqhXIuSvKhGrg5YmwmwK9V5DYUjygcVSRO2QCOZGB7kWmp2cF5xcQ9UM6nidjCoRur2PCKMB5i/h/lJE7/RMJhR2K9h38NWfTEiynsF2tDEcdN+A6aBfybHcnwX2teB9FBJSq5H3qKpndNDycWAIDtfIxIc2iqIKJFs5HtOJdLH3nZyzMv9jP8mnOtWpflSlQhQaiSGaWpAjSkcUixamdwj4GIwodA75HiI7SqGP7G2l5LVQKjqCY2BjdEBbF46PP7iNxQVvjq8hNwTCgolDhIIFUQfXbuRWG02S6OgkUAQkcNMk4pYPfcBbi1JGnOg6waOjSK5iM1+c0D664LU2UVz1EIbj+J42GpNoUp0SSAhBMVhxYh/Fa23QR4FVgTaoJEMlGqXlH/9Ui0McLTc7SqUo745OD2MMOkmOWBBtAkkaPe8R/eKiq90PEgzrvGBItJYQGRObGgfWdpKmDCHg8fIaIb5WkH3SWiajjk2VeM7lnBxQNPI6EPn4ztINLXW9o+/3JNpQFgHvUoYw4JwXHFDkzmsNRsm0QpZnZFkujQal8R5M36HUwGAsqu8F+ROGT3j6su+HYFe0sPiVDijlCUoaGdKoEXFdxwBWxSEA9ODWcQy98D8/bS4crq1DiYvdHJ3o8MwqP6Bc5Gf+KPh/2kwRcV2QLj742ECySHBvEl32cZsjrgglExISmHoS0U91qlOd6lSn+jFVlhuSVPO0ED63tYGqmvKLX/x9/uZvfiUs6FFKCMlx3brd7ji/KBmPxtzfP5DlmtV6w3Q6O7p5jUnwgejYbXl6WvL1N1/x61/9mhcvXrBarUWnijky0zxnu9lJYGiakOQJd/f3aK1Y7/ZU4ymJ0aAD2hiGQcLUsyyn6wfapo0Tj4GiGKG1ph8sWZZxdjbHOkuSJoSQEQiUVSVGFGC/r8nzguAM7b5nu96TmJTF0xNlMabMK7wNfPH6a95+e8e+t7TbjpfXX/Of/aN/QqISdt2e8egMvEzGSkYSgKGazLj/+JY01wy+ZrUZUInj7HyC95a3P7xnva4ZT6aU4xylC5JU41xLkiRsNhtGo5LtZiWIHd+y3W745stvuLl5yWg0Zrvdslgs6Pue9x/e8/LlS6qqYhgGXr9+hfeW0aQS97TO+Xj7QSZPNbjgaIeOvMoJBKazKeNJxWq1wg6e4GG/35MkiUwg+IC1PZCT5xmb7YqiKHj3/h3z+YzHxR1oWG/v+O7tv+LXv/lzdvUdk//qv+HXb/8ZQ+8oshGv06+4yW8wcRJX1sQHs1REWCqZTgXPZPyCf//P/hN+9bd/CWaHdR1D25EZTdd3DM5iEk1e5Pjg2WzXXF1dsW9qlNaslju0Urx8+YLlckXwntFojLNrZtMZzjl2+z0KxdAPjMcT8jxltXrCuoHRqKIoCrIsQ2sJLA0BlsvVcaL0xYtLkgTads9oPKIsc/a1YrtrReBPEvb7J6bTMbvdntF4zHa3I4RAlmWCA03To4lpsPYPfob/aBH9/FwEzelYwjUDEt65WYHfSrjjNoH9Emih90APWOijO7dtRKy1VsIunY9u9Ri4aYOI8K4XsV4bMWK5VkRt30CfQpPJe3dLcVN/9gpsK8/tGgnLBKAQ5/dgRTRWWpzOwYngjAMqEZbTVITjhRfnetGDcdDgyQpBgzR7wXqMpjBGHNtDbBKcnYsIu9/FJkB0GtuFCO12JcfCK8G90MH0GhSGura0LQyxETAMglopSyg8jHMRpvcBmAITccMbFYXkFjY7YbNbK7zws5Hs19kUthtxyuPB78GdwXIJq63gRHDSDClKEaRTDRjZ/qKQ41fXcgzHE9CtOPbxgr65fSdBpOfX8rh+JQ2RwUoDQ3jk0DWKp/uARUT+upXmwG4L9Aq3D/Q9/PDBM8qkydAtgByWTwMPCxHvX57lOD/ges/QQzVRjKYZN9cTktyxr3dYm5IWI3xaY7IBq+RaSFO5FpUW1EuvHJTARq6h5EL2eb0GPRLHeh+v46YHNHz+cwlMHZDfn59DvQWnW2ZzxeMTfP+dnOvQiKP9/I18VvJSHv/9D4JkubmRnw+DvI/R0HkPiXSNkglcXObg4f6+I0tg6DWLpcNoWHfwxUtDnpxwLqc61an+bgnGRf7OQymU1jE0UwkeJDrQ/adu2WhqDiEQfGR4Ky3uX010fSiCt9HZLGOc1g7H54SgICiUSjAKUD4K6/Ls4EQwDyGCVVQSueeBoDWJMVEg1iQJ0bkuAqQxQbjuOmC9MD9CCLgQMTAmiWgOEVoFp2JQ2qCCQhtPQEehWQORqW4KlJHFRwgaaz39IE77A2bEhYD+NMBUKel+64BOI+s8HDjcAZMkpPGGwTuHizx2dHTM4AnBCpPcKojNDVR0MTsfA8MVGoVR+jCpKjcLiUElIiArG9AYacwGT4gnWJolsRmiNanW5EkandgiopvIYj+wwYOWJVKwDjc4/BAICTgbcNahEhN58B4V+fmBEM0EKr52gtEyThwU6NRgGGhpYHB4NO6wYAs6XnNyAWoNCmHUH46T87KodV645+GTxo5JTAzZUngfyGI4bOgCLgwHsr00GWJgkjIpJinQSSqBpofjHZsYJgScdxLwijSJjNbkaYaPQbnOOsEOxVwBQohseDj0F1wU510IOMD6wOA9NuKJTnWqU53qVKc61Y+niiKj6yxVNcZbBT4lS8a8++GOsqhAebq+kcm/kGCtJy9KhsGx2e4pylHEuYyoRmPu7u4ZVSNMotntG3a7HbPZjMenJ/recXX9giTNqJstZVmw2eyYTMYy8ZdIFk2aS64PWnF5/YIP7z8SVAImZV9vyLKCapSy3eyoqoqmaWm7HjrLm9dvWCwXaA/zswploO1FjHbeY70nRHG9HwaM0XS9Y71sOJu8YOh7bL9mNB7TtR04BYWhnE549eIz/tP/uOSf/flfYXTJf/SP/lOcU6zrJ7oO8iwhOJk8LcoqogE9SVpw9/jA9atLfvv9X/PufkXXb6m7HXmeMJlWrNZLoKMqz+n7lkQlKJ9gSDEqZTQacXd3T5YkwuLuO+4fb7m8nJNmOe/eLSmrnM8//4y//Ku/5Gx+zmhcUjc7jEkYqZS//uWv6Puey4sLnhZPAHz5xZcEHVhv11xen1OOC4beslnVpEkF7EkSTdftmEymbHcbsixhPClwvkeRsl5vSNOUUTWiKmVfqqpku3vi//J//T/x9PRIZx/5539l+Jvv/pLH+w3n05dcXl2wWj3x4moEwUA0DSmtjyx0FSSbChRGjRhXV1ycv2S5+R7v9sLN955yNKdpWx6entBawm2bpuNx8chsdkZd71mutnz11Vfc3T5wc3PDZrNh6C1ffPEVi8WC8XjMdDLDOZl2tc5ijGYyGTMej1mv16xWS+q6FlpFNTpOej48PHBxOWcyLXn//necn5/TNDXb3ZIQHF999RmLxYIkSbi8PKfvO54WS16/+QxtEprIXQ8oxpMpj4+PZLklSbM/+Bn+o0X06RTOMxEhk0QEQB9EPCfiQwojIY59QPgXGTCIM/opAxMF4pDCfhB3t29glwBaEB1FIc9relgsEWyGjuzyDnwNbRS12z3kE3mON+L43heQa2i9bOuojEzrXsThF3NxZQ+9vE+SyWM0EuRZbyXk0yPu3mws4v56Jds0eOhWIv4enP7dHrapCLxpKm4vCqgHWNyBXwJroBQhN7SyL9tH+E1tcVZc1H3kiFcjEb1dFJCbvRyrEG/KMHLspxPYLGHdg/bi9ncIGmZ2EW+CB1juYTQSlni/h4dbQb/kZ+Ikn8Zg0aGXRkk1kvO7WMDQyT7e30Gw0V1v4vdJbCz0gohpWzm/44hlsQPMxuLav/0I9YPCm8DN1+K4zjM5ruUI8nmg3gvrPBCd7pVs5/RcuOWXqWB7piODtQObFbz6QuFDSjUqGFUl6/WOpnY0g2O97+n2llZJQ2E2lebG40MUju6ACznm7CFMpcFRL6WBY4w4/ImTF4OV7bu4gMeVPG8/wLyUhsvT00BeybEbTeRYBi0oHJXA9EoaRpsNnI1hu4XtQs5rNRJUy76DeQZFnnC3sFxeplzP53y8X9J2cl03rSNJwSVwcaG4eT3mu99s/tiP8qlOdaofUfkQno23UeBUiNgcRBEXAZ3fR7gcBcFP8BTPzHQRro8YF6w0Nr24ixG5Nzp/jWA4cII6ic5e7wPeHdjdgtQIGMF26JQsyyIOJDrPNRBZ3cIxl21K0oSQCJ7EO48PGqNEFNYhoHB4p0g0YJRg5PAo5SJyRqNNjkkLkkwEVQ9YGxhcxI5Yh/NWGNcKtHa/hwKRZHuNU1aEcudwg6BK1KchlhEPIsGfgRBkbk4pfxydRLl4buS466AI1mP7Ae8GEecPgvgRgyMO/SLPSROwQ0S/xOCdwwlMouvkgPJJTUYSA0wP7u9nNr7HaUtqEjKTodOIH/EKa+X4xZXZkaOolJKJg0RFUV4aIRrhxaMVQXuSoDAhkCqFValco4cLVAmuR3A1RC798/7aIDx774Xxf8SvaBUnLKILKciEhfU92glrPsTXCV6aCcakGJOjtIlNJI87wPwRsd54mT6QwNSASRRpSJ+vN+Ux2pCYRCY3fDhigEBhD8cS4TF21tJ7h/UiqKuTiH6qU53qVKc61Y+s5N/+UTWi64Qh7Jxns9lSFClFmVM3O8FRzK8wOmXohxjIOKJtOz58+MA33/zkGKjoXaDrGsqqZDwWl/QwWBaLBWmaMgyWoiho244XL17w8HhPWZXUdcPFxQVd27FcLnn16rUwqyfjGMy4Ji8MZVVwf//A09MTV1fXTMYTvPfs93uatj6uxYqioB8kN+b169c0TcNmsyHPZTqxaRogoSwr6l3DdHzGYB2vXr9hsXzEBsfQ1Vy/eMXj4wOr9YqgPF9+8zn/4E//A969u2W5uePxccVm3fLv/3v/iK4fMDrlw+0d19fXeDew3z2yWj/QD1sGW7PePDGdVXidsd1tURjG4xEQ2O12YgoKimHweN9idEaWFVjrWS7XgKIsK7abFffN9ojWGYYBpXsuLiaAZbW+ZzKdsl5vaFrDN998zWq1oq5rQgjkeS6TqwS++OIzbm9v+fLLL7m7u6dKMtLUkOc5q9UyYnoajNFyLZzNWS7l5z+8vSVNJSR0u91xfn7Ofr/nd29/x2QyBjzL5ZL/9r/7bynKnPn0infvP/D9d7/js5ufy/XgA8rEtXEAVPgEj3O4TjXz2QXBa5yFqpoIumcYMNpwdX7FeDQmSTPapuVsMuf9+w+oiWKz3HJ5dkXf9EwmE5arBZeXl3z48B4fLNOZ4G2ur19wd3fL9fU1Hz9+ZL1eHY1dWZbhvefNmzf89re/5bPPPqPv+/hzx363Y+06yrIgBE9R5AzDQAieruuOTPWmqSmKktGo5Pb2o0w/o0gSzXq9BODi4ow0Tbm9vfuDn+A/WkQfjUVYbBrBl3hEdMbA7FIERw94TeRTENOigL0g1Q9BjDqF+RhcAY8fBQejK3EsT8biUrabKPr2guHY14KBUUqcxNtaxO3KyOOyVLAvZQlFIq5yrWESmdaLpVwcgxdx05yJMOu9iLzdICK83cGqg8kshokmIgzniTyv2UJ9mAYeJIjTdbDfwHgqwnvvZFvLVJzxOGAC6qUgQPqtHAcQHjkNuFSc4PsFbL2ItyoRx/1+CWER33Msr10V4kTfLaLjOQUfp5ZVBo0T3nlTi6j76hU8fYRVDcN7Od5nE8i0BKNuekgTCRA1GXz4ANO5CNl9JyI4VhA4ZxdwWUrzYdjL+64EBUvnxJ3d9dKgwIn4HHZAFzj/Upoig4XUy/csExH5/Er+3zkJjPVeWOBlDA2ezgQn88P7mhBku85mAe8l7HRo17z9W5hcQTWGYeehQ5oXyDFzWqYJfIAhHsvRRMI7QxYnKRwUc6h3ka0veQj4iAbKDFydCfZn30ozZRWvV4ccpxARBTpic4YYhrrfwXwkTYjVUrZlei4flTay5usAs4mjzKFI4WGxJCt6bqbw8CgTAa9fZTw+DczGsN/VdPvTSPipTnWqv1v+4IqF6PQVxreO7vJDuOYh8PMgpB+kSx/EbXzAoQQn/699EPHcWXwQ8fiAdVFKwmoOoaDGyCLsUxSGhOj43+Ol6yiKJml6xJQIruPgmI7CamT3yWuFqBEfBG4TxdEkvraLiA8tvHQn42gOF/dVYZKUJEljYKbBORcd3M+s9gNb2xiF1yG64eXxSZYTUAxBHMYuPvbQMDiUUhqjDYL4djgfUHgIwiBPtMEohdKCBfEugAdrB4ahw7lBHP2KYzCswqCUfGVpjjcHnvlzSOxzo+JwDKXRkKV5DM48hGrqI/vbWkuWZpRFAfj4/hbv5fcAWifH86dj48Ik/2Zz9fFYfnK+0wDOBKwTVIrWmkSr6PSPz0MEb3lNYbt/+nqH/QIiKkdCT2VBakl9gvcZ1qnnQE/P8VzLjUxsNh347JF/f2goHAJHQwgYDV772CB4bjoYczjenuCfJ8ME43L4LArWRVxSz42BU53qVKc61alO9eOpd+/e87Of/oLHxwV13YobNgSm0ynL5SOD7WmbjrzIybKMei9hjMYY+l7CzW9ubphMpqxWa7bbDYnJGE9EPP/888/ZbrdcXV0BcHV1xWKxOK6DQNak+31NXTdcXV2htKaua+Gfe1njLhaPtG1HXswJQRjZ19eX5HlGWRUMdqDrO6wdmE4nNE3N0+KJ7XbNbDbj48eP1HXN69evsdbSdR0hBLquwzlHWY0AxW63Y7F85PLyHPDkeUHdbHn15pr/2//jv+Prr39KWUz4V7/8c1arLXW3pms9ihTrN2ij6bsaT81m5+n7lsfFO8qR5u7tPb3dslw9opIZ682Sq+sXPD0uefHiBU+LBX0/0HUdi8WSPM/IsiwGqHrG4wlFUcb7GyjLHGtbiqKkHwZG4xG7/QptAtYOjKoRs9mYLDe8/d07/uE//DOSxHB3Z5nNpsxmMyCQpslRGD/gNosy5/HxAaUgz3Pm8zn7/Y66aUgSmZB1LlCWFS9evOD29pZf/OIX3N19pG16rq6uj0K9uKwL9vuarnVkFyNGVcqvf/23/Mf/6L9iGOTeTSUh3g95mbpF/53r9fz8Upzo63d41/H61Wc8PN3T9wMXF+eUZcV8dh7Pd8tkMgVgNBqTJAl938X1sqeuBVFTVRXffvsts9mMthUeeV3XvHz5Emt7Hh4e6PteQmwnE/b7Pa9fv2YYBqbT6ZGjnuU52j7fFxVFcWyKNE0b75G0IIi6jrIs+eGHj3z55Su22y0vXlzR9x0fP35kNpsxHo/Qf/cQ/J36o0X0v/g/nkS6U53qVKc61an+t1SfusuVEna0QnAuR0yLE2SGIM6dBEQqS4hoDOvUEQEj3PKAdg6U8Lqt6/FhADwHD4M2YLRC60/CIWMdBMrD90/FZnUMFhU0ByDudqOOLnR5jdgYH5xkpjhZ+imTRBFfgkGDVhgtiBgfHAr7yXsRRWXze+xrwZQ8//7ZlCFOIY0YByQwKcFogz0ItCFEPE3c9vgz+TlHhTmg4+snGJOQmIQkureVMgRlcTgRznFoI8xtYV7qo3Au+ypfMgEgmDitEhHo8UdAevCCiyHo6NwGk0R0iorYlHAYeFMEDHnIBJtjDX3fY617xv58IgQfw2o1RzHaWgvBYA6if3ysMYYsivf4gHVOXN/mIJaHIys8ACZep1oZtHZH17eKyBgOeKEjU11BdKYf3PABH5tEHo7nXJxAgqEJcXeEn67U4Zo8nPzIldcqutL1MVTUOdl25/4Nq27vo7FdJi/wARUChufw3FOd6lSnOtWpTvXjqSwr+f7735HnZRRsS+q6Zrfb0veCO8nzgsl4wmazpW1E/BOhfUIIgbZteXx8jFzolPFoyv3DHWVZcHt7G8Moe16+fMlyuWS73ZIkCWdnZ3z//feMpxOGweKcY7VaMZ3OeDRPR9f64+MDWmvefPaK3X5N33uck2DLzWZNXe9lfRMCVVWiVKBtG16/eQWIMaLrOkG49D0hhOgQlu9d1zGbnrPdbTHGMEor2rZFa0Vd7xmNRvzu7W+5ur7hr//mL3j16g1952iagST11PuBxWLDn/z0c9I055e//BV/8vXP+PVvf8nl5Tl198Bi857v3/6Kr7/5jMmsBOX46usveXhYUDc1TdNxHY/PZDIh+IDWcv9R1w113fDFF18SQuDD+/dcX1/RdR3b3YqrTETwNE3Y73ecnc2i+F3Sdnum0ynXL66O07M/+ck3LJfieJY1ckLbNnz99dd8++23lGVF33f0vUwKvH//jsfHhyO32ySG29s7RqMqok3yo/FnNBqLu9qf0bUDSmmGwVPXG5arDT//+d/jw4d7Xly8IXhZO/d9j3UDymcRl/nJ7Q7Iujj+JEsq/rN//E/461/+M1wYePFiRJpkrNaLOBnRs9s2eC/rfRGsxS1+d3fH1dUlIE2GLMsoy5J3796R5xl1vSfPc7IsxXu5x/j48TY2ZRqSJGG1WmGM4fLykmEYuL295ebm5mhOGs/PWSyeyLIcUOz39VFEf/Xq1dFkI1kBO66u5qRpgkwhbFmtliSJkC7W6xV9//9HJvqpTnWqU53qVKf6314pRKzVKjuGxeij0KzROsSQRsVgRbg1Ojm6xa0FLTyPKKAGrOuBAR+ssNBxaK1IEhGkRRA+iKLPrOuDoGztc6jiIVTyiCYJz48TJ73HWRFAn18vvs4g2BPnQhRFVeRrx22NHDR14LwfkRs6uuT1ccH8LOyLm3ywLmJRhmcXcwiIoJpE17OIuCKm+uM+HbApChWZ8+LwcBEZchCTkyQjTfMjlzyNQZwB4ao7HdAIn9v7Axv+mcmulEKToEgii54YsJkej7lzFuf88XuSGPSBuf5JM+N5/6QOwaUg7iNZrD+7qQ84F5lmSOI2CT7Ie3GzB69w+OiaP7yuLFzxjsS56Gg/oGaepwxAQsCfEUNyc5NmGVqL5d3EaQV05MKrT8498bn64CQ/XIchTjBIw0R98ucDmP2A0xGDuiKo2Lg5nt9nlMyhjsczBEzE5R+mLz51tnv3vH98crxPdapTnepUpzrVv/1V5CKYfvHFF/yzP/8Lbm5ecnY2o65rcd3We5loVAmr9ZrJeAJA3/fH5n2SJEd033g85hAWf3jcfD4/itVlWVKWJc45mqYRgf2pp4jCfJqmXF1dst/veHx8JEkSPvvsDdvtBm0U69WKs/MZ4/GYvu8pyynDYIW9Pp9j3UBRFEwmE2azOU9Pj6xWK87PzxmGgQ8fPvD69WumU+FOH6b8FosnpuMzkiylafZY13F1dU63aVmuFyit2eyeyErNavtAkiRs6y3fve15dfM5Luy4vf2Ox+UTf/EXf8Fi846zszmPqw0//PAt/bCnGht2zZrxpORvfvk3NF2N0ik//8XPaZpe1o5ohsHStYMYPbKM9XrNdDJjvdpQliVFUdH3ltFoTNPuWCzWnM3PIGhevnxNmqZkWc733/2Os/Nz8rxi6O9oW2HDLxYLtNaMRiOGYeDm5obf/OY3tG3LYrHg5iZnMhmx2azE6V+WZJlw2VerFWEIjEcTrq+vub9/xOiCoij4/vvv+eyzz3AucH//SAhQlSP6bmDdr0l0RlmMmc9Tnp6W/Nm/8x8ym82od3J/k3iHMvoTjIuKxpLnP4egeHn9JXk2ZbPb8/S4Is9yXt2IM7zrO/rGkufi9t5stsznc4xJ6JqeUTnm+7d/yxdffsb9/W10+Tu8PzRbWgCGYWAYerIsJU1T2rZltVodHeibjeCLr6+vGQYRvK+vL1A6MJlMsNayWq24urqirmtGo1FsCg3sdjuGYeDq6oKyrFgul4zHE3a7HdPpFK0Nd3d3OOf48svP/uBn+GSBOdWpTnWqU53q39IS969GqwStE5RKUaRAcuR1A0f8SggO53qcF4E84HBe/hyCO34pDrgWESgPXGitDUYbtNFH94WIh7I1Smm8lwCgw59/z0l8FNOPWw/I44fB0vcDfT+ICO/8s3AKnyBM/CcoEz4RTzUxnjOK1inGZMeGgTxOmgQHgVVcEf4oegNo9cz8Pmy3CLfyPv64H5rD4lP2OUQR2h8F4DQ7YGSEmW70J19GkWYakxADYkNE20TxOuJcxDeuj42Jw3uLKB2d7QGsdQyDxdpnQffIP48Il8MN2eEG51NsymG8VRwjGUmSkKYpafaMhPHe46yVm5Guo2072lbYmP1waEaE45k1Snj1qTEkRqGCOOc/IRDFfVXHMdPEyOL6gP0xJnnG/6jncyLM+cP5DPEa8fEchYjocfjI9vfeSmipf8a6HJ4b4rTDoUnyaX16LA9fh+v3cFwPX+7AzHefvs+pTnWqU53qVKf6sZTRKVU5YbPeYYxms9lQ1/XzemFwaGWYTGZcnF9gjKEoRDS9u7uTNVAU0QUFw3Gys+s6ptPpkSXdNM1xrbdYLMiyjPl8zuXlFVdXVxRFwbt379jtdlhr0Ubx5VdfUDc7yqrA2o6z8xnWDlxcXBzX73leMAyOoXc0dUvw8ObNZ3z37bdHXMdh/VhVFZvNhu12y8ePH5lOp1xcXJBlOd570iRBKciynL4/OKkHkiThafFAP9SARWvLZvtIUSr++pd/gQt7/va7f8nbd79m8Dv+xV/9U+4ffsevf/NXfLz/nt7VtP2OtttjUs31iyumU0GNbPc70LBvapx3VKOSr77+krarKcqMV69vGI1L+qFlu13z+s0N2ijm8wvubrf85tfvubi4YT67wlnFZt3grOH9u3uc1ZzN5PgeUDfOWabTCcYYFosnvHdcXl4wDD15nmGtJUmE0940NaNRxWeffUaSJHRdz5dfShDnixcvBd+sAhcXZygV6PuWNDUkiWY8HnF3dwdo/vRP/z6z2ZzHxycUijRNuL6+ivcXCT6IAcp5mXb1x/uZEJGeAR/AB01RTPnpT/4Bs+klKCO/94LOubg4x3mLNrDf7yiK4ogGKoqMyWTMxcUF2+2W2WzGZrM5rokPQvpms0FrTd/3jEYj9vs9IQTevHlDnudcXFwcr2FjDMMw8OrVa8qq4vz8gjTNBW2ZFRiTMh5P2Gx2JEkap2hletmYlMViRdv2pGmGMRltO1BVY5IkpWl6Fov1H/wMn0T0U53qVKc61an+LS3BVqRonaF1hlIJwRuCfw6ZPIif3rm44CUK6T1KOZxrGYaGYWhFaIwi4GEBdBBrQUc2dhr/THSnJ6RJhtGpoEQwJCYVAThoyUEZvCA6eBbVnfNR2HXHmwpnA33fR2F2ODpyDoLkwXVzGGPtuu73QjZ9dIU/i5mWYeg/EU1FgBaBOIrbkckdgotc8QSFwrlA31u6Vt5DnqtI4s3OQWw+MrtjaJFJM3SSkSQ5WiXxeUiAqokgFQVZYtA64Jw0MLSGJDXHx+nIYXl2k/++kxyIrn/7e2KtuKX9cd+HoaPvW/q+xdoe5wacG7C2P6JLDteJ1oo0TcnznKIoGI1GlGVJnuekqTDSnZcwn7quaZqGvu8ZBuEaDsMQ8Sfiui8yQ54aWfwbCSDx3jIMHYPtsNbRdZZ+sPGmQUVOvlxTaZqTJhlamziR4CJ2po/O++dGiLjje+CAjlFx6kCmKfqhl6mDGPgq1/jz1+HQPrvK+aQB4Y4OfRPRQIeAWTsMDH3P0PfYYcBZe/zykS9/qlOd6lSnOtWpfhw1DJ4sK9hud9zc3FCNKgY7RJyEhFiGoLm/fyTPC5xzZFnGfr9nGPq4zrFHg0fbNnH6zzMajajr+hi+2LYtaZqyXC7JsgyAs7Mzbm5uCAGyNAdgt9tR1zWz2YS7u494b6nrHW1bc0BkvHv3Ee9hOp3RNh2j0YTJZEpZVuz3DU3kYR/W3c45uq47hmEqpfj666+xVgJPrbU0zR6TGpz3jKoxbTfEwHd4+8MPXF5d4oPD+p6n5QPKeG7v36ONxzPw3dtf87S8ZTzJqesFH27f4nxHN9ToBFCeoCTfqaxKrHN8+PgBHxxd32KdpapyVqsF3lu++OILksSQZQmvXt2QJAbnBW9zdXXFF59/yTdff83rV2+Yzy5J04oin+CsYj674O///X+X9WrLMASapqOuGz5+/Eie59zf37PdbiRkc7nk48ePEccyIstSsjzFOctsNiNJknh8WrIs5cOHD/zsZz+nLCuKsqAoUwKWosxo2h1JYri4vEBrxWQyZjSqWK9XRz55NSq5urrgf/7z/4lf/vJf0fc1BMdge4xWMiDqA893ESFOgoJWGWky5j/6D/93JGaEd4oiL+g6adC8f/8DNy+vaZqa169vMBq++PwzEmMI3vPhw3tW6xXDMPD+/TsuLs5Zr9dHRGdd7ymKgsvLCzHnpAmz2Yy7O3Hy13XNDz/8gFLq9ww/ZVnS1C31vsEYQ9d2TOP1WJYVo9GYxWJJ3w2Mx1OKvEBhSJOcV68+Yxg8Td0xDJ7fff8DX3zxNVlW0jbDH/wMn0T0U53qVKc61an+LS2tjbCklUFF7IdSmhD0UXQ9YCz80WnuCVhCGAgMoBwBiw9O3OnhIFpH1EXQEcGiIBgIhuB1/HN0M+tn1/uzM/25RAj+113k4SiM9/0Qneg9Q+/E/TKI+CmBlyKWys2FMAVFsD24gAeGoY3C7EEotoRgj9iSg5v64H4XAVnCfowxcVQzI03T6O6OzHgrLnlBkpiINkmOLvCD494YCTA9MLmV1sfjIkz2gCKQJuImUXwa4vnpdojAf0CfDLan61v6QfbtcEwO5zVJRKTOskQW6Vl65KCH32soPDdGPm1MHIRi+dmz2/qZg67j9stxs9GFXtc1db2nbVuGYTgKxs8YlGdHeqIUqdZHR/5B/O8HEbf74dnJfXCD68gUPzRxDlMJSqmITBFnzadTBc/nWVjoci3L1+8Ftir1yfX4jHY5hoeGw7TCs6v/+VqOGBfrsP2AHQZs3zN0PUPX4YaB4OT33p5E9FOd6lSnOtWpfkzVtT3Wetq2xZiE/X7LMHSYRDMej4+hlp+9+RxjZFqy73u01pydnbPZbGia5igmHpy9WTQ4NE3Dhw8fotFF0TQSHlpVFdvtNqIwNvTdwMXFJd988w0fPnxAqcAPP7zl/ft3MQxS7gnquuazz74kBM352SUhSHpO23Tsdw0vX74mz0uatufDh1tG1ei4dpxMJiil2O8lULIsS+7u7gRrYjuarub27pb52Zy7h3uqakzbDpTlGFRCCIJQTLMM/f9j78+aZbkS7Ezs29vnmCPOfEdMCWQWWawqssk2sZtit0zWphf9Wj3pgTKjWk0zFtkssoasTCABXNx77pljDp/3oIftHhEHmUXwQTLJkL7M3M7FGSLct7vDPNZe+1uex5u3b3maPxHFIRZLWZfMF0/kRcY23RD3QtJ8xy7dsNmuiJKI8/Nzyqrm6uoFk8mEz3/xBQBhGBDHAVmeUtcFm82Sk5MJvi8ZDntstitevXrBYNBDSksUOT637we8/eQt69WGsiib9PSAk5NTptMZ/f6AzWZDXSukFERRxO3tLefn5yyXSzzPYzAYMBqNGAwG9Pt9LJqyLLAY+v3ePjH/Z3/2Z5ydnXN9/REpfd69e0cQ+Gy3K/Jih5CGh4c7lC7p92POL84oq4K6LjFGMzuZ8MUvPkFKgx8IksTnf/3f/i0fb97jhx4O0anZpVmDMHRtQWAOq0KtwGifX3315/zi819RlYqyrHj18oq3b17RSyLyPKUqM1arOaNRn8EgJghkg4MUvH71EmMU4Bj6QeCRZSmnpzOGwwFa1+x2W9brJauVM9x7vR63t7ckScJ2u0VKyXg8fraKNs9zHh+fsAam0xnr9YYszXi4f2Sz3hD4IYPBkOsP1+x2Kff3jwRBxN3tA5PxjLOzC5K4T11rrBEkcZ/BYPST93DHRO/UqVOnTp1+pnLIEr8x052J7pjhEmsEpsVV0GA2BAhpEW5NH2AAibUK0xjm7r9dqqPFuVg8sAJjJMY0Jql11ZZSCKSQGGGecaSf86Rp0tLyiMPHnsHt9k007OrGtPRASIUzqR2L2miL0nWzX55LjBuNVgZt3NaiShCOP+753t6gPuZXu33k8FpHSBdrLbpJJxtrsG15a8t9p0HJNAasK79xD5MWi/R8V9bapNyFbJE0bmklxhUvGWOb0ihJW2TaNs2799INi10gUKgGp+MS3QYhHALGWJcykY1R/Txh/xxFcmwMO5PdNIb8EetdiAOr3I1Is/8WpRVFWVCVCk8qrHaTCIEf4DcfBoWUiLbEVACyKT3V7XXQGNHN8lKDK4lty1c9z8dRfA4FuTQTIOboOA4oF3dttwl+2+BcLAZkM1ki28mgdh8ESmkEggPC/Pmkw+F6OVw3hwkgd/1ac+Cg76+94wu/U6dOnTp16vRHo9PTc4xVjhldlnuzOwgCTk5OWSxW9JIeQvhst9t9yGEymWCt3aNdHKO7v19BOZ7MHMe8WbG32+1IkmRfhr5er4miiA8fPjgERhix2WwYjQecnZ1yf39HksRYawijgMlkzNdff03gJ4RBzNPjAlW7ByLfDwn8iLJU5FmJNZLJeMZut6WsKoIgoNfrsVqtODk5oS0UbVP1RVEQhSGZKuj3e9R1zfn5Bev1hiTus1w6A/Tm5g6vCTg4s7PmF7/4kjyrCIOQr778Jb/+9a9JkpgXL16SphlFkWOtW9GaZRkXF1dcf7xFmScm0xmj0QgLrNYrsOD7kv4gYbVekPQitFHkReo++1gYDGOqquL9h3f04j5aV1RVzuXlKet1zma74O3bt2y3C25vr5lMp+x2a8AwO5nx29/+liD0WG+WeL6gViWbbc1oPCCKA+IkZDCYMZmMqaqC1XrB2dkFWer49VdXLxiNJq5LyQswRnF6NqMsC8bjIdYqHh6eWK0WbLdbXr9+yWq1ZjqdUhQFf/kf/h1ffPYLkigk9EKuP3zHr//+lH/0j/6cMPTZbnMenlZ8+ukv0Eo78KWU7hnZus8bYRBisPxf/pf/Kw+Lr7l//IbFoiLpxcRxSFXlfPHFZzw+PoHQrNYLXr9+wTe/+x1JErLbORN8OBwShiEXFxfkec7d3R1ffvklSrlreTAYcHvrEujb7ZZXr17tP3dsNhum0yllWaKUoigKtFb0+z2MMUynU6rm2pNS8umnn+5XJBdFwWg0JcsqPBkS+DFnZ5d8vP5IXWmm0zM+Xt9R14pXr1795D3cJdE7derUqVOnn6l8zxmXrZHuktFNivqoSHOPMmk5555FSAPC7JPobXp7z5I2rlzSFZQ2r2vkHhdjjzjhx0z04+TwcXrYZZIb09MK7N64PE76yr2ZvTeEfZCeQErrlm1ahVIVVVVQlBllmVHVObXK0abEohDCGcxCHLjXzixu9/Ng9P94M4Ymfe4wHjQPm+2+amMODOwj41R6Eun5BEHkkDeeWxVgjo7RGIOqHP7DaI3v+YRhTBhGBEHQ8L89PNma2BZwKwjalQSOZd+eOwvC0nRv4vkC6bn9dcz69nUOaXch2KNeHDaneLY099gsdkmQHxnLWj8zjd04sv8Q5zA5rtxUNugaXwoa1HvD8W8mJGgRLSVlWTVomLrBpxwVuSL3Ewv7F4Ej898dN8JNDO2vKeEmSFqGP22ha3Oc7r1dsr4sqmYf6uaD4LFx/nxMlFLN0my1x7z8/gTF8wR7p06dOnXq1Onnr9VqjdGWPMvZbrdEUcRgMNgnxdM0o98fkaY78tyhWrIsa4zENb1eD6UU4EpEfd8jTVMW8zl3d3dUVUWv19unefv9PvP5HCEEZVlydnbGsElBh2FInuVsNpuGU+5wGpvNiixLOT8/I/ADkrjPaDjGaOglQwb9EefnVyil2ax39HoDBv0hL1++xJOSIAjY7Xb0+32m0+l+f8uy3IcZEDCZTaiVIs0ziqpksVyz3makWUHSG/L6zaf0h2OUttTasksLLJK8rCjrmuubWzd5sMvY7HYEDbJGSo+PH2+R0uN/+3f/DmsF282O7cZha+q6ZjabYXGGb1FknJ+fIiVk2Y7NZkVRZGw2S5IkJs8zLi/PMLbkT/7RLxiNe0jPMF/cIj3DejOn1w/RJsfzNHm5oddzzPfXr1/z2WefMZ/Pef36Nf1+f18oaq1lNpvh+z7D4YCLiwvKsiJJEl69eoXWumHInxLHCdLzCMOAssrYpWukZxmN+oBGei4UpVTNZrNiMOjxNH/g5ctL8mJHnASs1nN++ctfEIQSrev9c3O6yxqMpsFo06TRbYNKBNF8hpvNzphOThgOh5RlzuLpgc8/+xRfQllkjIZ9JqMhnoT50z2vXlyw267wPMF4POK0mayJ4oi6rhBScHd/SxD4/PDDOwCCwOfh4WFfRJtlWfMZyNt3MrXc//F4TBzHjEYjtNZNIWvI+fk5Wmt+85vfoLXmiy++4MWLl0jhs9vlVJXi9uae+/snNpuUdJfj+xGDwYj1eveT93Bnonfq1KlTp04/U/n+AS1yXNTpUtKNsXeUOn5u6hks+ij93eI99MFsxmFiZFtYaltUzAHnYn702i2+pDXCD8lmt297E7IpwXQ6FFxK6TUIFoc9cWZwmzZ2iWMwKFVR1y3ru0SpAq1LrHGIGiFMY7q3zHXHqwaaFI9LjkvvsH+HcdD7xLTDsvh7vIsLQEs3CdAsQ/X9YD+h0RZitoWozsA/YFTqusJaTeA7wz2KEqIowW/Y357n4Qd+c+xgcca5G1v2ha5uHJpx4fn3LYYfp+7bffmvFY4em+h7BIxWbmsLsbR2aXsp96Wfnuc/w+H43vFEgEA2+2qtxhq38kAK92ivtKYqHc7HYXx08+9DCerxVdsEz7H2MJlhOayCaNnubTGq73v7c9sOokMe2abQtqYsKoqipCwOEwr783+EKXJYpAaBo/R/5ZoXz675Tp06derUqdMfh6w1XL24wg/8ppxRsV5vWa22WODy8gKt3XNdkkQMR31OT0/IspReL9mXvPu+t2dql2VJGEWcnZ0zm50AgtFoTFlWZFmOUpogCEh3KZ7nUurgTHhjII4SJpMJYRjhShgdc3q3SxkMxixWO07PXyBkANKjUorziwuUMZycnzEaj3l8eqIoSvK8whpBkVdkaUGRVwwHzoA/mZ1RVxopfCaTE9brLXf392y3KU9PCy6vrrh68YJ/+k//At/3sFg83+PkdObGqq7J8gzPExgUZZVRq4LheMiLl68YjadoDUkzeRDHSVMyuUFrxd3djQt0eB55lhH6PmHkEYSCKJak2Zo02zAa99GmZrvb4HmC2WzKx4/XXFxe8Le//mv6g4gs3xI1xZm+L/nrv/kvnF+coXRFFIdUdYmUcHt7hzGGX/ziCwaDAXVdcX5+hlIVxtSMRn2KYkdRFuR5zsuXL7m8uGI+XwCC+XxOlu24u7thvZ6z3W6wFvq9QdMJZJlMxrhyVg/PF3zy2RvWmyWvXl3ie4bT0zHbzZLJdMB6u6CqUjzfunCR1aTpmjzbYY3GGgPt5x1d43kWbSxSSEK/x5/+yX+P1CNGg1MG/RFGay4uL1itFozHAwaDmPv7a4JQ8vHmA54v0FoxmUyZPz3R6/X2vPx+r894NMEYSxTHFGVJGAbMZlN6vR7D4ZDVakX7GXa5XLNeb6lrTVm60NH86YnhYMB8Psfz3D2x3e3Ii4K3n7pJmKflivl8yadvP8MTEl/6lHmJUZo/+eWvKLKcDz/8wHQ8Jt1ufvIe7kz0Tp06derU6Werpnyy+a8GNNI8i0isA2qA9YEAawOs9bBWNo3sdv83Zl+mCMZIrAmwzd9J6RrOW35hi36xuFS68xtbNvghGd8m0V2JUruPyhWbaoXVBqxA4opL3a636QiXAnd/K/alpaJBy2AN1jjuuTW1M89NhbUVWIVAITFgNEY5A91qg7C2SScHSD90+xsESN9v3tklNIQEz5dIabG2RpsapWtnogqLMgaLQEgfZAAyRHghvh/iewGeazzCGprySlfoaaxC+pYwCgiDiMCL8WWILwN86eNJD0/6e664NRZrDEIYPGnxpMATEg+JdBltpBB4QiCFRaD3WJJjY/w5nuQoLS0sWhg0Bo3GCDcC2hoqVTtuuaqpTE1tNG7ew0N6nkPlBB5+4L4Kz12PRgiMkFjpYYRAAxqLFmBECxKyGGUQ2mCVbo6TfRGtrhtUisGda6sxtsZol+LXymCUcWNjW6yQS+H7zQeo0G/GtCm5FVbiSb+Z5HDXkVKWqtZUdU2tLFqDMaK5P5p9bd/faLRRaKNB4F7L8wn8wE0iBAGB7zVMeg38dHlRp06dOnXq1OnnIxlY0nwLEs4vLxlPptTKsN5sqeqauBdTqhxNRV6mgOXh8R6LYTjqs9ms9sGIosj36JSz03MCP2S72bHbppyenGENZGnOdrMjChPG4ylJ3KOX9MmyHBCEYURRVGA9iqImivoIAopccTK7oKqgKC1Jb0yaVwgv4Ob2jof5HOl7pMWOv/rrv+J+/kia5UwnM3w/ZDgcI5uQSVnWRGFClhVcnF8xHk2Jgh7jwQlJ2MfDJ/QCPAS79ZIw8JhOhnzz27+jKnaURUoSBUyGA4RVSKnwpaIqNyzmt5zOxtRFAY1Rn2UFYRQznZ3sVzSOhgMklvfffwdGU+U529WaqkoRsmaXrRhPepxfTNCmwJiSi4YxHoQes9mUPM+R0qeqNdtdRlnW9PoDBsMJ/cEIpSEIeywXO8IwJMu2/Nmf/Qme54JNq9XKldlXBUkvdAn2gU+c+Nx8/MDFxTnb7YYffvgBawXz+YL7+zt26YZduiKMIOlFQMBynRJGfaKkzyeffk5RVkxPZuRlhvTg4emWNF0hqJHU1FWKtRVZscFQg1QolVLXW77/7u+QlHjSIIGqrEi3G9LdCqNzrEnd32uPf/LLf80/+vJ/YjR8wenZK3ZZyXqzZXY643F+R21LlC1RtqA3CJCe5erqCmMsSdJnsVg16E73+a/fd5MBvWTAZDxlu93SH/SQEs7Pz1FK8+LFS4aDMVHYIwwS+r0Rn3/2JZ99+jmfvn1LVZRopViv13z8eEsYJ3hBRFZUfLy9Q/oRV1cvqcsSqzW31x/4xWefcjab4kv41Vdf8OUXn1GXKY/3H3/yHu6Y6J06derUqdPPVBaBNRYhDFporGmwGtJvErc+4CFavIotsdYhqo21zpS0LbZFOFPYtkgYH2N9Z6QLf18I6XjjbUFja5K3aJTDvrXGbVtw2e6xNQqDRIgDy126X4J2AsAaZ6iaAGMEGIloWOoSgxWW0LdoYVHaYLTGEdpBGAVaYqXACg8rawxuQkEEwr2OBYvnEB+BPBjKWKTw8JsPL54nQVi0qqmNQhlXMIppLGyJ+2p9jPDxpO9MbSldew9t4rkxsLUgDD2EDBAeSBG4clYpEEiX6MeZ5m6z+6WYUpg9HgXhJhza1QbHJZ7WaIwRKO1Y3W3q3BhDEAT7Y21LTV1cvEXpC3Q7iYFG14dEu1IKbQ2IAOl7YMHzfaQv8Xw3FlZYdJt0x61laBeNIgTW87CewcpmPJRCGotnQWjbzqO4a8ezDhtkbLOCwKB1hdKVO67G4ZZIPAleg2vxPUHgeY6xCa54VwqUcUfmxtO9kbHa4XmOSlHdpBMgHE5GNwWmBoMyCq3NHhPji/BwXVu3sqMZVIw5pNk7derUqVOnTn8cSpKYNHWokzzPefXq1b5PpSxL1us1cRyx2azBgmrK4sMwRNWurHO73bJabQiCkOvra1St2W633N7e8+WXv+Djx4/keUFdO9Z0vz9ACEGWZQyHQ+5uPhJFEdZagsAnDCPSNKfIK2bTM1bLDVXpntOLQjGf3/LVV1/x8uVL8jxjMpmQZxnTyZQszXn54gohBHVdkxoNVvD0+MR4PCYKY9I0RWO4v7/n8vKSdoWpUpp+f8DZ2Rmr1Yp+v48xivfv36O1YjwZs0t3GGuJwggQ7HYpX/3yK25vbjHGMJk4XExV1dRVje+FhGHCYvGE5wVUVc1oNEFKD2OgKEqenhacnJyQ7nYsF2suLs8oiwqGAq0MVZlzdfWS9XrDarWm1+sThiHGaE5OZvR6fX73u+/Q2hXEVpVj3GdZynRywuXlFXEsWCweSNMcra0L9TSfk+I44mleMBz2+PDhBwaDAZPJmLJMeXy855//d/8CreG7775zq06tZjweoZQrhT07f4FSBoGPViWD6YggiFhvlnz77bf8q3/1PzCdTsBq4iRhsVxijeTyxWt+992vOTm75Otv/w5BzHy+Aa8gKxZ4XkRRlgjhnrUXyzmL1RNCSvr9HoPBkP5gxr/4F/+K//Uv33Nz91vGkz6eB5vtslltCcPhACkFcRITx45Zvl5vmM+fmExmDAZDgmDOyckpZVkhpY/nSdbrDWEY8eLqBTf2hiDw95MPr16+4elpzm63o64VWMvLl2fstjvWmw1pnnF6ds5wJEmSBK0MYRCRJL0GLepWi+Z5zsvXr/DDgMFoyP3DHS9eXPL2k9f87d/+DV9+9YufvIc7E71Tp06dOnX6mUpru08sg26wFQd0hWxY3u1XiwI8tHGmLdb9vsNuSFdOKoIG4eI3JY9eYzw6VvkBu3KEavkD+/aHURYGrVWDH3FJc4HFNgao+5Nj9EvTOrk3kgV4jn8tjEQJkBK0FGjrNXatANnwtq2m1gor3ESCaBLKWItoipmMFY3Jrfdj5fjZDf7EHMo3bfM9h3h37BSHe3GbbNEfTemlsYfUt8AiZPM3zrd23PCmzFQ3yBRj1DOedosFaVEhNG/txsIta7Rtgaw2GKuoakNd/5jXramV/+w1mxkOBOA3Ey20+Bll3OoE0zITDUY7jIvwPCSuCFQi9ka7OTajRfP9prxWcHjP59zwY/NfNUx9f1/iarTGao22bmwc2ubA13fjJPYMTiH8fZHrfqLg6Jq0FmzLhTR2z113583SdggI+Rwds+ezN3x/T7oCqBar45LyokEbuQJZa//QPdCpU6dOnTp1+rmqqiq01nieR1mWzOdzqqri9evX+0LFfr/PZDKhKFKqsqCqXE/N09OCXtLj8vKSzWbLarXkzZu3bDY7Pry/ochrvvn6Oy4uLqhrxenp2bPn14uLC7IsxTbPZWdnZ1xfXxNFMXle4MmAj9d3vH3zOcvliuvray4uX7LdrVku51R1ie95DGczqrqmlySsN2vCMHT4l82GqizwPI83b95wfX3N5eUlRVEQxzGXl5dNv5Ar/ZRSEEXRvizS930eHudMpxPCcMBFdM67d++aNH3Gyxcv0WrFu+9/4PzsnDCI+fDhhqr8wBdffMkP767R2uIHLhBxcnLKZrMjilzafjFfcnF5ydPjApCsVlsuLl4SBgFRFKNqQRwNeXh8IM9vSJKEOI4Jw5DJZIInPRaLJXVdY62h3+sRJwnr1YblcsXl5RXv3r1jNB7z8eMtg4FL/z89PtHr9VmtlmRZhlIVYRjQ68ekqUPGgKUqFcNRzA/vvyVLy4brLhmNRwwGQ+7ubhtm94K6tqgaVqstUvoURcZ6vWQ6mfLw8Mh4PEYK6Ccx3377HednV2x3Gz759DUfb95xc/c9dW0pcsUu3zBffWAyOefjzQ2D4RAhBP/5P//v/OpXv+Lm5pZ/+S//JQ6XWeL7LoCy2ax58fKcPN8ShhHnFycsl3OiKEJrQ78/oKpq1ut7+oMRo9GI9Xq155o77OKha+vxcc4/+Sf/mOvr95RlhecFTZ9SgB/4gOHq6qqZiEhYLu/Bas7OzlBNMe5sdsJmt0Ur9xnrZHbC7e0duc65OjvH8yRv377B8zzA8uLFC3a7LVme8U//2T9jt9v+5D3c4Vw6derUqVOnn6m0tk3xo0Fr2yRd3M8OfGjZmOHyyPz2DoWh1kMgm8Sut8ddONa198wMPzbQf/z945+15nu7Hd5XHLHCDcYolG4wKS1qxAANhsZoi9GuiLRNvrcMcuk3GBYvQPoBnh/g+yFRGBNHPaIoIQzj5lgPyJs9WqYtBJWH4tNj1MmB3V43D3/O8m9f73A8zsgX8lDcCnZvite1cun/liHuBQ3uxv+98Toe0+OJCtkY9FIeuPHyR8z5Y865MbVLbauSunabUhVlmTdbRlnm1FWJ0cotANivJGiS8w2+xJnCxk0CuPkJVxS6L+tsGPJauWJaVWO1AtOWgzZp+HZrU/ZHpaTtJIEnHSImCFzxVTtp4Mpc6waJ45Aqjlvfbna/aW1QzSRCXdfP2OqtjDHP37cZSyGPOeat6S/3CZf2Wve8pqzUc+PgefKIrd+OpVvG2qlTp06dOnX645HW2oU0jGE2m7HZbPA8j7u7O4QQ5HnO3d0dT09PqNo9/06nsz3veb1ek+c5FxcX9Pt9oigkDEKKomI8nrJe74iiHpv1jt02wxjYbDb4vs/t7Q1lVfDm7Zv9CsQ0zZjNTuglfS4uXtDrDRkMJgR+Qhj0mD8tGA561KrcFzuCYDadsd1uqetqf1xFUTCZTBBCMBgMiOMYrTXn5+cURUFRFKRpSpIkSClI0x2DwQBjDJ7nMZ/Pkc0YbLc7fvvbr9lsdvT7QwI/oSgUZamIoz7ffvuOJOlzenqOlB6b9Q6lNFmWk6UFeV6C9bAGrBFIEZCmBb1kQBjG1JWi3xtxe/NEXUFVGr779j3GCJrHVHa7jCAIqeuajx8/Utc1VVUxHg85OzvFWE1dl+R5tj/GqqpYzOcOY1Mo/v7Xv2G7TZk/LVDKGctxHCOl+1yxXC75cP0D6/WcLN8yGvcRUlPWGevNgiD0Uapmu90xHk8ZDadgPQb9CavlFmsk1ghub+/w/YAk6bHd7hBCUNWqCXAIbu9uKYqcy6szegOf5fqW+8d3PC3eIf2M65vf8O33f8OH69+SZXO+/e5vub1/RxhDUa25uf2estoipGK1fuTq6oz+IOHx8b4pRh3ycP+IUpo47pHECboxyOMkQmnFZDJhOByw2+0YDofM53OenubMZieO2++HbLc795lKyCb9bxqGek6cJJyezlCq5ttvf0dd1wRhyGQ2ZTAcIoULQEVBSJqmjAYDFvM5b9++pSwLwtB1IX3//ff8m3/zbzDGsdkthjiOubm54euvv/7Je7gz0Tt16tSpU6efqYy2KGWaWf5DWvbHepb4tUBL025MaSkDl+AVfmNWH8o2jwsSf5xCb178mUH54797XrAokKI1fdkXmbaGc5tUMFrsGdZCeE0q3vHGgyDC90I8P2q2EM8P978TBDFRlJAkPeI4IQwjPD84Mpptw7Y+cMLbQtHjEsnj4k2ldGM0e0eFle1Eg2y80pbb3R7PwcB1KXtvz4tvmfHPTfx2hJ4XUh6b+iDwG5O5neT48WSHaQznuq6oqoKqypuvJVVdUFY5RWOmV1XeFB8pjD6Y5m4/3Lk1RoM1B964lHhS7otDW3SN1gpVV9R1hToq52zHUavn5aTGtOx52RS9+ni+tz8+z3cJElfKWmOMRmndsCbLplC23p+jqqpRylBXirIsqarq2T64ze7Pt5Tes8kiKdw94c6HK+p1Bb2HiZV2ssEhkzxXjirE3nxvJyGO77lOnTp16tSp0x+PWmRLFEVsNhuklAyHQ5RSCCE4OztjPB43Ce6SXjJACp+yqBG4YvTNZoO1hvPzU+q6JIwChJTc3jzwySefNc+Snlt1qTVxnFAUJePxhPPzC4SA7XaNMWZfIFoUFUVRIYXHb3/zDUVR8eLqNVeXlwz6MUZV+AL6cUwcxBRpziDpMez1mQxHpJs1qq7IsoyiKFBKcX5+TlVV3N3d4XleU4jqik17vT5xHON5Hnmek6YpeZ6z22XUlWYxX1HkNZ70WcxX9JIBdaU5mZ0jRchkcsLXv/0WVbsgi1LuqyuuNPR7Qx4fn4jjPkJ4rFYbLi9f8Pd//zX9/pDdLiOOe/R7Y4pc43sJgd/j8WHFarVjtyuactSculJsNhuSXkJRFkgpSdOU9XpNkiScn59xejpjvV7y5Ve/YDweO2N5PEZbUMbwu+++I44TlDZYAf3hkKfFE0m/R5rvCJOAIPLwQ4+HpzumJ0M+++ITJrMJf/4Xf0FVK/wg4tNPv2A2O8PzQu7uHpnNThoTe0QURXz66WfEccJqtUEpRZ6XaAPS81htFlhRMz0Zcv/0nk32wDq9ZTST/Mf/8m9Zpx+p7JwPd3/H/eJbkqHmP/yn/wfDqeDdx7/jP/31/5PH1fdc33/NfHFHELjPemEUkqYZ0+mMMIjBSrQ2++u5yHNWqwUIePv2LWHos16v6PeHnJyc8PDwSC/pM5udcH19Q1m6CSHfD/n88y8YDl05rFIVj4+PzbUU0uv3QEos7rzXSlFVFaPRiNl0ymq1IkkSfnj3jjdv3lBr9xlDW82LV1ck/R5lXVCUBcIDZer/Joe8w7l06tSpU6dOP1O5gnVn7lnpihnbyPUxEsQ0qIx2yWdbAipFa+Y2fHMOJaCW5w7gcWrXaxAkrbm7Z6X/A2q53e73wZnKbbpc7k1O8PaoEosk8MLGdJd7jIiQgDUYKxEWEBahJdJzpZxBEBIE4d6objEk1oLSGkuNkM856C2uw+FPWmP/UMjpTFdnYPu+3+BG5CHBLNxraGPQSrtltNpgMQeEyNEKAWyLdTmk8o8nIp6nyhvTV3jY4GD+CvTe1LWYPTqknRCwtIntgwncHkt7LoUM3ESFVhgpENa6saYp1dTa8eab8Q+8loPv3ssTDunirjHjOOPGYKR7fyEapE1zLWlt0EdlpzSFoJ70QLhJiDb5rrVucCwGbTXa6IZNX6Oqqvl74bArSmO0JAgiNAZjaEqe5LPxFPtrv8W/QHvZ2j0M6DAZ4SYkmnPZTj61KyI8iWwnEYSrSnWFuy0GSDw7p506derUqVOnn7/aNPNsNkNr7VjnSgEO9RLHMavViqqqCPyI5XJFGLqOlfF4vEfBRFFEmqaUZYkQHhfnpxT5Db1eTFWV1HXNNJ7wd3/3N7x9+4ayLJjNZkRRSFmVvHz1in6/z+eff06RlyyXG7TW9AcDgiCmKEo8zydKfCItyLKcvEhdkMELiKKIp6dHBoMem43721evXvLdd99jjGG73VJVFUmSUJYlt7e3hGHIYDBgMBigtabXS9huN/R6idvf6QwpJcvlkiiKMQayLOUXv/glm3XK3d0dr169xBjLerUhz3PCMHbJ8mYSoq5rkiTh8vKS6XTK+/fvMcbw6tWrJpDj2OxRFJHnOdYIZrMz1uslg8EQz/N43e9zc/uxCU1IPny4Zjqb8p/+9//El199uQ9FPD098jd/83f863/9f+Tl2Ws+fLgmy1Im0zFC2n3w4+ryBXGccHd/z3DYJwh8ttsFaZZxfnFKGEQ8Ps45Oztjt9sg8Pjmm28YDiaMRhNubm4YjZxJnuU54/GIm5uvyfOi4bN/zfn5CUHoJmQ8/zV5lhLHEVmauXNelvi+z2L5xGAwYLPNGI2GgODx6Yb58pp/9+//7xR5zdXVFXlekGU5d481l68Siirn62++Rtk5q/UDZb1GqdqtYK0yrLUsFks8T7LZbDk9PXHXel0RJxFFVeL7kqouAVDKhVrG4wlYgecFzOcL+v0+w2Gf9XqN58v9KoWiKBgOh2it2aVbalUi5BBtrTPGjXb3TJNCHwxG1LViOBggrMBoxYfraz7//HP++q//mouLM5Sq9v1cy+WS9XrFbrf7yXu4M9E7derUqVOnn6naJLkQrhyUtl7SioOZ2jCdHZlCYLVLsO/LRK0rIHU8dMc/P05Dt4YjHBAjbbrcGLNHjbQ6LrI8NqHdz57vf2seOwyN43NLAcL38b2QwAvwhIfXpJSl52OxGKtR1kNoh+DwjHbpaK/luPstRR3fE894fLVqTPR9iph9mlnrGpcoPxxDa5467ErgUDJSNoiZA8vd4cXt3gS2bdHp0c+0dqsG3MQFwGESoh1rIQVW2T1Kpv0+wt/zuVsjXEr5e+WVx+n/lhN+fD6Oz5NSquG8u+8Zr2GeIw9lqMaZ274fEASBS1/jlmF6QtLy3a1RKGOhKVB1DrnFCDe5Y6ylNhZ9fLz7QtWGZQ9N4rxGWuPGwlpqXaNNhdY1lSpRddXw19vjdCgYa+sm8Q9uMqjeT3S4MXO/565nu18NcVipcegUMNqgGq58e0+1D+K+3yBdhGjuvcN1r5Smrtux+2+8kTt16tSpU6dOPwudnp7S6/VQShHHMXmeE8cxxhhWqxUXFxcEgTOpiywny4r982scxwCMRiN2qTP79B7RBycnY4bDHl9/81uiMKQsM7766kuCwGe1Wja4Osl6tcHzPVarFUVRNlhBQb/fR2vDbDbh5uaO/iChqrcgNEJa3r55zXab43uODZ4kPYIg5OHxjrOzGWXpjM6iKHh4eKAoCsIwpNfrcX5+ThAEjXneoyxLBoN+w3t3xurV1RVZllMUBYvFsjH/a8qi4uryJXHcByxlmQMQxwlpumO5XPPmdUJZFo2xHnJ3d0ccx/vPK0kS88MP7/F9j15v6Njw1uJ5Ho8PD8wXc05OZgwGA4IgJI5itNFcXb5ksViglSUIAj5+/MhkMuH09ITtdsu/+lf/2gUoQstwOOD+/o6Li0tms+k+JDIYDnh8emIwHNIf9Fmt5oDbD6UMSTLE80Ly3CX5e0mfKI5Yb9e8/eRTnh5XvHz1ivu7R4wx+IHH5dUpk2mfMPS5urpguVxwejajrjUf3l/z2Wef8v79O4bDAQiYncz4eH1NGIYsl3MWiwWDQY88z7BW8/nnb/n7X/+aV6/eUOsltSlI8yVv3r7h3/6//m9MJ1O26Yqw9xlP333PYv6RWlVst1tGoxFJnFCWNXEccXp6RpalvHnzmtV6iZSCs/MziiKnrms83+P6w0dOTy+oK01RlGhtmwS/x2azY73ecHIyRQjBdrttrl3BfD4nTVNOTmbs0h1JP6GsK4SUzE5OWa/W9Pt9lssFl5dXrNdrTk/O+Pbb3+H5ksGwz4sXlwxHQ7I0xQ88ojjg4eEBgLPz85+8hzucS6dOnTp16vQzlcBhKJyR7tIUcChubAszHaZD0ybAnYnrI0WIJ0OkCJrNff/4NY4T7fuSzGOD/chAP+aI10dIj+O/a5nRrY4NZGfSB0gZ4skAT/j40nHE/eZ7/h7rEhOEMVHUI4p77mvUI477hGFCEMR4ng/ClX5yxA1X5pB0bjEyh6/6gB1p9r9NzUt5MKg936XiLa1B7cbX91v+/HPGuG7SMcf87gMz+zDOB8TNwWx2xrbYn1PHLH9ePCql3HPSW754u8oABLbhhRvTrFyw7goStil3NcYlvauSosj3xVhgHY+9wbl4UuJ7PoHvEQTuq5QCsz9GvZ+0AYdD2bPllWpKV52kkEcolPbaaHn5zfnRNbUqHZ6mLpr0VUFdF1RV0WBrKqpKURQVVVlRVS4F1qbB2mMxR6sLfsxM368+0G61hDtHNEa7bVYjyAZ/5B3hW9jfY+1xtteS1p2L3qlTp06dOv0xKQgCisKZvb7vkyQJWZYBsNvtmM/nGGMoyxIpfV68eEmS9Dg9PafX62GMcbzn0YCkF7HdbvGkIAw9osSjrFOkNHiBM44Xizm+70pMoyjGWphMp7x9+4kLItQ1k/GE6XTCarXA82C9WZIkIdfX7xDCsN2smE3G3N3eMhwMiMIIVWukkFRlBcai6prHh8e9gd7v9zk/P2c0GiGEYDQaIaWk3+/veezHz0VlWbJYLJyJ3Bvw8uUrwjBht02R0oeGs/7wcM90OiGOY0ajEf1+H2sN292aIJCMRkNevLjizZuXCGE4O5sxnz+idM2LFxecn58Cmul0jNYKzxf0Bz3+4i/+jLdv3xAEjlf/7t17qrLm48db/smf/gW7XUqaZdS1QwIiLP/4H/8jTmYzfv3rXxNGbrXAp59+ymq15ObuI0HkMxgP+Hj7kW265erFC05Oz4iSPpUyKCM4OXtBlIx4mm958fIN09kZwgsIoqhBvWScXZyjjSGMI7TVIGrevLni5GSENiXT2ZjXb16S5xlSSsbjCVWlWK031KoiCAOqumJ2ekKW5/hBxFe//BW7tEBpgxdEfLj+yC//5BdkxZq//fu/QpmU2VmPxeoGZI4MC/yk4uvv/jM3999Q1inDYY/LS4fssVaglWH+tCDPCpIk4e7unjzPKauSPE/J85Q0TQmDgDdv3lCWJdoY3rz5hMFgyJvXbzHGkqYpURQhhIeq3fPz2dkZQkAQeGhd8/Bwz2w2pdfvsVi4VP9258z2OI6ZjicYpcFY6qri008/pT8YcP9wh7aaqi4J44DNdoP0PQajAXEv4dPPPv3Je7hLonfq1KlTp04/W7VYFrkvnTwumWwNSZeMVo2ZbhskRWvAtka840A3uWIO+BVnBLap5+ME+rFaA13tjdTf56S3r39MfmkZ3HKfKPbxZIAULo3uy4bb3vDakSCkjxQSz0q09bEYJOwZ1+0+aq1BaSwgm2JLrCu23JvQ8oDucPut9/tvjEVKxz1vzeiDaX2YaDC2QZPgzkOLiHGFnD9O5zesbCs4pKHtEX9dUasDeqc1xX9v4qHBhhxWCrjNeB5S+3iewfccVsSNgGmKZMFal6S3RmAESGMxVu3PozOPxVEprL9PoEvpEugt1seX0j1cG9Okw9mb7RqNNRwmLNpS1+Z1pJV4wkM0kw9CtpgbMHvEjlsqWtclSleUqsCoyqFfjAA8NxEkbfNvD9/X+MHzUtsgCBC+dzS5pI7M8mbCwsomNa+bAlO3EsFdDzRJsQBP+vvVGu3rHZfQtqseOpxLp06dOnXq9Mclz/NYr9cAe7McWnZ5vF8JOBwOkbjAg5S26dzxnGGYpmy2a4bDAScnM7IsY5R4bLc74uQMRM1sds7DwwPjyYAkSTg9PSPPczzPp64Vj4+P7HY74jimKHO22y3pLsPz3PNLmmXESUhV5cSxT9KL2Wx2pGnKp598RlUpFos52tSEYdgwq5UzZT2PXq/HX/3V3/KLX3yCMaZJeAcuRb/bsdvtmpJS9hjIuq4ZDkdYK9hutg5DicQYS1EUbLdbXrx4sWdiX11d8bd/+zes1wtmsxnSE7x+/YowdAWb79//0HTqCPJ8R1G4VP94MkQIQRgFBH7A+fklNzc3DIdDer2Y4XDIn/7pnzIajRtcY8h/98/+Be/ef02/HzMeDwmCiEF/wDe/+y39fo+H+3tevLhqunReMZz0+f7ddwwHI7SpmU4nPD4+8urVa6aTGXme0+vFhH6f7aYgSyuM9ojjAUKUbDYbzk7PHTdfRAR+xO9+9y2fvH3N7e0HptMJ7374zpV1jkbUleL09LTB/0T4fsAXn3+O0gWj0ZD379/zi198yfffv0MpzXA4pa4sgghde2Rpzf3dA/3BgPfvf8DzPR4eH3j79jXC0yzXj1hreJrfEMYCU1uKMuf87JIffvjAeDRlPJ4QRTHD0YCiSAkCn8024/zsjOubjwR+iDGax8dHvvzyl4xGU3w/5u72HqMtsz+dcXP7kbOzc/I8J0kS+v0+SZKglOLh4Y4gCHjx4tIl+0dDbu9uiXsJSeRWGlxeXlHXNf3BgCIr2KzWqFrz1a++wg9k05tUkecFcRzy5u0bPn68RgjBp5++5f7+9ifv4S6J3qlTp06dOv1s1ZaDPk8eHxd3aq2oVYVS9TMmOkiwXmNONyiX1qjmOZf7eHte1Pgc23JsBD/HgTw31OUeqXHEbhctLqbdDx8PDyk8h3SR7QcNf/870vMb1EuA9Fy5qGhLU3HH5vshgR8R+OGzEs7D8f2YR87eGG0Nfqxo5hUE1uC2I3+0fa26riiKgrIo9obq4fhdWaozat35qWv1LPX+4xS/bkzp/SSEOKSif8ysP5xX0Rj/Ib4f4fsuue9KVyN83yU/jBFozT7t/ePz1DL03XsZEAYwWKMREjxP4Es3mdCm1KV03/Nkk/5vd4lm+I6S9Z7nOJJBEBA0Za1tYao7P8YZ6M3DsNvKJoneJtPdz8qyoiwqyqLcl4oej+Ef5s2bZ+Z3Xbty0rKs9u/nXsMVWYlm4sD3/GaySu7HSevamfxN0emPV3B06tSpU6dOnf44dHt7i+/7DmvRsJ7fvn27Dx/Udc1ut6MsS8IwxFpIU5fgvb9/bJjkkigKWS6XlFWJNgqtS/qDkI83PzAcJdzeXjMYJBijeXh4IMsypPT48OEaT3p7PnlrTvu+x8XVGdpUWGryfEPSC7i9v0apCtt8Rtjttnz99TcIITg5OcH3PZIkaZ6Z3LPV1dUV1lrevn1BlmUopbi+vma5XPLu3TseHx+PnqUtWZZxeXlJGEZUVU1d1cxmJ3v8jJQeg74rX02SiLLKGQwHfLj+gfOLUz7/4nMGg4TLyzM2myVZtuPp6QGlKrJsx/n5CWHos1qtHDu8STM/Pt7x7odvubv/QNLzSbMNWb5jt9tijGY4HGKtYLPZkuclvh8wHo9Zr9eUZcn9wz0AZ2en9PoJL1++oKoqxxK/vWU6myA8w4uXl1SqJOklJL0+nhfwxRdfcXH5Eovk9PSK//l/+l+4uXliMjljMp4xnc7I8oyk1+e777/l5vaG8WTMYvlEmi2RnqasdlgURZHun4cvLy8RQhJHrsQ0iHy26YbZyYw0z6hqRRz3ub19ZDCcEcUjFouUy8u3ICKqyhCEfTabjLJSCOmx2W7ZbDYIKah1xcXFOYPhgLOzE6zVfPXVl/u0eJIkAM11nKK12mN9PF/y8uVLXr16RZ47tItAcH523jxfu1W5YRiS53mzKvfwWXC93hBFIVprrq4uyYucxXJJHMcorbm4vODk5KThm6/Z7XbuvdKM1XLJeDJmuVrR6/coq5JXr19ze3dLfzhgm+5Yrpa8++GHn7yHOxO9U6dOnTp1+plKtEn0PWZCHhnp7ndc0lYf4VKaTbRG4AEDAwfMx3MuuH3238dm+TH25Meolz+s40Q37FEmbUK7YbKDwDTfB4Fs+O8OWXI8YeC59LKUGATKWJRpiORSghQITyK8Buki2rTz8T7QJFHa1PlhrNwYthiW32e9t69hrUZpRV1XR5v7sHGMwzkev9Ywf44A0c8mIQ5GvCu4PP75H5rccIlqsZ9E8GTgkv0yaBLlRxMn7blwe7c/b0IcUkPHEyTmyBSW4sDH9zyJ7/tEjRHejqFofs/3fXyvwdw0Y+vvjXOfIPD3r3VId7fXVo3WztA3Terf2ENR7n4MjN6fJzcRIo6Op1k18GziR//oPBzM86pscDBV3XDQ2/uqveYOhbDPJ5rMj85HZ6J36tSpU6dOf0ySUjKbnfDpp59yenpGrzdgs9kyGAzp9QZobXn58jVaG6q6oqxyqrpgs92Qpime75NmOVGY0B8MOD+/RCnNarMkjDyiKEBIwWA4aIxI0zxDueeSi4uz5llGU1eGx8cnyrLk8eGJ+dOC3S5FSo8gCAnDCE/6GAOLhSt93GzWxHFEXZdYDLvdltV6yWq1ZDgaEUY+1rrS0LbMMooil3gvCvIipdeL6Q+ckSmlmxDI85zHx0cW8znGatbrFUWRc3FxShyHrDdL4jgkikIeHu7J83T/fHl1dUF/0KeqCpSqQRjKKmO9WZLlO7bbNWfnJ7z95CXz+ROeLwkCn14vYTQaobVis9nw+PjAbDZlu900uJmKPM8aLveGFy9esttlrNdbhkNn6nu+BAzb7ZKPNx8IIx9tK7bpmjTbOlSPsFxcXjKZTPj2229IsxRrDW/evCbd7bi8uOTDh4/M50s86VPVChAoVbvk+2SA51ukp+j1fQbDmDCSvP3kNVIadrs1CM1w1COOI8CyWi/xPcl6syaMQ7IiZ7PdcHF5ARL6gz7T2ZSiLLh6ccXs9BStLcPRBKUhCHtUteHu/hHPj6hqw+PjgiBIuL27JwgC0izjaT5nPl9ycXnJdrdDSh+jDVWlkJ7ks88+o6prBB67bU5ZlgRNUa5SNUq7laTnF6f4gWAyGbHbbd1HNCn2Ya8kifnzP/+zfRFvEPik6Xb/vH/94QP9Xp/tdku/P0Apjed7+IHPZDrm8emJXZrj+QHGQl4WXN/cMBpP0Nry6tVr3v3wnrLpm/qv3sP/X/0/RKdOnTp16tTp/2cSyMbga0xIHJJFCK9JPbvEubAeUgYOReG1pqrfIFw48Kv3xZgai8aisLZNJJsfGekWpS1Kg9HClUlaD4GPL0MEPtJ6yOarMG47/h5WNsWSjXnfeI4Wl/bWBrRtjP4mYd8WosrGQN+nzhtOtbGg94b8fqCaTTTGt0tStwgcT8rGpHdG/bHxLJBNilrsN6zA6Mbgt7jfMBpraoRVCKvA1FhdoXS1T3TvedxK7ZPVdV2htDOKXSq8NYVpcCMCrS21stTKoLRBtZMYRqOMQhuHHtG2QbG0OfVmcsGtVJD78RVC4MkGx9Iko2QzHu2JaAtDjXHmvTEOg0NTUIvnNikl0vfcFvgOmSJFE7yXSE827+Xj0TDVpSTwfXxf4vkS4ft74IxpDXStqOsSbXTDUXfxf89KhPXB+vtrvDXIj3n7wh5Nx1hXqKuVRiuN0aC1RStXHtqm0KtmMsOt3HCmuNGWQ4HvMfKonWhqWfPm2Tlu+wg6derUqVOnTn88evHiFWma4fsRo+GEF1evWMzXVKUhSyuE9REEBH5CrUsqk2OkJq9SRCBYbdcEcUypDJtdyWZX4fl94qjPerUmTTNub+558/pTt2pThkwmE8oyJ8u3bHcLfN8j9CMuzq6QNkSVMBqcoEpB5A8IvT7SRggTcnH2mmyniaIBaZphrGK9nZPlK7JsRRwHzGZToijBGs141Gf+9MB6tWQ8GuB7HsPBgIf7O7SqODs54ePHH1g3/PU026JNTVGkDIcJCMVo3GOXLpCeAlHx/sPvGAwSgsCjqkrOz8/I84yTE1dmCg4VGEUxSS+h14uIE59eL+DtJy+5enFOvx8znY6wuOCJNoqizJsyy4jJZMabN59QVTXD0ZBeP+b+4ZZPP3tLXRek2ZaHxwVlbSkqxS7LmZ1N8SOPTbbgP/31X5LXGwq1ZZevSJKIUX+ILzyKXYo0hjzdEfqS//6f/zmYkpNpj/EoIktXPD3c8/rqBYvHJxaPj/hS8urFJevVI0kEUmb0+xrpFSiTst3N8UMDUhHFks32CSEUcSIxtmC1eqSqHeIw321QVc5mNacqd3iiZrN+4P7uHf2eJIoUi6drktjHqIrPP3lNEvn82T/+x0xHU04mp/yzP//nTEcn2FpyMrlAegG9/hCEz6s3bylrxdN8ifQ9Pt7dYgRcvXzJ3//2t6RpxWR8Rl3BarVjs97geRJQ9Ho+Zb0hjBXXH79G6ZRapZyejdmlS9abBUJYyjLj9vYjZVkwm82oK0WVZwSeRaLpxRH3N7eEXkBda3a7jF2WUumKIPa5vb3j9vqebFuC9tCVIN3kFGmFrjR1qZgMp4z6o5+8hzsmeqdOnTp16vQzle9LPCHwGyMYC1YDwpmEjpLiyjklQJsU/1FI/JAId/gXgQIanInwHDsbUMgm4eshBZjGpHdGvXtZT1istFCrho3CPpGMEAhPOD+0TSobgWj41FpYlKeRsgYh8LwA6XnoNi3dND0aoxHSYIUzOD3hIax2hrYF9wbuwDzZklhc7toTuIQ7Ftc3KpqXbQxTK/GEj6BJPbeTEsg9jgUjEaJJ5wuFsRVW10hTg3UmtzHaTWIIiRGgdTPuGlSDiRHCOPSJkCilqCq1L8H0PA/hSbQBWxuM9Qh80aTpzZ5cDxZrdMMQbxjeFjTaLf81GisMCOuKULUrCvU8h11BGKRwH072HG9rUbXC2qaE1nfn1xrjilStdl8RICxWAr5D3Tj0SXtd+U0zp8IajURggaBJLR2vVlC1wRWgKqywbiKnmdARom34tEgEiKj5K+OmC2zDqRcghQGr3eSDAmldqarx3WUBjv/uGP6uSFYrg25S7rZZOXDgzbeJfn/PWG+mFwCBMRalNHX9fAVBp06dOnXq1OmPT6PRiPV6zcPDA0ZbJpMpQrj+mDhOKIsSrCCJE9J8iZSC8XBMWZYURUUYRoRhhNEg8SnzmiIvCX2XQA4Cief57HY7pIQs36Gfal68vEKpmqLICIOwway4DwUvXrxESo9vvvmG09MziqJiOj1hsViR567YUym3avX09ITVavVsFeVg2G9QNArpwfnFBVmaYa3gZHbKYr5gMpkChvV67bpoQp/5fMWrV69Yr9cMBgO0dp01t7cfGQz7VFXJZrumLAs2mxUWi+cn1LsKsChdE4YBeZ6RJD3WmzXL5ZzZya+o6wI/kDw93RPHCb/97ZLp9ASlCoSISJKYuna4wpcvL/ec+Ol0ui+wlFLw8HDDyemE7W6NAHw/pChKoiggTXfNZyHD6dkJfiDxfYmUoJVit92yWq4Jw5i63jCbneD7cHNzTVHs+Pq3v8FazWa9oNcLiWIfpUoXJPEkD/f37NIN0+mYwJcoZdhsViRJTJrt6PcG9HoRWsF2u2N2MuH+/halai6vznl6eiCKIjxPkmU5QeAzGg1QyjCdTlmvN8RxwHI5J+n1XIBEVwgJabrDGLeCdjabslgsqaqak5NT6roiLzL+5FdvWa12eJ7H//g//h/4y7/8D4RhyC++/JIPH37ghx9+ACRFXhEEMf3+AKzA8z3iOKGua7I8I44jrDWk2Y7JZMb9fU2eZ25VAZYw9KlrTRRFBEHAcrmkqiukdJ1X2S4FLEmS8PDwSL83RGtFXhTc3lZYa5nNpiyXS2h6lnpJgu8Fe4SSQHB5+YL/lpx5Z6J36tSpU6dOP1MdCjsPJZMt09tth2SuNfaouLFFkLTJcveg3P5b4DbrXhi8xgxvUthYCcLDa5LhUniINkVuLOLowbs1ZW0T4LVYROu4i7YY1XPv2PAT67rGClxaXViQGo2P8ARCWoRsX7N5HSxib0yLZ4lk3aTAHV+8NUYFxhwVg+4xIO3+QsuNF6I5ZtoCUItFu2PAGbaWGmvdxAPamddKuVJNKwS+B8YDoTSep/F8H+FJPCy62c/WRG8f+B0+BrCeI+54TQq+GSe7H1+HJjHmiFdvbVOK2Z6UY1TLAXMim6S6o5NIrNXN+7aYGrnnlx+X1rZjJqVs8DjtuX3+tdUxH9xvWOgH9nn7XhZrQGmQsj5gipqVCFIapDRY6zUJc+veVrac/QNepj1GYwwaQwP6wTbXmbEtE900HygMBnfc1hzKdL0jdNABH9SiXNin1Vszvv3vdrXD781WderUqVOnTp1+1mrZ570kwRhQqtojA10IRGCM5vbukdE4pp8Mubi44P7+kUHfkucFAumCHcB6teLTTz7n4f4GKUOwDoW33ixYrJyJKpVms5X4XkBZ1qRpRlUqqqrk8vKc6+sP/PKXv8IZliHb7XZfeqpUTRQHpKnjW+d5ThgGeJ6PtZbVasXT0yNaK66uLnh4uCPPS4IgdCsla/fs4wxLiZCWsizRtaHXS1ivV6SpQ6a8fv2a77//nul0ijGazWZDkiRUdcFy9cRkMqUodqTpZm84V3VBEvfIi5SqysmLHWBQusbzPeZPT5ye+QwHw+ZvKrSukVJydnZCFAdstxuM0SS9CGNUY97CbDZhs1lTlDnWGqI45PLqkvninlpVFEVGUebNOXUrO8uiQinFaDCkKmvCKESrGt8PyLIdw+GQd+++44svPiOMAlbrBUEgGQxiqiql1+vx4fqW648l//Sf/jmj0YAsz/ju2++xwNu3bzBGs1qvGA49ptMJ86clJyczfv3rXzMYjLi4OOfh4bHhkFtubj4yHk8IgpDVaoXn+YSBM6Qdd15grSYMI5RS5EXKcrlkNpsynz8xGPYpigJjDNvtlrIsiaKIb755z+XlS5aLFf/xP/4ndruU7faOf/JP/hG/+c1vuLo6p98f8fiwoq4Uw+EYaw1VVZEkPR4fH4miCG0Nq9Wa4XDIcrHk4uICa20zAeBhrWXZsM/d9ReyWq84OZmyvFuSpTVS+Az6E4ypqVVJf9CjmO8YjqZst1t6/YjFYsnl5QVaK8IwwFhDUSgmkwl5lhNFMePR9Cfv4c5E79SpU6dOnX7Gag1NVyjZpNExeyNwz4SGxnhsTcbGuLSmSWO75Y8ukd4iQVqetDPNbZPEdrztACl8x8ZujMXW2LbCNMiTJh2tNUZbEG5fnfHpktFIkA1/W3jthIBBofBthdAGoTRGKCSyyb03+yfa0kvA0CSsXbKkNWiVUgD78ZBS7lEvx6WTx8Y7bq4BCZijSQprLUbrJuRusUZhjMJSAxpQWKUaxIpLSCMlyiq0tiAknmfwjUH6HgZ7tJ96XzTq5h00IPGk2fPJ9+f6aAKkTWsfM9TbVDXaHbOxB2NYHF0T7rowbuJDcODat1eKkHsj2fP8fVloOylxbIIfm+uHi1P+XrnsMWv+uHjT83yMMIhmcqO5nNz13LLyhUv/uMkNfu8aP5jobVq8Abq0pbAtfx/7bJ+Axjx35/fYRD+kz5/fc9Za6lrtefbH3QDtcf6DtQCdOnXq1KlTp5+lXImnDwIWyyf6vSGTyYSqqiiKkrOzU+q65vTkFC9wz6JZViKERxT6xFGPoigpy4qLi0tWq5VLGfs9RsMBab5xCeV+zHAYI4QkCDxnHiuN74ckSUyvJ3l6mjfsc5+qKnjz5g2bzRpjLL1eD1cO+oqn+R3WGj58eM90OuXhIeWrr75it9s1KwIhDAOEEJydnbNcrkniHh8/3jbPigFCuJCEsQ6Rt9vt8MOAfr/P+/fvubi4YLvdsNttSZKY2WzGcDhASslg0Eeb2q0yBXr9mLLK2aUbhJDcP94xm84YjYf0+zFlmZPnO4bDEXESNYn8DGstk+nEPcNLwWCYuM4lNEJaQs9Hq5qz81Nub2+5f7il3+uxXi+bAs+MLNuR9CLm8yf6g5iIsGG/Ryzmc4bDMWVZEIY+WE1ZasIoYLvZAQLfl9zcfOT0bIpSFWWZMz0Z8f79e0bjmKzY8OLFJULAzc0NSZLgeR7D4YSiKMiygvn8kSRJuPl4Q54VTMYz7u4emE5ngKUsC4TA8dylod/vEwQBWmuSJGE0GrPbpRS7FGMMo9EIYwxFkTe/E5NlYTPJ4s5LWxyapilffvllU7Zq0coVw+Z5yeeff8HHjx/59//+L/mTX/0JX3/9W/K8YDyeoZTjpE8mI7LskffvP5AkvaZMVOFJn902JQgCnp6eePv2LUmS7E373W6HMa50dLFYUJUVYRDz9LgmiiJOT0/J84LNxvHU4zjE8wR5nrLbreklPfqDCIQh6UW8e/ee8XhMXdf4vs/lxRVplvLhw/VP3sOdid6pU6dOnTr9TNWaeVo3KIwmAWsaM/yg43R2k6xuEtyeJxvjr3EshftqTWMcCg8hHA/ameY+suGe+16AJ32X0HXR6L1p6qFRQjUMaoVuMCDGNmgOYfYL6qSUDbtbOla516SKpcHKGiMM0gFYHNXD4Az0Zr+tAIxANGmGfUlqCzw5Aq63Jms7eeDe/3hSwYFKgOeGMAfWuGjHvvmuO2aFMTVW1WirMZaG2Y5jy2MOWBghENa0AXeHYNHPJz5oMCTtyoI2SdIWbB4Ma7sv3mxNXNOYxfvVAfuVCQdMSXNVNJQUCzjz2KW7Jca0hreH77tEkvSkw7ccsb9bI/p5CaptADzy2e/9OCl+rANL/FCa6iYI3J5K4TkGu2MGYa340STIYWLAJcfbCQCHYuFHSXop3TVhbJvQEc8M/39oYuD4e89LYVtuvFvx0Zr6nTp16tSpU6c/Hq1WSwaDAUVREEURYeRzenLKhw/X7HZbgiBgOBySphm6NkzGEyQ+J5MTrj9+pJf0UJUiCkICz2c2mXJ9/ZEkHqK1R7orGQx90t0W0JRV4Qo5w4jNJidLc5I4wRjTIEssp2czpAf9fsLr16+a5063gi7N1vT7fVarJePxiDiO6PUSisLhQabTCff39wwGA3744Qeurq4wxvDhwzVBEJLnBZeXV+x2G4qixKLY7XbESYwQYv9MV5Ylk8mEs7Mzlsslvu+74k7P4+XLl7x794NjoFcVZeWS61WVUxQlaZozHo2IQkvU77FczQlDH2MU/X5CnmcubVykDIcjVK0oy5KrqxfcfLxDypiyLInjiPV6gR8IjKnQuqJWHnm+c4WXWcbt3TWTyYj7hxs8f0Sa7ogil5BWWu9xLsNhn+v1oik/lYwnI3a7HUHo8+VXX+D7ksVixXq9JOkHTKcjdumasqyQ0uP8/BxjNVmWUhQlV1cveXx85OlpjpSSJOlRFEsAqqpiNBqSpinWQlEUzd9ccn9/w8nJSYOgifF9n81mw83NLaPhmCiKWC7dNVmWroj2zZs3SAlhGPI0fyJNU66urgjDEGsFvd6AujL87pu/ZdnrU5Ylf/In/5jFYsFwOCTPU+7uHkjTnLdv32KNC9yoWuHJgDAI8QOfoqiIopA47lHXer9a+OTkhIeHBwaDAUIIer0eZ2dne0NdCMHLly+J44TxeILn+YzHUx4fH+n3e/R6CZvNiqou8Ar3OS7NNvT6faI4wJOS6XRIGIZMJmM+frylKt2zeq83/Ml7uDPRO3Xq1KlTp5+p2kRwi27ZIzdwRYytEQg0OIrWSLf71Ln79wEHYtH7tLJL/HpIIfGEhycCJD4eIYGMCD1npAvpTHSHAmlKLtEIocDWoMVRyWJTsmmtSx434BjXRekMcekJPCmwQmGFcKgNQFjZQDyAfWq+QbtY6ZLyPDeNpZTPTND2+I9NV5fib7Dbzb47g3Ufg3Z/i0OgyDaBjcBKxy1vDWCrFMpqVzqKxkWnPYTw8DwQQmOwbkzEwWRtSyp/XFrZYl1aE10pENIemdYaY5pS0jYJLZsUuRC/ZwbbZizaRDbtagPTrhwAKS2e5+P7AUEQHvArsr22eGY2H39Ictic9qDkUWLe7hP1x4ltd23KZsmzpqpaU1o3WBmJJ+QeYegwRAdz/vj92jGSUuL7jmHuN9cve5a5RWAQyGbFgVviqo1BK9UUlB6M/sP9dUDS7E30SqGUwWgclx3PjU/zbyE6E71Tp06dOnX6Y5IQlqLIieN4/xx6c/ORFy+dSfnw8NDwuksmozFBELDdbomiCFUr5tmC0WiEtYY03TkUI5YwCgEIg4DZbMb9w3sQGt/3UKpGqYxe0qefjJGeoK4rwsilfwFmsxlPj3MGgwFJ0ttzz6WQ7FKXDi/LAqVrAhnQH/Qw2lBWJbOTKVppZrMpw+GAPC8YjUdI4TcJdZfAn80mBGFCv98Ha9lsXMHkL3/5FXd3d+R5RhSHxHGEMS7Bne5SlHJpYa0VabYFFBcX5wwGCR8/fiQIPKI4JOnFLBZLkl7IYvFEvy+aZ0hNlmWs122i+gKtDavVEqUr0mzLarkCDMYq8jwlilwS26Jd8WmZMZ2Nmc/nIDRVVbDd4hCNuiZJ+u6ZsPl8sN1tyPMMrQ2DwQijbbNa0gCa27trt+pAFaTpBs8ThJFHVTsc4sPDPWEYIT0PYwxp6lLjvhcihGG5XNFL+ljjxjbPSy4uLsiyHIBer8e7d++YTkfc3d0RBCGjkescGo3GFEVJVdZsNht6vR7n5+fc3t0wGAxAGOIkYrVasdttefXqNf1+n8lkwtPTnN/85jeslmuurq4QQpKmO+rape+LIkcImM/nvHjxkvv7B05m50ipuLi44LPPP2WzXXFycsJ2u6GuK7755mum0ynD4ZC6rlmtVgDc3d0xm80Yj8dIKRmPx4zHY6y1PDw8kOcl0+kpn3zyCR8+vCdJYjxPUlUFQehTLHI8TxIEjn0+nz/S6yXs0gzP89hs3KRFVZX0+wkPD09NT8B/Xd3Te6dOnTp16vQz1XEiuDVRWwb4gfXtDEyXam6S6DhkiuM5q70ZazkyDJv3EE0K2KWqBR4evvDxRYAvAwIZEMqA0A9dasYPCWRA4LkEsOOdy4Z5DtraBjfSMMYb092VfRoMTWGotAjPgjRYq/ZJb2Pqg+mvNda0hjW/d8zt9+CIDb83p82zn7XoDs/zDniZ5meiTVBLifzR7zlO9uF8tPugGrZ7VVWout5jQrTWaKUbBnq1N5TdOQLTMLqVapndB6PYvfbzAsv2Nfcp9Pa19qsU9LPviSPUTZt0N/tJANOMhf/MPPc9h+1px+mYj35snh+jTeraHV9d18/2v/299vtaa6qqoiyL/eaMdGfouwkOl4Z3ZVtxk4wPGv6m7wxxKxBIpHTmv98sLbbNdba/BoT7PZq/scZidLOZH0+u8Oz+asfRlZI26B1j98b976XsjxeDdOrUqVOnTp1+9tJaNUWZG4oiR6maqq4QwvX0BEHAZrMhiiL8wCfPc3Y7xyiP4xiEwfMFYeQzmQyRwtDrRayWT+zSDUEUoI1il2ZoAwIfkGw3GzxfEsU+UeRTFCmgOTmdEASSNNuijeLp6ZHdbsdutyPLcnbpll4SkWZb4iQiCDzC0LHFa1WSZVt++OF7jFUIaVlv1gyHA/7iL/6cqi6p65LBoM90OiFOYsAyGg0R0jI7mZDlKav1ksGwD8KyXC6IkxBjXfhkMh0RhD5RFPL09MhyuWC1WpPnBcvVkuFwwHg8pqoqwIUp0tRNDAyHQ8qyJAxD4jgmSRLCMCRNU6SULJdLlFI8PT0hJFRVyXA4YLGcO0594O3T0UkSY6wmTkKqKmc4HFCrkigKCcOAMAgAQ5blVFXFer1CCIsxit1uzW63YTDoUZY5QrqAUlUXDEd9jFEgDFVVICVMZ2O3MmDQwxjHVS/KnCSJeXp6oK4VvaTPZDKlrhWDwRDfd9eK73tUVUUUBQSBY7UPBgPyPKcoCv7jf/yPPD25yZLRaMTl5SXD4ZCn+SPL5Zybm2tOThwWZjIZ88UXX2CtZbfbcX19zf39PUmScHp6RlVVXF+/x/Ml2tQo5c73w+Pd/r17vR6PTw9EkZvk+OabrymKAmsNStV888039Ps9Hp/u6fd7jMcjgiAgSdxky2azQWvNcrnk8fGRzWbD9bVb5SCl4/xXVeVe4/EJ3Hpb6rpGSo/Xr99wcXHFarVGa4esOTmZEQQ+g0EfKcH3PcIo4E//9E949erFT97DXRK9U6dOnTp1+pnqeQq3MfOMw6HA86SvFS4nS8NHb9PNDpnhst5tUeVzIIpsUusgEXhC4LXJdNqUuitcdIgVC9JitGNRO6PcmcrKaJfSFhqEKwMV1pWDeuJQCmqxGGFdchuHqLFCYxAIKx2+xh7QKqJB0xxjT1om9YF7/dwYfW6yt1trgNq94cqP0ujQppf/IeC1w8wYrV0JqXCpeoQHWh8QLy5YdLRaQPyht2qPcI91se1kQ3PufmzyguN7S2kwtjXLjzExBzO7+fTlClpxqxWCIHAlVdKZ5y7R7R/M4bZQld9HmxxPCJimtLM9F16TtDnml/u+vz8faj+xUFPXx0l16c6KbI1v2STJG64+Hjyf8kE02B8sjk2PASkdwqXBw1gL1hg3iaN/v0Og3a/fOxtN6r4tpWq7CH5cbOqQMh0UvVOnTp06dfpjUrvqbrlc0uv18DxBEPhcX1+z2+2YTU+aInLNYr5gOhlRlQVJHCBIwNZsVgvOzk8BRVVl9JKA5XJO0vfJ84KLizGBF6FKQMPJbEa6c0Z9mm0oy4Jev0dVFWRZyng8RWvFcrlgMpkhpeDkZEaapuRFjvQADMNhjzzP8DzJ09MD4/GYwaDPZDJiNpvg+z5RlOB7Edvtmivq6J4AAEDwSURBVLdvX+F5HlWVE8UBeb7DWuNKQOuS6cmEwaBHlmUEQcC7d++IoojB4OQIh6fYbDao2iClQGvDixeXbLc7Bv0hWmtOTiZcX39kNBwTBCF1XnJ2dsHNzR0XF5fEcczNzR1hEGE0pEXGaDRmOBxT5CXeMCAMQ4QQrFdbRsMJZenKKrHQ7w3YrLeIAHr9hHSX8vLVJU9PD4xGIz5+vCXPC+KoB1hnWhcZvV6ERZPuMobDCRbl0CnDAWEYUFU56/XKdULZwK0AFZbVasnT0xNFUeB7AXGc0O8PWC5XXFycI4TH69dvmc/nTKcn3NzcEIbhfswcUiVrEv2GzWazf36/urqiLEukkKRptuelq7rm4uKM+fyJ1WrBaDRgOBzx/ffvuLu7BST9fp8wdGx1zxMYIxgMe9R1QZbtEJLG9K559foF8/mSsixIkpDl6ola5cznc1ee+uF7AF6+uiLLdpydnfDh+gd6vT7T6XSfRldKsVqtGAwGaK3ZbrecnJwQxzH390+8fPmSLE/xPMl0NkRIKIqs+ewkmc9X9Ht9ZrMzlssFQRBxe3tHWZRcXFyilCJNd6xWC4oip9dLfvIe7pLonTp16tSp089UhxSw2qeTa6WepXz3fOkGC6KbxK1LIR9vpkGDODyIsRqDxtiGTW30HmXiCYHX2JWiwagLC9K2FubBrG0N2zadXNeaWmlUa/xa2yTRNVba/bJVV4/qykORuDc4SpobbbDagLH78skfp7CP2eGt4fzjBPrzhPqBa33AxBxexx4lv23Ldm+KWFu1jPF28kC3vOyjc7HnwiOOkC3tfokG/eLKPFu+937VwVEK+jjtfVxuecwVN23Kv93JH61ScJMvHr4fEkURURQRBBFh2CbRg3256KGA9jAB4Qo2D8nyNmFflMX+mvxxWr0sy4bnWDQp9HK/tel8pRxPXyuL1q7YqP33YUWFPPrq+O0g3O83if52lQJIrG3Z5835tmB0exxwPNFyfE0cr/g4cNC14617Pp4X7DcpfWTTE/APT7R06tSpU6dOnX6OipMIpdW+FyVNM3a7HWCYTCYsV0tWqwVpuqWqCooiJwwDNts1D493VHVOEEjKMiNNN4BB24rZaZ+sWDKdDdntUr74/FeMhmfUlaSuQIoArRVh6IEwaFPz8HiPRbNaLTDGoV/AslzOWa/XWGsZjUekacZkMm7wGzW73Rbf9yiKHGM0r1+/YjIZN0WZGevNE1m+xRjlzMkyw1qF7wtAIaVhMhmxWq0IgoC6rnl6egKg3++T5/k+jezKSy1ZljOfL7m6fEEQhFxff0RrS1UplNKMRxPC0BWp9vtDrJVkWclyuebDh4/kWd6ksSvKoiZLc1TtunVGozFlWTEeT7AWyrIiTTOKokQpw9PTgl6/jxSCNF1TVjlpukUpN4Za1xRFjrWGIAjAWlarFUWZUZYltaowVrFLt/SHPYS0ZPmOqioBw3QyQQrPBZiaVZ9nZ2dMp2MQhrLKUboCNHmREYYh2+2WMAw5Pz/nT//0T5tAk+HkZEaW7QDLbDbbhzuMMeR5zmeffQZAFEW8fft2z+d3gaaKMAyQnmC1WrBYzMnznH5/gFIK3/cJg9AFWpTCYjk9PUGpiizbUFUZfiCI4gDZkBLzImM46rPdrliuFpydnbpjr0sGQ/f9fj/BWsVgkOyvqyAIMMZQVdV+ZUF7LFVVUVUVk8kIIY17/3xHkkTNpFTAYrEkDCK3RtqLwXoEfkiRV2AF5+cXpGlKWZUYq/E8ePHygl26/sl7uDPRO3Xq1KlTp5+pjnEerYHZbj8uQ2yN2hYR8txwNnueuTPQHT5FG7X/nnXgZ2fCGmcoW2NcmlkbRPNwJxoDXEqBlC5ZjnBmuMUlsFvERmtctka6K/VsktbWvZ97NVe06bjZpknwOPwJRoCVtC2dx3gT9QwvUj8zm/8hMx0OuWbbjJ09GuMfoz1+Pzne1o3afVr+OULmYKC379uiXI6NXGfGHqNT2E8eHCYl6v12bKbXtaJWh4kVe2QEm6NjbtPgQRA0W9gYwC3Cp33vBt1ijoz5o2vveD/a/VJKPRvfHxvRVVU9M9KLotib6L8/CXTMhz+aEPgH7gl1NJGEYH9MbnVBe769pvzUjXmbJD++Z3789fjct+P34xR6+xo/LiTt1KlTp06dOv38JQRIIej3e82zkqauS6wF3/epqpIgDOj3e8RRwP39LRcXpwS+xJOWs9MTosjH82hY2obAh9E4QpuUx6cbrIWqMmBDPJkgRUQv6ZNlGbt0TZpuCAKPOA4dlqWuUKpEeoLBYMBwNMQYTZql+xBNFIVordx+xRFxHFGWBUEYsN1u2O62COl6ecqyQOuK7W5Nr5dQlrljmQtDWWUgDH4g8TxJnuf7FPjFxQV5njMYDAiCgCxzBaKDgSuyFPiEYcTt7T3n55fM5wuEkHz922/Q2rLbpqjakMR9dtuUMAgx2jCdnKCV4f7uiXSXE4Yx/f4IpQxVpcjzglev3rBcOka25wUsFiseHh5RSrPbZRR5yW6XUlUVWis+fHhPr5eQxDFZljKZjN2YpTuklPR6CUpVCGHp9RKs1U1x6dKltgXYJozUFt1naQY45E+WZWRZhlI1RZFhTE2abZnNZmRZRp4XfPz4kV//+tfM53P6/T7r9ZrBwE1CjMcj3r37HiEESZIwmUyYTCZ8//33KFWT5Rn39/csl0uWyyUuQd+nqktWqwV54bA00+kErRWvX79mOp1S1TUX5xfM50+MRgOybMvZ+Yww9vAD8Hz4/PNP8Bru/qtXV+x2WwbDAb4vXbeWsFir8DyYTIbUqsAPPFarBZvNhrKs2G63SOlQNJ7ngjphGDKbzej3+yjlCmq/++53DAYJL15c0OvFXF1dslgsiMKIOOoxHEy4+XhHntVMJiecnp7h+wGLxQIh3IqL8/NzPF/y+HhPXRc/eQ93OJdOnTp16tTpZ6rnhYdHyA5+jBtpTdjm90Rj5oq2nNM0zPGWi97gW4SHQWKEwuBjdI2xEi2cQS2tQPgWYV2yxYqDsb+nqouDSS08gbTyYJhbkK1J3bC+aYxb6Qm0cWa8sG3avEGTWFdHKqTESokwAiPM3tC2tmFwN6WpwP6DjEPgmD1ixPf9xvgUe7a5ED/ClVgQ1mKsQWgNQuB5onn9Hxu6jQkuJFYejr393X3aXDqjun0f93fPsw+HlLxoDNnGzDdtwvv517boE2uwVoI4GMMt0oUmKd8uN36OL7H793BIlNa0Zv8hy0qzx5TsOfx/oFT02Dg/vl6PDfh239pEvGnQKtaKg9Ftj0tXLVJYhPeHE97H77dHq3BArAghoZ3IODq/x3LJ/MPY/xgBdCjqlc0t1957okmfG8c91eYPXBudOnXq1KlTp5+ziiIn6UVUpQsITCZjdrsdq9WSKIo5OzsjTVPqWqHrgqJIKfIMbTRnZ6fOdFcVQSApipQwGiE9S5ZvSZKQonCojfV6xWg0paoqgiAgL1xKxfc9kiRmuVxwenriVt8ZgScDwJLnzsgtygzP88FaJtMJuzRlOBoyHA6I6wgpBWm2o65LPF+y3a6ZTqeApd9PSJLEFcLXJWnmjGWlJEVZYIwh3aUMhqOGjGiOMCEeaZqy2+3wPM9xzIOE2WxGXdcsl2u0MhhtieOE3S5lNpthjOHu7q5BmbRIQPf55uPHG15cvcTznphOZ+x2aVPAKZrU9znffvst1hqiyBW+DocjiqLg/ftrPM9nvd6ibMHF9ITVakFZVmy2O5IkIYoS8rwkDCKU0mTpjtEooa539PoDsJLHxznjsbsGlNIkSUxVFsRxQppmGOPCQK73p3YTEiJiOOyzWq2I45DLywuKwjHQy7Lk8vKSIAiw1iFgwjDk9vaWIAhYLpdIKVBa4XmSfr/HfD6nyHPOzy/44d0PnJ9f8Nmnn3J/f8f7Dz+QZxt2mx3ZMCPwAuI4ZrXauNLR23vHnq8rbu9vmEyGKFWAsFR1iZRQlgUvXrxks95RU+MHguVqQV3VDUs/ROua8XjI42PBdrdBqZrpdEqe55ydnyFEQJFVLBZzPv/8c+bzjNF4hCc90jTjiy++4O7uDq21mzAoMh6fHonjAIQly925/eUvf8l//qvfoLVLnVd1iRCC77//nuFwSK/XZzgccH19zXg8Io4Ttts15+dnP3kPd0n0Tp06derU6WevQxFomzY3v4drocFYsC+UdHxtu89771nbpuWjHBLn1pqDeWsUyiiUqR3r3GqUNehmM7SpZ7N/j3Y/wWCtS2a0aXdXctoaso2Z35i91uAKN/dp8CaRbEEb4WguVmC0RSnTpJzrfbL5eWK7Rco4XrdDgXhN4tvHkwGeDBwjsUGFWNugadr9OJ642Jv2B6SImwTwENJD+r5Ln4gDbd3NKuBKMIWHxEfgt3CcI0Y6e0B6y603Vu/PUVsE6lYKtOPc8O2NxhrVTIjoozQ1e5yL1ppKqwb/o6krjapV88HFMSndeLbYmJqqSXe7Is5mVYNqJi/a68qAEBLvKP39h0pIlVLPMC6qxd7sr1HHP2/PgzPED9f7wbw+fD026PfHjCuz3SOATIuzUftr0Bjd3BzNuFm9f82Wq9+OuWkmcdrVD3a/EqK9XtrVEu6669SpU6dOnTr98cjoEmsqqjqj349YrxZEgU8Sx1RFSZmVrBcbfOkhPEPU93l/+wOL9YLr21vmqxVGSNIyZzSbUJuKrEwJI4fZu7w8Jy83rHd3bNN78nLOYnmD5xvCyN/jSrbblKenOULAYNCjKHcEocELKjy/5PxiSNKzzJePbLcFcTLAD0JqXVOpkqfFI0k/JowjlLH0B2OyrGa7K6k1rLdbgijk7OKUKInwAkFWpJxfXpD0eghPUFYZVZ2zS9fs0jXz+SMnJzO01lxeXjKbnaCUIcuKJkDhU1eaOO7R6/XZbrfUdcVg2MNSIX2F8HSTtt9R11WToK8ZDPvN5MKa8XjMer1GSsnJyYS//83fIKQCNJvNkvPzM5IkoZf0qSuF0Zb1esdunVNlBlsHDJIZdS5IVzXT4TnS+OS7AlXUJGFEtstRtUTVUJY1u22G0TAZT5u+HslisaEqLaPRCYP+iH5/SFEUjEZDoigkL1KUrhFCUBQ1dW1Zr9Zsd0vCULJLN2iteHh4YLfLiKKYoihI0y0WzWQyQlUlWboDq+n3Il6/vMKoitPZhDzdYlVJ7HucTab8+a/+nP/T//B/ptzUCCVYPq0oshwpBFmeYqVFC0XY80nLFdJXnF9M3cRKPMAan922wBiHs4lij9nJED8Q5MUO6VmMqSmrnFevX4IVDPoj8qxE4CGEj+9JVpsHvECTFRv8QBAEHkWZc3p2si++LcsKBOT5DtAURYaUgu12g8V9fgxCS38Qst48EYaWSqVEsaQ/iKhVidI1nvSRwmc+X2CMIIp6P3kPd0n0Tp06derU6WeqWpmmdLFFVQhXuihcoaIwIKRAaYPQxv1ctEl0BcJt1iqMrTCmxlqLxENYzwVthSsFtV6FNRItBVpIFJbaKqSukdoj8H2kJxuz1DWy18oZ7dpojFVoW6JtBlIjGoSLKl3aPLAhqsGLGK2oKovvCfD9vYnZGtpYge95LvluLcKohvlu9ml6IZzJXTcmr7WgtcAYD+EJhPVRJkDaAG2cae4MUgnSQ3geQvsIW2JU5fjwwqWwjZQY0ybXLcYKjPEdC52GiW5bdIrdF6ZKr0F/IBFGIoQ7NnHEKW8NYmFBN4gcTwqsMI0Tb4+2lmGvm4dwmiWRGqzBaA+LDzRoFjxaRLhbbgzKSHTleOLub32UdogdFxS3R/gbjagNYeCW5TouPTjyjsQTvkuMNwa2+QNJ/Ta5fpxQdwxEAy1R/wglIziUxEoPpDTNRIIz9NuSJjeJ0ibZcemqZpLEWIk0FiFaNn4zYaMU1tRg3QcrhyGiWSXQFNZaC1Y65n9r6CMRVlArizGHCRSj24kClzIyRv1/7mbv1KlTp06dOv3/vdr+mH6vT1FUAERxxG6bIuKEwbDPZrtBSLDCIDzYbNbNc5dP0u9RVSVR5JMXOcNRj+VyDsIghEdRlHgyIIr8PV6jVgV9PyJOEubzOVlWMhqO0Fozf1rAiUMser4zO42tGyRMTVVppGeRuSCK3HNplmVorRiPJwRBQL/XJ01zwjDC90PquuL+/p7pdIoxhrIscSXzFdtNShz3qXdrtNbsdhu22x3T6ZT++aDhYbtVoE9PC/r9Po8PS6SISeKE2cmUh4c75vMF0+mMsspYr5dUVUUcx2w2S0ajqSsq/eEdV1dXfPrpp3z48B6LJokj0nRLHAckvZDvvv+Wfj9hMOjj+wFVWfPdd9/S6w0YDoeMRiMGgyHL5QpjPaIwYbfNOD09Y71eE0cxRhvmT0tOT0/Z7VLOz8+ZzqbM50/EUY+iKDk9PeX6+paXL6/QWhOFiUv/W/f8XZYZUkj6/QEAnicBNw5VVbFYLJvPFpLpdIgxFikESRJTFMk+kb7ZbOgPEnq9HlVZkmUZVVVze3vLJ28/oSpL0u2a8/MLbm9u+ev/8lf8ya9+xfnZFxS7io/vP6JqRZ4VzM5OqFXN/cM9ST/mw8cf+OqXvyTPU7SpENJSFAW7XYZWljCMCMOIp6cngtBZzVm2ZTQacHNzy5df/oK7u3uMMURRhJSSOI6pqpo0dYiaKA64uDjn48ePDAY9RqMxq6XDAt3f39Pr9Xnx4gXv33/g5uaGs7MzlHKGelW54/U8yXKxYDIZobXC8wSb7YqqyhgMhnz8+IG3bz8ly3K3ulYI1qtNw30PfvIe7kz0Tp06derU6Weqls0sjAB54E8ff91zu4VBWNkUdNL8zBVeOmafS+C6IsbWGASDQaEd1sVWKOnsRmU1nlZ4no9vAzQaaZq/0YZa1WilqVTlEuvGmfVtWrrlc7tkt0DUCiE9lKcRwnOH0+ynlLLBeRzwHEeQEJyPKvbp9RZ9cigTpWGmS4fcMG2yvOV9t2lpsFY2ieo2hS6wBnTDhPf8FsECntcmpD1XsGMNSgmsg83QIj4cG14i9+fE5f6dcfwcy+NQI63TbRHGAhKLaZZ+iv3vHdLWh6S5+752RroBKdwxu2FqkTd2P16eaF9fYAUobZDW4nlyv0Kgrl1SXAhLGPhoqffGt5t4cGfG2AP2BMAK+zxZf3RdHh+3k+fmCEQ7Vq2RLverJtwKhMMKBq0cRsWVuNqjdL675qXwEDiXX0pzNHb6kNxvNtp2WmHdtSJEcy7EnlffznEIYbBCNFz+dp9+zM1XnYneqVOnTp06/ZFJa0G6S0l6CbttStSYsEK4sMZ2uyFJIsAShqFL1eo1QghGDU5lPs8JQp/VakMQSpIkIc9ywiBCa00SD3BWn8D3fdI0pdfrkaY7NpsNvWSAlD51rYiihKKoKIqcXj9GqRrf9ynLgiSJ8H2LtZIoCjBWUaQFYeiM8t0uBVuQJAP6/RFgWS6fmE6nnJ2d8fDwSBTFPD0tGQ6GpGmFIGQ4jFgttwxHCdPpFN8PKIqSumpL5hVJ0mO1WpLnOSenZ5Q5LObzhq0dMByMEMJD1YaT0ym73ZbNZoMQYh/IOD8/Q0rJZrMiy7bMTk7Js7wxXhVpuiZJoj3O8fHxiSBwhZvv3n0AK+j3+9R1jedJPBFwd3dHWZa8efOGPHdlpY6B3kMpxWg02hvanucRRQmeF7Dd7ppJgx2ffPIJ6S4lz0vG4wlZ5hA6taoZDAZstyvGkxF5ngMwmUzQWrBYrBBIgsCnKEp6SR+tDXHsEuiqKfvUWpPnucNg4iFFgDHw+DRn2O/THwwIw4AoDsmzlB9++N71A1mPk5Mzvro4YbFesljOCeOQp/kDbz59S1kXWKvZbFYEfsj8aYkxMByMqeuSIHBInl6v1zznGqbTKWmaM5lMGI/H/P3f/4avvvqKx8cnrLVNcaz7fOA6kFJevLzaT8CEQYjneSyXrug2jiJ++OEHpPRRTYcSGKI4YLfbEQTOhNfKMp1M0BoGgz7Sgx9+WBIEIUoplsule704YrFYMBi48tSnx6efvIc7E71Tp06dOnX6mWpvokvReugcs70PasxK0aBaWhY6LQfbYpHOQG6qPbXRzlzGOHa5tCg0nqcx1uIbg+fV+F7oDE3j7ZnerfmqtUYZhTa1Kyu1pmGDH7AyDvniEvJCuQdrKT0QFo0A1N5IdZgQ8QyxseeJmyPuuXFp4NZodj6tdLgWzwNpn7Gu3bRAY2j/AQxO+3uOd90WSjqT/BjU4o69MWGPikNbU5h2fIzFiPZ9j83k1hxvkSTOyN3vs3VJoh+nu4+55vsf2XYfXbob5H7c2/d0frFBWLVnpLevYYzbt9ZE11ojpcD3JFq71Q4tOqi9voyxe8SMtRbpewhhkfK5iS6l3J+/9vprV1JIQTO+XjMBIJqCIpc0txzKbx3a/Kh0dY8yaq4p3IcLYxzD/lD6qZ9dP4fz7P4t9ga63J/PtmPAYo+QMOxXArTpeqU0SjXYG3NgxHfq1KlTp06dfv4S+GgFm3VKXSmGgwRjDL2ev0/FhmHA7GTKavNIlrvnT6VqwFDXJWEUsl6vCAJnHBqj8aRPGMQYzyKE5xAYedkYuTGbzYa6rhmORmRpSRgkGO0CBVVVc3d3z6vXL+j3E3Y7l8r1PEkQhlSlYrtbE4YBeZ4zHI64uLhis0nJs5LhIKQsHANbSsFqtQJoeOE9rBHkecXp6QVlUfL4sOLN6894eHqPEIK6diWRDmnSJ0lc6WoYBkRRyP3dHYE/4PT0jJubGwaDPuv1hiSJUcqwmK/YbB1fezab4XkeeZ5zenqK0hV+IJlMx0ipGQwjPF/ztHik3+/TH8SsV1seHh5Ikh6B7/Phwwdub284P7/k9OQUISR1PWC7W2GMYjAYcHt7uy/zNMYwn88Zj8f7zze9Xo/lYo0nA4QQxFHCyxcv8TyPLC2oa01V1ny8vuXTzz4B4QIWj4+PnJ7OWK83nJ2dsVquybIcY9x4Yl0Bbb/v40mfxWJBkvSQzWpd3/Op65r5fM7JySk29LFGksQJWhmGwxFVVfLx5gaB5erlJdfX15ycnFCVbmXtNl2z3iwZjUdkRcZ0NiFJIsLI59277xgOBwyHE7I0pSwLwqBiu9m6FajWkiSxCzhZxWIxRwiffr/PX/7lX2Kt5fHxERBEUURZllRVTRCEDIdDyiojTV2a//7+oRnnQcPbF5RFhVKKqiqI44iPHz9ycXGGyTWLxZzBYEAYRsyflpyfXyLw3T1iFbPZSXMvpRSFmwB58+aTxsCvieOY8Xj6k/dwx0Tv1KlTp06dfqba85+PzMDjUkQ4Tjm35uaRSWta81LuWXVSBoC3/xutW1553RRYtluFUhW1KqhVQVnlVHW7FQ2LrkHEoEBoxN4kPiSQnxvADXu6KZhsk8Xtv1355MF8bdnazrhs9+vA3K5bhnebYhfSJU08x0E/NtL3eJFnSf5DwWVr7B6M8eNJA7E3Yt33Djx1azl8NTTGa7sd87vts3OmtUbpgzl7MHzb3WyxJ66gtGWQt8lpY8Dow/tjZbMf0qXsjZtA0crsSzCPjXC3xLduxtCl1UWT+jbGNtzvthCUA6veWLS2R+fskAxvz8Hz7TAZIY4mHuQRT/34Ot5fw4elCM+S+dY6tItuuOTHW7s/xwb+AStjf/R6Py4TbdL1++vO3Q+H66y59pRCqfaeOdrJTp06derUqdPPXqqGk5NzrPGwTbeLK5pMKMuSKApRqqLIM9yzj8fJ7IRezxmlaZaidc39/R11XeN7AdZI4qhHnleUpWK7SYnCmEF/SC/pE4URSdIjiiKGgwFxlKA1TKenXFy8wBrL5eUlIJDSwxhLr9fDhQc0RZmiVIXWijRNieMYrUHVhjBICPyIIi8bRnlNVRX0ej3CICLdpaha4Xvh/7u9c91x40jT9BMReU6eWQdJZdk97ZlxX+Hewl7bYvcCBgOM3W1LKpXqRBaZ54yI/RGZSZbahrHYH4u14wEISaUqMpkkgaw33ng+AhXz8eMdm/UlP/34Dx4fnoijhJeXI9bAfD4HmHbrXVxsqOuKIAiwpqcsi2lnaFU19L0lCGLiKOP66g1SKMIwxBhDURS0bUsUBWjd0zQlYEBoet0QhgJjnF5PSkEQKqQUtF3D4fDCYjF3LfbDC58/34JwIbYxlqenJ3a7HS8vLxyPR8qyRAjBfD6fvvb58x3z+RIpFX1vyLKcrtNEUcJsNuf29g5r3QLC7adPpEnCfD5nPp9zOBS0Tc/d53vu7r7w9u07lsvFsIPRXZs+PT1S1zVVVfL4+EhVVcMA0XwY6tqTpTlCKKI4JYxipIp4et7RtC1SKRarFWVVUNVHnp+faPsWGQq2lxdEcUivO7qu5f37G25vXegOhrouqWvnMU/ijCiKmc+XxLF7D0dxTNd3JEnMdrtltVoRxzFv375lu91wfX1NECiSJJnmUuV5PhWfoiiagvTZbEbbthhjqOv6rL1uQbjvnc1mGOMa+Uopus4NKy3LanhtJN9//69EUUzbtuS5a8onSURdlyAs+SwHBF++3P/uZ9iH6B6Px+Px/EE5DVI8hdLumuMUPrqvnWtCToMYR33JpDpBgVWDMPuk+NDWDQnVQ8u7t90wVLSl0w1NV1G3FVVTUrclbVfT9g1d39DpxnnRbYe2/auw2Q30VCiphrb56FPXw20IQrshmOxPoewYsI/DRLuuc0GpeR08a+1axwKG9vQpFB3Pk1KjD9yFt2NjWcoxyOUsND9pTMygWpmCdjHez2nopNGge3sWMp8NeLV2ekW+vhmcM37U3jCoZaZmuxwXBIJpUUDKYGi9j0NSI5SICGRMoGKSMCMOE6IwIQpToiAlVDFKBQihhuem6drTYsSrMNzaYcho++qcjwH168D/q8WJrzg1+sdFDTm4z1+/d8+HkzL93+lmrdOquOB6eP+cLa6MCwGn4zwP0086nfGz83oXwmmgqPOtnxZnum5YROoauq6lbZuzRZvXn0mPx+PxeDx/DqwRXF+94+2bb9huLhFIyrIhDCP2+xeKosIYy48//sTT446+t8xmC9I0RWtDXdVEYTx4tFuKoiZLZyRxjrWSttEEKmKzvmS7vUTKACkVNzc3xPGge0lTAIqi4Pl5B0iqqiLPcrpWk2U58/mCvjdOTSgFZXkky1LSJAMkTdVRVR1N03E4FCwWS8qiZL1e8e7dO8qy4vLqivl8ThTFCCFompYf/v1vPD4+07aab775jrpuaOqGb755P2n1DocDHz58IM9zoiigbgpm84wwktzdfebp6Ymryyu6ruPt23dEcUyWZfzwww90nQtJ37y5Joqc33q/fyZJI6SCpqnRuqNuSl5entntnhHSfb1tG968uWaz3XB5eUHTVMRxRBRHFMVxGD6pWa1WLBYLPn78SBiG1HXN5eUlxhjCMGS73RJFEd999x2Hgwt827bj6uqKpmnY718IQ7f7dbFYcnl1yee7z3Rdy3K5pK5rjIE4TsnzGVVVcTweB2+9YrfbkecztNYcDgcWi8V0bTmfzxFCDK+xAKGIkwyL5OXlwMdPn+k6Tdd31E3N7e1nlqsVQRQyW8z413//Vz5/+YwVliiO0FbT6x4hBNvthrIsUFJRHiseH3dugSCdkWczmrpl97zjeDgMTnd4fHxEa83d3Z1zyCcpHz58IAwj6rpmNpvRNO4auSgKDocDm80GgLqueXx85Pn5GXDzjMbBsG/eXNP3PTc3N+R5jlJqWogKgoB8NuPy4oK7O+dg/8fff+bpyd1PEAQIAff39zRNMygoNfP5gsVi8bufYa9z8Xg8Ho/nD8qoc0HgvOgwqE7Eq0DQBY/WCVzOmulTsD40qhnUGRgBGOd+HhrEdlCpC2vQugcEUkik1WcB59jEdhGntcOAzaFd7vQtX6k9zgZIOr+1C86ttaBca1pI61zow/+fh6gMPzmGok73YV89d6e8GdvhbhimGLQoY4grhBzcgpwpPYYWN6OCZDiPxqJ7DcEY8sphCCcYa1DS0AuNEP3Uuhib6M4sM3i2JTC9PgIGbYmdWtFDyG5dWO+ODedWHx5zfJ2FsO47hESp0L02MkDJmEBFKBUSqGhwxp92JPTaoifryNgy710riFNz273fAHsKxs9D4tdaFHeexgWGE2MTfdwxwfQc3ODT39DjjEsLZ+fy68+BHlvxw84F9z4X04IFjC57yaTusa8XnybtEaNbflTbWIx2f+97PT3GqG/Rup+a8K93DPyTV8nj8Xg8Hs8fGCEkh8ORxcIF44+PD3RtT9cZAhWRJBltW+OGvofUVUkYRiwWc/q+I44tx2PJfL7i6vKauq7I8wXFsUCgXGPcQFFUgKCuGxaLFWEYMQ6elDJ2+pUgIE1X3Nzc8PT0wPPzjjiOyLKUumqJ45RABaSJ87Z3XU/b9TR1x5f7J7I058MvH/jmm4A8zwjDkLbtsNby+PjAcrlmv3thNp/RtW6Y5OPTI/ksIwi3pGk6DRV1Puya9Xo9tcmjKCJN3eDJui2ZZXOSJOL+/olZPme72RKGIRf5BU1bYYxrlbsWulPZfPvtez5+/GXwhJdDIHtJGCieD0eUdAqPtm35+OkDQRAikDw97Vgu19ze3rLdXiCloGlqpIxJUxcEj2WQKIpYLpdst1vu7+9ZLBYsl2uenp4Iw5AgCCbFjHOEF4RhyGKxQClJ2zYkSUxd17Rty/v33/Llyz1PT8+DxsS19JumIc9mFEe3G0DgBpEKIciyjCBQGGOc9z7L2O12GCLKlwOLRcbNzQ0vhx2BUihpybKE1WZDnmdEcUyS5PyP//U/2e/3SClYrdeowPnchRA8PDzytx9+oCprAhGxWkGgQkDSdW4o53y+mH5H6LoOYwxlUZJlGfv9njhOpvC873uur68pCueEv76+Zrd/4unpia7rWK3WPD/vSJKUqqpo235wxp886p8+fWKzWWHRtG1LmqYkScJisaBrNX/961+R0u1GqKqaNF1hjPO6by82IDRKhVxeXvFw/0wYRL/7GfYhusfj8Xg8f1BGt6JrVZ82n7ncUk6Nj/Mg0oWyZgr9xuD79PMGK/TUepZiDBjlWbBs3ZZDEbhg/Uwh8/WfY4D+Wl0yHqechmUKKSeVijHGhfK40FPhcv7Ri+0C63HIqBwC7tNjWeuGgJ5azmPT2YW4o0B+DD1BI4Rrt1vD1L4ez4+UilHZcmrxW6cUUcOxCAlYlAzR0iBF73zjQ446esnPQ+DJaX/Wvv767+Nx2iEUduH74L0XEiEMJ+2MRApJoALUoOYJgpgoTAiCCCWjSYky6kiUAmMZnq85e2x37gVm0q4IhAvRxwUK3ALO+DxcQO2CboamuWu4u+fNcF9jcO7O4+sdEzCE9FK6carGOJe/dcNiJ5+/Pb2GJ489U5Du7lczbl12x3YK5V8PNeXV18Ym/RiEGy1OLffutLAw7nwwWruFAMnZAogP0D0ej8fj+bMxn88pioKmaQZFRYMQkqqsefv2BoCq7Lh59y1BFHE4vhAoSVE0SGlZr7Z8uf/CdntJEMRstwuKY0XfuuuT9WrL7e0dkpB3N+8QQlEUJZvNiiRJ2e0qosiwWM7Y73bIYYNpkqQ8PT0TRylCBFRV5ULbPCMIY95cr9jvj1xevCFLF5TFL3xz8x6tNUEoadqKXvdDEzgcdnFKojhiNoupq5aqOtJ2JWkasVzNCUOB1jFZ5lQeXXegKApWqyVSwdPzA3ESkbQhcRRyf39LGEnev7/m/bffsN/vubv7zPWbS3Tf8fD4hdvbj6zX26HwIfnll38Aljxz/nKlAvp+HOQKWRpTFEeSJOHm5oayLJjNFmRZQt+3U8i92WzI85wkicjznDiOCYKA77//nqpy2pC2bdlut/zHf/wH8/mc2WxGEEjX5heWMFRDuz1kNst48+YNHz9+RIUWrTuiKORwcK782WzBdnMJwtA0NWVZ8vh4z7t3N1xcXGCtYDFfsF5v+Mc/fhna7g3rtWvJd13LsThiiUiSlCiOKeqKpmnZVSVpGvPzL7+wWMwIo5T9/oXP9080bcd6tUJIweNuR9v12NaVQg6HA/PZ3O3k7CveXL8bwv4XgkCxXq+oqoK+71mtF9R1gZQCqSTr9Xoa5imlW0jCCu7v75nP59zefubNm2vCIJi0QGEYUVc1s3xOUVTUdYUd2vaHwwvX19fc3X3i4uKC3f6Zoii4ubnh06dbPn38iNawXKxRKqBtW5I4RUrldmNkKUEgeX5+5PrqHT///DPYgCTOf/cz7EN0j8fj8Xj+oPR9Pw1ptJhJieGC2tMlgJ0ULv/cIHYpqBjK6OcDFMVwg1MSLN0AUiTGyqE8bac2tRBT4n567EEDogcHu2tjW6SQTI6VcfjlGOQP6pLR1e2OeVSxaKAfBvmcQl8l3JBHPbSspTwdk5Qnx7aUTlEznoe+76fG/ejT7nt9FqSeAtpRleNCazMdq5DuDqWUGDO2qUcHuTn7GaZGv7VghHFbNzlpZISr+7smNdZtCsAiDGehtB0a1ubsdJ80L6GKiIOEcFC3BCpGihAVKKwRbhCs6uj6Ho3bLXAajOm0Nl3XuU0KQjHE2ZN//+zVHcJzMb1HrHH+dTkszoyh9Lig8fVCyvg6uFBeDs/JLWaYIby3uEURa53rcnxM5yAfVS5m0Ps4jYsLzwNO3vWznRvj54B/Po5x8eT0fsMpeUZlTK+n92Wve87Dd9t/3cz3QbrH4/F4PH8mxrA1DINBY9GTZfkwV0fx/LRD9yBExPFQs15dEoSSuimR0oW/gYqpqw5BSBTkHF4q1qsNfd8RhTFtqwmCiDhKePf2hrar6LqeOIqZzWYoJZnPZ0jpSi8//vRfrFZr8jzHGGgbw3Kx5cuXez5+unODG43geCyZ5SHWCt68eUddN+R5Tts2IAxCWJIkmZrXfd+S5yk///wLlxfXXFyumS9cI7ksazKRUtcNs1lImiYEwSVRFKG1ZrFYUJYldV3RdS3z2Yxvv3vH8/OR+WxFVbu2eVkWYLfESQRoVuslcRwQxyFt2w4LATOEUAQqJFDOoR7FEdYGCKCqasA5z7vODX5fb9a0Tce3337HTz/9nSAIXLgv3a7Xi4uL4bre8P3330/NdGstYRjy/v03PO+eCaOALEt5enzk4uKCtutIk5jPd7csyjlCWKIoAiEoh90DbdvS94b5bM6XL3ckaUQURahAslwuaWrN4+MzYRjxLk0njcnxWNJ13fAaK5q248efPpCkCc+7R6SQ5HlG0zZcXFzRVA1JnFNXrXt9i4o4SSmqmtVqifOPlyjlmu5RFFMUlVP3tDVV1bLfH0jTjDTJBk99D42mqirAvR/KsiYMXNtcSklZloRhz3q1oaoqhJDMZjN2uz1v310hhOXh4YG+71iulqxWK9q2R4iasippmpbr6zccDnuyLHt13tu2JcsyXl4KtNbsdjsWi9Xw2IooipAy5CJdgRh+z5NQ1xWXFzfo/vc/wz5E93g8Ho/nD8qroYjaXfQ5R52cmr7gyruWntEBPbaWT63nMTwfdCVjeGzGBvFIwCknd75vl3s7nch43+d/nnQkLrR0HnGwQxv865jxpA6xCMwURBpjp1DWWoFRoOR4mWNBuhDd2MlNcqYSOYW87uty+qcLTfWpHT40jkeFihAnJYgdWtauNO8GFI0PL6bBmHLwU0qkURiL87aMYfNoy8FiBq3JSblzcoW7c3HWeue8Oe3Cea3toIA5/ZxSAUEYEYUpcZQQRSlSuGGxUijs8NpjBVYJsBoMGE7h+Bggv3pdzKAAMmY63teqltNrN+4sOLXXh10DSPd6WwPIwW/jzokYS+zD/dghNJdSTn8fdwG498Np8Ox4rOcDTE+7D04NdBfmi9MDjY/1VSv99WBRnPKmd4srThszuur1q/fZ6/t49TAej8fj8Xj+BKRZRNvWrlVrQSmnkxMojBas1xckSUqaZnTdA8ZIpAjdHB3czrfRQZ7nSyBAyoiyqEFYAhXz5votWZbx9PRMkkSA4OXlhbZrSNOUqi7ZvzwTRRFlcWS3e+LN9VvmsyVaW7q25/b2nq7t2G6uWK+3dF3PxTZDCEWaZLy5jglCxd///iPvv31H21ZsNiuybIYxmiRJaNsWpTR934DoOByfSZKENAuGtnrAYvGOonCDLeu6Io5jsjxlf/tM3/dstxuapqKsXyiLmuurb+g7TRhGCGGJk4i2qylfDlh63ry5pKpqojBBazME4z1FURKGCVVVY4wikAmLecR+/zQMbXUllO12Tdv0kw7ENeNXxHGElDlB4FrNbdsSxzF5nvPw8EBRFGRZhtaav/zlL+z2O+q65PLyijRNqeuM3f6Zv/zlX/j8+TNhGNDrjjiJMKbH2J7Lywu0Fnz88InN5nLwwmdkacrjU8Fms6YsSyBktVpRFiX/+Z//iTFOhWKMoWncgkaSJGijkUqgbY/uXDpcNzWrxYLD4YhSIUpFNE3Nw+MOFUfsX45stxvSdMb+ZY9SIS/7HXGc0ra9e773j6RRRtM0LJcrjDbUTQOY4fxUSCmJooCHhweqqmOWL1itVry8vHA4HNhuthRFQVVV7PcvXFxcYowmDNx5GT3ybtCtYbfbEUUJSgVsNxco5YbIJmk0HMeCKAqGYbIRWWZ56Y6DGqZnvd66QaNFyWo9p25qqrpkuVwSRzHv3r5D94JfPn383c+wD9E9Ho/H4/mDctKRuMa4Cwztq5sxZgi6NWCmbXZuIGXgAtgzF7UdA3g7OKQZAuLJd34Kg62wYAwCw2D2GNzowjXUx1B+uJ+vQ8XzINa41JJxGrvL5sfWsZ20JdYOjW4DRo7tbOiNC/LHQaD/pPxwT3R4bsblt5PH2g4hvbt/OwXIrjY+tZKxKDWG8KfgtO9dAO2CXTM0nhVCuHM+utvt8NdpgUNZLKfn7zYDDMfBWeud8efstHAyBc3DcUupCMOAKHQt9CiICYOYQEYIGSCsOmtGK8C9bhKLFW6BQnDyeY/Kn8k1b4fA3rxOh09tf/lKT+M880zHBmI6z8YMdfzJkc6gbDnpVrQ1Jwf+mc7FtdKZhnwa7VztWjsv+vgL67iQcRoOevLcj9n/rzXRR84Det27Jro5G7I6+v5/bXjopPrxeDwej8fzp0IIQ5Yn7HYvNHXLarUmy3KiKGG/O7BY5DRNSxylbDYXBIHg7u6Wpi1YLGccDgeUCnj/zXeUZcXd5y8URUmgII5Doijm6uqSum4GzYhFSIjjwKlXAkkUueA0ikKapuHy8oosy5yXPQiH4aQhQRoRhRFN48LM+XxG07R0fYfue6SK2GzWKCUIAsVqvSQKEz59+kTfdUghUFJy8+7t4H9/JE8TlIRASdI0BwFJEmOtput6Xg4HwjCgKuvhmpehpa5YLpf0uuPp+YAUEYEK+Zd/+Y48T/j4qSHLLzBGczwcSVLD1eXVtOhgLWw2F3z88AmsoO/dc726dB7utm0ByWazpQnHofPuujXLMqcymedEUch6vSLNEoqiQCrJbvfM9mLDfvcyNaJnsxlSQt933N3tiOOYJI2pa6dS6fUMrV0o/XJ4Rms9+OQlF5eX7vwHbsdknKQgJKu1c4RHKh2c9ycVpBByujbt+pauazEWNps1XdeSRDEARVG6XavGoIIQIdXk0E+kZLN1rnp3LS4Ig4ggiKjKBqUkZVGRphl5knP/5Z7t9oJWt1RlRRSHaG1JkpQgCDGmp+v6YbhsRNu2GGP45psb2rZzDndryPN8+j2h7XryPMNakFKwWMx5eLhnNsvc/KYgcsNNVYAQgiRJWK7m3N5+om1byrLgzZt3pImiLlv+9re/cTgcubjYkmUJn+8+YZ5aVus5s3yGVE43gxUcC9de/z3k736Hx+PxeDye/08ZW+S4ZvGrprKdhnR2XecUHqZ/5Safhl5OTXR3kWGsQFs5RKwSKyQGhT3XuSDRVqCNcH8OPzN+rTfQGUNnLL2xaOvc26O/++TxPnnbp4avdp7pMbB0qg47BdS6t3SduwB233PyWEuhhgD3dL8nB/wwTHM4J6dbS9d1tE1L3/VnTWQ3RBJO7WTsqGpxx9N1PV3X0rYNbecuavu+n47r9cBNTgE1Q5orxjK2xWAxw3madgtIObjP3SWdO0cn/YkUAYEKicKYJM7clss4JQwilAqQQqIGV7prhTtbzHgDhmGjp4b1qDQZFSmT4sT8c/AM40KGePWnkmoK0EfFkDH2VQh9HsBPOp3he3TvQvKu7169Vn3nfJxN09K1enhf6OmcW3sK9sfX7fycwanpP7bgz1+j8fmOf7rBocPNnP1d98Nt/EWsn+7ndIq8zsXj8Xg8nj8TZf1C19fEccxf//pX3r17ixCaY/GIoQDRMJvFXF5viLOAD59/5vPDLTKU7A97vnz5zNXVJWmc8OnDB/a7HbrvsEITpQFNV9Lqmk5XGNFxrA70psMKd/1YVjX7fYExAW3ndtPFacLzyzP7l2cen77Q9g2L1YIf/vYDN+/fY4Wg6Vr2xx0yNFhq6vYFrWu2myV12fB4v8N0Bt136K4lCgKiIED0htBKlBZczFZEVhGbENsYsjglVIK6KqjrAmEty9kG3SkkMYEMMbpnuVoShDl1a+l6wfXbG+IhFO7ansNLQRrn5OkC3YFSCYKI+WxNUdQ8P+85vBTUVYtSbjEhSVwIXpYNTW3QWpIm7jq5aZvhGk6zWq0wxrrr+b5FBoaX4yNBaAlDy7F4oqj29H2DtYamacmyGX2vWS4XaNPRdhUqAGNa9i+PWHqiSBFGAXEcYY1AyYQoyvjy5Z4oCjHCsNquqdqGsu0IopyygqfnAmMhCiO6tkfKgIuLKy62V/SdpWk0gYrpOqirjjSe0TeWvoO+EyznW4qiodMCFUVYAb3tyWYJgRTkSUJdlFRFRRJEpFHGerHGdJq6LKnLgjgMKKqSMAl53D1xrEqQCoNgvlwhVMThUNP1gijJkULx8HBPECpmswwhLUEg6IZz9u233w7N8pj9ruR47JAyJI5TsjRB64amORKFkCUBUhiyOB6sn4Kua0mziMNxz3y+QPeCpu5pGvc7wWKR8fPH/6K3Nekspu4qnvd7qrrmeCjRnUYJhZKCy4vt736GfRPd4/F4PJ4/KNaooUUM1gqkHS3mGjebUQ8mEzdMUkjrGtLgBoMiJsf2GLzDmX5lDCSHAZ4COeWDp9awQwimIaN2qFtb4QLFqbE7qE0EbmCpNmCGFrdzmJ/uU6vTsUg5aDysmI7J+a7FoOxQBNKpZqR0GhipBve1GENwgTbjuRkWERjMIsNxGWPc8UrXzJbiFPhLxBA2W0SvhwRau+dmtPs/6wab9oOX3Uo3bNMYBpe4OzYhlRsAikRY8SrQBpBCIr8aDGsGrQx2HM4pUVISKEUQBARKoYRC2BCDwiCdBV+4walgMEOAr3U/7CCwTr9uDLrv6fuOvuvQelicsE5ZYscKPXZyokspUUJghUBbO+1lUGGIkhI5NNHPdSrDszsz6xi3UGEsWo+LOaNLXE8Bt2FcANGTTmYcJmrM6fV0jffT8NbRGiOlmFyk026DcVFjWMUwRky7ILQBY91cgbFx7l7boX2OwQozLbBMzXZz7s5XnIb1ejwej8fj+TPQNCVxlLOYL4ii2DWXI4W2EEYB80VC18Ldl1senh9BWt5/e0NVFXRVw3fffcdqteT+/h4pJT/88G8URcWHTz+RpluMMTw83NG2LbPZjKI4kOcp+/2exWJB3/dUlWa7vaaqCowRXFysub29dY1gwNieIFCoIKBtO/L5gqo40OuOvm9Ik4imLYgjRR8quq6nLBvyvKVpaqSQvL1+g+56gijBakMkI1YXl1htCG3E08ueh/sHUE5BIoAsy1Eypqpa+s6im4a2cYF+pwNe9i376ogxkqura+q6RklFGCiKo/v6xfaaIMgJw9DNIbISJUMAwjBks9mw3+/Z7XZUVcVivqYoKg6HF+zlBW/enopGURQThhHz+YI8M8hA0/ctdV0QRRHG9kO5pMMYQ5zEWCuJ44SXl2dMHBAEiiSJnTNeGoIwIIrc71Zl2RCGK9brLff393StRkk1FGkMSPe7ktaG3gqiMOPq8i1RIKj7FiEUeT4jTTL63hCGEWmQIiQ0TU1rtWueI1EyQmtDHGcUxRNaG/e+0z297lhvVkQqpCkrsjihrWpEnLJarrj74hZ9uk4gpHEu+qpltdpwPBakSUZZOp97VTWYYTFBKIjjlN3TM01doQJJliWAoe1qsmxG13UIAUEQkCYz7r488vi4493bG9I04OPHfxAEkqY58vTYsdlckSYxUgjSJEVrw+G4R+uOLE2YzXK6VpBlc96//wv7l2d6XRElAVVbUtUFQRjQdi2m14RhCMhhjphgtZr/7mfYh+gej8fj8fxhkSf58hCkn1QcAG6Lp7UGIS3SMmhR1OAqd21bt9J/SnHNed/YSqwcfd0uAD25pxkGSQJyDK1Hr/oQgnNqvdsptQZj3EBOYezUVD4P0SXOHe62/7l7wr4OJcfG8RT0D0c//PQrbYcxo87G+San4zPgQlc53L9BWPe4VorJjONC9eEZjfc1BPLG9EPYazHWDSwa3dxj8H4KVwVqep5fhaznDfQhKJ/O3dBWP61iiGFR4XyI6eBvF24hwFinRbGAmAbGujB6bJz3RqP7Uxu/693NGBcWjwsg46spXx/u69djcJBPPnRx0gSdvntwsjOG5WN4LYfQ/FwhY07HYU5fHyrx7l4H1Y4xTnwz6lrEsNNBCokUJz+6EKMTXZy1xsdG+nAbjpPpe8+e6Vm5/LQYIKZFphEp1at/ezwej8fj+ePjQuaIeR7Qtg1aa6fBMCFNU9H3HdYGNI3zeG8v1mjtdkhmeQ7AbrejqXqiKGaxWGAtLFdLuq6b3NxSyilIl1KSZRnWWhccL+YEQUgYhsxnM9q2nXb0CRRlWZEmTh1zKA8uxO47F7h2DUo6rV1V1ygZuutRY5z2RUCezXj79h1f7u5I4wzbOwXferXmeCjoB72e1hqlJHEU0XYNi8Wc3a6YBndqU9Hryrm+2wapJIlK6bqOuqmnYzgeW4riSJyEhKEiUIokSTgcDiRJwvW1C9y7rqNpGsqyJE1Td67S+eAtT2jbhq5rCcNwGo6qtSZNUzcMMwio23IIfsfdsu76s21bojAhzzOCIKTXmqI4ogJX1KjrGiEsh8OBKIyJ45j9/ji8XmPob4iiiL7vafuWIIzddal1161xHBEFc/ZP9ywXa47HkuPxyOXFFW3bIoQkDEO0cf5zIQR5PgNbDbtznTrFWst+/8zLS8FqtRgC8galFEpp2q6l7zR13Z7pcSKUUhirORwOCBUihHDnMnHDPbMspyyPpEnKbv9MXTdkWcpsNgNruLu7I89Trq6uBh/+EqOhrEo3YJSAXvdcXV1SFEceHg6EUUBZHgiCgOOxYLXSzJYzhFHsDgeMMBjbsn955i/ffcfnzw9k6ZrVaoUQkrIoaDrnXs/znK7rWCw2PD48IkNIkgXL5ZKyqAmDkDTNfvcz7EN0j8fj8Xj+oLwe7Hhqbo9uamMtwriA/NfEEuffe3KYvw7R5TgQ81d/3ulFxBBqngfxYgh9R5/2GFha6wJUIVzsbcV5uPq63W6lGIZBjvoNw0kH8392jszgQTeGs1B7dFcPLnl7PjDTnHzsjF+XU5jLFDCf9Dhm+NqolmFwep8PvBzP+6/dzvUm421qoo8B+ldP3T0Hg32lxXGN9/HcTef0TF1ijKHXvbt1PX3/+jY518dFkMk1fzqn54sY58d8Cqt//fX4NY/41HSf9Cv61DJ/FaLrKel2jzEO0T1/7/zzMZ2O1S2U/NM5PLtNz+0331Xj94yPNeh27PgL1/DYQv3mPXg8Ho/H4/nj8d//24//rw/B4/H8XyDsr/+m4vF4PB6Px+PxeDwej8fj8Xg8Hs+fHr+P1OPxeDwej8fj8Xg8Ho/H4/F4PJ7fwIfoHo/H4/F4PB6Px+PxeDwej8fj8fwGPkT3eDwej8fj8Xg8Ho/H4/F4PB6P5zfwIbrH4/F4PB6Px+PxeDwej8fj8Xg8v4EP0T0ej8fj8Xg8Ho/H4/F4PB6Px+P5DXyI7vF4PB6Px+PxeDwej8fj8Xg8Hs9v4EN0j8fj8Xg8Ho/H4/F4PB6Px+PxeH4DH6J7PB6Px+PxeDwej8fj8Xg8Ho/H8xv4EN3j8Xg8Ho/H4/F4PB6Px+PxeDye3+B/A16EonDf1NeYAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -821,23 +890,115 @@ "output_type": "display_data" }, { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=3343, ip=10.0.102.235)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n" + "\u001b[36m(autoscaler +12m47s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", + "\u001b[36m(autoscaler +12m52s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", + "\u001b[36m(autoscaler +13m37s)\u001b[0m [autoscaler] Cluster upscaled to {104 CPU, 8 GPU}.\n", + "\u001b[36m(autoscaler +15m32s)\u001b[0m [autoscaler] [8CPU-32GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", + "\u001b[36m(autoscaler +15m32s)\u001b[0m [autoscaler] [8CPU-32GB|m5.2xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", + "\u001b[36m(autoscaler +16m2s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 1 to 2).\n", + "\u001b[36m(autoscaler +16m2s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", + "\u001b[36m(autoscaler +16m7s)\u001b[0m [autoscaler] Cluster upscaled to {112 CPU, 8 GPU}.\n", + "\u001b[36m(autoscaler +16m52s)\u001b[0m [autoscaler] Cluster upscaled to {160 CPU, 12 GPU}.\n", + "\u001b[36m(autoscaler +19m52s)\u001b[0m [autoscaler] Downscaling node i-0e941ed71ef3480ee (node IP: 10.0.34.27) due to node idle termination.\n", + "\u001b[36m(autoscaler +19m52s)\u001b[0m [autoscaler] Cluster resized to {112 CPU, 8 GPU}.\n", + "\u001b[36m(autoscaler +20m42s)\u001b[0m [autoscaler] [1xT4:8CPU-32GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", + "\u001b[36m(autoscaler +20m47s)\u001b[0m [autoscaler] [1xT4:8CPU-32GB|g4dn.2xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", + "\u001b[36m(autoscaler +20m47s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 1 to 2).\n", + "\u001b[36m(autoscaler +20m52s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", + "\u001b[36m(autoscaler +21m32s)\u001b[0m [autoscaler] Cluster upscaled to {120 CPU, 9 GPU}.\n", + "\u001b[36m(autoscaler +21m37s)\u001b[0m [autoscaler] Cluster upscaled to {168 CPU, 13 GPU}.\n", + "\u001b[36m(autoscaler +25m22s)\u001b[0m [autoscaler] Downscaling node i-0ffe5abae6e899f5a (node IP: 10.0.60.138) due to node idle termination.\n", + "\u001b[36m(autoscaler +25m27s)\u001b[0m [autoscaler] Cluster resized to {120 CPU, 9 GPU}.\n", + "\u001b[36m(autoscaler +28m22s)\u001b[0m [autoscaler] Downscaling node i-0aa72cef9b8921af5 (node IP: 10.0.31.199) due to node idle termination.\n", + "\u001b[36m(autoscaler +28m27s)\u001b[0m [autoscaler] Cluster resized to {112 CPU, 8 GPU}.\n" ] }, { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(autoscaler +16m26s)\u001b[0m [autoscaler] [4xL4:48CPU-192GB] Attempting to add 2 node(s) to the cluster (increasing from 1 to 3).\n", - "\u001b[36m(autoscaler +16m26s)\u001b[0m [autoscaler] [4xL4:48CPU-192GB] Launched 2 instances.\n", - "\u001b[36m(autoscaler +17m11s)\u001b[0m [autoscaler] Cluster upscaled to {152 CPU, 14 GPU}.\n", - "\u001b[33m(raylet)\u001b[0m WARNING: 4 PYTHON worker processes have been started on node: 97b39558bc8a3057162823cead1b8e035f1be130c49bb311e538ed2d with address: 10.0.52.172. This could be a result of using a large number of actors, or due to tasks blocked in ray.get() calls (see https://github.com/ray-project/ray/issues/3644 for some discussion of workarounds).\n", - "\u001b[36m(autoscaler +1h19m21s)\u001b[0m [autoscaler] Downscaling node i-03a133888407b8cf8 (node IP: 10.0.103.152) due to node idle termination.\n", - "\u001b[36m(autoscaler +1h19m21s)\u001b[0m [autoscaler] Downscaling node i-06023e83fb012b7ae (node IP: 10.0.90.122) due to node idle termination.\n", - "\u001b[36m(autoscaler +1h19m26s)\u001b[0m [autoscaler] Cluster resized to {56 CPU, 6 GPU}.\n" + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Using CPython 3.12.11 interpreter at: /home/ray/anaconda3/bin/python3.12\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Creating virtual environment at: .venv\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Building doggos @ file:///tmp/ray/session_2025-08-21_18-48-13_464408_2298/runtime_resources/working_dir_files/_ray_pkg_f79228c33bd2a431/doggos\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pillow (6.3MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading grpcio (5.9MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sqlalchemy (3.2MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pydantic-core (1.9MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading jedi (1.5MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading virtualenv (5.7MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pandas (11.4MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading setuptools (1.1MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading uvloop (4.5MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cuda-nvrtc-cu12 (22.6MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sympy (6.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading numpy (15.9MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading kiwisolver (1.4MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading tokenizers (3.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pyarrow (38.2MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading botocore (13.3MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading fonttools (4.7MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading widgetsnbextension (2.1MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow-skinny (5.6MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading aiohttp (1.6MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading networkx (1.9MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pygments (1.2MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading debugpy (4.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading py-spy (2.6MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scikit-learn (12.5MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading hf-xet (3.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading matplotlib (8.2MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading torch (783.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading transformers (10.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scipy (33.5MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading polars (36.7MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow (26.1MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading triton (148.5MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Built doggos @ file:///tmp/ray/session_2025-08-21_18-48-13_464408_2298/runtime_resources/working_dir_files/_ray_pkg_f79228c33bd2a431/doggos\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pillow\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading grpcio\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sqlalchemy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pydantic-core\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading jedi\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading virtualenv\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading setuptools\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading uvloop\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cuda-cupti-cu12\u001b[32m [repeated 13x across cluster]\u001b[0m\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sympy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading kiwisolver\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading tokenizers\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading fonttools\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading widgetsnbextension\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow-skinny\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading aiohttp\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading networkx\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pygments\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading debugpy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading py-spy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading hf-xet\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading matplotlib\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading transformers\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scikit-learn\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading numpy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading botocore\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pandas\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading polars\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cuda-nvrtc-cu12\u001b[32m [repeated 2x across cluster]\u001b[0m\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scipy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pyarrow\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-curand-cu12\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cusparselt-cu12\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading triton\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cublas-cu12\u001b[32m [repeated 5x across cluster]\u001b[0m\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading torch\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m warning: Failed to hardlink files; falling back to full copy. This may lead to degraded performance.\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m If the cache and target directories are on different filesystems, hardlinking may not be supported.\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m If this is intentional, set `export UV_LINK_MODE=copy` or use `--link-mode=copy` to suppress this warning.\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cudnn-cu12\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Installed 172 packages in 1.96s\n" ] } ], @@ -845,7 +1006,7 @@ "# Top matches by embedding similarity.\n", "embeddings_ds = ray.data.read_parquet(embeddings_path)\n", "top_matches = get_top_matches(embedding, embeddings_ds, n=5)\n", - "display_top_matches(url, top_matches)" + "display_top_matches(url, top_matches)\n" ] }, { diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/02-Distributed-Training.ipynb b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/02-Distributed-Training.ipynb index 59c16348c27a..26a8ca0766ed 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/02-Distributed-Training.ipynb +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/02-Distributed-Training.ipynb @@ -32,14 +32,35 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[92mSuccessfully registered `matplotlib, torch` and 4 other packages to be installed on all cluster nodes.\u001b[0m\n", - "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_eys8cskj5aivghbf773dp2vmcd?workspace-tab=dependencies\u001b[0m\n" + "\u001b[92mSuccessfully registered `ipywidgets, matplotlib` and 4 other packages to be installed on all cluster nodes.\u001b[0m\n", + "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_1dp3fa7w5hu3i83ldsi7lqvp9t?workspace-tab=dependencies\u001b[0m\n", + "\u001b[92mSuccessfully registered `doggos` package to be installed on all cluster nodes.\u001b[0m\n", + "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_1dp3fa7w5hu3i83ldsi7lqvp9t?workspace-tab=dependencies\u001b[0m\n" ] } ], "source": [ "%%bash\n", - "pip install -q \"matplotlib==3.10.0\" \"torch==2.7.0\" \"transformers==4.52.3\" \"scikit-learn==1.6.0\" \"mlflow==2.19.0\" \"ipywidgets==8.1.3\"" + "pip install -q -r /home/ray/default/requirements.txt\n", + "pip install -q -e /home/ray/default/doggos\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note**: A kernel restart may be required for all dependencies to become available. \n", + "\n", + "If using **uv**, then:\n", + "1. Turn off the runtime dependencies (`Dependencies` tab up top > Toggle off `Pip packages`). And no need to run the `pip install` commands above.\n", + "2. Change the python kernel of this notebook to use the `venv` (Click on `base (Python x.yy.zz)` on top right cordern of notebook > `Select another Kernel` > `Python Environments...` > `Create Python Environment` > `Venv` > `Use Existing`) and done! Now all the notebook's cells will use the virtual env.\n", + "3. Change the py executable to use `uv run` instead of `python` by adding this line after importing ray.\n", + "```python\n", + "import os\n", + "os.environ.pop(\"RAY_RUNTIME_ENV_HOOK\", None)\n", + "import ray\n", + "ray.init(runtime_env={\"py_executable\": \"uv run\", \"working_dir\": \"/home/ray/default\"})\n", + "```" ] }, { @@ -49,7 +70,7 @@ "outputs": [], "source": [ "%load_ext autoreload\n", - "%autoreload all" + "%autoreload all\n" ] }, { @@ -61,7 +82,17 @@ "import os\n", "import ray\n", "import sys\n", - "sys.path.append(os.path.abspath(\"..\"))" + "sys.path.append(os.path.abspath(\"../doggos/\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# If using UV\n", + "# os.environ.pop(\"RAY_RUNTIME_ENV_HOOK\", None)\n" ] }, { @@ -73,17 +104,19 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:26:58,662\tINFO worker.py:1723 -- Connecting to existing Ray cluster at address: 10.0.52.172:6379...\n", - "2025-06-23 14:26:58,674\tINFO worker.py:1908 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttps://session-gcwehd9xxjzkv5lxv8lgcdgx2n.i.anyscaleuserdata.com \u001b[39m\u001b[22m\n", - "2025-06-23 14:26:58,721\tINFO packaging.py:588 -- Creating a file package for local module '../'.\n", - "2025-06-23 14:26:58,781\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_df54fa2aa282ae62.zip' (13.77MiB) to Ray cluster...\n", - "2025-06-23 14:26:58,845\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_df54fa2aa282ae62.zip'.\n" + "2025-08-22 00:26:10,081\tINFO worker.py:1747 -- Connecting to existing Ray cluster at address: 10.0.52.10:6379...\n", + "2025-08-22 00:26:10,093\tINFO worker.py:1918 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttps://session-466hy7cqu1gzrp8zk8l4byz7l7.i.anyscaleuserdata.com \u001b[39m\u001b[22m\n", + "2025-08-22 00:26:10,108\tINFO packaging.py:588 -- Creating a file package for local module '/home/ray/default/doggos/doggos'.\n", + "2025-08-22 00:26:10,114\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_86fb5903fc73ddf5.zip' (0.03MiB) to Ray cluster...\n", + "2025-08-22 00:26:10,115\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_86fb5903fc73ddf5.zip'.\n", + "2025-08-22 00:26:10,118\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_e23a31aacd3983306ac777f59fba2eff0e5e9963.zip' (1.16MiB) to Ray cluster...\n", + "2025-08-22 00:26:10,124\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_e23a31aacd3983306ac777f59fba2eff0e5e9963.zip'.\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0ee0ee1a3af84e0481f83f2e8802e581", + "model_id": "143376de26ec4095b9e1f3956e0a86f5", "version_major": 2, "version_minor": 0 }, @@ -111,11 +144,11 @@ " \n", " \n", " Ray version:\n", - " 2.47.1\n", + " 2.48.0\n", " \n", " \n", " Dashboard:\n", - " http://session-gcwehd9xxjzkv5lxv8lgcdgx2n.i.anyscaleuserdata.com\n", + " http://session-466hy7cqu1gzrp8zk8l4byz7l7.i.anyscaleuserdata.com\n", "\n", "\n", "\n", @@ -124,7 +157,7 @@ "\n" ], "text/plain": [ - "RayContext(dashboard_url='session-gcwehd9xxjzkv5lxv8lgcdgx2n.i.anyscaleuserdata.com', python_version='3.12.11', ray_version='2.47.1', ray_commit='e06f523c450fb1c99d8f347f8bfcc4085cc68b66')" + "RayContext(dashboard_url='session-466hy7cqu1gzrp8zk8l4byz7l7.i.anyscaleuserdata.com', python_version='3.12.11', ray_version='2.48.0', ray_commit='61d1966b0b02ce07f95d8f046ea7f6f92f7be190')" ] }, "execution_count": null, @@ -139,11 +172,11 @@ " # connect to existing ray runtime (from previous notebook if still running)\n", " address=os.environ.get(\"RAY_ADDRESS\", \"auto\"),\n", " runtime_env={\n", - " \"env_vars\": {\"RAY_TRAIN_V2_ENABLED\": \"1\"}, \n", - " # working_dir to import doggos (default working_dir=\".\")\n", - " \"working_dir\": \"../\",\n", + " \"env_vars\": {\"RAY_TRAIN_V2_ENABLED\": \"1\"},\n", + " # \"py_executable\": \"uv run\", # if using uv \n", + " # \"working_dir\": \"/home/ray/default\", # if using uv \n", " },\n", - ")" + ")\n" ] }, { @@ -154,7 +187,7 @@ "source": [ "%%bash\n", "# This will be removed once Ray Train v2 is enabled by default.\n", - "echo \"RAY_TRAIN_V2_ENABLED=1\" > /home/ray/default/.env" + "echo \"RAY_TRAIN_V2_ENABLED=1\" > /home/ray/default/.env\n" ] }, { @@ -176,7 +209,7 @@ "source": [ "# Load env vars in notebooks.\n", "from dotenv import load_dotenv\n", - "load_dotenv()" + "load_dotenv()\n" ] }, { @@ -201,7 +234,7 @@ "source": [ "def add_class(row):\n", " row[\"class\"] = row[\"path\"].rsplit(\"/\", 3)[-2]\n", - " return row" + " return row\n" ] }, { @@ -214,7 +247,7 @@ "train_ds = ray.data.read_images(\"s3://doggos-dataset/train\", include_paths=True, shuffle=\"files\")\n", "train_ds = train_ds.map(add_class)\n", "val_ds = ray.data.read_images(\"s3://doggos-dataset/val\", include_paths=True)\n", - "val_ds = val_ds.map(add_class)" + "val_ds = val_ds.map(add_class)\n" ] }, { @@ -237,7 +270,7 @@ "def convert_to_label(row, class_to_label):\n", " if \"class\" in row:\n", " row[\"label\"] = class_to_label[row[\"class\"]]\n", - " return row" + " return row\n" ] }, { @@ -250,7 +283,7 @@ "from PIL import Image\n", "import torch\n", "from transformers import CLIPModel, CLIPProcessor\n", - "from doggos.embed import EmbedImages" + "from doggos.embed import EmbedImages\n" ] }, { @@ -285,14 +318,14 @@ " concurrency=4,\n", " batch_size=64,\n", " num_gpus=1,\n", - " accelerator_type=\"L4\",\n", + " accelerator_type=\"T4\",\n", " )\n", " ds = ds.drop_columns([\"image\"])\n", " return ds\n", "\n", " def save(self, fp):\n", " with open(fp, \"w\") as f:\n", - " json.dump(self.class_to_label, f)" + " json.dump(self.class_to_label, f)\n" ] }, { @@ -304,16 +337,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:27:10,597\tINFO dataset.py:3048 -- Tip: Use `take_batch()` instead of `take() / show()` to return records in pandas or numpy batch format.\n", - "2025-06-23 14:27:10,599\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_65_0\n", - "2025-06-23 14:27:10,612\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_65_0. Full logs are in /tmp/ray/session_2025-06-23_13-49-50_102769_2149/logs/ray-data\n", - "2025-06-23 14:27:10,613\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_65_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)] -> AllToAllOperator[Aggregate] -> LimitOperator[limit=1]\n" + "2025-08-22 00:26:17,487\tINFO dataset.py:3057 -- Tip: Use `take_batch()` instead of `take() / show()` to return records in pandas or numpy batch format.\n", + "2025-08-22 00:26:17,490\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_72_0\n", + "2025-08-22 00:26:17,522\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_72_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", + "2025-08-22 00:26:17,523\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_72_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)] -> AllToAllOperator[Aggregate] -> LimitOperator[limit=1]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6c2629752381401bb193d0d84fa68963", + "model_id": "aa17a81a379547a6b397907ffafec33b", "version_major": 2, "version_minor": 0 }, @@ -327,7 +360,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ffc73aed04544803a19756d5fc09c575", + "model_id": "08f021fb0f014335ab09e4baf362d6c3", "version_major": 2, "version_minor": 0 }, @@ -341,7 +374,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7cb26d7641104cfdabb606292026da04", + "model_id": "bb9749220ab644db83ab9ac57348c6df", "version_major": 2, "version_minor": 0 }, @@ -355,7 +388,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "cbd97058f69741b0a40e8bb312a88065", + "model_id": "7cd4f59d163149d8b1200348eaabebab", "version_major": 2, "version_minor": 0 }, @@ -369,7 +402,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "654ec007f7184ec0a9c2c487dd3df860", + "model_id": "9a7fe9e6350a4ce88f4c4a12c3e83fc1", "version_major": 2, "version_minor": 0 }, @@ -383,7 +416,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "58acb9149bf644be8386a8da980ea125", + "model_id": "38470d9ffbeb4f4ca11c2c7d4e70886b", "version_major": 2, "version_minor": 0 }, @@ -397,7 +430,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "19656545491b4ae4b239bb7773341210", + "model_id": "ea5ecfb1d6fb4eadb1d55130cdf0ec04", "version_major": 2, "version_minor": 0 }, @@ -411,7 +444,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "54a23a3f33054741981ad75230221b54", + "model_id": "e75cd967f3494a7cad6204c9f523ccad", "version_major": 2, "version_minor": 0 }, @@ -425,7 +458,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6ec35ea4ab2244748e2c6fc2d1b280d8", + "model_id": "37294f34620d4ed99e4c6db7f489e870", "version_major": 2, "version_minor": 0 }, @@ -440,7 +473,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:27:17,996\tINFO streaming_executor.py:227 -- ✔️ Dataset dataset_65_0 execution finished in 7.38 seconds\n" + "2025-08-22 00:26:17,662\tWARNING resource_manager.py:130 -- ⚠️ Ray's object store is configured to use only 28.2% of available memory (67.8GB out of 240.5GB total). For optimal Ray Data performance, we recommend setting the object store to at least 50% of available memory. You can do this by setting the 'object_store_memory' parameter when calling ray.init() or by setting the RAY_DEFAULT_OBJECT_STORE_MEMORY_PROPORTION environment variable.\n", + "2025-08-22 00:26:29,748\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_72_0 execution finished in 12.22 seconds\n" ] } ], @@ -449,7 +483,7 @@ "preprocessor = Preprocessor()\n", "preprocessor = preprocessor.fit(train_ds, column=\"class\")\n", "train_ds = preprocessor.transform(ds=train_ds)\n", - "val_ds = preprocessor.transform(ds=val_ds)" + "val_ds = preprocessor.transform(ds=val_ds)\n" ] }, { @@ -467,7 +501,7 @@ "metadata": {}, "outputs": [], "source": [ - "import shutil" + "import shutil\n" ] }, { @@ -479,15 +513,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:19:45,048\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_40_0\n", - "2025-06-23 14:19:45,067\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_40_0. Full logs are in /tmp/ray/session_2025-06-23_13-49-50_102769_2149/logs/ray-data\n", - "2025-06-23 14:19:45,069\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_40_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" + "2025-08-22 00:26:30,402\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_80_0\n", + "2025-08-22 00:26:30,433\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_80_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", + "2025-08-22 00:26:30,435\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_80_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a676da85e459434b82d231b8cf23a213", + "model_id": "9a4fc2809c524fdea65f33187089279c", "version_major": 2, "version_minor": 0 }, @@ -498,17 +532,10 @@ "metadata": {}, "output_type": "display_data" }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-06-23 14:19:45,088\tINFO actor_pool_map_operator.py:633 -- Scaling up actor pool by 4 (reason=scaling to min size, running=0, restarting=0, pending=0)\n" - ] - }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f3e0d36a35444d60b6234498fa910777", + "model_id": "7ed00402d404430596be9d367c53de16", "version_major": 2, "version_minor": 0 }, @@ -522,7 +549,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "befd7f324d954c7f8ca324b50d807239", + "model_id": "cb833a76da3245a0bfcf5d7d7959e847", "version_major": 2, "version_minor": 0 }, @@ -536,7 +563,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "39a445c091c5457ea4bfb31a88b9215b", + "model_id": "c2223e2ee8e046079a76f8cdcef8abf5", "version_major": 2, "version_minor": 0 }, @@ -550,7 +577,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "dd118260e28a42e0a4325a8f5036bc85", + "model_id": "14d3a76ba8f84b3ba6729d43dcd221d8", "version_major": 2, "version_minor": 0 }, @@ -564,7 +591,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a49e0bbbbf0e424da8b3a514885c0148", + "model_id": "5b5ea2e535b04e95b3bb8739e478a678", "version_major": 2, "version_minor": 0 }, @@ -575,23 +602,35 @@ "metadata": {}, "output_type": "display_data" }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(autoscaler +25s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n", + "\u001b[36m(autoscaler +25s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", + "\u001b[36m(autoscaler +30s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", + "\u001b[36m(autoscaler +1m15s)\u001b[0m [autoscaler] Cluster upscaled to {104 CPU, 8 GPU}.\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=18628, ip=10.0.102.235)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "2025-06-23 14:19:57,926\tINFO actor_pool_map_operator.py:661 -- Scaled down actor pool by 1 (reason=None; running=3, restarting=0, pending=0)\n", - "2025-06-23 14:19:58,259\tINFO streaming_executor.py:227 -- ✔️ Dataset dataset_40_0 execution finished in 13.19 seconds\n", - "2025-06-23 14:19:58,573\tINFO dataset.py:4603 -- Data sink Parquet finished. 2880 rows and 5.9MB data written.\n", - "2025-06-23 14:19:58,584\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_43_0\n", - "2025-06-23 14:19:58,602\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_43_0. Full logs are in /tmp/ray/session_2025-06-23_13-49-50_102769_2149/logs/ray-data\n", - "2025-06-23 14:19:58,603\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_43_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" + "\u001b[36m(_MapWorker pid=3320, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "\u001b[36m(MapBatches(drop_columns)->Write pid=44781, ip=10.0.171.239)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\n", + "\u001b[36m(_MapWorker pid=3323, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)\u001b[0m\n", + "\u001b[36m(MapBatches(drop_columns)->Write pid=44781, ip=10.0.171.239)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\u001b[32m [repeated 32x across cluster]\u001b[0m\n", + "2025-08-22 00:29:10,480\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_80_0 execution finished in 160.04 seconds\n", + "2025-08-22 00:29:10,570\tINFO dataset.py:4621 -- Data sink Parquet finished. 2880 rows and 5.9MB data written.\n", + "2025-08-22 00:29:10,582\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_83_0\n", + "2025-08-22 00:29:10,601\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_83_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", + "2025-08-22 00:29:10,603\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_83_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8cac399a609346a89cab141cb4bd91af", + "model_id": "0abbbeeb2a35431a95d00c4c921ee61b", "version_major": 2, "version_minor": 0 }, @@ -602,17 +641,10 @@ "metadata": {}, "output_type": "display_data" }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-06-23 14:19:58,620\tINFO actor_pool_map_operator.py:633 -- Scaling up actor pool by 4 (reason=scaling to min size, running=0, restarting=0, pending=0)\n" - ] - }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "231149ef4ba34ab9bb7c0346956bfb21", + "model_id": "bf9288444a51499a8b1083a1b784b210", "version_major": 2, "version_minor": 0 }, @@ -626,7 +658,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "50bcce7953b944eca37c79f5c024c982", + "model_id": "8ec81536f155464c9a2ecf135cf83d80", "version_major": 2, "version_minor": 0 }, @@ -640,7 +672,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "82da80edde0546d5935e0426960a904a", + "model_id": "512fded93ca04523b3ed812aec1ada77", "version_major": 2, "version_minor": 0 }, @@ -654,7 +686,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1ca5f7798fd744e99513b5cdfbf144f4", + "model_id": "b3b9c393001d49f4b0535b625a132edb", "version_major": 2, "version_minor": 0 }, @@ -668,7 +700,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5b1f7618f9e94098a64c33bf49f4d12c", + "model_id": "5f38287bfd4b4795a3839f44b0d2b1a8", "version_major": 2, "version_minor": 0 }, @@ -683,11 +715,36 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=33082, ip=10.0.102.235)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 4x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)\u001b[0m\n", - "2025-06-23 14:20:07,331\tINFO actor_pool_map_operator.py:661 -- Scaled down actor pool by 1 (reason=None; running=3, restarting=0, pending=0)\n", - "2025-06-23 14:20:07,854\tINFO actor_pool_map_operator.py:661 -- Scaled down actor pool by 1 (reason=None; running=2, restarting=0, pending=0)\n", - "2025-06-23 14:20:08,323\tINFO streaming_executor.py:227 -- ✔️ Dataset dataset_43_0 execution finished in 9.72 seconds\n", - "2025-06-23 14:20:08,372\tINFO dataset.py:4603 -- Data sink Parquet finished. 720 rows and 1.5MB data written.\n" + "2025-08-22 00:29:12,374\tWARNING streaming_executor_state.py:764 -- Operator produced a RefBundle with a different schema than the previous one. Previous schema: image: extension>\n", + "path: string, new schema: image: extension>\n", + "path: string. This may lead to unexpected behavior.\n", + "2025-08-22 00:29:13,110\tWARNING streaming_executor_state.py:764 -- Operator produced a RefBundle with a different schema than the previous one. Previous schema: image: extension>\n", + "path: string\n", + "class: string\n", + "label: int64, new schema: image: extension>\n", + "path: string\n", + "class: string\n", + "label: int64. This may lead to unexpected behavior.\n", + "\u001b[36m(_MapWorker pid=3910, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "\u001b[36m(MapBatches(drop_columns)->Write pid=121066)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\u001b[32m [repeated 7x across cluster]\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(autoscaler +3m10s)\u001b[0m [autoscaler] [8CPU-32GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", + "\u001b[36m(autoscaler +3m10s)\u001b[0m [autoscaler] [8CPU-32GB|m5.2xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(_MapWorker pid=4731, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(drop_columns)->Write pid=45557, ip=10.0.171.239)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\u001b[32m [repeated 6x across cluster]\u001b[0m\n", + "2025-08-22 00:29:24,485\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_83_0 execution finished in 13.88 seconds\n", + "2025-08-22 00:29:24,531\tINFO dataset.py:4621 -- Data sink Parquet finished. 720 rows and 1.5MB data written.\n" ] } ], @@ -699,7 +756,7 @@ "preprocessed_train_path = os.path.join(preprocessed_data_path, \"preprocessed_train\")\n", "preprocessed_val_path = os.path.join(preprocessed_data_path, \"preprocessed_val\")\n", "train_ds.write_parquet(preprocessed_train_path)\n", - "val_ds.write_parquet(preprocessed_val_path)" + "val_ds.write_parquet(preprocessed_val_path)\n" ] }, { @@ -738,7 +795,7 @@ "from pathlib import Path\n", "import torch\n", "import torch.nn as nn\n", - "import torch.nn.functional as F" + "import torch.nn.functional as F\n" ] }, { @@ -799,7 +856,7 @@ " with open(args_fp, \"r\") as fp:\n", " model = cls(**json.load(fp))\n", " model.load_state_dict(torch.load(state_dict_fp, map_location=device))\n", - " return model" + " return model\n" ] }, { @@ -830,7 +887,7 @@ " dropout_p=0.3, \n", " num_classes=num_classes,\n", ")\n", - "print (model)" + "print (model)\n" ] }, { @@ -853,7 +910,7 @@ "metadata": {}, "outputs": [], "source": [ - "from ray.train.torch import get_device" + "from ray.train.torch import get_device\n" ] }, { @@ -872,7 +929,7 @@ " dtype=dtypes[key],\n", " device=get_device(),\n", " )\n", - " return tensor_batch" + " return tensor_batch\n" ] }, { @@ -884,15 +941,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:27:26,458\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_72_0\n", - "2025-06-23 14:27:26,469\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_72_0. Full logs are in /tmp/ray/session_2025-06-23_13-49-50_102769_2149/logs/ray-data\n", - "2025-06-23 14:27:26,470\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_72_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> LimitOperator[limit=3]\n" + "2025-08-22 00:29:25,511\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_85_0\n", + "2025-08-22 00:29:25,523\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_85_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-08-22 00:29:25,524\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_85_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> LimitOperator[limit=3]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f7d09fbef089477397fd9c9745974185", + "model_id": "c596b6e4878f41a9ac527bfb3925c95e", "version_major": 2, "version_minor": 0 }, @@ -903,17 +966,10 @@ "metadata": {}, "output_type": "display_data" }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-06-23 14:27:26,489\tINFO actor_pool_map_operator.py:633 -- Scaling up actor pool by 4 (reason=scaling to min size, running=0, restarting=0, pending=0)\n" - ] - }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ae6a71a193b94308be98fe3bb49e830e", + "model_id": "124f996b75cd452d89bb404100035d45", "version_major": 2, "version_minor": 0 }, @@ -927,7 +983,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "23fbb838b4f4413188a21f963216d9b3", + "model_id": "f76037bae4334040bf81176c7ddab96d", "version_major": 2, "version_minor": 0 }, @@ -941,7 +997,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "505daeea69ac49f3b0eb712b855f4dbd", + "model_id": "323cdef1b95f405da16d7478a2295072", "version_major": 2, "version_minor": 0 }, @@ -955,7 +1011,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9acc85ac1e8e44a3b672e6bd0bb38995", + "model_id": "2f9da19c9b7046b68d748780426d7886", "version_major": 2, "version_minor": 0 }, @@ -969,7 +1025,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "620184f23bf74c1c9af7475e1bd291e4", + "model_id": "3ff417b842ae472f8ef0e640443d3897", "version_major": 2, "version_minor": 0 }, @@ -983,7 +1039,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "939b699564104c0b8048c3fe78a235bc", + "model_id": "b3b34ef888164ae8a618828d9832fd8f", "version_major": 2, "version_minor": 0 }, @@ -998,37 +1054,30 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=18053, ip=10.0.90.122)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "2025-06-23 14:27:33,774\tINFO streaming_executor.py:227 -- ✔️ Dataset dataset_72_0 execution finished in 7.30 seconds\n", - "/tmp/ipykernel_18629/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", + "\u001b[36m(_MapWorker pid=4911, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "2025-08-22 00:29:36,437\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_85_0 execution finished in 10.91 seconds\n", + "/tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", " tensor_batch[key] = torch.as_tensor(\n" ] }, { "data": { "text/plain": [ - "{'embedding': tensor([[-0.1921, 0.1182, -0.1963, ..., 0.7892, -0.2841, -0.0829],\n", - " [-0.0389, -0.1284, -0.5749, ..., 0.4360, 0.0745, -0.1555],\n", - " [-0.1139, 0.1539, -0.1519, ..., 0.8438, 0.3064, -0.1918]]),\n", - " 'label': tensor([22, 11, 33])}" + "{'embedding': tensor([[ 0.4219, 0.3688, -0.1833, ..., 0.6288, 0.2298, -0.3989],\n", + " [ 0.0385, 0.3297, 0.2076, ..., 0.3434, -0.5492, 0.0362],\n", + " [ 0.1881, 0.1737, -0.3069, ..., 0.3336, 0.1783, -0.0299]]),\n", + " 'label': tensor([11, 34, 7])}" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m(autoscaler +35s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n" - ] } ], "source": [ "# Sample batch\n", "sample_batch = train_ds.take_batch(batch_size=3)\n", - "collate_fn(batch=sample_batch)" + "collate_fn(batch=sample_batch)\n" ] }, { @@ -1049,19 +1098,9 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m(autoscaler +57m1s)\u001b[0m [autoscaler] Downscaling node i-03a133888407b8cf8 (node IP: 10.0.103.152) due to node idle termination.\n", - "\u001b[36m(autoscaler +57m1s)\u001b[0m [autoscaler] Downscaling node i-06023e83fb012b7ae (node IP: 10.0.90.122) due to node idle termination.\n", - "\u001b[36m(autoscaler +57m6s)\u001b[0m [autoscaler] Cluster resized to {56 CPU, 6 GPU}.\n" - ] - } - ], + "outputs": [], "source": [ - "import shutil" + "import shutil\n" ] }, { @@ -1073,7 +1112,7 @@ "model_registry = \"/mnt/cluster_storage/mlflow/doggos\"\n", "if os.path.isdir(model_registry):\n", " shutil.rmtree(model_registry) # clean up\n", - "os.makedirs(model_registry, exist_ok=True)" + "os.makedirs(model_registry, exist_ok=True)\n" ] }, { @@ -1120,7 +1159,7 @@ " \"lr_patience\": 3,\n", " \"num_epochs\": 20,\n", " \"batch_size\": 256,\n", - "}" + "}\n" ] }, { @@ -1135,8 +1174,8 @@ " num_workers=num_workers,\n", " use_gpu=True,\n", " resources_per_worker={\"CPU\": 8, \"GPU\": 2},\n", - " accelerator_type=\"L4\",\n", - ")" + " accelerator_type=\"T4\",\n", + ")\n" ] }, { @@ -1148,7 +1187,7 @@ "import tempfile\n", "import mlflow\n", "import numpy as np\n", - "from ray.train.torch import TorchTrainer" + "from ray.train.torch import TorchTrainer\n" ] }, { @@ -1169,7 +1208,7 @@ " J.backward() # Backward pass.\n", " optimizer.step() # Update weights.\n", " loss += (J.detach().item() - loss) / (i + 1) # Cumulative loss\n", - " return loss" + " return loss\n" ] }, { @@ -1191,7 +1230,7 @@ " loss += (J - loss) / (i + 1)\n", " y_trues.extend(batch[\"label\"].cpu().numpy())\n", " y_preds.extend(torch.argmax(z, dim=1).cpu().numpy())\n", - " return loss, np.vstack(y_trues), np.vstack(y_preds)" + " return loss, np.vstack(y_trues), np.vstack(y_preds)\n" ] }, { @@ -1266,7 +1305,7 @@ "\n", " # End experiment tracking.\n", " if ray.train.get_context().get_world_rank() == 0:\n", - " mlflow.end_run()" + " mlflow.end_run()\n" ] }, { @@ -1286,7 +1325,7 @@ "source": [ "# Load preprocessed datasets.\n", "preprocessed_train_ds = ray.data.read_parquet(preprocessed_train_path)\n", - "preprocessed_val_ds = ray.data.read_parquet(preprocessed_val_path)" + "preprocessed_val_ds = ray.data.read_parquet(preprocessed_val_path)\n" ] }, { @@ -1303,70 +1342,7 @@ " train_loop_config=train_loop_config,\n", " scaling_config=scaling_config,\n", " datasets={\"train\": preprocessed_train_ds, \"val\": preprocessed_val_ds},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Train.\n", - "results = trainer.fit()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Ray Train" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- automatically handles **multi-node, multi-GPU** setup with no manual SSH setup or hostfile configs. \n", - "- define **per-worker fractional resource requirements**, for example, 2 CPUs and 0.5 GPU per worker.\n", - "- run on **heterogeneous machines** and scale flexibly, for example, CPU for preprocessing and GPU for training. \n", - "- built-in **fault tolerance** with retry of failed workers and continue from last checkpoint.\n", - "- supports Data Parallel, Model Parallel, Parameter Server, and even custom strategies.\n", - "- [Ray Compiled graphs](https://docs.ray.io/en/latest/ray-core/compiled-graph/ray-compiled-graph.html) allow you to even define different parallelism for jointly optimizing multiple models like Megatron, DeepSpeed, etc., or only allow for one global setting.\n", - "- You can also use Torch DDP, FSPD, DeepSpeed, etc., under the hood." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "🔥 [RayTurbo Train](https://docs.anyscale.com/rayturbo/rayturbo-train) offers even more improvement to the price-performance ratio, performance monitoring and more:\n", - "- **elastic training** to scale to a dynamic number of workers, continue training on fewer resources, even on spot instances.\n", - "- **purpose-built dashboard** designed to streamline the debugging of Ray Train workloads:\n", - " - Monitoring: View the status of training runs and train workers.\n", - " - Metrics: See insights on training throughput and training system operation time.\n", - " - Profiling: Investigate bottlenecks, hangs, or errors from individual training worker processes.\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can view experiment metrics and model artifacts in the model registry. You're using OSS MLflow so you can run the server by pointing to the model registry location:\n", - "\n", - "```bash\n", - "mlflow server -h 0.0.0.0 -p 8080 --backend-store-uri /mnt/cluster_storage/mlflow/doggos\n", - "```\n", - "\n", - "You can view the dashboard by going to the **Overview tab** > **Open Ports**. \n", - "\n", - "\n", - "\n", - "You also have the preceding Ray Dashboard and Train workload specific dashboards.\n", - "\n", - "\n" + ")\n" ] }, { @@ -1374,116 +1350,2556 @@ "execution_count": null, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(TrainController pid=125066)\u001b[0m [State Transition] INITIALIZING -> SCHEDULING.\n", + "\u001b[36m(TrainController pid=125066)\u001b[0m Attempting to start training worker group of size 4 with the following resources: [{'CPU': 8, 'GPU': 2, 'accelerator_type:T4': 0.001}] * 4\n", + "\u001b[36m(TrainController pid=125066)\u001b[0m Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(autoscaler +3m40s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 1 to 2).\n", + "\u001b[36m(autoscaler +3m40s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", + "\u001b[36m(autoscaler +3m45s)\u001b[0m [autoscaler] Cluster upscaled to {112 CPU, 8 GPU}.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(TrainController pid=125066)\u001b[0m Retrying the launch of the training worker group. The previous launch attempt encountered the following failure:\n", + "\u001b[36m(TrainController pid=125066)\u001b[0m The worker group startup timed out after 30.0 seconds waiting for 4 workers. Potential causes include: (1) temporary insufficient cluster resources while waiting for autoscaling (ignore this warning in this case), (2) infeasible resource request where the provided `ScalingConfig` cannot be satisfied), and (3) transient network issues. Set the RAY_TRAIN_WORKER_GROUP_START_TIMEOUT_S environment variable to increase the timeout.\n", + "\u001b[36m(TrainController pid=125066)\u001b[0m [State Transition] SCHEDULING -> RESCHEDULING.\n", + "\u001b[36m(TrainController pid=125066)\u001b[0m [State Transition] RESCHEDULING -> SCHEDULING.\n", + "\u001b[36m(TrainController pid=125066)\u001b[0m Attempting to start training worker group of size 4 with the following resources: [{'CPU': 8, 'GPU': 2, 'accelerator_type:T4': 0.001}] * 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(autoscaler +4m30s)\u001b[0m [autoscaler] Cluster upscaled to {160 CPU, 12 GPU}.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m Setting up process group for: env:// [rank=0, world_size=4]\n", + "\u001b[36m(RayTrainWorker pid=16056, ip=10.0.4.102)\u001b[0m Moving model to device: cuda:0\n", + "\u001b[36m(TrainController pid=125066)\u001b[0m Started training worker group of size 4: \n", + "\u001b[36m(TrainController pid=125066)\u001b[0m - (ip=10.0.34.27, pid=3319) world_rank=0, local_rank=0, node_rank=0\n", + "\u001b[36m(TrainController pid=125066)\u001b[0m - (ip=10.0.34.27, pid=3320) world_rank=1, local_rank=1, node_rank=0\n", + "\u001b[36m(TrainController pid=125066)\u001b[0m - (ip=10.0.4.102, pid=16056) world_rank=2, local_rank=0, node_rank=1\n", + "\u001b[36m(TrainController pid=125066)\u001b[0m - (ip=10.0.4.102, pid=16055) world_rank=3, local_rank=1, node_rank=1\n", + "\u001b[36m(TrainController pid=125066)\u001b[0m [State Transition] SCHEDULING -> RUNNING.\n", + "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m 2025/08/22 00:32:11 INFO mlflow.tracking.fluent: Experiment with name 'doggos' does not exist. Creating a new experiment.\n", + "\u001b[36m(RayTrainWorker pid=16056, ip=10.0.4.102)\u001b[0m Wrapping provided model in DistributedDataParallel.\n", + "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m /home/ray/anaconda3/lib/python3.12/site-packages/ray/data/iterator.py:445: RayDeprecationWarning: Passing a function to `iter_torch_batches(collate_fn)` is deprecated in Ray 2.47. Please switch to using a callable class that inherits from `ArrowBatchCollateFn`, `NumpyBatchCollateFn`, or `PandasBatchCollateFn`.\n", + "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m warnings.warn(\n" + ] + }, { "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2f9b807b07e24754b872e832186a7ecc", + "version_major": 2, + "version_minor": 0 + }, "text/plain": [ - "run_id c65d5aba186c4ee58bf8188493cd047c\n", - "experiment_id 477478897635232497\n", - "status FINISHED\n", - "artifact_uri file:///mnt/cluster_storage/mlflow/doggos/4774...\n", - "start_time 2025-06-23 14:23:03.775000+00:00\n", - "end_time 2025-06-23 14:23:21.440000+00:00\n", - "metrics.train_loss 0.388298\n", - "metrics.lr 0.001\n", - "metrics.val_loss 0.664968\n", - "params.batch_size 256\n", - "params.num_epochs 20\n", - "params.lr 0.001\n", - "params.hidden_dim 256\n", - "params.experiment_name doggos\n", - "params.dropout_p 0.3\n", - "params.embedding_dim 512\n", - "params.lr_patience 3\n", - "params.class_to_label {'doberman': 0, 'collie': 1, 'dingo': 2, 'pome...\n", - "params.lr_factor 0.8\n", - "params.model_registry /mnt/cluster_storage/mlflow/doggos\n", - "params.num_classes 36\n", - "tags.mlflow.source.name /home/ray/anaconda3/lib/python3.12/site-packag...\n", - "tags.mlflow.user ray\n", - "tags.mlflow.source.type LOCAL\n", - "tags.mlflow.runName abrasive-newt-588\n", - "Name: 0, dtype: object" + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" ] }, - "execution_count": null, "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Sorted runs\n", - "mlflow.set_tracking_uri(f\"file:{model_registry}\")\n", - "sorted_runs = mlflow.search_runs(\n", - " experiment_names=[experiment_name], \n", - " order_by=[\"metrics.val_loss ASC\"])\n", - "best_run = sorted_runs.iloc[0]\n", - "best_run" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Production Job" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can easily wrap the training workload as a production grade [Anyscale Job](https://docs.anyscale.com/platform/jobs/) ([API ref](https://docs.anyscale.com/reference/job-api/)).\n", - "\n", - "**Note**: \n", - "- This Job uses a `containerfile` to define dependencies, but you could easily use a pre-built image as well.\n", - "- You can specify the compute as a [compute config](https://docs.anyscale.com/configuration/compute-configuration/) or inline in a [job config](https://docs.anyscale.com/reference/job-api#job-cli) file.\n", - "- When you don't specify compute while launching from a workspace, this configuration defaults to the compute configuration of the workspace." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "```bash\n", - "# Production batch job.\n", - "anyscale job submit --name=train-doggos-model \\\n", - " --containerfile=\"/home/ray/default/containerfile\" \\\n", - " --compute-config=\"/home/ray/default/configs/aws.yaml\" \\\n", - " --working-dir=\"/home/ray/default\" \\\n", - " --exclude=\"\" \\\n", - " --max-retries=0 \\\n", - " -- python doggos/train.py\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Evaluation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This tutorial concludes by evaluating the trained model on the test dataset. Evaluation is essentially the same as the batch inference workload where you apply the model on batches of data and then calculate metrics using the predictions versus true labels. Ray Data is hyper optimized for throughput so preserving order isn't a priority. But for evaluation, this approach is crucial. Achieve this approach by preserving the entire row and adding the predicted label as another column to each row." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from urllib.parse import urlparse\n", - "from sklearn.metrics import multilabel_confusion_matrix" + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "477c2f56fd9e4c31974d33eec3c722a0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3b3d7e99c85a496ebad1cba791a6bcd1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2263907df3504600a2281cb5bb1feb81", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Registered dataset logger for dataset train_88_0\n", + "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Starting execution of Dataset train_88_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", + "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Execution plan of Dataset train_88_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> OutputSplitter[split(4, equal=True)]\n", + "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m ⚠️ Ray's object store is configured to use only 28.5% of available memory (195.9GB out of 687.2GB total). For optimal Ray Data performance, we recommend setting the object store to at least 50% of available memory. You can do this by setting the 'object_store_memory' parameter when calling ray.init() or by setting the RAY_DEFAULT_OBJECT_STORE_MEMORY_PROPORTION environment variable.\n", + "\u001b[36m(RayTrainWorker pid=16056, ip=10.0.4.102)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", + "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m Moving model to device: cuda:0\n", + "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m Wrapping provided model in DistributedDataParallel.\n", + "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m ✔️ Dataset train_88_0 execution finished in 2.84 seconds\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "dbfdddd23f0a43658486e78ef5db13ec", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4a0cd95efca346bcbd9c962648fb8d18", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9d0c2a55758742729e3e4c41aff6daf7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fbab3701238a4fc3a295d6116e1908f4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8cca0efc0fdf42309956588e2ebad8d9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c4a5eedd70c540c68156ae60bd821773", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "692887626f444deaa309ee332a270796", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "364d9c81b14d44b18703999214217018", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(RayTrainWorker pid=16055, ip=10.0.4.102)\u001b[0m /home/ray/anaconda3/lib/python3.12/site-packages/ray/data/iterator.py:445: RayDeprecationWarning: Passing a function to `iter_torch_batches(collate_fn)` is deprecated in Ray 2.47. Please switch to using a callable class that inherits from `ArrowBatchCollateFn`, `NumpyBatchCollateFn`, or `PandasBatchCollateFn`.\u001b[32m [repeated 3x across cluster]\u001b[0m\n", + "\u001b[36m(RayTrainWorker pid=16055, ip=10.0.4.102)\u001b[0m warnings.warn(\u001b[32m [repeated 3x across cluster]\u001b[0m\n", + "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Registered dataset logger for dataset train_88_1\u001b[32m [repeated 2x across cluster]\u001b[0m\n", + "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Starting execution of Dataset train_88_1. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\u001b[32m [repeated 2x across cluster]\u001b[0m\n", + "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Execution plan of Dataset train_88_1: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> OutputSplitter[split(4, equal=True)]\u001b[32m [repeated 2x across cluster]\u001b[0m\n", + "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m ⚠️ Ray's object store is configured to use only 28.5% of available memory (195.9GB out of 687.2GB total). For optimal Ray Data performance, we recommend setting the object store to at least 50% of available memory. You can do this by setting the 'object_store_memory' parameter when calling ray.init() or by setting the RAY_DEFAULT_OBJECT_STORE_MEMORY_PROPORTION environment variable.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c94637fd36f7408887c53978632c81d6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d9c770c92271481e824deabb97479d02", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e9394ea598c44b8eaed72c3a567e8f80", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "78f5d148c07843d6ba79aa4443fac4c2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f3c0cca34393418db7a85bf3b5da8de0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1cf9d449bed34bd58633604913c4b6da", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0f9207fa9fe74fb480d20ab6792412e8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d2d38403e0ea479c8745ba86e479e5b6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9245ec966b7142a39ccc3fb881ea1895", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "547f6b9e985a4fbf942e01dd9687245d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "03a5c5eeddbc4c0dbc4b2ae694a2cf23", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a12199d167f24ecab2b9dfe37adafee1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f4a3036c6d42495db3704671a5913e97", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(RayTrainWorker pid=3320, ip=10.0.34.27)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 3x across cluster]\u001b[0m\n", + "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m ✔️ Dataset val_89_2 execution finished in 0.14 seconds\u001b[32m [repeated 5x across cluster]\u001b[0m\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8daad09e465a4c95b282f6bff58488c4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1c61f75ca7ca467c8995ea64d5fbe622", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9686329d65264f15b583f6a26966ca46", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f06b9dd78b594b3e8abbede8e584d6f6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2dac4a6c6b69497cbfb1c2730ae4c84b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "310c75a42eef4481bf92376c8225732a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "19eca161119f4c38839e2b2ff2a4bb36", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "59f241a3d0364ebbbae4e7a5a94037a5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7c6147789fbc4007bbc309f7537782fa", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ae5a1436d1e148cdb8a4aac594e2ee5a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0e6c03e36b9b4373b9e61c5e73e3943a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5d154effa9e84089b1fac7bc66574802", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e8ce7b2eef094a078cb3b516e6d381d2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fc4f18d5ca0747f49cb7d070fa984f13", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8a02cba1818049cabd118a8cb4ad16fd", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2faf9980bf2f44d98a4ba61cd943ca2e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "17eb0e1b001444d5b9b089e3540143ed", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c614080c873a4658b47a79ea793ec211", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "59bed3f576ba412091b1431810674dc1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6c28eaa475dd461c9a4c3524ab635758", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4a396b6c1f8243b289ac96f5b8c5e354", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b7a446b66d4447c68b6cd69ae370aec9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fff93ed630644210b0e7fdb218c0fba4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "df70761adc79445ea206305bdbef50fa", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e9917e5b9629429eb6423cb23d1ac8ca", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "02466fb2e05e40f781ba101d2e1c5394", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cba46412a0e1470093b37dd44571c867", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d5c3e448d3254eb2bd6ac87c2ee60ff7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4a46866464514d189d040bff9c170371", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f77002b71fd641f79a0b1320045ce8bf", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1b58d2b8f06c4d688d1062b11ab9ad1c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m Registered dataset logger for dataset val_89_6\u001b[32m [repeated 11x across cluster]\u001b[0m\n", + "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m Starting execution of Dataset val_89_6. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\u001b[32m [repeated 11x across cluster]\u001b[0m\n", + "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m Execution plan of Dataset val_89_6: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> OutputSplitter[split(4, equal=True)]\u001b[32m [repeated 11x across cluster]\u001b[0m\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "072e8ee325c546f1ae10cb085631db3e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a0ed51b8f63d43c3a75bd861e63d4e23", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a0f4b65c405644829fd3e560487979ba", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1743714b36eb4c3a9e066854bc55e9a0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9c91384d8ab642eda0883c097edeea4f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "abfa908329d34ee192c35a5aec2b38b0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8c1c6421140e4ea99f06bd917697fadf", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6ce5252ebcb244fa9493dde86699516f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f8a34c741f594ef89fd99172e59a34c5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9cb4841bc8fa49448e31612719799b03", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "50f1a2962113459884deef4975308b07", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "59ec7aee99934c6bbc3c73a093e3b4fe", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a1bc4dd100464a549bc322eedf97512f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f9be5454b006445d8a75769c7264d770", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "82326f0ca01b4f5c92731f21a14e7bbb", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2ce3abb1f2ac4320a05ab07b9e530679", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "774c642be6a143e482201d2a60b4f725", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ea41e9ce5d4749269ab068ee3fc4c3f1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fdeb575b9f12491d85437877bf16b0f0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "05fdb6464b7542bd894d27043329da01", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6cbeb9d41ce746e6af9f74ea3db2dd58", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fe06543c2f074c4a8778f11d493b40f6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c35a3a74fb4a42eaba77b7c9bf57835f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "aec3501a42bf4e7488ef6d6fdf931b7c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "838a6b8cb0f8462bafd777ba17ca0145", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2735cdc36bb34dd68340fbbe8e4a007d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bb69d8033e3e44fcba6bd2c12b076344", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c8de0eb20bc34c7799e701bd8dfdf093", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m ✔️ Dataset val_89_9 execution finished in 0.12 seconds\u001b[32m [repeated 14x across cluster]\u001b[0m\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "94b5b81bb4d549ff852bb292d5eabe0a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1f2a660cfbe5483683fdaa6de6e83731", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ed36ce7aeeba40bea644c1ec216917cb", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7f549b806ec443fab242e9734063166f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5ca785575831425182c8606cef4c22ed", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "44e5707abbc545e3b52a327e338d42c8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8f2719cac015410883ffbb3e5bf95f35", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a19c11e6356345c2b1e255492d1c1077", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "21ba9584d021403dacca550bb11ee8a2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "63be175948ae4d41b17c7a4c2c3de9d9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3cb193c2285649bd8f987b98ae9b4705", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1d227a26a0514824a0ea7c2964688e6b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "543c26f1127447cf90765a009caf2b67", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2bc02cb3fe2846f983128f974a822fba", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f71f650ff2884e8fa4538cd059a7408f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6fa5b191ae2c45d2a0dac2f718cbed08", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1bc7f991b39444bcaed694e25020c82e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0c7df79581f94f4eb66f0fcb95260e1e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "adab6b6ea3ca44f6be524a1deb1ef639", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e757b1e650e04c6abf451db64a698bb5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c042c13431d34e08bfbd78e31d3d5de6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d09c9c885b60411fa6331b4cbb0725dc", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ce85530f583f4b40a46ac81f83abeb36", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "236a34616ba048bbb2695c5aaf9416f5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "24fca83b96a842318cf502782e1ed1b6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "057b1ec3b01a478f80a58686bd171735", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4f862da0dfbb45e5a60f80b9636e071b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "58a97c021f5a49e69202f99f38bad1c0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d7eb25150fb74086a63d7fe5ac160cb1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f17b808f250d41929e4870bb83fc477c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "feb8925122f74f11a0b4d406549a8998", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9cb0e78fa32d417bbd237ea1b790ad86", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "aae13b8832ff40c4a768e08fc1489292", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8663407dc9a745408be1707fdcb3de64", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1f3fe18d3a114499bb0e8aedc9087516", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "dde2396c4c4743828a5b45b5ae8dd077", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9c28151437dc45b19eea8a774a565d54", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d4da9d395b3b40b7beaf40f96b020e4d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "72f25e200d844c3c8fb2e46bceba5e0e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "745b8abd99a148cfb35a207634053f68", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Registered dataset logger for dataset train_88_15\u001b[32m [repeated 17x across cluster]\u001b[0m\n", + "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Starting execution of Dataset train_88_15. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\u001b[32m [repeated 17x across cluster]\u001b[0m\n", + "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Execution plan of Dataset train_88_15: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> OutputSplitter[split(4, equal=True)]\u001b[32m [repeated 17x across cluster]\u001b[0m\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ea923fad3a6b427e95fc73aca1de7d0b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ee86e41631394e14a447e76d51b0b55b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b1b038c8a53f437b962ea4b2728f7ee2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ab1e205e2d2a4e40b0399497be2d3eaf", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9b0bc3fcded44cbcada0d64d6c03dd9c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c90593c23a5342cb9fb45b59a6d23954", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9c01556845fd474abd85d9bf39424908", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c35afae5f34c43b88d1461be949ab7e2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "63741d7ac05b462893c79533aab90adb", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e9a9894f3518447c8d977460cde7fd2b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a198c557b1e84c15975bafcc67ca2501", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6cfd359d3fc24709933a740eb38fe408", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2dd17bebcef148bfb4267274ea6538bf", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6381b5c191cd4a608067dad981936f29", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e54bd73a22084efaa21246f9ab88be18", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "efe59371cb0b40489158ba929bfcadd9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9d0fb221f5fb4f1c865fb2d383f3f66f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ee62c3c15ec5423baf0613424d66b960", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3d1adb75395c4367ac14d510c6f9e891", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4cd34884675440ffafd0334783920856", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "472913bf0ccb485b90de55a05501629f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "81d1e24ae643455d92707226660390fe", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a7f776485f714d66922fffe6f75d4aeb", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c753d595da314ff687d8357a56484ac2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ffd47b8311b44a6d9ef72e99ebda6b3e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "99cf3b9ba6c042a39dafe5cef3e49349", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "644e2243a1724d40bc6ad9ac5cd349b0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "933dc5a56bd64ee69ba1bd23c806c434", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "59a2b83763e04146aeec9edacaf72ed8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "801a70882a9546b1b9c2e7998178d6cf", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "526adc86f7674fae8b0e44cde52fd920", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "18414e84c96048a196a579fe5e5cdf79", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m ✔️ Dataset val_89_18 execution finished in 0.12 seconds\u001b[32m [repeated 18x across cluster]\u001b[0m\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1e1a34e9e96c4109b1ff5b38e23eef3d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c377ec61738f42dfa1ee8638ce0fb4ed", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "086c0e1c3c8443659adcdb4d5ec55d85", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f7fe6c34adb44a4780cb4c7f320ab3f4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(TrainController pid=125066)\u001b[0m [State Transition] RUNNING -> FINISHED.\n" + ] + } + ], + "source": [ + "# Train.\n", + "results = trainer.fit()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Ray Train" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- automatically handles **multi-node, multi-GPU** setup with no manual SSH setup or hostfile configs. \n", + "- define **per-worker fractional resource requirements**, for example, 2 CPUs and 0.5 GPU per worker.\n", + "- run on **heterogeneous machines** and scale flexibly, for example, CPU for preprocessing and GPU for training. \n", + "- built-in **fault tolerance** with retry of failed workers and continue from last checkpoint.\n", + "- supports Data Parallel, Model Parallel, Parameter Server, and even custom strategies.\n", + "- [Ray Compiled graphs](https://docs.ray.io/en/latest/ray-core/compiled-graph/ray-compiled-graph.html) allow you to even define different parallelism for jointly optimizing multiple models like Megatron, DeepSpeed, etc., or only allow for one global setting.\n", + "- You can also use Torch DDP, FSPD, DeepSpeed, etc., under the hood." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "🔥 [RayTurbo Train](https://docs.anyscale.com/rayturbo/rayturbo-train) offers even more improvement to the price-performance ratio, performance monitoring and more:\n", + "- **elastic training** to scale to a dynamic number of workers, continue training on fewer resources, even on spot instances.\n", + "- **purpose-built dashboard** designed to streamline the debugging of Ray Train workloads:\n", + " - Monitoring: View the status of training runs and train workers.\n", + " - Metrics: See insights on training throughput and training system operation time.\n", + " - Profiling: Investigate bottlenecks, hangs, or errors from individual training worker processes.\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can view experiment metrics and model artifacts in the model registry. You're using OSS MLflow so you can run the server by pointing to the model registry location:\n", + "\n", + "```bash\n", + "mlflow server -h 0.0.0.0 -p 8080 --backend-store-uri /mnt/cluster_storage/mlflow/doggos\n", + "```\n", + "\n", + "You can view the dashboard by going to the **Overview tab** > **Open Ports**. \n", + "\n", + "\n", + "\n", + "You also have the preceding Ray Dashboard and Train workload specific dashboards.\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "run_id fcb9ef8c96f844f08bcd0185601f3dbd\n", + "experiment_id 858816514880031760\n", + "status FINISHED\n", + "artifact_uri file:///mnt/cluster_storage/mlflow/doggos/8588...\n", + "start_time 2025-08-22 00:32:11.522000+00:00\n", + "end_time 2025-08-22 00:32:32.895000+00:00\n", + "metrics.train_loss 0.35504\n", + "metrics.val_loss 0.593301\n", + "metrics.lr 0.001\n", + "params.lr_patience 3\n", + "params.dropout_p 0.3\n", + "params.num_epochs 20\n", + "params.lr 0.001\n", + "params.num_classes 36\n", + "params.hidden_dim 256\n", + "params.experiment_name doggos\n", + "params.batch_size 256\n", + "params.model_registry /mnt/cluster_storage/mlflow/doggos\n", + "params.class_to_label {'border_collie': 0, 'pomeranian': 1, 'basset'...\n", + "params.lr_factor 0.8\n", + "params.embedding_dim 512\n", + "tags.mlflow.user ray\n", + "tags.mlflow.source.type LOCAL\n", + "tags.mlflow.runName enthused-donkey-931\n", + "tags.mlflow.source.name /home/ray/anaconda3/lib/python3.12/site-packag...\n", + "Name: 0, dtype: object" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Sorted runs\n", + "mlflow.set_tracking_uri(f\"file:{model_registry}\")\n", + "sorted_runs = mlflow.search_runs(\n", + " experiment_names=[experiment_name], \n", + " order_by=[\"metrics.val_loss ASC\"])\n", + "best_run = sorted_runs.iloc[0]\n", + "best_run\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Production Job" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can easily wrap the training workload as a production grade [Anyscale Job](https://docs.anyscale.com/platform/jobs/) ([API ref](https://docs.anyscale.com/reference/job-api/)).\n", + "\n", + "**Note**: \n", + "- This Job uses a `containerfile` to define dependencies, but you could easily use a pre-built image as well.\n", + "- You can specify the compute as a [compute config](https://docs.anyscale.com/configuration/compute-configuration/) or inline in a [job config](https://docs.anyscale.com/reference/job-api#job-cli) file.\n", + "- When you don't specify compute while launching from a workspace, this configuration defaults to the compute configuration of the workspace." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Output\n", + "(anyscale +0.9s) Submitting job with config JobConfig(name='train-image-model', image_uri='anyscale/ray:2.48.0-slim-py312-cu128', compute_config=None, env_vars=None, py_modules=['/home/ray/default/doggos'], py_executable=None, cloud=None, project=None, ray_version=None, job_queue_config=None).\n", + "(anyscale +2.8s) Uploading local dir '/home/ray/default' to cloud storage.\n", + "(anyscale +4.3s) Uploading local dir '/home/ray/default/doggos' to cloud storage.\n", + "(anyscale +5.4s) Job 'train-image-model' submitted, ID: 'prodjob_ac1sxbql2i2vah66k2462bhxie'.\n", + "(anyscale +5.4s) View the job in the UI: https://console.anyscale.com/jobs/prodjob_ac1sxbql2i2vah66k2462bhxie\n", + "(anyscale +5.4s) Use `--wait` to wait for the job to run and stream logs.\n" + ] + } + ], + "source": [ + "%%bash\n", + "# Production model training job\n", + "anyscale job submit -f /home/ray/default/configs/train_model.yaml\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This tutorial concludes by evaluating the trained model on the test dataset. Evaluation is essentially the same as the batch inference workload where you apply the model on batches of data and then calculate metrics using the predictions versus true labels. Ray Data is hyper optimized for throughput so preserving order isn't a priority. But for evaluation, this approach is crucial. Achieve this approach by preserving the entire row and adding the predicted label as another column to each row." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from urllib.parse import urlparse\n", + "from sklearn.metrics import multilabel_confusion_matrix\n" ] }, { @@ -1524,7 +3940,7 @@ " args_fp=os.path.join(artifacts_dir, \"args.json\"), \n", " state_dict_fp=os.path.join(artifacts_dir, \"model.pt\"),\n", " )\n", - " return cls(preprocessor=preprocessor, model=model)" + " return cls(preprocessor=preprocessor, model=model)\n" ] }, { @@ -1538,7 +3954,7 @@ "predictor = TorchPredictor.from_artifacts_dir(artifacts_dir=artifacts_dir)\n", "test_ds = ray.data.read_images(\"s3://doggos-dataset/test\", include_paths=True)\n", "test_ds = test_ds.map(add_class)\n", - "test_ds = predictor.preprocessor.transform(ds=test_ds)" + "test_ds = predictor.preprocessor.transform(ds=test_ds)\n" ] }, { @@ -1550,15 +3966,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:25:17,471\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_56_0\n", - "2025-06-23 14:25:17,483\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_56_0. Full logs are in /tmp/ray/session_2025-06-23_13-49-50_102769_2149/logs/ray-data\n", - "2025-06-23 14:25:17,484\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_56_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> TaskPoolMapOperator[MapBatches(TorchPredictor)] -> LimitOperator[limit=1]\n" + "2025-08-22 00:34:12,802\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_96_0\n", + "2025-08-22 00:34:12,814\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_96_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", + "2025-08-22 00:34:12,815\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_96_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> TaskPoolMapOperator[MapBatches(TorchPredictor)] -> LimitOperator[limit=1]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9b9a801dfa75419b9f307a00a3397677", + "model_id": "50d1c62a744146a398da57614e787e8c", "version_major": 2, "version_minor": 0 }, @@ -1569,17 +3985,10 @@ "metadata": {}, "output_type": "display_data" }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-06-23 14:25:17,504\tINFO actor_pool_map_operator.py:633 -- Scaling up actor pool by 4 (reason=scaling to min size, running=0, restarting=0, pending=0)\n" - ] - }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9354ed2fc7644cb7bacb97aa620d76fa", + "model_id": "b2d9b9453d0f40928a76a188f7a30eb4", "version_major": 2, "version_minor": 0 }, @@ -1593,7 +4002,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1980c5b233994a82b79c1b5853333de4", + "model_id": "85b1f60100d8451995792b7da3f8ac83", "version_major": 2, "version_minor": 0 }, @@ -1607,7 +4016,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a78cd2dd17df4f72b3aa28f40e36a04b", + "model_id": "d6f46bc81e674ba38e39f807dae62551", "version_major": 2, "version_minor": 0 }, @@ -1621,7 +4030,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9a240953f8a6401cb9e060439e3c7432", + "model_id": "9133c6ae847f4d52955482803d33c67f", "version_major": 2, "version_minor": 0 }, @@ -1635,7 +4044,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c1ead75a9a74448fb96776643b93b769", + "model_id": "9ecc392709b442f4b123fcad7fc7e60b", "version_major": 2, "version_minor": 0 }, @@ -1649,7 +4058,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "17d189a8c3534c7fbea57d6b4680337c", + "model_id": "5df8844d4afa424f8e750c4b362e3667", "version_major": 2, "version_minor": 0 }, @@ -1663,7 +4072,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "969c3c3cc23946238bae3b1682aa2ade", + "model_id": "d2838a72543d43f4a41520dc98f9dd57", "version_major": 2, "version_minor": 0 }, @@ -1678,148 +4087,164 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=41895, ip=10.0.102.235)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=7131, ip=10.0.90.122)\u001b[0m /tmp/ipykernel_14938/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", - "\u001b[36m(_MapWorker pid=6304, ip=10.0.90.122)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster]\u001b[0m\n", - "2025-06-23 14:25:31,572\tINFO streaming_executor.py:227 -- ✔️ Dataset dataset_56_0 execution finished in 14.08 seconds\n" + "\u001b[36m(_MapWorker pid=18066, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(autoscaler +8m20s)\u001b[0m [autoscaler] [1xT4:8CPU-32GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", + "\u001b[36m(autoscaler +8m25s)\u001b[0m [autoscaler] [1xT4:8CPU-32GB|g4dn.2xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", + "\u001b[36m(autoscaler +8m25s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 1 to 2).\n", + "\u001b[36m(autoscaler +8m30s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(MapBatches(TorchPredictor) pid=19185, ip=10.0.4.102)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", + "\u001b[36m(_MapWorker pid=18062, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster]\u001b[0m\n", + "2025-08-22 00:34:50,050\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_96_0 execution finished in 37.23 seconds\n" ] }, { "data": { "text/plain": [ - "[{'path': 'doggos-dataset/test/basset/basset_10288.jpg',\n", + "[{'path': 'doggos-dataset/test/basset/basset_10005.jpg',\n", " 'class': 'basset',\n", - " 'label': 26,\n", - " 'embedding': array([-1.04914151e-01, -2.44789988e-01, -9.95982289e-02, 1.35369569e-01,\n", - " -5.52587211e-02, -5.80722839e-02, 1.91796571e-01, 1.56359702e-01,\n", - " -6.07913733e-01, 2.08769619e-01, -3.80898006e-02, -1.11314066e-01,\n", - " -1.96144834e-01, -6.14988208e-02, 5.18053114e-01, 2.08482340e-01,\n", - " 1.18680000e+00, 2.00228021e-01, -2.38505289e-01, 7.44116083e-02,\n", - " -1.17921010e-01, 1.65986642e-02, 4.06986564e-01, 1.73043087e-02,\n", - " -7.19358325e-02, -2.49894068e-01, 5.69958836e-02, -2.07780451e-02,\n", - " -2.98084527e-01, -1.49073690e-01, 2.44870782e-02, 4.86774921e-01,\n", - " 3.78374428e-01, -2.37518042e-01, 1.26714706e-01, 1.10405624e-01,\n", - " 1.23483673e-01, -2.53296018e-01, -1.41814440e-01, 1.88360083e+00,\n", - " -4.67942834e-01, -1.71202213e-01, 2.93785512e-01, 9.53243077e-02,\n", - " -1.08036891e-01, -1.05388820e+00, 2.12952226e-01, 3.43122423e-01,\n", - " -9.08568352e-02, -6.02110699e-02, 1.57682300e-02, 1.13998428e-01,\n", - " -9.61582065e-02, 1.91040933e-01, 3.62998173e-02, -1.67396963e-02,\n", - " 4.08946127e-01, 4.58516389e-01, -4.09091681e-01, -3.85877311e-01,\n", - " 9.77702141e-01, -1.69139802e-02, 1.93179488e-01, 1.36374593e-01,\n", - " -2.66537070e-01, -6.00859582e-01, -5.44146113e-02, 1.52056739e-01,\n", - " -2.88875699e-01, 2.30367318e-01, 6.66391551e-02, -3.48750651e-01,\n", - " 1.32896990e-01, 2.43517846e-01, -3.36779654e-03, 2.86127269e-01,\n", - " -3.56745601e-01, -1.14945844e-01, 1.51565939e-01, 4.90366817e-02,\n", - " 7.63746500e-02, -2.27382034e-02, 2.54388422e-01, -5.34341276e-01,\n", - " 3.07917655e-01, 4.43625525e-02, 3.23391706e-02, -3.16016555e-01,\n", - " 3.49402249e-01, 1.40896916e-01, -3.93401146e-01, -6.98464215e-01,\n", - " -7.05318165e+00, -9.64104384e-02, -1.29345521e-01, 1.01153195e-01,\n", - " 1.66721642e-03, 2.46858150e-01, -6.62657797e-01, 8.84700537e-01,\n", - " -2.41105676e-01, -1.67729586e-01, -2.76175410e-01, -1.06329188e-01,\n", - " 4.68529433e-01, -2.96109051e-01, 5.00090122e-01, -1.51693597e-02,\n", - " 1.84735969e-01, -4.76171166e-01, 2.78874516e-01, -7.43267417e-01,\n", - " 3.29548061e-01, 9.67882574e-03, -2.46126920e-01, -2.13637024e-01,\n", - " -5.42725086e-01, 3.51180196e-01, -2.11806729e-01, 3.27730656e-01,\n", - " 1.95189789e-01, 1.26086920e-01, 6.48027122e-01, 2.56954640e-01,\n", - " 4.22701418e-01, -2.30529577e-01, -1.10486835e-01, -1.01444468e-01,\n", - " 7.89555907e-03, -2.47240350e-01, 1.73558876e-01, 3.03944647e-01,\n", - " -5.77825531e-02, 9.45507646e-01, -4.95145559e-01, 2.86680222e-01,\n", - " -7.24357292e-02, -8.29979897e-01, 4.94338155e-01, 2.54262447e-01,\n", - " 2.29299828e-01, -2.25470066e-02, 5.62191963e-01, 3.00550222e-01,\n", - " -2.83117369e-02, 3.84202749e-01, 2.89719075e-01, 3.54923964e-01,\n", - " 2.66314894e-01, -3.58392656e-01, -3.72334182e-01, 5.86691260e-01,\n", - " -1.24578431e-01, -4.04101044e-01, -5.07451952e-01, 5.48313916e-01,\n", - " -3.14691275e-01, -1.80745274e-01, 2.89481759e-01, 5.75179756e-02,\n", - " -1.80967286e-01, 9.15101022e-02, 4.65520680e-01, 7.72555918e-02,\n", - " 2.23801851e-01, -1.68022275e-01, 1.34750500e-01, 2.97952116e-01,\n", - " 2.26987794e-01, 3.05612266e-01, 8.25502351e-02, 1.27266854e-01,\n", - " 4.45461750e-01, 4.75219965e-01, 2.56610662e-02, -4.94095474e-01,\n", - " 6.80846751e-01, 6.35496229e-02, 2.54889160e-01, -1.44209296e-01,\n", - " -5.48627734e-01, 3.29704136e-02, 4.15674299e-02, -2.43748799e-02,\n", - " -2.19443023e-01, -1.42820716e-01, -2.50694096e-01, -2.07656205e-01,\n", - " -1.79199561e-01, 3.50940913e-01, 6.33473039e-01, 3.80550534e-01,\n", - " -2.89176375e-01, 2.02112049e-01, -4.48559523e-01, 2.72922575e-01,\n", - " 2.24376589e-01, -2.83806473e-01, -4.37651068e-01, -9.45880890e-01,\n", - " 1.22266248e-01, 4.01376486e-02, 3.55452418e-01, 2.14725018e-01,\n", - " -3.82868618e-01, -3.58605623e-01, 1.33403972e-01, 3.17366868e-02,\n", - " 8.55787545e-02, 8.59863982e-02, 9.54705626e-02, -3.47019404e-01,\n", - " -7.17684031e-02, 2.91243881e-01, 2.65088528e-01, -9.42258835e-02,\n", - " -1.77515849e-01, 2.28757620e-01, 9.07460928e-01, -1.03129521e-01,\n", - " 7.33332276e-01, 2.64944017e-01, -1.47793442e-01, 3.05287898e-01,\n", - " -2.62915194e-01, 1.97677180e-01, 6.06525466e-02, -1.16444737e-01,\n", - " 7.31713697e-03, 1.67819709e-01, 9.79746133e-02, 1.47581011e-01,\n", - " -4.00336832e-01, 4.21648145e-01, -8.30136314e-02, -6.39808178e-01,\n", - " -1.41640380e-01, 4.65202779e-02, 7.18399584e-02, -4.38913584e-01,\n", - " 2.07775518e-01, 4.70566414e-02, -8.90242606e-02, -4.53150421e-01,\n", - " -2.14878619e-01, 2.44945884e-01, 3.16962540e-01, -3.41699839e-01,\n", - " -1.91379115e-01, -2.09521651e-02, 2.30608553e-01, 3.33673239e-01,\n", - " 2.77272910e-01, -2.96298712e-01, 1.22105137e-01, -2.16433048e-01,\n", - " 5.48319101e-01, 2.72968113e-01, 1.73093528e-01, 1.80758208e-01,\n", - " -3.40644240e-01, 2.62541264e-01, 1.24807566e-01, -7.05128908e-01,\n", - " -1.10303462e-02, -1.81341395e-01, -1.78187087e-01, 1.32017612e-01,\n", - " -4.31975611e-02, 3.50797176e-03, 1.59508839e-01, 9.21480432e-02,\n", - " 4.54917192e-01, 2.72805333e-01, -5.77595115e-01, -2.87324011e-01,\n", - " 1.66138291e-01, 8.66501480e-02, 9.02174413e-03, -3.78495932e-01,\n", - " -3.07204783e-01, 1.98499486e-02, -2.17410654e-01, -3.29564735e-02,\n", - " -9.36664641e-03, 1.02078244e-01, -5.64144492e-01, 2.59325683e-01,\n", - " -1.29754335e-01, 1.67371452e-01, 3.65311772e-01, 1.91542730e-02,\n", - " -1.80281848e-01, -1.50442168e-01, 3.04976612e-01, 3.71464863e-02,\n", - " 1.42819434e-02, 1.84083462e-01, 2.46860430e-01, 1.05640769e-01,\n", - " 4.84380722e-02, -3.53347808e-02, -4.98287007e-02, 2.02643886e-01,\n", - " -1.73173457e-01, -3.63763243e-01, -2.20462531e-01, 3.16181600e-01,\n", - " 6.26130402e-02, 7.24823922e-02, -1.47105128e-01, 3.08875024e-01,\n", - " 9.42751825e-01, 1.98151171e-02, -1.21707544e-02, -2.04986826e-01,\n", - " 2.55928785e-01, -9.34749842e-02, -1.57368124e-01, -9.39193606e-01,\n", - " 7.99043655e-01, 7.17637539e-01, -3.75674933e-01, 5.69818616e-01,\n", - " -1.33306235e-02, 5.30459285e-01, -5.34143746e-01, 2.46586412e-01,\n", - " -1.07142270e-01, 3.60272974e-02, -2.97878295e-01, -4.83343840e-01,\n", - " 6.04178667e-01, -5.00948548e-01, 3.49492311e-01, 2.63357386e-02,\n", - " 9.19313729e-02, 4.02335197e-01, 1.58837855e-01, -6.79962993e-01,\n", - " -2.58434951e-01, -4.40313041e-01, 3.03083509e-01, 3.24987084e-01,\n", - " 5.39690614e-01, 5.20520747e-01, 4.50525880e-01, 4.25642878e-01,\n", - " -3.66918445e-01, 3.89405370e-01, -1.27459884e+00, 1.07019678e-01,\n", - " -2.60990173e-01, -1.43924609e-01, 7.54836053e-02, 9.26972032e-01,\n", - " 3.27434987e-01, -1.17758155e+00, 1.98659331e-01, -2.22037435e-02,\n", - " 7.09707081e-01, 2.66087234e-01, 1.21972881e-01, 3.83028030e-01,\n", - " -7.28927612e-01, 2.53533423e-01, -4.85364050e-01, -2.49552578e-01,\n", - " -6.45122454e-02, -7.29703009e-01, 4.32397306e-01, 2.20177278e-01,\n", - " 2.00846434e-01, -9.86097157e-02, -1.90976754e-01, 2.79123753e-01,\n", - " 1.66312551e+00, 4.78211313e-01, -2.51018330e-02, 2.72021592e-01,\n", - " 7.38141775e-01, -1.70819223e-01, 8.71482790e-02, 5.43940544e-01,\n", - " 1.69077605e-01, -3.87216598e-01, -2.42075190e-01, 2.69218534e-01,\n", - " 3.44690025e-01, -8.90391588e-01, -7.69253790e-01, -3.58836114e-01,\n", - " 5.44936597e-01, -5.26414633e-01, -7.02109337e-02, -9.80197862e-02,\n", - " 1.44381337e-02, 2.74508834e-01, -2.26176381e-01, -4.58218932e-01,\n", - " -1.67408079e-01, 9.71819162e-02, -4.52373654e-01, 2.12075204e-01,\n", - " 3.00378114e-01, -4.85782117e-01, -8.94452184e-02, -3.76136094e-01,\n", - " 6.35548115e-01, -5.96615791e-01, 4.56892580e-01, 8.58041495e-02,\n", - " -4.65728045e-01, 2.77835429e-02, 3.81691009e-02, -2.30244100e-01,\n", - " 2.88146824e-01, 4.18678313e-01, 2.95979947e-01, -3.73036146e-01,\n", - " 2.28022650e-01, 3.33540946e-01, -1.05593085e-01, -3.15681905e-01,\n", - " -1.58446252e-01, -1.87164396e-01, -2.52391577e-01, -2.95362055e-01,\n", - " 8.43314469e-01, 1.14071526e-01, -2.23938376e-02, 1.09957650e-01,\n", - " -3.88728201e-01, 1.39827147e-01, 2.20899284e-03, -1.90839812e-01,\n", - " -9.09137726e-01, 1.57145649e-01, -1.39061660e-02, -2.81439349e-02,\n", - " 1.31379187e-01, 1.93342119e-02, -3.97078514e-01, 4.37840447e-02,\n", - " 5.70612431e-01, -3.71424943e-01, 1.27987966e-01, -1.53837383e-01,\n", - " -1.62056446e-01, -2.61603892e-02, -9.74950790e-01, -2.85338938e-01,\n", - " 1.48266554e-06, -5.19999146e-01, -1.39436916e-01, -1.61675125e-01,\n", - " 2.82035142e-01, 5.65708935e-01, 1.78672537e-01, 2.84627140e-01,\n", - " -1.29202381e-02, -5.35536408e-01, 6.67068288e-02, 1.26034901e-01,\n", - " 4.77381468e-01, 4.13616210e-01, -8.82375419e-01, 2.16037527e-01,\n", - " -7.70060718e-03, -1.17288813e-01, 3.86771172e-01, 3.40055674e-01,\n", - " -3.02813143e-01, -2.90828168e-01, -4.41879481e-01, -3.02490562e-01,\n", - " 1.14623025e-01, 5.78140691e-02, -5.26804924e-01, -1.41756445e-01,\n", - " 2.43902951e-03, 6.49944693e-02, -2.29362592e-01, -5.48198938e-01,\n", - " -7.99068272e-01, -3.52486148e-02, 4.28467467e-02, -5.25768399e-01,\n", - " 1.63442969e-01, -2.11263120e-01, -6.78404570e-02, -2.00107336e-01,\n", - " 4.71601546e-01, -4.66121018e-01, 2.91595191e-01, -5.46462014e-02,\n", - " -5.07597744e-01, 6.30303860e-01, -7.32594371e-01, 1.00498527e-01,\n", - " -7.07668364e-01, -8.52217302e-02, -5.60935438e-02, -1.76870823e-03,\n", - " 3.38252485e-01, -1.68113291e-01, -1.64995581e-01, 1.30709872e-01,\n", - " -9.02270138e-01, 1.71258092e-01, -5.64923435e-02, -2.03939527e-01],\n", + " 'label': 2,\n", + " 'embedding': array([ 8.86104554e-02, -5.89382686e-02, 1.15464866e-01, 2.15815112e-01,\n", + " -3.43266308e-01, -3.35150540e-01, 1.48883224e-01, -1.02369718e-01,\n", + " -1.69915810e-01, 4.34856862e-03, 2.41593361e-01, 1.79200619e-01,\n", + " 4.34402555e-01, 4.59785998e-01, 1.59284808e-02, 4.16959971e-01,\n", + " 5.20779848e-01, 1.86366066e-01, -3.43496174e-01, -4.00813907e-01,\n", + " -1.15213782e-01, -3.04853529e-01, 1.77998394e-01, 1.82090014e-01,\n", + " -3.56360346e-01, -2.30711952e-01, 1.69025257e-01, 3.78455579e-01,\n", + " 8.37044120e-02, -4.81875241e-02, 3.17967087e-01, -1.40099749e-01,\n", + " -2.15949178e-01, -4.72761095e-01, -3.01893711e-01, 7.59940967e-02,\n", + " -2.64865339e-01, 5.89084566e-01, -3.75831634e-01, 3.11807573e-01,\n", + " -3.82964134e-01, -1.86417520e-01, 1.07007243e-01, 4.81416702e-01,\n", + " -3.70819569e-01, 9.12090182e-01, 3.13470632e-01, -3.69494259e-02,\n", + " -2.21142501e-01, 3.32214013e-02, 8.51379186e-02, 3.64337176e-01,\n", + " -3.90754700e-01, 4.39904258e-02, 5.39945886e-02, -5.02359867e-01,\n", + " -4.76054996e-02, 3.87604594e-01, -3.71239424e-01, -8.79095644e-02,\n", + " 5.62141061e-01, 1.96927994e-01, 3.54419112e-01, -6.80974126e-03,\n", + " 2.86425143e-01, -3.24660867e-01, -4.56204057e-01, 6.41017914e-01,\n", + " -1.67037442e-01, -2.29641497e-01, 4.71122622e-01, 5.03865302e-01,\n", + " -9.06585157e-03, -1.23926058e-01, -3.32888782e-01, 1.59683321e-02,\n", + " -5.00816345e-01, -3.53796408e-02, -1.60535276e-01, -2.88702995e-01,\n", + " 5.51706925e-02, -3.47863048e-01, -3.01085338e-02, -6.00592375e-01,\n", + " 2.04530790e-01, -1.17298350e-01, 8.88321698e-01, -3.18641007e-01,\n", + " 2.02193573e-01, -1.50856599e-01, -2.96603352e-01, -5.45758486e-01,\n", + " -7.55531311e+00, -3.07271361e-01, -7.33374238e-01, 2.76708573e-01,\n", + " -3.76666151e-02, -4.25825119e-01, -5.56892097e-01, 7.15545475e-01,\n", + " 1.02834240e-01, -1.19939610e-01, 1.94998607e-01, -2.46950224e-01,\n", + " 2.61530429e-01, -4.19263542e-01, 1.31001920e-01, -2.49398082e-01,\n", + " -3.26750994e-01, -3.92482489e-01, 3.30219358e-01, -5.78646958e-01,\n", + " 1.53134540e-01, -3.10127169e-01, -3.67199332e-01, -7.94161111e-02,\n", + " -2.93402106e-01, 2.62198240e-01, 2.91103810e-01, 1.32868871e-01,\n", + " -5.78317158e-02, -4.26885992e-01, 2.99195677e-01, 4.23972368e-01,\n", + " 2.30407149e-01, -2.98300147e-01, -1.55886114e-01, -1.24661736e-01,\n", + " -1.17139973e-01, -4.21351314e-01, -1.45010501e-02, -3.06388348e-01,\n", + " 2.89572328e-01, 9.73405361e-01, -5.52814901e-01, 2.36222595e-01,\n", + " -2.13898420e-01, -1.00043082e+00, -3.57041806e-01, -1.50843680e-01,\n", + " 4.69288528e-02, 2.08646134e-01, -2.70194232e-01, 2.63797104e-01,\n", + " 1.31332219e-01, 2.82329589e-01, 2.69341841e-02, -1.21627375e-01,\n", + " 3.80910456e-01, 2.65330970e-01, -3.01948935e-01, -6.39178753e-02,\n", + " -3.13922286e-01, -4.14075851e-01, -2.19056532e-01, 2.22424790e-01,\n", + " 8.13730657e-02, -3.03519934e-01, 9.32400897e-02, -3.76873404e-01,\n", + " 8.34950879e-02, 1.01878762e-01, 2.87054926e-01, 2.09415853e-02,\n", + " -1.22204229e-01, 1.64302550e-02, -2.41174936e-01, 1.78844824e-01,\n", + " 9.15416703e-03, 1.66462481e-01, -1.45732313e-01, -5.85511327e-04,\n", + " 2.25536823e-01, 3.30472469e-01, -1.25101686e-01, 1.13093004e-01,\n", + " 1.52094781e-01, 4.37459409e-01, 3.22061956e-01, 1.37893021e-01,\n", + " -2.53650725e-01, -1.94988877e-01, -2.72130489e-01, -2.57504702e-01,\n", + " 1.92389667e-01, -2.07393348e-01, 1.73574477e-01, 2.59756446e-02,\n", + " 2.20320046e-01, 6.48344308e-02, 3.96853566e-01, 1.11773282e-01,\n", + " -4.38930988e-01, -5.10937572e-02, 5.92644155e-01, 6.10140711e-03,\n", + " -3.97206768e-02, 7.65584633e-02, -7.68468618e-01, 1.23042464e-01,\n", + " 3.48037392e-01, 1.49242997e-01, 2.86662281e-02, 2.79642552e-01,\n", + " -2.26151049e-01, -6.73239648e-01, -8.07924390e-01, 8.62701386e-02,\n", + " 4.94999364e-02, 1.61207989e-02, -1.30242959e-01, 1.77768275e-01,\n", + " 3.62961054e-01, -3.20745975e-01, 3.67820978e-01, -9.77848917e-02,\n", + " -2.64019221e-01, 6.74475431e-01, 9.26629007e-01, -4.54470068e-02,\n", + " 9.59405363e-01, 3.02993000e-01, -5.81385851e-01, 3.98850322e-01,\n", + " 7.40434751e-02, 1.79926023e-01, 9.12196040e-02, 2.77938917e-02,\n", + " -2.20950916e-02, -1.98561847e-01, -4.33019698e-01, 1.35872006e-01,\n", + " -3.84440348e-02, 1.63487554e-01, 5.38927615e-02, 8.52212310e-01,\n", + " -8.64772916e-01, -3.00439209e-01, 1.66039094e-02, -4.84181255e-01,\n", + " -2.57156193e-01, 4.46582437e-01, 3.71635705e-02, -7.58354291e-02,\n", + " -1.38248950e-02, 1.01295078e+00, 2.14489758e-01, -1.17217854e-01,\n", + " -2.82662451e-01, 7.08411038e-01, 2.08262652e-01, -1.69240460e-02,\n", + " 1.02334268e-01, 4.20059741e-01, 1.07706316e-01, -3.89203757e-01,\n", + " -5.91410846e-02, -1.77690476e-01, -1.26772380e+00, 1.75859511e-01,\n", + " -2.49499828e-01, 1.60166726e-01, 8.72884393e-02, -4.53421593e-01,\n", + " 1.96858853e-01, -2.25365251e-01, -1.31235719e-02, -4.58204031e-01,\n", + " -1.54087022e-01, -1.87472761e-01, 2.73187131e-01, 4.14693624e-01,\n", + " 6.00348413e-01, 5.16499318e-02, -2.52319247e-01, -2.08351701e-01,\n", + " -3.85643661e-01, -6.44139796e-02, -2.70672083e-01, -5.09124994e-02,\n", + " -1.17392734e-01, -1.16136428e-02, -1.69710606e-01, 2.30101690e-01,\n", + " -6.31506741e-02, 2.20495850e-01, 4.81231391e-01, 3.76428038e-01,\n", + " -2.14597031e-01, -4.70009223e-02, 4.38644290e-01, 2.72557199e-01,\n", + " -1.89499091e-02, 6.36664629e-02, -4.86765429e-02, -6.02428794e-01,\n", + " 5.40002957e-02, -9.60005671e-02, 4.63560931e-02, -3.55034113e-01,\n", + " 2.27724269e-01, -1.30642965e-01, -5.17771959e-01, 7.08835796e-02,\n", + " -2.57462114e-01, -4.82860744e-01, 1.13421358e-01, 9.88648832e-02,\n", + " 6.21988237e-01, 2.64641732e-01, -9.67874378e-03, 1.94528699e-01,\n", + " 9.72453296e-01, -4.36969042e-01, -5.50681949e-02, 1.42934144e-01,\n", + " 1.37221038e-01, 5.63952804e-01, -3.20022464e-01, -5.56031644e-01,\n", + " 9.09894407e-01, 1.02216589e+00, -2.79887915e-01, 1.69066399e-01,\n", + " 6.48921371e-01, 1.68456510e-02, -2.58911937e-01, 4.62736428e-01,\n", + " 8.00172612e-03, 1.66315883e-01, -5.30062854e-01, -3.96020412e-01,\n", + " 4.43380117e-01, -4.35658276e-01, -1.11912012e-01, -5.91614306e-01,\n", + " -7.02220649e-02, 1.41544282e-01, -5.65246567e-02, -1.19229007e+00,\n", + " -1.00026041e-01, 1.35173336e-01, -1.37986809e-01, 4.58395988e-01,\n", + " 2.99769610e-01, 1.13845997e-01, -3.23149785e-02, 4.82394725e-01,\n", + " -6.13934547e-03, 3.68614852e-01, -4.91497517e-01, -4.97332066e-01,\n", + " 8.73729736e-02, 3.60586494e-01, -2.91166097e-01, 1.89481646e-01,\n", + " 2.87948608e-01, 1.90306157e-01, 4.15048778e-01, 3.93784940e-01,\n", + " 6.75817132e-02, 1.18251920e-01, 2.03508779e-01, 3.09830695e-01,\n", + " -1.03927016e+00, 1.00612268e-01, -3.46988708e-01, -7.09752440e-01,\n", + " 2.20241398e-01, -3.74946982e-01, -1.48783788e-01, -1.31232068e-01,\n", + " 3.87498319e-01, 1.67044029e-01, -2.79640555e-01, 3.40543866e-01,\n", + " 1.28378880e+00, 4.47215438e-01, -5.00054121e-01, 6.85076341e-02,\n", + " 1.93691164e-01, -4.66935217e-01, -3.24348718e-01, 4.53348368e-01,\n", + " 6.36629641e-01, -5.52294970e-01, -3.59640062e-01, 2.45728597e-01,\n", + " 4.48195577e-01, -1.36022663e+00, -6.26060665e-01, -4.96963590e-01,\n", + " -2.55071461e-01, -2.31453001e-01, -4.22013104e-01, 5.81141561e-02,\n", + " 1.66424632e-01, -1.81557357e-01, -2.85358205e-02, -1.10628068e+00,\n", + " -2.42026821e-01, -4.49676067e-03, 5.53836450e-02, 4.92810488e-01,\n", + " 5.83105981e-01, 6.97781667e-02, -1.33217961e-01, -1.25093237e-01,\n", + " 1.17499933e-01, -5.19634366e-01, 1.42042309e-01, 2.34404474e-01,\n", + " -2.55929470e-01, 3.23758684e-02, -2.34450802e-01, -7.54091814e-02,\n", + " 1.83672294e-01, -2.25883007e-01, -4.76478487e-02, -4.84889567e-01,\n", + " 1.12959743e-03, 1.80705532e-01, -5.87785244e-02, 4.82457250e-01,\n", + " -1.88920692e-01, 1.47517592e-01, 1.10182568e-01, -2.28278339e-02,\n", + " 8.62778306e-01, 4.46689427e-02, 4.16403189e-02, -1.07179873e-01,\n", + " -1.42522454e+00, -2.31161788e-02, 3.05959303e-02, -6.58722073e-02,\n", + " -3.69132429e-01, 3.49290550e-01, -1.39178723e-01, -3.51127565e-01,\n", + " 5.00785351e-01, 2.31236637e-01, 6.77590072e-02, -3.59323025e-02,\n", + " 2.69076526e-01, -3.60533416e-01, 1.48107335e-01, -1.11518174e-01,\n", + " 1.65307403e-01, -1.74086124e-01, 6.01880312e-01, -5.95235109e-01,\n", + " 5.29538319e-02, 3.12422097e-01, -1.14403330e-01, 2.30422497e-01,\n", + " -9.48345065e-02, 3.76421027e-02, 4.77573276e-02, 3.89954895e-01,\n", + " -1.91829026e-01, -6.26232028e-01, 1.29549801e-01, -2.84714490e-01,\n", + " 2.88834363e-01, 6.25569642e-01, -2.44193405e-01, 3.08956832e-01,\n", + " -4.79587227e-01, 1.59115836e-01, -1.07442781e-01, 1.57203451e-01,\n", + " -8.51369202e-02, -1.20136715e-01, -2.91232206e-02, 1.08408488e-01,\n", + " -5.97195402e-02, -1.21715315e-01, -5.79822421e-01, 3.90639007e-01,\n", + " -2.83878148e-01, -2.72939146e-01, 3.87672335e-04, -2.62640566e-01,\n", + " -1.67415068e-01, 1.97720259e-01, 3.60535234e-01, -1.85247302e-01,\n", + " -2.80813038e-01, 3.32875013e-01, -3.98125350e-01, -3.53022516e-02,\n", + " 5.48863769e-01, -1.35882646e-01, 2.50048220e-01, -1.27448589e-01,\n", + " -3.03174406e-01, 3.85489166e-02, -7.27320850e-01, 5.22592783e-01,\n", + " -1.97360516e-01, -1.98229402e-01, -1.42074719e-01, 4.11824808e-02,\n", + " -2.92105675e-01, 2.07964912e-01, 4.97746691e-02, 1.48062438e-01,\n", + " -2.94304550e-01, 7.31720269e-01, 1.14105418e-02, 5.50758056e-02],\n", " dtype=float32),\n", - " 'prediction': 26}]" + " 'prediction': 8}]" ] }, "execution_count": null, @@ -1834,9 +4259,9 @@ " concurrency=4,\n", " batch_size=64,\n", " num_gpus=1,\n", - " accelerator_type=\"L4\",\n", + " accelerator_type=\"T4\",\n", ")\n", - "pred_ds.take(1)" + "pred_ds.take(1)\n" ] }, { @@ -1867,21 +4292,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:25:31,814\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_59_0\n" + "2025-08-22 00:34:50,290\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_99_0\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025-06-23 14:25:31,828\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_59_0. Full logs are in /tmp/ray/session_2025-06-23_13-49-50_102769_2149/logs/ray-data\n", - "2025-06-23 14:25:31,829\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_59_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> TaskPoolMapOperator[MapBatches(TorchPredictor)] -> TaskPoolMapOperator[MapBatches(batch_metric)] -> AllToAllOperator[Aggregate] -> LimitOperator[limit=1]\n" + "2025-08-22 00:34:50,303\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_99_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", + "2025-08-22 00:34:50,304\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_99_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> TaskPoolMapOperator[MapBatches(TorchPredictor)] -> TaskPoolMapOperator[MapBatches(batch_metric)] -> AllToAllOperator[Aggregate] -> LimitOperator[limit=1]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "419085884b1849758482929023d6eb50", + "model_id": "1bf87bfd70924161a7f4f956a92eb23f", "version_major": 2, "version_minor": 0 }, @@ -1892,17 +4317,10 @@ "metadata": {}, "output_type": "display_data" }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-06-23 14:25:31,856\tINFO actor_pool_map_operator.py:633 -- Scaling up actor pool by 4 (reason=scaling to min size, running=0, restarting=0, pending=0)\n" - ] - }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fbde64d675d4412597d9ace64aa3ac38", + "model_id": "2114df52a7ac4646aabfda7f7802a648", "version_major": 2, "version_minor": 0 }, @@ -1916,7 +4334,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8325afb46d644ecc9027b7f152341021", + "model_id": "d889fe01be0545939617a037455180df", "version_major": 2, "version_minor": 0 }, @@ -1930,7 +4348,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e6a774a2b1f84b4086e750dc8ac348ed", + "model_id": "e8b23a2321514f21a50825b660f670bf", "version_major": 2, "version_minor": 0 }, @@ -1944,7 +4362,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b4770e750930416cbe1629bbf698f4a2", + "model_id": "627c443e2450449c8683775fc89d7a8f", "version_major": 2, "version_minor": 0 }, @@ -1958,7 +4376,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "89ffb7899dcc47a3b36daa13da9cfe4d", + "model_id": "2d41291adfbf4c86817b203dc9e6f181", "version_major": 2, "version_minor": 0 }, @@ -1972,7 +4390,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "85af9e24a5fa41189b94044860db6ee7", + "model_id": "a2831719e1324270a3662420aff4c1e0", "version_major": 2, "version_minor": 0 }, @@ -1986,7 +4404,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5d9111eece5c4d9381953e84c53de7b0", + "model_id": "120a18eec3a64dfda631fa6dbff06232", "version_major": 2, "version_minor": 0 }, @@ -2000,7 +4418,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1242eb6344e2407b9c9ca909ffc59816", + "model_id": "18834b310df94e338ad7ad76aaf77ec5", "version_major": 2, "version_minor": 0 }, @@ -2014,7 +4432,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "02d595ac51494d12bb2428a788d73b7f", + "model_id": "898fc65b6d6e491f9bd6dd33572f0d6d", "version_major": 2, "version_minor": 0 }, @@ -2028,7 +4446,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2f749b365c194b4a8da80f1c7679a040", + "model_id": "7c7712b3872448c49f04dd6ed1af48f6", "version_major": 2, "version_minor": 0 }, @@ -2042,7 +4460,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "cd5adf33abcd4a6f86a3411387dc62e6", + "model_id": "970b96aca5db4f1aab1e487105e61cae", "version_major": 2, "version_minor": 0 }, @@ -2056,7 +4474,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6efdb6f918a24510b0d69e060da5e2de", + "model_id": "aecfe233dbf249b7ab34fc9d26184bc5", "version_major": 2, "version_minor": 0 }, @@ -2071,23 +4489,35 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=7186, ip=10.0.90.122)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "2025-06-23 14:25:43,855\tINFO actor_pool_map_operator.py:661 -- Scaled down actor pool by 1 (reason=None; running=3, restarting=0, pending=0)\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=7259, ip=10.0.90.122)\u001b[0m /tmp/ipykernel_14938/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", - "\u001b[36m(_MapWorker pid=14469, ip=10.0.103.152)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster]\u001b[0m\n", - "2025-06-23 14:25:44,370\tINFO actor_pool_map_operator.py:661 -- Scaled down actor pool by 1 (reason=None; running=2, restarting=0, pending=0)\n", - "2025-06-23 14:25:44,899\tINFO actor_pool_map_operator.py:661 -- Scaled down actor pool by 1 (reason=None; running=1, restarting=0, pending=0)\n", - "2025-06-23 14:25:45,419\tINFO actor_pool_map_operator.py:661 -- Scaled down actor pool by 1 (reason=None; running=0, restarting=0, pending=0)\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=7393, ip=10.0.90.122)\u001b[0m /tmp/ipykernel_14938/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=46643, ip=10.0.102.235)\u001b[0m /tmp/ipykernel_14938/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=15409, ip=10.0.69.70)\u001b[0m /tmp/ipykernel_14938/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=16788, ip=10.0.90.122)\u001b[0m /tmp/ipykernel_14938/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=15462, ip=10.0.67.42)\u001b[0m /tmp/ipykernel_14938/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=47017, ip=10.0.102.235)\u001b[0m /tmp/ipykernel_14938/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=15584, ip=10.0.69.70)\u001b[0m /tmp/ipykernel_14938/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=17097, ip=10.0.103.152)\u001b[0m /tmp/ipykernel_14938/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=17183, ip=10.0.90.122)\u001b[0m /tmp/ipykernel_14938/3214280880.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "2025-06-23 14:26:35,251\tINFO streaming_executor.py:227 -- ✔️ Dataset dataset_59_0 execution finished in 63.42 seconds\n" + "\u001b[36m(_MapWorker pid=19193, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "\u001b[36m(_MapWorker pid=25926, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 2x across cluster]\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(autoscaler +9m10s)\u001b[0m [autoscaler] Cluster upscaled to {120 CPU, 9 GPU}.\n", + "\u001b[36m(autoscaler +9m15s)\u001b[0m [autoscaler] Cluster upscaled to {168 CPU, 13 GPU}.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(MapBatches(TorchPredictor) pid=2582, ip=10.0.31.199)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", + "\u001b[36m(_MapWorker pid=27577, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=2578, ip=10.0.31.199)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=2576, ip=10.0.31.199)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=3977, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 3x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=4229, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 2x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=2579, ip=10.0.31.199)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=2581, ip=10.0.31.199)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=5094, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=5289, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=5548, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=5816, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "2025-08-22 00:38:03,968\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_99_0 execution finished in 193.66 seconds\n" ] } ], @@ -2106,7 +4536,7 @@ "precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n", "recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n", "f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0\n", - "accuracy = (tp + tn) / (tp + tn + fp + fn)" + "accuracy = (tp + tn) / (tp + tn + fp + fn)\n" ] }, { @@ -2123,13 +4553,108 @@ "F1: 0.84\n", "Accuracy: 0.98\n" ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(autoscaler +13m0s)\u001b[0m [autoscaler] Downscaling node i-0ffe5abae6e899f5a (node IP: 10.0.60.138) due to node idle termination.\n", + "\u001b[36m(autoscaler +13m5s)\u001b[0m [autoscaler] Cluster resized to {120 CPU, 9 GPU}.\n", + "\u001b[36m(autoscaler +16m0s)\u001b[0m [autoscaler] Downscaling node i-0aa72cef9b8921af5 (node IP: 10.0.31.199) due to node idle termination.\n", + "\u001b[36m(autoscaler +16m5s)\u001b[0m [autoscaler] Cluster resized to {112 CPU, 8 GPU}.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Using CPython 3.12.11 interpreter at: /home/ray/anaconda3/bin/python3.12\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Creating virtual environment at: .venv\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Building doggos @ file:///tmp/ray/session_2025-08-21_18-48-13_464408_2298/runtime_resources/working_dir_files/_ray_pkg_f79228c33bd2a431/doggos\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pillow (6.3MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading grpcio (5.9MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sqlalchemy (3.2MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pydantic-core (1.9MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading jedi (1.5MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading virtualenv (5.7MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pandas (11.4MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading setuptools (1.1MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading uvloop (4.5MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cuda-nvrtc-cu12 (22.6MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sympy (6.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading numpy (15.9MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading kiwisolver (1.4MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading tokenizers (3.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pyarrow (38.2MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading botocore (13.3MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading fonttools (4.7MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading widgetsnbextension (2.1MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow-skinny (5.6MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading aiohttp (1.6MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading networkx (1.9MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pygments (1.2MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading debugpy (4.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading py-spy (2.6MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scikit-learn (12.5MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading hf-xet (3.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading matplotlib (8.2MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading torch (783.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading transformers (10.0MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scipy (33.5MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading polars (36.7MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow (26.1MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading triton (148.5MiB)\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Built doggos @ file:///tmp/ray/session_2025-08-21_18-48-13_464408_2298/runtime_resources/working_dir_files/_ray_pkg_f79228c33bd2a431/doggos\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pillow\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading grpcio\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sqlalchemy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pydantic-core\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading jedi\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading virtualenv\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading setuptools\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading uvloop\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cuda-cupti-cu12\u001b[32m [repeated 13x across cluster]\u001b[0m\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sympy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading kiwisolver\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading tokenizers\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading fonttools\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading widgetsnbextension\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow-skinny\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading aiohttp\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading networkx\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pygments\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading debugpy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading py-spy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading hf-xet\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading matplotlib\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading transformers\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scikit-learn\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading numpy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading botocore\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pandas\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading polars\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cuda-nvrtc-cu12\u001b[32m [repeated 2x across cluster]\u001b[0m\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scipy\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pyarrow\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-curand-cu12\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cusparselt-cu12\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading triton\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cublas-cu12\u001b[32m [repeated 5x across cluster]\u001b[0m\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading torch\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m warning: Failed to hardlink files; falling back to full copy. This may lead to degraded performance.\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m If the cache and target directories are on different filesystems, hardlinking may not be supported.\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m If this is intentional, set `export UV_LINK_MODE=copy` or use `--link-mode=copy` to suppress this warning.\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cudnn-cu12\n", + "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Installed 172 packages in 1.96s\n" + ] } ], "source": [ "print(f\"Precision: {precision:.2f}\")\n", "print(f\"Recall: {recall:.2f}\")\n", "print(f\"F1: {f1:.2f}\")\n", - "print(f\"Accuracy: {accuracy:.2f}\")" + "print(f\"Accuracy: {accuracy:.2f}\")\n" ] }, { diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/03-Online-Serving.ipynb b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/03-Online-Serving.ipynb index 587a257dd3a5..0dfd026ca0d8 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/03-Online-Serving.ipynb +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/03-Online-Serving.ipynb @@ -24,14 +24,35 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[92mSuccessfully registered `matplotlib, torch` and 4 other packages to be installed on all cluster nodes.\u001b[0m\n", - "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_eys8cskj5aivghbf773dp2vmcd?workspace-tab=dependencies\u001b[0m\n" + "\u001b[92mSuccessfully registered `ipywidgets, matplotlib` and 4 other packages to be installed on all cluster nodes.\u001b[0m\n", + "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_1dp3fa7w5hu3i83ldsi7lqvp9t?workspace-tab=dependencies\u001b[0m\n", + "\u001b[92mSuccessfully registered `doggos` package to be installed on all cluster nodes.\u001b[0m\n", + "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_1dp3fa7w5hu3i83ldsi7lqvp9t?workspace-tab=dependencies\u001b[0m\n" ] } ], "source": [ "%%bash\n", - "pip install -q \"matplotlib==3.10.0\" \"torch==2.7.0\" \"transformers==4.52.3\" \"scikit-learn==1.6.0\" \"mlflow==2.19.0\" \"ipywidgets==8.1.3\"" + "pip install -q -r /home/ray/default/requirements.txt\n", + "pip install -q -e /home/ray/default/doggos\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note**: A kernel restart may be required for all dependencies to become available. \n", + "\n", + "If using **uv**, then:\n", + "1. Turn off the runtime dependencies (`Dependencies` tab up top > Toggle off `Pip packages`). And no need to run the `pip install` commands above.\n", + "2. Change the python kernel of this notebook to use the `venv` (Click on `base (Python x.yy.zz)` on top right cordern of notebook > `Select another Kernel` > `Python Environments...` > `Create Python Environment` > `Venv` > `Use Existing`) and done! Now all the notebook's cells will use the virtual env.\n", + "3. Change the py executable to use `uv run` instead of `python` by adding this line after importing ray.\n", + "```python\n", + "import os\n", + "os.environ.pop(\"RAY_RUNTIME_ENV_HOOK\", None)\n", + "import ray\n", + "ray.init(runtime_env={\"py_executable\": \"uv run\", \"working_dir\": \"/home/ray/default\"})\n", + "```" ] }, { @@ -41,83 +62,30 @@ "outputs": [], "source": [ "%load_ext autoreload\n", - "%autoreload all" + "%autoreload all\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-06-23 20:03:54,080\tINFO worker.py:1723 -- Connecting to existing Ray cluster at address: 10.0.61.28:6379...\n", - "2025-06-23 20:03:54,091\tINFO worker.py:1908 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttps://session-gcwehd9xxjzkv5lxv8lgcdgx2n.i.anyscaleuserdata.com \u001b[39m\u001b[22m\n", - "2025-06-23 20:03:54,133\tINFO packaging.py:588 -- Creating a file package for local module '../'.\n", - "2025-06-23 20:03:54,190\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_60b8ab9607f9a287.zip' (12.99MiB) to Ray cluster...\n", - "2025-06-23 20:03:54,250\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_60b8ab9607f9a287.zip'.\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "aa33be7c5f98450283f661adb61a3c6b", - "version_major": 2, - "version_minor": 0 - }, - "text/html": [ - "
\n", - "
\n", - "
\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - "\n", - "
Python version:3.12.11
Ray version:2.47.1
Dashboard:http://session-gcwehd9xxjzkv5lxv8lgcdgx2n.i.anyscaleuserdata.com
\n", - "\n", - "
\n", - "
\n" - ], - "text/plain": [ - "RayContext(dashboard_url='session-gcwehd9xxjzkv5lxv8lgcdgx2n.i.anyscaleuserdata.com', python_version='3.12.11', ray_version='2.47.1', ray_commit='e06f523c450fb1c99d8f347f8bfcc4085cc68b66')" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import os\n", "import ray\n", "import sys\n", - "sys.path.append(os.path.abspath(\"..\"))\n", - "ray.init(runtime_env={\"working_dir\": \"../\"})" + "sys.path.append(os.path.abspath(\"../doggos/\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# If using UV\n", + "# os.environ.pop(\"RAY_RUNTIME_ENV_HOOK\", None)\n", + "# ray.init(runtime_env={\"py_executable\": \"uv run\", \"working_dir\": \"/home/ray/default\"})\n" ] }, { @@ -132,7 +100,7 @@ "import requests\n", "from starlette.requests import Request\n", "from urllib.parse import urlparse\n", - "from ray import serve" + "from ray import serve\n" ] }, { @@ -144,7 +112,7 @@ "import numpy as np\n", "from PIL import Image\n", "import torch\n", - "from transformers import CLIPModel, CLIPProcessor" + "from transformers import CLIPModel, CLIPProcessor\n" ] }, { @@ -155,7 +123,7 @@ "source": [ "from doggos.infer import TorchPredictor\n", "from doggos.model import collate_fn\n", - "from doggos.utils import url_to_array" + "from doggos.utils import url_to_array\n" ] }, { @@ -182,7 +150,7 @@ " num_replicas=\"1\", \n", " ray_actor_options={\n", " \"num_gpus\": 1, \n", - " \"accelerator_type\": \"L4\",\n", + " \"accelerator_type\": \"T4\",\n", " },\n", ")\n", "class ClassPredictor:\n", @@ -205,7 +173,7 @@ " embedding = self.model.get_image_features(**inputs).cpu().numpy()\n", " outputs = self.predictor.predict_probabilities(\n", " collate_fn({\"embedding\": embedding}))\n", - " return {\"probabilities\": outputs[\"probabilities\"][0]}" + " return {\"probabilities\": outputs[\"probabilities\"][0]}\n" ] }, { @@ -237,7 +205,7 @@ " title=\"doggos\", \n", " description=\"classify your dog\", \n", " version=\"0.1\",\n", - ")" + ")\n" ] }, { @@ -256,7 +224,7 @@ " async def predict(self, request: Request):\n", " data = await request.json()\n", " probabilities = await self.classifier.get_probabilities.remote(url=data[\"url\"])\n", - " return probabilities" + " return probabilities\n" ] }, { @@ -268,7 +236,7 @@ "# Model registry.\n", "model_registry = \"/mnt/cluster_storage/mlflow/doggos\"\n", "experiment_name = \"doggos\"\n", - "mlflow.set_tracking_uri(f\"file:{model_registry}\")" + "mlflow.set_tracking_uri(f\"file:{model_registry}\")\n" ] }, { @@ -278,12 +246,11 @@ "outputs": [], "source": [ "# Get best_run's artifact_dir.\n", - "mlflow.set_tracking_uri(f\"file:{model_registry}\")\n", "sorted_runs = mlflow.search_runs(\n", " experiment_names=[experiment_name], \n", " order_by=[\"metrics.val_loss ASC\"])\n", "best_run = sorted_runs.iloc[0]\n", - "artifacts_dir = urlparse(best_run.artifact_uri).path" + "artifacts_dir = urlparse(best_run.artifact_uri).path\n" ] }, { @@ -299,7 +266,7 @@ " artifacts_dir=artifacts_dir,\n", " device=\"cuda\"\n", " )\n", - ")" + ")\n" ] }, { @@ -311,27 +278,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(ProxyActor pid=75693)\u001b[0m INFO 2025-06-23 20:04:07,726 proxy 10.0.61.28 -- Proxy starting on node b4c1ef3393280e7df5c15725708ef231f52e1e31e050f75f5d32a41a (HTTP port: 8000).\n", - "\u001b[36m(ProxyActor pid=75693)\u001b[0m INFO 2025-06-23 20:04:07,794 proxy 10.0.61.28 -- Got updated endpoints: {}.\n", - "INFO 2025-06-23 20:04:07,815 serve 75456 -- Started Serve in namespace \"serve\".\n", - "\u001b[36m(ServeController pid=75629)\u001b[0m INFO 2025-06-23 20:04:07,905 controller 75629 -- Deploying new version of Deployment(name='ClassPredictor', app='default') (initial target replicas: 1).\n", - "\u001b[36m(ServeController pid=75629)\u001b[0m INFO 2025-06-23 20:04:07,907 controller 75629 -- Deploying new version of Deployment(name='Doggos', app='default') (initial target replicas: 1).\n", - "\u001b[36m(ProxyActor pid=75693)\u001b[0m INFO 2025-06-23 20:04:07,910 proxy 10.0.61.28 -- Got updated endpoints: {Deployment(name='Doggos', app='default'): EndpointInfo(route='/', app_is_cross_language=False)}.\n", - "\u001b[36m(ServeController pid=75629)\u001b[0m INFO 2025-06-23 20:04:08,013 controller 75629 -- Adding 1 replica to Deployment(name='ClassPredictor', app='default').\n", - "\u001b[36m(ServeController pid=75629)\u001b[0m INFO 2025-06-23 20:04:08,014 controller 75629 -- Adding 1 replica to Deployment(name='Doggos', app='default').\n", - "\u001b[36m(ProxyActor pid=75693)\u001b[0m INFO 2025-06-23 20:04:07,922 proxy 10.0.61.28 -- Started .\n", - "\u001b[36m(ServeController pid=75629)\u001b[0m WARNING 2025-06-23 20:04:38,040 controller 75629 -- Deployment 'ClassPredictor' in application 'default' has 1 replicas that have taken more than 30s to be scheduled. This may be due to waiting for the cluster to auto-scale or for a runtime environment to be installed. Resources required for each replica: {\"CPU\": 1, \"GPU\": 1, \"accelerator_type:L4\": 0.001}, total resources available: {\"accelerator_type:L4\": 0.999, \"CPU\": 2.0}. Use `ray status` for more details.\n", - "\u001b[36m(ServeController pid=75629)\u001b[0m WARNING 2025-06-23 20:04:38,041 controller 75629 -- Deployment 'Doggos' in application 'default' has 1 replicas that have taken more than 30s to be scheduled. This may be due to waiting for the cluster to auto-scale or for a runtime environment to be installed. Resources required for each replica: {\"CPU\": 1}, total resources available: {\"CPU\": 2.0}. Use `ray status` for more details.\n", - "\u001b[36m(ServeReplica:default:Doggos pid=19668, ip=10.0.95.114)\u001b[0m INFO 2025-06-23 20:05:03,231 default_Doggos 21c29nfb -- Direct ingress is disabled, skipping direct ingress server start\n", - "\u001b[36m(ProxyActor pid=19768, ip=10.0.95.114)\u001b[0m INFO 2025-06-23 20:05:05,037 proxy 10.0.95.114 -- Proxy starting on node 760a1c063ba581ef6100d697d1e1d263b0b354b603658541229768ae (HTTP port: 8000).\n", - "\u001b[36m(ProxyActor pid=19768, ip=10.0.95.114)\u001b[0m INFO 2025-06-23 20:05:05,092 proxy 10.0.95.114 -- Got updated endpoints: {Deployment(name='Doggos', app='default'): EndpointInfo(route='/', app_is_cross_language=False)}.\n", - "\u001b[36m(ProxyActor pid=19768, ip=10.0.95.114)\u001b[0m INFO 2025-06-23 20:05:05,105 proxy 10.0.95.114 -- Started .\n", - "\u001b[36m(ServeReplica:default:ClassPredictor pid=19669, ip=10.0.95.114)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "\u001b[36m(ServeController pid=75629)\u001b[0m WARNING 2025-06-23 20:05:08,122 controller 75629 -- Deployment 'ClassPredictor' in application 'default' has 1 replicas that have taken more than 30s to initialize.\n", - "\u001b[36m(ServeController pid=75629)\u001b[0m This may be caused by a slow __init__ or reconfigure method.\n", - "\u001b[36m(ServeReplica:default:ClassPredictor pid=19669, ip=10.0.95.114)\u001b[0m INFO 2025-06-23 20:05:09,415 default_ClassPredictor fyf5xp23 -- Direct ingress is disabled, skipping direct ingress server start\n", - "INFO 2025-06-23 20:05:10,065 serve 75456 -- Application 'default' is ready at http://127.0.0.1:8000/.\n", - "INFO 2025-06-23 20:05:10,071 serve 75456 -- Started .\n" + "2025-08-22 00:51:12,070\tINFO worker.py:1747 -- Connecting to existing Ray cluster at address: 10.0.52.10:6379...\n", + "2025-08-22 00:51:12,082\tINFO worker.py:1918 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttps://session-466hy7cqu1gzrp8zk8l4byz7l7.i.anyscaleuserdata.com \u001b[39m\u001b[22m\n", + "2025-08-22 00:51:12,091\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_bb1ea558d334a00804951688d29c76ff051341cc.zip' (1.11MiB) to Ray cluster...\n", + "2025-08-22 00:51:12,096\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_bb1ea558d334a00804951688d29c76ff051341cc.zip'.\n", + "INFO 2025-08-22 00:51:12,153 serve 133557 -- Connecting to existing Serve app in namespace \"serve\". New http options will not be applied.\n", + "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,275 controller 61167 -- Deploying new version of Deployment(name='ClassPredictor', app='default') (initial target replicas: 1).\n", + "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,276 controller 61167 -- Deploying new version of Deployment(name='Doggos', app='default') (initial target replicas: 1).\n", + "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,400 controller 61167 -- Stopping 1 replicas of Deployment(name='ClassPredictor', app='default') with outdated versions.\n", + "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,401 controller 61167 -- Adding 1 replica to Deployment(name='ClassPredictor', app='default').\n", + "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,403 controller 61167 -- Stopping 1 replicas of Deployment(name='Doggos', app='default') with outdated versions.\n", + "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,404 controller 61167 -- Adding 1 replica to Deployment(name='Doggos', app='default').\n", + "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,489 controller 61167 -- Draining proxy on node '7cf4feced6fe6a3166528c758e7aea63f8414ff9b95e3f69304a0bbb'.\n", + "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:14,489 controller 61167 -- Replica(id='m8hm2lqw', deployment='ClassPredictor', app='default') is stopped.\n", + "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:14,491 controller 61167 -- Replica(id='r17z6dkp', deployment='Doggos', app='default') is stopped.\n", + "\u001b[36m(ServeReplica:default:Doggos pid=133722)\u001b[0m INFO 2025-08-22 00:51:14,611 default_Doggos 0qpk1kw9 -- Direct ingress is disabled, skipping direct ingress server start\n", + "\u001b[36m(ServeReplica:default:ClassPredictor pid=30024, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:19,249 controller 61167 -- No longer draining proxy on node '7cf4feced6fe6a3166528c758e7aea63f8414ff9b95e3f69304a0bbb'.\n", + "\u001b[36m(ServeReplica:default:ClassPredictor pid=30024, ip=10.0.4.102)\u001b[0m INFO 2025-08-22 00:51:21,500 default_ClassPredictor qtsnu3yv -- Direct ingress is disabled, skipping direct ingress server start\n", + "INFO 2025-08-22 00:51:22,301 serve 133557 -- Application 'default' is ready at http://127.0.0.1:8000/.\n", + "INFO 2025-08-22 00:51:22,313 serve 133557 -- Started .\n" ] }, { @@ -347,7 +313,7 @@ ], "source": [ "# Run service locally.\n", - "serve.run(app, route_prefix=\"/\")" + "serve.run(app, route_prefix=\"/\")\n" ] }, { @@ -355,23 +321,40 @@ "execution_count": null, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(ServeReplica:default:ClassPredictor pid=30024, ip=10.0.4.102)\u001b[0m /home/ray/anaconda3/lib/python3.12/site-packages/ray/serve/_private/replica.py:1376: UserWarning: Calling sync method 'get_probabilities' directly on the asyncio loop. In a future version, sync methods will be run in a threadpool by default. Ensure your sync methods are thread safe or keep the existing behavior by making them `async def`. Opt into the new behavior by setting RAY_SERVE_RUN_SYNC_IN_THREADPOOL=1.\n", + "\u001b[36m(ServeReplica:default:ClassPredictor pid=30024, ip=10.0.4.102)\u001b[0m warnings.warn(\n", + "\u001b[36m(ServeReplica:default:Doggos pid=133722)\u001b[0m INFO 2025-08-22 00:51:22,490 default_Doggos 0qpk1kw9 daf11861-073e-4758-82b7-0b222066b668 -- Started .\n" + ] + }, { "data": { "text/plain": [ - "[('collie', 0.2568000853061676),\n", - " ('border_collie', 0.16908691823482513),\n", - " ('bernese_mountain_dog', 0.0767023041844368)]" + "[('collie', 0.2292557954788208),\n", + " ('border_collie', 0.1228194534778595),\n", + " ('german_shepherd', 0.07383470982313156)]" ] }, "execution_count": null, "metadata": {}, "output_type": "execute_result" }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(ServeReplica:default:ClassPredictor pid=30024, ip=10.0.4.102)\u001b[0m INFO 2025-08-22 00:51:23,011 default_ClassPredictor qtsnu3yv daf11861-073e-4758-82b7-0b222066b668 -- CALL /predict/ OK 504.9ms\n", + "\u001b[36m(ServeReplica:default:Doggos pid=133722)\u001b[0m INFO 2025-08-22 00:51:23,013 default_Doggos 0qpk1kw9 daf11861-073e-4758-82b7-0b222066b668 -- POST /predict/ 200 567.1ms\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[36m(autoscaler +38m14s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n" + "\u001b[36m(autoscaler +13m35s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n" ] } ], @@ -382,7 +365,7 @@ "response = requests.post(\"http://127.0.0.1:8000/predict/\", json=data)\n", "probabilities = response.json()[\"probabilities\"]\n", "sorted_probabilities = sorted(probabilities.items(), key=lambda x: x[1], reverse=True)\n", - "sorted_probabilities[0:3]" + "sorted_probabilities[0:3]\n" ] }, { @@ -476,13 +459,8 @@ "source": [ "```bash\n", "# Production online service.\n", - "anyscale service deploy doggos.serve:app --name=doggos-app \\\n", - " --containerfile=\"/home/ray/default/containerfile\" \\\n", - " --compute-config=\"/home/ray/default/configs/aws.yaml\" \\\n", - " --working-dir=\"/home/ray/default\" \\\n", - " --exclude=\"\"\n", + "anyscale service deploy -f /home/ray/default/configs/service.yaml\n", "```\n", - "\n", "```\n", "(anyscale +1.9s) Restarting existing service 'doggos-app'.\n", "(anyscale +3.2s) Uploading local dir '/home/ray/default' to cloud storage.\n", diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/pyproject.toml b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/pyproject.toml new file mode 100644 index 000000000000..b63d44cd0ee8 --- /dev/null +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "default" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "boto3>=1.40.9", + "doggos", + "ipykernel>=6.30.1", + "ipywidgets==8.1.3", + "matplotlib==3.10.0", + "mlflow==2.19.0", + "ray[data,serve,train,tune]", + "scikit-learn==1.6.0", + "torch==2.7.1", + "transformers==4.52.3", +] + +[tool.uv.sources] +ray = { url = "http://localhost:9478/ray/ray-2.48.0-cp312-cp312-manylinux2014_x86_64.whl" } +doggos = { path = "doggos" } diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/requirements.txt b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/requirements.txt index 42a1a0e489ce..603b6a10aafe 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/requirements.txt +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/requirements.txt @@ -1,6 +1,6 @@ +ipywidgets==8.1.3 matplotlib==3.10.0 +mlflow==2.19.0 torch==2.7.1 transformers==4.52.3 -scikit-learn==1.6.0 -mlflow==2.19.0 -ipywidgets==8.1.3 +scikit-learn==1.6.0 \ No newline at end of file diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/uv.lock b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/uv.lock new file mode 100644 index 000000000000..bf67a0f858bc --- /dev/null +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/uv.lock @@ -0,0 +1,3506 @@ +version = 1 +revision = 2 +requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.13' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform != 'win32'", + "python_full_version >= '3.13' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", +] + +[[package]] +name = "aiohappyeyeballs" +version = "2.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.12.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/e7/d92a237d8802ca88483906c388f7c201bbe96cd80a165ffd0ac2f6a8d59f/aiohttp-3.12.15.tar.gz", hash = "sha256:4fc61385e9c98d72fcdf47e6dd81833f47b2f77c114c29cd64a361be57a763a2", size = 7823716, upload-time = "2025-07-29T05:52:32.215Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/97/77cb2450d9b35f517d6cf506256bf4f5bda3f93a66b4ad64ba7fc917899c/aiohttp-3.12.15-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:802d3868f5776e28f7bf69d349c26fc0efadb81676d0afa88ed00d98a26340b7", size = 702333, upload-time = "2025-07-29T05:50:46.507Z" }, + { url = "https://files.pythonhosted.org/packages/83/6d/0544e6b08b748682c30b9f65640d006e51f90763b41d7c546693bc22900d/aiohttp-3.12.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2800614cd560287be05e33a679638e586a2d7401f4ddf99e304d98878c29444", size = 476948, upload-time = "2025-07-29T05:50:48.067Z" }, + { url = "https://files.pythonhosted.org/packages/3a/1d/c8c40e611e5094330284b1aea8a4b02ca0858f8458614fa35754cab42b9c/aiohttp-3.12.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8466151554b593909d30a0a125d638b4e5f3836e5aecde85b66b80ded1cb5b0d", size = 469787, upload-time = "2025-07-29T05:50:49.669Z" }, + { url = "https://files.pythonhosted.org/packages/38/7d/b76438e70319796bfff717f325d97ce2e9310f752a267bfdf5192ac6082b/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e5a495cb1be69dae4b08f35a6c4579c539e9b5706f606632102c0f855bcba7c", size = 1716590, upload-time = "2025-07-29T05:50:51.368Z" }, + { url = "https://files.pythonhosted.org/packages/79/b1/60370d70cdf8b269ee1444b390cbd72ce514f0d1cd1a715821c784d272c9/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6404dfc8cdde35c69aaa489bb3542fb86ef215fc70277c892be8af540e5e21c0", size = 1699241, upload-time = "2025-07-29T05:50:53.628Z" }, + { url = "https://files.pythonhosted.org/packages/a3/2b/4968a7b8792437ebc12186db31523f541943e99bda8f30335c482bea6879/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ead1c00f8521a5c9070fcb88f02967b1d8a0544e6d85c253f6968b785e1a2ab", size = 1754335, upload-time = "2025-07-29T05:50:55.394Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c1/49524ed553f9a0bec1a11fac09e790f49ff669bcd14164f9fab608831c4d/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6990ef617f14450bc6b34941dba4f12d5613cbf4e33805932f853fbd1cf18bfb", size = 1800491, upload-time = "2025-07-29T05:50:57.202Z" }, + { url = "https://files.pythonhosted.org/packages/de/5e/3bf5acea47a96a28c121b167f5ef659cf71208b19e52a88cdfa5c37f1fcc/aiohttp-3.12.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd736ed420f4db2b8148b52b46b88ed038d0354255f9a73196b7bbce3ea97545", size = 1719929, upload-time = "2025-07-29T05:50:59.192Z" }, + { url = "https://files.pythonhosted.org/packages/39/94/8ae30b806835bcd1cba799ba35347dee6961a11bd507db634516210e91d8/aiohttp-3.12.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c5092ce14361a73086b90c6efb3948ffa5be2f5b6fbcf52e8d8c8b8848bb97c", size = 1635733, upload-time = "2025-07-29T05:51:01.394Z" }, + { url = "https://files.pythonhosted.org/packages/7a/46/06cdef71dd03acd9da7f51ab3a9107318aee12ad38d273f654e4f981583a/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:aaa2234bb60c4dbf82893e934d8ee8dea30446f0647e024074237a56a08c01bd", size = 1696790, upload-time = "2025-07-29T05:51:03.657Z" }, + { url = "https://files.pythonhosted.org/packages/02/90/6b4cfaaf92ed98d0ec4d173e78b99b4b1a7551250be8937d9d67ecb356b4/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6d86a2fbdd14192e2f234a92d3b494dd4457e683ba07e5905a0b3ee25389ac9f", size = 1718245, upload-time = "2025-07-29T05:51:05.911Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e6/2593751670fa06f080a846f37f112cbe6f873ba510d070136a6ed46117c6/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a041e7e2612041a6ddf1c6a33b883be6a421247c7afd47e885969ee4cc58bd8d", size = 1658899, upload-time = "2025-07-29T05:51:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/8f/28/c15bacbdb8b8eb5bf39b10680d129ea7410b859e379b03190f02fa104ffd/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5015082477abeafad7203757ae44299a610e89ee82a1503e3d4184e6bafdd519", size = 1738459, upload-time = "2025-07-29T05:51:09.56Z" }, + { url = "https://files.pythonhosted.org/packages/00/de/c269cbc4faa01fb10f143b1670633a8ddd5b2e1ffd0548f7aa49cb5c70e2/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:56822ff5ddfd1b745534e658faba944012346184fbfe732e0d6134b744516eea", size = 1766434, upload-time = "2025-07-29T05:51:11.423Z" }, + { url = "https://files.pythonhosted.org/packages/52/b0/4ff3abd81aa7d929b27d2e1403722a65fc87b763e3a97b3a2a494bfc63bc/aiohttp-3.12.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b2acbbfff69019d9014508c4ba0401822e8bae5a5fdc3b6814285b71231b60f3", size = 1726045, upload-time = "2025-07-29T05:51:13.689Z" }, + { url = "https://files.pythonhosted.org/packages/71/16/949225a6a2dd6efcbd855fbd90cf476052e648fb011aa538e3b15b89a57a/aiohttp-3.12.15-cp312-cp312-win32.whl", hash = "sha256:d849b0901b50f2185874b9a232f38e26b9b3d4810095a7572eacea939132d4e1", size = 423591, upload-time = "2025-07-29T05:51:15.452Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d8/fa65d2a349fe938b76d309db1a56a75c4fb8cc7b17a398b698488a939903/aiohttp-3.12.15-cp312-cp312-win_amd64.whl", hash = "sha256:b390ef5f62bb508a9d67cb3bba9b8356e23b3996da7062f1a57ce1a79d2b3d34", size = 450266, upload-time = "2025-07-29T05:51:17.239Z" }, + { url = "https://files.pythonhosted.org/packages/f2/33/918091abcf102e39d15aba2476ad9e7bd35ddb190dcdd43a854000d3da0d/aiohttp-3.12.15-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9f922ffd05034d439dde1c77a20461cf4a1b0831e6caa26151fe7aa8aaebc315", size = 696741, upload-time = "2025-07-29T05:51:19.021Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2a/7495a81e39a998e400f3ecdd44a62107254803d1681d9189be5c2e4530cd/aiohttp-3.12.15-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2ee8a8ac39ce45f3e55663891d4b1d15598c157b4d494a4613e704c8b43112cd", size = 474407, upload-time = "2025-07-29T05:51:21.165Z" }, + { url = "https://files.pythonhosted.org/packages/49/fc/a9576ab4be2dcbd0f73ee8675d16c707cfc12d5ee80ccf4015ba543480c9/aiohttp-3.12.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3eae49032c29d356b94eee45a3f39fdf4b0814b397638c2f718e96cfadf4c4e4", size = 466703, upload-time = "2025-07-29T05:51:22.948Z" }, + { url = "https://files.pythonhosted.org/packages/09/2f/d4bcc8448cf536b2b54eed48f19682031ad182faa3a3fee54ebe5b156387/aiohttp-3.12.15-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97752ff12cc12f46a9b20327104448042fce5c33a624f88c18f66f9368091c7", size = 1705532, upload-time = "2025-07-29T05:51:25.211Z" }, + { url = "https://files.pythonhosted.org/packages/f1/f3/59406396083f8b489261e3c011aa8aee9df360a96ac8fa5c2e7e1b8f0466/aiohttp-3.12.15-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:894261472691d6fe76ebb7fcf2e5870a2ac284c7406ddc95823c8598a1390f0d", size = 1686794, upload-time = "2025-07-29T05:51:27.145Z" }, + { url = "https://files.pythonhosted.org/packages/dc/71/164d194993a8d114ee5656c3b7ae9c12ceee7040d076bf7b32fb98a8c5c6/aiohttp-3.12.15-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5fa5d9eb82ce98959fc1031c28198b431b4d9396894f385cb63f1e2f3f20ca6b", size = 1738865, upload-time = "2025-07-29T05:51:29.366Z" }, + { url = "https://files.pythonhosted.org/packages/1c/00/d198461b699188a93ead39cb458554d9f0f69879b95078dce416d3209b54/aiohttp-3.12.15-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0fa751efb11a541f57db59c1dd821bec09031e01452b2b6217319b3a1f34f3d", size = 1788238, upload-time = "2025-07-29T05:51:31.285Z" }, + { url = "https://files.pythonhosted.org/packages/85/b8/9e7175e1fa0ac8e56baa83bf3c214823ce250d0028955dfb23f43d5e61fd/aiohttp-3.12.15-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5346b93e62ab51ee2a9d68e8f73c7cf96ffb73568a23e683f931e52450e4148d", size = 1710566, upload-time = "2025-07-29T05:51:33.219Z" }, + { url = "https://files.pythonhosted.org/packages/59/e4/16a8eac9df39b48ae102ec030fa9f726d3570732e46ba0c592aeeb507b93/aiohttp-3.12.15-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:049ec0360f939cd164ecbfd2873eaa432613d5e77d6b04535e3d1fbae5a9e645", size = 1624270, upload-time = "2025-07-29T05:51:35.195Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f8/cd84dee7b6ace0740908fd0af170f9fab50c2a41ccbc3806aabcb1050141/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b52dcf013b57464b6d1e51b627adfd69a8053e84b7103a7cd49c030f9ca44461", size = 1677294, upload-time = "2025-07-29T05:51:37.215Z" }, + { url = "https://files.pythonhosted.org/packages/ce/42/d0f1f85e50d401eccd12bf85c46ba84f947a84839c8a1c2c5f6e8ab1eb50/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:9b2af240143dd2765e0fb661fd0361a1b469cab235039ea57663cda087250ea9", size = 1708958, upload-time = "2025-07-29T05:51:39.328Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6b/f6fa6c5790fb602538483aa5a1b86fcbad66244997e5230d88f9412ef24c/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ac77f709a2cde2cc71257ab2d8c74dd157c67a0558a0d2799d5d571b4c63d44d", size = 1651553, upload-time = "2025-07-29T05:51:41.356Z" }, + { url = "https://files.pythonhosted.org/packages/04/36/a6d36ad545fa12e61d11d1932eef273928b0495e6a576eb2af04297fdd3c/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:47f6b962246f0a774fbd3b6b7be25d59b06fdb2f164cf2513097998fc6a29693", size = 1727688, upload-time = "2025-07-29T05:51:43.452Z" }, + { url = "https://files.pythonhosted.org/packages/aa/c8/f195e5e06608a97a4e52c5d41c7927301bf757a8e8bb5bbf8cef6c314961/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:760fb7db442f284996e39cf9915a94492e1896baac44f06ae551974907922b64", size = 1761157, upload-time = "2025-07-29T05:51:45.643Z" }, + { url = "https://files.pythonhosted.org/packages/05/6a/ea199e61b67f25ba688d3ce93f63b49b0a4e3b3d380f03971b4646412fc6/aiohttp-3.12.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad702e57dc385cae679c39d318def49aef754455f237499d5b99bea4ef582e51", size = 1710050, upload-time = "2025-07-29T05:51:48.203Z" }, + { url = "https://files.pythonhosted.org/packages/b4/2e/ffeb7f6256b33635c29dbed29a22a723ff2dd7401fff42ea60cf2060abfb/aiohttp-3.12.15-cp313-cp313-win32.whl", hash = "sha256:f813c3e9032331024de2eb2e32a88d86afb69291fbc37a3a3ae81cc9917fb3d0", size = 422647, upload-time = "2025-07-29T05:51:50.718Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8e/78ee35774201f38d5e1ba079c9958f7629b1fd079459aea9467441dbfbf5/aiohttp-3.12.15-cp313-cp313-win_amd64.whl", hash = "sha256:1a649001580bdb37c6fdb1bebbd7e3bc688e8ec2b5c6f52edbb664662b17dc84", size = 449067, upload-time = "2025-07-29T05:51:52.549Z" }, +] + +[[package]] +name = "aiohttp-cors" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/d89e846a5444b3d5eb8985a6ddb0daef3774928e1bfbce8e84ec97b0ffa7/aiohttp_cors-0.8.1.tar.gz", hash = "sha256:ccacf9cb84b64939ea15f859a146af1f662a6b1d68175754a07315e305fb1403", size = 38626, upload-time = "2025-03-31T14:16:20.048Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/3b/40a68de458904bcc143622015fff2352b6461cd92fd66d3527bf1c6f5716/aiohttp_cors-0.8.1-py3-none-any.whl", hash = "sha256:3180cf304c5c712d626b9162b195b1db7ddf976a2a25172b35bb2448b890a80d", size = 25231, upload-time = "2025-03-31T14:16:18.478Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + +[[package]] +name = "alembic" +version = "1.16.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/52/72e791b75c6b1efa803e491f7cbab78e963695e76d4ada05385252927e76/alembic-1.16.4.tar.gz", hash = "sha256:efab6ada0dd0fae2c92060800e0bf5c1dc26af15a10e02fb4babff164b4725e2", size = 1968161, upload-time = "2025-07-10T16:17:20.192Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/62/96b5217b742805236614f05904541000f55422a6060a90d7fd4ce26c172d/alembic-1.16.4-py3-none-any.whl", hash = "sha256:b05e51e8e82efc1abd14ba2af6392897e145930c3e0a2faf2b0da2f7f7fd660d", size = 247026, upload-time = "2025-07-10T16:17:21.845Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anyio" +version = "4.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "sniffio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/b4/636b3b65173d3ce9a38ef5f0522789614e590dab6a8d505340a4efe4c567/anyio-4.10.0.tar.gz", hash = "sha256:3f3fae35c96039744587aa5b8371e7e8e603c0702999535961dd336026973ba6", size = 213252, upload-time = "2025-08-04T08:54:26.451Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload-time = "2025-08-04T08:54:24.882Z" }, +] + +[[package]] +name = "appnope" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/5d/752690df9ef5b76e169e68d6a129fa6d08a7100ca7f754c89495db3c6019/appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", size = 4170, upload-time = "2024-02-06T09:43:11.258Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, +] + +[[package]] +name = "asttokens" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/e7/82da0a03e7ba5141f05cce0d302e6eed121ae055e0456ca228bf693984bc/asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7", size = 61978, upload-time = "2024-11-30T04:30:14.439Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2", size = 26918, upload-time = "2024-11-30T04:30:10.946Z" }, +] + +[[package]] +name = "attrs" +version = "25.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, +] + +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, +] + +[[package]] +name = "boto3" +version = "1.40.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/63/b263070ba4a2815de633d71dd4c5c04c9eb7000d33c510036c9557692324/boto3-1.40.9.tar.gz", hash = "sha256:af3f77a548b3dd7db5046609598a28a9ad5d062437b1783da9b526cc67c38b79", size = 111953, upload-time = "2025-08-13T19:20:32.495Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/6d/79fad38fcd7e1fc6961061b46cc87706c5c946088bc4620abf0d0aa49420/boto3-1.40.9-py3-none-any.whl", hash = "sha256:516f5e3f7552b2a7ca4d2c89b338fb4684998c676b11b906e2ab694c91716ba6", size = 140061, upload-time = "2025-08-13T19:20:30.652Z" }, +] + +[[package]] +name = "botocore" +version = "1.40.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/f3/7bf4913b4b61416c014cfee38211d071f75894cca37f7234519c4d8676d1/botocore-1.40.9.tar.gz", hash = "sha256:f4a9c6ed08e8637138e1b5534f89d38c02650974b6458a07690493130e295f68", size = 14325768, upload-time = "2025-08-13T19:20:22.393Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/e9/367e81e114deb92a6e0d5740f0bff4548af710be318af65265b9aad72237/botocore-1.40.9-py3-none-any.whl", hash = "sha256:d4960a39aab9658bcd0272490003001cb4a8d12b89bb297ccef994ee023fb638", size = 13990592, upload-time = "2025-08-13T19:20:16.942Z" }, +] + +[[package]] +name = "cachetools" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/81/3747dad6b14fa2cf53fcf10548cf5aea6913e96fab41a3c198676f8948a5/cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", size = 28380, upload-time = "2025-02-20T21:01:19.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080, upload-time = "2025-02-20T21:01:16.647Z" }, +] + +[[package]] +name = "certifi" +version = "2025.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, +] + +[[package]] +name = "cffi" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, + { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, + { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, + { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, + { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, + { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, + { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, + { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, + { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, + { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, + { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, + { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, + { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, + { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" }, + { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" }, + { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" }, + { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" }, + { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" }, + { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" }, + { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326, upload-time = "2025-08-09T07:56:24.721Z" }, + { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008, upload-time = "2025-08-09T07:56:26.004Z" }, + { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196, upload-time = "2025-08-09T07:56:27.25Z" }, + { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819, upload-time = "2025-08-09T07:56:28.515Z" }, + { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350, upload-time = "2025-08-09T07:56:29.716Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644, upload-time = "2025-08-09T07:56:30.984Z" }, + { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468, upload-time = "2025-08-09T07:56:32.252Z" }, + { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187, upload-time = "2025-08-09T07:56:33.481Z" }, + { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699, upload-time = "2025-08-09T07:56:34.739Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580, upload-time = "2025-08-09T07:56:35.981Z" }, + { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366, upload-time = "2025-08-09T07:56:37.339Z" }, + { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342, upload-time = "2025-08-09T07:56:38.687Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995, upload-time = "2025-08-09T07:56:40.048Z" }, + { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640, upload-time = "2025-08-09T07:56:41.311Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636, upload-time = "2025-08-09T07:56:43.195Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939, upload-time = "2025-08-09T07:56:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580, upload-time = "2025-08-09T07:56:46.684Z" }, + { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870, upload-time = "2025-08-09T07:56:47.941Z" }, + { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797, upload-time = "2025-08-09T07:56:49.756Z" }, + { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" }, +] + +[[package]] +name = "click" +version = "8.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/6c/8ca2efa64cf75a977a0d7fac081354553ebe483345c734fb6b6515d96bbc/click-8.2.1.tar.gz", hash = "sha256:27c491cc05d968d271d5a1db13e3b5a184636d9d930f148c50b038f0d0646202", size = 286342, upload-time = "2025-05-20T23:19:49.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/39/069100b84d7418bc358d81669d5748efb14b9cceacd2f9c75f550424132f/cloudpickle-3.1.1.tar.gz", hash = "sha256:b216fa8ae4019d5482a8ac3c95d8f6346115d8835911fd4aefd1a445e4242c64", size = 22113, upload-time = "2025-01-14T17:02:05.085Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/e8/64c37fadfc2816a7701fa8a6ed8d87327c7d54eacfbfb6edab14a2f2be75/cloudpickle-3.1.1-py3-none-any.whl", hash = "sha256:c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e", size = 20992, upload-time = "2025-01-14T17:02:02.417Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "colorful" +version = "0.5.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/0c/d180ebf230b771907f46981023a80f62cf592d49673cc5f8a5993aa67bb6/colorful-0.5.7.tar.gz", hash = "sha256:c5452179b56601c178b03d468a5326cc1fe37d9be81d24d0d6bdab36c4b93ad8", size = 209487, upload-time = "2025-06-30T15:24:03.936Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/98/0d791b3d1eaed89d7d370b5cf9b8079b124da0545559417f394ba21b5532/colorful-0.5.7-py2.py3-none-any.whl", hash = "sha256:495dd3a23151a9568cee8a90fc1174c902ad7ef06655f50b6bddf9e80008da69", size = 201475, upload-time = "2025-06-30T15:24:02.693Z" }, +] + +[[package]] +name = "comm" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/13/7d740c5849255756bc17888787313b61fd38a0a8304fc4f073dfc46122aa/comm-0.2.3.tar.gz", hash = "sha256:2dc8048c10962d55d7ad693be1e7045d891b7ce8d999c97963a5e3e99c055971", size = 6319, upload-time = "2025-07-25T14:02:04.452Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, +] + +[[package]] +name = "contourpy" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/45/adfee365d9ea3d853550b2e735f9d66366701c65db7855cd07621732ccfc/contourpy-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb", size = 293419, upload-time = "2025-07-26T12:01:21.16Z" }, + { url = "https://files.pythonhosted.org/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6", size = 273979, upload-time = "2025-07-26T12:01:22.448Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/a12359b9b2ca3a845e8f7f9ac08bdf776114eb931392fcad91743e2ea17b/contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7", size = 332653, upload-time = "2025-07-26T12:01:24.155Z" }, + { url = "https://files.pythonhosted.org/packages/63/12/897aeebfb475b7748ea67b61e045accdfcf0d971f8a588b67108ed7f5512/contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8", size = 379536, upload-time = "2025-07-26T12:01:25.91Z" }, + { url = "https://files.pythonhosted.org/packages/43/8a/a8c584b82deb248930ce069e71576fc09bd7174bbd35183b7943fb1064fd/contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea", size = 384397, upload-time = "2025-07-26T12:01:27.152Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1", size = 362601, upload-time = "2025-07-26T12:01:28.808Z" }, + { url = "https://files.pythonhosted.org/packages/05/0a/a3fe3be3ee2dceb3e615ebb4df97ae6f3828aa915d3e10549ce016302bd1/contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7", size = 1331288, upload-time = "2025-07-26T12:01:31.198Z" }, + { url = "https://files.pythonhosted.org/packages/33/1d/acad9bd4e97f13f3e2b18a3977fe1b4a37ecf3d38d815333980c6c72e963/contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411", size = 1403386, upload-time = "2025-07-26T12:01:33.947Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8f/5847f44a7fddf859704217a99a23a4f6417b10e5ab1256a179264561540e/contourpy-1.3.3-cp312-cp312-win32.whl", hash = "sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69", size = 185018, upload-time = "2025-07-26T12:01:35.64Z" }, + { url = "https://files.pythonhosted.org/packages/19/e8/6026ed58a64563186a9ee3f29f41261fd1828f527dd93d33b60feca63352/contourpy-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b", size = 226567, upload-time = "2025-07-26T12:01:36.804Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e2/f05240d2c39a1ed228d8328a78b6f44cd695f7ef47beb3e684cf93604f86/contourpy-1.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc", size = 193655, upload-time = "2025-07-26T12:01:37.999Z" }, + { url = "https://files.pythonhosted.org/packages/68/35/0167aad910bbdb9599272bd96d01a9ec6852f36b9455cf2ca67bd4cc2d23/contourpy-1.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:177fb367556747a686509d6fef71d221a4b198a3905fe824430e5ea0fda54eb5", size = 293257, upload-time = "2025-07-26T12:01:39.367Z" }, + { url = "https://files.pythonhosted.org/packages/96/e4/7adcd9c8362745b2210728f209bfbcf7d91ba868a2c5f40d8b58f54c509b/contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1", size = 274034, upload-time = "2025-07-26T12:01:40.645Z" }, + { url = "https://files.pythonhosted.org/packages/73/23/90e31ceeed1de63058a02cb04b12f2de4b40e3bef5e082a7c18d9c8ae281/contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286", size = 334672, upload-time = "2025-07-26T12:01:41.942Z" }, + { url = "https://files.pythonhosted.org/packages/ed/93/b43d8acbe67392e659e1d984700e79eb67e2acb2bd7f62012b583a7f1b55/contourpy-1.3.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5", size = 381234, upload-time = "2025-07-26T12:01:43.499Z" }, + { url = "https://files.pythonhosted.org/packages/46/3b/bec82a3ea06f66711520f75a40c8fc0b113b2a75edb36aa633eb11c4f50f/contourpy-1.3.3-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67", size = 385169, upload-time = "2025-07-26T12:01:45.219Z" }, + { url = "https://files.pythonhosted.org/packages/4b/32/e0f13a1c5b0f8572d0ec6ae2f6c677b7991fafd95da523159c19eff0696a/contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9", size = 362859, upload-time = "2025-07-26T12:01:46.519Z" }, + { url = "https://files.pythonhosted.org/packages/33/71/e2a7945b7de4e58af42d708a219f3b2f4cff7386e6b6ab0a0fa0033c49a9/contourpy-1.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659", size = 1332062, upload-time = "2025-07-26T12:01:48.964Z" }, + { url = "https://files.pythonhosted.org/packages/12/fc/4e87ac754220ccc0e807284f88e943d6d43b43843614f0a8afa469801db0/contourpy-1.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7", size = 1403932, upload-time = "2025-07-26T12:01:51.979Z" }, + { url = "https://files.pythonhosted.org/packages/a6/2e/adc197a37443f934594112222ac1aa7dc9a98faf9c3842884df9a9d8751d/contourpy-1.3.3-cp313-cp313-win32.whl", hash = "sha256:b20c7c9a3bf701366556e1b1984ed2d0cedf999903c51311417cf5f591d8c78d", size = 185024, upload-time = "2025-07-26T12:01:53.245Z" }, + { url = "https://files.pythonhosted.org/packages/18/0b/0098c214843213759692cc638fce7de5c289200a830e5035d1791d7a2338/contourpy-1.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:1cadd8b8969f060ba45ed7c1b714fe69185812ab43bd6b86a9123fe8f99c3263", size = 226578, upload-time = "2025-07-26T12:01:54.422Z" }, + { url = "https://files.pythonhosted.org/packages/8a/9a/2f6024a0c5995243cd63afdeb3651c984f0d2bc727fd98066d40e141ad73/contourpy-1.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:fd914713266421b7536de2bfa8181aa8c699432b6763a0ea64195ebe28bff6a9", size = 193524, upload-time = "2025-07-26T12:01:55.73Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b3/f8a1a86bd3298513f500e5b1f5fd92b69896449f6cab6a146a5d52715479/contourpy-1.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:88df9880d507169449d434c293467418b9f6cbe82edd19284aa0409e7fdb933d", size = 306730, upload-time = "2025-07-26T12:01:57.051Z" }, + { url = "https://files.pythonhosted.org/packages/3f/11/4780db94ae62fc0c2053909b65dc3246bd7cecfc4f8a20d957ad43aa4ad8/contourpy-1.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d06bb1f751ba5d417047db62bca3c8fde202b8c11fb50742ab3ab962c81e8216", size = 287897, upload-time = "2025-07-26T12:01:58.663Z" }, + { url = "https://files.pythonhosted.org/packages/ae/15/e59f5f3ffdd6f3d4daa3e47114c53daabcb18574a26c21f03dc9e4e42ff0/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae", size = 326751, upload-time = "2025-07-26T12:02:00.343Z" }, + { url = "https://files.pythonhosted.org/packages/0f/81/03b45cfad088e4770b1dcf72ea78d3802d04200009fb364d18a493857210/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20", size = 375486, upload-time = "2025-07-26T12:02:02.128Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ba/49923366492ffbdd4486e970d421b289a670ae8cf539c1ea9a09822b371a/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99", size = 388106, upload-time = "2025-07-26T12:02:03.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/52/5b00ea89525f8f143651f9f03a0df371d3cbd2fccd21ca9b768c7a6500c2/contourpy-1.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b", size = 352548, upload-time = "2025-07-26T12:02:05.165Z" }, + { url = "https://files.pythonhosted.org/packages/32/1d/a209ec1a3a3452d490f6b14dd92e72280c99ae3d1e73da74f8277d4ee08f/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a", size = 1322297, upload-time = "2025-07-26T12:02:07.379Z" }, + { url = "https://files.pythonhosted.org/packages/bc/9e/46f0e8ebdd884ca0e8877e46a3f4e633f6c9c8c4f3f6e72be3fe075994aa/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e", size = 1391023, upload-time = "2025-07-26T12:02:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/b9/70/f308384a3ae9cd2209e0849f33c913f658d3326900d0ff5d378d6a1422d2/contourpy-1.3.3-cp313-cp313t-win32.whl", hash = "sha256:283edd842a01e3dcd435b1c5116798d661378d83d36d337b8dde1d16a5fc9ba3", size = 196157, upload-time = "2025-07-26T12:02:11.488Z" }, + { url = "https://files.pythonhosted.org/packages/b2/dd/880f890a6663b84d9e34a6f88cded89d78f0091e0045a284427cb6b18521/contourpy-1.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:87acf5963fc2b34825e5b6b048f40e3635dd547f590b04d2ab317c2619ef7ae8", size = 240570, upload-time = "2025-07-26T12:02:12.754Z" }, + { url = "https://files.pythonhosted.org/packages/80/99/2adc7d8ffead633234817ef8e9a87115c8a11927a94478f6bb3d3f4d4f7d/contourpy-1.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:3c30273eb2a55024ff31ba7d052dde990d7d8e5450f4bbb6e913558b3d6c2301", size = 199713, upload-time = "2025-07-26T12:02:14.4Z" }, + { url = "https://files.pythonhosted.org/packages/72/8b/4546f3ab60f78c514ffb7d01a0bd743f90de36f0019d1be84d0a708a580a/contourpy-1.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fde6c716d51c04b1c25d0b90364d0be954624a0ee9d60e23e850e8d48353d07a", size = 292189, upload-time = "2025-07-26T12:02:16.095Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e1/3542a9cb596cadd76fcef413f19c79216e002623158befe6daa03dbfa88c/contourpy-1.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cbedb772ed74ff5be440fa8eee9bd49f64f6e3fc09436d9c7d8f1c287b121d77", size = 273251, upload-time = "2025-07-26T12:02:17.524Z" }, + { url = "https://files.pythonhosted.org/packages/b1/71/f93e1e9471d189f79d0ce2497007731c1e6bf9ef6d1d61b911430c3db4e5/contourpy-1.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5", size = 335810, upload-time = "2025-07-26T12:02:18.9Z" }, + { url = "https://files.pythonhosted.org/packages/91/f9/e35f4c1c93f9275d4e38681a80506b5510e9327350c51f8d4a5a724d178c/contourpy-1.3.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4", size = 382871, upload-time = "2025-07-26T12:02:20.418Z" }, + { url = "https://files.pythonhosted.org/packages/b5/71/47b512f936f66a0a900d81c396a7e60d73419868fba959c61efed7a8ab46/contourpy-1.3.3-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36", size = 386264, upload-time = "2025-07-26T12:02:21.916Z" }, + { url = "https://files.pythonhosted.org/packages/04/5f/9ff93450ba96b09c7c2b3f81c94de31c89f92292f1380261bd7195bea4ea/contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3", size = 363819, upload-time = "2025-07-26T12:02:23.759Z" }, + { url = "https://files.pythonhosted.org/packages/3e/a6/0b185d4cc480ee494945cde102cb0149ae830b5fa17bf855b95f2e70ad13/contourpy-1.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b", size = 1333650, upload-time = "2025-07-26T12:02:26.181Z" }, + { url = "https://files.pythonhosted.org/packages/43/d7/afdc95580ca56f30fbcd3060250f66cedbde69b4547028863abd8aa3b47e/contourpy-1.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36", size = 1404833, upload-time = "2025-07-26T12:02:28.782Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e2/366af18a6d386f41132a48f033cbd2102e9b0cf6345d35ff0826cd984566/contourpy-1.3.3-cp314-cp314-win32.whl", hash = "sha256:66c8a43a4f7b8df8b71ee1840e4211a3c8d93b214b213f590e18a1beca458f7d", size = 189692, upload-time = "2025-07-26T12:02:30.128Z" }, + { url = "https://files.pythonhosted.org/packages/7d/c2/57f54b03d0f22d4044b8afb9ca0e184f8b1afd57b4f735c2fa70883dc601/contourpy-1.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:cf9022ef053f2694e31d630feaacb21ea24224be1c3ad0520b13d844274614fd", size = 232424, upload-time = "2025-07-26T12:02:31.395Z" }, + { url = "https://files.pythonhosted.org/packages/18/79/a9416650df9b525737ab521aa181ccc42d56016d2123ddcb7b58e926a42c/contourpy-1.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:95b181891b4c71de4bb404c6621e7e2390745f887f2a026b2d99e92c17892339", size = 198300, upload-time = "2025-07-26T12:02:32.956Z" }, + { url = "https://files.pythonhosted.org/packages/1f/42/38c159a7d0f2b7b9c04c64ab317042bb6952b713ba875c1681529a2932fe/contourpy-1.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:33c82d0138c0a062380332c861387650c82e4cf1747aaa6938b9b6516762e772", size = 306769, upload-time = "2025-07-26T12:02:34.2Z" }, + { url = "https://files.pythonhosted.org/packages/c3/6c/26a8205f24bca10974e77460de68d3d7c63e282e23782f1239f226fcae6f/contourpy-1.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ea37e7b45949df430fe649e5de8351c423430046a2af20b1c1961cae3afcda77", size = 287892, upload-time = "2025-07-26T12:02:35.807Z" }, + { url = "https://files.pythonhosted.org/packages/66/06/8a475c8ab718ebfd7925661747dbb3c3ee9c82ac834ccb3570be49d129f4/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13", size = 326748, upload-time = "2025-07-26T12:02:37.193Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a3/c5ca9f010a44c223f098fccd8b158bb1cb287378a31ac141f04730dc49be/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe", size = 375554, upload-time = "2025-07-26T12:02:38.894Z" }, + { url = "https://files.pythonhosted.org/packages/80/5b/68bd33ae63fac658a4145088c1e894405e07584a316738710b636c6d0333/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f", size = 388118, upload-time = "2025-07-26T12:02:40.642Z" }, + { url = "https://files.pythonhosted.org/packages/40/52/4c285a6435940ae25d7410a6c36bda5145839bc3f0beb20c707cda18b9d2/contourpy-1.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0", size = 352555, upload-time = "2025-07-26T12:02:42.25Z" }, + { url = "https://files.pythonhosted.org/packages/24/ee/3e81e1dd174f5c7fefe50e85d0892de05ca4e26ef1c9a59c2a57e43b865a/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4", size = 1322295, upload-time = "2025-07-26T12:02:44.668Z" }, + { url = "https://files.pythonhosted.org/packages/3c/b2/6d913d4d04e14379de429057cd169e5e00f6c2af3bb13e1710bcbdb5da12/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f", size = 1391027, upload-time = "2025-07-26T12:02:47.09Z" }, + { url = "https://files.pythonhosted.org/packages/93/8a/68a4ec5c55a2971213d29a9374913f7e9f18581945a7a31d1a39b5d2dfe5/contourpy-1.3.3-cp314-cp314t-win32.whl", hash = "sha256:e74a9a0f5e3fff48fb5a7f2fd2b9b70a3fe014a67522f79b7cca4c0c7e43c9ae", size = 202428, upload-time = "2025-07-26T12:02:48.691Z" }, + { url = "https://files.pythonhosted.org/packages/fa/96/fd9f641ffedc4fa3ace923af73b9d07e869496c9cc7a459103e6e978992f/contourpy-1.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:13b68d6a62db8eafaebb8039218921399baf6e47bf85006fd8529f2a08ef33fc", size = 250331, upload-time = "2025-07-26T12:02:50.137Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8c/469afb6465b853afff216f9528ffda78a915ff880ed58813ba4faf4ba0b6/contourpy-1.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b7448cb5a725bb1e35ce88771b86fba35ef418952474492cf7c764059933ff8b", size = 203831, upload-time = "2025-07-26T12:02:51.449Z" }, +] + +[[package]] +name = "cycler" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, +] + +[[package]] +name = "databricks-sdk" +version = "0.63.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/af/f77be9ed096e7e83732e80d87997256d7bd7d9aff62fa0d51e4068ae786c/databricks_sdk-0.63.0.tar.gz", hash = "sha256:f141bc810b4145e93e628a0e159ea41806440aed0cd17adddd252d65d1968465", size = 732112, upload-time = "2025-08-13T09:00:08.722Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/fd/2bb9bdf083a6943630762108e3e4e24b83f0571dd39cfa4fd467adaa1921/databricks_sdk-0.63.0-py3-none-any.whl", hash = "sha256:3ea569dcd0a4395c17221a5da39db4da85c6fc91b5fc14546514e329451d5eb3", size = 688018, upload-time = "2025-08-13T09:00:07.071Z" }, +] + +[[package]] +name = "debugpy" +version = "1.8.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/d4/722d0bcc7986172ac2ef3c979ad56a1030e3afd44ced136d45f8142b1f4a/debugpy-1.8.16.tar.gz", hash = "sha256:31e69a1feb1cf6b51efbed3f6c9b0ef03bc46ff050679c4be7ea6d2e23540870", size = 1643809, upload-time = "2025-08-06T18:00:02.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/fb/0387c0e108d842c902801bc65ccc53e5b91d8c169702a9bbf4f7efcedf0c/debugpy-1.8.16-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:b202e2843e32e80b3b584bcebfe0e65e0392920dc70df11b2bfe1afcb7a085e4", size = 2511822, upload-time = "2025-08-06T18:00:18.526Z" }, + { url = "https://files.pythonhosted.org/packages/37/44/19e02745cae22bf96440141f94e15a69a1afaa3a64ddfc38004668fcdebf/debugpy-1.8.16-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64473c4a306ba11a99fe0bb14622ba4fbd943eb004847d9b69b107bde45aa9ea", size = 4230135, upload-time = "2025-08-06T18:00:19.997Z" }, + { url = "https://files.pythonhosted.org/packages/f3/0b/19b1ba5ee4412f303475a2c7ad5858efb99c90eae5ec627aa6275c439957/debugpy-1.8.16-cp312-cp312-win32.whl", hash = "sha256:833a61ed446426e38b0dd8be3e9d45ae285d424f5bf6cd5b2b559c8f12305508", size = 5281271, upload-time = "2025-08-06T18:00:21.281Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e0/bc62e2dc141de53bd03e2c7cb9d7011de2e65e8bdcdaa26703e4d28656ba/debugpy-1.8.16-cp312-cp312-win_amd64.whl", hash = "sha256:75f204684581e9ef3dc2f67687c3c8c183fde2d6675ab131d94084baf8084121", size = 5323149, upload-time = "2025-08-06T18:00:23.033Z" }, + { url = "https://files.pythonhosted.org/packages/62/66/607ab45cc79e60624df386e233ab64a6d8d39ea02e7f80e19c1d451345bb/debugpy-1.8.16-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:85df3adb1de5258dca910ae0bb185e48c98801ec15018a263a92bb06be1c8787", size = 2496157, upload-time = "2025-08-06T18:00:24.361Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a0/c95baae08a75bceabb79868d663a0736655e427ab9c81fb848da29edaeac/debugpy-1.8.16-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bee89e948bc236a5c43c4214ac62d28b29388453f5fd328d739035e205365f0b", size = 4222491, upload-time = "2025-08-06T18:00:25.806Z" }, + { url = "https://files.pythonhosted.org/packages/5b/2f/1c8db6ddd8a257c3cd2c46413b267f1d5fa3df910401c899513ce30392d6/debugpy-1.8.16-cp313-cp313-win32.whl", hash = "sha256:cf358066650439847ec5ff3dae1da98b5461ea5da0173d93d5e10f477c94609a", size = 5281126, upload-time = "2025-08-06T18:00:27.207Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ba/c3e154ab307366d6c5a9c1b68de04914e2ce7fa2f50d578311d8cc5074b2/debugpy-1.8.16-cp313-cp313-win_amd64.whl", hash = "sha256:b5aea1083f6f50023e8509399d7dc6535a351cc9f2e8827d1e093175e4d9fa4c", size = 5323094, upload-time = "2025-08-06T18:00:29.03Z" }, + { url = "https://files.pythonhosted.org/packages/52/57/ecc9ae29fa5b2d90107cd1d9bf8ed19aacb74b2264d986ae9d44fe9bdf87/debugpy-1.8.16-py2.py3-none-any.whl", hash = "sha256:19c9521962475b87da6f673514f7fd610328757ec993bf7ec0d8c96f9a325f9e", size = 5287700, upload-time = "2025-08-06T18:00:42.333Z" }, +] + +[[package]] +name = "decorator" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/fa/6d96a0978d19e17b68d634497769987b16c8f4cd0a7a05048bec693caa6b/decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360", size = 56711, upload-time = "2025-02-24T04:41:34.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" }, +] + +[[package]] +name = "default" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "boto3" }, + { name = "doggos" }, + { name = "ipykernel" }, + { name = "ipywidgets" }, + { name = "matplotlib" }, + { name = "mlflow" }, + { name = "ray", extra = ["data", "serve", "train", "tune"] }, + { name = "scikit-learn" }, + { name = "torch" }, + { name = "transformers" }, +] + +[package.metadata] +requires-dist = [ + { name = "boto3", specifier = ">=1.40.9" }, + { name = "doggos", directory = "doggos" }, + { name = "ipykernel", specifier = ">=6.30.1" }, + { name = "ipywidgets", specifier = "==8.1.3" }, + { name = "matplotlib", specifier = "==3.10.0" }, + { name = "mlflow", specifier = "==2.19.0" }, + { name = "ray", extras = ["data", "serve", "train", "tune"], url = "http://localhost:9478/ray/ray-2.48.0-cp312-cp312-manylinux2014_x86_64.whl" }, + { name = "scikit-learn", specifier = "==1.6.0" }, + { name = "torch", specifier = "==2.7.1" }, + { name = "transformers", specifier = "==4.52.3" }, +] + +[[package]] +name = "distlib" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, +] + +[[package]] +name = "docker" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, +] + +[[package]] +name = "doggos" +version = "0.1.0" +source = { directory = "doggos" } + +[[package]] +name = "executing" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/50/a9d80c47ff289c611ff12e63f7c5d13942c65d68125160cefd768c73e6e4/executing-2.2.0.tar.gz", hash = "sha256:5d108c028108fe2551d1a7b2e8b713341e2cb4fc0aa7dcf966fa4327a5226755", size = 978693, upload-time = "2025-01-22T15:41:29.403Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/8f/c4d9bafc34ad7ad5d8dc16dd1347ee0e507a52c3adb6bfa8887e1c6a26ba/executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa", size = 26702, upload-time = "2025-01-22T15:41:25.929Z" }, +] + +[[package]] +name = "fastapi" +version = "0.116.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/d7/6c8b3bfe33eeffa208183ec037fee0cce9f7f024089ab1c5d12ef04bd27c/fastapi-0.116.1.tar.gz", hash = "sha256:ed52cbf946abfd70c5a0dccb24673f0670deeb517a88b3544d03c2a6bf283143", size = 296485, upload-time = "2025-07-11T16:22:32.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/47/d63c60f59a59467fda0f93f46335c9d18526d7071f025cb5b89d5353ea42/fastapi-0.116.1-py3-none-any.whl", hash = "sha256:c46ac7c312df840f0c9e220f7964bada936781bc4e2e6eb71f1c4d7553786565", size = 95631, upload-time = "2025-07-11T16:22:30.485Z" }, +] + +[[package]] +name = "filelock" +version = "3.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", size = 17687, upload-time = "2025-08-14T16:56:03.016Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" }, +] + +[[package]] +name = "flask" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "blinker" }, + { name = "click" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "markupsafe" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/de/e47735752347f4128bcf354e0da07ef311a78244eba9e3dc1d4a5ab21a98/flask-3.1.1.tar.gz", hash = "sha256:284c7b8f2f58cb737f0cf1c30fd7eaf0ccfcde196099d24ecede3fc2005aa59e", size = 753440, upload-time = "2025-05-13T15:01:17.447Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/68/9d4508e893976286d2ead7f8f571314af6c2037af34853a30fd769c02e9d/flask-3.1.1-py3-none-any.whl", hash = "sha256:07aae2bb5eaf77993ef57e357491839f5fd9f4dc281593a81a9e4d79a24f295c", size = 103305, upload-time = "2025-05-13T15:01:15.591Z" }, +] + +[[package]] +name = "fonttools" +version = "4.59.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/7f/29c9c3fe4246f6ad96fee52b88d0dc3a863c7563b0afc959e36d78b965dc/fonttools-4.59.1.tar.gz", hash = "sha256:74995b402ad09822a4c8002438e54940d9f1ecda898d2bb057729d7da983e4cb", size = 3534394, upload-time = "2025-08-14T16:28:14.266Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/fe/6e069cc4cb8881d164a9bd956e9df555bc62d3eb36f6282e43440200009c/fonttools-4.59.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:43ab814bbba5f02a93a152ee61a04182bb5809bd2bc3609f7822e12c53ae2c91", size = 2769172, upload-time = "2025-08-14T16:26:45.729Z" }, + { url = "https://files.pythonhosted.org/packages/b9/98/ec4e03f748fefa0dd72d9d95235aff6fef16601267f4a2340f0e16b9330f/fonttools-4.59.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4f04c3ffbfa0baafcbc550657cf83657034eb63304d27b05cff1653b448ccff6", size = 2337281, upload-time = "2025-08-14T16:26:47.921Z" }, + { url = "https://files.pythonhosted.org/packages/8b/b1/890360a7e3d04a30ba50b267aca2783f4c1364363797e892e78a4f036076/fonttools-4.59.1-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d601b153e51a5a6221f0d4ec077b6bfc6ac35bfe6c19aeaa233d8990b2b71726", size = 4909215, upload-time = "2025-08-14T16:26:49.682Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ec/2490599550d6c9c97a44c1e36ef4de52d6acf742359eaa385735e30c05c4/fonttools-4.59.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c735e385e30278c54f43a0d056736942023c9043f84ee1021eff9fd616d17693", size = 4951958, upload-time = "2025-08-14T16:26:51.616Z" }, + { url = "https://files.pythonhosted.org/packages/d1/40/bd053f6f7634234a9b9805ff8ae4f32df4f2168bee23cafd1271ba9915a9/fonttools-4.59.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1017413cdc8555dce7ee23720da490282ab7ec1cf022af90a241f33f9a49afc4", size = 4894738, upload-time = "2025-08-14T16:26:53.836Z" }, + { url = "https://files.pythonhosted.org/packages/ac/a1/3cd12a010d288325a7cfcf298a84825f0f9c29b01dee1baba64edfe89257/fonttools-4.59.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5c6d8d773470a5107052874341ed3c487c16ecd179976d81afed89dea5cd7406", size = 5045983, upload-time = "2025-08-14T16:26:56.153Z" }, + { url = "https://files.pythonhosted.org/packages/a2/af/8a2c3f6619cc43cf87951405337cc8460d08a4e717bb05eaa94b335d11dc/fonttools-4.59.1-cp312-cp312-win32.whl", hash = "sha256:2a2d0d33307f6ad3a2086a95dd607c202ea8852fa9fb52af9b48811154d1428a", size = 2203407, upload-time = "2025-08-14T16:26:58.165Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f2/a19b874ddbd3ebcf11d7e25188ef9ac3f68b9219c62263acb34aca8cde05/fonttools-4.59.1-cp312-cp312-win_amd64.whl", hash = "sha256:0b9e4fa7eaf046ed6ac470f6033d52c052481ff7a6e0a92373d14f556f298dc0", size = 2251561, upload-time = "2025-08-14T16:27:00.646Z" }, + { url = "https://files.pythonhosted.org/packages/19/5e/94a4d7f36c36e82f6a81e0064d148542e0ad3e6cf51fc5461ca128f3658d/fonttools-4.59.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:89d9957b54246c6251345297dddf77a84d2c19df96af30d2de24093bbdf0528b", size = 2760192, upload-time = "2025-08-14T16:27:03.024Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a5/f50712fc33ef9d06953c660cefaf8c8fe4b8bc74fa21f44ee5e4f9739439/fonttools-4.59.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8156b11c0d5405810d216f53907bd0f8b982aa5f1e7e3127ab3be1a4062154ff", size = 2332694, upload-time = "2025-08-14T16:27:04.883Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a2/5a9fc21c354bf8613215ce233ab0d933bd17d5ff4c29693636551adbc7b3/fonttools-4.59.1-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8387876a8011caec52d327d5e5bca705d9399ec4b17afb8b431ec50d47c17d23", size = 4889254, upload-time = "2025-08-14T16:27:07.02Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e5/54a6dc811eba018d022ca2e8bd6f2969291f9586ccf9a22a05fc55f91250/fonttools-4.59.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb13823a74b3a9204a8ed76d3d6d5ec12e64cc5bc44914eb9ff1cdac04facd43", size = 4949109, upload-time = "2025-08-14T16:27:09.3Z" }, + { url = "https://files.pythonhosted.org/packages/db/15/b05c72a248a95bea0fd05fbd95acdf0742945942143fcf961343b7a3663a/fonttools-4.59.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e1ca10da138c300f768bb68e40e5b20b6ecfbd95f91aac4cc15010b6b9d65455", size = 4888428, upload-time = "2025-08-14T16:27:11.514Z" }, + { url = "https://files.pythonhosted.org/packages/63/71/c7d6840f858d695adc0c4371ec45e3fb1c8e060b276ba944e2800495aca4/fonttools-4.59.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2beb5bfc4887a3130f8625349605a3a45fe345655ce6031d1bac11017454b943", size = 5032668, upload-time = "2025-08-14T16:27:13.872Z" }, + { url = "https://files.pythonhosted.org/packages/90/54/57be4aca6f1312e2bc4d811200dd822325794e05bdb26eeff0976edca651/fonttools-4.59.1-cp313-cp313-win32.whl", hash = "sha256:419f16d750d78e6d704bfe97b48bba2f73b15c9418f817d0cb8a9ca87a5b94bf", size = 2201832, upload-time = "2025-08-14T16:27:16.126Z" }, + { url = "https://files.pythonhosted.org/packages/fc/1f/1899a6175a5f900ed8730a0d64f53ca1b596ed7609bfda033cf659114258/fonttools-4.59.1-cp313-cp313-win_amd64.whl", hash = "sha256:c536f8a852e8d3fa71dde1ec03892aee50be59f7154b533f0bf3c1174cfd5126", size = 2250673, upload-time = "2025-08-14T16:27:18.033Z" }, + { url = "https://files.pythonhosted.org/packages/15/07/f6ba82c22f118d9985c37fea65d8d715ca71300d78b6c6e90874dc59f11d/fonttools-4.59.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:d5c3bfdc9663f3d4b565f9cb3b8c1efb3e178186435b45105bde7328cfddd7fe", size = 2758606, upload-time = "2025-08-14T16:27:20.064Z" }, + { url = "https://files.pythonhosted.org/packages/3a/81/84aa3d0ce27b0112c28b67b637ff7a47cf401cf5fbfee6476e4bc9777580/fonttools-4.59.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ea03f1da0d722fe3c2278a05957e6550175571a4894fbf9d178ceef4a3783d2b", size = 2330187, upload-time = "2025-08-14T16:27:22.42Z" }, + { url = "https://files.pythonhosted.org/packages/17/41/b3ba43f78afb321e2e50232c87304c8d0f5ab39b64389b8286cc39cdb824/fonttools-4.59.1-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:57a3708ca6bfccb790f585fa6d8f29432ec329618a09ff94c16bcb3c55994643", size = 4832020, upload-time = "2025-08-14T16:27:24.214Z" }, + { url = "https://files.pythonhosted.org/packages/67/b1/3af871c7fb325a68938e7ce544ca48bfd2c6bb7b357f3c8252933b29100a/fonttools-4.59.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:729367c91eb1ee84e61a733acc485065a00590618ca31c438e7dd4d600c01486", size = 4930687, upload-time = "2025-08-14T16:27:26.484Z" }, + { url = "https://files.pythonhosted.org/packages/c5/4f/299fc44646b30d9ef03ffaa78b109c7bd32121f0d8f10009ee73ac4514bc/fonttools-4.59.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8f8ef66ac6db450193ed150e10b3b45dde7aded10c5d279968bc63368027f62b", size = 4875794, upload-time = "2025-08-14T16:27:28.887Z" }, + { url = "https://files.pythonhosted.org/packages/90/cf/a0a3d763ab58f5f81ceff104ddb662fd9da94248694862b9c6cbd509fdd5/fonttools-4.59.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:075f745d539a998cd92cb84c339a82e53e49114ec62aaea8307c80d3ad3aef3a", size = 4985780, upload-time = "2025-08-14T16:27:30.858Z" }, + { url = "https://files.pythonhosted.org/packages/72/c5/ba76511aaae143d89c29cd32ce30bafb61c477e8759a1590b8483f8065f8/fonttools-4.59.1-cp314-cp314-win32.whl", hash = "sha256:c2b0597522d4c5bb18aa5cf258746a2d4a90f25878cbe865e4d35526abd1b9fc", size = 2205610, upload-time = "2025-08-14T16:27:32.578Z" }, + { url = "https://files.pythonhosted.org/packages/a9/65/b250e69d6caf35bc65cddbf608be0662d741c248f2e7503ab01081fc267e/fonttools-4.59.1-cp314-cp314-win_amd64.whl", hash = "sha256:e9ad4ce044e3236f0814c906ccce8647046cc557539661e35211faadf76f283b", size = 2255376, upload-time = "2025-08-14T16:27:34.653Z" }, + { url = "https://files.pythonhosted.org/packages/11/f3/0bc63a23ac0f8175e23d82f85d6ee693fbd849de7ad739f0a3622182ad29/fonttools-4.59.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:652159e8214eb4856e8387ebcd6b6bd336ee258cbeb639c8be52005b122b9609", size = 2826546, upload-time = "2025-08-14T16:27:36.783Z" }, + { url = "https://files.pythonhosted.org/packages/e9/46/a3968205590e068fdf60e926be329a207782576cb584d3b7dcd2d2844957/fonttools-4.59.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:43d177cd0e847ea026fedd9f099dc917da136ed8792d142298a252836390c478", size = 2359771, upload-time = "2025-08-14T16:27:39.678Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ff/d14b4c283879e8cb57862d9624a34fe6522b6fcdd46ccbfc58900958794a/fonttools-4.59.1-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e54437651e1440ee53a95e6ceb6ee440b67a3d348c76f45f4f48de1a5ecab019", size = 4831575, upload-time = "2025-08-14T16:27:41.885Z" }, + { url = "https://files.pythonhosted.org/packages/9c/04/a277d9a584a49d98ca12d3b2c6663bdf333ae97aaa83bd0cdabf7c5a6c84/fonttools-4.59.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6065fdec8ff44c32a483fd44abe5bcdb40dd5e2571a5034b555348f2b3a52cea", size = 5069962, upload-time = "2025-08-14T16:27:44.284Z" }, + { url = "https://files.pythonhosted.org/packages/16/6f/3d2ae69d96c4cdee6dfe7598ca5519a1514487700ca3d7c49c5a1ad65308/fonttools-4.59.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42052b56d176f8b315fbc09259439c013c0cb2109df72447148aeda677599612", size = 4942926, upload-time = "2025-08-14T16:27:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d3/c17379e0048d03ce26b38e4ab0e9a98280395b00529e093fe2d663ac0658/fonttools-4.59.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:bcd52eaa5c4c593ae9f447c1d13e7e4a00ca21d755645efa660b6999425b3c88", size = 4958678, upload-time = "2025-08-14T16:27:48.555Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3f/c5543a1540abdfb4d375e3ebeb84de365ab9b153ec14cb7db05f537dd1e7/fonttools-4.59.1-cp314-cp314t-win32.whl", hash = "sha256:02e4fdf27c550dded10fe038a5981c29f81cb9bc649ff2eaa48e80dab8998f97", size = 2266706, upload-time = "2025-08-14T16:27:50.556Z" }, + { url = "https://files.pythonhosted.org/packages/3e/99/85bff6e674226bc8402f983e365f07e76d990e7220ba72bcc738fef52391/fonttools-4.59.1-cp314-cp314t-win_amd64.whl", hash = "sha256:412a5fd6345872a7c249dac5bcce380393f40c1c316ac07f447bc17d51900922", size = 2329994, upload-time = "2025-08-14T16:27:52.36Z" }, + { url = "https://files.pythonhosted.org/packages/0f/64/9d606e66d498917cd7a2ff24f558010d42d6fd4576d9dd57f0bd98333f5a/fonttools-4.59.1-py3-none-any.whl", hash = "sha256:647db657073672a8330608970a984d51573557f328030566521bc03415535042", size = 1130094, upload-time = "2025-08-14T16:28:12.048Z" }, +] + +[[package]] +name = "frozenlist" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/79/b1/b64018016eeb087db503b038296fd782586432b9c077fc5c7839e9cb6ef6/frozenlist-1.7.0.tar.gz", hash = "sha256:2e310d81923c2437ea8670467121cc3e9b0f76d3043cc1d2331d56c7fb7a3a8f", size = 45078, upload-time = "2025-06-09T23:02:35.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a2/c8131383f1e66adad5f6ecfcce383d584ca94055a34d683bbb24ac5f2f1c/frozenlist-1.7.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3dbf9952c4bb0e90e98aec1bd992b3318685005702656bc6f67c1a32b76787f2", size = 81424, upload-time = "2025-06-09T23:00:42.24Z" }, + { url = "https://files.pythonhosted.org/packages/4c/9d/02754159955088cb52567337d1113f945b9e444c4960771ea90eb73de8db/frozenlist-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1f5906d3359300b8a9bb194239491122e6cf1444c2efb88865426f170c262cdb", size = 47952, upload-time = "2025-06-09T23:00:43.481Z" }, + { url = "https://files.pythonhosted.org/packages/01/7a/0046ef1bd6699b40acd2067ed6d6670b4db2f425c56980fa21c982c2a9db/frozenlist-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3dabd5a8f84573c8d10d8859a50ea2dec01eea372031929871368c09fa103478", size = 46688, upload-time = "2025-06-09T23:00:44.793Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a2/a910bafe29c86997363fb4c02069df4ff0b5bc39d33c5198b4e9dd42d8f8/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa57daa5917f1738064f302bf2626281a1cb01920c32f711fbc7bc36111058a8", size = 243084, upload-time = "2025-06-09T23:00:46.125Z" }, + { url = "https://files.pythonhosted.org/packages/64/3e/5036af9d5031374c64c387469bfcc3af537fc0f5b1187d83a1cf6fab1639/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c193dda2b6d49f4c4398962810fa7d7c78f032bf45572b3e04dd5249dff27e08", size = 233524, upload-time = "2025-06-09T23:00:47.73Z" }, + { url = "https://files.pythonhosted.org/packages/06/39/6a17b7c107a2887e781a48ecf20ad20f1c39d94b2a548c83615b5b879f28/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe2b675cf0aaa6d61bf8fbffd3c274b3c9b7b1623beb3809df8a81399a4a9c4", size = 248493, upload-time = "2025-06-09T23:00:49.742Z" }, + { url = "https://files.pythonhosted.org/packages/be/00/711d1337c7327d88c44d91dd0f556a1c47fb99afc060ae0ef66b4d24793d/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fc5d5cda37f62b262405cf9652cf0856839c4be8ee41be0afe8858f17f4c94b", size = 244116, upload-time = "2025-06-09T23:00:51.352Z" }, + { url = "https://files.pythonhosted.org/packages/24/fe/74e6ec0639c115df13d5850e75722750adabdc7de24e37e05a40527ca539/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0d5ce521d1dd7d620198829b87ea002956e4319002ef0bc8d3e6d045cb4646e", size = 224557, upload-time = "2025-06-09T23:00:52.855Z" }, + { url = "https://files.pythonhosted.org/packages/8d/db/48421f62a6f77c553575201e89048e97198046b793f4a089c79a6e3268bd/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:488d0a7d6a0008ca0db273c542098a0fa9e7dfaa7e57f70acef43f32b3f69dca", size = 241820, upload-time = "2025-06-09T23:00:54.43Z" }, + { url = "https://files.pythonhosted.org/packages/1d/fa/cb4a76bea23047c8462976ea7b7a2bf53997a0ca171302deae9d6dd12096/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:15a7eaba63983d22c54d255b854e8108e7e5f3e89f647fc854bd77a237e767df", size = 236542, upload-time = "2025-06-09T23:00:56.409Z" }, + { url = "https://files.pythonhosted.org/packages/5d/32/476a4b5cfaa0ec94d3f808f193301debff2ea42288a099afe60757ef6282/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1eaa7e9c6d15df825bf255649e05bd8a74b04a4d2baa1ae46d9c2d00b2ca2cb5", size = 249350, upload-time = "2025-06-09T23:00:58.468Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ba/9a28042f84a6bf8ea5dbc81cfff8eaef18d78b2a1ad9d51c7bc5b029ad16/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4389e06714cfa9d47ab87f784a7c5be91d3934cd6e9a7b85beef808297cc025", size = 225093, upload-time = "2025-06-09T23:01:00.015Z" }, + { url = "https://files.pythonhosted.org/packages/bc/29/3a32959e68f9cf000b04e79ba574527c17e8842e38c91d68214a37455786/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:73bd45e1488c40b63fe5a7df892baf9e2a4d4bb6409a2b3b78ac1c6236178e01", size = 245482, upload-time = "2025-06-09T23:01:01.474Z" }, + { url = "https://files.pythonhosted.org/packages/80/e8/edf2f9e00da553f07f5fa165325cfc302dead715cab6ac8336a5f3d0adc2/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99886d98e1643269760e5fe0df31e5ae7050788dd288947f7f007209b8c33f08", size = 249590, upload-time = "2025-06-09T23:01:02.961Z" }, + { url = "https://files.pythonhosted.org/packages/1c/80/9a0eb48b944050f94cc51ee1c413eb14a39543cc4f760ed12657a5a3c45a/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:290a172aae5a4c278c6da8a96222e6337744cd9c77313efe33d5670b9f65fc43", size = 237785, upload-time = "2025-06-09T23:01:05.095Z" }, + { url = "https://files.pythonhosted.org/packages/f3/74/87601e0fb0369b7a2baf404ea921769c53b7ae00dee7dcfe5162c8c6dbf0/frozenlist-1.7.0-cp312-cp312-win32.whl", hash = "sha256:426c7bc70e07cfebc178bc4c2bf2d861d720c4fff172181eeb4a4c41d4ca2ad3", size = 39487, upload-time = "2025-06-09T23:01:06.54Z" }, + { url = "https://files.pythonhosted.org/packages/0b/15/c026e9a9fc17585a9d461f65d8593d281fedf55fbf7eb53f16c6df2392f9/frozenlist-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:563b72efe5da92e02eb68c59cb37205457c977aa7a449ed1b37e6939e5c47c6a", size = 43874, upload-time = "2025-06-09T23:01:07.752Z" }, + { url = "https://files.pythonhosted.org/packages/24/90/6b2cebdabdbd50367273c20ff6b57a3dfa89bd0762de02c3a1eb42cb6462/frozenlist-1.7.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee80eeda5e2a4e660651370ebffd1286542b67e268aa1ac8d6dbe973120ef7ee", size = 79791, upload-time = "2025-06-09T23:01:09.368Z" }, + { url = "https://files.pythonhosted.org/packages/83/2e/5b70b6a3325363293fe5fc3ae74cdcbc3e996c2a11dde2fd9f1fb0776d19/frozenlist-1.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d1a81c85417b914139e3a9b995d4a1c84559afc839a93cf2cb7f15e6e5f6ed2d", size = 47165, upload-time = "2025-06-09T23:01:10.653Z" }, + { url = "https://files.pythonhosted.org/packages/f4/25/a0895c99270ca6966110f4ad98e87e5662eab416a17e7fd53c364bf8b954/frozenlist-1.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cbb65198a9132ebc334f237d7b0df163e4de83fb4f2bdfe46c1e654bdb0c5d43", size = 45881, upload-time = "2025-06-09T23:01:12.296Z" }, + { url = "https://files.pythonhosted.org/packages/19/7c/71bb0bbe0832793c601fff68cd0cf6143753d0c667f9aec93d3c323f4b55/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dab46c723eeb2c255a64f9dc05b8dd601fde66d6b19cdb82b2e09cc6ff8d8b5d", size = 232409, upload-time = "2025-06-09T23:01:13.641Z" }, + { url = "https://files.pythonhosted.org/packages/c0/45/ed2798718910fe6eb3ba574082aaceff4528e6323f9a8570be0f7028d8e9/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6aeac207a759d0dedd2e40745575ae32ab30926ff4fa49b1635def65806fddee", size = 225132, upload-time = "2025-06-09T23:01:15.264Z" }, + { url = "https://files.pythonhosted.org/packages/ba/e2/8417ae0f8eacb1d071d4950f32f229aa6bf68ab69aab797b72a07ea68d4f/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bd8c4e58ad14b4fa7802b8be49d47993182fdd4023393899632c88fd8cd994eb", size = 237638, upload-time = "2025-06-09T23:01:16.752Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b7/2ace5450ce85f2af05a871b8c8719b341294775a0a6c5585d5e6170f2ce7/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04fb24d104f425da3540ed83cbfc31388a586a7696142004c577fa61c6298c3f", size = 233539, upload-time = "2025-06-09T23:01:18.202Z" }, + { url = "https://files.pythonhosted.org/packages/46/b9/6989292c5539553dba63f3c83dc4598186ab2888f67c0dc1d917e6887db6/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a5c505156368e4ea6b53b5ac23c92d7edc864537ff911d2fb24c140bb175e60", size = 215646, upload-time = "2025-06-09T23:01:19.649Z" }, + { url = "https://files.pythonhosted.org/packages/72/31/bc8c5c99c7818293458fe745dab4fd5730ff49697ccc82b554eb69f16a24/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bd7eb96a675f18aa5c553eb7ddc24a43c8c18f22e1f9925528128c052cdbe00", size = 232233, upload-time = "2025-06-09T23:01:21.175Z" }, + { url = "https://files.pythonhosted.org/packages/59/52/460db4d7ba0811b9ccb85af996019f5d70831f2f5f255f7cc61f86199795/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:05579bf020096fe05a764f1f84cd104a12f78eaab68842d036772dc6d4870b4b", size = 227996, upload-time = "2025-06-09T23:01:23.098Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c9/f4b39e904c03927b7ecf891804fd3b4df3db29b9e487c6418e37988d6e9d/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:376b6222d114e97eeec13d46c486facd41d4f43bab626b7c3f6a8b4e81a5192c", size = 242280, upload-time = "2025-06-09T23:01:24.808Z" }, + { url = "https://files.pythonhosted.org/packages/b8/33/3f8d6ced42f162d743e3517781566b8481322be321b486d9d262adf70bfb/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0aa7e176ebe115379b5b1c95b4096fb1c17cce0847402e227e712c27bdb5a949", size = 217717, upload-time = "2025-06-09T23:01:26.28Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e8/ad683e75da6ccef50d0ab0c2b2324b32f84fc88ceee778ed79b8e2d2fe2e/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3fbba20e662b9c2130dc771e332a99eff5da078b2b2648153a40669a6d0e36ca", size = 236644, upload-time = "2025-06-09T23:01:27.887Z" }, + { url = "https://files.pythonhosted.org/packages/b2/14/8d19ccdd3799310722195a72ac94ddc677541fb4bef4091d8e7775752360/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f3f4410a0a601d349dd406b5713fec59b4cee7e71678d5b17edda7f4655a940b", size = 238879, upload-time = "2025-06-09T23:01:29.524Z" }, + { url = "https://files.pythonhosted.org/packages/ce/13/c12bf657494c2fd1079a48b2db49fa4196325909249a52d8f09bc9123fd7/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e2cdfaaec6a2f9327bf43c933c0319a7c429058e8537c508964a133dffee412e", size = 232502, upload-time = "2025-06-09T23:01:31.287Z" }, + { url = "https://files.pythonhosted.org/packages/d7/8b/e7f9dfde869825489382bc0d512c15e96d3964180c9499efcec72e85db7e/frozenlist-1.7.0-cp313-cp313-win32.whl", hash = "sha256:5fc4df05a6591c7768459caba1b342d9ec23fa16195e744939ba5914596ae3e1", size = 39169, upload-time = "2025-06-09T23:01:35.503Z" }, + { url = "https://files.pythonhosted.org/packages/35/89/a487a98d94205d85745080a37860ff5744b9820a2c9acbcdd9440bfddf98/frozenlist-1.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:52109052b9791a3e6b5d1b65f4b909703984b770694d3eb64fad124c835d7cba", size = 43219, upload-time = "2025-06-09T23:01:36.784Z" }, + { url = "https://files.pythonhosted.org/packages/56/d5/5c4cf2319a49eddd9dd7145e66c4866bdc6f3dbc67ca3d59685149c11e0d/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a6f86e4193bb0e235ef6ce3dde5cbabed887e0b11f516ce8a0f4d3b33078ec2d", size = 84345, upload-time = "2025-06-09T23:01:38.295Z" }, + { url = "https://files.pythonhosted.org/packages/a4/7d/ec2c1e1dc16b85bc9d526009961953df9cec8481b6886debb36ec9107799/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:82d664628865abeb32d90ae497fb93df398a69bb3434463d172b80fc25b0dd7d", size = 48880, upload-time = "2025-06-09T23:01:39.887Z" }, + { url = "https://files.pythonhosted.org/packages/69/86/f9596807b03de126e11e7d42ac91e3d0b19a6599c714a1989a4e85eeefc4/frozenlist-1.7.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:912a7e8375a1c9a68325a902f3953191b7b292aa3c3fb0d71a216221deca460b", size = 48498, upload-time = "2025-06-09T23:01:41.318Z" }, + { url = "https://files.pythonhosted.org/packages/5e/cb/df6de220f5036001005f2d726b789b2c0b65f2363b104bbc16f5be8084f8/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9537c2777167488d539bc5de2ad262efc44388230e5118868e172dd4a552b146", size = 292296, upload-time = "2025-06-09T23:01:42.685Z" }, + { url = "https://files.pythonhosted.org/packages/83/1f/de84c642f17c8f851a2905cee2dae401e5e0daca9b5ef121e120e19aa825/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f34560fb1b4c3e30ba35fa9a13894ba39e5acfc5f60f57d8accde65f46cc5e74", size = 273103, upload-time = "2025-06-09T23:01:44.166Z" }, + { url = "https://files.pythonhosted.org/packages/88/3c/c840bfa474ba3fa13c772b93070893c6e9d5c0350885760376cbe3b6c1b3/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:acd03d224b0175f5a850edc104ac19040d35419eddad04e7cf2d5986d98427f1", size = 292869, upload-time = "2025-06-09T23:01:45.681Z" }, + { url = "https://files.pythonhosted.org/packages/a6/1c/3efa6e7d5a39a1d5ef0abeb51c48fb657765794a46cf124e5aca2c7a592c/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2038310bc582f3d6a09b3816ab01737d60bf7b1ec70f5356b09e84fb7408ab1", size = 291467, upload-time = "2025-06-09T23:01:47.234Z" }, + { url = "https://files.pythonhosted.org/packages/4f/00/d5c5e09d4922c395e2f2f6b79b9a20dab4b67daaf78ab92e7729341f61f6/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8c05e4c8e5f36e5e088caa1bf78a687528f83c043706640a92cb76cd6999384", size = 266028, upload-time = "2025-06-09T23:01:48.819Z" }, + { url = "https://files.pythonhosted.org/packages/4e/27/72765be905619dfde25a7f33813ac0341eb6b076abede17a2e3fbfade0cb/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:765bb588c86e47d0b68f23c1bee323d4b703218037765dcf3f25c838c6fecceb", size = 284294, upload-time = "2025-06-09T23:01:50.394Z" }, + { url = "https://files.pythonhosted.org/packages/88/67/c94103a23001b17808eb7dd1200c156bb69fb68e63fcf0693dde4cd6228c/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:32dc2e08c67d86d0969714dd484fd60ff08ff81d1a1e40a77dd34a387e6ebc0c", size = 281898, upload-time = "2025-06-09T23:01:52.234Z" }, + { url = "https://files.pythonhosted.org/packages/42/34/a3e2c00c00f9e2a9db5653bca3fec306349e71aff14ae45ecc6d0951dd24/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:c0303e597eb5a5321b4de9c68e9845ac8f290d2ab3f3e2c864437d3c5a30cd65", size = 290465, upload-time = "2025-06-09T23:01:53.788Z" }, + { url = "https://files.pythonhosted.org/packages/bb/73/f89b7fbce8b0b0c095d82b008afd0590f71ccb3dee6eee41791cf8cd25fd/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a47f2abb4e29b3a8d0b530f7c3598badc6b134562b1a5caee867f7c62fee51e3", size = 266385, upload-time = "2025-06-09T23:01:55.769Z" }, + { url = "https://files.pythonhosted.org/packages/cd/45/e365fdb554159462ca12df54bc59bfa7a9a273ecc21e99e72e597564d1ae/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:3d688126c242a6fabbd92e02633414d40f50bb6002fa4cf995a1d18051525657", size = 288771, upload-time = "2025-06-09T23:01:57.4Z" }, + { url = "https://files.pythonhosted.org/packages/00/11/47b6117002a0e904f004d70ec5194fe9144f117c33c851e3d51c765962d0/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4e7e9652b3d367c7bd449a727dc79d5043f48b88d0cbfd4f9f1060cf2b414104", size = 288206, upload-time = "2025-06-09T23:01:58.936Z" }, + { url = "https://files.pythonhosted.org/packages/40/37/5f9f3c3fd7f7746082ec67bcdc204db72dad081f4f83a503d33220a92973/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1a85e345b4c43db8b842cab1feb41be5cc0b10a1830e6295b69d7310f99becaf", size = 282620, upload-time = "2025-06-09T23:02:00.493Z" }, + { url = "https://files.pythonhosted.org/packages/0b/31/8fbc5af2d183bff20f21aa743b4088eac4445d2bb1cdece449ae80e4e2d1/frozenlist-1.7.0-cp313-cp313t-win32.whl", hash = "sha256:3a14027124ddb70dfcee5148979998066897e79f89f64b13328595c4bdf77c81", size = 43059, upload-time = "2025-06-09T23:02:02.072Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ed/41956f52105b8dbc26e457c5705340c67c8cc2b79f394b79bffc09d0e938/frozenlist-1.7.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3bf8010d71d4507775f658e9823210b7427be36625b387221642725b515dcf3e", size = 47516, upload-time = "2025-06-09T23:02:03.779Z" }, + { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" }, +] + +[[package]] +name = "fsspec" +version = "2025.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/02/0835e6ab9cfc03916fe3f78c0956cfcdb6ff2669ffa6651065d5ebf7fc98/fsspec-2025.7.0.tar.gz", hash = "sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58", size = 304432, upload-time = "2025-07-15T16:05:21.19Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/e0/014d5d9d7a4564cf1c40b5039bc882db69fd881111e03ab3657ac0b218e2/fsspec-2025.7.0-py3-none-any.whl", hash = "sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21", size = 199597, upload-time = "2025-07-15T16:05:19.529Z" }, +] + +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, +] + +[[package]] +name = "gitpython" +version = "3.1.45" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/c8/dd58967d119baab745caec2f9d853297cec1989ec1d63f677d3880632b88/gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c", size = 215076, upload-time = "2025-07-24T03:45:54.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/61/d4b89fec821f72385526e1b9d9a3a0385dda4a72b206d28049e2c7cd39b8/gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77", size = 208168, upload-time = "2025-07-24T03:45:52.517Z" }, +] + +[[package]] +name = "google-api-core" +version = "2.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/21/e9d043e88222317afdbdb567165fdbc3b0aad90064c7e0c9eb0ad9955ad8/google_api_core-2.25.1.tar.gz", hash = "sha256:d2aaa0b13c78c61cb3f4282c464c046e45fbd75755683c9c525e6e8f7ed0a5e8", size = 165443, upload-time = "2025-06-12T20:52:20.439Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/4b/ead00905132820b623732b175d66354e9d3e69fcf2a5dcdab780664e7896/google_api_core-2.25.1-py3-none-any.whl", hash = "sha256:8a2a56c1fef82987a524371f99f3bd0143702fecc670c72e600c1cda6bf8dbb7", size = 160807, upload-time = "2025-06-12T20:52:19.334Z" }, +] + +[[package]] +name = "google-auth" +version = "2.40.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "pyasn1-modules" }, + { name = "rsa" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/9b/e92ef23b84fa10a64ce4831390b7a4c2e53c0132568d99d4ae61d04c8855/google_auth-2.40.3.tar.gz", hash = "sha256:500c3a29adedeb36ea9cf24b8d10858e152f2412e3ca37829b3fa18e33d63b77", size = 281029, upload-time = "2025-06-04T18:04:57.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/63/b19553b658a1692443c62bd07e5868adaa0ad746a0751ba62c59568cd45b/google_auth-2.40.3-py2.py3-none-any.whl", hash = "sha256:1370d4593e86213563547f97a92752fc658456fe4514c809544f330fed45a7ca", size = 216137, upload-time = "2025-06-04T18:04:55.573Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/24/33db22342cf4a2ea27c9955e6713140fedd51e8b141b5ce5260897020f1a/googleapis_common_protos-1.70.0.tar.gz", hash = "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257", size = 145903, upload-time = "2025-04-14T10:17:02.924Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" }, +] + +[[package]] +name = "graphene" +version = "3.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "graphql-core" }, + { name = "graphql-relay" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/f6/bf62ff950c317ed03e77f3f6ddd7e34aaa98fe89d79ebd660c55343d8054/graphene-3.4.3.tar.gz", hash = "sha256:2a3786948ce75fe7e078443d37f609cbe5bb36ad8d6b828740ad3b95ed1a0aaa", size = 44739, upload-time = "2024-11-09T20:44:25.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/e0/61d8e98007182e6b2aca7cf65904721fb2e4bce0192272ab9cb6f69d8812/graphene-3.4.3-py2.py3-none-any.whl", hash = "sha256:820db6289754c181007a150db1f7fff544b94142b556d12e3ebc777a7bf36c71", size = 114894, upload-time = "2024-11-09T20:44:23.851Z" }, +] + +[[package]] +name = "graphql-core" +version = "3.2.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c4/16/7574029da84834349b60ed71614d66ca3afe46e9bf9c7b9562102acb7d4f/graphql_core-3.2.6.tar.gz", hash = "sha256:c08eec22f9e40f0bd61d805907e3b3b1b9a320bc606e23dc145eebca07c8fbab", size = 505353, upload-time = "2025-01-26T16:36:27.374Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/4f/7297663840621022bc73c22d7d9d80dbc78b4db6297f764b545cd5dd462d/graphql_core-3.2.6-py3-none-any.whl", hash = "sha256:78b016718c161a6fb20a7d97bbf107f331cd1afe53e45566c59f776ed7f0b45f", size = 203416, upload-time = "2025-01-26T16:36:24.868Z" }, +] + +[[package]] +name = "graphql-relay" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "graphql-core" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/13/98fbf8d67552f102488ffc16c6f559ce71ea15f6294728d33928ab5ff14d/graphql-relay-3.2.0.tar.gz", hash = "sha256:1ff1c51298356e481a0be009ccdff249832ce53f30559c1338f22a0e0d17250c", size = 50027, upload-time = "2022-04-16T11:03:45.447Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/16/a4cf06adbc711bd364a73ce043b0b08d8fa5aae3df11b6ee4248bcdad2e0/graphql_relay-3.2.0-py3-none-any.whl", hash = "sha256:c9b22bd28b170ba1fe674c74384a8ff30a76c8e26f88ac3aa1584dd3179953e5", size = 16940, upload-time = "2022-04-16T11:03:43.895Z" }, +] + +[[package]] +name = "greenlet" +version = "3.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/b8/704d753a5a45507a7aab61f18db9509302ed3d0a27ac7e0359ec2905b1a6/greenlet-3.2.4.tar.gz", hash = "sha256:0dca0d95ff849f9a364385f36ab49f50065d76964944638be9691e1832e9f86d", size = 188260, upload-time = "2025-08-07T13:24:33.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, + { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, + { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, + { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, + { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, + { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, + { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, + { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, + { url = "https://files.pythonhosted.org/packages/49/e8/58c7f85958bda41dafea50497cbd59738c5c43dbbea5ee83d651234398f4/greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31", size = 272814, upload-time = "2025-08-07T13:15:50.011Z" }, + { url = "https://files.pythonhosted.org/packages/62/dd/b9f59862e9e257a16e4e610480cfffd29e3fae018a68c2332090b53aac3d/greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945", size = 641073, upload-time = "2025-08-07T13:42:57.23Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0b/bc13f787394920b23073ca3b6c4a7a21396301ed75a655bcb47196b50e6e/greenlet-3.2.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:710638eb93b1fa52823aa91bf75326f9ecdfd5e0466f00789246a5280f4ba0fc", size = 655191, upload-time = "2025-08-07T13:45:29.752Z" }, + { url = "https://files.pythonhosted.org/packages/f2/d6/6adde57d1345a8d0f14d31e4ab9c23cfe8e2cd39c3baf7674b4b0338d266/greenlet-3.2.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c5111ccdc9c88f423426df3fd1811bfc40ed66264d35aa373420a34377efc98a", size = 649516, upload-time = "2025-08-07T13:53:16.314Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3b/3a3328a788d4a473889a2d403199932be55b1b0060f4ddd96ee7cdfcad10/greenlet-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76383238584e9711e20ebe14db6c88ddcedc1829a9ad31a584389463b5aa504", size = 652169, upload-time = "2025-08-07T13:18:32.861Z" }, + { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, + { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/a2/15/0d5e4e1a66fab130d98168fe984c509249c833c1a3c16806b90f253ce7b9/greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae", size = 1149210, upload-time = "2025-08-07T13:18:24.072Z" }, + { url = "https://files.pythonhosted.org/packages/0b/55/2321e43595e6801e105fcfdee02b34c0f996eb71e6ddffca6b10b7e1d771/greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b", size = 299685, upload-time = "2025-08-07T13:24:38.824Z" }, + { url = "https://files.pythonhosted.org/packages/22/5c/85273fd7cc388285632b0498dbbab97596e04b154933dfe0f3e68156c68c/greenlet-3.2.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:49a30d5fda2507ae77be16479bdb62a660fa51b1eb4928b524975b3bde77b3c0", size = 273586, upload-time = "2025-08-07T13:16:08.004Z" }, + { url = "https://files.pythonhosted.org/packages/d1/75/10aeeaa3da9332c2e761e4c50d4c3556c21113ee3f0afa2cf5769946f7a3/greenlet-3.2.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:299fd615cd8fc86267b47597123e3f43ad79c9d8a22bebdce535e53550763e2f", size = 686346, upload-time = "2025-08-07T13:42:59.944Z" }, + { url = "https://files.pythonhosted.org/packages/c0/aa/687d6b12ffb505a4447567d1f3abea23bd20e73a5bed63871178e0831b7a/greenlet-3.2.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c17b6b34111ea72fc5a4e4beec9711d2226285f0386ea83477cbb97c30a3f3a5", size = 699218, upload-time = "2025-08-07T13:45:30.969Z" }, + { url = "https://files.pythonhosted.org/packages/dc/8b/29aae55436521f1d6f8ff4e12fb676f3400de7fcf27fccd1d4d17fd8fecd/greenlet-3.2.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b4a1870c51720687af7fa3e7cda6d08d801dae660f75a76f3845b642b4da6ee1", size = 694659, upload-time = "2025-08-07T13:53:17.759Z" }, + { url = "https://files.pythonhosted.org/packages/92/2e/ea25914b1ebfde93b6fc4ff46d6864564fba59024e928bdc7de475affc25/greenlet-3.2.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:061dc4cf2c34852b052a8620d40f36324554bc192be474b9e9770e8c042fd735", size = 695355, upload-time = "2025-08-07T13:18:34.517Z" }, + { url = "https://files.pythonhosted.org/packages/72/60/fc56c62046ec17f6b0d3060564562c64c862948c9d4bc8aa807cf5bd74f4/greenlet-3.2.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44358b9bf66c8576a9f57a590d5f5d6e72fa4228b763d0e43fee6d3b06d3a337", size = 657512, upload-time = "2025-08-07T13:18:33.969Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a5/6ddab2b4c112be95601c13428db1d8b6608a8b6039816f2ba09c346c08fc/greenlet-3.2.4-cp314-cp314-win_amd64.whl", hash = "sha256:e37ab26028f12dbb0ff65f29a8d3d44a765c61e729647bf2ddfbbed621726f01", size = 303425, upload-time = "2025-08-07T13:32:27.59Z" }, +] + +[[package]] +name = "grpcio" +version = "1.74.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/b4/35feb8f7cab7239c5b94bd2db71abb3d6adb5f335ad8f131abb6060840b6/grpcio-1.74.0.tar.gz", hash = "sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1", size = 12756048, upload-time = "2025-07-24T18:54:23.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/5d/e504d5d5c4469823504f65687d6c8fb97b7f7bf0b34873b7598f1df24630/grpcio-1.74.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8", size = 5445551, upload-time = "2025-07-24T18:53:23.641Z" }, + { url = "https://files.pythonhosted.org/packages/43/01/730e37056f96f2f6ce9f17999af1556df62ee8dab7fa48bceeaab5fd3008/grpcio-1.74.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6", size = 10979810, upload-time = "2025-07-24T18:53:25.349Z" }, + { url = "https://files.pythonhosted.org/packages/79/3d/09fd100473ea5c47083889ca47ffd356576173ec134312f6aa0e13111dee/grpcio-1.74.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5", size = 5941946, upload-time = "2025-07-24T18:53:27.387Z" }, + { url = "https://files.pythonhosted.org/packages/8a/99/12d2cca0a63c874c6d3d195629dcd85cdf5d6f98a30d8db44271f8a97b93/grpcio-1.74.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49", size = 6621763, upload-time = "2025-07-24T18:53:29.193Z" }, + { url = "https://files.pythonhosted.org/packages/9d/2c/930b0e7a2f1029bbc193443c7bc4dc2a46fedb0203c8793dcd97081f1520/grpcio-1.74.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7", size = 6180664, upload-time = "2025-07-24T18:53:30.823Z" }, + { url = "https://files.pythonhosted.org/packages/db/d5/ff8a2442180ad0867717e670f5ec42bfd8d38b92158ad6bcd864e6d4b1ed/grpcio-1.74.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3", size = 6301083, upload-time = "2025-07-24T18:53:32.454Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ba/b361d390451a37ca118e4ec7dccec690422e05bc85fba2ec72b06cefec9f/grpcio-1.74.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707", size = 6994132, upload-time = "2025-07-24T18:53:34.506Z" }, + { url = "https://files.pythonhosted.org/packages/3b/0c/3a5fa47d2437a44ced74141795ac0251bbddeae74bf81df3447edd767d27/grpcio-1.74.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b", size = 6489616, upload-time = "2025-07-24T18:53:36.217Z" }, + { url = "https://files.pythonhosted.org/packages/ae/95/ab64703b436d99dc5217228babc76047d60e9ad14df129e307b5fec81fd0/grpcio-1.74.0-cp312-cp312-win32.whl", hash = "sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c", size = 3807083, upload-time = "2025-07-24T18:53:37.911Z" }, + { url = "https://files.pythonhosted.org/packages/84/59/900aa2445891fc47a33f7d2f76e00ca5d6ae6584b20d19af9c06fa09bf9a/grpcio-1.74.0-cp312-cp312-win_amd64.whl", hash = "sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc", size = 4490123, upload-time = "2025-07-24T18:53:39.528Z" }, + { url = "https://files.pythonhosted.org/packages/d4/d8/1004a5f468715221450e66b051c839c2ce9a985aa3ee427422061fcbb6aa/grpcio-1.74.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89", size = 5449488, upload-time = "2025-07-24T18:53:41.174Z" }, + { url = "https://files.pythonhosted.org/packages/94/0e/33731a03f63740d7743dced423846c831d8e6da808fcd02821a4416df7fa/grpcio-1.74.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01", size = 10974059, upload-time = "2025-07-24T18:53:43.066Z" }, + { url = "https://files.pythonhosted.org/packages/0d/c6/3d2c14d87771a421205bdca991467cfe473ee4c6a1231c1ede5248c62ab8/grpcio-1.74.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e", size = 5945647, upload-time = "2025-07-24T18:53:45.269Z" }, + { url = "https://files.pythonhosted.org/packages/c5/83/5a354c8aaff58594eef7fffebae41a0f8995a6258bbc6809b800c33d4c13/grpcio-1.74.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91", size = 6626101, upload-time = "2025-07-24T18:53:47.015Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ca/4fdc7bf59bf6994aa45cbd4ef1055cd65e2884de6113dbd49f75498ddb08/grpcio-1.74.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249", size = 6182562, upload-time = "2025-07-24T18:53:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/fd/48/2869e5b2c1922583686f7ae674937986807c2f676d08be70d0a541316270/grpcio-1.74.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362", size = 6303425, upload-time = "2025-07-24T18:53:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/a6/0e/bac93147b9a164f759497bc6913e74af1cb632c733c7af62c0336782bd38/grpcio-1.74.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f", size = 6996533, upload-time = "2025-07-24T18:53:52.747Z" }, + { url = "https://files.pythonhosted.org/packages/84/35/9f6b2503c1fd86d068b46818bbd7329db26a87cdd8c01e0d1a9abea1104c/grpcio-1.74.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20", size = 6491489, upload-time = "2025-07-24T18:53:55.06Z" }, + { url = "https://files.pythonhosted.org/packages/75/33/a04e99be2a82c4cbc4039eb3a76f6c3632932b9d5d295221389d10ac9ca7/grpcio-1.74.0-cp313-cp313-win32.whl", hash = "sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa", size = 3805811, upload-time = "2025-07-24T18:53:56.798Z" }, + { url = "https://files.pythonhosted.org/packages/34/80/de3eb55eb581815342d097214bed4c59e806b05f1b3110df03b2280d6dfd/grpcio-1.74.0-cp313-cp313-win_amd64.whl", hash = "sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24", size = 4489214, upload-time = "2025-07-24T18:53:59.771Z" }, +] + +[[package]] +name = "gunicorn" +version = "23.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging", marker = "sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031, upload-time = "2024-08-10T20:25:27.378Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload-time = "2024-08-10T20:25:24.996Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.1.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/0a/a0f56735940fde6dd627602fec9ab3bad23f66a272397560abd65aba416e/hf_xet-1.1.7.tar.gz", hash = "sha256:20cec8db4561338824a3b5f8c19774055b04a8df7fff0cb1ff2cb1a0c1607b80", size = 477719, upload-time = "2025-08-06T00:30:55.741Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/7c/8d7803995caf14e7d19a392a486a040f923e2cfeff824e9b800b92072f76/hf_xet-1.1.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:60dae4b44d520819e54e216a2505685248ec0adbdb2dd4848b17aa85a0375cde", size = 2761743, upload-time = "2025-08-06T00:30:50.634Z" }, + { url = "https://files.pythonhosted.org/packages/51/a3/fa5897099454aa287022a34a30e68dbff0e617760f774f8bd1db17f06bd4/hf_xet-1.1.7-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:b109f4c11e01c057fc82004c9e51e6cdfe2cb230637644ade40c599739067b2e", size = 2624331, upload-time = "2025-08-06T00:30:49.212Z" }, + { url = "https://files.pythonhosted.org/packages/86/50/2446a132267e60b8a48b2e5835d6e24fd988000d0f5b9b15ebd6d64ef769/hf_xet-1.1.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6efaaf1a5a9fc3a501d3e71e88a6bfebc69ee3a716d0e713a931c8b8d920038f", size = 3183844, upload-time = "2025-08-06T00:30:47.582Z" }, + { url = "https://files.pythonhosted.org/packages/20/8f/ccc670616bb9beee867c6bb7139f7eab2b1370fe426503c25f5cbb27b148/hf_xet-1.1.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:751571540f9c1fbad9afcf222a5fb96daf2384bf821317b8bfb0c59d86078513", size = 3074209, upload-time = "2025-08-06T00:30:45.509Z" }, + { url = "https://files.pythonhosted.org/packages/21/0a/4c30e1eb77205565b854f5e4a82cf1f056214e4dc87f2918ebf83d47ae14/hf_xet-1.1.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:18b61bbae92d56ae731b92087c44efcac216071182c603fc535f8e29ec4b09b8", size = 3239602, upload-time = "2025-08-06T00:30:52.41Z" }, + { url = "https://files.pythonhosted.org/packages/f5/1e/fc7e9baf14152662ef0b35fa52a6e889f770a7ed14ac239de3c829ecb47e/hf_xet-1.1.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:713f2bff61b252f8523739969f247aa354ad8e6d869b8281e174e2ea1bb8d604", size = 3348184, upload-time = "2025-08-06T00:30:54.105Z" }, + { url = "https://files.pythonhosted.org/packages/a3/73/e354eae84ceff117ec3560141224724794828927fcc013c5b449bf0b8745/hf_xet-1.1.7-cp37-abi3-win_amd64.whl", hash = "sha256:2e356da7d284479ae0f1dea3cf5a2f74fdf925d6dca84ac4341930d892c7cb34", size = 2820008, upload-time = "2025-08-06T00:30:57.056Z" }, +] + +[[package]] +name = "httptools" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/9a/ce5e1f7e131522e6d3426e8e7a490b3a01f39a6696602e1c4f33f9e94277/httptools-0.6.4.tar.gz", hash = "sha256:4e93eee4add6493b59a5c514da98c939b244fce4a0d8879cd3f466562f4b7d5c", size = 240639, upload-time = "2024-10-16T19:45:08.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/0e/d0b71465c66b9185f90a091ab36389a7352985fe857e352801c39d6127c8/httptools-0.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2", size = 200683, upload-time = "2024-10-16T19:44:30.175Z" }, + { url = "https://files.pythonhosted.org/packages/e2/b8/412a9bb28d0a8988de3296e01efa0bd62068b33856cdda47fe1b5e890954/httptools-0.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44", size = 104337, upload-time = "2024-10-16T19:44:31.786Z" }, + { url = "https://files.pythonhosted.org/packages/9b/01/6fb20be3196ffdc8eeec4e653bc2a275eca7f36634c86302242c4fbb2760/httptools-0.6.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69422b7f458c5af875922cdb5bd586cc1f1033295aa9ff63ee196a87519ac8e1", size = 508796, upload-time = "2024-10-16T19:44:32.825Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d8/b644c44acc1368938317d76ac991c9bba1166311880bcc0ac297cb9d6bd7/httptools-0.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16e603a3bff50db08cd578d54f07032ca1631450ceb972c2f834c2b860c28ea2", size = 510837, upload-time = "2024-10-16T19:44:33.974Z" }, + { url = "https://files.pythonhosted.org/packages/52/d8/254d16a31d543073a0e57f1c329ca7378d8924e7e292eda72d0064987486/httptools-0.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec4f178901fa1834d4a060320d2f3abc5c9e39766953d038f1458cb885f47e81", size = 485289, upload-time = "2024-10-16T19:44:35.111Z" }, + { url = "https://files.pythonhosted.org/packages/5f/3c/4aee161b4b7a971660b8be71a92c24d6c64372c1ab3ae7f366b3680df20f/httptools-0.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f", size = 489779, upload-time = "2024-10-16T19:44:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/12/b7/5cae71a8868e555f3f67a50ee7f673ce36eac970f029c0c5e9d584352961/httptools-0.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:db78cb9ca56b59b016e64b6031eda5653be0589dba2b1b43453f6e8b405a0970", size = 88634, upload-time = "2024-10-16T19:44:37.357Z" }, + { url = "https://files.pythonhosted.org/packages/94/a3/9fe9ad23fd35f7de6b91eeb60848986058bd8b5a5c1e256f5860a160cc3e/httptools-0.6.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ade273d7e767d5fae13fa637f4d53b6e961fb7fd93c7797562663f0171c26660", size = 197214, upload-time = "2024-10-16T19:44:38.738Z" }, + { url = "https://files.pythonhosted.org/packages/ea/d9/82d5e68bab783b632023f2fa31db20bebb4e89dfc4d2293945fd68484ee4/httptools-0.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:856f4bc0478ae143bad54a4242fccb1f3f86a6e1be5548fecfd4102061b3a083", size = 102431, upload-time = "2024-10-16T19:44:39.818Z" }, + { url = "https://files.pythonhosted.org/packages/96/c1/cb499655cbdbfb57b577734fde02f6fa0bbc3fe9fb4d87b742b512908dff/httptools-0.6.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:322d20ea9cdd1fa98bd6a74b77e2ec5b818abdc3d36695ab402a0de8ef2865a3", size = 473121, upload-time = "2024-10-16T19:44:41.189Z" }, + { url = "https://files.pythonhosted.org/packages/af/71/ee32fd358f8a3bb199b03261f10921716990808a675d8160b5383487a317/httptools-0.6.4-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d87b29bd4486c0093fc64dea80231f7c7f7eb4dc70ae394d70a495ab8436071", size = 473805, upload-time = "2024-10-16T19:44:42.384Z" }, + { url = "https://files.pythonhosted.org/packages/8a/0a/0d4df132bfca1507114198b766f1737d57580c9ad1cf93c1ff673e3387be/httptools-0.6.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:342dd6946aa6bda4b8f18c734576106b8a31f2fe31492881a9a160ec84ff4bd5", size = 448858, upload-time = "2024-10-16T19:44:43.959Z" }, + { url = "https://files.pythonhosted.org/packages/1e/6a/787004fdef2cabea27bad1073bf6a33f2437b4dbd3b6fb4a9d71172b1c7c/httptools-0.6.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b36913ba52008249223042dca46e69967985fb4051951f94357ea681e1f5dc0", size = 452042, upload-time = "2024-10-16T19:44:45.071Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/7decab5c404d1d2cdc1bb330b1bf70e83d6af0396fd4fc76fc60c0d522bf/httptools-0.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:28908df1b9bb8187393d5b5db91435ccc9c8e891657f9cbb42a2541b44c82fc8", size = 87682, upload-time = "2024-10-16T19:44:46.46Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "0.34.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/45/c9/bdbe19339f76d12985bc03572f330a01a93c04dffecaaea3061bdd7fb892/huggingface_hub-0.34.4.tar.gz", hash = "sha256:a4228daa6fb001be3f4f4bdaf9a0db00e1739235702848df00885c9b5742c85c", size = 459768, upload-time = "2025-08-08T09:14:52.365Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/7b/bb06b061991107cd8783f300adff3e7b7f284e330fd82f507f2a1417b11d/huggingface_hub-0.34.4-py3-none-any.whl", hash = "sha256:9b365d781739c93ff90c359844221beef048403f1bc1f1c123c191257c3c890a", size = 561452, upload-time = "2025-08-08T09:14:50.159Z" }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "8.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, +] + +[[package]] +name = "ipykernel" +version = "6.30.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "appnope", marker = "sys_platform == 'darwin'" }, + { name = "comm" }, + { name = "debugpy" }, + { name = "ipython" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "matplotlib-inline" }, + { name = "nest-asyncio" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/76/11082e338e0daadc89c8ff866185de11daf67d181901038f9e139d109761/ipykernel-6.30.1.tar.gz", hash = "sha256:6abb270161896402e76b91394fcdce5d1be5d45f456671e5080572f8505be39b", size = 166260, upload-time = "2025-08-04T15:47:35.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/c7/b445faca8deb954fe536abebff4ece5b097b923de482b26e78448c89d1dd/ipykernel-6.30.1-py3-none-any.whl", hash = "sha256:aa6b9fb93dca949069d8b85b6c79b2518e32ac583ae9c7d37c51d119e18b3fb4", size = 117484, upload-time = "2025-08-04T15:47:32.622Z" }, +] + +[[package]] +name = "ipython" +version = "9.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/80/406f9e3bde1c1fd9bf5a0be9d090f8ae623e401b7670d8f6fdf2ab679891/ipython-9.4.0.tar.gz", hash = "sha256:c033c6d4e7914c3d9768aabe76bbe87ba1dc66a92a05db6bfa1125d81f2ee270", size = 4385338, upload-time = "2025-07-01T11:11:30.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/f8/0031ee2b906a15a33d6bfc12dd09c3dfa966b3cb5b284ecfb7549e6ac3c4/ipython-9.4.0-py3-none-any.whl", hash = "sha256:25850f025a446d9b359e8d296ba175a36aedd32e83ca9b5060430fe16801f066", size = 611021, upload-time = "2025-07-01T11:11:27.85Z" }, +] + +[[package]] +name = "ipython-pygments-lexers" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, +] + +[[package]] +name = "ipywidgets" +version = "8.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "comm" }, + { name = "ipython" }, + { name = "jupyterlab-widgets" }, + { name = "traitlets" }, + { name = "widgetsnbextension" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/0a/7e2069d2cf55307b37a6a5195e873968dea965252976c32515d4e300efb0/ipywidgets-8.1.3.tar.gz", hash = "sha256:f5f9eeaae082b1823ce9eac2575272952f40d748893972956dc09700a6392d9c", size = 116515, upload-time = "2024-05-28T09:32:19.319Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/17/8b2ce5765dd423433d2e0727712629c46152fb0bc706b0977f847480f262/ipywidgets-8.1.3-py3-none-any.whl", hash = "sha256:efafd18f7a142248f7cb0ba890a68b96abd4d6e88ddbda483c9130d12667eaf2", size = 139410, upload-time = "2024-05-28T09:32:16.041Z" }, +] + +[[package]] +name = "itsdangerous" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, +] + +[[package]] +name = "jedi" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "parso" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/3a/79a912fbd4d8dd6fbb02bf69afd3bb72cf0c729bb3063c6f4498603db17a/jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0", size = 1231287, upload-time = "2024-11-11T01:41:42.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278, upload-time = "2024-11-11T01:41:40.175Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, +] + +[[package]] +name = "joblib" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/fe/0f5a938c54105553436dbff7a61dc4fed4b1b2c98852f8833beaf4d5968f/joblib-1.5.1.tar.gz", hash = "sha256:f4f86e351f39fe3d0d32a9f2c3d8af1ee4cec285aafcb27003dda5205576b444", size = 330475, upload-time = "2025-05-23T12:04:37.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/4f/1195bbac8e0c2acc5f740661631d8d750dc38d4a32b23ee5df3cde6f4e0d/joblib-1.5.1-py3-none-any.whl", hash = "sha256:4719a31f054c7d766948dcd83e9613686b27114f190f717cec7eaa2084f8a74a", size = 307746, upload-time = "2025-05-23T12:04:35.124Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.25.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/00/a297a868e9d0784450faa7365c2172a7d6110c763e30ba861867c32ae6a9/jsonschema-4.25.0.tar.gz", hash = "sha256:e63acf5c11762c0e6672ffb61482bdf57f0876684d8d249c0fe2d730d48bc55f", size = 356830, upload-time = "2025-07-18T15:39:45.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/54/c86cd8e011fe98803d7e382fd67c0df5ceab8d2b7ad8c5a81524f791551c/jsonschema-4.25.0-py3-none-any.whl", hash = "sha256:24c2e8da302de79c8b9382fee3e76b355e44d2a4364bb207159ce10b517bd716", size = 89184, upload-time = "2025-07-18T15:39:42.956Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bf/ce/46fbd9c8119cfc3581ee5643ea49464d168028cfb5caff5fc0596d0cf914/jsonschema_specifications-2025.4.1.tar.gz", hash = "sha256:630159c9f4dbea161a6a2205c3011cc4f18ff381b189fff48bb39b9bf26ae608", size = 15513, upload-time = "2025-04-23T12:34:07.418Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/0e/b27cdbaccf30b890c40ed1da9fd4a3593a5cf94dae54fb34f8a4b74fcd3f/jsonschema_specifications-2025.4.1-py3-none-any.whl", hash = "sha256:4653bffbd6584f7de83a67e0d620ef16900b390ddc7939d56684d6c81e33f1af", size = 18437, upload-time = "2025-04-23T12:34:05.422Z" }, +] + +[[package]] +name = "jupyter-client" +version = "8.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jupyter-core" }, + { name = "python-dateutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/22/bf9f12fdaeae18019a468b68952a60fe6dbab5d67cd2a103cac7659b41ca/jupyter_client-8.6.3.tar.gz", hash = "sha256:35b3a0947c4a6e9d589eb97d7d4cd5e90f910ee73101611f01283732bd6d9419", size = 342019, upload-time = "2024-09-17T10:44:17.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/85/b0394e0b6fcccd2c1eeefc230978a6f8cb0c5df1e4cd3e7625735a0d7d1e/jupyter_client-8.6.3-py3-none-any.whl", hash = "sha256:e8a19cc986cc45905ac3362915f410f3af85424b4c0905e94fa5f2cb08e8f23f", size = 106105, upload-time = "2024-09-17T10:44:15.218Z" }, +] + +[[package]] +name = "jupyter-core" +version = "5.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "platformdirs" }, + { name = "pywin32", marker = "platform_python_implementation != 'PyPy' and sys_platform == 'win32'" }, + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/1b/72906d554acfeb588332eaaa6f61577705e9ec752ddb486f302dafa292d9/jupyter_core-5.8.1.tar.gz", hash = "sha256:0a5f9706f70e64786b75acba995988915ebd4601c8a52e534a40b51c95f59941", size = 88923, upload-time = "2025-05-27T07:38:16.655Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/57/6bffd4b20b88da3800c5d691e0337761576ee688eb01299eae865689d2df/jupyter_core-5.8.1-py3-none-any.whl", hash = "sha256:c28d268fc90fb53f1338ded2eb410704c5449a358406e8a948b75706e24863d0", size = 28880, upload-time = "2025-05-27T07:38:15.137Z" }, +] + +[[package]] +name = "jupyterlab-widgets" +version = "3.0.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/7d/160595ca88ee87ac6ba95d82177d29ec60aaa63821d3077babb22ce031a5/jupyterlab_widgets-3.0.15.tar.gz", hash = "sha256:2920888a0c2922351a9202817957a68c07d99673504d6cd37345299e971bb08b", size = 213149, upload-time = "2025-05-05T12:32:31.004Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/6a/ca128561b22b60bd5a0c4ea26649e68c8556b82bc70a0c396eebc977fe86/jupyterlab_widgets-3.0.15-py3-none-any.whl", hash = "sha256:d59023d7d7ef71400d51e6fee9a88867f6e65e10a4201605d2d7f3e8f012a31c", size = 216571, upload-time = "2025-05-05T12:32:29.534Z" }, +] + +[[package]] +name = "kiwisolver" +version = "1.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/3c/85844f1b0feb11ee581ac23fe5fce65cd049a200c1446708cc1b7f922875/kiwisolver-1.4.9.tar.gz", hash = "sha256:c3b22c26c6fd6811b0ae8363b95ca8ce4ea3c202d3d0975b2914310ceb1bcc4d", size = 97564, upload-time = "2025-08-10T21:27:49.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/c9/13573a747838aeb1c76e3267620daa054f4152444d1f3d1a2324b78255b5/kiwisolver-1.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ac5a486ac389dddcc5bef4f365b6ae3ffff2c433324fb38dd35e3fab7c957999", size = 123686, upload-time = "2025-08-10T21:26:10.034Z" }, + { url = "https://files.pythonhosted.org/packages/51/ea/2ecf727927f103ffd1739271ca19c424d0e65ea473fbaeea1c014aea93f6/kiwisolver-1.4.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2ba92255faa7309d06fe44c3a4a97efe1c8d640c2a79a5ef728b685762a6fd2", size = 66460, upload-time = "2025-08-10T21:26:11.083Z" }, + { url = "https://files.pythonhosted.org/packages/5b/5a/51f5464373ce2aeb5194508298a508b6f21d3867f499556263c64c621914/kiwisolver-1.4.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a2899935e724dd1074cb568ce7ac0dce28b2cd6ab539c8e001a8578eb106d14", size = 64952, upload-time = "2025-08-10T21:26:12.058Z" }, + { url = "https://files.pythonhosted.org/packages/70/90/6d240beb0f24b74371762873e9b7f499f1e02166a2d9c5801f4dbf8fa12e/kiwisolver-1.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f6008a4919fdbc0b0097089f67a1eb55d950ed7e90ce2cc3e640abadd2757a04", size = 1474756, upload-time = "2025-08-10T21:26:13.096Z" }, + { url = "https://files.pythonhosted.org/packages/12/42/f36816eaf465220f683fb711efdd1bbf7a7005a2473d0e4ed421389bd26c/kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67bb8b474b4181770f926f7b7d2f8c0248cbcb78b660fdd41a47054b28d2a752", size = 1276404, upload-time = "2025-08-10T21:26:14.457Z" }, + { url = "https://files.pythonhosted.org/packages/2e/64/bc2de94800adc830c476dce44e9b40fd0809cddeef1fde9fcf0f73da301f/kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2327a4a30d3ee07d2fbe2e7933e8a37c591663b96ce42a00bc67461a87d7df77", size = 1294410, upload-time = "2025-08-10T21:26:15.73Z" }, + { url = "https://files.pythonhosted.org/packages/5f/42/2dc82330a70aa8e55b6d395b11018045e58d0bb00834502bf11509f79091/kiwisolver-1.4.9-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a08b491ec91b1d5053ac177afe5290adacf1f0f6307d771ccac5de30592d198", size = 1343631, upload-time = "2025-08-10T21:26:17.045Z" }, + { url = "https://files.pythonhosted.org/packages/22/fd/f4c67a6ed1aab149ec5a8a401c323cee7a1cbe364381bb6c9c0d564e0e20/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d8fc5c867c22b828001b6a38d2eaeb88160bf5783c6cb4a5e440efc981ce286d", size = 2224963, upload-time = "2025-08-10T21:26:18.737Z" }, + { url = "https://files.pythonhosted.org/packages/45/aa/76720bd4cb3713314677d9ec94dcc21ced3f1baf4830adde5bb9b2430a5f/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3b3115b2581ea35bb6d1f24a4c90af37e5d9b49dcff267eeed14c3893c5b86ab", size = 2321295, upload-time = "2025-08-10T21:26:20.11Z" }, + { url = "https://files.pythonhosted.org/packages/80/19/d3ec0d9ab711242f56ae0dc2fc5d70e298bb4a1f9dfab44c027668c673a1/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:858e4c22fb075920b96a291928cb7dea5644e94c0ee4fcd5af7e865655e4ccf2", size = 2487987, upload-time = "2025-08-10T21:26:21.49Z" }, + { url = "https://files.pythonhosted.org/packages/39/e9/61e4813b2c97e86b6fdbd4dd824bf72d28bcd8d4849b8084a357bc0dd64d/kiwisolver-1.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ed0fecd28cc62c54b262e3736f8bb2512d8dcfdc2bcf08be5f47f96bf405b145", size = 2291817, upload-time = "2025-08-10T21:26:22.812Z" }, + { url = "https://files.pythonhosted.org/packages/a0/41/85d82b0291db7504da3c2defe35c9a8a5c9803a730f297bd823d11d5fb77/kiwisolver-1.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:f68208a520c3d86ea51acf688a3e3002615a7f0238002cccc17affecc86a8a54", size = 73895, upload-time = "2025-08-10T21:26:24.37Z" }, + { url = "https://files.pythonhosted.org/packages/e2/92/5f3068cf15ee5cb624a0c7596e67e2a0bb2adee33f71c379054a491d07da/kiwisolver-1.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:2c1a4f57df73965f3f14df20b80ee29e6a7930a57d2d9e8491a25f676e197c60", size = 64992, upload-time = "2025-08-10T21:26:25.732Z" }, + { url = "https://files.pythonhosted.org/packages/31/c1/c2686cda909742ab66c7388e9a1a8521a59eb89f8bcfbee28fc980d07e24/kiwisolver-1.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5d0432ccf1c7ab14f9949eec60c5d1f924f17c037e9f8b33352fa05799359b8", size = 123681, upload-time = "2025-08-10T21:26:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f0/f44f50c9f5b1a1860261092e3bc91ecdc9acda848a8b8c6abfda4a24dd5c/kiwisolver-1.4.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efb3a45b35622bb6c16dbfab491a8f5a391fe0e9d45ef32f4df85658232ca0e2", size = 66464, upload-time = "2025-08-10T21:26:27.733Z" }, + { url = "https://files.pythonhosted.org/packages/2d/7a/9d90a151f558e29c3936b8a47ac770235f436f2120aca41a6d5f3d62ae8d/kiwisolver-1.4.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1a12cf6398e8a0a001a059747a1cbf24705e18fe413bc22de7b3d15c67cffe3f", size = 64961, upload-time = "2025-08-10T21:26:28.729Z" }, + { url = "https://files.pythonhosted.org/packages/e9/e9/f218a2cb3a9ffbe324ca29a9e399fa2d2866d7f348ec3a88df87fc248fc5/kiwisolver-1.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b67e6efbf68e077dd71d1a6b37e43e1a99d0bff1a3d51867d45ee8908b931098", size = 1474607, upload-time = "2025-08-10T21:26:29.798Z" }, + { url = "https://files.pythonhosted.org/packages/d9/28/aac26d4c882f14de59041636292bc838db8961373825df23b8eeb807e198/kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5656aa670507437af0207645273ccdfee4f14bacd7f7c67a4306d0dcaeaf6eed", size = 1276546, upload-time = "2025-08-10T21:26:31.401Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ad/8bfc1c93d4cc565e5069162f610ba2f48ff39b7de4b5b8d93f69f30c4bed/kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bfc08add558155345129c7803b3671cf195e6a56e7a12f3dde7c57d9b417f525", size = 1294482, upload-time = "2025-08-10T21:26:32.721Z" }, + { url = "https://files.pythonhosted.org/packages/da/f1/6aca55ff798901d8ce403206d00e033191f63d82dd708a186e0ed2067e9c/kiwisolver-1.4.9-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:40092754720b174e6ccf9e845d0d8c7d8e12c3d71e7fc35f55f3813e96376f78", size = 1343720, upload-time = "2025-08-10T21:26:34.032Z" }, + { url = "https://files.pythonhosted.org/packages/d1/91/eed031876c595c81d90d0f6fc681ece250e14bf6998c3d7c419466b523b7/kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:497d05f29a1300d14e02e6441cf0f5ee81c1ff5a304b0d9fb77423974684e08b", size = 2224907, upload-time = "2025-08-10T21:26:35.824Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ec/4d1925f2e49617b9cca9c34bfa11adefad49d00db038e692a559454dfb2e/kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bdd1a81a1860476eb41ac4bc1e07b3f07259e6d55bbf739b79c8aaedcf512799", size = 2321334, upload-time = "2025-08-10T21:26:37.534Z" }, + { url = "https://files.pythonhosted.org/packages/43/cb/450cd4499356f68802750c6ddc18647b8ea01ffa28f50d20598e0befe6e9/kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:e6b93f13371d341afee3be9f7c5964e3fe61d5fa30f6a30eb49856935dfe4fc3", size = 2488313, upload-time = "2025-08-10T21:26:39.191Z" }, + { url = "https://files.pythonhosted.org/packages/71/67/fc76242bd99f885651128a5d4fa6083e5524694b7c88b489b1b55fdc491d/kiwisolver-1.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d75aa530ccfaa593da12834b86a0724f58bff12706659baa9227c2ccaa06264c", size = 2291970, upload-time = "2025-08-10T21:26:40.828Z" }, + { url = "https://files.pythonhosted.org/packages/75/bd/f1a5d894000941739f2ae1b65a32892349423ad49c2e6d0771d0bad3fae4/kiwisolver-1.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:dd0a578400839256df88c16abddf9ba14813ec5f21362e1fe65022e00c883d4d", size = 73894, upload-time = "2025-08-10T21:26:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/95/38/dce480814d25b99a391abbddadc78f7c117c6da34be68ca8b02d5848b424/kiwisolver-1.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:d4188e73af84ca82468f09cadc5ac4db578109e52acb4518d8154698d3a87ca2", size = 64995, upload-time = "2025-08-10T21:26:43.889Z" }, + { url = "https://files.pythonhosted.org/packages/e2/37/7d218ce5d92dadc5ebdd9070d903e0c7cf7edfe03f179433ac4d13ce659c/kiwisolver-1.4.9-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:5a0f2724dfd4e3b3ac5a82436a8e6fd16baa7d507117e4279b660fe8ca38a3a1", size = 126510, upload-time = "2025-08-10T21:26:44.915Z" }, + { url = "https://files.pythonhosted.org/packages/23/b0/e85a2b48233daef4b648fb657ebbb6f8367696a2d9548a00b4ee0eb67803/kiwisolver-1.4.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:1b11d6a633e4ed84fc0ddafd4ebfd8ea49b3f25082c04ad12b8315c11d504dc1", size = 67903, upload-time = "2025-08-10T21:26:45.934Z" }, + { url = "https://files.pythonhosted.org/packages/44/98/f2425bc0113ad7de24da6bb4dae1343476e95e1d738be7c04d31a5d037fd/kiwisolver-1.4.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61874cdb0a36016354853593cffc38e56fc9ca5aa97d2c05d3dcf6922cd55a11", size = 66402, upload-time = "2025-08-10T21:26:47.101Z" }, + { url = "https://files.pythonhosted.org/packages/98/d8/594657886df9f34c4177cc353cc28ca7e6e5eb562d37ccc233bff43bbe2a/kiwisolver-1.4.9-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:60c439763a969a6af93b4881db0eed8fadf93ee98e18cbc35bc8da868d0c4f0c", size = 1582135, upload-time = "2025-08-10T21:26:48.665Z" }, + { url = "https://files.pythonhosted.org/packages/5c/c6/38a115b7170f8b306fc929e166340c24958347308ea3012c2b44e7e295db/kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92a2f997387a1b79a75e7803aa7ded2cfbe2823852ccf1ba3bcf613b62ae3197", size = 1389409, upload-time = "2025-08-10T21:26:50.335Z" }, + { url = "https://files.pythonhosted.org/packages/bf/3b/e04883dace81f24a568bcee6eb3001da4ba05114afa622ec9b6fafdc1f5e/kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a31d512c812daea6d8b3be3b2bfcbeb091dbb09177706569bcfc6240dcf8b41c", size = 1401763, upload-time = "2025-08-10T21:26:51.867Z" }, + { url = "https://files.pythonhosted.org/packages/9f/80/20ace48e33408947af49d7d15c341eaee69e4e0304aab4b7660e234d6288/kiwisolver-1.4.9-cp313-cp313t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:52a15b0f35dad39862d376df10c5230155243a2c1a436e39eb55623ccbd68185", size = 1453643, upload-time = "2025-08-10T21:26:53.592Z" }, + { url = "https://files.pythonhosted.org/packages/64/31/6ce4380a4cd1f515bdda976a1e90e547ccd47b67a1546d63884463c92ca9/kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a30fd6fdef1430fd9e1ba7b3398b5ee4e2887783917a687d86ba69985fb08748", size = 2330818, upload-time = "2025-08-10T21:26:55.051Z" }, + { url = "https://files.pythonhosted.org/packages/fa/e9/3f3fcba3bcc7432c795b82646306e822f3fd74df0ee81f0fa067a1f95668/kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cc9617b46837c6468197b5945e196ee9ca43057bb7d9d1ae688101e4e1dddf64", size = 2419963, upload-time = "2025-08-10T21:26:56.421Z" }, + { url = "https://files.pythonhosted.org/packages/99/43/7320c50e4133575c66e9f7dadead35ab22d7c012a3b09bb35647792b2a6d/kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:0ab74e19f6a2b027ea4f845a78827969af45ce790e6cb3e1ebab71bdf9f215ff", size = 2594639, upload-time = "2025-08-10T21:26:57.882Z" }, + { url = "https://files.pythonhosted.org/packages/65/d6/17ae4a270d4a987ef8a385b906d2bdfc9fce502d6dc0d3aea865b47f548c/kiwisolver-1.4.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dba5ee5d3981160c28d5490f0d1b7ed730c22470ff7f6cc26cfcfaacb9896a07", size = 2391741, upload-time = "2025-08-10T21:26:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/2a/8f/8f6f491d595a9e5912971f3f863d81baddccc8a4d0c3749d6a0dd9ffc9df/kiwisolver-1.4.9-cp313-cp313t-win_arm64.whl", hash = "sha256:0749fd8f4218ad2e851e11cc4dc05c7cbc0cbc4267bdfdb31782e65aace4ee9c", size = 68646, upload-time = "2025-08-10T21:27:00.52Z" }, + { url = "https://files.pythonhosted.org/packages/6b/32/6cc0fbc9c54d06c2969faa9c1d29f5751a2e51809dd55c69055e62d9b426/kiwisolver-1.4.9-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:9928fe1eb816d11ae170885a74d074f57af3a0d65777ca47e9aeb854a1fba386", size = 123806, upload-time = "2025-08-10T21:27:01.537Z" }, + { url = "https://files.pythonhosted.org/packages/b2/dd/2bfb1d4a4823d92e8cbb420fe024b8d2167f72079b3bb941207c42570bdf/kiwisolver-1.4.9-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d0005b053977e7b43388ddec89fa567f43d4f6d5c2c0affe57de5ebf290dc552", size = 66605, upload-time = "2025-08-10T21:27:03.335Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/00aafdb4e4509c2ca6064646cba9cd4b37933898f426756adb2cb92ebbed/kiwisolver-1.4.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2635d352d67458b66fd0667c14cb1d4145e9560d503219034a18a87e971ce4f3", size = 64925, upload-time = "2025-08-10T21:27:04.339Z" }, + { url = "https://files.pythonhosted.org/packages/43/dc/51acc6791aa14e5cb6d8a2e28cefb0dc2886d8862795449d021334c0df20/kiwisolver-1.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:767c23ad1c58c9e827b649a9ab7809fd5fd9db266a9cf02b0e926ddc2c680d58", size = 1472414, upload-time = "2025-08-10T21:27:05.437Z" }, + { url = "https://files.pythonhosted.org/packages/3d/bb/93fa64a81db304ac8a246f834d5094fae4b13baf53c839d6bb6e81177129/kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72d0eb9fba308b8311685c2268cf7d0a0639a6cd027d8128659f72bdd8a024b4", size = 1281272, upload-time = "2025-08-10T21:27:07.063Z" }, + { url = "https://files.pythonhosted.org/packages/70/e6/6df102916960fb8d05069d4bd92d6d9a8202d5a3e2444494e7cd50f65b7a/kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f68e4f3eeca8fb22cc3d731f9715a13b652795ef657a13df1ad0c7dc0e9731df", size = 1298578, upload-time = "2025-08-10T21:27:08.452Z" }, + { url = "https://files.pythonhosted.org/packages/7c/47/e142aaa612f5343736b087864dbaebc53ea8831453fb47e7521fa8658f30/kiwisolver-1.4.9-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d84cd4061ae292d8ac367b2c3fa3aad11cb8625a95d135fe93f286f914f3f5a6", size = 1345607, upload-time = "2025-08-10T21:27:10.125Z" }, + { url = "https://files.pythonhosted.org/packages/54/89/d641a746194a0f4d1a3670fb900d0dbaa786fb98341056814bc3f058fa52/kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a60ea74330b91bd22a29638940d115df9dc00af5035a9a2a6ad9399ffb4ceca5", size = 2230150, upload-time = "2025-08-10T21:27:11.484Z" }, + { url = "https://files.pythonhosted.org/packages/aa/6b/5ee1207198febdf16ac11f78c5ae40861b809cbe0e6d2a8d5b0b3044b199/kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:ce6a3a4e106cf35c2d9c4fa17c05ce0b180db622736845d4315519397a77beaf", size = 2325979, upload-time = "2025-08-10T21:27:12.917Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ff/b269eefd90f4ae14dcc74973d5a0f6d28d3b9bb1afd8c0340513afe6b39a/kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:77937e5e2a38a7b48eef0585114fe7930346993a88060d0bf886086d2aa49ef5", size = 2491456, upload-time = "2025-08-10T21:27:14.353Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d4/10303190bd4d30de547534601e259a4fbf014eed94aae3e5521129215086/kiwisolver-1.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:24c175051354f4a28c5d6a31c93906dc653e2bf234e8a4bbfb964892078898ce", size = 2294621, upload-time = "2025-08-10T21:27:15.808Z" }, + { url = "https://files.pythonhosted.org/packages/28/e0/a9a90416fce5c0be25742729c2ea52105d62eda6c4be4d803c2a7be1fa50/kiwisolver-1.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:0763515d4df10edf6d06a3c19734e2566368980d21ebec439f33f9eb936c07b7", size = 75417, upload-time = "2025-08-10T21:27:17.436Z" }, + { url = "https://files.pythonhosted.org/packages/1f/10/6949958215b7a9a264299a7db195564e87900f709db9245e4ebdd3c70779/kiwisolver-1.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:0e4e2bf29574a6a7b7f6cb5fa69293b9f96c928949ac4a53ba3f525dffb87f9c", size = 66582, upload-time = "2025-08-10T21:27:18.436Z" }, + { url = "https://files.pythonhosted.org/packages/ec/79/60e53067903d3bc5469b369fe0dfc6b3482e2133e85dae9daa9527535991/kiwisolver-1.4.9-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d976bbb382b202f71c67f77b0ac11244021cfa3f7dfd9e562eefcea2df711548", size = 126514, upload-time = "2025-08-10T21:27:19.465Z" }, + { url = "https://files.pythonhosted.org/packages/25/d1/4843d3e8d46b072c12a38c97c57fab4608d36e13fe47d47ee96b4d61ba6f/kiwisolver-1.4.9-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2489e4e5d7ef9a1c300a5e0196e43d9c739f066ef23270607d45aba368b91f2d", size = 67905, upload-time = "2025-08-10T21:27:20.51Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ae/29ffcbd239aea8b93108de1278271ae764dfc0d803a5693914975f200596/kiwisolver-1.4.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e2ea9f7ab7fbf18fffb1b5434ce7c69a07582f7acc7717720f1d69f3e806f90c", size = 66399, upload-time = "2025-08-10T21:27:21.496Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ae/d7ba902aa604152c2ceba5d352d7b62106bedbccc8e95c3934d94472bfa3/kiwisolver-1.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b34e51affded8faee0dfdb705416153819d8ea9250bbbf7ea1b249bdeb5f1122", size = 1582197, upload-time = "2025-08-10T21:27:22.604Z" }, + { url = "https://files.pythonhosted.org/packages/f2/41/27c70d427eddb8bc7e4f16420a20fefc6f480312122a59a959fdfe0445ad/kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8aacd3d4b33b772542b2e01beb50187536967b514b00003bdda7589722d2a64", size = 1390125, upload-time = "2025-08-10T21:27:24.036Z" }, + { url = "https://files.pythonhosted.org/packages/41/42/b3799a12bafc76d962ad69083f8b43b12bf4fe78b097b12e105d75c9b8f1/kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7cf974dd4e35fa315563ac99d6287a1024e4dc2077b8a7d7cd3d2fb65d283134", size = 1402612, upload-time = "2025-08-10T21:27:25.773Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b5/a210ea073ea1cfaca1bb5c55a62307d8252f531beb364e18aa1e0888b5a0/kiwisolver-1.4.9-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:85bd218b5ecfbee8c8a82e121802dcb519a86044c9c3b2e4aef02fa05c6da370", size = 1453990, upload-time = "2025-08-10T21:27:27.089Z" }, + { url = "https://files.pythonhosted.org/packages/5f/ce/a829eb8c033e977d7ea03ed32fb3c1781b4fa0433fbadfff29e39c676f32/kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0856e241c2d3df4efef7c04a1e46b1936b6120c9bcf36dd216e3acd84bc4fb21", size = 2331601, upload-time = "2025-08-10T21:27:29.343Z" }, + { url = "https://files.pythonhosted.org/packages/e0/4b/b5e97eb142eb9cd0072dacfcdcd31b1c66dc7352b0f7c7255d339c0edf00/kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9af39d6551f97d31a4deebeac6f45b156f9755ddc59c07b402c148f5dbb6482a", size = 2422041, upload-time = "2025-08-10T21:27:30.754Z" }, + { url = "https://files.pythonhosted.org/packages/40/be/8eb4cd53e1b85ba4edc3a9321666f12b83113a178845593307a3e7891f44/kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:bb4ae2b57fc1d8cbd1cf7b1d9913803681ffa903e7488012be5b76dedf49297f", size = 2594897, upload-time = "2025-08-10T21:27:32.803Z" }, + { url = "https://files.pythonhosted.org/packages/99/dd/841e9a66c4715477ea0abc78da039832fbb09dac5c35c58dc4c41a407b8a/kiwisolver-1.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:aedff62918805fb62d43a4aa2ecd4482c380dc76cd31bd7c8878588a61bd0369", size = 2391835, upload-time = "2025-08-10T21:27:34.23Z" }, + { url = "https://files.pythonhosted.org/packages/0c/28/4b2e5c47a0da96896fdfdb006340ade064afa1e63675d01ea5ac222b6d52/kiwisolver-1.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:1fa333e8b2ce4d9660f2cda9c0e1b6bafcfb2457a9d259faa82289e73ec24891", size = 79988, upload-time = "2025-08-10T21:27:35.587Z" }, + { url = "https://files.pythonhosted.org/packages/80/be/3578e8afd18c88cdf9cb4cffde75a96d2be38c5a903f1ed0ceec061bd09e/kiwisolver-1.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:4a48a2ce79d65d363597ef7b567ce3d14d68783d2b2263d98db3d9477805ba32", size = 70260, upload-time = "2025-08-10T21:27:36.606Z" }, +] + +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, +] + +[[package]] +name = "markdown" +version = "3.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/c2/4ab49206c17f75cb08d6311171f2d65798988db4360c4d1485bd0eedd67c/markdown-3.8.2.tar.gz", hash = "sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45", size = 362071, upload-time = "2025-06-19T17:12:44.483Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/2b/34cc11786bc00d0f04d0f5fdc3a2b1ae0b6239eef72d3d345805f9ad92a1/markdown-3.8.2-py3-none-any.whl", hash = "sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24", size = 106827, upload-time = "2025-06-19T17:12:42.994Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537, upload-time = "2024-10-18T15:21:54.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274, upload-time = "2024-10-18T15:21:13.777Z" }, + { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348, upload-time = "2024-10-18T15:21:14.822Z" }, + { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149, upload-time = "2024-10-18T15:21:15.642Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118, upload-time = "2024-10-18T15:21:17.133Z" }, + { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993, upload-time = "2024-10-18T15:21:18.064Z" }, + { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178, upload-time = "2024-10-18T15:21:18.859Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319, upload-time = "2024-10-18T15:21:19.671Z" }, + { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352, upload-time = "2024-10-18T15:21:20.971Z" }, + { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097, upload-time = "2024-10-18T15:21:22.646Z" }, + { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601, upload-time = "2024-10-18T15:21:23.499Z" }, + { url = "https://files.pythonhosted.org/packages/83/0e/67eb10a7ecc77a0c2bbe2b0235765b98d164d81600746914bebada795e97/MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", size = 14274, upload-time = "2024-10-18T15:21:24.577Z" }, + { url = "https://files.pythonhosted.org/packages/2b/6d/9409f3684d3335375d04e5f05744dfe7e9f120062c9857df4ab490a1031a/MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", size = 12352, upload-time = "2024-10-18T15:21:25.382Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f5/6eadfcd3885ea85fe2a7c128315cc1bb7241e1987443d78c8fe712d03091/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", size = 24122, upload-time = "2024-10-18T15:21:26.199Z" }, + { url = "https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", size = 23085, upload-time = "2024-10-18T15:21:27.029Z" }, + { url = "https://files.pythonhosted.org/packages/c2/cf/c9d56af24d56ea04daae7ac0940232d31d5a8354f2b457c6d856b2057d69/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", size = 22978, upload-time = "2024-10-18T15:21:27.846Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9f/8619835cd6a711d6272d62abb78c033bda638fdc54c4e7f4272cf1c0962b/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", size = 24208, upload-time = "2024-10-18T15:21:28.744Z" }, + { url = "https://files.pythonhosted.org/packages/f9/bf/176950a1792b2cd2102b8ffeb5133e1ed984547b75db47c25a67d3359f77/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", size = 23357, upload-time = "2024-10-18T15:21:29.545Z" }, + { url = "https://files.pythonhosted.org/packages/ce/4f/9a02c1d335caabe5c4efb90e1b6e8ee944aa245c1aaaab8e8a618987d816/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", size = 23344, upload-time = "2024-10-18T15:21:30.366Z" }, + { url = "https://files.pythonhosted.org/packages/ee/55/c271b57db36f748f0e04a759ace9f8f759ccf22b4960c270c78a394f58be/MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1", size = 15101, upload-time = "2024-10-18T15:21:31.207Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/07df22d2dd4df40aba9f3e402e6dc1b8ee86297dddbad4872bd5e7b0094f/MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f", size = 15603, upload-time = "2024-10-18T15:21:32.032Z" }, + { url = "https://files.pythonhosted.org/packages/62/6a/8b89d24db2d32d433dffcd6a8779159da109842434f1dd2f6e71f32f738c/MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", size = 14510, upload-time = "2024-10-18T15:21:33.625Z" }, + { url = "https://files.pythonhosted.org/packages/7a/06/a10f955f70a2e5a9bf78d11a161029d278eeacbd35ef806c3fd17b13060d/MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", size = 12486, upload-time = "2024-10-18T15:21:34.611Z" }, + { url = "https://files.pythonhosted.org/packages/34/cf/65d4a571869a1a9078198ca28f39fba5fbb910f952f9dbc5220afff9f5e6/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", size = 25480, upload-time = "2024-10-18T15:21:35.398Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e3/90e9651924c430b885468b56b3d597cabf6d72be4b24a0acd1fa0e12af67/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", size = 23914, upload-time = "2024-10-18T15:21:36.231Z" }, + { url = "https://files.pythonhosted.org/packages/66/8c/6c7cf61f95d63bb866db39085150df1f2a5bd3335298f14a66b48e92659c/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", size = 23796, upload-time = "2024-10-18T15:21:37.073Z" }, + { url = "https://files.pythonhosted.org/packages/bb/35/cbe9238ec3f47ac9a7c8b3df7a808e7cb50fe149dc7039f5f454b3fba218/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", size = 25473, upload-time = "2024-10-18T15:21:37.932Z" }, + { url = "https://files.pythonhosted.org/packages/e6/32/7621a4382488aa283cc05e8984a9c219abad3bca087be9ec77e89939ded9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", size = 24114, upload-time = "2024-10-18T15:21:39.799Z" }, + { url = "https://files.pythonhosted.org/packages/0d/80/0985960e4b89922cb5a0bac0ed39c5b96cbc1a536a99f30e8c220a996ed9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", size = 24098, upload-time = "2024-10-18T15:21:40.813Z" }, + { url = "https://files.pythonhosted.org/packages/82/78/fedb03c7d5380df2427038ec8d973587e90561b2d90cd472ce9254cf348b/MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6", size = 15208, upload-time = "2024-10-18T15:21:41.814Z" }, + { url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739, upload-time = "2024-10-18T15:21:42.784Z" }, +] + +[[package]] +name = "matplotlib" +version = "3.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "contourpy" }, + { name = "cycler" }, + { name = "fonttools" }, + { name = "kiwisolver" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyparsing" }, + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/dd/fa2e1a45fce2d09f4aea3cee169760e672c8262325aa5796c49d543dc7e6/matplotlib-3.10.0.tar.gz", hash = "sha256:b886d02a581b96704c9d1ffe55709e49b4d2d52709ccebc4be42db856e511278", size = 36686418, upload-time = "2024-12-14T06:32:51.547Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/c7/6b2d8cb7cc251d53c976799cacd3200add56351c175ba89ab9cbd7c1e68a/matplotlib-3.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4659665bc7c9b58f8c00317c3c2a299f7f258eeae5a5d56b4c64226fca2f7c59", size = 8172465, upload-time = "2024-12-14T06:31:24.727Z" }, + { url = "https://files.pythonhosted.org/packages/42/2a/6d66d0fba41e13e9ca6512a0a51170f43e7e7ed3a8dfa036324100775612/matplotlib-3.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d44cb942af1693cced2604c33a9abcef6205601c445f6d0dc531d813af8a2f5a", size = 8043300, upload-time = "2024-12-14T06:31:28.55Z" }, + { url = "https://files.pythonhosted.org/packages/90/60/2a60342b27b90a16bada939a85e29589902b41073f59668b904b15ea666c/matplotlib-3.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a994f29e968ca002b50982b27168addfd65f0105610b6be7fa515ca4b5307c95", size = 8448936, upload-time = "2024-12-14T06:31:32.223Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b2/d872fc3d753516870d520595ddd8ce4dd44fa797a240999f125f58521ad7/matplotlib-3.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b0558bae37f154fffda54d779a592bc97ca8b4701f1c710055b609a3bac44c8", size = 8594151, upload-time = "2024-12-14T06:31:34.894Z" }, + { url = "https://files.pythonhosted.org/packages/f4/bd/b2f60cf7f57d014ab33e4f74602a2b5bdc657976db8196bbc022185f6f9c/matplotlib-3.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:503feb23bd8c8acc75541548a1d709c059b7184cde26314896e10a9f14df5f12", size = 9400347, upload-time = "2024-12-14T06:31:39.552Z" }, + { url = "https://files.pythonhosted.org/packages/9f/6e/264673e64001b99d747aff5a288eca82826c024437a3694e19aed1decf46/matplotlib-3.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:c40ba2eb08b3f5de88152c2333c58cee7edcead0a2a0d60fcafa116b17117adc", size = 8039144, upload-time = "2024-12-14T06:31:44.128Z" }, + { url = "https://files.pythonhosted.org/packages/72/11/1b2a094d95dcb6e6edd4a0b238177c439006c6b7a9fe8d31801237bf512f/matplotlib-3.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96f2886f5c1e466f21cc41b70c5a0cd47bfa0015eb2d5793c88ebce658600e25", size = 8173073, upload-time = "2024-12-14T06:31:46.592Z" }, + { url = "https://files.pythonhosted.org/packages/0d/c4/87b6ad2723070511a411ea719f9c70fde64605423b184face4e94986de9d/matplotlib-3.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:12eaf48463b472c3c0f8dbacdbf906e573013df81a0ab82f0616ea4b11281908", size = 8043892, upload-time = "2024-12-14T06:31:49.14Z" }, + { url = "https://files.pythonhosted.org/packages/57/69/cb0812a136550b21361335e9ffb7d459bf6d13e03cb7b015555d5143d2d6/matplotlib-3.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fbbabc82fde51391c4da5006f965e36d86d95f6ee83fb594b279564a4c5d0d2", size = 8450532, upload-time = "2024-12-14T06:31:53.005Z" }, + { url = "https://files.pythonhosted.org/packages/ea/3a/bab9deb4fb199c05e9100f94d7f1c702f78d3241e6a71b784d2b88d7bebd/matplotlib-3.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad2e15300530c1a94c63cfa546e3b7864bd18ea2901317bae8bbf06a5ade6dcf", size = 8593905, upload-time = "2024-12-14T06:31:59.022Z" }, + { url = "https://files.pythonhosted.org/packages/8b/66/742fd242f989adc1847ddf5f445815f73ad7c46aa3440690cc889cfa423c/matplotlib-3.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3547d153d70233a8496859097ef0312212e2689cdf8d7ed764441c77604095ae", size = 9399609, upload-time = "2024-12-14T06:32:05.151Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d6/54cee7142cef7d910a324a7aedf335c0c147b03658b54d49ec48166f10a6/matplotlib-3.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:c55b20591ced744aa04e8c3e4b7543ea4d650b6c3c4b208c08a05b4010e8b442", size = 8039076, upload-time = "2024-12-14T06:32:08.38Z" }, + { url = "https://files.pythonhosted.org/packages/43/14/815d072dc36e88753433bfd0385113405efb947e6895ff7b4d2e8614a33b/matplotlib-3.10.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:9ade1003376731a971e398cc4ef38bb83ee8caf0aee46ac6daa4b0506db1fd06", size = 8211000, upload-time = "2024-12-14T06:32:12.383Z" }, + { url = "https://files.pythonhosted.org/packages/9a/76/34e75f364194ec352678adcb540964be6f35ec7d3d8c75ebcb17e6839359/matplotlib-3.10.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:95b710fea129c76d30be72c3b38f330269363fbc6e570a5dd43580487380b5ff", size = 8087707, upload-time = "2024-12-14T06:32:15.773Z" }, + { url = "https://files.pythonhosted.org/packages/c3/2b/b6bc0dff6a72d333bc7df94a66e6ce662d224e43daa8ad8ae4eaa9a77f55/matplotlib-3.10.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdbaf909887373c3e094b0318d7ff230b2ad9dcb64da7ade654182872ab2593", size = 8477384, upload-time = "2024-12-14T06:32:20.311Z" }, + { url = "https://files.pythonhosted.org/packages/c2/2d/b5949fb2b76e9b47ab05e25a5f5f887c70de20d8b0cbc704a4e2ee71c786/matplotlib-3.10.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d907fddb39f923d011875452ff1eca29a9e7f21722b873e90db32e5d8ddff12e", size = 8610334, upload-time = "2024-12-14T06:32:25.779Z" }, + { url = "https://files.pythonhosted.org/packages/d6/9a/6e3c799d5134d9af44b01c787e1360bee38cf51850506ea2e743a787700b/matplotlib-3.10.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3b427392354d10975c1d0f4ee18aa5844640b512d5311ef32efd4dd7db106ede", size = 9406777, upload-time = "2024-12-14T06:32:28.919Z" }, + { url = "https://files.pythonhosted.org/packages/0e/dd/e6ae97151e5ed648ab2ea48885bc33d39202b640eec7a2910e2c843f7ac0/matplotlib-3.10.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5fd41b0ec7ee45cd960a8e71aea7c946a28a0b8a4dcee47d2856b2af051f334c", size = 8109742, upload-time = "2024-12-14T06:32:32.115Z" }, +] + +[[package]] +name = "matplotlib-inline" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/5b/a36a337438a14116b16480db471ad061c36c3694df7c2084a0da7ba538b7/matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90", size = 8159, upload-time = "2024-04-15T13:44:44.803Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899, upload-time = "2024-04-15T13:44:43.265Z" }, +] + +[[package]] +name = "mlflow" +version = "2.19.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "alembic" }, + { name = "docker" }, + { name = "flask" }, + { name = "graphene" }, + { name = "gunicorn", marker = "sys_platform != 'win32'" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "matplotlib" }, + { name = "mlflow-skinny" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "scikit-learn" }, + { name = "scipy" }, + { name = "sqlalchemy" }, + { name = "waitress", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/76/f623312328a8b642fba8b9683e07904ee9f9c59b9e58528e9a9f5bbdcfea/mlflow-2.19.0.tar.gz", hash = "sha256:b860e9d2599a32460968a0a90efdf960b6a6237a08bff44cc5508830017cf70e", size = 26813362, upload-time = "2024-12-11T09:49:38.38Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/39/e051e58f35077500fea62adb67c0ff32cab768a5bbc1e0d8c682e30d56ee/mlflow-2.19.0-py3-none-any.whl", hash = "sha256:875364a9c37d2e6e5b6256a3cee314e1e6ada0c253f46b6fcb37d986a2dc2514", size = 27397174, upload-time = "2024-12-11T09:49:32.119Z" }, +] + +[[package]] +name = "mlflow-skinny" +version = "2.19.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "click" }, + { name = "cloudpickle" }, + { name = "databricks-sdk" }, + { name = "gitpython" }, + { name = "importlib-metadata" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sqlparse" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/69/53c19be8f05574b9955a2930d0c9b04403d5dd35afce05fbe664b5bfbbfc/mlflow_skinny-2.19.0.tar.gz", hash = "sha256:55a464082ecd48961f73f9a0a58b8d44bf2e77bd32632998f1dffd43ef48623c", size = 5503927, upload-time = "2024-12-11T08:53:47.849Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/95/75f59715e39aa2224e5ecd8c52d5a305467e16a843ade2235a215599a1fa/mlflow_skinny-2.19.0-py3-none-any.whl", hash = "sha256:72c652545460db09dc5716241d2fcd9a211b7875444632fbe2d0b62a1f057694", size = 5854771, upload-time = "2024-12-11T08:53:44.16Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "msgpack" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/b1/ea4f68038a18c77c9467400d166d74c4ffa536f34761f7983a104357e614/msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd", size = 173555, upload-time = "2025-06-13T06:52:51.324Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/26/389b9c593eda2b8551b2e7126ad3a06af6f9b44274eb3a4f054d48ff7e47/msgpack-1.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae497b11f4c21558d95de9f64fff7053544f4d1a17731c866143ed6bb4591238", size = 82359, upload-time = "2025-06-13T06:52:03.909Z" }, + { url = "https://files.pythonhosted.org/packages/ab/65/7d1de38c8a22cf8b1551469159d4b6cf49be2126adc2482de50976084d78/msgpack-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:33be9ab121df9b6b461ff91baac6f2731f83d9b27ed948c5b9d1978ae28bf157", size = 79172, upload-time = "2025-06-13T06:52:05.246Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bd/cacf208b64d9577a62c74b677e1ada005caa9b69a05a599889d6fc2ab20a/msgpack-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f64ae8fe7ffba251fecb8408540c34ee9df1c26674c50c4544d72dbf792e5ce", size = 425013, upload-time = "2025-06-13T06:52:06.341Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ec/fd869e2567cc9c01278a736cfd1697941ba0d4b81a43e0aa2e8d71dab208/msgpack-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a494554874691720ba5891c9b0b39474ba43ffb1aaf32a5dac874effb1619e1a", size = 426905, upload-time = "2025-06-13T06:52:07.501Z" }, + { url = "https://files.pythonhosted.org/packages/55/2a/35860f33229075bce803a5593d046d8b489d7ba2fc85701e714fc1aaf898/msgpack-1.1.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb643284ab0ed26f6957d969fe0dd8bb17beb567beb8998140b5e38a90974f6c", size = 407336, upload-time = "2025-06-13T06:52:09.047Z" }, + { url = "https://files.pythonhosted.org/packages/8c/16/69ed8f3ada150bf92745fb4921bd621fd2cdf5a42e25eb50bcc57a5328f0/msgpack-1.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d275a9e3c81b1093c060c3837e580c37f47c51eca031f7b5fb76f7b8470f5f9b", size = 409485, upload-time = "2025-06-13T06:52:10.382Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b6/0c398039e4c6d0b2e37c61d7e0e9d13439f91f780686deb8ee64ecf1ae71/msgpack-1.1.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fd6b577e4541676e0cc9ddc1709d25014d3ad9a66caa19962c4f5de30fc09ef", size = 412182, upload-time = "2025-06-13T06:52:11.644Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d0/0cf4a6ecb9bc960d624c93effaeaae75cbf00b3bc4a54f35c8507273cda1/msgpack-1.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb29aaa613c0a1c40d1af111abf025f1732cab333f96f285d6a93b934738a68a", size = 419883, upload-time = "2025-06-13T06:52:12.806Z" }, + { url = "https://files.pythonhosted.org/packages/62/83/9697c211720fa71a2dfb632cad6196a8af3abea56eece220fde4674dc44b/msgpack-1.1.1-cp312-cp312-win32.whl", hash = "sha256:870b9a626280c86cff9c576ec0d9cbcc54a1e5ebda9cd26dab12baf41fee218c", size = 65406, upload-time = "2025-06-13T06:52:14.271Z" }, + { url = "https://files.pythonhosted.org/packages/c0/23/0abb886e80eab08f5e8c485d6f13924028602829f63b8f5fa25a06636628/msgpack-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:5692095123007180dca3e788bb4c399cc26626da51629a31d40207cb262e67f4", size = 72558, upload-time = "2025-06-13T06:52:15.252Z" }, + { url = "https://files.pythonhosted.org/packages/a1/38/561f01cf3577430b59b340b51329803d3a5bf6a45864a55f4ef308ac11e3/msgpack-1.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3765afa6bd4832fc11c3749be4ba4b69a0e8d7b728f78e68120a157a4c5d41f0", size = 81677, upload-time = "2025-06-13T06:52:16.64Z" }, + { url = "https://files.pythonhosted.org/packages/09/48/54a89579ea36b6ae0ee001cba8c61f776451fad3c9306cd80f5b5c55be87/msgpack-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8ddb2bcfd1a8b9e431c8d6f4f7db0773084e107730ecf3472f1dfe9ad583f3d9", size = 78603, upload-time = "2025-06-13T06:52:17.843Z" }, + { url = "https://files.pythonhosted.org/packages/a0/60/daba2699b308e95ae792cdc2ef092a38eb5ee422f9d2fbd4101526d8a210/msgpack-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:196a736f0526a03653d829d7d4c5500a97eea3648aebfd4b6743875f28aa2af8", size = 420504, upload-time = "2025-06-13T06:52:18.982Z" }, + { url = "https://files.pythonhosted.org/packages/20/22/2ebae7ae43cd8f2debc35c631172ddf14e2a87ffcc04cf43ff9df9fff0d3/msgpack-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d592d06e3cc2f537ceeeb23d38799c6ad83255289bb84c2e5792e5a8dea268a", size = 423749, upload-time = "2025-06-13T06:52:20.211Z" }, + { url = "https://files.pythonhosted.org/packages/40/1b/54c08dd5452427e1179a40b4b607e37e2664bca1c790c60c442c8e972e47/msgpack-1.1.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4df2311b0ce24f06ba253fda361f938dfecd7b961576f9be3f3fbd60e87130ac", size = 404458, upload-time = "2025-06-13T06:52:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/2e/60/6bb17e9ffb080616a51f09928fdd5cac1353c9becc6c4a8abd4e57269a16/msgpack-1.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4141c5a32b5e37905b5940aacbc59739f036930367d7acce7a64e4dec1f5e0b", size = 405976, upload-time = "2025-06-13T06:52:22.995Z" }, + { url = "https://files.pythonhosted.org/packages/ee/97/88983e266572e8707c1f4b99c8fd04f9eb97b43f2db40e3172d87d8642db/msgpack-1.1.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b1ce7f41670c5a69e1389420436f41385b1aa2504c3b0c30620764b15dded2e7", size = 408607, upload-time = "2025-06-13T06:52:24.152Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/36c78af2efaffcc15a5a61ae0df53a1d025f2680122e2a9eb8442fed3ae4/msgpack-1.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4147151acabb9caed4e474c3344181e91ff7a388b888f1e19ea04f7e73dc7ad5", size = 424172, upload-time = "2025-06-13T06:52:25.704Z" }, + { url = "https://files.pythonhosted.org/packages/8c/87/a75eb622b555708fe0427fab96056d39d4c9892b0c784b3a721088c7ee37/msgpack-1.1.1-cp313-cp313-win32.whl", hash = "sha256:500e85823a27d6d9bba1d057c871b4210c1dd6fb01fbb764e37e4e8847376323", size = 65347, upload-time = "2025-06-13T06:52:26.846Z" }, + { url = "https://files.pythonhosted.org/packages/ca/91/7dc28d5e2a11a5ad804cf2b7f7a5fcb1eb5a4966d66a5d2b41aee6376543/msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69", size = 72341, upload-time = "2025-06-13T06:52:27.835Z" }, +] + +[[package]] +name = "multidict" +version = "6.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/7f/0652e6ed47ab288e3756ea9c0df8b14950781184d4bd7883f4d87dd41245/multidict-6.6.4.tar.gz", hash = "sha256:d2d4e4787672911b48350df02ed3fa3fffdc2f2e8ca06dd6afdf34189b76a9dd", size = 101843, upload-time = "2025-08-11T12:08:48.217Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/f6/512ffd8fd8b37fb2680e5ac35d788f1d71bbaf37789d21a820bdc441e565/multidict-6.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0ffb87be160942d56d7b87b0fdf098e81ed565add09eaa1294268c7f3caac4c8", size = 76516, upload-time = "2025-08-11T12:06:53.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/58/45c3e75deb8855c36bd66cc1658007589662ba584dbf423d01df478dd1c5/multidict-6.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d191de6cbab2aff5de6c5723101705fd044b3e4c7cfd587a1929b5028b9714b3", size = 45394, upload-time = "2025-08-11T12:06:54.555Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ca/e8c4472a93a26e4507c0b8e1f0762c0d8a32de1328ef72fd704ef9cc5447/multidict-6.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38a0956dd92d918ad5feff3db8fcb4a5eb7dba114da917e1a88475619781b57b", size = 43591, upload-time = "2025-08-11T12:06:55.672Z" }, + { url = "https://files.pythonhosted.org/packages/05/51/edf414f4df058574a7265034d04c935aa84a89e79ce90fcf4df211f47b16/multidict-6.6.4-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6865f6d3b7900ae020b495d599fcf3765653bc927951c1abb959017f81ae8287", size = 237215, upload-time = "2025-08-11T12:06:57.213Z" }, + { url = "https://files.pythonhosted.org/packages/c8/45/8b3d6dbad8cf3252553cc41abea09ad527b33ce47a5e199072620b296902/multidict-6.6.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a2088c126b6f72db6c9212ad827d0ba088c01d951cee25e758c450da732c138", size = 258299, upload-time = "2025-08-11T12:06:58.946Z" }, + { url = "https://files.pythonhosted.org/packages/3c/e8/8ca2e9a9f5a435fc6db40438a55730a4bf4956b554e487fa1b9ae920f825/multidict-6.6.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0f37bed7319b848097085d7d48116f545985db988e2256b2e6f00563a3416ee6", size = 242357, upload-time = "2025-08-11T12:07:00.301Z" }, + { url = "https://files.pythonhosted.org/packages/0f/84/80c77c99df05a75c28490b2af8f7cba2a12621186e0a8b0865d8e745c104/multidict-6.6.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:01368e3c94032ba6ca0b78e7ccb099643466cf24f8dc8eefcfdc0571d56e58f9", size = 268369, upload-time = "2025-08-11T12:07:01.638Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e9/920bfa46c27b05fb3e1ad85121fd49f441492dca2449c5bcfe42e4565d8a/multidict-6.6.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe323540c255db0bffee79ad7f048c909f2ab0edb87a597e1c17da6a54e493c", size = 269341, upload-time = "2025-08-11T12:07:02.943Z" }, + { url = "https://files.pythonhosted.org/packages/af/65/753a2d8b05daf496f4a9c367fe844e90a1b2cac78e2be2c844200d10cc4c/multidict-6.6.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8eb3025f17b0a4c3cd08cda49acf312a19ad6e8a4edd9dbd591e6506d999402", size = 256100, upload-time = "2025-08-11T12:07:04.564Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/655be13ae324212bf0bc15d665a4e34844f34c206f78801be42f7a0a8aaa/multidict-6.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bbc14f0365534d35a06970d6a83478b249752e922d662dc24d489af1aa0d1be7", size = 253584, upload-time = "2025-08-11T12:07:05.914Z" }, + { url = "https://files.pythonhosted.org/packages/5c/74/ab2039ecc05264b5cec73eb018ce417af3ebb384ae9c0e9ed42cb33f8151/multidict-6.6.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:75aa52fba2d96bf972e85451b99d8e19cc37ce26fd016f6d4aa60da9ab2b005f", size = 251018, upload-time = "2025-08-11T12:07:08.301Z" }, + { url = "https://files.pythonhosted.org/packages/af/0a/ccbb244ac848e56c6427f2392741c06302bbfba49c0042f1eb3c5b606497/multidict-6.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fefd4a815e362d4f011919d97d7b4a1e566f1dde83dc4ad8cfb5b41de1df68d", size = 251477, upload-time = "2025-08-11T12:07:10.248Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b0/0ed49bba775b135937f52fe13922bc64a7eaf0a3ead84a36e8e4e446e096/multidict-6.6.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:db9801fe021f59a5b375ab778973127ca0ac52429a26e2fd86aa9508f4d26eb7", size = 263575, upload-time = "2025-08-11T12:07:11.928Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d9/7fb85a85e14de2e44dfb6a24f03c41e2af8697a6df83daddb0e9b7569f73/multidict-6.6.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a650629970fa21ac1fb06ba25dabfc5b8a2054fcbf6ae97c758aa956b8dba802", size = 259649, upload-time = "2025-08-11T12:07:13.244Z" }, + { url = "https://files.pythonhosted.org/packages/03/9e/b3a459bcf9b6e74fa461a5222a10ff9b544cb1cd52fd482fb1b75ecda2a2/multidict-6.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:452ff5da78d4720d7516a3a2abd804957532dd69296cb77319c193e3ffb87e24", size = 251505, upload-time = "2025-08-11T12:07:14.57Z" }, + { url = "https://files.pythonhosted.org/packages/86/a2/8022f78f041dfe6d71e364001a5cf987c30edfc83c8a5fb7a3f0974cff39/multidict-6.6.4-cp312-cp312-win32.whl", hash = "sha256:8c2fcb12136530ed19572bbba61b407f655e3953ba669b96a35036a11a485793", size = 41888, upload-time = "2025-08-11T12:07:15.904Z" }, + { url = "https://files.pythonhosted.org/packages/c7/eb/d88b1780d43a56db2cba24289fa744a9d216c1a8546a0dc3956563fd53ea/multidict-6.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:047d9425860a8c9544fed1b9584f0c8bcd31bcde9568b047c5e567a1025ecd6e", size = 46072, upload-time = "2025-08-11T12:07:17.045Z" }, + { url = "https://files.pythonhosted.org/packages/9f/16/b929320bf5750e2d9d4931835a4c638a19d2494a5b519caaaa7492ebe105/multidict-6.6.4-cp312-cp312-win_arm64.whl", hash = "sha256:14754eb72feaa1e8ae528468f24250dd997b8e2188c3d2f593f9eba259e4b364", size = 43222, upload-time = "2025-08-11T12:07:18.328Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5d/e1db626f64f60008320aab00fbe4f23fc3300d75892a3381275b3d284580/multidict-6.6.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f46a6e8597f9bd71b31cc708195d42b634c8527fecbcf93febf1052cacc1f16e", size = 75848, upload-time = "2025-08-11T12:07:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/4c/aa/8b6f548d839b6c13887253af4e29c939af22a18591bfb5d0ee6f1931dae8/multidict-6.6.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:22e38b2bc176c5eb9c0a0e379f9d188ae4cd8b28c0f53b52bce7ab0a9e534657", size = 45060, upload-time = "2025-08-11T12:07:21.163Z" }, + { url = "https://files.pythonhosted.org/packages/eb/c6/f5e97e5d99a729bc2aa58eb3ebfa9f1e56a9b517cc38c60537c81834a73f/multidict-6.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5df8afd26f162da59e218ac0eefaa01b01b2e6cd606cffa46608f699539246da", size = 43269, upload-time = "2025-08-11T12:07:22.392Z" }, + { url = "https://files.pythonhosted.org/packages/dc/31/d54eb0c62516776f36fe67f84a732f97e0b0e12f98d5685bebcc6d396910/multidict-6.6.4-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:49517449b58d043023720aa58e62b2f74ce9b28f740a0b5d33971149553d72aa", size = 237158, upload-time = "2025-08-11T12:07:23.636Z" }, + { url = "https://files.pythonhosted.org/packages/c4/1c/8a10c1c25b23156e63b12165a929d8eb49a6ed769fdbefb06e6f07c1e50d/multidict-6.6.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae9408439537c5afdca05edd128a63f56a62680f4b3c234301055d7a2000220f", size = 257076, upload-time = "2025-08-11T12:07:25.049Z" }, + { url = "https://files.pythonhosted.org/packages/ad/86/90e20b5771d6805a119e483fd3d1e8393e745a11511aebca41f0da38c3e2/multidict-6.6.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:87a32d20759dc52a9e850fe1061b6e41ab28e2998d44168a8a341b99ded1dba0", size = 240694, upload-time = "2025-08-11T12:07:26.458Z" }, + { url = "https://files.pythonhosted.org/packages/e7/49/484d3e6b535bc0555b52a0a26ba86e4d8d03fd5587d4936dc59ba7583221/multidict-6.6.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:52e3c8d43cdfff587ceedce9deb25e6ae77daba560b626e97a56ddcad3756879", size = 266350, upload-time = "2025-08-11T12:07:27.94Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b4/aa4c5c379b11895083d50021e229e90c408d7d875471cb3abf721e4670d6/multidict-6.6.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ad8850921d3a8d8ff6fbef790e773cecfc260bbfa0566998980d3fa8f520bc4a", size = 267250, upload-time = "2025-08-11T12:07:29.303Z" }, + { url = "https://files.pythonhosted.org/packages/80/e5/5e22c5bf96a64bdd43518b1834c6d95a4922cc2066b7d8e467dae9b6cee6/multidict-6.6.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:497a2954adc25c08daff36f795077f63ad33e13f19bfff7736e72c785391534f", size = 254900, upload-time = "2025-08-11T12:07:30.764Z" }, + { url = "https://files.pythonhosted.org/packages/17/38/58b27fed927c07035abc02befacab42491e7388ca105e087e6e0215ead64/multidict-6.6.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:024ce601f92d780ca1617ad4be5ac15b501cc2414970ffa2bb2bbc2bd5a68fa5", size = 252355, upload-time = "2025-08-11T12:07:32.205Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a1/dad75d23a90c29c02b5d6f3d7c10ab36c3197613be5d07ec49c7791e186c/multidict-6.6.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a693fc5ed9bdd1c9e898013e0da4dcc640de7963a371c0bd458e50e046bf6438", size = 250061, upload-time = "2025-08-11T12:07:33.623Z" }, + { url = "https://files.pythonhosted.org/packages/b8/1a/ac2216b61c7f116edab6dc3378cca6c70dc019c9a457ff0d754067c58b20/multidict-6.6.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:190766dac95aab54cae5b152a56520fd99298f32a1266d66d27fdd1b5ac00f4e", size = 249675, upload-time = "2025-08-11T12:07:34.958Z" }, + { url = "https://files.pythonhosted.org/packages/d4/79/1916af833b800d13883e452e8e0977c065c4ee3ab7a26941fbfdebc11895/multidict-6.6.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:34d8f2a5ffdceab9dcd97c7a016deb2308531d5f0fced2bb0c9e1df45b3363d7", size = 261247, upload-time = "2025-08-11T12:07:36.588Z" }, + { url = "https://files.pythonhosted.org/packages/c5/65/d1f84fe08ac44a5fc7391cbc20a7cedc433ea616b266284413fd86062f8c/multidict-6.6.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:59e8d40ab1f5a8597abcef00d04845155a5693b5da00d2c93dbe88f2050f2812", size = 257960, upload-time = "2025-08-11T12:07:39.735Z" }, + { url = "https://files.pythonhosted.org/packages/13/b5/29ec78057d377b195ac2c5248c773703a6b602e132a763e20ec0457e7440/multidict-6.6.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:467fe64138cfac771f0e949b938c2e1ada2b5af22f39692aa9258715e9ea613a", size = 250078, upload-time = "2025-08-11T12:07:41.525Z" }, + { url = "https://files.pythonhosted.org/packages/c4/0e/7e79d38f70a872cae32e29b0d77024bef7834b0afb406ddae6558d9e2414/multidict-6.6.4-cp313-cp313-win32.whl", hash = "sha256:14616a30fe6d0a48d0a48d1a633ab3b8bec4cf293aac65f32ed116f620adfd69", size = 41708, upload-time = "2025-08-11T12:07:43.405Z" }, + { url = "https://files.pythonhosted.org/packages/9d/34/746696dffff742e97cd6a23da953e55d0ea51fa601fa2ff387b3edcfaa2c/multidict-6.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:40cd05eaeb39e2bc8939451f033e57feaa2ac99e07dbca8afe2be450a4a3b6cf", size = 45912, upload-time = "2025-08-11T12:07:45.082Z" }, + { url = "https://files.pythonhosted.org/packages/c7/87/3bac136181e271e29170d8d71929cdeddeb77f3e8b6a0c08da3a8e9da114/multidict-6.6.4-cp313-cp313-win_arm64.whl", hash = "sha256:f6eb37d511bfae9e13e82cb4d1af36b91150466f24d9b2b8a9785816deb16605", size = 43076, upload-time = "2025-08-11T12:07:46.746Z" }, + { url = "https://files.pythonhosted.org/packages/64/94/0a8e63e36c049b571c9ae41ee301ada29c3fee9643d9c2548d7d558a1d99/multidict-6.6.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:6c84378acd4f37d1b507dfa0d459b449e2321b3ba5f2338f9b085cf7a7ba95eb", size = 82812, upload-time = "2025-08-11T12:07:48.402Z" }, + { url = "https://files.pythonhosted.org/packages/25/1a/be8e369dfcd260d2070a67e65dd3990dd635cbd735b98da31e00ea84cd4e/multidict-6.6.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0e0558693063c75f3d952abf645c78f3c5dfdd825a41d8c4d8156fc0b0da6e7e", size = 48313, upload-time = "2025-08-11T12:07:49.679Z" }, + { url = "https://files.pythonhosted.org/packages/26/5a/dd4ade298674b2f9a7b06a32c94ffbc0497354df8285f27317c66433ce3b/multidict-6.6.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3f8e2384cb83ebd23fd07e9eada8ba64afc4c759cd94817433ab8c81ee4b403f", size = 46777, upload-time = "2025-08-11T12:07:51.318Z" }, + { url = "https://files.pythonhosted.org/packages/89/db/98aa28bc7e071bfba611ac2ae803c24e96dd3a452b4118c587d3d872c64c/multidict-6.6.4-cp313-cp313t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f996b87b420995a9174b2a7c1a8daf7db4750be6848b03eb5e639674f7963773", size = 229321, upload-time = "2025-08-11T12:07:52.965Z" }, + { url = "https://files.pythonhosted.org/packages/c7/bc/01ddda2a73dd9d167bd85d0e8ef4293836a8f82b786c63fb1a429bc3e678/multidict-6.6.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc356250cffd6e78416cf5b40dc6a74f1edf3be8e834cf8862d9ed5265cf9b0e", size = 249954, upload-time = "2025-08-11T12:07:54.423Z" }, + { url = "https://files.pythonhosted.org/packages/06/78/6b7c0f020f9aa0acf66d0ab4eb9f08375bac9a50ff5e3edb1c4ccd59eafc/multidict-6.6.4-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:dadf95aa862714ea468a49ad1e09fe00fcc9ec67d122f6596a8d40caf6cec7d0", size = 228612, upload-time = "2025-08-11T12:07:55.914Z" }, + { url = "https://files.pythonhosted.org/packages/00/44/3faa416f89b2d5d76e9d447296a81521e1c832ad6e40b92f990697b43192/multidict-6.6.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dd57515bebffd8ebd714d101d4c434063322e4fe24042e90ced41f18b6d3395", size = 257528, upload-time = "2025-08-11T12:07:57.371Z" }, + { url = "https://files.pythonhosted.org/packages/05/5f/77c03b89af0fcb16f018f668207768191fb9dcfb5e3361a5e706a11db2c9/multidict-6.6.4-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:967af5f238ebc2eb1da4e77af5492219fbd9b4b812347da39a7b5f5c72c0fa45", size = 256329, upload-time = "2025-08-11T12:07:58.844Z" }, + { url = "https://files.pythonhosted.org/packages/cf/e9/ed750a2a9afb4f8dc6f13dc5b67b514832101b95714f1211cd42e0aafc26/multidict-6.6.4-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a4c6875c37aae9794308ec43e3530e4aa0d36579ce38d89979bbf89582002bb", size = 247928, upload-time = "2025-08-11T12:08:01.037Z" }, + { url = "https://files.pythonhosted.org/packages/1f/b5/e0571bc13cda277db7e6e8a532791d4403dacc9850006cb66d2556e649c0/multidict-6.6.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:7f683a551e92bdb7fac545b9c6f9fa2aebdeefa61d607510b3533286fcab67f5", size = 245228, upload-time = "2025-08-11T12:08:02.96Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a3/69a84b0eccb9824491f06368f5b86e72e4af54c3067c37c39099b6687109/multidict-6.6.4-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:3ba5aaf600edaf2a868a391779f7a85d93bed147854925f34edd24cc70a3e141", size = 235869, upload-time = "2025-08-11T12:08:04.746Z" }, + { url = "https://files.pythonhosted.org/packages/a9/9d/28802e8f9121a6a0804fa009debf4e753d0a59969ea9f70be5f5fdfcb18f/multidict-6.6.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:580b643b7fd2c295d83cad90d78419081f53fd532d1f1eb67ceb7060f61cff0d", size = 243446, upload-time = "2025-08-11T12:08:06.332Z" }, + { url = "https://files.pythonhosted.org/packages/38/ea/6c98add069b4878c1d66428a5f5149ddb6d32b1f9836a826ac764b9940be/multidict-6.6.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:37b7187197da6af3ee0b044dbc9625afd0c885f2800815b228a0e70f9a7f473d", size = 252299, upload-time = "2025-08-11T12:08:07.931Z" }, + { url = "https://files.pythonhosted.org/packages/3a/09/8fe02d204473e14c0af3affd50af9078839dfca1742f025cca765435d6b4/multidict-6.6.4-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:e1b93790ed0bc26feb72e2f08299691ceb6da5e9e14a0d13cc74f1869af327a0", size = 246926, upload-time = "2025-08-11T12:08:09.467Z" }, + { url = "https://files.pythonhosted.org/packages/37/3d/7b1e10d774a6df5175ecd3c92bff069e77bed9ec2a927fdd4ff5fe182f67/multidict-6.6.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a506a77ddee1efcca81ecbeae27ade3e09cdf21a8ae854d766c2bb4f14053f92", size = 243383, upload-time = "2025-08-11T12:08:10.981Z" }, + { url = "https://files.pythonhosted.org/packages/50/b0/a6fae46071b645ae98786ab738447de1ef53742eaad949f27e960864bb49/multidict-6.6.4-cp313-cp313t-win32.whl", hash = "sha256:f93b2b2279883d1d0a9e1bd01f312d6fc315c5e4c1f09e112e4736e2f650bc4e", size = 47775, upload-time = "2025-08-11T12:08:12.439Z" }, + { url = "https://files.pythonhosted.org/packages/b2/0a/2436550b1520091af0600dff547913cb2d66fbac27a8c33bc1b1bccd8d98/multidict-6.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:6d46a180acdf6e87cc41dc15d8f5c2986e1e8739dc25dbb7dac826731ef381a4", size = 53100, upload-time = "2025-08-11T12:08:13.823Z" }, + { url = "https://files.pythonhosted.org/packages/97/ea/43ac51faff934086db9c072a94d327d71b7d8b40cd5dcb47311330929ef0/multidict-6.6.4-cp313-cp313t-win_arm64.whl", hash = "sha256:756989334015e3335d087a27331659820d53ba432befdef6a718398b0a8493ad", size = 45501, upload-time = "2025-08-11T12:08:15.173Z" }, + { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313, upload-time = "2025-08-11T12:08:46.891Z" }, +] + +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + +[[package]] +name = "networkx" +version = "3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, +] + +[[package]] +name = "numpy" +version = "2.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/7d/3fec4199c5ffb892bed55cff901e4f39a58c81df9c44c280499e92cad264/numpy-2.3.2.tar.gz", hash = "sha256:e0486a11ec30cdecb53f184d496d1c6a20786c81e55e41640270130056f8ee48", size = 20489306, upload-time = "2025-07-24T21:32:07.553Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/6d/745dd1c1c5c284d17725e5c802ca4d45cfc6803519d777f087b71c9f4069/numpy-2.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bc3186bea41fae9d8e90c2b4fb5f0a1f5a690682da79b92574d63f56b529080b", size = 20956420, upload-time = "2025-07-24T20:28:18.002Z" }, + { url = "https://files.pythonhosted.org/packages/bc/96/e7b533ea5740641dd62b07a790af5d9d8fec36000b8e2d0472bd7574105f/numpy-2.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f4f0215edb189048a3c03bd5b19345bdfa7b45a7a6f72ae5945d2a28272727f", size = 14184660, upload-time = "2025-07-24T20:28:39.522Z" }, + { url = "https://files.pythonhosted.org/packages/2b/53/102c6122db45a62aa20d1b18c9986f67e6b97e0d6fbc1ae13e3e4c84430c/numpy-2.3.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:8b1224a734cd509f70816455c3cffe13a4f599b1bf7130f913ba0e2c0b2006c0", size = 5113382, upload-time = "2025-07-24T20:28:48.544Z" }, + { url = "https://files.pythonhosted.org/packages/2b/21/376257efcbf63e624250717e82b4fae93d60178f09eb03ed766dbb48ec9c/numpy-2.3.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:3dcf02866b977a38ba3ec10215220609ab9667378a9e2150615673f3ffd6c73b", size = 6647258, upload-time = "2025-07-24T20:28:59.104Z" }, + { url = "https://files.pythonhosted.org/packages/91/ba/f4ebf257f08affa464fe6036e13f2bf9d4642a40228781dc1235da81be9f/numpy-2.3.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:572d5512df5470f50ada8d1972c5f1082d9a0b7aa5944db8084077570cf98370", size = 14281409, upload-time = "2025-07-24T20:40:30.298Z" }, + { url = "https://files.pythonhosted.org/packages/59/ef/f96536f1df42c668cbacb727a8c6da7afc9c05ece6d558927fb1722693e1/numpy-2.3.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8145dd6d10df13c559d1e4314df29695613575183fa2e2d11fac4c208c8a1f73", size = 16641317, upload-time = "2025-07-24T20:40:56.625Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a7/af813a7b4f9a42f498dde8a4c6fcbff8100eed00182cc91dbaf095645f38/numpy-2.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:103ea7063fa624af04a791c39f97070bf93b96d7af7eb23530cd087dc8dbe9dc", size = 16056262, upload-time = "2025-07-24T20:41:20.797Z" }, + { url = "https://files.pythonhosted.org/packages/8b/5d/41c4ef8404caaa7f05ed1cfb06afe16a25895260eacbd29b4d84dff2920b/numpy-2.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc927d7f289d14f5e037be917539620603294454130b6de200091e23d27dc9be", size = 18579342, upload-time = "2025-07-24T20:41:50.753Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4f/9950e44c5a11636f4a3af6e825ec23003475cc9a466edb7a759ed3ea63bd/numpy-2.3.2-cp312-cp312-win32.whl", hash = "sha256:d95f59afe7f808c103be692175008bab926b59309ade3e6d25009e9a171f7036", size = 6320610, upload-time = "2025-07-24T20:42:01.551Z" }, + { url = "https://files.pythonhosted.org/packages/7c/2f/244643a5ce54a94f0a9a2ab578189c061e4a87c002e037b0829dd77293b6/numpy-2.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:9e196ade2400c0c737d93465327d1ae7c06c7cb8a1756121ebf54b06ca183c7f", size = 12786292, upload-time = "2025-07-24T20:42:20.738Z" }, + { url = "https://files.pythonhosted.org/packages/54/cd/7b5f49d5d78db7badab22d8323c1b6ae458fbf86c4fdfa194ab3cd4eb39b/numpy-2.3.2-cp312-cp312-win_arm64.whl", hash = "sha256:ee807923782faaf60d0d7331f5e86da7d5e3079e28b291973c545476c2b00d07", size = 10194071, upload-time = "2025-07-24T20:42:36.657Z" }, + { url = "https://files.pythonhosted.org/packages/1c/c0/c6bb172c916b00700ed3bf71cb56175fd1f7dbecebf8353545d0b5519f6c/numpy-2.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c8d9727f5316a256425892b043736d63e89ed15bbfe6556c5ff4d9d4448ff3b3", size = 20949074, upload-time = "2025-07-24T20:43:07.813Z" }, + { url = "https://files.pythonhosted.org/packages/20/4e/c116466d22acaf4573e58421c956c6076dc526e24a6be0903219775d862e/numpy-2.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:efc81393f25f14d11c9d161e46e6ee348637c0a1e8a54bf9dedc472a3fae993b", size = 14177311, upload-time = "2025-07-24T20:43:29.335Z" }, + { url = "https://files.pythonhosted.org/packages/78/45/d4698c182895af189c463fc91d70805d455a227261d950e4e0f1310c2550/numpy-2.3.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:dd937f088a2df683cbb79dda9a772b62a3e5a8a7e76690612c2737f38c6ef1b6", size = 5106022, upload-time = "2025-07-24T20:43:37.999Z" }, + { url = "https://files.pythonhosted.org/packages/9f/76/3e6880fef4420179309dba72a8c11f6166c431cf6dee54c577af8906f914/numpy-2.3.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:11e58218c0c46c80509186e460d79fbdc9ca1eb8d8aee39d8f2dc768eb781089", size = 6640135, upload-time = "2025-07-24T20:43:49.28Z" }, + { url = "https://files.pythonhosted.org/packages/34/fa/87ff7f25b3c4ce9085a62554460b7db686fef1e0207e8977795c7b7d7ba1/numpy-2.3.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5ad4ebcb683a1f99f4f392cc522ee20a18b2bb12a2c1c42c3d48d5a1adc9d3d2", size = 14278147, upload-time = "2025-07-24T20:44:10.328Z" }, + { url = "https://files.pythonhosted.org/packages/1d/0f/571b2c7a3833ae419fe69ff7b479a78d313581785203cc70a8db90121b9a/numpy-2.3.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:938065908d1d869c7d75d8ec45f735a034771c6ea07088867f713d1cd3bbbe4f", size = 16635989, upload-time = "2025-07-24T20:44:34.88Z" }, + { url = "https://files.pythonhosted.org/packages/24/5a/84ae8dca9c9a4c592fe11340b36a86ffa9fd3e40513198daf8a97839345c/numpy-2.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:66459dccc65d8ec98cc7df61307b64bf9e08101f9598755d42d8ae65d9a7a6ee", size = 16053052, upload-time = "2025-07-24T20:44:58.872Z" }, + { url = "https://files.pythonhosted.org/packages/57/7c/e5725d99a9133b9813fcf148d3f858df98511686e853169dbaf63aec6097/numpy-2.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a7af9ed2aa9ec5950daf05bb11abc4076a108bd3c7db9aa7251d5f107079b6a6", size = 18577955, upload-time = "2025-07-24T20:45:26.714Z" }, + { url = "https://files.pythonhosted.org/packages/ae/11/7c546fcf42145f29b71e4d6f429e96d8d68e5a7ba1830b2e68d7418f0bbd/numpy-2.3.2-cp313-cp313-win32.whl", hash = "sha256:906a30249315f9c8e17b085cc5f87d3f369b35fedd0051d4a84686967bdbbd0b", size = 6311843, upload-time = "2025-07-24T20:49:24.444Z" }, + { url = "https://files.pythonhosted.org/packages/aa/6f/a428fd1cb7ed39b4280d057720fed5121b0d7754fd2a9768640160f5517b/numpy-2.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:c63d95dc9d67b676e9108fe0d2182987ccb0f11933c1e8959f42fa0da8d4fa56", size = 12782876, upload-time = "2025-07-24T20:49:43.227Z" }, + { url = "https://files.pythonhosted.org/packages/65/85/4ea455c9040a12595fb6c43f2c217257c7b52dd0ba332c6a6c1d28b289fe/numpy-2.3.2-cp313-cp313-win_arm64.whl", hash = "sha256:b05a89f2fb84d21235f93de47129dd4f11c16f64c87c33f5e284e6a3a54e43f2", size = 10192786, upload-time = "2025-07-24T20:49:59.443Z" }, + { url = "https://files.pythonhosted.org/packages/80/23/8278f40282d10c3f258ec3ff1b103d4994bcad78b0cba9208317f6bb73da/numpy-2.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4e6ecfeddfa83b02318f4d84acf15fbdbf9ded18e46989a15a8b6995dfbf85ab", size = 21047395, upload-time = "2025-07-24T20:45:58.821Z" }, + { url = "https://files.pythonhosted.org/packages/1f/2d/624f2ce4a5df52628b4ccd16a4f9437b37c35f4f8a50d00e962aae6efd7a/numpy-2.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:508b0eada3eded10a3b55725b40806a4b855961040180028f52580c4729916a2", size = 14300374, upload-time = "2025-07-24T20:46:20.207Z" }, + { url = "https://files.pythonhosted.org/packages/f6/62/ff1e512cdbb829b80a6bd08318a58698867bca0ca2499d101b4af063ee97/numpy-2.3.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:754d6755d9a7588bdc6ac47dc4ee97867271b17cee39cb87aef079574366db0a", size = 5228864, upload-time = "2025-07-24T20:46:30.58Z" }, + { url = "https://files.pythonhosted.org/packages/7d/8e/74bc18078fff03192d4032cfa99d5a5ca937807136d6f5790ce07ca53515/numpy-2.3.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:a9f66e7d2b2d7712410d3bc5684149040ef5f19856f20277cd17ea83e5006286", size = 6737533, upload-time = "2025-07-24T20:46:46.111Z" }, + { url = "https://files.pythonhosted.org/packages/19/ea/0731efe2c9073ccca5698ef6a8c3667c4cf4eea53fcdcd0b50140aba03bc/numpy-2.3.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de6ea4e5a65d5a90c7d286ddff2b87f3f4ad61faa3db8dabe936b34c2275b6f8", size = 14352007, upload-time = "2025-07-24T20:47:07.1Z" }, + { url = "https://files.pythonhosted.org/packages/cf/90/36be0865f16dfed20f4bc7f75235b963d5939707d4b591f086777412ff7b/numpy-2.3.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3ef07ec8cbc8fc9e369c8dcd52019510c12da4de81367d8b20bc692aa07573a", size = 16701914, upload-time = "2025-07-24T20:47:32.459Z" }, + { url = "https://files.pythonhosted.org/packages/94/30/06cd055e24cb6c38e5989a9e747042b4e723535758e6153f11afea88c01b/numpy-2.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:27c9f90e7481275c7800dc9c24b7cc40ace3fdb970ae4d21eaff983a32f70c91", size = 16132708, upload-time = "2025-07-24T20:47:58.129Z" }, + { url = "https://files.pythonhosted.org/packages/9a/14/ecede608ea73e58267fd7cb78f42341b3b37ba576e778a1a06baffbe585c/numpy-2.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:07b62978075b67eee4065b166d000d457c82a1efe726cce608b9db9dd66a73a5", size = 18651678, upload-time = "2025-07-24T20:48:25.402Z" }, + { url = "https://files.pythonhosted.org/packages/40/f3/2fe6066b8d07c3685509bc24d56386534c008b462a488b7f503ba82b8923/numpy-2.3.2-cp313-cp313t-win32.whl", hash = "sha256:c771cfac34a4f2c0de8e8c97312d07d64fd8f8ed45bc9f5726a7e947270152b5", size = 6441832, upload-time = "2025-07-24T20:48:37.181Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ba/0937d66d05204d8f28630c9c60bc3eda68824abde4cf756c4d6aad03b0c6/numpy-2.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:72dbebb2dcc8305c431b2836bcc66af967df91be793d63a24e3d9b741374c450", size = 12927049, upload-time = "2025-07-24T20:48:56.24Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ed/13542dd59c104d5e654dfa2ac282c199ba64846a74c2c4bcdbc3a0f75df1/numpy-2.3.2-cp313-cp313t-win_arm64.whl", hash = "sha256:72c6df2267e926a6d5286b0a6d556ebe49eae261062059317837fda12ddf0c1a", size = 10262935, upload-time = "2025-07-24T20:49:13.136Z" }, + { url = "https://files.pythonhosted.org/packages/c9/7c/7659048aaf498f7611b783e000c7268fcc4dcf0ce21cd10aad7b2e8f9591/numpy-2.3.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:448a66d052d0cf14ce9865d159bfc403282c9bc7bb2a31b03cc18b651eca8b1a", size = 20950906, upload-time = "2025-07-24T20:50:30.346Z" }, + { url = "https://files.pythonhosted.org/packages/80/db/984bea9d4ddf7112a04cfdfb22b1050af5757864cfffe8e09e44b7f11a10/numpy-2.3.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:546aaf78e81b4081b2eba1d105c3b34064783027a06b3ab20b6eba21fb64132b", size = 14185607, upload-time = "2025-07-24T20:50:51.923Z" }, + { url = "https://files.pythonhosted.org/packages/e4/76/b3d6f414f4eca568f469ac112a3b510938d892bc5a6c190cb883af080b77/numpy-2.3.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:87c930d52f45df092f7578889711a0768094debf73cfcde105e2d66954358125", size = 5114110, upload-time = "2025-07-24T20:51:01.041Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d2/6f5e6826abd6bca52392ed88fe44a4b52aacb60567ac3bc86c67834c3a56/numpy-2.3.2-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:8dc082ea901a62edb8f59713c6a7e28a85daddcb67454c839de57656478f5b19", size = 6642050, upload-time = "2025-07-24T20:51:11.64Z" }, + { url = "https://files.pythonhosted.org/packages/c4/43/f12b2ade99199e39c73ad182f103f9d9791f48d885c600c8e05927865baf/numpy-2.3.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af58de8745f7fa9ca1c0c7c943616c6fe28e75d0c81f5c295810e3c83b5be92f", size = 14296292, upload-time = "2025-07-24T20:51:33.488Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f9/77c07d94bf110a916b17210fac38680ed8734c236bfed9982fd8524a7b47/numpy-2.3.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed5527c4cf10f16c6d0b6bee1f89958bccb0ad2522c8cadc2efd318bcd545f5", size = 16638913, upload-time = "2025-07-24T20:51:58.517Z" }, + { url = "https://files.pythonhosted.org/packages/9b/d1/9d9f2c8ea399cc05cfff8a7437453bd4e7d894373a93cdc46361bbb49a7d/numpy-2.3.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:095737ed986e00393ec18ec0b21b47c22889ae4b0cd2d5e88342e08b01141f58", size = 16071180, upload-time = "2025-07-24T20:52:22.827Z" }, + { url = "https://files.pythonhosted.org/packages/4c/41/82e2c68aff2a0c9bf315e47d61951099fed65d8cb2c8d9dc388cb87e947e/numpy-2.3.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b5e40e80299607f597e1a8a247ff8d71d79c5b52baa11cc1cce30aa92d2da6e0", size = 18576809, upload-time = "2025-07-24T20:52:51.015Z" }, + { url = "https://files.pythonhosted.org/packages/14/14/4b4fd3efb0837ed252d0f583c5c35a75121038a8c4e065f2c259be06d2d8/numpy-2.3.2-cp314-cp314-win32.whl", hash = "sha256:7d6e390423cc1f76e1b8108c9b6889d20a7a1f59d9a60cac4a050fa734d6c1e2", size = 6366410, upload-time = "2025-07-24T20:56:44.949Z" }, + { url = "https://files.pythonhosted.org/packages/11/9e/b4c24a6b8467b61aced5c8dc7dcfce23621baa2e17f661edb2444a418040/numpy-2.3.2-cp314-cp314-win_amd64.whl", hash = "sha256:b9d0878b21e3918d76d2209c924ebb272340da1fb51abc00f986c258cd5e957b", size = 12918821, upload-time = "2025-07-24T20:57:06.479Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0f/0dc44007c70b1007c1cef86b06986a3812dd7106d8f946c09cfa75782556/numpy-2.3.2-cp314-cp314-win_arm64.whl", hash = "sha256:2738534837c6a1d0c39340a190177d7d66fdf432894f469728da901f8f6dc910", size = 10477303, upload-time = "2025-07-24T20:57:22.879Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3e/075752b79140b78ddfc9c0a1634d234cfdbc6f9bbbfa6b7504e445ad7d19/numpy-2.3.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:4d002ecf7c9b53240be3bb69d80f86ddbd34078bae04d87be81c1f58466f264e", size = 21047524, upload-time = "2025-07-24T20:53:22.086Z" }, + { url = "https://files.pythonhosted.org/packages/fe/6d/60e8247564a72426570d0e0ea1151b95ce5bd2f1597bb878a18d32aec855/numpy-2.3.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:293b2192c6bcce487dbc6326de5853787f870aeb6c43f8f9c6496db5b1781e45", size = 14300519, upload-time = "2025-07-24T20:53:44.053Z" }, + { url = "https://files.pythonhosted.org/packages/4d/73/d8326c442cd428d47a067070c3ac6cc3b651a6e53613a1668342a12d4479/numpy-2.3.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:0a4f2021a6da53a0d580d6ef5db29947025ae8b35b3250141805ea9a32bbe86b", size = 5228972, upload-time = "2025-07-24T20:53:53.81Z" }, + { url = "https://files.pythonhosted.org/packages/34/2e/e71b2d6dad075271e7079db776196829019b90ce3ece5c69639e4f6fdc44/numpy-2.3.2-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:9c144440db4bf3bb6372d2c3e49834cc0ff7bb4c24975ab33e01199e645416f2", size = 6737439, upload-time = "2025-07-24T20:54:04.742Z" }, + { url = "https://files.pythonhosted.org/packages/15/b0/d004bcd56c2c5e0500ffc65385eb6d569ffd3363cb5e593ae742749b2daa/numpy-2.3.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f92d6c2a8535dc4fe4419562294ff957f83a16ebdec66df0805e473ffaad8bd0", size = 14352479, upload-time = "2025-07-24T20:54:25.819Z" }, + { url = "https://files.pythonhosted.org/packages/11/e3/285142fcff8721e0c99b51686426165059874c150ea9ab898e12a492e291/numpy-2.3.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cefc2219baa48e468e3db7e706305fcd0c095534a192a08f31e98d83a7d45fb0", size = 16702805, upload-time = "2025-07-24T20:54:50.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/c3/33b56b0e47e604af2c7cd065edca892d180f5899599b76830652875249a3/numpy-2.3.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:76c3e9501ceb50b2ff3824c3589d5d1ab4ac857b0ee3f8f49629d0de55ecf7c2", size = 16133830, upload-time = "2025-07-24T20:55:17.306Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ae/7b1476a1f4d6a48bc669b8deb09939c56dd2a439db1ab03017844374fb67/numpy-2.3.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:122bf5ed9a0221b3419672493878ba4967121514b1d7d4656a7580cd11dddcbf", size = 18652665, upload-time = "2025-07-24T20:55:46.665Z" }, + { url = "https://files.pythonhosted.org/packages/14/ba/5b5c9978c4bb161034148ade2de9db44ec316fab89ce8c400db0e0c81f86/numpy-2.3.2-cp314-cp314t-win32.whl", hash = "sha256:6f1ae3dcb840edccc45af496f312528c15b1f79ac318169d094e85e4bb35fdf1", size = 6514777, upload-time = "2025-07-24T20:55:57.66Z" }, + { url = "https://files.pythonhosted.org/packages/eb/46/3dbaf0ae7c17cdc46b9f662c56da2054887b8d9e737c1476f335c83d33db/numpy-2.3.2-cp314-cp314t-win_amd64.whl", hash = "sha256:087ffc25890d89a43536f75c5fe8770922008758e8eeeef61733957041ed2f9b", size = 13111856, upload-time = "2025-07-24T20:56:17.318Z" }, + { url = "https://files.pythonhosted.org/packages/c1/9e/1652778bce745a67b5fe05adde60ed362d38eb17d919a540e813d30f6874/numpy-2.3.2-cp314-cp314t-win_arm64.whl", hash = "sha256:092aeb3449833ea9c0bf0089d70c29ae480685dd2377ec9cdbbb620257f84631", size = 10544226, upload-time = "2025-07-24T20:56:34.509Z" }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.6.4.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/eb/ff4b8c503fa1f1796679dce648854d58751982426e4e4b37d6fce49d259c/nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08ed2686e9875d01b58e3cb379c6896df8e76c75e0d4a7f7dace3d7b6d9ef8eb", size = 393138322, upload-time = "2024-11-20T17:40:25.65Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.6.80" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/60/7b6497946d74bcf1de852a21824d63baad12cd417db4195fc1bfe59db953/nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6768bad6cab4f19e8292125e5f1ac8aa7d1718704012a0e3272a6f61c4bce132", size = 8917980, upload-time = "2024-11-20T17:36:04.019Z" }, + { url = "https://files.pythonhosted.org/packages/a5/24/120ee57b218d9952c379d1e026c4479c9ece9997a4fb46303611ee48f038/nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a3eff6cdfcc6a4c35db968a06fcadb061cbc7d6dde548609a941ff8701b98b73", size = 8917972, upload-time = "2024-10-01T16:58:06.036Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.6.77" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/2e/46030320b5a80661e88039f59060d1790298b4718944a65a7f2aeda3d9e9/nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:35b0cc6ee3a9636d5409133e79273ce1f3fd087abb0532d2d2e8fff1fe9efc53", size = 23650380, upload-time = "2024-10-01T17:00:14.643Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.6.77" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/23/e717c5ac26d26cf39a27fbc076240fad2e3b817e5889d671b67f4f9f49c5/nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba3b56a4f896141e25e19ab287cd71e52a6a0f4b29d0d31609f60e3b4d5219b7", size = 897690, upload-time = "2024-11-20T17:35:30.697Z" }, + { url = "https://files.pythonhosted.org/packages/f0/62/65c05e161eeddbafeca24dc461f47de550d9fa8a7e04eb213e32b55cfd99/nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a84d15d5e1da416dd4774cb42edf5e954a3e60cc945698dc1d5be02321c44dc8", size = 897678, upload-time = "2024-10-01T16:57:33.821Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.5.1.17" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12", marker = "sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/78/4535c9c7f859a64781e43c969a3a7e84c54634e319a996d43ef32ce46f83/nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2", size = 570988386, upload-time = "2024-10-25T19:54:26.39Z" }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/16/73727675941ab8e6ffd86ca3a4b7b47065edcca7a997920b831f8147c99d/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ccba62eb9cef5559abd5e0d54ceed2d9934030f51163df018532142a8ec533e5", size = 200221632, upload-time = "2024-11-20T17:41:32.357Z" }, + { url = "https://files.pythonhosted.org/packages/60/de/99ec247a07ea40c969d904fc14f3a356b3e2a704121675b75c366b694ee1/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.whl", hash = "sha256:768160ac89f6f7b459bee747e8d175dbf53619cfe74b2a5636264163138013ca", size = 200221622, upload-time = "2024-10-01T17:03:58.79Z" }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.11.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/66/cc9876340ac68ae71b15c743ddb13f8b30d5244af344ec8322b449e35426/nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc23469d1c7e52ce6c1d55253273d32c565dd22068647f3aa59b3c6b005bf159", size = 1142103, upload-time = "2024-11-20T17:42:11.83Z" }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.7.77" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/1b/44a01c4e70933637c93e6e1a8063d1e998b50213a6b65ac5a9169c47e98e/nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a42cd1344297f70b9e39a1e4f467a4e1c10f1da54ff7a85c12197f6c652c8bdf", size = 56279010, upload-time = "2024-11-20T17:42:50.958Z" }, + { url = "https://files.pythonhosted.org/packages/4a/aa/2c7ff0b5ee02eaef890c0ce7d4f74bc30901871c5e45dee1ae6d0083cd80/nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:99f1a32f1ac2bd134897fc7a203f779303261268a65762a623bf30cc9fe79117", size = 56279000, upload-time = "2024-10-01T17:04:45.274Z" }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12", marker = "sys_platform != 'win32'" }, + { name = "nvidia-cusparse-cu12", marker = "sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/6e/c2cf12c9ff8b872e92b4a5740701e51ff17689c4d726fca91875b07f655d/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c", size = 158229790, upload-time = "2024-11-20T17:43:43.211Z" }, + { url = "https://files.pythonhosted.org/packages/9f/81/baba53585da791d043c10084cf9553e074548408e04ae884cfe9193bd484/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6cf28f17f64107a0c4d7802be5ff5537b2130bfc112f25d5a30df227058ca0e6", size = 158229780, upload-time = "2024-10-01T17:05:39.875Z" }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/1e/b8b7c2f4099a37b96af5c9bb158632ea9e5d9d27d7391d7eb8fc45236674/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7556d9eca156e18184b94947ade0fba5bb47d69cec46bf8660fd2c71a4b48b73", size = 216561367, upload-time = "2024-11-20T17:44:54.824Z" }, + { url = "https://files.pythonhosted.org/packages/43/ac/64c4316ba163e8217a99680c7605f779accffc6a4bcd0c778c12948d3707/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:23749a6571191a215cb74d1cdbff4a86e7b19f1200c071b3fcf844a5bea23a2f", size = 216561357, upload-time = "2024-10-01T17:06:29.861Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/9a/72ef35b399b0e183bc2e8f6f558036922d453c4d8237dab26c666a04244b/nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46", size = 156785796, upload-time = "2024-10-15T21:29:17.709Z" }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.26.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/ca/f42388aed0fddd64ade7493dbba36e1f534d4e6fdbdd355c6a90030ae028/nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6", size = 201319755, upload-time = "2025-03-13T00:29:55.296Z" }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.6.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/d7/c5383e47c7e9bf1c99d5bd2a8c935af2b6d705ad831a7ec5c97db4d82f4f/nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:eedc36df9e88b682efe4309aa16b5b4e78c2407eac59e8c10a6a47535164369a", size = 19744971, upload-time = "2024-11-20T17:46:53.366Z" }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.6.77" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/9a/fff8376f8e3d084cd1530e1ef7b879bb7d6d265620c95c1b322725c694f4/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b90bed3df379fa79afbd21be8e04a0314336b8ae16768b58f2d34cb1d04cd7d2", size = 89276, upload-time = "2024-11-20T17:38:27.621Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4e/0d0c945463719429b7bd21dece907ad0bde437a2ff12b9b12fee94722ab0/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6574241a3ec5fdc9334353ab8c479fe75841dbe8f4532a8fc97ce63503330ba1", size = 89265, upload-time = "2024-10-01T17:00:38.172Z" }, +] + +[[package]] +name = "opencensus" +version = "0.11.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "opencensus-context" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/15/a7/a46dcffa1b63084f9f17fe3c8cb20724c4c8f91009fd0b2cfdb27d5d2b35/opencensus-0.11.4.tar.gz", hash = "sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2", size = 64966, upload-time = "2024-01-03T18:04:07.085Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/ed/9fbdeb23a09e430d87b7d72d430484b88184633dc50f6bfb792354b6f661/opencensus-0.11.4-py2.py3-none-any.whl", hash = "sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864", size = 128225, upload-time = "2024-01-03T18:04:05.127Z" }, +] + +[[package]] +name = "opencensus-context" +version = "0.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/96/3b6f638f6275a8abbd45e582448723bffa29c1fb426721dedb5c72f7d056/opencensus-context-0.1.3.tar.gz", hash = "sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c", size = 4066, upload-time = "2022-08-03T22:20:22.359Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/68/162c97ea78c957d68ecf78a5c5041d2e25bd5562bdf5d89a6cbf7f8429bf/opencensus_context-0.1.3-py2.py3-none-any.whl", hash = "sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039", size = 5060, upload-time = "2022-08-03T22:20:20.352Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/d2/c782c88b8afbf961d6972428821c302bd1e9e7bc361352172f0ca31296e2/opentelemetry_api-1.36.0.tar.gz", hash = "sha256:9a72572b9c416d004d492cbc6e61962c0501eaf945ece9b5a0f56597d8348aa0", size = 64780, upload-time = "2025-07-29T15:12:06.02Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/ee/6b08dde0a022c463b88f55ae81149584b125a42183407dc1045c486cc870/opentelemetry_api-1.36.0-py3-none-any.whl", hash = "sha256:02f20bcacf666e1333b6b1f04e647dc1d5111f86b8e510238fcc56d7762cda8c", size = 65564, upload-time = "2025-07-29T15:11:47.998Z" }, +] + +[[package]] +name = "opentelemetry-exporter-prometheus" +version = "0.57b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, + { name = "prometheus-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d6/d8/5f04c6d51c0823c3d8ac973a2a38db6fcf2d040ca3f08fc66b3c14b6e164/opentelemetry_exporter_prometheus-0.57b0.tar.gz", hash = "sha256:9eb15bdc189235cf03c3f93abf56f8ff0ab57a493a189263bd7fe77a4249e689", size = 14906, upload-time = "2025-07-29T15:12:09.96Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/1c/40fb93a7b7e495985393bbc734104d5d20e470811644dd56c2402d683739/opentelemetry_exporter_prometheus-0.57b0-py3-none-any.whl", hash = "sha256:c5b893d1cdd593fb022af2c7de3258c2d5a4d04402ae80d9fa35675fed77f05c", size = 12922, upload-time = "2025-07-29T15:11:54.055Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/02/f6556142301d136e3b7e95ab8ea6a5d9dc28d879a99f3dd673b5f97dca06/opentelemetry_proto-1.36.0.tar.gz", hash = "sha256:0f10b3c72f74c91e0764a5ec88fd8f1c368ea5d9c64639fb455e2854ef87dd2f", size = 46152, upload-time = "2025-07-29T15:12:15.717Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/57/3361e06136225be8180e879199caea520f38026f8071366241ac458beb8d/opentelemetry_proto-1.36.0-py3-none-any.whl", hash = "sha256:151b3bf73a09f94afc658497cf77d45a565606f62ce0c17acb08cd9937ca206e", size = 72537, upload-time = "2025-07-29T15:12:02.243Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/85/8567a966b85a2d3f971c4d42f781c305b2b91c043724fa08fd37d158e9dc/opentelemetry_sdk-1.36.0.tar.gz", hash = "sha256:19c8c81599f51b71670661ff7495c905d8fdf6976e41622d5245b791b06fa581", size = 162557, upload-time = "2025-07-29T15:12:16.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/59/7bed362ad1137ba5886dac8439e84cd2df6d087be7c09574ece47ae9b22c/opentelemetry_sdk-1.36.0-py3-none-any.whl", hash = "sha256:19fe048b42e98c5c1ffe85b569b7073576ad4ce0bcb6e9b4c6a39e890a6c45fb", size = 119995, upload-time = "2025-07-29T15:12:03.181Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.57b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/31/67dfa252ee88476a29200b0255bda8dfc2cf07b56ad66dc9a6221f7dc787/opentelemetry_semantic_conventions-0.57b0.tar.gz", hash = "sha256:609a4a79c7891b4620d64c7aac6898f872d790d75f22019913a660756f27ff32", size = 124225, upload-time = "2025-07-29T15:12:17.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/75/7d591371c6c39c73de5ce5da5a2cc7b72d1d1cd3f8f4638f553c01c37b11/opentelemetry_semantic_conventions-0.57b0-py3-none-any.whl", hash = "sha256:757f7e76293294f124c827e514c2a3144f191ef175b069ce8d1211e1e38e9e78", size = 201627, upload-time = "2025-07-29T15:12:04.174Z" }, +] + +[[package]] +name = "packaging" +version = "24.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/63/68dbb6eb2de9cb10ee4c9c14a0148804425e13c4fb20d61cce69f53106da/packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f", size = 163950, upload-time = "2024-11-08T09:47:47.202Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451, upload-time = "2024-11-08T09:47:44.722Z" }, +] + +[[package]] +name = "pandas" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/6f/75aa71f8a14267117adeeed5d21b204770189c0a0025acbdc03c337b28fc/pandas-2.3.1.tar.gz", hash = "sha256:0a95b9ac964fe83ce317827f80304d37388ea77616b1425f0ae41c9d2d0d7bb2", size = 4487493, upload-time = "2025-07-07T19:20:04.079Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/de/b8445e0f5d217a99fe0eeb2f4988070908979bec3587c0633e5428ab596c/pandas-2.3.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:689968e841136f9e542020698ee1c4fbe9caa2ed2213ae2388dc7b81721510d3", size = 11588172, upload-time = "2025-07-07T19:18:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/1e/e0/801cdb3564e65a5ac041ab99ea6f1d802a6c325bb6e58c79c06a3f1cd010/pandas-2.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:025e92411c16cbe5bb2a4abc99732a6b132f439b8aab23a59fa593eb00704232", size = 10717365, upload-time = "2025-07-07T19:18:54.785Z" }, + { url = "https://files.pythonhosted.org/packages/51/a5/c76a8311833c24ae61a376dbf360eb1b1c9247a5d9c1e8b356563b31b80c/pandas-2.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b7ff55f31c4fcb3e316e8f7fa194566b286d6ac430afec0d461163312c5841e", size = 11280411, upload-time = "2025-07-07T19:18:57.045Z" }, + { url = "https://files.pythonhosted.org/packages/da/01/e383018feba0a1ead6cf5fe8728e5d767fee02f06a3d800e82c489e5daaf/pandas-2.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7dcb79bf373a47d2a40cf7232928eb7540155abbc460925c2c96d2d30b006eb4", size = 11988013, upload-time = "2025-07-07T19:18:59.771Z" }, + { url = "https://files.pythonhosted.org/packages/5b/14/cec7760d7c9507f11c97d64f29022e12a6cc4fc03ac694535e89f88ad2ec/pandas-2.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56a342b231e8862c96bdb6ab97170e203ce511f4d0429589c8ede1ee8ece48b8", size = 12767210, upload-time = "2025-07-07T19:19:02.944Z" }, + { url = "https://files.pythonhosted.org/packages/50/b9/6e2d2c6728ed29fb3d4d4d302504fb66f1a543e37eb2e43f352a86365cdf/pandas-2.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ca7ed14832bce68baef331f4d7f294411bed8efd032f8109d690df45e00c4679", size = 13440571, upload-time = "2025-07-07T19:19:06.82Z" }, + { url = "https://files.pythonhosted.org/packages/80/a5/3a92893e7399a691bad7664d977cb5e7c81cf666c81f89ea76ba2bff483d/pandas-2.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:ac942bfd0aca577bef61f2bc8da8147c4ef6879965ef883d8e8d5d2dc3e744b8", size = 10987601, upload-time = "2025-07-07T19:19:09.589Z" }, + { url = "https://files.pythonhosted.org/packages/32/ed/ff0a67a2c5505e1854e6715586ac6693dd860fbf52ef9f81edee200266e7/pandas-2.3.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9026bd4a80108fac2239294a15ef9003c4ee191a0f64b90f170b40cfb7cf2d22", size = 11531393, upload-time = "2025-07-07T19:19:12.245Z" }, + { url = "https://files.pythonhosted.org/packages/c7/db/d8f24a7cc9fb0972adab0cc80b6817e8bef888cfd0024eeb5a21c0bb5c4a/pandas-2.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6de8547d4fdb12421e2d047a2c446c623ff4c11f47fddb6b9169eb98ffba485a", size = 10668750, upload-time = "2025-07-07T19:19:14.612Z" }, + { url = "https://files.pythonhosted.org/packages/0f/b0/80f6ec783313f1e2356b28b4fd8d2148c378370045da918c73145e6aab50/pandas-2.3.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:782647ddc63c83133b2506912cc6b108140a38a37292102aaa19c81c83db2928", size = 11342004, upload-time = "2025-07-07T19:19:16.857Z" }, + { url = "https://files.pythonhosted.org/packages/e9/e2/20a317688435470872885e7fc8f95109ae9683dec7c50be29b56911515a5/pandas-2.3.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ba6aff74075311fc88504b1db890187a3cd0f887a5b10f5525f8e2ef55bfdb9", size = 12050869, upload-time = "2025-07-07T19:19:19.265Z" }, + { url = "https://files.pythonhosted.org/packages/55/79/20d746b0a96c67203a5bee5fb4e00ac49c3e8009a39e1f78de264ecc5729/pandas-2.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e5635178b387bd2ba4ac040f82bc2ef6e6b500483975c4ebacd34bec945fda12", size = 12750218, upload-time = "2025-07-07T19:19:21.547Z" }, + { url = "https://files.pythonhosted.org/packages/7c/0f/145c8b41e48dbf03dd18fdd7f24f8ba95b8254a97a3379048378f33e7838/pandas-2.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6f3bf5ec947526106399a9e1d26d40ee2b259c66422efdf4de63c848492d91bb", size = 13416763, upload-time = "2025-07-07T19:19:23.939Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c0/54415af59db5cdd86a3d3bf79863e8cc3fa9ed265f0745254061ac09d5f2/pandas-2.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:1c78cf43c8fde236342a1cb2c34bcff89564a7bfed7e474ed2fffa6aed03a956", size = 10987482, upload-time = "2025-07-07T19:19:42.699Z" }, + { url = "https://files.pythonhosted.org/packages/48/64/2fd2e400073a1230e13b8cd604c9bc95d9e3b962e5d44088ead2e8f0cfec/pandas-2.3.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8dfc17328e8da77be3cf9f47509e5637ba8f137148ed0e9b5241e1baf526e20a", size = 12029159, upload-time = "2025-07-07T19:19:26.362Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0a/d84fd79b0293b7ef88c760d7dca69828d867c89b6d9bc52d6a27e4d87316/pandas-2.3.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ec6c851509364c59a5344458ab935e6451b31b818be467eb24b0fe89bd05b6b9", size = 11393287, upload-time = "2025-07-07T19:19:29.157Z" }, + { url = "https://files.pythonhosted.org/packages/50/ae/ff885d2b6e88f3c7520bb74ba319268b42f05d7e583b5dded9837da2723f/pandas-2.3.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:911580460fc4884d9b05254b38a6bfadddfcc6aaef856fb5859e7ca202e45275", size = 11309381, upload-time = "2025-07-07T19:19:31.436Z" }, + { url = "https://files.pythonhosted.org/packages/85/86/1fa345fc17caf5d7780d2699985c03dbe186c68fee00b526813939062bb0/pandas-2.3.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f4d6feeba91744872a600e6edbbd5b033005b431d5ae8379abee5bcfa479fab", size = 11883998, upload-time = "2025-07-07T19:19:34.267Z" }, + { url = "https://files.pythonhosted.org/packages/81/aa/e58541a49b5e6310d89474333e994ee57fea97c8aaa8fc7f00b873059bbf/pandas-2.3.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fe37e757f462d31a9cd7580236a82f353f5713a80e059a29753cf938c6775d96", size = 12704705, upload-time = "2025-07-07T19:19:36.856Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f9/07086f5b0f2a19872554abeea7658200824f5835c58a106fa8f2ae96a46c/pandas-2.3.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5db9637dbc24b631ff3707269ae4559bce4b7fd75c1c4d7e13f40edc42df4444", size = 13189044, upload-time = "2025-07-07T19:19:39.999Z" }, +] + +[[package]] +name = "parso" +version = "0.8.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/94/68e2e17afaa9169cf6412ab0f28623903be73d1b32e208d9e8e541bb086d/parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d", size = 400609, upload-time = "2024-04-05T09:43:55.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650, upload-time = "2024-04-05T09:43:53.299Z" }, +] + +[[package]] +name = "pexpect" +version = "4.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ptyprocess", marker = "sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" }, +] + +[[package]] +name = "pillow" +version = "11.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/d0d6dea55cd152ce3d6767bb38a8fc10e33796ba4ba210cbab9354b6d238/pillow-11.3.0.tar.gz", hash = "sha256:3828ee7586cd0b2091b6209e5ad53e20d0649bbe87164a459d0676e035e8f523", size = 47113069, upload-time = "2025-07-01T09:16:30.666Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/fe/1bc9b3ee13f68487a99ac9529968035cca2f0a51ec36892060edcc51d06a/pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4", size = 5278800, upload-time = "2025-07-01T09:14:17.648Z" }, + { url = "https://files.pythonhosted.org/packages/2c/32/7e2ac19b5713657384cec55f89065fb306b06af008cfd87e572035b27119/pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69", size = 4686296, upload-time = "2025-07-01T09:14:19.828Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1e/b9e12bbe6e4c2220effebc09ea0923a07a6da1e1f1bfbc8d7d29a01ce32b/pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d", size = 5871726, upload-time = "2025-07-03T13:10:04.448Z" }, + { url = "https://files.pythonhosted.org/packages/8d/33/e9200d2bd7ba00dc3ddb78df1198a6e80d7669cce6c2bdbeb2530a74ec58/pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6", size = 7644652, upload-time = "2025-07-03T13:10:10.391Z" }, + { url = "https://files.pythonhosted.org/packages/41/f1/6f2427a26fc683e00d985bc391bdd76d8dd4e92fac33d841127eb8fb2313/pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7", size = 5977787, upload-time = "2025-07-01T09:14:21.63Z" }, + { url = "https://files.pythonhosted.org/packages/e4/c9/06dd4a38974e24f932ff5f98ea3c546ce3f8c995d3f0985f8e5ba48bba19/pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024", size = 6645236, upload-time = "2025-07-01T09:14:23.321Z" }, + { url = "https://files.pythonhosted.org/packages/40/e7/848f69fb79843b3d91241bad658e9c14f39a32f71a301bcd1d139416d1be/pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809", size = 6086950, upload-time = "2025-07-01T09:14:25.237Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1a/7cff92e695a2a29ac1958c2a0fe4c0b2393b60aac13b04a4fe2735cad52d/pillow-11.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6be31e3fc9a621e071bc17bb7de63b85cbe0bfae91bb0363c893cbe67247780d", size = 6723358, upload-time = "2025-07-01T09:14:27.053Z" }, + { url = "https://files.pythonhosted.org/packages/26/7d/73699ad77895f69edff76b0f332acc3d497f22f5d75e5360f78cbcaff248/pillow-11.3.0-cp312-cp312-win32.whl", hash = "sha256:7b161756381f0918e05e7cb8a371fff367e807770f8fe92ecb20d905d0e1c149", size = 6275079, upload-time = "2025-07-01T09:14:30.104Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ce/e7dfc873bdd9828f3b6e5c2bbb74e47a98ec23cc5c74fc4e54462f0d9204/pillow-11.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a6444696fce635783440b7f7a9fc24b3ad10a9ea3f0ab66c5905be1c19ccf17d", size = 6986324, upload-time = "2025-07-01T09:14:31.899Z" }, + { url = "https://files.pythonhosted.org/packages/16/8f/b13447d1bf0b1f7467ce7d86f6e6edf66c0ad7cf44cf5c87a37f9bed9936/pillow-11.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:2aceea54f957dd4448264f9bf40875da0415c83eb85f55069d89c0ed436e3542", size = 2423067, upload-time = "2025-07-01T09:14:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/1e/93/0952f2ed8db3a5a4c7a11f91965d6184ebc8cd7cbb7941a260d5f018cd2d/pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1c627742b539bba4309df89171356fcb3cc5a9178355b2727d1b74a6cf155fbd", size = 2128328, upload-time = "2025-07-01T09:14:35.276Z" }, + { url = "https://files.pythonhosted.org/packages/4b/e8/100c3d114b1a0bf4042f27e0f87d2f25e857e838034e98ca98fe7b8c0a9c/pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:30b7c02f3899d10f13d7a48163c8969e4e653f8b43416d23d13d1bbfdc93b9f8", size = 2170652, upload-time = "2025-07-01T09:14:37.203Z" }, + { url = "https://files.pythonhosted.org/packages/aa/86/3f758a28a6e381758545f7cdb4942e1cb79abd271bea932998fc0db93cb6/pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f", size = 2227443, upload-time = "2025-07-01T09:14:39.344Z" }, + { url = "https://files.pythonhosted.org/packages/01/f4/91d5b3ffa718df2f53b0dc109877993e511f4fd055d7e9508682e8aba092/pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c", size = 5278474, upload-time = "2025-07-01T09:14:41.843Z" }, + { url = "https://files.pythonhosted.org/packages/f9/0e/37d7d3eca6c879fbd9dba21268427dffda1ab00d4eb05b32923d4fbe3b12/pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd", size = 4686038, upload-time = "2025-07-01T09:14:44.008Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b0/3426e5c7f6565e752d81221af9d3676fdbb4f352317ceafd42899aaf5d8a/pillow-11.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2d6fcc902a24ac74495df63faad1884282239265c6839a0a6416d33faedfae7e", size = 5864407, upload-time = "2025-07-03T13:10:15.628Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c1/c6c423134229f2a221ee53f838d4be9d82bab86f7e2f8e75e47b6bf6cd77/pillow-11.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0f5d8f4a08090c6d6d578351a2b91acf519a54986c055af27e7a93feae6d3f1", size = 7639094, upload-time = "2025-07-03T13:10:21.857Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c9/09e6746630fe6372c67c648ff9deae52a2bc20897d51fa293571977ceb5d/pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805", size = 5973503, upload-time = "2025-07-01T09:14:45.698Z" }, + { url = "https://files.pythonhosted.org/packages/d5/1c/a2a29649c0b1983d3ef57ee87a66487fdeb45132df66ab30dd37f7dbe162/pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8", size = 6642574, upload-time = "2025-07-01T09:14:47.415Z" }, + { url = "https://files.pythonhosted.org/packages/36/de/d5cc31cc4b055b6c6fd990e3e7f0f8aaf36229a2698501bcb0cdf67c7146/pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2", size = 6084060, upload-time = "2025-07-01T09:14:49.636Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ea/502d938cbaeec836ac28a9b730193716f0114c41325db428e6b280513f09/pillow-11.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:45dfc51ac5975b938e9809451c51734124e73b04d0f0ac621649821a63852e7b", size = 6721407, upload-time = "2025-07-01T09:14:51.962Z" }, + { url = "https://files.pythonhosted.org/packages/45/9c/9c5e2a73f125f6cbc59cc7087c8f2d649a7ae453f83bd0362ff7c9e2aee2/pillow-11.3.0-cp313-cp313-win32.whl", hash = "sha256:a4d336baed65d50d37b88ca5b60c0fa9d81e3a87d4a7930d3880d1624d5b31f3", size = 6273841, upload-time = "2025-07-01T09:14:54.142Z" }, + { url = "https://files.pythonhosted.org/packages/23/85/397c73524e0cd212067e0c969aa245b01d50183439550d24d9f55781b776/pillow-11.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0bce5c4fd0921f99d2e858dc4d4d64193407e1b99478bc5cacecba2311abde51", size = 6978450, upload-time = "2025-07-01T09:14:56.436Z" }, + { url = "https://files.pythonhosted.org/packages/17/d2/622f4547f69cd173955194b78e4d19ca4935a1b0f03a302d655c9f6aae65/pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580", size = 2423055, upload-time = "2025-07-01T09:14:58.072Z" }, + { url = "https://files.pythonhosted.org/packages/dd/80/a8a2ac21dda2e82480852978416cfacd439a4b490a501a288ecf4fe2532d/pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e", size = 5281110, upload-time = "2025-07-01T09:14:59.79Z" }, + { url = "https://files.pythonhosted.org/packages/44/d6/b79754ca790f315918732e18f82a8146d33bcd7f4494380457ea89eb883d/pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d", size = 4689547, upload-time = "2025-07-01T09:15:01.648Z" }, + { url = "https://files.pythonhosted.org/packages/49/20/716b8717d331150cb00f7fdd78169c01e8e0c219732a78b0e59b6bdb2fd6/pillow-11.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1019b04af07fc0163e2810167918cb5add8d74674b6267616021ab558dc98ced", size = 5901554, upload-time = "2025-07-03T13:10:27.018Z" }, + { url = "https://files.pythonhosted.org/packages/74/cf/a9f3a2514a65bb071075063a96f0a5cf949c2f2fce683c15ccc83b1c1cab/pillow-11.3.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f944255db153ebb2b19c51fe85dd99ef0ce494123f21b9db4877ffdfc5590c7c", size = 7669132, upload-time = "2025-07-03T13:10:33.01Z" }, + { url = "https://files.pythonhosted.org/packages/98/3c/da78805cbdbee9cb43efe8261dd7cc0b4b93f2ac79b676c03159e9db2187/pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8", size = 6005001, upload-time = "2025-07-01T09:15:03.365Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fa/ce044b91faecf30e635321351bba32bab5a7e034c60187fe9698191aef4f/pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59", size = 6668814, upload-time = "2025-07-01T09:15:05.655Z" }, + { url = "https://files.pythonhosted.org/packages/7b/51/90f9291406d09bf93686434f9183aba27b831c10c87746ff49f127ee80cb/pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe", size = 6113124, upload-time = "2025-07-01T09:15:07.358Z" }, + { url = "https://files.pythonhosted.org/packages/cd/5a/6fec59b1dfb619234f7636d4157d11fb4e196caeee220232a8d2ec48488d/pillow-11.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83e1b0161c9d148125083a35c1c5a89db5b7054834fd4387499e06552035236c", size = 6747186, upload-time = "2025-07-01T09:15:09.317Z" }, + { url = "https://files.pythonhosted.org/packages/49/6b/00187a044f98255225f172de653941e61da37104a9ea60e4f6887717e2b5/pillow-11.3.0-cp313-cp313t-win32.whl", hash = "sha256:2a3117c06b8fb646639dce83694f2f9eac405472713fcb1ae887469c0d4f6788", size = 6277546, upload-time = "2025-07-01T09:15:11.311Z" }, + { url = "https://files.pythonhosted.org/packages/e8/5c/6caaba7e261c0d75bab23be79f1d06b5ad2a2ae49f028ccec801b0e853d6/pillow-11.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:857844335c95bea93fb39e0fa2726b4d9d758850b34075a7e3ff4f4fa3aa3b31", size = 6985102, upload-time = "2025-07-01T09:15:13.164Z" }, + { url = "https://files.pythonhosted.org/packages/f3/7e/b623008460c09a0cb38263c93b828c666493caee2eb34ff67f778b87e58c/pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e", size = 2424803, upload-time = "2025-07-01T09:15:15.695Z" }, + { url = "https://files.pythonhosted.org/packages/73/f4/04905af42837292ed86cb1b1dabe03dce1edc008ef14c473c5c7e1443c5d/pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12", size = 5278520, upload-time = "2025-07-01T09:15:17.429Z" }, + { url = "https://files.pythonhosted.org/packages/41/b0/33d79e377a336247df6348a54e6d2a2b85d644ca202555e3faa0cf811ecc/pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a", size = 4686116, upload-time = "2025-07-01T09:15:19.423Z" }, + { url = "https://files.pythonhosted.org/packages/49/2d/ed8bc0ab219ae8768f529597d9509d184fe8a6c4741a6864fea334d25f3f/pillow-11.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0743841cabd3dba6a83f38a92672cccbd69af56e3e91777b0ee7f4dba4385632", size = 5864597, upload-time = "2025-07-03T13:10:38.404Z" }, + { url = "https://files.pythonhosted.org/packages/b5/3d/b932bb4225c80b58dfadaca9d42d08d0b7064d2d1791b6a237f87f661834/pillow-11.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2465a69cf967b8b49ee1b96d76718cd98c4e925414ead59fdf75cf0fd07df673", size = 7638246, upload-time = "2025-07-03T13:10:44.987Z" }, + { url = "https://files.pythonhosted.org/packages/09/b5/0487044b7c096f1b48f0d7ad416472c02e0e4bf6919541b111efd3cae690/pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027", size = 5973336, upload-time = "2025-07-01T09:15:21.237Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2d/524f9318f6cbfcc79fbc004801ea6b607ec3f843977652fdee4857a7568b/pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77", size = 6642699, upload-time = "2025-07-01T09:15:23.186Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d2/a9a4f280c6aefedce1e8f615baaa5474e0701d86dd6f1dede66726462bbd/pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874", size = 6083789, upload-time = "2025-07-01T09:15:25.1Z" }, + { url = "https://files.pythonhosted.org/packages/fe/54/86b0cd9dbb683a9d5e960b66c7379e821a19be4ac5810e2e5a715c09a0c0/pillow-11.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:98a9afa7b9007c67ed84c57c9e0ad86a6000da96eaa638e4f8abe5b65ff83f0a", size = 6720386, upload-time = "2025-07-01T09:15:27.378Z" }, + { url = "https://files.pythonhosted.org/packages/e7/95/88efcaf384c3588e24259c4203b909cbe3e3c2d887af9e938c2022c9dd48/pillow-11.3.0-cp314-cp314-win32.whl", hash = "sha256:02a723e6bf909e7cea0dac1b0e0310be9d7650cd66222a5f1c571455c0a45214", size = 6370911, upload-time = "2025-07-01T09:15:29.294Z" }, + { url = "https://files.pythonhosted.org/packages/2e/cc/934e5820850ec5eb107e7b1a72dd278140731c669f396110ebc326f2a503/pillow-11.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:a418486160228f64dd9e9efcd132679b7a02a5f22c982c78b6fc7dab3fefb635", size = 7117383, upload-time = "2025-07-01T09:15:31.128Z" }, + { url = "https://files.pythonhosted.org/packages/d6/e9/9c0a616a71da2a5d163aa37405e8aced9a906d574b4a214bede134e731bc/pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6", size = 2511385, upload-time = "2025-07-01T09:15:33.328Z" }, + { url = "https://files.pythonhosted.org/packages/1a/33/c88376898aff369658b225262cd4f2659b13e8178e7534df9e6e1fa289f6/pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae", size = 5281129, upload-time = "2025-07-01T09:15:35.194Z" }, + { url = "https://files.pythonhosted.org/packages/1f/70/d376247fb36f1844b42910911c83a02d5544ebd2a8bad9efcc0f707ea774/pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653", size = 4689580, upload-time = "2025-07-01T09:15:37.114Z" }, + { url = "https://files.pythonhosted.org/packages/eb/1c/537e930496149fbac69efd2fc4329035bbe2e5475b4165439e3be9cb183b/pillow-11.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee92f2fd10f4adc4b43d07ec5e779932b4eb3dbfbc34790ada5a6669bc095aa6", size = 5902860, upload-time = "2025-07-03T13:10:50.248Z" }, + { url = "https://files.pythonhosted.org/packages/bd/57/80f53264954dcefeebcf9dae6e3eb1daea1b488f0be8b8fef12f79a3eb10/pillow-11.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96d333dcf42d01f47b37e0979b6bd73ec91eae18614864622d9b87bbd5bbf36", size = 7670694, upload-time = "2025-07-03T13:10:56.432Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/4727d3b71a8578b4587d9c276e90efad2d6fe0335fd76742a6da08132e8c/pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b", size = 6005888, upload-time = "2025-07-01T09:15:39.436Z" }, + { url = "https://files.pythonhosted.org/packages/05/ae/716592277934f85d3be51d7256f3636672d7b1abfafdc42cf3f8cbd4b4c8/pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477", size = 6670330, upload-time = "2025-07-01T09:15:41.269Z" }, + { url = "https://files.pythonhosted.org/packages/e7/bb/7fe6cddcc8827b01b1a9766f5fdeb7418680744f9082035bdbabecf1d57f/pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50", size = 6114089, upload-time = "2025-07-01T09:15:43.13Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f5/06bfaa444c8e80f1a8e4bff98da9c83b37b5be3b1deaa43d27a0db37ef84/pillow-11.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a1bc6ba083b145187f648b667e05a2534ecc4b9f2784c2cbe3089e44868f2b9b", size = 6748206, upload-time = "2025-07-01T09:15:44.937Z" }, + { url = "https://files.pythonhosted.org/packages/f0/77/bc6f92a3e8e6e46c0ca78abfffec0037845800ea38c73483760362804c41/pillow-11.3.0-cp314-cp314t-win32.whl", hash = "sha256:118ca10c0d60b06d006be10a501fd6bbdfef559251ed31b794668ed569c87e12", size = 6377370, upload-time = "2025-07-01T09:15:46.673Z" }, + { url = "https://files.pythonhosted.org/packages/4a/82/3a721f7d69dca802befb8af08b7c79ebcab461007ce1c18bd91a5d5896f9/pillow-11.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8924748b688aa210d79883357d102cd64690e56b923a186f35a82cbc10f997db", size = 7121500, upload-time = "2025-07-01T09:15:48.512Z" }, + { url = "https://files.pythonhosted.org/packages/89/c7/5572fa4a3f45740eaab6ae86fcdf7195b55beac1371ac8c619d880cfe948/pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa", size = 2512835, upload-time = "2025-07-01T09:15:50.399Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/8b/3c73abc9c759ecd3f1f7ceff6685840859e8070c4d947c93fae71f6a0bf2/platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc", size = 21362, upload-time = "2025-05-07T22:47:42.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" }, +] + +[[package]] +name = "polars" +version = "1.32.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/f2/1a76a8bd902bc4942e435a480f362c8687bba60d438ff3283191e38568fa/polars-1.32.3.tar.gz", hash = "sha256:57c500dc1b5cba49b0589034478db031815f3d57a20cb830b05ecee1a9ba56b1", size = 4838448, upload-time = "2025-08-14T17:28:10.702Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/9b/5937ab9f8fa49c8e00617aeb817a5ffa5740434d5bb8a90f2afa657875aa/polars-1.32.3-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c7c472ea1d50a5104079cb64e34f78f85774bcc69b875ba8daf21233f4c70d42", size = 37935794, upload-time = "2025-08-14T17:26:55.565Z" }, + { url = "https://files.pythonhosted.org/packages/6e/e9/88f5332001b9dd5c8e0a4fab51015f740e01715a081c41bc0f7ad2bf76a5/polars-1.32.3-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd87275f0cc795e72a2030b58293198cfa748d4b009cf52218e27db5397ed07f", size = 34621102, upload-time = "2025-08-14T17:27:00.521Z" }, + { url = "https://files.pythonhosted.org/packages/ab/8a/6f56af7e535c34c95decc8654786bfce4632ba32817dc2f8bad18571ef9a/polars-1.32.3-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9a9b9668ef310e5a77a7e7daa9c753874779c8da52e93f654bfd7953eb4b60b", size = 38443071, upload-time = "2025-08-14T17:27:08.382Z" }, + { url = "https://files.pythonhosted.org/packages/46/aa/63536ea5780edc0ef6850679dc81d519f3966c7bb11a5cf10ccecb541095/polars-1.32.3-cp39-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:c8f5d2f43b80b68e39bfaa2948ce632563633466576f12e74e8560d6481f5851", size = 35639598, upload-time = "2025-08-14T17:27:12.261Z" }, + { url = "https://files.pythonhosted.org/packages/d7/c8/226953cda6cf9ae63aa9714d396a9138029e31db3c504c15d6711b618f8f/polars-1.32.3-cp39-abi3-win_amd64.whl", hash = "sha256:db56a7cb4898e173d62634e182f74bdff744c62be5470e0fe20df8d10f659af7", size = 38038192, upload-time = "2025-08-14T17:27:15.993Z" }, + { url = "https://files.pythonhosted.org/packages/ec/99/6b93c854e602927a778eabd7550204f700cc4e6c07be73372371583dda3e/polars-1.32.3-cp39-abi3-win_arm64.whl", hash = "sha256:a2e3f87c60f54eefe67b1bebd3105918d84df0fd6d59cc6b870c2f16d2d26ca1", size = 34198919, upload-time = "2025-08-14T17:27:21.423Z" }, +] + +[[package]] +name = "prometheus-client" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/cf/40dde0a2be27cc1eb41e333d1a674a74ce8b8b0457269cc640fd42b07cf7/prometheus_client-0.22.1.tar.gz", hash = "sha256:190f1331e783cf21eb60bca559354e0a4d4378facecf78f5428c39b675d20d28", size = 69746, upload-time = "2025-06-02T14:29:01.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/ae/ec06af4fe3ee72d16973474f122541746196aaa16cea6f66d18b963c6177/prometheus_client-0.22.1-py3-none-any.whl", hash = "sha256:cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094", size = 58694, upload-time = "2025-06-02T14:29:00.068Z" }, +] + +[[package]] +name = "prompt-toolkit" +version = "3.0.51" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/6e/9d084c929dfe9e3bfe0c6a47e31f78a25c54627d64a66e884a8bf5474f1c/prompt_toolkit-3.0.51.tar.gz", hash = "sha256:931a162e3b27fc90c86f1b48bb1fb2c528c2761475e57c9c06de13311c7b54ed", size = 428940, upload-time = "2025-04-15T09:18:47.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/4f/5249960887b1fbe561d9ff265496d170b55a735b76724f10ef19f9e40716/prompt_toolkit-3.0.51-py3-none-any.whl", hash = "sha256:52742911fde84e2d423e2f9a4cf1de7d7ac4e51958f648d9540e0fb8db077b07", size = 387810, upload-time = "2025-04-15T09:18:44.753Z" }, +] + +[[package]] +name = "propcache" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/16/43264e4a779dd8588c21a70f0709665ee8f611211bdd2c87d952cfa7c776/propcache-0.3.2.tar.gz", hash = "sha256:20d7d62e4e7ef05f221e0db2856b979540686342e7dd9973b815599c7057e168", size = 44139, upload-time = "2025-06-09T22:56:06.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/42/9ca01b0a6f48e81615dca4765a8f1dd2c057e0540f6116a27dc5ee01dfb6/propcache-0.3.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8de106b6c84506b31c27168582cd3cb3000a6412c16df14a8628e5871ff83c10", size = 73674, upload-time = "2025-06-09T22:54:30.551Z" }, + { url = "https://files.pythonhosted.org/packages/af/6e/21293133beb550f9c901bbece755d582bfaf2176bee4774000bd4dd41884/propcache-0.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:28710b0d3975117239c76600ea351934ac7b5ff56e60953474342608dbbb6154", size = 43570, upload-time = "2025-06-09T22:54:32.296Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c8/0393a0a3a2b8760eb3bde3c147f62b20044f0ddac81e9d6ed7318ec0d852/propcache-0.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce26862344bdf836650ed2487c3d724b00fbfec4233a1013f597b78c1cb73615", size = 43094, upload-time = "2025-06-09T22:54:33.929Z" }, + { url = "https://files.pythonhosted.org/packages/37/2c/489afe311a690399d04a3e03b069225670c1d489eb7b044a566511c1c498/propcache-0.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bca54bd347a253af2cf4544bbec232ab982f4868de0dd684246b67a51bc6b1db", size = 226958, upload-time = "2025-06-09T22:54:35.186Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ca/63b520d2f3d418c968bf596839ae26cf7f87bead026b6192d4da6a08c467/propcache-0.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55780d5e9a2ddc59711d727226bb1ba83a22dd32f64ee15594b9392b1f544eb1", size = 234894, upload-time = "2025-06-09T22:54:36.708Z" }, + { url = "https://files.pythonhosted.org/packages/11/60/1d0ed6fff455a028d678df30cc28dcee7af77fa2b0e6962ce1df95c9a2a9/propcache-0.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:035e631be25d6975ed87ab23153db6a73426a48db688070d925aa27e996fe93c", size = 233672, upload-time = "2025-06-09T22:54:38.062Z" }, + { url = "https://files.pythonhosted.org/packages/37/7c/54fd5301ef38505ab235d98827207176a5c9b2aa61939b10a460ca53e123/propcache-0.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee6f22b6eaa39297c751d0e80c0d3a454f112f5c6481214fcf4c092074cecd67", size = 224395, upload-time = "2025-06-09T22:54:39.634Z" }, + { url = "https://files.pythonhosted.org/packages/ee/1a/89a40e0846f5de05fdc6779883bf46ba980e6df4d2ff8fb02643de126592/propcache-0.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ca3aee1aa955438c4dba34fc20a9f390e4c79967257d830f137bd5a8a32ed3b", size = 212510, upload-time = "2025-06-09T22:54:41.565Z" }, + { url = "https://files.pythonhosted.org/packages/5e/33/ca98368586c9566a6b8d5ef66e30484f8da84c0aac3f2d9aec6d31a11bd5/propcache-0.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7a4f30862869fa2b68380d677cc1c5fcf1e0f2b9ea0cf665812895c75d0ca3b8", size = 222949, upload-time = "2025-06-09T22:54:43.038Z" }, + { url = "https://files.pythonhosted.org/packages/ba/11/ace870d0aafe443b33b2f0b7efdb872b7c3abd505bfb4890716ad7865e9d/propcache-0.3.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b77ec3c257d7816d9f3700013639db7491a434644c906a2578a11daf13176251", size = 217258, upload-time = "2025-06-09T22:54:44.376Z" }, + { url = "https://files.pythonhosted.org/packages/5b/d2/86fd6f7adffcfc74b42c10a6b7db721d1d9ca1055c45d39a1a8f2a740a21/propcache-0.3.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cab90ac9d3f14b2d5050928483d3d3b8fb6b4018893fc75710e6aa361ecb2474", size = 213036, upload-time = "2025-06-09T22:54:46.243Z" }, + { url = "https://files.pythonhosted.org/packages/07/94/2d7d1e328f45ff34a0a284cf5a2847013701e24c2a53117e7c280a4316b3/propcache-0.3.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0b504d29f3c47cf6b9e936c1852246c83d450e8e063d50562115a6be6d3a2535", size = 227684, upload-time = "2025-06-09T22:54:47.63Z" }, + { url = "https://files.pythonhosted.org/packages/b7/05/37ae63a0087677e90b1d14710e532ff104d44bc1efa3b3970fff99b891dc/propcache-0.3.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ce2ac2675a6aa41ddb2a0c9cbff53780a617ac3d43e620f8fd77ba1c84dcfc06", size = 234562, upload-time = "2025-06-09T22:54:48.982Z" }, + { url = "https://files.pythonhosted.org/packages/a4/7c/3f539fcae630408d0bd8bf3208b9a647ccad10976eda62402a80adf8fc34/propcache-0.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b4239611205294cc433845b914131b2a1f03500ff3c1ed093ed216b82621e1", size = 222142, upload-time = "2025-06-09T22:54:50.424Z" }, + { url = "https://files.pythonhosted.org/packages/7c/d2/34b9eac8c35f79f8a962546b3e97e9d4b990c420ee66ac8255d5d9611648/propcache-0.3.2-cp312-cp312-win32.whl", hash = "sha256:df4a81b9b53449ebc90cc4deefb052c1dd934ba85012aa912c7ea7b7e38b60c1", size = 37711, upload-time = "2025-06-09T22:54:52.072Z" }, + { url = "https://files.pythonhosted.org/packages/19/61/d582be5d226cf79071681d1b46b848d6cb03d7b70af7063e33a2787eaa03/propcache-0.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7046e79b989d7fe457bb755844019e10f693752d169076138abf17f31380800c", size = 41479, upload-time = "2025-06-09T22:54:53.234Z" }, + { url = "https://files.pythonhosted.org/packages/dc/d1/8c747fafa558c603c4ca19d8e20b288aa0c7cda74e9402f50f31eb65267e/propcache-0.3.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ca592ed634a73ca002967458187109265e980422116c0a107cf93d81f95af945", size = 71286, upload-time = "2025-06-09T22:54:54.369Z" }, + { url = "https://files.pythonhosted.org/packages/61/99/d606cb7986b60d89c36de8a85d58764323b3a5ff07770a99d8e993b3fa73/propcache-0.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ecb0aad4020e275652ba3975740f241bd12a61f1a784df044cf7477a02bc252", size = 42425, upload-time = "2025-06-09T22:54:55.642Z" }, + { url = "https://files.pythonhosted.org/packages/8c/96/ef98f91bbb42b79e9bb82bdd348b255eb9d65f14dbbe3b1594644c4073f7/propcache-0.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7f08f1cc28bd2eade7a8a3d2954ccc673bb02062e3e7da09bc75d843386b342f", size = 41846, upload-time = "2025-06-09T22:54:57.246Z" }, + { url = "https://files.pythonhosted.org/packages/5b/ad/3f0f9a705fb630d175146cd7b1d2bf5555c9beaed54e94132b21aac098a6/propcache-0.3.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1a342c834734edb4be5ecb1e9fb48cb64b1e2320fccbd8c54bf8da8f2a84c33", size = 208871, upload-time = "2025-06-09T22:54:58.975Z" }, + { url = "https://files.pythonhosted.org/packages/3a/38/2085cda93d2c8b6ec3e92af2c89489a36a5886b712a34ab25de9fbca7992/propcache-0.3.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8a544caaae1ac73f1fecfae70ded3e93728831affebd017d53449e3ac052ac1e", size = 215720, upload-time = "2025-06-09T22:55:00.471Z" }, + { url = "https://files.pythonhosted.org/packages/61/c1/d72ea2dc83ac7f2c8e182786ab0fc2c7bd123a1ff9b7975bee671866fe5f/propcache-0.3.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310d11aa44635298397db47a3ebce7db99a4cc4b9bbdfcf6c98a60c8d5261cf1", size = 215203, upload-time = "2025-06-09T22:55:01.834Z" }, + { url = "https://files.pythonhosted.org/packages/af/81/b324c44ae60c56ef12007105f1460d5c304b0626ab0cc6b07c8f2a9aa0b8/propcache-0.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c1396592321ac83157ac03a2023aa6cc4a3cc3cfdecb71090054c09e5a7cce3", size = 206365, upload-time = "2025-06-09T22:55:03.199Z" }, + { url = "https://files.pythonhosted.org/packages/09/73/88549128bb89e66d2aff242488f62869014ae092db63ccea53c1cc75a81d/propcache-0.3.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cabf5b5902272565e78197edb682017d21cf3b550ba0460ee473753f28d23c1", size = 196016, upload-time = "2025-06-09T22:55:04.518Z" }, + { url = "https://files.pythonhosted.org/packages/b9/3f/3bdd14e737d145114a5eb83cb172903afba7242f67c5877f9909a20d948d/propcache-0.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0a2f2235ac46a7aa25bdeb03a9e7060f6ecbd213b1f9101c43b3090ffb971ef6", size = 205596, upload-time = "2025-06-09T22:55:05.942Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ca/2f4aa819c357d3107c3763d7ef42c03980f9ed5c48c82e01e25945d437c1/propcache-0.3.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:92b69e12e34869a6970fd2f3da91669899994b47c98f5d430b781c26f1d9f387", size = 200977, upload-time = "2025-06-09T22:55:07.792Z" }, + { url = "https://files.pythonhosted.org/packages/cd/4a/e65276c7477533c59085251ae88505caf6831c0e85ff8b2e31ebcbb949b1/propcache-0.3.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:54e02207c79968ebbdffc169591009f4474dde3b4679e16634d34c9363ff56b4", size = 197220, upload-time = "2025-06-09T22:55:09.173Z" }, + { url = "https://files.pythonhosted.org/packages/7c/54/fc7152e517cf5578278b242396ce4d4b36795423988ef39bb8cd5bf274c8/propcache-0.3.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4adfb44cb588001f68c5466579d3f1157ca07f7504fc91ec87862e2b8e556b88", size = 210642, upload-time = "2025-06-09T22:55:10.62Z" }, + { url = "https://files.pythonhosted.org/packages/b9/80/abeb4a896d2767bf5f1ea7b92eb7be6a5330645bd7fb844049c0e4045d9d/propcache-0.3.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fd3e6019dc1261cd0291ee8919dd91fbab7b169bb76aeef6c716833a3f65d206", size = 212789, upload-time = "2025-06-09T22:55:12.029Z" }, + { url = "https://files.pythonhosted.org/packages/b3/db/ea12a49aa7b2b6d68a5da8293dcf50068d48d088100ac016ad92a6a780e6/propcache-0.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4c181cad81158d71c41a2bce88edce078458e2dd5ffee7eddd6b05da85079f43", size = 205880, upload-time = "2025-06-09T22:55:13.45Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e5/9076a0bbbfb65d1198007059c65639dfd56266cf8e477a9707e4b1999ff4/propcache-0.3.2-cp313-cp313-win32.whl", hash = "sha256:8a08154613f2249519e549de2330cf8e2071c2887309a7b07fb56098f5170a02", size = 37220, upload-time = "2025-06-09T22:55:15.284Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f5/b369e026b09a26cd77aa88d8fffd69141d2ae00a2abaaf5380d2603f4b7f/propcache-0.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:e41671f1594fc4ab0a6dec1351864713cb3a279910ae8b58f884a88a0a632c05", size = 40678, upload-time = "2025-06-09T22:55:16.445Z" }, + { url = "https://files.pythonhosted.org/packages/a4/3a/6ece377b55544941a08d03581c7bc400a3c8cd3c2865900a68d5de79e21f/propcache-0.3.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:9a3cf035bbaf035f109987d9d55dc90e4b0e36e04bbbb95af3055ef17194057b", size = 76560, upload-time = "2025-06-09T22:55:17.598Z" }, + { url = "https://files.pythonhosted.org/packages/0c/da/64a2bb16418740fa634b0e9c3d29edff1db07f56d3546ca2d86ddf0305e1/propcache-0.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:156c03d07dc1323d8dacaa221fbe028c5c70d16709cdd63502778e6c3ccca1b0", size = 44676, upload-time = "2025-06-09T22:55:18.922Z" }, + { url = "https://files.pythonhosted.org/packages/36/7b/f025e06ea51cb72c52fb87e9b395cced02786610b60a3ed51da8af017170/propcache-0.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74413c0ba02ba86f55cf60d18daab219f7e531620c15f1e23d95563f505efe7e", size = 44701, upload-time = "2025-06-09T22:55:20.106Z" }, + { url = "https://files.pythonhosted.org/packages/a4/00/faa1b1b7c3b74fc277f8642f32a4c72ba1d7b2de36d7cdfb676db7f4303e/propcache-0.3.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f066b437bb3fa39c58ff97ab2ca351db465157d68ed0440abecb21715eb24b28", size = 276934, upload-time = "2025-06-09T22:55:21.5Z" }, + { url = "https://files.pythonhosted.org/packages/74/ab/935beb6f1756e0476a4d5938ff44bf0d13a055fed880caf93859b4f1baf4/propcache-0.3.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1304b085c83067914721e7e9d9917d41ad87696bf70f0bc7dee450e9c71ad0a", size = 278316, upload-time = "2025-06-09T22:55:22.918Z" }, + { url = "https://files.pythonhosted.org/packages/f8/9d/994a5c1ce4389610838d1caec74bdf0e98b306c70314d46dbe4fcf21a3e2/propcache-0.3.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab50cef01b372763a13333b4e54021bdcb291fc9a8e2ccb9c2df98be51bcde6c", size = 282619, upload-time = "2025-06-09T22:55:24.651Z" }, + { url = "https://files.pythonhosted.org/packages/2b/00/a10afce3d1ed0287cef2e09506d3be9822513f2c1e96457ee369adb9a6cd/propcache-0.3.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad3b2a085ec259ad2c2842666b2a0a49dea8463579c606426128925af1ed725", size = 265896, upload-time = "2025-06-09T22:55:26.049Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a8/2aa6716ffa566ca57c749edb909ad27884680887d68517e4be41b02299f3/propcache-0.3.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:261fa020c1c14deafd54c76b014956e2f86991af198c51139faf41c4d5e83892", size = 252111, upload-time = "2025-06-09T22:55:27.381Z" }, + { url = "https://files.pythonhosted.org/packages/36/4f/345ca9183b85ac29c8694b0941f7484bf419c7f0fea2d1e386b4f7893eed/propcache-0.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:46d7f8aa79c927e5f987ee3a80205c987717d3659f035c85cf0c3680526bdb44", size = 268334, upload-time = "2025-06-09T22:55:28.747Z" }, + { url = "https://files.pythonhosted.org/packages/3e/ca/fcd54f78b59e3f97b3b9715501e3147f5340167733d27db423aa321e7148/propcache-0.3.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:6d8f3f0eebf73e3c0ff0e7853f68be638b4043c65a70517bb575eff54edd8dbe", size = 255026, upload-time = "2025-06-09T22:55:30.184Z" }, + { url = "https://files.pythonhosted.org/packages/8b/95/8e6a6bbbd78ac89c30c225210a5c687790e532ba4088afb8c0445b77ef37/propcache-0.3.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:03c89c1b14a5452cf15403e291c0ccd7751d5b9736ecb2c5bab977ad6c5bcd81", size = 250724, upload-time = "2025-06-09T22:55:31.646Z" }, + { url = "https://files.pythonhosted.org/packages/ee/b0/0dd03616142baba28e8b2d14ce5df6631b4673850a3d4f9c0f9dd714a404/propcache-0.3.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:0cc17efde71e12bbaad086d679ce575268d70bc123a5a71ea7ad76f70ba30bba", size = 268868, upload-time = "2025-06-09T22:55:33.209Z" }, + { url = "https://files.pythonhosted.org/packages/c5/98/2c12407a7e4fbacd94ddd32f3b1e3d5231e77c30ef7162b12a60e2dd5ce3/propcache-0.3.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:acdf05d00696bc0447e278bb53cb04ca72354e562cf88ea6f9107df8e7fd9770", size = 271322, upload-time = "2025-06-09T22:55:35.065Z" }, + { url = "https://files.pythonhosted.org/packages/35/91/9cb56efbb428b006bb85db28591e40b7736847b8331d43fe335acf95f6c8/propcache-0.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4445542398bd0b5d32df908031cb1b30d43ac848e20470a878b770ec2dcc6330", size = 265778, upload-time = "2025-06-09T22:55:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4c/b0fe775a2bdd01e176b14b574be679d84fc83958335790f7c9a686c1f468/propcache-0.3.2-cp313-cp313t-win32.whl", hash = "sha256:f86e5d7cd03afb3a1db8e9f9f6eff15794e79e791350ac48a8c924e6f439f394", size = 41175, upload-time = "2025-06-09T22:55:38.436Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ff/47f08595e3d9b5e149c150f88d9714574f1a7cbd89fe2817158a952674bf/propcache-0.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:9704bedf6e7cbe3c65eca4379a9b53ee6a83749f047808cbb5044d40d7d72198", size = 44857, upload-time = "2025-06-09T22:55:39.687Z" }, + { url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663, upload-time = "2025-06-09T22:56:04.484Z" }, +] + +[[package]] +name = "proto-plus" +version = "1.26.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/ac/87285f15f7cce6d4a008f33f1757fb5a13611ea8914eb58c3d0d26243468/proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012", size = 56142, upload-time = "2025-03-10T15:54:38.843Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/6d/280c4c2ce28b1593a19ad5239c8b826871fc6ec275c21afc8e1820108039/proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66", size = 50163, upload-time = "2025-03-10T15:54:37.335Z" }, +] + +[[package]] +name = "protobuf" +version = "5.29.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/29/d09e70352e4e88c9c7a198d5645d7277811448d76c23b00345670f7c8a38/protobuf-5.29.5.tar.gz", hash = "sha256:bc1463bafd4b0929216c35f437a8e28731a2b7fe3d98bb77a600efced5a15c84", size = 425226, upload-time = "2025-05-28T23:51:59.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/11/6e40e9fc5bba02988a214c07cf324595789ca7820160bfd1f8be96e48539/protobuf-5.29.5-cp310-abi3-win32.whl", hash = "sha256:3f1c6468a2cfd102ff4703976138844f78ebd1fb45f49011afc5139e9e283079", size = 422963, upload-time = "2025-05-28T23:51:41.204Z" }, + { url = "https://files.pythonhosted.org/packages/81/7f/73cefb093e1a2a7c3ffd839e6f9fcafb7a427d300c7f8aef9c64405d8ac6/protobuf-5.29.5-cp310-abi3-win_amd64.whl", hash = "sha256:3f76e3a3675b4a4d867b52e4a5f5b78a2ef9565549d4037e06cf7b0942b1d3fc", size = 434818, upload-time = "2025-05-28T23:51:44.297Z" }, + { url = "https://files.pythonhosted.org/packages/dd/73/10e1661c21f139f2c6ad9b23040ff36fee624310dc28fba20d33fdae124c/protobuf-5.29.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e38c5add5a311f2a6eb0340716ef9b039c1dfa428b28f25a7838ac329204a671", size = 418091, upload-time = "2025-05-28T23:51:45.907Z" }, + { url = "https://files.pythonhosted.org/packages/6c/04/98f6f8cf5b07ab1294c13f34b4e69b3722bb609c5b701d6c169828f9f8aa/protobuf-5.29.5-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:fa18533a299d7ab6c55a238bf8629311439995f2e7eca5caaff08663606e9015", size = 319824, upload-time = "2025-05-28T23:51:47.545Z" }, + { url = "https://files.pythonhosted.org/packages/85/e4/07c80521879c2d15f321465ac24c70efe2381378c00bf5e56a0f4fbac8cd/protobuf-5.29.5-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:63848923da3325e1bf7e9003d680ce6e14b07e55d0473253a690c3a8b8fd6e61", size = 319942, upload-time = "2025-05-28T23:51:49.11Z" }, + { url = "https://files.pythonhosted.org/packages/7e/cc/7e77861000a0691aeea8f4566e5d3aa716f2b1dece4a24439437e41d3d25/protobuf-5.29.5-py3-none-any.whl", hash = "sha256:6cf42630262c59b2d8de33954443d94b746c952b01434fc58a417fdbd2e84bd5", size = 172823, upload-time = "2025-05-28T23:51:58.157Z" }, +] + +[[package]] +name = "psutil" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456", size = 497003, upload-time = "2025-02-13T21:54:07.946Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25", size = 238051, upload-time = "2025-02-13T21:54:12.36Z" }, + { url = "https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da", size = 239535, upload-time = "2025-02-13T21:54:16.07Z" }, + { url = "https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91", size = 275004, upload-time = "2025-02-13T21:54:18.662Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34", size = 277986, upload-time = "2025-02-13T21:54:21.811Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993", size = 279544, upload-time = "2025-02-13T21:54:24.68Z" }, + { url = "https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99", size = 241053, upload-time = "2025-02-13T21:54:34.31Z" }, + { url = "https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553", size = 244885, upload-time = "2025-02-13T21:54:37.486Z" }, +] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762, upload-time = "2020-12-28T15:15:30.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993, upload-time = "2020-12-28T15:15:28.35Z" }, +] + +[[package]] +name = "pure-eval" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752, upload-time = "2024-07-21T12:58:21.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, +] + +[[package]] +name = "py-spy" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/e2/ff811a367028b87e86714945bb9ecb5c1cc69114a8039a67b3a862cef921/py_spy-0.4.1.tar.gz", hash = "sha256:e53aa53daa2e47c2eef97dd2455b47bb3a7e7f962796a86cc3e7dbde8e6f4db4", size = 244726, upload-time = "2025-07-31T19:33:25.172Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/e3/3a32500d845bdd94f6a2b4ed6244982f42ec2bc64602ea8fcfe900678ae7/py_spy-0.4.1-py2.py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:809094208c6256c8f4ccadd31e9a513fe2429253f48e20066879239ba12cd8cc", size = 3682508, upload-time = "2025-07-31T19:33:13.753Z" }, + { url = "https://files.pythonhosted.org/packages/4f/bf/e4d280e9e0bec71d39fc646654097027d4bbe8e04af18fb68e49afcff404/py_spy-0.4.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:1fb8bf71ab8df95a95cc387deed6552934c50feef2cf6456bc06692a5508fd0c", size = 1796395, upload-time = "2025-07-31T19:33:15.325Z" }, + { url = "https://files.pythonhosted.org/packages/df/79/9ed50bb0a9de63ed023aa2db8b6265b04a7760d98c61eb54def6a5fddb68/py_spy-0.4.1-py2.py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee776b9d512a011d1ad3907ed53ae32ce2f3d9ff3e1782236554e22103b5c084", size = 2034938, upload-time = "2025-07-31T19:33:17.194Z" }, + { url = "https://files.pythonhosted.org/packages/53/a5/36862e3eea59f729dfb70ee6f9e14b051d8ddce1aa7e70e0b81d9fe18536/py_spy-0.4.1-py2.py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:532d3525538254d1859b49de1fbe9744df6b8865657c9f0e444bf36ce3f19226", size = 2658968, upload-time = "2025-07-31T19:33:18.916Z" }, + { url = "https://files.pythonhosted.org/packages/08/f8/9ea0b586b065a623f591e5e7961282ec944b5fbbdca33186c7c0296645b3/py_spy-0.4.1-py2.py3-none-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4972c21890b6814017e39ac233c22572c4a61fd874524ebc5ccab0f2237aee0a", size = 2147541, upload-time = "2025-07-31T19:33:20.565Z" }, + { url = "https://files.pythonhosted.org/packages/68/fb/bc7f639aed026bca6e7beb1e33f6951e16b7d315594e7635a4f7d21d63f4/py_spy-0.4.1-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6a80ec05eb8a6883863a367c6a4d4f2d57de68466f7956b6367d4edd5c61bb29", size = 2763338, upload-time = "2025-07-31T19:33:22.202Z" }, + { url = "https://files.pythonhosted.org/packages/e1/da/fcc9a9fcd4ca946ff402cff20348e838b051d69f50f5d1f5dca4cd3c5eb8/py_spy-0.4.1-py2.py3-none-win_amd64.whl", hash = "sha256:d92e522bd40e9bf7d87c204033ce5bb5c828fca45fa28d970f58d71128069fdc", size = 1818784, upload-time = "2025-07-31T19:33:23.802Z" }, +] + +[[package]] +name = "pyarrow" +version = "18.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7f/7b/640785a9062bb00314caa8a387abce547d2a420cf09bd6c715fe659ccffb/pyarrow-18.1.0.tar.gz", hash = "sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73", size = 1118671, upload-time = "2024-11-26T02:01:48.62Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/50/12829e7111b932581e51dda51d5cb39207a056c30fe31ef43f14c63c4d7e/pyarrow-18.1.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d", size = 29514620, upload-time = "2024-11-26T01:59:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/d1/41/468c944eab157702e96abab3d07b48b8424927d4933541ab43788bb6964d/pyarrow-18.1.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee", size = 30856494, upload-time = "2024-11-26T01:59:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/68/f9/29fb659b390312a7345aeb858a9d9c157552a8852522f2c8bad437c29c0a/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992", size = 39203624, upload-time = "2024-11-26T01:59:49.189Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f6/19360dae44200e35753c5c2889dc478154cd78e61b1f738514c9f131734d/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54", size = 40139341, upload-time = "2024-11-26T01:59:54.849Z" }, + { url = "https://files.pythonhosted.org/packages/bb/e6/9b3afbbcf10cc724312e824af94a2e993d8ace22994d823f5c35324cebf5/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33", size = 38618629, upload-time = "2024-11-26T01:59:59.966Z" }, + { url = "https://files.pythonhosted.org/packages/3a/2e/3b99f8a3d9e0ccae0e961978a0d0089b25fb46ebbcfb5ebae3cca179a5b3/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30", size = 40078661, upload-time = "2024-11-26T02:00:04.55Z" }, + { url = "https://files.pythonhosted.org/packages/76/52/f8da04195000099d394012b8d42c503d7041b79f778d854f410e5f05049a/pyarrow-18.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99", size = 25092330, upload-time = "2024-11-26T02:00:09.576Z" }, + { url = "https://files.pythonhosted.org/packages/cb/87/aa4d249732edef6ad88899399047d7e49311a55749d3c373007d034ee471/pyarrow-18.1.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:84e314d22231357d473eabec709d0ba285fa706a72377f9cc8e1cb3c8013813b", size = 29497406, upload-time = "2024-11-26T02:00:14.469Z" }, + { url = "https://files.pythonhosted.org/packages/3c/c7/ed6adb46d93a3177540e228b5ca30d99fc8ea3b13bdb88b6f8b6467e2cb7/pyarrow-18.1.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:f591704ac05dfd0477bb8f8e0bd4b5dc52c1cadf50503858dce3a15db6e46ff2", size = 30835095, upload-time = "2024-11-26T02:00:19.347Z" }, + { url = "https://files.pythonhosted.org/packages/41/d7/ed85001edfb96200ff606943cff71d64f91926ab42828676c0fc0db98963/pyarrow-18.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acb7564204d3c40babf93a05624fc6a8ec1ab1def295c363afc40b0c9e66c191", size = 39194527, upload-time = "2024-11-26T02:00:24.085Z" }, + { url = "https://files.pythonhosted.org/packages/59/16/35e28eab126342fa391593415d79477e89582de411bb95232f28b131a769/pyarrow-18.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74de649d1d2ccb778f7c3afff6085bd5092aed4c23df9feeb45dd6b16f3811aa", size = 40131443, upload-time = "2024-11-26T02:00:29.483Z" }, + { url = "https://files.pythonhosted.org/packages/0c/95/e855880614c8da20f4cd74fa85d7268c725cf0013dc754048593a38896a0/pyarrow-18.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f96bd502cb11abb08efea6dab09c003305161cb6c9eafd432e35e76e7fa9b90c", size = 38608750, upload-time = "2024-11-26T02:00:34.069Z" }, + { url = "https://files.pythonhosted.org/packages/54/9d/f253554b1457d4fdb3831b7bd5f8f00f1795585a606eabf6fec0a58a9c38/pyarrow-18.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:36ac22d7782554754a3b50201b607d553a8d71b78cdf03b33c1125be4b52397c", size = 40066690, upload-time = "2024-11-26T02:00:39.603Z" }, + { url = "https://files.pythonhosted.org/packages/2f/58/8912a2563e6b8273e8aa7b605a345bba5a06204549826f6493065575ebc0/pyarrow-18.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:25dbacab8c5952df0ca6ca0af28f50d45bd31c1ff6fcf79e2d120b4a65ee7181", size = 25081054, upload-time = "2024-11-26T02:00:43.611Z" }, + { url = "https://files.pythonhosted.org/packages/82/f9/d06ddc06cab1ada0c2f2fd205ac8c25c2701182de1b9c4bf7a0a44844431/pyarrow-18.1.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6a276190309aba7bc9d5bd2933230458b3521a4317acfefe69a354f2fe59f2bc", size = 29525542, upload-time = "2024-11-26T02:00:48.094Z" }, + { url = "https://files.pythonhosted.org/packages/ab/94/8917e3b961810587ecbdaa417f8ebac0abb25105ae667b7aa11c05876976/pyarrow-18.1.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:ad514dbfcffe30124ce655d72771ae070f30bf850b48bc4d9d3b25993ee0e386", size = 30829412, upload-time = "2024-11-26T02:00:52.458Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e3/3b16c3190f3d71d3b10f6758d2d5f7779ef008c4fd367cedab3ed178a9f7/pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aebc13a11ed3032d8dd6e7171eb6e86d40d67a5639d96c35142bd568b9299324", size = 39119106, upload-time = "2024-11-26T02:00:57.219Z" }, + { url = "https://files.pythonhosted.org/packages/1d/d6/5d704b0d25c3c79532f8c0639f253ec2803b897100f64bcb3f53ced236e5/pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6cf5c05f3cee251d80e98726b5c7cc9f21bab9e9783673bac58e6dfab57ecc8", size = 40090940, upload-time = "2024-11-26T02:01:02.31Z" }, + { url = "https://files.pythonhosted.org/packages/37/29/366bc7e588220d74ec00e497ac6710c2833c9176f0372fe0286929b2d64c/pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:11b676cd410cf162d3f6a70b43fb9e1e40affbc542a1e9ed3681895f2962d3d9", size = 38548177, upload-time = "2024-11-26T02:01:07.371Z" }, + { url = "https://files.pythonhosted.org/packages/c8/11/fabf6ecabb1fe5b7d96889228ca2a9158c4c3bb732e3b8ee3f7f6d40b703/pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:b76130d835261b38f14fc41fdfb39ad8d672afb84c447126b84d5472244cfaba", size = 40043567, upload-time = "2024-11-26T02:01:12.931Z" }, +] + +[[package]] +name = "pyasn1" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/e9/01f1a64245b89f039897cb0130016d79f77d52669aae6ee7b159a6c4c018/pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034", size = 145322, upload-time = "2024-09-10T22:41:42.55Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/f1/d6a797abb14f6283c0ddff96bbdd46937f64122b8c925cab503dd37f8214/pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", size = 83135, upload-time = "2024-09-11T16:00:36.122Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + +[[package]] +name = "pycparser" +version = "2.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, +] + +[[package]] +name = "pydantic" +version = "2.11.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/dd/4325abf92c39ba8623b5af936ddb36ffcfe0beae70405d456ab1fb2f5b8c/pydantic-2.11.7.tar.gz", hash = "sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db", size = 788350, upload-time = "2025-06-14T08:33:17.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/c0/ec2b1c8712ca690e5d61979dee872603e92b8a32f94cc1b72d53beab008a/pydantic-2.11.7-py3-none-any.whl", hash = "sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b", size = 444782, upload-time = "2025-06-14T08:33:14.905Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.33.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/88/5f2260bdfae97aabf98f1778d43f69574390ad787afb646292a638c923d4/pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc", size = 435195, upload-time = "2025-04-23T18:33:52.104Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/8a/2b41c97f554ec8c71f2a8a5f85cb56a8b0956addfe8b0efb5b3d77e8bdc3/pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc", size = 2009000, upload-time = "2025-04-23T18:31:25.863Z" }, + { url = "https://files.pythonhosted.org/packages/a1/02/6224312aacb3c8ecbaa959897af57181fb6cf3a3d7917fd44d0f2917e6f2/pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7", size = 1847996, upload-time = "2025-04-23T18:31:27.341Z" }, + { url = "https://files.pythonhosted.org/packages/d6/46/6dcdf084a523dbe0a0be59d054734b86a981726f221f4562aed313dbcb49/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025", size = 1880957, upload-time = "2025-04-23T18:31:28.956Z" }, + { url = "https://files.pythonhosted.org/packages/ec/6b/1ec2c03837ac00886ba8160ce041ce4e325b41d06a034adbef11339ae422/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011", size = 1964199, upload-time = "2025-04-23T18:31:31.025Z" }, + { url = "https://files.pythonhosted.org/packages/2d/1d/6bf34d6adb9debd9136bd197ca72642203ce9aaaa85cfcbfcf20f9696e83/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f", size = 2120296, upload-time = "2025-04-23T18:31:32.514Z" }, + { url = "https://files.pythonhosted.org/packages/e0/94/2bd0aaf5a591e974b32a9f7123f16637776c304471a0ab33cf263cf5591a/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88", size = 2676109, upload-time = "2025-04-23T18:31:33.958Z" }, + { url = "https://files.pythonhosted.org/packages/f9/41/4b043778cf9c4285d59742281a769eac371b9e47e35f98ad321349cc5d61/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1", size = 2002028, upload-time = "2025-04-23T18:31:39.095Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d5/7bb781bf2748ce3d03af04d5c969fa1308880e1dca35a9bd94e1a96a922e/pydantic_core-2.33.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b", size = 2100044, upload-time = "2025-04-23T18:31:41.034Z" }, + { url = "https://files.pythonhosted.org/packages/fe/36/def5e53e1eb0ad896785702a5bbfd25eed546cdcf4087ad285021a90ed53/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1", size = 2058881, upload-time = "2025-04-23T18:31:42.757Z" }, + { url = "https://files.pythonhosted.org/packages/01/6c/57f8d70b2ee57fc3dc8b9610315949837fa8c11d86927b9bb044f8705419/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6", size = 2227034, upload-time = "2025-04-23T18:31:44.304Z" }, + { url = "https://files.pythonhosted.org/packages/27/b9/9c17f0396a82b3d5cbea4c24d742083422639e7bb1d5bf600e12cb176a13/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea", size = 2234187, upload-time = "2025-04-23T18:31:45.891Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6a/adf5734ffd52bf86d865093ad70b2ce543415e0e356f6cacabbc0d9ad910/pydantic_core-2.33.2-cp312-cp312-win32.whl", hash = "sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290", size = 1892628, upload-time = "2025-04-23T18:31:47.819Z" }, + { url = "https://files.pythonhosted.org/packages/43/e4/5479fecb3606c1368d496a825d8411e126133c41224c1e7238be58b87d7e/pydantic_core-2.33.2-cp312-cp312-win_amd64.whl", hash = "sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2", size = 1955866, upload-time = "2025-04-23T18:31:49.635Z" }, + { url = "https://files.pythonhosted.org/packages/0d/24/8b11e8b3e2be9dd82df4b11408a67c61bb4dc4f8e11b5b0fc888b38118b5/pydantic_core-2.33.2-cp312-cp312-win_arm64.whl", hash = "sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab", size = 1888894, upload-time = "2025-04-23T18:31:51.609Z" }, + { url = "https://files.pythonhosted.org/packages/46/8c/99040727b41f56616573a28771b1bfa08a3d3fe74d3d513f01251f79f172/pydantic_core-2.33.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f", size = 2015688, upload-time = "2025-04-23T18:31:53.175Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cc/5999d1eb705a6cefc31f0b4a90e9f7fc400539b1a1030529700cc1b51838/pydantic_core-2.33.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6", size = 1844808, upload-time = "2025-04-23T18:31:54.79Z" }, + { url = "https://files.pythonhosted.org/packages/6f/5e/a0a7b8885c98889a18b6e376f344da1ef323d270b44edf8174d6bce4d622/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef", size = 1885580, upload-time = "2025-04-23T18:31:57.393Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2a/953581f343c7d11a304581156618c3f592435523dd9d79865903272c256a/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a", size = 1973859, upload-time = "2025-04-23T18:31:59.065Z" }, + { url = "https://files.pythonhosted.org/packages/e6/55/f1a813904771c03a3f97f676c62cca0c0a4138654107c1b61f19c644868b/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916", size = 2120810, upload-time = "2025-04-23T18:32:00.78Z" }, + { url = "https://files.pythonhosted.org/packages/aa/c3/053389835a996e18853ba107a63caae0b9deb4a276c6b472931ea9ae6e48/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a", size = 2676498, upload-time = "2025-04-23T18:32:02.418Z" }, + { url = "https://files.pythonhosted.org/packages/eb/3c/f4abd740877a35abade05e437245b192f9d0ffb48bbbbd708df33d3cda37/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d", size = 2000611, upload-time = "2025-04-23T18:32:04.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/a7/63ef2fed1837d1121a894d0ce88439fe3e3b3e48c7543b2a4479eb99c2bd/pydantic_core-2.33.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56", size = 2107924, upload-time = "2025-04-23T18:32:06.129Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/2551964ef045669801675f1cfc3b0d74147f4901c3ffa42be2ddb1f0efc4/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5", size = 2063196, upload-time = "2025-04-23T18:32:08.178Z" }, + { url = "https://files.pythonhosted.org/packages/26/bd/d9602777e77fc6dbb0c7db9ad356e9a985825547dce5ad1d30ee04903918/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e", size = 2236389, upload-time = "2025-04-23T18:32:10.242Z" }, + { url = "https://files.pythonhosted.org/packages/42/db/0e950daa7e2230423ab342ae918a794964b053bec24ba8af013fc7c94846/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162", size = 2239223, upload-time = "2025-04-23T18:32:12.382Z" }, + { url = "https://files.pythonhosted.org/packages/58/4d/4f937099c545a8a17eb52cb67fe0447fd9a373b348ccfa9a87f141eeb00f/pydantic_core-2.33.2-cp313-cp313-win32.whl", hash = "sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849", size = 1900473, upload-time = "2025-04-23T18:32:14.034Z" }, + { url = "https://files.pythonhosted.org/packages/a0/75/4a0a9bac998d78d889def5e4ef2b065acba8cae8c93696906c3a91f310ca/pydantic_core-2.33.2-cp313-cp313-win_amd64.whl", hash = "sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9", size = 1955269, upload-time = "2025-04-23T18:32:15.783Z" }, + { url = "https://files.pythonhosted.org/packages/f9/86/1beda0576969592f1497b4ce8e7bc8cbdf614c352426271b1b10d5f0aa64/pydantic_core-2.33.2-cp313-cp313-win_arm64.whl", hash = "sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9", size = 1893921, upload-time = "2025-04-23T18:32:18.473Z" }, + { url = "https://files.pythonhosted.org/packages/a4/7d/e09391c2eebeab681df2b74bfe6c43422fffede8dc74187b2b0bf6fd7571/pydantic_core-2.33.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac", size = 1806162, upload-time = "2025-04-23T18:32:20.188Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3d/847b6b1fed9f8ed3bb95a9ad04fbd0b212e832d4f0f50ff4d9ee5a9f15cf/pydantic_core-2.33.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5", size = 1981560, upload-time = "2025-04-23T18:32:22.354Z" }, + { url = "https://files.pythonhosted.org/packages/6f/9a/e73262f6c6656262b5fdd723ad90f518f579b7bc8622e43a942eec53c938/pydantic_core-2.33.2-cp313-cp313t-win_amd64.whl", hash = "sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9", size = 1935777, upload-time = "2025-04-23T18:32:25.088Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pyparsing" +version = "3.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/22/f1129e69d94ffff626bdb5c835506b3a5b4f3d070f17ea295e12c2c6f60f/pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be", size = 1088608, upload-time = "2025-03-25T05:01:28.114Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/e7/df2285f3d08fee213f2d041540fa4fc9ca6c2d44cf36d3a035bf2a8d2bcc/pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf", size = 111120, upload-time = "2025-03-25T05:01:24.908Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/b0/4bc07ccd3572a2f9df7e6782f52b0c6c90dcbb803ac4a167702d7d0dfe1e/python_dotenv-1.1.1.tar.gz", hash = "sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab", size = 41978, upload-time = "2025-06-24T04:21:07.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/ed/539768cf28c661b5b068d66d96a2f155c4971a5d55684a514c1a0e0dec2f/python_dotenv-1.1.1-py3-none-any.whl", hash = "sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc", size = 20556, upload-time = "2025-06-24T04:21:06.073Z" }, +] + +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + +[[package]] +name = "pywin32" +version = "311" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/ab/01ea1943d4eba0f850c3c61e78e8dd59757ff815ff3ccd0a84de5f541f42/pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31", size = 8706543, upload-time = "2025-07-14T20:13:20.765Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a8/a0e8d07d4d051ec7502cd58b291ec98dcc0c3fff027caad0470b72cfcc2f/pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067", size = 9495040, upload-time = "2025-07-14T20:13:22.543Z" }, + { url = "https://files.pythonhosted.org/packages/ba/3a/2ae996277b4b50f17d61f0603efd8253cb2d79cc7ae159468007b586396d/pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852", size = 8710102, upload-time = "2025-07-14T20:13:24.682Z" }, + { url = "https://files.pythonhosted.org/packages/a5/be/3fd5de0979fcb3994bfee0d65ed8ca9506a8a1260651b86174f6a86f52b3/pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d", size = 8705700, upload-time = "2025-07-14T20:13:26.471Z" }, + { url = "https://files.pythonhosted.org/packages/e3/28/e0a1909523c6890208295a29e05c2adb2126364e289826c0a8bc7297bd5c/pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d", size = 9494700, upload-time = "2025-07-14T20:13:28.243Z" }, + { url = "https://files.pythonhosted.org/packages/04/bf/90339ac0f55726dce7d794e6d79a18a91265bdf3aa70b6b9ca52f35e022a/pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a", size = 8709318, upload-time = "2025-07-14T20:13:30.348Z" }, + { url = "https://files.pythonhosted.org/packages/c9/31/097f2e132c4f16d99a22bfb777e0fd88bd8e1c634304e102f313af69ace5/pywin32-311-cp314-cp314-win32.whl", hash = "sha256:b7a2c10b93f8986666d0c803ee19b5990885872a7de910fc460f9b0c2fbf92ee", size = 8840714, upload-time = "2025-07-14T20:13:32.449Z" }, + { url = "https://files.pythonhosted.org/packages/90/4b/07c77d8ba0e01349358082713400435347df8426208171ce297da32c313d/pywin32-311-cp314-cp314-win_amd64.whl", hash = "sha256:3aca44c046bd2ed8c90de9cb8427f581c479e594e99b5c0bb19b29c10fd6cb87", size = 9656800, upload-time = "2025-07-14T20:13:34.312Z" }, + { url = "https://files.pythonhosted.org/packages/c0/d2/21af5c535501a7233e734b8af901574572da66fcc254cb35d0609c9080dd/pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42", size = 8932540, upload-time = "2025-07-14T20:13:36.379Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, + { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, + { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, + { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, + { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, + { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, + { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, + { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, + { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, +] + +[[package]] +name = "pyzmq" +version = "27.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "implementation_name == 'pypy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/66/159f38d184f08b5f971b467f87b1ab142ab1320d5200825c824b32b84b66/pyzmq-27.0.2.tar.gz", hash = "sha256:b398dd713b18de89730447347e96a0240225e154db56e35b6bb8447ffdb07798", size = 281440, upload-time = "2025-08-21T04:23:26.334Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/69/b3a729e7b03e412bee2b1823ab8d22e20a92593634f664afd04c6c9d9ac0/pyzmq-27.0.2-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:5da05e3c22c95e23bfc4afeee6ff7d4be9ff2233ad6cb171a0e8257cd46b169a", size = 1305910, upload-time = "2025-08-21T04:21:27.609Z" }, + { url = "https://files.pythonhosted.org/packages/15/b7/f6a6a285193d489b223c340b38ee03a673467cb54914da21c3d7849f1b10/pyzmq-27.0.2-cp312-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4e4520577971d01d47e2559bb3175fce1be9103b18621bf0b241abe0a933d040", size = 895507, upload-time = "2025-08-21T04:21:29.005Z" }, + { url = "https://files.pythonhosted.org/packages/17/e6/c4ed2da5ef9182cde1b1f5d0051a986e76339d71720ec1a00be0b49275ad/pyzmq-27.0.2-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56d7de7bf73165b90bd25a8668659ccb134dd28449116bf3c7e9bab5cf8a8ec9", size = 652670, upload-time = "2025-08-21T04:21:30.71Z" }, + { url = "https://files.pythonhosted.org/packages/0e/66/d781ab0636570d32c745c4e389b1c6b713115905cca69ab6233508622edd/pyzmq-27.0.2-cp312-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:340e7cddc32f147c6c00d116a3f284ab07ee63dbd26c52be13b590520434533c", size = 840581, upload-time = "2025-08-21T04:21:32.008Z" }, + { url = "https://files.pythonhosted.org/packages/a6/df/f24790caf565d72544f5c8d8500960b9562c1dc848d6f22f3c7e122e73d4/pyzmq-27.0.2-cp312-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ba95693f9df8bb4a9826464fb0fe89033936f35fd4a8ff1edff09a473570afa0", size = 1641931, upload-time = "2025-08-21T04:21:33.371Z" }, + { url = "https://files.pythonhosted.org/packages/65/65/77d27b19fc5e845367f9100db90b9fce924f611b14770db480615944c9c9/pyzmq-27.0.2-cp312-abi3-musllinux_1_2_i686.whl", hash = "sha256:ca42a6ce2d697537da34f77a1960d21476c6a4af3e539eddb2b114c3cf65a78c", size = 2021226, upload-time = "2025-08-21T04:21:35.301Z" }, + { url = "https://files.pythonhosted.org/packages/5b/65/1ed14421ba27a4207fa694772003a311d1142b7f543179e4d1099b7eb746/pyzmq-27.0.2-cp312-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3e44e665d78a07214b2772ccbd4b9bcc6d848d7895f1b2d7653f047b6318a4f6", size = 1878047, upload-time = "2025-08-21T04:21:36.749Z" }, + { url = "https://files.pythonhosted.org/packages/dd/dc/e578549b89b40dc78a387ec471c2a360766690c0a045cd8d1877d401012d/pyzmq-27.0.2-cp312-abi3-win32.whl", hash = "sha256:272d772d116615397d2be2b1417b3b8c8bc8671f93728c2f2c25002a4530e8f6", size = 558757, upload-time = "2025-08-21T04:21:38.2Z" }, + { url = "https://files.pythonhosted.org/packages/b5/89/06600980aefcc535c758414da969f37a5194ea4cdb73b745223f6af3acfb/pyzmq-27.0.2-cp312-abi3-win_amd64.whl", hash = "sha256:734be4f44efba0aa69bf5f015ed13eb69ff29bf0d17ea1e21588b095a3147b8e", size = 619281, upload-time = "2025-08-21T04:21:39.909Z" }, + { url = "https://files.pythonhosted.org/packages/30/84/df8a5c089552d17c9941d1aea4314b606edf1b1622361dae89aacedc6467/pyzmq-27.0.2-cp312-abi3-win_arm64.whl", hash = "sha256:41f0bd56d9279392810950feb2785a419c2920bbf007fdaaa7f4a07332ae492d", size = 552680, upload-time = "2025-08-21T04:21:41.571Z" }, + { url = "https://files.pythonhosted.org/packages/b4/7b/b79e976508517ab80dc800f7021ef1fb602a6d55e4caa2d47fb3dca5d8b6/pyzmq-27.0.2-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:7f01118133427cd7f34ee133b5098e2af5f70303fa7519785c007bca5aa6f96a", size = 1122259, upload-time = "2025-08-21T04:21:43.063Z" }, + { url = "https://files.pythonhosted.org/packages/2b/1c/777217b9940ebcb7e71c924184ca5f31e410580a58d9fd93798589f0d31c/pyzmq-27.0.2-cp313-cp313-android_24_x86_64.whl", hash = "sha256:e4b860edf6379a7234ccbb19b4ed2c57e3ff569c3414fadfb49ae72b61a8ef07", size = 1156113, upload-time = "2025-08-21T04:21:44.566Z" }, + { url = "https://files.pythonhosted.org/packages/59/7d/654657a4c6435f41538182e71b61eac386a789a2bbb6f30171915253a9a7/pyzmq-27.0.2-cp313-cp313t-macosx_10_15_universal2.whl", hash = "sha256:cb77923ea163156da14295c941930bd525df0d29c96c1ec2fe3c3806b1e17cb3", size = 1341437, upload-time = "2025-08-21T04:21:46.019Z" }, + { url = "https://files.pythonhosted.org/packages/20/a0/5ed7710037f9c096017adc748bcb1698674a2d297f8b9422d38816f7b56a/pyzmq-27.0.2-cp313-cp313t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:61678b7407b04df8f9423f188156355dc94d0fb52d360ae79d02ed7e0d431eea", size = 897888, upload-time = "2025-08-21T04:21:47.362Z" }, + { url = "https://files.pythonhosted.org/packages/2c/8a/6e4699a60931c17e7406641d201d7f2c121e2a38979bc83226a6d8f1ba32/pyzmq-27.0.2-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3c824b70925963bdc8e39a642672c15ffaa67e7d4b491f64662dd56d6271263", size = 660727, upload-time = "2025-08-21T04:21:48.734Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d8/d761e438c186451bd89ce63a665cde5690c084b61cd8f5d7b51e966e875a/pyzmq-27.0.2-cp313-cp313t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c4833e02fcf2751975457be1dfa2f744d4d09901a8cc106acaa519d868232175", size = 848136, upload-time = "2025-08-21T04:21:50.416Z" }, + { url = "https://files.pythonhosted.org/packages/43/f1/a0f31684efdf3eb92f46b7dd2117e752208115e89d278f8ca5f413c5bb85/pyzmq-27.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b18045668d09cf0faa44918af2a67f0dbbef738c96f61c2f1b975b1ddb92ccfc", size = 1650402, upload-time = "2025-08-21T04:21:52.235Z" }, + { url = "https://files.pythonhosted.org/packages/41/fd/0d7f2a1732812df02c85002770da4a7864c79b210084bcdab01ea57e8d92/pyzmq-27.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:bbbb7e2f3ac5a22901324e7b086f398b8e16d343879a77b15ca3312e8cd8e6d5", size = 2024587, upload-time = "2025-08-21T04:21:54.07Z" }, + { url = "https://files.pythonhosted.org/packages/f1/73/358be69e279a382dd09e46dda29df8446365cddee4f79ef214e71e5b2b5a/pyzmq-27.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:b751914a73604d40d88a061bab042a11d4511b3ddbb7624cd83c39c8a498564c", size = 1885493, upload-time = "2025-08-21T04:21:55.588Z" }, + { url = "https://files.pythonhosted.org/packages/c5/7b/e9951ad53b3dfed8cfb4c2cfd6e0097c9b454e5c0d0e6df5f2b60d7c8c3d/pyzmq-27.0.2-cp313-cp313t-win32.whl", hash = "sha256:3e8f833dd82af11db5321c414638045c70f61009f72dd61c88db4a713c1fb1d2", size = 574934, upload-time = "2025-08-21T04:21:57.52Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/1a7fc3a92f2124a63e6e2a6afa0af471a5c0c713e776b476d4eda5111b13/pyzmq-27.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:5b45153cb8eadcab14139970643a84f7a7b08dda541fbc1f6f4855c49334b549", size = 640932, upload-time = "2025-08-21T04:21:59.527Z" }, + { url = "https://files.pythonhosted.org/packages/2a/52/2598a94ac251a7c83f3887866225eea1952b0d4463a68df5032eb00ff052/pyzmq-27.0.2-cp313-cp313t-win_arm64.whl", hash = "sha256:86898f5c9730df23427c1ee0097d8aa41aa5f89539a79e48cd0d2c22d059f1b7", size = 561315, upload-time = "2025-08-21T04:22:01.295Z" }, + { url = "https://files.pythonhosted.org/packages/42/7d/10ef02ea36590b29d48ef88eb0831f0af3eb240cccca2752556faec55f59/pyzmq-27.0.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d2b4b261dce10762be5c116b6ad1f267a9429765b493c454f049f33791dd8b8a", size = 1341463, upload-time = "2025-08-21T04:22:02.712Z" }, + { url = "https://files.pythonhosted.org/packages/94/36/115d18dade9a3d4d3d08dd8bfe5459561b8e02815f99df040555fdd7768e/pyzmq-27.0.2-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4e4d88b6cff156fed468903006b24bbd85322612f9c2f7b96e72d5016fd3f543", size = 897840, upload-time = "2025-08-21T04:22:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/39/66/083b37839b95c386a95f1537bb41bdbf0c002b7c55b75ee737949cecb11f/pyzmq-27.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8426c0ebbc11ed8416a6e9409c194142d677c2c5c688595f2743664e356d9e9b", size = 660704, upload-time = "2025-08-21T04:22:06.389Z" }, + { url = "https://files.pythonhosted.org/packages/76/5a/196ab46e549ba35bf3268f575e10cfac0dc86b78dcaa7a3e36407ecda752/pyzmq-27.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:565bee96a155fe6452caed5fb5f60c9862038e6b51a59f4f632562081cdb4004", size = 848037, upload-time = "2025-08-21T04:22:07.817Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/a27b9eb44b2e615a9ecb8510ebb023cc1d2d251181e4a1e50366bfbf94d6/pyzmq-27.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5de735c745ca5cefe9c2d1547d8f28cfe1b1926aecb7483ab1102fd0a746c093", size = 1650278, upload-time = "2025-08-21T04:22:09.269Z" }, + { url = "https://files.pythonhosted.org/packages/62/ac/3e9af036bfaf718ab5e69ded8f6332da392c5450ad43e8e3ca66797f145a/pyzmq-27.0.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ea4f498f8115fd90d7bf03a3e83ae3e9898e43362f8e8e8faec93597206e15cc", size = 2024504, upload-time = "2025-08-21T04:22:10.778Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e9/3202d31788df8ebaa176b23d846335eb9c768d8b43c0506bbd6265ad36a0/pyzmq-27.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d00e81cb0afd672915257a3927124ee2ad117ace3c256d39cd97ca3f190152ad", size = 1885381, upload-time = "2025-08-21T04:22:12.718Z" }, + { url = "https://files.pythonhosted.org/packages/4b/ed/42de80b7ab4e8fcf13376f81206cf8041740672ac1fd2e1c598d63f595bf/pyzmq-27.0.2-cp314-cp314t-win32.whl", hash = "sha256:0f6e9b00d81b58f859fffc112365d50413954e02aefe36c5b4c8fb4af79f8cc3", size = 587526, upload-time = "2025-08-21T04:22:14.18Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c8/8f3c72d6f0bfbf090aa5e283576073ca5c59839b85a5cc8c66ddb9b59801/pyzmq-27.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:2e73cf3b127a437fef4100eb3ac2ebe6b49e655bb721329f667f59eca0a26221", size = 661368, upload-time = "2025-08-21T04:22:15.677Z" }, + { url = "https://files.pythonhosted.org/packages/69/a4/7ee652ea1c77d872f5d99ed937fa8bbd1f6f4b7a39a6d3a0076c286e0c3e/pyzmq-27.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:4108785f2e5ac865d06f678a07a1901e3465611356df21a545eeea8b45f56265", size = 574901, upload-time = "2025-08-21T04:22:17.423Z" }, +] + +[[package]] +name = "ray" +version = "2.48.0" +source = { url = "http://localhost:9478/ray/ray-2.48.0-cp312-cp312-manylinux2014_x86_64.whl" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "jsonschema" }, + { name = "msgpack" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "pyyaml" }, + { name = "requests" }, +] +wheels = [ + { url = "http://localhost:9478/ray/ray-2.48.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:f93c6f2e7a91d5dfc6442c390005509c933dbc49d544d9b73bd81687529f4b57" }, +] + +[package.optional-dependencies] +data = [ + { name = "fsspec" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "polars" }, + { name = "pyarrow" }, +] +serve = [ + { name = "aiohttp" }, + { name = "aiohttp-cors" }, + { name = "colorful" }, + { name = "fastapi" }, + { name = "grpcio" }, + { name = "opencensus" }, + { name = "opentelemetry-exporter-prometheus" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "prometheus-client" }, + { name = "py-spy" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "smart-open" }, + { name = "starlette" }, + { name = "uvicorn", extra = ["standard"] }, + { name = "virtualenv" }, + { name = "watchfiles" }, +] +train = [ + { name = "fsspec" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "tensorboardx" }, +] +tune = [ + { name = "fsspec" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "requests" }, + { name = "tensorboardx" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiohttp", marker = "extra == 'air'", specifier = ">=3.7" }, + { name = "aiohttp", marker = "extra == 'all'", specifier = ">=3.7" }, + { name = "aiohttp", marker = "extra == 'default'", specifier = ">=3.7" }, + { name = "aiohttp", marker = "extra == 'llm'", specifier = ">=3.7" }, + { name = "aiohttp", marker = "extra == 'serve'", specifier = ">=3.7" }, + { name = "aiohttp", marker = "extra == 'serve-grpc'", specifier = ">=3.7" }, + { name = "aiohttp-cors", marker = "extra == 'air'" }, + { name = "aiohttp-cors", marker = "extra == 'all'" }, + { name = "aiohttp-cors", marker = "extra == 'default'" }, + { name = "aiohttp-cors", marker = "extra == 'llm'" }, + { name = "aiohttp-cors", marker = "extra == 'serve'" }, + { name = "aiohttp-cors", marker = "extra == 'serve-grpc'" }, + { name = "async-timeout", marker = "python_full_version < '3.11' and extra == 'llm'" }, + { name = "click", specifier = ">=7.0" }, + { name = "colorful", marker = "extra == 'air'" }, + { name = "colorful", marker = "extra == 'all'" }, + { name = "colorful", marker = "extra == 'default'" }, + { name = "colorful", marker = "extra == 'llm'" }, + { name = "colorful", marker = "extra == 'serve'" }, + { name = "colorful", marker = "extra == 'serve-grpc'" }, + { name = "cupy-cuda12x", marker = "sys_platform != 'darwin' and extra == 'adag'" }, + { name = "cupy-cuda12x", marker = "sys_platform != 'darwin' and extra == 'all'" }, + { name = "cupy-cuda12x", marker = "sys_platform != 'darwin' and extra == 'cgraph'" }, + { name = "dm-tree", marker = "extra == 'all'" }, + { name = "dm-tree", marker = "extra == 'rllib'" }, + { name = "fastapi", marker = "extra == 'air'" }, + { name = "fastapi", marker = "extra == 'all'" }, + { name = "fastapi", marker = "extra == 'llm'" }, + { name = "fastapi", marker = "extra == 'serve'" }, + { name = "fastapi", marker = "extra == 'serve-grpc'" }, + { name = "filelock" }, + { name = "fsspec", marker = "extra == 'air'" }, + { name = "fsspec", marker = "extra == 'all'" }, + { name = "fsspec", marker = "extra == 'data'" }, + { name = "fsspec", marker = "extra == 'llm'" }, + { name = "fsspec", marker = "extra == 'rllib'" }, + { name = "fsspec", marker = "extra == 'train'" }, + { name = "fsspec", marker = "extra == 'tune'" }, + { name = "grpcio", marker = "python_full_version >= '3.10' and extra == 'air'", specifier = ">=1.42.0" }, + { name = "grpcio", marker = "python_full_version >= '3.10' and extra == 'all'", specifier = ">=1.42.0" }, + { name = "grpcio", marker = "python_full_version >= '3.10' and extra == 'default'", specifier = ">=1.42.0" }, + { name = "grpcio", marker = "python_full_version >= '3.10' and extra == 'llm'", specifier = ">=1.42.0" }, + { name = "grpcio", marker = "python_full_version >= '3.10' and extra == 'serve'", specifier = ">=1.42.0" }, + { name = "grpcio", marker = "python_full_version >= '3.10' and extra == 'serve-grpc'", specifier = ">=1.42.0" }, + { name = "grpcio", marker = "python_full_version < '3.10' and extra == 'air'", specifier = ">=1.32.0" }, + { name = "grpcio", marker = "python_full_version < '3.10' and extra == 'all'", specifier = ">=1.32.0" }, + { name = "grpcio", marker = "python_full_version < '3.10' and extra == 'default'", specifier = ">=1.32.0" }, + { name = "grpcio", marker = "python_full_version < '3.10' and extra == 'llm'", specifier = ">=1.32.0" }, + { name = "grpcio", marker = "python_full_version < '3.10' and extra == 'serve'", specifier = ">=1.32.0" }, + { name = "grpcio", marker = "python_full_version < '3.10' and extra == 'serve-grpc'", specifier = ">=1.32.0" }, + { name = "grpcio", marker = "sys_platform == 'darwin' and extra == 'all'", specifier = "!=1.56.0" }, + { name = "grpcio", marker = "sys_platform == 'darwin' and extra == 'client'", specifier = "!=1.56.0" }, + { name = "grpcio", marker = "extra == 'all'" }, + { name = "grpcio", marker = "extra == 'client'" }, + { name = "gymnasium", marker = "extra == 'all'", specifier = "==1.0.0" }, + { name = "gymnasium", marker = "extra == 'rllib'", specifier = "==1.0.0" }, + { name = "jsonref", marker = "extra == 'llm'", specifier = ">=1.1.0" }, + { name = "jsonschema" }, + { name = "jsonschema", marker = "extra == 'llm'" }, + { name = "lz4", marker = "extra == 'all'" }, + { name = "lz4", marker = "extra == 'rllib'" }, + { name = "memray", marker = "sys_platform != 'win32' and extra == 'all'" }, + { name = "memray", marker = "sys_platform != 'win32' and extra == 'observability'" }, + { name = "msgpack", specifier = ">=1.0.0,<2.0.0" }, + { name = "ninja", marker = "extra == 'llm'" }, + { name = "numpy", marker = "extra == 'air'", specifier = ">=1.20" }, + { name = "numpy", marker = "extra == 'all'", specifier = ">=1.20" }, + { name = "numpy", marker = "extra == 'data'", specifier = ">=1.20" }, + { name = "numpy", marker = "extra == 'llm'", specifier = ">=1.20" }, + { name = "opencensus", marker = "extra == 'air'" }, + { name = "opencensus", marker = "extra == 'all'" }, + { name = "opencensus", marker = "extra == 'default'" }, + { name = "opencensus", marker = "extra == 'llm'" }, + { name = "opencensus", marker = "extra == 'serve'" }, + { name = "opencensus", marker = "extra == 'serve-grpc'" }, + { name = "opentelemetry-exporter-prometheus", marker = "extra == 'air'" }, + { name = "opentelemetry-exporter-prometheus", marker = "extra == 'all'" }, + { name = "opentelemetry-exporter-prometheus", marker = "extra == 'default'" }, + { name = "opentelemetry-exporter-prometheus", marker = "extra == 'llm'" }, + { name = "opentelemetry-exporter-prometheus", marker = "extra == 'serve'" }, + { name = "opentelemetry-exporter-prometheus", marker = "extra == 'serve-grpc'" }, + { name = "opentelemetry-proto", marker = "extra == 'air'" }, + { name = "opentelemetry-proto", marker = "extra == 'all'" }, + { name = "opentelemetry-proto", marker = "extra == 'default'" }, + { name = "opentelemetry-proto", marker = "extra == 'llm'" }, + { name = "opentelemetry-proto", marker = "extra == 'serve'" }, + { name = "opentelemetry-proto", marker = "extra == 'serve-grpc'" }, + { name = "opentelemetry-sdk", marker = "extra == 'air'", specifier = ">=1.30.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'all'", specifier = ">=1.30.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'default'", specifier = ">=1.30.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'llm'", specifier = ">=1.30.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'serve'", specifier = ">=1.30.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'serve-grpc'", specifier = ">=1.30.0" }, + { name = "ormsgpack", marker = "extra == 'all'", specifier = "==1.7.0" }, + { name = "ormsgpack", marker = "extra == 'rllib'", specifier = "==1.7.0" }, + { name = "packaging" }, + { name = "pandas", marker = "extra == 'air'" }, + { name = "pandas", marker = "extra == 'air'", specifier = ">=1.3" }, + { name = "pandas", marker = "extra == 'all'" }, + { name = "pandas", marker = "extra == 'all'", specifier = ">=1.3" }, + { name = "pandas", marker = "extra == 'data'", specifier = ">=1.3" }, + { name = "pandas", marker = "extra == 'llm'", specifier = ">=1.3" }, + { name = "pandas", marker = "extra == 'rllib'" }, + { name = "pandas", marker = "extra == 'train'" }, + { name = "pandas", marker = "extra == 'tune'" }, + { name = "polars", marker = "extra == 'air'", specifier = ">=1.30.0,<2.0.0" }, + { name = "polars", marker = "extra == 'all'", specifier = ">=1.30.0,<2.0.0" }, + { name = "polars", marker = "extra == 'data'", specifier = ">=1.30.0,<2.0.0" }, + { name = "polars", marker = "extra == 'llm'", specifier = ">=1.30.0,<2.0.0" }, + { name = "prometheus-client", marker = "extra == 'air'", specifier = ">=0.7.1" }, + { name = "prometheus-client", marker = "extra == 'all'", specifier = ">=0.7.1" }, + { name = "prometheus-client", marker = "extra == 'default'", specifier = ">=0.7.1" }, + { name = "prometheus-client", marker = "extra == 'llm'", specifier = ">=0.7.1" }, + { name = "prometheus-client", marker = "extra == 'serve'", specifier = ">=0.7.1" }, + { name = "prometheus-client", marker = "extra == 'serve-grpc'", specifier = ">=0.7.1" }, + { name = "protobuf", specifier = ">=3.15.3,!=3.19.5" }, + { name = "py-spy", marker = "python_full_version >= '3.12' and extra == 'air'", specifier = ">=0.4.0" }, + { name = "py-spy", marker = "python_full_version >= '3.12' and extra == 'all'", specifier = ">=0.4.0" }, + { name = "py-spy", marker = "python_full_version >= '3.12' and extra == 'default'", specifier = ">=0.4.0" }, + { name = "py-spy", marker = "python_full_version >= '3.12' and extra == 'llm'", specifier = ">=0.4.0" }, + { name = "py-spy", marker = "python_full_version >= '3.12' and extra == 'serve'", specifier = ">=0.4.0" }, + { name = "py-spy", marker = "python_full_version >= '3.12' and extra == 'serve-grpc'", specifier = ">=0.4.0" }, + { name = "py-spy", marker = "python_full_version < '3.12' and extra == 'air'", specifier = ">=0.2.0" }, + { name = "py-spy", marker = "python_full_version < '3.12' and extra == 'all'", specifier = ">=0.2.0" }, + { name = "py-spy", marker = "python_full_version < '3.12' and extra == 'default'", specifier = ">=0.2.0" }, + { name = "py-spy", marker = "python_full_version < '3.12' and extra == 'llm'", specifier = ">=0.2.0" }, + { name = "py-spy", marker = "python_full_version < '3.12' and extra == 'serve'", specifier = ">=0.2.0" }, + { name = "py-spy", marker = "python_full_version < '3.12' and extra == 'serve-grpc'", specifier = ">=0.2.0" }, + { name = "pyarrow", marker = "extra == 'air'", specifier = ">=9.0.0" }, + { name = "pyarrow", marker = "extra == 'all'", specifier = ">=9.0.0" }, + { name = "pyarrow", marker = "extra == 'data'", specifier = ">=9.0.0" }, + { name = "pyarrow", marker = "extra == 'llm'", specifier = ">=9.0.0" }, + { name = "pyarrow", marker = "extra == 'rllib'", specifier = ">=9.0.0" }, + { name = "pyarrow", marker = "extra == 'train'", specifier = ">=9.0.0" }, + { name = "pyarrow", marker = "extra == 'tune'", specifier = ">=9.0.0" }, + { name = "pydantic", marker = "extra == 'air'", specifier = "!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3" }, + { name = "pydantic", marker = "extra == 'all'", specifier = "!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3" }, + { name = "pydantic", marker = "extra == 'default'", specifier = "!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3" }, + { name = "pydantic", marker = "extra == 'llm'", specifier = "!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3" }, + { name = "pydantic", marker = "extra == 'serve'", specifier = "!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3" }, + { name = "pydantic", marker = "extra == 'serve-grpc'", specifier = "!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3" }, + { name = "pydantic", marker = "extra == 'train'", specifier = "!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3" }, + { name = "pyopenssl", marker = "extra == 'all'" }, + { name = "pyopenssl", marker = "extra == 'serve-grpc'" }, + { name = "pyyaml" }, + { name = "pyyaml", marker = "extra == 'all'" }, + { name = "pyyaml", marker = "extra == 'rllib'" }, + { name = "requests" }, + { name = "requests", marker = "extra == 'air'" }, + { name = "requests", marker = "extra == 'all'" }, + { name = "requests", marker = "extra == 'default'" }, + { name = "requests", marker = "extra == 'llm'" }, + { name = "requests", marker = "extra == 'rllib'" }, + { name = "requests", marker = "extra == 'serve'" }, + { name = "requests", marker = "extra == 'serve-grpc'" }, + { name = "requests", marker = "extra == 'train'" }, + { name = "requests", marker = "extra == 'tune'" }, + { name = "scipy", marker = "extra == 'all'" }, + { name = "scipy", marker = "extra == 'rllib'" }, + { name = "smart-open", marker = "extra == 'air'" }, + { name = "smart-open", marker = "extra == 'all'" }, + { name = "smart-open", marker = "extra == 'default'" }, + { name = "smart-open", marker = "extra == 'llm'" }, + { name = "smart-open", marker = "extra == 'serve'" }, + { name = "smart-open", marker = "extra == 'serve-grpc'" }, + { name = "starlette", marker = "extra == 'air'" }, + { name = "starlette", marker = "extra == 'all'" }, + { name = "starlette", marker = "extra == 'llm'" }, + { name = "starlette", marker = "extra == 'serve'" }, + { name = "starlette", marker = "extra == 'serve-grpc'" }, + { name = "tensorboardx", marker = "extra == 'air'", specifier = ">=1.9" }, + { name = "tensorboardx", marker = "extra == 'all'", specifier = ">=1.9" }, + { name = "tensorboardx", marker = "extra == 'rllib'", specifier = ">=1.9" }, + { name = "tensorboardx", marker = "extra == 'train'", specifier = ">=1.9" }, + { name = "tensorboardx", marker = "extra == 'tune'", specifier = ">=1.9" }, + { name = "typer", marker = "extra == 'llm'" }, + { name = "uvicorn", extras = ["standard"], marker = "extra == 'air'" }, + { name = "uvicorn", extras = ["standard"], marker = "extra == 'all'" }, + { name = "uvicorn", extras = ["standard"], marker = "extra == 'llm'" }, + { name = "uvicorn", extras = ["standard"], marker = "extra == 'serve'" }, + { name = "uvicorn", extras = ["standard"], marker = "extra == 'serve-grpc'" }, + { name = "virtualenv", marker = "extra == 'air'", specifier = ">=20.0.24,!=20.21.1" }, + { name = "virtualenv", marker = "extra == 'all'", specifier = ">=20.0.24,!=20.21.1" }, + { name = "virtualenv", marker = "extra == 'default'", specifier = ">=20.0.24,!=20.21.1" }, + { name = "virtualenv", marker = "extra == 'llm'", specifier = ">=20.0.24,!=20.21.1" }, + { name = "virtualenv", marker = "extra == 'serve'", specifier = ">=20.0.24,!=20.21.1" }, + { name = "virtualenv", marker = "extra == 'serve-grpc'", specifier = ">=20.0.24,!=20.21.1" }, + { name = "vllm", marker = "extra == 'llm'", specifier = ">=0.9.2" }, + { name = "watchfiles", marker = "extra == 'air'" }, + { name = "watchfiles", marker = "extra == 'all'" }, + { name = "watchfiles", marker = "extra == 'llm'" }, + { name = "watchfiles", marker = "extra == 'serve'" }, + { name = "watchfiles", marker = "extra == 'serve-grpc'" }, +] +provides-extras = ["cgraph", "client", "data", "default", "observability", "serve", "tune", "adag", "serve-grpc", "rllib", "train", "air", "all", "llm"] + +[[package]] +name = "referencing" +version = "0.36.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/db/98b5c277be99dd18bfd91dd04e1b759cad18d1a338188c936e92f921c7e2/referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa", size = 74744, upload-time = "2025-01-25T08:48:16.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/b1/3baf80dc6d2b7bc27a95a67752d0208e410351e3feb4eb78de5f77454d8d/referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0", size = 26775, upload-time = "2025-01-25T08:48:14.241Z" }, +] + +[[package]] +name = "regex" +version = "2025.7.34" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/de/e13fa6dc61d78b30ba47481f99933a3b49a57779d625c392d8036770a60d/regex-2025.7.34.tar.gz", hash = "sha256:9ead9765217afd04a86822dfcd4ed2747dfe426e887da413b15ff0ac2457e21a", size = 400714, upload-time = "2025-07-31T00:21:16.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/f0/31d62596c75a33f979317658e8d261574785c6cd8672c06741ce2e2e2070/regex-2025.7.34-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7f7211a746aced993bef487de69307a38c5ddd79257d7be83f7b202cb59ddb50", size = 485492, upload-time = "2025-07-31T00:19:35.57Z" }, + { url = "https://files.pythonhosted.org/packages/d8/16/b818d223f1c9758c3434be89aa1a01aae798e0e0df36c1f143d1963dd1ee/regex-2025.7.34-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fb31080f2bd0681484b275461b202b5ad182f52c9ec606052020fe13eb13a72f", size = 290000, upload-time = "2025-07-31T00:19:37.175Z" }, + { url = "https://files.pythonhosted.org/packages/cd/70/69506d53397b4bd6954061bae75677ad34deb7f6ca3ba199660d6f728ff5/regex-2025.7.34-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0200a5150c4cf61e407038f4b4d5cdad13e86345dac29ff9dab3d75d905cf130", size = 286072, upload-time = "2025-07-31T00:19:38.612Z" }, + { url = "https://files.pythonhosted.org/packages/b0/73/536a216d5f66084fb577bb0543b5cb7de3272eb70a157f0c3a542f1c2551/regex-2025.7.34-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:739a74970e736df0773788377969c9fea3876c2fc13d0563f98e5503e5185f46", size = 797341, upload-time = "2025-07-31T00:19:40.119Z" }, + { url = "https://files.pythonhosted.org/packages/26/af/733f8168449e56e8f404bb807ea7189f59507cbea1b67a7bbcd92f8bf844/regex-2025.7.34-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4fef81b2f7ea6a2029161ed6dea9ae13834c28eb5a95b8771828194a026621e4", size = 862556, upload-time = "2025-07-31T00:19:41.556Z" }, + { url = "https://files.pythonhosted.org/packages/19/dd/59c464d58c06c4f7d87de4ab1f590e430821345a40c5d345d449a636d15f/regex-2025.7.34-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea74cf81fe61a7e9d77989050d0089a927ab758c29dac4e8e1b6c06fccf3ebf0", size = 910762, upload-time = "2025-07-31T00:19:43Z" }, + { url = "https://files.pythonhosted.org/packages/37/a8/b05ccf33ceca0815a1e253693b2c86544932ebcc0049c16b0fbdf18b688b/regex-2025.7.34-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4636a7f3b65a5f340ed9ddf53585c42e3ff37101d383ed321bfe5660481744b", size = 801892, upload-time = "2025-07-31T00:19:44.645Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9a/b993cb2e634cc22810afd1652dba0cae156c40d4864285ff486c73cd1996/regex-2025.7.34-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cef962d7834437fe8d3da6f9bfc6f93f20f218266dcefec0560ed7765f5fe01", size = 786551, upload-time = "2025-07-31T00:19:46.127Z" }, + { url = "https://files.pythonhosted.org/packages/2d/79/7849d67910a0de4e26834b5bb816e028e35473f3d7ae563552ea04f58ca2/regex-2025.7.34-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:cbe1698e5b80298dbce8df4d8d1182279fbdaf1044e864cbc9d53c20e4a2be77", size = 856457, upload-time = "2025-07-31T00:19:47.562Z" }, + { url = "https://files.pythonhosted.org/packages/91/c6/de516bc082524b27e45cb4f54e28bd800c01efb26d15646a65b87b13a91e/regex-2025.7.34-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:32b9f9bcf0f605eb094b08e8da72e44badabb63dde6b83bd530580b488d1c6da", size = 848902, upload-time = "2025-07-31T00:19:49.312Z" }, + { url = "https://files.pythonhosted.org/packages/7d/22/519ff8ba15f732db099b126f039586bd372da6cd4efb810d5d66a5daeda1/regex-2025.7.34-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:524c868ba527eab4e8744a9287809579f54ae8c62fbf07d62aacd89f6026b282", size = 788038, upload-time = "2025-07-31T00:19:50.794Z" }, + { url = "https://files.pythonhosted.org/packages/3f/7d/aabb467d8f57d8149895d133c88eb809a1a6a0fe262c1d508eb9dfabb6f9/regex-2025.7.34-cp312-cp312-win32.whl", hash = "sha256:d600e58ee6d036081c89696d2bdd55d507498a7180df2e19945c6642fac59588", size = 264417, upload-time = "2025-07-31T00:19:52.292Z" }, + { url = "https://files.pythonhosted.org/packages/3b/39/bd922b55a4fc5ad5c13753274e5b536f5b06ec8eb9747675668491c7ab7a/regex-2025.7.34-cp312-cp312-win_amd64.whl", hash = "sha256:9a9ab52a466a9b4b91564437b36417b76033e8778e5af8f36be835d8cb370d62", size = 275387, upload-time = "2025-07-31T00:19:53.593Z" }, + { url = "https://files.pythonhosted.org/packages/f7/3c/c61d2fdcecb754a40475a3d1ef9a000911d3e3fc75c096acf44b0dfb786a/regex-2025.7.34-cp312-cp312-win_arm64.whl", hash = "sha256:c83aec91af9c6fbf7c743274fd952272403ad9a9db05fe9bfc9df8d12b45f176", size = 268482, upload-time = "2025-07-31T00:19:55.183Z" }, + { url = "https://files.pythonhosted.org/packages/15/16/b709b2119975035169a25aa8e4940ca177b1a2e25e14f8d996d09130368e/regex-2025.7.34-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c3c9740a77aeef3f5e3aaab92403946a8d34437db930a0280e7e81ddcada61f5", size = 485334, upload-time = "2025-07-31T00:19:56.58Z" }, + { url = "https://files.pythonhosted.org/packages/94/a6/c09136046be0595f0331bc58a0e5f89c2d324cf734e0b0ec53cf4b12a636/regex-2025.7.34-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:69ed3bc611540f2ea70a4080f853741ec698be556b1df404599f8724690edbcd", size = 289942, upload-time = "2025-07-31T00:19:57.943Z" }, + { url = "https://files.pythonhosted.org/packages/36/91/08fc0fd0f40bdfb0e0df4134ee37cfb16e66a1044ac56d36911fd01c69d2/regex-2025.7.34-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d03c6f9dcd562c56527c42b8530aad93193e0b3254a588be1f2ed378cdfdea1b", size = 285991, upload-time = "2025-07-31T00:19:59.837Z" }, + { url = "https://files.pythonhosted.org/packages/be/2f/99dc8f6f756606f0c214d14c7b6c17270b6bbe26d5c1f05cde9dbb1c551f/regex-2025.7.34-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6164b1d99dee1dfad33f301f174d8139d4368a9fb50bf0a3603b2eaf579963ad", size = 797415, upload-time = "2025-07-31T00:20:01.668Z" }, + { url = "https://files.pythonhosted.org/packages/62/cf/2fcdca1110495458ba4e95c52ce73b361cf1cafd8a53b5c31542cde9a15b/regex-2025.7.34-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1e4f4f62599b8142362f164ce776f19d79bdd21273e86920a7b604a4275b4f59", size = 862487, upload-time = "2025-07-31T00:20:03.142Z" }, + { url = "https://files.pythonhosted.org/packages/90/38/899105dd27fed394e3fae45607c1983e138273ec167e47882fc401f112b9/regex-2025.7.34-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:72a26dcc6a59c057b292f39d41465d8233a10fd69121fa24f8f43ec6294e5415", size = 910717, upload-time = "2025-07-31T00:20:04.727Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f6/4716198dbd0bcc9c45625ac4c81a435d1c4d8ad662e8576dac06bab35b17/regex-2025.7.34-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5273fddf7a3e602695c92716c420c377599ed3c853ea669c1fe26218867002f", size = 801943, upload-time = "2025-07-31T00:20:07.1Z" }, + { url = "https://files.pythonhosted.org/packages/40/5d/cff8896d27e4e3dd11dd72ac78797c7987eb50fe4debc2c0f2f1682eb06d/regex-2025.7.34-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c1844be23cd40135b3a5a4dd298e1e0c0cb36757364dd6cdc6025770363e06c1", size = 786664, upload-time = "2025-07-31T00:20:08.818Z" }, + { url = "https://files.pythonhosted.org/packages/10/29/758bf83cf7b4c34f07ac3423ea03cee3eb3176941641e4ccc05620f6c0b8/regex-2025.7.34-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dde35e2afbbe2272f8abee3b9fe6772d9b5a07d82607b5788e8508974059925c", size = 856457, upload-time = "2025-07-31T00:20:10.328Z" }, + { url = "https://files.pythonhosted.org/packages/d7/30/c19d212b619963c5b460bfed0ea69a092c6a43cba52a973d46c27b3e2975/regex-2025.7.34-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f3f6e8e7af516a7549412ce57613e859c3be27d55341a894aacaa11703a4c31a", size = 849008, upload-time = "2025-07-31T00:20:11.823Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b8/3c35da3b12c87e3cc00010ef6c3a4ae787cff0bc381aa3d251def219969a/regex-2025.7.34-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:469142fb94a869beb25b5f18ea87646d21def10fbacb0bcb749224f3509476f0", size = 788101, upload-time = "2025-07-31T00:20:13.729Z" }, + { url = "https://files.pythonhosted.org/packages/47/80/2f46677c0b3c2b723b2c358d19f9346e714113865da0f5f736ca1a883bde/regex-2025.7.34-cp313-cp313-win32.whl", hash = "sha256:da7507d083ee33ccea1310447410c27ca11fb9ef18c95899ca57ff60a7e4d8f1", size = 264401, upload-time = "2025-07-31T00:20:15.233Z" }, + { url = "https://files.pythonhosted.org/packages/be/fa/917d64dd074682606a003cba33585c28138c77d848ef72fc77cbb1183849/regex-2025.7.34-cp313-cp313-win_amd64.whl", hash = "sha256:9d644de5520441e5f7e2db63aec2748948cc39ed4d7a87fd5db578ea4043d997", size = 275368, upload-time = "2025-07-31T00:20:16.711Z" }, + { url = "https://files.pythonhosted.org/packages/65/cd/f94383666704170a2154a5df7b16be28f0c27a266bffcd843e58bc84120f/regex-2025.7.34-cp313-cp313-win_arm64.whl", hash = "sha256:7bf1c5503a9f2cbd2f52d7e260acb3131b07b6273c470abb78568174fe6bde3f", size = 268482, upload-time = "2025-07-31T00:20:18.189Z" }, + { url = "https://files.pythonhosted.org/packages/ac/23/6376f3a23cf2f3c00514b1cdd8c990afb4dfbac3cb4a68b633c6b7e2e307/regex-2025.7.34-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:8283afe7042d8270cecf27cca558873168e771183d4d593e3c5fe5f12402212a", size = 485385, upload-time = "2025-07-31T00:20:19.692Z" }, + { url = "https://files.pythonhosted.org/packages/73/5b/6d4d3a0b4d312adbfd6d5694c8dddcf1396708976dd87e4d00af439d962b/regex-2025.7.34-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6c053f9647e3421dd2f5dff8172eb7b4eec129df9d1d2f7133a4386319b47435", size = 289788, upload-time = "2025-07-31T00:20:21.941Z" }, + { url = "https://files.pythonhosted.org/packages/92/71/5862ac9913746e5054d01cb9fb8125b3d0802c0706ef547cae1e7f4428fa/regex-2025.7.34-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a16dd56bbcb7d10e62861c3cd000290ddff28ea142ffb5eb3470f183628011ac", size = 286136, upload-time = "2025-07-31T00:20:26.146Z" }, + { url = "https://files.pythonhosted.org/packages/27/df/5b505dc447eb71278eba10d5ec940769ca89c1af70f0468bfbcb98035dc2/regex-2025.7.34-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69c593ff5a24c0d5c1112b0df9b09eae42b33c014bdca7022d6523b210b69f72", size = 797753, upload-time = "2025-07-31T00:20:27.919Z" }, + { url = "https://files.pythonhosted.org/packages/86/38/3e3dc953d13998fa047e9a2414b556201dbd7147034fbac129392363253b/regex-2025.7.34-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98d0ce170fcde1a03b5df19c5650db22ab58af375aaa6ff07978a85c9f250f0e", size = 863263, upload-time = "2025-07-31T00:20:29.803Z" }, + { url = "https://files.pythonhosted.org/packages/68/e5/3ff66b29dde12f5b874dda2d9dec7245c2051f2528d8c2a797901497f140/regex-2025.7.34-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d72765a4bff8c43711d5b0f5b452991a9947853dfa471972169b3cc0ba1d0751", size = 910103, upload-time = "2025-07-31T00:20:31.313Z" }, + { url = "https://files.pythonhosted.org/packages/9e/fe/14176f2182125977fba3711adea73f472a11f3f9288c1317c59cd16ad5e6/regex-2025.7.34-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4494f8fd95a77eb434039ad8460e64d57baa0434f1395b7da44015bef650d0e4", size = 801709, upload-time = "2025-07-31T00:20:33.323Z" }, + { url = "https://files.pythonhosted.org/packages/5a/0d/80d4e66ed24f1ba876a9e8e31b709f9fd22d5c266bf5f3ab3c1afe683d7d/regex-2025.7.34-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4f42b522259c66e918a0121a12429b2abcf696c6f967fa37bdc7b72e61469f98", size = 786726, upload-time = "2025-07-31T00:20:35.252Z" }, + { url = "https://files.pythonhosted.org/packages/12/75/c3ebb30e04a56c046f5c85179dc173818551037daae2c0c940c7b19152cb/regex-2025.7.34-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:aaef1f056d96a0a5d53ad47d019d5b4c66fe4be2da87016e0d43b7242599ffc7", size = 857306, upload-time = "2025-07-31T00:20:37.12Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b2/a4dc5d8b14f90924f27f0ac4c4c4f5e195b723be98adecc884f6716614b6/regex-2025.7.34-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:656433e5b7dccc9bc0da6312da8eb897b81f5e560321ec413500e5367fcd5d47", size = 848494, upload-time = "2025-07-31T00:20:38.818Z" }, + { url = "https://files.pythonhosted.org/packages/0d/21/9ac6e07a4c5e8646a90b56b61f7e9dac11ae0747c857f91d3d2bc7c241d9/regex-2025.7.34-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e91eb2c62c39705e17b4d42d4b86c4e86c884c0d15d9c5a47d0835f8387add8e", size = 787850, upload-time = "2025-07-31T00:20:40.478Z" }, + { url = "https://files.pythonhosted.org/packages/be/6c/d51204e28e7bc54f9a03bb799b04730d7e54ff2718862b8d4e09e7110a6a/regex-2025.7.34-cp314-cp314-win32.whl", hash = "sha256:f978ddfb6216028c8f1d6b0f7ef779949498b64117fc35a939022f67f810bdcb", size = 269730, upload-time = "2025-07-31T00:20:42.253Z" }, + { url = "https://files.pythonhosted.org/packages/74/52/a7e92d02fa1fdef59d113098cb9f02c5d03289a0e9f9e5d4d6acccd10677/regex-2025.7.34-cp314-cp314-win_amd64.whl", hash = "sha256:4b7dc33b9b48fb37ead12ffc7bdb846ac72f99a80373c4da48f64b373a7abeae", size = 278640, upload-time = "2025-07-31T00:20:44.42Z" }, + { url = "https://files.pythonhosted.org/packages/d1/78/a815529b559b1771080faa90c3ab401730661f99d495ab0071649f139ebd/regex-2025.7.34-cp314-cp314-win_arm64.whl", hash = "sha256:4b8c4d39f451e64809912c82392933d80fe2e4a87eeef8859fcc5380d0173c64", size = 271757, upload-time = "2025-07-31T00:20:46.355Z" }, +] + +[[package]] +name = "requests" +version = "2.32.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/e4/56027c4a6b4ae70ca9de302488c5ca95ad4a39e190093d6c1a8ace08341b/requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", size = 64847, upload-time = "2025-06-09T16:43:05.728Z" }, +] + +[[package]] +name = "rpds-py" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/d9/991a0dee12d9fc53ed027e26a26a64b151d77252ac477e22666b9688bc16/rpds_py-0.27.0.tar.gz", hash = "sha256:8b23cf252f180cda89220b378d917180f29d313cd6a07b2431c0d3b776aae86f", size = 27420, upload-time = "2025-08-07T08:26:39.624Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/17/e67309ca1ac993fa1888a0d9b2f5ccc1f67196ace32e76c9f8e1dbbbd50c/rpds_py-0.27.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:19c990fdf5acecbf0623e906ae2e09ce1c58947197f9bced6bbd7482662231c4", size = 362611, upload-time = "2025-08-07T08:23:44.773Z" }, + { url = "https://files.pythonhosted.org/packages/93/2e/28c2fb84aa7aa5d75933d1862d0f7de6198ea22dfd9a0cca06e8a4e7509e/rpds_py-0.27.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6c27a7054b5224710fcfb1a626ec3ff4f28bcb89b899148c72873b18210e446b", size = 347680, upload-time = "2025-08-07T08:23:46.014Z" }, + { url = "https://files.pythonhosted.org/packages/44/3e/9834b4c8f4f5fe936b479e623832468aa4bd6beb8d014fecaee9eac6cdb1/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09965b314091829b378b60607022048953e25f0b396c2b70e7c4c81bcecf932e", size = 384600, upload-time = "2025-08-07T08:23:48Z" }, + { url = "https://files.pythonhosted.org/packages/19/78/744123c7b38865a965cd9e6f691fde7ef989a00a256fa8bf15b75240d12f/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:14f028eb47f59e9169bfdf9f7ceafd29dd64902141840633683d0bad5b04ff34", size = 400697, upload-time = "2025-08-07T08:23:49.407Z" }, + { url = "https://files.pythonhosted.org/packages/32/97/3c3d32fe7daee0a1f1a678b6d4dfb8c4dcf88197fa2441f9da7cb54a8466/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6168af0be75bba990a39f9431cdfae5f0ad501f4af32ae62e8856307200517b8", size = 517781, upload-time = "2025-08-07T08:23:50.557Z" }, + { url = "https://files.pythonhosted.org/packages/b2/be/28f0e3e733680aa13ecec1212fc0f585928a206292f14f89c0b8a684cad1/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab47fe727c13c09d0e6f508e3a49e545008e23bf762a245b020391b621f5b726", size = 406449, upload-time = "2025-08-07T08:23:51.732Z" }, + { url = "https://files.pythonhosted.org/packages/95/ae/5d15c83e337c082d0367053baeb40bfba683f42459f6ebff63a2fd7e5518/rpds_py-0.27.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa01b3d5e3b7d97efab65bd3d88f164e289ec323a8c033c5c38e53ee25c007e", size = 386150, upload-time = "2025-08-07T08:23:52.822Z" }, + { url = "https://files.pythonhosted.org/packages/bf/65/944e95f95d5931112829e040912b25a77b2e7ed913ea5fe5746aa5c1ce75/rpds_py-0.27.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:6c135708e987f46053e0a1246a206f53717f9fadfba27174a9769ad4befba5c3", size = 406100, upload-time = "2025-08-07T08:23:54.339Z" }, + { url = "https://files.pythonhosted.org/packages/21/a4/1664b83fae02894533cd11dc0b9f91d673797c2185b7be0f7496107ed6c5/rpds_py-0.27.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc327f4497b7087d06204235199daf208fd01c82d80465dc5efa4ec9df1c5b4e", size = 421345, upload-time = "2025-08-07T08:23:55.832Z" }, + { url = "https://files.pythonhosted.org/packages/7c/26/b7303941c2b0823bfb34c71378249f8beedce57301f400acb04bb345d025/rpds_py-0.27.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e57906e38583a2cba67046a09c2637e23297618dc1f3caddbc493f2be97c93f", size = 561891, upload-time = "2025-08-07T08:23:56.951Z" }, + { url = "https://files.pythonhosted.org/packages/9b/c8/48623d64d4a5a028fa99576c768a6159db49ab907230edddc0b8468b998b/rpds_py-0.27.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f4f69d7a4300fbf91efb1fb4916421bd57804c01ab938ab50ac9c4aa2212f03", size = 591756, upload-time = "2025-08-07T08:23:58.146Z" }, + { url = "https://files.pythonhosted.org/packages/b3/51/18f62617e8e61cc66334c9fb44b1ad7baae3438662098efbc55fb3fda453/rpds_py-0.27.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b4c4fbbcff474e1e5f38be1bf04511c03d492d42eec0babda5d03af3b5589374", size = 557088, upload-time = "2025-08-07T08:23:59.6Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4c/e84c3a276e2496a93d245516be6b49e20499aa8ca1c94d59fada0d79addc/rpds_py-0.27.0-cp312-cp312-win32.whl", hash = "sha256:27bac29bbbf39601b2aab474daf99dbc8e7176ca3389237a23944b17f8913d97", size = 221926, upload-time = "2025-08-07T08:24:00.695Z" }, + { url = "https://files.pythonhosted.org/packages/83/89/9d0fbcef64340db0605eb0a0044f258076f3ae0a3b108983b2c614d96212/rpds_py-0.27.0-cp312-cp312-win_amd64.whl", hash = "sha256:8a06aa1197ec0281eb1d7daf6073e199eb832fe591ffa329b88bae28f25f5fe5", size = 233235, upload-time = "2025-08-07T08:24:01.846Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b0/e177aa9f39cbab060f96de4a09df77d494f0279604dc2f509263e21b05f9/rpds_py-0.27.0-cp312-cp312-win_arm64.whl", hash = "sha256:e14aab02258cb776a108107bd15f5b5e4a1bbaa61ef33b36693dfab6f89d54f9", size = 223315, upload-time = "2025-08-07T08:24:03.337Z" }, + { url = "https://files.pythonhosted.org/packages/81/d2/dfdfd42565a923b9e5a29f93501664f5b984a802967d48d49200ad71be36/rpds_py-0.27.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:443d239d02d9ae55b74015234f2cd8eb09e59fbba30bf60baeb3123ad4c6d5ff", size = 362133, upload-time = "2025-08-07T08:24:04.508Z" }, + { url = "https://files.pythonhosted.org/packages/ac/4a/0a2e2460c4b66021d349ce9f6331df1d6c75d7eea90df9785d333a49df04/rpds_py-0.27.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b8a7acf04fda1f30f1007f3cc96d29d8cf0a53e626e4e1655fdf4eabc082d367", size = 347128, upload-time = "2025-08-07T08:24:05.695Z" }, + { url = "https://files.pythonhosted.org/packages/35/8d/7d1e4390dfe09d4213b3175a3f5a817514355cb3524593380733204f20b9/rpds_py-0.27.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d0f92b78cfc3b74a42239fdd8c1266f4715b573204c234d2f9fc3fc7a24f185", size = 384027, upload-time = "2025-08-07T08:24:06.841Z" }, + { url = "https://files.pythonhosted.org/packages/c1/65/78499d1a62172891c8cd45de737b2a4b84a414b6ad8315ab3ac4945a5b61/rpds_py-0.27.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ce4ed8e0c7dbc5b19352b9c2c6131dd23b95fa8698b5cdd076307a33626b72dc", size = 399973, upload-time = "2025-08-07T08:24:08.143Z" }, + { url = "https://files.pythonhosted.org/packages/10/a1/1c67c1d8cc889107b19570bb01f75cf49852068e95e6aee80d22915406fc/rpds_py-0.27.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fde355b02934cc6b07200cc3b27ab0c15870a757d1a72fd401aa92e2ea3c6bfe", size = 515295, upload-time = "2025-08-07T08:24:09.711Z" }, + { url = "https://files.pythonhosted.org/packages/df/27/700ec88e748436b6c7c4a2262d66e80f8c21ab585d5e98c45e02f13f21c0/rpds_py-0.27.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13bbc4846ae4c993f07c93feb21a24d8ec637573d567a924b1001e81c8ae80f9", size = 406737, upload-time = "2025-08-07T08:24:11.182Z" }, + { url = "https://files.pythonhosted.org/packages/33/cc/6b0ee8f0ba3f2df2daac1beda17fde5cf10897a7d466f252bd184ef20162/rpds_py-0.27.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be0744661afbc4099fef7f4e604e7f1ea1be1dd7284f357924af12a705cc7d5c", size = 385898, upload-time = "2025-08-07T08:24:12.798Z" }, + { url = "https://files.pythonhosted.org/packages/e8/7e/c927b37d7d33c0a0ebf249cc268dc2fcec52864c1b6309ecb960497f2285/rpds_py-0.27.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:069e0384a54f427bd65d7fda83b68a90606a3835901aaff42185fcd94f5a9295", size = 405785, upload-time = "2025-08-07T08:24:14.906Z" }, + { url = "https://files.pythonhosted.org/packages/5b/d2/8ed50746d909dcf402af3fa58b83d5a590ed43e07251d6b08fad1a535ba6/rpds_py-0.27.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4bc262ace5a1a7dc3e2eac2fa97b8257ae795389f688b5adf22c5db1e2431c43", size = 419760, upload-time = "2025-08-07T08:24:16.129Z" }, + { url = "https://files.pythonhosted.org/packages/d3/60/2b2071aee781cb3bd49f94d5d35686990b925e9b9f3e3d149235a6f5d5c1/rpds_py-0.27.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2fe6e18e5c8581f0361b35ae575043c7029d0a92cb3429e6e596c2cdde251432", size = 561201, upload-time = "2025-08-07T08:24:17.645Z" }, + { url = "https://files.pythonhosted.org/packages/98/1f/27b67304272521aaea02be293fecedce13fa351a4e41cdb9290576fc6d81/rpds_py-0.27.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d93ebdb82363d2e7bec64eecdc3632b59e84bd270d74fe5be1659f7787052f9b", size = 591021, upload-time = "2025-08-07T08:24:18.999Z" }, + { url = "https://files.pythonhosted.org/packages/db/9b/a2fadf823164dd085b1f894be6443b0762a54a7af6f36e98e8fcda69ee50/rpds_py-0.27.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0954e3a92e1d62e83a54ea7b3fdc9efa5d61acef8488a8a3d31fdafbfb00460d", size = 556368, upload-time = "2025-08-07T08:24:20.54Z" }, + { url = "https://files.pythonhosted.org/packages/24/f3/6d135d46a129cda2e3e6d4c5e91e2cc26ea0428c6cf152763f3f10b6dd05/rpds_py-0.27.0-cp313-cp313-win32.whl", hash = "sha256:2cff9bdd6c7b906cc562a505c04a57d92e82d37200027e8d362518df427f96cd", size = 221236, upload-time = "2025-08-07T08:24:22.144Z" }, + { url = "https://files.pythonhosted.org/packages/c5/44/65d7494f5448ecc755b545d78b188440f81da98b50ea0447ab5ebfdf9bd6/rpds_py-0.27.0-cp313-cp313-win_amd64.whl", hash = "sha256:dc79d192fb76fc0c84f2c58672c17bbbc383fd26c3cdc29daae16ce3d927e8b2", size = 232634, upload-time = "2025-08-07T08:24:23.642Z" }, + { url = "https://files.pythonhosted.org/packages/70/d9/23852410fadab2abb611733933401de42a1964ce6600a3badae35fbd573e/rpds_py-0.27.0-cp313-cp313-win_arm64.whl", hash = "sha256:5b3a5c8089eed498a3af23ce87a80805ff98f6ef8f7bdb70bd1b7dae5105f6ac", size = 222783, upload-time = "2025-08-07T08:24:25.098Z" }, + { url = "https://files.pythonhosted.org/packages/15/75/03447917f78512b34463f4ef11066516067099a0c466545655503bed0c77/rpds_py-0.27.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:90fb790138c1a89a2e58c9282fe1089638401f2f3b8dddd758499041bc6e0774", size = 359154, upload-time = "2025-08-07T08:24:26.249Z" }, + { url = "https://files.pythonhosted.org/packages/6b/fc/4dac4fa756451f2122ddaf136e2c6aeb758dc6fdbe9ccc4bc95c98451d50/rpds_py-0.27.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:010c4843a3b92b54373e3d2291a7447d6c3fc29f591772cc2ea0e9f5c1da434b", size = 343909, upload-time = "2025-08-07T08:24:27.405Z" }, + { url = "https://files.pythonhosted.org/packages/7b/81/723c1ed8e6f57ed9d8c0c07578747a2d3d554aaefc1ab89f4e42cfeefa07/rpds_py-0.27.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9ce7a9e967afc0a2af7caa0d15a3e9c1054815f73d6a8cb9225b61921b419bd", size = 379340, upload-time = "2025-08-07T08:24:28.714Z" }, + { url = "https://files.pythonhosted.org/packages/98/16/7e3740413de71818ce1997df82ba5f94bae9fff90c0a578c0e24658e6201/rpds_py-0.27.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aa0bf113d15e8abdfee92aa4db86761b709a09954083afcb5bf0f952d6065fdb", size = 391655, upload-time = "2025-08-07T08:24:30.223Z" }, + { url = "https://files.pythonhosted.org/packages/e0/63/2a9f510e124d80660f60ecce07953f3f2d5f0b96192c1365443859b9c87f/rpds_py-0.27.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb91d252b35004a84670dfeafadb042528b19842a0080d8b53e5ec1128e8f433", size = 513017, upload-time = "2025-08-07T08:24:31.446Z" }, + { url = "https://files.pythonhosted.org/packages/2c/4e/cf6ff311d09776c53ea1b4f2e6700b9d43bb4e99551006817ade4bbd6f78/rpds_py-0.27.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db8a6313dbac934193fc17fe7610f70cd8181c542a91382531bef5ed785e5615", size = 402058, upload-time = "2025-08-07T08:24:32.613Z" }, + { url = "https://files.pythonhosted.org/packages/88/11/5e36096d474cb10f2a2d68b22af60a3bc4164fd8db15078769a568d9d3ac/rpds_py-0.27.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce96ab0bdfcef1b8c371ada2100767ace6804ea35aacce0aef3aeb4f3f499ca8", size = 383474, upload-time = "2025-08-07T08:24:33.767Z" }, + { url = "https://files.pythonhosted.org/packages/db/a2/3dff02805b06058760b5eaa6d8cb8db3eb3e46c9e452453ad5fc5b5ad9fe/rpds_py-0.27.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:7451ede3560086abe1aa27dcdcf55cd15c96b56f543fb12e5826eee6f721f858", size = 400067, upload-time = "2025-08-07T08:24:35.021Z" }, + { url = "https://files.pythonhosted.org/packages/67/87/eed7369b0b265518e21ea836456a4ed4a6744c8c12422ce05bce760bb3cf/rpds_py-0.27.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:32196b5a99821476537b3f7732432d64d93a58d680a52c5e12a190ee0135d8b5", size = 412085, upload-time = "2025-08-07T08:24:36.267Z" }, + { url = "https://files.pythonhosted.org/packages/8b/48/f50b2ab2fbb422fbb389fe296e70b7a6b5ea31b263ada5c61377e710a924/rpds_py-0.27.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a029be818059870664157194e46ce0e995082ac49926f1423c1f058534d2aaa9", size = 555928, upload-time = "2025-08-07T08:24:37.573Z" }, + { url = "https://files.pythonhosted.org/packages/98/41/b18eb51045d06887666c3560cd4bbb6819127b43d758f5adb82b5f56f7d1/rpds_py-0.27.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3841f66c1ffdc6cebce8aed64e36db71466f1dc23c0d9a5592e2a782a3042c79", size = 585527, upload-time = "2025-08-07T08:24:39.391Z" }, + { url = "https://files.pythonhosted.org/packages/be/03/a3dd6470fc76499959b00ae56295b76b4bdf7c6ffc60d62006b1217567e1/rpds_py-0.27.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:42894616da0fc0dcb2ec08a77896c3f56e9cb2f4b66acd76fc8992c3557ceb1c", size = 554211, upload-time = "2025-08-07T08:24:40.6Z" }, + { url = "https://files.pythonhosted.org/packages/bf/d1/ee5fd1be395a07423ac4ca0bcc05280bf95db2b155d03adefeb47d5ebf7e/rpds_py-0.27.0-cp313-cp313t-win32.whl", hash = "sha256:b1fef1f13c842a39a03409e30ca0bf87b39a1e2a305a9924deadb75a43105d23", size = 216624, upload-time = "2025-08-07T08:24:42.204Z" }, + { url = "https://files.pythonhosted.org/packages/1c/94/4814c4c858833bf46706f87349c37ca45e154da7dbbec9ff09f1abeb08cc/rpds_py-0.27.0-cp313-cp313t-win_amd64.whl", hash = "sha256:183f5e221ba3e283cd36fdfbe311d95cd87699a083330b4f792543987167eff1", size = 230007, upload-time = "2025-08-07T08:24:43.329Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a5/8fffe1c7dc7c055aa02df310f9fb71cfc693a4d5ccc5de2d3456ea5fb022/rpds_py-0.27.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:f3cd110e02c5bf17d8fb562f6c9df5c20e73029d587cf8602a2da6c5ef1e32cb", size = 362595, upload-time = "2025-08-07T08:24:44.478Z" }, + { url = "https://files.pythonhosted.org/packages/bc/c7/4e4253fd2d4bb0edbc0b0b10d9f280612ca4f0f990e3c04c599000fe7d71/rpds_py-0.27.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8d0e09cf4863c74106b5265c2c310f36146e2b445ff7b3018a56799f28f39f6f", size = 347252, upload-time = "2025-08-07T08:24:45.678Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c8/3d1a954d30f0174dd6baf18b57c215da03cf7846a9d6e0143304e784cddc/rpds_py-0.27.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64f689ab822f9b5eb6dfc69893b4b9366db1d2420f7db1f6a2adf2a9ca15ad64", size = 384886, upload-time = "2025-08-07T08:24:46.86Z" }, + { url = "https://files.pythonhosted.org/packages/e0/52/3c5835f2df389832b28f9276dd5395b5a965cea34226e7c88c8fbec2093c/rpds_py-0.27.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e36c80c49853b3ffda7aa1831bf175c13356b210c73128c861f3aa93c3cc4015", size = 399716, upload-time = "2025-08-07T08:24:48.174Z" }, + { url = "https://files.pythonhosted.org/packages/40/73/176e46992461a1749686a2a441e24df51ff86b99c2d34bf39f2a5273b987/rpds_py-0.27.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6de6a7f622860af0146cb9ee148682ff4d0cea0b8fd3ad51ce4d40efb2f061d0", size = 517030, upload-time = "2025-08-07T08:24:49.52Z" }, + { url = "https://files.pythonhosted.org/packages/79/2a/7266c75840e8c6e70effeb0d38922a45720904f2cd695e68a0150e5407e2/rpds_py-0.27.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4045e2fc4b37ec4b48e8907a5819bdd3380708c139d7cc358f03a3653abedb89", size = 408448, upload-time = "2025-08-07T08:24:50.727Z" }, + { url = "https://files.pythonhosted.org/packages/e6/5f/a7efc572b8e235093dc6cf39f4dbc8a7f08e65fdbcec7ff4daeb3585eef1/rpds_py-0.27.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9da162b718b12c4219eeeeb68a5b7552fbc7aadedf2efee440f88b9c0e54b45d", size = 387320, upload-time = "2025-08-07T08:24:52.004Z" }, + { url = "https://files.pythonhosted.org/packages/a2/eb/9ff6bc92efe57cf5a2cb74dee20453ba444b6fdc85275d8c99e0d27239d1/rpds_py-0.27.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:0665be515767dc727ffa5f74bd2ef60b0ff85dad6bb8f50d91eaa6b5fb226f51", size = 407414, upload-time = "2025-08-07T08:24:53.664Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bd/3b9b19b00d5c6e1bd0f418c229ab0f8d3b110ddf7ec5d9d689ef783d0268/rpds_py-0.27.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:203f581accef67300a942e49a37d74c12ceeef4514874c7cede21b012613ca2c", size = 420766, upload-time = "2025-08-07T08:24:55.917Z" }, + { url = "https://files.pythonhosted.org/packages/17/6b/521a7b1079ce16258c70805166e3ac6ec4ee2139d023fe07954dc9b2d568/rpds_py-0.27.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7873b65686a6471c0037139aa000d23fe94628e0daaa27b6e40607c90e3f5ec4", size = 562409, upload-time = "2025-08-07T08:24:57.17Z" }, + { url = "https://files.pythonhosted.org/packages/8b/bf/65db5bfb14ccc55e39de8419a659d05a2a9cd232f0a699a516bb0991da7b/rpds_py-0.27.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:249ab91ceaa6b41abc5f19513cb95b45c6f956f6b89f1fe3d99c81255a849f9e", size = 590793, upload-time = "2025-08-07T08:24:58.388Z" }, + { url = "https://files.pythonhosted.org/packages/db/b8/82d368b378325191ba7aae8f40f009b78057b598d4394d1f2cdabaf67b3f/rpds_py-0.27.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d2f184336bc1d6abfaaa1262ed42739c3789b1e3a65a29916a615307d22ffd2e", size = 558178, upload-time = "2025-08-07T08:24:59.756Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ff/f270bddbfbc3812500f8131b1ebbd97afd014cd554b604a3f73f03133a36/rpds_py-0.27.0-cp314-cp314-win32.whl", hash = "sha256:d3c622c39f04d5751408f5b801ecb527e6e0a471b367f420a877f7a660d583f6", size = 222355, upload-time = "2025-08-07T08:25:01.027Z" }, + { url = "https://files.pythonhosted.org/packages/bf/20/fdab055b1460c02ed356a0e0b0a78c1dd32dc64e82a544f7b31c9ac643dc/rpds_py-0.27.0-cp314-cp314-win_amd64.whl", hash = "sha256:cf824aceaeffff029ccfba0da637d432ca71ab21f13e7f6f5179cd88ebc77a8a", size = 234007, upload-time = "2025-08-07T08:25:02.268Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a8/694c060005421797a3be4943dab8347c76c2b429a9bef68fb2c87c9e70c7/rpds_py-0.27.0-cp314-cp314-win_arm64.whl", hash = "sha256:86aca1616922b40d8ac1b3073a1ead4255a2f13405e5700c01f7c8d29a03972d", size = 223527, upload-time = "2025-08-07T08:25:03.45Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f9/77f4c90f79d2c5ca8ce6ec6a76cb4734ee247de6b3a4f337e289e1f00372/rpds_py-0.27.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:341d8acb6724c0c17bdf714319c393bb27f6d23d39bc74f94221b3e59fc31828", size = 359469, upload-time = "2025-08-07T08:25:04.648Z" }, + { url = "https://files.pythonhosted.org/packages/c0/22/b97878d2f1284286fef4172069e84b0b42b546ea7d053e5fb7adb9ac6494/rpds_py-0.27.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6b96b0b784fe5fd03beffff2b1533dc0d85e92bab8d1b2c24ef3a5dc8fac5669", size = 343960, upload-time = "2025-08-07T08:25:05.863Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b0/dfd55b5bb480eda0578ae94ef256d3061d20b19a0f5e18c482f03e65464f/rpds_py-0.27.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c431bfb91478d7cbe368d0a699978050d3b112d7f1d440a41e90faa325557fd", size = 380201, upload-time = "2025-08-07T08:25:07.513Z" }, + { url = "https://files.pythonhosted.org/packages/28/22/e1fa64e50d58ad2b2053077e3ec81a979147c43428de9e6de68ddf6aff4e/rpds_py-0.27.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e222a44ae9f507d0f2678ee3dd0c45ec1e930f6875d99b8459631c24058aec", size = 392111, upload-time = "2025-08-07T08:25:09.149Z" }, + { url = "https://files.pythonhosted.org/packages/49/f9/43ab7a43e97aedf6cea6af70fdcbe18abbbc41d4ae6cdec1bfc23bbad403/rpds_py-0.27.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:184f0d7b342967f6cda94a07d0e1fae177d11d0b8f17d73e06e36ac02889f303", size = 515863, upload-time = "2025-08-07T08:25:10.431Z" }, + { url = "https://files.pythonhosted.org/packages/38/9b/9bd59dcc636cd04d86a2d20ad967770bf348f5eb5922a8f29b547c074243/rpds_py-0.27.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a00c91104c173c9043bc46f7b30ee5e6d2f6b1149f11f545580f5d6fdff42c0b", size = 402398, upload-time = "2025-08-07T08:25:11.819Z" }, + { url = "https://files.pythonhosted.org/packages/71/bf/f099328c6c85667aba6b66fa5c35a8882db06dcd462ea214be72813a0dd2/rpds_py-0.27.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7a37dd208f0d658e0487522078b1ed68cd6bce20ef4b5a915d2809b9094b410", size = 384665, upload-time = "2025-08-07T08:25:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c5/9c1f03121ece6634818490bd3c8be2c82a70928a19de03467fb25a3ae2a8/rpds_py-0.27.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:92f3b3ec3e6008a1fe00b7c0946a170f161ac00645cde35e3c9a68c2475e8156", size = 400405, upload-time = "2025-08-07T08:25:14.417Z" }, + { url = "https://files.pythonhosted.org/packages/b5/b8/e25d54af3e63ac94f0c16d8fe143779fe71ff209445a0c00d0f6984b6b2c/rpds_py-0.27.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a1b3db5fae5cbce2131b7420a3f83553d4d89514c03d67804ced36161fe8b6b2", size = 413179, upload-time = "2025-08-07T08:25:15.664Z" }, + { url = "https://files.pythonhosted.org/packages/f9/d1/406b3316433fe49c3021546293a04bc33f1478e3ec7950215a7fce1a1208/rpds_py-0.27.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5355527adaa713ab693cbce7c1e0ec71682f599f61b128cf19d07e5c13c9b1f1", size = 556895, upload-time = "2025-08-07T08:25:17.061Z" }, + { url = "https://files.pythonhosted.org/packages/5f/bc/3697c0c21fcb9a54d46ae3b735eb2365eea0c2be076b8f770f98e07998de/rpds_py-0.27.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:fcc01c57ce6e70b728af02b2401c5bc853a9e14eb07deda30624374f0aebfe42", size = 585464, upload-time = "2025-08-07T08:25:18.406Z" }, + { url = "https://files.pythonhosted.org/packages/63/09/ee1bb5536f99f42c839b177d552f6114aa3142d82f49cef49261ed28dbe0/rpds_py-0.27.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3001013dae10f806380ba739d40dee11db1ecb91684febb8406a87c2ded23dae", size = 555090, upload-time = "2025-08-07T08:25:20.461Z" }, + { url = "https://files.pythonhosted.org/packages/7d/2c/363eada9e89f7059199d3724135a86c47082cbf72790d6ba2f336d146ddb/rpds_py-0.27.0-cp314-cp314t-win32.whl", hash = "sha256:0f401c369186a5743694dd9fc08cba66cf70908757552e1f714bfc5219c655b5", size = 218001, upload-time = "2025-08-07T08:25:21.761Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3f/d6c216ed5199c9ef79e2a33955601f454ed1e7420a93b89670133bca5ace/rpds_py-0.27.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8a1dca5507fa1337f75dcd5070218b20bc68cf8844271c923c1b79dfcbc20391", size = 230993, upload-time = "2025-08-07T08:25:23.34Z" }, +] + +[[package]] +name = "rsa" +version = "4.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/05/d52bf1e65044b4e5e27d4e63e8d1579dbdec54fce685908ae09bc3720030/s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf", size = 150589, upload-time = "2025-07-18T19:22:42.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/4f/d073e09df851cfa251ef7840007d04db3293a0482ce607d2b993926089be/s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724", size = 85308, upload-time = "2025-07-18T19:22:40.947Z" }, +] + +[[package]] +name = "safetensors" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/cc/738f3011628920e027a11754d9cae9abec1aed00f7ae860abbf843755233/safetensors-0.6.2.tar.gz", hash = "sha256:43ff2aa0e6fa2dc3ea5524ac7ad93a9839256b8703761e76e2d0b2a3fa4f15d9", size = 197968, upload-time = "2025-08-08T13:13:58.654Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/b1/3f5fd73c039fc87dba3ff8b5d528bfc5a32b597fea8e7a6a4800343a17c7/safetensors-0.6.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9c85ede8ec58f120bad982ec47746981e210492a6db876882aa021446af8ffba", size = 454797, upload-time = "2025-08-08T13:13:52.066Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c9/bb114c158540ee17907ec470d01980957fdaf87b4aa07914c24eba87b9c6/safetensors-0.6.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d6675cf4b39c98dbd7d940598028f3742e0375a6b4d4277e76beb0c35f4b843b", size = 432206, upload-time = "2025-08-08T13:13:50.931Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8e/f70c34e47df3110e8e0bb268d90db8d4be8958a54ab0336c9be4fe86dac8/safetensors-0.6.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d2d2b3ce1e2509c68932ca03ab8f20570920cd9754b05063d4368ee52833ecd", size = 473261, upload-time = "2025-08-08T13:13:41.259Z" }, + { url = "https://files.pythonhosted.org/packages/2a/f5/be9c6a7c7ef773e1996dc214e73485286df1836dbd063e8085ee1976f9cb/safetensors-0.6.2-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:93de35a18f46b0f5a6a1f9e26d91b442094f2df02e9fd7acf224cfec4238821a", size = 485117, upload-time = "2025-08-08T13:13:43.506Z" }, + { url = "https://files.pythonhosted.org/packages/c9/55/23f2d0a2c96ed8665bf17a30ab4ce5270413f4d74b6d87dd663258b9af31/safetensors-0.6.2-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89a89b505f335640f9120fac65ddeb83e40f1fd081cb8ed88b505bdccec8d0a1", size = 616154, upload-time = "2025-08-08T13:13:45.096Z" }, + { url = "https://files.pythonhosted.org/packages/98/c6/affb0bd9ce02aa46e7acddbe087912a04d953d7a4d74b708c91b5806ef3f/safetensors-0.6.2-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc4d0d0b937e04bdf2ae6f70cd3ad51328635fe0e6214aa1fc811f3b576b3bda", size = 520713, upload-time = "2025-08-08T13:13:46.25Z" }, + { url = "https://files.pythonhosted.org/packages/fe/5d/5a514d7b88e310c8b146e2404e0dc161282e78634d9358975fd56dfd14be/safetensors-0.6.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8045db2c872db8f4cbe3faa0495932d89c38c899c603f21e9b6486951a5ecb8f", size = 485835, upload-time = "2025-08-08T13:13:49.373Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7b/4fc3b2ba62c352b2071bea9cfbad330fadda70579f617506ae1a2f129cab/safetensors-0.6.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:81e67e8bab9878bb568cffbc5f5e655adb38d2418351dc0859ccac158f753e19", size = 521503, upload-time = "2025-08-08T13:13:47.651Z" }, + { url = "https://files.pythonhosted.org/packages/5a/50/0057e11fe1f3cead9254315a6c106a16dd4b1a19cd247f7cc6414f6b7866/safetensors-0.6.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b0e4d029ab0a0e0e4fdf142b194514695b1d7d3735503ba700cf36d0fc7136ce", size = 652256, upload-time = "2025-08-08T13:13:53.167Z" }, + { url = "https://files.pythonhosted.org/packages/e9/29/473f789e4ac242593ac1656fbece6e1ecd860bb289e635e963667807afe3/safetensors-0.6.2-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:fa48268185c52bfe8771e46325a1e21d317207bcabcb72e65c6e28e9ffeb29c7", size = 747281, upload-time = "2025-08-08T13:13:54.656Z" }, + { url = "https://files.pythonhosted.org/packages/68/52/f7324aad7f2df99e05525c84d352dc217e0fa637a4f603e9f2eedfbe2c67/safetensors-0.6.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:d83c20c12c2d2f465997c51b7ecb00e407e5f94d7dec3ea0cc11d86f60d3fde5", size = 692286, upload-time = "2025-08-08T13:13:55.884Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fe/cad1d9762868c7c5dc70c8620074df28ebb1a8e4c17d4c0cb031889c457e/safetensors-0.6.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d944cea65fad0ead848b6ec2c37cc0b197194bec228f8020054742190e9312ac", size = 655957, upload-time = "2025-08-08T13:13:57.029Z" }, + { url = "https://files.pythonhosted.org/packages/59/a7/e2158e17bbe57d104f0abbd95dff60dda916cf277c9f9663b4bf9bad8b6e/safetensors-0.6.2-cp38-abi3-win32.whl", hash = "sha256:cab75ca7c064d3911411461151cb69380c9225798a20e712b102edda2542ddb1", size = 308926, upload-time = "2025-08-08T13:14:01.095Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c3/c0be1135726618dc1e28d181b8c442403d8dbb9e273fd791de2d4384bcdd/safetensors-0.6.2-cp38-abi3-win_amd64.whl", hash = "sha256:c7b214870df923cbc1593c3faee16bec59ea462758699bd3fee399d00aac072c", size = 320192, upload-time = "2025-08-08T13:13:59.467Z" }, +] + +[[package]] +name = "scikit-learn" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/19/5aa2002044afc297ecaf1e3517ed07bba4aece3b5613b5160c1212995fc8/scikit_learn-1.6.0.tar.gz", hash = "sha256:9d58481f9f7499dff4196927aedd4285a0baec8caa3790efbe205f13de37dd6e", size = 7074944, upload-time = "2024-12-09T16:02:23.639Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/0c/a5de627aa57b028aea7026cb3bbeaf63be3158adc118212d6cc7843d939a/scikit_learn-1.6.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:04a5ba45c12a5ff81518aa4f1604e826a45d20e53da47b15871526cda4ff5174", size = 12096999, upload-time = "2024-12-09T16:01:31.659Z" }, + { url = "https://files.pythonhosted.org/packages/a3/7d/02a96e6fb28ddb213e84b1b4a44148d26ec96fc9db9c74e050277e009892/scikit_learn-1.6.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:21fadfc2ad7a1ce8bd1d90f23d17875b84ec765eecbbfc924ff11fb73db582ce", size = 11160579, upload-time = "2024-12-09T16:01:34.693Z" }, + { url = "https://files.pythonhosted.org/packages/70/28/77b071f541d75247e6c3403f19aaa634371e972691f6aa1838ca9fd4cc52/scikit_learn-1.6.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30f34bb5fde90e020653bb84dcb38b6c83f90c70680dbd8c38bd9becbad7a127", size = 12246543, upload-time = "2024-12-09T16:01:37.241Z" }, + { url = "https://files.pythonhosted.org/packages/17/0e/e6bb84074f1081245a165c0ee775ecef24beae9d2f2e24bcac0c9f155f13/scikit_learn-1.6.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1dad624cffe3062276a0881d4e441bc9e3b19d02d17757cd6ae79a9d192a0027", size = 13140402, upload-time = "2024-12-09T16:01:40.15Z" }, + { url = "https://files.pythonhosted.org/packages/21/1d/3df58df8bd425f425df9f90b316618ace62b7f1f838ac1580191025cc735/scikit_learn-1.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:2fce7950a3fad85e0a61dc403df0f9345b53432ac0e47c50da210d22c60b6d85", size = 11103596, upload-time = "2024-12-09T16:01:43.205Z" }, + { url = "https://files.pythonhosted.org/packages/2e/f4/c3b51920cf310169d19d07855a7bdf51a9b065314877d9a58c0c60d08eea/scikit_learn-1.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e5453b2e87ef8accedc5a8a4e6709f887ca01896cd7cc8a174fe39bd4bb00aef", size = 12002532, upload-time = "2024-12-09T16:01:46.199Z" }, + { url = "https://files.pythonhosted.org/packages/e4/76/cfb0778a84c30df272f1c41fc7b3bd3ffac6e8b02ee6a078a592d35cf73f/scikit_learn-1.6.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5fe11794236fb83bead2af26a87ced5d26e3370b8487430818b915dafab1724e", size = 11088997, upload-time = "2024-12-09T16:01:48.57Z" }, + { url = "https://files.pythonhosted.org/packages/2b/8d/4563419d742b852e50871fa3494a8dd0304610601359209a2e614e200260/scikit_learn-1.6.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61fe3dcec0d82ae280877a818ab652f4988371e32dd5451e75251bece79668b1", size = 12203192, upload-time = "2024-12-09T16:01:52.024Z" }, + { url = "https://files.pythonhosted.org/packages/15/a4/f4fdcdd11d82837804c888097ad02aa6381c4bbd57b9d3074ecf9eba8f42/scikit_learn-1.6.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b44e3a51e181933bdf9a4953cc69c6025b40d2b49e238233f149b98849beb4bf", size = 13164436, upload-time = "2024-12-09T16:01:54.447Z" }, + { url = "https://files.pythonhosted.org/packages/1a/e1/32bdcf8f918de5a156da6886aba24a3b5718d267954bd34555be896289f0/scikit_learn-1.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:a17860a562bac54384454d40b3f6155200c1c737c9399e6a97962c63fce503ac", size = 11064779, upload-time = "2024-12-09T16:01:56.756Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/14464bea220bc02879f9e8d905c4b0a44b5c12afde6c375720b6f41d9407/scikit_learn-1.6.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:98717d3c152f6842d36a70f21e1468fb2f1a2f8f2624d9a3f382211798516426", size = 11962472, upload-time = "2024-12-09T16:01:59.129Z" }, + { url = "https://files.pythonhosted.org/packages/b4/69/66899cdc65986188e0e255e52ee93dee5101a72f139ee05f263dfff2053a/scikit_learn-1.6.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:34e20bfac8ff0ebe0ff20fb16a4d6df5dc4cc9ce383e00c2ab67a526a3c67b18", size = 11104864, upload-time = "2024-12-09T16:02:01.457Z" }, + { url = "https://files.pythonhosted.org/packages/3c/32/2c63bc108cc5438b116a0c6fd25c6126dd14c03118724385f10a3d218ee8/scikit_learn-1.6.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eba06d75815406091419e06dd650b91ebd1c5f836392a0d833ff36447c2b1bfa", size = 12435734, upload-time = "2024-12-09T16:02:04.317Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f5/9434dff19e04a334bfb30df90511904263c48a422a9952d91d8de5c3aa62/scikit_learn-1.6.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b6916d1cec1ff163c7d281e699d7a6a709da2f2c5ec7b10547e08cc788ddd3ae", size = 11329803, upload-time = "2024-12-09T16:02:07.43Z" }, +] + +[[package]] +name = "scipy" +version = "1.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/4a/b927028464795439faec8eaf0b03b011005c487bb2d07409f28bf30879c4/scipy-1.16.1.tar.gz", hash = "sha256:44c76f9e8b6e8e488a586190ab38016e4ed2f8a038af7cd3defa903c0a2238b3", size = 30580861, upload-time = "2025-07-27T16:33:30.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/d9/ec4864f5896232133f51382b54a08de91a9d1af7a76dfa372894026dfee2/scipy-1.16.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:81b433bbeaf35728dad619afc002db9b189e45eebe2cd676effe1fb93fef2b9c", size = 36575194, upload-time = "2025-07-27T16:27:41.321Z" }, + { url = "https://files.pythonhosted.org/packages/5c/6d/40e81ecfb688e9d25d34a847dca361982a6addf8e31f0957b1a54fbfa994/scipy-1.16.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:886cc81fdb4c6903a3bb0464047c25a6d1016fef77bb97949817d0c0d79f9e04", size = 28594590, upload-time = "2025-07-27T16:27:49.204Z" }, + { url = "https://files.pythonhosted.org/packages/0e/37/9f65178edfcc629377ce9a64fc09baebea18c80a9e57ae09a52edf84880b/scipy-1.16.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:15240c3aac087a522b4eaedb09f0ad061753c5eebf1ea430859e5bf8640d5919", size = 20866458, upload-time = "2025-07-27T16:27:54.98Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7b/749a66766871ea4cb1d1ea10f27004db63023074c22abed51f22f09770e0/scipy-1.16.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:65f81a25805f3659b48126b5053d9e823d3215e4a63730b5e1671852a1705921", size = 23539318, upload-time = "2025-07-27T16:28:01.604Z" }, + { url = "https://files.pythonhosted.org/packages/c4/db/8d4afec60eb833a666434d4541a3151eedbf2494ea6d4d468cbe877f00cd/scipy-1.16.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c62eea7f607f122069b9bad3f99489ddca1a5173bef8a0c75555d7488b6f725", size = 33292899, upload-time = "2025-07-27T16:28:09.147Z" }, + { url = "https://files.pythonhosted.org/packages/51/1e/79023ca3bbb13a015d7d2757ecca3b81293c663694c35d6541b4dca53e98/scipy-1.16.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f965bbf3235b01c776115ab18f092a95aa74c271a52577bcb0563e85738fd618", size = 35162637, upload-time = "2025-07-27T16:28:17.535Z" }, + { url = "https://files.pythonhosted.org/packages/b6/49/0648665f9c29fdaca4c679182eb972935b3b4f5ace41d323c32352f29816/scipy-1.16.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f006e323874ffd0b0b816d8c6a8e7f9a73d55ab3b8c3f72b752b226d0e3ac83d", size = 35490507, upload-time = "2025-07-27T16:28:25.705Z" }, + { url = "https://files.pythonhosted.org/packages/62/8f/66cbb9d6bbb18d8c658f774904f42a92078707a7c71e5347e8bf2f52bb89/scipy-1.16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e8fd15fc5085ab4cca74cb91fe0a4263b1f32e4420761ddae531ad60934c2119", size = 37923998, upload-time = "2025-07-27T16:28:34.339Z" }, + { url = "https://files.pythonhosted.org/packages/14/c3/61f273ae550fbf1667675701112e380881905e28448c080b23b5a181df7c/scipy-1.16.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7b8013c6c066609577d910d1a2a077021727af07b6fab0ee22c2f901f22352a", size = 38508060, upload-time = "2025-07-27T16:28:43.242Z" }, + { url = "https://files.pythonhosted.org/packages/93/0b/b5c99382b839854a71ca9482c684e3472badc62620287cbbdab499b75ce6/scipy-1.16.1-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:5451606823a5e73dfa621a89948096c6528e2896e40b39248295d3a0138d594f", size = 36533717, upload-time = "2025-07-27T16:28:51.706Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e5/69ab2771062c91e23e07c12e7d5033a6b9b80b0903ee709c3c36b3eb520c/scipy-1.16.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:89728678c5ca5abd610aee148c199ac1afb16e19844401ca97d43dc548a354eb", size = 28570009, upload-time = "2025-07-27T16:28:57.017Z" }, + { url = "https://files.pythonhosted.org/packages/f4/69/bd75dbfdd3cf524f4d753484d723594aed62cfaac510123e91a6686d520b/scipy-1.16.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e756d688cb03fd07de0fffad475649b03cb89bee696c98ce508b17c11a03f95c", size = 20841942, upload-time = "2025-07-27T16:29:01.152Z" }, + { url = "https://files.pythonhosted.org/packages/ea/74/add181c87663f178ba7d6144b370243a87af8476664d5435e57d599e6874/scipy-1.16.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:5aa2687b9935da3ed89c5dbed5234576589dd28d0bf7cd237501ccfbdf1ad608", size = 23498507, upload-time = "2025-07-27T16:29:05.202Z" }, + { url = "https://files.pythonhosted.org/packages/1d/74/ece2e582a0d9550cee33e2e416cc96737dce423a994d12bbe59716f47ff1/scipy-1.16.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0851f6a1e537fe9399f35986897e395a1aa61c574b178c0d456be5b1a0f5ca1f", size = 33286040, upload-time = "2025-07-27T16:29:10.201Z" }, + { url = "https://files.pythonhosted.org/packages/e4/82/08e4076df538fb56caa1d489588d880ec7c52d8273a606bb54d660528f7c/scipy-1.16.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fedc2cbd1baed37474b1924c331b97bdff611d762c196fac1a9b71e67b813b1b", size = 35176096, upload-time = "2025-07-27T16:29:17.091Z" }, + { url = "https://files.pythonhosted.org/packages/fa/79/cd710aab8c921375711a8321c6be696e705a120e3011a643efbbcdeeabcc/scipy-1.16.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2ef500e72f9623a6735769e4b93e9dcb158d40752cdbb077f305487e3e2d1f45", size = 35490328, upload-time = "2025-07-27T16:29:22.928Z" }, + { url = "https://files.pythonhosted.org/packages/71/73/e9cc3d35ee4526d784520d4494a3e1ca969b071fb5ae5910c036a375ceec/scipy-1.16.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:978d8311674b05a8f7ff2ea6c6bce5d8b45a0cb09d4c5793e0318f448613ea65", size = 37939921, upload-time = "2025-07-27T16:29:29.108Z" }, + { url = "https://files.pythonhosted.org/packages/21/12/c0efd2941f01940119b5305c375ae5c0fcb7ec193f806bd8f158b73a1782/scipy-1.16.1-cp313-cp313-win_amd64.whl", hash = "sha256:81929ed0fa7a5713fcdd8b2e6f73697d3b4c4816d090dd34ff937c20fa90e8ab", size = 38479462, upload-time = "2025-07-27T16:30:24.078Z" }, + { url = "https://files.pythonhosted.org/packages/7a/19/c3d08b675260046a991040e1ea5d65f91f40c7df1045fffff412dcfc6765/scipy-1.16.1-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:bcc12db731858abda693cecdb3bdc9e6d4bd200213f49d224fe22df82687bdd6", size = 36938832, upload-time = "2025-07-27T16:29:35.057Z" }, + { url = "https://files.pythonhosted.org/packages/81/f2/ce53db652c033a414a5b34598dba6b95f3d38153a2417c5a3883da429029/scipy-1.16.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:744d977daa4becb9fc59135e75c069f8d301a87d64f88f1e602a9ecf51e77b27", size = 29093084, upload-time = "2025-07-27T16:29:40.201Z" }, + { url = "https://files.pythonhosted.org/packages/a9/ae/7a10ff04a7dc15f9057d05b33737ade244e4bd195caa3f7cc04d77b9e214/scipy-1.16.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:dc54f76ac18073bcecffb98d93f03ed6b81a92ef91b5d3b135dcc81d55a724c7", size = 21365098, upload-time = "2025-07-27T16:29:44.295Z" }, + { url = "https://files.pythonhosted.org/packages/36/ac/029ff710959932ad3c2a98721b20b405f05f752f07344622fd61a47c5197/scipy-1.16.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:367d567ee9fc1e9e2047d31f39d9d6a7a04e0710c86e701e053f237d14a9b4f6", size = 23896858, upload-time = "2025-07-27T16:29:48.784Z" }, + { url = "https://files.pythonhosted.org/packages/71/13/d1ef77b6bd7898720e1f0b6b3743cb945f6c3cafa7718eaac8841035ab60/scipy-1.16.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4cf5785e44e19dcd32a0e4807555e1e9a9b8d475c6afff3d21c3c543a6aa84f4", size = 33438311, upload-time = "2025-07-27T16:29:54.164Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e0/e64a6821ffbb00b4c5b05169f1c1fddb4800e9307efe3db3788995a82a2c/scipy-1.16.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3d0b80fb26d3e13a794c71d4b837e2a589d839fd574a6bbb4ee1288c213ad4a3", size = 35279542, upload-time = "2025-07-27T16:30:00.249Z" }, + { url = "https://files.pythonhosted.org/packages/57/59/0dc3c8b43e118f1e4ee2b798dcc96ac21bb20014e5f1f7a8e85cc0653bdb/scipy-1.16.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:8503517c44c18d1030d666cb70aaac1cc8913608816e06742498833b128488b7", size = 35667665, upload-time = "2025-07-27T16:30:05.916Z" }, + { url = "https://files.pythonhosted.org/packages/45/5f/844ee26e34e2f3f9f8febb9343748e72daeaec64fe0c70e9bf1ff84ec955/scipy-1.16.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:30cc4bb81c41831ecfd6dc450baf48ffd80ef5aed0f5cf3ea775740e80f16ecc", size = 38045210, upload-time = "2025-07-27T16:30:11.655Z" }, + { url = "https://files.pythonhosted.org/packages/8d/d7/210f2b45290f444f1de64bc7353aa598ece9f0e90c384b4a156f9b1a5063/scipy-1.16.1-cp313-cp313t-win_amd64.whl", hash = "sha256:c24fa02f7ed23ae514460a22c57eca8f530dbfa50b1cfdbf4f37c05b5309cc39", size = 38593661, upload-time = "2025-07-27T16:30:17.825Z" }, + { url = "https://files.pythonhosted.org/packages/81/ea/84d481a5237ed223bd3d32d6e82d7a6a96e34756492666c260cef16011d1/scipy-1.16.1-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:796a5a9ad36fa3a782375db8f4241ab02a091308eb079746bc0f874c9b998318", size = 36525921, upload-time = "2025-07-27T16:30:30.081Z" }, + { url = "https://files.pythonhosted.org/packages/4e/9f/d9edbdeff9f3a664807ae3aea383e10afaa247e8e6255e6d2aa4515e8863/scipy-1.16.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:3ea0733a2ff73fd6fdc5fecca54ee9b459f4d74f00b99aced7d9a3adb43fb1cc", size = 28564152, upload-time = "2025-07-27T16:30:35.336Z" }, + { url = "https://files.pythonhosted.org/packages/3b/95/8125bcb1fe04bc267d103e76516243e8d5e11229e6b306bda1024a5423d1/scipy-1.16.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:85764fb15a2ad994e708258bb4ed8290d1305c62a4e1ef07c414356a24fcfbf8", size = 20836028, upload-time = "2025-07-27T16:30:39.421Z" }, + { url = "https://files.pythonhosted.org/packages/77/9c/bf92e215701fc70bbcd3d14d86337cf56a9b912a804b9c776a269524a9e9/scipy-1.16.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ca66d980469cb623b1759bdd6e9fd97d4e33a9fad5b33771ced24d0cb24df67e", size = 23489666, upload-time = "2025-07-27T16:30:43.663Z" }, + { url = "https://files.pythonhosted.org/packages/5e/00/5e941d397d9adac41b02839011594620d54d99488d1be5be755c00cde9ee/scipy-1.16.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e7cc1ffcc230f568549fc56670bcf3df1884c30bd652c5da8138199c8c76dae0", size = 33358318, upload-time = "2025-07-27T16:30:48.982Z" }, + { url = "https://files.pythonhosted.org/packages/0e/87/8db3aa10dde6e3e8e7eb0133f24baa011377d543f5b19c71469cf2648026/scipy-1.16.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3ddfb1e8d0b540cb4ee9c53fc3dea3186f97711248fb94b4142a1b27178d8b4b", size = 35185724, upload-time = "2025-07-27T16:30:54.26Z" }, + { url = "https://files.pythonhosted.org/packages/89/b4/6ab9ae443216807622bcff02690262d8184078ea467efee2f8c93288a3b1/scipy-1.16.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4dc0e7be79e95d8ba3435d193e0d8ce372f47f774cffd882f88ea4e1e1ddc731", size = 35554335, upload-time = "2025-07-27T16:30:59.765Z" }, + { url = "https://files.pythonhosted.org/packages/9c/9a/d0e9dc03c5269a1afb60661118296a32ed5d2c24298af61b676c11e05e56/scipy-1.16.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f23634f9e5adb51b2a77766dac217063e764337fbc816aa8ad9aaebcd4397fd3", size = 37960310, upload-time = "2025-07-27T16:31:06.151Z" }, + { url = "https://files.pythonhosted.org/packages/5e/00/c8f3130a50521a7977874817ca89e0599b1b4ee8e938bad8ae798a0e1f0d/scipy-1.16.1-cp314-cp314-win_amd64.whl", hash = "sha256:57d75524cb1c5a374958a2eae3d84e1929bb971204cc9d52213fb8589183fc19", size = 39319239, upload-time = "2025-07-27T16:31:59.942Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f2/1ca3eda54c3a7e4c92f6acef7db7b3a057deb135540d23aa6343ef8ad333/scipy-1.16.1-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:d8da7c3dd67bcd93f15618938f43ed0995982eb38973023d46d4646c4283ad65", size = 36939460, upload-time = "2025-07-27T16:31:11.865Z" }, + { url = "https://files.pythonhosted.org/packages/80/30/98c2840b293a132400c0940bb9e140171dcb8189588619048f42b2ce7b4f/scipy-1.16.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:cc1d2f2fd48ba1e0620554fe5bc44d3e8f5d4185c8c109c7fbdf5af2792cfad2", size = 29093322, upload-time = "2025-07-27T16:31:17.045Z" }, + { url = "https://files.pythonhosted.org/packages/c1/e6/1e6e006e850622cf2a039b62d1a6ddc4497d4851e58b68008526f04a9a00/scipy-1.16.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:21a611ced9275cb861bacadbada0b8c0623bc00b05b09eb97f23b370fc2ae56d", size = 21365329, upload-time = "2025-07-27T16:31:21.188Z" }, + { url = "https://files.pythonhosted.org/packages/8e/02/72a5aa5b820589dda9a25e329ca752842bfbbaf635e36bc7065a9b42216e/scipy-1.16.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:8dfbb25dffc4c3dd9371d8ab456ca81beeaf6f9e1c2119f179392f0dc1ab7695", size = 23897544, upload-time = "2025-07-27T16:31:25.408Z" }, + { url = "https://files.pythonhosted.org/packages/2b/dc/7122d806a6f9eb8a33532982234bed91f90272e990f414f2830cfe656e0b/scipy-1.16.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f0ebb7204f063fad87fc0a0e4ff4a2ff40b2a226e4ba1b7e34bf4b79bf97cd86", size = 33442112, upload-time = "2025-07-27T16:31:30.62Z" }, + { url = "https://files.pythonhosted.org/packages/24/39/e383af23564daa1021a5b3afbe0d8d6a68ec639b943661841f44ac92de85/scipy-1.16.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f1b9e5962656f2734c2b285a8745358ecb4e4efbadd00208c80a389227ec61ff", size = 35286594, upload-time = "2025-07-27T16:31:36.112Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/1a0b0aff40c3056d955f38b0df5d178350c3d74734ec54f9c68d23910be5/scipy-1.16.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e1a106f8c023d57a2a903e771228bf5c5b27b5d692088f457acacd3b54511e4", size = 35665080, upload-time = "2025-07-27T16:31:42.025Z" }, + { url = "https://files.pythonhosted.org/packages/64/df/ce88803e9ed6e27fe9b9abefa157cf2c80e4fa527cf17ee14be41f790ad4/scipy-1.16.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:709559a1db68a9abc3b2c8672c4badf1614f3b440b3ab326d86a5c0491eafae3", size = 38050306, upload-time = "2025-07-27T16:31:48.109Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6c/a76329897a7cae4937d403e623aa6aaea616a0bb5b36588f0b9d1c9a3739/scipy-1.16.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c0c804d60492a0aad7f5b2bb1862f4548b990049e27e828391ff2bf6f7199998", size = 39427705, upload-time = "2025-07-27T16:31:53.96Z" }, +] + +[[package]] +name = "setuptools" +version = "80.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "smart-open" +version = "7.3.0.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/2b/5e7234c68ed5bc872ad6ae77b8a421c2ed70dcb1190b44dc1abdeed5e347/smart_open-7.3.0.post1.tar.gz", hash = "sha256:ce6a3d9bc1afbf6234ad13c010b77f8cd36d24636811e3c52c3b5160f5214d1e", size = 51557, upload-time = "2025-07-03T10:06:31.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/5b/a2a3d4514c64818925f4e886d39981f1926eeb5288a4549c6b3c17ed66bb/smart_open-7.3.0.post1-py3-none-any.whl", hash = "sha256:c73661a2c24bf045c1e04e08fffc585b59af023fe783d57896f590489db66fb4", size = 61946, upload-time = "2025-07-03T10:06:29.599Z" }, +] + +[[package]] +name = "smmap" +version = "5.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/cd/a040c4b3119bbe532e5b0732286f805445375489fceaec1f48306068ee3b/smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5", size = 22329, upload-time = "2025-01-02T07:14:40.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303, upload-time = "2025-01-02T07:14:38.724Z" }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.43" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "(python_full_version < '3.14' and platform_machine == 'AMD64') or (python_full_version < '3.14' and platform_machine == 'WIN32') or (python_full_version < '3.14' and platform_machine == 'aarch64') or (python_full_version < '3.14' and platform_machine == 'amd64') or (python_full_version < '3.14' and platform_machine == 'ppc64le') or (python_full_version < '3.14' and platform_machine == 'win32') or (python_full_version < '3.14' and platform_machine == 'x86_64')" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d7/bc/d59b5d97d27229b0e009bd9098cd81af71c2fa5549c580a0a67b9bed0496/sqlalchemy-2.0.43.tar.gz", hash = "sha256:788bfcef6787a7764169cfe9859fe425bf44559619e1d9f56f5bddf2ebf6f417", size = 9762949, upload-time = "2025-08-11T14:24:58.438Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/db/20c78f1081446095450bdc6ee6cc10045fce67a8e003a5876b6eaafc5cc4/sqlalchemy-2.0.43-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:20d81fc2736509d7a2bd33292e489b056cbae543661bb7de7ce9f1c0cd6e7f24", size = 2134891, upload-time = "2025-08-11T15:51:13.019Z" }, + { url = "https://files.pythonhosted.org/packages/45/0a/3d89034ae62b200b4396f0f95319f7d86e9945ee64d2343dcad857150fa2/sqlalchemy-2.0.43-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:25b9fc27650ff5a2c9d490c13c14906b918b0de1f8fcbb4c992712d8caf40e83", size = 2123061, upload-time = "2025-08-11T15:51:14.319Z" }, + { url = "https://files.pythonhosted.org/packages/cb/10/2711f7ff1805919221ad5bee205971254845c069ee2e7036847103ca1e4c/sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6772e3ca8a43a65a37c88e2f3e2adfd511b0b1da37ef11ed78dea16aeae85bd9", size = 3320384, upload-time = "2025-08-11T15:52:35.088Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0e/3d155e264d2ed2778484006ef04647bc63f55b3e2d12e6a4f787747b5900/sqlalchemy-2.0.43-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a113da919c25f7f641ffbd07fbc9077abd4b3b75097c888ab818f962707eb48", size = 3329648, upload-time = "2025-08-11T15:56:34.153Z" }, + { url = "https://files.pythonhosted.org/packages/5b/81/635100fb19725c931622c673900da5efb1595c96ff5b441e07e3dd61f2be/sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4286a1139f14b7d70141c67a8ae1582fc2b69105f1b09d9573494eb4bb4b2687", size = 3258030, upload-time = "2025-08-11T15:52:36.933Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ed/a99302716d62b4965fded12520c1cbb189f99b17a6d8cf77611d21442e47/sqlalchemy-2.0.43-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:529064085be2f4d8a6e5fab12d36ad44f1909a18848fcfbdb59cc6d4bbe48efe", size = 3294469, upload-time = "2025-08-11T15:56:35.553Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a2/3a11b06715149bf3310b55a98b5c1e84a42cfb949a7b800bc75cb4e33abc/sqlalchemy-2.0.43-cp312-cp312-win32.whl", hash = "sha256:b535d35dea8bbb8195e7e2b40059e2253acb2b7579b73c1b432a35363694641d", size = 2098906, upload-time = "2025-08-11T15:55:00.645Z" }, + { url = "https://files.pythonhosted.org/packages/bc/09/405c915a974814b90aa591280623adc6ad6b322f61fd5cff80aeaef216c9/sqlalchemy-2.0.43-cp312-cp312-win_amd64.whl", hash = "sha256:1c6d85327ca688dbae7e2b06d7d84cfe4f3fffa5b5f9e21bb6ce9d0e1a0e0e0a", size = 2126260, upload-time = "2025-08-11T15:55:02.965Z" }, + { url = "https://files.pythonhosted.org/packages/41/1c/a7260bd47a6fae7e03768bf66451437b36451143f36b285522b865987ced/sqlalchemy-2.0.43-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e7c08f57f75a2bb62d7ee80a89686a5e5669f199235c6d1dac75cd59374091c3", size = 2130598, upload-time = "2025-08-11T15:51:15.903Z" }, + { url = "https://files.pythonhosted.org/packages/8e/84/8a337454e82388283830b3586ad7847aa9c76fdd4f1df09cdd1f94591873/sqlalchemy-2.0.43-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:14111d22c29efad445cd5021a70a8b42f7d9152d8ba7f73304c4d82460946aaa", size = 2118415, upload-time = "2025-08-11T15:51:17.256Z" }, + { url = "https://files.pythonhosted.org/packages/cf/ff/22ab2328148492c4d71899d62a0e65370ea66c877aea017a244a35733685/sqlalchemy-2.0.43-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21b27b56eb2f82653168cefe6cb8e970cdaf4f3a6cb2c5e3c3c1cf3158968ff9", size = 3248707, upload-time = "2025-08-11T15:52:38.444Z" }, + { url = "https://files.pythonhosted.org/packages/dc/29/11ae2c2b981de60187f7cbc84277d9d21f101093d1b2e945c63774477aba/sqlalchemy-2.0.43-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c5a9da957c56e43d72126a3f5845603da00e0293720b03bde0aacffcf2dc04f", size = 3253602, upload-time = "2025-08-11T15:56:37.348Z" }, + { url = "https://files.pythonhosted.org/packages/b8/61/987b6c23b12c56d2be451bc70900f67dd7d989d52b1ee64f239cf19aec69/sqlalchemy-2.0.43-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d79f9fdc9584ec83d1b3c75e9f4595c49017f5594fee1a2217117647225d738", size = 3183248, upload-time = "2025-08-11T15:52:39.865Z" }, + { url = "https://files.pythonhosted.org/packages/86/85/29d216002d4593c2ce1c0ec2cec46dda77bfbcd221e24caa6e85eff53d89/sqlalchemy-2.0.43-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9df7126fd9db49e3a5a3999442cc67e9ee8971f3cb9644250107d7296cb2a164", size = 3219363, upload-time = "2025-08-11T15:56:39.11Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e4/bd78b01919c524f190b4905d47e7630bf4130b9f48fd971ae1c6225b6f6a/sqlalchemy-2.0.43-cp313-cp313-win32.whl", hash = "sha256:7f1ac7828857fcedb0361b48b9ac4821469f7694089d15550bbcf9ab22564a1d", size = 2096718, upload-time = "2025-08-11T15:55:05.349Z" }, + { url = "https://files.pythonhosted.org/packages/ac/a5/ca2f07a2a201f9497de1928f787926613db6307992fe5cda97624eb07c2f/sqlalchemy-2.0.43-cp313-cp313-win_amd64.whl", hash = "sha256:971ba928fcde01869361f504fcff3b7143b47d30de188b11c6357c0505824197", size = 2123200, upload-time = "2025-08-11T15:55:07.932Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d9/13bdde6521f322861fab67473cec4b1cc8999f3871953531cf61945fad92/sqlalchemy-2.0.43-py3-none-any.whl", hash = "sha256:1681c21dd2ccee222c2fe0bef671d1aef7c504087c9c4e800371cfcc8ac966fc", size = 1924759, upload-time = "2025-08-11T15:39:53.024Z" }, +] + +[[package]] +name = "sqlparse" +version = "0.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/40/edede8dd6977b0d3da179a342c198ed100dd2aba4be081861ee5911e4da4/sqlparse-0.5.3.tar.gz", hash = "sha256:09f67787f56a0b16ecdbde1bfc7f5d9c3371ca683cfeaa8e6ff60b4807ec9272", size = 84999, upload-time = "2024-12-10T12:05:30.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/5c/bfd6bd0bf979426d405cc6e71eceb8701b148b16c21d2dc3c261efc61c7b/sqlparse-0.5.3-py3-none-any.whl", hash = "sha256:cf2196ed3418f3ba5de6af7e82c694a9fbdbfecccdfc72e281548517081f16ca", size = 44415, upload-time = "2024-12-10T12:05:27.824Z" }, +] + +[[package]] +name = "stack-data" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asttokens" }, + { name = "executing" }, + { name = "pure-eval" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, +] + +[[package]] +name = "starlette" +version = "0.47.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/57/d062573f391d062710d4088fa1369428c38d51460ab6fedff920efef932e/starlette-0.47.2.tar.gz", hash = "sha256:6ae9aa5db235e4846decc1e7b79c4f346adf41e9777aebeb49dfd09bbd7023d8", size = 2583948, upload-time = "2025-07-20T17:31:58.522Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/1f/b876b1f83aef204198a42dc101613fefccb32258e5428b5f9259677864b4/starlette-0.47.2-py3-none-any.whl", hash = "sha256:c5847e96134e5c5371ee9fac6fdf1a67336d5815e09eb2a01fdb57a351ef915b", size = 72984, upload-time = "2025-07-20T17:31:56.738Z" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "tensorboardx" +version = "2.6.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2b/c5/d4cc6e293fb837aaf9f76dd7745476aeba8ef7ef5146c3b3f9ee375fe7a5/tensorboardx-2.6.4.tar.gz", hash = "sha256:b163ccb7798b31100b9f5fa4d6bc22dad362d7065c2f24b51e50731adde86828", size = 4769801, upload-time = "2025-06-10T22:37:07.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/1d/b5d63f1a6b824282b57f7b581810d20b7a28ca951f2d5b59f1eb0782c12b/tensorboardx-2.6.4-py3-none-any.whl", hash = "sha256:5970cf3a1f0a6a6e8b180ccf46f3fe832b8a25a70b86e5a237048a7c0beb18e2", size = 87201, upload-time = "2025-06-10T22:37:05.44Z" }, +] + +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.21.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/2f/402986d0823f8d7ca139d969af2917fefaa9b947d1fb32f6168c509f2492/tokenizers-0.21.4.tar.gz", hash = "sha256:fa23f85fbc9a02ec5c6978da172cdcbac23498c3ca9f3645c5c68740ac007880", size = 351253, upload-time = "2025-07-28T15:48:54.325Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/c6/fdb6f72bf6454f52eb4a2510be7fb0f614e541a2554d6210e370d85efff4/tokenizers-0.21.4-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2ccc10a7c3bcefe0f242867dc914fc1226ee44321eb618cfe3019b5df3400133", size = 2863987, upload-time = "2025-07-28T15:48:44.877Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a6/28975479e35ddc751dc1ddc97b9b69bf7fcf074db31548aab37f8116674c/tokenizers-0.21.4-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:5e2f601a8e0cd5be5cc7506b20a79112370b9b3e9cb5f13f68ab11acd6ca7d60", size = 2732457, upload-time = "2025-07-28T15:48:43.265Z" }, + { url = "https://files.pythonhosted.org/packages/aa/8f/24f39d7b5c726b7b0be95dca04f344df278a3fe3a4deb15a975d194cbb32/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39b376f5a1aee67b4d29032ee85511bbd1b99007ec735f7f35c8a2eb104eade5", size = 3012624, upload-time = "2025-07-28T13:22:43.895Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/26358925717687a58cb74d7a508de96649544fad5778f0cd9827398dc499/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2107ad649e2cda4488d41dfd031469e9da3fcbfd6183e74e4958fa729ffbf9c6", size = 2939681, upload-time = "2025-07-28T13:22:47.499Z" }, + { url = "https://files.pythonhosted.org/packages/99/6f/cc300fea5db2ab5ddc2c8aea5757a27b89c84469899710c3aeddc1d39801/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c73012da95afafdf235ba80047699df4384fdc481527448a078ffd00e45a7d9", size = 3247445, upload-time = "2025-07-28T15:48:39.711Z" }, + { url = "https://files.pythonhosted.org/packages/be/bf/98cb4b9c3c4afd8be89cfa6423704337dc20b73eb4180397a6e0d456c334/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f23186c40395fc390d27f519679a58023f368a0aad234af145e0f39ad1212732", size = 3428014, upload-time = "2025-07-28T13:22:49.569Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/96c1cc780e6ca7f01a57c13235dd05b7bc1c0f3588512ebe9d1331b5f5ae/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc88bb34e23a54cc42713d6d98af5f1bf79c07653d24fe984d2d695ba2c922a2", size = 3193197, upload-time = "2025-07-28T13:22:51.471Z" }, + { url = "https://files.pythonhosted.org/packages/f2/90/273b6c7ec78af547694eddeea9e05de771278bd20476525ab930cecaf7d8/tokenizers-0.21.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51b7eabb104f46c1c50b486520555715457ae833d5aee9ff6ae853d1130506ff", size = 3115426, upload-time = "2025-07-28T15:48:41.439Z" }, + { url = "https://files.pythonhosted.org/packages/91/43/c640d5a07e95f1cf9d2c92501f20a25f179ac53a4f71e1489a3dcfcc67ee/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:714b05b2e1af1288bd1bc56ce496c4cebb64a20d158ee802887757791191e6e2", size = 9089127, upload-time = "2025-07-28T15:48:46.472Z" }, + { url = "https://files.pythonhosted.org/packages/44/a1/dd23edd6271d4dca788e5200a807b49ec3e6987815cd9d0a07ad9c96c7c2/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1340ff877ceedfa937544b7d79f5b7becf33a4cfb58f89b3b49927004ef66f78", size = 9055243, upload-time = "2025-07-28T15:48:48.539Z" }, + { url = "https://files.pythonhosted.org/packages/21/2b/b410d6e9021c4b7ddb57248304dc817c4d4970b73b6ee343674914701197/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:3c1f4317576e465ac9ef0d165b247825a2a4078bcd01cba6b54b867bdf9fdd8b", size = 9298237, upload-time = "2025-07-28T15:48:50.443Z" }, + { url = "https://files.pythonhosted.org/packages/b7/0a/42348c995c67e2e6e5c89ffb9cfd68507cbaeb84ff39c49ee6e0a6dd0fd2/tokenizers-0.21.4-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:c212aa4e45ec0bb5274b16b6f31dd3f1c41944025c2358faaa5782c754e84c24", size = 9461980, upload-time = "2025-07-28T15:48:52.325Z" }, + { url = "https://files.pythonhosted.org/packages/3d/d3/dacccd834404cd71b5c334882f3ba40331ad2120e69ded32cf5fda9a7436/tokenizers-0.21.4-cp39-abi3-win32.whl", hash = "sha256:6c42a930bc5f4c47f4ea775c91de47d27910881902b0f20e4990ebe045a415d0", size = 2329871, upload-time = "2025-07-28T15:48:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/41/f2/fd673d979185f5dcbac4be7d09461cbb99751554ffb6718d0013af8604cb/tokenizers-0.21.4-cp39-abi3-win_amd64.whl", hash = "sha256:475d807a5c3eb72c59ad9b5fcdb254f6e17f53dfcbb9903233b0dfa9c943b597", size = 2507568, upload-time = "2025-07-28T15:48:55.456Z" }, +] + +[[package]] +name = "torch" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/93/fb505a5022a2e908d81fe9a5e0aa84c86c0d5f408173be71c6018836f34e/torch-2.7.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:27ea1e518df4c9de73af7e8a720770f3628e7f667280bce2be7a16292697e3fa", size = 98948276, upload-time = "2025-06-04T17:39:12.852Z" }, + { url = "https://files.pythonhosted.org/packages/56/7e/67c3fe2b8c33f40af06326a3d6ae7776b3e3a01daa8f71d125d78594d874/torch-2.7.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c33360cfc2edd976c2633b3b66c769bdcbbf0e0b6550606d188431c81e7dd1fc", size = 821025792, upload-time = "2025-06-04T17:34:58.747Z" }, + { url = "https://files.pythonhosted.org/packages/a1/37/a37495502bc7a23bf34f89584fa5a78e25bae7b8da513bc1b8f97afb7009/torch-2.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:d8bf6e1856ddd1807e79dc57e54d3335f2b62e6f316ed13ed3ecfe1fc1df3d8b", size = 216050349, upload-time = "2025-06-04T17:38:59.709Z" }, + { url = "https://files.pythonhosted.org/packages/3a/60/04b77281c730bb13460628e518c52721257814ac6c298acd25757f6a175c/torch-2.7.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:787687087412c4bd68d315e39bc1223f08aae1d16a9e9771d95eabbb04ae98fb", size = 68645146, upload-time = "2025-06-04T17:38:52.97Z" }, + { url = "https://files.pythonhosted.org/packages/66/81/e48c9edb655ee8eb8c2a6026abdb6f8d2146abd1f150979ede807bb75dcb/torch-2.7.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:03563603d931e70722dce0e11999d53aa80a375a3d78e6b39b9f6805ea0a8d28", size = 98946649, upload-time = "2025-06-04T17:38:43.031Z" }, + { url = "https://files.pythonhosted.org/packages/3a/24/efe2f520d75274fc06b695c616415a1e8a1021d87a13c68ff9dce733d088/torch-2.7.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:d632f5417b6980f61404a125b999ca6ebd0b8b4bbdbb5fbbba44374ab619a412", size = 821033192, upload-time = "2025-06-04T17:38:09.146Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d9/9c24d230333ff4e9b6807274f6f8d52a864210b52ec794c5def7925f4495/torch-2.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:23660443e13995ee93e3d844786701ea4ca69f337027b05182f5ba053ce43b38", size = 216055668, upload-time = "2025-06-04T17:38:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/95/bf/e086ee36ddcef9299f6e708d3b6c8487c1651787bb9ee2939eb2a7f74911/torch-2.7.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:0da4f4dba9f65d0d203794e619fe7ca3247a55ffdcbd17ae8fb83c8b2dc9b585", size = 68925988, upload-time = "2025-06-04T17:38:29.273Z" }, + { url = "https://files.pythonhosted.org/packages/69/6a/67090dcfe1cf9048448b31555af6efb149f7afa0a310a366adbdada32105/torch-2.7.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e08d7e6f21a617fe38eeb46dd2213ded43f27c072e9165dc27300c9ef9570934", size = 99028857, upload-time = "2025-06-04T17:37:50.956Z" }, + { url = "https://files.pythonhosted.org/packages/90/1c/48b988870823d1cc381f15ec4e70ed3d65e043f43f919329b0045ae83529/torch-2.7.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:30207f672328a42df4f2174b8f426f354b2baa0b7cca3a0adb3d6ab5daf00dc8", size = 821098066, upload-time = "2025-06-04T17:37:33.939Z" }, + { url = "https://files.pythonhosted.org/packages/7b/eb/10050d61c9d5140c5dc04a89ed3257ef1a6b93e49dd91b95363d757071e0/torch-2.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:79042feca1c634aaf6603fe6feea8c6b30dfa140a6bbc0b973e2260c7e79a22e", size = 216336310, upload-time = "2025-06-04T17:36:09.862Z" }, + { url = "https://files.pythonhosted.org/packages/b1/29/beb45cdf5c4fc3ebe282bf5eafc8dfd925ead7299b3c97491900fe5ed844/torch-2.7.1-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:988b0cbc4333618a1056d2ebad9eb10089637b659eb645434d0809d8d937b946", size = 68645708, upload-time = "2025-06-04T17:34:39.852Z" }, +] + +[[package]] +name = "tornado" +version = "6.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/09/ce/1eb500eae19f4648281bb2186927bb062d2438c2e5093d1360391afd2f90/tornado-6.5.2.tar.gz", hash = "sha256:ab53c8f9a0fa351e2c0741284e06c7a45da86afb544133201c5cc8578eb076a0", size = 510821, upload-time = "2025-08-08T18:27:00.78Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/48/6a7529df2c9cc12efd2e8f5dd219516184d703b34c06786809670df5b3bd/tornado-6.5.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:2436822940d37cde62771cff8774f4f00b3c8024fe482e16ca8387b8a2724db6", size = 442563, upload-time = "2025-08-08T18:26:42.945Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b5/9b575a0ed3e50b00c40b08cbce82eb618229091d09f6d14bce80fc01cb0b/tornado-6.5.2-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:583a52c7aa94ee046854ba81d9ebb6c81ec0fd30386d96f7640c96dad45a03ef", size = 440729, upload-time = "2025-08-08T18:26:44.473Z" }, + { url = "https://files.pythonhosted.org/packages/1b/4e/619174f52b120efcf23633c817fd3fed867c30bff785e2cd5a53a70e483c/tornado-6.5.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0fe179f28d597deab2842b86ed4060deec7388f1fd9c1b4a41adf8af058907e", size = 444295, upload-time = "2025-08-08T18:26:46.021Z" }, + { url = "https://files.pythonhosted.org/packages/95/fa/87b41709552bbd393c85dd18e4e3499dcd8983f66e7972926db8d96aa065/tornado-6.5.2-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b186e85d1e3536d69583d2298423744740986018e393d0321df7340e71898882", size = 443644, upload-time = "2025-08-08T18:26:47.625Z" }, + { url = "https://files.pythonhosted.org/packages/f9/41/fb15f06e33d7430ca89420283a8762a4e6b8025b800ea51796ab5e6d9559/tornado-6.5.2-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e792706668c87709709c18b353da1f7662317b563ff69f00bab83595940c7108", size = 443878, upload-time = "2025-08-08T18:26:50.599Z" }, + { url = "https://files.pythonhosted.org/packages/11/92/fe6d57da897776ad2e01e279170ea8ae726755b045fe5ac73b75357a5a3f/tornado-6.5.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:06ceb1300fd70cb20e43b1ad8aaee0266e69e7ced38fa910ad2e03285009ce7c", size = 444549, upload-time = "2025-08-08T18:26:51.864Z" }, + { url = "https://files.pythonhosted.org/packages/9b/02/c8f4f6c9204526daf3d760f4aa555a7a33ad0e60843eac025ccfd6ff4a93/tornado-6.5.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:74db443e0f5251be86cbf37929f84d8c20c27a355dd452a5cfa2aada0d001ec4", size = 443973, upload-time = "2025-08-08T18:26:53.625Z" }, + { url = "https://files.pythonhosted.org/packages/ae/2d/f5f5707b655ce2317190183868cd0f6822a1121b4baeae509ceb9590d0bd/tornado-6.5.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b5e735ab2889d7ed33b32a459cac490eda71a1ba6857b0118de476ab6c366c04", size = 443954, upload-time = "2025-08-08T18:26:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/e8/59/593bd0f40f7355806bf6573b47b8c22f8e1374c9b6fd03114bd6b7a3dcfd/tornado-6.5.2-cp39-abi3-win32.whl", hash = "sha256:c6f29e94d9b37a95013bb669616352ddb82e3bfe8326fccee50583caebc8a5f0", size = 445023, upload-time = "2025-08-08T18:26:56.677Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2a/f609b420c2f564a748a2d80ebfb2ee02a73ca80223af712fca591386cafb/tornado-6.5.2-cp39-abi3-win_amd64.whl", hash = "sha256:e56a5af51cc30dd2cae649429af65ca2f6571da29504a07995175df14c18f35f", size = 445427, upload-time = "2025-08-08T18:26:57.91Z" }, + { url = "https://files.pythonhosted.org/packages/5e/4f/e1f65e8f8c76d73658b33d33b81eed4322fb5085350e4328d5c956f0c8f9/tornado-6.5.2-cp39-abi3-win_arm64.whl", hash = "sha256:d6c33dc3672e3a1f3618eb63b7ef4683a7688e7b9e6e8f0d9aa5726360a004af", size = 444456, upload-time = "2025-08-08T18:26:59.207Z" }, +] + +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, +] + +[[package]] +name = "traitlets" +version = "5.14.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/79/72064e6a701c2183016abbbfedaba506d81e30e232a68c9f0d6f6fcd1574/traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7", size = 161621, upload-time = "2024-04-19T11:11:49.746Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359, upload-time = "2024-04-19T11:11:46.763Z" }, +] + +[[package]] +name = "transformers" +version = "4.52.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "requests" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/42/271bcf364788337ac24e7f200005ac7142aaf022206bd6119d2daca22c04/transformers-4.52.3.tar.gz", hash = "sha256:2e1de29374f27920aaf6d589d4e6339f33def2fb08809e1a1d792e040e9fbce7", size = 8951324, upload-time = "2025-05-22T14:40:52.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f8/1f086942bc6a044e4e68dacf6de761a45367795efd5f57ad356765691c79/transformers-4.52.3-py3-none-any.whl", hash = "sha256:cd04059da50e7cf2a617ce3143ba8beffbf119f8c25a0717c3454fd9d0f19609", size = 10460322, upload-time = "2025-05-22T14:40:49.583Z" }, +] + +[[package]] +name = "triton" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "setuptools", marker = "sys_platform != 'win32'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/5f/950fb373bf9c01ad4eb5a8cd5eaf32cdf9e238c02f9293557a2129b9c4ac/triton-3.3.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9999e83aba21e1a78c1f36f21bce621b77bcaa530277a50484a7cb4a822f6e43", size = 155669138, upload-time = "2025-05-29T23:39:51.771Z" }, + { url = "https://files.pythonhosted.org/packages/74/1f/dfb531f90a2d367d914adfee771babbd3f1a5b26c3f5fbc458dee21daa78/triton-3.3.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b89d846b5a4198317fec27a5d3a609ea96b6d557ff44b56c23176546023c4240", size = 155673035, upload-time = "2025-05-29T23:40:02.468Z" }, + { url = "https://files.pythonhosted.org/packages/28/71/bd20ffcb7a64c753dc2463489a61bf69d531f308e390ad06390268c4ea04/triton-3.3.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3198adb9d78b77818a5388bff89fa72ff36f9da0bc689db2f0a651a67ce6a42", size = 155735832, upload-time = "2025-05-29T23:40:10.522Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/b1/0c11f5058406b3af7609f121aaa6b609744687f1d158b3c3a5bf4cc94238/typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28", size = 75726, upload-time = "2025-05-21T18:55:23.885Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552, upload-time = "2025-05-21T18:55:22.152Z" }, +] + +[[package]] +name = "tzdata" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.35.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/42/e0e305207bb88c6b8d3061399c6a961ffe5fbb7e2aa63c9234df7259e9cd/uvicorn-0.35.0.tar.gz", hash = "sha256:bc662f087f7cf2ce11a1d7fd70b90c9f98ef2e2831556dd078d131b96cc94a01", size = 78473, upload-time = "2025-06-28T16:15:46.058Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/e2/dc81b1bd1dcfe91735810265e9d26bc8ec5da45b4c0f6237e286819194c3/uvicorn-0.35.0-py3-none-any.whl", hash = "sha256:197535216b25ff9b785e29a0b79199f55222193d47f820816e7da751e9bc8d4a", size = 66406, upload-time = "2025-06-28T16:15:44.816Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + +[[package]] +name = "uvloop" +version = "0.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/c0/854216d09d33c543f12a44b393c402e89a920b1a0a7dc634c42de91b9cf6/uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3", size = 2492741, upload-time = "2024-10-14T23:38:35.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/4c/03f93178830dc7ce8b4cdee1d36770d2f5ebb6f3d37d354e061eefc73545/uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c", size = 1471284, upload-time = "2024-10-14T23:37:47.833Z" }, + { url = "https://files.pythonhosted.org/packages/43/3e/92c03f4d05e50f09251bd8b2b2b584a2a7f8fe600008bcc4523337abe676/uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2", size = 821349, upload-time = "2024-10-14T23:37:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ef/a02ec5da49909dbbfb1fd205a9a1ac4e88ea92dcae885e7c961847cd51e2/uvloop-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d", size = 4580089, upload-time = "2024-10-14T23:37:51.703Z" }, + { url = "https://files.pythonhosted.org/packages/06/a7/b4e6a19925c900be9f98bec0a75e6e8f79bb53bdeb891916609ab3958967/uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc", size = 4693770, upload-time = "2024-10-14T23:37:54.122Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0c/f07435a18a4b94ce6bd0677d8319cd3de61f3a9eeb1e5f8ab4e8b5edfcb3/uvloop-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb", size = 4451321, upload-time = "2024-10-14T23:37:55.766Z" }, + { url = "https://files.pythonhosted.org/packages/8f/eb/f7032be105877bcf924709c97b1bf3b90255b4ec251f9340cef912559f28/uvloop-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f", size = 4659022, upload-time = "2024-10-14T23:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/3f/8d/2cbef610ca21539f0f36e2b34da49302029e7c9f09acef0b1c3b5839412b/uvloop-0.21.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:bfd55dfcc2a512316e65f16e503e9e450cab148ef11df4e4e679b5e8253a5281", size = 1468123, upload-time = "2024-10-14T23:38:00.688Z" }, + { url = "https://files.pythonhosted.org/packages/93/0d/b0038d5a469f94ed8f2b2fce2434a18396d8fbfb5da85a0a9781ebbdec14/uvloop-0.21.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:787ae31ad8a2856fc4e7c095341cccc7209bd657d0e71ad0dc2ea83c4a6fa8af", size = 819325, upload-time = "2024-10-14T23:38:02.309Z" }, + { url = "https://files.pythonhosted.org/packages/50/94/0a687f39e78c4c1e02e3272c6b2ccdb4e0085fda3b8352fecd0410ccf915/uvloop-0.21.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ee4d4ef48036ff6e5cfffb09dd192c7a5027153948d85b8da7ff705065bacc6", size = 4582806, upload-time = "2024-10-14T23:38:04.711Z" }, + { url = "https://files.pythonhosted.org/packages/d2/19/f5b78616566ea68edd42aacaf645adbf71fbd83fc52281fba555dc27e3f1/uvloop-0.21.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816", size = 4701068, upload-time = "2024-10-14T23:38:06.385Z" }, + { url = "https://files.pythonhosted.org/packages/47/57/66f061ee118f413cd22a656de622925097170b9380b30091b78ea0c6ea75/uvloop-0.21.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bd53ecc9a0f3d87ab847503c2e1552b690362e005ab54e8a48ba97da3924c0dc", size = 4454428, upload-time = "2024-10-14T23:38:08.416Z" }, + { url = "https://files.pythonhosted.org/packages/63/9a/0962b05b308494e3202d3f794a6e85abe471fe3cafdbcf95c2e8c713aabd/uvloop-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553", size = 4660018, upload-time = "2024-10-14T23:38:10.888Z" }, +] + +[[package]] +name = "virtualenv" +version = "20.34.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/14/37fcdba2808a6c615681cd216fecae00413c9dab44fb2e57805ecf3eaee3/virtualenv-20.34.0.tar.gz", hash = "sha256:44815b2c9dee7ed86e387b842a84f20b93f7f417f95886ca1996a72a4138eb1a", size = 6003808, upload-time = "2025-08-13T14:24:07.464Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/06/04c8e804f813cf972e3262f3f8584c232de64f0cde9f703b46cf53a45090/virtualenv-20.34.0-py3-none-any.whl", hash = "sha256:341f5afa7eee943e4984a9207c025feedd768baff6753cd660c857ceb3e36026", size = 5983279, upload-time = "2025-08-13T14:24:05.111Z" }, +] + +[[package]] +name = "waitress" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/cb/04ddb054f45faa306a230769e868c28b8065ea196891f09004ebace5b184/waitress-3.0.2.tar.gz", hash = "sha256:682aaaf2af0c44ada4abfb70ded36393f0e307f4ab9456a215ce0020baefc31f", size = 179901, upload-time = "2024-11-16T20:02:35.195Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/57/a27182528c90ef38d82b636a11f606b0cbb0e17588ed205435f8affe3368/waitress-3.0.2-py3-none-any.whl", hash = "sha256:c56d67fd6e87c2ee598b76abdd4e96cfad1f24cacdea5078d382b1f9d7b5ed2e", size = 56232, upload-time = "2024-11-16T20:02:33.858Z" }, +] + +[[package]] +name = "watchfiles" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2a/9a/d451fcc97d029f5812e898fd30a53fd8c15c7bbd058fd75cfc6beb9bd761/watchfiles-1.1.0.tar.gz", hash = "sha256:693ed7ec72cbfcee399e92c895362b6e66d63dac6b91e2c11ae03d10d503e575", size = 94406, upload-time = "2025-06-15T19:06:59.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/b8/858957045a38a4079203a33aaa7d23ea9269ca7761c8a074af3524fbb240/watchfiles-1.1.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9dc001c3e10de4725c749d4c2f2bdc6ae24de5a88a339c4bce32300a31ede179", size = 402339, upload-time = "2025-06-15T19:05:24.516Z" }, + { url = "https://files.pythonhosted.org/packages/80/28/98b222cca751ba68e88521fabd79a4fab64005fc5976ea49b53fa205d1fa/watchfiles-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d9ba68ec283153dead62cbe81872d28e053745f12335d037de9cbd14bd1877f5", size = 394409, upload-time = "2025-06-15T19:05:25.469Z" }, + { url = "https://files.pythonhosted.org/packages/86/50/dee79968566c03190677c26f7f47960aff738d32087087bdf63a5473e7df/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:130fc497b8ee68dce163e4254d9b0356411d1490e868bd8790028bc46c5cc297", size = 450939, upload-time = "2025-06-15T19:05:26.494Z" }, + { url = "https://files.pythonhosted.org/packages/40/45/a7b56fb129700f3cfe2594a01aa38d033b92a33dddce86c8dfdfc1247b72/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:50a51a90610d0845a5931a780d8e51d7bd7f309ebc25132ba975aca016b576a0", size = 457270, upload-time = "2025-06-15T19:05:27.466Z" }, + { url = "https://files.pythonhosted.org/packages/b5/c8/fa5ef9476b1d02dc6b5e258f515fcaaecf559037edf8b6feffcbc097c4b8/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc44678a72ac0910bac46fa6a0de6af9ba1355669b3dfaf1ce5f05ca7a74364e", size = 483370, upload-time = "2025-06-15T19:05:28.548Z" }, + { url = "https://files.pythonhosted.org/packages/98/68/42cfcdd6533ec94f0a7aab83f759ec11280f70b11bfba0b0f885e298f9bd/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a543492513a93b001975ae283a51f4b67973662a375a403ae82f420d2c7205ee", size = 598654, upload-time = "2025-06-15T19:05:29.997Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/b2a1544224118cc28df7e59008a929e711f9c68ce7d554e171b2dc531352/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ac164e20d17cc285f2b94dc31c384bc3aa3dd5e7490473b3db043dd70fbccfd", size = 478667, upload-time = "2025-06-15T19:05:31.172Z" }, + { url = "https://files.pythonhosted.org/packages/8c/77/e3362fe308358dc9f8588102481e599c83e1b91c2ae843780a7ded939a35/watchfiles-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7590d5a455321e53857892ab8879dce62d1f4b04748769f5adf2e707afb9d4f", size = 452213, upload-time = "2025-06-15T19:05:32.299Z" }, + { url = "https://files.pythonhosted.org/packages/6e/17/c8f1a36540c9a1558d4faf08e909399e8133599fa359bf52ec8fcee5be6f/watchfiles-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:37d3d3f7defb13f62ece99e9be912afe9dd8a0077b7c45ee5a57c74811d581a4", size = 626718, upload-time = "2025-06-15T19:05:33.415Z" }, + { url = "https://files.pythonhosted.org/packages/26/45/fb599be38b4bd38032643783d7496a26a6f9ae05dea1a42e58229a20ac13/watchfiles-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7080c4bb3efd70a07b1cc2df99a7aa51d98685be56be6038c3169199d0a1c69f", size = 623098, upload-time = "2025-06-15T19:05:34.534Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e7/fdf40e038475498e160cd167333c946e45d8563ae4dd65caf757e9ffe6b4/watchfiles-1.1.0-cp312-cp312-win32.whl", hash = "sha256:cbcf8630ef4afb05dc30107bfa17f16c0896bb30ee48fc24bf64c1f970f3b1fd", size = 279209, upload-time = "2025-06-15T19:05:35.577Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d3/3ae9d5124ec75143bdf088d436cba39812122edc47709cd2caafeac3266f/watchfiles-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:cbd949bdd87567b0ad183d7676feb98136cde5bb9025403794a4c0db28ed3a47", size = 292786, upload-time = "2025-06-15T19:05:36.559Z" }, + { url = "https://files.pythonhosted.org/packages/26/2f/7dd4fc8b5f2b34b545e19629b4a018bfb1de23b3a496766a2c1165ca890d/watchfiles-1.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:0a7d40b77f07be87c6faa93d0951a0fcd8cbca1ddff60a1b65d741bac6f3a9f6", size = 284343, upload-time = "2025-06-15T19:05:37.5Z" }, + { url = "https://files.pythonhosted.org/packages/d3/42/fae874df96595556a9089ade83be34a2e04f0f11eb53a8dbf8a8a5e562b4/watchfiles-1.1.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5007f860c7f1f8df471e4e04aaa8c43673429047d63205d1630880f7637bca30", size = 402004, upload-time = "2025-06-15T19:05:38.499Z" }, + { url = "https://files.pythonhosted.org/packages/fa/55/a77e533e59c3003d9803c09c44c3651224067cbe7fb5d574ddbaa31e11ca/watchfiles-1.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:20ecc8abbd957046f1fe9562757903f5eaf57c3bce70929fda6c7711bb58074a", size = 393671, upload-time = "2025-06-15T19:05:39.52Z" }, + { url = "https://files.pythonhosted.org/packages/05/68/b0afb3f79c8e832e6571022611adbdc36e35a44e14f129ba09709aa4bb7a/watchfiles-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2f0498b7d2a3c072766dba3274fe22a183dbea1f99d188f1c6c72209a1063dc", size = 449772, upload-time = "2025-06-15T19:05:40.897Z" }, + { url = "https://files.pythonhosted.org/packages/ff/05/46dd1f6879bc40e1e74c6c39a1b9ab9e790bf1f5a2fe6c08b463d9a807f4/watchfiles-1.1.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:239736577e848678e13b201bba14e89718f5c2133dfd6b1f7846fa1b58a8532b", size = 456789, upload-time = "2025-06-15T19:05:42.045Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ca/0eeb2c06227ca7f12e50a47a3679df0cd1ba487ea19cf844a905920f8e95/watchfiles-1.1.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eff4b8d89f444f7e49136dc695599a591ff769300734446c0a86cba2eb2f9895", size = 482551, upload-time = "2025-06-15T19:05:43.781Z" }, + { url = "https://files.pythonhosted.org/packages/31/47/2cecbd8694095647406645f822781008cc524320466ea393f55fe70eed3b/watchfiles-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12b0a02a91762c08f7264e2e79542f76870c3040bbc847fb67410ab81474932a", size = 597420, upload-time = "2025-06-15T19:05:45.244Z" }, + { url = "https://files.pythonhosted.org/packages/d9/7e/82abc4240e0806846548559d70f0b1a6dfdca75c1b4f9fa62b504ae9b083/watchfiles-1.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29e7bc2eee15cbb339c68445959108803dc14ee0c7b4eea556400131a8de462b", size = 477950, upload-time = "2025-06-15T19:05:46.332Z" }, + { url = "https://files.pythonhosted.org/packages/25/0d/4d564798a49bf5482a4fa9416dea6b6c0733a3b5700cb8a5a503c4b15853/watchfiles-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9481174d3ed982e269c090f780122fb59cee6c3796f74efe74e70f7780ed94c", size = 451706, upload-time = "2025-06-15T19:05:47.459Z" }, + { url = "https://files.pythonhosted.org/packages/81/b5/5516cf46b033192d544102ea07c65b6f770f10ed1d0a6d388f5d3874f6e4/watchfiles-1.1.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:80f811146831c8c86ab17b640801c25dc0a88c630e855e2bef3568f30434d52b", size = 625814, upload-time = "2025-06-15T19:05:48.654Z" }, + { url = "https://files.pythonhosted.org/packages/0c/dd/7c1331f902f30669ac3e754680b6edb9a0dd06dea5438e61128111fadd2c/watchfiles-1.1.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:60022527e71d1d1fda67a33150ee42869042bce3d0fcc9cc49be009a9cded3fb", size = 622820, upload-time = "2025-06-15T19:05:50.088Z" }, + { url = "https://files.pythonhosted.org/packages/1b/14/36d7a8e27cd128d7b1009e7715a7c02f6c131be9d4ce1e5c3b73d0e342d8/watchfiles-1.1.0-cp313-cp313-win32.whl", hash = "sha256:32d6d4e583593cb8576e129879ea0991660b935177c0f93c6681359b3654bfa9", size = 279194, upload-time = "2025-06-15T19:05:51.186Z" }, + { url = "https://files.pythonhosted.org/packages/25/41/2dd88054b849aa546dbeef5696019c58f8e0774f4d1c42123273304cdb2e/watchfiles-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:f21af781a4a6fbad54f03c598ab620e3a77032c5878f3d780448421a6e1818c7", size = 292349, upload-time = "2025-06-15T19:05:52.201Z" }, + { url = "https://files.pythonhosted.org/packages/c8/cf/421d659de88285eb13941cf11a81f875c176f76a6d99342599be88e08d03/watchfiles-1.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:5366164391873ed76bfdf618818c82084c9db7fac82b64a20c44d335eec9ced5", size = 283836, upload-time = "2025-06-15T19:05:53.265Z" }, + { url = "https://files.pythonhosted.org/packages/45/10/6faf6858d527e3599cc50ec9fcae73590fbddc1420bd4fdccfebffeedbc6/watchfiles-1.1.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:17ab167cca6339c2b830b744eaf10803d2a5b6683be4d79d8475d88b4a8a4be1", size = 400343, upload-time = "2025-06-15T19:05:54.252Z" }, + { url = "https://files.pythonhosted.org/packages/03/20/5cb7d3966f5e8c718006d0e97dfe379a82f16fecd3caa7810f634412047a/watchfiles-1.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:328dbc9bff7205c215a7807da7c18dce37da7da718e798356212d22696404339", size = 392916, upload-time = "2025-06-15T19:05:55.264Z" }, + { url = "https://files.pythonhosted.org/packages/8c/07/d8f1176328fa9e9581b6f120b017e286d2a2d22ae3f554efd9515c8e1b49/watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7208ab6e009c627b7557ce55c465c98967e8caa8b11833531fdf95799372633", size = 449582, upload-time = "2025-06-15T19:05:56.317Z" }, + { url = "https://files.pythonhosted.org/packages/66/e8/80a14a453cf6038e81d072a86c05276692a1826471fef91df7537dba8b46/watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a8f6f72974a19efead54195bc9bed4d850fc047bb7aa971268fd9a8387c89011", size = 456752, upload-time = "2025-06-15T19:05:57.359Z" }, + { url = "https://files.pythonhosted.org/packages/5a/25/0853b3fe0e3c2f5af9ea60eb2e781eade939760239a72c2d38fc4cc335f6/watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d181ef50923c29cf0450c3cd47e2f0557b62218c50b2ab8ce2ecaa02bd97e670", size = 481436, upload-time = "2025-06-15T19:05:58.447Z" }, + { url = "https://files.pythonhosted.org/packages/fe/9e/4af0056c258b861fbb29dcb36258de1e2b857be4a9509e6298abcf31e5c9/watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:adb4167043d3a78280d5d05ce0ba22055c266cf8655ce942f2fb881262ff3cdf", size = 596016, upload-time = "2025-06-15T19:05:59.59Z" }, + { url = "https://files.pythonhosted.org/packages/c5/fa/95d604b58aa375e781daf350897aaaa089cff59d84147e9ccff2447c8294/watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5701dc474b041e2934a26d31d39f90fac8a3dee2322b39f7729867f932b1d4", size = 476727, upload-time = "2025-06-15T19:06:01.086Z" }, + { url = "https://files.pythonhosted.org/packages/65/95/fe479b2664f19be4cf5ceeb21be05afd491d95f142e72d26a42f41b7c4f8/watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b067915e3c3936966a8607f6fe5487df0c9c4afb85226613b520890049deea20", size = 451864, upload-time = "2025-06-15T19:06:02.144Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8a/3c4af14b93a15ce55901cd7a92e1a4701910f1768c78fb30f61d2b79785b/watchfiles-1.1.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:9c733cda03b6d636b4219625a4acb5c6ffb10803338e437fb614fef9516825ef", size = 625626, upload-time = "2025-06-15T19:06:03.578Z" }, + { url = "https://files.pythonhosted.org/packages/da/f5/cf6aa047d4d9e128f4b7cde615236a915673775ef171ff85971d698f3c2c/watchfiles-1.1.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:cc08ef8b90d78bfac66f0def80240b0197008e4852c9f285907377b2947ffdcb", size = 622744, upload-time = "2025-06-15T19:06:05.066Z" }, + { url = "https://files.pythonhosted.org/packages/2c/00/70f75c47f05dea6fd30df90f047765f6fc2d6eb8b5a3921379b0b04defa2/watchfiles-1.1.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9974d2f7dc561cce3bb88dfa8eb309dab64c729de85fba32e98d75cf24b66297", size = 402114, upload-time = "2025-06-15T19:06:06.186Z" }, + { url = "https://files.pythonhosted.org/packages/53/03/acd69c48db4a1ed1de26b349d94077cca2238ff98fd64393f3e97484cae6/watchfiles-1.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c68e9f1fcb4d43798ad8814c4c1b61547b014b667216cb754e606bfade587018", size = 393879, upload-time = "2025-06-15T19:06:07.369Z" }, + { url = "https://files.pythonhosted.org/packages/2f/c8/a9a2a6f9c8baa4eceae5887fecd421e1b7ce86802bcfc8b6a942e2add834/watchfiles-1.1.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95ab1594377effac17110e1352989bdd7bdfca9ff0e5eeccd8c69c5389b826d0", size = 450026, upload-time = "2025-06-15T19:06:08.476Z" }, + { url = "https://files.pythonhosted.org/packages/fe/51/d572260d98388e6e2b967425c985e07d47ee6f62e6455cefb46a6e06eda5/watchfiles-1.1.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fba9b62da882c1be1280a7584ec4515d0a6006a94d6e5819730ec2eab60ffe12", size = 457917, upload-time = "2025-06-15T19:06:09.988Z" }, + { url = "https://files.pythonhosted.org/packages/c6/2d/4258e52917bf9f12909b6ec314ff9636276f3542f9d3807d143f27309104/watchfiles-1.1.0-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3434e401f3ce0ed6b42569128b3d1e3af773d7ec18751b918b89cd49c14eaafb", size = 483602, upload-time = "2025-06-15T19:06:11.088Z" }, + { url = "https://files.pythonhosted.org/packages/84/99/bee17a5f341a4345fe7b7972a475809af9e528deba056f8963d61ea49f75/watchfiles-1.1.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa257a4d0d21fcbca5b5fcba9dca5a78011cb93c0323fb8855c6d2dfbc76eb77", size = 596758, upload-time = "2025-06-15T19:06:12.197Z" }, + { url = "https://files.pythonhosted.org/packages/40/76/e4bec1d59b25b89d2b0716b41b461ed655a9a53c60dc78ad5771fda5b3e6/watchfiles-1.1.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7fd1b3879a578a8ec2076c7961076df540b9af317123f84569f5a9ddee64ce92", size = 477601, upload-time = "2025-06-15T19:06:13.391Z" }, + { url = "https://files.pythonhosted.org/packages/1f/fa/a514292956f4a9ce3c567ec0c13cce427c158e9f272062685a8a727d08fc/watchfiles-1.1.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62cc7a30eeb0e20ecc5f4bd113cd69dcdb745a07c68c0370cea919f373f65d9e", size = 451936, upload-time = "2025-06-15T19:06:14.656Z" }, + { url = "https://files.pythonhosted.org/packages/32/5d/c3bf927ec3bbeb4566984eba8dd7a8eb69569400f5509904545576741f88/watchfiles-1.1.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:891c69e027748b4a73847335d208e374ce54ca3c335907d381fde4e41661b13b", size = 626243, upload-time = "2025-06-15T19:06:16.232Z" }, + { url = "https://files.pythonhosted.org/packages/e6/65/6e12c042f1a68c556802a84d54bb06d35577c81e29fba14019562479159c/watchfiles-1.1.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:12fe8eaffaf0faa7906895b4f8bb88264035b3f0243275e0bf24af0436b27259", size = 623073, upload-time = "2025-06-15T19:06:17.457Z" }, + { url = "https://files.pythonhosted.org/packages/89/ab/7f79d9bf57329e7cbb0a6fd4c7bd7d0cee1e4a8ef0041459f5409da3506c/watchfiles-1.1.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:bfe3c517c283e484843cb2e357dd57ba009cff351edf45fb455b5fbd1f45b15f", size = 400872, upload-time = "2025-06-15T19:06:18.57Z" }, + { url = "https://files.pythonhosted.org/packages/df/d5/3f7bf9912798e9e6c516094db6b8932df53b223660c781ee37607030b6d3/watchfiles-1.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a9ccbf1f129480ed3044f540c0fdbc4ee556f7175e5ab40fe077ff6baf286d4e", size = 392877, upload-time = "2025-06-15T19:06:19.55Z" }, + { url = "https://files.pythonhosted.org/packages/0d/c5/54ec7601a2798604e01c75294770dbee8150e81c6e471445d7601610b495/watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba0e3255b0396cac3cc7bbace76404dd72b5438bf0d8e7cefa2f79a7f3649caa", size = 449645, upload-time = "2025-06-15T19:06:20.66Z" }, + { url = "https://files.pythonhosted.org/packages/0a/04/c2f44afc3b2fce21ca0b7802cbd37ed90a29874f96069ed30a36dfe57c2b/watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4281cd9fce9fc0a9dbf0fc1217f39bf9cf2b4d315d9626ef1d4e87b84699e7e8", size = 457424, upload-time = "2025-06-15T19:06:21.712Z" }, + { url = "https://files.pythonhosted.org/packages/9f/b0/eec32cb6c14d248095261a04f290636da3df3119d4040ef91a4a50b29fa5/watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6d2404af8db1329f9a3c9b79ff63e0ae7131986446901582067d9304ae8aaf7f", size = 481584, upload-time = "2025-06-15T19:06:22.777Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e2/ca4bb71c68a937d7145aa25709e4f5d68eb7698a25ce266e84b55d591bbd/watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e78b6ed8165996013165eeabd875c5dfc19d41b54f94b40e9fff0eb3193e5e8e", size = 596675, upload-time = "2025-06-15T19:06:24.226Z" }, + { url = "https://files.pythonhosted.org/packages/a1/dd/b0e4b7fb5acf783816bc950180a6cd7c6c1d2cf7e9372c0ea634e722712b/watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:249590eb75ccc117f488e2fabd1bfa33c580e24b96f00658ad88e38844a040bb", size = 477363, upload-time = "2025-06-15T19:06:25.42Z" }, + { url = "https://files.pythonhosted.org/packages/69/c4/088825b75489cb5b6a761a4542645718893d395d8c530b38734f19da44d2/watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d05686b5487cfa2e2c28ff1aa370ea3e6c5accfe6435944ddea1e10d93872147", size = 452240, upload-time = "2025-06-15T19:06:26.552Z" }, + { url = "https://files.pythonhosted.org/packages/10/8c/22b074814970eeef43b7c44df98c3e9667c1f7bf5b83e0ff0201b0bd43f9/watchfiles-1.1.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:d0e10e6f8f6dc5762adee7dece33b722282e1f59aa6a55da5d493a97282fedd8", size = 625607, upload-time = "2025-06-15T19:06:27.606Z" }, + { url = "https://files.pythonhosted.org/packages/32/fa/a4f5c2046385492b2273213ef815bf71a0d4c1943b784fb904e184e30201/watchfiles-1.1.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:af06c863f152005c7592df1d6a7009c836a247c9d8adb78fef8575a5a98699db", size = 623315, upload-time = "2025-06-15T19:06:29.076Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.2.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5", size = 101301, upload-time = "2024-01-06T02:10:57.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166, upload-time = "2024-01-06T02:10:55.763Z" }, +] + +[[package]] +name = "websockets" +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/6b/4545a0d843594f5d0771e86463606a3988b5a09ca5123136f8a76580dd63/websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3", size = 175437, upload-time = "2025-03-05T20:02:16.706Z" }, + { url = "https://files.pythonhosted.org/packages/f4/71/809a0f5f6a06522af902e0f2ea2757f71ead94610010cf570ab5c98e99ed/websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665", size = 173096, upload-time = "2025-03-05T20:02:18.832Z" }, + { url = "https://files.pythonhosted.org/packages/3d/69/1a681dd6f02180916f116894181eab8b2e25b31e484c5d0eae637ec01f7c/websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2", size = 173332, upload-time = "2025-03-05T20:02:20.187Z" }, + { url = "https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215", size = 183152, upload-time = "2025-03-05T20:02:22.286Z" }, + { url = "https://files.pythonhosted.org/packages/74/45/c205c8480eafd114b428284840da0b1be9ffd0e4f87338dc95dc6ff961a1/websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5", size = 182096, upload-time = "2025-03-05T20:02:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65", size = 182523, upload-time = "2025-03-05T20:02:25.669Z" }, + { url = "https://files.pythonhosted.org/packages/ec/6d/0267396610add5bc0d0d3e77f546d4cd287200804fe02323797de77dbce9/websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe", size = 182790, upload-time = "2025-03-05T20:02:26.99Z" }, + { url = "https://files.pythonhosted.org/packages/02/05/c68c5adbf679cf610ae2f74a9b871ae84564462955d991178f95a1ddb7dd/websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4", size = 182165, upload-time = "2025-03-05T20:02:30.291Z" }, + { url = "https://files.pythonhosted.org/packages/29/93/bb672df7b2f5faac89761cb5fa34f5cec45a4026c383a4b5761c6cea5c16/websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597", size = 182160, upload-time = "2025-03-05T20:02:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/ff/83/de1f7709376dc3ca9b7eeb4b9a07b4526b14876b6d372a4dc62312bebee0/websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9", size = 176395, upload-time = "2025-03-05T20:02:33.017Z" }, + { url = "https://files.pythonhosted.org/packages/7d/71/abf2ebc3bbfa40f391ce1428c7168fb20582d0ff57019b69ea20fa698043/websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7", size = 176841, upload-time = "2025-03-05T20:02:34.498Z" }, + { url = "https://files.pythonhosted.org/packages/cb/9f/51f0cf64471a9d2b4d0fc6c534f323b664e7095640c34562f5182e5a7195/websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931", size = 175440, upload-time = "2025-03-05T20:02:36.695Z" }, + { url = "https://files.pythonhosted.org/packages/8a/05/aa116ec9943c718905997412c5989f7ed671bc0188ee2ba89520e8765d7b/websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675", size = 173098, upload-time = "2025-03-05T20:02:37.985Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0b/33cef55ff24f2d92924923c99926dcce78e7bd922d649467f0eda8368923/websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151", size = 173329, upload-time = "2025-03-05T20:02:39.298Z" }, + { url = "https://files.pythonhosted.org/packages/31/1d/063b25dcc01faa8fada1469bdf769de3768b7044eac9d41f734fd7b6ad6d/websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22", size = 183111, upload-time = "2025-03-05T20:02:40.595Z" }, + { url = "https://files.pythonhosted.org/packages/93/53/9a87ee494a51bf63e4ec9241c1ccc4f7c2f45fff85d5bde2ff74fcb68b9e/websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f", size = 182054, upload-time = "2025-03-05T20:02:41.926Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b2/83a6ddf56cdcbad4e3d841fcc55d6ba7d19aeb89c50f24dd7e859ec0805f/websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8", size = 182496, upload-time = "2025-03-05T20:02:43.304Z" }, + { url = "https://files.pythonhosted.org/packages/98/41/e7038944ed0abf34c45aa4635ba28136f06052e08fc2168520bb8b25149f/websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375", size = 182829, upload-time = "2025-03-05T20:02:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/e0/17/de15b6158680c7623c6ef0db361da965ab25d813ae54fcfeae2e5b9ef910/websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d", size = 182217, upload-time = "2025-03-05T20:02:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/33/2b/1f168cb6041853eef0362fb9554c3824367c5560cbdaad89ac40f8c2edfc/websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4", size = 182195, upload-time = "2025-03-05T20:02:51.561Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/20b6cdf273913d0ad05a6a14aed4b9a85591c18a987a3d47f20fa13dcc47/websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa", size = 176393, upload-time = "2025-03-05T20:02:53.814Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561", size = 176837, upload-time = "2025-03-05T20:02:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, +] + +[[package]] +name = "werkzeug" +version = "3.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" }, +] + +[[package]] +name = "widgetsnbextension" +version = "4.0.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/53/2e0253c5efd69c9656b1843892052a31c36d37ad42812b5da45c62191f7e/widgetsnbextension-4.0.14.tar.gz", hash = "sha256:a3629b04e3edb893212df862038c7232f62973373869db5084aed739b437b5af", size = 1097428, upload-time = "2025-04-10T13:01:25.628Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/51/5447876806d1088a0f8f71e16542bf350918128d0a69437df26047c8e46f/widgetsnbextension-4.0.14-py3-none-any.whl", hash = "sha256:4875a9eaf72fbf5079dc372a51a9f268fc38d46f767cbf85c43a36da5cb9b575", size = 2196503, upload-time = "2025-04-10T13:01:23.086Z" }, +] + +[[package]] +name = "wrapt" +version = "1.17.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", size = 53998, upload-time = "2025-08-12T05:51:47.138Z" }, + { url = "https://files.pythonhosted.org/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", size = 39020, upload-time = "2025-08-12T05:51:35.906Z" }, + { url = "https://files.pythonhosted.org/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", size = 39098, upload-time = "2025-08-12T05:51:57.474Z" }, + { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" }, + { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" }, + { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" }, + { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", size = 36705, upload-time = "2025-08-12T05:53:07.123Z" }, + { url = "https://files.pythonhosted.org/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", size = 38877, upload-time = "2025-08-12T05:53:05.436Z" }, + { url = "https://files.pythonhosted.org/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", size = 36885, upload-time = "2025-08-12T05:52:54.367Z" }, + { url = "https://files.pythonhosted.org/packages/fc/f6/759ece88472157acb55fc195e5b116e06730f1b651b5b314c66291729193/wrapt-1.17.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a47681378a0439215912ef542c45a783484d4dd82bac412b71e59cf9c0e1cea0", size = 54003, upload-time = "2025-08-12T05:51:48.627Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a9/49940b9dc6d47027dc850c116d79b4155f15c08547d04db0f07121499347/wrapt-1.17.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a30837587c6ee3cd1a4d1c2ec5d24e77984d44e2f34547e2323ddb4e22eb77", size = 39025, upload-time = "2025-08-12T05:51:37.156Z" }, + { url = "https://files.pythonhosted.org/packages/45/35/6a08de0f2c96dcdd7fe464d7420ddb9a7655a6561150e5fc4da9356aeaab/wrapt-1.17.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:16ecf15d6af39246fe33e507105d67e4b81d8f8d2c6598ff7e3ca1b8a37213f7", size = 39108, upload-time = "2025-08-12T05:51:58.425Z" }, + { url = "https://files.pythonhosted.org/packages/0c/37/6faf15cfa41bf1f3dba80cd3f5ccc6622dfccb660ab26ed79f0178c7497f/wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", size = 88072, upload-time = "2025-08-12T05:52:37.53Z" }, + { url = "https://files.pythonhosted.org/packages/78/f2/efe19ada4a38e4e15b6dff39c3e3f3f73f5decf901f66e6f72fe79623a06/wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", size = 88214, upload-time = "2025-08-12T05:52:15.886Z" }, + { url = "https://files.pythonhosted.org/packages/40/90/ca86701e9de1622b16e09689fc24b76f69b06bb0150990f6f4e8b0eeb576/wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", size = 87105, upload-time = "2025-08-12T05:52:17.914Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e0/d10bd257c9a3e15cbf5523025252cc14d77468e8ed644aafb2d6f54cb95d/wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", size = 87766, upload-time = "2025-08-12T05:52:39.243Z" }, + { url = "https://files.pythonhosted.org/packages/e8/cf/7d848740203c7b4b27eb55dbfede11aca974a51c3d894f6cc4b865f42f58/wrapt-1.17.3-cp313-cp313-win32.whl", hash = "sha256:53e5e39ff71b3fc484df8a522c933ea2b7cdd0d5d15ae82e5b23fde87d44cbd8", size = 36711, upload-time = "2025-08-12T05:53:10.074Z" }, + { url = "https://files.pythonhosted.org/packages/57/54/35a84d0a4d23ea675994104e667ceff49227ce473ba6a59ba2c84f250b74/wrapt-1.17.3-cp313-cp313-win_amd64.whl", hash = "sha256:1f0b2f40cf341ee8cc1a97d51ff50dddb9fcc73241b9143ec74b30fc4f44f6cb", size = 38885, upload-time = "2025-08-12T05:53:08.695Z" }, + { url = "https://files.pythonhosted.org/packages/01/77/66e54407c59d7b02a3c4e0af3783168fff8e5d61def52cda8728439d86bc/wrapt-1.17.3-cp313-cp313-win_arm64.whl", hash = "sha256:7425ac3c54430f5fc5e7b6f41d41e704db073309acfc09305816bc6a0b26bb16", size = 36896, upload-time = "2025-08-12T05:52:55.34Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/cd864b2a14f20d14f4c496fab97802001560f9f41554eef6df201cd7f76c/wrapt-1.17.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cf30f6e3c077c8e6a9a7809c94551203c8843e74ba0c960f4a98cd80d4665d39", size = 54132, upload-time = "2025-08-12T05:51:49.864Z" }, + { url = "https://files.pythonhosted.org/packages/d5/46/d011725b0c89e853dc44cceb738a307cde5d240d023d6d40a82d1b4e1182/wrapt-1.17.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e228514a06843cae89621384cfe3a80418f3c04aadf8a3b14e46a7be704e4235", size = 39091, upload-time = "2025-08-12T05:51:38.935Z" }, + { url = "https://files.pythonhosted.org/packages/2e/9e/3ad852d77c35aae7ddebdbc3b6d35ec8013af7d7dddad0ad911f3d891dae/wrapt-1.17.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5ea5eb3c0c071862997d6f3e02af1d055f381b1d25b286b9d6644b79db77657c", size = 39172, upload-time = "2025-08-12T05:51:59.365Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f7/c983d2762bcce2326c317c26a6a1e7016f7eb039c27cdf5c4e30f4160f31/wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", size = 87163, upload-time = "2025-08-12T05:52:40.965Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0f/f673f75d489c7f22d17fe0193e84b41540d962f75fce579cf6873167c29b/wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", size = 87963, upload-time = "2025-08-12T05:52:20.326Z" }, + { url = "https://files.pythonhosted.org/packages/df/61/515ad6caca68995da2fac7a6af97faab8f78ebe3bf4f761e1b77efbc47b5/wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", size = 86945, upload-time = "2025-08-12T05:52:21.581Z" }, + { url = "https://files.pythonhosted.org/packages/d3/bd/4e70162ce398462a467bc09e768bee112f1412e563620adc353de9055d33/wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", size = 86857, upload-time = "2025-08-12T05:52:43.043Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b8/da8560695e9284810b8d3df8a19396a6e40e7518059584a1a394a2b35e0a/wrapt-1.17.3-cp314-cp314-win32.whl", hash = "sha256:fbd3c8319de8e1dc79d346929cd71d523622da527cca14e0c1d257e31c2b8b10", size = 37178, upload-time = "2025-08-12T05:53:12.605Z" }, + { url = "https://files.pythonhosted.org/packages/db/c8/b71eeb192c440d67a5a0449aaee2310a1a1e8eca41676046f99ed2487e9f/wrapt-1.17.3-cp314-cp314-win_amd64.whl", hash = "sha256:e1a4120ae5705f673727d3253de3ed0e016f7cd78dc463db1b31e2463e1f3cf6", size = 39310, upload-time = "2025-08-12T05:53:11.106Z" }, + { url = "https://files.pythonhosted.org/packages/45/20/2cda20fd4865fa40f86f6c46ed37a2a8356a7a2fde0773269311f2af56c7/wrapt-1.17.3-cp314-cp314-win_arm64.whl", hash = "sha256:507553480670cab08a800b9463bdb881b2edeed77dc677b0a5915e6106e91a58", size = 37266, upload-time = "2025-08-12T05:52:56.531Z" }, + { url = "https://files.pythonhosted.org/packages/77/ed/dd5cf21aec36c80443c6f900449260b80e2a65cf963668eaef3b9accce36/wrapt-1.17.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ed7c635ae45cfbc1a7371f708727bf74690daedc49b4dba310590ca0bd28aa8a", size = 56544, upload-time = "2025-08-12T05:51:51.109Z" }, + { url = "https://files.pythonhosted.org/packages/8d/96/450c651cc753877ad100c7949ab4d2e2ecc4d97157e00fa8f45df682456a/wrapt-1.17.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:249f88ed15503f6492a71f01442abddd73856a0032ae860de6d75ca62eed8067", size = 40283, upload-time = "2025-08-12T05:51:39.912Z" }, + { url = "https://files.pythonhosted.org/packages/d1/86/2fcad95994d9b572db57632acb6f900695a648c3e063f2cd344b3f5c5a37/wrapt-1.17.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a03a38adec8066d5a37bea22f2ba6bbf39fcdefbe2d91419ab864c3fb515454", size = 40366, upload-time = "2025-08-12T05:52:00.693Z" }, + { url = "https://files.pythonhosted.org/packages/64/0e/f4472f2fdde2d4617975144311f8800ef73677a159be7fe61fa50997d6c0/wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", size = 108571, upload-time = "2025-08-12T05:52:44.521Z" }, + { url = "https://files.pythonhosted.org/packages/cc/01/9b85a99996b0a97c8a17484684f206cbb6ba73c1ce6890ac668bcf3838fb/wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", size = 113094, upload-time = "2025-08-12T05:52:22.618Z" }, + { url = "https://files.pythonhosted.org/packages/25/02/78926c1efddcc7b3aa0bc3d6b33a822f7d898059f7cd9ace8c8318e559ef/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", size = 110659, upload-time = "2025-08-12T05:52:24.057Z" }, + { url = "https://files.pythonhosted.org/packages/dc/ee/c414501ad518ac3e6fe184753632fe5e5ecacdcf0effc23f31c1e4f7bfcf/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", size = 106946, upload-time = "2025-08-12T05:52:45.976Z" }, + { url = "https://files.pythonhosted.org/packages/be/44/a1bd64b723d13bb151d6cc91b986146a1952385e0392a78567e12149c7b4/wrapt-1.17.3-cp314-cp314t-win32.whl", hash = "sha256:41b1d2bc74c2cac6f9074df52b2efbef2b30bdfe5f40cb78f8ca22963bc62977", size = 38717, upload-time = "2025-08-12T05:53:15.214Z" }, + { url = "https://files.pythonhosted.org/packages/79/d9/7cfd5a312760ac4dd8bf0184a6ee9e43c33e47f3dadc303032ce012b8fa3/wrapt-1.17.3-cp314-cp314t-win_amd64.whl", hash = "sha256:73d496de46cd2cdbdbcce4ae4bcdb4afb6a11234a1df9c085249d55166b95116", size = 41334, upload-time = "2025-08-12T05:53:14.178Z" }, + { url = "https://files.pythonhosted.org/packages/46/78/10ad9781128ed2f99dbc474f43283b13fea8ba58723e98844367531c18e9/wrapt-1.17.3-cp314-cp314t-win_arm64.whl", hash = "sha256:f38e60678850c42461d4202739f9bf1e3a737c7ad283638251e79cc49effb6b6", size = 38471, upload-time = "2025-08-12T05:52:57.784Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, +] + +[[package]] +name = "yarl" +version = "1.20.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/fb/efaa23fa4e45537b827620f04cf8f3cd658b76642205162e072703a5b963/yarl-1.20.1.tar.gz", hash = "sha256:d017a4997ee50c91fd5466cef416231bb82177b93b029906cefc542ce14c35ac", size = 186428, upload-time = "2025-06-10T00:46:09.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/9a/cb7fad7d73c69f296eda6815e4a2c7ed53fc70c2f136479a91c8e5fbdb6d/yarl-1.20.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdcc4cd244e58593a4379fe60fdee5ac0331f8eb70320a24d591a3be197b94a9", size = 133667, upload-time = "2025-06-10T00:43:44.369Z" }, + { url = "https://files.pythonhosted.org/packages/67/38/688577a1cb1e656e3971fb66a3492501c5a5df56d99722e57c98249e5b8a/yarl-1.20.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b29a2c385a5f5b9c7d9347e5812b6f7ab267193c62d282a540b4fc528c8a9d2a", size = 91025, upload-time = "2025-06-10T00:43:46.295Z" }, + { url = "https://files.pythonhosted.org/packages/50/ec/72991ae51febeb11a42813fc259f0d4c8e0507f2b74b5514618d8b640365/yarl-1.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1112ae8154186dfe2de4732197f59c05a83dc814849a5ced892b708033f40dc2", size = 89709, upload-time = "2025-06-10T00:43:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/99/da/4d798025490e89426e9f976702e5f9482005c548c579bdae792a4c37769e/yarl-1.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90bbd29c4fe234233f7fa2b9b121fb63c321830e5d05b45153a2ca68f7d310ee", size = 352287, upload-time = "2025-06-10T00:43:49.924Z" }, + { url = "https://files.pythonhosted.org/packages/1a/26/54a15c6a567aac1c61b18aa0f4b8aa2e285a52d547d1be8bf48abe2b3991/yarl-1.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:680e19c7ce3710ac4cd964e90dad99bf9b5029372ba0c7cbfcd55e54d90ea819", size = 345429, upload-time = "2025-06-10T00:43:51.7Z" }, + { url = "https://files.pythonhosted.org/packages/d6/95/9dcf2386cb875b234353b93ec43e40219e14900e046bf6ac118f94b1e353/yarl-1.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a979218c1fdb4246a05efc2cc23859d47c89af463a90b99b7c56094daf25a16", size = 365429, upload-time = "2025-06-10T00:43:53.494Z" }, + { url = "https://files.pythonhosted.org/packages/91/b2/33a8750f6a4bc224242a635f5f2cff6d6ad5ba651f6edcccf721992c21a0/yarl-1.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255b468adf57b4a7b65d8aad5b5138dce6a0752c139965711bdcb81bc370e1b6", size = 363862, upload-time = "2025-06-10T00:43:55.766Z" }, + { url = "https://files.pythonhosted.org/packages/98/28/3ab7acc5b51f4434b181b0cee8f1f4b77a65919700a355fb3617f9488874/yarl-1.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a97d67108e79cfe22e2b430d80d7571ae57d19f17cda8bb967057ca8a7bf5bfd", size = 355616, upload-time = "2025-06-10T00:43:58.056Z" }, + { url = "https://files.pythonhosted.org/packages/36/a3/f666894aa947a371724ec7cd2e5daa78ee8a777b21509b4252dd7bd15e29/yarl-1.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8570d998db4ddbfb9a590b185a0a33dbf8aafb831d07a5257b4ec9948df9cb0a", size = 339954, upload-time = "2025-06-10T00:43:59.773Z" }, + { url = "https://files.pythonhosted.org/packages/f1/81/5f466427e09773c04219d3450d7a1256138a010b6c9f0af2d48565e9ad13/yarl-1.20.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97c75596019baae7c71ccf1d8cc4738bc08134060d0adfcbe5642f778d1dca38", size = 365575, upload-time = "2025-06-10T00:44:02.051Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e3/e4b0ad8403e97e6c9972dd587388940a032f030ebec196ab81a3b8e94d31/yarl-1.20.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1c48912653e63aef91ff988c5432832692ac5a1d8f0fb8a33091520b5bbe19ef", size = 365061, upload-time = "2025-06-10T00:44:04.196Z" }, + { url = "https://files.pythonhosted.org/packages/ac/99/b8a142e79eb86c926f9f06452eb13ecb1bb5713bd01dc0038faf5452e544/yarl-1.20.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4c3ae28f3ae1563c50f3d37f064ddb1511ecc1d5584e88c6b7c63cf7702a6d5f", size = 364142, upload-time = "2025-06-10T00:44:06.527Z" }, + { url = "https://files.pythonhosted.org/packages/34/f2/08ed34a4a506d82a1a3e5bab99ccd930a040f9b6449e9fd050320e45845c/yarl-1.20.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e9642f27036283550f5f57dc6156c51084b458570b9d0d96100c8bebb186a8", size = 381894, upload-time = "2025-06-10T00:44:08.379Z" }, + { url = "https://files.pythonhosted.org/packages/92/f8/9a3fbf0968eac704f681726eff595dce9b49c8a25cd92bf83df209668285/yarl-1.20.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2c26b0c49220d5799f7b22c6838409ee9bc58ee5c95361a4d7831f03cc225b5a", size = 383378, upload-time = "2025-06-10T00:44:10.51Z" }, + { url = "https://files.pythonhosted.org/packages/af/85/9363f77bdfa1e4d690957cd39d192c4cacd1c58965df0470a4905253b54f/yarl-1.20.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564ab3d517e3d01c408c67f2e5247aad4019dcf1969982aba3974b4093279004", size = 374069, upload-time = "2025-06-10T00:44:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/35/99/9918c8739ba271dcd935400cff8b32e3cd319eaf02fcd023d5dcd487a7c8/yarl-1.20.1-cp312-cp312-win32.whl", hash = "sha256:daea0d313868da1cf2fac6b2d3a25c6e3a9e879483244be38c8e6a41f1d876a5", size = 81249, upload-time = "2025-06-10T00:44:14.731Z" }, + { url = "https://files.pythonhosted.org/packages/eb/83/5d9092950565481b413b31a23e75dd3418ff0a277d6e0abf3729d4d1ce25/yarl-1.20.1-cp312-cp312-win_amd64.whl", hash = "sha256:48ea7d7f9be0487339828a4de0360d7ce0efc06524a48e1810f945c45b813698", size = 86710, upload-time = "2025-06-10T00:44:16.716Z" }, + { url = "https://files.pythonhosted.org/packages/8a/e1/2411b6d7f769a07687acee88a062af5833cf1966b7266f3d8dfb3d3dc7d3/yarl-1.20.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:0b5ff0fbb7c9f1b1b5ab53330acbfc5247893069e7716840c8e7d5bb7355038a", size = 131811, upload-time = "2025-06-10T00:44:18.933Z" }, + { url = "https://files.pythonhosted.org/packages/b2/27/584394e1cb76fb771371770eccad35de400e7b434ce3142c2dd27392c968/yarl-1.20.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:14f326acd845c2b2e2eb38fb1346c94f7f3b01a4f5c788f8144f9b630bfff9a3", size = 90078, upload-time = "2025-06-10T00:44:20.635Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9a/3246ae92d4049099f52d9b0fe3486e3b500e29b7ea872d0f152966fc209d/yarl-1.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f60e4ad5db23f0b96e49c018596707c3ae89f5d0bd97f0ad3684bcbad899f1e7", size = 88748, upload-time = "2025-06-10T00:44:22.34Z" }, + { url = "https://files.pythonhosted.org/packages/a3/25/35afe384e31115a1a801fbcf84012d7a066d89035befae7c5d4284df1e03/yarl-1.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:49bdd1b8e00ce57e68ba51916e4bb04461746e794e7c4d4bbc42ba2f18297691", size = 349595, upload-time = "2025-06-10T00:44:24.314Z" }, + { url = "https://files.pythonhosted.org/packages/28/2d/8aca6cb2cabc8f12efcb82749b9cefecbccfc7b0384e56cd71058ccee433/yarl-1.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:66252d780b45189975abfed839616e8fd2dbacbdc262105ad7742c6ae58f3e31", size = 342616, upload-time = "2025-06-10T00:44:26.167Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/1312633d16b31acf0098d30440ca855e3492d66623dafb8e25b03d00c3da/yarl-1.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59174e7332f5d153d8f7452a102b103e2e74035ad085f404df2e40e663a22b28", size = 361324, upload-time = "2025-06-10T00:44:27.915Z" }, + { url = "https://files.pythonhosted.org/packages/bc/a0/688cc99463f12f7669eec7c8acc71ef56a1521b99eab7cd3abb75af887b0/yarl-1.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e3968ec7d92a0c0f9ac34d5ecfd03869ec0cab0697c91a45db3fbbd95fe1b653", size = 359676, upload-time = "2025-06-10T00:44:30.041Z" }, + { url = "https://files.pythonhosted.org/packages/af/44/46407d7f7a56e9a85a4c207724c9f2c545c060380718eea9088f222ba697/yarl-1.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1a4fbb50e14396ba3d375f68bfe02215d8e7bc3ec49da8341fe3157f59d2ff5", size = 352614, upload-time = "2025-06-10T00:44:32.171Z" }, + { url = "https://files.pythonhosted.org/packages/b1/91/31163295e82b8d5485d31d9cf7754d973d41915cadce070491778d9c9825/yarl-1.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11a62c839c3a8eac2410e951301309426f368388ff2f33799052787035793b02", size = 336766, upload-time = "2025-06-10T00:44:34.494Z" }, + { url = "https://files.pythonhosted.org/packages/b4/8e/c41a5bc482121f51c083c4c2bcd16b9e01e1cf8729e380273a952513a21f/yarl-1.20.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:041eaa14f73ff5a8986b4388ac6bb43a77f2ea09bf1913df7a35d4646db69e53", size = 364615, upload-time = "2025-06-10T00:44:36.856Z" }, + { url = "https://files.pythonhosted.org/packages/e3/5b/61a3b054238d33d70ea06ebba7e58597891b71c699e247df35cc984ab393/yarl-1.20.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:377fae2fef158e8fd9d60b4c8751387b8d1fb121d3d0b8e9b0be07d1b41e83dc", size = 360982, upload-time = "2025-06-10T00:44:39.141Z" }, + { url = "https://files.pythonhosted.org/packages/df/a3/6a72fb83f8d478cb201d14927bc8040af901811a88e0ff2da7842dd0ed19/yarl-1.20.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1c92f4390e407513f619d49319023664643d3339bd5e5a56a3bebe01bc67ec04", size = 369792, upload-time = "2025-06-10T00:44:40.934Z" }, + { url = "https://files.pythonhosted.org/packages/7c/af/4cc3c36dfc7c077f8dedb561eb21f69e1e9f2456b91b593882b0b18c19dc/yarl-1.20.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d25ddcf954df1754ab0f86bb696af765c5bfaba39b74095f27eececa049ef9a4", size = 382049, upload-time = "2025-06-10T00:44:42.854Z" }, + { url = "https://files.pythonhosted.org/packages/19/3a/e54e2c4752160115183a66dc9ee75a153f81f3ab2ba4bf79c3c53b33de34/yarl-1.20.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:909313577e9619dcff8c31a0ea2aa0a2a828341d92673015456b3ae492e7317b", size = 384774, upload-time = "2025-06-10T00:44:45.275Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/200ae86dabfca89060ec6447649f219b4cbd94531e425e50d57e5f5ac330/yarl-1.20.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:793fd0580cb9664548c6b83c63b43c477212c0260891ddf86809e1c06c8b08f1", size = 374252, upload-time = "2025-06-10T00:44:47.31Z" }, + { url = "https://files.pythonhosted.org/packages/83/75/11ee332f2f516b3d094e89448da73d557687f7d137d5a0f48c40ff211487/yarl-1.20.1-cp313-cp313-win32.whl", hash = "sha256:468f6e40285de5a5b3c44981ca3a319a4b208ccc07d526b20b12aeedcfa654b7", size = 81198, upload-time = "2025-06-10T00:44:49.164Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ba/39b1ecbf51620b40ab402b0fc817f0ff750f6d92712b44689c2c215be89d/yarl-1.20.1-cp313-cp313-win_amd64.whl", hash = "sha256:495b4ef2fea40596bfc0affe3837411d6aa3371abcf31aac0ccc4bdd64d4ef5c", size = 86346, upload-time = "2025-06-10T00:44:51.182Z" }, + { url = "https://files.pythonhosted.org/packages/43/c7/669c52519dca4c95153c8ad96dd123c79f354a376346b198f438e56ffeb4/yarl-1.20.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:f60233b98423aab21d249a30eb27c389c14929f47be8430efa7dbd91493a729d", size = 138826, upload-time = "2025-06-10T00:44:52.883Z" }, + { url = "https://files.pythonhosted.org/packages/6a/42/fc0053719b44f6ad04a75d7f05e0e9674d45ef62f2d9ad2c1163e5c05827/yarl-1.20.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6f3eff4cc3f03d650d8755c6eefc844edde99d641d0dcf4da3ab27141a5f8ddf", size = 93217, upload-time = "2025-06-10T00:44:54.658Z" }, + { url = "https://files.pythonhosted.org/packages/4f/7f/fa59c4c27e2a076bba0d959386e26eba77eb52ea4a0aac48e3515c186b4c/yarl-1.20.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:69ff8439d8ba832d6bed88af2c2b3445977eba9a4588b787b32945871c2444e3", size = 92700, upload-time = "2025-06-10T00:44:56.784Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d4/062b2f48e7c93481e88eff97a6312dca15ea200e959f23e96d8ab898c5b8/yarl-1.20.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cf34efa60eb81dd2645a2e13e00bb98b76c35ab5061a3989c7a70f78c85006d", size = 347644, upload-time = "2025-06-10T00:44:59.071Z" }, + { url = "https://files.pythonhosted.org/packages/89/47/78b7f40d13c8f62b499cc702fdf69e090455518ae544c00a3bf4afc9fc77/yarl-1.20.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8e0fe9364ad0fddab2688ce72cb7a8e61ea42eff3c7caeeb83874a5d479c896c", size = 323452, upload-time = "2025-06-10T00:45:01.605Z" }, + { url = "https://files.pythonhosted.org/packages/eb/2b/490d3b2dc66f52987d4ee0d3090a147ea67732ce6b4d61e362c1846d0d32/yarl-1.20.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f64fbf81878ba914562c672024089e3401974a39767747691c65080a67b18c1", size = 346378, upload-time = "2025-06-10T00:45:03.946Z" }, + { url = "https://files.pythonhosted.org/packages/66/ad/775da9c8a94ce925d1537f939a4f17d782efef1f973039d821cbe4bcc211/yarl-1.20.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6342d643bf9a1de97e512e45e4b9560a043347e779a173250824f8b254bd5ce", size = 353261, upload-time = "2025-06-10T00:45:05.992Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/0ed0922b47a4f5c6eb9065d5ff1e459747226ddce5c6a4c111e728c9f701/yarl-1.20.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56dac5f452ed25eef0f6e3c6a066c6ab68971d96a9fb441791cad0efba6140d3", size = 335987, upload-time = "2025-06-10T00:45:08.227Z" }, + { url = "https://files.pythonhosted.org/packages/3e/49/bc728a7fe7d0e9336e2b78f0958a2d6b288ba89f25a1762407a222bf53c3/yarl-1.20.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7d7f497126d65e2cad8dc5f97d34c27b19199b6414a40cb36b52f41b79014be", size = 329361, upload-time = "2025-06-10T00:45:10.11Z" }, + { url = "https://files.pythonhosted.org/packages/93/8f/b811b9d1f617c83c907e7082a76e2b92b655400e61730cd61a1f67178393/yarl-1.20.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:67e708dfb8e78d8a19169818eeb5c7a80717562de9051bf2413aca8e3696bf16", size = 346460, upload-time = "2025-06-10T00:45:12.055Z" }, + { url = "https://files.pythonhosted.org/packages/70/fd/af94f04f275f95da2c3b8b5e1d49e3e79f1ed8b6ceb0f1664cbd902773ff/yarl-1.20.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:595c07bc79af2494365cc96ddeb772f76272364ef7c80fb892ef9d0649586513", size = 334486, upload-time = "2025-06-10T00:45:13.995Z" }, + { url = "https://files.pythonhosted.org/packages/84/65/04c62e82704e7dd0a9b3f61dbaa8447f8507655fd16c51da0637b39b2910/yarl-1.20.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7bdd2f80f4a7df852ab9ab49484a4dee8030023aa536df41f2d922fd57bf023f", size = 342219, upload-time = "2025-06-10T00:45:16.479Z" }, + { url = "https://files.pythonhosted.org/packages/91/95/459ca62eb958381b342d94ab9a4b6aec1ddec1f7057c487e926f03c06d30/yarl-1.20.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c03bfebc4ae8d862f853a9757199677ab74ec25424d0ebd68a0027e9c639a390", size = 350693, upload-time = "2025-06-10T00:45:18.399Z" }, + { url = "https://files.pythonhosted.org/packages/a6/00/d393e82dd955ad20617abc546a8f1aee40534d599ff555ea053d0ec9bf03/yarl-1.20.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:344d1103e9c1523f32a5ed704d576172d2cabed3122ea90b1d4e11fe17c66458", size = 355803, upload-time = "2025-06-10T00:45:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ed/c5fb04869b99b717985e244fd93029c7a8e8febdfcffa06093e32d7d44e7/yarl-1.20.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:88cab98aa4e13e1ade8c141daeedd300a4603b7132819c484841bb7af3edce9e", size = 341709, upload-time = "2025-06-10T00:45:23.221Z" }, + { url = "https://files.pythonhosted.org/packages/24/fd/725b8e73ac2a50e78a4534ac43c6addf5c1c2d65380dd48a9169cc6739a9/yarl-1.20.1-cp313-cp313t-win32.whl", hash = "sha256:b121ff6a7cbd4abc28985b6028235491941b9fe8fe226e6fdc539c977ea1739d", size = 86591, upload-time = "2025-06-10T00:45:25.793Z" }, + { url = "https://files.pythonhosted.org/packages/94/c3/b2e9f38bc3e11191981d57ea08cab2166e74ea770024a646617c9cddd9f6/yarl-1.20.1-cp313-cp313t-win_amd64.whl", hash = "sha256:541d050a355bbbc27e55d906bc91cb6fe42f96c01413dd0f4ed5a5240513874f", size = 93003, upload-time = "2025-06-10T00:45:27.752Z" }, + { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" }, +] + +[[package]] +name = "zipp" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, +] From 2acaf29e67823f48de70949bc65a517cd357623c Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Fri, 22 Aug 2025 14:07:31 -0700 Subject: [PATCH 231/634] [core] Minor bug fixes in scheduler path (#55806) Spotted a couple places in the scheduler path that didn't look right and did the opposite of what the comment states. --------- Signed-off-by: joshlee --- .../core_worker/task_submission/normal_task_submitter.cc | 2 +- src/ray/raylet/worker.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index ad7619849889..773852cd3837 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -169,7 +169,7 @@ void NormalTaskSubmitter::OnWorkerIdle( } else { auto client = core_worker_client_pool_->GetOrConnect(addr); - while (!current_queue.empty() && !lease_entry.is_busy) { + if (!current_queue.empty() && !lease_entry.is_busy) { auto task_spec = std::move(current_queue.front()); current_queue.pop_front(); diff --git a/src/ray/raylet/worker.h b/src/ray/raylet/worker.h index 2e0ef7f13f64..4c61f4860c3d 100644 --- a/src/ray/raylet/worker.h +++ b/src/ray/raylet/worker.h @@ -245,10 +245,10 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa bool IsRegistered() { return rpc_client_ != nullptr; } bool IsAvailableForScheduling() const { - return !IsDead() // Not dead - && !GetAssignedTaskId().IsNil() // No assigned task - && !IsBlocked() // Not blocked - && GetActorId().IsNil(); // No assigned actor + return !IsDead() // Not dead + && GetAssignedTaskId().IsNil() // No assigned task + && !IsBlocked() // Not blocked + && GetActorId().IsNil(); // No assigned actor } rpc::CoreWorkerClientInterface *rpc_client() { From cd1825a8c322870843cc2ccd01e93e85c7f429e2 Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Sat, 23 Aug 2025 02:39:05 +0530 Subject: [PATCH 232/634] [core] Remove `core_worker_lib` dependency from `shutdown_coordinator_test` (#55807) ## Why are these changes needed? Create and use a minimal `:shutdown_coordinator` target; point `shutdown_coordinator_test` to it directly. Pure build hygiene; no behavior changes. --------- Signed-off-by: Sagar Sumit --- src/ray/core_worker/BUILD.bazel | 18 ++++++++++++++++-- src/ray/core_worker/shutdown_coordinator.cc | 13 ++++++------- src/ray/core_worker/shutdown_coordinator.h | 7 +++---- src/ray/core_worker/tests/BUILD.bazel | 2 +- .../tests/shutdown_coordinator_test.cc | 19 ++++++++++--------- 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 428ab30eace4..5b6ff9734078 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -6,14 +6,12 @@ ray_cc_library( "core_worker.cc", "core_worker_process.cc", "core_worker_shutdown_executor.cc", - "shutdown_coordinator.cc", ], hdrs = [ "core_worker.h", "core_worker_process.h", "core_worker_rpc_proxy.h", "core_worker_shutdown_executor.h", - "shutdown_coordinator.h", ], deps = [ ":actor_handle", @@ -30,6 +28,7 @@ ray_cc_library( ":plasma_store_provider", ":profile_event", ":reference_count", + ":shutdown_coordinator", ":task_event_buffer", "//src/ray/common/cgroup:cgroup_context", "//src/ray/common/cgroup:cgroup_manager", @@ -63,6 +62,21 @@ ray_cc_library( ], ) +ray_cc_library( + name = "shutdown_coordinator", + srcs = [ + "shutdown_coordinator.cc", + ], + hdrs = [ + "shutdown_coordinator.h", + ], + visibility = [":__subpackages__"], + deps = [ + "//src/ray/common:buffer", + "//src/ray/protobuf:common_cc_proto", + ], +) + ray_cc_library( name = "core_worker_options", hdrs = ["core_worker_options.h"], diff --git a/src/ray/core_worker/shutdown_coordinator.cc b/src/ray/core_worker/shutdown_coordinator.cc index 75e13a2fbef9..93e5489b4144 100644 --- a/src/ray/core_worker/shutdown_coordinator.cc +++ b/src/ray/core_worker/shutdown_coordinator.cc @@ -22,14 +22,13 @@ #include #include -#include "ray/common/buffer.h" // LocalMemoryBuffer -#include "ray/core_worker/common.h" // for WorkerType alias +#include "ray/common/buffer.h" // LocalMemoryBuffer namespace ray { namespace core { ShutdownCoordinator::ShutdownCoordinator( - std::unique_ptr executor, WorkerType worker_type) + std::unique_ptr executor, rpc::WorkerType worker_type) : executor_(std::move(executor)), worker_type_(worker_type) { RAY_CHECK(executor_) << "ShutdownCoordinator requires a non-null ShutdownExecutorInterface. " @@ -156,12 +155,12 @@ void ShutdownCoordinator::ExecuteShutdownSequence( std::chrono::milliseconds timeout_ms, const std::shared_ptr &creation_task_exception_pb_bytes) { switch (worker_type_) { - case WorkerType::DRIVER: + case rpc::WorkerType::DRIVER: ExecuteDriverShutdown(force_shutdown, detail, timeout_ms); break; - case WorkerType::WORKER: - case WorkerType::SPILL_WORKER: - case WorkerType::RESTORE_WORKER: + case rpc::WorkerType::WORKER: + case rpc::WorkerType::SPILL_WORKER: + case rpc::WorkerType::RESTORE_WORKER: ExecuteWorkerShutdown( force_shutdown, detail, timeout_ms, creation_task_exception_pb_bytes); break; diff --git a/src/ray/core_worker/shutdown_coordinator.h b/src/ray/core_worker/shutdown_coordinator.h index b2a68ec4af39..f685261a5989 100644 --- a/src/ray/core_worker/shutdown_coordinator.h +++ b/src/ray/core_worker/shutdown_coordinator.h @@ -21,8 +21,7 @@ #include #include -// Bring in WorkerType alias and common types -#include "ray/core_worker/common.h" +#include "src/ray/protobuf/common.pb.h" namespace ray { class LocalMemoryBuffer; @@ -137,7 +136,7 @@ class ShutdownCoordinator { /// \param executor Shutdown executor implementation /// \param worker_type Type of worker for shutdown behavior customization explicit ShutdownCoordinator(std::unique_ptr executor, - WorkerType worker_type = WorkerType::WORKER); + rpc::WorkerType worker_type = rpc::WorkerType::WORKER); ~ShutdownCoordinator() = default; @@ -278,7 +277,7 @@ class ShutdownCoordinator { // Executor and configuration std::unique_ptr executor_; - WorkerType worker_type_; + rpc::WorkerType worker_type_; // Mutex-guarded shutdown state mutable std::mutex mu_; diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index 96b6769cc948..c36833757685 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -17,7 +17,7 @@ ray_cc_test( srcs = ["shutdown_coordinator_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/core_worker:core_worker_lib", + "//src/ray/core_worker:shutdown_coordinator", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/core_worker/tests/shutdown_coordinator_test.cc b/src/ray/core_worker/tests/shutdown_coordinator_test.cc index 6f0250ad3b8f..4079fbbe8020 100644 --- a/src/ray/core_worker/tests/shutdown_coordinator_test.cc +++ b/src/ray/core_worker/tests/shutdown_coordinator_test.cc @@ -25,6 +25,7 @@ #include #include "ray/common/buffer.h" +#include "src/ray/protobuf/common.pb.h" namespace ray { namespace core { @@ -106,7 +107,7 @@ class ShutdownCoordinatorTest : public ::testing::Test { protected: // Helper to create coordinator with specific worker type std::unique_ptr CreateCoordinator( - WorkerType worker_type = WorkerType::WORKER) { + rpc::WorkerType worker_type = rpc::WorkerType::WORKER) { auto fake = std::make_unique(); return std::make_unique(std::move(fake), worker_type); } @@ -165,7 +166,7 @@ TEST_F(ShutdownCoordinatorTest, TEST_F(ShutdownCoordinatorTest, RequestShutdown_Graceful_SetsDisconnecting_ThenTryTransitionToShutdown_Succeeds) { auto coordinator = std::make_unique( - std::make_unique(), WorkerType::WORKER); + std::make_unique(), rpc::WorkerType::WORKER); // Running -> ShuttingDown -> Disconnecting EXPECT_TRUE( @@ -241,7 +242,7 @@ TEST_F(ShutdownCoordinatorTest, } TEST_F(ShutdownCoordinatorTest, Driver_GracefulReasonRecorded) { - auto coordinator = CreateCoordinator(WorkerType::DRIVER); + auto coordinator = CreateCoordinator(rpc::WorkerType::DRIVER); EXPECT_TRUE(coordinator->RequestShutdown(false, // graceful ShutdownReason::kGracefulExit)); @@ -250,7 +251,7 @@ TEST_F(ShutdownCoordinatorTest, Driver_GracefulReasonRecorded) { } TEST_F(ShutdownCoordinatorTest, Driver_ForceReasonRecorded) { - auto coordinator = CreateCoordinator(WorkerType::DRIVER); + auto coordinator = CreateCoordinator(rpc::WorkerType::DRIVER); EXPECT_TRUE(coordinator->RequestShutdown(true, // force ShutdownReason::kForcedExit)); @@ -259,21 +260,21 @@ TEST_F(ShutdownCoordinatorTest, Driver_ForceReasonRecorded) { } TEST_F(ShutdownCoordinatorTest, Worker_GracefulInitiates) { - auto coordinator = CreateCoordinator(WorkerType::WORKER); + auto coordinator = CreateCoordinator(rpc::WorkerType::WORKER); EXPECT_TRUE(coordinator->RequestShutdown(false, // graceful ShutdownReason::kGracefulExit)); } TEST_F(ShutdownCoordinatorTest, Worker_ExecuteWorkerExit_OnUserError) { - auto coordinator = CreateCoordinator(WorkerType::WORKER); + auto coordinator = CreateCoordinator(rpc::WorkerType::WORKER); EXPECT_TRUE(coordinator->RequestShutdown(false, // graceful ShutdownReason::kUserError)); } TEST_F(ShutdownCoordinatorTest, Worker_HandleExit_OnIdleTimeout) { - auto coordinator = CreateCoordinator(WorkerType::WORKER); + auto coordinator = CreateCoordinator(rpc::WorkerType::WORKER); EXPECT_TRUE(coordinator->RequestShutdown(false, // graceful ShutdownReason::kIdleTimeout)); @@ -366,7 +367,7 @@ TEST_F(ShutdownCoordinatorTest, Concurrent_GracefulVsForce_ForceExecutesOnce) { auto fake = std::make_unique(); auto *fake_ptr = fake.get(); auto coordinator = - std::make_unique(std::move(fake), WorkerType::WORKER); + std::make_unique(std::move(fake), rpc::WorkerType::WORKER); std::thread t1([&] { coordinator->RequestShutdown(false, ShutdownReason::kGracefulExit, "graceful"); @@ -386,7 +387,7 @@ TEST_F(ShutdownCoordinatorTest, Concurrent_DoubleForce_ForceExecutesOnce) { auto fake = std::make_unique(); auto *fake_ptr = fake.get(); auto coordinator = - std::make_unique(std::move(fake), WorkerType::WORKER); + std::make_unique(std::move(fake), rpc::WorkerType::WORKER); std::thread t1( [&] { coordinator->RequestShutdown(true, ShutdownReason::kForcedExit, "force1"); }); From 5f6d8558f4495de28334dcef18e29f5db3ce50a1 Mon Sep 17 00:00:00 2001 From: goutamvenkat-anyscale Date: Fri, 22 Aug 2025 14:43:44 -0700 Subject: [PATCH 233/634] [Data] Upgrade Polars to 1.32.3 (#55847) ## Why are these changes needed? Upgrading Polars to 1.32.3 ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Goutam V --- python/requirements/test-requirements.txt | 2 +- python/requirements_compiled.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/requirements/test-requirements.txt b/python/requirements/test-requirements.txt index d08dc4ad4215..480b1eab6eb4 100644 --- a/python/requirements/test-requirements.txt +++ b/python/requirements/test-requirements.txt @@ -86,7 +86,7 @@ pytest-docker-tools==3.1.3 pytest-forked==1.4.0 # For dataset tests -polars>=1.30.0,<2.0.0 +polars>=1.32.3,<2.0.0 importlib-metadata==6.11.0 diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt index da646a8d36f4..27231510d8bc 100644 --- a/python/requirements_compiled.txt +++ b/python/requirements_compiled.txt @@ -1485,7 +1485,7 @@ plotly==5.23.0 # via ax-platform pluggy==1.3.0 # via pytest -polars==1.30.0 +polars==1.32.3 # via -r python/requirements/test-requirements.txt portalocker==2.8.2 # via From 951ce84d3229afdfe7b798d73ee66fdf030afb33 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 22 Aug 2025 16:13:39 -0700 Subject: [PATCH 234/634] [Core] Use _Exit instead of std::quick_exit which may not exist on mac (#55840) Signed-off-by: Jiajun Yao --- src/ray/core_worker/core_worker_shutdown_executor.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ray/core_worker/core_worker_shutdown_executor.cc b/src/ray/core_worker/core_worker_shutdown_executor.cc index 03cfc937d81a..f8680d709219 100644 --- a/src/ray/core_worker/core_worker_shutdown_executor.cc +++ b/src/ray/core_worker/core_worker_shutdown_executor.cc @@ -301,8 +301,7 @@ void CoreWorkerShutdownExecutor::DisconnectServices( void CoreWorkerShutdownExecutor::QuickExit() { RAY_LOG(WARNING) << "Quick exit - terminating process immediately"; - RAY_LOG(WARNING) << "Quick exit - calling std::quick_exit(1)"; - std::quick_exit(1); + ray::QuickExit(); RAY_LOG(WARNING) << "Quick exit - this line should never be reached"; } } // namespace core From df92a501147cb5b64c7ded4eafbfa0d0c126f2dc Mon Sep 17 00:00:00 2001 From: mcoder6425 Date: Sat, 23 Aug 2025 05:23:03 +0600 Subject: [PATCH 235/634] [Core] Add AMD-Instinct-MI50 support (#55365) Signed-off-by: mcoder6425 --- python/ray/_private/accelerators/amd_gpu.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ray/_private/accelerators/amd_gpu.py b/python/ray/_private/accelerators/amd_gpu.py index 662e858c71b9..ec870193c8c6 100644 --- a/python/ray/_private/accelerators/amd_gpu.py +++ b/python/ray/_private/accelerators/amd_gpu.py @@ -11,6 +11,7 @@ NOSET_HIP_VISIBLE_DEVICES_ENV_VAR = "RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES" amd_product_dict = { + "0x66a1": "AMD-Instinct-MI50", "0x738c": "AMD-Instinct-MI100", "0x7408": "AMD-Instinct-MI250X", "0x740c": "AMD-Instinct-MI250X-MI250", From 8f0ad90b28548b55e076a4264938ba5d879ff1dd Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 22 Aug 2025 19:12:38 -0700 Subject: [PATCH 236/634] [Core] Remove dead code (#55841) Signed-off-by: Jiajun Yao --- python/ray/_raylet.pyx | 6 --- .../ray/dashboard/client/src/type/worker.d.ts | 1 - .../ray/dashboard/modules/node/datacenter.py | 4 -- python/ray/includes/libcoreworker.pxd | 1 - src/ray/core_worker/core_worker.cc | 6 --- src/ray/core_worker/core_worker.h | 5 -- src/ray/protobuf/common.proto | 47 +++++++++---------- 7 files changed, 22 insertions(+), 48 deletions(-) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index 83798c1da252..282c39a466a5 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -1918,9 +1918,6 @@ cdef void execute_task( if (task_type == TASK_TYPE_ACTOR_CREATION_TASK): actor_id = core_worker.get_actor_id() actor = worker.actors[actor_id] - class_name = actor.__class__.__name__ - actor_title = f"{class_name}({args!r}, {kwargs!r})" - core_worker.set_actor_title(actor_title.encode("utf-8")) worker.record_task_log_start(task_id, attempt_number) @@ -3195,9 +3192,6 @@ cdef class CoreWorker: def set_webui_display(self, key, message): CCoreWorkerProcess.GetCoreWorker().SetWebuiDisplay(key, message) - def set_actor_title(self, title): - CCoreWorkerProcess.GetCoreWorker().SetActorTitle(title) - def set_actor_repr_name(self, repr_name): CCoreWorkerProcess.GetCoreWorker().SetActorReprName(repr_name) diff --git a/python/ray/dashboard/client/src/type/worker.d.ts b/python/ray/dashboard/client/src/type/worker.d.ts index 8f4d89e685e9..f8822f75b733 100644 --- a/python/ray/dashboard/client/src/type/worker.d.ts +++ b/python/ray/dashboard/client/src/type/worker.d.ts @@ -15,7 +15,6 @@ export type CoreWorkerStats = { numExecutedTasks: number; numPendingTasks: number; workerId: string; - actorTitle: string; jobId: string; numObjectRefsInScope: number; numInPlasma: number; diff --git a/python/ray/dashboard/modules/node/datacenter.py b/python/ray/dashboard/modules/node/datacenter.py index 6b32e4c545d5..941702822980 100644 --- a/python/ray/dashboard/modules/node/datacenter.py +++ b/python/ray/dashboard/modules/node/datacenter.py @@ -205,10 +205,6 @@ async def _get_actor_info(actor: Optional[dict]) -> Optional[dict]: actor = actor.copy() worker_id = actor["address"]["workerId"] core_worker_stats = DataSource.core_worker_stats.get(worker_id, {}) - actor_constructor = core_worker_stats.get( - "actorTitle", "Unknown actor constructor" - ) - actor["actorConstructor"] = actor_constructor actor.update(core_worker_stats) # TODO(fyrestone): remove this, give a link from actor diff --git a/python/ray/includes/libcoreworker.pxd b/python/ray/includes/libcoreworker.pxd index 5402a7e2f48d..2b401369f777 100644 --- a/python/ray/includes/libcoreworker.pxd +++ b/python/ray/includes/libcoreworker.pxd @@ -212,7 +212,6 @@ cdef extern from "ray/core_worker/core_worker.h" nogil: c_bool ShouldCaptureChildTasksInPlacementGroup() CActorID GetActorId() const const c_string GetActorName() - void SetActorTitle(const c_string &title) void SetActorReprName(const c_string &repr_name) void SetWebuiDisplay(const c_string &key, const c_string &message) const ResourceMappingType &GetResourceIDs() const diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 8efe6a7878bf..ae467adc6ca0 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -4054,7 +4054,6 @@ void CoreWorker::HandleGetCoreWorkerStats(rpc::GetCoreWorkerStatsRequest request } (*used_resources_map)[resource_name] = allocations; } - stats->set_actor_title(actor_title_); google::protobuf::Map webui_map(webui_display_.begin(), webui_display_.end()); (*stats->mutable_webui_display()) = webui_map; @@ -4388,11 +4387,6 @@ void CoreWorker::SetWebuiDisplay(const std::string &key, const std::string &mess webui_display_[key] = message; } -void CoreWorker::SetActorTitle(const std::string &title) { - absl::MutexLock lock(&mutex_); - actor_title_ = title; -} - void CoreWorker::SetActorReprName(const std::string &repr_name) { RAY_CHECK(task_receiver_ != nullptr); task_receiver_->SetActorReprName(repr_name); diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index cffda26e8c85..b44df2b5ee05 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -351,8 +351,6 @@ class CoreWorker { void SetWebuiDisplay(const std::string &key, const std::string &message); - void SetActorTitle(const std::string &title); - /// Sets the actor's repr name. /// /// This is set explicitly rather than included as part of actor creation task spec @@ -1849,9 +1847,6 @@ class CoreWorker { /// Key value pairs to be displayed on Web UI. std::unordered_map webui_display_ ABSL_GUARDED_BY(mutex_); - /// Actor title that consists of class name, args, kwargs for actor construction. - std::string actor_title_ ABSL_GUARDED_BY(mutex_); - /// Actor repr name if overrides by the user, empty string if not. std::string actor_repr_name_ ABSL_GUARDED_BY(mutex_); diff --git a/src/ray/protobuf/common.proto b/src/ray/protobuf/common.proto index 0fb1ade03022..d1dae8dd7c68 100644 --- a/src/ray/protobuf/common.proto +++ b/src/ray/protobuf/common.proto @@ -933,53 +933,50 @@ message ResourceAllocations { // Debug info returned from the core worker. message CoreWorkerStats { - reserved 1; // Number of pending normal and actor tasks. - int32 num_pending_tasks = 2; + int32 num_pending_tasks = 1; // Number of object refs in local scope. - int32 num_object_refs_in_scope = 3; + int32 num_object_refs_in_scope = 2; // IP address of the core worker. - string ip_address = 7; + string ip_address = 3; // Port of the core worker. - int64 port = 8; + int64 port = 4; // Actor ID. - bytes actor_id = 9; + bytes actor_id = 5; // A map from the resource name (e.g. "CPU") to its allocation. - map used_resources = 10; + map used_resources = 6; // A string displayed on Dashboard. - map webui_display = 11; + map webui_display = 7; // Number of objects that are IN_PLASMA_ERROR in the local memory store. - int32 num_in_plasma = 12; + int32 num_in_plasma = 8; // Number of objects stored in local memory. - int32 num_local_objects = 13; + int32 num_local_objects = 9; // Used local object store memory. - int64 used_object_store_memory = 14; + int64 used_object_store_memory = 10; // Length of the task queue. - int32 task_queue_length = 15; + int32 task_queue_length = 11; // Number of executed tasks. - int32 num_executed_tasks = 16; - // Actor constructor. - string actor_title = 17; + int32 num_executed_tasks = 12; // Local reference table. - repeated ObjectRefInfo object_refs = 18; + repeated ObjectRefInfo object_refs = 13; // Job ID. - bytes job_id = 19; + bytes job_id = 14; // Worker id of core worker. - bytes worker_id = 20; + bytes worker_id = 15; // Language - Language language = 21; + Language language = 16; // PID of the worker process. - uint32 pid = 22; + uint32 pid = 17; // The worker type. - WorkerType worker_type = 23; + WorkerType worker_type = 18; // Length of the number of objects without truncation. - int64 objects_total = 24; + int64 objects_total = 19; // Number of objects owned by the worker. - int64 num_owned_objects = 25; + int64 num_owned_objects = 20; // Number of actors owned by the worker. - int64 num_owned_actors = 26; + int64 num_owned_actors = 21; // Number of running tasks - int64 num_running_tasks = 27; + int64 num_running_tasks = 22; } // Resource usage reported by the node reporter. From 3b3d99731a5109e3b10fcfe9d04cde2b81e5d63b Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Sat, 23 Aug 2025 02:35:20 -0500 Subject: [PATCH 237/634] [core] Split `node_manager` target out from `gcs_server_lib` (#55814) Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 29 +++++++++++++++++-- src/ray/gcs/gcs_server/gcs_node_manager.cc | 5 ++-- src/ray/gcs/gcs_server/gcs_node_manager.h | 17 ++++------- .../gcs/gcs_server/gcs_placement_group_mgr.h | 1 + src/ray/gcs/gcs_server/tests/BUILD.bazel | 13 +++++---- .../gcs_node_manager_export_event_test.cc | 17 +++++------ .../gcs_server/tests/gcs_node_manager_test.cc | 22 +++++++------- 7 files changed, 59 insertions(+), 45 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index f81fbdcb7479..74cdbd2067c7 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -63,6 +63,32 @@ ray_cc_library( ], ) +ray_cc_library( + name = "gcs_node_manager", + srcs = ["gcs_node_manager.cc"], + hdrs = ["gcs_node_manager.h"], + implementation_deps = [ + "//src/ray/gcs:gcs_pb_util", + "@com_google_absl//absl/container:flat_hash_set", + ], + deps = [ + ":gcs_init_data", + ":gcs_table_storage", + "//src/ray/common:asio", + "//src/ray/common:id", + "//src/ray/common:ray_config", + "//src/ray/gcs/pubsub:gcs_pub_sub_lib", + "//src/ray/protobuf:gcs_service_cc_proto", + "//src/ray/protobuf:ray_syncer_cc_proto", + "//src/ray/rpc:gcs_server", + "//src/ray/rpc:node_manager_client", + "//src/ray/util:event", + "//src/ray/util:logging", + "//src/ray/util:time", + "@com_google_absl//absl/container:flat_hash_map", + ], +) + ray_cc_library( name = "gcs_usage_stats_client", srcs = ["usage_stats_client.cc"], @@ -199,7 +225,6 @@ ray_cc_library( "gcs_actor_manager.cc", "gcs_actor_scheduler.cc", "gcs_autoscaler_state_manager.cc", - "gcs_node_manager.cc", "gcs_placement_group_mgr.cc", "gcs_placement_group_scheduler.cc", "gcs_resource_manager.cc", @@ -209,7 +234,6 @@ ray_cc_library( "gcs_actor_manager.h", "gcs_actor_scheduler.h", "gcs_autoscaler_state_manager.h", - "gcs_node_manager.h", "gcs_placement_group_mgr.h", "gcs_placement_group_scheduler.h", "gcs_resource_manager.h", @@ -221,6 +245,7 @@ ray_cc_library( ":gcs_init_data", ":gcs_job_manager", ":gcs_kv_manager", + ":gcs_node_manager", ":gcs_pubsub_handler", ":gcs_runtime_env_handler", ":gcs_server_io_context_policy", diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.cc b/src/ray/gcs/gcs_server/gcs_node_manager.cc index bd14e119811a..8a6f6ffbb00c 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_node_manager.cc @@ -21,9 +21,8 @@ #include #include -#include "ray/common/ray_config.h" +#include "absl/container/flat_hash_set.h" #include "ray/gcs/pb_util.h" -#include "ray/util/event.h" #include "ray/util/logging.h" #include "ray/util/time.h" #include "src/ray/protobuf/gcs.pb.h" @@ -530,7 +529,7 @@ std::string GcsNodeManager::DebugString() const { void GcsNodeManager::UpdateAliveNode( const NodeID &node_id, - const syncer::ResourceViewSyncMessage &resource_view_sync_message) { + const rpc::syncer::ResourceViewSyncMessage &resource_view_sync_message) { auto maybe_node_info = GetAliveNode(node_id); if (maybe_node_info == absl::nullopt) { return; diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.h b/src/ray/gcs/gcs_server/gcs_node_manager.h index ff196f60b263..02c3b6723d55 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/ray/gcs/gcs_server/gcs_node_manager.h @@ -14,11 +14,6 @@ #pragma once -#include - -#include -#include -#include #include #include #include @@ -26,24 +21,21 @@ #include #include "absl/container/flat_hash_map.h" -#include "absl/container/flat_hash_set.h" #include "ray/common/id.h" -#include "ray/common/ray_syncer/ray_syncer.h" #include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" -#include "ray/rpc/client_call.h" #include "ray/rpc/gcs/gcs_rpc_server.h" -#include "ray/rpc/node_manager/node_manager_client.h" #include "ray/rpc/node_manager/raylet_client_pool.h" #include "ray/util/event.h" #include "src/ray/protobuf/gcs.pb.h" +#include "src/ray/protobuf/ray_syncer.pb.h" namespace ray::gcs { class GcsAutoscalerStateManagerTest; class GcsStateTest; + /// GcsNodeManager is responsible for managing and monitoring nodes as well as handing /// node and resource related rpc requests. /// This class is not thread-safe. @@ -176,8 +168,9 @@ class GcsNodeManager : public rpc::NodeInfoHandler { /// /// \param node_id The ID of the node to update. /// \param resource_view_sync_message The sync message containing the new state. - void UpdateAliveNode(const NodeID &node_id, - const syncer::ResourceViewSyncMessage &resource_view_sync_message); + void UpdateAliveNode( + const NodeID &node_id, + const rpc::syncer::ResourceViewSyncMessage &resource_view_sync_message); private: /// Add the dead node to the cache. If the cache is full, the earliest dead node is diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h index d5956a93609a..00dc3b0baeac 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h @@ -30,6 +30,7 @@ #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" +#include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/usage_stats_client.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index 60b092b2778a..e4db2be85604 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -112,9 +112,9 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - ":gcs_server_test_util", + "//:ray_fakes", "//src/mock/ray/pubsub:mock_publisher", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_node_manager", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], @@ -396,7 +396,7 @@ ray_cc_test( ) ray_cc_test( - name = "gcs_node_manager_export_event_test", + name = "node_manager_export_event_test", size = "small", srcs = ["export_api/gcs_node_manager_export_event_test.cc"], tags = [ @@ -404,11 +404,12 @@ ray_cc_test( "team:core", ], deps = [ - ":gcs_server_test_util", + "//:ray_fakes", "//src/mock/ray/pubsub:mock_publisher", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_node_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", - "@com_google_googletest//:gtest_main", + "//src/ray/util:string_utils", + "@com_google_googletest//:gtest", ], ) diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc index a310f656b7af..03237fe82167 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc @@ -20,18 +20,14 @@ #include #include -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "fakes/ray/rpc/raylet/raylet_client.h" +#include "mock/ray/pubsub/publisher.h" +#include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/util/event.h" #include "ray/util/string_utils.h" -// clang-format off -#include "ray/rpc/node_manager/node_manager_client.h" -#include "ray/rpc/node_manager/raylet_client_pool.h" -#include "mock/ray/pubsub/publisher.h" -// clang-format on - using json = nlohmann::json; namespace ray { @@ -46,9 +42,11 @@ std::string GenerateLogDir() { class GcsNodeManagerExportAPITest : public ::testing::Test { public: GcsNodeManagerExportAPITest() { - raylet_client_ = std::make_shared(); + auto raylet_client = std::make_shared(); client_pool_ = std::make_unique( - [this](const rpc::Address &) { return raylet_client_; }); + [raylet_client = std::move(raylet_client)](const rpc::Address &) { + return raylet_client; + }); gcs_publisher_ = std::make_unique( std::make_unique()); gcs_table_storage_ = std::make_unique( @@ -78,7 +76,6 @@ class GcsNodeManagerExportAPITest : public ::testing::Test { protected: std::unique_ptr gcs_table_storage_; - std::shared_ptr raylet_client_; std::unique_ptr client_pool_; std::shared_ptr gcs_publisher_; instrumented_io_context io_service_; diff --git a/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc index ced40b4863e1..7b8472a385bf 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc @@ -12,28 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "ray/gcs/gcs_server/gcs_node_manager.h" + +#include + #include #include #include -// clang-format off -#include "gtest/gtest.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" -#include "ray/gcs/tests/gcs_test_util.h" -#include "ray/rpc/node_manager/node_manager_client.h" -#include "ray/rpc/node_manager/raylet_client_pool.h" +#include "fakes/ray/rpc/raylet/raylet_client.h" #include "mock/ray/pubsub/publisher.h" -#include "ray/common/asio/asio_util.h" -#include "ray/common/ray_syncer/ray_syncer.h" -// clang-format on +#include "ray/gcs/tests/gcs_test_util.h" namespace ray { class GcsNodeManagerTest : public ::testing::Test { public: GcsNodeManagerTest() { - raylet_client_ = std::make_shared(); + auto raylet_client = std::make_shared(); client_pool_ = std::make_unique( - [this](const rpc::Address &) { return raylet_client_; }); + [raylet_client = std::move(raylet_client)](const rpc::Address &) { + return raylet_client; + }); gcs_publisher_ = std::make_unique( std::make_unique()); io_context_ = std::make_unique("GcsNodeManagerTest"); @@ -41,7 +40,6 @@ class GcsNodeManagerTest : public ::testing::Test { protected: std::unique_ptr gcs_table_storage_; - std::shared_ptr raylet_client_; std::unique_ptr client_pool_; std::unique_ptr gcs_publisher_; std::unique_ptr io_context_; From 9bc526d8c9580ef9c03e04ddd968921a3d0e4ae8 Mon Sep 17 00:00:00 2001 From: Kai-Hsun Chen Date: Sat, 23 Aug 2025 13:01:02 -0700 Subject: [PATCH 238/634] [core][chore] Avoid unnecessary deserialization (#55866) Signed-off-by: kaihsun --- .../core_worker/task_submission/dependency_resolver.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ray/core_worker/task_submission/dependency_resolver.cc b/src/ray/core_worker/task_submission/dependency_resolver.cc index e11b0c95645e..be99c9ddacb7 100644 --- a/src/ray/core_worker/task_submission/dependency_resolver.cc +++ b/src/ray/core_worker/task_submission/dependency_resolver.cc @@ -101,12 +101,12 @@ void LocalDependencyResolver::ResolveDependencies( if (task.ArgByRef(i)) { local_dependency_ids.insert(task.ArgObjectId(i)); } - for (const auto &in : task.ArgInlinedRefs(i)) { - auto object_id = ObjectID::FromBinary(in.object_id()); + for (const auto &inlined_ref : task.ArgInlinedRefs(i)) { + const auto object_id = ObjectID::FromBinary(inlined_ref.object_id()); if (ObjectID::IsActorID(object_id)) { - auto actor_id = ObjectID::ToActorID(object_id); + const auto actor_id = ObjectID::ToActorID(object_id); if (actor_creator_.IsActorInRegistering(actor_id)) { - actor_dependency_ids.insert(ObjectID::ToActorID(object_id)); + actor_dependency_ids.insert(actor_id); } } } From 729a3c098fc74f20fb44d8aed229ae44b513d7b8 Mon Sep 17 00:00:00 2001 From: Justin Yu Date: Sat, 23 Aug 2025 14:41:08 -0700 Subject: [PATCH 239/634] [data][train] Minor rework of `get_dataset_shard` (#55825) Adds a slim wrapper around dataset shard iterators passed to each worker. --------- Signed-off-by: Justin Yu --- .../train/v2/_internal/callbacks/datasets.py | 59 ++++++++++++++----- .../v2/_internal/data_integration/__init__.py | 0 .../_internal/data_integration/interfaces.py | 29 +++++++++ .../train/v2/_internal/execution/context.py | 18 +++--- .../v2/_internal/execution/train_fn_utils.py | 8 ++- .../execution/worker_group/worker.py | 10 ++-- .../train/v2/tests/test_data_integration.py | 22 ++++--- 7 files changed, 108 insertions(+), 38 deletions(-) create mode 100644 python/ray/train/v2/_internal/data_integration/__init__.py create mode 100644 python/ray/train/v2/_internal/data_integration/interfaces.py diff --git a/python/ray/train/v2/_internal/callbacks/datasets.py b/python/ray/train/v2/_internal/callbacks/datasets.py index a51b633d457a..9a6ce9d76cab 100644 --- a/python/ray/train/v2/_internal/callbacks/datasets.py +++ b/python/ray/train/v2/_internal/callbacks/datasets.py @@ -1,18 +1,37 @@ import copy -from typing import Any, Callable, Dict, List, Union +from typing import Dict, List import ray.train -from ray.data import Dataset +from ray.data import DataIterator from ray.data.context import DataContext +from ray.train.v2._internal.data_integration.interfaces import ( + DatasetShardMetadata, + DatasetShardProvider, + GenDataset, +) from ray.train.v2._internal.execution.callback import WorkerGroupCallback from ray.train.v2._internal.execution.worker_group.worker_group import ( Worker, WorkerGroup, ) -# A type representing either a ray.data.Dataset or a function that returns a -# ray.data.Dataset and accepts no arguments. -GenDataset = Union[Dataset, Callable[[], Dataset]] + +class RayDatasetShardProvider: + """A shard provider that Train workers use to access a DataIterator for a dataset.""" + + def __init__(self, ds_iterators: Dict[str, DataIterator]): + # Maps dataset_name to a DataIterator. + self._dataset_iterators = ds_iterators + + def get_dataset_shard(self, dataset_info: DatasetShardMetadata) -> DataIterator: + if dataset_info.dataset_name not in self._dataset_iterators: + raise KeyError( + f"Dataset shard for '{dataset_info.dataset_name}' not found. " + "Please ensure that the dataset is passed through the Trainer `datasets` " + "argument." + ) + + return self._dataset_iterators[dataset_info.dataset_name] class DatasetsSetupCallback(WorkerGroupCallback): @@ -45,10 +64,15 @@ def get_train_total_resources( these resources logically from its available pool.""" return scaling_config.total_resources - def before_init_train_context(self, workers: List[Worker]) -> Dict[str, List[Any]]: - # Configure dataset shards - datasets = {k: v() if callable(v) else v for k, v in self._datasets.items()} - node_ids = [worker.metadata.node_id for worker in workers] + # -------------------------- + # WorkerGroupCallback + # -------------------------- + + def before_init_train_context( + self, workers: List[Worker] + ) -> Dict[str, List[DatasetShardProvider]]: + world_size = len(workers) + worker_node_ids = [worker.metadata.node_id for worker in workers] # Notify the DataConfig about the total resources reserved for training. total_train_resources = self.get_train_total_resources(self._scaling_config) @@ -56,15 +80,20 @@ def before_init_train_context(self, workers: List[Worker]) -> Dict[str, List[Any total_train_resources.get("CPU", 0), total_train_resources.get("GPU", 0) ) - dataset_shards = self._data_config.configure( - datasets, - world_size=len(workers), + datasets = {k: v() if callable(v) else v for k, v in self._datasets.items()} + ds_iterators_per_rank = self._data_config.configure( + datasets=datasets, + world_size=world_size, worker_handles=None, - worker_node_ids=node_ids, + worker_node_ids=worker_node_ids, ) - assert len(dataset_shards) == len(workers) + assert len(ds_iterators_per_rank) == world_size - return {"dataset_shards": dataset_shards} + shard_providers_per_rank = [ + RayDatasetShardProvider(ds_iterators=ds_iterators_per_rank[rank]) + for rank in range(world_size) + ] + return {"dataset_shard_provider": shard_providers_per_rank} def after_worker_group_start(self, worker_group: WorkerGroup): # Propagate DataContext diff --git a/python/ray/train/v2/_internal/data_integration/__init__.py b/python/ray/train/v2/_internal/data_integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/ray/train/v2/_internal/data_integration/interfaces.py b/python/ray/train/v2/_internal/data_integration/interfaces.py new file mode 100644 index 000000000000..73b37854fee6 --- /dev/null +++ b/python/ray/train/v2/_internal/data_integration/interfaces.py @@ -0,0 +1,29 @@ +from dataclasses import dataclass +from typing import Callable, Protocol, Union + +from ray.data import DataIterator, Dataset + +# A type representing either a ray.data.Dataset or a function that returns a +# ray.data.Dataset and accepts no arguments. +GenDataset = Union[Dataset, Callable[[], Dataset]] + + +@dataclass +class DatasetShardMetadata: + """Metadata about a dataset shard used for lookup and configuration.""" + + dataset_name: str + + +class DatasetShardProvider(Protocol): + def get_dataset_shard(self, dataset_info: DatasetShardMetadata) -> DataIterator: + """Get the dataset shard for the given dataset info. + Args: + dataset_info: The metadata of the shard to retrieve, + including the dataset name. + Returns: + The :class:`~ray.data.DataIterator` shard for the given dataset info. + Raises: + KeyError: If the dataset shard for the given dataset info is not found. + """ + ... diff --git a/python/ray/train/v2/_internal/execution/context.py b/python/ray/train/v2/_internal/execution/context.py index fd2f0df8e23e..a19b308b0709 100644 --- a/python/ray/train/v2/_internal/execution/context.py +++ b/python/ray/train/v2/_internal/execution/context.py @@ -17,6 +17,10 @@ from ray.train.v2.api.config import RunConfig, ScalingConfig if TYPE_CHECKING: + from ray.train.v2._internal.data_integration.interfaces import ( + DatasetShardMetadata, + DatasetShardProvider, + ) from ray.train.v2._internal.execution.callback import TrainContextCallback from ray.train.v2._internal.execution.worker_group.thread_runner import ThreadRunner @@ -92,7 +96,7 @@ class TrainContext: distributed_context: DistributedContext execution_context: ExecutionContext storage_context: StorageContext - dataset_shards: Dict[str, DataIterator] + dataset_shard_provider: "DatasetShardProvider" checkpoint: Optional[Checkpoint] = None @_copy_doc(session.get_experiment_name) @@ -133,7 +137,7 @@ def get_synchronization_actor(self): def get_checkpoint(self): return self.checkpoint - def get_dataset_shard(self, dataset_name: str) -> DataIterator: + def get_dataset_shard(self, dataset_info: "DatasetShardMetadata") -> DataIterator: """Returns the :class:`ray.data.DataIterator` shard for this worker. Call :meth:`~ray.data.DataIterator.iter_torch_batches` or @@ -141,19 +145,13 @@ def get_dataset_shard(self, dataset_name: str) -> DataIterator: appropriate framework-specific data type. Args: - dataset_name: Name of the dataset shard. + dataset_info: The shard metadata, including the dataset name and worker rank. Returns: The ``DataIterator`` shard with the given name for this worker. Raises: KeyError: If the dataset shard with the given name is not found. """ - try: - return self.dataset_shards[dataset_name] - except KeyError: - raise KeyError( - f"Dataset {dataset_name} not found. Available datasets: " - f"{list(self.dataset_shards.keys())}." - ) + return self.dataset_shard_provider.get_dataset_shard(dataset_info) def get_context_callbacks(self) -> List["TrainContextCallback"]: return self.execution_context.train_context_callbacks diff --git a/python/ray/train/v2/_internal/execution/train_fn_utils.py b/python/ray/train/v2/_internal/execution/train_fn_utils.py index c960b32b5f45..6038e6655465 100644 --- a/python/ray/train/v2/_internal/execution/train_fn_utils.py +++ b/python/ray/train/v2/_internal/execution/train_fn_utils.py @@ -58,7 +58,13 @@ def get_dataset_shard(self, dataset_name: str) -> DataIterator: Returns: The DataIterator shard for this worker. """ - return get_internal_train_context().get_dataset_shard(dataset_name) + from ray.train.v2._internal.data_integration.interfaces import ( + DatasetShardMetadata, + ) + + return get_internal_train_context().get_dataset_shard( + DatasetShardMetadata(dataset_name=dataset_name) + ) def get_context(self) -> ExternalTrainContext: return ExternalTrainContext() diff --git a/python/ray/train/v2/_internal/execution/worker_group/worker.py b/python/ray/train/v2/_internal/execution/worker_group/worker.py index 667ab318296b..be2e935ca40e 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/worker.py +++ b/python/ray/train/v2/_internal/execution/worker_group/worker.py @@ -4,13 +4,12 @@ import socket from dataclasses import dataclass from functools import cached_property -from typing import Callable, Dict, List, Optional, TypeVar, Union +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, TypeVar, Union import ray import ray._private.ray_constants as ray_constants from .thread_runner import ThreadRunner from ray.actor import ActorHandle -from ray.data.iterator import DataIterator from ray.train import Checkpoint from ray.train.v2._internal.constants import ( DEFAULT_ENABLE_WORKER_LOGGING, @@ -40,6 +39,9 @@ from ray.train.v2._internal.util import ObjectRefWrapper from ray.types import ObjectRef +if TYPE_CHECKING: + from ray.train.v2._internal.data_integration.interfaces import DatasetShardProvider + T = TypeVar("T") logger = logging.getLogger(__name__) @@ -192,7 +194,7 @@ def init_train_context( synchronization_actor: SynchronizationActor, storage_context: StorageContext, worker_callbacks: List[Union[WorkerCallback, TrainContextCallback]], - dataset_shards: Dict[str, DataIterator] = None, + dataset_shard_provider: Optional["DatasetShardProvider"] = None, checkpoint: Optional[Checkpoint] = None, ): self._callbacks = [c for c in worker_callbacks if isinstance(c, WorkerCallback)] @@ -211,8 +213,8 @@ def init_train_context( train_context_callbacks=context_callbacks_to_propagate, ), storage_context=storage_context, - dataset_shards=dataset_shards or {}, checkpoint=checkpoint, + dataset_shard_provider=dataset_shard_provider, ) # Configure the train and root logger for the worker processes. if ray_constants.env_bool( diff --git a/python/ray/train/v2/tests/test_data_integration.py b/python/ray/train/v2/tests/test_data_integration.py index fe8159d5190f..bf7b0f4e694b 100644 --- a/python/ray/train/v2/tests/test_data_integration.py +++ b/python/ray/train/v2/tests/test_data_integration.py @@ -7,7 +7,8 @@ from ray.data import DataContext, ExecutionResources from ray.data._internal.iterator.stream_split_iterator import StreamSplitDataIterator from ray.data.tests.conftest import restore_data_context # noqa: F401 -from ray.train.v2._internal.callbacks import DatasetsSetupCallback +from ray.train.v2._internal.callbacks.datasets import DatasetsSetupCallback +from ray.train.v2._internal.data_integration.interfaces import DatasetShardMetadata from ray.train.v2._internal.execution.context import TrainRunContext from ray.train.v2._internal.execution.worker_group.worker_group import ( WorkerGroupContext, @@ -87,13 +88,18 @@ def test_dataset_setup_callback(ray_start_4_cpus): data_config=data_config, scaling_config=scaling_config, ) - dataset_shards = callback.before_init_train_context(worker_group.get_workers())[ - "dataset_shards" - ] - assert len(dataset_shards) == NUM_WORKERS - - processed_train_ds = dataset_shards[0]["train"] - processed_valid_ds = dataset_shards[0]["valid"] + dataset_manager_for_each_worker = callback.before_init_train_context( + worker_group.get_workers() + )["dataset_shard_provider"] + assert len(dataset_manager_for_each_worker) == NUM_WORKERS + + dataset_manager = dataset_manager_for_each_worker[0] + processed_train_ds = dataset_manager.get_dataset_shard( + DatasetShardMetadata(dataset_name="train") + ) + processed_valid_ds = dataset_manager.get_dataset_shard( + DatasetShardMetadata(dataset_name="valid") + ) assert isinstance(processed_train_ds, StreamSplitDataIterator) assert not isinstance(processed_valid_ds, StreamSplitDataIterator) From 0b76e5d18c1984d4666d938087f3daf881cbf136 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Sat, 23 Aug 2025 15:50:44 -0700 Subject: [PATCH 240/634] [image] add extra-test stage in image building (#55725) this removes the requirement to build the extra test layer after release image, and allows caching of everything other than the ray wheel and ray wheel installation. Signed-off-by: Lonnie Liu --- .buildkite/release/build.rayci.yml | 83 +++++++++++++++++-- ci/build/build-anyscale-docker.sh | 12 +-- .../ray.cpu.base-extra-testdeps.wanda.yaml | 10 +++ .../ray.cuda.base-extra-testdeps.wanda.yaml | 10 +++ ci/ray_ci/anyscale_docker_container.py | 14 +--- ci/ray_ci/builder.py | 2 +- ci/ray_ci/ray_docker_container.py | 4 +- ci/ray_ci/test_anyscale_docker_container.py | 18 +--- 8 files changed, 105 insertions(+), 48 deletions(-) create mode 100644 ci/docker/ray.cpu.base-extra-testdeps.wanda.yaml create mode 100644 ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml diff --git a/.buildkite/release/build.rayci.yml b/.buildkite/release/build.rayci.yml index d3e04eb91848..1a9137189753 100644 --- a/.buildkite/release/build.rayci.yml +++ b/.buildkite/release/build.rayci.yml @@ -1,10 +1,77 @@ group: release build -tags: - - oss steps: + - name: raycpubaseextra-testdeps + label: "wanda: ray.py{{matrix}}.cpu.base-extra-testdeps" + wanda: ci/docker/ray.cpu.base-extra-testdeps.wanda.yaml + matrix: + - "3.9" + - "3.11" + - "3.12" + env: + PYTHON_VERSION: "{{matrix}}" + IMAGE_TYPE: "ray" + REQUIREMENTS_FILE: "requirements_byod_{{matrix}}.txt" + depends_on: + - raycpubaseextra + + - name: raycudabaseextra-testdeps + label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra-testdeps" + wanda: ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml + matrix: + setup: + python: + - "3.9" + - "3.11" + - "3.12" + cuda: + - "12.3.2-cudnn9" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + IMAGE_TYPE: "ray" + REQUIREMENTS_FILE: "requirements_byod_{{matrix.python}}.txt" + depends_on: + - raycudabaseextra + + - name: ray-llmbaseextra-testdeps + label: "wanda: ray.py{{matrix.python}}.llm.base-extra-testdeps (cuda {{matrix.cuda}})" + wanda: ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml + matrix: + setup: + python: + - "3.11" + cuda: + - "12.8.1-cudnn" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + IMAGE_TYPE: "ray-llm" + REQUIREMENTS_FILE: "requirements_llm_byod_{{matrix.python}}.txt" + depends_on: + - ray-llmbaseextra + + - name: ray-mlcudabaseextra-testdeps + label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.ml.base-extra-testdeps" + wanda: ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml + matrix: + setup: + python: + - "3.9" + cuda: + - "12.1.1-cudnn8" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + IMAGE_TYPE: "ray-ml" + REQUIREMENTS_FILE: "requirements_ml_byod_{{matrix.python}}.txt" + depends_on: + - ray-mlcudabaseextra + - label: ":tapioca: build: anyscale py{{matrix.python}}-{{matrix.platform}} docker" key: anyscalebuild instance_type: release-medium + tags: + - oss commands: - bazel run //ci/ray_ci:build_in_docker -- anyscale --python-version {{matrix.python}} --platform {{matrix.platform}} @@ -12,8 +79,8 @@ steps: depends_on: - manylinux - forge - - raycudabaseextra - - raycpubaseextra + - raycpubaseextra-testdeps + - raycudabaseextra-testdeps matrix: setup: python: @@ -29,26 +96,30 @@ steps: - label: ":tapioca: build: anyscale-llm py{{matrix}} docker" key: anyscalellmbuild instance_type: release-medium + tags: + - oss commands: - bazel run //ci/ray_ci:build_in_docker -- anyscale --python-version {{matrix}} --platform cu12.8.1-cudnn --image-type ray-llm --upload depends_on: - manylinux - forge - - ray-llmbaseextra + - ray-llmbaseextra-testdeps matrix: - "3.11" - label: ":tapioca: build: anyscale-ml py{{matrix}} docker" key: anyscalemlbuild instance_type: release-medium + tags: + - oss commands: - bazel run //ci/ray_ci:build_in_docker -- anyscale --python-version {{matrix}} --platform cu12.1.1-cudnn8 --image-type ray-ml --upload depends_on: - manylinux - forge - - ray-mlcudabaseextra + - ray-mlcudabaseextra-testdeps matrix: # This list should be kept in sync with the list of supported Python in # release test suite. We don't have ray-ml release tests for Python 3.10 and 3.11 diff --git a/ci/build/build-anyscale-docker.sh b/ci/build/build-anyscale-docker.sh index 2e6db88d15f3..bcee1703e44c 100755 --- a/ci/build/build-anyscale-docker.sh +++ b/ci/build/build-anyscale-docker.sh @@ -3,17 +3,11 @@ set -euo pipefail SOURCE_IMAGE="$1" DEST_IMAGE="$2" -REQUIREMENTS="$3" -ECR="$4" - -DOCKER_BUILDKIT=1 docker build \ - --build-arg BASE_IMAGE="$SOURCE_IMAGE" \ - --build-arg PIP_REQUIREMENTS="$REQUIREMENTS" \ - -t "$DEST_IMAGE" \ - -f release/ray_release/byod/byod.Dockerfile \ - release/ray_release/byod +ECR="$3" # publish anyscale image aws ecr get-login-password --region us-west-2 | \ docker login --username AWS --password-stdin "$ECR" + +docker tag "$SOURCE_IMAGE" "$DEST_IMAGE" docker push "$DEST_IMAGE" diff --git a/ci/docker/ray.cpu.base-extra-testdeps.wanda.yaml b/ci/docker/ray.cpu.base-extra-testdeps.wanda.yaml new file mode 100644 index 000000000000..dbfa11ce5b11 --- /dev/null +++ b/ci/docker/ray.cpu.base-extra-testdeps.wanda.yaml @@ -0,0 +1,10 @@ +name: "$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra-testdeps" +froms: ["cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra"] +dockerfile: release/ray_release/byod/byod.Dockerfile +srcs: + - release/ray_release/byod/requirements_byod_$PYTHON_VERSION.txt +build_args: + - BASE_IMAGE=cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra + - PIP_REQUIREMENTS=release/ray_release/byod/$REQUIREMENTS_FILE +tags: + - cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra-testdeps diff --git a/ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml b/ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml new file mode 100644 index 000000000000..c27e49f812dd --- /dev/null +++ b/ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml @@ -0,0 +1,10 @@ +name: "$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra-testdeps" +froms: ["cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra"] +dockerfile: release/ray_release/byod/byod.Dockerfile +srcs: + - release/ray_release/byod/$REQUIREMENTS_FILE +build_args: + - BASE_IMAGE=cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra + - PIP_REQUIREMENTS=release/ray_release/byod/$REQUIREMENTS_FILE +tags: + - cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra-testdeps diff --git a/ci/ray_ci/anyscale_docker_container.py b/ci/ray_ci/anyscale_docker_container.py index ccf847590ceb..ff8499233daa 100644 --- a/ci/ray_ci/anyscale_docker_container.py +++ b/ci/ray_ci/anyscale_docker_container.py @@ -17,13 +17,12 @@ def run(self) -> None: tag = self._get_canonical_tag() ray_image = f"rayproject/{self.image_type}:{tag}" anyscale_image = f"{aws_registry}/anyscale/{self.image_type}:{tag}" - requirement = self._get_requirement_file() gce_credentials = get_global_config()["aws2gce_credentials"] cmds = [ # build docker image "./ci/build/build-anyscale-docker.sh " - + f"{ray_image} {anyscale_image} {requirement} {aws_registry}", + + f"{ray_image} {anyscale_image} {aws_registry}", # gcloud login f"./release/gcloud_docker_login.sh {gce_credentials}", "export PATH=$(pwd)/google-cloud-sdk/bin:$PATH", @@ -45,14 +44,3 @@ def run(self) -> None: def _should_upload(self) -> bool: return self.upload - - def _get_requirement_file(self) -> str: - if self.image_type == "ray-ml": - prefix = "requirements_ml" - elif self.image_type == "ray-llm": - prefix = "requirements_llm" - else: - prefix = "requirements" - postfix = self.python_version - - return f"{prefix}_byod_{postfix}.txt" diff --git a/ci/ray_ci/builder.py b/ci/ray_ci/builder.py index 66fbe735f4ae..e6d02b251c73 100644 --- a/ci/ray_ci/builder.py +++ b/ci/ray_ci/builder.py @@ -172,7 +172,7 @@ def build_anyscale( for p in platform: RayDockerContainer( python_version, p, image_type, architecture, canonical_tag, upload=False - ).run(use_base_extra=True) + ).run(use_base_extra_testdeps=True) AnyscaleDockerContainer( python_version, p, image_type, architecture, canonical_tag, upload ).run() diff --git a/ci/ray_ci/ray_docker_container.py b/ci/ray_ci/ray_docker_container.py index 799310e009bb..e58532b10692 100644 --- a/ci/ray_ci/ray_docker_container.py +++ b/ci/ray_ci/ray_docker_container.py @@ -14,13 +14,13 @@ class RayDockerContainer(DockerContainer): Container for building and publishing ray docker images """ - def run(self, use_base_extra: bool = False) -> None: + def run(self, use_base_extra_testdeps: bool = False) -> None: """ Build and publish ray docker images """ assert "RAYCI_BUILD_ID" in os.environ, "RAYCI_BUILD_ID not set" rayci_build_id = os.environ["RAYCI_BUILD_ID"] - base_name = "base" if not use_base_extra else "base-extra" + base_name = "base" if not use_base_extra_testdeps else "base-extra-testdeps" if self.architecture == DEFAULT_ARCHITECTURE: suffix = base_name else: diff --git a/ci/ray_ci/test_anyscale_docker_container.py b/ci/ray_ci/test_anyscale_docker_container.py index 192fcfa2c051..5214b2c6a056 100644 --- a/ci/ray_ci/test_anyscale_docker_container.py +++ b/ci/ray_ci/test_anyscale_docker_container.py @@ -58,29 +58,13 @@ def _mock_run_script(input: List[str]) -> None: == [ "./ci/build/build-anyscale-docker.sh " f"rayproject/ray-ml:123456-{pv}-cu121 " - f"{aws_prj}:123456-{pv}-cu121 requirements_ml_byod_{v}.txt {aws_ecr}", + f"{aws_prj}:123456-{pv}-cu121 {aws_ecr}", f"./release/gcloud_docker_login.sh {gce_credentials}", "export PATH=$(pwd)/google-cloud-sdk/bin:$PATH", ] + push_cmds_want ) - def test_requirements_file(self) -> None: - container = AnyscaleDockerContainer("3.11", "cu12.1.1-cudnn8", "ray-ml") - assert container._get_requirement_file() == "requirements_ml_byod_3.11.txt" - - container = AnyscaleDockerContainer("3.9", "cu12.1.1-cudnn8", "ray-ml") - assert container._get_requirement_file() == "requirements_ml_byod_3.9.txt" - - container = AnyscaleDockerContainer("3.11", "cu12.4.1-cudnn", "ray-llm") - assert container._get_requirement_file() == "requirements_llm_byod_3.11.txt" - - container = AnyscaleDockerContainer("3.9", "cpu", "ray") - assert container._get_requirement_file() == "requirements_byod_3.9.txt" - - container = AnyscaleDockerContainer("3.12", "cpu", "ray") - assert container._get_requirement_file() == "requirements_byod_3.12.txt" - if __name__ == "__main__": sys.exit(pytest.main(["-vv", __file__])) From 1818e21f25e110afdd22d247e395039a7a44b27b Mon Sep 17 00:00:00 2001 From: Jiang Wu Date: Sun, 24 Aug 2025 15:05:20 -0700 Subject: [PATCH 241/634] [data.llm] Skip safetensor file downloads for runai streamer mode (#55662) Signed-off-by: Jiang Wu Signed-off-by: Jiang Wu --- .../_internal/batch/stages/vllm_engine_stage.py | 11 ++++++++++- .../llm/_internal/common/utils/cloud_utils.py | 17 ++++++++++++++++- .../_internal/common/utils/download_utils.py | 16 ++++++++++++++-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py b/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py index 5395e448621e..092622d70bea 100644 --- a/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py +++ b/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py @@ -465,11 +465,20 @@ def __init__( if self.max_pending_requests > 0: logger.info("Max pending requests is set to %d", self.max_pending_requests) + exclude_safetensors = self.engine_kwargs.get("load_format") in [ + "runai_streamer", + "tensorizer", + ] + if exclude_safetensors: + download_model = NodeModelDownloadable.EXCLUDE_SAFETENSORS + else: + download_model = NodeModelDownloadable.MODEL_AND_TOKENIZER + # Download the model if needed. model_source = download_model_files( model_id=self.model, mirror_config=None, - download_model=NodeModelDownloadable.MODEL_AND_TOKENIZER, + download_model=download_model, download_extra_files=False, ) diff --git a/python/ray/llm/_internal/common/utils/cloud_utils.py b/python/ray/llm/_internal/common/utils/cloud_utils.py index 209b3a30ec15..0db75809efd5 100644 --- a/python/ray/llm/_internal/common/utils/cloud_utils.py +++ b/python/ray/llm/_internal/common/utils/cloud_utils.py @@ -233,6 +233,7 @@ def download_files( path: str, bucket_uri: str, substrings_to_include: Optional[List[str]] = None, + suffixes_to_exclude: Optional[List[str]] = None, ) -> None: """Download files from cloud storage to a local directory. @@ -240,6 +241,7 @@ def download_files( path: Local directory where files will be downloaded bucket_uri: URI of cloud directory substrings_to_include: Only include files containing these substrings + suffixes_to_exclude: Exclude certain files from download (e.g .safetensors) """ try: fs, source_path = CloudFileSystem.get_fs_and_path(bucket_uri) @@ -266,6 +268,11 @@ def download_files( ): continue + # Check if file matches suffixes to exclude filter + if suffixes_to_exclude: + if any(rel_path.endswith(suffix) for suffix in suffixes_to_exclude): + continue + # Create destination directory if needed if "/" in rel_path: dest_dir = os.path.join(path, os.path.dirname(rel_path)) @@ -283,7 +290,10 @@ def download_files( @staticmethod def download_model( - destination_path: str, bucket_uri: str, tokenizer_only: bool + destination_path: str, + bucket_uri: str, + tokenizer_only: bool, + exclude_safetensors: bool = False, ) -> None: """Download a model from cloud storage. @@ -294,6 +304,7 @@ def download_model( destination_path: Path where the model will be stored bucket_uri: URI of the cloud directory containing the model tokenizer_only: If True, only download tokenizer-related files + exclude_safetensors: If True, skip download of safetensor files """ try: fs, source_path = CloudFileSystem.get_fs_and_path(bucket_uri) @@ -333,10 +344,14 @@ def download_model( tokenizer_file_substrings = ( ["tokenizer", "config.json"] if tokenizer_only else [] ) + + safetensors_to_exclude = [".safetensors"] if exclude_safetensors else None + CloudFileSystem.download_files( path=destination_dir, bucket_uri=bucket_uri, substrings_to_include=tokenizer_file_substrings, + suffixes_to_exclude=safetensors_to_exclude, ) except Exception as e: diff --git a/python/ray/llm/_internal/common/utils/download_utils.py b/python/ray/llm/_internal/common/utils/download_utils.py index 2d4e0db908d0..88d8e208a226 100644 --- a/python/ray/llm/_internal/common/utils/download_utils.py +++ b/python/ray/llm/_internal/common/utils/download_utils.py @@ -24,6 +24,7 @@ class NodeModelDownloadable(enum.Enum): MODEL_AND_TOKENIZER = enum.auto() TOKENIZER_ONLY = enum.auto() + EXCLUDE_SAFETENSORS = enum.auto() NONE = enum.auto() def __bool__(self): @@ -36,7 +37,11 @@ def union(self, other: "NodeModelDownloadable") -> "NodeModelDownloadable": or other == NodeModelDownloadable.MODEL_AND_TOKENIZER ): return NodeModelDownloadable.MODEL_AND_TOKENIZER - + if ( + self == NodeModelDownloadable.EXCLUDE_SAFETENSORS + or other == NodeModelDownloadable.EXCLUDE_SAFETENSORS + ): + return NodeModelDownloadable.EXCLUDE_SAFETENSORS if ( self == NodeModelDownloadable.TOKENIZER_ONLY or other == NodeModelDownloadable.TOKENIZER_ONLY @@ -111,11 +116,13 @@ class CloudModelDownloader(CloudModelAccessor): def get_model( self, tokenizer_only: bool, + exclude_safetensors: bool = False, ) -> str: """Gets a model from cloud storage and stores it locally. Args: tokenizer_only: whether to download only the tokenizer files. + exclude_safetensors: whether to download safetensors files to disk. Returns: file path of model if downloaded, else the model id. """ @@ -135,10 +142,13 @@ def get_model( # This ensures that subsequent processes don't duplicate work. with FileLock(lock_path, timeout=0): try: + if exclude_safetensors: + logger.info("Skipping download of safetensors files.") CloudFileSystem.download_model( destination_path=path, bucket_uri=bucket_uri, tokenizer_only=tokenizer_only, + exclude_safetensors=exclude_safetensors, ) logger.info( "Finished downloading %s for %s from %s storage", @@ -282,7 +292,9 @@ def download_model_files( if download_model != NodeModelDownloadable.NONE: model_path_or_id = downloader.get_model( - tokenizer_only=download_model == NodeModelDownloadable.TOKENIZER_ONLY + tokenizer_only=download_model == NodeModelDownloadable.TOKENIZER_ONLY, + exclude_safetensors=download_model + == NodeModelDownloadable.EXCLUDE_SAFETENSORS, ) if download_extra_files: From fb37604a0dcf91afd5e08d620c296a0b67a3f0ca Mon Sep 17 00:00:00 2001 From: Kai-Hsun Chen Date: Sun, 24 Aug 2025 22:15:14 -0700 Subject: [PATCH 242/634] [core][chore] Rename `obj_ref` to `object_refs` because `invocation` returns a list of ObjectRef in most cases (#55868) Signed-off-by: kaihsun --- python/ray/actor.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/ray/actor.py b/python/ray/actor.py index fba4a4d39bb2..4e9e22f1a7dd 100644 --- a/python/ray/actor.py +++ b/python/ray/actor.py @@ -855,14 +855,18 @@ def invocation(args, kwargs): if self._decorator is not None: invocation = self._decorator(invocation) - obj_ref = invocation(args, kwargs) + object_refs = invocation(args, kwargs) if tensor_transport != TensorTransportEnum.OBJECT_STORE: + # Currently, we only support transfer tensor out-of-band when + # num_returns is 1. + assert isinstance(object_refs, ObjectRef) + object_ref = object_refs gpu_object_manager = ray._private.worker.global_worker.gpu_object_manager gpu_object_manager.add_gpu_object_ref( - obj_ref, self._actor, tensor_transport + object_ref, self._actor, tensor_transport ) - return obj_ref + return object_refs def __getstate__(self): return { From b47383c4f22f810ec505342f053b184ff33f6ac3 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 25 Aug 2025 09:26:49 -0700 Subject: [PATCH 243/634] [Core] [Doc] Lifecycle of a task (#55496) Signed-off-by: Jiajun Yao --- doc/source/ray-core/internals.rst | 12 +++ .../ray-core/internals/task-lifecycle.rst | 83 +++++++++++++++++++ doc/source/ray-core/walkthrough.rst | 3 +- 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 doc/source/ray-core/internals.rst create mode 100644 doc/source/ray-core/internals/task-lifecycle.rst diff --git a/doc/source/ray-core/internals.rst b/doc/source/ray-core/internals.rst new file mode 100644 index 000000000000..69505a23c1ad --- /dev/null +++ b/doc/source/ray-core/internals.rst @@ -0,0 +1,12 @@ +.. _ray-core-internals: + +Internals +========= + +This section provides a look into some of Ray Core internals. It's primarily intended for advanced users and developers of Ray Core. +For the high level architecture overview, please refer to the `whitepaper `__. + +.. toctree:: + :maxdepth: 1 + + internals/task-lifecycle.rst diff --git a/doc/source/ray-core/internals/task-lifecycle.rst b/doc/source/ray-core/internals/task-lifecycle.rst new file mode 100644 index 000000000000..a3d47d9f7017 --- /dev/null +++ b/doc/source/ray-core/internals/task-lifecycle.rst @@ -0,0 +1,83 @@ +.. _task-lifecycle: + +Task Lifecycle +============== + +This doc talks about the lifecycle of a task in Ray Core, including how tasks are defined, scheduled and executed. +We will use the following code as an example and the internals are based on Ray 2.48. + + +.. testcode:: + + import ray + + @ray.remote + def my_task(arg): + return f"Hello, {arg}!" + + obj_ref = my_task.remote("Ray") + print(ray.get(obj_ref)) + +.. testoutput:: + + Hello, Ray! + + +Defining a remote function +-------------------------- + +The first step in the task lifecycle is defining a remote function using the :func:`ray.remote` decorator. :func:`ray.remote` wraps the Python function and returns an instance of `RemoteFunction `__. +``RemoteFunction`` stores the underlying function and all the user specified Ray task :meth:`options ` such as ``num_cpus``. + + +Invoking a remote function +-------------------------- + +Once a remote function is defined, it can be invoked using the `.remote()` method. Each invocation of a remote function creates a Ray task. This method submits the task for execution and returns an object reference (``ObjectRef``) that can be used to retrieve the result later. +Under the hood, `.remote()` does the following: + +1. `Pickles the underlying function `__ into bytes and `stores the bytes in GCS key-value store `__ with a `key `__ so that, later on, the remote executor (the core worker process that will execute the task) can get the bytes, unpickle, and execute the function. This is done once per remote function definition instead of once per invocation. +2. `Calls `__ Cython `submit_task `__ which `prepares `__ the arguments (3 types) and calls the C++ `CoreWorker::SubmitTask `__. + + 1. Pass-by-reference argument: the argument is an ``ObjectRef``. + 2. Pass-by-value inline argument: the argument is a `small `__ Python object and the total size of such arguments so far is below the `threshold `__. In this case, it will be pickled, sent to the remote executor (as part of the ``PushTask`` RPC), and unpickled there. This is called inlining and plasma store is not involved in this case. + 3. Pass-by-value non-inline argument: the argument is a normal Python object but it doesn't meet the inline criteria (e.g. size is too big), it is `put `__ in the local plasma store and the argument is replaced by the generated ``ObjectRef``, so it's effectively equivalent to ``.remote(ray.put(arg))``. + +3. ``CoreWorker`` `builds `__ a `TaskSpecification `__ that contains all the information about the task including the `ID `__ of the function, all the user specified options and the arguments. This spec will be sent to the executor for execution. +4. The TaskSpecification is `submitted `__ to `NormalTaskSubmitter `__ asynchronously. This means the ``.remote()`` call returns immediately and the task is scheduled and executed asynchronously. + +Scheduling a task +----------------- + +Once the task is submitted to ``NormalTaskSubmitter``, a worker process on some Ray node is selected to execute the task and this process is called scheduling. + +1. ``NormalTaskSubmitter`` first `waits `__ for all the ``ObjectRef`` arguments to be available. Available means tasks that produce those ``ObjectRef``\s finished execution and the data is available somewhere in the cluster. + + 1. If the object pointed to by the ``ObjectRef`` is in the plasma store, the ``ObjectRef`` itself is sent to the executor and the executor will resolve the ``ObjectRef`` to the actual data (pull from remote plasma store if needed) before calling the user function. + 2. If the object pointed to by the ``ObjectRef`` is in the caller memory store, the data is `inlined `__ and sent to the executor as part of the ``PushTask`` RPC just like other pass-by-value inline arguments. + +2. Once all the arguments are available, ``NormalTaskSubmitter`` will try to find an idle worker to execute the task. ``NormalTaskSubmitter`` gets workers for task execution from raylet via a process called worker lease and this is where scheduling happens. + Specifically, it will `send `__ a ``RequestWorkerLease`` RPC to a `selected `__ (it's either the local raylet or a data-locality-favored raylet) raylet for a worker lease. +3. Raylet `handles `__ the ``RequestWorkerLease`` RPC. +4. When the ``RequestWorkerLease`` RPC returns and a leased worker address is included in the response, a worker lease is granted to the caller to execute the task. If the ``RequestWorkerLease`` response contains another raylet address instead, ``NormalTaskSubmitter`` will then worker lease from the specified raylet. This process continues until a worker lease is obtained. + +Executing a task +---------------- + +Once a leased worker is obtained, the task execution starts. + +1. ``NormalTaskSubmitter`` `sends `__ a ``PushTask`` RPC to the leased worker with the ``TaskSpecification`` to execute. +2. The executor `receives `__ the ``PushTask`` RPC and executes (`1 `__ -> `2 `__ -> `3 `__ -> `4 `__ -> `5 `__) the task. +3. First step of executing the task is `getting `__ all the pass-by-reference arguments from the local plasma store (data is already pulled from remote plasma store to the local plasma store during scheduling). +4. Then the executor `gets `__ the pickled function bytes from GCS key-value store and unpickles it. +5. The next step is `unpickling `__ the arguments. +6. Finally, the user function is `called `__. + +Getting the return value +------------------------ + +After the user function is executed, the caller can get the return values. + +1. After the user function returns, the executor `gets and stores `__ all the return values. If the return value is a `small `__ object and the total size of such return values so far is below the `threshold `__, it is returned directly to the caller as part of the ``PushTask`` RPC response. `Otherwise `__, it is put in the local plasma store and the reference is returned to the caller. +2. When the caller `receives `__ the ``PushTask`` RPC response, it `stores `__ the return values (actual data if the return value is small or a special value indicating the data is in plasma store if the return value is big) in the local memory store. +3. When the return value is `added `__ to the local memory store, ``ray.get()`` is `unblocked `__ and returns the value directly if the object is small, or it will `get `__ from the local plasma store (pull from remote plasma store first if needed) if the object is big. diff --git a/doc/source/ray-core/walkthrough.rst b/doc/source/ray-core/walkthrough.rst index 1fb7f8a8b3c6..938b219dedc4 100644 --- a/doc/source/ray-core/walkthrough.rst +++ b/doc/source/ray-core/walkthrough.rst @@ -11,6 +11,7 @@ What's Ray Core? User Guides Examples api/index + Internals Ray Core is a powerful distributed computing framework that provides a small set of essential primitives (tasks, actors, and objects) for building and scaling distributed applications. @@ -58,7 +59,7 @@ Here's a simple example: Calling an Actor ---------------- -While tasks are stateless, Ray actors allow you to create stateful workers that maintain their internal state between method calls. +While tasks are stateless, Ray actors allow you to create stateful workers that maintain their internal state between method calls. When you instantiate a Ray actor: 1. Ray starts a dedicated worker process somewhere in your cluster From 5b3f4a03cb2d1fb66acdeef19081911bab4bd1af Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Mon, 25 Aug 2025 10:00:44 -0700 Subject: [PATCH 244/634] [serve] Allow same event loop handle shutdown from sync context (#55551) ## Why are these changes needed? In the case user calls `serve.shutdown()`, we'd still want to be able to shutdown the handle if user has initialized it running in the same event loop. The current behaior may throw a runtime error. In order to block on the shutdown result in the same event loop without causing deadlock, the shutdown sequence in `CurrentLoopRouter` needs to happen in a separate thread (instead of the same event loop). ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: akyang-anyscale Signed-off-by: alexyang --- ci/docker/min.build.Dockerfile | 1 + doc/source/serve/api/index.md | 1 + python/ray/serve/__init__.py | 2 + python/ray/serve/_private/client.py | 45 ++++- python/ray/serve/api.py | 20 +++ python/ray/serve/handle.py | 12 +- python/ray/serve/tests/conftest.py | 12 +- .../ray/serve/tests/test_handle_same_loop.py | 16 +- python/ray/serve/tests/test_standalone.py | 155 ++++++++++++++++++ 9 files changed, 246 insertions(+), 18 deletions(-) diff --git a/ci/docker/min.build.Dockerfile b/ci/docker/min.build.Dockerfile index 00e6082788d6..fa88dbfb435d 100644 --- a/ci/docker/min.build.Dockerfile +++ b/ci/docker/min.build.Dockerfile @@ -31,6 +31,7 @@ elif [[ "${EXTRA_DEPENDENCY}" == "default" ]]; then pip-compile -o min_requirements.txt python/setup.py --extra default elif [[ "${EXTRA_DEPENDENCY}" == "serve" ]]; then echo "httpx==0.27.2" >> /tmp/min_build_requirements.txt + echo "pytest-asyncio==1.1.0" >> /tmp/min_build_requirements.txt pip-compile -o min_requirements.txt /tmp/min_build_requirements.txt python/setup.py --extra "serve-grpc" rm /tmp/min_build_requirements.txt fi diff --git a/doc/source/serve/api/index.md b/doc/source/serve/api/index.md index 75e60e8577d6..5b5620c86022 100644 --- a/doc/source/serve/api/index.md +++ b/doc/source/serve/api/index.md @@ -70,6 +70,7 @@ See the [model composition guide](serve-model-composition) for how to update cod serve.delete serve.status serve.shutdown + serve.shutdown_async ``` ### Configurations diff --git a/python/ray/serve/__init__.py b/python/ray/serve/__init__.py index 0d5b38cf84fe..379b118ac1f5 100644 --- a/python/ray/serve/__init__.py +++ b/python/ray/serve/__init__.py @@ -19,6 +19,7 @@ run, run_many, shutdown, + shutdown_async, start, status, ) @@ -47,6 +48,7 @@ "HTTPOptions", "get_replica_context", "shutdown", + "shutdown_async", "ingress", "deployment", "run", diff --git a/python/ray/serve/_private/client.py b/python/ray/serve/_private/client.py index ffec15fa89c2..b871553abda0 100644 --- a/python/ray/serve/_private/client.py +++ b/python/ray/serve/_private/client.py @@ -1,3 +1,4 @@ +import asyncio import logging import random import time @@ -80,18 +81,31 @@ def http_config(self): def __reduce__(self): raise RayServeException(("Ray Serve client cannot be serialized.")) - def shutdown_cached_handles(self, _skip_asyncio_check: bool = False): + def shutdown_cached_handles(self): """Shuts down all cached handles. Remove the reference to the cached handles so that they can be garbage collected. """ for cache_key in list(self.handle_cache): - self.handle_cache[cache_key].shutdown( - _skip_asyncio_check=_skip_asyncio_check - ) + self.handle_cache[cache_key].shutdown() + del self.handle_cache[cache_key] + + async def shutdown_cached_handles_async(self): + """Shuts down all cached handles asynchronously. + + Remove the reference to the cached handles so that they can be + garbage collected. + """ + + async def shutdown_task(cache_key): + await self.handle_cache[cache_key].shutdown_async() del self.handle_cache[cache_key] + await asyncio.gather( + *[shutdown_task(cache_key) for cache_key in list(self.handle_cache)] + ) + def shutdown(self, timeout_s: float = 30.0) -> None: """Completely shut down the connected Serve instance. @@ -113,6 +127,29 @@ def shutdown(self, timeout_s: float = 30.0) -> None: ) self._shutdown = True + async def shutdown_async(self, timeout_s: float = 30.0) -> None: + """Completely shut down the connected Serve instance. + + Shuts down all processes and deletes all state associated with the + instance. + """ + await self.shutdown_cached_handles_async() + + if ray.is_initialized() and not self._shutdown: + try: + await asyncio.wait_for( + self._controller.graceful_shutdown.remote(), timeout=timeout_s + ) + except ray.exceptions.RayActorError: + # Controller has been shut down. + pass + except TimeoutError: + logger.warning( + f"Controller failed to shut down within {timeout_s}s. " + "Check controller logs for more details." + ) + self._shutdown = True + def _wait_for_deployment_healthy(self, name: str, timeout_s: int = -1): """Waits for the named deployment to enter "HEALTHY" status. diff --git a/python/ray/serve/api.py b/python/ray/serve/api.py index 8a852d6a1dc9..5ae14f3c8afd 100644 --- a/python/ray/serve/api.py +++ b/python/ray/serve/api.py @@ -138,6 +138,26 @@ def shutdown(): _set_global_client(None) +@PublicAPI(stability="alpha") +async def shutdown_async(): + """Completely shut down Serve on the cluster asynchronously. + + Deletes all applications and shuts down Serve system actors. + """ + + try: + client = _get_global_client() + except RayServeException: + logger.info( + "Nothing to shut down. There's no Serve application " + "running on this Ray cluster." + ) + return + + await client.shutdown_async() + _set_global_client(None) + + @DeveloperAPI def get_replica_context() -> ReplicaContext: """Returns the deployment and replica tag from within a replica at runtime. diff --git a/python/ray/serve/handle.py b/python/ray/serve/handle.py index 5356a18db2de..929cb6682c27 100644 --- a/python/ray/serve/handle.py +++ b/python/ray/serve/handle.py @@ -212,17 +212,17 @@ def _remote( def __getattr__(self, name): return self.options(method_name=name) - def shutdown(self, _skip_asyncio_check: bool = False): + def shutdown(self): if self._router: shutdown_future = self._router.shutdown() if self._is_router_running_in_separate_loop(): shutdown_future.result() else: - if not _skip_asyncio_check: - raise RuntimeError( - "Sync methods should not be called from within an `asyncio` event " - "loop. Use `await handle.shutdown_async()` instead." - ) + logger.warning( + "Synchronously shutting down a router that's running in the same " + "event loop can only be done best effort. Please use " + "`shutdown_async` instead." + ) async def shutdown_async(self): if self._router: diff --git a/python/ray/serve/tests/conftest.py b/python/ray/serve/tests/conftest.py index 3ccfb9e45dc5..5de04057db06 100644 --- a/python/ray/serve/tests/conftest.py +++ b/python/ray/serve/tests/conftest.py @@ -7,6 +7,7 @@ import httpx import pytest +import pytest_asyncio import ray from ray import serve @@ -152,13 +153,22 @@ def _shared_serve_instance(): yield _get_global_client() +@pytest_asyncio.fixture +async def serve_instance_async(_shared_serve_instance): + yield _shared_serve_instance + # Clear all state for 2.x applications and deployments. + _shared_serve_instance.delete_all_apps() + # Clear the ServeHandle cache between tests to avoid them piling up. + await _shared_serve_instance.shutdown_cached_handles_async() + + @pytest.fixture def serve_instance(_shared_serve_instance): yield _shared_serve_instance # Clear all state for 2.x applications and deployments. _shared_serve_instance.delete_all_apps() # Clear the ServeHandle cache between tests to avoid them piling up. - _shared_serve_instance.shutdown_cached_handles(_skip_asyncio_check=True) + _shared_serve_instance.shutdown_cached_handles() @pytest.fixture diff --git a/python/ray/serve/tests/test_handle_same_loop.py b/python/ray/serve/tests/test_handle_same_loop.py index 78dcde1963cb..056703fe4365 100644 --- a/python/ray/serve/tests/test_handle_same_loop.py +++ b/python/ray/serve/tests/test_handle_same_loop.py @@ -24,7 +24,7 @@ def _skip_test_if_router_running_in_separate_loop(): @pytest.mark.asyncio async def test_deployment_handle_works_with_await_when_router_in_same_loop( - serve_instance, _skip_test_if_router_running_in_separate_loop + serve_instance_async, _skip_test_if_router_running_in_separate_loop ): @serve.deployment class F: @@ -50,7 +50,7 @@ def __call__(self): @pytest.mark.asyncio async def test_deployment_handle_result_fails_in_async_context_but_await_succeeds( - serve_instance, _skip_test_if_router_running_in_separate_loop + serve_instance_async, _skip_test_if_router_running_in_separate_loop ): @serve.deployment class F: @@ -81,7 +81,9 @@ def __call__(self): @pytest.mark.asyncio -async def test_deployment_handle_configured_for_same_loop_via_init(serve_instance): +async def test_deployment_handle_configured_for_same_loop_via_init( + serve_instance_async, +): @serve.deployment class F: def __call__(self): @@ -119,7 +121,7 @@ async def __call__(self): @pytest.mark.asyncio async def test_deployment_handle_exception_propagation_in_same_loop( - serve_instance, _skip_test_if_router_running_in_separate_loop + serve_instance_async, _skip_test_if_router_running_in_separate_loop ): """Test that exceptions are properly propagated when router runs in same loop.""" @@ -136,7 +138,7 @@ def __call__(self): @pytest.mark.asyncio async def test_streaming_response_generator_in_same_loop( - serve_instance, _skip_test_if_router_running_in_separate_loop + serve_instance_async, _skip_test_if_router_running_in_separate_loop ): """Test that streaming responses work correctly when router runs in same loop.""" @@ -159,7 +161,7 @@ def generate_numbers(self, limit: int): @pytest.mark.asyncio async def test_concurrent_requests_in_same_loop( - serve_instance, _skip_test_if_router_running_in_separate_loop + serve_instance_async, _skip_test_if_router_running_in_separate_loop ): """Test that multiple concurrent requests work correctly in same loop mode.""" @@ -185,7 +187,7 @@ async def slow_operation(self, delay: float, value: str): @pytest.mark.asyncio async def test_request_cancellation_in_same_loop( - serve_instance, _skip_test_if_router_running_in_separate_loop + serve_instance_async, _skip_test_if_router_running_in_separate_loop ): """Test that request cancellation works correctly when router runs in same loop.""" signal_actor = SignalActor.remote() diff --git a/python/ray/serve/tests/test_standalone.py b/python/ray/serve/tests/test_standalone.py index ba4b506ded42..2ddb36190acc 100644 --- a/python/ray/serve/tests/test_standalone.py +++ b/python/ray/serve/tests/test_standalone.py @@ -125,6 +125,53 @@ def check_dead(): wait_for_condition(check_dead) +@pytest.mark.asyncio +async def test_shutdown_async(ray_shutdown): + ray.init(num_cpus=8) + serve.start(http_options=dict(port=8003)) + gcs_client = GcsClient(address=ray.get_runtime_context().gcs_address) + cluster_node_info_cache = create_cluster_node_info_cache(gcs_client) + cluster_node_info_cache.update() + + @serve.deployment + def f(): + pass + + serve.run(f.bind()) + + actor_names = [ + SERVE_CONTROLLER_NAME, + format_actor_name( + SERVE_PROXY_NAME, + cluster_node_info_cache.get_alive_nodes()[0][0], + ), + ] + + def check_alive(): + alive = True + for actor_name in actor_names: + try: + ray.get_actor(actor_name, namespace=SERVE_NAMESPACE) + except ValueError: + alive = False + return alive + + wait_for_condition(check_alive) + + await serve.shutdown_async() + + def check_dead(): + for actor_name in actor_names: + try: + ray.get_actor(actor_name, namespace=SERVE_NAMESPACE) + return False + except ValueError: + pass + return True + + wait_for_condition(check_dead) + + def test_single_app_shutdown_actors(ray_shutdown): """Tests serve.shutdown() works correctly in single-app case @@ -165,6 +212,47 @@ def check_dead(): wait_for_condition(check_dead) +@pytest.mark.asyncio +async def test_single_app_shutdown_actors_async(ray_shutdown): + """Tests serve.shutdown_async() works correctly in single-app case + + Ensures that after deploying a (nameless) app using serve.run(), serve.shutdown_async() + deletes all actors (controller, http proxy, all replicas) in the "serve" namespace. + """ + address = ray.init(num_cpus=8)["address"] + serve.start(http_options=dict(port=8003)) + + @serve.deployment + def f(): + pass + + serve.run(f.bind(), name="app") + + actor_names = { + "ServeController", + "ProxyActor", + "ServeReplica:app:f", + } + + def check_alive(): + actors = list_actors( + address=address, + filters=[("ray_namespace", "=", SERVE_NAMESPACE), ("state", "=", "ALIVE")], + ) + return {actor["class_name"] for actor in actors} == actor_names + + def check_dead(): + actors = list_actors( + address=address, + filters=[("ray_namespace", "=", SERVE_NAMESPACE), ("state", "=", "ALIVE")], + ) + return len(actors) == 0 + + wait_for_condition(check_alive) + await serve.shutdown_async() + wait_for_condition(check_dead) + + def test_multi_app_shutdown_actors(ray_shutdown): """Tests serve.shutdown() works correctly in multi-app case. @@ -207,6 +295,49 @@ def check_dead(): wait_for_condition(check_dead) +@pytest.mark.asyncio +async def test_multi_app_shutdown_actors_async(ray_shutdown): + """Tests serve.shutdown_async() works correctly in multi-app case. + + Ensures that after deploying multiple distinct applications, serve.shutdown_async() + deletes all actors (controller, http proxy, all replicas) in the "serve" namespace. + """ + address = ray.init(num_cpus=8)["address"] + serve.start(http_options=dict(port=8003)) + + @serve.deployment + def f(): + pass + + serve.run(f.bind(), name="app1", route_prefix="/app1") + serve.run(f.bind(), name="app2", route_prefix="/app2") + + actor_names = { + "ServeController", + "ProxyActor", + "ServeReplica:app1:f", + "ServeReplica:app2:f", + } + + def check_alive(): + actors = list_actors( + address=address, + filters=[("ray_namespace", "=", SERVE_NAMESPACE), ("state", "=", "ALIVE")], + ) + return {actor["class_name"] for actor in actors} == actor_names + + def check_dead(): + actors = list_actors( + address=address, + filters=[("ray_namespace", "=", SERVE_NAMESPACE), ("state", "=", "ALIVE")], + ) + return len(actors) == 0 + + wait_for_condition(check_alive) + await serve.shutdown_async() + wait_for_condition(check_dead) + + def test_deployment(ray_cluster): # https://github.com/ray-project/ray/issues/11437 @@ -518,6 +649,30 @@ def __call__(self, *args): assert len(serve.status().applications) == 1 +@pytest.mark.asyncio +async def test_serve_shutdown_async(ray_shutdown): + ray.init(namespace="serve") + serve.start() + + @serve.deployment + class A: + def __call__(self, *args): + return "hi" + + serve.run(A.bind()) + + assert len(serve.status().applications) == 1 + + await serve.shutdown_async() + serve.start() + + assert len(serve.status().applications) == 0 + + serve.run(A.bind()) + + assert len(serve.status().applications) == 1 + + def test_instance_in_non_anonymous_namespace(ray_shutdown): # Can start instance in non-anonymous namespace. ray.init(namespace="foo") From dd68eee648917d94d134781796f6f66116594f85 Mon Sep 17 00:00:00 2001 From: Potato Date: Tue, 26 Aug 2025 01:06:19 +0800 Subject: [PATCH 245/634] [DOC] Fixed a number of typos in doc by gpt5 agent (#55873) --- .../vms/user-guides/large-cluster-best-practices.rst | 2 +- doc/source/data/aggregations.rst | 2 +- doc/source/data/batch_inference.rst | 6 +++--- doc/source/data/working-with-pytorch.rst | 2 +- doc/source/ray-core/compiled-graph/ray-compiled-graph.rst | 2 +- doc/source/ray-core/handling-dependencies.rst | 4 ++-- .../e2e-multimodal-ai-workloads/doggos/doggos/serve.py | 2 +- doc/source/rllib/multi-agent-envs.rst | 2 +- doc/source/rllib/rllib-algorithms.rst | 2 +- doc/source/rllib/rllib-examples.rst | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/source/cluster/vms/user-guides/large-cluster-best-practices.rst b/doc/source/cluster/vms/user-guides/large-cluster-best-practices.rst index 278673bf7734..6dcff6758ab0 100644 --- a/doc/source/cluster/vms/user-guides/large-cluster-best-practices.rst +++ b/doc/source/cluster/vms/user-guides/large-cluster-best-practices.rst @@ -127,7 +127,7 @@ General recommendations with AWS instance types: should help with this). If your CPU utilization is low add GPUs, or vice versa. * The exact ratio will be very dependent on your workload. -* Once you find a good ratio, you should be able to scale up and and keep the +* Once you find a good ratio, you should be able to scale up and keep the same ratio. * You can’t infinitely scale forever. Eventually, as you add more machines your performance improvements will become sub-linear/not worth it. There may not diff --git a/doc/source/data/aggregations.rst b/doc/source/data/aggregations.rst index ffaa2263736c..3ab3da449acd 100644 --- a/doc/source/data/aggregations.rst +++ b/doc/source/data/aggregations.rst @@ -145,7 +145,7 @@ Here's an example of creating a custom aggregator that calculates the Mean of va .. note:: Internally, aggregations support both the :ref:`hash-shuffle backend ` and the :ref:`range based backend `. - Hash-shuffling can provide better performance for aggregations in certain cases. For more information see `comparision between hash based shuffling and Range Based shuffling approach `_ . + Hash-shuffling can provide better performance for aggregations in certain cases. For more information see `comparison between hash based shuffling and Range Based shuffling approach `_ . To use the hash-shuffle algorithm for aggregations, you need to set the shuffle strategy explicitly: ``ray.data.DataContext.get_current().shuffle_strategy = ShuffleStrategy.HASH_SHUFFLE`` before creating a ``Dataset`` diff --git a/doc/source/data/batch_inference.rst b/doc/source/data/batch_inference.rst index 1e6b1f9c996b..8d80808e5465 100644 --- a/doc/source/data/batch_inference.rst +++ b/doc/source/data/batch_inference.rst @@ -55,7 +55,7 @@ For how to configure batch inference, see :ref:`the configuration guide`_ - `Ray Compiled Graph talk at Ray Summit `_ -- `Heterogenous training with Ray Compiled Graph `_ +- `Heterogeneous training with Ray Compiled Graph `_ - `Distributed LLM inference with Ray Compiled Graph `_ Table of Contents diff --git a/doc/source/ray-core/handling-dependencies.rst b/doc/source/ray-core/handling-dependencies.rst index 440b0dad0507..272cb5aebe36 100644 --- a/doc/source/ray-core/handling-dependencies.rst +++ b/doc/source/ray-core/handling-dependencies.rst @@ -39,7 +39,7 @@ Preparing an environment using the Ray Cluster launcher The first way to set up dependencies is to prepare a single environment across the cluster before starting the Ray runtime. -- You can build all your files and dependencies into a container image and specify this in your your :ref:`Cluster YAML Configuration `. +- You can build all your files and dependencies into a container image and specify this in your :ref:`Cluster YAML Configuration `. - You can also install packages using ``setup_commands`` in the Ray Cluster configuration file (:ref:`reference `); these commands will be run as each node joins the cluster. Note that for production settings, it is recommended to build any necessary packages into a container image instead. @@ -613,7 +613,7 @@ The ``runtime_env`` is a Python dictionary or a Python class :class:`ray.runtime - Example: ``{"LD_LIBRARY_PATH": "${LD_LIBRARY_PATH}:/home/admin/my_lib"}`` - - Non-existant variable example: ``{"ENV_VAR_NOT_EXIST": "${ENV_VAR_NOT_EXIST}:/home/admin/my_lib"}`` -> ``ENV_VAR_NOT_EXIST=":/home/admin/my_lib"``. + - Non-existent variable example: ``{"ENV_VAR_NOT_EXIST": "${ENV_VAR_NOT_EXIST}:/home/admin/my_lib"}`` -> ``ENV_VAR_NOT_EXIST=":/home/admin/my_lib"``. - ``nsight`` (Union[str, Dict[str, str]]): specifies the config for the Nsight System Profiler. The value is either (1) "default", which refers to the `default config `_, or (2) a dict of Nsight System Profiler options and their values. See :ref:`here ` for more details on setup and usage. diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/serve.py b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/serve.py index 350d2a0cc789..da8ff6a1b778 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/serve.py +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/doggos/doggos/serve.py @@ -24,7 +24,7 @@ class ClassPredictor: def __init__(self, model_id, artifacts_dir, device="cuda"): """Initialize the model.""" - # Embdding model + # Embedding model self.processor = CLIPProcessor.from_pretrained(model_id) self.model = CLIPModel.from_pretrained(model_id) self.model.to(device=device) diff --git a/doc/source/rllib/multi-agent-envs.rst b/doc/source/rllib/multi-agent-envs.rst index 9c4a48e10a55..232f9516c23e 100644 --- a/doc/source/rllib/multi-agent-envs.rst +++ b/doc/source/rllib/multi-agent-envs.rst @@ -300,7 +300,7 @@ receives +1 reward. The losing player receives a -1 reward. To make the implementation easier, the aberration from the original game is that trying to place a piece on an already occupied field results in the board not changing at all, but the moving player receiving a -5 reward as a penalty (in the original game, this move is -simply not allowed and therefor can never happen). +simply not allowed and therefore can never happen). Here is your initial class scaffold for the Tic-Tac-Toe game: diff --git a/doc/source/rllib/rllib-algorithms.rst b/doc/source/rllib/rllib-algorithms.rst index dd8e84713c03..e5743f8e306a 100644 --- a/doc/source/rllib/rllib-algorithms.rst +++ b/doc/source/rllib/rllib-algorithms.rst @@ -218,7 +218,7 @@ Importance Weighted Actor-Learner Architecture (IMPALA) **IMPALA architecture:** In a training iteration, IMPALA requests samples from all EnvRunners asynchronously and the collected episodes are returned to the main algorithm process as Ray references rather than actual objects available on the local process. IMPALA then passes these episode references to the Learners for asynchronous updates of the model. - RLlib doesn't always synch back the weights to the EnvRunners right after a new model version is available. + RLlib doesn't always sync back the weights to the EnvRunners right after a new model version is available. To account for the EnvRunners being off-policy, IMPALA uses a procedure called v-trace, `described in the paper `__. IMPALA scales out on both axes, supporting multiple EnvRunners for sample collection and multiple GPU- or CPU-based Learners diff --git a/doc/source/rllib/rllib-examples.rst b/doc/source/rllib/rllib-examples.rst index f5842747d60c..50a894e33b2d 100644 --- a/doc/source/rllib/rllib-examples.rst +++ b/doc/source/rllib/rllib-examples.rst @@ -158,7 +158,7 @@ Curiosity Using curiosity is beneficial in sparse-reward environments where agents may struggle to find rewarding paths. However, count-based methods are only feasible for environments with small observation spaces. -- `Euclidian distance-based curiosity `__: +- `Euclidean distance-based curiosity `__: Uses Euclidean distance between states and the initial state to measure novelty, encouraging exploration by rewarding the agent for reaching "far away" regions of the environment. Suitable for sparse-reward tasks, where diverse exploration is key to success. From a215e177cfa1e0bec48cf43497807458e9c5a4c9 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 25 Aug 2025 10:32:51 -0700 Subject: [PATCH 246/634] [llm] disable sglang release test (#55884) sglang installation fails on the image Signed-off-by: Lonnie Liu --- release/release_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/release_tests.yaml b/release/release_tests.yaml index d62589e6f1b9..bfdb10067dd8 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -4586,7 +4586,7 @@ pytest -sv test_batch_vllm.py - name: llm_batch_sglang_llama - frequency: nightly + frequency: manual # TODO(ray-llm): fix this test and re-enable it. python: "3.11" group: llm-batch team: llm From 6d27fc246a647820ec5bf8ee822de8771ac50fa4 Mon Sep 17 00:00:00 2001 From: Ibrahim Rabbani Date: Mon, 25 Aug 2025 10:33:04 -0700 Subject: [PATCH 247/634] [core] Fixing broken links for public docs in error messages (#55823) Signed-off-by: irabbani --- python/ray/util/client/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/util/client/worker.py b/python/ray/util/client/worker.py index babd60a79012..c5a09c8b51eb 100644 --- a/python/ray/util/client/worker.py +++ b/python/ray/util/client/worker.py @@ -66,9 +66,9 @@ # Links to the Ray Design Pattern doc to use in the task overhead warning # message -DESIGN_PATTERN_FINE_GRAIN_TASKS_LINK = "https://docs.google.com/document/d/167rnnDFIVRhHhK4mznEIemOtj63IOhtIPvSYaPgI4Fg/edit#heading=h.f7ins22n6nyl" # noqa E501 +DESIGN_PATTERN_FINE_GRAIN_TASKS_LINK = "https://docs.ray.io/en/latest/ray-core/patterns/too-fine-grained-tasks.html" # noqa E501 -DESIGN_PATTERN_LARGE_OBJECTS_LINK = "https://docs.google.com/document/d/167rnnDFIVRhHhK4mznEIemOtj63IOhtIPvSYaPgI4Fg/edit#heading=h.1afmymq455wu" # noqa E501 +DESIGN_PATTERN_LARGE_OBJECTS_LINK = "https://docs.ray.io/en/latest/ray-core/patterns/closure-capture-large-objects.html" # noqa E501 def backoff(timeout: int) -> int: From 2059a906bc96fe5fcb49486fbec6799587ba7a5b Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Mon, 25 Aug 2025 11:07:22 -0700 Subject: [PATCH 248/634] [Data] Fix `UnboundLocalError` when `read_parquet` with columns and no partitioning (#55820) ## Why are these changes needed? If you call `read_parquet` with `partitioning=None` and non-empty `columns`, then Ray Data raises a error because of a missing branch. This PR fixes that issue. ``` Traceback (most recent call last): File "/root/lab42_vr/test.py", line 6, in ds = ray.data.read_parquet(input_s3_path, partitioning=None, columns=columns) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/root/.venv/lib/python3.12/site-packages/ray/data/read_api.py", line 950, in read_parquet datasource = ParquetDatasource( ^^^^^^^^^^^^^^^^^^ File "/root/.venv/lib/python3.12/site-packages/ray/data/_internal/datasource/parquet_datasource.py", line 262, in __init__ data_columns, partition_columns = _infer_data_and_partition_columns( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/root/.venv/lib/python3.12/site-packages/ray/data/_internal/datasource/parquet_datasource.py", line 817, in _infer_data_and_partition_columns return data_columns, partition_columns ^^^^^^^^^^^^^^^^^ UnboundLocalError: cannot access local variable 'partition_columns' where it is not associated with a value ``` ## Related issue number Fixes https://github.com/ray-project/ray/issues/55279 ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Balaji Veeramani --- .../data/_internal/datasource/parquet_datasource.py | 3 +++ python/ray/data/tests/test_parquet.py | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/python/ray/data/_internal/datasource/parquet_datasource.py b/python/ray/data/_internal/datasource/parquet_datasource.py index 548f6b6a88dc..44ccbd1e55b2 100644 --- a/python/ray/data/_internal/datasource/parquet_datasource.py +++ b/python/ray/data/_internal/datasource/parquet_datasource.py @@ -825,4 +825,7 @@ def _infer_data_and_partition_columns( partition_columns = [ column for column in user_specified_columns if column in partitions ] + else: + partition_columns = [] + return data_columns, partition_columns diff --git a/python/ray/data/tests/test_parquet.py b/python/ray/data/tests/test_parquet.py index dd43f22e1ee2..bacf2c649941 100644 --- a/python/ray/data/tests/test_parquet.py +++ b/python/ray/data/tests/test_parquet.py @@ -2223,6 +2223,17 @@ def test_parquet_write_parallel_overwrite( assert result.count() == 1000 +def test_read_parquet_with_none_partitioning_and_columns(tmp_path): + # Test for https://github.com/ray-project/ray/issues/55279. + table = pa.table({"column": [42]}) + path = os.path.join(tmp_path, "file.parquet") + pq.write_table(table, path) + + ds = ray.data.read_parquet(path, partitioning=None, columns=["column"]) + + assert ds.take_all() == [{"column": 42}] + + if __name__ == "__main__": import sys From 158474e199596572de9f952ee431a1c0e8e80909 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 25 Aug 2025 11:10:16 -0700 Subject: [PATCH 249/634] [serve] install httpx for running container runtime env tests (#55869) required in runtime env test. Signed-off-by: Lonnie Liu --- ci/docker/runtime_env_container/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/docker/runtime_env_container/Dockerfile b/ci/docker/runtime_env_container/Dockerfile index 45501d3e0f5a..ad7972761bce 100644 --- a/ci/docker/runtime_env_container/Dockerfile +++ b/ci/docker/runtime_env_container/Dockerfile @@ -4,4 +4,5 @@ FROM $BASE_IMAGE COPY python/ray/tests/runtime_env_container/ /home/ray/tests/ # Install podman +RUN pip install --no-cache-dir -c /home/ray/requirements_compiled.txt httpx RUN sudo apt-get update && sudo apt-get install podman -y From 79aad9b68e2504918b3a5f57165b19b9113fe3ab Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 25 Aug 2025 14:20:47 -0500 Subject: [PATCH 250/634] [core] Remove unnecessary `gcs_client` dependency (#55883) Removing more developer productivity sabotage. --------- Signed-off-by: Edward Oakes --- src/ray/raylet/BUILD.bazel | 1 + src/ray/raylet/scheduling/BUILD.bazel | 2 +- src/ray/raylet/scheduling/local_resource_manager.cc | 1 + src/ray/raylet/scheduling/local_resource_manager.h | 2 -- src/ray/raylet/scheduling/tests/BUILD.bazel | 1 + 5 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index f5305944d209..371381a182bb 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -61,6 +61,7 @@ ray_cc_library( deps = [ "//src/ray/common:id", "//src/ray/common:task_common", + "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "//src/ray/util:container_util", "@com_google_absl//absl/container:flat_hash_map", diff --git a/src/ray/raylet/scheduling/BUILD.bazel b/src/ray/raylet/scheduling/BUILD.bazel index eef39d32e4e5..b8616a7a4882 100644 --- a/src/ray/raylet/scheduling/BUILD.bazel +++ b/src/ray/raylet/scheduling/BUILD.bazel @@ -124,9 +124,9 @@ ray_cc_library( "//src/ray/common:ray_config", "//src/ray/common:ray_syncer", "//src/ray/common:task_common", - "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/protobuf:gcs_cc_proto", "//src/ray/protobuf:node_manager_cc_proto", + "//src/ray/stats:stats_metric", "//src/ray/util:logging", "@com_google_absl//absl/container:flat_hash_map", "@com_google_googletest//:gtest_prod", diff --git a/src/ray/raylet/scheduling/local_resource_manager.cc b/src/ray/raylet/scheduling/local_resource_manager.cc index 68374d60e23e..5ccec894dd4a 100644 --- a/src/ray/raylet/scheduling/local_resource_manager.cc +++ b/src/ray/raylet/scheduling/local_resource_manager.cc @@ -24,6 +24,7 @@ #include "ray/common/grpc_util.h" #include "ray/common/ray_config.h" +#include "ray/stats/metric_defs.h" namespace ray { diff --git a/src/ray/raylet/scheduling/local_resource_manager.h b/src/ray/raylet/scheduling/local_resource_manager.h index 7b78327efc49..72c15ceb6514 100644 --- a/src/ray/raylet/scheduling/local_resource_manager.h +++ b/src/ray/raylet/scheduling/local_resource_manager.h @@ -26,8 +26,6 @@ #include "ray/common/scheduling/cluster_resource_data.h" #include "ray/common/scheduling/fixed_point.h" #include "ray/common/scheduling/resource_set.h" -#include "ray/gcs/gcs_client/accessor.h" -#include "ray/gcs/gcs_client/gcs_client.h" #include "ray/util/logging.h" #include "src/ray/protobuf/gcs.pb.h" #include "src/ray/protobuf/node_manager.pb.h" diff --git a/src/ray/raylet/scheduling/tests/BUILD.bazel b/src/ray/raylet/scheduling/tests/BUILD.bazel index 1e3d0cdae67f..de54b3c34d61 100644 --- a/src/ray/raylet/scheduling/tests/BUILD.bazel +++ b/src/ray/raylet/scheduling/tests/BUILD.bazel @@ -12,6 +12,7 @@ ray_cc_test( "//src/ray/common:ray_config", "//src/ray/common:task_common", "//src/ray/common:test_util", + "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "@com_google_googletest//:gtest_main", ], From 01b4f0de71c36fea73e1f7dacdcb7e5f955ea06d Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Mon, 25 Aug 2025 12:33:18 -0700 Subject: [PATCH 251/634] [ci] raydepsets: remove build_arg_sets arg from raydepsets (#55852) - Build arg sets arg is no longer used or needed in the cli - All build arg sets are processed before execution and are included in the names of each depset in the config - Removing unused build arg sets arg from raydepsets - updating unit test Signed-off-by: elliot-barn --- ci/raydepsets/cli.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index 4d1037387a07..70d5287ab680 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -41,11 +41,6 @@ def cli(): default=None, help="The name of the dependency set to load. If not specified, all dependency sets will be loaded.", ) -@click.option( - "--build-arg-set", - default=None, - help="The name of the build arg set to use. If not specified, a depset matching the name with no build arg set will be loaded.", -) @click.option( "--uv-cache-dir", default=None, help="The directory to cache uv dependencies" ) @@ -53,7 +48,6 @@ def build( config_path: str, workspace_dir: Optional[str], name: Optional[str], - build_arg_set: Optional[str], uv_cache_dir: Optional[str], ): """ From 235c403d210a805f9a9e80d171234b5ccfa3a048 Mon Sep 17 00:00:00 2001 From: Carolyn Wang Date: Mon, 25 Aug 2025 13:25:08 -0700 Subject: [PATCH 252/634] [observability] Prometheus http service discovery API (#55656) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? Adds an http endpoint for prometheus service discovery for metrics targets. Manual testing: Started local prometheus with config ``` global: scrape_interval: 10s # Set the scrape interval to every 10 seconds. Default is every 1 minute. evaluation_interval: 10s # Evaluate rules every 10 seconds. The default is every 1 minute. scrape_configs: - job_name: 'ray' http_sd_configs: - url: 'http://127.0.0.1:8265/api/prometheus/sd' refresh_interval: 60s ``` and local ray cluster with `ray start --head --metrics-export-port=8080`. check api response: ``` % curl http://127.0.0.1:8265/api/prometheus/sd [{"labels": {"job": "ray"}, "targets": ["127.0.0.1:8080", "127.0.0.1:44217", "127.0.0.1:44227"]}] ``` ensure prometheus can scrape for these targets: Screenshot 2025-08-15 at 2 25 34 PM ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [x] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: carolynwang --- doc/source/cluster/metrics.md | 16 ++++++++++++++ python/ray/_private/metrics_agent.py | 17 +++++++++++--- .../modules/reporter/reporter_head.py | 22 +++++++++++++++++++ python/ray/tests/test_metrics_agent.py | 10 +++++---- 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/doc/source/cluster/metrics.md b/doc/source/cluster/metrics.md index 7956a74ddcbb..c4ae16abe89e 100644 --- a/doc/source/cluster/metrics.md +++ b/doc/source/cluster/metrics.md @@ -188,6 +188,22 @@ scrape_configs: - '/tmp/ray/prom_metrics_service_discovery.json' ``` +#### HTTP service discovery +Ray also exposes the same list of addresses to scrape over an HTTP endpoint, compatible with [Prometheus HTTP Service Discovery](https://prometheus.io/docs/prometheus/latest/http_sd/). + +Use the following in your Prometheus config to use the HTTP endpoint for service discovery ([HTTP SD docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config)): + +```yaml +scrape_configs: +- job_name: 'ray' + http_sd_configs: + - url: 'http://:/api/prometheus/sd' + refresh_interval: 60s +``` + +- `` is `8265` by default. See [Configuring and Managing Ray Dashboard](https://docs.ray.io/en/latest/cluster/configure-manage-dashboard.html) for more details. +- The endpoint returns a JSON list of targets for Prometheus metrics. When no targets are available, it returns `[]`. + ### Manually discovering metrics endpoints If you know the IP addresses of the nodes in your Ray Cluster, you can configure Prometheus to read metrics from a static list of endpoints. diff --git a/python/ray/_private/metrics_agent.py b/python/ray/_private/metrics_agent.py index 1de034dfd49c..6116687e391a 100644 --- a/python/ray/_private/metrics_agent.py +++ b/python/ray/_private/metrics_agent.py @@ -775,8 +775,18 @@ def __init__(self, gcs_address, temp_dir): ray._private.state.state._initialize_global_state(gcs_client_options) self.temp_dir = temp_dir self.default_service_discovery_flush_period = 5 + + # The last service discovery content that PrometheusServiceDiscoveryWriter has seen + self.latest_service_discovery_content = [] + self._content_lock = threading.RLock() + super().__init__() + def get_latest_service_discovery_content(self): + """Return the latest stored service discovery content.""" + with self._content_lock: + return self.latest_service_discovery_content + def get_file_discovery_content(self): """Return the content for Prometheus service discovery.""" nodes = ray.nodes() @@ -792,9 +802,10 @@ def get_file_discovery_content(self): dashboard_addr = gcs_client.internal_kv_get(b"DashboardMetricsAddress", None) if dashboard_addr: metrics_export_addresses.append(dashboard_addr.decode("utf-8")) - return json.dumps( - [{"labels": {"job": "ray"}, "targets": metrics_export_addresses}] - ) + content = [{"labels": {"job": "ray"}, "targets": metrics_export_addresses}] + with self._content_lock: + self.latest_service_discovery_content = content + return json.dumps(content) def write(self): # Write a file based on https://prometheus.io/docs/guides/file-sd/ diff --git a/python/ray/dashboard/modules/reporter/reporter_head.py b/python/ray/dashboard/modules/reporter/reporter_head.py index d8e76b25bb07..7ec0f59be092 100644 --- a/python/ray/dashboard/modules/reporter/reporter_head.py +++ b/python/ray/dashboard/modules/reporter/reporter_head.py @@ -785,6 +785,28 @@ async def kill_actor_gcs(self, req: aiohttp.web.Request) -> aiohttp.web.Response status_code=status_code, message=message ) + @routes.get("/api/prometheus/sd") + async def prometheus_service_discovery(self, req) -> aiohttp.web.Response: + """ + Expose Prometheus metrics targets through HTTP Service Discovery. + """ + content = self.service_discovery.get_latest_service_discovery_content() + if not isinstance(content, list): + error_message = "service discovery error: content is not a list" + logger.warning(error_message) + return aiohttp.web.json_response( + {"error": error_message}, + status=dashboard_utils.HTTPStatusCode.INTERNAL_ERROR, + headers={"Cache-Control": "no-store"}, + ) + return aiohttp.web.Response( + text=json.dumps(content), + content_type="application/json", + charset="utf-8", + status=dashboard_utils.HTTPStatusCode.OK, + headers={"Cache-Control": "no-store"}, + ) + async def _get_stub_address_by_node_id( self, node_id: NodeID ) -> Optional[Tuple[NodeID, str, int, int]]: diff --git a/python/ray/tests/test_metrics_agent.py b/python/ray/tests/test_metrics_agent.py index 9f5fbc059431..9767ba79091d 100644 --- a/python/ray/tests/test_metrics_agent.py +++ b/python/ray/tests/test_metrics_agent.py @@ -971,9 +971,10 @@ def get_metrics_export_address_from_node(nodes): ) return node_export_addrs + [autoscaler_export_addr, dashboard_export_addr] - loaded_json_data = json.loads(writer.get_file_discovery_content())[0] + loaded_json_data = json.loads(writer.get_file_discovery_content()) + assert loaded_json_data == writer.get_latest_service_discovery_content() assert set(get_metrics_export_address_from_node(nodes)) == set( - loaded_json_data["targets"] + loaded_json_data[0]["targets"] ) # Let's update nodes. @@ -981,9 +982,10 @@ def get_metrics_export_address_from_node(nodes): nodes.append(cluster.add_node()) # Make sure service discovery file content is correctly updated. - loaded_json_data = json.loads(writer.get_file_discovery_content())[0] + loaded_json_data = json.loads(writer.get_file_discovery_content()) + assert loaded_json_data == writer.get_latest_service_discovery_content() assert set(get_metrics_export_address_from_node(nodes)) == set( - loaded_json_data["targets"] + loaded_json_data[0]["targets"] ) From a1cd755cd77372e84118d94f06ccc7dd38a31c5a Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 25 Aug 2025 13:36:21 -0700 Subject: [PATCH 253/634] [serve] remove regular checkpointing (#55848) ## Why are these changes needed? In https://github.com/ray-project/ray/pull/49168, support was added for bulk deploys. To support performant bulk deploys, checkpointing was added into the regular control loop (so that we wouldn't checkpoint for every application + deployment that was deployed, which is expensive). However this will cause the KV store memory to grow over time. Instead, we should only checkpoint when it is **confirmed** that target state has changed. --------- Signed-off-by: Cindy Zhang --- .../ray/serve/_private/application_state.py | 69 +++++++++++++++---- python/ray/serve/_private/controller.py | 7 -- python/ray/serve/_private/deployment_state.py | 38 ++++++---- .../tests/unit/test_application_state.py | 2 +- .../serve/tests/unit/test_deployment_state.py | 16 +++-- 5 files changed, 91 insertions(+), 41 deletions(-) diff --git a/python/ray/serve/_private/application_state.py b/python/ray/serve/_private/application_state.py index 41108f62edd4..21123b9be039 100644 --- a/python/ray/serve/_private/application_state.py +++ b/python/ray/serve/_private/application_state.py @@ -400,10 +400,18 @@ def _clear_target_state_and_store_config( deleting=False, ) - def _delete_deployment(self, name): + def _delete_deployment(self, name: str) -> bool: + """Delete a deployment in the application. + + Args: + name: The name of the deployment to delete. + + Returns: + Whether the target state has changed. + """ id = DeploymentID(name=name, app_name=self._name) self._endpoint_state.delete_endpoint(id) - self._deployment_state_manager.delete_deployment(id) + return self._deployment_state_manager.delete_deployment(id) def delete(self): """Delete the application""" @@ -426,8 +434,16 @@ def apply_deployment_info( self, deployment_name: str, deployment_info: DeploymentInfo, - ) -> None: - """Deploys a deployment in the application.""" + ) -> bool: + """Deploys a deployment in the application. + + Args: + deployment_name: The name of the deployment to apply. + deployment_info: The deployment info to apply. + + Returns: + Whether the target state has changed. + """ route_prefix = deployment_info.route_prefix if route_prefix is not None and not route_prefix.startswith("/"): raise RayServeException( @@ -436,7 +452,9 @@ def apply_deployment_info( deployment_id = DeploymentID(name=deployment_name, app_name=self._name) - self._deployment_state_manager.deploy(deployment_id, deployment_info) + target_state_changed = self._deployment_state_manager.deploy( + deployment_id, deployment_info + ) if deployment_info.route_prefix is not None: config = deployment_info.deployment_config @@ -457,6 +475,8 @@ def apply_deployment_info( else: self._endpoint_state.delete_endpoint(deployment_id) + return target_state_changed + def deploy_app(self, deployment_infos: Dict[str, DeploymentInfo]): """(Re-)deploy the application from list of deployment infos. @@ -761,6 +781,7 @@ def _reconcile_target_deployments(self) -> None: Ensure each deployment is running on up-to-date info, and remove outdated deployments from the application. """ + target_state_changed = False # Set target state for each deployment for deployment_name, info in self._target_state.deployment_infos.items(): @@ -784,26 +805,34 @@ def _reconcile_target_deployments(self) -> None: deploy_info.deployment_config.logging_config = ( self._target_state.config.logging_config ) - self.apply_deployment_info(deployment_name, deploy_info) + target_state_changed = ( + self.apply_deployment_info(deployment_name, deploy_info) + or target_state_changed + ) # Delete outdated deployments for deployment_name in self._get_live_deployments(): if deployment_name not in self.target_deployments: - self._delete_deployment(deployment_name) + target_state_changed = ( + self._delete_deployment(deployment_name) or target_state_changed + ) - def update(self) -> bool: + return target_state_changed + + def update(self) -> Tuple[bool, bool]: """Attempts to reconcile this application to match its target state. Updates the application status and status message based on the current state of the system. Returns: - A boolean indicating whether the application is ready to be - deleted. + Whether the target state has changed. """ infos, task_status, msg = self._reconcile_build_app_task() + target_state_changed = False if task_status == BuildAppStatus.SUCCEEDED: + target_state_changed = True self._set_target_state( deployment_infos=infos, code_version=self._build_app_task_info.code_version, @@ -821,14 +850,16 @@ def update(self) -> bool: # it's not finished, we don't know what the target list of deployments # is, so we don't perform any reconciliation. if self._target_state.deployment_infos is not None: - self._reconcile_target_deployments() + target_state_changed = ( + self._reconcile_target_deployments() or target_state_changed + ) status, status_msg = self._determine_app_status() self._update_status(status, status_msg) # Check if app is ready to be deleted if self._target_state.deleting: - return self.is_deleted() - return False + return self.is_deleted(), target_state_changed + return False, target_state_changed def get_checkpoint_data(self) -> ApplicationTargetState: return self._target_state @@ -1087,10 +1118,14 @@ def list_deployment_details(self, name: str) -> Dict[str, DeploymentDetails]: return self._application_states[name].list_deployment_details() def update(self): - """Update each application state""" + """Update each application state.""" apps_to_be_deleted = [] + any_target_state_changed = False for name, app in self._application_states.items(): - ready_to_be_deleted = app.update() + ready_to_be_deleted, app_target_state_changed = app.update() + any_target_state_changed = ( + any_target_state_changed or app_target_state_changed + ) if ready_to_be_deleted: apps_to_be_deleted.append(name) logger.debug(f"Application '{name}' deleted successfully.") @@ -1100,6 +1135,10 @@ def update(self): del self._application_states[app_name] ServeUsageTag.NUM_APPS.record(str(len(self._application_states))) + if any_target_state_changed: + self.save_checkpoint() + self._deployment_state_manager.save_checkpoint() + def shutdown(self) -> None: self._shutting_down = True diff --git a/python/ray/serve/_private/controller.py b/python/ray/serve/_private/controller.py index cc0724f20565..3caad988f715 100644 --- a/python/ray/serve/_private/controller.py +++ b/python/ray/serve/_private/controller.py @@ -442,8 +442,6 @@ async def run_control_loop_step( dsm_update_start_time = time.time() any_recovering = self.deployment_state_manager.update() - self.deployment_state_manager.save_checkpoint() - self.dsm_update_duration_gauge_s.set(time.time() - dsm_update_start_time) if not self.done_recovering_event.is_set() and not any_recovering: self.done_recovering_event.set() @@ -461,11 +459,6 @@ async def run_control_loop_step( asm_update_start_time = time.time() self.application_state_manager.update() - self.application_state_manager.save_checkpoint() - # ApplicationStateManager.update() can also mutate the - # DeploymentStateManager so we need to checkpoint that as well - self.deployment_state_manager.save_checkpoint() - self.asm_update_duration_gauge_s.set(time.time() - asm_update_start_time) except Exception: logger.exception("Exception updating application state.") diff --git a/python/ray/serve/_private/deployment_state.py b/python/ray/serve/_private/deployment_state.py index 8a33bc5eedc9..87b1ee46502d 100644 --- a/python/ray/serve/_private/deployment_state.py +++ b/python/ray/serve/_private/deployment_state.py @@ -1994,7 +1994,7 @@ def deploy(self, deployment_info: DeploymentInfo) -> bool: this method returns False. Returns: - bool: Whether or not the deployment is being updated. + bool: Whether the target state has changed. """ curr_deployment_info = self._target_state.info @@ -2078,10 +2078,14 @@ def deploy(self, deployment_info: DeploymentInfo) -> bool: return True def autoscale(self) -> int: - """Autoscale the deployment based on metrics.""" + """Autoscale the deployment based on metrics. + + Returns: + Whether the target state has changed. + """ if self._target_state.deleting: - return + return False decision_num_replicas = self._autoscaling_state_manager.get_target_num_replicas( deployment_id=self._id, @@ -2092,7 +2096,7 @@ def autoscale(self) -> int: decision_num_replicas is None or decision_num_replicas == self._target_state.target_num_replicas ): - return + return False new_info = copy(self._target_state.info) new_info.version = self._target_state.version.code_version @@ -2108,7 +2112,7 @@ def autoscale(self) -> int: states=[ReplicaState.RUNNING], version=self._target_state.version ), ): - return + return True curr_stats_str = ( f"Current ongoing requests: " @@ -2135,10 +2139,14 @@ def autoscale(self) -> int: trigger=DeploymentStatusInternalTrigger.AUTOSCALE_DOWN, message=f"Downscaling from {old_num} to {new_num} replicas.", ) + return True - def delete(self) -> None: + def delete(self) -> bool: if not self._target_state.deleting: self._set_target_state_deleting() + return True + + return False def _stop_or_update_outdated_version_replicas(self, max_to_stop=math.inf) -> bool: """Stop or update replicas with outdated versions. @@ -3068,7 +3076,7 @@ def deploy( this is a no-op and returns False. Returns: - bool: Whether or not the deployment is being updated. + bool: Whether the target state has changed. """ if deployment_id not in self._deployment_states: self._deployment_states[deployment_id] = self._create_deployment_state( @@ -3087,7 +3095,9 @@ def delete_deployment(self, id: DeploymentID): # This method must be idempotent. We should validate that the # specified deployment exists on the client. if id in self._deployment_states: - self._deployment_states[id].delete() + return self._deployment_states[id].delete() + + return False def update(self) -> bool: """Updates the state of all deployments to match their goal state. @@ -3099,11 +3109,14 @@ def update(self) -> bool: any_recovering = False upscales: Dict[DeploymentID, List[ReplicaSchedulingRequest]] = {} downscales: Dict[DeploymentID, DeploymentDownscaleRequest] = {} + target_state_changed = False # STEP 1: Update current state for deployment_state in self._deployment_states.values(): if deployment_state.should_autoscale(): - deployment_state.autoscale() + target_state_changed = ( + deployment_state.autoscale() or target_state_changed + ) deployment_state.check_and_update_replicas() @@ -3155,10 +3168,6 @@ def update(self) -> bool: deleted_ids.append(deployment_id) any_recovering |= any_replicas_recovering - # Take a checkpoint before actually affecting the state of the cluster - # by starting/stopping replicas. - self.save_checkpoint() - # STEP 6: Schedule all STARTING replicas and stop all STOPPING replicas deployment_to_replicas_to_stop = self._deployment_scheduler.schedule( upscales, downscales @@ -3198,6 +3207,9 @@ def update(self) -> bool: if len(deleted_ids): self._record_deployment_usage() + if target_state_changed: + self.save_checkpoint() + return any_recovering def _handle_scheduling_request_failures( diff --git a/python/ray/serve/tests/unit/test_application_state.py b/python/ray/serve/tests/unit/test_application_state.py index 1147e3c04f23..4fb9a47b373c 100644 --- a/python/ray/serve/tests/unit/test_application_state.py +++ b/python/ray/serve/tests/unit/test_application_state.py @@ -551,7 +551,7 @@ def test_deploy_and_delete_app(mocked_application_state): app_state.update() deployment_state_manager.set_deployment_deleted(d1_id) - ready_to_be_deleted = app_state.update() + ready_to_be_deleted, _ = app_state.update() assert not ready_to_be_deleted assert app_state.status == ApplicationStatus.DELETING diff --git a/python/ray/serve/tests/unit/test_deployment_state.py b/python/ray/serve/tests/unit/test_deployment_state.py index 553a12bff588..ad093c08ab3d 100644 --- a/python/ray/serve/tests/unit/test_deployment_state.py +++ b/python/ray/serve/tests/unit/test_deployment_state.py @@ -2388,7 +2388,9 @@ def test_recover_state_from_replica_names(mock_deployment_state_manager): # Deploy deployment with version "1" and one replica info1, v1 = deployment_info(version="1") - assert dsm.deploy(TEST_DEPLOYMENT_ID, info1) + target_state_changed = dsm.deploy(TEST_DEPLOYMENT_ID, info1) + assert target_state_changed + dsm.save_checkpoint() ds = dsm._deployment_states[TEST_DEPLOYMENT_ID] # Single replica of version `version1` should be created and in STARTING state @@ -2437,7 +2439,9 @@ def test_recover_during_rolling_update(mock_deployment_state_manager): # Step 1: Create some deployment info with actors in running state info1, v1 = deployment_info(version="1") - assert dsm.deploy(TEST_DEPLOYMENT_ID, info1) + target_state_changed = dsm.deploy(TEST_DEPLOYMENT_ID, info1) + assert target_state_changed + dsm.save_checkpoint() ds = dsm._deployment_states[TEST_DEPLOYMENT_ID] # Single replica of version `version1` should be created and in STARTING state @@ -2452,8 +2456,8 @@ def test_recover_during_rolling_update(mock_deployment_state_manager): # Now execute a rollout: upgrade the version to "2". info2, v2 = deployment_info(version="2") - assert dsm.deploy(TEST_DEPLOYMENT_ID, info2) - + target_state_changed = dsm.deploy(TEST_DEPLOYMENT_ID, info2) + assert target_state_changed # In real code this checkpoint would be done by the caller of .deploy() dsm.save_checkpoint() @@ -2518,7 +2522,9 @@ def test_actor_died_before_recover(mock_deployment_state_manager): # Create some deployment info with actors in running state info1, v1 = deployment_info(version="1") - assert dsm.deploy(TEST_DEPLOYMENT_ID, info1) + target_state_changed = dsm.deploy(TEST_DEPLOYMENT_ID, info1) + assert target_state_changed + dsm.save_checkpoint() ds = dsm._deployment_states[TEST_DEPLOYMENT_ID] # Single replica of version `version1` should be created and in STARTING state From 44bf1ef381899b2a8f66889e645d96dff21e7a81 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Mon, 25 Aug 2025 13:36:34 -0700 Subject: [PATCH 254/634] Add perf metrics for 2.49.0 (#55815) ``` REGRESSION 35.18%: client__put_gigabytes (THROUGHPUT) regresses from 0.1559990403715773 to 0.1011154460629056 in microbenchmark.json REGRESSION 19.86%: actors_per_second (THROUGHPUT) regresses from 657.1702061376596 to 526.6643292032776 in benchmarks/many_actors.json REGRESSION 11.54%: client__get_calls (THROUGHPUT) regresses from 1159.3513798913632 to 1025.614536261544 in microbenchmark.json REGRESSION 10.21%: 1_1_actor_calls_concurrent (THROUGHPUT) regresses from 5775.020315522301 to 5185.592351945926 in microbenchmark.json REGRESSION 7.22%: single_client_tasks_and_get_batch (THROUGHPUT) regresses from 5.800654754787365 to 5.381854147597904 in microbenchmark.json REGRESSION 6.72%: client__tasks_and_put_batch (THROUGHPUT) regresses from 14560.030073574557 to 13581.806864962535 in microbenchmark.json REGRESSION 5.84%: multi_client_put_calls_Plasma_Store (THROUGHPUT) regresses from 16526.35985553258 to 15560.620089508539 in microbenchmark.json REGRESSION 5.76%: single_client_put_gigabytes (THROUGHPUT) regresses from 19.85639156989914 to 18.71278038275444 in microbenchmark.json REGRESSION 4.78%: tasks_per_second (THROUGHPUT) regresses from 364.43726497335643 to 347.0074587793457 in benchmarks/many_tasks.json REGRESSION 4.71%: single_client_get_calls_Plasma_Store (THROUGHPUT) regresses from 10620.405550394937 to 10119.7301338237 in microbenchmark.json REGRESSION 3.55%: 1_n_actor_calls_async (THROUGHPUT) regresses from 8038.166251679982 to 7753.148948018643 in microbenchmark.json REGRESSION 3.54%: single_client_tasks_sync (THROUGHPUT) regresses from 980.7121217208985 to 946.027654634476 in microbenchmark.json REGRESSION 3.41%: n_n_actor_calls_async (THROUGHPUT) regresses from 27375.624367126635 to 26441.297568592032 in microbenchmark.json REGRESSION 3.24%: 1_1_actor_calls_async (THROUGHPUT) regresses from 8663.654839458402 to 8382.98999101571 in microbenchmark.json REGRESSION 3.15%: pgs_per_second (THROUGHPUT) regresses from 13.215254403739163 to 12.799625675009633 in benchmarks/many_pgs.json REGRESSION 2.50%: single_client_wait_1k_refs (THROUGHPUT) regresses from 5.079952667320649 to 4.952999927332959 in microbenchmark.json REGRESSION 2.00%: n_n_actor_calls_with_arg_async (THROUGHPUT) regresses from 2759.3212097473174 to 2704.110675027102 in microbenchmark.json REGRESSION 1.84%: multi_client_put_gigabytes (THROUGHPUT) regresses from 38.137310138893675 to 37.43614465888436 in microbenchmark.json REGRESSION 1.35%: 1_1_async_actor_calls_sync (THROUGHPUT) regresses from 1459.7289131365046 to 1440.0280310845594 in microbenchmark.json REGRESSION 1.13%: placement_group_create/removal (THROUGHPUT) regresses from 764.5677165695956 to 755.9481741578835 in microbenchmark.json REGRESSION 1.11%: n_n_async_actor_calls_async (THROUGHPUT) regresses from 23674.50106467489 to 23412.17782093146 in microbenchmark.json REGRESSION 1.02%: single_client_tasks_async (THROUGHPUT) regresses from 8040.530786886751 to 7958.403181954658 in microbenchmark.json REGRESSION 0.79%: single_client_get_object_containing_10k_refs (THROUGHPUT) regresses from 13.371722002108683 to 13.265651211414822 in microbenchmark.json REGRESSION 37.92%: dashboard_p50_latency_ms (LATENCY) regresses from 5.277 to 7.278 in benchmarks/many_tasks.json REGRESSION 35.95%: dashboard_p99_latency_ms (LATENCY) regresses from 2572.496 to 3497.265 in benchmarks/many_actors.json REGRESSION 34.32%: dashboard_p95_latency_ms (LATENCY) regresses from 7.454 to 10.012 in benchmarks/many_pgs.json REGRESSION 29.32%: dashboard_p95_latency_ms (LATENCY) regresses from 2197.485 to 2841.741 in benchmarks/many_actors.json REGRESSION 22.60%: dashboard_p50_latency_ms (LATENCY) regresses from 5.204 to 6.38 in benchmarks/many_nodes.json REGRESSION 10.19%: dashboard_p95_latency_ms (LATENCY) regresses from 492.608 to 542.827 in benchmarks/many_tasks.json REGRESSION 10.10%: avg_pg_remove_time_ms (LATENCY) regresses from 1.2057934429429915 to 1.3276310030028873 in stress_tests/stress_test_placement_group.json REGRESSION 9.71%: stage_3_creation_time (LATENCY) regresses from 1.8526091575622559 to 2.032559394836426 in stress_tests/stress_test_many_tasks.json REGRESSION 8.77%: dashboard_p50_latency_ms (LATENCY) regresses from 4.194 to 4.562 in benchmarks/many_pgs.json REGRESSION 6.57%: avg_pg_create_time_ms (LATENCY) regresses from 1.4493968768766456 to 1.544663453452921 in stress_tests/stress_test_placement_group.json REGRESSION 3.77%: 10000_get_time (LATENCY) regresses from 23.075941746000012 to 23.946038821000002 in scalability/single_node.json REGRESSION 3.39%: dashboard_p50_latency_ms (LATENCY) regresses from 9.35 to 9.667 in benchmarks/many_actors.json REGRESSION 2.28%: stage_1_avg_iteration_time (LATENCY) regresses from 12.96969530582428 to 13.265494346618652 in stress_tests/stress_test_many_tasks.json REGRESSION 1.74%: 10000_args_time (LATENCY) regresses from 18.84486551900001 to 19.17184469900002 in scalability/single_node.json REGRESSION 0.57%: stage_2_avg_iteration_time (LATENCY) regresses from 33.957920932769774 to 34.152964401245114 in stress_tests/stress_test_many_tasks.json REGRESSION 0.38%: avg_iteration_time (LATENCY) regresses from 1.1874613547325135 to 1.1919621157646179 in stress_tests/stress_test_dead_actors.json ``` Signed-off-by: elliot-barn Co-authored-by: elliot-barn Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> --- .../perf_metrics/benchmarks/many_actors.json | 18 +- .../perf_metrics/benchmarks/many_nodes.json | 18 +- release/perf_metrics/benchmarks/many_pgs.json | 18 +- .../perf_metrics/benchmarks/many_tasks.json | 18 +- release/perf_metrics/metadata.json | 2 +- release/perf_metrics/microbenchmark.json | 186 +++++++++--------- .../scalability/object_store.json | 4 +- .../perf_metrics/scalability/single_node.json | 20 +- .../stress_tests/stress_test_dead_actors.json | 10 +- .../stress_tests/stress_test_many_tasks.json | 36 ++-- .../stress_test_placement_group.json | 8 +- 11 files changed, 169 insertions(+), 169 deletions(-) diff --git a/release/perf_metrics/benchmarks/many_actors.json b/release/perf_metrics/benchmarks/many_actors.json index 3887e4d4e6fb..cae597479e54 100644 --- a/release/perf_metrics/benchmarks/many_actors.json +++ b/release/perf_metrics/benchmarks/many_actors.json @@ -1,32 +1,32 @@ { - "_dashboard_memory_usage_mb": 110.235648, + "_dashboard_memory_usage_mb": 102.928384, "_dashboard_test_success": true, - "_peak_memory": 4.85, - "_peak_process_memory": "PID\tMEM\tCOMMAND\n1129\t7.16GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3533\t2.04GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n4961\t1.06GiB\tpython distributed/test_many_actors.py\n3034\t0.46GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3734\t0.32GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n585\t0.2GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n3649\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n4254\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3001\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n4256\t0.08GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru", - "actors_per_second": 657.1702061376596, + "_peak_memory": 4.56, + "_peak_process_memory": "PID\tMEM\tCOMMAND\n1147\t7.55GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3549\t2.03GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n4987\t0.83GiB\tpython distributed/test_many_actors.py\n3056\t0.39GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3751\t0.27GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n584\t0.19GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n4235\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3664\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n2979\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n4237\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru", + "actors_per_second": 526.6643292032776, "num_actors": 10000, "perf_metrics": [ { "perf_metric_name": "actors_per_second", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 657.1702061376596 + "perf_metric_value": 526.6643292032776 }, { "perf_metric_name": "dashboard_p50_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 9.35 + "perf_metric_value": 9.667 }, { "perf_metric_name": "dashboard_p95_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 2197.485 + "perf_metric_value": 2841.741 }, { "perf_metric_name": "dashboard_p99_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 2572.496 + "perf_metric_value": 3497.265 } ], "success": "1", - "time": 15.216758012771606 + "time": 18.987426042556763 } diff --git a/release/perf_metrics/benchmarks/many_nodes.json b/release/perf_metrics/benchmarks/many_nodes.json index 5359354d5066..0b2e8ccb2f6a 100644 --- a/release/perf_metrics/benchmarks/many_nodes.json +++ b/release/perf_metrics/benchmarks/many_nodes.json @@ -1,14 +1,14 @@ { - "_dashboard_memory_usage_mb": 96.54272, + "_dashboard_memory_usage_mb": 95.510528, "_dashboard_test_success": true, - "_peak_memory": 2.26, - "_peak_process_memory": "PID\tMEM\tCOMMAND\n3357\t0.51GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2796\t0.28GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n5171\t0.17GiB\tpython distributed/test_many_tasks.py --num-tasks=1000\n3555\t0.14GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n1083\t0.13GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n4094\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n2769\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n5398\t0.09GiB\tray::StateAPIGeneratorActor.start\n3473\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n4096\t0.08GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru", + "_peak_memory": 2.29, + "_peak_process_memory": "PID\tMEM\tCOMMAND\n3521\t0.51GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n3063\t0.26GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n1069\t0.19GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n5012\t0.17GiB\tpython distributed/test_many_tasks.py --num-tasks=1000\n3719\t0.14GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n4200\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3637\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n3019\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n5257\t0.09GiB\tray::StateAPIGeneratorActor.start\n4202\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru", "num_tasks": 1000, "perf_metrics": [ { "perf_metric_name": "tasks_per_second", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 191.95909855877267 + "perf_metric_value": 210.13682904062856 }, { "perf_metric_name": "used_cpus_by_deadline", @@ -18,21 +18,21 @@ { "perf_metric_name": "dashboard_p50_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 5.204 + "perf_metric_value": 6.38 }, { "perf_metric_name": "dashboard_p95_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 13.297 + "perf_metric_value": 13.055 }, { "perf_metric_name": "dashboard_p99_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 54.703 + "perf_metric_value": 38.335 } ], "success": "1", - "tasks_per_second": 191.95909855877267, - "time": 305.2094430923462, + "tasks_per_second": 210.13682904062856, + "time": 304.7588040828705, "used_cpus": 250.0 } diff --git a/release/perf_metrics/benchmarks/many_pgs.json b/release/perf_metrics/benchmarks/many_pgs.json index d6c4288e5a3d..66b733a787ba 100644 --- a/release/perf_metrics/benchmarks/many_pgs.json +++ b/release/perf_metrics/benchmarks/many_pgs.json @@ -1,32 +1,32 @@ { - "_dashboard_memory_usage_mb": 93.515776, + "_dashboard_memory_usage_mb": 90.968064, "_dashboard_test_success": true, - "_peak_memory": 2.69, - "_peak_process_memory": "PID\tMEM\tCOMMAND\n1130\t7.9GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3522\t0.91GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n4967\t0.36GiB\tpython distributed/test_many_pgs.py\n2980\t0.32GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n580\t0.19GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n4243\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3724\t0.09GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n2794\t0.09GiB\t/app/go/infra/activityprobe/activityprobe ray --port=5903 --metrics_server_port=9092 --raylet_addr=l\n3106\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n3642\t0.08GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash", + "_peak_memory": 2.88, + "_peak_process_memory": "PID\tMEM\tCOMMAND\n1111\t8.31GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3486\t0.98GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2906\t0.4GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n5166\t0.39GiB\tpython distributed/test_many_pgs.py\n581\t0.18GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n3690\t0.14GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n2765\t0.11GiB\t/app/go/infra/activityprobe/activityprobe ray --port=5903 --metrics_server_port=9092 --raylet_addr=l\n4176\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3039\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n4178\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru", "num_pgs": 1000, "perf_metrics": [ { "perf_metric_name": "pgs_per_second", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 13.215254403739163 + "perf_metric_value": 12.799625675009633 }, { "perf_metric_name": "dashboard_p50_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 4.194 + "perf_metric_value": 4.562 }, { "perf_metric_name": "dashboard_p95_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 7.454 + "perf_metric_value": 10.012 }, { "perf_metric_name": "dashboard_p99_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 219.05 + "perf_metric_value": 186.207 } ], - "pgs_per_second": 13.215254403739163, + "pgs_per_second": 12.799625675009633, "success": "1", - "time": 75.67012858390808 + "time": 78.12728476524353 } diff --git a/release/perf_metrics/benchmarks/many_tasks.json b/release/perf_metrics/benchmarks/many_tasks.json index 0a045a0a839e..b16a12b02b26 100644 --- a/release/perf_metrics/benchmarks/many_tasks.json +++ b/release/perf_metrics/benchmarks/many_tasks.json @@ -1,14 +1,14 @@ { - "_dashboard_memory_usage_mb": 95.08864, + "_dashboard_memory_usage_mb": 104.685568, "_dashboard_test_success": true, - "_peak_memory": 3.91, - "_peak_process_memory": "PID\tMEM\tCOMMAND\n3526\t1.07GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n5070\t0.75GiB\tpython distributed/test_many_tasks.py --num-tasks=10000\n3724\t0.45GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n3054\t0.29GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3727\t0.2GiB\tray-dashboard-StateHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import s\n1120\t0.12GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n4243\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3642\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n3021\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n5380\t0.09GiB\tray::StateAPIGeneratorActor.start", + "_peak_memory": 3.96, + "_peak_process_memory": "PID\tMEM\tCOMMAND\n3526\t1.11GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n8718\t0.76GiB\tpython distributed/test_many_tasks.py --num-tasks=10000\n3726\t0.45GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n3040\t0.28GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3729\t0.15GiB\tray-dashboard-StateHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import s\n1122\t0.12GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n4209\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3641\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n2964\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n9009\t0.09GiB\tray::StateAPIGeneratorActor.start", "num_tasks": 10000, "perf_metrics": [ { "perf_metric_name": "tasks_per_second", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 364.43726497335643 + "perf_metric_value": 347.0074587793457 }, { "perf_metric_name": "used_cpus_by_deadline", @@ -18,21 +18,21 @@ { "perf_metric_name": "dashboard_p50_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 5.277 + "perf_metric_value": 7.278 }, { "perf_metric_name": "dashboard_p95_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 492.608 + "perf_metric_value": 542.827 }, { "perf_metric_name": "dashboard_p99_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 787.075 + "perf_metric_value": 736.395 } ], "success": "1", - "tasks_per_second": 364.43726497335643, - "time": 327.4395649433136, + "tasks_per_second": 347.0074587793457, + "time": 328.8178243637085, "used_cpus": 2500.0 } diff --git a/release/perf_metrics/metadata.json b/release/perf_metrics/metadata.json index f185fe20125d..aa69a1c1e5cc 100644 --- a/release/perf_metrics/metadata.json +++ b/release/perf_metrics/metadata.json @@ -1 +1 @@ -{"release_version": "2.48.0"} +{"release_version": "2.49.0"} diff --git a/release/perf_metrics/microbenchmark.json b/release/perf_metrics/microbenchmark.json index abd2e5ee7bcb..a58f3baee28e 100644 --- a/release/perf_metrics/microbenchmark.json +++ b/release/perf_metrics/microbenchmark.json @@ -1,283 +1,283 @@ { "1_1_actor_calls_async": [ - 8663.654839458402, - 182.98906836658583 + 8382.98999101571, + 117.79830230521418 ], "1_1_actor_calls_concurrent": [ - 5775.020315522301, - 166.03207664123752 + 5185.592351945926, + 90.30753238314676 ], "1_1_actor_calls_sync": [ - 2011.916260420167, - 34.20258828426277 + 2091.781722688228, + 16.9518585087781 ], "1_1_async_actor_calls_async": [ - 4259.771844696956, - 244.58821485834815 + 4261.744798110754, + 180.0414155637879 ], "1_1_async_actor_calls_sync": [ - 1459.7289131365046, - 14.372300668103277 + 1440.0280310845594, + 32.16094111992805 ], "1_1_async_actor_calls_with_args_async": [ - 2836.298297310687, - 165.56556787435736 + 2842.6446804977995, + 86.87223380532768 ], "1_n_actor_calls_async": [ - 8038.166251679982, - 223.66382715772104 + 7753.148948018643, + 157.64457409741257 ], "1_n_async_actor_calls_async": [ - 7382.681881276498, - 130.69555045858203 + 7607.418617152194, + 89.2638829207031 ], "client__1_1_actor_calls_async": [ - 1098.863141897179, - 11.579112801667774 + 1156.2740440851749, + 16.48223271213866 ], "client__1_1_actor_calls_concurrent": [ - 1085.0288964711467, - 4.148700210547401 + 1144.7059953730677, + 4.145550595536418 ], "client__1_1_actor_calls_sync": [ - 537.8164788509748, - 4.282391401398279 + 564.4854333177283, + 3.857233665824203 ], "client__get_calls": [ - 1159.3513798913632, - 25.153079890432657 + 1025.614536261544, + 21.381968599178617 ], "client__put_calls": [ - 817.4136861603523, - 35.13575238987404 + 869.4133022105747, + 28.436164429083114 ], "client__put_gigabytes": [ - 0.1559990403715773, - 0.0006899703405647251 + 0.1011154460629056, + 0.0022480613263383856 ], "client__tasks_and_get_batch": [ - 1.009944931213749, - 0.0320718636380897 + 1.040122205853533, + 0.03858532336305511 ], "client__tasks_and_put_batch": [ - 14560.030073574557, - 146.72299114824276 + 13581.806864962535, + 520.4510627044893 ], "multi_client_put_calls_Plasma_Store": [ - 16526.35985553258, - 400.3514368958908 + 15560.620089508539, + 96.2603254838798 ], "multi_client_put_gigabytes": [ - 38.137310138893675, - 1.3860853941620797 + 37.43614465888436, + 2.1762237421647583 ], "multi_client_tasks_async": [ - 21229.843138559452, - 1404.0869837056882 + 24135.499735285084, + 1599.8731087846127 ], "n_n_actor_calls_async": [ - 27375.624367126635, - 674.8368191945152 + 26441.297568592032, + 1050.529175955539 ], "n_n_actor_calls_with_arg_async": [ - 2759.3212097473174, - 60.45186810112816 + 2704.110675027102, + 28.368337606599287 ], "n_n_async_actor_calls_async": [ - 23674.50106467489, - 547.7052271058876 + 23412.17782093146, + 617.218384004416 ], "perf_metrics": [ { "perf_metric_name": "single_client_get_calls_Plasma_Store", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 10620.405550394937 + "perf_metric_value": 10119.7301338237 }, { "perf_metric_name": "single_client_put_calls_Plasma_Store", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 5173.290112206238 + "perf_metric_value": 5278.091294531883 }, { "perf_metric_name": "multi_client_put_calls_Plasma_Store", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 16526.35985553258 + "perf_metric_value": 15560.620089508539 }, { "perf_metric_name": "single_client_put_gigabytes", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 19.85639156989914 + "perf_metric_value": 18.71278038275444 }, { "perf_metric_name": "single_client_tasks_and_get_batch", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 5.800654754787365 + "perf_metric_value": 5.381854147597904 }, { "perf_metric_name": "multi_client_put_gigabytes", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 38.137310138893675 + "perf_metric_value": 37.43614465888436 }, { "perf_metric_name": "single_client_get_object_containing_10k_refs", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 13.371722002108683 + "perf_metric_value": 13.265651211414822 }, { "perf_metric_name": "single_client_wait_1k_refs", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 5.079952667320649 + "perf_metric_value": 4.952999927332959 }, { "perf_metric_name": "single_client_tasks_sync", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 980.7121217208985 + "perf_metric_value": 946.027654634476 }, { "perf_metric_name": "single_client_tasks_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 8040.530786886751 + "perf_metric_value": 7958.403181954658 }, { "perf_metric_name": "multi_client_tasks_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 21229.843138559452 + "perf_metric_value": 24135.499735285084 }, { "perf_metric_name": "1_1_actor_calls_sync", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 2011.916260420167 + "perf_metric_value": 2091.781722688228 }, { "perf_metric_name": "1_1_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 8663.654839458402 + "perf_metric_value": 8382.98999101571 }, { "perf_metric_name": "1_1_actor_calls_concurrent", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 5775.020315522301 + "perf_metric_value": 5185.592351945926 }, { "perf_metric_name": "1_n_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 8038.166251679982 + "perf_metric_value": 7753.148948018643 }, { "perf_metric_name": "n_n_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 27375.624367126635 + "perf_metric_value": 26441.297568592032 }, { "perf_metric_name": "n_n_actor_calls_with_arg_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 2759.3212097473174 + "perf_metric_value": 2704.110675027102 }, { "perf_metric_name": "1_1_async_actor_calls_sync", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 1459.7289131365046 + "perf_metric_value": 1440.0280310845594 }, { "perf_metric_name": "1_1_async_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 4259.771844696956 + "perf_metric_value": 4261.744798110754 }, { "perf_metric_name": "1_1_async_actor_calls_with_args_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 2836.298297310687 + "perf_metric_value": 2842.6446804977995 }, { "perf_metric_name": "1_n_async_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 7382.681881276498 + "perf_metric_value": 7607.418617152194 }, { "perf_metric_name": "n_n_async_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 23674.50106467489 + "perf_metric_value": 23412.17782093146 }, { "perf_metric_name": "placement_group_create/removal", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 764.5677165695956 + "perf_metric_value": 755.9481741578835 }, { "perf_metric_name": "client__get_calls", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 1159.3513798913632 + "perf_metric_value": 1025.614536261544 }, { "perf_metric_name": "client__put_calls", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 817.4136861603523 + "perf_metric_value": 869.4133022105747 }, { "perf_metric_name": "client__put_gigabytes", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 0.1559990403715773 + "perf_metric_value": 0.1011154460629056 }, { "perf_metric_name": "client__tasks_and_put_batch", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 14560.030073574557 + "perf_metric_value": 13581.806864962535 }, { "perf_metric_name": "client__1_1_actor_calls_sync", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 537.8164788509748 + "perf_metric_value": 564.4854333177283 }, { "perf_metric_name": "client__1_1_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 1098.863141897179 + "perf_metric_value": 1156.2740440851749 }, { "perf_metric_name": "client__1_1_actor_calls_concurrent", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 1085.0288964711467 + "perf_metric_value": 1144.7059953730677 }, { "perf_metric_name": "client__tasks_and_get_batch", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 1.009944931213749 + "perf_metric_value": 1.040122205853533 } ], "placement_group_create/removal": [ - 764.5677165695956, - 11.50741876717501 + 755.9481741578835, + 12.306329174758009 ], "single_client_get_calls_Plasma_Store": [ - 10620.405550394937, - 95.5780186318987 + 10119.7301338237, + 102.04801817695024 ], "single_client_get_object_containing_10k_refs": [ - 13.371722002108683, - 0.2715300404352367 + 13.265651211414822, + 0.01756980511554709 ], "single_client_put_calls_Plasma_Store": [ - 5173.290112206238, - 50.54867941540244 + 5278.091294531883, + 27.18593982260196 ], "single_client_put_gigabytes": [ - 19.85639156989914, - 8.982486882151242 + 18.71278038275444, + 8.13387701917967 ], "single_client_tasks_and_get_batch": [ - 5.800654754787365, - 3.260748466569974 + 5.381854147597904, + 3.1676227294432446 ], "single_client_tasks_async": [ - 8040.530786886751, - 508.5067401143829 + 7958.403181954658, + 437.13498052024147 ], "single_client_tasks_sync": [ - 980.7121217208985, - 15.070879654529714 + 946.027654634476, + 12.53937510184865 ], "single_client_wait_1k_refs": [ - 5.079952667320649, - 0.11950057107198113 + 4.952999927332959, + 0.036988477470103795 ] } diff --git a/release/perf_metrics/scalability/object_store.json b/release/perf_metrics/scalability/object_store.json index 367cb088c4bb..a2e424a23abd 100644 --- a/release/perf_metrics/scalability/object_store.json +++ b/release/perf_metrics/scalability/object_store.json @@ -1,12 +1,12 @@ { - "broadcast_time": 17.324763202, + "broadcast_time": 13.201167912000003, "num_nodes": 50, "object_size": 1073741824, "perf_metrics": [ { "perf_metric_name": "time_to_broadcast_1073741824_bytes_to_50_nodes", "perf_metric_type": "LATENCY", - "perf_metric_value": 17.324763202 + "perf_metric_value": 13.201167912000003 } ], "success": "1" diff --git a/release/perf_metrics/scalability/single_node.json b/release/perf_metrics/scalability/single_node.json index bc2cd08fe8df..be655d551555 100644 --- a/release/perf_metrics/scalability/single_node.json +++ b/release/perf_metrics/scalability/single_node.json @@ -1,8 +1,8 @@ { - "args_time": 18.84486551900001, - "get_time": 23.075941746000012, + "args_time": 19.17184469900002, + "get_time": 23.946038821000002, "large_object_size": 107374182400, - "large_object_time": 32.03462247800002, + "large_object_time": 31.283377702999985, "num_args": 10000, "num_get_args": 10000, "num_queued": 1000000, @@ -11,30 +11,30 @@ { "perf_metric_name": "10000_args_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 18.84486551900001 + "perf_metric_value": 19.17184469900002 }, { "perf_metric_name": "3000_returns_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 6.088559257 + "perf_metric_value": 5.863585175999987 }, { "perf_metric_name": "10000_get_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 23.075941746000012 + "perf_metric_value": 23.946038821000002 }, { "perf_metric_name": "1000000_queued_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 199.176572467 + "perf_metric_value": 190.35062810999997 }, { "perf_metric_name": "107374182400_large_object_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 32.03462247800002 + "perf_metric_value": 31.283377702999985 } ], - "queued_time": 199.176572467, - "returns_time": 6.088559257, + "queued_time": 190.35062810999997, + "returns_time": 5.863585175999987, "success": "1" } diff --git a/release/perf_metrics/stress_tests/stress_test_dead_actors.json b/release/perf_metrics/stress_tests/stress_test_dead_actors.json index 7daf8903fe7f..98efeb61e846 100644 --- a/release/perf_metrics/stress_tests/stress_test_dead_actors.json +++ b/release/perf_metrics/stress_tests/stress_test_dead_actors.json @@ -1,14 +1,14 @@ { - "avg_iteration_time": 1.1874613547325135, - "max_iteration_time": 3.250436544418335, - "min_iteration_time": 0.05550789833068848, + "avg_iteration_time": 1.1919621157646179, + "max_iteration_time": 3.34515118598938, + "min_iteration_time": 0.6364881992340088, "perf_metrics": [ { "perf_metric_name": "avg_iteration_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 1.1874613547325135 + "perf_metric_value": 1.1919621157646179 } ], "success": 1, - "total_time": 118.7462546825409 + "total_time": 119.196359872818 } diff --git a/release/perf_metrics/stress_tests/stress_test_many_tasks.json b/release/perf_metrics/stress_tests/stress_test_many_tasks.json index accdf6d571e9..1951e0e15b5e 100644 --- a/release/perf_metrics/stress_tests/stress_test_many_tasks.json +++ b/release/perf_metrics/stress_tests/stress_test_many_tasks.json @@ -3,45 +3,45 @@ { "perf_metric_name": "stage_0_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 7.344109535217285 + "perf_metric_value": 6.86789870262146 }, { "perf_metric_name": "stage_1_avg_iteration_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 12.96969530582428 + "perf_metric_value": 13.265494346618652 }, { "perf_metric_name": "stage_2_avg_iteration_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 33.957920932769774 + "perf_metric_value": 34.152964401245114 }, { "perf_metric_name": "stage_3_creation_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 1.8526091575622559 + "perf_metric_value": 2.032559394836426 }, { "perf_metric_name": "stage_3_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 1826.5975222587585 + "perf_metric_value": 1814.6457602977753 }, { "perf_metric_name": "stage_4_spread", "perf_metric_type": "LATENCY", - "perf_metric_value": 0.48570817077228695 + "perf_metric_value": 0.4687484200099014 } ], - "stage_0_time": 7.344109535217285, - "stage_1_avg_iteration_time": 12.96969530582428, - "stage_1_max_iteration_time": 13.717556715011597, - "stage_1_min_iteration_time": 11.527287244796753, - "stage_1_time": 129.69700860977173, - "stage_2_avg_iteration_time": 33.957920932769774, - "stage_2_max_iteration_time": 34.32049250602722, - "stage_2_min_iteration_time": 33.68821382522583, - "stage_2_time": 169.79015111923218, - "stage_3_creation_time": 1.8526091575622559, - "stage_3_time": 1826.5975222587585, - "stage_4_spread": 0.48570817077228695, + "stage_0_time": 6.86789870262146, + "stage_1_avg_iteration_time": 13.265494346618652, + "stage_1_max_iteration_time": 13.843246221542358, + "stage_1_min_iteration_time": 11.710993766784668, + "stage_1_time": 132.65499782562256, + "stage_2_avg_iteration_time": 34.152964401245114, + "stage_2_max_iteration_time": 34.738978147506714, + "stage_2_min_iteration_time": 33.831342458724976, + "stage_2_time": 170.76539039611816, + "stage_3_creation_time": 2.032559394836426, + "stage_3_time": 1814.6457602977753, + "stage_4_spread": 0.4687484200099014, "success": 1 } diff --git a/release/perf_metrics/stress_tests/stress_test_placement_group.json b/release/perf_metrics/stress_tests/stress_test_placement_group.json index 2ef542254e31..49a763bceb42 100644 --- a/release/perf_metrics/stress_tests/stress_test_placement_group.json +++ b/release/perf_metrics/stress_tests/stress_test_placement_group.json @@ -1,16 +1,16 @@ { - "avg_pg_create_time_ms": 1.4493968768766456, - "avg_pg_remove_time_ms": 1.2057934429429915, + "avg_pg_create_time_ms": 1.544663453452921, + "avg_pg_remove_time_ms": 1.3276310030028873, "perf_metrics": [ { "perf_metric_name": "avg_pg_create_time_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 1.4493968768766456 + "perf_metric_value": 1.544663453452921 }, { "perf_metric_name": "avg_pg_remove_time_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 1.2057934429429915 + "perf_metric_value": 1.3276310030028873 } ], "success": 1 From 0ed6da20380716a75848ce777f76951383252fe9 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 25 Aug 2025 13:38:15 -0700 Subject: [PATCH 255/634] [image] add slim image to the image build matrix (#55723) slim image is a smaller image than normal ray image. it has fewer packages installed, and does not contain the cuda dev sdk. Signed-off-by: Lonnie Liu --- .buildkite/_forge.rayci.yml | 45 ++++++++ ci/docker/ray-slim.cpu.base.wanda.yaml | 10 ++ ci/docker/ray-slim.cuda.base.wanda.yaml | 10 ++ docker/base-slim/Dockerfile | 146 ++++++++++++++++++++++++ 4 files changed, 211 insertions(+) create mode 100644 ci/docker/ray-slim.cpu.base.wanda.yaml create mode 100644 ci/docker/ray-slim.cuda.base.wanda.yaml create mode 100644 docker/base-slim/Dockerfile diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml index e74f90edf048..97acc658cc5a 100644 --- a/.buildkite/_forge.rayci.yml +++ b/.buildkite/_forge.rayci.yml @@ -1,4 +1,5 @@ group: forge +sort_key: "_forge" steps: - name: forge wanda: ci/docker/forge.wanda.yaml @@ -184,3 +185,47 @@ steps: IMAGE_TYPE: "ray-ml" ARCH_SUFFIX: "" depends_on: ray-mlcpubase + + - name: ray-slimcpubase + label: "wanda: ray-slim.py{{matrix}}.cpu.base" + tags: + - python_dependencies + - docker + wanda: ci/docker/ray-slim.cpu.base.wanda.yaml + depends_on: raycpubase + matrix: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + env: + PYTHON_VERSION: "{{matrix}}" + ARCH_SUFFIX: "" + + - name: ray-slimcudabase + label: "wanda: ray-slim.py{{matrix.python}}.cu{{matrix.cuda}}.base" + tags: + - python_dependencies + - docker + wanda: ci/docker/ray-slim.cuda.base.wanda.yaml + depends_on: raycudabase + matrix: + setup: + python: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + cuda: + - "11.7.1" + - "11.8.0" + - "12.1.1" + - "12.3.2" + - "12.4.1" + - "12.5.1" + - "12.6.3" + - "12.8.1" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + ARCH_SUFFIX: "" diff --git a/ci/docker/ray-slim.cpu.base.wanda.yaml b/ci/docker/ray-slim.cpu.base.wanda.yaml new file mode 100644 index 000000000000..b6d80f13992a --- /dev/null +++ b/ci/docker/ray-slim.cpu.base.wanda.yaml @@ -0,0 +1,10 @@ +name: "ray-slim.py$PYTHON_VERSION.cpu.base$ARCH_SUFFIX" +froms: ["ubuntu:22.04"] +dockerfile: docker/base-slim/Dockerfile +srcs: + - python/requirements_compiled.txt +build_args: + - PYTHON_VERSION + - BASE_IMAGE=ubuntu:22.04 +tags: + - cr.ray.io/rayproject/ray-slim.py$PYTHON_VERSION.cpu.base$ARCH_SUFFIX diff --git a/ci/docker/ray-slim.cuda.base.wanda.yaml b/ci/docker/ray-slim.cuda.base.wanda.yaml new file mode 100644 index 000000000000..8ae98a48d11a --- /dev/null +++ b/ci/docker/ray-slim.cuda.base.wanda.yaml @@ -0,0 +1,10 @@ +name: "ray-slim.py$PYTHON_VERSION.cu$CUDA_VERSION.base$ARCH_SUFFIX" +froms: ["nvidia/cuda:$CUDA_VERSION-runtime-ubuntu22.04"] +dockerfile: docker/base-slim/Dockerfile +srcs: + - python/requirements_compiled.txt +build_args: + - PYTHON_VERSION + - BASE_IMAGE=nvidia/cuda:$CUDA_VERSION-runtime-ubuntu22.04 +tags: + - cr.ray.io/rayproject/ray-slim.py$PYTHON_VERSION.cu$CUDA_VERSION.base$ARCH_SUFFIX diff --git a/docker/base-slim/Dockerfile b/docker/base-slim/Dockerfile new file mode 100644 index 000000000000..259bfba3d19c --- /dev/null +++ b/docker/base-slim/Dockerfile @@ -0,0 +1,146 @@ +# syntax=docker/dockerfile:1.3-labs + +# This Dockerfile is used to build the slim Ray image +# Mainly for use on Anyscale. + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +ARG PYTHON_VERSION + +RUN <> /etc/sudoers + +# Install uv +curl -sSL -o- https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="/usr/local/bin" sh + +# Determine the architecture of the host +if [[ "${HOSTTYPE}" =~ ^x86_64 ]]; then + ARCH="x86_64" +elif [[ "${HOSTTYPE}" =~ ^aarch64 ]]; then + ARCH="aarch64" +else + echo "Unsupported architecture ${HOSTTYPE}" >/dev/stderr + exit 1 +fi + +# Install dynolog +if [[ "$ARCH" == "x86_64" ]]; then + DYNOLOG_TMP="$(mktemp -d)" + ( + cd "${DYNOLOG_TMP}" + curl -sSL https://github.com/facebookincubator/dynolog/releases/download/v0.3.2/dynolog_0.3.2-0-amd64.deb -o dynolog_0.3.2-0-amd64.deb + sudo dpkg -i dynolog_0.3.2-0-amd64.deb + ) + rm -rf "${DYNOLOG_TMP}" +fi + +# Install azcopy +AZCOPY_VERSION="10.30.0" +AZCOPY_TMP="$(mktemp -d)" +( + cd "${AZCOPY_TMP}" + if [[ "$ARCH" == "x86_64" ]]; then + curl -sSfL "https://github.com/Azure/azure-storage-azcopy/releases/download/v${AZCOPY_VERSION}/azcopy_linux_amd64_${AZCOPY_VERSION}.tar.gz" \ + -o- | tar -xz "azcopy_linux_amd64_${AZCOPY_VERSION}/azcopy" + sudo mv "azcopy_linux_amd64_${AZCOPY_VERSION}/azcopy" /usr/local/bin/azcopy + else + curl -sSfL "https://github.com/Azure/azure-storage-azcopy/releases/download/v${AZCOPY_VERSION}/azcopy_linux_arm64_${AZCOPY_VERSION}.tar.gz" \ + -o- | tar -xz "azcopy_linux_arm64_${AZCOPY_VERSION}/azcopy" + sudo mv "azcopy_linux_arm64_${AZCOPY_VERSION}/azcopy" /usr/local/bin/azcopy + fi +) +rm -rf "${AZCOPY_TMP}" + +# Install awscli +AWSCLI_TMP="$(mktemp -d)" +( + cd "${AWSCLI_TMP}" + curl -sfL "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip" + unzip -q awscliv2.zip + sudo ./aws/install +) +rm -rf "${AWSCLI_TMP}" +aws --version + +EOF + +# Switch to ray user +USER ray +ENV HOME=/home/ray +WORKDIR /home/ray + +COPY python/requirements_compiled.txt /home/ray/requirements_compiled.txt + +RUN < Date: Mon, 25 Aug 2025 13:45:08 -0700 Subject: [PATCH 256/634] [core] move the update for node labels and total resources to ray syncer (#55727) ## Why are these changes needed? Currently, each Raylet gets the scheduling information of other nodes from two sources: GCS PubSub and Ray Syncer. Such as node total resources and node labels come from GCS PubSub, and node available resources come from Ray Syncer. This PR moves the updates for node labels and node total resources to the Ray Syncer from GCS PubSub. The goal is to make Ray Syncer the only one who is responsible for broadcasting all the node scheduling information. ### Changes: 1. LocalResourceManager will now carry labels when populating ResourceViewSyncMessage. 2. NodeManager will NOT update node labels and node total resources on NodeAdded, which is triggered when consuming GCS PubSub. 3. NodeManager will update node labels and node total resources on ConsumeSyncMessage. 4. ClusterResourceManager::UpdateNode will also update node labels. It is triggered by GcsResourceManager::ConsumeSyncMessage. ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Rueian Signed-off-by: Rueian Co-authored-by: Dhyey Shah --- src/ray/protobuf/ray_syncer.proto | 2 ++ src/ray/raylet/node_manager.cc | 28 ++++++++----------- .../scheduling/cluster_resource_manager.cc | 11 +++++--- .../scheduling/local_resource_manager.cc | 4 +++ .../tests/cluster_resource_manager_test.cc | 25 +++++++++++++++++ .../tests/local_resource_manager_test.cc | 21 ++++++++++++++ src/ray/raylet/tests/node_manager_test.cc | 27 ++++++++++++++++++ 7 files changed, 98 insertions(+), 20 deletions(-) diff --git a/src/ray/protobuf/ray_syncer.proto b/src/ray/protobuf/ray_syncer.proto index fe239695f129..36da8decc794 100644 --- a/src/ray/protobuf/ray_syncer.proto +++ b/src/ray/protobuf/ray_syncer.proto @@ -45,6 +45,8 @@ message ResourceViewSyncMessage { int64 draining_deadline_timestamp_ms = 6; // Why the node is not idle. repeated string node_activity = 7; + // The key-value labels of this node. + map labels = 8; } message RaySyncMessage { diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 3880f3fe3215..61fffd8cee91 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -783,21 +783,6 @@ void NodeManager::NodeAdded(const GcsNodeInfo &node_info) { remote_node_manager_addresses_[node_id] = std::make_pair(node_info.node_manager_address(), node_info.node_manager_port()); - // Set node labels when node added. - absl::flat_hash_map labels(node_info.labels().begin(), - node_info.labels().end()); - cluster_resource_scheduler_.GetClusterResourceManager().SetNodeLabels( - scheduling::NodeID(node_id.Binary()), labels); - - // TODO: Always use the message from ray syncer. // NOLINT - ResourceRequest resources; - for (auto &resource_entry : node_info.resources_total()) { - resources.Set(scheduling::ResourceID(resource_entry.first), - FixedPoint(resource_entry.second)); - } - if (ResourceCreateUpdated(node_id, resources)) { - cluster_task_manager_.ScheduleAndDispatchTasks(); - } // Update the resource view if a new message has been sent. if (auto sync_msg = ray_syncer_.GetSyncMessage(node_id.Binary(), syncer::MessageType::RESOURCE_VIEW)) { @@ -2779,7 +2764,18 @@ void NodeManager::ConsumeSyncMessage( syncer::ResourceViewSyncMessage resource_view_sync_message; resource_view_sync_message.ParseFromString(message->sync_message()); NodeID node_id = NodeID::FromBinary(message->node_id()); - if (UpdateResourceUsage(node_id, resource_view_sync_message)) { + // Set node labels when node added. + auto node_labels = MapFromProtobuf(resource_view_sync_message.labels()); + cluster_resource_scheduler_.GetClusterResourceManager().SetNodeLabels( + scheduling::NodeID(node_id.Binary()), node_labels); + ResourceRequest resources; + for (auto &resource_entry : resource_view_sync_message.resources_total()) { + resources.Set(scheduling::ResourceID(resource_entry.first), + FixedPoint(resource_entry.second)); + } + const bool capacity_updated = ResourceCreateUpdated(node_id, resources); + const bool usage_update = UpdateResourceUsage(node_id, resource_view_sync_message); + if (capacity_updated || usage_update) { cluster_task_manager_.ScheduleAndDispatchTasks(); } } else if (message->message_type() == syncer::MessageType::COMMANDS) { diff --git a/src/ray/raylet/scheduling/cluster_resource_manager.cc b/src/ray/raylet/scheduling/cluster_resource_manager.cc index 225beb0cfbe6..de7f10825c84 100644 --- a/src/ray/raylet/scheduling/cluster_resource_manager.cc +++ b/src/ray/raylet/scheduling/cluster_resource_manager.cc @@ -77,16 +77,19 @@ bool ClusterResourceManager::UpdateNode( return false; } - auto resources_total = MapFromProtobuf(resource_view_sync_message.resources_total()); - auto resources_available = + const auto resources_total = + MapFromProtobuf(resource_view_sync_message.resources_total()); + const auto resources_available = MapFromProtobuf(resource_view_sync_message.resources_available()); + auto node_labels = MapFromProtobuf(resource_view_sync_message.labels()); NodeResources node_resources = ResourceMapToNodeResources(resources_total, resources_available); NodeResources local_view; RAY_CHECK(GetNodeResources(node_id, &local_view)); - local_view.total = node_resources.total; - local_view.available = node_resources.available; + local_view.total = std::move(node_resources.total); + local_view.available = std::move(node_resources.available); + local_view.labels = std::move(node_labels); local_view.object_pulls_queued = resource_view_sync_message.object_pulls_queued(); // Update the idle duration for the node in terms of resources usage. diff --git a/src/ray/raylet/scheduling/local_resource_manager.cc b/src/ray/raylet/scheduling/local_resource_manager.cc index 5ccec894dd4a..f30a8997920d 100644 --- a/src/ray/raylet/scheduling/local_resource_manager.cc +++ b/src/ray/raylet/scheduling/local_resource_manager.cc @@ -306,6 +306,10 @@ void LocalResourceManager::PopulateResourceViewSyncMessage( syncer::ResourceViewSyncMessage &resource_view_sync_message) const { NodeResources resources = ToNodeResources(); + // Populate node labels. + resource_view_sync_message.mutable_labels()->insert(resources.labels.begin(), + resources.labels.end()); + auto total = resources.total.GetResourceMap(); resource_view_sync_message.mutable_resources_total()->insert(total.begin(), total.end()); diff --git a/src/ray/raylet/scheduling/tests/cluster_resource_manager_test.cc b/src/ray/raylet/scheduling/tests/cluster_resource_manager_test.cc index 0324c84bb31a..f7d4506dd4e5 100644 --- a/src/ray/raylet/scheduling/tests/cluster_resource_manager_test.cc +++ b/src/ray/raylet/scheduling/tests/cluster_resource_manager_test.cc @@ -60,6 +60,31 @@ struct ClusterResourceManagerTest : public ::testing::Test { std::unique_ptr manager; }; +TEST_F(ClusterResourceManagerTest, UpdateNode) { + // Prepare a sync message with updated totals/available, labels and flags. + syncer::ResourceViewSyncMessage payload; + payload.mutable_resources_total()->insert({"CPU", 10.0}); + payload.mutable_resources_available()->insert({"CPU", 5.0}); + payload.mutable_labels()->insert({"zone", "us-east-1a"}); + payload.set_object_pulls_queued(true); + payload.set_idle_duration_ms(42); + payload.set_is_draining(true); + payload.set_draining_deadline_timestamp_ms(123456); + + // Update existing node and validate the local view reflects the payload. + ASSERT_TRUE(manager->UpdateNode(node0, payload)); + + const auto &node_resources = manager->GetNodeResources(node0); + ASSERT_EQ(node_resources.total.Get(scheduling::ResourceID("CPU")), 10); + ASSERT_EQ(node_resources.available.Get(scheduling::ResourceID("CPU")), 5); + ASSERT_EQ(node_resources.labels.at("zone"), "us-east-1a"); + ASSERT_TRUE(node_resources.object_pulls_queued); + ASSERT_EQ(node_resources.idle_resource_duration_ms, 42); + ASSERT_TRUE(node_resources.is_draining); + ASSERT_EQ(node_resources.draining_deadline_timestamp_ms, 123456); + ASSERT_TRUE(node_resources.last_resource_update_time.has_value()); +} + TEST_F(ClusterResourceManagerTest, DebugStringTest) { // Test max_num_nodes_to_include parameter is working. ASSERT_EQ(std::vector(absl::StrSplit(manager->DebugString(), "node id:")) diff --git a/src/ray/raylet/scheduling/tests/local_resource_manager_test.cc b/src/ray/raylet/scheduling/tests/local_resource_manager_test.cc index 30b30e573f13..17d9d260c10b 100644 --- a/src/ray/raylet/scheduling/tests/local_resource_manager_test.cc +++ b/src/ray/raylet/scheduling/tests/local_resource_manager_test.cc @@ -371,4 +371,25 @@ TEST_F(LocalResourceManagerTest, CreateSyncMessageNegativeResourceAvailability) ASSERT_EQ(resource_view_sync_messge.resources_available().at("CPU"), 0); } +TEST_F(LocalResourceManagerTest, PopulateResourceViewSyncMessage) { + // Prepare node resources with labels. + NodeResources resources = CreateNodeResources({{ResourceID::CPU(), 2.0}}); + resources.labels = {{"label1", "value1"}, {"label2", "value2"}}; + + manager = std::make_unique( + local_node_id, resources, nullptr, nullptr, nullptr, nullptr); + + // Populate the sync message and verify labels are copied over. + syncer::ResourceViewSyncMessage msg; + manager->PopulateResourceViewSyncMessage(msg); + + // Verify total resources are populated. + ASSERT_EQ(msg.resources_total_size(), 1); + ASSERT_EQ(msg.resources_total().at("CPU"), 2.0); + // Verify labels are populated. + ASSERT_EQ(msg.labels_size(), 2); + ASSERT_EQ(msg.labels().at("label1"), "value1"); + ASSERT_EQ(msg.labels().at("label2"), "value2"); +} + } // namespace ray diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index 36b263b87207..ffc17e1af5e6 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -749,6 +749,33 @@ TEST_F(NodeManagerTest, TestPinningAnObjectPendingDeletionFails) { EXPECT_FALSE(failed_pin_reply.successes(0)); } +TEST_F(NodeManagerTest, TestConsumeSyncMessage) { + // Create and wrap a mock resource view sync message. + syncer::ResourceViewSyncMessage payload; + payload.mutable_resources_total()->insert({"CPU", 10.0}); + payload.mutable_resources_available()->insert({"CPU", 10.0}); + payload.mutable_labels()->insert({"label1", "value1"}); + + std::string serialized; + ASSERT_TRUE(payload.SerializeToString(&serialized)); + + auto node_id = NodeID::FromRandom(); + syncer::RaySyncMessage msg; + msg.set_node_id(node_id.Binary()); + msg.set_message_type(syncer::MessageType::RESOURCE_VIEW); + msg.set_sync_message(serialized); + + node_manager_->ConsumeSyncMessage(std::make_shared(msg)); + + // Verify node resources and labels were updated. + const auto &node_resources = + cluster_resource_scheduler_->GetClusterResourceManager().GetNodeResources( + scheduling::NodeID(node_id.Binary())); + EXPECT_EQ(node_resources.labels.at("label1"), "value1"); + EXPECT_EQ(node_resources.total.Get(scheduling::ResourceID("CPU")).Double(), 10.0); + EXPECT_EQ(node_resources.available.Get(scheduling::ResourceID("CPU")).Double(), 10.0); +} + TEST_F(NodeManagerTest, TestResizeLocalResourceInstancesSuccessful) { // Test 1: Up scaling (increasing resource capacity) rpc::ResizeLocalResourceInstancesRequest request; From 48a993785c4cc6cdc557ec23ab2aa9269e9fda3e Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 25 Aug 2025 15:45:50 -0500 Subject: [PATCH 257/634] [core] Remove experimental `max_cpu_frac_per_node` (#53610) [Introduced](https://github.com/ray-project/ray/pull/26397) for Ray AIR, no longer used by any Ray libraries, but never finished ([does not work with autoscaler](https://github.com/ray-project/ray/issues/26791)). This is exposed in the public `ray.util.placement_group` API, but it is marked experimental with a strong caveat and we aren't aware of any external usage, so going to directly remove it. --------- Signed-off-by: Edward Oakes --- .../ray/runtime/task/native_task_submitter.cc | 3 +- python/ray/_raylet.pyx | 2 - python/ray/includes/common.pxd | 1 - python/ray/util/placement_group.py | 28 --- src/ray/common/placement_group.cc | 4 - src/ray/common/placement_group.h | 4 - src/ray/core_worker/common.h | 4 - src/ray/core_worker/core_worker.cc | 1 - ...io_ray_runtime_task_NativeTaskSubmitter.cc | 3 +- .../gcs/gcs_server/gcs_placement_group_mgr.cc | 4 - .../gcs/gcs_server/gcs_placement_group_mgr.h | 5 - .../gcs_placement_group_scheduler.cc | 9 +- .../gcs_placement_group_scheduler.h | 1 - src/ray/gcs/tests/gcs_test_util.h | 1 - src/ray/protobuf/common.proto | 2 - src/ray/protobuf/gcs.proto | 2 - .../policy/bundle_scheduling_policy.cc | 174 ++---------------- .../policy/bundle_scheduling_policy.h | 9 +- .../scheduling/policy/scheduling_options.h | 27 +-- .../tests/hybrid_scheduling_policy_test.cc | 1 - .../policy/tests/scheduling_policy_test.cc | 132 +------------ .../cluster_resource_scheduler_2_test.cc | 2 - 22 files changed, 31 insertions(+), 388 deletions(-) diff --git a/cpp/src/ray/runtime/task/native_task_submitter.cc b/cpp/src/ray/runtime/task/native_task_submitter.cc index b2e035cc415e..c42ecf725120 100644 --- a/cpp/src/ray/runtime/task/native_task_submitter.cc +++ b/cpp/src/ray/runtime/task/native_task_submitter.cc @@ -200,8 +200,7 @@ ray::PlacementGroup NativeTaskSubmitter::CreatePlacementGroup( create_options.name, (ray::core::PlacementStrategy)create_options.strategy, create_options.bundles, - false, - 1.0); + false); ray::PlacementGroupID placement_group_id; auto status = CoreWorkerProcess::GetCoreWorker().CreatePlacementGroup( options, &placement_group_id); diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index 282c39a466a5..f513243a660f 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -3880,7 +3880,6 @@ cdef class CoreWorker: c_vector[unordered_map[c_string, double]] bundles, c_string strategy, c_bool is_detached, - double max_cpu_fraction_per_node, soft_target_node_id, c_vector[unordered_map[c_string, c_string]] bundle_label_selector): cdef: @@ -3912,7 +3911,6 @@ cdef class CoreWorker: c_strategy, bundles, is_detached, - max_cpu_fraction_per_node, c_soft_target_node_id, bundle_label_selector), &c_placement_group_id)) diff --git a/python/ray/includes/common.pxd b/python/ray/includes/common.pxd index 82d14d92e63a..6e50fd8ae35d 100644 --- a/python/ray/includes/common.pxd +++ b/python/ray/includes/common.pxd @@ -372,7 +372,6 @@ cdef extern from "ray/core_worker/common.h" nogil: CPlacementStrategy strategy, const c_vector[unordered_map[c_string, double]] &bundles, c_bool is_detached, - double max_cpu_fraction_per_node, CNodeID soft_target_node_id, const c_vector[unordered_map[c_string, c_string]] &bundle_label_selector, ) diff --git a/python/ray/util/placement_group.py b/python/ray/util/placement_group.py index 02c45f380484..286036118934 100644 --- a/python/ray/util/placement_group.py +++ b/python/ray/util/placement_group.py @@ -148,7 +148,6 @@ def placement_group( strategy: str = "PACK", name: str = "", lifetime: Optional[str] = None, - _max_cpu_fraction_per_node: float = 1.0, _soft_target_node_id: Optional[str] = None, bundle_label_selector: List[Dict[str, str]] = None, ) -> PlacementGroup: @@ -170,14 +169,6 @@ def placement_group( will fate share with its creator and will be deleted once its creator is dead, or "detached", which means the placement group will live as a global object independent of the creator. - _max_cpu_fraction_per_node: (Experimental) Disallow placing bundles on nodes - if it would cause the fraction of CPUs used by bundles from *any* placement - group on the node to exceed this fraction. This effectively sets aside - CPUs that placement groups cannot occupy on nodes. when - `max_cpu_fraction_per_node < 1.0`, at least 1 CPU will be excluded from - placement group scheduling. Note: This feature is experimental and is not - recommended for use with autoscaling clusters (scale-up will not trigger - properly). _soft_target_node_id: (Private, Experimental) Soft hint where bundles of this placement group should be placed. The target node is specified by it's hex ID. @@ -202,7 +193,6 @@ def placement_group( bundles=bundles, strategy=strategy, lifetime=lifetime, - _max_cpu_fraction_per_node=_max_cpu_fraction_per_node, _soft_target_node_id=_soft_target_node_id, bundle_label_selector=bundle_label_selector, ) @@ -220,7 +210,6 @@ def placement_group( bundles, strategy, detached, - _max_cpu_fraction_per_node, _soft_target_node_id, bundle_label_selector, ) @@ -353,7 +342,6 @@ def validate_placement_group( bundles: List[Dict[str, float]], strategy: str = "PACK", lifetime: Optional[str] = None, - _max_cpu_fraction_per_node: float = 1.0, _soft_target_node_id: Optional[str] = None, bundle_label_selector: List[Dict[str, str]] = None, ) -> bool: @@ -361,22 +349,6 @@ def validate_placement_group( Raises ValueError if inputs are invalid. """ - - assert _max_cpu_fraction_per_node is not None - - if _max_cpu_fraction_per_node != 1.0: - warnings.warn( - "The experimental '_max_cpu_fraction_per_node' option for placement groups " - "is deprecated and will be removed in a future version of Ray." - ) - - if _max_cpu_fraction_per_node <= 0 or _max_cpu_fraction_per_node > 1: - raise ValueError( - "Invalid argument `_max_cpu_fraction_per_node`: " - f"{_max_cpu_fraction_per_node}. " - "_max_cpu_fraction_per_node must be a float between 0 and 1. " - ) - if _soft_target_node_id and strategy != "STRICT_PACK": raise ValueError( "_soft_target_node_id currently only works " diff --git a/src/ray/common/placement_group.cc b/src/ray/common/placement_group.cc index 93431cf4c0f4..15c1d825aa15 100644 --- a/src/ray/common/placement_group.cc +++ b/src/ray/common/placement_group.cc @@ -44,8 +44,4 @@ BundleSpecification PlacementGroupSpecification::GetBundle(int position) const { std::string PlacementGroupSpecification::GetName() const { return std::string(message_->name()); } - -double PlacementGroupSpecification::GetMaxCpuFractionPerNode() const { - return message_->max_cpu_fraction_per_node(); -} } // namespace ray diff --git a/src/ray/common/placement_group.h b/src/ray/common/placement_group.h index e20776e3aa5b..f053a917ac70 100644 --- a/src/ray/common/placement_group.h +++ b/src/ray/common/placement_group.h @@ -60,8 +60,6 @@ class PlacementGroupSpecification : public MessageWrapper> &bundles, const rpc::PlacementStrategy strategy, const bool is_detached, - double max_cpu_fraction_per_node, NodeID soft_target_node_id, const JobID &creator_job_id, const ActorID &creator_actor_id, @@ -105,7 +102,6 @@ class PlacementGroupSpecBuilder { message_->set_creator_actor_id(creator_actor_id.Binary()); message_->set_creator_actor_dead(creator_actor_id.IsNil()); message_->set_is_detached(is_detached); - message_->set_max_cpu_fraction_per_node(max_cpu_fraction_per_node); message_->set_soft_target_node_id(soft_target_node_id.Binary()); for (size_t i = 0; i < bundles.size(); i++) { diff --git a/src/ray/core_worker/common.h b/src/ray/core_worker/common.h index 2cbdbf5bdafa..afd98d20a568 100644 --- a/src/ray/core_worker/common.h +++ b/src/ray/core_worker/common.h @@ -219,7 +219,6 @@ struct PlacementGroupCreationOptions { PlacementStrategy strategy, std::vector> bundles, bool is_detached_p, - double max_cpu_fraction_per_node, NodeID soft_target_node_id = NodeID::Nil(), std::vector> bundle_label_selector = {}) @@ -227,7 +226,6 @@ struct PlacementGroupCreationOptions { strategy_(strategy), bundles_(std::move(bundles)), is_detached_(is_detached_p), - max_cpu_fraction_per_node_(max_cpu_fraction_per_node), soft_target_node_id_(soft_target_node_id), bundle_label_selector_(std::move(bundle_label_selector)) { RAY_CHECK(soft_target_node_id_.IsNil() || strategy_ == PlacementStrategy::STRICT_PACK) @@ -242,8 +240,6 @@ struct PlacementGroupCreationOptions { const std::vector> bundles_; /// Whether to keep the placement group persistent after its creator dead. const bool is_detached_ = false; - /// The maximum fraction of CPU cores this placement group can take up on each node. - const double max_cpu_fraction_per_node_; /// ID of the target node where bundles should be placed /// iff the target node has enough available resources and alive. /// Otherwise, the bundles can be placed elsewhere. diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index ae467adc6ca0..8f1b0dd9bd4f 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -2219,7 +2219,6 @@ Status CoreWorker::CreatePlacementGroup( placement_group_creation_options.bundles_, placement_group_creation_options.strategy_, placement_group_creation_options.is_detached_, - placement_group_creation_options.max_cpu_fraction_per_node_, placement_group_creation_options.soft_target_node_id_, worker_context_->GetCurrentJobID(), worker_context_->GetCurrentActorID(), diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc index fc565d26e8c0..363d51234a12 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskSubmitter.cc @@ -358,8 +358,7 @@ inline PlacementGroupCreationOptions ToPlacementGroupCreationOptions( return PlacementGroupCreationOptions(name, ConvertStrategy(java_strategy), bundles, - /*is_detached=*/false, - /*max_cpu_fraction_per_node*/ 1.0); + /*is_detached=*/false); } #ifdef __cplusplus diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc index 71b200e2cd7c..3cd7af20aaac 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc @@ -166,10 +166,6 @@ bool GcsPlacementGroup::IsDetached() const { return placement_group_table_data_.is_detached(); } -double GcsPlacementGroup::GetMaxCpuFractionPerNode() const { - return placement_group_table_data_.max_cpu_fraction_per_node(); -} - NodeID GcsPlacementGroup::GetSoftTargetNodeID() const { return NodeID::FromBinary(placement_group_table_data_.soft_target_node_id()); } diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h index 00dc3b0baeac..80365c542c87 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h @@ -85,8 +85,6 @@ class GcsPlacementGroup { placement_group_table_data_.set_creator_actor_dead( placement_group_spec.creator_actor_dead()); placement_group_table_data_.set_is_detached(placement_group_spec.is_detached()); - placement_group_table_data_.set_max_cpu_fraction_per_node( - placement_group_spec.max_cpu_fraction_per_node()); placement_group_table_data_.set_soft_target_node_id( placement_group_spec.soft_target_node_id()); placement_group_table_data_.set_ray_namespace(ray_namespace); @@ -162,9 +160,6 @@ class GcsPlacementGroup { /// Returns whether or not this is a detached placement group. bool IsDetached() const; - /// Returns the maximum CPU fraction per node for this placement group. - double GetMaxCpuFractionPerNode() const; - /// Return the target node ID where bundles of this placement group should be placed. /// Only works for STRICT_PACK placement group. NodeID GetSoftTargetNodeID() const; diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc index 6a981400adf5..703d72c7a4dd 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc @@ -72,7 +72,6 @@ void GcsPlacementGroupScheduler::ScheduleUnplacedBundles( auto scheduling_options = CreateSchedulingOptions(placement_group->GetPlacementGroupID(), strategy, - placement_group->GetMaxCpuFractionPerNode(), placement_group->GetSoftTargetNodeID()); auto scheduling_result = cluster_resource_scheduler_.Schedule(resource_request_list, scheduling_options); @@ -475,22 +474,20 @@ GcsPlacementGroupScheduler::CreateSchedulingContext( SchedulingOptions GcsPlacementGroupScheduler::CreateSchedulingOptions( const PlacementGroupID &placement_group_id, rpc::PlacementStrategy strategy, - double max_cpu_fraction_per_node, NodeID soft_target_node_id) { switch (strategy) { case rpc::PlacementStrategy::PACK: - return SchedulingOptions::BundlePack(max_cpu_fraction_per_node); + return SchedulingOptions::BundlePack(); case rpc::PlacementStrategy::SPREAD: - return SchedulingOptions::BundleSpread(max_cpu_fraction_per_node); + return SchedulingOptions::BundleSpread(); case rpc::PlacementStrategy::STRICT_PACK: return SchedulingOptions::BundleStrictPack( - max_cpu_fraction_per_node, soft_target_node_id.IsNil() ? scheduling::NodeID::Nil() : scheduling::NodeID(soft_target_node_id.Binary())); case rpc::PlacementStrategy::STRICT_SPREAD: return SchedulingOptions::BundleStrictSpread( - max_cpu_fraction_per_node, CreateSchedulingContext(placement_group_id)); + CreateSchedulingContext(placement_group_id)); default: RAY_LOG(FATAL) << "Unsupported scheduling type: " << rpc::PlacementStrategy_Name(strategy); diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h index ebd5e9e460e9..6c668fccb8a8 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h @@ -465,7 +465,6 @@ class GcsPlacementGroupScheduler : public GcsPlacementGroupSchedulerInterface { /// Create scheduling options. SchedulingOptions CreateSchedulingOptions(const PlacementGroupID &placement_group_id, rpc::PlacementStrategy strategy, - double max_cpu_fraction_per_node, NodeID soft_target_node_id); /// Try to release bundle resource to cluster resource manager. diff --git a/src/ray/gcs/tests/gcs_test_util.h b/src/ray/gcs/tests/gcs_test_util.h index 4b5b125a97a7..0d27f047190c 100644 --- a/src/ray/gcs/tests/gcs_test_util.h +++ b/src/ray/gcs/tests/gcs_test_util.h @@ -174,7 +174,6 @@ struct Mocker { bundles, strategy, /* is_detached */ false, - /* max_cpu_fraction_per_node */ 1.0, /* soft_target_node_id */ NodeID::Nil(), job_id, actor_id, diff --git a/src/ray/protobuf/common.proto b/src/ray/protobuf/common.proto index d1dae8dd7c68..3c990b2f0a53 100644 --- a/src/ray/protobuf/common.proto +++ b/src/ray/protobuf/common.proto @@ -655,8 +655,6 @@ message PlacementGroupSpec { bool creator_actor_dead = 8; // Whether the placement group is persistent. bool is_detached = 9; - // The maximum fraction of CPU cores that this placement group can use on each node. - double max_cpu_fraction_per_node = 10; // Binary ID of the target node where bundles should be placed // iff the target node has enough available resources and alive. // Otherwise, the bundles can be placed elsewhere. diff --git a/src/ray/protobuf/gcs.proto b/src/ray/protobuf/gcs.proto index 972c80489b25..9d350bca72e8 100644 --- a/src/ray/protobuf/gcs.proto +++ b/src/ray/protobuf/gcs.proto @@ -642,8 +642,6 @@ message PlacementGroupTableData { // The placement group's stats / information such as when it is created or // what's the current scheduling state. PlacementGroupStats stats = 12; - // The maximum fraction of CPU cores that this placement group can use on each node. - double max_cpu_fraction_per_node = 13; // Binary ID of the target node where bundles should be placed // iff the target node has enough available resources and alive. // Otherwise, the bundles can be placed elsewhere. diff --git a/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.cc b/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.cc index 2eac56977c51..d01871377f3d 100644 --- a/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.cc +++ b/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.cc @@ -14,81 +14,6 @@ #include "ray/raylet/scheduling/policy/bundle_scheduling_policy.h" -namespace { - -/// Return true if scheduling this bundle (with resource_request) will exceed the -/// max cpu fraction for placement groups. This is per node. -/// -/// \param node_resources The resource of the current node. -/// \param bundle_resource_request The requested resources for the current bundle. -/// \param max_cpu_fraction_per_node Highest CPU fraction the bundles can take up. -/// \param available_cpus_before_curernt_pg_request Available CPUs on this node before -/// scheduling the current pg request. It is used to calculate how many CPUs are -/// allocated by the current bundles so far. It will help us figuring out -/// the total CPU allocation from the current bundles for this node. -bool AllocationWillExceedMaxCpuFraction( - const ray::NodeResources &node_resources, - const ray::ResourceRequest &bundle_resource_request, - double max_cpu_fraction_per_node, - double available_cpus_before_curernt_pg_request) { - if (max_cpu_fraction_per_node == 1.0) { - // Allocation will never exceed the threshold if the fraction == 1.0. - return false; - } - - auto cpu_id = ray::ResourceID::CPU(); - auto total_cpus = node_resources.total.Get(cpu_id).Double(); - - // Calculate max_reservable_cpus - auto max_reservable_cpus = - max_cpu_fraction_per_node * node_resources.total.Get(cpu_id).Double(); - - // If the max reservable cpu < 1, we allow at least 1 CPU. - if (max_reservable_cpus < 1) { - max_reservable_cpus = 1; - } - - // We guarantee at least 1 CPU is excluded from the placement group - // when max_cpu_fraction_per_node is specified. - if (max_reservable_cpus > total_cpus - 1) { - max_reservable_cpus = total_cpus - 1; - } - - /* - To calculate if allocating a new bundle will exceed the pg max_fraction, - we need a sum of - - - CPUs used by placement groups before. - - CPUs that will be allocated by the current pg request. - */ - - // Get the sum of all cpu allocated by placement group on this node. - FixedPoint cpus_used_by_pg_before(0); - for (const auto &resource_id : node_resources.total.ExplicitResourceIds()) { - if (ray::GetOriginalResourceNameFromWildcardResource(resource_id.Binary()) == "CPU") { - cpus_used_by_pg_before += node_resources.total.Get(resource_id); - } - } - - // Get the CPUs allocated by current pg request so far. - // Note that when we schedule the current pg, we allocate resources - // temporarily meaning `node_resources.available` will contain - // available CPUs after allocating CPUs for the current pg request. - auto cpus_allocated_by_current_pg_request = - (available_cpus_before_curernt_pg_request - - node_resources.available.Get(cpu_id).Double()); - - auto cpus_to_allocate_by_current_pg_request = - (cpus_allocated_by_current_pg_request + - bundle_resource_request.Get(cpu_id).Double()); - - auto cpus_used_by_pg_after = - cpus_used_by_pg_before.Double() + cpus_to_allocate_by_current_pg_request; - return cpus_used_by_pg_after > max_reservable_cpus; -} - -} // namespace - namespace ray { namespace raylet_scheduling_policy { @@ -117,19 +42,6 @@ BundleSchedulingPolicy::SelectCandidateNodes(const SchedulingContext *context) c return result; } -/// Return the map of node id -> available cpus before the current bundle scheduling. -/// It is used to calculate how many CPUs have been allocated for the current bundles. -const absl::flat_hash_map -BundleSchedulingPolicy::GetAvailableCpusBeforeBundleScheduling() const { - absl::flat_hash_map result; - for (const auto &entry : cluster_resource_manager_.GetResourceView()) { - result.emplace( - entry.first, - entry.second.GetLocalView().available.Get(ray::ResourceID::CPU()).Double()); - } - return result; -} - std::pair, std::vector> BundleSchedulingPolicy::SortRequiredResources( const std::vector &resource_request_list) { @@ -203,9 +115,7 @@ BundleSchedulingPolicy::SortRequiredResources( std::pair BundleSchedulingPolicy::GetBestNode( const ResourceRequest &required_resources, const absl::flat_hash_map &candidate_nodes, - const SchedulingOptions &options, - const absl::flat_hash_map - &available_cpus_before_bundle_scheduling) const { + const SchedulingOptions &options) const { double best_node_score = -1; auto best_node_id = scheduling::NodeID::Nil(); const Node *best_node = nullptr; @@ -213,14 +123,6 @@ std::pair BundleSchedulingPolicy::GetBestNode( // Score the nodes. for (const auto &[node_id, node] : candidate_nodes) { const auto &node_resources = node->GetLocalView(); - if (AllocationWillExceedMaxCpuFraction( - node_resources, - required_resources, - options.max_cpu_fraction_per_node_, - available_cpus_before_bundle_scheduling.at(node_id))) { - continue; - } - double node_score = node_scorer_->Score(required_resources, node_resources); if (best_node_id.IsNil() || best_node_score < node_score) { best_node_id = node_id; @@ -246,9 +148,6 @@ SchedulingResult BundlePackSchedulingPolicy::Schedule( return SchedulingResult::Infeasible(); } - const auto available_cpus_before_bundle_scheduling = - GetAvailableCpusBeforeBundleScheduling(); - // First schedule scarce resources (such as GPU) and large capacity resources to improve // the scheduling success rate. auto sorted_result = SortRequiredResources(resource_request_list); @@ -266,10 +165,7 @@ SchedulingResult BundlePackSchedulingPolicy::Schedule( while (!required_resources_list_copy.empty()) { const auto &required_resources_index = required_resources_list_copy.front().first; const auto &required_resources = required_resources_list_copy.front().second; - auto best_node = GetBestNode(*required_resources, - candidate_nodes, - options, - available_cpus_before_bundle_scheduling); + auto best_node = GetBestNode(*required_resources, candidate_nodes, options); if (best_node.first.IsNil()) { // There is no node to meet the scheduling requirements. break; @@ -285,14 +181,8 @@ SchedulingResult BundlePackSchedulingPolicy::Schedule( // We try to schedule more resources on one node. for (auto iter = required_resources_list_copy.begin(); iter != required_resources_list_copy.end();) { - if (node_resources.IsAvailable(*iter->second) // If the node has enough resources. - && !AllocationWillExceedMaxCpuFraction( // and allocating resources won't - // exceed max cpu fraction. - node_resources, - *iter->second, - options.max_cpu_fraction_per_node_, - available_cpus_before_bundle_scheduling.at(best_node.first))) { - // Then allocate it. + // If the node has sufficient resources, allocate it. + if (node_resources.IsAvailable(*iter->second)) { RAY_CHECK(cluster_resource_manager_.SubtractNodeAvailableResources( best_node.first, *iter->second)); result_nodes[iter->first] = best_node.first; @@ -335,9 +225,6 @@ SchedulingResult BundleSpreadSchedulingPolicy::Schedule( return SchedulingResult::Infeasible(); } - const auto available_cpus_before_bundle_scheduling = - GetAvailableCpusBeforeBundleScheduling(); - // First schedule scarce resources (such as GPU) and large capacity resources to improve // the scheduling success rate. auto sorted_result = SortRequiredResources(resource_request_list); @@ -348,10 +235,7 @@ SchedulingResult BundleSpreadSchedulingPolicy::Schedule( absl::flat_hash_map selected_nodes; for (const auto &resource_request : sorted_resource_request_list) { // Score and sort nodes. - auto best_node = GetBestNode(*resource_request, - candidate_nodes, - options, - available_cpus_before_bundle_scheduling); + auto best_node = GetBestNode(*resource_request, candidate_nodes, options); // There are nodes to meet the scheduling requirements. if (!best_node.first.IsNil()) { @@ -362,10 +246,7 @@ SchedulingResult BundleSpreadSchedulingPolicy::Schedule( selected_nodes.emplace(best_node); } else { // Scheduling from selected nodes. - best_node = GetBestNode(*resource_request, - selected_nodes, - options, - available_cpus_before_bundle_scheduling); + best_node = GetBestNode(*resource_request, selected_nodes, options); if (!best_node.first.IsNil()) { result_nodes.emplace_back(best_node.first); RAY_CHECK(cluster_resource_manager_.SubtractNodeAvailableResources( @@ -405,9 +286,6 @@ SchedulingResult BundleStrictPackSchedulingPolicy::Schedule( return SchedulingResult::Infeasible(); } - const auto available_cpus_before_bundle_scheduling = - GetAvailableCpusBeforeBundleScheduling(); - // Aggregate required resources. ResourceRequest aggregated_resource_request; for (const auto &resource_request : resource_request_list) { @@ -418,23 +296,13 @@ SchedulingResult BundleStrictPackSchedulingPolicy::Schedule( } } - const auto &right_node_it = std::find_if( - candidate_nodes.begin(), - candidate_nodes.end(), - [&aggregated_resource_request, &options, &available_cpus_before_bundle_scheduling]( - const auto &entry) { - const auto &node_resources = entry.second->GetLocalView(); - auto allocatable = - (node_resources.IsFeasible( - aggregated_resource_request) // If the resource is available - && !AllocationWillExceedMaxCpuFraction( // and allocating resources won't - // exceed max cpu fraction. - node_resources, - aggregated_resource_request, - options.max_cpu_fraction_per_node_, - available_cpus_before_bundle_scheduling.at(entry.first))); - return allocatable; - }); + const auto &right_node_it = + std::find_if(candidate_nodes.begin(), + candidate_nodes.end(), + [&aggregated_resource_request](const auto &entry) { + const auto &node_resources = entry.second->GetLocalView(); + return node_resources.IsFeasible(aggregated_resource_request); + }); if (right_node_it == candidate_nodes.end()) { RAY_LOG(DEBUG) << "The required resource is bigger than the maximum resource in the " @@ -451,16 +319,12 @@ SchedulingResult BundleStrictPackSchedulingPolicy::Schedule( absl::flat_hash_map{ {options.bundle_strict_pack_soft_target_node_id_, candidate_nodes[options.bundle_strict_pack_soft_target_node_id_]}}, - options, - available_cpus_before_bundle_scheduling); + options); } } if (best_node.first.IsNil()) { - best_node = GetBestNode(aggregated_resource_request, - candidate_nodes, - options, - available_cpus_before_bundle_scheduling); + best_node = GetBestNode(aggregated_resource_request, candidate_nodes, options); } // Select the node with the highest score. @@ -491,9 +355,6 @@ SchedulingResult BundleStrictSpreadSchedulingPolicy::Schedule( return SchedulingResult::Infeasible(); } - const auto available_cpus_before_bundle_scheduling = - GetAvailableCpusBeforeBundleScheduling(); - if (resource_request_list.size() > candidate_nodes.size()) { RAY_LOG(DEBUG) << "The number of required resources " << resource_request_list.size() << " is greater than the number of candidate nodes " @@ -510,10 +371,7 @@ SchedulingResult BundleStrictSpreadSchedulingPolicy::Schedule( std::vector result_nodes; for (const auto &resource_request : sorted_resource_request_list) { // Score and sort nodes. - auto best_node = GetBestNode(*resource_request, - candidate_nodes, - options, - available_cpus_before_bundle_scheduling); + auto best_node = GetBestNode(*resource_request, candidate_nodes, options); // There are nodes to meet the scheduling requirements. if (!best_node.first.IsNil()) { diff --git a/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.h b/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.h index 255a11957d70..fe82f9fd55e5 100644 --- a/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.h +++ b/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.h @@ -61,14 +61,7 @@ class BundleSchedulingPolicy : public IBundleSchedulingPolicy { std::pair GetBestNode( const ResourceRequest &required_resources, const absl::flat_hash_map &candidate_nodes, - const SchedulingOptions &options, - const absl::flat_hash_map - &available_cpus_before_bundle_scheduling) const; - - /// Return the map of node id -> available cpus before the current bundle scheduling. - /// It is used to calculate how many CPUs have been allocated for the current bundles. - const absl::flat_hash_map - GetAvailableCpusBeforeBundleScheduling() const; + const SchedulingOptions &options) const; protected: /// The cluster resource manager. diff --git a/src/ray/raylet/scheduling/policy/scheduling_options.h b/src/ray/raylet/scheduling/policy/scheduling_options.h index fe1d1f2bb8ad..b8f8804e3be2 100644 --- a/src/ray/raylet/scheduling/policy/scheduling_options.h +++ b/src/ray/raylet/scheduling/policy/scheduling_options.h @@ -68,7 +68,6 @@ struct SchedulingOptions { avoid_local_node, require_node_available, RayConfig::instance().scheduler_avoid_gpu_nodes(), - /*max_cpu_fraction_per_node*/ 1.0, /*scheduling_context*/ nullptr, preferred_node_id); } @@ -105,7 +104,6 @@ struct SchedulingOptions { /*avoid_local_node*/ false, /*require_node_available*/ true, /*avoid_gpu_nodes*/ RayConfig::instance().scheduler_avoid_gpu_nodes(), - /*max_cpu_fraction_per_node*/ 0, std::move(scheduling_context)); } @@ -119,7 +117,6 @@ struct SchedulingOptions { /*avoid_local_node*/ false, /*require_node_available*/ true, /*avoid_gpu_nodes*/ RayConfig::instance().scheduler_avoid_gpu_nodes(), - /*max_cpu_fraction_per_node*/ 0, std::move(scheduling_context)); } /* @@ -127,50 +124,44 @@ struct SchedulingOptions { */ // construct option for soft pack scheduling policy. - static SchedulingOptions BundlePack(double max_cpu_fraction_per_node = 1.0) { + static SchedulingOptions BundlePack() { return SchedulingOptions(SchedulingType::BUNDLE_PACK, /*spread_threshold*/ 0, /*avoid_local_node*/ false, /*require_node_available*/ true, - /*avoid_gpu_nodes*/ false, - /*max_cpu_fraction_per_node*/ max_cpu_fraction_per_node); + /*avoid_gpu_nodes*/ false); } // construct option for strict spread scheduling policy. - static SchedulingOptions BundleSpread(double max_cpu_fraction_per_node = 1.0) { + static SchedulingOptions BundleSpread() { return SchedulingOptions(SchedulingType::BUNDLE_SPREAD, /*spread_threshold*/ 0, /*avoid_local_node*/ false, /*require_node_available*/ true, - /*avoid_gpu_nodes*/ false, - /*max_cpu_fraction_per_node*/ max_cpu_fraction_per_node); + /*avoid_gpu_nodes*/ false); } // construct option for strict pack scheduling policy. static SchedulingOptions BundleStrictPack( - double max_cpu_fraction_per_node = 1.0, scheduling::NodeID soft_target_node_id = scheduling::NodeID::Nil()) { SchedulingOptions scheduling_options = SchedulingOptions(SchedulingType::BUNDLE_STRICT_PACK, /*spread_threshold*/ 0, /*avoid_local_node*/ false, /*require_node_available*/ true, - /*avoid_gpu_nodes*/ false, - /*max_cpu_fraction_per_node*/ max_cpu_fraction_per_node); + /*avoid_gpu_nodes*/ false); scheduling_options.bundle_strict_pack_soft_target_node_id_ = soft_target_node_id; return scheduling_options; } // construct option for strict spread scheduling policy. static SchedulingOptions BundleStrictSpread( - double max_cpu_fraction_per_node = 1.0, std::unique_ptr scheduling_context = nullptr) { return SchedulingOptions(SchedulingType::BUNDLE_STRICT_SPREAD, /*spread_threshold*/ 0, /*avoid_local_node*/ false, /*require_node_available*/ true, /*avoid_gpu_nodes*/ false, - /*max_cpu_fraction_per_node*/ max_cpu_fraction_per_node, /*scheduling_context*/ std::move(scheduling_context)); } @@ -179,12 +170,6 @@ struct SchedulingOptions { bool avoid_local_node_; bool require_node_available_; bool avoid_gpu_nodes_; - // Maximum reservable CPU fraction per node. It is applied across multiple - // bundles, individually. E.g., when you have 2 bundles {CPU: 4} from 2 different - // scheduilng request, and there's one node with {CPU: 8}, only 1 bundle from 1 request - // can be scheduled on this node. This is only used for bundle scheduling policies - // (bundle pack, spread). - double max_cpu_fraction_per_node_; // ID of the target node where bundles should be placed // iff the target node has enough available resources. // Otherwise, the bundles can be placed elsewhere. @@ -208,7 +193,6 @@ struct SchedulingOptions { bool avoid_local_node, bool require_node_available, bool avoid_gpu_nodes, - double max_cpu_fraction_per_node = 1.0, std::shared_ptr scheduling_context = nullptr, const std::string &preferred_node_id = std::string(), int32_t schedule_top_k_absolute = RayConfig::instance().scheduler_top_k_absolute(), @@ -218,7 +202,6 @@ struct SchedulingOptions { avoid_local_node_(avoid_local_node), require_node_available_(require_node_available), avoid_gpu_nodes_(avoid_gpu_nodes), - max_cpu_fraction_per_node_(max_cpu_fraction_per_node), scheduling_context_(std::move(scheduling_context)), preferred_node_id_(preferred_node_id), schedule_top_k_absolute_(schedule_top_k_absolute), diff --git a/src/ray/raylet/scheduling/policy/tests/hybrid_scheduling_policy_test.cc b/src/ray/raylet/scheduling/policy/tests/hybrid_scheduling_policy_test.cc index 786fc52aac61..f0a0042ae3ac 100644 --- a/src/ray/raylet/scheduling/policy/tests/hybrid_scheduling_policy_test.cc +++ b/src/ray/raylet/scheduling/policy/tests/hybrid_scheduling_policy_test.cc @@ -62,7 +62,6 @@ class HybridSchedulingPolicyTest : public ::testing::Test { avoid_local_node, require_node_available, avoid_gpu_nodes, - /*max_cpu_fraction_per_node*/ 1.0, /*scheduling_context*/ nullptr, /*preferred_node*/ "", schedule_top_k_absolute, diff --git a/src/ray/raylet/scheduling/policy/tests/scheduling_policy_test.cc b/src/ray/raylet/scheduling/policy/tests/scheduling_policy_test.cc index 4cce097edd8f..5d20b6e29c4c 100644 --- a/src/ray/raylet/scheduling/policy/tests/scheduling_policy_test.cc +++ b/src/ray/raylet/scheduling/policy/tests/scheduling_policy_test.cc @@ -59,7 +59,6 @@ class SchedulingPolicyTest : public ::testing::Test { avoid_local_node, require_node_available, avoid_gpu_nodes, - /*max_cpu_fraction_per_node*/ 1.0, /*scheduling_context*/ nullptr, /*preferred node*/ "", schedule_top_k_absolute, @@ -524,8 +523,7 @@ TEST_F(SchedulingPolicyTest, StrictPackBundleSchedulingTest) { req_list.push_back(&req); // No target node. - auto strict_pack_op = SchedulingOptions::BundleStrictPack( - /*max_cpu_fraction_per_node*/ 1.0, scheduling::NodeID::Nil()); + auto strict_pack_op = SchedulingOptions::BundleStrictPack(scheduling::NodeID::Nil()); auto to_schedule = raylet_scheduling_policy::BundleStrictPackSchedulingPolicy( *cluster_resource_manager, [](auto) { return true; }) .Schedule(req_list, strict_pack_op); @@ -533,8 +531,7 @@ TEST_F(SchedulingPolicyTest, StrictPackBundleSchedulingTest) { ASSERT_EQ(to_schedule.selected_nodes[0], local_node); // Target node has enough available resources. - strict_pack_op = SchedulingOptions::BundleStrictPack(/*max_cpu_fraction_per_node*/ 1.0, - remote_node_2); + strict_pack_op = SchedulingOptions::BundleStrictPack(remote_node_2); to_schedule = raylet_scheduling_policy::BundleStrictPackSchedulingPolicy( *cluster_resource_manager, [](auto) { return true; }) .Schedule(req_list, strict_pack_op); @@ -542,8 +539,7 @@ TEST_F(SchedulingPolicyTest, StrictPackBundleSchedulingTest) { ASSERT_EQ(to_schedule.selected_nodes[0], remote_node_2); // Target node doesn't have enough available resources. - strict_pack_op = - SchedulingOptions::BundleStrictPack(/*max_cpu_fraction_per_node*/ 1.0, remote_node); + strict_pack_op = SchedulingOptions::BundleStrictPack(remote_node); to_schedule = raylet_scheduling_policy::BundleStrictPackSchedulingPolicy( *cluster_resource_manager, [](auto) { return true; }) .Schedule(req_list, strict_pack_op); @@ -551,8 +547,7 @@ TEST_F(SchedulingPolicyTest, StrictPackBundleSchedulingTest) { ASSERT_EQ(to_schedule.selected_nodes[0], local_node); // Target node doesn't exist. - strict_pack_op = SchedulingOptions::BundleStrictPack(/*max_cpu_fraction_per_node*/ 1.0, - scheduling::NodeID(888)); + strict_pack_op = SchedulingOptions::BundleStrictPack(scheduling::NodeID(888)); to_schedule = raylet_scheduling_policy::BundleStrictPackSchedulingPolicy( *cluster_resource_manager, [](auto) { return true; }) .Schedule(req_list, strict_pack_op); @@ -560,125 +555,6 @@ TEST_F(SchedulingPolicyTest, StrictPackBundleSchedulingTest) { ASSERT_EQ(to_schedule.selected_nodes[0], local_node); } -TEST_F(SchedulingPolicyTest, BundleSchedulingMaxFractionTest) { - /* - * Test the bundle scheduling policy respects the max fraction request. - */ - - ResourceRequest req = ResourceMapToResourceRequest({{"CPU", 2}, {"GPU", 1}}, false); - std::vector req_list; - req_list.push_back(&req); - req_list.push_back(&req); - auto pack_op = SchedulingOptions::BundlePack(/*max_cpu_fraction_per_node*/ 0.5); - auto strict_pack_op = - SchedulingOptions::BundleStrictPack(/*max_cpu_fraction_per_node*/ 0.5); - auto spread_op = SchedulingOptions::BundleSpread(/*max_cpu_fraction_per_node*/ 0.5); - auto strict_spread_op = - SchedulingOptions::BundleStrictSpread(/*max_cpu_fraction_per_node*/ 0.5); - - nodes.emplace(local_node, CreateNodeResources(7, 7, 0, 0, 2, 2)); - - auto cluster_resource_manager = MockClusterResourceManager(nodes); - // req is unscheduleable because the max cpu fraction reaches 0.5. - auto unscheduable = raylet_scheduling_policy::BundlePackSchedulingPolicy( - *cluster_resource_manager, [](auto) { return true; }) - .Schedule(req_list, pack_op); - ASSERT_TRUE(unscheduable.status.IsFailed()); - - unscheduable = raylet_scheduling_policy::BundleSpreadSchedulingPolicy( - *cluster_resource_manager, [](auto) { return true; }) - .Schedule(req_list, spread_op); - ASSERT_TRUE(unscheduable.status.IsFailed()); - - unscheduable = raylet_scheduling_policy::BundleStrictPackSchedulingPolicy( - *cluster_resource_manager, [](auto) { return true; }) - .Schedule(req_list, strict_pack_op); - ASSERT_TRUE(unscheduable.status.IsInfeasible()); - - unscheduable = raylet_scheduling_policy::BundleStrictSpreadSchedulingPolicy( - *cluster_resource_manager, [](auto) { return true; }) - .Schedule(req_list, strict_spread_op); - ASSERT_TRUE(unscheduable.status.IsInfeasible()); -} - -TEST_F(SchedulingPolicyTest, BundleSchedulingMaxFractionOneCpuReservationGuaranteeTest) { - /* - * Test that when the max cpu fraction is provided, it reserves at least 1 CPU. - */ - - ResourceRequest req = ResourceMapToResourceRequest({{"CPU", 1}}, false); - std::vector req_list; - req_list.push_back(&req); - - // NOTE: We can only reserve up to 0.4 CPU, but it will round up to 1, - // which means the placement group is schedulable. - auto pack_op = SchedulingOptions::BundlePack(/*max_cpu_fraction_per_node*/ 0.1); - nodes.emplace(local_node, CreateNodeResources(4, 4, 0, 0, 0, 0)); - - auto cluster_resource_manager = MockClusterResourceManager(nodes); - // req is unscheduleable because the max cpu fraction reaches 0.5. - auto to_schedule = raylet_scheduling_policy::BundlePackSchedulingPolicy( - *cluster_resource_manager, [](auto) { return true; }) - .Schedule(req_list, pack_op); - ASSERT_TRUE(to_schedule.status.IsSuccess()); -} - -TEST_F(SchedulingPolicyTest, - BundleSchedulingMinFractionExcludeOneCpuReservationGuaranteeTest) { - /* - * Test that when the max cpu fraction is high, it excludes at least 1 CPU. - */ - - ResourceRequest req = ResourceMapToResourceRequest({{"CPU", 3}}, false); - std::vector req_list; - req_list.push_back(&req); - - // NOTE: We can reserve up to 3.96 CPU, but it will round down to 3 (exclude 1 CPU), - // which means a regular task with 1 CPU can be scheduled. - auto pack_op = SchedulingOptions::BundlePack(/*max_cpu_fraction_per_node*/ 0.99); - nodes.emplace(local_node, CreateNodeResources(4, 4, 0, 0, 0, 0)); - - auto cluster_resource_manager = MockClusterResourceManager(nodes); - // req is unscheduleable because the max cpu fraction reaches 0.5. - auto to_schedule = raylet_scheduling_policy::BundlePackSchedulingPolicy( - *cluster_resource_manager, [](auto) { return true; }) - .Schedule(req_list, pack_op); - ASSERT_TRUE(to_schedule.status.IsSuccess()); - - req = ResourceMapToResourceRequest({{"CPU", 1}}, false); - - auto to_schedule_task = - raylet_scheduling_policy::CompositeSchedulingPolicy( - local_node, *cluster_resource_manager, [](auto) { return true; }) - .Schedule(req, HybridOptions(0.50, false, false)); - ASSERT_TRUE(!to_schedule_task.IsNil()); -} - -TEST_F(SchedulingPolicyTest, BundleSchedulingMaxFractionWorkingWhenNormalResourceUsed) { - /* - * Test that it can schedule placement group correctly when there are non-pg - * resources occupying resources. - */ - - ResourceRequest req = ResourceMapToResourceRequest({{"CPU", 1}}, false); - std::vector req_list; - req_list.push_back(&req); - - // 2 CPUs / 4 CPUs is used by a regular task/actor. - // It means that when the fraction is 0.5, we still should - // be able to schedule a pg because 50% of CPUs still can be - // used for the placement group. - auto pack_op = SchedulingOptions::BundlePack(/*max_cpu_fraction_per_node*/ 0.5); - nodes.emplace(local_node, CreateNodeResources(2, 4, 0, 0, 0, 0)); - - auto cluster_resource_manager = MockClusterResourceManager(nodes); - // req is unscheduleable because the max cpu fraction reaches 0.5. - auto to_schedule = raylet_scheduling_policy::BundlePackSchedulingPolicy( - *cluster_resource_manager, [](auto) { return true; }) - .Schedule(req_list, pack_op); - ASSERT_TRUE(to_schedule.status.IsSuccess()); -} - int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_2_test.cc b/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_2_test.cc index f2a19f15474b..06db0f82085a 100644 --- a/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_2_test.cc +++ b/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_2_test.cc @@ -229,7 +229,6 @@ TEST_F(GcsResourceSchedulerTest, TestNodeFilter) { auto result1 = cluster_resource_scheduler_->Schedule( resource_request_list, SchedulingOptions::BundleStrictSpread( - /*max_cpu_fraction_per_node*/ 1.0, std::make_unique(bundle_locations))); ASSERT_TRUE(result1.status.IsInfeasible()); ASSERT_EQ(result1.selected_nodes.size(), 0); @@ -238,7 +237,6 @@ TEST_F(GcsResourceSchedulerTest, TestNodeFilter) { auto result2 = cluster_resource_scheduler_->Schedule( resource_request_list, SchedulingOptions::BundleStrictSpread( - /*max_cpu_fraction_per_node*/ 1.0, std::make_unique(nullptr))); ASSERT_TRUE(result2.status.IsSuccess()); ASSERT_EQ(result2.selected_nodes.size(), 1); From 183f9332bc8fd399ae5e02f82b36936e8e36b725 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Mon, 25 Aug 2025 13:48:51 -0700 Subject: [PATCH 258/634] [Core] Fix variable shadowing in local_object_manager_test.cc (#55888) otherwise ubsan test is not compiling. Signed-off-by: Jiajun Yao --- src/ray/raylet/tests/local_object_manager_test.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ray/raylet/tests/local_object_manager_test.cc b/src/ray/raylet/tests/local_object_manager_test.cc index 2a366322e74a..a5d7192e69b4 100644 --- a/src/ray/raylet/tests/local_object_manager_test.cc +++ b/src/ray/raylet/tests/local_object_manager_test.cc @@ -229,11 +229,12 @@ class MockIOWorker : public MockWorker { MockIOWorker(WorkerID worker_id, int port, std::shared_ptr io_worker) - : MockWorker(worker_id, port), io_worker(io_worker) {} + : MockWorker(worker_id, port), io_worker_(io_worker) {} - rpc::CoreWorkerClientInterface *rpc_client() { return io_worker.get(); } + rpc::CoreWorkerClientInterface *rpc_client() { return io_worker_.get(); } - std::shared_ptr io_worker; + private: + std::shared_ptr io_worker_; }; class MockIOWorkerPool : public IOWorkerPoolInterface { From 83d4464785f566b45b49c19b6051fd6c514f9145 Mon Sep 17 00:00:00 2001 From: Matthew Owen Date: Mon, 25 Aug 2025 14:18:24 -0700 Subject: [PATCH 259/634] [Data] Add large parquet release test (#55889) ## Why are these changes needed? Add another `read_parquet` release test on a dataset with larger parquet files. This dataset contains 103 files, 2.2GB each, ~56 row groups per file. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Matthew Owen --- release/release_data_tests.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/release/release_data_tests.yaml b/release/release_data_tests.yaml index 5a1bf0debacc..ce36281a8e50 100644 --- a/release/release_data_tests.yaml +++ b/release/release_data_tests.yaml @@ -35,6 +35,22 @@ s3://ray-benchmark-data-internal-us-west-2/imagenet/parquet --format parquet --iter-bundles +- name: "read_large_parquet_{{scaling}}" + + cluster: + cluster_compute: "{{scaling}}_cpu_compute.yaml" + + matrix: + setup: + scaling: [fixed_size, autoscaling] + + run: + timeout: 3600 + script: > + python read_and_consume_benchmark.py + s3://ray-benchmark-data-internal-us-west-2/large-parquet/ --format parquet + --iter-bundles + - name: "read_images_{{scaling}}" cluster: From d9598ba2f211f638a33f8f33b9b0dc96570d8b01 Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Tue, 26 Aug 2025 03:08:55 +0530 Subject: [PATCH 260/634] [core] Fix flaky `shutdown_coordinator_test` (#55893) Signed-off-by: Sagar Sumit --- src/ray/core_worker/tests/shutdown_coordinator_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ray/core_worker/tests/shutdown_coordinator_test.cc b/src/ray/core_worker/tests/shutdown_coordinator_test.cc index 4079fbbe8020..d20cda359eeb 100644 --- a/src/ray/core_worker/tests/shutdown_coordinator_test.cc +++ b/src/ray/core_worker/tests/shutdown_coordinator_test.cc @@ -398,9 +398,10 @@ TEST_F(ShutdownCoordinatorTest, Concurrent_DoubleForce_ForceExecutesOnce) { EXPECT_EQ(coordinator->GetState(), ShutdownState::kShutdown); EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kForcedExit); + // Verify that only one forced shutdown was called EXPECT_EQ(fake_ptr->force_calls.load(), 1); EXPECT_EQ(fake_ptr->graceful_calls.load(), 0); - EXPECT_EQ(fake_ptr->last_detail, "force1"); + EXPECT_TRUE(fake_ptr->last_detail == "force1" || fake_ptr->last_detail == "force2"); } } // namespace core From db9b20d3ec920d7681d8b34ae53ba5caa777df81 Mon Sep 17 00:00:00 2001 From: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Date: Mon, 25 Aug 2025 15:37:45 -0700 Subject: [PATCH 261/634] [Data] Add time to first batch metric for dataset iterators (#55758) The time to first batch usually takes longer time than the subsequent batches. This is because the time to first batch includes the time needed for the pipeline to warm up. The iterator receives the batch once the first few blocks have made it through all stages of the data pipeline and piped to the train worker consumers. Since we do prefetching and the data pipeline is in a steady state, so the time to produce subsequent batches is much lower. In this PR, we added a metric to track the time to first batch. --------- Signed-off-by: xgui Signed-off-by: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Justin Yu --- .../_internal/block_batching/iter_batches.py | 21 ++++++++++++++++--- python/ray/data/_internal/stats.py | 17 +++++++++++++++ python/ray/data/tests/test_stats.py | 4 ++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/python/ray/data/_internal/block_batching/iter_batches.py b/python/ray/data/_internal/block_batching/iter_batches.py index f807ae2078dc..9dc052d12aaa 100644 --- a/python/ray/data/_internal/block_batching/iter_batches.py +++ b/python/ray/data/_internal/block_batching/iter_batches.py @@ -135,6 +135,7 @@ def __init__( if actor_prefetcher_enabled else WaitBlockPrefetcher() ) + self._yielded_first_batch = False def _prefetch_blocks( self, ref_bundles: Iterator[RefBundle] @@ -235,15 +236,29 @@ def __iter__(self) -> Iterator[DataBatch]: return self._iter_batches() def before_epoch_start(self): - pass + self._yielded_first_batch = False def after_epoch_end(self): StatsManager.clear_iteration_metrics(self._dataset_tag) @contextmanager def get_next_batch_context(self): - with self._stats.iter_total_blocked_s.timer() if self._stats else nullcontext(): - yield + try: + if self._stats: + # Always track total blocked time + total_timer = self._stats.iter_total_blocked_s.timer() + # Also track the time until the first batch is ready + first_batch_ready_timer = ( + self._stats.iter_time_to_first_batch_s.timer() + if not self._yielded_first_batch + else nullcontext() + ) + with total_timer, first_batch_ready_timer: + yield + else: + yield + finally: + self._yielded_first_batch = True @contextmanager def yield_batch_context(self, batch: Batch): diff --git a/python/ray/data/_internal/stats.py b/python/ray/data/_internal/stats.py index d00b45c89b8a..7f4222f68426 100644 --- a/python/ray/data/_internal/stats.py +++ b/python/ray/data/_internal/stats.py @@ -280,6 +280,12 @@ def __init__(self, max_stats=1000): description="Seconds user thread is blocked by iter_batches()", tag_keys=iter_tag_keys, ) + self.time_to_first_batch_s = Gauge( + "data_iter_time_to_first_batch_seconds", + description="Total time spent waiting for the first batch after starting iteration. " + "This includes the dataset pipeline warmup time. This metric is accumulated across different epochs.", + tag_keys=iter_tag_keys, + ) self.iter_user_s = Gauge( "data_iter_user_seconds", description="Seconds spent in user code", @@ -469,6 +475,7 @@ def update_iteration_metrics( ): tags = self._create_tags(dataset_tag) self.iter_total_blocked_s.set(stats.iter_total_blocked_s.get(), tags) + self.time_to_first_batch_s.set(stats.iter_time_to_first_batch_s.get(), tags) self.iter_user_s.set(stats.iter_user_s.get(), tags) self.iter_initialize_s.set(stats.iter_initialize_s.get(), tags) @@ -948,6 +955,7 @@ def __init__( self.iter_format_batch_s: Timer = Timer() self.iter_collate_batch_s: Timer = Timer() self.iter_finalize_batch_s: Timer = Timer() + self.iter_time_to_first_batch_s: Timer = Timer() self.iter_total_blocked_s: Timer = Timer() self.iter_user_s: Timer = Timer() self.iter_initialize_s: Timer = Timer() @@ -1003,6 +1011,7 @@ def to_summary(self) -> "DatasetStatsSummary": self.iter_format_batch_s, self.iter_collate_batch_s, self.iter_finalize_batch_s, + self.iter_time_to_first_batch_s, self.iter_total_blocked_s, self.iter_user_s, self.iter_initialize_s, @@ -1642,6 +1651,8 @@ class IterStatsSummary: collate_time: Timer # Time spent in finalize_fn, in seconds finalize_batch_time: Timer + # Time user thread is blocked waiting for first batch + time_to_first_batch: Timer # Total time user thread is blocked by iter_batches block_time: Timer # Time spent in user code, in seconds @@ -1665,6 +1676,7 @@ def to_string(self) -> str: out = "" if ( self.block_time.get() + or self.time_to_first_batch.get() or self.total_time.get() or self.get_time.get() or self.next_time.get() @@ -1685,6 +1697,11 @@ def to_string(self) -> str: " * Total time user thread is blocked by Ray Data iter_batches: " "{}\n".format(fmt(self.block_time.get())) ) + if self.time_to_first_batch.get(): + out += ( + " * Total time spent waiting for the first batch after starting iteration: " + "{}\n".format(fmt(self.time_to_first_batch.get())) + ) if self.user_time.get(): out += " * Total execution time for user thread: {}\n".format( fmt(self.user_time.get()) diff --git a/python/ray/data/tests/test_stats.py b/python/ray/data/tests/test_stats.py index 0a3a32e9d63e..cbe5b5dfa8ba 100644 --- a/python/ray/data/tests/test_stats.py +++ b/python/ray/data/tests/test_stats.py @@ -395,6 +395,7 @@ def test_streaming_split_stats(ray_start_regular_shared, restore_data_context): * Total time overall: T * Total time in Ray Data iterator initialization code: T * Total time user thread is blocked by Ray Data iter_batches: T + * Total time spent waiting for the first batch after starting iteration: T * Total execution time for user thread: T * Batch iteration time breakdown (summed across prefetch threads): * In ray.get(): T min, T max, T avg, T total @@ -577,6 +578,7 @@ def test_dataset_stats_basic( f"* Total time overall: T\n" f" * Total time in Ray Data iterator initialization code: T\n" f" * Total time user thread is blocked by Ray Data iter_batches: T\n" + f" * Total time spent waiting for the first batch after starting iteration: T\n" f" * Total execution time for user thread: T\n" f"* Batch iteration time breakdown (summed across prefetch threads):\n" f" * In ray.get(): T min, T max, T avg, T total\n" @@ -618,6 +620,7 @@ def test_block_location_nums(ray_start_regular_shared, restore_data_context): f"* Total time overall: T\n" f" * Total time in Ray Data iterator initialization code: T\n" f" * Total time user thread is blocked by Ray Data iter_batches: T\n" + f" * Total time spent waiting for the first batch after starting iteration: T\n" f" * Total execution time for user thread: T\n" f"* Batch iteration time breakdown (summed across prefetch threads):\n" f" * In ray.get(): T min, T max, T avg, T total\n" @@ -1363,6 +1366,7 @@ def test_streaming_stats_full(ray_start_regular_shared, restore_data_context): * Total time overall: T * Total time in Ray Data iterator initialization code: T * Total time user thread is blocked by Ray Data iter_batches: T + * Total time spent waiting for the first batch after starting iteration: T * Total execution time for user thread: T * Batch iteration time breakdown (summed across prefetch threads): * In ray.get(): T min, T max, T avg, T total From 7f19c11447f6731eae5299cc07c4dc5e8a4fa712 Mon Sep 17 00:00:00 2001 From: Kai-Hsun Chen Date: Mon, 25 Aug 2025 15:51:41 -0700 Subject: [PATCH 262/634] [core][gpu-objects] tensor shape/dtype is not necessary for send_multiple_tensors (#55870) Signed-off-by: Kai-Hsun Chen --- .../experimental/collective/collective_tensor_transport.py | 1 - python/ray/experimental/collective/nixl_tensor_transport.py | 1 - .../ray/experimental/collective/tensor_transport_manager.py | 5 ----- .../experimental/gpu_object_manager/gpu_object_manager.py | 1 - .../ray/experimental/gpu_object_manager/gpu_object_store.py | 2 -- 5 files changed, 10 deletions(-) diff --git a/python/ray/experimental/collective/collective_tensor_transport.py b/python/ray/experimental/collective/collective_tensor_transport.py index d4dece6f1808..64bc0991db26 100644 --- a/python/ray/experimental/collective/collective_tensor_transport.py +++ b/python/ray/experimental/collective/collective_tensor_transport.py @@ -129,7 +129,6 @@ def recv_multiple_tensors( @staticmethod def send_multiple_tensors( tensors: List["torch.Tensor"], - tensor_transport_metadata: CollectiveTransportMetadata, communicator_metadata: CollectiveCommunicatorMetadata, device: "torch.device", ): diff --git a/python/ray/experimental/collective/nixl_tensor_transport.py b/python/ray/experimental/collective/nixl_tensor_transport.py index 5723e7b1d686..eb86f0cb9a3d 100644 --- a/python/ray/experimental/collective/nixl_tensor_transport.py +++ b/python/ray/experimental/collective/nixl_tensor_transport.py @@ -109,7 +109,6 @@ def recv_multiple_tensors( @staticmethod def send_multiple_tensors( tensors: List["torch.Tensor"], - tensor_transport_metadata: NixlTransportMetadata, communicator_metadata: NixlCommunicatorMetadata, device: "torch.device", ): diff --git a/python/ray/experimental/collective/tensor_transport_manager.py b/python/ray/experimental/collective/tensor_transport_manager.py index 302b6998b699..9f3896699393 100644 --- a/python/ray/experimental/collective/tensor_transport_manager.py +++ b/python/ray/experimental/collective/tensor_transport_manager.py @@ -64,7 +64,6 @@ def get_communicator_metadata( def send_object( src_actor: "ray.actor.ActorHandle", obj_id: str, - tensor_transport_metadata_ref: TensorTransportMetadata, communicator_metadata_ref: CommunicatorMetadata, ): """ @@ -73,7 +72,6 @@ def send_object( Args: src_actor: The actor that runs this function. obj_id: The ID of the GPU object to send. - tensor_transport_metadata_ref: The ObjectRef of tensor transport metadata for the GPU object. communicator_metadata_ref: The ObjectRef of communicator metadata for the send/recv operation. """ from ray.experimental.gpu_object_manager.gpu_object_store import __ray_send__ @@ -85,7 +83,6 @@ def send_object( src_actor.__ray_call__.options(concurrency_group="_ray_system").remote( __ray_send__, obj_id, - tensor_transport_metadata_ref, communicator_metadata_ref, ) @@ -145,7 +142,6 @@ def recv_multiple_tensors( @abstractmethod def send_multiple_tensors( tensors: List["torch.Tensor"], - tensor_transport_metadata: TensorTransportMetadata, communicator_metadata: CommunicatorMetadata, device: "torch.device", ): @@ -154,7 +150,6 @@ def send_multiple_tensors( Args: tensors: The tensors to send. - tensor_transport_metadata: The tensor transport metadata for the GPU object. communicator_metadata: The communicator metadata for the send/recv operation. device: The device to send the tensors to. """ diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py index 7d7f031269e6..9102edb86566 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py @@ -222,7 +222,6 @@ def trigger_out_of_band_tensor_transfer( tensor_transport_manager.send_object( src_actor, obj_id, - tensor_transport_meta, communicator_meta, ) tensor_transport_manager.recv_object( diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_store.py b/python/ray/experimental/gpu_object_manager/gpu_object_store.py index 6b39a5856554..92e706f50af0 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_store.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_store.py @@ -48,7 +48,6 @@ def _tensor_transport_to_collective_backend( def __ray_send__( self, obj_id: str, - tensor_transport_meta: TensorTransportMetadata, communicator_meta: CommunicatorMetadata, ): """Helper function that runs on the src actor to send tensors to the dst actor.""" @@ -69,7 +68,6 @@ def __ray_send__( tensor_transport_manager = get_tensor_transport_manager(backend) tensor_transport_manager.send_multiple_tensors( tensors, - tensor_transport_meta, communicator_meta, device=device, ) From b32f2949033fb7717658ae47f382f62f3746031b Mon Sep 17 00:00:00 2001 From: matthewdeng Date: Mon, 25 Aug 2025 15:53:14 -0700 Subject: [PATCH 263/634] [train] add usage tag key for JaxTrainer (#55887) Signed-off-by: Matthew Deng --- python/ray/air/_internal/usage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ray/air/_internal/usage.py b/python/ray/air/_internal/usage.py index 9e921d40f803..4933a7517631 100644 --- a/python/ray/air/_internal/usage.py +++ b/python/ray/air/_internal/usage.py @@ -24,6 +24,7 @@ TRAIN_V2_TRAINERS = { "DataParallelTrainer", + "JaxTrainer", "LightGBMTrainer", "TensorflowTrainer", "TorchTrainer", From b76addb37f98beddb39a05170874c95e82874d62 Mon Sep 17 00:00:00 2001 From: goutamvenkat-anyscale Date: Mon, 25 Aug 2025 17:23:23 -0700 Subject: [PATCH 264/634] [Data] - Hash Partitioning test, make assert less sensitive to exact number of partitions (#55905) ## Why are these changes needed? Make `test_hash_partitioning` assert statement less sensitive to exact number of partitions ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Goutam V --- python/ray/data/tests/test_transform_pyarrow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/data/tests/test_transform_pyarrow.py b/python/ray/data/tests/test_transform_pyarrow.py index f2a58f1c6bcc..2d0ba5f2f75c 100644 --- a/python/ray/data/tests/test_transform_pyarrow.py +++ b/python/ray/data/tests/test_transform_pyarrow.py @@ -101,7 +101,7 @@ def _concat_and_sort_partitions(parts: Iterable[pa.Table]) -> pa.Table: t, hash_cols=["structs"], num_partitions=101 ) - assert len(_structs_partition_dict) == 34 + assert len(_structs_partition_dict) <= 101 assert t == _concat_and_sort_partitions(_structs_partition_dict.values()) From 490f9283a7430c1863eb9e00172d2501e3e37b20 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 25 Aug 2025 20:26:33 -0500 Subject: [PATCH 265/634] [core] Remove `gcs_rpc_server.h` dependency from `GcsNodeManager` (#55886) Drops compilation time of `gcs_node_manager` target from ~24s to ~14s (measured locally on a change to `gcs_node_manager.cc`). --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 36 ++++++++- src/ray/gcs/gcs_server/gcs_node_manager.h | 4 +- src/ray/gcs/gcs_server/gcs_server.cc | 5 +- .../gcs/gcs_server/grpc_service_interfaces.h | 68 +++++++++++++++++ src/ray/gcs/gcs_server/grpc_services.cc | 40 ++++++++++ src/ray/gcs/gcs_server/grpc_services.h | 62 ++++++++++++++++ src/ray/rpc/gcs/gcs_rpc_server.h | 73 ------------------- 7 files changed, 211 insertions(+), 77 deletions(-) create mode 100644 src/ray/gcs/gcs_server/grpc_service_interfaces.h create mode 100644 src/ray/gcs/gcs_server/grpc_services.cc create mode 100644 src/ray/gcs/gcs_server/grpc_services.h diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 74cdbd2067c7..7bdcecd2c0c6 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -74,13 +74,13 @@ ray_cc_library( deps = [ ":gcs_init_data", ":gcs_table_storage", + ":grpc_service_interfaces", "//src/ray/common:asio", "//src/ray/common:id", "//src/ray/common:ray_config", "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/protobuf:ray_syncer_cc_proto", - "//src/ray/rpc:gcs_server", "//src/ray/rpc:node_manager_client", "//src/ray/util:event", "//src/ray/util:logging", @@ -219,6 +219,38 @@ ray_cc_library( ], ) +ray_cc_library( + name = "grpc_service_interfaces", + hdrs = [ + "grpc_service_interfaces.h", + ], + visibility = ["//visibility:private"], + deps = [ + "//src/ray/common:status", + "//src/ray/protobuf:gcs_service_cc_grpc", + ], +) + +ray_cc_library( + name = "grpc_services", + srcs = [ + "grpc_services.cc", + ], + hdrs = [ + "grpc_services.h", + ], + visibility = ["//visibility:private"], + deps = [ + ":grpc_service_interfaces", + "//src/ray/common:asio", + "//src/ray/common:id", + "//src/ray/protobuf:gcs_service_cc_grpc", + "//src/ray/rpc:grpc_server", + "//src/ray/rpc:server_call", + "@com_github_grpc_grpc//:grpc++", + ], +) + ray_cc_library( name = "gcs_server_lib", srcs = [ @@ -255,6 +287,8 @@ ray_cc_library( ":gcs_task_manager", ":gcs_usage_stats_client", ":gcs_worker_manager", + ":grpc_service_interfaces", + ":grpc_services", "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/gcs/store_client", "//src/ray/gcs/store_client:in_memory_store_client", diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.h b/src/ray/gcs/gcs_server/gcs_node_manager.h index 02c3b6723d55..d54536c4ec49 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/ray/gcs/gcs_server/gcs_node_manager.h @@ -24,8 +24,8 @@ #include "ray/common/id.h" #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" #include "ray/rpc/node_manager/raylet_client_pool.h" #include "ray/util/event.h" #include "src/ray/protobuf/gcs.pb.h" @@ -39,7 +39,7 @@ class GcsStateTest; /// GcsNodeManager is responsible for managing and monitoring nodes as well as handing /// node and resource related rpc requests. /// This class is not thread-safe. -class GcsNodeManager : public rpc::NodeInfoHandler { +class GcsNodeManager : public rpc::NodeInfoGcsServiceHandler { public: /// Create a GcsNodeManager. /// diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 503414c9685a..9e15a8f79429 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -29,6 +29,7 @@ #include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/gcs_worker_manager.h" +#include "ray/gcs/gcs_server/grpc_services.h" #include "ray/gcs/gcs_server/store_client_kv.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/store_client/observable_store_client.h" @@ -356,7 +357,9 @@ void GcsServer::InitGcsNodeManager(const GcsInitData &gcs_init_data) { // Initialize by gcs tables data. gcs_node_manager_->Initialize(gcs_init_data); rpc_server_.RegisterService(std::make_unique( - io_context_provider_.GetDefaultIOContext(), *gcs_node_manager_)); + io_context_provider_.GetDefaultIOContext(), + *gcs_node_manager_, + RayConfig::instance().gcs_max_active_rpcs_per_handler())); } void GcsServer::InitGcsHealthCheckManager(const GcsInitData &gcs_init_data) { diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h new file mode 100644 index 000000000000..0deb908cae5d --- /dev/null +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -0,0 +1,68 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + * This file defines the gRPC service *INTERFACES* only. + * The subcomponent that handles a given interface should inherit from the relevant + * class. The target for the subcomponent should depend only on this file, not on + * grpc_services.h. + */ + +#pragma once + +#include "ray/common/status.h" +#include "src/ray/protobuf/gcs_service.grpc.pb.h" + +namespace ray { +namespace rpc { + +using SendReplyCallback = std::function success, std::function failure)>; + +#define GCS_RPC_SEND_REPLY(send_reply_callback, reply, status) \ + reply->mutable_status()->set_code(static_cast(status.code())); \ + reply->mutable_status()->set_message(status.message()); \ + send_reply_callback(ray::Status::OK(), nullptr, nullptr) + +class NodeInfoGcsServiceHandler { + public: + virtual ~NodeInfoGcsServiceHandler() = default; + + virtual void HandleGetClusterId(GetClusterIdRequest request, + GetClusterIdReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleRegisterNode(RegisterNodeRequest request, + RegisterNodeReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleUnregisterNode(UnregisterNodeRequest request, + UnregisterNodeReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleCheckAlive(CheckAliveRequest request, + CheckAliveReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleDrainNode(DrainNodeRequest request, + DrainNodeReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetAllNodeInfo(GetAllNodeInfoRequest request, + GetAllNodeInfoReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + +} // namespace rpc +} // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc new file mode 100644 index 000000000000..098299683462 --- /dev/null +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -0,0 +1,40 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "ray/gcs/gcs_server/grpc_services.h" + +#include +#include + +namespace ray { +namespace rpc { + +void NodeInfoGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + // We only allow one cluster ID in the lifetime of a client. + // So, if a client connects, it should not have a pre-existing different ID. + RPC_SERVICE_HANDLER_CUSTOM_AUTH(NodeInfoGcsService, + GetClusterId, + max_active_rpcs_per_handler_, + AuthType::EMPTY_AUTH); + RPC_SERVICE_HANDLER(NodeInfoGcsService, RegisterNode, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(NodeInfoGcsService, UnregisterNode, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(NodeInfoGcsService, DrainNode, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(NodeInfoGcsService, GetAllNodeInfo, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(NodeInfoGcsService, CheckAlive, max_active_rpcs_per_handler_) +} + +} // namespace rpc +} // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/gcs_server/grpc_services.h new file mode 100644 index 000000000000..e4785f1dbe65 --- /dev/null +++ b/src/ray/gcs/gcs_server/grpc_services.h @@ -0,0 +1,62 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + * This file defines the gRPC service handlers for the GCS server binary. + * Subcomponents that implement a given interface should inherit from the relevant + * class in grpc_service_interfaces.h. + * + * The GCS server main binary should be the only user of this target. + */ + +#pragma once + +#include +#include + +#include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/id.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" +#include "ray/rpc/grpc_server.h" +#include "ray/rpc/server_call.h" +#include "src/ray/protobuf/gcs_service.grpc.pb.h" + +namespace ray { +namespace rpc { + +class NodeInfoGrpcService : public GrpcService { + public: + explicit NodeInfoGrpcService(instrumented_io_context &io_service, + NodeInfoGcsServiceHandler &service_handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(service_handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler){}; + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + NodeInfoGcsService::AsyncService service_; + NodeInfoGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + +} // namespace rpc +} // namespace ray diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index e91b8161a8af..6e5835ae032f 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -122,11 +122,6 @@ namespace rpc { HANDLER, \ RayConfig::instance().gcs_max_active_rpcs_per_handler()) -#define NODE_INFO_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(NodeInfoGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - #define TASK_INFO_SERVICE_RPC_HANDLER(HANDLER) \ RPC_SERVICE_HANDLER(TaskInfoGcsService, \ HANDLER, \ @@ -319,73 +314,6 @@ class ActorInfoGrpcService : public GrpcService { ActorInfoGcsServiceHandler &service_handler_; }; -class NodeInfoGcsServiceHandler { - public: - virtual ~NodeInfoGcsServiceHandler() = default; - - virtual void HandleGetClusterId(rpc::GetClusterIdRequest request, - rpc::GetClusterIdReply *reply, - rpc::SendReplyCallback send_reply_callback) = 0; - - virtual void HandleRegisterNode(RegisterNodeRequest request, - RegisterNodeReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleUnregisterNode(UnregisterNodeRequest request, - UnregisterNodeReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleCheckAlive(CheckAliveRequest request, - CheckAliveReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleDrainNode(DrainNodeRequest request, - DrainNodeReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetAllNodeInfo(GetAllNodeInfoRequest request, - GetAllNodeInfoReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -/// The `GrpcService` for `NodeInfoGcsService`. -class NodeInfoGrpcService : public GrpcService { - public: - /// Constructor. - /// - /// \param[in] handler The service handler that actually handle the requests. - explicit NodeInfoGrpcService(instrumented_io_context &io_service, - NodeInfoGcsServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler){}; - - protected: - grpc::Service &GetGrpcService() override { return service_; } - - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - // We only allow one cluster ID in the lifetime of a client. - // So, if a client connects, it should not have a pre-existing different ID. - RPC_SERVICE_HANDLER_CUSTOM_AUTH( - NodeInfoGcsService, - GetClusterId, - RayConfig::instance().gcs_max_active_rpcs_per_handler(), - AuthType::EMPTY_AUTH); - NODE_INFO_SERVICE_RPC_HANDLER(RegisterNode); - NODE_INFO_SERVICE_RPC_HANDLER(UnregisterNode); - NODE_INFO_SERVICE_RPC_HANDLER(DrainNode); - NODE_INFO_SERVICE_RPC_HANDLER(GetAllNodeInfo); - NODE_INFO_SERVICE_RPC_HANDLER(CheckAlive); - } - - private: - /// The grpc async service object. - NodeInfoGcsService::AsyncService service_; - /// The service handler that actually handle the requests. - NodeInfoGcsServiceHandler &service_handler_; -}; - class NodeResourceInfoGcsServiceHandler { public: virtual ~NodeResourceInfoGcsServiceHandler() = default; @@ -767,7 +695,6 @@ class InternalPubSubGrpcService : public GrpcService { using JobInfoHandler = JobInfoGcsServiceHandler; using ActorInfoHandler = ActorInfoGcsServiceHandler; -using NodeInfoHandler = NodeInfoGcsServiceHandler; using NodeResourceInfoHandler = NodeResourceInfoGcsServiceHandler; using WorkerInfoHandler = WorkerInfoGcsServiceHandler; using PlacementGroupInfoHandler = PlacementGroupInfoGcsServiceHandler; From 6dae9f3c93234b566c1183f0c2b5e7564eddfd00 Mon Sep 17 00:00:00 2001 From: vickytsang Date: Mon, 25 Aug 2025 18:52:26 -0700 Subject: [PATCH 266/634] [Core][AMD] Add AMD Instinct MI350 and MI355 products (#55853) Signed-off-by: root --- python/ray/_private/accelerators/amd_gpu.py | 2 ++ python/ray/util/accelerators/accelerators.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/python/ray/_private/accelerators/amd_gpu.py b/python/ray/_private/accelerators/amd_gpu.py index ec870193c8c6..0e820b28a666 100644 --- a/python/ray/_private/accelerators/amd_gpu.py +++ b/python/ray/_private/accelerators/amd_gpu.py @@ -21,6 +21,8 @@ "0x74a2": "AMD-Instinct-MI308X-OAM", "0x74a9": "AMD-Instinct-MI300X-HF", "0x74a5": "AMD-Instinct-MI325X-OAM", + "0x75a0": "AMD-Instinct-MI350X-OAM", + "0x75a3": "AMD-Instinct-MI355X-OAM", "0x6798": "AMD-Radeon-R9-200-HD-7900", "0x6799": "AMD-Radeon-HD-7900", "0x679A": "AMD-Radeon-HD-7900", diff --git a/python/ray/util/accelerators/accelerators.py b/python/ray/util/accelerators/accelerators.py index aaa5b8f86f81..b68d0460b538 100644 --- a/python/ray/util/accelerators/accelerators.py +++ b/python/ray/util/accelerators/accelerators.py @@ -23,6 +23,8 @@ AMD_INSTINCT_MI300x_HF = "AMD-Instinct-MI300X-HF" AMD_INSTINCT_MI308x = "AMD-Instinct-MI308X" AMD_INSTINCT_MI325x = "AMD-Instinct-MI325X-OAM" +AMD_INSTINCT_MI350x = "AMD-Instinct-MI350X-OAM" +AMD_INSTINCT_MI355x = "AMD-Instinct-MI355X-OAM" AMD_RADEON_R9_200_HD_7900 = "AMD-Radeon-R9-200-HD-7900" AMD_RADEON_HD_7900 = "AMD-Radeon-HD-7900" AWS_NEURON_CORE = "aws-neuron-core" From 8b2c8d42028637ffa8f6ce8f716617ce70b07187 Mon Sep 17 00:00:00 2001 From: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Date: Mon, 25 Aug 2025 20:11:16 -0700 Subject: [PATCH 267/634] [Serve.llm] Add start/stop_profile method to LLMServer (#55920) Signed-off-by: Rui Qiao --- .../serve/deployments/llm/llm_server.py | 30 ++++++++++++ .../serve/deployments/llm/vllm/vllm_engine.py | 8 ++++ .../cpu/deployments/llm/test_llm_server.py | 48 ++++++++++++++++++- .../llm/tests/serve/mocks/mock_vllm_engine.py | 10 ++++ 4 files changed, 95 insertions(+), 1 deletion(-) diff --git a/python/ray/llm/_internal/serve/deployments/llm/llm_server.py b/python/ray/llm/_internal/serve/deployments/llm/llm_server.py index f7f4d274e611..72a02525286e 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/llm_server.py +++ b/python/ray/llm/_internal/serve/deployments/llm/llm_server.py @@ -108,6 +108,14 @@ async def check_health(self) -> None: async def reset_prefix_cache(self) -> None: """Reset the prefix cache of the underlying engine""" + @abstractmethod + async def start_profile(self) -> None: + """Start profiling""" + + @abstractmethod + async def stop_profile(self) -> None: + """Stop profiling""" + # TODO (Kourosh): This does not belong here. async def llm_config(self) -> Optional[LLMConfig]: return None @@ -410,6 +418,28 @@ async def reset_prefix_cache(self) -> None: ) raise e + async def start_profile(self) -> None: + """Start profiling""" + if self.engine is None: + return + try: + await self.engine.start_profile() + except Exception as e: + logger.error( + "Engine start profile failed in LLMServer.start_profile: %s", e + ) + raise e + + async def stop_profile(self) -> None: + """Stop profiling""" + if self.engine is None: + return + try: + await self.engine.stop_profile() + except Exception as e: + logger.error("Engine stop profile failed in LLMServer.stop_profile: %s", e) + raise e + async def llm_config(self) -> Optional[LLMConfig]: return self._llm_config diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py index 28ad58b09bb5..9e67fdbe30dd 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py @@ -454,3 +454,11 @@ async def check_health(self) -> None: async def reset_prefix_cache(self) -> None: assert self._engine_client is not None, "engine_client is not initialized" await self._engine_client.reset_prefix_cache() + + async def start_profile(self) -> None: + assert self._engine_client is not None, "engine_client is not initialized" + await self._engine_client.start_profile() + + async def stop_profile(self) -> None: + assert self._engine_client is not None, "engine_client is not initialized" + await self._engine_client.stop_profile() diff --git a/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py b/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py index 6b7627980490..20c517bd23b6 100644 --- a/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py +++ b/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py @@ -192,12 +192,58 @@ async def reset_prefix_cache(self): server = LLMServer.sync_init(mock_llm_config, engine_cls=LocalMockEngine) await server.start() - # Perform the health check, no exceptions should be raised + # Reset prefix cache, no exceptions should be raised await server.reset_prefix_cache() # Check that the reset prefix cache method was called assert server.engine.reset_prefix_cache_called + @pytest.mark.asyncio + async def test_start_profile(self, mock_llm_config): + """Test start profile functionality.""" + + # Mock the engine's start_profile method + class LocalMockEngine(MockVLLMEngine): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.start_profile_called = False + + async def start_profile(self): + self.start_profile_called = True + + # Create a server with a mocked engine + server = LLMServer.sync_init(mock_llm_config, engine_cls=LocalMockEngine) + await server.start() + + # Start profile, no exceptions should be raised + await server.start_profile() + + # Check that the start profile method was called + assert server.engine.start_profile_called + + @pytest.mark.asyncio + async def test_stop_profile(self, mock_llm_config): + """Test stop profile functionality.""" + + # Mock the engine's stop_profile method + class LocalMockEngine(MockVLLMEngine): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.stop_profile_called = False + + async def stop_profile(self): + self.stop_profile_called = True + + # Create a server with a mocked engine + server = LLMServer.sync_init(mock_llm_config, engine_cls=LocalMockEngine) + await server.start() + + # Stop profile, no exceptions should be raised + await server.stop_profile() + + # Check that the stop profile method was called + assert server.engine.stop_profile_called + @pytest.mark.asyncio async def test_llm_config_property(self, mock_llm_config): """Test the llm_config property.""" diff --git a/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py b/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py index 5fd900d1c6fb..6879d7db1272 100644 --- a/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py +++ b/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py @@ -56,6 +56,16 @@ async def reset_prefix_cache(self) -> None: if not self.started: raise RuntimeError("Engine not started") + async def start_profile(self) -> None: + """Start profiling of the mock engine.""" + if not self.started: + raise RuntimeError("Engine not started") + + async def stop_profile(self) -> None: + """Stop profiling of the mock engine.""" + if not self.started: + raise RuntimeError("Engine not started") + async def chat( self, request: ChatCompletionRequest ) -> AsyncGenerator[Union[str, ChatCompletionResponse, ErrorResponse], None]: From ec5d4108737ead7dccb2930439842e0b2de012bb Mon Sep 17 00:00:00 2001 From: Rueian Date: Mon, 25 Aug 2025 21:56:53 -0700 Subject: [PATCH 268/634] [core] avoid copying node labels when updating them from sync messages (#55909) Signed-off-by: Rueian --- src/ray/gcs/gcs_server/gcs_resource_manager.cc | 2 +- src/ray/raylet/node_manager.cc | 2 +- src/ray/raylet/scheduling/cluster_resource_manager.cc | 4 ++-- src/ray/raylet/scheduling/cluster_resource_manager.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ray/gcs/gcs_server/gcs_resource_manager.cc b/src/ray/gcs/gcs_server/gcs_resource_manager.cc index 088c32cc2b15..373ec4c5dae5 100644 --- a/src/ray/gcs/gcs_server/gcs_resource_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_resource_manager.cc @@ -301,7 +301,7 @@ void GcsResourceManager::OnNodeAdd(const rpc::GcsNodeInfo &node) { absl::flat_hash_map labels(node.labels().begin(), node.labels().end()); - cluster_resource_manager_.SetNodeLabels(scheduling_node_id, labels); + cluster_resource_manager_.SetNodeLabels(scheduling_node_id, std::move(labels)); rpc::ResourcesData data; data.set_node_id(node_id.Binary()); diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 61fffd8cee91..0f536647b7cd 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -2767,7 +2767,7 @@ void NodeManager::ConsumeSyncMessage( // Set node labels when node added. auto node_labels = MapFromProtobuf(resource_view_sync_message.labels()); cluster_resource_scheduler_.GetClusterResourceManager().SetNodeLabels( - scheduling::NodeID(node_id.Binary()), node_labels); + scheduling::NodeID(node_id.Binary()), std::move(node_labels)); ResourceRequest resources; for (auto &resource_entry : resource_view_sync_message.resources_total()) { resources.Set(scheduling::ResourceID(resource_entry.first), diff --git a/src/ray/raylet/scheduling/cluster_resource_manager.cc b/src/ray/raylet/scheduling/cluster_resource_manager.cc index de7f10825c84..7ed06e6b96f5 100644 --- a/src/ray/raylet/scheduling/cluster_resource_manager.cc +++ b/src/ray/raylet/scheduling/cluster_resource_manager.cc @@ -293,13 +293,13 @@ BundleLocationIndex &ClusterResourceManager::GetBundleLocationIndex() { void ClusterResourceManager::SetNodeLabels( const scheduling::NodeID &node_id, - const absl::flat_hash_map &labels) { + absl::flat_hash_map labels) { auto it = nodes_.find(node_id); if (it == nodes_.end()) { NodeResources node_resources; it = nodes_.emplace(node_id, node_resources).first; } - it->second.GetMutableLocalView()->labels = labels; + it->second.GetMutableLocalView()->labels = std::move(labels); } } // namespace ray diff --git a/src/ray/raylet/scheduling/cluster_resource_manager.h b/src/ray/raylet/scheduling/cluster_resource_manager.h index a83c6a608624..1b3f01528031 100644 --- a/src/ray/raylet/scheduling/cluster_resource_manager.h +++ b/src/ray/raylet/scheduling/cluster_resource_manager.h @@ -138,7 +138,7 @@ class ClusterResourceManager { BundleLocationIndex &GetBundleLocationIndex(); void SetNodeLabels(const scheduling::NodeID &node_id, - const absl::flat_hash_map &labels); + absl::flat_hash_map labels); private: friend class ClusterResourceScheduler; From fd9bf528baf020f607b91fa71e3e60309bbcdd82 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Mon, 25 Aug 2025 22:17:08 -0700 Subject: [PATCH 269/634] [ci] organizing llm lock files (#55714) Moving llm lock files to a better directory in anticipation of generating many more lock files --------- Signed-off-by: elliot-barn Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> --- ci/docker/llm.build.Dockerfile | 2 +- ci/docker/llm.build.wanda.yaml | 4 +- ci/docker/ray-llm.base.wanda.yaml | 2 +- ci/pipeline/test_rules.txt | 2 +- ci/raydepsets/rayllm.depsets.yaml | 10 +- ci/test_compile_llm_requirements.sh | 8 +- .../examples/e2e-audio/requirements.txt | 2 +- docker/ray-llm/Dockerfile | 4 +- .../llm/ray_py311_cpu.lock} | 216 ++++----- .../llm/ray_py311_cu121.lock} | 216 ++++----- .../llm/ray_py311_cu128.lock} | 216 ++++----- .../llm/ray_test_py311_cpu.lock} | 2 +- .../llm/ray_test_py311_cu121.lock} | 2 +- .../llm/ray_test_py311_cu128.lock} | 2 +- .../llm/rayllm_py311_cpu.lock} | 350 +++++++------- .../llm/rayllm_py311_cu121.lock} | 378 +++++++-------- .../llm/rayllm_py311_cu128.lock} | 378 +++++++-------- .../llm/rayllm_test_py311_cpu.lock} | 446 +++++++++--------- .../llm/rayllm_test_py311_cu121.lock} | 446 +++++++++--------- .../llm/rayllm_test_py311_cu128.lock} | 446 +++++++++--------- 20 files changed, 1566 insertions(+), 1566 deletions(-) rename python/{requirements_compiled_ray_py311_cpu.txt => deplocks/llm/ray_py311_cpu.lock} (95%) rename python/{requirements_compiled_ray_py311_cu121.txt => deplocks/llm/ray_py311_cu121.lock} (95%) rename python/{requirements_compiled_ray_py311_cu128.txt => deplocks/llm/ray_py311_cu128.lock} (95%) rename python/{requirements_compiled_ray_test_py311_cpu.txt => deplocks/llm/ray_test_py311_cpu.lock} (99%) rename python/{requirements_compiled_ray_test_py311_cu121.txt => deplocks/llm/ray_test_py311_cu121.lock} (99%) rename python/{requirements_compiled_ray_test_py311_cu128.txt => deplocks/llm/ray_test_py311_cu128.lock} (99%) rename python/{requirements_compiled_rayllm_py311_cpu.txt => deplocks/llm/rayllm_py311_cpu.lock} (95%) rename python/{requirements_compiled_rayllm_py311_cu121.txt => deplocks/llm/rayllm_py311_cu121.lock} (95%) rename python/{requirements_compiled_rayllm_py311_cu128.txt => deplocks/llm/rayllm_py311_cu128.lock} (95%) rename python/{requirements_compiled_rayllm_test_py311_cpu.txt => deplocks/llm/rayllm_test_py311_cpu.lock} (95%) rename python/{requirements_compiled_rayllm_test_py311_cu121.txt => deplocks/llm/rayllm_test_py311_cu121.lock} (95%) rename python/{requirements_compiled_rayllm_test_py311_cu128.txt => deplocks/llm/rayllm_test_py311_cu128.lock} (95%) diff --git a/ci/docker/llm.build.Dockerfile b/ci/docker/llm.build.Dockerfile index 42e1dca1ac03..312d31c5e94b 100644 --- a/ci/docker/llm.build.Dockerfile +++ b/ci/docker/llm.build.Dockerfile @@ -17,6 +17,6 @@ set -euo pipefail SKIP_PYTHON_PACKAGES=1 ./ci/env/install-dependencies.sh -pip install --no-deps -r python/requirements_compiled_rayllm_test_py311_$RAY_CUDA_CODE.txt +pip install --no-deps -r python/deplocks/llm/rayllm_test_py311_${RAY_CUDA_CODE}.lock EOF diff --git a/ci/docker/llm.build.wanda.yaml b/ci/docker/llm.build.wanda.yaml index 5779c145fcf9..6d89370977a3 100644 --- a/ci/docker/llm.build.wanda.yaml +++ b/ci/docker/llm.build.wanda.yaml @@ -5,8 +5,8 @@ srcs: - ci/env/install-dependencies.sh - ci/env/install-llvm-binaries.sh - ci/suppress_output - - python/requirements_compiled_rayllm_test_py311_cpu.txt - - python/requirements_compiled_rayllm_test_py311_cu128.txt + - python/deplocks/llm/rayllm_test_py311_cpu.lock + - python/deplocks/llm/rayllm_test_py311_cu128.lock tags: - cr.ray.io/rayproject/$IMAGE_TO build_args: diff --git a/ci/docker/ray-llm.base.wanda.yaml b/ci/docker/ray-llm.base.wanda.yaml index ad7db3ea04ec..f1f91c738382 100644 --- a/ci/docker/ray-llm.base.wanda.yaml +++ b/ci/docker/ray-llm.base.wanda.yaml @@ -3,7 +3,7 @@ froms: ["cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base"] dockerfile: docker/ray-llm/Dockerfile srcs: - python/requirements.txt - - python/requirements_compiled_rayllm_py311_cu128.txt + - python/deplocks/llm/rayllm_py311_cu128.lock build_args: - BASE_IMAGE=cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base tags: diff --git a/ci/pipeline/test_rules.txt b/ci/pipeline/test_rules.txt index bc102a99c74e..1e3246f2a8bc 100644 --- a/ci/pipeline/test_rules.txt +++ b/ci/pipeline/test_rules.txt @@ -18,7 +18,7 @@ python/ray/llm/ doc/source/llm/ .buildkite/llm.rayci.yml ci/docker/llm.build.Dockerfile -python/requirements_compiled_*.txt +python/deplocks/llm/*.lock @ llm ; diff --git a/ci/raydepsets/rayllm.depsets.yaml b/ci/raydepsets/rayllm.depsets.yaml index e50b8fc4837d..d96df6168dc7 100644 --- a/ci/raydepsets/rayllm.depsets.yaml +++ b/ci/raydepsets/rayllm.depsets.yaml @@ -32,7 +32,7 @@ depsets: - python/requirements/base-test-requirements.txt constraints: - /tmp/ray-deps/requirements_compiled.txt - output: python/requirements_compiled_ray_test_${PYTHON_VERSION}_${CUDA_CODE}.txt + output: python/deplocks/llm/ray_test_${PYTHON_VERSION}_${CUDA_CODE}.lock operation: compile # Second, expand it into LLM test dependencies. @@ -46,8 +46,8 @@ depsets: - python/requirements/llm/llm-requirements.txt - python/requirements/llm/llm-test-requirements.txt constraints: - - python/requirements_compiled_ray_test_${PYTHON_VERSION}_${CUDA_CODE}.txt - output: python/requirements_compiled_rayllm_test_${PYTHON_VERSION}_${CUDA_CODE}.txt + - python/deplocks/llm/ray_test_${PYTHON_VERSION}_${CUDA_CODE}.lock + output: python/deplocks/llm/rayllm_test_${PYTHON_VERSION}_${CUDA_CODE}.lock # Third, subset the base test dependencies into Ray dependencies. - name: compiled_ray_depset_${PYTHON_VERSION}_${CUDA_CODE} @@ -56,7 +56,7 @@ depsets: source_depset: ray_base_test_depset_${PYTHON_VERSION}_${CUDA_CODE} requirements: - python/requirements.txt - output: python/requirements_compiled_ray_${PYTHON_VERSION}_${CUDA_CODE}.txt + output: python/deplocks/llm/ray_${PYTHON_VERSION}_${CUDA_CODE}.lock # Fourth, subset the LLM test dependencies into RayLLM dependencies. - name: compiled_ray_llm_depset_${PYTHON_VERSION}_${CUDA_CODE} @@ -66,4 +66,4 @@ depsets: requirements: - python/requirements.txt - python/requirements/llm/llm-requirements.txt - output: python/requirements_compiled_rayllm_${PYTHON_VERSION}_${CUDA_CODE}.txt + output: python/deplocks/llm/rayllm_${PYTHON_VERSION}_${CUDA_CODE}.lock diff --git a/ci/test_compile_llm_requirements.sh b/ci/test_compile_llm_requirements.sh index d9e08caf3c77..0843b7bcaac4 100755 --- a/ci/test_compile_llm_requirements.sh +++ b/ci/test_compile_llm_requirements.sh @@ -18,7 +18,7 @@ VARIANTS=(cpu cu121 cu128) for LOCK_TYPE in "${LOCK_TYPES[@]}"; do for VARIANT in "${VARIANTS[@]}"; do - cp ./python/requirements_compiled_"${LOCK_TYPE}"_py311_"${VARIANT}".txt "$TEMP_DIR/requirements_compiled_${LOCK_TYPE}_py311_${VARIANT}_backup.txt" + cp ./python/deplocks/llm/"${LOCK_TYPE}"_py311_"${VARIANT}".lock "$TEMP_DIR/${LOCK_TYPE}_py311_${VARIANT}_backup.lock" done done @@ -27,7 +27,7 @@ done # Copy files to artifact mount on Buildkite for LOCK_TYPE in "${LOCK_TYPES[@]}"; do for VARIANT in "${VARIANTS[@]}"; do - cp ./python/requirements_compiled_"${LOCK_TYPE}"_py311_"${VARIANT}".txt /artifact-mount/ + cp ./python/deplocks/llm/"${LOCK_TYPE}"_py311_"${VARIANT}".lock /artifact-mount/ done done @@ -35,8 +35,8 @@ done FAILED=0 for LOCK_TYPE in "${LOCK_TYPES[@]}"; do for VARIANT in "${VARIANTS[@]}"; do - diff --color -u ./python/requirements_compiled_"${LOCK_TYPE}"_py311_"${VARIANT}".txt "$TEMP_DIR/requirements_compiled_${LOCK_TYPE}_py311_${VARIANT}_backup.txt" || { - echo "requirements_compiled_${LOCK_TYPE}_py311_${VARIANT}.txt is not up to date. Please download it from Artifacts tab and git push the changes." + diff --color -u ./python/deplocks/llm/"${LOCK_TYPE}"_py311_"${VARIANT}".lock "$TEMP_DIR/${LOCK_TYPE}_py311_${VARIANT}_backup.lock" || { + echo "${LOCK_TYPE}_py311_${VARIANT}.lock is not up to date. Please download it from Artifacts tab and git push the changes." FAILED=1 } done diff --git a/doc/source/ray-overview/examples/e2e-audio/requirements.txt b/doc/source/ray-overview/examples/e2e-audio/requirements.txt index 4f2606eeaa9e..0c47ff4c5f00 100644 --- a/doc/source/ray-overview/examples/e2e-audio/requirements.txt +++ b/doc/source/ray-overview/examples/e2e-audio/requirements.txt @@ -1,7 +1,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --constraint=python/requirements_compiled_rayllm_py311_cu128.txt --no-annotate --no-emit-index-url --no-emit-trusted-host --output-file=requirements.txt --strip-extras doc/source/ray-overview/examples/e2e-audio/requirements.in +# pip-compile --constraint=python/deplocks/llm/rayllm_py311_cu128.lock --no-annotate --no-emit-index-url --no-emit-trusted-host --output-file=requirements.txt --strip-extras doc/source/ray-overview/examples/e2e-audio/requirements.in # # ... and then slimmed down by Ricardo accelerate==1.7.0 diff --git a/docker/ray-llm/Dockerfile b/docker/ray-llm/Dockerfile index d7038f92b883..e933928bcf10 100644 --- a/docker/ray-llm/Dockerfile +++ b/docker/ray-llm/Dockerfile @@ -3,7 +3,7 @@ ARG BASE_IMAGE FROM "$BASE_IMAGE" -COPY python/requirements_*.txt ./ +COPY python/deplocks/llm/rayllm_*.lock ./ ARG KVER="5.15.0-139-generic" ARG ROOT_DIR="/usr/local" @@ -28,7 +28,7 @@ fi uv pip install --system --no-cache-dir --no-deps \ --index-strategy unsafe-best-match \ - -r "requirements_compiled_rayllm_${PYTHON_CODE}_${CUDA_CODE}.txt" + -r "rayllm_${PYTHON_CODE}_${CUDA_CODE}.lock" # Export installed packages $HOME/anaconda3/bin/pip freeze > /home/ray/pip-freeze.txt diff --git a/python/requirements_compiled_ray_py311_cpu.txt b/python/deplocks/llm/ray_py311_cpu.lock similarity index 95% rename from python/requirements_compiled_ray_py311_cpu.txt rename to python/deplocks/llm/ray_py311_cpu.lock index ad9a94b18e3b..cd1fdcbbc79f 100644 --- a/python/requirements_compiled_ray_py311_cpu.txt +++ b/python/deplocks/llm/ray_py311_cpu.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cpu.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cpu.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu @@ -7,7 +7,7 @@ aiohappyeyeballs==2.6.1 \ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp aiohttp==3.11.16 \ --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ @@ -92,51 +92,51 @@ aiohttp==3.11.16 \ --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # aiohttp-cors aiohttp-cors==0.7.0 \ --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt aiorwlock==1.3.0 \ --hash=sha256:45baf8e4fa9a23e0bb325fbd67da80de1fd7ae1d4f59a6381754c60cec7b289b \ --hash=sha256:83f12d87df4b9728a0b8fda1756585ab0d652b107bab59c6084e1b1ad692ab45 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp amqp==5.3.1 \ --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # pydantic anyio==3.7.1 \ --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # starlette # watchfiles attrs==25.1.0 \ --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp # jsonschema # referencing @@ -144,25 +144,25 @@ billiard==4.2.1 \ --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # celery cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-auth celery==5.5.3 \ --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # requests cffi==1.16.0 ; platform_python_implementation != 'PyPy' \ --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ @@ -218,7 +218,7 @@ cffi==1.16.0 ; platform_python_implementation != 'PyPy' \ --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # cryptography charset-normalizer==3.3.2 \ --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ @@ -312,13 +312,13 @@ charset-normalizer==3.3.2 \ --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # requests click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # celery # click-didyoumean @@ -330,31 +330,31 @@ click-didyoumean==0.3.1 \ --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # celery click-plugins==1.1.1.2 \ --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # celery click-repl==0.3.0 \ --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # gymnasium colorful==0.5.5 \ --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt cryptography==44.0.3 \ --hash=sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259 \ @@ -395,7 +395,7 @@ cryptography==44.0.3 \ --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # pyopenssl cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:230f8a8e99c81a653baa0ed00819990c0ed1f0cf0298214786b5e323461dc61a \ @@ -411,13 +411,13 @@ cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt distlib==0.3.7 \ --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # virtualenv dm-tree==0.1.8 \ --hash=sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6 \ @@ -467,19 +467,19 @@ dm-tree==0.1.8 \ --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt farama-notifications==0.0.4 \ --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # gymnasium fastapi==0.115.12 \ --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ @@ -558,13 +558,13 @@ fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # cupy-cuda12x filelock==3.17.0 \ --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # virtualenv frozenlist==1.4.1 \ @@ -646,32 +646,32 @@ frozenlist==1.4.1 \ --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp # aiosignal fsspec==2023.5.0 \ --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt google-api-core==2.24.2 \ --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # opencensus google-auth==2.23.4 \ --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-api-core googleapis-common-protos==1.61.0 \ --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-api-core grpcio==1.66.2 \ --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ @@ -730,25 +730,25 @@ grpcio==1.66.2 \ --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt gymnasium==1.1.1 \ --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # uvicorn idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # anyio # requests # yarl @@ -756,43 +756,43 @@ imageio==2.34.2 \ --hash=sha256:5c0c0ee8faa018a1c42f649b90395dd4d3bb6187c09053a0cd6f1fdd51bbff5e \ --hash=sha256:a0bb27ec9d5bab36a9f4835e51b21d2cb099e1f78451441f94687ff3404b79f8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # scikit-image importlib-metadata==6.11.0 \ --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # opentelemetry-api jinja2==3.1.6 ; sys_platform != 'win32' \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # memray jsonschema==4.23.0 \ --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt jsonschema-specifications==2024.10.1 \ --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema kombu==5.5.4 \ --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # celery lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # scikit-image lz4==4.3.3 \ --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ @@ -832,13 +832,13 @@ lz4==4.3.3 \ --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt markdown-it-py==2.2.0 \ --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # rich markupsafe==2.1.3 ; sys_platform != 'win32' \ --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ @@ -902,13 +902,13 @@ markupsafe==2.1.3 ; sys_platform != 'win32' \ --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jinja2 mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # markdown-it-py memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ @@ -947,7 +947,7 @@ memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt msgpack==1.0.7 \ --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ @@ -1007,7 +1007,7 @@ msgpack==1.0.7 \ --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt multidict==6.0.5 \ --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ @@ -1101,14 +1101,14 @@ multidict==6.0.5 \ --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp # yarl networkx==3.2.1 \ --hash=sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6 \ --hash=sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # scikit-image numpy==1.26.4 \ --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ @@ -1148,7 +1148,7 @@ numpy==1.26.4 \ --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # cupy-cuda12x # gymnasium @@ -1162,19 +1162,19 @@ opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt opencensus-context==0.1.3 \ --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # opencensus opentelemetry-api==1.34.1 \ --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # opentelemetry-sdk @@ -1183,32 +1183,32 @@ opentelemetry-exporter-prometheus==0.55b1 \ --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt opentelemetry-proto==1.27.0 \ --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt opentelemetry-sdk==1.34.1 \ --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus opentelemetry-semantic-conventions==0.55b1 \ --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # opentelemetry-sdk packaging==23.0 \ --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # kombu # lazy-loader @@ -1243,7 +1243,7 @@ pandas==1.5.3 \ --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt pillow==10.3.0 \ --hash=sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c \ @@ -1316,27 +1316,27 @@ pillow==10.3.0 \ --hash=sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27 \ --hash=sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # imageio # scikit-image platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # virtualenv prometheus-client==0.19.0 \ --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus prompt-toolkit==3.0.41 \ --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -1438,14 +1438,14 @@ propcache==0.3.0 \ --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp # yarl proto-plus==1.22.3 \ --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-api-core protobuf==4.25.8 \ --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ @@ -1460,7 +1460,7 @@ protobuf==4.25.8 \ --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # google-api-core # googleapis-common-protos @@ -1477,7 +1477,7 @@ py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt pyarrow==19.0.1 \ --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ @@ -1523,32 +1523,32 @@ pyarrow==19.0.1 \ --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt pyasn1==0.5.1 \ --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # pyasn1-modules # rsa pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-auth pycparser==2.21 ; platform_python_implementation != 'PyPy' \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # cffi pydantic==2.10.0 \ --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # fastapi pydantic-core==2.27.0 \ @@ -1653,32 +1653,32 @@ pydantic-core==2.27.0 \ --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # pydantic pygments==2.18.0 \ --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # rich pyopenssl==25.0.0 \ --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # celery # pandas pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # pandas pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ @@ -1733,27 +1733,27 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt referencing==0.36.2 \ --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema # jsonschema-specifications requests==2.32.3 \ --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # google-api-core rich==13.3.2 \ --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # memray # typer @@ -1862,14 +1862,14 @@ rpds-py==0.22.3 \ --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema # referencing rsa==4.7.2 \ --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-auth scikit-image==0.24.0 \ --hash=sha256:18836a18d3a7b6aca5376a2d805f0045826bc6c9fc85331659c33b4813e0b563 \ @@ -1894,7 +1894,7 @@ scikit-image==0.24.0 \ --hash=sha256:ef04360eda372ee5cd60aebe9be91258639c86ae2ea24093fb9182118008d009 \ --hash=sha256:fa27b3a0dbad807b966b8db2d78da734cb812ca4787f7fbb143764800ce2fa9c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt scipy==1.11.4 \ --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ @@ -1923,64 +1923,64 @@ scipy==1.11.4 \ --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # scikit-image shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # typer six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # opencensus # python-dateutil smart-open==6.2.0 \ --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # anyio starlette==0.46.2 \ --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # fastapi tensorboardx==2.6.2.2 \ --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt tifffile==2024.7.21 \ --hash=sha256:7f335b5d6ca49401fe0f1d87deb206f5dae47297e47b1ed52a676d05d6d26798 \ --hash=sha256:818b577d49350421fb511f389f937984f9feaa2cd8177fa00823001920bf3483 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # scikit-image typer==0.12.3 \ --hash=sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914 \ --hash=sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # fastapi # gymnasium # opentelemetry-api @@ -1995,25 +1995,25 @@ tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # requests uvicorn==0.22.0 \ --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt vine==5.1.0 \ --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # amqp # celery # kombu @@ -2021,7 +2021,7 @@ virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt watchfiles==0.19.0 \ --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ @@ -2047,13 +2047,13 @@ watchfiles==0.19.0 \ --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt wcwidth==0.2.13 \ --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # prompt-toolkit yarl==1.18.3 \ --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ @@ -2139,11 +2139,11 @@ yarl==1.18.3 \ --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp zipp==3.19.2 \ --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # importlib-metadata diff --git a/python/requirements_compiled_ray_py311_cu121.txt b/python/deplocks/llm/ray_py311_cu121.lock similarity index 95% rename from python/requirements_compiled_ray_py311_cu121.txt rename to python/deplocks/llm/ray_py311_cu121.lock index 40b0814f0d07..3417dd6ec83c 100644 --- a/python/requirements_compiled_ray_py311_cu121.txt +++ b/python/deplocks/llm/ray_py311_cu121.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu121.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cu121.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 @@ -7,7 +7,7 @@ aiohappyeyeballs==2.6.1 \ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp aiohttp==3.11.16 \ --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ @@ -92,51 +92,51 @@ aiohttp==3.11.16 \ --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # aiohttp-cors aiohttp-cors==0.7.0 \ --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt aiorwlock==1.3.0 \ --hash=sha256:45baf8e4fa9a23e0bb325fbd67da80de1fd7ae1d4f59a6381754c60cec7b289b \ --hash=sha256:83f12d87df4b9728a0b8fda1756585ab0d652b107bab59c6084e1b1ad692ab45 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp amqp==5.3.1 \ --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # pydantic anyio==3.7.1 \ --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # starlette # watchfiles attrs==25.1.0 \ --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp # jsonschema # referencing @@ -144,25 +144,25 @@ billiard==4.2.1 \ --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # celery cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-auth celery==5.5.3 \ --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # requests cffi==1.16.0 ; platform_python_implementation != 'PyPy' \ --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ @@ -218,7 +218,7 @@ cffi==1.16.0 ; platform_python_implementation != 'PyPy' \ --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # cryptography charset-normalizer==3.3.2 \ --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ @@ -312,13 +312,13 @@ charset-normalizer==3.3.2 \ --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # requests click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # celery # click-didyoumean @@ -330,31 +330,31 @@ click-didyoumean==0.3.1 \ --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # celery click-plugins==1.1.1.2 \ --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # celery click-repl==0.3.0 \ --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # gymnasium colorful==0.5.5 \ --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt cryptography==44.0.3 \ --hash=sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259 \ @@ -395,7 +395,7 @@ cryptography==44.0.3 \ --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # pyopenssl cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:230f8a8e99c81a653baa0ed00819990c0ed1f0cf0298214786b5e323461dc61a \ @@ -411,13 +411,13 @@ cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt distlib==0.3.7 \ --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # virtualenv dm-tree==0.1.8 \ --hash=sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6 \ @@ -467,19 +467,19 @@ dm-tree==0.1.8 \ --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt farama-notifications==0.0.4 \ --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # gymnasium fastapi==0.115.12 \ --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ @@ -558,13 +558,13 @@ fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # cupy-cuda12x filelock==3.17.0 \ --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # virtualenv frozenlist==1.4.1 \ @@ -646,32 +646,32 @@ frozenlist==1.4.1 \ --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp # aiosignal fsspec==2023.5.0 \ --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt google-api-core==2.24.2 \ --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # opencensus google-auth==2.23.4 \ --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-api-core googleapis-common-protos==1.61.0 \ --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-api-core grpcio==1.66.2 \ --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ @@ -730,25 +730,25 @@ grpcio==1.66.2 \ --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt gymnasium==1.1.1 \ --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # uvicorn idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # anyio # requests # yarl @@ -756,43 +756,43 @@ imageio==2.34.2 \ --hash=sha256:5c0c0ee8faa018a1c42f649b90395dd4d3bb6187c09053a0cd6f1fdd51bbff5e \ --hash=sha256:a0bb27ec9d5bab36a9f4835e51b21d2cb099e1f78451441f94687ff3404b79f8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # scikit-image importlib-metadata==6.11.0 \ --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # opentelemetry-api jinja2==3.1.6 ; sys_platform != 'win32' \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # memray jsonschema==4.23.0 \ --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt jsonschema-specifications==2024.10.1 \ --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema kombu==5.5.4 \ --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # celery lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # scikit-image lz4==4.3.3 \ --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ @@ -832,13 +832,13 @@ lz4==4.3.3 \ --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt markdown-it-py==2.2.0 \ --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # rich markupsafe==2.1.3 ; sys_platform != 'win32' \ --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ @@ -902,13 +902,13 @@ markupsafe==2.1.3 ; sys_platform != 'win32' \ --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jinja2 mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # markdown-it-py memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ @@ -947,7 +947,7 @@ memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt msgpack==1.0.7 \ --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ @@ -1007,7 +1007,7 @@ msgpack==1.0.7 \ --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt multidict==6.0.5 \ --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ @@ -1101,14 +1101,14 @@ multidict==6.0.5 \ --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp # yarl networkx==3.2.1 \ --hash=sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6 \ --hash=sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # scikit-image numpy==1.26.4 \ --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ @@ -1148,7 +1148,7 @@ numpy==1.26.4 \ --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # cupy-cuda12x # gymnasium @@ -1162,19 +1162,19 @@ opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt opencensus-context==0.1.3 \ --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # opencensus opentelemetry-api==1.34.1 \ --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # opentelemetry-sdk @@ -1183,32 +1183,32 @@ opentelemetry-exporter-prometheus==0.55b1 \ --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt opentelemetry-proto==1.27.0 \ --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt opentelemetry-sdk==1.34.1 \ --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus opentelemetry-semantic-conventions==0.55b1 \ --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # opentelemetry-sdk packaging==23.0 \ --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # kombu # lazy-loader @@ -1243,7 +1243,7 @@ pandas==1.5.3 \ --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt pillow==10.3.0 \ --hash=sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c \ @@ -1316,27 +1316,27 @@ pillow==10.3.0 \ --hash=sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27 \ --hash=sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # imageio # scikit-image platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # virtualenv prometheus-client==0.19.0 \ --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus prompt-toolkit==3.0.41 \ --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -1438,14 +1438,14 @@ propcache==0.3.0 \ --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp # yarl proto-plus==1.22.3 \ --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-api-core protobuf==4.25.8 \ --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ @@ -1460,7 +1460,7 @@ protobuf==4.25.8 \ --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # google-api-core # googleapis-common-protos @@ -1477,7 +1477,7 @@ py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt pyarrow==19.0.1 \ --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ @@ -1523,32 +1523,32 @@ pyarrow==19.0.1 \ --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt pyasn1==0.5.1 \ --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # pyasn1-modules # rsa pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-auth pycparser==2.21 ; platform_python_implementation != 'PyPy' \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # cffi pydantic==2.10.0 \ --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # fastapi pydantic-core==2.27.0 \ @@ -1653,32 +1653,32 @@ pydantic-core==2.27.0 \ --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # pydantic pygments==2.18.0 \ --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # rich pyopenssl==25.0.0 \ --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # celery # pandas pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # pandas pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ @@ -1733,27 +1733,27 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt referencing==0.36.2 \ --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema # jsonschema-specifications requests==2.32.3 \ --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # google-api-core rich==13.3.2 \ --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # memray # typer @@ -1862,14 +1862,14 @@ rpds-py==0.22.3 \ --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema # referencing rsa==4.7.2 \ --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-auth scikit-image==0.24.0 \ --hash=sha256:18836a18d3a7b6aca5376a2d805f0045826bc6c9fc85331659c33b4813e0b563 \ @@ -1894,7 +1894,7 @@ scikit-image==0.24.0 \ --hash=sha256:ef04360eda372ee5cd60aebe9be91258639c86ae2ea24093fb9182118008d009 \ --hash=sha256:fa27b3a0dbad807b966b8db2d78da734cb812ca4787f7fbb143764800ce2fa9c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt scipy==1.11.4 \ --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ @@ -1923,64 +1923,64 @@ scipy==1.11.4 \ --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # scikit-image shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # typer six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # opencensus # python-dateutil smart-open==6.2.0 \ --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # anyio starlette==0.46.2 \ --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # fastapi tensorboardx==2.6.2.2 \ --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt tifffile==2024.7.21 \ --hash=sha256:7f335b5d6ca49401fe0f1d87deb206f5dae47297e47b1ed52a676d05d6d26798 \ --hash=sha256:818b577d49350421fb511f389f937984f9feaa2cd8177fa00823001920bf3483 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # scikit-image typer==0.12.3 \ --hash=sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914 \ --hash=sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # fastapi # gymnasium # opentelemetry-api @@ -1995,25 +1995,25 @@ tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # requests uvicorn==0.22.0 \ --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt vine==5.1.0 \ --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # amqp # celery # kombu @@ -2021,7 +2021,7 @@ virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt watchfiles==0.19.0 \ --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ @@ -2047,13 +2047,13 @@ watchfiles==0.19.0 \ --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt wcwidth==0.2.13 \ --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # prompt-toolkit yarl==1.18.3 \ --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ @@ -2139,11 +2139,11 @@ yarl==1.18.3 \ --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp zipp==3.19.2 \ --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # importlib-metadata diff --git a/python/requirements_compiled_ray_py311_cu128.txt b/python/deplocks/llm/ray_py311_cu128.lock similarity index 95% rename from python/requirements_compiled_ray_py311_cu128.txt rename to python/deplocks/llm/ray_py311_cu128.lock index e03e8073bde2..411cb5498708 100644 --- a/python/requirements_compiled_ray_py311_cu128.txt +++ b/python/deplocks/llm/ray_py311_cu128.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt -o python/requirements_compiled_ray_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu128.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cu128.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 @@ -7,7 +7,7 @@ aiohappyeyeballs==2.6.1 \ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp aiohttp==3.11.16 \ --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ @@ -92,51 +92,51 @@ aiohttp==3.11.16 \ --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # aiohttp-cors aiohttp-cors==0.7.0 \ --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt aiorwlock==1.3.0 \ --hash=sha256:45baf8e4fa9a23e0bb325fbd67da80de1fd7ae1d4f59a6381754c60cec7b289b \ --hash=sha256:83f12d87df4b9728a0b8fda1756585ab0d652b107bab59c6084e1b1ad692ab45 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp amqp==5.3.1 \ --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # pydantic anyio==3.7.1 \ --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # starlette # watchfiles attrs==25.1.0 \ --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp # jsonschema # referencing @@ -144,25 +144,25 @@ billiard==4.2.1 \ --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # celery cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-auth celery==5.5.3 \ --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # requests cffi==1.16.0 ; platform_python_implementation != 'PyPy' \ --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ @@ -218,7 +218,7 @@ cffi==1.16.0 ; platform_python_implementation != 'PyPy' \ --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # cryptography charset-normalizer==3.3.2 \ --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ @@ -312,13 +312,13 @@ charset-normalizer==3.3.2 \ --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # requests click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # celery # click-didyoumean @@ -330,31 +330,31 @@ click-didyoumean==0.3.1 \ --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # celery click-plugins==1.1.1.2 \ --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # celery click-repl==0.3.0 \ --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # gymnasium colorful==0.5.5 \ --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt cryptography==44.0.3 \ --hash=sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259 \ @@ -395,7 +395,7 @@ cryptography==44.0.3 \ --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # pyopenssl cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:230f8a8e99c81a653baa0ed00819990c0ed1f0cf0298214786b5e323461dc61a \ @@ -411,13 +411,13 @@ cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt distlib==0.3.7 \ --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # virtualenv dm-tree==0.1.8 \ --hash=sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6 \ @@ -467,19 +467,19 @@ dm-tree==0.1.8 \ --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt farama-notifications==0.0.4 \ --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # gymnasium fastapi==0.115.12 \ --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ @@ -558,13 +558,13 @@ fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # cupy-cuda12x filelock==3.17.0 \ --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # virtualenv frozenlist==1.4.1 \ @@ -646,32 +646,32 @@ frozenlist==1.4.1 \ --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp # aiosignal fsspec==2023.5.0 \ --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt google-api-core==2.24.2 \ --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # opencensus google-auth==2.23.4 \ --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-api-core googleapis-common-protos==1.61.0 \ --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-api-core grpcio==1.66.2 \ --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ @@ -730,25 +730,25 @@ grpcio==1.66.2 \ --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt gymnasium==1.1.1 \ --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # uvicorn idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # anyio # requests # yarl @@ -756,43 +756,43 @@ imageio==2.34.2 \ --hash=sha256:5c0c0ee8faa018a1c42f649b90395dd4d3bb6187c09053a0cd6f1fdd51bbff5e \ --hash=sha256:a0bb27ec9d5bab36a9f4835e51b21d2cb099e1f78451441f94687ff3404b79f8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # scikit-image importlib-metadata==6.11.0 \ --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # opentelemetry-api jinja2==3.1.6 ; sys_platform != 'win32' \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # memray jsonschema==4.23.0 \ --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt jsonschema-specifications==2024.10.1 \ --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema kombu==5.5.4 \ --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # celery lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # scikit-image lz4==4.3.3 \ --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ @@ -832,13 +832,13 @@ lz4==4.3.3 \ --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt markdown-it-py==2.2.0 \ --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # rich markupsafe==2.1.3 ; sys_platform != 'win32' \ --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ @@ -867,13 +867,13 @@ markupsafe==2.1.3 ; sys_platform != 'win32' \ --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jinja2 mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # markdown-it-py memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ @@ -912,7 +912,7 @@ memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt msgpack==1.0.7 \ --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ @@ -972,7 +972,7 @@ msgpack==1.0.7 \ --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt multidict==6.0.5 \ --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ @@ -1066,13 +1066,13 @@ multidict==6.0.5 \ --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp # yarl networkx==3.2.1 \ --hash=sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # scikit-image numpy==1.26.4 \ --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ @@ -1112,7 +1112,7 @@ numpy==1.26.4 \ --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # cupy-cuda12x # gymnasium @@ -1126,19 +1126,19 @@ opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt opencensus-context==0.1.3 \ --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # opencensus opentelemetry-api==1.34.1 \ --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # opentelemetry-sdk @@ -1147,32 +1147,32 @@ opentelemetry-exporter-prometheus==0.55b1 \ --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt opentelemetry-proto==1.27.0 \ --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt opentelemetry-sdk==1.34.1 \ --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus opentelemetry-semantic-conventions==0.55b1 \ --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # opentelemetry-sdk packaging==23.0 \ --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # kombu # lazy-loader @@ -1207,7 +1207,7 @@ pandas==1.5.3 \ --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt pillow==10.3.0 \ --hash=sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c \ @@ -1280,27 +1280,27 @@ pillow==10.3.0 \ --hash=sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27 \ --hash=sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # imageio # scikit-image platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # virtualenv prometheus-client==0.19.0 \ --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus prompt-toolkit==3.0.41 \ --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -1402,14 +1402,14 @@ propcache==0.3.0 \ --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp # yarl proto-plus==1.22.3 \ --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-api-core protobuf==4.25.8 \ --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ @@ -1424,7 +1424,7 @@ protobuf==4.25.8 \ --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # google-api-core # googleapis-common-protos @@ -1441,7 +1441,7 @@ py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt pyarrow==19.0.1 \ --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ @@ -1487,32 +1487,32 @@ pyarrow==19.0.1 \ --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt pyasn1==0.5.1 \ --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # pyasn1-modules # rsa pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-auth pycparser==2.21 ; platform_python_implementation != 'PyPy' \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # cffi pydantic==2.10.0 \ --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # fastapi pydantic-core==2.27.0 \ @@ -1617,32 +1617,32 @@ pydantic-core==2.27.0 \ --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # pydantic pygments==2.18.0 \ --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # rich pyopenssl==25.0.0 \ --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # celery # pandas pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # pandas pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ @@ -1697,27 +1697,27 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt referencing==0.36.2 \ --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema # jsonschema-specifications requests==2.32.3 \ --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # google-api-core rich==13.3.2 \ --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # memray # typer @@ -1826,14 +1826,14 @@ rpds-py==0.22.3 \ --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema # referencing rsa==4.7.2 \ --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-auth scikit-image==0.24.0 \ --hash=sha256:18836a18d3a7b6aca5376a2d805f0045826bc6c9fc85331659c33b4813e0b563 \ @@ -1858,7 +1858,7 @@ scikit-image==0.24.0 \ --hash=sha256:ef04360eda372ee5cd60aebe9be91258639c86ae2ea24093fb9182118008d009 \ --hash=sha256:fa27b3a0dbad807b966b8db2d78da734cb812ca4787f7fbb143764800ce2fa9c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt scipy==1.11.4 \ --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ @@ -1887,63 +1887,63 @@ scipy==1.11.4 \ --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # scikit-image shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # typer six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # opencensus # python-dateutil smart-open==6.2.0 \ --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # anyio starlette==0.46.2 \ --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # fastapi tensorboardx==2.6.2.2 \ --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt tifffile==2024.7.21 \ --hash=sha256:7f335b5d6ca49401fe0f1d87deb206f5dae47297e47b1ed52a676d05d6d26798 \ --hash=sha256:818b577d49350421fb511f389f937984f9feaa2cd8177fa00823001920bf3483 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # scikit-image typer==0.12.3 \ --hash=sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914 \ --hash=sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # fastapi # gymnasium # opentelemetry-api @@ -1958,25 +1958,25 @@ tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # requests uvicorn==0.22.0 \ --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt vine==5.1.0 \ --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # amqp # celery # kombu @@ -1984,7 +1984,7 @@ virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt watchfiles==0.19.0 \ --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ @@ -2010,13 +2010,13 @@ watchfiles==0.19.0 \ --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt wcwidth==0.2.13 \ --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # prompt-toolkit yarl==1.18.3 \ --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ @@ -2102,11 +2102,11 @@ yarl==1.18.3 \ --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp zipp==3.19.2 \ --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # importlib-metadata diff --git a/python/requirements_compiled_ray_test_py311_cpu.txt b/python/deplocks/llm/ray_test_py311_cpu.lock similarity index 99% rename from python/requirements_compiled_ray_test_py311_cpu.txt rename to python/deplocks/llm/ray_test_py311_cpu.lock index 6d33de854539..a5778d763b58 100644 --- a/python/requirements_compiled_ray_test_py311_cpu.txt +++ b/python/deplocks/llm/ray_test_py311_cpu.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/deplocks/llm/ray_test_py311_cpu.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/requirements_compiled_ray_test_py311_cu121.txt b/python/deplocks/llm/ray_test_py311_cu121.lock similarity index 99% rename from python/requirements_compiled_ray_test_py311_cu121.txt rename to python/deplocks/llm/ray_test_py311_cu121.lock index 8c0d7d9b187a..9d7dd16bc2b9 100644 --- a/python/requirements_compiled_ray_test_py311_cu121.txt +++ b/python/deplocks/llm/ray_test_py311_cu121.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/deplocks/llm/ray_test_py311_cu121.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/requirements_compiled_ray_test_py311_cu128.txt b/python/deplocks/llm/ray_test_py311_cu128.lock similarity index 99% rename from python/requirements_compiled_ray_test_py311_cu128.txt rename to python/deplocks/llm/ray_test_py311_cu128.lock index b61e16287344..8bde8ef1045a 100644 --- a/python/requirements_compiled_ray_test_py311_cu128.txt +++ b/python/deplocks/llm/ray_test_py311_cu128.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/requirements_compiled_ray_test_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/deplocks/llm/ray_test_py311_cu128.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/python/requirements_compiled_rayllm_py311_cpu.txt b/python/deplocks/llm/rayllm_py311_cpu.lock similarity index 95% rename from python/requirements_compiled_rayllm_py311_cpu.txt rename to python/deplocks/llm/rayllm_py311_cpu.lock index 9ba0d45b6817..fafc05f0902e 100644 --- a/python/requirements_compiled_rayllm_py311_cpu.txt +++ b/python/deplocks/llm/rayllm_py311_cpu.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/requirements_compiled_rayllm_test_py311_cpu.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/deplocks/llm/rayllm_test_py311_cpu.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cpu.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu @@ -7,7 +7,7 @@ aiohappyeyeballs==2.6.1 \ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # aiohttp aiohttp==3.11.16 \ --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ @@ -92,7 +92,7 @@ aiohttp==3.11.16 \ --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # aiohttp-cors # vllm @@ -100,37 +100,37 @@ aiohttp-cors==0.7.0 \ --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt aiorwlock==1.3.0 \ --hash=sha256:45baf8e4fa9a23e0bb325fbd67da80de1fd7ae1d4f59a6381754c60cec7b289b \ --hash=sha256:83f12d87df4b9728a0b8fda1756585ab0d652b107bab59c6084e1b1ad692ab45 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # aiohttp amqp==5.3.1 \ --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # pydantic anyio==3.7.1 \ --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # httpx # openai # starlette @@ -139,13 +139,13 @@ astor==0.8.1 \ --hash=sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5 \ --hash=sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # depyf attrs==25.1.0 \ --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # aiohttp # jsonschema # referencing @@ -153,7 +153,7 @@ billiard==4.2.1 \ --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # celery blake3==1.0.4 \ --hash=sha256:00605aa59923205c6a4f21131840840eb2d9a754c59b163357d890566755b97a \ @@ -242,13 +242,13 @@ blake3==1.0.4 \ --hash=sha256:fedc326cac4476d2eab88413a4bf56e491040ae11ea98ddadaa5487cecda9b93 \ --hash=sha256:ff0e96f61b16b365ad5bb7c6272754f83d8a59c95d3b2f70c3bb6324ddf5bc0c # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # google-auth # vllm cbor2==5.6.5 \ @@ -297,19 +297,19 @@ cbor2==5.6.5 \ --hash=sha256:fde21ac1cf29336a31615a2c469a9cb03cf0add3ae480672d4d38cda467d07fc \ --hash=sha256:fe11c2eb518c882cfbeed456e7a552e544893c17db66fe5d3230dbeaca6b615c # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm celery==5.5.3 \ --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # httpcore # httpx # requests @@ -367,7 +367,7 @@ cffi==1.16.0 \ --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # cryptography # soundfile charset-normalizer==3.3.2 \ @@ -462,13 +462,13 @@ charset-normalizer==3.3.2 \ --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # requests click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # celery # click-didyoumean @@ -481,38 +481,38 @@ click-didyoumean==0.3.1 \ --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # celery click-plugins==1.1.1.2 \ --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # celery click-repl==0.3.0 \ --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # gymnasium # vllm colorful==0.5.5 \ --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt compressed-tensors==0.10.2 \ --hash=sha256:6de13ac535d7ffdd8890fad3d229444c33076170acaa8fab6bab8ecfa96c1d8f \ --hash=sha256:e1b4d9bc2006e3fd3a938e59085f318fdb280c5af64688a4792bf1bc263e579d # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm cryptography==44.0.3 \ --hash=sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259 \ @@ -553,7 +553,7 @@ cryptography==44.0.3 \ --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # pyopenssl cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:230f8a8e99c81a653baa0ed00819990c0ed1f0cf0298214786b5e323461dc61a \ @@ -569,38 +569,38 @@ cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # ray depyf==0.19.0 \ --hash=sha256:040b35fc0997d49df024b7d094f2a7836f91e9ed02f49982dd37e70aa3285ad5 \ --hash=sha256:afed0916b32d141cc90fa6220df01885eda442ca43b297d5050eeb90b4a5cb44 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm dill==0.3.9 \ --hash=sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a \ --hash=sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # depyf diskcache==5.6.3 \ --hash=sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc \ --hash=sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm distlib==0.3.7 \ --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # virtualenv distro==1.9.0 \ --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # openai dm-tree==0.1.8 \ --hash=sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6 \ @@ -650,44 +650,44 @@ dm-tree==0.1.8 \ --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt dnspython==2.7.0 \ --hash=sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86 \ --hash=sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # email-validator einops==0.8.1 \ --hash=sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737 \ --hash=sha256:de5d960a7a761225532e0f1959e5315ebeafc0cd43394732f103ca44b9837e84 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm email-validator==2.2.0 \ --hash=sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631 \ --hash=sha256:cb690f344c617a714f22e66ae771445a1ceb46821152df8e165c5f9a364582b7 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # fastapi farama-notifications==0.0.4 \ --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # gymnasium fastapi==0.115.12 \ --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # vllm fastapi-cli==0.0.5 \ --hash=sha256:d30e1239c6f46fcb95e606f02cdda59a1e2fa778a54b64686b3ff27f6211ff9f \ --hash=sha256:e94d847524648c748a5350673546bbf9bcaeb086b33c24f2e82e021436866a46 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # fastapi fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ @@ -766,13 +766,13 @@ fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # cupy-cuda12x filelock==3.17.0 \ --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # huggingface-hub # ray @@ -859,14 +859,14 @@ frozenlist==1.4.1 \ --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # aiohttp # aiosignal fsspec==2023.5.0 \ --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # huggingface-hub # torch @@ -874,25 +874,25 @@ gguf==0.16.2 \ --hash=sha256:0fc956289a30d0f1f3afd75ec0d493f73ae2629a3f21f3846dd1687d8791c7c1 \ --hash=sha256:e73eb19b30fcc7c7f32894345024dda8b1a0c959b94a12b7c40ded8dd3f96810 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm google-api-core==2.24.2 \ --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # opencensus google-auth==2.23.4 \ --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # google-api-core googleapis-common-protos==1.61.0 \ --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # google-api-core grpcio==1.66.2 \ --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ @@ -951,19 +951,19 @@ grpcio==1.66.2 \ --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt gymnasium==1.1.1 \ --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # httpcore # uvicorn hf-transfer==0.1.9 \ @@ -993,7 +993,7 @@ hf-transfer==0.1.9 \ --hash=sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f \ --hash=sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt hf-xet==1.1.5 \ --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 \ @@ -1005,13 +1005,13 @@ hf-xet==1.1.5 \ --hash=sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23 \ --hash=sha256:fc874b5c843e642f45fd85cda1ce599e123308ad2901ead23d3510a47ff506d1 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # huggingface-hub httpcore==1.0.9 \ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # httpx httptools==0.6.4 \ --hash=sha256:0614154d5454c21b6410fdf5262b4a3ddb0f53f1e1721cfd59d55f32138c578a \ @@ -1058,20 +1058,20 @@ httptools==0.6.4 \ --hash=sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f \ --hash=sha256:fc411e1c0a7dcd2f902c7c48cf079947a7e65b5485dea9decb82b9105ca71a43 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # uvicorn httpx==0.28.1 \ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # fastapi # openai huggingface-hub==0.34.3 \ --hash=sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492 \ --hash=sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # tokenizers # transformers # vllm @@ -1079,7 +1079,7 @@ idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # anyio # email-validator # httpx @@ -1089,25 +1089,25 @@ imageio==2.34.2 \ --hash=sha256:5c0c0ee8faa018a1c42f649b90395dd4d3bb6187c09053a0cd6f1fdd51bbff5e \ --hash=sha256:a0bb27ec9d5bab36a9f4835e51b21d2cb099e1f78451441f94687ff3404b79f8 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # scikit-image importlib-metadata==6.11.0 \ --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # opentelemetry-api interegular==0.3.3 \ --hash=sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c \ --hash=sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # lm-format-enforcer jinja2==3.1.6 \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # fastapi # memray # torch @@ -1189,19 +1189,19 @@ jiter==0.8.2 \ --hash=sha256:fc9043259ee430ecd71d178fccabd8c332a3bf1e81e50cae43cc2b28d19e4cb7 \ --hash=sha256:ffd9fee7d0775ebaba131f7ca2e2d83839a62ad65e8e02fe2bd8fc975cedeb9e # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # openai jsonref==1.1.0 \ --hash=sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552 \ --hash=sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt jsonschema==4.23.0 \ --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt # mistral-common @@ -1210,25 +1210,25 @@ jsonschema-specifications==2024.10.1 \ --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # jsonschema kombu==5.5.4 \ --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ --hash=sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # scikit-image llguidance==0.7.26 ; platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:1895ff449c8ec0a5f1d3b142d723fc9b26a85b021b72d7f1173f8b7507f528c0 \ @@ -1240,7 +1240,7 @@ llguidance==0.7.26 ; platform_machine == 'aarch64' or platform_machine == 'arm64 --hash=sha256:e4e552eb3193b56ca3347f96c1382779e438b7dfc1d234323e202fd7c7a98d28 \ --hash=sha256:fa8ca0660df03934027b87d7e574edf1f8651493f77c0932f3f66d6effbed2b1 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm llvmlite==0.44.0 \ --hash=sha256:07667d66a5d150abed9157ab6c0b9393c9356f229784a4385c02f99e94fc94d4 \ @@ -1265,13 +1265,13 @@ llvmlite==0.44.0 \ --hash=sha256:eed7d5f29136bda63b6d7804c279e2b72e08c952b7c5df61f45db408e0ee52f3 \ --hash=sha256:f01a394e9c9b7b1d4e63c327b096d10f6f0ed149ef53d38a09b3749dcf8c9610 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # numba lm-format-enforcer==0.10.11 \ --hash=sha256:563e0dbc930a6d50fb687951506c5de098c6e962601be0ce723f3b7d0b916a1b \ --hash=sha256:8ab371924e166a1df68f243aca73a8a647bea5909f37edd6a53a694e7e7c3274 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm lz4==4.3.3 \ --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ @@ -1311,13 +1311,13 @@ lz4==4.3.3 \ --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt markdown-it-py==2.2.0 \ --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # rich markupsafe==2.1.3 \ --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ @@ -1381,13 +1381,13 @@ markupsafe==2.1.3 \ --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # jinja2 mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # markdown-it-py memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ @@ -1426,25 +1426,25 @@ memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt meson==1.8.3 \ --hash=sha256:ef02b806ce0c5b6becd5bb5dc9fa67662320b29b337e7ace73e4354500590233 \ --hash=sha256:f118aa910fc0a137cc2dd0122232dbf82153d9a12fb5b0f5bb64896f6a157abf # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt mistral-common==1.8.3 \ --hash=sha256:0d1979d82227b625f6d71b3c828176f059da8d0f5a3307cdf53b48409a3970a4 \ --hash=sha256:846b6e4bbe016dc2e64fd3169fa704a548f6c74467e0cb18dc165b7a7669abd6 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm mpmath==1.3.0 \ --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # sympy msgpack==1.0.7 \ --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ @@ -1504,7 +1504,7 @@ msgpack==1.0.7 \ --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # ray msgspec==0.19.0 \ @@ -1545,7 +1545,7 @@ msgspec==0.19.0 \ --hash=sha256:f98bd8962ad549c27d63845b50af3f53ec468b6318400c9f1adfe8b092d7b62f \ --hash=sha256:fe2c4bf29bf4e89790b3117470dea2c20b59932772483082c468b990d45fb947 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm multidict==6.0.5 \ --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ @@ -1639,14 +1639,14 @@ multidict==6.0.5 \ --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # aiohttp # yarl networkx==3.2.1 \ --hash=sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6 \ --hash=sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # scikit-image # torch ninja==1.11.1.3 \ @@ -1668,7 +1668,7 @@ ninja==1.11.1.3 \ --hash=sha256:bc3ebc8b2e47716149f3541742b5cd8e0b08f51013b825c05baca3e34854370d \ --hash=sha256:edfa0d2e9d7ead1635b03e40a32ad56cc8f56798b6e2e9848d8300b174897076 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt # vllm # xgrammar @@ -1682,7 +1682,7 @@ nixl==0.4.1 \ --hash=sha256:e33102b85b3f95a8c95e59b59b29aabd03d47b5bce619de506b9bb83739cf60d \ --hash=sha256:f16092dd445542e82e3db3553f6c7697ec5a2e837f19d416401283ae245826f9 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt numba==0.61.2 \ --hash=sha256:34fba9406078bac7ab052efbf0d13939426c753ad72946baaa5bf9ae0ebb8dd2 \ @@ -1707,7 +1707,7 @@ numba==0.61.2 \ --hash=sha256:ea0247617edcb5dd61f6106a56255baab031acc4257bddaeddb3a1003b4ca3fd \ --hash=sha256:efd3db391df53aaa5cfbee189b6c910a5b471488749fd6606c3f33fc984c2ae2 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm numpy==1.26.4 \ --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ @@ -1747,7 +1747,7 @@ numpy==1.26.4 \ --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # cupy-cuda12x # gguf @@ -1772,19 +1772,19 @@ openai==1.90.0 \ --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ --hash=sha256:e5dcb5498ea6b42fec47546d10f1bcc05fb854219a7d953a5ba766718b212a02 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt opencensus-context==0.1.3 \ --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # opencensus opencv-python-headless==4.11.0.86 \ --hash=sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b \ @@ -1795,14 +1795,14 @@ opencv-python-headless==4.11.0.86 \ --hash=sha256:a66c1b286a9de872c343ee7c3553b084244299714ebb50fbdcd76f07ebbe6c81 \ --hash=sha256:f447d8acbb0b6f2808da71fddd29c1cdd448d2bc98f72d9bb78a7a898fc9621b # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # mistral-common # vllm opentelemetry-api==1.34.1 \ --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # opentelemetry-sdk @@ -1811,26 +1811,26 @@ opentelemetry-exporter-prometheus==0.55b1 \ --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt opentelemetry-proto==1.27.0 \ --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt opentelemetry-sdk==1.34.1 \ --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus opentelemetry-semantic-conventions==0.55b1 \ --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # opentelemetry-sdk outlines-core==0.2.10 \ --hash=sha256:0a9e4b192ca837a472a1bb1428397509f543db08e1aeeee30252525cec34093a \ @@ -1875,13 +1875,13 @@ outlines-core==0.2.10 \ --hash=sha256:f895834da0a577120dcb8d979c12c0690fe912095413bf0070a73e9ff363b7bf \ --hash=sha256:faf5b43181b1d033871364e74e9d348362c6a77b1d054d7af35e09fdfcff5b16 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm packaging==23.0 \ --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # huggingface-hub # kombu @@ -1920,13 +1920,13 @@ pandas==1.5.3 \ --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt partial-json-parser==0.2.1.1.post5 \ --hash=sha256:627715aaa3cb3fb60a65b0d62223243acaa6c70846520a90326fef3a2f0b61ca \ --hash=sha256:992710ac67e90b367921d52727698928040f7713ba7ecb33b96371ea7aec82ca # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm pillow==10.3.0 \ --hash=sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c \ @@ -1999,7 +1999,7 @@ pillow==10.3.0 \ --hash=sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27 \ --hash=sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # imageio # mistral-common # scikit-image @@ -2009,13 +2009,13 @@ platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # virtualenv prometheus-client==0.19.0 \ --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # prometheus-fastapi-instrumentator @@ -2024,13 +2024,13 @@ prometheus-fastapi-instrumentator==7.0.2 \ --hash=sha256:8a4d8fb13dbe19d2882ac6af9ce236e4e1f98dc48e3fa44fe88d8e23ac3c953f \ --hash=sha256:975e39992acb7a112758ff13ba95317e6c54d1bbf605f9156f31ac9f2800c32d # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm prompt-toolkit==3.0.41 \ --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -2132,14 +2132,14 @@ propcache==0.3.0 \ --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # aiohttp # yarl proto-plus==1.22.3 \ --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # google-api-core protobuf==4.25.8 \ --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ @@ -2154,7 +2154,7 @@ protobuf==4.25.8 \ --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # google-api-core # googleapis-common-protos @@ -2181,13 +2181,13 @@ psutil==5.9.6 \ --hash=sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d \ --hash=sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm py-cpuinfo==9.0.0 \ --hash=sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690 \ --hash=sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:47cdda4c34d9b6cb01f3aaeceb2e88faf57da880207fe72ff6ff97e9bb6cc8a9 \ @@ -2199,7 +2199,7 @@ py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt pyarrow==19.0.1 \ --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ @@ -2245,20 +2245,20 @@ pyarrow==19.0.1 \ --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt pyasn1==0.5.1 \ --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # pyasn1-modules # rsa pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # google-auth pybase64==1.4.1 \ --hash=sha256:011a54ff6ca44c5d03746aec3f1f492fce3155bd3f943fb2ceaea92416d40eeb \ @@ -2405,31 +2405,31 @@ pybase64==1.4.1 \ --hash=sha256:fc9504c4c2e893e0a6c1cc80bce51907e3461288289f630eab22b5735eba1104 \ --hash=sha256:ff172a4dacbd964e5edcf1c2152dae157aabf856508aed15276f46d04a22128e # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm pybind11==2.13.6 \ --hash=sha256:237c41e29157b962835d356b370ededd57594a26d5894a795960f0047cb5caf5 \ --hash=sha256:ba6af10348c12b24e92fa086b39cfba0eff619b61ac77c406167d813b096d39a # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt pycountry==24.6.1 \ --hash=sha256:b61b3faccea67f87d10c1f2b0fc0be714409e8fcdcc1315613174f6466c10221 \ --hash=sha256:f1a4fb391cd7214f8eefd39556d740adcc233c778a27f8942c8dca351d6ce06f # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # pydantic-extra-types pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # cffi pydantic==2.10.0 \ --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # compressed-tensors # fastapi @@ -2541,56 +2541,56 @@ pydantic-core==2.27.0 \ --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # pydantic pydantic-extra-types==2.10.5 \ --hash=sha256:1dcfa2c0cf741a422f088e0dbb4690e7bfadaaf050da3d6f80d6c3cf58a2bad8 \ --hash=sha256:b60c4e23d573a69a4f1a16dd92888ecc0ef34fb0e655b4f305530377fa70e7a8 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # mistral-common pygments==2.18.0 \ --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # rich pyopenssl==25.0.0 \ --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # celery # pandas python-dotenv==1.0.1 \ --hash=sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca \ --hash=sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # uvicorn python-json-logger==2.0.7 \ --hash=sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c \ --hash=sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm python-multipart==0.0.20 \ --hash=sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104 \ --hash=sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # fastapi pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # pandas pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ @@ -2645,7 +2645,7 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # gguf # huggingface-hub @@ -2744,13 +2744,13 @@ pyzmq==26.0.3 \ --hash=sha256:f6b1d1c631e5940cac5a0b22c5379c86e8df6a4ec277c7a856b714021ab6cfad \ --hash=sha256:f6c21c00478a7bea93caaaef9e7629145d4153b15a8653e8bb4609d4bc70dbfc # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm referencing==0.36.2 \ --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # jsonschema # jsonschema-specifications regex==2024.11.6 \ @@ -2849,7 +2849,7 @@ regex==2024.11.6 \ --hash=sha256:fdd6028445d2460f33136c55eeb1f601ab06d74cb3347132e1c24250187500d9 \ --hash=sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # tiktoken # transformers # vllm @@ -2857,7 +2857,7 @@ requests==2.32.3 \ --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # google-api-core # huggingface-hub @@ -2870,7 +2870,7 @@ rich==13.3.2 \ --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # memray # typer @@ -2979,14 +2979,14 @@ rpds-py==0.22.3 \ --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # jsonschema # referencing rsa==4.7.2 \ --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # google-auth safetensors==0.5.2 \ --hash=sha256:03c937100f38c9ff4c1507abea9928a6a9b02c9c1c9c3609ed4fb2bf413d4975 \ @@ -3005,7 +3005,7 @@ safetensors==0.5.2 \ --hash=sha256:d3a06fae62418ec8e5c635b61a8086032c9e281f16c63c3af46a6efbab33156f \ --hash=sha256:fe55c039d97090d1f85277d402954dd6ad27f63034fa81985a9cc59655ac3ee2 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # transformers scikit-image==0.24.0 \ --hash=sha256:18836a18d3a7b6aca5376a2d805f0045826bc6c9fc85331659c33b4813e0b563 \ @@ -3030,7 +3030,7 @@ scikit-image==0.24.0 \ --hash=sha256:ef04360eda372ee5cd60aebe9be91258639c86ae2ea24093fb9182118008d009 \ --hash=sha256:fa27b3a0dbad807b966b8db2d78da734cb812ca4787f7fbb143764800ce2fa9c # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt scipy==1.11.4 \ --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ @@ -3059,7 +3059,7 @@ scipy==1.11.4 \ --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # scikit-image # vllm @@ -3118,7 +3118,7 @@ sentencepiece==0.2.0 \ --hash=sha256:fb89f811e5efd18bab141afc3fea3de141c3f69f3fe9e898f710ae7fe3aab251 \ --hash=sha256:ff88712338b01031910e8e61e7239aff3ce8869ee31a47df63cb38aadd591bea # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # gguf # mistral-common # vllm @@ -3126,26 +3126,26 @@ shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # typer six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # opencensus # python-dateutil smart-open==6.2.0 \ --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # anyio # openai soundfile==0.13.1 \ @@ -3158,7 +3158,7 @@ soundfile==0.13.1 \ --hash=sha256:b2c68dab1e30297317080a5b43df57e302584c49e2942defdde0acccc53f0e5b \ --hash=sha256:c734564fab7c5ddf8e9be5bf70bab68042cd17e9c214c06e365e20d64f9a69d5 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # mistral-common soxr==0.5.0.post1 \ --hash=sha256:39e0f791ba178d69cd676485dbee37e75a34f20daa478d90341ecb7f6d9d690f \ @@ -3183,13 +3183,13 @@ soxr==0.5.0.post1 \ --hash=sha256:fcc049b0a151a65aa75b92f0ac64bb2dba785d16b78c31c2b94e68c141751d6d \ --hash=sha256:fef509466c9c25f65eae0ce1e4b9ac9705d22c6038c914160ddaf459589c6e31 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # mistral-common starlette==0.46.2 \ --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # fastapi # prometheus-fastapi-instrumentator @@ -3197,19 +3197,19 @@ sympy==1.14.0 \ --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # torch tensorboardx==2.6.2.2 \ --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt tifffile==2024.7.21 \ --hash=sha256:7f335b5d6ca49401fe0f1d87deb206f5dae47297e47b1ed52a676d05d6d26798 \ --hash=sha256:818b577d49350421fb511f389f937984f9feaa2cd8177fa00823001920bf3483 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # scikit-image tiktoken==0.9.0 \ --hash=sha256:03935988a91d6d3216e2ec7c645afbb3d870b37bcb67ada1943ec48678e7ee33 \ @@ -3244,7 +3244,7 @@ tiktoken==0.9.0 \ --hash=sha256:f0968d5beeafbca2a72c595e8385a1a1f8af58feaebb02b227229b69ca5357fd \ --hash=sha256:f32cc56168eac4851109e9b5d327637f15fd662aa30dd79f964b7c39fbadd26e # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # mistral-common # vllm tokenizers==0.21.1 \ @@ -3264,7 +3264,7 @@ tokenizers==0.21.1 \ --hash=sha256:e78e413e9e668ad790a29456e677d9d3aa50a9ad311a40905d6861ba7692cf41 \ --hash=sha256:ed248ab5279e601a30a4d67bdb897ecbe955a50f1e7bb62bd99f07dd11c2f5b6 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # transformers # vllm torch==2.7.1+cpu \ @@ -3288,7 +3288,7 @@ torch==2.7.1+cpu \ --hash=sha256:d25435bdc4780d3cb512aad55142aca9584ae1fe8f8691cda6d32f19faf5d58e \ --hash=sha256:eb17646792ac4374ffc87e42369f45d21eff17c790868963b90483ef0b6db4ef # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # compressed-tensors # nixl # torchaudio @@ -3310,7 +3310,7 @@ torchaudio==2.7.1+cpu \ --hash=sha256:deb19d2a1cbbe49f9d14a9fe3dce65fef8dd98570aa8b6a65d7f5d1e0d16d0f3 \ --hash=sha256:e169a2b62e55342f2f30e17640054707c8e339045a1ccc2db33517e9debb2767 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm torchvision==0.22.1+cpu \ --hash=sha256:34c914ad4728b81848ac802c5fc5eeb8de8ff4058cc59c1463a74ce4f4fbf0d8 \ @@ -3326,13 +3326,13 @@ torchvision==0.22.1+cpu \ --hash=sha256:c852e61bc903351169017e2e96389f28f6cfb52ca7c3945acceb31e7fe1b21e6 \ --hash=sha256:e31f1273a8dd9760906288036ac3c8f5fef25eed393da0491db150d7be78910d # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm tqdm==4.67.1 \ --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # gguf # huggingface-hub # openai @@ -3342,7 +3342,7 @@ transformers==4.53.2 \ --hash=sha256:6c3ed95edfb1cba71c4245758f1b4878c93bf8cde77d076307dacb2cbbd72be2 \ --hash=sha256:db8f4819bb34f000029c73c3c557e7d06fc1b8e612ec142eecdae3947a9c78bf # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt # compressed-tensors # vllm @@ -3361,13 +3361,13 @@ triton==3.2.0 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:f1679fde231fb04c96cb5a01b160c8d0294ce6f7c122565d8b33ad8a910422d7 \ --hash=sha256:f24212d12744266f6229f90f820f34c43a538a69d6511b8e92ee392d2dc0d38b # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # xgrammar typer==0.12.3 \ --hash=sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914 \ --hash=sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt # fastapi-cli @@ -3375,7 +3375,7 @@ typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # fastapi # gymnasium # huggingface-hub @@ -3396,19 +3396,19 @@ tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # requests uvicorn==0.22.0 \ --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # fastapi # fastapi-cli @@ -3451,13 +3451,13 @@ uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'c --hash=sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816 \ --hash=sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # uvicorn vine==5.1.0 \ --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # amqp # celery # kombu @@ -3465,13 +3465,13 @@ virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt vllm==0.10.0 \ --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ --hash=sha256:a44e9013db26082a82c3931ed8772ac884d6d60566d36ecdb0e8dc01c65b241a # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt watchfiles==0.19.0 \ --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ @@ -3497,7 +3497,7 @@ watchfiles==0.19.0 \ --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt # uvicorn # vllm @@ -3505,7 +3505,7 @@ wcwidth==0.2.13 \ --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # prompt-toolkit websockets==15.0 \ --hash=sha256:0e389efe46ccb25a1f93d08c7a74e8123a2517f7b7458f043bd7529d1a63ffeb \ @@ -3578,14 +3578,14 @@ websockets==15.0 \ --hash=sha256:ffc02b159b65c05f2ed9ec176b715b66918a674bd4daed48a9a7a590dd4be1aa \ --hash=sha256:ffc5ae23ada6515f31604f700009e2df90b091b67d463a8401c1d8a37f76c1d7 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # uvicorn xformers==0.0.31 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:23331bdb9831ba0df96f55258537ca0df7ad888efc75cea97a0de79b5e2291c4 \ --hash=sha256:3fccb159c6327c13fc1b08f8b963c2779ca526e2e50755dee9bcc1bac67d20c6 \ --hash=sha256:50aedaea82a38d7d28631f77617d1ed1f6f37c60bdc4bf167a69cbc0e39cee76 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm xgrammar==0.1.21 ; platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:140628376fc701a535600dc64752603ddaed619461dc50669e90626e9f61b8aa \ @@ -3613,7 +3613,7 @@ xgrammar==0.1.21 ; platform_machine == 'aarch64' or platform_machine == 'arm64' --hash=sha256:f89d9ddb4d00fadcffa4bcabd0c3ae75d47c844c728bbb6be695056df3767524 \ --hash=sha256:f9247641c73eec6e972cec15156a8844957334204ba79ad1abdb0d7b03def8a1 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm yarl==1.18.3 \ --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ @@ -3699,13 +3699,13 @@ yarl==1.18.3 \ --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # aiohttp zipp==3.19.2 \ --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via - # -c python/requirements_compiled_rayllm_test_py311_cpu.txt + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # importlib-metadata # The following packages were excluded from the output: diff --git a/python/requirements_compiled_rayllm_py311_cu121.txt b/python/deplocks/llm/rayllm_py311_cu121.lock similarity index 95% rename from python/requirements_compiled_rayllm_py311_cu121.txt rename to python/deplocks/llm/rayllm_py311_cu121.lock index 4ad206d3186f..4f260cad18b3 100644 --- a/python/requirements_compiled_rayllm_py311_cu121.txt +++ b/python/deplocks/llm/rayllm_py311_cu121.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_rayllm_test_py311_cu121.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/rayllm_test_py311_cu121.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cu121.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 @@ -7,7 +7,7 @@ aiohappyeyeballs==2.6.1 \ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # aiohttp aiohttp==3.11.16 \ --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ @@ -92,7 +92,7 @@ aiohttp==3.11.16 \ --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # aiohttp-cors # vllm @@ -100,37 +100,37 @@ aiohttp-cors==0.7.0 \ --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt aiorwlock==1.3.0 \ --hash=sha256:45baf8e4fa9a23e0bb325fbd67da80de1fd7ae1d4f59a6381754c60cec7b289b \ --hash=sha256:83f12d87df4b9728a0b8fda1756585ab0d652b107bab59c6084e1b1ad692ab45 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # aiohttp amqp==5.3.1 \ --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # pydantic anyio==3.7.1 \ --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # httpx # openai # starlette @@ -139,13 +139,13 @@ astor==0.8.1 \ --hash=sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5 \ --hash=sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # depyf attrs==25.1.0 \ --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # aiohttp # jsonschema # referencing @@ -153,7 +153,7 @@ billiard==4.2.1 \ --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # celery blake3==1.0.4 \ --hash=sha256:00605aa59923205c6a4f21131840840eb2d9a754c59b163357d890566755b97a \ @@ -242,13 +242,13 @@ blake3==1.0.4 \ --hash=sha256:fedc326cac4476d2eab88413a4bf56e491040ae11ea98ddadaa5487cecda9b93 \ --hash=sha256:ff0e96f61b16b365ad5bb7c6272754f83d8a59c95d3b2f70c3bb6324ddf5bc0c # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # google-auth # vllm cbor2==5.6.5 \ @@ -297,19 +297,19 @@ cbor2==5.6.5 \ --hash=sha256:fde21ac1cf29336a31615a2c469a9cb03cf0add3ae480672d4d38cda467d07fc \ --hash=sha256:fe11c2eb518c882cfbeed456e7a552e544893c17db66fe5d3230dbeaca6b615c # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm celery==5.5.3 \ --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # httpcore # httpx # requests @@ -367,7 +367,7 @@ cffi==1.16.0 \ --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # cryptography # soundfile charset-normalizer==3.3.2 \ @@ -462,13 +462,13 @@ charset-normalizer==3.3.2 \ --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # requests click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # celery # click-didyoumean @@ -481,38 +481,38 @@ click-didyoumean==0.3.1 \ --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # celery click-plugins==1.1.1.2 \ --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # celery click-repl==0.3.0 \ --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # gymnasium # vllm colorful==0.5.5 \ --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt compressed-tensors==0.10.2 \ --hash=sha256:6de13ac535d7ffdd8890fad3d229444c33076170acaa8fab6bab8ecfa96c1d8f \ --hash=sha256:e1b4d9bc2006e3fd3a938e59085f318fdb280c5af64688a4792bf1bc263e579d # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm cryptography==44.0.3 \ --hash=sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259 \ @@ -553,7 +553,7 @@ cryptography==44.0.3 \ --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # pyopenssl cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:230f8a8e99c81a653baa0ed00819990c0ed1f0cf0298214786b5e323461dc61a \ @@ -569,38 +569,38 @@ cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # ray depyf==0.19.0 \ --hash=sha256:040b35fc0997d49df024b7d094f2a7836f91e9ed02f49982dd37e70aa3285ad5 \ --hash=sha256:afed0916b32d141cc90fa6220df01885eda442ca43b297d5050eeb90b4a5cb44 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm dill==0.3.9 \ --hash=sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a \ --hash=sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # depyf diskcache==5.6.3 \ --hash=sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc \ --hash=sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm distlib==0.3.7 \ --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # virtualenv distro==1.9.0 \ --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # openai dm-tree==0.1.8 \ --hash=sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6 \ @@ -650,44 +650,44 @@ dm-tree==0.1.8 \ --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt dnspython==2.7.0 \ --hash=sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86 \ --hash=sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # email-validator einops==0.8.1 \ --hash=sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737 \ --hash=sha256:de5d960a7a761225532e0f1959e5315ebeafc0cd43394732f103ca44b9837e84 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm email-validator==2.2.0 \ --hash=sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631 \ --hash=sha256:cb690f344c617a714f22e66ae771445a1ceb46821152df8e165c5f9a364582b7 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # fastapi farama-notifications==0.0.4 \ --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # gymnasium fastapi==0.115.12 \ --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # vllm fastapi-cli==0.0.5 \ --hash=sha256:d30e1239c6f46fcb95e606f02cdda59a1e2fa778a54b64686b3ff27f6211ff9f \ --hash=sha256:e94d847524648c748a5350673546bbf9bcaeb086b33c24f2e82e021436866a46 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # fastapi fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ @@ -766,13 +766,13 @@ fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # cupy-cuda12x filelock==3.17.0 \ --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # huggingface-hub # ray @@ -859,14 +859,14 @@ frozenlist==1.4.1 \ --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # aiohttp # aiosignal fsspec==2023.5.0 \ --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # huggingface-hub # torch @@ -874,25 +874,25 @@ gguf==0.16.2 \ --hash=sha256:0fc956289a30d0f1f3afd75ec0d493f73ae2629a3f21f3846dd1687d8791c7c1 \ --hash=sha256:e73eb19b30fcc7c7f32894345024dda8b1a0c959b94a12b7c40ded8dd3f96810 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm google-api-core==2.24.2 \ --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # opencensus google-auth==2.23.4 \ --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # google-api-core googleapis-common-protos==1.61.0 \ --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # google-api-core grpcio==1.66.2 \ --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ @@ -951,19 +951,19 @@ grpcio==1.66.2 \ --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt gymnasium==1.1.1 \ --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # httpcore # uvicorn hf-transfer==0.1.9 \ @@ -993,7 +993,7 @@ hf-transfer==0.1.9 \ --hash=sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f \ --hash=sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt hf-xet==1.1.5 \ --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 \ @@ -1005,13 +1005,13 @@ hf-xet==1.1.5 \ --hash=sha256:f52c2fa3635b8c37c7764d8796dfa72706cc4eded19d638331161e82b0792e23 \ --hash=sha256:fc874b5c843e642f45fd85cda1ce599e123308ad2901ead23d3510a47ff506d1 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # huggingface-hub httpcore==1.0.9 \ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # httpx httptools==0.6.4 \ --hash=sha256:0614154d5454c21b6410fdf5262b4a3ddb0f53f1e1721cfd59d55f32138c578a \ @@ -1058,20 +1058,20 @@ httptools==0.6.4 \ --hash=sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f \ --hash=sha256:fc411e1c0a7dcd2f902c7c48cf079947a7e65b5485dea9decb82b9105ca71a43 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # uvicorn httpx==0.28.1 \ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # fastapi # openai huggingface-hub==0.34.3 \ --hash=sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492 \ --hash=sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # tokenizers # transformers # vllm @@ -1079,7 +1079,7 @@ idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # anyio # email-validator # httpx @@ -1089,25 +1089,25 @@ imageio==2.34.2 \ --hash=sha256:5c0c0ee8faa018a1c42f649b90395dd4d3bb6187c09053a0cd6f1fdd51bbff5e \ --hash=sha256:a0bb27ec9d5bab36a9f4835e51b21d2cb099e1f78451441f94687ff3404b79f8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # scikit-image importlib-metadata==6.11.0 \ --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # opentelemetry-api interegular==0.3.3 \ --hash=sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c \ --hash=sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # lm-format-enforcer jinja2==3.1.6 \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # fastapi # memray # torch @@ -1189,19 +1189,19 @@ jiter==0.8.2 \ --hash=sha256:fc9043259ee430ecd71d178fccabd8c332a3bf1e81e50cae43cc2b28d19e4cb7 \ --hash=sha256:ffd9fee7d0775ebaba131f7ca2e2d83839a62ad65e8e02fe2bd8fc975cedeb9e # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # openai jsonref==1.1.0 \ --hash=sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552 \ --hash=sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt jsonschema==4.23.0 \ --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt # mistral-common @@ -1210,25 +1210,25 @@ jsonschema-specifications==2024.10.1 \ --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # jsonschema kombu==5.5.4 \ --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ --hash=sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # scikit-image llguidance==0.7.26 ; platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:1895ff449c8ec0a5f1d3b142d723fc9b26a85b021b72d7f1173f8b7507f528c0 \ @@ -1240,7 +1240,7 @@ llguidance==0.7.26 ; platform_machine == 'aarch64' or platform_machine == 'arm64 --hash=sha256:e4e552eb3193b56ca3347f96c1382779e438b7dfc1d234323e202fd7c7a98d28 \ --hash=sha256:fa8ca0660df03934027b87d7e574edf1f8651493f77c0932f3f66d6effbed2b1 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm llvmlite==0.44.0 \ --hash=sha256:07667d66a5d150abed9157ab6c0b9393c9356f229784a4385c02f99e94fc94d4 \ @@ -1265,13 +1265,13 @@ llvmlite==0.44.0 \ --hash=sha256:eed7d5f29136bda63b6d7804c279e2b72e08c952b7c5df61f45db408e0ee52f3 \ --hash=sha256:f01a394e9c9b7b1d4e63c327b096d10f6f0ed149ef53d38a09b3749dcf8c9610 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # numba lm-format-enforcer==0.10.11 \ --hash=sha256:563e0dbc930a6d50fb687951506c5de098c6e962601be0ce723f3b7d0b916a1b \ --hash=sha256:8ab371924e166a1df68f243aca73a8a647bea5909f37edd6a53a694e7e7c3274 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm lz4==4.3.3 \ --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ @@ -1311,13 +1311,13 @@ lz4==4.3.3 \ --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt markdown-it-py==2.2.0 \ --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # rich markupsafe==2.1.3 \ --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ @@ -1381,13 +1381,13 @@ markupsafe==2.1.3 \ --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # jinja2 mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # markdown-it-py memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ @@ -1426,25 +1426,25 @@ memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt meson==1.8.3 \ --hash=sha256:ef02b806ce0c5b6becd5bb5dc9fa67662320b29b337e7ace73e4354500590233 \ --hash=sha256:f118aa910fc0a137cc2dd0122232dbf82153d9a12fb5b0f5bb64896f6a157abf # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt mistral-common==1.8.3 \ --hash=sha256:0d1979d82227b625f6d71b3c828176f059da8d0f5a3307cdf53b48409a3970a4 \ --hash=sha256:846b6e4bbe016dc2e64fd3169fa704a548f6c74467e0cb18dc165b7a7669abd6 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm mpmath==1.3.0 \ --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # sympy msgpack==1.0.7 \ --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ @@ -1504,7 +1504,7 @@ msgpack==1.0.7 \ --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # ray msgspec==0.19.0 \ @@ -1545,7 +1545,7 @@ msgspec==0.19.0 \ --hash=sha256:f98bd8962ad549c27d63845b50af3f53ec468b6318400c9f1adfe8b092d7b62f \ --hash=sha256:fe2c4bf29bf4e89790b3117470dea2c20b59932772483082c468b990d45fb947 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm multidict==6.0.5 \ --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ @@ -1639,14 +1639,14 @@ multidict==6.0.5 \ --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # aiohttp # yarl networkx==3.2.1 \ --hash=sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6 \ --hash=sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # scikit-image # torch ninja==1.11.1.3 \ @@ -1668,7 +1668,7 @@ ninja==1.11.1.3 \ --hash=sha256:bc3ebc8b2e47716149f3541742b5cd8e0b08f51013b825c05baca3e34854370d \ --hash=sha256:edfa0d2e9d7ead1635b03e40a32ad56cc8f56798b6e2e9848d8300b174897076 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt # vllm # xgrammar @@ -1682,7 +1682,7 @@ nixl==0.4.1 \ --hash=sha256:e33102b85b3f95a8c95e59b59b29aabd03d47b5bce619de506b9bb83739cf60d \ --hash=sha256:f16092dd445542e82e3db3553f6c7697ec5a2e837f19d416401283ae245826f9 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt numba==0.61.2 \ --hash=sha256:34fba9406078bac7ab052efbf0d13939426c753ad72946baaa5bf9ae0ebb8dd2 \ @@ -1707,7 +1707,7 @@ numba==0.61.2 \ --hash=sha256:ea0247617edcb5dd61f6106a56255baab031acc4257bddaeddb3a1003b4ca3fd \ --hash=sha256:efd3db391df53aaa5cfbee189b6c910a5b471488749fd6606c3f33fc984c2ae2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm numpy==1.26.4 \ --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ @@ -1747,7 +1747,7 @@ numpy==1.26.4 \ --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # cupy-cuda12x # gguf @@ -1773,7 +1773,7 @@ nvidia-cublas-cu12==12.6.4.1 ; platform_machine == 'x86_64' and sys_platform == --hash=sha256:235f728d6e2a409eddf1df58d5b0921cf80cfa9e72b9f2775ccb7b4a87984668 \ --hash=sha256:9e4fa264f4d8a4eb0cdbd34beadc029f453b3bafae02401e999cf3d5a5af75f8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # nvidia-cudnn-cu12 # nvidia-cusolver-cu12 # torch @@ -1784,14 +1784,14 @@ nvidia-cuda-cupti-cu12==12.6.80 ; platform_machine == 'x86_64' and sys_platform --hash=sha256:a3eff6cdfcc6a4c35db968a06fcadb061cbc7d6dde548609a941ff8701b98b73 \ --hash=sha256:bbe6ae76e83ce5251b56e8c8e61a964f757175682bbad058b170b136266ab00a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch nvidia-cuda-nvrtc-cu12==12.6.77 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:35b0cc6ee3a9636d5409133e79273ce1f3fd087abb0532d2d2e8fff1fe9efc53 \ --hash=sha256:5847f1d6e5b757f1d2b3991a01082a44aad6f10ab3c5c0213fa3e25bddc25a13 \ --hash=sha256:f7007dbd914c56bd80ea31bc43e8e149da38f68158f423ba845fc3292684e45a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch nvidia-cuda-runtime-cu12==12.6.77 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:6116fad3e049e04791c0256a9778c16237837c08b27ed8c8401e2e45de8d60cd \ @@ -1800,14 +1800,14 @@ nvidia-cuda-runtime-cu12==12.6.77 ; platform_machine == 'x86_64' and sys_platfor --hash=sha256:ba3b56a4f896141e25e19ab287cd71e52a6a0f4b29d0d31609f60e3b4d5219b7 \ --hash=sha256:d461264ecb429c84c8879a7153499ddc7b19b5f8d84c204307491989a365588e # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch nvidia-cudnn-cu12==9.5.1.17 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2 \ --hash=sha256:9fd4584468533c61873e5fda8ca41bac3a38bcb2d12350830c69b0a96a7e4def \ --hash=sha256:d7af0f8a4f3b4b9dbb3122f2ef553b45694ed9c384d5a75bab197b8eefb79ab8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch nvidia-cufft-cu12==11.3.0.4 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:6048ebddfb90d09d2707efb1fd78d4e3a77cb3ae4dc60e19aab6be0ece2ae464 \ @@ -1816,13 +1816,13 @@ nvidia-cufft-cu12==11.3.0.4 ; platform_machine == 'x86_64' and sys_platform == ' --hash=sha256:ccba62eb9cef5559abd5e0d54ceed2d9934030f51163df018532142a8ec533e5 \ --hash=sha256:d16079550df460376455cba121db6564089176d9bac9e4f360493ca4741b22a6 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch nvidia-cufile-cu12==1.11.1.6 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:8f57a0051dcf2543f6dc2b98a98cb2719c37d3cee1baba8965d57f3bbc90d4db \ --hash=sha256:cc23469d1c7e52ce6c1d55253273d32c565dd22068647f3aa59b3c6b005bf159 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch nvidia-curand-cu12==10.3.7.77 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:6d6d935ffba0f3d439b7cd968192ff068fafd9018dbf1b85b37261b13cfc9905 \ @@ -1831,7 +1831,7 @@ nvidia-curand-cu12==10.3.7.77 ; platform_machine == 'x86_64' and sys_platform == --hash=sha256:99f1a32f1ac2bd134897fc7a203f779303261268a65762a623bf30cc9fe79117 \ --hash=sha256:a42cd1344297f70b9e39a1e4f467a4e1c10f1da54ff7a85c12197f6c652c8bdf # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch nvidia-cusolver-cu12==11.7.1.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:0ce237ef60acde1efc457335a2ddadfd7610b892d94efee7b776c64bb1cac9e0 \ @@ -1840,7 +1840,7 @@ nvidia-cusolver-cu12==11.7.1.2 ; platform_machine == 'x86_64' and sys_platform = --hash=sha256:dbbe4fc38ec1289c7e5230e16248365e375c3673c9c8bac5796e2e20db07f56e \ --hash=sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch nvidia-cusparse-cu12==12.5.4.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:23749a6571191a215cb74d1cdbff4a86e7b19f1200c071b3fcf844a5bea23a2f \ @@ -1849,7 +1849,7 @@ nvidia-cusparse-cu12==12.5.4.2 ; platform_machine == 'x86_64' and sys_platform = --hash=sha256:7aa32fa5470cf754f72d1116c7cbc300b4e638d3ae5304cfa4a638a5b87161b1 \ --hash=sha256:d25b62fb18751758fe3c93a4a08eff08effedfe4edf1c6bb5afd0890fe88f887 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # nvidia-cusolver-cu12 # torch nvidia-cusparselt-cu12==0.6.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ @@ -1857,20 +1857,20 @@ nvidia-cusparselt-cu12==0.6.3 ; platform_machine == 'x86_64' and sys_platform == --hash=sha256:8371549623ba601a06322af2133c4a44350575f5a3108fb75f3ef20b822ad5f1 \ --hash=sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch nvidia-nccl-cu12==2.26.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:5c196e95e832ad30fbbb50381eb3cbd1fadd5675e587a548563993609af19522 \ --hash=sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch nvidia-nvjitlink-cu12==12.6.85 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:cf4eaa7d4b6b543ffd69d6abfb11efdeb2db48270d94dfd3a452c24150829e41 \ --hash=sha256:e61120e52ed675747825cdd16febc6a0730537451d867ee58bee3853b1b13d1c \ --hash=sha256:eedc36df9e88b682efe4309aa16b5b4e78c2407eac59e8c10a6a47535164369a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # nvidia-cufft-cu12 # nvidia-cusolver-cu12 # nvidia-cusparse-cu12 @@ -1882,25 +1882,25 @@ nvidia-nvtx-cu12==12.6.77 ; platform_machine == 'x86_64' and sys_platform == 'li --hash=sha256:b90bed3df379fa79afbd21be8e04a0314336b8ae16768b58f2d34cb1d04cd7d2 \ --hash=sha256:f44f8d86bb7d5629988d61c8d3ae61dddb2015dee142740536bc7481b022fe4b # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch openai==1.90.0 \ --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ --hash=sha256:e5dcb5498ea6b42fec47546d10f1bcc05fb854219a7d953a5ba766718b212a02 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt opencensus-context==0.1.3 \ --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # opencensus opencv-python-headless==4.11.0.86 \ --hash=sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b \ @@ -1911,14 +1911,14 @@ opencv-python-headless==4.11.0.86 \ --hash=sha256:a66c1b286a9de872c343ee7c3553b084244299714ebb50fbdcd76f07ebbe6c81 \ --hash=sha256:f447d8acbb0b6f2808da71fddd29c1cdd448d2bc98f72d9bb78a7a898fc9621b # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # mistral-common # vllm opentelemetry-api==1.34.1 \ --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # opentelemetry-sdk @@ -1927,26 +1927,26 @@ opentelemetry-exporter-prometheus==0.55b1 \ --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt opentelemetry-proto==1.27.0 \ --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt opentelemetry-sdk==1.34.1 \ --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus opentelemetry-semantic-conventions==0.55b1 \ --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # opentelemetry-sdk outlines-core==0.2.10 \ --hash=sha256:0a9e4b192ca837a472a1bb1428397509f543db08e1aeeee30252525cec34093a \ @@ -1991,13 +1991,13 @@ outlines-core==0.2.10 \ --hash=sha256:f895834da0a577120dcb8d979c12c0690fe912095413bf0070a73e9ff363b7bf \ --hash=sha256:faf5b43181b1d033871364e74e9d348362c6a77b1d054d7af35e09fdfcff5b16 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm packaging==23.0 \ --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # huggingface-hub # kombu @@ -2036,13 +2036,13 @@ pandas==1.5.3 \ --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt partial-json-parser==0.2.1.1.post5 \ --hash=sha256:627715aaa3cb3fb60a65b0d62223243acaa6c70846520a90326fef3a2f0b61ca \ --hash=sha256:992710ac67e90b367921d52727698928040f7713ba7ecb33b96371ea7aec82ca # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm pillow==10.3.0 \ --hash=sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c \ @@ -2115,7 +2115,7 @@ pillow==10.3.0 \ --hash=sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27 \ --hash=sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # imageio # mistral-common # scikit-image @@ -2125,13 +2125,13 @@ platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # virtualenv prometheus-client==0.19.0 \ --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # prometheus-fastapi-instrumentator @@ -2140,13 +2140,13 @@ prometheus-fastapi-instrumentator==7.0.2 \ --hash=sha256:8a4d8fb13dbe19d2882ac6af9ce236e4e1f98dc48e3fa44fe88d8e23ac3c953f \ --hash=sha256:975e39992acb7a112758ff13ba95317e6c54d1bbf605f9156f31ac9f2800c32d # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm prompt-toolkit==3.0.41 \ --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -2248,14 +2248,14 @@ propcache==0.3.0 \ --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # aiohttp # yarl proto-plus==1.22.3 \ --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # google-api-core protobuf==4.25.8 \ --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ @@ -2270,7 +2270,7 @@ protobuf==4.25.8 \ --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # google-api-core # googleapis-common-protos @@ -2297,13 +2297,13 @@ psutil==5.9.6 \ --hash=sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d \ --hash=sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm py-cpuinfo==9.0.0 \ --hash=sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690 \ --hash=sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:47cdda4c34d9b6cb01f3aaeceb2e88faf57da880207fe72ff6ff97e9bb6cc8a9 \ @@ -2315,7 +2315,7 @@ py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt pyarrow==19.0.1 \ --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ @@ -2361,20 +2361,20 @@ pyarrow==19.0.1 \ --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt pyasn1==0.5.1 \ --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # pyasn1-modules # rsa pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # google-auth pybase64==1.4.1 \ --hash=sha256:011a54ff6ca44c5d03746aec3f1f492fce3155bd3f943fb2ceaea92416d40eeb \ @@ -2521,31 +2521,31 @@ pybase64==1.4.1 \ --hash=sha256:fc9504c4c2e893e0a6c1cc80bce51907e3461288289f630eab22b5735eba1104 \ --hash=sha256:ff172a4dacbd964e5edcf1c2152dae157aabf856508aed15276f46d04a22128e # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm pybind11==2.13.6 \ --hash=sha256:237c41e29157b962835d356b370ededd57594a26d5894a795960f0047cb5caf5 \ --hash=sha256:ba6af10348c12b24e92fa086b39cfba0eff619b61ac77c406167d813b096d39a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt pycountry==24.6.1 \ --hash=sha256:b61b3faccea67f87d10c1f2b0fc0be714409e8fcdcc1315613174f6466c10221 \ --hash=sha256:f1a4fb391cd7214f8eefd39556d740adcc233c778a27f8942c8dca351d6ce06f # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # pydantic-extra-types pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # cffi pydantic==2.10.0 \ --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # compressed-tensors # fastapi @@ -2657,56 +2657,56 @@ pydantic-core==2.27.0 \ --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # pydantic pydantic-extra-types==2.10.5 \ --hash=sha256:1dcfa2c0cf741a422f088e0dbb4690e7bfadaaf050da3d6f80d6c3cf58a2bad8 \ --hash=sha256:b60c4e23d573a69a4f1a16dd92888ecc0ef34fb0e655b4f305530377fa70e7a8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # mistral-common pygments==2.18.0 \ --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # rich pyopenssl==25.0.0 \ --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # celery # pandas python-dotenv==1.0.1 \ --hash=sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca \ --hash=sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # uvicorn python-json-logger==2.0.7 \ --hash=sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c \ --hash=sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm python-multipart==0.0.20 \ --hash=sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104 \ --hash=sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # fastapi pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # pandas pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ @@ -2761,7 +2761,7 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # gguf # huggingface-hub @@ -2860,13 +2860,13 @@ pyzmq==26.0.3 \ --hash=sha256:f6b1d1c631e5940cac5a0b22c5379c86e8df6a4ec277c7a856b714021ab6cfad \ --hash=sha256:f6c21c00478a7bea93caaaef9e7629145d4153b15a8653e8bb4609d4bc70dbfc # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm referencing==0.36.2 \ --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # jsonschema # jsonschema-specifications regex==2024.11.6 \ @@ -2965,7 +2965,7 @@ regex==2024.11.6 \ --hash=sha256:fdd6028445d2460f33136c55eeb1f601ab06d74cb3347132e1c24250187500d9 \ --hash=sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # tiktoken # transformers # vllm @@ -2973,7 +2973,7 @@ requests==2.32.3 \ --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # google-api-core # huggingface-hub @@ -2986,7 +2986,7 @@ rich==13.3.2 \ --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # memray # typer @@ -3095,14 +3095,14 @@ rpds-py==0.22.3 \ --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # jsonschema # referencing rsa==4.7.2 \ --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # google-auth safetensors==0.5.2 \ --hash=sha256:03c937100f38c9ff4c1507abea9928a6a9b02c9c1c9c3609ed4fb2bf413d4975 \ @@ -3121,7 +3121,7 @@ safetensors==0.5.2 \ --hash=sha256:d3a06fae62418ec8e5c635b61a8086032c9e281f16c63c3af46a6efbab33156f \ --hash=sha256:fe55c039d97090d1f85277d402954dd6ad27f63034fa81985a9cc59655ac3ee2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # transformers scikit-image==0.24.0 \ --hash=sha256:18836a18d3a7b6aca5376a2d805f0045826bc6c9fc85331659c33b4813e0b563 \ @@ -3146,7 +3146,7 @@ scikit-image==0.24.0 \ --hash=sha256:ef04360eda372ee5cd60aebe9be91258639c86ae2ea24093fb9182118008d009 \ --hash=sha256:fa27b3a0dbad807b966b8db2d78da734cb812ca4787f7fbb143764800ce2fa9c # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt scipy==1.11.4 \ --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ @@ -3175,7 +3175,7 @@ scipy==1.11.4 \ --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # scikit-image # vllm @@ -3234,7 +3234,7 @@ sentencepiece==0.2.0 \ --hash=sha256:fb89f811e5efd18bab141afc3fea3de141c3f69f3fe9e898f710ae7fe3aab251 \ --hash=sha256:ff88712338b01031910e8e61e7239aff3ce8869ee31a47df63cb38aadd591bea # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # gguf # mistral-common # vllm @@ -3242,26 +3242,26 @@ shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # typer six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # opencensus # python-dateutil smart-open==6.2.0 \ --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # anyio # openai soundfile==0.13.1 \ @@ -3274,7 +3274,7 @@ soundfile==0.13.1 \ --hash=sha256:b2c68dab1e30297317080a5b43df57e302584c49e2942defdde0acccc53f0e5b \ --hash=sha256:c734564fab7c5ddf8e9be5bf70bab68042cd17e9c214c06e365e20d64f9a69d5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # mistral-common soxr==0.5.0.post1 \ --hash=sha256:39e0f791ba178d69cd676485dbee37e75a34f20daa478d90341ecb7f6d9d690f \ @@ -3299,13 +3299,13 @@ soxr==0.5.0.post1 \ --hash=sha256:fcc049b0a151a65aa75b92f0ac64bb2dba785d16b78c31c2b94e68c141751d6d \ --hash=sha256:fef509466c9c25f65eae0ce1e4b9ac9705d22c6038c914160ddaf459589c6e31 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # mistral-common starlette==0.46.2 \ --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # fastapi # prometheus-fastapi-instrumentator @@ -3313,19 +3313,19 @@ sympy==1.14.0 \ --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch tensorboardx==2.6.2.2 \ --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt tifffile==2024.7.21 \ --hash=sha256:7f335b5d6ca49401fe0f1d87deb206f5dae47297e47b1ed52a676d05d6d26798 \ --hash=sha256:818b577d49350421fb511f389f937984f9feaa2cd8177fa00823001920bf3483 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # scikit-image tiktoken==0.9.0 \ --hash=sha256:03935988a91d6d3216e2ec7c645afbb3d870b37bcb67ada1943ec48678e7ee33 \ @@ -3360,7 +3360,7 @@ tiktoken==0.9.0 \ --hash=sha256:f0968d5beeafbca2a72c595e8385a1a1f8af58feaebb02b227229b69ca5357fd \ --hash=sha256:f32cc56168eac4851109e9b5d327637f15fd662aa30dd79f964b7c39fbadd26e # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # mistral-common # vllm tokenizers==0.21.1 \ @@ -3380,7 +3380,7 @@ tokenizers==0.21.1 \ --hash=sha256:e78e413e9e668ad790a29456e677d9d3aa50a9ad311a40905d6861ba7692cf41 \ --hash=sha256:ed248ab5279e601a30a4d67bdb897ecbe955a50f1e7bb62bd99f07dd11c2f5b6 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # transformers # vllm torch==2.7.1 \ @@ -3409,7 +3409,7 @@ torch==2.7.1 \ --hash=sha256:e0d81e9a12764b6f3879a866607c8ae93113cbcad57ce01ebde63eb48a576369 \ --hash=sha256:fe955951bdf32d182ee8ead6c3186ad54781492bf03d547d31771a01b3d6fb7d # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # compressed-tensors # nixl # torchaudio @@ -3443,7 +3443,7 @@ torchaudio==2.7.1 \ --hash=sha256:edb4deaa6f95acd5522912ed643303d0b86d79a6f15914362f5a5d49baaf5d13 \ --hash=sha256:f8bd69354a397753b9dea9699d9e1251f8496fbbdf3028c7086a57a615bf33c3 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm torchvision==0.22.1 \ --hash=sha256:043d9e35ed69c2e586aff6eb9e2887382e7863707115668ac9d140da58f42cba \ @@ -3471,13 +3471,13 @@ torchvision==0.22.1 \ --hash=sha256:ef46e065502f7300ad6abc98554131c35dc4c837b978d91306658f1a65c00baa \ --hash=sha256:ef7dee376f42900c0e7b0e34624f391d9ece70ab90ee74b42de0c1fffe371284 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm tqdm==4.67.1 \ --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # gguf # huggingface-hub # openai @@ -3487,7 +3487,7 @@ transformers==4.53.2 \ --hash=sha256:6c3ed95edfb1cba71c4245758f1b4878c93bf8cde77d076307dacb2cbbd72be2 \ --hash=sha256:db8f4819bb34f000029c73c3c557e7d06fc1b8e612ec142eecdae3947a9c78bf # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt # compressed-tensors # vllm @@ -3495,14 +3495,14 @@ transformers==4.53.2 \ triton==3.3.1 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:b31e3aa26f8cb3cc5bf4e187bf737cbacf17311e1112b781d4a059353dfd731b # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch # xgrammar typer==0.12.3 \ --hash=sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914 \ --hash=sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt # fastapi-cli @@ -3510,7 +3510,7 @@ typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # fastapi # gymnasium # huggingface-hub @@ -3531,19 +3531,19 @@ tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # requests uvicorn==0.22.0 \ --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # fastapi # fastapi-cli @@ -3586,13 +3586,13 @@ uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'c --hash=sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816 \ --hash=sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # uvicorn vine==5.1.0 \ --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # amqp # celery # kombu @@ -3600,13 +3600,13 @@ virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt vllm==0.10.0 \ --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ --hash=sha256:a44e9013db26082a82c3931ed8772ac884d6d60566d36ecdb0e8dc01c65b241a # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt watchfiles==0.19.0 \ --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ @@ -3632,7 +3632,7 @@ watchfiles==0.19.0 \ --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt # uvicorn # vllm @@ -3640,7 +3640,7 @@ wcwidth==0.2.13 \ --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # prompt-toolkit websockets==15.0 \ --hash=sha256:0e389efe46ccb25a1f93d08c7a74e8123a2517f7b7458f043bd7529d1a63ffeb \ @@ -3713,14 +3713,14 @@ websockets==15.0 \ --hash=sha256:ffc02b159b65c05f2ed9ec176b715b66918a674bd4daed48a9a7a590dd4be1aa \ --hash=sha256:ffc5ae23ada6515f31604f700009e2df90b091b67d463a8401c1d8a37f76c1d7 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # uvicorn xformers==0.0.31 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:23331bdb9831ba0df96f55258537ca0df7ad888efc75cea97a0de79b5e2291c4 \ --hash=sha256:3fccb159c6327c13fc1b08f8b963c2779ca526e2e50755dee9bcc1bac67d20c6 \ --hash=sha256:50aedaea82a38d7d28631f77617d1ed1f6f37c60bdc4bf167a69cbc0e39cee76 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm xgrammar==0.1.21 ; platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:140628376fc701a535600dc64752603ddaed619461dc50669e90626e9f61b8aa \ @@ -3748,7 +3748,7 @@ xgrammar==0.1.21 ; platform_machine == 'aarch64' or platform_machine == 'arm64' --hash=sha256:f89d9ddb4d00fadcffa4bcabd0c3ae75d47c844c728bbb6be695056df3767524 \ --hash=sha256:f9247641c73eec6e972cec15156a8844957334204ba79ad1abdb0d7b03def8a1 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm yarl==1.18.3 \ --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ @@ -3834,13 +3834,13 @@ yarl==1.18.3 \ --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # aiohttp zipp==3.19.2 \ --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via - # -c python/requirements_compiled_rayllm_test_py311_cu121.txt + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # importlib-metadata # The following packages were excluded from the output: diff --git a/python/requirements_compiled_rayllm_py311_cu128.txt b/python/deplocks/llm/rayllm_py311_cu128.lock similarity index 95% rename from python/requirements_compiled_rayllm_py311_cu128.txt rename to python/deplocks/llm/rayllm_py311_cu128.lock index 47338e202f4b..48efbd5be7dd 100644 --- a/python/requirements_compiled_rayllm_py311_cu128.txt +++ b/python/deplocks/llm/rayllm_py311_cu128.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_rayllm_test_py311_cu128.txt python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/requirements_compiled_rayllm_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/rayllm_test_py311_cu128.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cu128.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 @@ -7,7 +7,7 @@ aiohappyeyeballs==2.6.1 \ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # aiohttp aiohttp==3.11.16 \ --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ @@ -92,7 +92,7 @@ aiohttp==3.11.16 \ --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # aiohttp-cors # vllm @@ -100,37 +100,37 @@ aiohttp-cors==0.7.0 \ --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt aiorwlock==1.3.0 \ --hash=sha256:45baf8e4fa9a23e0bb325fbd67da80de1fd7ae1d4f59a6381754c60cec7b289b \ --hash=sha256:83f12d87df4b9728a0b8fda1756585ab0d652b107bab59c6084e1b1ad692ab45 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # aiohttp amqp==5.3.1 \ --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # pydantic anyio==3.7.1 \ --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # httpx # openai # starlette @@ -139,13 +139,13 @@ astor==0.8.1 \ --hash=sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5 \ --hash=sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # depyf attrs==25.1.0 \ --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # aiohttp # jsonschema # referencing @@ -153,7 +153,7 @@ billiard==4.2.1 \ --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # celery blake3==1.0.5 \ --hash=sha256:03638a6dc8546365c3576fdb293fb2c53b898ac80525b5742d9cf00b4f44dea5 \ @@ -242,13 +242,13 @@ blake3==1.0.5 \ --hash=sha256:fe333852c5bbafd7735d36da2d60d44a022247bd180f2c43facb2585134c1792 \ --hash=sha256:feb0d1558d720a476f888566ddf2faf91d9147ada9261f3ccf11400ca3798661 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # google-auth # vllm cbor2==5.6.5 \ @@ -297,19 +297,19 @@ cbor2==5.6.5 \ --hash=sha256:fde21ac1cf29336a31615a2c469a9cb03cf0add3ae480672d4d38cda467d07fc \ --hash=sha256:fe11c2eb518c882cfbeed456e7a552e544893c17db66fe5d3230dbeaca6b615c # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm celery==5.5.3 \ --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # httpcore # httpx # requests @@ -367,7 +367,7 @@ cffi==1.16.0 \ --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # cryptography # soundfile charset-normalizer==3.3.2 \ @@ -462,13 +462,13 @@ charset-normalizer==3.3.2 \ --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # requests click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # celery # click-didyoumean @@ -481,38 +481,38 @@ click-didyoumean==0.3.1 \ --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # celery click-plugins==1.1.1.2 \ --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # celery click-repl==0.3.0 \ --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # gymnasium # vllm colorful==0.5.5 \ --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt compressed-tensors==0.10.2 \ --hash=sha256:6de13ac535d7ffdd8890fad3d229444c33076170acaa8fab6bab8ecfa96c1d8f \ --hash=sha256:e1b4d9bc2006e3fd3a938e59085f318fdb280c5af64688a4792bf1bc263e579d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm cryptography==44.0.3 \ --hash=sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259 \ @@ -553,7 +553,7 @@ cryptography==44.0.3 \ --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # pyopenssl cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:230f8a8e99c81a653baa0ed00819990c0ed1f0cf0298214786b5e323461dc61a \ @@ -569,38 +569,38 @@ cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # ray depyf==0.19.0 \ --hash=sha256:040b35fc0997d49df024b7d094f2a7836f91e9ed02f49982dd37e70aa3285ad5 \ --hash=sha256:afed0916b32d141cc90fa6220df01885eda442ca43b297d5050eeb90b4a5cb44 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm dill==0.4.0 \ --hash=sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0 \ --hash=sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # depyf diskcache==5.6.3 \ --hash=sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc \ --hash=sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm distlib==0.3.7 \ --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # virtualenv distro==1.9.0 \ --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # openai dm-tree==0.1.8 \ --hash=sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6 \ @@ -650,44 +650,44 @@ dm-tree==0.1.8 \ --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt dnspython==2.7.0 \ --hash=sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86 \ --hash=sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # email-validator einops==0.8.1 \ --hash=sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737 \ --hash=sha256:de5d960a7a761225532e0f1959e5315ebeafc0cd43394732f103ca44b9837e84 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm email-validator==2.2.0 \ --hash=sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631 \ --hash=sha256:cb690f344c617a714f22e66ae771445a1ceb46821152df8e165c5f9a364582b7 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # fastapi farama-notifications==0.0.4 \ --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # gymnasium fastapi==0.115.12 \ --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # vllm fastapi-cli==0.0.5 \ --hash=sha256:d30e1239c6f46fcb95e606f02cdda59a1e2fa778a54b64686b3ff27f6211ff9f \ --hash=sha256:e94d847524648c748a5350673546bbf9bcaeb086b33c24f2e82e021436866a46 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # fastapi fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ @@ -766,13 +766,13 @@ fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # cupy-cuda12x filelock==3.17.0 \ --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # huggingface-hub # ray @@ -859,14 +859,14 @@ frozenlist==1.4.1 \ --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # aiohttp # aiosignal fsspec==2023.5.0 \ --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # huggingface-hub # torch @@ -874,25 +874,25 @@ gguf==0.17.0 \ --hash=sha256:52f2759c6e0ab3d228d4d44f871e3eb140004712c31aed72e2ae82f61aa5aa05 \ --hash=sha256:e3f88278e6f6778e0348fbc97313a4a2f8af63b08fe25dc381251d9c611dae03 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm google-api-core==2.24.2 \ --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # opencensus google-auth==2.23.4 \ --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # google-api-core googleapis-common-protos==1.61.0 \ --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # google-api-core grpcio==1.66.2 \ --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ @@ -951,19 +951,19 @@ grpcio==1.66.2 \ --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt gymnasium==1.1.1 \ --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # httpcore # uvicorn hf-transfer==0.1.9 \ @@ -993,7 +993,7 @@ hf-transfer==0.1.9 \ --hash=sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f \ --hash=sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt hf-xet==1.1.3 \ --hash=sha256:30c575a5306f8e6fda37edb866762140a435037365eba7a17ce7bd0bc0216a8b \ @@ -1005,13 +1005,13 @@ hf-xet==1.1.3 \ --hash=sha256:c3b508b5f583a75641aebf732853deb058953370ce8184f5dabc49f803b0819b \ --hash=sha256:fd2da210856444a34aad8ada2fc12f70dabed7cc20f37e90754d1d9b43bc0534 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # huggingface-hub httpcore==1.0.9 \ --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # httpx httptools==0.6.4 \ --hash=sha256:0614154d5454c21b6410fdf5262b4a3ddb0f53f1e1721cfd59d55f32138c578a \ @@ -1058,20 +1058,20 @@ httptools==0.6.4 \ --hash=sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f \ --hash=sha256:fc411e1c0a7dcd2f902c7c48cf079947a7e65b5485dea9decb82b9105ca71a43 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # uvicorn httpx==0.28.1 \ --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # fastapi # openai huggingface-hub==0.34.3 \ --hash=sha256:5444550099e2d86e68b2898b09e85878fbd788fc2957b506c6a79ce060e39492 \ --hash=sha256:d58130fd5aa7408480681475491c0abd7e835442082fbc3ef4d45b6c39f83853 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # tokenizers # transformers # vllm @@ -1079,7 +1079,7 @@ idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # anyio # email-validator # httpx @@ -1089,25 +1089,25 @@ imageio==2.34.2 \ --hash=sha256:5c0c0ee8faa018a1c42f649b90395dd4d3bb6187c09053a0cd6f1fdd51bbff5e \ --hash=sha256:a0bb27ec9d5bab36a9f4835e51b21d2cb099e1f78451441f94687ff3404b79f8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # scikit-image importlib-metadata==6.11.0 \ --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # opentelemetry-api interegular==0.3.3 \ --hash=sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c \ --hash=sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # lm-format-enforcer jinja2==3.1.6 \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # fastapi # memray # torch @@ -1190,19 +1190,19 @@ jiter==0.10.0 \ --hash=sha256:fcedb049bdfc555e261d6f65a6abe1d5ad68825b7202ccb9692636c70fcced86 \ --hash=sha256:ff76d8887c8c8ee1e772274fcf8cc1071c2c58590d13e33bd12d02dc9a560397 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # openai jsonref==1.1.0 \ --hash=sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552 \ --hash=sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt jsonschema==4.23.0 \ --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt # mistral-common @@ -1211,25 +1211,25 @@ jsonschema-specifications==2024.10.1 \ --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # jsonschema kombu==5.5.4 \ --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ --hash=sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # scikit-image llguidance==0.7.29 ; platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:17fd439957d6ca5f459d0dec755a2d040c2dc946ed7e3c332b469ef6861292f8 \ @@ -1241,7 +1241,7 @@ llguidance==0.7.29 ; platform_machine == 'aarch64' or platform_machine == 'arm64 --hash=sha256:c97f16ddd6be28f4d176eaaa493102b981ba5470299253903de9a764e2501ef3 \ --hash=sha256:d1aa68a54f9496d36750018e7edad3bf624ee2fbcf671a7483883790d798c4fe # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm llvmlite==0.44.0 \ --hash=sha256:07667d66a5d150abed9157ab6c0b9393c9356f229784a4385c02f99e94fc94d4 \ @@ -1266,13 +1266,13 @@ llvmlite==0.44.0 \ --hash=sha256:eed7d5f29136bda63b6d7804c279e2b72e08c952b7c5df61f45db408e0ee52f3 \ --hash=sha256:f01a394e9c9b7b1d4e63c327b096d10f6f0ed149ef53d38a09b3749dcf8c9610 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # numba lm-format-enforcer==0.10.11 \ --hash=sha256:563e0dbc930a6d50fb687951506c5de098c6e962601be0ce723f3b7d0b916a1b \ --hash=sha256:8ab371924e166a1df68f243aca73a8a647bea5909f37edd6a53a694e7e7c3274 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm lz4==4.3.3 \ --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ @@ -1312,13 +1312,13 @@ lz4==4.3.3 \ --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt markdown-it-py==2.2.0 \ --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # rich markupsafe==2.1.3 \ --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ @@ -1347,13 +1347,13 @@ markupsafe==2.1.3 \ --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # jinja2 mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # markdown-it-py memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ @@ -1392,24 +1392,24 @@ memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt meson==1.8.3 \ --hash=sha256:ef02b806ce0c5b6becd5bb5dc9fa67662320b29b337e7ace73e4354500590233 \ --hash=sha256:f118aa910fc0a137cc2dd0122232dbf82153d9a12fb5b0f5bb64896f6a157abf # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt mistral-common==1.8.3 \ --hash=sha256:0d1979d82227b625f6d71b3c828176f059da8d0f5a3307cdf53b48409a3970a4 \ --hash=sha256:846b6e4bbe016dc2e64fd3169fa704a548f6c74467e0cb18dc165b7a7669abd6 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm mpmath==1.3.0 \ --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # sympy msgpack==1.0.7 \ --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ @@ -1469,7 +1469,7 @@ msgpack==1.0.7 \ --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # ray msgspec==0.19.0 \ @@ -1510,7 +1510,7 @@ msgspec==0.19.0 \ --hash=sha256:f98bd8962ad549c27d63845b50af3f53ec468b6318400c9f1adfe8b092d7b62f \ --hash=sha256:fe2c4bf29bf4e89790b3117470dea2c20b59932772483082c468b990d45fb947 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm multidict==6.0.5 \ --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ @@ -1604,13 +1604,13 @@ multidict==6.0.5 \ --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # aiohttp # yarl networkx==3.2.1 \ --hash=sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # scikit-image # torch ninja==1.11.1.4 \ @@ -1632,7 +1632,7 @@ ninja==1.11.1.4 \ --hash=sha256:ecce44a00325a93631792974659cf253a815cc6da4ec96f89742925dfc295a0d \ --hash=sha256:f6186d7607bb090c3be1e10c8a56b690be238f953616626f5032238c66e56867 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt # vllm # xgrammar @@ -1646,7 +1646,7 @@ nixl==0.4.1 \ --hash=sha256:e33102b85b3f95a8c95e59b59b29aabd03d47b5bce619de506b9bb83739cf60d \ --hash=sha256:f16092dd445542e82e3db3553f6c7697ec5a2e837f19d416401283ae245826f9 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt numba==0.61.2 \ --hash=sha256:34fba9406078bac7ab052efbf0d13939426c753ad72946baaa5bf9ae0ebb8dd2 \ @@ -1671,7 +1671,7 @@ numba==0.61.2 \ --hash=sha256:ea0247617edcb5dd61f6106a56255baab031acc4257bddaeddb3a1003b4ca3fd \ --hash=sha256:efd3db391df53aaa5cfbee189b6c910a5b471488749fd6606c3f33fc984c2ae2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm numpy==1.26.4 \ --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ @@ -1711,7 +1711,7 @@ numpy==1.26.4 \ --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # cupy-cuda12x # gguf @@ -1735,70 +1735,70 @@ numpy==1.26.4 \ nvidia-cublas-cu12==12.8.3.14 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:3f0e05e7293598cf61933258b73e66a160c27d59c4422670bf0b79348c04be44 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # nvidia-cudnn-cu12 # nvidia-cusolver-cu12 # torch nvidia-cuda-cupti-cu12==12.8.57 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:8e0b2eb847de260739bee4a3f66fac31378f4ff49538ff527a38a01a9a39f950 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch nvidia-cuda-nvrtc-cu12==12.8.61 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:a0fa9c2a21583105550ebd871bd76e2037205d56f33f128e69f6d2a55e0af9ed # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch nvidia-cuda-runtime-cu12==12.8.57 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:75342e28567340b7428ce79a5d6bb6ca5ff9d07b69e7ce00d2c7b4dc23eff0be # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch nvidia-cudnn-cu12==9.7.1.26 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:6d011159a158f3cfc47bf851aea79e31bcff60d530b70ef70474c84cac484d07 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch nvidia-cufft-cu12==11.3.3.41 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:da650080ab79fcdf7a4b06aa1b460e99860646b176a43f6208099bdc17836b6a # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch nvidia-cufile-cu12==1.13.0.11 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:483f434c541806936b98366f6d33caef5440572de8ddf38d453213729da3e7d4 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch nvidia-curand-cu12==10.3.9.55 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:8387d974240c91f6a60b761b83d4b2f9b938b7e0b9617bae0f0dafe4f5c36b86 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch nvidia-cusolver-cu12==11.7.2.55 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:4d1354102f1e922cee9db51920dba9e2559877cf6ff5ad03a00d853adafb191b # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch nvidia-cusparse-cu12==12.5.7.53 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:3c1b61eb8c85257ea07e9354606b26397612627fdcd327bfd91ccf6155e7c86d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # nvidia-cusolver-cu12 # torch nvidia-cusparselt-cu12==0.6.3 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch nvidia-nccl-cu12==2.26.2 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch nvidia-nvjitlink-cu12==12.8.61 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:45fd79f2ae20bd67e8bc411055939049873bfd8fac70ff13bd4865e0b9bdab17 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # nvidia-cufft-cu12 # nvidia-cusolver-cu12 # nvidia-cusparse-cu12 @@ -1806,25 +1806,25 @@ nvidia-nvjitlink-cu12==12.8.61 ; platform_machine == 'x86_64' and sys_platform = nvidia-nvtx-cu12==12.8.55 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:2dd0780f1a55c21d8e06a743de5bd95653de630decfff40621dbde78cc307102 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch openai==1.90.0 \ --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ --hash=sha256:e5dcb5498ea6b42fec47546d10f1bcc05fb854219a7d953a5ba766718b212a02 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt opencensus-context==0.1.3 \ --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # opencensus opencv-python-headless==4.11.0.86 \ --hash=sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b \ @@ -1835,14 +1835,14 @@ opencv-python-headless==4.11.0.86 \ --hash=sha256:a66c1b286a9de872c343ee7c3553b084244299714ebb50fbdcd76f07ebbe6c81 \ --hash=sha256:f447d8acbb0b6f2808da71fddd29c1cdd448d2bc98f72d9bb78a7a898fc9621b # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # mistral-common # vllm opentelemetry-api==1.34.1 \ --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # opentelemetry-sdk @@ -1851,26 +1851,26 @@ opentelemetry-exporter-prometheus==0.55b1 \ --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt opentelemetry-proto==1.27.0 \ --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt opentelemetry-sdk==1.34.1 \ --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus opentelemetry-semantic-conventions==0.55b1 \ --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # opentelemetry-sdk outlines-core==0.2.10 \ --hash=sha256:0a9e4b192ca837a472a1bb1428397509f543db08e1aeeee30252525cec34093a \ @@ -1915,13 +1915,13 @@ outlines-core==0.2.10 \ --hash=sha256:f895834da0a577120dcb8d979c12c0690fe912095413bf0070a73e9ff363b7bf \ --hash=sha256:faf5b43181b1d033871364e74e9d348362c6a77b1d054d7af35e09fdfcff5b16 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm packaging==23.0 \ --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # huggingface-hub # kombu @@ -1960,13 +1960,13 @@ pandas==1.5.3 \ --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt partial-json-parser==0.2.1.1.post5 \ --hash=sha256:627715aaa3cb3fb60a65b0d62223243acaa6c70846520a90326fef3a2f0b61ca \ --hash=sha256:992710ac67e90b367921d52727698928040f7713ba7ecb33b96371ea7aec82ca # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm pillow==10.3.0 \ --hash=sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c \ @@ -2039,7 +2039,7 @@ pillow==10.3.0 \ --hash=sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27 \ --hash=sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # imageio # mistral-common # scikit-image @@ -2049,13 +2049,13 @@ platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # virtualenv prometheus-client==0.19.0 \ --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # prometheus-fastapi-instrumentator @@ -2064,13 +2064,13 @@ prometheus-fastapi-instrumentator==7.1.0 \ --hash=sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9 \ --hash=sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm prompt-toolkit==3.0.41 \ --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # click-repl propcache==0.3.0 \ --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ @@ -2172,14 +2172,14 @@ propcache==0.3.0 \ --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # aiohttp # yarl proto-plus==1.22.3 \ --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # google-api-core protobuf==4.25.8 \ --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ @@ -2194,7 +2194,7 @@ protobuf==4.25.8 \ --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # google-api-core # googleapis-common-protos @@ -2221,13 +2221,13 @@ psutil==5.9.6 \ --hash=sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d \ --hash=sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm py-cpuinfo==9.0.0 \ --hash=sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690 \ --hash=sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:47cdda4c34d9b6cb01f3aaeceb2e88faf57da880207fe72ff6ff97e9bb6cc8a9 \ @@ -2239,7 +2239,7 @@ py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt pyarrow==19.0.1 \ --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ @@ -2285,20 +2285,20 @@ pyarrow==19.0.1 \ --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt pyasn1==0.5.1 \ --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # pyasn1-modules # rsa pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # google-auth pybase64==1.4.1 \ --hash=sha256:011a54ff6ca44c5d03746aec3f1f492fce3155bd3f943fb2ceaea92416d40eeb \ @@ -2445,31 +2445,31 @@ pybase64==1.4.1 \ --hash=sha256:fc9504c4c2e893e0a6c1cc80bce51907e3461288289f630eab22b5735eba1104 \ --hash=sha256:ff172a4dacbd964e5edcf1c2152dae157aabf856508aed15276f46d04a22128e # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm pybind11==2.13.6 \ --hash=sha256:237c41e29157b962835d356b370ededd57594a26d5894a795960f0047cb5caf5 \ --hash=sha256:ba6af10348c12b24e92fa086b39cfba0eff619b61ac77c406167d813b096d39a # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt pycountry==24.6.1 \ --hash=sha256:b61b3faccea67f87d10c1f2b0fc0be714409e8fcdcc1315613174f6466c10221 \ --hash=sha256:f1a4fb391cd7214f8eefd39556d740adcc233c778a27f8942c8dca351d6ce06f # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # pydantic-extra-types pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # cffi pydantic==2.10.0 \ --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # compressed-tensors # fastapi @@ -2581,56 +2581,56 @@ pydantic-core==2.27.0 \ --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # pydantic pydantic-extra-types==2.10.5 \ --hash=sha256:1dcfa2c0cf741a422f088e0dbb4690e7bfadaaf050da3d6f80d6c3cf58a2bad8 \ --hash=sha256:b60c4e23d573a69a4f1a16dd92888ecc0ef34fb0e655b4f305530377fa70e7a8 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # mistral-common pygments==2.18.0 \ --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # rich pyopenssl==25.0.0 \ --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # celery # pandas python-dotenv==1.1.0 \ --hash=sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5 \ --hash=sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # uvicorn python-json-logger==2.0.7 \ --hash=sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c \ --hash=sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm python-multipart==0.0.20 \ --hash=sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104 \ --hash=sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # fastapi pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # pandas pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ @@ -2685,7 +2685,7 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # gguf # huggingface-hub @@ -2784,13 +2784,13 @@ pyzmq==26.0.3 \ --hash=sha256:f6b1d1c631e5940cac5a0b22c5379c86e8df6a4ec277c7a856b714021ab6cfad \ --hash=sha256:f6c21c00478a7bea93caaaef9e7629145d4153b15a8653e8bb4609d4bc70dbfc # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm referencing==0.36.2 \ --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # jsonschema # jsonschema-specifications regex==2024.11.6 \ @@ -2889,7 +2889,7 @@ regex==2024.11.6 \ --hash=sha256:fdd6028445d2460f33136c55eeb1f601ab06d74cb3347132e1c24250187500d9 \ --hash=sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # tiktoken # transformers # vllm @@ -2897,7 +2897,7 @@ requests==2.32.3 \ --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # google-api-core # huggingface-hub @@ -2910,7 +2910,7 @@ rich==13.3.2 \ --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # memray # typer @@ -3019,14 +3019,14 @@ rpds-py==0.22.3 \ --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # jsonschema # referencing rsa==4.7.2 \ --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # google-auth safetensors==0.5.3 \ --hash=sha256:1077f3e94182d72618357b04b5ced540ceb71c8a813d3319f1aba448e68a770d \ @@ -3045,7 +3045,7 @@ safetensors==0.5.3 \ --hash=sha256:cfc0ec0846dcf6763b0ed3d1846ff36008c6e7290683b61616c4b040f6a54ace \ --hash=sha256:df26da01aaac504334644e1b7642fa000bfec820e7cef83aeac4e355e03195ff # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # transformers scikit-image==0.24.0 \ --hash=sha256:18836a18d3a7b6aca5376a2d805f0045826bc6c9fc85331659c33b4813e0b563 \ @@ -3070,7 +3070,7 @@ scikit-image==0.24.0 \ --hash=sha256:ef04360eda372ee5cd60aebe9be91258639c86ae2ea24093fb9182118008d009 \ --hash=sha256:fa27b3a0dbad807b966b8db2d78da734cb812ca4787f7fbb143764800ce2fa9c # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt scipy==1.11.4 \ --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ @@ -3099,7 +3099,7 @@ scipy==1.11.4 \ --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # scikit-image # vllm @@ -3158,7 +3158,7 @@ sentencepiece==0.2.0 \ --hash=sha256:fb89f811e5efd18bab141afc3fea3de141c3f69f3fe9e898f710ae7fe3aab251 \ --hash=sha256:ff88712338b01031910e8e61e7239aff3ce8869ee31a47df63cb38aadd591bea # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # gguf # mistral-common # vllm @@ -3166,26 +3166,26 @@ shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # typer six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # opencensus # python-dateutil smart-open==6.2.0 \ --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # anyio # openai soundfile==0.13.1 \ @@ -3198,7 +3198,7 @@ soundfile==0.13.1 \ --hash=sha256:b2c68dab1e30297317080a5b43df57e302584c49e2942defdde0acccc53f0e5b \ --hash=sha256:c734564fab7c5ddf8e9be5bf70bab68042cd17e9c214c06e365e20d64f9a69d5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # mistral-common soxr==0.5.0.post1 \ --hash=sha256:39e0f791ba178d69cd676485dbee37e75a34f20daa478d90341ecb7f6d9d690f \ @@ -3223,13 +3223,13 @@ soxr==0.5.0.post1 \ --hash=sha256:fcc049b0a151a65aa75b92f0ac64bb2dba785d16b78c31c2b94e68c141751d6d \ --hash=sha256:fef509466c9c25f65eae0ce1e4b9ac9705d22c6038c914160ddaf459589c6e31 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # mistral-common starlette==0.46.2 \ --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # fastapi # prometheus-fastapi-instrumentator @@ -3237,19 +3237,19 @@ sympy==1.14.0 \ --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch tensorboardx==2.6.2.2 \ --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt tifffile==2024.7.21 \ --hash=sha256:7f335b5d6ca49401fe0f1d87deb206f5dae47297e47b1ed52a676d05d6d26798 \ --hash=sha256:818b577d49350421fb511f389f937984f9feaa2cd8177fa00823001920bf3483 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # scikit-image tiktoken==0.9.0 \ --hash=sha256:03935988a91d6d3216e2ec7c645afbb3d870b37bcb67ada1943ec48678e7ee33 \ @@ -3284,7 +3284,7 @@ tiktoken==0.9.0 \ --hash=sha256:f0968d5beeafbca2a72c595e8385a1a1f8af58feaebb02b227229b69ca5357fd \ --hash=sha256:f32cc56168eac4851109e9b5d327637f15fd662aa30dd79f964b7c39fbadd26e # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # mistral-common # vllm tokenizers==0.21.1 \ @@ -3304,7 +3304,7 @@ tokenizers==0.21.1 \ --hash=sha256:e78e413e9e668ad790a29456e677d9d3aa50a9ad311a40905d6861ba7692cf41 \ --hash=sha256:ed248ab5279e601a30a4d67bdb897ecbe955a50f1e7bb62bd99f07dd11c2f5b6 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # transformers # vllm torch==2.7.1+cu128 \ @@ -3327,7 +3327,7 @@ torch==2.7.1+cu128 \ --hash=sha256:e27e5f7e74179fb5d814a0412e5026e4b50c9e0081e9050bc4c28c992a276eb1 \ --hash=sha256:f112465fdf42eb1297c6dddda1a8b7f411914428b704e1b8a47870c52e290909 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # compressed-tensors # nixl # torchaudio @@ -3349,7 +3349,7 @@ torchaudio==2.7.1+cu128 \ --hash=sha256:b1e56a999a06a5deaebfb991dc676aaa60d98139907d99badbc6dca6456637ee \ --hash=sha256:cb435329019d441d8177db2d84e8d397881896d100efb4f4c15f0d3732f92a81 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm torchvision==0.22.1+cu128 \ --hash=sha256:02faf51fbf5070592768fa935327d13a484b745faef38b0fee01d85cfb35f5bc \ @@ -3365,13 +3365,13 @@ torchvision==0.22.1+cu128 \ --hash=sha256:eb784cc75a66f3336a04ff3a992bf74160842132db69e8bdbb58b5ab9422c345 \ --hash=sha256:f64ef9bb91d71ab35d8384912a19f7419e35928685bc67544d58f45148334373 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm tqdm==4.67.1 \ --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # gguf # huggingface-hub # openai @@ -3381,7 +3381,7 @@ transformers==4.53.2 \ --hash=sha256:6c3ed95edfb1cba71c4245758f1b4878c93bf8cde77d076307dacb2cbbd72be2 \ --hash=sha256:db8f4819bb34f000029c73c3c557e7d06fc1b8e612ec142eecdae3947a9c78bf # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt # compressed-tensors # vllm @@ -3389,21 +3389,21 @@ transformers==4.53.2 \ triton==3.3.1 ; sys_platform == 'linux' \ --hash=sha256:b31e3aa26f8cb3cc5bf4e187bf737cbacf17311e1112b781d4a059353dfd731b # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch # xgrammar typer==0.12.3 \ --hash=sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914 \ --hash=sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt # fastapi-cli typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # fastapi # gymnasium # huggingface-hub @@ -3424,19 +3424,19 @@ tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # kombu urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # requests uvicorn==0.22.0 \ --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # fastapi # fastapi-cli @@ -3479,13 +3479,13 @@ uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'c --hash=sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816 \ --hash=sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # uvicorn vine==5.1.0 \ --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # amqp # celery # kombu @@ -3493,13 +3493,13 @@ virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt vllm==0.10.0 \ --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ --hash=sha256:a44e9013db26082a82c3931ed8772ac884d6d60566d36ecdb0e8dc01c65b241a # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt watchfiles==0.19.0 \ --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ @@ -3525,7 +3525,7 @@ watchfiles==0.19.0 \ --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt # uvicorn # vllm @@ -3533,7 +3533,7 @@ wcwidth==0.2.13 \ --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # prompt-toolkit websockets==15.0.1 \ --hash=sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2 \ @@ -3606,12 +3606,12 @@ websockets==15.0.1 \ --hash=sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f \ --hash=sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # uvicorn xformers==0.0.31 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:b2ea87e0651f46164cb3cd74face021bd1654229ca4f8c0baa03b8c477515c7a # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm xgrammar==0.1.21 ; platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:140628376fc701a535600dc64752603ddaed619461dc50669e90626e9f61b8aa \ @@ -3639,7 +3639,7 @@ xgrammar==0.1.21 ; platform_machine == 'aarch64' or platform_machine == 'arm64' --hash=sha256:f89d9ddb4d00fadcffa4bcabd0c3ae75d47c844c728bbb6be695056df3767524 \ --hash=sha256:f9247641c73eec6e972cec15156a8844957334204ba79ad1abdb0d7b03def8a1 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm yarl==1.18.3 \ --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ @@ -3725,13 +3725,13 @@ yarl==1.18.3 \ --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # aiohttp zipp==3.19.2 \ --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via - # -c python/requirements_compiled_rayllm_test_py311_cu128.txt + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # importlib-metadata # The following packages were excluded from the output: diff --git a/python/requirements_compiled_rayllm_test_py311_cpu.txt b/python/deplocks/llm/rayllm_test_py311_cpu.lock similarity index 95% rename from python/requirements_compiled_rayllm_test_py311_cpu.txt rename to python/deplocks/llm/rayllm_test_py311_cpu.lock index bcacf5033722..d52e9ae3b25a 100644 --- a/python/requirements_compiled_rayllm_test_py311_cpu.txt +++ b/python/deplocks/llm/rayllm_test_py311_cpu.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cpu.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cpu.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cpu.lock python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/deplocks/llm/rayllm_test_py311_cpu.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu @@ -7,13 +7,13 @@ aiofiles==22.1.0 \ --hash=sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad \ --hash=sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ypy-websocket aiohappyeyeballs==2.6.1 \ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp aiohttp==3.11.16 \ --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ @@ -98,7 +98,7 @@ aiohttp==3.11.16 \ --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements/llm/llm-test-requirements.txt # -r python/requirements.txt @@ -109,25 +109,25 @@ aiohttp-cors==0.7.0 \ --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt aiorwlock==1.3.0 \ --hash=sha256:45baf8e4fa9a23e0bb325fbd67da80de1fd7ae1d4f59a6381754c60cec7b289b \ --hash=sha256:83f12d87df4b9728a0b8fda1756585ab0d652b107bab59c6084e1b1ad692ab45 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp aiosqlite==0.19.0 \ --hash=sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d \ --hash=sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ypy-websocket alabaster==0.7.16 \ --hash=sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65 \ @@ -137,19 +137,19 @@ amqp==5.3.1 \ --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # pydantic anyio==3.7.1 \ --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # httpx # jupyter-server # openai @@ -159,7 +159,7 @@ argon2-cffi==23.1.0 \ --hash=sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08 \ --hash=sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-server # nbclassic # notebook @@ -186,13 +186,13 @@ argon2-cffi-bindings==21.2.0 \ --hash=sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e \ --hash=sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # argon2-cffi arrow==1.3.0 \ --hash=sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80 \ --hash=sha256:d4540617648cb5f895730f1ad8c82a65f2dad0166f57b75f3ca54759c4d67a85 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # isoduration astor==0.8.1 \ --hash=sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5 \ @@ -202,13 +202,13 @@ asttokens==2.4.1 \ --hash=sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24 \ --hash=sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # stack-data attrs==25.1.0 \ --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp # jsonschema # referencing @@ -216,13 +216,13 @@ azure-common==1.1.28 \ --hash=sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3 \ --hash=sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # smart-open azure-core==1.29.5 \ --hash=sha256:0fa04b7b1f7d44a4fb8468c4093deb2ea01fdf4faddbf802ed9205615f99d68c \ --hash=sha256:52983c89d394c6f881a121e5101c5fa67278ca3b1f339c8fb2ef39230c70e9ac # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # azure-identity # azure-storage-blob # smart-open @@ -230,26 +230,26 @@ azure-identity==1.17.1 \ --hash=sha256:32ecc67cc73f4bd0595e4f64b1ca65cd05186f4fe6f98ed2ae9f1aa32646efea \ --hash=sha256:db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt azure-storage-blob==12.22.0 \ --hash=sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e \ --hash=sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # smart-open babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ --hash=sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyterlab-server # sphinx backcall==0.2.0 \ --hash=sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e \ --hash=sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipython backoff==2.2.1 \ --hash=sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba \ @@ -259,13 +259,13 @@ beautifulsoup4==4.11.1 \ --hash=sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30 \ --hash=sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbconvert billiard==4.2.1 \ --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # celery blake3==1.0.4 \ --hash=sha256:00605aa59923205c6a4f21131840840eb2d9a754c59b163357d890566755b97a \ @@ -358,20 +358,20 @@ bleach==6.1.0 \ --hash=sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe \ --hash=sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbconvert boto3==1.26.76 \ --hash=sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027 \ --hash=sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # smart-open botocore==1.29.76 \ --hash=sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7 \ --hash=sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # boto3 # s3transfer @@ -379,7 +379,7 @@ cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-auth # vllm cbor2==5.6.5 \ @@ -432,13 +432,13 @@ celery==5.5.3 \ --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # httpcore # httpx @@ -497,7 +497,7 @@ cffi==1.16.0 \ --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # argon2-cffi-bindings # cryptography # soundfile @@ -593,13 +593,13 @@ charset-normalizer==3.3.2 \ --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # requests click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # celery @@ -613,32 +613,32 @@ click-didyoumean==0.3.1 \ --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # celery click-plugins==1.1.1.2 \ --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # celery click-repl==0.3.0 \ --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # gymnasium # vllm colorama==0.4.6 \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # halo # log-symbols @@ -646,13 +646,13 @@ colorful==0.5.5 \ --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt comm==0.2.0 \ --hash=sha256:2da8d9ebb8dd7bfc247adaff99f24dce705638a8042b85cb995066793e391001 \ --hash=sha256:a517ea2ca28931c7007a7a99c562a0fa5883cfb48963140cf642c41c948498be # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipykernel # ipywidgets compressed-tensors==0.10.2 \ @@ -698,7 +698,7 @@ cryptography==44.0.3 \ --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # azure-identity # azure-storage-blob # msal @@ -718,7 +718,7 @@ cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # ray debugpy==1.8.0 \ @@ -741,19 +741,19 @@ debugpy==1.8.0 \ --hash=sha256:ef54404365fae8d45cf450d0544ee40cefbcb9cb85ea7afe89a963c27028261e \ --hash=sha256:ef9ab7df0b9a42ed9c878afd3eaaff471fce3fa73df96022e1f5c9f8f8c87ada # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipykernel decorator==5.1.1 \ --hash=sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330 \ --hash=sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipython defusedxml==0.7.1 \ --hash=sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69 \ --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbconvert depyf==0.19.0 \ --hash=sha256:040b35fc0997d49df024b7d094f2a7836f91e9ed02f49982dd37e70aa3285ad5 \ @@ -771,7 +771,7 @@ distlib==0.3.7 \ --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # virtualenv distro==1.9.0 \ --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ @@ -825,7 +825,7 @@ dm-tree==0.1.8 \ --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt dnspython==2.7.0 \ --hash=sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86 \ @@ -847,26 +847,26 @@ entrypoints==0.4 \ --hash=sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4 \ --hash=sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-client # nbconvert executing==2.0.1 \ --hash=sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147 \ --hash=sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # stack-data farama-notifications==0.0.4 \ --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # gymnasium fastapi==0.115.12 \ --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # vllm fastapi-cli==0.0.5 \ @@ -877,7 +877,7 @@ fastjsonschema==2.19.0 \ --hash=sha256:b9fd1a2dd6971dbc7fee280a95bd199ae0dd9ce22beb91cc75e9c1c528a5170e \ --hash=sha256:e25df6647e1bc4a26070b700897b07b542ec898dd4f1f6ea013e7f6a88417225 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbformat fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ @@ -956,13 +956,13 @@ fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # cupy-cuda12x filelock==3.17.0 \ --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # huggingface-hub # ray @@ -974,7 +974,7 @@ fqdn==1.5.1 \ --hash=sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f \ --hash=sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema frozenlist==1.4.1 \ --hash=sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7 \ @@ -1055,14 +1055,14 @@ frozenlist==1.4.1 \ --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp # aiosignal fsspec==2023.5.0 \ --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # huggingface-hub # torch @@ -1074,19 +1074,19 @@ gitdb==4.0.11 \ --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # gitpython gitpython==3.1.44 \ --hash=sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110 \ --hash=sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt google-api-core==2.24.2 \ --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-cloud-core # google-cloud-storage # opencensus @@ -1094,7 +1094,7 @@ google-auth==2.23.4 \ --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # google-api-core # google-cloud-core @@ -1103,13 +1103,13 @@ google-cloud-core==2.4.1 \ --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-cloud-storage google-cloud-storage==2.14.0 \ --hash=sha256:2d23fcf59b55e7b45336729c148bb1c464468c69d5efbaee30f7201dd90eb97e \ --hash=sha256:8641243bbf2a2042c16a6399551fbb13f062cbc9a2de38d6c0bb5426962e9dbd # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # smart-open google-crc32c==1.5.0 \ @@ -1182,20 +1182,20 @@ google-crc32c==1.5.0 \ --hash=sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556 \ --hash=sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-cloud-storage # google-resumable-media google-resumable-media==2.6.0 \ --hash=sha256:972852f6c65f933e15a4a210c2b96930763b47197cdf4aa5f5bea435efb626e7 \ --hash=sha256:fc03d344381970f79eebb632a3c18bb1828593a2dc5572b5f90115ef7d11e81b # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-cloud-storage googleapis-common-protos==1.61.0 \ --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-api-core grpcio==1.66.2 \ --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ @@ -1254,7 +1254,7 @@ grpcio==1.66.2 \ --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # grpcio-tools @@ -1308,26 +1308,26 @@ grpcio-tools==1.62.3 \ --hash=sha256:f4b1615adf67bd8bb71f3464146a6f9949972d06d21a4f5e87e73f6464d97f57 \ --hash=sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt gymnasium==1.1.1 \ --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # httpcore # uvicorn halo==0.0.31 \ --hash=sha256:5350488fb7d2aa7c31a1344120cee67a872901ce8858f60da7946cef96c208ab \ --hash=sha256:7b67a3521ee91d53b7152d4ee3452811e1d2a6321975137762eb3d70063cc9d6 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt hf-transfer==0.1.9 \ --hash=sha256:035572865dab29d17e783fbf1e84cf1cb24f3fcf8f1b17db1cfc7fdf139f02bf \ @@ -1374,7 +1374,7 @@ httplib2==0.20.4 \ --hash=sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585 \ --hash=sha256:8b6a905cb1c79eefd03f8669fd993c36dc341f7c558f056cb5a33b5c2f458543 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # oauth2client httptools==0.6.4 \ --hash=sha256:0614154d5454c21b6410fdf5262b4a3ddb0f53f1e1721cfd59d55f32138c578a \ @@ -1439,13 +1439,13 @@ humanize==4.12.1 \ --hash=sha256:1338ba97415c96556758a6e2f65977ed406dddf4620d4c6db9bbdfd07f0f1232 \ --hash=sha256:86014ca5c52675dffa1d404491952f1f5bf03b07c175a51891a343daebf01fea # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # anyio # email-validator # httpx @@ -1456,7 +1456,7 @@ imageio==2.34.2 \ --hash=sha256:5c0c0ee8faa018a1c42f649b90395dd4d3bb6187c09053a0cd6f1fdd51bbff5e \ --hash=sha256:a0bb27ec9d5bab36a9f4835e51b21d2cb099e1f78451441f94687ff3404b79f8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # scikit-image imagesize==1.4.1 \ --hash=sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b \ @@ -1466,13 +1466,13 @@ importlib-metadata==6.11.0 \ --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # opentelemetry-api iniconfig==2.0.0 \ --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # pytest interegular==0.3.3 \ --hash=sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c \ @@ -1482,14 +1482,14 @@ ipykernel==6.27.1 \ --hash=sha256:7d5d594b6690654b4d299edba5e872dc17bb7396a8d0609c97cb7b8a1c605de6 \ --hash=sha256:dab88b47f112f9f7df62236511023c9bdeef67abc73af7c652e4ce4441601686 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbclassic # notebook ipython==8.12.3 \ --hash=sha256:3910c4b54543c2ad73d06579aa771041b7d5707b033bd488669b4cf544e3b363 \ --hash=sha256:b0340d46a933d27c657b211a329d0be23793c36595acf9e6ef4164bc01a1804c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipykernel # ipywidgets # jupyterlab @@ -1497,38 +1497,38 @@ ipython-genutils==0.2.0 \ --hash=sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8 \ --hash=sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbclassic # notebook ipywidgets==8.1.3 \ --hash=sha256:efafd18f7a142248f7cb0ba890a68b96abd4d6e88ddbda483c9130d12667eaf2 \ --hash=sha256:f5f9eeaae082b1823ce9eac2575272952f40d748893972956dc09700a6392d9c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt isodate==0.6.1 \ --hash=sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96 \ --hash=sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # azure-storage-blob isoduration==20.11.0 \ --hash=sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9 \ --hash=sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema jedi==0.19.1 \ --hash=sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd \ --hash=sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipython jinja2==3.1.6 \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # fastapi # jupyter-server # jupyterlab @@ -1621,26 +1621,26 @@ jmespath==1.0.1 \ --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # boto3 # botocore json5==0.9.14 \ --hash=sha256:740c7f1b9e584a468dbb2939d8d458db3427f2c93ae2139d05f47e453eae964f \ --hash=sha256:9ed66c3a6ca3510a976a9ef9b8c0787de24802724ab1860bc0153c7fdd589b02 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyterlab-server jsonpatch==1.32 \ --hash=sha256:26ac385719ac9f54df8a2f0827bb8253aa3ea8ab7b3368457bcdb8c14595a397 \ --hash=sha256:b6ddfe6c3db30d81a96aaeceb6baf916094ffa23d7dd5fa2c13e13f8b6e600c2 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt jsonpointer==2.4 \ --hash=sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a \ --hash=sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonpatch # jsonschema jsonref==1.1.0 \ @@ -1651,7 +1651,7 @@ jsonschema==4.23.0 \ --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt @@ -1664,13 +1664,13 @@ jsonschema-specifications==2024.10.1 \ --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema jupyter-client==7.3.4 \ --hash=sha256:17d74b0d0a7b24f1c8c527b24fcf4607c56bee542ffe8e3418e50b21e514b621 \ --hash=sha256:aa9a6c32054b290374f95f73bb0cae91455c58dfb84f65c8591912b8f65e6d56 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipykernel # jupyter-server # nbclassic @@ -1680,7 +1680,7 @@ jupyter-core==5.5.0 \ --hash=sha256:880b86053bf298a8724994f95e99b99130659022a4f7f45f563084b6223861d3 \ --hash=sha256:e11e02cd8ae0a9de5c6c44abf5727df9f2581055afe00b22183f621ba3585805 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipykernel # jupyter-client # jupyter-server @@ -1693,13 +1693,13 @@ jupyter-events==0.6.3 \ --hash=sha256:57a2749f87ba387cd1bfd9b22a0875b889237dbf2edc2121ebb22bde47036c17 \ --hash=sha256:9a6e9995f75d1b7146b436ea24d696ce3a35bfa8bfe45e0c33c334c79464d0b3 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-server-fileid jupyter-server==1.24.0 \ --hash=sha256:23368e8e214baf82b313d4c5a0d828ca73015e1a192ce3829bd74e62fab8d046 \ --hash=sha256:c88ddbe862966ea1aea8c3ccb89a5903abd8fbcfe5cd14090ef549d403332c37 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-server-fileid # jupyterlab # jupyterlab-server @@ -1709,44 +1709,44 @@ jupyter-server-fileid==0.9.0 \ --hash=sha256:171538b7c7d08d11dbc57d4e6da196e0c258e4c2cd29249ef1e032bb423677f8 \ --hash=sha256:5b489c6fe6783c41174a728c7b81099608518387e53c3d53451a67f46a0cb7b0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-server-ydoc jupyter-server-ydoc==0.6.1 \ --hash=sha256:18275ff1ce7e93bbda2301ca066273b3951fc50b0d9c8fc33788374134ad7920 \ --hash=sha256:ab10864708c81fa41ab9f2ed3626b54ff6926eaf14545d1d439714978dad6e9f # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyterlab jupyter-ydoc==0.2.5 \ --hash=sha256:5759170f112c70320a84217dd98d287699076ae65a7f88d458d57940a9f2b882 \ --hash=sha256:5a02ca7449f0d875f73e8cb8efdf695dddef15a8e71378b1f4eda6b7c90f5382 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-server-ydoc # jupyterlab jupyterlab==3.6.1 \ --hash=sha256:ad6707dd0149b629d0ed5b56916cfcdb816b376c6af3190337faba09e27ea29e \ --hash=sha256:aee98c174180e98a30470297d10b959e8e64f2288970c0de65f0a6d2b4807034 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt jupyterlab-pygments==0.3.0 \ --hash=sha256:721aca4d9029252b11cfa9d185e5b5af4d54772bb8072f9b7036f4170054d35d \ --hash=sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbconvert jupyterlab-server==2.24.0 \ --hash=sha256:4e6f99e0a5579bbbc32e449c4dbb039561d4f1a7827d5733273ed56738f21f07 \ --hash=sha256:5f077e142bb8dc9b843d960f940c513581bceca3793a0d80f9c67d9522c4e876 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyterlab jupyterlab-widgets==3.0.11 \ --hash=sha256:78287fd86d20744ace330a61625024cf5521e1c012a352ddc0a3cdc2348becd0 \ --hash=sha256:dd5ac679593c969af29c9bed054c24f26842baa51352114736756bc035deee27 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipywidgets jupytext==1.16.7 \ --hash=sha256:912f9d9af7bd3f15470105e5c5dddf1669b2d8c17f0c55772687fc5a4a73fe69 \ @@ -1756,7 +1756,7 @@ kombu==5.5.4 \ --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ @@ -1766,7 +1766,7 @@ lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # scikit-image llguidance==0.7.26 ; platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:1895ff449c8ec0a5f1d3b142d723fc9b26a85b021b72d7f1173f8b7507f528c0 \ @@ -1809,7 +1809,7 @@ log-symbols==0.0.14 \ --hash=sha256:4952106ff8b605ab7d5081dd2c7e6ca7374584eff7086f499c06edd1ce56dcca \ --hash=sha256:cf0bbc6fe1a8e53f0d174a716bc625c4f87043cc21eb55dd8a740cfe22680556 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # halo lxml==4.9.4 \ --hash=sha256:00e91573183ad273e242db5585b52670eddf92bacad095ce25c1e682da14ed91 \ @@ -1906,7 +1906,7 @@ lxml==4.9.4 \ --hash=sha256:fd814847901df6e8de13ce69b84c31fc9b3fb591224d6762d0b256d510cbf382 \ --hash=sha256:fdb325b7fba1e2c40b9b1db407f85642e32404131c08480dd652110fc908561b # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbconvert lz4==4.3.3 \ --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ @@ -1946,13 +1946,13 @@ lz4==4.3.3 \ --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt markdown-it-py==2.2.0 \ --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupytext # mdit-py-plugins # rich @@ -2018,14 +2018,14 @@ markupsafe==2.1.3 \ --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jinja2 # nbconvert matplotlib-inline==0.1.6 \ --hash=sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311 \ --hash=sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipykernel # ipython mdit-py-plugins==0.4.2 \ @@ -2036,7 +2036,7 @@ mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # markdown-it-py memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ @@ -2075,7 +2075,7 @@ memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt meson==1.8.3 \ --hash=sha256:ef02b806ce0c5b6becd5bb5dc9fa67662320b29b337e7ace73e4354500590233 \ @@ -2089,7 +2089,7 @@ mistune==0.8.4 \ --hash=sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e \ --hash=sha256:88a1051873018da288eee8538d476dffe1262495144b33ecb586c4ab266bb8d4 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbconvert mpmath==1.3.0 \ --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ @@ -2099,14 +2099,14 @@ msal==1.28.1 \ --hash=sha256:563c2d70de77a2ca9786aab84cb4e133a38a6897e6676774edc23d610bfc9e7b \ --hash=sha256:d72bbfe2d5c2f2555f4bc6205be4450ddfd12976610dd9a16a9ab0f05c68b64d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # azure-identity # msal-extensions msal-extensions==1.2.0b1 \ --hash=sha256:217f391bb549de11b19abe8029a8375fe3ca0556aa8cce004b2083f00a569b71 \ --hash=sha256:3658b3814cd6a7759e83cb0ec145f30330ee249a92444adaf9aa4eb4f5bbcbbc # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # azure-identity msgpack==1.0.7 \ --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ @@ -2166,7 +2166,7 @@ msgpack==1.0.7 \ --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # ray msgspec==0.19.0 \ @@ -2299,27 +2299,27 @@ multidict==6.0.5 \ --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp # yarl nbclassic==1.0.0 \ --hash=sha256:0ae11eb2319455d805596bf320336cda9554b41d99ab9a3c31bf8180bffa30e3 \ --hash=sha256:f99e4769b4750076cd4235c044b61232110733322384a94a63791d2e7beacc66 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyterlab # notebook nbclient==0.5.13 \ --hash=sha256:40c52c9b5e3c31faecaee69f202b3f53e38d7c1c563de0fadde9d7eda0fdafe8 \ --hash=sha256:47ac905af59379913c1f8f541098d2550153cf8dc58553cbe18c702b181518b0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbconvert nbconvert==6.5.4 \ --hash=sha256:9e3c7c6d491374cbdd5f35d268c05809357716d346f4573186bbeab32ee50bc1 \ --hash=sha256:d679a947f849a966cbbd0bf6e7fedcfdb64be3b20ce7cef11ad55c13f5820e19 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-server # nbclassic # notebook @@ -2327,7 +2327,7 @@ nbformat==5.9.2 \ --hash=sha256:1c5172d786a41b82bcfd0c23f9e6b6f072e8fb49c39250219e4acfff1efe89e9 \ --hash=sha256:5f98b5ba1997dff175e77e0c17d5c10a96eaed2cbd1de3533d1fc35d5e111192 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-server # jupytext # nbclassic @@ -2338,7 +2338,7 @@ nest-asyncio==1.5.8 \ --hash=sha256:25aa2ca0d2a5b5531956b9e273b45cf664cae2b145101d73b86b199978d48fdb \ --hash=sha256:accda7a339a70599cb08f9dd09a67e0c2ef8d8d6f4c07f96ab203f2ae254e48d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipykernel # jupyter-client # nbclassic @@ -2348,7 +2348,7 @@ networkx==3.2.1 \ --hash=sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6 \ --hash=sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # scikit-image # torch ninja==1.11.1.3 \ @@ -2387,13 +2387,13 @@ notebook==6.5.7 \ --hash=sha256:04eb9011dfac634fbd4442adaf0a8c27cd26beef831fe1d19faf930c327768e4 \ --hash=sha256:a6afa9a4ff4d149a0771ff8b8c881a7a73b3835f9add0606696d6e9d98ac1cd0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyterlab notebook-shim==0.2.3 \ --hash=sha256:a83496a43341c1674b093bfcebf0fe8e74cbe7eda5fd2bbc56f8e39e1486c0c7 \ --hash=sha256:f69388ac283ae008cd506dda10d0288b09a017d822d5e8c7129a152cbd3ce7e9 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbclassic numba==0.61.2 \ --hash=sha256:34fba9406078bac7ab052efbf0d13939426c753ad72946baaa5bf9ae0ebb8dd2 \ @@ -2456,7 +2456,7 @@ numpy==1.26.4 \ --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # cupy-cuda12x # gguf @@ -2485,7 +2485,7 @@ oauth2client==4.1.3 \ --hash=sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac \ --hash=sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt openai==1.90.0 \ --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ @@ -2495,13 +2495,13 @@ opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt opencensus-context==0.1.3 \ --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # opencensus opencv-python-headless==4.11.0.86 \ --hash=sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b \ @@ -2518,7 +2518,7 @@ opentelemetry-api==1.34.1 \ --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # opentelemetry-sdk @@ -2527,26 +2527,26 @@ opentelemetry-exporter-prometheus==0.55b1 \ --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt opentelemetry-proto==1.27.0 \ --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt opentelemetry-sdk==1.34.1 \ --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus opentelemetry-semantic-conventions==0.55b1 \ --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # opentelemetry-sdk outlines-core==0.2.10 \ --hash=sha256:0a9e4b192ca837a472a1bb1428397509f543db08e1aeeee30252525cec34093a \ @@ -2595,7 +2595,7 @@ packaging==23.0 \ --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # huggingface-hub @@ -2643,19 +2643,19 @@ pandas==1.5.3 \ --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt pandocfilters==1.5.0 \ --hash=sha256:0b679503337d233b4339a817bfc8c50064e2eff681314376a47cb582305a7a38 \ --hash=sha256:33aae3f25fd1a026079f5d27bdd52496f0e0803b3469282162bafdcbdf6ef14f # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbconvert parso==0.8.3 \ --hash=sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0 \ --hash=sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jedi partial-json-parser==0.2.1.1.post5 \ --hash=sha256:627715aaa3cb3fb60a65b0d62223243acaa6c70846520a90326fef3a2f0b61ca \ @@ -2665,19 +2665,19 @@ pathspec==0.11.2 \ --hash=sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20 \ --hash=sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt pexpect==4.8.0 ; sys_platform != 'win32' \ --hash=sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937 \ --hash=sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipython pickleshare==0.7.5 \ --hash=sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca \ --hash=sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipython pillow==10.3.0 \ --hash=sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c \ @@ -2750,7 +2750,7 @@ pillow==10.3.0 \ --hash=sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27 \ --hash=sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/llm/llm-test-requirements.txt # imageio # mistral-common @@ -2761,26 +2761,26 @@ platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-core # virtualenv pluggy==1.3.0 \ --hash=sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12 \ --hash=sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # pytest portalocker==2.8.2 \ --hash=sha256:2b035aa7828e46c58e9b31390ee1f169b98e1066ab10b9a6a861fe7e25ee4f33 \ --hash=sha256:cfb86acc09b9aa7c3b43594e19be1345b9d16af3feb08bf92f23d4dce513a28e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # msal-extensions prometheus-client==0.19.0 \ --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # jupyter-server # nbclassic @@ -2796,7 +2796,7 @@ prompt-toolkit==3.0.41 \ --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # click-repl # ipython propcache==0.3.0 \ @@ -2899,14 +2899,14 @@ propcache==0.3.0 \ --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp # yarl proto-plus==1.22.3 \ --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-api-core protobuf==4.25.8 \ --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ @@ -2921,7 +2921,7 @@ protobuf==4.25.8 \ --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # google-api-core # googleapis-common-protos @@ -2949,21 +2949,21 @@ psutil==5.9.6 \ --hash=sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d \ --hash=sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipykernel # vllm ptyprocess==0.7.0 ; os_name != 'nt' or sys_platform != 'win32' \ --hash=sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35 \ --hash=sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # pexpect # terminado pure-eval==0.2.2 \ --hash=sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350 \ --hash=sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # stack-data py-cpuinfo==9.0.0 \ --hash=sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690 \ @@ -2979,7 +2979,7 @@ py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt pyarrow==19.0.1 \ --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ @@ -3025,13 +3025,13 @@ pyarrow==19.0.1 \ --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt pyasn1==0.5.1 \ --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # oauth2client # pyasn1-modules # rsa @@ -3039,7 +3039,7 @@ pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-auth # oauth2client pybase64==1.4.1 \ @@ -3199,7 +3199,7 @@ pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # cffi pycurl==7.45.3 \ --hash=sha256:0c41a172d5e8a5cdd8328cc8134f47b2a57960ac677f7cda8520eaa9fbe7d990 \ @@ -3239,13 +3239,13 @@ pycurl==7.45.3 \ --hash=sha256:fa7751b614d9aa82d7a0f49ca90924c29c6cedf85a2f8687fb6a772dbfe48711 \ --hash=sha256:fbd4a6b8654b779089c5a44af1c65c1419c2cd60718780df6d8f354eb35d6d55 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt pydantic==2.10.0 \ --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # compressed-tensors # fastapi @@ -3357,7 +3357,7 @@ pydantic-core==2.27.0 \ --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # pydantic pydantic-extra-types==2.10.5 \ --hash=sha256:1dcfa2c0cf741a422f088e0dbb4690e7bfadaaf050da3d6f80d6c3cf58a2bad8 \ @@ -3367,7 +3367,7 @@ pygments==2.18.0 \ --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipython # nbconvert # rich @@ -3376,7 +3376,7 @@ pyjwt==2.8.0 \ --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # msal pynvml==12.0.0 \ --hash=sha256:299ce2451a6a17e6822d6faee750103e25b415f06f59abb8db65d30f794166f5 \ @@ -3386,20 +3386,20 @@ pyopenssl==25.0.0 \ --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt pyparsing==3.1.1 \ --hash=sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb \ --hash=sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # httplib2 pytest==7.4.4 \ --hash=sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280 \ --hash=sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/base-test-requirements.txt # -r python/requirements/llm/llm-test-requirements.txt # pytest-aiohttp @@ -3408,20 +3408,20 @@ pytest-aiohttp==1.1.0 \ --hash=sha256:147de8cb164f3fc9d7196967f109ab3c0b93ea3463ab50631e56438eab7b5adc \ --hash=sha256:f39a11693a0dce08dd6c542d241e199dd8047a6e6596b2bcfa60d373f143456d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/base-test-requirements.txt pytest-asyncio==0.17.2 \ --hash=sha256:6d895b02432c028e6957d25fc936494e78c6305736e785d9fee408b1efbc7ff4 \ --hash=sha256:e0fe5dbea40516b661ef1bcfe0bd9461c2847c4ef4bb40012324f2454fb7d56d # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/base-test-requirements.txt # pytest-aiohttp python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # arrow # botocore @@ -3436,7 +3436,7 @@ python-json-logger==2.0.7 \ --hash=sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c \ --hash=sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-events # vllm python-multipart==0.0.20 \ @@ -3447,7 +3447,7 @@ pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # pandas pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ @@ -3502,7 +3502,7 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # gguf @@ -3604,7 +3604,7 @@ pyzmq==26.0.3 \ --hash=sha256:f6b1d1c631e5940cac5a0b22c5379c86e8df6a4ec277c7a856b714021ab6cfad \ --hash=sha256:f6c21c00478a7bea93caaaef9e7629145d4153b15a8653e8bb4609d4bc70dbfc # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipykernel # jupyter-client # jupyter-server @@ -3615,7 +3615,7 @@ referencing==0.36.2 \ --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema # jsonschema-specifications regex==2024.11.6 \ @@ -3721,7 +3721,7 @@ requests==2.32.3 \ --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # azure-core @@ -3741,21 +3741,21 @@ rfc3339-validator==0.1.4 \ --hash=sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b \ --hash=sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema # jupyter-events rfc3986-validator==0.1.1 \ --hash=sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9 \ --hash=sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema # jupyter-events rich==13.3.2 \ --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # memray @@ -3865,21 +3865,21 @@ rpds-py==0.22.3 \ --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema # referencing rsa==4.7.2 \ --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-auth # oauth2client s3transfer==0.6.2 \ --hash=sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084 \ --hash=sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # boto3 safetensors==0.5.2 \ --hash=sha256:03c937100f38c9ff4c1507abea9928a6a9b02c9c1c9c3609ed4fb2bf413d4975 \ @@ -3921,7 +3921,7 @@ scikit-image==0.24.0 \ --hash=sha256:ef04360eda372ee5cd60aebe9be91258639c86ae2ea24093fb9182118008d009 \ --hash=sha256:fa27b3a0dbad807b966b8db2d78da734cb812ca4787f7fbb143764800ce2fa9c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt scipy==1.11.4 \ --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ @@ -3950,7 +3950,7 @@ scipy==1.11.4 \ --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # scikit-image # vllm @@ -3958,7 +3958,7 @@ send2trash==1.8.3 \ --hash=sha256:0c31227e0bd08961c7665474a3d1ef7193929fedda4233843689baa056be46c9 \ --hash=sha256:b18e7a3966d99871aefeb00cfbcfdced55ce4871194810fc71f4aa484b953abf # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-server # nbclassic # notebook @@ -4024,13 +4024,13 @@ shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # typer six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # asttokens # azure-core @@ -4045,20 +4045,20 @@ smart-open==6.2.0 \ --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt smmap==5.0.1 \ --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # gitdb sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # anyio # openai snowballstemmer==2.2.0 \ @@ -4079,7 +4079,7 @@ soupsieve==2.5 \ --hash=sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690 \ --hash=sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # beautifulsoup4 soxr==0.5.0.post1 \ --hash=sha256:39e0f791ba178d69cd676485dbee37e75a34f20daa478d90341ecb7f6d9d690f \ @@ -4136,19 +4136,19 @@ spinners==0.0.24 \ --hash=sha256:1eb6aeb4781d72ab42ed8a01dcf20f3002bf50740d7154d12fb8c9769bf9e27f \ --hash=sha256:2fa30d0b72c9650ad12bbe031c9943b8d441e41b4f5602b0ec977a19f3290e98 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # halo stack-data==0.6.3 \ --hash=sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9 \ --hash=sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipython starlette==0.46.2 \ --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # fastapi # prometheus-fastapi-instrumentator @@ -4160,25 +4160,25 @@ tabulate==0.9.0 \ --hash=sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c \ --hash=sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt tensorboardx==2.6.2.2 \ --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt termcolor==2.4.0 \ --hash=sha256:9297c0df9c99445c2412e832e882a7884038a25617c60cea2ad69488d4040d63 \ --hash=sha256:aab9e56047c8ac41ed798fa36d892a37aca6b3e9159f3e0c24bc64a9b3ac7b7a # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # halo terminado==0.18.1 \ --hash=sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0 \ --hash=sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-server # nbclassic # notebook @@ -4186,7 +4186,7 @@ tifffile==2024.7.21 \ --hash=sha256:7f335b5d6ca49401fe0f1d87deb206f5dae47297e47b1ed52a676d05d6d26798 \ --hash=sha256:818b577d49350421fb511f389f937984f9feaa2cd8177fa00823001920bf3483 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # scikit-image tiktoken==0.9.0 \ --hash=sha256:03935988a91d6d3216e2ec7c645afbb3d870b37bcb67ada1943ec48678e7ee33 \ @@ -4227,7 +4227,7 @@ tinycss2==1.3.0 \ --hash=sha256:152f9acabd296a8375fbca5b84c961ff95971fcfc32e79550c8df8e29118c54d \ --hash=sha256:54a8dbdffb334d536851be0226030e9505965bb2f30f21a4a82c55fb2a80fae7 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbconvert tokenizers==0.21.1 \ --hash=sha256:0f0dcbcc9f6e13e675a66d7a5f2f225a736745ce484c1a4e07476a89ccdad382 \ @@ -4347,7 +4347,7 @@ tornado==6.1 \ --hash=sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68 \ --hash=sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipykernel # jupyter-client # jupyter-server @@ -4359,7 +4359,7 @@ tqdm==4.67.1 \ --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # gguf # huggingface-hub @@ -4370,7 +4370,7 @@ traitlets==5.14.3 \ --hash=sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7 \ --hash=sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # comm # ipykernel # ipython @@ -4411,7 +4411,7 @@ typer==0.12.3 \ --hash=sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914 \ --hash=sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt # fastapi-cli @@ -4419,13 +4419,13 @@ types-python-dateutil==2.9.0.20240316 \ --hash=sha256:5d2f2e240b86905e40944dd787db6da9263f0deabef1076ddaed797351ec0202 \ --hash=sha256:6b8cb66d960771ce5ff974e9dd45e38facb81718cc1e208b10b1baccbfdbee3b # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # arrow typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # azure-core # azure-identity # azure-storage-blob @@ -4449,25 +4449,25 @@ tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # kombu tzlocal==5.3 \ --hash=sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2 \ --hash=sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt uri-template==1.3.0 \ --hash=sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7 \ --hash=sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # botocore # requests @@ -4475,7 +4475,7 @@ uvicorn==0.22.0 \ --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # fastapi # fastapi-cli @@ -4522,7 +4522,7 @@ vine==5.1.0 \ --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # amqp # celery # kombu @@ -4530,7 +4530,7 @@ virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt vllm==0.10.0 \ --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ @@ -4560,7 +4560,7 @@ watchfiles==0.19.0 \ --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # uvicorn # vllm @@ -4568,26 +4568,26 @@ wcwidth==0.2.13 \ --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # prompt-toolkit webcolors==24.6.0 \ --hash=sha256:1d160d1de46b3e81e58d0a280d0c78b467dc80f47294b91b1ad8029d2cedb55b \ --hash=sha256:8cf5bc7e28defd1d48b9e83d5fc30741328305a8195c29a8e668fa45586568a1 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jsonschema webencodings==0.5.1 \ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # bleach # tinycss2 websocket-client==1.8.0 \ --hash=sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526 \ --hash=sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-server websockets==15.0 \ --hash=sha256:0e389efe46ccb25a1f93d08c7a74e8123a2517f7b7458f043bd7529d1a63ffeb \ @@ -4664,7 +4664,7 @@ widgetsnbextension==4.0.11 \ --hash=sha256:55d4d6949d100e0d08b94948a42efc3ed6dfdc0e9468b2c4b128c9a2ce3a7a36 \ --hash=sha256:8b22a8f1910bfd188e596fe7fc05dcbd87e810c8a4ba010bdb3da86637398474 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # ipywidgets wrapt==1.14.1 \ --hash=sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3 \ @@ -4742,7 +4742,7 @@ wrapt==1.14.1 \ --hash=sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015 \ --hash=sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt xformers==0.0.31 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:23331bdb9831ba0df96f55258537ca0df7ad888efc75cea97a0de79b5e2291c4 \ @@ -4853,7 +4853,7 @@ y-py==0.6.2 \ --hash=sha256:e92878cc05e844c8da937204bc34c2e6caf66709ce5936802fbfb35f04132892 \ --hash=sha256:ff32548e45e45bf3280ac1d28b3148337a5c6714c28db23aeb0693e33eba257e # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-ydoc # ypy-websocket yarl==1.18.3 \ @@ -4940,19 +4940,19 @@ yarl==1.18.3 \ --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp ypy-websocket==0.8.4 \ --hash=sha256:43a001473f5c8abcf182f603049cf305cbc855ad8deaa9dfa0f3b5a7cea9d0ff \ --hash=sha256:b1ba0dfcc9762f0ca168d2378062d3ca1299d39076b0f145d961359121042be5 # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # jupyter-server-ydoc zipp==3.19.2 \ --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via - # -c python/requirements_compiled_ray_test_py311_cpu.txt + # -c python/deplocks/llm/ray_test_py311_cpu.lock # importlib-metadata # The following packages were excluded from the output: diff --git a/python/requirements_compiled_rayllm_test_py311_cu121.txt b/python/deplocks/llm/rayllm_test_py311_cu121.lock similarity index 95% rename from python/requirements_compiled_rayllm_test_py311_cu121.txt rename to python/deplocks/llm/rayllm_test_py311_cu121.lock index 70d213fe4e0c..eea2023989b0 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu121.txt +++ b/python/deplocks/llm/rayllm_test_py311_cu121.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cu121.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu121.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu121.lock python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/deplocks/llm/rayllm_test_py311_cu121.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 @@ -7,13 +7,13 @@ aiofiles==22.1.0 \ --hash=sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad \ --hash=sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ypy-websocket aiohappyeyeballs==2.6.1 \ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp aiohttp==3.11.16 \ --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ @@ -98,7 +98,7 @@ aiohttp==3.11.16 \ --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements/llm/llm-test-requirements.txt # -r python/requirements.txt @@ -109,25 +109,25 @@ aiohttp-cors==0.7.0 \ --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt aiorwlock==1.3.0 \ --hash=sha256:45baf8e4fa9a23e0bb325fbd67da80de1fd7ae1d4f59a6381754c60cec7b289b \ --hash=sha256:83f12d87df4b9728a0b8fda1756585ab0d652b107bab59c6084e1b1ad692ab45 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp aiosqlite==0.19.0 \ --hash=sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d \ --hash=sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ypy-websocket alabaster==0.7.16 \ --hash=sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65 \ @@ -137,19 +137,19 @@ amqp==5.3.1 \ --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # pydantic anyio==3.7.1 \ --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # httpx # jupyter-server # openai @@ -159,7 +159,7 @@ argon2-cffi==23.1.0 \ --hash=sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08 \ --hash=sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-server # nbclassic # notebook @@ -186,13 +186,13 @@ argon2-cffi-bindings==21.2.0 \ --hash=sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e \ --hash=sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # argon2-cffi arrow==1.3.0 \ --hash=sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80 \ --hash=sha256:d4540617648cb5f895730f1ad8c82a65f2dad0166f57b75f3ca54759c4d67a85 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # isoduration astor==0.8.1 \ --hash=sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5 \ @@ -202,13 +202,13 @@ asttokens==2.4.1 \ --hash=sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24 \ --hash=sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # stack-data attrs==25.1.0 \ --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp # jsonschema # referencing @@ -216,13 +216,13 @@ azure-common==1.1.28 \ --hash=sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3 \ --hash=sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # smart-open azure-core==1.29.5 \ --hash=sha256:0fa04b7b1f7d44a4fb8468c4093deb2ea01fdf4faddbf802ed9205615f99d68c \ --hash=sha256:52983c89d394c6f881a121e5101c5fa67278ca3b1f339c8fb2ef39230c70e9ac # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # azure-identity # azure-storage-blob # smart-open @@ -230,26 +230,26 @@ azure-identity==1.17.1 \ --hash=sha256:32ecc67cc73f4bd0595e4f64b1ca65cd05186f4fe6f98ed2ae9f1aa32646efea \ --hash=sha256:db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt azure-storage-blob==12.22.0 \ --hash=sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e \ --hash=sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # smart-open babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ --hash=sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyterlab-server # sphinx backcall==0.2.0 \ --hash=sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e \ --hash=sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipython backoff==2.2.1 \ --hash=sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba \ @@ -259,13 +259,13 @@ beautifulsoup4==4.11.1 \ --hash=sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30 \ --hash=sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbconvert billiard==4.2.1 \ --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # celery blake3==1.0.4 \ --hash=sha256:00605aa59923205c6a4f21131840840eb2d9a754c59b163357d890566755b97a \ @@ -358,20 +358,20 @@ bleach==6.1.0 \ --hash=sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe \ --hash=sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbconvert boto3==1.26.76 \ --hash=sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027 \ --hash=sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # smart-open botocore==1.29.76 \ --hash=sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7 \ --hash=sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # boto3 # s3transfer @@ -379,7 +379,7 @@ cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-auth # vllm cbor2==5.6.5 \ @@ -432,13 +432,13 @@ celery==5.5.3 \ --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # httpcore # httpx @@ -497,7 +497,7 @@ cffi==1.16.0 \ --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # argon2-cffi-bindings # cryptography # soundfile @@ -593,13 +593,13 @@ charset-normalizer==3.3.2 \ --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # requests click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # celery @@ -613,32 +613,32 @@ click-didyoumean==0.3.1 \ --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # celery click-plugins==1.1.1.2 \ --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # celery click-repl==0.3.0 \ --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # gymnasium # vllm colorama==0.4.6 \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # halo # log-symbols @@ -646,13 +646,13 @@ colorful==0.5.5 \ --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt comm==0.2.0 \ --hash=sha256:2da8d9ebb8dd7bfc247adaff99f24dce705638a8042b85cb995066793e391001 \ --hash=sha256:a517ea2ca28931c7007a7a99c562a0fa5883cfb48963140cf642c41c948498be # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipykernel # ipywidgets compressed-tensors==0.10.2 \ @@ -698,7 +698,7 @@ cryptography==44.0.3 \ --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # azure-identity # azure-storage-blob # msal @@ -718,7 +718,7 @@ cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # ray debugpy==1.8.0 \ @@ -741,19 +741,19 @@ debugpy==1.8.0 \ --hash=sha256:ef54404365fae8d45cf450d0544ee40cefbcb9cb85ea7afe89a963c27028261e \ --hash=sha256:ef9ab7df0b9a42ed9c878afd3eaaff471fce3fa73df96022e1f5c9f8f8c87ada # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipykernel decorator==5.1.1 \ --hash=sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330 \ --hash=sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipython defusedxml==0.7.1 \ --hash=sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69 \ --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbconvert depyf==0.19.0 \ --hash=sha256:040b35fc0997d49df024b7d094f2a7836f91e9ed02f49982dd37e70aa3285ad5 \ @@ -771,7 +771,7 @@ distlib==0.3.7 \ --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # virtualenv distro==1.9.0 \ --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ @@ -825,7 +825,7 @@ dm-tree==0.1.8 \ --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt dnspython==2.7.0 \ --hash=sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86 \ @@ -847,26 +847,26 @@ entrypoints==0.4 \ --hash=sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4 \ --hash=sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-client # nbconvert executing==2.0.1 \ --hash=sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147 \ --hash=sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # stack-data farama-notifications==0.0.4 \ --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # gymnasium fastapi==0.115.12 \ --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # vllm fastapi-cli==0.0.5 \ @@ -877,7 +877,7 @@ fastjsonschema==2.19.0 \ --hash=sha256:b9fd1a2dd6971dbc7fee280a95bd199ae0dd9ce22beb91cc75e9c1c528a5170e \ --hash=sha256:e25df6647e1bc4a26070b700897b07b542ec898dd4f1f6ea013e7f6a88417225 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbformat fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ @@ -956,13 +956,13 @@ fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # cupy-cuda12x filelock==3.17.0 \ --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # huggingface-hub # ray @@ -974,7 +974,7 @@ fqdn==1.5.1 \ --hash=sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f \ --hash=sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema frozenlist==1.4.1 \ --hash=sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7 \ @@ -1055,14 +1055,14 @@ frozenlist==1.4.1 \ --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp # aiosignal fsspec==2023.5.0 \ --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # huggingface-hub # torch @@ -1074,19 +1074,19 @@ gitdb==4.0.11 \ --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # gitpython gitpython==3.1.44 \ --hash=sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110 \ --hash=sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt google-api-core==2.24.2 \ --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-cloud-core # google-cloud-storage # opencensus @@ -1094,7 +1094,7 @@ google-auth==2.23.4 \ --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # google-api-core # google-cloud-core @@ -1103,13 +1103,13 @@ google-cloud-core==2.4.1 \ --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-cloud-storage google-cloud-storage==2.14.0 \ --hash=sha256:2d23fcf59b55e7b45336729c148bb1c464468c69d5efbaee30f7201dd90eb97e \ --hash=sha256:8641243bbf2a2042c16a6399551fbb13f062cbc9a2de38d6c0bb5426962e9dbd # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # smart-open google-crc32c==1.5.0 \ @@ -1182,20 +1182,20 @@ google-crc32c==1.5.0 \ --hash=sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556 \ --hash=sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-cloud-storage # google-resumable-media google-resumable-media==2.6.0 \ --hash=sha256:972852f6c65f933e15a4a210c2b96930763b47197cdf4aa5f5bea435efb626e7 \ --hash=sha256:fc03d344381970f79eebb632a3c18bb1828593a2dc5572b5f90115ef7d11e81b # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-cloud-storage googleapis-common-protos==1.61.0 \ --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-api-core grpcio==1.66.2 \ --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ @@ -1254,7 +1254,7 @@ grpcio==1.66.2 \ --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # grpcio-tools @@ -1308,26 +1308,26 @@ grpcio-tools==1.62.3 \ --hash=sha256:f4b1615adf67bd8bb71f3464146a6f9949972d06d21a4f5e87e73f6464d97f57 \ --hash=sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt gymnasium==1.1.1 \ --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # httpcore # uvicorn halo==0.0.31 \ --hash=sha256:5350488fb7d2aa7c31a1344120cee67a872901ce8858f60da7946cef96c208ab \ --hash=sha256:7b67a3521ee91d53b7152d4ee3452811e1d2a6321975137762eb3d70063cc9d6 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt hf-transfer==0.1.9 \ --hash=sha256:035572865dab29d17e783fbf1e84cf1cb24f3fcf8f1b17db1cfc7fdf139f02bf \ @@ -1374,7 +1374,7 @@ httplib2==0.20.4 \ --hash=sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585 \ --hash=sha256:8b6a905cb1c79eefd03f8669fd993c36dc341f7c558f056cb5a33b5c2f458543 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # oauth2client httptools==0.6.4 \ --hash=sha256:0614154d5454c21b6410fdf5262b4a3ddb0f53f1e1721cfd59d55f32138c578a \ @@ -1439,13 +1439,13 @@ humanize==4.12.1 \ --hash=sha256:1338ba97415c96556758a6e2f65977ed406dddf4620d4c6db9bbdfd07f0f1232 \ --hash=sha256:86014ca5c52675dffa1d404491952f1f5bf03b07c175a51891a343daebf01fea # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # anyio # email-validator # httpx @@ -1456,7 +1456,7 @@ imageio==2.34.2 \ --hash=sha256:5c0c0ee8faa018a1c42f649b90395dd4d3bb6187c09053a0cd6f1fdd51bbff5e \ --hash=sha256:a0bb27ec9d5bab36a9f4835e51b21d2cb099e1f78451441f94687ff3404b79f8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # scikit-image imagesize==1.4.1 \ --hash=sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b \ @@ -1466,13 +1466,13 @@ importlib-metadata==6.11.0 \ --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # opentelemetry-api iniconfig==2.0.0 \ --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # pytest interegular==0.3.3 \ --hash=sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c \ @@ -1482,14 +1482,14 @@ ipykernel==6.27.1 \ --hash=sha256:7d5d594b6690654b4d299edba5e872dc17bb7396a8d0609c97cb7b8a1c605de6 \ --hash=sha256:dab88b47f112f9f7df62236511023c9bdeef67abc73af7c652e4ce4441601686 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbclassic # notebook ipython==8.12.3 \ --hash=sha256:3910c4b54543c2ad73d06579aa771041b7d5707b033bd488669b4cf544e3b363 \ --hash=sha256:b0340d46a933d27c657b211a329d0be23793c36595acf9e6ef4164bc01a1804c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipykernel # ipywidgets # jupyterlab @@ -1497,38 +1497,38 @@ ipython-genutils==0.2.0 \ --hash=sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8 \ --hash=sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbclassic # notebook ipywidgets==8.1.3 \ --hash=sha256:efafd18f7a142248f7cb0ba890a68b96abd4d6e88ddbda483c9130d12667eaf2 \ --hash=sha256:f5f9eeaae082b1823ce9eac2575272952f40d748893972956dc09700a6392d9c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt isodate==0.6.1 \ --hash=sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96 \ --hash=sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # azure-storage-blob isoduration==20.11.0 \ --hash=sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9 \ --hash=sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema jedi==0.19.1 \ --hash=sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd \ --hash=sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipython jinja2==3.1.6 \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # fastapi # jupyter-server # jupyterlab @@ -1621,26 +1621,26 @@ jmespath==1.0.1 \ --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # boto3 # botocore json5==0.9.14 \ --hash=sha256:740c7f1b9e584a468dbb2939d8d458db3427f2c93ae2139d05f47e453eae964f \ --hash=sha256:9ed66c3a6ca3510a976a9ef9b8c0787de24802724ab1860bc0153c7fdd589b02 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyterlab-server jsonpatch==1.32 \ --hash=sha256:26ac385719ac9f54df8a2f0827bb8253aa3ea8ab7b3368457bcdb8c14595a397 \ --hash=sha256:b6ddfe6c3db30d81a96aaeceb6baf916094ffa23d7dd5fa2c13e13f8b6e600c2 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt jsonpointer==2.4 \ --hash=sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a \ --hash=sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonpatch # jsonschema jsonref==1.1.0 \ @@ -1651,7 +1651,7 @@ jsonschema==4.23.0 \ --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt @@ -1664,13 +1664,13 @@ jsonschema-specifications==2024.10.1 \ --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema jupyter-client==7.3.4 \ --hash=sha256:17d74b0d0a7b24f1c8c527b24fcf4607c56bee542ffe8e3418e50b21e514b621 \ --hash=sha256:aa9a6c32054b290374f95f73bb0cae91455c58dfb84f65c8591912b8f65e6d56 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipykernel # jupyter-server # nbclassic @@ -1680,7 +1680,7 @@ jupyter-core==5.5.0 \ --hash=sha256:880b86053bf298a8724994f95e99b99130659022a4f7f45f563084b6223861d3 \ --hash=sha256:e11e02cd8ae0a9de5c6c44abf5727df9f2581055afe00b22183f621ba3585805 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipykernel # jupyter-client # jupyter-server @@ -1693,13 +1693,13 @@ jupyter-events==0.6.3 \ --hash=sha256:57a2749f87ba387cd1bfd9b22a0875b889237dbf2edc2121ebb22bde47036c17 \ --hash=sha256:9a6e9995f75d1b7146b436ea24d696ce3a35bfa8bfe45e0c33c334c79464d0b3 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-server-fileid jupyter-server==1.24.0 \ --hash=sha256:23368e8e214baf82b313d4c5a0d828ca73015e1a192ce3829bd74e62fab8d046 \ --hash=sha256:c88ddbe862966ea1aea8c3ccb89a5903abd8fbcfe5cd14090ef549d403332c37 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-server-fileid # jupyterlab # jupyterlab-server @@ -1709,44 +1709,44 @@ jupyter-server-fileid==0.9.0 \ --hash=sha256:171538b7c7d08d11dbc57d4e6da196e0c258e4c2cd29249ef1e032bb423677f8 \ --hash=sha256:5b489c6fe6783c41174a728c7b81099608518387e53c3d53451a67f46a0cb7b0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-server-ydoc jupyter-server-ydoc==0.6.1 \ --hash=sha256:18275ff1ce7e93bbda2301ca066273b3951fc50b0d9c8fc33788374134ad7920 \ --hash=sha256:ab10864708c81fa41ab9f2ed3626b54ff6926eaf14545d1d439714978dad6e9f # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyterlab jupyter-ydoc==0.2.5 \ --hash=sha256:5759170f112c70320a84217dd98d287699076ae65a7f88d458d57940a9f2b882 \ --hash=sha256:5a02ca7449f0d875f73e8cb8efdf695dddef15a8e71378b1f4eda6b7c90f5382 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-server-ydoc # jupyterlab jupyterlab==3.6.1 \ --hash=sha256:ad6707dd0149b629d0ed5b56916cfcdb816b376c6af3190337faba09e27ea29e \ --hash=sha256:aee98c174180e98a30470297d10b959e8e64f2288970c0de65f0a6d2b4807034 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt jupyterlab-pygments==0.3.0 \ --hash=sha256:721aca4d9029252b11cfa9d185e5b5af4d54772bb8072f9b7036f4170054d35d \ --hash=sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbconvert jupyterlab-server==2.24.0 \ --hash=sha256:4e6f99e0a5579bbbc32e449c4dbb039561d4f1a7827d5733273ed56738f21f07 \ --hash=sha256:5f077e142bb8dc9b843d960f940c513581bceca3793a0d80f9c67d9522c4e876 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyterlab jupyterlab-widgets==3.0.11 \ --hash=sha256:78287fd86d20744ace330a61625024cf5521e1c012a352ddc0a3cdc2348becd0 \ --hash=sha256:dd5ac679593c969af29c9bed054c24f26842baa51352114736756bc035deee27 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipywidgets jupytext==1.16.7 \ --hash=sha256:912f9d9af7bd3f15470105e5c5dddf1669b2d8c17f0c55772687fc5a4a73fe69 \ @@ -1756,7 +1756,7 @@ kombu==5.5.4 \ --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ @@ -1766,7 +1766,7 @@ lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # scikit-image llguidance==0.7.26 ; platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:1895ff449c8ec0a5f1d3b142d723fc9b26a85b021b72d7f1173f8b7507f528c0 \ @@ -1809,7 +1809,7 @@ log-symbols==0.0.14 \ --hash=sha256:4952106ff8b605ab7d5081dd2c7e6ca7374584eff7086f499c06edd1ce56dcca \ --hash=sha256:cf0bbc6fe1a8e53f0d174a716bc625c4f87043cc21eb55dd8a740cfe22680556 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # halo lxml==4.9.4 \ --hash=sha256:00e91573183ad273e242db5585b52670eddf92bacad095ce25c1e682da14ed91 \ @@ -1906,7 +1906,7 @@ lxml==4.9.4 \ --hash=sha256:fd814847901df6e8de13ce69b84c31fc9b3fb591224d6762d0b256d510cbf382 \ --hash=sha256:fdb325b7fba1e2c40b9b1db407f85642e32404131c08480dd652110fc908561b # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbconvert lz4==4.3.3 \ --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ @@ -1946,13 +1946,13 @@ lz4==4.3.3 \ --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt markdown-it-py==2.2.0 \ --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupytext # mdit-py-plugins # rich @@ -2018,14 +2018,14 @@ markupsafe==2.1.3 \ --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jinja2 # nbconvert matplotlib-inline==0.1.6 \ --hash=sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311 \ --hash=sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipykernel # ipython mdit-py-plugins==0.4.2 \ @@ -2036,7 +2036,7 @@ mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # markdown-it-py memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ @@ -2075,7 +2075,7 @@ memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt meson==1.8.3 \ --hash=sha256:ef02b806ce0c5b6becd5bb5dc9fa67662320b29b337e7ace73e4354500590233 \ @@ -2089,7 +2089,7 @@ mistune==0.8.4 \ --hash=sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e \ --hash=sha256:88a1051873018da288eee8538d476dffe1262495144b33ecb586c4ab266bb8d4 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbconvert mpmath==1.3.0 \ --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ @@ -2099,14 +2099,14 @@ msal==1.28.1 \ --hash=sha256:563c2d70de77a2ca9786aab84cb4e133a38a6897e6676774edc23d610bfc9e7b \ --hash=sha256:d72bbfe2d5c2f2555f4bc6205be4450ddfd12976610dd9a16a9ab0f05c68b64d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # azure-identity # msal-extensions msal-extensions==1.2.0b1 \ --hash=sha256:217f391bb549de11b19abe8029a8375fe3ca0556aa8cce004b2083f00a569b71 \ --hash=sha256:3658b3814cd6a7759e83cb0ec145f30330ee249a92444adaf9aa4eb4f5bbcbbc # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # azure-identity msgpack==1.0.7 \ --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ @@ -2166,7 +2166,7 @@ msgpack==1.0.7 \ --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # ray msgspec==0.19.0 \ @@ -2299,27 +2299,27 @@ multidict==6.0.5 \ --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp # yarl nbclassic==1.0.0 \ --hash=sha256:0ae11eb2319455d805596bf320336cda9554b41d99ab9a3c31bf8180bffa30e3 \ --hash=sha256:f99e4769b4750076cd4235c044b61232110733322384a94a63791d2e7beacc66 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyterlab # notebook nbclient==0.5.13 \ --hash=sha256:40c52c9b5e3c31faecaee69f202b3f53e38d7c1c563de0fadde9d7eda0fdafe8 \ --hash=sha256:47ac905af59379913c1f8f541098d2550153cf8dc58553cbe18c702b181518b0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbconvert nbconvert==6.5.4 \ --hash=sha256:9e3c7c6d491374cbdd5f35d268c05809357716d346f4573186bbeab32ee50bc1 \ --hash=sha256:d679a947f849a966cbbd0bf6e7fedcfdb64be3b20ce7cef11ad55c13f5820e19 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-server # nbclassic # notebook @@ -2327,7 +2327,7 @@ nbformat==5.9.2 \ --hash=sha256:1c5172d786a41b82bcfd0c23f9e6b6f072e8fb49c39250219e4acfff1efe89e9 \ --hash=sha256:5f98b5ba1997dff175e77e0c17d5c10a96eaed2cbd1de3533d1fc35d5e111192 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-server # jupytext # nbclassic @@ -2338,7 +2338,7 @@ nest-asyncio==1.5.8 \ --hash=sha256:25aa2ca0d2a5b5531956b9e273b45cf664cae2b145101d73b86b199978d48fdb \ --hash=sha256:accda7a339a70599cb08f9dd09a67e0c2ef8d8d6f4c07f96ab203f2ae254e48d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipykernel # jupyter-client # nbclassic @@ -2348,7 +2348,7 @@ networkx==3.2.1 \ --hash=sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6 \ --hash=sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # scikit-image # torch ninja==1.11.1.3 \ @@ -2387,13 +2387,13 @@ notebook==6.5.7 \ --hash=sha256:04eb9011dfac634fbd4442adaf0a8c27cd26beef831fe1d19faf930c327768e4 \ --hash=sha256:a6afa9a4ff4d149a0771ff8b8c881a7a73b3835f9add0606696d6e9d98ac1cd0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyterlab notebook-shim==0.2.3 \ --hash=sha256:a83496a43341c1674b093bfcebf0fe8e74cbe7eda5fd2bbc56f8e39e1486c0c7 \ --hash=sha256:f69388ac283ae008cd506dda10d0288b09a017d822d5e8c7129a152cbd3ce7e9 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbclassic numba==0.61.2 \ --hash=sha256:34fba9406078bac7ab052efbf0d13939426c753ad72946baaa5bf9ae0ebb8dd2 \ @@ -2456,7 +2456,7 @@ numpy==1.26.4 \ --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # cupy-cuda12x # gguf @@ -2576,7 +2576,7 @@ oauth2client==4.1.3 \ --hash=sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac \ --hash=sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt openai==1.90.0 \ --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ @@ -2586,13 +2586,13 @@ opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt opencensus-context==0.1.3 \ --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # opencensus opencv-python-headless==4.11.0.86 \ --hash=sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b \ @@ -2609,7 +2609,7 @@ opentelemetry-api==1.34.1 \ --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # opentelemetry-sdk @@ -2618,26 +2618,26 @@ opentelemetry-exporter-prometheus==0.55b1 \ --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt opentelemetry-proto==1.27.0 \ --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt opentelemetry-sdk==1.34.1 \ --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus opentelemetry-semantic-conventions==0.55b1 \ --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # opentelemetry-sdk outlines-core==0.2.10 \ --hash=sha256:0a9e4b192ca837a472a1bb1428397509f543db08e1aeeee30252525cec34093a \ @@ -2686,7 +2686,7 @@ packaging==23.0 \ --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # huggingface-hub @@ -2734,19 +2734,19 @@ pandas==1.5.3 \ --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt pandocfilters==1.5.0 \ --hash=sha256:0b679503337d233b4339a817bfc8c50064e2eff681314376a47cb582305a7a38 \ --hash=sha256:33aae3f25fd1a026079f5d27bdd52496f0e0803b3469282162bafdcbdf6ef14f # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbconvert parso==0.8.3 \ --hash=sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0 \ --hash=sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jedi partial-json-parser==0.2.1.1.post5 \ --hash=sha256:627715aaa3cb3fb60a65b0d62223243acaa6c70846520a90326fef3a2f0b61ca \ @@ -2756,19 +2756,19 @@ pathspec==0.11.2 \ --hash=sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20 \ --hash=sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt pexpect==4.8.0 ; sys_platform != 'win32' \ --hash=sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937 \ --hash=sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipython pickleshare==0.7.5 \ --hash=sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca \ --hash=sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipython pillow==10.3.0 \ --hash=sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c \ @@ -2841,7 +2841,7 @@ pillow==10.3.0 \ --hash=sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27 \ --hash=sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/llm/llm-test-requirements.txt # imageio # mistral-common @@ -2852,26 +2852,26 @@ platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-core # virtualenv pluggy==1.3.0 \ --hash=sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12 \ --hash=sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # pytest portalocker==2.8.2 \ --hash=sha256:2b035aa7828e46c58e9b31390ee1f169b98e1066ab10b9a6a861fe7e25ee4f33 \ --hash=sha256:cfb86acc09b9aa7c3b43594e19be1345b9d16af3feb08bf92f23d4dce513a28e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # msal-extensions prometheus-client==0.19.0 \ --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # jupyter-server # nbclassic @@ -2887,7 +2887,7 @@ prompt-toolkit==3.0.41 \ --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # click-repl # ipython propcache==0.3.0 \ @@ -2990,14 +2990,14 @@ propcache==0.3.0 \ --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp # yarl proto-plus==1.22.3 \ --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-api-core protobuf==4.25.8 \ --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ @@ -3012,7 +3012,7 @@ protobuf==4.25.8 \ --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # google-api-core # googleapis-common-protos @@ -3040,21 +3040,21 @@ psutil==5.9.6 \ --hash=sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d \ --hash=sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipykernel # vllm ptyprocess==0.7.0 ; os_name != 'nt' or sys_platform != 'win32' \ --hash=sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35 \ --hash=sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # pexpect # terminado pure-eval==0.2.2 \ --hash=sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350 \ --hash=sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # stack-data py-cpuinfo==9.0.0 \ --hash=sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690 \ @@ -3070,7 +3070,7 @@ py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt pyarrow==19.0.1 \ --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ @@ -3116,13 +3116,13 @@ pyarrow==19.0.1 \ --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt pyasn1==0.5.1 \ --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # oauth2client # pyasn1-modules # rsa @@ -3130,7 +3130,7 @@ pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-auth # oauth2client pybase64==1.4.1 \ @@ -3290,7 +3290,7 @@ pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # cffi pycurl==7.45.3 \ --hash=sha256:0c41a172d5e8a5cdd8328cc8134f47b2a57960ac677f7cda8520eaa9fbe7d990 \ @@ -3330,13 +3330,13 @@ pycurl==7.45.3 \ --hash=sha256:fa7751b614d9aa82d7a0f49ca90924c29c6cedf85a2f8687fb6a772dbfe48711 \ --hash=sha256:fbd4a6b8654b779089c5a44af1c65c1419c2cd60718780df6d8f354eb35d6d55 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt pydantic==2.10.0 \ --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # compressed-tensors # fastapi @@ -3448,7 +3448,7 @@ pydantic-core==2.27.0 \ --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # pydantic pydantic-extra-types==2.10.5 \ --hash=sha256:1dcfa2c0cf741a422f088e0dbb4690e7bfadaaf050da3d6f80d6c3cf58a2bad8 \ @@ -3458,7 +3458,7 @@ pygments==2.18.0 \ --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipython # nbconvert # rich @@ -3467,7 +3467,7 @@ pyjwt==2.8.0 \ --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # msal pynvml==12.0.0 \ --hash=sha256:299ce2451a6a17e6822d6faee750103e25b415f06f59abb8db65d30f794166f5 \ @@ -3477,20 +3477,20 @@ pyopenssl==25.0.0 \ --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt pyparsing==3.1.1 \ --hash=sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb \ --hash=sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # httplib2 pytest==7.4.4 \ --hash=sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280 \ --hash=sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/base-test-requirements.txt # -r python/requirements/llm/llm-test-requirements.txt # pytest-aiohttp @@ -3499,20 +3499,20 @@ pytest-aiohttp==1.1.0 \ --hash=sha256:147de8cb164f3fc9d7196967f109ab3c0b93ea3463ab50631e56438eab7b5adc \ --hash=sha256:f39a11693a0dce08dd6c542d241e199dd8047a6e6596b2bcfa60d373f143456d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/base-test-requirements.txt pytest-asyncio==0.17.2 \ --hash=sha256:6d895b02432c028e6957d25fc936494e78c6305736e785d9fee408b1efbc7ff4 \ --hash=sha256:e0fe5dbea40516b661ef1bcfe0bd9461c2847c4ef4bb40012324f2454fb7d56d # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/base-test-requirements.txt # pytest-aiohttp python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # arrow # botocore @@ -3527,7 +3527,7 @@ python-json-logger==2.0.7 \ --hash=sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c \ --hash=sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-events # vllm python-multipart==0.0.20 \ @@ -3538,7 +3538,7 @@ pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # pandas pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ @@ -3593,7 +3593,7 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # gguf @@ -3695,7 +3695,7 @@ pyzmq==26.0.3 \ --hash=sha256:f6b1d1c631e5940cac5a0b22c5379c86e8df6a4ec277c7a856b714021ab6cfad \ --hash=sha256:f6c21c00478a7bea93caaaef9e7629145d4153b15a8653e8bb4609d4bc70dbfc # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipykernel # jupyter-client # jupyter-server @@ -3706,7 +3706,7 @@ referencing==0.36.2 \ --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema # jsonschema-specifications regex==2024.11.6 \ @@ -3812,7 +3812,7 @@ requests==2.32.3 \ --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # azure-core @@ -3832,21 +3832,21 @@ rfc3339-validator==0.1.4 \ --hash=sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b \ --hash=sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema # jupyter-events rfc3986-validator==0.1.1 \ --hash=sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9 \ --hash=sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema # jupyter-events rich==13.3.2 \ --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # memray @@ -3956,21 +3956,21 @@ rpds-py==0.22.3 \ --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema # referencing rsa==4.7.2 \ --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-auth # oauth2client s3transfer==0.6.2 \ --hash=sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084 \ --hash=sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # boto3 safetensors==0.5.2 \ --hash=sha256:03c937100f38c9ff4c1507abea9928a6a9b02c9c1c9c3609ed4fb2bf413d4975 \ @@ -4012,7 +4012,7 @@ scikit-image==0.24.0 \ --hash=sha256:ef04360eda372ee5cd60aebe9be91258639c86ae2ea24093fb9182118008d009 \ --hash=sha256:fa27b3a0dbad807b966b8db2d78da734cb812ca4787f7fbb143764800ce2fa9c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt scipy==1.11.4 \ --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ @@ -4041,7 +4041,7 @@ scipy==1.11.4 \ --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # scikit-image # vllm @@ -4049,7 +4049,7 @@ send2trash==1.8.3 \ --hash=sha256:0c31227e0bd08961c7665474a3d1ef7193929fedda4233843689baa056be46c9 \ --hash=sha256:b18e7a3966d99871aefeb00cfbcfdced55ce4871194810fc71f4aa484b953abf # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-server # nbclassic # notebook @@ -4115,13 +4115,13 @@ shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # typer six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # asttokens # azure-core @@ -4136,20 +4136,20 @@ smart-open==6.2.0 \ --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt smmap==5.0.1 \ --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # gitdb sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # anyio # openai snowballstemmer==2.2.0 \ @@ -4170,7 +4170,7 @@ soupsieve==2.5 \ --hash=sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690 \ --hash=sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # beautifulsoup4 soxr==0.5.0.post1 \ --hash=sha256:39e0f791ba178d69cd676485dbee37e75a34f20daa478d90341ecb7f6d9d690f \ @@ -4227,19 +4227,19 @@ spinners==0.0.24 \ --hash=sha256:1eb6aeb4781d72ab42ed8a01dcf20f3002bf50740d7154d12fb8c9769bf9e27f \ --hash=sha256:2fa30d0b72c9650ad12bbe031c9943b8d441e41b4f5602b0ec977a19f3290e98 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # halo stack-data==0.6.3 \ --hash=sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9 \ --hash=sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipython starlette==0.46.2 \ --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # fastapi # prometheus-fastapi-instrumentator @@ -4251,25 +4251,25 @@ tabulate==0.9.0 \ --hash=sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c \ --hash=sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt tensorboardx==2.6.2.2 \ --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt termcolor==2.4.0 \ --hash=sha256:9297c0df9c99445c2412e832e882a7884038a25617c60cea2ad69488d4040d63 \ --hash=sha256:aab9e56047c8ac41ed798fa36d892a37aca6b3e9159f3e0c24bc64a9b3ac7b7a # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # halo terminado==0.18.1 \ --hash=sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0 \ --hash=sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-server # nbclassic # notebook @@ -4277,7 +4277,7 @@ tifffile==2024.7.21 \ --hash=sha256:7f335b5d6ca49401fe0f1d87deb206f5dae47297e47b1ed52a676d05d6d26798 \ --hash=sha256:818b577d49350421fb511f389f937984f9feaa2cd8177fa00823001920bf3483 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # scikit-image tiktoken==0.9.0 \ --hash=sha256:03935988a91d6d3216e2ec7c645afbb3d870b37bcb67ada1943ec48678e7ee33 \ @@ -4318,7 +4318,7 @@ tinycss2==1.3.0 \ --hash=sha256:152f9acabd296a8375fbca5b84c961ff95971fcfc32e79550c8df8e29118c54d \ --hash=sha256:54a8dbdffb334d536851be0226030e9505965bb2f30f21a4a82c55fb2a80fae7 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbconvert tokenizers==0.21.1 \ --hash=sha256:0f0dcbcc9f6e13e675a66d7a5f2f225a736745ce484c1a4e07476a89ccdad382 \ @@ -4467,7 +4467,7 @@ tornado==6.1 \ --hash=sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68 \ --hash=sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipykernel # jupyter-client # jupyter-server @@ -4479,7 +4479,7 @@ tqdm==4.67.1 \ --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # gguf # huggingface-hub @@ -4490,7 +4490,7 @@ traitlets==5.14.3 \ --hash=sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7 \ --hash=sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # comm # ipykernel # ipython @@ -4522,7 +4522,7 @@ typer==0.12.3 \ --hash=sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914 \ --hash=sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt # fastapi-cli @@ -4530,13 +4530,13 @@ types-python-dateutil==2.9.0.20240316 \ --hash=sha256:5d2f2e240b86905e40944dd787db6da9263f0deabef1076ddaed797351ec0202 \ --hash=sha256:6b8cb66d960771ce5ff974e9dd45e38facb81718cc1e208b10b1baccbfdbee3b # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # arrow typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # azure-core # azure-identity # azure-storage-blob @@ -4560,25 +4560,25 @@ tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # kombu tzlocal==5.3 \ --hash=sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2 \ --hash=sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt uri-template==1.3.0 \ --hash=sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7 \ --hash=sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # botocore # requests @@ -4586,7 +4586,7 @@ uvicorn==0.22.0 \ --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # fastapi # fastapi-cli @@ -4633,7 +4633,7 @@ vine==5.1.0 \ --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # amqp # celery # kombu @@ -4641,7 +4641,7 @@ virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt vllm==0.10.0 \ --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ @@ -4671,7 +4671,7 @@ watchfiles==0.19.0 \ --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # uvicorn # vllm @@ -4679,26 +4679,26 @@ wcwidth==0.2.13 \ --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # prompt-toolkit webcolors==24.6.0 \ --hash=sha256:1d160d1de46b3e81e58d0a280d0c78b467dc80f47294b91b1ad8029d2cedb55b \ --hash=sha256:8cf5bc7e28defd1d48b9e83d5fc30741328305a8195c29a8e668fa45586568a1 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jsonschema webencodings==0.5.1 \ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # bleach # tinycss2 websocket-client==1.8.0 \ --hash=sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526 \ --hash=sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-server websockets==15.0 \ --hash=sha256:0e389efe46ccb25a1f93d08c7a74e8123a2517f7b7458f043bd7529d1a63ffeb \ @@ -4775,7 +4775,7 @@ widgetsnbextension==4.0.11 \ --hash=sha256:55d4d6949d100e0d08b94948a42efc3ed6dfdc0e9468b2c4b128c9a2ce3a7a36 \ --hash=sha256:8b22a8f1910bfd188e596fe7fc05dcbd87e810c8a4ba010bdb3da86637398474 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # ipywidgets wrapt==1.14.1 \ --hash=sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3 \ @@ -4853,7 +4853,7 @@ wrapt==1.14.1 \ --hash=sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015 \ --hash=sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt xformers==0.0.31 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:23331bdb9831ba0df96f55258537ca0df7ad888efc75cea97a0de79b5e2291c4 \ @@ -4964,7 +4964,7 @@ y-py==0.6.2 \ --hash=sha256:e92878cc05e844c8da937204bc34c2e6caf66709ce5936802fbfb35f04132892 \ --hash=sha256:ff32548e45e45bf3280ac1d28b3148337a5c6714c28db23aeb0693e33eba257e # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-ydoc # ypy-websocket yarl==1.18.3 \ @@ -5051,19 +5051,19 @@ yarl==1.18.3 \ --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp ypy-websocket==0.8.4 \ --hash=sha256:43a001473f5c8abcf182f603049cf305cbc855ad8deaa9dfa0f3b5a7cea9d0ff \ --hash=sha256:b1ba0dfcc9762f0ca168d2378062d3ca1299d39076b0f145d961359121042be5 # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # jupyter-server-ydoc zipp==3.19.2 \ --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via - # -c python/requirements_compiled_ray_test_py311_cu121.txt + # -c python/deplocks/llm/ray_test_py311_cu121.lock # importlib-metadata # The following packages were excluded from the output: diff --git a/python/requirements_compiled_rayllm_test_py311_cu128.txt b/python/deplocks/llm/rayllm_test_py311_cu128.lock similarity index 95% rename from python/requirements_compiled_rayllm_test_py311_cu128.txt rename to python/deplocks/llm/rayllm_test_py311_cu128.lock index c8ee9734ba19..f750627c7e8a 100644 --- a/python/requirements_compiled_rayllm_test_py311_cu128.txt +++ b/python/deplocks/llm/rayllm_test_py311_cu128.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/requirements_compiled_ray_test_py311_cu128.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/requirements_compiled_rayllm_test_py311_cu128.txt +# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu128.lock python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/deplocks/llm/rayllm_test_py311_cu128.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 @@ -7,13 +7,13 @@ aiofiles==22.1.0 \ --hash=sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad \ --hash=sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ypy-websocket aiohappyeyeballs==2.6.1 \ --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp aiohttp==3.11.16 \ --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ @@ -98,7 +98,7 @@ aiohttp==3.11.16 \ --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements/llm/llm-test-requirements.txt # -r python/requirements.txt @@ -109,25 +109,25 @@ aiohttp-cors==0.7.0 \ --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt aiorwlock==1.3.0 \ --hash=sha256:45baf8e4fa9a23e0bb325fbd67da80de1fd7ae1d4f59a6381754c60cec7b289b \ --hash=sha256:83f12d87df4b9728a0b8fda1756585ab0d652b107bab59c6084e1b1ad692ab45 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt aiosignal==1.3.1 \ --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp aiosqlite==0.19.0 \ --hash=sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d \ --hash=sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ypy-websocket alabaster==0.7.16 \ --hash=sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65 \ @@ -137,19 +137,19 @@ amqp==5.3.1 \ --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # kombu annotated-types==0.6.0 \ --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # pydantic anyio==3.7.1 \ --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # httpx # jupyter-server # openai @@ -159,7 +159,7 @@ argon2-cffi==23.1.0 \ --hash=sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08 \ --hash=sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-server # nbclassic # notebook @@ -186,13 +186,13 @@ argon2-cffi-bindings==21.2.0 \ --hash=sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e \ --hash=sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # argon2-cffi arrow==1.3.0 \ --hash=sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80 \ --hash=sha256:d4540617648cb5f895730f1ad8c82a65f2dad0166f57b75f3ca54759c4d67a85 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # isoduration astor==0.8.1 \ --hash=sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5 \ @@ -202,13 +202,13 @@ asttokens==2.4.1 \ --hash=sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24 \ --hash=sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # stack-data attrs==25.1.0 \ --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp # jsonschema # referencing @@ -216,13 +216,13 @@ azure-common==1.1.28 \ --hash=sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3 \ --hash=sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # smart-open azure-core==1.29.5 \ --hash=sha256:0fa04b7b1f7d44a4fb8468c4093deb2ea01fdf4faddbf802ed9205615f99d68c \ --hash=sha256:52983c89d394c6f881a121e5101c5fa67278ca3b1f339c8fb2ef39230c70e9ac # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # azure-identity # azure-storage-blob # smart-open @@ -230,26 +230,26 @@ azure-identity==1.17.1 \ --hash=sha256:32ecc67cc73f4bd0595e4f64b1ca65cd05186f4fe6f98ed2ae9f1aa32646efea \ --hash=sha256:db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt azure-storage-blob==12.22.0 \ --hash=sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e \ --hash=sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # smart-open babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ --hash=sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyterlab-server # sphinx backcall==0.2.0 \ --hash=sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e \ --hash=sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipython backoff==2.2.1 \ --hash=sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba \ @@ -259,13 +259,13 @@ beautifulsoup4==4.11.1 \ --hash=sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30 \ --hash=sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbconvert billiard==4.2.1 \ --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # celery blake3==1.0.5 \ --hash=sha256:03638a6dc8546365c3576fdb293fb2c53b898ac80525b5742d9cf00b4f44dea5 \ @@ -358,20 +358,20 @@ bleach==6.1.0 \ --hash=sha256:0a31f1837963c41d46bbf1331b8778e1308ea0791db03cc4e7357b97cf42a8fe \ --hash=sha256:3225f354cfc436b9789c66c4ee030194bee0568fbf9cbdad3bc8b5c26c5f12b6 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbconvert boto3==1.26.76 \ --hash=sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027 \ --hash=sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # smart-open botocore==1.29.76 \ --hash=sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7 \ --hash=sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # boto3 # s3transfer @@ -379,7 +379,7 @@ cachetools==5.5.2 \ --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-auth # vllm cbor2==5.6.5 \ @@ -432,13 +432,13 @@ celery==5.5.3 \ --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt certifi==2025.1.31 \ --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # httpcore # httpx @@ -497,7 +497,7 @@ cffi==1.16.0 \ --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # argon2-cffi-bindings # cryptography # soundfile @@ -593,13 +593,13 @@ charset-normalizer==3.3.2 \ --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # requests click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # celery @@ -613,31 +613,31 @@ click-didyoumean==0.3.1 \ --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # celery click-plugins==1.1.1.2 \ --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # celery click-repl==0.3.0 \ --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # celery cloudpickle==2.2.0 \ --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # gymnasium # vllm colorama==0.4.6 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # halo # log-symbols @@ -645,13 +645,13 @@ colorful==0.5.5 \ --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt comm==0.2.0 \ --hash=sha256:2da8d9ebb8dd7bfc247adaff99f24dce705638a8042b85cb995066793e391001 \ --hash=sha256:a517ea2ca28931c7007a7a99c562a0fa5883cfb48963140cf642c41c948498be # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipykernel # ipywidgets compressed-tensors==0.10.2 \ @@ -697,7 +697,7 @@ cryptography==44.0.3 \ --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # azure-identity # azure-storage-blob # msal @@ -717,7 +717,7 @@ cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # ray debugpy==1.8.0 \ @@ -740,19 +740,19 @@ debugpy==1.8.0 \ --hash=sha256:ef54404365fae8d45cf450d0544ee40cefbcb9cb85ea7afe89a963c27028261e \ --hash=sha256:ef9ab7df0b9a42ed9c878afd3eaaff471fce3fa73df96022e1f5c9f8f8c87ada # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipykernel decorator==5.1.1 \ --hash=sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330 \ --hash=sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipython defusedxml==0.7.1 \ --hash=sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69 \ --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbconvert depyf==0.19.0 \ --hash=sha256:040b35fc0997d49df024b7d094f2a7836f91e9ed02f49982dd37e70aa3285ad5 \ @@ -770,7 +770,7 @@ distlib==0.3.7 \ --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # virtualenv distro==1.9.0 \ --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ @@ -824,7 +824,7 @@ dm-tree==0.1.8 \ --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt dnspython==2.7.0 \ --hash=sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86 \ @@ -846,26 +846,26 @@ entrypoints==0.4 \ --hash=sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4 \ --hash=sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-client # nbconvert executing==2.0.1 \ --hash=sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147 \ --hash=sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # stack-data farama-notifications==0.0.4 \ --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # gymnasium fastapi==0.115.12 \ --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # vllm fastapi-cli==0.0.5 \ @@ -876,7 +876,7 @@ fastjsonschema==2.19.0 \ --hash=sha256:b9fd1a2dd6971dbc7fee280a95bd199ae0dd9ce22beb91cc75e9c1c528a5170e \ --hash=sha256:e25df6647e1bc4a26070b700897b07b542ec898dd4f1f6ea013e7f6a88417225 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbformat fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ @@ -955,13 +955,13 @@ fastrlock==0.8.2 ; sys_platform != 'darwin' \ --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # cupy-cuda12x filelock==3.17.0 \ --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # huggingface-hub # ray @@ -973,7 +973,7 @@ fqdn==1.5.1 \ --hash=sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f \ --hash=sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema frozenlist==1.4.1 \ --hash=sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7 \ @@ -1054,14 +1054,14 @@ frozenlist==1.4.1 \ --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp # aiosignal fsspec==2023.5.0 \ --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # huggingface-hub # torch @@ -1073,19 +1073,19 @@ gitdb==4.0.11 \ --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # gitpython gitpython==3.1.44 \ --hash=sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110 \ --hash=sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt google-api-core==2.24.2 \ --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-cloud-core # google-cloud-storage # opencensus @@ -1093,7 +1093,7 @@ google-auth==2.23.4 \ --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # google-api-core # google-cloud-core @@ -1102,13 +1102,13 @@ google-cloud-core==2.4.1 \ --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-cloud-storage google-cloud-storage==2.14.0 \ --hash=sha256:2d23fcf59b55e7b45336729c148bb1c464468c69d5efbaee30f7201dd90eb97e \ --hash=sha256:8641243bbf2a2042c16a6399551fbb13f062cbc9a2de38d6c0bb5426962e9dbd # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # smart-open google-crc32c==1.5.0 \ @@ -1181,20 +1181,20 @@ google-crc32c==1.5.0 \ --hash=sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556 \ --hash=sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-cloud-storage # google-resumable-media google-resumable-media==2.6.0 \ --hash=sha256:972852f6c65f933e15a4a210c2b96930763b47197cdf4aa5f5bea435efb626e7 \ --hash=sha256:fc03d344381970f79eebb632a3c18bb1828593a2dc5572b5f90115ef7d11e81b # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-cloud-storage googleapis-common-protos==1.61.0 \ --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-api-core grpcio==1.66.2 \ --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ @@ -1253,7 +1253,7 @@ grpcio==1.66.2 \ --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # grpcio-tools @@ -1307,26 +1307,26 @@ grpcio-tools==1.62.3 \ --hash=sha256:f4b1615adf67bd8bb71f3464146a6f9949972d06d21a4f5e87e73f6464d97f57 \ --hash=sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt gymnasium==1.1.1 \ --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # httpcore # uvicorn halo==0.0.31 \ --hash=sha256:5350488fb7d2aa7c31a1344120cee67a872901ce8858f60da7946cef96c208ab \ --hash=sha256:7b67a3521ee91d53b7152d4ee3452811e1d2a6321975137762eb3d70063cc9d6 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt hf-transfer==0.1.9 \ --hash=sha256:035572865dab29d17e783fbf1e84cf1cb24f3fcf8f1b17db1cfc7fdf139f02bf \ @@ -1373,7 +1373,7 @@ httplib2==0.20.4 \ --hash=sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585 \ --hash=sha256:8b6a905cb1c79eefd03f8669fd993c36dc341f7c558f056cb5a33b5c2f458543 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # oauth2client httptools==0.6.4 \ --hash=sha256:0614154d5454c21b6410fdf5262b4a3ddb0f53f1e1721cfd59d55f32138c578a \ @@ -1438,13 +1438,13 @@ humanize==4.12.1 \ --hash=sha256:1338ba97415c96556758a6e2f65977ed406dddf4620d4c6db9bbdfd07f0f1232 \ --hash=sha256:86014ca5c52675dffa1d404491952f1f5bf03b07c175a51891a343daebf01fea # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # anyio # email-validator # httpx @@ -1455,7 +1455,7 @@ imageio==2.34.2 \ --hash=sha256:5c0c0ee8faa018a1c42f649b90395dd4d3bb6187c09053a0cd6f1fdd51bbff5e \ --hash=sha256:a0bb27ec9d5bab36a9f4835e51b21d2cb099e1f78451441f94687ff3404b79f8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # scikit-image imagesize==1.4.1 \ --hash=sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b \ @@ -1465,13 +1465,13 @@ importlib-metadata==6.11.0 \ --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # opentelemetry-api iniconfig==2.0.0 \ --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # pytest interegular==0.3.3 \ --hash=sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c \ @@ -1481,14 +1481,14 @@ ipykernel==6.27.1 \ --hash=sha256:7d5d594b6690654b4d299edba5e872dc17bb7396a8d0609c97cb7b8a1c605de6 \ --hash=sha256:dab88b47f112f9f7df62236511023c9bdeef67abc73af7c652e4ce4441601686 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbclassic # notebook ipython==8.12.3 \ --hash=sha256:3910c4b54543c2ad73d06579aa771041b7d5707b033bd488669b4cf544e3b363 \ --hash=sha256:b0340d46a933d27c657b211a329d0be23793c36595acf9e6ef4164bc01a1804c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipykernel # ipywidgets # jupyterlab @@ -1496,38 +1496,38 @@ ipython-genutils==0.2.0 \ --hash=sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8 \ --hash=sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbclassic # notebook ipywidgets==8.1.3 \ --hash=sha256:efafd18f7a142248f7cb0ba890a68b96abd4d6e88ddbda483c9130d12667eaf2 \ --hash=sha256:f5f9eeaae082b1823ce9eac2575272952f40d748893972956dc09700a6392d9c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt isodate==0.6.1 \ --hash=sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96 \ --hash=sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # azure-storage-blob isoduration==20.11.0 \ --hash=sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9 \ --hash=sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema jedi==0.19.1 \ --hash=sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd \ --hash=sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipython jinja2==3.1.6 \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # fastapi # jupyter-server # jupyterlab @@ -1621,26 +1621,26 @@ jmespath==1.0.1 \ --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # boto3 # botocore json5==0.9.14 \ --hash=sha256:740c7f1b9e584a468dbb2939d8d458db3427f2c93ae2139d05f47e453eae964f \ --hash=sha256:9ed66c3a6ca3510a976a9ef9b8c0787de24802724ab1860bc0153c7fdd589b02 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyterlab-server jsonpatch==1.32 \ --hash=sha256:26ac385719ac9f54df8a2f0827bb8253aa3ea8ab7b3368457bcdb8c14595a397 \ --hash=sha256:b6ddfe6c3db30d81a96aaeceb6baf916094ffa23d7dd5fa2c13e13f8b6e600c2 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt jsonpointer==2.4 \ --hash=sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a \ --hash=sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonpatch # jsonschema jsonref==1.1.0 \ @@ -1651,7 +1651,7 @@ jsonschema==4.23.0 \ --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt @@ -1664,13 +1664,13 @@ jsonschema-specifications==2024.10.1 \ --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema jupyter-client==7.3.4 \ --hash=sha256:17d74b0d0a7b24f1c8c527b24fcf4607c56bee542ffe8e3418e50b21e514b621 \ --hash=sha256:aa9a6c32054b290374f95f73bb0cae91455c58dfb84f65c8591912b8f65e6d56 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipykernel # jupyter-server # nbclassic @@ -1680,7 +1680,7 @@ jupyter-core==5.5.0 \ --hash=sha256:880b86053bf298a8724994f95e99b99130659022a4f7f45f563084b6223861d3 \ --hash=sha256:e11e02cd8ae0a9de5c6c44abf5727df9f2581055afe00b22183f621ba3585805 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipykernel # jupyter-client # jupyter-server @@ -1693,13 +1693,13 @@ jupyter-events==0.6.3 \ --hash=sha256:57a2749f87ba387cd1bfd9b22a0875b889237dbf2edc2121ebb22bde47036c17 \ --hash=sha256:9a6e9995f75d1b7146b436ea24d696ce3a35bfa8bfe45e0c33c334c79464d0b3 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-server-fileid jupyter-server==1.24.0 \ --hash=sha256:23368e8e214baf82b313d4c5a0d828ca73015e1a192ce3829bd74e62fab8d046 \ --hash=sha256:c88ddbe862966ea1aea8c3ccb89a5903abd8fbcfe5cd14090ef549d403332c37 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-server-fileid # jupyterlab # jupyterlab-server @@ -1709,44 +1709,44 @@ jupyter-server-fileid==0.9.0 \ --hash=sha256:171538b7c7d08d11dbc57d4e6da196e0c258e4c2cd29249ef1e032bb423677f8 \ --hash=sha256:5b489c6fe6783c41174a728c7b81099608518387e53c3d53451a67f46a0cb7b0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-server-ydoc jupyter-server-ydoc==0.6.1 \ --hash=sha256:18275ff1ce7e93bbda2301ca066273b3951fc50b0d9c8fc33788374134ad7920 \ --hash=sha256:ab10864708c81fa41ab9f2ed3626b54ff6926eaf14545d1d439714978dad6e9f # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyterlab jupyter-ydoc==0.2.5 \ --hash=sha256:5759170f112c70320a84217dd98d287699076ae65a7f88d458d57940a9f2b882 \ --hash=sha256:5a02ca7449f0d875f73e8cb8efdf695dddef15a8e71378b1f4eda6b7c90f5382 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-server-ydoc # jupyterlab jupyterlab==3.6.1 \ --hash=sha256:ad6707dd0149b629d0ed5b56916cfcdb816b376c6af3190337faba09e27ea29e \ --hash=sha256:aee98c174180e98a30470297d10b959e8e64f2288970c0de65f0a6d2b4807034 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt jupyterlab-pygments==0.3.0 \ --hash=sha256:721aca4d9029252b11cfa9d185e5b5af4d54772bb8072f9b7036f4170054d35d \ --hash=sha256:841a89020971da1d8693f1a99997aefc5dc424bb1b251fd6322462a1b8842780 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbconvert jupyterlab-server==2.24.0 \ --hash=sha256:4e6f99e0a5579bbbc32e449c4dbb039561d4f1a7827d5733273ed56738f21f07 \ --hash=sha256:5f077e142bb8dc9b843d960f940c513581bceca3793a0d80f9c67d9522c4e876 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyterlab jupyterlab-widgets==3.0.11 \ --hash=sha256:78287fd86d20744ace330a61625024cf5521e1c012a352ddc0a3cdc2348becd0 \ --hash=sha256:dd5ac679593c969af29c9bed054c24f26842baa51352114736756bc035deee27 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipywidgets jupytext==1.17.2 \ --hash=sha256:4f85dc43bb6a24b75491c5c434001ad5ef563932f68f15dd3e1c8ce12a4a426b \ @@ -1756,7 +1756,7 @@ kombu==5.5.4 \ --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # celery lark==1.2.2 \ --hash=sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c \ @@ -1766,7 +1766,7 @@ lazy-loader==0.4 \ --hash=sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc \ --hash=sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # scikit-image llguidance==0.7.29 ; platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:17fd439957d6ca5f459d0dec755a2d040c2dc946ed7e3c332b469ef6861292f8 \ @@ -1809,7 +1809,7 @@ log-symbols==0.0.14 \ --hash=sha256:4952106ff8b605ab7d5081dd2c7e6ca7374584eff7086f499c06edd1ce56dcca \ --hash=sha256:cf0bbc6fe1a8e53f0d174a716bc625c4f87043cc21eb55dd8a740cfe22680556 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # halo lxml==4.9.4 \ --hash=sha256:00e91573183ad273e242db5585b52670eddf92bacad095ce25c1e682da14ed91 \ @@ -1906,7 +1906,7 @@ lxml==4.9.4 \ --hash=sha256:fd814847901df6e8de13ce69b84c31fc9b3fb591224d6762d0b256d510cbf382 \ --hash=sha256:fdb325b7fba1e2c40b9b1db407f85642e32404131c08480dd652110fc908561b # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbconvert lz4==4.3.3 \ --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ @@ -1946,13 +1946,13 @@ lz4==4.3.3 \ --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt markdown-it-py==2.2.0 \ --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupytext # mdit-py-plugins # rich @@ -1983,14 +1983,14 @@ markupsafe==2.1.3 \ --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jinja2 # nbconvert matplotlib-inline==0.1.6 \ --hash=sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311 \ --hash=sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipykernel # ipython mdit-py-plugins==0.4.2 \ @@ -2001,7 +2001,7 @@ mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # markdown-it-py memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ @@ -2040,7 +2040,7 @@ memray==1.10.0 ; sys_platform != 'win32' \ --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt meson==1.8.3 \ --hash=sha256:ef02b806ce0c5b6becd5bb5dc9fa67662320b29b337e7ace73e4354500590233 \ @@ -2054,7 +2054,7 @@ mistune==0.8.4 \ --hash=sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e \ --hash=sha256:88a1051873018da288eee8538d476dffe1262495144b33ecb586c4ab266bb8d4 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbconvert mpmath==1.3.0 \ --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c @@ -2063,14 +2063,14 @@ msal==1.28.1 \ --hash=sha256:563c2d70de77a2ca9786aab84cb4e133a38a6897e6676774edc23d610bfc9e7b \ --hash=sha256:d72bbfe2d5c2f2555f4bc6205be4450ddfd12976610dd9a16a9ab0f05c68b64d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # azure-identity # msal-extensions msal-extensions==1.2.0b1 \ --hash=sha256:217f391bb549de11b19abe8029a8375fe3ca0556aa8cce004b2083f00a569b71 \ --hash=sha256:3658b3814cd6a7759e83cb0ec145f30330ee249a92444adaf9aa4eb4f5bbcbbc # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # azure-identity msgpack==1.0.7 \ --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ @@ -2130,7 +2130,7 @@ msgpack==1.0.7 \ --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # ray msgspec==0.19.0 \ @@ -2263,27 +2263,27 @@ multidict==6.0.5 \ --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp # yarl nbclassic==1.0.0 \ --hash=sha256:0ae11eb2319455d805596bf320336cda9554b41d99ab9a3c31bf8180bffa30e3 \ --hash=sha256:f99e4769b4750076cd4235c044b61232110733322384a94a63791d2e7beacc66 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyterlab # notebook nbclient==0.5.13 \ --hash=sha256:40c52c9b5e3c31faecaee69f202b3f53e38d7c1c563de0fadde9d7eda0fdafe8 \ --hash=sha256:47ac905af59379913c1f8f541098d2550153cf8dc58553cbe18c702b181518b0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbconvert nbconvert==6.5.4 \ --hash=sha256:9e3c7c6d491374cbdd5f35d268c05809357716d346f4573186bbeab32ee50bc1 \ --hash=sha256:d679a947f849a966cbbd0bf6e7fedcfdb64be3b20ce7cef11ad55c13f5820e19 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-server # nbclassic # notebook @@ -2291,7 +2291,7 @@ nbformat==5.9.2 \ --hash=sha256:1c5172d786a41b82bcfd0c23f9e6b6f072e8fb49c39250219e4acfff1efe89e9 \ --hash=sha256:5f98b5ba1997dff175e77e0c17d5c10a96eaed2cbd1de3533d1fc35d5e111192 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-server # jupytext # nbclassic @@ -2302,7 +2302,7 @@ nest-asyncio==1.5.8 \ --hash=sha256:25aa2ca0d2a5b5531956b9e273b45cf664cae2b145101d73b86b199978d48fdb \ --hash=sha256:accda7a339a70599cb08f9dd09a67e0c2ef8d8d6f4c07f96ab203f2ae254e48d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipykernel # jupyter-client # nbclassic @@ -2311,7 +2311,7 @@ nest-asyncio==1.5.8 \ networkx==3.2.1 \ --hash=sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # scikit-image # torch ninja==1.11.1.4 \ @@ -2350,13 +2350,13 @@ notebook==6.5.7 \ --hash=sha256:04eb9011dfac634fbd4442adaf0a8c27cd26beef831fe1d19faf930c327768e4 \ --hash=sha256:a6afa9a4ff4d149a0771ff8b8c881a7a73b3835f9add0606696d6e9d98ac1cd0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyterlab notebook-shim==0.2.3 \ --hash=sha256:a83496a43341c1674b093bfcebf0fe8e74cbe7eda5fd2bbc56f8e39e1486c0c7 \ --hash=sha256:f69388ac283ae008cd506dda10d0288b09a017d822d5e8c7129a152cbd3ce7e9 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbclassic numba==0.61.2 \ --hash=sha256:34fba9406078bac7ab052efbf0d13939426c753ad72946baaa5bf9ae0ebb8dd2 \ @@ -2419,7 +2419,7 @@ numpy==1.26.4 \ --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # cupy-cuda12x # gguf @@ -2499,7 +2499,7 @@ oauth2client==4.1.3 \ --hash=sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac \ --hash=sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt openai==1.90.0 \ --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ @@ -2509,13 +2509,13 @@ opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt opencensus-context==0.1.3 \ --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # opencensus opencv-python-headless==4.11.0.86 \ --hash=sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b \ @@ -2532,7 +2532,7 @@ opentelemetry-api==1.34.1 \ --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus # opentelemetry-sdk @@ -2541,26 +2541,26 @@ opentelemetry-exporter-prometheus==0.55b1 \ --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt opentelemetry-proto==1.27.0 \ --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt opentelemetry-sdk==1.34.1 \ --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # opentelemetry-exporter-prometheus opentelemetry-semantic-conventions==0.55b1 \ --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # opentelemetry-sdk outlines-core==0.2.10 \ --hash=sha256:0a9e4b192ca837a472a1bb1428397509f543db08e1aeeee30252525cec34093a \ @@ -2609,7 +2609,7 @@ packaging==23.0 \ --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # huggingface-hub @@ -2657,19 +2657,19 @@ pandas==1.5.3 \ --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt pandocfilters==1.5.0 \ --hash=sha256:0b679503337d233b4339a817bfc8c50064e2eff681314376a47cb582305a7a38 \ --hash=sha256:33aae3f25fd1a026079f5d27bdd52496f0e0803b3469282162bafdcbdf6ef14f # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbconvert parso==0.8.3 \ --hash=sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0 \ --hash=sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jedi partial-json-parser==0.2.1.1.post5 \ --hash=sha256:627715aaa3cb3fb60a65b0d62223243acaa6c70846520a90326fef3a2f0b61ca \ @@ -2679,19 +2679,19 @@ pathspec==0.11.2 \ --hash=sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20 \ --hash=sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt pexpect==4.8.0 ; sys_platform != 'win32' \ --hash=sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937 \ --hash=sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipython pickleshare==0.7.5 \ --hash=sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca \ --hash=sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipython pillow==10.3.0 \ --hash=sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c \ @@ -2764,7 +2764,7 @@ pillow==10.3.0 \ --hash=sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27 \ --hash=sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/llm/llm-test-requirements.txt # imageio # mistral-common @@ -2775,26 +2775,26 @@ platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-core # virtualenv pluggy==1.3.0 \ --hash=sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12 \ --hash=sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # pytest portalocker==2.8.2 \ --hash=sha256:2b035aa7828e46c58e9b31390ee1f169b98e1066ab10b9a6a861fe7e25ee4f33 \ --hash=sha256:cfb86acc09b9aa7c3b43594e19be1345b9d16af3feb08bf92f23d4dce513a28e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # msal-extensions prometheus-client==0.19.0 \ --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # jupyter-server # nbclassic @@ -2810,7 +2810,7 @@ prompt-toolkit==3.0.41 \ --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # click-repl # ipython propcache==0.3.0 \ @@ -2913,14 +2913,14 @@ propcache==0.3.0 \ --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp # yarl proto-plus==1.22.3 \ --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-api-core protobuf==4.25.8 \ --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ @@ -2935,7 +2935,7 @@ protobuf==4.25.8 \ --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # google-api-core # googleapis-common-protos @@ -2963,21 +2963,21 @@ psutil==5.9.6 \ --hash=sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d \ --hash=sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipykernel # vllm ptyprocess==0.7.0 ; os_name != 'nt' or sys_platform != 'win32' \ --hash=sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35 \ --hash=sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # pexpect # terminado pure-eval==0.2.2 \ --hash=sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350 \ --hash=sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # stack-data py-cpuinfo==9.0.0 \ --hash=sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690 \ @@ -2993,7 +2993,7 @@ py-spy==0.4.0 ; python_full_version < '3.12' \ --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt pyarrow==19.0.1 \ --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ @@ -3039,13 +3039,13 @@ pyarrow==19.0.1 \ --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt pyasn1==0.5.1 \ --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # oauth2client # pyasn1-modules # rsa @@ -3053,7 +3053,7 @@ pyasn1-modules==0.3.0 \ --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-auth # oauth2client pybase64==1.4.1 \ @@ -3213,7 +3213,7 @@ pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # cffi pycurl==7.45.3 \ --hash=sha256:0c41a172d5e8a5cdd8328cc8134f47b2a57960ac677f7cda8520eaa9fbe7d990 \ @@ -3253,13 +3253,13 @@ pycurl==7.45.3 \ --hash=sha256:fa7751b614d9aa82d7a0f49ca90924c29c6cedf85a2f8687fb6a772dbfe48711 \ --hash=sha256:fbd4a6b8654b779089c5a44af1c65c1419c2cd60718780df6d8f354eb35d6d55 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt pydantic==2.10.0 \ --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # compressed-tensors # fastapi @@ -3371,7 +3371,7 @@ pydantic-core==2.27.0 \ --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # pydantic pydantic-extra-types==2.10.5 \ --hash=sha256:1dcfa2c0cf741a422f088e0dbb4690e7bfadaaf050da3d6f80d6c3cf58a2bad8 \ @@ -3381,7 +3381,7 @@ pygments==2.18.0 \ --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipython # nbconvert # rich @@ -3390,7 +3390,7 @@ pyjwt==2.8.0 \ --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # msal pynvml==12.0.0 \ --hash=sha256:299ce2451a6a17e6822d6faee750103e25b415f06f59abb8db65d30f794166f5 \ @@ -3400,20 +3400,20 @@ pyopenssl==25.0.0 \ --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt pyparsing==3.1.1 \ --hash=sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb \ --hash=sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # httplib2 pytest==7.4.4 \ --hash=sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280 \ --hash=sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/base-test-requirements.txt # -r python/requirements/llm/llm-test-requirements.txt # pytest-aiohttp @@ -3422,20 +3422,20 @@ pytest-aiohttp==1.1.0 \ --hash=sha256:147de8cb164f3fc9d7196967f109ab3c0b93ea3463ab50631e56438eab7b5adc \ --hash=sha256:f39a11693a0dce08dd6c542d241e199dd8047a6e6596b2bcfa60d373f143456d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/base-test-requirements.txt pytest-asyncio==0.17.2 \ --hash=sha256:6d895b02432c028e6957d25fc936494e78c6305736e785d9fee408b1efbc7ff4 \ --hash=sha256:e0fe5dbea40516b661ef1bcfe0bd9461c2847c4ef4bb40012324f2454fb7d56d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/base-test-requirements.txt # pytest-aiohttp python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # arrow # botocore @@ -3450,7 +3450,7 @@ python-json-logger==2.0.7 \ --hash=sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c \ --hash=sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-events # vllm python-multipart==0.0.20 \ @@ -3461,7 +3461,7 @@ pytz==2022.7.1 \ --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # pandas pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ @@ -3516,7 +3516,7 @@ pyyaml==6.0.1 \ --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # gguf @@ -3618,7 +3618,7 @@ pyzmq==26.0.3 \ --hash=sha256:f6b1d1c631e5940cac5a0b22c5379c86e8df6a4ec277c7a856b714021ab6cfad \ --hash=sha256:f6c21c00478a7bea93caaaef9e7629145d4153b15a8653e8bb4609d4bc70dbfc # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipykernel # jupyter-client # jupyter-server @@ -3629,7 +3629,7 @@ referencing==0.36.2 \ --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema # jsonschema-specifications regex==2024.11.6 \ @@ -3735,7 +3735,7 @@ requests==2.32.3 \ --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # azure-core @@ -3755,21 +3755,21 @@ rfc3339-validator==0.1.4 \ --hash=sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b \ --hash=sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema # jupyter-events rfc3986-validator==0.1.1 \ --hash=sha256:2f235c432ef459970b4306369336b9d5dbdda31b510ca1e327636e01f528bfa9 \ --hash=sha256:3d44bde7921b3b9ec3ae4e3adca370438eccebc676456449b145d533b240d055 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema # jupyter-events rich==13.3.2 \ --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # memray @@ -3879,21 +3879,21 @@ rpds-py==0.22.3 \ --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema # referencing rsa==4.7.2 \ --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-auth # oauth2client s3transfer==0.6.2 \ --hash=sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084 \ --hash=sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # boto3 safetensors==0.5.3 \ --hash=sha256:1077f3e94182d72618357b04b5ced540ceb71c8a813d3319f1aba448e68a770d \ @@ -3935,7 +3935,7 @@ scikit-image==0.24.0 \ --hash=sha256:ef04360eda372ee5cd60aebe9be91258639c86ae2ea24093fb9182118008d009 \ --hash=sha256:fa27b3a0dbad807b966b8db2d78da734cb812ca4787f7fbb143764800ce2fa9c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt scipy==1.11.4 \ --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ @@ -3964,7 +3964,7 @@ scipy==1.11.4 \ --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # scikit-image # vllm @@ -3972,7 +3972,7 @@ send2trash==1.8.3 \ --hash=sha256:0c31227e0bd08961c7665474a3d1ef7193929fedda4233843689baa056be46c9 \ --hash=sha256:b18e7a3966d99871aefeb00cfbcfdced55ce4871194810fc71f4aa484b953abf # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-server # nbclassic # notebook @@ -4038,13 +4038,13 @@ shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # typer six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # asttokens # azure-core @@ -4059,20 +4059,20 @@ smart-open==6.2.0 \ --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt smmap==5.0.1 \ --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # gitdb sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # anyio # openai snowballstemmer==3.0.1 \ @@ -4093,7 +4093,7 @@ soupsieve==2.5 \ --hash=sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690 \ --hash=sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # beautifulsoup4 soxr==0.5.0.post1 \ --hash=sha256:39e0f791ba178d69cd676485dbee37e75a34f20daa478d90341ecb7f6d9d690f \ @@ -4150,19 +4150,19 @@ spinners==0.0.24 \ --hash=sha256:1eb6aeb4781d72ab42ed8a01dcf20f3002bf50740d7154d12fb8c9769bf9e27f \ --hash=sha256:2fa30d0b72c9650ad12bbe031c9943b8d441e41b4f5602b0ec977a19f3290e98 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # halo stack-data==0.6.3 \ --hash=sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9 \ --hash=sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipython starlette==0.46.2 \ --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # fastapi # prometheus-fastapi-instrumentator @@ -4174,25 +4174,25 @@ tabulate==0.9.0 \ --hash=sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c \ --hash=sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt tensorboardx==2.6.2.2 \ --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt termcolor==2.4.0 \ --hash=sha256:9297c0df9c99445c2412e832e882a7884038a25617c60cea2ad69488d4040d63 \ --hash=sha256:aab9e56047c8ac41ed798fa36d892a37aca6b3e9159f3e0c24bc64a9b3ac7b7a # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # halo terminado==0.18.1 \ --hash=sha256:a4468e1b37bb318f8a86514f65814e1afc977cf29b3992a4500d9dd305dcceb0 \ --hash=sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-server # nbclassic # notebook @@ -4200,7 +4200,7 @@ tifffile==2024.7.21 \ --hash=sha256:7f335b5d6ca49401fe0f1d87deb206f5dae47297e47b1ed52a676d05d6d26798 \ --hash=sha256:818b577d49350421fb511f389f937984f9feaa2cd8177fa00823001920bf3483 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # scikit-image tiktoken==0.9.0 \ --hash=sha256:03935988a91d6d3216e2ec7c645afbb3d870b37bcb67ada1943ec48678e7ee33 \ @@ -4241,7 +4241,7 @@ tinycss2==1.3.0 \ --hash=sha256:152f9acabd296a8375fbca5b84c961ff95971fcfc32e79550c8df8e29118c54d \ --hash=sha256:54a8dbdffb334d536851be0226030e9505965bb2f30f21a4a82c55fb2a80fae7 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbconvert tokenizers==0.21.1 \ --hash=sha256:0f0dcbcc9f6e13e675a66d7a5f2f225a736745ce484c1a4e07476a89ccdad382 \ @@ -4360,7 +4360,7 @@ tornado==6.1 \ --hash=sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68 \ --hash=sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipykernel # jupyter-client # jupyter-server @@ -4372,7 +4372,7 @@ tqdm==4.67.1 \ --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # gguf # huggingface-hub @@ -4383,7 +4383,7 @@ traitlets==5.14.3 \ --hash=sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7 \ --hash=sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # comm # ipykernel # ipython @@ -4415,7 +4415,7 @@ typer==0.12.3 \ --hash=sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914 \ --hash=sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt # -r python/requirements.txt # fastapi-cli @@ -4423,12 +4423,12 @@ types-python-dateutil==2.9.0.20240316 \ --hash=sha256:5d2f2e240b86905e40944dd787db6da9263f0deabef1076ddaed797351ec0202 \ --hash=sha256:6b8cb66d960771ce5ff974e9dd45e38facb81718cc1e208b10b1baccbfdbee3b # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # arrow typing-extensions==4.12.2 \ --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # azure-core # azure-identity # azure-storage-blob @@ -4452,25 +4452,25 @@ tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # kombu tzlocal==5.3 \ --hash=sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2 \ --hash=sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt uri-template==1.3.0 \ --hash=sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7 \ --hash=sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # botocore # requests @@ -4478,7 +4478,7 @@ uvicorn==0.22.0 \ --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # fastapi # fastapi-cli @@ -4525,7 +4525,7 @@ vine==5.1.0 \ --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # amqp # celery # kombu @@ -4533,7 +4533,7 @@ virtualenv==20.29.1 \ --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt vllm==0.10.0 \ --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ @@ -4563,7 +4563,7 @@ watchfiles==0.19.0 \ --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # uvicorn # vllm @@ -4571,26 +4571,26 @@ wcwidth==0.2.13 \ --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # prompt-toolkit webcolors==24.6.0 \ --hash=sha256:1d160d1de46b3e81e58d0a280d0c78b467dc80f47294b91b1ad8029d2cedb55b \ --hash=sha256:8cf5bc7e28defd1d48b9e83d5fc30741328305a8195c29a8e668fa45586568a1 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jsonschema webencodings==0.5.1 \ --hash=sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78 \ --hash=sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # bleach # tinycss2 websocket-client==1.8.0 \ --hash=sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526 \ --hash=sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-server websockets==15.0.1 \ --hash=sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2 \ @@ -4667,7 +4667,7 @@ widgetsnbextension==4.0.11 \ --hash=sha256:55d4d6949d100e0d08b94948a42efc3ed6dfdc0e9468b2c4b128c9a2ce3a7a36 \ --hash=sha256:8b22a8f1910bfd188e596fe7fc05dcbd87e810c8a4ba010bdb3da86637398474 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # ipywidgets wrapt==1.14.1 \ --hash=sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3 \ @@ -4745,7 +4745,7 @@ wrapt==1.14.1 \ --hash=sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015 \ --hash=sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt xformers==0.0.31 ; platform_machine == 'x86_64' and sys_platform == 'linux' \ --hash=sha256:b2ea87e0651f46164cb3cd74face021bd1654229ca4f8c0baa03b8c477515c7a @@ -4854,7 +4854,7 @@ y-py==0.6.2 \ --hash=sha256:e92878cc05e844c8da937204bc34c2e6caf66709ce5936802fbfb35f04132892 \ --hash=sha256:ff32548e45e45bf3280ac1d28b3148337a5c6714c28db23aeb0693e33eba257e # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-ydoc # ypy-websocket yarl==1.18.3 \ @@ -4941,19 +4941,19 @@ yarl==1.18.3 \ --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp ypy-websocket==0.8.4 \ --hash=sha256:43a001473f5c8abcf182f603049cf305cbc855ad8deaa9dfa0f3b5a7cea9d0ff \ --hash=sha256:b1ba0dfcc9762f0ca168d2378062d3ca1299d39076b0f145d961359121042be5 # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # jupyter-server-ydoc zipp==3.19.2 \ --hash=sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19 \ --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c # via - # -c python/requirements_compiled_ray_test_py311_cu128.txt + # -c python/deplocks/llm/ray_test_py311_cu128.lock # importlib-metadata # The following packages were excluded from the output: From 31e1fb2276de05c50d5a04b567bf4367bc6ef673 Mon Sep 17 00:00:00 2001 From: Yevet Date: Mon, 25 Aug 2025 22:19:12 -0700 Subject: [PATCH 270/634] Add GKE GPU compat paths to ray image: PATH, LD_LIBRARY_PATH (temporarily) (#55569) The ray base image does not include the GPU operator compatibility paths included in the [runtime image](https://hub.docker.com/layers/nvidia/cuda/12.8.0-runtime-ubuntu22.04/images/sha256-80c946064fc2db73957af8fa6a7a8bec3e3fd3241d130fb668cad6a62510aeda). Temporarily fix GPU detection problem on GPU for basic ray image. This is a follow up for https://github.com/ray-project/ray/issues/54551 --------- Signed-off-by: Yiwen Xiang Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> --- docker/base-deps/Dockerfile | 13 ++++++++++++- docker/ray-llm/Dockerfile | 14 ++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index 1320aed23d53..2857ec1cb34f 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -9,8 +9,19 @@ FROM ${BASE_IMAGE} ENV TZ=America/Los_Angeles ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 + # TODO(ilr) $HOME seems to point to result in "" instead of "/home/ray" -ENV PATH "/home/ray/anaconda3/bin:$PATH" +# Q: Why add paths like /usr/local/nvidia/lib64 and /usr/local/nvidia/bin? +# A: The NVIDIA GPU operator version used by GKE injects these into the container +# after it's mounted to a pod. +# Issue is tracked here: +# https://github.com/GoogleCloudPlatform/compute-gpu-installation/issues/46 +# More context here: +# https://github.com/NVIDIA/nvidia-container-toolkit/issues/275 +# and here: +# https://gitlab.com/nvidia/container-images/cuda/-/issues/27 +ENV PATH "/home/ray/anaconda3/bin:$PATH:/usr/local/nvidia/bin" +ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/usr/local/nvidia/lib64" ARG DEBIAN_FRONTEND=noninteractive ARG PYTHON_VERSION=3.9 diff --git a/docker/ray-llm/Dockerfile b/docker/ray-llm/Dockerfile index e933928bcf10..554466d3a7a8 100644 --- a/docker/ray-llm/Dockerfile +++ b/docker/ray-llm/Dockerfile @@ -190,16 +190,6 @@ sudo apt-get clean EOF -# Q: Why add paths that don't exist in the base image, like /usr/local/nvidia/lib64 -# and /usr/local/nvidia/bin? -# A: The NVIDIA GPU operator version used by GKE injects these into the container -# after it's mounted to a pod. -# Issue is tracked here: -# https://github.com/GoogleCloudPlatform/compute-gpu-installation/issues/46 -# More context here: -# https://github.com/NVIDIA/nvidia-container-toolkit/issues/275 -# and here: -# https://gitlab.com/nvidia/container-images/cuda/-/issues/27 -ENV PATH="${PATH}:${UCX_HOME}/bin:${NIXL_HOME}/bin:/usr/local/nvidia/bin" -ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${UCX_HOME}/lib:${NIXL_HOME}/lib/x86_64-linux-gnu:/usr/local/nvidia/lib64" +ENV PATH="${PATH}:${UCX_HOME}/bin:${NIXL_HOME}/bin" +ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${UCX_HOME}/lib:${NIXL_HOME}/lib/x86_64-linux-gnu" ENV NIXL_PLUGIN_DIR="${NIXL_HOME}/lib/x86_64-linux-gnu/plugins/" From 1c236a40e37b44a17a2e4a03c8fb8bdb920216c1 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Mon, 25 Aug 2025 23:06:24 -0700 Subject: [PATCH 271/634] [ci] removing python 3.13 wheel verification for mac (#55907) removing python 3.13 wheel verification for macos due to this bug: https://github.com/ray-project/ray/issues/54047 Signed-off-by: elliot-barn Signed-off-by: Lonnie Liu --- .buildkite/release-automation/verify-macos-wheels.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.buildkite/release-automation/verify-macos-wheels.sh b/.buildkite/release-automation/verify-macos-wheels.sh index a3a06dafca58..e867ff0d6b74 100755 --- a/.buildkite/release-automation/verify-macos-wheels.sh +++ b/.buildkite/release-automation/verify-macos-wheels.sh @@ -4,7 +4,10 @@ set -euo pipefail set -x -PYTHON_VERSIONS=("3.9" "3.10" "3.11" "3.12" "3.13") +# TODO(#54047): Python 3.13 is skipped due to the bug +# we should re-enable it when the bug is fixed. + +PYTHON_VERSIONS=("3.9" "3.10" "3.11" "3.12") BAZELISK_VERSION="v1.16.0" # Check arguments From 0f70eb24aab91191624c369ac74e0807bca470e7 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 26 Aug 2025 06:41:50 -0700 Subject: [PATCH 272/634] [java] convert `gen_maven_deps` to py_binary (#55928) and use bazel run to run it this is the last local genrule that is not yet converted. Signed-off-by: Lonnie Liu --- BUILD.bazel | 2 ++ java/BUILD.bazel | 55 ++++++++++++++++----------------- java/build-jar-multiplatform.sh | 2 +- java/gen_maven_deps.py | 4 +++ java/test.sh | 2 +- 5 files changed, 35 insertions(+), 30 deletions(-) create mode 100644 java/gen_maven_deps.py diff --git a/BUILD.bazel b/BUILD.bazel index 65c06e57345c..ae56691ed2f7 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -469,6 +469,8 @@ genrule( sed -i -E 's/from opencensus.proto.resource.v1 import/from . import/' "$${files[@]}" $(location //bazel:pyzip) "$$tmpdir" $@ + + rm -rf "$$tmpdir" """, tools = [ "//bazel:pyzip", diff --git a/java/BUILD.bazel b/java/BUILD.bazel index df4c0d1aceed..7f2bc61fa6d4 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -6,7 +6,7 @@ load("@rules_java//java:java_binary.bzl", "java_binary") load("@rules_java//java:java_import.bzl", "java_import") load("@rules_java//java:java_library.bzl", "java_library") load("@rules_java//java:java_test.bzl", "java_test") -load("@rules_pkg//pkg:mappings.bzl", "pkg_files") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") load("@rules_proto_grpc//java:defs.bzl", "java_proto_compile") load("@rules_python//python:defs.bzl", "py_binary") @@ -398,39 +398,38 @@ py_binary( deps = ["//bazel:gen_extract"], ) -# Generates the dependencies needed by maven. -genrule( - name = "gen_maven_deps", +pkg_files( + name = "maven_deps_files", srcs = [ - ":pom_files.zip", - ":proto_files.zip", ":java_native_deps", ], - outs = ["gen_maven_deps.out"], - cmd = """ - WORK_DIR="$${PWD}" - # Copy native dependencies. - OS_NAME="" - case "$${OSTYPE}" in - linux*) OS_NAME="linux";; - darwin*) OS_NAME="darwin";; - *) echo "$${OSTYPE} is not supported currently"; exit 1;; - esac - NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/native/$$OS_NAME" - rm -rf "$$NATIVE_DEPS_DIR" - mkdir -p "$$NATIVE_DEPS_DIR" - echo "# gen_maven_deps" > $@ - for f in $(locations //java:java_native_deps); do - chmod +w "$$f" - cp "$$f" "$$NATIVE_DEPS_DIR" - if [[ "$$OSTYPE" =~ ^darwin ]]; then shasum "$$f" >> $@ ; else sha1sum "$$f" >> $@ ; fi - done - """, - local = 1, - tags = ["no-cache"], + attributes = pkg_attributes(mode = "755"), + prefix = select( + { + "@platforms//os:linux": "runtime/native_dependencies/native/linux", + "@platforms//os:macos": "runtime/native_dependencies/native/darwin", + }, + no_match_error = "Unsupported platform", + ), visibility = ["//visibility:private"], ) +pkg_zip( + name = "maven_deps", + srcs = [ + ":maven_deps_files", + ], + visibility = ["//visibility:private"], +) + +py_binary( + name = "gen_maven_deps", + srcs = ["gen_maven_deps.py"], + data = [":maven_deps.zip"], + visibility = ["//visibility:private"], + deps = ["//bazel:gen_extract"], +) + java_binary( name = "ray_dist", # This rule is used to package all Ray Java code and the third-party dependencies into a diff --git a/java/build-jar-multiplatform.sh b/java/build-jar-multiplatform.sh index 38baefba6e8b..c54d20dc861a 100755 --- a/java/build-jar-multiplatform.sh +++ b/java/build-jar-multiplatform.sh @@ -38,7 +38,7 @@ build_jars() { bazel run ":gen_proto_files" if [[ $bazel_build == "true" ]]; then echo "Starting building java native dependencies for $p" - bazel build ":gen_maven_deps" + bazel run ":gen_maven_deps" echo "Finished building java native dependencies for $p" fi echo "Start building jars for $p" diff --git a/java/gen_maven_deps.py b/java/gen_maven_deps.py new file mode 100644 index 000000000000..be2e7238c22a --- /dev/null +++ b/java/gen_maven_deps.py @@ -0,0 +1,4 @@ +from bazel.gen_extract import gen_extract + +if __name__ == "__main__": + gen_extract(["java/maven_deps.zip"], sub_dir="java") diff --git a/java/test.sh b/java/test.sh index fa1864083554..ab8dd0bbd528 100755 --- a/java/test.sh +++ b/java/test.sh @@ -66,7 +66,7 @@ fi echo "Build java maven deps." bazel run //java:gen_pom_files bazel run //java:gen_proto_files -bazel build //java:gen_maven_deps +bazel run //java:gen_maven_deps echo "Build ray core." bazel run //:gen_ray_pkg From 09fd1de850a08c81bbddd829072f383906903863 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 26 Aug 2025 08:57:57 -0500 Subject: [PATCH 273/634] [core] Remove `gcs_rpc_server.h` dependency from `RuntimeEnvHandler` (#55901) Depends on: https://github.com/ray-project/ray/pull/55886 --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 4 ++- src/ray/gcs/gcs_server/gcs_server.cc | 4 ++- .../gcs/gcs_server/grpc_service_interfaces.h | 9 ++++++ src/ray/gcs/gcs_server/grpc_services.cc | 8 +++++ src/ray/gcs/gcs_server/grpc_services.h | 24 +++++++++++++- src/ray/gcs/gcs_server/runtime_env_handler.h | 7 ++-- src/ray/rpc/gcs/gcs_rpc_server.h | 32 ------------------- 7 files changed, 51 insertions(+), 37 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 7bdcecd2c0c6..b0af984c2388 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -128,10 +128,12 @@ ray_cc_library( srcs = ["runtime_env_handler.cc"], hdrs = ["runtime_env_handler.h"], deps = [ + ":grpc_service_interfaces", + "//src/ray/common:asio", "//src/ray/common:runtime_env", "//src/ray/protobuf:gcs_cc_proto", - "//src/ray/rpc:gcs_server", "//src/ray/util:thread_checker", + "@boost//:asio", ], ) diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 9e15a8f79429..a9b9f9d381f3 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -697,7 +697,9 @@ void GcsServer::InitRuntimeEnvManager() { std::chrono::milliseconds(delay_ms)); }); rpc_server_.RegisterService(std::make_unique( - io_context_provider_.GetDefaultIOContext(), *runtime_env_handler_)); + io_context_provider_.GetDefaultIOContext(), + *runtime_env_handler_, + /*max_active_rpcs_per_handler=*/-1)); } void GcsServer::InitGcsWorkerManager() { diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h index 0deb908cae5d..74f1eda145a2 100644 --- a/src/ray/gcs/gcs_server/grpc_service_interfaces.h +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -64,5 +64,14 @@ class NodeInfoGcsServiceHandler { SendReplyCallback send_reply_callback) = 0; }; +class RuntimeEnvGcsServiceHandler { + public: + virtual ~RuntimeEnvGcsServiceHandler() = default; + + virtual void HandlePinRuntimeEnvURI(PinRuntimeEnvURIRequest request, + PinRuntimeEnvURIReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc index 098299683462..841c7ad3814a 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -36,5 +36,13 @@ void NodeInfoGrpcService::InitServerCallFactories( RPC_SERVICE_HANDLER(NodeInfoGcsService, CheckAlive, max_active_rpcs_per_handler_) } +void RuntimeEnvGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + RPC_SERVICE_HANDLER( + RuntimeEnvGcsService, PinRuntimeEnvURI, max_active_rpcs_per_handler_); +} + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/gcs_server/grpc_services.h index e4785f1dbe65..f438e717cf9b 100644 --- a/src/ray/gcs/gcs_server/grpc_services.h +++ b/src/ray/gcs/gcs_server/grpc_services.h @@ -42,7 +42,7 @@ class NodeInfoGrpcService : public GrpcService { int64_t max_active_rpcs_per_handler) : GrpcService(io_service), service_handler_(service_handler), - max_active_rpcs_per_handler_(max_active_rpcs_per_handler){}; + max_active_rpcs_per_handler_(max_active_rpcs_per_handler) {} protected: grpc::Service &GetGrpcService() override { return service_; } @@ -58,5 +58,27 @@ class NodeInfoGrpcService : public GrpcService { int64_t max_active_rpcs_per_handler_; }; +class RuntimeEnvGrpcService : public GrpcService { + public: + explicit RuntimeEnvGrpcService(instrumented_io_context &io_service, + RuntimeEnvGcsServiceHandler &handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler) {} + + protected: + grpc::Service &GetGrpcService() override { return service_; } + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + RuntimeEnvGcsService::AsyncService service_; + RuntimeEnvGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/runtime_env_handler.h b/src/ray/gcs/gcs_server/runtime_env_handler.h index 946ca0327568..15eb88ee95ca 100644 --- a/src/ray/gcs/gcs_server/runtime_env_handler.h +++ b/src/ray/gcs/gcs_server/runtime_env_handler.h @@ -13,11 +13,14 @@ // limitations under the License. #pragma once +#include #include #include +#include "ray/common/asio/instrumented_io_context.h" #include "ray/common/runtime_env_manager.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" + namespace ray { namespace gcs { @@ -25,7 +28,7 @@ typedef std::function(std::function uint32_t delay_ms)> DelayExecutorFn; -class RuntimeEnvHandler : public rpc::RuntimeEnvHandler { +class RuntimeEnvHandler : public rpc::RuntimeEnvGcsServiceHandler { public: RuntimeEnvHandler(instrumented_io_context &io_service, RuntimeEnvManager &runtime_env_manager, diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index 6e5835ae032f..ce83ef508d72 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -155,9 +155,6 @@ namespace rpc { #define INTERNAL_KV_SERVICE_RPC_HANDLER(HANDLER) \ RPC_SERVICE_HANDLER(InternalKVGcsService, HANDLER, -1) -#define RUNTIME_ENV_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(RuntimeEnvGcsService, HANDLER, -1) - // Unlimited max active RPCs, because of long poll. #define INTERNAL_PUBSUB_SERVICE_RPC_HANDLER(HANDLER) \ RPC_SERVICE_HANDLER(InternalPubSubGcsService, HANDLER, -1) @@ -548,34 +545,6 @@ class InternalKVGrpcService : public GrpcService { InternalKVGcsServiceHandler &service_handler_; }; -class RuntimeEnvGcsServiceHandler { - public: - virtual ~RuntimeEnvGcsServiceHandler() = default; - virtual void HandlePinRuntimeEnvURI(PinRuntimeEnvURIRequest request, - PinRuntimeEnvURIReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -class RuntimeEnvGrpcService : public GrpcService { - public: - explicit RuntimeEnvGrpcService(instrumented_io_context &io_service, - RuntimeEnvGcsServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler) {} - - protected: - grpc::Service &GetGrpcService() override { return service_; } - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - RUNTIME_ENV_SERVICE_RPC_HANDLER(PinRuntimeEnvURI); - } - - private: - RuntimeEnvGcsService::AsyncService service_; - RuntimeEnvGcsServiceHandler &service_handler_; -}; - class TaskInfoGcsServiceHandler { public: virtual ~TaskInfoGcsServiceHandler() = default; @@ -700,7 +669,6 @@ using WorkerInfoHandler = WorkerInfoGcsServiceHandler; using PlacementGroupInfoHandler = PlacementGroupInfoGcsServiceHandler; using InternalKVHandler = InternalKVGcsServiceHandler; using InternalPubSubHandler = InternalPubSubGcsServiceHandler; -using RuntimeEnvHandler = RuntimeEnvGcsServiceHandler; using TaskInfoHandler = TaskInfoGcsServiceHandler; using RayEventExportHandler = RayEventExportGcsServiceHandler; From 2191b0663270d922548b90225d29e5a63af1c7f4 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 26 Aug 2025 09:25:37 -0500 Subject: [PATCH 274/634] [core] Fix TSAN build for `redis_store_client_test` (#55951) Prior to my PR combining the `RedisClient` and `RedisStoreClient`, `RedisClient::Disconnect` did nothing: https://github.com/ray-project/ray/pull/55655/files#diff-2d6d8cd30f5c1efd0051a66b40bddb2e72497afae5aa1fa12ef1491d476b04e5L52 This was called in the `DisconnectStoreClient` method of the test. I changed it to do `store_client.reset()`. This is invalid because there might still be async callbacks scheduled on the loop that depend on the primary Redis context. Reverting to doing nothing for disconnection and directly stopping the loop instead. Signed-off-by: Edward Oakes --- src/ray/gcs/store_client/tests/in_memory_store_client_test.cc | 2 -- .../gcs/store_client/tests/observable_store_client_test.cc | 2 -- src/ray/gcs/store_client/tests/redis_store_client_test.cc | 2 -- src/ray/gcs/store_client/tests/store_client_test_base.h | 4 ---- 4 files changed, 10 deletions(-) diff --git a/src/ray/gcs/store_client/tests/in_memory_store_client_test.cc b/src/ray/gcs/store_client/tests/in_memory_store_client_test.cc index 0fa5a47116f6..fdd6d24b4673 100644 --- a/src/ray/gcs/store_client/tests/in_memory_store_client_test.cc +++ b/src/ray/gcs/store_client/tests/in_memory_store_client_test.cc @@ -27,8 +27,6 @@ class InMemoryStoreClientTest : public StoreClientTestBase { void InitStoreClient() override { store_client_ = std::make_shared(); } - - void DisconnectStoreClient() override {} }; TEST_F(InMemoryStoreClientTest, AsyncPutAndAsyncGetTest) { TestAsyncPutAndAsyncGet(); } diff --git a/src/ray/gcs/store_client/tests/observable_store_client_test.cc b/src/ray/gcs/store_client/tests/observable_store_client_test.cc index 39f084b522af..013601fd8741 100644 --- a/src/ray/gcs/store_client/tests/observable_store_client_test.cc +++ b/src/ray/gcs/store_client/tests/observable_store_client_test.cc @@ -29,8 +29,6 @@ class ObservableStoreClientTest : public StoreClientTestBase { store_client_ = std::make_shared(std::make_unique()); } - - void DisconnectStoreClient() override {} }; TEST_F(ObservableStoreClientTest, AsyncPutAndAsyncGetTest) { TestAsyncPutAndAsyncGet(); } diff --git a/src/ray/gcs/store_client/tests/redis_store_client_test.cc b/src/ray/gcs/store_client/tests/redis_store_client_test.cc index 0e3f0ea2a4c0..1d3cb4c3f7a9 100644 --- a/src/ray/gcs/store_client/tests/redis_store_client_test.cc +++ b/src/ray/gcs/store_client/tests/redis_store_client_test.cc @@ -79,8 +79,6 @@ class RedisStoreClientTest : public StoreClientTestBase { store_client_ = std::make_shared(io_context, options); } - void DisconnectStoreClient() override { store_client_.reset(); } - protected: std::unique_ptr t_; std::atomic stopped_ = false; diff --git a/src/ray/gcs/store_client/tests/store_client_test_base.h b/src/ray/gcs/store_client/tests/store_client_test_base.h index f115ff0292ec..c687a7d96e53 100644 --- a/src/ray/gcs/store_client/tests/store_client_test_base.h +++ b/src/ray/gcs/store_client/tests/store_client_test_base.h @@ -48,8 +48,6 @@ class StoreClientTestBase : public ::testing::Test { } void TearDown() override { - DisconnectStoreClient(); - io_service_pool_->Stop(); key_to_value_.clear(); @@ -57,8 +55,6 @@ class StoreClientTestBase : public ::testing::Test { virtual void InitStoreClient() = 0; - virtual void DisconnectStoreClient() = 0; - protected: void Put() { auto put_callback = [this](auto) { --pending_count_; }; From af077a90e7e1feadf5dccc0eb005234f546e1c90 Mon Sep 17 00:00:00 2001 From: Mao Yancan Date: Wed, 27 Aug 2025 00:01:55 +0800 Subject: [PATCH 275/634] [core][dashboard] use ray node id instead of ip for profilinglink (#55439) Signed-off-by: Mao Yancan Signed-off-by: Mao Yancan Co-authored-by: Mao Yancan Co-authored-by: Jiajun Yao --- ci/lint/pydoclint-baseline.txt | 4 +- .../client/src/common/ProfilingLink.tsx | 27 +++-- .../client/src/components/ActorTable.tsx | 6 +- .../client/src/pages/actor/ActorDetail.tsx | 6 +- .../src/pages/job/JobDetailInfoPage.tsx | 6 +- .../dashboard/client/src/pages/job/JobRow.tsx | 6 +- .../client/src/pages/node/NodeRow.tsx | 7 +- .../modules/reporter/reporter_head.py | 106 +++++++++++++----- python/ray/tests/test_dashboard_profiler.py | 69 +++++++++--- 9 files changed, 163 insertions(+), 74 deletions(-) diff --git a/ci/lint/pydoclint-baseline.txt b/ci/lint/pydoclint-baseline.txt index 99adbd1f2fe2..56df1bd45d5c 100644 --- a/ci/lint/pydoclint-baseline.txt +++ b/ci/lint/pydoclint-baseline.txt @@ -957,10 +957,10 @@ python/ray/dashboard/modules/reporter/reporter_head.py DOC101: Method `ReportHead.get_task_cpu_profile`: Docstring contains fewer arguments than in function signature. DOC103: Method `ReportHead.get_task_cpu_profile`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [req: aiohttp.web.Request]. DOC102: Method `ReportHead.get_traceback`: Docstring contains more arguments than in function signature. - DOC103: Method `ReportHead.get_traceback`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [req: aiohttp.web.Request]. Arguments in the docstring but not in the function signature: [ip: , pid: ]. + DOC103: Method `ReportHead.get_traceback`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [req: aiohttp.web.Request]. Arguments in the docstring but not in the function signature: [ip or node_id: , pid: ]. DOC201: Method `ReportHead.get_traceback` does not have a return section in docstring DOC102: Method `ReportHead.cpu_profile`: Docstring contains more arguments than in function signature. - DOC103: Method `ReportHead.cpu_profile`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [req: aiohttp.web.Request]. Arguments in the docstring but not in the function signature: [duration: , format: , ip: , native: , pid: ]. + DOC103: Method `ReportHead.cpu_profile`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [req: aiohttp.web.Request]. Arguments in the docstring but not in the function signature: [duration: , format: , ip or node_id: , native: , pid: ]. DOC201: Method `ReportHead.cpu_profile` does not have a return section in docstring DOC101: Method `ReportHead.memory_profile`: Docstring contains fewer arguments than in function signature. DOC103: Method `ReportHead.memory_profile`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [req: aiohttp.web.Request]. diff --git a/python/ray/dashboard/client/src/common/ProfilingLink.tsx b/python/ray/dashboard/client/src/common/ProfilingLink.tsx index 5c44c4547a61..5639bf827556 100644 --- a/python/ray/dashboard/client/src/common/ProfilingLink.tsx +++ b/python/ray/dashboard/client/src/common/ProfilingLink.tsx @@ -20,7 +20,7 @@ import { ClassNameProps } from "./props"; type CpuProfilingLinkProps = PropsWithChildren< { pid: string | number | null | undefined; - ip: string | null | undefined; + nodeId: string | null | undefined; type: string | null; } & ClassNameProps >; @@ -34,7 +34,7 @@ type TaskProfilingStackTraceProps = { type MemoryProfilingProps = PropsWithChildren< { pid: string | number | null | undefined; - ip: string | null | undefined; + nodeId: string | null | undefined; type?: string | null; } & ClassNameProps >; @@ -92,15 +92,20 @@ export const TaskCpuStackTraceLink = ({ export const CpuStackTraceLink = ({ pid, - ip, + nodeId, type = "", }: CpuProfilingLinkProps) => { - if (!pid || !ip || typeof pid === "undefined" || typeof ip === "undefined") { + if ( + !pid || + !nodeId || + typeof pid === "undefined" || + typeof nodeId === "undefined" + ) { return
; } return ( { - if (!pid || !ip) { + if (!pid || !nodeId) { return
; } return ( { - if (!pid || !ip) { + if (!pid || !nodeId) { return
; } - const profilerUrl = `memory_profile?pid=${pid}&ip=${ip}`; + const profilerUrl = `memory_profile?pid=${pid}&node_id=${nodeId}`; return ; }; diff --git a/python/ray/dashboard/client/src/components/ActorTable.tsx b/python/ray/dashboard/client/src/components/ActorTable.tsx index ec4f1d745234..bff6b40352ad 100644 --- a/python/ray/dashboard/client/src/components/ActorTable.tsx +++ b/python/ray/dashboard/client/src/components/ActorTable.tsx @@ -655,19 +655,19 @@ const ActorTable = ({


diff --git a/python/ray/dashboard/client/src/pages/actor/ActorDetail.tsx b/python/ray/dashboard/client/src/pages/actor/ActorDetail.tsx index 6fb16c1d6cdf..6cf3096272e0 100644 --- a/python/ray/dashboard/client/src/pages/actor/ActorDetail.tsx +++ b/python/ray/dashboard/client/src/pages/actor/ActorDetail.tsx @@ -191,19 +191,19 @@ const ActorDetailPage = () => {


diff --git a/python/ray/dashboard/client/src/pages/job/JobDetailInfoPage.tsx b/python/ray/dashboard/client/src/pages/job/JobDetailInfoPage.tsx index 112b19ed5df6..4c0c1562267b 100644 --- a/python/ray/dashboard/client/src/pages/job/JobDetailInfoPage.tsx +++ b/python/ray/dashboard/client/src/pages/job/JobDetailInfoPage.tsx @@ -172,19 +172,19 @@ export const JobMetadataSection = ({ job }: JobMetadataSectionProps) => {


diff --git a/python/ray/dashboard/client/src/pages/job/JobRow.tsx b/python/ray/dashboard/client/src/pages/job/JobRow.tsx index 8440dcc26edf..dfb571fb5ec4 100644 --- a/python/ray/dashboard/client/src/pages/job/JobRow.tsx +++ b/python/ray/dashboard/client/src/pages/job/JobRow.tsx @@ -116,19 +116,19 @@ export const JobRow = ({ job }: JobRowProps) => { )}

diff --git a/python/ray/dashboard/client/src/pages/node/NodeRow.tsx b/python/ray/dashboard/client/src/pages/node/NodeRow.tsx index d8a135510503..b873884564a0 100644 --- a/python/ray/dashboard/client/src/pages/node/NodeRow.tsx +++ b/python/ray/dashboard/client/src/pages/node/NodeRow.tsx @@ -227,7 +227,6 @@ type WorkerRowProps = { */ export const WorkerRow = ({ node, worker }: WorkerRowProps) => { const { - ip, mem, raylet: { nodeId }, } = node; @@ -278,11 +277,11 @@ export const WorkerRow = ({ node, worker }: WorkerRowProps) => { Log
- +
- +
- + diff --git a/python/ray/dashboard/modules/reporter/reporter_head.py b/python/ray/dashboard/modules/reporter/reporter_head.py index 7ec0f59be092..bbcaef0a863c 100644 --- a/python/ray/dashboard/modules/reporter/reporter_head.py +++ b/python/ray/dashboard/modules/reporter/reporter_head.py @@ -413,21 +413,32 @@ async def get_traceback(self, req: aiohttp.web.Request) -> aiohttp.web.Response: Params: pid: Required. The PID of the worker. - ip: Required. The IP address of the node. + ip or node_id: Required. The IP address or hex ID of the node. """ pid = req.query.get("pid") ip = req.query.get("ip") + node_id_hex = req.query.get("node_id") if not pid: raise ValueError("pid is required") - if not ip: - raise ValueError("ip is required") + if not node_id_hex and not ip: + raise ValueError("ip or node_id is required") - addrs = await self._get_stub_address_by_ip(ip) - if not addrs: - raise aiohttp.web.HTTPInternalServerError( - text=f"Failed to get agent address for node at IP {ip}" + if node_id_hex: + addrs = await self._get_stub_address_by_node_id( + NodeID.from_hex(node_id_hex) ) + if not addrs: + raise aiohttp.web.HTTPInternalServerError( + text=f"Failed to get agent address for node at node_id {node_id_hex}" + ) + else: + addrs = await self._get_stub_address_by_ip(ip) + if not addrs: + raise aiohttp.web.HTTPInternalServerError( + text=f"Failed to get agent address for node at IP {ip}" + ) + node_id, ip, http_port, grpc_port = addrs reporter_stub = self._make_stub(build_address(ip, grpc_port)) # Default not using `--native` for profiling @@ -451,29 +462,40 @@ async def cpu_profile(self, req: aiohttp.web.Request) -> aiohttp.web.Response: Params: pid: Required. The PID of the worker. - ip: Required. The IP address of the node. + ip or node_id: Required. The IP address or hex ID of the node. duration: Optional. Duration in seconds for profiling (default: 5, max: 60). format: Optional. Output format (default: "flamegraph"). native: Optional. Whether to use native profiling (default: false). Raises: ValueError: If pid is not provided. - ValueError: If ip is not provided. + ValueError: If ip or node_id is not provided. ValueError: If duration exceeds 60 seconds. aiohttp.web.HTTPInternalServerError: If there is an internal server error during the profile retrieval. """ pid = req.query.get("pid") ip = req.query.get("ip") + node_id_hex = req.query.get("node_id") if not pid: raise ValueError("pid is required") - if not ip: - raise ValueError("ip is required") + if not node_id_hex and not ip: + raise ValueError("ip or node_id is required") - addrs = await self._get_stub_address_by_ip(ip) - if not addrs: - raise aiohttp.web.HTTPInternalServerError( - text=f"Failed to get agent address for node at IP {ip}" + if node_id_hex: + addrs = await self._get_stub_address_by_node_id( + NodeID.from_hex(node_id_hex) ) + if not addrs: + raise aiohttp.web.HTTPInternalServerError( + text=f"Failed to get agent address for node at node_id {node_id_hex}" + ) + else: + addrs = await self._get_stub_address_by_ip(ip) + if not addrs: + raise aiohttp.web.HTTPInternalServerError( + text=f"Failed to get agent address for node at IP {ip}" + ) + node_id, ip, http_port, grpc_port = addrs reporter_stub = self._make_stub(build_address(ip, grpc_port)) @@ -517,7 +539,7 @@ async def gpu_profile(self, req: aiohttp.web.Request) -> aiohttp.web.Response: Params: req: A request with the following query parameters: pid: Required. The PID of the GPU training worker. - ip: Required. The IP address of the node where the GPU training worker is running. + ip or node_id: Required. The IP address or hex ID of the node where the GPU training worker is running. num_iterations: Number of training steps for profiling. Defaults to 4 This is the number of calls to the torch Optimizer.step(). @@ -536,16 +558,27 @@ async def gpu_profile(self, req: aiohttp.web.Request) -> aiohttp.web.Response: pid = req.query.get("pid") ip = req.query.get("ip") + node_id_hex = req.query.get("node_id") if not pid: raise ValueError("pid is required") - if not ip: - raise ValueError("ip is required") + if not node_id_hex and not ip: + raise ValueError("ip or node_id is required") - addrs = await self._get_stub_address_by_ip(ip) - if not addrs: - raise aiohttp.web.HTTPInternalServerError( - text=f"Failed to get agent address for node at IP {ip}, pid {pid}" + if node_id_hex: + addrs = await self._get_stub_address_by_node_id( + NodeID.from_hex(node_id_hex) ) + if not addrs: + raise aiohttp.web.HTTPInternalServerError( + text=f"Failed to get agent address for node at node_id {node_id_hex}, pid {pid}" + ) + else: + addrs = await self._get_stub_address_by_ip(ip) + if not addrs: + raise aiohttp.web.HTTPInternalServerError( + text=f"Failed to get agent address for node at IP {ip}, pid {pid}" + ) + node_id, ip, http_port, grpc_port = addrs reporter_stub = self._make_stub(build_address(ip, grpc_port)) @@ -592,7 +625,7 @@ async def memory_profile(self, req: aiohttp.web.Request) -> aiohttp.web.Response Params (1): pid: The PID of the worker. - ip: The IP address of the node. + ip or node_id: The IP address or hex ID of the node. Params (2): task_id: The ID of the task. @@ -601,7 +634,7 @@ async def memory_profile(self, req: aiohttp.web.Request) -> aiohttp.web.Response Raises: aiohttp.web.HTTPInternalServerError: If no stub - found from the given IP value + found from the given IP address or hex ID value aiohttp.web.HTTPInternalServerError: If the "task_id" parameter exists but either "attempt_number" or "node id" is missing in the request query. @@ -652,12 +685,27 @@ async def memory_profile(self, req: aiohttp.web.Request) -> aiohttp.web.Response else: pid = int(req.query["pid"]) ip = req.query.get("ip") - addrs = await self._get_stub_address_by_ip(ip) - if not addrs: - return aiohttp.web.HTTPInternalServerError( - text=f"Failed to execute: no agent address found for node IP {ip}" + node_id_hex = req.query.get("node_id") + + if not node_id_hex and not ip: + raise ValueError("ip or node_id is required") + + if node_id_hex: + addrs = await self._get_stub_address_by_node_id( + NodeID.from_hex(node_id_hex) ) - _, ip, _, grpc_port = addrs + if not addrs: + return aiohttp.web.HTTPInternalServerError( + text=f"Failed to execute: no agent address found for node {node_id_hex}" + ) + _, ip, _, grpc_port = addrs + else: + addrs = await self._get_stub_address_by_ip(ip) + if not addrs: + return aiohttp.web.HTTPInternalServerError( + text=f"Failed to execute: no agent address found for node IP {ip}" + ) + _, ip, _, grpc_port = addrs assert pid is not None ip_port = build_address(ip, grpc_port) diff --git a/python/ray/tests/test_dashboard_profiler.py b/python/ray/tests/test_dashboard_profiler.py index 721dffff7fc2..0a261e5f5951 100644 --- a/python/ray/tests/test_dashboard_profiler.py +++ b/python/ray/tests/test_dashboard_profiler.py @@ -23,7 +23,8 @@ reason="Fails on OSX: https://github.com/ray-project/ray/issues/30114", ) @pytest.mark.parametrize("native", ["0", "1"]) -def test_profiler_endpoints(ray_start_with_dashboard, native): +@pytest.mark.parametrize("node_info", ["node_id", "ip"]) +def test_profiler_endpoints(ray_start_with_dashboard, native, node_info): # Sanity check py-spy are installed. subprocess.check_call(["py-spy", "--version"]) @@ -45,10 +46,19 @@ def do_stuff_infinite(self): pid = ray.get(a.getpid.remote()) a.do_stuff_infinite.remote() + node_id = ray_start_with_dashboard.address_info["node_id"] node_ip = ray_start_with_dashboard.address_info["node_ip_address"] + def get_node_info(): + if node_info == "node_id": + return f"node_id={node_id}" + else: + return f"ip={node_ip}" + def get_actor_stack(): - url = f"{webui_url}/worker/traceback?pid={pid}&ip={node_ip}&native={native}" + url = ( + f"{webui_url}/worker/traceback?pid={pid}&{get_node_info()}&native={native}" + ) print("GET URL", url) response = requests.get(url) print("STATUS CODE", response.status_code) @@ -73,7 +83,7 @@ def get_actor_stack(): def get_actor_flamegraph(): response = requests.get( - f"{webui_url}/worker/cpu_profile?pid={pid}&ip={node_ip}&native={native}" + f"{webui_url}/worker/cpu_profile?pid={pid}&{get_node_info()}&native={native}" ) response.raise_for_status() assert response.headers["Content-Type"] == "image/svg+xml", response.headers @@ -106,7 +116,8 @@ def get_actor_flamegraph(): reason="Fails on OSX, requires memray & lldb installed in osx image", ) @pytest.mark.parametrize("leaks", ["0", "1"]) -def test_memory_profiler_endpoint(ray_start_with_dashboard, leaks): +@pytest.mark.parametrize("node_info", ["node_id", "ip"]) +def test_memory_profiler_endpoint(ray_start_with_dashboard, leaks, node_info): # Sanity check memray are installed. subprocess.check_call(["memray", "--version"]) @@ -128,11 +139,18 @@ def do_stuff_infinite(self): pid = ray.get(a.getpid.remote()) a.do_stuff_infinite.remote() + node_id = ray_start_with_dashboard.address_info["node_id"] node_ip = ray_start_with_dashboard.address_info["node_ip_address"] + def get_node_info(): + if node_info == "node_id": + return f"node_id={node_id}" + else: + return f"ip={node_ip}" + def get_actor_memory_flamegraph(): response = requests.get( - f"{webui_url}/memory_profile?pid={pid}&ip={node_ip}&leaks={leaks}&duration=5" + f"{webui_url}/memory_profile?pid={pid}&{get_node_info()}&leaks={leaks}&duration=5" ) response.raise_for_status() @@ -156,7 +174,7 @@ def get_actor_memory_flamegraph(): def get_actor_memory_multiple_flamegraphs(): response = requests.get( - f"{webui_url}/memory_profile?pid={pid}&ip={node_ip}&leaks={leaks}&duration=5" + f"{webui_url}/memory_profile?pid={pid}&{get_node_info()}&leaks={leaks}&duration=5" ) response.raise_for_status() @@ -189,7 +207,8 @@ def get_actor_memory_multiple_flamegraphs(): sys.platform == "darwin", reason="Fails on OSX, requires memray & lldb installed in osx image", ) -def test_profiler_failure_message(ray_start_with_dashboard): +@pytest.mark.parametrize("node_info", ["node_id", "ip"]) +def test_profiler_failure_message(ray_start_with_dashboard, node_info): # Sanity check py-spy and memray is installed. subprocess.check_call(["py-spy", "--version"]) subprocess.check_call(["memray", "--version"]) @@ -212,10 +231,19 @@ def do_stuff_infinite(self): pid = ray.get(a.getpid.remote()) a.do_stuff_infinite.remote() + node_id = ray_start_with_dashboard.address_info["node_id"] node_ip = ray_start_with_dashboard.address_info["node_ip_address"] + def get_node_info(): + if node_info == "node_id": + return f"node_id={node_id}" + else: + return f"ip={node_ip}" + def get_actor_stack(): - response = requests.get(f"{webui_url}/worker/traceback?pid={pid}&ip={node_ip}") + response = requests.get( + f"{webui_url}/worker/traceback?pid={pid}&{get_node_info()}" + ) response.raise_for_status() content = response.content.decode("utf-8") print("CONTENT", content) @@ -230,33 +258,42 @@ def get_actor_stack(): ) # Check we return the right status code and error message on failure. - response = requests.get(f"{webui_url}/worker/traceback?pid=1234567&ip={node_ip}") + response = requests.get( + f"{webui_url}/worker/traceback?pid=1234567&{get_node_info()}" + ) content = response.content.decode("utf-8") print(content) assert "text/plain" in response.headers["Content-Type"], response.headers assert "Failed to execute" in content, content # Check we return the right status code and error message on failure. - response = requests.get(f"{webui_url}/worker/cpu_profile?pid=1234567&ip={node_ip}") + response = requests.get( + f"{webui_url}/worker/cpu_profile?pid=1234567&{get_node_info()}" + ) content = response.content.decode("utf-8") print(content) assert "text/plain" in response.headers["Content-Type"], response.headers assert "Failed to execute" in content, content # Check we return the right status code and error message on failure. - response = requests.get(f"{webui_url}/memory_profile?pid=1234567&ip={node_ip}") + response = requests.get(f"{webui_url}/memory_profile?pid=1234567&{get_node_info()}") content = response.content.decode("utf-8") print(content) assert "text/plain" in response.headers["Content-Type"], response.headers assert "Failed to execute" in content, content - # Check wrong ip failure - response = requests.get(f"{webui_url}/memory_profile?pid=1234567&ip=1.2.3.4") + # Check wrong ID/ip failure + if node_info == "node_id": + wrong_param = "node_id=DUMMY_ID" + expect_msg = "Failed to execute: no agent address found for node DUMMY_ID" + else: + wrong_param = "ip=1.2.3.4" + expect_msg = "Failed to execute: no agent address found for node IP 1.2.3.4" + + response = requests.get(f"{webui_url}/memory_profile?pid=1234567&{wrong_param}") content = response.content.decode("utf-8") print(content) - assert ( - "Failed to execute: no agent address found for node IP 1.2.3.4" in content - ), content + assert expect_msg in content, content if __name__ == "__main__": From 306db86dad3f124792bb8eba2b44d3e903dfca63 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Tue, 26 Aug 2025 09:29:12 -0700 Subject: [PATCH 276/634] [core][obsclean/04] move metric registration to constructor (#55544) Currently, the `Metric` class registers the metric name each time it records a new value, due to the [C++ static initialization order fiasco](https://en.cppreference.com/w/cpp/language/siof.html). The base of this PR move all static metrics to runtime initialization, allowing us to safely perform metric registration in the constructor. Note that I'm only doing this for the open-telemetry stack. The opencensus has a complex registration sequence using view+measure and I have hard time make it to work so I don't bother fixing it. Test: - CI Signed-off-by: Cuong Nguyen --- src/ray/stats/metric.cc | 14 -------------- src/ray/stats/metric.h | 24 ++++++++++++++++++++---- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/ray/stats/metric.cc b/src/ray/stats/metric.cc index c2dcd38ddc26..65e7032226bc 100644 --- a/src/ray/stats/metric.cc +++ b/src/ray/stats/metric.cc @@ -114,20 +114,6 @@ void Metric::Record(double value, TagsType tags) { } if (::RayConfig::instance().enable_open_telemetry()) { - // Register the metric if it hasn't been registered yet; otherwise, this is a no-op. - // We defer metric registration until the first time it's recorded, rather than during - // construction, to avoid issues with static initialization order. Specifically, our - // internal Metric objects (see metric_defs.h) are declared as static, and - // constructing another static object within their constructor can lead to crashes at - // program exit due to unpredictable destruction order. - // - // Once these internal Metric objects are migrated to use DEFINE_stats, we can - // safely move the registration logic to the constructor. See - // https://github.com/ray-project/ray/issues/54538 for the backlog of Ray metric infra - // improvements. - // - // This function is thread-safe. - RegisterOpenTelemetryMetric(); // Collect tags from both the metric-specific tags and the global tags. absl::flat_hash_map open_telemetry_tags; std::unordered_set tag_keys_set; diff --git a/src/ray/stats/metric.h b/src/ray/stats/metric.h index f53056492dcc..8ac54d521f31 100644 --- a/src/ray/stats/metric.h +++ b/src/ray/stats/metric.h @@ -163,7 +163,11 @@ class Gauge : public Metric { const std::string &description, const std::string &unit, const std::vector &tag_keys = {}) - : Metric(name, description, unit, tag_keys) {} + : Metric(name, description, unit, tag_keys) { + if (::RayConfig::instance().enable_open_telemetry()) { + RegisterOpenTelemetryMetric(); + } + } private: void RegisterView() override; @@ -178,7 +182,11 @@ class Histogram : public Metric { const std::string &unit, const std::vector &boundaries, const std::vector &tag_keys = {}) - : Metric(name, description, unit, tag_keys), boundaries_(boundaries) {} + : Metric(name, description, unit, tag_keys), boundaries_(boundaries) { + if (::RayConfig::instance().enable_open_telemetry()) { + RegisterOpenTelemetryMetric(); + } + } private: void RegisterView() override; @@ -195,7 +203,11 @@ class Count : public Metric { const std::string &description, const std::string &unit, const std::vector &tag_keys = {}) - : Metric(name, description, unit, tag_keys) {} + : Metric(name, description, unit, tag_keys) { + if (::RayConfig::instance().enable_open_telemetry()) { + RegisterOpenTelemetryMetric(); + } + } private: void RegisterView() override; @@ -209,7 +221,11 @@ class Sum : public Metric { const std::string &description, const std::string &unit, const std::vector &tag_keys = {}) - : Metric(name, description, unit, tag_keys) {} + : Metric(name, description, unit, tag_keys) { + if (::RayConfig::instance().enable_open_telemetry()) { + RegisterOpenTelemetryMetric(); + } + } private: void RegisterView() override; From 08ad2a3fa6edc3c870ce005dade68cb4fb6a9574 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 26 Aug 2025 11:46:50 -0500 Subject: [PATCH 277/634] [core] Remove `node_manager_server.h` header from `node_manager_client` target (#55953) I don't know... Signed-off-by: Edward Oakes --- src/ray/rpc/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ray/rpc/BUILD.bazel b/src/ray/rpc/BUILD.bazel index 41e9e92f0121..70d7a6cc97d2 100644 --- a/src/ray/rpc/BUILD.bazel +++ b/src/ray/rpc/BUILD.bazel @@ -128,7 +128,6 @@ ray_cc_library( srcs = ["node_manager/raylet_client_pool.cc"], hdrs = [ "node_manager/node_manager_client.h", - "node_manager/node_manager_server.h", "node_manager/raylet_client_pool.h", ] + [ # TODO(eoakes): these are needed due to a circular dependency: From 3119676929ef83820770c5e986426c2303ef0fde Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 26 Aug 2025 11:48:44 -0500 Subject: [PATCH 278/634] [core] Remove `gcs_client` dependency in `gcs_server` tests (#55902) Unused. --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/tests/BUILD.bazel | 2 - .../gcs_server/tests/gcs_server_test_util.h | 50 ------------------- 2 files changed, 52 deletions(-) diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index e4db2be85604..d5417e0647f3 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -81,7 +81,6 @@ ray_cc_library( ], deps = [ "//:ray_fakes", - "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/store_client:in_memory_store_client", ], ) @@ -202,7 +201,6 @@ ray_cc_test( deps = [ ":gcs_server_test_util", "//src/mock/ray/pubsub:mock_publisher", - "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h index 43c2bb1a163d..eb8cf7da9eba 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h +++ b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h @@ -27,7 +27,6 @@ #include "ray/common/task/task.h" #include "ray/common/task/task_util.h" #include "ray/common/test_util.h" -#include "ray/gcs/gcs_client/accessor.h" #include "ray/gcs/gcs_server/gcs_actor_manager.h" #include "ray/gcs/gcs_server/gcs_actor_scheduler.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" @@ -382,55 +381,6 @@ struct GcsServerMocker { std::shared_ptr store_client_ = std::make_shared(); }; - - class MockedNodeInfoAccessor : public gcs::NodeInfoAccessor { - public: - Status RegisterSelf(const rpc::GcsNodeInfo &local_node_info, - const gcs::StatusCallback &callback) override { - return Status::NotImplemented(""); - } - - const NodeID &GetSelfId() const override { - static NodeID node_id; - return node_id; - } - - const rpc::GcsNodeInfo &GetSelfInfo() const override { - static rpc::GcsNodeInfo node_info; - return node_info; - } - - void AsyncRegister(const rpc::GcsNodeInfo &node_info, - const gcs::StatusCallback &callback) override {} - - void AsyncGetAll(const gcs::MultiItemCallback &callback, - int64_t timeout_ms, - const std::vector &node_ids = {}) override { - if (callback) { - callback(Status::OK(), {}); - } - } - - void AsyncSubscribeToNodeChange( - std::function subscribe, - gcs::StatusCallback done) override { - RAY_LOG(FATAL) << "Not implemented"; - } - - const rpc::GcsNodeInfo *Get(const NodeID &node_id, - bool filter_dead_nodes = true) const override { - return nullptr; - } - - const absl::flat_hash_map &GetAll() const override { - static absl::flat_hash_map node_info_list; - return node_info_list; - } - - bool IsNodeDead(const NodeID &node_id) const override { return false; } - - void AsyncResubscribe() override {} - }; }; } // namespace ray From 17bc93a1b310288cb3071049ad56270dcc4b8c58 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 26 Aug 2025 11:51:47 -0500 Subject: [PATCH 279/634] [core] Remove `gcs_rpc_server.h` dependency from `GcsWorkerManager` (#55954) Splitting gRPC service interface from implementation. --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 2 +- src/ray/gcs/gcs_server/gcs_server.cc | 4 +- src/ray/gcs/gcs_server/gcs_worker_manager.h | 5 +- .../gcs/gcs_server/grpc_service_interfaces.h | 30 ++++++++ src/ray/gcs/gcs_server/grpc_services.cc | 16 +++++ src/ray/gcs/gcs_server/grpc_services.h | 23 +++++++ src/ray/rpc/gcs/gcs_rpc_server.h | 68 ------------------- 7 files changed, 75 insertions(+), 73 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index b0af984c2388..1620daef6773 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -145,8 +145,8 @@ ray_cc_library( ":gcs_kv_manager", ":gcs_table_storage", ":gcs_usage_stats_client", + ":grpc_service_interfaces", "//src/ray/gcs/pubsub:gcs_pub_sub_lib", - "//src/ray/rpc:gcs_server", "//src/ray/stats:stats_metric", ], ) diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index a9b9f9d381f3..f52499d22886 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -706,7 +706,9 @@ void GcsServer::InitGcsWorkerManager() { gcs_worker_manager_ = std::make_unique( *gcs_table_storage_, io_context_provider_.GetDefaultIOContext(), *gcs_publisher_); rpc_server_.RegisterService(std::make_unique( - io_context_provider_.GetDefaultIOContext(), *gcs_worker_manager_)); + io_context_provider_.GetDefaultIOContext(), + *gcs_worker_manager_, + RayConfig::instance().gcs_max_active_rpcs_per_handler())); } void GcsServer::InitGcsAutoscalerStateManager(const GcsInitData &gcs_init_data) { diff --git a/src/ray/gcs/gcs_server/gcs_worker_manager.h b/src/ray/gcs/gcs_server/gcs_worker_manager.h index fb33d80ddb6c..0912e625d1e6 100644 --- a/src/ray/gcs/gcs_server/gcs_worker_manager.h +++ b/src/ray/gcs/gcs_server/gcs_worker_manager.h @@ -18,15 +18,14 @@ #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/usage_stats_client.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" namespace ray { namespace gcs { -/// This implementation class of `WorkerInfoHandler`. -class GcsWorkerManager : public rpc::WorkerInfoHandler { +class GcsWorkerManager : public rpc::WorkerInfoGcsServiceHandler { public: GcsWorkerManager(gcs::GcsTableStorage &gcs_table_storage, instrumented_io_context &io_context, diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h index 74f1eda145a2..89aab76879a8 100644 --- a/src/ray/gcs/gcs_server/grpc_service_interfaces.h +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -73,5 +73,35 @@ class RuntimeEnvGcsServiceHandler { SendReplyCallback send_reply_callback) = 0; }; +class WorkerInfoGcsServiceHandler { + public: + virtual ~WorkerInfoGcsServiceHandler() = default; + + virtual void HandleReportWorkerFailure(ReportWorkerFailureRequest request, + ReportWorkerFailureReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetWorkerInfo(GetWorkerInfoRequest request, + GetWorkerInfoReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetAllWorkerInfo(GetAllWorkerInfoRequest request, + GetAllWorkerInfoReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleAddWorkerInfo(AddWorkerInfoRequest request, + AddWorkerInfoReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleUpdateWorkerDebuggerPort(UpdateWorkerDebuggerPortRequest request, + UpdateWorkerDebuggerPortReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleUpdateWorkerNumPausedThreads( + UpdateWorkerNumPausedThreadsRequest request, + UpdateWorkerNumPausedThreadsReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc index 841c7ad3814a..a0de6d357d8b 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -44,5 +44,21 @@ void RuntimeEnvGrpcService::InitServerCallFactories( RuntimeEnvGcsService, PinRuntimeEnvURI, max_active_rpcs_per_handler_); } +void WorkerInfoGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + RPC_SERVICE_HANDLER( + WorkerInfoGcsService, ReportWorkerFailure, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(WorkerInfoGcsService, GetWorkerInfo, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + WorkerInfoGcsService, GetAllWorkerInfo, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(WorkerInfoGcsService, AddWorkerInfo, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + WorkerInfoGcsService, UpdateWorkerDebuggerPort, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + WorkerInfoGcsService, UpdateWorkerNumPausedThreads, max_active_rpcs_per_handler_) +} + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/gcs_server/grpc_services.h index f438e717cf9b..98268354000b 100644 --- a/src/ray/gcs/gcs_server/grpc_services.h +++ b/src/ray/gcs/gcs_server/grpc_services.h @@ -80,5 +80,28 @@ class RuntimeEnvGrpcService : public GrpcService { int64_t max_active_rpcs_per_handler_; }; +class WorkerInfoGrpcService : public GrpcService { + public: + explicit WorkerInfoGrpcService(instrumented_io_context &io_service, + WorkerInfoGcsServiceHandler &handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler){}; + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + WorkerInfoGcsService::AsyncService service_; + WorkerInfoGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + } // namespace rpc } // namespace ray diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index ce83ef508d72..db0d3bb9db14 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -142,11 +142,6 @@ namespace rpc { HANDLER, \ RayConfig::instance().gcs_max_active_rpcs_per_handler()) -#define WORKER_INFO_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(WorkerInfoGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - #define PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(HANDLER) \ RPC_SERVICE_HANDLER(PlacementGroupInfoGcsService, \ HANDLER, \ @@ -363,68 +358,6 @@ class NodeResourceInfoGrpcService : public GrpcService { NodeResourceInfoGcsServiceHandler &service_handler_; }; -class WorkerInfoGcsServiceHandler { - public: - virtual ~WorkerInfoGcsServiceHandler() = default; - - virtual void HandleReportWorkerFailure(ReportWorkerFailureRequest request, - ReportWorkerFailureReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetWorkerInfo(GetWorkerInfoRequest request, - GetWorkerInfoReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetAllWorkerInfo(GetAllWorkerInfoRequest request, - GetAllWorkerInfoReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleAddWorkerInfo(AddWorkerInfoRequest request, - AddWorkerInfoReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleUpdateWorkerDebuggerPort(UpdateWorkerDebuggerPortRequest request, - UpdateWorkerDebuggerPortReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleUpdateWorkerNumPausedThreads( - UpdateWorkerNumPausedThreadsRequest request, - UpdateWorkerNumPausedThreadsReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -/// The `GrpcService` for `WorkerInfoGcsService`. -class WorkerInfoGrpcService : public GrpcService { - public: - /// Constructor. - /// - /// \param[in] handler The service handler that actually handle the requests. - explicit WorkerInfoGrpcService(instrumented_io_context &io_service, - WorkerInfoGcsServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler){}; - - protected: - grpc::Service &GetGrpcService() override { return service_; } - - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - WORKER_INFO_SERVICE_RPC_HANDLER(ReportWorkerFailure); - WORKER_INFO_SERVICE_RPC_HANDLER(GetWorkerInfo); - WORKER_INFO_SERVICE_RPC_HANDLER(GetAllWorkerInfo); - WORKER_INFO_SERVICE_RPC_HANDLER(AddWorkerInfo); - WORKER_INFO_SERVICE_RPC_HANDLER(UpdateWorkerDebuggerPort); - WORKER_INFO_SERVICE_RPC_HANDLER(UpdateWorkerNumPausedThreads); - } - - private: - /// The grpc async service object. - WorkerInfoGcsService::AsyncService service_; - /// The service handler that actually handle the requests. - WorkerInfoGcsServiceHandler &service_handler_; -}; - class PlacementGroupInfoGcsServiceHandler { public: virtual ~PlacementGroupInfoGcsServiceHandler() = default; @@ -665,7 +598,6 @@ class InternalPubSubGrpcService : public GrpcService { using JobInfoHandler = JobInfoGcsServiceHandler; using ActorInfoHandler = ActorInfoGcsServiceHandler; using NodeResourceInfoHandler = NodeResourceInfoGcsServiceHandler; -using WorkerInfoHandler = WorkerInfoGcsServiceHandler; using PlacementGroupInfoHandler = PlacementGroupInfoGcsServiceHandler; using InternalKVHandler = InternalKVGcsServiceHandler; using InternalPubSubHandler = InternalPubSubGcsServiceHandler; From 0b58406d91db1216c70add64339ca8bddea92a27 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 26 Aug 2025 11:57:45 -0500 Subject: [PATCH 280/634] [core] Remove `gcs_rpc_server.h` dependency from `GcsJobManager` (#55899) Depends on: https://github.com/ray-project/ray/pull/55886 --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 11 ++-- src/ray/gcs/gcs_server/gcs_job_manager.cc | 4 ++ src/ray/gcs/gcs_server/gcs_job_manager.h | 19 +++--- src/ray/gcs/gcs_server/gcs_server.cc | 39 +---------- .../gcs/gcs_server/grpc_service_interfaces.h | 29 ++++++++ src/ray/gcs/gcs_server/grpc_services.cc | 13 +++- src/ray/gcs/gcs_server/grpc_services.h | 24 +++++++ src/ray/rpc/gcs/gcs_rpc_server.h | 66 ------------------- 8 files changed, 88 insertions(+), 117 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 1620daef6773..d8d1aac34681 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -204,20 +204,21 @@ ray_cc_library( name = "gcs_job_manager", srcs = ["gcs_job_manager.cc"], hdrs = ["gcs_job_manager.h"], - deps = [ + implementation_deps = [ ":gcs_function_manager", ":gcs_init_data", ":gcs_table_storage", - "//src/ray/common:runtime_env", "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/pubsub:gcs_pub_sub_lib", - "//src/ray/rpc:core_worker_client", - "//src/ray/rpc:gcs_server", "//src/ray/stats:stats_metric", + ], + deps = [ + ":grpc_service_interfaces", + "//src/ray/common:runtime_env", + "//src/ray/rpc:core_worker_client", "//src/ray/util:event", "//src/ray/util:thread_checker", "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", ], ) diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.cc b/src/ray/gcs/gcs_server/gcs_job_manager.cc index e71c69bccedc..ae55183c416c 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_job_manager.cc @@ -22,7 +22,11 @@ #include #include "absl/strings/match.h" +#include "ray/gcs/gcs_server/gcs_function_manager.h" +#include "ray/gcs/gcs_server/gcs_init_data.h" +#include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/pb_util.h" +#include "ray/gcs/pubsub/gcs_pub_sub.h" #include "ray/stats/metric.h" #include "ray/util/time.h" diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.h b/src/ray/gcs/gcs_server/gcs_job_manager.h index a9484e7c0277..97da33b4b478 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.h +++ b/src/ray/gcs/gcs_server/gcs_job_manager.h @@ -21,13 +21,8 @@ #include #include "absl/container/flat_hash_map.h" -#include "absl/container/flat_hash_set.h" #include "ray/common/runtime_env_manager.h" -#include "ray/gcs/gcs_server/gcs_function_manager.h" -#include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/event.h" @@ -46,10 +41,16 @@ inline std::string JobDataKey(const std::string &submission_id) { return kJobDataKeyPrefix + submission_id; } -using JobFinishListenerCallback = rpc::JobInfoHandler::JobFinishListenerCallback; +using JobFinishListenerCallback = + rpc::JobInfoGcsServiceHandler::JobFinishListenerCallback; -/// This implementation class of `JobInfoHandler`. -class GcsJobManager : public rpc::JobInfoHandler { +class GcsInitData; +class GcsTableStorage; +class GcsPublisher; +class GCSFunctionManager; +class InternalKVInterface; + +class GcsJobManager : public rpc::JobInfoGcsServiceHandler { public: explicit GcsJobManager(GcsTableStorage &gcs_table_storage, GcsPublisher &gcs_publisher, diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index f52499d22886..9a5b55ef5561 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -236,58 +236,23 @@ void GcsServer::GetOrGenerateClusterId( } void GcsServer::DoStart(const GcsInitData &gcs_init_data) { - // Init cluster resource scheduler. InitClusterResourceScheduler(); - - // Init gcs node manager. InitGcsNodeManager(gcs_init_data); - - // Init cluster task manager. InitClusterTaskManager(); - - // Init gcs resource manager. InitGcsResourceManager(gcs_init_data); - - // Init gcs health check manager. InitGcsHealthCheckManager(gcs_init_data); - - // Init synchronization service InitRaySyncer(gcs_init_data); - - // Init KV service. InitKVService(); - - // Init function manager InitFunctionManager(); - - // Init Pub/Sub handler InitPubSubHandler(); - - // Init RuntimeEnv manager InitRuntimeEnvManager(); - - // Init gcs job manager. InitGcsJobManager(gcs_init_data); - - // Init gcs placement group manager. InitGcsPlacementGroupManager(gcs_init_data); - - // Init gcs actor manager. InitGcsActorManager(gcs_init_data); - - // Init gcs worker manager. InitGcsWorkerManager(); - - // Init GCS task manager. InitGcsTaskManager(); - - // Install event listeners. InstallEventListeners(); - - // Init autoscaling manager InitGcsAutoscalerStateManager(gcs_init_data); - - // Init usage stats client. InitUsageStatsClient(); // Init OpenTelemetry exporter. @@ -483,7 +448,9 @@ void GcsServer::InitGcsJobManager(const GcsInitData &gcs_init_data) { gcs_job_manager_->Initialize(gcs_init_data); rpc_server_.RegisterService(std::make_unique( - io_context_provider_.GetDefaultIOContext(), *gcs_job_manager_)); + io_context_provider_.GetDefaultIOContext(), + *gcs_job_manager_, + RayConfig::instance().gcs_max_active_rpcs_per_handler())); } void GcsServer::InitGcsActorManager(const GcsInitData &gcs_init_data) { diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h index 89aab76879a8..d8dbb7febf4d 100644 --- a/src/ray/gcs/gcs_server/grpc_service_interfaces.h +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -64,6 +64,35 @@ class NodeInfoGcsServiceHandler { SendReplyCallback send_reply_callback) = 0; }; +class JobInfoGcsServiceHandler { + public: + using JobFinishListenerCallback = std::function; + + virtual ~JobInfoGcsServiceHandler() = default; + + virtual void HandleAddJob(AddJobRequest request, + AddJobReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleMarkJobFinished(MarkJobFinishedRequest request, + MarkJobFinishedReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetAllJobInfo(GetAllJobInfoRequest request, + GetAllJobInfoReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void AddJobFinishedListener(JobFinishListenerCallback listener) = 0; + + virtual void HandleReportJobError(ReportJobErrorRequest request, + ReportJobErrorReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetNextJobID(GetNextJobIDRequest request, + GetNextJobIDReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + class RuntimeEnvGcsServiceHandler { public: virtual ~RuntimeEnvGcsServiceHandler() = default; diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc index a0de6d357d8b..a35a43e68c42 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -36,12 +36,23 @@ void NodeInfoGrpcService::InitServerCallFactories( RPC_SERVICE_HANDLER(NodeInfoGcsService, CheckAlive, max_active_rpcs_per_handler_) } +void JobInfoGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + RPC_SERVICE_HANDLER(JobInfoGcsService, AddJob, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(JobInfoGcsService, MarkJobFinished, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(JobInfoGcsService, GetAllJobInfo, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(JobInfoGcsService, ReportJobError, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(JobInfoGcsService, GetNextJobID, max_active_rpcs_per_handler_) +} + void RuntimeEnvGrpcService::InitServerCallFactories( const std::unique_ptr &cq, std::vector> *server_call_factories, const ClusterID &cluster_id) { RPC_SERVICE_HANDLER( - RuntimeEnvGcsService, PinRuntimeEnvURI, max_active_rpcs_per_handler_); + RuntimeEnvGcsService, PinRuntimeEnvURI, max_active_rpcs_per_handler_) } void WorkerInfoGrpcService::InitServerCallFactories( diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/gcs_server/grpc_services.h index 98268354000b..90ef1d75a4e3 100644 --- a/src/ray/gcs/gcs_server/grpc_services.h +++ b/src/ray/gcs/gcs_server/grpc_services.h @@ -58,6 +58,29 @@ class NodeInfoGrpcService : public GrpcService { int64_t max_active_rpcs_per_handler_; }; +class JobInfoGrpcService : public GrpcService { + public: + explicit JobInfoGrpcService(instrumented_io_context &io_service, + JobInfoGcsServiceHandler &handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler){}; + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + JobInfoGcsService::AsyncService service_; + JobInfoGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + class RuntimeEnvGrpcService : public GrpcService { public: explicit RuntimeEnvGrpcService(instrumented_io_context &io_service, @@ -69,6 +92,7 @@ class RuntimeEnvGrpcService : public GrpcService { protected: grpc::Service &GetGrpcService() override { return service_; } + void InitServerCallFactories( const std::unique_ptr &cq, std::vector> *server_call_factories, diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index db0d3bb9db14..65e214245162 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -109,11 +109,6 @@ using AutoscalerStateHandler = AutoscalerStateServiceHandler; namespace ray { namespace rpc { -#define JOB_INFO_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(JobInfoGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - #define ACTOR_INFO_SERVICE_RPC_HANDLER(HANDLER, MAX_ACTIVE_RPCS) \ RPC_SERVICE_HANDLER(ActorInfoGcsService, HANDLER, MAX_ACTIVE_RPCS) @@ -159,66 +154,6 @@ namespace rpc { reply->mutable_status()->set_message(status.message()); \ send_reply_callback(ray::Status::OK(), nullptr, nullptr) -class JobInfoGcsServiceHandler { - public: - using JobFinishListenerCallback = std::function; - - virtual ~JobInfoGcsServiceHandler() = default; - - virtual void HandleAddJob(AddJobRequest request, - AddJobReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleMarkJobFinished(MarkJobFinishedRequest request, - MarkJobFinishedReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetAllJobInfo(GetAllJobInfoRequest request, - GetAllJobInfoReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void AddJobFinishedListener(JobFinishListenerCallback listener) = 0; - - virtual void HandleReportJobError(ReportJobErrorRequest request, - ReportJobErrorReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetNextJobID(GetNextJobIDRequest request, - GetNextJobIDReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -/// The `GrpcService` for `JobInfoGcsService`. -class JobInfoGrpcService : public GrpcService { - public: - /// Constructor. - /// - /// \param[in] handler The service handler that actually handle the requests. - explicit JobInfoGrpcService(instrumented_io_context &io_service, - JobInfoGcsServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler){}; - - protected: - grpc::Service &GetGrpcService() override { return service_; } - - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - JOB_INFO_SERVICE_RPC_HANDLER(AddJob); - JOB_INFO_SERVICE_RPC_HANDLER(MarkJobFinished); - JOB_INFO_SERVICE_RPC_HANDLER(GetAllJobInfo); - JOB_INFO_SERVICE_RPC_HANDLER(ReportJobError); - JOB_INFO_SERVICE_RPC_HANDLER(GetNextJobID); - } - - private: - /// The grpc async service object. - JobInfoGcsService::AsyncService service_; - /// The service handler that actually handle the requests. - JobInfoGcsServiceHandler &service_handler_; -}; - class ActorInfoGcsServiceHandler { public: virtual ~ActorInfoGcsServiceHandler() = default; @@ -595,7 +530,6 @@ class InternalPubSubGrpcService : public GrpcService { InternalPubSubGcsServiceHandler &service_handler_; }; -using JobInfoHandler = JobInfoGcsServiceHandler; using ActorInfoHandler = ActorInfoGcsServiceHandler; using NodeResourceInfoHandler = NodeResourceInfoGcsServiceHandler; using PlacementGroupInfoHandler = PlacementGroupInfoGcsServiceHandler; From 7c5310644833482c2525fd70253805fa591d1756 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 26 Aug 2025 10:12:30 -0700 Subject: [PATCH 281/634] [Core] Bind dashboard agent http server to localhost in addition to the node ip (#55910) Signed-off-by: Jiajun Yao --- python/ray/_common/network_utils.py | 12 ++ .../ray/_common/tests/test_network_utils.py | 115 +++++++++--------- python/ray/dashboard/agent.py | 4 +- python/ray/dashboard/http_server_agent.py | 9 +- python/ray/util/client/server/proxier.py | 4 +- python/ray/util/client/server/server.py | 4 +- 6 files changed, 85 insertions(+), 63 deletions(-) diff --git a/python/ray/_common/network_utils.py b/python/ray/_common/network_utils.py index 8b3c510a09e7..952559142398 100644 --- a/python/ray/_common/network_utils.py +++ b/python/ray/_common/network_utils.py @@ -42,3 +42,15 @@ def build_address(host: str, port: Union[int, str]) -> str: return f"[{host}]:{port}" # IPv4 address or hostname return f"{host}:{port}" + + +def is_localhost(host: str) -> bool: + """Check if the given host string represents a localhost address. + + Args: + host: The hostname or IP address to check. + + Returns: + True if the host is a localhost address, False otherwise. + """ + return host in ("localhost", "127.0.0.1", "::1") diff --git a/python/ray/_common/tests/test_network_utils.py b/python/ray/_common/tests/test_network_utils.py index 2eee263f9a29..98b070cf7b6c 100644 --- a/python/ray/_common/tests/test_network_utils.py +++ b/python/ray/_common/tests/test_network_utils.py @@ -1,70 +1,73 @@ import pytest import sys -from ray._common.network_utils import parse_address, build_address +from ray._common.network_utils import parse_address, build_address, is_localhost -class TestBuildAddress: - """Test cases for build_address function, matching C++ tests exactly.""" +def test_is_localhost(): + assert is_localhost("localhost") + assert is_localhost("127.0.0.1") + assert is_localhost("::1") + assert not is_localhost("8.8.8.8") + assert not is_localhost("2001:db8::1") + - @pytest.mark.parametrize( - "host,port,expected", - [ - # IPv4 - ("192.168.1.1", 8080, "192.168.1.1:8080"), - ("192.168.1.1", "8080", "192.168.1.1:8080"), - # IPv6 - ("::1", 8080, "[::1]:8080"), - ("::1", "8080", "[::1]:8080"), - ("2001:db8::1", 8080, "[2001:db8::1]:8080"), - ("2001:db8::1", "8080", "[2001:db8::1]:8080"), - # Hostname - ("localhost", 9000, "localhost:9000"), - ("localhost", "9000", "localhost:9000"), - ], - ) - def test_build_address(self, host, port, expected): - """Test building address strings from host and port.""" - result = build_address(host, port) - assert result == expected +@pytest.mark.parametrize( + "host,port,expected", + [ + # IPv4 + ("192.168.1.1", 8080, "192.168.1.1:8080"), + ("192.168.1.1", "8080", "192.168.1.1:8080"), + # IPv6 + ("::1", 8080, "[::1]:8080"), + ("::1", "8080", "[::1]:8080"), + ("2001:db8::1", 8080, "[2001:db8::1]:8080"), + ("2001:db8::1", "8080", "[2001:db8::1]:8080"), + # Hostname + ("localhost", 9000, "localhost:9000"), + ("localhost", "9000", "localhost:9000"), + ], +) +def test_build_address(host, port, expected): + """Test cases for build_address function, matching C++ tests exactly.""" + result = build_address(host, port) + assert result == expected -class TestParseAddress: +@pytest.mark.parametrize( + "address,expected", + [ + # IPv4 + ("192.168.1.1:8080", ("192.168.1.1", "8080")), + # IPv6:loopback address + ("[::1]:8080", ("::1", "8080")), + # IPv6 + ("[2001:db8::1]:8080", ("2001:db8::1", "8080")), + # Hostname:Port + ("localhost:9000", ("localhost", "9000")), + ], +) +def test_parse_valid_addresses(address, expected): """Test cases for parse_address function, matching C++ tests exactly.""" + result = parse_address(address) + assert result == expected - @pytest.mark.parametrize( - "address,expected", - [ - # IPv4 - ("192.168.1.1:8080", ("192.168.1.1", "8080")), - # IPv6:loopback address - ("[::1]:8080", ("::1", "8080")), - # IPv6 - ("[2001:db8::1]:8080", ("2001:db8::1", "8080")), - # Hostname:Port - ("localhost:9000", ("localhost", "9000")), - ], - ) - def test_parse_valid_addresses(self, address, expected): - """Test parsing valid addresses.""" - result = parse_address(address) - assert result == expected - @pytest.mark.parametrize( - "address", - [ - # bare IP or hostname - # should return None when no port is found - "::1", - "2001:db8::1", - "192.168.1.1", - "localhost", - ], - ) - def test_parse_bare_addresses(self, address): - """Test parsing bare addresses returns None.""" - result = parse_address(address) - assert result is None +@pytest.mark.parametrize( + "address", + [ + # bare IP or hostname + # should return None when no port is found + "::1", + "2001:db8::1", + "192.168.1.1", + "localhost", + ], +) +def test_parse_bare_addresses(address): + """Test parsing bare addresses returns None.""" + result = parse_address(address) + assert result is None if __name__ == "__main__": diff --git a/python/ray/dashboard/agent.py b/python/ray/dashboard/agent.py index 976a9ee7e1a3..624efc5742bb 100644 --- a/python/ray/dashboard/agent.py +++ b/python/ray/dashboard/agent.py @@ -11,7 +11,7 @@ import ray.dashboard.utils as dashboard_utils from ray._common.utils import get_or_create_event_loop from ray._private import logging_utils -from ray._common.network_utils import build_address +from ray._common.network_utils import build_address, is_localhost from ray._private.process_watcher import create_check_raylet_task from ray._private.ray_constants import AGENT_GRPC_MAX_MESSAGE_LENGTH from ray._private.ray_logging import setup_component_logger @@ -113,7 +113,7 @@ def _init_non_minimal(self): ) try: add_port_to_grpc_server(self.server, build_address(self.ip, self.grpc_port)) - if self.ip != "127.0.0.1" and self.ip != "localhost": + if not is_localhost(self.ip): add_port_to_grpc_server(self.server, f"127.0.0.1:{self.grpc_port}") except Exception: # TODO(SongGuyang): Catch the exception here because there is diff --git a/python/ray/dashboard/http_server_agent.py b/python/ray/dashboard/http_server_agent.py index b71ba6c5a38b..cc3e20d85b87 100644 --- a/python/ray/dashboard/http_server_agent.py +++ b/python/ray/dashboard/http_server_agent.py @@ -7,7 +7,7 @@ import ray.dashboard.optional_utils as dashboard_optional_utils from ray._common.utils import get_or_create_event_loop -from ray._common.network_utils import build_address +from ray._common.network_utils import build_address, is_localhost from ray.dashboard.optional_deps import aiohttp, aiohttp_cors, hdrs logger = logging.getLogger(__name__) @@ -48,6 +48,13 @@ async def _start_site_with_retry( self.listen_port, ) await site.start() + if not is_localhost(self.ip): + local_site = aiohttp.web.TCPSite( + self.runner, + "127.0.0.1", + self.listen_port, + ) + await local_site.start() if attempt > 0: logger.info( f"Successfully started agent on port {self.listen_port} " diff --git a/python/ray/util/client/server/proxier.py b/python/ray/util/client/server/proxier.py index a952cffa8f58..8be870c566d7 100644 --- a/python/ray/util/client/server/proxier.py +++ b/python/ray/util/client/server/proxier.py @@ -28,7 +28,7 @@ from ray._private.services import ProcessInfo, start_ray_client_server from ray._private.tls_utils import add_port_to_grpc_server from ray._private.utils import detect_fate_sharing_support -from ray._common.network_utils import build_address +from ray._common.network_utils import build_address, is_localhost from ray.cloudpickle.compat import pickle from ray.job_config import JobConfig from ray.util.client.common import ( @@ -860,7 +860,7 @@ def serve_proxier( ray_client_pb2_grpc.add_RayletDriverServicer_to_server(task_servicer, server) ray_client_pb2_grpc.add_RayletDataStreamerServicer_to_server(data_servicer, server) ray_client_pb2_grpc.add_RayletLogStreamerServicer_to_server(logs_servicer, server) - if host != "127.0.0.1" and host != "localhost": + if not is_localhost(host): add_port_to_grpc_server(server, f"127.0.0.1:{port}") add_port_to_grpc_server(server, f"{host}:{port}") server.start() diff --git a/python/ray/util/client/server/server.py b/python/ray/util/client/server/server.py index 768ae2c2118a..f4194ae83fa0 100644 --- a/python/ray/util/client/server/server.py +++ b/python/ray/util/client/server/server.py @@ -27,7 +27,7 @@ from ray._private.ray_logging import setup_logger from ray._private.services import canonicalize_bootstrap_address_or_die from ray._private.tls_utils import add_port_to_grpc_server -from ray._common.network_utils import build_address +from ray._common.network_utils import build_address, is_localhost from ray.job_config import JobConfig from ray.util.client.common import ( CLIENT_SERVER_MAX_THREADS, @@ -787,7 +787,7 @@ def default_connect_handler( ray_client_pb2_grpc.add_RayletDriverServicer_to_server(task_servicer, server) ray_client_pb2_grpc.add_RayletDataStreamerServicer_to_server(data_servicer, server) ray_client_pb2_grpc.add_RayletLogStreamerServicer_to_server(logs_servicer, server) - if host != "127.0.0.1" and host != "localhost": + if not is_localhost(host): add_port_to_grpc_server(server, f"127.0.0.1:{port}") add_port_to_grpc_server(server, f"{host}:{port}") current_handle = ClientServerHandle( From 399081db12e398055a5929d6b53ce8a4cafab53f Mon Sep 17 00:00:00 2001 From: Srinath Krishnamachari <68668616+srinathk10@users.noreply.github.com> Date: Tue, 26 Aug 2025 10:27:49 -0700 Subject: [PATCH 282/634] [data] Update `repartition` docs for streaming repartition (#55808) --- doc/source/data/images/dataset-shuffle.svg | 2 +- python/ray/data/dataset.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/source/data/images/dataset-shuffle.svg b/doc/source/data/images/dataset-shuffle.svg index 6745724c722d..018f877fb728 100644 --- a/doc/source/data/images/dataset-shuffle.svg +++ b/doc/source/data/images/dataset-shuffle.svg @@ -1 +1 @@ - + \ No newline at end of file diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 2d619381c66f..2254cc7fb90a 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -1494,7 +1494,6 @@ def filter( logical_plan = LogicalPlan(op, self.context) return Dataset(plan, logical_plan) - @AllToAllAPI @PublicAPI(api_group=SSR_API_GROUP) def repartition( self, @@ -1517,9 +1516,11 @@ def repartition( .. note:: - Repartition has two modes. If ``shuffle=False``, Ray Data performs the - minimal data movement needed to equalize block sizes. Otherwise, Ray Data - performs a full distributed shuffle. + Repartition has three modes: + + * When ``num_blocks`` and ``shuffle=True`` are specified Ray Data performs a full distributed shuffle producing exactly ``num_blocks`` blocks. + * When ``num_blocks`` and ``shuffle=False`` are specified, Ray Data does NOT perform full shuffle, instead opting in for splitting and combining of the blocks attempting to minimize the necessary data movement (relative to full-blown shuffle). Exactly ``num_blocks`` will be produced. + * If ``target_num_rows_per_block`` is set (exclusive with ``num_blocks`` and ``shuffle``), streaming repartitioning will be executed, where blocks will be made to carry no more than ``target_num_rows_per_block``. Smaller blocks will be combined into bigger ones up to ``target_num_rows_per_block`` as well. .. image:: /data/images/dataset-shuffle.svg :align: center @@ -1538,7 +1539,8 @@ def repartition( Args: num_blocks: Number of blocks after repartitioning. target_num_rows_per_block: [Experimental] The target number of rows per block to - repartition. Note that either `num_blocks` or + repartition. Performs streaming repartitioning of the dataset (no shuffling). + Note that either `num_blocks` or `target_num_rows_per_block` must be set, but not both. When `target_num_rows_per_block` is set, it only repartitions :class:`Dataset` :ref:`blocks ` that are larger than From 2937445c5f25280b50c11fdef598772108afc8f7 Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Tue, 26 Aug 2025 10:41:15 -0700 Subject: [PATCH 283/634] [data] truncate sequences in sanitize for struct (#55818) ## Why are these changes needed? `input_files` and other lists can be quite large. We assume that user args aren't that many (and therefore dictionaries shouldn't need to be truncated), and that Logical Operators aren't deeply nested. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu Signed-off-by: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Alexey Kudinkin --- python/ray/data/_internal/metadata_exporter.py | 16 +++++++++++----- python/ray/data/tests/test_state_export.py | 6 ++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/python/ray/data/_internal/metadata_exporter.py b/python/ray/data/_internal/metadata_exporter.py index 17150c5b006c..3ce64c3aef6f 100644 --- a/python/ray/data/_internal/metadata_exporter.py +++ b/python/ray/data/_internal/metadata_exporter.py @@ -157,7 +157,7 @@ class DatasetMetadata: state: str -def _add_ellipsis(s: str, truncate_length: int) -> str: +def _add_ellipsis_for_string(s: str, truncate_length: int) -> str: if len(s) > truncate_length: return s[:truncate_length] + "..." return s @@ -175,18 +175,24 @@ def sanitize_for_struct(obj, truncate_length=DEFAULT_TRUNCATION_LENGTH): # protobuf Struct key names must be strings. return {str(k): sanitize_for_struct(v, truncate_length) for k, v in obj.items()} elif isinstance(obj, str): - return _add_ellipsis(obj, truncate_length) + return _add_ellipsis_for_string(obj, truncate_length) elif isinstance(obj, (Sequence, set)): # Convert all sequence-like types (lists, tuples, sets, bytes, other sequences) to lists - return [sanitize_for_struct(v, truncate_length=truncate_length) for v in obj] + res = [] + for i, v in enumerate(obj): + if i >= truncate_length: + res.append("...") + break + res.append(sanitize_for_struct(v, truncate_length)) + return res else: try: if is_dataclass(obj): return sanitize_for_struct(asdict(obj), truncate_length) - return _add_ellipsis(str(obj), truncate_length) + return _add_ellipsis_for_string(str(obj), truncate_length) except Exception: unk_name = f"{UNKNOWN}: {type(obj).__name__}" - return _add_ellipsis(unk_name, truncate_length) + return _add_ellipsis_for_string(unk_name, truncate_length) def dataset_metadata_to_proto(dataset_metadata: DatasetMetadata) -> Any: diff --git a/python/ray/data/tests/test_state_export.py b/python/ray/data/tests/test_state_export.py index 570927446d55..4bacfbbb6ffe 100644 --- a/python/ray/data/tests/test_state_export.py +++ b/python/ray/data/tests/test_state_export.py @@ -624,6 +624,12 @@ def __str__(self): }, 100, ), + # Test sequence truncation - list longer than truncate_length gets truncated + ( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + ["1", "2", "3", "..."], # Only first 3 elements after truncation + ... + 3, + ), ], ) def test_sanitize_for_struct(input_obj, expected_output, truncate_length): From a6e9955c367e927c568294df7564690301d79ef1 Mon Sep 17 00:00:00 2001 From: dragongu <38997200+dragongu@users.noreply.github.com> Date: Wed, 27 Aug 2025 01:47:19 +0800 Subject: [PATCH 284/634] [data] Add DownstreamCapacityBackpressurePolicy based on downstream processing capacity (#55463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Implement a downstream processing capacity-based backpressure mechanism to address stability and performance issues caused by unbalanced processing speeds across pipeline operators due to user misconfigurations, instance preemptions, and cluster resource scaling. ## Problem Statement Current Ray Data pipelines face several critical challenges: ### 1. Performance & Stability Issues - Large amounts of objects accumulate in memory while downstream operators cannot consume them timely - Memory resource waste and potential spilling leads to significant performance degradation - Pipeline instability due to memory pressure ### 2. Resource Waste in Dynamic Environments - In preemption scenarios, the situation becomes worse as large amounts of objects are repeatedly rebuilt when workers are preempted - Inefficient resource utilization due to upstream-downstream speed mismatch - Wasted compute resources on processing data that cannot be consumed ### 3. Complex Configuration Requirements - Users find it difficult to configure reasonable parallelism ratios - Inappropriate configurations lead to resource waste or insufficient throughput - Especially challenging on elastic resources where capacity changes dynamically ## Solution This PR introduces `DownstreamCapacityBackpressurePolicy` that provides: ### 1. Simplified User Configuration with Adaptive Concurrency - Ray Data automatically adjusts parallelism based on actual pipeline performance - When upstream is blocked due to backpressure, resources are released to allow downstream scaling up - Self-adaptive mechanism reduces the need for manual tuning and complex configuration ### 2. Consistent Pipeline Throughput - Objects output by upstream operators are consumed by downstream as quickly as possible - Ensures stability, saves memory resources, and avoids unnecessary object rebuilding risks - Maintains balanced flow throughout the entire pipeline ## Key Benefits ### 🚀 Performance Improvements - Prevents memory bloat and reduces object spilling - Maintains optimal memory utilization across the pipeline - Eliminates performance degradation from memory pressure ### 🛡️ Enhanced Stability - Handles instance preemptions gracefully - Reduces object rebuilding in dynamic environments - Maintains pipeline stability under varying cluster conditions ### ⚙️ Simplified Operations - Reduces complex configuration requirements - Provides self-adaptive parallelism adjustment - Works effectively on elastic resources ### 💰 Resource Efficiency - Prevents resource waste from unbalanced processing - Optimizes resource allocation across pipeline stages - Reduces unnecessary compute overhead ## Configuration Users can configure the backpressure behavior via DataContext: ```python ctx = ray.data.DataContext.get_current() # Set ratio threshold (default: inf, disabled) ctx.downstream_capacity_backpressure_ratio = 2.0 # Set absolute threshold (default: sys.maxsize, disabled) ctx.downstream_capacity_backpressure_max_queued_bundles = 4000 ``` ### Default Behavior - By default, backpressure is disabled (thresholds set to infinity) to maintain backward compatibility - Users can enable it by setting appropriate threshold values ## Impact & Results This implementation successfully addresses the core challenges: ✅ **Performance & Stability**: Eliminates memory pressure and spilling issues ✅ **Resource Efficiency**: Prevents waste in preemption scenarios and dynamic environments ✅ **Configuration Simplicity**: Reduces complex user configuration requirements ✅ **Adaptive Throughput**: Maintains consistent pipeline performance The solution provides a foundation for more intelligent, self-adaptive Ray Data pipelines that can handle dynamic cluster conditions while maintaining optimal performance and resource utilization. --- ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: dragongu --- python/ray/data/BUILD | 14 ++ .../autoscaling_actor_pool.py | 3 + .../execution/backpressure_policy/__init__.py | 5 + ...downstream_capacity_backpressure_policy.py | 94 +++++++++++++ python/ray/data/context.py | 3 + ...downstream_capacity_backpressure_policy.py | 127 ++++++++++++++++++ 6 files changed, 246 insertions(+) create mode 100644 python/ray/data/_internal/execution/backpressure_policy/downstream_capacity_backpressure_policy.py create mode 100644 python/ray/data/tests/test_downstream_capacity_backpressure_policy.py diff --git a/python/ray/data/BUILD b/python/ray/data/BUILD index 59bc8b57c741..0383364d6ced 100644 --- a/python/ray/data/BUILD +++ b/python/ray/data/BUILD @@ -1323,6 +1323,20 @@ py_test( ], ) +py_test( + name = "test_downstream_capacity_backpressure_policy", + size = "medium", + srcs = ["tests/test_downstream_capacity_backpressure_policy.py"], + tags = [ + "exclusive", + "team:data", + ], + deps = [ + ":conftest", + "//:ray_lib", + ], +) + py_test( name = "test_backpressure_e2e", size = "large", diff --git a/python/ray/data/_internal/actor_autoscaler/autoscaling_actor_pool.py b/python/ray/data/_internal/actor_autoscaler/autoscaling_actor_pool.py index 97d4dc589a18..f6145a4c175f 100644 --- a/python/ray/data/_internal/actor_autoscaler/autoscaling_actor_pool.py +++ b/python/ray/data/_internal/actor_autoscaler/autoscaling_actor_pool.py @@ -109,3 +109,6 @@ def per_actor_resource_usage(self) -> ExecutionResources: def get_pool_util(self) -> float: """Calculate the utilization of the given actor pool.""" ... + + def max_concurrent_tasks(self) -> int: + return self.max_actor_concurrency() * self.num_running_actors() diff --git a/python/ray/data/_internal/execution/backpressure_policy/__init__.py b/python/ray/data/_internal/execution/backpressure_policy/__init__.py index e08fd3a5f55b..c0aad671df10 100644 --- a/python/ray/data/_internal/execution/backpressure_policy/__init__.py +++ b/python/ray/data/_internal/execution/backpressure_policy/__init__.py @@ -2,6 +2,9 @@ from .backpressure_policy import BackpressurePolicy from .concurrency_cap_backpressure_policy import ConcurrencyCapBackpressurePolicy +from .downstream_capacity_backpressure_policy import ( + DownstreamCapacityBackpressurePolicy, +) from .resource_budget_backpressure_policy import ResourceBudgetBackpressurePolicy from ray.data.context import DataContext @@ -14,6 +17,7 @@ ENABLED_BACKPRESSURE_POLICIES = [ ConcurrencyCapBackpressurePolicy, ResourceBudgetBackpressurePolicy, + DownstreamCapacityBackpressurePolicy, ] ENABLED_BACKPRESSURE_POLICIES_CONFIG_KEY = "backpressure_policies.enabled" @@ -33,6 +37,7 @@ def get_backpressure_policies( __all__ = [ "BackpressurePolicy", "ConcurrencyCapBackpressurePolicy", + "DownstreamCapacityBackpressurePolicy", "ENABLED_BACKPRESSURE_POLICIES_CONFIG_KEY", "get_backpressure_policies", ] diff --git a/python/ray/data/_internal/execution/backpressure_policy/downstream_capacity_backpressure_policy.py b/python/ray/data/_internal/execution/backpressure_policy/downstream_capacity_backpressure_policy.py new file mode 100644 index 000000000000..bf1fb57dc852 --- /dev/null +++ b/python/ray/data/_internal/execution/backpressure_policy/downstream_capacity_backpressure_policy.py @@ -0,0 +1,94 @@ +import logging +from typing import TYPE_CHECKING + +from .backpressure_policy import BackpressurePolicy +from ray.data._internal.execution.operators.actor_pool_map_operator import ( + ActorPoolMapOperator, +) +from ray.data.context import DataContext + +if TYPE_CHECKING: + from ray.data._internal.execution.interfaces.physical_operator import ( + PhysicalOperator, + ) + from ray.data._internal.execution.resource_manager import ResourceManager + from ray.data._internal.execution.streaming_executor_state import Topology + +logger = logging.getLogger(__name__) + + +class DownstreamCapacityBackpressurePolicy(BackpressurePolicy): + """Backpressure policy based on downstream processing capacity. + + This policy triggers backpressure when the output bundles size exceeds both: + 1. A ratio threshold multiplied by the number of running tasks in downstream operators + 2. An absolute threshold for the output bundles size + + The policy monitors actual downstream processing capacity by tracking the number + of currently running tasks rather than configured parallelism. This approach + ensures effective backpressure even when cluster resources are insufficient or + scaling is slow, preventing memory pressure and maintaining pipeline stability. + + Key benefits: + - Prevents memory bloat from unprocessed output objects + - Adapts to actual cluster conditions and resource availability + - Maintains balanced throughput across pipeline operators + - Reduces object spilling and unnecessary rebuilds + """ + + def __init__( + self, + data_context: DataContext, + topology: "Topology", + resource_manager: "ResourceManager", + ): + super().__init__(data_context, topology, resource_manager) + self._backpressure_concurrency_ratio = ( + self._data_context.downstream_capacity_backpressure_ratio + ) + self._backpressure_max_queued_bundles = ( + self._data_context.downstream_capacity_backpressure_max_queued_bundles + ) + self._backpressure_disabled = ( + self._backpressure_concurrency_ratio is None + or self._backpressure_max_queued_bundles is None + ) + + def _max_concurrent_tasks(self, op: "PhysicalOperator") -> int: + if isinstance(op, ActorPoolMapOperator): + return sum( + [ + actor_pool.max_concurrent_tasks() + for actor_pool in op.get_autoscaling_actor_pools() + ] + ) + return op.num_active_tasks() + + def can_add_input(self, op: "PhysicalOperator") -> bool: + """Determine if we can add input to the operator based on downstream capacity.""" + if self._backpressure_disabled: + return True + for output_dependency in op.output_dependencies: + total_enqueued_input_bundles = self._topology[ + output_dependency + ].total_enqueued_input_bundles() + + avg_inputs_per_task = ( + output_dependency.metrics.num_task_inputs_processed + / max(output_dependency.metrics.num_tasks_finished, 1) + ) + outstanding_tasks = total_enqueued_input_bundles / max( + avg_inputs_per_task, 1 + ) + max_allowed_outstanding = ( + self._max_concurrent_tasks(output_dependency) + * self._backpressure_concurrency_ratio + ) + + if ( + total_enqueued_input_bundles > self._backpressure_max_queued_bundles + and outstanding_tasks > max_allowed_outstanding + ): + return False + + return True diff --git a/python/ray/data/context.py b/python/ray/data/context.py index 8c8b2c9023a5..2bbca500edcf 100644 --- a/python/ray/data/context.py +++ b/python/ray/data/context.py @@ -534,6 +534,9 @@ class DataContext: default_factory=_issue_detectors_config_factory ) + downstream_capacity_backpressure_ratio: float = None + downstream_capacity_backpressure_max_queued_bundles: int = None + def __post_init__(self): # The additonal ray remote args that should be added to # the task-pool-based data tasks. diff --git a/python/ray/data/tests/test_downstream_capacity_backpressure_policy.py b/python/ray/data/tests/test_downstream_capacity_backpressure_policy.py new file mode 100644 index 000000000000..4acec6c01852 --- /dev/null +++ b/python/ray/data/tests/test_downstream_capacity_backpressure_policy.py @@ -0,0 +1,127 @@ +from unittest.mock import MagicMock + +import pytest + +from ray.data._internal.execution.backpressure_policy.downstream_capacity_backpressure_policy import ( + DownstreamCapacityBackpressurePolicy, +) +from ray.data._internal.execution.interfaces.physical_operator import ( + OpRuntimeMetrics, + PhysicalOperator, +) +from ray.data._internal.execution.operators.actor_pool_map_operator import ( + ActorPoolMapOperator, +) +from ray.data._internal.execution.streaming_executor_state import OpState, Topology +from ray.data.context import DataContext + + +class TestDownstreamCapacityBackpressurePolicy: + def _mock_operator( + self, + op_class: PhysicalOperator = PhysicalOperator, + num_enqueued_input_bundles: int = 0, + num_task_inputs_processed: int = 0, + num_tasks_finished: int = 0, + max_concurrent_tasks: int = 100, + ): + """Helper method to create mock operator.""" + mock_operator = MagicMock(spec=op_class) + mock_operator.metrics = MagicMock(spec=OpRuntimeMetrics) + mock_operator.metrics.num_task_inputs_processed = num_task_inputs_processed + mock_operator.metrics.num_tasks_finished = num_tasks_finished + mock_operator.num_active_tasks.return_value = max_concurrent_tasks + + op_state = MagicMock(spec=OpState) + op_state.total_enqueued_input_bundles.return_value = num_enqueued_input_bundles + return mock_operator, op_state + + def _mock_actor_pool_map_operator( + self, + num_enqueued_input_bundles: int, + num_task_inputs_processed: int, + num_tasks_finished: int, + max_concurrent_tasks: int = 100, + ): + """Helper method to create mock actor pool map operator.""" + op, op_state = self._mock_operator( + ActorPoolMapOperator, + num_enqueued_input_bundles, + num_task_inputs_processed, + num_tasks_finished, + max_concurrent_tasks, + ) + actor_pool = MagicMock( + spec="ray.data._internal.execution.operators.actor_pool_map_operator._ActorPool" + ) + actor_pool.max_concurrent_tasks = MagicMock(return_value=max_concurrent_tasks) + op.get_autoscaling_actor_pools.return_value = [actor_pool] + return op, op_state + + def _create_policy( + self, data_context: DataContext = None, topology: Topology = None + ): + """Helper method to create policy instance.""" + context = data_context or self.context + return DownstreamCapacityBackpressurePolicy( + data_context=context, + topology=topology, + resource_manager=MagicMock(), + ) + + @pytest.mark.parametrize( + "mock_method", + [ + (_mock_operator), + (_mock_actor_pool_map_operator), + ], + ) + @pytest.mark.parametrize( + "num_enqueued, num_task_inputs_processed, num_tasks_finished, backpressure_ratio, max_queued_bundles, expected_result, test_name", + [ + (100, 100, 10, 2, 4000, True, "no_backpressure_low_queue"), + (5000, 100, 10, 2, 4000, False, "high_queue_pressure"), + (100, 0, 0, 2, 400, True, "zero_inputs_protection"), + (1000000, 1, 1, None, None, True, "default disabled"), + ], + ) + def test_backpressure_conditions( + self, + mock_method, + num_enqueued, + num_task_inputs_processed, + num_tasks_finished, + backpressure_ratio, + max_queued_bundles, + expected_result, + test_name, + ): + """Parameterized test covering various backpressure conditions.""" + context = DataContext() + context.downstream_capacity_backpressure_ratio = backpressure_ratio + context.downstream_capacity_backpressure_max_queued_bundles = max_queued_bundles + + op, op_state = self._mock_operator(PhysicalOperator) + op_output_dep, op_output_state = mock_method( + self, + num_enqueued_input_bundles=num_enqueued, + num_task_inputs_processed=num_task_inputs_processed, + num_tasks_finished=num_tasks_finished, + ) + op.output_dependencies = [op_output_dep] + + policy = self._create_policy( + context, topology={op: op_state, op_output_dep: op_output_state} + ) + result = policy.can_add_input(op) + + assert result == expected_result, test_name + assert ( + backpressure_ratio is None or max_queued_bundles is None + ) == policy._backpressure_disabled, test_name + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) From 36cfce8f69b3d06a072d035883a7d4543f9e1b5c Mon Sep 17 00:00:00 2001 From: Andrew Grosser Date: Tue, 26 Aug 2025 10:50:03 -0700 Subject: [PATCH 285/634] added ssl to ray serve (#55228) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? Ray serve is exposed to man in the middle attacks #55219 --------- Signed-off-by: Goutam V Signed-off-by: Andrew Grosser Signed-off-by: Justin Yu Signed-off-by: abrar Signed-off-by: Lonnie Liu Signed-off-by: Kourosh Hakhamaneshi Signed-off-by: elliot-barn Signed-off-by: Yicheng-Lu-llll Signed-off-by: Potato Signed-off-by: Cuong Nguyen Signed-off-by: Zac Policzer Signed-off-by: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Signed-off-by: Alexey Kudinkin Signed-off-by: Edward Oakes Signed-off-by: Seiji Eicher Signed-off-by: 杨睿 <595403043@qq.com> Signed-off-by: dayshah Signed-off-by: Sagar Sumit Signed-off-by: axreldable Signed-off-by: kaihsun Signed-off-by: Kai-Hsun Chen Signed-off-by: Kai-Hsun Chen Signed-off-by: Jiajun Yao Signed-off-by: Timothy Seah Signed-off-by: kevin Signed-off-by: joshlee Signed-off-by: 400Ping Signed-off-by: xgui Signed-off-by: Dhyey Shah Signed-off-by: harshit Signed-off-by: anmol Signed-off-by: Kit Lee <7000003+wingkitlee0@users.noreply.github.com> Signed-off-by: JasonLi1909 Signed-off-by: Jason Li <57246540+JasonLi1909@users.noreply.github.com> Signed-off-by: MengqingCao Signed-off-by: sampan Signed-off-by: iamjustinhsu Signed-off-by: zac Signed-off-by: Elliot Barnwell Signed-off-by: Mengjin Yan Signed-off-by: myan Signed-off-by: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Signed-off-by: Rui Qiao Signed-off-by: Linkun Signed-off-by: Balaji Veeramani Signed-off-by: Markus Signed-off-by: Gagandeep Singh Signed-off-by: akyang-anyscale Signed-off-by: Alan Guo Signed-off-by: haotian Signed-off-by: Howie Tien Signed-off-by: will.lin Signed-off-by: Richard Liaw Signed-off-by: Ryan O'Leary Signed-off-by: Andrew Sy Kim Signed-off-by: Matvei Pashkovskii Signed-off-by: Kishanthan Thangarajah Signed-off-by: my-vegetable-has-exploded Signed-off-by: Neil Girdhar Signed-off-by: Nikhil Ghosh Signed-off-by: win5923 Signed-off-by: Stephanie wang Signed-off-by: Stephanie Wang Signed-off-by: cong.qian Signed-off-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Signed-off-by: doyoung Signed-off-by: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Signed-off-by: simonsays1980 Signed-off-by: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Signed-off-by: Sampan S Nayak Signed-off-by: vincenthhan Signed-off-by: jeffreyjeffreywang Signed-off-by: irabbani Signed-off-by: Ibrahim Rabbani Signed-off-by: avigyabb Signed-off-by: avibasnet31 Signed-off-by: Tanner Wood Signed-off-by: avigyabb <98926738+avigyabb@users.noreply.github.com> Signed-off-by: Ricardo Decal Signed-off-by: Matthew Signed-off-by: Rueian Signed-off-by: tianyi-ge Co-authored-by: goutamvenkat-anyscale Co-authored-by: Justin Yu Co-authored-by: Abrar Sheikh Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Co-authored-by: kourosh hakhamaneshi <31483498+kouroshHakha@users.noreply.github.com> Co-authored-by: Elliot Barnwell Co-authored-by: Yicheng-Lu-llll <51814063+Yicheng-Lu-llll@users.noreply.github.com> Co-authored-by: Potato Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Co-authored-by: Zac Policzer Co-authored-by: Edward Oakes Co-authored-by: Alexey Kudinkin Co-authored-by: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Co-authored-by: 杨睿 <595403043@qq.com> Co-authored-by: Ibrahim Rabbani Co-authored-by: Dhyey Shah Co-authored-by: Sagar Sumit Co-authored-by: Aleksei Starikov Co-authored-by: Kai-Hsun Chen Co-authored-by: Stephanie Wang Co-authored-by: Jiajun Yao Co-authored-by: Timothy Seah Co-authored-by: Timothy Seah Co-authored-by: Kevin H. Luu Co-authored-by: Qiaolin Yu Co-authored-by: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Co-authored-by: Ping Co-authored-by: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Co-authored-by: harshit-anyscale Co-authored-by: Anmol Singh Co-authored-by: anmol Co-authored-by: Kit Lee <7000003+wingkitlee0@users.noreply.github.com> Co-authored-by: Jason Li <57246540+JasonLi1909@users.noreply.github.com> Co-authored-by: matthewdeng Co-authored-by: matthewdeng Co-authored-by: Mengqing Cao Co-authored-by: Sampan S Nayak Co-authored-by: sampan Co-authored-by: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Co-authored-by: Sven Mika Co-authored-by: Mengjin Yan Co-authored-by: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Co-authored-by: Nary Yeh <60069744+machichima@users.noreply.github.com> Co-authored-by: lkchen Co-authored-by: Balaji Veeramani Co-authored-by: Markus <44006014+minosvasilias@users.noreply.github.com> Co-authored-by: czgdp1807 Co-authored-by: akyang-anyscale Co-authored-by: Alan Guo Co-authored-by: Howie Tien Co-authored-by: Balaji Veeramani Co-authored-by: simonsays1980 Co-authored-by: William Lin Co-authored-by: Richard Liaw Co-authored-by: Ryan O'Leary <113500783+ryanaoleary@users.noreply.github.com> Co-authored-by: Andrew Sy Kim Co-authored-by: Matvei Pashkovskii Co-authored-by: Kourosh Hakhamaneshi Co-authored-by: Kishanthan Thangarajah Co-authored-by: yi wang <48236141+my-vegetable-has-exploded@users.noreply.github.com> Co-authored-by: Neil Girdhar Co-authored-by: Nikhil G Co-authored-by: Jun-Hao Wan Co-authored-by: Kai-Hsun Chen Co-authored-by: Stephanie Wang Co-authored-by: coqian Co-authored-by: angelinalg <122562471+angelinalg@users.noreply.github.com> Co-authored-by: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Co-authored-by: vincenthhan <46981434+BestVIncent@users.noreply.github.com> Co-authored-by: vincenthhan Co-authored-by: Jeffrey Wang Co-authored-by: jeffreyjeffreywang Co-authored-by: Ibrahim Rabbani Co-authored-by: avigyabb <98926738+avigyabb@users.noreply.github.com> Co-authored-by: avibasnet31 Co-authored-by: tannerdwood <71387269+tannerdwood@users.noreply.github.com> Co-authored-by: Tanner Wood Co-authored-by: Ricardo Decal Co-authored-by: Kamil Kaczmarek Co-authored-by: Hassam Ullah Sheikh Co-authored-by: MatthewCWeston <61944935+MatthewCWeston@users.noreply.github.com> Co-authored-by: Artur Niederfahrenhorst Co-authored-by: Artur Niederfahrenhorst Co-authored-by: Rueian Co-authored-by: Tianyi --- python/ray/serve/_private/http_util.py | 18 + python/ray/serve/_private/utils.py | 24 +- python/ray/serve/config.py | 18 + python/ray/serve/schema.py | 27 +- python/ray/serve/tests/BUILD | 1 + python/ray/serve/tests/test_certs/ca.crt | 21 + python/ray/serve/tests/test_certs/server.crt | 21 + python/ray/serve/tests/test_certs/server.csr | 17 + python/ray/serve/tests/test_certs/server.key | 28 ++ python/ray/serve/tests/test_https_proxy.py | 495 +++++++++++++++++++ 10 files changed, 667 insertions(+), 3 deletions(-) create mode 100644 python/ray/serve/tests/test_certs/ca.crt create mode 100644 python/ray/serve/tests/test_certs/server.crt create mode 100644 python/ray/serve/tests/test_certs/server.csr create mode 100644 python/ray/serve/tests/test_certs/server.key create mode 100644 python/ray/serve/tests/test_https_proxy.py diff --git a/python/ray/serve/_private/http_util.py b/python/ray/serve/_private/http_util.py index 2da5b8223964..93880b65f77b 100644 --- a/python/ray/serve/_private/http_util.py +++ b/python/ray/serve/_private/http_util.py @@ -714,6 +714,23 @@ async def start_asgi_http_server( # has no use to us. logging.getLogger("uvicorn.error").level = logging.CRITICAL + # Configure SSL if certificates are provided + ssl_kwargs = {} + if http_options.ssl_keyfile and http_options.ssl_certfile: + ssl_kwargs = { + "ssl_keyfile": http_options.ssl_keyfile, + "ssl_certfile": http_options.ssl_certfile, + } + if http_options.ssl_keyfile_password: + ssl_kwargs["ssl_keyfile_password"] = http_options.ssl_keyfile_password + if http_options.ssl_ca_certs: + ssl_kwargs["ssl_ca_certs"] = http_options.ssl_ca_certs + + logger.info( + f"Starting HTTPS server on {http_options.host}:{http_options.port} " + f"with SSL certificate: {http_options.ssl_certfile}" + ) + # NOTE: We have to use lower level uvicorn Config and Server # class because we want to run the server as a coroutine. The only # alternative is to call uvicorn.run which is blocking. @@ -730,6 +747,7 @@ async def start_asgi_http_server( access_log=False, log_level=None, log_config=None, + **ssl_kwargs, ) ) diff --git a/python/ray/serve/_private/utils.py b/python/ray/serve/_private/utils.py index 3cfad95ff0cd..924702c704da 100644 --- a/python/ray/serve/_private/utils.py +++ b/python/ray/serve/_private/utils.py @@ -27,7 +27,6 @@ from ray.actor import ActorHandle from ray.serve._private.common import RequestMetadata, ServeComponentType from ray.serve._private.constants import HTTP_PROXY_TIMEOUT, SERVE_LOGGER_NAME -from ray.serve.config import gRPCOptions from ray.types import ObjectRef from ray.util.serialization import StandaloneSerializationContext @@ -44,6 +43,27 @@ FILE_NAME_REGEX = r"[^\x20-\x7E]|[<>:\"/\\|?*]" MESSAGE_PACK_OFFSET = 9 + + +def validate_ssl_config( + ssl_certfile: Optional[str], ssl_keyfile: Optional[str] +) -> None: + """Validate SSL configuration for HTTPS support. + + Args: + ssl_certfile: Path to SSL certificate file + ssl_keyfile: Path to SSL private key file + + Raises: + ValueError: If only one of ssl_certfile or ssl_keyfile is provided + """ + if (ssl_certfile and not ssl_keyfile) or (ssl_keyfile and not ssl_certfile): + raise ValueError( + "Both ssl_keyfile and ssl_certfile must be provided together " + "to enable HTTPS." + ) + + GENERATOR_COMPOSITION_NOT_SUPPORTED_ERROR = RuntimeError( "Streaming deployment handle results cannot be passed to " "downstream handle calls. If you have a use case requiring " @@ -612,7 +632,7 @@ def wait_for_interrupt() -> None: raise -def is_grpc_enabled(grpc_config: gRPCOptions) -> bool: +def is_grpc_enabled(grpc_config) -> bool: return grpc_config.port > 0 and len(grpc_config.grpc_servicer_functions) > 0 diff --git a/python/ray/serve/config.py b/python/ray/serve/config.py index c58570eff39b..31b3ba84e825 100644 --- a/python/ray/serve/config.py +++ b/python/ray/serve/config.py @@ -28,6 +28,7 @@ DEFAULT_UVICORN_KEEP_ALIVE_TIMEOUT_S, SERVE_LOGGER_NAME, ) +from ray.serve._private.utils import validate_ssl_config from ray.util.annotations import Deprecated, PublicAPI logger = logging.getLogger(SERVE_LOGGER_NAME) @@ -410,6 +411,13 @@ class HTTPOptions(BaseModel): - request_timeout_s: End-to-end timeout for HTTP requests. - keep_alive_timeout_s: Duration to keep idle connections alive when no requests are ongoing. + - ssl_keyfile: Path to the SSL key file for HTTPS. If provided with + ssl_certfile, the HTTP server will use HTTPS. + - ssl_certfile: Path to the SSL certificate file for HTTPS. If provided + with ssl_keyfile, the HTTP server will use HTTPS. + - ssl_keyfile_password: Optional password for the SSL key file. + - ssl_ca_certs: Optional path to CA certificate file for client certificate + verification. - location: [DEPRECATED: use `proxy_location` field instead] The deployment location of HTTP servers: @@ -433,6 +441,10 @@ class HTTPOptions(BaseModel): root_path: str = "" request_timeout_s: Optional[float] = None keep_alive_timeout_s: int = DEFAULT_UVICORN_KEEP_ALIVE_TIMEOUT_S + ssl_keyfile: Optional[str] = None + ssl_certfile: Optional[str] = None + ssl_keyfile_password: Optional[str] = None + ssl_ca_certs: Optional[str] = None @validator("location", always=True) def location_backfill_no_server(cls, v, values): @@ -441,6 +453,12 @@ def location_backfill_no_server(cls, v, values): return v + @validator("ssl_certfile") + def validate_ssl_certfile(cls, v, values): + ssl_keyfile = values.get("ssl_keyfile") + validate_ssl_config(v, ssl_keyfile) + return v + @validator("middlewares", always=True) def warn_for_middlewares(cls, v, values): if v: diff --git a/python/ray/serve/schema.py b/python/ray/serve/schema.py index 5ba0d8e0a344..f9d49d079b07 100644 --- a/python/ray/serve/schema.py +++ b/python/ray/serve/schema.py @@ -32,7 +32,7 @@ SERVE_DEFAULT_APP_NAME, ) from ray.serve._private.deployment_info import DeploymentInfo -from ray.serve._private.utils import DEFAULT +from ray.serve._private.utils import DEFAULT, validate_ssl_config from ray.serve.config import ProxyLocation, RequestRouterConfig from ray.util.annotations import PublicAPI @@ -713,6 +713,31 @@ class HTTPOptionsSchema(BaseModel): "before closing them when no requests are ongoing. Defaults to " f"{DEFAULT_UVICORN_KEEP_ALIVE_TIMEOUT_S} seconds.", ) + ssl_keyfile: Optional[str] = Field( + default=None, + description="Path to the SSL key file for HTTPS. If provided with ssl_certfile, " + "the HTTP server will use HTTPS. Cannot be updated once Serve has started.", + ) + ssl_certfile: Optional[str] = Field( + default=None, + description="Path to the SSL certificate file for HTTPS. If provided with " + "ssl_keyfile, the HTTP server will use HTTPS. Cannot be updated once Serve " + "has started.", + ) + ssl_keyfile_password: Optional[str] = Field( + default=None, + description="Password for the SSL key file, if encrypted.", + ) + ssl_ca_certs: Optional[str] = Field( + default=None, + description="Path to the CA certificate file for verifying client certificates.", + ) + + @validator("ssl_certfile") + def validate_ssl_certfile(cls, v, values): + ssl_keyfile = values.get("ssl_keyfile") + validate_ssl_config(v, ssl_keyfile) + return v @PublicAPI(stability="stable") diff --git a/python/ray/serve/tests/BUILD b/python/ray/serve/tests/BUILD index 997cc2de7a2b..174d782d925d 100644 --- a/python/ray/serve/tests/BUILD +++ b/python/ray/serve/tests/BUILD @@ -82,6 +82,7 @@ py_test_module_list( "test_healthcheck.py", "test_http_headers.py", "test_http_routes.py", + "test_https_proxy.py", "test_max_replicas_per_node.py", "test_multiplex.py", "test_proxy.py", diff --git a/python/ray/serve/tests/test_certs/ca.crt b/python/ray/serve/tests/test_certs/ca.crt new file mode 100644 index 000000000000..5b0a5e11bf42 --- /dev/null +++ b/python/ray/serve/tests/test_certs/ca.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDfTCCAmWgAwIBAgIUYcUOt0aN1Ml/1WnFPB9gveNNniQwDQYJKoZIhvcNAQEL +BQAwZzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xFzAVBgNVBAoMDlJheSBTZXJ2ZSBUZXN0MRIwEAYDVQQD +DAlsb2NhbGhvc3QwHhcNMjUwODIwMTgxODUzWhcNMjYwODIwMTgxODUzWjBnMQsw +CQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZy +YW5jaXNjbzEXMBUGA1UECgwOUmF5IFNlcnZlIFRlc3QxEjAQBgNVBAMMCWxvY2Fs +aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKYXcIirTR5AHb5V +T6yijOR8mvc6AXSKkmIKu7n2vaJ3Jrt7d6mPz/ScXlLYxq+mgt4avX/VozES0ARM +NcbqlHOcahgfyyN+/02q/Aimwbaf/FwiS5qyQfMXzFg70kydqlDlUsyE49qdFHEv +xx4ostLnTeyIpS7AS14qJXGeg5NE9Pm+XSs0HVBPZBaM6VCJl8/Pjog0qqffovGo +/qN8gVxnydg4ayTZ9nl+NNMivFJ/f5MUXmJiuFYAoZnwMiCy2QAU9TmdA5mCOGNZ +pv/KSSdqkVh7X6JNGB6OLgikCsObWxAJqq7WZgiHoc2WlXuN+U2SLuA0JLZZZr+t +zpw1DH0CAwEAAaMhMB8wHQYDVR0OBBYEFIey4ZBoVICZ7kAJv7K5kY/SHP6wMA0G +CSqGSIb3DQEBCwUAA4IBAQAg47MfYFykzDdynJnKf/Aqlp4bnT3GVEW3lRk8AMv9 +yrjwQeVKihiQLgC6b7ChyLUQWxcxJPqhzAIe/+sn9bAxz448oGMtU6ghHtxt13T2 +9VKsyyrjgZ3fbiFT5AFMYxwYlcaf1hJPE+PKKU3oUhYxUlEBKweDjTw7+7xym/Ix +hNYv36lDst/zwA1HKmvorDhCVOT3Y90deVA31NxFQbqNpeCjG6uiURAtO3jMan50 +m9U60cHjJBkSxCKCw4SQXOan9VKePIsHnZgIiDPmO25KYSJxeat92sHVtI3FZfrh +pN3cjQaXhMbJFO9ySv5tqr0KxUbymN56ynWkScMGbI0W +-----END CERTIFICATE----- diff --git a/python/ray/serve/tests/test_certs/server.crt b/python/ray/serve/tests/test_certs/server.crt new file mode 100644 index 000000000000..5b0a5e11bf42 --- /dev/null +++ b/python/ray/serve/tests/test_certs/server.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDfTCCAmWgAwIBAgIUYcUOt0aN1Ml/1WnFPB9gveNNniQwDQYJKoZIhvcNAQEL +BQAwZzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xFzAVBgNVBAoMDlJheSBTZXJ2ZSBUZXN0MRIwEAYDVQQD +DAlsb2NhbGhvc3QwHhcNMjUwODIwMTgxODUzWhcNMjYwODIwMTgxODUzWjBnMQsw +CQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZy +YW5jaXNjbzEXMBUGA1UECgwOUmF5IFNlcnZlIFRlc3QxEjAQBgNVBAMMCWxvY2Fs +aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKYXcIirTR5AHb5V +T6yijOR8mvc6AXSKkmIKu7n2vaJ3Jrt7d6mPz/ScXlLYxq+mgt4avX/VozES0ARM +NcbqlHOcahgfyyN+/02q/Aimwbaf/FwiS5qyQfMXzFg70kydqlDlUsyE49qdFHEv +xx4ostLnTeyIpS7AS14qJXGeg5NE9Pm+XSs0HVBPZBaM6VCJl8/Pjog0qqffovGo +/qN8gVxnydg4ayTZ9nl+NNMivFJ/f5MUXmJiuFYAoZnwMiCy2QAU9TmdA5mCOGNZ +pv/KSSdqkVh7X6JNGB6OLgikCsObWxAJqq7WZgiHoc2WlXuN+U2SLuA0JLZZZr+t +zpw1DH0CAwEAAaMhMB8wHQYDVR0OBBYEFIey4ZBoVICZ7kAJv7K5kY/SHP6wMA0G +CSqGSIb3DQEBCwUAA4IBAQAg47MfYFykzDdynJnKf/Aqlp4bnT3GVEW3lRk8AMv9 +yrjwQeVKihiQLgC6b7ChyLUQWxcxJPqhzAIe/+sn9bAxz448oGMtU6ghHtxt13T2 +9VKsyyrjgZ3fbiFT5AFMYxwYlcaf1hJPE+PKKU3oUhYxUlEBKweDjTw7+7xym/Ix +hNYv36lDst/zwA1HKmvorDhCVOT3Y90deVA31NxFQbqNpeCjG6uiURAtO3jMan50 +m9U60cHjJBkSxCKCw4SQXOan9VKePIsHnZgIiDPmO25KYSJxeat92sHVtI3FZfrh +pN3cjQaXhMbJFO9ySv5tqr0KxUbymN56ynWkScMGbI0W +-----END CERTIFICATE----- diff --git a/python/ray/serve/tests/test_certs/server.csr b/python/ray/serve/tests/test_certs/server.csr new file mode 100644 index 000000000000..3d26126664ef --- /dev/null +++ b/python/ray/serve/tests/test_certs/server.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICrDCCAZQCAQAwZzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xFzAVBgNVBAoMDlJheSBTZXJ2ZSBUZXN0 +MRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCmF3CIq00eQB2+VU+soozkfJr3OgF0ipJiCru59r2idya7e3epj8/0nF5S +2MavpoLeGr1/1aMxEtAETDXG6pRznGoYH8sjfv9NqvwIpsG2n/xcIkuaskHzF8xY +O9JMnapQ5VLMhOPanRRxL8ceKLLS503siKUuwEteKiVxnoOTRPT5vl0rNB1QT2QW +jOlQiZfPz46INKqn36LxqP6jfIFcZ8nYOGsk2fZ5fjTTIrxSf3+TFF5iYrhWAKGZ +8DIgstkAFPU5nQOZgjhjWab/ykknapFYe1+iTRgeji4IpArDm1sQCaqu1mYIh6HN +lpV7jflNki7gNCS2WWa/rc6cNQx9AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEA +igYR2ZQ4fmp339T/BGvXSDIjQQkecd9MeifdcXuN/2FZ7dhyfDWHjQadtohgXSZw +LwfUx43L+JcebMY8GyN/4JIAKA5hVqqvAiaMb+vRUItgku5M2WIpnPLVKQJHTUGC +aaDq6u7aS4eFcvuYGaFTUD7tNMOfRP8SfQL/sk2UqZVOCIxCFX9gLS/p4IyorUsb +VjdQBHRvOZnZCFMwmisquXXeGxtAPabUWMPLvSqcP/93WdjFwtrcscyY68s+AC6o +9sx1x3qjnTxnx+a8ho5f0p/JSUqye+G/gzqzB5WMZK5U7oiYgP0rEajU9odGIPSK +AqzWpVDtZBSr8FFamw4uqQ== +-----END CERTIFICATE REQUEST----- diff --git a/python/ray/serve/tests/test_certs/server.key b/python/ray/serve/tests/test_certs/server.key new file mode 100644 index 000000000000..de16d5454e9d --- /dev/null +++ b/python/ray/serve/tests/test_certs/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCmF3CIq00eQB2+ +VU+soozkfJr3OgF0ipJiCru59r2idya7e3epj8/0nF5S2MavpoLeGr1/1aMxEtAE +TDXG6pRznGoYH8sjfv9NqvwIpsG2n/xcIkuaskHzF8xYO9JMnapQ5VLMhOPanRRx +L8ceKLLS503siKUuwEteKiVxnoOTRPT5vl0rNB1QT2QWjOlQiZfPz46INKqn36Lx +qP6jfIFcZ8nYOGsk2fZ5fjTTIrxSf3+TFF5iYrhWAKGZ8DIgstkAFPU5nQOZgjhj +Wab/ykknapFYe1+iTRgeji4IpArDm1sQCaqu1mYIh6HNlpV7jflNki7gNCS2WWa/ +rc6cNQx9AgMBAAECggEAFj7SHLaiJ+i7KHCcBj7ok1Bjyl8OLizCebfUTY5QTH/x +mRoVUd7oIcFbxMmHUE6t/STPDV3GHgmAq5gFeonqrigHRwnjFvL91h+OOB5q7ZSJ ++VEX7TVDg1VEUkEDjq1t+qhsVDuBmm3VfL9tx4qjQNTSvq536UYUvMefp5MX2P54 +/7IDM9osP5VgeFIUx/d7QYymhgmVaSv+xcxxlZCwT3ib/wW7eU964FjkuRG8eein +zlyOwRufmg+eEvOUHN/4Fth0AUUirCMpflgRdcQtKs77FARiG8LybMGyDDsE7YBt +5f/UBZea2TQG9q4aGNUIHA869CCNKg2R27AtBpTtBQKBgQDd95GDIZMlEmR3GzpJ +6rqRHtfXj7BHUlzew+RCI1KhWkjRZqv2bavmeqRLpRdKd36aC+l+QallSW/MME+7 +JSgRMqqdQK2myLJnZOIcONjMlOn9xzEQGYUsKL4IiPkdP0lWdzJ6iqAHm/Xq7GxE +BJF5XkYD1NP2+y3dlZYNrmUGHwKBgQC/jrOCV7Y34IriUPHSQA1JaPZQDBBxwiNo +ifPYkRc5C0zwskiELnJGF34Y/fK06n73JyBh6WqMdu7+V/QKNCEgcKU45n+pnlAL +vx+xflfMknWEOhLdT31ca0kvxtGEomOD1MNV+b1cRYBlL/oMC2IpIKd0N/HFa3Nc +pDmLcBWB4wKBgAIHXD4dlXG2TFLGXe8FBTWEWaavuoW8W/rxQWnVVtEAuT+ot5Om +BvcxUcUbOi5FD1QrHbQ4t2qklDAClQf52/bkRqjvSWcH2JGXW3W0k06zYbwfEPS7 +tvrjWHFNhzFcPbhbmIuELthC9alzBb5NaGL6mJs6W8GbJB0tW9S+LlAzAoGBAIlB +h/B6Rs+s7fcSBuQfDyYttmhO7K2GbPan+niQJfKy3TOOm5VS7oC4rprbw7/MUqNn +frWJmdYCFmdawDtbdO0Yqdqmlo0EKdjw3pXAsMqdmuTe88tt/KZvHWbFcDU4YlQA +7OI662slRcW7ZdChi3lqs3H78BoETwnvhmgaLN7/AoGBAIVtEVcieOsasQ3Cje4L +mZxo9WFwtX4llH/CTZZeyek6VZBEWP8b3i1uh0uOzeiR7nDiwGEbHfXdvIvWrZqf +IC9Lo1D24uzE14XcKypFsYL5GAwtNhTAuP52tfV9V7DlS2QmxQt6hzx0/MhtdM3X +1XCsMrmi/WleIy611H2j0gUj +-----END PRIVATE KEY----- diff --git a/python/ray/serve/tests/test_https_proxy.py b/python/ray/serve/tests/test_https_proxy.py new file mode 100644 index 000000000000..051960eafd1d --- /dev/null +++ b/python/ray/serve/tests/test_https_proxy.py @@ -0,0 +1,495 @@ +import asyncio +import json +import os +import ssl +import tempfile + +import pytest +import requests +import websockets + +import ray +from ray import serve +from ray._private.tls_utils import generate_self_signed_tls_certs +from ray.serve.config import HTTPOptions + + +@pytest.fixture(scope="session") +def ssl_cert_and_key(): + """Generate SSL certificates using Ray's built-in utilities for testing.""" + # Generate certificate and key using Ray's utility + cert_contents, key_contents = generate_self_signed_tls_certs() + + # Create temp directory that persists for the session + temp_dir = tempfile.mkdtemp(prefix="ray_serve_https_test_") + + # Write server certificate and key + cert_path = os.path.join(temp_dir, "server.crt") + key_path = os.path.join(temp_dir, "server.key") + + with open(cert_path, "w") as f: + f.write(cert_contents) + with open(key_path, "w") as f: + f.write(key_contents) + + yield { + "key_path": key_path, + "cert_path": cert_path, + "temp_dir": temp_dir, + } + + # Cleanup + import shutil + + try: + shutil.rmtree(temp_dir) + except Exception: + pass # Ignore cleanup errors + + +@pytest.fixture +def https_serve_instance(ssl_cert_and_key): + """Start Ray Serve with HTTPS enabled.""" + # Ensure Ray is shutdown before starting + try: + ray.shutdown() + except Exception: + pass + + # Disable runtime env upload (dashboard should work now that it's built) + ray.init(runtime_env={"working_dir": None}) + serve.start( + http_options=HTTPOptions( + ssl_keyfile=ssl_cert_and_key["key_path"], + ssl_certfile=ssl_cert_and_key["cert_path"], + ) + ) + yield serve + serve.shutdown() + ray.shutdown() + + +class TestHTTPSProxy: + def test_https_basic_deployment(self, https_serve_instance): + """Test basic HTTPS deployment functionality.""" + + @serve.deployment + def hello(): + return "Hello HTTPS!" + + serve.run(hello.bind()) + + # Test HTTPS request with certificate verification disabled for self-signed cert + response = requests.get( + "https://localhost:8000/hello", + verify=False, # Skip cert verification for self-signed + ) + assert response.status_code == 200 + assert response.text == "Hello HTTPS!" + + def test_https_vs_http_requests(self, https_serve_instance): + """Test that HTTP requests fail when HTTPS is enabled.""" + + @serve.deployment + def echo(): + return "echo" + + serve.run(echo.bind()) + + # HTTPS request should succeed + https_response = requests.get("https://localhost:8000/echo", verify=False) + assert https_response.status_code == 200 + + # HTTP request should fail with connection error + with pytest.raises(requests.exceptions.ConnectionError): + requests.get("http://localhost:8000/echo", timeout=5) + + def test_https_with_fastapi_deployment(self, https_serve_instance): + """Test HTTPS with FastAPI-based deployment.""" + from fastapi import FastAPI + + app = FastAPI() + + @app.get("/items/{item_id}") + async def read_item(item_id: int): + return {"item_id": item_id, "secure": True} + + @serve.deployment + @serve.ingress(app) + class FastAPIDeployment: + pass + + serve.run(FastAPIDeployment.bind()) + + response = requests.get("https://localhost:8000/items/42", verify=False) + assert response.status_code == 200 + assert response.json() == {"item_id": 42, "secure": True} + + def test_https_concurrent_requests(self, https_serve_instance): + """Test HTTPS with concurrent requests.""" + import concurrent.futures + + @serve.deployment + def concurrent_handler(): + import time + + time.sleep(0.1) # Small delay to test concurrency + return "concurrent" + + serve.run(concurrent_handler.bind()) + + def make_request(): + return requests.get( + "https://localhost:8000/concurrent_handler", verify=False + ) + + # Send 10 concurrent requests + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + futures = [executor.submit(make_request) for _ in range(10)] + responses = [f.result() for f in futures] + + # All requests should succeed + for response in responses: + assert response.status_code == 200 + assert response.text == "concurrent" + + def test_https_large_payload(self, https_serve_instance): + """Test HTTPS with large payloads.""" + + @serve.deployment + class LargePayloadHandler: + def __call__(self, request): + # Return a large response (1MB) + large_data = "x" * (1024 * 1024) # 1MB string + return {"data": large_data, "size": len(large_data)} + + serve.run(LargePayloadHandler.bind()) + + response = requests.get( + "https://localhost:8000/LargePayloadHandler", verify=False + ) + assert response.status_code == 200 + data = response.json() + assert data["size"] == 1024 * 1024 + assert len(data["data"]) == 1024 * 1024 + + def test_https_websocket_with_fastapi(self, https_serve_instance): + """Test WebSocket functionality with FastAPI over HTTPS.""" + from fastapi import FastAPI, WebSocket, WebSocketDisconnect + + app = FastAPI() + + @app.websocket("/ws") + async def websocket_endpoint(websocket: WebSocket): + await websocket.accept() + try: + while True: + # Receive message from client + data = await websocket.receive_text() + message = json.loads(data) + + # Echo back with modification + response = { + "echo": message.get("message", ""), + "secure": True, + "protocol": "wss", + } + await websocket.send_text(json.dumps(response)) + except WebSocketDisconnect: + pass + + @serve.deployment + @serve.ingress(app) + class WebSocketDeployment: + pass + + serve.run(WebSocketDeployment.bind()) + + # Test WebSocket connection over HTTPS (wss://) + async def test_websocket(): + # Create SSL context that doesn't verify certificates (for self-signed certs) + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + + uri = "wss://localhost:8000/ws" + async with websockets.connect(uri, ssl=ssl_context) as websocket: + # Send test message + test_message = {"message": "Hello WebSocket over HTTPS!"} + await websocket.send(json.dumps(test_message)) + + # Receive response + response = await websocket.recv() + data = json.loads(response) + + # Verify response + assert data["echo"] == "Hello WebSocket over HTTPS!" + assert data["secure"] is True + assert data["protocol"] == "wss" + + # Send another message to test bidirectional communication + test_message2 = {"message": "Second message"} + await websocket.send(json.dumps(test_message2)) + + response2 = await websocket.recv() + data2 = json.loads(response2) + assert data2["echo"] == "Second message" + + # Run the async test + asyncio.run(test_websocket()) + + def test_https_websocket_multiple_connections(self, https_serve_instance): + """Test multiple WebSocket connections over HTTPS.""" + from fastapi import FastAPI, WebSocket, WebSocketDisconnect + + app = FastAPI() + + # Store active connections + connections = [] + + @app.websocket("/ws/broadcast") + async def websocket_broadcast(websocket: WebSocket): + await websocket.accept() + connections.append(websocket) + try: + while True: + data = await websocket.receive_text() + message = json.loads(data) + + # Broadcast to all connections + broadcast_message = { + "type": "broadcast", + "message": message.get("message", ""), + "connections": len(connections), + "secure": True, + } + + # Send to all connected clients + disconnected = [] + for conn in connections: + try: + await conn.send_text(json.dumps(broadcast_message)) + except Exception: + disconnected.append(conn) + + # Remove disconnected clients + for conn in disconnected: + connections.remove(conn) + + except WebSocketDisconnect: + if websocket in connections: + connections.remove(websocket) + + @serve.deployment + @serve.ingress(app) + class WebSocketBroadcastDeployment: + pass + + serve.run(WebSocketBroadcastDeployment.bind()) + + async def test_multiple_websockets(): + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + + uri = "wss://localhost:8000/ws/broadcast" + + # Connect multiple clients + websocket1 = await websockets.connect(uri, ssl=ssl_context) + websocket2 = await websockets.connect(uri, ssl=ssl_context) + + try: + # Send message from client 1 + test_message = {"message": "Hello from client 1"} + await websocket1.send(json.dumps(test_message)) + + # Both clients should receive the broadcast + response1 = await websocket1.recv() + response2 = await websocket2.recv() + + data1 = json.loads(response1) + data2 = json.loads(response2) + + # Verify both received the same broadcast + assert data1["type"] == "broadcast" + assert data1["message"] == "Hello from client 1" + assert data1["connections"] == 2 + assert data1["secure"] is True + + assert data2["type"] == "broadcast" + assert data2["message"] == "Hello from client 1" + assert data2["connections"] == 2 + assert data2["secure"] is True + + finally: + await websocket1.close() + await websocket2.close() + + # Run the async test + asyncio.run(test_multiple_websockets()) + + +class TestSSLConfiguration: + def test_ssl_config_validation_success(self, ssl_cert_and_key): + """Test successful SSL configuration validation.""" + key_path = ssl_cert_and_key["key_path"] + cert_path = ssl_cert_and_key["cert_path"] + + # Should not raise exception + options = HTTPOptions(ssl_keyfile=key_path, ssl_certfile=cert_path) + assert options.ssl_keyfile == key_path + assert options.ssl_certfile == cert_path + + def test_ssl_config_validation_missing_key(self): + """Test SSL configuration validation with missing key file.""" + with tempfile.TemporaryDirectory() as temp_dir: + cert_path = os.path.join(temp_dir, "test.crt") + with open(cert_path, "w") as f: + f.write("dummy cert") + + with pytest.raises(ValueError) as exc_info: + HTTPOptions(ssl_keyfile=None, ssl_certfile=cert_path) + + assert "Both ssl_keyfile and ssl_certfile must be provided together" in str( + exc_info.value + ) + + def test_ssl_config_validation_missing_cert(self): + """Test SSL configuration validation with missing cert file.""" + with tempfile.TemporaryDirectory() as temp_dir: + key_path = os.path.join(temp_dir, "test.key") + with open(key_path, "w") as f: + f.write("dummy key") + + with pytest.raises(ValueError) as exc_info: + HTTPOptions(ssl_keyfile=key_path, ssl_certfile=None) + + assert "Both ssl_keyfile and ssl_certfile must be provided together" in str( + exc_info.value + ) + + def test_ssl_config_with_password(self, ssl_cert_and_key): + """Test SSL configuration with key file password.""" + key_path = ssl_cert_and_key["key_path"] + cert_path = ssl_cert_and_key["cert_path"] + + options = HTTPOptions( + ssl_keyfile=key_path, ssl_certfile=cert_path, ssl_keyfile_password="secret" + ) + assert options.ssl_keyfile_password == "secret" + + def test_ssl_config_with_ca_certs(self, ssl_cert_and_key): + """Test SSL configuration with CA certificates.""" + key_path = ssl_cert_and_key["key_path"] + cert_path = ssl_cert_and_key["cert_path"] + # Use cert as CA for testing purposes + ca_path = cert_path + + options = HTTPOptions( + ssl_keyfile=key_path, ssl_certfile=cert_path, ssl_ca_certs=ca_path + ) + assert options.ssl_ca_certs == ca_path + + +class TestHTTPSErrorHandling: + def test_ssl_file_paths_validation(self): + """Test that SSL file paths are properly configured in HTTPOptions.""" + with tempfile.TemporaryDirectory() as temp_dir: + key_path = os.path.join(temp_dir, "test.key") + cert_path = os.path.join(temp_dir, "test.crt") + + # Create dummy files (content doesn't matter for this test) + with open(key_path, "w") as f: + f.write("dummy key") + with open(cert_path, "w") as f: + f.write("dummy cert") + + # Test that HTTPOptions accepts valid file paths + options = HTTPOptions(ssl_keyfile=key_path, ssl_certfile=cert_path) + assert options.ssl_keyfile == key_path + assert options.ssl_certfile == cert_path + + def test_https_requires_both_cert_and_key_files(self): + """Test that HTTPS configuration requires both certificate and key files.""" + # This test validates our SSL validation logic works correctly + + # Should work with both files + options = HTTPOptions(ssl_keyfile="key.pem", ssl_certfile="cert.pem") + assert options.ssl_keyfile == "key.pem" + assert options.ssl_certfile == "cert.pem" + + # Should work with neither file + options = HTTPOptions() + assert options.ssl_keyfile is None + assert options.ssl_certfile is None + + +class TestHTTPSIntegration: + def test_https_with_custom_port(self, ssl_cert_and_key): + """Test HTTPS on custom port.""" + # Ensure Ray is shutdown before starting + try: + ray.shutdown() + except Exception: + pass + + # Disable dashboard to prevent SSL conflicts and disable runtime env upload + ray.init(include_dashboard=False, runtime_env={"working_dir": None}) + + try: + serve.start( + http_options=HTTPOptions( + host="127.0.0.1", + port=8443, + ssl_keyfile=ssl_cert_and_key["key_path"], + ssl_certfile=ssl_cert_and_key["cert_path"], + ) + ) + + @serve.deployment + def custom_port_handler(): + return "custom port" + + serve.run(custom_port_handler.bind()) + + response = requests.get( + "https://127.0.0.1:8443/custom_port_handler", verify=False + ) + assert response.status_code == 200 + assert response.text == "custom port" + finally: + try: + serve.shutdown() + except Exception: + pass + ray.shutdown() + + def test_https_deployment_update(self, https_serve_instance): + """Test deployment updates work correctly with HTTPS.""" + + @serve.deployment + def updatable(): + return "version 1" + + serve.run(updatable.bind()) + + # Test initial version + response = requests.get("https://localhost:8000/updatable", verify=False) + assert response.text == "version 1" + + # Update deployment + @serve.deployment + def updatable(): + return "version 2" + + serve.run(updatable.bind()) + + # Test updated version + response = requests.get("https://localhost:8000/updatable", verify=False) + assert response.text == "version 2" + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", "-s", __file__])) From 669ffcca2e7e5deccf3c4c6fdb70d1e24b265052 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 26 Aug 2025 11:05:20 -0700 Subject: [PATCH 286/634] [Core] Print out docker run output when it fails (#55918) Signed-off-by: Jiajun Yao --- python/ray/tests/conftest_docker.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/ray/tests/conftest_docker.py b/python/ray/tests/conftest_docker.py index ffe5d5c72ac4..12be797a23c6 100644 --- a/python/ray/tests/conftest_docker.py +++ b/python/ray/tests/conftest_docker.py @@ -221,7 +221,16 @@ def podman_docker_cluster(): "-f", "/dev/null", ] - container_id = subprocess.check_output(start_container_command).decode("utf-8") + try: + container_id = subprocess.check_output( + start_container_command, stderr=subprocess.STDOUT + ).decode("utf-8") + except subprocess.CalledProcessError as e: + error_output = e.output.decode("utf-8") if e.output else "No output" + print(f"Command failed with return code {e.returncode}") + print(f"Full error output:\n{error_output}") + raise + container_id = container_id.strip() # Get group id that owns the docker socket file. Add user `ray` to From 3ece65bc5ff9034578da1c7b9bc87a2bd6ec4c23 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 26 Aug 2025 15:18:55 -0500 Subject: [PATCH 287/634] [core] Remove `gcs_rpc_server.h` dependency from `InternalPubSubHandler` (#55952) Splitting gRPC service interface from implementation. --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 2 +- src/ray/gcs/gcs_server/gcs_server.cc | 6 ++- .../gcs/gcs_server/grpc_service_interfaces.h | 21 ++++++++ src/ray/gcs/gcs_server/grpc_services.cc | 13 +++++ src/ray/gcs/gcs_server/grpc_services.h | 23 +++++++++ src/ray/gcs/gcs_server/pubsub_handler.h | 4 +- src/ray/rpc/gcs/gcs_rpc_server.h | 49 ------------------- 7 files changed, 64 insertions(+), 54 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index d8d1aac34681..7c9a8109e48f 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -115,9 +115,9 @@ ray_cc_library( srcs = ["pubsub_handler.cc"], hdrs = ["pubsub_handler.h"], deps = [ + "//src/ray/gcs/gcs_server:grpc_service_interfaces", "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/protobuf:gcs_service_cc_proto", - "//src/ray/rpc:gcs_server", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", ], diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 9a5b55ef5561..a63cc1b7e2c2 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -621,8 +621,10 @@ void GcsServer::InitKVService() { void GcsServer::InitPubSubHandler() { auto &io_context = io_context_provider_.GetIOContext(); pubsub_handler_ = std::make_unique(io_context, *gcs_publisher_); - rpc_server_.RegisterService( - std::make_unique(io_context, *pubsub_handler_)); + + // This service is used to handle long poll requests, so we don't limit active RPCs. + rpc_server_.RegisterService(std::make_unique( + io_context, *pubsub_handler_, /*max_active_rpcs_per_handler_=*/-1)); } void GcsServer::InitRuntimeEnvManager() { diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h index d8dbb7febf4d..743017f20ed0 100644 --- a/src/ray/gcs/gcs_server/grpc_service_interfaces.h +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -64,6 +64,27 @@ class NodeInfoGcsServiceHandler { SendReplyCallback send_reply_callback) = 0; }; +class InternalPubSubGcsServiceHandler { + public: + virtual ~InternalPubSubGcsServiceHandler() = default; + + virtual void HandleGcsPublish(GcsPublishRequest request, + GcsPublishReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGcsSubscriberPoll(GcsSubscriberPollRequest request, + GcsSubscriberPollReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGcsSubscriberCommandBatch(GcsSubscriberCommandBatchRequest request, + GcsSubscriberCommandBatchReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGcsUnregisterSubscriber(GcsUnregisterSubscriberRequest request, + GcsUnregisterSubscriberReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + class JobInfoGcsServiceHandler { public: using JobFinishListenerCallback = std::function; diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc index a35a43e68c42..6ac013f7502a 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -36,6 +36,19 @@ void NodeInfoGrpcService::InitServerCallFactories( RPC_SERVICE_HANDLER(NodeInfoGcsService, CheckAlive, max_active_rpcs_per_handler_) } +void InternalPubSubGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + RPC_SERVICE_HANDLER(InternalPubSubGcsService, GcsPublish, max_active_rpcs_per_handler_); + RPC_SERVICE_HANDLER( + InternalPubSubGcsService, GcsSubscriberPoll, max_active_rpcs_per_handler_); + RPC_SERVICE_HANDLER( + InternalPubSubGcsService, GcsSubscriberCommandBatch, max_active_rpcs_per_handler_); + RPC_SERVICE_HANDLER( + InternalPubSubGcsService, GcsUnregisterSubscriber, max_active_rpcs_per_handler_); +} + void JobInfoGrpcService::InitServerCallFactories( const std::unique_ptr &cq, std::vector> *server_call_factories, diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/gcs_server/grpc_services.h index 90ef1d75a4e3..d825ad7112bb 100644 --- a/src/ray/gcs/gcs_server/grpc_services.h +++ b/src/ray/gcs/gcs_server/grpc_services.h @@ -58,6 +58,29 @@ class NodeInfoGrpcService : public GrpcService { int64_t max_active_rpcs_per_handler_; }; +class InternalPubSubGrpcService : public GrpcService { + public: + InternalPubSubGrpcService(instrumented_io_context &io_service, + InternalPubSubGcsServiceHandler &handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler) {} + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + InternalPubSubGcsService::AsyncService service_; + InternalPubSubGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + class JobInfoGrpcService : public GrpcService { public: explicit JobInfoGrpcService(instrumented_io_context &io_service, diff --git a/src/ray/gcs/gcs_server/pubsub_handler.h b/src/ray/gcs/gcs_server/pubsub_handler.h index 34554c74bc39..521863db6ed9 100644 --- a/src/ray/gcs/gcs_server/pubsub_handler.h +++ b/src/ray/gcs/gcs_server/pubsub_handler.h @@ -18,8 +18,8 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" namespace ray { namespace gcs { @@ -27,7 +27,7 @@ namespace gcs { /// This is the implementation class of `InternalPubsubHandler`. /// It supports subscribing updates from GCS with long poll, and registering / /// de-registering subscribers. -class InternalPubSubHandler : public rpc::InternalPubSubHandler { +class InternalPubSubHandler : public rpc::InternalPubSubGcsServiceHandler { public: InternalPubSubHandler(instrumented_io_context &io_service, gcs::GcsPublisher &gcs_publisher); diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index 65e214245162..3cbc18558d07 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -145,10 +145,6 @@ namespace rpc { #define INTERNAL_KV_SERVICE_RPC_HANDLER(HANDLER) \ RPC_SERVICE_HANDLER(InternalKVGcsService, HANDLER, -1) -// Unlimited max active RPCs, because of long poll. -#define INTERNAL_PUBSUB_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(InternalPubSubGcsService, HANDLER, -1) - #define GCS_RPC_SEND_REPLY(send_reply_callback, reply, status) \ reply->mutable_status()->set_code(static_cast(status.code())); \ reply->mutable_status()->set_message(status.message()); \ @@ -486,55 +482,10 @@ class RayEventExportGrpcService : public GrpcService { RayEventExportGcsServiceHandler &service_handler_; }; -class InternalPubSubGcsServiceHandler { - public: - virtual ~InternalPubSubGcsServiceHandler() = default; - - virtual void HandleGcsPublish(GcsPublishRequest request, - GcsPublishReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGcsSubscriberPoll(GcsSubscriberPollRequest request, - GcsSubscriberPollReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGcsSubscriberCommandBatch(GcsSubscriberCommandBatchRequest request, - GcsSubscriberCommandBatchReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGcsUnregisterSubscriber(GcsUnregisterSubscriberRequest request, - GcsUnregisterSubscriberReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -class InternalPubSubGrpcService : public GrpcService { - public: - InternalPubSubGrpcService(instrumented_io_context &io_service, - InternalPubSubGcsServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler) {} - - protected: - grpc::Service &GetGrpcService() override { return service_; } - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - INTERNAL_PUBSUB_SERVICE_RPC_HANDLER(GcsPublish); - INTERNAL_PUBSUB_SERVICE_RPC_HANDLER(GcsSubscriberPoll); - INTERNAL_PUBSUB_SERVICE_RPC_HANDLER(GcsSubscriberCommandBatch); - INTERNAL_PUBSUB_SERVICE_RPC_HANDLER(GcsUnregisterSubscriber); - } - - private: - InternalPubSubGcsService::AsyncService service_; - InternalPubSubGcsServiceHandler &service_handler_; -}; - using ActorInfoHandler = ActorInfoGcsServiceHandler; using NodeResourceInfoHandler = NodeResourceInfoGcsServiceHandler; using PlacementGroupInfoHandler = PlacementGroupInfoGcsServiceHandler; using InternalKVHandler = InternalKVGcsServiceHandler; -using InternalPubSubHandler = InternalPubSubGcsServiceHandler; using TaskInfoHandler = TaskInfoGcsServiceHandler; using RayEventExportHandler = RayEventExportGcsServiceHandler; From e0fa662147c370635d488ef42558a85b5f6858d1 Mon Sep 17 00:00:00 2001 From: gangsf Date: Tue, 26 Aug 2025 13:36:32 -0700 Subject: [PATCH 288/634] feat(runtime_env): add Azure ABFSS Blob Storage support (#55963) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit abfss:// Context: This stands for Azure Blob File System Secure and is used specifically with Azure Data Lake Storage Gen2.Usage: It is the actual protocol used by Spark, Hadoop, and other big data tools to securely access files in ADLS Gen2.Security: The abfss:// scheme ensures secure communication (HTTPS) and is preferred over abfs://, which is non-secure. Example: abfss://myfilesystem@myaccount.dfs.core.windows.net/myfolder/myfile.csv --------- Co-authored-by: Gang Zhao --- python/ray/_private/runtime_env/protocol.py | 46 ++++++++++++++++++- .../ray/tests/test_runtime_env_packaging.py | 15 ++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/python/ray/_private/runtime_env/protocol.py b/python/ray/_private/runtime_env/protocol.py index b61dea8f71fa..9d9887b409a5 100644 --- a/python/ray/_private/runtime_env/protocol.py +++ b/python/ray/_private/runtime_env/protocol.py @@ -30,13 +30,15 @@ def get_protocols(cls): "gs", # Remote azure blob storage path, assumes everything packed in one zip file. "azure", + # Remote Azure Blob File System Secure path, assumes everything packed in one zip file. + "abfss", # File storage path, assumes everything packed in one zip file. "file", } @classmethod def get_remote_protocols(cls): - return {"https", "s3", "gs", "azure", "file"} + return {"https", "s3", "gs", "azure", "abfss", "file"} @classmethod def _handle_s3_protocol(cls): @@ -122,6 +124,46 @@ def _handle_azure_protocol(cls): return open_file, transport_params + @classmethod + def _handle_abfss_protocol(cls): + """Set up Azure Blob File System Secure (ABFSS) protocol handling. + + Returns: + tuple: (open_file function, transport_params) + + Raises: + ImportError: If required dependencies are not installed. + ValueError: If required environment variables are not set. + """ + try: + from azure.identity import DefaultAzureCredential + from azure.storage.blob import BlobServiceClient # noqa: F401 + from smart_open import open as open_file + except ImportError: + raise ImportError( + "You must `pip install azure-storage-blob azure-identity smart_open[azure]` " + "to fetch URIs in Azure Blob File System Secure. " + + cls._MISSING_DEPENDENCIES_WARNING + ) + + # Define authentication variable + azure_storage_account_name = os.getenv("AZURE_STORAGE_ACCOUNT") + + if not azure_storage_account_name: + raise ValueError( + "Azure Blob File System Secure authentication requires " + "AZURE_STORAGE_ACCOUNT environment variable to be set." + ) + + account_url = f"https://{azure_storage_account_name}.dfs.core.windows.net/" + transport_params = { + "client": BlobServiceClient( + account_url=account_url, credential=DefaultAzureCredential() + ) + } + + return open_file, transport_params + @classmethod def download_remote_uri(cls, protocol: str, source_uri: str, dest_file: str): """Download file from remote URI to destination file. @@ -151,6 +193,8 @@ def open_file(uri, mode, *, transport_params=None): open_file, tp = cls._handle_gs_protocol() elif protocol == "azure": open_file, tp = cls._handle_azure_protocol() + elif protocol == "abfss": + open_file, tp = cls._handle_abfss_protocol() else: try: from smart_open import open as open_file diff --git a/python/ray/tests/test_runtime_env_packaging.py b/python/ray/tests/test_runtime_env_packaging.py index 21f226b94965..4b56a71c9d03 100644 --- a/python/ray/tests/test_runtime_env_packaging.py +++ b/python/ray/tests/test_runtime_env_packaging.py @@ -491,6 +491,11 @@ class TestParseUri: ("https://test.com/file.zip", Protocol.HTTPS, "https_test_com_file.zip"), ("gs://bucket/file.zip", Protocol.GS, "gs_bucket_file.zip"), ("azure://container/file.zip", Protocol.AZURE, "azure_container_file.zip"), + ( + "abfss://container@account.dfs.core.windows.net/file.zip", + Protocol.ABFSS, + "abfss_container_account_dfs_core_windows_net_file.zip", + ), ( "https://test.com/package-0.0.1-py2.py3-none-any.whl?param=value", Protocol.HTTPS, @@ -553,6 +558,11 @@ def test_parse_private_git_https_uris(self, parsing_tuple): Protocol.AZURE, "azure_fake_2022-10-21T13_11_35_00_00_package.zip", ), + ( + "abfss://container@account.dfs.core.windows.net/2022-10-21T13:11:35+00:00/package.zip", + Protocol.ABFSS, + "abfss_container_account_dfs_core_windows_net_2022-10-21T13_11_35_00_00_package.zip", + ), ( "file:///fake/2022-10-21T13:11:35+00:00/package.zip", Protocol.FILE, @@ -594,6 +604,11 @@ def test_parse_uris_with_disallowed_chars(self, parsing_tuple): Protocol.AZURE, "package.whl", ), + ( + "abfss://container@account.dfs.core.windows.net/2022-10-21T13:11:35+00:00/package.whl", + Protocol.ABFSS, + "package.whl", + ), ( "file:///fake/2022-10-21T13:11:35+00:00/package.whl", Protocol.FILE, From f9fdd07d5c93cf48b6f6ec47ca22db115f1f0ee5 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 26 Aug 2025 14:37:55 -0700 Subject: [PATCH 289/634] Revert "[Core] Bind grpc servers to specified ip instead of 0.0.0.0 (#55484)" (#55962) --- python/ray/tests/test_autoscaler.py | 2 +- python/ray/tests/test_multi_node_3.py | 2 +- src/ray/core_worker/core_worker_process.cc | 6 ++++-- src/ray/core_worker/tests/core_worker_test.cc | 4 ++-- src/ray/gcs/gcs_server/gcs_server.cc | 2 +- .../gcs_server/tests/gcs_health_check_manager_test.cc | 2 +- src/ray/object_manager/object_manager.cc | 2 +- src/ray/raylet/node_manager.cc | 5 +++-- src/ray/raylet/tests/node_manager_test.cc | 2 -- src/ray/rpc/grpc_server.cc | 3 ++- src/ray/rpc/grpc_server.h | 11 ++++++----- src/ray/rpc/tests/grpc_bench/grpc_bench.cc | 2 +- src/ray/rpc/tests/grpc_server_client_test.cc | 2 +- 13 files changed, 24 insertions(+), 21 deletions(-) diff --git a/python/ray/tests/test_autoscaler.py b/python/ray/tests/test_autoscaler.py index 506327bd6980..a728071528c2 100644 --- a/python/ray/tests/test_autoscaler.py +++ b/python/ray/tests/test_autoscaler.py @@ -3517,7 +3517,7 @@ def __init__(self, *args, **kwargs): _internal_kv_initialized=Mock(return_value=False), ): monitor = Monitor( - address=f"{ray.util.get_node_ip_address()}:12345", + address="localhost:12345", autoscaling_config="", log_dir=self.tmpdir, ) diff --git a/python/ray/tests/test_multi_node_3.py b/python/ray/tests/test_multi_node_3.py index 8c89115e2bda..f741baa7bef0 100644 --- a/python/ray/tests/test_multi_node_3.py +++ b/python/ray/tests/test_multi_node_3.py @@ -241,7 +241,7 @@ def f(): indirect=True, ) def test_using_hostnames(call_ray_start): - ray.init(address=call_ray_start) + ray.init(_node_ip_address="localhost", address="localhost:6379") @ray.remote def f(): diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index fa3f1ef415e1..c870835c47bd 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -247,8 +247,10 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( std::move(raylet_address), *client_call_manager, /*raylet_unavailable_timeout_callback=*/[] {}); - auto core_worker_server = std::make_unique( - WorkerTypeString(options.worker_type), assigned_port, options.node_ip_address); + auto core_worker_server = + std::make_unique(WorkerTypeString(options.worker_type), + assigned_port, + options.node_ip_address == "127.0.0.1"); // Start RPC server after all the task receivers are properly initialized and we have // our assigned port from the raylet. core_worker_server->RegisterService( diff --git a/src/ray/core_worker/tests/core_worker_test.cc b/src/ray/core_worker/tests/core_worker_test.cc index f3f538e4191e..0fa7170c1838 100644 --- a/src/ray/core_worker/tests/core_worker_test.cc +++ b/src/ray/core_worker/tests/core_worker_test.cc @@ -104,8 +104,8 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { auto service_handler = std::make_unique(); auto worker_context = std::make_unique( WorkerType::WORKER, WorkerID::FromRandom(), JobID::FromInt(1)); - auto core_worker_server = std::make_unique( - WorkerTypeString(options.worker_type), 0, "127.0.0.1"); + auto core_worker_server = + std::make_unique(WorkerTypeString(options.worker_type), 0, true); core_worker_server->RegisterService( std::make_unique(io_service_, *service_handler), false /* token_auth */); diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index a63cc1b7e2c2..d49029f4dc93 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -62,7 +62,7 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, storage_type_(GetStorageType()), rpc_server_(config.grpc_server_name, config.grpc_server_port, - config.node_ip_address, + config.node_ip_address == "127.0.0.1", ClusterID::Nil(), config.grpc_server_thread_num, /*keepalive_time_ms=*/RayConfig::instance().grpc_keepalive_time_ms()), diff --git a/src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc index ed376e4224b0..8ca66a12522f 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc @@ -88,7 +88,7 @@ class GcsHealthCheckManagerTest : public ::testing::Test { auto node_id = NodeID::FromRandom(); auto port = GetFreePort(); RAY_LOG(INFO) << "Get port " << port; - auto server = std::make_shared(node_id.Hex(), port, "127.0.0.1"); + auto server = std::make_shared(node_id.Hex(), port, true); auto channel = grpc::CreateChannel(BuildAddress("localhost", port), grpc::InsecureChannelCredentials()); diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index 6abe66bd2ea6..53b18b32d11c 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -107,7 +107,7 @@ ObjectManager::ObjectManager( rpc_work_(rpc_service_.get_executor()), object_manager_server_("ObjectManager", config_.object_manager_port, - config_.object_manager_address, + config_.object_manager_address == "127.0.0.1", ClusterID::Nil(), config_.rpc_service_threads_number), client_call_manager_(main_service, diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 0f536647b7cd..0806b072b696 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -160,8 +160,9 @@ NodeManager::NodeManager( RAY_UNUSED(execute_after( io_service_, fn, std::chrono::milliseconds(delay_ms))); }), - node_manager_server_( - "NodeManager", config.node_manager_port, config.node_manager_address), + node_manager_server_("NodeManager", + config.node_manager_port, + config.node_manager_address == "127.0.0.1"), local_object_manager_(local_object_manager), leased_workers_(leased_workers), high_plasma_storage_usage_(RayConfig::instance().high_plasma_storage_usage()), diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index ffc17e1af5e6..2f041bf2825b 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -389,8 +389,6 @@ class NodeManagerTest : public ::testing::Test { })"); NodeManagerConfig node_manager_config{}; - node_manager_config.node_manager_address = "127.0.0.1"; - node_manager_config.node_manager_port = 0; node_manager_config.maximum_startup_concurrency = 1; node_manager_config.store_socket_name = "test_store_socket"; diff --git a/src/ray/rpc/grpc_server.cc b/src/ray/rpc/grpc_server.cc index c644dc85280e..047ff734e94b 100644 --- a/src/ray/rpc/grpc_server.cc +++ b/src/ray/rpc/grpc_server.cc @@ -62,7 +62,8 @@ void GrpcServer::Shutdown() { void GrpcServer::Run() { uint32_t specified_port = port_; - std::string server_address = BuildAddress(ip_address_, port_); + std::string server_address = + BuildAddress((listen_to_localhost_only_ ? "127.0.0.1" : "0.0.0.0"), port_); grpc::ServerBuilder builder; // Disable the SO_REUSEPORT option. We don't need it in ray. If the option is enabled // (default behavior in grpc), we may see multiple workers listen on the same port and diff --git a/src/ray/rpc/grpc_server.h b/src/ray/rpc/grpc_server.h index 56566569c359..686c4a68b2a4 100644 --- a/src/ray/rpc/grpc_server.h +++ b/src/ray/rpc/grpc_server.h @@ -95,14 +95,14 @@ class GrpcServer { /// GrpcServer(std::string name, const uint32_t port, - std::string ip_address, + bool listen_to_localhost_only, const ClusterID &cluster_id = ClusterID::Nil(), int num_threads = 1, int64_t keepalive_time_ms = 7200000 /*2 hours, grpc default*/) : name_(std::move(name)), port_(port), - ip_address_(std::move(ip_address)), - cluster_id_(cluster_id), + listen_to_localhost_only_(listen_to_localhost_only), + cluster_id_(ClusterID::Nil()), is_shutdown_(true), num_threads_(num_threads), keepalive_time_ms_(keepalive_time_ms) { @@ -161,8 +161,9 @@ class GrpcServer { const std::string name_; /// Port of this server. int port_; - /// IP address of this server. - const std::string ip_address_; + /// Listen to localhost (127.0.0.1) only if it's true, otherwise listen to all network + /// interfaces (0.0.0.0) + const bool listen_to_localhost_only_; /// Token representing ID of this cluster. ClusterID cluster_id_; /// Indicates whether this server is in shutdown state. diff --git a/src/ray/rpc/tests/grpc_bench/grpc_bench.cc b/src/ray/rpc/tests/grpc_bench/grpc_bench.cc index f1268c93d764..86b8e7ef4e27 100644 --- a/src/ray/rpc/tests/grpc_bench/grpc_bench.cc +++ b/src/ray/rpc/tests/grpc_bench/grpc_bench.cc @@ -71,7 +71,7 @@ int main() { const auto env = std::getenv("GRPC_SERVER_CPUS"); const auto parallelism = env ? std::atoi(env) : std::thread::hardware_concurrency(); - GrpcServer server("grpc_bench", 50051, "0.0.0.0", ClusterID::Nil(), parallelism); + GrpcServer server("grpc_bench", 50051, false, ClusterID::Nil(), parallelism); instrumented_io_context main_service; std::thread t([&main_service] { boost::asio::executor_work_guard work( diff --git a/src/ray/rpc/tests/grpc_server_client_test.cc b/src/ray/rpc/tests/grpc_server_client_test.cc index c4bbd1e42033..b020b7b45b8b 100644 --- a/src/ray/rpc/tests/grpc_server_client_test.cc +++ b/src/ray/rpc/tests/grpc_server_client_test.cc @@ -111,7 +111,7 @@ class TestGrpcServerClientFixture : public ::testing::Test { handler_io_service_work_(handler_io_service_.get_executor()); handler_io_service_.run(); }); - grpc_server_.reset(new GrpcServer("test", 0, "127.0.0.1")); + grpc_server_.reset(new GrpcServer("test", 0, true)); grpc_server_->RegisterService( std::make_unique(handler_io_service_, test_service_handler_), false); From 52b932568df0ebca672f93cb314187d3ad3d39a5 Mon Sep 17 00:00:00 2001 From: Stephanie Wang Date: Tue, 26 Aug 2025 14:47:12 -0700 Subject: [PATCH 290/634] [core][gpu-objects] Automatically enable tensor transport for the actor if any method specifies one (#55324) This PR makes the `@ray.remote(enable_tensor_transport=True)` flag optional by checking if any of the actor's methods have `@ray.method(tensor_transport=...)` and automatically enabling tensor transport if so. This step is needed before actor creation because we need to create a background concurrency group on the actor if tensor transport is enabled. It also adds some validation checks for these corner cases: - `@ray.remote(enable_tensor_transport=True)` and actor method dynamically specifies `.options(tensor_transport=...)` -> succeeds - `@ray.remote` and actor method dynamically specifies `.options(tensor_transport=...)` -> fails, user needs to specify `enable_tensor_transport=True` - Actor method with tensor transport is called, but no collective group is created yet -> fails, user needs to create a collective group first ## Related issue number Closes #55269. --------- Signed-off-by: Stephanie wang --- ci/lint/pydoclint-baseline.txt | 2 +- python/ray/_common/ray_option_utils.py | 2 +- python/ray/_raylet.pyx | 6 + python/ray/actor.py | 115 ++++++++++----- .../collective/collective_tensor_transport.py | 2 +- .../gpu_object_manager/gpu_object_manager.py | 26 ++++ python/ray/tests/test_gpu_objects_gloo.py | 131 +++++++++++++++++- 7 files changed, 248 insertions(+), 36 deletions(-) diff --git a/ci/lint/pydoclint-baseline.txt b/ci/lint/pydoclint-baseline.txt index 56df1bd45d5c..84b97b4f4724 100644 --- a/ci/lint/pydoclint-baseline.txt +++ b/ci/lint/pydoclint-baseline.txt @@ -307,7 +307,7 @@ python/ray/actor.py DOC201: Method `ActorMethod.options` does not have a return section in docstring DOC101: Method `_ActorClassMetadata.__init__`: Docstring contains fewer arguments than in function signature. DOC107: Method `_ActorClassMetadata.__init__`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints - DOC103: Method `_ActorClassMetadata.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [accelerator_type: , actor_creation_function_descriptor: , class_id: , concurrency_groups: , enable_tensor_transport: bool, label_selector: , language: , max_restarts: , max_task_retries: , memory: , modified_class: , num_cpus: , num_gpus: , object_store_memory: , resources: , runtime_env: , scheduling_strategy: SchedulingStrategyT]. + DOC103: Method `_ActorClassMetadata.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [accelerator_type: , actor_creation_function_descriptor: , class_id: , concurrency_groups: , enable_tensor_transport: bool, label_selector: , language: , max_restarts: , max_task_retries: , memory: , method_meta: , modified_class: , num_cpus: , num_gpus: , object_store_memory: , resources: , runtime_env: , scheduling_strategy: SchedulingStrategyT]. DOC101: Method `ActorClass.__init__`: Docstring contains fewer arguments than in function signature. DOC106: Method `ActorClass.__init__`: The option `--arg-type-hints-in-signature` is `True` but there are no argument type hints in the signature DOC107: Method `ActorClass.__init__`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints diff --git a/python/ray/_common/ray_option_utils.py b/python/ray/_common/ray_option_utils.py index 6ff64108f795..8d56f0d0f675 100644 --- a/python/ray/_common/ray_option_utils.py +++ b/python/ray/_common/ray_option_utils.py @@ -222,7 +222,7 @@ def issubclass_safe(obj: Any, cls_: type) -> bool: _actor_only_options = { "concurrency_groups": Option((list, dict, type(None))), - "enable_tensor_transport": Option(bool, default_value=False), + "enable_tensor_transport": Option((bool, type(None)), default_value=None), "lifetime": Option( (str, type(None)), lambda x: None diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index f513243a660f..fdd0dea05c72 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -4155,6 +4155,11 @@ cdef class CoreWorker: method_meta.retry_exceptions, method_meta.generator_backpressure_num_objects, # noqa method_meta.enable_task_events, + # TODO(swang): Pass + # enable_tensor_transport when + # serializing an ActorHandle and + # sending to another actor. + False, # enable_tensor_transport method_meta.method_name_to_tensor_transport, actor_method_cpu, actor_creation_function_descriptor, @@ -4173,6 +4178,7 @@ cdef class CoreWorker: {}, # method retry_exceptions {}, # generator_backpressure_num_objects {}, # enable_task_events + False, # enable_tensor_transport None, # method_name_to_tensor_transport 0, # actor method cpu actor_creation_function_descriptor, diff --git a/python/ray/actor.py b/python/ray/actor.py index 4e9e22f1a7dd..15b8df351618 100644 --- a/python/ray/actor.py +++ b/python/ray/actor.py @@ -815,11 +815,26 @@ def _remote( if tensor_transport is None: tensor_transport = self._tensor_transport - if tensor_transport != TensorTransportEnum.OBJECT_STORE and num_returns != 1: - raise ValueError( - f"Currently, methods with tensor_transport={tensor_transport.name} only support 1 return value. " - "Please make sure the actor method is decorated with `@ray.method(num_returns=1)` (the default)." - ) + if tensor_transport != TensorTransportEnum.OBJECT_STORE: + if num_returns != 1: + raise ValueError( + f"Currently, methods with tensor_transport={tensor_transport.name} only support 1 return value. " + "Please make sure the actor method is decorated with `@ray.method(num_returns=1)` (the default)." + ) + if not self._actor._ray_enable_tensor_transport: + raise ValueError( + f'Currently, methods with .options(tensor_transport="{tensor_transport.name}") are not supported when enable_tensor_transport=False. ' + "Please set @ray.remote(enable_tensor_transport=True) on the actor class definition." + ) + gpu_object_manager = ray._private.worker.global_worker.gpu_object_manager + if not gpu_object_manager.actor_has_tensor_transport( + self._actor, tensor_transport + ): + raise ValueError( + f"{self._actor} does not have tensor transport {tensor_transport.name} available. Please create a communicator with " + "`ray.experimental.collective.create_collective_group` " + "before calling actor tasks with non-default tensor_transport." + ) args = args or [] kwargs = kwargs or {} @@ -933,7 +948,6 @@ def create( cls, modified_class, actor_creation_function_descriptor, - enable_tensor_transport: bool = False, ): # Try to create an instance from cache. cached_meta = cls._cache.get(actor_creation_function_descriptor) @@ -960,6 +974,15 @@ def create( self.concurrency_group_for_methods = {} self.method_name_to_tensor_transport: Dict[str, TensorTransportEnum] = {} + # Check whether any actor methods specify a non-default tensor transport. + self.has_tensor_transport_methods = any( + getattr( + method, "__ray_tensor_transport__", TensorTransportEnum.OBJECT_STORE + ) + != TensorTransportEnum.OBJECT_STORE + for _, method in actor_methods + ) + for method_name, method in actor_methods: # Whether or not this method requires binding of its first # argument. For class and static methods, we do not want to bind @@ -1019,15 +1042,6 @@ def create( method_name ] = method.__ray_tensor_transport__ - method_tensor_transport = self.method_name_to_tensor_transport.get( - method_name, None - ) - if not enable_tensor_transport and method_tensor_transport is not None: - if method_tensor_transport != TensorTransportEnum.OBJECT_STORE: - raise ValueError( - f"Method {method_name} has tensor_transport={method_tensor_transport.name} but enable_tensor_transport is False" - ) - # Update cache. cls._cache[actor_creation_function_descriptor] = self return self @@ -1043,6 +1057,7 @@ class _ActorClassMetadata: actor_creation_function_descriptor: The function descriptor for the actor creation task. class_id: The ID of this actor class. + method_meta: The actor method metadata. class_name: The name of this class. num_cpus: The default number of CPUs required by the actor creation task. @@ -1058,14 +1073,14 @@ class _ActorClassMetadata: See :ref:`accelerator types `. runtime_env: The runtime environment for this actor. scheduling_strategy: Strategy about how to schedule this actor. - enable_tensor_transport: Whether to enable out-of-band tensor transport for this actor. last_export_cluster_and_job: A pair of the last exported cluster and job to help us to know whether this function was exported. This is an imperfect mechanism used to determine if we need to export the remote function again. It is imperfect in the sense that the actor class definition could be exported multiple times by different workers. - method_meta: The actor method metadata. + enable_tensor_transport: Whether to enable out-of-band tensor transport + for this actor. """ def __init__( @@ -1074,6 +1089,7 @@ def __init__( modified_class, actor_creation_function_descriptor, class_id, + method_meta, max_restarts, max_task_retries, num_cpus, @@ -1086,11 +1102,12 @@ def __init__( runtime_env, concurrency_groups, scheduling_strategy: SchedulingStrategyT, - enable_tensor_transport: bool = False, + enable_tensor_transport: bool, ): self.language = language self.modified_class = modified_class self.actor_creation_function_descriptor = actor_creation_function_descriptor + self.method_meta = method_meta self.class_name = actor_creation_function_descriptor.class_name self.is_cross_language = language != Language.PYTHON self.class_id = class_id @@ -1106,13 +1123,8 @@ def __init__( self.runtime_env = runtime_env self.concurrency_groups = concurrency_groups self.scheduling_strategy = scheduling_strategy - self.enable_tensor_transport = enable_tensor_transport self.last_export_cluster_and_job = None - self.method_meta = _ActorClassMethodMetadata.create( - modified_class, - actor_creation_function_descriptor, - self.enable_tensor_transport, - ) + self.enable_tensor_transport = enable_tensor_transport @PublicAPI @@ -1120,7 +1132,7 @@ class ActorClassInheritanceException(TypeError): pass -def _process_option_dict(actor_options): +def _process_option_dict(actor_options, has_tensor_transport_methods): _filled_options = {} arg_names = set(inspect.getfullargspec(_ActorClassMetadata.__init__)[0]) for k, v in ray_option_utils.actor_options.items(): @@ -1129,15 +1141,28 @@ def _process_option_dict(actor_options): _filled_options["runtime_env"] = parse_runtime_env_for_task_or_actor( _filled_options["runtime_env"] ) + # If any actor method has a non-default tensor transport, automatically + # enable tensor transport, unless it was explicitly set to False by the + # user. + if has_tensor_transport_methods: + if _filled_options["enable_tensor_transport"] is False: + raise ValueError( + "Actor class has methods with @ray.method(tensor_transport=...) decorator but @ray.remote(enable_tensor_transport=False). " + "Either set enable_tensor_transport=True or remove the @ray.method(tensor_transport=...) decorator from the methods." + ) + _filled_options["enable_tensor_transport"] = True # Ray GPU objects requires a background thread for data transfer. However, # currently by default the background thread will be blocked if the main - # thread does not yield. For now, we explicitly create the background - # thread, which forces Ray to execute all tasks on background threads - # instead of the main thread. + # thread does not yield. For now, we explicitly create the background thread + # if `@ray.remote(enable_tensor_transport=True)` or if any methods are + # decorated with `@ray.method(tensor_transport=...)` and a non-default + # tensor transport. This forces Ray to execute all tasks on background + # threads instead of the main thread. # TODO(swang): Remove this code once # https://github.com/ray-project/ray/issues/54639 is fixed. - if _filled_options.get("enable_tensor_transport", False): + enable_tensor_transport = _filled_options.get("enable_tensor_transport", False) + if enable_tensor_transport: if _filled_options.get("concurrency_groups", None) is None: _filled_options["concurrency_groups"] = {} _filled_options["concurrency_groups"]["_ray_system"] = 1 @@ -1253,12 +1278,19 @@ def __init__(self, *args, **kwargs): modified_class.__ray_actor_class__ ) + actor_method_meta = _ActorClassMethodMetadata.create( + modified_class, + actor_creation_function_descriptor, + ) self.__ray_metadata__ = _ActorClassMetadata( Language.PYTHON, modified_class, actor_creation_function_descriptor, class_id, - **_process_option_dict(actor_options), + actor_method_meta, + **_process_option_dict( + actor_options, actor_method_meta.has_tensor_transport_methods + ), ) self._default_options = actor_options if "runtime_env" in self._default_options: @@ -1274,12 +1306,20 @@ def _ray_from_function_descriptor( actor_options, ): self = ActorClass.__new__(ActorClass) + modified_class = None + actor_method_meta = _ActorClassMethodMetadata.create( + modified_class, + actor_creation_function_descriptor, + ) self.__ray_metadata__ = _ActorClassMetadata( language, - None, + modified_class, actor_creation_function_descriptor, None, - **_process_option_dict(actor_options), + actor_method_meta, + **_process_option_dict( + actor_options, actor_method_meta.has_tensor_transport_methods + ), ) self._default_options = actor_options if "runtime_env" in self._default_options: @@ -1775,6 +1815,7 @@ def _remote(self, args=None, kwargs=None, **actor_options) -> ActorProxy[T]: meta.method_meta.retry_exceptions, meta.method_meta.generator_backpressure_num_objects, meta.method_meta.enable_task_events, + meta.enable_tensor_transport, meta.method_meta.method_name_to_tensor_transport, actor_method_cpu, meta.actor_creation_function_descriptor, @@ -1867,6 +1908,7 @@ def __init__( method_retry_exceptions: Dict[str, Union[bool, list, tuple]], method_generator_backpressure_num_objects: Dict[str, int], method_enable_task_events: Dict[str, bool], + enable_tensor_transport: bool, method_name_to_tensor_transport: Dict[str, TensorTransportEnum], actor_method_cpus: int, actor_creation_function_descriptor, @@ -1890,6 +1932,10 @@ def __init__( method_retry_exceptions: Dictionary mapping method names to their retry exception settings. method_generator_backpressure_num_objects: Dictionary mapping method names to their generator backpressure settings. method_enable_task_events: Dictionary mapping method names to whether task events are enabled. + enable_tensor_transport: Whether tensor transport is enabled for + this actor. If True, then methods can be called with + .options(tensor_transport=...) to specify a non-default tensor + transport. method_name_to_tensor_transport: Dictionary mapping method names to their tensor transport settings. actor_method_cpus: The number of CPUs required by actor methods. actor_creation_function_descriptor: The function descriptor for actor creation. @@ -1916,6 +1962,7 @@ def __init__( method_generator_backpressure_num_objects ) self._ray_method_enable_task_events = method_enable_task_events + self._ray_enable_tensor_transport = enable_tensor_transport self._ray_method_name_to_tensor_transport = method_name_to_tensor_transport self._ray_actor_method_cpus = actor_method_cpus self._ray_cluster_and_job = cluster_and_job @@ -2238,6 +2285,8 @@ def _serialization_helper(self): self._ray_method_generator_backpressure_num_objects ), "method_enable_task_events": self._ray_method_enable_task_events, + "enable_tensor_transport": self._ray_enable_tensor_transport, + "method_name_to_tensor_transport": self._ray_method_name_to_tensor_transport, "actor_method_cpus": self._ray_actor_method_cpus, "actor_creation_function_descriptor": self._ray_actor_creation_function_descriptor, # noqa: E501 }, @@ -2287,6 +2336,8 @@ def _deserialization_helper(cls, state, weak_ref: bool, outer_object_ref=None): state["method_retry_exceptions"], state["method_generator_backpressure_num_objects"], state["method_enable_task_events"], + state["enable_tensor_transport"], + state["method_name_to_tensor_transport"], state["actor_method_cpus"], state["actor_creation_function_descriptor"], state["current_cluster_and_job"], diff --git a/python/ray/experimental/collective/collective_tensor_transport.py b/python/ray/experimental/collective/collective_tensor_transport.py index 64bc0991db26..7bf39aed7b25 100644 --- a/python/ray/experimental/collective/collective_tensor_transport.py +++ b/python/ray/experimental/collective/collective_tensor_transport.py @@ -74,7 +74,7 @@ def get_communicator_metadata( f"No communicators found for actors {src_actor} and {dst_actor}. " "Create a communicator with " "`ray.experimental.collective.create_collective_group` " - "before calling actor tasks." + "before calling actor tasks. with non-default tensor_transport." ) elif len(communicators) > 1: raise ValueError( diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py index 9102edb86566..6c3157b13134 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py @@ -253,3 +253,29 @@ def get_gpu_object(self, object_id: str) -> List["torch.Tensor"]: object_id, timeout=ray_constants.FETCH_FAIL_TIMEOUT_SECONDS ) return gpu_object + + def actor_has_tensor_transport( + self, actor: "ray.actor.ActorHandle", tensor_transport: TensorTransportEnum + ): + """ + Check if the actor has a communicator for the given tensor transport backend. + + Args: + actor: The actor to check. + tensor_transport: The tensor transport backend to check. + + Returns: + True if the actor has a communicator for the given tensor transport backend, False otherwise. + """ + # Import get_collective_groups here to avoid dependency on + # collective libraries for default Ray installation. + from ray.experimental.collective import get_collective_groups + from ray.experimental.gpu_object_manager.gpu_object_store import ( + _tensor_transport_to_collective_backend, + ) + + tensor_transport_backend = _tensor_transport_to_collective_backend( + tensor_transport + ) + communicators = get_collective_groups([actor], backend=tensor_transport_backend) + return len(communicators) > 0 diff --git a/python/ray/tests/test_gpu_objects_gloo.py b/python/ray/tests/test_gpu_objects_gloo.py index ff0c233a43bb..65f5550e2896 100644 --- a/python/ray/tests/test_gpu_objects_gloo.py +++ b/python/ray/tests/test_gpu_objects_gloo.py @@ -8,6 +8,7 @@ from ray.experimental.collective import create_collective_group from ray._private.custom_types import TensorTransportEnum from ray._common.test_utils import wait_for_condition +from ray._common.test_utils import SignalActor # tensordict is not supported on macos ci, so we skip the tests support_tensordict = sys.platform != "darwin" @@ -16,7 +17,11 @@ from tensordict import TensorDict -@ray.remote(enable_tensor_transport=True) +# TODO: check whether concurrency groups are created correctly if +# enable_tensor_transport is True or if any methods are decorated with +# @ray.method(tensor_transport=...). Check that specifying +# .options(tensor_transport=...) fails if enable_tensor_transport is False. +@ray.remote class GPUTestActor: @ray.method(tensor_transport="gloo") def echo(self, data): @@ -207,6 +212,78 @@ def test_p2p(ray_start_regular): assert ray.get(result) == pytest.approx(medium_tensor * 2) +def test_p2p_errors_before_group_creation(ray_start_regular): + world_size = 2 + actors = [GPUTestActor.remote() for _ in range(world_size)] + + small_tensor = torch.randn((1,)) + sender = actors[0] + + with pytest.raises( + ValueError, + match="Actor.* does not have tensor transport GLOO available. Please create a communicator with `ray.experimental.collective.create_collective_group` before calling actor tasks with non-default tensor_transport.", + ): + sender.echo.remote(small_tensor) + + +@pytest.mark.parametrize("has_tensor_transport_method", [True, False]) +def test_p2p_blocking(ray_start_regular, has_tensor_transport_method): + """Test that p2p transfers still work when sender is blocked in another + task. This should work whether the actor has (a) a tensor transport method + (a method decorated with @ray.method(tensor_transport=...)) or (b) an actor-level decorator + @ray.remote(enable_tensor_transport=True).""" + + class _GPUTestActor: + def double(self, data): + if isinstance(data, list): + return [self.double(d) for d in data] + if support_tensordict and isinstance(data, TensorDict): + return data.apply(lambda x: x * 2) + return data * 2 + + def infinite_sleep(self, signal): + signal.send.remote() + while True: + time.sleep(0.1) + + if has_tensor_transport_method: + # Test tensor transport annotation via ray.method. + @ray.remote + class GPUTestActor(_GPUTestActor): + @ray.method(tensor_transport="gloo") + def echo(self, data): + return data + + else: + # Test tensor transport annotation via ray.remote. + @ray.remote(enable_tensor_transport=True) + class GPUTestActor(_GPUTestActor): + def echo(self, data): + return data + + sender, receiver = GPUTestActor.remote(), GPUTestActor.remote() + signal = SignalActor.remote() + create_collective_group([sender, receiver], backend="torch_gloo") + tensor = torch.randn((500, 500)) + # If the actor does not have a tensor transport method declared, declare it + # dynamically using .options(). + sender_fn = ( + sender.echo + if has_tensor_transport_method + else sender.echo.options(tensor_transport="gloo") + ) + ref = sender_fn.remote(tensor) + + # Start a blocking task on the sender actor. + sender.infinite_sleep.remote(signal) + ray.get(signal.wait.remote(), timeout=10) + + # Ensure that others can still receive the object. + result = receiver.double.remote(ref) + result = ray.get(result, timeout=10) + assert result == pytest.approx(tensor * 2) + + def test_p2p_with_cpu_data(ray_start_regular): world_size = 2 actors = [GPUTestActor.remote() for _ in range(world_size)] @@ -494,6 +571,58 @@ def test_tensor_extracted_from_tensordict_in_gpu_object_store(ray_start_regular) assert torch.equal(ret_val_src[1], td["reward"]) +@pytest.mark.parametrize("enable_tensor_transport", [True, False]) +def test_dynamic_tensor_transport_via_options( + ray_start_regular, enable_tensor_transport +): + """Test that tensor_transport can be set dynamically via .options() at call + time, if enable_tensor_transport is set to True in @ray.remote.""" + + class TestActor: + def __init__(self): + pass + + def normal_method(self): + return "normal" + + def tensor_method(self): + return torch.randn(5, 5) + + def double(self, data): + return data * 2 + + if enable_tensor_transport: + TestActor = ray.remote(enable_tensor_transport=True)(TestActor) + else: + TestActor = ray.remote(TestActor) + + # Create actor without any tensor_transport decorators + sender = TestActor.remote() + receiver = TestActor.remote() + create_collective_group([sender, receiver], backend="torch_gloo") + + # Test normal method call + result = ray.get(sender.normal_method.remote()) + assert result == "normal" + + # Test method call with tensor_transport specified via .options() + if enable_tensor_transport: + # If enable_tensor_transport is set to True, then it's okay to use + # dynamic tensor_transport. + ref = sender.tensor_method.options(tensor_transport="gloo").remote() + tensor = ray.get(ref) + result = ray.get(receiver.double.remote(ref)) + assert result == pytest.approx(tensor * 2) + else: + # If enable_tensor_transport is not set, then user cannot use + # dynamic tensor_transport. + with pytest.raises( + ValueError, + match='Currently, methods with .options\\(tensor_transport="GLOO"\\) are not supported when enable_tensor_transport=False. Please set @ray.remote\\(enable_tensor_transport=True\\) on the actor class definition.', + ): + ref = sender.tensor_method.options(tensor_transport="gloo").remote() + + def test_gpu_object_ref_in_list_throws_exception(ray_start_regular): """Test that passing GPU ObjectRefs inside lists as task arguments raises an error.""" From 031266f73d293351ddad41c49653c89cb229ae15 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Tue, 26 Aug 2025 14:48:53 -0700 Subject: [PATCH 291/634] [docker] Update latest Docker dependencies for 2.49.0 release (#55966) Created by release automation bot. Update with commit ec5d4108737ead7dccb2930439842e0b2de012bb Signed-off-by: elliot-barn Co-authored-by: elliot-barn --- doc/source/ray-overview/installation.rst | 2 +- .../pip_freeze_ray-ml-py39-cpu.txt | 48 +++++++++++-------- .../ray-overview/pip_freeze_ray-py39-cpu.txt | 34 +++++++++---- 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/doc/source/ray-overview/installation.rst b/doc/source/ray-overview/installation.rst index a6131d6ab8b2..1bb7e8fcc552 100644 --- a/doc/source/ray-overview/installation.rst +++ b/doc/source/ray-overview/installation.rst @@ -436,7 +436,7 @@ We publish the dependencies that are installed in our ``ray`` Docker images for .. tab-item:: ray (Python 3.9) :sync: ray (Python 3.9) - Ray version: nightly (`f99d0ea `_) + Ray version: nightly (`ec5d410 `_) .. literalinclude:: ./pip_freeze_ray-py39-cpu.txt diff --git a/doc/source/ray-overview/pip_freeze_ray-ml-py39-cpu.txt b/doc/source/ray-overview/pip_freeze_ray-ml-py39-cpu.txt index 74d74f03ee00..01830b5aa586 100644 --- a/doc/source/ray-overview/pip_freeze_ray-ml-py39-cpu.txt +++ b/doc/source/ray-overview/pip_freeze_ray-ml-py39-cpu.txt @@ -12,11 +12,12 @@ aiohappyeyeballs==2.6.1 aiohttp==3.11.16 aiohttp-cors==0.7.0 aioitertools==0.11.0 -aiorwlock==1.5.0 +aiorwlock==1.3.0 aiosignal==1.3.1 aiosqlite==0.19.0 ale-py==0.10.1 alembic==1.12.1 +amqp==5.3.1 annotated-types==0.6.0 antlr4-python3-runtime==4.11.1 anyio==3.7.1 @@ -32,11 +33,15 @@ astunparse==1.6.3 async-timeout==4.0.3 attrs==25.1.0 ax-platform==0.3.2 +azure-common==1.1.28 +azure-core==1.29.5 +azure-storage-blob==12.22.0 Babel==2.13.1 backcall==0.2.0 base58==2.0.1 bayesian-optimization==1.4.3 beautifulsoup4==4.11.1 +billiard==4.2.1 bleach==6.1.0 bokeh==2.4.3 boltons @ file:///home/conda/feedstock_root/build_artifacts/boltons_1733827268945/work @@ -45,12 +50,15 @@ boto3==1.26.76 botocore==1.29.76 botorch==0.8.5 Brotli @ file:///home/conda/feedstock_root/build_artifacts/brotli-split_1749229842835/work -build==1.2.2.post1 cachetools==5.5.2 +celery==5.5.3 certifi==2025.1.31 cffi==1.16.0 charset-normalizer==3.3.2 click==8.1.7 +click-didyoumean==0.3.1 +click-plugins==1.1.1.2 +click-repl==0.3.0 cloudpickle==2.2.0 cma==3.2.2 cmdstanpy==1.2.0 @@ -60,7 +68,7 @@ colorful==0.5.5 colorlog==6.7.0 comet-ml==3.44.1 comm==0.2.0 -conda @ file:///home/conda/feedstock_root/build_artifacts/conda_1749201703459/work/conda-src +conda @ file:///home/conda/feedstock_root/build_artifacts/conda_1754405245494/work/conda-src conda-libmamba-solver @ file:///home/conda/feedstock_root/build_artifacts/conda-libmamba-solver_1745834476052/work/src conda-package-handling @ file:///home/conda/feedstock_root/build_artifacts/conda-package-handling_1736345463896/work conda_package_streaming @ file:///home/conda/feedstock_root/build_artifacts/conda-package-streaming_1729004031731/work @@ -121,7 +129,7 @@ gast==0.6.0 gcs-oauth2-boto-plugin==3.0 getdaft==0.4.3 gitdb==4.0.11 -GitPython==3.1.40 +GitPython==3.1.44 glfw==2.6.3 google-api-core==2.24.2 google-api-python-client==2.111.0 @@ -129,9 +137,13 @@ google-apitools==0.5.32 google-auth==2.23.4 google-auth-httplib2==0.1.1 google-auth-oauthlib==1.0.0 +google-cloud-core==2.4.1 +google-cloud-storage==2.14.0 +google-crc32c==1.5.0 google-oauth==1.0.1 google-pasta==0.2.0 google-reauth==0.1.1 +google-resumable-media==2.6.0 googleapis-common-protos==1.61.0 GPy==1.13.1 gpytorch==1.10 @@ -139,10 +151,10 @@ graphene==3.4.3 graphql-core==3.2.3 graphql-relay==3.2.0 greenlet==3.0.1 -grpcio==1.66.2 +grpcio==1.74.0 gsutil==5.27 gunicorn==20.1.0 -gymnasium==1.0.0 +gymnasium==1.1.1 h11==0.16.0 h2 @ file:///home/conda/feedstock_root/build_artifacts/h2_1733298745555/work h5py==3.10.0 @@ -166,6 +178,7 @@ ipykernel==6.27.1 ipython==8.12.3 ipython-genutils==0.2.0 ipywidgets==8.1.3 +isodate==0.6.1 isoduration==20.11.0 itsdangerous==2.1.2 jedi==0.19.1 @@ -190,14 +203,14 @@ jupyterlab_server==2.24.0 jupyterlab_widgets==3.0.11 keras==2.15.0 kiwisolver==1.4.5 +kombu==5.5.4 labmaze==1.0.6 lazy_loader==0.4 libclang==18.1.1 -libmambapy @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_libmambapy_1750078835/work/libmambapy +libmambapy @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_libmambapy_1753776969/work/libmambapy lightgbm==4.6.0 lightning-utilities==0.11.2 linear-operator==0.4.0 -linkify-it-py==2.0.3 llvmlite==0.42.0 locket==1.0.0 lxml==4.9.4 @@ -208,10 +221,9 @@ markdown-it-py==2.2.0 MarkupSafe==2.1.3 matplotlib==3.7.4 matplotlib-inline==0.1.6 -mdit-py-plugins==0.4.2 mdurl==0.1.2 memray==1.10.0 -menuinst @ file:///home/conda/feedstock_root/build_artifacts/menuinst_1750792275478/work +menuinst @ file:///home/conda/feedstock_root/build_artifacts/menuinst_1753546271769/work minigrid==2.3.1 mistune==0.8.4 ml-dtypes==0.3.2 @@ -240,6 +252,7 @@ netifaces==0.11.0 networkx==3.2.1 nevergrad==0.4.3.post7 ninja==1.11.1.1 +nixl==0.4.0 notebook==6.5.7 notebook_shim==0.2.3 numba==0.59.1 @@ -284,7 +297,6 @@ pettingzoo==1.24.3 pexpect==4.8.0 pickleshare==0.7.5 pillow==10.3.0 -pip-tools==7.4.1 platformdirs==3.11.0 plotly==5.23.0 pluggy==1.3.0 @@ -316,7 +328,6 @@ pynvml==11.5.0 PyOpenGL==3.1.7 pyOpenSSL==25.0.0 pyparsing==3.1.1 -pyproject_hooks==1.2.0 pyro-api==0.1.2 pyro-ppl==1.9.1 Pyro4==4.82 @@ -334,11 +345,10 @@ pyu2f==0.1.5 PyYAML==6.0.1 pyzmq==26.0.3 qpd==0.4.4 -ray @ file:///home/ray/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl#sha256=5a7ca35580e97891618705a7a7efcc556b30cc1d3e6155605d690b18d50c4383 -redis==4.4.2 +ray @ file:///home/ray/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl#sha256=6dd296fe192d3c9953867ef02ab2645e2f5e48f49e7d4e7c1fecc7689d139cf1 referencing==0.36.2 regex==2024.5.15 -requests==2.32.3 +requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1733217035951/work requests-oauthlib==2.0.0 requests-toolbelt==1.0.0 responses==0.13.4 @@ -396,9 +406,8 @@ tensorflow-metadata==1.13.1 tensorflow-probability==0.23.0 termcolor==2.4.0 terminado==0.18.1 -textual==4.0.0 threadpoolctl==3.1.0 -tifffile==2024.8.30 +tifffile==2024.7.21 timm==0.9.2 tinycss2==1.3.0 tinyscaler==1.2.8 @@ -423,18 +432,17 @@ transformers==4.36.2 triad==0.9.8 triton==2.3.0 typeguard==2.13.3 -typer==0.16.0 +typer==0.12.3 types-python-dateutil==2.9.0.20240316 -typing-inspection==0.4.1 typing_extensions==4.12.2 tzdata==2025.2 -uc-micro-py==1.0.3 uri-template==1.3.0 uritemplate==4.1.1 urllib3==1.26.19 utilsforecast==0.2.0 uvicorn==0.22.0 uvloop==0.21.0 +vine==5.1.0 virtualenv==20.29.1 wandb==0.17.0 watchfiles==0.19.0 diff --git a/doc/source/ray-overview/pip_freeze_ray-py39-cpu.txt b/doc/source/ray-overview/pip_freeze_ray-py39-cpu.txt index e2ca596c5740..da45ccee1874 100644 --- a/doc/source/ray-overview/pip_freeze_ray-py39-cpu.txt +++ b/doc/source/ray-overview/pip_freeze_ray-py39-cpu.txt @@ -2,24 +2,33 @@ aiohappyeyeballs==2.6.1 aiohttp==3.11.16 aiohttp-cors==0.7.0 aiosignal==1.3.1 +amqp==5.3.1 annotated-types==0.6.0 anyio==3.7.1 archspec @ file:///home/conda/feedstock_root/build_artifacts/archspec_1737352602016/work async-timeout==4.0.3 attrs==25.1.0 +azure-common==1.1.28 +azure-core==1.29.5 +azure-storage-blob==12.22.0 +billiard==4.2.1 boltons @ file:///home/conda/feedstock_root/build_artifacts/boltons_1733827268945/work boto3==1.26.76 botocore==1.29.76 Brotli @ file:///home/conda/feedstock_root/build_artifacts/brotli-split_1749229842835/work cachetools==5.5.2 +celery==5.5.3 certifi==2025.1.31 cffi==1.16.0 charset-normalizer==3.3.2 click==8.1.7 +click-didyoumean==0.3.1 +click-plugins==1.1.1.2 +click-repl==0.3.0 cloudpickle==2.2.0 colorama @ file:///home/conda/feedstock_root/build_artifacts/colorama_1733218098505/work colorful==0.5.5 -conda @ file:///home/conda/feedstock_root/build_artifacts/conda_1749201703459/work/conda-src +conda @ file:///home/conda/feedstock_root/build_artifacts/conda_1754405245494/work/conda-src conda-libmamba-solver @ file:///home/conda/feedstock_root/build_artifacts/conda-libmamba-solver_1745834476052/work/src conda-package-handling @ file:///home/conda/feedstock_root/build_artifacts/conda-package-handling_1736345463896/work conda_package_streaming @ file:///home/conda/feedstock_root/build_artifacts/conda-package-streaming_1729004031731/work @@ -42,10 +51,14 @@ google-api-core==2.24.2 google-api-python-client==2.111.0 google-auth==2.23.4 google-auth-httplib2==0.1.1 +google-cloud-core==2.4.1 +google-cloud-storage==2.14.0 +google-crc32c==1.5.0 google-oauth==1.0.1 +google-resumable-media==2.6.0 googleapis-common-protos==1.61.0 -grpcio==1.66.2 -gymnasium==1.0.0 +grpcio==1.74.0 +gymnasium==1.1.1 h11==0.16.0 h2 @ file:///home/conda/feedstock_root/build_artifacts/h2_1733298745555/work hpack @ file:///home/conda/feedstock_root/build_artifacts/hpack_1733299205993/work @@ -54,19 +67,21 @@ httptools==0.6.4 hyperframe @ file:///home/conda/feedstock_root/build_artifacts/hyperframe_1733298771451/work idna==3.7 importlib-metadata==6.11.0 +isodate==0.6.1 Jinja2==3.1.6 jmespath==1.0.1 jsonpatch @ file:///home/conda/feedstock_root/build_artifacts/jsonpatch_1733814567314/work jsonpointer @ file:///home/conda/feedstock_root/build_artifacts/jsonpointer_1725302957584/work jsonschema==4.23.0 jsonschema-specifications==2024.10.1 -libmambapy @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_libmambapy_1750078835/work/libmambapy +kombu==5.5.4 +libmambapy @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_libmambapy_1753776969/work/libmambapy lz4==4.3.3 markdown-it-py==2.2.0 MarkupSafe==2.1.3 mdurl==0.1.2 memray==1.10.0 -menuinst @ file:///home/conda/feedstock_root/build_artifacts/menuinst_1750792275478/work +menuinst @ file:///home/conda/feedstock_root/build_artifacts/menuinst_1753546271769/work msgpack==1.0.7 multidict==6.0.5 numpy==1.26.4 @@ -83,6 +98,7 @@ pandas==1.5.3 platformdirs==3.11.0 pluggy @ file:///home/conda/feedstock_root/build_artifacts/pluggy_1733222765875/work prometheus-client==0.19.0 +prompt-toolkit==3.0.41 propcache==0.3.0 proto-plus==1.22.3 protobuf==4.25.8 @@ -103,14 +119,13 @@ python-dateutil==2.8.2 python-dotenv==1.1.1 pytz==2022.7.1 PyYAML==6.0.1 -ray @ file:///home/ray/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl#sha256=6d5b553cf00ee5e32ad8f0e02333a0c4ecafd471fa52f4ee1e71ed00944120ec -redis==4.4.2 +ray @ file:///home/ray/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl#sha256=f882349a99e35a6628b064cfcb031919f026933731262d982bbdf7664003fbd3 referencing==0.36.2 requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1733217035951/work rich==13.3.2 rpds-py==0.22.3 rsa==4.7.2 -ruamel.yaml @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml_1749479929034/work +ruamel.yaml @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml_1755625023823/work ruamel.yaml.clib @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml.clib_1728724456970/work s3transfer==0.6.2 scipy==1.11.4 @@ -121,12 +136,15 @@ starlette==0.46.2 tensorboardX==2.6.2.2 tqdm @ file:///home/conda/feedstock_root/build_artifacts/tqdm_1735661334605/work typing_extensions==4.12.2 +tzdata==2025.2 uritemplate==4.1.1 urllib3==1.26.19 uvicorn==0.22.0 uvloop==0.21.0 +vine==5.1.0 virtualenv==20.29.1 watchfiles==0.19.0 +wcwidth==0.2.13 websockets==11.0.3 yarl==1.18.3 zipp==3.19.2 From f7089b947a7c1d51ffd77a2734261e81a1c35a1b Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Tue, 26 Aug 2025 15:07:57 -0700 Subject: [PATCH 292/634] [core][obsclean/05] add an interface for metric (#55911) Add an interface for Metric class. This interface will allow us to move metric from being statically initialized to be a class members, etc, both reduce build time + avoid static destruction issues. - I add the `MetricInterface` under the `observability` namespace; all metrics, events and logs will be migrated to this new namespace, so any new file I also add to this new namespace - Fix `tag_defs.h` which currently doesn't have namespace and is included in a weird way to have namespace Test: - CI Signed-off-by: Cuong Nguyen --- src/ray/observability/BUILD.bazel | 8 ++++ src/ray/observability/metric_interface.h | 49 ++++++++++++++++++++++++ src/ray/stats/BUILD.bazel | 15 +++++++- src/ray/stats/metric.cc | 5 ++- src/ray/stats/metric.h | 17 ++++---- src/ray/stats/tag_defs.cc | 2 +- src/ray/stats/tag_defs.h | 9 ++++- 7 files changed, 90 insertions(+), 15 deletions(-) create mode 100644 src/ray/observability/metric_interface.h diff --git a/src/ray/observability/BUILD.bazel b/src/ray/observability/BUILD.bazel index a55197186043..623bd3ab6a53 100644 --- a/src/ray/observability/BUILD.bazel +++ b/src/ray/observability/BUILD.bazel @@ -16,3 +16,11 @@ ray_cc_library( "@io_opentelemetry_cpp//sdk/src/metrics", ], ) + +ray_cc_library( + name = "metric_interface", + hdrs = ["metric_interface.h"], + deps = [ + "@io_opencensus_cpp//opencensus/stats", + ], +) diff --git a/src/ray/observability/metric_interface.h b/src/ray/observability/metric_interface.h new file mode 100644 index 000000000000..4ba235e0e0b7 --- /dev/null +++ b/src/ray/observability/metric_interface.h @@ -0,0 +1,49 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "opencensus/tags/tag_key.h" + +namespace ray { + +// TODO(can-anyscale): Use stats namespace for backward compatibility. We will remove +// these types soon when opencensus is removed, and then we can remove this namespace. +namespace stats { + +using TagKeyType = opencensus::tags::TagKey; +using TagsType = std::vector>; + +} // namespace stats + +namespace observability { + +class MetricInterface { + public: + virtual ~MetricInterface() = default; + + virtual void Record(double value) = 0; + virtual void Record(double value, stats::TagsType tags) = 0; + virtual void Record(double value, + const std::unordered_map &tags) = 0; + virtual void Record(double value, + const std::unordered_map &tags) = 0; +}; + +} // namespace observability +} // namespace ray diff --git a/src/ray/stats/BUILD.bazel b/src/ray/stats/BUILD.bazel index cdabcc11f1d7..2be6459bb0fd 100644 --- a/src/ray/stats/BUILD.bazel +++ b/src/ray/stats/BUILD.bazel @@ -10,10 +10,11 @@ ray_cc_library( hdrs = [ "metric.h", "metric_defs.h", - "tag_defs.h", ], deps = [ + ":tag_defs", "//src/ray/common:ray_config", + "//src/ray/observability:metric_interface", "//src/ray/observability:open_telemetry_metric_recorder", "//src/ray/util:logging", "//src/ray/util:size_literals", @@ -37,7 +38,6 @@ ray_cc_library( "metric.h", "metric_exporter.h", "stats.h", - "tag_defs.h", ], linkopts = select({ "@platforms//os:windows": [ @@ -48,9 +48,20 @@ ray_cc_library( }), deps = [ ":stats_metric", + ":tag_defs", + "//src/ray/observability:metric_interface", "//src/ray/rpc:metrics_agent_client", "//src/ray/util:network_util", "//src/ray/util:size_literals", "@com_github_grpc_grpc//:grpc_opencensus_plugin", ], ) + +ray_cc_library( + name = "tag_defs", + srcs = ["tag_defs.cc"], + hdrs = ["tag_defs.h"], + deps = [ + "//src/ray/observability:metric_interface", + ], +) diff --git a/src/ray/stats/metric.cc b/src/ray/stats/metric.cc index 65e7032226bc..4903352ab0d2 100644 --- a/src/ray/stats/metric.cc +++ b/src/ray/stats/metric.cc @@ -161,7 +161,7 @@ void Metric::Record(double value, TagsType tags) { } void Metric::Record(double value, - std::unordered_map tags) { + const std::unordered_map &tags) { TagsType tags_pair_vec; tags_pair_vec.reserve(tags.size()); std::for_each(tags.begin(), tags.end(), [&tags_pair_vec](auto &tag) { @@ -171,7 +171,8 @@ void Metric::Record(double value, Record(value, std::move(tags_pair_vec)); } -void Metric::Record(double value, std::unordered_map tags) { +void Metric::Record(double value, + const std::unordered_map &tags) { TagsType tags_pair_vec; tags_pair_vec.reserve(tags.size()); std::for_each(tags.begin(), tags.end(), [&tags_pair_vec](auto &tag) { diff --git a/src/ray/stats/metric.h b/src/ray/stats/metric.h index 8ac54d521f31..cb1d3d83702a 100644 --- a/src/ray/stats/metric.h +++ b/src/ray/stats/metric.h @@ -29,16 +29,15 @@ #include "opencensus/stats/stats_exporter.h" #include "opencensus/tags/tag_key.h" #include "ray/common/ray_config.h" +#include "ray/observability/metric_interface.h" #include "ray/observability/open_telemetry_metric_recorder.h" +#include "ray/stats/tag_defs.h" #include "ray/util/logging.h" namespace ray { namespace stats { -/// Include tag_defs.h to define tag items -#include "ray/stats/tag_defs.h" - using OpenTelemetryMetricRecorder = ray::observability::OpenTelemetryMetricRecorder; /// StatsConfig per process. @@ -107,7 +106,7 @@ class StatsConfig final { }; /// A thin wrapper that wraps the `opencensus::tag::measure` for using it simply. -class Metric { +class Metric : public observability::MetricInterface { public: Metric(const std::string &name, std::string description, @@ -124,20 +123,22 @@ class Metric { const std::string &GetName() const { return name_; } /// Record the value for this metric. - void Record(double value) { Record(value, TagsType{}); } + void Record(double value) override { Record(value, TagsType{}); } /// Record the value for this metric. /// /// \param value The value that we record. /// \param tags The tag values that we want to record for this metric record. - void Record(double value, TagsType tags); + void Record(double value, TagsType tags) override; /// Record the value for this metric. /// /// \param value The value that we record. /// \param tags The map tag values that we want to record for this metric record. - void Record(double value, std::unordered_map tags); - void Record(double value, std::unordered_map tags); + void Record(double value, + const std::unordered_map &tags) override; + void Record(double value, + const std::unordered_map &tags) override; protected: virtual void RegisterView() = 0; diff --git a/src/ray/stats/tag_defs.cc b/src/ray/stats/tag_defs.cc index 527a06007c87..600d5f1ef95b 100644 --- a/src/ray/stats/tag_defs.cc +++ b/src/ray/stats/tag_defs.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/stats/metric.h" +#include "ray/stats/tag_defs.h" namespace ray { namespace stats { diff --git a/src/ray/stats/tag_defs.h b/src/ray/stats/tag_defs.h index 38c2df149dfc..47d197f71161 100644 --- a/src/ray/stats/tag_defs.h +++ b/src/ray/stats/tag_defs.h @@ -14,11 +14,13 @@ #pragma once +#include "ray/observability/metric_interface.h" + /// The definitions of tag keys that you can use every where. /// You can follow these examples to define and register your tag keys. -using TagKeyType = opencensus::tags::TagKey; -using TagsType = std::vector>; +namespace ray { +namespace stats { extern const TagKeyType ComponentKey; @@ -66,3 +68,6 @@ constexpr char kObjectUnsealed[] = "UNSEALED"; // GCS task manager tags constexpr char kGcsTaskStatusEventDropped[] = "STATUS_EVENT"; constexpr char kGcsProfileEventDropped[] = "PROFILE_EVENT"; + +} // namespace stats +} // namespace ray From 4f95dfb29b06a9a4baf2896aee96200588d2c9ae Mon Sep 17 00:00:00 2001 From: Timothy Seah Date: Tue, 26 Aug 2025 16:44:16 -0700 Subject: [PATCH 293/634] [train] ThreadRunner captures exceptions from nested threads (#55756) The `ThreadRunner` is an abstraction used by Ray Train to capture errors raised by the training function so they can be polled by the Ray Train controller. This PR extends the `ThreadRunner` to also capture errors raised by threads created by the training function e.g. async checkpoint upload threads (https://github.com/ray-project/ray/pull/55637). --------- Signed-off-by: Timothy Seah --- .../execution/worker_group/thread_runner.py | 50 +++++++++++-------- python/ray/train/v2/_internal/util.py | 28 +++++++++++ .../ray/train/v2/tests/test_thread_runner.py | 48 +++++++++++++++++- 3 files changed, 104 insertions(+), 22 deletions(-) diff --git a/python/ray/train/v2/_internal/execution/worker_group/thread_runner.py b/python/ray/train/v2/_internal/execution/worker_group/thread_runner.py index ef19e66583d4..460b2f59c18a 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/thread_runner.py +++ b/python/ray/train/v2/_internal/execution/worker_group/thread_runner.py @@ -1,10 +1,13 @@ import logging +import queue import threading -import traceback from typing import Callable, Optional, TypeVar from ray.train.v2._internal.exceptions import UserExceptionWithTraceback -from ray.train.v2._internal.util import get_callable_name +from ray.train.v2._internal.util import ( + construct_user_exception_with_traceback, + get_callable_name, +) T = TypeVar("T") @@ -21,7 +24,9 @@ def __init__(self): self._exc: Optional[UserExceptionWithTraceback] = None self._thread: Optional[threading.Thread] = None + self._monitor_thread: Optional[threading.Thread] = None self._lock = threading.Lock() + self._exc_queue: queue.SimpleQueue[Optional[Exception]] = queue.SimpleQueue() self._is_running = False @@ -37,19 +42,13 @@ def _run_target(): result = target() with self._lock: self._ret = result + self._exc_queue.put(None) except BaseException as e: - with self._lock: - # Exclude the first 2 frames from the traceback, which are - # the `ThreadRunner._run_target` and `construct_train_func` calls. - # TODO(justinvyu): This is brittle and may break if the call stack - # changes. Figure out a more robust way to exclude these frames. - exc_traceback_str = traceback.format_exc( - limit=-(len(traceback.extract_tb(e.__traceback__)) - 2) - ) - logger.error(f"Error in training function:\n{exc_traceback_str}") - self._exc = UserExceptionWithTraceback( - e, traceback_str=exc_traceback_str - ) + # Exclude the first 2 frames from the traceback, which are + # the `ThreadRunner._run_target` and `construct_train_func` calls. + self._exc_queue.put( + construct_user_exception_with_traceback(e, exclude_frames=2) + ) with self._lock: self._is_running = False @@ -61,7 +60,20 @@ def _run_target(): ) self._thread.start() + def _monitor_target(): + exc = self._exc_queue.get() + with self._lock: + self._exc = exc + + self._monitor_thread = threading.Thread( + target=_monitor_target, + daemon=True, + name=f"MonitoringThread({get_callable_name(target)})", + ) + self._monitor_thread.start() + def is_running(self) -> bool: + """Returns whether the target function is still running.""" with self._lock: return self._is_running @@ -73,10 +85,6 @@ def get_return_value(self) -> Optional[T]: with self._lock: return self._ret - def join(self, timeout: Optional[float] = None) -> T: - if self._thread is None: - raise RuntimeError("Must call `run` before trying to `join`.") - - self._thread.join(timeout=timeout) - - return self.get_return_value() + def get_exception_queue(self) -> queue.SimpleQueue: + """Returns a queue that nested threads can add exceptions to.""" + return self._exc_queue diff --git a/python/ray/train/v2/_internal/util.py b/python/ray/train/v2/_internal/util.py index 8ce512af8d90..4f8a7c32732a 100644 --- a/python/ray/train/v2/_internal/util.py +++ b/python/ray/train/v2/_internal/util.py @@ -1,6 +1,8 @@ import contextlib import functools +import logging import time +import traceback from datetime import datetime from typing import ( Any, @@ -17,8 +19,12 @@ import ray from ray.train._internal.utils import count_required_parameters +from ray.train.v2._internal.exceptions import UserExceptionWithTraceback from ray.types import ObjectRef +logger = logging.getLogger(__name__) + + T = TypeVar("T") @@ -210,3 +216,25 @@ def get_callable_name(fn: Callable) -> str: # Fallback to the class name for objects that implement __call__ return fn.__class__.__name__ + + +def construct_user_exception_with_traceback( + e: BaseException, exclude_frames: int = 0 +) -> UserExceptionWithTraceback: + """Construct a UserExceptionWithTraceback from a base exception. + + Args: + e: The base exception to construct a UserExceptionWithTraceback from. + exclude_frames: The number of frames to exclude from the beginnning of + the traceback. + + Returns: + A UserExceptionWithTraceback object. + """ + # TODO(justinvyu): This is brittle and may break if the call stack + # changes. Figure out a more robust way to exclude these frames. + exc_traceback_str = traceback.format_exc( + limit=-(len(traceback.extract_tb(e.__traceback__)) - exclude_frames) + ) + logger.error(f"Error in training function:\n{exc_traceback_str}") + return UserExceptionWithTraceback(e, traceback_str=exc_traceback_str) diff --git a/python/ray/train/v2/tests/test_thread_runner.py b/python/ray/train/v2/tests/test_thread_runner.py index 788150a8af14..9d0c7f3d730b 100644 --- a/python/ray/train/v2/tests/test_thread_runner.py +++ b/python/ray/train/v2/tests/test_thread_runner.py @@ -1,14 +1,32 @@ +import threading import time import pytest from ray.train.v2._internal.exceptions import UserExceptionWithTraceback from ray.train.v2._internal.execution.worker_group.thread_runner import ThreadRunner +from ray.train.v2._internal.util import construct_user_exception_with_traceback + + +class ThreadRunnerWithJoin(ThreadRunner): + def join(self): + """Join both the target thread and the monitor thread. + + Do not include this with the main ThreadRunner class because: + * It is tricky to avoid hangs when nested threads raise errors + * We don't need to join in that case since the controller will see the + error and shut down the worker + """ + if self._monitor_thread is None or self._thread is None: + raise RuntimeError("Must call `run` before trying to `join`.") + self._monitor_thread.join() + self._thread.join() + return self.get_return_value() @pytest.fixture() def thread_runner(): - return ThreadRunner() + return ThreadRunnerWithJoin() def test_successful_return(thread_runner): @@ -48,6 +66,34 @@ def nested(): assert "_run_target" not in error._traceback_str +def test_nested_thread_error(thread_runner): + """Checks that we capture exceptions from threads kicked off by target function.""" + + def target(): + def nested(): + try: + raise ValueError + except ValueError as e: + thread_runner.get_exception_queue().put( + construct_user_exception_with_traceback(e) + ) + + thread = threading.Thread(target=nested) + thread.start() + thread.join() + + thread_runner.run(target) + assert not thread_runner.join() + + assert thread_runner.get_return_value() is None + assert not thread_runner.is_running() + + error = thread_runner.get_error() + + assert isinstance(error, UserExceptionWithTraceback) + assert isinstance(error._base_exc, ValueError) + + def test_running(thread_runner, tmp_path): """Checks that the running status can be queried.""" From 7eb706f55c68f26834633978553fb82c7bbd3db1 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Tue, 26 Aug 2025 17:38:32 -0700 Subject: [PATCH 294/634] [core][otel] remove RAY_enable_open_telemetry from release tests (#55974) Remove the `RAY_enable_open_telemetry` environment variable from all release tests. This feature has been turned on staging (where the release tests run) by default so we no longer need this flag. Test: - CI Signed-off-by: Cuong Nguyen --- release/release_tests.yaml | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/release/release_tests.yaml b/release/release_tests.yaml index bfdb10067dd8..2c84db0f9c98 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -2856,9 +2856,7 @@ team: core cluster: - byod: - runtime_env: - - RAY_enable_open_telemetry=1 + byod: {} cluster_compute: stress_tests/placement_group_tests_compute.yaml run: @@ -2930,9 +2928,7 @@ working_dir: microbenchmark cluster: - byod: - runtime_env: - - RAY_enable_open_telemetry=1 + byod: {} cluster_compute: tpl_64.yaml run: @@ -3211,9 +3207,7 @@ frequency: nightly team: core cluster: - byod: - runtime_env: - - RAY_enable_open_telemetry=1 + byod: {} cluster_compute: stress_tests/stress_tests_compute.yaml run: @@ -3253,9 +3247,7 @@ frequency: nightly team: core cluster: - byod: - runtime_env: - - RAY_enable_open_telemetry=1 + byod: {} cluster_compute: stress_tests/stress_tests_compute.yaml run: @@ -3387,9 +3379,7 @@ team: core env: aws_perf cluster: - byod: - runtime_env: - - RAY_enable_open_telemetry=1 + byod: {} cluster_compute: stress_tests/stress_tests_single_node_oom_compute.yaml run: @@ -3553,7 +3543,6 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_enable_open_telemetry=1 cluster_compute: single_node.yaml run: @@ -3581,7 +3570,6 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_enable_open_telemetry=1 cluster_compute: object_store.yaml run: @@ -3610,7 +3598,6 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_enable_open_telemetry=1 cluster_compute: object_store/small_objects.yaml run: @@ -3634,7 +3621,6 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_enable_open_telemetry=1 cluster_compute: object_store/large_objects.yaml run: @@ -3658,7 +3644,6 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_enable_open_telemetry=1 cluster_compute: distributed.yaml run: @@ -3707,7 +3692,6 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_enable_open_telemetry=1 cluster_compute: distributed.yaml run: @@ -3736,7 +3720,6 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_enable_open_telemetry=1 cluster_compute: distributed.yaml run: @@ -3786,7 +3769,6 @@ type: gpu runtime_env: - LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so - - RAY_enable_open_telemetry=1 cluster_compute: many_nodes.yaml run: From 2b92c28c072043d189c39b512cdc28194dc656f5 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Tue, 26 Aug 2025 17:49:07 -0700 Subject: [PATCH 295/634] [deps] updating byod requirements (#55977) updating byod requirements (gynamisum 1.0.0 -> 1.1.1) Signed-off-by: elliot-barn --- release/ray_release/byod/requirements_byod_3.9.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release/ray_release/byod/requirements_byod_3.9.txt b/release/ray_release/byod/requirements_byod_3.9.txt index 3613efcea7f5..d74b06abc2d7 100644 --- a/release/ray_release/byod/requirements_byod_3.9.txt +++ b/release/ray_release/byod/requirements_byod_3.9.txt @@ -1263,9 +1263,9 @@ gsutil==5.27 \ # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_byod_3.9.in -gymnasium[atari]==1.0.0 \ - --hash=sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403 \ - --hash=sha256:b6f40e1e24c5bd419361e1a5b86a9117d2499baecc3a660d44dfff4c465393ad +gymnasium[atari]==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_byod_3.9.in From 36eb2c547a27ebeecdbb078752ab6fc2432911a9 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Tue, 26 Aug 2025 17:56:16 -0700 Subject: [PATCH 296/634] [ci] creating new buildkite group for dependency testing (#55755) creating new buildkite group for dependency compilation testing renaming compile job --------- Signed-off-by: elliot-barn --- .buildkite/dependencies.rayci.yml | 28 ++++++++++++++++++++++++++++ .buildkite/others.rayci.yml | 25 ------------------------- 2 files changed, 28 insertions(+), 25 deletions(-) create mode 100644 .buildkite/dependencies.rayci.yml diff --git a/.buildkite/dependencies.rayci.yml b/.buildkite/dependencies.rayci.yml new file mode 100644 index 000000000000..97f96c3e752b --- /dev/null +++ b/.buildkite/dependencies.rayci.yml @@ -0,0 +1,28 @@ +group: dependencies +depends_on: + - forge +steps: + # dependencies + - label: ":tapioca: build: pip-compile dependencies" + key: pip_compile_dependencies + tags: always + instance_type: small + commands: + # uncomment the following line to update the pinned versions of pip dependencies + # to the latest versions; otherwise, the pinned versions will be re-used as much + # as possible + # - rm ./python/requirements_compiled.txt + - cp ./python/requirements_compiled.txt requirements_compiled_backup.txt + - ./ci/ci.sh compile_pip_dependencies + - cp -f ./python/requirements_compiled.txt /artifact-mount/ + - diff ./python/requirements_compiled.txt requirements_compiled_backup.txt || (echo "requirements_compiled.txt is not up to date. Please download it from Artifacts tab and git push the changes." && exit 1) + job_env: oss-ci-base_test-py3.11 + depends_on: oss-ci-base_test-multipy + + - label: ":tapioca: build: raydepsets: compile LLM dependencies" + key: raydepsets_compile_llm_dependencies + tags: always + instance_type: small + command: ./ci/test_compile_llm_requirements.sh + job_env: oss-ci-base_test-py3.11 + depends_on: oss-ci-base_test-multipy diff --git a/.buildkite/others.rayci.yml b/.buildkite/others.rayci.yml index 3f4551474e30..08fbf7944b8b 100644 --- a/.buildkite/others.rayci.yml +++ b/.buildkite/others.rayci.yml @@ -2,31 +2,6 @@ group: others depends_on: - forge steps: - # dependencies - - label: ":tapioca: build: pip-compile dependencies" - key: pip_compile_dependencies - tags: always - instance_type: small - commands: - # uncomment the following line to update the pinned versions of pip dependencies - # to the latest versions; otherwise, the pinned versions will be re-used as much - # as possible - # - rm ./python/requirements_compiled.txt - - cp ./python/requirements_compiled.txt requirements_compiled_backup.txt - - ./ci/ci.sh compile_pip_dependencies - - cp -f ./python/requirements_compiled.txt /artifact-mount/ - - diff ./python/requirements_compiled.txt requirements_compiled_backup.txt || (echo "requirements_compiled.txt is not up to date. Please download it from Artifacts tab and git push the changes." && exit 1) - job_env: oss-ci-base_test-py3.11 - depends_on: oss-ci-base_test-multipy - - - label: ":tapioca: build: uv pip compile LLM dependencies" - key: uv_pip_compile_llm_dependencies - tags: always - instance_type: small - command: ./ci/test_compile_llm_requirements.sh - job_env: oss-ci-base_test-py3.11 - depends_on: oss-ci-base_test-multipy - # docs - name: doctestbuild wanda: ci/docker/doctest.build.wanda.yaml From 3cea1a437564640fccae5a11d6cedb0021d2801d Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Tue, 26 Aug 2025 19:54:56 -0700 Subject: [PATCH 297/634] [core] Cleanup dead grpc server code (#55965) Signed-off-by: dayshah --- BUILD.bazel | 14 +------------- python/ray/includes/ray_config.pxd | 8 -------- python/ray/includes/ray_config.pxi | 16 ---------------- src/ray/common/ray_config_def.h | 6 ------ src/ray/gcs/gcs_server/gcs_server.cc | 1 - src/ray/object_manager/object_manager.cc | 3 +-- src/ray/rpc/grpc_server.h | 2 -- src/ray/rpc/server_call.h | 14 ++------------ 8 files changed, 4 insertions(+), 60 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index ae56691ed2f7..b0a4b3bb17cb 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -116,20 +116,8 @@ alias( # https://github.com/hedronvision/bazel-compile-commands-extractor?tab=readme-ov-file#vscode - directions for clangd config refresh_compile_commands( name = "refresh_compile_commands", - exclude_external_sources = True, # removed below to have lsp index external cc files at the cost of 2x index time - # Specify the targets of interest. - # For example, specify a dict of targets and any flags required to build. targets = { - "//:ray_pkg_zip": "", - }, - # No need to add flags already in .bazelrc. They're automatically picked up. -) - -# bazel run :refresh_compile_commands_external_sources for generation with external source files (cc files) -refresh_compile_commands( - name = "refresh_compile_commands_external_sources", - targets = { - "//:ray_pkg_zip": "", + "//:ray_pkg": "", }, ) diff --git a/python/ray/includes/ray_config.pxd b/python/ray/includes/ray_config.pxd index 9459cbbef77b..01e9827b98f9 100644 --- a/python/ray/includes/ray_config.pxd +++ b/python/ray/includes/ray_config.pxd @@ -69,12 +69,6 @@ cdef extern from "ray/common/ray_config.h" nogil: int64_t health_check_failure_threshold() const - uint64_t memory_monitor_refresh_ms() const - - int64_t grpc_keepalive_time_ms() const - - int64_t grpc_keepalive_timeout_ms() const - int64_t grpc_client_keepalive_time_ms() const int64_t grpc_client_keepalive_timeout_ms() const @@ -89,8 +83,6 @@ cdef extern from "ray/common/ray_config.h" nogil: int64_t py_gcs_connect_timeout_s() const - int gcs_rpc_server_reconnect_timeout_s() const - int maximum_gcs_destroyed_actor_cached_count() const c_bool record_task_actor_creation_sites() const diff --git a/python/ray/includes/ray_config.pxi b/python/ray/includes/ray_config.pxi index d1506678bae4..26236f2cce2c 100644 --- a/python/ray/includes/ray_config.pxi +++ b/python/ray/includes/ray_config.pxi @@ -117,18 +117,6 @@ cdef class Config: def health_check_failure_threshold(): return RayConfig.instance().health_check_failure_threshold() - @staticmethod - def memory_monitor_refresh_ms(): - return (RayConfig.instance().memory_monitor_refresh_ms()) - - @staticmethod - def grpc_keepalive_time_ms(): - return RayConfig.instance().grpc_keepalive_time_ms() - - @staticmethod - def grpc_keepalive_timeout_ms(): - return RayConfig.instance().grpc_keepalive_timeout_ms() - @staticmethod def grpc_client_keepalive_time_ms(): return RayConfig.instance().grpc_client_keepalive_time_ms() @@ -149,10 +137,6 @@ cdef class Config: def py_gcs_connect_timeout_s(): return RayConfig.instance().py_gcs_connect_timeout_s() - @staticmethod - def gcs_rpc_server_reconnect_timeout_s(): - return RayConfig.instance().gcs_rpc_server_reconnect_timeout_s() - @staticmethod def maximum_gcs_destroyed_actor_cached_count(): return RayConfig.instance().maximum_gcs_destroyed_actor_cached_count() diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index 74a932c136a5..85adf608b1cc 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -750,12 +750,6 @@ RAY_CONFIG(std::string, custom_unit_instance_resources, "neuron_cores,TPU,NPU,HP /// Ray-internal auxiliary tasks (e.g., compiled graph workers). RAY_CONFIG(std::string, system_concurrency_group_name, "_ray_system") -// Maximum size of the batches when broadcasting resources to raylet. -RAY_CONFIG(uint64_t, resource_broadcast_batch_size, 512) - -// Maximum ray sync message batch size in bytes (1MB by default) between nodes. -RAY_CONFIG(uint64_t, max_sync_message_batch_bytes, 1 * 1024 * 1024) - /// ServerCall instance number of each RPC service handler /// /// NOTE: Default value is temporarily pegged at `gcs_server_rpc_server_thread_num * 100` diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index d49029f4dc93..00ce1088abe0 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -63,7 +63,6 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, rpc_server_(config.grpc_server_name, config.grpc_server_port, config.node_ip_address == "127.0.0.1", - ClusterID::Nil(), config.grpc_server_thread_num, /*keepalive_time_ms=*/RayConfig::instance().grpc_keepalive_time_ms()), client_call_manager_(main_service, diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index 53b18b32d11c..c8c3deb29311 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -108,13 +108,12 @@ ObjectManager::ObjectManager( object_manager_server_("ObjectManager", config_.object_manager_port, config_.object_manager_address == "127.0.0.1", - ClusterID::Nil(), config_.rpc_service_threads_number), client_call_manager_(main_service, /*record_stats=*/true, ClusterID::Nil(), config_.rpc_service_threads_number), - restore_spilled_object_(restore_spilled_object), + restore_spilled_object_(std::move(restore_spilled_object)), get_spilled_object_url_(std::move(get_spilled_object_url)), pull_retry_timer_(*main_service_, boost::posix_time::milliseconds(config.timer_freq_ms)), diff --git a/src/ray/rpc/grpc_server.h b/src/ray/rpc/grpc_server.h index 686c4a68b2a4..93c16457f9cb 100644 --- a/src/ray/rpc/grpc_server.h +++ b/src/ray/rpc/grpc_server.h @@ -96,13 +96,11 @@ class GrpcServer { GrpcServer(std::string name, const uint32_t port, bool listen_to_localhost_only, - const ClusterID &cluster_id = ClusterID::Nil(), int num_threads = 1, int64_t keepalive_time_ms = 7200000 /*2 hours, grpc default*/) : name_(std::move(name)), port_(port), listen_to_localhost_only_(listen_to_localhost_only), - cluster_id_(ClusterID::Nil()), is_shutdown_(true), num_threads_(num_threads), keepalive_time_ms_(keepalive_time_ms) { diff --git a/src/ray/rpc/server_call.h b/src/ray/rpc/server_call.h index a2e7cd2ceea5..698767d7e25b 100644 --- a/src/ray/rpc/server_call.h +++ b/src/ray/rpc/server_call.h @@ -109,9 +109,6 @@ class ServerCall { /// Get the state of this `ServerCall`. virtual ServerCallState GetState() const = 0; - /// Set state of this `ServerCall`. - virtual void SetState(const ServerCallState &new_state) = 0; - /// Handle the requst. This is the callback function to be called by /// `GrpcServer` when the request is received. virtual void HandleRequest() = 0; @@ -201,12 +198,8 @@ class ServerCallImpl : public ServerCall { } } - ~ServerCallImpl() override = default; - ServerCallState GetState() const override { return state_; } - void SetState(const ServerCallState &new_state) override { state_ = new_state; } - void HandleRequest() override { stats_handle_ = io_service_.stats().RecordStart(call_name_); bool auth_success = true; @@ -262,15 +255,12 @@ class ServerCallImpl : public ServerCall { } } state_ = ServerCallState::PROCESSING; - // NOTE(hchen): This `factory` local variable is needed. Because `SendReply` runs in - // a different thread, and will cause `this` to be deleted. - const auto &factory = factory_; - if (factory.GetMaxActiveRPCs() == -1) { + if (factory_.GetMaxActiveRPCs() == -1) { // Create a new `ServerCall` to accept the next incoming request. // We create this before handling the request only when no back pressure limit is // set. So that the it can be populated by the completion queue in the background if // a new request comes in. - factory.CreateCall(); + factory_.CreateCall(); } if (!auth_success) { boost::asio::post(GetServerCallExecutor(), [this]() { From 91dbbcfb37522e86e8fa24003c9bb3e0d110863b Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Tue, 26 Aug 2025 20:14:05 -0700 Subject: [PATCH 298/634] [core][proto] split runtime_env_common proto to private+public (#55913) The `RuntimeEnvInfo` and its dependencies (part of`runtime_env_common`) is a public proto because it is used inside another public proto. Move `RuntimeEnfInfo` to the `public` directory. The `RuntimeEnvState` remains private. Test: - CI Signed-off-by: Cuong Nguyen Signed-off-by: can --- ci/pipeline/determine_tests_to_run.py | 2 +- cpp/src/ray/runtime/runtime_env.cc | 2 +- java/BUILD.bazel | 5 +- .../runtime/runtimeenv/RuntimeEnvImpl.java | 14 +++--- python/ray/_private/utils.py | 2 +- .../aggregator/tests/test_aggregator_agent.py | 2 +- python/ray/runtime_env/runtime_env.py | 2 +- src/ray/protobuf/BUILD.bazel | 9 ++-- src/ray/protobuf/common.proto | 2 +- .../events_actor_task_definition_event.proto | 2 +- .../events_task_definition_event.proto | 2 +- src/ray/protobuf/node_manager.proto | 2 +- src/ray/protobuf/public/BUILD.bazel | 14 ++++-- .../events_driver_job_definition_event.proto | 2 +- .../events_driver_job_execution_event.proto | 2 - .../protobuf/public/runtime_environment.proto | 47 +++++++++++++++++++ src/ray/protobuf/runtime_env_agent.proto | 1 + src/ray/protobuf/runtime_env_common.proto | 28 ----------- src/ray/raylet/runtime_env_agent_client.h | 2 +- 19 files changed, 87 insertions(+), 55 deletions(-) create mode 100644 src/ray/protobuf/public/runtime_environment.proto diff --git a/ci/pipeline/determine_tests_to_run.py b/ci/pipeline/determine_tests_to_run.py index 61939c5adacb..e32d548668f6 100644 --- a/ci/pipeline/determine_tests_to_run.py +++ b/ci/pipeline/determine_tests_to_run.py @@ -14,7 +14,7 @@ lint python cpp core_cpp java workflow compiled_graphs dashboard ray_client data dask serve ml tune train llm rllib rllib_gpu rllib_directly linux_wheels macos_wheels docker doc python_dependencies tools - release_tests compiled_python spark_on_ray + release_tests compiled_python spark_on_ray runtime_env_container """.split() ) diff --git a/cpp/src/ray/runtime/runtime_env.cc b/cpp/src/ray/runtime/runtime_env.cc index df69dbfd36d3..437238bd2f9c 100644 --- a/cpp/src/ray/runtime/runtime_env.cc +++ b/cpp/src/ray/runtime/runtime_env.cc @@ -16,7 +16,7 @@ #include #include -#include "src/ray/protobuf/runtime_env_common.pb.h" +#include "src/ray/protobuf/public/runtime_environment.pb.h" namespace ray { diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 7f2bc61fa6d4..7833c83d8fa3 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -254,7 +254,10 @@ java_proto_compile( java_proto_compile( name = "runtime_env_common_java_proto", - deps = ["@io_ray//src/ray/protobuf:runtime_env_common_proto"], + deps = [ + "@io_ray//src/ray/protobuf:runtime_env_common_proto", + "@io_ray//src/ray/protobuf/public:runtime_environment_proto", + ], ) java_proto_compile( diff --git a/java/runtime/src/main/java/io/ray/runtime/runtimeenv/RuntimeEnvImpl.java b/java/runtime/src/main/java/io/ray/runtime/runtimeenv/RuntimeEnvImpl.java index 81c1cfde5657..b25566964b9d 100644 --- a/java/runtime/src/main/java/io/ray/runtime/runtimeenv/RuntimeEnvImpl.java +++ b/java/runtime/src/main/java/io/ray/runtime/runtimeenv/RuntimeEnvImpl.java @@ -10,7 +10,7 @@ import io.ray.api.exception.RuntimeEnvException; import io.ray.api.runtimeenv.RuntimeEnv; import io.ray.api.runtimeenv.RuntimeEnvConfig; -import io.ray.runtime.generated.RuntimeEnvCommon; +import io.ray.runtime.generated.RuntimeEnvironment; import java.io.IOException; public class RuntimeEnvImpl implements RuntimeEnv { @@ -100,7 +100,7 @@ public boolean isEmpty() { @Override public String serializeToRuntimeEnvInfo() throws RuntimeEnvException { - RuntimeEnvCommon.RuntimeEnvInfo protoRuntimeEnvInfo = GenerateRuntimeEnvInfo(); + RuntimeEnvironment.RuntimeEnvInfo protoRuntimeEnvInfo = GenerateRuntimeEnvInfo(); JsonFormat.Printer printer = JsonFormat.printer(); try { @@ -123,15 +123,15 @@ public RuntimeEnvConfig getConfig() { return get(CONFIG_FIELD_NAME, RuntimeEnvConfig.class); } - public RuntimeEnvCommon.RuntimeEnvInfo GenerateRuntimeEnvInfo() throws RuntimeEnvException { + public RuntimeEnvironment.RuntimeEnvInfo GenerateRuntimeEnvInfo() throws RuntimeEnvException { String serializeRuntimeEnv = serialize(); - RuntimeEnvCommon.RuntimeEnvInfo.Builder protoRuntimeEnvInfoBuilder = - RuntimeEnvCommon.RuntimeEnvInfo.newBuilder(); + RuntimeEnvironment.RuntimeEnvInfo.Builder protoRuntimeEnvInfoBuilder = + RuntimeEnvironment.RuntimeEnvInfo.newBuilder(); protoRuntimeEnvInfoBuilder.setSerializedRuntimeEnv(serializeRuntimeEnv); RuntimeEnvConfig runtimeEnvConfig = getConfig(); if (runtimeEnvConfig != null) { - RuntimeEnvCommon.RuntimeEnvConfig.Builder protoRuntimeEnvConfigBuilder = - RuntimeEnvCommon.RuntimeEnvConfig.newBuilder(); + RuntimeEnvironment.RuntimeEnvConfig.Builder protoRuntimeEnvConfigBuilder = + RuntimeEnvironment.RuntimeEnvConfig.newBuilder(); protoRuntimeEnvConfigBuilder.setSetupTimeoutSeconds( runtimeEnvConfig.getSetupTimeoutSeconds()); protoRuntimeEnvConfigBuilder.setEagerInstall(runtimeEnvConfig.getEagerInstall()); diff --git a/python/ray/_private/utils.py b/python/ray/_private/utils.py index 9204f46b6463..7793787e488f 100644 --- a/python/ray/_private/utils.py +++ b/python/ray/_private/utils.py @@ -35,7 +35,7 @@ get_ray_address_file, get_system_memory, ) -from ray.core.generated.runtime_env_common_pb2 import ( +from ray.core.generated.runtime_environment_pb2 import ( RuntimeEnvInfo as ProtoRuntimeEnvInfo, ) diff --git a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py index 1853029f006f..8ab41de85aeb 100644 --- a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py @@ -40,7 +40,7 @@ from ray.core.generated.events_driver_job_execution_event_pb2 import ( DriverJobExecutionEvent, ) -from ray.core.generated.runtime_env_common_pb2 import ( +from ray.core.generated.runtime_environment_pb2 import ( RuntimeEnvInfo, RuntimeEnvUris, RuntimeEnvConfig, diff --git a/python/ray/runtime_env/runtime_env.py b/python/ray/runtime_env/runtime_env.py index 0682c48539fb..c43685a921d0 100644 --- a/python/ray/runtime_env/runtime_env.py +++ b/python/ray/runtime_env/runtime_env.py @@ -17,7 +17,7 @@ OPTION_TO_NO_PATH_VALIDATION_FN, ) from ray._private.thirdparty.dacite import from_dict -from ray.core.generated.runtime_env_common_pb2 import ( +from ray.core.generated.runtime_environment_pb2 import ( RuntimeEnvConfig as ProtoRuntimeEnvConfig, ) from ray.util.annotations import PublicAPI diff --git a/src/ray/protobuf/BUILD.bazel b/src/ray/protobuf/BUILD.bazel index 01cc6a39f45e..f1c6bad93acf 100644 --- a/src/ray/protobuf/BUILD.bazel +++ b/src/ray/protobuf/BUILD.bazel @@ -13,7 +13,7 @@ proto_library( "//java:__subpackages__", ], deps = [ - ":runtime_env_common_proto", + "//src/ray/protobuf/public:runtime_environment_proto", ], ) @@ -107,7 +107,7 @@ proto_library( ":autoscaler_proto", ":common_proto", ":gcs_proto", - ":runtime_env_common_proto", + "//src/ray/protobuf/public:runtime_environment_proto", ], ) @@ -402,6 +402,7 @@ proto_library( deps = [ ":common_proto", ":runtime_env_common_proto", + "//src/ray/protobuf/public:runtime_environment_proto", ], ) @@ -425,7 +426,6 @@ proto_library( srcs = ["autoscaler.proto"], deps = [ ":common_proto", - ":runtime_env_common_proto", ], ) @@ -449,6 +449,7 @@ proto_library( deps = [ ":common_proto", ":runtime_env_common_proto", + "//src/ray/protobuf/public:runtime_environment_proto", ], ) @@ -463,6 +464,7 @@ proto_library( deps = [ ":common_proto", ":runtime_env_common_proto", + "//src/ray/protobuf/public:runtime_environment_proto", ], ) @@ -559,6 +561,7 @@ python_grpc_compile( ":runtime_env_agent_proto", ":runtime_env_common_proto", ":usage_proto", + "//src/ray/protobuf/public:runtime_environment_proto", ], ) diff --git a/src/ray/protobuf/common.proto b/src/ray/protobuf/common.proto index 3c990b2f0a53..56dc85bb7020 100644 --- a/src/ray/protobuf/common.proto +++ b/src/ray/protobuf/common.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package ray.rpc; -import "src/ray/protobuf/runtime_env_common.proto"; +import "src/ray/protobuf/public/runtime_environment.proto"; option java_package = "io.ray.runtime.generated"; diff --git a/src/ray/protobuf/events_actor_task_definition_event.proto b/src/ray/protobuf/events_actor_task_definition_event.proto index c201c1602804..12cda5260379 100644 --- a/src/ray/protobuf/events_actor_task_definition_event.proto +++ b/src/ray/protobuf/events_actor_task_definition_event.proto @@ -14,7 +14,7 @@ syntax = "proto3"; -import "src/ray/protobuf/runtime_env_common.proto"; +import "src/ray/protobuf/public/runtime_environment.proto"; import "src/ray/protobuf/common.proto"; package ray.rpc.events; diff --git a/src/ray/protobuf/events_task_definition_event.proto b/src/ray/protobuf/events_task_definition_event.proto index 2c63c7559b34..7ed83ae87938 100644 --- a/src/ray/protobuf/events_task_definition_event.proto +++ b/src/ray/protobuf/events_task_definition_event.proto @@ -14,7 +14,7 @@ syntax = "proto3"; -import "src/ray/protobuf/runtime_env_common.proto"; +import "src/ray/protobuf/public/runtime_environment.proto"; import "src/ray/protobuf/common.proto"; package ray.rpc.events; diff --git a/src/ray/protobuf/node_manager.proto b/src/ray/protobuf/node_manager.proto index 16ddfd364243..0585ad27c087 100644 --- a/src/ray/protobuf/node_manager.proto +++ b/src/ray/protobuf/node_manager.proto @@ -19,7 +19,7 @@ package ray.rpc; import "src/ray/protobuf/common.proto"; import "src/ray/protobuf/gcs.proto"; import "src/ray/protobuf/autoscaler.proto"; -import "src/ray/protobuf/runtime_env_common.proto"; +import "src/ray/protobuf/public/runtime_environment.proto"; message WorkerBacklogReport { // TaskSpec indicating the scheduling class. diff --git a/src/ray/protobuf/public/BUILD.bazel b/src/ray/protobuf/public/BUILD.bazel index 991c911f16f1..dc7712fbd8e9 100644 --- a/src/ray/protobuf/public/BUILD.bazel +++ b/src/ray/protobuf/public/BUILD.bazel @@ -7,8 +7,8 @@ proto_library( name = "events_driver_job_definition_event_proto", srcs = ["events_driver_job_definition_event.proto"], deps = [ + ":runtime_environment_proto", "//src/ray/protobuf:common_proto", - "//src/ray/protobuf:runtime_env_common_proto", "@com_google_protobuf//:timestamp_proto", ], ) @@ -22,8 +22,6 @@ proto_library( name = "events_driver_job_execution_event_proto", srcs = ["events_driver_job_execution_event.proto"], deps = [ - "//src/ray/protobuf:common_proto", - "//src/ray/protobuf:runtime_env_common_proto", "@com_google_protobuf//:timestamp_proto", ], ) @@ -32,3 +30,13 @@ cc_proto_library( name = "events_driver_job_execution_event_cc_proto", deps = [":events_driver_job_execution_event_proto"], ) + +proto_library( + name = "runtime_environment_proto", + srcs = ["runtime_environment.proto"], +) + +cc_proto_library( + name = "runtime_environment_cc_proto", + deps = [":runtime_environment_proto"], +) diff --git a/src/ray/protobuf/public/events_driver_job_definition_event.proto b/src/ray/protobuf/public/events_driver_job_definition_event.proto index 498982ee220d..a9f17714c7a1 100644 --- a/src/ray/protobuf/public/events_driver_job_definition_event.proto +++ b/src/ray/protobuf/public/events_driver_job_definition_event.proto @@ -16,7 +16,7 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; -import "src/ray/protobuf/runtime_env_common.proto"; +import "src/ray/protobuf/public/runtime_environment.proto"; package ray.rpc.events; diff --git a/src/ray/protobuf/public/events_driver_job_execution_event.proto b/src/ray/protobuf/public/events_driver_job_execution_event.proto index 73153b7b05e2..2d6c58f1c760 100644 --- a/src/ray/protobuf/public/events_driver_job_execution_event.proto +++ b/src/ray/protobuf/public/events_driver_job_execution_event.proto @@ -16,8 +16,6 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; -import "src/ray/protobuf/runtime_env_common.proto"; -import "src/ray/protobuf/common.proto"; package ray.rpc.events; diff --git a/src/ray/protobuf/public/runtime_environment.proto b/src/ray/protobuf/public/runtime_environment.proto new file mode 100644 index 000000000000..f707d888e5fd --- /dev/null +++ b/src/ray/protobuf/public/runtime_environment.proto @@ -0,0 +1,47 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ray.rpc; + +option java_package = "io.ray.runtime.generated"; + +message RuntimeEnvUris { + /// working dir uri + string working_dir_uri = 1; + /// python modules uris + repeated string py_modules_uris = 2; +} + +/// The runtime env config, include some fields that do not +/// participate in the calculation of the runtime_env hash. +message RuntimeEnvConfig { + /// The timeout of runtime env creation. + int32 setup_timeout_seconds = 1; + /// Indicates whether to install runtime env eagerly before the workers are leased. + bool eager_install = 2; + /// A list of files to stream the runtime env setup logs to. + repeated string log_files = 3; +} + +/// The runtime env information which is transferred between ray core processes. +message RuntimeEnvInfo { + /// The serialized runtime env passed from the user. + string serialized_runtime_env = 1; + /// URIs used in this runtime env. These will be used for reference counting. + RuntimeEnvUris uris = 2; + /// The serialized runtime env config passed from the user. + RuntimeEnvConfig runtime_env_config = 3; +} diff --git a/src/ray/protobuf/runtime_env_agent.proto b/src/ray/protobuf/runtime_env_agent.proto index 707b818d3279..161f844a0d77 100644 --- a/src/ray/protobuf/runtime_env_agent.proto +++ b/src/ray/protobuf/runtime_env_agent.proto @@ -17,6 +17,7 @@ syntax = "proto3"; package ray.rpc; import "src/ray/protobuf/runtime_env_common.proto"; +import "src/ray/protobuf/public/runtime_environment.proto"; enum AgentRpcStatus { // OK. diff --git a/src/ray/protobuf/runtime_env_common.proto b/src/ray/protobuf/runtime_env_common.proto index b11021ef6ab8..c7f01fd493b4 100644 --- a/src/ray/protobuf/runtime_env_common.proto +++ b/src/ray/protobuf/runtime_env_common.proto @@ -18,34 +18,6 @@ package ray.rpc; option java_package = "io.ray.runtime.generated"; -message RuntimeEnvUris { - /// working dir uri - string working_dir_uri = 1; - /// python modules uris - repeated string py_modules_uris = 2; -} - -/// The runtime env config, include some fields that do not -/// participate in the calculation of the runtime_env hash. -message RuntimeEnvConfig { - /// The timeout of runtime env creation. - int32 setup_timeout_seconds = 1; - /// Indicates whether to install runtime env eagerly before the workers are leased. - bool eager_install = 2; - /// A list of files to stream the runtime env setup logs to. - repeated string log_files = 3; -} - -/// The runtime env information which is transferred between ray core processes. -message RuntimeEnvInfo { - /// The serialized runtime env passed from the user. - string serialized_runtime_env = 1; - /// URIs used in this runtime env. These will be used for reference counting. - RuntimeEnvUris uris = 2; - /// The serialized runtime env config passed from the user. - RuntimeEnvConfig runtime_env_config = 3; -} - message RuntimeEnvState { /// The serialized runtime env. string runtime_env = 1; diff --git a/src/ray/raylet/runtime_env_agent_client.h b/src/ray/raylet/runtime_env_agent_client.h index f86b0fd3ddbf..feec543aed55 100644 --- a/src/ray/raylet/runtime_env_agent_client.h +++ b/src/ray/raylet/runtime_env_agent_client.h @@ -25,7 +25,7 @@ #include "ray/common/id.h" #include "ray/common/ray_config.h" #include "src/ray/protobuf/gcs.pb.h" -#include "src/ray/protobuf/runtime_env_common.pb.h" +#include "src/ray/protobuf/public/runtime_environment.pb.h" namespace ray { namespace raylet { From b9ecd136af303c45f122e01333260b4c60a52019 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 26 Aug 2025 20:19:49 -0700 Subject: [PATCH 299/634] [wheel] add `gen_py_proto` script (#55957) as a replacement for `install_py_proto`, for people who only wants to regenerate the protobuf python files but not the ray core bits Signed-off-by: Lonnie Liu --- BUILD.bazel | 12 ++++++++++++ bazel/gen_extract.py | 4 +++- gen_py_proto.py | 12 ++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 gen_py_proto.py diff --git a/BUILD.bazel b/BUILD.bazel index b0a4b3bb17cb..f2ce28aa7e9b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -503,6 +503,18 @@ genrule( local = 1, ) +py_binary( + name = "gen_py_proto", + srcs = ["gen_py_proto.py"], + data = [ + ":ray_py_proto_zip", + ], + visibility = ["//visibility:private"], + deps = [ + "//bazel:gen_extract", + ], +) + py_binary( name = "gen_ray_pkg", srcs = ["gen_ray_pkg.py"], diff --git a/bazel/gen_extract.py b/bazel/gen_extract.py index 1930440a4235..80402bc9f39c 100644 --- a/bazel/gen_extract.py +++ b/bazel/gen_extract.py @@ -16,7 +16,9 @@ def gen_extract( root_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY") if not root_dir: - raise ValueError("BUILD_WORKSPACE_DIRECTORY not set") + raise ValueError( + "BUILD_WORKSPACE_DIRECTORY not set; please run this script from 'bazelisk run'" + ) if sub_dir: extract_dir = os.path.join(root_dir, sub_dir) diff --git a/gen_py_proto.py b/gen_py_proto.py new file mode 100644 index 000000000000..cbc71e0a2542 --- /dev/null +++ b/gen_py_proto.py @@ -0,0 +1,12 @@ +from bazel.gen_extract import gen_extract + +if __name__ == "__main__": + gen_extract( + [ + "ray_py_proto.zip", + ], + clear_dir_first=[ + "ray/core/generated", + "ray/serve/generated", + ], + ) From 594e1d96e63362515523dc227d1d5552977e467e Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Wed, 27 Aug 2025 07:06:24 -0700 Subject: [PATCH 300/634] [serve] Cache router metrics (#55897) ## Why are these changes needed? We can improve performance by caching metrics at the router and updating the metrics async. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: akyang-anyscale --- python/ray/serve/_private/router.py | 64 +++++++++++++++++++--- python/ray/serve/tests/unit/test_router.py | 38 ++++++++++--- 2 files changed, 86 insertions(+), 16 deletions(-) diff --git a/python/ray/serve/_private/router.py b/python/ray/serve/_private/router.py index 82f90674184b..29658716439d 100644 --- a/python/ray/serve/_private/router.py +++ b/python/ray/serve/_private/router.py @@ -37,6 +37,7 @@ RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE, RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S, RAY_SERVE_HANDLE_AUTOSCALING_METRIC_RECORD_INTERVAL_S, + RAY_SERVE_METRICS_EXPORT_INTERVAL_MS, RAY_SERVE_PROXY_PREFER_LOCAL_AZ_ROUTING, SERVE_LOGGER_NAME, ) @@ -80,6 +81,7 @@ def __init__( router_requests_counter: metrics.Counter, queued_requests_gauge: metrics.Gauge, running_requests_gauge: metrics.Gauge, + event_loop: asyncio.BaseEventLoop, ): self._handle_id = handle_id self._deployment_id = deployment_id @@ -139,6 +141,14 @@ def __init__( # Track whether the metrics manager has been shutdown self._shutdown: bool = False + # If the interval is set to 0, eagerly sets all metrics. + self._cached_metrics_enabled = RAY_SERVE_METRICS_EXPORT_INTERVAL_MS != 0 + self._cached_metrics_interval_s = RAY_SERVE_METRICS_EXPORT_INTERVAL_MS / 1000 + + if self._cached_metrics_enabled: + self._cached_num_router_requests = defaultdict(int) + event_loop.create_task(self._report_cached_metrics_forever()) + @contextmanager def wrap_request_assignment(self, request_meta: RequestMetadata): max_queued_requests = ( @@ -271,30 +281,65 @@ def update_deployment_config( if self.metrics_pusher: self.metrics_pusher.stop_tasks() + def _report_cached_metrics(self): + for route, count in self._cached_num_router_requests.items(): + self.num_router_requests.inc(count, tags={"route": route}) + self._cached_num_router_requests.clear() + + self.num_queued_requests_gauge.set(self.num_queued_requests) + + self.num_running_requests_gauge.set( + sum(self.num_requests_sent_to_replicas.values()) + ) + + async def _report_cached_metrics_forever(self): + assert self._cached_metrics_interval_s > 0 + + consecutive_errors = 0 + while True: + try: + await asyncio.sleep(self._cached_metrics_interval_s) + self._report_cached_metrics() + consecutive_errors = 0 + except Exception: + logger.exception("Unexpected error reporting metrics.") + + # Exponential backoff starting at 1s and capping at 10s. + backoff_time_s = min(10, 2**consecutive_errors) + consecutive_errors += 1 + await asyncio.sleep(backoff_time_s) + def inc_num_total_requests(self, route: str): - self.num_router_requests.inc(tags={"route": route}) + if self._cached_metrics_enabled: + self._cached_num_router_requests[route] += 1 + else: + self.num_router_requests.inc(tags={"route": route}) def inc_num_queued_requests(self): self.num_queued_requests += 1 - self.num_queued_requests_gauge.set(self.num_queued_requests) + if not self._cached_metrics_enabled: + self.num_queued_requests_gauge.set(self.num_queued_requests) def dec_num_queued_requests(self): self.num_queued_requests -= 1 - self.num_queued_requests_gauge.set(self.num_queued_requests) + if not self._cached_metrics_enabled: + self.num_queued_requests_gauge.set(self.num_queued_requests) def inc_num_running_requests_for_replica(self, replica_id: ReplicaID): with self._queries_lock: self.num_requests_sent_to_replicas[replica_id] += 1 - self.num_running_requests_gauge.set( - sum(self.num_requests_sent_to_replicas.values()) - ) + if not self._cached_metrics_enabled: + self.num_running_requests_gauge.set( + sum(self.num_requests_sent_to_replicas.values()) + ) def dec_num_running_requests_for_replica(self, replica_id: ReplicaID): with self._queries_lock: self.num_requests_sent_to_replicas[replica_id] -= 1 - self.num_running_requests_gauge.set( - sum(self.num_requests_sent_to_replicas.values()) - ) + if not self._cached_metrics_enabled: + self.num_running_requests_gauge.set( + sum(self.num_requests_sent_to_replicas.values()) + ) def should_send_scaled_to_zero_optimized_push(self, curr_num_replicas: int) -> bool: return ( @@ -478,6 +523,7 @@ def __init__( ), tag_keys=("deployment", "application", "handle", "actor_id"), ), + event_loop, ) # The Router needs to stay informed about changes to the target deployment's diff --git a/python/ray/serve/tests/unit/test_router.py b/python/ray/serve/tests/unit/test_router.py index 6e8949b1e11b..5b849c965f78 100644 --- a/python/ray/serve/tests/unit/test_router.py +++ b/python/ray/serve/tests/unit/test_router.py @@ -22,7 +22,10 @@ RunningReplicaInfo, ) from ray.serve._private.config import DeploymentConfig -from ray.serve._private.constants import RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE +from ray.serve._private.constants import ( + RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE, + RAY_SERVE_METRICS_EXPORT_INTERVAL_MS, +) from ray.serve._private.replica_result import ReplicaResult from ray.serve._private.request_router import ( PendingRequest, @@ -769,7 +772,8 @@ def running_replica_info(replica_id: ReplicaID) -> RunningReplicaInfo: class TestRouterMetricsManager: - def test_num_router_requests(self): + @pytest.mark.asyncio + async def test_num_router_requests(self): tags = { "deployment": "a", "application": "b", @@ -788,15 +792,19 @@ def test_num_router_requests(self): ), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), + event_loop=asyncio.get_event_loop(), ) assert metrics_manager.num_router_requests.get_count(tags) is None n = random.randint(1, 10) for _ in range(n): metrics_manager.inc_num_total_requests(route="/alice") + + await asyncio.sleep(RAY_SERVE_METRICS_EXPORT_INTERVAL_MS * 2 / 1000) assert metrics_manager.num_router_requests.get_count(tags) == n - def test_num_queued_requests_gauge(self): + @pytest.mark.asyncio + async def test_num_queued_requests_gauge(self): tags = { "deployment": "a", "application": "b", @@ -814,18 +822,23 @@ def test_num_queued_requests_gauge(self): ), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), + event_loop=asyncio.get_event_loop(), ) assert metrics_manager.num_queued_requests_gauge.get_value(tags) == 0 n, m = random.randint(0, 10), random.randint(0, 5) for _ in range(n): metrics_manager.inc_num_queued_requests() + await asyncio.sleep(RAY_SERVE_METRICS_EXPORT_INTERVAL_MS * 2 / 1000) assert metrics_manager.num_queued_requests_gauge.get_value(tags) == n for _ in range(m): metrics_manager.dec_num_queued_requests() + + await asyncio.sleep(RAY_SERVE_METRICS_EXPORT_INTERVAL_MS * 2 / 1000) assert metrics_manager.num_queued_requests_gauge.get_value(tags) == n - m - def test_track_requests_sent_to_replicas(self): + @pytest.mark.asyncio + async def test_track_requests_sent_to_replicas(self): d_id = DeploymentID(name="a", app_name="b") metrics_manager = RouterMetricsManager( d_id, @@ -838,6 +851,7 @@ def test_track_requests_sent_to_replicas(self): ), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), + event_loop=asyncio.get_event_loop(), ) # r1: number requests -> 0, removed from list of running replicas -> prune @@ -854,6 +868,7 @@ def test_track_requests_sent_to_replicas(self): for i in range(4): for _ in range(i + 1): metrics_manager.inc_num_running_requests_for_replica(replica_ids[i]) + await asyncio.sleep(RAY_SERVE_METRICS_EXPORT_INTERVAL_MS * 2 / 1000) # All 4 replicas should have a positive number of requests for i, r in enumerate(replica_ids): @@ -875,6 +890,7 @@ def test_track_requests_sent_to_replicas(self): metrics_manager.dec_num_running_requests_for_replica(r1) for _ in range(2): metrics_manager.dec_num_running_requests_for_replica(r2) + await asyncio.sleep(RAY_SERVE_METRICS_EXPORT_INTERVAL_MS * 2 / 1000) assert metrics_manager.num_requests_sent_to_replicas[r1] == 0 assert metrics_manager.num_requests_sent_to_replicas[r2] == 0 @@ -898,6 +914,7 @@ def test_track_requests_sent_to_replicas(self): running_replica_info(r4), ] ) + await asyncio.sleep(RAY_SERVE_METRICS_EXPORT_INTERVAL_MS * 2 / 1000) # Only r1 should be pruned, the rest should still be tracked. assert r1 not in metrics_manager.num_requests_sent_to_replicas @@ -905,7 +922,8 @@ def test_track_requests_sent_to_replicas(self): assert r3 in metrics_manager.num_requests_sent_to_replicas assert r4 in metrics_manager.num_requests_sent_to_replicas - def test_should_send_scaled_to_zero_optimized_push(self): + @pytest.mark.asyncio + async def test_should_send_scaled_to_zero_optimized_push(self): metrics_manager = RouterMetricsManager( DeploymentID(name="a", app_name="b"), "random", @@ -917,6 +935,7 @@ def test_should_send_scaled_to_zero_optimized_push(self): ), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), + event_loop=asyncio.get_event_loop(), ) # Not an autoscaling deployment, should not push metrics @@ -935,10 +954,11 @@ def test_should_send_scaled_to_zero_optimized_push(self): # All 3 conditions satisfied, should push metrics assert metrics_manager.should_send_scaled_to_zero_optimized_push(0) + @pytest.mark.asyncio @patch( "ray.serve._private.router.RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE", "1" ) - def test_push_autoscaling_metrics_to_controller(self): + async def test_push_autoscaling_metrics_to_controller(self): timer = MockTimer() start = random.randint(50, 100) timer.reset(start) @@ -965,6 +985,7 @@ def test_push_autoscaling_metrics_to_controller(self): ), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), + event_loop=asyncio.get_event_loop(), ) metrics_manager._deployment_config = DeploymentConfig( autoscaling_config=AutoscalingConfig() @@ -1023,6 +1044,7 @@ async def test_memory_cleared(self): ), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), + event_loop=asyncio.get_event_loop(), ) metrics_manager.update_deployment_config( deployment_config=DeploymentConfig( @@ -1065,11 +1087,12 @@ def check_database(expected: Set[ReplicaID]): check_database, expected={r1, r2, QUEUED_REQUESTS_KEY} ) + @pytest.mark.asyncio @patch( "ray.serve._private.router.RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE", "1" ) @patch("ray.serve._private.router.MetricsPusher") - def test_update_deployment_config(self, metrics_pusher_mock): + async def test_update_deployment_config(self, metrics_pusher_mock): metrics_manager = RouterMetricsManager( DeploymentID(name="a", app_name="b"), "random", @@ -1081,6 +1104,7 @@ def test_update_deployment_config(self, metrics_pusher_mock): ), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), FakeGauge(tag_keys=("deployment", "application", "handle", "actor_id")), + event_loop=asyncio.get_event_loop(), ) # Without autoscaling config, do nothing From d7ced7a91f7ffcccca31d5bf1583c2ad9b8ac25e Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Wed, 27 Aug 2025 07:06:41 -0700 Subject: [PATCH 301/634] [serve] Add microbenchmark for throughput optimized configuration (#55900) ## Why are these changes needed? Adding a benchmark with the throughput optimized mode enabled. Also adding httpx to the release test dependencies. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: akyang-anyscale --- .../ray_release/byod/requirements_byod_3.9.in | 1 + .../byod/requirements_byod_3.9.txt | 23 ++++++++++++++ release/release_tests.yaml | 31 +++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/release/ray_release/byod/requirements_byod_3.9.in b/release/ray_release/byod/requirements_byod_3.9.in index 248863c233c5..be42c3f1f682 100644 --- a/release/ray_release/byod/requirements_byod_3.9.in +++ b/release/ray_release/byod/requirements_byod_3.9.in @@ -11,6 +11,7 @@ gcsfs==2023.5.0 gsutil gymnasium gymnasium[atari] +httpx importlib-metadata jsonschema lightgbm diff --git a/release/ray_release/byod/requirements_byod_3.9.txt b/release/ray_release/byod/requirements_byod_3.9.txt index d74b06abc2d7..91b7f97daaad 100644 --- a/release/ray_release/byod/requirements_byod_3.9.txt +++ b/release/ray_release/byod/requirements_byod_3.9.txt @@ -167,6 +167,7 @@ anyio==3.7.1 \ --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 # via # -c release/ray_release/byod/requirements_compiled.txt + # httpx # starlette argcomplete==3.3.0 \ --hash=sha256:c168c3723482c031df3c207d4ba8fa702717ccb9fc0bfe4117166c1f537b4a54 \ @@ -314,6 +315,8 @@ certifi==2025.1.31 \ # via # -c release/ray_release/byod/requirements_compiled.txt # geventhttpclient + # httpcore + # httpx # requests cffi==1.16.0 \ --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ @@ -1269,6 +1272,12 @@ gymnasium[atari]==1.1.1 \ # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_byod_3.9.in +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # -c release/ray_release/byod/requirements_compiled.txt + # httpcore h5py==3.10.0 \ --hash=sha256:012ab448590e3c4f5a8dd0f3533255bc57f80629bf7c5054cf4c87b30085063c \ --hash=sha256:212bb997a91e6a895ce5e2f365ba764debeaef5d2dca5c6fb7098d66607adf99 \ @@ -1298,6 +1307,12 @@ h5py==3.10.0 \ # via # -c release/ray_release/byod/requirements_compiled.txt # tensorflow +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via + # -c release/ray_release/byod/requirements_compiled.txt + # httpx httplib2==0.20.4 \ --hash=sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585 \ --hash=sha256:8b6a905cb1c79eefd03f8669fd993c36dc341f7c558f056cb5a33b5c2f458543 @@ -1307,12 +1322,19 @@ httplib2==0.20.4 \ # google-apitools # gsutil # oauth2client +httpx==0.27.2 \ + --hash=sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0 \ + --hash=sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2 + # via + # -c release/ray_release/byod/requirements_compiled.txt + # -r release/ray_release/byod/requirements_byod_3.9.in idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 # via # -c release/ray_release/byod/requirements_compiled.txt # anyio + # httpx # requests # yarl importlib-metadata==6.11.0 \ @@ -2723,6 +2745,7 @@ sniffio==1.3.1 \ # via # -c release/ray_release/byod/requirements_compiled.txt # anyio + # httpx starlette==0.46.2 \ --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 diff --git a/release/release_tests.yaml b/release/release_tests.yaml index 2c84db0f9c98..8cd1f1bee9e4 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -2077,6 +2077,37 @@ cluster: cluster_compute: compute_tpl_single_node_gce.yaml +- name: serve_throughput_optimized_microbenchmarks + group: Serve tests + working_dir: serve_tests + + frequency: nightly + team: serve + + cluster: + byod: + runtime_env: + - RAY_SERVE_THROUGHPUT_OPTIMIZED=1 + - RAY_SERVE_DISABLE_SHUTTING_DOWN_INGRESS_REPLICAS_FORCEFULLY=0 + cluster_compute: compute_tpl_single_node_16_cpu.yaml + cloud_id: cld_wy5a6nhazplvu32526ams61d98 + project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq + + run: + timeout: 7200 + long_running: false + script: python workloads/microbenchmarks.py --run-all + + alert: default + + variations: + - __suffix__: aws + - __suffix__: gce + env: gce + frequency: manual + cluster: + cluster_compute: compute_tpl_single_node_gce.yaml + - name: serve_resnet_benchmark group: Serve tests working_dir: serve_tests From ea8b6ae2fb3e9558b24e32737cb3e72a89132c2c Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Wed, 27 Aug 2025 11:54:15 -0500 Subject: [PATCH 302/634] [core] Split `GcsActor` class into its own file (#55997) Breaks the circular dependency between `gcs_actor_manager.h` and `gcs_actor_scheduler.h` --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 21 ++ src/ray/gcs/gcs_server/gcs_actor.cc | 143 +++++++++ src/ray/gcs/gcs_server/gcs_actor.h | 276 ++++++++++++++++++ src/ray/gcs/gcs_server/gcs_actor_manager.cc | 117 -------- src/ray/gcs/gcs_server/gcs_actor_manager.h | 245 +--------------- src/ray/gcs/gcs_server/gcs_actor_scheduler.cc | 1 - src/ray/gcs/gcs_server/gcs_actor_scheduler.h | 3 +- 7 files changed, 442 insertions(+), 364 deletions(-) create mode 100644 src/ray/gcs/gcs_server/gcs_actor.cc create mode 100644 src/ray/gcs/gcs_server/gcs_actor.h diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 7c9a8109e48f..bc0a46a00635 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -254,6 +254,26 @@ ray_cc_library( ], ) +ray_cc_library( + name = "gcs_actor", + srcs = [ + "gcs_actor.cc", + ], + hdrs = [ + "gcs_actor.h", + ], + deps = [ + "//src/ray/common:id", + "//src/ray/common:task_common", + "//src/ray/protobuf:core_worker_cc_proto", + "//src/ray/protobuf:export_event_cc_proto", + "//src/ray/protobuf:gcs_service_cc_proto", + "//src/ray/util:counter_map", + "//src/ray/util:event", + "//src/ray/util:logging", + ], +) + ray_cc_library( name = "gcs_server_lib", srcs = [ @@ -275,6 +295,7 @@ ray_cc_library( "gcs_server.h", ], deps = [ + ":gcs_actor", ":gcs_function_manager", ":gcs_health_check_manager", ":gcs_init_data", diff --git a/src/ray/gcs/gcs_server/gcs_actor.cc b/src/ray/gcs/gcs_server/gcs_actor.cc new file mode 100644 index 000000000000..0c4c44bbdc54 --- /dev/null +++ b/src/ray/gcs/gcs_server/gcs_actor.cc @@ -0,0 +1,143 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/gcs/gcs_server/gcs_actor.h" + +#include +#include + +#include "ray/util/logging.h" + +namespace ray { +namespace gcs { + +NodeID GcsActor::GetNodeID() const { + const auto &node_id_binary = actor_table_data_.address().node_id(); + if (node_id_binary.empty()) { + return NodeID::Nil(); + } + return NodeID::FromBinary(node_id_binary); +} + +void GcsActor::UpdateAddress(const rpc::Address &address) { + actor_table_data_.mutable_address()->CopyFrom(address); +} + +const rpc::Address &GcsActor::GetAddress() const { return actor_table_data_.address(); } + +WorkerID GcsActor::GetWorkerID() const { + const auto &address = actor_table_data_.address(); + if (address.worker_id().empty()) { + return WorkerID::Nil(); + } + return WorkerID::FromBinary(address.worker_id()); +} + +WorkerID GcsActor::GetOwnerID() const { + return WorkerID::FromBinary(GetOwnerAddress().worker_id()); +} + +NodeID GcsActor::GetOwnerNodeID() const { + return NodeID::FromBinary(GetOwnerAddress().node_id()); +} + +const rpc::Address &GcsActor::GetOwnerAddress() const { + return actor_table_data_.owner_address(); +} + +void GcsActor::UpdateState(rpc::ActorTableData::ActorState state) { + actor_table_data_.set_state(state); + RefreshMetrics(); +} + +rpc::ActorTableData::ActorState GcsActor::GetState() const { + return actor_table_data_.state(); +} + +ActorID GcsActor::GetActorID() const { + return ActorID::FromBinary(actor_table_data_.actor_id()); +} + +bool GcsActor::IsDetached() const { return actor_table_data_.is_detached(); } + +std::string GcsActor::GetName() const { return actor_table_data_.name(); } + +std::string GcsActor::GetRayNamespace() const { + return actor_table_data_.ray_namespace(); +} + +TaskSpecification GcsActor::GetCreationTaskSpecification() const { + // The task spec is not available when the actor is dead. + RAY_CHECK(actor_table_data_.state() != rpc::ActorTableData::DEAD); + return TaskSpecification(*task_spec_); +} + +const rpc::ActorTableData &GcsActor::GetActorTableData() const { + return actor_table_data_; +} + +rpc::ActorTableData *GcsActor::GetMutableActorTableData() { return &actor_table_data_; } + +void GcsActor::WriteActorExportEvent() const { + /// Verify actor export events should be written to file + /// and then write actor_table_data_ as an export event. + if (!export_event_write_enabled_) { + return; + } + std::shared_ptr export_actor_data_ptr = + std::make_shared(); + + export_actor_data_ptr->set_actor_id(actor_table_data_.actor_id()); + export_actor_data_ptr->set_job_id(actor_table_data_.job_id()); + export_actor_data_ptr->set_state(ConvertActorStateToExport(actor_table_data_.state())); + export_actor_data_ptr->set_is_detached(actor_table_data_.is_detached()); + export_actor_data_ptr->set_name(actor_table_data_.name()); + export_actor_data_ptr->set_pid(actor_table_data_.pid()); + export_actor_data_ptr->set_ray_namespace(actor_table_data_.ray_namespace()); + export_actor_data_ptr->set_serialized_runtime_env( + actor_table_data_.serialized_runtime_env()); + export_actor_data_ptr->set_class_name(actor_table_data_.class_name()); + export_actor_data_ptr->mutable_death_cause()->CopyFrom(actor_table_data_.death_cause()); + export_actor_data_ptr->mutable_required_resources()->insert( + actor_table_data_.required_resources().begin(), + actor_table_data_.required_resources().end()); + export_actor_data_ptr->set_node_id(actor_table_data_.node_id()); + export_actor_data_ptr->set_placement_group_id(actor_table_data_.placement_group_id()); + export_actor_data_ptr->set_repr_name(actor_table_data_.repr_name()); + export_actor_data_ptr->mutable_labels()->insert(task_spec_.get()->labels().begin(), + task_spec_.get()->labels().end()); + export_actor_data_ptr->mutable_label_selector()->insert( + actor_table_data_.label_selector().begin(), + actor_table_data_.label_selector().end()); + + RayExportEvent(export_actor_data_ptr).SendEvent(); +} + +rpc::TaskSpec *GcsActor::GetMutableTaskSpec() { return task_spec_.get(); } + +const ResourceRequest &GcsActor::GetAcquiredResources() const { + return acquired_resources_; +} +void GcsActor::SetAcquiredResources(ResourceRequest &&resource_request) { + acquired_resources_ = std::move(resource_request); +} + +bool GcsActor::GetGrantOrReject() const { return grant_or_reject_; } + +void GcsActor::SetGrantOrReject(bool grant_or_reject) { + grant_or_reject_ = grant_or_reject; +} + +} // namespace gcs +} // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_actor.h b/src/ray/gcs/gcs_server/gcs_actor.h new file mode 100644 index 000000000000..b9aa9bea5019 --- /dev/null +++ b/src/ray/gcs/gcs_server/gcs_actor.h @@ -0,0 +1,276 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include + +#include "ray/common/id.h" +#include "ray/common/scheduling/cluster_resource_data.h" +#include "ray/common/task/task_spec.h" +#include "ray/util/counter_map.h" +#include "ray/util/event.h" +#include "src/ray/protobuf/core_worker.pb.h" +#include "src/ray/protobuf/export_actor_data.pb.h" +#include "src/ray/protobuf/gcs_service.pb.h" + +namespace ray { +namespace gcs { + +/// GcsActor just wraps `ActorTableData` and provides some convenient interfaces to access +/// the fields inside `ActorTableData`. +/// This class is not thread-safe. +class GcsActor { + public: + /// Create a GcsActor by actor_table_data. + /// + /// \param actor_table_data Data of the actor (see gcs.proto). + /// \param counter The counter to report metrics to. + explicit GcsActor( + rpc::ActorTableData actor_table_data, + std::shared_ptr>> + counter) + : actor_table_data_(std::move(actor_table_data)), + counter_(std::move(counter)), + export_event_write_enabled_(IsExportAPIEnabledActor()) { + RefreshMetrics(); + } + + /// Create a GcsActor by actor_table_data and task_spec. + /// This is only for ALIVE actors. + /// + /// \param actor_table_data Data of the actor (see gcs.proto). + /// \param task_spec Task spec of the actor. + /// \param counter The counter to report metrics to. + explicit GcsActor( + rpc::ActorTableData actor_table_data, + rpc::TaskSpec task_spec, + std::shared_ptr>> + counter) + : actor_table_data_(std::move(actor_table_data)), + task_spec_(std::make_unique(std::move(task_spec))), + counter_(std::move(counter)), + export_event_write_enabled_(IsExportAPIEnabledActor()) { + RAY_CHECK(actor_table_data_.state() != rpc::ActorTableData::DEAD); + RefreshMetrics(); + } + + /// Create a GcsActor by TaskSpec. + /// + /// \param task_spec Contains the actor creation task specification. + /// \param ray_namespace Namespace of the actor. + /// \param counter The counter to report metrics to. + explicit GcsActor( + rpc::TaskSpec task_spec, + std::string ray_namespace, + std::shared_ptr>> + counter) + : task_spec_(std::make_unique(std::move(task_spec))), + counter_(std::move(counter)), + export_event_write_enabled_(IsExportAPIEnabledActor()) { + RAY_CHECK(task_spec_->type() == TaskType::ACTOR_CREATION_TASK); + const auto &actor_creation_task_spec = task_spec_->actor_creation_task_spec(); + actor_table_data_.set_actor_id(actor_creation_task_spec.actor_id()); + actor_table_data_.set_job_id(task_spec_->job_id()); + actor_table_data_.set_max_restarts(actor_creation_task_spec.max_actor_restarts()); + actor_table_data_.set_num_restarts(0); + actor_table_data_.set_num_restarts_due_to_lineage_reconstruction(0); + + actor_table_data_.mutable_function_descriptor()->CopyFrom( + task_spec_->function_descriptor()); + + actor_table_data_.set_is_detached(actor_creation_task_spec.is_detached()); + actor_table_data_.set_name(actor_creation_task_spec.name()); + actor_table_data_.mutable_owner_address()->CopyFrom(task_spec_->caller_address()); + + actor_table_data_.set_state(rpc::ActorTableData::DEPENDENCIES_UNREADY); + + actor_table_data_.mutable_address()->set_node_id(NodeID::Nil().Binary()); + actor_table_data_.mutable_address()->set_worker_id(WorkerID::Nil().Binary()); + + actor_table_data_.set_ray_namespace(ray_namespace); + if (task_spec_->scheduling_strategy().scheduling_strategy_case() == + rpc::SchedulingStrategy::SchedulingStrategyCase:: + kPlacementGroupSchedulingStrategy) { + actor_table_data_.set_placement_group_id(task_spec_->scheduling_strategy() + .placement_group_scheduling_strategy() + .placement_group_id()); + } + + // Set required resources. + auto resource_map = + GetCreationTaskSpecification().GetRequiredResources().GetResourceMap(); + actor_table_data_.mutable_required_resources()->insert(resource_map.begin(), + resource_map.end()); + + const auto &function_descriptor = task_spec_->function_descriptor(); + switch (function_descriptor.function_descriptor_case()) { + case rpc::FunctionDescriptor::FunctionDescriptorCase::kJavaFunctionDescriptor: + actor_table_data_.set_class_name( + function_descriptor.java_function_descriptor().class_name()); + break; + case rpc::FunctionDescriptor::FunctionDescriptorCase::kPythonFunctionDescriptor: + actor_table_data_.set_class_name( + function_descriptor.python_function_descriptor().class_name()); + break; + default: + // TODO(Alex): Handle the C++ case, which we currently don't have an + // easy equivalent to class_name for. + break; + } + + actor_table_data_.set_serialized_runtime_env( + task_spec_->runtime_env_info().serialized_runtime_env()); + if (task_spec_->call_site().size() > 0) { + actor_table_data_.set_call_site(task_spec_->call_site()); + } + if (task_spec_->label_selector().size() > 0) { + actor_table_data_.mutable_label_selector()->insert( + task_spec_->label_selector().begin(), task_spec_->label_selector().end()); + } + RefreshMetrics(); + } + + ~GcsActor() { + // We don't decrement the value when it becomes DEAD because we don't want to + // lose the # of dead actors count when this class is GC'ed. + if (last_metric_state_ && last_metric_state_.value() != rpc::ActorTableData::DEAD) { + RAY_LOG(DEBUG) << "Decrementing state at " + << rpc::ActorTableData::ActorState_Name(last_metric_state_.value()) + << " " << GetActorTableData().class_name(); + counter_->Decrement( + std::make_pair(last_metric_state_.value(), GetActorTableData().class_name())); + } + } + + /// Get the node id on which this actor is created. + NodeID GetNodeID() const; + /// Get the id of the worker on which this actor is created. + WorkerID GetWorkerID() const; + /// Get the actor's owner ID. + WorkerID GetOwnerID() const; + /// Get the node ID of the actor's owner. + NodeID GetOwnerNodeID() const; + /// Get the address of the actor's owner. + const rpc::Address &GetOwnerAddress() const; + + /// Update the `Address` of this actor (see gcs.proto). + void UpdateAddress(const rpc::Address &address); + /// Get the `Address` of this actor. + const rpc::Address &GetAddress() const; + + /// Update the state of this actor and refreshes metrics. Do not update the + /// state of the underlying proto directly via set_state(), otherwise metrics + /// will get out of sync. + void UpdateState(rpc::ActorTableData::ActorState state); + /// Get the state of this gcs actor. + rpc::ActorTableData::ActorState GetState() const; + + /// Get the id of this actor. + ActorID GetActorID() const; + /// Returns whether or not this is a detached actor. + bool IsDetached() const; + /// Get the name of this actor. + std::string GetName() const; + /// Get the namespace of this actor. + std::string GetRayNamespace() const; + /// Get the task specification of this actor. + TaskSpecification GetCreationTaskSpecification() const; + + /// Get the immutable ActorTableData of this actor. + const rpc::ActorTableData &GetActorTableData() const; + /// Get the mutable ActorTableData of this actor. + rpc::ActorTableData *GetMutableActorTableData(); + rpc::TaskSpec *GetMutableTaskSpec(); + /// Write an event containing this actor's ActorTableData + /// to file for the Export API. + void WriteActorExportEvent() const; + // Verify if export events should be written for EXPORT_ACTOR source types + bool IsExportAPIEnabledActor() const { + return IsExportAPIEnabledSourceType( + "EXPORT_ACTOR", + RayConfig::instance().enable_export_api_write(), + RayConfig::instance().enable_export_api_write_config()); + } + + const ResourceRequest &GetAcquiredResources() const; + void SetAcquiredResources(ResourceRequest &&resource_request); + bool GetGrantOrReject() const; + void SetGrantOrReject(bool grant_or_reject); + + private: + void RefreshMetrics() { + auto cur_state = GetState(); + if (last_metric_state_) { + RAY_LOG(DEBUG) << "Swapping state from " + << rpc::ActorTableData::ActorState_Name(last_metric_state_.value()) + << " to " << rpc::ActorTableData::ActorState_Name(cur_state) + << " for : " << GetActorID(); + counter_->Swap( + std::make_pair(last_metric_state_.value(), GetActorTableData().class_name()), + std::make_pair(cur_state, GetActorTableData().class_name())); + } else { + RAY_LOG(DEBUG) << "Incrementing state at " + << rpc::ActorTableData::ActorState_Name(cur_state) << " " + << GetActorTableData().class_name(); + counter_->Increment(std::make_pair(cur_state, GetActorTableData().class_name())); + } + last_metric_state_ = cur_state; + } + + rpc::ExportActorData::ActorState ConvertActorStateToExport( + rpc::ActorTableData::ActorState actor_state) const { + switch (actor_state) { + case rpc::ActorTableData::DEPENDENCIES_UNREADY: + return rpc::ExportActorData::DEPENDENCIES_UNREADY; + case rpc::ActorTableData::PENDING_CREATION: + return rpc::ExportActorData::PENDING_CREATION; + case rpc::ActorTableData::ALIVE: + return rpc::ExportActorData::ALIVE; + case rpc::ActorTableData::RESTARTING: + return rpc::ExportActorData::RESTARTING; + case rpc::ActorTableData::DEAD: + return rpc::ExportActorData::DEAD; + default: + // Unknown rpc::ActorTableData::ActorState value + RAY_LOG(FATAL) << "Invalid value for rpc::ActorTableData::ActorState" + << rpc::ActorTableData::ActorState_Name(actor_state); + return rpc::ExportActorData::DEAD; + } + } + + /// The actor meta data which contains the task specification as well as the state of + /// the gcs actor and so on (see gcs.proto). + rpc::ActorTableData actor_table_data_; + const std::unique_ptr task_spec_; + /// Resources acquired by this actor. + ResourceRequest acquired_resources_; + /// Reference to the counter to use for actor state metrics tracking. + std::shared_ptr>> + counter_; + /// Whether the actor's target node only grants or rejects the lease request. + bool grant_or_reject_ = false; + /// The last recorded metric state. + std::optional last_metric_state_; + /// If true, actor events are exported for Export API + bool export_event_write_enabled_ = false; +}; + +using RestartActorForLineageReconstructionCallback = + std::function)>; +using CreateActorCallback = std::function, const rpc::PushTaskReply &reply, const Status &status)>; + +} // namespace gcs +} // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index 626631f91dc1..399dacf8b5b6 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -180,122 +180,6 @@ bool is_uuid(const std::string &str) { return regex_match(str, e); // note: case sensitive now } -NodeID GcsActor::GetNodeID() const { - const auto &node_id_binary = actor_table_data_.address().node_id(); - if (node_id_binary.empty()) { - return NodeID::Nil(); - } - return NodeID::FromBinary(node_id_binary); -} - -void GcsActor::UpdateAddress(const rpc::Address &address) { - actor_table_data_.mutable_address()->CopyFrom(address); -} - -const rpc::Address &GcsActor::GetAddress() const { return actor_table_data_.address(); } - -WorkerID GcsActor::GetWorkerID() const { - const auto &address = actor_table_data_.address(); - if (address.worker_id().empty()) { - return WorkerID::Nil(); - } - return WorkerID::FromBinary(address.worker_id()); -} - -WorkerID GcsActor::GetOwnerID() const { - return WorkerID::FromBinary(GetOwnerAddress().worker_id()); -} - -NodeID GcsActor::GetOwnerNodeID() const { - return NodeID::FromBinary(GetOwnerAddress().node_id()); -} - -const rpc::Address &GcsActor::GetOwnerAddress() const { - return actor_table_data_.owner_address(); -} - -void GcsActor::UpdateState(rpc::ActorTableData::ActorState state) { - actor_table_data_.set_state(state); - RefreshMetrics(); -} - -rpc::ActorTableData::ActorState GcsActor::GetState() const { - return actor_table_data_.state(); -} - -ActorID GcsActor::GetActorID() const { - return ActorID::FromBinary(actor_table_data_.actor_id()); -} - -bool GcsActor::IsDetached() const { return actor_table_data_.is_detached(); } - -std::string GcsActor::GetName() const { return actor_table_data_.name(); } - -std::string GcsActor::GetRayNamespace() const { - return actor_table_data_.ray_namespace(); -} - -TaskSpecification GcsActor::GetCreationTaskSpecification() const { - // The task spec is not available when the actor is dead. - RAY_CHECK(actor_table_data_.state() != rpc::ActorTableData::DEAD); - return TaskSpecification(*task_spec_); -} - -const rpc::ActorTableData &GcsActor::GetActorTableData() const { - return actor_table_data_; -} - -rpc::ActorTableData *GcsActor::GetMutableActorTableData() { return &actor_table_data_; } - -void GcsActor::WriteActorExportEvent() const { - /// Verify actor export events should be written to file - /// and then write actor_table_data_ as an export event. - if (!export_event_write_enabled_) { - return; - } - std::shared_ptr export_actor_data_ptr = - std::make_shared(); - - export_actor_data_ptr->set_actor_id(actor_table_data_.actor_id()); - export_actor_data_ptr->set_job_id(actor_table_data_.job_id()); - export_actor_data_ptr->set_state(ConvertActorStateToExport(actor_table_data_.state())); - export_actor_data_ptr->set_is_detached(actor_table_data_.is_detached()); - export_actor_data_ptr->set_name(actor_table_data_.name()); - export_actor_data_ptr->set_pid(actor_table_data_.pid()); - export_actor_data_ptr->set_ray_namespace(actor_table_data_.ray_namespace()); - export_actor_data_ptr->set_serialized_runtime_env( - actor_table_data_.serialized_runtime_env()); - export_actor_data_ptr->set_class_name(actor_table_data_.class_name()); - export_actor_data_ptr->mutable_death_cause()->CopyFrom(actor_table_data_.death_cause()); - export_actor_data_ptr->mutable_required_resources()->insert( - actor_table_data_.required_resources().begin(), - actor_table_data_.required_resources().end()); - export_actor_data_ptr->set_node_id(actor_table_data_.node_id()); - export_actor_data_ptr->set_placement_group_id(actor_table_data_.placement_group_id()); - export_actor_data_ptr->set_repr_name(actor_table_data_.repr_name()); - export_actor_data_ptr->mutable_labels()->insert(task_spec_.get()->labels().begin(), - task_spec_.get()->labels().end()); - export_actor_data_ptr->mutable_label_selector()->insert( - actor_table_data_.label_selector().begin(), - actor_table_data_.label_selector().end()); - - RayExportEvent(export_actor_data_ptr).SendEvent(); -} - -rpc::TaskSpec *GcsActor::GetMutableTaskSpec() { return task_spec_.get(); } - -const ResourceRequest &GcsActor::GetAcquiredResources() const { - return acquired_resources_; -} -void GcsActor::SetAcquiredResources(ResourceRequest &&resource_request) { - acquired_resources_ = std::move(resource_request); -} - -bool GcsActor::GetGrantOrReject() const { return grant_or_reject_; } -void GcsActor::SetGrantOrReject(bool grant_or_reject) { - grant_or_reject_ = grant_or_reject; -} - const ray::rpc::ActorDeathCause GcsActorManager::GenNodeDiedCause( const ray::gcs::GcsActor *actor, std::shared_ptr node) { ray::rpc::ActorDeathCause death_cause; @@ -332,7 +216,6 @@ const ray::rpc::ActorDeathCause GcsActorManager::GenNodeDiedCause( return death_cause; } -///////////////////////////////////////////////////////////////////////////////////////// GcsActorManager::GcsActorManager( std::unique_ptr scheduler, GcsTableStorage *gcs_table_storage, diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.h b/src/ray/gcs/gcs_server/gcs_actor_manager.h index 69a06df1f6cc..5fd24a07995d 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.h +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.h @@ -25,6 +25,7 @@ #include "ray/common/id.h" #include "ray/common/runtime_env_manager.h" #include "ray/common/task/task_spec.h" +#include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_actor_scheduler.h" #include "ray/gcs/gcs_server/gcs_function_manager.h" #include "ray/gcs/gcs_server/gcs_init_data.h" @@ -41,250 +42,6 @@ namespace ray { namespace gcs { -/// GcsActor just wraps `ActorTableData` and provides some convenient interfaces to access -/// the fields inside `ActorTableData`. -/// This class is not thread-safe. -class GcsActor { - public: - /// Create a GcsActor by actor_table_data. - /// - /// \param actor_table_data Data of the actor (see gcs.proto). - /// \param counter The counter to report metrics to. - explicit GcsActor( - rpc::ActorTableData actor_table_data, - std::shared_ptr>> - counter) - : actor_table_data_(std::move(actor_table_data)), - counter_(std::move(counter)), - export_event_write_enabled_(IsExportAPIEnabledActor()) { - RefreshMetrics(); - } - - /// Create a GcsActor by actor_table_data and task_spec. - /// This is only for ALIVE actors. - /// - /// \param actor_table_data Data of the actor (see gcs.proto). - /// \param task_spec Task spec of the actor. - /// \param counter The counter to report metrics to. - explicit GcsActor( - rpc::ActorTableData actor_table_data, - rpc::TaskSpec task_spec, - std::shared_ptr>> - counter) - : actor_table_data_(std::move(actor_table_data)), - task_spec_(std::make_unique(std::move(task_spec))), - counter_(std::move(counter)), - export_event_write_enabled_(IsExportAPIEnabledActor()) { - RAY_CHECK(actor_table_data_.state() != rpc::ActorTableData::DEAD); - RefreshMetrics(); - } - - /// Create a GcsActor by TaskSpec. - /// - /// \param task_spec Contains the actor creation task specification. - /// \param ray_namespace Namespace of the actor. - /// \param counter The counter to report metrics to. - explicit GcsActor( - rpc::TaskSpec task_spec, - std::string ray_namespace, - std::shared_ptr>> - counter) - : task_spec_(std::make_unique(std::move(task_spec))), - counter_(std::move(counter)), - export_event_write_enabled_(IsExportAPIEnabledActor()) { - RAY_CHECK(task_spec_->type() == TaskType::ACTOR_CREATION_TASK); - const auto &actor_creation_task_spec = task_spec_->actor_creation_task_spec(); - actor_table_data_.set_actor_id(actor_creation_task_spec.actor_id()); - actor_table_data_.set_job_id(task_spec_->job_id()); - actor_table_data_.set_max_restarts(actor_creation_task_spec.max_actor_restarts()); - actor_table_data_.set_num_restarts(0); - actor_table_data_.set_num_restarts_due_to_lineage_reconstruction(0); - - actor_table_data_.mutable_function_descriptor()->CopyFrom( - task_spec_->function_descriptor()); - - actor_table_data_.set_is_detached(actor_creation_task_spec.is_detached()); - actor_table_data_.set_name(actor_creation_task_spec.name()); - actor_table_data_.mutable_owner_address()->CopyFrom(task_spec_->caller_address()); - - actor_table_data_.set_state(rpc::ActorTableData::DEPENDENCIES_UNREADY); - - actor_table_data_.mutable_address()->set_node_id(NodeID::Nil().Binary()); - actor_table_data_.mutable_address()->set_worker_id(WorkerID::Nil().Binary()); - - actor_table_data_.set_ray_namespace(ray_namespace); - if (task_spec_->scheduling_strategy().scheduling_strategy_case() == - rpc::SchedulingStrategy::SchedulingStrategyCase:: - kPlacementGroupSchedulingStrategy) { - actor_table_data_.set_placement_group_id(task_spec_->scheduling_strategy() - .placement_group_scheduling_strategy() - .placement_group_id()); - } - - // Set required resources. - auto resource_map = - GetCreationTaskSpecification().GetRequiredResources().GetResourceMap(); - actor_table_data_.mutable_required_resources()->insert(resource_map.begin(), - resource_map.end()); - - const auto &function_descriptor = task_spec_->function_descriptor(); - switch (function_descriptor.function_descriptor_case()) { - case rpc::FunctionDescriptor::FunctionDescriptorCase::kJavaFunctionDescriptor: - actor_table_data_.set_class_name( - function_descriptor.java_function_descriptor().class_name()); - break; - case rpc::FunctionDescriptor::FunctionDescriptorCase::kPythonFunctionDescriptor: - actor_table_data_.set_class_name( - function_descriptor.python_function_descriptor().class_name()); - break; - default: - // TODO(Alex): Handle the C++ case, which we currently don't have an - // easy equivalent to class_name for. - break; - } - - actor_table_data_.set_serialized_runtime_env( - task_spec_->runtime_env_info().serialized_runtime_env()); - if (task_spec_->call_site().size() > 0) { - actor_table_data_.set_call_site(task_spec_->call_site()); - } - if (task_spec_->label_selector().size() > 0) { - actor_table_data_.mutable_label_selector()->insert( - task_spec_->label_selector().begin(), task_spec_->label_selector().end()); - } - RefreshMetrics(); - } - - ~GcsActor() { - // We don't decrement the value when it becomes DEAD because we don't want to - // lose the # of dead actors count when this class is GC'ed. - if (last_metric_state_ && last_metric_state_.value() != rpc::ActorTableData::DEAD) { - RAY_LOG(DEBUG) << "Decrementing state at " - << rpc::ActorTableData::ActorState_Name(last_metric_state_.value()) - << " " << GetActorTableData().class_name(); - counter_->Decrement( - std::make_pair(last_metric_state_.value(), GetActorTableData().class_name())); - } - } - - /// Get the node id on which this actor is created. - NodeID GetNodeID() const; - /// Get the id of the worker on which this actor is created. - WorkerID GetWorkerID() const; - /// Get the actor's owner ID. - WorkerID GetOwnerID() const; - /// Get the node ID of the actor's owner. - NodeID GetOwnerNodeID() const; - /// Get the address of the actor's owner. - const rpc::Address &GetOwnerAddress() const; - - /// Update the `Address` of this actor (see gcs.proto). - void UpdateAddress(const rpc::Address &address); - /// Get the `Address` of this actor. - const rpc::Address &GetAddress() const; - - /// Update the state of this actor and refreshes metrics. Do not update the - /// state of the underlying proto directly via set_state(), otherwise metrics - /// will get out of sync. - void UpdateState(rpc::ActorTableData::ActorState state); - /// Get the state of this gcs actor. - rpc::ActorTableData::ActorState GetState() const; - - /// Get the id of this actor. - ActorID GetActorID() const; - /// Returns whether or not this is a detached actor. - bool IsDetached() const; - /// Get the name of this actor. - std::string GetName() const; - /// Get the namespace of this actor. - std::string GetRayNamespace() const; - /// Get the task specification of this actor. - TaskSpecification GetCreationTaskSpecification() const; - - /// Get the immutable ActorTableData of this actor. - const rpc::ActorTableData &GetActorTableData() const; - /// Get the mutable ActorTableData of this actor. - rpc::ActorTableData *GetMutableActorTableData(); - rpc::TaskSpec *GetMutableTaskSpec(); - /// Write an event containing this actor's ActorTableData - /// to file for the Export API. - void WriteActorExportEvent() const; - // Verify if export events should be written for EXPORT_ACTOR source types - bool IsExportAPIEnabledActor() const { - return IsExportAPIEnabledSourceType( - "EXPORT_ACTOR", - RayConfig::instance().enable_export_api_write(), - RayConfig::instance().enable_export_api_write_config()); - } - - const ResourceRequest &GetAcquiredResources() const; - void SetAcquiredResources(ResourceRequest &&resource_request); - bool GetGrantOrReject() const; - void SetGrantOrReject(bool grant_or_reject); - - private: - void RefreshMetrics() { - auto cur_state = GetState(); - if (last_metric_state_) { - RAY_LOG(DEBUG) << "Swapping state from " - << rpc::ActorTableData::ActorState_Name(last_metric_state_.value()) - << " to " << rpc::ActorTableData::ActorState_Name(cur_state) - << " for : " << GetActorID(); - counter_->Swap( - std::make_pair(last_metric_state_.value(), GetActorTableData().class_name()), - std::make_pair(cur_state, GetActorTableData().class_name())); - } else { - RAY_LOG(DEBUG) << "Incrementing state at " - << rpc::ActorTableData::ActorState_Name(cur_state) << " " - << GetActorTableData().class_name(); - counter_->Increment(std::make_pair(cur_state, GetActorTableData().class_name())); - } - last_metric_state_ = cur_state; - } - - rpc::ExportActorData::ActorState ConvertActorStateToExport( - rpc::ActorTableData::ActorState actor_state) const { - switch (actor_state) { - case rpc::ActorTableData::DEPENDENCIES_UNREADY: - return rpc::ExportActorData::DEPENDENCIES_UNREADY; - case rpc::ActorTableData::PENDING_CREATION: - return rpc::ExportActorData::PENDING_CREATION; - case rpc::ActorTableData::ALIVE: - return rpc::ExportActorData::ALIVE; - case rpc::ActorTableData::RESTARTING: - return rpc::ExportActorData::RESTARTING; - case rpc::ActorTableData::DEAD: - return rpc::ExportActorData::DEAD; - default: - // Unknown rpc::ActorTableData::ActorState value - RAY_LOG(FATAL) << "Invalid value for rpc::ActorTableData::ActorState" - << rpc::ActorTableData::ActorState_Name(actor_state); - return rpc::ExportActorData::DEAD; - } - } - - /// The actor meta data which contains the task specification as well as the state of - /// the gcs actor and so on (see gcs.proto). - rpc::ActorTableData actor_table_data_; - const std::unique_ptr task_spec_; - /// Resources acquired by this actor. - ResourceRequest acquired_resources_; - /// Reference to the counter to use for actor state metrics tracking. - std::shared_ptr>> - counter_; - /// Whether the actor's target node only grants or rejects the lease request. - bool grant_or_reject_ = false; - /// The last recorded metric state. - std::optional last_metric_state_; - /// If true, actor events are exported for Export API - bool export_event_write_enabled_ = false; -}; - -using RestartActorForLineageReconstructionCallback = - std::function)>; -using CreateActorCallback = std::function, const rpc::PushTaskReply &reply, const Status &status)>; - /// GcsActorManager is responsible for managing the lifecycle of all actors. /// This class is not thread-safe. /// Actor State Transition Diagram: diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc index 746de6923558..a970061bd243 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc @@ -22,7 +22,6 @@ #include "ray/common/asio/asio_util.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/ray_config.h" -#include "ray/gcs/gcs_server/gcs_actor_manager.h" #include "ray/util/time.h" #include "src/ray/protobuf/node_manager.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h index b3a3c708debc..218bf9f3b035 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h @@ -27,6 +27,7 @@ #include "ray/common/id.h" #include "ray/common/scheduling/scheduling_ids.h" #include "ray/common/task/task_spec.h" +#include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/raylet/scheduling/cluster_task_manager.h" @@ -41,8 +42,6 @@ namespace ray { using raylet::ClusterTaskManager; namespace gcs { -class GcsActor; - using GcsActorSchedulerFailureCallback = std::function, rpc::RequestWorkerLeaseReply::SchedulingFailureType, From 216fed93304d467eec1c008a07b17f37b47ede7d Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Wed, 27 Aug 2025 12:49:55 -0500 Subject: [PATCH 303/634] [core] Remove `gcs_rpc_server.h` dependency from `InternalKVManager` (#55999) Splitting gRPC service interface from implementation. --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 4 +- src/ray/gcs/gcs_server/gcs_function_manager.h | 1 + src/ray/gcs/gcs_server/gcs_kv_manager.h | 5 +- src/ray/gcs/gcs_server/gcs_server.cc | 4 +- src/ray/gcs/gcs_server/gcs_task_manager.h | 1 + .../gcs/gcs_server/grpc_service_interfaces.h | 32 ++++++++++ src/ray/gcs/gcs_server/grpc_services.cc | 16 +++++ src/ray/gcs/gcs_server/grpc_services.h | 23 +++++++ src/ray/gcs/gcs_server/tests/BUILD.bazel | 2 +- .../tests/gcs_function_manager_test.cc | 6 +- src/ray/rpc/gcs/gcs_rpc_server.h | 62 ------------------- 11 files changed, 85 insertions(+), 71 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index bc0a46a00635..bcf7cafe32e7 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -47,8 +47,8 @@ ray_cc_library( deps = [ "//src/ray/common:asio", "//src/ray/common:status", + "//src/ray/gcs/gcs_server:grpc_service_interfaces", "//src/ray/protobuf:gcs_cc_proto", - "//src/ray/rpc:gcs_server", ], ) @@ -59,6 +59,7 @@ ray_cc_library( ":gcs_kv_manager", "//src/ray/common:asio", "//src/ray/common:constants", + "//src/ray/common:id", "@com_google_absl//absl/container:flat_hash_map", ], ) @@ -180,6 +181,7 @@ ray_cc_library( "//src/ray/gcs:gcs_pb_util", "//src/ray/protobuf:events_event_aggregator_service_cc_proto", "//src/ray/protobuf:gcs_cc_proto", + "//src/ray/rpc:gcs_server", "//src/ray/util:counter_map", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", diff --git a/src/ray/gcs/gcs_server/gcs_function_manager.h b/src/ray/gcs/gcs_server/gcs_function_manager.h index 10eac00744e3..27380c052e83 100644 --- a/src/ray/gcs/gcs_server/gcs_function_manager.h +++ b/src/ray/gcs/gcs_server/gcs_function_manager.h @@ -17,6 +17,7 @@ #include "absl/container/flat_hash_map.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/constants.h" +#include "ray/common/id.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/gcs_kv_manager.h b/src/ray/gcs/gcs_server/gcs_kv_manager.h index eb1ca3b302f7..bacbf74a8768 100644 --- a/src/ray/gcs/gcs_server/gcs_kv_manager.h +++ b/src/ray/gcs/gcs_server/gcs_kv_manager.h @@ -22,7 +22,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/asio/postable.h" #include "ray/common/status.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" namespace ray { namespace gcs { @@ -100,8 +100,7 @@ class InternalKVInterface { virtual ~InternalKVInterface() = default; }; -/// This implementation class of `InternalKVHandler`. -class GcsInternalKVManager : public rpc::InternalKVHandler { +class GcsInternalKVManager : public rpc::InternalKVGcsServiceHandler { public: explicit GcsInternalKVManager(std::unique_ptr kv_instance, std::string raylet_config_list, diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 00ce1088abe0..f38223c44afb 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -613,7 +613,9 @@ void GcsServer::InitKVService() { RAY_CHECK(kv_manager_); rpc_server_.RegisterService( std::make_unique( - io_context_provider_.GetIOContext(), *kv_manager_), + io_context_provider_.GetIOContext(), + *kv_manager_, + /*max_active_rpcs_per_handler_=*/-1), false /* token_auth */); } diff --git a/src/ray/gcs/gcs_server/gcs_task_manager.h b/src/ray/gcs/gcs_server/gcs_task_manager.h index 89ca5c8b611c..a98d30c0f036 100644 --- a/src/ray/gcs/gcs_server/gcs_task_manager.h +++ b/src/ray/gcs/gcs_server/gcs_task_manager.h @@ -26,6 +26,7 @@ #include "absl/synchronization/mutex.h" #include "ray/gcs/gcs_server/usage_stats_client.h" #include "ray/gcs/pb_util.h" +#include "ray/rpc/gcs/gcs_rpc_server.h" #include "ray/util/counter_map.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h index 743017f20ed0..4721344af4ae 100644 --- a/src/ray/gcs/gcs_server/grpc_service_interfaces.h +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -153,5 +153,37 @@ class WorkerInfoGcsServiceHandler { SendReplyCallback send_reply_callback) = 0; }; +class InternalKVGcsServiceHandler { + public: + virtual ~InternalKVGcsServiceHandler() = default; + virtual void HandleInternalKVKeys(InternalKVKeysRequest request, + InternalKVKeysReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleInternalKVGet(InternalKVGetRequest request, + InternalKVGetReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleInternalKVMultiGet(InternalKVMultiGetRequest request, + InternalKVMultiGetReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleInternalKVPut(InternalKVPutRequest request, + InternalKVPutReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleInternalKVDel(InternalKVDelRequest request, + InternalKVDelReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleInternalKVExists(InternalKVExistsRequest request, + InternalKVExistsReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetInternalConfig(GetInternalConfigRequest request, + GetInternalConfigReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc index 6ac013f7502a..897cf18e234c 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -84,5 +84,21 @@ void WorkerInfoGrpcService::InitServerCallFactories( WorkerInfoGcsService, UpdateWorkerNumPausedThreads, max_active_rpcs_per_handler_) } +void InternalKVGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + RPC_SERVICE_HANDLER(InternalKVGcsService, InternalKVGet, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + InternalKVGcsService, InternalKVMultiGet, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(InternalKVGcsService, InternalKVPut, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(InternalKVGcsService, InternalKVDel, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + InternalKVGcsService, InternalKVExists, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(InternalKVGcsService, InternalKVKeys, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + InternalKVGcsService, GetInternalConfig, max_active_rpcs_per_handler_) +} + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/gcs_server/grpc_services.h index d825ad7112bb..b8f6b69f8180 100644 --- a/src/ray/gcs/gcs_server/grpc_services.h +++ b/src/ray/gcs/gcs_server/grpc_services.h @@ -150,5 +150,28 @@ class WorkerInfoGrpcService : public GrpcService { int64_t max_active_rpcs_per_handler_; }; +class InternalKVGrpcService : public GrpcService { + public: + explicit InternalKVGrpcService(instrumented_io_context &io_service, + InternalKVGcsServiceHandler &handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler){}; + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + InternalKVGcsService::AsyncService service_; + InternalKVGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index d5417e0647f3..ccb97f0489f0 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -6,7 +6,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_function_manager", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc index 2e82f08330d6..f2db28735bad 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc @@ -14,12 +14,12 @@ #include "ray/gcs/gcs_server/gcs_function_manager.h" +#include + #include -// clang-format off -#include "gtest/gtest.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" -// clang-format on +#include "ray/gcs/gcs_server/gcs_function_manager.h" namespace ray { diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index 3cbc18558d07..7843b9ad8d17 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -142,9 +142,6 @@ namespace rpc { HANDLER, \ RayConfig::instance().gcs_max_active_rpcs_per_handler()) -#define INTERNAL_KV_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(InternalKVGcsService, HANDLER, -1) - #define GCS_RPC_SEND_REPLY(send_reply_callback, reply, status) \ reply->mutable_status()->set_code(static_cast(status.code())); \ reply->mutable_status()->set_message(status.message()); \ @@ -351,64 +348,6 @@ class PlacementGroupInfoGrpcService : public GrpcService { PlacementGroupInfoGcsServiceHandler &service_handler_; }; -class InternalKVGcsServiceHandler { - public: - virtual ~InternalKVGcsServiceHandler() = default; - virtual void HandleInternalKVKeys(InternalKVKeysRequest request, - InternalKVKeysReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleInternalKVGet(InternalKVGetRequest request, - InternalKVGetReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleInternalKVMultiGet(InternalKVMultiGetRequest request, - InternalKVMultiGetReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleInternalKVPut(InternalKVPutRequest request, - InternalKVPutReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleInternalKVDel(InternalKVDelRequest request, - InternalKVDelReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleInternalKVExists(InternalKVExistsRequest request, - InternalKVExistsReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetInternalConfig(GetInternalConfigRequest request, - GetInternalConfigReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -class InternalKVGrpcService : public GrpcService { - public: - explicit InternalKVGrpcService(instrumented_io_context &io_service, - InternalKVGcsServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler) {} - - protected: - grpc::Service &GetGrpcService() override { return service_; } - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - INTERNAL_KV_SERVICE_RPC_HANDLER(InternalKVGet); - INTERNAL_KV_SERVICE_RPC_HANDLER(InternalKVMultiGet); - INTERNAL_KV_SERVICE_RPC_HANDLER(InternalKVPut); - INTERNAL_KV_SERVICE_RPC_HANDLER(InternalKVDel); - INTERNAL_KV_SERVICE_RPC_HANDLER(InternalKVExists); - INTERNAL_KV_SERVICE_RPC_HANDLER(InternalKVKeys); - INTERNAL_KV_SERVICE_RPC_HANDLER(GetInternalConfig); - } - - private: - InternalKVGcsService::AsyncService service_; - InternalKVGcsServiceHandler &service_handler_; -}; - class TaskInfoGcsServiceHandler { public: virtual ~TaskInfoGcsServiceHandler() = default; @@ -485,7 +424,6 @@ class RayEventExportGrpcService : public GrpcService { using ActorInfoHandler = ActorInfoGcsServiceHandler; using NodeResourceInfoHandler = NodeResourceInfoGcsServiceHandler; using PlacementGroupInfoHandler = PlacementGroupInfoGcsServiceHandler; -using InternalKVHandler = InternalKVGcsServiceHandler; using TaskInfoHandler = TaskInfoGcsServiceHandler; using RayEventExportHandler = RayEventExportGcsServiceHandler; From 5f83c073ea4104924d06ac8b9ca33116d59fb804 Mon Sep 17 00:00:00 2001 From: Mengjin Yan Date: Wed, 27 Aug 2025 11:27:37 -0700 Subject: [PATCH 304/634] [Core] Add Comment on the Remaining Dynamic Labels Logic (#55982) Dynamic labels are a deprecated feature in Ray. However, Ray's autoscaler depends on the feature for placement group strict-spread scheduling. So for backward compatibility purposes, we keep this only use case for now in the codebase. This PR add comments to clarify the above in the code to avoid confusion from people. Signed-off-by: Mengjin Yan --- python/ray/autoscaler/v2/scheduler.py | 7 ++++++- src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc | 4 ++++ src/ray/gcs/pb_util.h | 2 ++ src/ray/protobuf/autoscaler.proto | 6 +++++- 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/python/ray/autoscaler/v2/scheduler.py b/python/ray/autoscaler/v2/scheduler.py index 9fafab0ca7a4..6c87dc4e85f5 100644 --- a/python/ray/autoscaler/v2/scheduler.py +++ b/python/ray/autoscaler/v2/scheduler.py @@ -161,6 +161,8 @@ class SchedulingNode: # The node's current resource capacity. total_resources: Dict[str, float] = field(default_factory=dict) # Node's labels, including static or dynamic labels. + # Note that dynamic labels are a deprecated feature. And it is only used for the + # autoscaler’s strict-spread placement group scheduling (antiaffinity) labels: Dict[str, str] = field(default_factory=dict) # Observability descriptive message for why the node was launched in the # first place. @@ -278,6 +280,9 @@ def new( available_resources=dict(instance.ray_node.available_resources), labels={ **(instance.ray_node.labels or {}), + # DEPRECATED: Dynamic labels are a deprecated feature. This field + # is used here only for the autoscaler’s strict-spread placement + # group scheduling (antiaffinity). **(instance.ray_node.dynamic_labels or {}), }, status=SchedulingNodeStatus.SCHEDULABLE, @@ -606,7 +611,7 @@ def _try_schedule_one( # Add the request to the node. self.add_sched_request(request, resource_request_source) - # Update the dynamic labels if there's any + # Update the placement group in labels if there's any for constraint in request.placement_constraints: # We don't need to check for affinity constraints here since # we have already combined resource requests with the affinity diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc index 8a84b8a00c63..c6de00a352f6 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc @@ -413,6 +413,10 @@ void GcsAutoscalerStateManager::GetNodeStates( node_state_proto->mutable_total_resources()->insert(total.begin(), total.end()); // Add dynamic PG labels. + // DEPRECATED: Dynamic labels feature is deprecated. Do not introduce new usages. + // This assignment is kept only for backward compatibility in the autoscaler, where + // the placement group ID is needed to enforce antiaffinity constraints for + // strict-spread placement group scheduling. const auto &pgs_on_node = gcs_placement_group_manager_.GetBundlesOnNode(node_id); for (const auto &[pg_id, _bundle_indices] : pgs_on_node) { node_state_proto->mutable_dynamic_labels()->insert( diff --git a/src/ray/gcs/pb_util.h b/src/ray/gcs/pb_util.h index 52f222ba443a..f934ea51309d 100644 --- a/src/ray/gcs/pb_util.h +++ b/src/ray/gcs/pb_util.h @@ -454,6 +454,8 @@ GenPlacementConstraintForPlacementGroup(const std::string &pg_id, // We are embedding the PG id into the key for the same reasons as we do for // dynamic labels (a node will have multiple PGs thus having a common PG key // is not enough). + // Note that this is only use case for dynamic labels and is retained + // purely for backward compatibility purposes. const std::string name = FormatPlacementGroupLabelName(pg_id); switch (strategy) { case rpc::PlacementStrategy::STRICT_SPREAD: { diff --git a/src/ray/protobuf/autoscaler.proto b/src/ray/protobuf/autoscaler.proto index 6666e131828c..e463b08c1bed 100644 --- a/src/ray/protobuf/autoscaler.proto +++ b/src/ray/protobuf/autoscaler.proto @@ -150,7 +150,11 @@ message NodeState { // The corresponding total resources on the node. map total_resources = 5; - // Dynamic labels associated with the node. + // DEPRECATED: This field is part of the deprecated dynamic labels feature and + // must not be used in new code. It is retained solely for backward compatibility + // in the autoscaler, where it is required to retrieve the placement group ID for + // enforcing antiaffinity constraints in strict-spread placement group scheduling. + // // Reserved dynamic label names: _PG map dynamic_labels = 6; From d115cdaaa71642f874a32b692572f01bc993154b Mon Sep 17 00:00:00 2001 From: avigyabb <98926738+avigyabb@users.noreply.github.com> Date: Wed, 27 Aug 2025 11:58:56 -0700 Subject: [PATCH 305/634] [core][gpu-objects] Handle multiple transfers of the same object to an actor (#55628) There exists a race condition where if a gpu obj is being passed twice to an actor. Since the send/recv is occurring on a background thread, an order of operations may occur like: actor1.recv1 --> actor1.recv2 --> actor1.pop1 --> actor1.pop2. In this case, the obj_id gets popped out of the dictionary in pop1, and pop2 will hang waiting for the gpu_obj to be received. Our solution is to implement a queue that accepts multiple recvs so each pop always has a relevant list of tensors in the object store. --------- Signed-off-by: avigyabb --- .../gpu_object_manager/gpu_object_store.py | 42 ++++++++++++------- python/ray/tests/test_gpu_objects_gloo.py | 39 +++++++++++++++++ 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_store.py b/python/ray/experimental/gpu_object_manager/gpu_object_store.py index 92e706f50af0..93fcbc20f54d 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_store.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_store.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Dict, List, Optional, Set import threading -from collections import defaultdict +from collections import defaultdict, deque import ray.util.collective as collective from ray._private.custom_types import TensorTransportEnum @@ -139,10 +139,10 @@ class GPUObjectStore: """ def __init__(self): - # A dictionary that maps from an object ID to a list of tensors. + # A dictionary that maps from an object ID to a queue of tensor lists. # - # Note: Currently, `gpu_object_store` is only supported for Ray Actors. - self._gpu_object_store: Dict[str, _GPUObject] = {} + # Note: Currently, `_gpu_object_store` is only supported for Ray Actors. + self._gpu_object_store: Dict[str, deque[_GPUObject]] = defaultdict(deque) # Mapping from tensor to the IDs of objects that contain it. self._tensor_to_object_ids: Dict["torch.Tensor", Set[str]] = defaultdict(set) # Synchronization for GPU object store. @@ -154,7 +154,10 @@ def __init__(self): def has_object(self, obj_id: str) -> bool: with self._lock: - return obj_id in self._gpu_object_store + existed = obj_id in self._gpu_object_store + if existed: + return len(self._gpu_object_store[obj_id]) > 0 + return existed def has_tensor(self, tensor: "torch.Tensor") -> bool: with self._lock: @@ -162,7 +165,7 @@ def has_tensor(self, tensor: "torch.Tensor") -> bool: def get_object(self, obj_id: str) -> Optional[List["torch.Tensor"]]: with self._lock: - return self._gpu_object_store[obj_id].data + return self._gpu_object_store[obj_id][0].data def add_object( self, @@ -181,17 +184,19 @@ def add_object( with self._object_present_cv: for tensor in gpu_object: self._tensor_to_object_ids[tensor].add(obj_id) - self._gpu_object_store[obj_id] = _GPUObject( - gpu_object, - is_primary, + # Append to the queue instead of overwriting + self._gpu_object_store[obj_id].append( + _GPUObject( + gpu_object, + is_primary, + ) ) self._object_present_cv.notify_all() def is_primary_copy(self, obj_id: str) -> bool: with self._lock: return ( - obj_id in self._gpu_object_store - and self._gpu_object_store[obj_id].is_primary + self.has_object(obj_id) and self._gpu_object_store[obj_id][0].is_primary ) def wait_and_get_object( @@ -246,7 +251,8 @@ def _wait_object(self, obj_id: str, timeout: Optional[float] = None) -> None: """ with self._object_present_cv: if not self._object_present_cv.wait_for( - lambda: obj_id in self._gpu_object_store, timeout=timeout + lambda: self.has_object(obj_id), + timeout=timeout, ): raise TimeoutError( f"ObjectRef({obj_id}) not found in GPU object store after {timeout}s, transfer may have failed. Please report this issue on GitHub: https://github.com/ray-project/ray/issues/new/choose" @@ -254,10 +260,13 @@ def _wait_object(self, obj_id: str, timeout: Optional[float] = None) -> None: def pop_object(self, obj_id: str) -> List["torch.Tensor"]: with self._lock: - assert ( - obj_id in self._gpu_object_store + assert self.has_object( + obj_id ), f"obj_id={obj_id} not found in GPU object store" - gpu_object = self._gpu_object_store.pop(obj_id) + queue = self._gpu_object_store.get(obj_id) + gpu_object = queue.popleft() + if len(queue) == 0: + del self._gpu_object_store[obj_id] for tensor in gpu_object.data: self._tensor_to_object_ids[tensor].remove(obj_id) if len(self._tensor_to_object_ids[tensor]) == 0: @@ -284,4 +293,5 @@ def get_num_objects(self) -> int: Return the number of objects in the GPU object store. """ with self._lock: - return len(self._gpu_object_store) + # Count total objects across all queues + return sum(len(queue) for queue in self._gpu_object_store.values()) diff --git a/python/ray/tests/test_gpu_objects_gloo.py b/python/ray/tests/test_gpu_objects_gloo.py index 65f5550e2896..4b01babd9a69 100644 --- a/python/ray/tests/test_gpu_objects_gloo.py +++ b/python/ray/tests/test_gpu_objects_gloo.py @@ -37,6 +37,10 @@ def double(self, data): return data.apply(lambda x: x * 2) return data * 2 + def increment(self, data): + data += 1 + return data + def get_out_of_band_tensors(self, obj_id: str, timeout=None): gpu_object_store = ( ray._private.worker.global_worker.gpu_object_manager.gpu_object_store @@ -812,5 +816,40 @@ def gc(obj_id): assert not gpu_object_store.has_object(obj_id2) +def test_duplicate_objectref_transfer(ray_start_regular): + world_size = 2 + actors = [GPUTestActor.remote() for _ in range(world_size)] + create_collective_group(actors, backend="torch_gloo") + actor0, actor1 = actors[0], actors[1] + + small_tensor = torch.randn((1,)) + + # Store the original value for comparison + original_value = small_tensor + + ref = actor0.echo.remote(small_tensor) + + # Pass the same ref to actor1 twice + result1 = actor1.increment.remote(ref) + result2 = actor1.increment.remote(ref) + + # Both should return original_value + 1 because each increment task should receive the same object value. + val1 = ray.get(result1) + val2 = ray.get(result2) + + # Check for correctness + assert val1 == pytest.approx( + original_value + 1 + ), f"Result1 incorrect: got {val1}, expected {original_value + 1}" + assert val2 == pytest.approx( + original_value + 1 + ), f"Result2 incorrect: got {val2}, expected {original_value + 1}" + + # Additional check: results should be equal (both got clean copies) + assert val1 == pytest.approx( + val2 + ), f"Results differ: result1={val1}, result2={val2}" + + if __name__ == "__main__": sys.exit(pytest.main(["-sv", __file__])) From 2fd22314b4920bcf911fe8727a58de1426d1cf01 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Wed, 27 Aug 2025 13:44:42 -0700 Subject: [PATCH 306/634] [ci] point refresh compile commands to :ray_pkg_zip (#56004) deprecating `:ray_pkg` rule Signed-off-by: Lonnie Liu --- BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD.bazel b/BUILD.bazel index f2ce28aa7e9b..47afe3ea98a3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -117,7 +117,7 @@ alias( refresh_compile_commands( name = "refresh_compile_commands", targets = { - "//:ray_pkg": "", + "//:ray_pkg_zip": "", }, ) From 5d9932309bace2292cc2fd94cd565ec24b28c065 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Wed, 27 Aug 2025 15:53:49 -0500 Subject: [PATCH 307/634] [core] Split `gcs_resource_manager` target from `gcs_server_lib` (#55846) Stacked on: https://github.com/ray-project/ray/pull/55814 --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 47 +++++++++++---- src/ray/gcs/gcs_server/gcs_job_manager.cc | 3 - src/ray/gcs/gcs_server/gcs_job_manager.h | 9 +-- src/ray/gcs/gcs_server/gcs_node_manager.h | 13 +++-- .../gcs/gcs_server/gcs_placement_group_mgr.h | 1 + .../gcs/gcs_server/gcs_resource_manager.cc | 7 ++- src/ray/gcs/gcs_server/gcs_resource_manager.h | 23 +++----- src/ray/gcs/gcs_server/gcs_server.cc | 4 +- .../gcs/gcs_server/grpc_service_interfaces.h | 21 +++++++ src/ray/gcs/gcs_server/grpc_services.cc | 14 +++++ src/ray/gcs/gcs_server/grpc_services.h | 23 ++++++++ src/ray/gcs/gcs_server/tests/BUILD.bazel | 32 ++++++---- .../gcs_actor_manager_export_event_test.cc | 13 ++--- .../gcs_job_manager_export_event_test.cc | 4 +- .../gcs_autoscaler_state_manager_test.cc | 2 + .../tests/gcs_function_manager_test.cc | 1 - .../gcs_server/tests/gcs_job_manager_test.cc | 1 - .../tests/gcs_worker_manager_test.cc | 14 ++--- .../tests/usage_stats_client_test.cc | 5 +- src/ray/rpc/gcs/gcs_rpc_server.h | 58 ------------------- 20 files changed, 163 insertions(+), 132 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index bcf7cafe32e7..8d6de0908ca3 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -68,10 +68,6 @@ ray_cc_library( name = "gcs_node_manager", srcs = ["gcs_node_manager.cc"], hdrs = ["gcs_node_manager.h"], - implementation_deps = [ - "//src/ray/gcs:gcs_pb_util", - "@com_google_absl//absl/container:flat_hash_set", - ], deps = [ ":gcs_init_data", ":gcs_table_storage", @@ -79,14 +75,43 @@ ray_cc_library( "//src/ray/common:asio", "//src/ray/common:id", "//src/ray/common:ray_config", + "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/pubsub:gcs_pub_sub_lib", + "//src/ray/protobuf:autoscaler_cc_proto", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/protobuf:ray_syncer_cc_proto", "//src/ray/rpc:node_manager_client", + "//src/ray/stats:stats_metric", "//src/ray/util:event", "//src/ray/util:logging", "//src/ray/util:time", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + ], +) + +ray_cc_library( + name = "gcs_resource_manager", + srcs = ["gcs_resource_manager.cc"], + hdrs = ["gcs_resource_manager.h"], + deps = [ + ":gcs_init_data", + ":gcs_node_manager", + ":gcs_state_util", + ":grpc_service_interfaces", + "//src/ray/common:asio", + "//src/ray/common:id", + "//src/ray/common:ray_config", + "//src/ray/common:ray_syncer", + "//src/ray/common:task_common", + "//src/ray/protobuf:gcs_service_cc_proto", + "//src/ray/protobuf:ray_syncer_cc_proto", + "//src/ray/raylet/scheduling:cluster_resource_manager", + "//src/ray/raylet/scheduling:cluster_task_manager", + "//src/ray/rpc:gcs_server", + "//src/ray/stats:stats_metric", + "//src/ray/util:logging", + "@com_google_absl//absl/container:flat_hash_map", ], ) @@ -206,18 +231,16 @@ ray_cc_library( name = "gcs_job_manager", srcs = ["gcs_job_manager.cc"], hdrs = ["gcs_job_manager.h"], - implementation_deps = [ + deps = [ ":gcs_function_manager", ":gcs_init_data", ":gcs_table_storage", - "//src/ray/gcs:gcs_pb_util", - "//src/ray/gcs/pubsub:gcs_pub_sub_lib", - "//src/ray/stats:stats_metric", - ], - deps = [ ":grpc_service_interfaces", "//src/ray/common:runtime_env", + "//src/ray/gcs:gcs_pb_util", + "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/rpc:core_worker_client", + "//src/ray/stats:stats_metric", "//src/ray/util:event", "//src/ray/util:thread_checker", "@com_google_absl//absl/container:flat_hash_map", @@ -284,7 +307,6 @@ ray_cc_library( "gcs_autoscaler_state_manager.cc", "gcs_placement_group_mgr.cc", "gcs_placement_group_scheduler.cc", - "gcs_resource_manager.cc", "gcs_server.cc", ], hdrs = [ @@ -293,7 +315,6 @@ ray_cc_library( "gcs_autoscaler_state_manager.h", "gcs_placement_group_mgr.h", "gcs_placement_group_scheduler.h", - "gcs_resource_manager.h", "gcs_server.h", ], deps = [ @@ -305,6 +326,7 @@ ray_cc_library( ":gcs_kv_manager", ":gcs_node_manager", ":gcs_pubsub_handler", + ":gcs_resource_manager", ":gcs_runtime_env_handler", ":gcs_server_io_context_policy", ":gcs_state_util", @@ -350,6 +372,7 @@ ray_cc_binary( deps = [ ":gcs_server_lib", "//src/ray/stats:stats_lib", + "//src/ray/util:event", "//src/ray/util:raii", "//src/ray/util:stream_redirection", "//src/ray/util:stream_redirection_options", diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.cc b/src/ray/gcs/gcs_server/gcs_job_manager.cc index ae55183c416c..ec0e251d5b20 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_job_manager.cc @@ -22,9 +22,6 @@ #include #include "absl/strings/match.h" -#include "ray/gcs/gcs_server/gcs_function_manager.h" -#include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/pb_util.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" #include "ray/stats/metric.h" diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.h b/src/ray/gcs/gcs_server/gcs_job_manager.h index 97da33b4b478..00b21286a372 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.h +++ b/src/ray/gcs/gcs_server/gcs_job_manager.h @@ -22,7 +22,12 @@ #include "absl/container/flat_hash_map.h" #include "ray/common/runtime_env_manager.h" +#include "ray/gcs/gcs_server/gcs_function_manager.h" +#include "ray/gcs/gcs_server/gcs_init_data.h" +#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" +#include "ray/gcs/pubsub/gcs_pub_sub.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/event.h" @@ -44,11 +49,7 @@ inline std::string JobDataKey(const std::string &submission_id) { using JobFinishListenerCallback = rpc::JobInfoGcsServiceHandler::JobFinishListenerCallback; -class GcsInitData; -class GcsTableStorage; class GcsPublisher; -class GCSFunctionManager; -class InternalKVInterface; class GcsJobManager : public rpc::JobInfoGcsServiceHandler { public: diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.h b/src/ray/gcs/gcs_server/gcs_node_manager.h index d54536c4ec49..710899260779 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/ray/gcs/gcs_server/gcs_node_manager.h @@ -21,17 +21,21 @@ #include #include "absl/container/flat_hash_map.h" +#include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" #include "ray/rpc/node_manager/raylet_client_pool.h" +#include "ray/stats/metric_defs.h" #include "ray/util/event.h" +#include "src/ray/protobuf/autoscaler.pb.h" #include "src/ray/protobuf/gcs.pb.h" #include "src/ray/protobuf/ray_syncer.pb.h" -namespace ray::gcs { +namespace ray { +namespace gcs { class GcsAutoscalerStateManagerTest; class GcsStateTest; @@ -46,7 +50,7 @@ class GcsNodeManager : public rpc::NodeInfoGcsServiceHandler { /// \param gcs_publisher GCS message publisher. /// \param gcs_table_storage GCS table external storage accessor. GcsNodeManager(GcsPublisher *gcs_publisher, - gcs::GcsTableStorage *gcs_table_storage, + GcsTableStorage *gcs_table_storage, instrumented_io_context &io_context, rpc::RayletClientPool *raylet_client_pool, const ClusterID &cluster_id); @@ -260,7 +264,7 @@ class GcsNodeManager : public rpc::NodeInfoGcsServiceHandler { /// A publisher for publishing gcs messages. GcsPublisher *gcs_publisher_; /// Storage for GCS tables. - gcs::GcsTableStorage *gcs_table_storage_; + GcsTableStorage *gcs_table_storage_; instrumented_io_context &io_context_; /// Raylet client pool. rpc::RayletClientPool *raylet_client_pool_; @@ -289,4 +293,5 @@ class GcsNodeManager : public rpc::NodeInfoGcsServiceHandler { friend GcsStateTest; }; -} // namespace ray::gcs +} // namespace gcs +} // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h index 80365c542c87..37d1afdcf58d 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h @@ -34,6 +34,7 @@ #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/usage_stats_client.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" +#include "ray/rpc/gcs/gcs_rpc_server.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/util/counter_map.h" #include "ray/util/exponential_backoff.h" diff --git a/src/ray/gcs/gcs_server/gcs_resource_manager.cc b/src/ray/gcs/gcs_server/gcs_resource_manager.cc index 373ec4c5dae5..e73f834bf816 100644 --- a/src/ray/gcs/gcs_server/gcs_resource_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_resource_manager.cc @@ -19,7 +19,8 @@ #include #include "ray/common/ray_config.h" -#include "ray/stats/metric_defs.h" +#include "ray/gcs/gcs_server/state_util.h" +#include "ray/util/logging.h" namespace ray { namespace gcs { @@ -28,7 +29,7 @@ GcsResourceManager::GcsResourceManager(instrumented_io_context &io_context, ClusterResourceManager &cluster_resource_manager, GcsNodeManager &gcs_node_manager, NodeID local_node_id, - ClusterTaskManager *cluster_task_manager) + raylet::ClusterTaskManager *cluster_task_manager) : io_context_(io_context), cluster_resource_manager_(cluster_resource_manager), gcs_node_manager_(gcs_node_manager), @@ -36,7 +37,7 @@ GcsResourceManager::GcsResourceManager(instrumented_io_context &io_context, cluster_task_manager_(cluster_task_manager) {} void GcsResourceManager::ConsumeSyncMessage( - std::shared_ptr message) { + std::shared_ptr message) { // ConsumeSyncMessage is called by ray_syncer which might not run // in a dedicated thread for performance. // GcsResourceManager is a module always run in the main thread, so we just diff --git a/src/ray/gcs/gcs_server/gcs_resource_manager.h b/src/ray/gcs/gcs_server/gcs_resource_manager.h index ab523b33bd23..730141ff220f 100644 --- a/src/ray/gcs/gcs_server/gcs_resource_manager.h +++ b/src/ray/gcs/gcs_server/gcs_resource_manager.h @@ -20,27 +20,21 @@ #include #include "absl/container/flat_hash_map.h" -#include "absl/container/flat_hash_set.h" +#include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" #include "ray/common/ray_syncer/ray_syncer.h" #include "ray/common/scheduling/cluster_resource_data.h" #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/gcs_server/state_util.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" #include "ray/raylet/scheduling/cluster_task_manager.h" -#include "ray/rpc/client_call.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" +#include "ray/stats/metric_defs.h" #include "src/ray/protobuf/gcs.pb.h" +#include "src/ray/protobuf/ray_syncer.pb.h" namespace ray { - -using raylet::ClusterTaskManager; - namespace gcs { -class GcsNodeManager; -class GcsServer; /// Ideally, the logic related to resource calculation should be moved from /// `gcs_resource_manager` to `cluster_resource_manager`, and all logic related to @@ -60,7 +54,7 @@ class GcsServer; /// It is responsible for handing node resource related rpc requests and it is used for /// actor and placement group scheduling. It obtains the available resources of nodes /// through heartbeat reporting. Non-thread safe. -class GcsResourceManager : public rpc::NodeResourceInfoHandler, +class GcsResourceManager : public rpc::NodeResourceInfoGcsServiceHandler, public syncer::ReceiverInterface { public: /// Create a GcsResourceManager. @@ -68,12 +62,13 @@ class GcsResourceManager : public rpc::NodeResourceInfoHandler, ClusterResourceManager &cluster_resource_manager, GcsNodeManager &gcs_node_manager, NodeID local_node_id, - ClusterTaskManager *cluster_task_manager = nullptr); + raylet::ClusterTaskManager *cluster_task_manager = nullptr); virtual ~GcsResourceManager() = default; /// Handle the resource update. - void ConsumeSyncMessage(std::shared_ptr message) override; + void ConsumeSyncMessage( + std::shared_ptr message) override; /// Handle get available resources of all nodes. /// Autoscaler-specific RPC called from Python. @@ -202,7 +197,7 @@ class GcsResourceManager : public rpc::NodeResourceInfoHandler, ClusterResourceManager &cluster_resource_manager_; GcsNodeManager &gcs_node_manager_; NodeID local_node_id_; - ClusterTaskManager *cluster_task_manager_; + raylet::ClusterTaskManager *cluster_task_manager_; /// Num of alive nodes in the cluster. size_t num_alive_nodes_ = 0; }; diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index f38223c44afb..890d958d72c5 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -361,7 +361,9 @@ void GcsServer::InitGcsResourceManager(const GcsInitData &gcs_init_data) { // Initialize by gcs tables data. gcs_resource_manager_->Initialize(gcs_init_data); rpc_server_.RegisterService(std::make_unique( - io_context_provider_.GetDefaultIOContext(), *gcs_resource_manager_)); + io_context_provider_.GetDefaultIOContext(), + *gcs_resource_manager_, + RayConfig::instance().gcs_max_active_rpcs_per_handler())); periodical_runner_->RunFnPeriodically( [this] { diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h index 4721344af4ae..f1cf9b70f538 100644 --- a/src/ray/gcs/gcs_server/grpc_service_interfaces.h +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -64,6 +64,27 @@ class NodeInfoGcsServiceHandler { SendReplyCallback send_reply_callback) = 0; }; +class NodeResourceInfoGcsServiceHandler { + public: + virtual ~NodeResourceInfoGcsServiceHandler() = default; + + virtual void HandleGetAllAvailableResources(GetAllAvailableResourcesRequest request, + GetAllAvailableResourcesReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetAllTotalResources(GetAllTotalResourcesRequest request, + GetAllTotalResourcesReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetDrainingNodes(GetDrainingNodesRequest request, + GetDrainingNodesReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetAllResourceUsage(GetAllResourceUsageRequest request, + GetAllResourceUsageReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + class InternalPubSubGcsServiceHandler { public: virtual ~InternalPubSubGcsServiceHandler() = default; diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc index 897cf18e234c..4157256be205 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -36,6 +36,20 @@ void NodeInfoGrpcService::InitServerCallFactories( RPC_SERVICE_HANDLER(NodeInfoGcsService, CheckAlive, max_active_rpcs_per_handler_) } +void NodeResourceInfoGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + RPC_SERVICE_HANDLER( + NodeResourceInfoGcsService, GetAllAvailableResources, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + NodeResourceInfoGcsService, GetAllTotalResources, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + NodeResourceInfoGcsService, GetDrainingNodes, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + NodeResourceInfoGcsService, GetAllResourceUsage, max_active_rpcs_per_handler_) +} + void InternalPubSubGrpcService::InitServerCallFactories( const std::unique_ptr &cq, std::vector> *server_call_factories, diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/gcs_server/grpc_services.h index b8f6b69f8180..0a463c2b6b6f 100644 --- a/src/ray/gcs/gcs_server/grpc_services.h +++ b/src/ray/gcs/gcs_server/grpc_services.h @@ -58,6 +58,29 @@ class NodeInfoGrpcService : public GrpcService { int64_t max_active_rpcs_per_handler_; }; +class NodeResourceInfoGrpcService : public GrpcService { + public: + explicit NodeResourceInfoGrpcService(instrumented_io_context &io_service, + NodeResourceInfoGcsServiceHandler &handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler){}; + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + NodeResourceInfoGcsService::AsyncService service_; + NodeResourceInfoGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + class InternalPubSubGrpcService : public GrpcService { public: InternalPubSubGrpcService(instrumented_io_context &io_service, diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index ccb97f0489f0..417989a559c4 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -68,7 +68,10 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_kv_manager", + "//src/ray/gcs/gcs_server:gcs_store_client_kv", + "//src/ray/gcs/store_client:in_memory_store_client", + "//src/ray/gcs/store_client:redis_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest", ], @@ -96,7 +99,8 @@ ray_cc_test( "team:core", ], deps = [ - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_health_check_manager", + "//src/ray/rpc:grpc_server", "//src/ray/util:network_util", "@boost//:thread", "@com_google_googletest//:gtest_main", @@ -127,10 +131,11 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - ":gcs_server_test_util", "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_job_manager", + "//src/ray/gcs/gcs_server:gcs_kv_manager", + "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], @@ -146,7 +151,7 @@ ray_cc_test( deps = [ ":gcs_server_test_util", "//:ray_mock", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_task_manager", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], @@ -250,9 +255,10 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - ":gcs_server_test_util", "//src/mock/ray/pubsub:mock_publisher", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_store_client_kv", + "//src/ray/gcs/gcs_server:gcs_worker_manager", + "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:process", "@com_google_googletest//:gtest_main", @@ -334,8 +340,10 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_node_manager", + "//src/ray/gcs/gcs_server:gcs_resource_manager", "//src/ray/gcs/tests:gcs_test_util_lib", + "//src/ray/raylet/scheduling:cluster_resource_manager", "@com_google_googletest//:gtest_main", ], ) @@ -348,9 +356,9 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - ":gcs_server_test_util", "//:ray_mock", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/common:asio", + "//src/ray/gcs/gcs_server:gcs_usage_stats_client", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], @@ -365,10 +373,10 @@ ray_cc_test( "team:core", ], deps = [ - ":gcs_server_test_util", "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_job_manager", + "//src/ray/gcs/gcs_server:gcs_kv_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc index 0b5c6494ea3a..dfa789f79805 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include #include #include @@ -20,18 +22,15 @@ #include #include -// clang-format off -#include "gtest/gtest.h" +#include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" +#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" -#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/pubsub/publisher.h" #include "ray/util/event.h" -// clang-format on namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc index c4cd529da5df..7d2581b5cf44 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include #include #include -#include "gtest/gtest.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/pubsub/publisher.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/gcs/gcs_server/gcs_job_manager.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/tests/gcs_test_util.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc index 5f3f587c17b6..5507a24b0461 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc @@ -30,6 +30,8 @@ #include "mock/ray/gcs/store_client/store_client.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/gcs/gcs_server/gcs_init_data.h" +#include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/store_client_kv.h" #include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/tests/gcs_test_util.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc index f2db28735bad..10b8409dc844 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc @@ -19,7 +19,6 @@ #include #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" -#include "ray/gcs/gcs_server/gcs_function_manager.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc index b41e4d727e8f..5fe6fad4aa96 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc @@ -22,7 +22,6 @@ #include "mock/ray/pubsub/publisher.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/tests/gcs_test_util.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc index 0d34847e9ced..def15f75e97d 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc @@ -19,17 +19,15 @@ #include #include -#include "ray/util/process.h" - -// clang-format off +#include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/gcs_server/store_client_kv.h" +#include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/tests/gcs_test_util.h" -#include "mock/ray/pubsub/publisher.h" -#include "src/ray/protobuf/gcs.pb.h" +#include "ray/util/process.h" #include "src/ray/protobuf/common.pb.h" -#include "ray/gcs/gcs_server/store_client_kv.h" -// clang-format on +#include "src/ray/protobuf/gcs.pb.h" + using namespace ::testing; // NOLINT using namespace ray::gcs; // NOLINT using namespace ray; // NOLINT diff --git a/src/ray/gcs/gcs_server/tests/usage_stats_client_test.cc b/src/ray/gcs/gcs_server/tests/usage_stats_client_test.cc index fbe054f4bebc..7a0b7ffea376 100644 --- a/src/ray/gcs/gcs_server/tests/usage_stats_client_test.cc +++ b/src/ray/gcs/gcs_server/tests/usage_stats_client_test.cc @@ -14,13 +14,14 @@ #include "ray/gcs/gcs_server/usage_stats_client.h" +#include + #include #include -#include "gtest/gtest.h" #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/common/asio/asio_util.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" -#include "ray/gcs/gcs_server/gcs_server.h" using namespace ray; // NOLINT diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index 7843b9ad8d17..7b812c13739e 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -127,11 +127,6 @@ namespace rpc { HANDLER, \ RayConfig::instance().gcs_max_active_rpcs_per_handler()) -#define NODE_RESOURCE_INFO_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(NodeResourceInfoGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - #define OBJECT_INFO_SERVICE_RPC_HANDLER(HANDLER) \ RPC_SERVICE_HANDLER(ObjectInfoGcsService, \ HANDLER, \ @@ -234,58 +229,6 @@ class ActorInfoGrpcService : public GrpcService { ActorInfoGcsServiceHandler &service_handler_; }; -class NodeResourceInfoGcsServiceHandler { - public: - virtual ~NodeResourceInfoGcsServiceHandler() = default; - - virtual void HandleGetAllAvailableResources( - rpc::GetAllAvailableResourcesRequest request, - rpc::GetAllAvailableResourcesReply *reply, - rpc::SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetAllTotalResources(rpc::GetAllTotalResourcesRequest request, - rpc::GetAllTotalResourcesReply *reply, - rpc::SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetDrainingNodes(rpc::GetDrainingNodesRequest request, - rpc::GetDrainingNodesReply *reply, - rpc::SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetAllResourceUsage(GetAllResourceUsageRequest request, - GetAllResourceUsageReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -/// The `GrpcService` for `NodeResourceInfoGcsService`. -class NodeResourceInfoGrpcService : public GrpcService { - public: - /// Constructor. - /// - /// \param[in] handler The service handler that actually handle the requests. - explicit NodeResourceInfoGrpcService(instrumented_io_context &io_service, - NodeResourceInfoGcsServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler){}; - - protected: - grpc::Service &GetGrpcService() override { return service_; } - - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - NODE_RESOURCE_INFO_SERVICE_RPC_HANDLER(GetAllAvailableResources); - NODE_RESOURCE_INFO_SERVICE_RPC_HANDLER(GetAllTotalResources); - NODE_RESOURCE_INFO_SERVICE_RPC_HANDLER(GetDrainingNodes); - NODE_RESOURCE_INFO_SERVICE_RPC_HANDLER(GetAllResourceUsage); - } - - private: - /// The grpc async service object. - NodeResourceInfoGcsService::AsyncService service_; - /// The service handler that actually handle the requests. - NodeResourceInfoGcsServiceHandler &service_handler_; -}; - class PlacementGroupInfoGcsServiceHandler { public: virtual ~PlacementGroupInfoGcsServiceHandler() = default; @@ -422,7 +365,6 @@ class RayEventExportGrpcService : public GrpcService { }; using ActorInfoHandler = ActorInfoGcsServiceHandler; -using NodeResourceInfoHandler = NodeResourceInfoGcsServiceHandler; using PlacementGroupInfoHandler = PlacementGroupInfoGcsServiceHandler; using TaskInfoHandler = TaskInfoGcsServiceHandler; using RayEventExportHandler = RayEventExportGcsServiceHandler; From 028965089883cd56dcb0cc5a0e2e683e9f684fe1 Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Wed, 27 Aug 2025 15:06:13 -0700 Subject: [PATCH 308/634] Support cpu tensor transfer with NIXL in GPU Objects (#55793) Since nixl actually supports cpu tensor transfer, we should also support it in ray gpu objects. Close #55587 --- .../channel/serialization_context.py | 4 ++- .../collective/collective_tensor_transport.py | 16 ++++++++-- .../collective/nixl_tensor_transport.py | 13 ++++++++- .../collective/tensor_transport_manager.py | 2 -- python/ray/experimental/collective/util.py | 17 ++++++++++- .../gpu_object_manager/gpu_object_store.py | 29 +++++++++---------- python/ray/tests/test_gpu_objects_nixl.py | 24 ++++++++++----- python/ray/util/collective/types.py | 3 ++ 8 files changed, 79 insertions(+), 29 deletions(-) diff --git a/python/ray/experimental/channel/serialization_context.py b/python/ray/experimental/channel/serialization_context.py index 40784b38f409..d06a2d9e098b 100644 --- a/python/ray/experimental/channel/serialization_context.py +++ b/python/ray/experimental/channel/serialization_context.py @@ -97,7 +97,9 @@ def serialize_tensor( from ray.experimental.channel import ChannelContext ctx = ChannelContext.get_current() - if self._use_external_transport and tensor.device == ctx.torch_device: + if self._use_external_transport and ( + ctx._torch_device is None or ctx._torch_device == tensor.device + ): # External transport is enabled and we found a tensor that matches # our device. Add the actual tensor to a buffer. The buffer of # tensors should later be popped by the caller and sent via diff --git a/python/ray/experimental/collective/collective_tensor_transport.py b/python/ray/experimental/collective/collective_tensor_transport.py index 7bf39aed7b25..f6766cbd6f36 100644 --- a/python/ray/experimental/collective/collective_tensor_transport.py +++ b/python/ray/experimental/collective/collective_tensor_transport.py @@ -40,8 +40,19 @@ def __ray_get_tensor_transport_metadata__( # it could take arbitrarily long and we don't want to trigger a spurious # timeout. gpu_object = gpu_object_store.wait_and_get_object(obj_id) + tensor_meta = [] + device = None + if gpu_object: + device = gpu_object[0].device + for t in gpu_object: + if t.device.type != device.type: + raise ValueError( + "All tensors in one GPU object must be the same device type." + ) + tensor_meta.append((t.shape, t.dtype)) return CollectiveTransportMetadata( - tensor_meta=[(t.shape, t.dtype) for t in gpu_object], + tensor_meta=tensor_meta, + tensor_device=device, ) # Submit a Ray actor task to the source actor to get the tensor metadata. @@ -130,10 +141,11 @@ def recv_multiple_tensors( def send_multiple_tensors( tensors: List["torch.Tensor"], communicator_metadata: CollectiveCommunicatorMetadata, - device: "torch.device", ): import ray.util.collective as collective + device = tensor_transport_metadata.tensor_device + for tensor in tensors: if tensor.device.type != device.type: # TODO(swang): Right now there is no way to catch this error diff --git a/python/ray/experimental/collective/nixl_tensor_transport.py b/python/ray/experimental/collective/nixl_tensor_transport.py index eb86f0cb9a3d..dcedbc11bf41 100644 --- a/python/ray/experimental/collective/nixl_tensor_transport.py +++ b/python/ray/experimental/collective/nixl_tensor_transport.py @@ -45,14 +45,25 @@ def __ray_get_tensor_transport_metadata__( from ray.util.collective.collective import get_group_handle nixl_backend: NixlBackend = get_group_handle(NIXL_GROUP_NAME) + device = None + tensor_meta = [] if gpu_object: serialized_descs, agent_meta = nixl_backend.get_nixl_metadata( gpu_object ) + # We assume all tensors in one GPU object have the same device type. + device = gpu_object[0].device + for t in gpu_object: + if t.device.type != device.type: + raise ValueError( + "All tensors in one GPU object must be the same device type." + ) + tensor_meta.append((t.shape, t.dtype)) else: serialized_descs, agent_meta = None, None return NixlTransportMetadata( - tensor_meta=[(t.shape, t.dtype) for t in gpu_object], + tensor_meta=tensor_meta, + tensor_device=device, nixl_serialized_descs=serialized_descs, nixl_agent_meta=agent_meta, ) diff --git a/python/ray/experimental/collective/tensor_transport_manager.py b/python/ray/experimental/collective/tensor_transport_manager.py index 9f3896699393..18d554d2c6d7 100644 --- a/python/ray/experimental/collective/tensor_transport_manager.py +++ b/python/ray/experimental/collective/tensor_transport_manager.py @@ -143,7 +143,6 @@ def recv_multiple_tensors( def send_multiple_tensors( tensors: List["torch.Tensor"], communicator_metadata: CommunicatorMetadata, - device: "torch.device", ): """ Send multiple tensors to the destination actor. @@ -151,5 +150,4 @@ def send_multiple_tensors( Args: tensors: The tensors to send. communicator_metadata: The communicator metadata for the send/recv operation. - device: The device to send the tensors to. """ diff --git a/python/ray/experimental/collective/util.py b/python/ray/experimental/collective/util.py index fc6ef64229fb..6e9297dba37c 100644 --- a/python/ray/experimental/collective/util.py +++ b/python/ray/experimental/collective/util.py @@ -1,4 +1,4 @@ -from typing import Tuple +from typing import Tuple, TYPE_CHECKING from contextlib import closing import socket @@ -11,6 +11,9 @@ CollectiveTensorTransport, ) +if TYPE_CHECKING: + import torch + # Singleton instances for tensor transport managers _nixl_tensor_transport_manager = None _collective_tensor_transport_manager = None @@ -41,6 +44,18 @@ def get_tensor_transport_manager( raise ValueError(f"Unsupported tensor transport protocol: {tensor_transport}") +def device_match_transport(device: "torch.device", tensor_transport: Backend) -> bool: + """Check if the device matches the transport.""" + if tensor_transport == Backend.NIXL: + return device.type == "cuda" or device.type == "cpu" + elif tensor_transport == Backend.TORCH_GLOO: + return device.type == "cpu" + elif tensor_transport == Backend.NCCL: + return device.type == "cuda" + else: + raise ValueError(f"Unsupported tensor transport protocol: {tensor_transport}") + + def find_free_port() -> int: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(("", 0)) diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_store.py b/python/ray/experimental/gpu_object_manager/gpu_object_store.py index 93fcbc20f54d..01fdbccaf43e 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_store.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_store.py @@ -11,6 +11,9 @@ TensorTransportMetadata, ) +from ray.experimental.collective import get_tensor_transport_manager +from ray.experimental.collective.util import device_match_transport + try: import torch except ImportError: @@ -25,14 +28,6 @@ TensorTransportEnum.NIXL: Backend.NIXL, } -COLLECTIVE_BACKEND_TO_TORCH_DEVICE = { - Backend.NCCL: torch.device("cuda"), - Backend.TORCH_GLOO: torch.device("cpu"), - # TODO(Qiaolin-Yu): NIXL could also transfer tensors from CPU to CPU. - # More details in https://github.com/ray-project/ray/issues/55587. - Backend.NIXL: torch.device("cuda"), -} - def _tensor_transport_to_collective_backend( tensor_transport: TensorTransportEnum, @@ -61,15 +56,17 @@ def __ray_send__( tensors = gpu_object_store.get_object(obj_id) backend = collective.get_group_handle(communicator_meta.communicator_name).backend() - device = COLLECTIVE_BACKEND_TO_TORCH_DEVICE[backend] - - from ray.experimental.collective import get_tensor_transport_manager tensor_transport_manager = get_tensor_transport_manager(backend) + if tensors and not device_match_transport( + tensor_transport_meta.tensor_device, backend + ): + raise ValueError( + f"Tensor transport backend {backend} does not support tensor transfer on device {tensor_transport_meta.tensor_device}." + ) tensor_transport_manager.send_multiple_tensors( tensors, communicator_meta, - device=device, ) @@ -82,14 +79,16 @@ def __ray_recv__( """Helper function that runs on the dst actor to receive tensors from the src actor.""" from ray._private.worker import global_worker - from ray.experimental.collective import get_tensor_transport_manager - backend = collective.get_group_handle(communicator_meta.communicator_name).backend() - device = COLLECTIVE_BACKEND_TO_TORCH_DEVICE[backend] + device = tensor_transport_meta.tensor_device tensor_meta = tensor_transport_meta.tensor_meta gpu_object_store = global_worker.gpu_object_manager.gpu_object_store + if tensor_meta and not device_match_transport(device, backend): + raise ValueError( + f"Tensor transport backend {backend} does not support tensor transfer on device {device}." + ) tensors = [] for meta in tensor_meta: shape, dtype = meta diff --git a/python/ray/tests/test_gpu_objects_nixl.py b/python/ray/tests/test_gpu_objects_nixl.py index e46a24358d37..3a65e8ea7eb9 100644 --- a/python/ray/tests/test_gpu_objects_nixl.py +++ b/python/ray/tests/test_gpu_objects_nixl.py @@ -7,10 +7,11 @@ @ray.remote(num_gpus=1, num_cpus=0, enable_tensor_transport=True) class GPUTestActor: @ray.method(tensor_transport="nixl") - def echo(self, data): - return data.to("cuda") + def echo(self, data, device): + return data.to(device) - def sum(self, data): + def sum(self, data, device): + assert data.device.type == device return data.sum().item() @@ -23,12 +24,21 @@ def test_p2p(ray_start_regular): # Create test tensor tensor = torch.tensor([1, 2, 3]) - ref = src_actor.echo.remote(tensor) + + tensor1 = torch.tensor([4, 5, 6]) + + # Test GPU to GPU transfer + ref = src_actor.echo.remote(tensor, "cuda") # Trigger tensor transfer from src to dst actor - result = dst_actor.sum.remote(ref) + result = dst_actor.sum.remote(ref, "cuda") assert tensor.sum().item() == ray.get(result) + # Test CPU to CPU transfer + ref1 = src_actor.echo.remote(tensor1, "cpu") + result1 = dst_actor.sum.remote(ref1, "cpu") + assert tensor1.sum().item() == ray.get(result1) + @pytest.mark.parametrize("ray_start_regular", [{"num_gpus": 1}], indirect=True) def test_intra_gpu_tensor_transfer(ray_start_regular): @@ -37,8 +47,8 @@ def test_intra_gpu_tensor_transfer(ray_start_regular): tensor = torch.tensor([1, 2, 3]) # Intra-actor communication for pure GPU tensors - ref = actor.echo.remote(tensor) - result = actor.sum.remote(ref) + ref = actor.echo.remote(tensor, "cuda") + result = actor.sum.remote(ref, "cuda") assert tensor.sum().item() == ray.get(result) diff --git a/python/ray/util/collective/types.py b/python/ray/util/collective/types.py index 06a05ae71549..e46737c5a033 100644 --- a/python/ray/util/collective/types.py +++ b/python/ray/util/collective/types.py @@ -61,9 +61,12 @@ class TensorTransportMetadata: Args: tensor_meta: A list of tuples, each containing the shape and dtype of a tensor. + tensor_device: The device of the tensor. Currently, we require all tensors in the + list have the same device type. """ tensor_meta: List[Tuple["torch.Size", "torch.dtype"]] + tensor_device: Optional["torch.device"] = None @dataclass From b4488cdc278ec0249ddd04a367f6f9810a5035dc Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Wed, 27 Aug 2025 15:06:33 -0700 Subject: [PATCH 309/634] [deps] upgrading orjson to 3.9.15 (#55972) upgrading orjson due to security vulnerability https://github.com/ray-project/ray/security/dependabot/3692 --------- Signed-off-by: elliot-barn --- python/requirements_compiled.txt | 2 +- .../byod/requirements_byod_3.9.txt | 102 +++++++++--------- .../byod/requirements_ml_byod_3.9.txt | 102 +++++++++--------- 3 files changed, 103 insertions(+), 103 deletions(-) diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt index 27231510d8bc..f66c65c7b33f 100644 --- a/python/requirements_compiled.txt +++ b/python/requirements_compiled.txt @@ -1350,7 +1350,7 @@ opt-einsum==3.3.0 # tensorflow optuna==4.1.0 # via -r python/requirements/ml/tune-requirements.txt -orjson==3.9.10 +orjson==3.9.15 # via # gradio # tensordict diff --git a/release/ray_release/byod/requirements_byod_3.9.txt b/release/ray_release/byod/requirements_byod_3.9.txt index 91b7f97daaad..982e72c18612 100644 --- a/release/ray_release/byod/requirements_byod_3.9.txt +++ b/release/ray_release/byod/requirements_byod_3.9.txt @@ -1837,57 +1837,57 @@ opt-einsum==3.3.0 \ # via # -c release/ray_release/byod/requirements_compiled.txt # tensorflow -orjson==3.9.10 \ - --hash=sha256:06ad5543217e0e46fd7ab7ea45d506c76f878b87b1b4e369006bdb01acc05a83 \ - --hash=sha256:0a73160e823151f33cdc05fe2cea557c5ef12fdf276ce29bb4f1c571c8368a60 \ - --hash=sha256:1234dc92d011d3554d929b6cf058ac4a24d188d97be5e04355f1b9223e98bbe9 \ - --hash=sha256:1d0dc4310da8b5f6415949bd5ef937e60aeb0eb6b16f95041b5e43e6200821fb \ - --hash=sha256:2a11b4b1a8415f105d989876a19b173f6cdc89ca13855ccc67c18efbd7cbd1f8 \ - --hash=sha256:2e2ecd1d349e62e3960695214f40939bbfdcaeaaa62ccc638f8e651cf0970e5f \ - --hash=sha256:3a2ce5ea4f71681623f04e2b7dadede3c7435dfb5e5e2d1d0ec25b35530e277b \ - --hash=sha256:3e892621434392199efb54e69edfff9f699f6cc36dd9553c5bf796058b14b20d \ - --hash=sha256:3fb205ab52a2e30354640780ce4587157a9563a68c9beaf52153e1cea9aa0921 \ - --hash=sha256:4689270c35d4bb3102e103ac43c3f0b76b169760aff8bcf2d401a3e0e58cdb7f \ - --hash=sha256:49f8ad582da6e8d2cf663c4ba5bf9f83cc052570a3a767487fec6af839b0e777 \ - --hash=sha256:4bd176f528a8151a6efc5359b853ba3cc0e82d4cd1fab9c1300c5d957dc8f48c \ - --hash=sha256:4cf7837c3b11a2dfb589f8530b3cff2bd0307ace4c301e8997e95c7468c1378e \ - --hash=sha256:4fd72fab7bddce46c6826994ce1e7de145ae1e9e106ebb8eb9ce1393ca01444d \ - --hash=sha256:5148bab4d71f58948c7c39d12b14a9005b6ab35a0bdf317a8ade9a9e4d9d0bd5 \ - --hash=sha256:5869e8e130e99687d9e4be835116c4ebd83ca92e52e55810962446d841aba8de \ - --hash=sha256:602a8001bdf60e1a7d544be29c82560a7b49319a0b31d62586548835bbe2c862 \ - --hash=sha256:61804231099214e2f84998316f3238c4c2c4aaec302df12b21a64d72e2a135c7 \ - --hash=sha256:666c6fdcaac1f13eb982b649e1c311c08d7097cbda24f32612dae43648d8db8d \ - --hash=sha256:674eb520f02422546c40401f4efaf8207b5e29e420c17051cddf6c02783ff5ca \ - --hash=sha256:7ec960b1b942ee3c69323b8721df2a3ce28ff40e7ca47873ae35bfafeb4555ca \ - --hash=sha256:7f433be3b3f4c66016d5a20e5b4444ef833a1f802ced13a2d852c637f69729c1 \ - --hash=sha256:7f8fb7f5ecf4f6355683ac6881fd64b5bb2b8a60e3ccde6ff799e48791d8f864 \ - --hash=sha256:81a3a3a72c9811b56adf8bcc829b010163bb2fc308877e50e9910c9357e78521 \ - --hash=sha256:858379cbb08d84fe7583231077d9a36a1a20eb72f8c9076a45df8b083724ad1d \ - --hash=sha256:8b9ba0ccd5a7f4219e67fbbe25e6b4a46ceef783c42af7dbc1da548eb28b6531 \ - --hash=sha256:92af0d00091e744587221e79f68d617b432425a7e59328ca4c496f774a356071 \ - --hash=sha256:9ebbdbd6a046c304b1845e96fbcc5559cd296b4dfd3ad2509e33c4d9ce07d6a1 \ - --hash=sha256:9edd2856611e5050004f4722922b7b1cd6268da34102667bd49d2a2b18bafb81 \ - --hash=sha256:a353bf1f565ed27ba71a419b2cd3db9d6151da426b61b289b6ba1422a702e643 \ - --hash=sha256:b5b7d4a44cc0e6ff98da5d56cde794385bdd212a86563ac321ca64d7f80c80d1 \ - --hash=sha256:b90f340cb6397ec7a854157fac03f0c82b744abdd1c0941a024c3c29d1340aff \ - --hash=sha256:c18a4da2f50050a03d1da5317388ef84a16013302a5281d6f64e4a3f406aabc4 \ - --hash=sha256:c338ed69ad0b8f8f8920c13f529889fe0771abbb46550013e3c3d01e5174deef \ - --hash=sha256:c5a02360e73e7208a872bf65a7554c9f15df5fe063dc047f79738998b0506a14 \ - --hash=sha256:c62b6fa2961a1dcc51ebe88771be5319a93fd89bd247c9ddf732bc250507bc2b \ - --hash=sha256:c812312847867b6335cfb264772f2a7e85b3b502d3a6b0586aa35e1858528ab1 \ - --hash=sha256:c943b35ecdf7123b2d81d225397efddf0bce2e81db2f3ae633ead38e85cd5ade \ - --hash=sha256:ce0a29c28dfb8eccd0f16219360530bc3cfdf6bf70ca384dacd36e6c650ef8e8 \ - --hash=sha256:cf80b550092cc480a0cbd0750e8189247ff45457e5a023305f7ef1bcec811616 \ - --hash=sha256:cff7570d492bcf4b64cc862a6e2fb77edd5e5748ad715f487628f102815165e9 \ - --hash=sha256:d2c1e559d96a7f94a4f581e2a32d6d610df5840881a8cba8f25e446f4d792df3 \ - --hash=sha256:deeb3922a7a804755bbe6b5be9b312e746137a03600f488290318936c1a2d4dc \ - --hash=sha256:e28a50b5be854e18d54f75ef1bb13e1abf4bc650ab9d635e4258c58e71eb6ad5 \ - --hash=sha256:e99c625b8c95d7741fe057585176b1b8783d46ed4b8932cf98ee145c4facf499 \ - --hash=sha256:ec6f18f96b47299c11203edfbdc34e1b69085070d9a3d1f302810cc23ad36bf3 \ - --hash=sha256:ed8bc367f725dfc5cabeed1ae079d00369900231fbb5a5280cf0736c30e2adf7 \ - --hash=sha256:ee5926746232f627a3be1cc175b2cfad24d0170d520361f4ce3fa2fd83f09e1d \ - --hash=sha256:f295efcd47b6124b01255d1491f9e46f17ef40d3d7eabf7364099e463fb45f0f \ - --hash=sha256:fb0b361d73f6b8eeceba47cd37070b5e6c9de5beaeaa63a1cb35c7e1a73ef088 +orjson==3.9.15 \ + --hash=sha256:001f4eb0ecd8e9ebd295722d0cbedf0748680fb9998d3993abaed2f40587257a \ + --hash=sha256:05a1f57fb601c426635fcae9ddbe90dfc1ed42245eb4c75e4960440cac667262 \ + --hash=sha256:10c57bc7b946cf2efa67ac55766e41764b66d40cbd9489041e637c1304400494 \ + --hash=sha256:12365576039b1a5a47df01aadb353b68223da413e2e7f98c02403061aad34bde \ + --hash=sha256:2973474811db7b35c30248d1129c64fd2bdf40d57d84beed2a9a379a6f57d0ab \ + --hash=sha256:2b5c0f532905e60cf22a511120e3719b85d9c25d0e1c2a8abb20c4dede3b05a5 \ + --hash=sha256:2c51378d4a8255b2e7c1e5cc430644f0939539deddfa77f6fac7b56a9784160a \ + --hash=sha256:2d99e3c4c13a7b0fb3792cc04c2829c9db07838fb6973e578b85c1745e7d0ce7 \ + --hash=sha256:2f256d03957075fcb5923410058982aea85455d035607486ccb847f095442bda \ + --hash=sha256:34cbcd216e7af5270f2ffa63a963346845eb71e174ea530867b7443892d77180 \ + --hash=sha256:4228aace81781cc9d05a3ec3a6d2673a1ad0d8725b4e915f1089803e9efd2b99 \ + --hash=sha256:4feeb41882e8aa17634b589533baafdceb387e01e117b1ec65534ec724023d04 \ + --hash=sha256:57d5d8cf9c27f7ef6bc56a5925c7fbc76b61288ab674eb352c26ac780caa5b10 \ + --hash=sha256:5bb399e1b49db120653a31463b4a7b27cf2fbfe60469546baf681d1b39f4edf2 \ + --hash=sha256:62482873e0289cf7313461009bf62ac8b2e54bc6f00c6fabcde785709231a5d7 \ + --hash=sha256:67384f588f7f8daf040114337d34a5188346e3fae6c38b6a19a2fe8c663a2f9b \ + --hash=sha256:6ae4e06be04dc00618247c4ae3f7c3e561d5bc19ab6941427f6d3722a0875ef7 \ + --hash=sha256:6f7b65bfaf69493c73423ce9db66cfe9138b2f9ef62897486417a8fcb0a92bfe \ + --hash=sha256:6fc2fe4647927070df3d93f561d7e588a38865ea0040027662e3e541d592811e \ + --hash=sha256:71c6b009d431b3839d7c14c3af86788b3cfac41e969e3e1c22f8a6ea13139404 \ + --hash=sha256:7413070a3e927e4207d00bd65f42d1b780fb0d32d7b1d951f6dc6ade318e1b5a \ + --hash=sha256:76bc6356d07c1d9f4b782813094d0caf1703b729d876ab6a676f3aaa9a47e37c \ + --hash=sha256:7f6cbd8e6e446fb7e4ed5bac4661a29e43f38aeecbf60c4b900b825a353276a1 \ + --hash=sha256:8055ec598605b0077e29652ccfe9372247474375e0e3f5775c91d9434e12d6b1 \ + --hash=sha256:809d653c155e2cc4fd39ad69c08fdff7f4016c355ae4b88905219d3579e31eb7 \ + --hash=sha256:82425dd5c7bd3adfe4e94c78e27e2fa02971750c2b7ffba648b0f5d5cc016a73 \ + --hash=sha256:87f1097acb569dde17f246faa268759a71a2cb8c96dd392cd25c668b104cad2f \ + --hash=sha256:920fa5a0c5175ab14b9c78f6f820b75804fb4984423ee4c4f1e6d748f8b22bc1 \ + --hash=sha256:92255879280ef9c3c0bcb327c5a1b8ed694c290d61a6a532458264f887f052cb \ + --hash=sha256:946c3a1ef25338e78107fba746f299f926db408d34553b4754e90a7de1d44068 \ + --hash=sha256:95cae920959d772f30ab36d3b25f83bb0f3be671e986c72ce22f8fa700dae061 \ + --hash=sha256:9cf1596680ac1f01839dba32d496136bdd5d8ffb858c280fa82bbfeb173bdd40 \ + --hash=sha256:9fe41b6f72f52d3da4db524c8653e46243c8c92df826ab5ffaece2dba9cccd58 \ + --hash=sha256:b17f0f14a9c0ba55ff6279a922d1932e24b13fc218a3e968ecdbf791b3682b25 \ + --hash=sha256:b3d336ed75d17c7b1af233a6561cf421dee41d9204aa3cfcc6c9c65cd5bb69a8 \ + --hash=sha256:b66bcc5670e8a6b78f0313bcb74774c8291f6f8aeef10fe70e910b8040f3ab75 \ + --hash=sha256:b725da33e6e58e4a5d27958568484aa766e825e93aa20c26c91168be58e08cbb \ + --hash=sha256:b72758f3ffc36ca566ba98a8e7f4f373b6c17c646ff8ad9b21ad10c29186f00d \ + --hash=sha256:bcef128f970bb63ecf9a65f7beafd9b55e3aaf0efc271a4154050fc15cdb386e \ + --hash=sha256:c8e8fe01e435005d4421f183038fc70ca85d2c1e490f51fb972db92af6e047c2 \ + --hash=sha256:d61f7ce4727a9fa7680cd6f3986b0e2c732639f46a5e0156e550e35258aa313a \ + --hash=sha256:d6768a327ea1ba44c9114dba5fdda4a214bdb70129065cd0807eb5f010bfcbb5 \ + --hash=sha256:e18668f1bd39e69b7fed19fa7cd1cd110a121ec25439328b5c89934e6d30d357 \ + --hash=sha256:e88b97ef13910e5f87bcbc4dd7979a7de9ba8702b54d3204ac587e83639c0c2b \ + --hash=sha256:ea0b183a5fe6b2b45f3b854b0d19c4e932d6f5934ae1f723b07cf9560edd4ec7 \ + --hash=sha256:ede0bde16cc6e9b96633df1631fbcd66491d1063667f260a4f2386a098393790 \ + --hash=sha256:f541587f5c558abd93cb0de491ce99a9ef8d1ae29dd6ab4dbb5a13281ae04cbd \ + --hash=sha256:fbbeb3c9b2edb5fd044b2a070f127a0ac456ffd079cb82746fc84af01ef021a4 \ + --hash=sha256:fdfa97090e2d6f73dced247a2f2d8004ac6449df6568f30e7fa1a045767c69a6 \ + --hash=sha256:ff0f9913d82e1d1fadbd976424c316fbc4d9c525c81d047bbdd16bd27dd98cfc # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_byod_3.9.in diff --git a/release/ray_release/byod/requirements_ml_byod_3.9.txt b/release/ray_release/byod/requirements_ml_byod_3.9.txt index e907ad35cae5..fe20bf45e379 100644 --- a/release/ray_release/byod/requirements_ml_byod_3.9.txt +++ b/release/ray_release/byod/requirements_ml_byod_3.9.txt @@ -2310,57 +2310,57 @@ openskill==6.0.0 \ --hash=sha256:eee2d0b3c1648663a480cf4680654dfd12bdc749a96d611b1904e191f2632f62 \ --hash=sha256:f89b18930c2befd580407e7cf80a480bc69c3b25d2841346be6d875c8c4bc92e # via -r release/ray_release/byod/requirements_ml_byod_3.9.in -orjson==3.9.10 \ - --hash=sha256:06ad5543217e0e46fd7ab7ea45d506c76f878b87b1b4e369006bdb01acc05a83 \ - --hash=sha256:0a73160e823151f33cdc05fe2cea557c5ef12fdf276ce29bb4f1c571c8368a60 \ - --hash=sha256:1234dc92d011d3554d929b6cf058ac4a24d188d97be5e04355f1b9223e98bbe9 \ - --hash=sha256:1d0dc4310da8b5f6415949bd5ef937e60aeb0eb6b16f95041b5e43e6200821fb \ - --hash=sha256:2a11b4b1a8415f105d989876a19b173f6cdc89ca13855ccc67c18efbd7cbd1f8 \ - --hash=sha256:2e2ecd1d349e62e3960695214f40939bbfdcaeaaa62ccc638f8e651cf0970e5f \ - --hash=sha256:3a2ce5ea4f71681623f04e2b7dadede3c7435dfb5e5e2d1d0ec25b35530e277b \ - --hash=sha256:3e892621434392199efb54e69edfff9f699f6cc36dd9553c5bf796058b14b20d \ - --hash=sha256:3fb205ab52a2e30354640780ce4587157a9563a68c9beaf52153e1cea9aa0921 \ - --hash=sha256:4689270c35d4bb3102e103ac43c3f0b76b169760aff8bcf2d401a3e0e58cdb7f \ - --hash=sha256:49f8ad582da6e8d2cf663c4ba5bf9f83cc052570a3a767487fec6af839b0e777 \ - --hash=sha256:4bd176f528a8151a6efc5359b853ba3cc0e82d4cd1fab9c1300c5d957dc8f48c \ - --hash=sha256:4cf7837c3b11a2dfb589f8530b3cff2bd0307ace4c301e8997e95c7468c1378e \ - --hash=sha256:4fd72fab7bddce46c6826994ce1e7de145ae1e9e106ebb8eb9ce1393ca01444d \ - --hash=sha256:5148bab4d71f58948c7c39d12b14a9005b6ab35a0bdf317a8ade9a9e4d9d0bd5 \ - --hash=sha256:5869e8e130e99687d9e4be835116c4ebd83ca92e52e55810962446d841aba8de \ - --hash=sha256:602a8001bdf60e1a7d544be29c82560a7b49319a0b31d62586548835bbe2c862 \ - --hash=sha256:61804231099214e2f84998316f3238c4c2c4aaec302df12b21a64d72e2a135c7 \ - --hash=sha256:666c6fdcaac1f13eb982b649e1c311c08d7097cbda24f32612dae43648d8db8d \ - --hash=sha256:674eb520f02422546c40401f4efaf8207b5e29e420c17051cddf6c02783ff5ca \ - --hash=sha256:7ec960b1b942ee3c69323b8721df2a3ce28ff40e7ca47873ae35bfafeb4555ca \ - --hash=sha256:7f433be3b3f4c66016d5a20e5b4444ef833a1f802ced13a2d852c637f69729c1 \ - --hash=sha256:7f8fb7f5ecf4f6355683ac6881fd64b5bb2b8a60e3ccde6ff799e48791d8f864 \ - --hash=sha256:81a3a3a72c9811b56adf8bcc829b010163bb2fc308877e50e9910c9357e78521 \ - --hash=sha256:858379cbb08d84fe7583231077d9a36a1a20eb72f8c9076a45df8b083724ad1d \ - --hash=sha256:8b9ba0ccd5a7f4219e67fbbe25e6b4a46ceef783c42af7dbc1da548eb28b6531 \ - --hash=sha256:92af0d00091e744587221e79f68d617b432425a7e59328ca4c496f774a356071 \ - --hash=sha256:9ebbdbd6a046c304b1845e96fbcc5559cd296b4dfd3ad2509e33c4d9ce07d6a1 \ - --hash=sha256:9edd2856611e5050004f4722922b7b1cd6268da34102667bd49d2a2b18bafb81 \ - --hash=sha256:a353bf1f565ed27ba71a419b2cd3db9d6151da426b61b289b6ba1422a702e643 \ - --hash=sha256:b5b7d4a44cc0e6ff98da5d56cde794385bdd212a86563ac321ca64d7f80c80d1 \ - --hash=sha256:b90f340cb6397ec7a854157fac03f0c82b744abdd1c0941a024c3c29d1340aff \ - --hash=sha256:c18a4da2f50050a03d1da5317388ef84a16013302a5281d6f64e4a3f406aabc4 \ - --hash=sha256:c338ed69ad0b8f8f8920c13f529889fe0771abbb46550013e3c3d01e5174deef \ - --hash=sha256:c5a02360e73e7208a872bf65a7554c9f15df5fe063dc047f79738998b0506a14 \ - --hash=sha256:c62b6fa2961a1dcc51ebe88771be5319a93fd89bd247c9ddf732bc250507bc2b \ - --hash=sha256:c812312847867b6335cfb264772f2a7e85b3b502d3a6b0586aa35e1858528ab1 \ - --hash=sha256:c943b35ecdf7123b2d81d225397efddf0bce2e81db2f3ae633ead38e85cd5ade \ - --hash=sha256:ce0a29c28dfb8eccd0f16219360530bc3cfdf6bf70ca384dacd36e6c650ef8e8 \ - --hash=sha256:cf80b550092cc480a0cbd0750e8189247ff45457e5a023305f7ef1bcec811616 \ - --hash=sha256:cff7570d492bcf4b64cc862a6e2fb77edd5e5748ad715f487628f102815165e9 \ - --hash=sha256:d2c1e559d96a7f94a4f581e2a32d6d610df5840881a8cba8f25e446f4d792df3 \ - --hash=sha256:deeb3922a7a804755bbe6b5be9b312e746137a03600f488290318936c1a2d4dc \ - --hash=sha256:e28a50b5be854e18d54f75ef1bb13e1abf4bc650ab9d635e4258c58e71eb6ad5 \ - --hash=sha256:e99c625b8c95d7741fe057585176b1b8783d46ed4b8932cf98ee145c4facf499 \ - --hash=sha256:ec6f18f96b47299c11203edfbdc34e1b69085070d9a3d1f302810cc23ad36bf3 \ - --hash=sha256:ed8bc367f725dfc5cabeed1ae079d00369900231fbb5a5280cf0736c30e2adf7 \ - --hash=sha256:ee5926746232f627a3be1cc175b2cfad24d0170d520361f4ce3fa2fd83f09e1d \ - --hash=sha256:f295efcd47b6124b01255d1491f9e46f17ef40d3d7eabf7364099e463fb45f0f \ - --hash=sha256:fb0b361d73f6b8eeceba47cd37070b5e6c9de5beaeaa63a1cb35c7e1a73ef088 +orjson==3.9.15 \ + --hash=sha256:001f4eb0ecd8e9ebd295722d0cbedf0748680fb9998d3993abaed2f40587257a \ + --hash=sha256:05a1f57fb601c426635fcae9ddbe90dfc1ed42245eb4c75e4960440cac667262 \ + --hash=sha256:10c57bc7b946cf2efa67ac55766e41764b66d40cbd9489041e637c1304400494 \ + --hash=sha256:12365576039b1a5a47df01aadb353b68223da413e2e7f98c02403061aad34bde \ + --hash=sha256:2973474811db7b35c30248d1129c64fd2bdf40d57d84beed2a9a379a6f57d0ab \ + --hash=sha256:2b5c0f532905e60cf22a511120e3719b85d9c25d0e1c2a8abb20c4dede3b05a5 \ + --hash=sha256:2c51378d4a8255b2e7c1e5cc430644f0939539deddfa77f6fac7b56a9784160a \ + --hash=sha256:2d99e3c4c13a7b0fb3792cc04c2829c9db07838fb6973e578b85c1745e7d0ce7 \ + --hash=sha256:2f256d03957075fcb5923410058982aea85455d035607486ccb847f095442bda \ + --hash=sha256:34cbcd216e7af5270f2ffa63a963346845eb71e174ea530867b7443892d77180 \ + --hash=sha256:4228aace81781cc9d05a3ec3a6d2673a1ad0d8725b4e915f1089803e9efd2b99 \ + --hash=sha256:4feeb41882e8aa17634b589533baafdceb387e01e117b1ec65534ec724023d04 \ + --hash=sha256:57d5d8cf9c27f7ef6bc56a5925c7fbc76b61288ab674eb352c26ac780caa5b10 \ + --hash=sha256:5bb399e1b49db120653a31463b4a7b27cf2fbfe60469546baf681d1b39f4edf2 \ + --hash=sha256:62482873e0289cf7313461009bf62ac8b2e54bc6f00c6fabcde785709231a5d7 \ + --hash=sha256:67384f588f7f8daf040114337d34a5188346e3fae6c38b6a19a2fe8c663a2f9b \ + --hash=sha256:6ae4e06be04dc00618247c4ae3f7c3e561d5bc19ab6941427f6d3722a0875ef7 \ + --hash=sha256:6f7b65bfaf69493c73423ce9db66cfe9138b2f9ef62897486417a8fcb0a92bfe \ + --hash=sha256:6fc2fe4647927070df3d93f561d7e588a38865ea0040027662e3e541d592811e \ + --hash=sha256:71c6b009d431b3839d7c14c3af86788b3cfac41e969e3e1c22f8a6ea13139404 \ + --hash=sha256:7413070a3e927e4207d00bd65f42d1b780fb0d32d7b1d951f6dc6ade318e1b5a \ + --hash=sha256:76bc6356d07c1d9f4b782813094d0caf1703b729d876ab6a676f3aaa9a47e37c \ + --hash=sha256:7f6cbd8e6e446fb7e4ed5bac4661a29e43f38aeecbf60c4b900b825a353276a1 \ + --hash=sha256:8055ec598605b0077e29652ccfe9372247474375e0e3f5775c91d9434e12d6b1 \ + --hash=sha256:809d653c155e2cc4fd39ad69c08fdff7f4016c355ae4b88905219d3579e31eb7 \ + --hash=sha256:82425dd5c7bd3adfe4e94c78e27e2fa02971750c2b7ffba648b0f5d5cc016a73 \ + --hash=sha256:87f1097acb569dde17f246faa268759a71a2cb8c96dd392cd25c668b104cad2f \ + --hash=sha256:920fa5a0c5175ab14b9c78f6f820b75804fb4984423ee4c4f1e6d748f8b22bc1 \ + --hash=sha256:92255879280ef9c3c0bcb327c5a1b8ed694c290d61a6a532458264f887f052cb \ + --hash=sha256:946c3a1ef25338e78107fba746f299f926db408d34553b4754e90a7de1d44068 \ + --hash=sha256:95cae920959d772f30ab36d3b25f83bb0f3be671e986c72ce22f8fa700dae061 \ + --hash=sha256:9cf1596680ac1f01839dba32d496136bdd5d8ffb858c280fa82bbfeb173bdd40 \ + --hash=sha256:9fe41b6f72f52d3da4db524c8653e46243c8c92df826ab5ffaece2dba9cccd58 \ + --hash=sha256:b17f0f14a9c0ba55ff6279a922d1932e24b13fc218a3e968ecdbf791b3682b25 \ + --hash=sha256:b3d336ed75d17c7b1af233a6561cf421dee41d9204aa3cfcc6c9c65cd5bb69a8 \ + --hash=sha256:b66bcc5670e8a6b78f0313bcb74774c8291f6f8aeef10fe70e910b8040f3ab75 \ + --hash=sha256:b725da33e6e58e4a5d27958568484aa766e825e93aa20c26c91168be58e08cbb \ + --hash=sha256:b72758f3ffc36ca566ba98a8e7f4f373b6c17c646ff8ad9b21ad10c29186f00d \ + --hash=sha256:bcef128f970bb63ecf9a65f7beafd9b55e3aaf0efc271a4154050fc15cdb386e \ + --hash=sha256:c8e8fe01e435005d4421f183038fc70ca85d2c1e490f51fb972db92af6e047c2 \ + --hash=sha256:d61f7ce4727a9fa7680cd6f3986b0e2c732639f46a5e0156e550e35258aa313a \ + --hash=sha256:d6768a327ea1ba44c9114dba5fdda4a214bdb70129065cd0807eb5f010bfcbb5 \ + --hash=sha256:e18668f1bd39e69b7fed19fa7cd1cd110a121ec25439328b5c89934e6d30d357 \ + --hash=sha256:e88b97ef13910e5f87bcbc4dd7979a7de9ba8702b54d3204ac587e83639c0c2b \ + --hash=sha256:ea0b183a5fe6b2b45f3b854b0d19c4e932d6f5934ae1f723b07cf9560edd4ec7 \ + --hash=sha256:ede0bde16cc6e9b96633df1631fbcd66491d1063667f260a4f2386a098393790 \ + --hash=sha256:f541587f5c558abd93cb0de491ce99a9ef8d1ae29dd6ab4dbb5a13281ae04cbd \ + --hash=sha256:fbbeb3c9b2edb5fd044b2a070f127a0ac456ffd079cb82746fc84af01ef021a4 \ + --hash=sha256:fdfa97090e2d6f73dced247a2f2d8004ac6449df6568f30e7fa1a045767c69a6 \ + --hash=sha256:ff0f9913d82e1d1fadbd976424c316fbc4d9c525c81d047bbdd16bd27dd98cfc # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_ml_byod_3.9.in From 894fb976ea8ab74b42d8a87ea89608e4f1476c28 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:21:51 -0700 Subject: [PATCH 310/634] [core][proto/2] move task+actor event proto to public (#56012) As title, task+actor event proto are supposed to be public. Test: - CI --------- Signed-off-by: Cuong Nguyen --- src/ray/protobuf/BUILD.bazel | 50 ++----------------- src/ray/protobuf/events_base_event.proto | 6 +-- src/ray/protobuf/public/BUILD.bazel | 42 ++++++++++++++++ .../events_actor_task_definition_event.proto | 0 .../events_task_definition_event.proto | 0 .../events_task_execution_event.proto | 0 6 files changed, 48 insertions(+), 50 deletions(-) rename src/ray/protobuf/{ => public}/events_actor_task_definition_event.proto (100%) rename src/ray/protobuf/{ => public}/events_task_definition_event.proto (100%) rename src/ray/protobuf/{ => public}/events_task_execution_event.proto (100%) diff --git a/src/ray/protobuf/BUILD.bazel b/src/ray/protobuf/BUILD.bazel index f1c6bad93acf..7e89c341b679 100644 --- a/src/ray/protobuf/BUILD.bazel +++ b/src/ray/protobuf/BUILD.bazel @@ -443,50 +443,6 @@ cc_grpc_library( ], ) -proto_library( - name = "events_actor_task_definition_event_proto", - srcs = ["events_actor_task_definition_event.proto"], - deps = [ - ":common_proto", - ":runtime_env_common_proto", - "//src/ray/protobuf/public:runtime_environment_proto", - ], -) - -cc_proto_library( - name = "events_actor_task_definition_event_cc_proto", - deps = [":events_actor_task_definition_event_proto"], -) - -proto_library( - name = "events_task_definition_event_proto", - srcs = ["events_task_definition_event.proto"], - deps = [ - ":common_proto", - ":runtime_env_common_proto", - "//src/ray/protobuf/public:runtime_environment_proto", - ], -) - -cc_proto_library( - name = "events_task_definition_event_cc_proto", - deps = [":events_task_definition_event_proto"], -) - -proto_library( - name = "events_task_execution_event_proto", - srcs = ["events_task_execution_event.proto"], - deps = [ - ":common_proto", - "@com_google_protobuf//:timestamp_proto", - ], -) - -cc_proto_library( - name = "events_task_execution_event_cc_proto", - deps = [":events_task_execution_event_proto"], -) - proto_library( name = "events_task_profile_events_proto", srcs = ["events_task_profile_events.proto"], @@ -504,12 +460,12 @@ proto_library( name = "events_base_event_proto", srcs = ["events_base_event.proto"], deps = [ - ":events_actor_task_definition_event_proto", - ":events_task_definition_event_proto", - ":events_task_execution_event_proto", ":events_task_profile_events_proto", + "//src/ray/protobuf/public:events_actor_task_definition_event_proto", "//src/ray/protobuf/public:events_driver_job_definition_event_proto", "//src/ray/protobuf/public:events_driver_job_execution_event_proto", + "//src/ray/protobuf/public:events_task_definition_event_proto", + "//src/ray/protobuf/public:events_task_execution_event_proto", "@com_google_protobuf//:timestamp_proto", ], ) diff --git a/src/ray/protobuf/events_base_event.proto b/src/ray/protobuf/events_base_event.proto index fab95867c473..859e498d3a10 100644 --- a/src/ray/protobuf/events_base_event.proto +++ b/src/ray/protobuf/events_base_event.proto @@ -17,10 +17,10 @@ syntax = "proto3"; package ray.rpc.events; import "google/protobuf/timestamp.proto"; -import "src/ray/protobuf/events_actor_task_definition_event.proto"; -import "src/ray/protobuf/events_task_definition_event.proto"; -import "src/ray/protobuf/events_task_execution_event.proto"; import "src/ray/protobuf/events_task_profile_events.proto"; +import "src/ray/protobuf/public/events_actor_task_definition_event.proto"; +import "src/ray/protobuf/public/events_task_definition_event.proto"; +import "src/ray/protobuf/public/events_task_execution_event.proto"; import "src/ray/protobuf/public/events_driver_job_definition_event.proto"; import "src/ray/protobuf/public/events_driver_job_execution_event.proto"; diff --git a/src/ray/protobuf/public/BUILD.bazel b/src/ray/protobuf/public/BUILD.bazel index dc7712fbd8e9..683e604ffffa 100644 --- a/src/ray/protobuf/public/BUILD.bazel +++ b/src/ray/protobuf/public/BUILD.bazel @@ -3,6 +3,48 @@ load("@rules_proto//proto:defs.bzl", "proto_library") package(default_visibility = ["//visibility:public"]) +proto_library( + name = "events_actor_task_definition_event_proto", + srcs = ["events_actor_task_definition_event.proto"], + deps = [ + ":runtime_environment_proto", + "//src/ray/protobuf:common_proto", + ], +) + +cc_proto_library( + name = "events_actor_task_definition_event_cc_proto", + deps = [":events_actor_task_definition_event_proto"], +) + +proto_library( + name = "events_task_definition_event_proto", + srcs = ["events_task_definition_event.proto"], + deps = [ + ":runtime_environment_proto", + "//src/ray/protobuf:common_proto", + ], +) + +cc_proto_library( + name = "events_task_definition_event_cc_proto", + deps = [":events_task_definition_event_proto"], +) + +proto_library( + name = "events_task_execution_event_proto", + srcs = ["events_task_execution_event.proto"], + deps = [ + "//src/ray/protobuf:common_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + +cc_proto_library( + name = "events_task_execution_event_cc_proto", + deps = [":events_task_execution_event_proto"], +) + proto_library( name = "events_driver_job_definition_event_proto", srcs = ["events_driver_job_definition_event.proto"], diff --git a/src/ray/protobuf/events_actor_task_definition_event.proto b/src/ray/protobuf/public/events_actor_task_definition_event.proto similarity index 100% rename from src/ray/protobuf/events_actor_task_definition_event.proto rename to src/ray/protobuf/public/events_actor_task_definition_event.proto diff --git a/src/ray/protobuf/events_task_definition_event.proto b/src/ray/protobuf/public/events_task_definition_event.proto similarity index 100% rename from src/ray/protobuf/events_task_definition_event.proto rename to src/ray/protobuf/public/events_task_definition_event.proto diff --git a/src/ray/protobuf/events_task_execution_event.proto b/src/ray/protobuf/public/events_task_execution_event.proto similarity index 100% rename from src/ray/protobuf/events_task_execution_event.proto rename to src/ray/protobuf/public/events_task_execution_event.proto From 22b3552294079c043c19488adfe5b0b97c6f0f65 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Wed, 27 Aug 2025 15:42:41 -0700 Subject: [PATCH 311/634] [core] Kill copy whenever sending a retryable grpc request (#55991) Signed-off-by: dayshah --- src/mock/ray/rpc/worker/core_worker_client.h | 4 +- src/ray/core_worker/core_worker.cc | 23 ++- src/ray/core_worker/future_resolver.cc | 3 +- src/ray/gcs/gcs_client/accessor.cc | 162 ++++++++++-------- src/ray/gcs/gcs_client/gcs_client.cc | 7 +- .../gcs_server/tests/gcs_server_rpc_test.cc | 53 +++--- .../ownership_object_directory.cc | 2 +- .../tests/ownership_object_directory_test.cc | 2 +- .../raylet/tests/local_object_manager_test.cc | 2 +- src/ray/rpc/gcs/gcs_rpc_client.h | 12 +- src/ray/rpc/retryable_grpc_client.h | 22 +-- src/ray/rpc/worker/core_worker_client.h | 6 +- 12 files changed, 161 insertions(+), 137 deletions(-) diff --git a/src/mock/ray/rpc/worker/core_worker_client.h b/src/mock/ray/rpc/worker/core_worker_client.h index 019d4b42842c..26aed0495833 100644 --- a/src/mock/ray/rpc/worker/core_worker_client.h +++ b/src/mock/ray/rpc/worker/core_worker_client.h @@ -47,7 +47,7 @@ class MockCoreWorkerClientInterface : public CoreWorkerClientInterface { (override)); MOCK_METHOD(void, GetObjectStatus, - (const GetObjectStatusRequest &request, + (GetObjectStatusRequest && request, const ClientCallback &callback), (override)); MOCK_METHOD(void, @@ -67,7 +67,7 @@ class MockCoreWorkerClientInterface : public CoreWorkerClientInterface { (override)); MOCK_METHOD(void, UpdateObjectLocationBatch, - (const UpdateObjectLocationBatchRequest &request, + (UpdateObjectLocationBatchRequest && request, const ClientCallback &callback), (override)); MOCK_METHOD(void, diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 8f1b0dd9bd4f..fcac4106e5dd 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -2213,17 +2213,16 @@ Status CoreWorker::CreatePlacementGroup( } const PlacementGroupID placement_group_id = PlacementGroupID::Of(GetCurrentJobId()); PlacementGroupSpecBuilder builder; - builder.SetPlacementGroupSpec( - placement_group_id, - placement_group_creation_options.name_, - placement_group_creation_options.bundles_, - placement_group_creation_options.strategy_, - placement_group_creation_options.is_detached_, - placement_group_creation_options.soft_target_node_id_, - worker_context_->GetCurrentJobID(), - worker_context_->GetCurrentActorID(), - worker_context_->CurrentActorDetached(), - placement_group_creation_options.bundle_label_selector_); + builder.SetPlacementGroupSpec(placement_group_id, + placement_group_creation_options.name_, + placement_group_creation_options.bundles_, + placement_group_creation_options.strategy_, + placement_group_creation_options.is_detached_, + placement_group_creation_options.soft_target_node_id_, + worker_context_->GetCurrentJobID(), + worker_context_->GetCurrentActorID(), + worker_context_->CurrentActorDetached(), + placement_group_creation_options.bundle_label_selector_); PlacementGroupSpecification placement_group_spec = builder.Build(); *return_placement_group_id = placement_group_id; RAY_LOG(INFO).WithField(placement_group_id) @@ -3097,7 +3096,7 @@ Status CoreWorker::ReportGeneratorItemReturns( waiter->IncrementObjectGenerated(); client->ReportGeneratorItemReturns( - request, + std::move(request), [waiter, generator_id, return_id, item_index]( const Status &status, const rpc::ReportGeneratorItemReturnsReply &reply) { RAY_LOG(DEBUG) << "ReportGeneratorItemReturns replied. " << generator_id diff --git a/src/ray/core_worker/future_resolver.cc b/src/ray/core_worker/future_resolver.cc index 3231b9595bd9..153c06a0f84f 100644 --- a/src/ray/core_worker/future_resolver.cc +++ b/src/ray/core_worker/future_resolver.cc @@ -15,6 +15,7 @@ #include "ray/core_worker/future_resolver.h" #include +#include namespace ray { namespace core { @@ -32,7 +33,7 @@ void FutureResolver::ResolveFutureAsync(const ObjectID &object_id, request.set_object_id(object_id.Binary()); request.set_owner_worker_id(owner_address.worker_id()); conn->GetObjectStatus( - request, + std::move(request), [this, object_id, owner_address](const Status &status, const rpc::GetObjectStatusReply &reply) { ProcessResolvedObject(object_id, owner_address, status, reply); diff --git a/src/ray/gcs/gcs_client/accessor.cc b/src/ray/gcs/gcs_client/accessor.cc index 1656f86eabdf..75f97ea938bf 100644 --- a/src/ray/gcs/gcs_client/accessor.cc +++ b/src/ray/gcs/gcs_client/accessor.cc @@ -43,7 +43,8 @@ void JobInfoAccessor::AsyncAdd(const std::shared_ptr &data_pt rpc::AddJobRequest request; request.mutable_data()->CopyFrom(*data_ptr); client_impl_->GetGcsRpcClient().AddJob( - request, [job_id, data_ptr, callback](const Status &status, rpc::AddJobReply &&) { + std::move(request), + [job_id, data_ptr, callback](const Status &status, rpc::AddJobReply &&) { if (callback) { callback(status); } @@ -58,7 +59,8 @@ void JobInfoAccessor::AsyncMarkFinished(const JobID &job_id, rpc::MarkJobFinishedRequest request; request.set_job_id(job_id.Binary()); client_impl_->GetGcsRpcClient().MarkJobFinished( - request, [job_id, callback](const Status &status, rpc::MarkJobFinishedReply &&) { + std::move(request), + [job_id, callback](const Status &status, rpc::MarkJobFinishedReply &&) { if (callback) { callback(status); } @@ -123,7 +125,7 @@ void JobInfoAccessor::AsyncGetAll(const std::optional &job_or_submi request.set_job_or_submission_id(job_or_submission_id.value()); } client_impl_->GetGcsRpcClient().GetAllJobInfo( - request, + std::move(request), [callback](const Status &status, rpc::GetAllJobInfoReply &&reply) { callback(status, VectorFromProtobuf(std::move(*reply.mutable_job_info_list()))); RAY_LOG(DEBUG) << "Finished getting all job info."; @@ -143,8 +145,8 @@ Status JobInfoAccessor::GetAll(const std::optional &job_or_submissi request.set_job_or_submission_id(job_or_submission_id.value()); } rpc::GetAllJobInfoReply reply; - RAY_RETURN_NOT_OK( - client_impl_->GetGcsRpcClient().SyncGetAllJobInfo(request, &reply, timeout_ms)); + RAY_RETURN_NOT_OK(client_impl_->GetGcsRpcClient().SyncGetAllJobInfo( + std::move(request), &reply, timeout_ms)); job_data_list = VectorFromProtobuf(std::move(*reply.mutable_job_info_list())); return Status::OK(); } @@ -153,7 +155,8 @@ void JobInfoAccessor::AsyncGetNextJobID(const ItemCallback &callback) { RAY_LOG(DEBUG) << "Getting next job id"; rpc::GetNextJobIDRequest request; client_impl_->GetGcsRpcClient().GetNextJobID( - request, [callback](const Status &status, rpc::GetNextJobIDReply &&reply) { + std::move(request), + [callback](const Status &status, rpc::GetNextJobIDReply &&reply) { RAY_CHECK_OK(status); auto job_id = JobID::FromInt(reply.job_id()); RAY_LOG(DEBUG) << "Finished getting next job id = " << job_id; @@ -170,7 +173,7 @@ void ActorInfoAccessor::AsyncGet( rpc::GetActorInfoRequest request; request.set_actor_id(actor_id.Binary()); client_impl_->GetGcsRpcClient().GetActorInfo( - request, + std::move(request), [actor_id, callback](const Status &status, rpc::GetActorInfoReply &&reply) { if (reply.has_actor_table_data()) { callback(status, reply.actor_table_data()); @@ -203,7 +206,7 @@ void ActorInfoAccessor::AsyncGetAllByFilter( } client_impl_->GetGcsRpcClient().GetAllActorInfo( - request, + std::move(request), [callback](const Status &status, rpc::GetAllActorInfoReply &&reply) { callback(status, VectorFromProtobuf(std::move(*reply.mutable_actor_table_data()))); @@ -222,7 +225,7 @@ void ActorInfoAccessor::AsyncGetByName( request.set_name(name); request.set_ray_namespace(ray_namespace); client_impl_->GetGcsRpcClient().GetNamedActorInfo( - request, + std::move(request), [name, callback](const Status &status, rpc::GetNamedActorInfoReply &&reply) { if (reply.has_actor_table_data()) { callback(status, reply.actor_table_data()); @@ -244,10 +247,10 @@ Status ActorInfoAccessor::SyncGetByName(const std::string &name, request.set_name(name); request.set_ray_namespace(ray_namespace); auto status = client_impl_->GetGcsRpcClient().SyncGetNamedActorInfo( - request, &reply, GetGcsTimeoutMs()); + std::move(request), &reply, GetGcsTimeoutMs()); if (status.ok()) { - actor_table_data = reply.actor_table_data(); - task_spec = reply.task_spec(); + actor_table_data = std::move(*reply.mutable_actor_table_data()); + task_spec = std::move(*reply.mutable_task_spec()); } return status; } @@ -261,14 +264,15 @@ Status ActorInfoAccessor::SyncListNamedActors( request.set_ray_namespace(ray_namespace); rpc::ListNamedActorsReply reply; auto status = client_impl_->GetGcsRpcClient().SyncListNamedActors( - request, &reply, GetGcsTimeoutMs()); + std::move(request), &reply, GetGcsTimeoutMs()); if (!status.ok()) { return status; } actors.reserve(reply.named_actors_list_size()); - for (const auto &actor_info : + for (auto &actor_info : VectorFromProtobuf(std::move(*reply.mutable_named_actors_list()))) { - actors.emplace_back(actor_info.ray_namespace(), actor_info.name()); + actors.emplace_back(std::move(*actor_info.mutable_ray_namespace()), + std::move(*actor_info.mutable_name())); } return status; } @@ -283,7 +287,7 @@ void ActorInfoAccessor::AsyncRestartActorForLineageReconstruction( request.set_num_restarts_due_to_lineage_reconstruction( num_restarts_due_to_lineage_reconstruction); client_impl_->GetGcsRpcClient().RestartActorForLineageReconstruction( - request, + std::move(request), [callback](const Status &status, rpc::RestartActorForLineageReconstructionReply &&reply) { callback(status); @@ -314,7 +318,7 @@ void ActorInfoAccessor::AsyncRegisterActor(const ray::TaskSpecification &task_sp rpc::RegisterActorRequest request; request.mutable_task_spec()->CopyFrom(task_spec.GetMessage()); client_impl_->GetGcsRpcClient().RegisterActor( - request, + std::move(request), [callback](const Status &status, rpc::RegisterActorReply &&reply) { callback(ComputeGcsStatus(status, reply.status())); }, @@ -327,7 +331,7 @@ Status ActorInfoAccessor::SyncRegisterActor(const ray::TaskSpecification &task_s rpc::RegisterActorReply reply; request.mutable_task_spec()->CopyFrom(task_spec.GetMessage()); auto status = client_impl_->GetGcsRpcClient().SyncRegisterActor( - request, &reply, GetGcsTimeoutMs()); + std::move(request), &reply, GetGcsTimeoutMs()); return ComputeGcsStatus(status, reply.status()); } @@ -341,7 +345,7 @@ void ActorInfoAccessor::AsyncKillActor(const ActorID &actor_id, request.set_force_kill(force_kill); request.set_no_restart(no_restart); client_impl_->GetGcsRpcClient().KillActorViaGcs( - request, + std::move(request), [callback](const Status &status, rpc::KillActorViaGcsReply &&reply) { if (callback) { callback(status); @@ -357,7 +361,8 @@ void ActorInfoAccessor::AsyncCreateActor( rpc::CreateActorRequest request; request.mutable_task_spec()->CopyFrom(task_spec.GetMessage()); client_impl_->GetGcsRpcClient().CreateActor( - request, [callback](const Status &status, rpc::CreateActorReply &&reply) { + std::move(request), + [callback](const Status &status, rpc::CreateActorReply &&reply) { callback(status, std::move(reply)); }); } @@ -372,7 +377,7 @@ void ActorInfoAccessor::AsyncReportActorOutOfScope( request.set_num_restarts_due_to_lineage_reconstruction( num_restarts_due_to_lineage_reconstruction); client_impl_->GetGcsRpcClient().ReportActorOutOfScope( - request, + std::move(request), [callback](const Status &status, rpc::ReportActorOutOfScopeReply &&reply) { if (callback) { callback(status); @@ -468,7 +473,7 @@ Status NodeInfoAccessor::RegisterSelf(const rpc::GcsNodeInfo &local_node_info, rpc::RegisterNodeRequest request; request.mutable_node_info()->CopyFrom(local_node_info); client_impl_->GetGcsRpcClient().RegisterNode( - request, + std::move(request), [this, node_id, local_node_info, callback](const Status &status, rpc::RegisterNodeReply &&reply) { if (status.ok()) { @@ -498,7 +503,7 @@ void NodeInfoAccessor::UnregisterSelf(const rpc::NodeDeathInfo &node_death_info, request.set_node_id(local_node_info_.node_id()); request.mutable_node_death_info()->CopyFrom(node_death_info); client_impl_->GetGcsRpcClient().UnregisterNode( - request, + std::move(request), [this, node_id, unregister_done_callback](const Status &status, rpc::UnregisterNodeReply &&reply) { if (status.ok()) { @@ -522,7 +527,8 @@ void NodeInfoAccessor::AsyncRegister(const rpc::GcsNodeInfo &node_info, rpc::RegisterNodeRequest request; request.mutable_node_info()->CopyFrom(node_info); client_impl_->GetGcsRpcClient().RegisterNode( - request, [node_id, callback](const Status &status, rpc::RegisterNodeReply &&reply) { + std::move(request), + [node_id, callback](const Status &status, rpc::RegisterNodeReply &&reply) { if (callback) { callback(status); } @@ -557,7 +563,7 @@ void NodeInfoAccessor::AsyncCheckAlive(const std::vector &node_ids, } size_t num_raylets = node_ids.size(); client_impl_->GetGcsRpcClient().CheckAlive( - request, + std::move(request), [num_raylets, callback](const Status &status, rpc::CheckAliveReply &&reply) { if (status.ok()) { RAY_CHECK_EQ(static_cast(reply.raylet_alive().size()), num_raylets); @@ -584,8 +590,8 @@ Status NodeInfoAccessor::DrainNodes(const std::vector &node_ids, auto draining_request = request.add_drain_node_data(); draining_request->set_node_id(node_id.Binary()); } - RAY_RETURN_NOT_OK( - client_impl_->GetGcsRpcClient().SyncDrainNode(request, &reply, timeout_ms)); + RAY_RETURN_NOT_OK(client_impl_->GetGcsRpcClient().SyncDrainNode( + std::move(request), &reply, timeout_ms)); drained_node_ids.clear(); for (const auto &s : reply.drain_node_status()) { drained_node_ids.push_back(s.node_id()); @@ -602,7 +608,7 @@ void NodeInfoAccessor::AsyncGetAll(const MultiItemCallback &ca request.add_node_selectors()->set_node_id(node_id.Binary()); } client_impl_->GetGcsRpcClient().GetAllNodeInfo( - request, + std::move(request), [callback](const Status &status, rpc::GetAllNodeInfoReply &&reply) { std::vector result; result.reserve((reply.node_info_list_size())); @@ -684,8 +690,8 @@ StatusOr> NodeInfoAccessor::GetAllNoCache( *request.add_node_selectors() = std::move(node_selector.value()); } rpc::GetAllNodeInfoReply reply; - RAY_RETURN_NOT_OK( - client_impl_->GetGcsRpcClient().SyncGetAllNodeInfo(request, &reply, timeout_ms)); + RAY_RETURN_NOT_OK(client_impl_->GetGcsRpcClient().SyncGetAllNodeInfo( + std::move(request), &reply, timeout_ms)); return VectorFromProtobuf(std::move(*reply.mutable_node_info_list())); } @@ -784,7 +790,7 @@ void NodeResourceInfoAccessor::AsyncGetAllAvailableResources( const MultiItemCallback &callback) { rpc::GetAllAvailableResourcesRequest request; client_impl_->GetGcsRpcClient().GetAllAvailableResources( - request, + std::move(request), [callback](const Status &status, rpc::GetAllAvailableResourcesReply &&reply) { callback(status, VectorFromProtobuf(std::move(*reply.mutable_resources_list()))); RAY_LOG(DEBUG) << "Finished getting available resources of all nodes, status = " @@ -796,7 +802,8 @@ void NodeResourceInfoAccessor::AsyncGetAllTotalResources( const MultiItemCallback &callback) { rpc::GetAllTotalResourcesRequest request; client_impl_->GetGcsRpcClient().GetAllTotalResources( - request, [callback](const Status &status, rpc::GetAllTotalResourcesReply &&reply) { + std::move(request), + [callback](const Status &status, rpc::GetAllTotalResourcesReply &&reply) { callback(status, VectorFromProtobuf(std::move(*reply.mutable_resources_list()))); RAY_LOG(DEBUG) << "Finished getting total resources of all nodes, status = " << status; @@ -807,7 +814,8 @@ void NodeResourceInfoAccessor::AsyncGetDrainingNodes( const ItemCallback> &callback) { rpc::GetDrainingNodesRequest request; client_impl_->GetGcsRpcClient().GetDrainingNodes( - request, [callback](const Status &status, rpc::GetDrainingNodesReply &&reply) { + std::move(request), + [callback](const Status &status, rpc::GetDrainingNodesReply &&reply) { RAY_CHECK_OK(status); std::unordered_map draining_nodes; for (const auto &draining_node : reply.draining_nodes()) { @@ -832,7 +840,8 @@ void NodeResourceInfoAccessor::AsyncGetAllResourceUsage( const ItemCallback &callback) { rpc::GetAllResourceUsageRequest request; client_impl_->GetGcsRpcClient().GetAllResourceUsage( - request, [callback](const Status &status, rpc::GetAllResourceUsageReply &&reply) { + std::move(request), + [callback](const Status &status, rpc::GetAllResourceUsageReply &&reply) { callback(std::move(*reply.mutable_resource_usage_data())); RAY_LOG(DEBUG) << "Finished getting resource usage of all nodes, status = " << status; @@ -843,7 +852,7 @@ Status NodeResourceInfoAccessor::GetAllResourceUsage( int64_t timeout_ms, rpc::GetAllResourceUsageReply &reply) { rpc::GetAllResourceUsageRequest request; return client_impl_->GetGcsRpcClient().SyncGetAllResourceUsage( - request, &reply, timeout_ms); + std::move(request), &reply, timeout_ms); } void TaskInfoAccessor::AsyncAddTaskEventData(std::unique_ptr data_ptr, @@ -852,7 +861,8 @@ void TaskInfoAccessor::AsyncAddTaskEventData(std::unique_ptr // Prevent copy here request.mutable_data()->Swap(data_ptr.get()); client_impl_->GetGcsRpcClient().AddTaskEventData( - request, [callback](const Status &status, rpc::AddTaskEventDataReply &&reply) { + std::move(request), + [callback](const Status &status, rpc::AddTaskEventDataReply &&reply) { if (callback) { callback(status); } @@ -866,7 +876,8 @@ void TaskInfoAccessor::AsyncGetTaskEvents( RAY_CHECK(callback); rpc::GetTaskEventsRequest request; client_impl_->GetGcsRpcClient().GetTaskEvents( - request, [callback](const Status &status, rpc::GetTaskEventsReply &&reply) { + std::move(request), + [callback](const Status &status, rpc::GetTaskEventsReply &&reply) { callback(status, VectorFromProtobuf(std::move(*reply.mutable_events_by_task()))); }); } @@ -880,7 +891,8 @@ void ErrorInfoAccessor::AsyncReportJobError(rpc::ErrorTableData data) { rpc::ReportJobErrorRequest request; *request.mutable_job_error() = std::move(data); client_impl_->GetGcsRpcClient().ReportJobError( - request, [job_id](const Status &status, rpc::ReportJobErrorReply &&reply) { + std::move(request), + [job_id](const Status &status, rpc::ReportJobErrorReply &&reply) { RAY_LOG(DEBUG) << "Finished publishing job error, job id = " << job_id; }); } @@ -916,7 +928,7 @@ void WorkerInfoAccessor::AsyncReportWorkerFailure( rpc::ReportWorkerFailureRequest request; request.mutable_worker_failure()->CopyFrom(*data_ptr); client_impl_->GetGcsRpcClient().ReportWorkerFailure( - request, + std::move(request), [worker_address, callback](const Status &status, rpc::ReportWorkerFailureReply &&reply) { if (callback) { @@ -934,7 +946,7 @@ void WorkerInfoAccessor::AsyncGet( rpc::GetWorkerInfoRequest request; request.set_worker_id(worker_id.Binary()); client_impl_->GetGcsRpcClient().GetWorkerInfo( - request, + std::move(request), [worker_id, callback](const Status &status, rpc::GetWorkerInfoReply &&reply) { if (reply.has_worker_table_data()) { callback(status, reply.worker_table_data()); @@ -950,7 +962,8 @@ void WorkerInfoAccessor::AsyncGetAll( RAY_LOG(DEBUG) << "Getting all worker info."; rpc::GetAllWorkerInfoRequest request; client_impl_->GetGcsRpcClient().GetAllWorkerInfo( - request, [callback](const Status &status, rpc::GetAllWorkerInfoReply &&reply) { + std::move(request), + [callback](const Status &status, rpc::GetAllWorkerInfoReply &&reply) { callback(status, VectorFromProtobuf(std::move(*reply.mutable_worker_table_data()))); RAY_LOG(DEBUG) << "Finished getting all worker info, status = " << status; @@ -962,7 +975,8 @@ void WorkerInfoAccessor::AsyncAdd(const std::shared_ptr &d rpc::AddWorkerInfoRequest request; request.mutable_worker_data()->CopyFrom(*data_ptr); client_impl_->GetGcsRpcClient().AddWorkerInfo( - request, [callback](const Status &status, rpc::AddWorkerInfoReply &&reply) { + std::move(request), + [callback](const Status &status, rpc::AddWorkerInfoReply &&reply) { if (callback) { callback(status); } @@ -978,7 +992,7 @@ void WorkerInfoAccessor::AsyncUpdateDebuggerPort(const WorkerID &worker_id, RAY_LOG(DEBUG) << "Updating the worker debugger port, worker id = " << worker_id << ", port = " << debugger_port << "."; client_impl_->GetGcsRpcClient().UpdateWorkerDebuggerPort( - request, + std::move(request), [callback](const Status &status, rpc::UpdateWorkerDebuggerPortReply &&reply) { if (callback) { callback(status); @@ -996,7 +1010,7 @@ void WorkerInfoAccessor::AsyncUpdateWorkerNumPausedThreads( RAY_LOG(DEBUG).WithField(worker_id) << "Update the num paused threads by delta = " << num_paused_threads_delta << "."; client_impl_->GetGcsRpcClient().UpdateWorkerNumPausedThreads( - request, + std::move(request), [callback](const Status &status, rpc::UpdateWorkerNumPausedThreadsReply &&reply) { if (callback) { callback(status); @@ -1013,7 +1027,7 @@ Status PlacementGroupInfoAccessor::SyncCreatePlacementGroup( rpc::CreatePlacementGroupReply reply; request.mutable_placement_group_spec()->CopyFrom(placement_group_spec.GetMessage()); auto status = client_impl_->GetGcsRpcClient().SyncCreatePlacementGroup( - request, &reply, GetGcsTimeoutMs()); + std::move(request), &reply, GetGcsTimeoutMs()); if (status.ok()) { RAY_LOG(DEBUG).WithField(placement_group_spec.PlacementGroupId()) << "Finished registering placement group."; @@ -1030,7 +1044,7 @@ Status PlacementGroupInfoAccessor::SyncRemovePlacementGroup( rpc::RemovePlacementGroupReply reply; request.set_placement_group_id(placement_group_id.Binary()); auto status = client_impl_->GetGcsRpcClient().SyncRemovePlacementGroup( - request, &reply, GetGcsTimeoutMs()); + std::move(request), &reply, GetGcsTimeoutMs()); return status; } @@ -1041,7 +1055,7 @@ void PlacementGroupInfoAccessor::AsyncGet( rpc::GetPlacementGroupRequest request; request.set_placement_group_id(placement_group_id.Binary()); client_impl_->GetGcsRpcClient().GetPlacementGroup( - request, + std::move(request), [placement_group_id, callback](const Status &status, rpc::GetPlacementGroupReply &&reply) { if (reply.has_placement_group_table_data()) { @@ -1064,7 +1078,7 @@ void PlacementGroupInfoAccessor::AsyncGetByName( request.set_name(name); request.set_ray_namespace(ray_namespace); client_impl_->GetGcsRpcClient().GetNamedPlacementGroup( - request, + std::move(request), [name, callback](const Status &status, rpc::GetNamedPlacementGroupReply &&reply) { if (reply.has_placement_group_table_data()) { callback(status, reply.placement_group_table_data()); @@ -1082,7 +1096,8 @@ void PlacementGroupInfoAccessor::AsyncGetAll( RAY_LOG(DEBUG) << "Getting all placement group info."; rpc::GetAllPlacementGroupRequest request; client_impl_->GetGcsRpcClient().GetAllPlacementGroup( - request, [callback](const Status &status, rpc::GetAllPlacementGroupReply &&reply) { + std::move(request), + [callback](const Status &status, rpc::GetAllPlacementGroupReply &&reply) { callback( status, VectorFromProtobuf(std::move(*reply.mutable_placement_group_table_data()))); @@ -1097,7 +1112,9 @@ Status PlacementGroupInfoAccessor::SyncWaitUntilReady( rpc::WaitPlacementGroupUntilReadyReply reply; request.set_placement_group_id(placement_group_id.Binary()); auto status = client_impl_->GetGcsRpcClient().SyncWaitPlacementGroupUntilReady( - request, &reply, absl::ToInt64Milliseconds(absl::Seconds(timeout_seconds))); + std::move(request), + &reply, + absl::ToInt64Milliseconds(absl::Seconds(timeout_seconds))); RAY_LOG(DEBUG).WithField(placement_group_id) << "Finished waiting placement group until ready"; return status; @@ -1115,7 +1132,7 @@ void InternalKVAccessor::AsyncInternalKVGet( req.set_key(key); req.set_namespace_(ns); client_impl_->GetGcsRpcClient().InternalKVGet( - req, + std::move(req), [callback](const Status &status, rpc::InternalKVGetReply &&reply) { if (reply.status().code() == static_cast(StatusCode::NotFound)) { callback(status, std::nullopt); @@ -1137,7 +1154,7 @@ void InternalKVAccessor::AsyncInternalKVMultiGet( } req.set_namespace_(ns); client_impl_->GetGcsRpcClient().InternalKVMultiGet( - req, + std::move(req), [callback](const Status &status, rpc::InternalKVMultiGetReply &&reply) { std::unordered_map map; if (!status.ok()) { @@ -1167,7 +1184,7 @@ void InternalKVAccessor::AsyncInternalKVPut(const std::string &ns, req.set_value(value); req.set_overwrite(overwrite); client_impl_->GetGcsRpcClient().InternalKVPut( - req, + std::move(req), [callback](const Status &status, rpc::InternalKVPutReply &&reply) { callback(status, reply.added()); }, @@ -1183,7 +1200,7 @@ void InternalKVAccessor::AsyncInternalKVExists( req.set_namespace_(ns); req.set_key(key); client_impl_->GetGcsRpcClient().InternalKVExists( - req, + std::move(req), [callback](const Status &status, rpc::InternalKVExistsReply &&reply) { callback(status, reply.exists()); }, @@ -1200,7 +1217,7 @@ void InternalKVAccessor::AsyncInternalKVDel(const std::string &ns, req.set_key(key); req.set_del_by_prefix(del_by_prefix); client_impl_->GetGcsRpcClient().InternalKVDel( - req, + std::move(req), [callback](const Status &status, rpc::InternalKVDelReply &&reply) { callback(status, reply.deleted_num()); }, @@ -1216,7 +1233,7 @@ void InternalKVAccessor::AsyncInternalKVKeys( req.set_namespace_(ns); req.set_prefix(prefix); client_impl_->GetGcsRpcClient().InternalKVKeys( - req, + std::move(req), [callback](const Status &status, rpc::InternalKVKeysReply &&reply) { if (!status.ok()) { callback(status, std::nullopt); @@ -1348,7 +1365,8 @@ void InternalKVAccessor::AsyncGetInternalConfig( const OptionalItemCallback &callback) { rpc::GetInternalConfigRequest request; client_impl_->GetGcsRpcClient().GetInternalConfig( - request, [callback](const Status &status, rpc::GetInternalConfigReply &&reply) { + std::move(request), + [callback](const Status &status, rpc::GetInternalConfigReply &&reply) { if (status.ok()) { RAY_LOG(DEBUG) << "Fetched internal config: " << reply.config(); } else { @@ -1368,8 +1386,8 @@ Status RuntimeEnvAccessor::PinRuntimeEnvUri(const std::string &uri, request.set_uri(uri); request.set_expiration_s(expiration_s); rpc::PinRuntimeEnvURIReply reply; - auto status = - client_impl_->GetGcsRpcClient().SyncPinRuntimeEnvURI(request, &reply, timeout_ms); + auto status = client_impl_->GetGcsRpcClient().SyncPinRuntimeEnvURI( + std::move(request), &reply, timeout_ms); return status; } @@ -1396,7 +1414,7 @@ Status AutoscalerStateAccessor::RequestClusterResourceConstraint( } return client_impl_->GetGcsRpcClient().SyncRequestClusterResourceConstraint( - request, &reply, timeout_ms); + std::move(request), &reply, timeout_ms); } Status AutoscalerStateAccessor::GetClusterResourceState(int64_t timeout_ms, @@ -1405,7 +1423,7 @@ Status AutoscalerStateAccessor::GetClusterResourceState(int64_t timeout_ms, rpc::autoscaler::GetClusterResourceStateReply reply; RAY_RETURN_NOT_OK(client_impl_->GetGcsRpcClient().SyncGetClusterResourceState( - request, &reply, timeout_ms)); + std::move(request), &reply, timeout_ms)); if (!reply.SerializeToString(&serialized_reply)) { return Status::IOError("Failed to serialize GetClusterResourceState"); @@ -1418,8 +1436,8 @@ Status AutoscalerStateAccessor::GetClusterStatus(int64_t timeout_ms, rpc::autoscaler::GetClusterStatusRequest request; rpc::autoscaler::GetClusterStatusReply reply; - RAY_RETURN_NOT_OK( - client_impl_->GetGcsRpcClient().SyncGetClusterStatus(request, &reply, timeout_ms)); + RAY_RETURN_NOT_OK(client_impl_->GetGcsRpcClient().SyncGetClusterStatus( + std::move(request), &reply, timeout_ms)); if (!reply.SerializeToString(&serialized_reply)) { return Status::IOError("Failed to serialize GetClusterStatusReply"); @@ -1432,7 +1450,7 @@ void AutoscalerStateAccessor::AsyncGetClusterStatus( const OptionalItemCallback &callback) { rpc::autoscaler::GetClusterStatusRequest request; client_impl_->GetGcsRpcClient().GetClusterStatus( - request, + std::move(request), [callback](const Status &status, rpc::autoscaler::GetClusterStatusReply &&reply) { if (!status.ok()) { callback(status, std::nullopt); @@ -1452,7 +1470,7 @@ Status AutoscalerStateAccessor::ReportAutoscalingState( return Status::IOError("Failed to parse ReportAutoscalingState"); } return client_impl_->GetGcsRpcClient().SyncReportAutoscalingState( - request, &reply, timeout_ms); + std::move(request), &reply, timeout_ms); } Status AutoscalerStateAccessor::ReportClusterConfig( @@ -1464,7 +1482,7 @@ Status AutoscalerStateAccessor::ReportClusterConfig( return Status::IOError("Failed to parse ClusterConfig"); } return client_impl_->GetGcsRpcClient().SyncReportClusterConfig( - request, &reply, timeout_ms); + std::move(request), &reply, timeout_ms); } Status AutoscalerStateAccessor::DrainNode(const std::string &node_id, @@ -1482,8 +1500,8 @@ Status AutoscalerStateAccessor::DrainNode(const std::string &node_id, rpc::autoscaler::DrainNodeReply reply; - RAY_RETURN_NOT_OK( - client_impl_->GetGcsRpcClient().SyncDrainNode(request, &reply, timeout_ms)); + RAY_RETURN_NOT_OK(client_impl_->GetGcsRpcClient().SyncDrainNode( + std::move(request), &reply, timeout_ms)); is_accepted = reply.is_accepted(); if (!is_accepted) { @@ -1504,7 +1522,8 @@ Status PublisherAccessor::PublishError(std::string key_id, pub_message->set_key_id(std::move(key_id)); *(pub_message->mutable_error_info_message()) = std::move(data); rpc::GcsPublishReply reply; - return client_impl_->GetGcsRpcClient().SyncGcsPublish(request, &reply, timeout_ms); + return client_impl_->GetGcsRpcClient().SyncGcsPublish( + std::move(request), &reply, timeout_ms); } Status PublisherAccessor::PublishLogs(std::string key_id, @@ -1516,7 +1535,8 @@ Status PublisherAccessor::PublishLogs(std::string key_id, pub_message->set_key_id(std::move(key_id)); *(pub_message->mutable_log_batch_message()) = std::move(data); rpc::GcsPublishReply reply; - return client_impl_->GetGcsRpcClient().SyncGcsPublish(request, &reply, timeout_ms); + return client_impl_->GetGcsRpcClient().SyncGcsPublish( + std::move(request), &reply, timeout_ms); } void PublisherAccessor::AsyncPublishNodeResourceUsage( @@ -1530,7 +1550,7 @@ void PublisherAccessor::AsyncPublishNodeResourceUsage( pub_message->mutable_node_resource_usage_message()->set_json( std::move(node_resource_usage_json)); client_impl_->GetGcsRpcClient().GcsPublish( - request, + std::move(request), [done](const Status &status, rpc::GcsPublishReply &&reply) { done(status); }); } diff --git a/src/ray/gcs/gcs_client/gcs_client.cc b/src/ray/gcs/gcs_client/gcs_client.cc index b5d3b1f2bf4f..43d960e0bdf9 100644 --- a/src/ray/gcs/gcs_client/gcs_client.cc +++ b/src/ray/gcs/gcs_client/gcs_client.cc @@ -58,7 +58,8 @@ void GcsSubscriberClient::PubsubLongPolling( req.set_max_processed_sequence_id(request.max_processed_sequence_id()); req.set_publisher_id(request.publisher_id()); rpc_client_->GcsSubscriberPoll( - req, [callback](const Status &status, rpc::GcsSubscriberPollReply &&poll_reply) { + std::move(req), + [callback](const Status &status, rpc::GcsSubscriberPollReply &&poll_reply) { rpc::PubsubLongPollingReply reply; reply.mutable_pub_messages()->Swap(poll_reply.mutable_pub_messages()); *reply.mutable_publisher_id() = std::move(*poll_reply.mutable_publisher_id()); @@ -73,7 +74,7 @@ void GcsSubscriberClient::PubsubCommandBatch( req.set_subscriber_id(request.subscriber_id()); *req.mutable_commands() = request.commands(); rpc_client_->GcsSubscriberCommandBatch( - req, + std::move(req), [callback](const Status &status, rpc::GcsSubscriberCommandBatchReply &&batch_reply) { rpc::PubsubCommandBatchReply reply; @@ -177,7 +178,7 @@ Status GcsClient::FetchClusterId(int64_t timeout_ms) { rpc::GetClusterIdReply reply; RAY_LOG(DEBUG) << "Cluster ID is nil, getting cluster ID from GCS server."; - Status s = gcs_rpc_client_->SyncGetClusterId(request, &reply, timeout_ms); + Status s = gcs_rpc_client_->SyncGetClusterId(std::move(request), &reply, timeout_ms); if (!s.ok()) { RAY_LOG(WARNING) << "Failed to get cluster ID from GCS server: " << s; gcs_rpc_client_.reset(); diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc b/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc index 81798c28be7d..2df5c2681201 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc @@ -69,9 +69,9 @@ class GcsServerTest : public ::testing::Test { rpc::ResetServerCallExecutor(); } - bool AddJob(const rpc::AddJobRequest &request) { + bool AddJob(rpc::AddJobRequest request) { std::promise promise; - client_->AddJob(request, + client_->AddJob(std::move(request), [&promise](const Status &status, const rpc::AddJobReply &reply) { RAY_CHECK_OK(status); promise.set_value(true); @@ -79,10 +79,10 @@ class GcsServerTest : public ::testing::Test { return WaitReady(promise.get_future(), timeout_ms_); } - bool MarkJobFinished(const rpc::MarkJobFinishedRequest &request) { + bool MarkJobFinished(rpc::MarkJobFinishedRequest request) { std::promise promise; client_->MarkJobFinished( - request, + std::move(request), [&promise](const Status &status, const rpc::MarkJobFinishedReply &reply) { RAY_CHECK_OK(status); promise.set_value(true); @@ -95,7 +95,7 @@ class GcsServerTest : public ::testing::Test { request.set_actor_id(actor_id); std::optional actor_table_data_opt; std::promise promise; - client_->GetActorInfo(request, + client_->GetActorInfo(std::move(request), [&actor_table_data_opt, &promise]( const Status &status, const rpc::GetActorInfoReply &reply) { RAY_CHECK_OK(status); @@ -110,10 +110,11 @@ class GcsServerTest : public ::testing::Test { return actor_table_data_opt; } - bool RegisterNode(const rpc::RegisterNodeRequest &request) { + bool RegisterNode(rpc::RegisterNodeRequest request) { std::promise promise; client_->RegisterNode( - request, [&promise](const Status &status, const rpc::RegisterNodeReply &reply) { + std::move(request), + [&promise](const Status &status, const rpc::RegisterNodeReply &reply) { RAY_CHECK_OK(status); promise.set_value(true); }); @@ -121,10 +122,11 @@ class GcsServerTest : public ::testing::Test { return WaitReady(promise.get_future(), timeout_ms_); } - bool UnregisterNode(const rpc::UnregisterNodeRequest &request) { + bool UnregisterNode(rpc::UnregisterNodeRequest request) { std::promise promise; client_->UnregisterNode( - request, [&promise](const Status &status, const rpc::UnregisterNodeReply &reply) { + std::move(request), + [&promise](const Status &status, const rpc::UnregisterNodeReply &reply) { RAY_CHECK_OK(status); promise.set_value(true); }); @@ -137,7 +139,7 @@ class GcsServerTest : public ::testing::Test { rpc::GetAllNodeInfoRequest request; std::promise promise; client_->GetAllNodeInfo( - request, + std::move(request), [&node_info_list, &promise](const Status &status, const rpc::GetAllNodeInfoReply &reply) { RAY_CHECK_OK(status); @@ -150,10 +152,10 @@ class GcsServerTest : public ::testing::Test { return node_info_list; } - bool ReportWorkerFailure(const rpc::ReportWorkerFailureRequest &request) { + bool ReportWorkerFailure(rpc::ReportWorkerFailureRequest request) { std::promise promise; client_->ReportWorkerFailure( - request, + std::move(request), [&promise](const Status &status, const rpc::ReportWorkerFailureReply &reply) { RAY_CHECK_OK(status); promise.set_value(status.ok()); @@ -167,7 +169,7 @@ class GcsServerTest : public ::testing::Test { std::optional worker_table_data_opt; std::promise promise; client_->GetWorkerInfo( - request, + std::move(request), [&worker_table_data_opt, &promise](const Status &status, const rpc::GetWorkerInfoReply &reply) { RAY_CHECK_OK(status); @@ -187,7 +189,7 @@ class GcsServerTest : public ::testing::Test { rpc::GetAllWorkerInfoRequest request; std::promise promise; client_->GetAllWorkerInfo( - request, + std::move(request), [&worker_table_data, &promise](const Status &status, const rpc::GetAllWorkerInfoReply &reply) { RAY_CHECK_OK(status); @@ -200,10 +202,11 @@ class GcsServerTest : public ::testing::Test { return worker_table_data; } - bool AddWorkerInfo(const rpc::AddWorkerInfoRequest &request) { + bool AddWorkerInfo(rpc::AddWorkerInfoRequest request) { std::promise promise; client_->AddWorkerInfo( - request, [&promise](const Status &status, const rpc::AddWorkerInfoReply &reply) { + std::move(request), + [&promise](const Status &status, const rpc::AddWorkerInfoReply &reply) { RAY_CHECK_OK(status); promise.set_value(true); }); @@ -330,7 +333,7 @@ TEST_F(GcsServerTest, TestNodeInfoFilters) { // Get all rpc::GetAllNodeInfoRequest request; rpc::GetAllNodeInfoReply reply; - RAY_CHECK_OK(client_->SyncGetAllNodeInfo(request, &reply)); + RAY_CHECK_OK(client_->SyncGetAllNodeInfo(std::move(request), &reply)); ASSERT_EQ(reply.node_info_list_size(), 3); ASSERT_EQ(reply.num_filtered(), 0); @@ -342,7 +345,7 @@ TEST_F(GcsServerTest, TestNodeInfoFilters) { request.add_node_selectors()->set_node_id(node1->node_id()); request.add_node_selectors()->set_node_id(node2->node_id()); rpc::GetAllNodeInfoReply reply; - RAY_CHECK_OK(client_->SyncGetAllNodeInfo(request, &reply)); + RAY_CHECK_OK(client_->SyncGetAllNodeInfo(std::move(request), &reply)); ASSERT_EQ(reply.node_info_list_size(), 2); ASSERT_EQ(reply.num_filtered(), 1); @@ -353,7 +356,7 @@ TEST_F(GcsServerTest, TestNodeInfoFilters) { rpc::GetAllNodeInfoRequest request; request.set_state_filter(rpc::GcsNodeInfo::ALIVE); rpc::GetAllNodeInfoReply reply; - RAY_CHECK_OK(client_->SyncGetAllNodeInfo(request, &reply)); + RAY_CHECK_OK(client_->SyncGetAllNodeInfo(std::move(request), &reply)); ASSERT_EQ(reply.node_info_list_size(), 2); ASSERT_EQ(reply.num_filtered(), 1); @@ -365,7 +368,7 @@ TEST_F(GcsServerTest, TestNodeInfoFilters) { rpc::GetAllNodeInfoRequest request; request.set_state_filter(rpc::GcsNodeInfo::DEAD); rpc::GetAllNodeInfoReply reply; - RAY_CHECK_OK(client_->SyncGetAllNodeInfo(request, &reply)); + RAY_CHECK_OK(client_->SyncGetAllNodeInfo(std::move(request), &reply)); ASSERT_EQ(reply.node_info_list_size(), 1); ASSERT_EQ(reply.num_filtered(), 2); @@ -378,7 +381,7 @@ TEST_F(GcsServerTest, TestNodeInfoFilters) { request.add_node_selectors()->set_node_name("node1"); request.add_node_selectors()->set_node_name("node2"); rpc::GetAllNodeInfoReply reply; - RAY_CHECK_OK(client_->SyncGetAllNodeInfo(request, &reply)); + RAY_CHECK_OK(client_->SyncGetAllNodeInfo(std::move(request), &reply)); ASSERT_EQ(reply.node_info_list_size(), 2); ASSERT_EQ(reply.num_filtered(), 1); @@ -391,7 +394,7 @@ TEST_F(GcsServerTest, TestNodeInfoFilters) { request.add_node_selectors()->set_node_ip_address("127.0.0.1"); request.add_node_selectors()->set_node_ip_address("127.0.0.2"); rpc::GetAllNodeInfoReply reply; - RAY_CHECK_OK(client_->SyncGetAllNodeInfo(request, &reply)); + RAY_CHECK_OK(client_->SyncGetAllNodeInfo(std::move(request), &reply)); ASSERT_EQ(reply.node_info_list_size(), 2); ASSERT_EQ(reply.num_filtered(), 1); @@ -404,7 +407,7 @@ TEST_F(GcsServerTest, TestNodeInfoFilters) { request.add_node_selectors()->set_node_id(node1->node_id()); request.add_node_selectors()->set_node_name("node2"); rpc::GetAllNodeInfoReply reply; - RAY_CHECK_OK(client_->SyncGetAllNodeInfo(request, &reply)); + RAY_CHECK_OK(client_->SyncGetAllNodeInfo(std::move(request), &reply)); ASSERT_EQ(reply.node_info_list_size(), 2); ASSERT_EQ(reply.num_filtered(), 1); ASSERT_EQ(reply.total(), 3); @@ -417,7 +420,7 @@ TEST_F(GcsServerTest, TestNodeInfoFilters) { request.add_node_selectors()->set_node_id(node3->node_id()); request.set_state_filter(rpc::GcsNodeInfo::ALIVE); rpc::GetAllNodeInfoReply reply; - RAY_CHECK_OK(client_->SyncGetAllNodeInfo(request, &reply)); + RAY_CHECK_OK(client_->SyncGetAllNodeInfo(std::move(request), &reply)); ASSERT_EQ(reply.node_info_list_size(), 1); ASSERT_EQ(reply.num_filtered(), 2); ASSERT_EQ(reply.total(), 3); @@ -430,7 +433,7 @@ TEST_F(GcsServerTest, TestNodeInfoFilters) { request.add_node_selectors()->set_node_name("node3"); request.set_state_filter(rpc::GcsNodeInfo::DEAD); rpc::GetAllNodeInfoReply reply; - RAY_CHECK_OK(client_->SyncGetAllNodeInfo(request, &reply)); + RAY_CHECK_OK(client_->SyncGetAllNodeInfo(std::move(request), &reply)); ASSERT_EQ(reply.node_info_list_size(), 1); ASSERT_EQ(reply.num_filtered(), 2); ASSERT_EQ(reply.total(), 3); diff --git a/src/ray/object_manager/ownership_object_directory.cc b/src/ray/object_manager/ownership_object_directory.cc index 3e3d958f5a17..dfa9615b6679 100644 --- a/src/ray/object_manager/ownership_object_directory.cc +++ b/src/ray/object_manager/ownership_object_directory.cc @@ -246,7 +246,7 @@ void OwnershipBasedObjectDirectory::SendObjectLocationUpdateBatchIfNeeded( in_flight_requests_.emplace(worker_id); auto owner_client = GetClient(owner_address); owner_client->UpdateObjectLocationBatch( - request, + std::move(request), [this, worker_id, node_id, owner_address]( const Status &status, const rpc::UpdateObjectLocationBatchReply &reply) { RAY_CHECK(in_flight_requests_.erase(worker_id) > 0); diff --git a/src/ray/object_manager/tests/ownership_object_directory_test.cc b/src/ray/object_manager/tests/ownership_object_directory_test.cc index 4156ed167954..1d152f0aadba 100644 --- a/src/ray/object_manager/tests/ownership_object_directory_test.cc +++ b/src/ray/object_manager/tests/ownership_object_directory_test.cc @@ -37,7 +37,7 @@ using ::testing::Return; class MockWorkerClient : public rpc::CoreWorkerClientInterface { public: void UpdateObjectLocationBatch( - const rpc::UpdateObjectLocationBatchRequest &request, + rpc::UpdateObjectLocationBatchRequest &&request, const rpc::ClientCallback &callback) override { const auto &worker_id = WorkerID::FromBinary(request.intended_worker_id()); const auto &object_location_updates = request.object_location_updates(); diff --git a/src/ray/raylet/tests/local_object_manager_test.cc b/src/ray/raylet/tests/local_object_manager_test.cc index a5d7192e69b4..9fd1fe7e2490 100644 --- a/src/ray/raylet/tests/local_object_manager_test.cc +++ b/src/ray/raylet/tests/local_object_manager_test.cc @@ -107,7 +107,7 @@ class MockSubscriber : public pubsub::SubscriberInterface { class MockWorkerClient : public rpc::CoreWorkerClientInterface { public: void UpdateObjectLocationBatch( - const rpc::UpdateObjectLocationBatchRequest &request, + rpc::UpdateObjectLocationBatchRequest &&request, const rpc::ClientCallback &callback) override { for (const auto &object_location_update : request.object_location_updates()) { ASSERT_TRUE(object_location_update.has_spilled_location_update()); diff --git a/src/ray/rpc/gcs/gcs_rpc_client.h b/src/ray/rpc/gcs/gcs_rpc_client.h index cd804f1fac41..17f023e3c116 100644 --- a/src/ray/rpc/gcs/gcs_rpc_client.h +++ b/src/ray/rpc/gcs/gcs_rpc_client.h @@ -88,7 +88,7 @@ namespace rpc { method_timeout_ms, \ handle_payload_status, \ SPECS) \ - void METHOD(const METHOD_NAMESPACE::METHOD##Request &request, \ + void METHOD(METHOD_NAMESPACE::METHOD##Request &&request, \ const ClientCallback &callback, \ const int64_t timeout_ms = method_timeout_ms) SPECS { \ invoke_async_method promise; \ METHOD( \ - request, \ + std::move(request), \ [&promise, reply_in](const Status &status, \ const METHOD_NAMESPACE::METHOD##Reply &reply) { \ reply_in->CopyFrom(reply); \ @@ -223,14 +223,14 @@ class GcsRpcClient { PrepareAsyncFunction prepare_async_function, std::shared_ptr> grpc_client, const std::string &call_name, - const Request &request, + Request &&request, const ClientCallback &callback, const int64_t timeout_ms) { retryable_grpc_client_->template CallMethod( prepare_async_function, std::move(grpc_client), call_name, - request, + std::forward(request), [callback](const Status &status, Reply &&reply) { if (status.ok()) { if constexpr (handle_payload_status) { diff --git a/src/ray/rpc/retryable_grpc_client.h b/src/ray/rpc/retryable_grpc_client.h index 6bb6df4477f5..6e7558d118fb 100644 --- a/src/ray/rpc/retryable_grpc_client.h +++ b/src/ray/rpc/retryable_grpc_client.h @@ -32,17 +32,17 @@ namespace ray::rpc { // Define a void retryable RPC client method. -#define VOID_RETRYABLE_RPC_CLIENT_METHOD( \ - retryable_rpc_client, SERVICE, METHOD, rpc_client, method_timeout_ms, SPECS) \ - void METHOD(const METHOD##Request &request, \ - const ClientCallback &callback) SPECS { \ - retryable_rpc_client->CallMethod( \ - &SERVICE::Stub::PrepareAsync##METHOD, \ - rpc_client, \ - #SERVICE ".grpc_client." #METHOD, \ - request, \ - callback, \ - method_timeout_ms); \ +#define VOID_RETRYABLE_RPC_CLIENT_METHOD( \ + retryable_rpc_client, SERVICE, METHOD, rpc_client, method_timeout_ms, SPECS) \ + void METHOD(METHOD##Request &&request, const ClientCallback &callback) \ + SPECS { \ + retryable_rpc_client->CallMethod( \ + &SERVICE::Stub::PrepareAsync##METHOD, \ + rpc_client, \ + #SERVICE ".grpc_client." #METHOD, \ + std::move(request), \ + callback, \ + method_timeout_ms); \ } /** diff --git a/src/ray/rpc/worker/core_worker_client.h b/src/ray/rpc/worker/core_worker_client.h index 341f07e9786a..0f099dd3e86e 100644 --- a/src/ray/rpc/worker/core_worker_client.h +++ b/src/ray/rpc/worker/core_worker_client.h @@ -109,7 +109,7 @@ class CoreWorkerClientInterface : public pubsub::SubscriberClientInterface { const ClientCallback &callback) {} /// Ask the owner of an object about the object's current status. - virtual void GetObjectStatus(const GetObjectStatusRequest &request, + virtual void GetObjectStatus(GetObjectStatusRequest &&request, const ClientCallback &callback) {} /// Ask the actor's owner to reply when the actor has no references. @@ -128,7 +128,7 @@ class CoreWorkerClientInterface : public pubsub::SubscriberClientInterface { const ClientCallback &callback) {} virtual void UpdateObjectLocationBatch( - const UpdateObjectLocationBatchRequest &request, + UpdateObjectLocationBatchRequest &&request, const ClientCallback &callback) {} virtual void GetObjectLocationsOwner( @@ -136,7 +136,7 @@ class CoreWorkerClientInterface : public pubsub::SubscriberClientInterface { const ClientCallback &callback) {} virtual void ReportGeneratorItemReturns( - const ReportGeneratorItemReturnsRequest &request, + ReportGeneratorItemReturnsRequest &&request, const ClientCallback &callback) {} /// Tell this actor to exit immediately. From 4a6c264e0e2b0402d94a20a6ac89dab4a547eb1c Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:53:57 -0700 Subject: [PATCH 312/634] [data] dedupe schemas (#55854) ## Why are these changes needed? results: TBD - 2180 columns, 1000 schemas deduping is 1 second ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu Signed-off-by: Alexey Kudinkin Co-authored-by: Alexey Kudinkin --- python/ray/air/util/object_extensions/arrow.py | 3 +++ python/ray/air/util/tensor_extensions/arrow.py | 8 ++++++++ .../data/_internal/arrow_ops/transform_pyarrow.py | 8 ++++++++ .../_internal/execution/interfaces/ref_bundle.py | 7 +++++++ python/ray/data/tests/test_transform_pyarrow.py | 12 ------------ 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/python/ray/air/util/object_extensions/arrow.py b/python/ray/air/util/object_extensions/arrow.py index 180fcfc96367..b7e2e569c61b 100644 --- a/python/ray/air/util/object_extensions/arrow.py +++ b/python/ray/air/util/object_extensions/arrow.py @@ -71,6 +71,9 @@ def __reduce__(self): self.__arrow_ext_serialize__(), ) + def __hash__(self) -> int: + return hash((type(self), self.storage_type.id, self.extension_name)) + @PublicAPI(stability="alpha") class ArrowPythonObjectScalar(pa.ExtensionScalar): diff --git a/python/ray/air/util/tensor_extensions/arrow.py b/python/ray/air/util/tensor_extensions/arrow.py index be0a88c83bb8..0c1772b46d78 100644 --- a/python/ray/air/util/tensor_extensions/arrow.py +++ b/python/ray/air/util/tensor_extensions/arrow.py @@ -574,6 +574,9 @@ def _need_variable_shaped_tensor_array( shape = arr_type.shape return False + def __hash__(self) -> int: + return hash((type(self), self.extension_name, self.storage_type, self._shape)) + @PublicAPI(stability="beta") class ArrowTensorType(_BaseFixedShapeArrowTensorType): @@ -584,6 +587,7 @@ class ArrowTensorType(_BaseFixedShapeArrowTensorType): """ OFFSET_DTYPE = np.int32 + __hash__ = _BaseFixedShapeArrowTensorType.__hash__ def __init__(self, shape: Tuple[int, ...], dtype: pa.DataType): """ @@ -614,6 +618,7 @@ class ArrowTensorTypeV2(_BaseFixedShapeArrowTensorType): """Arrow ExtensionType (v2) for tensors (supporting tensors > 4Gb).""" OFFSET_DTYPE = np.int64 + __hash__ = _BaseFixedShapeArrowTensorType.__hash__ def __init__(self, shape: Tuple[int, ...], dtype: pa.DataType): """ @@ -1125,6 +1130,9 @@ def _extension_scalar_to_ndarray(self, scalar: "pa.ExtensionScalar") -> np.ndarr data_buffer = raw_values.buffers()[1] return _to_ndarray_helper(shape, value_type, offset, data_buffer) + def __hash__(self) -> int: + return hash((type(self), self.extension_name, self.storage_type, self._ndim)) + # NOTE: We need to inherit from the mixin before pa.ExtensionArray to ensure that the # mixin's overriding methods appear first in the MRO. diff --git a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py index b97cf4d641c6..453ba23e4e08 100644 --- a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py +++ b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py @@ -179,6 +179,14 @@ def unify_schemas( ArrowVariableShapedTensorType, ) + try: + if len(set(schemas)) == 1: + # Early exit because unifying can be expensive + return schemas[0] + except Exception as e: + # Unsure if there are cases where schemas are NOT hashable + logger.warning(f"Failed to hash the schemas (for deduplication): {e}") + schemas_to_unify = [] schema_field_overrides = {} diff --git a/python/ray/data/_internal/execution/interfaces/ref_bundle.py b/python/ray/data/_internal/execution/interfaces/ref_bundle.py index 50c905803d2a..310acf160b07 100644 --- a/python/ray/data/_internal/execution/interfaces/ref_bundle.py +++ b/python/ray/data/_internal/execution/interfaces/ref_bundle.py @@ -63,6 +63,13 @@ def __post_init__(self): "The size in bytes of the block must be known: {}".format(b) ) + import pyarrow as pa + + # The schema metadata might be unhashable. + # We need schemas to be hashable for unification + if isinstance(self.schema, pa.lib.Schema): + self.schema = self.schema.remove_metadata() + def __setattr__(self, key, value): if hasattr(self, key) and key in ["blocks", "owns_blocks"]: raise ValueError(f"The `{key}` field of RefBundle cannot be updated.") diff --git a/python/ray/data/tests/test_transform_pyarrow.py b/python/ray/data/tests/test_transform_pyarrow.py index 2d0ba5f2f75c..9c253e4ea4d0 100644 --- a/python/ray/data/tests/test_transform_pyarrow.py +++ b/python/ray/data/tests/test_transform_pyarrow.py @@ -603,12 +603,6 @@ def test_unify_schemas_object_types(unify_schemas_object_types_schemas): assert result == schemas["expected"] -def test_unify_schemas_duplicate_fields(unify_schemas_duplicate_fields_schema): - """Test error handling for duplicate field names.""" - with pytest.raises(ValueError, match="has multiple fields with the same name"): - unify_schemas([unify_schemas_duplicate_fields_schema]) - - @pytest.mark.skipif( get_pyarrow_version() < parse_version("17.0.0"), reason="Requires PyArrow version 17 or higher", @@ -2788,12 +2782,6 @@ def unify_schemas_object_types_schemas(): } -@pytest.fixture -def unify_schemas_duplicate_fields_schema(): - """Fixture for duplicate fields unify schemas test data.""" - return pa.schema([("col", pa.int32()), ("col", pa.int64())]) # Duplicate name - - @pytest.fixture def unify_schemas_incompatible_tensor_schemas(): """Fixture for incompatible tensor dtypes unify schemas test data.""" From 777370ea963b51bd872d5d2df8af02b6ea4a7dcc Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Wed, 27 Aug 2025 15:58:30 -0700 Subject: [PATCH 313/634] [release] Add option to filter test by name prefix (#55670) - This filter test func can now use either or both of name prefix filter or regex filter. If both are used at the same time, test matches either of the filter is accepted --------- Signed-off-by: kevin --- release/ray_release/buildkite/filter.py | 28 +++-- release/ray_release/buildkite/settings.py | 36 +++--- release/ray_release/scripts/build_pipeline.py | 6 +- release/ray_release/tests/test_buildkite.py | 110 +++++++++++------- 4 files changed, 105 insertions(+), 75 deletions(-) diff --git a/release/ray_release/buildkite/filter.py b/release/ray_release/buildkite/filter.py index 727b0711896e..c2685f124918 100644 --- a/release/ray_release/buildkite/filter.py +++ b/release/ray_release/buildkite/filter.py @@ -22,28 +22,38 @@ def _unflattened_lookup(lookup: Dict, flat_key: str, delimiter: str = "/") -> An def filter_tests( test_collection: List[Test], frequency: Frequency, - test_attr_regex_filters: Optional[Dict[str, str]] = None, + test_filters: Optional[Dict[str, str]] = None, prefer_smoke_tests: bool = False, run_jailed_tests: bool = False, run_unstable_tests: bool = False, ) -> List[Tuple[Test, bool]]: - if test_attr_regex_filters is None: - test_attr_regex_filters = {} + if test_filters is None: + test_filters = {} tests_to_run = [] for test in test_collection: + attr_mismatch = False # Skip kuberay tests for now. # TODO: (khluu) Remove this once we start running KubeRay release tests. if test.is_kuberay() and get_global_config()["kuberay_disabled"]: continue - # First, filter by string attributes - attr_mismatch = False - for attr, regex in test_attr_regex_filters.items(): - if not re.fullmatch(regex, _unflattened_lookup(test, attr) or ""): - attr_mismatch = True - break + + # Check if any test attributes match filters + if test_filters: + for attr, value in test_filters.items(): + # Only prefix filter doesn't use regex + if attr == "prefix": + if not test.get_name().startswith(value): + attr_mismatch = True + break + else: # Match filters using regex + attr_value = _unflattened_lookup(test, attr) or "" + if not re.fullmatch(value, attr_value): + attr_mismatch = True + break if attr_mismatch: continue + if not run_jailed_tests: clone_test = copy.deepcopy(test) clone_test.update_from_s3() diff --git a/release/ray_release/buildkite/settings.py b/release/ray_release/buildkite/settings.py index 17ab6f9db657..07cfbd70fcf2 100644 --- a/release/ray_release/buildkite/settings.py +++ b/release/ray_release/buildkite/settings.py @@ -63,11 +63,11 @@ def get_priority(priority_str: str) -> Priority: return priority_str_to_enum[priority_str] -def get_test_attr_regex_filters(filters_str: str) -> Dict[str, str]: +def get_test_filters(filters_str: str) -> Dict[str, str]: if not filters_str: return {} - test_attr_regex_filters = {} + test_filters = {} for line in filters_str.splitlines(): line = line.strip() if not line: @@ -75,11 +75,10 @@ def get_test_attr_regex_filters(filters_str: str) -> Dict[str, str]: parts = line.split(":", maxsplit=1) if len(parts) != 2: raise ReleaseTestConfigError( - f"Invalid test attr regex filter: {line}. " - "Should be of the form attr:regex" + f"Invalid test filter: {line}. " "Should be of the form attr:value" ) - test_attr_regex_filters[parts[0]] = parts[1] - return test_attr_regex_filters + test_filters[parts[0]] = parts[1] + return test_filters def split_ray_repo_str(repo_str: str) -> Tuple[str, str]: @@ -127,7 +126,7 @@ def get_default_settings() -> Dict: settings = { "frequency": Frequency.ANY, "prefer_smoke_tests": False, - "test_attr_regex_filters": None, + "test_filters": None, "ray_test_repo": None, "ray_test_branch": None, "priority": Priority.DEFAULT, @@ -158,12 +157,13 @@ def update_settings_from_environment(settings: Dict) -> Dict: if "TEST_NAME" in os.environ: # This is for backward compatibility. - settings["test_attr_regex_filters"] = get_test_attr_regex_filters( - "name:" + os.environ["TEST_NAME"] - ) + settings["test_filters"] = get_test_filters("name:" + os.environ["TEST_NAME"]) + + if "TEST_FILTERS" in os.environ: + settings["test_filters"] = os.environ["TEST_FILTERS"] if "TEST_ATTR_REGEX_FILTERS" in os.environ: - settings["test_attr_regex_filters"] = get_test_attr_regex_filters( + settings["test_filters"] = get_test_filters( os.environ["TEST_ATTR_REGEX_FILTERS"] ) @@ -191,17 +191,11 @@ def update_settings_from_buildkite(settings: Dict): test_name_filter = get_buildkite_prompt_value("release-test-name") if test_name_filter: - settings["test_attr_regex_filters"] = get_test_attr_regex_filters( - "name:" + test_name_filter - ) + settings["test_filters"] = get_test_filters("name:" + test_name_filter) - test_attr_regex_filters = get_buildkite_prompt_value( - "release-test-attr-regex-filters" - ) - if test_attr_regex_filters: - settings["test_attr_regex_filters"] = get_test_attr_regex_filters( - test_attr_regex_filters - ) + test_filters = get_buildkite_prompt_value("release-test-filters") + if test_filters: + settings["test_filters"] = get_test_filters(test_filters) test_priority = get_buildkite_prompt_value("release-priority") if test_priority: diff --git a/release/ray_release/scripts/build_pipeline.py b/release/ray_release/scripts/build_pipeline.py index 5c6cc5b5998f..4cf21864da0a 100644 --- a/release/ray_release/scripts/build_pipeline.py +++ b/release/ray_release/scripts/build_pipeline.py @@ -78,14 +78,14 @@ def main( env = {} frequency = settings["frequency"] prefer_smoke_tests = settings["prefer_smoke_tests"] - test_attr_regex_filters = settings["test_attr_regex_filters"] + test_filters = settings["test_filters"] priority = settings["priority"] logger.info( f"Found the following buildkite pipeline settings:\n\n" f" frequency = {settings['frequency']}\n" f" prefer_smoke_tests = {settings['prefer_smoke_tests']}\n" - f" test_attr_regex_filters = {settings['test_attr_regex_filters']}\n" + f" test_filters = {settings['test_filters']}\n" f" ray_test_repo = {settings['ray_test_repo']}\n" f" ray_test_branch = {settings['ray_test_branch']}\n" f" priority = {settings['priority']}\n" @@ -110,7 +110,7 @@ def main( filtered_tests = filter_tests( test_collection, frequency=frequency, - test_attr_regex_filters=test_attr_regex_filters, + test_filters=test_filters, prefer_smoke_tests=prefer_smoke_tests, run_jailed_tests=run_jailed_tests, run_unstable_tests=run_unstable_tests, diff --git a/release/ray_release/tests/test_buildkite.py b/release/ray_release/tests/test_buildkite.py index 79e709cfb434..08128ee3c8f0 100644 --- a/release/ray_release/tests/test_buildkite.py +++ b/release/ray_release/tests/test_buildkite.py @@ -20,7 +20,7 @@ Frequency, update_settings_from_buildkite, Priority, - get_test_attr_regex_filters, + get_test_filters, ) from ray_release.buildkite.step import ( get_step, @@ -110,23 +110,23 @@ def testSplitRayRepoStr(self): self.assertEqual(branch, DEFAULT_BRANCH) def testGetTestAttrRegexFilters(self): - test_attr_regex_filters = get_test_attr_regex_filters("") - self.assertDictEqual(test_attr_regex_filters, {}) + test_filters = get_test_filters("") + self.assertDictEqual(test_filters, {}) - test_attr_regex_filters = get_test_attr_regex_filters("name:xxx") - self.assertDictEqual(test_attr_regex_filters, {"name": "xxx"}) + test_filters = get_test_filters("name:xxx") + self.assertDictEqual(test_filters, {"name": "xxx"}) - test_attr_regex_filters = get_test_attr_regex_filters("name:xxx\n") - self.assertDictEqual(test_attr_regex_filters, {"name": "xxx"}) + test_filters = get_test_filters("name:xxx\n") + self.assertDictEqual(test_filters, {"name": "xxx"}) - test_attr_regex_filters = get_test_attr_regex_filters("name:xxx\n\nteam:yyy") - self.assertDictEqual(test_attr_regex_filters, {"name": "xxx", "team": "yyy"}) + test_filters = get_test_filters("name:xxx\n\nteam:yyy") + self.assertDictEqual(test_filters, {"name": "xxx", "team": "yyy"}) - test_attr_regex_filters = get_test_attr_regex_filters("name:xxx\n \nteam:yyy\n") - self.assertDictEqual(test_attr_regex_filters, {"name": "xxx", "team": "yyy"}) + test_filters = get_test_filters("name:xxx\n \nteam:yyy\n") + self.assertDictEqual(test_filters, {"name": "xxx", "team": "yyy"}) with self.assertRaises(ReleaseTestConfigError): - get_test_attr_regex_filters("xxx") + get_test_filters("xxx") def testSettingsOverrideEnv(self): settings = get_default_settings() @@ -168,8 +168,9 @@ def testSettingsOverrideEnv(self): os.environ["TEST_ATTR_REGEX_FILTERS"] = "name:xxx\nteam:yyy\n" updated_settings = settings.copy() update_settings_from_environment(updated_settings) + print(updated_settings) self.assertDictEqual( - updated_settings["test_attr_regex_filters"], + updated_settings["test_filters"], { "name": "xxx", "team": "yyy", @@ -191,7 +192,7 @@ def testSettingsOverrideEnv(self): { "frequency": Frequency.NIGHTLY, "prefer_smoke_tests": False, - "test_attr_regex_filters": {"name": "name_filter"}, + "test_filters": {"name": "name_filter"}, "ray_test_repo": "https://github.com/user/ray.git", "ray_test_branch": "sub/branch", "priority": Priority.MANUAL, @@ -206,7 +207,7 @@ def testSettingsOverrideEnv(self): { "frequency": Frequency.ANY, "prefer_smoke_tests": True, - "test_attr_regex_filters": {"name": "name_filter"}, + "test_filters": {"name": "name_filter"}, "ray_test_repo": "https://github.com/user/ray.git", "ray_test_branch": "sub/branch", "priority": Priority.MANUAL, @@ -321,18 +322,18 @@ def testSettingsOverrideBuildkite(self): # Invalid test attr regex filters self.buildkite.clear() self.buildkite.update(buildkite) - self.buildkite["release-test-attr-regex-filters"] = "xxxx" + self.buildkite["release-test-filters"] = "xxxx" updated_settings = settings.copy() with self.assertRaises(ReleaseTestConfigError): update_settings_from_buildkite(updated_settings) self.buildkite.clear() self.buildkite.update(buildkite) - self.buildkite["release-test-attr-regex-filters"] = "name:xxx\ngroup:yyy" + self.buildkite["release-test-filters"] = "name:xxx\ngroup:yyy" updated_settings = settings.copy() update_settings_from_buildkite(updated_settings) self.assertDictEqual( - updated_settings["test_attr_regex_filters"], + updated_settings["test_filters"], { "name": "xxx", "group": "yyy", @@ -353,7 +354,7 @@ def testSettingsOverrideBuildkite(self): { "frequency": Frequency.NIGHTLY, "prefer_smoke_tests": False, - "test_attr_regex_filters": {"name": "name_filter"}, + "test_filters": {"name": "name_filter"}, "ray_test_repo": "https://github.com/user/ray.git", "ray_test_branch": "sub/branch", "priority": Priority.MANUAL, @@ -369,7 +370,7 @@ def testSettingsOverrideBuildkite(self): { "frequency": Frequency.ANY, "prefer_smoke_tests": True, - "test_attr_regex_filters": {"name": "name_filter"}, + "test_filters": {"name": "name_filter"}, "ray_test_repo": "https://github.com/user/ray.git", "ray_test_branch": "sub/branch", "priority": Priority.MANUAL, @@ -377,7 +378,7 @@ def testSettingsOverrideBuildkite(self): }, ) - def _filter_names_smoke(self, *args, **kwargs): + def _filter_names(self, *args, **kwargs): filtered = filter_tests(*args, **kwargs) return [(t[0]["name"], t[1]) for t in filtered] @@ -429,7 +430,32 @@ def testFilterTests(self, *args): ), ] - filtered = self._filter_names_smoke(tests, frequency=Frequency.ANY) + # Test filter by prefix alone + filtered = self._filter_names( + tests, frequency=Frequency.ANY, test_filters={"prefix": "test"} + ) + self.assertSequenceEqual( + filtered, + [ + ("test_1", False), + ("test_2", False), + ("test_3", False), + ("test_4.kuberay", False), + ], + ) + + # Test filter by prefix and regex together + filtered = self._filter_names( + tests, + frequency=Frequency.NIGHTLY, + test_filters={"prefix": "test", "name": "other.*"}, + ) + self.assertSequenceEqual( + filtered, + [], + ) + + filtered = self._filter_names(tests, frequency=Frequency.ANY) self.assertSequenceEqual( filtered, [ @@ -444,7 +470,7 @@ def testFilterTests(self, *args): ) assert not test.get("update_from_s3") - filtered = self._filter_names_smoke( + filtered = self._filter_names( tests, frequency=Frequency.ANY, prefer_smoke_tests=True, @@ -462,7 +488,7 @@ def testFilterTests(self, *args): ], ) - filtered = self._filter_names_smoke(tests, frequency=Frequency.NIGHTLY) + filtered = self._filter_names(tests, frequency=Frequency.NIGHTLY) self.assertSequenceEqual( filtered, [ @@ -474,7 +500,7 @@ def testFilterTests(self, *args): ], ) - filtered = self._filter_names_smoke( + filtered = self._filter_names( tests, frequency=Frequency.NIGHTLY, prefer_smoke_tests=True, @@ -490,13 +516,13 @@ def testFilterTests(self, *args): ], ) - filtered = self._filter_names_smoke(tests, frequency=Frequency.WEEKLY) + filtered = self._filter_names(tests, frequency=Frequency.WEEKLY) self.assertSequenceEqual(filtered, [("test_2", False), ("other_1", False)]) - filtered = self._filter_names_smoke( + filtered = self._filter_names( tests, frequency=Frequency.NIGHTLY, - test_attr_regex_filters={"name": "other.*"}, + test_filters={"name": "other.*"}, ) self.assertSequenceEqual( filtered, @@ -505,10 +531,10 @@ def testFilterTests(self, *args): ], ) - filtered = self._filter_names_smoke( + filtered = self._filter_names( tests, frequency=Frequency.NIGHTLY, - test_attr_regex_filters={"name": "test.*"}, + test_filters={"name": "test.*"}, ) self.assertSequenceEqual( filtered, @@ -520,46 +546,46 @@ def testFilterTests(self, *args): ], ) - filtered = self._filter_names_smoke( - tests, frequency=Frequency.NIGHTLY, test_attr_regex_filters={"name": "test"} + filtered = self._filter_names( + tests, frequency=Frequency.NIGHTLY, test_filters={"name": "test"} ) self.assertSequenceEqual(filtered, []) - filtered = self._filter_names_smoke( + filtered = self._filter_names( tests, frequency=Frequency.NIGHTLY, - test_attr_regex_filters={"name": "test.*", "team": "team_1"}, + test_filters={"name": "test.*", "team": "team_1"}, ) self.assertSequenceEqual(filtered, [("test_1", False)]) - filtered = self._filter_names_smoke( + filtered = self._filter_names( tests, frequency=Frequency.NIGHTLY, - test_attr_regex_filters={"name": "test_1|test_2"}, + test_filters={"name": "test_1|test_2"}, ) self.assertSequenceEqual(filtered, [("test_1", False), ("test_2", True)]) # Filter by nested properties - filtered = self._filter_names_smoke( + filtered = self._filter_names( tests, frequency=Frequency.ANY, - test_attr_regex_filters={"run/type": "job"}, + test_filters={"run/type": "job"}, ) self.assertSequenceEqual( filtered, [("test_1", False), ("other_2", False), ("test_4.kuberay", False)] ) - filtered = self._filter_names_smoke( + filtered = self._filter_names( tests, frequency=Frequency.ANY, - test_attr_regex_filters={"run/type": "client"}, + test_filters={"run/type": "client"}, ) self.assertSequenceEqual(filtered, [("test_2", False)]) - filtered = self._filter_names_smoke( + filtered = self._filter_names( tests, frequency=Frequency.ANY, - test_attr_regex_filters={"run/invalid": "xxx"}, + test_filters={"run/invalid": "xxx"}, ) self.assertSequenceEqual(filtered, []) From ce02a626e0b56e22f296761e9a5280f38e8b148a Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Wed, 27 Aug 2025 16:04:03 -0700 Subject: [PATCH 314/634] [image] move release wanda yaml files to docker/ (#55959) out of `ci/docker`, so that they are closer to the related `Dockerfile` and build contexts. Signed-off-by: Lonnie Liu --- .buildkite/_forge.rayci.yml | 24 +++++++++---------- .buildkite/linux_aarch64.rayci.yml | 8 +++---- .buildkite/release/build.rayci.yml | 8 +++---- ci/pipeline/test_rules.txt | 4 ---- .../base-deps/cpu.wanda.yaml | 0 .../base-deps/cuda.wanda.yaml | 0 .../base-extra-testdeps/cpu.wanda.yaml | 0 .../base-extra-testdeps/cuda.wanda.yaml | 0 .../base-extra/cpu.wanda.yaml | 0 .../base-extra/cuda.wanda.yaml | 0 .../base-slim/cpu.wanda.yaml | 0 .../base-slim/cuda.wanda.yaml | 0 .../ray-llm/cuda.wanda.yaml | 0 .../ray-ml/cpu.wanda.yaml | 0 .../ray-ml/cuda.wanda.yaml | 0 15 files changed, 20 insertions(+), 24 deletions(-) rename ci/docker/ray.cpu.base.wanda.yaml => docker/base-deps/cpu.wanda.yaml (100%) rename ci/docker/ray.cuda.base.wanda.yaml => docker/base-deps/cuda.wanda.yaml (100%) rename ci/docker/ray.cpu.base-extra-testdeps.wanda.yaml => docker/base-extra-testdeps/cpu.wanda.yaml (100%) rename ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml => docker/base-extra-testdeps/cuda.wanda.yaml (100%) rename ci/docker/ray.cpu.base-extra.wanda.yaml => docker/base-extra/cpu.wanda.yaml (100%) rename ci/docker/ray.cuda.base-extra.wanda.yaml => docker/base-extra/cuda.wanda.yaml (100%) rename ci/docker/ray-slim.cpu.base.wanda.yaml => docker/base-slim/cpu.wanda.yaml (100%) rename ci/docker/ray-slim.cuda.base.wanda.yaml => docker/base-slim/cuda.wanda.yaml (100%) rename ci/docker/ray-llm.base.wanda.yaml => docker/ray-llm/cuda.wanda.yaml (100%) rename ci/docker/ray-ml.cpu.base.wanda.yaml => docker/ray-ml/cpu.wanda.yaml (100%) rename ci/docker/ray-ml.cuda.base.wanda.yaml => docker/ray-ml/cuda.wanda.yaml (100%) diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml index 97acc658cc5a..220f556651f7 100644 --- a/.buildkite/_forge.rayci.yml +++ b/.buildkite/_forge.rayci.yml @@ -12,7 +12,7 @@ steps: tags: - python_dependencies - docker - wanda: ci/docker/ray.cpu.base.wanda.yaml + wanda: docker/base-deps/cpu.wanda.yaml matrix: - "3.9" - "3.10" @@ -24,7 +24,7 @@ steps: - name: raycpubaseextra label: "wanda: ray.py{{matrix}}.cpu.base-extra" - wanda: ci/docker/ray.cpu.base-extra.wanda.yaml + wanda: docker/base-extra/cpu.wanda.yaml matrix: - "3.9" - "3.10" @@ -41,7 +41,7 @@ steps: tags: - python_dependencies - docker - wanda: ci/docker/ray.cuda.base.wanda.yaml + wanda: docker/base-deps/cuda.wanda.yaml matrix: setup: python: @@ -65,7 +65,7 @@ steps: - name: raycudabaseextra label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" - wanda: ci/docker/ray.cuda.base-extra.wanda.yaml + wanda: docker/base-extra/cuda.wanda.yaml matrix: setup: python: @@ -94,7 +94,7 @@ steps: tags: - python_dependencies - docker - wanda: ci/docker/ray-llm.base.wanda.yaml + wanda: docker/ray-llm/cuda.wanda.yaml depends_on: raycudabase matrix: setup: @@ -108,7 +108,7 @@ steps: - name: ray-llmbaseextra label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" - wanda: ci/docker/ray.cuda.base-extra.wanda.yaml + wanda: docker/base-extra/cuda.wanda.yaml matrix: setup: python: @@ -127,7 +127,7 @@ steps: tags: - python_dependencies - docker - wanda: ci/docker/ray-ml.cuda.base.wanda.yaml + wanda: docker/ray-ml/cuda.wanda.yaml depends_on: raycudabase matrix: setup: @@ -143,7 +143,7 @@ steps: - name: ray-mlcudabaseextra label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" - wanda: ci/docker/ray.cuda.base-extra.wanda.yaml + wanda: docker/base-extra/cuda.wanda.yaml matrix: setup: python: @@ -164,7 +164,7 @@ steps: tags: - python_dependencies - docker - wanda: ci/docker/ray-ml.cpu.base.wanda.yaml + wanda: docker/ray-ml/cpu.wanda.yaml depends_on: raycpubase matrix: - "3.9" @@ -175,7 +175,7 @@ steps: - name: ray-mlcpubaseextra label: "wanda: ray-ml.py{{matrix}}.cpu.base-extra" - wanda: ci/docker/ray.cpu.base-extra.wanda.yaml + wanda: docker/base-extra/cpu.wanda.yaml matrix: - "3.9" - "3.10" @@ -191,7 +191,7 @@ steps: tags: - python_dependencies - docker - wanda: ci/docker/ray-slim.cpu.base.wanda.yaml + wanda: docker/base-slim/cpu.wanda.yaml depends_on: raycpubase matrix: - "3.9" @@ -207,7 +207,7 @@ steps: tags: - python_dependencies - docker - wanda: ci/docker/ray-slim.cuda.base.wanda.yaml + wanda: docker/base-slim/cuda.wanda.yaml depends_on: raycudabase matrix: setup: diff --git a/.buildkite/linux_aarch64.rayci.yml b/.buildkite/linux_aarch64.rayci.yml index 91bab540ab12..14c397b3d9db 100644 --- a/.buildkite/linux_aarch64.rayci.yml +++ b/.buildkite/linux_aarch64.rayci.yml @@ -18,7 +18,7 @@ steps: tags: - python_dependencies - docker - wanda: ci/docker/ray.cpu.base.wanda.yaml + wanda: docker/base-deps/cpu.wanda.yaml matrix: - "3.9" - "3.10" @@ -31,7 +31,7 @@ steps: - name: raycpubaseextra-aarch64 label: "wanda: ray.py{{matrix}}.cpu.base-extra (aarch64)" - wanda: ci/docker/ray.cpu.base-extra.wanda.yaml + wanda: docker/base-extra/cpu.wanda.yaml matrix: - "3.9" - "3.10" @@ -49,7 +49,7 @@ steps: tags: - python_dependencies - docker - wanda: ci/docker/ray.cuda.base.wanda.yaml + wanda: docker/base-deps/cuda.wanda.yaml matrix: setup: python: @@ -74,7 +74,7 @@ steps: - name: raycudabaseextra-aarch64 label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra (aarch64)" - wanda: ci/docker/ray.cuda.base-extra.wanda.yaml + wanda: docker/base-extra/cuda.wanda.yaml matrix: setup: python: diff --git a/.buildkite/release/build.rayci.yml b/.buildkite/release/build.rayci.yml index 1a9137189753..9c7a816c6eae 100644 --- a/.buildkite/release/build.rayci.yml +++ b/.buildkite/release/build.rayci.yml @@ -2,7 +2,7 @@ group: release build steps: - name: raycpubaseextra-testdeps label: "wanda: ray.py{{matrix}}.cpu.base-extra-testdeps" - wanda: ci/docker/ray.cpu.base-extra-testdeps.wanda.yaml + wanda: docker/base-extra-testdeps/cpu.wanda.yaml matrix: - "3.9" - "3.11" @@ -16,7 +16,7 @@ steps: - name: raycudabaseextra-testdeps label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra-testdeps" - wanda: ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml + wanda: docker/base-extra-testdeps/cuda.wanda.yaml matrix: setup: python: @@ -35,7 +35,7 @@ steps: - name: ray-llmbaseextra-testdeps label: "wanda: ray.py{{matrix.python}}.llm.base-extra-testdeps (cuda {{matrix.cuda}})" - wanda: ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml + wanda: docker/base-extra-testdeps/cuda.wanda.yaml matrix: setup: python: @@ -52,7 +52,7 @@ steps: - name: ray-mlcudabaseextra-testdeps label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.ml.base-extra-testdeps" - wanda: ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml + wanda: docker/base-extra-testdeps/cuda.wanda.yaml matrix: setup: python: diff --git a/ci/pipeline/test_rules.txt b/ci/pipeline/test_rules.txt index 1e3246f2a8bc..f19d274cfee0 100644 --- a/ci/pipeline/test_rules.txt +++ b/ci/pipeline/test_rules.txt @@ -203,10 +203,6 @@ ci/docker/forge.aarch64.wanda.yaml ci/docker/manylinux.Dockerfile ci/docker/manylinux.wanda.yaml ci/docker/manylinux.aarch64.wanda.yaml -ci/docker/ray.cpu.base.wanda.yaml -ci/docker/ray.cpu.base.aarch64.wanda.yaml -ci/docker/ray.cuda.base.wanda.yaml -ci/docker/ray.cuda.base.aarch64.wanda.yaml ci/docker/windows.build.Dockerfile ci/docker/windows.build.wanda.yaml build-docker.sh diff --git a/ci/docker/ray.cpu.base.wanda.yaml b/docker/base-deps/cpu.wanda.yaml similarity index 100% rename from ci/docker/ray.cpu.base.wanda.yaml rename to docker/base-deps/cpu.wanda.yaml diff --git a/ci/docker/ray.cuda.base.wanda.yaml b/docker/base-deps/cuda.wanda.yaml similarity index 100% rename from ci/docker/ray.cuda.base.wanda.yaml rename to docker/base-deps/cuda.wanda.yaml diff --git a/ci/docker/ray.cpu.base-extra-testdeps.wanda.yaml b/docker/base-extra-testdeps/cpu.wanda.yaml similarity index 100% rename from ci/docker/ray.cpu.base-extra-testdeps.wanda.yaml rename to docker/base-extra-testdeps/cpu.wanda.yaml diff --git a/ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml b/docker/base-extra-testdeps/cuda.wanda.yaml similarity index 100% rename from ci/docker/ray.cuda.base-extra-testdeps.wanda.yaml rename to docker/base-extra-testdeps/cuda.wanda.yaml diff --git a/ci/docker/ray.cpu.base-extra.wanda.yaml b/docker/base-extra/cpu.wanda.yaml similarity index 100% rename from ci/docker/ray.cpu.base-extra.wanda.yaml rename to docker/base-extra/cpu.wanda.yaml diff --git a/ci/docker/ray.cuda.base-extra.wanda.yaml b/docker/base-extra/cuda.wanda.yaml similarity index 100% rename from ci/docker/ray.cuda.base-extra.wanda.yaml rename to docker/base-extra/cuda.wanda.yaml diff --git a/ci/docker/ray-slim.cpu.base.wanda.yaml b/docker/base-slim/cpu.wanda.yaml similarity index 100% rename from ci/docker/ray-slim.cpu.base.wanda.yaml rename to docker/base-slim/cpu.wanda.yaml diff --git a/ci/docker/ray-slim.cuda.base.wanda.yaml b/docker/base-slim/cuda.wanda.yaml similarity index 100% rename from ci/docker/ray-slim.cuda.base.wanda.yaml rename to docker/base-slim/cuda.wanda.yaml diff --git a/ci/docker/ray-llm.base.wanda.yaml b/docker/ray-llm/cuda.wanda.yaml similarity index 100% rename from ci/docker/ray-llm.base.wanda.yaml rename to docker/ray-llm/cuda.wanda.yaml diff --git a/ci/docker/ray-ml.cpu.base.wanda.yaml b/docker/ray-ml/cpu.wanda.yaml similarity index 100% rename from ci/docker/ray-ml.cpu.base.wanda.yaml rename to docker/ray-ml/cpu.wanda.yaml diff --git a/ci/docker/ray-ml.cuda.base.wanda.yaml b/docker/ray-ml/cuda.wanda.yaml similarity index 100% rename from ci/docker/ray-ml.cuda.base.wanda.yaml rename to docker/ray-ml/cuda.wanda.yaml From c63d5a3f2e0cda07982c2091386453bca72f7c7f Mon Sep 17 00:00:00 2001 From: Kai-Hsun Chen Date: Wed, 27 Aug 2025 16:05:39 -0700 Subject: [PATCH 315/634] =?UTF-8?q?[core][gpu-objects]=20tensor=5Ftranspor?= =?UTF-8?q?t=20doesn=E2=80=99t=20transfer=20correctly=20when=20the=20argum?= =?UTF-8?q?ent=20is=20not=20inlined=20(#55876)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Step 1: Call an actor method to create an object ref. ```python ref = sender.produce.remote() ``` * `TaskManager::AddPendingTask`: Adds `tensor_transport` from the task spec to the object ref. * `submit_actor_task` calls VectorToObjectRefs to convert `CObjectReference` (C++) into an `ObjectRef` (Python). We also pass `tensor_transport` to the `ObjectRef`. * Step 2: Pass the GPU object ref from the driver to the receiver actor ```python receiver.consume.remote(ref) ``` * Driver calls `prepare_args_internal` to convert an `ObjectRef` (Python) into a `CTaskArgByReference` (C++). * The `TaskArgByReference` constructor converts an integer into `rpc::TensorTransport`. * `SubmitActorTask` -> `BuildCommonTaskSpec` -> `AddArg` -> `arg.ToProto` * `TaskArgByReference::ToProto` sets `tensor_transport` in `TaskArgByReference` into the gRPC message. * Step 3: Receiver actor * Pass `tensor_transport` from `message ObjectReference` to `message TaskArg`. * `deserialize_objects`: use `tensor_transport` from the object ref instead of Ray objects in some cases. Closes #54281 --------- Signed-off-by: kaihsun --- python/ray/_raylet.pxd | 3 +++ python/ray/_raylet.pyx | 20 ++++++++++++---- python/ray/includes/common.pxd | 7 +++++- python/ray/includes/object_ref.pxi | 10 ++++++-- python/ray/tests/test_gpu_objects_gloo.py | 23 +++++++++++++++++++ src/ray/common/task/task_util.h | 15 ++++++++---- src/ray/core_worker/task_manager.cc | 1 + .../task_submission/dependency_resolver.cc | 3 +++ src/ray/protobuf/common.proto | 3 +++ 9 files changed, 73 insertions(+), 12 deletions(-) diff --git a/python/ray/_raylet.pxd b/python/ray/_raylet.pxd index a45f127a3291..89ef3261db2a 100644 --- a/python/ray/_raylet.pxd +++ b/python/ray/_raylet.pxd @@ -110,9 +110,12 @@ cdef class ObjectRef(BaseID): # it up. c_bool in_core_worker c_string call_site_data + int tensor_transport_val cdef CObjectID native(self) + cdef CTensorTransport c_tensor_transport(self) + cdef class ActorID(BaseID): cdef CActorID data diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index fdd0dea05c72..4b4ce13e0205 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -598,7 +598,7 @@ class SerializedRayObject(NamedTuple): cdef RayObjectsToSerializedRayObjects( - const c_vector[shared_ptr[CRayObject]] objects): + const c_vector[shared_ptr[CRayObject]] objects, object_refs: Optional[List[ObjectRef]] = None): serialized_ray_objects = [] for i in range(objects.size()): # core_worker will return a nullptr for objects that couldn't be @@ -614,6 +614,11 @@ cdef RayObjectsToSerializedRayObjects( metadata = Buffer.make( objects[i].get().GetMetadata()).to_pybytes() tensor_transport = TensorTransportEnum((objects[i].get().GetTensorTransport())) + if ( + tensor_transport == TensorTransportEnum.OBJECT_STORE + and object_refs is not None + ): + tensor_transport = TensorTransportEnum(object_refs[i].tensor_transport()) serialized_ray_objects.append(SerializedRayObject(data, metadata, tensor_transport)) return serialized_ray_objects @@ -622,11 +627,13 @@ cdef VectorToObjectRefs(const c_vector[CObjectReference] &object_refs, skip_adding_local_ref): result = [] for i in range(object_refs.size()): + tensor_transport_val = object_refs[i].tensor_transport() result.append(ObjectRef( object_refs[i].object_id(), object_refs[i].owner_address().SerializeAsString(), object_refs[i].call_site(), - skip_adding_local_ref=skip_adding_local_ref)) + skip_adding_local_ref=skip_adding_local_ref, + tensor_transport_val=tensor_transport_val)) return result @@ -937,11 +944,13 @@ cdef prepare_args_internal( op_status = CCoreWorkerProcess.GetCoreWorker().GetOwnerAddress( c_arg, &c_owner_address) check_status(op_status) + c_tensor_transport = (arg).c_tensor_transport() args_vector.push_back( unique_ptr[CTaskArg](new CTaskArgByReference( c_arg, c_owner_address, - arg.call_site()))) + arg.call_site(), + c_tensor_transport))) else: try: @@ -998,7 +1007,8 @@ cdef prepare_args_internal( new CTaskArgByReference( put_id, CCoreWorkerProcess.GetCoreWorker().GetRpcAddress(), - put_arg_call_site + put_arg_call_site, + TENSOR_TRANSPORT_OBJECT_STORE ))) incremented_put_arg_ids.push_back(put_id) @@ -1882,10 +1892,10 @@ cdef void execute_task( if c_args.empty(): args, kwargs = [], {} else: - metadata_pairs = RayObjectsToSerializedRayObjects(c_args) object_refs = VectorToObjectRefs( c_arg_refs, skip_adding_local_ref=False) + metadata_pairs = RayObjectsToSerializedRayObjects(c_args, object_refs) if core_worker.current_actor_is_asyncio(): # We deserialize objects in event loop thread to # prevent segfaults. See #7799 diff --git a/python/ray/includes/common.pxd b/python/ray/includes/common.pxd index 6e50fd8ae35d..d15612e6dd20 100644 --- a/python/ray/includes/common.pxd +++ b/python/ray/includes/common.pxd @@ -204,6 +204,7 @@ cdef extern from "src/ray/protobuf/common.pb.h" nogil: CAddress owner_address() const const c_string &object_id() const const c_string &call_site() const + CTensorTransport tensor_transport() const cdef cppclass CNodeLabelSchedulingStrategy "ray::rpc::NodeLabelSchedulingStrategy": # noqa: E501 CNodeLabelSchedulingStrategy() CLabelMatchExpressions* mutable_hard() @@ -263,6 +264,9 @@ cdef extern from "src/ray/protobuf/common.pb.h" nogil: cdef extern from "src/ray/protobuf/common.pb.h" nogil: cdef CTensorTransport TENSOR_TRANSPORT_OBJECT_STORE "ray::rpc::TensorTransport::OBJECT_STORE" + cdef CTensorTransport TENSOR_TRANSPORT_NCCL "ray::rpc::TensorTransport::NCCL" + cdef CTensorTransport TENSOR_TRANSPORT_GLOO "ray::rpc::TensorTransport::GLOO" + cdef CTensorTransport TENSOR_TRANSPORT_NIXL "ray::rpc::TensorTransport::NIXL" cdef extern from "src/ray/protobuf/common.pb.h" nogil: cdef CPlacementStrategy PLACEMENT_STRATEGY_PACK \ @@ -318,7 +322,8 @@ cdef extern from "ray/core_worker/common.h" nogil: cdef cppclass CTaskArgByReference "ray::TaskArgByReference": CTaskArgByReference(const CObjectID &object_id, const CAddress &owner_address, - const c_string &call_site) + const c_string &call_site, + const CTensorTransport &tensor_transport) cdef cppclass CTaskArgByValue "ray::TaskArgByValue": CTaskArgByValue(const shared_ptr[CRayObject] &data) diff --git a/python/ray/includes/object_ref.pxi b/python/ray/includes/object_ref.pxi index f447c9aaa0ce..829ec790c4cf 100644 --- a/python/ray/includes/object_ref.pxi +++ b/python/ray/includes/object_ref.pxi @@ -40,12 +40,12 @@ cdef class ObjectRef(BaseID): def __init__( self, id, owner_addr="", call_site_data="", - skip_adding_local_ref=False): + skip_adding_local_ref=False, tensor_transport_val=0): self._set_id(id) self.owner_addr = owner_addr self.in_core_worker = False self.call_site_data = call_site_data - + self.tensor_transport_val = tensor_transport_val worker = ray._private.worker.global_worker # TODO(edoakes): We should be able to remove the in_core_worker flag. # But there are still some dummy object refs being created outside the @@ -152,3 +152,9 @@ cdef class ObjectRef(BaseID): core_worker = ray._private.worker.global_worker.core_worker core_worker.set_get_async_callback(self, py_callback) return self + + def tensor_transport(self): + return self.tensor_transport_val + + cdef CTensorTransport c_tensor_transport(self): + return self.tensor_transport_val diff --git a/python/ray/tests/test_gpu_objects_gloo.py b/python/ray/tests/test_gpu_objects_gloo.py index 4b01babd9a69..c15b05e2ade6 100644 --- a/python/ray/tests/test_gpu_objects_gloo.py +++ b/python/ray/tests/test_gpu_objects_gloo.py @@ -388,6 +388,29 @@ def test_mix_cpu_gpu_data(ray_start_regular): tensor = torch.randn((1,)) cpu_data = random.randint(0, 100) + + data = [tensor, cpu_data] + + sender, receiver = actors[0], actors[1] + ref = sender.echo.remote(data) + ref = receiver.double.remote(ref) + result = ray.get(ref) + + assert result[0] == pytest.approx(tensor * 2) + assert result[1] == cpu_data * 2 + + +def test_object_in_plasma(ray_start_regular): + """ + This test uses a CPU object that is large enough to be stored + in plasma instead of being inlined in the gRPC message. + """ + world_size = 2 + actors = [GPUTestActor.remote() for _ in range(world_size)] + create_collective_group(actors, backend="torch_gloo") + + tensor = torch.randn((1,)) + cpu_data = b"1" * 1000 * 1000 data = [tensor, cpu_data] sender, receiver = actors[0], actors[1] diff --git a/src/ray/common/task/task_util.h b/src/ray/common/task/task_util.h index 5779cd2ca3f2..265aed3fb51e 100644 --- a/src/ray/common/task/task_util.h +++ b/src/ray/common/task/task_util.h @@ -56,16 +56,22 @@ class TaskArgByReference : public TaskArg { /// /// \param[in] object_id Id of the argument. /// \return The task argument. - TaskArgByReference(const ObjectID &object_id, - const rpc::Address &owner_address, - const std::string &call_site) - : id_(object_id), owner_address_(owner_address), call_site_(call_site) {} + TaskArgByReference( + const ObjectID &object_id, + const rpc::Address &owner_address, + const std::string &call_site, + const rpc::TensorTransport &tensor_transport = rpc::TensorTransport::OBJECT_STORE) + : id_(object_id), + owner_address_(owner_address), + call_site_(call_site), + tensor_transport_(tensor_transport) {} void ToProto(rpc::TaskArg *arg_proto) const { auto ref = arg_proto->mutable_object_ref(); ref->set_object_id(id_.Binary()); ref->mutable_owner_address()->CopyFrom(owner_address_); ref->set_call_site(call_site_); + ref->set_tensor_transport(tensor_transport_); } private: @@ -73,6 +79,7 @@ class TaskArgByReference : public TaskArg { const ObjectID id_; const rpc::Address owner_address_; const std::string call_site_; + const rpc::TensorTransport tensor_transport_; }; class TaskArgByValue : public TaskArg { diff --git a/src/ray/core_worker/task_manager.cc b/src/ray/core_worker/task_manager.cc index 75f84cd21ad4..56aa59d5b495 100644 --- a/src/ray/core_worker/task_manager.cc +++ b/src/ray/core_worker/task_manager.cc @@ -277,6 +277,7 @@ std::vector TaskManager::AddPendingTask( ref.set_object_id(return_object_id.Binary()); ref.mutable_owner_address()->CopyFrom(caller_address); ref.set_call_site(call_site); + ref.set_tensor_transport(spec.TensorTransport()); // Register the callback to free the GPU object when it is out of scope. auto tensor_transport = reference_counter_.GetTensorTransport(return_object_id); diff --git a/src/ray/core_worker/task_submission/dependency_resolver.cc b/src/ray/core_worker/task_submission/dependency_resolver.cc index be99c9ddacb7..3b3c521cb8d1 100644 --- a/src/ray/core_worker/task_submission/dependency_resolver.cc +++ b/src/ray/core_worker/task_submission/dependency_resolver.cc @@ -77,6 +77,9 @@ void InlineDependencies( mutable_arg->add_nested_inlined_refs()->CopyFrom(nested_ref); contained_ids->push_back(ObjectID::FromBinary(nested_ref.object_id())); } + } else { + auto tensor_transport = mutable_arg->object_ref().tensor_transport(); + mutable_arg->set_tensor_transport(tensor_transport); } found++; } diff --git a/src/ray/protobuf/common.proto b/src/ray/protobuf/common.proto index 56dc85bb7020..84d91cf599b9 100644 --- a/src/ray/protobuf/common.proto +++ b/src/ray/protobuf/common.proto @@ -671,6 +671,9 @@ message ObjectReference { // Used to print debugging information if there is an error retrieving the // object. string call_site = 3; + // The tensor transport to use for this object. If not specified, then use the + // default object store. + optional TensorTransport tensor_transport = 4; } message ObjectReferenceCount { From 06393df8be6662c0e8c86f87b6d97e0300411ae7 Mon Sep 17 00:00:00 2001 From: Alan Guo Date: Wed, 27 Aug 2025 16:22:46 -0700 Subject: [PATCH 316/634] Add error_type to job failures (#55578) ## Why are these changes needed? Error type enums are useful as they are statically known values that should not contain sensitive information. The status message is dynamic and could contain sensitive pointers to logs or code and may be a field that should be hidden from others. This re-uses the existing error_type field and fills it in in all known error scenarios. ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Alan Guo --- python/ray/dashboard/modules/job/common.py | 37 ++++++++++++++-- .../ray/dashboard/modules/job/job_manager.py | 8 +++- .../dashboard/modules/job/job_supervisor.py | 4 +- .../modules/job/tests/test_common.py | 5 ++- .../modules/job/tests/test_job_manager.py | 44 ++++++++++++++----- python/ray/job_submission/__init__.py | 3 +- 6 files changed, 83 insertions(+), 18 deletions(-) diff --git a/python/ray/dashboard/modules/job/common.py b/python/ray/dashboard/modules/job/common.py index 9b543cd049d2..ee55cb78de13 100644 --- a/python/ray/dashboard/modules/job/common.py +++ b/python/ray/dashboard/modules/job/common.py @@ -66,6 +66,26 @@ def is_terminal(self) -> bool: return self.value in {"STOPPED", "SUCCEEDED", "FAILED"} +@PublicAPI(stability="stable") +class JobErrorType(str, Enum): + """An enumeration for describing the error type of a job.""" + + # Runtime environment failed to be set up + RUNTIME_ENV_SETUP_FAILURE = "RUNTIME_ENV_SETUP_FAILURE" + # Job supervisor actor launched, but job failed to start within timeout + JOB_SUPERVISOR_ACTOR_START_TIMEOUT = "JOB_SUPERVISOR_ACTOR_START_TIMEOUT" + # Job supervisor actor failed to start + JOB_SUPERVISOR_ACTOR_START_FAILURE = "JOB_SUPERVISOR_ACTOR_START_FAILURE" + # Job supervisor actor failed to be scheduled + JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE = "JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE" + # Job supervisor actor failed for unknown exception + JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE = "JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE" + # Job driver script failed to start due to exception + JOB_ENTRYPOINT_COMMAND_START_ERROR = "JOB_ENTRYPOINT_COMMAND_START_ERROR" + # Job driver script failed due to non-zero exit code + JOB_ENTRYPOINT_COMMAND_ERROR = "JOB_ENTRYPOINT_COMMAND_ERROR" + + # TODO(aguo): Convert to pydantic model @PublicAPI(stability="stable") @dataclass @@ -81,9 +101,8 @@ class JobInfo: entrypoint: str #: A message describing the status in more detail. message: Optional[str] = None - # TODO(architkulkarni): Populate this field with e.g. Runtime env setup failure, #: Internal error, user script error - error_type: Optional[str] = None + error_type: Optional[JobErrorType] = None #: The time when the job was started. A Unix timestamp in ms. start_time: Optional[int] = None #: The time when the job moved into a terminal state. A Unix timestamp in ms. @@ -157,6 +176,9 @@ def to_json(self) -> Dict[str, Any]: # Convert enum values to strings. json_dict["status"] = str(json_dict["status"]) + json_dict["error_type"] = ( + json_dict["error_type"].value if json_dict.get("error_type") else None + ) # Convert runtime_env to a JSON-serialized string. if "runtime_env" in json_dict: @@ -181,6 +203,11 @@ def from_json(cls, json_dict: Dict[str, Any]) -> None: """ # Convert enum values to enum objects. json_dict["status"] = JobStatus(json_dict["status"]) + json_dict["error_type"] = ( + JobErrorType(json_dict["error_type"]) + if json_dict.get("error_type") + else None + ) # Convert runtime_env from a JSON-serialized string to a dictionary. if "runtime_env_json" in json_dict: @@ -322,6 +349,7 @@ async def put_status( status: JobStatus, message: Optional[str] = None, driver_exit_code: Optional[int] = None, + error_type: Optional[JobErrorType] = None, jobinfo_replace_kwargs: Optional[Dict[str, Any]] = None, ): """Puts or updates job status. Sets end_time if status is terminal.""" @@ -331,7 +359,10 @@ async def put_status( if jobinfo_replace_kwargs is None: jobinfo_replace_kwargs = dict() jobinfo_replace_kwargs.update( - status=status, message=message, driver_exit_code=driver_exit_code + status=status, + message=message, + driver_exit_code=driver_exit_code, + error_type=error_type, ) if old_info is not None: if status != old_info.status and old_info.status.is_terminal(): diff --git a/python/ray/dashboard/modules/job/job_manager.py b/python/ray/dashboard/modules/job/job_manager.py index 5479757ea1cd..8c3a13690768 100644 --- a/python/ray/dashboard/modules/job/job_manager.py +++ b/python/ray/dashboard/modules/job/job_manager.py @@ -33,7 +33,7 @@ from ray.dashboard.modules.job.utils import get_head_node_id from ray.dashboard.utils import close_logger_file_descriptor from ray.exceptions import ActorUnschedulableError, RuntimeEnvSetupError -from ray.job_submission import JobStatus +from ray.job_submission import JobStatus, JobErrorType from ray.runtime_env import RuntimeEnvConfig from ray.util.scheduling_strategies import ( NodeAffinitySchedulingStrategy, @@ -208,6 +208,7 @@ async def _monitor_job_internal( job_id, JobStatus.FAILED, message=err_msg, + error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_TIMEOUT, ) is_alive = False logger.error(err_msg) @@ -234,6 +235,7 @@ async def _monitor_job_internal( "Unexpected error occurred: " "failed to get job supervisor." ), + error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_FAILURE, ) is_alive = False continue @@ -264,6 +266,7 @@ async def _monitor_job_internal( job_id, job_status, message=job_error_message, + error_type=JobErrorType.RUNTIME_ENV_SETUP_FAILURE, ) elif isinstance(e, ActorUnschedulableError): logger.info( @@ -277,6 +280,7 @@ async def _monitor_job_internal( job_id, JobStatus.FAILED, message=job_error_message, + error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE, ) else: logger.warning( @@ -288,6 +292,7 @@ async def _monitor_job_internal( job_id, job_status, message=job_error_message, + error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE, ) # Log error message to the job driver file for easy access. @@ -575,6 +580,7 @@ async def submit_job( f"Failed to start supervisor actor {submission_id}: '{e}'" f". Full traceback:\n{tb_str}" ), + error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_FAILURE, ) finally: close_logger_file_descriptor(driver_logger) diff --git a/python/ray/dashboard/modules/job/job_supervisor.py b/python/ray/dashboard/modules/job/job_supervisor.py index 9b9536a4129e..1ffaad752f9c 100644 --- a/python/ray/dashboard/modules/job/job_supervisor.py +++ b/python/ray/dashboard/modules/job/job_supervisor.py @@ -24,7 +24,7 @@ JobInfoStorageClient, ) from ray.dashboard.modules.job.job_log_storage_client import JobLogStorageClient -from ray.job_submission import JobStatus +from ray.job_submission import JobStatus, JobErrorType from ray._common.network_utils import build_address import psutil @@ -450,6 +450,7 @@ async def run( JobStatus.FAILED, message=message, driver_exit_code=return_code, + error_type=JobErrorType.JOB_ENTRYPOINT_COMMAND_ERROR, ) except Exception: self._logger.error( @@ -461,6 +462,7 @@ async def run( self._job_id, JobStatus.FAILED, message=traceback.format_exc(), + error_type=JobErrorType.JOB_ENTRYPOINT_COMMAND_START_ERROR, ) except Exception: self._logger.error( diff --git a/python/ray/dashboard/modules/job/tests/test_common.py b/python/ray/dashboard/modules/job/tests/test_common.py index 1bd9d51b9f87..03ca01dc5282 100644 --- a/python/ray/dashboard/modules/job/tests/test_common.py +++ b/python/ray/dashboard/modules/job/tests/test_common.py @@ -7,6 +7,7 @@ from ray.dashboard.modules.job.common import ( JobInfo, JobStatus, + JobErrorType, JobSubmitRequest, http_uri_components_to_uri, uri_to_http_components, @@ -179,7 +180,7 @@ def test_job_info_json_to_proto(): info = JobInfo( status=JobStatus.PENDING, entrypoint="echo hi", - error_type="error_type", + error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE, start_time=123, end_time=456, metadata={"hi": "hi2"}, @@ -208,7 +209,7 @@ def test_job_info_json_to_proto(): "(CPUs, GPUs, memory, custom resources) to become available. " "It may be waiting for the runtime environment to be set up." ) - assert info_proto.error_type == "error_type" + assert info_proto.error_type == "JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE" assert info_proto.driver_agent_http_address == "http://localhost:1234" assert info_proto.driver_node_id == "node_id" diff --git a/python/ray/dashboard/modules/job/tests/test_job_manager.py b/python/ray/dashboard/modules/job/tests/test_job_manager.py index 4481acb667b8..d0ed6271a8ff 100644 --- a/python/ray/dashboard/modules/job/tests/test_job_manager.py +++ b/python/ray/dashboard/modules/job/tests/test_job_manager.py @@ -37,7 +37,7 @@ create_job_manager, create_ray_cluster, ) -from ray.job_submission import JobStatus +from ray.job_submission import JobStatus, JobErrorType from ray.tests.conftest import call_ray_start # noqa: F401 from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy # noqa: F401 from ray.util.state import list_tasks @@ -412,9 +412,12 @@ async def check_job_succeeded(job_manager, job_id): return status == JobStatus.SUCCEEDED -async def check_job_failed(job_manager, job_id): - status = await job_manager.get_job_status(job_id) +async def check_job_failed(job_manager, job_id, expected_error_type=None): + data = await job_manager.get_job_info(job_id) + status = data.status assert status in {JobStatus.PENDING, JobStatus.RUNNING, JobStatus.FAILED} + if expected_error_type: + assert data.error_type == expected_error_type return status == JobStatus.FAILED @@ -720,7 +723,10 @@ async def test_failed_runtime_env_setup(self, job_manager): ) await async_wait_for_condition( - check_job_failed, job_manager=job_manager, job_id=job_id + check_job_failed, + job_manager=job_manager, + job_id=job_id, + expected_error_type=JobErrorType.RUNTIME_ENV_SETUP_FAILURE, ) data = await job_manager.get_job_info(job_id) @@ -880,7 +886,10 @@ async def test_kill_job_actor_in_before_driver_finish(self, job_manager): actor = job_manager._get_actor_for_job(job_id) ray.kill(actor, no_restart=True) await async_wait_for_condition( - check_job_failed, job_manager=job_manager, job_id=job_id + check_job_failed, + job_manager=job_manager, + job_id=job_id, + expected_error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE, ) data = await job_manager.get_job_info(job_id) assert data.driver_exit_code is None @@ -934,7 +943,10 @@ async def test_kill_job_actor_in_pending(self, job_manager): actor = job_manager._get_actor_for_job(job_id) ray.kill(actor, no_restart=True) await async_wait_for_condition( - check_job_failed, job_manager=job_manager, job_id=job_id + check_job_failed, + job_manager=job_manager, + job_id=job_id, + expected_error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE, ) data = await job_manager.get_job_info(job_id) assert data.driver_exit_code is None @@ -1040,7 +1052,10 @@ async def test_failed_job(self, job_manager): print(lines, end="") await async_wait_for_condition( - check_job_failed, job_manager=job_manager, job_id=job_id + check_job_failed, + job_manager=job_manager, + job_id=job_id, + expected_error_type=JobErrorType.JOB_ENTRYPOINT_COMMAND_ERROR, ) # check if the driver is killed data = await job_manager.get_job_info(job_id) @@ -1255,7 +1270,10 @@ async def test_failed_job_logs_max_char(job_manager): ) await async_wait_for_condition( - check_job_failed, job_manager=job_manager, job_id=job_id + check_job_failed, + job_manager=job_manager, + job_id=job_id, + expected_error_type=JobErrorType.JOB_ENTRYPOINT_COMMAND_ERROR, ) # Verify the status message length @@ -1330,7 +1348,10 @@ async def test_job_pending_timeout(job_manager, monkeypatch): # Wait for the job to timeout. await async_wait_for_condition( - check_job_failed, job_manager=job_manager, job_id=job_id + check_job_failed, + job_manager=job_manager, + job_id=job_id, + expected_error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_TIMEOUT, ) # Check that the job timed out. @@ -1355,7 +1376,10 @@ async def test_failed_driver_exit_code(job_manager): job_id = await job_manager.submit_job(entrypoint=exit_code_cmd) # Wait for the job to timeout. await async_wait_for_condition( - check_job_failed, job_manager=job_manager, job_id=job_id + check_job_failed, + job_manager=job_manager, + job_id=job_id, + expected_error_type=JobErrorType.JOB_ENTRYPOINT_COMMAND_ERROR, ) # Check that the job failed diff --git a/python/ray/job_submission/__init__.py b/python/ray/job_submission/__init__.py index 6a86cf73c329..b3a76be1e535 100644 --- a/python/ray/job_submission/__init__.py +++ b/python/ray/job_submission/__init__.py @@ -1,10 +1,11 @@ -from ray.dashboard.modules.job.common import JobInfo, JobStatus +from ray.dashboard.modules.job.common import JobErrorType, JobInfo, JobStatus from ray.dashboard.modules.job.pydantic_models import DriverInfo, JobDetails, JobType from ray.dashboard.modules.job.sdk import JobSubmissionClient __all__ = [ "JobSubmissionClient", "JobStatus", + "JobErrorType", "JobInfo", "JobDetails", "DriverInfo", From c3b6d49c928912ccc48e7e659642a3974f5e831d Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Wed, 27 Aug 2025 16:26:41 -0700 Subject: [PATCH 317/634] [cli] Ray Symmetric Run Script (#55111) ## Why are these changes needed? Implements symmetric-run for HPC and other settings. This aims to bridge the gap between running Ray and other styles of execution like torchrun. ## Related issue number Closes https://github.com/ray-project/ray/issues/55119 ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Richard Liaw Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python/ray/scripts/symmetric_run.py | 261 +++++++++++++++++++++++++ python/ray/tests/BUILD | 1 + python/ray/tests/test_symmetric_run.py | 169 ++++++++++++++++ 3 files changed, 431 insertions(+) create mode 100644 python/ray/scripts/symmetric_run.py create mode 100644 python/ray/tests/test_symmetric_run.py diff --git a/python/ray/scripts/symmetric_run.py b/python/ray/scripts/symmetric_run.py new file mode 100644 index 000000000000..e93087574369 --- /dev/null +++ b/python/ray/scripts/symmetric_run.py @@ -0,0 +1,261 @@ +"""Symmetric Run for Ray.""" + +from typing import List + +import click +import ray +import socket +import psutil +import subprocess +import sys +import time + +from ray._private.ray_constants import env_integer +from ray._raylet import GcsClient + +CLUSTER_WAIT_TIMEOUT = env_integer("RAY_SYMMETRIC_RUN_CLUSTER_WAIT_TIMEOUT", 30) + + +def check_ray_already_started() -> bool: + import ray._private.services as services + + # Try auto-detecting the Ray instance. + running_gcs_addresses = services.find_gcs_addresses() + return len(running_gcs_addresses) > 0 + + +def check_cluster_ready(nnodes, timeout=CLUSTER_WAIT_TIMEOUT): + """Wait for all nodes to start. + + Raises an exception if the nodes don't start in time. + """ + start_time = time.time() + current_nodes = 1 + ray.init(ignore_reinit_error=True) + + while time.time() - start_time < timeout: + time.sleep(5) + current_nodes = len(ray.nodes()) + if current_nodes == nnodes: + return True + else: + click.echo( + f"Waiting for nodes to start... {current_nodes}/{nnodes} nodes started" + ) + return False + + +def check_head_node_ready(address: str, timeout=CLUSTER_WAIT_TIMEOUT): + start_time = time.time() + gcs_client = GcsClient(address=address) + while time.time() - start_time < timeout: + if gcs_client.check_alive([], timeout=1): + click.echo("Ray cluster is ready!") + return True + time.sleep(5) + return False + + +def curate_and_validate_ray_start_args(run_and_start_args: List[str]) -> List[str]: + # Reparse the arguments to remove symmetric_run arguments. + ctx = symmetric_run.make_context("_", run_and_start_args, resilient_parsing=True) + cleaned_args = list(ctx.params["ray_args_and_entrypoint"]) + + for arg in cleaned_args: + if arg == "--head": + raise click.ClickException("Cannot use --head option in symmetric_run.") + if arg == "--node-ip-address": + raise click.ClickException( + "Cannot use --node-ip-address option in symmetric_run." + ) + if arg == "--port": + raise click.ClickException("Cannot use --port option in symmetric_run.") + if arg == "--block": + raise click.ClickException("Cannot use --block option in symmetric_run.") + + return cleaned_args + + +@click.command( + name="symmetric_run", + context_settings={"ignore_unknown_options": True, "allow_extra_args": True}, + help="""Command to start Ray across all nodes and execute an entrypoint command. + +USAGE: + + python -m ray.scripts.symmetric_run --address ADDRESS +[--min-nodes NUM_NODES] [RAY_START_OPTIONS] -- [ENTRYPOINT_COMMAND] + +DESCRIPTION: + + This command (1) starts a Ray cluster across all nodes, +(2) runs a command on the head node, and (3) stops the Ray cluster. + + The '--' separator is required to distinguish between Ray start arguments +and the entrypoint command. The --min-nodes option is optional and +can be used to wait for a specific number of nodes to start. + +EXAMPLES: + + # Start Ray with default settings and run a Python script + + python -m ray.scripts.symmetric_run --address 127.0.0.1:6379 -- python my_script.py + + # Start Ray with specific head node and run a command + + python -m ray.scripts.symmetric_run --address 127.0.0.1:6379 --min-nodes 4 -- python train_model.py --epochs=100 + + # Start Ray and run a multi-word command + + python -m ray.scripts.symmetric_run --address 127.0.0.1:6379 --min-nodes 4 --num-cpus=4 -- python -m my_module --config=prod + +RAY START OPTIONS: + + Most ray start command options are supported. Arguments that are not +supported are: --head, --node-ip-address, --port, --block. + +SEPARATOR REQUIREMENT: + + The '--' separator is mandatory and must appear between Ray start + arguments and the entrypoint command. This ensures clear separation + between the two sets of arguments. +""", +) +@click.option( + "--address", required=True, type=str, help="The address of the Ray cluster." +) +@click.option( + "--min-nodes", + type=int, + help="If provided, wait for this number of nodes to start.", +) +@click.argument("ray_args_and_entrypoint", nargs=-1, type=click.UNPROCESSED) +def symmetric_run(address, min_nodes, ray_args_and_entrypoint): + all_args = sys.argv[1:] + separator = all_args.index("--") + + if separator == -1: + raise click.ClickException("No separator '--' found in arguments.") + + run_and_start_args, entrypoint_on_head = ( + all_args[:separator], + all_args[separator + 1 :], + ) + + ray_start_args = curate_and_validate_ray_start_args(run_and_start_args) + + min_nodes = 1 if min_nodes is None else min_nodes + + if not entrypoint_on_head: + raise click.ClickException("No entrypoint command provided.") + + if check_ray_already_started(): + raise click.ClickException("Ray is already started on this node.") + + # 1. Parse address and check if we are on the head node. + gcs_host_port = ray._common.network_utils.parse_address(address) + if gcs_host_port is None: + raise click.ClickException( + f"Invalid address format: {address}, should be `host:port`" + ) + gcs_host, gcs_port = gcs_host_port + + try: + # AF_UNSPEC allows resolving both IPv4 and IPv6 + addrinfo = socket.getaddrinfo( + gcs_host, gcs_port, socket.AF_UNSPEC, socket.SOCK_STREAM + ) + resolved_gcs_host = addrinfo[0][4][0] + except socket.gaierror: + raise click.ClickException(f"Could not resolve hostname: {gcs_host}") + + my_ips = [] + for iface, addrs in psutil.net_if_addrs().items(): + for addr in addrs: + # Look for AF_INET (IPv4) or AF_INET6 (IPv6) + if addr.family in [ + socket.AddressFamily.AF_INET, + socket.AddressFamily.AF_INET6, + ]: + my_ips.append(addr.address) + + if min_nodes > 1: + # Ban localhost ips if we are not running on a single node + # to avoid starting N head nodes + my_ips = [ip for ip in my_ips if ip != "127.0.0.1" and ip != "::1"] + + is_head = resolved_gcs_host in my_ips + + result = None + # 2. Start Ray and run commands. + try: + if is_head: + # On the head node, start Ray, run the command, then stop Ray. + click.echo("On head node. Starting Ray cluster head...") + + # Build the ray start command with all parameters + ray_start_cmd = [ + "ray", + "start", + "--head", + f"--node-ip-address={resolved_gcs_host}", + f"--port={gcs_port}", + *ray_start_args, + ] + + # Start Ray head. This runs in the background and hides output. + subprocess.run(ray_start_cmd, check=True, capture_output=True) + click.echo("Head node started.") + click.echo("=======================") + if min_nodes > 1 and not check_cluster_ready(min_nodes): + raise click.ClickException( + "Timed out waiting for other nodes to start." + ) + + click.echo( + f"Running command on head node: {entrypoint_on_head}", + ) + click.echo("=======================") + result = subprocess.run(entrypoint_on_head) + click.echo("=======================") + else: + # On a worker node, start Ray and connect to the head. + click.echo(f"On worker node. Connecting to Ray cluster at {address}...") + + if not check_head_node_ready(address): + raise click.ClickException("Timed out waiting for head node to start.") + + # Build the ray start command for worker nodes with all parameters + ray_start_cmd = [ + "ray", + "start", + "--address", + address, + "--block", + *ray_start_args, + ] + + # This command will block until the Ray cluster is stopped. + subprocess.run(ray_start_cmd, check=True) + + except subprocess.CalledProcessError as e: + click.echo(f"Failed to start Ray: {e}", err=True) + if e.stdout: + click.echo(f"stdout:\n{e.stdout.decode()}", err=True) + if e.stderr: + click.echo(f"stderr:\n{e.stderr.decode()}", err=True) + except KeyboardInterrupt: + # This can be triggered by ctrl-c on the user's side. + click.echo("Interrupted by user.", err=True) + finally: + # Stop Ray cluster. + subprocess.run(["ray", "stop"]) + + # Propagate the exit code of the user script. + if result is not None and result.returncode != 0: + click.echo(f"Command failed with return code {result.returncode}", err=True) + sys.exit(result.returncode) + + +if __name__ == "__main__": + symmetric_run() diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD index e6c86d9adac5..1040d7411cc6 100644 --- a/python/ray/tests/BUILD +++ b/python/ray/tests/BUILD @@ -731,6 +731,7 @@ py_test_module_list( size = "medium", files = [ "test_autoscaler.py", + "test_symmetric_run.py", ], tags = [ "exclusive", diff --git a/python/ray/tests/test_symmetric_run.py b/python/ray/tests/test_symmetric_run.py new file mode 100644 index 000000000000..524e4c52b6a1 --- /dev/null +++ b/python/ray/tests/test_symmetric_run.py @@ -0,0 +1,169 @@ +import ray +import sys +import pytest +from contextlib import contextmanager +from unittest.mock import patch, MagicMock +from click.testing import CliRunner +import ray.scripts.scripts as scripts + + +@contextmanager +def _setup_mock_network_utils(curr_ip, head_ip): + import socket + + # Mock socket.getaddrinfo to return a valid IP + with patch("socket.getaddrinfo") as mock_getaddrinfo: + mock_getaddrinfo.return_value = [("", "", "", "", (curr_ip, 6379))] + + # Mock psutil.net_if_addrs to return localhost IP + with patch("psutil.net_if_addrs") as mock_net_if_addrs: + mock_net_if_addrs.return_value = { + "lo": [ + type( + "addr", + (), + {"family": socket.AF_INET, "address": head_ip}, + )() + ] + } + yield + + +@pytest.fixture +def cleanup_ray(): + """Shutdown all ray instances""" + yield + runner = CliRunner() + runner.invoke(scripts.stop, ["--force"]) + ray.shutdown() + + +def test_symmetric_run_basic_interface(monkeypatch, cleanup_ray): + """Test basic symmetric_run interface with minimal arguments.""" + from ray.scripts.symmetric_run import symmetric_run + + runner = CliRunner() + + # Mock subprocess.run to avoid actually starting Ray + with patch("subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + with _setup_mock_network_utils("127.0.0.1", "127.0.0.1"): + args = ["--address", "127.0.0.1:6379", "--", "echo", "test"] + + with patch("sys.argv", ["ray.scripts.symmetric_run", *args]): + # Test basic symmetric_run call using CliRunner + result = runner.invoke(symmetric_run, args) + assert result.exit_code == 0 + + # Verify that subprocess.run was called for ray start + assert mock_run.called + calls = mock_run.call_args_list + + # Should have called ray start with --head + ray_start_calls = [ + call for call in calls if "ray" in str(call) and "start" in str(call) + ] + assert len(ray_start_calls) > 0 + + # Should have called ray stop + ray_stop_calls = [ + call for call in calls if "ray" in str(call) and "stop" in str(call) + ] + assert len(ray_stop_calls) > 0 + + +def test_symmetric_run_worker_node_behavior(monkeypatch, cleanup_ray): + """Test symmetric_run behavior when not on the head node.""" + from ray.scripts.symmetric_run import symmetric_run + + runner = CliRunner() + + with patch("subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + + with _setup_mock_network_utils("192.168.1.100", "192.168.1.101"): + # Mock socket connection check to simulate head node ready + with patch("socket.socket") as mock_socket: + mock_socket_instance = MagicMock() + mock_socket_instance.connect_ex.return_value = 0 + mock_socket.return_value.__enter__.return_value = mock_socket_instance + + # Test worker node behavior + args = ["--address", "192.168.1.100:6379", "--", "echo", "test"] + with patch("sys.argv", ["ray.scripts.symmetric_run", *args]): + with patch( + "ray.scripts.symmetric_run.check_head_node_ready" + ) as mock_check_head_node_ready: + mock_check_head_node_ready.return_value = True + result = runner.invoke(symmetric_run, args) + assert result.exit_code == 0 + + # Verify that subprocess.run was called + assert mock_run.called + calls = mock_run.call_args_list + + # Should have called ray start with --address (worker mode) + ray_start_calls = [ + call + for call in calls + if "ray" in str(call) and "start" in str(call) + ] + assert len(ray_start_calls) > 0 + + # Check that it's in worker mode (--address instead of --head) + start_call = ray_start_calls[0] + start_args = start_call[0][0] + assert "--address" in start_args + assert "192.168.1.100:6379" in start_args + assert "--head" not in start_args + assert "--block" in start_args # Worker nodes should block + + +def test_symmetric_run_arg_validation(monkeypatch, cleanup_ray): + """Test that symmetric_run validates arguments.""" + from ray.scripts.symmetric_run import symmetric_run + + runner = CliRunner() + + # Mock subprocess.run to avoid actually starting Ray + with _setup_mock_network_utils("127.0.0.1", "127.0.0.1"): + + with patch("subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + args = ["--address", "127.0.0.1:6379", "--", "echo", "test"] + + with patch("sys.argv", ["ray.scripts.symmetric_run", *args]): + # Test basic symmetric_run call using CliRunner + result = runner.invoke(symmetric_run, args) + assert result.exit_code == 0 + + # Test that invalid arguments are rejected + with patch("subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + + args = ["--address", "127.0.0.1:6379", "--head", "--", "echo", "test"] + with patch("sys.argv", ["ray.scripts.symmetric_run", *args]): + result = runner.invoke(symmetric_run, args) + assert result.exit_code == 1 + assert "Cannot use --head option in symmetric_run." in result.output + + with patch("subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + + # Test args with "=" are passed to ray start + args = ["--address", "127.0.0.1:6379", "--num-cpus=4", "--", "echo", "test"] + with patch("sys.argv", ["ray.scripts.symmetric_run", *args]): + result = runner.invoke(symmetric_run, args) + assert result.exit_code == 0 + + ray_start_calls = [ + call + for call in mock_run.call_args_list + if "ray" in str(call) and "start" in str(call) + ] + assert len(ray_start_calls) > 0 + assert "--num-cpus=4" in ray_start_calls[0][0][0] + + +if __name__ == "__main__": + sys.exit(pytest.main(["-sv", __file__])) From 227dfb3d8b29cabb7a8eaabaf22a1e6d74880c3a Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Wed, 27 Aug 2025 18:29:13 -0500 Subject: [PATCH 318/634] [core] Remove `gcs_rpc_server.h` dependency from `GcsTaskManager` (#56001) Splitting gRPC service interface from implementation. Stacked on: https://github.com/ray-project/ray/pull/55999 --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 3 +- src/ray/gcs/gcs_server/gcs_server.cc | 12 ++- src/ray/gcs/gcs_server/gcs_task_manager.h | 6 +- .../gcs/gcs_server/grpc_service_interfaces.h | 21 +++++ src/ray/gcs/gcs_server/grpc_services.cc | 18 ++++ src/ray/gcs/gcs_server/grpc_services.h | 46 ++++++++++ src/ray/gcs/gcs_server/tests/BUILD.bazel | 1 - src/ray/rpc/gcs/gcs_rpc_server.h | 85 ------------------- 8 files changed, 99 insertions(+), 93 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 8d6de0908ca3..5e3de7e6ed16 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -199,6 +199,7 @@ ray_cc_library( hdrs = ["gcs_task_manager.h"], deps = [ ":gcs_usage_stats_client", + ":grpc_service_interfaces", "//src/ray/common:asio", "//src/ray/common:id", "//src/ray/common:ray_config", @@ -206,7 +207,7 @@ ray_cc_library( "//src/ray/gcs:gcs_pb_util", "//src/ray/protobuf:events_event_aggregator_service_cc_proto", "//src/ray/protobuf:gcs_cc_proto", - "//src/ray/rpc:gcs_server", + "//src/ray/stats:stats_metric", "//src/ray/util:counter_map", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 890d958d72c5..9648b6082ed5 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -735,10 +735,14 @@ void GcsServer::InitGcsTaskManager() { auto &io_context = io_context_provider_.GetIOContext(); gcs_task_manager_ = std::make_unique(io_context); // Register service. - rpc_server_.RegisterService( - std::make_unique(io_context, *gcs_task_manager_)); - rpc_server_.RegisterService( - std::make_unique(io_context, *gcs_task_manager_)); + rpc_server_.RegisterService(std::make_unique( + io_context, + *gcs_task_manager_, + RayConfig::instance().gcs_max_active_rpcs_per_handler())); + rpc_server_.RegisterService(std::make_unique( + io_context, + *gcs_task_manager_, + RayConfig::instance().gcs_max_active_rpcs_per_handler())); } void GcsServer::InstallEventListeners() { diff --git a/src/ray/gcs/gcs_server/gcs_task_manager.h b/src/ray/gcs/gcs_server/gcs_task_manager.h index a98d30c0f036..d6e5686100c7 100644 --- a/src/ray/gcs/gcs_server/gcs_task_manager.h +++ b/src/ray/gcs/gcs_server/gcs_task_manager.h @@ -24,9 +24,10 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/synchronization/mutex.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/usage_stats_client.h" #include "ray/gcs/pb_util.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" +#include "ray/stats/metric_defs.h" #include "ray/util/counter_map.h" #include "src/ray/protobuf/gcs.pb.h" @@ -92,7 +93,8 @@ class FinishedTaskActorTaskGcPolicy : public TaskEventsGcPolicyInterface { /// /// This class has its own io_context and io_thread, that's separate from other GCS /// services. All handling of all rpc should be posted to the single thread it owns. -class GcsTaskManager : public rpc::TaskInfoHandler, public rpc::RayEventExportHandler { +class GcsTaskManager : public rpc::TaskInfoGcsServiceHandler, + public rpc::RayEventExportGcsServiceHandler { public: /// Create a GcsTaskManager. explicit GcsTaskManager(instrumented_io_context &io_service); diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h index f1cf9b70f538..0c8d68bc7aad 100644 --- a/src/ray/gcs/gcs_server/grpc_service_interfaces.h +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -206,5 +206,26 @@ class InternalKVGcsServiceHandler { SendReplyCallback send_reply_callback) = 0; }; +class TaskInfoGcsServiceHandler { + public: + virtual ~TaskInfoGcsServiceHandler() = default; + + virtual void HandleAddTaskEventData(AddTaskEventDataRequest request, + AddTaskEventDataReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetTaskEvents(GetTaskEventsRequest request, + GetTaskEventsReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + +class RayEventExportGcsServiceHandler { + public: + virtual ~RayEventExportGcsServiceHandler() = default; + virtual void HandleAddEvents(events::AddEventsRequest request, + events::AddEventsReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc index 4157256be205..cf6fad71f43b 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -114,5 +114,23 @@ void InternalKVGrpcService::InitServerCallFactories( InternalKVGcsService, GetInternalConfig, max_active_rpcs_per_handler_) } +void TaskInfoGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + RPC_SERVICE_HANDLER(TaskInfoGcsService, AddTaskEventData, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(TaskInfoGcsService, GetTaskEvents, max_active_rpcs_per_handler_) +} + +using events::AddEventsReply; +using events::AddEventsRequest; + +void RayEventExportGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + RPC_SERVICE_HANDLER(RayEventExportGcsService, AddEvents, max_active_rpcs_per_handler_) +} + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/gcs_server/grpc_services.h index 0a463c2b6b6f..a4faaf4bf453 100644 --- a/src/ray/gcs/gcs_server/grpc_services.h +++ b/src/ray/gcs/gcs_server/grpc_services.h @@ -196,5 +196,51 @@ class InternalKVGrpcService : public GrpcService { int64_t max_active_rpcs_per_handler_; }; +class TaskInfoGrpcService : public GrpcService { + public: + explicit TaskInfoGrpcService(instrumented_io_context &io_service, + TaskInfoGcsServiceHandler &handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler){}; + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + TaskInfoGcsService::AsyncService service_; + TaskInfoGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + +class RayEventExportGrpcService : public GrpcService { + public: + explicit RayEventExportGrpcService(instrumented_io_context &io_service, + RayEventExportGcsServiceHandler &handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler){}; + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + RayEventExportGcsService::AsyncService service_; + RayEventExportGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index 417989a559c4..87fb68703f42 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -149,7 +149,6 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_task_manager", "//src/ray/gcs/tests:gcs_test_util_lib", diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index 7b812c13739e..4175e6271bf3 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -117,16 +117,6 @@ namespace rpc { HANDLER, \ RayConfig::instance().gcs_max_active_rpcs_per_handler()) -#define TASK_INFO_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(TaskInfoGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - -#define RAY_EVENT_EXPORT_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(RayEventExportGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - #define OBJECT_INFO_SERVICE_RPC_HANDLER(HANDLER) \ RPC_SERVICE_HANDLER(ObjectInfoGcsService, \ HANDLER, \ @@ -291,83 +281,8 @@ class PlacementGroupInfoGrpcService : public GrpcService { PlacementGroupInfoGcsServiceHandler &service_handler_; }; -class TaskInfoGcsServiceHandler { - public: - virtual ~TaskInfoGcsServiceHandler() = default; - - virtual void HandleAddTaskEventData(AddTaskEventDataRequest request, - AddTaskEventDataReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetTaskEvents(GetTaskEventsRequest request, - GetTaskEventsReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -/// The `GrpcService` for `TaskInfoGcsService`. -class TaskInfoGrpcService : public GrpcService { - public: - /// Constructor. - /// - /// \param[in] io_service IO service to run the handler. - /// \param[in] handler The service handler that actually handle the requests. - explicit TaskInfoGrpcService(instrumented_io_context &io_service, - TaskInfoGcsServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler){}; - - protected: - grpc::Service &GetGrpcService() override { return service_; } - - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - TASK_INFO_SERVICE_RPC_HANDLER(AddTaskEventData); - TASK_INFO_SERVICE_RPC_HANDLER(GetTaskEvents); - } - - private: - /// The grpc async service object. - TaskInfoGcsService::AsyncService service_; - /// The service handler that actually handle the requests. - TaskInfoGcsServiceHandler &service_handler_; -}; - -class RayEventExportGcsServiceHandler { - public: - virtual ~RayEventExportGcsServiceHandler() = default; - virtual void HandleAddEvents(AddEventsRequest request, - AddEventsReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -/// The `GrpcService` for `RayEventExportGcsService`. -class RayEventExportGrpcService : public GrpcService { - public: - explicit RayEventExportGrpcService(instrumented_io_context &io_service, - RayEventExportGcsServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler) {} - - protected: - grpc::Service &GetGrpcService() override { return service_; } - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - RAY_EVENT_EXPORT_SERVICE_RPC_HANDLER(AddEvents); - } - - private: - /// The grpc async service object. - RayEventExportGcsService::AsyncService service_; - /// The service handler that actually handle the requests. - RayEventExportGcsServiceHandler &service_handler_; -}; - using ActorInfoHandler = ActorInfoGcsServiceHandler; using PlacementGroupInfoHandler = PlacementGroupInfoGcsServiceHandler; -using TaskInfoHandler = TaskInfoGcsServiceHandler; -using RayEventExportHandler = RayEventExportGcsServiceHandler; } // namespace rpc } // namespace ray From b975ed086b022ad630ae4d4567ba0d5c31ea4119 Mon Sep 17 00:00:00 2001 From: ahao-anyscale Date: Wed, 27 Aug 2025 16:57:48 -0700 Subject: [PATCH 319/634] [serve.llm] Score API Integration for Serve LLM (#55914) Signed-off-by: ahao-anyscale Co-authored-by: Nikhil G Co-authored-by: angelinalg <122562471+angelinalg@users.noreply.github.com> --- doc/source/serve/llm/index.md | 1 + .../serve/configs/openai_api_models.py | 14 +++++++ .../serve/deployments/llm/llm_server.py | 25 ++++++++++- .../serve/deployments/llm/vllm/vllm_engine.py | 30 +++++++++++++- .../serve/deployments/routers/router.py | 41 ++++++++++++++++++- python/ray/llm/tests/serve/conftest.py | 11 +++++ .../cpu/deployments/llm/test_llm_engine.py | 15 +++++++ .../cpu/deployments/llm/test_llm_server.py | 28 +++++++++++++ .../llm/tests/serve/mocks/mock_vllm_engine.py | 37 +++++++++++++++++ .../llm/tests/serve/utils/testing_utils.py | 14 +++++++ 10 files changed, 212 insertions(+), 4 deletions(-) diff --git a/doc/source/serve/llm/index.md b/doc/source/serve/llm/index.md index 0cf2a27aa989..893b4a5d370a 100644 --- a/doc/source/serve/llm/index.md +++ b/doc/source/serve/llm/index.md @@ -36,6 +36,7 @@ This deployment provides an OpenAI-compatible FastAPI ingress and routes traffic - `/v1/chat/completions`: Chat interface (ChatGPT-style) - `/v1/completions`: Text completion - `/v1/embeddings`: Text embeddings +- `/v1/score`: Text comparison - `/v1/models`: List available models - `/v1/models/{model}`: Model information diff --git a/python/ray/llm/_internal/serve/configs/openai_api_models.py b/python/ray/llm/_internal/serve/configs/openai_api_models.py index bb0b195d93f4..2d118ab4742e 100644 --- a/python/ray/llm/_internal/serve/configs/openai_api_models.py +++ b/python/ray/llm/_internal/serve/configs/openai_api_models.py @@ -22,6 +22,8 @@ EmbeddingCompletionRequest as vLLMEmbeddingCompletionRequest, EmbeddingResponse as vLLMEmbeddingResponse, ErrorResponse as vLLMErrorResponse, + ScoreRequest as vLLMScoreRequest, + ScoreResponse as vLLMScoreResponse, ) from vllm.utils import random_uuid @@ -89,12 +91,24 @@ class EmbeddingResponse(vLLMEmbeddingResponse): model_config = ConfigDict(arbitrary_types_allowed=True) +class ScoreRequest(vLLMScoreRequest): + model_config = ConfigDict(arbitrary_types_allowed=True) + + +class ScoreResponse(vLLMScoreResponse): + model_config = ConfigDict(arbitrary_types_allowed=True) + + EmbeddingRequest = Union[EmbeddingCompletionRequest, EmbeddingChatRequest] LLMEmbeddingsResponse = Union[ AsyncGenerator[Union[EmbeddingResponse, ErrorResponse], None], ] +LLMScoreResponse = Union[ + AsyncGenerator[Union[ScoreResponse, ErrorResponse], None], +] + LLMChatResponse = Union[ AsyncGenerator[Union[str, ChatCompletionResponse, ErrorResponse], None], ] diff --git a/python/ray/llm/_internal/serve/deployments/llm/llm_server.py b/python/ray/llm/_internal/serve/deployments/llm/llm_server.py index 72a02525286e..faa9f98f87b5 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/llm_server.py +++ b/python/ray/llm/_internal/serve/deployments/llm/llm_server.py @@ -52,6 +52,8 @@ EmbeddingRequest, EmbeddingResponse, ErrorResponse, + ScoreRequest, + ScoreResponse, ) logger = get_logger(__name__) @@ -306,7 +308,10 @@ def _batch_output_stream( async def _run_request( self, request: Union[ - "ChatCompletionRequest", "CompletionRequest", "EmbeddingRequest" + "ChatCompletionRequest", + "CompletionRequest", + "EmbeddingRequest", + "ScoreRequest", ], *, engine_method: str, @@ -392,6 +397,24 @@ async def embeddings( request, engine_method="embeddings", batch_output_stream=False ) + async def score( + self, request: "ScoreRequest" + ) -> AsyncGenerator[Union["ScoreResponse", "ErrorResponse"], None]: + """Runs a score request to the engine and returns the response. + + Returns an AsyncGenerator over the ScoreResponse object. This is so that the caller can have a consistent interface across all the methods of chat, completions, embeddings, and score. + + Args: + request: A ScoreRequest object. + + Returns: + An AsyncGenerator over the ScoreResponse object. + """ + # NOTE: Score does not need batching, similar to embeddings. + return await self._run_request( + request, engine_method="score", batch_output_stream=False + ) + async def check_health(self) -> None: """ Check the health of the replica. Does not return anything. Raise error when diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py index 9e67fdbe30dd..9df60582bf2a 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py @@ -19,6 +19,8 @@ EmbeddingRequest, EmbeddingResponse, ErrorResponse, + ScoreRequest, + ScoreResponse, ) from ray.llm._internal.serve.configs.server_models import ( DiskMultiplexConfig, @@ -43,6 +45,7 @@ from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding from vllm.entrypoints.openai.serving_models import OpenAIServingModels + from vllm.entrypoints.openai.serving_score import ServingScores vllm = try_import("vllm") logger = get_logger(__name__) @@ -134,6 +137,7 @@ def __init__( self._oai_serving_chat: Optional["OpenAIServingChat"] = None self._oai_serving_completion: Optional["OpenAIServingCompletion"] = None self._oai_serving_embedding: Optional["OpenAIServingEmbedding"] = None + self._oai_serving_scores: Optional["ServingScores"] = None async def start(self) -> None: """Start the vLLM engine. @@ -189,6 +193,7 @@ async def start(self) -> None: self._oai_serving_chat = state.openai_serving_chat self._oai_serving_completion = state.openai_serving_completion self._oai_serving_embedding = state.openai_serving_embedding + self._oai_serving_scores = state.openai_serving_scores self._validate_openai_serving_models() self._validate_engine_client() @@ -221,6 +226,11 @@ def _validate_openai_serving_embedding(self): self._oai_serving_embedding, "create_embedding" ), "oai_serving_embedding must have a create_embedding attribute" + def _validate_openai_serving_scores(self): + assert hasattr( + self._oai_serving_scores, "create_score" + ), "oai_serving_scores must have a create_score attribute" + def _validate_engine_client(self): assert hasattr( self._engine_client, "check_health" @@ -354,7 +364,9 @@ async def resolve_lora(self, disk_lora_model: DiskMultiplexConfig): def _create_raw_request( self, - request: Union[CompletionRequest, ChatCompletionRequest, EmbeddingRequest], + request: Union[ + CompletionRequest, ChatCompletionRequest, EmbeddingRequest, ScoreRequest + ], path: str, ) -> Request: scope = { @@ -442,6 +454,22 @@ async def embeddings( else: yield EmbeddingResponse(**embedding_response.model_dump()) + async def score( + self, request: ScoreRequest + ) -> AsyncGenerator[Union[ScoreResponse, ErrorResponse], None]: + self._validate_openai_serving_scores() + + raw_request = self._create_raw_request(request, "/score") + + score_response = await self._oai_serving_scores.create_score( + request, raw_request=raw_request + ) + + if isinstance(score_response, VLLMErrorResponse): + yield ErrorResponse(**score_response.model_dump()) + else: + yield ScoreResponse(**score_response.model_dump()) + async def check_health(self) -> None: assert self._engine_client is not None, "engine_client is not initialized" diff --git a/python/ray/llm/_internal/serve/deployments/routers/router.py b/python/ray/llm/_internal/serve/deployments/routers/router.py index 0d22410c6ef8..d9af5b28e327 100644 --- a/python/ray/llm/_internal/serve/deployments/routers/router.py +++ b/python/ray/llm/_internal/serve/deployments/routers/router.py @@ -46,9 +46,12 @@ LLMChatResponse, LLMCompletionsResponse, LLMEmbeddingsResponse, + LLMScoreResponse, ModelCard, ModelList, OpenAIHTTPException, + ScoreRequest, + ScoreResponse, to_model_metadata, ) from ray.llm._internal.serve.configs.server_models import LLMConfig @@ -310,10 +313,18 @@ def _get_configured_serve_handle(self, model_id: str): async def _get_response( self, *, - body: Union[CompletionRequest, ChatCompletionRequest, EmbeddingRequest], + body: Union[ + CompletionRequest, ChatCompletionRequest, EmbeddingRequest, ScoreRequest + ], call_method: str, ) -> AsyncGenerator[ - Union[LLMChatResponse, LLMCompletionsResponse, LLMEmbeddingsResponse], None + Union[ + LLMChatResponse, + LLMCompletionsResponse, + LLMEmbeddingsResponse, + LLMScoreResponse, + ], + None, ]: """Calls the model deployment and returns the stream.""" model: str = body.model @@ -478,6 +489,32 @@ async def embeddings(self, body: EmbeddingRequest) -> Response: if isinstance(result, EmbeddingResponse): return JSONResponse(content=result.model_dump()) + @fastapi_router_app.post("/v1/score") + async def score(self, body: ScoreRequest) -> Response: + """Create scores for the provided text pairs. + + Note: This is a vLLM specific endpoint. + + Args: + body: The score request containing input text pairs to score. + + Returns: + A response object with scores. + """ + + async with timeout(DEFAULT_LLM_ROUTER_HTTP_TIMEOUT): + results = self._get_response(body=body, call_method="score") + result = await results.__anext__() + if isinstance(result, ErrorResponse): + raise OpenAIHTTPException( + message=result.message, + status_code=result.code, + type=result.type, + ) + + if isinstance(result, ScoreResponse): + return JSONResponse(content=result.model_dump()) + @classmethod def as_deployment( cls, llm_configs: Optional[List[LLMConfig]] = None diff --git a/python/ray/llm/tests/serve/conftest.py b/python/ray/llm/tests/serve/conftest.py index 55d94387f88c..f39a4bfc8483 100644 --- a/python/ray/llm/tests/serve/conftest.py +++ b/python/ray/llm/tests/serve/conftest.py @@ -15,6 +15,7 @@ ChatCompletionRequest, CompletionRequest, EmbeddingCompletionRequest, + ScoreRequest, ) from ray.llm._internal.serve.deployments.llm.vllm.vllm_models import ( VLLMEngineConfig, @@ -112,6 +113,16 @@ def mock_embedding_request(dimensions): return request +@pytest.fixture +def mock_score_request(): + """Fixture for creating score requests for mock testing.""" + return ScoreRequest( + model=MOCK_MODEL_ID, + text_1="What is the capital of France?", + text_2="The capital of France is Paris.", + ) + + def get_test_model_path(yaml_file: str) -> pathlib.Path: current_file_dir = pathlib.Path(__file__).absolute().parent test_model_path = current_file_dir / yaml_file diff --git a/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_engine.py b/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_engine.py index ef3603cfde2a..e473d6f4e14f 100644 --- a/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_engine.py +++ b/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_engine.py @@ -81,3 +81,18 @@ async def test_embedding_mock_engine( async for response in engine.embeddings(request): LLMResponseValidator.validate_embedding_response(response, dimensions) + + @pytest.mark.asyncio + async def test_score_mock_engine(self, mock_llm_config, mock_score_request): + """Test score API for text similarity.""" + # Create and start the engine + engine = MockVLLMEngine(mock_llm_config) + await engine.start() + + # Create score request + request = mock_score_request + + print("\n\n_____ SCORE _____\n\n") + + async for response in engine.score(request): + LLMResponseValidator.validate_score_response(response) diff --git a/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py b/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py index 20c517bd23b6..44404585003b 100644 --- a/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py +++ b/python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py @@ -152,6 +152,34 @@ async def test_embedding_llm_server( # Validate embedding response LLMResponseValidator.validate_embedding_response(chunks[0], dimensions) + @pytest.mark.asyncio + async def test_score_llm_server( + self, + serve_handle, + mock_llm_config, + mock_score_request, + ): + """Test score API from LLMServer perspective.""" + + # Create score request + request = mock_score_request + + print("\n\n_____ SCORE SERVER _____\n\n") + + # Get the response + batched_chunks = serve_handle.score.remote(request) + + # Collect responses (should be just one) + chunks = [] + async for batch in batched_chunks: + chunks.append(batch) + + # Check that we got one response + assert len(chunks) == 1 + + # Validate score response + LLMResponseValidator.validate_score_response(chunks[0]) + @pytest.mark.asyncio async def test_check_health(self, mock_llm_config): """Test health check functionality.""" diff --git a/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py b/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py index 6879d7db1272..068621fbfa71 100644 --- a/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py +++ b/python/ray/llm/tests/serve/mocks/mock_vllm_engine.py @@ -13,6 +13,8 @@ EmbeddingRequest, EmbeddingResponse, ErrorResponse, + ScoreRequest, + ScoreResponse, ) from ray.llm._internal.serve.configs.server_models import ( DiskMultiplexConfig, @@ -135,6 +137,41 @@ async def embeddings( ) yield response + async def score( + self, request: ScoreRequest + ) -> AsyncGenerator[Union[str, ScoreResponse, ErrorResponse], None]: + """Mock score generation for text pairs.""" + if not self.started: + raise RuntimeError("Engine not started") + + # Extract text_1 and text_2 from the request + text_1 = getattr(request, "text_1", "") + text_2 = getattr(request, "text_2", "") + + # Convert to lists if they aren't already + text_1_list = text_1 if isinstance(text_1, list) else [text_1] + text_2_list = text_2 if isinstance(text_2, list) else [text_2] + + # Generate mock scores for each pair + score_data = [] + for i, (t1, t2) in enumerate(zip(text_1_list, text_2_list)): + # Generate a random score (can be any float value) + score = random.uniform(-10.0, 10.0) + + score_data.append({"object": "score", "score": score, "index": i}) + + # Create the response + response = ScoreResponse( + object="list", + data=score_data, + model=getattr(request, "model", "mock-model"), + usage={ + "prompt_tokens": len(str(text_1).split()) + len(str(text_2).split()), + "total_tokens": len(str(text_1).split()) + len(str(text_2).split()), + }, + ) + yield response + async def _generate_chat_response( self, request: ChatCompletionRequest, prompt_text: str, max_tokens: int ) -> AsyncGenerator[Union[str, ChatCompletionResponse], None]: diff --git a/python/ray/llm/tests/serve/utils/testing_utils.py b/python/ray/llm/tests/serve/utils/testing_utils.py index fd62a3034deb..496dd2b50b0b 100644 --- a/python/ray/llm/tests/serve/utils/testing_utils.py +++ b/python/ray/llm/tests/serve/utils/testing_utils.py @@ -11,6 +11,7 @@ ChatCompletionResponse, CompletionResponse, EmbeddingResponse, + ScoreResponse, ) @@ -94,3 +95,16 @@ def validate_embedding_response( # Check dimensions if specified if expected_dimensions: assert len(response.data[0].embedding) == expected_dimensions + + @staticmethod + def validate_score_response(response: ScoreResponse): + """Validate score responses.""" + assert isinstance(response, ScoreResponse) + assert response.object == "list" + assert len(response.data) >= 1 + + # Validate each score data element + for i, score_data in enumerate(response.data): + assert score_data.object == "score" + assert isinstance(score_data.score, float) + assert score_data.index == i # Index should match position in list From 90230cee72e660a9bc7b072ebf3bd56f8d4d9d9f Mon Sep 17 00:00:00 2001 From: Abrar Sheikh Date: Wed, 27 Aug 2025 17:22:28 -0700 Subject: [PATCH 320/634] Add rank and world size in replica context (#55827) 1. `rank` and `world_size` are now available on `ReplicaContext`. 2. Replica initialization now requires providing a rank 3. Any change to the replicas rank will be communicated from controller via `.reconfigure()` method. 4. Assigned rank to replica can be fetched from `get_metadata()` function, this will be useful during controller recovery to reconstruct the state. This PR fills in a dummy rank value, in the future PR we will fetch the replica from DeploymentRankManager and pass in the correct value. Part 2 of https://github.com/ray-project/ray/pull/54938 Next diff in pipeline https://github.com/ray-project/ray/pull/55829 --------- Signed-off-by: abrar --- python/ray/serve/_private/deployment_state.py | 16 ++++++++- python/ray/serve/_private/replica.py | 35 ++++++++++++++----- python/ray/serve/context.py | 8 +++++ .../serve/tests/test_controller_recovery.py | 4 +-- python/ray/serve/tests/test_multiplex.py | 2 ++ python/ray/serve/tests/unit/test_batching.py | 2 ++ 6 files changed, 56 insertions(+), 11 deletions(-) diff --git a/python/ray/serve/_private/deployment_state.py b/python/ray/serve/_private/deployment_state.py index 87b1ee46502d..96658abd2cb9 100644 --- a/python/ray/serve/_private/deployment_state.py +++ b/python/ray/serve/_private/deployment_state.py @@ -250,6 +250,8 @@ def __init__( self._initialization_latency_s: Optional[float] = None self._port: Optional[int] = None self._docs_path: Optional[str] = None + # Rank assigned to the replica. + self._rank: Optional[int] = None # Populated in `on_scheduled` or `recover`. self._actor_handle: ActorHandle = None self._placement_group: PlacementGroup = None @@ -282,6 +284,10 @@ def replica_id(self) -> str: def deployment_name(self) -> str: return self._deployment_id.name + @property + def rank(self) -> Optional[int]: + return self._rank + @property def app_name(self) -> str: return self._deployment_id.app_name @@ -454,6 +460,8 @@ def start(self, deployment_info: DeploymentInfo) -> ReplicaSchedulingRequest: if self._deployment_is_cross_language else deployment_info.replica_config.serialized_init_args ) + # TODO(abrar): Fill in the correct rank + rank = 0 init_args = ( self.replica_id, cloudpickle.dumps(deployment_info.replica_config.deployment_def) @@ -467,6 +475,7 @@ def start(self, deployment_info: DeploymentInfo) -> ReplicaSchedulingRequest: self._version, deployment_info.ingress, deployment_info.route_prefix, + rank, ) # TODO(simon): unify the constructor arguments across language elif ( @@ -598,8 +607,12 @@ def reconfigure(self, version: DeploymentVersion) -> bool: deployment_config.user_config = self._format_user_config( deployment_config.user_config ) + # TODO(abrar): FIll in the correct rank + rank = 0 self._ready_obj_ref = self._actor_handle.reconfigure.remote( - deployment_config, version.route_prefix + deployment_config, + rank, + version.route_prefix, ) self._version = version @@ -729,6 +742,7 @@ def check_ready(self) -> Tuple[ReplicaStartupStatus, Optional[str]]: self._initialization_latency_s, self._port, self._docs_path, + self._rank, ) = ray.get(self._ready_obj_ref) except RayTaskError as e: logger.exception( diff --git a/python/ray/serve/_private/replica.py b/python/ray/serve/_private/replica.py index 5ce22606f4b9..6ddfe873d97b 100644 --- a/python/ray/serve/_private/replica.py +++ b/python/ray/serve/_private/replica.py @@ -115,6 +115,7 @@ Optional[float], Optional[int], Optional[str], + int, ] @@ -356,6 +357,7 @@ def __init__( version: DeploymentVersion, ingress: bool, route_prefix: str, + rank: int, ): self._version = version self._replica_id = replica_id @@ -402,7 +404,7 @@ def __init__( # Set metadata for logs and metrics. # servable_object will be populated in `initialize_and_get_metadata`. - self._set_internal_replica_context(servable_object=None) + self._set_internal_replica_context(servable_object=None, rank=rank) self._metrics_manager = create_replica_metrics_manager( replica_id=replica_id, @@ -422,19 +424,27 @@ def get_num_ongoing_requests(self) -> int: return self._metrics_manager.get_num_ongoing_requests() def get_metadata(self) -> ReplicaMetadata: + current_rank = ray.serve.context._get_internal_replica_context().rank return ( self._version.deployment_config, self._version, self._initialization_latency, self._port, self._docs_path, + current_rank, ) - def _set_internal_replica_context(self, *, servable_object: Callable = None): + def _set_internal_replica_context( + self, *, servable_object: Callable = None, rank: int = None + ): + # Calculate world_size from deployment config instead of storing it + world_size = self._deployment_config.num_replicas ray.serve.context._set_internal_replica_context( replica_id=self._replica_id, servable_object=servable_object, _deployment_config=self._deployment_config, + rank=rank, + world_size=world_size, ) def _configure_logger_and_profilers( @@ -752,7 +762,10 @@ async def initialize(self, deployment_config: DeploymentConfig): raise RuntimeError(traceback.format_exc()) from None async def reconfigure( - self, deployment_config: DeploymentConfig, route_prefix: Optional[str] = None + self, + deployment_config: DeploymentConfig, + rank: int, + route_prefix: Optional[str] = None, ): try: user_config_changed = ( @@ -782,9 +795,10 @@ async def reconfigure( ) # We need to update internal replica context to reflect the new - # deployment_config. + # deployment_config and rank. self._set_internal_replica_context( - servable_object=self._user_callable_wrapper.user_callable + servable_object=self._user_callable_wrapper.user_callable, + rank=rank, ) self._route_prefix = self._version.route_prefix @@ -894,8 +908,11 @@ async def record_routing_stats(self) -> Dict[str, Any]: class Replica(ReplicaBase): async def _on_initialized(self): + # Get current rank from replica context during initialization + current_rank = ray.serve.context._get_internal_replica_context().rank self._set_internal_replica_context( - servable_object=self._user_callable_wrapper.user_callable + servable_object=self._user_callable_wrapper.user_callable, + rank=current_rank, ) # Save the initialization latency if the replica is initializing @@ -969,6 +986,7 @@ async def __init__( version: DeploymentVersion, ingress: bool, route_prefix: str, + rank: int, ): deployment_config = DeploymentConfig.from_proto_bytes( deployment_config_proto_bytes @@ -985,6 +1003,7 @@ async def __init__( version=version, ingress=ingress, route_prefix=route_prefix, + rank=rank, ) def push_proxy_handle(self, handle: ActorHandle): @@ -1047,9 +1066,9 @@ async def record_routing_stats(self) -> Dict[str, Any]: return await self._replica_impl.record_routing_stats() async def reconfigure( - self, deployment_config, route_prefix: Optional[str] = None + self, deployment_config, rank: int, route_prefix: Optional[str] = None ) -> ReplicaMetadata: - await self._replica_impl.reconfigure(deployment_config, route_prefix) + await self._replica_impl.reconfigure(deployment_config, rank, route_prefix) return self._replica_impl.get_metadata() def _preprocess_request_args( diff --git a/python/ray/serve/context.py b/python/ray/serve/context.py index ecb412aab37b..99b8e5f0d9e8 100644 --- a/python/ray/serve/context.py +++ b/python/ray/serve/context.py @@ -41,11 +41,15 @@ class ReplicaContext: - deployment: name of the deployment the replica is a part of. - replica_tag: unique ID for the replica. - servable_object: instance of the user class/function this replica is running. + - rank: the rank of the replica. + - world_size: the number of replicas in the deployment. """ replica_id: ReplicaID servable_object: Callable _deployment_config: DeploymentConfig + rank: int + world_size: int @property def app_name(self) -> str: @@ -108,12 +112,16 @@ def _set_internal_replica_context( replica_id: ReplicaID, servable_object: Callable, _deployment_config: DeploymentConfig, + rank: int, + world_size: int, ): global _INTERNAL_REPLICA_CONTEXT _INTERNAL_REPLICA_CONTEXT = ReplicaContext( replica_id=replica_id, servable_object=servable_object, _deployment_config=_deployment_config, + rank=rank, + world_size=world_size, ) diff --git a/python/ray/serve/tests/test_controller_recovery.py b/python/ray/serve/tests/test_controller_recovery.py index 493931872ea9..52f405153a1e 100644 --- a/python/ray/serve/tests/test_controller_recovery.py +++ b/python/ray/serve/tests/test_controller_recovery.py @@ -65,7 +65,7 @@ def __call__(self, *args): replica_version_hash = None for replica in deployment_dict[id]: ref = replica.actor_handle.initialize_and_get_metadata.remote() - _, version, _, _, _ = ray.get(ref) + _, version, _, _, _, _ = ray.get(ref) if replica_version_hash is None: replica_version_hash = hash(version) assert replica_version_hash == hash(version), ( @@ -118,7 +118,7 @@ def __call__(self, *args): for replica_name in recovered_replica_names: actor_handle = ray.get_actor(replica_name, namespace=SERVE_NAMESPACE) ref = actor_handle.initialize_and_get_metadata.remote() - _, version, _, _, _ = ray.get(ref) + _, version, _, _, _, _ = ray.get(ref) assert replica_version_hash == hash( version ), "Replica version hash should be the same after recover from actor names" diff --git a/python/ray/serve/tests/test_multiplex.py b/python/ray/serve/tests/test_multiplex.py index 1ebc29066181..b93857f12c03 100644 --- a/python/ray/serve/tests/test_multiplex.py +++ b/python/ray/serve/tests/test_multiplex.py @@ -34,6 +34,8 @@ def start_serve_with_context(): ), servable_object=None, _deployment_config=DeploymentConfig(), + rank=0, + world_size=1, ) try: yield diff --git a/python/ray/serve/tests/unit/test_batching.py b/python/ray/serve/tests/unit/test_batching.py index fa83a018c85a..9c08ffdf6813 100644 --- a/python/ray/serve/tests/unit/test_batching.py +++ b/python/ray/serve/tests/unit/test_batching.py @@ -20,6 +20,8 @@ replica_id=ReplicaID(unique_id="test", deployment_id=DeploymentID(name="test")), servable_object=None, _deployment_config=default_deployment_config, + rank=0, + world_size=1, ) From a5d032ba1a69105902697390f553d3ed3afed5af Mon Sep 17 00:00:00 2001 From: Jeffrey Wang Date: Wed, 27 Aug 2025 17:27:58 -0700 Subject: [PATCH 321/634] [data.llm] Allow vLLM engine to be shared by sequential processors with serve deployments (#55179) Signed-off-by: jeffreyjeffreywang Co-authored-by: jeffreyjeffreywang --- python/ray/data/llm.py | 105 ++++++++ .../llm/_internal/batch/processor/__init__.py | 2 + .../batch/processor/serve_deployment_proc.py | 78 ++++++ .../llm/_internal/batch/stages/__init__.py | 2 + .../batch/stages/serve_deployment_stage.py | 156 ++++++++++++ .../serve/builders/application_builders.py | 4 + .../processor/test_serve_deployment_proc.py | 236 ++++++++++++++++++ .../gpu/stages/test_serve_deployment_stage.py | 177 +++++++++++++ python/ray/llm/tests/conftest.py | 66 +++++ python/ray/serve/llm/__init__.py | 12 +- 10 files changed, 836 insertions(+), 2 deletions(-) create mode 100644 python/ray/llm/_internal/batch/processor/serve_deployment_proc.py create mode 100644 python/ray/llm/_internal/batch/stages/serve_deployment_stage.py create mode 100644 python/ray/llm/tests/batch/gpu/processor/test_serve_deployment_proc.py create mode 100644 python/ray/llm/tests/batch/gpu/stages/test_serve_deployment_stage.py diff --git a/python/ray/data/llm.py b/python/ray/data/llm.py index bdae67778cfb..0e8055771795 100644 --- a/python/ray/data/llm.py +++ b/python/ray/data/llm.py @@ -5,6 +5,7 @@ HttpRequestProcessorConfig as _HttpRequestProcessorConfig, Processor, ProcessorConfig as _ProcessorConfig, + ServeDeploymentProcessorConfig as _ServeDeploymentProcessorConfig, SGLangEngineProcessorConfig as _SGLangEngineProcessorConfig, vLLMEngineProcessorConfig as _vLLMEngineProcessorConfig, ) @@ -244,6 +245,109 @@ class SGLangEngineProcessorConfig(_SGLangEngineProcessorConfig): pass +@PublicAPI(stability="alpha") +class ServeDeploymentProcessorConfig(_ServeDeploymentProcessorConfig): + """The configuration for the serve deployment processor. + + This processor enables sharing serve deployments across multiple processors. This is useful + for sharing the same LLM engine across multiple processors. + + Args: + deployment_name: The name of the serve deployment to use. + app_name: The name of the serve application to use. + batch_size: The batch size to send to the serve deployment. Large batch sizes are + likely to saturate the compute resources and could achieve higher throughput. + On the other hand, small batch sizes are more fault-tolerant and could + reduce bubbles in the data pipeline. You can tune the batch size to balance + the throughput and fault-tolerance based on your use case. + dtype_mapping: The mapping of the request class name to the request class. If this is + not provided, the serve deployment is expected to accept a dict as the request. + concurrency: The number of workers for data parallelism. Default to 1. Note that this is + not the concurrency of the underlying serve deployment. + + Examples: + + .. testcode:: + :skipif: True + + import ray + from ray import serve + from ray.data.llm import ServeDeploymentProcessorConfig, build_llm_processor + from ray.serve.llm import ( + LLMConfig, + ModelLoadingConfig, + build_llm_deployment, + ) + from ray.serve.llm.openai_api_models import CompletionRequest + + llm_config = LLMConfig( + model_loading_config=ModelLoadingConfig( + model_id="facebook/opt-1.3b", + model_source="facebook/opt-1.3b", + ), + accelerator_type="A10G", + deployment_config=dict( + name="facebook", + autoscaling_config=dict( + min_replicas=1, + max_replicas=1, + ), + ), + engine_kwargs=dict( + enable_prefix_caching=True, + enable_chunked_prefill=True, + max_num_batched_tokens=4096, + ), + ) + + APP_NAME = "facebook_opt_app" + DEPLOYMENT_NAME = "facebook_deployment" + override_serve_options = dict(name=DEPLOYMENT_NAME) + + llm_app = build_llm_deployment( + llm_config, override_serve_options=override_serve_options + ) + app = serve.run(llm_app, name=APP_NAME) + + config = ServeDeploymentProcessorConfig( + deployment_name=DEPLOYMENT_NAME, + app_name=APP_NAME, + dtype_mapping={ + "CompletionRequest": CompletionRequest, + }, + concurrency=1, + batch_size=64, + ) + processor = build_llm_processor( + config, + preprocess=lambda row: dict( + method="completions", + dtype="CompletionRequest", + request_kwargs=dict( + model="facebook/opt-1.3b", + prompt=f"This is a prompt for {row['id']}", + stream=False, + ), + ), + postprocess=lambda row: dict( + resp=row["choices"][0]["text"], + ), + ) + + # The processor requires specific input columns, which depend on + # your processor config. You can use the following API to check + # the required input columns: + processor.log_input_column_names() + + ds = ray.data.range(10) + ds = processor(ds) + for row in ds.take_all(): + print(row) + """ + + pass + + @PublicAPI(stability="alpha") def build_llm_processor( config: ProcessorConfig, @@ -324,5 +428,6 @@ def build_llm_processor( "HttpRequestProcessorConfig", "vLLMEngineProcessorConfig", "SGLangEngineProcessorConfig", + "ServeDeploymentProcessorConfig", "build_llm_processor", ] diff --git a/python/ray/llm/_internal/batch/processor/__init__.py b/python/ray/llm/_internal/batch/processor/__init__.py index 99388bbbaaef..fed7d021fe8e 100644 --- a/python/ray/llm/_internal/batch/processor/__init__.py +++ b/python/ray/llm/_internal/batch/processor/__init__.py @@ -1,5 +1,6 @@ from .base import Processor, ProcessorBuilder, ProcessorConfig from .http_request_proc import HttpRequestProcessorConfig +from .serve_deployment_proc import ServeDeploymentProcessorConfig from .sglang_engine_proc import SGLangEngineProcessorConfig from .vllm_engine_proc import vLLMEngineProcessorConfig @@ -9,5 +10,6 @@ "HttpRequestProcessorConfig", "vLLMEngineProcessorConfig", "SGLangEngineProcessorConfig", + "ServeDeploymentProcessorConfig", "Processor", ] diff --git a/python/ray/llm/_internal/batch/processor/serve_deployment_proc.py b/python/ray/llm/_internal/batch/processor/serve_deployment_proc.py new file mode 100644 index 000000000000..5a0b4e930318 --- /dev/null +++ b/python/ray/llm/_internal/batch/processor/serve_deployment_proc.py @@ -0,0 +1,78 @@ +"""The processor that runs serve deployment.""" + +from typing import Any, Dict, Optional, Type + +from pydantic import Field + +from ray.data.block import UserDefinedFunction +from ray.llm._internal.batch.processor.base import ( + Processor, + ProcessorBuilder, + ProcessorConfig, +) +from ray.llm._internal.batch.stages import ( + ServeDeploymentStage, +) + + +class ServeDeploymentProcessorConfig(ProcessorConfig): + """The configuration for the serve deployment processor.""" + + # Configurations used to build the serve deployment + deployment_name: str = Field( + description="The name of the serve deployment to use.", + ) + app_name: str = Field( + description="The name of the serve application to use.", + default="default", + ) + dtype_mapping: Dict[str, Type[Any]] = Field( + description="A dictionary mapping data type names to their corresponding request classes for the serve deployment.", + default=None, + ) + + +def build_serve_deployment_processor( + config: ServeDeploymentProcessorConfig, + preprocess: Optional[UserDefinedFunction] = None, + postprocess: Optional[UserDefinedFunction] = None, +) -> Processor: + """ + Construct a processor that runs a serve deployment. + + Args: + config: The configuration for the processor. + preprocess: An optional lambda function that takes a row (dict) as input + and returns a preprocessed row (dict). The output row must contain the + required fields for the following processing stages. + postprocess: An optional lambda function that takes a row (dict) as input + and returns a postprocessed row (dict). + + Returns: + The constructed processor. + """ + stages = [ + ServeDeploymentStage( + fn_constructor_kwargs=dict( + deployment_name=config.deployment_name, + app_name=config.app_name, + dtype_mapping=config.dtype_mapping, + ), + map_batches_kwargs=dict( + concurrency=config.concurrency, + ), + ) + ] + # TODO (Kourosh): Add telemetry for ServeDeploymentStage + processor = Processor( + config, + stages, + preprocess=preprocess, + postprocess=postprocess, + ) + return processor + + +ProcessorBuilder.register( + ServeDeploymentProcessorConfig, build_serve_deployment_processor +) diff --git a/python/ray/llm/_internal/batch/stages/__init__.py b/python/ray/llm/_internal/batch/stages/__init__.py index 0742784cf592..a45d21fc7670 100644 --- a/python/ray/llm/_internal/batch/stages/__init__.py +++ b/python/ray/llm/_internal/batch/stages/__init__.py @@ -6,6 +6,7 @@ from ray.llm._internal.batch.stages.chat_template_stage import ChatTemplateStage from ray.llm._internal.batch.stages.http_request_stage import HttpRequestStage from ray.llm._internal.batch.stages.prepare_image_stage import PrepareImageStage +from ray.llm._internal.batch.stages.serve_deployment_stage import ServeDeploymentStage from ray.llm._internal.batch.stages.sglang_engine_stage import SGLangEngineStage from ray.llm._internal.batch.stages.tokenize_stage import DetokenizeStage, TokenizeStage from ray.llm._internal.batch.stages.vllm_engine_stage import vLLMEngineStage @@ -18,6 +19,7 @@ "DetokenizeStage", "vLLMEngineStage", "SGLangEngineStage", + "ServeDeploymentStage", "wrap_preprocess", "wrap_postprocess", "PrepareImageStage", diff --git a/python/ray/llm/_internal/batch/stages/serve_deployment_stage.py b/python/ray/llm/_internal/batch/stages/serve_deployment_stage.py new file mode 100644 index 000000000000..04626e734cd7 --- /dev/null +++ b/python/ray/llm/_internal/batch/stages/serve_deployment_stage.py @@ -0,0 +1,156 @@ +"""The stage that runs serve deployment.""" + +import asyncio +import logging +import time +import uuid +from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Type + +from pydantic import BaseModel + +from ray import serve +from ray.llm._internal.batch.stages.base import ( + StatefulStage, + StatefulStageUDF, +) + +logger = logging.getLogger(__name__) + + +class ServeDeploymentStageUDF(StatefulStageUDF): + def __init__( + self, + data_column: str, + expected_input_keys: List[str], + *, + deployment_name: str, + app_name: str, + dtype_mapping: Dict[str, Type[Any]], + ): + """ + Initialize the ServeDeploymentStageUDF. + + Args: + data_column: The data column name. + expected_input_keys: The expected input keys of the stage. + deployment_name: The name of the deployment. + app_name: The name of the deployment app. + dtype_mapping: The mapping of the request class name to the request class. + """ + super().__init__(data_column, expected_input_keys) + self._dtype_mapping = dtype_mapping + + # Using stream=True as LLM serve deployments return async generators. + # TODO (Kourosh): Generalize this to support non-streaming deployments. + self._dh = serve.get_deployment_handle(deployment_name, app_name).options( + stream=True + ) + self.request_id = 0 + + def _prepare_request( + self, row: Dict[str, Any] + ) -> Tuple[Dict[str, Any], Optional[Type[Any]], str]: + """ + Decorate the request with metadata related to the batch. + + Args: + row: The row. + + Returns: + A tuple of (decorated_request, dtype, method_name). dtype is the class of the request object and + can be None if the serve deployment accepts a raw dict. method_name is the name of the method to + invoke on the serve deployment. + """ + method = row.get("method") + dtype_name = row.get("dtype") + + dtype = None + if dtype_name is not None: + if not self._dtype_mapping or dtype_name not in self._dtype_mapping: + raise ValueError( + f"{dtype_name} must be provided in ServeDeploymentProcessorConfig's dtype_mapping." + ) + dtype = self._dtype_mapping[dtype_name] + + request_kwargs = row.pop("request_kwargs") + request = { + "request_id": str(self.request_id), + "idx_in_batch": row[self.IDX_IN_BATCH_COLUMN], + **request_kwargs, + } + self.request_id += 1 + + return request, dtype, method + + async def generate_async( + self, row: Dict[str, Any] + ) -> Tuple[Dict[str, Any], Dict[str, Any], float]: + """ + Run the serve deployment. + + Args: + row: The row to run the serve deployment on. + + Returns: + The response from the serve deployment. + """ + request, dtype, method = self._prepare_request(row) + request_obj = dtype(**request) if dtype else request + + if getattr(self._dh, method) is None: + raise ValueError(f"Method {method} not found in the serve deployment.") + + t = time.perf_counter() + # Directly using anext() requires python3.10 and above + output_data = await getattr(self._dh, method).remote(request_obj).__anext__() + time_taken = time.perf_counter() - t + + # Convert the output data to a dict if it is a Pydantic model. + if isinstance(output_data, BaseModel): + output_data = output_data.model_dump() + + return request, output_data, time_taken + + async def udf(self, batch: List[Dict[str, Any]]) -> AsyncIterator[Dict[str, Any]]: + """ + Run the serve deployment. + + Args: + batch: A list of rows to run the serve deployment on. + + Yields: + Dict[str, Any]: A dictionary containing the response from the serve deployment + along with processing metadata. + """ + batch_uuid = uuid.uuid4() + t = time.perf_counter() + tasks = [asyncio.create_task(self.generate_async(row)) for row in batch] + + for resp in asyncio.as_completed(tasks): + request, output, time_taken = await resp + + yield { + "request_id": request["request_id"], + self.IDX_IN_BATCH_COLUMN: request["idx_in_batch"], + "batch_uuid": batch_uuid.hex, + "time_taken": time_taken, + **output, + } + + batch_time_taken = time.perf_counter() - t + logger.info( + "[LLM Batch - Serve Deployment] Elapsed time for batch %s with size %d: %s", + batch_uuid.hex, + len(batch), + batch_time_taken, + ) + + +class ServeDeploymentStage(StatefulStage): + fn: Type[StatefulStageUDF] = ServeDeploymentStageUDF + + def get_required_input_keys(self) -> Dict[str, str]: + return { + "method": "Name of the method to invoke on the serve deployment.", + "request_kwargs": "The request_kwargs to construct the request to the serve deployment.", + } diff --git a/python/ray/llm/_internal/serve/builders/application_builders.py b/python/ray/llm/_internal/serve/builders/application_builders.py index 769477044e1d..c8de292f5cda 100644 --- a/python/ray/llm/_internal/serve/builders/application_builders.py +++ b/python/ray/llm/_internal/serve/builders/application_builders.py @@ -21,6 +21,7 @@ def build_llm_deployment( *, name_prefix: Optional[str] = None, deployment_kwargs: Optional[dict] = None, + override_serve_options: Optional[dict] = None, ) -> Application: name_prefix = name_prefix or "LLMServer:" deployment_kwargs = deployment_kwargs or {} @@ -29,6 +30,9 @@ def build_llm_deployment( name_prefix=name_prefix, ) + if override_serve_options: + deployment_options.update(override_serve_options) + return LLMDeployment.options(**deployment_options).bind( llm_config=llm_config, **deployment_kwargs ) diff --git a/python/ray/llm/tests/batch/gpu/processor/test_serve_deployment_proc.py b/python/ray/llm/tests/batch/gpu/processor/test_serve_deployment_proc.py new file mode 100644 index 000000000000..74f72b1eb916 --- /dev/null +++ b/python/ray/llm/tests/batch/gpu/processor/test_serve_deployment_proc.py @@ -0,0 +1,236 @@ +import sys +from typing import Any, Dict + +import pytest + +import ray +from ray import serve +from ray.llm._internal.batch.processor import ProcessorBuilder +from ray.llm._internal.batch.processor.serve_deployment_proc import ( + ServeDeploymentProcessorConfig, +) +from ray.serve.llm.openai_api_models import ChatCompletionRequest, CompletionRequest + + +@pytest.mark.parametrize( + "dtype_mapping", [None, {"CompletionRequest": CompletionRequest}] +) +def test_serve_deployment_processor(dtype_mapping): + app_name = "test_serve_deployment_processor_app" + deployment_name = "test_serve_deployment_name" + + config_kwargs = dict( + deployment_name=deployment_name, + app_name=app_name, + batch_size=16, + concurrency=1, + ) + if dtype_mapping is not None: + config_kwargs["dtype_mapping"] = dtype_mapping + config = ServeDeploymentProcessorConfig(**config_kwargs) + + processor = ProcessorBuilder.build(config) + assert processor.list_stage_names() == [ + "ServeDeploymentStage", + ] + + stage = processor.get_stage_by_name("ServeDeploymentStage") + assert stage.fn_constructor_kwargs == { + "deployment_name": deployment_name, + "app_name": app_name, + "dtype_mapping": dtype_mapping, + } + + assert stage.map_batches_kwargs == { + "concurrency": 1, + } + + +def test_simple_serve_deployment(serve_cleanup): + @serve.deployment + class SimpleServeDeployment: + # ServeDeploymentStageUDF expects an async generator. + async def add(self, request: Dict[str, Any]): + yield {"result": request["x"] + 1} + + app_name = "simple_serve_deployment_app" + deployment_name = "SimpleServeDeployment" + + serve.run(SimpleServeDeployment.bind(), name=app_name) + + config = ServeDeploymentProcessorConfig( + deployment_name=deployment_name, + app_name=app_name, + batch_size=16, + concurrency=1, + ) + + processor = ProcessorBuilder.build( + config, + preprocess=lambda row: dict( + method="add", + dtype=None, # Empty dtype since output is already dict format + request_kwargs=dict(x=row["id"]), + ), + postprocess=lambda row: dict( + resp=row["result"], + id=row["id"], + ), + ) + + ds = ray.data.range(60) + ds = ds.map(lambda x: {"id": x["id"]}) + ds = processor(ds) + + outs = ds.take_all() + assert len(outs) == 60 + assert all("resp" in out for out in outs) + assert all(out["resp"] == out["id"] + 1 for out in outs) + + +def test_completion_model(model_opt_125m, create_model_opt_125m_deployment): + deployment_name, app_name = create_model_opt_125m_deployment + config = ServeDeploymentProcessorConfig( + deployment_name=deployment_name, + app_name=app_name, + dtype_mapping={ + "CompletionRequest": CompletionRequest, + }, + batch_size=16, + concurrency=1, + ) + + processor = ProcessorBuilder.build( + config, + preprocess=lambda row: dict( + method="completions", + dtype="CompletionRequest", + request_kwargs=dict( + model=model_opt_125m, + prompt=row["prompt"], + stream=False, + ), + ), + postprocess=lambda row: dict( + resp=row["choices"][0]["text"], + ), + ) + + ds = ray.data.range(60) + ds = ds.map(lambda x: {"prompt": f"Hello {x['id']}"}) + ds = processor(ds) + ds = ds.materialize() + outs = ds.take_all() + assert len(outs) == 60 + assert all("resp" in out for out in outs) + + +def test_multi_turn_completion_model(model_opt_125m, create_model_opt_125m_deployment): + deployment_name, app_name = create_model_opt_125m_deployment + + config1 = ServeDeploymentProcessorConfig( + deployment_name=deployment_name, + app_name=app_name, + dtype_mapping={ + "CompletionRequest": CompletionRequest, + }, + # Use lower batch size to reduce resource usage as there are multiple processors + batch_size=4, + concurrency=1, + ) + + processor1 = ProcessorBuilder.build( + config1, + preprocess=lambda row: dict( + dtype="CompletionRequest", + method="completions", + request_kwargs=dict( + model=model_opt_125m, + prompt=row["prompt"], + stream=False, + ), + ), + postprocess=lambda row: dict( + prompt=row["choices"][0]["text"], + ), + ) + + config2 = ServeDeploymentProcessorConfig( + deployment_name=deployment_name, + app_name=app_name, + dtype_mapping={ + "CompletionRequest": CompletionRequest, + }, + batch_size=4, + concurrency=1, + ) + + processor2 = ProcessorBuilder.build( + config2, + preprocess=lambda row: dict( + dtype="CompletionRequest", + method="completions", + request_kwargs=dict( + model=model_opt_125m, + prompt=row["prompt"], + stream=False, + ), + ), + postprocess=lambda row: dict( + resp=row["choices"][0]["text"], + ), + ) + + ds = ray.data.range(60) + ds = ds.map(lambda x: {"prompt": f"Hello {x['id']}"}) + ds = processor1(ds) + ds = processor2(ds) + + ds = ds.materialize() + outs = ds.take_all() + assert len(outs) == 60 + assert all("resp" in out for out in outs) + + +def test_chat_model(model_opt_125m, create_model_opt_125m_deployment): + deployment_name, app_name = create_model_opt_125m_deployment + config = ServeDeploymentProcessorConfig( + deployment_name=deployment_name, + app_name=app_name, + dtype_mapping={ + "ChatCompletionRequest": ChatCompletionRequest, + }, + batch_size=16, + concurrency=1, + ) + + processor = ProcessorBuilder.build( + config, + preprocess=lambda row: dict( + dtype="ChatCompletionRequest", + method="chat", + request_kwargs=dict( + model=model_opt_125m, + messages=[ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": f"Hello {row['id']}"}, + ], + stream=False, + ), + ), + postprocess=lambda row: dict( + resp=row["choices"][0]["message"]["content"], + ), + ) + + ds = ray.data.range(60) + ds = ds.map(lambda x: {"id": x["id"]}) + ds = processor(ds) + ds = ds.materialize() + outs = ds.take_all() + assert len(outs) == 60 + assert all("resp" in out for out in outs) + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/llm/tests/batch/gpu/stages/test_serve_deployment_stage.py b/python/ray/llm/tests/batch/gpu/stages/test_serve_deployment_stage.py new file mode 100644 index 000000000000..8e07fa739d92 --- /dev/null +++ b/python/ray/llm/tests/batch/gpu/stages/test_serve_deployment_stage.py @@ -0,0 +1,177 @@ +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from ray.llm._internal.batch.stages.serve_deployment_stage import ( + ServeDeploymentStageUDF, +) +from ray.serve.llm.openai_api_models import ChatCompletionRequest, CompletionRequest + + +@pytest.fixture +def mock_serve_deployment_handle(): + """Mock the serve deployment handle and its methods.""" + with patch("ray.serve.get_deployment_handle") as mock_get_handle: + mock_handle = MagicMock() + mock_handle.options.return_value = mock_handle + + # Mock the chat and completions methods + mock_handle.chat = MagicMock() + mock_handle.completions = MagicMock() + + mock_get_handle.return_value = mock_handle + yield mock_handle + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,test_data", + [ + ( + "completions", + [ + { + "method": "completions", + "dtype": "CompletionRequest", + "request_kwargs": {"prompt": "Hello", "temperature": 0.7}, + }, + ], + ), + ( + "chat", + [ + { + "method": "chat", + "dtype": "ChatCompletionRequest", + "request_kwargs": { + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant", + }, + {"role": "user", "content": "Hello 1"}, + ] + }, + }, + ], + ), + ], +) +async def test_serve_deployment_udf_methods( + mock_serve_deployment_handle, method, test_data +): + """Test both completions and chat methods.""" + # Create a mock response that will be returned directly + mock_response = {"test": "response"} + + def mock_remote_call(*args, **kwargs): + async def mock_async_iterator(): + yield mock_response + + return mock_async_iterator() + + getattr(mock_serve_deployment_handle, method).remote.side_effect = mock_remote_call + + udf = ServeDeploymentStageUDF( + data_column="__data", + expected_input_keys=["method", "request_kwargs"], + deployment_name="test_deployment", + app_name="test_app", + dtype_mapping={ + "CompletionRequest": CompletionRequest, + "ChatCompletionRequest": ChatCompletionRequest, + }, + ) + + batch = {"__data": test_data} + + responses = [] + async for response in udf(batch): + responses.append(response) + + assert len(responses) == 1 + assert "__data" in responses[0] + assert len(responses[0]["__data"]) == len(test_data) + + for i, item in enumerate(responses[0]["__data"]): + assert "batch_uuid" in item + assert "time_taken" in item + assert item["request_id"] == str(i) + assert "test" in item # From the mock response + + assert getattr(mock_serve_deployment_handle, method).remote.call_count == len( + test_data + ) + + +@pytest.mark.asyncio +async def test_serve_deployment_invalid_method(mock_serve_deployment_handle): + """Test that invalid method raises error at runtime.""" + # Set up the mock to simulate a method that doesn't exist + mock_serve_deployment_handle.invalid_method = None + + udf = ServeDeploymentStageUDF( + data_column="__data", + expected_input_keys=["method", "request_kwargs"], + deployment_name="test_deployment", + app_name="test_app", + dtype_mapping={ + "CompletionRequest": CompletionRequest, + }, + ) + + batch = { + "__data": [ + { + "method": "invalid_method", + "dtype": "CompletionRequest", + "request_kwargs": {"prompt": "Hello", "temperature": 0.7}, + } + ] + } + + with pytest.raises( + ValueError, match="Method invalid_method not found in the serve deployment." + ): + async for _ in udf(batch): + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "dtype_mapping", [None, {"ChatCompletionRequest": ChatCompletionRequest}] +) +async def test_serve_deployment_missing_dtype( + mock_serve_deployment_handle, dtype_mapping +): + """Test that missing dtype raises error at runtime.""" + + udf = ServeDeploymentStageUDF( + data_column="__data", + expected_input_keys=["method", "request_kwargs"], + deployment_name="test_deployment", + app_name="test_app", + dtype_mapping=dtype_mapping, + ) + + batch = { + "__data": [ + { + "method": "completions", + "dtype": "CompletionRequest", + "request_kwargs": {"prompt": "Hello", "temperature": 0.7}, + } + ] + } + + with pytest.raises( + ValueError, + match="CompletionRequest must be provided in ServeDeploymentProcessorConfig's dtype_mapping.", + ): + async for _ in udf(batch): + pass + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/llm/tests/conftest.py b/python/ray/llm/tests/conftest.py index 65291b778714..778c32131de5 100644 --- a/python/ray/llm/tests/conftest.py +++ b/python/ray/llm/tests/conftest.py @@ -5,6 +5,9 @@ import pytest import requests +from ray import serve +from ray.serve.llm import LLMConfig, ModelLoadingConfig, build_llm_deployment + S3_ARTIFACT_URL = "https://air-example-data.s3.amazonaws.com/" S3_ARTIFACT_LLM_OSSCI_URL = S3_ARTIFACT_URL + "rayllm-ossci/" @@ -167,3 +170,66 @@ def gpu_type(): print("Failed to import torch to get GPU type", flush=True) except ValueError as err: print(f"Failed to get the GPU type: {err}", flush=True) + + +@pytest.fixture +def serve_cleanup(): + yield + serve.shutdown() + + +@pytest.fixture +def create_model_opt_125m_deployment(gpu_type, model_opt_125m, serve_cleanup): + """Create a serve deployment for testing.""" + app_name = "test_serve_deployment_processor_app" + deployment_name = "test_deployment_name" + + chat_template = """ +{% if messages[0]['role'] == 'system' %} + {% set offset = 1 %} +{% else %} + {% set offset = 0 %} +{% endif %} + +{{ bos_token }} +{% for message in messages %} + {% if (message['role'] == 'user') != (loop.index0 % 2 == offset) %} + {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }} + {% endif %} + + {{ '<|im_start|>' + message['role'] + '\n' + message['content'] | trim + '<|im_end|>\n' }} +{% endfor %} + +{% if add_generation_prompt %} + {{ '<|im_start|>assistant\n' }} +{% endif %} + """ + + # Create a vLLM serve deployment + llm_config = LLMConfig( + model_loading_config=ModelLoadingConfig( + model_id=model_opt_125m, + model_source=model_opt_125m, + ), + accelerator_type=gpu_type, + deployment_config=dict( + name="test_deployment_name", # This is not necessarily the final deployment name + autoscaling_config=dict( + min_replicas=1, + max_replicas=1, + ), + ), + engine_kwargs=dict( + enable_prefix_caching=True, + enable_chunked_prefill=True, + max_num_batched_tokens=4096, + # Add chat template for OPT model to enable chat API + chat_template=chat_template, + ), + ) + + llm_app = build_llm_deployment( + llm_config, override_serve_options=dict(name=deployment_name) + ) + serve.run(llm_app, name=app_name) + yield deployment_name, app_name diff --git a/python/ray/serve/llm/__init__.py b/python/ray/serve/llm/__init__.py index 3b04709a0bae..fea4ce70c2f2 100644 --- a/python/ray/serve/llm/__init__.py +++ b/python/ray/serve/llm/__init__.py @@ -93,7 +93,10 @@ class LLMRouter(_LLMRouter): @PublicAPI(stability="alpha") def build_llm_deployment( - llm_config: "LLMConfig", *, name_prefix: Optional[str] = None + llm_config: "LLMConfig", + *, + name_prefix: Optional[str] = None, + override_serve_options: Optional[dict] = None, ) -> "Application": """Helper to build a single vllm deployment from the given llm config. @@ -150,13 +153,18 @@ async def query_model(model_handle): Args: llm_config: The llm config to build vllm deployment. name_prefix: Optional prefix to be used for the deployment name. + override_serve_options: Optional serve options to override the original serve options based on the llm_config. Returns: The configured Ray Serve Application for vllm deployment. """ from ray.llm._internal.serve.builders import build_llm_deployment - return build_llm_deployment(llm_config=llm_config, name_prefix=name_prefix) + return build_llm_deployment( + llm_config=llm_config, + name_prefix=name_prefix, + override_serve_options=override_serve_options, + ) @PublicAPI(stability="alpha") From 92419304797a1b3123f2169cd50d6caad334a59f Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Wed, 27 Aug 2025 17:56:56 -0700 Subject: [PATCH 322/634] Revert "Support cpu tensor transfer with NIXL in GPU Objects" (#56026) Reverts ray-project/ray#55793 fixing lint check --- .../channel/serialization_context.py | 4 +-- .../collective/collective_tensor_transport.py | 16 ++-------- .../collective/nixl_tensor_transport.py | 13 +-------- .../collective/tensor_transport_manager.py | 2 ++ python/ray/experimental/collective/util.py | 17 +---------- .../gpu_object_manager/gpu_object_store.py | 29 ++++++++++--------- python/ray/tests/test_gpu_objects_nixl.py | 24 +++++---------- python/ray/util/collective/types.py | 3 -- 8 files changed, 29 insertions(+), 79 deletions(-) diff --git a/python/ray/experimental/channel/serialization_context.py b/python/ray/experimental/channel/serialization_context.py index d06a2d9e098b..40784b38f409 100644 --- a/python/ray/experimental/channel/serialization_context.py +++ b/python/ray/experimental/channel/serialization_context.py @@ -97,9 +97,7 @@ def serialize_tensor( from ray.experimental.channel import ChannelContext ctx = ChannelContext.get_current() - if self._use_external_transport and ( - ctx._torch_device is None or ctx._torch_device == tensor.device - ): + if self._use_external_transport and tensor.device == ctx.torch_device: # External transport is enabled and we found a tensor that matches # our device. Add the actual tensor to a buffer. The buffer of # tensors should later be popped by the caller and sent via diff --git a/python/ray/experimental/collective/collective_tensor_transport.py b/python/ray/experimental/collective/collective_tensor_transport.py index f6766cbd6f36..7bf39aed7b25 100644 --- a/python/ray/experimental/collective/collective_tensor_transport.py +++ b/python/ray/experimental/collective/collective_tensor_transport.py @@ -40,19 +40,8 @@ def __ray_get_tensor_transport_metadata__( # it could take arbitrarily long and we don't want to trigger a spurious # timeout. gpu_object = gpu_object_store.wait_and_get_object(obj_id) - tensor_meta = [] - device = None - if gpu_object: - device = gpu_object[0].device - for t in gpu_object: - if t.device.type != device.type: - raise ValueError( - "All tensors in one GPU object must be the same device type." - ) - tensor_meta.append((t.shape, t.dtype)) return CollectiveTransportMetadata( - tensor_meta=tensor_meta, - tensor_device=device, + tensor_meta=[(t.shape, t.dtype) for t in gpu_object], ) # Submit a Ray actor task to the source actor to get the tensor metadata. @@ -141,11 +130,10 @@ def recv_multiple_tensors( def send_multiple_tensors( tensors: List["torch.Tensor"], communicator_metadata: CollectiveCommunicatorMetadata, + device: "torch.device", ): import ray.util.collective as collective - device = tensor_transport_metadata.tensor_device - for tensor in tensors: if tensor.device.type != device.type: # TODO(swang): Right now there is no way to catch this error diff --git a/python/ray/experimental/collective/nixl_tensor_transport.py b/python/ray/experimental/collective/nixl_tensor_transport.py index dcedbc11bf41..eb86f0cb9a3d 100644 --- a/python/ray/experimental/collective/nixl_tensor_transport.py +++ b/python/ray/experimental/collective/nixl_tensor_transport.py @@ -45,25 +45,14 @@ def __ray_get_tensor_transport_metadata__( from ray.util.collective.collective import get_group_handle nixl_backend: NixlBackend = get_group_handle(NIXL_GROUP_NAME) - device = None - tensor_meta = [] if gpu_object: serialized_descs, agent_meta = nixl_backend.get_nixl_metadata( gpu_object ) - # We assume all tensors in one GPU object have the same device type. - device = gpu_object[0].device - for t in gpu_object: - if t.device.type != device.type: - raise ValueError( - "All tensors in one GPU object must be the same device type." - ) - tensor_meta.append((t.shape, t.dtype)) else: serialized_descs, agent_meta = None, None return NixlTransportMetadata( - tensor_meta=tensor_meta, - tensor_device=device, + tensor_meta=[(t.shape, t.dtype) for t in gpu_object], nixl_serialized_descs=serialized_descs, nixl_agent_meta=agent_meta, ) diff --git a/python/ray/experimental/collective/tensor_transport_manager.py b/python/ray/experimental/collective/tensor_transport_manager.py index 18d554d2c6d7..9f3896699393 100644 --- a/python/ray/experimental/collective/tensor_transport_manager.py +++ b/python/ray/experimental/collective/tensor_transport_manager.py @@ -143,6 +143,7 @@ def recv_multiple_tensors( def send_multiple_tensors( tensors: List["torch.Tensor"], communicator_metadata: CommunicatorMetadata, + device: "torch.device", ): """ Send multiple tensors to the destination actor. @@ -150,4 +151,5 @@ def send_multiple_tensors( Args: tensors: The tensors to send. communicator_metadata: The communicator metadata for the send/recv operation. + device: The device to send the tensors to. """ diff --git a/python/ray/experimental/collective/util.py b/python/ray/experimental/collective/util.py index 6e9297dba37c..fc6ef64229fb 100644 --- a/python/ray/experimental/collective/util.py +++ b/python/ray/experimental/collective/util.py @@ -1,4 +1,4 @@ -from typing import Tuple, TYPE_CHECKING +from typing import Tuple from contextlib import closing import socket @@ -11,9 +11,6 @@ CollectiveTensorTransport, ) -if TYPE_CHECKING: - import torch - # Singleton instances for tensor transport managers _nixl_tensor_transport_manager = None _collective_tensor_transport_manager = None @@ -44,18 +41,6 @@ def get_tensor_transport_manager( raise ValueError(f"Unsupported tensor transport protocol: {tensor_transport}") -def device_match_transport(device: "torch.device", tensor_transport: Backend) -> bool: - """Check if the device matches the transport.""" - if tensor_transport == Backend.NIXL: - return device.type == "cuda" or device.type == "cpu" - elif tensor_transport == Backend.TORCH_GLOO: - return device.type == "cpu" - elif tensor_transport == Backend.NCCL: - return device.type == "cuda" - else: - raise ValueError(f"Unsupported tensor transport protocol: {tensor_transport}") - - def find_free_port() -> int: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(("", 0)) diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_store.py b/python/ray/experimental/gpu_object_manager/gpu_object_store.py index 01fdbccaf43e..93fcbc20f54d 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_store.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_store.py @@ -11,9 +11,6 @@ TensorTransportMetadata, ) -from ray.experimental.collective import get_tensor_transport_manager -from ray.experimental.collective.util import device_match_transport - try: import torch except ImportError: @@ -28,6 +25,14 @@ TensorTransportEnum.NIXL: Backend.NIXL, } +COLLECTIVE_BACKEND_TO_TORCH_DEVICE = { + Backend.NCCL: torch.device("cuda"), + Backend.TORCH_GLOO: torch.device("cpu"), + # TODO(Qiaolin-Yu): NIXL could also transfer tensors from CPU to CPU. + # More details in https://github.com/ray-project/ray/issues/55587. + Backend.NIXL: torch.device("cuda"), +} + def _tensor_transport_to_collective_backend( tensor_transport: TensorTransportEnum, @@ -56,17 +61,15 @@ def __ray_send__( tensors = gpu_object_store.get_object(obj_id) backend = collective.get_group_handle(communicator_meta.communicator_name).backend() + device = COLLECTIVE_BACKEND_TO_TORCH_DEVICE[backend] + + from ray.experimental.collective import get_tensor_transport_manager tensor_transport_manager = get_tensor_transport_manager(backend) - if tensors and not device_match_transport( - tensor_transport_meta.tensor_device, backend - ): - raise ValueError( - f"Tensor transport backend {backend} does not support tensor transfer on device {tensor_transport_meta.tensor_device}." - ) tensor_transport_manager.send_multiple_tensors( tensors, communicator_meta, + device=device, ) @@ -79,16 +82,14 @@ def __ray_recv__( """Helper function that runs on the dst actor to receive tensors from the src actor.""" from ray._private.worker import global_worker + from ray.experimental.collective import get_tensor_transport_manager + backend = collective.get_group_handle(communicator_meta.communicator_name).backend() - device = tensor_transport_meta.tensor_device + device = COLLECTIVE_BACKEND_TO_TORCH_DEVICE[backend] tensor_meta = tensor_transport_meta.tensor_meta gpu_object_store = global_worker.gpu_object_manager.gpu_object_store - if tensor_meta and not device_match_transport(device, backend): - raise ValueError( - f"Tensor transport backend {backend} does not support tensor transfer on device {device}." - ) tensors = [] for meta in tensor_meta: shape, dtype = meta diff --git a/python/ray/tests/test_gpu_objects_nixl.py b/python/ray/tests/test_gpu_objects_nixl.py index 3a65e8ea7eb9..e46a24358d37 100644 --- a/python/ray/tests/test_gpu_objects_nixl.py +++ b/python/ray/tests/test_gpu_objects_nixl.py @@ -7,11 +7,10 @@ @ray.remote(num_gpus=1, num_cpus=0, enable_tensor_transport=True) class GPUTestActor: @ray.method(tensor_transport="nixl") - def echo(self, data, device): - return data.to(device) + def echo(self, data): + return data.to("cuda") - def sum(self, data, device): - assert data.device.type == device + def sum(self, data): return data.sum().item() @@ -24,21 +23,12 @@ def test_p2p(ray_start_regular): # Create test tensor tensor = torch.tensor([1, 2, 3]) - - tensor1 = torch.tensor([4, 5, 6]) - - # Test GPU to GPU transfer - ref = src_actor.echo.remote(tensor, "cuda") + ref = src_actor.echo.remote(tensor) # Trigger tensor transfer from src to dst actor - result = dst_actor.sum.remote(ref, "cuda") + result = dst_actor.sum.remote(ref) assert tensor.sum().item() == ray.get(result) - # Test CPU to CPU transfer - ref1 = src_actor.echo.remote(tensor1, "cpu") - result1 = dst_actor.sum.remote(ref1, "cpu") - assert tensor1.sum().item() == ray.get(result1) - @pytest.mark.parametrize("ray_start_regular", [{"num_gpus": 1}], indirect=True) def test_intra_gpu_tensor_transfer(ray_start_regular): @@ -47,8 +37,8 @@ def test_intra_gpu_tensor_transfer(ray_start_regular): tensor = torch.tensor([1, 2, 3]) # Intra-actor communication for pure GPU tensors - ref = actor.echo.remote(tensor, "cuda") - result = actor.sum.remote(ref, "cuda") + ref = actor.echo.remote(tensor) + result = actor.sum.remote(ref) assert tensor.sum().item() == ray.get(result) diff --git a/python/ray/util/collective/types.py b/python/ray/util/collective/types.py index e46737c5a033..06a05ae71549 100644 --- a/python/ray/util/collective/types.py +++ b/python/ray/util/collective/types.py @@ -61,12 +61,9 @@ class TensorTransportMetadata: Args: tensor_meta: A list of tuples, each containing the shape and dtype of a tensor. - tensor_device: The device of the tensor. Currently, we require all tensors in the - list have the same device type. """ tensor_meta: List[Tuple["torch.Size", "torch.dtype"]] - tensor_device: Optional["torch.device"] = None @dataclass From bbdce364e36dbcb71a6d85705a8d327a4475b944 Mon Sep 17 00:00:00 2001 From: Alan Guo Date: Wed, 27 Aug 2025 19:42:11 -0700 Subject: [PATCH 323/634] Fix GPU metrics (#56009) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? Bugs introduced in #52102 Two bugs: - proc is a TypedDict so it needs to be fetched via `proc["pid"]` instead of `proc.pid`. - Changing `processes_pid` is backwards-incompatible change that ends up changing the dashboard APIs that power the ray dashboard. Maintain backwards-compatibility Verified fix: Metrics work again: Screenshot 2025-08-27 at 12 22 40 PM Ray Dashboard works again: Screenshot 2025-08-27 at 12 21 51 PM ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Alan Guo --- .../ray/dashboard/modules/node/datacenter.py | 2 +- .../modules/reporter/gpu_providers.py | 6 +- .../modules/reporter/reporter_agent.py | 19 +- .../modules/reporter/tests/test_reporter.py | 326 +++++++++++++++--- 4 files changed, 299 insertions(+), 54 deletions(-) diff --git a/python/ray/dashboard/modules/node/datacenter.py b/python/ray/dashboard/modules/node/datacenter.py index 941702822980..3fa86de65e92 100644 --- a/python/ray/dashboard/modules/node/datacenter.py +++ b/python/ray/dashboard/modules/node/datacenter.py @@ -221,7 +221,7 @@ async def _get_actor_info(actor: Optional[dict]) -> Optional[dict]: break for gpu_stats in node_physical_stats.get("gpus", []): - # gpu_stats.get("processes") can be None, an empty list or a + # gpu_stats.get("processesPids") can be None, an empty list or a # list of dictionaries. for process in gpu_stats.get("processesPids") or []: if process["pid"] == pid: diff --git a/python/ray/dashboard/modules/reporter/gpu_providers.py b/python/ray/dashboard/modules/reporter/gpu_providers.py index b50a417c1c48..431bc9f10beb 100644 --- a/python/ray/dashboard/modules/reporter/gpu_providers.py +++ b/python/ray/dashboard/modules/reporter/gpu_providers.py @@ -230,9 +230,9 @@ def _parse_nvsmi_pmon_output( 1 7175 C 86 26 - - - - ray::TorchGPUWo 2 - - - - - - - - - - Returns a dict mapping GPU index to list of ProcessGPUInfo. + Returns a dict mapping GPU index to dict of pid to ProcessGPUInfo. """ - process_utilizations = defaultdict(list) + process_utilizations = defaultdict(dict) lines = nvsmi_stdout.splitlines() # Get the first line that is started with # table_header = None @@ -275,7 +275,7 @@ def _parse_nvsmi_pmon_output( ), # Convert percentage to MB gpu_utilization=sm, ) - process_utilizations[gpu_id].append(process_info) + process_utilizations[gpu_id][pid] = process_info return process_utilizations def _get_pynvml_gpu_usage(self) -> List[GpuUtilizationInfo]: diff --git a/python/ray/dashboard/modules/reporter/reporter_agent.py b/python/ray/dashboard/modules/reporter/reporter_agent.py index ea842e8eacc7..7a7b1ec83d07 100644 --- a/python/ray/dashboard/modules/reporter/reporter_agent.py +++ b/python/ray/dashboard/modules/reporter/reporter_agent.py @@ -893,7 +893,7 @@ def _get_agent_proc(self) -> psutil.Process: def _generate_worker_key(self, proc: psutil.Process) -> Tuple[int, float]: return (proc.pid, proc.create_time()) - def _get_workers(self, gpus: Optional[List[GpuUtilizationInfo]] = None): + def _get_worker_processes(self): raylet_proc = self._get_raylet_proc() if raylet_proc is None: return [] @@ -910,7 +910,13 @@ def _get_workers(self, gpus: Optional[List[GpuUtilizationInfo]] = None): self._generate_worker_key(proc): proc for proc in raylet_proc.children() } + return workers + def _get_workers(self, gpus: Optional[List[GpuUtilizationInfo]] = None): + workers = self._get_worker_processes() + if not workers: + return [] + else: # We should keep `raylet_proc.children()` in `self` because # when `cpu_percent` is first called, it returns the meaningless 0. # See more: https://github.com/ray-project/ray/issues/29848 @@ -937,7 +943,7 @@ def _get_workers(self, gpus: Optional[List[GpuUtilizationInfo]] = None): processes = gpu.get("processes_pids") if processes: for proc in processes.values(): - gpu_pid_mapping[proc.pid].append(proc) + gpu_pid_mapping[proc["pid"]].append(proc) result = [] for w in self._workers.values(): @@ -1763,6 +1769,15 @@ def _compose_stats_payload( self._metrics_agent.clean_all_dead_worker_metrics() + # Convert processes_pids back to a list of dictionaries to maintain backwards-compatibility + for gpu in stats["gpus"]: + if isinstance(gpu.get("processes_pids"), dict): + gpu["processes_pids"] = list(gpu["processes_pids"].values()) + + # TODO(aguo): Add a pydantic model for this dict to maintain compatibility + # with the Ray Dashboard API and UI code. + + # NOTE: This converts keys to "Google style", (e.g: "processes_pids" -> "processesPids") return jsonify_asdict(stats) async def run(self, server): diff --git a/python/ray/dashboard/modules/reporter/tests/test_reporter.py b/python/ray/dashboard/modules/reporter/tests/test_reporter.py index 9390275b0297..58fd949584a7 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_reporter.py +++ b/python/ray/dashboard/modules/reporter/tests/test_reporter.py @@ -509,58 +509,256 @@ def test_report_per_component_stats_gpu(): """ GPU_MEMORY = 22731 - STATS_TEMPLATE["gpus"] = [ - { - "index": 0, - "uuid": "GPU-36e1567d-37ed-051e-f8ff-df807517b396", - "name": "NVIDIA A10G", - "utilization_gpu": 0, # NOTE: this is a dummy value - "memory_used": 0, - "memory_total": GPU_MEMORY, - "processes_pids": { - 2297322: { - "pid": 2297322, - "gpu_memory_usage": 26, - "gpu_utilization": None, - } + # Prepare the stats data that would be collected by _collect_stats + mock_collected_stats = { + "now": 1614826393.975763, + "hostname": "fake_hostname.local", + "ip": "127.0.0.1", + "cpu": 57.4, + "cpus": (8, 4), + "mem": (17179869184, 5723353088, 66.7, 9234341888), + "shm": 456, + "workers": [ + { + "memory_info": Bunch( + rss=55934976, vms=7026937856, pfaults=15354, pageins=0 + ), + "memory_full_info": Bunch(uss=51428381), + "cpu_percent": 0.0, + "num_fds": 10, + "cmdline": ["ray::IDLE", "", "", "", "", "", "", "", "", "", "", ""], + "create_time": 1614826391.338613, + "pid": 7174, + "cpu_times": Bunch( + user=0.607899328, + system=0.274044032, + children_user=0.0, + children_system=0.0, + ), }, - }, - { - "index": 1, - "uuid": "GPU-36e1567d-37ed-051e-f8ff-df807517b397", - "name": "NVIDIA A10G", - "utilization_gpu": 1, - "memory_used": 1, - "memory_total": GPU_MEMORY, - "processes_pids": { - 2297332: { - "pid": 2297332, - "gpu_memory_usage": 26, - "gpu_utilization": None, - } + { + "memory_info": Bunch( + rss=55934976, vms=7026937856, pfaults=15354, pageins=0 + ), + "memory_full_info": Bunch(uss=51428381), + "cpu_percent": 10.0, + "num_fds": 5, + "cmdline": [ + "ray::TorchGPUWorker.dummy_method", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + ], + "create_time": 1614826391.338613, + "pid": 7175, + "cpu_times": Bunch( + user=0.607899328, + system=0.274044032, + children_user=0.0, + children_system=0.0, + ), }, + ], + "gcs": { + "memory_info": Bunch(rss=18354171, vms=6921486336, pfaults=6203, pageins=2), + "memory_full_info": Bunch(uss=51428384), + "cpu_percent": 5.0, + "num_fds": 14, + "cmdline": ["fake gcs cmdline"], + "create_time": 1614826395.274854, + "pid": 7154, + "cpu_times": Bunch( + user=0.01683138, + system=0.045913716, + children_user=0.0, + children_system=0.0, + ), }, - ] - gpu_worker = STATS_TEMPLATE["workers"][0].copy() - gpu_worker.update( - {"pid": 7175, "cmdline": ["ray::TorchGPUWorker.dummy_method", ""]} - ) + "raylet": { + "memory_info": Bunch(rss=18354176, vms=6921486336, pfaults=6206, pageins=3), + "cpu_percent": 0.0, + "num_fds": 10, + "cmdline": ["fake raylet cmdline"], + "create_time": 1614826390.274854, + "pid": 7153, + "cpu_times": Bunch( + user=0.03683138, + system=0.035913716, + children_user=0.0, + children_system=0.0, + ), + }, + "agent": { + "memory_info": Bunch(rss=18354176, vms=6921486336, pfaults=6206, pageins=3), + "cpu_percent": 0.0, + "num_fds": 10, + "cmdline": ["fake raylet cmdline"], + "create_time": 1614826390.274854, + "pid": 7154, + "cpu_times": Bunch( + user=0.03683138, + system=0.035913716, + children_user=0.0, + children_system=0.0, + ), + }, + "bootTime": 1612934656.0, + "loadAvg": ((4.4521484375, 3.61083984375, 3.5400390625), (0.56, 0.45, 0.44)), + "disk_io": (100, 100, 100, 100), + "disk_io_speed": (100, 100, 100, 100), + "disk": { + "/": Bunch( + total=250790436864, used=11316781056, free=22748921856, percent=33.2 + ), + "/tmp": Bunch( + total=250790436864, used=209532035072, free=22748921856, percent=90.2 + ), + }, + "gpus": [ + { + "index": 0, + "uuid": "GPU-36e1567d-37ed-051e-f8ff-df807517b396", + "name": "NVIDIA A10G", + "utilization_gpu": 0, # NOTE: this is a dummy value + "memory_used": 0, + "memory_total": GPU_MEMORY, + "processes_pids": { + 2297322: { + "pid": 2297322, + "gpu_memory_usage": 26, + "gpu_utilization": None, + } + }, + }, + { + "index": 1, + "uuid": "GPU-36e1567d-37ed-051e-f8ff-df807517b397", + "name": "NVIDIA A10G", + "utilization_gpu": 1, + "memory_used": 1, + "memory_total": GPU_MEMORY, + "processes_pids": { + 2297332: { + "pid": 2297332, + "gpu_memory_usage": 26, + "gpu_utilization": None, + } + }, + }, + ], + "gpu_processes": {}, + "tpus": [], + "network": (13621160960, 11914936320), + "network_speed": (8.435062128545095, 7.378462703142336), + "cmdline": ["fake raylet cmdline"], + } + gpu_metrics_aggregatd = { "component_gpu_utilization": 0, "component_gpu_memory_usage": 0, } - STATS_TEMPLATE["workers"].append(gpu_worker) + def create_mock_agent_proc(): + """Helper function to create a mock agent process.""" + mock_agent_proc = MagicMock() + mock_agent_proc.pid = agent_proc_pid + mock_agent_proc.create_time.return_value = agent_proc_create_time + return mock_agent_proc + + agent_proc_pid = 22334 + agent_proc_create_time = 1614826392.338613 + agent_proc_mock = create_mock_agent_proc() + + def create_mock_worker_processes(): + """Helper function to create mock worker processes for testing.""" + mock_workers = {} + + # Create mock worker processes that match what _get_workers expects + for i, worker_data in enumerate(mock_collected_stats["workers"]): + mock_proc = MagicMock() + mock_proc.status.return_value = psutil.STATUS_RUNNING + mock_proc.as_dict.return_value = { + "pid": worker_data["pid"], + "cmdline": worker_data["cmdline"], + "cpu_percent": worker_data["cpu_percent"], + "memory_info": worker_data["memory_info"], + "memory_full_info": worker_data["memory_full_info"], + "num_fds": worker_data["num_fds"], + "create_time": worker_data["create_time"], + "cpu_times": worker_data["cpu_times"], + } + mock_workers[f"worker_{i}"] = mock_proc + + # Add the agent process to the mock workers + mock_workers[agent._generate_worker_key(agent_proc_mock)] = agent_proc_mock + return mock_workers + + # Mock all the individual methods that _collect_stats calls to return predictable data + mock_patches = { + "_get_network_stats": lambda: (13621160960, 11914936320), + "_get_disk_io_stats": lambda: (100, 100, 100, 100), + "_get_gpu_usage": lambda: mock_collected_stats["gpus"], + "_get_cpu_percent": lambda _: 57.4, + "_get_mem_usage": lambda: (17179869184, 5723353088, 66.7, 9234341888), + "_get_shm_usage": lambda: 456, + "_get_raylet": lambda: mock_collected_stats["raylet"], + "_get_agent": lambda: mock_collected_stats["agent"], + "_get_boot_time": lambda: 1612934656.0, + "_get_load_avg": lambda: ( + (4.4521484375, 3.61083984375, 3.5400390625), + (0.56, 0.45, 0.44), + ), + "_get_disk_usage": lambda: mock_collected_stats["disk"], + "_get_tpu_usage": lambda: [], + "_get_gcs": lambda: mock_collected_stats["gcs"], + "_get_worker_processes": lambda: create_mock_worker_processes(), + "_get_agent_proc": lambda: agent_proc_mock, + } + + with patch.multiple(agent, **mock_patches): + # Call _collect_stats to actually run through the collection process + collected_stats_result = agent._collect_stats() + + # Verify that _collect_stats was called and returned the expected structure + assert "gpus" in collected_stats_result + assert "workers" in collected_stats_result + assert "gcs" in collected_stats_result # Should be present for head node + assert len(collected_stats_result["gpus"]) == 2 + assert len(collected_stats_result["workers"]) == 2 + assert collected_stats_result["cpu"] == 57.4 + assert collected_stats_result["mem"] == ( + 17179869184, + 5723353088, + 66.7, + 9234341888, + ) + assert collected_stats_result["shm"] == 456 + assert collected_stats_result["network"] == (13621160960, 11914936320) + assert collected_stats_result["disk_io"] == (100, 100, 100, 100) + + # Now add the GPU processes data to the collected stats result NVSMI_OUTPUT_TWO_TASK_ON_TWO_GPUS = ( "# gpu pid type sm mem enc dec jpg ofa command \n" "# Idx # C/G % % % % % % name \n" " 0 7175 C 84 26 - - - - ray::TorchGPUWo\n" " 1 7175 C 86 26 - - - - ray::TorchGPUWo\n" ) - STATS_TEMPLATE["gpu_processes"] = NvidiaGpuProvider._parse_nvsmi_pmon_output( - NVSMI_OUTPUT_TWO_TASK_ON_TWO_GPUS, STATS_TEMPLATE["gpus"] + collected_stats_result[ + "gpu_processes" + ] = NvidiaGpuProvider._parse_nvsmi_pmon_output( + NVSMI_OUTPUT_TWO_TASK_ON_TWO_GPUS, collected_stats_result["gpus"] ) - records = agent._to_records(STATS_TEMPLATE, {}) + + # Use the collected stats result for _to_records instead of STATS_TEMPLATE + records = agent._to_records(collected_stats_result, {}) gpu_component_records = defaultdict(list) @@ -587,21 +785,50 @@ def test_report_per_component_stats_gpu(): " 0 7176 C 77 22 - - - - ray::TorchGPUWo\n" " 1 - - - - - - - - - \n" ) - STATS_TEMPLATE["gpu_processes"] = NvidiaGpuProvider._parse_nvsmi_pmon_output( - NVSMI_OUTPUT_TWO_TASK_ON_ONE_GPUS, STATS_TEMPLATE["gpus"] + + # Update the collected stats result for the second test scenario + collected_stats_result[ + "gpu_processes" + ] = NvidiaGpuProvider._parse_nvsmi_pmon_output( + NVSMI_OUTPUT_TWO_TASK_ON_ONE_GPUS, collected_stats_result["gpus"] ) # Move process from GPU 1 to GPU 0 - gpu1_process = STATS_TEMPLATE["gpus"][1]["processes_pids"][2297332] - STATS_TEMPLATE["gpus"][0]["processes_pids"][2297332] = gpu1_process - STATS_TEMPLATE["gpus"][1]["processes_pids"] = {} + gpu1_process = collected_stats_result["gpus"][1]["processes_pids"][2297332] + collected_stats_result["gpus"][0]["processes_pids"][2297332] = gpu1_process + collected_stats_result["gpus"][1]["processes_pids"] = {} - gpu_worker = gpu_worker.copy() - gpu_worker.update( - {"pid": 7176, "cmdline": ["ray::TorchGPUWorker.dummy_method_2", ""]} - ) - STATS_TEMPLATE["workers"].append(gpu_worker) + # Add the second GPU worker to the collected stats result + gpu_worker_2 = { + "memory_info": Bunch(rss=55934976, vms=7026937856, pfaults=15354, pageins=0), + "memory_full_info": Bunch(uss=51428381), + "cpu_percent": 15.0, + "num_fds": 6, + "cmdline": [ + "ray::TorchGPUWorker.dummy_method_2", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + ], + "create_time": 1614826391.338613, + "pid": 7176, + "cpu_times": Bunch( + user=0.607899328, + system=0.274044032, + children_user=0.0, + children_system=0.0, + ), + } + collected_stats_result["workers"].append(gpu_worker_2) - records = agent._to_records(STATS_TEMPLATE, {}) + records = agent._to_records(collected_stats_result, {}) gpu_component_records = defaultdict(list) for record in records: @@ -1017,6 +1244,9 @@ def _get_agent_proc(self): def _generate_worker_key(self, proc): return (proc.pid, proc.create_time()) + def _get_worker_processes(self): + return ReporterAgent._get_worker_processes(self) + obj = ReporterAgentDummy() try: From af5833890194197a8beda5e694f624e2490490f4 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Wed, 27 Aug 2025 20:09:31 -0700 Subject: [PATCH 324/634] [release test] change job title to release test images (#56024) these images are specifically built for running release tests on anyscale. they contain test dependencies that are not part of a normal container image for release only job label text change; no function change. Signed-off-by: Lonnie Liu --- .buildkite/release/build.rayci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/release/build.rayci.yml b/.buildkite/release/build.rayci.yml index 9c7a816c6eae..708b7b622aaf 100644 --- a/.buildkite/release/build.rayci.yml +++ b/.buildkite/release/build.rayci.yml @@ -67,7 +67,7 @@ steps: depends_on: - ray-mlcudabaseextra - - label: ":tapioca: build: anyscale py{{matrix.python}}-{{matrix.platform}} docker" + - label: ":tapioca: build: ray py{{matrix.python}}-{{matrix.platform}} image for release tests" key: anyscalebuild instance_type: release-medium tags: @@ -93,7 +93,7 @@ steps: - cu12.3.2-cudnn9 - cpu - - label: ":tapioca: build: anyscale-llm py{{matrix}} docker" + - label: ":tapioca: build: ray-llm py{{matrix}} image for release tests" key: anyscalellmbuild instance_type: release-medium tags: @@ -108,7 +108,7 @@ steps: matrix: - "3.11" - - label: ":tapioca: build: anyscale-ml py{{matrix}} docker" + - label: ":tapioca: build: ray-ml py{{matrix}} image for release tests" key: anyscalemlbuild instance_type: release-medium tags: From 1fbb951567f90c54b801021455d91c2391f20cca Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Wed, 27 Aug 2025 21:48:41 -0700 Subject: [PATCH 325/634] [core] use prebuilt redis CLI and server binaries for linux (#55975) avoid building from source. the binaries are built with ubuntu 20.04, which is debian bookworm compatible. Signed-off-by: Lonnie Liu --- BUILD.bazel | 24 ++++++------------------ WORKSPACE | 14 ++++++++++++++ bazel/BUILD.bazel | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 47afe3ea98a3..f9a5539a67a4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -87,27 +87,11 @@ config_setting( flag_values = {":jemalloc_flag": "true"}, ) -config_setting( - name = "linux_x86_64", - constraint_values = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], -) - -config_setting( - name = "darwin_aarch64", - constraint_values = [ - "@platforms//os:osx", - "@platforms//cpu:aarch64", - ], -) - alias( name = "uv_file", actual = select({ - "//:linux_x86_64": "@uv_x86_64-linux//:file", - "//:darwin_aarch64": "@uv_aarch64-darwin//:file", + "//bazel:linux_x86_64_config": "@uv_x86_64-linux//:file", + "//bazel:osx_arm64_config": "@uv_aarch64-darwin//:file", "//conditions:default": "@uv_x86_64-linux//:file", }), ) @@ -296,6 +280,8 @@ alias( name = "redis-server", actual = select({ "@platforms//os:windows": "@com_github_tporadowski_redis_bin//:redis-server.exe", + "//bazel:linux_x86_64_config": "@redis_linux_x86_64//:redis-server", + "//bazel:linux_arm64_config": "@redis_linux_arm64//:redis-server", "//conditions:default": "@com_github_antirez_redis//:redis-server", }), ) @@ -304,6 +290,8 @@ alias( name = "redis-cli", actual = select({ "@platforms//os:windows": "@com_github_tporadowski_redis_bin//:redis-cli.exe", + "//bazel:linux_x86_64_config": "@redis_linux_x86_64//:redis-cli", + "//bazel:linux_arm64_config": "@redis_linux_arm64//:redis-cli", "//conditions:default": "@com_github_antirez_redis//:redis-cli", }), ) diff --git a/WORKSPACE b/WORKSPACE index 086d69f189ca..44b7942314b0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -151,6 +151,20 @@ http_archive( ], ) +http_archive( + name = "redis_linux_x86_64", + build_file_content = """exports_files(["redis-server", "redis-cli"])""", + sha256 = "4ae33c10059ed52202a12929d269deea46fac81b8e02e722d30cb22ceb3ed678", + urls = ["https://github.com/ray-project/redis/releases/download/7.2.3/redis-linux-x86_64.tar.gz"], +) + +http_archive( + name = "redis_linux_arm64", + build_file_content = """exports_files(["redis-server", "redis-cli"])""", + sha256 = "2d1085a4f69477e1f44cbddd531e593f0712532b1ade9beab0b221a0cb01f298", + urls = ["https://github.com/ray-project/redis/releases/download/7.2.3/redis-linux-arm64.tar.gz"], +) + load("@com_github_storypku_bazel_iwyu//bazel:dependencies.bzl", "bazel_iwyu_dependencies") bazel_iwyu_dependencies() diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 2e4ce8f51b18..7aa94b909bc0 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -20,3 +20,43 @@ py_library( ], visibility = ["//visibility:public"], ) + +config_setting( + name = "linux_x86_64_config", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) + +config_setting( + name = "linux_arm64_config", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:arm64", + ], +) + +config_setting( + name = "osx_x86_64_config", + constraint_values = [ + "@platforms//os:osx", + "@platforms//cpu:x86_64", + ], +) + +config_setting( + name = "osx_arm64_config", + constraint_values = [ + "@platforms//os:osx", + "@platforms//cpu:arm64", + ], +) + +config_setting( + name = "windows_x86_64_config", + constraint_values = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], +) From bfa342f8a9ba9d632f82564454d4b53c9fc447cb Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Wed, 27 Aug 2025 23:05:25 -0700 Subject: [PATCH 326/634] [core] Turn off -wshadow for tests (#56030) Signed-off-by: dayshah --- bazel/ray.bzl | 9 +- .../tests/normal_task_submitter_test.cc | 55 ++--- .../core_worker/tests/memory_store_test.cc | 4 +- .../core_worker/tests/task_manager_test.cc | 4 +- .../gcs/gcs_client/tests/gcs_client_test.cc | 6 +- .../gcs_actor_manager_export_event_test.cc | 12 +- .../tests/gcs_actor_manager_test.cc | 206 +++++++++--------- .../plasma/tests/object_store_test.cc | 8 +- 8 files changed, 151 insertions(+), 153 deletions(-) diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 2c273128fd74..f2215880338b 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -3,7 +3,7 @@ load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public") load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") -COPTS_WITHOUT_LOG = select({ +COPTS_TESTS = select({ "//:opt": ["-DBAZEL_OPT"], "//conditions:default": [], }) + select({ @@ -16,7 +16,6 @@ COPTS_WITHOUT_LOG = select({ "-Wconversion-null", "-Wno-misleading-indentation", "-Wimplicit-fallthrough", - "-Wshadow", ], }) + select({ "//:clang-cl": [ @@ -26,7 +25,9 @@ COPTS_WITHOUT_LOG = select({ "//conditions:default": [], }) -COPTS = COPTS_WITHOUT_LOG +COPTS = COPTS_TESTS + select({ + "//conditions:default": ["-Wshadow"], +}) PYX_COPTS = select({ "//:msvc-cl": [], @@ -145,7 +146,7 @@ def ray_cc_library(name, strip_include_prefix = "/src", copts = [], visibility = def ray_cc_test(name, linkopts = [], copts = [], **kwargs): cc_test( name = name, - copts = COPTS + copts, + copts = COPTS_TESTS + copts, linkopts = linkopts + ["-pie"], **kwargs ) diff --git a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc index fc58a3bdfb4e..5798da1df518 100644 --- a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc @@ -39,11 +39,10 @@ namespace { class DynamicRateLimiter : public LeaseRequestRateLimiter { public: - explicit DynamicRateLimiter(size_t limit) : limit(limit) {} - size_t GetMaxPendingLeaseRequestsPerSchedulingCategory() override { return limit; } + explicit DynamicRateLimiter(size_t limit) : limit_(limit) {} + size_t GetMaxPendingLeaseRequestsPerSchedulingCategory() override { return limit_; } - public: - size_t limit; + size_t limit_; }; // Wait (and halt the thread) until object_id appears in memory_store. @@ -925,7 +924,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamic) { ASSERT_EQ(raylet_client->reported_backlog_size, tasks.size() - 2); // Increase max concurrency. Should request leases up to the max concurrency. - rateLimiter->limit = concurrency; + rateLimiter->limit_ = concurrency; ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, NodeID::Nil())); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2 + concurrency); ASSERT_EQ(raylet_client->num_workers_requested, 2 + concurrency); @@ -935,7 +934,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamic) { // Decrease max concurrency again. Should not request any more leases even as // previous requests are granted, since we are still over the current // concurrency. - rateLimiter->limit = 1; + rateLimiter->limit_ = 1; for (int i = 0; i < concurrency - 1; i++) { ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", i, NodeID::Nil())); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2 + concurrency); @@ -1010,7 +1009,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamicWithSpillback) ASSERT_EQ(raylet_client->reported_backlog_size, tasks.size() - 2); // Increase max concurrency. - rateLimiter->limit = concurrency; + rateLimiter->limit_ = concurrency; // The outstanding lease request is spilled back to a remote raylet. ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, NodeID::FromRandom())); // We should request one lease request from the spillback raylet and then the @@ -1023,7 +1022,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamicWithSpillback) // Decrease max concurrency again. Should not request any more leases even as // previous requests are granted, since we are still over the current // concurrency. - rateLimiter->limit = 1; + rateLimiter->limit_ = 1; for (int i = 0; i < concurrency - 1; i++) { ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", i, NodeID::Nil())); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, concurrency + 1); @@ -1388,12 +1387,12 @@ TEST_F(NormalTaskSubmitterTest, TestSpillbackRoundTrip) { remote_raylet_clients[addr.port()] = client; return client; }; - auto store = DefaultCoreWorkerMemoryStoreWithThread::CreateShared(); + auto memory_store = DefaultCoreWorkerMemoryStoreWithThread::CreateShared(); auto submitter = CreateNormalTaskSubmitter(std::make_shared(1), WorkerType::WORKER, raylet_client_factory, - store, + memory_store, kLongTimeout); TaskSpecification task = BuildEmptyTaskSpec(); @@ -1533,7 +1532,7 @@ void TestSchedulingKey(const std::shared_ptr store, TEST(NormalTaskSubmitterSchedulingKeyTest, TestSchedulingKeys) { InstrumentedIOContextWithThread io_context("TestSchedulingKeys"); - auto store = std::make_shared(io_context.GetIoService()); + auto memory_store = std::make_shared(io_context.GetIoService()); std::unordered_map resources1({{"a", 1.0}}); std::unordered_map resources2({{"b", 2.0}}); @@ -1544,28 +1543,28 @@ TEST(NormalTaskSubmitterSchedulingKeyTest, TestSchedulingKeys) { // Tasks with different resources should request different worker leases. RAY_LOG(INFO) << "Test different resources"; - TestSchedulingKey(store, + TestSchedulingKey(memory_store, BuildTaskSpec(resources1, descriptor1), BuildTaskSpec(resources1, descriptor1), BuildTaskSpec(resources2, descriptor1)); // Tasks with different functions should request different worker leases. RAY_LOG(INFO) << "Test different functions"; - TestSchedulingKey(store, + TestSchedulingKey(memory_store, BuildTaskSpec(resources1, descriptor1), BuildTaskSpec(resources1, descriptor1), BuildTaskSpec(resources1, descriptor2)); // Tasks with different depths should request different worker leases. RAY_LOG(INFO) << "Test different depths"; - TestSchedulingKey(store, + TestSchedulingKey(memory_store, BuildTaskSpec(resources1, descriptor1, 0), BuildTaskSpec(resources1, descriptor1, 0), BuildTaskSpec(resources1, descriptor1, 1)); // Tasks with different runtime envs do not request different workers. RAY_LOG(INFO) << "Test different runtimes"; - TestSchedulingKey(store, + TestSchedulingKey(memory_store, BuildTaskSpec(resources1, descriptor1, 0, "a"), BuildTaskSpec(resources1, descriptor1, 0, "b"), BuildTaskSpec(resources1, descriptor1, 1, "a")); @@ -1576,16 +1575,16 @@ TEST(NormalTaskSubmitterSchedulingKeyTest, TestSchedulingKeys) { ObjectID plasma2 = ObjectID::FromRandom(); // Ensure the data is already present in the local store for direct call objects. auto data = GenerateRandomObject(); - store->Put(*data, direct1); - store->Put(*data, direct2); + memory_store->Put(*data, direct1); + memory_store->Put(*data, direct2); // Force plasma objects to be promoted. std::string meta = std::to_string(static_cast(rpc::ErrorType::OBJECT_IN_PLASMA)); auto metadata = const_cast(reinterpret_cast(meta.data())); auto meta_buffer = std::make_shared(metadata, meta.size()); auto plasma_data = RayObject(nullptr, meta_buffer, std::vector()); - store->Put(plasma_data, plasma1); - store->Put(plasma_data, plasma2); + memory_store->Put(plasma_data, plasma1); + memory_store->Put(plasma_data, plasma2); TaskSpecification same_deps_1 = BuildTaskSpec(resources1, descriptor1); same_deps_1.GetMutableMessage().add_args()->mutable_object_ref()->set_object_id( @@ -1611,17 +1610,18 @@ TEST(NormalTaskSubmitterSchedulingKeyTest, TestSchedulingKeys) { // Tasks with different plasma dependencies should request different worker leases, // but direct call dependencies shouldn't be considered. RAY_LOG(INFO) << "Test different dependencies"; - TestSchedulingKey(store, same_deps_1, same_deps_2, different_deps); + TestSchedulingKey(memory_store, same_deps_1, same_deps_2, different_deps); } TEST_F(NormalTaskSubmitterTest, TestBacklogReport) { InstrumentedIOContextWithThread store_io_context("TestBacklogReport"); - auto store = std::make_shared(store_io_context.GetIoService()); + auto memory_store = + std::make_shared(store_io_context.GetIoService()); auto submitter = CreateNormalTaskSubmitter(std::make_shared(1), WorkerType::WORKER, /*raylet_client_factory=*/nullptr, - store); + memory_store); TaskSpecification task1 = BuildEmptyTaskSpec(); @@ -1638,8 +1638,8 @@ TEST_F(NormalTaskSubmitterTest, TestBacklogReport) { auto metadata = const_cast(reinterpret_cast(meta.data())); auto meta_buffer = std::make_shared(metadata, meta.size()); auto plasma_data = RayObject(nullptr, meta_buffer, std::vector()); - store->Put(plasma_data, plasma1); - store->Put(plasma_data, plasma2); + memory_store->Put(plasma_data, plasma1); + memory_store->Put(plasma_data, plasma2); // Same SchedulingClass, different SchedulingKey TaskSpecification task2 = BuildTaskSpec(resources1, descriptor1); @@ -1648,7 +1648,8 @@ TEST_F(NormalTaskSubmitterTest, TestBacklogReport) { TaskSpecification task3 = BuildTaskSpec(resources1, descriptor1); task3.GetMutableMessage().add_args()->mutable_object_ref()->set_object_id( plasma2.Binary()); - TestSchedulingKey(store, WithRandomTaskId(task2), WithRandomTaskId(task2), task3); + TestSchedulingKey( + memory_store, WithRandomTaskId(task2), WithRandomTaskId(task2), task3); TaskSpecification task4 = BuildTaskSpec(resources2, descriptor2); @@ -1677,12 +1678,12 @@ TEST_F(NormalTaskSubmitterTest, TestBacklogReport) { } TEST_F(NormalTaskSubmitterTest, TestWorkerLeaseTimeout) { - auto store = DefaultCoreWorkerMemoryStoreWithThread::CreateShared(); + auto memory_store = DefaultCoreWorkerMemoryStoreWithThread::CreateShared(); auto submitter = CreateNormalTaskSubmitter(std::make_shared(1), WorkerType::WORKER, /*raylet_client_factory=*/nullptr, - store, + memory_store, /*lease_timeout_ms=*/5); TaskSpecification task1 = BuildEmptyTaskSpec(); TaskSpecification task2 = BuildEmptyTaskSpec(); diff --git a/src/ray/core_worker/tests/memory_store_test.cc b/src/ray/core_worker/tests/memory_store_test.cc index 257330888afe..65da14ceec22 100644 --- a/src/ray/core_worker/tests/memory_store_test.cc +++ b/src/ray/core_worker/tests/memory_store_test.cc @@ -195,8 +195,8 @@ TEST(TestMemoryStore, TestObjectAllocator) { auto buf = object.GetData(); mock_buffer_manager.AcquireMemory(buf->Size()); auto data_factory = [&mock_buffer_manager, object]() -> std::shared_ptr { - auto buf = object.GetData(); - std::string data(reinterpret_cast(buf->Data()), buf->Size()); + auto inner_buf = object.GetData(); + std::string data(reinterpret_cast(inner_buf->Data()), inner_buf->Size()); return std::make_shared(mock_buffer_manager, data); }; diff --git a/src/ray/core_worker/tests/task_manager_test.cc b/src/ray/core_worker/tests/task_manager_test.cc index b8e4c5722253..e2704d4a3cb7 100644 --- a/src/ray/core_worker/tests/task_manager_test.cc +++ b/src/ray/core_worker/tests/task_manager_test.cc @@ -2560,10 +2560,10 @@ TEST_F(TaskManagerTest, TestObjectRefStreamBackpressure) { bool signal_called = false; ASSERT_TRUE(manager_.HandleReportGeneratorItemReturns( req, - /*execution_signal_callback*/ [&signal_called](Status status, + /*execution_signal_callback*/ [&signal_called](Status callback_status, int64_t num_objects_consumed) { signal_called = true; - ASSERT_TRUE(status.ok()); + ASSERT_TRUE(callback_status.ok()); ASSERT_EQ(num_objects_consumed, 0); })); ASSERT_TRUE(signal_called); diff --git a/src/ray/gcs/gcs_client/tests/gcs_client_test.cc b/src/ray/gcs/gcs_client/tests/gcs_client_test.cc index 7b68e37d1516..ff78ae00d4ab 100644 --- a/src/ray/gcs/gcs_client/tests/gcs_client_test.cc +++ b/src/ray/gcs/gcs_client/tests/gcs_client_test.cc @@ -519,7 +519,7 @@ TEST_P(GcsClientTest, TestActorInfo) { ActorID actor_id = ActorID::FromBinary(actor_table_data->actor_id()); // Subscribe to any update operations of an actor. - auto on_subscribe = [](const ActorID &actor_id, const rpc::ActorTableData &data) {}; + auto on_subscribe = [](const ActorID &, const rpc::ActorTableData &) {}; ASSERT_TRUE(SubscribeActor(actor_id, on_subscribe)); // Register an actor to GCS. @@ -689,7 +689,7 @@ TEST_P(GcsClientTest, TestActorTableResubscribe) { // All the notifications for the following `SubscribeActor` operation. std::vector subscribe_one_notifications; auto actor_subscribe = [&num_subscribe_one_notifications, &subscribe_one_notifications]( - const ActorID &actor_id, const rpc::ActorTableData &data) { + const ActorID &, const rpc::ActorTableData &data) { subscribe_one_notifications.emplace_back(data); ++num_subscribe_one_notifications; RAY_LOG(INFO) << "The number of actor subscription messages received is " @@ -829,7 +829,7 @@ TEST_P(GcsClientTest, TestMultiThreadSubAndUnsub) { auto job_id = JobID::FromInt(1); for (int index = 0; index < size; ++index) { threads[index].reset(new std::thread([this, sub_and_unsub_loop_count, job_id] { - for (int index = 0; index < sub_and_unsub_loop_count; ++index) { + for (int inner_index = 0; inner_index < sub_and_unsub_loop_count; ++inner_index) { auto actor_id = ActorID::Of(job_id, RandomTaskId(), 0); ASSERT_TRUE(SubscribeActor( actor_id, [](const ActorID &id, const rpc::ActorTableData &result) {})); diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc index dfa789f79805..a70c40231aba 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc @@ -53,8 +53,8 @@ class MockActorScheduler : public gcs::GcsActorSchedulerInterface { auto pending_it = std::find_if(actors.begin(), actors.end(), - [actor_id](const std::shared_ptr &actor) { - return actor->GetActorID() == actor_id; + [actor_id](const std::shared_ptr ¤t_actor) { + return current_actor->GetActorID() == actor_id; }); if (pending_it != actors.end()) { actors.erase(pending_it); @@ -234,7 +234,7 @@ class GcsActorManagerTest : public ::testing::Test { io_service_.post( [this, request, &promise]() { auto status = gcs_actor_manager_->RegisterActor( - request, [this, request, &promise](const Status &status) { + request, [this, request, &promise](const Status &) { auto actor_id = ActorID::FromBinary( request.task_spec().actor_creation_task_spec().actor_id()); promise.set_value( @@ -285,9 +285,9 @@ TEST_F(GcsActorManagerTest, TestBasic) { std::vector> finished_actors; Status status = gcs_actor_manager_->CreateActor( create_actor_request, - [&finished_actors](const std::shared_ptr &actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); }); + [&finished_actors](const std::shared_ptr &result_actor, + const rpc::PushTaskReply &, + const Status &) { finished_actors.emplace_back(result_actor); }); RAY_CHECK_OK(status); RAY_CHECK_EQ(gcs_actor_manager_->CountFor(rpc::ActorTableData::PENDING_CREATION, ""), 1); diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc index e639ac58fc4c..891343111c0b 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc @@ -49,8 +49,8 @@ class MockActorScheduler : public gcs::GcsActorSchedulerInterface { auto pending_it = std::find_if(actors.begin(), actors.end(), - [actor_id](const std::shared_ptr &actor) { - return actor->GetActorID() == actor_id; + [actor_id](const std::shared_ptr ¤t_actor) { + return current_actor->GetActorID() == actor_id; }); if (pending_it != actors.end()) { actors.erase(pending_it); @@ -178,7 +178,7 @@ class GcsActorManagerTest : public ::testing::Test { } auto request = Mocker::GenRegisterActorRequest( job_id, max_restarts, detached, name, ray_namespace); - auto status = gcs_actor_manager_->RegisterActor(request, [](const Status &status) {}); + auto status = gcs_actor_manager_->RegisterActor(request, [](const Status &) {}); io_service_.run_one(); io_service_.run_one(); auto actor_id = @@ -244,7 +244,7 @@ TEST_F(GcsActorManagerTest, TestBasic) { create_actor_request, [&finished_actors](const std::shared_ptr &actor, const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); }); + const Status &) { finished_actors.emplace_back(actor); }); RAY_CHECK_OK(status); RAY_CHECK_EQ(gcs_actor_manager_->CountFor(rpc::ActorTableData::PENDING_CREATION, ""), 1); @@ -289,7 +289,7 @@ TEST_F(GcsActorManagerTest, TestDeadCount) { gcs_actor_manager_->CreateActor(create_actor_request, [](const std::shared_ptr &actor, const rpc::PushTaskReply &reply, - const Status &status) {}); + const Status &) {}); RAY_CHECK_OK(status); auto actor = mock_actor_scheduler_->actors.back(); mock_actor_scheduler_->actors.pop_back(); @@ -314,9 +314,11 @@ TEST_F(GcsActorManagerTest, TestSchedulingFailed) { std::vector> finished_actors; RAY_CHECK_OK(gcs_actor_manager_->CreateActor( create_actor_request, - [&finished_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); })); + [&finished_actors](std::shared_ptr result_actor, + const rpc::PushTaskReply &, + const Status &) { + finished_actors.emplace_back(result_actor); + })); ASSERT_EQ(finished_actors.size(), 0); ASSERT_EQ(mock_actor_scheduler_->actors.size(), 1); @@ -341,9 +343,11 @@ TEST_F(GcsActorManagerTest, TestWorkerFailure) { std::vector> finished_actors; RAY_CHECK_OK(gcs_actor_manager_->CreateActor( create_actor_request, - [&finished_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); })); + [&finished_actors](std::shared_ptr result_actor, + const rpc::PushTaskReply &, + const Status &) { + finished_actors.emplace_back(result_actor); + })); ASSERT_EQ(finished_actors.size(), 0); ASSERT_EQ(mock_actor_scheduler_->actors.size(), 1); @@ -388,8 +392,8 @@ TEST_F(GcsActorManagerTest, TestNodeFailure) { Status status = gcs_actor_manager_->CreateActor( create_actor_request, [&finished_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); }); + const rpc::PushTaskReply &, + const Status &) { finished_actors.emplace_back(actor); }); RAY_CHECK_OK(status); ASSERT_EQ(finished_actors.size(), 0); @@ -438,8 +442,8 @@ TEST_F(GcsActorManagerTest, TestActorReconstruction) { Status status = gcs_actor_manager_->CreateActor( create_actor_request, [&finished_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); }); + const rpc::PushTaskReply &, + const Status &) { finished_actors.emplace_back(actor); }); RAY_CHECK_OK(status); ASSERT_EQ(finished_actors.size(), 0); @@ -505,9 +509,11 @@ TEST_F(GcsActorManagerTest, TestActorRestartWhenOwnerDead) { std::vector> finished_actors; RAY_CHECK_OK(gcs_actor_manager_->CreateActor( create_actor_request, - [&finished_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); })); + [&finished_actors](std::shared_ptr result_actor, + const rpc::PushTaskReply &, + const Status &) { + finished_actors.emplace_back(result_actor); + })); ASSERT_EQ(finished_actors.size(), 0); ASSERT_EQ(mock_actor_scheduler_->actors.size(), 1); @@ -555,9 +561,11 @@ TEST_F(GcsActorManagerTest, TestDetachedActorRestartWhenCreatorDead) { std::vector> finished_actors; RAY_CHECK_OK(gcs_actor_manager_->CreateActor( create_actor_request, - [&finished_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); })); + [&finished_actors](std::shared_ptr result_actor, + const rpc::PushTaskReply &, + const Status &) { + finished_actors.emplace_back(result_actor); + })); ASSERT_EQ(finished_actors.size(), 0); ASSERT_EQ(mock_actor_scheduler_->actors.size(), 1); @@ -589,8 +597,7 @@ TEST_F(GcsActorManagerTest, TestActorWithEmptyName) { /*detached=*/true, /*name=*/""); - Status status = - gcs_actor_manager_->RegisterActor(request1, [](const Status &status) {}); + Status status = gcs_actor_manager_->RegisterActor(request1, [](const Status &) {}); io_service_.run_one(); // Ensure successful registration. @@ -604,7 +611,7 @@ TEST_F(GcsActorManagerTest, TestActorWithEmptyName) { /*max_restarts=*/0, /*detached=*/true, /*name=*/""); - status = gcs_actor_manager_->RegisterActor(request2, [](const Status &status) {}); + status = gcs_actor_manager_->RegisterActor(request2, [](const Status &) {}); io_service_.run_one(); // Ensure successful registration. ASSERT_TRUE(status.ok()); @@ -619,8 +626,7 @@ TEST_F(GcsActorManagerTest, TestNamedActors) { /*detached=*/true, /*name=*/"actor1", /*ray_namespace=*/"test_named_actor"); - Status status = - gcs_actor_manager_->RegisterActor(request1, [](const Status &status) {}); + Status status = gcs_actor_manager_->RegisterActor(request1, [](const Status &) {}); io_service_.run_one(); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor1", "test_named_actor").Binary(), @@ -631,7 +637,7 @@ TEST_F(GcsActorManagerTest, TestNamedActors) { /*detached=*/true, /*name=*/"actor2", /*ray_namesapce=*/"test_named_actor"); - status = gcs_actor_manager_->RegisterActor(request2, [](const Status &status) {}); + status = gcs_actor_manager_->RegisterActor(request2, [](const Status &) {}); io_service_.run_one(); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor2", "test_named_actor").Binary(), @@ -647,7 +653,7 @@ TEST_F(GcsActorManagerTest, TestNamedActors) { /*detached=*/true, /*name=*/"actor2", /*ray_namesapce=*/"test_named_actor"); - status = gcs_actor_manager_->RegisterActor(request3, [](const Status &status) {}); + status = gcs_actor_manager_->RegisterActor(request3, [](const Status &) {}); io_service_.run_one(); ASSERT_TRUE(status.IsAlreadyExists()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor2", "test_named_actor").Binary(), @@ -659,7 +665,7 @@ TEST_F(GcsActorManagerTest, TestNamedActors) { /*detached=*/true, /*name=*/"actor2", /*ray_namesapce=*/"test_named_actor"); - status = gcs_actor_manager_->RegisterActor(request4, [](const Status &status) {}); + status = gcs_actor_manager_->RegisterActor(request4, [](const Status &) {}); io_service_.run_one(); ASSERT_TRUE(status.IsAlreadyExists()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor2", "test_named_actor").Binary(), @@ -678,10 +684,9 @@ TEST_F(GcsActorManagerTest, TestNamedActorDeletionWorkerFailure) { request1.mutable_task_spec()->CopyFrom( registered_actor_1->GetCreationTaskSpecification().GetMessage()); - Status status = gcs_actor_manager_->CreateActor(request1, - [](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) {}); + Status status = gcs_actor_manager_->CreateActor( + request1, + [](std::shared_ptr, const rpc::PushTaskReply &, const Status &) {}); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName(actor_name, "test").Binary(), request1.task_spec().actor_creation_task_spec().actor_id()); @@ -716,10 +721,9 @@ TEST_F(GcsActorManagerTest, TestNamedActorDeletionWorkerFailure) { request2.mutable_task_spec()->CopyFrom( registered_actor_2->GetCreationTaskSpecification().GetMessage()); - status = gcs_actor_manager_->CreateActor(request2, - [](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) {}); + status = gcs_actor_manager_->CreateActor( + request2, + [](std::shared_ptr, const rpc::PushTaskReply &, const Status &) {}); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName(actor_name, "test").Binary(), request2.task_spec().actor_creation_task_spec().actor_id()); @@ -736,10 +740,9 @@ TEST_F(GcsActorManagerTest, TestNamedActorDeletionNodeFailure) { request1.mutable_task_spec()->CopyFrom( registered_actor_1->GetCreationTaskSpecification().GetMessage()); - Status status = gcs_actor_manager_->CreateActor(request1, - [](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) {}); + Status status = gcs_actor_manager_->CreateActor( + request1, + [](std::shared_ptr, const rpc::PushTaskReply &, const Status &) {}); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor", "test").Binary(), request1.task_spec().actor_creation_task_spec().actor_id()); @@ -773,10 +776,9 @@ TEST_F(GcsActorManagerTest, TestNamedActorDeletionNodeFailure) { request2.mutable_task_spec()->CopyFrom( registered_actor_2->GetCreationTaskSpecification().GetMessage()); - status = gcs_actor_manager_->CreateActor(request2, - [](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) {}); + status = gcs_actor_manager_->CreateActor( + request2, + [](std::shared_ptr, const rpc::PushTaskReply &, const Status &) {}); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor", "test").Binary(), request2.task_spec().actor_creation_task_spec().actor_id()); @@ -797,7 +799,7 @@ TEST_F(GcsActorManagerTest, TestNamedActorDeletionNotHappendWhenReconstructed) { Status status = gcs_actor_manager_->CreateActor(request1, [](std::shared_ptr actor, const rpc::PushTaskReply &reply, - const Status &status) {}); + const Status &) {}); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor", "test").Binary(), request1.task_spec().actor_creation_task_spec().actor_id()); @@ -826,7 +828,7 @@ TEST_F(GcsActorManagerTest, TestNamedActorDeletionNotHappendWhenReconstructed) { /*max_restarts=*/0, /*detached=*/true, /*name=*/"actor"); - status = gcs_actor_manager_->RegisterActor(request2, [](const Status &status) {}); + status = gcs_actor_manager_->RegisterActor(request2, [](const Status &) {}); io_service_.run_one(); ASSERT_TRUE(status.IsAlreadyExists()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor", "test").Binary(), @@ -843,9 +845,11 @@ TEST_F(GcsActorManagerTest, TestDestroyActorBeforeActorCreationCompletes) { std::vector> finished_actors; RAY_CHECK_OK(gcs_actor_manager_->CreateActor( create_actor_request, - [&finished_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); })); + [&finished_actors](std::shared_ptr result_actor, + const rpc::PushTaskReply &, + const Status &) { + finished_actors.emplace_back(result_actor); + })); ASSERT_EQ(finished_actors.size(), 0); ASSERT_EQ(mock_actor_scheduler_->actors.size(), 1); @@ -878,9 +882,11 @@ TEST_F(GcsActorManagerTest, TestRaceConditionCancelLease) { std::vector> finished_actors; RAY_CHECK_OK(gcs_actor_manager_->CreateActor( create_actor_request, - [&finished_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); })); + [&finished_actors](std::shared_ptr result_actor, + const rpc::PushTaskReply &, + const Status &) { + finished_actors.emplace_back(result_actor); + })); ASSERT_EQ(finished_actors.size(), 0); ASSERT_EQ(mock_actor_scheduler_->actors.size(), 1); @@ -922,10 +928,10 @@ TEST_F(GcsActorManagerTest, TestRegisterActor) { registered_actor->GetCreationTaskSpecification().GetMessage()); RAY_CHECK_OK(gcs_actor_manager_->CreateActor( request, - [&finished_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { - finished_actors.emplace_back(std::move(actor)); + [&finished_actors](std::shared_ptr result_actor, + const rpc::PushTaskReply &, + const Status &) { + finished_actors.emplace_back(result_actor); })); // Make sure the actor is scheduling. ASSERT_EQ(mock_actor_scheduler_->actors.size(), 1); @@ -1048,9 +1054,11 @@ TEST_F(GcsActorManagerTest, TestOwnerAndChildDiedAtTheSameTimeRaceCondition) { std::vector> finished_actors; RAY_CHECK_OK(gcs_actor_manager_->CreateActor( create_actor_request, - [&finished_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); })); + [&finished_actors](std::shared_ptr result_actor, + const rpc::PushTaskReply &, + const Status &) { + finished_actors.emplace_back(result_actor); + })); auto actor = mock_actor_scheduler_->actors.back(); mock_actor_scheduler_->actors.pop_back(); @@ -1084,8 +1092,7 @@ TEST_F(GcsActorManagerTest, TestRayNamespace) { /*max_restarts=*/0, /*detached=*/true, /*name=*/"actor"); - Status status = - gcs_actor_manager_->RegisterActor(request1, [](const Status &status) {}); + Status status = gcs_actor_manager_->RegisterActor(request1, [](const Status &) {}); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor", "test").Binary(), request1.task_spec().actor_creation_task_spec().actor_id()); @@ -1098,7 +1105,7 @@ TEST_F(GcsActorManagerTest, TestRayNamespace) { second_namespace); // Create a second actor of the same name. Its job id belongs to a different // namespace though. - status = gcs_actor_manager_->RegisterActor(request2, [](const Status &status) {}); + status = gcs_actor_manager_->RegisterActor(request2, [](const Status &) {}); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor", second_namespace).Binary(), request2.task_spec().actor_creation_task_spec().actor_id()); @@ -1112,7 +1119,7 @@ TEST_F(GcsActorManagerTest, TestRayNamespace) { /*detached=*/true, /*name=*/"actor", /*ray_namespace=*/"test"); - status = gcs_actor_manager_->RegisterActor(request3, [](const Status &status) {}); + status = gcs_actor_manager_->RegisterActor(request3, [](const Status &) {}); ASSERT_TRUE(status.IsAlreadyExists()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor", "test").Binary(), request1.task_spec().actor_creation_task_spec().actor_id()); @@ -1128,8 +1135,7 @@ TEST_F(GcsActorManagerTest, TestReuseActorNameInNamespace) { Mocker::GenRegisterActorRequest(job_id_1, 0, true, actor_name, ray_namespace); auto actor_id_1 = ActorID::FromBinary(request_1.task_spec().actor_creation_task_spec().actor_id()); - Status status = - gcs_actor_manager_->RegisterActor(request_1, [](const Status &status) {}); + Status status = gcs_actor_manager_->RegisterActor(request_1, [](const Status &) {}); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName(actor_name, ray_namespace).Binary(), actor_id_1.Binary()); @@ -1148,7 +1154,7 @@ TEST_F(GcsActorManagerTest, TestReuseActorNameInNamespace) { Mocker::GenRegisterActorRequest(job_id_2, 0, true, actor_name, ray_namespace); auto actor_id_2 = ActorID::FromBinary(request_2.task_spec().actor_creation_task_spec().actor_id()); - status = gcs_actor_manager_->RegisterActor(request_2, [](const Status &status) {}); + status = gcs_actor_manager_->RegisterActor(request_2, [](const Status &) {}); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName(actor_name, ray_namespace).Binary(), actor_id_2.Binary()); @@ -1166,9 +1172,9 @@ TEST_F(GcsActorManagerTest, TestGetAllActorInfoFilters) { std::vector> finished_actors; Status create_status = gcs_actor_manager_->CreateActor( create_actor_request, - [&finished_actors](const std::shared_ptr &actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); }); + [&finished_actors](const std::shared_ptr &result_actor, + const rpc::PushTaskReply &, + const Status &) { finished_actors.emplace_back(result_actor); }); ASSERT_TRUE(create_status.ok()); auto actor = mock_actor_scheduler_->actors.back(); @@ -1194,8 +1200,7 @@ TEST_F(GcsActorManagerTest, TestGetAllActorInfoFilters) { io_service_.run_one(); } - auto callback = - [](Status status, std::function success, std::function failure) {}; + auto callback = [](Status, std::function, std::function) {}; // Filter with actor id { rpc::GetAllActorInfoRequest request; @@ -1266,8 +1271,7 @@ TEST_F(GcsActorManagerTest, TestGetAllActorInfoLimit) { auto request1 = Mocker::GenRegisterActorRequest(job_id_1, /*max_restarts=*/0, /*detached=*/false); - Status status = - gcs_actor_manager_->RegisterActor(request1, [](const Status &status) {}); + Status status = gcs_actor_manager_->RegisterActor(request1, [](const Status &) {}); ASSERT_TRUE(status.ok()); io_service_.run_one(); } @@ -1276,9 +1280,7 @@ TEST_F(GcsActorManagerTest, TestGetAllActorInfoLimit) { rpc::GetAllActorInfoRequest request; auto &reply = *google::protobuf::Arena::CreateMessage(&arena); - auto callback = [](Status status, - std::function success, - std::function failure) {}; + auto callback = [](Status, std::function, std::function) {}; gcs_actor_manager_->HandleGetAllActorInfo(request, &reply, callback); ASSERT_EQ(reply.actor_table_data().size(), 3); @@ -1301,9 +1303,9 @@ TEST_F(GcsActorManagerTest, TestKillActorWhenActorIsCreating) { std::vector> finished_actors; Status status = gcs_actor_manager_->CreateActor( create_actor_request, - [&finished_actors](const std::shared_ptr &actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); }); + [&finished_actors](const std::shared_ptr &result_actor, + const rpc::PushTaskReply &, + const Status &) { finished_actors.emplace_back(result_actor); }); RAY_CHECK_OK(status); ASSERT_EQ(finished_actors.size(), 0); @@ -1329,7 +1331,7 @@ TEST_F(GcsActorManagerTest, TestKillActorWhenActorIsCreating) { request, &reply, /*send_reply_callback*/ - [](Status status, std::function success, std::function failure) {}); + [](Status, std::function, std::function) {}); io_service_.run_one(); // Make sure the `KillActor` rpc is send. @@ -1350,9 +1352,9 @@ TEST_F(GcsActorManagerTest, TestRestartActorForLineageReconstruction) { std::vector> created_actors; RAY_CHECK_OK(gcs_actor_manager_->CreateActor( create_actor_request, - [&created_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { created_actors.emplace_back(actor); })); + [&created_actors](std::shared_ptr result_actor, + const rpc::PushTaskReply &, + const Status &) { created_actors.emplace_back(result_actor); })); ASSERT_EQ(created_actors.size(), 0); ASSERT_EQ(mock_actor_scheduler_->actors.size(), 1); @@ -1431,9 +1433,9 @@ TEST_F(GcsActorManagerTest, TestRestartPermanentlyDeadActorForLineageReconstruct std::vector> created_actors; RAY_CHECK_OK(gcs_actor_manager_->CreateActor( create_actor_request, - [&created_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { created_actors.emplace_back(actor); })); + [&created_actors](std::shared_ptr result_actor, + const rpc::PushTaskReply &, + const Status &) { created_actors.emplace_back(result_actor); })); ASSERT_EQ(created_actors.size(), 0); ASSERT_EQ(mock_actor_scheduler_->actors.size(), 1); @@ -1487,9 +1489,9 @@ TEST_F(GcsActorManagerTest, TestIdempotencyOfRestartActorForLineageReconstructio std::vector> created_actors; RAY_CHECK_OK(gcs_actor_manager_->CreateActor( create_actor_request, - [&created_actors](std::shared_ptr actor, - const rpc::PushTaskReply &reply, - const Status &status) { created_actors.emplace_back(actor); })); + [&created_actors](std::shared_ptr result_actor, + const rpc::PushTaskReply &, + const Status &) { created_actors.emplace_back(result_actor); })); ASSERT_EQ(created_actors.size(), 0); ASSERT_EQ(mock_actor_scheduler_->actors.size(), 1); @@ -1520,17 +1522,11 @@ TEST_F(GcsActorManagerTest, TestIdempotencyOfRestartActorForLineageReconstructio rpc::RestartActorForLineageReconstructionReply reply2; gcs_actor_manager_->HandleRestartActorForLineageReconstruction( - request, - &reply1, - [&reply1]( - Status status, std::function success, std::function failure) { + request, &reply1, [&reply1](Status, std::function, std::function) { ASSERT_EQ(reply1.status().code(), static_cast(StatusCode::OK)); }); gcs_actor_manager_->HandleRestartActorForLineageReconstruction( - request, - &reply2, - [&reply2]( - Status status, std::function success, std::function failure) { + request, &reply2, [&reply2](Status, std::function, std::function) { ASSERT_EQ(reply2.status().code(), static_cast(StatusCode::OK)); }); io_service_.run_one(); @@ -1580,9 +1576,9 @@ TEST_F(GcsActorManagerTest, TestDestroyActorWhenActorIsCreating) { std::vector> finished_actors; Status status = gcs_actor_manager_->CreateActor( create_actor_request, - [&finished_actors](const std::shared_ptr &actor, - const rpc::PushTaskReply &reply, - const Status &status) { finished_actors.emplace_back(actor); }); + [&finished_actors](const std::shared_ptr &result_actor, + const rpc::PushTaskReply &, + const Status &) { finished_actors.emplace_back(result_actor); }); RAY_CHECK_OK(status); ASSERT_EQ(finished_actors.size(), 0); @@ -1608,7 +1604,7 @@ TEST_F(GcsActorManagerTest, TestDestroyActorWhenActorIsCreating) { request, &reply, /*send_reply_callback*/ - [](Status status, std::function success, std::function failure) {}); + [](Status, std::function, std::function) {}); io_service_.run_one(); io_service_.run_one(); @@ -1659,9 +1655,9 @@ TEST_F(GcsActorManagerTest, TestRestartPreemptedActor) { Status status = gcs_actor_manager_->CreateActor(create_actor_request, - [](const std::shared_ptr &actor, - const rpc::PushTaskReply &reply, - const Status &status) {}); + [](const std::shared_ptr &, + const rpc::PushTaskReply &, + const Status &) {}); RAY_CHECK_OK(status); ASSERT_EQ(mock_actor_scheduler_->actors.size(), 1); diff --git a/src/ray/object_manager/plasma/tests/object_store_test.cc b/src/ray/object_manager/plasma/tests/object_store_test.cc index 67c4f3e44039..20d4267d9bd3 100644 --- a/src/ray/object_manager/plasma/tests/object_store_test.cc +++ b/src/ray/object_manager/plasma/tests/object_store_test.cc @@ -129,8 +129,8 @@ TEST(ObjectStoreTest, PassThroughTest) { EXPECT_EQ(nullptr, store.SealObject(kId2)); // delete sealed - EXPECT_CALL(allocator, Free(_)).Times(1).WillOnce(Invoke([&](auto &&allocation) { - EXPECT_EQ(alloc_str, Serialize(allocation)); + EXPECT_CALL(allocator, Free(_)).Times(1).WillOnce(Invoke([&](auto &&allocation_arg) { + EXPECT_EQ(alloc_str, Serialize(allocation_arg)); })); EXPECT_TRUE(store.DeleteObject(kId1)); @@ -175,8 +175,8 @@ TEST(ObjectStoreTest, PassThroughTest) { EXPECT_TRUE(entry->allocation_.fallback_allocated_); // delete unsealed - EXPECT_CALL(allocator, Free(_)).Times(1).WillOnce(Invoke([&](auto &&allocation) { - EXPECT_EQ(alloc_str, Serialize(allocation)); + EXPECT_CALL(allocator, Free(_)).Times(1).WillOnce(Invoke([&](auto &&allocation_arg) { + EXPECT_EQ(alloc_str, Serialize(allocation_arg)); })); EXPECT_TRUE(store.DeleteObject(kId2)); From 235b43fd4ab81aab92010261d775dd48983ae983 Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Thu, 28 Aug 2025 12:07:05 +0530 Subject: [PATCH 327/634] [core] Remove perf test in `shutdown_coordinator_test` (#56033) Signed-off-by: Sagar Sumit --- .../tests/shutdown_coordinator_test.cc | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/src/ray/core_worker/tests/shutdown_coordinator_test.cc b/src/ray/core_worker/tests/shutdown_coordinator_test.cc index d20cda359eeb..f01b80dba9d7 100644 --- a/src/ray/core_worker/tests/shutdown_coordinator_test.cc +++ b/src/ray/core_worker/tests/shutdown_coordinator_test.cc @@ -280,28 +280,6 @@ TEST_F(ShutdownCoordinatorTest, Worker_HandleExit_OnIdleTimeout) { ShutdownReason::kIdleTimeout)); } -TEST_F(ShutdownCoordinatorTest, ShouldEarlyExit_Performance_IsFast) { - auto coordinator = CreateCoordinator(); - auto start = std::chrono::high_resolution_clock::now(); - constexpr int iterations = 1000000; - volatile bool result = false; - - for (int i = 0; i < iterations; ++i) { - result = coordinator->ShouldEarlyExit(); - } - - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - - // Should be very fast (less than 100ns per call on modern hardware) - double ns_per_call = static_cast(duration.count()) / iterations; - EXPECT_LT(ns_per_call, 100.0) - << "ShouldEarlyExit too slow: " << ns_per_call << "ns per call"; - - // Prevent unused variable warning - (void)result; -} - TEST_F(ShutdownCoordinatorTest, StringRepresentations_StateAndReason_AreReadable) { auto coordinator = CreateCoordinator(); From f7ee0fcc5404b5eef74a824be9c572fe8f5af482 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Thu, 28 Aug 2025 08:35:34 -0500 Subject: [PATCH 328/634] [core] Split `gcs_placement_group_scheduler` out from `gcs_server_lib` (#55955) Involved splitting `GcsPlacementGroup` into its own file to break circular dependency. Also eliminated `gcs_server_test_util.h` by: 1. Updating the global fakes with the implementation that was defined in the file. 2. Moving other utilities into the single file that depended on them. --------- Signed-off-by: Edward Oakes --- src/fakes/ray/rpc/raylet/BUILD.bazel | 9 + src/fakes/ray/rpc/raylet/raylet_client.h | 189 +++++++- src/fakes/ray/rpc/worker/BUILD.bazel | 10 + src/fakes/ray/rpc/worker/core_worker_client.h | 58 +++ .../task_submission/tests/BUILD.bazel | 1 + src/ray/core_worker/tests/BUILD.bazel | 3 + src/ray/gcs/gcs_server/BUILD.bazel | 41 +- src/ray/gcs/gcs_server/gcs_placement_group.cc | 151 +++++++ src/ray/gcs/gcs_server/gcs_placement_group.h | 212 +++++++++ .../gcs/gcs_server/gcs_placement_group_mgr.cc | 126 ------ .../gcs/gcs_server/gcs_placement_group_mgr.h | 178 +------- .../gcs_placement_group_scheduler.cc | 2 - .../gcs_placement_group_scheduler.h | 8 +- src/ray/gcs/gcs_server/tests/BUILD.bazel | 45 +- .../gcs_actor_manager_export_event_test.cc | 10 +- .../tests/gcs_actor_manager_test.cc | 18 +- .../tests/gcs_actor_scheduler_mock_test.cc | 1 + .../tests/gcs_actor_scheduler_test.cc | 80 +++- .../gcs_autoscaler_state_manager_test.cc | 6 +- .../tests/gcs_placement_group_mgr_test.cc | 11 +- .../gcs_placement_group_scheduler_test.cc | 409 +++++++++--------- .../gcs_server/tests/gcs_server_test_util.h | 386 ----------------- src/ray/raylet/tests/BUILD.bazel | 1 + src/ray/rpc/node_manager/tests/BUILD.bazel | 2 +- 24 files changed, 1002 insertions(+), 955 deletions(-) create mode 100644 src/fakes/ray/rpc/raylet/BUILD.bazel create mode 100644 src/fakes/ray/rpc/worker/BUILD.bazel create mode 100644 src/fakes/ray/rpc/worker/core_worker_client.h create mode 100644 src/ray/gcs/gcs_server/gcs_placement_group.cc create mode 100644 src/ray/gcs/gcs_server/gcs_placement_group.h delete mode 100644 src/ray/gcs/gcs_server/tests/gcs_server_test_util.h diff --git a/src/fakes/ray/rpc/raylet/BUILD.bazel b/src/fakes/ray/rpc/raylet/BUILD.bazel new file mode 100644 index 000000000000..6e2730394697 --- /dev/null +++ b/src/fakes/ray/rpc/raylet/BUILD.bazel @@ -0,0 +1,9 @@ +load("//bazel:ray.bzl", "ray_cc_library") + +ray_cc_library( + name = "fake_raylet_client", + hdrs = ["raylet_client.h"], + deps = [ + "//src/ray/raylet_client:raylet_client_lib", + ], +) diff --git a/src/fakes/ray/rpc/raylet/raylet_client.h b/src/fakes/ray/rpc/raylet/raylet_client.h index 3c2bcb70a8cd..b18394c56695 100644 --- a/src/fakes/ray/rpc/raylet/raylet_client.h +++ b/src/fakes/ray/rpc/raylet/raylet_client.h @@ -31,13 +31,21 @@ class FakeRayletClient : public RayletClientInterface { bool grant_or_reject, const ray::rpc::ClientCallback &callback, const int64_t backlog_size = -1, - const bool is_selected_based_on_locality = false) override {} + const bool is_selected_based_on_locality = false) override { + num_workers_requested += 1; + callbacks.push_back(callback); + } ray::Status ReturnWorker(int worker_port, const WorkerID &worker_id, bool disconnect_worker, const std::string &disconnect_worker_error_detail, bool worker_exiting) override { + if (disconnect_worker) { + num_workers_disconnected++; + } else { + num_workers_returned++; + } return Status::OK(); } @@ -48,30 +56,166 @@ class FakeRayletClient : public RayletClientInterface { void ReleaseUnusedActorWorkers( const std::vector &workers_in_use, const rpc::ClientCallback &callback) override { + num_release_unused_workers += 1; + release_callbacks.push_back(callback); } void CancelWorkerLease( const TaskID &task_id, - const rpc::ClientCallback &callback) override {} + const rpc::ClientCallback &callback) override { + num_leases_canceled += 1; + cancel_callbacks.push_back(callback); + } + + bool GrantWorkerLease() { + return GrantWorkerLease("", 0, WorkerID::FromRandom(), node_id_, NodeID::Nil()); + } + + bool GrantWorkerLease(const std::string &address, + int port, + const WorkerID &worker_id, + const NodeID &node_id, + const NodeID &retry_at_node_id, + Status status = Status::OK(), + bool rejected = false) { + rpc::RequestWorkerLeaseReply reply; + if (!retry_at_node_id.IsNil()) { + reply.mutable_retry_at_raylet_address()->set_ip_address(address); + reply.mutable_retry_at_raylet_address()->set_port(port); + reply.mutable_retry_at_raylet_address()->set_node_id(retry_at_node_id.Binary()); + } else { + reply.mutable_worker_address()->set_ip_address(address); + reply.mutable_worker_address()->set_port(port); + reply.mutable_worker_address()->set_node_id(node_id.Binary()); + reply.mutable_worker_address()->set_worker_id(worker_id.Binary()); + } + if (rejected) { + reply.set_rejected(true); + auto resources_data = reply.mutable_resources_data(); + resources_data->set_node_id(node_id.Binary()); + resources_data->set_resources_normal_task_changed(true); + auto &normal_task_map = *(resources_data->mutable_resources_normal_task()); + normal_task_map[kMemory_ResourceLabel] = + static_cast(std::numeric_limits::max()); + resources_data->set_resources_normal_task_timestamp(absl::GetCurrentTimeNanos()); + } + + if (callbacks.size() == 0) { + return false; + } else { + auto callback = callbacks.front(); + callback(status, std::move(reply)); + callbacks.pop_front(); + return true; + } + } + + bool ReplyCancelWorkerLease(bool success = true) { + rpc::CancelWorkerLeaseReply reply; + reply.set_success(success); + if (cancel_callbacks.size() == 0) { + return false; + } else { + auto callback = cancel_callbacks.front(); + callback(Status::OK(), std::move(reply)); + cancel_callbacks.pop_front(); + return true; + } + } + + bool ReplyReleaseUnusedActorWorkers() { + rpc::ReleaseUnusedActorWorkersReply reply; + if (release_callbacks.size() == 0) { + return false; + } else { + auto callback = release_callbacks.front(); + callback(Status::OK(), std::move(reply)); + release_callbacks.pop_front(); + return true; + } + } + + bool ReplyDrainRaylet() { + if (drain_raylet_callbacks.size() == 0) { + return false; + } else { + rpc::DrainRayletReply reply; + reply.set_is_accepted(true); + auto callback = drain_raylet_callbacks.front(); + callback(Status::OK(), std::move(reply)); + drain_raylet_callbacks.pop_front(); + return true; + } + } void PrepareBundleResources( const std::vector> &bundle_specs, const ray::rpc::ClientCallback &callback) - override {} + override { + num_lease_requested += 1; + lease_callbacks.push_back(callback); + } void CommitBundleResources( const std::vector> &bundle_specs, const ray::rpc::ClientCallback &callback) - override {} + override { + num_commit_requested += 1; + commit_callbacks.push_back(callback); + } void CancelResourceReserve( const BundleSpecification &bundle_spec, const ray::rpc::ClientCallback &callback) - override {} + override { + num_return_requested += 1; + return_callbacks.push_back(callback); + } void ReleaseUnusedBundles( const std::vector &bundles_in_use, - const rpc::ClientCallback &callback) override {} + const rpc::ClientCallback &callback) override { + ++num_release_unused_bundles_requested; + } + + bool GrantPrepareBundleResources(bool success = true, + const Status &status = Status::OK()) { + rpc::PrepareBundleResourcesReply reply; + reply.set_success(success); + if (lease_callbacks.size() == 0) { + return false; + } else { + auto callback = lease_callbacks.front(); + callback(status, std::move(reply)); + lease_callbacks.pop_front(); + return true; + } + } + + bool GrantCommitBundleResources(const Status &status = Status::OK()) { + rpc::CommitBundleResourcesReply reply; + if (commit_callbacks.size() == 0) { + return false; + } else { + auto callback = commit_callbacks.front(); + callback(status, std::move(reply)); + commit_callbacks.pop_front(); + return true; + } + } + + bool GrantCancelResourceReserve(bool success = true) { + Status status = Status::OK(); + rpc::CancelResourceReserveReply reply; + if (return_callbacks.size() == 0) { + return false; + } else { + auto callback = return_callbacks.front(); + callback(status, std::move(reply)); + return_callbacks.pop_front(); + return true; + } + } void ReportWorkerBacklog( const WorkerID &worker_id, @@ -96,7 +240,11 @@ class FakeRayletClient : public RayletClientInterface { void GetTaskFailureCause( const TaskID &task_id, - const rpc::ClientCallback &callback) override {} + const rpc::ClientCallback &callback) override { + ray::rpc::GetTaskFailureCauseReply reply; + callback(Status::OK(), std::move(reply)); + num_get_task_failure_causes += 1; + } void GetSystemConfig( const rpc::ClientCallback &callback) override {} @@ -112,7 +260,11 @@ class FakeRayletClient : public RayletClientInterface { void DrainRaylet(const rpc::autoscaler::DrainNodeReason &reason, const std::string &reason_message, int64_t deadline_timestamp_ms, - const rpc::ClientCallback &callback) override {} + const rpc::ClientCallback &callback) override { + rpc::DrainRayletReply reply; + reply.set_is_accepted(true); + drain_raylet_callbacks.push_back(callback); + } void CancelTasksWithResourceShapes( const std::vector> &resource_shapes, @@ -132,6 +284,27 @@ class FakeRayletClient : public RayletClientInterface { void GlobalGC(const rpc::ClientCallback &callback) override {} int64_t GetPinsInFlight() const override { return 0; } + + int num_workers_requested = 0; + int num_workers_returned = 0; + int num_workers_disconnected = 0; + int num_leases_canceled = 0; + int num_release_unused_workers = 0; + int num_get_task_failure_causes = 0; + NodeID node_id_ = NodeID::FromRandom(); + std::list> drain_raylet_callbacks = {}; + std::list> callbacks = {}; + std::list> cancel_callbacks = {}; + std::list> release_callbacks = + {}; + int num_lease_requested = 0; + int num_return_requested = 0; + int num_commit_requested = 0; + + int num_release_unused_bundles_requested = 0; + std::list> lease_callbacks = {}; + std::list> commit_callbacks = {}; + std::list> return_callbacks = {}; }; } // namespace ray diff --git a/src/fakes/ray/rpc/worker/BUILD.bazel b/src/fakes/ray/rpc/worker/BUILD.bazel new file mode 100644 index 000000000000..b5bd45196c99 --- /dev/null +++ b/src/fakes/ray/rpc/worker/BUILD.bazel @@ -0,0 +1,10 @@ +load("//bazel:ray.bzl", "ray_cc_library") + +ray_cc_library( + name = "fake_core_worker_client", + hdrs = ["core_worker_client.h"], + deps = [ + "//src/ray/rpc:core_worker_client", + "@com_google_absl//absl/synchronization", + ], +) diff --git a/src/fakes/ray/rpc/worker/core_worker_client.h b/src/fakes/ray/rpc/worker/core_worker_client.h new file mode 100644 index 000000000000..e4f5f27f1aad --- /dev/null +++ b/src/fakes/ray/rpc/worker/core_worker_client.h @@ -0,0 +1,58 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "absl/synchronization/mutex.h" +#include "ray/rpc/worker/core_worker_client.h" + +namespace ray { + +class FakeCoreWorkerClient : public rpc::CoreWorkerClientInterface { + public: + void PushNormalTask(std::unique_ptr request, + const rpc::ClientCallback &callback) override { + absl::MutexLock lock(&mutex_); + callbacks_.push_back(callback); + } + + bool ReplyPushTask(Status status = Status::OK(), bool exit = false) { + rpc::ClientCallback callback = nullptr; + { + absl::MutexLock lock(&mutex_); + if (callbacks_.size() == 0) { + return false; + } + callback = callbacks_.front(); + callbacks_.pop_front(); + } + // call the callback without the lock to avoid deadlock. + auto reply = rpc::PushTaskReply(); + if (exit) { + reply.set_worker_exiting(true); + } + callback(status, std::move(reply)); + return true; + } + + size_t GetNumCallbacks() { + absl::MutexLock lock(&mutex_); + return callbacks_.size(); + } + + std::list> callbacks_ ABSL_GUARDED_BY(mutex_); + absl::Mutex mutex_; +}; + +} // namespace ray diff --git a/src/ray/core_worker/task_submission/tests/BUILD.bazel b/src/ray/core_worker/task_submission/tests/BUILD.bazel index 707a4d310589..517b2fb220f3 100644 --- a/src/ray/core_worker/task_submission/tests/BUILD.bazel +++ b/src/ray/core_worker/task_submission/tests/BUILD.bazel @@ -65,6 +65,7 @@ ray_cc_test( deps = [ "//:ray_fakes", "//:ray_mock", + "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/common:task_common", "//src/ray/common:test_util", "//src/ray/core_worker:memory_store", diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index c36833757685..b9625c7b2469 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -68,6 +68,7 @@ ray_cc_test( deps = [ "//:ray_fakes", "//:ray_mock", + "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/common:task_common", "//src/ray/common:test_util", @@ -224,6 +225,7 @@ ray_cc_test( deps = [ "//:ray_fakes", "//:ray_mock", + "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/core_worker:experimental_mutable_object_provider", "//src/ray/object_manager:object_manager_common", "//src/ray/object_manager/plasma:plasma_client", @@ -244,6 +246,7 @@ ray_cc_test( deps = [ "//:ray_fakes", "//:ray_mock", + "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/common:test_util", "//src/ray/core_worker:core_worker_lib", "//src/ray/core_worker:memory_store", diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 5e3de7e6ed16..5ce6e18de9e6 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -248,6 +248,43 @@ ray_cc_library( ], ) +ray_cc_library( + name = "gcs_placement_group", + srcs = ["gcs_placement_group.cc"], + hdrs = ["gcs_placement_group.h"], + implementation_deps = [ + "//src/ray/stats:stats_lib", + ], + deps = [ + "//src/ray/common:id", + "//src/ray/common:task_common", + "//src/ray/protobuf:gcs_service_cc_proto", + "//src/ray/util:counter_map", + "//src/ray/util:time", + ], +) + +ray_cc_library( + name = "gcs_placement_group_scheduler", + srcs = ["gcs_placement_group_scheduler.cc"], + hdrs = ["gcs_placement_group_scheduler.h"], + deps = [ + ":gcs_node_manager", + ":gcs_placement_group", + ":gcs_table_storage", + "//src/ray/common:asio", + "//src/ray/common:id", + "//src/ray/common:task_common", + "//src/ray/raylet/scheduling:cluster_resource_scheduler", + "//src/ray/raylet/scheduling:scheduling_context", + "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/rpc:core_worker_client", + "//src/ray/rpc:node_manager_client", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + ], +) + ray_cc_library( name = "grpc_service_interfaces", hdrs = [ @@ -307,7 +344,6 @@ ray_cc_library( "gcs_actor_scheduler.cc", "gcs_autoscaler_state_manager.cc", "gcs_placement_group_mgr.cc", - "gcs_placement_group_scheduler.cc", "gcs_server.cc", ], hdrs = [ @@ -315,7 +351,6 @@ ray_cc_library( "gcs_actor_scheduler.h", "gcs_autoscaler_state_manager.h", "gcs_placement_group_mgr.h", - "gcs_placement_group_scheduler.h", "gcs_server.h", ], deps = [ @@ -326,6 +361,8 @@ ray_cc_library( ":gcs_job_manager", ":gcs_kv_manager", ":gcs_node_manager", + ":gcs_placement_group", + ":gcs_placement_group_scheduler", ":gcs_pubsub_handler", ":gcs_resource_manager", ":gcs_runtime_env_handler", diff --git a/src/ray/gcs/gcs_server/gcs_placement_group.cc b/src/ray/gcs/gcs_server/gcs_placement_group.cc new file mode 100644 index 000000000000..a238e5c51000 --- /dev/null +++ b/src/ray/gcs/gcs_server/gcs_placement_group.cc @@ -0,0 +1,151 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/gcs/gcs_server/gcs_placement_group.h" + +#include +#include +#include + +#include "ray/stats/metric_defs.h" + +namespace ray { +namespace gcs { + +void GcsPlacementGroup::UpdateState( + rpc::PlacementGroupTableData::PlacementGroupState state) { + if (state == rpc::PlacementGroupTableData::CREATED) { + RAY_CHECK_EQ(placement_group_table_data_.state(), + rpc::PlacementGroupTableData::PREPARED); + placement_group_table_data_.set_placement_group_final_bundle_placement_timestamp_ms( + current_sys_time_ms()); + + double duration_s = + (placement_group_table_data_ + .placement_group_final_bundle_placement_timestamp_ms() - + placement_group_table_data_.placement_group_creation_timestamp_ms()) / + 1000; + stats::STATS_scheduler_placement_time_s.Record(duration_s, + {{"WorkloadType", "PlacementGroup"}}); + } + placement_group_table_data_.set_state(state); + RefreshMetrics(); +} + +rpc::PlacementGroupTableData::PlacementGroupState GcsPlacementGroup::GetState() const { + return placement_group_table_data_.state(); +} + +PlacementGroupID GcsPlacementGroup::GetPlacementGroupID() const { + return PlacementGroupID::FromBinary(placement_group_table_data_.placement_group_id()); +} + +std::string GcsPlacementGroup::GetName() const { + return placement_group_table_data_.name(); +} + +std::string GcsPlacementGroup::GetRayNamespace() const { + return placement_group_table_data_.ray_namespace(); +} + +std::vector> &GcsPlacementGroup::GetBundles() + const { + // Fill the cache if it wasn't. + if (cached_bundle_specs_.empty()) { + const auto &bundles = placement_group_table_data_.bundles(); + for (const auto &bundle : bundles) { + cached_bundle_specs_.push_back(std::make_shared(bundle)); + } + } + return cached_bundle_specs_; +} + +std::vector> +GcsPlacementGroup::GetUnplacedBundles() const { + const auto &bundle_specs = GetBundles(); + + std::vector> unplaced_bundles; + for (const auto &bundle : bundle_specs) { + if (bundle->NodeId().IsNil()) { + unplaced_bundles.push_back(bundle); + } + } + return unplaced_bundles; +} + +bool GcsPlacementGroup::HasUnplacedBundles() const { + return !GetUnplacedBundles().empty(); +} + +rpc::PlacementStrategy GcsPlacementGroup::GetStrategy() const { + return placement_group_table_data_.strategy(); +} + +const rpc::PlacementGroupTableData &GcsPlacementGroup::GetPlacementGroupTableData() + const { + return placement_group_table_data_; +} + +std::string GcsPlacementGroup::DebugString() const { + std::stringstream stream; + stream << "placement group id = " << GetPlacementGroupID() << ", name = " << GetName() + << ", strategy = " << GetStrategy(); + return stream.str(); +} + +rpc::Bundle *GcsPlacementGroup::GetMutableBundle(int bundle_index) { + // Invalidate the cache. + cached_bundle_specs_.clear(); + return placement_group_table_data_.mutable_bundles(bundle_index); +} + +const ActorID GcsPlacementGroup::GetCreatorActorId() const { + return ActorID::FromBinary(placement_group_table_data_.creator_actor_id()); +} + +const JobID GcsPlacementGroup::GetCreatorJobId() const { + return JobID::FromBinary(placement_group_table_data_.creator_job_id()); +} + +void GcsPlacementGroup::MarkCreatorJobDead() { + placement_group_table_data_.set_creator_job_dead(true); +} + +void GcsPlacementGroup::MarkCreatorActorDead() { + placement_group_table_data_.set_creator_actor_dead(true); +} + +bool GcsPlacementGroup::IsPlacementGroupLifetimeDone() const { + return !IsDetached() && placement_group_table_data_.creator_job_dead() && + placement_group_table_data_.creator_actor_dead(); +} + +bool GcsPlacementGroup::IsDetached() const { + return placement_group_table_data_.is_detached(); +} + +NodeID GcsPlacementGroup::GetSoftTargetNodeID() const { + return NodeID::FromBinary(placement_group_table_data_.soft_target_node_id()); +} + +const rpc::PlacementGroupStats &GcsPlacementGroup::GetStats() const { + return placement_group_table_data_.stats(); +} + +rpc::PlacementGroupStats *GcsPlacementGroup::GetMutableStats() { + return placement_group_table_data_.mutable_stats(); +} + +} // namespace gcs +} // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_placement_group.h b/src/ray/gcs/gcs_server/gcs_placement_group.h new file mode 100644 index 000000000000..61f41fcbbcc4 --- /dev/null +++ b/src/ray/gcs/gcs_server/gcs_placement_group.h @@ -0,0 +1,212 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include + +#include +#include +#include +#include +#include + +#include "ray/common/bundle_spec.h" +#include "ray/common/id.h" +#include "ray/util/counter_map.h" +#include "ray/util/time.h" +#include "src/ray/protobuf/gcs_service.pb.h" + +namespace ray { +namespace gcs { + +/// GcsPlacementGroup just wraps `PlacementGroupTableData` and provides some convenient +/// interfaces to access the fields inside `PlacementGroupTableData`. This class is not +/// thread-safe. +class GcsPlacementGroup { + public: + /// Create a GcsPlacementGroup by placement_group_table_data. + /// + /// \param placement_group_table_data Data of the placement_group (see gcs.proto). + explicit GcsPlacementGroup( + rpc::PlacementGroupTableData placement_group_table_data, + std::shared_ptr> + counter) + : placement_group_table_data_(std::move(placement_group_table_data)), + counter_(counter) { + SetupStates(); + } + + /// Create a GcsPlacementGroup by CreatePlacementGroupRequest. + /// + /// \param request Contains the placement group creation task specification. + explicit GcsPlacementGroup( + const ray::rpc::CreatePlacementGroupRequest &request, + std::string ray_namespace, + std::shared_ptr> + counter) + : counter_(counter) { + const auto &placement_group_spec = request.placement_group_spec(); + placement_group_table_data_.set_placement_group_id( + placement_group_spec.placement_group_id()); + placement_group_table_data_.set_name(placement_group_spec.name()); + placement_group_table_data_.set_state(rpc::PlacementGroupTableData::PENDING); + placement_group_table_data_.mutable_bundles()->CopyFrom( + placement_group_spec.bundles()); + placement_group_table_data_.set_strategy(placement_group_spec.strategy()); + placement_group_table_data_.set_creator_job_id(placement_group_spec.creator_job_id()); + placement_group_table_data_.set_creator_actor_id( + placement_group_spec.creator_actor_id()); + placement_group_table_data_.set_creator_job_dead( + placement_group_spec.creator_job_dead()); + placement_group_table_data_.set_creator_actor_dead( + placement_group_spec.creator_actor_dead()); + placement_group_table_data_.set_is_detached(placement_group_spec.is_detached()); + placement_group_table_data_.set_soft_target_node_id( + placement_group_spec.soft_target_node_id()); + placement_group_table_data_.set_ray_namespace(ray_namespace); + placement_group_table_data_.set_placement_group_creation_timestamp_ms( + current_sys_time_ms()); + SetupStates(); + } + + ~GcsPlacementGroup() { + if (last_metric_state_ && + last_metric_state_.value() != rpc::PlacementGroupTableData::REMOVED) { + RAY_LOG(DEBUG) << "Decrementing state at " + << rpc::PlacementGroupTableData::PlacementGroupState_Name( + last_metric_state_.value()); + // Retain groups in the REMOVED state so we have a history of past groups. + counter_->Decrement(last_metric_state_.value()); + } + } + + /// Get the immutable PlacementGroupTableData of this placement group. + const rpc::PlacementGroupTableData &GetPlacementGroupTableData() const; + + /// Get the mutable bundle of this placement group. + rpc::Bundle *GetMutableBundle(int bundle_index); + + /// Update the state of this placement_group. + void UpdateState(rpc::PlacementGroupTableData::PlacementGroupState state); + + /// Get the state of this gcs placement_group. + rpc::PlacementGroupTableData::PlacementGroupState GetState() const; + + /// Get the id of this placement_group. + PlacementGroupID GetPlacementGroupID() const; + + /// Get the name of this placement_group. + std::string GetName() const; + + /// Get the name of this placement_group. + std::string GetRayNamespace() const; + + /// Get the bundles of this placement_group (including unplaced). + std::vector> &GetBundles() const; + + /// Get the unplaced bundles of this placement group. + std::vector> GetUnplacedBundles() const; + + /// Check if there are unplaced bundles. + bool HasUnplacedBundles() const; + + /// Get the Strategy + rpc::PlacementStrategy GetStrategy() const; + + /// Get debug string for the placement group. + std::string DebugString() const; + + /// Below fields are used for automatic cleanup of placement groups. + + /// Get the actor id that created the placement group. + const ActorID GetCreatorActorId() const; + + /// Get the job id that created the placement group. + const JobID GetCreatorJobId() const; + + /// Mark that the creator job of this placement group is dead. + void MarkCreatorJobDead(); + + /// Mark that the creator actor of this placement group is dead. + void MarkCreatorActorDead(); + + /// Return True if the placement group lifetime is done. False otherwise. + bool IsPlacementGroupLifetimeDone() const; + + /// Returns whether or not this is a detached placement group. + bool IsDetached() const; + + /// Return the target node ID where bundles of this placement group should be placed. + /// Only works for STRICT_PACK placement group. + NodeID GetSoftTargetNodeID() const; + + const rpc::PlacementGroupStats &GetStats() const; + + rpc::PlacementGroupStats *GetMutableStats(); + + private: + // XXX. + FRIEND_TEST(GcsPlacementGroupManagerTest, TestPlacementGroupBundleCache); + + /// Setup states other than placement_group_table_data_. + void SetupStates() { + auto stats = placement_group_table_data_.mutable_stats(); + // The default value for the field is 0 + if (stats->creation_request_received_ns() == 0) { + auto now = absl::GetCurrentTimeNanos(); + stats->set_creation_request_received_ns(now); + } + // The default value for the field is 0 + // Only set the state to the QUEUED when the state wasn't persisted before. + if (stats->scheduling_state() == 0) { + stats->set_scheduling_state(rpc::PlacementGroupStats::QUEUED); + } + RefreshMetrics(); + } + + /// Record metric updates if there have been any state changes. + void RefreshMetrics() { + auto cur_state = GetState(); + if (last_metric_state_) { + RAY_LOG(DEBUG) << "Swapping state from " + << rpc::PlacementGroupTableData::PlacementGroupState_Name( + last_metric_state_.value()) + << " to " + << rpc::PlacementGroupTableData::PlacementGroupState_Name(cur_state); + counter_->Swap(last_metric_state_.value(), cur_state); + } else { + RAY_LOG(DEBUG) << "Incrementing state at " + << rpc::PlacementGroupTableData::PlacementGroupState_Name(cur_state); + counter_->Increment(cur_state); + } + last_metric_state_ = cur_state; + } + + /// The placement_group meta data which contains the task specification as well as the + /// state of the gcs placement_group and so on (see gcs.proto). + rpc::PlacementGroupTableData placement_group_table_data_; + /// Creating bundle specification requires heavy computation because it needs to compute + /// formatted strings for all resources (heavy string operations). To optimize the CPU + /// usage, we cache bundle specs. + mutable std::vector> cached_bundle_specs_; + + /// Reference to the counter to use for placement group state metrics tracking. + std::shared_ptr> counter_; + + /// The last recorded metric state. + std::optional last_metric_state_; +}; + +} // namespace gcs +} // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc index 3cd7af20aaac..9c4007df4a8e 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc @@ -54,132 +54,6 @@ ExponentialBackoff CreateDefaultBackoff() { } } // namespace -void GcsPlacementGroup::UpdateState( - rpc::PlacementGroupTableData::PlacementGroupState state) { - if (state == rpc::PlacementGroupTableData::CREATED) { - RAY_CHECK_EQ(placement_group_table_data_.state(), - rpc::PlacementGroupTableData::PREPARED); - placement_group_table_data_.set_placement_group_final_bundle_placement_timestamp_ms( - current_sys_time_ms()); - - double duration_s = - (placement_group_table_data_ - .placement_group_final_bundle_placement_timestamp_ms() - - placement_group_table_data_.placement_group_creation_timestamp_ms()) / - 1000; - stats::STATS_scheduler_placement_time_s.Record(duration_s, - {{"WorkloadType", "PlacementGroup"}}); - } - placement_group_table_data_.set_state(state); - RefreshMetrics(); -} - -rpc::PlacementGroupTableData::PlacementGroupState GcsPlacementGroup::GetState() const { - return placement_group_table_data_.state(); -} - -PlacementGroupID GcsPlacementGroup::GetPlacementGroupID() const { - return PlacementGroupID::FromBinary(placement_group_table_data_.placement_group_id()); -} - -std::string GcsPlacementGroup::GetName() const { - return placement_group_table_data_.name(); -} - -std::string GcsPlacementGroup::GetRayNamespace() const { - return placement_group_table_data_.ray_namespace(); -} - -std::vector> &GcsPlacementGroup::GetBundles() - const { - // Fill the cache if it wasn't. - if (cached_bundle_specs_.empty()) { - const auto &bundles = placement_group_table_data_.bundles(); - for (const auto &bundle : bundles) { - cached_bundle_specs_.push_back(std::make_shared(bundle)); - } - } - return cached_bundle_specs_; -} - -std::vector> -GcsPlacementGroup::GetUnplacedBundles() const { - const auto &bundle_specs = GetBundles(); - - std::vector> unplaced_bundles; - for (const auto &bundle : bundle_specs) { - if (bundle->NodeId().IsNil()) { - unplaced_bundles.push_back(bundle); - } - } - return unplaced_bundles; -} - -bool GcsPlacementGroup::HasUnplacedBundles() const { - return !GetUnplacedBundles().empty(); -} - -rpc::PlacementStrategy GcsPlacementGroup::GetStrategy() const { - return placement_group_table_data_.strategy(); -} - -const rpc::PlacementGroupTableData &GcsPlacementGroup::GetPlacementGroupTableData() - const { - return placement_group_table_data_; -} - -std::string GcsPlacementGroup::DebugString() const { - std::stringstream stream; - stream << "placement group id = " << GetPlacementGroupID() << ", name = " << GetName() - << ", strategy = " << GetStrategy(); - return stream.str(); -} - -rpc::Bundle *GcsPlacementGroup::GetMutableBundle(int bundle_index) { - // Invalidate the cache. - cached_bundle_specs_.clear(); - return placement_group_table_data_.mutable_bundles(bundle_index); -} - -const ActorID GcsPlacementGroup::GetCreatorActorId() const { - return ActorID::FromBinary(placement_group_table_data_.creator_actor_id()); -} - -const JobID GcsPlacementGroup::GetCreatorJobId() const { - return JobID::FromBinary(placement_group_table_data_.creator_job_id()); -} - -void GcsPlacementGroup::MarkCreatorJobDead() { - placement_group_table_data_.set_creator_job_dead(true); -} - -void GcsPlacementGroup::MarkCreatorActorDead() { - placement_group_table_data_.set_creator_actor_dead(true); -} - -bool GcsPlacementGroup::IsPlacementGroupLifetimeDone() const { - return !IsDetached() && placement_group_table_data_.creator_job_dead() && - placement_group_table_data_.creator_actor_dead(); -} - -bool GcsPlacementGroup::IsDetached() const { - return placement_group_table_data_.is_detached(); -} - -NodeID GcsPlacementGroup::GetSoftTargetNodeID() const { - return NodeID::FromBinary(placement_group_table_data_.soft_target_node_id()); -} - -const rpc::PlacementGroupStats &GcsPlacementGroup::GetStats() const { - return placement_group_table_data_.stats(); -} - -rpc::PlacementGroupStats *GcsPlacementGroup::GetMutableStats() { - return placement_group_table_data_.mutable_stats(); -} - -///////////////////////////////////////////////////////////////////////////////////////// - GcsPlacementGroupManager::GcsPlacementGroupManager( instrumented_io_context &io_context, GcsResourceManager &gcs_resource_manager) : io_context_(io_context), gcs_resource_manager_(gcs_resource_manager) {} diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h index 37d1afdcf58d..3ee4264d42df 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h @@ -29,6 +29,7 @@ #include "ray/common/task/task_spec.h" #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" +#include "ray/gcs/gcs_server/gcs_placement_group.h" #include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" @@ -44,183 +45,6 @@ namespace ray { namespace gcs { -/// GcsPlacementGroup just wraps `PlacementGroupTableData` and provides some convenient -/// interfaces to access the fields inside `PlacementGroupTableData`. This class is not -/// thread-safe. -class GcsPlacementGroup { - public: - /// Create a GcsPlacementGroup by placement_group_table_data. - /// - /// \param placement_group_table_data Data of the placement_group (see gcs.proto). - explicit GcsPlacementGroup( - rpc::PlacementGroupTableData placement_group_table_data, - std::shared_ptr> - counter) - : placement_group_table_data_(std::move(placement_group_table_data)), - counter_(counter) { - SetupStates(); - } - - /// Create a GcsPlacementGroup by CreatePlacementGroupRequest. - /// - /// \param request Contains the placement group creation task specification. - explicit GcsPlacementGroup( - const ray::rpc::CreatePlacementGroupRequest &request, - std::string ray_namespace, - std::shared_ptr> - counter) - : counter_(counter) { - const auto &placement_group_spec = request.placement_group_spec(); - placement_group_table_data_.set_placement_group_id( - placement_group_spec.placement_group_id()); - placement_group_table_data_.set_name(placement_group_spec.name()); - placement_group_table_data_.set_state(rpc::PlacementGroupTableData::PENDING); - placement_group_table_data_.mutable_bundles()->CopyFrom( - placement_group_spec.bundles()); - placement_group_table_data_.set_strategy(placement_group_spec.strategy()); - placement_group_table_data_.set_creator_job_id(placement_group_spec.creator_job_id()); - placement_group_table_data_.set_creator_actor_id( - placement_group_spec.creator_actor_id()); - placement_group_table_data_.set_creator_job_dead( - placement_group_spec.creator_job_dead()); - placement_group_table_data_.set_creator_actor_dead( - placement_group_spec.creator_actor_dead()); - placement_group_table_data_.set_is_detached(placement_group_spec.is_detached()); - placement_group_table_data_.set_soft_target_node_id( - placement_group_spec.soft_target_node_id()); - placement_group_table_data_.set_ray_namespace(ray_namespace); - placement_group_table_data_.set_placement_group_creation_timestamp_ms( - current_sys_time_ms()); - SetupStates(); - } - - ~GcsPlacementGroup() { - if (last_metric_state_ && - last_metric_state_.value() != rpc::PlacementGroupTableData::REMOVED) { - RAY_LOG(DEBUG) << "Decrementing state at " - << rpc::PlacementGroupTableData::PlacementGroupState_Name( - last_metric_state_.value()); - // Retain groups in the REMOVED state so we have a history of past groups. - counter_->Decrement(last_metric_state_.value()); - } - } - - /// Get the immutable PlacementGroupTableData of this placement group. - const rpc::PlacementGroupTableData &GetPlacementGroupTableData() const; - - /// Get the mutable bundle of this placement group. - rpc::Bundle *GetMutableBundle(int bundle_index); - - /// Update the state of this placement_group. - void UpdateState(rpc::PlacementGroupTableData::PlacementGroupState state); - - /// Get the state of this gcs placement_group. - rpc::PlacementGroupTableData::PlacementGroupState GetState() const; - - /// Get the id of this placement_group. - PlacementGroupID GetPlacementGroupID() const; - - /// Get the name of this placement_group. - std::string GetName() const; - - /// Get the name of this placement_group. - std::string GetRayNamespace() const; - - /// Get the bundles of this placement_group (including unplaced). - std::vector> &GetBundles() const; - - /// Get the unplaced bundles of this placement group. - std::vector> GetUnplacedBundles() const; - - /// Check if there are unplaced bundles. - bool HasUnplacedBundles() const; - - /// Get the Strategy - rpc::PlacementStrategy GetStrategy() const; - - /// Get debug string for the placement group. - std::string DebugString() const; - - /// Below fields are used for automatic cleanup of placement groups. - - /// Get the actor id that created the placement group. - const ActorID GetCreatorActorId() const; - - /// Get the job id that created the placement group. - const JobID GetCreatorJobId() const; - - /// Mark that the creator job of this placement group is dead. - void MarkCreatorJobDead(); - - /// Mark that the creator actor of this placement group is dead. - void MarkCreatorActorDead(); - - /// Return True if the placement group lifetime is done. False otherwise. - bool IsPlacementGroupLifetimeDone() const; - - /// Returns whether or not this is a detached placement group. - bool IsDetached() const; - - /// Return the target node ID where bundles of this placement group should be placed. - /// Only works for STRICT_PACK placement group. - NodeID GetSoftTargetNodeID() const; - - const rpc::PlacementGroupStats &GetStats() const; - - rpc::PlacementGroupStats *GetMutableStats(); - - private: - FRIEND_TEST(GcsPlacementGroupManagerTest, TestPlacementGroupBundleCache); - - /// Setup states other than placement_group_table_data_. - void SetupStates() { - auto stats = placement_group_table_data_.mutable_stats(); - // The default value for the field is 0 - if (stats->creation_request_received_ns() == 0) { - auto now = absl::GetCurrentTimeNanos(); - stats->set_creation_request_received_ns(now); - } - // The default value for the field is 0 - // Only set the state to the QUEUED when the state wasn't persisted before. - if (stats->scheduling_state() == 0) { - stats->set_scheduling_state(rpc::PlacementGroupStats::QUEUED); - } - RefreshMetrics(); - } - - /// Record metric updates if there have been any state changes. - void RefreshMetrics() { - auto cur_state = GetState(); - if (last_metric_state_) { - RAY_LOG(DEBUG) << "Swapping state from " - << rpc::PlacementGroupTableData::PlacementGroupState_Name( - last_metric_state_.value()) - << " to " - << rpc::PlacementGroupTableData::PlacementGroupState_Name(cur_state); - counter_->Swap(last_metric_state_.value(), cur_state); - } else { - RAY_LOG(DEBUG) << "Incrementing state at " - << rpc::PlacementGroupTableData::PlacementGroupState_Name(cur_state); - counter_->Increment(cur_state); - } - last_metric_state_ = cur_state; - } - - /// The placement_group meta data which contains the task specification as well as the - /// state of the gcs placement_group and so on (see gcs.proto). - rpc::PlacementGroupTableData placement_group_table_data_; - /// Creating bundle specification requires heavy computation because it needs to compute - /// formatted strings for all resources (heavy string operations). To optimize the CPU - /// usage, we cache bundle specs. - mutable std::vector> cached_bundle_specs_; - - /// Reference to the counter to use for placement group state metrics tracking. - std::shared_ptr> counter_; - - /// The last recorded metric state. - std::optional last_metric_state_; -}; - /// GcsPlacementGroupManager is responsible for managing the lifecycle of all placement /// group. This class is not thread-safe. /// The placementGroup will be added into queue and set the status as pending first and diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc index 703d72c7a4dd..855139ee8e15 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc @@ -21,8 +21,6 @@ #include #include "ray/common/asio/asio_util.h" -#include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" -#include "src/ray/protobuf/gcs.pb.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h index 6c668fccb8a8..61b7fa651a79 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h @@ -26,6 +26,7 @@ #include "ray/common/id.h" #include "ray/common/scheduling/scheduling_ids.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" +#include "ray/gcs/gcs_server/gcs_placement_group.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/scheduling/policy/scheduling_context.h" @@ -38,8 +39,6 @@ namespace ray { namespace gcs { -class GcsPlacementGroup; - using PGSchedulingFailureCallback = std::function, bool)>; using PGSchedulingSuccessfulCallback = @@ -513,6 +512,11 @@ class GcsPlacementGroupScheduler : public GcsPlacementGroupSchedulerInterface { /// The bundles that waiting to be destroyed and release resources. std::list>> waiting_removed_bundles_; + + friend class GcsPlacementGroupSchedulerTest; + FRIEND_TEST(GcsPlacementGroupSchedulerTest, TestCheckingWildcardResource); + FRIEND_TEST(GcsPlacementGroupSchedulerTest, TestWaitingRemovedBundles); + FRIEND_TEST(GcsPlacementGroupSchedulerTest, TestBundlesRemovedWhenNodeDead); }; } // namespace gcs diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index 87fb68703f42..ced5b241df0e 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -22,6 +22,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", + "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], ) @@ -77,17 +78,6 @@ ray_cc_test( ], ) -ray_cc_library( - name = "gcs_server_test_util", - hdrs = [ - "gcs_server_test_util.h", - ], - deps = [ - "//:ray_fakes", - "//src/ray/gcs/store_client:in_memory_store_client", - ], -) - ray_cc_test( name = "gcs_health_check_manager_test", size = "medium", @@ -116,6 +106,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_fakes", + "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_server:gcs_node_manager", "//src/ray/gcs/tests:gcs_test_util_lib", @@ -167,11 +158,12 @@ ray_cc_test( "team:core", ], deps = [ - ":gcs_server_test_util", "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", + "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], ) @@ -187,10 +179,17 @@ ray_cc_test( "team:core", ], deps = [ - ":gcs_server_test_util", + "//src/fakes/ray/rpc/raylet:fake_raylet_client", + "//src/fakes/ray/rpc/worker:fake_core_worker_client", "//src/mock/ray/pubsub:mock_publisher", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_node_manager", + "//src/ray/gcs/gcs_server:gcs_placement_group", + "//src/ray/gcs/gcs_server:gcs_placement_group_scheduler", + "//src/ray/gcs/gcs_server:gcs_resource_manager", + "//src/ray/gcs/gcs_server:gcs_table_storage", + "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", + "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], ) @@ -203,11 +202,18 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - ":gcs_server_test_util", + "//src/fakes/ray/rpc/raylet:fake_raylet_client", + "//src/fakes/ray/rpc/worker:fake_core_worker_client", "//src/mock/ray/pubsub:mock_publisher", + "//src/ray/gcs/gcs_server:gcs_function_manager", + "//src/ray/gcs/gcs_server:gcs_node_manager", + "//src/ray/gcs/gcs_server:gcs_resource_manager", "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_table_storage", + "//src/ray/gcs/store_client", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", + "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], ) @@ -223,6 +229,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:test_util", "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], ) @@ -237,8 +244,8 @@ ray_cc_test( "team:core", ], deps = [ - ":gcs_server_test_util", "//:ray_mock", + "//src/ray/gcs/gcs_server:gcs_actor", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/pubsub:publisher", @@ -322,8 +329,9 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - ":gcs_server_test_util", + "//:ray_fakes", "//:ray_mock", + "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", @@ -391,7 +399,6 @@ ray_cc_test( "team:core", ], deps = [ - ":gcs_server_test_util", "//:ray_mock", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/store_client:in_memory_store_client", @@ -409,7 +416,7 @@ ray_cc_test( "team:core", ], deps = [ - "//:ray_fakes", + "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/gcs/gcs_server:gcs_node_manager", "//src/ray/gcs/store_client:in_memory_store_client", diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc index a70c40231aba..fa8465fda45d 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - #include #include @@ -25,15 +24,19 @@ #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/runtime_env_manager.h" +#include "ray/gcs/gcs_server/gcs_actor.h" +#include "ray/gcs/gcs_server/gcs_actor_manager.h" +#include "ray/gcs/gcs_server/gcs_function_manager.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/pubsub/publisher.h" +#include "ray/rpc/worker/core_worker_client.h" +#include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/event.h" namespace ray { - namespace gcs { using ::testing::_; @@ -351,5 +354,4 @@ TEST_F(GcsActorManagerTest, TestBasic) { } } // namespace gcs - } // namespace ray diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc index 891343111c0b..659c9a5c53e0 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc @@ -12,23 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "ray/gcs/gcs_server/gcs_actor_manager.h" + +#include + #include #include #include #include #include -// clang-format off -#include "gtest/gtest.h" +#include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" +#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" -#include "ray/gcs/tests/gcs_test_util.h" +#include "ray/common/runtime_env_manager.h" +#include "ray/gcs/gcs_server/gcs_actor.h" +#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" +#include "ray/gcs/gcs_server/gcs_function_manager.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" +#include "ray/gcs/tests/gcs_test_util.h" #include "ray/pubsub/publisher.h" -#include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" -#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" -// clang-format on namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc index c0333a646c0b..8d04a3b18465 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc @@ -25,6 +25,7 @@ #include "ray/common/test_util.h" #include "ray/gcs/gcs_server/gcs_actor_manager.h" #include "ray/gcs/gcs_server/gcs_actor_scheduler.h" +#include "ray/util/counter_map.h" using namespace ::testing; // NOLINT diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc index 72ed153bc9b8..9be76e0dd12b 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" + #include #include @@ -20,27 +22,78 @@ #include #include -// clang-format off +#include "fakes/ray/rpc/raylet/raylet_client.h" +#include "fakes/ray/rpc/worker/core_worker_client.h" +#include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/asio_util.h" -#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/gcs_server/gcs_actor_manager.h" +#include "ray/gcs/gcs_server/gcs_resource_manager.h" +#include "ray/gcs/store_client/in_memory_store_client.h" +#include "ray/gcs/store_client/store_client.h" #include "ray/gcs/tests/gcs_test_util.h" -#include "mock/ray/pubsub/publisher.h" -// clang-format on +#include "ray/util/counter_map.h" namespace ray { using raylet::NoopLocalTaskManager; namespace gcs { +class MockedGcsActorScheduler : public gcs::GcsActorScheduler { + public: + using gcs::GcsActorScheduler::GcsActorScheduler; + + void TryLeaseWorkerFromNodeAgain(std::shared_ptr actor, + std::shared_ptr node) { + DoRetryLeasingWorkerFromNode(std::move(actor), std::move(node)); + } + + protected: + void RetryLeasingWorkerFromNode(std::shared_ptr actor, + std::shared_ptr node) override { + ++num_retry_leasing_count_; + if (num_retry_leasing_count_ <= 1) { + DoRetryLeasingWorkerFromNode(actor, node); + } + } + + void RetryCreatingActorOnWorker(std::shared_ptr actor, + std::shared_ptr worker) override { + ++num_retry_creating_count_; + DoRetryCreatingActorOnWorker(actor, worker); + } + + public: + int num_retry_leasing_count_ = 0; + int num_retry_creating_count_ = 0; +}; + +class FakeGcsActorTable : public gcs::GcsActorTable { + public: + // The store_client and io_context args are NOT used. + explicit FakeGcsActorTable(std::shared_ptr store_client) + : GcsActorTable(store_client) {} + + Status Put(const ActorID &key, + const rpc::ActorTableData &value, + Postable callback) override { + auto status = Status::OK(); + std::move(callback).Post("FakeGcsActorTable.Put", status); + return status; + } + + private: + std::shared_ptr store_client_ = + std::make_shared(); +}; + class GcsActorSchedulerTest : public ::testing::Test { public: void SetUp() override { io_context_ = std::make_unique("GcsActorSchedulerTest"); - raylet_client_ = std::make_shared(); + raylet_client_ = std::make_shared(); raylet_client_pool_ = std::make_shared( [this](const rpc::Address &addr) { return raylet_client_; }); - worker_client_ = std::make_shared(); + worker_client_ = std::make_shared(); gcs_publisher_ = std::make_shared( std::make_unique()); store_client_ = std::make_shared(); @@ -51,8 +104,7 @@ class GcsActorSchedulerTest : public ::testing::Test { io_context_->GetIoService(), raylet_client_pool_.get(), ClusterID::Nil()); - gcs_actor_table_ = - std::make_shared(store_client_); + gcs_actor_table_ = std::make_shared(store_client_); local_node_id_ = NodeID::FromRandom(); cluster_resource_scheduler_ = std::make_unique( io_context_->GetIoService(), @@ -81,7 +133,7 @@ class GcsActorSchedulerTest : public ::testing::Test { local_node_id_); worker_client_pool_ = std::make_unique( [this](const rpc::Address &address) { return worker_client_; }); - gcs_actor_scheduler_ = std::make_shared( + gcs_actor_scheduler_ = std::make_shared( io_context_->GetIoService(), *gcs_actor_table_, *gcs_node_manager_, @@ -157,15 +209,15 @@ class GcsActorSchedulerTest : public ::testing::Test { protected: std::unique_ptr io_context_; std::shared_ptr store_client_; - std::shared_ptr gcs_actor_table_; - std::shared_ptr raylet_client_; - std::shared_ptr worker_client_; + std::shared_ptr gcs_actor_table_; + std::shared_ptr raylet_client_; + std::shared_ptr worker_client_; std::unique_ptr worker_client_pool_; std::shared_ptr gcs_node_manager_; std::unique_ptr local_task_manager_; std::unique_ptr cluster_resource_scheduler_; std::shared_ptr cluster_task_manager_; - std::shared_ptr gcs_actor_scheduler_; + std::shared_ptr gcs_actor_scheduler_; std::shared_ptr>> counter; std::vector> failure_actors_; diff --git a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc index 5507a24b0461..0149b081b648 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc @@ -22,6 +22,7 @@ #include #include +#include "fakes/ray/rpc/raylet/raylet_client.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include "mock/ray/gcs/gcs_server/gcs_actor_manager.h" @@ -33,7 +34,6 @@ #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/store_client_kv.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" @@ -54,7 +54,7 @@ class GcsAutoscalerStateManagerTest : public ::testing::Test { protected: static constexpr char kRayletConfig[] = R"({"raylet_config":"this is a config"})"; instrumented_io_context io_service_; - std::shared_ptr raylet_client_; + std::shared_ptr raylet_client_; std::shared_ptr client_pool_; std::unique_ptr cluster_resource_manager_; std::shared_ptr gcs_resource_manager_; @@ -68,7 +68,7 @@ class GcsAutoscalerStateManagerTest : public ::testing::Test { std::unique_ptr worker_client_pool_; void SetUp() override { - raylet_client_ = std::make_shared(); + raylet_client_ = std::make_shared(); client_pool_ = std::make_unique( [this](const rpc::Address &) { return raylet_client_; }); cluster_resource_manager_ = std::make_unique(io_service_); diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc index ca7355faf94e..359814b61514 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc @@ -14,20 +14,19 @@ #include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" +#include + #include #include #include -// clang-format off -#include "gtest/gtest.h" +#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" +#include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" #include "ray/util/counter_map.h" -#include "mock/ray/pubsub/publisher.h" -#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" -// clang-format on namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc index cfeef60a5fa6..af3db9c5f798 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc @@ -13,22 +13,29 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" + +#include + #include #include #include #include -// clang-format off -#include "gtest/gtest.h" +#include "fakes/ray/rpc/raylet/raylet_client.h" +#include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/tests/gcs_server_test_util.h" +#include "ray/gcs/gcs_server/gcs_node_manager.h" +#include "ray/gcs/gcs_server/gcs_placement_group.h" +#include "ray/gcs/gcs_server/gcs_resource_manager.h" +#include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/util/counter_map.h" -#include "mock/ray/pubsub/publisher.h" -// clang-format on namespace ray { +namespace gcs { enum class GcsPlacementGroupStatus : int32_t { SUCCESS = 0, @@ -44,12 +51,12 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { io_service_.run(); })); for (int index = 0; index < 3; ++index) { - raylet_clients_.push_back(std::make_shared()); + raylet_clients_.push_back(std::make_shared()); } - gcs_table_storage_ = std::make_unique( - std::make_unique()); - gcs_publisher_ = std::make_shared( - std::make_unique()); + gcs_table_storage_ = + std::make_unique(std::make_unique()); + gcs_publisher_ = + std::make_shared(std::make_unique()); auto local_node_id = NodeID::FromRandom(); cluster_resource_scheduler_ = std::make_shared( io_service_, @@ -58,25 +65,25 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { /*is_node_available_fn=*/ [](auto) { return true; }, /*is_local_node_with_raylet=*/false); - gcs_node_manager_ = std::make_shared(gcs_publisher_.get(), - gcs_table_storage_.get(), - io_service_, - raylet_client_pool_.get(), - ClusterID::Nil()); - gcs_resource_manager_ = std::make_shared( + gcs_node_manager_ = std::make_shared(gcs_publisher_.get(), + gcs_table_storage_.get(), + io_service_, + raylet_client_pool_.get(), + ClusterID::Nil()); + gcs_resource_manager_ = std::make_shared( io_service_, cluster_resource_scheduler_->GetClusterResourceManager(), *gcs_node_manager_, local_node_id); - store_client_ = std::make_shared(); + store_client_ = std::make_shared(); raylet_client_pool_ = std::make_unique( [this](const rpc::Address &addr) { return raylet_clients_[addr.port()]; }); - scheduler_ = std::make_shared( - io_service_, - *gcs_table_storage_, - *gcs_node_manager_, - *cluster_resource_scheduler_, - *raylet_client_pool_); + scheduler_ = + std::make_unique(io_service_, + *gcs_table_storage_, + *gcs_node_manager_, + *cluster_resource_scheduler_, + *raylet_client_pool_); counter_.reset(new CounterMap()); } @@ -105,9 +112,8 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { } } - void CheckEqWithPlacementGroupFront( - std::shared_ptr placement_group, - const GcsPlacementGroupStatus status) { + void CheckEqWithPlacementGroupFront(std::shared_ptr placement_group, + const GcsPlacementGroupStatus status) { absl::MutexLock lock(&placement_group_requests_mutex_); if (status == GcsPlacementGroupStatus::SUCCESS) { ASSERT_EQ(placement_group, success_placement_groups_.front()); @@ -140,21 +146,19 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { void ScheduleFailedWithZeroNodeTest(rpc::PlacementStrategy strategy) { ASSERT_EQ(0, gcs_node_manager_->GetAllAliveNodes().size()); auto request = Mocker::GenCreatePlacementGroupRequest("", strategy); - auto placement_group = - std::make_shared(request, "", counter_); + auto placement_group = std::make_shared(request, "", counter_); // Schedule the placement_group with zero node. - scheduler_->ScheduleUnplacedBundles( + scheduler_->ScheduleUnplacedBundles(SchedulePgRequest{ placement_group, - [this](std::shared_ptr placement_group, - bool is_insfeasble) { + [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }, - [this](std::shared_ptr placement_group) { + [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); - }); + }}); // The lease request should not be send and the scheduling of placement_group should // fail as there are no available nodes. @@ -170,22 +174,20 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto request = Mocker::GenCreatePlacementGroupRequest("", strategy); - auto placement_group = - std::make_shared(request, "", counter_); + auto placement_group = std::make_shared(request, "", counter_); // Schedule the placement_group with 1 available node, and the lease request should be // send to the node. - scheduler_->ScheduleUnplacedBundles( + scheduler_->ScheduleUnplacedBundles(SchedulePgRequest{ placement_group, - [this](std::shared_ptr placement_group, - bool is_insfeasble) { + [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }, - [this](std::shared_ptr placement_group) { + [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); - }); + }}); ASSERT_EQ(1, raylet_clients_[0]->num_lease_requested); ASSERT_EQ(1, raylet_clients_[0]->lease_callbacks.size()); @@ -199,30 +201,28 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { void ReschedulingWhenNodeAddTest(rpc::PlacementStrategy strategy) { AddNode(Mocker::GenNodeInfo(0), 1); - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = - [this](std::shared_ptr placement_group) { - absl::MutexLock lock(&placement_group_requests_mutex_); - success_placement_groups_.emplace_back(std::move(placement_group)); - }; + auto success_handler = [this](std::shared_ptr placement_group) { + absl::MutexLock lock(&placement_group_requests_mutex_); + success_placement_groups_.emplace_back(std::move(placement_group)); + }; // Failed to schedule the placement group, because the node resources is not enough. auto request = Mocker::GenCreatePlacementGroupRequest("", strategy); - auto placement_group = - std::make_shared(request, "", counter_); + auto placement_group = std::make_shared(request, "", counter_); scheduler_->ScheduleUnplacedBundles( - placement_group, failure_handler, success_handler); + SchedulePgRequest{placement_group, failure_handler, success_handler}); WaitPlacementGroupPendingDone(1, GcsPlacementGroupStatus::FAILURE); CheckPlacementGroupSize(0, GcsPlacementGroupStatus::SUCCESS); // A new node is added, and the rescheduling is successful. AddNode(Mocker::GenNodeInfo(0), 2); scheduler_->ScheduleUnplacedBundles( - placement_group, failure_handler, success_handler); + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); ASSERT_TRUE(raylet_clients_[0]->GrantCommitBundleResources()); @@ -250,18 +250,17 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { } void ScheduleUnplacedBundles( - const std::shared_ptr &placement_group) { - scheduler_->ScheduleUnplacedBundles( + const std::shared_ptr &placement_group) { + scheduler_->ScheduleUnplacedBundles(SchedulePgRequest{ placement_group, - [this](std::shared_ptr placement_group, - bool is_insfeasble) { + [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }, - [this](std::shared_ptr placement_group) { + [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); - }); + }}); } void GrantPrepareBundleResources(const std::pair &grant0, @@ -290,19 +289,19 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { absl::Mutex placement_group_requests_mutex_; std::unique_ptr thread_io_service_; instrumented_io_context io_service_; - std::shared_ptr store_client_; + std::shared_ptr store_client_; - std::vector> raylet_clients_; - std::shared_ptr gcs_resource_manager_; + std::vector> raylet_clients_; + std::shared_ptr gcs_resource_manager_; std::shared_ptr cluster_resource_scheduler_; - std::shared_ptr gcs_node_manager_; - std::shared_ptr scheduler_; - std::vector> success_placement_groups_ + std::shared_ptr gcs_node_manager_; + std::unique_ptr scheduler_; + std::vector> success_placement_groups_ ABSL_GUARDED_BY(placement_group_requests_mutex_); - std::vector> failure_placement_groups_ + std::vector> failure_placement_groups_ ABSL_GUARDED_BY(placement_group_requests_mutex_); - std::shared_ptr gcs_publisher_; - std::shared_ptr gcs_table_storage_; + std::shared_ptr gcs_publisher_; + std::shared_ptr gcs_table_storage_; std::unique_ptr raylet_client_pool_; std::shared_ptr> counter_; }; @@ -341,21 +340,20 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestSchedulePlacementGroupReplyFailure) { ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared(request, "", counter_); + auto placement_group = std::make_shared(request, "", counter_); // Schedule the placement_group with 1 available node, and the lease request should be // send to the node. - scheduler_->ScheduleUnplacedBundles( + scheduler_->ScheduleUnplacedBundles(SchedulePgRequest{ placement_group, - [this](std::shared_ptr placement_group, - bool is_insfeasble) { + [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }, - [this](std::shared_ptr placement_group) { + [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); - }); + }}); ASSERT_EQ(1, raylet_clients_[0]->num_lease_requested); ASSERT_EQ(1, raylet_clients_[0]->lease_callbacks.size()); @@ -371,24 +369,26 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestSchedulePlacementGroupReplyFailure) { TEST_F(GcsPlacementGroupSchedulerTest, TestSpreadStrategyResourceCheck) { auto node = Mocker::GenNodeInfo(0); AddNode(node, 2); - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; auto request = Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::SPREAD, 3, 2); - auto placement_group = std::make_shared(request, "", counter_); - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + auto placement_group = std::make_shared(request, "", counter_); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); // The node resource is not enough, scheduling failed. WaitPlacementGroupPendingDone(1, GcsPlacementGroupStatus::FAILURE); - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); // The node resource is not enough, scheduling failed. WaitPlacementGroupPendingDone(2, GcsPlacementGroupStatus::FAILURE); @@ -400,21 +400,20 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestSchedulePlacementGroupReturnResource) ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared(request, "", counter_); + auto placement_group = std::make_shared(request, "", counter_); // Schedule the placement_group with 1 available node, and the lease request should be // send to the node. - scheduler_->ScheduleUnplacedBundles( + scheduler_->ScheduleUnplacedBundles(SchedulePgRequest{ placement_group, - [this](std::shared_ptr placement_group, - bool is_insfeasble) { + [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }, - [this](std::shared_ptr placement_group) { + [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); - }); + }}); ASSERT_EQ(1, raylet_clients_[0]->num_lease_requested); ASSERT_EQ(1, raylet_clients_[0]->lease_callbacks.size()); @@ -431,12 +430,12 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestSchedulePlacementGroupReturnResource) TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyBalancedScheduling) { AddNode(Mocker::GenNodeInfo(0)); AddNode(Mocker::GenNodeInfo(1)); - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; @@ -448,10 +447,9 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyBalancedScheduling) for (int index = 0; index < 10; ++index) { auto request = Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::STRICT_PACK); - auto placement_group = - std::make_shared(request, "", counter_); + auto placement_group = std::make_shared(request, "", counter_); scheduler_->ScheduleUnplacedBundles( - placement_group, failure_handler, success_handler); + SchedulePgRequest{placement_group, failure_handler, success_handler}); node_index = !raylet_clients_[0]->lease_callbacks.empty() ? 0 : 1; ++node_select_count[node_index]; @@ -478,19 +476,20 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyReschedulingWhenNod TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyResourceCheck) { auto node0 = Mocker::GenNodeInfo(0); AddNode(node0); - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; auto request = Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::STRICT_PACK); - auto placement_group = std::make_shared(request, "", counter_); - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + auto placement_group = std::make_shared(request, "", counter_); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); ASSERT_TRUE(raylet_clients_[0]->GrantCommitBundleResources()); @@ -502,9 +501,10 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyResourceCheck) { AddNode(node1, 1); auto create_placement_group_request2 = Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::STRICT_PACK); - auto placement_group2 = std::make_shared( - create_placement_group_request2, "", counter_); - scheduler_->ScheduleUnplacedBundles(placement_group2, failure_handler, success_handler); + auto placement_group2 = + std::make_shared(create_placement_group_request2, "", counter_); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group2, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); ASSERT_TRUE(raylet_clients_[0]->GrantCommitBundleResources()); @@ -517,22 +517,21 @@ TEST_F(GcsPlacementGroupSchedulerTest, DestroyPlacementGroup) { ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement_group with 1 available node, and the lease request should be // send to the node. - scheduler_->ScheduleUnplacedBundles( + scheduler_->ScheduleUnplacedBundles(SchedulePgRequest{ placement_group, - [this](std::shared_ptr placement_group, - bool is_insfeasble) { + [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }, - [this](std::shared_ptr placement_group) { + [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); - }); + }}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); ASSERT_TRUE(raylet_clients_[0]->GrantCommitBundleResources()); @@ -556,23 +555,22 @@ TEST_F(GcsPlacementGroupSchedulerTest, DestroyCancelledPlacementGroup) { ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); const auto &placement_group_id = placement_group->GetPlacementGroupID(); // Schedule the placement_group with 1 available node, and the lease request should be // send to the node. - scheduler_->ScheduleUnplacedBundles( + scheduler_->ScheduleUnplacedBundles(SchedulePgRequest{ placement_group, - [this](std::shared_ptr placement_group, - bool is_insfeasble) { + [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }, - [this](std::shared_ptr placement_group) { + [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); - }); + }}); // Now, cancel the schedule request. ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); @@ -591,23 +589,22 @@ TEST_F(GcsPlacementGroupSchedulerTest, PlacementGroupCancelledDuringCommit) { ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); const auto &placement_group_id = placement_group->GetPlacementGroupID(); // Schedule the placement_group with 1 available node, and the lease request should be // send to the node. - scheduler_->ScheduleUnplacedBundles( + scheduler_->ScheduleUnplacedBundles(SchedulePgRequest{ placement_group, - [this](std::shared_ptr placement_group, - bool is_insfeasble) { + [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }, - [this](std::shared_ptr placement_group) { + [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); - }); + }}); // Now, cancel the schedule request. ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); @@ -634,21 +631,22 @@ TEST_F(GcsPlacementGroupSchedulerTest, PlacementGroupCancelledDuringPreparedPut) ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement group successfully. - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); scheduler_->MarkScheduleCancelled(placement_group->GetPlacementGroupID()); ASSERT_TRUE(raylet_clients_[1]->GrantPrepareBundleResources()); @@ -679,12 +677,12 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPackStrategyReschedulingWhenNodeAdd) TEST_F(GcsPlacementGroupSchedulerTest, TestPackStrategyLargeBundlesScheduling) { AddNode(Mocker::GenNodeInfo(0)); AddNode(Mocker::GenNodeInfo(1)); - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; @@ -693,8 +691,9 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPackStrategyLargeBundlesScheduling) { // One node does not have enough resources, so we will divide bundles to two nodes. auto request = Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::PACK, 15); - auto placement_group = std::make_shared(request, "", counter_); - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + auto placement_group = std::make_shared(request, "", counter_); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); // Prepared resource is batched! ASSERT_EQ(raylet_clients_[0]->num_lease_requested, 1); ASSERT_EQ(raylet_clients_[1]->num_lease_requested, 1); @@ -720,21 +719,22 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictSpreadRescheduleWhenNodeDead) { auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest( "pg1", rpc::PlacementStrategy::STRICT_SPREAD); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement group successfully. - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); // Prepare bundle resources. for (int index = 0; index < node_count; ++index) { @@ -767,7 +767,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictSpreadRescheduleWhenNodeDead) { // One node is dead, reschedule the placement group. auto bundle_on_dead_node = placement_group->GetMutableBundle(0); bundle_on_dead_node->clear_node_id(); - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); // Prepare bundle resources. for (int index = 0; index < node_count; ++index) { @@ -791,19 +792,20 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictSpreadRescheduleWhenNodeDead) { TEST_F(GcsPlacementGroupSchedulerTest, TestStrictSpreadStrategyResourceCheck) { auto node0 = Mocker::GenNodeInfo(0); AddNode(node0); - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; auto request = Mocker::GenCreatePlacementGroupRequest( "", rpc::PlacementStrategy::STRICT_SPREAD, 2, 2); - auto placement_group = std::make_shared(request, "", counter_); - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + auto placement_group = std::make_shared(request, "", counter_); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); // The number of nodes is less than the number of bundles, scheduling failed. WaitPlacementGroupPendingDone(1, GcsPlacementGroupStatus::FAILURE); @@ -811,13 +813,15 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictSpreadStrategyResourceCheck) { // Node1 resource is insufficient, scheduling failed. auto node1 = Mocker::GenNodeInfo(1); AddNode(node1, 1); - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); WaitPlacementGroupPendingDone(2, GcsPlacementGroupStatus::FAILURE); // The node2 resource is enough and the scheduling is successful. auto node2 = Mocker::GenNodeInfo(2); AddNode(node2); - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); ASSERT_TRUE(raylet_clients_[2]->GrantPrepareBundleResources()); WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); @@ -911,23 +915,24 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestNodeDeadDuringPreparingResources) { ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement group. // One node is dead, so one bundle failed to schedule. - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); ASSERT_EQ(placement_group->GetUnplacedBundles().size(), 2); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); RemoveNode(node1); // This should fail because the node is dead. @@ -948,23 +953,24 @@ TEST_F(GcsPlacementGroupSchedulerTest, ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement group. // One node is dead, so one bundle failed to schedule. - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); ASSERT_EQ(placement_group->GetUnplacedBundles().size(), 1); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); RemoveNode(node1); // If node is dead right after raylet succeds to create a bundle, it will reply that @@ -989,23 +995,24 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestNodeDeadBeforeCommittingResources) { ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement group. // One node is dead, so one bundle failed to schedule. - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); ASSERT_EQ(placement_group->GetUnplacedBundles().size(), 1); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); // node1 dead right after prepare succeeded. To simulate gcs_placement_group_scheduler // finding the node dead before it tries to commit all nodes, we remove node *before* @@ -1027,23 +1034,24 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestNodeErrorDuringCommittingResources) { ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement group. // One node is dead, so one bundle failed to schedule. - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); ASSERT_EQ(placement_group->GetUnplacedBundles().size(), 1); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); ASSERT_TRUE(raylet_clients_[1]->GrantPrepareBundleResources()); WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); @@ -1063,21 +1071,22 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestNodeDeadDuringRescheduling) { ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement group successfully. - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); ASSERT_TRUE(raylet_clients_[1]->GrantPrepareBundleResources()); WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); @@ -1095,7 +1104,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestNodeDeadDuringRescheduling) { // All nodes are dead, reschedule the placement group. placement_group->GetMutableBundle(0)->clear_node_id(); placement_group->GetMutableBundle(1)->clear_node_id(); - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); // Before prepare requests are done, suppose a node is dead. @@ -1118,21 +1128,22 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPGCancelledDuringReschedulingCommit) ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement group successfully. - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); ASSERT_TRUE(raylet_clients_[1]->GrantPrepareBundleResources()); WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); @@ -1150,7 +1161,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPGCancelledDuringReschedulingCommit) // All nodes are dead, reschedule the placement group. placement_group->GetMutableBundle(0)->clear_node_id(); placement_group->GetMutableBundle(1)->clear_node_id(); - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); // Rescheduling happening. ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); @@ -1175,21 +1187,22 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPGCancelledDuringReschedulingCommitPr ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement group successfully. - auto failure_handler = [this](std::shared_ptr placement_group, + auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }; - auto success_handler = [this](std::shared_ptr placement_group) { + auto success_handler = [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); ASSERT_TRUE(raylet_clients_[1]->GrantPrepareBundleResources()); WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); @@ -1207,7 +1220,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPGCancelledDuringReschedulingCommitPr // All nodes are dead, reschedule the placement group. placement_group->GetMutableBundle(0)->clear_node_id(); placement_group->GetMutableBundle(1)->clear_node_id(); - scheduler_->ScheduleUnplacedBundles(placement_group, failure_handler, success_handler); + scheduler_->ScheduleUnplacedBundles( + SchedulePgRequest{placement_group, failure_handler, success_handler}); // Rescheduling happening. // Cancel the placement group scheduling before prepare requests are granted. @@ -1237,8 +1251,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestInitialize) { ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); placement_group->GetMutableBundle(0)->set_node_id(node0->node_id()); placement_group->GetMutableBundle(1)->set_node_id(node1->node_id()); @@ -1270,7 +1284,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPrepareFromDeadNodes) { ASSERT_TRUE(EnsureClusterResourcesAreNotInUse()); // Create a placement group. - auto placement_group = std::make_shared( + auto placement_group = std::make_shared( Mocker::GenCreatePlacementGroupRequest(), "", counter_); // Schedule the unplaced bundles of the placement_group. @@ -1298,7 +1312,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPrepareFromNodeWithInsufficientResour ASSERT_TRUE(EnsureClusterResourcesAreNotInUse()); // Create a placement group. - auto placement_group = std::make_shared( + auto placement_group = std::make_shared( Mocker::GenCreatePlacementGroupRequest(), "", counter_); // Schedule the unplaced bundles of the placement_group. @@ -1326,7 +1340,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestCommitToDeadNodes) { ASSERT_TRUE(EnsureClusterResourcesAreNotInUse()); // Create a placement group. - auto placement_group = std::make_shared( + auto placement_group = std::make_shared( Mocker::GenCreatePlacementGroupRequest(), "", counter_); // Schedule the unplaced bundles of the placement_group. @@ -1354,8 +1368,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestCommitToDeadNodes) { TEST_F(GcsPlacementGroupSchedulerTest, TestCheckingWildcardResource) { auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest( /*name=*/"", /*strategy=*/rpc::PlacementStrategy::SPREAD, /*bundles_count=*/1); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); int wildcard_resource_count = 0; for (const auto &bundle_spec : placement_group->GetBundles()) { for (const auto &resource_entry : bundle_spec->GetFormattedResources()) { @@ -1378,22 +1392,21 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestWaitingRemovedBundles) { ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement_group with 1 available node, and the lease request should be // send to the node. - scheduler_->ScheduleUnplacedBundles( + scheduler_->ScheduleUnplacedBundles(SchedulePgRequest{ placement_group, - [this](std::shared_ptr placement_group, - bool is_insfeasble) { + [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }, - [this](std::shared_ptr placement_group) { + [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); - }); + }}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); ASSERT_TRUE(raylet_clients_[0]->GrantCommitBundleResources()); @@ -1418,7 +1431,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestWaitingRemovedBundles) { ASSERT_TRUE(raylet_clients_[0]->GrantCancelResourceReserve()); // Because actors have not released the bundle resources, bundles have to keep waiting. - ASSERT_EQ(scheduler_->GetWaitingRemovedBundlesSize(), 2); + ASSERT_EQ(scheduler_->waiting_removed_bundles_.size(), 2); const auto &node_resources = cluster_resource_scheduler_->GetClusterResourceManager().GetNodeResources( scheduling::NodeID(node->node_id())); @@ -1437,7 +1450,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestWaitingRemovedBundles) { scheduler_->HandleWaitingRemovedBundles(); // The waiting bundles are removed, and resources are successfully returned to node. - ASSERT_EQ(scheduler_->GetWaitingRemovedBundlesSize(), 0); + ASSERT_EQ(scheduler_->waiting_removed_bundles_.size(), 0); ASSERT_EQ(node_resources.available.Get(scheduling::ResourceID::CPU()), node_resources.total.Get(scheduling::ResourceID::CPU())); } @@ -1448,22 +1461,21 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestBundlesRemovedWhenNodeDead) { ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); - auto placement_group = std::make_shared( - create_placement_group_request, "", counter_); + auto placement_group = + std::make_shared(create_placement_group_request, "", counter_); // Schedule the placement_group with 1 available node, and the lease request should be // send to the node. - scheduler_->ScheduleUnplacedBundles( + scheduler_->ScheduleUnplacedBundles(SchedulePgRequest{ placement_group, - [this](std::shared_ptr placement_group, - bool is_insfeasble) { + [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); failure_placement_groups_.emplace_back(std::move(placement_group)); }, - [this](std::shared_ptr placement_group) { + [this](std::shared_ptr placement_group) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); - }); + }}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); WaitPendingDone(raylet_clients_[0]->commit_callbacks, 1); ASSERT_TRUE(raylet_clients_[0]->GrantCommitBundleResources()); @@ -1479,7 +1491,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestBundlesRemovedWhenNodeDead) { // There shouldn't be any remaining bundles to be removed since the node is // already removed. The bundles are already removed when the node is removed. - ASSERT_EQ(scheduler_->GetWaitingRemovedBundlesSize(), 0); + ASSERT_EQ(scheduler_->waiting_removed_bundles_.size(), 0); } +} // namespace gcs } // namespace ray diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h deleted file mode 100644 index eb8cf7da9eba..000000000000 --- a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h +++ /dev/null @@ -1,386 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "absl/base/thread_annotations.h" -#include "absl/synchronization/mutex.h" -#include "fakes/ray/rpc/raylet/raylet_client.h" -#include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/task/task.h" -#include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" -#include "ray/gcs/gcs_server/gcs_actor_manager.h" -#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" -#include "ray/gcs/gcs_server/gcs_node_manager.h" -#include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" -#include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" -#include "ray/gcs/gcs_server/gcs_resource_manager.h" -#include "ray/gcs/store_client/in_memory_store_client.h" - -namespace ray { - -struct GcsServerMocker { - class MockWorkerClient : public rpc::CoreWorkerClientInterface { - public: - void PushNormalTask( - std::unique_ptr request, - const rpc::ClientCallback &callback) override { - absl::MutexLock lock(&mutex_); - callbacks_.push_back(callback); - } - - bool ReplyPushTask(Status status = Status::OK(), bool exit = false) { - rpc::ClientCallback callback = nullptr; - { - absl::MutexLock lock(&mutex_); - if (callbacks_.size() == 0) { - return false; - } - callback = callbacks_.front(); - callbacks_.pop_front(); - } - // call the callback without the lock to avoid deadlock. - auto reply = rpc::PushTaskReply(); - if (exit) { - reply.set_worker_exiting(true); - } - callback(status, std::move(reply)); - return true; - } - - size_t GetNumCallbacks() { - absl::MutexLock lock(&mutex_); - return callbacks_.size(); - } - - std::list> callbacks_ ABSL_GUARDED_BY(mutex_); - absl::Mutex mutex_; - }; - - class MockRayletClient : public FakeRayletClient { - public: - ray::Status ReturnWorker(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) override { - if (disconnect_worker) { - num_workers_disconnected++; - } else { - num_workers_returned++; - } - return Status::OK(); - } - - void GetTaskFailureCause( - const TaskID &task_id, - const ray::rpc::ClientCallback &callback) - override { - ray::rpc::GetTaskFailureCauseReply reply; - callback(Status::OK(), std::move(reply)); - num_get_task_failure_causes += 1; - } - - void RequestWorkerLease( - const rpc::TaskSpec &spec, - bool grant_or_reject, - const rpc::ClientCallback &callback, - const int64_t backlog_size, - const bool is_selected_based_on_locality) override { - num_workers_requested += 1; - callbacks.push_back(callback); - } - - void PrestartWorkers( - const rpc::PrestartWorkersRequest &request, - const rpc::ClientCallback &callback) override { - RAY_LOG(FATAL) << "Not implemented"; - } - - void ReleaseUnusedActorWorkers( - const std::vector &workers_in_use, - const rpc::ClientCallback &callback) - override { - num_release_unused_workers += 1; - release_callbacks.push_back(callback); - } - - void CancelWorkerLease( - const TaskID &task_id, - const rpc::ClientCallback &callback) override { - num_leases_canceled += 1; - cancel_callbacks.push_back(callback); - } - - bool GrantWorkerLease() { - return GrantWorkerLease("", 0, WorkerID::FromRandom(), node_id_, NodeID::Nil()); - } - - bool GrantWorkerLease(const std::string &address, - int port, - const WorkerID &worker_id, - const NodeID &node_id, - const NodeID &retry_at_node_id, - Status status = Status::OK(), - bool rejected = false) { - rpc::RequestWorkerLeaseReply reply; - if (!retry_at_node_id.IsNil()) { - reply.mutable_retry_at_raylet_address()->set_ip_address(address); - reply.mutable_retry_at_raylet_address()->set_port(port); - reply.mutable_retry_at_raylet_address()->set_node_id(retry_at_node_id.Binary()); - } else { - reply.mutable_worker_address()->set_ip_address(address); - reply.mutable_worker_address()->set_port(port); - reply.mutable_worker_address()->set_node_id(node_id.Binary()); - reply.mutable_worker_address()->set_worker_id(worker_id.Binary()); - } - if (rejected) { - reply.set_rejected(true); - auto resources_data = reply.mutable_resources_data(); - resources_data->set_node_id(node_id.Binary()); - resources_data->set_resources_normal_task_changed(true); - auto &normal_task_map = *(resources_data->mutable_resources_normal_task()); - normal_task_map[kMemory_ResourceLabel] = - static_cast(std::numeric_limits::max()); - resources_data->set_resources_normal_task_timestamp(absl::GetCurrentTimeNanos()); - } - - if (callbacks.size() == 0) { - return false; - } else { - auto callback = callbacks.front(); - callback(status, std::move(reply)); - callbacks.pop_front(); - return true; - } - } - - bool ReplyCancelWorkerLease(bool success = true) { - rpc::CancelWorkerLeaseReply reply; - reply.set_success(success); - if (cancel_callbacks.size() == 0) { - return false; - } else { - auto callback = cancel_callbacks.front(); - callback(Status::OK(), std::move(reply)); - cancel_callbacks.pop_front(); - return true; - } - } - - bool ReplyReleaseUnusedActorWorkers() { - rpc::ReleaseUnusedActorWorkersReply reply; - if (release_callbacks.size() == 0) { - return false; - } else { - auto callback = release_callbacks.front(); - callback(Status::OK(), std::move(reply)); - release_callbacks.pop_front(); - return true; - } - } - - bool ReplyDrainRaylet() { - if (drain_raylet_callbacks.size() == 0) { - return false; - } else { - rpc::DrainRayletReply reply; - reply.set_is_accepted(true); - auto callback = drain_raylet_callbacks.front(); - callback(Status::OK(), std::move(reply)); - drain_raylet_callbacks.pop_front(); - return true; - } - } - - void PrepareBundleResources( - const std::vector> &bundle_specs, - const ray::rpc::ClientCallback &callback) - override { - num_lease_requested += 1; - lease_callbacks.push_back(callback); - } - - void CommitBundleResources( - const std::vector> &bundle_specs, - const ray::rpc::ClientCallback &callback) - override { - num_commit_requested += 1; - commit_callbacks.push_back(callback); - } - - void CancelResourceReserve( - const BundleSpecification &bundle_spec, - const ray::rpc::ClientCallback &callback) - override { - num_return_requested += 1; - return_callbacks.push_back(callback); - } - - void ReleaseUnusedBundles( - const std::vector &bundles_in_use, - const rpc::ClientCallback &callback) override { - ++num_release_unused_bundles_requested; - } - - bool GrantPrepareBundleResources(bool success = true, - const Status &status = Status::OK()) { - rpc::PrepareBundleResourcesReply reply; - reply.set_success(success); - if (lease_callbacks.size() == 0) { - return false; - } else { - auto callback = lease_callbacks.front(); - callback(status, std::move(reply)); - lease_callbacks.pop_front(); - return true; - } - } - - bool GrantCommitBundleResources(const Status &status = Status::OK()) { - rpc::CommitBundleResourcesReply reply; - if (commit_callbacks.size() == 0) { - return false; - } else { - auto callback = commit_callbacks.front(); - callback(status, std::move(reply)); - commit_callbacks.pop_front(); - return true; - } - } - - bool GrantCancelResourceReserve(bool success = true) { - Status status = Status::OK(); - rpc::CancelResourceReserveReply reply; - if (return_callbacks.size() == 0) { - return false; - } else { - auto callback = return_callbacks.front(); - callback(status, std::move(reply)); - return_callbacks.pop_front(); - return true; - } - } - - void DrainRaylet( - const rpc::autoscaler::DrainNodeReason &reason, - const std::string &reason_message, - int64_t deadline_timestamp_ms, - const rpc::ClientCallback &callback) override { - rpc::DrainRayletReply reply; - reply.set_is_accepted(true); - drain_raylet_callbacks.push_back(callback); - }; - - ~MockRayletClient() {} - - int num_workers_requested = 0; - int num_workers_returned = 0; - int num_workers_disconnected = 0; - int num_leases_canceled = 0; - int num_release_unused_workers = 0; - int num_get_task_failure_causes = 0; - NodeID node_id_ = NodeID::FromRandom(); - std::list> drain_raylet_callbacks = {}; - std::list> callbacks = {}; - std::list> cancel_callbacks = {}; - std::list> - release_callbacks = {}; - int num_lease_requested = 0; - int num_return_requested = 0; - int num_commit_requested = 0; - - int num_release_unused_bundles_requested = 0; - std::list> lease_callbacks = {}; - std::list> commit_callbacks = {}; - std::list> return_callbacks = {}; - }; - - class MockedGcsActorScheduler : public gcs::GcsActorScheduler { - public: - using gcs::GcsActorScheduler::GcsActorScheduler; - - void TryLeaseWorkerFromNodeAgain(std::shared_ptr actor, - std::shared_ptr node) { - DoRetryLeasingWorkerFromNode(std::move(actor), std::move(node)); - } - - protected: - void RetryLeasingWorkerFromNode(std::shared_ptr actor, - std::shared_ptr node) override { - ++num_retry_leasing_count_; - if (num_retry_leasing_count_ <= 1) { - DoRetryLeasingWorkerFromNode(actor, node); - } - } - - void RetryCreatingActorOnWorker(std::shared_ptr actor, - std::shared_ptr worker) override { - ++num_retry_creating_count_; - DoRetryCreatingActorOnWorker(actor, worker); - } - - public: - int num_retry_leasing_count_ = 0; - int num_retry_creating_count_ = 0; - }; - - class MockedGcsPlacementGroupScheduler : public gcs::GcsPlacementGroupScheduler { - public: - using gcs::GcsPlacementGroupScheduler::GcsPlacementGroupScheduler; - - size_t GetWaitingRemovedBundlesSize() { return waiting_removed_bundles_.size(); } - - using gcs::GcsPlacementGroupScheduler::ScheduleUnplacedBundles; - // Extra conveinence overload for the mock tests to keep using the old interface. - void ScheduleUnplacedBundles( - const std::shared_ptr &placement_group, - gcs::PGSchedulingFailureCallback failure_callback, - gcs::PGSchedulingSuccessfulCallback success_callback) { - ScheduleUnplacedBundles( - gcs::SchedulePgRequest{placement_group, failure_callback, success_callback}); - }; - - protected: - friend class GcsPlacementGroupSchedulerTest; - FRIEND_TEST(GcsPlacementGroupSchedulerTest, TestCheckingWildcardResource); - }; - class MockedGcsActorTable : public gcs::GcsActorTable { - public: - // The store_client and io_context args are NOT used. - explicit MockedGcsActorTable(std::shared_ptr store_client) - : GcsActorTable(store_client) {} - - Status Put(const ActorID &key, - const rpc::ActorTableData &value, - Postable callback) override { - auto status = Status::OK(); - std::move(callback).Post("FakeGcsActorTable.Put", status); - return status; - } - - private: - std::shared_ptr store_client_ = - std::make_shared(); - }; -}; - -} // namespace ray diff --git a/src/ray/raylet/tests/BUILD.bazel b/src/ray/raylet/tests/BUILD.bazel index d85e2e164fea..52321de9f933 100644 --- a/src/ray/raylet/tests/BUILD.bazel +++ b/src/ray/raylet/tests/BUILD.bazel @@ -180,6 +180,7 @@ ray_cc_test( ":util", "//:ray_fakes", "//:ray_mock", + "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/common:ray_object", "//src/ray/object_manager/plasma:plasma_client", "//src/ray/raylet:local_object_manager_interface", diff --git a/src/ray/rpc/node_manager/tests/BUILD.bazel b/src/ray/rpc/node_manager/tests/BUILD.bazel index e8c9d9dd6b2d..d26b0fdb0a0d 100644 --- a/src/ray/rpc/node_manager/tests/BUILD.bazel +++ b/src/ray/rpc/node_manager/tests/BUILD.bazel @@ -6,7 +6,7 @@ ray_cc_test( srcs = ["raylet_client_pool_test.cc"], tags = ["team:core"], deps = [ - "//:ray_fakes", + "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/rpc:node_manager_client", "@com_google_googletest//:gtest", From 23fc36bf5f94283fb2788b4fcf682d099bb4a585 Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Thu, 28 Aug 2025 09:44:31 -0700 Subject: [PATCH 329/634] [serve] use shutdown async for microbenchmark cleanup (#56013) ## Why are these changes needed? `serve.shutdown()` is best effort in async contexts. we should use `shutdown_async()` ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: akyang-anyscale --- .../serve_tests/workloads/microbenchmarks.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/release/serve_tests/workloads/microbenchmarks.py b/release/serve_tests/workloads/microbenchmarks.py index 983cef1d5f3a..d75a204f14cf 100644 --- a/release/serve_tests/workloads/microbenchmarks.py +++ b/release/serve_tests/workloads/microbenchmarks.py @@ -145,7 +145,7 @@ async def _main( num_requests=NUM_REQUESTS, ) perf_metrics.extend(convert_latencies_to_perf_metrics(name, latencies)) - serve.shutdown() + await serve.shutdown_async() if run_throughput: # Microbenchmark: HTTP throughput @@ -177,7 +177,7 @@ async def _main( perf_metrics.extend( convert_throughput_to_perf_metrics(test_name, mean, std) ) - serve.shutdown() + await serve.shutdown_async() if run_streaming: # Direct streaming between replica @@ -214,7 +214,7 @@ async def _main( perf_metrics.extend( convert_latencies_to_perf_metrics("http_streaming", latencies) ) - serve.shutdown() + await serve.shutdown_async() # Streaming with intermediate router serve.run( @@ -248,7 +248,7 @@ async def _main( "http_intermediate_streaming", latencies ) ) - serve.shutdown() + await serve.shutdown_async() # GRPC if run_grpc: @@ -280,7 +280,7 @@ async def _main( num_requests=NUM_REQUESTS, ) perf_metrics.extend(convert_latencies_to_perf_metrics(name, latencies)) - serve.shutdown() + await serve.shutdown_async() if run_throughput: # Microbenchmark: GRPC throughput @@ -315,7 +315,7 @@ async def _main( perf_metrics.extend( convert_throughput_to_perf_metrics(test_name, mean, std) ) - serve.shutdown() + await serve.shutdown_async() # Handle if run_handle: @@ -330,7 +330,7 @@ async def _main( num_requests=NUM_REQUESTS, payload=payload ) perf_metrics.extend(convert_latencies_to_perf_metrics(name, latencies)) - serve.shutdown() + await serve.shutdown_async() if run_throughput: # Microbenchmark: Handle throughput @@ -367,7 +367,7 @@ async def _main( perf_metrics.extend( convert_throughput_to_perf_metrics(test_name, mean, std) ) - serve.shutdown() + await serve.shutdown_async() if run_streaming: h: DeploymentHandle = serve.run( @@ -394,7 +394,7 @@ async def _main( perf_metrics.extend( convert_latencies_to_perf_metrics("handle_streaming", latencies) ) - serve.shutdown() + await serve.shutdown_async() logging.info(f"Perf metrics:\n {json.dumps(perf_metrics, indent=4)}") results = {"perf_metrics": perf_metrics} From 0a2bacf106d8e937de16473c5f49ea879d9e9b5f Mon Sep 17 00:00:00 2001 From: Lehui Liu Date: Thu, 28 Aug 2025 09:45:35 -0700 Subject: [PATCH 330/634] Calculate _num_batches_to_skip based on global_rows_processed_this_epoch (#55964) 1. Previously, we use `_restored_train_batch_idx` as a run_state to determine how many batches to skip when resuming training. 2. In this PR we introduced a `_global_rows_processed_this_epoch` and `_num_batches_to_skip` instead so that it will be easier for us to calculate the num_batches to skip when resuming training with different number of workers. 3. Added one `checkpoint_every_n_steps: int = -1` config so that we can separate validation and checkpoint frequency. 4. release test run: https://buildkite.com/ray-project/release/builds/55274 Signed-off-by: Lehui Liu --- release/train_tests/benchmark/config.py | 27 ++++---- release/train_tests/benchmark/runner.py | 68 +++++++++++++------ .../train_tests/benchmark/train_benchmark.py | 24 ++++--- 3 files changed, 79 insertions(+), 40 deletions(-) diff --git a/release/train_tests/benchmark/config.py b/release/train_tests/benchmark/config.py index 03f55de0ce6d..b0686d8c4d23 100644 --- a/release/train_tests/benchmark/config.py +++ b/release/train_tests/benchmark/config.py @@ -83,6 +83,9 @@ class BenchmarkConfig(BaseModel): num_epochs: int = 1 skip_train_step: bool = False + # Checkpointing + checkpoint_every_n_steps: int = -1 + # Validation validate_every_n_steps: int = -1 skip_validation_step: bool = False @@ -109,11 +112,11 @@ def _add_field_to_parser(parser: argparse.ArgumentParser, field: str, field_info parser.add_argument(f"--{field}", type=field_type, default=field_info.default) -def cli_to_config() -> BenchmarkConfig: +def cli_to_config(benchmark_config_cls=BenchmarkConfig) -> BenchmarkConfig: parser = argparse.ArgumentParser() nested_fields = [] - for field, field_info in BenchmarkConfig.model_fields.items(): + for field, field_info in benchmark_config_cls.model_fields.items(): # Skip nested configs for now if _is_pydantic_model(field_info.annotation): nested_fields.append(field) @@ -127,24 +130,24 @@ def cli_to_config() -> BenchmarkConfig: nested_configs = {} for nested_field in nested_fields: nested_parser = argparse.ArgumentParser() - config_cls = BenchmarkConfig.model_fields[nested_field].annotation + nested_config_cls = benchmark_config_cls.model_fields[nested_field].annotation - if config_cls == DataLoaderConfig: + if nested_config_cls == DataLoaderConfig: if top_level_args.dataloader_type == DataloaderType.RAY_DATA: - config_cls = RayDataConfig + nested_config_cls = RayDataConfig elif top_level_args.dataloader_type == DataloaderType.TORCH: - config_cls = TorchConfig + nested_config_cls = TorchConfig - if config_cls == TaskConfig: + if nested_config_cls == TaskConfig: if top_level_args.task == ImageClassificationConfig.TASK_NAME: - config_cls = ImageClassificationConfig + nested_config_cls = ImageClassificationConfig elif top_level_args.task == RecsysConfig.TASK_NAME: - config_cls = RecsysConfig + nested_config_cls = RecsysConfig - for field, field_info in config_cls.model_fields.items(): + for field, field_info in nested_config_cls.model_fields.items(): _add_field_to_parser(nested_parser, field, field_info) args, _ = nested_parser.parse_known_args() - nested_configs[nested_field] = config_cls(**vars(args)) + nested_configs[nested_field] = nested_config_cls(**vars(args)) - return BenchmarkConfig(**vars(top_level_args), **nested_configs) + return benchmark_config_cls(**vars(top_level_args), **nested_configs) diff --git a/release/train_tests/benchmark/runner.py b/release/train_tests/benchmark/runner.py index 5842059fae6a..ab20c9205a40 100644 --- a/release/train_tests/benchmark/runner.py +++ b/release/train_tests/benchmark/runner.py @@ -32,7 +32,7 @@ def __init__(self, factory: BenchmarkFactory): # Training progress state. self._train_batch_idx: int = 0 self._train_epoch_idx: int = 0 - self._restored_train_batch_idx: Optional[int] = None + self._global_rows_processed_this_epoch: int = 0 # Performance metrics self._metrics = collections.defaultdict(lambda: Timer()) @@ -121,6 +121,17 @@ def dataloader_with_timers(): return dataloader_with_timers() + @property + def _num_batches_to_skip(self) -> int: + """Calculate the number of batches to skip based on the number of rows already processed in this epoch.""" + + global_batch_size = ( + self.benchmark_config.dataloader_config.train_batch_size + * ray.train.get_context().get_world_size() + ) + + return self._global_rows_processed_this_epoch // global_batch_size + def _train_epoch(self): """Subclasses can override the entrire `_train_epoch` method for more training logic customization.""" @@ -132,11 +143,11 @@ def _train_epoch(self): # Skip through batches if we restored to a middle of the epoch. # TODO: Compare this baseline to the data checkpointing approach once we have it. - if self._restored_train_batch_idx is not None: + if self._num_batches_to_skip: if ray.train.get_context().get_world_rank() == 0: - logger.info(f"Skipping {self._restored_train_batch_idx + 1} batches...") + logger.info(f"Skipping {self._num_batches_to_skip} batches...") - for _ in range(self._restored_train_batch_idx + 1): + for _ in range(self._num_batches_to_skip): with self._metrics["train/iter_skip_batch"].timer(): next(train_dataloader) @@ -146,18 +157,27 @@ def _train_epoch(self): self._train_step(batch) # TODO: This is slightly off if the last batch is a partial batch (if drop_last=False) - self._metrics["train/rows_processed"].add( + global_batch_size = ( self.benchmark_config.dataloader_config.train_batch_size + * ray.train.get_context().get_world_size() ) + self._metrics["train/rows_processed"].add(global_batch_size) + + self._global_rows_processed_this_epoch += global_batch_size + + if self._should_checkpoint_during_epoch(): + self._checkpoint() if self._should_validate_during_epoch(): - self._validate_and_checkpoint() + validation_metrics = self._validate() + self._checkpoint(validation_metrics) if self._should_log_metrics(): logger.info(pprint.pformat(self.get_metrics(), indent=2)) self._train_epoch_idx += 1 self._train_batch_idx = 0 + self._global_rows_processed_this_epoch = 0 def _validate_epoch(self) -> Dict[str, float]: if ray.train.get_context().get_world_rank() == 0: @@ -184,6 +204,14 @@ def _validate_epoch(self) -> Dict[str, float]: return {"validation/loss": total_loss.item() / num_rows} + def _should_checkpoint_during_epoch(self) -> bool: + """Handles the checkpoint_every_n_steps logic.""" + return ( + self.benchmark_config.checkpoint_every_n_steps > 0 + and self._train_batch_idx % self.benchmark_config.checkpoint_every_n_steps + == 0 + ) + def _should_validate_during_epoch(self) -> bool: """Handles the validate_every_n_steps logic.""" return ( @@ -200,10 +228,12 @@ def _should_log_metrics(self) -> bool: == 0 ) - def _validate_and_checkpoint(self): + def _validate(self) -> Dict[str, float]: with self._metrics["validation/epoch"].timer(): validation_metrics = self._validate_epoch() + return validation_metrics + def _checkpoint(self, metrics: Optional[Dict[str, float]] = None): with tempfile.TemporaryDirectory( dir="/mnt/local_storage" ) as temp_checkpoint_dir: @@ -212,7 +242,7 @@ def _validate_and_checkpoint(self): with self._metrics["checkpoint/report"].timer(): self._report_checkpoint( - metrics=validation_metrics, + metrics=metrics or {}, checkpoint=ray.train.Checkpoint.from_directory(temp_checkpoint_dir), ) @@ -221,7 +251,10 @@ def _load_checkpoint(self, local_dir: str): run_state = torch.load(os.path.join(local_dir, "run_state.pt")) self._train_epoch_idx = run_state["epoch"] - self._restored_train_batch_idx = run_state["batch_idx"] + self._train_batch_idx = run_state["batch_idx"] + self._global_rows_processed_this_epoch = run_state[ + "global_rows_processed_this_epoch" + ] with open(os.path.join(local_dir, "metrics.json"), "r") as f: metrics_json = json.load(f) @@ -232,7 +265,7 @@ def _load_checkpoint(self, local_dir: str): if ray.train.get_context().get_world_rank() == 0: logger.info( f"Restored to epoch={self._train_epoch_idx}, " - f"train_batch_idx={self._restored_train_batch_idx} from checkpoint: " + f"train_batch_idx={self._train_batch_idx} from checkpoint: " f"{ray.train.get_checkpoint()}" ) @@ -248,6 +281,7 @@ def _save_checkpoint(self, local_dir: str): run_state = { "epoch": self._train_epoch_idx, "batch_idx": self._train_batch_idx, + "global_rows_processed_this_epoch": self._global_rows_processed_this_epoch, } torch.save(run_state, os.path.join(local_dir, "run_state.pt")) @@ -279,7 +313,8 @@ def run(self): self._train_epoch() if not self.benchmark_config.skip_validation_at_epoch_end: - self._validate_and_checkpoint() + validation_metrics = self._validate() + self._checkpoint(validation_metrics) if ray.train.get_context().get_world_rank() == 0: logger.info(pprint.pformat(self.get_metrics(), indent=2)) @@ -304,7 +339,6 @@ def get_metrics(self, dataset_creation_time: float = 0.0) -> Dict[str, float]: # Throughput # TODO: Ray Data can provide these throughput metrics automatically. - num_workers = ray.train.get_context().get_world_size() train_time = ( metrics["train/dataset_creation_time"] + self._metrics["train/step"].get() @@ -313,11 +347,8 @@ def get_metrics(self, dataset_creation_time: float = 0.0) -> Dict[str, float]: + self._metrics["train/iter_batch"].get() ) if train_time > 0: - metrics["train/local_throughput"] = ( - self._metrics["train/rows_processed"].get() / train_time - ) metrics["train/global_throughput"] = ( - metrics["train/local_throughput"] * num_workers + self._metrics["train/rows_processed"].get() / train_time ) validation_time = ( @@ -328,11 +359,8 @@ def get_metrics(self, dataset_creation_time: float = 0.0) -> Dict[str, float]: + self._metrics["validation/iter_batch"].get() ) if validation_time > 0: - metrics["validation/local_throughput"] = ( - self._metrics["validation/rows_processed"].get() / validation_time - ) metrics["validation/global_throughput"] = ( - metrics["validation/local_throughput"] * num_workers + self._metrics["validation/rows_processed"].get() / validation_time ) # Extra time that each worker spends to restore from checkpoint, diff --git a/release/train_tests/benchmark/train_benchmark.py b/release/train_tests/benchmark/train_benchmark.py index b2650baf985b..f2378a19b32b 100644 --- a/release/train_tests/benchmark/train_benchmark.py +++ b/release/train_tests/benchmark/train_benchmark.py @@ -34,12 +34,26 @@ def train_fn_per_worker(config): runner.run() - metrics = runner.get_metrics(dataset_creation_time=config["dataset_creation_time"]) + metrics = runner.get_metrics( + dataset_creation_time=config.get("dataset_creation_time", 0) + ) if ray.train.get_context().get_world_rank() == 0: with open(METRICS_OUTPUT_PATH, "w") as f: json.dump(metrics, f) +def get_datasets_and_data_config(factory: BenchmarkFactory): + dataloader_factory = factory.get_dataloader_factory() + if isinstance(dataloader_factory, RayDataLoaderFactory): + datasets = dataloader_factory.get_ray_datasets() + data_config = dataloader_factory.get_ray_data_config() + else: + datasets = {} + data_config = None + + return datasets, data_config + + def main(): start_time = time.perf_counter() logging.basicConfig(level=logging.INFO) @@ -60,13 +74,7 @@ def main(): else: raise ValueError(f"Unknown task: {benchmark_config.task}") - dataloader_factory = factory.get_dataloader_factory() - if isinstance(dataloader_factory, RayDataLoaderFactory): - datasets = dataloader_factory.get_ray_datasets() - data_config = dataloader_factory.get_ray_data_config() - else: - datasets = {} - data_config = None + datasets, data_config = get_datasets_and_data_config(factory) dataset_creation_time = time.perf_counter() - start_time From 15d9d2debf036af22cc581cd388ae0d3d421653e Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Thu, 28 Aug 2025 09:59:05 -0700 Subject: [PATCH 331/634] [ci] enable bazel strict action env by default (#55985) so that action env vars are properly captured and sandboxed, makes caching working properly. Signed-off-by: Lonnie Liu --- .bazelrc | 24 +++++++++++++++--------- python/setup.py | 7 ------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.bazelrc b/.bazelrc index 5e4727932249..c71651133b52 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,15 +1,11 @@ # Must be first. Enables build:windows, build:linux, build:macos, build:freebsd, build:openbsd build --enable_platform_specific_config -build:linux --workspace_status_command="bash ./bazel/workspace_status.sh" - -# Provides users an option to turn on strict action env. -# TODO(aslonnie): make this default, fix the python tests. -# NOTE(edoakes): enable this by default locally by adding a .user.bazelrc file with: -# build --config=strict -# test --config=strict +build --incompatible_strict_action_env build:strict --incompatible_strict_action_env +build:linux --workspace_status_command="bash ./bazel/workspace_status.sh" + # To distinguish different incompatible environments. build --action_env=RAY_BUILD_ENV @@ -84,8 +80,6 @@ build:iwyu --output_groups=report build:windows --attempt_to_print_relative_paths # Save disk space by hardlinking cache hits instead of copying build:windows --experimental_repository_cache_hardlinks -# Clean the environment before building, to make builds more deterministic -build:windows --incompatible_strict_action_env # For colored output (seems necessary on Windows) build:windows --color=yes # For compiler colored output (seems necessary on Windows) @@ -168,6 +162,18 @@ test:ci-base --test_output=errors test:ci-base --test_verbose_timeout_warnings test:ci-base --flaky_test_attempts=3 +# Sending in PATH is required for tests to run on CI, after we enable +# --incompatible_strict_action_env, until we either convert all Python tests to +# hermetic tests -- which not only requires pinning all Python dependencies with bazel, +# but also requires building ray(test) wheel with bazel. Alternatively, we can +# also stop using bazel test to run ray's Python tests. +# +# This PATH test_env is intentionally not enabled on non-CI so that C/C++ +# tests, which are all hermetic, can build, test and cache as intended, ray +# Python developers do not really use bazel test to run tests locally, but more +# often just run tests with "pytest" directly. +test:ci-base --test_env=PATH + build:ci --color=yes build:ci --curses=no build:ci --keep_going diff --git a/python/setup.py b/python/setup.py index 8918efef130b..bae70a9a265e 100644 --- a/python/setup.py +++ b/python/setup.py @@ -635,13 +635,6 @@ def build(build_python, build_java, build_cpp): bazel_precmd_flags = [] if sys.platform == "win32": bazel_precmd_flags = ["--output_user_root=C:/tmp"] - # Using --incompatible_strict_action_env so that the build is more - # cache-able We cannot turn this on for Python tests yet, as Ray's - # Python bazel tests are not hermetic. - # - # And we put it here so that does not change behavior of - # conda-forge build. - bazel_flags.append("--incompatible_strict_action_env") bazel_targets = [] bazel_targets += ["//:gen_ray_pkg"] if build_python else [] From 2064742b3c0abf4470b1c8d456218270057acbe0 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 28 Aug 2025 09:59:54 -0700 Subject: [PATCH 332/634] [release] Read test filters from both release-test-filters and release-test-attr-regex-filters (#56039) `release-test-attr-regex-filters` is the metadata being used in the input box for Release pipeline which we need to make it backward compatible still after adding `release-test-filters` Signed-off-by: kevin --- release/ray_release/buildkite/settings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/ray_release/buildkite/settings.py b/release/ray_release/buildkite/settings.py index 07cfbd70fcf2..6e6579fe4faa 100644 --- a/release/ray_release/buildkite/settings.py +++ b/release/ray_release/buildkite/settings.py @@ -193,7 +193,9 @@ def update_settings_from_buildkite(settings: Dict): if test_name_filter: settings["test_filters"] = get_test_filters("name:" + test_name_filter) - test_filters = get_buildkite_prompt_value("release-test-filters") + test_filters = get_buildkite_prompt_value( + "release-test-filters" + ) or get_buildkite_prompt_value("release-test-attr-regex-filters") if test_filters: settings["test_filters"] = get_test_filters(test_filters) From d3597cba8fe20a223295c68bbcc4c8f45805bfb8 Mon Sep 17 00:00:00 2001 From: matthewdeng Date: Thu, 28 Aug 2025 10:26:03 -0700 Subject: [PATCH 333/634] [air] update `/python/ray/air` CODEOWNERS (#55891) Add Ray Train maintainers as the CODEOWNERS for `/python/ray/air` sub-directory. Without this change, this directory will have Ray Core assigned as the CODEOWNER due to [this line](https://github.com/matthewdeng/ray/blob/ca2d866a95385aea4fd3de1c3b9f5148f899c282/.github/CODEOWNERS#L20-L21). Signed-off-by: Matthew Deng --- .github/CODEOWNERS | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 723c50deedda..6d12a001d401 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -51,14 +51,17 @@ /rllib/ @ray-project/ray-rllib /doc/source/rllib/ @ray-project/ray-rllib @ray-project/ray-docs -# Tune +# Ray Tune /python/ray/tune/ @ray-project/ray-tune /doc/source/tune/ @ray-project/ray-tune @ray-project/ray-docs -# Train +# Ray Train /python/ray/train/ @ray-project/ray-train /doc/source/train/ @ray-project/ray-train @ray-project/ray-docs +# Ray AIR +/python/ray/air/ @ray-project/ray-train + # LLM /python/ray/llm/ @ray-project/ray-llm From 21b77f66a1dd8ed89e3efe67460d42c2dca911bf Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Thu, 28 Aug 2025 10:26:20 -0700 Subject: [PATCH 334/634] [data] Remove unification of schemas (#55926) ## Why are these changes needed? - unification of schemas is slow - revert back to pre https://github.com/ray-project/ray/pull/53454/files commit. if no unification before, no unification after. if unification before, we can leave it there or add it back. If I removed it I added a comment with `# NOTE` ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu Signed-off-by: Alexey Kudinkin Co-authored-by: Alexey Kudinkin --- python/ray/data/BUILD | 14 ++ .../_internal/arrow_ops/transform_pyarrow.py | 2 +- python/ray/data/_internal/equalize.py | 10 +- .../data/_internal/execution/legacy_compat.py | 21 +-- .../execution/operators/map_operator.py | 16 +- .../execution/streaming_executor_state.py | 11 +- .../logical/operators/from_operators.py | 13 +- .../logical/operators/input_data_operator.py | 4 +- .../logical/operators/read_operator.py | 5 +- python/ray/data/_internal/plan.py | 15 +- .../ray/data/_internal/planner/aggregate.py | 1 + .../push_based_shuffle_task_scheduler.py | 5 +- python/ray/data/_internal/planner/sort.py | 1 + python/ray/data/block.py | 15 ++ python/ray/data/context.py | 4 + python/ray/data/tests/test_deduping_schema.py | 19 ++- .../tests/test_unify_schemas_performance.py | 140 ++++++++++++++++++ 17 files changed, 244 insertions(+), 52 deletions(-) create mode 100644 python/ray/data/tests/test_unify_schemas_performance.py diff --git a/python/ray/data/BUILD b/python/ray/data/BUILD index 0383364d6ced..bbe95eb076a1 100644 --- a/python/ray/data/BUILD +++ b/python/ray/data/BUILD @@ -1239,6 +1239,20 @@ py_test( ], ) +py_test( + name = "test_unify_schemas_performance", + size = "small", + srcs = ["tests/test_unify_schemas_performance.py"], + tags = [ + "exclusive", + "team:data", + ], + deps = [ + ":conftest", + "//:ray_lib", + ], +) + py_test( name = "test_util", size = "small", diff --git a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py index 453ba23e4e08..04f8b5468f3f 100644 --- a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py +++ b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py @@ -182,7 +182,7 @@ def unify_schemas( try: if len(set(schemas)) == 1: # Early exit because unifying can be expensive - return schemas[0] + return schemas.pop() except Exception as e: # Unsure if there are cases where schemas are NOT hashable logger.warning(f"Failed to hash the schemas (for deduplication): {e}") diff --git a/python/ray/data/_internal/equalize.py b/python/ray/data/_internal/equalize.py index 92f6ff6c2afa..52561020dfab 100644 --- a/python/ray/data/_internal/equalize.py +++ b/python/ray/data/_internal/equalize.py @@ -2,8 +2,12 @@ from ray.data._internal.execution.interfaces import RefBundle from ray.data._internal.split import _calculate_blocks_rows, _split_at_indices -from ray.data._internal.util import unify_ref_bundles_schema -from ray.data.block import Block, BlockMetadata, BlockPartition +from ray.data.block import ( + Block, + BlockMetadata, + BlockPartition, + _take_first_non_empty_schema, +) from ray.types import ObjectRef @@ -41,7 +45,7 @@ def _equalize( # phase 2: based on the num rows needed for each shaved split, split the leftovers # in the shape that exactly matches the rows needed. - schema = unify_ref_bundles_schema(per_split_bundles) + schema = _take_first_non_empty_schema(bundle.schema for bundle in per_split_bundles) leftover_bundle = RefBundle(leftovers, owns_blocks=owned_by_consumer, schema=schema) leftover_splits = _split_leftovers(leftover_bundle, per_split_needed_rows) diff --git a/python/ray/data/_internal/execution/legacy_compat.py b/python/ray/data/_internal/execution/legacy_compat.py index af651c797352..fefb1592c736 100644 --- a/python/ray/data/_internal/execution/legacy_compat.py +++ b/python/ray/data/_internal/execution/legacy_compat.py @@ -16,10 +16,11 @@ from ray.data._internal.logical.util import record_operators_usage from ray.data._internal.plan import ExecutionPlan from ray.data._internal.stats import DatasetStats -from ray.data._internal.util import ( - unify_schemas_with_validation, +from ray.data.block import ( + BlockMetadata, + BlockMetadataWithSchema, + _take_first_non_empty_schema, ) -from ray.data.block import BlockMetadata, BlockMetadataWithSchema # Warn about tasks larger than this. TASK_SIZE_WARN_THRESHOLD_BYTES = 100000 @@ -171,18 +172,18 @@ def _get_initial_stats_from_plan(plan: ExecutionPlan) -> DatasetStats: def _bundles_to_block_list(bundles: Iterator[RefBundle]) -> BlockList: blocks, metadata = [], [] owns_blocks = True - schemas = [] + bundle_list = list(bundles) + schema = _take_first_non_empty_schema( + ref_bundle.schema for ref_bundle in bundle_list + ) - for ref_bundle in bundles: + for ref_bundle in bundle_list: if not ref_bundle.owns_blocks: owns_blocks = False blocks.extend(ref_bundle.block_refs) metadata.extend(ref_bundle.metadata) - schemas.append(ref_bundle.schema) - unified_schema = unify_schemas_with_validation(schemas) - return BlockList( - blocks, metadata, owned_by_consumer=owns_blocks, schema=unified_schema - ) + + return BlockList(blocks, metadata, owned_by_consumer=owns_blocks, schema=schema) def _set_stats_uuid_recursive(stats: DatasetStats, dataset_uuid: str) -> None: diff --git a/python/ray/data/_internal/execution/operators/map_operator.py b/python/ray/data/_internal/execution/operators/map_operator.py index 3a9f9bd1bced..3efa53e28bb1 100644 --- a/python/ray/data/_internal/execution/operators/map_operator.py +++ b/python/ray/data/_internal/execution/operators/map_operator.py @@ -48,13 +48,14 @@ ) from ray.data._internal.execution.util import memory_string from ray.data._internal.stats import StatsDict -from ray.data._internal.util import MemoryProfiler, unify_ref_bundles_schema +from ray.data._internal.util import MemoryProfiler from ray.data.block import ( Block, BlockAccessor, BlockExecStats, BlockMetadataWithSchema, BlockStats, + _take_first_non_empty_schema, to_stats, ) from ray.data.context import DataContext @@ -541,8 +542,6 @@ def _map_task( A generator of blocks, followed by the list of BlockMetadata for the blocks as the last generator return. """ - from ray.data.block import BlockMetadataWithSchema - logger.debug( "Executing map task of operator %s with task index %d", ctx.op_name, @@ -662,14 +661,13 @@ def _get_bundle_size(bundle: RefBundle): def _merge_ref_bundles(*bundles: RefBundle) -> RefBundle: """Merge N ref bundles into a single bundle of multiple blocks.""" # Check that at least one bundle is non-null. - assert any(bundle is not None for bundle in bundles) + bundles = [bundle for bundle in bundles if bundle is not None] + assert len(bundles) > 0 blocks = list( - itertools.chain( - block for bundle in bundles if bundle is not None for block in bundle.blocks - ) + itertools.chain(block for bundle in bundles for block in bundle.blocks) ) - owns_blocks = all(bundle.owns_blocks for bundle in bundles if bundle is not None) - schema = unify_ref_bundles_schema(bundles) + owns_blocks = all(bundle.owns_blocks for bundle in bundles) + schema = _take_first_non_empty_schema(bundle.schema for bundle in bundles) return RefBundle(blocks, owns_blocks=owns_blocks, schema=schema) diff --git a/python/ray/data/_internal/execution/streaming_executor_state.py b/python/ray/data/_internal/execution/streaming_executor_state.py index 586e6817e81f..bd4634900651 100644 --- a/python/ray/data/_internal/execution/streaming_executor_state.py +++ b/python/ray/data/_internal/execution/streaming_executor_state.py @@ -284,7 +284,10 @@ def add_output(self, ref: RefBundle) -> None: """Move a bundle produced by the operator to its outqueue.""" ref, diverged = dedupe_schemas_with_validation( - self._schema, ref, warn=not self._warned_on_schema_divergence + self._schema, + ref, + warn=not self._warned_on_schema_divergence, + enforce_schemas=self.op.data_context.enforce_schemas, ) self._schema = ref.schema self._warned_on_schema_divergence |= diverged @@ -757,7 +760,7 @@ def dedupe_schemas_with_validation( old_schema: Optional["Schema"], bundle: "RefBundle", warn: bool = True, - allow_divergent: bool = False, + enforce_schemas: bool = False, ) -> Tuple["RefBundle", bool]: """Unify/Dedupe two schemas, warning if warn=True @@ -766,7 +769,7 @@ def dedupe_schemas_with_validation( the new schema will be used as the old schema. bundle: The new `RefBundle` to unify with the old schema. warn: Raise a warning if the schemas diverge. - allow_divergent: If `True`, allow the schemas to diverge and return unified schema. + enforce_schemas: If `True`, allow the schemas to diverge and return unified schema. If `False`, but keep the old schema. Returns: @@ -793,7 +796,7 @@ def dedupe_schemas_with_validation( f"than the previous one. Previous schema: {old_schema}, " f"new schema: {bundle.schema}. This may lead to unexpected behavior." ) - if allow_divergent: + if enforce_schemas: old_schema = unify_schemas_with_validation([old_schema, bundle.schema]) return ( diff --git a/python/ray/data/_internal/logical/operators/from_operators.py b/python/ray/data/_internal/logical/operators/from_operators.py index 05d0d668d3a6..0f6ec1a7a2d2 100644 --- a/python/ray/data/_internal/logical/operators/from_operators.py +++ b/python/ray/data/_internal/logical/operators/from_operators.py @@ -4,8 +4,12 @@ from ray.data._internal.execution.interfaces import RefBundle from ray.data._internal.logical.interfaces import LogicalOperator, SourceOperator -from ray.data._internal.util import unify_block_metadata_schema -from ray.data.block import Block, BlockMetadata, BlockMetadataWithSchema +from ray.data._internal.util import unify_ref_bundles_schema +from ray.data.block import ( + Block, + BlockMetadata, + BlockMetadataWithSchema, +) from ray.types import ObjectRef if TYPE_CHECKING: @@ -28,12 +32,11 @@ def __init__( len(input_metadata), ) # `owns_blocks` is False because this op may be shared by multiple Datasets. - self._schema = unify_block_metadata_schema(input_metadata) self._input_data = [ RefBundle( [(input_blocks[i], input_metadata[i])], owns_blocks=False, - schema=self._schema, + schema=input_metadata[i].schema, ) for i in range(len(input_blocks)) ] @@ -71,7 +74,7 @@ def infer_metadata(self) -> BlockMetadata: return self._cached_output_metadata def infer_schema(self): - return self._schema + return unify_ref_bundles_schema(self._input_data) def is_lineage_serializable(self) -> bool: # This operator isn't serializable because it contains ObjectRefs. diff --git a/python/ray/data/_internal/logical/operators/input_data_operator.py b/python/ray/data/_internal/logical/operators/input_data_operator.py index f779d582706b..373a12e84961 100644 --- a/python/ray/data/_internal/logical/operators/input_data_operator.py +++ b/python/ray/data/_internal/logical/operators/input_data_operator.py @@ -3,7 +3,7 @@ from ray.data._internal.execution.interfaces import RefBundle from ray.data._internal.logical.interfaces import LogicalOperator, SourceOperator -from ray.data._internal.util import unify_ref_bundles_schema +from ray.data._internal.util import unify_schemas_with_validation from ray.data.block import BlockMetadata @@ -49,7 +49,7 @@ def _size_bytes(self): return None def infer_schema(self): - return unify_ref_bundles_schema(self.input_data) + return unify_schemas_with_validation([data.schema for data in self.input_data]) def is_lineage_serializable(self) -> bool: # This operator isn't serializable because it contains ObjectRefs. diff --git a/python/ray/data/_internal/logical/operators/read_operator.py b/python/ray/data/_internal/logical/operators/read_operator.py index b3ba9c42b498..aef39c554d23 100644 --- a/python/ray/data/_internal/logical/operators/read_operator.py +++ b/python/ray/data/_internal/logical/operators/read_operator.py @@ -3,7 +3,10 @@ from ray.data._internal.logical.interfaces import SourceOperator from ray.data._internal.logical.operators.map_operator import AbstractMap -from ray.data.block import BlockMetadata, BlockMetadataWithSchema +from ray.data.block import ( + BlockMetadata, + BlockMetadataWithSchema, +) from ray.data.datasource.datasource import Datasource, Reader diff --git a/python/ray/data/_internal/plan.py b/python/ray/data/_internal/plan.py index a471949d89e3..ba4d6df6727f 100644 --- a/python/ray/data/_internal/plan.py +++ b/python/ray/data/_internal/plan.py @@ -14,8 +14,7 @@ from ray.data._internal.logical.interfaces.operator import Operator from ray.data._internal.logical.operators.read_operator import Read from ray.data._internal.stats import DatasetStats -from ray.data._internal.util import unify_ref_bundles_schema -from ray.data.block import BlockMetadataWithSchema +from ray.data.block import BlockMetadataWithSchema, _take_first_non_empty_schema from ray.data.context import DataContext from ray.data.exceptions import omit_traceback_stdout from ray.util.debug import log_once @@ -401,10 +400,9 @@ def schema( iter_ref_bundles, _, executor = self.execute_to_iterator() # Make sure executor is fully shutdown upon exiting with executor: - for bundle in iter_ref_bundles: - if bundle.schema is not None: - schema = bundle.schema - break + schema = _take_first_non_empty_schema( + bundle.schema for bundle in iter_ref_bundles + ) self.cache_schema(schema) return self._schema @@ -516,9 +514,10 @@ def execute( # `List[RefBundle]` instead of `RefBundle`. Among other reasons, it'd # allow us to remove the unwrapping logic below. output_bundles = self._logical_plan.dag.output_data() - schema = self._logical_plan.dag.infer_schema() owns_blocks = all(bundle.owns_blocks for bundle in output_bundles) - schema = unify_ref_bundles_schema(output_bundles) + schema = _take_first_non_empty_schema( + bundle.schema for bundle in output_bundles + ) bundle = RefBundle( [ (block, metadata) diff --git a/python/ray/data/_internal/planner/aggregate.py b/python/ray/data/_internal/planner/aggregate.py index 75c42657391f..199382e226f0 100644 --- a/python/ray/data/_internal/planner/aggregate.py +++ b/python/ray/data/_internal/planner/aggregate.py @@ -52,6 +52,7 @@ def fn( metadata.extend(ref_bundle.metadata) if len(blocks) == 0: return (blocks, {}) + unified_schema = unify_ref_bundles_schema(refs) for agg_fn in aggs: agg_fn._validate(unified_schema) diff --git a/python/ray/data/_internal/planner/exchange/push_based_shuffle_task_scheduler.py b/python/ray/data/_internal/planner/exchange/push_based_shuffle_task_scheduler.py index eac3c1e88e30..901eb69bd969 100644 --- a/python/ray/data/_internal/planner/exchange/push_based_shuffle_task_scheduler.py +++ b/python/ray/data/_internal/planner/exchange/push_based_shuffle_task_scheduler.py @@ -14,7 +14,6 @@ from ray.data._internal.stats import StatsDict from ray.data._internal.util import ( convert_bytes_to_human_readable_str, - unify_schemas_with_validation, unzip, ) from ray.data.block import ( @@ -23,6 +22,7 @@ BlockExecStats, BlockMetadata, BlockMetadataWithSchema, + _take_first_non_empty_schema, to_stats, ) from ray.data.context import DataContext @@ -743,13 +743,14 @@ def _merge( del block schemas.append(meta_with_schema.schema) + schema = _take_first_non_empty_schema(iter(schemas)) + meta = BlockMetadata( num_rows=num_rows, size_bytes=size_bytes, input_files=None, exec_stats=stats.build(), ) - schema = unify_schemas_with_validation(schemas) meta_with_schema = BlockMetadataWithSchema(metadata=meta, schema=schema) yield meta_with_schema diff --git a/python/ray/data/_internal/planner/sort.py b/python/ray/data/_internal/planner/sort.py index ece9a76e01ce..852154c66c36 100644 --- a/python/ray/data/_internal/planner/sort.py +++ b/python/ray/data/_internal/planner/sort.py @@ -36,6 +36,7 @@ def fn( blocks.extend(ref_bundle.block_refs) if len(blocks) == 0: return (blocks, {}) + sort_key.validate_schema(unify_ref_bundles_schema(refs)) num_mappers = len(blocks) diff --git a/python/ray/data/block.py b/python/ray/data/block.py index 64b09334f4f2..d0edaf4a8574 100644 --- a/python/ray/data/block.py +++ b/python/ray/data/block.py @@ -114,6 +114,21 @@ def _is_empty_schema(schema: Optional[Schema]) -> bool: ) +def _take_first_non_empty_schema(schemas: Iterator["Schema"]) -> Optional["Schema"]: + """Return the first non-empty schema from an iterator of schemas. + + Args: + schemas: Iterator of schemas to check. + + Returns: + The first non-empty schema, or None if all schemas are empty. + """ + for schema in schemas: + if not _is_empty_schema(schema): + return schema + return None + + def _apply_batch_format(given_batch_format: Optional[str]) -> str: if given_batch_format == "default": given_batch_format = DEFAULT_BATCH_FORMAT diff --git a/python/ray/data/context.py b/python/ray/data/context.py index 2bbca500edcf..f39c5cd700c1 100644 --- a/python/ray/data/context.py +++ b/python/ray/data/context.py @@ -140,6 +140,8 @@ class ShuffleStrategy(str, enum.Enum): "RAY_DATA_ENABLE_PROGRESS_BAR_NAME_TRUNCATION", True ) +DEFAULT_ENFORCE_SCHEMAS = env_bool("RAY_DATA_ALLOW_ENFORCE_SCHEMAS", False) + DEFAULT_ENABLE_GET_OBJECT_LOCATIONS_FOR_METRICS = False @@ -537,6 +539,8 @@ class DataContext: downstream_capacity_backpressure_ratio: float = None downstream_capacity_backpressure_max_queued_bundles: int = None + enforce_schemas: bool = DEFAULT_ENFORCE_SCHEMAS + def __post_init__(self): # The additonal ray remote args that should be added to # the task-pool-based data tasks. diff --git a/python/ray/data/tests/test_deduping_schema.py b/python/ray/data/tests/test_deduping_schema.py index 651efaf86223..2574c7195161 100644 --- a/python/ray/data/tests/test_deduping_schema.py +++ b/python/ray/data/tests/test_deduping_schema.py @@ -7,7 +7,8 @@ from ray.data._internal.execution.streaming_executor_state import ( dedupe_schemas_with_validation, ) -from ray.data.block import Schema +from ray.data._internal.pandas_block import PandasBlockSchema +from ray.data.block import Schema, _is_empty_schema @pytest.mark.parametrize( @@ -15,6 +16,8 @@ [ pa.schema([pa.field("uuid", pa.string())]), # NOTE: diff from old_schema pa.schema([]), # Empty Schema + PandasBlockSchema(names=["col1"], types=[int]), + PandasBlockSchema(names=[], types=[]), None, # Null Schema ], ) @@ -23,6 +26,8 @@ [ pa.schema([pa.field("id", pa.int64())]), pa.schema([]), # Empty Schema + PandasBlockSchema(names=["col2"], types=[int]), + PandasBlockSchema(names=[], types=[]), None, # Null Schema ], ) @@ -33,10 +38,10 @@ def test_dedupe_schema_handle_empty( incoming_bundle = RefBundle([], owns_blocks=False, schema=incoming_schema) out_bundle, diverged = dedupe_schemas_with_validation( - old_schema, incoming_bundle, allow_divergent=False, warn=False + old_schema, incoming_bundle, enforce_schemas=False, warn=False ) - if old_schema is None or len(old_schema) == 0: + if _is_empty_schema(old_schema): # old_schema is invalid assert not diverged, (old_schema, incoming_schema) assert out_bundle.schema == incoming_schema, (old_schema, incoming_schema) @@ -47,25 +52,25 @@ def test_dedupe_schema_handle_empty( assert old_schema == out_bundle.schema, (old_schema, incoming_schema) -@pytest.mark.parametrize("allow_divergent", [False, True]) +@pytest.mark.parametrize("enforce_schemas", [False, True]) @pytest.mark.parametrize( "incoming_schema", [pa.schema([pa.field("uuid", pa.string())])] ) @pytest.mark.parametrize("old_schema", [pa.schema([pa.field("id", pa.int64())])]) def test_dedupe_schema_divergence( - allow_divergent: bool, + enforce_schemas: bool, old_schema: Optional["Schema"], incoming_schema: Optional["Schema"], ): incoming_bundle = RefBundle([], owns_blocks=False, schema=incoming_schema) out_bundle, diverged = dedupe_schemas_with_validation( - old_schema, incoming_bundle, allow_divergent=allow_divergent, warn=False + old_schema, incoming_bundle, enforce_schemas=enforce_schemas, warn=False ) assert diverged - if allow_divergent: + if enforce_schemas: assert out_bundle.schema == pa.schema(list(old_schema) + list(incoming_schema)) else: assert out_bundle.schema == old_schema diff --git a/python/ray/data/tests/test_unify_schemas_performance.py b/python/ray/data/tests/test_unify_schemas_performance.py new file mode 100644 index 000000000000..704f15de5167 --- /dev/null +++ b/python/ray/data/tests/test_unify_schemas_performance.py @@ -0,0 +1,140 @@ +import pyarrow as pa +import pytest + +from ray.data._internal.arrow_ops.transform_pyarrow import ( + unify_schemas, +) +from ray.data.extensions import ( + ArrowPythonObjectType, + ArrowTensorType, + ArrowVariableShapedTensorType, +) + + +# Schema factory functions - just return schemas +def _create_simple_schema(num_columns): + return pa.schema([(f"col_{i}", pa.int64()) for i in range(num_columns)]) + + +def _create_tensor_fixed_schema(num_columns): + return pa.schema( + [ + (f"tensor_{i}", ArrowTensorType((2, 2), pa.float32())) + for i in range(num_columns) + ] + ) + + +def _create_tensor_variable_schema(num_columns): + return pa.schema( + [ + (f"tensor_{i}", ArrowVariableShapedTensorType(pa.float32(), 2)) + for i in range(num_columns) + ] + ) + + +def _create_object_schema(num_columns): + return pa.schema( + [(f"obj_{i}", ArrowPythonObjectType()) for i in range(num_columns)] + ) + + +def _create_nested_struct_schema(num_columns): + fields = [] + for i in range(num_columns): + inner_struct = pa.struct( + [("x", pa.int32()), ("y", pa.string()), ("z", pa.float64())] + ) + fields.append((f"struct_{i}", inner_struct)) + return pa.schema(fields) + + +def _create_deep_nested_schema(num_columns): + fields = [] + for i in range(num_columns): + level4 = pa.struct([("data", pa.int32()), ("meta", pa.string())]) + level3 = pa.struct([("level4", level4), ("id3", pa.int64())]) + level2 = pa.struct([("level3", level3), ("id2", pa.int64())]) + level1 = pa.struct([("level2", level2), ("id1", pa.int64())]) + fields.append((f"deep_{i}", level1)) + return pa.schema(fields) + + +def _create_mixed_complex_schema(num_columns): + fields = [] + for i in range(num_columns): + field_type = i % 5 + if field_type == 0: + fields.append((f"col_{i}", pa.int64())) + elif field_type == 1: + fields.append((f"col_{i}", ArrowTensorType((3, 3), pa.int32()))) + elif field_type == 2: + fields.append((f"col_{i}", ArrowPythonObjectType())) + elif field_type == 3: + inner_struct = pa.struct([("a", pa.int32()), ("b", pa.string())]) + fields.append((f"col_{i}", inner_struct)) + else: + fields.append((f"col_{i}", pa.list_(pa.float64()))) + return pa.schema(fields) + + +@pytest.mark.parametrize("num_schemas", [10, 100]) +@pytest.mark.parametrize("num_columns", [10, 100, 1000, 5000]) +@pytest.mark.parametrize( + "schema_factory,expected_time_per_schema_per_column", + [ + (_create_simple_schema, 0.00001), + (_create_tensor_fixed_schema, 0.00005), + (_create_tensor_variable_schema, 0.00005), + (_create_object_schema, 0.00005), + (_create_nested_struct_schema, 0.0001), + (_create_deep_nested_schema, 0.0002), + (_create_mixed_complex_schema, 0.0002), + ], +) +def test_unify_schemas_equivalent_performance( + num_schemas, num_columns, schema_factory, expected_time_per_schema_per_column +): + """Stress test for unify_schemas when ALL schemas are equivalent (identical). + + This tests the fast path where all schemas are the same and should be optimized + to return quickly without expensive comparisons. + """ + import time + + # Create the base schema + base_schema = schema_factory(num_columns) + + # Create list of identical schemas + schemas = [base_schema] * num_schemas + + # Time the unification + start_time = time.time() + unified = unify_schemas(schemas) + elapsed_time = time.time() - start_time + + # Verify the result is correct (should be identical to base schema) + assert unified == base_schema + + # Performance assertions with scaling based on complexity + scale_factor = num_schemas * num_columns + max_allowed_time = expected_time_per_schema_per_column * scale_factor + buffer_factor = 2 + assert elapsed_time < buffer_factor * max_allowed_time, ( + f"unify_schemas took {elapsed_time:.4f}s for {num_schemas} identical " + f"{schema_factory.__name__} schemas with {num_columns} columns, " + f"should be < {max_allowed_time:.4f}s" + ) + + # Print timing info for large cases + if num_schemas >= 1000 or num_columns >= 100: + print( + f"\n{schema_factory.__name__}: {num_schemas} schemas x {num_columns} cols = {elapsed_time:.4f}s" + ) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) From d44c59913ca1fbf83bb9afffb3a13698359c0fdf Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Thu, 28 Aug 2025 12:39:46 -0500 Subject: [PATCH 335/634] [core] Remove `core_worker_server.h` from `core_worker_client` target (#56045) Signed-off-by: Edward Oakes --- src/ray/rpc/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ray/rpc/BUILD.bazel b/src/ray/rpc/BUILD.bazel index 70d7a6cc97d2..a1bf901ab98f 100644 --- a/src/ray/rpc/BUILD.bazel +++ b/src/ray/rpc/BUILD.bazel @@ -241,7 +241,6 @@ ray_cc_library( hdrs = [ "worker/core_worker_client.h", "worker/core_worker_client_pool.h", - "worker/core_worker_server.h", ], deps = [ "//src/ray/common:id", From d97e9870822e3d8f48232312b5a2c06102948089 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Thu, 28 Aug 2025 10:41:58 -0700 Subject: [PATCH 336/634] [serve] skip running soft-fail tests on premerge (#56034) they never blocks merging practically, and when they are flaky, their retries just make PR merging take longer time without any meaningful blocking. Signed-off-by: Lonnie Liu --- .buildkite/serve.rayci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/serve.rayci.yml b/.buildkite/serve.rayci.yml index 6b45f68f4430..5a2f5a4c00f6 100644 --- a/.buildkite/serve.rayci.yml +++ b/.buildkite/serve.rayci.yml @@ -54,6 +54,7 @@ steps: tags: - serve - python + - skip-on-premerge instance_type: large soft_fail: true commands: @@ -68,6 +69,7 @@ steps: tags: - serve - python + - skip-on-premerge instance_type: large soft_fail: true commands: From d04c066073c2b8c3eced033155baa9c667dd11d9 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Thu, 28 Aug 2025 12:44:30 -0500 Subject: [PATCH 337/634] [core] Split `gcs_actor_scheduler` out from `gcs_server_lib` (#56002) Same treatment as: https://github.com/ray-project/ray/pull/55955 --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 32 +++++++++++++++++-- src/ray/gcs/gcs_server/gcs_actor_scheduler.cc | 2 -- src/ray/gcs/gcs_server/tests/BUILD.bazel | 11 +++---- .../tests/gcs_actor_scheduler_mock_test.cc | 2 +- .../tests/gcs_actor_scheduler_test.cc | 20 +++++------- 5 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 5ce6e18de9e6..3a7b5dd42443 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -337,24 +337,52 @@ ray_cc_library( ], ) +ray_cc_library( + name = "gcs_actor_scheduler", + srcs = [ + "gcs_actor_scheduler.cc", + ], + hdrs = [ + "gcs_actor_scheduler.h", + ], + deps = [ + ":gcs_actor", + ":gcs_node_manager", + ":gcs_table_storage", + "//src/ray/common:asio", + "//src/ray/common:id", + "//src/ray/common:ray_config", + "//src/ray/common:task_common", + "//src/ray/protobuf:gcs_service_cc_proto", + "//src/ray/raylet/scheduling:cluster_task_manager", + "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/rpc:core_worker_client", + "//src/ray/rpc:node_manager_client", + "//src/ray/util:logging", + "//src/ray/util:time", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_googletest//:gtest", + ], +) + ray_cc_library( name = "gcs_server_lib", srcs = [ "gcs_actor_manager.cc", - "gcs_actor_scheduler.cc", "gcs_autoscaler_state_manager.cc", "gcs_placement_group_mgr.cc", "gcs_server.cc", ], hdrs = [ "gcs_actor_manager.h", - "gcs_actor_scheduler.h", "gcs_autoscaler_state_manager.h", "gcs_placement_group_mgr.h", "gcs_server.h", ], deps = [ ":gcs_actor", + ":gcs_actor_scheduler", ":gcs_function_manager", ":gcs_health_check_manager", ":gcs_init_data", diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc index a970061bd243..9868f13cf08c 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc @@ -20,10 +20,8 @@ #include #include "ray/common/asio/asio_util.h" -#include "ray/common/asio/instrumented_io_context.h" #include "ray/common/ray_config.h" #include "ray/util/time.h" -#include "src/ray/protobuf/node_manager.pb.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index ced5b241df0e..0235d835391f 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -205,12 +205,10 @@ ray_cc_test( "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/fakes/ray/rpc/worker:fake_core_worker_client", "//src/mock/ray/pubsub:mock_publisher", - "//src/ray/gcs/gcs_server:gcs_function_manager", - "//src/ray/gcs/gcs_server:gcs_node_manager", + "//src/ray/common:test_util", + "//src/ray/gcs/gcs_server:gcs_actor", + "//src/ray/gcs/gcs_server:gcs_actor_scheduler", "//src/ray/gcs/gcs_server:gcs_resource_manager", - "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/gcs_server:gcs_table_storage", - "//src/ray/gcs/store_client", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:counter_map", @@ -228,7 +226,8 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:test_util", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_actor", + "//src/ray/gcs/gcs_server:gcs_actor_scheduler", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc index 8d04a3b18465..c5650fcf9671 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc @@ -23,7 +23,7 @@ #include "mock/ray/raylet_client/raylet_client.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/test_util.h" -#include "ray/gcs/gcs_server/gcs_actor_manager.h" +#include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_actor_scheduler.h" #include "ray/util/counter_map.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc index 9be76e0dd12b..0a6fd64244f4 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc @@ -26,10 +26,11 @@ #include "fakes/ray/rpc/worker/core_worker_client.h" #include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/asio_util.h" -#include "ray/gcs/gcs_server/gcs_actor_manager.h" +#include "ray/common/test_util.h" +#include "ray/gcs/gcs_server/gcs_actor.h" +#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/store_client/store_client.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/util/counter_map.h" @@ -41,11 +42,6 @@ class MockedGcsActorScheduler : public gcs::GcsActorScheduler { public: using gcs::GcsActorScheduler::GcsActorScheduler; - void TryLeaseWorkerFromNodeAgain(std::shared_ptr actor, - std::shared_ptr node) { - DoRetryLeasingWorkerFromNode(std::move(actor), std::move(node)); - } - protected: void RetryLeasingWorkerFromNode(std::shared_ptr actor, std::shared_ptr node) override { @@ -69,7 +65,7 @@ class MockedGcsActorScheduler : public gcs::GcsActorScheduler { class FakeGcsActorTable : public gcs::GcsActorTable { public: // The store_client and io_context args are NOT used. - explicit FakeGcsActorTable(std::shared_ptr store_client) + explicit FakeGcsActorTable(std::shared_ptr store_client) : GcsActorTable(store_client) {} Status Put(const ActorID &key, @@ -81,7 +77,7 @@ class FakeGcsActorTable : public gcs::GcsActorTable { } private: - std::shared_ptr store_client_ = + std::shared_ptr store_client_ = std::make_shared(); }; @@ -208,7 +204,7 @@ class GcsActorSchedulerTest : public ::testing::Test { protected: std::unique_ptr io_context_; - std::shared_ptr store_client_; + std::shared_ptr store_client_; std::shared_ptr gcs_actor_table_; std::shared_ptr raylet_client_; std::shared_ptr worker_client_; @@ -702,7 +698,7 @@ TEST_F(GcsActorSchedulerTest, TestReleaseUnusedActorWorkers) { // When `GcsActorScheduler` receives the `ReleaseUnusedActorWorkers` reply, it will send // out the `RequestWorkerLease` request. ASSERT_TRUE(raylet_client_->ReplyReleaseUnusedActorWorkers()); - gcs_actor_scheduler_->TryLeaseWorkerFromNodeAgain(actor, node); + gcs_actor_scheduler_->DoRetryLeasingWorkerFromNode(actor, node); ASSERT_EQ(raylet_client_->num_workers_requested, 1); } @@ -1257,7 +1253,7 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestReleaseUnusedActorWorkersByGc // When `GcsActorScheduler` receives the `ReleaseUnusedActorWorkers` reply, it will send // out the `RequestWorkerLease` request. ASSERT_TRUE(raylet_client_->ReplyReleaseUnusedActorWorkers()); - gcs_actor_scheduler_->TryLeaseWorkerFromNodeAgain(actor, node); + gcs_actor_scheduler_->DoRetryLeasingWorkerFromNode(actor, node); ASSERT_EQ(raylet_client_->num_workers_requested, 1); } From 6d19b2077483e2ae52e4f2cf4ae9cfb4a0dc2bfa Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Thu, 28 Aug 2025 12:44:46 -0500 Subject: [PATCH 338/634] [core] Remove unused RPC macros (#56052) Signed-off-by: Edward Oakes --- src/ray/rpc/gcs/gcs_rpc_server.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index 4175e6271bf3..dbbd1ddbf3a2 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -112,16 +112,6 @@ namespace rpc { #define ACTOR_INFO_SERVICE_RPC_HANDLER(HANDLER, MAX_ACTIVE_RPCS) \ RPC_SERVICE_HANDLER(ActorInfoGcsService, HANDLER, MAX_ACTIVE_RPCS) -#define MONITOR_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(MonitorGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - -#define OBJECT_INFO_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(ObjectInfoGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - #define PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(HANDLER) \ RPC_SERVICE_HANDLER(PlacementGroupInfoGcsService, \ HANDLER, \ From e4f070f47665331498045e244cf18a2e81c16515 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Thu, 28 Aug 2025 11:41:38 -0700 Subject: [PATCH 339/634] [Core] [Windows] Ray cluster commands (up, attach, status, etc) updates to work on Windows (#54982) Signed-off-by: Mark Rossetti Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../ray/autoscaler/_private/command_runner.py | 122 ++++++++++++------ python/ray/autoscaler/_private/commands.py | 19 ++- python/ray/autoscaler/_private/util.py | 7 + 3 files changed, 104 insertions(+), 44 deletions(-) diff --git a/python/ray/autoscaler/_private/command_runner.py b/python/ray/autoscaler/_private/command_runner.py index 44ca7369efe3..55ffd5d9c838 100644 --- a/python/ray/autoscaler/_private/command_runner.py +++ b/python/ray/autoscaler/_private/command_runner.py @@ -133,13 +133,18 @@ def __init__(self, ssh_key, control_path=None, **kwargs): "ServerAliveCountMax": 3, } if control_path: - self.arg_dict.update( - { - "ControlMaster": "auto", - "ControlPath": "{}/%C".format(control_path), - "ControlPersist": "10s", - } - ) + if sys.platform == "win32": + # Don't set any control path options on Windows + pass + else: + self.arg_dict.update( + { + "ControlMaster": "auto", + "ControlPath": "{}/%C".format(control_path), + "ControlPersist": "10s", + } + ) + self.arg_dict.update(kwargs) def to_ssh_options_list(self, *, timeout=60): @@ -170,9 +175,13 @@ def __init__( ssh_control_hash = hashlib.sha1(cluster_name.encode()).hexdigest() ssh_user_hash = hashlib.sha1(getuser().encode()).hexdigest() - ssh_control_path = "/tmp/ray_ssh_{}/{}".format( - ssh_user_hash[:HASH_MAX_LENGTH], ssh_control_hash[:HASH_MAX_LENGTH] - ) + if sys.platform == "win32": + # Disable SSH control paths on Windows - currently using it cause socket errors + ssh_control_path = None + else: + ssh_control_path = "/tmp/ray_ssh_{}/{}".format( + ssh_user_hash[:HASH_MAX_LENGTH], ssh_control_hash[:HASH_MAX_LENGTH] + ) self.cluster_name = cluster_name self.log_prefix = log_prefix @@ -238,10 +247,11 @@ def _set_ssh_ip_if_required(self): # This should run before any SSH commands and therefore ensure that # the ControlPath directory exists, allowing SSH to maintain # persistent sessions later on. - try: - os.makedirs(self.ssh_control_path, mode=0o700, exist_ok=True) - except OSError as e: - cli_logger.warning("{}", str(e)) # todo: msg + if self.ssh_control_path is not None: + try: + os.makedirs(self.ssh_control_path, mode=0o700, exist_ok=True) + except OSError as e: + cli_logger.warning("{}", str(e)) # todo: msg def _run_helper( self, @@ -406,32 +416,48 @@ def run_rsync_up(self, source, target, options=None): self._set_ssh_ip_if_required() options = options or {} - command = ["rsync"] - command += [ - "--rsh", - subprocess.list2cmdline( - ["ssh"] + self.ssh_options.to_ssh_options_list(timeout=120) - ), - ] - command += ["-avz"] - command += self._create_rsync_filter_args(options=options) - command += [source, "{}@{}:{}".format(self.ssh_user, self.ssh_ip, target)] + # on windows use scp -r instead of rsync + if sys.platform == "win32": + # Use scp as fallback for Windows + command = ["scp", "-r"] + command += self.ssh_options.to_ssh_options_list(timeout=120) + command += [source, "{}@{}:{}".format(self.ssh_user, self.ssh_ip, target)] + else: + command = ["rsync"] + command += [ + "--rsh", + subprocess.list2cmdline( + ["ssh"] + self.ssh_options.to_ssh_options_list(timeout=120) + ), + ] + command += ["-avz"] + command += self._create_rsync_filter_args(options=options) + command += [source, "{}@{}:{}".format(self.ssh_user, self.ssh_ip, target)] + cli_logger.verbose("Running `{}`", cf.bold(" ".join(command))) self._run_helper(command, silent=is_rsync_silent()) def run_rsync_down(self, source, target, options=None): self._set_ssh_ip_if_required() - command = ["rsync"] - command += [ - "--rsh", - subprocess.list2cmdline( - ["ssh"] + self.ssh_options.to_ssh_options_list(timeout=120) - ), - ] - command += ["-avz"] - command += self._create_rsync_filter_args(options=options) - command += ["{}@{}:{}".format(self.ssh_user, self.ssh_ip, source), target] + # on Windows use scp -r instead of rsync + if sys.platform == "win32": + # Use scp as fallback for Windows + command = ["scp", "-r"] + command += self.ssh_options.to_ssh_options_list(timeout=120) + command += ["{}@{}:{}".format(self.ssh_user, self.ssh_ip, source), target] + else: + command = ["rsync"] + command += [ + "--rsh", + subprocess.list2cmdline( + ["ssh"] + self.ssh_options.to_ssh_options_list(timeout=120) + ), + ] + command += ["-avz"] + command += self._create_rsync_filter_args(options=options) + command += ["{}@{}:{}".format(self.ssh_user, self.ssh_ip, source), target] + cli_logger.verbose("Running `{}`", cf.bold(" ".join(command))) self._run_helper(command, silent=is_rsync_silent()) @@ -510,8 +536,13 @@ def run_rsync_up(self, source, target, options=None): self._get_docker_host_mount_location(self.ssh_command_runner.cluster_name), target.lstrip("/"), ) - host_mount_location = os.path.dirname(host_destination.rstrip("/")) + if sys.platform == "win32": + # fix paths if running on Windows + source = source.replace("\\", "/") + host_mount_location = host_mount_location.replace("\\", "/") + host_destination = host_destination.replace("\\", "/") + self.ssh_command_runner.run( f"mkdir -p {host_mount_location} && chown -R " f"{self.ssh_command_runner.ssh_user} {host_mount_location}", @@ -558,9 +589,11 @@ def run_rsync_down(self, source, target, options=None): source.lstrip("/"), ) host_mount_location = os.path.dirname(host_source.rstrip("/")) + # Convert Windows paths to Unix-style for remote commands + host_mount_location_unix = host_mount_location.replace("\\", "/") self.ssh_command_runner.run( - f"mkdir -p {host_mount_location} && chown -R " - f"{self.ssh_command_runner.ssh_user} {host_mount_location}", + f"mkdir -p {host_mount_location_unix} && chown -R " + f"{self.ssh_command_runner.ssh_user} {host_mount_location_unix}", silent=is_rsync_silent(), ) if source[-1] == "/": @@ -575,7 +608,9 @@ def run_rsync_down(self, source, target, options=None): self.docker_cmd, self.container_name, self._docker_expand_user(source), - host_source, + host_source.replace( + "\\", "/" + ), # Convert Windows paths to Unix-style for rsync ), silent=is_rsync_silent(), ) @@ -728,7 +763,6 @@ def run_init( "{} pull {}".format(self.docker_cmd, specific_image), run_env="host" ) else: - self.run( f"{self.docker_cmd} image inspect {specific_image} " "1> /dev/null 2>&1 || " @@ -750,9 +784,9 @@ def run_init( specific_image, cleaned_bind_mounts ) if requires_re_init: - self.run( - f"{self.docker_cmd} stop {self.container_name}", run_env="host" - ) + docker_stop_cmd = f"{self.docker_cmd} stop {self.container_name}" + logger.info("Executing Docker command: %s", docker_stop_cmd) + self.run(docker_stop_cmd, run_env="host") if (not container_running) or requires_re_init: if not sync_run_yet: @@ -821,7 +855,9 @@ def run_init( self.ssh_command_runner.cluster_name ), mount, - ), + ).replace( + "\\", "/" + ), # Convert Windows paths to Unix-style for rsync container=self.container_name, dst=self._docker_expand_user(mount), ) diff --git a/python/ray/autoscaler/_private/commands.py b/python/ray/autoscaler/_private/commands.py index 52f87f2e31a0..fa571a641b48 100644 --- a/python/ray/autoscaler/_private/commands.py +++ b/python/ray/autoscaler/_private/commands.py @@ -926,6 +926,18 @@ def get_or_create_head_node( ) cli_logger.newline() + # Clean up temporary config file if it was created + # Clean up temporary config file if it was created on Windows + if ( + sys.platform == "win32" + and not no_monitor_on_head + and "remote_config_file" in locals() + ): + try: + os.remove(remote_config_file.name) + except OSError: + pass # Ignore cleanup errors + def _should_create_new_head( head_node_id: Optional[str], @@ -1025,9 +1037,14 @@ def _set_up_config_for_head_node( remote_config = provider.prepare_for_head_node(remote_config) # Now inject the rewritten config and SSH key into the head node - remote_config_file = tempfile.NamedTemporaryFile("w", prefix="ray-bootstrap-") + is_windows = sys.platform == "win32" + remote_config_file = tempfile.NamedTemporaryFile( + "w", prefix="ray-bootstrap-", delete=not is_windows + ) remote_config_file.write(json.dumps(remote_config)) remote_config_file.flush() + if is_windows: + remote_config_file.close() # Close the file handle to ensure it's accessible config["file_mounts"].update( {"~/ray_bootstrap_config.yaml": remote_config_file.name} ) diff --git a/python/ray/autoscaler/_private/util.py b/python/ray/autoscaler/_private/util.py index 17d463304957..66a7e5a8b900 100644 --- a/python/ray/autoscaler/_private/util.py +++ b/python/ray/autoscaler/_private/util.py @@ -4,6 +4,7 @@ import json import logging import os +import sys import threading from dataclasses import dataclass from datetime import datetime @@ -193,6 +194,12 @@ def validate_config(config: Dict[str, Any]) -> None: "sum of `min_workers` of all the available node types." ) + if sys.platform == "win32" and config.get("file_mounts_sync_continuously", False): + raise ValueError( + "`file_mounts_sync_continuously` is not supported on Windows. " + "Please set this to False when running on Windows." + ) + def check_legacy_fields(config: Dict[str, Any]) -> None: """For use in providers that have completed the migration to From 4f1fca4dc6c5100fe99a8aba772b77b5180adaf0 Mon Sep 17 00:00:00 2001 From: Jialing He Date: Fri, 29 Aug 2025 02:43:16 +0800 Subject: [PATCH 340/634] [Core] Fixed the issue of RemoveActorNameFromRegistry being called repeatedly. (#54955) Signed-off-by: hejialing.hjl Co-authored-by: hejialing.hjl --- python/ray/tests/test_actor.py | 59 +++++++++++++++++++ src/ray/gcs/gcs_server/gcs_actor_manager.cc | 1 - .../tests/gcs_actor_manager_test.cc | 18 ++++++ 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/python/ray/tests/test_actor.py b/python/ray/tests/test_actor.py index b1ff97c2a671..99197d8217e2 100644 --- a/python/ray/tests/test_actor.py +++ b/python/ray/tests/test_actor.py @@ -2,6 +2,7 @@ import random import sys import tempfile +import signal import numpy as np import pytest @@ -1676,5 +1677,63 @@ def method(self): assert result == "ok" +def test_get_actor_after_same_name_actor_dead(shutdown_only): + ACTOR_NAME = "test_actor" + NAMESPACE_NAME = "test_namespace" + + ray.init(namespace=NAMESPACE_NAME) + + @ray.remote + class Actor: + def get_pid(self): + return os.getpid() + + a = Actor.options(name=ACTOR_NAME, max_restarts=0, max_task_retries=-1).remote() + + pid = ray.get(a.get_pid.remote()) + os.kill(pid, signal.SIGKILL) + a_actor_id = a._actor_id.hex() + + wait_for_condition(lambda: ray.state.actors(a_actor_id)["State"] == "DEAD") + + # When a reference is held, the name cannot be reused. + with pytest.raises(ValueError): + Actor.options(name=ACTOR_NAME).remote() + + # Deleting the remaining reference so the name can be reused + del a + + b = None + + def wait_new_actor_ready(): + nonlocal b + b = Actor.options(name=ACTOR_NAME).remote() + return True + + wait_for_condition(wait_new_actor_ready) + + ray.get(b.__ray_ready__.remote()) + _ = ray.get_actor(ACTOR_NAME, namespace=NAMESPACE_NAME) + + # ray.kill can proactively release the name. + ray.kill(b) + wait_for_condition(lambda: ray.state.actors(b._actor_id.hex())["State"] == "DEAD") + + c = Actor.options(name=ACTOR_NAME, lifetime="detached").remote() + ray.get(c.__ray_ready__.remote()) + _ = ray.get_actor(ACTOR_NAME, namespace=NAMESPACE_NAME) + + pid = ray.get(c.get_pid.remote()) + os.kill(pid, signal.SIGKILL) + + wait_for_condition(lambda: ray.state.actors(c._actor_id.hex())["State"] == "DEAD") + + # Detached actors do not subscribe to reference counting, so + # they release the actor name when the actor is dead, without waiting for the reference count + # to be released or the execution of ray.kill. + d = Actor.options(name=ACTOR_NAME).remote() + ray.get(d.__ray_ready__.remote()) + + if __name__ == "__main__": sys.exit(pytest.main(["-sv", __file__])) diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index 399dacf8b5b6..ddad4fe2d02f 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -1376,7 +1376,6 @@ void GcsActorManager::RestartActor(const ActorID &actor_id, io_context_})); gcs_actor_scheduler_->Schedule(actor); } else { - RemoveActorNameFromRegistry(actor); actor->UpdateState(rpc::ActorTableData::DEAD); mutable_actor_table_data->mutable_death_cause()->CopyFrom(death_cause); auto time = current_sys_time_ms(); diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc index 659c9a5c53e0..b05cabf4141b 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc @@ -713,6 +713,24 @@ TEST_F(GcsActorManagerTest, TestNamedActorDeletionWorkerFailure) { ASSERT_TRUE(absl::StrContains( actor->GetActorTableData().death_cause().actor_died_error_context().error_message(), "worker process has died.")); + ASSERT_EQ(gcs_actor_manager_->GetActorIDByName(actor_name, "test"), + actor->GetActorID()); + + // Detached actor has no reply of WaitForActorRefDeleted request. + ASSERT_FALSE(worker_client_->Reply()); + // Kill this detached actor + rpc::KillActorViaGcsReply reply; + rpc::KillActorViaGcsRequest request; + request.set_actor_id(actor->GetActorID().Binary()); + request.set_force_kill(true); + request.set_no_restart(true); + gcs_actor_manager_->HandleKillActorViaGcs( + request, + &reply, + /*send_reply_callback*/ + [](Status status, std::function success, std::function failure) {}); + io_service_.run_one(); + ASSERT_EQ(gcs_actor_manager_->GetActorIDByName(actor_name, "test"), ActorID::Nil()); // Create an actor with the same name. This ensures that the name has been properly From 2ff6909822cd617049daaad6ec2af6845a296f44 Mon Sep 17 00:00:00 2001 From: Nikhil G Date: Thu, 28 Aug 2025 11:53:33 -0700 Subject: [PATCH 341/634] [docs] Batch inference for embedding models in ray data (#56027) --- doc/source/data/working-with-llms.rst | 58 +++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/doc/source/data/working-with-llms.rst b/doc/source/data/working-with-llms.rst index d9b052b03e83..cfd0c4bedf77 100644 --- a/doc/source/data/working-with-llms.rst +++ b/doc/source/data/working-with-llms.rst @@ -9,6 +9,7 @@ This guide shows you how to use :ref:`ray.data.llm ` to: * :ref:`Perform batch inference with LLMs ` * :ref:`Configure vLLM for LLM inference ` +* :ref:`Batch inference with embedding models ` * :ref:`Query deployed models with an OpenAI compatible API endpoint ` .. _batch_inference_llm: @@ -283,6 +284,63 @@ This example applies 2 adjustments on top of the previous example: vision_processed_ds = vision_processor(vision_dataset).materialize() vision_processed_ds.show(3) +.. _embedding_models: + +Batch inference with embedding models +--------------------------------------- + +Ray Data LLM supports batch inference with embedding models using vLLM: + +.. testcode:: + + import ray + from ray.data.llm import vLLMEngineProcessorConfig, build_llm_processor + + embedding_config = vLLMEngineProcessorConfig( + model_source="sentence-transformers/all-MiniLM-L6-v2", + task_type="embed", + engine_kwargs=dict( + enable_prefix_caching=False, + enable_chunked_prefill=False, + max_model_len=256, + enforce_eager=True, + ), + batch_size=32, + concurrency=1, + apply_chat_template=False, + detokenize=False, + ) + + embedding_processor = build_llm_processor( + embedding_config, + preprocess=lambda row: dict(prompt=row["text"]), + postprocess=lambda row: { + "text": row["prompt"], + "embedding": row["embeddings"], + }, + ) + + texts = [ + "Hello world", + "This is a test sentence", + "Embedding models convert text to vectors", + ] + ds = ray.data.from_items([{"text": text} for text in texts]) + + embedded_ds = embedding_processor(ds) + embedded_ds.show(limit=1) + +.. testoutput:: + :options: +MOCK + + {'text': 'Hello world', 'embedding': [0.1, -0.2, 0.3, ...]} + +Key differences for embedding models: + +- Set ``task_type="embed"`` +- Set ``apply_chat_template=False`` and ``detokenize=False`` +- Use direct ``prompt`` input instead of ``messages`` +- Access embeddings through``row["embeddings"]`` .. _openai_compatible_api_endpoint: From a7faf4cc9f9a8e20a4db7dbf7e421f38d353bba7 Mon Sep 17 00:00:00 2001 From: Sampan S Nayak Date: Fri, 29 Aug 2025 00:37:17 +0530 Subject: [PATCH 342/634] [core] gcs AddEvent support (#55528) This is a series of PRs that connects each worker's EventAggregator to GCS, completing the OneEvent implementation for task events. For more details, see: https://docs.google.com/document/d/1WjdlKprMqLqyPvmR1OGRQNhCD93SWrCXj21jZXXUcSk/edit?tab=t.0#heading=h.mmmgp2sapmts. Test: - CI - build --------- Signed-off-by: sampan Signed-off-by: Sampan S Nayak Signed-off-by: Cuong Nguyen Co-authored-by: sampan Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cuong Nguyen --- src/ray/common/grpc_util.h | 7 + src/ray/core_worker/task_event_buffer.cc | 2 + src/ray/gcs/gcs_server/BUILD.bazel | 14 + .../gcs/gcs_server/gcs_ray_event_converter.cc | 239 ++++++++++ .../gcs/gcs_server/gcs_ray_event_converter.h | 107 +++++ src/ray/gcs/gcs_server/gcs_task_manager.cc | 20 +- src/ray/gcs/gcs_server/gcs_task_manager.h | 9 + src/ray/gcs/gcs_server/tests/BUILD.bazel | 11 + .../tests/gcs_ray_event_converter_test.cc | 427 ++++++++++++++++++ .../gcs_server/tests/gcs_task_manager_test.cc | 85 ++++ src/ray/gcs/tests/gcs_test_util.h | 34 ++ .../public/events_task_execution_event.proto | 1 + 12 files changed, 952 insertions(+), 4 deletions(-) create mode 100644 src/ray/gcs/gcs_server/gcs_ray_event_converter.cc create mode 100644 src/ray/gcs/gcs_server/gcs_ray_event_converter.h create mode 100644 src/ray/gcs/gcs_server/tests/gcs_ray_event_converter_test.cc diff --git a/src/ray/common/grpc_util.h b/src/ray/common/grpc_util.h index 458c5d17d3e8..ea48af266fce 100644 --- a/src/ray/common/grpc_util.h +++ b/src/ray/common/grpc_util.h @@ -243,4 +243,11 @@ inline google::protobuf::Timestamp AbslTimeNanosToProtoTimestamp(int64_t nanos) return timestamp; } +// Conver a protobuf timestamp to an epoch time in nanoseconds +// Ref: https://protobuf.dev/reference/php/api-docs/Google/Protobuf/Timestamp.html +inline int64_t ProtoTimestampToAbslTimeNanos( + const google::protobuf::Timestamp ×tamp) { + return timestamp.seconds() * 1000000000LL + timestamp.nanos(); +} + } // namespace ray diff --git a/src/ray/core_worker/task_event_buffer.cc b/src/ray/core_worker/task_event_buffer.cc index e3ead75ec489..9187764b2768 100644 --- a/src/ray/core_worker/task_event_buffer.cc +++ b/src/ray/core_worker/task_event_buffer.cc @@ -254,6 +254,8 @@ void TaskStatusEvent::PopulateRpcRayTaskExecutionEvent( if (state_update_->pid_.has_value()) { execution_event_data.set_worker_pid(state_update_->pid_.value()); } + + execution_event_data.set_job_id(job_id_.Binary()); } void TaskStatusEvent::PopulateRpcRayEventBaseFields( diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 3a7b5dd42443..50e782c6b17a 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -193,11 +193,25 @@ ray_cc_library( ], ) +ray_cc_library( + name = "gcs_ray_event_converter", + srcs = ["gcs_ray_event_converter.cc"], + hdrs = ["gcs_ray_event_converter.h"], + deps = [ + "//src/ray/common:grpc_util", + "//src/ray/common:id", + "//src/ray/protobuf:events_event_aggregator_service_cc_proto", + "//src/ray/protobuf:gcs_service_cc_proto", + "//src/ray/util:logging", + ], +) + ray_cc_library( name = "gcs_task_manager", srcs = ["gcs_task_manager.cc"], hdrs = ["gcs_task_manager.h"], deps = [ + ":gcs_ray_event_converter", ":gcs_usage_stats_client", ":grpc_service_interfaces", "//src/ray/common:asio", diff --git a/src/ray/gcs/gcs_server/gcs_ray_event_converter.cc b/src/ray/gcs/gcs_server/gcs_ray_event_converter.cc new file mode 100644 index 000000000000..43b76e8e001f --- /dev/null +++ b/src/ray/gcs/gcs_server/gcs_ray_event_converter.cc @@ -0,0 +1,239 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/gcs/gcs_server/gcs_ray_event_converter.h" + +#include + +#include "absl/container/flat_hash_map.h" +#include "ray/common/grpc_util.h" +#include "ray/common/id.h" +#include "ray/util/logging.h" + +namespace ray { +namespace gcs { + +std::vector +GcsRayEventConverter::ConvertToTaskEventDataRequests( + rpc::events::AddEventsRequest &&request) { + std::vector requests_per_job_id; + absl::flat_hash_map job_id_to_index; + // convert RayEvents to TaskEvents and group by job id. + for (auto &event : *request.mutable_events_data()->mutable_events()) { + std::optional task_event = std::nullopt; + switch (event.event_type()) { + case rpc::events::RayEvent::TASK_DEFINITION_EVENT: { + task_event = ConvertToTaskEvents(std::move(*event.mutable_task_definition_event())); + break; + } + case rpc::events::RayEvent::TASK_EXECUTION_EVENT: { + task_event = ConvertToTaskEvents(std::move(*event.mutable_task_execution_event())); + break; + } + case rpc::events::RayEvent::TASK_PROFILE_EVENT: { + task_event = ConvertToTaskEvents(std::move(*event.mutable_task_profile_events())); + break; + } + case rpc::events::RayEvent::ACTOR_TASK_DEFINITION_EVENT: { + task_event = + ConvertToTaskEvents(std::move(*event.mutable_actor_task_definition_event())); + break; + } + default: + // TODO(can-anyscale): Handle other event types + break; + } + + // Groups all taskEvents belonging to same jobId into one AddTaskEventDataRequest + if (task_event) { + AddTaskEventToRequest(std::move(*task_event), requests_per_job_id, job_id_to_index); + } + } + + // Groups all taskEventMetadata belonging to same jobId into one + // AddTaskEventDataRequest + auto *metadata = request.mutable_events_data()->mutable_task_events_metadata(); + if (metadata->dropped_task_attempts_size() > 0) { + AddDroppedTaskAttemptsToRequest( + std::move(*metadata), requests_per_job_id, job_id_to_index); + } + return requests_per_job_id; +} + +void GcsRayEventConverter::AddTaskEventToRequest( + rpc::TaskEvents &&task_event, + std::vector &requests_per_job_id, + absl::flat_hash_map &job_id_to_index) { + const std::string job_id_key = task_event.job_id(); + auto it = job_id_to_index.find(job_id_key); + if (it == job_id_to_index.end()) { + // Create new AddTaskEventDataRequest entry and add index to map + size_t idx = requests_per_job_id.size(); + requests_per_job_id.emplace_back(); + auto *data = requests_per_job_id.back().mutable_data(); + data->set_job_id(job_id_key); + *data->add_events_by_task() = std::move(task_event); + job_id_to_index.emplace(job_id_key, idx); + } else { + // add taskEvent to existing AddTaskEventDataRequest with same job id + auto *data = requests_per_job_id[it->second].mutable_data(); + *data->add_events_by_task() = std::move(task_event); + } +} + +void GcsRayEventConverter::AddDroppedTaskAttemptsToRequest( + rpc::events::TaskEventsMetadata &&metadata, + std::vector &requests_per_job_id, + absl::flat_hash_map &job_id_to_index) { + // Process each dropped task attempt individually and route to the correct job ID + for (auto &dropped_attempt : *metadata.mutable_dropped_task_attempts()) { + const auto task_id = TaskID::FromBinary(dropped_attempt.task_id()); + const auto job_id_key = task_id.JobId().Binary(); + + auto it = job_id_to_index.find(job_id_key); + if (it == job_id_to_index.end()) { + // Create new request if job_id not found + size_t idx = requests_per_job_id.size(); + requests_per_job_id.emplace_back(); + auto *data = requests_per_job_id.back().mutable_data(); + data->set_job_id(job_id_key); + *data->add_dropped_task_attempts() = std::move(dropped_attempt); + job_id_to_index.emplace(job_id_key, idx); + } else { + // Add to existing request with same job_id + auto *data = requests_per_job_id[it->second].mutable_data(); + *data->add_dropped_task_attempts() = std::move(dropped_attempt); + } + } +} + +rpc::TaskEvents GcsRayEventConverter::ConvertToTaskEvents( + rpc::events::TaskDefinitionEvent &&event) { + rpc::TaskEvents task_event; + task_event.set_task_id(event.task_id()); + task_event.set_attempt_number(event.task_attempt()); + task_event.set_job_id(event.job_id()); + + rpc::TaskInfoEntry *task_info = task_event.mutable_task_info(); + task_info->set_type(event.task_type()); + task_info->set_name(event.task_name()); + task_info->set_task_id(event.task_id()); + task_info->set_job_id(event.job_id()); + task_info->set_parent_task_id(event.parent_task_id()); + if (!event.placement_group_id().empty()) { + task_info->set_placement_group_id(event.placement_group_id()); + } + + PopulateTaskRuntimeAndFunctionInfo(std::move(*event.mutable_runtime_env_info()), + std::move(*event.mutable_task_func()), + std::move(*event.mutable_required_resources()), + event.language(), + task_info); + return task_event; +} + +rpc::TaskEvents GcsRayEventConverter::ConvertToTaskEvents( + rpc::events::TaskExecutionEvent &&event) { + rpc::TaskEvents task_event; + task_event.set_task_id(event.task_id()); + task_event.set_attempt_number(event.task_attempt()); + task_event.set_job_id(event.job_id()); + + rpc::TaskStateUpdate *task_state_update = task_event.mutable_state_updates(); + task_state_update->set_node_id(event.node_id()); + task_state_update->set_worker_id(event.worker_id()); + task_state_update->set_worker_pid(event.worker_pid()); + task_state_update->mutable_error_info()->Swap(event.mutable_ray_error_info()); + + for (const auto &[state, timestamp] : event.task_state()) { + int64_t ns = ProtoTimestampToAbslTimeNanos(timestamp); + (*task_state_update->mutable_state_ts_ns())[state] = ns; + } + return task_event; +} + +rpc::TaskEvents GcsRayEventConverter::ConvertToTaskEvents( + rpc::events::ActorTaskDefinitionEvent &&event) { + rpc::TaskEvents task_event; + task_event.set_task_id(event.task_id()); + task_event.set_attempt_number(event.task_attempt()); + task_event.set_job_id(event.job_id()); + + rpc::TaskInfoEntry *task_info = task_event.mutable_task_info(); + task_info->set_type(rpc::TaskType::ACTOR_TASK); + task_info->set_name(event.actor_task_name()); + task_info->set_task_id(event.task_id()); + task_info->set_job_id(event.job_id()); + task_info->set_parent_task_id(event.parent_task_id()); + if (!event.placement_group_id().empty()) { + task_info->set_placement_group_id(event.placement_group_id()); + } + if (!event.actor_id().empty()) { + task_info->set_actor_id(event.actor_id()); + } + PopulateTaskRuntimeAndFunctionInfo(std::move(*event.mutable_runtime_env_info()), + std::move(*event.mutable_actor_func()), + std::move(*event.mutable_required_resources()), + event.language(), + task_info); + return task_event; +} + +rpc::TaskEvents GcsRayEventConverter::ConvertToTaskEvents( + rpc::events::TaskProfileEvents &&event) { + rpc::TaskEvents task_event; + task_event.set_task_id(event.task_id()); + task_event.set_attempt_number(event.attempt_number()); + task_event.set_job_id(event.job_id()); + + task_event.mutable_profile_events()->Swap(event.mutable_profile_events()); + return task_event; +} + +void GcsRayEventConverter::PopulateTaskRuntimeAndFunctionInfo( + rpc::RuntimeEnvInfo &&runtime_env_info, + rpc::FunctionDescriptor &&function_descriptor, + ::google::protobuf::Map &&required_resources, + rpc::Language language, + rpc::TaskInfoEntry *task_info) { + task_info->set_language(language); + task_info->mutable_runtime_env_info()->Swap(&runtime_env_info); + switch (language) { + case rpc::Language::CPP: + if (function_descriptor.has_cpp_function_descriptor()) { + task_info->set_func_or_class_name( + function_descriptor.cpp_function_descriptor().function_name()); + } + break; + case rpc::Language::PYTHON: + if (function_descriptor.has_python_function_descriptor()) { + task_info->set_func_or_class_name( + function_descriptor.python_function_descriptor().function_name()); + } + break; + case rpc::Language::JAVA: + if (function_descriptor.has_java_function_descriptor()) { + task_info->set_func_or_class_name( + function_descriptor.java_function_descriptor().function_name()); + } + break; + default: + // Other languages are not handled. + break; + } + task_info->mutable_required_resources()->swap(required_resources); +} + +} // namespace gcs +} // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_ray_event_converter.h b/src/ray/gcs/gcs_server/gcs_ray_event_converter.h new file mode 100644 index 000000000000..950d77c00dcb --- /dev/null +++ b/src/ray/gcs/gcs_server/gcs_ray_event_converter.h @@ -0,0 +1,107 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "absl/container/flat_hash_map.h" +#include "gtest/gtest_prod.h" +#include "src/ray/protobuf/events_event_aggregator_service.pb.h" +#include "src/ray/protobuf/gcs_service.pb.h" + +namespace ray { +namespace gcs { + +/// GcsRayEventConverter converts RayEvents to TaskEvents. +class GcsRayEventConverter { + public: + GcsRayEventConverter() = default; + ~GcsRayEventConverter() = default; + + /// Convert an AddEventsRequest to a list of AddTaskEventDataRequest objects, + /// grouping entries by job id. + /// + /// \param request The AddEventsRequest to convert. + /// \return A list of AddTaskEventDataRequest grouped by job id. + std::vector ConvertToTaskEventDataRequests( + rpc::events::AddEventsRequest &&request); + + private: + /// Convert a TaskDefinitionEvent to a TaskEvents. + /// + /// \param event The TaskDefinitionEvent to convert. + /// \return The output TaskEvents to populate. + rpc::TaskEvents ConvertToTaskEvents(rpc::events::TaskDefinitionEvent &&event); + + /// Convert ProfileEvents to a TaskEvents. + /// + /// \param event TaskProfileEvents object to convert. + /// \return The output TaskEvents to populate. + rpc::TaskEvents ConvertToTaskEvents(rpc::events::TaskProfileEvents &&event); + + /// Convert a TaskExecutionEvent to a TaskEvents. + /// + /// \param event The TaskExecutionEvent to convert. + /// \return The output TaskEvents to populate. + rpc::TaskEvents ConvertToTaskEvents(rpc::events::TaskExecutionEvent &&event); + + /// Convert an ActorTaskDefinitionEvent to a TaskEvents. + /// + /// \param event The ActorTaskDefinitionEvent to convert. + /// \return The output TaskEvents to populate. + rpc::TaskEvents ConvertToTaskEvents(rpc::events::ActorTaskDefinitionEvent &&event); + + /// Populate the TaskInfoEntry with the given runtime env info, function descriptor, + /// and required resources. This function is commonly used to convert the task + /// and actor task definition events to TaskEvents. + /// + /// \param runtime_env_info The runtime env info. + /// \param function_descriptor The function descriptor. + /// \param required_resources The required resources. + /// \param language The language of the task. + /// \param task_info The output TaskInfoEntry to populate. + void PopulateTaskRuntimeAndFunctionInfo( + rpc::RuntimeEnvInfo &&runtime_env_info, + rpc::FunctionDescriptor &&function_descriptor, + ::google::protobuf::Map &&required_resources, + rpc::Language language, + rpc::TaskInfoEntry *task_info); + + /// Add a task event to the appropriate job-grouped request. + /// + /// \param task_event The TaskEvents to add. + /// \param requests_per_job_id The list of requests grouped by job id. + /// \param job_id_to_index The map from job id to index in requests_per_job_id. + void AddTaskEventToRequest( + rpc::TaskEvents &&task_event, + std::vector &requests_per_job_id, + absl::flat_hash_map &job_id_to_index); + + /// Add dropped task attempts to the appropriate job-grouped request. + /// + /// \param metadata The task events metadata containing dropped task attempts. + /// \param requests_per_job_id The list of requests grouped by job id. + /// \param job_id_to_index The map from job id to index in requests_per_job_id. + void AddDroppedTaskAttemptsToRequest( + rpc::events::TaskEventsMetadata &&metadata, + std::vector &requests_per_job_id, + absl::flat_hash_map &job_id_to_index); + + FRIEND_TEST(GcsRayEventConverterTest, TestConvertTaskExecutionEvent); + FRIEND_TEST(GcsRayEventConverterTest, TestConvertActorTaskDefinitionEvent); +}; + +} // namespace gcs +} // namespace ray diff --git a/src/ray/gcs/gcs_server/gcs_task_manager.cc b/src/ray/gcs/gcs_server/gcs_task_manager.cc index 271f704322ac..bbea0fe45f37 100644 --- a/src/ray/gcs/gcs_server/gcs_task_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_task_manager.cc @@ -38,6 +38,7 @@ GcsTaskManager::GcsTaskManager(instrumented_io_context &io_service) RayConfig::instance().task_events_max_num_task_in_gcs(), stats_counter_, std::make_unique())), + ray_event_converter_(std::make_unique()), periodical_runner_(PeriodicalRunner::Create(io_service_)) { periodical_runner_->RunFnPeriodically([this] { task_event_storage_->GcJobSummary(); }, 5 * 1000, @@ -638,9 +639,7 @@ void GcsTaskManager::GcsTaskManagerStorage::RecordDataLossFromWorker( } } -void GcsTaskManager::HandleAddTaskEventData(rpc::AddTaskEventDataRequest request, - rpc::AddTaskEventDataReply *reply, - rpc::SendReplyCallback send_reply_callback) { +void GcsTaskManager::RecordTaskEventData(rpc::AddTaskEventDataRequest &request) { auto data = std::move(*request.mutable_data()); task_event_storage_->RecordDataLossFromWorker(data); @@ -648,6 +647,12 @@ void GcsTaskManager::HandleAddTaskEventData(rpc::AddTaskEventDataRequest request stats_counter_.Increment(kTotalNumTaskEventsReported); task_event_storage_->AddOrReplaceTaskEvent(std::move(events_by_task)); } +} + +void GcsTaskManager::HandleAddTaskEventData(rpc::AddTaskEventDataRequest request, + rpc::AddTaskEventDataReply *reply, + rpc::SendReplyCallback send_reply_callback) { + RecordTaskEventData(request); // Processed all the task events GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); @@ -656,7 +661,14 @@ void GcsTaskManager::HandleAddTaskEventData(rpc::AddTaskEventDataRequest request void GcsTaskManager::HandleAddEvents(rpc::events::AddEventsRequest request, rpc::events::AddEventsReply *reply, rpc::SendReplyCallback send_reply_callback) { - // TODO(can-anyscale): Implement this. + auto task_event_data_requests = + ray_event_converter_->ConvertToTaskEventDataRequests(std::move(request)); + + for (auto &task_event_data : task_event_data_requests) { + RecordTaskEventData(task_event_data); + } + + // Processed all the task events GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); } diff --git a/src/ray/gcs/gcs_server/gcs_task_manager.h b/src/ray/gcs/gcs_server/gcs_task_manager.h index d6e5686100c7..bb9c03e7fbc8 100644 --- a/src/ray/gcs/gcs_server/gcs_task_manager.h +++ b/src/ray/gcs/gcs_server/gcs_task_manager.h @@ -24,6 +24,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/synchronization/mutex.h" +#include "ray/gcs/gcs_server/gcs_ray_event_converter.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/usage_stats_client.h" #include "ray/gcs/pb_util.h" @@ -472,6 +473,7 @@ class GcsTaskManager : public rpc::TaskInfoGcsServiceHandler, std::vector> task_events_list_; friend class GcsTaskManager; + FRIEND_TEST(GcsTaskManagerTest, TestHandleAddEventBasic); FRIEND_TEST(GcsTaskManagerTest, TestHandleAddTaskEventBasic); FRIEND_TEST(GcsTaskManagerTest, TestMergeTaskEventsSameTaskAttempt); FRIEND_TEST(GcsTaskManagerMemoryLimitedTest, TestLimitTaskEvents); @@ -482,6 +484,8 @@ class GcsTaskManager : public rpc::TaskInfoGcsServiceHandler, }; private: + void RecordTaskEventData(rpc::AddTaskEventDataRequest &request); + /// Record data loss from worker. /// /// TODO(rickyx): This will be updated to record task attempt loss properly. @@ -522,10 +526,15 @@ class GcsTaskManager : public rpc::TaskInfoGcsServiceHandler, // the io_service_thread_. Access to it is *not* thread safe. std::unique_ptr task_event_storage_; + // Converter for converting RayEvents to TaskEvents. + std::unique_ptr ray_event_converter_; + /// The runner to run function periodically. std::shared_ptr periodical_runner_; + FRIEND_TEST(GcsTaskManagerTest, TestHandleAddEventBasic); FRIEND_TEST(GcsTaskManagerTest, TestHandleAddTaskEventBasic); + FRIEND_TEST(GcsTaskManagerTest, TestHandleAddEventsMultiJobGrouping); FRIEND_TEST(GcsTaskManagerTest, TestMergeTaskEventsSameTaskAttempt); FRIEND_TEST(GcsTaskManagerMemoryLimitedTest, TestLimitTaskEvents); FRIEND_TEST(GcsTaskManagerMemoryLimitedTest, TestIndexNoLeak); diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index 0235d835391f..d65aa6f83b35 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -424,3 +424,14 @@ ray_cc_test( "@com_google_googletest//:gtest", ], ) + +ray_cc_test( + name = "gcs_ray_event_converter_test", + size = "small", + srcs = ["gcs_ray_event_converter_test.cc"], + tags = ["team:core"], + deps = [ + "//src/ray/gcs/gcs_server:gcs_ray_event_converter", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/gcs/gcs_server/tests/gcs_ray_event_converter_test.cc b/src/ray/gcs/gcs_server/tests/gcs_ray_event_converter_test.cc new file mode 100644 index 000000000000..02738c4cce4c --- /dev/null +++ b/src/ray/gcs/gcs_server/tests/gcs_ray_event_converter_test.cc @@ -0,0 +1,427 @@ +// Copyright 2022 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/gcs/gcs_server/gcs_ray_event_converter.h" + +#include + +#include "gtest/gtest.h" +#include "ray/common/id.h" +#include "src/ray/protobuf/common.pb.h" +#include "src/ray/protobuf/events_base_event.pb.h" +#include "src/ray/protobuf/events_event_aggregator_service.pb.h" +#include "src/ray/protobuf/gcs_service.pb.h" + +namespace ray { +namespace gcs { + +class GcsRayEventConverterTest : public ::testing::Test { + public: + GcsRayEventConverterTest() = default; +}; + +TEST_F(GcsRayEventConverterTest, TestConvertToTaskEventData) { + rpc::events::AddEventsRequest request; + GcsRayEventConverter converter; + + // Convert empty request + auto task_event_data_requests = + converter.ConvertToTaskEventDataRequests(std::move(request)); + + // Test empty request + EXPECT_EQ(task_event_data_requests.size(), 0); +} + +TEST_F(GcsRayEventConverterTest, TestConvertTaskDefinitionEvent) { + rpc::events::AddEventsRequest request; + GcsRayEventConverter converter; + + // Create a task definition event + auto *event = request.mutable_events_data()->add_events(); + event->set_event_id("test_event_id"); + event->set_event_type(rpc::events::RayEvent::TASK_DEFINITION_EVENT); + event->set_source_type(rpc::events::RayEvent::CORE_WORKER); + event->set_severity(rpc::events::RayEvent::INFO); + event->set_message("test message"); + + auto *task_def_event = event->mutable_task_definition_event(); + + task_def_event->set_task_type(rpc::TaskType::NORMAL_TASK); + task_def_event->set_language(rpc::Language::PYTHON); + task_def_event->mutable_task_func() + ->mutable_python_function_descriptor() + ->set_function_name("test_task_name"); + task_def_event->set_task_id("test_task_id"); + task_def_event->set_task_attempt(1); + task_def_event->set_job_id("test_job_id"); + task_def_event->set_task_name("test_task_name"); + + task_def_event->set_parent_task_id("parent_task_id"); + task_def_event->set_placement_group_id("pg_id"); + + // Add some required resources + (*task_def_event->mutable_required_resources())["CPU"] = 1.0; + (*task_def_event->mutable_required_resources())["memory"] = 1024.0; + + // Set runtime env info + auto *runtime_env = task_def_event->mutable_runtime_env_info(); + runtime_env->set_serialized_runtime_env("test_env"); + + // Convert + auto task_event_data_requests = + converter.ConvertToTaskEventDataRequests(std::move(request)); + + // Verify conversion + ASSERT_EQ(task_event_data_requests.size(), 1); + const auto &task_event_data = task_event_data_requests[0]; + EXPECT_EQ(task_event_data.data().events_by_task_size(), 1); + const auto &converted_task = task_event_data.data().events_by_task(0); + EXPECT_EQ(converted_task.task_id(), "test_task_id"); + EXPECT_EQ(converted_task.attempt_number(), 1); + EXPECT_EQ(converted_task.job_id(), "test_job_id"); + EXPECT_EQ(task_event_data.data().job_id(), "test_job_id"); + + // Verify task info + ASSERT_TRUE(converted_task.has_task_info()); + const auto &task_info = converted_task.task_info(); + EXPECT_EQ(task_info.name(), "test_task_name"); + EXPECT_EQ(task_info.type(), rpc::TaskType::NORMAL_TASK); + EXPECT_EQ(task_info.language(), rpc::Language::PYTHON); + EXPECT_EQ(task_info.func_or_class_name(), "test_task_name"); + EXPECT_EQ(task_info.runtime_env_info().serialized_runtime_env(), "test_env"); + EXPECT_EQ(task_info.parent_task_id(), "parent_task_id"); + EXPECT_EQ(task_info.placement_group_id(), "pg_id"); + + // Verify required resources + EXPECT_EQ(task_info.required_resources().at("CPU"), 1.0); + EXPECT_EQ(task_info.required_resources().at("memory"), 1024.0); +} + +TEST_F(GcsRayEventConverterTest, TestConvertWithDroppedTaskAttempts) { + rpc::events::AddEventsRequest request; + GcsRayEventConverter converter; + + // Create a proper TaskID for testing + const auto job_id = JobID::FromInt(100); + const auto driver_task_id = TaskID::ForDriverTask(job_id); + const auto test_task_id = TaskID::ForNormalTask(job_id, driver_task_id, 1); + const auto task_id_binary = test_task_id.Binary(); + + // Add dropped task attempts to metadata + auto *dropped_attempt = request.mutable_events_data() + ->mutable_task_events_metadata() + ->add_dropped_task_attempts(); + dropped_attempt->set_task_id(task_id_binary); + dropped_attempt->set_attempt_number(2); + + // Convert + auto task_event_data_requests = + converter.ConvertToTaskEventDataRequests(std::move(request)); + + // Verify dropped task attempts are copied + ASSERT_FALSE(task_event_data_requests.empty()); + EXPECT_EQ(task_event_data_requests[0].data().dropped_task_attempts_size(), 1); + const auto &converted_dropped = + task_event_data_requests[0].data().dropped_task_attempts(0); + EXPECT_EQ(converted_dropped.task_id(), task_id_binary); + EXPECT_EQ(converted_dropped.attempt_number(), 2); +} + +TEST_F(GcsRayEventConverterTest, TestMultipleJobIds) { + rpc::events::AddEventsRequest request; + GcsRayEventConverter converter; + + // Create events with different job IDs + const auto job_id_1 = JobID::FromInt(100); + const auto job_id_2 = JobID::FromInt(200); + + // Create first task event + auto *event1 = request.mutable_events_data()->add_events(); + event1->set_event_id("test_event_1"); + event1->set_event_type(rpc::events::RayEvent::TASK_DEFINITION_EVENT); + auto *task_def_event1 = event1->mutable_task_definition_event(); + task_def_event1->set_task_type(rpc::TaskType::NORMAL_TASK); + task_def_event1->set_language(rpc::Language::PYTHON); + task_def_event1->set_task_id("task_1"); + task_def_event1->set_job_id(job_id_1.Binary()); + task_def_event1->set_task_name("task_1_name"); + + // Create second task event with different job ID + auto *event2 = request.mutable_events_data()->add_events(); + event2->set_event_id("test_event_2"); + event2->set_event_type(rpc::events::RayEvent::TASK_DEFINITION_EVENT); + auto *task_def_event2 = event2->mutable_task_definition_event(); + task_def_event2->set_task_type(rpc::TaskType::NORMAL_TASK); + task_def_event2->set_language(rpc::Language::PYTHON); + task_def_event2->set_task_id("task_2"); + task_def_event2->set_job_id(job_id_2.Binary()); + task_def_event2->set_task_name("task_2_name"); + + // Add dropped task attempts for both job IDs + const auto driver_task_id_1 = TaskID::ForDriverTask(job_id_1); + const auto test_task_id_1 = TaskID::ForNormalTask(job_id_1, driver_task_id_1, 1); + + const auto driver_task_id_2 = TaskID::ForDriverTask(job_id_2); + const auto test_task_id_2 = TaskID::ForNormalTask(job_id_2, driver_task_id_2, 1); + + // Add dropped task attempt for job_id_1 + auto *dropped_attempt_1 = request.mutable_events_data() + ->mutable_task_events_metadata() + ->add_dropped_task_attempts(); + dropped_attempt_1->set_task_id(test_task_id_1.Binary()); + dropped_attempt_1->set_attempt_number(3); + + // Add dropped task attempt for job_id_2 + auto *dropped_attempt_2 = request.mutable_events_data() + ->mutable_task_events_metadata() + ->add_dropped_task_attempts(); + dropped_attempt_2->set_task_id(test_task_id_2.Binary()); + dropped_attempt_2->set_attempt_number(4); + + // Convert + auto task_event_data_requests = + converter.ConvertToTaskEventDataRequests(std::move(request)); + + // Verify that we get two separate requests (one for each job ID) + ASSERT_EQ(task_event_data_requests.size(), 2); + + // Check that each request has the correct job ID and dropped task attempts + bool found_job_1 = false, found_job_2 = false; + for (const auto &req : task_event_data_requests) { + if (req.data().job_id() == job_id_1.Binary()) { + found_job_1 = true; + EXPECT_EQ(req.data().events_by_task_size(), 1); + EXPECT_EQ(req.data().events_by_task(0).job_id(), job_id_1.Binary()); + + // Verify dropped task attempt for job_id_1 + EXPECT_EQ(req.data().dropped_task_attempts_size(), 1); + const auto &dropped = req.data().dropped_task_attempts(0); + EXPECT_EQ(dropped.task_id(), test_task_id_1.Binary()); + EXPECT_EQ(dropped.attempt_number(), 3); + } else if (req.data().job_id() == job_id_2.Binary()) { + found_job_2 = true; + EXPECT_EQ(req.data().events_by_task_size(), 1); + EXPECT_EQ(req.data().events_by_task(0).job_id(), job_id_2.Binary()); + + // Verify dropped task attempt for job_id_2 + EXPECT_EQ(req.data().dropped_task_attempts_size(), 1); + const auto &dropped = req.data().dropped_task_attempts(0); + EXPECT_EQ(dropped.task_id(), test_task_id_2.Binary()); + EXPECT_EQ(dropped.attempt_number(), 4); + } + } + EXPECT_TRUE(found_job_1); + EXPECT_TRUE(found_job_2); +} + +TEST_F(GcsRayEventConverterTest, TestSameJobIdGrouping) { + rpc::events::AddEventsRequest request; + GcsRayEventConverter converter; + + // Create multiple events with the same job ID + const auto job_id = JobID::FromInt(100); + + // Create first task event + auto *event1 = request.mutable_events_data()->add_events(); + event1->set_event_id("test_event_1"); + event1->set_event_type(rpc::events::RayEvent::TASK_DEFINITION_EVENT); + auto *task_def_event1 = event1->mutable_task_definition_event(); + task_def_event1->set_task_type(rpc::TaskType::NORMAL_TASK); + task_def_event1->set_language(rpc::Language::PYTHON); + task_def_event1->set_task_id("task_1"); + task_def_event1->set_job_id(job_id.Binary()); + task_def_event1->set_task_name("task_1_name"); + + // Create second task event with same job ID + auto *event2 = request.mutable_events_data()->add_events(); + event2->set_event_id("test_event_2"); + event2->set_event_type(rpc::events::RayEvent::TASK_DEFINITION_EVENT); + auto *task_def_event2 = event2->mutable_task_definition_event(); + task_def_event2->set_task_type(rpc::TaskType::NORMAL_TASK); + task_def_event2->set_language(rpc::Language::PYTHON); + task_def_event2->set_task_id("task_2"); + task_def_event2->set_job_id(job_id.Binary()); + task_def_event2->set_task_name("task_2_name"); + + // Convert + auto task_event_data_requests = + converter.ConvertToTaskEventDataRequests(std::move(request)); + + // Verify that we get one request with both events grouped together + ASSERT_EQ(task_event_data_requests.size(), 1); + EXPECT_EQ(task_event_data_requests[0].data().job_id(), job_id.Binary()); + EXPECT_EQ(task_event_data_requests[0].data().events_by_task_size(), 2); + + // Verify both tasks are present + const auto &events = task_event_data_requests[0].data().events_by_task(); + EXPECT_EQ(events[0].job_id(), job_id.Binary()); + EXPECT_EQ(events[1].job_id(), job_id.Binary()); +} + +TEST_F(GcsRayEventConverterTest, TestConvertTaskProfileEvents) { + rpc::events::AddEventsRequest request; + GcsRayEventConverter converter; + + // Create a task profile event + auto *event = request.mutable_events_data()->add_events(); + event->set_event_id("test_event_id"); + event->set_event_type(rpc::events::RayEvent::TASK_PROFILE_EVENT); + event->set_source_type(rpc::events::RayEvent::CORE_WORKER); + event->set_severity(rpc::events::RayEvent::INFO); + event->set_message("test message"); + + auto *task_profile_events = event->mutable_task_profile_events(); + task_profile_events->set_task_id("test_task_id"); + task_profile_events->set_attempt_number(1); + task_profile_events->set_job_id("test_job_id"); + + // Add a profile event + auto *profile_events = task_profile_events->mutable_profile_events(); + profile_events->set_component_id("test_component_id"); + profile_events->set_component_type("worker"); + profile_events->set_node_ip_address("test_address"); + + // add a profile event entry + auto *ProfileEventEntry = profile_events->add_events(); + ProfileEventEntry->set_start_time(123456789); + ProfileEventEntry->set_end_time(123456799); + ProfileEventEntry->set_extra_data("{\"foo\": \"bar\"}"); + ProfileEventEntry->set_event_name("test_event"); + + // Convert + auto task_event_data_requests = + converter.ConvertToTaskEventDataRequests(std::move(request)); + + // Verify conversion + EXPECT_EQ(task_event_data_requests.size(), 1); + auto task_event_data = task_event_data_requests[0]; + EXPECT_EQ(task_event_data.data().events_by_task_size(), 1); + const auto &converted_task = task_event_data.data().events_by_task(0); + + EXPECT_EQ(converted_task.task_id(), "test_task_id"); + EXPECT_EQ(converted_task.attempt_number(), 1); + EXPECT_EQ(converted_task.job_id(), "test_job_id"); + EXPECT_EQ(converted_task.profile_events().events_size(), 1); + EXPECT_EQ(task_event_data.data().job_id(), "test_job_id"); + + // Check profile event fields + EXPECT_TRUE(converted_task.has_profile_events()); + const auto &profile_event = converted_task.profile_events(); + EXPECT_EQ(profile_event.component_id(), "test_component_id"); + EXPECT_EQ(profile_event.component_type(), "worker"); + EXPECT_EQ(profile_event.node_ip_address(), "test_address"); + + // verify that there is one profile event entry and values match our expectations + EXPECT_TRUE(profile_event.events().size() == 1); + const auto &entry = profile_event.events(0); + EXPECT_EQ(entry.start_time(), 123456789); + EXPECT_EQ(entry.end_time(), 123456799); + EXPECT_EQ(entry.extra_data(), "{\"foo\": \"bar\"}"); + EXPECT_EQ(entry.event_name(), "test_event"); +} + +TEST_F(GcsRayEventConverterTest, TestConvertTaskExecutionEvent) { + GcsRayEventConverter converter; + rpc::events::TaskExecutionEvent exec_event; + + // Set basic fields + exec_event.set_task_id("test_task_id"); + exec_event.set_task_attempt(3); + exec_event.set_job_id("test_job_id"); + exec_event.set_node_id("test_node_id"); + exec_event.set_worker_id("test_worker_id"); + exec_event.set_worker_pid(1234); + + // Set a RayErrorInfo + exec_event.mutable_ray_error_info()->set_error_message("error"); + + google::protobuf::Timestamp ts; + ts.set_seconds(42); + ts.set_nanos(123456789); + (*exec_event.mutable_task_state())[rpc::TaskStatus::SUBMITTED_TO_WORKER] = ts; + + // Call the converter + rpc::TaskEvents task_event = converter.ConvertToTaskEvents(std::move(exec_event)); + + // Check basic fields + EXPECT_EQ(task_event.attempt_number(), 3); + EXPECT_EQ(task_event.job_id(), "test_job_id"); + EXPECT_TRUE(task_event.has_state_updates()); + const auto &state_updates = task_event.state_updates(); + EXPECT_EQ(state_updates.node_id(), "test_node_id"); + EXPECT_EQ(state_updates.worker_id(), "test_worker_id"); + EXPECT_EQ(state_updates.worker_pid(), 1234); + EXPECT_EQ(state_updates.error_info().error_message(), "error"); + + // Check state_ts_ns + ASSERT_EQ(state_updates.state_ts_ns().size(), 1); + int64_t expected_ns = 42 * 1000000000LL + 123456789; + EXPECT_EQ(state_updates.state_ts_ns().at(5), expected_ns); +} + +TEST_F(GcsRayEventConverterTest, TestConvertActorTaskDefinitionEvent) { + GcsRayEventConverter converter; + rpc::events::ActorTaskDefinitionEvent actor_def_event; + + // Set basic fields + actor_def_event.set_task_id("test_actor_task_id"); + actor_def_event.set_task_attempt(2); + actor_def_event.set_job_id("test_job_id"); + actor_def_event.set_actor_task_name("test_actor_task"); + actor_def_event.set_language(rpc::Language::PYTHON); + actor_def_event.set_actor_id("actor-123"); + actor_def_event.set_parent_task_id("parent-actor-task"); + actor_def_event.set_placement_group_id("pg-actor"); + + // Set runtime env info + auto *runtime_env = actor_def_event.mutable_runtime_env_info(); + runtime_env->set_serialized_runtime_env("test_actor_env"); + + // Set actor function descriptor (Python) + auto *func_desc = actor_def_event.mutable_actor_func(); + auto *python_func = func_desc->mutable_python_function_descriptor(); + python_func->set_function_name("test_actor_function"); + python_func->set_class_name("TestActorClass"); + + // Add required resources + (*actor_def_event.mutable_required_resources())["CPU"] = 2.0; + (*actor_def_event.mutable_required_resources())["GPU"] = 1.0; + + // Call the converter + rpc::TaskEvents task_event = converter.ConvertToTaskEvents(std::move(actor_def_event)); + + // Check basic fields + EXPECT_EQ(task_event.task_id(), "test_actor_task_id"); + EXPECT_EQ(task_event.attempt_number(), 2); + EXPECT_EQ(task_event.job_id(), "test_job_id"); + + // Check task info + EXPECT_TRUE(task_event.has_task_info()); + const auto &task_info = task_event.task_info(); + EXPECT_EQ(task_info.type(), rpc::TaskType::ACTOR_TASK); + EXPECT_EQ(task_info.name(), "test_actor_task"); + EXPECT_EQ(task_info.language(), rpc::Language::PYTHON); + EXPECT_EQ(task_info.func_or_class_name(), "test_actor_function"); + EXPECT_EQ(task_info.runtime_env_info().serialized_runtime_env(), "test_actor_env"); + EXPECT_EQ(task_info.actor_id(), "actor-123"); + EXPECT_EQ(task_info.parent_task_id(), "parent-actor-task"); + EXPECT_EQ(task_info.placement_group_id(), "pg-actor"); + + // Check required resources + EXPECT_EQ(task_info.required_resources().at("CPU"), 2.0); + EXPECT_EQ(task_info.required_resources().at("GPU"), 1.0); +} + +} // namespace gcs +} // namespace ray diff --git a/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc index efb4c30b9a49..099ceadfad98 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc @@ -146,6 +146,32 @@ class GcsTaskManagerTest : public ::testing::Test { return reply; } + rpc::events::AddEventsReply SyncAddEvents( + const rpc::events::RayEventsData &events_data) { + rpc::events::AddEventsRequest request; + rpc::events::AddEventsReply reply; + std::promise promise; + + request.mutable_events_data()->CopyFrom(events_data); + // Dispatch so that it runs in GcsTaskManager's io service. + io_context_->GetIoService().dispatch( + [this, &promise, &request, &reply]() { + task_manager->HandleAddEvents( + request, + &reply, + [&promise](Status, std::function, std::function) { + promise.set_value(true); + }); + }, + "SyncAddEvent"); + + promise.get_future().get(); + + // Assert on RPC reply. + EXPECT_EQ(StatusCode(reply.status().code()), StatusCode::OK); + return reply; + } + rpc::GetTaskEventsReply SyncGetTaskEvents( const std::vector task_ids, const std::vector task_id_predicates, @@ -401,6 +427,21 @@ class GcsTaskManagerDroppedTaskAttemptsLimit : public GcsTaskManagerTest { } }; +TEST_F(GcsTaskManagerTest, TestHandleAddEventBasic) { + size_t num_task_events = 100; + auto task_ids = GenTaskIDs(num_task_events); + auto events = GenTaskEvents(task_ids, 0); + auto events_data = Mocker::GenRayEventsData(events, {}); + auto reply = SyncAddEvents(events_data); + + // Assert on RPC reply. + EXPECT_EQ(StatusCode(reply.status().code()), StatusCode::OK); + + // Assert on actual data. + EXPECT_EQ(task_manager->task_event_storage_->GetTaskEvents().size(), num_task_events); + EXPECT_EQ(task_manager->GetTotalNumTaskEventsReported(), num_task_events); +} + TEST_F(GcsTaskManagerTest, TestHandleAddTaskEventBasic) { size_t num_task_events = 100; int32_t num_status_events_dropped = 10; @@ -425,6 +466,50 @@ TEST_F(GcsTaskManagerTest, TestHandleAddTaskEventBasic) { } } +TEST_F(GcsTaskManagerTest, TestHandleAddEventsMultiJobGrouping) { + // Prepare events for two jobs in a single AddEvents request + auto task_ids_job0 = GenTaskIDs(3); + auto task_ids_job1 = GenTaskIDs(2); + + auto events_job0 = GenTaskEvents(task_ids_job0, /*attempt_number*/ 0, /*job_id*/ 0); + auto events_job1 = GenTaskEvents(task_ids_job1, /*attempt_number*/ 0, /*job_id*/ 1); + + // Build RayEventsData including dropped attempts for each job + std::vector all_events; + all_events.insert(all_events.end(), events_job0.begin(), events_job0.end()); + all_events.insert(all_events.end(), events_job1.begin(), events_job1.end()); + + std::vector dropped_attempts; + dropped_attempts.emplace_back(GenTaskIDForJob(0), 0); + dropped_attempts.emplace_back(GenTaskIDForJob(1), 0); + + auto ray_events_data = Mocker::GenRayEventsData(all_events, dropped_attempts); + + // Send AddEvents once; converter should group by job id and GCS should record all + auto reply = SyncAddEvents(ray_events_data); + EXPECT_EQ(StatusCode(reply.status().code()), StatusCode::OK); + + // Verify all events stored + EXPECT_EQ(task_manager->task_event_storage_->GetTaskEvents().size(), + task_ids_job0.size() + task_ids_job1.size()); + + // Verify per-job data loss counters populated from dropped attempts + { + auto reply_job0 = SyncGetTaskEvents(/* task_ids */ {}, JobID::FromInt(0)); + EXPECT_EQ(reply_job0.num_status_task_events_dropped(), 1); + } + { + auto reply_job1 = SyncGetTaskEvents(/* task_ids */ {}, JobID::FromInt(1)); + EXPECT_EQ(reply_job1.num_status_task_events_dropped(), 1); + } + + // Verify global counters reflect both drops + { + auto reply_all = SyncGetTaskEvents(/* task_ids */ {}); + EXPECT_EQ(reply_all.num_status_task_events_dropped(), 2); + } +} + TEST_F(GcsTaskManagerTest, TestMergeTaskEventsSameTaskAttempt) { size_t num_task_events = 20; // Same task id and attempt diff --git a/src/ray/gcs/tests/gcs_test_util.h b/src/ray/gcs/tests/gcs_test_util.h index 0d27f047190c..d820eae87261 100644 --- a/src/ray/gcs/tests/gcs_test_util.h +++ b/src/ray/gcs/tests/gcs_test_util.h @@ -308,6 +308,40 @@ struct Mocker { return data; } + static rpc::events::RayEventsData GenRayEventsData( + const std::vector &task_events, + const std::vector &drop_tasks) { + rpc::events::RayEventsData data; + rpc::events::TaskEventsMetadata metadata; + for (const auto &task_attempt : drop_tasks) { + rpc::TaskAttempt rpc_task_attempt; + rpc_task_attempt.set_task_id(task_attempt.first.Binary()); + rpc_task_attempt.set_attempt_number(task_attempt.second); + *(metadata.add_dropped_task_attempts()) = rpc_task_attempt; + } + data.mutable_task_events_metadata()->CopyFrom(metadata); + for (const auto &task_event : task_events) { + rpc::events::RayEvent ray_event; + rpc::events::TaskDefinitionEvent task_definition_event; + task_definition_event.set_task_id(task_event.task_id()); + task_definition_event.set_task_attempt(task_event.attempt_number()); + task_definition_event.set_job_id(task_event.job_id()); + if (task_event.has_task_info()) { + const auto &task_info = task_event.task_info(); + task_definition_event.set_task_type(task_info.type()); + task_definition_event.set_task_name(task_info.name()); + task_definition_event.set_language(task_info.language()); + } + ray_event.set_event_id(task_event.task_id()); + ray_event.set_event_type(rpc::events::RayEvent::TASK_DEFINITION_EVENT); + ray_event.set_message("test"); + ray_event.mutable_task_definition_event()->CopyFrom(task_definition_event); + *(data.add_events()) = ray_event; + } + + return data; + } + static rpc::TaskEventData GenTaskEventsDataLoss( const std::vector &drop_tasks, int job_id = 0) { rpc::TaskEventData data; diff --git a/src/ray/protobuf/public/events_task_execution_event.proto b/src/ray/protobuf/public/events_task_execution_event.proto index 49e3ee37d569..51724605fb7b 100644 --- a/src/ray/protobuf/public/events_task_execution_event.proto +++ b/src/ray/protobuf/public/events_task_execution_event.proto @@ -39,4 +39,5 @@ message TaskExecutionEvent { bytes node_id = 5; bytes worker_id = 6; int32 worker_pid = 7; + bytes job_id = 8; } From d9806f17e51e38d20c80b661b8a1269e06696c51 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Thu, 28 Aug 2025 14:24:56 -0500 Subject: [PATCH 343/634] [core] Split `gcs_placement_group_manager` out from `gcs_server_lib` (#56048) Also renamed it for consistency. I think previously this was an issue due to the test path length being too long on Windows, but I mitigated that recently. --------- Signed-off-by: Edward Oakes --- .../ray/gcs/gcs_server/gcs_actor_manager.h | 13 +- .../ray/gcs/gcs_server/gcs_actor_scheduler.h | 6 + src/mock/ray/gcs/gcs_server/gcs_init_data.h | 23 --- src/mock/ray/gcs/gcs_server/gcs_job_manager.h | 6 + src/mock/ray/gcs/gcs_server/gcs_kv_manager.h | 7 +- .../ray/gcs/gcs_server/gcs_node_manager.h | 3 +- ...up_mgr.h => gcs_placement_group_manager.h} | 11 +- .../gcs_placement_group_scheduler.h | 6 + .../ray/gcs/gcs_server/gcs_resource_manager.h | 6 + src/mock/ray/gcs/gcs_server/gcs_server.h | 33 ---- .../ray/gcs/gcs_server/gcs_table_storage.h | 154 ------------------ .../ray/gcs/gcs_server/gcs_task_manager.h | 6 + .../ray/gcs/gcs_server/gcs_worker_manager.h | 6 + src/ray/gcs/gcs_server/BUILD.bazel | 30 +++- .../gcs_autoscaler_state_manager.cc | 2 +- ..._mgr.cc => gcs_placement_group_manager.cc} | 2 +- ...up_mgr.h => gcs_placement_group_manager.h} | 6 +- src/ray/gcs/gcs_server/gcs_server.cc | 8 +- .../gcs/gcs_server/grpc_service_interfaces.h | 30 ++++ src/ray/gcs/gcs_server/grpc_services.cc | 19 +++ src/ray/gcs/gcs_server/grpc_services.h | 23 +++ src/ray/gcs/gcs_server/tests/BUILD.bazel | 12 +- .../gcs_autoscaler_state_manager_test.cc | 2 +- ... gcs_placement_group_manager_mock_test.cc} | 15 +- ...cc => gcs_placement_group_manager_test.cc} | 2 +- src/ray/rpc/gcs/gcs_rpc_server.h | 74 +-------- 26 files changed, 177 insertions(+), 328 deletions(-) delete mode 100644 src/mock/ray/gcs/gcs_server/gcs_init_data.h rename src/mock/ray/gcs/gcs_server/{gcs_placement_group_mgr.h => gcs_placement_group_manager.h} (93%) delete mode 100644 src/mock/ray/gcs/gcs_server/gcs_server.h delete mode 100644 src/mock/ray/gcs/gcs_server/gcs_table_storage.h rename src/ray/gcs/gcs_server/{gcs_placement_group_mgr.cc => gcs_placement_group_manager.cc} (99%) rename src/ray/gcs/gcs_server/{gcs_placement_group_mgr.h => gcs_placement_group_manager.h} (98%) rename src/ray/gcs/gcs_server/tests/{gcs_placement_group_mgr_mock_test.cc => gcs_placement_group_manager_mock_test.cc} (97%) rename src/ray/gcs/gcs_server/tests/{gcs_placement_group_mgr_test.cc => gcs_placement_group_manager_test.cc} (99%) diff --git a/src/mock/ray/gcs/gcs_server/gcs_actor_manager.h b/src/mock/ray/gcs/gcs_server/gcs_actor_manager.h index 7799fd7a9c01..b3984e0c36c1 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_actor_manager.h +++ b/src/mock/ray/gcs/gcs_server/gcs_actor_manager.h @@ -14,18 +14,9 @@ #pragma once -#include "gmock/gmock.h" -#include "ray/gcs/gcs_server/gcs_actor_manager.h" - -namespace ray { -namespace gcs { - -class MockGcsActor : public GcsActor { - public: -}; +#include -} // namespace gcs -} // namespace ray +#include "ray/gcs/gcs_server/gcs_actor_manager.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/mock/ray/gcs/gcs_server/gcs_actor_scheduler.h index 2715c57849eb..164be134d22b 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_actor_scheduler.h +++ b/src/mock/ray/gcs/gcs_server/gcs_actor_scheduler.h @@ -12,6 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#pragma once + +#include + +#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" + namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_init_data.h b/src/mock/ray/gcs/gcs_server/gcs_init_data.h deleted file mode 100644 index e784243ca5af..000000000000 --- a/src/mock/ray/gcs/gcs_server/gcs_init_data.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace ray { -namespace gcs { - -class MockGcsInitData : public GcsInitData { - public: -}; - -} // namespace gcs -} // namespace ray diff --git a/src/mock/ray/gcs/gcs_server/gcs_job_manager.h b/src/mock/ray/gcs/gcs_server/gcs_job_manager.h index 9b3b2ca2d1f2..9228c063e4f3 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_job_manager.h +++ b/src/mock/ray/gcs/gcs_server/gcs_job_manager.h @@ -12,6 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#pragma once + +#include + +#include "ray/gcs/gcs_server/gcs_job_manager.h" + namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_kv_manager.h b/src/mock/ray/gcs/gcs_server/gcs_kv_manager.h index 9004ffe59785..5a5a224e0199 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_kv_manager.h +++ b/src/mock/ray/gcs/gcs_server/gcs_kv_manager.h @@ -12,13 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "gmock/gmock.h" +#pragma once + +#include + #include "ray/gcs/gcs_server/gcs_kv_manager.h" namespace ray { namespace gcs { -class MockInternalKVInterface : public ray::gcs::InternalKVInterface { +class MockInternalKVInterface : public InternalKVInterface { public: MockInternalKVInterface() {} diff --git a/src/mock/ray/gcs/gcs_server/gcs_node_manager.h b/src/mock/ray/gcs/gcs_server/gcs_node_manager.h index 919324cb0996..67161c0a6456 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/mock/ray/gcs/gcs_server/gcs_node_manager.h @@ -14,7 +14,8 @@ #pragma once -#include "gmock/gmock.h" +#include + #include "ray/gcs/gcs_server/gcs_node_manager.h" namespace ray { diff --git a/src/mock/ray/gcs/gcs_server/gcs_placement_group_mgr.h b/src/mock/ray/gcs/gcs_server/gcs_placement_group_manager.h similarity index 93% rename from src/mock/ray/gcs/gcs_server/gcs_placement_group_mgr.h rename to src/mock/ray/gcs/gcs_server/gcs_placement_group_manager.h index 97d02a932d94..433bc132bddb 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_placement_group_mgr.h +++ b/src/mock/ray/gcs/gcs_server/gcs_placement_group_manager.h @@ -11,17 +11,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" -namespace ray { -namespace gcs { +#pragma once -class MockGcsPlacementGroup : public GcsPlacementGroup { - public: -}; +#include -} // namespace gcs -} // namespace ray +#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_placement_group_scheduler.h b/src/mock/ray/gcs/gcs_server/gcs_placement_group_scheduler.h index a0d6f84d1663..e35bcd45940b 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_placement_group_scheduler.h +++ b/src/mock/ray/gcs/gcs_server/gcs_placement_group_scheduler.h @@ -12,6 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#pragma once + +#include + +#include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" + namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_resource_manager.h b/src/mock/ray/gcs/gcs_server/gcs_resource_manager.h index eba879e1ad00..a3865ab02968 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_resource_manager.h +++ b/src/mock/ray/gcs/gcs_server/gcs_resource_manager.h @@ -12,10 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +#pragma once + +#include + #include "ray/common/asio/instrumented_io_context.h" +#include "ray/gcs/gcs_server/gcs_resource_manager.h" namespace ray { namespace gcs { + static instrumented_io_context __mock_io_context_; static ClusterResourceManager __mock_cluster_resource_manager_(__mock_io_context_); static GcsNodeManager __mock_gcs_node_manager_( diff --git a/src/mock/ray/gcs/gcs_server/gcs_server.h b/src/mock/ray/gcs/gcs_server/gcs_server.h deleted file mode 100644 index 8c80774b4078..000000000000 --- a/src/mock/ray/gcs/gcs_server/gcs_server.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace ray { -namespace gcs { - -class MockGcsServerConfig : public GcsServerConfig { - public: -}; - -} // namespace gcs -} // namespace ray - -namespace ray { -namespace gcs { - -class MockGcsServer : public GcsServer { - public: -}; - -} // namespace gcs -} // namespace ray diff --git a/src/mock/ray/gcs/gcs_server/gcs_table_storage.h b/src/mock/ray/gcs/gcs_server/gcs_table_storage.h deleted file mode 100644 index 4b229784b8cb..000000000000 --- a/src/mock/ray/gcs/gcs_server/gcs_table_storage.h +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace ray { -namespace gcs { - -template -class MockGcsTable : public GcsTable { - public: - MOCK_METHOD(Status, - Put, - (const Key &key, const Data &value, const StatusCallback &callback), - (override)); - MOCK_METHOD(Status, - Delete, - (const Key &key, const StatusCallback &callback), - (override)); - MOCK_METHOD(Status, - BatchDelete, - (const std::vector &keys, const StatusCallback &callback), - (override)); -}; - -} // namespace gcs -} // namespace ray - -namespace ray { -namespace gcs { - -template -class MockGcsTableWithJobId : public GcsTableWithJobId { - public: - MOCK_METHOD(Status, - Put, - (const Key &key, const Data &value, const StatusCallback &callback), - (override)); - MOCK_METHOD(Status, - Delete, - (const Key &key, const StatusCallback &callback), - (override)); - MOCK_METHOD(Status, - BatchDelete, - (const std::vector &keys, const StatusCallback &callback), - (override)); - MOCK_METHOD(JobID, GetJobIdFromKey, (const Key &key), (override)); -}; - -} // namespace gcs -} // namespace ray - -namespace ray { -namespace gcs { - -class MockGcsJobTable : public GcsJobTable { - public: -}; - -} // namespace gcs -} // namespace ray - -namespace ray { -namespace gcs { - -class MockGcsActorTable : public GcsActorTable { - public: - MockGcsActorTable() : GcsActorTable(nullptr) {} - - MOCK_METHOD(JobID, GetJobIdFromKey, (const ActorID &key), (override)); -}; - -} // namespace gcs -} // namespace ray - -namespace ray { -namespace gcs { - -class MockGcsPlacementGroupTable : public GcsPlacementGroupTable { - public: -}; - -} // namespace gcs -} // namespace ray - -namespace ray { -namespace gcs { - -class MockGcsNodeTable : public GcsNodeTable { - public: - MockGcsNodeTable() : GcsNodeTable(nullptr){}; - - MOCK_METHOD(Status, - Put, - (const NodeID &key, - const GcsNodeInfo &value, - const StatusCallback &callback), - (override)); -}; - -} // namespace gcs -} // namespace ray - -namespace ray { -namespace gcs { - -class MockGcsWorkerTable : public GcsWorkerTable { - public: -}; - -} // namespace gcs -} // namespace ray - -namespace ray { -namespace gcs { - -class MockGcsTableStorage : public GcsTableStorage { - public: - MockGcsTableStorage() : GcsTableStorage(nullptr) {} - - MOCK_METHOD((GcsNodeTable &), NodeTable, (), (override)); -}; - -} // namespace gcs -} // namespace ray - -namespace ray { -namespace gcs { - -class MockRedisGcsTableStorage : public RedisGcsTableStorage { - public: -}; - -} // namespace gcs -} // namespace ray - -namespace ray { -namespace gcs { - -class MockInMemoryGcsTableStorage : public InMemoryGcsTableStorage { - public: -}; - -} // namespace gcs -} // namespace ray diff --git a/src/mock/ray/gcs/gcs_server/gcs_task_manager.h b/src/mock/ray/gcs/gcs_server/gcs_task_manager.h index 67601dfd56a7..e3c8222a01d4 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_task_manager.h +++ b/src/mock/ray/gcs/gcs_server/gcs_task_manager.h @@ -12,6 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#pragma once + +#include + +#include "ray/gcs/gcs_server/gcs_task_manager.h" + namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_worker_manager.h b/src/mock/ray/gcs/gcs_server/gcs_worker_manager.h index 7e993fc4814a..deb459f53a65 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_worker_manager.h +++ b/src/mock/ray/gcs/gcs_server/gcs_worker_manager.h @@ -12,6 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#pragma once + +#include + +#include "ray/gcs/gcs_server/gcs_worker_manager.h" + namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 50e782c6b17a..d8791b891079 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -266,13 +266,11 @@ ray_cc_library( name = "gcs_placement_group", srcs = ["gcs_placement_group.cc"], hdrs = ["gcs_placement_group.h"], - implementation_deps = [ - "//src/ray/stats:stats_lib", - ], deps = [ "//src/ray/common:id", "//src/ray/common:task_common", "//src/ray/protobuf:gcs_service_cc_proto", + "//src/ray/stats:stats_lib", "//src/ray/util:counter_map", "//src/ray/util:time", ], @@ -299,6 +297,29 @@ ray_cc_library( ], ) +ray_cc_library( + name = "gcs_placement_group_manager", + srcs = ["gcs_placement_group_manager.cc"], + hdrs = ["gcs_placement_group_manager.h"], + deps = [ + ":gcs_init_data", + ":gcs_node_manager", + ":gcs_placement_group", + ":gcs_placement_group_scheduler", + ":gcs_resource_manager", + ":gcs_table_storage", + ":gcs_usage_stats_client", + "//src/ray/common:asio", + "//src/ray/common:id", + "//src/ray/common:task_common", + "//src/ray/protobuf:gcs_cc_proto", + "//src/ray/util:counter_map", + "//src/ray/util:exponential_backoff", + "//src/ray/util:time", + "@com_google_absl//absl/container:flat_hash_map", + ], +) + ray_cc_library( name = "grpc_service_interfaces", hdrs = [ @@ -385,13 +406,11 @@ ray_cc_library( srcs = [ "gcs_actor_manager.cc", "gcs_autoscaler_state_manager.cc", - "gcs_placement_group_mgr.cc", "gcs_server.cc", ], hdrs = [ "gcs_actor_manager.h", "gcs_autoscaler_state_manager.h", - "gcs_placement_group_mgr.h", "gcs_server.h", ], deps = [ @@ -404,6 +423,7 @@ ray_cc_library( ":gcs_kv_manager", ":gcs_node_manager", ":gcs_placement_group", + ":gcs_placement_group_manager", ":gcs_placement_group_scheduler", ":gcs_pubsub_handler", ":gcs_resource_manager", diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc index c6de00a352f6..706c3ab6cdba 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc @@ -21,7 +21,7 @@ #include "ray/gcs/gcs_server/gcs_actor_manager.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" -#include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" +#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" #include "ray/gcs/gcs_server/state_util.h" #include "ray/gcs/pb_util.h" #include "ray/util/string_utils.h" diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc b/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc rename to src/ray/gcs/gcs_server/gcs_placement_group_manager.cc index 9c4007df4a8e..c213394b923d 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" +#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h b/src/ray/gcs/gcs_server/gcs_placement_group_manager.h similarity index 98% rename from src/ray/gcs/gcs_server/gcs_placement_group_mgr.h rename to src/ray/gcs/gcs_server/gcs_placement_group_manager.h index 3ee4264d42df..24130ccc7df0 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_mgr.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_manager.h @@ -33,10 +33,8 @@ #include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/usage_stats_client.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" -#include "ray/rpc/worker/core_worker_client.h" #include "ray/util/counter_map.h" #include "ray/util/exponential_backoff.h" #include "ray/util/time.h" @@ -52,7 +50,7 @@ namespace gcs { /// the head of the queue and schedule it. If schedule success, using the /// SchedulePendingPlacementGroups() Immediately. else wait for a short time beforw using /// SchedulePendingPlacementGroups() next time. -class GcsPlacementGroupManager : public rpc::PlacementGroupInfoHandler { +class GcsPlacementGroupManager : public rpc::PlacementGroupInfoGcsServiceHandler { public: /// Create a GcsPlacementGroupManager /// diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 9648b6082ed5..be8a25d3b2f3 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -26,7 +26,7 @@ #include "ray/gcs/gcs_server/gcs_actor_manager.h" #include "ray/gcs/gcs_server/gcs_autoscaler_state_manager.h" #include "ray/gcs/gcs_server/gcs_job_manager.h" -#include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" +#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/gcs_worker_manager.h" #include "ray/gcs/gcs_server/grpc_services.h" @@ -522,10 +522,12 @@ void GcsServer::InitGcsPlacementGroupManager(const GcsInitData &gcs_init_data) { [this](const JobID &job_id) { return gcs_job_manager_->GetJobConfig(job_id)->ray_namespace(); }); - // Initialize by gcs tables data. + gcs_placement_group_manager_->Initialize(gcs_init_data); rpc_server_.RegisterService(std::make_unique( - io_context_provider_.GetDefaultIOContext(), *gcs_placement_group_manager_)); + io_context_provider_.GetDefaultIOContext(), + *gcs_placement_group_manager_, + RayConfig::instance().gcs_max_active_rpcs_per_handler())); } GcsServer::StorageType GcsServer::GetStorageType() const { diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h index 0c8d68bc7aad..1ce4129ca697 100644 --- a/src/ray/gcs/gcs_server/grpc_service_interfaces.h +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -227,5 +227,35 @@ class RayEventExportGcsServiceHandler { SendReplyCallback send_reply_callback) = 0; }; +class PlacementGroupInfoGcsServiceHandler { + public: + virtual ~PlacementGroupInfoGcsServiceHandler() = default; + + virtual void HandleCreatePlacementGroup(CreatePlacementGroupRequest request, + CreatePlacementGroupReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleRemovePlacementGroup(RemovePlacementGroupRequest request, + RemovePlacementGroupReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetPlacementGroup(GetPlacementGroupRequest request, + GetPlacementGroupReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetAllPlacementGroup(GetAllPlacementGroupRequest request, + GetAllPlacementGroupReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleWaitPlacementGroupUntilReady( + WaitPlacementGroupUntilReadyRequest request, + WaitPlacementGroupUntilReadyReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetNamedPlacementGroup(GetNamedPlacementGroupRequest request, + GetNamedPlacementGroupReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc index cf6fad71f43b..4e63a85a4e12 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -132,5 +132,24 @@ void RayEventExportGrpcService::InitServerCallFactories( RPC_SERVICE_HANDLER(RayEventExportGcsService, AddEvents, max_active_rpcs_per_handler_) } +void PlacementGroupInfoGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + RPC_SERVICE_HANDLER( + PlacementGroupInfoGcsService, CreatePlacementGroup, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + PlacementGroupInfoGcsService, RemovePlacementGroup, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + PlacementGroupInfoGcsService, GetPlacementGroup, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + PlacementGroupInfoGcsService, GetNamedPlacementGroup, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + PlacementGroupInfoGcsService, GetAllPlacementGroup, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(PlacementGroupInfoGcsService, + WaitPlacementGroupUntilReady, + max_active_rpcs_per_handler_) +} + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/gcs_server/grpc_services.h index a4faaf4bf453..8108dc3760ce 100644 --- a/src/ray/gcs/gcs_server/grpc_services.h +++ b/src/ray/gcs/gcs_server/grpc_services.h @@ -242,5 +242,28 @@ class RayEventExportGrpcService : public GrpcService { int64_t max_active_rpcs_per_handler_; }; +class PlacementGroupInfoGrpcService : public GrpcService { + public: + explicit PlacementGroupInfoGrpcService(instrumented_io_context &io_service, + PlacementGroupInfoGcsServiceHandler &handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler) {} + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + PlacementGroupInfoGcsService::AsyncService service_; + PlacementGroupInfoGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index d65aa6f83b35..51a7dde30e5b 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -12,15 +12,15 @@ ray_cc_test( ) ray_cc_test( - name = "gcs_placement_group_mgr_mock_test", + name = "gcs_placement_group_manager_mock_test", size = "small", srcs = [ - "gcs_placement_group_mgr_mock_test.cc", + "gcs_placement_group_manager_mock_test.cc", ], tags = ["team:core"], deps = [ "//:ray_mock", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_placement_group_manager", "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", @@ -148,10 +148,10 @@ ray_cc_test( ) ray_cc_test( - name = "gcs_placement_group_mgr_test", + name = "gcs_placement_group_manager_test", size = "small", srcs = [ - "gcs_placement_group_mgr_test.cc", + "gcs_placement_group_manager_test.cc", ], tags = [ "no_tsan", @@ -160,7 +160,7 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_placement_group_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:counter_map", diff --git a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc index 0149b081b648..5cb34e043dfa 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc @@ -27,7 +27,7 @@ #include "gtest/gtest.h" #include "mock/ray/gcs/gcs_server/gcs_actor_manager.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" -#include "mock/ray/gcs/gcs_server/gcs_placement_group_mgr.h" +#include "mock/ray/gcs/gcs_server/gcs_placement_group_manager.h" #include "mock/ray/gcs/store_client/store_client.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/asio/instrumented_io_context.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc similarity index 97% rename from src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_mock_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc index 643d4f454112..07caca6d2a56 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_mock_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc @@ -12,21 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include + #include #include -// clang-format off -#include "gtest/gtest.h" -#include "gmock/gmock.h" -#include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" -#include "ray/raylet/scheduling/cluster_resource_manager.h" + #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" -#include "mock/ray/gcs/gcs_server/gcs_placement_group_mgr.h" #include "mock/ray/gcs/gcs_server/gcs_placement_group_scheduler.h" #include "mock/ray/gcs/gcs_server/gcs_resource_manager.h" #include "mock/ray/gcs/store_client/store_client.h" -#include "ray/util/counter_map.h" +#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" #include "ray/gcs/tests/gcs_test_util.h" -// clang-format on +#include "ray/raylet/scheduling/cluster_resource_manager.h" +#include "ray/util/counter_map.h" using namespace ::testing; // NOLINT using namespace ray; // NOLINT diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc rename to src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc index 359814b61514..15876316cac2 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_mgr_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" +#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" #include diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index dbbd1ddbf3a2..2750b945c35a 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -112,9 +112,14 @@ namespace rpc { #define ACTOR_INFO_SERVICE_RPC_HANDLER(HANDLER, MAX_ACTIVE_RPCS) \ RPC_SERVICE_HANDLER(ActorInfoGcsService, HANDLER, MAX_ACTIVE_RPCS) -#define PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(PlacementGroupInfoGcsService, \ - HANDLER, \ +#define MONITOR_SERVICE_RPC_HANDLER(HANDLER) \ + RPC_SERVICE_HANDLER(MonitorGcsService, \ + HANDLER, \ + RayConfig::instance().gcs_max_active_rpcs_per_handler()) + +#define OBJECT_INFO_SERVICE_RPC_HANDLER(HANDLER) \ + RPC_SERVICE_HANDLER(ObjectInfoGcsService, \ + HANDLER, \ RayConfig::instance().gcs_max_active_rpcs_per_handler()) #define GCS_RPC_SEND_REPLY(send_reply_callback, reply, status) \ @@ -209,70 +214,7 @@ class ActorInfoGrpcService : public GrpcService { ActorInfoGcsServiceHandler &service_handler_; }; -class PlacementGroupInfoGcsServiceHandler { - public: - virtual ~PlacementGroupInfoGcsServiceHandler() = default; - - virtual void HandleCreatePlacementGroup(CreatePlacementGroupRequest request, - CreatePlacementGroupReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleRemovePlacementGroup(RemovePlacementGroupRequest request, - RemovePlacementGroupReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetPlacementGroup(GetPlacementGroupRequest request, - GetPlacementGroupReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetAllPlacementGroup(GetAllPlacementGroupRequest request, - GetAllPlacementGroupReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleWaitPlacementGroupUntilReady( - WaitPlacementGroupUntilReadyRequest request, - WaitPlacementGroupUntilReadyReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetNamedPlacementGroup(GetNamedPlacementGroupRequest request, - GetNamedPlacementGroupReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -/// The `GrpcService` for `PlacementGroupInfoGcsService`. -class PlacementGroupInfoGrpcService : public GrpcService { - public: - /// Constructor. - /// - /// \param[in] handler The service handler that actually handle the requests. - explicit PlacementGroupInfoGrpcService(instrumented_io_context &io_service, - PlacementGroupInfoGcsServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler) {} - - protected: - grpc::Service &GetGrpcService() override { return service_; } - - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(CreatePlacementGroup); - PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(RemovePlacementGroup); - PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(GetPlacementGroup); - PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(GetNamedPlacementGroup); - PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(GetAllPlacementGroup); - PLACEMENT_GROUP_INFO_SERVICE_RPC_HANDLER(WaitPlacementGroupUntilReady); - } - - private: - /// The grpc async service object. - PlacementGroupInfoGcsService::AsyncService service_; - /// The service handler that actually handle the requests. - PlacementGroupInfoGcsServiceHandler &service_handler_; -}; - using ActorInfoHandler = ActorInfoGcsServiceHandler; -using PlacementGroupInfoHandler = PlacementGroupInfoGcsServiceHandler; } // namespace rpc } // namespace ray From 3da782ab3899c4ff0e3e0cd92eff1c4f67739f8a Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Thu, 28 Aug 2025 13:19:15 -0700 Subject: [PATCH 344/634] Support cpu tensor transfer with NIXL in GPU Objects (#56055) The original pr #55793 has been reverted because of lint issues, this pr is to apply the changes again. --- .../channel/serialization_context.py | 4 ++- .../collective/collective_tensor_transport.py | 16 +++++++++-- .../collective/nixl_tensor_transport.py | 13 ++++++++- .../collective/tensor_transport_manager.py | 2 -- python/ray/experimental/collective/util.py | 17 +++++++++++- .../gpu_object_manager/gpu_object_store.py | 27 +++++++++---------- python/ray/tests/test_gpu_objects_nixl.py | 24 ++++++++++++----- python/ray/util/collective/types.py | 3 +++ 8 files changed, 77 insertions(+), 29 deletions(-) diff --git a/python/ray/experimental/channel/serialization_context.py b/python/ray/experimental/channel/serialization_context.py index 40784b38f409..d06a2d9e098b 100644 --- a/python/ray/experimental/channel/serialization_context.py +++ b/python/ray/experimental/channel/serialization_context.py @@ -97,7 +97,9 @@ def serialize_tensor( from ray.experimental.channel import ChannelContext ctx = ChannelContext.get_current() - if self._use_external_transport and tensor.device == ctx.torch_device: + if self._use_external_transport and ( + ctx._torch_device is None or ctx._torch_device == tensor.device + ): # External transport is enabled and we found a tensor that matches # our device. Add the actual tensor to a buffer. The buffer of # tensors should later be popped by the caller and sent via diff --git a/python/ray/experimental/collective/collective_tensor_transport.py b/python/ray/experimental/collective/collective_tensor_transport.py index 7bf39aed7b25..2edb06d32072 100644 --- a/python/ray/experimental/collective/collective_tensor_transport.py +++ b/python/ray/experimental/collective/collective_tensor_transport.py @@ -40,8 +40,19 @@ def __ray_get_tensor_transport_metadata__( # it could take arbitrarily long and we don't want to trigger a spurious # timeout. gpu_object = gpu_object_store.wait_and_get_object(obj_id) + tensor_meta = [] + device = None + if gpu_object: + device = gpu_object[0].device + for t in gpu_object: + if t.device.type != device.type: + raise ValueError( + "All tensors in one GPU object must be the same device type." + ) + tensor_meta.append((t.shape, t.dtype)) return CollectiveTransportMetadata( - tensor_meta=[(t.shape, t.dtype) for t in gpu_object], + tensor_meta=tensor_meta, + tensor_device=device, ) # Submit a Ray actor task to the source actor to get the tensor metadata. @@ -130,10 +141,11 @@ def recv_multiple_tensors( def send_multiple_tensors( tensors: List["torch.Tensor"], communicator_metadata: CollectiveCommunicatorMetadata, - device: "torch.device", ): import ray.util.collective as collective + device = tensors[0].device if tensors else None + for tensor in tensors: if tensor.device.type != device.type: # TODO(swang): Right now there is no way to catch this error diff --git a/python/ray/experimental/collective/nixl_tensor_transport.py b/python/ray/experimental/collective/nixl_tensor_transport.py index eb86f0cb9a3d..dcedbc11bf41 100644 --- a/python/ray/experimental/collective/nixl_tensor_transport.py +++ b/python/ray/experimental/collective/nixl_tensor_transport.py @@ -45,14 +45,25 @@ def __ray_get_tensor_transport_metadata__( from ray.util.collective.collective import get_group_handle nixl_backend: NixlBackend = get_group_handle(NIXL_GROUP_NAME) + device = None + tensor_meta = [] if gpu_object: serialized_descs, agent_meta = nixl_backend.get_nixl_metadata( gpu_object ) + # We assume all tensors in one GPU object have the same device type. + device = gpu_object[0].device + for t in gpu_object: + if t.device.type != device.type: + raise ValueError( + "All tensors in one GPU object must be the same device type." + ) + tensor_meta.append((t.shape, t.dtype)) else: serialized_descs, agent_meta = None, None return NixlTransportMetadata( - tensor_meta=[(t.shape, t.dtype) for t in gpu_object], + tensor_meta=tensor_meta, + tensor_device=device, nixl_serialized_descs=serialized_descs, nixl_agent_meta=agent_meta, ) diff --git a/python/ray/experimental/collective/tensor_transport_manager.py b/python/ray/experimental/collective/tensor_transport_manager.py index 9f3896699393..18d554d2c6d7 100644 --- a/python/ray/experimental/collective/tensor_transport_manager.py +++ b/python/ray/experimental/collective/tensor_transport_manager.py @@ -143,7 +143,6 @@ def recv_multiple_tensors( def send_multiple_tensors( tensors: List["torch.Tensor"], communicator_metadata: CommunicatorMetadata, - device: "torch.device", ): """ Send multiple tensors to the destination actor. @@ -151,5 +150,4 @@ def send_multiple_tensors( Args: tensors: The tensors to send. communicator_metadata: The communicator metadata for the send/recv operation. - device: The device to send the tensors to. """ diff --git a/python/ray/experimental/collective/util.py b/python/ray/experimental/collective/util.py index fc6ef64229fb..6e9297dba37c 100644 --- a/python/ray/experimental/collective/util.py +++ b/python/ray/experimental/collective/util.py @@ -1,4 +1,4 @@ -from typing import Tuple +from typing import Tuple, TYPE_CHECKING from contextlib import closing import socket @@ -11,6 +11,9 @@ CollectiveTensorTransport, ) +if TYPE_CHECKING: + import torch + # Singleton instances for tensor transport managers _nixl_tensor_transport_manager = None _collective_tensor_transport_manager = None @@ -41,6 +44,18 @@ def get_tensor_transport_manager( raise ValueError(f"Unsupported tensor transport protocol: {tensor_transport}") +def device_match_transport(device: "torch.device", tensor_transport: Backend) -> bool: + """Check if the device matches the transport.""" + if tensor_transport == Backend.NIXL: + return device.type == "cuda" or device.type == "cpu" + elif tensor_transport == Backend.TORCH_GLOO: + return device.type == "cpu" + elif tensor_transport == Backend.NCCL: + return device.type == "cuda" + else: + raise ValueError(f"Unsupported tensor transport protocol: {tensor_transport}") + + def find_free_port() -> int: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: s.bind(("", 0)) diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_store.py b/python/ray/experimental/gpu_object_manager/gpu_object_store.py index 93fcbc20f54d..a1e76392df26 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_store.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_store.py @@ -11,6 +11,9 @@ TensorTransportMetadata, ) +from ray.experimental.collective import get_tensor_transport_manager +from ray.experimental.collective.util import device_match_transport + try: import torch except ImportError: @@ -25,14 +28,6 @@ TensorTransportEnum.NIXL: Backend.NIXL, } -COLLECTIVE_BACKEND_TO_TORCH_DEVICE = { - Backend.NCCL: torch.device("cuda"), - Backend.TORCH_GLOO: torch.device("cpu"), - # TODO(Qiaolin-Yu): NIXL could also transfer tensors from CPU to CPU. - # More details in https://github.com/ray-project/ray/issues/55587. - Backend.NIXL: torch.device("cuda"), -} - def _tensor_transport_to_collective_backend( tensor_transport: TensorTransportEnum, @@ -61,15 +56,15 @@ def __ray_send__( tensors = gpu_object_store.get_object(obj_id) backend = collective.get_group_handle(communicator_meta.communicator_name).backend() - device = COLLECTIVE_BACKEND_TO_TORCH_DEVICE[backend] - - from ray.experimental.collective import get_tensor_transport_manager tensor_transport_manager = get_tensor_transport_manager(backend) + if tensors and not device_match_transport(tensors[0].device, backend): + raise ValueError( + f"Tensor transport backend {backend} does not support tensor transfer on device {tensors[0].device}." + ) tensor_transport_manager.send_multiple_tensors( tensors, communicator_meta, - device=device, ) @@ -82,14 +77,16 @@ def __ray_recv__( """Helper function that runs on the dst actor to receive tensors from the src actor.""" from ray._private.worker import global_worker - from ray.experimental.collective import get_tensor_transport_manager - backend = collective.get_group_handle(communicator_meta.communicator_name).backend() - device = COLLECTIVE_BACKEND_TO_TORCH_DEVICE[backend] + device = tensor_transport_meta.tensor_device tensor_meta = tensor_transport_meta.tensor_meta gpu_object_store = global_worker.gpu_object_manager.gpu_object_store + if tensor_meta and not device_match_transport(device, backend): + raise ValueError( + f"Tensor transport backend {backend} does not support tensor transfer on device {device}." + ) tensors = [] for meta in tensor_meta: shape, dtype = meta diff --git a/python/ray/tests/test_gpu_objects_nixl.py b/python/ray/tests/test_gpu_objects_nixl.py index e46a24358d37..3a65e8ea7eb9 100644 --- a/python/ray/tests/test_gpu_objects_nixl.py +++ b/python/ray/tests/test_gpu_objects_nixl.py @@ -7,10 +7,11 @@ @ray.remote(num_gpus=1, num_cpus=0, enable_tensor_transport=True) class GPUTestActor: @ray.method(tensor_transport="nixl") - def echo(self, data): - return data.to("cuda") + def echo(self, data, device): + return data.to(device) - def sum(self, data): + def sum(self, data, device): + assert data.device.type == device return data.sum().item() @@ -23,12 +24,21 @@ def test_p2p(ray_start_regular): # Create test tensor tensor = torch.tensor([1, 2, 3]) - ref = src_actor.echo.remote(tensor) + + tensor1 = torch.tensor([4, 5, 6]) + + # Test GPU to GPU transfer + ref = src_actor.echo.remote(tensor, "cuda") # Trigger tensor transfer from src to dst actor - result = dst_actor.sum.remote(ref) + result = dst_actor.sum.remote(ref, "cuda") assert tensor.sum().item() == ray.get(result) + # Test CPU to CPU transfer + ref1 = src_actor.echo.remote(tensor1, "cpu") + result1 = dst_actor.sum.remote(ref1, "cpu") + assert tensor1.sum().item() == ray.get(result1) + @pytest.mark.parametrize("ray_start_regular", [{"num_gpus": 1}], indirect=True) def test_intra_gpu_tensor_transfer(ray_start_regular): @@ -37,8 +47,8 @@ def test_intra_gpu_tensor_transfer(ray_start_regular): tensor = torch.tensor([1, 2, 3]) # Intra-actor communication for pure GPU tensors - ref = actor.echo.remote(tensor) - result = actor.sum.remote(ref) + ref = actor.echo.remote(tensor, "cuda") + result = actor.sum.remote(ref, "cuda") assert tensor.sum().item() == ray.get(result) diff --git a/python/ray/util/collective/types.py b/python/ray/util/collective/types.py index 06a05ae71549..e46737c5a033 100644 --- a/python/ray/util/collective/types.py +++ b/python/ray/util/collective/types.py @@ -61,9 +61,12 @@ class TensorTransportMetadata: Args: tensor_meta: A list of tuples, each containing the shape and dtype of a tensor. + tensor_device: The device of the tensor. Currently, we require all tensors in the + list have the same device type. """ tensor_meta: List[Tuple["torch.Size", "torch.dtype"]] + tensor_device: Optional["torch.device"] = None @dataclass From c44a7ca23b24133f54f0c1049d8addd4cbe162bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Q=E6=96=87=E4=B8=BE?= <39372502+orangeQWJ@users.noreply.github.com> Date: Fri, 29 Aug 2025 04:22:39 +0800 Subject: [PATCH 345/634] [Data] Prevent unbounded growth of_StatsActor.datasets (#55925) Background Closes #43924. The _StatsActor suffers from unbounded memory usage in long-running clusters. Its core metadata dictionaries, `datasets` and `dataset_metadatas`, lacked a proper garbage collection (GC) mechanism. This could lead to an Out-of-Memory (OOM) error in the _StatsActor when a large number of datasets are created. Solution This patch implements an eviction policy for _StatsActor based on the `max_stats` configuration to effectively limit the size of the `datasets` and `dataset_metadatas` dictionaries, preventing their unbounded growth. The implementation details are as follows: 1. **Optimize Queue Implementation** The old, unused `fifo_queue` field has been removed. It is replaced by a new `collections.deque`, `finished_datasets_queue`, which serves as a more efficient FIFO queue for storing the tags of completed datasets. 2. **Implement Eviction Logic** - When a dataset's status is updated to `FINISHED` or `FAILED`, its tag is appended to the `finished_datasets_queue`. - A check is then immediately performed to see if the total number of entries in the `datasets` dictionary exceeds `max_stats`. - If the limit is exceeded, the oldest dataset tag is popped from the front of the `finished_datasets_queue`, and the corresponding entries are synchronously deleted from the `datasets` and `dataset_metadatas` dictionaries. 3. **Clarify Limitation Strategy** `max_stats` is not a strict hard limit. Since the eviction logic is only triggered when a dataset completes (`FINISHED` or `FAILED`), it is possible for the number of `RUNNING` datasets to cause the total entry count to temporarily exceed `max_stats`. This design ensures that metadata for in-progress tasks is never evicted, while still effectively preventing unbounded memory growth and OOM errors by cleaning up the oldest completed data as soon as a task finishes. Testing To verify the correctness of this fix, a new unit test, `test_stats_actor_datasets_eviction`, has been added. This test sets a low `max_stats` value and asserts that the oldest finished dataset is correctly evicted when the limit is surpassed. --------- Signed-off-by: qiwenju Co-authored-by: qiwenju --- python/ray/data/_internal/stats.py | 13 ++++- python/ray/data/tests/test_stats.py | 75 +++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/python/ray/data/_internal/stats.py b/python/ray/data/_internal/stats.py index 7f4222f68426..cba592182e65 100644 --- a/python/ray/data/_internal/stats.py +++ b/python/ray/data/_internal/stats.py @@ -163,7 +163,6 @@ def __init__(self, max_stats=1000): self.last_time = {} self.start_time = {} self.max_stats = max_stats - self.fifo_queue = [] # Assign dataset uuids with a global counter. self.next_dataset_id = 0 @@ -177,6 +176,10 @@ def __init__(self, max_stats=1000): self._metadata_exporter = get_dataset_metadata_exporter() self.dataset_metadatas: Dict[str, DatasetMetadata] = {} + # A FIFO queue of dataset_tags for finished datasets. This is used to + # efficiently evict the oldest finished datasets when max_stats is reached. + self.finished_datasets_queue = collections.deque() + # Ray Data dashboard metrics # Everything is a gauge because we need to reset all of # a dataset's metrics to 0 after each finishes execution. @@ -570,6 +573,14 @@ def update_dataset(self, dataset_tag: str, state: Dict[str, Any]): self.update_dataset_metadata_operator_states(dataset_tag, operator_states) + # Evict the oldest finished datasets to ensure the `max_stats` limit is enforced. + if state["state"] in {DatasetState.FINISHED.name, DatasetState.FAILED.name}: + self.finished_datasets_queue.append(dataset_tag) + while len(self.datasets) > self.max_stats and self.finished_datasets_queue: + tag_to_evict = self.finished_datasets_queue.popleft() + self.datasets.pop(tag_to_evict, None) + self.dataset_metadatas.pop(tag_to_evict, None) + def get_datasets(self, job_id: Optional[str] = None): if not job_id: return self.datasets diff --git a/python/ray/data/tests/test_stats.py b/python/ray/data/tests/test_stats.py index cbe5b5dfa8ba..71edcc8f94a4 100644 --- a/python/ray/data/tests/test_stats.py +++ b/python/ray/data/tests/test_stats.py @@ -22,6 +22,7 @@ from ray.data._internal.execution.backpressure_policy.backpressure_policy import ( BackpressurePolicy, ) +from ray.data._internal.execution.dataset_state import DatasetState from ray.data._internal.execution.interfaces.op_runtime_metrics import TaskDurationStats from ray.data._internal.execution.interfaces.physical_operator import PhysicalOperator from ray.data._internal.stats import ( @@ -29,6 +30,7 @@ NodeMetrics, StatsManager, _get_or_create_stats_actor, + _StatsActor, ) from ray.data._internal.util import MemoryProfiler from ray.data.context import DataContext @@ -1915,6 +1917,79 @@ def test_stats_actor_datasets(ray_start_cluster): assert value["state"] == "FINISHED" +def test_stats_actor_datasets_eviction(ray_start_cluster): + """ + Tests that finished datasets are evicted from the _StatsActor when + the number of datasets exceeds the configured `max_stats` limit. + """ + # Set a low max_stats limit to easily trigger eviction. + max_stats = 2 + # Create a dedicated _StatsActor for this test to avoid interfering + # with the global actor. + stats_actor = _StatsActor.remote(max_stats=max_stats) + + # Patch the function that retrieves the stats actor to return our + # test-specific actor instance. + with patch( + "ray.data._internal.stats._get_or_create_stats_actor", + return_value=stats_actor, + ): + + def check_ds_finished(ds_name): + """Helper to check if a dataset is marked as FINISHED in the actor.""" + datasets = ray.get(stats_actor.get_datasets.remote()) + ds_tag = next((tag for tag in datasets if tag.startswith(ds_name)), None) + if not ds_tag: + return False + return datasets[ds_tag]["state"] == DatasetState.FINISHED.name + + # --- DS1 --- + # Create and materialize the first dataset. + ds1 = ray.data.range(1, override_num_blocks=1) + ds1.set_name("ds1") + ds1.materialize() + # Wait until the actor has been updated with the FINISHED state. + wait_for_condition(lambda: check_ds_finished("ds1")) + + # --- DS2 --- + # Create and materialize the second dataset. + # This brings the total number of datasets to the `max_stats` limit. + ds2 = ray.data.range(1, override_num_blocks=1) + ds2.set_name("ds2") + ds2.materialize() + wait_for_condition(lambda: check_ds_finished("ds2")) + + # --- Verify state before eviction --- + # At this point, both ds1 and ds2 should be in the actor. + datasets = ray.get(stats_actor.get_datasets.remote()) + names_in_actor = {k.split("_")[0] for k in datasets.keys()} + assert names_in_actor == {"ds1", "ds2"} + + # --- DS3 --- + # Create and materialize the third dataset. This should trigger the + # eviction of the oldest finished dataset (ds1). + ds3 = ray.data.range(1, override_num_blocks=1) + ds3.set_name("ds3") + ds3.materialize() + + def check_eviction(): + """ + Helper to check that the actor state reflects the eviction. + The actor should now contain ds2 and ds3, but not ds1. + """ + datasets = ray.get(stats_actor.get_datasets.remote()) + # The eviction happens asynchronously, so we might briefly see 3 datasets. + # We wait until the count is back to 2. + if len(datasets) == max_stats + 1: + return False + names = {k.split("_")[0] for k in datasets.keys()} + assert names == {"ds2", "ds3"} + return True + + # Wait until the eviction has occurred and the actor state is correct. + wait_for_condition(check_eviction) + + @patch.object(StatsManager, "STATS_ACTOR_UPDATE_INTERVAL_SECONDS", new=0.5) @patch.object(StatsManager, "_stats_actor_handle") @patch.object(StatsManager, "UPDATE_THREAD_INACTIVITY_LIMIT", new=1) From 2bd37f52d790b42325fc6c9621e25ed3d201bef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Q=E6=96=87=E4=B8=BE?= <39372502+orangeQWJ@users.noreply.github.com> Date: Fri, 29 Aug 2025 04:23:36 +0800 Subject: [PATCH 346/634] Update async_api.rst (#56042) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Q文举 <39372502+orangeQWJ@users.noreply.github.com> --- doc/source/ray-core/actors/async_api.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/source/ray-core/actors/async_api.rst b/doc/source/ray-core/actors/async_api.rst index 36f5296fdba4..7a64a774c85a 100644 --- a/doc/source/ray-core/actors/async_api.rst +++ b/doc/source/ray-core/actors/async_api.rst @@ -63,14 +63,14 @@ async frameworks like aiohttp, aioredis, etc. .. testoutput:: :options: +MOCK - (AsyncActor pid=40293) started - (AsyncActor pid=40293) started - (AsyncActor pid=40293) started - (AsyncActor pid=40293) started - (AsyncActor pid=40293) finished - (AsyncActor pid=40293) finished - (AsyncActor pid=40293) finished - (AsyncActor pid=40293) finished + (AsyncActor pid=9064) Waiting for other coroutines to start. + (AsyncActor pid=9064) Waiting for other coroutines to start. + (AsyncActor pid=9064) Waiting for other coroutines to start. + (AsyncActor pid=9064) All coroutines are executing concurrently, unblocking. + (AsyncActor pid=9064) All coroutines ran concurrently. + (AsyncActor pid=9064) All coroutines ran concurrently. + (AsyncActor pid=9064) All coroutines ran concurrently. + (AsyncActor pid=9064) All coroutines ran concurrently. .. testcode:: :hide: From f201564f2fd54dc8ff42257fdccd0698b0feef6c Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Thu, 28 Aug 2025 13:58:54 -0700 Subject: [PATCH 347/634] [data] add commas in data dashboard (#56014) ## Why are these changes needed? Add missing commas ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: iamjustinhsu --- .../dashboards/data_dashboard_panels.py | 104 +++++++++--------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py index f73736e3c56c..5ae50ac361ee 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py @@ -16,7 +16,7 @@ # targets=[ # Target( # expr=f"sum(ray_data_{metric.name}" -# + "{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)", +# + "{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)", # legend=legend, # ) # ], @@ -33,7 +33,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_spilled_bytes{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_spilled_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Spilled: {{dataset}}, {{operator}}", ) ], @@ -48,7 +48,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_freed_bytes{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_freed_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Freed: {{dataset}}, {{operator}}", ) ], @@ -63,7 +63,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_current_bytes{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_current_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Current Usage: {{dataset}}, {{operator}}", ) ], @@ -78,7 +78,7 @@ unit="cores", targets=[ Target( - expr='sum(ray_data_cpu_usage_cores{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_cpu_usage_cores{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="CPU Usage: {{dataset}}, {{operator}}", ) ], @@ -93,7 +93,7 @@ unit="cores", targets=[ Target( - expr='sum(ray_data_gpu_usage_cores{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_gpu_usage_cores{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="GPU Usage: {{dataset}}, {{operator}}", ) ], @@ -108,7 +108,7 @@ unit="Bps", targets=[ Target( - expr='sum(rate(ray_data_output_bytes{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_output_bytes{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Output / Second: {{dataset}}, {{operator}}", ) ], @@ -123,7 +123,7 @@ unit="rows/sec", targets=[ Target( - expr='sum(rate(ray_data_output_rows{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_output_rows{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Rows Output / Second: {{dataset}}, {{operator}}", ) ], @@ -139,7 +139,7 @@ unit="blocks/sec", targets=[ Target( - expr='sum(rate(ray_data_num_inputs_received{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_num_inputs_received{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Blocks Received / Second: {{dataset}}, {{operator}}", ) ], @@ -154,7 +154,7 @@ unit="Bps", targets=[ Target( - expr='sum(rate(ray_data_bytes_inputs_received{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_bytes_inputs_received{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Received / Second: {{dataset}}, {{operator}}", ) ], @@ -171,7 +171,7 @@ unit="blocks/sec", targets=[ Target( - expr='sum(rate(ray_data_num_task_inputs_processed{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_num_task_inputs_processed{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Blocks Processed / Second: {{dataset}}, {{operator}}", ) ], @@ -188,7 +188,7 @@ unit="Bps", targets=[ Target( - expr='sum(rate(ray_data_bytes_task_inputs_processed{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_bytes_task_inputs_processed{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Processed / Second: {{dataset}}, {{operator}}", ) ], @@ -203,7 +203,7 @@ unit="Bps", targets=[ Target( - expr='sum(rate(ray_data_bytes_inputs_of_submitted_tasks{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_bytes_inputs_of_submitted_tasks{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Submitted / Second: {{dataset}}, {{operator}}", ) ], @@ -219,7 +219,7 @@ unit="blocks/sec", targets=[ Target( - expr='sum(rate(ray_data_num_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_num_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Blocks Generated / Second: {{dataset}}, {{operator}}", ) ], @@ -234,7 +234,7 @@ unit="Bps", targets=[ Target( - expr='sum(rate(ray_data_bytes_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_bytes_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Generated / Second: {{dataset}}, {{operator}}", ) ], @@ -249,7 +249,7 @@ unit="rows/sec", targets=[ Target( - expr='sum(rate(ray_data_rows_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_rows_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Rows Generated / Second: {{dataset}}, {{operator}}", ) ], @@ -264,7 +264,7 @@ unit="blocks/sec", targets=[ Target( - expr='sum(rate(ray_data_num_outputs_taken{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_num_outputs_taken{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Blocks Taken / Second: {{dataset}}, {{operator}}", ) ], @@ -281,7 +281,7 @@ unit="Bps", targets=[ Target( - expr='sum(rate(ray_data_bytes_outputs_taken{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, operator)', + expr='sum(rate(ray_data_bytes_outputs_taken{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, operator)', legend="Bytes Taken / Second: {{dataset}}, {{operator}}", ) ], @@ -296,7 +296,7 @@ unit="bytes", targets=[ Target( - expr='increase(ray_data_bytes_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[5m]) / increase(ray_data_num_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[5m])', + expr='increase(ray_data_bytes_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[5m])', legend="Average Bytes Generated / Output Block: {{dataset}}, {{operator}}", ) ], @@ -311,7 +311,7 @@ unit="blocks", targets=[ Target( - expr='increase(ray_data_num_task_outputs_generated{{{global_filters} operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters} operator=~"$Operator"}}[5m])', + expr='increase(ray_data_num_task_outputs_generated{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', legend="Average Number of Output Blocks / Task: {{dataset}}, {{operator}}", ) ], @@ -328,7 +328,7 @@ unit="Bps", targets=[ Target( - expr='sum(rate(ray_data_bytes_outputs_of_finished_tasks_per_node{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, node_ip)', + expr='sum(rate(ray_data_bytes_outputs_of_finished_tasks_per_node{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, node_ip)', legend="Bytes output / Second: {{dataset}}, {{node_ip}}", ) ], @@ -345,7 +345,7 @@ unit="blocks/s", targets=[ Target( - expr='sum(rate(ray_data_blocks_outputs_of_finished_tasks_per_node{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, node_ip)', + expr='sum(rate(ray_data_blocks_outputs_of_finished_tasks_per_node{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, node_ip)', legend="Blocks output / Second: {{dataset}}, {{node_ip}}", ) ], @@ -361,7 +361,7 @@ unit="tasks", targets=[ Target( - expr='sum(ray_data_num_tasks_submitted{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_num_tasks_submitted{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Submitted Tasks: {{dataset}}, {{operator}}", ) ], @@ -376,7 +376,7 @@ unit="tasks", targets=[ Target( - expr='sum(ray_data_num_tasks_running{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_num_tasks_running{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Running Tasks: {{dataset}}, {{operator}}", ) ], @@ -391,7 +391,7 @@ unit="tasks", targets=[ Target( - expr='sum(ray_data_num_tasks_have_outputs{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_num_tasks_have_outputs{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Tasks with output blocks: {{dataset}}, {{operator}}", ) ], @@ -406,7 +406,7 @@ unit="tasks", targets=[ Target( - expr='sum(ray_data_num_tasks_finished{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Finished Tasks: {{dataset}}, {{operator}}", ) ], @@ -421,7 +421,7 @@ unit="tasks", targets=[ Target( - expr='sum(ray_data_num_tasks_failed{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_num_tasks_failed{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Failed Tasks: {{dataset}}, {{operator}}", ) ], @@ -436,7 +436,7 @@ unit="tasks/s", targets=[ Target( - expr='sum(rate(ray_data_num_tasks_finished_per_node{{{global_filters} operator=~"$Operator"}}[1m])) by (dataset, node_ip)', + expr='sum(rate(ray_data_num_tasks_finished_per_node{{{global_filters}, operator=~"$Operator"}}[1m])) by (dataset, node_ip)', legend="Finished Tasks: {{dataset}}, {{node_ip}}", ) ], @@ -451,7 +451,7 @@ unit="seconds", targets=[ Target( - expr='sum(ray_data_block_generation_time{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_block_generation_time{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Block Generation Time: {{dataset}}, {{operator}}", ) ], @@ -466,7 +466,7 @@ unit="seconds", targets=[ Target( - expr='sum(ray_data_task_submission_backpressure_time{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_task_submission_backpressure_time{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Backpressure Time: {{dataset}}, {{operator}}", ) ], @@ -482,7 +482,7 @@ unit="seconds", targets=[ Target( - expr='increase(ray_data_task_completion_time{{{global_filters} operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters} operator=~"$Operator"}}[5m])', + expr='increase(ray_data_task_completion_time{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', legend="Task Completion Time: {{dataset}}, {{operator}}", ), ], @@ -497,7 +497,7 @@ unit="seconds", targets=[ Target( - expr='increase(ray_data_task_output_backpressure_time{{{global_filters} operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters} operator=~"$Operator"}}[5m])', + expr='increase(ray_data_task_output_backpressure_time{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', legend="Task Output Backpressure Time: {{dataset}}, {{operator}}", ), ], @@ -512,7 +512,7 @@ unit="seconds", targets=[ Target( - expr='increase(ray_data_task_completion_time_without_backpressure{{{global_filters} operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters} operator=~"$Operator"}}[5m])', + expr='increase(ray_data_task_completion_time_without_backpressure{{{global_filters}, operator=~"$Operator"}}[5m]) / increase(ray_data_num_tasks_finished{{{global_filters}, operator=~"$Operator"}}[5m])', legend="Task Completion Time w/o Backpressure: {{dataset}}, {{operator}}", ), ], @@ -528,7 +528,7 @@ unit="blocks", targets=[ Target( - expr='sum(ray_data_obj_store_mem_internal_inqueue_blocks{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_obj_store_mem_internal_inqueue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Number of Blocks: {{dataset}}, {{operator}}", ) ], @@ -543,7 +543,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_obj_store_mem_internal_inqueue{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_obj_store_mem_internal_inqueue{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Size: {{dataset}}, {{operator}}", ) ], @@ -558,7 +558,7 @@ unit="blocks", targets=[ Target( - expr='sum(ray_data_obj_store_mem_internal_outqueue_blocks{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_obj_store_mem_internal_outqueue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Number of Blocks: {{dataset}}, {{operator}}", ) ], @@ -573,7 +573,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_obj_store_mem_internal_outqueue{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_obj_store_mem_internal_outqueue{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Size: {{dataset}}, {{operator}}", ) ], @@ -588,7 +588,7 @@ unit="blocks", targets=[ Target( - expr='sum(ray_data_num_external_inqueue_blocks{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_num_external_inqueue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Number of Blocks: {{dataset}}, {{operator}}", ) ], @@ -603,7 +603,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_num_external_inqueue_blocks{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_num_external_inqueue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Number of Bytes: {{dataset}}, {{operator}}", ) ], @@ -619,7 +619,7 @@ unit="blocks", targets=[ Target( - expr='sum(ray_data_obj_store_mem_internal_inqueue_blocks{{{global_filters} operator=~"$Operator"}} + ray_data_num_external_inqueue_blocks{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_obj_store_mem_internal_inqueue_blocks{{{global_filters}, operator=~"$Operator"}} + ray_data_num_external_inqueue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Combined Blocks: {{dataset}}, {{operator}}", ) ], @@ -634,7 +634,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_obj_store_mem_pending_task_inputs{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_obj_store_mem_pending_task_inputs{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Size: {{dataset}}, {{operator}}", ) ], @@ -649,7 +649,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_obj_store_mem_freed{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_obj_store_mem_freed{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Size: {{dataset}}, {{operator}}", ) ], @@ -664,7 +664,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_obj_store_mem_spilled{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_obj_store_mem_spilled{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Bytes Size: {{dataset}}, {{operator}}", ) ], @@ -680,7 +680,7 @@ unit="seconds", targets=[ Target( - expr='sum(ray_data_iter_initialize_seconds{{{global_filters} operator=~"$Operator"}}) by (dataset)', + expr='sum(ray_data_iter_initialize_seconds{{{global_filters}, operator=~"$Operator"}}) by (dataset)', legend="Seconds: {{dataset}}, {{operator}}", ) ], @@ -695,7 +695,7 @@ unit="seconds", targets=[ Target( - expr='sum(ray_data_iter_total_blocked_seconds{{{global_filters} operator=~"$Operator"}}) by (dataset)', + expr='sum(ray_data_iter_total_blocked_seconds{{{global_filters}, operator=~"$Operator"}}) by (dataset)', legend="Seconds: {{dataset}}", ) ], @@ -710,7 +710,7 @@ unit="seconds", targets=[ Target( - expr='sum(ray_data_iter_user_seconds{{{global_filters} operator=~"$Operator"}}) by (dataset)', + expr='sum(ray_data_iter_user_seconds{{{global_filters}, operator=~"$Operator"}}) by (dataset)', legend="Seconds: {{dataset}}", ) ], @@ -726,7 +726,7 @@ unit="seconds", targets=[ Target( - expr='sum(ray_data_sched_loop_duration_s{{{global_filters} operator=~"$Operator"}}) by (dataset)', + expr='sum(ray_data_sched_loop_duration_s{{{global_filters}, operator=~"$Operator"}}) by (dataset)', legend="Scheduling Loop Duration: {{dataset}}", ) ], @@ -741,7 +741,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_max_bytes_to_read{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_max_bytes_to_read{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Max Bytes to Read: {{dataset}}, {{operator}}", ) ], @@ -757,7 +757,7 @@ unit="cpu", targets=[ Target( - expr='sum(ray_data_cpu_budget{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_cpu_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Budget (CPU): {{dataset}}, {{operator}}", ) ], @@ -772,7 +772,7 @@ unit="gpu", targets=[ Target( - expr='sum(ray_data_gpu_budget{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_gpu_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Budget (GPU): {{dataset}}, {{operator}}", ) ], @@ -787,7 +787,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_memory_budget{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_memory_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Budget (Memory): {{dataset}}, {{operator}}", ) ], @@ -802,7 +802,7 @@ unit="bytes", targets=[ Target( - expr='sum(ray_data_object_store_memory_budget{{{global_filters} operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_object_store_memory_budget{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Budget (Object Store Memory): {{dataset}}, {{operator}}", ) ], From c036435d2afe822d03c7c8bd9d417a33f2396b7d Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Thu, 28 Aug 2025 16:16:40 -0500 Subject: [PATCH 348/634] [core] Split `gcs_actor_manager` out from `gcs_server_lib` (#56049) Stacked on: https://github.com/ray-project/ray/pull/56002 --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 49 ++++++++- src/ray/gcs/gcs_server/gcs_actor_manager.cc | 2 + src/ray/gcs/gcs_server/gcs_actor_manager.h | 10 +- src/ray/gcs/gcs_server/gcs_server.cc | 5 +- .../gcs/gcs_server/grpc_service_interfaces.h | 42 ++++++++ src/ray/gcs/gcs_server/grpc_services.cc | 20 ++++ src/ray/gcs/gcs_server/grpc_services.h | 23 ++++ src/ray/gcs/gcs_server/tests/BUILD.bazel | 17 ++- .../gcs_actor_manager_export_event_test.cc | 1 - .../tests/gcs_actor_manager_test.cc | 1 - src/ray/rpc/gcs/gcs_rpc_server.h | 100 ++---------------- 11 files changed, 165 insertions(+), 105 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index d8791b891079..3228dde7e6a7 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -360,6 +360,9 @@ ray_cc_library( hdrs = [ "gcs_actor.h", ], + implementation_deps = [ + "//src/ray/util:logging", + ], deps = [ "//src/ray/common:id", "//src/ray/common:task_common", @@ -368,7 +371,6 @@ ray_cc_library( "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/util:counter_map", "//src/ray/util:event", - "//src/ray/util:logging", ], ) @@ -380,13 +382,16 @@ ray_cc_library( hdrs = [ "gcs_actor_scheduler.h", ], + implementation_deps = [ + "//src/ray/common:ray_config", + "//src/ray/util:time", + ], deps = [ ":gcs_actor", ":gcs_node_manager", ":gcs_table_storage", "//src/ray/common:asio", "//src/ray/common:id", - "//src/ray/common:ray_config", "//src/ray/common:task_common", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/raylet/scheduling:cluster_task_manager", @@ -394,7 +399,44 @@ ray_cc_library( "//src/ray/rpc:core_worker_client", "//src/ray/rpc:node_manager_client", "//src/ray/util:logging", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_googletest//:gtest", + ], +) + +ray_cc_library( + name = "gcs_actor_manager", + srcs = [ + "gcs_actor_manager.cc", + ], + hdrs = [ + "gcs_actor_manager.h", + ], + implementation_deps = [ + "//src/ray/common:ray_config", + "//src/ray/common:task_common", + "//src/ray/gcs:gcs_pb_util", + "//src/ray/stats:stats_lib", + "//src/ray/util:logging", "//src/ray/util:time", + ], + deps = [ + ":gcs_actor", + ":gcs_actor_scheduler", + ":gcs_function_manager", + ":gcs_init_data", + ":gcs_table_storage", + ":gcs_usage_stats_client", + ":grpc_service_interfaces", + "//src/ray/common:asio", + "//src/ray/common:id", + "//src/ray/gcs/pubsub:gcs_pub_sub_lib", + "//src/ray/protobuf:gcs_service_cc_proto", + "//src/ray/rpc:core_worker_client", + "//src/ray/util:counter_map", + "//src/ray/util:logging", + "//src/ray/util:thread_checker", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_googletest//:gtest", @@ -404,17 +446,16 @@ ray_cc_library( ray_cc_library( name = "gcs_server_lib", srcs = [ - "gcs_actor_manager.cc", "gcs_autoscaler_state_manager.cc", "gcs_server.cc", ], hdrs = [ - "gcs_actor_manager.h", "gcs_autoscaler_state_manager.h", "gcs_server.h", ], deps = [ ":gcs_actor", + ":gcs_actor_manager", ":gcs_actor_scheduler", ":gcs_function_manager", ":gcs_health_check_manager", diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index ddad4fe2d02f..b8a45ed147e8 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -23,8 +23,10 @@ #include #include "ray/common/ray_config.h" +#include "ray/common/task/task_spec.h" #include "ray/gcs/pb_util.h" #include "ray/stats/metric_defs.h" +#include "ray/util/logging.h" #include "ray/util/time.h" namespace { diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.h b/src/ray/gcs/gcs_server/gcs_actor_manager.h index 5fd24a07995d..a0dab85a3903 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.h +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once + #include #include @@ -22,20 +23,21 @@ #include #include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" #include "ray/common/runtime_env_manager.h" -#include "ray/common/task/task_spec.h" #include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_actor_scheduler.h" #include "ray/gcs/gcs_server/gcs_function_manager.h" #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/usage_stats_client.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" #include "ray/rpc/worker/core_worker_client.h" +#include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/counter_map.h" -#include "ray/util/event.h" #include "ray/util/thread_checker.h" #include "src/ray/protobuf/gcs_service.pb.h" @@ -87,7 +89,7 @@ namespace gcs { /// will update its state to `DEAD` and remove it from `registered_actors_` and /// `created_actors_`. /// 9: A dead actor caused by out-of-scope is lineage reconstructed. -class GcsActorManager : public rpc::ActorInfoHandler { +class GcsActorManager : public rpc::ActorInfoGcsServiceHandler { public: /// Create a GcsActorManager /// diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index be8a25d3b2f3..041412b0b16d 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -499,10 +499,11 @@ void GcsServer::InitGcsActorManager(const GcsInitData &gcs_init_data) { }, worker_client_pool_); - // Initialize by gcs tables data. gcs_actor_manager_->Initialize(gcs_init_data); rpc_server_.RegisterService(std::make_unique( - io_context_provider_.GetDefaultIOContext(), *gcs_actor_manager_)); + io_context_provider_.GetDefaultIOContext(), + *gcs_actor_manager_, + RayConfig::instance().gcs_max_active_rpcs_per_handler())); } void GcsServer::InitGcsPlacementGroupManager(const GcsInitData &gcs_init_data) { diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h index 1ce4129ca697..3f4feb03b784 100644 --- a/src/ray/gcs/gcs_server/grpc_service_interfaces.h +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -35,6 +35,48 @@ using SendReplyCallback = std::functionmutable_status()->set_message(status.message()); \ send_reply_callback(ray::Status::OK(), nullptr, nullptr) +class ActorInfoGcsServiceHandler { + public: + virtual ~ActorInfoGcsServiceHandler() = default; + + virtual void HandleRegisterActor(RegisterActorRequest request, + RegisterActorReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleRestartActorForLineageReconstruction( + RestartActorForLineageReconstructionRequest request, + RestartActorForLineageReconstructionReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleCreateActor(CreateActorRequest request, + CreateActorReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetActorInfo(GetActorInfoRequest request, + GetActorInfoReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetNamedActorInfo(GetNamedActorInfoRequest request, + GetNamedActorInfoReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleListNamedActors(rpc::ListNamedActorsRequest request, + rpc::ListNamedActorsReply *reply, + rpc::SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetAllActorInfo(GetAllActorInfoRequest request, + GetAllActorInfoReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleKillActorViaGcs(KillActorViaGcsRequest request, + KillActorViaGcsReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleReportActorOutOfScope(ReportActorOutOfScopeRequest request, + ReportActorOutOfScopeReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + class NodeInfoGcsServiceHandler { public: virtual ~NodeInfoGcsServiceHandler() = default; diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc index 4e63a85a4e12..d1b4bb9d9f51 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -19,6 +19,26 @@ namespace ray { namespace rpc { +void ActorInfoGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + /// The register & create actor RPCs take a long time, so we shouldn't limit their + /// concurrency to avoid distributed deadlock. + RPC_SERVICE_HANDLER(ActorInfoGcsService, RegisterActor, -1) + RPC_SERVICE_HANDLER(ActorInfoGcsService, CreateActor, -1) + RPC_SERVICE_HANDLER(ActorInfoGcsService, RestartActorForLineageReconstruction, -1) + + RPC_SERVICE_HANDLER(ActorInfoGcsService, GetActorInfo, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(ActorInfoGcsService, GetAllActorInfo, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + ActorInfoGcsService, GetNamedActorInfo, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(ActorInfoGcsService, ListNamedActors, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(ActorInfoGcsService, KillActorViaGcs, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + ActorInfoGcsService, ReportActorOutOfScope, max_active_rpcs_per_handler_) +} + void NodeInfoGrpcService::InitServerCallFactories( const std::unique_ptr &cq, std::vector> *server_call_factories, diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/gcs_server/grpc_services.h index 8108dc3760ce..20040d8396aa 100644 --- a/src/ray/gcs/gcs_server/grpc_services.h +++ b/src/ray/gcs/gcs_server/grpc_services.h @@ -35,6 +35,29 @@ namespace ray { namespace rpc { +class ActorInfoGrpcService : public GrpcService { + public: + explicit ActorInfoGrpcService(instrumented_io_context &io_service, + ActorInfoGcsServiceHandler &service_handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(service_handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler) {} + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + ActorInfoGcsService::AsyncService service_; + ActorInfoGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + class NodeInfoGrpcService : public GrpcService { public: explicit NodeInfoGrpcService(instrumented_io_context &io_service, diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index 51a7dde30e5b..2bb7fc487143 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -244,8 +244,13 @@ ray_cc_test( ], deps = [ "//:ray_mock", + "//src/ray/common:asio", + "//src/ray/common:runtime_env", "//src/ray/gcs/gcs_server:gcs_actor", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs/gcs_server:gcs_actor_manager", + "//src/ray/gcs/gcs_server:gcs_actor_scheduler", + "//src/ray/gcs/gcs_server:gcs_function_manager", + "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/pubsub:publisher", "@com_google_googletest//:gtest_main", @@ -399,9 +404,17 @@ ray_cc_test( ], deps = [ "//:ray_mock", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/common:asio", + "//src/ray/common:runtime_env", + "//src/ray/gcs/gcs_server:gcs_actor", + "//src/ray/gcs/gcs_server:gcs_actor_manager", + "//src/ray/gcs/gcs_server:gcs_actor_scheduler", + "//src/ray/gcs/gcs_server:gcs_function_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/tests:gcs_test_util_lib", + "//src/ray/pubsub:publisher", + "//src/ray/rpc:core_worker_client", + "//src/ray/util:event", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc index fa8465fda45d..735acc7c9bbd 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc @@ -28,7 +28,6 @@ #include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_actor_manager.h" #include "ray/gcs/gcs_server/gcs_function_manager.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/pubsub/publisher.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc index b05cabf4141b..8efa1aa22544 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc @@ -29,7 +29,6 @@ #include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_actor_scheduler.h" #include "ray/gcs/gcs_server/gcs_function_manager.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/tests/gcs_test_util.h" #include "ray/pubsub/publisher.h" diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index 2750b945c35a..28b0c7e73c9f 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -109,8 +109,15 @@ using AutoscalerStateHandler = AutoscalerStateServiceHandler; namespace ray { namespace rpc { -#define ACTOR_INFO_SERVICE_RPC_HANDLER(HANDLER, MAX_ACTIVE_RPCS) \ - RPC_SERVICE_HANDLER(ActorInfoGcsService, HANDLER, MAX_ACTIVE_RPCS) +#define MONITOR_SERVICE_RPC_HANDLER(HANDLER) \ + RPC_SERVICE_HANDLER(MonitorGcsService, \ + HANDLER, \ + RayConfig::instance().gcs_max_active_rpcs_per_handler()) + +#define OBJECT_INFO_SERVICE_RPC_HANDLER(HANDLER) \ + RPC_SERVICE_HANDLER(ObjectInfoGcsService, \ + HANDLER, \ + RayConfig::instance().gcs_max_active_rpcs_per_handler()) #define MONITOR_SERVICE_RPC_HANDLER(HANDLER) \ RPC_SERVICE_HANDLER(MonitorGcsService, \ @@ -127,94 +134,5 @@ namespace rpc { reply->mutable_status()->set_message(status.message()); \ send_reply_callback(ray::Status::OK(), nullptr, nullptr) -class ActorInfoGcsServiceHandler { - public: - virtual ~ActorInfoGcsServiceHandler() = default; - - virtual void HandleRegisterActor(RegisterActorRequest request, - RegisterActorReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleRestartActorForLineageReconstruction( - RestartActorForLineageReconstructionRequest request, - RestartActorForLineageReconstructionReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleCreateActor(CreateActorRequest request, - CreateActorReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetActorInfo(GetActorInfoRequest request, - GetActorInfoReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetNamedActorInfo(GetNamedActorInfoRequest request, - GetNamedActorInfoReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleListNamedActors(rpc::ListNamedActorsRequest request, - rpc::ListNamedActorsReply *reply, - rpc::SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetAllActorInfo(GetAllActorInfoRequest request, - GetAllActorInfoReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleKillActorViaGcs(KillActorViaGcsRequest request, - KillActorViaGcsReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleReportActorOutOfScope(ReportActorOutOfScopeRequest request, - ReportActorOutOfScopeReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -/// The `GrpcService` for `ActorInfoGcsService`. -class ActorInfoGrpcService : public GrpcService { - public: - /// Constructor. - /// - /// \param[in] handler The service handler that actually handle the requests. - explicit ActorInfoGrpcService(instrumented_io_context &io_service, - ActorInfoGcsServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler){}; - - protected: - grpc::Service &GetGrpcService() override { return service_; } - - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - /// Register/Create Actor RPC takes long time, we shouldn't limit them to avoid - /// distributed deadlock. - ACTOR_INFO_SERVICE_RPC_HANDLER(RegisterActor, -1); - ACTOR_INFO_SERVICE_RPC_HANDLER(RestartActorForLineageReconstruction, -1); - ACTOR_INFO_SERVICE_RPC_HANDLER(CreateActor, -1); - - /// Others need back pressure. - ACTOR_INFO_SERVICE_RPC_HANDLER( - GetActorInfo, RayConfig::instance().gcs_max_active_rpcs_per_handler()); - ACTOR_INFO_SERVICE_RPC_HANDLER( - GetNamedActorInfo, RayConfig::instance().gcs_max_active_rpcs_per_handler()); - ACTOR_INFO_SERVICE_RPC_HANDLER( - ListNamedActors, RayConfig::instance().gcs_max_active_rpcs_per_handler()); - ACTOR_INFO_SERVICE_RPC_HANDLER( - GetAllActorInfo, RayConfig::instance().gcs_max_active_rpcs_per_handler()); - ACTOR_INFO_SERVICE_RPC_HANDLER( - KillActorViaGcs, RayConfig::instance().gcs_max_active_rpcs_per_handler()); - ACTOR_INFO_SERVICE_RPC_HANDLER( - ReportActorOutOfScope, RayConfig::instance().gcs_max_active_rpcs_per_handler()); - } - - private: - /// The grpc async service object. - ActorInfoGcsService::AsyncService service_; - /// The service handler that actually handle the requests. - ActorInfoGcsServiceHandler &service_handler_; -}; - -using ActorInfoHandler = ActorInfoGcsServiceHandler; - } // namespace rpc } // namespace ray From d0f14a9acb382b42c2416fae924c91509f149fde Mon Sep 17 00:00:00 2001 From: Rueian Date: Thu, 28 Aug 2025 14:17:38 -0700 Subject: [PATCH 349/634] [core][autoscaler] Enable autoscaler v2 on clusters launched by the cluster launcher (#55865) Turn on autoscaler v2 on cluster launcher by default in the next Ray release (2.50.0). Signed-off-by: Rueian --- python/ray/autoscaler/_private/commands.py | 12 ++++++++-- .../tests/test_cli_patterns/test_ray_up.txt | 1 + .../test_cli_patterns/test_ray_up_docker.txt | 1 + .../test_cli_patterns/test_ray_up_record.txt | 5 ++-- release/release_tests.yaml | 24 +++++++++++++++++++ 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/python/ray/autoscaler/_private/commands.py b/python/ray/autoscaler/_private/commands.py index fa571a641b48..fd15218e65e2 100644 --- a/python/ray/autoscaler/_private/commands.py +++ b/python/ray/autoscaler/_private/commands.py @@ -824,8 +824,16 @@ def get_or_create_head_node( # Use RAY_UP_enable_autoscaler_v2 instead of RAY_enable_autoscaler_v2 # to avoid accidentally enabling autoscaler v2 for ray up - # due to env inheritance. - if os.getenv("RAY_UP_enable_autoscaler_v2", "0") == "1": + # due to env inheritance. The default value is 1 since Ray 2.50.0. + if os.getenv("RAY_UP_enable_autoscaler_v2", "1") == "1": + if "RAY_UP_enable_autoscaler_v2" not in os.environ: + # TODO (rueian): Remove this notice after Ray 2.52.0. + cli_logger.print( + "Autoscaler v2 is now enabled by default (since Ray 2.50.0). " + "To switch back to v1, set {}=0. This message can be suppressed by setting {} explicitly.", + cf.bold("RAY_UP_enable_autoscaler_v2"), + cf.bold("RAY_UP_enable_autoscaler_v2"), + ) ray_start_commands = with_envs( ray_start_commands, { diff --git a/python/ray/tests/test_cli_patterns/test_ray_up.txt b/python/ray/tests/test_cli_patterns/test_ray_up.txt index 30a9f52d28e9..0da266aaed4a 100644 --- a/python/ray/tests/test_cli_patterns/test_ray_up.txt +++ b/python/ray/tests/test_cli_patterns/test_ray_up.txt @@ -19,6 +19,7 @@ Acquiring an up-to-date head node <1/1> Setting up head node Prepared bootstrap config + Autoscaler v2 is now enabled by default.+ New status: waiting-for-ssh \[1/7\] Waiting for SSH to become available Running `uptime` as a test\. diff --git a/python/ray/tests/test_cli_patterns/test_ray_up_docker.txt b/python/ray/tests/test_cli_patterns/test_ray_up_docker.txt index 30a9f52d28e9..0da266aaed4a 100644 --- a/python/ray/tests/test_cli_patterns/test_ray_up_docker.txt +++ b/python/ray/tests/test_cli_patterns/test_ray_up_docker.txt @@ -19,6 +19,7 @@ Acquiring an up-to-date head node <1/1> Setting up head node Prepared bootstrap config + Autoscaler v2 is now enabled by default.+ New status: waiting-for-ssh \[1/7\] Waiting for SSH to become available Running `uptime` as a test\. diff --git a/python/ray/tests/test_cli_patterns/test_ray_up_record.txt b/python/ray/tests/test_cli_patterns/test_ray_up_record.txt index 1f6ce5e93ce3..3bbbc3b98a13 100644 --- a/python/ray/tests/test_cli_patterns/test_ray_up_record.txt +++ b/python/ray/tests/test_cli_patterns/test_ray_up_record.txt @@ -18,6 +18,7 @@ .+\.py.*Fetching the new head node .+\.py.*<1/1> Setting up head node .+\.py.*Prepared bootstrap config +.+\.py.*Autoscaler v2 is now enabled by default.+ .+\.py.*AWSNodeProvider: Set tag ray-node-status=waiting-for-ssh on \['.+'\] \[LogTimer=.+\] .+\.py.*New status: waiting-for-ssh .+\.py.*\[1/7\] Waiting for SSH to become available @@ -73,9 +74,9 @@ .+\.py.*Full command is `ssh.+` .+\.py.*NodeUpdater: i-.+: Setup commands succeeded \[LogTimer=.+\] .+\.py.*\[7/7\] Starting the Ray runtime -.+\.py.*Running `export RAY_USAGE_STATS_ENABLED=1;export RAY_OVERRIDE_RESOURCES='{"CPU":1}';export RAY_OVERRIDE_LABELS='{"key1":"value1"}';ray stop` +.+\.py.*Running `export RAY_USAGE_STATS_ENABLED=1;export RAY_OVERRIDE_RESOURCES='{"CPU":1}';export RAY_OVERRIDE_LABELS='{"key1":"value1"}';export RAY_enable_autoscaler_v2=1; export RAY_CLOUD_INSTANCE_ID=i-.+; export RAY_NODE_TYPE_NAME=head_node; ray stop` .+\.py.*Full command is `ssh.+` -.+\.py.*Running `export RAY_USAGE_STATS_ENABLED=1;export RAY_OVERRIDE_RESOURCES='{"CPU":1}';export RAY_OVERRIDE_LABELS='{"key1":"value1"}';ray start --head --autoscaling-config=~/ray_bootstrap_config\.yaml` +.+\.py.*Running `export RAY_USAGE_STATS_ENABLED=1;export RAY_OVERRIDE_RESOURCES='{"CPU":1}';export RAY_OVERRIDE_LABELS='{"key1":"value1"}';export RAY_enable_autoscaler_v2=1; export RAY_CLOUD_INSTANCE_ID=i-.+; export RAY_NODE_TYPE_NAME=head_node; ray start --head --autoscaling-config=~/ray_bootstrap_config\.yaml` .+\.py.*Full command is `ssh.+` .+\.py.*NodeUpdater: i-.+: Ray start commands succeeded \[LogTimer=.+\] .+\.py.*NodeUpdater: i-.+: Applied config .+ \[LogTimer=.+\] diff --git a/release/release_tests.yaml b/release/release_tests.yaml index 8cd1f1bee9e4..835ceacbe3f6 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -4102,6 +4102,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 @@ -4122,6 +4124,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override nightly - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override nightly @@ -4142,6 +4146,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override latest - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override latest @@ -4162,6 +4168,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override commit - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override commit @@ -4183,6 +4191,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/example-minimal.yaml - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/example-minimal.yaml @@ -4203,6 +4213,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest @@ -4226,6 +4238,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py gcp/example-minimal-pinned.yaml - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py gcp/example-minimal-pinned.yaml @@ -4249,6 +4263,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 30 --docker-override latest - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 30 --docker-override latest @@ -4272,6 +4288,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest @@ -4295,6 +4313,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override nightly - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override nightly @@ -4318,6 +4338,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override commit - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override commit @@ -4341,6 +4363,8 @@ variations: - __suffix__: v1 + run: + script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py gcp/example-gpu-docker.yaml - __suffix__: v2 run: script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py gcp/example-gpu-docker.yaml From 4104c49cacbd4d266b716e8fc89b74b0397eb451 Mon Sep 17 00:00:00 2001 From: Len Strnad Date: Thu, 28 Aug 2025 15:45:09 -0600 Subject: [PATCH 350/634] [Data] - Handle the case when number of rows exceeds the number of bytes for a block (#55790) ## Why are these changes needed? When num rows is >> bytes int casts to 0 and we get division by zero. ## Related issue number Closes #54385 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: ljstrnadiii Signed-off-by: Alexey Kudinkin Co-authored-by: Alexey Kudinkin --- python/ray/data/_internal/arrow_block.py | 2 +- python/ray/data/tests/test_arrow_block.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ray/data/_internal/arrow_block.py b/python/ray/data/_internal/arrow_block.py index 9d18db5bf028..8f8058cc58c0 100644 --- a/python/ray/data/_internal/arrow_block.py +++ b/python/ray/data/_internal/arrow_block.py @@ -187,7 +187,7 @@ def _get_max_chunk_size( if table.nbytes == 0: return None else: - avg_row_size = int(table.nbytes / table.num_rows) + avg_row_size = table.nbytes / table.num_rows return max(1, int(max_chunk_size_bytes / avg_row_size)) diff --git a/python/ray/data/tests/test_arrow_block.py b/python/ray/data/tests/test_arrow_block.py index 3790964d19b3..f599bfc58ee5 100644 --- a/python/ray/data/tests/test_arrow_block.py +++ b/python/ray/data/tests/test_arrow_block.py @@ -542,6 +542,7 @@ def test_arrow_nan_element(): "table_data,max_chunk_size_bytes,expected", [ ({"a": []}, 100, None), + ({"a": list(range(100))}, 7, 1), ({"a": list(range(100))}, 10, 1), ({"a": list(range(100))}, 25, 3), ({"a": list(range(100))}, 50, 6), From 89ce367b13ea83a1bb53f12c78a7e153a68f3e8c Mon Sep 17 00:00:00 2001 From: Goku Mohandas Date: Thu, 28 Aug 2025 15:18:19 -0700 Subject: [PATCH 351/634] Mm ai 082725 (#56035) --- .../e2e-multimodal-ai-workloads/containerfile | 2 +- .../notebooks/01-Batch-Inference.ipynb | 436 +-- .../notebooks/02-Distributed-Training.ipynb | 3054 ++--------------- .../notebooks/03-Online-Serving.ipynb | 84 +- .../byod/byod_e2e_multimodal_ai_workloads.sh | 2 +- 5 files changed, 547 insertions(+), 3031 deletions(-) diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/containerfile b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/containerfile index 4a370bacac53..a9bc32fc2faa 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/containerfile +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/containerfile @@ -1,6 +1,6 @@ # Start with an Anyscale base image. # Use the drop-down above to browse through all available images. -FROM anyscale/ray:2.48.0-slim-py312-cu128 +FROM anyscale/ray:2.49.0-slim-py312-cu128 # Add your pip dependencies here. Disable cache for a smaller image to optimize build and cluster startup time. # RUN pip install --no-cache-dir --upgrade diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/01-Batch-Inference.ipynb b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/01-Batch-Inference.ipynb index f50bf23508fd..2b083decd67d 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/01-Batch-Inference.ipynb +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/01-Batch-Inference.ipynb @@ -31,9 +31,9 @@ "output_type": "stream", "text": [ "\u001b[92mSuccessfully registered `ipywidgets, matplotlib` and 4 other packages to be installed on all cluster nodes.\u001b[0m\n", - "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_1dp3fa7w5hu3i83ldsi7lqvp9t?workspace-tab=dependencies\u001b[0m\n", + "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_23ry3pgfn3jgq2jk3e5z25udhz?workspace-tab=dependencies\u001b[0m\n", "\u001b[92mSuccessfully registered `doggos` package to be installed on all cluster nodes.\u001b[0m\n", - "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_1dp3fa7w5hu3i83ldsi7lqvp9t?workspace-tab=dependencies\u001b[0m\n" + "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_23ry3pgfn3jgq2jk3e5z25udhz?workspace-tab=dependencies\u001b[0m\n" ] } ], @@ -121,23 +121,23 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-22 00:14:08,238\tINFO worker.py:1747 -- Connecting to existing Ray cluster at address: 10.0.52.10:6379...\n", - "2025-08-22 00:14:08,250\tINFO worker.py:1918 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttps://session-466hy7cqu1gzrp8zk8l4byz7l7.i.anyscaleuserdata.com \u001b[39m\u001b[22m\n", - "2025-08-22 00:14:08,255\tINFO packaging.py:588 -- Creating a file package for local module '/home/ray/default/doggos/doggos'.\n", - "2025-08-22 00:14:08,258\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_0193267f6c9951ce.zip' (0.02MiB) to Ray cluster...\n", - "2025-08-22 00:14:08,259\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_0193267f6c9951ce.zip'.\n", - "2025-08-22 00:14:08,262\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_6d26725922931a7a9e87fca928dfafe4f4e5e54b.zip' (1.18MiB) to Ray cluster...\n", - "2025-08-22 00:14:08,268\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_6d26725922931a7a9e87fca928dfafe4f4e5e54b.zip'.\n", - "2025-08-22 00:14:08,550\tINFO dataset.py:3057 -- Tip: Use `take_batch()` instead of `take() / show()` to return records in pandas or numpy batch format.\n", - "2025-08-22 00:14:08,552\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_59_0\n", - "2025-08-22 00:14:08,641\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_59_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", - "2025-08-22 00:14:08,642\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_59_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> LimitOperator[limit=1]\n" + "2025-08-28 05:00:43,606\tINFO worker.py:1771 -- Connecting to existing Ray cluster at address: 10.0.17.148:6379...\n", + "2025-08-28 05:00:43,617\tINFO worker.py:1942 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttps://session-jhxhj69d6ttkjctcxfnsfe7gwk.i.anyscaleuserdata.com \u001b[39m\u001b[22m\n", + "2025-08-28 05:00:43,621\tINFO packaging.py:588 -- Creating a file package for local module '/home/ray/default/doggos/doggos'.\n", + "2025-08-28 05:00:43,625\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_7400f2bea399eebc.zip' (0.02MiB) to Ray cluster...\n", + "2025-08-28 05:00:43,625\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_7400f2bea399eebc.zip'.\n", + "2025-08-28 05:00:43,628\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_a31dca6092632244a5c9467084f1b1f8df982200.zip' (1.10MiB) to Ray cluster...\n", + "2025-08-28 05:00:43,634\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_a31dca6092632244a5c9467084f1b1f8df982200.zip'.\n", + "2025-08-28 05:00:48,035\tINFO dataset.py:3248 -- Tip: Use `take_batch()` instead of `take() / show()` to return records in pandas or numpy batch format.\n", + "2025-08-28 05:00:48,039\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_1_0\n", + "2025-08-28 05:00:48,101\tINFO streaming_executor.py:159 -- Starting execution of Dataset dataset_1_0. Full logs are in /tmp/ray/session_2025-08-28_04-57-43_348032_12595/logs/ray-data\n", + "2025-08-28 05:00:48,102\tINFO streaming_executor.py:160 -- Execution plan of Dataset dataset_1_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> LimitOperator[limit=1] -> TaskPoolMapOperator[ReadFiles]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b6862146286847ef9294638c1aa3d311", + "model_id": "d08e184535944a6c8ea162eca5674cd1", "version_major": 2, "version_minor": 0 }, @@ -151,7 +151,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "86a135d7d9cd45bc8ad9e5f0e5c477ad", + "model_id": "ac866daca29b4379b67367b2c50c65f0", "version_major": 2, "version_minor": 0 }, @@ -165,12 +165,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4bfc88e39a7b450c945855a2d3f908e4", + "model_id": "724c3b66392442aebf0d756157799e44", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "- ReadFiles 2: 0.00 row [00:00, ? row/s]" + "- limit=1 2: 0.00 row [00:00, ? row/s]" ] }, "metadata": {}, @@ -179,12 +179,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1071a49d524e424498985dc424a029a1", + "model_id": "c1d6a10ed6c04fce8135dc2a98b8ebe3", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "- limit=1 3: 0.00 row [00:00, ? row/s]" + "- ReadFiles 3: 0.00 row [00:00, ? row/s]" ] }, "metadata": {}, @@ -194,63 +194,63 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-22 00:14:08,686\tWARNING resource_manager.py:130 -- ⚠️ Ray's object store is configured to use only 28.2% of available memory (67.8GB out of 240.5GB total). For optimal Ray Data performance, we recommend setting the object store to at least 50% of available memory. You can do this by setting the 'object_store_memory' parameter when calling ray.init() or by setting the RAY_DEFAULT_OBJECT_STORE_MEMORY_PROPORTION environment variable.\n", - "2025-08-22 00:15:25,802\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_59_0 execution finished in 77.16 seconds\n" + "2025-08-28 05:00:48,137\tWARNING resource_manager.py:134 -- ⚠️ Ray's object store is configured to use only 27.3% of available memory (8.7GiB out of 32.0GiB total). For optimal Ray Data performance, we recommend setting the object store to at least 50% of available memory. You can do this by setting the 'object_store_memory' parameter when calling ray.init() or by setting the RAY_DEFAULT_OBJECT_STORE_MEMORY_PROPORTION environment variable.\n", + "2025-08-28 05:00:52,084\tINFO streaming_executor.py:279 -- ✔️ Dataset dataset_1_0 execution finished in 3.98 seconds\n" ] }, { "data": { "text/plain": [ - "[{'image': array([[[123, 118, 78],\n", - " [125, 120, 80],\n", - " [128, 120, 83],\n", + "[{'image': array([[[ 71, 93, 81],\n", + " [ 71, 93, 81],\n", + " [ 71, 91, 79],\n", " ...,\n", - " [162, 128, 83],\n", - " [162, 128, 83],\n", - " [161, 127, 82]],\n", + " [ 99, 129, 137],\n", + " [101, 131, 139],\n", + " [102, 132, 140]],\n", " \n", - " [[123, 118, 78],\n", - " [125, 120, 80],\n", - " [127, 119, 82],\n", + " [[ 61, 81, 70],\n", + " [ 61, 81, 70],\n", + " [ 61, 81, 69],\n", " ...,\n", - " [162, 128, 83],\n", - " [162, 128, 83],\n", - " [161, 127, 82]],\n", + " [ 93, 123, 131],\n", + " [ 96, 125, 133],\n", + " [ 97, 127, 135]],\n", " \n", - " [[123, 118, 78],\n", - " [125, 120, 80],\n", - " [127, 119, 82],\n", + " [[ 51, 68, 58],\n", + " [ 51, 68, 58],\n", + " [ 50, 68, 56],\n", " ...,\n", - " [161, 128, 83],\n", - " [161, 128, 83],\n", - " [160, 127, 82]],\n", + " [ 82, 111, 117],\n", + " [ 85, 112, 119],\n", + " [ 86, 115, 121]],\n", " \n", " ...,\n", " \n", - " [[235, 234, 239],\n", - " [233, 232, 237],\n", - " [221, 220, 225],\n", + " [[ 83, 101, 103],\n", + " [ 83, 101, 103],\n", + " [ 84, 102, 106],\n", " ...,\n", - " [158, 95, 54],\n", - " [150, 85, 53],\n", - " [151, 88, 57]],\n", + " [ 94, 82, 56],\n", + " [ 97, 85, 59],\n", + " [ 99, 87, 61]],\n", " \n", - " [[219, 220, 222],\n", - " [227, 228, 230],\n", - " [222, 223, 225],\n", + " [[ 82, 100, 102],\n", + " [ 82, 100, 102],\n", + " [ 83, 101, 105],\n", " ...,\n", - " [153, 91, 54],\n", - " [146, 83, 52],\n", - " [149, 88, 59]],\n", + " [ 95, 83, 57],\n", + " [ 98, 86, 60],\n", + " [ 99, 87, 61]],\n", " \n", - " [[213, 217, 216],\n", - " [217, 221, 220],\n", - " [213, 214, 216],\n", + " [[ 85, 100, 103],\n", + " [ 85, 100, 103],\n", + " [ 83, 101, 103],\n", " ...,\n", - " [153, 91, 54],\n", - " [144, 83, 54],\n", - " [149, 88, 60]]], dtype=uint8),\n", - " 'path': 'doggos-dataset/train/border_collie/border_collie_1055.jpg'}]" + " [ 95, 84, 56],\n", + " [ 99, 88, 60],\n", + " [100, 89, 61]]], dtype=uint8),\n", + " 'path': 'doggos-dataset/train/malamute/malamute_11814.jpg'}]" ] }, "execution_count": null, @@ -498,15 +498,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-22 00:15:55,241\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_64_0\n", - "2025-08-22 00:15:55,265\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_64_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", - "2025-08-22 00:15:55,267\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_64_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" + "2025-08-28 05:00:55,737\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_6_0\n", + "2025-08-28 05:00:55,756\tINFO streaming_executor.py:159 -- Starting execution of Dataset dataset_6_0. Full logs are in /tmp/ray/session_2025-08-28_04-57-43_348032_12595/logs/ray-data\n", + "2025-08-28 05:00:55,757\tINFO streaming_executor.py:160 -- Execution plan of Dataset dataset_6_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6d183707412548d5acd113c34ed06a4c", + "model_id": "7dbb9f0a9c364c529da80ff9e3266eb4", "version_major": 2, "version_minor": 0 }, @@ -517,10 +517,17 @@ "metadata": {}, "output_type": "display_data" }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "{\"asctime\":\"2025-08-28 05:00:55,808\",\"levelname\":\"E\",\"message\":\"Actor with class name: 'MapWorker(MapBatches(EmbedImages))' and ID: '1e923c76f6e2b92256b942a802000000' has constructor arguments in the object store and max_restarts > 0. If the arguments in the object store go out of scope or are lost, the actor restart will fail. See https://github.com/ray-project/ray/issues/53727 for more details.\",\"filename\":\"core_worker.cc\",\"lineno\":2254}\n" + ] + }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "658e7e6b56fd4ddca63a85f903dc598c", + "model_id": "02bc199f5f074df19b376272e8c29ba8", "version_major": 2, "version_minor": 0 }, @@ -534,7 +541,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ea845b3839f341fe96882d806ad16146", + "model_id": "70ce40105ae34580a6ebb69dfada0de0", "version_major": 2, "version_minor": 0 }, @@ -548,7 +555,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "83e3d887e66844b7a414f95163268c4f", + "model_id": "36450487d0614de89dab8cb02e4e7180", "version_major": 2, "version_minor": 0 }, @@ -562,7 +569,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "dca912f45aab4457b4188f74fb21ab63", + "model_id": "1197937e481a43bb90094625f2c8a569", "version_major": 2, "version_minor": 0 }, @@ -576,7 +583,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1b0ab6aaffeb45aea51b8a3c7b75540a", + "model_id": "c6277d187bd345ff9a773b33bbc03ea6", "version_major": 2, "version_minor": 0 }, @@ -591,30 +598,32 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[36m(autoscaler +2m12s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n", - "\u001b[36m(autoscaler +2m17s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", - "\u001b[36m(autoscaler +2m17s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", - "\u001b[36m(autoscaler +2m57s)\u001b[0m [autoscaler] Cluster upscaled to {104 CPU, 8 GPU}.\n" + "\u001b[36m(autoscaler +20s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n", + "\u001b[36m(autoscaler +20s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", + "\u001b[36m(autoscaler +25s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=3333, ip=10.0.27.32)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "\u001b[36m(MapBatches(drop_columns)->Write pid=116142)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\n", - "\u001b[36m(_MapWorker pid=3332, ip=10.0.27.32)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)\u001b[0m\n", - "\u001b[36m(MapBatches(drop_columns)->Write pid=34034, ip=10.0.171.239)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\u001b[32m [repeated 32x across cluster]\u001b[0m\n", - "2025-08-22 00:18:30,236\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_64_0 execution finished in 154.97 seconds\n", - "2025-08-22 00:18:30,323\tINFO dataset.py:4621 -- Data sink Parquet finished. 2880 rows and 5.8MB data written.\n" + "2025-08-28 05:01:19,478\tWARNING resource_manager.py:551 -- Cluster resources are not engough to run any task from ActorPoolMapOperator[MapBatches(EmbedImages)]. The job may hang forever unless the cluster scales up.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[36m(autoscaler +6m52s)\u001b[0m [autoscaler] Downscaling node i-0b5c2c9a5a27cfba2 (node IP: 10.0.27.32) due to node idle termination.\n", - "\u001b[36m(autoscaler +6m52s)\u001b[0m [autoscaler] Cluster resized to {56 CPU, 4 GPU}.\n" + "\u001b[36m(autoscaler +1m10s)\u001b[0m [autoscaler] Cluster upscaled to {56 CPU, 4 GPU}.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(MapWorker(MapBatches(EmbedImages)) pid=3337, ip=10.0.5.252)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "2025-08-28 05:03:39,362\tINFO streaming_executor.py:279 -- ✔️ Dataset dataset_6_0 execution finished in 163.60 seconds\n", + "2025-08-28 05:03:39,422\tINFO dataset.py:4871 -- Data sink Parquet finished. 2880 rows and 5.8MB data written.\n" ] } ], @@ -718,12 +727,12 @@ "output_type": "stream", "text": [ "Output\n", - "(anyscale +0.9s) Submitting job with config JobConfig(name='image-batch-embeddings', image_uri='anyscale/ray:2.48.0-slim-py312-cu128', compute_config=None, env_vars=None, py_modules=['/home/ray/default/doggos'], py_executable=None, cloud=None, project=None, ray_version=None, job_queue_config=None).\n", - "(anyscale +3.0s) Uploading local dir '/home/ray/default' to cloud storage.\n", - "(anyscale +4.2s) Uploading local dir '/home/ray/default/doggos' to cloud storage.\n", - "(anyscale +5.2s) Job 'image-batch-embeddings' submitted, ID: 'prodjob_cmhr6w7l9fb42be6xjsz1rnxsl'.\n", - "(anyscale +5.2s) View the job in the UI: https://console.anyscale.com/jobs/prodjob_cmhr6w7l9fb42be6xjsz1rnxsl\n", - "(anyscale +5.2s) Use `--wait` to wait for the job to run and stream logs.\n" + "(anyscale +0.8s) Submitting job with config JobConfig(name='image-batch-embeddings', image_uri='anyscale/ray:2.48.0-slim-py312-cu128', compute_config=None, env_vars=None, py_modules=['/home/ray/default/doggos'], py_executable=None, cloud=None, project=None, ray_version=None, job_queue_config=None).\n", + "(anyscale +7.2s) Uploading local dir '/home/ray/default' to cloud storage.\n", + "(anyscale +7.9s) Uploading local dir '/home/ray/default/doggos' to cloud storage.\n", + "(anyscale +9.2s) Job 'image-batch-embeddings' submitted, ID: 'prodjob_7e1fsj9xzs2iryayj7hgbhifl8'.\n", + "(anyscale +9.2s) View the job in the UI: https://console.anyscale.com/jobs/prodjob_7e1fsj9xzs2iryayj7hgbhifl8\n", + "(anyscale +9.2s) Use `--wait` to wait for the job to run and stream logs.\n" ] } ], @@ -790,6 +799,118 @@ "Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n" ] }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "91c47446fb224d72987f0f9b4c9c5e90", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "preprocessor_config.json: 0%| | 0.00/316 [00:00 TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles]\n" + "/home/ray/anaconda3/lib/python3.12/site-packages/ray/data/_internal/datasource/parquet_datasource.py:750: FutureWarning: The default `file_extensions` for `read_parquet` will change from `None` to ['parquet'] after Ray 2.43, and your dataset contains files that don't match the new `file_extensions`. To maintain backwards compatibility, set `file_extensions=None` explicitly.\n", + " warnings.warn(\n", + "2025-08-28 05:03:56,303\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_8_0\n", + "2025-08-28 05:03:56,308\tINFO streaming_executor.py:159 -- Starting execution of Dataset dataset_8_0. Full logs are in /tmp/ray/session_2025-08-28_04-57-43_348032_12595/logs/ray-data\n", + "2025-08-28 05:03:56,309\tINFO streaming_executor.py:160 -- Execution plan of Dataset dataset_8_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7b04bc8e4d444e82950af699e0891b1e", + "model_id": "5d82b793825b412c9ab72693c6fb92ce", "version_major": 2, "version_minor": 0 }, @@ -847,7 +964,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5c7fffbb6cf44a29904a90f20015ab9b", + "model_id": "bdd005beb194490e8c641ef7548fdf09", "version_major": 2, "version_minor": 0 }, @@ -861,7 +978,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "57a3c6fc76d448b49941e9459d88b051", + "model_id": "884a8230054a42c88e655c178827a68f", "version_major": 2, "version_minor": 0 }, @@ -872,11 +989,25 @@ "metadata": {}, "output_type": "display_data" }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6f2d6a1e74fd41f6a9be6c2fefdadf64", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "model.safetensors: 0%| | 0.00/605M [00:00\n", " \n", " Ray version:\n", - " 2.48.0\n", + " 2.49.0\n", " \n", " \n", " Dashboard:\n", - " http://session-466hy7cqu1gzrp8zk8l4byz7l7.i.anyscaleuserdata.com\n", + " http://session-jhxhj69d6ttkjctcxfnsfe7gwk.i.anyscaleuserdata.com\n", "\n", "\n", "\n", @@ -157,7 +157,7 @@ "\n" ], "text/plain": [ - "RayContext(dashboard_url='session-466hy7cqu1gzrp8zk8l4byz7l7.i.anyscaleuserdata.com', python_version='3.12.11', ray_version='2.48.0', ray_commit='61d1966b0b02ce07f95d8f046ea7f6f92f7be190')" + "RayContext(dashboard_url='session-jhxhj69d6ttkjctcxfnsfe7gwk.i.anyscaleuserdata.com', python_version='3.12.11', ray_version='2.49.0', ray_commit='8b349d73c5d5c4b56dc719fcc447d18ae8571dd4')" ] }, "execution_count": null, @@ -337,16 +337,22 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-22 00:26:17,487\tINFO dataset.py:3057 -- Tip: Use `take_batch()` instead of `take() / show()` to return records in pandas or numpy batch format.\n", - "2025-08-22 00:26:17,490\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_72_0\n", - "2025-08-22 00:26:17,522\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_72_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", - "2025-08-22 00:26:17,523\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_72_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)] -> AllToAllOperator[Aggregate] -> LimitOperator[limit=1]\n" + "2025-08-28 05:06:54,182\tINFO dataset.py:3248 -- Tip: Use `take_batch()` instead of `take() / show()` to return records in pandas or numpy batch format.\n", + "2025-08-28 05:06:54,184\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_14_0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-08-28 05:06:54,206\tINFO streaming_executor.py:159 -- Starting execution of Dataset dataset_14_0. Full logs are in /tmp/ray/session_2025-08-28_04-57-43_348032_12595/logs/ray-data\n", + "2025-08-28 05:06:54,207\tINFO streaming_executor.py:160 -- Execution plan of Dataset dataset_14_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)] -> AllToAllOperator[Aggregate] -> LimitOperator[limit=1]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "aa17a81a379547a6b397907ffafec33b", + "model_id": "66271b6a5fb7493998bb818d81eb9d12", "version_major": 2, "version_minor": 0 }, @@ -360,7 +366,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "08f021fb0f014335ab09e4baf362d6c3", + "model_id": "342323504f9046f5ab79cc8fab75fd3d", "version_major": 2, "version_minor": 0 }, @@ -374,7 +380,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bb9749220ab644db83ab9ac57348c6df", + "model_id": "4ad27f68f4954b839ec614deb470dc2c", "version_major": 2, "version_minor": 0 }, @@ -388,7 +394,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7cd4f59d163149d8b1200348eaabebab", + "model_id": "ec8f2f8c07134885bcf1e339079e5602", "version_major": 2, "version_minor": 0 }, @@ -402,7 +408,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9a7fe9e6350a4ce88f4c4a12c3e83fc1", + "model_id": "8af9685130484d73bdcc36ff9a7b6742", "version_major": 2, "version_minor": 0 }, @@ -416,7 +422,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "38470d9ffbeb4f4ca11c2c7d4e70886b", + "model_id": "8e2ff9a70b3047b78d3421a9c6ba4a2a", "version_major": 2, "version_minor": 0 }, @@ -430,7 +436,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ea5ecfb1d6fb4eadb1d55130cdf0ec04", + "model_id": "5d47b1e760004a8ba310e0cda5de47f0", "version_major": 2, "version_minor": 0 }, @@ -444,7 +450,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e75cd967f3494a7cad6204c9f523ccad", + "model_id": "d4a4b8c2fd3240a1ad46f9abd4869497", "version_major": 2, "version_minor": 0 }, @@ -458,7 +464,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "37294f34620d4ed99e4c6db7f489e870", + "model_id": "c881bfe545de4d8e9384ad4a4c4a3346", "version_major": 2, "version_minor": 0 }, @@ -473,8 +479,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-22 00:26:17,662\tWARNING resource_manager.py:130 -- ⚠️ Ray's object store is configured to use only 28.2% of available memory (67.8GB out of 240.5GB total). For optimal Ray Data performance, we recommend setting the object store to at least 50% of available memory. You can do this by setting the 'object_store_memory' parameter when calling ray.init() or by setting the RAY_DEFAULT_OBJECT_STORE_MEMORY_PROPORTION environment variable.\n", - "2025-08-22 00:26:29,748\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_72_0 execution finished in 12.22 seconds\n" + "2025-08-28 05:06:54,275\tWARNING resource_manager.py:134 -- ⚠️ Ray's object store is configured to use only 28.5% of available memory (63.9GiB out of 224.0GiB total). For optimal Ray Data performance, we recommend setting the object store to at least 50% of available memory. You can do this by setting the 'object_store_memory' parameter when calling ray.init() or by setting the RAY_DEFAULT_OBJECT_STORE_MEMORY_PROPORTION environment variable.\n", + "2025-08-28 05:07:03,480\tINFO streaming_executor.py:279 -- ✔️ Dataset dataset_14_0 execution finished in 9.27 seconds\n" ] } ], @@ -513,15 +519,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-22 00:26:30,402\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_80_0\n", - "2025-08-22 00:26:30,433\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_80_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", - "2025-08-22 00:26:30,435\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_80_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" + "2025-08-28 05:07:04,254\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_22_0\n", + "2025-08-28 05:07:04,270\tINFO streaming_executor.py:159 -- Starting execution of Dataset dataset_22_0. Full logs are in /tmp/ray/session_2025-08-28_04-57-43_348032_12595/logs/ray-data\n", + "2025-08-28 05:07:04,271\tINFO streaming_executor.py:160 -- Execution plan of Dataset dataset_22_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9a4fc2809c524fdea65f33187089279c", + "model_id": "05ab1c63234c4c779fbca8267b744477", "version_major": 2, "version_minor": 0 }, @@ -535,7 +541,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7ed00402d404430596be9d367c53de16", + "model_id": "310392fbcf7f4b50bc122405fdfdaaac", "version_major": 2, "version_minor": 0 }, @@ -549,7 +555,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "cb833a76da3245a0bfcf5d7d7959e847", + "model_id": "b600d27a67dc42159f5a97a445538560", "version_major": 2, "version_minor": 0 }, @@ -563,7 +569,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c2223e2ee8e046079a76f8cdcef8abf5", + "model_id": "fbb9d22bf6c74df7b21af88c7363adad", "version_major": 2, "version_minor": 0 }, @@ -577,7 +583,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "14d3a76ba8f84b3ba6729d43dcd221d8", + "model_id": "5b6f5b9affbb4f458851974ec2811594", "version_major": 2, "version_minor": 0 }, @@ -591,7 +597,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5b5ea2e535b04e95b3bb8739e478a678", + "model_id": "936041324dbb49dda5872a3e6d3fa979", "version_major": 2, "version_minor": 0 }, @@ -602,35 +608,22 @@ "metadata": {}, "output_type": "display_data" }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m(autoscaler +25s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n", - "\u001b[36m(autoscaler +25s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", - "\u001b[36m(autoscaler +30s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", - "\u001b[36m(autoscaler +1m15s)\u001b[0m [autoscaler] Cluster upscaled to {104 CPU, 8 GPU}.\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=3320, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "\u001b[36m(MapBatches(drop_columns)->Write pid=44781, ip=10.0.171.239)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\n", - "\u001b[36m(_MapWorker pid=3323, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)\u001b[0m\n", - "\u001b[36m(MapBatches(drop_columns)->Write pid=44781, ip=10.0.171.239)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\u001b[32m [repeated 32x across cluster]\u001b[0m\n", - "2025-08-22 00:29:10,480\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_80_0 execution finished in 160.04 seconds\n", - "2025-08-22 00:29:10,570\tINFO dataset.py:4621 -- Data sink Parquet finished. 2880 rows and 5.9MB data written.\n", - "2025-08-22 00:29:10,582\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_83_0\n", - "2025-08-22 00:29:10,601\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_83_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", - "2025-08-22 00:29:10,603\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_83_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" + "\u001b[36m(MapWorker(MapBatches(EmbedImages)) pid=9215, ip=10.0.5.252)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "2025-08-28 05:07:20,682\tINFO streaming_executor.py:279 -- ✔️ Dataset dataset_22_0 execution finished in 16.41 seconds\n", + "2025-08-28 05:07:20,747\tINFO dataset.py:4871 -- Data sink Parquet finished. 2880 rows and 5.9MB data written.\n", + "2025-08-28 05:07:20,759\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_25_0\n", + "2025-08-28 05:07:20,774\tINFO streaming_executor.py:159 -- Starting execution of Dataset dataset_25_0. Full logs are in /tmp/ray/session_2025-08-28_04-57-43_348032_12595/logs/ray-data\n", + "2025-08-28 05:07:20,775\tINFO streaming_executor.py:160 -- Execution plan of Dataset dataset_25_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)->Write]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0abbbeeb2a35431a95d00c4c921ee61b", + "model_id": "cb6abfa554bd48a8a91f6bf67b72321d", "version_major": 2, "version_minor": 0 }, @@ -644,7 +637,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bf9288444a51499a8b1083a1b784b210", + "model_id": "ebd4d9a7ce0e494cbd256531e82c76a0", "version_major": 2, "version_minor": 0 }, @@ -658,7 +651,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8ec81536f155464c9a2ecf135cf83d80", + "model_id": "bbf925aa09a9434d9a0e1b0a0434a977", "version_major": 2, "version_minor": 0 }, @@ -672,7 +665,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "512fded93ca04523b3ed812aec1ada77", + "model_id": "c061d8e4bee4444fa9d1ddfafa9f99bc", "version_major": 2, "version_minor": 0 }, @@ -686,7 +679,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b3b9c393001d49f4b0535b625a132edb", + "model_id": "47cb0744205149d1860913bf2124338d", "version_major": 2, "version_minor": 0 }, @@ -700,7 +693,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5f38287bfd4b4795a3839f44b0d2b1a8", + "model_id": "a68e006347144beca6b0593d148d0f8d", "version_major": 2, "version_minor": 0 }, @@ -715,36 +708,19 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-22 00:29:12,374\tWARNING streaming_executor_state.py:764 -- Operator produced a RefBundle with a different schema than the previous one. Previous schema: image: extension>\n", + "2025-08-28 05:07:22,417\tWARNING streaming_executor_state.py:790 -- Operator produced a RefBundle with a different schema than the previous one. Previous schema: image: extension>\n", "path: string, new schema: image: extension>\n", "path: string. This may lead to unexpected behavior.\n", - "2025-08-22 00:29:13,110\tWARNING streaming_executor_state.py:764 -- Operator produced a RefBundle with a different schema than the previous one. Previous schema: image: extension>\n", + "2025-08-28 05:07:22,642\tWARNING streaming_executor_state.py:790 -- Operator produced a RefBundle with a different schema than the previous one. Previous schema: image: extension>\n", "path: string\n", "class: string\n", "label: int64, new schema: image: extension>\n", "path: string\n", "class: string\n", "label: int64. This may lead to unexpected behavior.\n", - "\u001b[36m(_MapWorker pid=3910, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "\u001b[36m(MapBatches(drop_columns)->Write pid=121066)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\u001b[32m [repeated 7x across cluster]\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m(autoscaler +3m10s)\u001b[0m [autoscaler] [8CPU-32GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", - "\u001b[36m(autoscaler +3m10s)\u001b[0m [autoscaler] [8CPU-32GB|m5.2xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(_MapWorker pid=4731, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(drop_columns)->Write pid=45557, ip=10.0.171.239)\u001b[0m FilenameProvider have to provide proper filename template including '{{i}}' macro to ensure unique filenames when writing multiple files. Appending '{{i}}' macro to the end of the file. For more details on the expected filename template checkout PyArrow's `write_to_dataset` API\u001b[32m [repeated 6x across cluster]\u001b[0m\n", - "2025-08-22 00:29:24,485\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_83_0 execution finished in 13.88 seconds\n", - "2025-08-22 00:29:24,531\tINFO dataset.py:4621 -- Data sink Parquet finished. 720 rows and 1.5MB data written.\n" + "\u001b[36m(MapWorker(MapBatches(EmbedImages)) pid=23307, ip=10.0.5.252)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 4x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)\u001b[0m\n", + "2025-08-28 05:07:33,184\tINFO streaming_executor.py:279 -- ✔️ Dataset dataset_25_0 execution finished in 12.41 seconds\n", + "2025-08-28 05:07:33,214\tINFO dataset.py:4871 -- Data sink Parquet finished. 720 rows and 1.5MB data written.\n" ] } ], @@ -941,21 +917,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-22 00:29:25,511\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_85_0\n", - "2025-08-22 00:29:25,523\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_85_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-08-22 00:29:25,524\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_85_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> LimitOperator[limit=3]\n" + "2025-08-28 05:07:34,380\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_27_0\n", + "2025-08-28 05:07:34,394\tINFO streaming_executor.py:159 -- Starting execution of Dataset dataset_27_0. Full logs are in /tmp/ray/session_2025-08-28_04-57-43_348032_12595/logs/ray-data\n", + "2025-08-28 05:07:34,395\tINFO streaming_executor.py:160 -- Execution plan of Dataset dataset_27_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> LimitOperator[limit=3]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c596b6e4878f41a9ac527bfb3925c95e", + "model_id": "a81b12e57aba4dc3b16c2fafcb91cade", "version_major": 2, "version_minor": 0 }, @@ -969,7 +939,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "124f996b75cd452d89bb404100035d45", + "model_id": "c176ff7a83b54b9b99fc7f5dc12e92c7", "version_major": 2, "version_minor": 0 }, @@ -983,7 +953,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f76037bae4334040bf81176c7ddab96d", + "model_id": "59f86a3defeb41de9be9874b6ae8a234", "version_major": 2, "version_minor": 0 }, @@ -997,7 +967,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "323cdef1b95f405da16d7478a2295072", + "model_id": "1acb1ef77faf42119ee3702dcaa2bcd7", "version_major": 2, "version_minor": 0 }, @@ -1011,7 +981,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2f9da19c9b7046b68d748780426d7886", + "model_id": "59526fe14e3a41cd8abb44b306127a7d", "version_major": 2, "version_minor": 0 }, @@ -1025,7 +995,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3ff417b842ae472f8ef0e640443d3897", + "model_id": "98c20dcbec0b4cb8871a4635a3d02815", "version_major": 2, "version_minor": 0 }, @@ -1039,7 +1009,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b3b34ef888164ae8a618828d9832fd8f", + "model_id": "09f2cbe0afe04bc1a046e4e45a387fc1", "version_major": 2, "version_minor": 0 }, @@ -1054,19 +1024,19 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=4911, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "2025-08-22 00:29:36,437\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_85_0 execution finished in 10.91 seconds\n", - "/tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", + "\u001b[36m(MapWorker(MapBatches(EmbedImages)) pid=26114, ip=10.0.5.252)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "2025-08-28 05:07:45,755\tINFO streaming_executor.py:279 -- ✔️ Dataset dataset_27_0 execution finished in 11.36 seconds\n", + "/tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", " tensor_batch[key] = torch.as_tensor(\n" ] }, { "data": { "text/plain": [ - "{'embedding': tensor([[ 0.4219, 0.3688, -0.1833, ..., 0.6288, 0.2298, -0.3989],\n", - " [ 0.0385, 0.3297, 0.2076, ..., 0.3434, -0.5492, 0.0362],\n", - " [ 0.1881, 0.1737, -0.3069, ..., 0.3336, 0.1783, -0.0299]]),\n", - " 'label': tensor([11, 34, 7])}" + "{'embedding': tensor([[ 0.0245, 0.6505, 0.0627, ..., 0.4001, -0.2721, -0.0673],\n", + " [-0.2416, 0.2315, 0.0255, ..., 0.4065, 0.2805, -0.1156],\n", + " [-0.2301, -0.3628, 0.1086, ..., 0.3038, 0.0543, 0.6214]]),\n", + " 'label': tensor([10, 29, 27])}" ] }, "execution_count": null, @@ -1321,7 +1291,16 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ray/anaconda3/lib/python3.12/site-packages/ray/data/_internal/datasource/parquet_datasource.py:750: FutureWarning: The default `file_extensions` for `read_parquet` will change from `None` to ['parquet'] after Ray 2.43, and your dataset contains files that don't match the new `file_extensions`. To maintain backwards compatibility, set `file_extensions=None` explicitly.\n", + " warnings.warn(\n" + ] + } + ], "source": [ "# Load preprocessed datasets.\n", "preprocessed_train_ds = ray.data.read_parquet(preprocessed_train_path)\n", @@ -1345,2523 +1324,155 @@ ")\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Train.\n", + "results = trainer.fit()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Ray Train" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- automatically handles **multi-node, multi-GPU** setup with no manual SSH setup or hostfile configs. \n", + "- define **per-worker fractional resource requirements**, for example, 2 CPUs and 0.5 GPU per worker.\n", + "- run on **heterogeneous machines** and scale flexibly, for example, CPU for preprocessing and GPU for training. \n", + "- built-in **fault tolerance** with retry of failed workers and continue from last checkpoint.\n", + "- supports Data Parallel, Model Parallel, Parameter Server, and even custom strategies.\n", + "- [Ray Compiled graphs](https://docs.ray.io/en/latest/ray-core/compiled-graph/ray-compiled-graph.html) allow you to even define different parallelism for jointly optimizing multiple models like Megatron, DeepSpeed, etc., or only allow for one global setting.\n", + "- You can also use Torch DDP, FSPD, DeepSpeed, etc., under the hood." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "🔥 [RayTurbo Train](https://docs.anyscale.com/rayturbo/rayturbo-train) offers even more improvement to the price-performance ratio, performance monitoring and more:\n", + "- **elastic training** to scale to a dynamic number of workers, continue training on fewer resources, even on spot instances.\n", + "- **purpose-built dashboard** designed to streamline the debugging of Ray Train workloads:\n", + " - Monitoring: View the status of training runs and train workers.\n", + " - Metrics: See insights on training throughput and training system operation time.\n", + " - Profiling: Investigate bottlenecks, hangs, or errors from individual training worker processes.\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can view experiment metrics and model artifacts in the model registry. You're using OSS MLflow so you can run the server by pointing to the model registry location:\n", + "\n", + "```bash\n", + "mlflow server -h 0.0.0.0 -p 8080 --backend-store-uri /mnt/cluster_storage/mlflow/doggos\n", + "```\n", + "\n", + "You can view the dashboard by going to the **Overview tab** > **Open Ports**. \n", + "\n", + "\n", + "\n", + "You also have the preceding Ray Dashboard and Train workload specific dashboards.\n", + "\n", + "\n" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(TrainController pid=125066)\u001b[0m [State Transition] INITIALIZING -> SCHEDULING.\n", - "\u001b[36m(TrainController pid=125066)\u001b[0m Attempting to start training worker group of size 4 with the following resources: [{'CPU': 8, 'GPU': 2, 'accelerator_type:T4': 0.001}] * 4\n", - "\u001b[36m(TrainController pid=125066)\u001b[0m Using blocking ray.get inside async actor. This blocks the event loop. Please use `await` on object ref with asyncio.gather if you want to yield execution to the event loop instead.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m(autoscaler +3m40s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 1 to 2).\n", - "\u001b[36m(autoscaler +3m40s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", - "\u001b[36m(autoscaler +3m45s)\u001b[0m [autoscaler] Cluster upscaled to {112 CPU, 8 GPU}.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(TrainController pid=125066)\u001b[0m Retrying the launch of the training worker group. The previous launch attempt encountered the following failure:\n", - "\u001b[36m(TrainController pid=125066)\u001b[0m The worker group startup timed out after 30.0 seconds waiting for 4 workers. Potential causes include: (1) temporary insufficient cluster resources while waiting for autoscaling (ignore this warning in this case), (2) infeasible resource request where the provided `ScalingConfig` cannot be satisfied), and (3) transient network issues. Set the RAY_TRAIN_WORKER_GROUP_START_TIMEOUT_S environment variable to increase the timeout.\n", - "\u001b[36m(TrainController pid=125066)\u001b[0m [State Transition] SCHEDULING -> RESCHEDULING.\n", - "\u001b[36m(TrainController pid=125066)\u001b[0m [State Transition] RESCHEDULING -> SCHEDULING.\n", - "\u001b[36m(TrainController pid=125066)\u001b[0m Attempting to start training worker group of size 4 with the following resources: [{'CPU': 8, 'GPU': 2, 'accelerator_type:T4': 0.001}] * 4\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m(autoscaler +4m30s)\u001b[0m [autoscaler] Cluster upscaled to {160 CPU, 12 GPU}.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m Setting up process group for: env:// [rank=0, world_size=4]\n", - "\u001b[36m(RayTrainWorker pid=16056, ip=10.0.4.102)\u001b[0m Moving model to device: cuda:0\n", - "\u001b[36m(TrainController pid=125066)\u001b[0m Started training worker group of size 4: \n", - "\u001b[36m(TrainController pid=125066)\u001b[0m - (ip=10.0.34.27, pid=3319) world_rank=0, local_rank=0, node_rank=0\n", - "\u001b[36m(TrainController pid=125066)\u001b[0m - (ip=10.0.34.27, pid=3320) world_rank=1, local_rank=1, node_rank=0\n", - "\u001b[36m(TrainController pid=125066)\u001b[0m - (ip=10.0.4.102, pid=16056) world_rank=2, local_rank=0, node_rank=1\n", - "\u001b[36m(TrainController pid=125066)\u001b[0m - (ip=10.0.4.102, pid=16055) world_rank=3, local_rank=1, node_rank=1\n", - "\u001b[36m(TrainController pid=125066)\u001b[0m [State Transition] SCHEDULING -> RUNNING.\n", - "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m 2025/08/22 00:32:11 INFO mlflow.tracking.fluent: Experiment with name 'doggos' does not exist. Creating a new experiment.\n", - "\u001b[36m(RayTrainWorker pid=16056, ip=10.0.4.102)\u001b[0m Wrapping provided model in DistributedDataParallel.\n", - "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m /home/ray/anaconda3/lib/python3.12/site-packages/ray/data/iterator.py:445: RayDeprecationWarning: Passing a function to `iter_torch_batches(collate_fn)` is deprecated in Ray 2.47. Please switch to using a callable class that inherits from `ArrowBatchCollateFn`, `NumpyBatchCollateFn`, or `PandasBatchCollateFn`.\n", - "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m warnings.warn(\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2f9b807b07e24754b872e832186a7ecc", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "477c2f56fd9e4c31974d33eec3c722a0", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "3b3d7e99c85a496ebad1cba791a6bcd1", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, { "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2263907df3504600a2281cb5bb1feb81", - "version_major": 2, - "version_minor": 0 - }, "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" + "run_id d54aa07059384d139ea572123ae9409c\n", + "experiment_id 653138458592289747\n", + "status FINISHED\n", + "artifact_uri file:///mnt/cluster_storage/mlflow/doggos/6531...\n", + "start_time 2025-08-28 05:10:15.049000+00:00\n", + "end_time 2025-08-28 05:10:33.936000+00:00\n", + "metrics.lr 0.001\n", + "metrics.val_loss 0.778273\n", + "metrics.train_loss 0.39104\n", + "params.lr_factor 0.8\n", + "params.hidden_dim 256\n", + "params.embedding_dim 512\n", + "params.dropout_p 0.3\n", + "params.experiment_name doggos\n", + "params.batch_size 256\n", + "params.lr 0.001\n", + "params.num_classes 36\n", + "params.class_to_label {'pomeranian': 0, 'rottweiler': 1, 'boxer': 2,...\n", + "params.num_epochs 20\n", + "params.lr_patience 3\n", + "params.model_registry /mnt/cluster_storage/mlflow/doggos\n", + "tags.mlflow.source.name /home/ray/anaconda3/lib/python3.12/site-packag...\n", + "tags.mlflow.source.type LOCAL\n", + "tags.mlflow.runName judicious-panda-916\n", + "tags.mlflow.user ray\n", + "Name: 0, dtype: object" ] }, + "execution_count": null, "metadata": {}, - "output_type": "display_data" - }, + "output_type": "execute_result" + } + ], + "source": [ + "# Sorted runs\n", + "mlflow.set_tracking_uri(f\"file:{model_registry}\")\n", + "sorted_runs = mlflow.search_runs(\n", + " experiment_names=[experiment_name], \n", + " order_by=[\"metrics.val_loss ASC\"])\n", + "best_run = sorted_runs.iloc[0]\n", + "best_run\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Production Job" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can easily wrap the training workload as a production grade [Anyscale Job](https://docs.anyscale.com/platform/jobs/) ([API ref](https://docs.anyscale.com/reference/job-api/)).\n", + "\n", + "**Note**: \n", + "- This Job uses a `containerfile` to define dependencies, but you could easily use a pre-built image as well.\n", + "- You can specify the compute as a [compute config](https://docs.anyscale.com/configuration/compute-configuration/) or inline in a [job config](https://docs.anyscale.com/reference/job-api#job-cli) file.\n", + "- When you don't specify compute while launching from a workspace, this configuration defaults to the compute configuration of the workspace." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Registered dataset logger for dataset train_88_0\n", - "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Starting execution of Dataset train_88_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", - "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Execution plan of Dataset train_88_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> OutputSplitter[split(4, equal=True)]\n", - "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m ⚠️ Ray's object store is configured to use only 28.5% of available memory (195.9GB out of 687.2GB total). For optimal Ray Data performance, we recommend setting the object store to at least 50% of available memory. You can do this by setting the 'object_store_memory' parameter when calling ray.init() or by setting the RAY_DEFAULT_OBJECT_STORE_MEMORY_PROPORTION environment variable.\n", - "\u001b[36m(RayTrainWorker pid=16056, ip=10.0.4.102)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", - "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m Moving model to device: cuda:0\n", - "\u001b[36m(RayTrainWorker pid=3319, ip=10.0.34.27)\u001b[0m Wrapping provided model in DistributedDataParallel.\n", - "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m ✔️ Dataset train_88_0 execution finished in 2.84 seconds\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "dbfdddd23f0a43658486e78ef5db13ec", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4a0cd95efca346bcbd9c962648fb8d18", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9d0c2a55758742729e3e4c41aff6daf7", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "fbab3701238a4fc3a295d6116e1908f4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8cca0efc0fdf42309956588e2ebad8d9", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c4a5eedd70c540c68156ae60bd821773", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "692887626f444deaa309ee332a270796", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "364d9c81b14d44b18703999214217018", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(RayTrainWorker pid=16055, ip=10.0.4.102)\u001b[0m /home/ray/anaconda3/lib/python3.12/site-packages/ray/data/iterator.py:445: RayDeprecationWarning: Passing a function to `iter_torch_batches(collate_fn)` is deprecated in Ray 2.47. Please switch to using a callable class that inherits from `ArrowBatchCollateFn`, `NumpyBatchCollateFn`, or `PandasBatchCollateFn`.\u001b[32m [repeated 3x across cluster]\u001b[0m\n", - "\u001b[36m(RayTrainWorker pid=16055, ip=10.0.4.102)\u001b[0m warnings.warn(\u001b[32m [repeated 3x across cluster]\u001b[0m\n", - "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Registered dataset logger for dataset train_88_1\u001b[32m [repeated 2x across cluster]\u001b[0m\n", - "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Starting execution of Dataset train_88_1. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\u001b[32m [repeated 2x across cluster]\u001b[0m\n", - "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Execution plan of Dataset train_88_1: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> OutputSplitter[split(4, equal=True)]\u001b[32m [repeated 2x across cluster]\u001b[0m\n", - "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m ⚠️ Ray's object store is configured to use only 28.5% of available memory (195.9GB out of 687.2GB total). For optimal Ray Data performance, we recommend setting the object store to at least 50% of available memory. You can do this by setting the 'object_store_memory' parameter when calling ray.init() or by setting the RAY_DEFAULT_OBJECT_STORE_MEMORY_PROPORTION environment variable.\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c94637fd36f7408887c53978632c81d6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d9c770c92271481e824deabb97479d02", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e9394ea598c44b8eaed72c3a567e8f80", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "78f5d148c07843d6ba79aa4443fac4c2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f3c0cca34393418db7a85bf3b5da8de0", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1cf9d449bed34bd58633604913c4b6da", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "0f9207fa9fe74fb480d20ab6792412e8", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d2d38403e0ea479c8745ba86e479e5b6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9245ec966b7142a39ccc3fb881ea1895", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "547f6b9e985a4fbf942e01dd9687245d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "03a5c5eeddbc4c0dbc4b2ae694a2cf23", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a12199d167f24ecab2b9dfe37adafee1", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f4a3036c6d42495db3704671a5913e97", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(RayTrainWorker pid=3320, ip=10.0.34.27)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 3x across cluster]\u001b[0m\n", - "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m ✔️ Dataset val_89_2 execution finished in 0.14 seconds\u001b[32m [repeated 5x across cluster]\u001b[0m\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8daad09e465a4c95b282f6bff58488c4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1c61f75ca7ca467c8995ea64d5fbe622", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9686329d65264f15b583f6a26966ca46", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f06b9dd78b594b3e8abbede8e584d6f6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2dac4a6c6b69497cbfb1c2730ae4c84b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "310c75a42eef4481bf92376c8225732a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "19eca161119f4c38839e2b2ff2a4bb36", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "59f241a3d0364ebbbae4e7a5a94037a5", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "7c6147789fbc4007bbc309f7537782fa", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ae5a1436d1e148cdb8a4aac594e2ee5a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "0e6c03e36b9b4373b9e61c5e73e3943a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "5d154effa9e84089b1fac7bc66574802", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e8ce7b2eef094a078cb3b516e6d381d2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "fc4f18d5ca0747f49cb7d070fa984f13", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8a02cba1818049cabd118a8cb4ad16fd", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2faf9980bf2f44d98a4ba61cd943ca2e", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "17eb0e1b001444d5b9b089e3540143ed", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c614080c873a4658b47a79ea793ec211", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "59bed3f576ba412091b1431810674dc1", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6c28eaa475dd461c9a4c3524ab635758", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4a396b6c1f8243b289ac96f5b8c5e354", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "b7a446b66d4447c68b6cd69ae370aec9", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "fff93ed630644210b0e7fdb218c0fba4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "df70761adc79445ea206305bdbef50fa", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e9917e5b9629429eb6423cb23d1ac8ca", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "02466fb2e05e40f781ba101d2e1c5394", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "cba46412a0e1470093b37dd44571c867", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d5c3e448d3254eb2bd6ac87c2ee60ff7", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4a46866464514d189d040bff9c170371", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f77002b71fd641f79a0b1320045ce8bf", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1b58d2b8f06c4d688d1062b11ab9ad1c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m Registered dataset logger for dataset val_89_6\u001b[32m [repeated 11x across cluster]\u001b[0m\n", - "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m Starting execution of Dataset val_89_6. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\u001b[32m [repeated 11x across cluster]\u001b[0m\n", - "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m Execution plan of Dataset val_89_6: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> OutputSplitter[split(4, equal=True)]\u001b[32m [repeated 11x across cluster]\u001b[0m\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "072e8ee325c546f1ae10cb085631db3e", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a0ed51b8f63d43c3a75bd861e63d4e23", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a0f4b65c405644829fd3e560487979ba", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1743714b36eb4c3a9e066854bc55e9a0", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9c91384d8ab642eda0883c097edeea4f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "abfa908329d34ee192c35a5aec2b38b0", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8c1c6421140e4ea99f06bd917697fadf", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6ce5252ebcb244fa9493dde86699516f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f8a34c741f594ef89fd99172e59a34c5", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9cb4841bc8fa49448e31612719799b03", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "50f1a2962113459884deef4975308b07", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "59ec7aee99934c6bbc3c73a093e3b4fe", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a1bc4dd100464a549bc322eedf97512f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f9be5454b006445d8a75769c7264d770", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "82326f0ca01b4f5c92731f21a14e7bbb", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2ce3abb1f2ac4320a05ab07b9e530679", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "774c642be6a143e482201d2a60b4f725", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ea41e9ce5d4749269ab068ee3fc4c3f1", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "fdeb575b9f12491d85437877bf16b0f0", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "05fdb6464b7542bd894d27043329da01", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6cbeb9d41ce746e6af9f74ea3db2dd58", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "fe06543c2f074c4a8778f11d493b40f6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c35a3a74fb4a42eaba77b7c9bf57835f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "aec3501a42bf4e7488ef6d6fdf931b7c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "838a6b8cb0f8462bafd777ba17ca0145", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2735cdc36bb34dd68340fbbe8e4a007d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "bb69d8033e3e44fcba6bd2c12b076344", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c8de0eb20bc34c7799e701bd8dfdf093", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m ✔️ Dataset val_89_9 execution finished in 0.12 seconds\u001b[32m [repeated 14x across cluster]\u001b[0m\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "94b5b81bb4d549ff852bb292d5eabe0a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1f2a660cfbe5483683fdaa6de6e83731", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ed36ce7aeeba40bea644c1ec216917cb", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "7f549b806ec443fab242e9734063166f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "5ca785575831425182c8606cef4c22ed", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "44e5707abbc545e3b52a327e338d42c8", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8f2719cac015410883ffbb3e5bf95f35", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a19c11e6356345c2b1e255492d1c1077", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "21ba9584d021403dacca550bb11ee8a2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "63be175948ae4d41b17c7a4c2c3de9d9", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "3cb193c2285649bd8f987b98ae9b4705", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1d227a26a0514824a0ea7c2964688e6b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "543c26f1127447cf90765a009caf2b67", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2bc02cb3fe2846f983128f974a822fba", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f71f650ff2884e8fa4538cd059a7408f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6fa5b191ae2c45d2a0dac2f718cbed08", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1bc7f991b39444bcaed694e25020c82e", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "0c7df79581f94f4eb66f0fcb95260e1e", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "adab6b6ea3ca44f6be524a1deb1ef639", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e757b1e650e04c6abf451db64a698bb5", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c042c13431d34e08bfbd78e31d3d5de6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d09c9c885b60411fa6331b4cbb0725dc", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ce85530f583f4b40a46ac81f83abeb36", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "236a34616ba048bbb2695c5aaf9416f5", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "24fca83b96a842318cf502782e1ed1b6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "057b1ec3b01a478f80a58686bd171735", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4f862da0dfbb45e5a60f80b9636e071b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "58a97c021f5a49e69202f99f38bad1c0", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d7eb25150fb74086a63d7fe5ac160cb1", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f17b808f250d41929e4870bb83fc477c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "feb8925122f74f11a0b4d406549a8998", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9cb0e78fa32d417bbd237ea1b790ad86", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "aae13b8832ff40c4a768e08fc1489292", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8663407dc9a745408be1707fdcb3de64", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1f3fe18d3a114499bb0e8aedc9087516", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "dde2396c4c4743828a5b45b5ae8dd077", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9c28151437dc45b19eea8a774a565d54", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d4da9d395b3b40b7beaf40f96b020e4d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "72f25e200d844c3c8fb2e46bceba5e0e", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "745b8abd99a148cfb35a207634053f68", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Registered dataset logger for dataset train_88_15\u001b[32m [repeated 17x across cluster]\u001b[0m\n", - "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Starting execution of Dataset train_88_15. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\u001b[32m [repeated 17x across cluster]\u001b[0m\n", - "\u001b[36m(SplitCoordinator pid=125821)\u001b[0m Execution plan of Dataset train_88_15: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> OutputSplitter[split(4, equal=True)]\u001b[32m [repeated 17x across cluster]\u001b[0m\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ea923fad3a6b427e95fc73aca1de7d0b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ee86e41631394e14a447e76d51b0b55b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "b1b038c8a53f437b962ea4b2728f7ee2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ab1e205e2d2a4e40b0399497be2d3eaf", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9b0bc3fcded44cbcada0d64d6c03dd9c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c90593c23a5342cb9fb45b59a6d23954", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9c01556845fd474abd85d9bf39424908", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c35afae5f34c43b88d1461be949ab7e2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "63741d7ac05b462893c79533aab90adb", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e9a9894f3518447c8d977460cde7fd2b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a198c557b1e84c15975bafcc67ca2501", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6cfd359d3fc24709933a740eb38fe408", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2dd17bebcef148bfb4267274ea6538bf", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6381b5c191cd4a608067dad981936f29", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e54bd73a22084efaa21246f9ab88be18", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "efe59371cb0b40489158ba929bfcadd9", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9d0fb221f5fb4f1c865fb2d383f3f66f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ee62c3c15ec5423baf0613424d66b960", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "3d1adb75395c4367ac14d510c6f9e891", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4cd34884675440ffafd0334783920856", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "472913bf0ccb485b90de55a05501629f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "81d1e24ae643455d92707226660390fe", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a7f776485f714d66922fffe6f75d4aeb", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c753d595da314ff687d8357a56484ac2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ffd47b8311b44a6d9ef72e99ebda6b3e", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "99cf3b9ba6c042a39dafe5cef3e49349", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "644e2243a1724d40bc6ad9ac5cd349b0", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "933dc5a56bd64ee69ba1bd23c806c434", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "59a2b83763e04146aeec9edacaf72ed8", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "801a70882a9546b1b9c2e7998178d6cf", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "526adc86f7674fae8b0e44cde52fd920", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "18414e84c96048a196a579fe5e5cdf79", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125821) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(SplitCoordinator pid=125822)\u001b[0m ✔️ Dataset val_89_18 execution finished in 0.12 seconds\u001b[32m [repeated 18x across cluster]\u001b[0m\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1e1a34e9e96c4109b1ff5b38e23eef3d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) Running 0: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c377ec61738f42dfa1ee8638ce0fb4ed", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ListFiles 1: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "086c0e1c3c8443659adcdb4d5ec55d85", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - ReadFiles 2: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "f7fe6c34adb44a4780cb4c7f320ab3f4", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "(pid=125822) - split(4, equal=True) 3: 0.00 row [00:00, ? row/s]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(TrainController pid=125066)\u001b[0m [State Transition] RUNNING -> FINISHED.\n" - ] - } - ], - "source": [ - "# Train.\n", - "results = trainer.fit()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Ray Train" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- automatically handles **multi-node, multi-GPU** setup with no manual SSH setup or hostfile configs. \n", - "- define **per-worker fractional resource requirements**, for example, 2 CPUs and 0.5 GPU per worker.\n", - "- run on **heterogeneous machines** and scale flexibly, for example, CPU for preprocessing and GPU for training. \n", - "- built-in **fault tolerance** with retry of failed workers and continue from last checkpoint.\n", - "- supports Data Parallel, Model Parallel, Parameter Server, and even custom strategies.\n", - "- [Ray Compiled graphs](https://docs.ray.io/en/latest/ray-core/compiled-graph/ray-compiled-graph.html) allow you to even define different parallelism for jointly optimizing multiple models like Megatron, DeepSpeed, etc., or only allow for one global setting.\n", - "- You can also use Torch DDP, FSPD, DeepSpeed, etc., under the hood." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "🔥 [RayTurbo Train](https://docs.anyscale.com/rayturbo/rayturbo-train) offers even more improvement to the price-performance ratio, performance monitoring and more:\n", - "- **elastic training** to scale to a dynamic number of workers, continue training on fewer resources, even on spot instances.\n", - "- **purpose-built dashboard** designed to streamline the debugging of Ray Train workloads:\n", - " - Monitoring: View the status of training runs and train workers.\n", - " - Metrics: See insights on training throughput and training system operation time.\n", - " - Profiling: Investigate bottlenecks, hangs, or errors from individual training worker processes.\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can view experiment metrics and model artifacts in the model registry. You're using OSS MLflow so you can run the server by pointing to the model registry location:\n", - "\n", - "```bash\n", - "mlflow server -h 0.0.0.0 -p 8080 --backend-store-uri /mnt/cluster_storage/mlflow/doggos\n", - "```\n", - "\n", - "You can view the dashboard by going to the **Overview tab** > **Open Ports**. \n", - "\n", - "\n", - "\n", - "You also have the preceding Ray Dashboard and Train workload specific dashboards.\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "run_id fcb9ef8c96f844f08bcd0185601f3dbd\n", - "experiment_id 858816514880031760\n", - "status FINISHED\n", - "artifact_uri file:///mnt/cluster_storage/mlflow/doggos/8588...\n", - "start_time 2025-08-22 00:32:11.522000+00:00\n", - "end_time 2025-08-22 00:32:32.895000+00:00\n", - "metrics.train_loss 0.35504\n", - "metrics.val_loss 0.593301\n", - "metrics.lr 0.001\n", - "params.lr_patience 3\n", - "params.dropout_p 0.3\n", - "params.num_epochs 20\n", - "params.lr 0.001\n", - "params.num_classes 36\n", - "params.hidden_dim 256\n", - "params.experiment_name doggos\n", - "params.batch_size 256\n", - "params.model_registry /mnt/cluster_storage/mlflow/doggos\n", - "params.class_to_label {'border_collie': 0, 'pomeranian': 1, 'basset'...\n", - "params.lr_factor 0.8\n", - "params.embedding_dim 512\n", - "tags.mlflow.user ray\n", - "tags.mlflow.source.type LOCAL\n", - "tags.mlflow.runName enthused-donkey-931\n", - "tags.mlflow.source.name /home/ray/anaconda3/lib/python3.12/site-packag...\n", - "Name: 0, dtype: object" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Sorted runs\n", - "mlflow.set_tracking_uri(f\"file:{model_registry}\")\n", - "sorted_runs = mlflow.search_runs(\n", - " experiment_names=[experiment_name], \n", - " order_by=[\"metrics.val_loss ASC\"])\n", - "best_run = sorted_runs.iloc[0]\n", - "best_run\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Production Job" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can easily wrap the training workload as a production grade [Anyscale Job](https://docs.anyscale.com/platform/jobs/) ([API ref](https://docs.anyscale.com/reference/job-api/)).\n", - "\n", - "**Note**: \n", - "- This Job uses a `containerfile` to define dependencies, but you could easily use a pre-built image as well.\n", - "- You can specify the compute as a [compute config](https://docs.anyscale.com/configuration/compute-configuration/) or inline in a [job config](https://docs.anyscale.com/reference/job-api#job-cli) file.\n", - "- When you don't specify compute while launching from a workspace, this configuration defaults to the compute configuration of the workspace." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Output\n", - "(anyscale +0.9s) Submitting job with config JobConfig(name='train-image-model', image_uri='anyscale/ray:2.48.0-slim-py312-cu128', compute_config=None, env_vars=None, py_modules=['/home/ray/default/doggos'], py_executable=None, cloud=None, project=None, ray_version=None, job_queue_config=None).\n", - "(anyscale +2.8s) Uploading local dir '/home/ray/default' to cloud storage.\n", - "(anyscale +4.3s) Uploading local dir '/home/ray/default/doggos' to cloud storage.\n", - "(anyscale +5.4s) Job 'train-image-model' submitted, ID: 'prodjob_ac1sxbql2i2vah66k2462bhxie'.\n", - "(anyscale +5.4s) View the job in the UI: https://console.anyscale.com/jobs/prodjob_ac1sxbql2i2vah66k2462bhxie\n", - "(anyscale +5.4s) Use `--wait` to wait for the job to run and stream logs.\n" + "Output\n", + "(anyscale +0.8s) Submitting job with config JobConfig(name='train-image-model', image_uri='anyscale/ray:2.48.0-slim-py312-cu128', compute_config=None, env_vars=None, py_modules=['/home/ray/default/doggos'], py_executable=None, cloud=None, project=None, ray_version=None, job_queue_config=None).\n", + "(anyscale +3.0s) Uploading local dir '/home/ray/default' to cloud storage.\n", + "(anyscale +3.8s) Uploading local dir '/home/ray/default/doggos' to cloud storage.\n", + "(anyscale +4.9s) Job 'train-image-model' submitted, ID: 'prodjob_zfy5ak9a5masjb4vuidtxvxpqt'.\n", + "(anyscale +4.9s) View the job in the UI: https://console.anyscale.com/jobs/prodjob_zfy5ak9a5masjb4vuidtxvxpqt\n", + "(anyscale +4.9s) Use `--wait` to wait for the job to run and stream logs.\n" ] } ], @@ -3966,15 +1577,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-22 00:34:12,802\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_96_0\n", - "2025-08-22 00:34:12,814\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_96_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", - "2025-08-22 00:34:12,815\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_96_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> TaskPoolMapOperator[MapBatches(TorchPredictor)] -> LimitOperator[limit=1]\n" + "2025-08-28 05:10:42,369\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_40_0\n", + "2025-08-28 05:10:42,388\tINFO streaming_executor.py:159 -- Starting execution of Dataset dataset_40_0. Full logs are in /tmp/ray/session_2025-08-28_04-57-43_348032_12595/logs/ray-data\n", + "2025-08-28 05:10:42,388\tINFO streaming_executor.py:160 -- Execution plan of Dataset dataset_40_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> TaskPoolMapOperator[MapBatches(TorchPredictor)] -> LimitOperator[limit=1]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "50d1c62a744146a398da57614e787e8c", + "model_id": "9c8deb98ca3d40cd8aea0fdaaa3abadc", "version_major": 2, "version_minor": 0 }, @@ -3988,7 +1599,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b2d9b9453d0f40928a76a188f7a30eb4", + "model_id": "34c194e15c044a308d3d89e3c99414be", "version_major": 2, "version_minor": 0 }, @@ -4002,7 +1613,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "85b1f60100d8451995792b7da3f8ac83", + "model_id": "7c6efd5cc49744a594a647449d67e6c5", "version_major": 2, "version_minor": 0 }, @@ -4016,7 +1627,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d6f46bc81e674ba38e39f807dae62551", + "model_id": "e9edf72e22d64cf6adaeda87459d0c0b", "version_major": 2, "version_minor": 0 }, @@ -4030,7 +1641,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9133c6ae847f4d52955482803d33c67f", + "model_id": "1fd89ff72f0e42689114fc21b8748658", "version_major": 2, "version_minor": 0 }, @@ -4044,7 +1655,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9ecc392709b442f4b123fcad7fc7e60b", + "model_id": "984b6bbdeaee4207aaa18b88cbaa2691", "version_major": 2, "version_minor": 0 }, @@ -4058,7 +1669,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5df8844d4afa424f8e750c4b362e3667", + "model_id": "6dca7e9346d34396b3d758f7e8eb34f6", "version_major": 2, "version_minor": 0 }, @@ -4072,7 +1683,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d2838a72543d43f4a41520dc98f9dd57", + "model_id": "cd539308a7ef471f927d664745cebb49", "version_major": 2, "version_minor": 0 }, @@ -4087,26 +1698,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=18066, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m(autoscaler +8m20s)\u001b[0m [autoscaler] [1xT4:8CPU-32GB] Attempting to add 1 node to the cluster (increasing from 0 to 1).\n", - "\u001b[36m(autoscaler +8m25s)\u001b[0m [autoscaler] [1xT4:8CPU-32GB|g4dn.2xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n", - "\u001b[36m(autoscaler +8m25s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB] Attempting to add 1 node to the cluster (increasing from 1 to 2).\n", - "\u001b[36m(autoscaler +8m30s)\u001b[0m [autoscaler] [4xT4:48CPU-192GB|g4dn.12xlarge] [us-west-2a] [on-demand] Launched 1 instance.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(MapBatches(TorchPredictor) pid=19185, ip=10.0.4.102)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", - "\u001b[36m(_MapWorker pid=18062, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster]\u001b[0m\n", - "2025-08-22 00:34:50,050\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_96_0 execution finished in 37.23 seconds\n" + "\u001b[36m(MapWorker(MapBatches(EmbedImages)) pid=33395, ip=10.0.5.252)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=34104, ip=10.0.5.252)\u001b[0m /tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", + "\u001b[36m(MapWorker(MapBatches(EmbedImages)) pid=6674, ip=10.0.5.20)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster]\u001b[0m\n", + "2025-08-28 05:10:59,374\tINFO streaming_executor.py:279 -- ✔️ Dataset dataset_40_0 execution finished in 16.98 seconds\n" ] }, { @@ -4114,7 +1709,7 @@ "text/plain": [ "[{'path': 'doggos-dataset/test/basset/basset_10005.jpg',\n", " 'class': 'basset',\n", - " 'label': 2,\n", + " 'label': 30,\n", " 'embedding': array([ 8.86104554e-02, -5.89382686e-02, 1.15464866e-01, 2.15815112e-01,\n", " -3.43266308e-01, -3.35150540e-01, 1.48883224e-01, -1.02369718e-01,\n", " -1.69915810e-01, 4.34856862e-03, 2.41593361e-01, 1.79200619e-01,\n", @@ -4292,21 +1887,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-22 00:34:50,290\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_99_0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-08-22 00:34:50,303\tINFO streaming_executor.py:117 -- Starting execution of Dataset dataset_99_0. Full logs are in /tmp/ray/session_2025-08-21_18-48-13_464408_2298/logs/ray-data\n", - "2025-08-22 00:34:50,304\tINFO streaming_executor.py:118 -- Execution plan of Dataset dataset_99_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> TaskPoolMapOperator[MapBatches(TorchPredictor)] -> TaskPoolMapOperator[MapBatches(batch_metric)] -> AllToAllOperator[Aggregate] -> LimitOperator[limit=1]\n" + "2025-08-28 05:10:59,627\tINFO logging.py:295 -- Registered dataset logger for dataset dataset_43_0\n", + "2025-08-28 05:10:59,639\tINFO streaming_executor.py:159 -- Starting execution of Dataset dataset_43_0. Full logs are in /tmp/ray/session_2025-08-28_04-57-43_348032_12595/logs/ray-data\n", + "2025-08-28 05:10:59,640\tINFO streaming_executor.py:160 -- Execution plan of Dataset dataset_43_0: InputDataBuffer[Input] -> TaskPoolMapOperator[ListFiles] -> TaskPoolMapOperator[ReadFiles] -> TaskPoolMapOperator[Map(add_class)->Map(convert_to_label)] -> ActorPoolMapOperator[MapBatches(EmbedImages)] -> TaskPoolMapOperator[MapBatches(drop_columns)] -> TaskPoolMapOperator[MapBatches(TorchPredictor)] -> TaskPoolMapOperator[MapBatches(batch_metric)] -> AllToAllOperator[Aggregate] -> LimitOperator[limit=1]\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "1bf87bfd70924161a7f4f956a92eb23f", + "model_id": "d6accaaab88244f09ad25c06860ef15f", "version_major": 2, "version_minor": 0 }, @@ -4320,7 +1909,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2114df52a7ac4646aabfda7f7802a648", + "model_id": "b5716638e4bf437399f7192ed356d610", "version_major": 2, "version_minor": 0 }, @@ -4334,7 +1923,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d889fe01be0545939617a037455180df", + "model_id": "d691aeac306d4249ad7cb71172b81f5c", "version_major": 2, "version_minor": 0 }, @@ -4348,7 +1937,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e8b23a2321514f21a50825b660f670bf", + "model_id": "f9175ec54ce64fd18afcc7d2b31b2e4b", "version_major": 2, "version_minor": 0 }, @@ -4362,7 +1951,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "627c443e2450449c8683775fc89d7a8f", + "model_id": "ac75a05d32484b918a46a7c082a6c88a", "version_major": 2, "version_minor": 0 }, @@ -4376,7 +1965,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "2d41291adfbf4c86817b203dc9e6f181", + "model_id": "e0f1ce17c2c54a9ea96762fc0004c543", "version_major": 2, "version_minor": 0 }, @@ -4390,7 +1979,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a2831719e1324270a3662420aff4c1e0", + "model_id": "b3fccdab34974505a9b518ff286f390f", "version_major": 2, "version_minor": 0 }, @@ -4404,7 +1993,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "120a18eec3a64dfda631fa6dbff06232", + "model_id": "3d9e617f49b847a0891fbbdb5185aae6", "version_major": 2, "version_minor": 0 }, @@ -4418,7 +2007,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "18834b310df94e338ad7ad76aaf77ec5", + "model_id": "52a0cd72c7be435894e9ad1981c50301", "version_major": 2, "version_minor": 0 }, @@ -4432,7 +2021,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "898fc65b6d6e491f9bd6dd33572f0d6d", + "model_id": "000b77243d824ddf8a2c199357ce6cf1", "version_major": 2, "version_minor": 0 }, @@ -4446,7 +2035,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7c7712b3872448c49f04dd6ed1af48f6", + "model_id": "51122559786d45c8856839c1818a7158", "version_major": 2, "version_minor": 0 }, @@ -4460,7 +2049,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "970b96aca5db4f1aab1e487105e61cae", + "model_id": "361ff1a618994ef2819407518534ecd7", "version_major": 2, "version_minor": 0 }, @@ -4474,7 +2063,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "aecfe233dbf249b7ab34fc9d26184bc5", + "model_id": "20681d4878dd485e8ca817c2ee333ccc", "version_major": 2, "version_minor": 0 }, @@ -4489,35 +2078,19 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(_MapWorker pid=19193, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "\u001b[36m(_MapWorker pid=25926, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 2x across cluster]\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m(autoscaler +9m10s)\u001b[0m [autoscaler] Cluster upscaled to {120 CPU, 9 GPU}.\n", - "\u001b[36m(autoscaler +9m15s)\u001b[0m [autoscaler] Cluster upscaled to {168 CPU, 13 GPU}.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[36m(MapBatches(TorchPredictor) pid=2582, ip=10.0.31.199)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", - "\u001b[36m(_MapWorker pid=27577, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=2578, ip=10.0.31.199)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=2576, ip=10.0.31.199)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=3977, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 3x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=4229, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 2x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=2579, ip=10.0.31.199)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=2581, ip=10.0.31.199)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=5094, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=5289, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=5548, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "\u001b[36m(MapBatches(TorchPredictor) pid=5816, ip=10.0.60.138)\u001b[0m /tmp/ipykernel_120810/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", - "2025-08-22 00:38:03,968\tINFO streaming_executor.py:231 -- ✔️ Dataset dataset_99_0 execution finished in 193.66 seconds\n" + "\u001b[36m(MapWorker(MapBatches(EmbedImages)) pid=34103, ip=10.0.5.252)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=8149, ip=10.0.5.20)\u001b[0m /tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\n", + "\u001b[36m(MapWorker(MapBatches(EmbedImages)) pid=40389, ip=10.0.5.252)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\u001b[32m [repeated 3x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=8263, ip=10.0.5.20)\u001b[0m /tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=8340, ip=10.0.5.20)\u001b[0m /tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=17879, ip=10.0.5.20)\u001b[0m /tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=18144, ip=10.0.5.20)\u001b[0m /tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=18411, ip=10.0.5.20)\u001b[0m /tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=18682, ip=10.0.5.20)\u001b[0m /tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=18950, ip=10.0.5.20)\u001b[0m /tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=19219, ip=10.0.5.20)\u001b[0m /tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "\u001b[36m(MapBatches(TorchPredictor) pid=19564, ip=10.0.5.20)\u001b[0m /tmp/ipykernel_31027/417303983.py:6: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:203.)\u001b[32m [repeated 4x across cluster]\u001b[0m\n", + "2025-08-28 05:12:20,741\tINFO streaming_executor.py:279 -- ✔️ Dataset dataset_43_0 execution finished in 81.10 seconds\n" ] } ], @@ -4553,101 +2126,6 @@ "F1: 0.84\n", "Accuracy: 0.98\n" ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m(autoscaler +13m0s)\u001b[0m [autoscaler] Downscaling node i-0ffe5abae6e899f5a (node IP: 10.0.60.138) due to node idle termination.\n", - "\u001b[36m(autoscaler +13m5s)\u001b[0m [autoscaler] Cluster resized to {120 CPU, 9 GPU}.\n", - "\u001b[36m(autoscaler +16m0s)\u001b[0m [autoscaler] Downscaling node i-0aa72cef9b8921af5 (node IP: 10.0.31.199) due to node idle termination.\n", - "\u001b[36m(autoscaler +16m5s)\u001b[0m [autoscaler] Cluster resized to {112 CPU, 8 GPU}.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Using CPython 3.12.11 interpreter at: /home/ray/anaconda3/bin/python3.12\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Creating virtual environment at: .venv\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Building doggos @ file:///tmp/ray/session_2025-08-21_18-48-13_464408_2298/runtime_resources/working_dir_files/_ray_pkg_f79228c33bd2a431/doggos\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pillow (6.3MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading grpcio (5.9MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sqlalchemy (3.2MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pydantic-core (1.9MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading jedi (1.5MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading virtualenv (5.7MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pandas (11.4MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading setuptools (1.1MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading uvloop (4.5MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cuda-nvrtc-cu12 (22.6MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sympy (6.0MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading numpy (15.9MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading kiwisolver (1.4MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading tokenizers (3.0MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pyarrow (38.2MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading botocore (13.3MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading fonttools (4.7MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading widgetsnbextension (2.1MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow-skinny (5.6MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading aiohttp (1.6MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading networkx (1.9MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pygments (1.2MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading debugpy (4.0MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading py-spy (2.6MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scikit-learn (12.5MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading hf-xet (3.0MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading matplotlib (8.2MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading torch (783.0MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading transformers (10.0MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scipy (33.5MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading polars (36.7MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow (26.1MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading triton (148.5MiB)\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Built doggos @ file:///tmp/ray/session_2025-08-21_18-48-13_464408_2298/runtime_resources/working_dir_files/_ray_pkg_f79228c33bd2a431/doggos\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pillow\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading grpcio\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sqlalchemy\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pydantic-core\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading jedi\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading virtualenv\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading setuptools\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading uvloop\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cuda-cupti-cu12\u001b[32m [repeated 13x across cluster]\u001b[0m\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading sympy\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading kiwisolver\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading tokenizers\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading fonttools\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading widgetsnbextension\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow-skinny\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading aiohttp\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading networkx\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pygments\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading debugpy\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading py-spy\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading hf-xet\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading matplotlib\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading transformers\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scikit-learn\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading numpy\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading botocore\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pandas\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading polars\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cuda-nvrtc-cu12\u001b[32m [repeated 2x across cluster]\u001b[0m\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading scipy\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading mlflow\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading pyarrow\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-curand-cu12\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cusparselt-cu12\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading triton\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cublas-cu12\u001b[32m [repeated 5x across cluster]\u001b[0m\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading torch\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m warning: Failed to hardlink files; falling back to full copy. This may lead to degraded performance.\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m If the cache and target directories are on different filesystems, hardlinking may not be supported.\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m If this is intentional, set `export UV_LINK_MODE=copy` or use `--link-mode=copy` to suppress this warning.\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Downloading nvidia-cudnn-cu12\n", - "\u001b[33m(raylet, ip=10.0.4.102)\u001b[0m Installed 172 packages in 1.96s\n" - ] } ], "source": [ diff --git a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/03-Online-Serving.ipynb b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/03-Online-Serving.ipynb index 0dfd026ca0d8..d60d62c52c13 100644 --- a/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/03-Online-Serving.ipynb +++ b/doc/source/ray-overview/examples/e2e-multimodal-ai-workloads/notebooks/03-Online-Serving.ipynb @@ -25,9 +25,9 @@ "output_type": "stream", "text": [ "\u001b[92mSuccessfully registered `ipywidgets, matplotlib` and 4 other packages to be installed on all cluster nodes.\u001b[0m\n", - "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_1dp3fa7w5hu3i83ldsi7lqvp9t?workspace-tab=dependencies\u001b[0m\n", + "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_23ry3pgfn3jgq2jk3e5z25udhz?workspace-tab=dependencies\u001b[0m\n", "\u001b[92mSuccessfully registered `doggos` package to be installed on all cluster nodes.\u001b[0m\n", - "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_1dp3fa7w5hu3i83ldsi7lqvp9t?workspace-tab=dependencies\u001b[0m\n" + "\u001b[92mView and update dependencies here: https://console.anyscale.com/cld_kvedZWag2qA8i5BjxUevf5i7/prj_cz951f43jjdybtzkx1s5sjgz99/workspaces/expwrk_23ry3pgfn3jgq2jk3e5z25udhz?workspace-tab=dependencies\u001b[0m\n" ] } ], @@ -278,26 +278,43 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-08-22 00:51:12,070\tINFO worker.py:1747 -- Connecting to existing Ray cluster at address: 10.0.52.10:6379...\n", - "2025-08-22 00:51:12,082\tINFO worker.py:1918 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttps://session-466hy7cqu1gzrp8zk8l4byz7l7.i.anyscaleuserdata.com \u001b[39m\u001b[22m\n", - "2025-08-22 00:51:12,091\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_bb1ea558d334a00804951688d29c76ff051341cc.zip' (1.11MiB) to Ray cluster...\n", - "2025-08-22 00:51:12,096\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_bb1ea558d334a00804951688d29c76ff051341cc.zip'.\n", - "INFO 2025-08-22 00:51:12,153 serve 133557 -- Connecting to existing Serve app in namespace \"serve\". New http options will not be applied.\n", - "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,275 controller 61167 -- Deploying new version of Deployment(name='ClassPredictor', app='default') (initial target replicas: 1).\n", - "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,276 controller 61167 -- Deploying new version of Deployment(name='Doggos', app='default') (initial target replicas: 1).\n", - "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,400 controller 61167 -- Stopping 1 replicas of Deployment(name='ClassPredictor', app='default') with outdated versions.\n", - "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,401 controller 61167 -- Adding 1 replica to Deployment(name='ClassPredictor', app='default').\n", - "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,403 controller 61167 -- Stopping 1 replicas of Deployment(name='Doggos', app='default') with outdated versions.\n", - "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,404 controller 61167 -- Adding 1 replica to Deployment(name='Doggos', app='default').\n", - "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:12,489 controller 61167 -- Draining proxy on node '7cf4feced6fe6a3166528c758e7aea63f8414ff9b95e3f69304a0bbb'.\n", - "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:14,489 controller 61167 -- Replica(id='m8hm2lqw', deployment='ClassPredictor', app='default') is stopped.\n", - "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:14,491 controller 61167 -- Replica(id='r17z6dkp', deployment='Doggos', app='default') is stopped.\n", - "\u001b[36m(ServeReplica:default:Doggos pid=133722)\u001b[0m INFO 2025-08-22 00:51:14,611 default_Doggos 0qpk1kw9 -- Direct ingress is disabled, skipping direct ingress server start\n", - "\u001b[36m(ServeReplica:default:ClassPredictor pid=30024, ip=10.0.4.102)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", - "\u001b[36m(ServeController pid=61167)\u001b[0m INFO 2025-08-22 00:51:19,249 controller 61167 -- No longer draining proxy on node '7cf4feced6fe6a3166528c758e7aea63f8414ff9b95e3f69304a0bbb'.\n", - "\u001b[36m(ServeReplica:default:ClassPredictor pid=30024, ip=10.0.4.102)\u001b[0m INFO 2025-08-22 00:51:21,500 default_ClassPredictor qtsnu3yv -- Direct ingress is disabled, skipping direct ingress server start\n", - "INFO 2025-08-22 00:51:22,301 serve 133557 -- Application 'default' is ready at http://127.0.0.1:8000/.\n", - "INFO 2025-08-22 00:51:22,313 serve 133557 -- Started .\n" + "2025-08-28 05:15:38,455\tINFO worker.py:1771 -- Connecting to existing Ray cluster at address: 10.0.17.148:6379...\n", + "2025-08-28 05:15:38,465\tINFO worker.py:1942 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32mhttps://session-jhxhj69d6ttkjctcxfnsfe7gwk.i.anyscaleuserdata.com \u001b[39m\u001b[22m\n", + "2025-08-28 05:15:38,471\tINFO packaging.py:588 -- Creating a file package for local module '/home/ray/default/doggos/doggos'.\n", + "2025-08-28 05:15:38,475\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_62e649352ce105b6.zip' (0.04MiB) to Ray cluster...\n", + "2025-08-28 05:15:38,476\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_62e649352ce105b6.zip'.\n", + "2025-08-28 05:15:38,478\tINFO packaging.py:380 -- Pushing file package 'gcs://_ray_pkg_c3f5a1927d401ecc93333d17727d37c3401aeed9.zip' (1.08MiB) to Ray cluster...\n", + "2025-08-28 05:15:38,484\tINFO packaging.py:393 -- Successfully pushed file package 'gcs://_ray_pkg_c3f5a1927d401ecc93333d17727d37c3401aeed9.zip'.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(autoscaler +9s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(ProxyActor pid=42150)\u001b[0m INFO 2025-08-28 05:15:42,208 proxy 10.0.17.148 -- Proxy starting on node 524d54fa7a3dfe7fcd55149e6efeaa7a697a4ce87282da72073206b6 (HTTP port: 8000).\n", + "INFO 2025-08-28 05:15:42,290 serve 41929 -- Started Serve in namespace \"serve\".\n", + "\u001b[36m(ProxyActor pid=42150)\u001b[0m INFO 2025-08-28 05:15:42,286 proxy 10.0.17.148 -- Got updated endpoints: {}.\n", + "\u001b[36m(ServeController pid=42086)\u001b[0m INFO 2025-08-28 05:15:47,403 controller 42086 -- Deploying new version of Deployment(name='ClassPredictor', app='default') (initial target replicas: 1).\n", + "\u001b[36m(ServeController pid=42086)\u001b[0m INFO 2025-08-28 05:15:47,404 controller 42086 -- Deploying new version of Deployment(name='Doggos', app='default') (initial target replicas: 1).\n", + "\u001b[36m(ProxyActor pid=42150)\u001b[0m INFO 2025-08-28 05:15:47,423 proxy 10.0.17.148 -- Got updated endpoints: {Deployment(name='Doggos', app='default'): EndpointInfo(route='/', app_is_cross_language=False)}.\n", + "\u001b[36m(ProxyActor pid=42150)\u001b[0m WARNING 2025-08-28 05:15:47,430 proxy 10.0.17.148 -- ANYSCALE_RAY_SERVE_GRPC_RUN_PROXY_ROUTER_SEPARATE_LOOP has been deprecated and will be removed in the ray v2.50.0. Please use RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP instead.\n", + "\u001b[36m(ProxyActor pid=42150)\u001b[0m INFO 2025-08-28 05:15:47,434 proxy 10.0.17.148 -- Started .\n", + "\u001b[36m(ServeController pid=42086)\u001b[0m INFO 2025-08-28 05:15:47,524 controller 42086 -- Adding 1 replica to Deployment(name='ClassPredictor', app='default').\n", + "\u001b[36m(ServeController pid=42086)\u001b[0m INFO 2025-08-28 05:15:47,525 controller 42086 -- Adding 1 replica to Deployment(name='Doggos', app='default').\n", + "\u001b[36m(ServeReplica:default:ClassPredictor pid=20055, ip=10.0.5.20)\u001b[0m Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n", + "\u001b[36m(ProxyActor pid=20172, ip=10.0.5.20)\u001b[0m INFO 2025-08-28 05:15:56,055 proxy 10.0.5.20 -- Proxy starting on node b84e244dca75c40ea981202cae7a1a06df9598ac29ad2b18e1bedb99 (HTTP port: 8000).\n", + "\u001b[36m(ProxyActor pid=20172, ip=10.0.5.20)\u001b[0m INFO 2025-08-28 05:15:56,131 proxy 10.0.5.20 -- Got updated endpoints: {Deployment(name='Doggos', app='default'): EndpointInfo(route='/', app_is_cross_language=False)}.\n", + "\u001b[36m(ProxyActor pid=20172, ip=10.0.5.20)\u001b[0m WARNING 2025-08-28 05:15:56,137 proxy 10.0.5.20 -- ANYSCALE_RAY_SERVE_GRPC_RUN_PROXY_ROUTER_SEPARATE_LOOP has been deprecated and will be removed in the ray v2.50.0. Please use RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP instead.\n", + "\u001b[36m(ProxyActor pid=20172, ip=10.0.5.20)\u001b[0m INFO 2025-08-28 05:15:56,141 proxy 10.0.5.20 -- Started .\n", + "INFO 2025-08-28 05:15:57,505 serve 41929 -- Application 'default' is ready at http://127.0.0.1:8000/.\n", + "INFO 2025-08-28 05:15:57,511 serve 41929 -- Started .\n" ] }, { @@ -325,17 +342,17 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(ServeReplica:default:ClassPredictor pid=30024, ip=10.0.4.102)\u001b[0m /home/ray/anaconda3/lib/python3.12/site-packages/ray/serve/_private/replica.py:1376: UserWarning: Calling sync method 'get_probabilities' directly on the asyncio loop. In a future version, sync methods will be run in a threadpool by default. Ensure your sync methods are thread safe or keep the existing behavior by making them `async def`. Opt into the new behavior by setting RAY_SERVE_RUN_SYNC_IN_THREADPOOL=1.\n", - "\u001b[36m(ServeReplica:default:ClassPredictor pid=30024, ip=10.0.4.102)\u001b[0m warnings.warn(\n", - "\u001b[36m(ServeReplica:default:Doggos pid=133722)\u001b[0m INFO 2025-08-22 00:51:22,490 default_Doggos 0qpk1kw9 daf11861-073e-4758-82b7-0b222066b668 -- Started .\n" + "\u001b[36m(ServeReplica:default:Doggos pid=42244)\u001b[0m INFO 2025-08-28 05:15:57,646 default_Doggos fs1weamq 31c15b70-89a9-4b2d-b4ab-f8424fe6d8d2 -- Started .\n", + "\u001b[36m(ServeReplica:default:ClassPredictor pid=20055, ip=10.0.5.20)\u001b[0m /home/ray/anaconda3/lib/python3.12/site-packages/ray/serve/_private/replica.py:1397: UserWarning: Calling sync method 'get_probabilities' directly on the asyncio loop. In a future version, sync methods will be run in a threadpool by default. Ensure your sync methods are thread safe or keep the existing behavior by making them `async def`. Opt into the new behavior by setting RAY_SERVE_RUN_SYNC_IN_THREADPOOL=1.\n", + "\u001b[36m(ServeReplica:default:ClassPredictor pid=20055, ip=10.0.5.20)\u001b[0m warnings.warn(\n" ] }, { "data": { "text/plain": [ - "[('collie', 0.2292557954788208),\n", - " ('border_collie', 0.1228194534778595),\n", - " ('german_shepherd', 0.07383470982313156)]" + "[('border_collie', 0.1990548074245453),\n", + " ('collie', 0.1363961398601532),\n", + " ('german_shepherd', 0.07545585185289383)]" ] }, "execution_count": null, @@ -346,15 +363,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[36m(ServeReplica:default:ClassPredictor pid=30024, ip=10.0.4.102)\u001b[0m INFO 2025-08-22 00:51:23,011 default_ClassPredictor qtsnu3yv daf11861-073e-4758-82b7-0b222066b668 -- CALL /predict/ OK 504.9ms\n", - "\u001b[36m(ServeReplica:default:Doggos pid=133722)\u001b[0m INFO 2025-08-22 00:51:23,013 default_Doggos 0qpk1kw9 daf11861-073e-4758-82b7-0b222066b668 -- POST /predict/ 200 567.1ms\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m(autoscaler +13m35s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n" + "\u001b[36m(ServeReplica:default:Doggos pid=42244)\u001b[0m INFO 2025-08-28 05:15:58,150 default_Doggos fs1weamq 31c15b70-89a9-4b2d-b4ab-f8424fe6d8d2 -- POST /predict/ 200 516.2ms\n", + "\u001b[36m(ServeReplica:default:ClassPredictor pid=20055, ip=10.0.5.20)\u001b[0m INFO 2025-08-28 05:15:58,148 default_ClassPredictor y7tebd3e 31c15b70-89a9-4b2d-b4ab-f8424fe6d8d2 -- CALL /predict/ OK 491.4ms\n" ] } ], diff --git a/release/ray_release/byod/byod_e2e_multimodal_ai_workloads.sh b/release/ray_release/byod/byod_e2e_multimodal_ai_workloads.sh index bda0ef917f47..05ff13248bd7 100755 --- a/release/ray_release/byod/byod_e2e_multimodal_ai_workloads.sh +++ b/release/ray_release/byod/byod_e2e_multimodal_ai_workloads.sh @@ -5,7 +5,7 @@ set -exo pipefail # Install Python dependencies pip3 install --no-cache-dir \ "matplotlib==3.10.0" \ - "torch==2.7.0" \ + "torch==2.7.1" \ "transformers==4.52.3" \ "scikit-learn==1.6.0" \ "mlflow==2.19.0" \ From 9461ac6b6c2a93e1193ca9641208913d09234674 Mon Sep 17 00:00:00 2001 From: Stephanie Wang Date: Thu, 28 Aug 2025 16:10:10 -0700 Subject: [PATCH 352/634] [core][gpu-objects] Fix #55987 (#56021) Miscellaneous bug fixes introduced by some hidden merge conflicts: - Check whether NIXL is available on actors instead of checking whether a collective group has been created - Add back `tensor_transport_metadata` as an argument when coordinating transfers Closes #55987. --------- Signed-off-by: Stephanie wang Signed-off-by: Stephanie Wang Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python/ray/actor.py | 2 +- .../collective/collective_tensor_transport.py | 17 +++++++++++++ .../collective/nixl_tensor_transport.py | 22 +++++++++++++++++ .../collective/tensor_transport_manager.py | 24 +++++++++++++++++++ python/ray/experimental/collective/util.py | 18 +++++++++----- .../gpu_object_manager/gpu_object_manager.py | 9 ++++--- .../gpu_object_manager/gpu_object_store.py | 2 ++ python/ray/tests/test_gpu_objects_gloo.py | 2 +- 8 files changed, 85 insertions(+), 11 deletions(-) diff --git a/python/ray/actor.py b/python/ray/actor.py index 15b8df351618..64058ee0b728 100644 --- a/python/ray/actor.py +++ b/python/ray/actor.py @@ -831,7 +831,7 @@ def _remote( self._actor, tensor_transport ): raise ValueError( - f"{self._actor} does not have tensor transport {tensor_transport.name} available. Please create a communicator with " + f'{self._actor} does not have tensor transport {tensor_transport.name} available. If using a collective-based transport ("nccl" or "gloo"), please create a communicator with ' "`ray.experimental.collective.create_collective_group` " "before calling actor tasks with non-default tensor_transport." ) diff --git a/python/ray/experimental/collective/collective_tensor_transport.py b/python/ray/experimental/collective/collective_tensor_transport.py index 2edb06d32072..1c0f84e05511 100644 --- a/python/ray/experimental/collective/collective_tensor_transport.py +++ b/python/ray/experimental/collective/collective_tensor_transport.py @@ -1,6 +1,7 @@ from typing import Optional, List, TYPE_CHECKING import ray +from ray.util.collective.types import Backend from ray.experimental.collective.tensor_transport_manager import ( TensorTransportManager, TensorTransportEnum, @@ -16,10 +17,25 @@ class CollectiveTensorTransport(TensorTransportManager): + def __init__(self, tensor_transport_backend: Backend): + self._tensor_transport_backend = tensor_transport_backend + + @property + def tensor_transport_backend(self) -> Backend: + return self._tensor_transport_backend + @staticmethod def is_one_sided() -> bool: return False + def actor_has_tensor_transport(self, actor: "ray.actor.ActorHandle") -> bool: + from ray.experimental.collective import get_collective_groups + + communicators = get_collective_groups( + [actor], backend=self.tensor_transport_backend + ) + return len(communicators) > 0 + @staticmethod def get_tensor_transport_metadata( src_actor: "ray.actor.ActorHandle", @@ -140,6 +156,7 @@ def recv_multiple_tensors( @staticmethod def send_multiple_tensors( tensors: List["torch.Tensor"], + tensor_transport_metadata: CollectiveTransportMetadata, communicator_metadata: CollectiveCommunicatorMetadata, ): import ray.util.collective as collective diff --git a/python/ray/experimental/collective/nixl_tensor_transport.py b/python/ray/experimental/collective/nixl_tensor_transport.py index dcedbc11bf41..b9938ccb8507 100644 --- a/python/ray/experimental/collective/nixl_tensor_transport.py +++ b/python/ray/experimental/collective/nixl_tensor_transport.py @@ -1,10 +1,12 @@ from typing import Optional, List, TYPE_CHECKING import ray +from ray.util.collective.types import Backend from ray.experimental.collective.tensor_transport_manager import ( TensorTransportManager, TensorTransportEnum, ) +from ray.util.collective.collective import get_group_handle from ray.util.collective.types import ( NIXL_GROUP_NAME, NixlTransportMetadata, @@ -16,10 +18,30 @@ class NixlTensorTransport(TensorTransportManager): + @property + def tensor_transport_backend(self) -> Backend: + return Backend.NIXL + @staticmethod def is_one_sided() -> bool: return True + def actor_has_tensor_transport(self, actor: "ray.actor.ActorHandle") -> bool: + def __ray_actor_has_tensor_transport__( + self: "ray.actor.ActorHandle", + ) -> bool: + try: + nixl_backend = get_group_handle(NIXL_GROUP_NAME) + return nixl_backend is not None + except Exception: + return False + + return ray.get( + actor.__ray_call__.options(concurrency_group="_ray_system").remote( + __ray_actor_has_tensor_transport__ + ) + ) + @staticmethod def get_tensor_transport_metadata( src_actor: "ray.actor.ActorHandle", diff --git a/python/ray/experimental/collective/tensor_transport_manager.py b/python/ray/experimental/collective/tensor_transport_manager.py index 18d554d2c6d7..5daf604f640f 100644 --- a/python/ray/experimental/collective/tensor_transport_manager.py +++ b/python/ray/experimental/collective/tensor_transport_manager.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from typing import List, Optional, TYPE_CHECKING from ray.util.collective.types import TensorTransportMetadata, CommunicatorMetadata +from ray.util.collective.types import Backend from ray._private.custom_types import TensorTransportEnum import ray @@ -10,6 +11,15 @@ class TensorTransportManager(ABC): + @property + @abstractmethod + def tensor_transport_backend(self) -> Backend: + """The tensor transport backend, e.g., NCCL. + + Returns: + Backend: The backend of the tensor transport. + """ + @staticmethod @abstractmethod def is_one_sided() -> bool: @@ -19,6 +29,17 @@ def is_one_sided() -> bool: bool: True if the backend is one-sided, False otherwise. """ + @abstractmethod + def actor_has_tensor_transport(self, actor: "ray.actor.ActorHandle") -> bool: + """Whether the actor has the tensor transport available. + + Args: + actor: The actor to check. + + Returns: + bool: True if the actor has the tensor transport available, False otherwise. + """ + @staticmethod @abstractmethod def get_tensor_transport_metadata( @@ -64,6 +85,7 @@ def get_communicator_metadata( def send_object( src_actor: "ray.actor.ActorHandle", obj_id: str, + tensor_transport_meta: TensorTransportMetadata, communicator_metadata_ref: CommunicatorMetadata, ): """ @@ -72,6 +94,7 @@ def send_object( Args: src_actor: The actor that runs this function. obj_id: The ID of the GPU object to send. + tensor_transport_meta: The tensor transport metadata for the GPU object. communicator_metadata_ref: The ObjectRef of communicator metadata for the send/recv operation. """ from ray.experimental.gpu_object_manager.gpu_object_store import __ray_send__ @@ -83,6 +106,7 @@ def send_object( src_actor.__ray_call__.options(concurrency_group="_ray_system").remote( __ray_send__, obj_id, + tensor_transport_meta, communicator_metadata_ref, ) diff --git a/python/ray/experimental/collective/util.py b/python/ray/experimental/collective/util.py index 6e9297dba37c..3539e5780766 100644 --- a/python/ray/experimental/collective/util.py +++ b/python/ray/experimental/collective/util.py @@ -16,7 +16,8 @@ # Singleton instances for tensor transport managers _nixl_tensor_transport_manager = None -_collective_tensor_transport_manager = None +_gloo_tensor_transport_manager = None +_nccl_tensor_transport_manager = None def get_tensor_transport_manager( @@ -35,11 +36,16 @@ def get_tensor_transport_manager( if _nixl_tensor_transport_manager is None: _nixl_tensor_transport_manager = NixlTensorTransport() return _nixl_tensor_transport_manager - elif tensor_transport == Backend.TORCH_GLOO or tensor_transport == Backend.NCCL: - global _collective_tensor_transport_manager - if _collective_tensor_transport_manager is None: - _collective_tensor_transport_manager = CollectiveTensorTransport() - return _collective_tensor_transport_manager + elif tensor_transport == Backend.TORCH_GLOO: + global _gloo_tensor_transport_manager + if _gloo_tensor_transport_manager is None: + _gloo_tensor_transport_manager = CollectiveTensorTransport(tensor_transport) + return _gloo_tensor_transport_manager + elif tensor_transport == Backend.NCCL: + global _nccl_tensor_transport_manager + if _nccl_tensor_transport_manager is None: + _nccl_tensor_transport_manager = CollectiveTensorTransport(tensor_transport) + return _nccl_tensor_transport_manager else: raise ValueError(f"Unsupported tensor transport protocol: {tensor_transport}") diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py index 6c3157b13134..5ce2a345b225 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py @@ -222,6 +222,7 @@ def trigger_out_of_band_tensor_transfer( tensor_transport_manager.send_object( src_actor, obj_id, + tensor_transport_meta, communicator_meta, ) tensor_transport_manager.recv_object( @@ -269,7 +270,7 @@ def actor_has_tensor_transport( """ # Import get_collective_groups here to avoid dependency on # collective libraries for default Ray installation. - from ray.experimental.collective import get_collective_groups + from ray.experimental.collective import get_tensor_transport_manager from ray.experimental.gpu_object_manager.gpu_object_store import ( _tensor_transport_to_collective_backend, ) @@ -277,5 +278,7 @@ def actor_has_tensor_transport( tensor_transport_backend = _tensor_transport_to_collective_backend( tensor_transport ) - communicators = get_collective_groups([actor], backend=tensor_transport_backend) - return len(communicators) > 0 + tensor_transport_manager = get_tensor_transport_manager( + tensor_transport_backend + ) + return tensor_transport_manager.actor_has_tensor_transport(actor) diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_store.py b/python/ray/experimental/gpu_object_manager/gpu_object_store.py index a1e76392df26..7f6309048d8b 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_store.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_store.py @@ -43,6 +43,7 @@ def _tensor_transport_to_collective_backend( def __ray_send__( self, obj_id: str, + tensor_transport_meta: TensorTransportMetadata, communicator_meta: CommunicatorMetadata, ): """Helper function that runs on the src actor to send tensors to the dst actor.""" @@ -64,6 +65,7 @@ def __ray_send__( ) tensor_transport_manager.send_multiple_tensors( tensors, + tensor_transport_meta, communicator_meta, ) diff --git a/python/ray/tests/test_gpu_objects_gloo.py b/python/ray/tests/test_gpu_objects_gloo.py index c15b05e2ade6..6298dbcac4b8 100644 --- a/python/ray/tests/test_gpu_objects_gloo.py +++ b/python/ray/tests/test_gpu_objects_gloo.py @@ -225,7 +225,7 @@ def test_p2p_errors_before_group_creation(ray_start_regular): with pytest.raises( ValueError, - match="Actor.* does not have tensor transport GLOO available. Please create a communicator with `ray.experimental.collective.create_collective_group` before calling actor tasks with non-default tensor_transport.", + match="Actor.* does not have tensor transport GLOO available.*", ): sender.echo.remote(small_tensor) From 3c9b3ed93077a1b78df58482d642adc0ef9b8ca1 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Thu, 28 Aug 2025 16:50:52 -0700 Subject: [PATCH 353/634] [core] use prebuilt redis for osx arm64 (#56036) avoids building from source. Signed-off-by: Lonnie Liu --- BUILD.bazel | 2 ++ WORKSPACE | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/BUILD.bazel b/BUILD.bazel index f9a5539a67a4..8fc6429846d3 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -282,6 +282,7 @@ alias( "@platforms//os:windows": "@com_github_tporadowski_redis_bin//:redis-server.exe", "//bazel:linux_x86_64_config": "@redis_linux_x86_64//:redis-server", "//bazel:linux_arm64_config": "@redis_linux_arm64//:redis-server", + "//bazel:osx_arm64_config": "@redis_osx_arm64//:redis-server", "//conditions:default": "@com_github_antirez_redis//:redis-server", }), ) @@ -292,6 +293,7 @@ alias( "@platforms//os:windows": "@com_github_tporadowski_redis_bin//:redis-cli.exe", "//bazel:linux_x86_64_config": "@redis_linux_x86_64//:redis-cli", "//bazel:linux_arm64_config": "@redis_linux_arm64//:redis-cli", + "//bazel:osx_arm64_config": "@redis_osx_arm64//:redis-cli", "//conditions:default": "@com_github_antirez_redis//:redis-cli", }), ) diff --git a/WORKSPACE b/WORKSPACE index 44b7942314b0..20fd81787491 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -165,6 +165,13 @@ http_archive( urls = ["https://github.com/ray-project/redis/releases/download/7.2.3/redis-linux-arm64.tar.gz"], ) +http_archive( + name = "redis_osx_arm64", + build_file_content = """exports_files(["redis-server", "redis-cli"])""", + sha256 = "74b76099c3600b538252cdd1731278e087e8e85eecc6c64318c860f3e9462506", + urls = ["https://github.com/ray-project/redis/releases/download/7.2.3/redis-osx-arm64.tar.gz"], +) + load("@com_github_storypku_bazel_iwyu//bazel:dependencies.bzl", "bazel_iwyu_dependencies") bazel_iwyu_dependencies() From 6fdf2cfb821e228eb17147b6c100c5a75e2caa79 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Thu, 28 Aug 2025 19:14:14 -0500 Subject: [PATCH 354/634] [core] Split `gcs_autoscaler_state_manager` out from `gcs_server_lib` (#56053) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🫡 --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 38 +++++++- .../gcs_autoscaler_state_manager.cc | 4 - .../gcs_server/gcs_autoscaler_state_manager.h | 16 ++-- src/ray/gcs/gcs_server/gcs_server.cc | 6 +- src/ray/gcs/gcs_server/gcs_task_manager.h | 2 +- .../gcs/gcs_server/grpc_service_interfaces.h | 55 +++++++++-- src/ray/gcs/gcs_server/grpc_services.cc | 43 +++++++-- src/ray/gcs/gcs_server/grpc_services.h | 58 ++++++++--- src/ray/gcs/gcs_server/tests/BUILD.bazel | 7 +- .../gcs_autoscaler_state_manager_test.cc | 5 +- src/ray/rpc/gcs/gcs_rpc_server.h | 95 ------------------- 11 files changed, 184 insertions(+), 145 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 3228dde7e6a7..4c24d3ea9dd0 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -328,6 +328,7 @@ ray_cc_library( visibility = ["//visibility:private"], deps = [ "//src/ray/common:status", + "//src/ray/protobuf:autoscaler_cc_grpc", "//src/ray/protobuf:gcs_service_cc_grpc", ], ) @@ -345,6 +346,7 @@ ray_cc_library( ":grpc_service_interfaces", "//src/ray/common:asio", "//src/ray/common:id", + "//src/ray/protobuf:autoscaler_cc_grpc", "//src/ray/protobuf:gcs_service_cc_grpc", "//src/ray/rpc:grpc_server", "//src/ray/rpc:server_call", @@ -444,19 +446,51 @@ ray_cc_library( ) ray_cc_library( - name = "gcs_server_lib", + name = "gcs_autoscaler_state_manager", srcs = [ "gcs_autoscaler_state_manager.cc", - "gcs_server.cc", ], hdrs = [ "gcs_autoscaler_state_manager.h", + ], + implementation_deps = [ + "//src/ray/common:ray_config", + "//src/ray/gcs:gcs_pb_util", + "//src/ray/util:logging", + "//src/ray/util:string_utils", + "//src/ray/util:time", + ], + deps = [ + ":gcs_actor_manager", + ":gcs_init_data", + ":gcs_kv_manager", + ":gcs_node_manager", + ":gcs_placement_group_manager", + ":gcs_state_util", + ":grpc_service_interfaces", + "//src/ray/common:asio", + "//src/ray/common:id", + "//src/ray/gcs/pubsub:gcs_pub_sub_lib", + "//src/ray/protobuf:gcs_cc_proto", + "//src/ray/util:thread_checker", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_googletest//:gtest", + ], +) + +ray_cc_library( + name = "gcs_server_lib", + srcs = [ + "gcs_server.cc", + ], + hdrs = [ "gcs_server.h", ], deps = [ ":gcs_actor", ":gcs_actor_manager", ":gcs_actor_scheduler", + ":gcs_autoscaler_state_manager", ":gcs_function_manager", ":gcs_health_check_manager", ":gcs_init_data", diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc index 706c3ab6cdba..6fc73a7ff351 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc @@ -19,10 +19,6 @@ #include #include -#include "ray/gcs/gcs_server/gcs_actor_manager.h" -#include "ray/gcs/gcs_server/gcs_node_manager.h" -#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" -#include "ray/gcs/gcs_server/state_util.h" #include "ray/gcs/pb_util.h" #include "ray/util/string_utils.h" #include "ray/util/time.h" diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h index b57cc5c46626..2dae21963fa2 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h @@ -14,16 +14,23 @@ #pragma once +#include + #include #include #include #include +#include "absl/container/flat_hash_map.h" +#include "ray/common/asio/instrumented_io_context.h" +#include "ray/gcs/gcs_server/gcs_actor_manager.h" #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_server/gcs_node_manager.h" +#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" +#include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/state_util.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" #include "ray/rpc/node_manager/raylet_client_pool.h" #include "ray/util/thread_checker.h" #include "src/ray/protobuf/gcs.pb.h" @@ -31,12 +38,7 @@ namespace ray { namespace gcs { -class GcsActorManager; -class GcsNodeManager; -class GcsPlacementGroupManager; -class GcsResourceManager; - -class GcsAutoscalerStateManager : public rpc::autoscaler::AutoscalerStateHandler { +class GcsAutoscalerStateManager : public rpc::autoscaler::AutoscalerStateServiceHandler { public: GcsAutoscalerStateManager(std::string session_name, GcsNodeManager &gcs_node_manager, diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 041412b0b16d..1cf6c2e8b9e8 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -731,7 +731,9 @@ void GcsServer::InitGcsAutoscalerStateManager(const GcsInitData &gcs_init_data) gcs_autoscaler_state_manager_->Initialize(gcs_init_data); rpc_server_.RegisterService( std::make_unique( - io_context_provider_.GetDefaultIOContext(), *gcs_autoscaler_state_manager_)); + io_context_provider_.GetDefaultIOContext(), + *gcs_autoscaler_state_manager_, + RayConfig::instance().gcs_max_active_rpcs_per_handler())); } void GcsServer::InitGcsTaskManager() { @@ -742,7 +744,7 @@ void GcsServer::InitGcsTaskManager() { io_context, *gcs_task_manager_, RayConfig::instance().gcs_max_active_rpcs_per_handler())); - rpc_server_.RegisterService(std::make_unique( + rpc_server_.RegisterService(std::make_unique( io_context, *gcs_task_manager_, RayConfig::instance().gcs_max_active_rpcs_per_handler())); diff --git a/src/ray/gcs/gcs_server/gcs_task_manager.h b/src/ray/gcs/gcs_server/gcs_task_manager.h index bb9c03e7fbc8..ca1edcc34ed0 100644 --- a/src/ray/gcs/gcs_server/gcs_task_manager.h +++ b/src/ray/gcs/gcs_server/gcs_task_manager.h @@ -95,7 +95,7 @@ class FinishedTaskActorTaskGcPolicy : public TaskEventsGcPolicyInterface { /// This class has its own io_context and io_thread, that's separate from other GCS /// services. All handling of all rpc should be posted to the single thread it owns. class GcsTaskManager : public rpc::TaskInfoGcsServiceHandler, - public rpc::RayEventExportGcsServiceHandler { + public rpc::events::RayEventExportGcsServiceHandler { public: /// Create a GcsTaskManager. explicit GcsTaskManager(instrumented_io_context &io_service); diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h index 3f4feb03b784..03dee80ce3fe 100644 --- a/src/ray/gcs/gcs_server/grpc_service_interfaces.h +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -22,6 +22,7 @@ #pragma once #include "ray/common/status.h" +#include "src/ray/protobuf/autoscaler.grpc.pb.h" #include "src/ray/protobuf/gcs_service.grpc.pb.h" namespace ray { @@ -261,14 +262,6 @@ class TaskInfoGcsServiceHandler { SendReplyCallback send_reply_callback) = 0; }; -class RayEventExportGcsServiceHandler { - public: - virtual ~RayEventExportGcsServiceHandler() = default; - virtual void HandleAddEvents(events::AddEventsRequest request, - events::AddEventsReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - class PlacementGroupInfoGcsServiceHandler { public: virtual ~PlacementGroupInfoGcsServiceHandler() = default; @@ -299,5 +292,51 @@ class PlacementGroupInfoGcsServiceHandler { SendReplyCallback send_reply_callback) = 0; }; +namespace autoscaler { + +class AutoscalerStateServiceHandler { + public: + virtual ~AutoscalerStateServiceHandler() = default; + + virtual void HandleGetClusterResourceState(GetClusterResourceStateRequest request, + GetClusterResourceStateReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleReportAutoscalingState(ReportAutoscalingStateRequest request, + ReportAutoscalingStateReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleRequestClusterResourceConstraint( + RequestClusterResourceConstraintRequest request, + RequestClusterResourceConstraintReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetClusterStatus(GetClusterStatusRequest request, + GetClusterStatusReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleDrainNode(DrainNodeRequest request, + DrainNodeReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleReportClusterConfig(ReportClusterConfigRequest request, + ReportClusterConfigReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + +} // namespace autoscaler + +namespace events { + +class RayEventExportGcsServiceHandler { + public: + virtual ~RayEventExportGcsServiceHandler() = default; + virtual void HandleAddEvents(events::AddEventsRequest request, + events::AddEventsReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + +} // namespace events + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc index d1b4bb9d9f51..1fd61662ba66 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -142,16 +142,6 @@ void TaskInfoGrpcService::InitServerCallFactories( RPC_SERVICE_HANDLER(TaskInfoGcsService, GetTaskEvents, max_active_rpcs_per_handler_) } -using events::AddEventsReply; -using events::AddEventsRequest; - -void RayEventExportGrpcService::InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) { - RPC_SERVICE_HANDLER(RayEventExportGcsService, AddEvents, max_active_rpcs_per_handler_) -} - void PlacementGroupInfoGrpcService::InitServerCallFactories( const std::unique_ptr &cq, std::vector> *server_call_factories, @@ -171,5 +161,38 @@ void PlacementGroupInfoGrpcService::InitServerCallFactories( max_active_rpcs_per_handler_) } +namespace autoscaler { + +void AutoscalerStateGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + RPC_SERVICE_HANDLER( + AutoscalerStateService, GetClusterResourceState, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + AutoscalerStateService, ReportAutoscalingState, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + AutoscalerStateService, ReportClusterConfig, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(AutoscalerStateService, + RequestClusterResourceConstraint, + max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER( + AutoscalerStateService, GetClusterStatus, max_active_rpcs_per_handler_) + RPC_SERVICE_HANDLER(AutoscalerStateService, DrainNode, max_active_rpcs_per_handler_) +} + +} // namespace autoscaler + +namespace events { + +void RayEventExportGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + RPC_SERVICE_HANDLER(RayEventExportGcsService, AddEvents, max_active_rpcs_per_handler_) +} + +} // namespace events + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/gcs_server/grpc_services.h index 20040d8396aa..12e745196163 100644 --- a/src/ray/gcs/gcs_server/grpc_services.h +++ b/src/ray/gcs/gcs_server/grpc_services.h @@ -30,6 +30,7 @@ #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/rpc/grpc_server.h" #include "ray/rpc/server_call.h" +#include "src/ray/protobuf/autoscaler.grpc.pb.h" #include "src/ray/protobuf/gcs_service.grpc.pb.h" namespace ray { @@ -242,11 +243,36 @@ class TaskInfoGrpcService : public GrpcService { int64_t max_active_rpcs_per_handler_; }; -class RayEventExportGrpcService : public GrpcService { +class PlacementGroupInfoGrpcService : public GrpcService { public: - explicit RayEventExportGrpcService(instrumented_io_context &io_service, - RayEventExportGcsServiceHandler &handler, - int64_t max_active_rpcs_per_handler) + explicit PlacementGroupInfoGrpcService(instrumented_io_context &io_service, + PlacementGroupInfoGcsServiceHandler &handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(io_service), + service_handler_(handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler) {} + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + PlacementGroupInfoGcsService::AsyncService service_; + PlacementGroupInfoGcsServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + +namespace autoscaler { + +class AutoscalerStateGrpcService : public GrpcService { + public: + explicit AutoscalerStateGrpcService(instrumented_io_context &io_service, + AutoscalerStateServiceHandler &handler, + int64_t max_active_rpcs_per_handler) : GrpcService(io_service), service_handler_(handler), max_active_rpcs_per_handler_(max_active_rpcs_per_handler){}; @@ -260,19 +286,23 @@ class RayEventExportGrpcService : public GrpcService { const ClusterID &cluster_id) override; private: - RayEventExportGcsService::AsyncService service_; - RayEventExportGcsServiceHandler &service_handler_; + AutoscalerStateService::AsyncService service_; + AutoscalerStateServiceHandler &service_handler_; int64_t max_active_rpcs_per_handler_; }; -class PlacementGroupInfoGrpcService : public GrpcService { +} // namespace autoscaler + +namespace events { + +class RayEventExportGrpcService : public GrpcService { public: - explicit PlacementGroupInfoGrpcService(instrumented_io_context &io_service, - PlacementGroupInfoGcsServiceHandler &handler, - int64_t max_active_rpcs_per_handler) + explicit RayEventExportGrpcService(instrumented_io_context &io_service, + RayEventExportGcsServiceHandler &handler, + int64_t max_active_rpcs_per_handler) : GrpcService(io_service), service_handler_(handler), - max_active_rpcs_per_handler_(max_active_rpcs_per_handler) {} + max_active_rpcs_per_handler_(max_active_rpcs_per_handler){}; protected: grpc::Service &GetGrpcService() override { return service_; } @@ -283,10 +313,12 @@ class PlacementGroupInfoGrpcService : public GrpcService { const ClusterID &cluster_id) override; private: - PlacementGroupInfoGcsService::AsyncService service_; - PlacementGroupInfoGcsServiceHandler &service_handler_; + RayEventExportGcsService::AsyncService service_; + RayEventExportGcsServiceHandler &service_handler_; int64_t max_active_rpcs_per_handler_; }; +} // namespace events + } // namespace rpc } // namespace ray diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index 2bb7fc487143..2bf6918e5fa0 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -336,8 +336,13 @@ ray_cc_test( "//:ray_fakes", "//:ray_mock", "//src/fakes/ray/rpc/raylet:fake_raylet_client", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/common:asio", + "//src/ray/gcs/gcs_server:gcs_autoscaler_state_manager", + "//src/ray/gcs/gcs_server:gcs_init_data", + "//src/ray/gcs/gcs_server:gcs_resource_manager", + "//src/ray/gcs/gcs_server:gcs_store_client_kv", "//src/ray/gcs/tests:gcs_test_util_lib", + "//src/ray/raylet/scheduling:cluster_resource_manager", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc index 5cb34e043dfa..68ed5f907e8a 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc @@ -14,6 +14,9 @@ #include "ray/gcs/gcs_server/gcs_autoscaler_state_manager.h" +#include +#include + #include #include #include @@ -23,8 +26,6 @@ #include #include "fakes/ray/rpc/raylet/raylet_client.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" #include "mock/ray/gcs/gcs_server/gcs_actor_manager.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "mock/ray/gcs/gcs_server/gcs_placement_group_manager.h" diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h index 28b0c7e73c9f..0e797263efda 100644 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ b/src/ray/rpc/gcs/gcs_rpc_server.h @@ -25,110 +25,15 @@ #include "src/ray/protobuf/events_event_aggregator_service.pb.h" #include "src/ray/protobuf/gcs_service.grpc.pb.h" -namespace ray { -namespace rpc { // Most of our RPC templates, if not all, expect messages in the ray::rpc protobuf // namespace. Since the following two messages are defined under the rpc::events // namespace, we treat them as if they were part of ray::rpc for compatibility. using ray::rpc::events::AddEventsReply; using ray::rpc::events::AddEventsRequest; -namespace autoscaler { - -#define AUTOSCALER_STATE_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(AutoscalerStateService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - -class AutoscalerStateServiceHandler { - public: - virtual ~AutoscalerStateServiceHandler() = default; - - virtual void HandleGetClusterResourceState(GetClusterResourceStateRequest request, - GetClusterResourceStateReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleReportAutoscalingState(ReportAutoscalingStateRequest request, - ReportAutoscalingStateReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleRequestClusterResourceConstraint( - RequestClusterResourceConstraintRequest request, - RequestClusterResourceConstraintReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGetClusterStatus(GetClusterStatusRequest request, - GetClusterStatusReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleDrainNode(DrainNodeRequest request, - DrainNodeReply *reply, - SendReplyCallback send_reply_callback) = 0; - - virtual void HandleReportClusterConfig(ReportClusterConfigRequest request, - ReportClusterConfigReply *reply, - SendReplyCallback send_reply_callback) = 0; -}; - -/// The `GrpcService` for `AutoscalerStateService`. -class AutoscalerStateGrpcService : public GrpcService { - public: - /// Constructor. - /// - /// \param[in] handler The service handler that actually handle the requests. - explicit AutoscalerStateGrpcService(instrumented_io_context &io_service, - AutoscalerStateServiceHandler &handler) - : GrpcService(io_service), service_handler_(handler){}; - - protected: - grpc::Service &GetGrpcService() override { return service_; } - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - AUTOSCALER_STATE_SERVICE_RPC_HANDLER(GetClusterResourceState); - AUTOSCALER_STATE_SERVICE_RPC_HANDLER(ReportAutoscalingState); - AUTOSCALER_STATE_SERVICE_RPC_HANDLER(ReportClusterConfig); - AUTOSCALER_STATE_SERVICE_RPC_HANDLER(RequestClusterResourceConstraint); - AUTOSCALER_STATE_SERVICE_RPC_HANDLER(GetClusterStatus); - AUTOSCALER_STATE_SERVICE_RPC_HANDLER(DrainNode); - } - - private: - /// The grpc async service object. - AutoscalerStateService::AsyncService service_; - /// The service handler that actually handle the requests. - AutoscalerStateServiceHandler &service_handler_; -}; - -using AutoscalerStateHandler = AutoscalerStateServiceHandler; - -} // namespace autoscaler -} // namespace rpc -} // namespace ray namespace ray { namespace rpc { -#define MONITOR_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(MonitorGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - -#define OBJECT_INFO_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(ObjectInfoGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - -#define MONITOR_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(MonitorGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - -#define OBJECT_INFO_SERVICE_RPC_HANDLER(HANDLER) \ - RPC_SERVICE_HANDLER(ObjectInfoGcsService, \ - HANDLER, \ - RayConfig::instance().gcs_max_active_rpcs_per_handler()) - #define GCS_RPC_SEND_REPLY(send_reply_callback, reply, status) \ reply->mutable_status()->set_code(static_cast(status.code())); \ reply->mutable_status()->set_message(status.message()); \ From f39a860436dca3ed5b9dfae84bd867ac10c84dc6 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Thu, 28 Aug 2025 17:41:37 -0700 Subject: [PATCH 355/634] [core] Kill GcsUnregisterSubscriber + publisher retry tests + doc (#56038) Signed-off-by: dayshah --- src/fakes/ray/pubsub/publisher.h | 14 +- src/mock/ray/pubsub/publisher.h | 4 +- .../core_worker/tests/reference_count_test.cc | 25 ++- .../gcs/gcs_server/grpc_service_interfaces.h | 4 - src/ray/gcs/gcs_server/grpc_services.cc | 2 - src/ray/gcs/gcs_server/pubsub_handler.cc | 9 -- src/ray/gcs/gcs_server/pubsub_handler.h | 4 - src/ray/gcs/pubsub/gcs_pub_sub.cc | 10 +- .../ownership_object_directory.cc | 8 +- src/ray/protobuf/gcs_service.proto | 10 -- src/ray/pubsub/README.md | 39 +++-- src/ray/pubsub/publisher.cc | 115 +++++-------- src/ray/pubsub/publisher.h | 32 ++-- src/ray/pubsub/publisher_interface.h | 12 +- src/ray/pubsub/subscriber.h | 1 - src/ray/pubsub/tests/publisher_test.cc | 151 ++++++++++++------ .../pubsub/tests/pubsub_integration_test.cc | 6 +- src/ray/util/BUILD.bazel | 8 - src/ray/util/sample.h | 50 ------ src/ray/util/tests/BUILD.bazel | 11 -- src/ray/util/tests/sample_test.cc | 83 ---------- 21 files changed, 217 insertions(+), 381 deletions(-) delete mode 100644 src/ray/util/sample.h delete mode 100644 src/ray/util/tests/sample_test.cc diff --git a/src/fakes/ray/pubsub/publisher.h b/src/fakes/ray/pubsub/publisher.h index d6a5c7a6dae0..fe10bc02550c 100644 --- a/src/fakes/ray/pubsub/publisher.h +++ b/src/fakes/ray/pubsub/publisher.h @@ -21,24 +21,20 @@ namespace pubsub { class FakePublisher : public Publisher { public: - bool RegisterSubscription(const rpc::ChannelType channel_type, + void RegisterSubscription(const rpc::ChannelType channel_type, const UniqueID &subscriber_id, - const std::optional &key_id) override { - return true; - } + const std::optional &key_id) override {} void Publish(rpc::PubMessage pub_message) override {} void PublishFailure(const rpc::ChannelType channel_type, const std::string &key_id) override {} - bool UnregisterSubscription(const rpc::ChannelType channel_type, + void UnregisterSubscription(const rpc::ChannelType channel_type, const UniqueID &subscriber_id, - const std::optional &key_id) override { - return true; - } + const std::optional &key_id) override {} - void UnregisterSubscriber(const UniqueID &subscriber_id) override { return; } + void UnregisterSubscriber(const UniqueID &subscriber_id) override {} std::string DebugString() const override { return "FakePublisher"; } }; diff --git a/src/mock/ray/pubsub/publisher.h b/src/mock/ray/pubsub/publisher.h index 1e384427db95..9a1d7c33635f 100644 --- a/src/mock/ray/pubsub/publisher.h +++ b/src/mock/ray/pubsub/publisher.h @@ -29,7 +29,7 @@ class MockPublisher : public PublisherInterface { google::protobuf::RepeatedPtrField *pub_messages, rpc::SendReplyCallback send_reply_callback), (override)); - MOCK_METHOD(bool, + MOCK_METHOD(void, RegisterSubscription, (const rpc::ChannelType channel_type, const UniqueID &subscriber_id, @@ -40,7 +40,7 @@ class MockPublisher : public PublisherInterface { PublishFailure, (const rpc::ChannelType channel_type, const std::string &key_id), (override)); - MOCK_METHOD(bool, + MOCK_METHOD(void, UnregisterSubscription, (const rpc::ChannelType channel_type, const UniqueID &subscriber_id, diff --git a/src/ray/core_worker/tests/reference_count_test.cc b/src/ray/core_worker/tests/reference_count_test.cc index 44f753bbe693..72f7dc6a1364 100644 --- a/src/ray/core_worker/tests/reference_count_test.cc +++ b/src/ray/core_worker/tests/reference_count_test.cc @@ -107,7 +107,7 @@ using SubscriptionFailureCallbackMap = // static maps are used to simulate distirubted environment. static SubscriptionCallbackMap subscription_callback_map; static SubscriptionFailureCallbackMap subscription_failure_callback_map; -static pubsub::pub_internal::SubscriptionIndex directory( +static pubsub::SubscriptionIndex directory( rpc::ChannelType::WORKER_OBJECT_LOCATIONS_CHANNEL); static std::string GenerateID(UniqueID publisher_id, UniqueID subscriber_id) { @@ -127,7 +127,7 @@ using PublisherFactoryFn = class MockDistributedSubscriber : public pubsub::SubscriberInterface { public: - MockDistributedSubscriber(pubsub::pub_internal::SubscriptionIndex *dict, + MockDistributedSubscriber(pubsub::SubscriptionIndex *dict, SubscriptionCallbackMap *sub_callback_map, SubscriptionFailureCallbackMap *sub_failure_callback_map, UniqueID subscriber_id, @@ -136,7 +136,7 @@ class MockDistributedSubscriber : public pubsub::SubscriberInterface { subscription_callback_map_(sub_callback_map), subscription_failure_callback_map_(sub_failure_callback_map), subscriber_id_(subscriber_id), - subscriber_(std::make_unique( + subscriber_(std::make_unique( subscriber_id, /*get_time_ms=*/[]() { return 1.0; }, /*subscriber_timeout_ms=*/1000, @@ -207,17 +207,17 @@ class MockDistributedSubscriber : public pubsub::SubscriberInterface { return ""; } - pubsub::pub_internal::SubscriptionIndex *directory_; + pubsub::SubscriptionIndex *directory_; SubscriptionCallbackMap *subscription_callback_map_; SubscriptionFailureCallbackMap *subscription_failure_callback_map_; UniqueID subscriber_id_; - std::unique_ptr subscriber_; + std::unique_ptr subscriber_; PublisherFactoryFn client_factory_; }; class MockDistributedPublisher : public pubsub::PublisherInterface { public: - MockDistributedPublisher(pubsub::pub_internal::SubscriptionIndex *dict, + MockDistributedPublisher(pubsub::SubscriptionIndex *dict, SubscriptionCallbackMap *sub_callback_map, SubscriptionFailureCallbackMap *sub_failure_callback_map, WorkerID publisher_id) @@ -227,11 +227,10 @@ class MockDistributedPublisher : public pubsub::PublisherInterface { publisher_id_(publisher_id) {} ~MockDistributedPublisher() = default; - bool RegisterSubscription(const rpc::ChannelType channel_type, + void RegisterSubscription(const rpc::ChannelType channel_type, const UniqueID &subscriber_id, const std::optional &key_id_binary) override { RAY_CHECK(false) << "No need to implement it for testing."; - return false; } void PublishFailure(const rpc::ChannelType channel_type, @@ -256,13 +255,11 @@ class MockDistributedPublisher : public pubsub::PublisherInterface { } } - bool UnregisterSubscription(const rpc::ChannelType channel_type, + void UnregisterSubscription(const rpc::ChannelType channel_type, const UniqueID &subscriber_id, - const std::optional &key_id_binary) override { - return true; - } + const std::optional &key_id_binary) override {} - void UnregisterSubscriber(const UniqueID &subscriber_id) override { return; } + void UnregisterSubscriber(const UniqueID &subscriber_id) override {} void ConnectToSubscriber( const rpc::PubsubLongPollingRequest &request, @@ -272,7 +269,7 @@ class MockDistributedPublisher : public pubsub::PublisherInterface { std::string DebugString() const override { return ""; } - pubsub::pub_internal::SubscriptionIndex *directory_; + pubsub::SubscriptionIndex *directory_; SubscriptionCallbackMap *subscription_callback_map_; SubscriptionFailureCallbackMap *subscription_failure_callback_map_; WorkerID publisher_id_; diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/gcs_server/grpc_service_interfaces.h index 03dee80ce3fe..e7d8dedf5a61 100644 --- a/src/ray/gcs/gcs_server/grpc_service_interfaces.h +++ b/src/ray/gcs/gcs_server/grpc_service_interfaces.h @@ -143,10 +143,6 @@ class InternalPubSubGcsServiceHandler { virtual void HandleGcsSubscriberCommandBatch(GcsSubscriberCommandBatchRequest request, GcsSubscriberCommandBatchReply *reply, SendReplyCallback send_reply_callback) = 0; - - virtual void HandleGcsUnregisterSubscriber(GcsUnregisterSubscriberRequest request, - GcsUnregisterSubscriberReply *reply, - SendReplyCallback send_reply_callback) = 0; }; class JobInfoGcsServiceHandler { diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/gcs_server/grpc_services.cc index 1fd61662ba66..b868c93f9425 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/gcs_server/grpc_services.cc @@ -79,8 +79,6 @@ void InternalPubSubGrpcService::InitServerCallFactories( InternalPubSubGcsService, GcsSubscriberPoll, max_active_rpcs_per_handler_); RPC_SERVICE_HANDLER( InternalPubSubGcsService, GcsSubscriberCommandBatch, max_active_rpcs_per_handler_); - RPC_SERVICE_HANDLER( - InternalPubSubGcsService, GcsUnregisterSubscriber, max_active_rpcs_per_handler_); } void JobInfoGrpcService::InitServerCallFactories( diff --git a/src/ray/gcs/gcs_server/pubsub_handler.cc b/src/ray/gcs/gcs_server/pubsub_handler.cc index 865acac8f9e7..51026d4dc0f0 100644 --- a/src/ray/gcs/gcs_server/pubsub_handler.cc +++ b/src/ray/gcs/gcs_server/pubsub_handler.cc @@ -93,15 +93,6 @@ void InternalPubSubHandler::HandleGcsSubscriberCommandBatch( send_reply_callback(Status::OK(), nullptr, nullptr); } -void InternalPubSubHandler::HandleGcsUnregisterSubscriber( - rpc::GcsUnregisterSubscriberRequest request, - rpc::GcsUnregisterSubscriberReply *reply, - rpc::SendReplyCallback send_reply_callback) { - const auto subscriber_id = UniqueID::FromBinary(request.subscriber_id()); - gcs_publisher_.GetPublisher().UnregisterSubscriber(subscriber_id); - send_reply_callback(Status::OK(), nullptr, nullptr); -} - void InternalPubSubHandler::AsyncRemoveSubscriberFrom(const std::string &sender_id) { io_service_.post( [this, sender_id]() { diff --git a/src/ray/gcs/gcs_server/pubsub_handler.h b/src/ray/gcs/gcs_server/pubsub_handler.h index 521863db6ed9..04a935fd949a 100644 --- a/src/ray/gcs/gcs_server/pubsub_handler.h +++ b/src/ray/gcs/gcs_server/pubsub_handler.h @@ -44,10 +44,6 @@ class InternalPubSubHandler : public rpc::InternalPubSubGcsServiceHandler { rpc::GcsSubscriberCommandBatchReply *reply, rpc::SendReplyCallback send_reply_callback) final; - void HandleGcsUnregisterSubscriber(rpc::GcsUnregisterSubscriberRequest request, - rpc::GcsUnregisterSubscriberReply *reply, - rpc::SendReplyCallback send_reply_callback) final; - /// This function is only for external callers. Internally, can just erase from /// sender_to_subscribers_ and everything should be on the Publisher's io_service_. void AsyncRemoveSubscriberFrom(const std::string &sender_id); diff --git a/src/ray/gcs/pubsub/gcs_pub_sub.cc b/src/ray/gcs/pubsub/gcs_pub_sub.cc index 976779ec6c94..12fd6a767f09 100644 --- a/src/ray/gcs/pubsub/gcs_pub_sub.cc +++ b/src/ray/gcs/pubsub/gcs_pub_sub.cc @@ -335,10 +335,14 @@ Status PythonGcsSubscriber::Close() { grpc::ClientContext context; - rpc::GcsUnregisterSubscriberRequest request; + rpc::GcsSubscriberCommandBatchRequest request; request.set_subscriber_id(subscriber_id_); - rpc::GcsUnregisterSubscriberReply reply; - grpc::Status status = pubsub_stub_->GcsUnregisterSubscriber(&context, request, &reply); + auto *command = request.add_commands(); + command->set_channel_type(channel_type_); + command->mutable_unsubscribe_message(); + rpc::GcsSubscriberCommandBatchReply reply; + grpc::Status status = + pubsub_stub_->GcsSubscriberCommandBatch(&context, request, &reply); if (!status.ok()) { RAY_LOG(WARNING) << "Error while unregistering the subscriber: " diff --git a/src/ray/object_manager/ownership_object_directory.cc b/src/ray/object_manager/ownership_object_directory.cc index dfa9615b6679..431bd14095a2 100644 --- a/src/ray/object_manager/ownership_object_directory.cc +++ b/src/ray/object_manager/ownership_object_directory.cc @@ -346,7 +346,6 @@ ray::Status OwnershipBasedObjectDirectory::SubscribeObjectLocations( auto failure_callback = [this, owner_address](const std::string &object_id_binary, const Status &status) { const auto obj_id = ObjectID::FromBinary(object_id_binary); - rpc::WorkerObjectLocationsPubMessage location_info; if (!status.ok()) { RAY_LOG(INFO).WithField(obj_id) << "Failed to get the location: " << status.ToString(); @@ -362,9 +361,10 @@ ray::Status OwnershipBasedObjectDirectory::SubscribeObjectLocations( // Location lookup can fail if the owner is reachable but no longer has a // record of this ObjectRef, most likely due to an issue with the // distributed reference counting protocol. - ObjectLocationSubscriptionCallback(location_info, - obj_id, - /*location_lookup_failed*/ true); + ObjectLocationSubscriptionCallback( + /*location_info=*/rpc::WorkerObjectLocationsPubMessage{}, + obj_id, + /*location_lookup_failed*/ true); }; auto sub_message = std::make_unique(); diff --git a/src/ray/protobuf/gcs_service.proto b/src/ray/protobuf/gcs_service.proto index d0c69395ed4c..3e25c48da3c3 100644 --- a/src/ray/protobuf/gcs_service.proto +++ b/src/ray/protobuf/gcs_service.proto @@ -680,12 +680,6 @@ message GcsSubscriberCommandBatchReply { GcsStatus status = 100; } -message GcsUnregisterSubscriberRequest { - bytes subscriber_id = 1; -} - -message GcsUnregisterSubscriberReply {} - /// This supports subscribing updates from GCS with long poll, and registering / /// de-registering subscribers. service InternalPubSubGcsService { @@ -699,10 +693,6 @@ service InternalPubSubGcsService { /// A batch of subscribe / unsubscribe requests sent by the subscriber. rpc GcsSubscriberCommandBatch(GcsSubscriberCommandBatchRequest) returns (GcsSubscriberCommandBatchReply); - /// Unregister a subscriber from GCS, removing all subscriptions as well as the - /// subscriber itself. - rpc GcsUnregisterSubscriber(GcsUnregisterSubscriberRequest) - returns (GcsUnregisterSubscriberReply); } message GetAllResourceUsageRequest {} diff --git a/src/ray/pubsub/README.md b/src/ray/pubsub/README.md index 75d32cead1f3..fd791fd1cddb 100644 --- a/src/ray/pubsub/README.md +++ b/src/ray/pubsub/README.md @@ -135,19 +135,26 @@ Note that this section ignores fault tolerance. ### Fault detection -Fault detection needed to be implemented in the component-agonistic manner, so -it doesn't use Ray's GCS for that. - -Subscriber detects the publisher failures from the long polling request. A -single long polling request is initiated from the subscriber, and it sends them -again and again whenever replied as long as there are subscribing entries. If -the publisher fails, the long polling request is also failed, so that the -subscriber can detect the failures of publishers. All metadata is cleaned up in -this case. - -Publishers always have received long polling request from a subscriber as long -as there are subscribing entries from them. If subscribers are failed, they are -not sending any more long polling requests. Publishers refreshes the long -polling request every 30 seconds to check if the subscriber is still alive. If -the subscriber doesn't initiate a long polling request for more than certain -threshold, the subscriber is considered failed and all metadata is cleaned up. +Both pubsub RPC's will be retried by the client on transient network failures using the +retryable grpc client used by other RPC's throughout. + +TODO(dayshah): Only the GCS client currently retries the requests, the core worker clients will in the future. + +Subscribing and unsubscribing are idempotent so the `PubsubCommandBatchRequest` can be resent. +Since we restrict it to one in-flight request, the commands will be ordered even with retries. + +The subscriber's `PubsubLongPollingRequest` can also be retried since it comes with a +max_processed_sequence_id. The retry will be sent with the same max_processed_sequence_id +and therefore the publisher will send back the all the messages from max_processed_sequence_id +to max_sequence_id in that subscriber's mailbox. Messages will not be removed from a subscriber's +mailbox until the subscriber sends a request with max_processed_sequence_id > sequence id of message. +Sequence id increments on every publish on a publisher, regardless of channel or entity. + +Publishers keep receiving long polling requests from a subscriber as long +as there are subscribing entries from them. If subscribers are "dead", they are +not sending any more long polling requests. Publishers check if there's been active +long polling requests every 30 seconds to check if the subscriber is still alive. If +there's no activity on a LongPollingRequest for subscriber_timeout_ms (300s by default), +we'll flush the request (we'll reply) and wait to see if the subscriber sends another one. +If there hasn't been an active long polling request for over subscriber_timeout_ms, the +subscriber is considered dead and all metadata is cleaned up. diff --git a/src/ray/pubsub/publisher.cc b/src/ray/pubsub/publisher.cc index 2f4cdd05f176..b9494c5b0034 100644 --- a/src/ray/pubsub/publisher.cc +++ b/src/ray/pubsub/publisher.cc @@ -25,8 +25,6 @@ namespace ray { namespace pubsub { -namespace pub_internal { - bool EntityState::Publish(const std::shared_ptr &msg, size_t msg_size) { if (subscribers_.empty()) { return false; @@ -92,12 +90,12 @@ bool EntityState::Publish(const std::shared_ptr &msg, size_t ms return true; } -bool EntityState::AddSubscriber(SubscriberState *subscriber) { - return subscribers_.emplace(subscriber->id(), subscriber).second; +void EntityState::AddSubscriber(SubscriberState *subscriber) { + subscribers_.emplace(subscriber->id(), subscriber); } -bool EntityState::RemoveSubscriber(const UniqueID &subscriber_id) { - return subscribers_.erase(subscriber_id) > 0; +void EntityState::RemoveSubscriber(const UniqueID &subscriber_id) { + subscribers_.erase(subscriber_id); } const absl::flat_hash_map &EntityState::Subscribers() const { @@ -131,22 +129,20 @@ bool SubscriptionIndex::Publish(const std::shared_ptr &pub_mess return publish_to_all || publish_to_entity; } -bool SubscriptionIndex::AddEntry(const std::string &key_id, SubscriberState *subscriber) { +void SubscriptionIndex::AddEntry(const std::string &key_id, SubscriberState *subscriber) { if (key_id.empty()) { - return subscribers_to_all_->AddSubscriber(subscriber); + subscribers_to_all_->AddSubscriber(subscriber); + return; } auto &subscribing_key_ids = subscribers_to_key_id_[subscriber->id()]; - const bool key_added = subscribing_key_ids.emplace(key_id).second; + subscribing_key_ids.emplace(key_id); auto sub_it = entities_.find(key_id); if (sub_it == entities_.end()) { sub_it = entities_.emplace(key_id, CreateEntityState(channel_type_)).first; } - const bool subscriber_added = sub_it->second->AddSubscriber(subscriber); - - RAY_CHECK(key_added == subscriber_added); - return key_added; + sub_it->second->AddSubscriber(subscriber); } std::vector SubscriptionIndex::GetSubscriberIdsByKeyId( @@ -166,15 +162,13 @@ std::vector SubscriptionIndex::GetSubscriberIdsByKeyId( return subscribers; } -bool SubscriptionIndex::EraseSubscriber(const UniqueID &subscriber_id) { +void SubscriptionIndex::EraseSubscriber(const UniqueID &subscriber_id) { // Erase subscriber of all keys. - if (subscribers_to_all_->RemoveSubscriber(subscriber_id)) { - return true; - } + subscribers_to_all_->RemoveSubscriber(subscriber_id); auto subscribing_key_it = subscribers_to_key_id_.find(subscriber_id); if (subscribing_key_it == subscribers_to_key_id_.end()) { - return false; + return; } // Erase subscriber of individual keys. @@ -192,46 +186,41 @@ bool SubscriptionIndex::EraseSubscriber(const UniqueID &subscriber_id) { } } subscribers_to_key_id_.erase(subscribing_key_it); - return true; } -bool SubscriptionIndex::EraseEntry(const std::string &key_id, +void SubscriptionIndex::EraseEntry(const std::string &key_id, const UniqueID &subscriber_id) { // Erase the subscriber of all keys. if (key_id.empty()) { - return subscribers_to_all_->RemoveSubscriber(subscriber_id); + subscribers_to_all_->RemoveSubscriber(subscriber_id); } // Erase keys from the subscriber of individual keys. - auto subscribers_to_message_it = subscribers_to_key_id_.find(subscriber_id); - if (subscribers_to_message_it == subscribers_to_key_id_.end()) { - return false; + auto subscribers_to_key_id_it = subscribers_to_key_id_.find(subscriber_id); + if (subscribers_to_key_id_it == subscribers_to_key_id_.end()) { + return; } - auto &objects = subscribers_to_message_it->second; + auto &objects = subscribers_to_key_id_it->second; auto object_it = objects.find(key_id); if (object_it == objects.end()) { - auto it = entities_.find(key_id); - if (it != entities_.end()) { - RAY_CHECK(!it->second->Subscribers().contains(subscriber_id)); - } - return false; + return; } objects.erase(object_it); if (objects.empty()) { - subscribers_to_key_id_.erase(subscribers_to_message_it); + subscribers_to_key_id_.erase(subscribers_to_key_id_it); } // Erase subscribers from keys (reverse index). auto entity_it = entities_.find(key_id); - // If code reaches this line, that means the object id was in the index. - RAY_CHECK(entity_it != entities_.end()); + if (entity_it == entities_.end()) { + return; + } auto &entity = *entity_it->second; // If code reaches this line, that means the subscriber id was in the index. - RAY_CHECK(entity.RemoveSubscriber(subscriber_id)); + entity.RemoveSubscriber(subscriber_id); if (entity.Subscribers().empty()) { entities_.erase(entity_it); } - return true; } bool SubscriptionIndex::HasKeyId(const std::string &key_id) const { @@ -371,8 +360,6 @@ bool SubscriberState::IsActive() const { return get_time_ms_() - last_connection_update_time_ms_ < connection_timeout_ms_; } -} // namespace pub_internal - void Publisher::ConnectToSubscriber( const rpc::PubsubLongPollingRequest &request, std::string *publisher_id, @@ -387,13 +374,12 @@ void Publisher::ConnectToSubscriber( auto it = subscribers_.find(subscriber_id); if (it == subscribers_.end()) { it = subscribers_ - .emplace( - subscriber_id, - std::make_unique(subscriber_id, - get_time_ms_, - subscriber_timeout_ms_, - publish_batch_size_, - publisher_id_)) + .emplace(subscriber_id, + std::make_unique(subscriber_id, + get_time_ms_, + subscriber_timeout_ms_, + publish_batch_size_, + publisher_id_)) .first; } auto &subscriber = it->second; @@ -403,26 +389,25 @@ void Publisher::ConnectToSubscriber( request, publisher_id, pub_messages, std::move(send_reply_callback)); } -bool Publisher::RegisterSubscription(const rpc::ChannelType channel_type, +void Publisher::RegisterSubscription(const rpc::ChannelType channel_type, const UniqueID &subscriber_id, const std::optional &key_id) { absl::MutexLock lock(&mutex_); auto it = subscribers_.find(subscriber_id); if (it == subscribers_.end()) { it = subscribers_ - .emplace( - subscriber_id, - std::make_unique(subscriber_id, - get_time_ms_, - subscriber_timeout_ms_, - publish_batch_size_, - publisher_id_)) + .emplace(subscriber_id, + std::make_unique(subscriber_id, + get_time_ms_, + subscriber_timeout_ms_, + publish_batch_size_, + publisher_id_)) .first; } - pub_internal::SubscriberState *subscriber = it->second.get(); + SubscriberState *subscriber = it->second.get(); auto subscription_index_it = subscription_index_map_.find(channel_type); RAY_CHECK(subscription_index_it != subscription_index_map_.end()); - return subscription_index_it->second.AddEntry(key_id.value_or(""), subscriber); + subscription_index_it->second.AddEntry(key_id.value_or(""), subscriber); } void Publisher::Publish(rpc::PubMessage pub_message) { @@ -430,8 +415,6 @@ void Publisher::Publish(rpc::PubMessage pub_message) { const auto channel_type = pub_message.channel_type(); absl::MutexLock lock(&mutex_); auto &subscription_index = subscription_index_map_.at(channel_type); - // TODO(sang): Currently messages are lost if publish happens - // before there's any subscriber for the object. pub_message.set_sequence_id(++next_sequence_id_); const size_t msg_size = pub_message.ByteSizeLong(); @@ -451,13 +434,14 @@ void Publisher::PublishFailure(const rpc::ChannelType channel_type, Publish(pub_message); } -bool Publisher::UnregisterSubscription(const rpc::ChannelType channel_type, +void Publisher::UnregisterSubscription(const rpc::ChannelType channel_type, const UniqueID &subscriber_id, const std::optional &key_id) { absl::MutexLock lock(&mutex_); auto subscription_index_it = subscription_index_map_.find(channel_type); - RAY_CHECK(subscription_index_it != subscription_index_map_.end()); - return subscription_index_it->second.EraseEntry(key_id.value_or(""), subscriber_id); + if (subscription_index_it != subscription_index_map_.end()) { + subscription_index_it->second.EraseEntry(key_id.value_or(""), subscriber_id); + } } void Publisher::UnregisterSubscriber(const UniqueID &subscriber_id) { @@ -465,19 +449,6 @@ void Publisher::UnregisterSubscriber(const UniqueID &subscriber_id) { UnregisterSubscriberInternal(subscriber_id); } -void Publisher::UnregisterAll() { - absl::MutexLock lock(&mutex_); - // Save the subscriber IDs to be removed, because UnregisterSubscriberInternal() - // erases from subscribers_. - std::vector ids; - for (const auto &[id, subscriber] : subscribers_) { - ids.push_back(id); - } - for (const auto &id : ids) { - UnregisterSubscriberInternal(id); - } -} - void Publisher::UnregisterSubscriberInternal(const UniqueID &subscriber_id) { RAY_LOG(DEBUG) << "Unregistering subscriber " << subscriber_id.Hex(); for (auto &index : subscription_index_map_) { diff --git a/src/ray/pubsub/publisher.h b/src/ray/pubsub/publisher.h index 81165e1a8c5e..9657fab90309 100644 --- a/src/ray/pubsub/publisher.h +++ b/src/ray/pubsub/publisher.h @@ -37,8 +37,6 @@ namespace ray { namespace pubsub { -namespace pub_internal { - class SubscriberState; /// State for an entity / topic in a pub/sub channel. @@ -54,8 +52,8 @@ class EntityState { bool Publish(const std::shared_ptr &pub_message, size_t msg_size); /// Manages the set of subscribers of this entity. - bool AddSubscriber(SubscriberState *subscriber); - bool RemoveSubscriber(const UniqueID &subscriber_id); + void AddSubscriber(SubscriberState *subscriber); + void RemoveSubscriber(const UniqueID &subscriber_id); /// Gets the current set of subscribers, keyed by subscriber IDs. const absl::flat_hash_map &Subscribers() const; @@ -101,16 +99,15 @@ class SubscriptionIndex { /// Adds a new subscriber and the key it subscribes to. /// When `key_id` is empty, the subscriber subscribes to all keys. - /// NOTE: The method is idempotent. If it adds a duplicated entry, it will be no-op. - bool AddEntry(const std::string &key_id, SubscriberState *subscriber); + void AddEntry(const std::string &key_id, SubscriberState *subscriber); /// Erases the subscriber from this index. /// Returns whether the subscriber exists before the call. - bool EraseSubscriber(const UniqueID &subscriber_id); + void EraseSubscriber(const UniqueID &subscriber_id); /// Erases the subscriber from the particular key. /// When `key_id` is empty, the subscriber subscribes to all keys. - bool EraseEntry(const std::string &key_id, const UniqueID &subscriber_id); + void EraseEntry(const std::string &key_id, const UniqueID &subscriber_id); /// Test only. /// Returns true if the entity id exists in the index. @@ -231,8 +228,6 @@ class SubscriberState { std::string publisher_id_binary_; }; -} // namespace pub_internal - /// Protocol detail /// /// - Subscriber always send a long polling connection as long as there are subscribed @@ -272,7 +267,7 @@ class Publisher : public PublisherInterface { publisher_id_(publisher_id) { // Insert index map for each channel. for (auto type : channels) { - subscription_index_map_.emplace(type, pub_internal::SubscriptionIndex(type)); + subscription_index_map_.emplace(type, SubscriptionIndex(type)); } periodical_runner_->RunFnPeriodically([this] { CheckDeadSubscribers(); }, @@ -286,7 +281,7 @@ class Publisher : public PublisherInterface { google::protobuf::RepeatedPtrField *pub_messages, rpc::SendReplyCallback send_reply_callback) override; - bool RegisterSubscription(const rpc::ChannelType channel_type, + void RegisterSubscription(const rpc::ChannelType channel_type, const UniqueID &subscriber_id, const std::optional &key_id) override; @@ -295,7 +290,7 @@ class Publisher : public PublisherInterface { void PublishFailure(const rpc::ChannelType channel_type, const std::string &key_id) override; - bool UnregisterSubscription(const rpc::ChannelType channel_type, + void UnregisterSubscription(const rpc::ChannelType channel_type, const UniqueID &subscriber_id, const std::optional &key_id) override; @@ -303,9 +298,6 @@ class Publisher : public PublisherInterface { std::string DebugString() const override; - /// Flushes all inflight pollings and unregisters all subscribers. - void UnregisterAll(); - /// Check all subscribers, detect which subscribers are dead or its connection is timed /// out, and clean up their metadata. This uses the goal-oriented logic to clean up all /// metadata that can happen by subscriber failures. It is how it works; @@ -373,12 +365,12 @@ class Publisher : public PublisherInterface { mutable absl::Mutex mutex_; /// Mapping of node id -> subscribers. - absl::flat_hash_map> - subscribers_ ABSL_GUARDED_BY(mutex_); + absl::flat_hash_map> subscribers_ + ABSL_GUARDED_BY(mutex_); /// Index that stores the mapping of messages <-> subscribers. - absl::flat_hash_map - subscription_index_map_ ABSL_GUARDED_BY(mutex_); + absl::flat_hash_map subscription_index_map_ + ABSL_GUARDED_BY(mutex_); /// The maximum number of objects to publish for each publish calls. const int64_t publish_batch_size_; diff --git a/src/ray/pubsub/publisher_interface.h b/src/ray/pubsub/publisher_interface.h index 2ff6bafe7ecc..35bde4fab94a 100644 --- a/src/ray/pubsub/publisher_interface.h +++ b/src/ray/pubsub/publisher_interface.h @@ -16,9 +16,7 @@ #include -#include #include -#include #include "ray/common/id.h" #include "ray/rpc/server_call.h" @@ -34,10 +32,6 @@ class PublisherInterface { virtual ~PublisherInterface() = default; /// Handle a long poll request from `subscriber_id`. - /// - /// TODO(sang): Currently, we need to pass the callback for connection because we are - /// using long polling internally. This should be changed once the bidirectional grpc - /// streaming is supported. virtual void ConnectToSubscriber( const rpc::PubsubLongPollingRequest &request, std::string *publisher_id, @@ -50,8 +44,7 @@ class PublisherInterface { /// \param subscriber_id The ID of the subscriber. /// \param key_id The key_id that the subscriber is subscribing to. std::nullopt if /// subscribing to all. - /// \return True if registration is new. False otherwise. - virtual bool RegisterSubscription(const rpc::ChannelType channel_type, + virtual void RegisterSubscription(const rpc::ChannelType channel_type, const UniqueID &subscriber_id, const std::optional &key_id) = 0; @@ -75,8 +68,7 @@ class PublisherInterface { /// \param channel_type The type of the channel. /// \param subscriber_id The ID of the subscriber. /// \param key_id The key_id of the subscriber. std::nullopt if subscribing to all. - /// \return True if erased. False otherwise. - virtual bool UnregisterSubscription(const rpc::ChannelType channel_type, + virtual void UnregisterSubscription(const rpc::ChannelType channel_type, const UniqueID &subscriber_id, const std::optional &key_id) = 0; diff --git a/src/ray/pubsub/subscriber.h b/src/ray/pubsub/subscriber.h index 584c5a849452..44920840e6da 100644 --- a/src/ray/pubsub/subscriber.h +++ b/src/ray/pubsub/subscriber.h @@ -264,7 +264,6 @@ class Subscriber : public SubscriberInterface { /// FRIEND_TEST(IntegrationTest, SubscribersToOneIDAndAllIDs); - FRIEND_TEST(IntegrationTest, GcsFailsOver); FRIEND_TEST(SubscriberTest, TestBasicSubscription); FRIEND_TEST(SubscriberTest, TestSingleLongPollingWithMultipleSubscriptions); FRIEND_TEST(SubscriberTest, TestMultiLongPollingWithTheSameSubscription); diff --git a/src/ray/pubsub/tests/publisher_test.cc b/src/ray/pubsub/tests/publisher_test.cc index 4ce919eb94e8..17b728a38c4a 100644 --- a/src/ray/pubsub/tests/publisher_test.cc +++ b/src/ray/pubsub/tests/publisher_test.cc @@ -19,7 +19,6 @@ #include #include -#include "gmock/gmock.h" #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/asio/periodical_runner.h" @@ -28,20 +27,16 @@ namespace ray { namespace pubsub { + namespace { const NodeID kDefaultPublisherId = NodeID::FromRandom(); } -using pub_internal::SubscriberState; -using pub_internal::SubscriptionIndex; - class PublisherTest : public ::testing::Test { public: PublisherTest() : periodical_runner_(PeriodicalRunner::Create(io_service_)) {} - ~PublisherTest() {} - - void SetUp() { + void SetUp() override { publisher_ = std::make_shared( /*channels=*/ std::vector{ @@ -60,26 +55,23 @@ class PublisherTest : public ::testing::Test { request_.set_publisher_id(kDefaultPublisherId.Binary()); } - void TearDown() {} - void ResetSequenceId() { sequence_id_ = 0; } int64_t GetNextSequenceId() { return ++sequence_id_; } - const rpc::PubMessage GeneratePubMessage(const ObjectID &object_id, - int64_t sequence_id = 0) { + rpc::PubMessage GeneratePubMessage(const ObjectID &object_id, int64_t sequence_id = 0) { rpc::PubMessage pub_message; auto *object_eviction_msg = pub_message.mutable_worker_object_eviction_message(); object_eviction_msg->set_object_id(object_id.Binary()); pub_message.set_key_id(object_id.Binary()); pub_message.set_channel_type(rpc::ChannelType::WORKER_OBJECT_EVICTION); - RAY_LOG(INFO) << "message sequence_id is" << sequence_id; + RAY_LOG(INFO) << "message sequence_id is " << sequence_id; pub_message.set_sequence_id(sequence_id); return pub_message; } - const rpc::PubMessage GenerateErrorInfoMessage(const std::string &id, - const std::string &text) { + rpc::PubMessage GenerateErrorInfoMessage(const std::string &id, + const std::string &text) { rpc::PubMessage pub_message; auto *error_msg = pub_message.mutable_error_info_message(); error_msg->set_error_message(text); @@ -232,7 +224,7 @@ TEST_F(PublisherTest, TestSubscriptionIndexErase) { auto current = it++; auto subscriber_id = *current; oid_subscribers.erase(current); - ASSERT_EQ(subscription_index.EraseEntry(oid.Binary(), subscriber_id), 1); + subscription_index.EraseEntry(oid.Binary(), subscriber_id); i++; } const auto &subscribers_from_index = @@ -272,8 +264,8 @@ TEST_F(PublisherTest, TestSubscriptionIndexEraseMultiSubscribers) { subscription_index.AddEntry(oid.Binary(), subscriber_1); subscription_index.AddEntry(oid2.Binary(), subscriber_1); subscription_index.AddEntry(oid.Binary(), subscriber_2); - ASSERT_TRUE(subscription_index.EraseEntry(oid.Binary(), subscriber_id)); - ASSERT_FALSE(subscription_index.EraseEntry(oid.Binary(), subscriber_id)); + subscription_index.EraseEntry(oid.Binary(), subscriber_id); + subscription_index.EraseEntry(oid.Binary(), subscriber_id); } TEST_F(PublisherTest, TestSubscriptionIndexEraseSubscriber) { @@ -1048,24 +1040,19 @@ TEST_F(PublisherTest, TestUnregisterSubscription) { ASSERT_EQ(long_polling_connection_replied, false); // Connection should be replied (removed) when the subscriber is unregistered. - int erased = publisher_->UnregisterSubscription( + publisher_->UnregisterSubscription( rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); - ASSERT_EQ(erased, 1); ASSERT_EQ(long_polling_connection_replied, false); // Make sure when the entries don't exist, it doesn't delete anything. - ASSERT_EQ(publisher_->UnregisterSubscription(rpc::ChannelType::WORKER_OBJECT_EVICTION, - subscriber_id_, - ObjectID::FromRandom().Binary()), - 0); - ASSERT_EQ( - publisher_->UnregisterSubscription( - rpc::ChannelType::WORKER_OBJECT_EVICTION, NodeID::FromRandom(), oid.Binary()), - 0); - ASSERT_EQ(publisher_->UnregisterSubscription(rpc::ChannelType::WORKER_OBJECT_EVICTION, - NodeID::FromRandom(), - ObjectID::FromRandom().Binary()), - 0); + publisher_->UnregisterSubscription(rpc::ChannelType::WORKER_OBJECT_EVICTION, + subscriber_id_, + ObjectID::FromRandom().Binary()); + publisher_->UnregisterSubscription( + rpc::ChannelType::WORKER_OBJECT_EVICTION, NodeID::FromRandom(), oid.Binary()); + publisher_->UnregisterSubscription(rpc::ChannelType::WORKER_OBJECT_EVICTION, + NodeID::FromRandom(), + ObjectID::FromRandom().Binary()); ASSERT_EQ(long_polling_connection_replied, false); // Metadata won't be removed until we unregsiter the subscriber. publisher_->UnregisterSubscriber(subscriber_id_); @@ -1115,25 +1102,93 @@ TEST_F(PublisherTest, TestUnregisterSubscriber) { // Test if registration / unregistration is idempotent. TEST_F(PublisherTest, TestRegistrationIdempotency) { const auto oid = ObjectID::FromRandom(); - ASSERT_TRUE(publisher_->RegisterSubscription( - rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary())); - ASSERT_FALSE(publisher_->RegisterSubscription( - rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary())); - ASSERT_FALSE(publisher_->RegisterSubscription( - rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary())); - ASSERT_FALSE(publisher_->RegisterSubscription( - rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary())); - ASSERT_FALSE(publisher_->CheckNoLeaks()); - ASSERT_TRUE(publisher_->UnregisterSubscription( - rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary())); - ASSERT_FALSE(publisher_->UnregisterSubscription( - rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary())); + + // Double register and assert publish + publisher_->RegisterSubscription( + rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); + publisher_->RegisterSubscription( + rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); + publisher_->ConnectToSubscriber( + request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + [](Status, std::function, std::function) {}); + publisher_->Publish(GeneratePubMessage(oid)); + ASSERT_EQ(reply.publisher_id(), kDefaultPublisherId.Binary()); + ASSERT_EQ(reply.pub_messages().size(), 1); + reply = rpc::PubsubLongPollingReply(); + + // Reconnect, unregister and assert no publish messages + request_.set_max_processed_sequence_id(1); + publisher_->ConnectToSubscriber( + request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + [](Status, std::function, std::function) {}); + publisher_->UnregisterSubscription( + rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); + publisher_->UnregisterSubscription( + rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); + auto pub_message = GeneratePubMessage(oid); + publisher_->Publish(pub_message); + ASSERT_TRUE(reply.pub_messages().empty()); ASSERT_TRUE(publisher_->CheckNoLeaks()); - ASSERT_TRUE(publisher_->RegisterSubscription( - rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary())); + + // Register and connect. Then unregister a couple times and make sure there's no + // publish. + publisher_->RegisterSubscription( + rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); + publisher_->ConnectToSubscriber( + request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + [](Status, std::function, std::function) {}); ASSERT_FALSE(publisher_->CheckNoLeaks()); - ASSERT_TRUE(publisher_->UnregisterSubscription( - rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary())); + publisher_->UnregisterSubscriber(subscriber_id_); + publisher_->UnregisterSubscriber(subscriber_id_); + publisher_->UnregisterSubscription( + rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); + ASSERT_TRUE(publisher_->CheckNoLeaks()); + publisher_->Publish(GeneratePubMessage(oid)); + ASSERT_TRUE(reply.pub_messages().empty()); +} + +TEST_F(PublisherTest, TestSubscriberLostAPublish) { + const auto oid = ObjectID::FromRandom(); + send_reply_callback = [](Status, std::function, std::function) {}; + + // Subscriber registers and connects and publisher publishes. + publisher_->RegisterSubscription( + rpc::ChannelType::WORKER_OBJECT_EVICTION, subscriber_id_, oid.Binary()); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); + publisher_->Publish(GeneratePubMessage(oid)); + ASSERT_EQ(reply.pub_messages().size(), 1); + reply = rpc::PubsubLongPollingReply(); + + // The publisher publishes while there's no active request, then the Subscriber retries + // the LongPollingRequest with the same max_sequence_id since it lost the reply from the + // publisher. The subscriber should get both the 1st and 2nd messages. + publisher_->Publish(GeneratePubMessage(oid)); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); + ASSERT_EQ(reply.pub_messages().size(), 2); + auto max_processed = reply.pub_messages(1).sequence_id(); + reply = rpc::PubsubLongPollingReply(); + + // Subscriber got the reply this time, sends another request with a higher + // max_sequence_id, and then the publisher publishes. + request_.set_max_processed_sequence_id(max_processed); + publisher_->ConnectToSubscriber(request_, + reply.mutable_publisher_id(), + reply.mutable_pub_messages(), + send_reply_callback); + publisher_->Publish(GeneratePubMessage(oid)); + ASSERT_EQ(reply.pub_messages().size(), 1); } TEST_F(PublisherTest, TestPublishFailure) { diff --git a/src/ray/pubsub/tests/pubsub_integration_test.cc b/src/ray/pubsub/tests/pubsub_integration_test.cc index 626242d3befc..4f88012c91d1 100644 --- a/src/ray/pubsub/tests/pubsub_integration_test.cc +++ b/src/ray/pubsub/tests/pubsub_integration_test.cc @@ -304,11 +304,15 @@ TEST_F(IntegrationTest, SubscribersToOneIDAndAllIDs) { int wait_count = 0; while (!(subscriber_1->CheckNoLeaks() && subscriber_2->CheckNoLeaks())) { // Flush all the inflight long polling. - subscriber_service_->GetPublisher().UnregisterAll(); + subscriber_service_->GetPublisher().UnregisterSubscriber( + subscriber_1->subscriber_id_); + subscriber_service_->GetPublisher().UnregisterSubscriber( + subscriber_2->subscriber_id_); ASSERT_LT(wait_count, 60) << "Subscribers still have inflight operations after 60s"; ++wait_count; absl::SleepFor(absl::Seconds(1)); } } + } // namespace pubsub } // namespace ray diff --git a/src/ray/util/BUILD.bazel b/src/ray/util/BUILD.bazel index 749c95438b70..98e0c7a74860 100644 --- a/src/ray/util/BUILD.bazel +++ b/src/ray/util/BUILD.bazel @@ -199,14 +199,6 @@ ray_cc_library( ], ) -ray_cc_library( - name = "sample", - hdrs = ["sample.h"], - deps = [ - "@com_google_absl//absl/time", - ], -) - ray_cc_library( name = "cmd_line_utils", srcs = ["cmd_line_utils.cc"], diff --git a/src/ray/util/sample.h b/src/ray/util/sample.h deleted file mode 100644 index 886a4a1104e2..000000000000 --- a/src/ray/util/sample.h +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include - -#include "absl/time/clock.h" - -// Randomly samples num_elements from the elements between first and last using reservoir -// sampling. -template ::value_type> -void random_sample(Iterator begin, - Iterator end, - size_t num_elements, - std::vector *out) { - out->resize(0); - if (num_elements == 0) { - return; - } - - std::default_random_engine gen(absl::GetCurrentTimeNanos()); - size_t current_index = 0; - for (auto it = begin; it != end; it++) { - if (current_index < num_elements) { - out->push_back(*it); - } else { - size_t random_index = std::uniform_int_distribution(0, current_index)(gen); - if (random_index < num_elements) { - out->at(random_index) = *it; - } - } - current_index++; - } - return; -} diff --git a/src/ray/util/tests/BUILD.bazel b/src/ray/util/tests/BUILD.bazel index 20c5edd2f103..88854484df15 100644 --- a/src/ray/util/tests/BUILD.bazel +++ b/src/ray/util/tests/BUILD.bazel @@ -125,17 +125,6 @@ ray_cc_test( ], ) -ray_cc_test( - name = "sample_test", - size = "small", - srcs = ["sample_test.cc"], - tags = ["team:core"], - deps = [ - "//src/ray/util:sample", - "@com_google_googletest//:gtest_main", - ], -) - ray_cc_test( name = "sequencer_test", size = "small", diff --git a/src/ray/util/tests/sample_test.cc b/src/ray/util/tests/sample_test.cc deleted file mode 100644 index 451cdf8f18b2..000000000000 --- a/src/ray/util/tests/sample_test.cc +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ray/util/sample.h" - -#include - -#include - -namespace ray { - -class RandomSampleTest : public ::testing::Test { - protected: - std::vector *sample; - std::vector *test_vector; - virtual void SetUp() { - sample = new std::vector(); - test_vector = new std::vector(); - for (int i = 0; i < 10; i++) { - test_vector->push_back(i); - } - } - - virtual void TearDown() { - delete sample; - delete test_vector; - } -}; - -TEST_F(RandomSampleTest, TestEmpty) { - random_sample(test_vector->begin(), test_vector->end(), 0, sample); - ASSERT_EQ(sample->size(), 0); -} - -TEST_F(RandomSampleTest, TestSmallerThanSampleSize) { - random_sample( - test_vector->begin(), test_vector->end(), test_vector->size() + 1, sample); - ASSERT_EQ(sample->size(), test_vector->size()); -} - -TEST_F(RandomSampleTest, TestEqualToSampleSize) { - random_sample(test_vector->begin(), test_vector->end(), test_vector->size(), sample); - ASSERT_EQ(sample->size(), test_vector->size()); -} - -TEST_F(RandomSampleTest, TestLargerThanSampleSize) { - random_sample( - test_vector->begin(), test_vector->end(), test_vector->size() - 1, sample); - ASSERT_EQ(sample->size(), test_vector->size() - 1); -} - -TEST_F(RandomSampleTest, TestEqualOccurrenceChance) { - int trials = 1000000; - std::vector occurrences(test_vector->size(), 0); - for (int i = 0; i < trials; i++) { - random_sample( - test_vector->begin(), test_vector->end(), test_vector->size() / 2, sample); - for (int idx : *sample) { - occurrences[idx]++; - } - } - for (int count : occurrences) { - ASSERT_NEAR(trials / 2, count, 0.05 * trials / 2); - } -} - -} // namespace ray - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} From 8a9e9949d5519c180e5d8df703fec565425a74ef Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Thu, 28 Aug 2025 18:43:46 -0700 Subject: [PATCH 356/634] [core] Skip actor name deletion test on ray client (#56063) Signed-off-by: dayshah --- python/ray/tests/test_actor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/ray/tests/test_actor.py b/python/ray/tests/test_actor.py index 99197d8217e2..5c738808ad8b 100644 --- a/python/ray/tests/test_actor.py +++ b/python/ray/tests/test_actor.py @@ -1677,6 +1677,10 @@ def method(self): assert result == "ok" +@pytest.mark.skipif( + client_test_enabled(), + reason="Out of scope actor cleanup doesn't work with Ray client.", +) def test_get_actor_after_same_name_actor_dead(shutdown_only): ACTOR_NAME = "test_actor" NAMESPACE_NAME = "test_namespace" From dcc750edd31538296eac471922ee5e5f0fadef22 Mon Sep 17 00:00:00 2001 From: Matthew Owen Date: Thu, 28 Aug 2025 18:59:41 -0700 Subject: [PATCH 357/634] [Data] Improved loading from column of URIs (#55824) ## Why are these changes needed? There are many use cases for Ray Data where you might have a column of URIs, that you want to transform into a column of bytes corresponding to the bytes in those URIs. Currently Ray Data does, not handle this use case very well because the incoming number of bytes is quite small relative to the size of the bytes loaded when loading the data from the URIs. This introduces a new code path that can be used to estimate the size of the files, allowing for more adequate block sizes to be used leading to increased performance. In followups, we will introduce additional code to allow users to efficiently decode the bytes that have been loaded from the URIs. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Matthew Owen --- doc/source/data/api/expressions.rst | 1 + python/ray/data/BUILD | 14 + .../logical/operators/map_operator.py | 26 ++ .../_internal/planner/plan_download_op.py | 220 ++++++++++++++++ .../data/_internal/planner/plan_udf_map_op.py | 48 +++- python/ray/data/_internal/planner/planner.py | 8 +- python/ray/data/dataset.py | 30 ++- python/ray/data/expressions.py | 44 ++++ .../data/tests/test_download_expression.py | 243 ++++++++++++++++++ 9 files changed, 615 insertions(+), 19 deletions(-) create mode 100644 python/ray/data/_internal/planner/plan_download_op.py create mode 100644 python/ray/data/tests/test_download_expression.py diff --git a/doc/source/data/api/expressions.rst b/doc/source/data/api/expressions.rst index 69c1a50c93d0..a613597fa087 100644 --- a/doc/source/data/api/expressions.rst +++ b/doc/source/data/api/expressions.rst @@ -19,6 +19,7 @@ Public API col lit + download Expression Classes ------------------ diff --git a/python/ray/data/BUILD b/python/ray/data/BUILD index bbe95eb076a1..c9273b2aec72 100644 --- a/python/ray/data/BUILD +++ b/python/ray/data/BUILD @@ -1449,6 +1449,20 @@ py_test( ], ) +py_test( + name = "test_download_expression", + size = "small", + srcs = ["tests/test_download_expression.py"], + tags = [ + "exclusive", + "team:data", + ], + deps = [ + ":conftest", + "//:ray_lib", + ], +) + py_test( name = "test_context", size = "small", diff --git a/python/ray/data/_internal/logical/operators/map_operator.py b/python/ray/data/_internal/logical/operators/map_operator.py index 6b1bcefba5ad..8afee463aef1 100644 --- a/python/ray/data/_internal/logical/operators/map_operator.py +++ b/python/ray/data/_internal/logical/operators/map_operator.py @@ -360,3 +360,29 @@ def target_num_rows_per_block(self) -> int: def can_modify_num_rows(self) -> bool: return False + + +class Download(AbstractMap): + """Logical operator for download operation.""" + + def __init__( + self, + input_op: LogicalOperator, + uri_column_name: str, + output_bytes_column_name: str, + ray_remote_args: Optional[Dict[str, Any]] = None, + ): + super().__init__("Download", input_op, ray_remote_args=ray_remote_args) + self._uri_column_name = uri_column_name + self._output_bytes_column_name = output_bytes_column_name + + def can_modify_num_rows(self) -> bool: + return False + + @property + def uri_column_name(self) -> str: + return self._uri_column_name + + @property + def output_bytes_column_name(self) -> str: + return self._output_bytes_column_name diff --git a/python/ray/data/_internal/planner/plan_download_op.py b/python/ray/data/_internal/planner/plan_download_op.py new file mode 100644 index 000000000000..98617c4593b6 --- /dev/null +++ b/python/ray/data/_internal/planner/plan_download_op.py @@ -0,0 +1,220 @@ +import logging +import math +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import List +from urllib.parse import urlparse + +import pyarrow as pa + +import ray +from ray.data._internal.compute import ActorPoolStrategy, get_compute +from ray.data._internal.execution.interfaces import PhysicalOperator +from ray.data._internal.execution.operators.map_operator import MapOperator +from ray.data._internal.execution.operators.map_transformer import ( + BlockMapTransformFn, + MapTransformer, +) +from ray.data._internal.logical.operators.map_operator import Download +from ray.data._internal.util import RetryingPyFileSystem, make_async_gen +from ray.data.block import BlockAccessor +from ray.data.context import DataContext +from ray.data.datasource.path_util import _resolve_paths_and_filesystem + +logger = logging.getLogger(__name__) + +URI_DOWNLOAD_MAX_WORKERS = 16 + + +def plan_download_op( + op: Download, + physical_children: List[PhysicalOperator], + data_context: DataContext, +) -> MapOperator: + """Plan the download operation with partitioning and downloading stages.""" + assert len(physical_children) == 1 + input_physical_dag = physical_children[0] + compute = get_compute(op._compute) + uri_column_name = op.uri_column_name + output_bytes_column_name = op.output_bytes_column_name + + # Import _get_udf from the main planner file + from ray.data._internal.planner.plan_udf_map_op import ( + _generate_transform_fn_for_map_batches, + _get_udf, + ) + + # PartitionActor is a callable class, so we need ActorPoolStrategy + partition_compute = ActorPoolStrategy(size=1) # Use single actor for partitioning + + fn, init_fn = _get_udf(PartitionActor, (), {}, (uri_column_name, data_context), {}) + block_fn = _generate_transform_fn_for_map_batches(fn) + partition_transform_fns = [ + BlockMapTransformFn(block_fn), + ] + partition_map_transformer = MapTransformer(partition_transform_fns, init_fn) + partition_map_operator = MapOperator.create( + partition_map_transformer, + input_physical_dag, + data_context, + name="URIPartitioner", + compute_strategy=partition_compute, # Use actor-based compute for callable class + ray_remote_args=op._ray_remote_args, + ray_remote_args_fn=op._ray_remote_args_fn, + ) + + fn, init_fn = _get_udf( + download_bytes_threaded, + (uri_column_name, output_bytes_column_name, data_context), + {}, + None, + None, + ) + download_transform_fn = _generate_transform_fn_for_map_batches(fn) + transform_fns = [ + BlockMapTransformFn(download_transform_fn), + ] + download_map_transformer = MapTransformer(transform_fns, init_fn) + download_map_operator = MapOperator.create( + download_map_transformer, + partition_map_operator, + data_context, + name="URIDownloader", + compute_strategy=compute, + ray_remote_args=op._ray_remote_args, + ray_remote_args_fn=op._ray_remote_args_fn, + ) + + return download_map_operator + + +def uri_to_path(uri: str) -> str: + """Convert a URI to a filesystem path.""" + # TODO(mowen): urlparse might be slow. in the future we could use a faster alternative. + parsed = urlparse(uri) + if parsed.scheme == "file": + return parsed.path + return parsed.netloc + parsed.path + + +def download_bytes_threaded( + block, + uri_column_name, + output_bytes_column_name, + data_context: DataContext, +): + """Optimized version that uses make_async_gen for concurrent downloads.""" + if not isinstance(block, pa.Table): + block = BlockAccessor.for_block(block).to_arrow() + + # Extract URIs from PyArrow table + uris = block.column(uri_column_name).to_pylist() + + if len(uris) == 0: + return block + + paths, fs = _resolve_paths_and_filesystem(uris) + fs = RetryingPyFileSystem.wrap(fs, retryable_errors=data_context.retried_io_errors) + + def load_uri_bytes(uri_path_iterator): + """Function that takes an iterator of URI paths and yields downloaded bytes for each.""" + for uri_path in uri_path_iterator: + with fs.open_input_file(uri_path) as f: + yield f.read() + + # Use make_async_gen to download URI bytes concurrently + # This preserves the order of results to match the input URIs + uri_bytes = list( + make_async_gen( + base_iterator=iter(paths), + fn=load_uri_bytes, + preserve_ordering=True, + num_workers=URI_DOWNLOAD_MAX_WORKERS, + ) + ) + + # Add the new column to the PyArrow table + return block.add_column( + len(block.column_names), output_bytes_column_name, pa.array(uri_bytes) + ) + + +class PartitionActor: + """Actor that partitions download operations based on estimated file sizes.""" + + INIT_SAMPLE_BATCH_SIZE = 25 + + def __init__(self, uri_column_name: str, data_context: DataContext): + self._uri_column_name = uri_column_name + self._data_context = data_context + self._batch_size_estimate = None + + def _sample_sizes(self, uris): + """Fetch file sizes in parallel using ThreadPoolExecutor.""" + + def get_file_size(uri_path, fs): + try: + return fs.get_file_info(uri_path).size + except Exception: + return None + + # If no URIs, return empty list + if not uris: + return [] + + # Get the filesystem from the first URI + paths, fs = _resolve_paths_and_filesystem(uris) + fs = RetryingPyFileSystem.wrap( + fs, retryable_errors=self._data_context.retried_io_errors + ) + + # Use ThreadPoolExecutor for concurrent size fetching + file_sizes = [] + with ThreadPoolExecutor(max_workers=URI_DOWNLOAD_MAX_WORKERS) as executor: + # Submit all size fetch tasks + futures = [ + executor.submit(get_file_size, uri_path, fs) for uri_path in paths + ] + + # Collect results as they complete (order doesn't matter) + for future in as_completed(futures): + try: + size = future.result() + if size is not None: + file_sizes.append(size) + except Exception as e: + logger.warning(f"Error fetching file size for download: {e}") + + return file_sizes + + def _arrow_batcher(self, table, n): + """Batch a PyArrow table into smaller tables of size n using zero-copy slicing.""" + num_rows = table.num_rows + for i in range(0, num_rows, n): + end_idx = min(i + n, num_rows) + # Use PyArrow's zero-copy slice operation + batch_table = table.slice(i, end_idx - i) + yield batch_table + + def __call__(self, block): + if not isinstance(block, pa.Table): + block = BlockAccessor.for_block(block).to_arrow() + + if self._batch_size_estimate is None: + # Extract URIs from PyArrow table for sampling + uris = block.column(self._uri_column_name).to_pylist() + sample_uris = uris[: self.INIT_SAMPLE_BATCH_SIZE] + file_sizes = self._sample_sizes(sample_uris) + if not file_sizes or sum(file_sizes) == 0: + # Fallback to incoming block size if no file sizes could be determined + # or if the total size sampled is 0 + logger.warning( + "No file sizes could be determined, using incoming block size" + ) + self._batch_size_estimate = block.num_rows + else: + file_size_estimate = sum(file_sizes) / len(file_sizes) + ctx = ray.data.context.DatasetContext.get_current() + max_bytes = ctx.target_max_block_size + self._batch_size_estimate = math.floor(max_bytes / file_size_estimate) + + yield from self._arrow_batcher(block, self._batch_size_estimate) diff --git a/python/ray/data/_internal/planner/plan_udf_map_op.py b/python/ray/data/_internal/planner/plan_udf_map_op.py index 945d67717f6b..940773dc0b43 100644 --- a/python/ray/data/_internal/planner/plan_udf_map_op.py +++ b/python/ray/data/_internal/planner/plan_udf_map_op.py @@ -5,7 +5,17 @@ import queue from threading import Thread from types import GeneratorType -from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, TypeVar +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Tuple, + TypeVar, +) import numpy as np import pandas as pd @@ -213,7 +223,14 @@ def filter_batch_fn(block: "pa.Table") -> "pa.Table": zero_copy_batch=True, ) else: - filter_fn, init_fn = _get_udf(op) + udf_is_callable_class = isinstance(op._fn, CallableClass) + filter_fn, init_fn = _get_udf( + op._fn, + op._fn_args, + op._fn_kwargs, + op._fn_constructor_args if udf_is_callable_class else None, + op._fn_constructor_kwargs if udf_is_callable_class else None, + ) transform_fn = _generate_transform_fn_for_filter(filter_fn) map_transformer = _create_map_transformer_for_row_based_map_op( transform_fn, init_fn @@ -244,7 +261,14 @@ def plan_udf_map_op( input_physical_dag = physical_children[0] compute = get_compute(op._compute) - fn, init_fn = _get_udf(op) + udf_is_callable_class = isinstance(op._fn, CallableClass) + fn, init_fn = _get_udf( + op._fn, + op._fn_args, + op._fn_kwargs, + op._fn_constructor_args if udf_is_callable_class else None, + op._fn_constructor_kwargs if udf_is_callable_class else None, + ) if isinstance(op, MapBatches): transform_fn = _generate_transform_fn_for_map_batches(fn) @@ -280,17 +304,23 @@ def plan_udf_map_op( ) -def _get_udf(op: AbstractUDFMap): +def _get_udf( + op_fn: Callable, + op_fn_args: Tuple[Any, ...], + op_fn_kwargs: Dict[str, Any], + op_fn_constructor_args: Optional[Tuple[Any, ...]], + op_fn_constructor_kwargs: Optional[Dict[str, Any]], +): # Note, it's important to define these standalone variables. # So the parsed functions won't need to capture the entire operator, which may not # be serializable. - udf = op._fn - fn_args = op._fn_args or () - fn_kwargs = op._fn_kwargs or {} + udf = op_fn + fn_args = op_fn_args or () + fn_kwargs = op_fn_kwargs or {} if isinstance(udf, CallableClass): - fn_constructor_args = op._fn_constructor_args or () - fn_constructor_kwargs = op._fn_constructor_kwargs or {} + fn_constructor_args = op_fn_constructor_args or () + fn_constructor_kwargs = op_fn_constructor_kwargs or {} is_async_udf = _is_async_udf(udf.__call__) diff --git a/python/ray/data/_internal/planner/planner.py b/python/ray/data/_internal/planner/planner.py index 3b42c4ab7f91..371a238c33b5 100644 --- a/python/ray/data/_internal/planner/planner.py +++ b/python/ray/data/_internal/planner/planner.py @@ -26,6 +26,7 @@ from ray.data._internal.logical.operators.join_operator import Join from ray.data._internal.logical.operators.map_operator import ( AbstractUDFMap, + Download, Filter, Project, StreamingRepartition, @@ -36,6 +37,7 @@ from ray.data._internal.logical.operators.streaming_split_operator import StreamingSplit from ray.data._internal.logical.operators.write_operator import Write from ray.data._internal.planner.plan_all_to_all_op import plan_all_to_all_op +from ray.data._internal.planner.plan_download_op import plan_download_op from ray.data._internal.planner.plan_read_op import plan_read_op from ray.data._internal.planner.plan_udf_map_op import ( plan_filter_op, @@ -157,6 +159,7 @@ class Planner: StreamingRepartition: plan_streaming_repartition_op, Join: plan_join_op, StreamingSplit: plan_streaming_split_op, + Download: plan_download_op, } def plan(self, logical_plan: LogicalPlan) -> PhysicalPlan: @@ -214,8 +217,11 @@ def _plan_recursively( break curr_physical_op.set_logical_operators(logical_op) - queue.extend(physical_op.input_dependencies) + # Add this operator to the op_map so optimizer can find it + op_map[curr_physical_op] = logical_op + queue.extend(curr_physical_op.input_dependencies) + # Also add the final operator (in case the loop didn't catch it) op_map[physical_op] = logical_op return physical_op, op_map diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 2254cc7fb90a..4c0aa0d465cc 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -808,17 +808,29 @@ def with_column(self, column_name: str, expr: Expr, **ray_remote_args) -> "Datas Returns: A new dataset with the added column evaluated via the expression. """ - from ray.data._internal.logical.operators.map_operator import Project + from ray.data._internal.logical.operators.map_operator import Download, Project + + # TODO: Once the expression API supports UDFs, we can clean up the code here. + from ray.data.expressions import DownloadExpr plan = self._plan.copy() - project_op = Project( - self._logical_plan.dag, - cols=None, - cols_rename=None, - exprs={column_name: expr}, - ray_remote_args=ray_remote_args, - ) - logical_plan = LogicalPlan(project_op, self.context) + if isinstance(expr, DownloadExpr): + download_op = Download( + self._logical_plan.dag, + uri_column_name=expr.uri_column_name, + output_bytes_column_name=column_name, + ray_remote_args=ray_remote_args, + ) + logical_plan = LogicalPlan(download_op, self.context) + else: + project_op = Project( + self._logical_plan.dag, + cols=None, + cols_rename=None, + exprs={column_name: expr}, + ray_remote_args=ray_remote_args, + ) + logical_plan = LogicalPlan(project_op, self.context) return Dataset(plan, logical_plan) @PublicAPI(api_group=BT_API_GROUP) diff --git a/python/ray/data/expressions.py b/python/ray/data/expressions.py index 3ba8f48356da..597b6c19effe 100644 --- a/python/ray/data/expressions.py +++ b/python/ray/data/expressions.py @@ -239,6 +239,20 @@ def structurally_equals(self, other: Any) -> bool: ) +@DeveloperAPI(stability="alpha") +@dataclass(frozen=True, eq=False) +class DownloadExpr(Expr): + """Expression that represents a download operation.""" + + uri_column_name: str + + def structurally_equals(self, other: Any) -> bool: + return ( + isinstance(other, DownloadExpr) + and self.uri_column_name == other.uri_column_name + ) + + @PublicAPI(stability="beta") def col(name: str) -> ColumnExpr: """ @@ -301,6 +315,34 @@ def lit(value: Any) -> LiteralExpr: return LiteralExpr(value) +@DeveloperAPI(stability="alpha") +def download(uri_column_name: str) -> DownloadExpr: + """ + Create a download expression that downloads content from URIs. + + This creates an expression that will download bytes from URIs stored in + a specified column. When evaluated, it will fetch the content from each URI + and return the downloaded bytes. + + Args: + uri_column_name: The name of the column containing URIs to download from + Returns: + A DownloadExpr that will download content from the specified URI column + + Example: + >>> from ray.data.expressions import download + >>> import ray + >>> # Create dataset with URIs + >>> ds = ray.data.from_items([ + ... {"uri": "s3://bucket/file1.jpg", "id": "1"}, + ... {"uri": "s3://bucket/file2.jpg", "id": "2"} + ... ]) + >>> # Add downloaded bytes column + >>> ds_with_bytes = ds.with_column("bytes", download("uri")) + """ + return DownloadExpr(uri_column_name=uri_column_name) + + # ────────────────────────────────────── # Public API for evaluation # ────────────────────────────────────── @@ -314,6 +356,8 @@ def lit(value: Any) -> LiteralExpr: "ColumnExpr", "LiteralExpr", "BinaryExpr", + "DownloadExpr", "col", "lit", + "download", ] diff --git a/python/ray/data/tests/test_download_expression.py b/python/ray/data/tests/test_download_expression.py new file mode 100644 index 000000000000..53aa28318cd4 --- /dev/null +++ b/python/ray/data/tests/test_download_expression.py @@ -0,0 +1,243 @@ +import io + +import pyarrow as pa +import pytest +from PIL import Image + +import ray +from ray.data.expressions import DownloadExpr, col, download + + +class TestDownloadExpressionStructure: + """Test DownloadExpr structural equality and basic properties.""" + + def test_download_expression_creation(self): + """Test that download() creates a DownloadExpr with correct properties.""" + expr = download("uri_column") + + assert isinstance(expr, DownloadExpr) + assert expr.uri_column_name == "uri_column" + + def test_download_expression_structural_equality(self): + """Test structural equality comparison for download expressions.""" + # Same expressions should be equal + expr1 = download("uri") + expr2 = download("uri") + assert expr1.structurally_equals(expr2) + assert expr2.structurally_equals(expr1) + + # Different URI column names should not be equal + expr3 = download("different_uri") + assert not expr1.structurally_equals(expr3) + assert not expr3.structurally_equals(expr1) + + # Compare with non-DownloadExpr + non_download_expr = col("uri") + assert not expr1.structurally_equals(non_download_expr) + assert not non_download_expr.structurally_equals(expr1) + + +class TestDownloadExpressionFunctionality: + """Test actual download functionality with real and mocked data.""" + + def test_download_expression_with_local_files(self, tmp_path): + """Test basic download expression functionality with local files.""" + # Create sample files with different content types + sample_data = [ + b"This is test file 1 content", + b"Different content for file 2", + b"File 3 has some binary data: \x00\x01\x02\x03", + ] + + file_paths = [] + for i, data in enumerate(sample_data): + file_path = tmp_path / f"test_file_{i}.txt" + file_path.write_bytes(data) + file_paths.append(str(file_path)) + + # Create dataset with file URIs and metadata + table = pa.Table.from_arrays( + [ + pa.array([f"local://{path}" for path in file_paths]), + pa.array([f"id_{i}" for i in range(len(file_paths))]), + pa.array([f"metadata_{i}" for i in range(len(file_paths))]), + pa.array(range(len(file_paths))), + ], + names=["file_uri", "file_id", "metadata", "index"], + ) + + ds = ray.data.from_arrow(table) + + # Add download column using expression + ds_with_downloads = ds.with_column("file_bytes", download("file_uri")) + + # Verify results + results = ds_with_downloads.take_all() + assert len(results) == len(sample_data) + + for i, result in enumerate(results): + # Download column should be added correctly + assert "file_bytes" in result + assert result["file_bytes"] == sample_data[i] + + # All original columns should be preserved + assert result["file_id"] == f"id_{i}" + assert result["metadata"] == f"metadata_{i}" + assert result["index"] == i + assert result["file_uri"] == f"local://{file_paths[i]}" + + def test_download_expression_empty_dataset(self): + """Test download expression with empty dataset.""" + # Create empty dataset with correct schema + table = pa.Table.from_arrays( + [ + pa.array([], type=pa.string()), + ], + names=["uri"], + ) + + ds = ray.data.from_arrow(table) + ds_with_downloads = ds.with_column("bytes", download("uri")) + + results = ds_with_downloads.take_all() + assert len(results) == 0 + + def test_download_expression_with_different_file_types(self, tmp_path): + """Test download expression with various file types including actual images.""" + # Create a small 8x8 RGB image + small_image = Image.new("RGB", (8, 8), color=(255, 0, 0)) # Red 8x8 image + image_buffer = io.BytesIO() + small_image.save(image_buffer, format="PNG") + image_bytes = image_buffer.getvalue() + + # Create files with different types of content + test_files = [ + ("text_file.txt", b"Simple text content"), + ("binary_file.dat", b"\x00\x01\x02\x03\x04\x05"), + ("json_file.json", b'{"key": "value", "number": 123}'), + ("small_image.png", image_bytes), # Actual PNG image (primary use case) + ("empty_file.txt", b""), # Empty file edge case + ] + + file_paths = [] + expected_data = [] + for filename, content in test_files: + file_path = tmp_path / filename + file_path.write_bytes(content) + file_paths.append(str(file_path)) + expected_data.append(content) + + # Create dataset + table = pa.Table.from_arrays( + [ + pa.array([f"local://{path}" for path in file_paths]), + pa.array( + [f.split(".")[0] for f, _ in test_files] + ), # filename without extension + ], + names=["file_uri", "file_type"], + ) + + ds = ray.data.from_arrow(table) + ds_with_downloads = ds.with_column("content", download("file_uri")) + + results = ds_with_downloads.take_all() + assert len(results) == len(test_files) + + for i, result in enumerate(results): + assert result["content"] == expected_data[i] + assert result["file_type"] == test_files[i][0].split(".")[0] + + # Special verification for image file - ensure it can be loaded as an image + if test_files[i][0].endswith(".png"): + downloaded_image = Image.open(io.BytesIO(result["content"])) + assert downloaded_image.size == (8, 8) + assert downloaded_image.mode == "RGB" + + +class TestDownloadExpressionErrors: + """Test error conditions and edge cases for download expressions.""" + + def test_download_expression_invalid_uri_column(self): + """Test download expression with non-existent URI column.""" + table = pa.Table.from_arrays( + [ + pa.array(["local://test.txt"]), + ], + names=["existing_column"], + ) + + ds = ray.data.from_arrow(table) + ds_with_downloads = ds.with_column("bytes", download("non_existent_column")) + + # Should raise error when trying to execute + with pytest.raises(Exception): # Could be KeyError or similar + ds_with_downloads.take_all() + + def test_download_expression_with_null_uris(self): + """Test download expression handling of null/empty URIs.""" + table = pa.Table.from_arrays( + [ + pa.array(["local://test.txt", None, ""]), + ], + names=["uri"], + ) + + ds = ray.data.from_arrow(table) + ds_with_downloads = ds.with_column("bytes", download("uri")) + + # Should handle nulls gracefully (exact behavior may vary) + # This test mainly ensures no crash occurs + try: + results = ds_with_downloads.take_all() + # If it succeeds, verify structure is reasonable + assert len(results) == 3 + for result in results: + assert "bytes" in result + except Exception as e: + # If it fails, should be a reasonable error (not a crash) + assert isinstance(e, (ValueError, KeyError, RuntimeError)) + + +class TestDownloadExpressionIntegration: + """Integration tests combining download expressions with other Ray Data operations.""" + + def test_download_expression_with_map_batches(self, tmpdir): + """Test download expression followed by map_batches processing.""" + # Create a test file + test_file = tmpdir.join("test.txt") + test_content = b"Hello, World!" + test_file.write_binary(test_content) + + # Create dataset + table = pa.Table.from_arrays( + [ + pa.array([f"local://{test_file}"]), + ], + names=["uri"], + ) + + ds = ray.data.from_arrow(table) + + # Download then process + ds_with_content = ds.with_column("raw_bytes", download("uri")) + + def decode_bytes(batch): + # Access the specific column containing the bytes data + batch["decoded_text"] = [ + data.decode("utf-8") for data in batch["raw_bytes"] + ] + return batch + + ds_decoded = ds_with_content.map_batches(decode_bytes) + results = ds_decoded.take_all() + + assert len(results) == 1 + assert results[0]["decoded_text"] == "Hello, World!" + assert results[0]["raw_bytes"] == test_content + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", __file__])) From e9c9a8fd0581a5911711b6c6e69ee64a939fdc4c Mon Sep 17 00:00:00 2001 From: goutamvenkat-anyscale Date: Thu, 28 Aug 2025 20:41:17 -0700 Subject: [PATCH 358/634] [Data] UDF Expression Support for with_column (#55788) --- doc/source/data/api/api.rst | 2 +- doc/source/data/api/expressions.rst | 3 +- python/ray/data/_expression_evaluator.py | 27 +- python/ray/data/block.py | 5 + python/ray/data/dataset.py | 37 ++- python/ray/data/expressions.py | 142 +++++++++- python/ray/data/tests/test_map.py | 344 ++++++++++++++++++++++- 7 files changed, 544 insertions(+), 16 deletions(-) diff --git a/doc/source/data/api/api.rst b/doc/source/data/api/api.rst index e0d0e94d9480..2926be209658 100644 --- a/doc/source/data/api/api.rst +++ b/doc/source/data/api/api.rst @@ -16,4 +16,4 @@ Ray Data API data_context.rst preprocessor.rst llm.rst - from_other_data_libs.rst + from_other_data_libs.rst \ No newline at end of file diff --git a/doc/source/data/api/expressions.rst b/doc/source/data/api/expressions.rst index a613597fa087..3e73a314b3bf 100644 --- a/doc/source/data/api/expressions.rst +++ b/doc/source/data/api/expressions.rst @@ -1,7 +1,7 @@ .. _expressions-api: Expressions API -=============== +================ .. currentmodule:: ray.data.expressions @@ -19,6 +19,7 @@ Public API col lit + udf download Expression Classes diff --git a/python/ray/data/_expression_evaluator.py b/python/ray/data/_expression_evaluator.py index 370ef90b83bc..26642055aa2e 100644 --- a/python/ray/data/_expression_evaluator.py +++ b/python/ray/data/_expression_evaluator.py @@ -3,16 +3,19 @@ import operator from typing import Any, Callable, Dict +import numpy as np import pandas as pd import pyarrow as pa import pyarrow.compute as pc +from ray.data.block import DataBatch from ray.data.expressions import ( BinaryExpr, ColumnExpr, Expr, LiteralExpr, Operation, + UDFExpr, ) _PANDAS_EXPR_OPS_MAP = { @@ -44,7 +47,9 @@ } -def _eval_expr_recursive(expr: "Expr", batch, ops: Dict["Operation", Callable]) -> Any: +def _eval_expr_recursive( + expr: "Expr", batch: DataBatch, ops: Dict["Operation", Callable[..., Any]] +) -> Any: """Generic recursive expression evaluator.""" # TODO: Separate unresolved expressions (arbitrary AST with unresolved refs) # and resolved expressions (bound to a schema) for better error handling @@ -58,10 +63,26 @@ def _eval_expr_recursive(expr: "Expr", batch, ops: Dict["Operation", Callable]) _eval_expr_recursive(expr.left, batch, ops), _eval_expr_recursive(expr.right, batch, ops), ) - raise TypeError(f"Unsupported expression node: {type(expr).__name__}") + if isinstance(expr, UDFExpr): + args = [_eval_expr_recursive(arg, batch, ops) for arg in expr.args] + kwargs = { + k: _eval_expr_recursive(v, batch, ops) for k, v in expr.kwargs.items() + } + result = expr.fn(*args, **kwargs) + # Can't perform type validation for unions if python version is < 3.10 + if not isinstance(result, (pd.Series, np.ndarray, pa.Array, pa.ChunkedArray)): + function_name = expr.fn.__name__ + raise TypeError( + f"UDF '{function_name}' returned invalid type {type(result).__name__}. " + f"Expected type (pandas.Series, numpy.ndarray, pyarrow.Array, or pyarrow.ChunkedArray)" + ) -def eval_expr(expr: "Expr", batch) -> Any: + return result + raise TypeError(f"Unsupported expression node: {type(expr).__name__}") + + +def eval_expr(expr: "Expr", batch: DataBatch) -> Any: """Recursively evaluate *expr* against a batch of the appropriate type.""" if isinstance(batch, pd.DataFrame): return _eval_expr_recursive(expr, batch, _PANDAS_EXPR_OPS_MAP) diff --git a/python/ray/data/block.py b/python/ray/data/block.py index d0edaf4a8574..073033d1ef1e 100644 --- a/python/ray/data/block.py +++ b/python/ray/data/block.py @@ -57,6 +57,11 @@ # Represents a single column of the ``Block`` BlockColumn = Union["pyarrow.ChunkedArray", "pyarrow.Array", "pandas.Series"] +# Represents a single column of the ``Batch`` +BatchColumn = Union[ + "pandas.Series", "np.ndarray", "pyarrow.Array", "pyarrow.ChunkedArray" +] + logger = logging.getLogger(__name__) diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 4c0aa0d465cc..079ac1f0fcf7 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -783,31 +783,50 @@ def _map_batches_without_batch_size_validation( return Dataset(plan, logical_plan) @PublicAPI(api_group=EXPRESSION_API_GROUP, stability="alpha") - def with_column(self, column_name: str, expr: Expr, **ray_remote_args) -> "Dataset": + def with_column( + self, + column_name: str, + expr: Expr, + **ray_remote_args, + ) -> "Dataset": """ Add a new column to the dataset via an expression. - Examples: + This method allows you to add a new column to a dataset by applying an + expression. The expression can be composed of existing columns, literals, + and user-defined functions (UDFs). + Examples: >>> import ray >>> from ray.data.expressions import col >>> ds = ray.data.range(100) - >>> ds.with_column("id_2", (col("id") * 2)).schema() - Column Type - ------ ---- - id int64 - id_2 int64 + >>> # Add a new column 'id_2' by multiplying 'id' by 2. + >>> ds.with_column("id_2", col("id") * 2).show(2) + {'id': 0, 'id_2': 0} + {'id': 1, 'id_2': 2} + + >>> # Using a UDF with with_column + >>> from ray.data.expressions import udf + >>> import pyarrow.compute as pc + >>> + >>> @udf() + ... def add_one(column): + ... return pc.add(column, 1) + >>> + >>> ds.with_column("id_plus_one", add_one(col("id"))).show(2) + {'id': 0, 'id_plus_one': 1} + {'id': 1, 'id_plus_one': 2} Args: column_name: The name of the new column. expr: An expression that defines the new column values. **ray_remote_args: Additional resource requirements to request from - Ray (e.g., num_gpus=1 to request GPUs for the map tasks). See - :func:`ray.remote` for details. + Ray for the map tasks (e.g., `num_gpus=1`). Returns: A new dataset with the added column evaluated via the expression. """ + # TODO: update schema based on the expression AST. from ray.data._internal.logical.operators.map_operator import Download, Project # TODO: Once the expression API supports UDFs, we can clean up the code here. diff --git a/python/ray/data/expressions.py b/python/ray/data/expressions.py index 597b6c19effe..fa99ae638eab 100644 --- a/python/ray/data/expressions.py +++ b/python/ray/data/expressions.py @@ -1,10 +1,12 @@ from __future__ import annotations +import functools from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum -from typing import Any +from typing import Any, Callable, Dict, List +from ray.data.block import BatchColumn from ray.util.annotations import DeveloperAPI, PublicAPI @@ -239,6 +241,142 @@ def structurally_equals(self, other: Any) -> bool: ) +@DeveloperAPI(stability="alpha") +@dataclass(frozen=True, eq=False) +class UDFExpr(Expr): + """Expression that represents a user-defined function call. + + This expression type wraps a UDF with schema inference capabilities, + allowing UDFs to be used seamlessly within the expression system. + + UDFs operate on batches of data, where each column argument is passed + as a PyArrow Array containing multiple values from that column across the batch. + + Args: + fn: The user-defined function to call + args: List of argument expressions (positional arguments) + kwargs: Dictionary of keyword argument expressions + function_name: Optional name for the function (for debugging) + + Example: + >>> from ray.data.expressions import col, udf + >>> import pyarrow as pa + >>> import pyarrow.compute as pc + >>> + >>> @udf() + ... def add_one(x: pa.Array) -> pa.Array: + ... return pc.add(x, 1) + >>> + >>> # Use in expressions + >>> expr = add_one(col("value")) + """ + + fn: Callable[..., BatchColumn] + args: List[Expr] + kwargs: Dict[str, Expr] + + def structurally_equals(self, other: Any) -> bool: + return ( + isinstance(other, UDFExpr) + and self.fn == other.fn + and len(self.args) == len(other.args) + and all(a.structurally_equals(b) for a, b in zip(self.args, other.args)) + and self.kwargs.keys() == other.kwargs.keys() + and all( + self.kwargs[k].structurally_equals(other.kwargs[k]) + for k in self.kwargs.keys() + ) + ) + + +def _create_udf_callable(fn: Callable[..., BatchColumn]) -> Callable[..., UDFExpr]: + """Create a callable that generates UDFExpr when called with expressions.""" + + def udf_callable(*args, **kwargs) -> UDFExpr: + # Convert arguments to expressions if they aren't already + expr_args = [] + for arg in args: + if isinstance(arg, Expr): + expr_args.append(arg) + else: + expr_args.append(LiteralExpr(arg)) + + expr_kwargs = {} + for k, v in kwargs.items(): + if isinstance(v, Expr): + expr_kwargs[k] = v + else: + expr_kwargs[k] = LiteralExpr(v) + + return UDFExpr( + fn=fn, + args=expr_args, + kwargs=expr_kwargs, + ) + + # Preserve original function metadata + functools.update_wrapper(udf_callable, fn) + + # Store the original function for access if needed + udf_callable._original_fn = fn + + return udf_callable + + +@PublicAPI(stability="alpha") +def udf() -> Callable[..., UDFExpr]: + """ + Decorator to convert a UDF into an expression-compatible function. + + This decorator allows UDFs to be used seamlessly within the expression system, + enabling schema inference and integration with other expressions. + + IMPORTANT: UDFs operate on batches of data, not individual rows. When your UDF + is called, each column argument will be passed as a PyArrow Array containing + multiple values from that column across the batch. Under the hood, when working + with multiple columns, they get translated to PyArrow arrays (one array per column). + + Returns: + A callable that creates UDFExpr instances when called with expressions + + Example: + >>> from ray.data.expressions import col, udf + >>> import pyarrow as pa + >>> import pyarrow.compute as pc + >>> import ray + >>> + >>> # UDF that operates on a batch of values (PyArrow Array) + >>> @udf() + ... def add_one(x: pa.Array) -> pa.Array: + ... return pc.add(x, 1) # Vectorized operation on the entire Array + >>> + >>> # UDF that combines multiple columns (each as a PyArrow Array) + >>> @udf() + ... def format_name(first: pa.Array, last: pa.Array) -> pa.Array: + ... return pc.binary_join_element_wise(first, last, " ") # Vectorized string concatenation + >>> + >>> # Use in dataset operations + >>> ds = ray.data.from_items([ + ... {"value": 5, "first": "John", "last": "Doe"}, + ... {"value": 10, "first": "Jane", "last": "Smith"} + ... ]) + >>> + >>> # Single column transformation (operates on batches) + >>> ds_incremented = ds.with_column("value_plus_one", add_one(col("value"))) + >>> + >>> # Multi-column transformation (each column becomes a PyArrow Array) + >>> ds_formatted = ds.with_column("full_name", format_name(col("first"), col("last"))) + >>> + >>> # Can also be used in complex expressions + >>> ds_complex = ds.with_column("doubled_plus_one", add_one(col("value")) * 2) + """ + + def decorator(func: Callable[..., BatchColumn]) -> Callable[..., UDFExpr]: + return _create_udf_callable(func) + + return decorator + + @DeveloperAPI(stability="alpha") @dataclass(frozen=True, eq=False) class DownloadExpr(Expr): @@ -356,6 +494,8 @@ def download(uri_column_name: str) -> DownloadExpr: "ColumnExpr", "LiteralExpr", "BinaryExpr", + "UDFExpr", + "udf", "DownloadExpr", "col", "lit", diff --git a/python/ray/data/tests/test_map.py b/python/ray/data/tests/test_map.py index 0bfb35f68f17..6b72f2e6b972 100644 --- a/python/ray/data/tests/test_map.py +++ b/python/ray/data/tests/test_map.py @@ -35,7 +35,7 @@ ) from ray.data.context import DataContext from ray.data.exceptions import UserCodeException -from ray.data.expressions import col, lit +from ray.data.expressions import col, lit, udf from ray.data.tests.conftest import * # noqa from ray.data.tests.test_util import ConcurrencyCounter # noqa from ray.data.tests.util import column_udf, extract_values @@ -2368,6 +2368,348 @@ def test_with_column_multiple_expressions( assert set(ds.schema().names) == {"id", "plus_one", "times_two", "ten_minus_id"} +@pytest.mark.skipif( + get_pyarrow_version() < parse_version("20.0.0"), + reason="with_column requires PyArrow >= 20.0.0", +) +@pytest.mark.parametrize( + "udf_function, column_name, expected_result", + [ + # Single column UDF - add one to each value + pytest.param( + lambda: udf()(lambda x: pc.add(x, 1)), + "add_one", + 1, # 0 + 1 = 1 + id="single_column_add_one", + ), + # Single column UDF - multiply by 2 + pytest.param( + lambda: udf()(lambda x: pc.multiply(x, 2)), + "times_two", + 0, # 0 * 2 = 0 + id="single_column_multiply", + ), + # Single column UDF - square the value + pytest.param( + lambda: udf()(lambda x: pc.multiply(x, x)), + "squared", + 0, # 0 * 0 = 0 + id="single_column_square", + ), + # Single column UDF with string return type + pytest.param( + lambda: udf()(lambda x: pc.cast(x, pa.string())), + "id_str", + "0", # Convert 0 to "0" + id="single_column_to_string", + ), + # Single column UDF with float return type + pytest.param( + lambda: udf()(lambda x: pc.divide(x, 2.0)), + "half", + 0.0, # 0 / 2.0 = 0.0 + id="single_column_divide_float", + ), + ], +) +def test_with_column_udf_single_column( + ray_start_regular_shared, + udf_function, + column_name, + expected_result, + target_max_block_size_infinite_or_default, +): + """Test UDFExpr functionality with single column operations in with_column.""" + ds = ray.data.range(5) + udf_fn = udf_function() + + # Apply the UDF to the "id" column + ds_with_udf = ds.with_column(column_name, udf_fn(col("id"))) + + result = ds_with_udf.take(1)[0] + assert result["id"] == 0 + assert result[column_name] == expected_result + + +@pytest.mark.skipif( + get_pyarrow_version() < parse_version("20.0.0"), + reason="with_column requires PyArrow >= 20.0.0", +) +@pytest.mark.parametrize( + "test_scenario", + [ + # Multi-column UDF - add two columns + pytest.param( + { + "data": [{"a": 1, "b": 2}, {"a": 3, "b": 4}], + "udf": lambda: udf()(lambda x, y: pc.add(x, y)), + "column_name": "sum_ab", + "expected_first": 3, # 1 + 2 = 3 + "expected_second": 7, # 3 + 4 = 7 + }, + id="multi_column_add", + ), + # Multi-column UDF - multiply two columns + pytest.param( + { + "data": [{"x": 2, "y": 3}, {"x": 4, "y": 5}], + "udf": lambda: udf()(lambda x, y: pc.multiply(x, y)), + "column_name": "product_xy", + "expected_first": 6, # 2 * 3 = 6 + "expected_second": 20, # 4 * 5 = 20 + }, + id="multi_column_multiply", + ), + # Multi-column UDF - string concatenation + pytest.param( + { + "data": [ + {"first": "John", "last": "Doe"}, + {"first": "Jane", "last": "Smith"}, + ], + "udf": lambda: udf()( + lambda first, last: pc.binary_join_element_wise(first, last, " ") + ), + "column_name": "full_name", + "expected_first": "John Doe", + "expected_second": "Jane Smith", + }, + id="multi_column_string_concat", + ), + ], +) +def test_with_column_udf_multi_column( + ray_start_regular_shared, + test_scenario, + target_max_block_size_infinite_or_default, +): + """Test UDFExpr functionality with multi-column operations in with_column.""" + data = test_scenario["data"] + udf_fn = test_scenario["udf"]() + column_name = test_scenario["column_name"] + expected_first = test_scenario["expected_first"] + expected_second = test_scenario["expected_second"] + + ds = ray.data.from_items(data) + + # Apply UDF to multiple columns based on the scenario + if "a" in data[0] and "b" in data[0]: + ds_with_udf = ds.with_column(column_name, udf_fn(col("a"), col("b"))) + elif "x" in data[0] and "y" in data[0]: + ds_with_udf = ds.with_column(column_name, udf_fn(col("x"), col("y"))) + else: # first/last name scenario + ds_with_udf = ds.with_column(column_name, udf_fn(col("first"), col("last"))) + + results = ds_with_udf.take(2) + assert results[0][column_name] == expected_first + assert results[1][column_name] == expected_second + + +@pytest.mark.skipif( + get_pyarrow_version() < parse_version("20.0.0"), + reason="with_column requires PyArrow >= 20.0.0", +) +@pytest.mark.parametrize( + "expression_scenario", + [ + # UDF in arithmetic expression + pytest.param( + { + "expression_factory": lambda add_one_udf: add_one_udf(col("id")) * 2, + "expected": 2, # (0 + 1) * 2 = 2 + "column_name": "udf_times_two", + }, + id="udf_in_arithmetic", + ), + # UDF with literal addition + pytest.param( + { + "expression_factory": lambda add_one_udf: add_one_udf(col("id")) + + lit(10), + "expected": 11, # (0 + 1) + 10 = 11 + "column_name": "udf_plus_literal", + }, + id="udf_plus_literal", + ), + # UDF in comparison + pytest.param( + { + "expression_factory": lambda add_one_udf: add_one_udf(col("id")) > 0, + "expected": True, # (0 + 1) > 0 = True + "column_name": "udf_comparison", + }, + id="udf_in_comparison", + ), + # Nested UDF operations (UDF + regular expression) + pytest.param( + { + "expression_factory": lambda add_one_udf: add_one_udf(col("id") + 5), + "expected": 6, # add_one(0 + 5) = add_one(5) = 6 + "column_name": "nested_udf", + }, + id="nested_udf_expression", + ), + ], +) +def test_with_column_udf_in_complex_expressions( + ray_start_regular_shared, + expression_scenario, + target_max_block_size_infinite_or_default, +): + """Test UDFExpr functionality in complex expressions with with_column.""" + ds = ray.data.range(5) + + # Create a simple add_one UDF for use in expressions + @udf() + def add_one(x: pa.Array) -> pa.Array: + return pc.add(x, 1) + + expression = expression_scenario["expression_factory"](add_one) + expected = expression_scenario["expected"] + column_name = expression_scenario["column_name"] + + ds_with_expr = ds.with_column(column_name, expression) + + result = ds_with_expr.take(1)[0] + assert result["id"] == 0 + assert result[column_name] == expected + + +@pytest.mark.skipif( + get_pyarrow_version() < parse_version("20.0.0"), + reason="with_column requires PyArrow >= 20.0.0", +) +def test_with_column_udf_multiple_udfs( + ray_start_regular_shared, target_max_block_size_infinite_or_default +): + """Test applying multiple UDFs in sequence with with_column.""" + ds = ray.data.range(5) + + # Define multiple UDFs + @udf() + def add_one(x: pa.Array) -> pa.Array: + return pc.add(x, 1) + + @udf() + def multiply_by_two(x: pa.Array) -> pa.Array: + return pc.multiply(x, 2) + + @udf() + def divide_by_three(x: pa.Array) -> pa.Array: + return pc.divide(x, 3.0) + + # Apply UDFs in sequence + ds = ds.with_column("plus_one", add_one(col("id"))) + ds = ds.with_column("times_two", multiply_by_two(col("plus_one"))) + ds = ds.with_column("div_three", divide_by_three(col("times_two"))) + + # Convert to pandas and compare with expected result + result_df = ds.to_pandas() + + expected_df = pd.DataFrame( + { + "id": [0, 1, 2, 3, 4], + "plus_one": [1, 2, 3, 4, 5], # id + 1 + "times_two": [2, 4, 6, 8, 10], # (id + 1) * 2 + "div_three": [ + 2.0 / 3.0, + 4.0 / 3.0, + 2.0, + 8.0 / 3.0, + 10.0 / 3.0, + ], # ((id + 1) * 2) / 3 + } + ) + + pd.testing.assert_frame_equal(result_df, expected_df) + + +@pytest.mark.skipif( + get_pyarrow_version() < parse_version("20.0.0"), + reason="with_column requires PyArrow >= 20.0.0", +) +def test_with_column_mixed_udf_and_regular_expressions( + ray_start_regular_shared, target_max_block_size_infinite_or_default +): + """Test mixing UDF expressions and regular expressions in with_column operations.""" + ds = ray.data.range(5) + + # Define a UDF for testing + @udf() + def multiply_by_three(x: pa.Array) -> pa.Array: + return pc.multiply(x, 3) + + # Mix regular expressions and UDF expressions + ds = ds.with_column("plus_ten", col("id") + 10) # Regular expression + ds = ds.with_column("times_three", multiply_by_three(col("id"))) # UDF expression + ds = ds.with_column("minus_five", col("id") - 5) # Regular expression + ds = ds.with_column( + "udf_plus_regular", multiply_by_three(col("id")) + col("plus_ten") + ) # Mixed: UDF + regular + ds = ds.with_column( + "comparison", col("times_three") > col("plus_ten") + ) # Regular expression using UDF result + + # Convert to pandas and compare with expected result + result_df = ds.to_pandas() + + expected_df = pd.DataFrame( + { + "id": [0, 1, 2, 3, 4], + "plus_ten": [10, 11, 12, 13, 14], # id + 10 + "times_three": [0, 3, 6, 9, 12], # id * 3 + "minus_five": [-5, -4, -3, -2, -1], # id - 5 + "udf_plus_regular": [10, 14, 18, 22, 26], # (id * 3) + (id + 10) + "comparison": [False, False, False, False, False], # times_three > plus_ten + } + ) + + pd.testing.assert_frame_equal(result_df, expected_df) + + +@pytest.mark.skipif( + get_pyarrow_version() < parse_version("20.0.0"), + reason="with_column requires PyArrow >= 20.0.0", +) +def test_with_column_udf_invalid_return_type_validation( + ray_start_regular_shared, target_max_block_size_infinite_or_default +): + """Test that UDFs returning invalid types raise TypeError with clear message.""" + ds = ray.data.range(3) + + # Test UDF returning invalid type (dict) + @udf() + def invalid_dict_return(x: pa.Array) -> dict: + return {"invalid": "return_type"} + + # Test UDF returning invalid type (str) + @udf() + def invalid_str_return(x: pa.Array) -> str: + return "invalid_string" + + # Test UDF returning invalid type (int) + @udf() + def invalid_int_return(x: pa.Array) -> int: + return 42 + + # Test each invalid return type + test_cases = [ + (invalid_dict_return, "dict"), + (invalid_str_return, "str"), + (invalid_int_return, "int"), + ] + + for invalid_udf, expected_type_name in test_cases: + with pytest.raises((RayTaskError, UserCodeException)) as exc_info: + ds.with_column("invalid_col", invalid_udf(col("id"))).take(1) + + # The actual TypeError gets wrapped, so we need to check the exception chain + error_message = str(exc_info.value) + assert f"returned invalid type {expected_type_name}" in error_message + assert "Expected type" in error_message + assert "pandas.Series" in error_message and "numpy.ndarray" in error_message + + if __name__ == "__main__": import sys From 1cb5d0fd190e71eecd96b5eca40e64f389d0539c Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Thu, 28 Aug 2025 20:59:39 -0700 Subject: [PATCH 359/634] [core] Create LeaseID (#55469) Signed-off-by: joshlee --- python/ray/tests/test_placement_group_5.py | 4 +- src/fakes/ray/rpc/raylet/raylet_client.h | 26 +- .../ray/gcs/gcs_server/gcs_actor_scheduler.h | 4 +- ...l_task_manager.h => local_lease_manager.h} | 30 +- src/mock/ray/raylet/worker_pool.h | 4 +- src/mock/ray/raylet_client/raylet_client.h | 16 +- src/ray/common/BUILD.bazel | 21 +- src/ray/common/function_descriptor.h | 2 +- src/ray/common/grpc_util.h | 1 + src/ray/common/id.cc | 33 +- src/ray/common/id.h | 96 +- src/ray/common/lease/lease.h | 82 + src/ray/common/lease/lease_spec.cc | 360 ++++ src/ray/common/lease/lease_spec.h | 114 ++ src/ray/common/task/task.cc | 57 - src/ray/common/task/task.h | 85 - src/ray/common/task/task_spec.cc | 14 +- src/ray/common/task/task_spec.h | 7 +- src/ray/common/tests/BUILD.bazel | 1 - src/ray/common/tests/id_test.cc | 28 +- src/ray/core_worker/BUILD.bazel | 2 +- src/ray/core_worker/lease_policy.cc | 14 +- src/ray/core_worker/lease_policy.h | 24 +- src/ray/core_worker/task_manager_interface.h | 2 +- .../core_worker/task_submission/BUILD.bazel | 1 + .../task_submission/normal_task_submitter.cc | 160 +- .../task_submission/normal_task_submitter.h | 29 +- .../tests/normal_task_submitter_test.cc | 50 +- src/ray/core_worker/tests/BUILD.bazel | 1 - .../core_worker/tests/lease_policy_test.cc | 59 +- src/ray/design_docs/id_specification.md | 17 +- src/ray/gcs/gcs_server/BUILD.bazel | 3 +- src/ray/gcs/gcs_server/gcs_actor.cc | 6 + src/ray/gcs/gcs_server/gcs_actor.h | 6 + src/ray/gcs/gcs_server/gcs_actor_manager.cc | 20 +- src/ray/gcs/gcs_server/gcs_actor_manager.h | 5 +- src/ray/gcs/gcs_server/gcs_actor_scheduler.cc | 64 +- src/ray/gcs/gcs_server/gcs_actor_scheduler.h | 18 +- .../gcs_autoscaler_state_manager.cc | 4 +- .../gcs/gcs_server/gcs_resource_manager.cc | 8 +- src/ray/gcs/gcs_server/gcs_resource_manager.h | 15 +- src/ray/gcs/gcs_server/gcs_server.cc | 24 +- src/ray/gcs/gcs_server/gcs_server.h | 20 +- .../gcs_actor_manager_export_event_test.cc | 2 +- .../tests/gcs_actor_manager_test.cc | 7 +- .../tests/gcs_actor_scheduler_mock_test.cc | 24 +- .../tests/gcs_actor_scheduler_test.cc | 60 +- .../gcs_server/tests/gcs_server_test_util.h | 386 +++++ src/ray/gcs/tests/gcs_test_util.h | 1 - src/ray/protobuf/common.proto | 25 + src/ray/protobuf/node_manager.proto | 41 +- src/ray/raylet/BUILD.bazel | 34 +- ...manager.cc => lease_dependency_manager.cc} | 157 +- ...y_manager.h => lease_dependency_manager.h} | 122 +- ...task_manager.cc => local_lease_manager.cc} | 775 ++++----- ...l_task_manager.h => local_lease_manager.h} | 291 ++-- src/ray/raylet/main.cc | 68 +- src/ray/raylet/node_manager.cc | 417 +++-- src/ray/raylet/node_manager.h | 86 +- src/ray/raylet/scheduling/BUILD.bazel | 44 +- ...sk_manager.cc => cluster_lease_manager.cc} | 256 +-- ...task_manager.h => cluster_lease_manager.h} | 141 +- ...ce.h => cluster_lease_manager_interface.h} | 75 +- .../scheduling/cluster_resource_manager.h | 6 +- .../scheduling/cluster_resource_scheduler.cc | 24 +- .../scheduling/cluster_resource_scheduler.h | 6 +- src/ray/raylet/scheduling/internal.h | 11 +- ...face.h => local_lease_manager_interface.h} | 89 +- .../scheduling/scheduler_resource_reporter.cc | 35 +- .../scheduling/scheduler_resource_reporter.h | 14 +- src/ray/raylet/scheduling/scheduler_stats.cc | 82 +- src/ray/raylet/scheduling/scheduler_stats.h | 34 +- src/ray/raylet/scheduling/tests/BUILD.bazel | 11 +- ..._test.cc => cluster_lease_manager_test.cc} | 1495 +++++++++-------- .../tests/cluster_resource_scheduler_test.cc | 14 +- src/ray/raylet/tests/BUILD.bazel | 26 +- .../raylet/tests/dependency_manager_test.cc | 399 ----- .../tests/lease_dependency_manager_test.cc | 403 +++++ ...er_test.cc => local_lease_manager_test.cc} | 133 +- src/ray/raylet/tests/node_manager_test.cc | 105 +- src/ray/raylet/tests/util.h | 40 +- ...rker_killing_policy_group_by_owner_test.cc | 30 +- ...rker_killing_policy_retriable_fifo_test.cc | 24 +- .../tests/worker_killing_policy_test.cc | 46 +- src/ray/raylet/tests/worker_pool_test.cc | 547 +++--- src/ray/raylet/worker.cc | 21 +- src/ray/raylet/worker.h | 78 +- src/ray/raylet/worker_killing_policy.cc | 17 +- .../worker_killing_policy_group_by_owner.cc | 36 +- .../worker_killing_policy_group_by_owner.h | 38 +- .../worker_killing_policy_retriable_fifo.cc | 8 +- src/ray/raylet/worker_pool.cc | 182 +- src/ray/raylet/worker_pool.h | 44 +- src/ray/raylet_client/raylet_client.cc | 55 +- src/ray/raylet_client/raylet_client.h | 52 +- .../rpc/node_manager/node_manager_client.h | 6 +- .../rpc/node_manager/node_manager_server.h | 70 +- src/ray/util/logging.h | 1 + 98 files changed, 4848 insertions(+), 3883 deletions(-) rename src/mock/ray/raylet/{local_task_manager.h => local_lease_manager.h} (76%) create mode 100644 src/ray/common/lease/lease.h create mode 100644 src/ray/common/lease/lease_spec.cc create mode 100644 src/ray/common/lease/lease_spec.h delete mode 100644 src/ray/common/task/task.cc delete mode 100644 src/ray/common/task/task.h create mode 100644 src/ray/gcs/gcs_server/tests/gcs_server_test_util.h rename src/ray/raylet/{dependency_manager.cc => lease_dependency_manager.cc} (65%) rename src/ray/raylet/{dependency_manager.h => lease_dependency_manager.h} (73%) rename src/ray/raylet/{local_task_manager.cc => local_lease_manager.cc} (57%) rename src/ray/raylet/{local_task_manager.h => local_lease_manager.h} (53%) rename src/ray/raylet/scheduling/{cluster_task_manager.cc => cluster_lease_manager.cc} (62%) rename src/ray/raylet/scheduling/{cluster_task_manager.h => cluster_lease_manager.h} (62%) rename src/ray/raylet/scheduling/{cluster_task_manager_interface.h => cluster_lease_manager_interface.h} (58%) rename src/ray/raylet/scheduling/{local_task_manager_interface.h => local_lease_manager_interface.h} (57%) rename src/ray/raylet/scheduling/tests/{cluster_task_manager_test.cc => cluster_lease_manager_test.cc} (63%) delete mode 100644 src/ray/raylet/tests/dependency_manager_test.cc create mode 100644 src/ray/raylet/tests/lease_dependency_manager_test.cc rename src/ray/raylet/tests/{local_task_manager_test.cc => local_lease_manager_test.cc} (75%) diff --git a/python/ray/tests/test_placement_group_5.py b/python/ray/tests/test_placement_group_5.py index 8440cee1ef37..7caf7a77c61f 100644 --- a/python/ray/tests/test_placement_group_5.py +++ b/python/ray/tests/test_placement_group_5.py @@ -510,7 +510,7 @@ def test_remove_placement_group_with_pending_worker_lease_waiting_for_pg_resourc Specific test steps: 1. Create a placement group with only 1 bundle. 2. Create two actors using the aforementioned pg. At this point, - the latter actor lease request will definitely be pending in local task manager dispatch queue due to + the latter actor lease request will definitely be pending in local lease manager leases_to_grant queue due to unavailable pg bundle resources. 3. Remove the pg while the latter actor lease request is pending. 4. Verify that the pending actor lease request is cancelled and the pg @@ -549,7 +549,7 @@ def wait_for_actor2_added_to_dispatch_queue(): return False for sample in samples: if sample.labels["State"] == "Dispatched" and sample.value == 1: - # actor2 is in the local task manager dispatch queue + # actor2 is in the local lease manager leases_to_grant queue return True return False diff --git a/src/fakes/ray/rpc/raylet/raylet_client.h b/src/fakes/ray/rpc/raylet/raylet_client.h index b18394c56695..1a434abc20a6 100644 --- a/src/fakes/ray/rpc/raylet/raylet_client.h +++ b/src/fakes/ray/rpc/raylet/raylet_client.h @@ -27,7 +27,7 @@ class FakeRayletClient : public RayletClientInterface { const ray::rpc::ClientCallback &callback) override {} void RequestWorkerLease( - const rpc::TaskSpec &task_spec, + const rpc::LeaseSpec &lease_spec, bool grant_or_reject, const ray::rpc::ClientCallback &callback, const int64_t backlog_size = -1, @@ -36,11 +36,11 @@ class FakeRayletClient : public RayletClientInterface { callbacks.push_back(callback); } - ray::Status ReturnWorker(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) override { + ray::Status ReturnWorkerLease(int worker_port, + const WorkerID &worker_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) override { if (disconnect_worker) { num_workers_disconnected++; } else { @@ -61,7 +61,7 @@ class FakeRayletClient : public RayletClientInterface { } void CancelWorkerLease( - const TaskID &task_id, + const LeaseID &lease_id, const rpc::ClientCallback &callback) override { num_leases_canceled += 1; cancel_callbacks.push_back(callback); @@ -238,10 +238,10 @@ class FakeRayletClient : public RayletClientInterface { void *metadata, const rpc::ClientCallback &callback) override {} - void GetTaskFailureCause( - const TaskID &task_id, - const rpc::ClientCallback &callback) override { - ray::rpc::GetTaskFailureCauseReply reply; + void GetWorkerFailureCause( + const LeaseID &lease_id, + const rpc::ClientCallback &callback) override { + ray::rpc::GetWorkerFailureCauseReply reply; callback(Status::OK(), std::move(reply)); num_get_task_failure_causes += 1; } @@ -266,9 +266,9 @@ class FakeRayletClient : public RayletClientInterface { drain_raylet_callbacks.push_back(callback); } - void CancelTasksWithResourceShapes( + void CancelLeasesWithResourceShapes( const std::vector> &resource_shapes, - const rpc::ClientCallback &callback) + const rpc::ClientCallback &callback) override {} void IsLocalWorkerDead( diff --git a/src/mock/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/mock/ray/gcs/gcs_server/gcs_actor_scheduler.h index 164be134d22b..f4edac842542 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_actor_scheduler.h +++ b/src/mock/ray/gcs/gcs_server/gcs_actor_scheduler.h @@ -28,7 +28,7 @@ class MockGcsActorSchedulerInterface : public GcsActorSchedulerInterface { MOCK_METHOD(std::vector, CancelOnNode, (const NodeID &node_id), (override)); MOCK_METHOD(void, CancelOnLeasing, - (const NodeID &node_id, const ActorID &actor_id, const TaskID &task_id), + (const NodeID &node_id, const ActorID &actor_id, const LeaseID &lease_id), (override)); MOCK_METHOD(ActorID, CancelOnWorker, @@ -68,7 +68,7 @@ class MockGcsActorScheduler : public GcsActorScheduler { MOCK_METHOD(std::vector, CancelOnNode, (const NodeID &node_id), (override)); MOCK_METHOD(void, CancelOnLeasing, - (const NodeID &node_id, const ActorID &actor_id, const TaskID &task_id), + (const NodeID &node_id, const ActorID &actor_id, const LeaseID &lease_id), (override)); MOCK_METHOD(ActorID, CancelOnWorker, diff --git a/src/mock/ray/raylet/local_task_manager.h b/src/mock/ray/raylet/local_lease_manager.h similarity index 76% rename from src/mock/ray/raylet/local_task_manager.h rename to src/mock/ray/raylet/local_lease_manager.h index 1dbbb8aea9ef..825dae47dde7 100644 --- a/src/mock/ray/raylet/local_task_manager.h +++ b/src/mock/ray/raylet/local_lease_manager.h @@ -15,25 +15,25 @@ #pragma once #include "gmock/gmock.h" -#include "ray/raylet/scheduling/local_task_manager_interface.h" +#include "ray/raylet/scheduling/local_lease_manager_interface.h" namespace ray::raylet { -class MockLocalTaskManager : public ILocalTaskManager { +class MockLocalLeaseManager : public LocalLeaseManagerInterface { public: MOCK_METHOD(void, - QueueAndScheduleTask, + QueueAndScheduleLease, (std::shared_ptr work), (override)); - MOCK_METHOD(void, ScheduleAndDispatchTasks, (), (override)); + MOCK_METHOD(void, ScheduleAndGrantLeases, (), (override)); MOCK_METHOD(bool, - CancelTasks, + CancelLeases, (std::function &)> predicate, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message), (override)); MOCK_METHOD((const absl::flat_hash_map>> &), - GetTaskToDispatch, + GetLeasesToGrant, (), (const, override)); MOCK_METHOD((const absl::flat_hash_map worker, RayTask *task), + CleanupLease, + (std::shared_ptr worker, RayLease *lease), (override)); - MOCK_METHOD(void, TasksUnblocked, (const std::vector &ready_ids), (override)); + MOCK_METHOD(void, LeasesUnblocked, (const std::vector &ready_ids), (override)); MOCK_METHOD(void, ReleaseWorkerResources, (std::shared_ptr worker), @@ -72,9 +72,9 @@ class MockLocalTaskManager : public ILocalTaskManager { MOCK_METHOD(ResourceSet, CalcNormalTaskResources, (), (const, override)); MOCK_METHOD(void, RecordMetrics, (), (const, override)); MOCK_METHOD(void, DebugStr, (std::stringstream & buffer), (const, override)); - MOCK_METHOD(size_t, GetNumTaskSpilled, (), (const, override)); - MOCK_METHOD(size_t, GetNumWaitingTaskSpilled, (), (const, override)); - MOCK_METHOD(size_t, GetNumUnschedulableTaskSpilled, (), (const, override)); + MOCK_METHOD(size_t, GetNumLeaseSpilled, (), (const, override)); + MOCK_METHOD(size_t, GetNumWaitingLeaseSpilled, (), (const, override)); + MOCK_METHOD(size_t, GetNumUnschedulableLeaseSpilled, (), (const, override)); }; } // namespace ray::raylet diff --git a/src/mock/ray/raylet/worker_pool.h b/src/mock/ray/raylet/worker_pool.h index 731e59abd424..6e8337aef2d4 100644 --- a/src/mock/ray/raylet/worker_pool.h +++ b/src/mock/ray/raylet/worker_pool.h @@ -22,7 +22,7 @@ class MockWorkerPool : public WorkerPoolInterface { public: MOCK_METHOD(void, PopWorker, - (const TaskSpecification &task_spec, const PopWorkerCallback &callback), + (const LeaseSpecification &lease_spec, const PopWorkerCallback &callback), (override)); MOCK_METHOD(void, PushWorker, @@ -100,7 +100,7 @@ class MockWorkerPool : public WorkerPoolInterface { (override)); MOCK_METHOD(void, PrestartWorkers, - (const TaskSpecification &task_spec, int64_t backlog_size), + (const LeaseSpecification &lease_spec, int64_t backlog_size), (override)); MOCK_METHOD(void, StartNewWorker, diff --git a/src/mock/ray/raylet_client/raylet_client.h b/src/mock/ray/raylet_client/raylet_client.h index dc804ff16207..dab5fb18dc33 100644 --- a/src/mock/ray/raylet_client/raylet_client.h +++ b/src/mock/ray/raylet_client/raylet_client.h @@ -25,14 +25,14 @@ class MockRayletClientInterface : public RayletClientInterface { MOCK_METHOD( void, RequestWorkerLease, - (const rpc::TaskSpec &resource_spec, + (const rpc::LeaseSpec &lease_spec, bool grant_or_reject, const ray::rpc::ClientCallback &callback, const int64_t backlog_size, const bool is_selected_based_on_locality), (override)); MOCK_METHOD(ray::Status, - ReturnWorker, + ReturnWorkerLease, (int worker_port, const WorkerID &worker_id, bool disconnect_worker, @@ -40,9 +40,9 @@ class MockRayletClientInterface : public RayletClientInterface { bool worker_exiting), (override)); MOCK_METHOD(void, - GetTaskFailureCause, - (const TaskID &task_id, - const rpc::ClientCallback &callback), + GetWorkerFailureCause, + (const LeaseID &lease_id, + const rpc::ClientCallback &callback), (override)); MOCK_METHOD(void, PrestartWorkers, @@ -56,7 +56,7 @@ class MockRayletClientInterface : public RayletClientInterface { (override)); MOCK_METHOD(void, CancelWorkerLease, - (const TaskID &task_id, + (const LeaseID &lease_id, const rpc::ClientCallback &callback), (override)); MOCK_METHOD( @@ -132,9 +132,9 @@ class MockRayletClientInterface : public RayletClientInterface { (override)); MOCK_METHOD( void, - CancelTasksWithResourceShapes, + CancelLeasesWithResourceShapes, ((const std::vector>)&resource_shapes, - const rpc::ClientCallback &callback), + const rpc::ClientCallback &callback), (override)); MOCK_METHOD(void, IsLocalWorkerDead, diff --git a/src/ray/common/BUILD.bazel b/src/ray/common/BUILD.bazel index 5eea6f6c8bea..9b8a89329019 100644 --- a/src/ray/common/BUILD.bazel +++ b/src/ray/common/BUILD.bazel @@ -139,6 +139,9 @@ ray_cc_library( ], ) +# TODO(#55922): split out the scheduling dependencies into a separate bazel target +# and have lease and task bazel targets depend on this + ray_cc_library( name = "task_common", srcs = [ @@ -152,7 +155,6 @@ ray_cc_library( "scheduling/resource_instance_set.cc", "scheduling/resource_set.cc", "scheduling/scheduling_ids.cc", - "task/task.cc", "task/task_spec.cc", ], hdrs = [ @@ -166,7 +168,6 @@ ray_cc_library( "scheduling/resource_instance_set.h", "scheduling/resource_set.h", "scheduling/scheduling_ids.h", - "task/task.h", "task/task_common.h", "task/task_spec.h", "task/task_util.h", @@ -188,6 +189,22 @@ ray_cc_library( ], ) +ray_cc_library( + name = "lease", + srcs = [ + "lease/lease_spec.cc", + ], + hdrs = [ + "lease/lease.h", + "lease/lease_spec.h", + ], + deps = [ + ":id", + ":task_common", + "//src/ray/protobuf:common_cc_proto", + ], +) + ray_cc_library( name = "asio", srcs = [ diff --git a/src/ray/common/function_descriptor.h b/src/ray/common/function_descriptor.h index b4f7ca3cd92a..452fc446ae6c 100644 --- a/src/ray/common/function_descriptor.h +++ b/src/ray/common/function_descriptor.h @@ -145,7 +145,7 @@ class JavaFunctionDescriptor : public FunctionDescriptorInterface { virtual std::string ClassName() const { return typed_message_->class_name(); } - const std::string &FunctionName() const { return typed_message_->function_name(); } + virtual std::string FunctionName() const { return typed_message_->function_name(); } const std::string &Signature() const { return typed_message_->signature(); } diff --git a/src/ray/common/grpc_util.h b/src/ray/common/grpc_util.h index ea48af266fce..ae99eaf79081 100644 --- a/src/ray/common/grpc_util.h +++ b/src/ray/common/grpc_util.h @@ -32,6 +32,7 @@ namespace ray { /// Wrap a protobuf message. template +// TODO(#55921): Remove MessageWrapper class and clean up LeaseSpec/TaskSpec classes class MessageWrapper { public: /// Construct an empty message wrapper. This should not be used directly. diff --git a/src/ray/common/id.cc b/src/ray/common/id.cc index 53b033dfb7f5..65ace9147bed 100644 --- a/src/ray/common/id.cc +++ b/src/ray/common/id.cc @@ -143,7 +143,6 @@ ActorID ActorID::Of(const JobID &job_id, absl::GetCurrentTimeNanos(), ActorID::kUniqueBytesLength); std::copy_n(job_id.Data(), JobID::kLength, std::back_inserter(data)); - RAY_CHECK(data.size() == kLength); return ActorID::FromBinary(data); } @@ -151,7 +150,6 @@ ActorID ActorID::NilFromJob(const JobID &job_id) { std::string data(kUniqueBytesLength, 0); FillNil(&data); std::copy_n(job_id.Data(), JobID::kLength, std::back_inserter(data)); - RAY_CHECK(data.size() == kLength); return ActorID::FromBinary(data); } @@ -166,7 +164,6 @@ TaskID TaskID::ForDriverTask(const JobID &job_id) { FillNil(&data); const auto dummy_actor_id = ActorID::NilFromJob(job_id); std::copy_n(dummy_actor_id.Data(), ActorID::kLength, std::back_inserter(data)); - RAY_CHECK(data.size() == TaskID::kLength); return TaskID::FromBinary(data); } @@ -181,7 +178,6 @@ TaskID TaskID::ForActorCreationTask(const ActorID &actor_id) { std::string data(kUniqueBytesLength, 0); FillNil(&data); std::copy_n(actor_id.Data(), ActorID::kLength, std::back_inserter(data)); - RAY_CHECK(data.size() == TaskID::kLength); return TaskID::FromBinary(data); } @@ -192,7 +188,6 @@ TaskID TaskID::ForActorTask(const JobID &job_id, std::string data = GenerateUniqueBytes( job_id, parent_task_id, parent_task_counter, 0, TaskID::kUniqueBytesLength); std::copy_n(actor_id.Data(), ActorID::kLength, std::back_inserter(data)); - RAY_CHECK(data.size() == TaskID::kLength); return TaskID::FromBinary(data); } @@ -203,7 +198,6 @@ TaskID TaskID::ForNormalTask(const JobID &job_id, job_id, parent_task_id, parent_task_counter, 0, TaskID::kUniqueBytesLength); const auto dummy_actor_id = ActorID::NilFromJob(job_id); std::copy_n(dummy_actor_id.Data(), ActorID::kLength, std::back_inserter(data)); - RAY_CHECK(data.size() == TaskID::kLength); return TaskID::FromBinary(data); } @@ -312,7 +306,6 @@ PlacementGroupID PlacementGroupID::Of(const JobID &job_id) { std::string data(PlacementGroupID::kUniqueBytesLength, 0); FillRandom(&data); std::copy_n(job_id.Data(), JobID::kLength, std::back_inserter(data)); - RAY_CHECK(data.size() == kLength); return PlacementGroupID::FromBinary(data); } @@ -322,6 +315,31 @@ JobID PlacementGroupID::JobId() const { reinterpret_cast(this->Data() + kUniqueBytesLength), JobID::kLength)); } +LeaseID LeaseID::FromRandom() { + std::string data(kLength, 0); + FillRandom(&data); + return LeaseID::FromBinary(data); +} + +LeaseID LeaseID::FromWorker(const WorkerID &worker_id, uint32_t counter) { + RAY_CHECK_GT(counter, 0u); + std::string data(kUniqueBytesLength, 0); + std::memcpy(data.data(), &counter, sizeof(counter)); + std::copy_n(worker_id.Data(), kUniqueIDSize, std::back_inserter(data)); + return LeaseID::FromBinary(data); +} + +LeaseID LeaseID::DriverLeaseId(const WorkerID &driver_id) { + std::string data(kUniqueBytesLength, 0); + std::copy_n(driver_id.Data(), kUniqueIDSize, std::back_inserter(data)); + return LeaseID::FromBinary(data); +} + +WorkerID LeaseID::WorkerId() const { + return WorkerID::FromBinary(std::string( + reinterpret_cast(id_ + kUniqueBytesLength), kUniqueIDSize)); +} + #define ID_OSTREAM_OPERATOR(id_type) \ std::ostream &operator<<(std::ostream &os, const id_type &id) { \ if (id.IsNil()) { \ @@ -338,6 +356,7 @@ ID_OSTREAM_OPERATOR(ActorID); ID_OSTREAM_OPERATOR(TaskID); ID_OSTREAM_OPERATOR(ObjectID); ID_OSTREAM_OPERATOR(PlacementGroupID); +ID_OSTREAM_OPERATOR(LeaseID); const NodeID kGCSNodeID = NodeID::FromBinary(std::string(kUniqueIDSize, 0)); diff --git a/src/ray/common/id.h b/src/ray/common/id.h index 736c116a341a..8a1f7e99d610 100644 --- a/src/ray/common/id.h +++ b/src/ray/common/id.h @@ -131,12 +131,8 @@ class ActorID : public BaseID { static constexpr size_t kUniqueBytesLength = 12; public: - /// Length of `ActorID` in bytes. static constexpr size_t kLength = kUniqueBytesLength + JobID::kLength; - /// Size of `ActorID` in bytes. - /// - /// \return Size of `ActorID` in bytes. static constexpr size_t Size() { return kLength; } /// Creates an `ActorID` by hashing the given information. @@ -150,22 +146,13 @@ class ActorID : public BaseID { const TaskID &parent_task_id, const size_t parent_task_counter); - /// Creates a nil ActorID with the given job. - /// - /// \param job_id The job id to which this actor belongs. - /// - /// \return The `ActorID` with unique bytes being nil. static ActorID NilFromJob(const JobID &job_id); // Warning: this can duplicate IDs after a fork() call. We assume this never happens. static ActorID FromRandom() = delete; - /// Constructor of `ActorID`. ActorID() : BaseID() {} - /// Get the job id to which this actor belongs. - /// - /// \return The job id to which this actor belongs. JobID JobId() const; MSGPACK_DEFINE(id_); @@ -190,18 +177,11 @@ class TaskID : public BaseID { // Warning: this can duplicate IDs after a fork() call. We assume this never happens. static TaskID FromRandom() = delete; - /// The ID generated for driver task. static TaskID ForDriverTask(const JobID &job_id); /// Generate driver task id for the given job. static TaskID FromRandom(const JobID &job_id); - /// Creates a TaskID for an actor creation task. - /// - /// \param actor_id The ID of the actor that will be created - /// by this actor creation task. - /// - /// \return The ID of the actor creation task. static TaskID ForActorCreationTask(const ActorID &actor_id); /// Creates a TaskID for actor task. @@ -241,17 +221,10 @@ class TaskID : public BaseID { /// \return The ID of the n-th execution of the task. static TaskID ForExecutionAttempt(const TaskID &task_id, uint64_t attempt_number); - /// Get the id of the actor to which this task belongs. - /// - /// \return The `ActorID` of the actor which creates this task. ActorID ActorId() const; - /// Returns whether this is the ID of an actor creation task. bool IsForActorCreationTask() const; - /// Get the id of the job to which this task belongs. - /// - /// \return The `JobID` of the job which creates this task. JobID JobId() const; MSGPACK_DEFINE(id_); @@ -268,7 +241,6 @@ class ObjectID : public BaseID { /// The maximum number of objects that can be returned or put by a task. static constexpr int64_t kMaxObjectIndex = ((int64_t)1 << kObjectIdIndexSize) - 1; - /// The length of ObjectID in bytes. static constexpr size_t kLength = kIndexBytesLength + TaskID::kLength; ObjectID() : BaseID() {} @@ -288,9 +260,6 @@ class ObjectID : public BaseID { /// this object. ObjectIDIndexType ObjectIndex() const; - /// Compute the task ID of the task that created the object. - /// - /// \return The task ID of the task that created this object. TaskID TaskId() const; /// Compute the object ID of an object created by a task, either via an object put @@ -302,12 +271,8 @@ class ObjectID : public BaseID { /// \return The computed object ID. static ObjectID FromIndex(const TaskID &task_id, ObjectIDIndexType index); - /// Create an object id randomly. - /// /// Warning: this can duplicate IDs after a fork() call. We assume this /// never happens. - /// - /// \return A random object id. static ObjectID FromRandom(); /// Compute the object ID that is used to track an actor's lifetime. This @@ -321,6 +286,7 @@ class ObjectID : public BaseID { /// Whether this ObjectID represents an actor handle. This is the ObjectID /// returned by the actor's creation task. static bool IsActorID(const ObjectID &object_id); + /// Return the ID of the actor that produces this object. For the actor /// creation task and for tasks executed by the actor, this will return a /// non-nil ActorID. @@ -329,7 +295,6 @@ class ObjectID : public BaseID { MSGPACK_DEFINE(id_); private: - /// A helper method to generate an ObjectID. static ObjectID GenerateObjectId(const std::string &task_id_binary, ObjectIDIndexType object_index = 0); @@ -342,12 +307,8 @@ class PlacementGroupID : public BaseID { static constexpr size_t kUniqueBytesLength = 14; public: - /// Length of `PlacementGroupID` in bytes. static constexpr size_t kLength = kUniqueBytesLength + JobID::kLength; - /// Size of `PlacementGroupID` in bytes. - /// - /// \return Size of `PlacementGroupID` in bytes. static constexpr size_t Size() { return kLength; } /// Creates a `PlacementGroupID` by hashing the given information. @@ -359,12 +320,8 @@ class PlacementGroupID : public BaseID { static PlacementGroupID FromRandom() = delete; - /// Constructor of `PlacementGroupID`. PlacementGroupID() : BaseID() {} - /// Get the job id to which this placement group belongs. - /// - /// \return The job id to which this placement group belongs. JobID JobId() const; MSGPACK_DEFINE(id_); @@ -375,6 +332,48 @@ class PlacementGroupID : public BaseID { typedef std::pair BundleID; +class LeaseID : public BaseID { + private: + static constexpr size_t kUniqueBytesLength = 4; + + public: + static constexpr size_t kLength = kUniqueBytesLength + kUniqueIDSize; + + static constexpr size_t Size() { return kLength; } + + /// Creates a `LeaseID` from a specific worker ID. + /// + /// \param worker_id The worker ID from which this lease is requested. + /// \param counter The n-th lease requested by this worker, staring from 1 + /// + /// \return The `LeaseID` for the worker lease. + static LeaseID FromWorker(const WorkerID &worker_id, uint32_t counter); + + /// Creates a `LeaseID` from a driver ID. The counter bits are nulled out only for + /// driver as we need a predetermined lease value that can be calculated indepently by + /// the raylet without having to send the ID over. + /// + /// \param driver_id The driver ID to which this lease belongs. + /// + /// \return The `LeaseID` for the worker lease. + static LeaseID DriverLeaseId(const WorkerID &driver_id); + + /// Creates a random `LeaseID`. + /// + /// \return A `LeaseID` generated with random bytes + /// Warning: this can duplicate IDs after a fork() call. We assume this never happens. + static LeaseID FromRandom(); + + LeaseID() : BaseID() {} + + WorkerID WorkerId() const; + + MSGPACK_DEFINE(id_); + + private: + uint8_t id_[kLength]; +}; + static_assert(sizeof(JobID) == JobID::kLength + sizeof(size_t), "JobID size is not as expected"); static_assert(sizeof(ActorID) == ActorID::kLength + sizeof(size_t), @@ -385,6 +384,8 @@ static_assert(sizeof(ObjectID) == ObjectID::kLength + sizeof(size_t), "ObjectID size is not as expected"); static_assert(sizeof(PlacementGroupID) == PlacementGroupID::kLength + sizeof(size_t), "PlacementGroupID size is not as expected"); +static_assert(sizeof(LeaseID) == LeaseID::kLength + sizeof(size_t), + "LeaseID size is not as expected"); std::ostream &operator<<(std::ostream &os, const UniqueID &id); std::ostream &operator<<(std::ostream &os, const JobID &id); @@ -392,6 +393,7 @@ std::ostream &operator<<(std::ostream &os, const ActorID &id); std::ostream &operator<<(std::ostream &os, const TaskID &id); std::ostream &operator<<(std::ostream &os, const ObjectID &id); std::ostream &operator<<(std::ostream &os, const PlacementGroupID &id); +std::ostream &operator<<(std::ostream &os, const LeaseID &id); #define DEFINE_UNIQUE_ID(type) \ class RAY_EXPORT type : public UniqueID { \ @@ -590,6 +592,11 @@ struct DefaultLogKey { constexpr static std::string_view key = kLogKeyPlacementGroupID; }; +template <> +struct DefaultLogKey { + constexpr static std::string_view key = kLogKeyLeaseID; +}; + } // namespace ray namespace std { @@ -606,6 +613,7 @@ DEFINE_UNIQUE_ID(ActorID); DEFINE_UNIQUE_ID(TaskID); DEFINE_UNIQUE_ID(ObjectID); DEFINE_UNIQUE_ID(PlacementGroupID); +DEFINE_UNIQUE_ID(LeaseID); #include "ray/common/id_def.h" #undef DEFINE_UNIQUE_ID diff --git a/src/ray/common/lease/lease.h b/src/ray/common/lease/lease.h new file mode 100644 index 000000000000..1dd4853c4064 --- /dev/null +++ b/src/ray/common/lease/lease.h @@ -0,0 +1,82 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include + +#include "ray/common/lease/lease_spec.h" + +namespace ray { + +/// \class RayLease +/// +/// A RayLease represents a Ray lease and a specification of its execution (e.g., +/// resource demands). The lease's specification contains both immutable fields, +/// determined at submission time, and mutable fields, determined at execution +/// time. +class RayLease { + public: + /// Construct an empty lease. This should only be used to pass a lease + /// as an out parameter to a function or method. + // TODO(#55923): Remove this constructor and refactor worker.h to use unique_ptr + RayLease() = default; + + /// Construct a `RayLease` object from a protobuf message. + explicit RayLease(rpc::LeaseSpec lease_spec) + : lease_spec_(LeaseSpecification(std::move(lease_spec))) {} + + /// Construct a `RayLease` object from a `LeaseSpecification`. + explicit RayLease(LeaseSpecification lease_spec) : lease_spec_(std::move(lease_spec)) {} + + RayLease(LeaseSpecification lease_spec, std::string preferred_node_id) + : lease_spec_(std::move(lease_spec)), + preferred_node_id_(std::move(preferred_node_id)) {} + + /// Get the immutable specification for the lease. + /// + /// \return The immutable specification for the lease. + const LeaseSpecification &GetLeaseSpecification() const { return lease_spec_; } + + /// Get the lease's object dependencies. This comprises the immutable lease + /// arguments and the mutable execution dependencies. + /// + /// \return The object dependencies. + const std::vector &GetDependencies() const { + return lease_spec_.GetDependencies(); + } + + /// Get the lease's preferred node id for scheduling. If the returned value + /// is empty, then it means the lease has no preferred node. + /// + /// \return The preferred node id. + const std::string &GetPreferredNodeID() const { return preferred_node_id_; } + + std::string DebugString() const { + return absl::StrFormat("lease_spec={%s}", lease_spec_.DebugString()); + } + + private: + /// RayLease specification object, consisting of immutable information about this + /// lease determined at submission time. Includes resource demand, object + /// dependencies, etc. + LeaseSpecification lease_spec_; + + std::string preferred_node_id_; +}; + +} // namespace ray diff --git a/src/ray/common/lease/lease_spec.cc b/src/ray/common/lease/lease_spec.cc new file mode 100644 index 000000000000..8dcf2552d472 --- /dev/null +++ b/src/ray/common/lease/lease_spec.cc @@ -0,0 +1,360 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/common/lease/lease_spec.h" + +#include "ray/common/common_protocol.h" +#include "ray/common/function_descriptor.h" +#include "ray/common/runtime_env_common.h" + +namespace ray { + +LeaseSpecification::LeaseSpecification(const rpc::TaskSpec &task_spec) + : MessageWrapper(std::make_shared()) { + RAY_CHECK(task_spec.type() == TaskType::NORMAL_TASK || + task_spec.type() == TaskType::ACTOR_CREATION_TASK); + message_->set_job_id(task_spec.job_id()); + message_->mutable_caller_address()->CopyFrom(task_spec.caller_address()); + message_->mutable_required_resources()->insert(task_spec.required_resources().begin(), + task_spec.required_resources().end()); + message_->mutable_required_placement_resources()->insert( + task_spec.required_placement_resources().begin(), + task_spec.required_placement_resources().end()); + message_->mutable_scheduling_strategy()->CopyFrom(task_spec.scheduling_strategy()); + message_->mutable_label_selector()->insert(task_spec.label_selector().begin(), + task_spec.label_selector().end()); + message_->set_depth(task_spec.depth()); + message_->set_parent_task_id(task_spec.parent_task_id()); + message_->mutable_dependencies()->Reserve(task_spec.args_size()); + for (size_t i = 0; i < static_cast(task_spec.args_size()); ++i) { + if (task_spec.args(i).has_object_ref() && !task_spec.args(i).is_inlined()) { + message_->add_dependencies()->CopyFrom(task_spec.args(i).object_ref()); + } + } + message_->mutable_function_descriptor()->CopyFrom(task_spec.function_descriptor()); + message_->set_language(task_spec.language()); + message_->mutable_runtime_env_info()->CopyFrom(task_spec.runtime_env_info()); + message_->set_attempt_number(task_spec.attempt_number()); + message_->set_root_detached_actor_id(task_spec.root_detached_actor_id()); + message_->set_task_name(task_spec.name()); + message_->set_type(task_spec.type()); + if (IsActorCreationTask()) { + message_->set_actor_id(task_spec.actor_creation_task_spec().actor_id()); + message_->set_is_detached_actor(task_spec.actor_creation_task_spec().is_detached()); + message_->set_max_actor_restarts( + task_spec.actor_creation_task_spec().max_actor_restarts()); + for (const auto &option : + task_spec.actor_creation_task_spec().dynamic_worker_options()) { + message_->add_dynamic_worker_options(option); + } + } else { + message_->set_max_retries(task_spec.max_retries()); + } + ComputeResources(); +} + +LeaseID LeaseSpecification::LeaseId() const { + return LeaseID::FromBinary(message_->lease_id()); +} + +JobID LeaseSpecification::JobId() const { return JobID::FromBinary(message_->job_id()); } + +const rpc::Address &LeaseSpecification::CallerAddress() const { + return message_->caller_address(); +} + +Language LeaseSpecification::GetLanguage() const { return message_->language(); } + +bool LeaseSpecification::IsNormalTask() const { + return message_->type() == TaskType::NORMAL_TASK; +} + +bool LeaseSpecification::IsActorCreationTask() const { + return message_->type() == TaskType::ACTOR_CREATION_TASK; +} + +bool LeaseSpecification::IsNodeAffinitySchedulingStrategy() const { + return GetSchedulingStrategy().scheduling_strategy_case() == + rpc::SchedulingStrategy::kNodeAffinitySchedulingStrategy; +} + +NodeID LeaseSpecification::GetNodeAffinitySchedulingStrategyNodeId() const { + if (!IsNodeAffinitySchedulingStrategy()) { + return NodeID::Nil(); + } + return NodeID::FromBinary( + GetSchedulingStrategy().node_affinity_scheduling_strategy().node_id()); +} + +bool LeaseSpecification::GetNodeAffinitySchedulingStrategySoft() const { + if (!IsNodeAffinitySchedulingStrategy()) { + return false; + } + return GetSchedulingStrategy().node_affinity_scheduling_strategy().soft(); +} + +std::vector LeaseSpecification::GetDependencyIds() const { + std::vector ids; + ids.reserve(dependencies_.size()); + for (const auto &ref : dependencies_) { + ids.emplace_back(ObjectRefToId(ref)); + } + return ids; +} + +const std::vector &LeaseSpecification::GetDependencies() const { + return dependencies_; +} + +WorkerID LeaseSpecification::CallerWorkerId() const { + return WorkerID::FromBinary(message_->caller_address().worker_id()); +} + +NodeID LeaseSpecification::CallerNodeId() const { + return NodeID::FromBinary(message_->caller_address().node_id()); +} + +BundleID LeaseSpecification::PlacementGroupBundleId() const { + if (GetSchedulingStrategy().scheduling_strategy_case() != + rpc::SchedulingStrategy::kPlacementGroupSchedulingStrategy) { + return std::make_pair(PlacementGroupID::Nil(), -1); + } + const auto &pg = GetSchedulingStrategy().placement_group_scheduling_strategy(); + return std::make_pair(PlacementGroupID::FromBinary(pg.placement_group_id()), + pg.placement_group_bundle_index()); +} + +int64_t LeaseSpecification::MaxActorRestarts() const { + RAY_CHECK(IsActorCreationTask()); + return message_->max_actor_restarts(); +} + +int32_t LeaseSpecification::MaxRetries() const { + RAY_CHECK(IsNormalTask()); + return message_->max_retries(); +} + +bool LeaseSpecification::IsRetriable() const { + if (IsActorCreationTask() && MaxActorRestarts() == 0) { + return false; + } + if (IsNormalTask() && MaxRetries() == 0) { + return false; + } + return true; +} + +uint64_t LeaseSpecification::AttemptNumber() const { return message_->attempt_number(); } + +bool LeaseSpecification::IsRetry() const { return AttemptNumber() > 0; } + +std::string LeaseSpecification::GetTaskName() const { return message_->task_name(); } + +std::string LeaseSpecification::GetFunctionOrActorName() const { + if (IsActorCreationTask()) { + return FunctionDescriptor()->ClassName(); + } + return FunctionDescriptor()->CallString(); +} + +TaskID LeaseSpecification::ParentTaskId() const { + // Set to Nil for driver tasks. + if (message_->parent_task_id().empty()) { + return TaskID::Nil(); + } + return TaskID::FromBinary(message_->parent_task_id()); +} + +ActorID LeaseSpecification::ActorId() const { + if (message_->actor_id().empty()) { + return ActorID::Nil(); + } + return ActorID::FromBinary(message_->actor_id()); +} + +ActorID LeaseSpecification::RootDetachedActorId() const { + if (message_->root_detached_actor_id().empty()) { + return ActorID::Nil(); + } + return ActorID::FromBinary(message_->root_detached_actor_id()); +} + +bool LeaseSpecification::IsDetachedActor() const { return message_->is_detached_actor(); } + +int LeaseSpecification::GetRuntimeEnvHash() const { return runtime_env_hash_; } + +std::string LeaseSpecification::DebugString() const { + std::ostringstream stream; + stream << "Type=" << TaskType_Name(message_->type()) + << ", Language=" << Language_Name(message_->language()); + + if (required_resources_ != nullptr) { + stream << ", Resources: {"; + + // Print resource description. + for (const auto &entry : GetRequiredResources().GetResourceMap()) { + stream << entry.first << ": " << entry.second << ", "; + } + stream << "}"; + } + + stream << ", function_descriptor="; + + // Print function descriptor. + stream << FunctionDescriptor()->ToString(); + + stream << ", lease_id=" << LeaseId() << ", task_name=" << GetTaskName() + << ", job_id=" << JobId() << ", depth=" << GetDepth() + << ", attempt_number=" << AttemptNumber(); + + if (IsActorCreationTask()) { + // Print actor creation task spec. + stream << ", actor_creation_task_spec={actor_id=" << ActorId() + << ", max_restarts=" << MaxActorRestarts() + << ", is_detached=" << IsDetachedActor() << "}"; + } else { + stream << ", normal_task_spec={max_retries=" << MaxRetries() << "}"; + } + + // Print non-sensitive runtime env info. + if (HasRuntimeEnv()) { + const auto &runtime_env_info = RuntimeEnvInfo(); + stream << ", runtime_env_hash=" << GetRuntimeEnvHash(); + if (runtime_env_info.has_runtime_env_config()) { + stream << ", eager_install=" + << runtime_env_info.runtime_env_config().eager_install(); + stream << ", setup_timeout_seconds=" + << runtime_env_info.runtime_env_config().setup_timeout_seconds(); + } + } + + return stream.str(); +} + +bool LeaseSpecification::HasRuntimeEnv() const { + return !IsRuntimeEnvEmpty(SerializedRuntimeEnv()); +} + +const std::string &LeaseSpecification::SerializedRuntimeEnv() const { + return message_->runtime_env_info().serialized_runtime_env(); +} + +const rpc::RuntimeEnvInfo &LeaseSpecification::RuntimeEnvInfo() const { + return message_->runtime_env_info(); +} + +int64_t LeaseSpecification::GetDepth() const { return message_->depth(); } + +const rpc::SchedulingStrategy &LeaseSpecification::GetSchedulingStrategy() const { + return message_->scheduling_strategy(); +} + +const ResourceSet &LeaseSpecification::GetRequiredResources() const { + return *required_resources_; +} + +const ResourceSet &LeaseSpecification::GetRequiredPlacementResources() const { + return *required_placement_resources_; +} + +const LabelSelector &LeaseSpecification::GetLabelSelector() const { + return *label_selector_; +} + +ray::FunctionDescriptor LeaseSpecification::FunctionDescriptor() const { + return ray::FunctionDescriptorBuilder::FromProto(message_->function_descriptor()); +} + +void LeaseSpecification::ComputeResources() { + auto &required_resources = message_->required_resources(); + + if (required_resources.empty()) { + // A static nil object is used here to avoid allocating the empty object every time. + required_resources_ = ResourceSet::Nil(); + } else { + required_resources_ = + std::make_shared(MapFromProtobuf(required_resources)); + } + + auto &required_placement_resources = message_->required_placement_resources().empty() + ? required_resources + : message_->required_placement_resources(); + + if (required_placement_resources.empty()) { + required_placement_resources_ = ResourceSet::Nil(); + } else { + required_placement_resources_ = + std::make_shared(MapFromProtobuf(required_placement_resources)); + } + + // Set LabelSelector required for scheduling if specified. Parses string map + // from proto to LabelSelector data type. + label_selector_ = std::make_shared(message_->label_selector()); + + // Copy dependencies from message + dependencies_.reserve(message_->dependencies_size()); + for (int i = 0; i < message_->dependencies_size(); ++i) { + dependencies_.push_back(message_->dependencies(i)); + } + + // There is no need to compute `SchedulingClass` for actor tasks since + // the actor tasks need not be scheduled. + const bool is_actor_creation_task = IsActorCreationTask(); + const bool should_report_placement_resources = + RayConfig::instance().report_actor_placement_resources(); + const auto &resource_set = (is_actor_creation_task && should_report_placement_resources) + ? GetRequiredPlacementResources() + : GetRequiredResources(); + auto depth = GetDepth(); + auto label_selector = GetLabelSelector(); + const auto &function_descriptor = FunctionDescriptor(); + auto sched_cls_desc = SchedulingClassDescriptor( + resource_set, label_selector, function_descriptor, depth, GetSchedulingStrategy()); + // Map the scheduling class descriptor to an integer for performance. + sched_cls_id_ = TaskSpecification::GetSchedulingClass(sched_cls_desc); + RAY_CHECK_GT(sched_cls_id_, 0); + + runtime_env_hash_ = CalculateRuntimeEnvHash(SerializedRuntimeEnv()); +} + +std::vector LeaseSpecification::DynamicWorkerOptionsOrEmpty() const { + if (!IsActorCreationTask()) { + return {}; + } + return VectorFromProtobuf(message_->dynamic_worker_options()); +} + +std::vector LeaseSpecification::DynamicWorkerOptions() const { + RAY_CHECK(IsActorCreationTask()); + return VectorFromProtobuf(message_->dynamic_worker_options()); +} + +size_t LeaseSpecification::DynamicWorkerOptionsSize() const { + return message_->dynamic_worker_options_size(); +} + +const rpc::RuntimeEnvConfig &LeaseSpecification::RuntimeEnvConfig() const { + return message_->runtime_env_info().runtime_env_config(); +} + +bool LeaseSpecification::IsSpreadSchedulingStrategy() const { + return message_->scheduling_strategy().scheduling_strategy_case() == + rpc::SchedulingStrategy::SchedulingStrategyCase::kSpreadSchedulingStrategy; +} + +SchedulingClass LeaseSpecification::GetSchedulingClass() const { return sched_cls_id_; } + +const rpc::LeaseSpec &LeaseSpecification::GetMessage() const { return *message_; } + +} // namespace ray diff --git a/src/ray/common/lease/lease_spec.h b/src/ray/common/lease/lease_spec.h new file mode 100644 index 000000000000..6a2c566b795f --- /dev/null +++ b/src/ray/common/lease/lease_spec.h @@ -0,0 +1,114 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "ray/common/grpc_util.h" +#include "ray/common/id.h" +#include "ray/common/scheduling/label_selector.h" +#include "ray/common/scheduling/resource_set.h" +#include "ray/common/task/task_spec.h" +#include "src/ray/protobuf/common.pb.h" + +namespace ray { + +// LeaseSpec captures only the subset of TaskSpec used by the raylet for +// leasing, scheduling, dependency resolution, and cancellation. +class LeaseSpecification : public MessageWrapper { + public: + explicit LeaseSpecification(const rpc::TaskSpec &task_spec); + + /// Construct an empty task specification. This should not be used directly. + LeaseSpecification() { ComputeResources(); } + + explicit LeaseSpecification(rpc::LeaseSpec lease_spec) + : MessageWrapper(std::move(lease_spec)) { + ComputeResources(); + } + + explicit LeaseSpecification(std::shared_ptr message) + : MessageWrapper(std::move(message)) { + ComputeResources(); + } + + LeaseID LeaseId() const; + JobID JobId() const; + + const ResourceSet &GetRequiredResources() const; + const ResourceSet &GetRequiredPlacementResources() const; + const LabelSelector &GetLabelSelector() const; + const rpc::SchedulingStrategy &GetSchedulingStrategy() const; + bool IsNodeAffinitySchedulingStrategy() const; + NodeID GetNodeAffinitySchedulingStrategyNodeId() const; + bool GetNodeAffinitySchedulingStrategySoft() const; + std::vector GetDependencyIds() const; + const std::vector &GetDependencies() const; + + bool IsNormalTask() const; + bool IsActorCreationTask() const; + ActorID ActorId() const; + + const rpc::Address &CallerAddress() const; + WorkerID CallerWorkerId() const; + NodeID CallerNodeId() const; + BundleID PlacementGroupBundleId() const; + bool IsRetriable() const; + TaskID ParentTaskId() const; + bool IsDetachedActor() const; + std::string DebugString() const; + int GetRuntimeEnvHash() const; + Language GetLanguage() const; + bool HasRuntimeEnv() const; + const rpc::RuntimeEnvInfo &RuntimeEnvInfo() const; + const std::string &SerializedRuntimeEnv() const; + int64_t GetDepth() const; + ActorID RootDetachedActorId() const; + ray::FunctionDescriptor FunctionDescriptor() const; + int64_t MaxActorRestarts() const; + int32_t MaxRetries() const; + uint64_t AttemptNumber() const; + bool IsRetry() const; + std::string GetTaskName() const; + std::string GetFunctionOrActorName() const; + std::vector DynamicWorkerOptionsOrEmpty() const; + std::vector DynamicWorkerOptions() const; + size_t DynamicWorkerOptionsSize() const; + const rpc::RuntimeEnvConfig &RuntimeEnvConfig() const; + bool IsSpreadSchedulingStrategy() const; + SchedulingClass GetSchedulingClass() const; + const rpc::LeaseSpec &GetMessage() const; + + private: + void ComputeResources(); + + SchedulingClass GetSchedulingClass(const SchedulingClassDescriptor &sched_cls); + + SchedulingClass sched_cls_id_ = 0; + std::shared_ptr required_resources_; + std::shared_ptr required_placement_resources_; + std::shared_ptr label_selector_; + + std::vector dependencies_; + + int runtime_env_hash_ = 0; +}; + +} // namespace ray diff --git a/src/ray/common/task/task.cc b/src/ray/common/task/task.cc deleted file mode 100644 index 28d28d6a8a12..000000000000 --- a/src/ray/common/task/task.cc +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2019-2020 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ray/common/task/task.h" - -#include -#include -#include - -#include "absl/strings/str_format.h" - -namespace ray { - -RayTask::RayTask(rpc::TaskSpec task_spec) : task_spec_(std::move(task_spec)) { - ComputeDependencies(); -} - -RayTask::RayTask(rpc::Task message) - : task_spec_(std::move(*message.mutable_task_spec())) { - ComputeDependencies(); -} - -RayTask::RayTask(TaskSpecification task_spec) : task_spec_(std::move(task_spec)) { - ComputeDependencies(); -} - -RayTask::RayTask(TaskSpecification task_spec, std::string preferred_node_id) - : task_spec_(std::move(task_spec)), preferred_node_id_(std::move(preferred_node_id)) { - ComputeDependencies(); -} - -const TaskSpecification &RayTask::GetTaskSpecification() const { return task_spec_; } - -const std::vector &RayTask::GetDependencies() const { - return dependencies_; -} - -const std::string &RayTask::GetPreferredNodeID() const { return preferred_node_id_; } - -void RayTask::ComputeDependencies() { dependencies_ = task_spec_.GetDependencies(); } - -std::string RayTask::DebugString() const { - return absl::StrFormat("task_spec={%s}", task_spec_.DebugString()); -} - -} // namespace ray diff --git a/src/ray/common/task/task.h b/src/ray/common/task/task.h deleted file mode 100644 index 9408990fb1de..000000000000 --- a/src/ray/common/task/task.h +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2019-2020 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include -#include - -#include "ray/common/task/task_common.h" -#include "ray/common/task/task_spec.h" - -namespace ray { - -/// \class RayTask -/// -/// A RayTask represents a Ray task and a specification of its execution (e.g., -/// resource demands). The task's specification contains both immutable fields, -/// determined at submission time, and mutable fields, determined at execution -/// time. -class RayTask { - public: - /// Construct an empty task. This should only be used to pass a task - /// as an out parameter to a function or method. - RayTask() = default; - - /// Construct a `RayTask` object from a protobuf message. - explicit RayTask(rpc::TaskSpec task_spec); - - /// Construct a `RayTask` object from a protobuf message. - /// - /// \param message The protobuf message. - explicit RayTask(rpc::Task message); - - /// Construct a `RayTask` object from a `TaskSpecification`. - explicit RayTask(TaskSpecification task_spec); - - RayTask(TaskSpecification task_spec, std::string preferred_node_id); - - /// Get the immutable specification for the task. - /// - /// \return The immutable specification for the task. - const TaskSpecification &GetTaskSpecification() const; - - /// Get the task's object dependencies. This comprises the immutable task - /// arguments and the mutable execution dependencies. - /// - /// \return The object dependencies. - const std::vector &GetDependencies() const; - - /// Get the task's preferred node id for scheduling. If the returned value - /// is empty, then it means the task has no preferred node. - /// - /// \return The preferred node id. - const std::string &GetPreferredNodeID() const; - - std::string DebugString() const; - - private: - void ComputeDependencies(); - - /// RayTask specification object, consisting of immutable information about this - /// task determined at submission time. Includes resource demand, object - /// dependencies, etc. - TaskSpecification task_spec_; - /// A cached copy of the task's object dependencies, including arguments from - /// the TaskSpecification. - std::vector dependencies_; - - std::string preferred_node_id_; -}; - -} // namespace ray diff --git a/src/ray/common/task/task_spec.cc b/src/ray/common/task/task_spec.cc index d231470f174a..dd4ea7cf7e93 100644 --- a/src/ray/common/task/task_spec.cc +++ b/src/ray/common/task/task_spec.cc @@ -99,7 +99,8 @@ void TaskSpecification::ComputeResources() { // A static nil object is used here to avoid allocating the empty object every time. required_resources_ = ResourceSet::Nil(); } else { - required_resources_.reset(new ResourceSet(MapFromProtobuf(required_resources))); + required_resources_ = + std::make_shared(MapFromProtobuf(required_resources)); } auto &required_placement_resources = message_->required_placement_resources().empty() @@ -109,8 +110,8 @@ void TaskSpecification::ComputeResources() { if (required_placement_resources.empty()) { required_placement_resources_ = ResourceSet::Nil(); } else { - required_placement_resources_.reset( - new ResourceSet(MapFromProtobuf(required_placement_resources))); + required_placement_resources_ = + std::make_shared(MapFromProtobuf(required_placement_resources)); } // Set LabelSelector required for scheduling if specified. Parses string map @@ -166,12 +167,7 @@ const std::string TaskSpecification::GetSerializedActorHandle() const { return message_->actor_creation_task_spec().serialized_actor_handle(); } -JobID TaskSpecification::JobId() const { - if (message_->job_id().empty() /* e.g., empty proto default */) { - return JobID::Nil(); - } - return JobID::FromBinary(message_->job_id()); -} +JobID TaskSpecification::JobId() const { return JobID::FromBinary(message_->job_id()); } const rpc::JobConfig &TaskSpecification::JobConfig() const { return message_->job_config(); diff --git a/src/ray/common/task/task_spec.h b/src/ray/common/task/task_spec.h index d23fd5bff286..18ad83f80a44 100644 --- a/src/ray/common/task/task_spec.h +++ b/src/ray/common/task/task_spec.h @@ -283,12 +283,7 @@ class TaskSpecification : public MessageWrapper { /// The input message will be copied/moved into this object. /// /// \param message The protobuf message. - explicit TaskSpecification(rpc::TaskSpec &&message) - : MessageWrapper(std::move(message)) { - ComputeResources(); - } - - explicit TaskSpecification(const rpc::TaskSpec &message) : MessageWrapper(message) { + explicit TaskSpecification(rpc::TaskSpec message) : MessageWrapper(std::move(message)) { ComputeResources(); } diff --git a/src/ray/common/tests/BUILD.bazel b/src/ray/common/tests/BUILD.bazel index 76b56ffbb30d..4822452c0d8b 100644 --- a/src/ray/common/tests/BUILD.bazel +++ b/src/ray/common/tests/BUILD.bazel @@ -112,7 +112,6 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/ray/common:id", - "//src/ray/common:task_common", "//src/ray/protobuf:common_cc_proto", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/common/tests/id_test.cc b/src/ray/common/tests/id_test.cc index 1264dcf0db30..eee4b98e5b5b 100644 --- a/src/ray/common/tests/id_test.cc +++ b/src/ray/common/tests/id_test.cc @@ -18,7 +18,6 @@ #include "absl/container/flat_hash_set.h" #include "ray/common/common_protocol.h" -#include "ray/common/task/task_spec.h" namespace ray { @@ -176,4 +175,31 @@ TEST(PlacementGroupIDTest, TestPlacementGroup) { } } +TEST(LeaseIDTest, TestLeaseID) { + // Test basic LeaseID creation, size, and worker extraction + const WorkerID worker_id = WorkerID::FromRandom(); + const LeaseID lease_id = LeaseID::FromWorker(worker_id, 2); + const size_t lease_id_size = 32; + ASSERT_FALSE(lease_id.IsNil()); + ASSERT_EQ(lease_id.WorkerId(), worker_id); + ASSERT_EQ(LeaseID::Size(), lease_id_size); + ASSERT_EQ(lease_id.Binary().size(), lease_id_size); + + const LeaseID random_lease = LeaseID::FromRandom(); + const LeaseID another_lease = LeaseID::FromWorker(worker_id, 1); + + ASSERT_FALSE(random_lease.IsNil()); + ASSERT_NE(lease_id, another_lease); + ASSERT_NE(lease_id, random_lease); + ASSERT_EQ(lease_id.WorkerId(), another_lease.WorkerId()); + + // Test serialization roundtrip + const LeaseID from_hex = LeaseID::FromHex(lease_id.Hex()); + const LeaseID from_binary = LeaseID::FromBinary(lease_id.Binary()); + + ASSERT_EQ(lease_id, from_hex); + ASSERT_EQ(lease_id, from_binary); + ASSERT_EQ(lease_id.WorkerId(), from_hex.WorkerId()); +} + } // namespace ray diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 5b6ff9734078..5a6c7c1963c3 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -190,7 +190,7 @@ ray_cc_library( visibility = [":__subpackages__"], deps = [ "//src/ray/common:id", - "//src/ray/common:task_common", + "//src/ray/common:lease", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", diff --git a/src/ray/core_worker/lease_policy.cc b/src/ray/core_worker/lease_policy.cc index 10e4471cf39c..f1efd15e951c 100644 --- a/src/ray/core_worker/lease_policy.cc +++ b/src/ray/core_worker/lease_policy.cc @@ -21,8 +21,8 @@ namespace ray { namespace core { -std::pair LocalityAwareLeasePolicy::GetBestNodeForTask( - const TaskSpecification &spec) { +std::pair LocalityAwareLeasePolicy::GetBestNodeForLease( + const LeaseSpecification &spec) { if (spec.GetMessage().scheduling_strategy().scheduling_strategy_case() == rpc::SchedulingStrategy::SchedulingStrategyCase::kSpreadSchedulingStrategy) { // The explicit spread scheduling strategy @@ -40,7 +40,7 @@ std::pair LocalityAwareLeasePolicy::GetBestNodeForTask( } // Pick node based on locality. - if (auto node_id = GetBestNodeIdForTask(spec)) { + if (auto node_id = GetBestNodeIdForLease(spec)) { if (auto addr = node_addr_factory_(node_id.value())) { return std::make_pair(addr.value(), true); } @@ -49,8 +49,8 @@ std::pair LocalityAwareLeasePolicy::GetBestNodeForTask( } /// Criteria for "best" node: The node with the most object bytes (from object_ids) local. -std::optional LocalityAwareLeasePolicy::GetBestNodeIdForTask( - const TaskSpecification &spec) { +std::optional LocalityAwareLeasePolicy::GetBestNodeIdForLease( + const LeaseSpecification &spec) { const auto object_ids = spec.GetDependencyIds(); // Number of object bytes (from object_ids) that a given node has local. absl::flat_hash_map bytes_local_table; @@ -76,8 +76,8 @@ std::optional LocalityAwareLeasePolicy::GetBestNodeIdForTask( return max_bytes_node; } -std::pair LocalLeasePolicy::GetBestNodeForTask( - const TaskSpecification &spec) { +std::pair LocalLeasePolicy::GetBestNodeForLease( + const LeaseSpecification &spec) { // Always return the local node. return std::make_pair(local_node_rpc_address_, false); } diff --git a/src/ray/core_worker/lease_policy.h b/src/ray/core_worker/lease_policy.h index 78c927802987..78ae5d4aefd6 100644 --- a/src/ray/core_worker/lease_policy.h +++ b/src/ray/core_worker/lease_policy.h @@ -18,7 +18,7 @@ #include "absl/container/flat_hash_set.h" #include "ray/common/id.h" -#include "ray/common/task/task_spec.h" +#include "ray/common/lease/lease_spec.h" #include "src/ray/protobuf/common.pb.h" namespace ray { @@ -41,9 +41,9 @@ class LocalityDataProviderInterface { /// Interface for mocking the lease policy. class LeasePolicyInterface { public: - /// Get the address of the best worker node for a lease request for the provided task. - virtual std::pair GetBestNodeForTask( - const TaskSpecification &spec) = 0; + /// Get the address of the best worker node for a lease request. + virtual std::pair GetBestNodeForLease( + const LeaseSpecification &spec) = 0; virtual ~LeasePolicyInterface() = default; }; @@ -63,13 +63,13 @@ class LocalityAwareLeasePolicy : public LeasePolicyInterface { ~LocalityAwareLeasePolicy() override = default; - /// Get the address of the best worker node for a lease request for the provided task. - std::pair GetBestNodeForTask( - const TaskSpecification &spec) override; + /// Get the address of the best worker node for a lease request. + std::pair GetBestNodeForLease( + const LeaseSpecification &spec) override; private: - /// Get the best worker node for a lease request for the provided task. - std::optional GetBestNodeIdForTask(const TaskSpecification &spec); + /// Get the best worker node for a lease request. + std::optional GetBestNodeIdForLease(const LeaseSpecification &spec); /// Provider of locality data that will be used in choosing the best lessor. LocalityDataProviderInterface &locality_data_provider_; @@ -90,9 +90,9 @@ class LocalLeasePolicy : public LeasePolicyInterface { ~LocalLeasePolicy() override = default; - /// Get the address of the local node for a lease request for the provided task. - std::pair GetBestNodeForTask( - const TaskSpecification &spec) override; + /// Get the address of the local node for a lease request. + std::pair GetBestNodeForLease( + const LeaseSpecification &spec) override; private: /// RPC address of the local node. diff --git a/src/ray/core_worker/task_manager_interface.h b/src/ray/core_worker/task_manager_interface.h index 9ba0260b40bd..34e04140984d 100644 --- a/src/ray/core_worker/task_manager_interface.h +++ b/src/ray/core_worker/task_manager_interface.h @@ -19,9 +19,9 @@ #include "absl/types/optional.h" #include "ray/common/id.h" +#include "ray/common/lease/lease.h" #include "ray/common/scheduling/scheduling_ids.h" #include "ray/common/status.h" -#include "ray/common/task/task.h" #include "ray/common/task/task_spec.h" #include "src/ray/protobuf/common.pb.h" #include "src/ray/protobuf/core_worker.pb.h" diff --git a/src/ray/core_worker/task_submission/BUILD.bazel b/src/ray/core_worker/task_submission/BUILD.bazel index 6cd702f312f6..09362fe45c2d 100644 --- a/src/ray/core_worker/task_submission/BUILD.bazel +++ b/src/ray/core_worker/task_submission/BUILD.bazel @@ -88,6 +88,7 @@ ray_cc_library( deps = [ ":dependency_resolver", "//src/ray/common:id", + "//src/ray/common:lease", "//src/ray/core_worker:lease_policy", "//src/ray/core_worker:memory_store", "//src/ray/core_worker:task_manager", diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 773852cd3837..0b7ba9e4c781 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -21,6 +21,7 @@ #include #include +#include "ray/common/lease/lease_spec.h" #include "ray/gcs/pb_util.h" namespace ray { @@ -62,9 +63,15 @@ Status NormalTaskSubmitter::SubmitTask(TaskSpecification task_spec) { const SchedulingKey scheduling_key(task_spec.GetSchedulingClass(), task_spec.GetDependencyIds(), task_spec.GetRuntimeEnvHash()); - auto &scheduling_key_entry = scheduling_key_entries_[scheduling_key]; - scheduling_key_entry.task_queue.push_back(task_spec); - scheduling_key_entry.resource_spec = std::move(task_spec); + auto [scheduler_key_entry_iter, new_scheduling_key_entry] = + scheduling_key_entries_.try_emplace(scheduling_key, SchedulingKeyEntry{}); + auto &scheduling_key_entry = scheduler_key_entry_iter->second; + + // Only set lease_spec if this is a new scheduling key entry + if (new_scheduling_key_entry) { + scheduling_key_entry.lease_spec = LeaseSpecification(task_spec.GetMessage()); + } + scheduling_key_entry.task_queue.push_back(std::move(task_spec)); if (!scheduling_key_entry.AllWorkersBusy()) { // There are idle workers, so we don't need more @@ -94,11 +101,11 @@ void NormalTaskSubmitter::AddWorkerLeaseClient( std::shared_ptr raylet_client, const google::protobuf::RepeatedPtrField &assigned_resources, const SchedulingKey &scheduling_key, - const TaskID &task_id) { + const LeaseID &lease_id) { core_worker_client_pool_->GetOrConnect(addr); int64_t expiration = current_time_ms() + lease_timeout_ms_; LeaseEntry new_lease_entry{ - std::move(raylet_client), expiration, assigned_resources, scheduling_key, task_id}; + std::move(raylet_client), expiration, assigned_resources, scheduling_key, lease_id}; worker_to_lease_entry_.emplace(addr, new_lease_entry); auto &scheduling_key_entry = scheduling_key_entries_[scheduling_key]; @@ -106,11 +113,11 @@ void NormalTaskSubmitter::AddWorkerLeaseClient( RAY_CHECK(scheduling_key_entry.active_workers.size() >= 1); } -void NormalTaskSubmitter::ReturnWorker(const rpc::Address &addr, - bool was_error, - const std::string &error_detail, - bool worker_exiting, - const SchedulingKey &scheduling_key) { +void NormalTaskSubmitter::ReturnWorkerLease(const rpc::Address &addr, + bool was_error, + const std::string &error_detail, + bool worker_exiting, + const SchedulingKey &scheduling_key) { RAY_LOG(DEBUG) << "Returning worker " << WorkerID::FromBinary(addr.worker_id()) << " to raylet " << NodeID::FromBinary(addr.node_id()); auto &scheduling_key_entry = scheduling_key_entries_[scheduling_key]; @@ -129,11 +136,11 @@ void NormalTaskSubmitter::ReturnWorker(const rpc::Address &addr, } auto status = - lease_entry.raylet_client->ReturnWorker(addr.port(), - WorkerID::FromBinary(addr.worker_id()), - was_error, - error_detail, - worker_exiting); + lease_entry.raylet_client->ReturnWorkerLease(addr.port(), + WorkerID::FromBinary(addr.worker_id()), + was_error, + error_detail, + worker_exiting); if (!status.ok()) { RAY_LOG(ERROR) << "Error returning worker to raylet: " << status.ToString(); } @@ -164,7 +171,7 @@ void NormalTaskSubmitter::OnWorkerIdle( // Return the worker only if there are no tasks to do. if (!lease_entry.is_busy) { - ReturnWorker(addr, was_error, error_detail, worker_exiting, scheduling_key); + ReturnWorkerLease(addr, was_error, error_detail, worker_exiting, scheduling_key); } } else { auto client = core_worker_client_pool_->GetOrConnect(addr); @@ -207,10 +214,10 @@ void NormalTaskSubmitter::CancelWorkerLeaseIfNeeded(const SchedulingKey &schedul // There is an in-flight lease request. Cancel it. auto raylet_client = raylet_client_pool_->GetOrConnectByAddress(pending_lease_request.second); - auto &task_id = pending_lease_request.first; - RAY_LOG(DEBUG) << "Canceling lease request " << task_id; + const auto &lease_id = pending_lease_request.first; + RAY_LOG(DEBUG) << "Canceling lease request " << lease_id; raylet_client->CancelWorkerLease( - task_id, + lease_id, [this, scheduling_key](const Status &status, const rpc::CancelWorkerLeaseReply &reply) { absl::MutexLock lock(&mu_); @@ -235,11 +242,11 @@ void NormalTaskSubmitter::ReportWorkerBacklog() { } void NormalTaskSubmitter::ReportWorkerBacklogInternal() { - absl::flat_hash_map> backlogs; + absl::flat_hash_map> backlogs; for (auto &scheduling_key_and_entry : scheduling_key_entries_) { const SchedulingClass scheduling_class = std::get<0>(scheduling_key_and_entry.first); if (backlogs.find(scheduling_class) == backlogs.end()) { - backlogs[scheduling_class].first = scheduling_key_and_entry.second.resource_spec; + backlogs[scheduling_class].first = scheduling_key_and_entry.second.lease_spec; backlogs[scheduling_class].second = 0; } // We report backlog size per scheduling class not per scheduling key @@ -253,7 +260,7 @@ void NormalTaskSubmitter::ReportWorkerBacklogInternal() { std::vector backlog_reports; for (const auto &backlog : backlogs) { rpc::WorkerBacklogReport backlog_report; - backlog_report.mutable_resource_spec()->CopyFrom(backlog.second.first.GetMessage()); + backlog_report.mutable_lease_spec()->CopyFrom(backlog.second.first.GetMessage()); backlog_report.set_backlog_size(backlog.second.second); backlog_reports.emplace_back(backlog_report); } @@ -302,36 +309,35 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli // All tasks have corresponding pending leases, no need to request more return; } - - // Create a TaskSpecification with an overwritten TaskID to make sure we don't reuse the - // same TaskID to request a worker - auto resource_spec_msg = scheduling_key_entry.resource_spec.GetMutableMessage(); - resource_spec_msg.set_task_id(TaskID::FromRandom(job_id_).Binary()); - const TaskSpecification resource_spec = TaskSpecification(std::move(resource_spec_msg)); + // Counter for generating unique lease IDs. + static uint32_t lease_id_counter = 1; + const LeaseID lease_id = LeaseID::FromWorker(worker_id_, lease_id_counter++); + rpc::LeaseSpec lease_spec_msg = scheduling_key_entry.lease_spec.GetMessage(); + lease_spec_msg.set_lease_id(lease_id.Binary()); + const LeaseSpecification lease_spec = LeaseSpecification(std::move(lease_spec_msg)); rpc::Address best_node_address; const bool is_spillback = (raylet_address != nullptr); bool is_selected_based_on_locality = false; if (raylet_address == nullptr) { // If no raylet address is given, find the best worker for our next lease request. std::tie(best_node_address, is_selected_based_on_locality) = - lease_policy_->GetBestNodeForTask(resource_spec); + lease_policy_->GetBestNodeForLease(lease_spec); raylet_address = &best_node_address; } auto raylet_client = raylet_client_pool_->GetOrConnectByAddress(*raylet_address); - const TaskID task_id = resource_spec.TaskId(); - const std::string task_name = resource_spec.GetName(); - RAY_LOG(DEBUG) << "Requesting lease from raylet " - << NodeID::FromBinary(raylet_address->node_id()) << " for task " - << task_id; + const std::string function_or_actor_name = lease_spec.GetFunctionOrActorName(); + RAY_LOG(DEBUG) << "Requesting lease " << lease_id << " from raylet " + << NodeID::FromBinary(raylet_address->node_id()) << " for " + << function_or_actor_name; raylet_client->RequestWorkerLease( - resource_spec.GetMessage(), + lease_spec.GetMessage(), /*grant_or_reject=*/is_spillback, [this, scheduling_key, - task_id, - task_name, + lease_id, + function_or_actor_name, is_spillback, raylet_address = *raylet_address](const Status &status, const rpc::RequestWorkerLeaseReply &reply) { @@ -345,12 +351,11 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli auto &sched_entry = scheduling_key_entries_[scheduling_key]; auto raylet_lease_client = raylet_client_pool_->GetOrConnectByAddress(raylet_address); - sched_entry.pending_lease_requests.erase(task_id); + sched_entry.pending_lease_requests.erase(lease_id); if (status.ok()) { if (reply.canceled()) { - RAY_LOG(DEBUG) << "Lease canceled for task: " << task_id - << ", canceled type: " + RAY_LOG(DEBUG) << "Lease canceled for: " << lease_id << ", canceled type: " << rpc::RequestWorkerLeaseReply::SchedulingFailureType_Name( reply.failure_type()); if (reply.failure_type() == @@ -383,10 +388,10 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli } error_info.set_error_message( absl::StrCat(reply.scheduling_failure_message(), - " task_id=", - task_id.Hex(), - ", task_name=", - task_name)); + " lease_id=", + lease_id.Hex(), + ", name=", + function_or_actor_name)); tasks_to_fail = std::move(sched_entry.task_queue); sched_entry.task_queue.clear(); @@ -397,7 +402,7 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli RequestNewWorkerIfNeeded(scheduling_key); } } else if (reply.rejected()) { - RAY_LOG(DEBUG) << "Lease rejected " << task_id; + RAY_LOG(DEBUG) << "Lease rejected " << lease_id; // It might happen when the first raylet has a stale view // of the spillback raylet resources. // Retry the request at the first raylet since the resource view may be @@ -407,7 +412,7 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli } else if (!reply.worker_address().node_id().empty()) { // We got a lease for a worker. Add the lease client state and try to // assign work to the worker. - RAY_LOG(DEBUG) << "Lease granted to task " << task_id << " from raylet " + RAY_LOG(DEBUG) << "Lease granted to task " << lease_id << " from raylet " << NodeID::FromBinary(reply.worker_address().node_id()) << " with worker " << WorkerID::FromBinary(reply.worker_address().worker_id()); @@ -416,7 +421,7 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli std::move(raylet_lease_client), reply.resource_mapping(), scheduling_key, - task_id); + lease_id); RAY_CHECK(sched_entry.active_workers.size() >= 1); OnWorkerIdle(reply.worker_address(), scheduling_key, @@ -427,11 +432,12 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli } else { // The raylet redirected us to a different raylet to retry at. RAY_CHECK(!is_spillback); - RAY_LOG(DEBUG) << "Redirect lease for task " << task_id << " from raylet " + RAY_LOG(DEBUG) << "Redirect lease " << lease_id << " from raylet " << NodeID::FromBinary(raylet_address.node_id()) << " to raylet " << NodeID::FromBinary( - reply.retry_at_raylet_address().node_id()); + reply.retry_at_raylet_address().node_id()) + << " for " << function_or_actor_name; RequestNewWorkerIfNeeded(scheduling_key, &reply.retry_at_raylet_address()); } @@ -440,8 +446,8 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli // still needed. // TODO(swang): Fail after some number of retries? RAY_LOG_EVERY_MS(INFO, 30 * 1000) - << "Retrying attempt to schedule task (id: " << task_id - << " name: " << task_name + << "Retrying attempt to schedule lease (id: " << lease_id + << " name: " << function_or_actor_name << ") at remote node (id: " << raylet_address.node_id() << " ip: " << raylet_address.ip_address() << "). Try again " @@ -499,7 +505,7 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli }, task_queue.size(), is_selected_based_on_locality); - scheduling_key_entry.pending_lease_requests.emplace(task_id, *raylet_address); + scheduling_key_entry.pending_lease_requests.emplace(lease_id, *raylet_address); ReportWorkerBacklogIfNeeded(scheduling_key); // Lease more workers if there are still pending tasks and @@ -565,16 +571,17 @@ void NormalTaskSubmitter::PushNormalTask( if (!status.ok()) { failed_tasks_pending_failure_cause_.insert(task_id); RAY_LOG(DEBUG) << "Getting error from raylet for task " << task_id; - const ray::rpc::ClientCallback callback = - [this, status, task_id, addr]( - const Status &get_task_failure_cause_reply_status, - const rpc::GetTaskFailureCauseReply &get_task_failure_cause_reply) { + const ray::rpc::ClientCallback + callback = [this, status, task_id, addr]( + const Status &get_task_failure_cause_reply_status, + const rpc::GetWorkerFailureCauseReply + &get_task_failure_cause_reply) { bool will_retry = - HandleGetTaskFailureCause(status, - task_id, - addr, - get_task_failure_cause_reply_status, - get_task_failure_cause_reply); + HandleGetWorkerFailureCause(status, + task_id, + addr, + get_task_failure_cause_reply_status, + get_task_failure_cause_reply); absl::MutexLock task_submission_state_lock(&mu_); if (!will_retry) { // Task submission and task cancellation are the only two other code @@ -587,8 +594,8 @@ void NormalTaskSubmitter::PushNormalTask( }; auto &cur_lease_entry = worker_to_lease_entry_[addr]; RAY_CHECK(cur_lease_entry.raylet_client); - cur_lease_entry.raylet_client->GetTaskFailureCause(cur_lease_entry.task_id, - callback); + cur_lease_entry.raylet_client->GetWorkerFailureCause(cur_lease_entry.lease_id, + callback); } OnWorkerIdle(addr, scheduling_key, @@ -619,31 +626,32 @@ void NormalTaskSubmitter::PushNormalTask( }); } -bool NormalTaskSubmitter::HandleGetTaskFailureCause( +bool NormalTaskSubmitter::HandleGetWorkerFailureCause( const Status &task_execution_status, const TaskID &task_id, const rpc::Address &addr, - const Status &get_task_failure_cause_reply_status, - const rpc::GetTaskFailureCauseReply &get_task_failure_cause_reply) { + const Status &get_worker_failure_cause_reply_status, + const rpc::GetWorkerFailureCauseReply &get_worker_failure_cause_reply) { rpc::ErrorType task_error_type = rpc::ErrorType::WORKER_DIED; std::unique_ptr error_info; bool fail_immediately = false; - if (get_task_failure_cause_reply_status.ok()) { - RAY_LOG(WARNING) << "Task failure cause for task " << task_id << ": " + if (get_worker_failure_cause_reply_status.ok()) { + RAY_LOG(WARNING) << "Worker failure cause for task " << task_id << ": " << ray::gcs::RayErrorInfoToString( - get_task_failure_cause_reply.failure_cause()) + get_worker_failure_cause_reply.failure_cause()) << " fail immedediately: " - << get_task_failure_cause_reply.fail_task_immediately(); - if (get_task_failure_cause_reply.has_failure_cause()) { - task_error_type = get_task_failure_cause_reply.failure_cause().error_type(); + << get_worker_failure_cause_reply.fail_task_immediately(); + if (get_worker_failure_cause_reply.has_failure_cause()) { + task_error_type = get_worker_failure_cause_reply.failure_cause().error_type(); error_info = std::make_unique( - get_task_failure_cause_reply.failure_cause()); + get_worker_failure_cause_reply.failure_cause()); // TODO(clarng): track and append task retry history to the error message. } - fail_immediately = get_task_failure_cause_reply.fail_task_immediately(); + fail_immediately = get_worker_failure_cause_reply.fail_task_immediately(); } else { - RAY_LOG(WARNING) << "Failed to fetch task result with status " - << get_task_failure_cause_reply_status.ToString() + RAY_LOG(WARNING) << "Failed to fetch worker failure cause with status " + << get_worker_failure_cause_reply_status.ToString() + << " worker id: " << WorkerID::FromBinary(addr.worker_id()) << " node id: " << NodeID::FromBinary(addr.node_id()) << " ip: " << addr.ip_address(); task_error_type = rpc::ErrorType::NODE_DIED; diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.h b/src/ray/core_worker/task_submission/normal_task_submitter.h index 1ff97333e5d7..856c322cfd23 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.h +++ b/src/ray/core_worker/task_submission/normal_task_submitter.h @@ -201,7 +201,7 @@ class NormalTaskSubmitter { std::shared_ptr raylet_client, const google::protobuf::RepeatedPtrField &assigned_resources, const SchedulingKey &scheduling_key, - const TaskID &task_id) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + const LeaseID &lease_id) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); /// This function takes care of returning a worker to the Raylet. /// \param[in] addr The address of the worker. @@ -209,11 +209,11 @@ class NormalTaskSubmitter { /// \param[in] error_detail The reason why it was errored. /// it is unused if was_error is false. /// \param[in] worker_exiting Whether the worker is exiting. - void ReturnWorker(const rpc::Address &addr, - bool was_error, - const std::string &error_detail, - bool worker_exiting, - const SchedulingKey &scheduling_key) + void ReturnWorkerLease(const rpc::Address &addr, + bool was_error, + const std::string &error_detail, + bool worker_exiting, + const SchedulingKey &scheduling_key) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); /// Check that the scheduling_key_entries_ hashmap is empty. @@ -229,14 +229,14 @@ class NormalTaskSubmitter { const google::protobuf::RepeatedPtrField &assigned_resources); - /// Handles result from GetTaskFailureCause. - /// \return true if the task should be retried, false otherwise. - bool HandleGetTaskFailureCause( + /// Handles result from GetWorkerFailureCause. + /// \return true if the task executing on the worker should be retried, false otherwise. + bool HandleGetWorkerFailureCause( const Status &task_execution_status, const TaskID &task_id, const rpc::Address &addr, - const Status &get_task_failure_cause_reply_status, - const rpc::GetTaskFailureCauseReply &get_task_failure_cause_reply); + const Status &get_worker_failure_cause_reply_status, + const rpc::GetWorkerFailureCauseReply &get_worker_failure_cause_reply); /// Address of our RPC server. rpc::Address rpc_address_; @@ -291,7 +291,7 @@ class NormalTaskSubmitter { int64_t lease_expiration_time; google::protobuf::RepeatedPtrField assigned_resources; SchedulingKey scheduling_key; - TaskID task_id; + LeaseID lease_id; bool is_busy = false; }; @@ -301,8 +301,9 @@ class NormalTaskSubmitter { struct SchedulingKeyEntry { // Keep track of pending worker lease requests to the raylet. - absl::flat_hash_map pending_lease_requests; - TaskSpecification resource_spec; + absl::flat_hash_map pending_lease_requests; + + LeaseSpecification lease_spec; // Tasks that are queued for execution. We keep an individual queue per // scheduling class to ensure fairness. std::deque task_queue; diff --git a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc index 5798da1df518..a3af7a15b838 100644 --- a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc @@ -223,11 +223,11 @@ class MockTaskManager : public MockTaskManagerInterface { class MockRayletClient : public FakeRayletClient { public: - Status ReturnWorker(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) override { + Status ReturnWorkerLease(int worker_port, + const WorkerID &worker_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) override { std::lock_guard lock(mu_); if (disconnect_worker) { num_workers_disconnected++; @@ -240,22 +240,22 @@ class MockRayletClient : public FakeRayletClient { return Status::OK(); } - void GetTaskFailureCause( - const TaskID &task_id, - const ray::rpc::ClientCallback &callback) + void GetWorkerFailureCause( + const LeaseID &lease_id, + const ray::rpc::ClientCallback &callback) override { std::lock_guard lock(mu_); get_task_failure_cause_callbacks.push_back(callback); num_get_task_failure_causes += 1; } - bool ReplyGetTaskFailureCause() { + bool ReplyGetWorkerFailureCause() { if (get_task_failure_cause_callbacks.size() == 0) { return false; } auto callback = std::move(get_task_failure_cause_callbacks.front()); get_task_failure_cause_callbacks.pop_front(); - rpc::GetTaskFailureCauseReply reply; + rpc::GetWorkerFailureCauseReply reply; callback(Status::OK(), std::move(reply)); return true; } @@ -268,14 +268,14 @@ class MockRayletClient : public FakeRayletClient { reported_backlogs.clear(); for (const auto &backlog_report : backlog_reports) { reported_backlog_size += backlog_report.backlog_size(); - const TaskSpecification resource_spec(backlog_report.resource_spec()); - const SchedulingClass scheduling_class = resource_spec.GetSchedulingClass(); + const LeaseSpecification lease_spec(backlog_report.lease_spec()); + const SchedulingClass scheduling_class = lease_spec.GetSchedulingClass(); reported_backlogs[scheduling_class] = backlog_report.backlog_size(); } } void RequestWorkerLease( - const rpc::TaskSpec &task_spec, + const rpc::LeaseSpec &lease_spec, bool grant_or_reject, const ray::rpc::ClientCallback &callback, const int64_t backlog_size, @@ -301,7 +301,7 @@ class MockRayletClient : public FakeRayletClient { } void CancelWorkerLease( - const TaskID &task_id, + const LeaseID &lease_id, const rpc::ClientCallback &callback) override { std::lock_guard lock(mu_); num_leases_canceled += 1; @@ -400,7 +400,7 @@ class MockRayletClient : public FakeRayletClient { std::map reported_backlogs; std::list> callbacks = {}; std::list> cancel_callbacks = {}; - std::list> + std::list> get_task_failure_cause_callbacks = {}; }; @@ -447,7 +447,7 @@ class MockLeasePolicy : public LeasePolicyInterface { public: void SetNodeID(NodeID node_id) { fallback_rpc_address_.set_node_id(node_id.Binary()); } - std::pair GetBestNodeForTask(const TaskSpecification &spec) { + std::pair GetBestNodeForLease(const LeaseSpecification &spec) { num_lease_policy_consults++; return std::make_pair(fallback_rpc_address_, is_locality_aware); }; @@ -660,7 +660,7 @@ TEST_F(NormalTaskSubmitterTest, TestHandleTaskFailure) { ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); // Simulate a system failure, i.e., worker died unexpectedly. ASSERT_TRUE(worker_client->ReplyPushTask(Status::IOError("oops"))); - ASSERT_TRUE(raylet_client->ReplyGetTaskFailureCause()); + ASSERT_TRUE(raylet_client->ReplyGetWorkerFailureCause()); ASSERT_EQ(worker_client->callbacks.size(), 0); ASSERT_EQ(raylet_client->num_workers_returned, 0); ASSERT_EQ(raylet_client->num_workers_disconnected, 1); @@ -677,7 +677,7 @@ TEST_F(NormalTaskSubmitterTest, TestHandleTaskFailure) { TEST_F(NormalTaskSubmitterTest, TestCancellationWhileHandlingTaskFailure) { // This test is a regression test for a bug where a crash happens when - // the task cancellation races between ReplyPushTask and ReplyGetTaskFailureCause. + // the task cancellation races between ReplyPushTask and ReplyGetWorkerFailureCause. // For an example of a python integration test, see // https://github.com/ray-project/ray/blob/2b6807f4d9c4572e6309f57bc404aa641bc4b185/python/ray/tests/test_cancel.py#L35 auto submitter = @@ -687,13 +687,13 @@ TEST_F(NormalTaskSubmitterTest, TestCancellationWhileHandlingTaskFailure) { ASSERT_TRUE(submitter.SubmitTask(task).ok()); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); // Simulate a system failure, i.e., worker died unexpectedly so that - // GetTaskFailureCause is called. + // GetWorkerFailureCause is called. ASSERT_TRUE(worker_client->ReplyPushTask(Status::IOError("oops"))); - // Cancel the task while GetTaskFailureCause has not been completed. + // Cancel the task while GetWorkerFailureCause has not been completed. ASSERT_TRUE(submitter.CancelTask(task, true, false).ok()); - // Completing the GetTaskFailureCause call. Check that the reply runs without error + // Completing the GetWorkerFailureCause call. Check that the reply runs without error // and FailPendingTask is not called. - ASSERT_TRUE(raylet_client->ReplyGetTaskFailureCause()); + ASSERT_TRUE(raylet_client->ReplyGetWorkerFailureCause()); ASSERT_EQ(task_manager->num_fail_pending_task_calls, 0); } @@ -1278,7 +1278,7 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerNotReusedOnError) { // Task 1 finishes with failure; the worker is returned. ASSERT_TRUE(worker_client->ReplyPushTask(Status::IOError("worker dead"))); - ASSERT_TRUE(raylet_client->ReplyGetTaskFailureCause()); + ASSERT_TRUE(raylet_client->ReplyGetWorkerFailureCause()); ASSERT_EQ(worker_client->callbacks.size(), 0); ASSERT_EQ(raylet_client->num_workers_returned, 0); ASSERT_EQ(raylet_client->num_workers_disconnected, 1); @@ -1701,7 +1701,7 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerLeaseTimeout) { // Task 1 finishes with failure; the worker is returned due to the error even though // it hasn't timed out. ASSERT_TRUE(worker_client->ReplyPushTask(Status::IOError("worker dead"))); - ASSERT_TRUE(raylet_client->ReplyGetTaskFailureCause()); + ASSERT_TRUE(raylet_client->ReplyGetWorkerFailureCause()); ASSERT_EQ(raylet_client->num_workers_returned, 0); ASSERT_EQ(raylet_client->num_workers_disconnected, 1); @@ -1741,7 +1741,7 @@ TEST_F(NormalTaskSubmitterTest, TestKillExecutingTask) { ASSERT_TRUE(submitter.CancelTask(task, true, false).ok()); ASSERT_EQ(worker_client->kill_requests.front().intended_task_id(), task.TaskIdBinary()); ASSERT_TRUE(worker_client->ReplyPushTask(Status::IOError("workerdying"), true)); - ASSERT_TRUE(raylet_client->ReplyGetTaskFailureCause()); + ASSERT_TRUE(raylet_client->ReplyGetWorkerFailureCause()); ASSERT_EQ(worker_client->callbacks.size(), 0); ASSERT_EQ(raylet_client->num_workers_returned, 0); ASSERT_EQ(raylet_client->num_workers_returned_exiting, 0); diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index b9625c7b2469..451bff46d50f 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -200,7 +200,6 @@ ray_cc_test( srcs = ["lease_policy_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/common:task_common", "//src/ray/core_worker:lease_policy", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", diff --git a/src/ray/core_worker/tests/lease_policy_test.cc b/src/ray/core_worker/tests/lease_policy_test.cc index 37e1a584cd2c..b1219bcf375a 100644 --- a/src/ray/core_worker/tests/lease_policy_test.cc +++ b/src/ray/core_worker/tests/lease_policy_test.cc @@ -18,22 +18,19 @@ #include #include "gtest/gtest.h" -#include "ray/common/task/task_spec.h" +#include "ray/common/lease/lease_spec.h" namespace ray { namespace core { -TaskSpecification CreateFakeTask(std::vector deps) { - TaskSpecification spec; - spec.GetMutableMessage().set_task_id(TaskID::FromRandom(JobID::FromInt(1)).Binary()); +LeaseSpecification CreateFakeLease(std::vector deps) { + rpc::LeaseSpec spec; for (auto &dep : deps) { - spec.GetMutableMessage().add_args()->mutable_object_ref()->set_object_id( - dep.Binary()); + spec.add_dependencies()->set_object_id(dep.Binary()); } - spec.GetMutableMessage() - .mutable_scheduling_strategy() - ->mutable_default_scheduling_strategy(); - return spec; + spec.set_lease_id(LeaseID::FromRandom().Binary()); + spec.mutable_scheduling_strategy()->mutable_default_scheduling_strategy(); + return LeaseSpecification(spec); } class MockLocalityDataProvider : public LocalityDataProviderInterface { @@ -73,9 +70,9 @@ TEST(LocalLeasePolicyTest, TestReturnFallback) { ObjectID obj1 = ObjectID::FromRandom(); ObjectID obj2 = ObjectID::FromRandom(); std::vector deps{obj1, obj2}; - auto task_spec = CreateFakeTask(deps); + auto lease_spec = CreateFakeLease(deps); auto [best_node_address, is_selected_based_on_locality] = - local_lease_policy.GetBestNodeForTask(task_spec); + local_lease_policy.GetBestNodeForLease(lease_spec); // Test that fallback node was chosen. ASSERT_EQ(NodeID::FromBinary(best_node_address.node_id()), fallback_node); ASSERT_FALSE(is_selected_based_on_locality); @@ -96,12 +93,12 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityFallbackSpreadSchedulingStrat LocalityAwareLeasePolicy locality_lease_policy( *mock_locality_data_provider, MockNodeAddrFactory, fallback_rpc_address); std::vector deps{obj1, obj2}; - auto task_spec = CreateFakeTask(deps); - task_spec.GetMutableMessage() + auto lease_spec = CreateFakeLease(deps); + lease_spec.GetMutableMessage() .mutable_scheduling_strategy() ->mutable_spread_scheduling_strategy(); auto [best_node_address, is_selected_based_on_locality] = - locality_lease_policy.GetBestNodeForTask(task_spec); + locality_lease_policy.GetBestNodeForLease(lease_spec); // Locality logic is not run since it's a spread scheduling strategy. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, 0); // Test that fallback node was chosen. @@ -125,14 +122,14 @@ TEST(LocalityAwareLeasePolicyTest, LocalityAwareLeasePolicy locality_lease_policy( *mock_locality_data_provider, MockNodeAddrFactory, fallback_rpc_address); std::vector deps{obj1, obj2}; - auto task_spec = CreateFakeTask(deps); + auto lease_spec = CreateFakeLease(deps); NodeID node_affinity_node = NodeID::FromRandom(); - task_spec.GetMutableMessage() + lease_spec.GetMutableMessage() .mutable_scheduling_strategy() ->mutable_node_affinity_scheduling_strategy() ->set_node_id(node_affinity_node.Binary()); auto [best_node_address, is_selected_based_on_locality] = - locality_lease_policy.GetBestNodeForTask(task_spec); + locality_lease_policy.GetBestNodeForLease(lease_spec); // Locality logic is not run since it's a node affinity scheduling strategy. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, 0); // Test that node affinity node was chosen. @@ -155,9 +152,9 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityDominatingNode) { LocalityAwareLeasePolicy locality_lease_policy( *mock_locality_data_provider, MockNodeAddrFactory, fallback_rpc_address); std::vector deps{obj1, obj2}; - auto task_spec = CreateFakeTask(deps); + auto lease_spec = CreateFakeLease(deps); auto [best_node_address, is_selected_based_on_locality] = - locality_lease_policy.GetBestNodeForTask(task_spec); + locality_lease_policy.GetBestNodeForLease(lease_spec); // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that best node was chosen. @@ -181,9 +178,9 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityBiggerObject) { LocalityAwareLeasePolicy locality_lease_policy( *mock_locality_data_provider, MockNodeAddrFactory, fallback_rpc_address); std::vector deps{obj1, obj2}; - auto task_spec = CreateFakeTask(deps); + auto lease_spec = CreateFakeLease(deps); auto [best_node_address, is_selected_based_on_locality] = - locality_lease_policy.GetBestNodeForTask(task_spec); + locality_lease_policy.GetBestNodeForLease(lease_spec); // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that best node was chosen. @@ -211,9 +208,9 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityBetterNode) { LocalityAwareLeasePolicy locality_lease_policy( *mock_locality_data_provider, MockNodeAddrFactory, fallback_rpc_address); std::vector deps{obj1, obj2, obj3}; - auto task_spec = CreateFakeTask(deps); + auto lease_spec = CreateFakeLease(deps); auto [best_node_address, is_selected_based_on_locality] = - locality_lease_policy.GetBestNodeForTask(task_spec); + locality_lease_policy.GetBestNodeForLease(lease_spec); // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that best node was chosen. @@ -235,9 +232,9 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityFallbackNoLocations) { LocalityAwareLeasePolicy locality_lease_policy( *mock_locality_data_provider, MockNodeAddrFactory, fallback_rpc_address); std::vector deps{obj1, obj2}; - auto task_spec = CreateFakeTask(deps); + auto lease_spec = CreateFakeLease(deps); auto [best_node_address, is_selected_based_on_locality] = - locality_lease_policy.GetBestNodeForTask(task_spec); + locality_lease_policy.GetBestNodeForLease(lease_spec); // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that fallback node was chosen. @@ -252,11 +249,11 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityFallbackNoDeps) { auto mock_locality_data_provider = std::make_shared(); LocalityAwareLeasePolicy locality_lease_policy( *mock_locality_data_provider, MockNodeAddrFactory, fallback_rpc_address); - // No task dependencies. + // No lease dependencies. std::vector deps; - auto task_spec = CreateFakeTask(deps); + auto lease_spec = CreateFakeLease(deps); auto [best_node_address, is_selected_based_on_locality] = - locality_lease_policy.GetBestNodeForTask(task_spec); + locality_lease_policy.GetBestNodeForLease(lease_spec); // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that fallback node was chosen. @@ -279,9 +276,9 @@ TEST(LocalityAwareLeasePolicyTest, TestBestLocalityFallbackAddrFetchFail) { LocalityAwareLeasePolicy locality_lease_policy( *mock_locality_data_provider, MockNodeAddrFactoryAlwaysNull, fallback_rpc_address); std::vector deps{obj1, obj2}; - auto task_spec = CreateFakeTask(deps); + auto lease_spec = CreateFakeLease(deps); auto [best_node_address, is_selected_based_on_locality] = - locality_lease_policy.GetBestNodeForTask(task_spec); + locality_lease_policy.GetBestNodeForLease(lease_spec); // Locality data provider should be called once for each dependency. ASSERT_EQ(mock_locality_data_provider->num_locality_data_fetches, deps.size()); // Test that fallback node was chosen. diff --git a/src/ray/design_docs/id_specification.md b/src/ray/design_docs/id_specification.md index e5a4e52368bb..8c56400f7a08 100644 --- a/src/ray/design_docs/id_specification.md +++ b/src/ray/design_docs/id_specification.md @@ -25,14 +25,19 @@ Ray ID Specification | TaskID | index bytes | ObjectID 28B +-----------------------------------------------------------------------+-----------------+ + 4B 28B ++-----------------+-----------------------------------------------------------------------+ +| unique bytes | WorkerID | LeaseID 32B ++-----------------+-----------------------------------------------------------------------+ + ``` #### JobID (4 bytes) `JobID` is generated by `GCS` to ensure uniqueness. Its length is 4 bytes. -#### ActorID (8 bytes) +#### ActorID (16 bytes) An `ActorID` contains two parts: 1) 12 unique bytes, and 2) its `JobID`. -#### TaskID (16 bytes) +#### TaskID (24 bytes) A `TaskID` contains two parts: 1) 8 unique bytes, and 2) its `ActorID`. If the task is a normal task or a driver task, the part 2 is its dummy actor id. @@ -58,3 +63,11 @@ An `ObjectID` contains 2 parts: and `n` is added to the `TaskID`'s unique bytes, where `n` is the number of times that task has executed so far. For task returns, the unique bytes are identical to the parent task. + +#### LeaseID (32 bytes) +A `LeaseID` contains 2 parts: +- `unique bytes`: 4 bytes generated via a counter unique to the lease requester +(worker or gcs). +- `WorkerID`: 28 bytes that represent the WorkerID of the lease requester. +In the case of the gcs it's randomly generated. Due to the possibility of GCS +restarts, we can't simply nil them out. diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 4c24d3ea9dd0..4ff406b335e5 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -106,8 +106,8 @@ ray_cc_library( "//src/ray/common:task_common", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/protobuf:ray_syncer_cc_proto", + "//src/ray/raylet/scheduling:cluster_lease_manager", "//src/ray/raylet/scheduling:cluster_resource_manager", - "//src/ray/raylet/scheduling:cluster_task_manager", "//src/ray/rpc:gcs_server", "//src/ray/stats:stats_metric", "//src/ray/util:logging", @@ -367,6 +367,7 @@ ray_cc_library( ], deps = [ "//src/ray/common:id", + "//src/ray/common:lease", "//src/ray/common:task_common", "//src/ray/protobuf:core_worker_cc_proto", "//src/ray/protobuf:export_event_cc_proto", diff --git a/src/ray/gcs/gcs_server/gcs_actor.cc b/src/ray/gcs/gcs_server/gcs_actor.cc index 0c4c44bbdc54..2e9b6769b052 100644 --- a/src/ray/gcs/gcs_server/gcs_actor.cc +++ b/src/ray/gcs/gcs_server/gcs_actor.cc @@ -126,6 +126,12 @@ void GcsActor::WriteActorExportEvent() const { rpc::TaskSpec *GcsActor::GetMutableTaskSpec() { return task_spec_.get(); } +rpc::LeaseSpec *GcsActor::GetMutableLeaseSpec() { + return &lease_spec_->GetMutableMessage(); +} + +const LeaseSpecification &GcsActor::GetLeaseSpecification() const { return *lease_spec_; } + const ResourceRequest &GcsActor::GetAcquiredResources() const { return acquired_resources_; } diff --git a/src/ray/gcs/gcs_server/gcs_actor.h b/src/ray/gcs/gcs_server/gcs_actor.h index b9aa9bea5019..4ea57bdfebfb 100644 --- a/src/ray/gcs/gcs_server/gcs_actor.h +++ b/src/ray/gcs/gcs_server/gcs_actor.h @@ -17,6 +17,7 @@ #include #include "ray/common/id.h" +#include "ray/common/lease/lease_spec.h" #include "ray/common/scheduling/cluster_resource_data.h" #include "ray/common/task/task_spec.h" #include "ray/util/counter_map.h" @@ -62,6 +63,7 @@ class GcsActor { task_spec_(std::make_unique(std::move(task_spec))), counter_(std::move(counter)), export_event_write_enabled_(IsExportAPIEnabledActor()) { + lease_spec_ = std::make_unique(*task_spec_); RAY_CHECK(actor_table_data_.state() != rpc::ActorTableData::DEAD); RefreshMetrics(); } @@ -139,6 +141,7 @@ class GcsActor { actor_table_data_.mutable_label_selector()->insert( task_spec_->label_selector().begin(), task_spec_->label_selector().end()); } + lease_spec_ = std::make_unique(*task_spec_); RefreshMetrics(); } @@ -187,12 +190,14 @@ class GcsActor { std::string GetRayNamespace() const; /// Get the task specification of this actor. TaskSpecification GetCreationTaskSpecification() const; + const LeaseSpecification &GetLeaseSpecification() const; /// Get the immutable ActorTableData of this actor. const rpc::ActorTableData &GetActorTableData() const; /// Get the mutable ActorTableData of this actor. rpc::ActorTableData *GetMutableActorTableData(); rpc::TaskSpec *GetMutableTaskSpec(); + rpc::LeaseSpec *GetMutableLeaseSpec(); /// Write an event containing this actor's ActorTableData /// to file for the Export API. void WriteActorExportEvent() const; @@ -265,6 +270,7 @@ class GcsActor { std::optional last_metric_state_; /// If true, actor events are exported for Export API bool export_event_write_enabled_ = false; + std::unique_ptr lease_spec_; }; using RestartActorForLineageReconstructionCallback = diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index b8a45ed147e8..e2b9b5cbf064 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -1000,7 +1000,7 @@ void GcsActorManager::DestroyActor(const ActorID &actor_id, // worker exit to avoid process and resource leak. NotifyCoreWorkerToKillActor(actor, death_cause, force_kill); } - CancelActorInScheduling(actor, TaskID::ForActorCreationTask(actor_id)); + CancelActorInScheduling(actor); } } @@ -1710,16 +1710,16 @@ void GcsActorManager::KillActor(const ActorID &actor_id, bool force_kill) { NotifyCoreWorkerToKillActor( actor, GenKilledByApplicationCause(GetActor(actor_id)), force_kill); } else { - const auto &task_id = actor->GetCreationTaskSpecification().TaskId(); - RAY_LOG(DEBUG).WithField(actor->GetActorID()).WithField(task_id) - << "The actor hasn't been created yet, cancel scheduling task"; + const auto &lease_id = actor->GetLeaseSpecification().LeaseId(); + RAY_LOG(DEBUG).WithField(actor->GetActorID()).WithField(lease_id) + << "The actor hasn't been created yet, cancel scheduling lease"; if (!worker_id.IsNil()) { // The actor is in phase of creating, so we need to notify the core // worker exit to avoid process and resource leak. NotifyCoreWorkerToKillActor( actor, GenKilledByApplicationCause(GetActor(actor_id)), force_kill); } - CancelActorInScheduling(actor, task_id); + CancelActorInScheduling(actor); RestartActor(actor_id, /*need_reschedule=*/true, GenKilledByApplicationCause(GetActor(actor_id))); @@ -1743,10 +1743,10 @@ void GcsActorManager::AddDestroyedActorToCache(const std::shared_ptr & } } -void GcsActorManager::CancelActorInScheduling(const std::shared_ptr &actor, - const TaskID &task_id) { - RAY_LOG(DEBUG).WithField(actor->GetActorID()).WithField(task_id) - << "Cancel actor in scheduling"; +void GcsActorManager::CancelActorInScheduling(const std::shared_ptr &actor) { + auto lease_id = actor->GetLeaseSpecification().LeaseId(); + RAY_LOG(DEBUG).WithField(actor->GetActorID()).WithField(lease_id) + << "Cancel actor in scheduling, this may be due to resource re-eviction"; const auto &actor_id = actor->GetActorID(); const auto &node_id = actor->GetNodeID(); // The actor has not been created yet. It is either being scheduled or is @@ -1763,7 +1763,7 @@ void GcsActorManager::CancelActorInScheduling(const std::shared_ptr &a // it doesn't responds, and the actor should be still in leasing state. // NOTE: We will cancel outstanding lease request by calling // `raylet_client->CancelWorkerLease`. - gcs_actor_scheduler_->CancelOnLeasing(node_id, actor_id, task_id); + gcs_actor_scheduler_->CancelOnLeasing(node_id, actor_id, lease_id); // Return the actor's acquired resources (if any). gcs_actor_scheduler_->OnActorDestruction(actor); } diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.h b/src/ray/gcs/gcs_server/gcs_actor_manager.h index a0dab85a3903..479d3818100b 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.h +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.h @@ -392,9 +392,8 @@ class GcsActorManager : public rpc::ActorInfoGcsServiceHandler { /// Cancel actor which is either being scheduled or is pending scheduling. /// /// \param actor The actor to be cancelled. - /// \param task_id The id of actor creation task to be cancelled. - void CancelActorInScheduling(const std::shared_ptr &actor, - const TaskID &task_id); + /// \param lease_id The lease id of actor creation task to be cancelled. + void CancelActorInScheduling(const std::shared_ptr &actor); /// Get the alive or dead actor of the actor id. /// NOTE: The return value is not meant to be passed to other scope. diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc index 9868f13cf08c..c8e2aff769dc 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc @@ -30,7 +30,7 @@ GcsActorScheduler::GcsActorScheduler( instrumented_io_context &io_context, GcsActorTable &gcs_actor_table, const GcsNodeManager &gcs_node_manager, - ClusterTaskManager &cluster_task_manager, + ClusterLeaseManager &cluster_lease_manager, GcsActorSchedulerFailureCallback schedule_failure_handler, GcsActorSchedulerSuccessCallback schedule_success_handler, rpc::RayletClientPool &raylet_client_pool, @@ -40,7 +40,7 @@ GcsActorScheduler::GcsActorScheduler( : io_context_(io_context), gcs_actor_table_(gcs_actor_table), gcs_node_manager_(gcs_node_manager), - cluster_task_manager_(cluster_task_manager), + cluster_lease_manager_(cluster_lease_manager), schedule_failure_handler_(std::move(schedule_failure_handler)), schedule_success_handler_(std::move(schedule_success_handler)), raylet_client_pool_(raylet_client_pool), @@ -54,7 +54,7 @@ void GcsActorScheduler::Schedule(std::shared_ptr actor) { RAY_CHECK(actor->GetNodeID().IsNil() && actor->GetWorkerID().IsNil()); if (RayConfig::instance().gcs_actor_scheduling_enabled() && - !actor->GetCreationTaskSpecification().GetRequiredResources().IsEmpty()) { + !actor->GetLeaseSpecification().GetRequiredResources().IsEmpty()) { ScheduleByGcs(actor); } else { ScheduleByRaylet(actor); @@ -90,8 +90,7 @@ void GcsActorScheduler::ScheduleByGcs(std::shared_ptr actor) { .second); actor->SetAcquiredResources(ResourceMapToResourceRequest( - actor->GetCreationTaskSpecification().GetRequiredResources().GetResourceMap(), - false)); + actor->GetLeaseSpecification().GetRequiredResources().GetResourceMap(), false)); // Lease worker directly from the node. actor->SetGrantOrReject(true); LeaseWorkerFromNode(actor, node.value()); @@ -99,13 +98,14 @@ void GcsActorScheduler::ScheduleByGcs(std::shared_ptr actor) { // Queue and schedule the actor locally (gcs). const auto &owner_node = gcs_node_manager_.GetAliveNode(actor->GetOwnerNodeID()); - RayTask task(actor->GetCreationTaskSpecification(), - owner_node.has_value() ? actor->GetOwnerNodeID().Binary() : std::string()); - cluster_task_manager_.QueueAndScheduleTask(std::move(task), - /*grant_or_reject*/ false, - /*is_selected_based_on_locality*/ false, - /*reply*/ reply.get(), - send_reply_callback); + RayLease lease( + actor->GetLeaseSpecification(), + owner_node.has_value() ? actor->GetOwnerNodeID().Binary() : std::string()); + cluster_lease_manager_.QueueAndScheduleLease(std::move(lease), + /*grant_or_reject=*/false, + /*is_selected_based_on_locality=*/false, + /*reply=*/reply.get(), + send_reply_callback); } void GcsActorScheduler::ScheduleByRaylet(std::shared_ptr actor) { @@ -142,8 +142,8 @@ NodeID GcsActorScheduler::SelectForwardingNode(std::shared_ptr actor) // If an actor has resource requirements, we will try to schedule it on the same node as // the owner if possible. - const auto &task_spec = actor->GetCreationTaskSpecification(); - if (!task_spec.GetRequiredResources().IsEmpty()) { + const auto &lease_spec = actor->GetLeaseSpecification(); + if (!lease_spec.GetRequiredResources().IsEmpty()) { auto maybe_node = gcs_node_manager_.GetAliveNode(actor->GetOwnerNodeID()); node = maybe_node.has_value() ? maybe_node.value() : SelectNodeRandomly(); } else { @@ -227,10 +227,10 @@ std::vector GcsActorScheduler::CancelOnNode(const NodeID &node_id) { void GcsActorScheduler::CancelOnLeasing(const NodeID &node_id, const ActorID &actor_id, - const TaskID &task_id) { + const LeaseID &lease_id) { // NOTE: This method will cancel the outstanding lease request and remove leasing // information from the internal state. - RAY_LOG(DEBUG) << "Canceling worker leasing of task " << task_id; + RAY_LOG(DEBUG) << "Canceling worker lease request " << lease_id; auto node_it = node_to_actors_when_leasing_.find(node_id); RAY_CHECK(node_it != node_to_actors_when_leasing_.end()); node_it->second.erase(actor_id); @@ -248,7 +248,7 @@ void GcsActorScheduler::CancelOnLeasing(const NodeID &node_id, address.set_port(node_info->node_manager_port()); auto raylet_client = GetOrConnectRayletClient(address); raylet_client->CancelWorkerLease( - task_id, [](const Status &status, const rpc::CancelWorkerLeaseReply &reply) {}); + lease_id, [](const Status &status, const rpc::CancelWorkerLeaseReply &reply) {}); } } @@ -327,8 +327,12 @@ void GcsActorScheduler::LeaseWorkerFromNode(std::shared_ptr actor, auto raylet_client = GetOrConnectRayletClient(remote_address); // Actor leases should be sent to the raylet immediately, so we should never build up a // backlog in GCS. + // Counter for generating unique lease IDs. + static uint32_t lease_id_counter = 1; + actor->GetMutableLeaseSpec()->set_lease_id( + LeaseID::FromWorker(WorkerID::FromRandom(), lease_id_counter++).Binary()); raylet_client->RequestWorkerLease( - actor->GetCreationTaskSpecification().GetMessage(), + actor->GetLeaseSpecification().GetMessage(), actor->GetGrantOrReject(), [this, actor, node](const Status &status, const rpc::RequestWorkerLeaseReply &reply) { @@ -432,13 +436,13 @@ void GcsActorScheduler::HandleRequestWorkerLeaseCanceled( const NodeID &node_id, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) { - RAY_LOG(INFO) - << "The lease worker request from node " << node_id << " for actor " - << actor->GetActorID() << "(" - << actor->GetCreationTaskSpecification().FunctionDescriptor()->CallString() << ")" - << " has been canceled, job id = " << actor->GetActorID().JobId() - << ", cancel type: " - << rpc::RequestWorkerLeaseReply::SchedulingFailureType_Name(failure_type); + RAY_LOG(INFO) << "The lease worker request from node " << node_id << " for actor " + << actor->GetActorID() << "(" + << actor->GetLeaseSpecification().FunctionDescriptor()->CallString() + << ")" + << " has been canceled, job id = " << actor->GetActorID().JobId() + << ", cancel type: " + << rpc::RequestWorkerLeaseReply::SchedulingFailureType_Name(failure_type); schedule_failure_handler_(actor, failure_type, scheduling_failure_message); } @@ -663,13 +667,13 @@ void GcsActorScheduler::HandleWorkerLeaseRejectedReply( void GcsActorScheduler::OnActorDestruction(std::shared_ptr actor) { if (!actor->GetAcquiredResources().IsEmpty()) { ReturnActorAcquiredResources(actor); - cluster_task_manager_.ScheduleAndDispatchTasks(); + cluster_lease_manager_.ScheduleAndGrantLeases(); } } void GcsActorScheduler::ReturnActorAcquiredResources(std::shared_ptr actor) { auto &cluster_resource_manager = - cluster_task_manager_.GetClusterResourceScheduler().GetClusterResourceManager(); + cluster_lease_manager_.GetClusterResourceScheduler().GetClusterResourceManager(); cluster_resource_manager.AddNodeAvailableResources( scheduling::NodeID(actor->GetNodeID().Binary()), actor->GetAcquiredResources().GetResourceSet()); @@ -677,13 +681,13 @@ void GcsActorScheduler::ReturnActorAcquiredResources(std::shared_ptr a } size_t GcsActorScheduler::GetPendingActorsCount() const { - return cluster_task_manager_.GetInfeasibleQueueSize() + - cluster_task_manager_.GetPendingQueueSize(); + return cluster_lease_manager_.GetInfeasibleQueueSize() + + cluster_lease_manager_.GetPendingQueueSize(); } bool GcsActorScheduler::CancelInFlightActorScheduling( const std::shared_ptr &actor) { - return cluster_task_manager_.CancelTask(actor->GetCreationTaskSpecification().TaskId()); + return cluster_lease_manager_.CancelLease(actor->GetLeaseSpecification().LeaseId()); } } // namespace gcs diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h index 218bf9f3b035..582957e99445 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h @@ -30,7 +30,7 @@ #include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/raylet/scheduling/cluster_task_manager.h" +#include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet_client/raylet_client.h" #include "ray/rpc/node_manager/node_manager_client.h" #include "ray/rpc/node_manager/raylet_client_pool.h" @@ -39,7 +39,7 @@ #include "src/ray/protobuf/gcs_service.pb.h" namespace ray { -using raylet::ClusterTaskManager; +using raylet::ClusterLeaseManager; namespace gcs { using GcsActorSchedulerFailureCallback = @@ -73,7 +73,7 @@ class GcsActorSchedulerInterface { /// \param actor_id ID of an actor. virtual void CancelOnLeasing(const NodeID &node_id, const ActorID &actor_id, - const TaskID &task_id) = 0; + const LeaseID &lease_id) = 0; /// Cancel the actor that is being scheduled to the specified worker. /// @@ -118,7 +118,7 @@ class GcsActorScheduler : public GcsActorSchedulerInterface { /// \param io_context The main event loop. /// \param gcs_actor_table Used to flush actor info to storage. /// \param gcs_node_manager The node manager which is used when scheduling. - /// \param cluster_task_manager The task manager that queues and schedules actor. + /// \param cluster_lease_manager The task manager that queues and schedules actor. /// creation tasks. /// \param schedule_failure_handler Invoked when there are no available /// nodes to schedule actors. @@ -131,7 +131,7 @@ class GcsActorScheduler : public GcsActorSchedulerInterface { instrumented_io_context &io_context, GcsActorTable &gcs_actor_table, const GcsNodeManager &gcs_node_manager, - ClusterTaskManager &cluster_task_manager_, + ClusterLeaseManager &cluster_lease_manager_, GcsActorSchedulerFailureCallback schedule_failure_handler, GcsActorSchedulerSuccessCallback schedule_success_handler, rpc::RayletClientPool &raylet_client_pool, @@ -143,7 +143,7 @@ class GcsActorScheduler : public GcsActorSchedulerInterface { /// Schedule the specified actor. /// If there is no available nodes then the actor would be queued in the - /// `cluster_task_manager_`. + /// `cluster_lease_manager_`. /// /// \param actor to be scheduled. void Schedule(std::shared_ptr actor) override; @@ -169,7 +169,7 @@ class GcsActorScheduler : public GcsActorSchedulerInterface { /// \param actor_id ID of an actor. void CancelOnLeasing(const NodeID &node_id, const ActorID &actor_id, - const TaskID &task_id) override; + const LeaseID &lease_id) override; /// Cancel the actor that is being scheduled to the specified worker. /// @@ -379,8 +379,8 @@ class GcsActorScheduler : public GcsActorSchedulerInterface { node_to_workers_when_creating_; /// Reference of GcsNodeManager. const GcsNodeManager &gcs_node_manager_; - /// The cluster task manager. - ClusterTaskManager &cluster_task_manager_; + /// The cluster lease manager. + ClusterLeaseManager &cluster_lease_manager_; /// The handler to handle the scheduling failures. GcsActorSchedulerFailureCallback schedule_failure_handler_; /// The handler to handle the successful scheduling. diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc index 6fc73a7ff351..5ad9d09c719d 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc @@ -617,10 +617,10 @@ void GcsAutoscalerStateManager::CancelInfeasibleRequests() const { RAY_LOG(WARNING) << "Canceling infeasible requests on node " << node_id << " with infeasible_shapes=" << resource_shapes_str; - raylet_client->CancelTasksWithResourceShapes( + raylet_client->CancelLeasesWithResourceShapes( infeasible_shapes, [node_id](const Status &status, - const rpc::CancelTasksWithResourceShapesReply &) { + const rpc::CancelLeasesWithResourceShapesReply &) { if (status.ok()) { RAY_LOG(INFO) << "Infeasible tasks cancelled on node " << node_id; } else { diff --git a/src/ray/gcs/gcs_server/gcs_resource_manager.cc b/src/ray/gcs/gcs_server/gcs_resource_manager.cc index e73f834bf816..0dca317fda20 100644 --- a/src/ray/gcs/gcs_server/gcs_resource_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_resource_manager.cc @@ -29,12 +29,12 @@ GcsResourceManager::GcsResourceManager(instrumented_io_context &io_context, ClusterResourceManager &cluster_resource_manager, GcsNodeManager &gcs_node_manager, NodeID local_node_id, - raylet::ClusterTaskManager *cluster_task_manager) + raylet::ClusterLeaseManager *cluster_lease_manager) : io_context_(io_context), cluster_resource_manager_(cluster_resource_manager), gcs_node_manager_(gcs_node_manager), local_node_id_(std::move(local_node_id)), - cluster_task_manager_(cluster_task_manager) {} + cluster_lease_manager_(cluster_lease_manager) {} void GcsResourceManager::ConsumeSyncMessage( std::shared_ptr message) { @@ -200,10 +200,10 @@ void GcsResourceManager::HandleGetAllResourceUsage( batch.add_batch()->CopyFrom(usage.second); } - if (cluster_task_manager_ != nullptr) { + if (cluster_lease_manager_ != nullptr) { // Fill the gcs info when gcs actor scheduler is enabled. rpc::ResourcesData gcs_resources_data; - cluster_task_manager_->FillPendingActorInfo(gcs_resources_data); + cluster_lease_manager_->FillPendingActorInfo(gcs_resources_data); // Aggregate the load (pending actor info) of gcs. FillAggregateLoad(gcs_resources_data, &aggregate_load); // We only export gcs's pending info without adding the corresponding diff --git a/src/ray/gcs/gcs_server/gcs_resource_manager.h b/src/ray/gcs/gcs_server/gcs_resource_manager.h index 730141ff220f..96242cb291a7 100644 --- a/src/ray/gcs/gcs_server/gcs_resource_manager.h +++ b/src/ray/gcs/gcs_server/gcs_resource_manager.h @@ -27,8 +27,8 @@ #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" +#include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" -#include "ray/raylet/scheduling/cluster_task_manager.h" #include "ray/stats/metric_defs.h" #include "src/ray/protobuf/gcs.pb.h" #include "src/ray/protobuf/ray_syncer.pb.h" @@ -58,11 +58,12 @@ class GcsResourceManager : public rpc::NodeResourceInfoGcsServiceHandler, public syncer::ReceiverInterface { public: /// Create a GcsResourceManager. - explicit GcsResourceManager(instrumented_io_context &io_context, - ClusterResourceManager &cluster_resource_manager, - GcsNodeManager &gcs_node_manager, - NodeID local_node_id, - raylet::ClusterTaskManager *cluster_task_manager = nullptr); + explicit GcsResourceManager( + instrumented_io_context &io_context, + ClusterResourceManager &cluster_resource_manager, + GcsNodeManager &gcs_node_manager, + NodeID local_node_id, + raylet::ClusterLeaseManager *cluster_lease_manager = nullptr); virtual ~GcsResourceManager() = default; @@ -197,7 +198,7 @@ class GcsResourceManager : public rpc::NodeResourceInfoGcsServiceHandler, ClusterResourceManager &cluster_resource_manager_; GcsNodeManager &gcs_node_manager_; NodeID local_node_id_; - raylet::ClusterTaskManager *cluster_task_manager_; + raylet::ClusterLeaseManager *cluster_lease_manager_; /// Num of alive nodes in the cluster. size_t num_alive_nodes_ = 0; }; diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 1cf6c2e8b9e8..3720f93847ff 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -237,7 +237,7 @@ void GcsServer::GetOrGenerateClusterId( void GcsServer::DoStart(const GcsInitData &gcs_init_data) { InitClusterResourceScheduler(); InitGcsNodeManager(gcs_init_data); - InitClusterTaskManager(); + InitClusterLeaseManager(); InitGcsResourceManager(gcs_init_data); InitGcsHealthCheckManager(gcs_init_data); InitRaySyncer(gcs_init_data); @@ -350,13 +350,13 @@ void GcsServer::InitGcsHealthCheckManager(const GcsInitData &gcs_init_data) { } void GcsServer::InitGcsResourceManager(const GcsInitData &gcs_init_data) { - RAY_CHECK(cluster_resource_scheduler_ && cluster_task_manager_); + RAY_CHECK(cluster_resource_scheduler_ && cluster_lease_manager_); gcs_resource_manager_ = std::make_unique( io_context_provider_.GetDefaultIOContext(), cluster_resource_scheduler_->GetClusterResourceManager(), *gcs_node_manager_, kGCSNodeID, - cluster_task_manager_.get()); + cluster_lease_manager_.get()); // Initialize by gcs tables data. gcs_resource_manager_->Initialize(gcs_init_data); @@ -422,9 +422,9 @@ void GcsServer::InitClusterResourceScheduler() { /*is_local_node_with_raylet=*/false); } -void GcsServer::InitClusterTaskManager() { +void GcsServer::InitClusterLeaseManager() { RAY_CHECK(cluster_resource_scheduler_); - cluster_task_manager_ = std::make_unique( + cluster_lease_manager_ = std::make_unique( kGCSNodeID, *cluster_resource_scheduler_, /*get_node_info=*/ @@ -433,7 +433,7 @@ void GcsServer::InitClusterTaskManager() { return node.has_value() ? node.value().get() : nullptr; }, /*announce_infeasible_task=*/nullptr, - /*local_task_manager=*/local_task_manager_); + /*local_lease_manager=*/local_lease_manager_); } void GcsServer::InitGcsJobManager(const GcsInitData &gcs_init_data) { @@ -473,12 +473,12 @@ void GcsServer::InitGcsActorManager(const GcsInitData &gcs_init_data) { gcs_actor_manager_->OnActorCreationSuccess(actor, reply); }; - RAY_CHECK(gcs_resource_manager_ && cluster_task_manager_); + RAY_CHECK(gcs_resource_manager_ && cluster_lease_manager_); scheduler = std::make_unique( io_context_provider_.GetDefaultIOContext(), gcs_table_storage_->ActorTable(), *gcs_node_manager_, - *cluster_task_manager_, + *cluster_lease_manager_, schedule_failure_handler, schedule_success_handler, raylet_client_pool_, @@ -772,7 +772,7 @@ void GcsServer::InstallEventListeners() { RAY_CHECK(channel != nullptr); gcs_healthcheck_manager_->AddNode(node_id, channel); } - cluster_task_manager_->ScheduleAndDispatchTasks(); + cluster_lease_manager_->ScheduleAndGrantLeases(); }); gcs_node_manager_->AddNodeRemovedListener( [this](const std::shared_ptr &node) { @@ -829,7 +829,7 @@ void GcsServer::InstallEventListeners() { // Because resources have been changed, we need to try to schedule the // pending placement groups and actors. gcs_placement_group_manager_->SchedulePendingPlacementGroups(); - cluster_task_manager_->ScheduleAndDispatchTasks(); + cluster_lease_manager_->ScheduleAndGrantLeases(); }, "GcsServer.SchedulePendingActors"); }); @@ -840,7 +840,7 @@ void GcsServer::InstallEventListeners() { // Because some placement group resources have been committed or deleted, we // need to try to schedule the pending placement groups and actors. gcs_placement_group_manager_->SchedulePendingPlacementGroups(); - cluster_task_manager_->ScheduleAndDispatchTasks(); + cluster_lease_manager_->ScheduleAndGrantLeases(); }, "GcsServer.SchedulePendingPGActors"); }); @@ -901,7 +901,7 @@ void GcsServer::PrintAsioStats() { } void GcsServer::TryGlobalGC() { - if (cluster_task_manager_->GetPendingQueueSize() == 0) { + if (cluster_lease_manager_->GetPendingQueueSize() == 0) { task_pending_schedule_detected_ = 0; return; } diff --git a/src/ray/gcs/gcs_server/gcs_server.h b/src/ray/gcs/gcs_server/gcs_server.h index 25b3bdea7d9a..5a4ece5e4b21 100644 --- a/src/ray/gcs/gcs_server/gcs_server.h +++ b/src/ray/gcs/gcs_server/gcs_server.h @@ -37,8 +37,8 @@ #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/store_client/observable_store_client.h" #include "ray/gcs/store_client/redis_store_client.h" +#include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" -#include "ray/raylet/scheduling/cluster_task_manager.h" #include "ray/rpc/client_call.h" #include "ray/rpc/gcs/gcs_rpc_server.h" #include "ray/rpc/metrics_agent_client.h" @@ -47,8 +47,8 @@ #include "ray/util/throttler.h" namespace ray { -using raylet::ClusterTaskManager; -using raylet::NoopLocalTaskManager; +using raylet::ClusterLeaseManager; +using raylet::NoopLocalLeaseManager; namespace gcs { @@ -149,8 +149,8 @@ class GcsServer { /// Initialize cluster resource scheduler. void InitClusterResourceScheduler(); - /// Initialize cluster task manager. - void InitClusterTaskManager(); + /// Initialize cluster lease manager. + void InitClusterLeaseManager(); /// Initialize gcs job manager. void InitGcsJobManager(const GcsInitData &gcs_init_data); @@ -235,13 +235,13 @@ class GcsServer { rpc::CoreWorkerClientPool worker_client_pool_; /// The cluster resource scheduler. std::shared_ptr cluster_resource_scheduler_; - /// Local task manager. - NoopLocalTaskManager local_task_manager_; + /// Local lease manager. + NoopLocalLeaseManager local_lease_manager_; /// The gcs table storage. std::unique_ptr gcs_table_storage_; - /// The cluster task manager. - std::unique_ptr cluster_task_manager_; - /// [gcs_resource_manager_] depends on [cluster_task_manager_]. + /// The cluster lease manager. + std::unique_ptr cluster_lease_manager_; + /// [gcs_resource_manager_] depends on [cluster_lease_manager_]. /// The gcs resource manager. std::unique_ptr gcs_resource_manager_; /// The autoscaler state manager. diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc index 735acc7c9bbd..1dc0fff9e892 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc @@ -74,7 +74,7 @@ class MockActorScheduler : public gcs::GcsActorSchedulerInterface { MOCK_METHOD3(CancelOnLeasing, void(const NodeID &node_id, const ActorID &actor_id, - const TaskID &task_id)); + const LeaseID &lease_id)); std::vector> actors; }; diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc index 8efa1aa22544..8dca97d5ad0c 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc @@ -71,7 +71,7 @@ class MockActorScheduler : public gcs::GcsActorSchedulerInterface { MOCK_METHOD3(CancelOnLeasing, void(const NodeID &node_id, const ActorID &actor_id, - const TaskID &task_id)); + const LeaseID &lease_id)); std::vector> actors; }; @@ -924,9 +924,8 @@ TEST_F(GcsActorManagerTest, TestRaceConditionCancelLease) { address.set_worker_id(worker_id.Binary()); actor->UpdateAddress(address); const auto &actor_id = actor->GetActorID(); - const auto &task_id = TaskID::FromBinary( - registered_actor->GetCreationTaskSpecification().GetMessage().task_id()); - EXPECT_CALL(*mock_actor_scheduler_, CancelOnLeasing(node_id, actor_id, task_id)); + // LeaseID is randomly generated, so we can't check for a specific lease ID. + EXPECT_CALL(*mock_actor_scheduler_, CancelOnLeasing(node_id, actor_id, _)); gcs_actor_manager_->OnWorkerDead(owner_node_id, owner_worker_id); io_service_.run_one(); ASSERT_TRUE(actor->GetActorTableData().death_cause().has_actor_died_error_context()); diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc index c5650fcf9671..70e1003cfeae 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc @@ -56,8 +56,8 @@ class GcsActorSchedulerMockTest : public Test { /*is_node_available_fn=*/ [](auto) { return true; }, /*is_local_node_with_raylet=*/false); - local_task_manager_ = std::make_unique(); - cluster_task_manager = std::make_unique( + local_lease_manager_ = std::make_unique(); + cluster_lease_manager = std::make_unique( local_node_id, *cluster_resource_scheduler, /*get_node_info=*/ @@ -65,8 +65,8 @@ class GcsActorSchedulerMockTest : public Test { auto node = gcs_node_manager->GetAliveNode(nid); return node.has_value() ? node.value().get() : nullptr; }, - /*announce_infeasible_task=*/nullptr, - /*local_task_manager=*/*local_task_manager_); + /*announce_infeasible_lease=*/nullptr, + *local_lease_manager_); counter.reset( new CounterMap>()); worker_client_pool_ = std::make_unique( @@ -75,7 +75,7 @@ class GcsActorSchedulerMockTest : public Test { io_context, *actor_table, *gcs_node_manager, - *cluster_task_manager, + *cluster_lease_manager, [this](auto a, auto b, auto c) { schedule_failure_handler(a); }, [this](auto a, const rpc::PushTaskReply) { schedule_success_handler(a); }, *client_pool, @@ -93,8 +93,8 @@ class GcsActorSchedulerMockTest : public Test { std::shared_ptr store_client; std::unique_ptr actor_table; std::unique_ptr gcs_node_manager; - std::unique_ptr local_task_manager_; - std::unique_ptr cluster_task_manager; + std::unique_ptr local_lease_manager_; + std::unique_ptr cluster_lease_manager; std::unique_ptr actor_scheduler; std::shared_ptr core_worker_client; std::unique_ptr worker_client_pool_; @@ -121,7 +121,8 @@ TEST_F(GcsActorSchedulerMockTest, KillWorkerLeak1) { actor_data.set_actor_id(actor_id.Binary()); auto actor = std::make_shared(actor_data, rpc::TaskSpec(), counter); rpc::ClientCallback cb; - EXPECT_CALL(*raylet_client, RequestWorkerLease(An(), _, _, _, _)) + EXPECT_CALL(*raylet_client, + RequestWorkerLease(An(), _, _, _, _)) .WillOnce(testing::SaveArg<2>(&cb)); // Ensure actor is killed EXPECT_CALL(*core_worker_client, KillActor(_, _)); @@ -138,8 +139,8 @@ TEST_F(GcsActorSchedulerMockTest, KillWorkerLeak2) { // Ensure worker is not leak in the following case: // 1. Actor is in pending creation // 2. Gcs push creation task to run in worker - // 3. Cancel the task - // 4. Task creating reply received + // 3. Cancel the lease + // 4. Lease creating reply received // We'd like to test the worker got released eventually. // Worker is released with actor killing auto actor_id = ActorID::FromHex("f4ce02420592ca68c1738a0d01000000"); @@ -150,7 +151,8 @@ TEST_F(GcsActorSchedulerMockTest, KillWorkerLeak2) { rpc::ClientCallback request_worker_lease_cb; // Ensure actor is killed EXPECT_CALL(*core_worker_client, KillActor(_, _)); - EXPECT_CALL(*raylet_client, RequestWorkerLease(An(), _, _, _, _)) + EXPECT_CALL(*raylet_client, + RequestWorkerLease(An(), _, _, _, _)) .WillOnce(testing::SaveArg<2>(&request_worker_lease_cb)); // Postable is not default constructable, so we use a unique_ptr to hold one. diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc index 0a6fd64244f4..049b7faa42c3 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc @@ -35,7 +35,7 @@ #include "ray/util/counter_map.h" namespace ray { -using raylet::NoopLocalTaskManager; +using raylet::NoopLocalLeaseManager; namespace gcs { class MockedGcsActorScheduler : public gcs::GcsActorScheduler { @@ -111,8 +111,8 @@ class GcsActorSchedulerTest : public ::testing::Test { /*is_local_node_with_raylet=*/false); counter.reset( new CounterMap>()); - local_task_manager_ = std::make_unique(); - cluster_task_manager_ = std::make_unique( + local_lease_manager_ = std::make_unique(); + cluster_lease_manager_ = std::make_unique( local_node_id_, *cluster_resource_scheduler_, /*get_node_info=*/ @@ -121,7 +121,7 @@ class GcsActorSchedulerTest : public ::testing::Test { return node.has_value() ? node.value().get() : nullptr; }, /*announce_infeasible_task=*/nullptr, - /*local_task_manager=*/*local_task_manager_); + /*local_lease_manager=*/*local_lease_manager_); auto gcs_resource_manager = std::make_shared( io_context_->GetIoService(), cluster_resource_scheduler_->GetClusterResourceManager(), @@ -133,7 +133,7 @@ class GcsActorSchedulerTest : public ::testing::Test { io_context_->GetIoService(), *gcs_actor_table_, *gcs_node_manager_, - *cluster_task_manager_, + *cluster_lease_manager_, /*schedule_failure_handler=*/ [this](std::shared_ptr actor, const rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, @@ -210,9 +210,9 @@ class GcsActorSchedulerTest : public ::testing::Test { std::shared_ptr worker_client_; std::unique_ptr worker_client_pool_; std::shared_ptr gcs_node_manager_; - std::unique_ptr local_task_manager_; + std::unique_ptr local_lease_manager_; std::unique_ptr cluster_resource_scheduler_; - std::shared_ptr cluster_task_manager_; + std::shared_ptr cluster_lease_manager_; std::shared_ptr gcs_actor_scheduler_; std::shared_ptr>> counter; @@ -440,8 +440,8 @@ TEST_F(GcsActorSchedulerTest, TestLeasingCancelledWhenLeasing) { ASSERT_EQ(1, raylet_client_->callbacks.size()); // Cancel the lease request. - const auto &task_id = TaskID::FromBinary(create_actor_request.task_spec().task_id()); - gcs_actor_scheduler_->CancelOnLeasing(node_id, actor->GetActorID(), task_id); + gcs_actor_scheduler_->CancelOnLeasing( + node_id, actor->GetActorID(), actor->GetLeaseSpecification().LeaseId()); ASSERT_EQ(1, raylet_client_->num_workers_requested); ASSERT_EQ(1, raylet_client_->callbacks.size()); @@ -726,7 +726,7 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestScheduleFailedWithZeroNodeByG // are no available nodes. ASSERT_EQ(raylet_client_->num_workers_requested, 0); ASSERT_EQ(0, success_actors_.size()); - ASSERT_EQ(1, cluster_task_manager_->GetInfeasibleQueueSize()); + ASSERT_EQ(1, cluster_lease_manager_->GetInfeasibleQueueSize()); ASSERT_TRUE(actor->GetNodeID().IsNil()); } @@ -748,7 +748,7 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestNotEnoughClusterResources) { // are not enough cluster resources. ASSERT_EQ(raylet_client_->num_workers_requested, 0); ASSERT_EQ(0, success_actors_.size()); - ASSERT_EQ(1, cluster_task_manager_->GetInfeasibleQueueSize()); + ASSERT_EQ(1, cluster_lease_manager_->GetInfeasibleQueueSize()); ASSERT_TRUE(actor->GetNodeID().IsNil()); } @@ -761,7 +761,7 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestScheduleAndDestroyOneActor) { scheduling::NodeID scheduling_node_id(node->node_id()); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); const auto &cluster_resource_manager = - cluster_task_manager_->GetClusterResourceScheduler().GetClusterResourceManager(); + cluster_lease_manager_->GetClusterResourceScheduler().GetClusterResourceManager(); auto resource_view_before_scheduling = cluster_resource_manager.GetResourceView(); ASSERT_TRUE(resource_view_before_scheduling.contains(scheduling_node_id)); @@ -789,8 +789,8 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestScheduleAndDestroyOneActor) { // Reply the actor creation request, then the actor should be scheduled successfully. ASSERT_TRUE(worker_client_->ReplyPushTask()); ASSERT_EQ(0, worker_client_->GetNumCallbacks()); - ASSERT_EQ(0, cluster_task_manager_->GetInfeasibleQueueSize()); - ASSERT_EQ(0, cluster_task_manager_->GetPendingQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetInfeasibleQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetPendingQueueSize()); ASSERT_EQ(1, success_actors_.size()); ASSERT_EQ(actor, success_actors_.front()); ASSERT_EQ(actor->GetNodeID(), node_id); @@ -928,8 +928,8 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestScheduleRetryWhenLeasingByGcs // Reply the actor creation request, then the actor should be scheduled successfully. ASSERT_TRUE(worker_client_->ReplyPushTask()); ASSERT_EQ(0, worker_client_->GetNumCallbacks()); - ASSERT_EQ(0, cluster_task_manager_->GetInfeasibleQueueSize()); - ASSERT_EQ(0, cluster_task_manager_->GetPendingQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetInfeasibleQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetPendingQueueSize()); ASSERT_EQ(1, success_actors_.size()); ASSERT_EQ(actor, success_actors_.front()); ASSERT_EQ(actor->GetNodeID(), node_id); @@ -975,8 +975,8 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestScheduleRetryWhenCreatingByGc // Reply the actor creation request, then the actor should be scheduled successfully. ASSERT_TRUE(worker_client_->ReplyPushTask()); ASSERT_EQ(0, worker_client_->GetNumCallbacks()); - ASSERT_EQ(0, cluster_task_manager_->GetInfeasibleQueueSize()); - ASSERT_EQ(0, cluster_task_manager_->GetPendingQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetInfeasibleQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetPendingQueueSize()); ASSERT_EQ(1, success_actors_.size()); ASSERT_EQ(actor, success_actors_.front()); ASSERT_EQ(actor->GetNodeID(), node_id); @@ -1024,8 +1024,8 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestNodeFailedWhenLeasingByGcs) { ASSERT_EQ(0, gcs_actor_scheduler_->num_retry_leasing_count_); ASSERT_EQ(0, success_actors_.size()); - ASSERT_EQ(0, cluster_task_manager_->GetInfeasibleQueueSize()); - ASSERT_EQ(0, cluster_task_manager_->GetPendingQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetInfeasibleQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetPendingQueueSize()); } TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestLeasingCancelledWhenLeasingByGcs) { @@ -1048,8 +1048,8 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestLeasingCancelledWhenLeasingBy ASSERT_EQ(1, raylet_client_->callbacks.size()); // Cancel the lease request. - const auto &task_id = actor->GetCreationTaskSpecification().TaskId(); - gcs_actor_scheduler_->CancelOnLeasing(node_id, actor->GetActorID(), task_id); + gcs_actor_scheduler_->CancelOnLeasing( + node_id, actor->GetActorID(), actor->GetLeaseSpecification().LeaseId()); ASSERT_EQ(1, raylet_client_->num_workers_requested); ASSERT_EQ(1, raylet_client_->callbacks.size()); @@ -1064,8 +1064,8 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestLeasingCancelledWhenLeasingBy ASSERT_EQ(0, gcs_actor_scheduler_->num_retry_leasing_count_); ASSERT_EQ(0, success_actors_.size()); - ASSERT_EQ(0, cluster_task_manager_->GetInfeasibleQueueSize()); - ASSERT_EQ(0, cluster_task_manager_->GetPendingQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetInfeasibleQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetPendingQueueSize()); } TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestNodeFailedWhenCreatingByGcs) { @@ -1113,8 +1113,8 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestNodeFailedWhenCreatingByGcs) ASSERT_EQ(0, gcs_actor_scheduler_->num_retry_creating_count_); ASSERT_EQ(0, success_actors_.size()); - ASSERT_EQ(0, cluster_task_manager_->GetInfeasibleQueueSize()); - ASSERT_EQ(0, cluster_task_manager_->GetPendingQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetInfeasibleQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetPendingQueueSize()); } TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestWorkerFailedWhenCreatingByGcs) { @@ -1158,8 +1158,8 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestWorkerFailedWhenCreatingByGcs ASSERT_EQ(0, gcs_actor_scheduler_->num_retry_creating_count_); ASSERT_EQ(0, success_actors_.size()); - ASSERT_EQ(0, cluster_task_manager_->GetInfeasibleQueueSize()); - ASSERT_EQ(0, cluster_task_manager_->GetPendingQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetInfeasibleQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetPendingQueueSize()); } TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestRescheduleByGcs) { @@ -1213,8 +1213,8 @@ TEST_F(GcsActorSchedulerTestWithGcsScheduling, TestRescheduleByGcs) { ASSERT_TRUE(worker_client_->ReplyPushTask()); ASSERT_EQ(0, worker_client_->GetNumCallbacks()); - ASSERT_EQ(0, cluster_task_manager_->GetInfeasibleQueueSize()); - ASSERT_EQ(0, cluster_task_manager_->GetPendingQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetInfeasibleQueueSize()); + ASSERT_EQ(0, cluster_lease_manager_->GetPendingQueueSize()); ASSERT_EQ(2, success_actors_.size()); } diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h new file mode 100644 index 000000000000..839bbddd31d3 --- /dev/null +++ b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h @@ -0,0 +1,386 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/mutex.h" +#include "fakes/ray/rpc/raylet/raylet_client.h" +#include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/lease/lease.h" +#include "ray/common/task/task_util.h" +#include "ray/common/test_util.h" +#include "ray/gcs/gcs_server/gcs_actor_manager.h" +#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" +#include "ray/gcs/gcs_server/gcs_node_manager.h" +#include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" +#include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" +#include "ray/gcs/gcs_server/gcs_resource_manager.h" +#include "ray/gcs/store_client/in_memory_store_client.h" + +namespace ray { + +struct GcsServerMocker { + class MockWorkerClient : public rpc::CoreWorkerClientInterface { + public: + void PushNormalTask( + std::unique_ptr request, + const rpc::ClientCallback &callback) override { + absl::MutexLock lock(&mutex_); + callbacks_.push_back(callback); + } + + bool ReplyPushTask(Status status = Status::OK(), bool exit = false) { + rpc::ClientCallback callback = nullptr; + { + absl::MutexLock lock(&mutex_); + if (callbacks_.size() == 0) { + return false; + } + callback = callbacks_.front(); + callbacks_.pop_front(); + } + // call the callback without the lock to avoid deadlock. + auto reply = rpc::PushTaskReply(); + if (exit) { + reply.set_worker_exiting(true); + } + callback(status, std::move(reply)); + return true; + } + + size_t GetNumCallbacks() { + absl::MutexLock lock(&mutex_); + return callbacks_.size(); + } + + std::list> callbacks_ ABSL_GUARDED_BY(mutex_); + absl::Mutex mutex_; + }; + + class MockRayletClient : public FakeRayletClient { + public: + ray::Status ReturnWorkerLease(int worker_port, + const WorkerID &worker_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) override { + if (disconnect_worker) { + num_workers_disconnected++; + } else { + num_workers_returned++; + } + return Status::OK(); + } + + void GetWorkerFailureCause( + const LeaseID &lease_id, + const ray::rpc::ClientCallback &callback) + override { + ray::rpc::GetWorkerFailureCauseReply reply; + callback(Status::OK(), std::move(reply)); + num_get_task_failure_causes += 1; + } + + void RequestWorkerLease( + const rpc::LeaseSpec &spec, + bool grant_or_reject, + const rpc::ClientCallback &callback, + const int64_t backlog_size, + const bool is_selected_based_on_locality) override { + num_workers_requested += 1; + callbacks.push_back(callback); + } + + void PrestartWorkers( + const rpc::PrestartWorkersRequest &request, + const rpc::ClientCallback &callback) override { + RAY_LOG(FATAL) << "Not implemented"; + } + + void ReleaseUnusedActorWorkers( + const std::vector &workers_in_use, + const rpc::ClientCallback &callback) + override { + num_release_unused_workers += 1; + release_callbacks.push_back(callback); + } + + void CancelWorkerLease( + const LeaseID &lease_id, + const rpc::ClientCallback &callback) override { + num_leases_canceled += 1; + cancel_callbacks.push_back(callback); + } + + bool GrantWorkerLease() { + return GrantWorkerLease("", 0, WorkerID::FromRandom(), node_id_, NodeID::Nil()); + } + + bool GrantWorkerLease(const std::string &address, + int port, + const WorkerID &worker_id, + const NodeID &node_id, + const NodeID &retry_at_node_id, + Status status = Status::OK(), + bool rejected = false) { + rpc::RequestWorkerLeaseReply reply; + if (!retry_at_node_id.IsNil()) { + reply.mutable_retry_at_raylet_address()->set_ip_address(address); + reply.mutable_retry_at_raylet_address()->set_port(port); + reply.mutable_retry_at_raylet_address()->set_node_id(retry_at_node_id.Binary()); + } else { + reply.mutable_worker_address()->set_ip_address(address); + reply.mutable_worker_address()->set_port(port); + reply.mutable_worker_address()->set_node_id(node_id.Binary()); + reply.mutable_worker_address()->set_worker_id(worker_id.Binary()); + } + if (rejected) { + reply.set_rejected(true); + auto resources_data = reply.mutable_resources_data(); + resources_data->set_node_id(node_id.Binary()); + resources_data->set_resources_normal_task_changed(true); + auto &normal_task_map = *(resources_data->mutable_resources_normal_task()); + normal_task_map[kMemory_ResourceLabel] = + static_cast(std::numeric_limits::max()); + resources_data->set_resources_normal_task_timestamp(absl::GetCurrentTimeNanos()); + } + + if (callbacks.size() == 0) { + return false; + } else { + auto callback = callbacks.front(); + callback(status, std::move(reply)); + callbacks.pop_front(); + return true; + } + } + + bool ReplyCancelWorkerLease(bool success = true) { + rpc::CancelWorkerLeaseReply reply; + reply.set_success(success); + if (cancel_callbacks.size() == 0) { + return false; + } else { + auto callback = cancel_callbacks.front(); + callback(Status::OK(), std::move(reply)); + cancel_callbacks.pop_front(); + return true; + } + } + + bool ReplyReleaseUnusedActorWorkers() { + rpc::ReleaseUnusedActorWorkersReply reply; + if (release_callbacks.size() == 0) { + return false; + } else { + auto callback = release_callbacks.front(); + callback(Status::OK(), std::move(reply)); + release_callbacks.pop_front(); + return true; + } + } + + bool ReplyDrainRaylet() { + if (drain_raylet_callbacks.size() == 0) { + return false; + } else { + rpc::DrainRayletReply reply; + reply.set_is_accepted(true); + auto callback = drain_raylet_callbacks.front(); + callback(Status::OK(), std::move(reply)); + drain_raylet_callbacks.pop_front(); + return true; + } + } + + void PrepareBundleResources( + const std::vector> &bundle_specs, + const ray::rpc::ClientCallback &callback) + override { + num_lease_requested += 1; + lease_callbacks.push_back(callback); + } + + void CommitBundleResources( + const std::vector> &bundle_specs, + const ray::rpc::ClientCallback &callback) + override { + num_commit_requested += 1; + commit_callbacks.push_back(callback); + } + + void CancelResourceReserve( + const BundleSpecification &bundle_spec, + const ray::rpc::ClientCallback &callback) + override { + num_return_requested += 1; + return_callbacks.push_back(callback); + } + + void ReleaseUnusedBundles( + const std::vector &bundles_in_use, + const rpc::ClientCallback &callback) override { + ++num_release_unused_bundles_requested; + } + + bool GrantPrepareBundleResources(bool success = true, + const Status &status = Status::OK()) { + rpc::PrepareBundleResourcesReply reply; + reply.set_success(success); + if (lease_callbacks.size() == 0) { + return false; + } else { + auto callback = lease_callbacks.front(); + callback(status, std::move(reply)); + lease_callbacks.pop_front(); + return true; + } + } + + bool GrantCommitBundleResources(const Status &status = Status::OK()) { + rpc::CommitBundleResourcesReply reply; + if (commit_callbacks.size() == 0) { + return false; + } else { + auto callback = commit_callbacks.front(); + callback(status, std::move(reply)); + commit_callbacks.pop_front(); + return true; + } + } + + bool GrantCancelResourceReserve(bool success = true) { + Status status = Status::OK(); + rpc::CancelResourceReserveReply reply; + if (return_callbacks.size() == 0) { + return false; + } else { + auto callback = return_callbacks.front(); + callback(status, std::move(reply)); + return_callbacks.pop_front(); + return true; + } + } + + void DrainRaylet( + const rpc::autoscaler::DrainNodeReason &reason, + const std::string &reason_message, + int64_t deadline_timestamp_ms, + const rpc::ClientCallback &callback) override { + rpc::DrainRayletReply reply; + reply.set_is_accepted(true); + drain_raylet_callbacks.push_back(callback); + }; + + ~MockRayletClient() {} + + int num_workers_requested = 0; + int num_workers_returned = 0; + int num_workers_disconnected = 0; + int num_leases_canceled = 0; + int num_release_unused_workers = 0; + int num_get_task_failure_causes = 0; + NodeID node_id_ = NodeID::FromRandom(); + std::list> drain_raylet_callbacks = {}; + std::list> callbacks = {}; + std::list> cancel_callbacks = {}; + std::list> + release_callbacks = {}; + int num_lease_requested = 0; + int num_return_requested = 0; + int num_commit_requested = 0; + + int num_release_unused_bundles_requested = 0; + std::list> lease_callbacks = {}; + std::list> commit_callbacks = {}; + std::list> return_callbacks = {}; + }; + + class MockedGcsActorScheduler : public gcs::GcsActorScheduler { + public: + using gcs::GcsActorScheduler::GcsActorScheduler; + + void TryLeaseWorkerFromNodeAgain(std::shared_ptr actor, + std::shared_ptr node) { + DoRetryLeasingWorkerFromNode(std::move(actor), std::move(node)); + } + + protected: + void RetryLeasingWorkerFromNode(std::shared_ptr actor, + std::shared_ptr node) override { + ++num_retry_leasing_count_; + if (num_retry_leasing_count_ <= 1) { + DoRetryLeasingWorkerFromNode(actor, node); + } + } + + void RetryCreatingActorOnWorker(std::shared_ptr actor, + std::shared_ptr worker) override { + ++num_retry_creating_count_; + DoRetryCreatingActorOnWorker(actor, worker); + } + + public: + int num_retry_leasing_count_ = 0; + int num_retry_creating_count_ = 0; + }; + + class MockedGcsPlacementGroupScheduler : public gcs::GcsPlacementGroupScheduler { + public: + using gcs::GcsPlacementGroupScheduler::GcsPlacementGroupScheduler; + + size_t GetWaitingRemovedBundlesSize() { return waiting_removed_bundles_.size(); } + + using gcs::GcsPlacementGroupScheduler::ScheduleUnplacedBundles; + // Extra conveinence overload for the mock tests to keep using the old interface. + void ScheduleUnplacedBundles( + const std::shared_ptr &placement_group, + gcs::PGSchedulingFailureCallback failure_callback, + gcs::PGSchedulingSuccessfulCallback success_callback) { + ScheduleUnplacedBundles( + gcs::SchedulePgRequest{placement_group, failure_callback, success_callback}); + }; + + protected: + friend class GcsPlacementGroupSchedulerTest; + FRIEND_TEST(GcsPlacementGroupSchedulerTest, TestCheckingWildcardResource); + }; + class MockedGcsActorTable : public gcs::GcsActorTable { + public: + // The store_client and io_context args are NOT used. + explicit MockedGcsActorTable(std::shared_ptr store_client) + : GcsActorTable(store_client) {} + + Status Put(const ActorID &key, + const rpc::ActorTableData &value, + Postable callback) override { + auto status = Status::OK(); + std::move(callback).Post("FakeGcsActorTable.Put", status); + return status; + } + + private: + std::shared_ptr store_client_ = + std::make_shared(); + }; +}; + +} // namespace ray diff --git a/src/ray/gcs/tests/gcs_test_util.h b/src/ray/gcs/tests/gcs_test_util.h index d820eae87261..cbe7486a04c2 100644 --- a/src/ray/gcs/tests/gcs_test_util.h +++ b/src/ray/gcs/tests/gcs_test_util.h @@ -26,7 +26,6 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/bundle_spec.h" #include "ray/common/placement_group.h" -#include "ray/common/task/task.h" #include "ray/common/task/task_util.h" #include "ray/common/test_util.h" #include "ray/gcs/pb_util.h" diff --git a/src/ray/protobuf/common.proto b/src/ray/protobuf/common.proto index 84d91cf599b9..410e6bfd5e05 100644 --- a/src/ray/protobuf/common.proto +++ b/src/ray/protobuf/common.proto @@ -456,6 +456,31 @@ message StreamingGeneratorReturnIdInfo { bool is_plasma_object = 2; } +message LeaseSpec { + bytes lease_id = 1; + bytes job_id = 2; + Address caller_address = 3; + TaskType type = 4; + bytes actor_id = 5; + bool is_detached_actor = 6; + bytes root_detached_actor_id = 7; + int64 max_actor_restarts = 8; + map required_resources = 9; + map required_placement_resources = 10; + SchedulingStrategy scheduling_strategy = 11; + map label_selector = 12; + int64 depth = 13; + RuntimeEnvInfo runtime_env_info = 14; + repeated ObjectReference dependencies = 15; + bytes parent_task_id = 16; + Language language = 17; + FunctionDescriptor function_descriptor = 18; + repeated string dynamic_worker_options = 19; + int32 max_retries = 20; + uint64 attempt_number = 21; + string task_name = 22; +} + /// The task specification encapsulates all immutable information about the /// task. message TaskSpec { diff --git a/src/ray/protobuf/node_manager.proto b/src/ray/protobuf/node_manager.proto index 0585ad27c087..624edac21496 100644 --- a/src/ray/protobuf/node_manager.proto +++ b/src/ray/protobuf/node_manager.proto @@ -22,10 +22,10 @@ import "src/ray/protobuf/autoscaler.proto"; import "src/ray/protobuf/public/runtime_environment.proto"; message WorkerBacklogReport { - // TaskSpec indicating the scheduling class. + // LeaseSpec indicating the scheduling class. // Cannot send scheduling class directly // since it's local to each process. - TaskSpec resource_spec = 1; + LeaseSpec lease_spec = 1; // Size of the backlog for the above scheduling class. int64 backlog_size = 2; } @@ -41,8 +41,8 @@ message ReportWorkerBacklogReply {} // Request a worker from the raylet with the specified resources. message RequestWorkerLeaseRequest { - // TaskSpec containing the requested resources. - TaskSpec resource_spec = 1; + // LeaseSpec containing the requested resources. + LeaseSpec lease_spec = 1; // Worker's backlog size for this spec's shape. int64 backlog_size = 2; // If it's true, either grant the lease if the task is @@ -144,10 +144,10 @@ message ResizeLocalResourceInstancesReply { } // Release a worker back to its raylet. -message ReturnWorkerRequest { +message ReturnWorkerLeaseRequest { // Port of the leased worker that we are now returning. int32 worker_port = 1; - // Unique id of the leased worker we are now returning. + // The worker id of the lease we are now returning. bytes worker_id = 2; // If true, there was some unrecoverable error and the raylet should // disconnect the worker. @@ -158,7 +158,7 @@ message ReturnWorkerRequest { string disconnect_worker_error_detail = 5; } -message ReturnWorkerReply {} +message ReturnWorkerLeaseReply {} message ReleaseUnusedActorWorkersRequest { repeated bytes worker_ids_in_use = 1; @@ -174,8 +174,8 @@ message ShutdownRayletRequest { message ShutdownRayletReply {} message CancelWorkerLeaseRequest { - // The task to cancel. - bytes task_id = 1; + // The lease to cancel. + bytes lease_id = 1; } message CancelWorkerLeaseReply { @@ -308,7 +308,7 @@ message GetResourceLoadReply { ResourcesData resources = 1; } -message CancelTasksWithResourceShapesRequest { +message CancelLeasesWithResourceShapesRequest { message ResourceShape { // A map from resource name to the quantity of that resource. This map represents // the resource request shape of a task. @@ -318,7 +318,7 @@ message CancelTasksWithResourceShapesRequest { repeated ResourceShape resource_shapes = 1; } -message CancelTasksWithResourceShapesReply { +message CancelLeasesWithResourceShapesReply { // Empty } @@ -326,11 +326,11 @@ message NotifyGCSRestartRequest {} message NotifyGCSRestartReply {} -message GetTaskFailureCauseRequest { - bytes task_id = 1; +message GetWorkerFailureCauseRequest { + bytes lease_id = 1; } -message GetTaskFailureCauseReply { +message GetWorkerFailureCauseReply { optional RayErrorInfo failure_cause = 1; bool fail_task_immediately = 2; } @@ -414,8 +414,8 @@ service NodeManagerService { // request. // Failure: This doesn't explicitly retry, only logs on failure, but autoscaler will // keep calling this so it will be retried at a layer above. - rpc CancelTasksWithResourceShapes(CancelTasksWithResourceShapesRequest) - returns (CancelTasksWithResourceShapesReply); + rpc CancelLeasesWithResourceShapes(CancelLeasesWithResourceShapesRequest) + returns (CancelLeasesWithResourceShapesReply); // Request a worker from the raylet. // Failure: Does retry if request to remote raylet fails. Just logs warning if request // to local raylet fails. @@ -430,7 +430,7 @@ service NodeManagerService { rpc ReportWorkerBacklog(ReportWorkerBacklogRequest) returns (ReportWorkerBacklogReply); // Release a worker back to its raylet. // Failure: TODO: Failure behavior needs to be fixed. - rpc ReturnWorker(ReturnWorkerRequest) returns (ReturnWorkerReply); + rpc ReturnWorkerLease(ReturnWorkerLeaseRequest) returns (ReturnWorkerLeaseReply); // This method is only used by GCS, and the purpose is to release leased workers // that may be leaked. When GCS restarts, it doesn't know which workers it has leased // in the previous lifecycle. In this case, GCS will send a list of worker ids that @@ -500,10 +500,11 @@ service NodeManagerService { // [State API] Get the all object information of the node. // Failure: State API user can retry. rpc GetObjectsInfo(GetObjectsInfoRequest) returns (GetObjectsInfoReply); - // Gets the task execution result. May contain a result if - // the task completed in error. + // Gets the worker failure cause. May contain a result if + // the worker executing the task failed. // Failure: Gives user error message on failure. - rpc GetTaskFailureCause(GetTaskFailureCauseRequest) returns (GetTaskFailureCauseReply); + rpc GetWorkerFailureCause(GetWorkerFailureCauseRequest) + returns (GetWorkerFailureCauseReply); // Failure: TODO: Handle network failure for cgraphs. rpc RegisterMutableObject(RegisterMutableObjectRequest) returns (RegisterMutableObjectReply); diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index 371381a182bb..aaabeefac51d 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -18,13 +18,13 @@ ray_cc_library( ) ray_cc_library( - name = "dependency_manager", - srcs = ["dependency_manager.cc"], - hdrs = ["dependency_manager.h"], + name = "lease_dependency_manager", + srcs = ["lease_dependency_manager.cc"], + hdrs = ["lease_dependency_manager.h"], visibility = [":__subpackages__"], deps = [ "//src/ray/common:id", - "//src/ray/common:task_common", + "//src/ray/common:lease", "//src/ray/object_manager", "//src/ray/util:counter_map", "@com_google_absl//absl/container:flat_hash_map", @@ -34,19 +34,19 @@ ray_cc_library( # TODO(edoakes): looks like this belongs under scheduling/... ray_cc_library( - name = "local_task_manager", - srcs = ["local_task_manager.cc"], - hdrs = ["local_task_manager.h"], + name = "local_lease_manager", + srcs = ["local_lease_manager.cc"], + hdrs = ["local_lease_manager.h"], visibility = [":__subpackages__"], deps = [ - ":dependency_manager", + ":lease_dependency_manager", ":worker", ":worker_pool", + "//src/ray/common:lease", "//src/ray/common:ray_object", - "//src/ray/common:task_common", "//src/ray/object_manager:object_manager_common", "//src/ray/raylet/scheduling:cluster_resource_scheduler", - "//src/ray/raylet/scheduling:local_task_manager_interface", + "//src/ray/raylet/scheduling:local_lease_manager_interface", "//src/ray/raylet/scheduling:scheduler_internal", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -60,7 +60,6 @@ ray_cc_library( visibility = [":__subpackages__"], deps = [ "//src/ray/common:id", - "//src/ray/common:task_common", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "//src/ray/util:container_util", @@ -86,7 +85,7 @@ ray_cc_library( visibility = [":__subpackages__"], deps = [ "//src/ray/common:id", - "//src/ray/common:task_common", + "//src/ray/common:lease", "//src/ray/flatbuffers:node_manager_generated", "//src/ray/ipc:client_connection", "//src/ray/raylet/scheduling:cluster_resource_scheduler", @@ -107,10 +106,10 @@ ray_cc_library( ":runtime_env_agent_client", ":worker", "//src/ray/common:constants", + "//src/ray/common:lease", "//src/ray/common:ray_config", "//src/ray/common:runtime_env", "//src/ray/common:status", - "//src/ray/common:task_common", "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/ipc:client_connection", @@ -216,9 +215,9 @@ ray_cc_library( visibility = [":__subpackages__"], deps = [ ":agent_manager", - ":dependency_manager", + ":lease_dependency_manager", + ":local_lease_manager", ":local_object_manager_interface", - ":local_task_manager", ":placement_group_resource_manager", ":runtime_env_agent_client", ":wait_manager", @@ -226,6 +225,7 @@ ray_cc_library( ":worker_killing_policy", ":worker_pool", "//src/ray/common:buffer", + "//src/ray/common:lease", "//src/ray/common:memory_monitor", "//src/ray/core_worker:experimental_mutable_object_provider", "//src/ray/flatbuffers:node_manager_generated", @@ -279,13 +279,13 @@ ray_cc_binary( ":local_object_manager_interface", ":raylet_lib", "//src/ray/common:asio", + "//src/ray/common:lease", "//src/ray/common:ray_config", "//src/ray/common:status", - "//src/ray/common:task_common", "//src/ray/common/cgroup:cgroup_manager", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/object_manager:ownership_object_directory", - "//src/ray/raylet/scheduling:cluster_task_manager", + "//src/ray/raylet/scheduling:cluster_lease_manager", "//src/ray/rpc:metrics_agent_client", "//src/ray/stats:stats_lib", "//src/ray/util:cmd_line_utils", diff --git a/src/ray/raylet/dependency_manager.cc b/src/ray/raylet/lease_dependency_manager.cc similarity index 65% rename from src/ray/raylet/dependency_manager.cc rename to src/ray/raylet/lease_dependency_manager.cc index 359f399f5a95..1cf2b602f381 100644 --- a/src/ray/raylet/dependency_manager.cc +++ b/src/ray/raylet/lease_dependency_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/raylet/dependency_manager.h" +#include "ray/raylet/lease_dependency_manager.h" #include #include @@ -23,12 +23,12 @@ namespace ray { namespace raylet { -bool DependencyManager::CheckObjectLocal(const ObjectID &object_id) const { +bool LeaseDependencyManager::CheckObjectLocal(const ObjectID &object_id) const { return local_objects_.count(object_id) == 1; } -bool DependencyManager::GetOwnerAddress(const ObjectID &object_id, - rpc::Address *owner_address) const { +bool LeaseDependencyManager::GetOwnerAddress(const ObjectID &object_id, + rpc::Address *owner_address) const { auto obj = required_objects_.find(object_id); if (obj == required_objects_.end()) { return false; @@ -38,8 +38,8 @@ bool DependencyManager::GetOwnerAddress(const ObjectID &object_id, return !owner_address->worker_id().empty(); } -void DependencyManager::RemoveObjectIfNotNeeded( - absl::flat_hash_map::iterator +void LeaseDependencyManager::RemoveObjectIfNotNeeded( + absl::flat_hash_map::iterator required_object_it) { const auto &object_id = required_object_it->first; if (required_object_it->second.Empty()) { @@ -53,9 +53,9 @@ void DependencyManager::RemoveObjectIfNotNeeded( } } -absl::flat_hash_map::iterator -DependencyManager::GetOrInsertRequiredObject(const ObjectID &object_id, - const rpc::ObjectReference &ref) { +absl::flat_hash_map::iterator +LeaseDependencyManager::GetOrInsertRequiredObject(const ObjectID &object_id, + const rpc::ObjectReference &ref) { auto it = required_objects_.find(object_id); if (it == required_objects_.end()) { it = required_objects_.emplace(object_id, ref).first; @@ -63,7 +63,7 @@ DependencyManager::GetOrInsertRequiredObject(const ObjectID &object_id, return it; } -void DependencyManager::StartOrUpdateWaitRequest( +void LeaseDependencyManager::StartOrUpdateWaitRequest( const WorkerID &worker_id, const std::vector &required_objects) { RAY_LOG(DEBUG) << "Starting wait request for worker " << worker_id; @@ -95,7 +95,7 @@ void DependencyManager::StartOrUpdateWaitRequest( } } -void DependencyManager::CancelWaitRequest(const WorkerID &worker_id) { +void LeaseDependencyManager::CancelWaitRequest(const WorkerID &worker_id) { RAY_LOG(DEBUG) << "Canceling wait request for worker " << worker_id; auto req_iter = wait_requests_.find(worker_id); if (req_iter == wait_requests_.end()) { @@ -112,7 +112,7 @@ void DependencyManager::CancelWaitRequest(const WorkerID &worker_id) { wait_requests_.erase(req_iter); } -void DependencyManager::StartOrUpdateGetRequest( +void LeaseDependencyManager::StartOrUpdateGetRequest( const WorkerID &worker_id, const std::vector &required_objects) { RAY_LOG(DEBUG) << "Starting get request for worker " << worker_id; @@ -150,7 +150,7 @@ void DependencyManager::StartOrUpdateGetRequest( } } -void DependencyManager::CancelGetRequest(const WorkerID &worker_id) { +void LeaseDependencyManager::CancelGetRequest(const WorkerID &worker_id) { RAY_LOG(DEBUG) << "Canceling get request for worker " << worker_id; auto req_iter = get_requests_.find(worker_id); if (req_iter == get_requests_.end()) { @@ -171,120 +171,121 @@ void DependencyManager::CancelGetRequest(const WorkerID &worker_id) { get_requests_.erase(req_iter); } -/// Request dependencies for a queued task. -bool DependencyManager::RequestTaskDependencies( - const TaskID &task_id, +/// Request dependencies for a queued lease. +bool LeaseDependencyManager::RequestLeaseDependencies( + const LeaseID &lease_id, const std::vector &required_objects, const TaskMetricsKey &task_key) { - RAY_LOG(DEBUG) << "Adding dependencies for task " << task_id + RAY_LOG(DEBUG) << "Adding dependencies for lease " << lease_id << ". Required objects length: " << required_objects.size(); const auto required_ids = ObjectRefsToIds(required_objects); absl::flat_hash_set deduped_ids(required_ids.begin(), required_ids.end()); - auto inserted = queued_task_requests_.emplace( - task_id, - std::make_unique( - std::move(deduped_ids), waiting_tasks_counter_, task_key)); - RAY_CHECK(inserted.second) << "Task depedencies can be requested only once per task. " - << task_id; - auto &task_entry = inserted.first->second; + auto inserted = queued_lease_requests_.emplace( + lease_id, + std::make_unique( + std::move(deduped_ids), waiting_leases_counter_, task_key)); + RAY_CHECK(inserted.second) << "Lease depedencies can be requested only once per lease. " + << lease_id; + auto &lease_entry = inserted.first->second; for (const auto &ref : required_objects) { const auto obj_id = ObjectRefToId(ref); - RAY_LOG(DEBUG) << "Task " << task_id << " blocked on object " << obj_id; + RAY_LOG(DEBUG).WithField(lease_id).WithField(obj_id) << "Lease blocked on object"; auto it = GetOrInsertRequiredObject(obj_id, ref); - it->second.dependent_tasks.insert(task_id); + it->second.dependent_leases.insert(lease_id); } - for (const auto &obj_id : task_entry->dependencies_) { + for (const auto &obj_id : lease_entry->dependencies_) { if (local_objects_.count(obj_id)) { - task_entry->DecrementMissingDependencies(); + lease_entry->DecrementMissingDependencies(); } } if (!required_objects.empty()) { - task_entry->pull_request_id_ = + lease_entry->pull_request_id_ = object_manager_.Pull(required_objects, BundlePriority::TASK_ARGS, task_key); - RAY_LOG(DEBUG) << "Started pull for dependencies of task " << task_id - << " request: " << task_entry->pull_request_id_; + RAY_LOG(DEBUG) << "Started pull for dependencies of lease " << lease_id + << " request: " << lease_entry->pull_request_id_; } - return task_entry->num_missing_dependencies_ == 0; + return lease_entry->num_missing_dependencies_ == 0; } -void DependencyManager::RemoveTaskDependencies(const TaskID &task_id) { - RAY_LOG(DEBUG) << "Removing dependencies for task " << task_id; - auto task_entry = queued_task_requests_.find(task_id); - RAY_CHECK(task_entry != queued_task_requests_.end()) +void LeaseDependencyManager::RemoveLeaseDependencies(const LeaseID &lease_id) { + RAY_LOG(DEBUG) << "Removing dependencies for lease " << lease_id; + auto lease_entry = queued_lease_requests_.find(lease_id); + RAY_CHECK(lease_entry != queued_lease_requests_.end()) << "Can't remove dependencies of tasks that are not queued."; - if (task_entry->second->pull_request_id_ > 0) { - RAY_LOG(DEBUG) << "Canceling pull for dependencies of task " << task_id - << " request: " << task_entry->second->pull_request_id_; - object_manager_.CancelPull(task_entry->second->pull_request_id_); + if (lease_entry->second->pull_request_id_ > 0) { + RAY_LOG(DEBUG) << "Canceling pull for dependencies of lease " << lease_id + << " request: " << lease_entry->second->pull_request_id_; + object_manager_.CancelPull(lease_entry->second->pull_request_id_); } - for (const auto &obj_id : task_entry->second->dependencies_) { + for (const auto &obj_id : lease_entry->second->dependencies_) { auto it = required_objects_.find(obj_id); RAY_CHECK(it != required_objects_.end()); - it->second.dependent_tasks.erase(task_id); + it->second.dependent_leases.erase(lease_id); RemoveObjectIfNotNeeded(it); } - queued_task_requests_.erase(task_entry); + queued_lease_requests_.erase(lease_entry); } -std::vector DependencyManager::HandleObjectMissing( +std::vector LeaseDependencyManager::HandleObjectMissing( const ray::ObjectID &object_id) { RAY_CHECK(local_objects_.erase(object_id)) << "Evicted object was not local " << object_id; - // Find any tasks that are dependent on the missing object. - std::vector waiting_task_ids; + // Find any leases that are dependent on the missing object. + std::vector waiting_lease_ids; auto object_entry = required_objects_.find(object_id); if (object_entry != required_objects_.end()) { - for (auto &dependent_task_id : object_entry->second.dependent_tasks) { - auto it = queued_task_requests_.find(dependent_task_id); - RAY_CHECK(it != queued_task_requests_.end()); - auto &task_entry = it->second; - // If the dependent task had all of its arguments ready, it was ready to + for (auto &dependent_lease_id : object_entry->second.dependent_leases) { + auto it = queued_lease_requests_.find(dependent_lease_id); + RAY_CHECK(it != queued_lease_requests_.end()); + auto &lease_entry = it->second; + // If the dependent lease had all of its arguments ready, it was ready to // run but must be switched to waiting since one of its arguments is now // missing. - if (task_entry->num_missing_dependencies_ == 0) { - waiting_task_ids.push_back(dependent_task_id); + if (lease_entry->num_missing_dependencies_ == 0) { + waiting_lease_ids.push_back(dependent_lease_id); // During normal execution we should be able to include the check - // RAY_CHECK(pending_tasks_.count(dependent_task_id) == 1); + // RAY_CHECK(pending_leases_.count(dependent_lease_id) == 1); // However, this invariant will not hold during unit test execution. } - task_entry->IncrementMissingDependencies(); + lease_entry->IncrementMissingDependencies(); } } - // Process callbacks for all of the tasks dependent on the object that are + // Process callbacks for all of the leases dependent on the object that are // now ready to run. - return waiting_task_ids; + return waiting_lease_ids; } -std::vector DependencyManager::HandleObjectLocal(const ray::ObjectID &object_id) { +std::vector LeaseDependencyManager::HandleObjectLocal( + const ray::ObjectID &object_id) { // Add the object to the table of locally available objects. auto inserted = local_objects_.insert(object_id); RAY_CHECK(inserted.second) << "Local object was already local " << object_id; - // Find all tasks and workers that depend on the newly available object. - std::vector ready_task_ids; + // Find all leases and workers that depend on the newly available object. + std::vector ready_lease_ids; auto object_entry = required_objects_.find(object_id); if (object_entry != required_objects_.end()) { - // Loop through all tasks that depend on the newly available object. - for (const auto &dependent_task_id : object_entry->second.dependent_tasks) { - auto it = queued_task_requests_.find(dependent_task_id); - RAY_CHECK(it != queued_task_requests_.end()); - auto &task_entry = it->second; - task_entry->DecrementMissingDependencies(); - // If the dependent task now has all of its arguments ready, it's ready + // Loop through all leases that depend on the newly available object. + for (const auto &dependent_lease_id : object_entry->second.dependent_leases) { + auto it = queued_lease_requests_.find(dependent_lease_id); + RAY_CHECK(it != queued_lease_requests_.end()); + auto &lease_entry = it->second; + lease_entry->DecrementMissingDependencies(); + // If the dependent lease now has all of its arguments ready, it's ready // to run. - if (task_entry->num_missing_dependencies_ == 0) { - ready_task_ids.push_back(dependent_task_id); + if (lease_entry->num_missing_dependencies_ == 0) { + ready_lease_ids.push_back(dependent_lease_id); } } @@ -310,29 +311,29 @@ std::vector DependencyManager::HandleObjectLocal(const ray::ObjectID &ob RemoveObjectIfNotNeeded(object_entry); } - return ready_task_ids; + return ready_lease_ids; } -bool DependencyManager::TaskDependenciesBlocked(const TaskID &task_id) const { - auto it = queued_task_requests_.find(task_id); - RAY_CHECK(it != queued_task_requests_.end()); +bool LeaseDependencyManager::LeaseDependenciesBlocked(const LeaseID &lease_id) const { + auto it = queued_lease_requests_.find(lease_id); + RAY_CHECK(it != queued_lease_requests_.end()); RAY_CHECK(it->second->pull_request_id_ != 0); return !object_manager_.PullRequestActiveOrWaitingForMetadata( it->second->pull_request_id_); } -std::string DependencyManager::DebugString() const { +std::string LeaseDependencyManager::DebugString() const { std::stringstream result; - result << "TaskDependencyManager:"; - result << "\n- task deps map size: " << queued_task_requests_.size(); + result << "LeaseDependencyManager:"; + result << "\n- lease deps map size: " << queued_lease_requests_.size(); result << "\n- get req map size: " << get_requests_.size(); result << "\n- wait req map size: " << wait_requests_.size(); result << "\n- local objects map size: " << local_objects_.size(); return result.str(); } -void DependencyManager::RecordMetrics() { - waiting_tasks_counter_.FlushOnChangeCallbacks(); +void LeaseDependencyManager::RecordMetrics() { + waiting_leases_counter_.FlushOnChangeCallbacks(); } } // namespace raylet diff --git a/src/ray/raylet/dependency_manager.h b/src/ray/raylet/lease_dependency_manager.h similarity index 73% rename from src/ray/raylet/dependency_manager.h rename to src/ray/raylet/lease_dependency_manager.h index 01ac2ec5ff53..7c051e4bbbfe 100644 --- a/src/ray/raylet/dependency_manager.h +++ b/src/ray/raylet/lease_dependency_manager.h @@ -24,7 +24,7 @@ #include "absl/container/flat_hash_set.h" #include "ray/common/common_protocol.h" #include "ray/common/id.h" -#include "ray/common/task/task.h" +#include "ray/common/lease/lease.h" #include "ray/object_manager/object_manager.h" #include "ray/util/counter_map.h" @@ -32,36 +32,36 @@ namespace ray { namespace raylet { -/// Used for unit-testing the ClusterTaskManager, which requests dependencies -/// for queued tasks. -class TaskDependencyManagerInterface { +/// Used for unit-testing the ClusterLeaseManager, which requests dependencies +/// for queued leases. +class LeaseDependencyManagerInterface { public: - virtual bool RequestTaskDependencies( - const TaskID &task_id, + virtual bool RequestLeaseDependencies( + const LeaseID &lease_id, const std::vector &required_objects, - const TaskMetricsKey &task_key) = 0; - virtual void RemoveTaskDependencies(const TaskID &task_id) = 0; - virtual bool TaskDependenciesBlocked(const TaskID &task_id) const = 0; + const TaskMetricsKey &lease_key) = 0; + virtual void RemoveLeaseDependencies(const LeaseID &lease_id) = 0; + virtual bool LeaseDependenciesBlocked(const LeaseID &lease_id) const = 0; virtual bool CheckObjectLocal(const ObjectID &object_id) const = 0; - virtual ~TaskDependencyManagerInterface(){}; + virtual ~LeaseDependencyManagerInterface(){}; }; -/// \class DependencyManager +/// \class LeaseDependencyManager /// /// Responsible for managing object dependencies for local workers calling /// `ray.get` or `ray.wait` and arguments of queued tasks. The caller can -/// request object dependencies for a task or worker. The task manager will +/// request object dependencies for a lease or worker. The lease manager will /// determine which object dependencies are remote and will request that these /// objects be made available locally, either via the object manager or by /// storing an error if the object is lost. -class DependencyManager : public TaskDependencyManagerInterface { +class LeaseDependencyManager : public LeaseDependencyManagerInterface { public: - /// Create a task dependency manager. - explicit DependencyManager(ObjectManagerInterface &object_manager) + /// Create a lease dependency manager. + explicit LeaseDependencyManager(ObjectManagerInterface &object_manager) : object_manager_(object_manager) { - waiting_tasks_counter_.SetOnChangeCallback( + waiting_leases_counter_.SetOnChangeCallback( [this](std::pair key) mutable { - int64_t num_total = waiting_tasks_counter_.Get(key); + int64_t num_total = waiting_leases_counter_.Get(key); // Of the waiting tasks of this name, some fraction may be inactive (blocked on // object store memory availability). Get this breakdown by querying the pull // manager. @@ -141,54 +141,54 @@ class DependencyManager : public TaskDependencyManagerInterface { const std::vector &required_objects); /// Cancel a worker's `ray.get` request. We will no longer attempt to fetch - /// any objects that this worker requested previously, if no other task or + /// any objects that this worker requested previously, if no other lease or /// worker requires them. /// /// \param worker_id The ID of the worker whose `ray.get` request we should /// cancel. void CancelGetRequest(const WorkerID &worker_id); - /// Request dependencies for a queued task. This will attempt to make any - /// remote objects local until the caller cancels the task's dependencies. + /// Request dependencies for a queued lease. This will attempt to make any + /// remote objects local until the caller cancels the lease's dependencies. /// - /// This method can only be called once per task, until the task has been + /// This method can only be called once per lease, until the lease has been /// canceled. /// - /// \param task_id The task that requires the objects. - /// \param required_objects The objects required by the task. - bool RequestTaskDependencies(const TaskID &task_id, - const std::vector &required_objects, - const TaskMetricsKey &task_key); - - /// Cancel a task's dependencies. We will no longer attempt to fetch any - /// remote dependencies, if no other task or worker requires them. + /// \param lease_id The lease that requires the objects. + /// \param required_objects The objects required by the lease. + bool RequestLeaseDependencies(const LeaseID &lease_id, + const std::vector &required_objects, + const TaskMetricsKey &task_key); + + /// Cancel a lease's dependencies. We will no longer attempt to fetch any + /// remote dependencies, if no other lease or worker requires them. /// - /// This method can only be called on a task whose dependencies were added. + /// This method can only be called on a lease whose dependencies were added. /// - /// \param task_id The task that requires the objects. - /// \param required_objects The objects required by the task. - void RemoveTaskDependencies(const TaskID &task_id); + /// \param lease_id The lease that requires the objects. + /// \param required_objects The objects required by the lease. + void RemoveLeaseDependencies(const LeaseID &lease_id); /// Handle an object becoming locally available. /// /// \param object_id The object ID of the object to mark as locally /// available. - /// \return A list of task IDs. This contains all added tasks that now have + /// \return A list of lease IDs. This contains all granted leases that now have /// all of their dependencies fulfilled. - std::vector HandleObjectLocal(const ray::ObjectID &object_id); + std::vector HandleObjectLocal(const ray::ObjectID &object_id); /// Handle an object that is no longer locally available. /// /// \param object_id The object ID of the object that was previously locally /// available. - /// \return A list of task IDs. This contains all added tasks that previously + /// \return A list of lease IDs. This contains all granted leases that previously /// had all of their dependencies fulfilled, but are now missing this object /// dependency. - std::vector HandleObjectMissing(const ray::ObjectID &object_id); + std::vector HandleObjectMissing(const ray::ObjectID &object_id); - /// Check whether a requested task's dependencies are not being fetched to + /// Check whether a requested lease's dependencies are not being fetched to /// the local node due to lack of memory. - bool TaskDependenciesBlocked(const TaskID &task_id) const; + bool LeaseDependenciesBlocked(const LeaseID &lease_id) const; /// Returns debug string for class. /// @@ -200,13 +200,13 @@ class DependencyManager : public TaskDependencyManagerInterface { private: /// Metadata for an object that is needed by at least one executing worker - /// and/or one queued task. + /// and/or one queued lease. struct ObjectDependencies { explicit ObjectDependencies(const rpc::ObjectReference &ref) : owner_address(ref.owner_address()) {} - /// The tasks that depend on this object, either because the object is a task argument - /// or because the task called `ray.get` on the object. - std::unordered_set dependent_tasks; + /// The leases that depend on this object, either because the object is a lease + /// argument or because the lease of the lease called `ray.get` on the object. + std::unordered_set dependent_leases; /// The workers that depend on this object because they called `ray.get` on the /// object. std::unordered_set dependent_get_requests; @@ -220,16 +220,16 @@ class DependencyManager : public TaskDependencyManagerInterface { rpc::Address owner_address; bool Empty() const { - return dependent_tasks.empty() && dependent_get_requests.empty() && + return dependent_leases.empty() && dependent_get_requests.empty() && dependent_wait_requests.empty(); } }; /// A struct to represent the object dependencies of a task. - struct TaskDependencies { - TaskDependencies(const absl::flat_hash_set &deps, - CounterMap> &counter_map, - const TaskMetricsKey &task_key) + struct LeaseDependencies { + LeaseDependencies(const absl::flat_hash_set &deps, + CounterMap> &counter_map, + const TaskMetricsKey &task_key) : dependencies_(std::move(deps)), num_missing_dependencies_(dependencies_.size()), waiting_task_counter_map_(counter_map), @@ -238,8 +238,8 @@ class DependencyManager : public TaskDependencyManagerInterface { waiting_task_counter_map_.Increment(task_key_); } } - /// The objects that the task depends on. These are the arguments to the - /// task. These must all be simultaneously local before the task is ready + /// The objects that the lease depends on. These are the arguments to the + /// lease. These must all be simultaneously local before the lease is ready /// to execute. Objects are removed from this set once /// UnsubscribeGetDependencies is called. absl::flat_hash_set dependencies_; @@ -268,7 +268,7 @@ class DependencyManager : public TaskDependencyManagerInterface { } } - ~TaskDependencies() { + ~LeaseDependencies() { if (num_missing_dependencies_ > 0) { waiting_task_counter_map_.Decrement(task_key_); } @@ -280,16 +280,16 @@ class DependencyManager : public TaskDependencyManagerInterface { void RemoveObjectIfNotNeeded( absl::flat_hash_map::iterator required_object_it); - /// Start tracking an object that is needed by a worker and/or queued task. + /// Start tracking an object that is needed by a worker and/or queued lease. absl::flat_hash_map::iterator GetOrInsertRequiredObject( const ObjectID &object_id, const rpc::ObjectReference &ref); /// The object manager, used to fetch required objects from remote nodes. ObjectManagerInterface &object_manager_; - /// A map from the ID of a queued task to metadata about whether the task's + /// A map from the ID of a queued lease to metadata about whether the lease's /// dependencies are all local or not. - absl::flat_hash_map> queued_task_requests_; + absl::flat_hash_map> queued_lease_requests_; /// A map from worker ID to the set of objects that the worker called /// `ray.get` on and a pull request ID for these objects. The pull request ID @@ -303,20 +303,20 @@ class DependencyManager : public TaskDependencyManagerInterface { /// or the worker cancels the `ray.wait` request. absl::flat_hash_map> wait_requests_; - /// Deduplicated pool of objects required by all queued tasks and workers. - /// Objects are removed from this set once there are no more tasks or workers + /// Deduplicated pool of objects required by all queued leases and workers. + /// Objects are removed from this set once there are no more leases or workers /// that require it. absl::flat_hash_map required_objects_; /// The set of locally available objects. This is used to determine which - /// tasks are ready to run and which `ray.wait` requests can be finished. + /// leases are ready to run and which `ray.wait` requests can be finished. std::unordered_set local_objects_; - /// Counts the number of active task dependency fetches by task name. The counter - /// total will be less than or equal to the size of queued_task_requests_. - CounterMap waiting_tasks_counter_; + /// Counts the number of active lease dependency fetches by lease name. The counter + /// total will be less than or equal to the size of queued_lease_requests_. + CounterMap waiting_leases_counter_; - friend class DependencyManagerTest; + friend class LeaseDependencyManagerTest; }; } // namespace raylet diff --git a/src/ray/raylet/local_task_manager.cc b/src/ray/raylet/local_lease_manager.cc similarity index 57% rename from src/ray/raylet/local_task_manager.cc rename to src/ray/raylet/local_lease_manager.cc index cf1956fa91dc..f1d516d3a642 100644 --- a/src/ray/raylet/local_task_manager.cc +++ b/src/ray/raylet/local_lease_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/raylet/local_task_manager.h" +#include "ray/raylet/local_lease_manager.h" #include @@ -31,163 +31,162 @@ namespace ray { namespace raylet { -LocalTaskManager::LocalTaskManager( +LocalLeaseManager::LocalLeaseManager( const NodeID &self_node_id, ClusterResourceScheduler &cluster_resource_scheduler, - TaskDependencyManagerInterface &task_dependency_manager, + LeaseDependencyManagerInterface &lease_dependency_manager, internal::NodeInfoGetter get_node_info, WorkerPoolInterface &worker_pool, absl::flat_hash_map> &leased_workers, std::function &object_ids, std::vector> *results)> - get_task_arguments, - size_t max_pinned_task_arguments_bytes, + get_lease_arguments, + size_t max_pinned_lease_arguments_bytes, std::function get_time_ms, int64_t sched_cls_cap_interval_ms) : self_node_id_(self_node_id), self_scheduling_node_id_(self_node_id.Binary()), cluster_resource_scheduler_(cluster_resource_scheduler), - task_dependency_manager_(task_dependency_manager), + lease_dependency_manager_(lease_dependency_manager), get_node_info_(get_node_info), max_resource_shapes_per_load_report_( RayConfig::instance().max_resource_shapes_per_load_report()), worker_pool_(worker_pool), leased_workers_(leased_workers), - get_task_arguments_(get_task_arguments), - max_pinned_task_arguments_bytes_(max_pinned_task_arguments_bytes), + get_lease_arguments_(get_lease_arguments), + max_pinned_lease_arguments_bytes_(max_pinned_lease_arguments_bytes), get_time_ms_(get_time_ms), sched_cls_cap_enabled_(RayConfig::instance().worker_cap_enabled()), sched_cls_cap_interval_ms_(sched_cls_cap_interval_ms), sched_cls_cap_max_ms_(RayConfig::instance().worker_cap_max_backoff_delay_ms()) {} -void LocalTaskManager::QueueAndScheduleTask(std::shared_ptr work) { - // If the local node is draining, the cluster task manager will +void LocalLeaseManager::QueueAndScheduleLease(std::shared_ptr work) { + // If the local node is draining, the cluster lease manager will // guarantee that the local node is not selected for scheduling. RAY_CHECK(!cluster_resource_scheduler_.GetLocalResourceManager().IsLocalNodeDraining()); - // The local node must be feasible if the cluster task manager decides to run the task + // The local node must be feasible if the cluster lease manager decides to run the task // locally. RAY_CHECK(cluster_resource_scheduler_.GetClusterResourceManager().HasFeasibleResources( self_scheduling_node_id_, - ResourceMapToResourceRequest(work->task_.GetTaskSpecification() + ResourceMapToResourceRequest(work->lease_.GetLeaseSpecification() .GetRequiredPlacementResources() .GetResourceMap(), /*requires_object_store_memory=*/false))) - << work->task_.GetTaskSpecification().DebugString() << " " + << work->lease_.GetLeaseSpecification().DebugString() << " " << cluster_resource_scheduler_.GetClusterResourceManager() .GetNodeResources(self_scheduling_node_id_) .DebugString(); - WaitForTaskArgsRequests(std::move(work)); - ScheduleAndDispatchTasks(); + WaitForLeaseArgsRequests(std::move(work)); + ScheduleAndGrantLeases(); } -void LocalTaskManager::WaitForTaskArgsRequests(std::shared_ptr work) { - const auto &task = work->task_; - const auto &task_id = task.GetTaskSpecification().TaskId(); - const auto &scheduling_key = task.GetTaskSpecification().GetSchedulingClass(); - auto object_ids = task.GetTaskSpecification().GetDependencies(); +void LocalLeaseManager::WaitForLeaseArgsRequests(std::shared_ptr work) { + const auto &lease = work->lease_; + const auto &lease_id = lease.GetLeaseSpecification().LeaseId(); + const auto &scheduling_key = lease.GetLeaseSpecification().GetSchedulingClass(); + auto object_ids = lease.GetLeaseSpecification().GetDependencies(); if (!object_ids.empty()) { - bool args_ready = task_dependency_manager_.RequestTaskDependencies( - task_id, - task.GetDependencies(), - {task.GetTaskSpecification().GetName(), task.GetTaskSpecification().IsRetry()}); + bool args_ready = lease_dependency_manager_.RequestLeaseDependencies( + lease_id, + lease.GetLeaseSpecification().GetDependencies(), + {lease.GetLeaseSpecification().GetTaskName(), + lease.GetLeaseSpecification().IsRetry()}); if (args_ready) { - RAY_LOG(DEBUG) << "Args already ready, task can be dispatched " << task_id; - tasks_to_dispatch_[scheduling_key].emplace_back(std::move(work)); + RAY_LOG(DEBUG) << "Args already ready, lease can be granted " << lease_id; + leases_to_grant_[scheduling_key].emplace_back(std::move(work)); } else { - RAY_LOG(DEBUG) << "Waiting for args for task: " - << task.GetTaskSpecification().TaskId(); - auto it = waiting_task_queue_.insert(waiting_task_queue_.end(), std::move(work)); - RAY_CHECK(waiting_tasks_index_.emplace(task_id, it).second); + RAY_LOG(DEBUG) << "Waiting for args for lease: " << lease_id; + auto it = waiting_lease_queue_.insert(waiting_lease_queue_.end(), std::move(work)); + RAY_CHECK(waiting_leases_index_.emplace(lease_id, it).second); } } else { - RAY_LOG(DEBUG) << "No args, task can be dispatched " - << task.GetTaskSpecification().TaskId(); - tasks_to_dispatch_[scheduling_key].emplace_back(std::move(work)); + RAY_LOG(DEBUG) << "No args, lease can be granted " << lease_id; + leases_to_grant_[scheduling_key].emplace_back(std::move(work)); } } -void LocalTaskManager::ScheduleAndDispatchTasks() { - DispatchScheduledTasksToWorkers(); +void LocalLeaseManager::ScheduleAndGrantLeases() { + GrantScheduledLeasesToWorkers(); // TODO(swang): Spill from waiting queue first? Otherwise, we may end up - // spilling a task whose args are already local. - // TODO(swang): Invoke ScheduleAndDispatchTasks() when we run out of memory + // spilling a lease whose args are already local. + // TODO(swang): Invoke ScheduleAndGrantLeases() when we run out of memory // in the PullManager or periodically, to make sure that we spill waiting - // tasks that are blocked. - SpillWaitingTasks(); + // leases that are blocked. + SpillWaitingLeases(); } -void LocalTaskManager::DispatchScheduledTasksToWorkers() { - // Check every task in task_to_dispatch queue to see - // whether it can be dispatched and ran. This avoids head-of-line - // blocking where a task which cannot be dispatched because +void LocalLeaseManager::GrantScheduledLeasesToWorkers() { + // Check every lease in leases_to_grant queue to see + // whether it can be granted and ran. This avoids head-of-line + // blocking where a lease which cannot be granted because // there are not enough available resources blocks other - // tasks from being dispatched. - for (auto shapes_it = tasks_to_dispatch_.begin(); - shapes_it != tasks_to_dispatch_.end();) { + // leases from being granted. + for (auto shapes_it = leases_to_grant_.begin(); shapes_it != leases_to_grant_.end();) { auto &scheduling_class = shapes_it->first; - auto &dispatch_queue = shapes_it->second; + auto &leases_to_grant_queue = shapes_it->second; auto sched_cls_iter = info_by_sched_cls_.find(scheduling_class); if (sched_cls_iter == info_by_sched_cls_.end()) { // Initialize the class info. - sched_cls_iter = info_by_sched_cls_ - .emplace(scheduling_class, - SchedulingClassInfo(MaxRunningTasksPerSchedulingClass( - scheduling_class))) - .first; + sched_cls_iter = + info_by_sched_cls_ + .emplace(scheduling_class, + SchedulingClassInfo( + MaxGrantedLeasesPerSchedulingClass(scheduling_class))) + .first; } auto &sched_cls_info = sched_cls_iter->second; // Fair scheduling is applied only when the total CPU requests exceed the node's - // capacity. This skips scheduling classes whose number of running tasks exceeds the - // average number of tasks per scheduling class. + // capacity. This skips scheduling classes whose number of granted leases exceeds the + // average number of granted leases per scheduling class. // The purpose of fair scheduling is to ensure that each scheduling class has an - // equal chance of being selected for dispatch. For instance, in a pipeline with both - // data producers and consumers, we aim for consumers to have the same chance to be - // dispatched as producers. This prevents memory peak caused by dispatching all - // producer tasks first. - // A scheduling class is skipped from dispatching if its number of running tasks - // exceeds the fair_share, which is the average number of running tasks among all + // equal chance of being selected for lease granting. For instance, in a pipeline with + // both data producers and consumers, we aim for consumers to have the same chance to + // be granted a lease as producers. This prevents memory peak caused by granting all + // producer leases first. + // A scheduling class is skipped from lease granting if its number of granted leases + // exceeds the fair_share, which is the average number of granted leases among all // scheduling classes. For example, consider a scenario where we have 3 CPUs and 2 - // scheduling classes, `f` and `g`, each with 4 tasks. - // Status 1: The queue init with [f, f, f, f, g, g, g, g], and 0 running tasks. - // Status 2: We dispatch 3 `f` tasks. Now the queue is [f, g, g, g, g], - // with 3 `f` tasks running. - // Status 3: Suppose 1 `f` task finishes. When choosing the next task to dispatch, - // the queue is [f, g, g, g, g], and there are 2 `f` tasks running. + // scheduling classes, `f` and `g`, each with 4 leases. + // Status 1: The queue init with [f, f, f, f, g, g, g, g], and 0 granted leases. + // Status 2: We grant 3 `f` leases. Now the queue is [f, g, g, g, g], + // with 3 `f` leases granted. + // Status 3: Suppose 1 `f` lease finishes. When choosing the next lease to grant, + // the queue is [f, g, g, g, g], and there are 2 `f` leases granted. // We calculate fair_share as follows: - // fair_share = number of running tasks / number of scheduling classes + // fair_share = number of granted leases / number of scheduling classes // = 2 / 2 = 1. - // Since the number of running `f` tasks (2) is greater than the - // fair_share (1), we skip `f` and choose to dispatch `g`. - // Note 1: Fair_share is calculated as (total number of running tasks with >0 CPU) - // / (number of scheduling classes in tasks_to_dispatch_). + // Since the number of granted `f` leases (2) is greater than the + // fair_share (1), we skip `f` and choose to grant `g`. + // Note 1: Fair_share is calculated as (total number of granted leases with >0 CPU) + // / (number of scheduling classes in leases_to_dispatch_). // Note 2: The decision to skip a scheduling class happens when loop through the - // scheduling classes (keys of tasks_to_dispatch_). This means we check for + // scheduling classes (keys of leases_to_grant_). This means we check for // fair dispatching when looping through the scheduling classes rather than - // for each individual task, reducing the number of checks required. - // This is why in Status 2 of the example, we dispatch 3 `f` tasks because - // we chose `f` for dispatch,and we continue dispatching all `f` - // tasks until resources are fully utilized. + // for each individual lease, reducing the number of checks required. + // This is why in Status 2 of the example, we grant 3 `f` leases because + // we chose `f` for grant, and we continue granting all `f` + // leases until resources are fully utilized. - // Currently, fair dispatching is implemented only for tasks that require CPU + // Currently, fair granting is implemented only for leases that require CPU // resources. CPU. For details, see https://github.com/ray-project/ray/pull/44733. - // Calculate the total CPU requests for all tasks in the tasks_to_dispatch queue. + // Calculate the total CPU requests for all leases in the leases_to_grant queue. double total_cpu_requests_ = 0.0; // Count the number of scheduling classes that require CPU and sum their total CPU // requests. size_t num_classes_with_cpu = 0; - for (const auto &[_, cur_dispatch_queue] : tasks_to_dispatch_) { + for (const auto &[_, cur_dispatch_queue] : leases_to_grant_) { // Only need to check the first because all tasks with the same scheduling class // have the same CPU resource requirements. RAY_CHECK(!cur_dispatch_queue.empty()); const auto &work = cur_dispatch_queue.front(); - const auto &task_spec = work->task_.GetTaskSpecification(); + const auto &lease_spec = work->lease_.GetLeaseSpecification(); auto cpu_request_ = - task_spec.GetRequiredResources().Get(scheduling::ResourceID::CPU()).Double(); + lease_spec.GetRequiredResources().Get(scheduling::ResourceID::CPU()).Double(); if (cpu_request_ > 0) { num_classes_with_cpu++; total_cpu_requests_ += cur_dispatch_queue.size() * cpu_request_; @@ -199,59 +198,62 @@ void LocalTaskManager::DispatchScheduledTasksToWorkers() { cluster_resource_scheduler_.GetLocalResourceManager().GetNumCpus(); // Compare total CPU requests with the node's total CPU capacity. If the requests - // exceed the capacity, check if fair dispatching is needed. + // exceed the capacity, check if fair granting is needed. if (sched_cls_desc.resource_set.Get(scheduling::ResourceID::CPU()).Double() > 0 && total_cpu_requests_ > total_cpus) { RAY_LOG(DEBUG) - << "Applying fairness policy. Total CPU requests in tasks_to_dispatch_ (" + << "Applying fairness policy. Total CPU requests in leases_to_grant_ (" << total_cpu_requests_ << ") exceed total CPUs available (" << total_cpus << ")."; - // Get the total number of running tasks requires CPU. - size_t total_cpu_running_tasks = 0; + // Get the total number of granted leases that require CPU. + size_t total_cpu_granted_leases = 0; for (auto &entry : info_by_sched_cls_) { // Only consider CPU requests const auto &cur_sched_cls_desc = TaskSpecification::GetSchedulingClassDescriptor(entry.first); if (cur_sched_cls_desc.resource_set.Get(scheduling::ResourceID::CPU()).Double() > 0) { - total_cpu_running_tasks += entry.second.running_tasks.size(); + total_cpu_granted_leases += entry.second.granted_leases.size(); } } // 1. We have confirmed that this is a scheduling class that requires CPU resources, // hence num_classes_with_cpu >= 1 (cannot be 0) as this scheduling class is in - // tasks_to_dispatch_. - // 2. We will compute fair_share as the ideal distribution of tasks among all - // scheduling classes in tasks_to_dispatch_. Then, we will check if the number of - // running tasks for this scheduling class exceeds its ideal fair_share. - // 3. Note: We should get the num_classes_with_cpu from tasks_to_dispatch_ - // instead of the info_by_sched_cls_ although total_cpu_running_tasks gets from - // the task running. First, info_by_sched_cls_ may not be initialized yet for - // some scheduling classes (as we initialize it in the loop). Second, we expect - // the number of running tasks for this scheduling class to not be much. However, - // if no tasks of this scheduling class are running, it will not be skipped. - - size_t fair_share = total_cpu_running_tasks / num_classes_with_cpu; - if (sched_cls_info.running_tasks.size() > fair_share) { - RAY_LOG(DEBUG) << "Skipping dispatch for scheduling class " << scheduling_class - << ". Running tasks (" << sched_cls_info.running_tasks.size() - << ") exceed fair share (" << fair_share << ")."; + // leases_to_grant_. + // 2. We will compute fair_share as the ideal distribution of leases among all + // scheduling classes in leases_to_grant_. Then, we will check if the number + // of granted leases for this scheduling class exceeds its ideal fair_share. + // 3. Note: We should get the num_classes_with_cpu from leases_to_grant_ + // instead of the info_by_sched_cls_ although total_cpu_granted_leases is + // obtained from the granted leases. First, info_by_sched_cls_ may not be + // initialized yet for some scheduling classes (as we initialize it in the loop). + // Second, we expect the number of granted leases for this scheduling class to + // not be much. However, if no leases of this scheduling class are granted, it + // will not be skipped. + + size_t fair_share = total_cpu_granted_leases / num_classes_with_cpu; + if (sched_cls_info.granted_leases.size() > fair_share) { + RAY_LOG(DEBUG) << "Skipping lease granting for scheduling class " + << scheduling_class << ". Granted leases (" + << sched_cls_info.granted_leases.size() << ") exceed fair share (" + << fair_share << ")."; shapes_it++; continue; } } - /// We cap the maximum running tasks of a scheduling class to avoid - /// scheduling too many tasks of a single type/depth, when there are + /// We cap the maximum granted leases of a scheduling class to avoid + /// granting too many leases of a single type/depth, when there are /// deeper/other functions that should be run. We need to apply back /// pressure to limit the number of worker processes started in scenarios /// with nested tasks. bool is_infeasible = false; - for (auto work_it = dispatch_queue.begin(); work_it != dispatch_queue.end();) { + for (auto work_it = leases_to_grant_queue.begin(); + work_it != leases_to_grant_queue.end();) { auto &work = *work_it; - const auto &task = work->task_; - const auto &spec = task.GetTaskSpecification(); - TaskID task_id = spec.TaskId(); + const auto &lease = work->lease_; + const auto &spec = lease.GetLeaseSpecification(); + LeaseID lease_id = spec.LeaseId(); if (work->GetState() == internal::WorkStatus::WAITING_FOR_WORKER) { work_it++; continue; @@ -259,14 +261,14 @@ void LocalTaskManager::DispatchScheduledTasksToWorkers() { // Check if the scheduling class is at capacity now. if (sched_cls_cap_enabled_ && - sched_cls_info.running_tasks.size() >= sched_cls_info.capacity && + sched_cls_info.granted_leases.size() >= sched_cls_info.capacity && work->GetState() == internal::WorkStatus::WAITING) { RAY_LOG(DEBUG) << "Hit cap! time=" << get_time_ms_() << " next update time=" << sched_cls_info.next_update_time; if (get_time_ms_() < sched_cls_info.next_update_time) { - // We're over capacity and it's not time to admit a new task yet. - // Calculate the next time we should admit a new task. - int64_t current_capacity = sched_cls_info.running_tasks.size(); + // We're over capacity and it's not time to grant a new lease yet. + // Calculate the next time we should grant a new lease. + int64_t current_capacity = sched_cls_info.granted_leases.size(); int64_t allowed_capacity = sched_cls_info.capacity; int64_t exp = current_capacity - allowed_capacity; int64_t wait_time = sched_cls_cap_interval_ms_ * (1L << exp); @@ -280,11 +282,11 @@ void LocalTaskManager::DispatchScheduledTasksToWorkers() { sched_cls_info.next_update_time = std::min(target_time, sched_cls_info.next_update_time); - // While we're over capacity and cannot run the task, - // try to spill to a node that can run it. + // While we're over capacity and cannot grant the lease, + // try to spill to a node that can. bool did_spill = TrySpillback(work, is_infeasible); if (did_spill) { - work_it = dispatch_queue.erase(work_it); + work_it = leases_to_grant_queue.erase(work_it); continue; } @@ -293,32 +295,32 @@ void LocalTaskManager::DispatchScheduledTasksToWorkers() { } bool args_missing = false; - bool success = PinTaskArgsIfMemoryAvailable(spec, &args_missing); - // An argument was evicted since this task was added to the dispatch + bool success = PinLeaseArgsIfMemoryAvailable(spec, &args_missing); + // An argument was evicted since this lease was added to the grant // queue. Move it back to the waiting queue. The caller is responsible - // for notifying us when the task is unblocked again. + // for notifying us when the lease is unblocked again. if (!success) { if (args_missing) { - // Insert the task at the head of the waiting queue because we + // Insert the lease at the head of the waiting queue because we // prioritize spilling from the end of the queue. // TODO(scv119): where does pulling happen? - auto it = waiting_task_queue_.insert(waiting_task_queue_.begin(), - std::move(*work_it)); - RAY_CHECK(waiting_tasks_index_.emplace(task_id, it).second); - work_it = dispatch_queue.erase(work_it); + auto it = waiting_lease_queue_.insert(waiting_lease_queue_.begin(), + std::move(*work_it)); + RAY_CHECK(waiting_leases_index_.emplace(lease_id, it).second); + work_it = leases_to_grant_queue.erase(work_it); } else { - // The task's args cannot be pinned due to lack of memory. We should - // retry dispatching the task once another task finishes and releases + // The lease's args cannot be pinned due to lack of memory. We should + // retry granting the lease once another lease finishes and releases // its arguments. - RAY_LOG(DEBUG) << "Dispatching task " << task_id + RAY_LOG(DEBUG) << "Granting lease " << lease_id << " would put this node over the max memory allowed for " - "arguments of executing tasks (" - << max_pinned_task_arguments_bytes_ - << "). Waiting to dispatch task until other tasks complete"; - RAY_CHECK(!executing_task_args_.empty() && !pinned_task_arguments_.empty()) - << "Cannot dispatch task " << task_id - << " until another task finishes and releases its arguments, but no other " - "task is running"; + "arguments of granted leases (" + << max_pinned_lease_arguments_bytes_ + << "). Waiting to grant lease until other leases are returned"; + RAY_CHECK(!granted_lease_args_.empty() && !pinned_lease_arguments_.empty()) + << "Cannot grant lease " << lease_id + << " until another lease is returned and releases its arguments, but no " + "other lease is granted"; work->SetStateWaiting( internal::UnscheduledWorkCause::WAITING_FOR_AVAILABLE_PLASMA_MEMORY); work_it++; @@ -335,41 +337,41 @@ void LocalTaskManager::DispatchScheduledTasksToWorkers() { .AllocateLocalTaskResources(spec.GetRequiredResources().GetResourceMap(), allocated_instances); if (!schedulable) { - ReleaseTaskArgs(task_id); - // The local node currently does not have the resources to run the task, so we + ReleaseLeaseArgs(lease_id); + // The local node currently does not have the resources to grant the lease, so we // should try spilling to another node. bool did_spill = TrySpillback(work, is_infeasible); if (!did_spill) { - // There must not be any other available nodes in the cluster, so the task + // There must not be any other available nodes in the cluster, so the lease // should stay on this node. We can skip the rest of the shape because the // scheduler will make the same decision. work->SetStateWaiting( internal::UnscheduledWorkCause::WAITING_FOR_RESOURCES_AVAILABLE); break; } - work_it = dispatch_queue.erase(work_it); + work_it = leases_to_grant_queue.erase(work_it); } else { // Force us to recalculate the next update time the next time a task // comes through this queue. We should only do this when we're // confident we're ready to dispatch the task after all checks have // passed. sched_cls_info.next_update_time = std::numeric_limits::max(); - sched_cls_info.running_tasks.insert(spec.TaskId()); - // The local node has the available resources to run the task, so we should run - // it. + sched_cls_info.granted_leases.insert(lease_id); + // The local node has the available resources to grant the lease, so we should + // grant it. work->allocated_instances_ = allocated_instances; work->SetStateWaitingForWorker(); bool is_detached_actor = spec.IsDetachedActor(); auto &owner_address = spec.CallerAddress(); - /// TODO(scv119): if a worker is not started, the resources is leaked and + /// TODO(scv119): if a worker is not started, the resources are leaked and // task might be hanging. worker_pool_.PopWorker( spec, - [this, task_id, scheduling_class, work, is_detached_actor, owner_address]( + [this, lease_id, scheduling_class, work, is_detached_actor, owner_address]( const std::shared_ptr worker, PopWorkerStatus status, const std::string &runtime_env_setup_error_message) -> bool { - // TODO(hjiang): After getting the ready-to-use worker and task id, we're + // TODO(hjiang): After getting the ready-to-use worker and lease id, we're // able to get physical execution context. // // ownership chain: raylet has-a node manager, node manager has-a local task @@ -381,7 +383,7 @@ void LocalTaskManager::DispatchScheduledTasksToWorkers() { return PoppedWorkerHandler(worker, status, - task_id, + lease_id, scheduling_class, work, is_detached_actor, @@ -394,77 +396,79 @@ void LocalTaskManager::DispatchScheduledTasksToWorkers() { // In the beginning of the loop, we add scheduling_class // to the `info_by_sched_cls_` map. // In cases like dead owners, we may not add any tasks - // to `running_tasks` so we can remove the map entry + // to `granted_leases` so we can remove the map entry // for that scheduling_class to prevent memory leaks. - if (sched_cls_info.running_tasks.size() == 0) { + if (sched_cls_info.granted_leases.size() == 0) { info_by_sched_cls_.erase(scheduling_class); } if (is_infeasible) { - const auto &front_task = dispatch_queue.front()->task_.GetTaskSpecification(); - RAY_LOG(ERROR) << "A task got scheduled to a node even though it was infeasible. " - "Please report an issue on GitHub.\nTask: " - << front_task.DebugString(); - auto dispatch_queue_iter = dispatch_queue.begin(); - while (dispatch_queue_iter != dispatch_queue.end()) { - CancelTaskToDispatch( - *dispatch_queue_iter, + const auto &front_lease = + leases_to_grant_queue.front()->lease_.GetLeaseSpecification(); + RAY_LOG(ERROR) << "A lease got granted to a node even though it was infeasible. " + "Please report an issue on GitHub.\nLease: " + << front_lease.DebugString(); + auto leases_to_grant_queue_iter = leases_to_grant_queue.begin(); + while (leases_to_grant_queue_iter != leases_to_grant_queue.end()) { + CancelLeaseToGrant( + *leases_to_grant_queue_iter, rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_UNSCHEDULABLE, - "Scheduling failed due to the task becoming infeasible."); - dispatch_queue_iter = dispatch_queue.erase(dispatch_queue_iter); + "Lease granting failed due to the lease becoming infeasible."); + leases_to_grant_queue_iter = + leases_to_grant_queue.erase(leases_to_grant_queue_iter); } - tasks_to_dispatch_.erase(shapes_it++); - } else if (dispatch_queue.empty()) { - tasks_to_dispatch_.erase(shapes_it++); + leases_to_grant_.erase(shapes_it++); + } else if (leases_to_grant_queue.empty()) { + leases_to_grant_.erase(shapes_it++); } else { shapes_it++; } } } -void LocalTaskManager::SpillWaitingTasks() { - // Try to spill waiting tasks to a remote node, prioritizing those at the end - // of the queue. Waiting tasks are spilled if there are enough remote +void LocalLeaseManager::SpillWaitingLeases() { + // Try to spill waiting leases to a remote node, prioritizing those at the end + // of the queue. Waiting leases are spilled if there are enough remote // resources AND (we have no resources available locally OR their - // dependencies are not being fetched). We should not spill tasks whose + // dependencies are not being fetched). We should not spill leases whose // dependencies are actively being fetched because some of their dependencies // may already be local or in-flight to this node. // // NOTE(swang): We do not iterate by scheduling class here, so if we break - // due to lack of remote resources, it is possible that a waiting task that + // due to lack of remote resources, it is possible that a waiting lease that // is earlier in the queue could have been scheduled to a remote node. // TODO(scv119): this looks very aggressive: we will try to spillback - // all the tasks in the waiting queue regardless of the wait time. - auto it = waiting_task_queue_.end(); - while (it != waiting_task_queue_.begin()) { + // all the leases in the waiting queue regardless of the wait time. + auto it = waiting_lease_queue_.end(); + while (it != waiting_lease_queue_.begin()) { it--; - const auto &task = (*it)->task_; - const auto &spec = task.GetTaskSpecification(); - const auto &task_id = spec.TaskId(); + const auto &lease = (*it)->lease_; + const auto &lease_spec = lease.GetLeaseSpecification(); + const auto &lease_id = lease_spec.LeaseId(); - // Check whether this task's dependencies are blocked (not being actively - // pulled). If this is true, then we should force the task onto a remote + // Check whether this lease's dependencies are blocked (not being actively + // pulled). If this is true, then we should force the lease onto a remote // feasible node, even if we have enough resources available locally for // placement. - bool task_dependencies_blocked = - task_dependency_manager_.TaskDependenciesBlocked(task_id); - RAY_LOG(DEBUG) << "Attempting to spill back waiting task " << task_id + bool lease_dependencies_blocked = + lease_dependency_manager_.LeaseDependenciesBlocked(lease_id); + RAY_LOG(DEBUG) << "Attempting to spill back waiting lease " << lease_id << " to remote node. Dependencies blocked? " - << task_dependencies_blocked; + << lease_dependencies_blocked; bool is_infeasible; // TODO(swang): The policy currently does not account for the amount of // object store memory availability. Ideally, we should pick the node with // the most memory availability. scheduling::NodeID scheduling_node_id; - if (!spec.IsSpreadSchedulingStrategy()) { + if (!lease_spec.IsSpreadSchedulingStrategy()) { scheduling_node_id = cluster_resource_scheduler_.GetBestSchedulableNode( - spec, + lease_spec, /*preferred_node_id*/ self_node_id_.Binary(), - /*exclude_local_node*/ task_dependencies_blocked, + /*exclude_local_node*/ lease_dependencies_blocked, /*requires_object_store_memory*/ true, &is_infeasible); } else { // If scheduling strategy is spread, we prefer honoring spread decision - // and waiting for task dependencies to be pulled + // and waiting for lease dependencies to be pulled // locally than spilling back and causing uneven spread. scheduling_node_id = self_scheduling_node_id_; } @@ -472,21 +476,21 @@ void LocalTaskManager::SpillWaitingTasks() { if (!scheduling_node_id.IsNil() && scheduling_node_id != self_scheduling_node_id_) { NodeID node_id = NodeID::FromBinary(scheduling_node_id.Binary()); Spillback(node_id, *it); - if (!spec.GetDependencies().empty()) { - task_dependency_manager_.RemoveTaskDependencies(spec.TaskId()); + if (!lease_spec.GetDependencies().empty()) { + lease_dependency_manager_.RemoveLeaseDependencies(lease_id); } - num_waiting_task_spilled_++; - waiting_tasks_index_.erase(task_id); - it = waiting_task_queue_.erase(it); + num_waiting_lease_spilled_++; + waiting_leases_index_.erase(lease_id); + it = waiting_lease_queue_.erase(it); } else { if (scheduling_node_id.IsNil()) { - RAY_LOG(DEBUG) << "RayTask " << task_id + RAY_LOG(DEBUG) << "RayLease " << lease_id << " has blocked dependencies, but no other node has resources, " - "keeping the task local"; + "keeping the lease local"; } else { - RAY_LOG(DEBUG) << "Keeping waiting task " << task_id << " local"; + RAY_LOG(DEBUG) << "Keeping waiting lease " << lease_id << " local"; } - // We should keep the task local. Note that an earlier task in the queue + // We should keep the lease local. Note that an earlier lease in the queue // may have different resource requirements and could actually be // scheduled on a remote node. break; @@ -494,9 +498,9 @@ void LocalTaskManager::SpillWaitingTasks() { } } -bool LocalTaskManager::TrySpillback(const std::shared_ptr &work, - bool &is_infeasible) { - const auto &spec = work->task_.GetTaskSpecification(); +bool LocalLeaseManager::TrySpillback(const std::shared_ptr &work, + bool &is_infeasible) { + const auto &spec = work->lease_.GetLeaseSpecification(); auto scheduling_node_id = cluster_resource_scheduler_.GetBestSchedulableNode( spec, // We should prefer to stay local if possible @@ -514,17 +518,17 @@ bool LocalTaskManager::TrySpillback(const std::shared_ptr &work, NodeID node_id = NodeID::FromBinary(scheduling_node_id.Binary()); Spillback(node_id, work); - num_unschedulable_task_spilled_++; + num_unschedulable_lease_spilled_++; if (!spec.GetDependencies().empty()) { - task_dependency_manager_.RemoveTaskDependencies(spec.TaskId()); + lease_dependency_manager_.RemoveLeaseDependencies(spec.LeaseId()); } return true; } -bool LocalTaskManager::PoppedWorkerHandler( +bool LocalLeaseManager::PoppedWorkerHandler( const std::shared_ptr worker, PopWorkerStatus status, - const TaskID &task_id, + const LeaseID &lease_id, SchedulingClass scheduling_class, const std::shared_ptr &work, bool is_detached_actor, @@ -533,12 +537,12 @@ bool LocalTaskManager::PoppedWorkerHandler( const auto &reply = work->reply_; const auto &callback = work->callback_; const bool canceled = work->GetState() == internal::WorkStatus::CANCELLED; - const auto &task = work->task_; - bool dispatched = false; + const auto &lease = work->lease_; + bool granted = false; if (!canceled) { const auto &required_resource = - task.GetTaskSpecification().GetRequiredResources().GetResourceMap(); + lease.GetLeaseSpecification().GetRequiredResources().GetResourceMap(); for (auto &entry : required_resource) { // This is to make sure PG resource is not deleted during popping worker // unless the lease request is cancelled. @@ -548,62 +552,63 @@ bool LocalTaskManager::PoppedWorkerHandler( } } - // Erases the work from task_to_dispatch_ queue, also removes the task dependencies. + // Erases the work from lease_to_grant_ queue, also removes the lease dependencies. // // IDEA(ryw): Make an RAII class to wrap the a shared_ptr and - // requests task dependency upon ctor, and remove task dependency upon dtor. - // I tried this, it works, but we expose the map via GetTaskToDispatch() used in + // requests lease dependency upon ctor, and remove lease dependency upon dtor. + // I tried this, it works, but we expose the map via GetLeasesToGrant() used in // scheduler_resource_reporter.cc. Maybe we can use `boost::any_range` to only expose // a view of the Work ptrs, but I got dependency issues // (can't include boost/range/any_range.hpp). - auto erase_from_dispatch_queue_fn = + auto erase_from_leases_to_grant_queue_fn = [this](const std::shared_ptr &work_to_erase, const SchedulingClass &_scheduling_class) { - auto shapes_it = tasks_to_dispatch_.find(_scheduling_class); - RAY_CHECK(shapes_it != tasks_to_dispatch_.end()); - auto &dispatch_queue = shapes_it->second; + auto shapes_it = leases_to_grant_.find(_scheduling_class); + RAY_CHECK(shapes_it != leases_to_grant_.end()); + auto &leases_to_grant_queue = shapes_it->second; bool erased = false; - for (auto work_it = dispatch_queue.begin(); work_it != dispatch_queue.end(); + for (auto work_it = leases_to_grant_queue.begin(); + work_it != leases_to_grant_queue.end(); work_it++) { if (*work_it == work_to_erase) { - dispatch_queue.erase(work_it); + leases_to_grant_queue.erase(work_it); erased = true; break; } } - if (dispatch_queue.empty()) { - tasks_to_dispatch_.erase(shapes_it); + if (leases_to_grant_queue.empty()) { + leases_to_grant_.erase(shapes_it); } RAY_CHECK(erased); - const auto &_task = work_to_erase->task_; - if (!_task.GetDependencies().empty()) { - task_dependency_manager_.RemoveTaskDependencies( - _task.GetTaskSpecification().TaskId()); + const auto &_lease = work_to_erase->lease_; + if (!_lease.GetLeaseSpecification().GetDependencies().empty()) { + lease_dependency_manager_.RemoveLeaseDependencies( + _lease.GetLeaseSpecification().LeaseId()); } }; if (canceled) { // Task has been canceled. - RAY_LOG(DEBUG) << "Task " << task_id << " has been canceled when worker popped"; - RemoveFromRunningTasksIfExists(task); - // All the cleaning work has been done when canceled task. Just return + RAY_LOG(DEBUG) << "Lease " << lease_id << " has been canceled when worker popped"; + RemoveFromGrantedLeasesIfExists(lease); + // All the cleaning work has been done when canceled lease. Just return // false without doing anything. return false; } if (!worker) { - dispatched = false; + granted = false; // We've already acquired resources so we need to release them. cluster_resource_scheduler_.GetLocalResourceManager().ReleaseWorkerResources( work->allocated_instances_); work->allocated_instances_ = nullptr; // Release pinned task args. - ReleaseTaskArgs(task_id); - RemoveFromRunningTasksIfExists(task); + ReleaseLeaseArgs(lease_id); + RemoveFromGrantedLeasesIfExists(lease); // Empty worker popped. - RAY_LOG(DEBUG).WithField(task_id) + RAY_LOG(DEBUG).WithField(lease_id) << "This node has available resources, but no worker processes " "to grant the lease: status " << status; @@ -612,17 +617,17 @@ bool LocalTaskManager::PoppedWorkerHandler( // directly and raise a `RuntimeEnvSetupError` exception to user // eventually. The task will be removed from dispatch queue in // `CancelTask`. - CancelTasks( - [task_id](const auto &w) { - return task_id == w->task_.GetTaskSpecification().TaskId(); + CancelLeases( + [lease_id](const auto &w) { + return lease_id == w->lease_.GetLeaseSpecification().LeaseId(); }, rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_RUNTIME_ENV_SETUP_FAILED, /*scheduling_failure_message*/ runtime_env_setup_error_message); } else if (status == PopWorkerStatus::JobFinished) { // The task job finished. // Just remove the task from dispatch queue. - RAY_LOG(DEBUG) << "Call back to a job finished task, task id = " << task_id; - erase_from_dispatch_queue_fn(work, scheduling_class); + RAY_LOG(DEBUG) << "Call back to a job finished lease, lease id = " << lease_id; + erase_from_leases_to_grant_queue_fn(work, scheduling_class); } else { // In other cases, set the work status `WAITING` to make this task // could be re-dispatched. @@ -639,21 +644,21 @@ bool LocalTaskManager::PoppedWorkerHandler( work->SetStateWaiting(cause); } } else { - // A worker has successfully popped for a valid task. Dispatch the task to + // A worker has successfully popped for a valid lease. Grant the lease to // the worker. - RAY_LOG(DEBUG) << "Dispatching task " << task_id << " to worker " + RAY_LOG(DEBUG) << "Granting lease " << lease_id << " to worker " << worker->WorkerId(); - Dispatch(worker, leased_workers_, work->allocated_instances_, task, reply, callback); - erase_from_dispatch_queue_fn(work, scheduling_class); - dispatched = true; + Grant(worker, leased_workers_, work->allocated_instances_, lease, reply, callback); + erase_from_leases_to_grant_queue_fn(work, scheduling_class); + granted = true; } - return dispatched; + return granted; } -void LocalTaskManager::Spillback(const NodeID &spillback_to, - const std::shared_ptr &work) { +void LocalLeaseManager::Spillback(const NodeID &spillback_to, + const std::shared_ptr &work) { auto send_reply_callback = work->callback_; if (work->grant_or_reject_) { @@ -662,15 +667,15 @@ void LocalTaskManager::Spillback(const NodeID &spillback_to, return; } - num_task_spilled_++; - const auto &task = work->task_; - const auto &task_spec = task.GetTaskSpecification(); - RAY_LOG(DEBUG) << "Spilling task " << task_spec.TaskId() << " to node " << spillback_to; + num_lease_spilled_++; + const auto &lease_spec = work->lease_.GetLeaseSpecification(); + RAY_LOG(DEBUG) << "Spilling lease " << lease_spec.LeaseId() << " to node " + << spillback_to; if (!cluster_resource_scheduler_.AllocateRemoteTaskResources( scheduling::NodeID(spillback_to.Binary()), - task_spec.GetRequiredResources().GetResourceMap())) { - RAY_LOG(DEBUG) << "Tried to allocate resources for request " << task_spec.TaskId() + lease_spec.GetRequiredResources().GetResourceMap())) { + RAY_LOG(DEBUG) << "Tried to allocate resources for request " << lease_spec.LeaseId() << " on a remote node that are no longer available"; } @@ -687,74 +692,74 @@ void LocalTaskManager::Spillback(const NodeID &spillback_to, send_reply_callback(); } -void LocalTaskManager::TasksUnblocked(const std::vector &ready_ids) { +void LocalLeaseManager::LeasesUnblocked(const std::vector &ready_ids) { if (ready_ids.empty()) { return; } - for (const auto &task_id : ready_ids) { - auto it = waiting_tasks_index_.find(task_id); - if (it != waiting_tasks_index_.end()) { + for (const auto &lease_id : ready_ids) { + auto it = waiting_leases_index_.find(lease_id); + if (it != waiting_leases_index_.end()) { auto work = *it->second; - const auto &task = work->task_; - const auto &scheduling_key = task.GetTaskSpecification().GetSchedulingClass(); - RAY_LOG(DEBUG) << "Args ready, task can be dispatched " - << task.GetTaskSpecification().TaskId(); - tasks_to_dispatch_[scheduling_key].push_back(work); - waiting_task_queue_.erase(it->second); - waiting_tasks_index_.erase(it); + const auto &lease = work->lease_; + const auto &scheduling_key = lease.GetLeaseSpecification().GetSchedulingClass(); + RAY_LOG(DEBUG) << "Args ready, lease can be granted " + << lease.GetLeaseSpecification().LeaseId(); + leases_to_grant_[scheduling_key].push_back(work); + waiting_lease_queue_.erase(it->second); + waiting_leases_index_.erase(it); } } - ScheduleAndDispatchTasks(); + ScheduleAndGrantLeases(); } -void LocalTaskManager::RemoveFromRunningTasksIfExists(const RayTask &task) { - auto sched_cls = task.GetTaskSpecification().GetSchedulingClass(); +void LocalLeaseManager::RemoveFromGrantedLeasesIfExists(const RayLease &lease) { + auto sched_cls = lease.GetLeaseSpecification().GetSchedulingClass(); auto it = info_by_sched_cls_.find(sched_cls); if (it != info_by_sched_cls_.end()) { - // TODO(hjiang): After remove the task id from `running_tasks`, corresponding cgroup + // TODO(hjiang): After remove the lease id from `granted_leases`, corresponding cgroup // will be updated. - it->second.running_tasks.erase(task.GetTaskSpecification().TaskId()); - if (it->second.running_tasks.size() == 0) { + it->second.granted_leases.erase(lease.GetLeaseSpecification().LeaseId()); + if (it->second.granted_leases.size() == 0) { info_by_sched_cls_.erase(it); } } } -void LocalTaskManager::TaskFinished(std::shared_ptr worker, - RayTask *task) { - RAY_CHECK(worker != nullptr && task != nullptr); - *task = worker->GetAssignedTask(); - RemoveFromRunningTasksIfExists(*task); +void LocalLeaseManager::CleanupLease(std::shared_ptr worker, + RayLease *lease) { + RAY_CHECK(worker != nullptr && lease != nullptr); + *lease = worker->GetGrantedLease(); + RemoveFromGrantedLeasesIfExists(*lease); - ReleaseTaskArgs(task->GetTaskSpecification().TaskId()); + ReleaseLeaseArgs(lease->GetLeaseSpecification().LeaseId()); if (worker->GetAllocatedInstances() != nullptr) { ReleaseWorkerResources(worker); } } -// TODO(scv119): task args related logic probaly belongs task dependency manager. -bool LocalTaskManager::PinTaskArgsIfMemoryAvailable(const TaskSpecification &spec, - bool *args_missing) { +// TODO(scv119): lease args related logic probaly belongs lease dependency manager. +bool LocalLeaseManager::PinLeaseArgsIfMemoryAvailable( + const LeaseSpecification &lease_spec, bool *args_missing) { std::vector> args; - const auto &deps = spec.GetDependencyIds(); + const auto &deps = lease_spec.GetDependencyIds(); if (!deps.empty()) { // This gets refs to the arguments stored in plasma. The refs should be // deleted once we no longer need to pin the arguments. - if (!get_task_arguments_(deps, &args)) { + if (!get_lease_arguments_(deps, &args)) { *args_missing = true; return false; } for (size_t i = 0; i < deps.size(); i++) { if (args[i] == nullptr) { - // This can happen if the task's arguments were all local at some - // point, but then at least one was evicted before the task could - // be dispatched to a worker. + // This can happen if the lease's arguments were all local at some + // point, but then at least one was evicted before the lease could + // be granted to a worker. RAY_LOG(DEBUG) - << "RayTask " << spec.TaskId() << " argument " << deps[i] - << " was evicted before the task could be dispatched. This can happen " - "when there are many objects needed on this node. The task will be " - "scheduled once all of its dependencies are local."; + << "RayLease " << lease_spec.LeaseId() << " argument " << deps[i] + << " was evicted before the lease could be granted. This can happen " + "when there are many objects needed on this node. The lease will be " + "granted once all of its dependencies are local."; *args_missing = true; return false; } @@ -762,76 +767,78 @@ bool LocalTaskManager::PinTaskArgsIfMemoryAvailable(const TaskSpecification &spe } *args_missing = false; - size_t task_arg_bytes = 0; + size_t lease_arg_bytes = 0; for (auto &arg : args) { - task_arg_bytes += arg->GetSize(); + lease_arg_bytes += arg->GetSize(); } - RAY_LOG(DEBUG) << "RayTask " << spec.TaskId() << " has args of size " << task_arg_bytes; - PinTaskArgs(spec, std::move(args)); - RAY_LOG(DEBUG) << "Size of pinned task args is now " << pinned_task_arguments_bytes_; - if (max_pinned_task_arguments_bytes_ == 0) { + RAY_LOG(DEBUG) << "RayLease " << lease_spec.LeaseId() << " has args of size " + << lease_arg_bytes; + PinLeaseArgs(lease_spec, std::move(args)); + RAY_LOG(DEBUG) << "Size of pinned task args is now " << pinned_lease_arguments_bytes_; + if (max_pinned_lease_arguments_bytes_ == 0) { // Max threshold for pinned args is not set. return true; } - if (task_arg_bytes > max_pinned_task_arguments_bytes_) { + if (lease_arg_bytes > max_pinned_lease_arguments_bytes_) { RAY_LOG(WARNING) - << "Dispatched task " << spec.TaskId() << " has arguments of size " - << task_arg_bytes - << ", but the max memory allowed for arguments of executing tasks is only " - << max_pinned_task_arguments_bytes_; - } else if (pinned_task_arguments_bytes_ > max_pinned_task_arguments_bytes_) { - ReleaseTaskArgs(spec.TaskId()); - RAY_LOG(DEBUG) << "Cannot dispatch task " << spec.TaskId() - << " with arguments of size " << task_arg_bytes - << " current pinned bytes is " << pinned_task_arguments_bytes_; + << "Granted lease " << lease_spec.LeaseId() << " has arguments of size " + << lease_arg_bytes + << ", but the max memory allowed for arguments of granted leases is only " + << max_pinned_lease_arguments_bytes_; + } else if (pinned_lease_arguments_bytes_ > max_pinned_lease_arguments_bytes_) { + ReleaseLeaseArgs(lease_spec.LeaseId()); + RAY_LOG(DEBUG) << "Cannot grant lease " << lease_spec.LeaseId() + << " with arguments of size " << lease_arg_bytes + << " current pinned bytes is " << pinned_lease_arguments_bytes_; return false; } return true; } -void LocalTaskManager::PinTaskArgs(const TaskSpecification &spec, - std::vector> args) { - const auto &deps = spec.GetDependencyIds(); +void LocalLeaseManager::PinLeaseArgs(const LeaseSpecification &lease_spec, + std::vector> args) { + const auto &deps = lease_spec.GetDependencyIds(); // TODO(swang): This should really be an assertion, but we can sometimes - // receive a duplicate task request if there is a failure and the original - // version of the task has not yet been canceled. - auto executed_task_inserted = executing_task_args_.emplace(spec.TaskId(), deps).second; - if (executed_task_inserted) { + // receive a duplicate lease request if there is a failure and the original + // version of the lease has not yet been canceled. + auto executed_lease_inserted = + granted_lease_args_.emplace(lease_spec.LeaseId(), deps).second; + if (executed_lease_inserted) { for (size_t i = 0; i < deps.size(); i++) { - auto [it, pinned_task_inserted] = - pinned_task_arguments_.emplace(deps[i], std::make_pair(std::move(args[i]), 0)); - if (pinned_task_inserted) { - // This is the first task that needed this argument. - pinned_task_arguments_bytes_ += it->second.first->GetSize(); + auto [it, pinned_lease_inserted] = + pinned_lease_arguments_.emplace(deps[i], std::make_pair(std::move(args[i]), 0)); + if (pinned_lease_inserted) { + // This is the first lease that needed this argument. + pinned_lease_arguments_bytes_ += it->second.first->GetSize(); } it->second.second++; } } else { - RAY_LOG(DEBUG) << "Scheduler received duplicate task " << spec.TaskId() + RAY_LOG(DEBUG) << "Scheduler received duplicate lease " << lease_spec.LeaseId() << ", most likely because the first execution failed"; } } -void LocalTaskManager::ReleaseTaskArgs(const TaskID &task_id) { - auto it = executing_task_args_.find(task_id); +void LocalLeaseManager::ReleaseLeaseArgs(const LeaseID &lease_id) { + auto it = granted_lease_args_.find(lease_id); // TODO(swang): This should really be an assertion, but we can sometimes - // receive a duplicate task request if there is a failure and the original - // version of the task has not yet been canceled. - if (it != executing_task_args_.end()) { + // receive a duplicate lease request if there is a failure and the original + // version of the lease has not yet been canceled. + if (it != granted_lease_args_.end()) { for (auto &arg : it->second) { - auto arg_it = pinned_task_arguments_.find(arg); - RAY_CHECK(arg_it != pinned_task_arguments_.end()); + auto arg_it = pinned_lease_arguments_.find(arg); + RAY_CHECK(arg_it != pinned_lease_arguments_.end()); RAY_CHECK(arg_it->second.second > 0); arg_it->second.second--; if (arg_it->second.second == 0) { - // This is the last task that needed this argument. - pinned_task_arguments_bytes_ -= arg_it->second.first->GetSize(); - pinned_task_arguments_.erase(arg_it); + // This is the last lease that needed this argument. + pinned_lease_arguments_bytes_ -= arg_it->second.first->GetSize(); + pinned_lease_arguments_.erase(arg_it); } } - executing_task_args_.erase(it); + granted_lease_args_.erase(it); } } @@ -848,31 +855,31 @@ void ReplyCancelled(const std::shared_ptr &work, } } // namespace -bool LocalTaskManager::CancelTasks( +bool LocalLeaseManager::CancelLeases( std::function &)> predicate, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) { bool tasks_cancelled = false; ray::erase_if>( - tasks_to_dispatch_, [&](const std::shared_ptr &work) { + leases_to_grant_, [&](const std::shared_ptr &work) { if (!predicate(work)) { return false; } - CancelTaskToDispatch(work, failure_type, scheduling_failure_message); + CancelLeaseToGrant(work, failure_type, scheduling_failure_message); tasks_cancelled = true; return true; }); ray::erase_if>( - waiting_task_queue_, [&](const std::shared_ptr &work) { + waiting_lease_queue_, [&](const std::shared_ptr &work) { if (predicate(work)) { ReplyCancelled(work, failure_type, scheduling_failure_message); - if (!work->task_.GetTaskSpecification().GetDependencies().empty()) { - task_dependency_manager_.RemoveTaskDependencies( - work->task_.GetTaskSpecification().TaskId()); + if (!work->lease_.GetLeaseSpecification().GetDependencies().empty()) { + lease_dependency_manager_.RemoveLeaseDependencies( + work->lease_.GetLeaseSpecification().LeaseId()); } - waiting_tasks_index_.erase(work->task_.GetTaskSpecification().TaskId()); + waiting_leases_index_.erase(work->lease_.GetLeaseSpecification().LeaseId()); tasks_cancelled = true; return true; } else { @@ -883,39 +890,39 @@ bool LocalTaskManager::CancelTasks( return tasks_cancelled; } -void LocalTaskManager::CancelTaskToDispatch( +void LocalLeaseManager::CancelLeaseToGrant( const std::shared_ptr &work, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) { - const TaskID task_id = work->task_.GetTaskSpecification().TaskId(); - RAY_LOG(DEBUG) << "Canceling task " << task_id << " from dispatch queue."; + const LeaseID lease_id = work->lease_.GetLeaseSpecification().LeaseId(); + RAY_LOG(DEBUG) << "Canceling lease " << lease_id << " from leases_to_grant_queue."; ReplyCancelled(work, failure_type, scheduling_failure_message); if (work->GetState() == internal::WorkStatus::WAITING_FOR_WORKER) { // We've already acquired resources so we need to release them. cluster_resource_scheduler_.GetLocalResourceManager().ReleaseWorkerResources( work->allocated_instances_); - // Release pinned task args. - ReleaseTaskArgs(task_id); + // Release pinned lease args. + ReleaseLeaseArgs(lease_id); } - if (!work->task_.GetTaskSpecification().GetDependencies().empty()) { - task_dependency_manager_.RemoveTaskDependencies( - work->task_.GetTaskSpecification().TaskId()); + if (!work->lease_.GetLeaseSpecification().GetDependencies().empty()) { + lease_dependency_manager_.RemoveLeaseDependencies( + work->lease_.GetLeaseSpecification().LeaseId()); } - RemoveFromRunningTasksIfExists(work->task_); + RemoveFromGrantedLeasesIfExists(work->lease_); work->SetStateCancelled(); } -const RayTask *LocalTaskManager::AnyPendingTasksForResourceAcquisition( - int *num_pending_actor_creation, int *num_pending_tasks) const { - const RayTask *exemplar = nullptr; - // We are guaranteed that these tasks are blocked waiting for resources after a - // call to ScheduleAndDispatchTasks(). They may be waiting for workers as well, but +const RayLease *LocalLeaseManager::AnyPendingLeasesForResourceAcquisition( + int *num_pending_actor_creation, int *num_pending_leases) const { + const RayLease *exemplar = nullptr; + // We are guaranteed that these leases are blocked waiting for resources after a + // call to ScheduleAndGrantLeases(). They may be waiting for workers as well, but // this should be a transient condition only. - for (const auto &shapes_it : tasks_to_dispatch_) { + for (const auto &shapes_it : leases_to_grant_) { auto &work_queue = shapes_it.second; for (const auto &work_it : work_queue) { const auto &work = *work_it; - const auto &task = work_it->task_; + const auto &lease = work_it->lease_; // If the work is not in the waiting state, it will be scheduled soon or won't be // scheduled. Consider as non-pending. @@ -934,36 +941,36 @@ const RayTask *LocalTaskManager::AnyPendingTasksForResourceAcquisition( continue; } - if (task.GetTaskSpecification().IsActorCreationTask()) { + if (lease.GetLeaseSpecification().IsActorCreationTask()) { *num_pending_actor_creation += 1; } else { - *num_pending_tasks += 1; + *num_pending_leases += 1; } if (exemplar == nullptr) { - exemplar = &task; + exemplar = &lease; } } } return exemplar; } -void LocalTaskManager::Dispatch( +void LocalLeaseManager::Grant( std::shared_ptr worker, absl::flat_hash_map> &leased_workers, const std::shared_ptr &allocated_instances, - const RayTask &task, + const RayLease &lease, rpc::RequestWorkerLeaseReply *reply, std::function send_reply_callback) { - const auto &task_spec = task.GetTaskSpecification(); + const auto &lease_spec = lease.GetLeaseSpecification(); - if (task_spec.IsActorCreationTask()) { + if (lease_spec.IsActorCreationTask()) { // The actor belongs to this worker now. worker->SetLifetimeAllocatedInstances(allocated_instances); } else { worker->SetAllocatedInstances(allocated_instances); } - worker->SetAssignedTask(task); + worker->GrantLease(lease); // Pass the contact info of the worker to use. reply->set_worker_pid(worker->GetProcess().GetId()); @@ -979,7 +986,7 @@ void LocalTaskManager::Dispatch( // Update our internal view of the cluster state. std::shared_ptr allocated_resources; - if (task_spec.IsActorCreationTask()) { + if (lease_spec.IsActorCreationTask()) { allocated_resources = worker->GetLifetimeAllocatedInstances(); } else { allocated_resources = worker->GetAllocatedInstances(); @@ -1006,7 +1013,7 @@ void LocalTaskManager::Dispatch( send_reply_callback(); } -void LocalTaskManager::ClearWorkerBacklog(const WorkerID &worker_id) { +void LocalLeaseManager::ClearWorkerBacklog(const WorkerID &worker_id) { for (auto it = backlog_tracker_.begin(); it != backlog_tracker_.end();) { it->second.erase(worker_id); if (it->second.empty()) { @@ -1017,9 +1024,9 @@ void LocalTaskManager::ClearWorkerBacklog(const WorkerID &worker_id) { } } -void LocalTaskManager::SetWorkerBacklog(SchedulingClass scheduling_class, - const WorkerID &worker_id, - int64_t backlog_size) { +void LocalLeaseManager::SetWorkerBacklog(SchedulingClass scheduling_class, + const WorkerID &worker_id, + int64_t backlog_size) { if (backlog_size == 0) { backlog_tracker_[scheduling_class].erase(worker_id); if (backlog_tracker_[scheduling_class].empty()) { @@ -1030,7 +1037,7 @@ void LocalTaskManager::SetWorkerBacklog(SchedulingClass scheduling_class, } } -void LocalTaskManager::ReleaseWorkerResources(std::shared_ptr worker) { +void LocalLeaseManager::ReleaseWorkerResources(std::shared_ptr worker) { RAY_CHECK(worker != nullptr); auto allocated_instances = worker->GetAllocatedInstances() ? worker->GetAllocatedInstances() @@ -1062,7 +1069,7 @@ void LocalTaskManager::ReleaseWorkerResources(std::shared_ptr w worker->ClearLifetimeAllocatedInstances(); } -bool LocalTaskManager::ReleaseCpuResourcesFromBlockedWorker( +bool LocalLeaseManager::ReleaseCpuResourcesFromBlockedWorker( std::shared_ptr worker) { if (!worker || worker->IsBlocked()) { return false; @@ -1091,7 +1098,7 @@ bool LocalTaskManager::ReleaseCpuResourcesFromBlockedWorker( } } -bool LocalTaskManager::ReturnCpuResourcesToUnblockedWorker( +bool LocalLeaseManager::ReturnCpuResourcesToUnblockedWorker( std::shared_ptr worker) { if (!worker || !worker->IsBlocked()) { return false; @@ -1123,18 +1130,17 @@ bool LocalTaskManager::ReturnCpuResourcesToUnblockedWorker( } } -ResourceSet LocalTaskManager::CalcNormalTaskResources() const { +ResourceSet LocalLeaseManager::CalcNormalTaskResources() const { ResourceSet total_normal_task_resources; for (auto &entry : leased_workers_) { std::shared_ptr worker = entry.second; - auto &task_spec = worker->GetAssignedTask().GetTaskSpecification(); - if (!task_spec.PlacementGroupBundleId().first.IsNil()) { + auto &lease_spec = worker->GetGrantedLease().GetLeaseSpecification(); + if (!lease_spec.PlacementGroupBundleId().first.IsNil()) { continue; } - auto task_id = worker->GetAssignedTaskId(); - auto actor_id = task_id.ActorId(); - if (!actor_id.IsNil() && task_id == TaskID::ForActorCreationTask(actor_id)) { + auto actor_id = worker->GetActorId(); + if (!actor_id.IsNil() && lease_spec.IsActorCreationTask()) { // This task ID corresponds to an actor creation task. continue; } @@ -1155,7 +1161,7 @@ ResourceSet LocalTaskManager::CalcNormalTaskResources() const { return total_normal_task_resources; } -uint64_t LocalTaskManager::MaxRunningTasksPerSchedulingClass( +uint64_t LocalLeaseManager::MaxGrantedLeasesPerSchedulingClass( SchedulingClass sched_cls_id) const { auto sched_cls = TaskSpecification::GetSchedulingClassDescriptor(sched_cls_id); double cpu_req = sched_cls.resource_set.Get(ResourceID::CPU()).Double(); @@ -1168,23 +1174,24 @@ uint64_t LocalTaskManager::MaxRunningTasksPerSchedulingClass( return static_cast(std::round(total_cpus / cpu_req)); } -void LocalTaskManager::RecordMetrics() const { - ray::stats::STATS_scheduler_tasks.Record(executing_task_args_.size(), "Executing"); - ray::stats::STATS_scheduler_tasks.Record(waiting_tasks_index_.size(), "Waiting"); +void LocalLeaseManager::RecordMetrics() const { + ray::stats::STATS_scheduler_tasks.Record(granted_lease_args_.size(), "Executing"); + ray::stats::STATS_scheduler_tasks.Record(waiting_leases_index_.size(), "Waiting"); } -void LocalTaskManager::DebugStr(std::stringstream &buffer) const { - buffer << "Waiting tasks size: " << waiting_tasks_index_.size() << "\n"; - buffer << "Number of executing tasks: " << executing_task_args_.size() << "\n"; - buffer << "Number of pinned task arguments: " << pinned_task_arguments_.size() << "\n"; - buffer << "Number of total spilled tasks: " << num_task_spilled_ << "\n"; - buffer << "Number of spilled waiting tasks: " << num_waiting_task_spilled_ << "\n"; - buffer << "Number of spilled unschedulable tasks: " << num_unschedulable_task_spilled_ +void LocalLeaseManager::DebugStr(std::stringstream &buffer) const { + buffer << "Waiting leases size: " << waiting_leases_index_.size() << "\n"; + buffer << "Number of granted lease arguments: " << granted_lease_args_.size() << "\n"; + buffer << "Number of pinned lease arguments: " << pinned_lease_arguments_.size() + << "\n"; + buffer << "Number of total spilled leases: " << num_lease_spilled_ << "\n"; + buffer << "Number of spilled waiting leases: " << num_waiting_lease_spilled_ << "\n"; + buffer << "Number of spilled unschedulable leases: " << num_unschedulable_lease_spilled_ << "\n"; buffer << "Resource usage {\n"; - // Calculates how much resources are occupied by tasks or actors. - // Only iterate upto this number to avoid excessive CPU usage. + // Calculates how much resources are occupied by leases. + // Only iterate up to this number to avoid excessive CPU usage. auto max_iteration = RayConfig::instance().worker_max_resource_analysis_iteration(); uint32_t iteration = 0; for (const auto &worker : worker_pool_.GetAllRegisteredWorkers( @@ -1194,24 +1201,26 @@ void LocalTaskManager::DebugStr(std::stringstream &buffer) const { } if (worker->IsDead() // worker is dead || worker->IsBlocked() // worker is blocked by blocking Ray API - || (worker->GetAssignedTaskId().IsNil() && - worker->GetActorId().IsNil())) { // Tasks or actors not assigned + || (worker->GetGrantedLeaseId().IsNil() && + worker->GetActorId().IsNil())) { // Lease not assigned + // TODO(#55923) probably don't need to above check for ActorId since LeaseId is not + // reset for actors either // Then this shouldn't have allocated resources. continue; } - const auto &task_or_actor_name = worker->GetAssignedTask() - .GetTaskSpecification() + const auto &task_or_actor_name = worker->GetGrantedLease() + .GetLeaseSpecification() .FunctionDescriptor() ->CallString(); buffer << " - (language=" << rpc::Language_descriptor()->FindValueByNumber(worker->GetLanguage())->name() << " " - << "actor_or_task=" << task_or_actor_name << " " + << "actor_or_task" << task_or_actor_name << " " << "pid=" << worker->GetProcess().GetId() << " " << "worker_id=" << worker->WorkerId() << "): " - << worker->GetAssignedTask() - .GetTaskSpecification() + << worker->GetGrantedLease() + .GetLeaseSpecification() .GetRequiredResources() .DebugString() << "\n"; @@ -1227,13 +1236,13 @@ void LocalTaskManager::DebugStr(std::stringstream &buffer) const { buffer << "\t}\n"; } buffer << "\n"; - buffer << "Running tasks by scheduling class:\n"; + buffer << "Granted leases by scheduling class:\n"; for (const auto &pair : info_by_sched_cls_) { const auto &sched_cls = pair.first; const auto &info = pair.second; const auto &descriptor = TaskSpecification::GetSchedulingClassDescriptor(sched_cls); - buffer << " - " << descriptor.DebugString() << ": " << info.running_tasks.size() + buffer << " - " << descriptor.DebugString() << ": " << info.granted_leases.size() << "/" << info.capacity << "\n"; } } diff --git a/src/ray/raylet/local_task_manager.h b/src/ray/raylet/local_lease_manager.h similarity index 53% rename from src/ray/raylet/local_task_manager.h rename to src/ray/raylet/local_lease_manager.h index ebba83089d23..65611c2c338e 100644 --- a/src/ray/raylet/local_task_manager.h +++ b/src/ray/raylet/local_lease_manager.h @@ -23,120 +23,122 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" +#include "ray/common/lease/lease.h" #include "ray/common/ray_object.h" -#include "ray/common/task/task.h" -#include "ray/raylet/dependency_manager.h" +#include "ray/raylet/lease_dependency_manager.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/scheduling/internal.h" -#include "ray/raylet/scheduling/local_task_manager_interface.h" +#include "ray/raylet/scheduling/local_lease_manager_interface.h" #include "ray/raylet/worker.h" #include "ray/raylet/worker_pool.h" namespace ray { namespace raylet { -/// Manages the lifetime of a task on the local node. It receives request from -/// cluster_task_manager (the distributed scheduler) and does the following +/// Manages the lifetime of a lease on the local node. It receives request from +/// cluster_lease_manager (the distributed scheduler) and does the following /// steps: -/// 1. Pulling task dependencies, add the task into waiting queue. -/// 2. Once task's dependencies are all pulled locally, the task be added into -/// dispatch queue. -/// 3. For all tasks in dispatch queue, we schedule them by first acquiring -/// local resources (including pinning the objects in memory and deduct -/// cpu/gpu and other resources from local reosource manager)) . -/// If a task failed to acquire resources in step 3, we will try to -/// spill it to an different remote node. -/// 4. If all resources are acquired, we start a worker and returns the worker +/// 1. Pulling lease dependencies, add the lease into waiting queue. +/// 2. Once lease's dependencies are all pulled locally, the lease is added into +/// the grant queue. +/// 3. For all leases in the grant queue, we schedule them by first acquiring +/// local resources (including pinning the objects in memory and deducting +/// cpu/gpu and other resources from the local resource manager). +/// If a lease failed to acquire resources in step 3, we will try to +/// spill it to a different remote node. +/// 4. If all resources are acquired, we start a worker and return the worker /// address to the client once worker starts up. /// 5. When a worker finishes executing its task(s), the requester will return -/// it and we should release the resources in our view of the node's state. -/// 6. If a task has been waiting for arguments for too long, it will also be +/// the lease and we should release the resources in our view of the node's state. +/// 6. If a lease has been waiting for arguments for too long, it will also be /// spilled back to a different node. /// /// TODO(scv119): ideally, the local scheduler shouldn't be responsible for spilling, /// as it should return the request to the distributed scheduler if -/// resource accusition failed, or a task has arguments pending resolution for too long +/// resource accusition failed, or a lease has arguments pending resolution for too long /// time. -class LocalTaskManager : public ILocalTaskManager { +class LocalLeaseManager : public LocalLeaseManagerInterface { public: + /// Create a local lease manager. /// \param self_node_id: ID of local node. /// \param cluster_resource_scheduler: The resource scheduler which contains /// the state of the cluster. - /// \param task_dependency_manager_ Used to fetch task's dependencies. + /// \param lease_dependency_manager_ Used to fetch lease's dependencies. /// \param get_node_info: Function that returns the node info for a node. /// \param worker_pool: A reference to the worker pool. /// \param leased_workers: A reference to the leased workers map. - /// \param get_task_arguments: A callback for getting a tasks' arguments by + /// \param get_lease_arguments: A callback for getting a leases' arguments by /// their ids. - /// \param max_pinned_task_arguments_bytes: The cap on pinned arguments. + /// \param max_pinned_lease_arguments_bytes: The cap on pinned arguments. /// \param get_time_ms: A callback which returns the current time in milliseconds. /// \param sched_cls_cap_interval_ms: The time before we increase the cap - /// on the number of tasks that can run per + /// on the number of leases that can run per /// scheduling class. If set to 0, there is no /// cap. If it's a large number, the cap is hard. - LocalTaskManager( + LocalLeaseManager( const NodeID &self_node_id, ClusterResourceScheduler &cluster_resource_scheduler, - TaskDependencyManagerInterface &task_dependency_manager, + LeaseDependencyManagerInterface &lease_dependency_manager, internal::NodeInfoGetter get_node_info, WorkerPoolInterface &worker_pool, absl::flat_hash_map> &leased_workers, std::function &object_ids, std::vector> *results)> - get_task_arguments, - size_t max_pinned_task_arguments_bytes, + get_lease_arguments, + size_t max_pinned_lease_arguments_bytes, std::function get_time_ms = []() { return static_cast(absl::GetCurrentTimeNanos() / 1e6); }, int64_t sched_cls_cap_interval_ms = RayConfig::instance().worker_cap_initial_backoff_delay_ms()); - /// Queue task and schedule. - void QueueAndScheduleTask(std::shared_ptr work) override; + /// Queue lease and schedule. + void QueueAndScheduleLease(std::shared_ptr work) override; - // Schedule and dispatch tasks. - void ScheduleAndDispatchTasks() override; + // Schedule and dispatch leases. + void ScheduleAndGrantLeases() override; - /// Move tasks from waiting to ready for dispatch. Called when a task's + /// Move leases from waiting to ready for dispatch. Called when a lease's /// dependencies are resolved. /// - /// \param ready_ids: The tasks which are now ready to be dispatched. - void TasksUnblocked(const std::vector &ready_ids) override; + /// \param ready_ids: The leases which are now ready to be granted. + void LeasesUnblocked(const std::vector &ready_ids) override; - /// Return the finished task and release the worker resources. + /// Cleanup the lease and release the worker resources. /// This method will be removed and can be replaced by `ReleaseWorkerResources` directly /// once we remove the legacy scheduler. /// - /// \param worker: The worker which was running the task. - /// \param task: Output parameter. - void TaskFinished(std::shared_ptr worker, RayTask *task) override; + /// \param worker: The worker which was granted the lease. + /// \param lease: Output parameter. + void CleanupLease(std::shared_ptr worker, RayLease *lease) override; - /// Attempt to cancel all queued tasks that match the predicate. + /// Attempt to cancel all queued leases that match the predicate. /// - /// \param predicate: A function that returns true if a task needs to be cancelled. + /// \param predicate: A function that returns true if a lease needs to be cancelled. /// \param failure_type: The reason for cancellation. /// \param scheduling_failure_message: The reason message for cancellation. - /// \return True if any task was successfully cancelled. - bool CancelTasks(std::function &)> predicate, - rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, - const std::string &scheduling_failure_message) override; + /// \return True if any lease was successfully cancelled. + bool CancelLeases( + std::function &)> predicate, + rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, + const std::string &scheduling_failure_message) override; - /// Return with an exemplar if any tasks are pending resource acquisition. + /// Return with an exemplar if any leases are pending resource acquisition. /// - /// \param[in,out] num_pending_actor_creation: Number of pending actor creation tasks. - /// \param[in,out] num_pending_tasks: Number of pending tasks. - /// \return An example task that is deadlocking if any tasks are pending resource + /// \param[in,out] num_pending_actor_creation: Number of pending actor creation leases. + /// \param[in,out] num_pending_leases: Number of pending leases. + /// \return An example lease that is deadlocking if any leases are pending resource /// acquisition. - const RayTask *AnyPendingTasksForResourceAcquisition( - int *num_pending_actor_creation, int *num_pending_tasks) const override; + const RayLease *AnyPendingLeasesForResourceAcquisition( + int *num_pending_actor_creation, int *num_pending_leases) const override; - /// Call once a task finishes (i.e. a worker is returned). + /// Call once a lease finishes (i.e. a worker is returned). /// - /// \param worker: The worker which was running the task. + /// \param worker: The worker which was granted the lease. void ReleaseWorkerResources(std::shared_ptr worker) override; - /// When a task is blocked in ray.get or ray.wait, the worker who is executing the task - /// should give up the CPU resources allocated for the running task for the time being - /// and the worker itself should also be marked as blocked. + /// When a lease is blocked in ray.get or ray.wait, the worker who is executing the + /// lease should give up the CPU resources allocated for the granted lease for the time + /// being and the worker itself should also be marked as blocked. /// /// \param worker: The worker who will give up the CPU resources. /// \return true if the cpu resources of the specified worker are released successfully, @@ -144,7 +146,7 @@ class LocalTaskManager : public ILocalTaskManager { bool ReleaseCpuResourcesFromBlockedWorker( std::shared_ptr worker) override; - /// When a task is no longer blocked in a ray.get or ray.wait, the CPU resources that + /// When a lease is no longer blocked in a ray.get or ray.wait, the CPU resources that /// the worker gave up should be returned to it. /// /// \param worker The blocked worker. @@ -165,8 +167,8 @@ class LocalTaskManager : public ILocalTaskManager { void ClearWorkerBacklog(const WorkerID &worker_id) override; const absl::flat_hash_map>> - &GetTaskToDispatch() const override { - return tasks_to_dispatch_; + &GetLeasesToGrant() const override { + return leases_to_grant_; } const absl::flat_hash_map> @@ -178,160 +180,162 @@ class LocalTaskManager : public ILocalTaskManager { void DebugStr(std::stringstream &buffer) const override; - size_t GetNumTaskSpilled() const override { return num_task_spilled_; } - size_t GetNumWaitingTaskSpilled() const override { return num_waiting_task_spilled_; } - size_t GetNumUnschedulableTaskSpilled() const override { - return num_unschedulable_task_spilled_; + size_t GetNumLeaseSpilled() const override { return num_lease_spilled_; } + size_t GetNumWaitingLeaseSpilled() const override { return num_waiting_lease_spilled_; } + size_t GetNumUnschedulableLeaseSpilled() const override { + return num_unschedulable_lease_spilled_; } private: struct SchedulingClassInfo; - void RemoveFromRunningTasksIfExists(const RayTask &task); + void RemoveFromGrantedLeasesIfExists(const RayLease &lease); /// Handle the popped worker from worker pool. bool PoppedWorkerHandler(const std::shared_ptr worker, PopWorkerStatus status, - const TaskID &task_id, + const LeaseID &lease_id, SchedulingClass scheduling_class, const std::shared_ptr &work, bool is_detached_actor, const rpc::Address &owner_address, const std::string &runtime_env_setup_error_message); - /// Cancels a task in tasks_to_dispatch_. Does not remove it from tasks_to_dispatch_. - void CancelTaskToDispatch( + /// Cancels a lease in leases_to_grant_. Does not remove it from leases_to_grant_. + void CancelLeaseToGrant( const std::shared_ptr &work, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type = rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_INTENDED, const std::string &scheduling_failure_message = ""); - /// Attempts to dispatch all tasks which are ready to run. A task - /// will be dispatched if it is on `tasks_to_dispatch_` and there are still + /// Attempts to grant all leases which are ready to run. A lease + /// will be granted if it is on `leases_to_grant_` and there are still /// available resources on the node. /// - /// If there are not enough resources locally, up to one task per resource - /// shape (the task at the head of the queue) will get spilled back to a + /// If there are not enough resources locally, up to one lease per resource + /// shape (the lease at the head of the queue) will get spilled back to a /// different node. - void DispatchScheduledTasksToWorkers(); + void GrantScheduledLeasesToWorkers(); /// Helper method when the current node does not have the available resources to run a - /// task. + /// lease. /// - /// \returns true if the task was spilled. The task may not be spilled if the + /// \returns true if the lease was spilled. The lease may not be spilled if the /// spillback policy specifies the local node (which may happen if no other nodes have /// the requested resources available). bool TrySpillback(const std::shared_ptr &work, bool &is_infeasible); - // Try to spill waiting tasks to a remote node, starting from the end of the + // Try to spill waiting leases to a remote node, starting from the end of the // queue. - void SpillWaitingTasks(); + void SpillWaitingLeases(); - /// Calculate the maximum number of running tasks for a given scheduling + /// Calculate the maximum number of granted leases for a given scheduling /// class. https://github.com/ray-project/ray/issues/16973 /// /// \param sched_cls_id The scheduling class in question. /// \returns The maximum number instances of that scheduling class that - /// should be running (or blocked) at once. - uint64_t MaxRunningTasksPerSchedulingClass(SchedulingClass sched_cls_id) const; + /// should be granted (or blocked) at once. + uint64_t MaxGrantedLeasesPerSchedulingClass(SchedulingClass sched_cls_id) const; /// Recompute the debug stats. - /// It is needed because updating the debug state is expensive for cluster_task_manager. + /// It is needed because updating the debug state is expensive for + /// cluster_lease_manager. /// TODO(sang): Update the internal states value dynamically instead of iterating the /// data structure. void RecomputeDebugStats() const; - void Dispatch( + void Grant( std::shared_ptr worker, absl::flat_hash_map> &leased_workers_, const std::shared_ptr &allocated_instances, - const RayTask &task, + const RayLease &lease, rpc::RequestWorkerLeaseReply *reply, std::function send_reply_callback); void Spillback(const NodeID &spillback_to, const std::shared_ptr &work); - // Helper function to pin a task's args immediately before dispatch. This + // Helper function to pin a lease's args immediately before being granted. This // returns false if there are missing args (due to eviction) or if there is - // not enough memory available to dispatch the task, due to other executing - // tasks' arguments. - bool PinTaskArgsIfMemoryAvailable(const TaskSpecification &spec, bool *args_missing); + // not enough memory available to grant the lease, due to other granted + // leases' arguments. + bool PinLeaseArgsIfMemoryAvailable(const LeaseSpecification &lease_spec, + bool *args_missing); - // Helper functions to pin and release an executing task's args. - void PinTaskArgs(const TaskSpecification &spec, - std::vector> args); - void ReleaseTaskArgs(const TaskID &task_id); + // Helper functions to pin and release a granted lease's args. + void PinLeaseArgs(const LeaseSpecification &lease_spec, + std::vector> args); + void ReleaseLeaseArgs(const LeaseID &lease_id); private: - /// Determine whether a task should be immediately dispatched, + /// Determine whether a lease should be immediately granted, /// or placed on a wait queue. - void WaitForTaskArgsRequests(std::shared_ptr work); + void WaitForLeaseArgsRequests(std::shared_ptr work); const NodeID &self_node_id_; const scheduling::NodeID self_scheduling_node_id_; /// Responsible for resource tracking/view of the cluster. ClusterResourceScheduler &cluster_resource_scheduler_; - /// Class to make task dependencies to be local. - TaskDependencyManagerInterface &task_dependency_manager_; + /// Class to make lease dependencies to be local. + LeaseDependencyManagerInterface &lease_dependency_manager_; /// Function to get the node information of a given node id. internal::NodeInfoGetter get_node_info_; const int max_resource_shapes_per_load_report_; - /// Tracking information about the currently running tasks in a scheduling - /// class. This information is used to place a cap on the number of running - /// running tasks per scheduling class. + /// Tracking information about the currently granted leases in a scheduling + /// class. This information is used to place a cap on the number of + /// granted leases per scheduling class. struct SchedulingClassInfo { explicit SchedulingClassInfo(int64_t cap) : capacity(cap), next_update_time(std::numeric_limits::max()) {} - /// Track the running task ids in this scheduling class. + /// Track the granted lease ids in this scheduling class. /// - /// TODO(hjiang): Store cgroup manager along with task id as the value for map. - absl::flat_hash_set running_tasks; - /// The total number of tasks that can run from this scheduling class. + /// TODO(hjiang): Store cgroup manager along with lease id as the value for map. + absl::flat_hash_set granted_leases; + /// The total number of leases that can run from this scheduling class. uint64_t capacity; - /// The next time that a new task of this scheduling class may be dispatched. + /// The next time that a new lease of this scheduling class may be dispatched. int64_t next_update_time; }; - /// Mapping from scheduling class to information about the running tasks of + /// Mapping from scheduling class to information about the granted leases of /// the scheduling class. See `struct SchedulingClassInfo` above for more /// details about what information is tracked. absl::flat_hash_map info_by_sched_cls_; /// Queue of lease requests that should be scheduled onto workers. - /// Tasks move from scheduled | waiting -> dispatch. - /// Tasks can also move from dispatch -> waiting if one of their arguments is + /// Leases move from scheduled | waiting -> granting. + /// Leases can also move from granting -> waiting if one of their arguments is /// evicted. - /// All tasks in this map that have dependencies should be registered with - /// the dependency manager, in case a dependency gets evicted while the task + /// All leases in this map that have dependencies should be registered with + /// the dependency manager, in case a dependency gets evicted while the lease /// is still queued. /// Note that if a queue exists, it should be guaranteed to be non-empty. absl::flat_hash_map>> - tasks_to_dispatch_; + leases_to_grant_; - /// Tasks waiting for arguments to be transferred locally. - /// Tasks move from waiting -> dispatch. - /// Tasks can also move from dispatch -> waiting if one of their arguments is + /// Leases waiting for arguments to be transferred locally. + /// Leases move from waiting -> granting. + /// Leases can also move from granting -> waiting if one of their arguments is /// evicted. - /// All tasks in this map that have dependencies should be registered with - /// the dependency manager, so that they can be moved to dispatch once their + /// All leases in this map that have dependencies should be registered with + /// the dependency manager, so that they can be moved to granting once their /// dependencies are local. - /// - /// We keep these in a queue so that tasks can be spilled back from the end - /// of the queue. This is to try to prioritize spilling tasks whose + + /// We keep these in a queue so that leases can be spilled back from the end + /// of the queue. This is to try to prioritize spilling leases whose /// dependencies may not be fetched locally yet. - /// - /// Note that because tasks can also move from dispatch -> waiting, the order + + /// Note that because leases can also move from grant -> waiting, the order /// in this queue may not match the order in which we initially received the - /// tasks. This also means that the PullManager may request dependencies for - /// these tasks in a different order than the waiting task queue. + /// leases. This also means that the PullManager may request dependencies for + /// these leases in a different order than the waiting lease queue. /// Note that if a queue exists, it should be guaranteed to be non-empty. - std::list> waiting_task_queue_; + std::list> waiting_lease_queue_; /// An index for the above queue. - absl::flat_hash_map>::iterator> - waiting_tasks_index_; + absl::flat_hash_map>::iterator> + waiting_leases_index_; /// Track the backlog of all workers belonging to this raylet. absl::flat_hash_map> @@ -342,30 +346,29 @@ class LocalTaskManager : public ILocalTaskManager { WorkerPoolInterface &worker_pool_; absl::flat_hash_map> &leased_workers_; - /// Callback to get references to task arguments. These will be pinned while - /// the task is running. + /// Callback to get references to lease arguments. These will be pinned while + /// the lease is granted. std::function &object_ids, std::vector> *results)> - get_task_arguments_; + get_lease_arguments_; - /// Arguments needed by currently granted lease requests. These should be - /// pinned before the lease is granted to ensure that the arguments are not - /// evicted before the task(s) start running. - absl::flat_hash_map> executing_task_args_; + /// Arguments needed by currently granted leases. These should be pinned before + /// the lease is granted to ensure that the arguments are not evicted. + absl::flat_hash_map> granted_lease_args_; - /// All arguments of running tasks, which are also pinned in the object - /// store. The value is a pair: (the pointer to the object store that should - /// be deleted once the object is no longer needed, number of tasks that - /// depend on the object). + /// All arguments of granted leases, which are also pinned in the object store. + /// The value is a pair: (the pointer to the object store that should be deleted + /// once the object is no longer needed, number of leases that depend on the + /// object). absl::flat_hash_map, size_t>> - pinned_task_arguments_; + pinned_lease_arguments_; - /// The total number of arguments pinned for running tasks. + /// The total number of arguments pinned for granted leases. /// Used for debug purposes. - size_t pinned_task_arguments_bytes_ = 0; + size_t pinned_lease_arguments_bytes_ = 0; - /// The maximum amount of bytes that can be used by executing task arguments. - size_t max_pinned_task_arguments_bytes_; + /// The maximum amount of bytes that can be used by granted lease arguments. + size_t max_pinned_lease_arguments_bytes_; /// Returns the current time in milliseconds. std::function get_time_ms_; @@ -378,16 +381,16 @@ class LocalTaskManager : public ILocalTaskManager { const int64_t sched_cls_cap_max_ms_; - size_t num_task_spilled_ = 0; - size_t num_waiting_task_spilled_ = 0; - size_t num_unschedulable_task_spilled_ = 0; + size_t num_lease_spilled_ = 0; + size_t num_waiting_lease_spilled_ = 0; + size_t num_unschedulable_lease_spilled_ = 0; friend class SchedulerResourceReporter; - friend class ClusterTaskManagerTest; + friend class ClusterLeaseManagerTest; friend class SchedulerStats; - friend class LocalTaskManagerTest; - FRIEND_TEST(ClusterTaskManagerTest, FeasibleToNonFeasible); - FRIEND_TEST(LocalTaskManagerTest, TestTaskDispatchingOrder); + friend class LocalLeaseManagerTest; + FRIEND_TEST(ClusterLeaseManagerTest, FeasibleToNonFeasible); + FRIEND_TEST(LocalLeaseManagerTest, TestLeaseGrantingOrder); }; } // namespace raylet } // namespace ray diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 4ce139301327..bc886a851236 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -26,9 +26,9 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/cgroup/cgroup_manager.h" #include "ray/common/id.h" +#include "ray/common/lease/lease.h" #include "ray/common/ray_config.h" #include "ray/common/status.h" -#include "ray/common/task/task_common.h" #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/object_manager/ownership_object_directory.h" #include "ray/raylet/local_object_manager.h" @@ -43,7 +43,7 @@ #include "ray/util/stream_redirection_options.h" #include "ray/util/subreaper.h" #include "ray/util/time.h" -#include "scheduling/cluster_task_manager.h" +#include "scheduling/cluster_lease_manager.h" using json = nlohmann::json; @@ -269,11 +269,11 @@ int main(int argc, char *argv[]) { std::unique_ptr local_object_manager; /// These classes make up the new scheduler. ClusterResourceScheduler is /// responsible for maintaining a view of the cluster state w.r.t resource - /// usage. ClusterTaskManager is responsible for queuing, spilling back, and - /// dispatching tasks. + /// usage. ClusterLeaseManager is responsible for queuing, spilling back, and + /// granting leases. std::unique_ptr cluster_resource_scheduler; - std::unique_ptr local_task_manager; - std::unique_ptr cluster_task_manager; + std::unique_ptr local_lease_manager; + std::unique_ptr cluster_lease_manager; /// The raylet client to initiate the pubsub to core workers (owners). /// It is used to subscribe objects to evict. std::unique_ptr core_worker_subscriber; @@ -282,13 +282,13 @@ int main(int argc, char *argv[]) { std::unique_ptr object_directory; /// Manages client requests for object transfers and availability. std::unique_ptr object_manager; - /// A manager to resolve objects needed by queued tasks and workers that + /// A manager to resolve objects needed by queued leases and workers that /// called `ray.get` or `ray.wait`. - std::unique_ptr dependency_manager; + std::unique_ptr lease_dependency_manager; /// The client to export metrics to the metrics agent. std::unique_ptr metrics_agent_client; /// Map of workers leased out to clients. - absl::flat_hash_map> + absl::flat_hash_map> leased_workers; // Enable subreaper. This is called in `AsyncGetInternalConfig` below, but MSVC does @@ -511,12 +511,12 @@ int main(int argc, char *argv[]) { return node_manager_config.num_workers_soft_limit; } // If no limit is provided, use the available number of CPUs, - // assuming that each incoming task will likely require 1 CPU. + // assuming that each incoming lease will likely require 1 CPU. // We floor the available CPUs to the nearest integer to avoid // starting too many workers when there is less than 1 CPU left. // Otherwise, we could end up repeatedly starting the worker, then // killing it because it idles for too long. The downside is that - // we will be slower to schedule tasks that could use a fraction + // we will be slower to schedule leases that could use a fraction // of a CPU. return static_cast( cluster_resource_scheduler->GetLocalResourceManager() @@ -531,7 +531,7 @@ int main(int argc, char *argv[]) { node_manager_config.worker_commands, node_manager_config.native_library_path, /*starting_worker_timeout_callback=*/ - [&] { cluster_task_manager->ScheduleAndDispatchTasks(); }, + [&] { cluster_lease_manager->ScheduleAndGrantLeases(); }, node_manager_config.ray_debugger_external, /*get_time=*/[]() { return absl::Now(); }, node_manager_config.enable_resource_isolation); @@ -669,8 +669,8 @@ int main(int argc, char *argv[]) { /*core_worker_subscriber_=*/core_worker_subscriber.get(), object_directory.get()); - dependency_manager = - std::make_unique(*object_manager); + lease_dependency_manager = + std::make_unique(*object_manager); cluster_resource_scheduler = std::make_unique( main_service, @@ -704,31 +704,31 @@ int main(int argc, char *argv[]) { auto get_node_info_func = [&](const NodeID &id) { return gcs_client->Nodes().Get(id); }; - auto announce_infeasible_task = [](const ray::RayTask &task) { - /// Publish the infeasible task error to GCS so that drivers can subscribe to it + auto announce_infeasible_lease = [](const ray::RayLease &lease) { + /// Publish the infeasible lease error to GCS so that drivers can subscribe to it /// and print. bool suppress_warning = false; - if (!task.GetTaskSpecification().PlacementGroupBundleId().first.IsNil()) { - // If the task is part of a placement group, do nothing. If necessary, the + if (!lease.GetLeaseSpecification().PlacementGroupBundleId().first.IsNil()) { + // If the lease is part of a placement group, do nothing. If necessary, the // infeasible warning should come from the placement group scheduling, not the - // task scheduling. + // lease scheduling. suppress_warning = true; } - // Push a warning to the task's driver that this task is currently infeasible. + // Push a warning to the lease's driver that this lease is currently infeasible. if (!suppress_warning) { std::ostringstream error_message; error_message - << "The actor or task with ID " << task.GetTaskSpecification().TaskId() + << "The lease with ID " << lease.GetLeaseSpecification().LeaseId() << " cannot be scheduled right now. It requires " - << task.GetTaskSpecification().GetRequiredPlacementResources().DebugString() + << lease.GetLeaseSpecification().GetRequiredPlacementResources().DebugString() << " for placement, however the cluster currently cannot provide the " "requested " "resources. The required resources may be added as autoscaling takes " "place " "or placement groups are scheduled. Otherwise, consider reducing the " - "resource requirements of the task."; + "resource requirements of the lease."; std::string error_message_str = error_message.str(); RAY_LOG(WARNING) << error_message_str; } @@ -751,10 +751,10 @@ int main(int argc, char *argv[]) { max_task_args_memory = 0; } - local_task_manager = std::make_unique( + local_lease_manager = std::make_unique( raylet_node_id, *cluster_resource_scheduler, - *dependency_manager, + *lease_dependency_manager, get_node_info_func, *worker_pool, leased_workers, @@ -764,12 +764,12 @@ int main(int argc, char *argv[]) { }, max_task_args_memory); - cluster_task_manager = - std::make_unique(raylet_node_id, - *cluster_resource_scheduler, - get_node_info_func, - announce_infeasible_task, - *local_task_manager); + cluster_lease_manager = + std::make_unique(raylet_node_id, + *cluster_resource_scheduler, + get_node_info_func, + announce_infeasible_lease, + *local_lease_manager); auto raylet_client_factory = [&](const NodeID &id) { const ray::rpc::GcsNodeInfo *node_info = gcs_client->Nodes().Get(id); @@ -791,12 +791,12 @@ int main(int argc, char *argv[]) { *raylet_client_pool, *core_worker_subscriber, *cluster_resource_scheduler, - *local_task_manager, - *cluster_task_manager, + *local_lease_manager, + *cluster_lease_manager, *object_directory, *object_manager, *local_object_manager, - *dependency_manager, + *lease_dependency_manager, *worker_pool, leased_workers, *plasma_client, diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 0806b072b696..6a958c060631 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -34,17 +34,16 @@ #include "ray/common/common_protocol.h" #include "ray/common/constants.h" #include "ray/common/grpc_util.h" +#include "ray/common/lease/lease.h" #include "ray/common/memory_monitor.h" #include "ray/common/scheduling/scheduling_ids.h" #include "ray/common/status.h" -#include "ray/common/task/task_common.h" -#include "ray/common/task/task_spec.h" #include "ray/flatbuffers/node_manager_generated.h" #include "ray/gcs/pb_util.h" #include "ray/ipc/client_connection.h" #include "ray/object_manager/ownership_object_directory.h" #include "ray/raylet/local_object_manager_interface.h" -#include "ray/raylet/scheduling/cluster_task_manager.h" +#include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/worker_killing_policy.h" #include "ray/raylet/worker_pool.h" #include "ray/rpc/node_manager/node_manager_client.h" @@ -121,12 +120,12 @@ NodeManager::NodeManager( rpc::RayletClientPool &raylet_client_pool, pubsub::SubscriberInterface &core_worker_subscriber, ClusterResourceScheduler &cluster_resource_scheduler, - ILocalTaskManager &local_task_manager, - ClusterTaskManagerInterface &cluster_task_manager, + LocalLeaseManagerInterface &local_lease_manager, + ClusterLeaseManagerInterface &cluster_lease_manager, IObjectDirectory &object_directory, ObjectManagerInterface &object_manager, LocalObjectManagerInterface &local_object_manager, - DependencyManager &dependency_manager, + LeaseDependencyManager &lease_dependency_manager, WorkerPoolInterface &worker_pool, absl::flat_hash_map> &leased_workers, plasma::PlasmaClientInterface &store_client, @@ -150,10 +149,10 @@ NodeManager::NodeManager( periodical_runner_(PeriodicalRunner::Create(io_service)), report_resources_period_ms_(config.report_resources_period_ms), initial_config_(config), - dependency_manager_(dependency_manager), + lease_dependency_manager_(lease_dependency_manager), wait_manager_(/*is_object_local*/ [this](const ObjectID &object_id) { - return dependency_manager_.CheckObjectLocal(object_id); + return lease_dependency_manager_.CheckObjectLocal(object_id); }, /*delay_executor*/ [this](std::function fn, int64_t delay_ms) { @@ -171,8 +170,8 @@ NodeManager::NodeManager( global_gc_throttler_(RayConfig::instance().global_gc_min_interval_s() * 1e9), local_gc_interval_ns_(RayConfig::instance().local_gc_interval_s() * 1e9), cluster_resource_scheduler_(cluster_resource_scheduler), - local_task_manager_(local_task_manager), - cluster_task_manager_(cluster_task_manager), + local_lease_manager_(local_lease_manager), + cluster_lease_manager_(cluster_lease_manager), record_metrics_period_ms_(config.record_metrics_period_ms), next_resource_seq_no_(0), ray_syncer_(io_service_, self_node_id_.Binary()), @@ -190,9 +189,9 @@ NodeManager::NodeManager( std::make_unique(cluster_resource_scheduler_); periodical_runner_->RunFnPeriodically( - [this]() { cluster_task_manager_.ScheduleAndDispatchTasks(); }, + [this]() { cluster_lease_manager_.ScheduleAndGrantLeases(); }, RayConfig::instance().worker_cap_initial_backoff_delay_ms(), - "NodeManager.ScheduleAndDispatchTasks"); + "NodeManager.ScheduleAndGrantLeases"); periodical_runner_->RunFnPeriodically( [this]() { CheckForUnexpectedWorkerDisconnects(); }, @@ -227,7 +226,7 @@ NodeManager::NodeManager( worker_pool_.SetRuntimeEnvAgentClient(std::move(runtime_env_agent_client)); worker_pool_.Start(); - periodical_runner_->RunFnPeriodically([this]() { GCTaskFailureReason(); }, + periodical_runner_->RunFnPeriodically([this]() { GCWorkerFailureReason(); }, RayConfig::instance().task_failure_entry_ttl_ms(), "NodeManager.GCTaskFailureReason"); } @@ -409,10 +408,10 @@ void NodeManager::HandleJobStarted(const JobID &job_id, const JobTableData &job_ << " is dead: " << job_data.is_dead() << " driver address: " << job_data.driver_address().ip_address(); worker_pool_.HandleJobStarted(job_id, job_data.config()); - // Tasks of this job may already arrived but failed to pop a worker because the job - // config is not local yet. So we trigger dispatching again here to try to - // reschedule these tasks. - cluster_task_manager_.ScheduleAndDispatchTasks(); + // Leases of this job may already arrived but failed to pop a worker because the job + // config is not local yet. So we trigger granting again here to try to + // reschedule these leases. + cluster_lease_manager_.ScheduleAndGrantLeases(); } void NodeManager::HandleJobFinished(const JobID &job_id, const JobTableData &job_data) { @@ -515,10 +514,10 @@ void NodeManager::HandleReleaseUnusedBundles(rpc::ReleaseUnusedBundlesRequest re // Cancel lease requests that are waiting for workers // to free the acquired pg bundle resources // so that pg bundle can be returned. - local_task_manager_.CancelTasks( + local_lease_manager_.CancelLeases( [&](const std::shared_ptr &work) { const auto bundle_id = - work->task_.GetTaskSpecification().PlacementGroupBundleId(); + work->lease_.GetLeaseSpecification().PlacementGroupBundleId(); return !bundle_id.first.IsNil() && (0 == in_use_bundles.count(bundle_id)) && (work->GetState() == internal::WorkStatus::WAITING_FOR_WORKER); }, @@ -544,7 +543,7 @@ void NodeManager::HandleReleaseUnusedBundles(rpc::ReleaseUnusedBundlesRequest re for (const auto &worker : workers_associated_with_unused_bundles) { RAY_LOG(DEBUG) .WithField(worker->GetBundleId().first) - .WithField(worker->GetAssignedTaskId()) + .WithField(worker->GetGrantedLeaseId()) .WithField(worker->GetActorId()) .WithField(worker->WorkerId()) << "Destroying worker since its bundle was unused, bundle index: " @@ -596,21 +595,23 @@ void NodeManager::HandleGetObjectsInfo(rpc::GetObjectsInfoRequest request, /*on_all_replied*/ [total, reply]() { reply->set_total(*total); }); } -void NodeManager::HandleGetTaskFailureCause(rpc::GetTaskFailureCauseRequest request, - rpc::GetTaskFailureCauseReply *reply, - rpc::SendReplyCallback send_reply_callback) { - const TaskID task_id = TaskID::FromBinary(request.task_id()); - RAY_LOG(DEBUG) << "Received a HandleGetTaskFailureCause request for task " << task_id; +void NodeManager::HandleGetWorkerFailureCause( + rpc::GetWorkerFailureCauseRequest request, + rpc::GetWorkerFailureCauseReply *reply, + rpc::SendReplyCallback send_reply_callback) { + const LeaseID lease_id = LeaseID::FromBinary(request.lease_id()); + RAY_LOG(DEBUG) << "Received a HandleGetWorkerFailureCause request for lease " + << lease_id; - auto it = task_failure_reasons_.find(task_id); - if (it != task_failure_reasons_.end()) { - RAY_LOG(DEBUG) << "task " << task_id << " has failure reason " + auto it = worker_failure_reasons_.find(lease_id); + if (it != worker_failure_reasons_.end()) { + RAY_LOG(DEBUG) << "lease " << lease_id << " has failure reason " << ray::gcs::RayErrorInfoToString(it->second.ray_error_info_) << ", fail immediately: " << !it->second.should_retry_; reply->mutable_failure_cause()->CopyFrom(it->second.ray_error_info_); reply->set_fail_task_immediately(!it->second.should_retry_); } else { - RAY_LOG(INFO) << "didn't find failure cause for task " << task_id; + RAY_LOG(INFO) << "didn't find failure cause for lease " << lease_id; } send_reply_callback(Status::OK(), nullptr, nullptr); @@ -710,31 +711,31 @@ void NodeManager::QueryAllWorkerStates( // This warns users that there could be the resource deadlock. It works this way; // - If there's no available workers for scheduling -// - But if there are still pending tasks waiting for resource acquisition +// - But if there are still pending leases waiting for resource acquisition // It means the cluster might not have enough resources to be in progress. // Note that this can print the false negative messages // e.g., there are many actors taking up resources for a long time. void NodeManager::WarnResourceDeadlock() { int pending_actor_creations = 0; - int pending_tasks = 0; + int pending_leases = 0; // Check if any progress is being made on this raylet. if (worker_pool_.IsWorkerAvailableForScheduling()) { - // Progress is being made in a task, don't warn. + // Progress is being made in a lease, don't warn. resource_deadlock_warned_ = 0; return; } - auto exemplar = cluster_task_manager_.AnyPendingTasksForResourceAcquisition( - &pending_actor_creations, &pending_tasks); - // Check if any tasks are blocked on resource acquisition. + auto exemplar = cluster_lease_manager_.AnyPendingLeasesForResourceAcquisition( + &pending_actor_creations, &pending_leases); + // Check if any leases are blocked on resource acquisition. if (exemplar == nullptr) { - // No pending tasks, no need to warn. + // No pending leases, no need to warn. resource_deadlock_warned_ = 0; return; } - // Push an warning to the driver that a task is blocked trying to acquire resources. + // Push an warning to the driver that a lease is blocked trying to acquire resources. // To avoid spurious triggers, only take action starting with the second time. // case resource_deadlock_warned_: 0 => first time, don't do anything yet // case resource_deadlock_warned_: 1 => second time, print a warning @@ -750,26 +751,26 @@ void NodeManager::WarnResourceDeadlock() { } RAY_LOG(WARNING) - << "The actor or task with ID " << exemplar->GetTaskSpecification().TaskId() + << "The lease with ID " << exemplar->GetLeaseSpecification().LeaseId() << " cannot be scheduled right now. You can ignore this message if this " << "Ray cluster is expected to auto-scale or if you specified a " - << "runtime_env for this actor or task, which may take time to install. " + << "runtime_env for this actor or lease, which may take time to install. " << "Otherwise, this is likely due to all cluster resources being claimed " << "by actors. To resolve the issue, consider creating fewer actors or " << "increasing the resources available to this Ray cluster.\n" - << "Required resources for this actor or task: " - << exemplar->GetTaskSpecification().GetRequiredPlacementResources().DebugString() + << "Required resources for this lease: " + << exemplar->GetLeaseSpecification().GetRequiredPlacementResources().DebugString() << "\n" << "Available resources on this node: " << cluster_resource_scheduler_.GetClusterResourceManager() .GetNodeResourceViewString(scheduling::NodeID(self_node_id_.Binary())) - << " In total there are " << pending_tasks << " pending tasks and " + << " In total there are " << pending_leases << " pending leases and " << pending_actor_creations << " pending actors on this node."; - RAY_LOG_EVERY_MS(WARNING, 10 * 1000) << cluster_task_manager_.DebugStr(); + RAY_LOG_EVERY_MS(WARNING, 10 * 1000) << cluster_lease_manager_.DebugStr(); } - // Try scheduling tasks. Without this, if there's no more tasks coming in, deadlocked - // tasks are never be scheduled. - cluster_task_manager_.ScheduleAndDispatchTasks(); + // Try scheduling leases. Without this, if there's no more leases coming in, deadlocked + // leases are never be scheduled. + cluster_lease_manager_.ScheduleAndGrantLeases(); } void NodeManager::NodeAdded(const GcsNodeInfo &node_info) { @@ -819,7 +820,7 @@ void NodeManager::NodeRemoved(const NodeID &node_id) { failed_nodes_cache_.insert(node_id); - cluster_task_manager_.CancelAllTasksOwnedBy(node_id); + cluster_lease_manager_.CancelAllLeasesOwnedBy(node_id); // Clean up workers that were owned by processes that were on the failed // node. @@ -863,7 +864,7 @@ void NodeManager::HandleUnexpectedWorkerFailure(const WorkerID &worker_id) { RAY_LOG(DEBUG).WithField(worker_id) << "Worker failed"; failed_workers_cache_.insert(worker_id); - cluster_task_manager_.CancelAllTasksOwnedBy(worker_id); + cluster_lease_manager_.CancelAllLeasesOwnedBy(worker_id); for (const auto &[_, worker] : leased_workers_) { const auto owner_worker_id = @@ -1143,10 +1144,10 @@ Status NodeManager::RegisterForNewWorker( Status status = worker_pool_.RegisterWorker(worker, pid, worker_startup_token, send_reply_callback); if (!status.ok()) { - // If the worker failed to register to Raylet, trigger task dispatching here to + // If the worker failed to register to Raylet, trigger lease granting here to // allow new worker processes to be started (if capped by // maximum_startup_concurrency). - cluster_task_manager_.ScheduleAndDispatchTasks(); + cluster_lease_manager_.ScheduleAndGrantLeases(); } return status; } @@ -1158,11 +1159,13 @@ Status NodeManager::RegisterForNewDriver( const ray::protocol::RegisterClientRequest *message, std::function send_reply_callback) { worker->SetProcess(Process::FromPid(pid)); - // Compute a dummy driver task id from a given driver. - // The task id set in the worker here should be consistent with the task + // Compute a dummy driver lease id from a given driver. + // The lease id set in the worker here should be consistent with the lease // id set in the core worker. - const TaskID driver_task_id = TaskID::ForDriverTask(job_id); - worker->AssignTaskId(driver_task_id); + // TODO(#56010): We shouldn't need to have a special lease id for the driver, just check + // the worker type instead + const LeaseID driver_lease_id = LeaseID::DriverLeaseId(worker->WorkerId()); + worker->GrantLeaseId(driver_lease_id); rpc::JobConfig job_config; job_config.ParseFromString(message->serialized_job_config()->str()); @@ -1262,9 +1265,9 @@ void NodeManager::HandleWorkerAvailable(const std::shared_ptr & bool worker_idle = true; - // If the worker was assigned a task, mark it as finished. - if (!worker->GetAssignedTaskId().IsNil()) { - worker_idle = FinishAssignedTask(worker); + // If the worker was granted a lease, clean up any lease resources and state + if (!worker->GetGrantedLeaseId().IsNil()) { + worker_idle = CleanupLease(worker); } if (worker_idle) { @@ -1272,7 +1275,7 @@ void NodeManager::HandleWorkerAvailable(const std::shared_ptr & worker_pool_.PushWorker(worker); } - cluster_task_manager_.ScheduleAndDispatchTasks(); + cluster_lease_manager_.ScheduleAndGrantLeases(); } void SendDisconnectClientReply(const WorkerID &worker_id, @@ -1324,8 +1327,8 @@ void NodeManager::DisconnectClient(const std::shared_ptr &clie RAY_CHECK(worker != nullptr); RAY_CHECK(!(is_worker && is_driver)); // Clean up any open ray.get or ray.wait calls that the worker made. - dependency_manager_.CancelGetRequest(worker->WorkerId()); - dependency_manager_.CancelWaitRequest(worker->WorkerId()); + lease_dependency_manager_.CancelGetRequest(worker->WorkerId()); + lease_dependency_manager_.CancelWaitRequest(worker->WorkerId()); // Erase any lease metadata. ReleaseWorker(worker->WorkerId()); @@ -1349,15 +1352,15 @@ void NodeManager::DisconnectClient(const std::shared_ptr &clie if (is_worker) { const ActorID &actor_id = worker->GetActorId(); - const TaskID &task_id = worker->GetAssignedTaskId(); - // If the worker was running a task or actor, clean up the task and push an + const LeaseID &lease_id = worker->GetGrantedLeaseId(); + // If the worker was granted a lease, clean up the lease and push an // error to the driver, unless the worker is already dead. - if ((!task_id.IsNil() || !actor_id.IsNil()) && !worker->IsDead()) { + if ((!lease_id.IsNil() || !actor_id.IsNil()) && !worker->IsDead()) { // If the worker was an actor, it'll be cleaned by GCS. if (actor_id.IsNil()) { // Return the resources that were being used by this worker. - RayTask task; - local_task_manager_.TaskFinished(worker, &task); + RayLease lease; + local_lease_manager_.CleanupLease(worker, &lease); } if (disconnect_type == rpc::WorkerExitType::SYSTEM_ERROR) { @@ -1370,8 +1373,7 @@ void NodeManager::DisconnectClient(const std::shared_ptr &clie "unexpected system " "error. To troubleshoot the problem, check the logs for the " "dead worker." - << " RayTask ID: " << task_id - << " Worker ID: " << worker->WorkerId() + << " Lease ID: " << lease_id << " Worker ID: " << worker->WorkerId() << " Node ID: " << self_node_id_ << " Worker IP address: " << worker->IpAddress() << " Worker port: " << worker->Port() @@ -1395,10 +1397,10 @@ void NodeManager::DisconnectClient(const std::shared_ptr &clie worker_pool_.DisconnectWorker(worker, disconnect_type); // Return the resources that were being used by this worker. - local_task_manager_.ReleaseWorkerResources(worker); + local_lease_manager_.ReleaseWorkerResources(worker); - // Since some resources may have been released, we can try to dispatch more tasks. - cluster_task_manager_.ScheduleAndDispatchTasks(); + // Since some resources may have been released, we can try to grant more leases. + cluster_lease_manager_.ScheduleAndGrantLeases(); } else if (is_driver) { // The client is a driver. const auto job_id = worker->GetAssignedJobId(); @@ -1419,8 +1421,8 @@ void NodeManager::DisconnectClient(const std::shared_ptr &clie } } - local_task_manager_.ClearWorkerBacklog(worker->WorkerId()); - cluster_task_manager_.CancelAllTasksOwnedBy(worker->WorkerId()); + local_lease_manager_.ClearWorkerBacklog(worker->WorkerId()); + cluster_lease_manager_.CancelAllLeasesOwnedBy(worker->WorkerId()); if (graceful) { // Graceful disconnects are initiated by a request from the worker and @@ -1477,7 +1479,7 @@ void NodeManager::ProcessWaitRequestMessage( bool all_objects_local = true; for (auto const &object_id : object_ids) { - if (!dependency_manager_.CheckObjectLocal(object_id)) { + if (!lease_dependency_manager_.CheckObjectLocal(object_id)) { all_objects_local = false; } } @@ -1592,13 +1594,13 @@ void NodeManager::HandleGetResourceLoad(rpc::GetResourceLoadRequest request, auto resources_data = reply->mutable_resources(); resources_data->set_node_id(self_node_id_.Binary()); resources_data->set_node_manager_address(initial_config_.node_manager_address); - cluster_task_manager_.FillResourceUsage(*resources_data); + cluster_lease_manager_.FillResourceUsage(*resources_data); send_reply_callback(Status::OK(), nullptr, nullptr); } -void NodeManager::HandleCancelTasksWithResourceShapes( - rpc::CancelTasksWithResourceShapesRequest request, - rpc::CancelTasksWithResourceShapesReply *reply, +void NodeManager::HandleCancelLeasesWithResourceShapes( + rpc::CancelLeasesWithResourceShapesRequest request, + rpc::CancelLeasesWithResourceShapesReply *reply, rpc::SendReplyCallback send_reply_callback) { const auto &resource_shapes = request.resource_shapes(); std::vector target_resource_shapes; @@ -1607,7 +1609,7 @@ void NodeManager::HandleCancelTasksWithResourceShapes( ResourceSet(MapFromProtobuf(resource_shape.resource_shape()))); } - cluster_task_manager_.CancelTasksWithResourceShapes(target_resource_shapes); + cluster_lease_manager_.CancelLeasesWithResourceShapes(target_resource_shapes); send_reply_callback(Status::OK(), nullptr, nullptr); } @@ -1615,14 +1617,15 @@ void NodeManager::HandleReportWorkerBacklog(rpc::ReportWorkerBacklogRequest requ rpc::ReportWorkerBacklogReply *reply, rpc::SendReplyCallback send_reply_callback) { HandleReportWorkerBacklog( - request, reply, send_reply_callback, worker_pool_, local_task_manager_); + request, reply, send_reply_callback, worker_pool_, local_lease_manager_); } -void NodeManager::HandleReportWorkerBacklog(rpc::ReportWorkerBacklogRequest request, - rpc::ReportWorkerBacklogReply *reply, - rpc::SendReplyCallback send_reply_callback, - WorkerPoolInterface &worker_pool, - ILocalTaskManager &local_task_manager) { +void NodeManager::HandleReportWorkerBacklog( + rpc::ReportWorkerBacklogRequest request, + rpc::ReportWorkerBacklogReply *reply, + rpc::SendReplyCallback send_reply_callback, + WorkerPoolInterface &worker_pool, + LocalLeaseManagerInterface &local_lease_manager) { const WorkerID worker_id = WorkerID::FromBinary(request.worker_id()); if (worker_pool.GetRegisteredWorker(worker_id) == nullptr && worker_pool.GetRegisteredDriver(worker_id) == nullptr) { @@ -1631,13 +1634,13 @@ void NodeManager::HandleReportWorkerBacklog(rpc::ReportWorkerBacklogRequest requ return; } - local_task_manager.ClearWorkerBacklog(worker_id); + local_lease_manager.ClearWorkerBacklog(worker_id); std::unordered_set seen; for (const auto &backlog_report : request.backlog_reports()) { - const TaskSpecification resource_spec(backlog_report.resource_spec()); - const SchedulingClass scheduling_class = resource_spec.GetSchedulingClass(); + const LeaseSpecification lease_spec(backlog_report.lease_spec()); + const SchedulingClass scheduling_class = lease_spec.GetSchedulingClass(); RAY_CHECK(seen.find(scheduling_class) == seen.end()); - local_task_manager.SetWorkerBacklog( + local_lease_manager.SetWorkerBacklog( scheduling_class, worker_id, backlog_report.backlog_size()); } send_reply_callback(Status::OK(), nullptr, nullptr); @@ -1646,13 +1649,12 @@ void NodeManager::HandleReportWorkerBacklog(rpc::ReportWorkerBacklogRequest requ void NodeManager::HandleRequestWorkerLease(rpc::RequestWorkerLeaseRequest request, rpc::RequestWorkerLeaseReply *reply, rpc::SendReplyCallback send_reply_callback) { - RayTask task{std::move(*request.mutable_resource_spec())}; - + RayLease lease{std::move(*request.mutable_lease_spec())}; const auto caller_worker = - WorkerID::FromBinary(task.GetTaskSpecification().CallerAddress().worker_id()); + WorkerID::FromBinary(lease.GetLeaseSpecification().CallerAddress().worker_id()); const auto caller_node = - NodeID::FromBinary(task.GetTaskSpecification().CallerAddress().node_id()); - if (!task.GetTaskSpecification().IsDetachedActor() && + NodeID::FromBinary(lease.GetLeaseSpecification().CallerAddress().node_id()); + if (!lease.GetLeaseSpecification().IsDetachedActor() && (failed_workers_cache_.contains(caller_worker) || failed_nodes_cache_.contains(caller_node))) { RAY_LOG(INFO).WithField(caller_worker).WithField(caller_node) @@ -1665,16 +1667,16 @@ void NodeManager::HandleRequestWorkerLease(rpc::RequestWorkerLeaseRequest reques return; }; - const bool is_actor_creation_task = task.GetTaskSpecification().IsActorCreationTask(); + const bool is_actor_creation_task = lease.GetLeaseSpecification().IsActorCreationTask(); ActorID actor_id = ActorID::Nil(); metrics_num_task_scheduled_ += 1; if (is_actor_creation_task) { - actor_id = task.GetTaskSpecification().ActorCreationId(); + actor_id = lease.GetLeaseSpecification().ActorId(); } - const auto &task_spec = task.GetTaskSpecification(); - worker_pool_.PrestartWorkers(task_spec, request.backlog_size()); + const auto &lease_spec = lease.GetLeaseSpecification(); + worker_pool_.PrestartWorkers(lease_spec, request.backlog_size()); auto send_reply_callback_wrapper = [this, is_actor_creation_task, actor_id, reply, send_reply_callback]( @@ -1687,7 +1689,7 @@ void NodeManager::HandleRequestWorkerLease(rpc::RequestWorkerLeaseRequest reques // with normal task resource usages so GCS can fast update // its resource view of this raylet. if (RayConfig::instance().gcs_actor_scheduling_enabled()) { - auto normal_task_resources = local_task_manager_.CalcNormalTaskResources(); + auto normal_task_resources = local_lease_manager_.CalcNormalTaskResources(); RAY_LOG(DEBUG).WithField(actor_id) << "Reject leasing as the raylet has no enough resources. " "normal_task_resources = " @@ -1706,11 +1708,11 @@ void NodeManager::HandleRequestWorkerLease(rpc::RequestWorkerLeaseRequest reques send_reply_callback(status, success, failure); }; - cluster_task_manager_.QueueAndScheduleTask(std::move(task), - request.grant_or_reject(), - request.is_selected_based_on_locality(), - reply, - std::move(send_reply_callback_wrapper)); + cluster_lease_manager_.QueueAndScheduleLease(std::move(lease), + request.grant_or_reject(), + request.is_selected_based_on_locality(), + reply, + std::move(send_reply_callback_wrapper)); } void NodeManager::HandlePrestartWorkers(rpc::PrestartWorkersRequest request, @@ -1776,7 +1778,7 @@ void NodeManager::HandleCommitBundleResources( placement_group_resource_manager_->CommitBundles(bundle_specs); send_reply_callback(Status::OK(), nullptr, nullptr); - cluster_task_manager_.ScheduleAndDispatchTasks(); + cluster_lease_manager_.ScheduleAndGrantLeases(); } void NodeManager::HandleCancelResourceReserve( @@ -1788,13 +1790,13 @@ void NodeManager::HandleCancelResourceReserve( << bundle_spec.DebugString(); // The PG bundle resource must be committed before a lease request asking for it - // can be added to local_task_manager and the only reason why we cancel + // can be added to local_lease_manager and the only reason why we cancel // a committed bundle is when the placement group is removed. // In the case of placement group removal, we should cancel all the lease requests. - local_task_manager_.CancelTasks( + local_lease_manager_.CancelLeases( [&](const std::shared_ptr &work) { const auto bundle_id = - work->task_.GetTaskSpecification().PlacementGroupBundleId(); + work->lease_.GetLeaseSpecification().PlacementGroupBundleId(); return bundle_id.first == bundle_spec.PlacementGroupId(); }, rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_PLACEMENT_GROUP_REMOVED, @@ -1819,7 +1821,7 @@ void NodeManager::HandleCancelResourceReserve( << "Destroying worker since its placement group was removed. Placement group id: " << worker->GetBundleId().first << ", bundle index: " << bundle_spec.BundleId().second - << ", task id: " << worker->GetAssignedTaskId() + << ", lease id: " << worker->GetGrantedLeaseId() << ", actor id: " << worker->GetActorId() << ", worker id: " << worker->WorkerId(); const auto &message = stream.str(); @@ -1828,7 +1830,7 @@ void NodeManager::HandleCancelResourceReserve( } RAY_CHECK_OK(placement_group_resource_manager_->ReturnBundle(bundle_spec)); - cluster_task_manager_.ScheduleAndDispatchTasks(); + cluster_lease_manager_.ScheduleAndGrantLeases(); send_reply_callback(Status::OK(), nullptr, nullptr); } @@ -1919,7 +1921,7 @@ void NodeManager::HandleResizeLocalResourceInstances( << debug_string(updated_total_map); RAY_LOG(INFO) << "Available resources: " << debug_string(updated_available_map); // Trigger scheduling to account for the new resources - cluster_task_manager_.ScheduleAndDispatchTasks(); + cluster_lease_manager_.ScheduleAndGrantLeases(); } // Populate the reply with the current resource state @@ -1929,16 +1931,15 @@ void NodeManager::HandleResizeLocalResourceInstances( send_reply_callback(Status::OK(), nullptr, nullptr); } -void NodeManager::HandleReturnWorker(rpc::ReturnWorkerRequest request, - rpc::ReturnWorkerReply *reply, - rpc::SendReplyCallback send_reply_callback) { +void NodeManager::HandleReturnWorkerLease(rpc::ReturnWorkerLeaseRequest request, + rpc::ReturnWorkerLeaseReply *reply, + rpc::SendReplyCallback send_reply_callback) { // Read the resource spec submitted by the client. auto worker_id = WorkerID::FromBinary(request.worker_id()); std::shared_ptr worker = leased_workers_[worker_id]; Status status; ReleaseWorker(worker_id); - if (worker) { if (request.disconnect_worker()) { // The worker should be destroyed. @@ -1955,7 +1956,7 @@ void NodeManager::HandleReturnWorker(rpc::ReturnWorkerRequest request, // unblock RPC by unblocking it immediately (unblock is idempotent). HandleDirectCallTaskUnblocked(worker); } - local_task_manager_.ReleaseWorkerResources(worker); + local_lease_manager_.ReleaseWorkerResources(worker); // If the worker is exiting, don't add it to our pool. The worker will cleanup // and terminate itself. if (!request.worker_exiting()) { @@ -2048,7 +2049,8 @@ void NodeManager::HandleReleaseUnusedActorWorkers( std::vector> unused_actor_workers; for (auto &iter : leased_workers_) { // We only kill *actor* workers. - if (!iter.second->GetActorId().IsNil() && !in_use_worker_ids.count(iter.first)) { + if (!iter.second->GetActorId().IsNil() && + !in_use_worker_ids.count(iter.second->WorkerId())) { unused_actor_workers.push_back(iter.second); } } @@ -2067,8 +2069,8 @@ void NodeManager::HandleReleaseUnusedActorWorkers( void NodeManager::HandleCancelWorkerLease(rpc::CancelWorkerLeaseRequest request, rpc::CancelWorkerLeaseReply *reply, rpc::SendReplyCallback send_reply_callback) { - const TaskID task_id = TaskID::FromBinary(request.task_id()); - bool canceled = cluster_task_manager_.CancelTask(task_id); + const LeaseID lease_id = LeaseID::FromBinary(request.lease_id()); + bool canceled = cluster_lease_manager_.CancelLease(lease_id); // The task cancellation failed if we did not have the task queued, since // this means that we may not have received the task request yet. It is // successful if we did have the task queued, since we have now replied to @@ -2122,27 +2124,27 @@ void NodeManager::MarkObjectsAsFailed( void NodeManager::HandleDirectCallTaskBlocked( const std::shared_ptr &worker) { - if (!worker || worker->IsBlocked() || worker->GetAssignedTaskId().IsNil()) { + if (!worker || worker->IsBlocked() || worker->GetGrantedLeaseId().IsNil()) { return; // The worker may have died or is no longer processing the task. } - local_task_manager_.ReleaseCpuResourcesFromBlockedWorker(worker); - cluster_task_manager_.ScheduleAndDispatchTasks(); + local_lease_manager_.ReleaseCpuResourcesFromBlockedWorker(worker); + cluster_lease_manager_.ScheduleAndGrantLeases(); } void NodeManager::HandleDirectCallTaskUnblocked( const std::shared_ptr &worker) { - if (!worker || worker->GetAssignedTaskId().IsNil()) { + if (!worker || worker->GetGrantedLeaseId().IsNil()) { return; // The worker may have died or is no longer processing the task. } // First, always release task dependencies. This ensures we don't leak resources even // if we don't need to unblock the worker below. - dependency_manager_.CancelGetRequest(worker->WorkerId()); + lease_dependency_manager_.CancelGetRequest(worker->WorkerId()); if (worker->IsBlocked()) { - local_task_manager_.ReturnCpuResourcesToUnblockedWorker(worker); - cluster_task_manager_.ScheduleAndDispatchTasks(); + local_lease_manager_.ReturnCpuResourcesToUnblockedWorker(worker); + cluster_lease_manager_.ScheduleAndGrantLeases(); } } @@ -2158,9 +2160,9 @@ void NodeManager::AsyncGetOrWait(const std::shared_ptr &client // Start an async request to get or wait for the objects. // The objects will be fetched locally unless the get or wait request is canceled. if (is_get_request) { - dependency_manager_.StartOrUpdateGetRequest(worker->WorkerId(), object_refs); + lease_dependency_manager_.StartOrUpdateGetRequest(worker->WorkerId(), object_refs); } else { - dependency_manager_.StartOrUpdateWaitRequest(worker->WorkerId(), object_refs); + lease_dependency_manager_.StartOrUpdateWaitRequest(worker->WorkerId(), object_refs); } } @@ -2171,52 +2173,48 @@ void NodeManager::CancelGetRequest(const std::shared_ptr &clie } RAY_CHECK(worker); - dependency_manager_.CancelGetRequest(worker->WorkerId()); + lease_dependency_manager_.CancelGetRequest(worker->WorkerId()); } -bool NodeManager::FinishAssignedTask(const std::shared_ptr &worker) { - TaskID task_id = worker->GetAssignedTaskId(); - RAY_LOG(DEBUG).WithField(task_id) << "Finished task "; +bool NodeManager::CleanupLease(const std::shared_ptr &worker) { + LeaseID lease_id = worker->GetGrantedLeaseId(); + RAY_LOG(DEBUG).WithField(lease_id) << "Cleaning up lease "; - RayTask task; - local_task_manager_.TaskFinished(worker, &task); + RayLease lease; + local_lease_manager_.CleanupLease(worker, &lease); - const auto &spec = task.GetTaskSpecification(); // - if ((spec.IsActorCreationTask())) { - // If this was an actor or actor creation task, handle the actor's new - // state. - FinishAssignedActorCreationTask(worker, task); + const auto &lease_spec = lease.GetLeaseSpecification(); + if ((lease_spec.IsActorCreationTask())) { + // If this was an actor or actor creation task, convert the worker to an actor. + ConvertWorkerToActor(worker, lease); } else { - // If this was a non-actor task, then cancel any ray.wait calls that were - // made during the task execution. - dependency_manager_.CancelWaitRequest(worker->WorkerId()); + // If this was a non-actor lease, cancel any ray.wait calls that were + // made during the lease execution. + lease_dependency_manager_.CancelWaitRequest(worker->WorkerId()); } - // Notify the task dependency manager that this task has finished execution. - dependency_manager_.CancelGetRequest(worker->WorkerId()); + // Notify the lease dependency manager that this lease has returned. + lease_dependency_manager_.CancelGetRequest(worker->WorkerId()); - if (!spec.IsActorCreationTask()) { - // Unset the worker's assigned task. We keep the assigned task ID for - // actor creation calls because this ID is used later if the actor - // requires objects from plasma. - worker->AssignTaskId(TaskID::Nil()); + if (!lease_spec.IsActorCreationTask()) { + worker->GrantLeaseId(LeaseID::Nil()); worker->SetOwnerAddress(rpc::Address()); } // Actors will be assigned tasks via the core worker and therefore are not idle. - return !spec.IsActorCreationTask(); + return !lease_spec.IsActorCreationTask(); } -void NodeManager::FinishAssignedActorCreationTask( - const std::shared_ptr &worker, const RayTask &task) { - RAY_LOG(DEBUG) << "Finishing assigned actor creation task"; - const TaskSpecification task_spec = task.GetTaskSpecification(); - ActorID actor_id = task_spec.ActorCreationId(); +void NodeManager::ConvertWorkerToActor(const std::shared_ptr &worker, + const RayLease &lease) { + RAY_LOG(DEBUG) << "Converting worker to actor"; + const LeaseSpecification lease_spec = lease.GetLeaseSpecification(); + ActorID actor_id = lease_spec.ActorId(); // This was an actor creation task. Convert the worker to an actor. worker->AssignActorId(actor_id); - if (task_spec.IsDetachedActor()) { - auto job_id = task.GetTaskSpecification().JobId(); + if (lease_spec.IsDetachedActor()) { + auto job_id = lease_spec.JobId(); auto job_config = worker_pool_.GetJobConfig(job_id); RAY_CHECK(job_config); } @@ -2242,10 +2240,10 @@ void NodeManager::SpillIfOverPrimaryObjectsThreshold() { void NodeManager::HandleObjectLocal(const ObjectInfo &object_info) { const ObjectID &object_id = object_info.object_id; // Notify the task dependency manager that this object is local. - const auto ready_task_ids = dependency_manager_.HandleObjectLocal(object_id); + const auto ready_lease_ids = lease_dependency_manager_.HandleObjectLocal(object_id); RAY_LOG(DEBUG).WithField(object_id).WithField(self_node_id_) - << "Object local on node, " << ready_task_ids.size() << " tasks ready"; - local_task_manager_.TasksUnblocked(ready_task_ids); + << "Object local on node, " << ready_lease_ids.size() << " tasks ready"; + local_lease_manager_.LeasesUnblocked(ready_lease_ids); // Notify the wait manager that this object is local. wait_manager_.HandleObjectLocal(object_id); @@ -2276,27 +2274,17 @@ void NodeManager::HandleObjectLocal(const ObjectInfo &object_info) { SpillIfOverPrimaryObjectsThreshold(); } -bool NodeManager::IsActorCreationTask(const TaskID &task_id) { - auto actor_id = task_id.ActorId(); - if (!actor_id.IsNil() && task_id == TaskID::ForActorCreationTask(actor_id)) { - // This task ID corresponds to an actor creation task. - return true; - } - - return false; -} - void NodeManager::HandleObjectMissing(const ObjectID &object_id) { - // Notify the task dependency manager that this object is no longer local. - const auto waiting_task_ids = dependency_manager_.HandleObjectMissing(object_id); + // Notify the lease dependency manager that this object is no longer local. + const auto waiting_lease_ids = lease_dependency_manager_.HandleObjectMissing(object_id); std::stringstream result; result << "Object missing " << object_id << ", " - << " on " << self_node_id_ << ", " << waiting_task_ids.size() - << " tasks waiting"; - if (waiting_task_ids.size() > 0) { - result << ", tasks: "; - for (const auto &task_id : waiting_task_ids) { - result << task_id << " "; + << " on " << self_node_id_ << ", " << waiting_lease_ids.size() + << " leases waiting"; + if (waiting_lease_ids.size() > 0) { + result << ", leases: "; + for (const auto &lease_id : waiting_lease_ids) { + result << lease_id << " "; } } RAY_LOG(DEBUG) << result.str(); @@ -2315,7 +2303,7 @@ void NodeManager::ProcessSubscribePlasmaReady( auto message = flatbuffers::GetRoot(message_data); auto id = from_flatbuf(*message->object_id()); - if (dependency_manager_.CheckObjectLocal(id)) { + if (lease_dependency_manager_.CheckObjectLocal(id)) { // Object is already local, so we directly fire the callback to tell the core worker // that the plasma object is ready. rpc::PlasmaObjectReadyRequest request; @@ -2342,7 +2330,8 @@ void NodeManager::ProcessSubscribePlasmaReady( // is local at this time but when the core worker was notified, the object is // is evicted. The core worker should be able to handle evicted object in this // case. - dependency_manager_.StartOrUpdateWaitRequest(associated_worker->WorkerId(), refs); + lease_dependency_manager_.StartOrUpdateWaitRequest(associated_worker->WorkerId(), + refs); // Add this worker to the listeners for the object ID. { @@ -2369,14 +2358,14 @@ std::string NodeManager::DebugString() const { result << "\nNode ID: " << self_node_id_; result << "\nNode name: " << self_node_name_; result << "\nInitialConfigResources: " << initial_config_.resource_config.DebugString(); - result << "\nClusterTaskManager:\n"; - result << cluster_task_manager_.DebugStr(); + result << "\nClusterLeaseManager:\n"; + result << cluster_lease_manager_.DebugStr(); result << "\nClusterResources:"; result << "\n" << local_object_manager_.DebugString(); result << "\n" << object_manager_.DebugString(); result << "\n" << gcs_client_.DebugString(); result << "\n" << worker_pool_.DebugString(); - result << "\n" << dependency_manager_.DebugString(); + result << "\n" << lease_dependency_manager_.DebugString(); result << "\n" << wait_manager_.DebugString(); result << "\n" << core_worker_subscriber_.DebugString(); { @@ -2748,7 +2737,7 @@ void NodeManager::RecordMetrics() { return; } - cluster_task_manager_.RecordMetrics(); + cluster_lease_manager_.RecordMetrics(); object_manager_.RecordMetrics(); local_object_manager_.RecordMetrics(); @@ -2756,7 +2745,7 @@ void NodeManager::RecordMetrics() { uint64_t duration_ms = current_time - last_metrics_recorded_at_ms_; last_metrics_recorded_at_ms_ = current_time; object_directory_.RecordMetrics(duration_ms); - dependency_manager_.RecordMetrics(); + lease_dependency_manager_.RecordMetrics(); } void NodeManager::ConsumeSyncMessage( @@ -2777,7 +2766,7 @@ void NodeManager::ConsumeSyncMessage( const bool capacity_updated = ResourceCreateUpdated(node_id, resources); const bool usage_update = UpdateResourceUsage(node_id, resource_view_sync_message); if (capacity_updated || usage_update) { - cluster_task_manager_.ScheduleAndDispatchTasks(); + cluster_lease_manager_.ScheduleAndGrantLeases(); } } else if (message->message_type() == syncer::MessageType::COMMANDS) { syncer::CommandsSyncMessage commands_sync_message; @@ -2821,7 +2810,7 @@ MemoryUsageRefreshCallback NodeManager::CreateMemoryUsageRefreshCallback() { if (!high_memory_eviction_target_->GetProcess().IsAlive()) { RAY_LOG(INFO) .WithField(high_memory_eviction_target_->WorkerId()) - .WithField(high_memory_eviction_target_->GetAssignedTaskId()) + .WithField(high_memory_eviction_target_->GetGrantedLeaseId()) << "Worker evicted and process killed to reclaim memory. " << "worker pid: " << high_memory_eviction_target_->GetProcess().GetId(); high_memory_eviction_target_ = nullptr; @@ -2830,7 +2819,7 @@ MemoryUsageRefreshCallback NodeManager::CreateMemoryUsageRefreshCallback() { if (is_usage_above_threshold) { if (high_memory_eviction_target_ != nullptr) { RAY_LOG_EVERY_MS(INFO, 1000) - .WithField(high_memory_eviction_target_->GetAssignedTaskId()) + .WithField(high_memory_eviction_target_->GetGrantedLeaseId()) .WithField(high_memory_eviction_target_->WorkerId()) << "Memory usage above threshold. " << "Still waiting for worker eviction to free up memory. " @@ -2863,7 +2852,7 @@ MemoryUsageRefreshCallback NodeManager::CreateMemoryUsageRefreshCallback() { RAY_LOG(INFO) << "Killing worker with task " - << worker_to_kill->GetAssignedTask().GetTaskSpecification().DebugString() + << worker_to_kill->GetGrantedLease().GetLeaseSpecification().DebugString() << "\n\n" << oom_kill_details << "\n\n" << oom_kill_suggestions; @@ -2878,13 +2867,13 @@ MemoryUsageRefreshCallback NodeManager::CreateMemoryUsageRefreshCallback() { // Rerpot the event to the dashboard. RAY_EVENT_EVERY_MS(ERROR, "Out of Memory", 10 * 1000) << worker_exit_message; - // Mark the task as failure and raise an exception from a caller. - rpc::RayErrorInfo task_failure_reason; - task_failure_reason.set_error_message(worker_exit_message); - task_failure_reason.set_error_type(rpc::ErrorType::OUT_OF_MEMORY); - SetTaskFailureReason(worker_to_kill->GetAssignedTaskId(), - std::move(task_failure_reason), - should_retry); + // Mark the worker as failure and raise an exception from a caller. + rpc::RayErrorInfo worker_failure_reason; + worker_failure_reason.set_error_message(worker_exit_message); + worker_failure_reason.set_error_type(rpc::ErrorType::OUT_OF_MEMORY); + SetWorkerFailureReason(worker_to_kill->GetGrantedLeaseId(), + std::move(worker_failure_reason), + should_retry); /// since we print the process memory in the message. Destroy should be called /// as soon as possible to free up memory. @@ -2898,17 +2887,17 @@ MemoryUsageRefreshCallback NodeManager::CreateMemoryUsageRefreshCallback() { ray::stats::STATS_memory_manager_worker_eviction_total.Record( 1, {{"Type", "MemoryManager.DriverEviction.Total"}, {"Name", ""}}); } else if (worker_to_kill->GetActorId().IsNil()) { - const auto &ray_task = worker_to_kill->GetAssignedTask(); + const auto &ray_lease = worker_to_kill->GetGrantedLease(); ray::stats::STATS_memory_manager_worker_eviction_total.Record( 1, {{"Type", "MemoryManager.TaskEviction.Total"}, - {"Name", ray_task.GetTaskSpecification().GetName()}}); + {"Name", ray_lease.GetLeaseSpecification().GetTaskName()}}); } else { - const auto &ray_task = worker_to_kill->GetAssignedTask(); + const auto &ray_lease = worker_to_kill->GetGrantedLease(); ray::stats::STATS_memory_manager_worker_eviction_total.Record( 1, {{"Type", "MemoryManager.ActorEviction.Total"}, - {"Name", ray_task.GetTaskSpecification().GetName()}}); + {"Name", ray_lease.GetLeaseSpecification().GetTaskName()}}); } } } @@ -2944,8 +2933,8 @@ const std::string NodeManager::CreateOomKillMessageDetails( oom_kill_details_ss << "Memory on the node (IP: " << worker->IpAddress() << ", ID: " << node_id - << ") where the task (" << worker->GetTaskOrActorIdAsDebugString() - << ", name=" << worker->GetAssignedTask().GetTaskSpecification().GetName() + << ") where the lease (" << worker->GetLeaseIdAsDebugString() + << ", name=" << worker->GetGrantedLease().GetLeaseSpecification().GetTaskName() << ", pid=" << worker->GetProcess().GetId() << ", memory used=" << process_used_bytes_gb << "GB) was running was " << used_bytes_gb << "GB / " << total_bytes_gb << "GB (" << usage_fraction @@ -2964,9 +2953,9 @@ const std::string NodeManager::CreateOomKillMessageDetails( const std::string NodeManager::CreateOomKillMessageSuggestions( const std::shared_ptr &worker, bool should_retry) const { std::stringstream not_retriable_recommendation_ss; - if (worker && !worker->GetAssignedTask().GetTaskSpecification().IsRetriable()) { + if (worker && !worker->GetGrantedLease().GetLeaseSpecification().IsRetriable()) { not_retriable_recommendation_ss << "Set "; - if (worker->GetAssignedTask().GetTaskSpecification().IsNormalTask()) { + if (worker->GetGrantedLease().GetLeaseSpecification().IsNormalTask()) { not_retriable_recommendation_ss << "max_retries"; } else { not_retriable_recommendation_ss << "max_restarts and max_task_retries"; @@ -2994,29 +2983,29 @@ const std::string NodeManager::CreateOomKillMessageSuggestions( return oom_kill_suggestions_ss.str(); } -void NodeManager::SetTaskFailureReason(const TaskID &task_id, - const rpc::RayErrorInfo &failure_reason, - bool should_retry) { - RAY_LOG(DEBUG).WithField(task_id) << "set failure reason for task "; +void NodeManager::SetWorkerFailureReason(const LeaseID &lease_id, + const rpc::RayErrorInfo &failure_reason, + bool should_retry) { + RAY_LOG(DEBUG).WithField(lease_id) << "set failure reason for lease "; ray::TaskFailureEntry entry(failure_reason, should_retry); - auto result = task_failure_reasons_.emplace(task_id, std::move(entry)); + auto result = worker_failure_reasons_.emplace(lease_id, std::move(entry)); if (!result.second) { - RAY_LOG(WARNING).WithField(task_id) + RAY_LOG(WARNING).WithField(lease_id) << "Trying to insert failure reason more than once for the same " - "task, the previous failure will be removed."; + "worker, the previous failure will be removed."; } } -void NodeManager::GCTaskFailureReason() { - for (const auto &entry : task_failure_reasons_) { +void NodeManager::GCWorkerFailureReason() { + for (const auto &entry : worker_failure_reasons_) { auto duration = static_cast( std::chrono::duration_cast( std::chrono::steady_clock::now() - entry.second.creation_time_) .count()); if (duration > RayConfig::instance().task_failure_entry_ttl_ms()) { RAY_LOG(INFO).WithField(entry.first) - << "Removing task failure reason since it expired"; - task_failure_reasons_.erase(entry.first); + << "Removing worker failure reason since it expired"; + worker_failure_reasons_.erase(entry.first); } } } diff --git a/src/ray/raylet/node_manager.h b/src/ray/raylet/node_manager.h index 2727f0d3d4a0..0344677df99e 100644 --- a/src/ray/raylet/node_manager.h +++ b/src/ray/raylet/node_manager.h @@ -24,11 +24,11 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/bundle_spec.h" #include "ray/common/id.h" +#include "ray/common/lease/lease.h" #include "ray/common/memory_monitor.h" #include "ray/common/ray_object.h" #include "ray/common/ray_syncer/ray_syncer.h" #include "ray/common/scheduling/resource_set.h" -#include "ray/common/task/task.h" #include "ray/common/task/task_util.h" #include "ray/core_worker/experimental_mutable_object_provider.h" #include "ray/flatbuffers/node_manager_generated.h" @@ -38,13 +38,13 @@ #include "ray/object_manager/plasma/client.h" #include "ray/pubsub/subscriber.h" #include "ray/raylet/agent_manager.h" -#include "ray/raylet/dependency_manager.h" +#include "ray/raylet/lease_dependency_manager.h" +#include "ray/raylet/local_lease_manager.h" #include "ray/raylet/local_object_manager_interface.h" -#include "ray/raylet/local_task_manager.h" #include "ray/raylet/placement_group_resource_manager.h" #include "ray/raylet/runtime_env_agent_client.h" +#include "ray/raylet/scheduling/cluster_lease_manager_interface.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" -#include "ray/raylet/scheduling/cluster_task_manager_interface.h" #include "ray/raylet/wait_manager.h" #include "ray/raylet/worker_killing_policy.h" #include "ray/raylet/worker_pool.h" @@ -143,12 +143,12 @@ class NodeManager : public rpc::NodeManagerServiceHandler, rpc::RayletClientPool &raylet_client_pool, pubsub::SubscriberInterface &core_worker_subscriber, ClusterResourceScheduler &cluster_resource_scheduler, - ILocalTaskManager &local_task_manager, - ClusterTaskManagerInterface &cluster_task_manager, + LocalLeaseManagerInterface &local_lease_manager, + ClusterLeaseManagerInterface &cluster_lease_manager, IObjectDirectory &object_directory, ObjectManagerInterface &object_manager, LocalObjectManagerInterface &local_object_manager, - DependencyManager &dependency_manager, + LeaseDependencyManager &lease_dependency_manager, WorkerPoolInterface &worker_pool, absl::flat_hash_map> &leased_workers, plasma::PlasmaClientInterface &store_client, @@ -359,18 +359,18 @@ class NodeManager : public rpc::NodeManagerServiceHandler, const NodeID &id, const syncer::ResourceViewSyncMessage &resource_view_sync_message); - /// Handle a worker finishing its assigned task. + /// Cleanup any lease resources and state for a worker that was granted a lease. /// - /// \param worker The worker that finished the task. + /// \param worker The worker that was granted the lease. /// \return Whether the worker should be returned to the idle pool. This is /// only false for actor creation calls, which should never be returned to idle. - bool FinishAssignedTask(const std::shared_ptr &worker); + bool CleanupLease(const std::shared_ptr &worker); - /// Handle a worker finishing an assigned actor creation task. - /// \param worker The worker that finished the task. - /// \param task The actor task or actor creation task. - void FinishAssignedActorCreationTask(const std::shared_ptr &worker, - const RayTask &task); + /// Convert a worker to an actor since it's finished an actor creation task. + /// \param worker The worker that was granted the actor creation lease. + /// \param lease The lease of the actor creation task. + void ConvertWorkerToActor(const std::shared_ptr &worker, + const RayLease &lease); /// Start a get or wait request for the requested objects. /// @@ -414,12 +414,6 @@ class NodeManager : public rpc::NodeManagerServiceHandler, const std::string &disconnect_detail, bool force = false); - /// When a job finished, loop over all of the queued tasks for that job and - /// treat them as failed. - /// - /// \param job_id The job that exited. - void CleanUpTasksForFinishedJob(const JobID &job_id); - /// Handles the event that a job is started. /// /// \param job_id ID of the started job. @@ -525,10 +519,10 @@ class NodeManager : public rpc::NodeManagerServiceHandler, rpc::GetResourceLoadReply *reply, rpc::SendReplyCallback send_reply_callback) override; - /// Handle a `CancelTasksWithResourceShapes` request. - void HandleCancelTasksWithResourceShapes( - rpc::CancelTasksWithResourceShapesRequest request, - rpc::CancelTasksWithResourceShapesReply *reply, + /// Handle a `CancelLeasesWithResourceShapes` request. + void HandleCancelLeasesWithResourceShapes( + rpc::CancelLeasesWithResourceShapesRequest request, + rpc::CancelLeasesWithResourceShapesReply *reply, rpc::SendReplyCallback send_reply_callback) override; /// Handle a `PrepareBundleResources` request. @@ -561,12 +555,12 @@ class NodeManager : public rpc::NodeManagerServiceHandler, rpc::ReportWorkerBacklogReply *reply, rpc::SendReplyCallback send_reply_callback, WorkerPoolInterface &worker_pool, - ILocalTaskManager &local_task_manager); + LocalLeaseManagerInterface &local_lease_manager); - /// Handle a `ReturnWorker` request. - void HandleReturnWorker(rpc::ReturnWorkerRequest request, - rpc::ReturnWorkerReply *reply, - rpc::SendReplyCallback send_reply_callback) override; + /// Handle a `ReturnWorkerLease` request. + void HandleReturnWorkerLease(rpc::ReturnWorkerLeaseRequest request, + rpc::ReturnWorkerLeaseReply *reply, + rpc::SendReplyCallback send_reply_callback) override; /// Handle a `ReleaseUnusedActorWorkers` request. // On GCS restart, there's a pruning effort. GCS sends raylet a list of actor workers it @@ -621,10 +615,10 @@ class NodeManager : public rpc::NodeManagerServiceHandler, rpc::GetSystemConfigReply *reply, rpc::SendReplyCallback send_reply_callback) override; - /// Handle a `GetTaskFailureCause` request. - void HandleGetTaskFailureCause(rpc::GetTaskFailureCauseRequest request, - rpc::GetTaskFailureCauseReply *reply, - rpc::SendReplyCallback send_reply_callback) override; + /// Handle a `GetWorkerFailureCause` request. + void HandleGetWorkerFailureCause(rpc::GetWorkerFailureCauseRequest request, + rpc::GetWorkerFailureCauseReply *reply, + rpc::SendReplyCallback send_reply_callback) override; void HandleRegisterMutableObject(rpc::RegisterMutableObjectRequest request, rpc::RegisterMutableObjectReply *reply, @@ -714,12 +708,12 @@ class NodeManager : public rpc::NodeManagerServiceHandler, /// Stores the failure reason for the task. The entry will be cleaned up by a periodic /// function post TTL. - void SetTaskFailureReason(const TaskID &task_id, - const rpc::RayErrorInfo &failure_reason, - bool should_retry); + void SetWorkerFailureReason(const LeaseID &lease_id, + const rpc::RayErrorInfo &failure_reason, + bool should_retry); - /// Checks the expiry time of the task failures and garbage collect them. - void GCTaskFailureReason(); + /// Checks the expiry time of the worker failures and garbage collect them. + void GCWorkerFailureReason(); /// Creates a AgentManager that creates and manages a dashboard agent. std::unique_ptr CreateDashboardAgentManager( @@ -776,7 +770,7 @@ class NodeManager : public rpc::NodeManagerServiceHandler, /// A manager to resolve objects needed by queued tasks and workers that /// called `ray.get` or `ray.wait`. - DependencyManager &dependency_manager_; + LeaseDependencyManager &lease_dependency_manager_; /// A manager for wait requests. WaitManager wait_manager_; @@ -801,11 +795,11 @@ class NodeManager : public rpc::NodeManagerServiceHandler, absl::flat_hash_map> remote_node_manager_addresses_; - /// Map of workers leased out to clients. + /// Map of workers to their worker ids. absl::flat_hash_map> &leased_workers_; - /// Optional extra information about why the task failed. - absl::flat_hash_map task_failure_reasons_; + /// Optional extra information about why the worker failed. + absl::flat_hash_map worker_failure_reasons_; /// Whether to trigger global GC in the next resource usage report. This will broadcast /// a global GC message to all raylets except for this one. @@ -835,11 +829,11 @@ class NodeManager : public rpc::NodeManagerServiceHandler, /// These classes make up the new scheduler. ClusterResourceScheduler is /// responsible for maintaining a view of the cluster state w.r.t resource - /// usage. ClusterTaskManager is responsible for queuing, spilling back, and + /// usage. ClusterLeaseManager is responsible for queuing, spilling back, and /// dispatching tasks. ClusterResourceScheduler &cluster_resource_scheduler_; - ILocalTaskManager &local_task_manager_; - ClusterTaskManagerInterface &cluster_task_manager_; + LocalLeaseManagerInterface &local_lease_manager_; + ClusterLeaseManagerInterface &cluster_lease_manager_; absl::flat_hash_map> pinned_objects_; diff --git a/src/ray/raylet/scheduling/BUILD.bazel b/src/ray/raylet/scheduling/BUILD.bazel index b8616a7a4882..ee89aba03ab6 100644 --- a/src/ray/raylet/scheduling/BUILD.bazel +++ b/src/ray/raylet/scheduling/BUILD.bazel @@ -16,9 +16,9 @@ ray_cc_library( deps = [ ":affinity_with_bundle_scheduling_policy", ":bundle_scheduling_policy", + ":cluster_lease_manager", ":cluster_resource_manager", ":cluster_resource_scheduler", - ":cluster_task_manager", ":composite_scheduling_policy", ":hybrid_scheduling_policy", ":local_resource_manager", @@ -33,8 +33,8 @@ ray_cc_library( name = "scheduler_internal", hdrs = ["internal.h"], deps = [ + "//src/ray/common:lease", "//src/ray/common:ray_object", - "//src/ray/common:task_common", "//src/ray/protobuf:node_manager_cc_proto", ], ) @@ -46,8 +46,8 @@ ray_cc_library( deps = [ ":local_resource_manager", "//src/ray/common:grpc_util", + "//src/ray/common:lease", "//src/ray/common:ray_config", - "//src/ray/common:task_common", "//src/ray/protobuf:gcs_cc_proto", "//src/ray/util:container_util", "//src/ray/util:logging", @@ -72,24 +72,24 @@ ray_cc_library( ) ray_cc_library( - name = "cluster_task_manager", + name = "cluster_lease_manager", srcs = [ - "cluster_task_manager.cc", + "cluster_lease_manager.cc", "scheduler_stats.cc", ], hdrs = [ - "cluster_task_manager.h", + "cluster_lease_manager.h", "scheduler_stats.h", ], deps = [ + ":cluster_lease_manager_interface", ":cluster_resource_scheduler", - ":cluster_task_manager_interface", - ":local_task_manager_interface", + ":local_lease_manager_interface", ":scheduler_internal", ":scheduler_resource_reporter", + "//src/ray/common:lease", "//src/ray/common:ray_config", "//src/ray/common:ray_object", - "//src/ray/common:task_common", "//src/ray/stats:stats_lib", "//src/ray/util:logging", "@com_google_absl//absl/container:flat_hash_map", @@ -97,8 +97,8 @@ ray_cc_library( ) ray_cc_library( - name = "cluster_task_manager_interface", - hdrs = ["cluster_task_manager_interface.h"], + name = "cluster_lease_manager_interface", + hdrs = ["cluster_lease_manager_interface.h"], deps = [ "//src/ray/protobuf:node_manager_cc_proto", "//src/ray/rpc:server_call", @@ -106,11 +106,11 @@ ray_cc_library( ) ray_cc_library( - name = "local_task_manager_interface", - hdrs = ["local_task_manager_interface.h"], + name = "local_lease_manager_interface", + hdrs = ["local_lease_manager_interface.h"], deps = [ ":scheduler_internal", - "//src/ray/common:task_common", + "//src/ray/common:lease", "@com_google_absl//absl/container:flat_hash_map", ], ) @@ -121,9 +121,9 @@ ray_cc_library( hdrs = ["local_resource_manager.h"], deps = [ "//src/ray/common:grpc_util", + "//src/ray/common:lease", "//src/ray/common:ray_config", "//src/ray/common:ray_syncer", - "//src/ray/common:task_common", "//src/ray/protobuf:gcs_cc_proto", "//src/ray/protobuf:node_manager_cc_proto", "//src/ray/stats:stats_metric", @@ -138,10 +138,10 @@ ray_cc_library( srcs = ["scheduler_resource_reporter.cc"], hdrs = ["scheduler_resource_reporter.h"], deps = [ - ":local_task_manager_interface", + ":local_lease_manager_interface", ":scheduler_internal", + "//src/ray/common:lease", "//src/ray/common:ray_config", - "//src/ray/common:task_common", "@com_google_absl//absl/container:flat_hash_map", ], ) @@ -160,7 +160,7 @@ ray_cc_library( hdrs = ["policy/scheduling_context.h"], deps = [ "//src/ray/common:id", - "//src/ray/common:task_common", + "//src/ray/common:lease", "@com_google_absl//absl/container:flat_hash_map", ], ) @@ -171,7 +171,7 @@ ray_cc_library( hdrs = ["policy/affinity_with_bundle_scheduling_policy.h"], deps = [ ":scheduling_policy", - "//src/ray/common:task_common", + "//src/ray/common:lease", ], ) @@ -184,7 +184,7 @@ ray_cc_library( ":scheduling_context", ":scheduling_policy", ":scorer", - "//src/ray/common:task_common", + "//src/ray/common:lease", ], ) @@ -258,7 +258,7 @@ ray_cc_library( name = "scorer", srcs = ["policy/scorer.cc"], hdrs = ["policy/scorer.h"], - deps = ["//src/ray/common:task_common"], + deps = ["//src/ray/common:lease"], ) ray_cc_library( @@ -266,6 +266,6 @@ ray_cc_library( hdrs = ["policy/scheduling_policy.h"], deps = [ ":scheduling_options", - "//src/ray/common:task_common", + "//src/ray/common:lease", ], ) diff --git a/src/ray/raylet/scheduling/cluster_task_manager.cc b/src/ray/raylet/scheduling/cluster_lease_manager.cc similarity index 62% rename from src/ray/raylet/scheduling/cluster_task_manager.cc rename to src/ray/raylet/scheduling/cluster_lease_manager.cc index 2eaf1edb2f55..f96973da9574 100644 --- a/src/ray/raylet/scheduling/cluster_task_manager.cc +++ b/src/ray/raylet/scheduling/cluster_lease_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/raylet/scheduling/cluster_task_manager.h" +#include "ray/raylet/scheduling/cluster_lease_manager.h" #include @@ -28,34 +28,34 @@ namespace ray { namespace raylet { -ClusterTaskManager::ClusterTaskManager( +ClusterLeaseManager::ClusterLeaseManager( const NodeID &self_node_id, ClusterResourceScheduler &cluster_resource_scheduler, internal::NodeInfoGetter get_node_info, - std::function announce_infeasible_task, - ILocalTaskManager &local_task_manager, + std::function announce_infeasible_lease, + LocalLeaseManagerInterface &local_lease_manager, std::function get_time_ms) : self_node_id_(self_node_id), cluster_resource_scheduler_(cluster_resource_scheduler), get_node_info_(std::move(get_node_info)), - announce_infeasible_task_(std::move(announce_infeasible_task)), - local_task_manager_(local_task_manager), + announce_infeasible_lease_(std::move(announce_infeasible_lease)), + local_lease_manager_(local_lease_manager), scheduler_resource_reporter_( - tasks_to_schedule_, infeasible_tasks_, local_task_manager_), - internal_stats_(*this, local_task_manager_), + leases_to_schedule_, infeasible_leases_, local_lease_manager_), + internal_stats_(*this, local_lease_manager_), get_time_ms_(std::move(get_time_ms)) {} -void ClusterTaskManager::QueueAndScheduleTask( - RayTask task, +void ClusterLeaseManager::QueueAndScheduleLease( + RayLease lease, bool grant_or_reject, bool is_selected_based_on_locality, rpc::RequestWorkerLeaseReply *reply, rpc::SendReplyCallback send_reply_callback) { - RAY_LOG(DEBUG) << "Queuing and scheduling task " - << task.GetTaskSpecification().TaskId(); - const auto scheduling_class = task.GetTaskSpecification().GetSchedulingClass(); + RAY_LOG(DEBUG) << "Queuing and scheduling lease " + << lease.GetLeaseSpecification().LeaseId(); + const auto scheduling_class = lease.GetLeaseSpecification().GetSchedulingClass(); auto work = std::make_shared( - std::move(task), + std::move(lease), grant_or_reject, is_selected_based_on_locality, reply, @@ -64,13 +64,13 @@ void ClusterTaskManager::QueueAndScheduleTask( }); // If the scheduling class is infeasible, just add the work to the infeasible queue // directly. - auto infeasible_tasks_iter = infeasible_tasks_.find(scheduling_class); - if (infeasible_tasks_iter != infeasible_tasks_.end()) { - infeasible_tasks_iter->second.emplace_back(std::move(work)); + auto infeasible_leases_iter = infeasible_leases_.find(scheduling_class); + if (infeasible_leases_iter != infeasible_leases_.end()) { + infeasible_leases_iter->second.emplace_back(std::move(work)); } else { - tasks_to_schedule_[scheduling_class].emplace_back(std::move(work)); + leases_to_schedule_[scheduling_class].emplace_back(std::move(work)); } - ScheduleAndDispatchTasks(); + ScheduleAndGrantLeases(); } namespace { @@ -86,20 +86,20 @@ void ReplyCancelled(const internal::Work &work, } } // namespace -bool ClusterTaskManager::CancelTasks( +bool ClusterLeaseManager::CancelLeases( std::function &)> predicate, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) { - bool tasks_cancelled = false; + bool leases_cancelled = false; ray::erase_if>( - tasks_to_schedule_, [&](const std::shared_ptr &work) { + leases_to_schedule_, [&](const std::shared_ptr &work) { if (predicate(work)) { - RAY_LOG(DEBUG) << "Canceling task " - << work->task_.GetTaskSpecification().TaskId() + RAY_LOG(DEBUG) << "Canceling lease " + << work->lease_.GetLeaseSpecification().LeaseId() << " from schedule queue."; ReplyCancelled(*work, failure_type, scheduling_failure_message); - tasks_cancelled = true; + leases_cancelled = true; return true; } else { return false; @@ -107,28 +107,28 @@ bool ClusterTaskManager::CancelTasks( }); ray::erase_if>( - infeasible_tasks_, [&](const std::shared_ptr &work) { + infeasible_leases_, [&](const std::shared_ptr &work) { if (predicate(work)) { - RAY_LOG(DEBUG) << "Canceling task " - << work->task_.GetTaskSpecification().TaskId() + RAY_LOG(DEBUG) << "Canceling lease " + << work->lease_.GetLeaseSpecification().LeaseId() << " from infeasible queue."; ReplyCancelled(*work, failure_type, scheduling_failure_message); - tasks_cancelled = true; + leases_cancelled = true; return true; } else { return false; } }); - if (local_task_manager_.CancelTasks( + if (local_lease_manager_.CancelLeases( predicate, failure_type, scheduling_failure_message)) { - tasks_cancelled = true; + leases_cancelled = true; } - return tasks_cancelled; + return leases_cancelled; } -bool ClusterTaskManager::CancelTasksWithResourceShapes( +bool ClusterLeaseManager::CancelLeasesWithResourceShapes( const std::vector target_resource_shapes) { auto predicate = [target_resource_shapes, this](const std::shared_ptr &work) { @@ -140,7 +140,7 @@ bool ClusterTaskManager::CancelTasksWithResourceShapes( RAY_LOG(WARNING) << "Cancelling infeasible tasks with resource shapes " << resource_shapes_str; - bool task_cancelled = CancelTasks( + bool lease_cancelled = CancelLeases( predicate, rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_UNSCHEDULABLE, absl::StrCat( @@ -149,17 +149,17 @@ bool ClusterTaskManager::CancelTasksWithResourceShapes( " failed to schedule because there are not enough resources for the tasks " "or actors on the whole cluster.")); - RAY_LOG(INFO) << "Infeasible tasks cancellation complete with result=" << task_cancelled - << ",resource shapes=" << resource_shapes_str; + RAY_LOG(INFO) << "Infeasible tasks cancellation complete with result=" + << lease_cancelled << ",resource shapes=" << resource_shapes_str; - return task_cancelled; + return lease_cancelled; } -bool ClusterTaskManager::IsWorkWithResourceShape( +bool ClusterLeaseManager::IsWorkWithResourceShape( const std::shared_ptr &work, const std::vector &target_resource_shapes) { SchedulingClass scheduling_class = - work->task_.GetTaskSpecification().GetSchedulingClass(); + work->lease_.GetLeaseSpecification().GetSchedulingClass(); ResourceSet resource_set = TaskSpecification::GetSchedulingClassDescriptor(scheduling_class).resource_set; for (const auto &target_resource_shape : target_resource_shapes) { @@ -170,56 +170,56 @@ bool ClusterTaskManager::IsWorkWithResourceShape( return false; } -bool ClusterTaskManager::CancelAllTasksOwnedBy( +bool ClusterLeaseManager::CancelAllLeasesOwnedBy( const NodeID &node_id, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) { // Only tasks and regular actors are canceled because their lifetime is // the same as the owner. auto predicate = [node_id](const std::shared_ptr &work) { - return !work->task_.GetTaskSpecification().IsDetachedActor() && - work->task_.GetTaskSpecification().CallerNodeId() == node_id; + return !work->lease_.GetLeaseSpecification().IsDetachedActor() && + work->lease_.GetLeaseSpecification().CallerNodeId() == node_id; }; - return CancelTasks(predicate, failure_type, scheduling_failure_message); + return CancelLeases(predicate, failure_type, scheduling_failure_message); } -bool ClusterTaskManager::CancelAllTasksOwnedBy( +bool ClusterLeaseManager::CancelAllLeasesOwnedBy( const WorkerID &worker_id, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) { // Only tasks and regular actors are canceled because their lifetime is // the same as the owner. auto predicate = [worker_id](const std::shared_ptr &work) { - return !work->task_.GetTaskSpecification().IsDetachedActor() && - work->task_.GetTaskSpecification().CallerWorkerId() == worker_id; + return !work->lease_.GetLeaseSpecification().IsDetachedActor() && + work->lease_.GetLeaseSpecification().CallerWorkerId() == worker_id; }; - return CancelTasks(predicate, failure_type, scheduling_failure_message); + return CancelLeases(predicate, failure_type, scheduling_failure_message); } -void ClusterTaskManager::ScheduleAndDispatchTasks() { +void ClusterLeaseManager::ScheduleAndGrantLeases() { // Always try to schedule infeasible tasks in case they are now feasible. - TryScheduleInfeasibleTask(); + TryScheduleInfeasibleLease(); std::deque> works_to_cancel; - for (auto shapes_it = tasks_to_schedule_.begin(); - shapes_it != tasks_to_schedule_.end();) { + for (auto shapes_it = leases_to_schedule_.begin(); + shapes_it != leases_to_schedule_.end();) { auto &work_queue = shapes_it->second; bool is_infeasible = false; for (auto work_it = work_queue.begin(); work_it != work_queue.end();) { - // Check every task in task_to_schedule queue to see + // Check every lease in lease_to_schedule queue to see // whether it can be scheduled. This avoids head-of-line - // blocking where a task which cannot be scheduled because + // blocking where a lease which cannot be scheduled because // there are not enough available resources blocks other - // tasks from being scheduled. + // leases from being scheduled. const std::shared_ptr &work = *work_it; - RayTask task = work->task_; - RAY_LOG(DEBUG) << "Scheduling pending task " - << task.GetTaskSpecification().TaskId(); + RayLease lease = work->lease_; + RAY_LOG(DEBUG) << "Scheduling pending lease " + << lease.GetLeaseSpecification().LeaseId(); auto scheduling_node_id = cluster_resource_scheduler_.GetBestSchedulableNode( - task.GetTaskSpecification(), + lease.GetLeaseSpecification(), /*preferred_node_id*/ work->PrioritizeLocalNode() ? self_node_id_.Binary() - : task.GetPreferredNodeID(), + : lease.GetPreferredNodeID(), /*exclude_local_node*/ false, /*requires_object_store_memory*/ false, &is_infeasible); @@ -227,14 +227,14 @@ void ClusterTaskManager::ScheduleAndDispatchTasks() { // There is no node that has available resources to run the request. // Move on to the next shape. if (scheduling_node_id.IsNil()) { - RAY_LOG(DEBUG) << "No node found to schedule a task " - << task.GetTaskSpecification().TaskId() << " is infeasible?" + RAY_LOG(DEBUG) << "No node found to schedule a lease " + << lease.GetLeaseSpecification().LeaseId() << " is infeasible?" << is_infeasible; - if (task.GetTaskSpecification().IsNodeAffinitySchedulingStrategy() && - !task.GetTaskSpecification().GetNodeAffinitySchedulingStrategySoft()) { + if (lease.GetLeaseSpecification().IsNodeAffinitySchedulingStrategy() && + !lease.GetLeaseSpecification().GetNodeAffinitySchedulingStrategySoft()) { // This can only happen if the target node doesn't exist or is infeasible. - // The task will never be schedulable in either case so we should fail it. + // The lease will never be schedulable in either case so we should fail it. if (cluster_resource_scheduler_.IsLocalNodeWithRaylet()) { ReplyCancelled( *work, @@ -246,9 +246,9 @@ void ClusterTaskManager::ScheduleAndDispatchTasks() { work_it = work_queue.erase(work_it); } else { // If scheduling is done by gcs, we can not `ReplyCancelled` now because it - // would synchronously call `ClusterTaskManager::CancelTask`, where - // `task_to_schedule_`'s iterator will be invalidated. So record this work and - // it will be handled below (out of the loop). + // would synchronously call `ClusterLeaseManager::CancelLease`, where + // `lease_to_schedule_`'s iterator will be invalidated. So record this work + // and it will be handled below (out of the loop). works_to_cancel.push_back(*work_it); work_it++; } @@ -269,15 +269,15 @@ void ClusterTaskManager::ScheduleAndDispatchTasks() { // Only announce the first item as infeasible. auto &cur_work_queue = shapes_it->second; const auto &work = cur_work_queue[0]; - const RayTask task = work->task_; - if (announce_infeasible_task_) { - announce_infeasible_task_(task); + const RayLease lease = work->lease_; + if (announce_infeasible_lease_) { + announce_infeasible_lease_(lease); } - infeasible_tasks_[shapes_it->first] = std::move(shapes_it->second); - tasks_to_schedule_.erase(shapes_it++); + infeasible_leases_[shapes_it->first] = std::move(shapes_it->second); + leases_to_schedule_.erase(shapes_it++); } else if (work_queue.empty()) { - tasks_to_schedule_.erase(shapes_it++); + leases_to_schedule_.erase(shapes_it++); } else { shapes_it++; } @@ -285,7 +285,7 @@ void ClusterTaskManager::ScheduleAndDispatchTasks() { for (const auto &work : works_to_cancel) { // All works in `works_to_cancel` are scheduled by gcs. So `ReplyCancelled` - // will synchronously call `ClusterTaskManager::CancelTask`, where works are + // will synchronously call `ClusterLeaseManager::CancelLease`, where works are // erased from the pending queue. ReplyCancelled(*work, rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_UNSCHEDULABLE, @@ -294,26 +294,27 @@ void ClusterTaskManager::ScheduleAndDispatchTasks() { } works_to_cancel.clear(); - local_task_manager_.ScheduleAndDispatchTasks(); + local_lease_manager_.ScheduleAndGrantLeases(); } -void ClusterTaskManager::TryScheduleInfeasibleTask() { - for (auto shapes_it = infeasible_tasks_.begin(); - shapes_it != infeasible_tasks_.end();) { +void ClusterLeaseManager::TryScheduleInfeasibleLease() { + for (auto shapes_it = infeasible_leases_.begin(); + shapes_it != infeasible_leases_.end();) { auto &work_queue = shapes_it->second; RAY_CHECK(!work_queue.empty()) << "Empty work queue shouldn't have been added as a infeasible shape."; // We only need to check the first item because every task has the same shape. // If the first entry is infeasible, that means everything else is the same. const auto work = work_queue[0]; - RayTask task = work->task_; - RAY_LOG(DEBUG) << "Check if the infeasible task is schedulable in any node. task_id:" - << task.GetTaskSpecification().TaskId(); + RayLease lease = work->lease_; + RAY_LOG(DEBUG) + << "Check if the infeasible lease is schedulable in any node. lease_id:" + << lease.GetLeaseSpecification().LeaseId(); bool is_infeasible; cluster_resource_scheduler_.GetBestSchedulableNode( - task.GetTaskSpecification(), + lease.GetLeaseSpecification(), /*preferred_node_id*/ work->PrioritizeLocalNode() ? self_node_id_.Binary() - : task.GetPreferredNodeID(), + : lease.GetPreferredNodeID(), /*exclude_local_node*/ false, /*requires_object_store_memory*/ false, &is_infeasible); @@ -321,31 +322,31 @@ void ClusterTaskManager::TryScheduleInfeasibleTask() { // There is no node that has available resources to run the request. // Move on to the next shape. if (is_infeasible) { - RAY_LOG(DEBUG) << "No feasible node found for task " - << task.GetTaskSpecification().TaskId(); + RAY_LOG(DEBUG) << "No feasible node found for lease " + << lease.GetLeaseSpecification().LeaseId(); shapes_it++; } else { - RAY_LOG(DEBUG) << "Infeasible task of task id " - << task.GetTaskSpecification().TaskId() - << " is now feasible. Move the entry back to tasks_to_schedule_"; - tasks_to_schedule_[shapes_it->first] = std::move(shapes_it->second); - infeasible_tasks_.erase(shapes_it++); + RAY_LOG(DEBUG) << "Infeasible lease of lease id " + << lease.GetLeaseSpecification().LeaseId() + << " is now feasible. Move the entry back to leases_to_schedule_"; + leases_to_schedule_[shapes_it->first] = std::move(shapes_it->second); + infeasible_leases_.erase(shapes_it++); } } } -bool ClusterTaskManager::CancelTask( - const TaskID &task_id, +bool ClusterLeaseManager::CancelLease( + const LeaseID &lease_id, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) { - auto predicate = [task_id](const std::shared_ptr &work) { - return work->task_.GetTaskSpecification().TaskId() == task_id; + auto predicate = [lease_id](const std::shared_ptr &work) { + return work->lease_.GetLeaseSpecification().LeaseId() == lease_id; }; - return CancelTasks(predicate, failure_type, scheduling_failure_message); + return CancelLeases(predicate, failure_type, scheduling_failure_message); } -void ClusterTaskManager::FillResourceUsage(rpc::ResourcesData &data) { +void ClusterLeaseManager::FillResourceUsage(rpc::ResourcesData &data) { // This populates load information. scheduler_resource_reporter_.FillResourceUsage(data); // This populates usage information. @@ -363,17 +364,17 @@ void ClusterTaskManager::FillResourceUsage(rpc::ResourcesData &data) { resource_view_sync_message.draining_deadline_timestamp_ms()); } -const RayTask *ClusterTaskManager::AnyPendingTasksForResourceAcquisition( - int *num_pending_actor_creation, int *num_pending_tasks) const { - const RayTask *exemplar = nullptr; - // We are guaranteed that these tasks are blocked waiting for resources after a - // call to ScheduleAndDispatchTasks(). They may be waiting for workers as well, but +const RayLease *ClusterLeaseManager::AnyPendingLeasesForResourceAcquisition( + int *num_pending_actor_creation, int *num_pending_leases) const { + const RayLease *exemplar = nullptr; + // We are guaranteed that these leases are blocked waiting for resources after a + // call to ScheduleAndGrantLeases(). They may be waiting for workers as well, but // this should be a transient condition only. - for (const auto &shapes_it : tasks_to_schedule_) { + for (const auto &shapes_it : leases_to_schedule_) { auto &work_queue = shapes_it.second; for (const auto &work_it : work_queue) { const auto &work = *work_it; - const auto &task = work_it->task_; + const auto &lease = work_it->lease_; // If the work is not in the waiting state, it will be scheduled soon or won't be // scheduled. Consider as non-pending. @@ -392,37 +393,37 @@ const RayTask *ClusterTaskManager::AnyPendingTasksForResourceAcquisition( continue; } - if (task.GetTaskSpecification().IsActorCreationTask()) { + if (lease.GetLeaseSpecification().IsActorCreationTask()) { *num_pending_actor_creation += 1; } else { - *num_pending_tasks += 1; + *num_pending_leases += 1; } if (exemplar == nullptr) { - exemplar = &task; + exemplar = &lease; } } } - auto local_task_exemplar = local_task_manager_.AnyPendingTasksForResourceAcquisition( - num_pending_actor_creation, num_pending_tasks); - // Prefer returning the cluster task manager exemplar if it exists. - return exemplar == nullptr ? local_task_exemplar : exemplar; + auto local_lease_exemplar = local_lease_manager_.AnyPendingLeasesForResourceAcquisition( + num_pending_actor_creation, num_pending_leases); + // Prefer returning the cluster lease manager exemplar if it exists. + return exemplar == nullptr ? local_lease_exemplar : exemplar; } -void ClusterTaskManager::RecordMetrics() const { +void ClusterLeaseManager::RecordMetrics() const { internal_stats_.RecordMetrics(); cluster_resource_scheduler_.GetLocalResourceManager().RecordMetrics(); } -std::string ClusterTaskManager::DebugStr() const { +std::string ClusterLeaseManager::DebugStr() const { return internal_stats_.ComputeAndReportDebugStr(); } -void ClusterTaskManager::ScheduleOnNode(const NodeID &spillback_to, - const std::shared_ptr &work) { +void ClusterLeaseManager::ScheduleOnNode(const NodeID &spillback_to, + const std::shared_ptr &work) { if (spillback_to == self_node_id_) { - local_task_manager_.QueueAndScheduleTask(work); + local_lease_manager_.QueueAndScheduleLease(work); return; } @@ -434,16 +435,17 @@ void ClusterTaskManager::ScheduleOnNode(const NodeID &spillback_to, return; } - internal_stats_.TaskSpilled(); + internal_stats_.LeaseSpilled(); - const auto &task = work->task_; - const auto &task_spec = task.GetTaskSpecification(); - RAY_LOG(DEBUG) << "Spilling task " << task_spec.TaskId() << " to node " << spillback_to; + const auto &lease = work->lease_; + const auto &lease_spec = lease.GetLeaseSpecification(); + RAY_LOG(DEBUG) << "Spilling lease " << lease_spec.LeaseId() << " to node " + << spillback_to; if (!cluster_resource_scheduler_.AllocateRemoteTaskResources( scheduling::NodeID(spillback_to.Binary()), - task_spec.GetRequiredResources().GetResourceMap())) { - RAY_LOG(DEBUG) << "Tried to allocate resources for request " << task_spec.TaskId() + lease_spec.GetRequiredResources().GetResourceMap())) { + RAY_LOG(DEBUG) << "Tried to allocate resources for request " << lease_spec.LeaseId() << " on a remote node that are no longer available"; } @@ -460,27 +462,27 @@ void ClusterTaskManager::ScheduleOnNode(const NodeID &spillback_to, send_reply_callback(); } -ClusterResourceScheduler &ClusterTaskManager::GetClusterResourceScheduler() const { +ClusterResourceScheduler &ClusterLeaseManager::GetClusterResourceScheduler() const { return cluster_resource_scheduler_; } -size_t ClusterTaskManager::GetInfeasibleQueueSize() const { +size_t ClusterLeaseManager::GetInfeasibleQueueSize() const { size_t count = 0; - for (const auto &cls_entry : infeasible_tasks_) { + for (const auto &cls_entry : infeasible_leases_) { count += cls_entry.second.size(); } return count; } -size_t ClusterTaskManager::GetPendingQueueSize() const { +size_t ClusterLeaseManager::GetPendingQueueSize() const { size_t count = 0; - for (const auto &cls_entry : tasks_to_schedule_) { + for (const auto &cls_entry : leases_to_schedule_) { count += cls_entry.second.size(); } return count; } -void ClusterTaskManager::FillPendingActorInfo(rpc::ResourcesData &data) const { +void ClusterLeaseManager::FillPendingActorInfo(rpc::ResourcesData &data) const { scheduler_resource_reporter_.FillPendingActorCountByShape(data); } diff --git a/src/ray/raylet/scheduling/cluster_task_manager.h b/src/ray/raylet/scheduling/cluster_lease_manager.h similarity index 62% rename from src/ray/raylet/scheduling/cluster_task_manager.h rename to src/ray/raylet/scheduling/cluster_lease_manager.h index 5137d1fb527a..89f15ba62417 100644 --- a/src/ray/raylet/scheduling/cluster_task_manager.h +++ b/src/ray/raylet/scheduling/cluster_lease_manager.h @@ -19,109 +19,108 @@ #include #include "absl/container/flat_hash_map.h" +#include "ray/common/lease/lease.h" #include "ray/common/ray_object.h" -#include "ray/common/task/task.h" -#include "ray/common/task/task_common.h" +#include "ray/raylet/scheduling/cluster_lease_manager_interface.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" -#include "ray/raylet/scheduling/cluster_task_manager_interface.h" -#include "ray/raylet/scheduling/internal.h" -#include "ray/raylet/scheduling/local_task_manager_interface.h" +#include "ray/raylet/scheduling/local_lease_manager_interface.h" #include "ray/raylet/scheduling/scheduler_resource_reporter.h" #include "ray/raylet/scheduling/scheduler_stats.h" namespace ray { namespace raylet { -/// Schedules a task onto one node of the cluster. The logic is as follows: -/// 1. Queue tasks for scheduling. +/// Schedules a lease onto one node of the cluster. The logic is as follows: +/// 1. Queue leases for scheduling. /// 2. Pick a node on the cluster which has the available resources to run a -/// task. +/// lease. /// * Step 2 should occur any time the state of the cluster is -/// changed, or a new task is queued. -/// 3. For tasks that's infeasable, put them into infeasible queue and reports -/// it to gcs, where the auto scaler will be notified and start new node +/// changed, or a new lease is queued. +/// 3. For leases that are infeasible, put them into infeasible queue and report +/// it to gcs, where the auto scaler will be notified and start a new node /// to accommodate the requirement. -class ClusterTaskManager : public ClusterTaskManagerInterface { +class ClusterLeaseManager : public ClusterLeaseManagerInterface { public: /// \param self_node_id: ID of local node. /// \param cluster_resource_scheduler: The resource scheduler which contains /// the state of the cluster. /// \param get_node_info: Function that returns the node info for a node. - /// \param announce_infeasible_task: Callback that informs the user if a task + /// \param announce_infeasible_lease: Callback that informs the user if a lease /// is infeasible. - /// \param local_task_manager: Manages local tasks. + /// \param local_lease_manager: Manages local leases. /// \param get_time_ms: A callback which returns the current time in milliseconds. - ClusterTaskManager( + ClusterLeaseManager( const NodeID &self_node_id, ClusterResourceScheduler &cluster_resource_scheduler, internal::NodeInfoGetter get_node_info, - std::function announce_infeasible_task, - ILocalTaskManager &local_task_manager, + std::function announce_infeasible_lease, + LocalLeaseManagerInterface &local_lease_manager, std::function get_time_ms = []() { return static_cast(absl::GetCurrentTimeNanos() / 1e6); }); - /// Queue task and schedule. This happens when processing the worker lease request. + /// Queue lease and schedule. This happens when processing the worker lease request. /// - /// \param task: The incoming task to be queued and scheduled. + /// \param lease: The incoming lease to be queued and scheduled. /// \param grant_or_reject: True if we we should either grant or reject the request /// but no spillback. /// \param is_selected_based_on_locality : should schedule on local node if possible. /// \param reply: The reply of the lease request. /// \param send_reply_callback: The function used during dispatching. - void QueueAndScheduleTask(RayTask task, - bool grant_or_reject, - bool is_selected_based_on_locality, - rpc::RequestWorkerLeaseReply *reply, - rpc::SendReplyCallback send_reply_callback) override; + void QueueAndScheduleLease(RayLease lease, + bool grant_or_reject, + bool is_selected_based_on_locality, + rpc::RequestWorkerLeaseReply *reply, + rpc::SendReplyCallback send_reply_callback) override; - /// Attempt to cancel an already queued task. + /// Attempt to cancel an already queued lease. /// - /// \param task_id: The id of the task to remove. + /// \param lease_id: The lease_id of the lease to remove. /// \param failure_type: The failure type. /// \param scheduling_failure_message: The failure message. /// - /// \return True if task was successfully removed. This function will return - /// false if the task is already running. - bool CancelTask(const TaskID &task_id, - rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type = - rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_INTENDED, - const std::string &scheduling_failure_message = "") override; - - bool CancelAllTasksOwnedBy( + /// \return True if lease was successfully cancelled. This function will return + /// false if the lease is already granted. + bool CancelLease(const LeaseID &lease_id, + rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type = + rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_INTENDED, + const std::string &scheduling_failure_message = "") override; + + bool CancelAllLeasesOwnedBy( const WorkerID &worker_id, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type = rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_INTENDED, const std::string &scheduling_failure_message = "") override; - bool CancelAllTasksOwnedBy( + bool CancelAllLeasesOwnedBy( const NodeID &node_id, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type = rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_INTENDED, const std::string &scheduling_failure_message = "") override; - /// Cancel all tasks that requires certain resource shape. - /// This function is intended to be used to cancel the infeasible tasks. To make it a + /// Cancel all leases that requires certain resource shape. + /// This function is intended to be used to cancel the infeasible leases. To make it a /// more general function, please modify the signature by adding parameters including /// the failure type and the failure message. /// /// \param target_resource_shapes: The resource shapes to cancel. /// - /// \return True if any task was successfully cancelled. This function will return - /// false if the task is already running. This shouldn't happen in noremal cases - /// because the infeasible tasks shouldn't be able to run due to resource constraints. - bool CancelTasksWithResourceShapes( + /// \return True if any lease was successfully cancelled. This function will return + /// false if the lease is already granted. This shouldn't happen in normal cases + /// because the infeasible leases shouldn't be granted due to resource constraints. + bool CancelLeasesWithResourceShapes( const std::vector target_resource_shapes) override; - /// Attempt to cancel all queued tasks that match the predicate. + /// Attempt to cancel all queued leases that match the predicate. /// - /// \param predicate: A function that returns true if a task needs to be cancelled. + /// \param predicate: A function that returns true if a lease needs to be cancelled. /// \param failure_type: The reason for cancellation. /// \param scheduling_failure_message: The reason message for cancellation. - /// \return True if any task was successfully cancelled. - bool CancelTasks(std::function &)> predicate, - rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, - const std::string &scheduling_failure_message) override; + /// \return True if any lease was successfully cancelled. + bool CancelLeases( + std::function &)> predicate, + rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, + const std::string &scheduling_failure_message) override; /// Populate the relevant parts of the heartbeat table. This is intended for /// sending resource usage of raylet to gcs. In particular, this should fill in @@ -131,29 +130,29 @@ class ClusterTaskManager : public ClusterTaskManagerInterface { /// the only fields used. void FillResourceUsage(rpc::ResourcesData &data) override; - /// Return with an exemplar if any tasks are pending resource acquisition. + /// Return with an exemplar if any leases are pending resource acquisition. /// - /// \param[in,out] num_pending_actor_creation: Number of pending actor creation tasks. - /// \param[in,out] num_pending_tasks: Number of pending tasks. - /// \return An example task that is deadlocking if any tasks are pending resource + /// \param[in,out] num_pending_actor_creation: Number of pending actor creation leases. + /// \param[in,out] num_pending_leases: Number of pending leases. + /// \return An example lease that is deadlocking if any leases are pending resource /// acquisition. - const RayTask *AnyPendingTasksForResourceAcquisition( - int *num_pending_actor_creation, int *num_pending_tasks) const override; + const RayLease *AnyPendingLeasesForResourceAcquisition( + int *num_pending_actor_creation, int *num_pending_leases) const override; - // Schedule and dispatch tasks. - void ScheduleAndDispatchTasks() override; + // Schedule and grant leases. + void ScheduleAndGrantLeases() override; /// Record the internal metrics. void RecordMetrics() const override; - /// The helper to dump the debug state of the cluster task manater. + /// The helper to dump the debug state of the cluster lease manater. std::string DebugStr() const override; ClusterResourceScheduler &GetClusterResourceScheduler() const; - /// Get the count of tasks in `infeasible_tasks_`. + /// Get the count of leases in `infeasible_leases_`. size_t GetInfeasibleQueueSize() const; - /// Get the count of tasks in `tasks_to_schedule_`. + /// Get the count of leases in `leases_to_schedule_`. size_t GetPendingQueueSize() const; /// Populate the info of pending and infeasible actors. This function @@ -164,14 +163,16 @@ class ClusterTaskManager : public ClusterTaskManagerInterface { void FillPendingActorInfo(rpc::ResourcesData &data) const; private: - void TryScheduleInfeasibleTask(); + void TryScheduleInfeasibleLease(); - // Schedule the task onto a node (which could be either remote or local). + // Schedule the lease onto a node (which could be to a worker thats in a local or remote + // node). void ScheduleOnNode(const NodeID &node_to_schedule, const std::shared_ptr &work); /// Recompute the debug stats. - /// It is needed because updating the debug state is expensive for cluster_task_manager. + /// It is needed because updating the debug state is expensive for + /// cluster_lease_manager. /// TODO(sang): Update the internal states value dynamically instead of iterating the /// data structure. void RecomputeDebugStats() const; @@ -194,20 +195,20 @@ class ClusterTaskManager : public ClusterTaskManagerInterface { /// Function to get the node information of a given node id. internal::NodeInfoGetter get_node_info_; - /// Function to announce infeasible task to GCS. - std::function announce_infeasible_task_; + /// Function to announce infeasible lease to GCS. + std::function announce_infeasible_lease_; - ILocalTaskManager &local_task_manager_; + LocalLeaseManagerInterface &local_lease_manager_; /// Queue of lease requests that are waiting for resources to become available. - /// Tasks move from scheduled -> dispatch | waiting. + /// Leases move from scheduled -> dispatch | waiting. absl::flat_hash_map>> - tasks_to_schedule_; + leases_to_schedule_; /// Queue of lease requests that are infeasible. - /// Tasks go between scheduling <-> infeasible. + /// Leases go between scheduling <-> infeasible. absl::flat_hash_map>> - infeasible_tasks_; + infeasible_leases_; const SchedulerResourceReporter scheduler_resource_reporter_; mutable SchedulerStats internal_stats_; @@ -216,8 +217,8 @@ class ClusterTaskManager : public ClusterTaskManagerInterface { std::function get_time_ms_; friend class SchedulerStats; - friend class ClusterTaskManagerTest; - FRIEND_TEST(ClusterTaskManagerTest, FeasibleToNonFeasible); + friend class ClusterLeaseManagerTest; + FRIEND_TEST(ClusterLeaseManagerTest, FeasibleToNonFeasible); }; } // namespace raylet } // namespace ray diff --git a/src/ray/raylet/scheduling/cluster_task_manager_interface.h b/src/ray/raylet/scheduling/cluster_lease_manager_interface.h similarity index 58% rename from src/ray/raylet/scheduling/cluster_task_manager_interface.h rename to src/ray/raylet/scheduling/cluster_lease_manager_interface.h index 7950706eb04e..e8b885c7a8ad 100644 --- a/src/ray/raylet/scheduling/cluster_task_manager_interface.h +++ b/src/ray/raylet/scheduling/cluster_lease_manager_interface.h @@ -17,17 +17,18 @@ #include #include +#include "ray/raylet/scheduling/internal.h" #include "ray/rpc/server_call.h" #include "src/ray/protobuf/node_manager.pb.h" namespace ray { namespace raylet { -class ClusterTaskManagerInterface { +class ClusterLeaseManagerInterface { public: - virtual ~ClusterTaskManagerInterface() = default; + virtual ~ClusterLeaseManagerInterface() = default; - // Schedule and dispatch tasks. - virtual void ScheduleAndDispatchTasks() = 0; + // Schedule and dispatch leases. + virtual void ScheduleAndGrantLeases() = 0; /// Populate the relevant parts of the heartbeat table. This is intended for /// sending raylet <-> gcs heartbeats. In particular, this should fill in @@ -37,81 +38,81 @@ class ClusterTaskManagerInterface { /// fields used. virtual void FillResourceUsage(rpc::ResourcesData &data) = 0; - /// Attempt to cancel an already queued task. + /// Attempt to cancel an already queued lease. /// - /// \param task_id: The id of the task to remove. + /// \param lease_id: The id of the lease to remove. /// \param failure_type: The failure type. /// \param scheduling_failure_message: The failure message. /// - /// \return True if task was successfully removed. This function will return - /// false if the task is already running. - virtual bool CancelTask( - const TaskID &task_id, + /// \return True if lease was successfully cancelled. This function will return + /// false if the lease is already granted. + virtual bool CancelLease( + const LeaseID &lease_id, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type = rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_INTENDED, const std::string &scheduling_failure_message = "") = 0; - /// Cancel all tasks owned by a specific worker. - virtual bool CancelAllTasksOwnedBy( + /// Cancel all leases owned by a specific worker. + virtual bool CancelAllLeasesOwnedBy( const WorkerID &worker_id, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type = rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_INTENDED, const std::string &scheduling_failure_message = "") = 0; - /// Cancel all tasks owned by a worker on the specific node. - virtual bool CancelAllTasksOwnedBy( + /// Cancel all leases owned by a worker on the specific node. + virtual bool CancelAllLeasesOwnedBy( const NodeID &node_id, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type = rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_INTENDED, const std::string &scheduling_failure_message = "") = 0; - /// Attempt to cancel all queued tasks that match the resource shapes. - /// This function is intended to be used to cancel the infeasible tasks. To make it a + /// Attempt to cancel all queued leases that match the resource shapes. + /// This function is intended to be used to cancel the infeasible leases. To make it a /// more general function, please modify the signature by adding parameters including /// the failure type and the failure message. /// /// \param target_resource_shapes: The resource shapes to cancel. /// - /// \return True if any task was successfully removed. This function will return false - /// if the task is already running. This shouldn't happen in noremal cases because the - /// infeasible tasks shouldn't be able to run due to resource constraints. - virtual bool CancelTasksWithResourceShapes( + /// \return True if any lease was successfully removed. This function will return false + /// if the lease is already running. This shouldn't happen in noremal cases because the + /// infeasible leases shouldn't be able to run due to resource constraints. + virtual bool CancelLeasesWithResourceShapes( const std::vector target_resource_shapes) = 0; - /// Attempt to cancel all queued tasks that match the predicate. + /// Attempt to cancel all queued leases that match the predicate. /// - /// \param predicate: A function that returns true if a task needs to be cancelled. + /// \param predicate: A function that returns true if a lease needs to be cancelled. /// \param failure_type: The reason for cancellation. /// \param scheduling_failure_message: The reason message for cancellation. - /// \return True if any task was successfully cancelled. - virtual bool CancelTasks( + /// \return True if any lease was successfully cancelled. + virtual bool CancelLeases( std::function &)> predicate, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) = 0; - /// Queue task and schedule. This happens when processing the worker lease request. + /// Queue lease and schedule. This happens when processing the worker lease request. /// - /// \param task: The incoming task to be queued and scheduled. + /// \param lease: The incoming lease to be queued and scheduled. /// \param grant_or_reject: True if we we should either grant or reject the request /// but no spillback. /// \param reply: The reply of the lease request. /// \param send_reply_callback: The function used during dispatching. - virtual void QueueAndScheduleTask(RayTask task, - bool grant_or_reject, - bool is_selected_based_on_locality, - rpc::RequestWorkerLeaseReply *reply, - rpc::SendReplyCallback send_reply_callback) = 0; + virtual void QueueAndScheduleLease(RayLease lease, + bool grant_or_reject, + bool is_selected_based_on_locality, + rpc::RequestWorkerLeaseReply *reply, + rpc::SendReplyCallback send_reply_callback) = 0; - /// Return with an exemplar if any tasks are pending resource acquisition. + /// Return with an exemplar if any leases are pending resource acquisition. /// /// \param[in] num_pending_actor_creation Number of pending actor creation tasks. - /// \param[in] num_pending_tasks Number of pending tasks. - /// \return An example task that is deadlocking if any tasks are pending resource + /// \param[in] num_pending_leases Number of pending leases. + /// \return An example lease that is deadlocking if any leases are pending resource /// acquisition. - virtual const RayTask *AnyPendingTasksForResourceAcquisition( - int *num_pending_actor_creation, int *num_pending_tasks) const = 0; + virtual const RayLease *AnyPendingLeasesForResourceAcquisition( + int *num_pending_actor_creation, int *num_pending_leases) const = 0; - /// The helper to dump the debug state of the cluster task manater. + /// The helper to dump the debug state of the cluster lease manater. virtual std::string DebugStr() const = 0; /// Record the internal metrics. diff --git a/src/ray/raylet/scheduling/cluster_resource_manager.h b/src/ray/raylet/scheduling/cluster_resource_manager.h index 1b3f01528031..58bde9688105 100644 --- a/src/ray/raylet/scheduling/cluster_resource_manager.h +++ b/src/ray/raylet/scheduling/cluster_resource_manager.h @@ -33,7 +33,7 @@ namespace ray { namespace raylet { -class ClusterTaskManagerTest; +class ClusterLeaseManagerTest; class SchedulingPolicyTest; } // namespace raylet namespace raylet_scheduling_policy { @@ -180,7 +180,7 @@ class ClusterResourceManager { friend class ClusterResourceSchedulerTest; friend struct ClusterResourceManagerTest; - friend class raylet::ClusterTaskManagerTest; + friend class raylet::ClusterLeaseManagerTest; FRIEND_TEST(ClusterResourceSchedulerTest, SchedulingDeleteClusterNodeTest); FRIEND_TEST(ClusterResourceSchedulerTest, SchedulingModifyClusterNodeTest); FRIEND_TEST(ClusterResourceSchedulerTest, SchedulingUpdateAvailableResourcesTest); @@ -199,7 +199,7 @@ class ClusterResourceManager { FRIEND_TEST(ClusterResourceSchedulerTest, AvailableResourceInstancesOpsTest); FRIEND_TEST(ClusterResourceSchedulerTest, DirtyLocalViewTest); FRIEND_TEST(ClusterResourceSchedulerTest, DynamicResourceTest); - FRIEND_TEST(ClusterTaskManagerTestWithGPUsAtHead, RleaseAndReturnWorkerCpuResources); + FRIEND_TEST(ClusterLeaseManagerTestWithGPUsAtHead, RleaseAndReturnWorkerCpuResources); FRIEND_TEST(ClusterResourceSchedulerTest, TestForceSpillback); FRIEND_TEST(ClusterResourceSchedulerTest, AffinityWithBundleScheduleTest); FRIEND_TEST(ClusterResourceSchedulerTest, LabelSelectorIsSchedulableOnNodeTest); diff --git a/src/ray/raylet/scheduling/cluster_resource_scheduler.cc b/src/ray/raylet/scheduling/cluster_resource_scheduler.cc index dff976ffcf62..4f1448320547 100644 --- a/src/ray/raylet/scheduling/cluster_resource_scheduler.cc +++ b/src/ray/raylet/scheduling/cluster_resource_scheduler.cc @@ -284,7 +284,7 @@ bool ClusterResourceScheduler::IsSchedulableOnNode( } scheduling::NodeID ClusterResourceScheduler::GetBestSchedulableNode( - const TaskSpecification &task_spec, + const LeaseSpecification &lease_spec, const std::string &preferred_node_id, bool exclude_local_node, bool requires_object_store_memory, @@ -293,8 +293,8 @@ scheduling::NodeID ClusterResourceScheduler::GetBestSchedulableNode( // going through the full hybrid policy since we don't want spillback. if (preferred_node_id == local_node_id_.Binary() && !exclude_local_node && IsSchedulableOnNode(local_node_id_, - task_spec.GetRequiredPlacementResources().GetResourceMap(), - task_spec.GetLabelSelector(), + lease_spec.GetRequiredPlacementResources().GetResourceMap(), + lease_spec.GetLabelSelector(), requires_object_store_memory)) { *is_infeasible = false; return local_node_id_; @@ -303,11 +303,11 @@ scheduling::NodeID ClusterResourceScheduler::GetBestSchedulableNode( // This argument is used to set violation, which is an unsupported feature now. int64_t _unused; scheduling::NodeID best_node = - GetBestSchedulableNode(task_spec.GetRequiredPlacementResources().GetResourceMap(), - task_spec.GetLabelSelector(), - task_spec.GetMessage().scheduling_strategy(), + GetBestSchedulableNode(lease_spec.GetRequiredPlacementResources().GetResourceMap(), + lease_spec.GetLabelSelector(), + lease_spec.GetMessage().scheduling_strategy(), requires_object_store_memory, - task_spec.IsActorCreationTask(), + lease_spec.IsActorCreationTask(), exclude_local_node, preferred_node_id, &_unused, @@ -316,16 +316,16 @@ scheduling::NodeID ClusterResourceScheduler::GetBestSchedulableNode( // There is no other available nodes. if (!best_node.IsNil() && !IsSchedulableOnNode(best_node, - task_spec.GetRequiredPlacementResources().GetResourceMap(), - task_spec.GetLabelSelector(), + lease_spec.GetRequiredPlacementResources().GetResourceMap(), + lease_spec.GetLabelSelector(), requires_object_store_memory)) { // Prefer waiting on the local node if possible // since the local node is chosen for a reason (e.g. spread). if ((preferred_node_id == local_node_id_.Binary()) && NodeAvailable(local_node_id_)) { auto resource_request = ResourceMapToResourceRequest( - task_spec.GetRequiredPlacementResources().GetResourceMap(), + lease_spec.GetRequiredPlacementResources().GetResourceMap(), requires_object_store_memory); - const auto &selector = task_spec.GetLabelSelector(); + const auto &selector = lease_spec.GetLabelSelector(); resource_request.SetLabelSelector(selector); if (cluster_resource_manager_->HasFeasibleResources(local_node_id_, resource_request)) { @@ -334,7 +334,7 @@ scheduling::NodeID ClusterResourceScheduler::GetBestSchedulableNode( } } // If the task is being scheduled by gcs, return nil to make it stay in the - // `cluster_task_manager`'s queue. + // `cluster_lease_manager`'s queue. if (!is_local_node_with_raylet_) { return scheduling::NodeID::Nil(); } diff --git a/src/ray/raylet/scheduling/cluster_resource_scheduler.h b/src/ray/raylet/scheduling/cluster_resource_scheduler.h index 39a7f0111e1b..2df66334975f 100644 --- a/src/ray/raylet/scheduling/cluster_resource_scheduler.h +++ b/src/ray/raylet/scheduling/cluster_resource_scheduler.h @@ -84,7 +84,7 @@ class ClusterResourceScheduler { /// Find a node in the cluster on which we can schedule a given resource request. /// In hybrid mode, see `scheduling_policy.h` for a description of the policy. /// - /// \param task_spec: Task/Actor to be scheduled. + /// \param lease_spec: Lease to be scheduled. /// \param preferred_node_id: the node where the task is preferred to be placed. An /// empty `preferred_node_id` (string) means no preferred node. /// \param exclude_local_node: true if we want to avoid local node. This will cancel @@ -96,7 +96,7 @@ class ClusterResourceScheduler { /// /// \return empty string, if no node can schedule the current request; otherwise, /// return the string name of a node that can schedule the resource request. - scheduling::NodeID GetBestSchedulableNode(const TaskSpecification &task_spec, + scheduling::NodeID GetBestSchedulableNode(const LeaseSpecification &lease_spec, const std::string &preferred_node_id, bool exclude_local_node, bool requires_object_store_memory, @@ -244,7 +244,7 @@ class ClusterResourceScheduler { FRIEND_TEST(ClusterResourceSchedulerTest, AvailableResourceInstancesOpsTest); FRIEND_TEST(ClusterResourceSchedulerTest, DirtyLocalViewTest); FRIEND_TEST(ClusterResourceSchedulerTest, DynamicResourceTest); - FRIEND_TEST(ClusterTaskManagerTestWithGPUsAtHead, RleaseAndReturnWorkerCpuResources); + FRIEND_TEST(ClusterLeaseManagerTestWithGPUsAtHead, RleaseAndReturnWorkerCpuResources); FRIEND_TEST(ClusterResourceSchedulerTest, TestForceSpillback); FRIEND_TEST(ClusterResourceSchedulerTest, AffinityWithBundleScheduleTest); FRIEND_TEST(ClusterResourceSchedulerTest, LabelSelectorIsSchedulableOnNodeTest); diff --git a/src/ray/raylet/scheduling/internal.h b/src/ray/raylet/scheduling/internal.h index 8e98eb342ff2..dfb6d130a618 100644 --- a/src/ray/raylet/scheduling/internal.h +++ b/src/ray/raylet/scheduling/internal.h @@ -17,10 +17,9 @@ #include #include +#include "ray/common/lease/lease.h" #include "ray/common/ray_object.h" #include "ray/common/scheduling/cluster_resource_data.h" -#include "ray/common/task/task.h" -#include "ray/common/task/task_common.h" #include "src/ray/protobuf/node_manager.pb.h" namespace ray::raylet::internal { @@ -51,23 +50,23 @@ enum class UnscheduledWorkCause { }; /// Work represents all the information needed to make a scheduling decision. -/// This includes the task, the information we need to communicate to +/// This includes the lease, the information we need to communicate to /// dispatch/spillback and the callback to trigger it. class Work { public: - RayTask task_; + RayLease lease_; bool grant_or_reject_; bool is_selected_based_on_locality_; rpc::RequestWorkerLeaseReply *reply_; std::function callback_; std::shared_ptr allocated_instances_; - Work(RayTask task, + Work(RayLease lease, bool grant_or_reject, bool is_selected_based_on_locality, rpc::RequestWorkerLeaseReply *reply, std::function callback, WorkStatus status = WorkStatus::WAITING) - : task_(std::move(task)), + : lease_(std::move(lease)), grant_or_reject_(grant_or_reject), is_selected_based_on_locality_(is_selected_based_on_locality), reply_(reply), diff --git a/src/ray/raylet/scheduling/local_task_manager_interface.h b/src/ray/raylet/scheduling/local_lease_manager_interface.h similarity index 57% rename from src/ray/raylet/scheduling/local_task_manager_interface.h rename to src/ray/raylet/scheduling/local_lease_manager_interface.h index 3eae10859b0d..dedf7ef53b98 100644 --- a/src/ray/raylet/scheduling/local_task_manager_interface.h +++ b/src/ray/raylet/scheduling/local_lease_manager_interface.h @@ -19,7 +19,7 @@ #include #include "absl/container/flat_hash_map.h" -#include "ray/common/task/task.h" +#include "ray/common/lease/lease.h" #include "ray/raylet/scheduling/internal.h" namespace ray { @@ -28,33 +28,33 @@ namespace raylet { // Forward declaration class WorkerInterface; -/// Manages the lifetime of a task on the local node. It receives request from -/// cluster_task_manager and tries to execute the task locally. -/// Read raylet/local_task_manager.h for more information. -class ILocalTaskManager { +/// Manages the lifetime of a lease on the local node. It receives request from +/// cluster_lease_manager and tries to execute the lease locally. +/// Read raylet/local_lease_manager.h for more information. +class LocalLeaseManagerInterface { public: - virtual ~ILocalTaskManager() = default; + virtual ~LocalLeaseManagerInterface() = default; - /// Queue task and schedule. - virtual void QueueAndScheduleTask(std::shared_ptr work) = 0; + /// Queue lease and schedule. + virtual void QueueAndScheduleLease(std::shared_ptr work) = 0; - // Schedule and dispatch tasks. - virtual void ScheduleAndDispatchTasks() = 0; + // Schedule and grant leases. + virtual void ScheduleAndGrantLeases() = 0; - /// Attempt to cancel all queued tasks that match the predicate. + /// Attempt to cancel all queued leases that match the predicate. /// - /// \param predicate: A function that returns true if a task needs to be cancelled. + /// \param predicate: A function that returns true if a lease needs to be cancelled. /// \param failure_type: The reason for cancellation. /// \param scheduling_failure_message: The reason message for cancellation. - /// \return True if any task was successfully cancelled. - virtual bool CancelTasks( + /// \return True if any lease was successfully cancelled. + virtual bool CancelLeases( std::function &)> predicate, rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, const std::string &scheduling_failure_message) = 0; virtual const absl::flat_hash_map>> - &GetTaskToDispatch() const = 0; + &GetLeasesToGrant() const = 0; virtual const absl::flat_hash_map> @@ -66,12 +66,12 @@ class ILocalTaskManager { virtual void ClearWorkerBacklog(const WorkerID &worker_id) = 0; - virtual const RayTask *AnyPendingTasksForResourceAcquisition( - int *num_pending_actor_creation, int *num_pending_tasks) const = 0; + virtual const RayLease *AnyPendingLeasesForResourceAcquisition( + int *num_pending_actor_creation, int *num_pending_leases) const = 0; - virtual void TasksUnblocked(const std::vector &ready_ids) = 0; + virtual void LeasesUnblocked(const std::vector &ready_ids) = 0; - virtual void TaskFinished(std::shared_ptr worker, RayTask *task) = 0; + virtual void CleanupLease(std::shared_ptr worker, RayLease *lease) = 0; virtual void ReleaseWorkerResources(std::shared_ptr worker) = 0; @@ -87,39 +87,38 @@ class ILocalTaskManager { virtual void DebugStr(std::stringstream &buffer) const = 0; - virtual size_t GetNumTaskSpilled() const = 0; - virtual size_t GetNumWaitingTaskSpilled() const = 0; - virtual size_t GetNumUnschedulableTaskSpilled() const = 0; + virtual size_t GetNumLeaseSpilled() const = 0; + virtual size_t GetNumWaitingLeaseSpilled() const = 0; + virtual size_t GetNumUnschedulableLeaseSpilled() const = 0; }; -/// A noop local task manager. It is a no-op class. We need this because there's no -/// "LocalTaskManager" when the `ClusterTaskManager` is used within GCS. In the long term, -/// we should make `ClusterTaskManager` not aware of `LocalTaskManager`. -class NoopLocalTaskManager : public ILocalTaskManager { +/// A noop local lease manager. It is a no-op class. We need this because there's no +/// "LocalLeaseManager" when the `ClusterLeaseManager` is used within GCS. In the long +/// term, we should make `ClusterLeaseManager` not aware of `LocalLeaseManager`. +class NoopLocalLeaseManager : public LocalLeaseManagerInterface { public: - NoopLocalTaskManager() = default; + NoopLocalLeaseManager() = default; - /// Queue task and schedule. - void QueueAndScheduleTask(std::shared_ptr work) override { + void QueueAndScheduleLease(std::shared_ptr work) override { RAY_CHECK(false) - << "This function should never be called by gcs' local task manager."; + << "This function should never be called by gcs' local lease manager."; } - // Schedule and dispatch tasks. - void ScheduleAndDispatchTasks() override {} + void ScheduleAndGrantLeases() override {} - bool CancelTasks(std::function &)> predicate, - rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, - const std::string &scheduling_failure_message) override { + bool CancelLeases( + std::function &)> predicate, + rpc::RequestWorkerLeaseReply::SchedulingFailureType failure_type, + const std::string &scheduling_failure_message) override { return false; } const absl::flat_hash_map>> - &GetTaskToDispatch() const override { + &GetLeasesToGrant() const override { static const absl::flat_hash_map>> - tasks_to_dispatch; - return tasks_to_dispatch; + leases_to_grant; + return leases_to_grant; } const absl::flat_hash_map> @@ -136,14 +135,14 @@ class NoopLocalTaskManager : public ILocalTaskManager { void ClearWorkerBacklog(const WorkerID &worker_id) override {} - const RayTask *AnyPendingTasksForResourceAcquisition( - int *num_pending_actor_creation, int *num_pending_tasks) const override { + const RayLease *AnyPendingLeasesForResourceAcquisition( + int *num_pending_actor_creation, int *num_pending_leases) const override { return nullptr; } - void TasksUnblocked(const std::vector &ready_ids) override {} + void LeasesUnblocked(const std::vector &ready_ids) override {} - void TaskFinished(std::shared_ptr worker, RayTask *task) override {} + void CleanupLease(std::shared_ptr worker, RayLease *lease) override {} void ReleaseWorkerResources(std::shared_ptr worker) override {} @@ -163,9 +162,9 @@ class NoopLocalTaskManager : public ILocalTaskManager { void DebugStr(std::stringstream &buffer) const override {} - size_t GetNumTaskSpilled() const override { return 0; } - size_t GetNumWaitingTaskSpilled() const override { return 0; } - size_t GetNumUnschedulableTaskSpilled() const override { return 0; } + size_t GetNumLeaseSpilled() const override { return 0; } + size_t GetNumWaitingLeaseSpilled() const override { return 0; } + size_t GetNumUnschedulableLeaseSpilled() const override { return 0; } }; } // namespace raylet diff --git a/src/ray/raylet/scheduling/scheduler_resource_reporter.cc b/src/ray/raylet/scheduling/scheduler_resource_reporter.cc index 597e7ddf5277..35be2849e6da 100644 --- a/src/ray/raylet/scheduling/scheduler_resource_reporter.cc +++ b/src/ray/raylet/scheduling/scheduler_resource_reporter.cc @@ -28,17 +28,17 @@ namespace raylet { SchedulerResourceReporter::SchedulerResourceReporter( const absl::flat_hash_map>> - &tasks_to_schedule, + &leases_to_schedule, const absl::flat_hash_map>> - &infeasible_tasks, - const ILocalTaskManager &local_task_manager) + &infeasible_leases, + const LocalLeaseManagerInterface &local_lease_manager) : max_resource_shapes_per_load_report_( RayConfig::instance().max_resource_shapes_per_load_report()), - tasks_to_schedule_(tasks_to_schedule), - tasks_to_dispatch_(local_task_manager.GetTaskToDispatch()), - infeasible_tasks_(infeasible_tasks), - backlog_tracker_(local_task_manager.GetBackLogTracker()) {} + leases_to_schedule_(leases_to_schedule), + leases_to_grant_(local_lease_manager.GetLeasesToGrant()), + infeasible_leases_(infeasible_leases), + backlog_tracker_(local_lease_manager.GetBackLogTracker()) {} int64_t SchedulerResourceReporter::TotalBacklogSize( SchedulingClass scheduling_class) const { @@ -133,22 +133,23 @@ void SchedulerResourceReporter::FillResourceUsage(rpc::ResourcesData &data) cons }; fill_resource_usage_helper( - tasks_to_schedule_ | boost::adaptors::transformed(transform_func), false); - auto tasks_to_dispatch_range = - tasks_to_dispatch_ | boost::adaptors::transformed([](const auto &pair) { + leases_to_schedule_ | boost::adaptors::transformed(transform_func), false); + auto leases_to_grant_range = + leases_to_grant_ | boost::adaptors::transformed([](const auto &pair) { auto cnt = pair.second.size(); - // We should only report dispatching tasks that do not have resources allocated. - for (const auto &task : pair.second) { - if (task->allocated_instances_) { + // We should only report leases to be granted that do not have resources + // allocated. + for (const auto &lease : pair.second) { + if (lease->allocated_instances_) { cnt--; } } return std::make_pair(pair.first, cnt); }); - fill_resource_usage_helper(tasks_to_dispatch_range, false); + fill_resource_usage_helper(leases_to_grant_range, false); fill_resource_usage_helper( - infeasible_tasks_ | boost::adaptors::transformed(transform_func), true); + infeasible_leases_ | boost::adaptors::transformed(transform_func), true); auto backlog_tracker_range = backlog_tracker_ | boost::adaptors::transformed([](const auto &pair) { return std::make_pair(pair.first, 0); @@ -169,10 +170,10 @@ void SchedulerResourceReporter::FillResourceUsage(rpc::ResourcesData &data) cons void SchedulerResourceReporter::FillPendingActorCountByShape( rpc::ResourcesData &data) const { absl::flat_hash_map> pending_count_by_shape; - for (const auto &[scheduling_class, queue] : infeasible_tasks_) { + for (const auto &[scheduling_class, queue] : infeasible_leases_) { pending_count_by_shape[scheduling_class].first = queue.size(); } - for (const auto &[scheduling_class, queue] : tasks_to_schedule_) { + for (const auto &[scheduling_class, queue] : leases_to_schedule_) { pending_count_by_shape[scheduling_class].second = queue.size(); } diff --git a/src/ray/raylet/scheduling/scheduler_resource_reporter.h b/src/ray/raylet/scheduling/scheduler_resource_reporter.h index 29c3f9818910..3357bd484a3f 100644 --- a/src/ray/raylet/scheduling/scheduler_resource_reporter.h +++ b/src/ray/raylet/scheduling/scheduler_resource_reporter.h @@ -20,7 +20,7 @@ #include "ray/common/ray_config.h" #include "ray/common/task/task_spec.h" #include "ray/raylet/scheduling/internal.h" -#include "ray/raylet/scheduling/local_task_manager_interface.h" +#include "ray/raylet/scheduling/local_lease_manager_interface.h" namespace ray { namespace raylet { @@ -31,11 +31,11 @@ class SchedulerResourceReporter { SchedulerResourceReporter( const absl::flat_hash_map>> - &tasks_to_schedule, + &leases_to_schedule, const absl::flat_hash_map>> - &infeasible_tasks, - const ILocalTaskManager &local_task_manager); + &infeasible_leases, + const LocalLeaseManagerInterface &local_lease_manager); /// Populate the relevant parts of the heartbeat table. This is intended for /// sending resource usage of raylet to gcs. In particular, this should fill in @@ -56,13 +56,13 @@ class SchedulerResourceReporter { const int64_t max_resource_shapes_per_load_report_; const absl::flat_hash_map>> - &tasks_to_schedule_; + &leases_to_schedule_; const absl::flat_hash_map>> - &tasks_to_dispatch_; + &leases_to_grant_; const absl::flat_hash_map>> - &infeasible_tasks_; + &infeasible_leases_; const absl::flat_hash_map> &backlog_tracker_; diff --git a/src/ray/raylet/scheduling/scheduler_stats.cc b/src/ray/raylet/scheduling/scheduler_stats.cc index 4b62d8e5fae4..0534c80dafd6 100644 --- a/src/ray/raylet/scheduling/scheduler_stats.cc +++ b/src/ray/raylet/scheduling/scheduler_stats.cc @@ -18,16 +18,16 @@ #include #include -#include "ray/raylet/scheduling/cluster_task_manager.h" +#include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/stats/metric_defs.h" namespace ray { namespace raylet { -SchedulerStats::SchedulerStats(const ClusterTaskManager &cluster_task_manager, - const ILocalTaskManager &local_task_manager) - : cluster_task_manager_(cluster_task_manager), - local_task_manager_(local_task_manager) {} +SchedulerStats::SchedulerStats(const ClusterLeaseManager &cluster_lease_manager, + const LocalLeaseManagerInterface &local_lease_manager) + : cluster_lease_manager_(cluster_lease_manager), + local_lease_manager_(local_lease_manager) {} void SchedulerStats::ComputeStats() { auto accumulator = @@ -41,11 +41,11 @@ void SchedulerStats::ComputeStats() { size_t num_worker_not_started_by_job_config_not_exist = 0; size_t num_worker_not_started_by_registration_timeout = 0; size_t num_tasks_waiting_for_workers = 0; - size_t num_cancelled_tasks = 0; + size_t num_cancelled_leases = 0; - size_t num_infeasible_tasks = - std::accumulate(cluster_task_manager_.infeasible_tasks_.begin(), - cluster_task_manager_.infeasible_tasks_.end(), + size_t num_infeasible_leases = + std::accumulate(cluster_lease_manager_.infeasible_leases_.begin(), + cluster_lease_manager_.infeasible_leases_.end(), static_cast(0), accumulator); @@ -58,7 +58,7 @@ void SchedulerStats::ComputeStats() { &num_worker_not_started_by_job_config_not_exist, &num_worker_not_started_by_registration_timeout, &num_tasks_waiting_for_workers, - &num_cancelled_tasks]( + &num_cancelled_leases]( size_t state, const std::pair< int, @@ -70,7 +70,7 @@ void SchedulerStats::ComputeStats() { if (work->GetState() == internal::WorkStatus::WAITING_FOR_WORKER) { num_tasks_waiting_for_workers += 1; } else if (work->GetState() == internal::WorkStatus::CANCELLED) { - num_cancelled_tasks += 1; + num_cancelled_leases += 1; } else if (work->GetUnscheduledCause() == internal::UnscheduledWorkCause::WAITING_FOR_RESOURCE_ACQUISITION) { num_waiting_for_resource += 1; @@ -90,14 +90,14 @@ void SchedulerStats::ComputeStats() { } return state + pair.second.size(); }; - size_t num_tasks_to_schedule = - std::accumulate(cluster_task_manager_.tasks_to_schedule_.begin(), - cluster_task_manager_.tasks_to_schedule_.end(), + size_t num_leases_to_schedule = + std::accumulate(cluster_lease_manager_.leases_to_schedule_.begin(), + cluster_lease_manager_.leases_to_schedule_.end(), static_cast(0), per_work_accumulator); - size_t num_tasks_to_dispatch = - std::accumulate(local_task_manager_.GetTaskToDispatch().begin(), - local_task_manager_.GetTaskToDispatch().end(), + size_t num_leases_to_grant = + std::accumulate(local_lease_manager_.GetLeasesToGrant().begin(), + local_lease_manager_.GetLeasesToGrant().end(), static_cast(0), per_work_accumulator); @@ -110,21 +110,21 @@ void SchedulerStats::ComputeStats() { num_worker_not_started_by_registration_timeout_ = num_worker_not_started_by_registration_timeout; num_tasks_waiting_for_workers_ = num_tasks_waiting_for_workers; - num_cancelled_tasks_ = num_cancelled_tasks; - num_infeasible_tasks_ = num_infeasible_tasks; - num_tasks_to_schedule_ = num_tasks_to_schedule; - num_tasks_to_dispatch_ = num_tasks_to_dispatch; + num_cancelled_leases_ = num_cancelled_leases; + num_infeasible_leases_ = num_infeasible_leases; + num_leases_to_schedule_ = num_leases_to_schedule; + num_leases_to_grant_ = num_leases_to_grant; } void SchedulerStats::RecordMetrics() { /// This method intentionally doesn't call ComputeStats() because /// that function is expensive. ComputeStats is called by ComputeAndReportDebugStr /// method and they are always periodically called by node manager. - ray_metric_num_spilled_tasks_.Record(metric_tasks_spilled_ + - local_task_manager_.GetNumTaskSpilled()); - local_task_manager_.RecordMetrics(); + ray_metric_num_spilled_tasks_.Record(metric_leases_spilled_ + + local_lease_manager_.GetNumLeaseSpilled()); + local_lease_manager_.RecordMetrics(); ray_metric_num_infeasible_scheduling_classes_.Record( - cluster_task_manager_.infeasible_tasks_.size()); + cluster_lease_manager_.infeasible_leases_.size()); /// Worker startup failure ray::stats::STATS_scheduler_failed_worker_startup_total.Record( num_worker_not_started_by_job_config_not_exist_, "JobConfigMissing"); @@ -134,16 +134,16 @@ void SchedulerStats::RecordMetrics() { num_worker_not_started_by_process_rate_limit_, "RateLimited"); /// Queued tasks. - ray::stats::STATS_scheduler_tasks.Record(num_cancelled_tasks_, "Cancelled"); - ray::stats::STATS_scheduler_tasks.Record(num_tasks_to_dispatch_, "Dispatched"); - ray::stats::STATS_scheduler_tasks.Record(num_tasks_to_schedule_, "Received"); - ray::stats::STATS_scheduler_tasks.Record(local_task_manager_.GetNumWaitingTaskSpilled(), - "SpilledWaiting"); + ray::stats::STATS_scheduler_tasks.Record(num_cancelled_leases_, "Cancelled"); + ray::stats::STATS_scheduler_tasks.Record(num_leases_to_grant_, "Dispatched"); + ray::stats::STATS_scheduler_tasks.Record(num_leases_to_schedule_, "Received"); ray::stats::STATS_scheduler_tasks.Record( - local_task_manager_.GetNumUnschedulableTaskSpilled(), "SpilledUnschedulable"); + local_lease_manager_.GetNumWaitingLeaseSpilled(), "SpilledWaiting"); + ray::stats::STATS_scheduler_tasks.Record( + local_lease_manager_.GetNumUnschedulableLeaseSpilled(), "SpilledUnschedulable"); /// Pending task count. - ray::stats::STATS_scheduler_unscheduleable_tasks.Record(num_infeasible_tasks_, + ray::stats::STATS_scheduler_unscheduleable_tasks.Record(num_infeasible_leases_, "Infeasible"); ray::stats::STATS_scheduler_unscheduleable_tasks.Record(num_waiting_for_resource_, "WaitingForResources"); @@ -157,17 +157,17 @@ void SchedulerStats::RecordMetrics() { std::string SchedulerStats::ComputeAndReportDebugStr() { ComputeStats(); - if (num_tasks_to_schedule_ + num_tasks_to_dispatch_ + num_infeasible_tasks_ > 1000) { + if (num_leases_to_schedule_ + num_leases_to_grant_ + num_infeasible_leases_ > 1000) { RAY_LOG(WARNING) << "More than 1000 tasks are queued for scheduling on this node. " "This can slow down the raylet."; } std::stringstream buffer; - buffer << "========== Node: " << cluster_task_manager_.self_node_id_ + buffer << "========== Node: " << cluster_lease_manager_.self_node_id_ << " =================\n"; - buffer << "Infeasible queue length: " << num_infeasible_tasks_ << "\n"; - buffer << "Schedule queue length: " << num_tasks_to_schedule_ << "\n"; - buffer << "Dispatch queue length: " << num_tasks_to_dispatch_ << "\n"; + buffer << "Infeasible queue length: " << num_infeasible_leases_ << "\n"; + buffer << "Schedule queue length: " << num_leases_to_schedule_ << "\n"; + buffer << "Grant queue length: " << num_leases_to_grant_ << "\n"; buffer << "num_waiting_for_resource: " << num_waiting_for_resource_ << "\n"; buffer << "num_waiting_for_plasma_memory: " << num_waiting_for_plasma_memory_ << "\n"; buffer << "num_waiting_for_remote_node_resources: " @@ -177,16 +177,16 @@ std::string SchedulerStats::ComputeAndReportDebugStr() { buffer << "num_worker_not_started_by_registration_timeout: " << num_worker_not_started_by_registration_timeout_ << "\n"; buffer << "num_tasks_waiting_for_workers: " << num_tasks_waiting_for_workers_ << "\n"; - buffer << "num_cancelled_tasks: " << num_cancelled_tasks_ << "\n"; + buffer << "num_cancelled_leases: " << num_cancelled_leases_ << "\n"; buffer << "cluster_resource_scheduler state: " - << cluster_task_manager_.cluster_resource_scheduler_.DebugString() << "\n"; - local_task_manager_.DebugStr(buffer); + << cluster_lease_manager_.cluster_resource_scheduler_.DebugString() << "\n"; + local_lease_manager_.DebugStr(buffer); buffer << "==================================================\n"; return buffer.str(); } -void SchedulerStats::TaskSpilled() { metric_tasks_spilled_++; } +void SchedulerStats::LeaseSpilled() { metric_leases_spilled_++; } } // namespace raylet } // namespace ray diff --git a/src/ray/raylet/scheduling/scheduler_stats.h b/src/ray/raylet/scheduling/scheduler_stats.h index 97dc0f9a1858..3974ba6f0a2e 100644 --- a/src/ray/raylet/scheduling/scheduler_stats.h +++ b/src/ray/raylet/scheduling/scheduler_stats.h @@ -20,19 +20,19 @@ #include "ray/common/ray_config.h" #include "ray/common/task/task_spec.h" #include "ray/raylet/scheduling/internal.h" -#include "ray/raylet/scheduling/local_task_manager_interface.h" +#include "ray/raylet/scheduling/local_lease_manager_interface.h" #include "ray/stats/metric.h" namespace ray { namespace raylet { -class ClusterTaskManager; +class ClusterLeaseManager; // Helper class that collects and reports scheduler's metrics into counters or human // readable string. class SchedulerStats { public: - explicit SchedulerStats(const ClusterTaskManager &cluster_task_manager, - const ILocalTaskManager &local_task_manager); + explicit SchedulerStats(const ClusterLeaseManager &cluster_lease_manager, + const LocalLeaseManagerInterface &local_lease_manager); // Report metrics doesn't recompute the stats. void RecordMetrics(); @@ -40,19 +40,19 @@ class SchedulerStats { // Recompute the stats and report the result as string. std::string ComputeAndReportDebugStr(); - // increase the task spilled counter. - void TaskSpilled(); + // increase the lease spilled counter. + void LeaseSpilled(); private: // recompute the metrics. void ComputeStats(); - const ClusterTaskManager &cluster_task_manager_; - const ILocalTaskManager &local_task_manager_; + const ClusterLeaseManager &cluster_lease_manager_; + const LocalLeaseManagerInterface &local_lease_manager_; /// Number of tasks that are spilled to other /// nodes because it cannot be scheduled locally. - int64_t metric_tasks_spilled_ = 0; + int64_t metric_leases_spilled_ = 0; /// Number of tasks that are waiting for /// resources to be available locally. int64_t num_waiting_for_resource_ = 0; @@ -71,14 +71,14 @@ class SchedulerStats { int64_t num_worker_not_started_by_process_rate_limit_ = 0; /// Number of tasks that are waiting for worker processes to start. int64_t num_tasks_waiting_for_workers_ = 0; - /// Number of cancelled tasks. - int64_t num_cancelled_tasks_ = 0; - /// Number of infeasible tasks. - int64_t num_infeasible_tasks_ = 0; - /// Number of tasks to schedule. - int64_t num_tasks_to_schedule_ = 0; - /// Number of tasks to dispatch. - int64_t num_tasks_to_dispatch_ = 0; + /// Number of cancelled leases. + int64_t num_cancelled_leases_ = 0; + /// Number of infeasible leases. + int64_t num_infeasible_leases_ = 0; + /// Number of leases to schedule. + int64_t num_leases_to_schedule_ = 0; + /// Number of leases to grant. + int64_t num_leases_to_grant_ = 0; /// Ray metrics ray::stats::Gauge ray_metric_num_spilled_tasks_{ diff --git a/src/ray/raylet/scheduling/tests/BUILD.bazel b/src/ray/raylet/scheduling/tests/BUILD.bazel index de54b3c34d61..661e52007f1f 100644 --- a/src/ray/raylet/scheduling/tests/BUILD.bazel +++ b/src/ray/raylet/scheduling/tests/BUILD.bazel @@ -9,8 +9,8 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", + "//src/ray/common:lease", "//src/ray/common:ray_config", - "//src/ray/common:task_common", "//src/ray/common:test_util", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/raylet/scheduling:cluster_resource_scheduler", @@ -47,20 +47,21 @@ ray_cc_test( ) ray_cc_test( - name = "cluster_task_manager_test", + name = "cluster_lease_manager_test", size = "small", srcs = [ - "cluster_task_manager_test.cc", + "cluster_lease_manager_test.cc", ], tags = ["team:core"], deps = [ "//:ray_mock", "//src/ray/common:id", + "//src/ray/common:lease", "//src/ray/common:task_common", "//src/ray/common:test_util", - "//src/ray/raylet:local_task_manager", + "//src/ray/raylet:local_lease_manager", + "//src/ray/raylet/scheduling:cluster_lease_manager", "//src/ray/raylet/scheduling:cluster_resource_scheduler", - "//src/ray/raylet/scheduling:cluster_task_manager", "//src/ray/raylet/tests:util", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/raylet/scheduling/tests/cluster_task_manager_test.cc b/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc similarity index 63% rename from src/ray/raylet/scheduling/tests/cluster_task_manager_test.cc rename to src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc index 9ce9ccaca635..8d6c3db5890f 100644 --- a/src/ray/raylet/scheduling/tests/cluster_task_manager_test.cc +++ b/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc @@ -14,7 +14,7 @@ // limitations under the License. // clang-format off -#include "ray/raylet/scheduling/cluster_task_manager.h" +#include "ray/raylet/scheduling/cluster_lease_manager.h" #include #include @@ -29,10 +29,10 @@ #include "ray/common/id.h" #include "ray/common/scheduling/resource_set.h" #include "ray/common/scheduling/scheduling_ids.h" -#include "ray/common/task/task.h" +#include "ray/common/lease/lease.h" #include "ray/common/task/task_util.h" #include "ray/common/test_util.h" -#include "ray/raylet/local_task_manager.h" +#include "ray/raylet/local_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/tests/util.h" #include "mock/ray/gcs/gcs_client/gcs_client.h" @@ -48,10 +48,10 @@ class MockWorkerPool : public WorkerPoolInterface { public: MockWorkerPool() : num_pops(0) {} - void PopWorker(const TaskSpecification &task_spec, + void PopWorker(const LeaseSpecification &lease_spec, const PopWorkerCallback &callback) override { num_pops++; - const int runtime_env_hash = task_spec.GetRuntimeEnvHash(); + const int runtime_env_hash = lease_spec.GetRuntimeEnvHash(); callbacks[runtime_env_hash].push_back(callback); } @@ -87,7 +87,7 @@ class MockWorkerPool : public WorkerPoolInterface { RAY_CHECK(status != PopWorkerStatus::OK); for (const auto &pair : callbacks) { for (const auto &callback : pair.second) { - // No task should be dispatched. + // No lease should be dispatched. ASSERT_FALSE( callback(nullptr, status, @@ -205,7 +205,7 @@ class MockWorkerPool : public WorkerPoolInterface { RAY_CHECK(false) << "Not used."; } - void PrestartWorkers(const TaskSpecification &task_spec, + void PrestartWorkers(const LeaseSpecification &lease_spec, int64_t backlog_size) override { RAY_CHECK(false) << "Not used."; } @@ -271,12 +271,13 @@ std::shared_ptr CreateSingleNodeScheduler( return scheduler; } -RayTask CreateTask( +RayLease CreateLease( const std::unordered_map &required_resources, int num_args = 0, std::vector args = {}, const std::shared_ptr runtime_env_info = nullptr, - rpc::SchedulingStrategy scheduling_strategy = rpc::SchedulingStrategy()) { + rpc::SchedulingStrategy scheduling_strategy = rpc::SchedulingStrategy(), + const LeaseID &lease_id = LeaseID::FromRandom()) { TaskSpecBuilder spec_builder; TaskID id = RandomTaskId(); JobID job_id = RandomJobId(); @@ -317,19 +318,21 @@ RayTask CreateTask( } spec_builder.SetNormalTaskSpec(0, false, "", scheduling_strategy, ActorID::Nil()); - - return RayTask(std::move(spec_builder).ConsumeAndBuild()); + TaskSpecification spec = std::move(spec_builder).ConsumeAndBuild(); + LeaseSpecification lease_spec(spec.GetMessage()); + lease_spec.GetMutableMessage().set_lease_id(lease_id.Binary()); + return RayLease(std::move(lease_spec)); } -class MockTaskDependencyManager : public TaskDependencyManagerInterface { +class MockLeaseDependencyManager : public LeaseDependencyManagerInterface { public: - explicit MockTaskDependencyManager(std::unordered_set &missing_objects) + explicit MockLeaseDependencyManager(std::unordered_set &missing_objects) : missing_objects_(missing_objects) {} - bool RequestTaskDependencies(const TaskID &task_id, - const std::vector &required_objects, - const TaskMetricsKey &task_key) { - RAY_CHECK(subscribed_tasks.insert(task_id).second); + bool RequestLeaseDependencies(const LeaseID &lease_id, + const std::vector &required_objects, + const TaskMetricsKey &task_key) { + RAY_CHECK(subscribed_leases.insert(lease_id).second); for (auto &obj_ref : required_objects) { if (missing_objects_.find(ObjectRefToId(obj_ref)) != missing_objects_.end()) { return false; @@ -338,19 +341,19 @@ class MockTaskDependencyManager : public TaskDependencyManagerInterface { return true; } - void RemoveTaskDependencies(const TaskID &task_id) { - RAY_CHECK(subscribed_tasks.erase(task_id)); + void RemoveLeaseDependencies(const LeaseID &lease_id) { + RAY_CHECK(subscribed_leases.erase(lease_id)); } - bool TaskDependenciesBlocked(const TaskID &task_id) const { - return blocked_tasks.count(task_id); + bool LeaseDependenciesBlocked(const LeaseID &lease_id) const { + return blocked_leases.count(lease_id); } bool CheckObjectLocal(const ObjectID &object_id) const { return true; } std::unordered_set &missing_objects_; - std::unordered_set subscribed_tasks; - std::unordered_set blocked_tasks; + std::unordered_set subscribed_leases; + std::unordered_set blocked_leases; }; class FeatureFlagEnvironment : public ::testing::Environment { @@ -369,19 +372,19 @@ class FeatureFlagEnvironment : public ::testing::Environment { testing::Environment *const env = ::testing::AddGlobalTestEnvironment(new FeatureFlagEnvironment); -class ClusterTaskManagerTest : public ::testing::Test { +class ClusterLeaseManagerTest : public ::testing::Test { public: - explicit ClusterTaskManagerTest(double num_cpus_at_head = 8.0, - double num_gpus_at_head = 0.0) + explicit ClusterLeaseManagerTest(double num_cpus_at_head = 8.0, + double num_gpus_at_head = 0.0) : gcs_client_(std::make_unique()), id_(NodeID::FromRandom()), scheduler_(CreateSingleNodeScheduler( id_.Binary(), num_cpus_at_head, num_gpus_at_head, *gcs_client_)), - dependency_manager_(missing_objects_), - local_task_manager_(std::make_unique( + lease_dependency_manager_(missing_objects_), + local_lease_manager_(std::make_unique( id_, *scheduler_, - dependency_manager_, + lease_dependency_manager_, /* get_node_info= */ [this](const NodeID &node_id) -> const rpc::GcsNodeInfo * { node_info_calls_++; @@ -392,7 +395,7 @@ class ClusterTaskManagerTest : public ::testing::Test { }, pool_, leased_workers_, - /* get_task_arguments= */ + /* get_lease_args= */ [this](const std::vector &object_ids, std::vector> *results) { for (auto &obj_id : object_ids) { @@ -404,9 +407,9 @@ class ClusterTaskManagerTest : public ::testing::Test { } return true; }, - /*max_pinned_task_arguments_bytes=*/1000, + /*max_pinned_lease_args_bytes=*/1000, /*get_time=*/[this]() { return current_time_ms_; })), - task_manager_( + lease_manager_( id_, *scheduler_, /* get_node_info= */ @@ -417,9 +420,9 @@ class ClusterTaskManagerTest : public ::testing::Test { } return nullptr; }, - /* announce_infeasible_task= */ - [this](const RayTask &task) { announce_infeasible_task_calls_++; }, - *local_task_manager_, + /* announce_infeasible_lease= */ + [this](const RayLease &lease) { announce_infeasible_lease_calls_++; }, + *local_lease_manager_, /*get_time=*/[this]() { return current_time_ms_; }) { RayConfig::instance().initialize("{\"scheduler_top_k_absolute\": 1}"); } @@ -455,31 +458,31 @@ class ClusterTaskManagerTest : public ::testing::Test { } void AssertNoLeaks() { - ASSERT_TRUE(task_manager_.tasks_to_schedule_.empty()); - ASSERT_TRUE(local_task_manager_->tasks_to_dispatch_.empty()); - ASSERT_TRUE(local_task_manager_->waiting_tasks_index_.empty()); - ASSERT_TRUE(local_task_manager_->waiting_task_queue_.empty()); - ASSERT_TRUE(task_manager_.infeasible_tasks_.empty()); - ASSERT_TRUE(local_task_manager_->executing_task_args_.empty()); - ASSERT_TRUE(local_task_manager_->pinned_task_arguments_.empty()); - ASSERT_TRUE(local_task_manager_->info_by_sched_cls_.empty()); - ASSERT_EQ(local_task_manager_->pinned_task_arguments_bytes_, 0); - ASSERT_TRUE(dependency_manager_.subscribed_tasks.empty()); - } - - void AssertPinnedTaskArgumentsPresent(const RayTask &task) { - const auto &expected_deps = task.GetTaskSpecification().GetDependencyIds(); - ASSERT_EQ( - local_task_manager_->executing_task_args_[task.GetTaskSpecification().TaskId()], - expected_deps); + ASSERT_TRUE(lease_manager_.leases_to_schedule_.empty()); + ASSERT_TRUE(local_lease_manager_->leases_to_grant_.empty()); + ASSERT_TRUE(local_lease_manager_->waiting_leases_index_.empty()); + ASSERT_TRUE(local_lease_manager_->waiting_lease_queue_.empty()); + ASSERT_TRUE(lease_manager_.infeasible_leases_.empty()); + ASSERT_TRUE(local_lease_manager_->granted_lease_args_.empty()); + ASSERT_TRUE(local_lease_manager_->pinned_lease_arguments_.empty()); + ASSERT_TRUE(local_lease_manager_->info_by_sched_cls_.empty()); + ASSERT_EQ(local_lease_manager_->pinned_lease_arguments_bytes_, 0); + ASSERT_TRUE(lease_dependency_manager_.subscribed_leases.empty()); + } + + void AssertPinnedLeaseArgumentsPresent(const RayLease &lease) { + const auto &expected_deps = lease.GetLeaseSpecification().GetDependencyIds(); + ASSERT_EQ(local_lease_manager_ + ->granted_lease_args_[lease.GetLeaseSpecification().LeaseId()], + expected_deps); for (auto &arg : expected_deps) { - ASSERT_TRUE(local_task_manager_->pinned_task_arguments_.count(arg)); + ASSERT_TRUE(local_lease_manager_->pinned_lease_arguments_.count(arg)); } } - int NumTasksToDispatchWithStatus(internal::WorkStatus status) { + int NumLeasesToDispatchWithStatus(internal::WorkStatus status) { int count = 0; - for (const auto &pair : local_task_manager_->tasks_to_dispatch_) { + for (const auto &pair : local_lease_manager_->leases_to_grant_) { for (const auto &work : pair.second) { if (work->GetState() == status) { count++; @@ -489,10 +492,10 @@ class ClusterTaskManagerTest : public ::testing::Test { return count; } - int NumRunningTasks() { + int NumRunningLeases() { int count = 0; - for (const auto &pair : local_task_manager_->info_by_sched_cls_) { - count += (pair.second.running_tasks.size()); + for (const auto &pair : local_lease_manager_->info_by_sched_cls_) { + count += (pair.second.granted_leases.size()); } return count; @@ -508,36 +511,36 @@ class ClusterTaskManagerTest : public ::testing::Test { int default_arg_size_ = 10; int node_info_calls_ = 0; - int announce_infeasible_task_calls_ = 0; + int announce_infeasible_lease_calls_ = 0; absl::flat_hash_map node_info_; int64_t current_time_ms_ = 0; - MockTaskDependencyManager dependency_manager_; - std::unique_ptr local_task_manager_; - ClusterTaskManager task_manager_; + MockLeaseDependencyManager lease_dependency_manager_; + std::unique_ptr local_lease_manager_; + ClusterLeaseManager lease_manager_; }; -// Same as ClusterTaskManagerTest, but the head node starts with 4.0 num gpus. -class ClusterTaskManagerTestWithGPUsAtHead : public ClusterTaskManagerTest { +// Same as ClusterLeaseManagerTest, but the head node starts with 4.0 num gpus. +class ClusterLeaseManagerTestWithGPUsAtHead : public ClusterLeaseManagerTest { public: - ClusterTaskManagerTestWithGPUsAtHead() - : ClusterTaskManagerTest(/*num_cpus_at_head=*/8.0, /*num_gpus_at_head=*/4.0) {} + ClusterLeaseManagerTestWithGPUsAtHead() + : ClusterLeaseManagerTest(/*num_cpus_at_head=*/8.0, /*num_gpus_at_head=*/4.0) {} }; -// Same as ClusterTaskManagerTest, but the head node starts with 0.0 num cpus. -class ClusterTaskManagerTestWithoutCPUsAtHead : public ClusterTaskManagerTest { +// Same as ClusterLeaseManagerTest, but the head node starts with 0.0 num cpus. +class ClusterLeaseManagerTestWithoutCPUsAtHead : public ClusterLeaseManagerTest { public: - ClusterTaskManagerTestWithoutCPUsAtHead() - : ClusterTaskManagerTest(/*num_cpus_at_head=*/0.0) {} + ClusterLeaseManagerTestWithoutCPUsAtHead() + : ClusterLeaseManagerTest(/*num_cpus_at_head=*/0.0) {} }; -TEST_F(ClusterTaskManagerTest, BasicTest) { +TEST_F(ClusterLeaseManagerTest, BasicTest) { /* Test basic scheduler functionality: 1. Queue and attempt to schedule/dispatch atest with no workers available 2. A worker becomes available, dispatch again. */ - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 4}}); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 4}}); rpc::RequestWorkerLeaseReply reply; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -546,7 +549,7 @@ TEST_F(ClusterTaskManagerTest, BasicTest) { *callback_occurred_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_FALSE(callback_occurred); ASSERT_EQ(leased_workers_.size(), 0); @@ -562,21 +565,21 @@ TEST_F(ClusterTaskManagerTest, BasicTest) { ASSERT_EQ(pool_.workers.size(), 0); ASSERT_EQ(node_info_calls_, 0); - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); - ASSERT_EQ(finished_task.GetTaskSpecification().TaskId(), - task.GetTaskSpecification().TaskId()); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); + ASSERT_EQ(finished_lease.GetLeaseSpecification().LeaseId(), + lease.GetLeaseSpecification().LeaseId()); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, IdempotencyTest) { +TEST_F(ClusterLeaseManagerTest, IdempotencyTest) { /* - A few task manager methods are meant to be idempotent. - * `TaskFinished` + A few lease manager methods are meant to be idempotent. + * `CleanupLease` * `ReleaseCpuResourcesFromBlockedWorker` * `ReturnCpuResourcesToUnblockedWorker` */ - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 4}}); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 4}}); rpc::RequestWorkerLeaseReply reply; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -585,7 +588,7 @@ TEST_F(ClusterTaskManagerTest, IdempotencyTest) { *callback_occurred_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_FALSE(callback_occurred); ASSERT_EQ(leased_workers_.size(), 0); @@ -603,30 +606,30 @@ TEST_F(ClusterTaskManagerTest, IdempotencyTest) { ASSERT_EQ(scheduler_->GetLocalResourceManager().GetLocalAvailableCpus(), 4.0); - local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker); - local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker); + local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker); + local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker); ASSERT_EQ(scheduler_->GetLocalResourceManager().GetLocalAvailableCpus(), 8.0); - local_task_manager_->ReturnCpuResourcesToUnblockedWorker(worker); - local_task_manager_->ReturnCpuResourcesToUnblockedWorker(worker); + local_lease_manager_->ReturnCpuResourcesToUnblockedWorker(worker); + local_lease_manager_->ReturnCpuResourcesToUnblockedWorker(worker); ASSERT_EQ(scheduler_->GetLocalResourceManager().GetLocalAvailableCpus(), 4.0); - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); - ASSERT_EQ(finished_task.GetTaskSpecification().TaskId(), - task.GetTaskSpecification().TaskId()); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); + ASSERT_EQ(finished_lease.GetLeaseSpecification().LeaseId(), + lease.GetLeaseSpecification().LeaseId()); ASSERT_EQ(scheduler_->GetLocalResourceManager().GetLocalAvailableCpus(), 8.0); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, DispatchQueueNonBlockingTest) { +TEST_F(ClusterLeaseManagerTest, GrantQueueNonBlockingTest) { /* - Test that if no worker is available for the first task in a dispatch - queue (because the runtime env in the task spec doesn't match any - available worker), other tasks in the dispatch queue can still be scheduled. + Test that if no worker is available for the first lease in a leases to grant + queue (because the runtime env in the lease spec doesn't match any + available worker), other leases in the grant queue can still be scheduled. https://github.com/ray-project/ray/issues/16226 */ @@ -639,8 +642,8 @@ TEST_F(ClusterTaskManagerTest, DispatchQueueNonBlockingTest) { runtime_env_info_A.reset(new rpc::RuntimeEnvInfo()); runtime_env_info_A->set_serialized_runtime_env(serialized_runtime_env_A); - RayTask task_A = - CreateTask(required_resources, /*num_args=*/0, /*args=*/{}, runtime_env_info_A); + RayLease lease_A = + CreateLease(required_resources, /*num_args=*/0, /*args=*/{}, runtime_env_info_A); rpc::RequestWorkerLeaseReply reply_A; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -654,18 +657,20 @@ TEST_F(ClusterTaskManagerTest, DispatchQueueNonBlockingTest) { runtime_env_info_B.reset(new rpc::RuntimeEnvInfo()); runtime_env_info_B->set_serialized_runtime_env(serialized_runtime_env_B); - RayTask task_B_1 = - CreateTask(required_resources, /*num_args=*/0, /*args=*/{}, runtime_env_info_B); - RayTask task_B_2 = - CreateTask(required_resources, /*num_args=*/0, /*args=*/{}, runtime_env_info_B); + RayLease lease_B_1 = + CreateLease(required_resources, /*num_args=*/0, /*args=*/{}, runtime_env_info_B); + RayLease lease_B_2 = + CreateLease(required_resources, /*num_args=*/0, /*args=*/{}, runtime_env_info_B); rpc::RequestWorkerLeaseReply reply_B_1; rpc::RequestWorkerLeaseReply reply_B_2; auto empty_callback = [](Status, std::function, std::function) {}; // Ensure task_A is not at the front of the queue. - task_manager_.QueueAndScheduleTask(task_B_1, false, false, &reply_B_1, empty_callback); - task_manager_.QueueAndScheduleTask(task_A, false, false, &reply_A, callback); - task_manager_.QueueAndScheduleTask(task_B_2, false, false, &reply_B_2, empty_callback); + lease_manager_.QueueAndScheduleLease( + lease_B_1, false, false, &reply_B_1, empty_callback); + lease_manager_.QueueAndScheduleLease(lease_A, false, false, &reply_A, callback); + lease_manager_.QueueAndScheduleLease( + lease_B_2, false, false, &reply_B_2, empty_callback); pool_.TriggerCallbacks(); // Push a worker that can only run task A. @@ -679,16 +684,16 @@ TEST_F(ClusterTaskManagerTest, DispatchQueueNonBlockingTest) { ASSERT_EQ(pool_.workers.size(), 0); ASSERT_EQ(node_info_calls_, 0); - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); - ASSERT_EQ(finished_task.GetTaskSpecification().TaskId(), - task_A.GetTaskSpecification().TaskId()); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); + ASSERT_EQ(finished_lease.GetLeaseSpecification().LeaseId(), + lease_A.GetLeaseSpecification().LeaseId()); // task_B_1 and task_B_2 remain in the dispatch queue, so don't call AssertNoLeaks(). // AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, BlockedWorkerDiesTest) { +TEST_F(ClusterLeaseManagerTest, BlockedWorkerDiesTest) { /* Tests the edge case in which a worker crashes while it's blocked. In this case, its CPU resources should not be double freed. @@ -700,9 +705,23 @@ TEST_F(ClusterTaskManagerTest, BlockedWorkerDiesTest) { scheduler_->GetLocalResourceManager().AddLocalResourceInstances( scheduling::ResourceID("CPU_group_0_aaa"), std::vector{FixedPoint(1)}); - RayTask task1 = CreateTask({{ray::kCPU_ResourceLabel, 4}}); + WorkerID worker_id1 = WorkerID::FromRandom(); + WorkerID worker_id2 = WorkerID::FromRandom(); + LeaseID lease_id1 = LeaseID::FromWorker(worker_id1, 1); + LeaseID lease_id2 = LeaseID::FromWorker(worker_id2, 1); + RayLease lease1 = CreateLease({{ray::kCPU_ResourceLabel, 4}}, + 0, + {}, + nullptr, + rpc::SchedulingStrategy(), + lease_id1); rpc::RequestWorkerLeaseReply reply1; - RayTask task2 = CreateTask({{"CPU_group_aaa", 1}, {"CPU_group_0_aaa", 1}}); + RayLease lease2 = CreateLease({{"CPU_group_aaa", 1}, {"CPU_group_0_aaa", 1}}, + 0, + {}, + nullptr, + rpc::SchedulingStrategy(), + lease_id2); rpc::RequestWorkerLeaseReply reply2; bool callback_occurred = false; @@ -712,25 +731,23 @@ TEST_F(ClusterTaskManagerTest, BlockedWorkerDiesTest) { *callback_occurred_ptr = true; }; - task_manager_.QueueAndScheduleTask(task1, false, false, &reply1, callback); + lease_manager_.QueueAndScheduleLease(lease1, false, false, &reply1, callback); pool_.TriggerCallbacks(); ASSERT_FALSE(callback_occurred); ASSERT_EQ(leased_workers_.size(), 0); ASSERT_EQ(pool_.workers.size(), 0); - std::shared_ptr worker1 = - std::make_shared(WorkerID::FromRandom(), 1234); - std::shared_ptr worker2 = - std::make_shared(WorkerID::FromRandom(), 5678); + std::shared_ptr worker1 = std::make_shared(worker_id1, 1234); + std::shared_ptr worker2 = std::make_shared(worker_id2, 5678); pool_.PushWorker(std::static_pointer_cast(worker1)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply2, callback); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply2, callback); pool_.PushWorker(std::static_pointer_cast(worker2)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_TRUE(callback_occurred); @@ -739,30 +756,28 @@ TEST_F(ClusterTaskManagerTest, BlockedWorkerDiesTest) { ASSERT_EQ(node_info_calls_, 0); // Block the worker. Which releases only the CPU resource. - local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1); - local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker2); + local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1); + local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker2); - RayTask finished_task1; - RayTask finished_task2; + RayLease finished_lease1; + RayLease finished_lease2; // If a resource was double-freed, we will crash in this call. - local_task_manager_->TaskFinished(leased_workers_[worker1->WorkerId()], - &finished_task1); - local_task_manager_->TaskFinished(leased_workers_[worker2->WorkerId()], - &finished_task2); - ASSERT_EQ(finished_task1.GetTaskSpecification().TaskId(), - task1.GetTaskSpecification().TaskId()); - ASSERT_EQ(finished_task2.GetTaskSpecification().TaskId(), - task2.GetTaskSpecification().TaskId()); + local_lease_manager_->CleanupLease(leased_workers_[worker_id1], &finished_lease1); + local_lease_manager_->CleanupLease(leased_workers_[worker_id2], &finished_lease2); + ASSERT_EQ(finished_lease1.GetLeaseSpecification().LeaseId(), + lease1.GetLeaseSpecification().LeaseId()); + ASSERT_EQ(finished_lease2.GetLeaseSpecification().LeaseId(), + lease2.GetLeaseSpecification().LeaseId()); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, BlockedWorkerDies2Test) { +TEST_F(ClusterLeaseManagerTest, BlockedWorkerDies2Test) { /* Same edge case as the previous test, but this time the block and finish requests happen in the opposite order. */ - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 4}}); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 4}}); rpc::RequestWorkerLeaseReply reply; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -771,7 +786,7 @@ TEST_F(ClusterTaskManagerTest, BlockedWorkerDies2Test) { *callback_occurred_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_FALSE(callback_occurred); @@ -782,7 +797,7 @@ TEST_F(ClusterTaskManagerTest, BlockedWorkerDies2Test) { std::make_shared(WorkerID::FromRandom(), 1234); pool_.PushWorker(std::static_pointer_cast(worker)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_TRUE(callback_occurred); @@ -790,23 +805,23 @@ TEST_F(ClusterTaskManagerTest, BlockedWorkerDies2Test) { ASSERT_EQ(pool_.workers.size(), 0); ASSERT_EQ(node_info_calls_, 0); - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); - ASSERT_EQ(finished_task.GetTaskSpecification().TaskId(), - task.GetTaskSpecification().TaskId()); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); + ASSERT_EQ(finished_lease.GetLeaseSpecification().LeaseId(), + lease.GetLeaseSpecification().LeaseId()); // Block the worker. Which releases only the CPU resource. - local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker); + local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, NoFeasibleNodeTest) { +TEST_F(ClusterLeaseManagerTest, NoFeasibleNodeTest) { std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); pool_.PushWorker(std::dynamic_pointer_cast(worker)); - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 999}}); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 999}}); rpc::RequestWorkerLeaseReply reply; bool callback_called = false; @@ -816,7 +831,7 @@ TEST_F(ClusterTaskManagerTest, NoFeasibleNodeTest) { *callback_called_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_FALSE(callback_called); @@ -826,13 +841,13 @@ TEST_F(ClusterTaskManagerTest, NoFeasibleNodeTest) { ASSERT_EQ(node_info_calls_, 0); } -TEST_F(ClusterTaskManagerTest, DrainingWhileResolving) { +TEST_F(ClusterLeaseManagerTest, DrainingWhileResolving) { /* - Test the race condition in which a task is assigned to a node, but cannot + Test the race condition in which a lease is assigned to a node, but cannot run because its dependencies are unresolved. Once its dependencies are resolved, the node is being drained. */ - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 1}}); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply reply; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -840,7 +855,7 @@ TEST_F(ClusterTaskManagerTest, DrainingWhileResolving) { Status, std::function, std::function) { *callback_occurred_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); std::shared_ptr worker2 = @@ -855,12 +870,12 @@ TEST_F(ClusterTaskManagerTest, DrainingWhileResolving) { auto remote_node_id = NodeID::FromRandom(); AddNode(remote_node_id, 5); - RayTask resolving_args_task = CreateTask({{ray::kCPU_ResourceLabel, 1}}, 1); - auto missing_arg = resolving_args_task.GetTaskSpecification().GetDependencyIds()[0]; + RayLease resolving_args_lease = CreateLease({{ray::kCPU_ResourceLabel, 1}}, 1); + auto missing_arg = resolving_args_lease.GetLeaseSpecification().GetDependencyIds()[0]; missing_objects_.insert(missing_arg); rpc::RequestWorkerLeaseReply spillback_reply; - task_manager_.QueueAndScheduleTask( - resolving_args_task, false, false, &spillback_reply, callback); + lease_manager_.QueueAndScheduleLease( + resolving_args_lease, false, false, &spillback_reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); @@ -872,14 +887,15 @@ TEST_F(ClusterTaskManagerTest, DrainingWhileResolving) { // Arg is resolved. missing_objects_.erase(missing_arg); - std::vector unblocked = {resolving_args_task.GetTaskSpecification().TaskId()}; - local_task_manager_->TasksUnblocked(unblocked); + std::vector unblocked = { + resolving_args_lease.GetLeaseSpecification().LeaseId()}; + local_lease_manager_->LeasesUnblocked(unblocked); ASSERT_EQ(spillback_reply.retry_at_raylet_address().node_id(), remote_node_id.Binary()); } -TEST_F(ClusterTaskManagerTest, ResourceTakenWhileResolving) { +TEST_F(ClusterLeaseManagerTest, ResourceTakenWhileResolving) { /* - Test the race condition in which a task is assigned to a node, but cannot + Test the race condition in which a lease is assigned to a node, but cannot run because its dependencies are unresolved. Once its dependencies are resolved, the node no longer has available resources. */ @@ -899,14 +915,14 @@ TEST_F(ClusterTaskManagerTest, ResourceTakenWhileResolving) { }; /* Blocked on dependencies */ - auto task = CreateTask({{ray::kCPU_ResourceLabel, 5}}, 2); - auto missing_arg = task.GetTaskSpecification().GetDependencyIds()[0]; + auto lease = CreateLease({{ray::kCPU_ResourceLabel, 5}}, 2); + auto missing_arg = lease.GetLeaseSpecification().GetDependencyIds()[0]; missing_objects_.insert(missing_arg); - std::unordered_set expected_subscribed_tasks = { - task.GetTaskSpecification().TaskId()}; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + std::unordered_set expected_subscribed_leases = { + lease.GetLeaseSpecification().LeaseId()}; + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); - ASSERT_EQ(dependency_manager_.subscribed_tasks, expected_subscribed_tasks); + ASSERT_EQ(lease_dependency_manager_.subscribed_leases, expected_subscribed_leases); ASSERT_EQ(num_callbacks, 0); ASSERT_EQ(leased_workers_.size(), 0); @@ -915,52 +931,52 @@ TEST_F(ClusterTaskManagerTest, ResourceTakenWhileResolving) { // https://github.com/ray-project/ray/issues/13725. ASSERT_EQ(pool_.num_pops, 0); - /* This task can run */ - auto task2 = CreateTask({{ray::kCPU_ResourceLabel, 5}}, 1); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply, callback); + /* This lease can run */ + auto lease2 = CreateLease({{ray::kCPU_ResourceLabel, 5}}, 1); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply, callback); pool_.TriggerCallbacks(); - ASSERT_EQ(dependency_manager_.subscribed_tasks, expected_subscribed_tasks); + ASSERT_EQ(lease_dependency_manager_.subscribed_leases, expected_subscribed_leases); - AssertPinnedTaskArgumentsPresent(task2); + AssertPinnedLeaseArgumentsPresent(lease2); ASSERT_EQ(num_callbacks, 1); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); ASSERT_EQ(pool_.num_pops, 1); - /* First task is unblocked now, but resources are no longer available */ + /* First lease is unblocked now, but resources are no longer available */ missing_objects_.erase(missing_arg); - auto id = task.GetTaskSpecification().TaskId(); - std::vector unblocked = {id}; - local_task_manager_->TasksUnblocked(unblocked); - ASSERT_EQ(dependency_manager_.subscribed_tasks, expected_subscribed_tasks); + auto id = lease.GetLeaseSpecification().LeaseId(); + std::vector unblocked = {id}; + local_lease_manager_->LeasesUnblocked(unblocked); + ASSERT_EQ(lease_dependency_manager_.subscribed_leases, expected_subscribed_leases); - AssertPinnedTaskArgumentsPresent(task2); + AssertPinnedLeaseArgumentsPresent(lease2); ASSERT_EQ(num_callbacks, 1); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); ASSERT_EQ(pool_.num_pops, 1); - /* Second task finishes, making space for the original task */ - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); + /* Second lease finishes, making space for the original lease */ + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); leased_workers_.clear(); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - ASSERT_TRUE(dependency_manager_.subscribed_tasks.empty()); + ASSERT_TRUE(lease_dependency_manager_.subscribed_leases.empty()); - // Task2 is now done so task can run. - AssertPinnedTaskArgumentsPresent(task); + // Lease2 is now done so lease can run. + AssertPinnedLeaseArgumentsPresent(lease); ASSERT_EQ(num_callbacks, 2); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 0); ASSERT_EQ(pool_.num_pops, 2); - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, TestIsSelectedBasedOnLocality) { +TEST_F(ClusterLeaseManagerTest, TestIsSelectedBasedOnLocality) { std::shared_ptr worker1 = std::make_shared(WorkerID::FromRandom(), 1234); std::shared_ptr worker2 = @@ -976,45 +992,45 @@ TEST_F(ClusterTaskManagerTest, TestIsSelectedBasedOnLocality) { auto remote_node_id = NodeID::FromRandom(); AddNode(remote_node_id, 8); - auto task1 = CreateTask({{ray::kCPU_ResourceLabel, 5}}); + auto lease1 = CreateLease({{ray::kCPU_ResourceLabel, 5}}); rpc::RequestWorkerLeaseReply local_reply; - task_manager_.QueueAndScheduleTask( - task1, false, /*is_selected_based_on_locality=*/false, &local_reply, callback); + lease_manager_.QueueAndScheduleLease( + lease1, false, /*is_selected_based_on_locality=*/false, &local_reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 1); - // The first task was dispatched. + // The first lease was dispatched. ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); - auto task2 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); + auto lease2 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply spillback_reply; - task_manager_.QueueAndScheduleTask( - task2, false, /*is_selected_based_on_locality=*/false, &spillback_reply, callback); + lease_manager_.QueueAndScheduleLease( + lease2, false, /*is_selected_based_on_locality=*/false, &spillback_reply, callback); pool_.TriggerCallbacks(); - // The second task was spilled. + // The second lease was spilled. ASSERT_EQ(num_callbacks, 2); ASSERT_EQ(spillback_reply.retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); - auto task3 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); - task_manager_.QueueAndScheduleTask( - task3, false, /*is_selected_based_on_locality=*/true, &local_reply, callback); + auto lease3 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); + lease_manager_.QueueAndScheduleLease( + lease3, false, /*is_selected_based_on_locality=*/true, &local_reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 3); - // The third task was dispatched. + // The third lease was dispatched. ASSERT_EQ(leased_workers_.size(), 2); ASSERT_EQ(pool_.workers.size(), 0); while (!leased_workers_.empty()) { - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); leased_workers_.erase(leased_workers_.begin()); } AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, TestGrantOrReject) { +TEST_F(ClusterLeaseManagerTest, TestGrantOrReject) { std::shared_ptr worker1 = std::make_shared(WorkerID::FromRandom(), 1234); std::shared_ptr worker2 = @@ -1030,49 +1046,49 @@ TEST_F(ClusterTaskManagerTest, TestGrantOrReject) { auto remote_node_id = NodeID::FromRandom(); AddNode(remote_node_id, 8); - auto task1 = CreateTask({{ray::kCPU_ResourceLabel, 5}}); + auto lease1 = CreateLease({{ray::kCPU_ResourceLabel, 5}}); rpc::RequestWorkerLeaseReply local_reply; - task_manager_.QueueAndScheduleTask( - task1, /*grant_or_reject=*/false, false, &local_reply, callback); + lease_manager_.QueueAndScheduleLease( + lease1, /*grant_or_reject=*/false, false, &local_reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 1); - // The first task was dispatched. + // The first lease was dispatched. ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); - auto task2 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); + auto lease2 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply spillback_reply; - task_manager_.QueueAndScheduleTask( - task2, /*grant_or_reject=*/false, false, &spillback_reply, callback); + lease_manager_.QueueAndScheduleLease( + lease2, /*grant_or_reject=*/false, false, &spillback_reply, callback); pool_.TriggerCallbacks(); - // The second task was spilled. + // The second lease was spilled. ASSERT_EQ(num_callbacks, 2); ASSERT_EQ(spillback_reply.retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); - auto task3 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); - task_manager_.QueueAndScheduleTask( - task3, /*grant_or_reject=*/true, false, &local_reply, callback); + auto lease3 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); + lease_manager_.QueueAndScheduleLease( + lease3, /*grant_or_reject=*/true, false, &local_reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 3); - // The third task was dispatched. + // The third lease was dispatched. ASSERT_EQ(leased_workers_.size(), 2); ASSERT_EQ(pool_.workers.size(), 0); while (!leased_workers_.empty()) { - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); leased_workers_.erase(leased_workers_.begin()); } AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, TestSpillAfterAssigned) { +TEST_F(ClusterLeaseManagerTest, TestSpillAfterAssigned) { /* - Test the race condition in which a task is assigned to the local node, but - it cannot be run because a different task gets assigned the resources - first. The un-runnable task should eventually get spilled back to another + Test the race condition in which a lease is assigned to the local node, but + it cannot be run because a different lease gets assigned the resources + first. The un-runnable lease should eventually get spilled back to another node. */ std::shared_ptr worker = @@ -1086,59 +1102,59 @@ TEST_F(ClusterTaskManagerTest, TestSpillAfterAssigned) { }; /* Blocked on starting a worker. */ - auto task = CreateTask({{ray::kCPU_ResourceLabel, 5}}); + auto lease = CreateLease({{ray::kCPU_ResourceLabel, 5}}); rpc::RequestWorkerLeaseReply local_reply; - task_manager_.QueueAndScheduleTask(task, false, false, &local_reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &local_reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 0); ASSERT_EQ(leased_workers_.size(), 0); // Resources are no longer available for the second. - auto task2 = CreateTask({{ray::kCPU_ResourceLabel, 5}}); + auto lease2 = CreateLease({{ray::kCPU_ResourceLabel, 5}}); rpc::RequestWorkerLeaseReply reject_reply; - task_manager_.QueueAndScheduleTask( - task2, /*grant_or_reject=*/true, false, &reject_reply, callback); + lease_manager_.QueueAndScheduleLease( + lease2, /*grant_or_reject=*/true, false, &reject_reply, callback); pool_.TriggerCallbacks(); - // The second task was rejected. + // The second lease was rejected. ASSERT_EQ(num_callbacks, 1); ASSERT_TRUE(reject_reply.rejected()); ASSERT_EQ(leased_workers_.size(), 0); // Resources are no longer available for the third. - auto task3 = CreateTask({{ray::kCPU_ResourceLabel, 5}}); + auto lease3 = CreateLease({{ray::kCPU_ResourceLabel, 5}}); rpc::RequestWorkerLeaseReply spillback_reply; - task_manager_.QueueAndScheduleTask(task3, false, false, &spillback_reply, callback); + lease_manager_.QueueAndScheduleLease(lease3, false, false, &spillback_reply, callback); pool_.TriggerCallbacks(); - // The third task was spilled. + // The third lease was spilled. ASSERT_EQ(num_callbacks, 2); ASSERT_EQ(spillback_reply.retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_EQ(leased_workers_.size(), 0); - // Two workers start. First task was dispatched now. + // Two workers start. First lease was dispatched now. pool_.PushWorker(std::static_pointer_cast(worker)); pool_.PushWorker(std::static_pointer_cast(worker)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - // Check that all tasks got removed from the queue. + // Check that all leases got removed from the queue. ASSERT_EQ(num_callbacks, 3); - // The first task was dispatched. + // The first lease was dispatched. ASSERT_EQ(leased_workers_.size(), 1); // Leave one alive worker. ASSERT_EQ(pool_.workers.size(), 1); - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); - ASSERT_EQ(finished_task.GetTaskSpecification().TaskId(), - task.GetTaskSpecification().TaskId()); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); + ASSERT_EQ(finished_lease.GetLeaseSpecification().LeaseId(), + lease.GetLeaseSpecification().LeaseId()); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, TestIdleNode) { - RayTask task = CreateTask({{}}); +TEST_F(ClusterLeaseManagerTest, TestIdleNode) { + RayLease lease = CreateLease({{}}); rpc::RequestWorkerLeaseReply reply; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -1147,7 +1163,7 @@ TEST_F(ClusterTaskManagerTest, TestIdleNode) { *callback_occurred_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_TRUE(scheduler_->GetLocalResourceManager().IsLocalNodeIdle()); ASSERT_FALSE(callback_occurred); @@ -1164,7 +1180,7 @@ TEST_F(ClusterTaskManagerTest, TestIdleNode) { ASSERT_EQ(node_info_calls_, 0); } -TEST_F(ClusterTaskManagerTest, NotOKPopWorkerAfterDrainingTest) { +TEST_F(ClusterLeaseManagerTest, NotOKPopWorkerAfterDrainingTest) { /* Test cases where the node is being drained after PopWorker is called and PopWorker fails. @@ -1180,8 +1196,8 @@ TEST_F(ClusterTaskManagerTest, NotOKPopWorkerAfterDrainingTest) { task_allocation); } - RayTask task1 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); - RayTask task2 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); + RayLease lease1 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); + RayLease lease2 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply reply1; rpc::RequestWorkerLeaseReply reply2; bool callback_called = false; @@ -1190,8 +1206,8 @@ TEST_F(ClusterTaskManagerTest, NotOKPopWorkerAfterDrainingTest) { Status, std::function, std::function) { *callback_called_ptr = true; }; - task_manager_.QueueAndScheduleTask(task1, false, false, &reply1, callback); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply2, callback); + lease_manager_.QueueAndScheduleLease(lease1, false, false, &reply1, callback); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply2, callback); auto remote_node_id = NodeID::FromRandom(); AddNode(remote_node_id, 5); @@ -1201,20 +1217,20 @@ TEST_F(ClusterTaskManagerTest, NotOKPopWorkerAfterDrainingTest) { drain_request.set_deadline_timestamp_ms(std::numeric_limits::max()); scheduler_->GetLocalResourceManager().SetLocalNodeDraining(drain_request); - pool_.callbacks[task1.GetTaskSpecification().GetRuntimeEnvHash()].front()( + pool_.callbacks[lease1.GetLeaseSpecification().GetRuntimeEnvHash()].front()( nullptr, PopWorkerStatus::WorkerPendingRegistration, ""); - pool_.callbacks[task1.GetTaskSpecification().GetRuntimeEnvHash()].back()( + pool_.callbacks[lease1.GetLeaseSpecification().GetRuntimeEnvHash()].back()( nullptr, PopWorkerStatus::RuntimeEnvCreationFailed, "runtime env setup error"); pool_.callbacks.clear(); - task_manager_.ScheduleAndDispatchTasks(); - // task1 is spilled and task2 is cancelled. + lease_manager_.ScheduleAndGrantLeases(); + // lease1 is spilled and lease2 is cancelled. ASSERT_EQ(reply1.retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_TRUE(reply2.canceled()); ASSERT_EQ(reply2.scheduling_failure_message(), "runtime env setup error"); } -TEST_F(ClusterTaskManagerTest, NotOKPopWorkerTest) { - RayTask task1 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); +TEST_F(ClusterLeaseManagerTest, NotOKPopWorkerTest) { + RayLease lease1 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply reply; bool callback_called = false; bool *callback_called_ptr = &callback_called; @@ -1222,61 +1238,61 @@ TEST_F(ClusterTaskManagerTest, NotOKPopWorkerTest) { Status, std::function, std::function) { *callback_called_ptr = true; }; - task_manager_.QueueAndScheduleTask(task1, false, false, &reply, callback); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 1); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING), 0); - ASSERT_EQ(NumRunningTasks(), 1); + lease_manager_.QueueAndScheduleLease(lease1, false, false, &reply, callback); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 1); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING), 0); + ASSERT_EQ(NumRunningLeases(), 1); pool_.TriggerCallbacksWithNotOKStatus(PopWorkerStatus::WorkerPendingRegistration); ASSERT_FALSE(callback_called); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 0); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING), 1); - ASSERT_EQ(NumRunningTasks(), 0); - ASSERT_TRUE(task_manager_.CancelTask(task1.GetTaskSpecification().TaskId())); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 0); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING), 1); + ASSERT_EQ(NumRunningLeases(), 0); + ASSERT_TRUE(lease_manager_.CancelLease(lease1.GetLeaseSpecification().LeaseId())); callback_called = false; reply.Clear(); - RayTask task2 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply, callback); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 1); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING), 0); - ASSERT_EQ(NumRunningTasks(), 1); - // The task should be cancelled. + RayLease lease2 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply, callback); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 1); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING), 0); + ASSERT_EQ(NumRunningLeases(), 1); + // The lease should be cancelled. const auto runtime_env_error_msg = "Runtime env error message"; pool_.TriggerCallbacksWithNotOKStatus(PopWorkerStatus::RuntimeEnvCreationFailed, runtime_env_error_msg); ASSERT_TRUE(callback_called); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 0); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING), 0); - ASSERT_EQ(NumRunningTasks(), 0); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 0); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING), 0); + ASSERT_EQ(NumRunningLeases(), 0); ASSERT_TRUE(reply.canceled()); ASSERT_EQ(reply.scheduling_failure_message(), runtime_env_error_msg); - // Test that local task manager handles PopWorkerStatus::JobFinished correctly. + // Test that local lease manager handles PopWorkerStatus::JobFinished correctly. callback_called = false; reply.Clear(); - RayTask task3 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); - task_manager_.QueueAndScheduleTask(task3, false, false, &reply, callback); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 1); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING), 0); - ASSERT_EQ(NumRunningTasks(), 1); + RayLease lease3 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); + lease_manager_.QueueAndScheduleLease(lease3, false, false, &reply, callback); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 1); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING), 0); + ASSERT_EQ(NumRunningLeases(), 1); pool_.TriggerCallbacksWithNotOKStatus(PopWorkerStatus::JobFinished); - // The task should be removed from the dispatch queue. + // The lease should be removed from the leases_to_grant queue. ASSERT_FALSE(callback_called); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 0); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING), 0); - ASSERT_EQ(NumRunningTasks(), 0); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 0); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING), 0); + ASSERT_EQ(NumRunningLeases(), 0); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, TaskUnschedulableTest) { - TaskSpecification task_spec = - CreateTask({{ray::kCPU_ResourceLabel, 1}}).GetTaskSpecification(); - task_spec.GetMutableMessage() +TEST_F(ClusterLeaseManagerTest, TaskUnschedulableTest) { + LeaseSpecification lease_spec = + CreateLease({{ray::kCPU_ResourceLabel, 1}}).GetLeaseSpecification(); + lease_spec.GetMutableMessage() .mutable_scheduling_strategy() ->mutable_node_affinity_scheduling_strategy() ->set_node_id(NodeID::FromRandom().Binary()); - task_spec.GetMutableMessage() + lease_spec.GetMutableMessage() .mutable_scheduling_strategy() ->mutable_node_affinity_scheduling_strategy() ->set_soft(false); @@ -1289,7 +1305,8 @@ TEST_F(ClusterTaskManagerTest, TaskUnschedulableTest) { *callback_called_ptr = true; }; - task_manager_.QueueAndScheduleTask(RayTask(task_spec), false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease( + RayLease(lease_spec), false, false, &reply, callback); ASSERT_TRUE(callback_called); ASSERT_TRUE(reply.canceled()); ASSERT_EQ(reply.failure_type(), @@ -1298,10 +1315,10 @@ TEST_F(ClusterTaskManagerTest, TaskUnschedulableTest) { AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, TaskCancellationTest) { +TEST_F(ClusterLeaseManagerTest, TaskCancellationTest) { std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); - RayTask task1 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); + RayLease lease1 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply reply; bool callback_called = false; @@ -1311,52 +1328,52 @@ TEST_F(ClusterTaskManagerTest, TaskCancellationTest) { *callback_called_ptr = true; }; - // Task1 not queued so we can't cancel it. - ASSERT_FALSE(task_manager_.CancelTask(task1.GetTaskSpecification().TaskId())); + // Lease1 not queued so we can't cancel it. + ASSERT_FALSE(lease_manager_.CancelLease(lease1.GetLeaseSpecification().LeaseId())); - task_manager_.QueueAndScheduleTask(task1, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease1, false, false, &reply, callback); pool_.TriggerCallbacks(); - // Task1 is now in dispatch queue. + // Lease1 is now in dispatch queue. callback_called = false; reply.Clear(); - ASSERT_TRUE(task_manager_.CancelTask(task1.GetTaskSpecification().TaskId())); + ASSERT_TRUE(lease_manager_.CancelLease(lease1.GetLeaseSpecification().LeaseId())); pool_.PushWorker(std::static_pointer_cast(worker)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - // Task1 will not execute. + // Lease1 will not be granted. ASSERT_TRUE(callback_called); ASSERT_TRUE(reply.canceled()); ASSERT_EQ(leased_workers_.size(), 0); - RayTask task2 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply, callback); + RayLease lease2 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply, callback); pool_.TriggerCallbacks(); - // Task2 is now running so we can't cancel it. + // Lease2 is now granted so we can't cancel it. callback_called = false; reply.Clear(); - ASSERT_FALSE(task_manager_.CancelTask(task2.GetTaskSpecification().TaskId())); + ASSERT_FALSE(lease_manager_.CancelLease(lease2.GetLeaseSpecification().LeaseId())); ASSERT_FALSE(reply.canceled()); ASSERT_FALSE(callback_called); ASSERT_EQ(pool_.workers.size(), 0); ASSERT_EQ(leased_workers_.size(), 1); - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); - ASSERT_EQ(finished_task.GetTaskSpecification().TaskId(), - task2.GetTaskSpecification().TaskId()); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); + ASSERT_EQ(finished_lease.GetLeaseSpecification().LeaseId(), + lease2.GetLeaseSpecification().LeaseId()); - RayTask task3 = CreateTask({{ray::kCPU_ResourceLabel, 2}}); + RayLease lease3 = CreateLease({{ray::kCPU_ResourceLabel, 2}}); rpc::RequestWorkerLeaseReply reply3; - RayTask task4 = CreateTask({{ray::kCPU_ResourceLabel, 200}}); + RayLease lease4 = CreateLease({{ray::kCPU_ResourceLabel, 200}}); rpc::RequestWorkerLeaseReply reply4; - // Task 3 should be popping worker - task_manager_.QueueAndScheduleTask(task3, false, false, &reply3, callback); - // Task 4 is infeasible - task_manager_.QueueAndScheduleTask(task4, false, false, &reply4, callback); + // Lease 3 should be popping worker + lease_manager_.QueueAndScheduleLease(lease3, false, false, &reply3, callback); + // Lease 4 is infeasible + lease_manager_.QueueAndScheduleLease(lease4, false, false, &reply4, callback); pool_.TriggerCallbacks(); - ASSERT_TRUE(task_manager_.CancelTasks( + ASSERT_TRUE(lease_manager_.CancelLeases( [](const std::shared_ptr &work) { return true; }, rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_INTENDED, "")); @@ -1366,13 +1383,13 @@ TEST_F(ClusterTaskManagerTest, TaskCancellationTest) { AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, TaskCancelInfeasibleTask) { - /* Make sure cancelTask works for infeasible tasks */ +TEST_F(ClusterLeaseManagerTest, TaskCancelInfeasibleTask) { + /* Make sure cancelLease works for infeasible leases */ std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); pool_.PushWorker(std::static_pointer_cast(worker)); - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 12}}); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 12}}); rpc::RequestWorkerLeaseReply reply; bool callback_called = false; @@ -1382,24 +1399,24 @@ TEST_F(ClusterTaskManagerTest, TaskCancelInfeasibleTask) { *callback_called_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); - // RayTask is now queued so cancellation works. - ASSERT_TRUE(task_manager_.CancelTask(task.GetTaskSpecification().TaskId())); - task_manager_.ScheduleAndDispatchTasks(); + // RayLease is now queued so cancellation works. + ASSERT_TRUE(lease_manager_.CancelLease(lease.GetLeaseSpecification().LeaseId())); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - // Task will not execute. + // Lease will not be granted. ASSERT_TRUE(callback_called); ASSERT_TRUE(reply.canceled()); ASSERT_EQ(leased_workers_.size(), 0); ASSERT_EQ(pool_.workers.size(), 1); - // Although the feasible node is added, task shouldn't be executed because it is + // Although the feasible node is added, lease shouldn't be granted because it is // cancelled. auto remote_node_id = NodeID::FromRandom(); AddNode(remote_node_id, 12); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_TRUE(callback_called); ASSERT_TRUE(reply.canceled()); @@ -1408,13 +1425,13 @@ TEST_F(ClusterTaskManagerTest, TaskCancelInfeasibleTask) { AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, TaskCancelWithResourceShape) { - // task1 doesn't match the resource shape so shouldn't be cancelled - // task2 matches the resource shape and should be cancelled +TEST_F(ClusterLeaseManagerTest, TaskCancelWithResourceShape) { + // lease1 doesn't match the resource shape so shouldn't be cancelled + // lease2 matches the resource shape and should be cancelled std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); - RayTask task1 = CreateTask({{ray::kCPU_ResourceLabel, 1}}); - RayTask task2 = CreateTask({{ray::kCPU_ResourceLabel, 10}}); + RayLease lease1 = CreateLease({{ray::kCPU_ResourceLabel, 1}}); + RayLease lease2 = CreateLease({{ray::kCPU_ResourceLabel, 10}}); absl::flat_hash_map resource_shape_1 = { {ray::kCPU_ResourceLabel, 10}}; absl::flat_hash_map resource_shape_2 = { @@ -1437,42 +1454,42 @@ TEST_F(ClusterTaskManagerTest, TaskCancelWithResourceShape) { *callback_called_ptr_2 = true; }; - task_manager_.QueueAndScheduleTask(task1, false, false, &reply1, callback1); + lease_manager_.QueueAndScheduleLease(lease1, false, false, &reply1, callback1); pool_.TriggerCallbacks(); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply2, callback2); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply2, callback2); pool_.TriggerCallbacks(); callback_called_1 = false; callback_called_2 = false; reply1.Clear(); reply2.Clear(); - ASSERT_TRUE(task_manager_.CancelTasksWithResourceShapes(target_resource_shapes)); + ASSERT_TRUE(lease_manager_.CancelLeasesWithResourceShapes(target_resource_shapes)); ASSERT_FALSE(reply1.canceled()); ASSERT_FALSE(callback_called_1); ASSERT_TRUE(reply2.canceled()); ASSERT_TRUE(callback_called_2); pool_.PushWorker(std::static_pointer_cast(worker)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_EQ(pool_.workers.size(), 0); ASSERT_EQ(leased_workers_.size(), 1); - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); - ASSERT_EQ(finished_task.GetTaskSpecification().TaskId(), - task1.GetTaskSpecification().TaskId()); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); + ASSERT_EQ(finished_lease.GetLeaseSpecification().LeaseId(), + lease1.GetLeaseSpecification().LeaseId()); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, HeartbeatTest) { +TEST_F(ClusterLeaseManagerTest, HeartbeatTest) { std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); pool_.PushWorker(std::static_pointer_cast(worker)); { - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 1}}); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply reply; bool callback_called = false; @@ -1482,14 +1499,14 @@ TEST_F(ClusterTaskManagerTest, HeartbeatTest) { *callback_called_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_TRUE(callback_called); // Now {CPU: 7, GPU: 4, MEM:128} } { - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 1}}); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply reply; bool callback_called = false; @@ -1499,15 +1516,15 @@ TEST_F(ClusterTaskManagerTest, HeartbeatTest) { *callback_called_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_FALSE(callback_called); // No worker available. - // Now {CPU: 7, GPU: 4, MEM:128} with 1 queued task. + // Now {CPU: 7, GPU: 4, MEM:128} with 1 queued lease. } { - RayTask task = - CreateTask({{ray::kCPU_ResourceLabel, 9}, {ray::kGPU_ResourceLabel, 5}}); + RayLease lease = + CreateLease({{ray::kCPU_ResourceLabel, 9}, {ray::kGPU_ResourceLabel, 5}}); rpc::RequestWorkerLeaseReply reply; bool callback_called = false; @@ -1517,15 +1534,15 @@ TEST_F(ClusterTaskManagerTest, HeartbeatTest) { *callback_called_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_FALSE(callback_called); // Infeasible. - // Now there is also an infeasible task {CPU: 9}. + // Now there is also an infeasible lease {CPU: 9}. } { - RayTask task = - CreateTask({{ray::kCPU_ResourceLabel, 10}, {ray::kGPU_ResourceLabel, 1}}); + RayLease lease = + CreateLease({{ray::kCPU_ResourceLabel, 10}, {ray::kGPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply reply; bool callback_called = false; @@ -1535,15 +1552,15 @@ TEST_F(ClusterTaskManagerTest, HeartbeatTest) { *callback_called_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_FALSE(callback_called); // Infeasible. - // Now there is also an infeasible task {CPU: 10}. + // Now there is also an infeasible lease {CPU: 10}. } { rpc::ResourcesData data; - task_manager_.FillResourceUsage(data); + lease_manager_.FillResourceUsage(data); auto load_by_shape = data.mutable_resource_load_by_shape()->mutable_resource_demands(); @@ -1585,7 +1602,7 @@ TEST_F(ClusterTaskManagerTest, HeartbeatTest) { } } -TEST_F(ClusterTaskManagerTest, ResourceReportForNodeAffinitySchedulingStrategyTasks) { +TEST_F(ClusterLeaseManagerTest, ResourceReportForNodeAffinitySchedulingStrategyTasks) { rpc::RequestWorkerLeaseReply reply; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -1594,39 +1611,39 @@ TEST_F(ClusterTaskManagerTest, ResourceReportForNodeAffinitySchedulingStrategyTa *callback_occurred_ptr = true; }; - // Feasible strict task won't be reported. + // Feasible strict lease won't be reported. rpc::SchedulingStrategy scheduling_strategy; scheduling_strategy.mutable_node_affinity_scheduling_strategy()->set_node_id( id_.Binary()); scheduling_strategy.mutable_node_affinity_scheduling_strategy()->set_soft(false); - RayTask task1 = - CreateTask({{ray::kCPU_ResourceLabel, 1}}, 0, {}, nullptr, scheduling_strategy); - task_manager_.QueueAndScheduleTask(task1, false, false, &reply, callback); + RayLease lease1 = + CreateLease({{ray::kCPU_ResourceLabel, 1}}, 0, {}, nullptr, scheduling_strategy); + lease_manager_.QueueAndScheduleLease(lease1, false, false, &reply, callback); - // Feasible soft task won't be reported. + // Feasible soft lease won't be reported. scheduling_strategy.mutable_node_affinity_scheduling_strategy()->set_node_id( id_.Binary()); scheduling_strategy.mutable_node_affinity_scheduling_strategy()->set_soft(true); - RayTask task2 = - CreateTask({{ray::kCPU_ResourceLabel, 2}}, 0, {}, nullptr, scheduling_strategy); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply, callback); + RayLease task2 = + CreateLease({{ray::kCPU_ResourceLabel, 2}}, 0, {}, nullptr, scheduling_strategy); + lease_manager_.QueueAndScheduleLease(task2, false, false, &reply, callback); - // Infeasible soft task will be reported. + // Infeasible soft lease will be reported. scheduling_strategy.mutable_node_affinity_scheduling_strategy()->set_node_id( id_.Binary()); scheduling_strategy.mutable_node_affinity_scheduling_strategy()->set_soft(true); - RayTask task3 = - CreateTask({{ray::kGPU_ResourceLabel, 1}}, 0, {}, nullptr, scheduling_strategy); - task_manager_.QueueAndScheduleTask(task3, false, false, &reply, callback); + RayLease task3 = + CreateLease({{ray::kGPU_ResourceLabel, 1}}, 0, {}, nullptr, scheduling_strategy); + lease_manager_.QueueAndScheduleLease(task3, false, false, &reply, callback); ASSERT_FALSE(callback_occurred); - // Infeasible strict task won't be reported (will fail immediately). + // Infeasible strict lease won't be reported (will fail immediately). scheduling_strategy.mutable_node_affinity_scheduling_strategy()->set_node_id( id_.Binary()); scheduling_strategy.mutable_node_affinity_scheduling_strategy()->set_soft(false); - RayTask task4 = - CreateTask({{ray::kGPU_ResourceLabel, 2}}, 0, {}, nullptr, scheduling_strategy); - task_manager_.QueueAndScheduleTask(task4, false, false, &reply, callback); + RayLease task4 = + CreateLease({{ray::kGPU_ResourceLabel, 2}}, 0, {}, nullptr, scheduling_strategy); + lease_manager_.QueueAndScheduleLease(task4, false, false, &reply, callback); ASSERT_TRUE(callback_occurred); ASSERT_TRUE(reply.canceled()); ASSERT_EQ(reply.failure_type(), @@ -1636,7 +1653,7 @@ TEST_F(ClusterTaskManagerTest, ResourceReportForNodeAffinitySchedulingStrategyTa ASSERT_EQ(pool_.workers.size(), 0); rpc::ResourcesData data; - task_manager_.FillResourceUsage(data); + lease_manager_.FillResourceUsage(data); auto resource_load_by_shape = data.resource_load_by_shape(); ASSERT_EQ(resource_load_by_shape.resource_demands().size(), 1); auto demand = resource_load_by_shape.resource_demands()[0]; @@ -1645,7 +1662,7 @@ TEST_F(ClusterTaskManagerTest, ResourceReportForNodeAffinitySchedulingStrategyTa ASSERT_EQ(demand.shape().at("GPU"), 1); } -TEST_F(ClusterTaskManagerTest, BacklogReportTest) { +TEST_F(ClusterLeaseManagerTest, BacklogReportTest) { /* Test basic scheduler functionality: 1. Queue and attempt to schedule/dispatch a test with no workers available @@ -1659,18 +1676,18 @@ TEST_F(ClusterTaskManagerTest, BacklogReportTest) { *callback_occurred_ptr = true; }; - std::vector to_cancel; + std::vector to_cancel; std::vector worker_ids; for (int i = 0; i < 10; i++) { - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 8}}); - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 8}}); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); worker_ids.push_back(WorkerID::FromRandom()); - local_task_manager_->SetWorkerBacklog( - task.GetTaskSpecification().GetSchedulingClass(), worker_ids.back(), 10 - i); + local_lease_manager_->SetWorkerBacklog( + lease.GetLeaseSpecification().GetSchedulingClass(), worker_ids.back(), 10 - i); pool_.TriggerCallbacks(); - // Don't add the fist task to `to_cancel`. + // Don't add the first lease to `to_cancel`. if (i != 0) { - to_cancel.push_back(task.GetTaskSpecification().TaskId()); + to_cancel.push_back(lease.GetLeaseSpecification().LeaseId()); } } @@ -1679,9 +1696,9 @@ TEST_F(ClusterTaskManagerTest, BacklogReportTest) { ASSERT_EQ(pool_.workers.size(), 0); ASSERT_EQ(node_info_calls_, 0); - { // 1 task has resources allocated, while remaining 9 is stuck. + { // 1 lease has resources allocated, while remaining 9 are stuck. rpc::ResourcesData data; - task_manager_.FillResourceUsage(data); + lease_manager_.FillResourceUsage(data); auto resource_load_by_shape = data.resource_load_by_shape(); auto shape1 = resource_load_by_shape.resource_demands()[0]; @@ -1690,17 +1707,17 @@ TEST_F(ClusterTaskManagerTest, BacklogReportTest) { ASSERT_EQ(shape1.num_ready_requests_queued(), 9); } - // Push a worker so the first task can run. + // Push a worker so the first lease can be granted. std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); pool_.PushWorker(worker); - task_manager_.ScheduleAndDispatchTasks(); - local_task_manager_->ClearWorkerBacklog(worker_ids[0]); + lease_manager_.ScheduleAndGrantLeases(); + local_lease_manager_->ClearWorkerBacklog(worker_ids[0]); pool_.TriggerCallbacks(); { rpc::ResourcesData data; - task_manager_.FillResourceUsage(data); + lease_manager_.FillResourceUsage(data); auto resource_load_by_shape = data.resource_load_by_shape(); auto shape1 = resource_load_by_shape.resource_demands()[0]; @@ -1711,32 +1728,33 @@ TEST_F(ClusterTaskManagerTest, BacklogReportTest) { } // Cancel the rest. - for (auto &task_id : to_cancel) { - ASSERT_TRUE(task_manager_.CancelTask(task_id)); + for (auto &lease_id : to_cancel) { + ASSERT_TRUE(lease_manager_.CancelLease(lease_id)); } for (size_t i = 1; i < worker_ids.size(); ++i) { - local_task_manager_->ClearWorkerBacklog(worker_ids[i]); + local_lease_manager_->ClearWorkerBacklog(worker_ids[i]); } { rpc::ResourcesData data; - task_manager_.FillResourceUsage(data); + lease_manager_.FillResourceUsage(data); auto resource_load_by_shape = data.resource_load_by_shape(); ASSERT_EQ(resource_load_by_shape.resource_demands().size(), 0); while (!leased_workers_.empty()) { - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, + &finished_lease); leased_workers_.erase(leased_workers_.begin()); } AssertNoLeaks(); } } -TEST_F(ClusterTaskManagerTest, OwnerDeadTest) { - // Test the case when the task owner (worker or node) dies, the task is cancelled. - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 4}}); +TEST_F(ClusterLeaseManagerTest, OwnerDeadTest) { + // Test the case when the lease owner (worker or node) dies, the lease is cancelled. + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 4}}); rpc::RequestWorkerLeaseReply reply; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -1745,64 +1763,64 @@ TEST_F(ClusterTaskManagerTest, OwnerDeadTest) { *callback_occurred_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_FALSE(callback_occurred); - task_manager_.CancelAllTasksOwnedBy(task.GetTaskSpecification().CallerWorkerId()); + lease_manager_.CancelAllLeasesOwnedBy(lease.GetLeaseSpecification().CallerWorkerId()); AssertNoLeaks(); callback_occurred = false; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_FALSE(callback_occurred); - task_manager_.CancelAllTasksOwnedBy(task.GetTaskSpecification().CallerNodeId()); + lease_manager_.CancelAllLeasesOwnedBy(lease.GetLeaseSpecification().CallerNodeId()); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, TestInfeasibleTaskWarning) { +TEST_F(ClusterLeaseManagerTest, TestInfeasibleLeaseWarning) { /* - Test if infeasible tasks warnings are printed. + Test if infeasible leases warnings are printed. */ - // Create an infeasible task. - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 12}}); + // Create an infeasible lease. + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 12}}); rpc::RequestWorkerLeaseReply reply; std::shared_ptr callback_occurred = std::make_shared(false); auto callback = [callback_occurred]( Status, std::function, std::function) { *callback_occurred = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); - ASSERT_EQ(announce_infeasible_task_calls_, 1); + ASSERT_EQ(announce_infeasible_lease_calls_, 1); - // Infeasible warning shouldn't be reprinted when the previous task is still infeasible + // Infeasible warning shouldn't be reprinted when the previous lease is still infeasible // after adding a new node. AddNode(NodeID::FromRandom(), 8); std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); pool_.PushWorker(std::static_pointer_cast(worker)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - // Task shouldn't be scheduled yet. - ASSERT_EQ(announce_infeasible_task_calls_, 1); + // Lease shouldn't be scheduled yet. + ASSERT_EQ(announce_infeasible_lease_calls_, 1); ASSERT_FALSE(*callback_occurred); ASSERT_EQ(leased_workers_.size(), 0); ASSERT_EQ(pool_.workers.size(), 1); - // Now we have a node that is feasible to schedule the task. Make sure the infeasible - // task is spillbacked properly. + // Now we have a node that is feasible to schedule the lease. Make sure the infeasible + // lease is spillbacked properly. auto remote_node_id = NodeID::FromRandom(); AddNode(remote_node_id, 12); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); // Make sure nothing happens locally. - ASSERT_EQ(announce_infeasible_task_calls_, 1); + ASSERT_EQ(announce_infeasible_lease_calls_, 1); ASSERT_TRUE(*callback_occurred); ASSERT_EQ(leased_workers_.size(), 0); ASSERT_EQ(pool_.workers.size(), 1); @@ -1811,90 +1829,90 @@ TEST_F(ClusterTaskManagerTest, TestInfeasibleTaskWarning) { AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, TestMultipleInfeasibleTasksWarnOnce) { +TEST_F(ClusterLeaseManagerTest, TestMultipleInfeasibleLeasesWarnOnce) { /* Test infeasible warning is printed only once when the same shape is queued again. */ - // Make sure the first infeasible task announces warning. - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 12}}); + // Make sure the first infeasible lease announces warning. + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 12}}); rpc::RequestWorkerLeaseReply reply; std::shared_ptr callback_occurred = std::make_shared(false); auto callback = [callback_occurred]( Status, std::function, std::function) { *callback_occurred = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); - ASSERT_EQ(announce_infeasible_task_calls_, 1); + ASSERT_EQ(announce_infeasible_lease_calls_, 1); - // Make sure the same shape infeasible task won't be announced. - RayTask task2 = CreateTask({{ray::kCPU_ResourceLabel, 12}}); + // Make sure the same shape infeasible lease won't be announced. + RayLease lease2 = CreateLease({{ray::kCPU_ResourceLabel, 12}}); rpc::RequestWorkerLeaseReply reply2; std::shared_ptr callback_occurred2 = std::make_shared(false); auto callback2 = [callback_occurred2]( Status, std::function, std::function) { *callback_occurred2 = true; }; - task_manager_.QueueAndScheduleTask(task2, false, false, &reply2, callback2); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply2, callback2); pool_.TriggerCallbacks(); - ASSERT_EQ(announce_infeasible_task_calls_, 1); + ASSERT_EQ(announce_infeasible_lease_calls_, 1); } -TEST_F(ClusterTaskManagerTest, TestAnyPendingTasksForResourceAcquisition) { +TEST_F(ClusterLeaseManagerTest, TestAnyPendingLeasesForResourceAcquisition) { /* - Check if the manager can correctly identify pending tasks. + Check if the manager can correctly identify pending leases. */ std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); pool_.PushWorker(std::static_pointer_cast(worker)); - // task1: running - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 6}}); + // lease1: running. + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 6}}); rpc::RequestWorkerLeaseReply reply; std::shared_ptr callback_occurred = std::make_shared(false); auto callback = [callback_occurred]( Status, std::function, std::function) { *callback_occurred = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_TRUE(*callback_occurred); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 0); - // task1: running. Progress is made, and there's no deadlock. - int pending_actor_creations = 0; - int pending_tasks = 0; - ASSERT_EQ(task_manager_.AnyPendingTasksForResourceAcquisition(&pending_actor_creations, - &pending_tasks), + // lease1: running. Progress is made, and there's no deadlock. + int pending_lease_creations = 0; + int pending_leases = 0; + ASSERT_EQ(lease_manager_.AnyPendingLeasesForResourceAcquisition( + &pending_lease_creations, &pending_leases), nullptr); - ASSERT_EQ(pending_actor_creations, 0); - ASSERT_EQ(pending_tasks, 0); + ASSERT_EQ(pending_lease_creations, 0); + ASSERT_EQ(pending_leases, 0); - // task1: running, task2: queued. - RayTask task2 = CreateTask({{ray::kCPU_ResourceLabel, 6}}); + // lease1: running, lease2: queued. + RayLease lease2 = CreateLease({{ray::kCPU_ResourceLabel, 6}}); rpc::RequestWorkerLeaseReply reply2; std::shared_ptr callback_occurred2 = std::make_shared(false); auto callback2 = [callback_occurred2]( Status, std::function, std::function) { *callback_occurred2 = true; }; - task_manager_.QueueAndScheduleTask(task2, false, false, &reply2, callback2); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply2, callback2); pool_.TriggerCallbacks(); ASSERT_FALSE(*callback_occurred2); - auto pending_task = task_manager_.AnyPendingTasksForResourceAcquisition( - &pending_actor_creations, &pending_tasks); - ASSERT_EQ(pending_task->GetTaskSpecification().TaskId(), - task2.GetTaskSpecification().TaskId()); - ASSERT_EQ(pending_actor_creations, 0); - ASSERT_EQ(pending_tasks, 1); + auto pending_lease = lease_manager_.AnyPendingLeasesForResourceAcquisition( + &pending_lease_creations, &pending_leases); + ASSERT_EQ(pending_lease->GetLeaseSpecification().LeaseId(), + lease2.GetLeaseSpecification().LeaseId()); + ASSERT_EQ(pending_lease_creations, 0); + ASSERT_EQ(pending_leases, 1); } -TEST_F(ClusterTaskManagerTest, ArgumentEvicted) { +TEST_F(ClusterLeaseManagerTest, ArgumentEvicted) { /* - Test the task's dependencies becoming local, then one of the arguments is - evicted. The task should go from waiting -> dispatch -> waiting. + Test the lease's dependencies becoming local, then one of the arguments is + evicted. The lease should go from waiting -> dispatch -> waiting. */ std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); @@ -1909,52 +1927,52 @@ TEST_F(ClusterTaskManagerTest, ArgumentEvicted) { }; /* Blocked on dependencies */ - auto task = CreateTask({{ray::kCPU_ResourceLabel, 5}}, 2); - auto missing_arg = task.GetTaskSpecification().GetDependencyIds()[0]; + auto lease = CreateLease({{ray::kCPU_ResourceLabel, 5}}, 2); + auto missing_arg = lease.GetLeaseSpecification().GetDependencyIds()[0]; missing_objects_.insert(missing_arg); - std::unordered_set expected_subscribed_tasks = { - task.GetTaskSpecification().TaskId()}; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + std::unordered_set expected_subscribed_leases = { + lease.GetLeaseSpecification().LeaseId()}; + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); - ASSERT_EQ(dependency_manager_.subscribed_tasks, expected_subscribed_tasks); + ASSERT_EQ(lease_dependency_manager_.subscribed_leases, expected_subscribed_leases); ASSERT_EQ(num_callbacks, 0); ASSERT_EQ(leased_workers_.size(), 0); - /* RayTask is unblocked now */ + /* RayLease is unblocked now */ missing_objects_.erase(missing_arg); pool_.workers.clear(); - auto id = task.GetTaskSpecification().TaskId(); - local_task_manager_->TasksUnblocked({id}); - ASSERT_EQ(dependency_manager_.subscribed_tasks, expected_subscribed_tasks); + auto id = lease.GetLeaseSpecification().LeaseId(); + local_lease_manager_->LeasesUnblocked({id}); + ASSERT_EQ(lease_dependency_manager_.subscribed_leases, expected_subscribed_leases); ASSERT_EQ(num_callbacks, 0); ASSERT_EQ(leased_workers_.size(), 0); /* Worker available and arguments available */ pool_.PushWorker(std::static_pointer_cast(worker)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 1); ASSERT_EQ(leased_workers_.size(), 1); - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); - ASSERT_EQ(finished_task.GetTaskSpecification().TaskId(), - task.GetTaskSpecification().TaskId()); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); + ASSERT_EQ(finished_lease.GetLeaseSpecification().LeaseId(), + lease.GetLeaseSpecification().LeaseId()); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, FeasibleToNonFeasible) { - // Test the case, when resources changes in local node, the feasible task should - // able to transfer to infeasible task +TEST_F(ClusterLeaseManagerTest, FeasibleToNonFeasible) { + // Test the case, when resources changes in local node, the feasible lease should + // able to transfer to infeasible lease std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); pool_.PushWorker(std::static_pointer_cast(worker)); - RayTask task1 = CreateTask({{ray::kCPU_ResourceLabel, 4}}); + RayLease lease1 = CreateLease({{ray::kCPU_ResourceLabel, 4}}); rpc::RequestWorkerLeaseReply reply1; bool callback_occurred1 = false; - task_manager_.QueueAndScheduleTask( - task1, + lease_manager_.QueueAndScheduleLease( + lease1, false, false, &reply1, @@ -1965,20 +1983,20 @@ TEST_F(ClusterTaskManagerTest, FeasibleToNonFeasible) { ASSERT_EQ(leased_workers_.size(), 1); ASSERT_TRUE(callback_occurred1); ASSERT_EQ(pool_.workers.size(), 0); - ASSERT_EQ(task_manager_.tasks_to_schedule_.size(), 0); - ASSERT_EQ(local_task_manager_->tasks_to_dispatch_.size(), 0); - ASSERT_EQ(task_manager_.infeasible_tasks_.size(), 0); + ASSERT_EQ(lease_manager_.leases_to_schedule_.size(), 0); + ASSERT_EQ(local_lease_manager_->leases_to_grant_.size(), 0); + ASSERT_EQ(lease_manager_.infeasible_leases_.size(), 0); - // Delete cpu resource of local node, then task 2 should be turned into + // Delete cpu resource of local node, then lease 2 should be turned into // infeasible. scheduler_->GetLocalResourceManager().DeleteLocalResource( scheduling::ResourceID(ray::kCPU_ResourceLabel)); - RayTask task2 = CreateTask({{ray::kCPU_ResourceLabel, 4}}); + RayLease lease2 = CreateLease({{ray::kCPU_ResourceLabel, 4}}); rpc::RequestWorkerLeaseReply reply2; bool callback_occurred2 = false; - task_manager_.QueueAndScheduleTask( - task2, + lease_manager_.QueueAndScheduleLease( + lease2, false, false, &reply2, @@ -1989,17 +2007,18 @@ TEST_F(ClusterTaskManagerTest, FeasibleToNonFeasible) { ASSERT_EQ(leased_workers_.size(), 1); ASSERT_FALSE(callback_occurred2); ASSERT_EQ(pool_.workers.size(), 0); - ASSERT_EQ(task_manager_.tasks_to_schedule_.size(), 0); - ASSERT_EQ(local_task_manager_->tasks_to_dispatch_.size(), 0); - ASSERT_EQ(task_manager_.infeasible_tasks_.size(), 1); - - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); - ASSERT_EQ(finished_task.GetTaskSpecification().TaskId(), - task1.GetTaskSpecification().TaskId()); + ASSERT_EQ(lease_manager_.leases_to_schedule_.size(), 0); + ASSERT_EQ(local_lease_manager_->leases_to_grant_.size(), 0); + ASSERT_EQ(local_lease_manager_->waiting_lease_queue_.size(), 0); + ASSERT_EQ(lease_manager_.infeasible_leases_.size(), 1); + + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); + ASSERT_EQ(finished_lease.GetLeaseSpecification().LeaseId(), + lease1.GetLeaseSpecification().LeaseId()); } -TEST_F(ClusterTaskManagerTest, NegativePlacementGroupCpuResources) { +TEST_F(ClusterLeaseManagerTest, NegativePlacementGroupCpuResources) { // Add PG CPU resources. scheduler_->GetLocalResourceManager().AddLocalResourceInstances( scheduling::ResourceID("CPU_group_aaa"), std::vector{FixedPoint(2)}); @@ -2018,7 +2037,7 @@ TEST_F(ClusterTaskManagerTest, NegativePlacementGroupCpuResources) { {{"CPU_group_aaa", 1.}, {"CPU_group_0_aaa", 1.}}, allocated_instances)); worker1->SetAllocatedInstances(allocated_instances); // worker1 calls ray.get() and release the CPU resource - ASSERT_TRUE(local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1)); + ASSERT_TRUE(local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1)); // the released CPU resource is acquired by worker2 auto worker2 = std::make_shared(WorkerID::FromRandom(), 5678); @@ -2028,7 +2047,7 @@ TEST_F(ClusterTaskManagerTest, NegativePlacementGroupCpuResources) { worker2->SetAllocatedInstances(allocated_instances); // ray.get() returns and worker1 acquires the CPU resource again - ASSERT_TRUE(local_task_manager_->ReturnCpuResourcesToUnblockedWorker(worker1)); + ASSERT_TRUE(local_lease_manager_->ReturnCpuResourcesToUnblockedWorker(worker1)); ASSERT_EQ(node_resources.available.Get(scheduling::ResourceID("CPU_group_aaa")), 0); ASSERT_EQ(node_resources.available.Get(scheduling::ResourceID("CPU_group_0_aaa")), -1); ASSERT_EQ(node_resources.available.Get(scheduling::ResourceID("CPU_group_1_aaa")), 1); @@ -2043,7 +2062,7 @@ TEST_F(ClusterTaskManagerTest, NegativePlacementGroupCpuResources) { ASSERT_EQ(node_resources.available.Get(scheduling::ResourceID("CPU_group_1_aaa")), 0); } -TEST_F(ClusterTaskManagerTestWithGPUsAtHead, ReleaseAndReturnWorkerCpuResources) { +TEST_F(ClusterLeaseManagerTestWithGPUsAtHead, ReleaseAndReturnWorkerCpuResources) { // Add PG CPU and GPU resources. scheduler_->GetLocalResourceManager().AddLocalResourceInstances( scheduling::ResourceID("CPU_group_aaa"), std::vector{FixedPoint(1)}); @@ -2064,8 +2083,8 @@ TEST_F(ClusterTaskManagerTestWithGPUsAtHead, ReleaseAndReturnWorkerCpuResources) auto worker2 = std::make_shared(WorkerID::FromRandom(), 5678); // Check failed as the worker has no allocated resource instances. - ASSERT_FALSE(local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1)); - ASSERT_FALSE(local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker2)); + ASSERT_FALSE(local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1)); + ASSERT_FALSE(local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker2)); auto node_resource_instances = scheduler_->GetLocalResourceManager().GetLocalResources(); @@ -2073,18 +2092,18 @@ TEST_F(ClusterTaskManagerTestWithGPUsAtHead, ReleaseAndReturnWorkerCpuResources) node_resource_instances.GetAvailableResourceInstances(); auto allocated_instances = std::make_shared(); - absl::flat_hash_map task_spec = {{"CPU", 1.}, {"GPU", 1.}}; + absl::flat_hash_map lease_spec = {{"CPU", 1.}, {"GPU", 1.}}; ASSERT_TRUE(scheduler_->GetLocalResourceManager().AllocateLocalTaskResources( - task_spec, allocated_instances)); + lease_spec, allocated_instances)); worker1->SetAllocatedInstances(allocated_instances); allocated_instances = std::make_shared(); - task_spec = {{"CPU_group_aaa", 1.}, - {"CPU_group_0_aaa", 1.}, - {"GPU_group_aaa", 1.}, - {"GPU_group_0_aaa", 1.}}; + lease_spec = {{"CPU_group_aaa", 1.}, + {"CPU_group_0_aaa", 1.}, + {"GPU_group_aaa", 1.}, + {"GPU_group_0_aaa", 1.}}; ASSERT_TRUE(scheduler_->GetLocalResourceManager().AllocateLocalTaskResources( - task_spec, allocated_instances)); + lease_spec, allocated_instances)); worker2->SetAllocatedInstances(allocated_instances); // Check that the resources are allocated successfully. @@ -2096,8 +2115,8 @@ TEST_F(ClusterTaskManagerTestWithGPUsAtHead, ReleaseAndReturnWorkerCpuResources) ASSERT_EQ(node_resources.available.Get(scheduling::ResourceID("GPU_group_0_aaa")), 0); // Check that the cpu resources are released successfully. - ASSERT_TRUE(local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1)); - ASSERT_TRUE(local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker2)); + ASSERT_TRUE(local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1)); + ASSERT_TRUE(local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker2)); // Check that only cpu resources are released. ASSERT_EQ(node_resources.available.Get(ResourceID::CPU()), 8); @@ -2111,8 +2130,8 @@ TEST_F(ClusterTaskManagerTestWithGPUsAtHead, ReleaseAndReturnWorkerCpuResources) worker1->MarkBlocked(); worker2->MarkBlocked(); // Check failed as the worker is blocked. - ASSERT_FALSE(local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1)); - ASSERT_FALSE(local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker2)); + ASSERT_FALSE(local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1)); + ASSERT_FALSE(local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker2)); // Check nothing will be changed. ASSERT_EQ(node_resources.available.Get(ResourceID::CPU()), 8); ASSERT_EQ(node_resources.available.Get(ResourceID::GPU()), 3); @@ -2122,8 +2141,8 @@ TEST_F(ClusterTaskManagerTestWithGPUsAtHead, ReleaseAndReturnWorkerCpuResources) ASSERT_EQ(node_resources.available.Get(scheduling::ResourceID("GPU_group_0_aaa")), 0); // Check that the cpu resources are returned back to worker successfully. - ASSERT_TRUE(local_task_manager_->ReturnCpuResourcesToUnblockedWorker(worker1)); - ASSERT_TRUE(local_task_manager_->ReturnCpuResourcesToUnblockedWorker(worker2)); + ASSERT_TRUE(local_lease_manager_->ReturnCpuResourcesToUnblockedWorker(worker1)); + ASSERT_TRUE(local_lease_manager_->ReturnCpuResourcesToUnblockedWorker(worker2)); // Check that only cpu resources are returned back to the worker. ASSERT_EQ(node_resources.available.Get(ResourceID::CPU()), 7); @@ -2136,8 +2155,8 @@ TEST_F(ClusterTaskManagerTestWithGPUsAtHead, ReleaseAndReturnWorkerCpuResources) // Mark worker as unblocked. worker1->MarkUnblocked(); worker2->MarkUnblocked(); - ASSERT_FALSE(local_task_manager_->ReturnCpuResourcesToUnblockedWorker(worker1)); - ASSERT_FALSE(local_task_manager_->ReturnCpuResourcesToUnblockedWorker(worker2)); + ASSERT_FALSE(local_lease_manager_->ReturnCpuResourcesToUnblockedWorker(worker1)); + ASSERT_FALSE(local_lease_manager_->ReturnCpuResourcesToUnblockedWorker(worker2)); // Check nothing will be changed. ASSERT_EQ(node_resources.available.Get(ResourceID::CPU()), 7); ASSERT_EQ(node_resources.available.Get(ResourceID::GPU()), 3); @@ -2147,104 +2166,104 @@ TEST_F(ClusterTaskManagerTestWithGPUsAtHead, ReleaseAndReturnWorkerCpuResources) ASSERT_EQ(node_resources.available.Get(scheduling::ResourceID("GPU_group_0_aaa")), 0); } -TEST_F(ClusterTaskManagerTest, TestSpillWaitingTasks) { +TEST_F(ClusterLeaseManagerTest, TestSpillWaitingLeases) { // Cases to check: - // - resources available locally, task dependencies being fetched -> do not spill. - // - resources available locally, task dependencies blocked -> spill. + // - resources available locally, lease dependencies being fetched -> do not spill. + // - resources available locally, lease dependencies blocked -> spill. // - resources not available locally -> spill. - std::vector tasks; + std::vector leases; std::vector> replies; int num_callbacks = 0; auto callback = [&](Status, std::function, std::function) { num_callbacks++; }; for (int i = 0; i < 5; i++) { - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 8}}, /*num_args=*/1); - tasks.push_back(task); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 8}}, /*num_args=*/1); + leases.push_back(lease); replies.push_back(std::make_unique()); - // All tasks except the last one added are waiting for dependencies. + // All leases except the last one added are waiting for dependencies. if (i < 4) { - auto missing_arg = task.GetTaskSpecification().GetDependencyIds()[0]; + auto missing_arg = lease.GetLeaseSpecification().GetDependencyIds()[0]; missing_objects_.insert(missing_arg); } if (i == 0) { - const_cast(task.GetTaskSpecification()) + const_cast(lease.GetLeaseSpecification()) .GetMutableMessage() .mutable_scheduling_strategy() ->mutable_spread_scheduling_strategy(); } - task_manager_.QueueAndScheduleTask(task, false, false, replies[i].get(), callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, replies[i].get(), callback); pool_.TriggerCallbacks(); } ASSERT_EQ(num_callbacks, 0); - // Local resources could only dispatch one task. - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 1); + // Local resources could only dispatch one lease. + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING_FOR_WORKER), 1); auto remote_node_id = NodeID::FromRandom(); AddNode(remote_node_id, 16); - // We are fetching dependencies for all waiting tasks but we have no enough - // resources available locally to schedule tasks except the first. + // We are fetching dependencies for all waiting leases but we have no enough + // resources available locally to schedule leases except the first. // We should only spill up to the remote node's resource availability. - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); ASSERT_EQ(num_callbacks, 2); // Spill from the back of the waiting queue. ASSERT_EQ(replies[0]->retry_at_raylet_address().node_id(), ""); ASSERT_EQ(replies[1]->retry_at_raylet_address().node_id(), ""); ASSERT_EQ(replies[2]->retry_at_raylet_address().node_id(), remote_node_id.Binary()); ASSERT_EQ(replies[3]->retry_at_raylet_address().node_id(), remote_node_id.Binary()); - ASSERT_FALSE(task_manager_.CancelTask(tasks[2].GetTaskSpecification().TaskId())); - ASSERT_FALSE(task_manager_.CancelTask(tasks[3].GetTaskSpecification().TaskId())); - // Do not spill back tasks ready to dispatch. + ASSERT_FALSE(lease_manager_.CancelLease(leases[2].GetLeaseSpecification().LeaseId())); + ASSERT_FALSE(lease_manager_.CancelLease(leases[3].GetLeaseSpecification().LeaseId())); + // Do not spill back leases ready to dispatch. ASSERT_EQ(replies[4]->retry_at_raylet_address().node_id(), ""); AddNode(remote_node_id, 8); - // Dispatch the ready task. + // Dispatch the ready lease. std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); pool_.PushWorker(std::dynamic_pointer_cast(worker)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 4); - // One waiting task spilled. + // One waiting lease spilled. ASSERT_EQ(replies[0]->retry_at_raylet_address().node_id(), ""); ASSERT_EQ(replies[1]->retry_at_raylet_address().node_id(), remote_node_id.Binary()); - ASSERT_FALSE(task_manager_.CancelTask(tasks[1].GetTaskSpecification().TaskId())); - // One task dispatched. + ASSERT_FALSE(lease_manager_.CancelLease(leases[1].GetLeaseSpecification().LeaseId())); + // One lease dispatched. ASSERT_EQ(replies[4]->worker_address().port(), 1234); // Spillback is idempotent. - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 4); - // One waiting task spilled. + // One waiting lease spilled. ASSERT_EQ(replies[0]->retry_at_raylet_address().node_id(), ""); ASSERT_EQ(replies[1]->retry_at_raylet_address().node_id(), remote_node_id.Binary()); - ASSERT_FALSE(task_manager_.CancelTask(tasks[1].GetTaskSpecification().TaskId())); - // One task dispatched. + ASSERT_FALSE(lease_manager_.CancelLease(leases[1].GetLeaseSpecification().LeaseId())); + // One lease dispatched. ASSERT_EQ(replies[4]->worker_address().port(), 1234); - // Spread task won't be spilled due to waiting for dependencies. + // Spread lease won't be spilled due to waiting for dependencies. AddNode(remote_node_id, 8); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); ASSERT_EQ(num_callbacks, 4); ASSERT_EQ(replies[0]->retry_at_raylet_address().node_id(), ""); - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); leased_workers_.clear(); - ASSERT_TRUE(task_manager_.CancelTask(tasks[0].GetTaskSpecification().TaskId())); + ASSERT_TRUE(lease_manager_.CancelLease(leases[0].GetLeaseSpecification().LeaseId())); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, PinnedArgsMemoryTest) { +TEST_F(ClusterLeaseManagerTest, PinnedArgsMemoryTest) { /* - Total memory required by executing tasks' args stays under the specified + Total memory required by granted lease args stays under the specified threshold. */ - std::shared_ptr worker = - std::make_shared(WorkerID::FromRandom(), 1234); - std::shared_ptr worker2 = - std::make_shared(WorkerID::FromRandom(), 12345); + auto worker_id1 = WorkerID::FromRandom(); + auto worker_id2 = WorkerID::FromRandom(); + std::shared_ptr worker = std::make_shared(worker_id1, 1234); + std::shared_ptr worker2 = std::make_shared(worker_id2, 12345); pool_.PushWorker(std::static_pointer_cast(worker2)); pool_.PushWorker(std::static_pointer_cast(worker)); @@ -2256,44 +2275,56 @@ TEST_F(ClusterTaskManagerTest, PinnedArgsMemoryTest) { (*num_callbacks_ptr) = *num_callbacks_ptr + 1; }; - // This task can run. + // This lease can run. + auto lease_id1 = LeaseID::FromWorker(worker_id1, 1); default_arg_size_ = 600; - auto task = CreateTask({{ray::kCPU_ResourceLabel, 1}}, 1); - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + auto lease1 = CreateLease({{ray::kCPU_ResourceLabel, 1}}, + 1, + {}, + nullptr, + rpc::SchedulingStrategy(), + lease_id1); + lease_manager_.QueueAndScheduleLease(lease1, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 1); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); - AssertPinnedTaskArgumentsPresent(task); - - // This task cannot run because it would put us over the memory threshold. - auto task2 = CreateTask({{ray::kCPU_ResourceLabel, 1}}, 1); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply, callback); + AssertPinnedLeaseArgumentsPresent(lease1); + + // This lease cannot run because it would put us over the memory threshold. + auto lease_id2 = LeaseID::FromWorker(worker_id2, 1); + auto lease2 = CreateLease({{ray::kCPU_ResourceLabel, 1}}, + 1, + {}, + nullptr, + rpc::SchedulingStrategy(), + lease_id2); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 1); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); - /* First task finishes, freeing memory for the second task */ - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); + /* First lease finishes, freeing memory for the second lease */ + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); leased_workers_.clear(); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - AssertPinnedTaskArgumentsPresent(task2); + AssertPinnedLeaseArgumentsPresent(lease2); ASSERT_EQ(num_callbacks, 2); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 0); - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); leased_workers_.clear(); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, PinnedArgsSameMemoryTest) { +TEST_F(ClusterLeaseManagerTest, PinnedArgsSameMemoryTest) { /* - * Two tasks that depend on the same object can run concurrently. + * Two leases that depend on the same object can run concurrently. */ std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); @@ -2310,33 +2341,34 @@ TEST_F(ClusterTaskManagerTest, PinnedArgsSameMemoryTest) { (*num_callbacks_ptr) = *num_callbacks_ptr + 1; }; - // This task can run. + // This lease can run. default_arg_size_ = 600; - auto task = CreateTask({{ray::kCPU_ResourceLabel, 1}}, 1); - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + auto lease = CreateLease({{ray::kCPU_ResourceLabel, 1}}, 1); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 1); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); - AssertPinnedTaskArgumentsPresent(task); + AssertPinnedLeaseArgumentsPresent(lease); - // This task can run because it depends on the same object as the first task. - auto task2 = CreateTask( - {{ray::kCPU_ResourceLabel, 1}}, 1, task.GetTaskSpecification().GetDependencyIds()); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply, callback); + // This lease can run because it depends on the same object as the first lease. + auto lease2 = CreateLease({{ray::kCPU_ResourceLabel, 1}}, + 1, + lease.GetLeaseSpecification().GetDependencyIds()); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 2); ASSERT_EQ(leased_workers_.size(), 2); ASSERT_EQ(pool_.workers.size(), 0); - RayTask finished_task; + RayLease finished_lease; for (auto &cur_worker : leased_workers_) { - local_task_manager_->TaskFinished(cur_worker.second, &finished_task); + local_lease_manager_->CleanupLease(cur_worker.second, &finished_lease); } AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, LargeArgsNoStarvationTest) { +TEST_F(ClusterLeaseManagerTest, LargeArgsNoStarvationTest) { std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); pool_.PushWorker(std::static_pointer_cast(worker)); @@ -2350,29 +2382,29 @@ TEST_F(ClusterTaskManagerTest, LargeArgsNoStarvationTest) { }; default_arg_size_ = 2000; - auto task = CreateTask({{ray::kCPU_ResourceLabel, 1}}, 1); + auto lease = CreateLease({{ray::kCPU_ResourceLabel, 1}}, 1); pool_.PushWorker(std::static_pointer_cast(worker)); - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 1); ASSERT_EQ(leased_workers_.size(), 1); - AssertPinnedTaskArgumentsPresent(task); + AssertPinnedLeaseArgumentsPresent(lease); - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, PopWorkerExactlyOnce) { - // Create and queue one task. +TEST_F(ClusterLeaseManagerTest, PopWorkerExactlyOnce) { + // Create and queue one lease. std::string serialized_runtime_env = "mock_env"; std::shared_ptr runtime_env_info = nullptr; runtime_env_info.reset(new rpc::RuntimeEnvInfo()); runtime_env_info->set_serialized_runtime_env(serialized_runtime_env); - RayTask task = CreateTask( + RayLease lease = CreateLease( {{ray::kCPU_ResourceLabel, 4}}, /*num_args=*/0, /*args=*/{}, runtime_env_info); - auto runtime_env_hash = task.GetTaskSpecification().GetRuntimeEnvHash(); + auto runtime_env_hash = lease.GetLeaseSpecification().GetRuntimeEnvHash(); rpc::RequestWorkerLeaseReply reply; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -2381,7 +2413,7 @@ TEST_F(ClusterTaskManagerTest, PopWorkerExactlyOnce) { *callback_occurred_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); // Make sure callback doesn't occurred. ASSERT_FALSE(callback_occurred); @@ -2389,8 +2421,8 @@ TEST_F(ClusterTaskManagerTest, PopWorkerExactlyOnce) { ASSERT_EQ(pool_.workers.size(), 0); // Popworker was called once. ASSERT_EQ(pool_.CallbackSize(runtime_env_hash), 1); - // Try to schedule and dispatch tasks. - task_manager_.ScheduleAndDispatchTasks(); + // Try to schedule and dispatch leases. + lease_manager_.ScheduleAndGrantLeases(); // Popworker has been called once, don't call it repeatedly. ASSERT_EQ(pool_.CallbackSize(runtime_env_hash), 1); // Push a worker and try to call back. @@ -2402,31 +2434,34 @@ TEST_F(ClusterTaskManagerTest, PopWorkerExactlyOnce) { ASSERT_TRUE(callback_occurred); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 0); - // Try to schedule and dispatch tasks. - task_manager_.ScheduleAndDispatchTasks(); + // Try to schedule and dispatch leases. + lease_manager_.ScheduleAndGrantLeases(); // Worker has been popped. Don't call `PopWorker` repeatedly. ASSERT_EQ(pool_.CallbackSize(runtime_env_hash), 0); - RayTask finished_task; - local_task_manager_->TaskFinished(leased_workers_.begin()->second, &finished_task); - ASSERT_EQ(finished_task.GetTaskSpecification().TaskId(), - task.GetTaskSpecification().TaskId()); + RayLease finished_lease; + local_lease_manager_->CleanupLease(leased_workers_.begin()->second, &finished_lease); + ASSERT_EQ(finished_lease.GetLeaseSpecification().LeaseId(), + lease.GetLeaseSpecification().LeaseId()); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, CapRunningOnDispatchQueue) { +TEST_F(ClusterLeaseManagerTest, CapRunningOnDispatchQueue) { scheduler_->GetLocalResourceManager().AddLocalResourceInstances( scheduling::ResourceID(ray::kGPU_ResourceLabel), {1, 1, 1}); - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 4}, {ray::kGPU_ResourceLabel, 1}}, - /*num_args=*/0, - /*args=*/{}); - RayTask task2 = CreateTask({{ray::kCPU_ResourceLabel, 4}, {ray::kGPU_ResourceLabel, 1}}, - /*num_args=*/0, - /*args=*/{}); - RayTask task3 = CreateTask({{ray::kCPU_ResourceLabel, 4}, {ray::kGPU_ResourceLabel, 1}}, - /*num_args=*/0, - /*args=*/{}); - auto runtime_env_hash = task.GetTaskSpecification().GetRuntimeEnvHash(); + RayLease lease = + CreateLease({{ray::kCPU_ResourceLabel, 4}, {ray::kGPU_ResourceLabel, 1}}, + /*num_args=*/0, + /*args=*/{}); + RayLease lease2 = + CreateLease({{ray::kCPU_ResourceLabel, 4}, {ray::kGPU_ResourceLabel, 1}}, + /*num_args=*/0, + /*args=*/{}); + RayLease lease3 = + CreateLease({{ray::kCPU_ResourceLabel, 4}, {ray::kGPU_ResourceLabel, 1}}, + /*num_args=*/0, + /*args=*/{}); + auto runtime_env_hash = lease.GetLeaseSpecification().GetRuntimeEnvHash(); std::vector> workers; for (int i = 0; i < 3; i++) { std::shared_ptr worker = @@ -2440,42 +2475,42 @@ TEST_F(ClusterTaskManagerTest, CapRunningOnDispatchQueue) { auto callback = [&num_callbacks](Status, std::function, std::function) { num_callbacks++; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply, callback); - task_manager_.QueueAndScheduleTask(task3, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease3, false, false, &reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 2); - local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(workers[0]); - local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(workers[1]); + local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(workers[0]); + local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(workers[1]); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); // Even though there are free resources, we've hit our cap of (8/4=)2 workers - // of the given scheduling class so we shouldn't dispatch the remaining task. + // of the given scheduling class so we shouldn't dispatch the remaining lease. ASSERT_EQ(num_callbacks, 2); - RayTask buf; - local_task_manager_->TaskFinished(workers[1], &buf); + RayLease buf; + local_lease_manager_->CleanupLease(workers[1], &buf); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 3); - local_task_manager_->TaskFinished(workers[0], &buf); - local_task_manager_->TaskFinished(workers[2], &buf); + local_lease_manager_->CleanupLease(workers[0], &buf); + local_lease_manager_->CleanupLease(workers[2], &buf); AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, ZeroCPUTasks) { +TEST_F(ClusterLeaseManagerTest, ZeroCPULeases) { scheduler_->GetLocalResourceManager().AddLocalResourceInstances( scheduling::ResourceID(ray::kGPU_ResourceLabel), {1, 1, 1}); - RayTask task = CreateTask({{"GPU", 1}}, /*num_args=*/0, /*args=*/{}); - RayTask task2 = CreateTask({{"GPU", 1}}, /*num_args=*/0, /*args=*/{}); - RayTask task3 = CreateTask({{"GPU", 1}}, /*num_args=*/0, /*args=*/{}); - auto runtime_env_hash = task.GetTaskSpecification().GetRuntimeEnvHash(); + RayLease lease = CreateLease({{"GPU", 1}}, /*num_args=*/0, /*args=*/{}); + RayLease lease2 = CreateLease({{"GPU", 1}}, /*num_args=*/0, /*args=*/{}); + RayLease lease3 = CreateLease({{"GPU", 1}}, /*num_args=*/0, /*args=*/{}); + auto runtime_env_hash = lease.GetLeaseSpecification().GetRuntimeEnvHash(); std::vector> workers; for (int i = 0; i < 3; i++) { std::shared_ptr worker = @@ -2489,28 +2524,28 @@ TEST_F(ClusterTaskManagerTest, ZeroCPUTasks) { auto callback = [&num_callbacks](Status, std::function, std::function) { num_callbacks++; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply, callback); - task_manager_.QueueAndScheduleTask(task3, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease3, false, false, &reply, callback); pool_.TriggerCallbacks(); - // We shouldn't cap anything for zero cpu tasks (and shouldn't crash before + // We shouldn't cap anything for zero cpu leases (and shouldn't crash before // this point). ASSERT_EQ(num_callbacks, 3); for (auto &worker : workers) { - RayTask buf; - local_task_manager_->TaskFinished(worker, &buf); + RayLease buf; + local_lease_manager_->CleanupLease(worker, &buf); } AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTestWithoutCPUsAtHead, ZeroCPUNode) { - RayTask task = CreateTask({}, /*num_args=*/0, /*args=*/{}); - RayTask task2 = CreateTask({}, /*num_args=*/0, /*args=*/{}); - RayTask task3 = CreateTask({}, /*num_args=*/0, /*args=*/{}); - auto runtime_env_hash = task.GetTaskSpecification().GetRuntimeEnvHash(); +TEST_F(ClusterLeaseManagerTestWithoutCPUsAtHead, ZeroCPUNode) { + RayLease lease = CreateLease({}, /*num_args=*/0, /*args=*/{}); + RayLease lease2 = CreateLease({}, /*num_args=*/0, /*args=*/{}); + RayLease lease3 = CreateLease({}, /*num_args=*/0, /*args=*/{}); + auto runtime_env_hash = lease.GetLeaseSpecification().GetRuntimeEnvHash(); std::vector> workers; for (int i = 0; i < 3; i++) { std::shared_ptr worker = @@ -2524,60 +2559,60 @@ TEST_F(ClusterTaskManagerTestWithoutCPUsAtHead, ZeroCPUNode) { auto callback = [&num_callbacks](Status, std::function, std::function) { num_callbacks++; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); - task_manager_.QueueAndScheduleTask(task2, false, false, &reply, callback); - task_manager_.QueueAndScheduleTask(task3, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease2, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease3, false, false, &reply, callback); pool_.TriggerCallbacks(); - // We shouldn't cap anything for zero cpu tasks (and shouldn't crash before + // We shouldn't cap anything for zero cpu leases (and shouldn't crash before // this point). ASSERT_EQ(num_callbacks, 3); for (auto &worker : workers) { - RayTask buf; - local_task_manager_->TaskFinished(worker, &buf); + RayLease buf; + local_lease_manager_->CleanupLease(worker, &buf); } AssertNoLeaks(); } -/// Test that we are able to spillback tasks +/// Test that we are able to spillback leases /// while hitting the scheduling class cap. -TEST_F(ClusterTaskManagerTest, SchedulingClassCapSpillback) { +TEST_F(ClusterLeaseManagerTest, SchedulingClassCapSpillback) { std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); pool_.PushWorker(std::dynamic_pointer_cast(worker)); - std::vector tasks; + std::vector leases; std::vector> replies; int num_callbacks = 0; auto callback = [&](Status, std::function, std::function) { num_callbacks++; }; - // The first task will be dispatched right away, - // and the second task will hit the scheduling class cap. + // The first lease will be dispatched right away, + // and the second lease will hit the scheduling class cap. for (int i = 0; i < 2; ++i) { - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 8}}); - tasks.push_back(task); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 8}}); + leases.push_back(lease); replies.push_back(std::make_unique()); - task_manager_.QueueAndScheduleTask(task, false, false, replies[i].get(), callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, replies[i].get(), callback); pool_.TriggerCallbacks(); } ASSERT_EQ(replies[0]->worker_address().port(), 1234); ASSERT_EQ(num_callbacks, 1); - ASSERT_EQ(NumTasksToDispatchWithStatus(internal::WorkStatus::WAITING), 1); + ASSERT_EQ(NumLeasesToDispatchWithStatus(internal::WorkStatus::WAITING), 1); // A new node is added so we should be able to spillback to it. auto remote_node_id = NodeID::FromRandom(); AddNode(remote_node_id, 8); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); ASSERT_EQ(num_callbacks, 2); ASSERT_EQ(replies[1]->retry_at_raylet_address().node_id(), remote_node_id.Binary()); } /// Test that we exponentially increase the amount of time it takes to increase /// the dispatch cap for a scheduling class. -TEST_F(ClusterTaskManagerTest, SchedulingClassCapIncrease) { +TEST_F(ClusterLeaseManagerTest, SchedulingClassCapIncrease) { auto get_unblocked_worker = [](std::vector> &workers) -> std::shared_ptr { for (auto &worker : workers) { @@ -2589,12 +2624,12 @@ TEST_F(ClusterTaskManagerTest, SchedulingClassCapIncrease) { }; int64_t UNIT = RayConfig::instance().worker_cap_initial_backoff_delay_ms(); - std::vector tasks; + std::vector leases; for (int i = 0; i < 3; i++) { - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 8}}, - /*num_args=*/0, - /*args=*/{}); - tasks.emplace_back(task); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 8}}, + /*num_args=*/0, + /*args=*/{}); + leases.emplace_back(lease); } rpc::RequestWorkerLeaseReply reply; @@ -2602,11 +2637,11 @@ TEST_F(ClusterTaskManagerTest, SchedulingClassCapIncrease) { auto callback = [&num_callbacks](Status, std::function, std::function) { num_callbacks++; }; - for (const auto &task : tasks) { - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + for (const auto &lease : leases) { + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); } - auto runtime_env_hash = tasks[0].GetTaskSpecification().GetRuntimeEnvHash(); + auto runtime_env_hash = leases[0].GetLeaseSpecification().GetRuntimeEnvHash(); std::vector> workers; for (int i = 0; i < 3; i++) { std::shared_ptr worker = @@ -2615,40 +2650,40 @@ TEST_F(ClusterTaskManagerTest, SchedulingClassCapIncrease) { pool_.TriggerCallbacks(); workers.push_back(worker); } - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); ASSERT_EQ(num_callbacks, 1); current_time_ms_ += UNIT; ASSERT_FALSE(workers.back()->IsBlocked()); - ASSERT_TRUE(local_task_manager_->ReleaseCpuResourcesFromBlockedWorker( + ASSERT_TRUE(local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker( get_unblocked_worker(workers))); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); ASSERT_EQ(num_callbacks, 2); // Since we're increasing exponentially, increasing by a unit show no longer be enough. current_time_ms_ += UNIT; - ASSERT_TRUE(local_task_manager_->ReleaseCpuResourcesFromBlockedWorker( + ASSERT_TRUE(local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker( get_unblocked_worker(workers))); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); ASSERT_EQ(num_callbacks, 2); // Now it should run current_time_ms_ += UNIT; - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); ASSERT_EQ(num_callbacks, 3); - // Let just one task finish. + // Let just one lease finish. for (auto it = workers.begin(); it != workers.end(); it++) { if (!(*it)->IsBlocked()) { - RayTask buf; - local_task_manager_->TaskFinished(*it, &buf); + RayLease buf; + local_lease_manager_->CleanupLease(*it, &buf); workers.erase(it); break; } @@ -2656,11 +2691,11 @@ TEST_F(ClusterTaskManagerTest, SchedulingClassCapIncrease) { current_time_ms_ += UNIT; - // Now schedule another task of the same scheduling class. - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 8}}, - /*num_args=*/0, - /*args=*/{}); - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + // Now schedule another lease of the same scheduling class. + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 8}}, + /*num_args=*/0, + /*args=*/{}); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); std::shared_ptr new_worker = std::make_shared(WorkerID::FromRandom(), 1234, runtime_env_hash); @@ -2669,31 +2704,31 @@ TEST_F(ClusterTaskManagerTest, SchedulingClassCapIncrease) { workers.push_back(new_worker); // It can't run for another 2 units (doesn't increase to 4, because one of - // the tasks finished). + // the leases finished). ASSERT_EQ(num_callbacks, 3); current_time_ms_ += 2 * UNIT; - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 4); for (auto &worker : workers) { - RayTask buf; - local_task_manager_->TaskFinished(worker, &buf); + RayLease buf; + local_lease_manager_->CleanupLease(worker, &buf); } AssertNoLeaks(); } -/// Ensure we reset the cap after we've finished executing through the queue. -TEST_F(ClusterTaskManagerTest, SchedulingClassCapResetTest) { +/// Ensure we reset the cap after we've granted all leases in the queue. +TEST_F(ClusterLeaseManagerTest, SchedulingClassCapResetTest) { int64_t UNIT = RayConfig::instance().worker_cap_initial_backoff_delay_ms(); - std::vector tasks; + std::vector leases; for (int i = 0; i < 2; i++) { - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 8}}, - /*num_args=*/0, - /*args=*/{}); - tasks.emplace_back(task); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 8}}, + /*num_args=*/0, + /*args=*/{}); + leases.emplace_back(lease); } rpc::RequestWorkerLeaseReply reply; @@ -2701,97 +2736,97 @@ TEST_F(ClusterTaskManagerTest, SchedulingClassCapResetTest) { auto callback = [&num_callbacks](Status, std::function, std::function) { num_callbacks++; }; - for (const auto &task : tasks) { - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + for (const auto &lease : leases) { + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); } - auto runtime_env_hash = tasks[0].GetTaskSpecification().GetRuntimeEnvHash(); + auto runtime_env_hash = leases[0].GetLeaseSpecification().GetRuntimeEnvHash(); std::shared_ptr worker1 = std::make_shared(WorkerID::FromRandom(), 1234, runtime_env_hash); pool_.PushWorker(std::static_pointer_cast(worker1)); pool_.TriggerCallbacks(); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); - ASSERT_TRUE(local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1)); + ASSERT_TRUE(local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker1)); current_time_ms_ += UNIT; std::shared_ptr worker2 = std::make_shared(WorkerID::FromRandom(), 1234, runtime_env_hash); pool_.PushWorker(std::static_pointer_cast(worker2)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 2); - RayTask buf; - local_task_manager_->TaskFinished(worker1, &buf); - local_task_manager_->TaskFinished(worker2, &buf); + RayLease buf; + local_lease_manager_->CleanupLease(worker1, &buf); + local_lease_manager_->CleanupLease(worker2, &buf); AssertNoLeaks(); for (int i = 0; i < 2; i++) { - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 8}}, - /*num_args=*/0, - /*args=*/{}); - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 8}}, + /*num_args=*/0, + /*args=*/{}); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); } std::shared_ptr worker3 = std::make_shared(WorkerID::FromRandom(), 1234, runtime_env_hash); pool_.PushWorker(std::static_pointer_cast(worker3)); pool_.TriggerCallbacks(); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); ASSERT_EQ(num_callbacks, 3); - ASSERT_TRUE(local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker3)); + ASSERT_TRUE(local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker3)); current_time_ms_ += UNIT; std::shared_ptr worker4 = std::make_shared(WorkerID::FromRandom(), 1234, runtime_env_hash); pool_.PushWorker(std::static_pointer_cast(worker4)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 4); { // Ensure a class of a different scheduling class can still be scheduled. - RayTask task5 = CreateTask({}, - /*num_args=*/0, - /*args=*/{}); - task_manager_.QueueAndScheduleTask(task5, false, false, &reply, callback); + RayLease lease5 = CreateLease({}, + /*num_args=*/0, + /*args=*/{}); + lease_manager_.QueueAndScheduleLease(lease5, false, false, &reply, callback); std::shared_ptr worker5 = std::make_shared(WorkerID::FromRandom(), 1234, runtime_env_hash); pool_.PushWorker(std::static_pointer_cast(worker5)); - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 5); - local_task_manager_->TaskFinished(worker5, &buf); + local_lease_manager_->CleanupLease(worker5, &buf); } - local_task_manager_->TaskFinished(worker3, &buf); - local_task_manager_->TaskFinished(worker4, &buf); + local_lease_manager_->CleanupLease(worker3, &buf); + local_lease_manager_->CleanupLease(worker4, &buf); AssertNoLeaks(); } /// Test that scheduling classes which have reached their running cap start -/// their timer after the new task is submitted, not before. -TEST_F(ClusterTaskManagerTest, DispatchTimerAfterRequestTest) { +/// their timer after the new lease is submitted, not before. +TEST_F(ClusterLeaseManagerTest, DispatchTimerAfterRequestTest) { int64_t UNIT = RayConfig::instance().worker_cap_initial_backoff_delay_ms(); - RayTask first_task = CreateTask({{ray::kCPU_ResourceLabel, 8}}, - /*num_args=*/0, - /*args=*/{}); + RayLease first_lease = CreateLease({{ray::kCPU_ResourceLabel, 8}}, + /*num_args=*/0, + /*args=*/{}); rpc::RequestWorkerLeaseReply reply; int num_callbacks = 0; auto callback = [&num_callbacks](Status, std::function, std::function) { num_callbacks++; }; - task_manager_.QueueAndScheduleTask(first_task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(first_lease, false, false, &reply, callback); - auto runtime_env_hash = first_task.GetTaskSpecification().GetRuntimeEnvHash(); + auto runtime_env_hash = first_lease.GetLeaseSpecification().GetRuntimeEnvHash(); std::vector> workers; for (int i = 0; i < 3; i++) { std::shared_ptr worker = @@ -2800,68 +2835,68 @@ TEST_F(ClusterTaskManagerTest, DispatchTimerAfterRequestTest) { pool_.TriggerCallbacks(); workers.push_back(worker); } - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); ASSERT_EQ(num_callbacks, 1); - RayTask second_task = CreateTask({{ray::kCPU_ResourceLabel, 8}}, - /*num_args=*/0, - /*args=*/{}); - task_manager_.QueueAndScheduleTask(second_task, false, false, &reply, callback); + RayLease second_lease = CreateLease({{ray::kCPU_ResourceLabel, 8}}, + /*num_args=*/0, + /*args=*/{}); + lease_manager_.QueueAndScheduleLease(second_lease, false, false, &reply, callback); pool_.TriggerCallbacks(); /// Can't schedule yet due to the cap. ASSERT_EQ(num_callbacks, 1); for (auto &worker : workers) { if (worker->GetAllocatedInstances() && !worker->IsBlocked()) { - local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker); + local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker); } } current_time_ms_ += UNIT; - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 2); for (auto &worker : workers) { if (worker->GetAllocatedInstances() && !worker->IsBlocked()) { - local_task_manager_->ReleaseCpuResourcesFromBlockedWorker(worker); + local_lease_manager_->ReleaseCpuResourcesFromBlockedWorker(worker); } } /// A lot of time passes, definitely more than the timeout. current_time_ms_ += 100000 * UNIT; - RayTask third_task = CreateTask({{ray::kCPU_ResourceLabel, 8}}, - /*num_args=*/0, - /*args=*/{}); - task_manager_.QueueAndScheduleTask(third_task, false, false, &reply, callback); + RayLease third_lease = CreateLease({{ray::kCPU_ResourceLabel, 8}}, + /*num_args=*/0, + /*args=*/{}); + lease_manager_.QueueAndScheduleLease(third_lease, false, false, &reply, callback); pool_.TriggerCallbacks(); - /// We still can't schedule the third task since the timer doesn't start - /// until after the task is queued. + /// We still can't schedule the third lease since the timer doesn't start + /// until after the lease is queued. ASSERT_EQ(num_callbacks, 2); current_time_ms_ += 2 * UNIT; - task_manager_.ScheduleAndDispatchTasks(); + lease_manager_.ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); ASSERT_EQ(num_callbacks, 3); for (auto &worker : workers) { - RayTask buf; - local_task_manager_->TaskFinished(worker, &buf); + RayLease buf; + local_lease_manager_->CleanupLease(worker, &buf); } AssertNoLeaks(); } -TEST_F(ClusterTaskManagerTest, PopWorkerBeforeDraining) { +TEST_F(ClusterLeaseManagerTest, PopWorkerBeforeDraining) { /* Test that if PopWorker happens before draining, the lease request can still succeed. */ - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 1}}); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply reply; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -2869,7 +2904,7 @@ TEST_F(ClusterTaskManagerTest, PopWorkerBeforeDraining) { Status, std::function, std::function) { *callback_occurred_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); // Drain the local node. rpc::DrainRayletRequest drain_request; @@ -2884,11 +2919,11 @@ TEST_F(ClusterTaskManagerTest, PopWorkerBeforeDraining) { ASSERT_EQ(leased_workers_.size(), 1); } -TEST_F(ClusterTaskManagerTest, UnscheduleableWhileDraining) { +TEST_F(ClusterLeaseManagerTest, UnscheduleableWhileDraining) { /* - Test that new tasks are not scheduled onto draining nodes. + Test that new leases are not scheduled onto draining nodes. */ - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, 1}}); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply reply; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -2896,7 +2931,7 @@ TEST_F(ClusterTaskManagerTest, UnscheduleableWhileDraining) { Status, std::function, std::function) { *callback_occurred_ptr = true; }; - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); std::shared_ptr worker = std::make_shared(WorkerID::FromRandom(), 1234); std::shared_ptr worker2 = @@ -2916,10 +2951,10 @@ TEST_F(ClusterTaskManagerTest, UnscheduleableWhileDraining) { drain_request.set_deadline_timestamp_ms(std::numeric_limits::max()); scheduler_->GetLocalResourceManager().SetLocalNodeDraining(drain_request); - RayTask spillback_task = CreateTask({{ray::kCPU_ResourceLabel, 1}}); + RayLease spillback_lease = CreateLease({{ray::kCPU_ResourceLabel, 1}}); rpc::RequestWorkerLeaseReply spillback_reply; - task_manager_.QueueAndScheduleTask( - spillback_task, false, false, &spillback_reply, callback); + lease_manager_.QueueAndScheduleLease( + spillback_lease, false, false, &spillback_reply, callback); pool_.TriggerCallbacks(); ASSERT_EQ(leased_workers_.size(), 1); ASSERT_EQ(pool_.workers.size(), 1); @@ -2927,9 +2962,9 @@ TEST_F(ClusterTaskManagerTest, UnscheduleableWhileDraining) { } // Regression test for https://github.com/ray-project/ray/issues/16935: -// When a task requires 1 CPU and is infeasible because head node has 0 CPU, -// make sure the task's resource demand is reported. -TEST_F(ClusterTaskManagerTestWithoutCPUsAtHead, OneCpuInfeasibleTask) { +// When a lease requires 1 CPU and is infeasible because head node has 0 CPU, +// make sure the lease's resource demand is reported. +TEST_F(ClusterLeaseManagerTestWithoutCPUsAtHead, OneCpuInfeasibleLease) { rpc::RequestWorkerLeaseReply reply; bool callback_occurred = false; bool *callback_occurred_ptr = &callback_occurred; @@ -2940,7 +2975,7 @@ TEST_F(ClusterTaskManagerTestWithoutCPUsAtHead, OneCpuInfeasibleTask) { }; constexpr int num_cases = 5; - // Create 5 tasks with different CPU requests. + // Create 5 leases with different CPU requests. const std::array cpu_request = {1, 2, 1, 3, 1}; // Each type of CPU request corresponds to a types of resource demand. const std::array demand_types = {1, 2, 2, 3, 3}; @@ -2948,18 +2983,18 @@ TEST_F(ClusterTaskManagerTestWithoutCPUsAtHead, OneCpuInfeasibleTask) { const std::array num_infeasible_1cpu = {1, 1, 2, 2, 3}; for (int i = 0; i < num_cases; ++i) { - RayTask task = CreateTask({{ray::kCPU_ResourceLabel, cpu_request[i]}}); - task_manager_.QueueAndScheduleTask(task, false, false, &reply, callback); + RayLease lease = CreateLease({{ray::kCPU_ResourceLabel, cpu_request[i]}}); + lease_manager_.QueueAndScheduleLease(lease, false, false, &reply, callback); pool_.TriggerCallbacks(); - // The task cannot run because there is only 1 node (head) with 0 CPU. + // The lease cannot run because there is only 1 node (head) with 0 CPU. ASSERT_FALSE(callback_occurred); ASSERT_EQ(leased_workers_.size(), 0); ASSERT_EQ(pool_.workers.size(), 0); ASSERT_EQ(node_info_calls_, 0); rpc::ResourcesData data; - task_manager_.FillResourceUsage(data); + lease_manager_.FillResourceUsage(data); const auto &resource_load_by_shape = data.resource_load_by_shape(); ASSERT_EQ(resource_load_by_shape.resource_demands().size(), demand_types[i]); diff --git a/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc b/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc index cafc79dfbeab..f4b0d92bdaed 100644 --- a/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc +++ b/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc @@ -508,8 +508,10 @@ TEST_F(ClusterResourceSchedulerTest, SchedulingWithPreferredNodeTest) { // Remote node is feasible but has no available resource. resource_scheduler.GetClusterResourceManager().AddOrUpdateNode( remote_node_id, remote_resource_total, {{"CPU", 0}}); + LeaseSpecification lease_spec1( + std::move(spec_builder_1).ConsumeAndBuild().GetMessage()); auto node_id_3 = resource_scheduler.GetBestSchedulableNode( - std::move(spec_builder_1).ConsumeAndBuild(), + lease_spec1, /*preferred_node_id=*/local_node_id.Binary(), false, false, @@ -550,8 +552,10 @@ TEST_F(ClusterResourceSchedulerTest, SchedulingWithPreferredNodeTest) { "", nullptr); spec_builder_2.SetNormalTaskSpec(0, false, "", scheduling_strategy, ActorID::Nil()); + LeaseSpecification lease_spec2( + std::move(spec_builder_2).ConsumeAndBuild().GetMessage()); auto node_id_4 = resource_scheduler.GetBestSchedulableNode( - std::move(spec_builder_2).ConsumeAndBuild(), + lease_spec2, /*preferred_node_id=*/local_node_id.Binary(), false, false, @@ -1848,8 +1852,8 @@ TEST_F(ClusterResourceSchedulerTest, LabelSelectorIsSchedulableOnNodeTest) { label_selector_spec.SetNormalTaskSpec( 0, false, "", scheduling_strategy, ActorID::Nil()); auto built_label_selector = std::move(label_selector_spec).ConsumeAndBuild(); - resource_scheduler.GetBestSchedulableNode( - built_label_selector, "", false, false, &is_infeasible); + LeaseSpecification lease_spec(built_label_selector.GetMessage()); + resource_scheduler.GetBestSchedulableNode(lease_spec, "", false, false, &is_infeasible); ASSERT_TRUE(is_infeasible); // Set node labels - node should now be schedulable @@ -1858,7 +1862,7 @@ TEST_F(ClusterResourceSchedulerTest, LabelSelectorIsSchedulableOnNodeTest) { }; resource_scheduler.GetClusterResourceManager().SetNodeLabels(node_1, test_labels); auto best_node_2 = resource_scheduler.GetBestSchedulableNode( - built_label_selector, "", false, false, &is_infeasible); + lease_spec, "", false, false, &is_infeasible); ASSERT_EQ(best_node_2, node_1); ASSERT_FALSE(is_infeasible); } diff --git a/src/ray/raylet/tests/BUILD.bazel b/src/ray/raylet/tests/BUILD.bazel index 52321de9f933..f0b0cbc877ff 100644 --- a/src/ray/raylet/tests/BUILD.bazel +++ b/src/ray/raylet/tests/BUILD.bazel @@ -70,7 +70,6 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:id", - "//src/ray/common:task_common", "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/raylet:placement_group_resource_manager", "@com_google_googletest//:gtest_main", @@ -96,31 +95,32 @@ ray_cc_test( ) ray_cc_test( - name = "dependency_manager_test", + name = "lease_dependency_manager_test", size = "small", - srcs = ["dependency_manager_test.cc"], + srcs = ["lease_dependency_manager_test.cc"], tags = ["team:core"], deps = [ "//:ray_mock", - "//src/ray/common:task_common", + "//src/ray/common:lease", "//src/ray/common:test_util", - "//src/ray/raylet:dependency_manager", + "//src/ray/raylet:lease_dependency_manager", "@com_google_googletest//:gtest_main", ], ) ray_cc_test( - name = "local_task_manager_test", + name = "local_lease_manager_test", size = "small", - srcs = ["local_task_manager_test.cc"], + srcs = ["local_lease_manager_test.cc"], tags = ["team:core"], deps = [ ":util", "//:ray_mock", "//src/ray/common:id", + "//src/ray/common:lease", "//src/ray/common:task_common", "//src/ray/common:test_util", - "//src/ray/raylet:local_task_manager", + "//src/ray/raylet:local_lease_manager", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "@com_google_googletest//:gtest_main", ], @@ -135,7 +135,7 @@ ray_cc_test( tags = ["team:core"], deps = [ ":util", - "//src/ray/common:task_common", + "//src/ray/common:lease", "//src/ray/raylet:worker_killing_policy", "@com_google_googletest//:gtest_main", ], @@ -150,7 +150,7 @@ ray_cc_test( tags = ["team:core"], deps = [ ":util", - "//src/ray/common:task_common", + "//src/ray/common:lease", "//src/ray/raylet:worker_killing_policy", "@com_google_googletest//:gtest_main", ], @@ -165,7 +165,7 @@ ray_cc_test( tags = ["team:core"], deps = [ ":util", - "//src/ray/common:task_common", + "//src/ray/common:lease", "//src/ray/raylet:worker_killing_policy", "@com_google_googletest//:gtest_main", ], @@ -181,11 +181,13 @@ ray_cc_test( "//:ray_fakes", "//:ray_mock", "//src/fakes/ray/rpc/raylet:fake_raylet_client", + "//src/ray/common:lease", "//src/ray/common:ray_object", + "//src/ray/common:task_common", "//src/ray/object_manager/plasma:plasma_client", "//src/ray/raylet:local_object_manager_interface", "//src/ray/raylet:node_manager", - "//src/ray/raylet/scheduling:cluster_task_manager", + "//src/ray/raylet/scheduling:cluster_lease_manager", "//src/ray/util:macros", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/raylet/tests/dependency_manager_test.cc b/src/ray/raylet/tests/dependency_manager_test.cc deleted file mode 100644 index 9ad14a15df91..000000000000 --- a/src/ray/raylet/tests/dependency_manager_test.cc +++ /dev/null @@ -1,399 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ray/raylet/dependency_manager.h" - -#include -#include -#include -#include -#include - -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "mock/ray/object_manager/object_manager.h" -#include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" - -namespace ray { - -namespace raylet { - -using ::testing::_; -using ::testing::InSequence; -using ::testing::Return; - -class CustomMockObjectManager : public MockObjectManager { - public: - uint64_t Pull(const std::vector &object_refs, - BundlePriority prio, - const TaskMetricsKey &task_key) override { - if (prio == BundlePriority::GET_REQUEST) { - active_get_requests.insert(req_id); - } else if (prio == BundlePriority::WAIT_REQUEST) { - active_wait_requests.insert(req_id); - } else { - active_task_requests.insert(req_id); - } - return req_id++; - } - - void CancelPull(uint64_t request_id) override { - ASSERT_TRUE(active_get_requests.erase(request_id) || - active_wait_requests.erase(request_id) || - active_task_requests.erase(request_id)); - } - - bool PullRequestActiveOrWaitingForMetadata(uint64_t request_id) const override { - return active_get_requests.count(request_id) || - active_wait_requests.count(request_id) || - active_task_requests.count(request_id); - } - - uint64_t req_id = 1; - std::unordered_set active_get_requests; - std::unordered_set active_wait_requests; - std::unordered_set active_task_requests; -}; - -class DependencyManagerTest : public ::testing::Test { - public: - DependencyManagerTest() - : object_manager_mock_(), dependency_manager_(object_manager_mock_) {} - - int64_t NumWaiting(const std::string &task_name) { - return dependency_manager_.waiting_tasks_counter_.Get({task_name, false}); - } - - int64_t NumWaitingTotal() { return dependency_manager_.waiting_tasks_counter_.Total(); } - - void AssertNoLeaks() { - ASSERT_TRUE(dependency_manager_.required_objects_.empty()); - ASSERT_TRUE(dependency_manager_.queued_task_requests_.empty()); - ASSERT_TRUE(dependency_manager_.get_requests_.empty()); - ASSERT_TRUE(dependency_manager_.wait_requests_.empty()); - ASSERT_EQ(dependency_manager_.waiting_tasks_counter_.Total(), 0); - // All pull requests are canceled. - ASSERT_TRUE(object_manager_mock_.active_task_requests.empty()); - ASSERT_TRUE(object_manager_mock_.active_get_requests.empty()); - ASSERT_TRUE(object_manager_mock_.active_wait_requests.empty()); - } - - CustomMockObjectManager object_manager_mock_; - DependencyManager dependency_manager_; -}; - -/// Test requesting the dependencies for a task. The dependency manager should -/// return the task ID as ready once all of its arguments are local. -TEST_F(DependencyManagerTest, TestSimpleTask) { - // Create a task with 3 arguments. - int num_arguments = 3; - std::vector arguments; - for (int i = 0; i < num_arguments; i++) { - arguments.push_back(ObjectID::FromRandom()); - } - TaskID task_id = RandomTaskId(); - bool ready = dependency_manager_.RequestTaskDependencies( - task_id, ObjectIdsToRefs(arguments), {"foo", false}); - ASSERT_FALSE(ready); - ASSERT_EQ(NumWaiting("bar"), 0); - ASSERT_EQ(NumWaiting("foo"), 1); - ASSERT_EQ(NumWaitingTotal(), 1); - - // For each argument, tell the task dependency manager that the argument is - // local. All arguments should be canceled as they become available locally. - auto ready_task_ids = dependency_manager_.HandleObjectLocal(arguments[0]); - ASSERT_TRUE(ready_task_ids.empty()); - ready_task_ids = dependency_manager_.HandleObjectLocal(arguments[1]); - ASSERT_TRUE(ready_task_ids.empty()); - // The task is ready to run. - ready_task_ids = dependency_manager_.HandleObjectLocal(arguments[2]); - ASSERT_EQ(ready_task_ids.size(), 1); - ASSERT_EQ(ready_task_ids.front(), task_id); - ASSERT_EQ(NumWaiting("bar"), 0); - ASSERT_EQ(NumWaiting("foo"), 0); - ASSERT_EQ(NumWaitingTotal(), 0); - - // Remove the task. - dependency_manager_.RemoveTaskDependencies(task_id); - AssertNoLeaks(); -} - -/// Test multiple tasks that depend on the same object. The dependency manager -/// should return all task IDs as ready once the object is local. -TEST_F(DependencyManagerTest, TestMultipleTasks) { - // Create 3 tasks that are dependent on the same object. - ObjectID argument_id = ObjectID::FromRandom(); - std::vector dependent_tasks; - int num_dependent_tasks = 3; - for (int i = 0; i < num_dependent_tasks; i++) { - TaskID task_id = RandomTaskId(); - dependent_tasks.push_back(task_id); - bool ready = dependency_manager_.RequestTaskDependencies( - task_id, ObjectIdsToRefs({argument_id}), {"foo", false}); - ASSERT_FALSE(ready); - // The object should be requested from the object manager once for each task. - ASSERT_EQ(object_manager_mock_.active_task_requests.size(), i + 1); - } - ASSERT_EQ(NumWaiting("bar"), 0); - ASSERT_EQ(NumWaiting("foo"), 3); - ASSERT_EQ(NumWaitingTotal(), 3); - - // Tell the task dependency manager that the object is local. - auto ready_task_ids = dependency_manager_.HandleObjectLocal(argument_id); - // Check that all tasks are now ready to run. - std::unordered_set added_tasks(dependent_tasks.begin(), dependent_tasks.end()); - for (auto &id : ready_task_ids) { - ASSERT_TRUE(added_tasks.erase(id)); - } - ASSERT_TRUE(added_tasks.empty()); - - for (auto &id : dependent_tasks) { - dependency_manager_.RemoveTaskDependencies(id); - } - AssertNoLeaks(); -} - -/// Test task with multiple dependencies. The dependency manager should return -/// the task ID as ready once all dependencies are local. If a dependency is -/// later evicted, the dependency manager should return the task ID as waiting. -TEST_F(DependencyManagerTest, TestTaskArgEviction) { - // Add a task with 3 arguments. - int num_arguments = 3; - std::vector arguments; - for (int i = 0; i < num_arguments; i++) { - arguments.push_back(ObjectID::FromRandom()); - } - TaskID task_id = RandomTaskId(); - bool ready = dependency_manager_.RequestTaskDependencies( - task_id, ObjectIdsToRefs(arguments), {"", false}); - ASSERT_FALSE(ready); - - // Tell the task dependency manager that each of the arguments is now - // available. - for (size_t i = 0; i < arguments.size(); i++) { - std::vector ready_tasks; - ready_tasks = dependency_manager_.HandleObjectLocal(arguments[i]); - if (i == arguments.size() - 1) { - ASSERT_EQ(ready_tasks.size(), 1); - ASSERT_EQ(ready_tasks.front(), task_id); - } else { - ASSERT_TRUE(ready_tasks.empty()); - } - } - - // Simulate each of the arguments getting evicted. Each object should now be - // considered remote. - for (size_t i = 0; i < arguments.size(); i++) { - std::vector waiting_tasks; - waiting_tasks = dependency_manager_.HandleObjectMissing(arguments[i]); - if (i == 0) { - // The first eviction should cause the task to go back to the waiting - // state. - ASSERT_EQ(waiting_tasks.size(), 1); - ASSERT_EQ(waiting_tasks.front(), task_id); - } else { - // The subsequent evictions shouldn't cause any more tasks to go back to - // the waiting state. - ASSERT_TRUE(waiting_tasks.empty()); - } - } - - // Tell the task dependency manager that each of the arguments is available - // again. - for (size_t i = 0; i < arguments.size(); i++) { - std::vector ready_tasks; - ready_tasks = dependency_manager_.HandleObjectLocal(arguments[i]); - if (i == arguments.size() - 1) { - ASSERT_EQ(ready_tasks.size(), 1); - ASSERT_EQ(ready_tasks.front(), task_id); - } else { - ASSERT_TRUE(ready_tasks.empty()); - } - } - - dependency_manager_.RemoveTaskDependencies(task_id); - AssertNoLeaks(); -} - -/// Test `ray.get`. Worker calls ray.get on {oid1}, then {oid1, oid2}, then -/// {oid1, oid2, oid3}. -TEST_F(DependencyManagerTest, TestGet) { - WorkerID worker_id = WorkerID::FromRandom(); - int num_arguments = 3; - std::vector arguments; - for (int i = 0; i < num_arguments; i++) { - // Add the new argument to the list of dependencies to subscribe to. - ObjectID argument_id = ObjectID::FromRandom(); - arguments.push_back(argument_id); - // Subscribe to the task's dependencies. All arguments except the last are - // duplicates of previous subscription calls. Each argument should only be - // requested from the node manager once. - auto prev_pull_reqs = object_manager_mock_.active_get_requests; - dependency_manager_.StartOrUpdateGetRequest(worker_id, ObjectIdsToRefs(arguments)); - // Previous pull request for this get should be canceled upon each new - // bundle. - ASSERT_EQ(object_manager_mock_.active_get_requests.size(), 1); - ASSERT_NE(object_manager_mock_.active_get_requests, prev_pull_reqs); - } - - // Nothing happens if the same bundle is requested. - auto prev_pull_reqs = object_manager_mock_.active_get_requests; - dependency_manager_.StartOrUpdateGetRequest(worker_id, ObjectIdsToRefs(arguments)); - ASSERT_EQ(object_manager_mock_.active_get_requests, prev_pull_reqs); - - // Cancel the pull request once the worker cancels the `ray.get`. - dependency_manager_.CancelGetRequest(worker_id); - AssertNoLeaks(); -} - -/// Test that when one of the objects becomes local after a `ray.wait` call, -/// all requests to remote nodes associated with the object are canceled. -TEST_F(DependencyManagerTest, TestWait) { - // Generate a random worker and objects to wait on. - WorkerID worker_id = WorkerID::FromRandom(); - int num_objects = 3; - std::vector oids; - for (int i = 0; i < num_objects; i++) { - oids.push_back(ObjectID::FromRandom()); - } - dependency_manager_.StartOrUpdateWaitRequest(worker_id, ObjectIdsToRefs(oids)); - ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects); - - for (int i = 0; i < num_objects; i++) { - // Object is local. - auto ready_task_ids = dependency_manager_.HandleObjectLocal(oids[i]); - - // Local object gets evicted. The `ray.wait` call should not be - // reactivated. - auto waiting_task_ids = dependency_manager_.HandleObjectMissing(oids[i]); - ASSERT_TRUE(waiting_task_ids.empty()); - ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects - i - 1); - } - AssertNoLeaks(); -} - -/// Test that when no objects are locally available, a `ray.wait` call makes -/// the correct requests to remote nodes and correctly cancels the requests -/// when the `ray.wait` call is canceled. -TEST_F(DependencyManagerTest, TestWaitThenCancel) { - // Generate a random worker and objects to wait on. - WorkerID worker_id = WorkerID::FromRandom(); - int num_objects = 3; - std::vector oids; - for (int i = 0; i < num_objects; i++) { - oids.push_back(ObjectID::FromRandom()); - } - // Simulate a worker calling `ray.wait` on some objects. - dependency_manager_.StartOrUpdateWaitRequest(worker_id, ObjectIdsToRefs(oids)); - ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects); - // Check that it's okay to call `ray.wait` on the same objects again. No new - // calls should be made to try and make the objects local. - dependency_manager_.StartOrUpdateWaitRequest(worker_id, ObjectIdsToRefs(oids)); - ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects); - // Cancel the worker's `ray.wait`. - dependency_manager_.CancelWaitRequest(worker_id); - AssertNoLeaks(); -} - -/// Test that when one of the objects is already local at the time of the -/// `ray.wait` call, the `ray.wait` call does not trigger any requests to -/// remote nodes for that object. -TEST_F(DependencyManagerTest, TestWaitObjectLocal) { - // Generate a random worker and objects to wait on. - WorkerID worker_id = WorkerID::FromRandom(); - int num_objects = 3; - std::vector oids; - for (int i = 0; i < num_objects; i++) { - oids.push_back(ObjectID::FromRandom()); - } - // Simulate one of the objects becoming local. The later `ray.wait` call - // should have no effect because the object is already local. - const ObjectID local_object_id = std::move(oids.back()); - auto ready_task_ids = dependency_manager_.HandleObjectLocal(local_object_id); - ASSERT_TRUE(ready_task_ids.empty()); - dependency_manager_.StartOrUpdateWaitRequest(worker_id, ObjectIdsToRefs(oids)); - ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects - 1); - // Simulate the local object getting evicted. The `ray.wait` call should not - // be reactivated. - auto waiting_task_ids = dependency_manager_.HandleObjectMissing(local_object_id); - ASSERT_TRUE(waiting_task_ids.empty()); - ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects - 1); - // Cancel the worker's `ray.wait`. - dependency_manager_.CancelWaitRequest(worker_id); - AssertNoLeaks(); -} - -/// Test requesting the dependencies for a task. The dependency manager should -/// return the task ID as ready once all of its unique arguments are local. -TEST_F(DependencyManagerTest, TestDuplicateTaskArgs) { - // Create a task with 3 arguments. - int num_arguments = 3; - auto obj_id = ObjectID::FromRandom(); - std::vector arguments; - for (int i = 0; i < num_arguments; i++) { - arguments.push_back(obj_id); - } - TaskID task_id = RandomTaskId(); - bool ready = dependency_manager_.RequestTaskDependencies( - task_id, ObjectIdsToRefs(arguments), {"", false}); - ASSERT_FALSE(ready); - ASSERT_EQ(object_manager_mock_.active_task_requests.size(), 1); - - auto ready_task_ids = dependency_manager_.HandleObjectLocal(obj_id); - ASSERT_EQ(ready_task_ids.size(), 1); - ASSERT_EQ(ready_task_ids.front(), task_id); - dependency_manager_.RemoveTaskDependencies(task_id); - - TaskID task_id2 = RandomTaskId(); - ready = dependency_manager_.RequestTaskDependencies( - task_id2, ObjectIdsToRefs(arguments), {"", false}); - ASSERT_TRUE(ready); - ASSERT_EQ(object_manager_mock_.active_task_requests.size(), 1); - dependency_manager_.RemoveTaskDependencies(task_id2); - - AssertNoLeaks(); -} - -/// Test that RemoveTaskDependencies is called before objects -/// becoming local (e.g. the task is cancelled). -TEST_F(DependencyManagerTest, TestRemoveTaskDependenciesBeforeLocal) { - int num_arguments = 3; - std::vector arguments; - for (int i = 0; i < num_arguments; i++) { - arguments.push_back(ObjectID::FromRandom()); - } - TaskID task_id = RandomTaskId(); - bool ready = dependency_manager_.RequestTaskDependencies( - task_id, ObjectIdsToRefs(arguments), {"foo", false}); - ASSERT_FALSE(ready); - ASSERT_EQ(NumWaiting("bar"), 0); - ASSERT_EQ(NumWaiting("foo"), 1); - ASSERT_EQ(NumWaitingTotal(), 1); - - // The task is cancelled - dependency_manager_.RemoveTaskDependencies(task_id); - ASSERT_EQ(NumWaiting("foo"), 0); - ASSERT_EQ(NumWaitingTotal(), 0); - AssertNoLeaks(); -} - -} // namespace raylet - -} // namespace ray - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/src/ray/raylet/tests/lease_dependency_manager_test.cc b/src/ray/raylet/tests/lease_dependency_manager_test.cc new file mode 100644 index 000000000000..d240d45566db --- /dev/null +++ b/src/ray/raylet/tests/lease_dependency_manager_test.cc @@ -0,0 +1,403 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/raylet/lease_dependency_manager.h" + +#include +#include +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "mock/ray/object_manager/object_manager.h" +#include "ray/common/test_util.h" + +namespace ray { + +namespace raylet { + +using ::testing::_; +using ::testing::InSequence; +using ::testing::Return; + +class CustomMockObjectManager : public MockObjectManager { + public: + uint64_t Pull(const std::vector &object_refs, + BundlePriority prio, + const TaskMetricsKey &task_key) override { + if (prio == BundlePriority::GET_REQUEST) { + active_get_requests.insert(req_id); + } else if (prio == BundlePriority::WAIT_REQUEST) { + active_wait_requests.insert(req_id); + } else { + active_lease_requests.insert(req_id); + } + return req_id++; + } + + void CancelPull(uint64_t request_id) override { + ASSERT_TRUE(active_get_requests.erase(request_id) || + active_wait_requests.erase(request_id) || + active_lease_requests.erase(request_id)); + } + + bool PullRequestActiveOrWaitingForMetadata(uint64_t request_id) const override { + return active_get_requests.count(request_id) || + active_wait_requests.count(request_id) || + active_lease_requests.count(request_id); + } + + uint64_t req_id = 1; + std::unordered_set active_get_requests; + std::unordered_set active_wait_requests; + std::unordered_set active_lease_requests; +}; + +class LeaseDependencyManagerTest : public ::testing::Test { + public: + LeaseDependencyManagerTest() + : object_manager_mock_(), lease_dependency_manager_(object_manager_mock_) {} + + int64_t NumWaiting(const std::string &lease_name) { + return lease_dependency_manager_.waiting_leases_counter_.Get({lease_name, false}); + } + + int64_t NumWaitingTotal() { + return lease_dependency_manager_.waiting_leases_counter_.Total(); + } + + void AssertNoLeaks() { + ASSERT_TRUE(lease_dependency_manager_.required_objects_.empty()); + ASSERT_TRUE(lease_dependency_manager_.queued_lease_requests_.empty()); + ASSERT_TRUE(lease_dependency_manager_.get_requests_.empty()); + ASSERT_TRUE(lease_dependency_manager_.wait_requests_.empty()); + ASSERT_EQ(lease_dependency_manager_.waiting_leases_counter_.Total(), 0); + // All pull requests are canceled. + ASSERT_TRUE(object_manager_mock_.active_lease_requests.empty()); + ASSERT_TRUE(object_manager_mock_.active_get_requests.empty()); + ASSERT_TRUE(object_manager_mock_.active_wait_requests.empty()); + } + + CustomMockObjectManager object_manager_mock_; + LeaseDependencyManager lease_dependency_manager_; +}; + +/// Test requesting the dependencies for a lease. The dependency manager should +/// return the lease ID as ready once all of its arguments are local. +TEST_F(LeaseDependencyManagerTest, TestSimpleLease) { + // Create a lease with 3 arguments. + int num_arguments = 3; + std::vector arguments; + for (int i = 0; i < num_arguments; i++) { + arguments.push_back(ObjectID::FromRandom()); + } + LeaseID lease_id = LeaseID::FromRandom(); + bool ready = lease_dependency_manager_.RequestLeaseDependencies( + lease_id, ObjectIdsToRefs(arguments), {"foo", false}); + ASSERT_FALSE(ready); + ASSERT_EQ(NumWaiting("bar"), 0); + ASSERT_EQ(NumWaiting("foo"), 1); + ASSERT_EQ(NumWaitingTotal(), 1); + + // For each argument, tell the lease dependency manager that the argument is + // local. All arguments should be canceled as they become available locally. + auto ready_lease_ids = lease_dependency_manager_.HandleObjectLocal(arguments[0]); + ASSERT_TRUE(ready_lease_ids.empty()); + ready_lease_ids = lease_dependency_manager_.HandleObjectLocal(arguments[1]); + ASSERT_TRUE(ready_lease_ids.empty()); + // The lease is ready to run. + ready_lease_ids = lease_dependency_manager_.HandleObjectLocal(arguments[2]); + ASSERT_EQ(ready_lease_ids.size(), 1); + ASSERT_EQ(ready_lease_ids.front(), lease_id); + ASSERT_EQ(NumWaiting("bar"), 0); + ASSERT_EQ(NumWaiting("foo"), 0); + ASSERT_EQ(NumWaitingTotal(), 0); + + // Remove the lease. + lease_dependency_manager_.RemoveLeaseDependencies(lease_id); + AssertNoLeaks(); +} + +/// Test multiple leases that depend on the same object. The dependency manager +/// should return all lease IDs as ready once the object is local. +TEST_F(LeaseDependencyManagerTest, TestMultipleLeases) { + // Create 3 leases that are dependent on the same object. + ObjectID argument_id = ObjectID::FromRandom(); + std::vector dependent_leases; + int num_dependent_leases = 3; + for (int i = 0; i < num_dependent_leases; i++) { + LeaseID lease_id = LeaseID::FromRandom(); + dependent_leases.push_back(lease_id); + bool ready = lease_dependency_manager_.RequestLeaseDependencies( + lease_id, ObjectIdsToRefs({argument_id}), {"foo", false}); + ASSERT_FALSE(ready); + // The object should be requested from the object manager once for each lease. + ASSERT_EQ(object_manager_mock_.active_lease_requests.size(), i + 1); + } + ASSERT_EQ(NumWaiting("bar"), 0); + ASSERT_EQ(NumWaiting("foo"), 3); + ASSERT_EQ(NumWaitingTotal(), 3); + + // Tell the lease dependency manager that the object is local. + auto ready_lease_ids = lease_dependency_manager_.HandleObjectLocal(argument_id); + // Check that all leases are now ready to run. + std::unordered_set added_leases(dependent_leases.begin(), + dependent_leases.end()); + for (auto &id : ready_lease_ids) { + ASSERT_TRUE(added_leases.erase(id)); + } + ASSERT_TRUE(added_leases.empty()); + + for (auto &id : dependent_leases) { + lease_dependency_manager_.RemoveLeaseDependencies(id); + } + AssertNoLeaks(); +} + +/// Test lease with multiple dependencies. The dependency manager should return +/// the lease ID as ready once all dependencies are local. If a dependency is +/// later evicted, the dependency manager should return the lease ID as waiting. +TEST_F(LeaseDependencyManagerTest, TestLeaseArgEviction) { + // Add a lease with 3 arguments. + int num_arguments = 3; + std::vector arguments; + for (int i = 0; i < num_arguments; i++) { + arguments.push_back(ObjectID::FromRandom()); + } + LeaseID lease_id = LeaseID::FromRandom(); + bool ready = lease_dependency_manager_.RequestLeaseDependencies( + lease_id, ObjectIdsToRefs(arguments), {"", false}); + ASSERT_FALSE(ready); + + // Tell the lease dependency manager that each of the arguments is now + // available. + for (size_t i = 0; i < arguments.size(); i++) { + std::vector ready_leases; + ready_leases = lease_dependency_manager_.HandleObjectLocal(arguments[i]); + if (i == arguments.size() - 1) { + ASSERT_EQ(ready_leases.size(), 1); + ASSERT_EQ(ready_leases.front(), lease_id); + } else { + ASSERT_TRUE(ready_leases.empty()); + } + } + + // Simulate each of the arguments getting evicted. Each object should now be + // considered remote. + for (size_t i = 0; i < arguments.size(); i++) { + std::vector waiting_leases; + waiting_leases = lease_dependency_manager_.HandleObjectMissing(arguments[i]); + if (i == 0) { + // The first eviction should cause the lease to go back to the waiting + // state. + ASSERT_EQ(waiting_leases.size(), 1); + ASSERT_EQ(waiting_leases.front(), lease_id); + } else { + // The subsequent evictions shouldn't cause any more leases to go back to + // the waiting state. + ASSERT_TRUE(waiting_leases.empty()); + } + } + + // Tell the lease dependency manager that each of the arguments is available + // again. + for (size_t i = 0; i < arguments.size(); i++) { + std::vector ready_leases; + ready_leases = lease_dependency_manager_.HandleObjectLocal(arguments[i]); + if (i == arguments.size() - 1) { + ASSERT_EQ(ready_leases.size(), 1); + ASSERT_EQ(ready_leases.front(), lease_id); + } else { + ASSERT_TRUE(ready_leases.empty()); + } + } + + lease_dependency_manager_.RemoveLeaseDependencies(lease_id); + AssertNoLeaks(); +} + +/// Test `ray.get`. Worker calls ray.get on {oid1}, then {oid1, oid2}, then +/// {oid1, oid2, oid3}. +TEST_F(LeaseDependencyManagerTest, TestGet) { + WorkerID worker_id = WorkerID::FromRandom(); + int num_arguments = 3; + std::vector arguments; + for (int i = 0; i < num_arguments; i++) { + // Add the new argument to the list of dependencies to subscribe to. + ObjectID argument_id = ObjectID::FromRandom(); + arguments.push_back(argument_id); + // Subscribe to the lease's dependencies. All arguments except the last are + // duplicates of previous subscription calls. Each argument should only be + // requested from the node manager once. + auto prev_pull_reqs = object_manager_mock_.active_get_requests; + lease_dependency_manager_.StartOrUpdateGetRequest(worker_id, + ObjectIdsToRefs(arguments)); + // Previous pull request for this get should be canceled upon each new + // bundle. + ASSERT_EQ(object_manager_mock_.active_get_requests.size(), 1); + ASSERT_NE(object_manager_mock_.active_get_requests, prev_pull_reqs); + } + + // Nothing happens if the same bundle is requested. + auto prev_pull_reqs = object_manager_mock_.active_get_requests; + lease_dependency_manager_.StartOrUpdateGetRequest(worker_id, + ObjectIdsToRefs(arguments)); + ASSERT_EQ(object_manager_mock_.active_get_requests, prev_pull_reqs); + + // Cancel the pull request once the worker cancels the `ray.get`. + lease_dependency_manager_.CancelGetRequest(worker_id); + AssertNoLeaks(); +} + +/// Test that when one of the objects becomes local after a `ray.wait` call, +/// all requests to remote nodes associated with the object are canceled. +TEST_F(LeaseDependencyManagerTest, TestWait) { + // Generate a random worker and objects to wait on. + WorkerID worker_id = WorkerID::FromRandom(); + int num_objects = 3; + std::vector oids; + for (int i = 0; i < num_objects; i++) { + oids.push_back(ObjectID::FromRandom()); + } + lease_dependency_manager_.StartOrUpdateWaitRequest(worker_id, ObjectIdsToRefs(oids)); + ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects); + + for (int i = 0; i < num_objects; i++) { + // Object is local. + auto ready_lease_ids = lease_dependency_manager_.HandleObjectLocal(oids[i]); + + // Local object gets evicted. The `ray.wait` call should not be + // reactivated. + auto waiting_lease_ids = lease_dependency_manager_.HandleObjectMissing(oids[i]); + ASSERT_TRUE(waiting_lease_ids.empty()); + ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects - i - 1); + } + AssertNoLeaks(); +} + +/// Test that when no objects are locally available, a `ray.wait` call makes +/// the correct requests to remote nodes and correctly cancels the requests +/// when the `ray.wait` call is canceled. +TEST_F(LeaseDependencyManagerTest, TestWaitThenCancel) { + // Generate a random worker and objects to wait on. + WorkerID worker_id = WorkerID::FromRandom(); + int num_objects = 3; + std::vector oids; + for (int i = 0; i < num_objects; i++) { + oids.push_back(ObjectID::FromRandom()); + } + // Simulate a worker calling `ray.wait` on some objects. + lease_dependency_manager_.StartOrUpdateWaitRequest(worker_id, ObjectIdsToRefs(oids)); + ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects); + // Check that it's okay to call `ray.wait` on the same objects again. No new + // calls should be made to try and make the objects local. + lease_dependency_manager_.StartOrUpdateWaitRequest(worker_id, ObjectIdsToRefs(oids)); + ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects); + // Cancel the worker's `ray.wait`. + lease_dependency_manager_.CancelWaitRequest(worker_id); + AssertNoLeaks(); +} + +/// Test that when one of the objects is already local at the time of the +/// `ray.wait` call, the `ray.wait` call does not trigger any requests to +/// remote nodes for that object. +TEST_F(LeaseDependencyManagerTest, TestWaitObjectLocal) { + // Generate a random worker and objects to wait on. + WorkerID worker_id = WorkerID::FromRandom(); + int num_objects = 3; + std::vector oids; + for (int i = 0; i < num_objects; i++) { + oids.push_back(ObjectID::FromRandom()); + } + // Simulate one of the objects becoming local. The later `ray.wait` call + // should have no effect because the object is already local. + const ObjectID local_object_id = std::move(oids.back()); + auto ready_lease_ids = lease_dependency_manager_.HandleObjectLocal(local_object_id); + ASSERT_TRUE(ready_lease_ids.empty()); + lease_dependency_manager_.StartOrUpdateWaitRequest(worker_id, ObjectIdsToRefs(oids)); + ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects - 1); + // Simulate the local object getting evicted. The `ray.wait` call should not + // be reactivated. + auto waiting_lease_ids = lease_dependency_manager_.HandleObjectMissing(local_object_id); + ASSERT_TRUE(waiting_lease_ids.empty()); + ASSERT_EQ(object_manager_mock_.active_wait_requests.size(), num_objects - 1); + // Cancel the worker's `ray.wait`. + lease_dependency_manager_.CancelWaitRequest(worker_id); + AssertNoLeaks(); +} + +/// Test requesting the dependencies for a lease. The dependency manager should +/// return the lease ID as ready once all of its unique arguments are local. +TEST_F(LeaseDependencyManagerTest, TestDuplicateLeaseArgs) { + // Create a lease with 3 arguments. + int num_arguments = 3; + auto obj_id = ObjectID::FromRandom(); + std::vector arguments; + for (int i = 0; i < num_arguments; i++) { + arguments.push_back(obj_id); + } + LeaseID lease_id = LeaseID::FromRandom(); + bool ready = lease_dependency_manager_.RequestLeaseDependencies( + lease_id, ObjectIdsToRefs(arguments), {"", false}); + ASSERT_FALSE(ready); + ASSERT_EQ(object_manager_mock_.active_lease_requests.size(), 1); + + auto ready_lease_ids = lease_dependency_manager_.HandleObjectLocal(obj_id); + ASSERT_EQ(ready_lease_ids.size(), 1); + ASSERT_EQ(ready_lease_ids.front(), lease_id); + lease_dependency_manager_.RemoveLeaseDependencies(lease_id); + + LeaseID lease_id2 = LeaseID::FromRandom(); + ready = lease_dependency_manager_.RequestLeaseDependencies( + lease_id2, ObjectIdsToRefs(arguments), {"", false}); + ASSERT_TRUE(ready); + ASSERT_EQ(object_manager_mock_.active_lease_requests.size(), 1); + lease_dependency_manager_.RemoveLeaseDependencies(lease_id2); + + AssertNoLeaks(); +} + +/// Test that RemoveLeaseDependencies is called before objects +/// becoming local (e.g. the lease is cancelled). +TEST_F(LeaseDependencyManagerTest, TestRemoveLeaseDependenciesBeforeLocal) { + int num_arguments = 3; + std::vector arguments; + for (int i = 0; i < num_arguments; i++) { + arguments.push_back(ObjectID::FromRandom()); + } + LeaseID lease_id = LeaseID::FromRandom(); + bool ready = lease_dependency_manager_.RequestLeaseDependencies( + lease_id, ObjectIdsToRefs(arguments), {"foo", false}); + ASSERT_FALSE(ready); + ASSERT_EQ(NumWaiting("bar"), 0); + ASSERT_EQ(NumWaiting("foo"), 1); + ASSERT_EQ(NumWaitingTotal(), 1); + + // The lease is cancelled + lease_dependency_manager_.RemoveLeaseDependencies(lease_id); + ASSERT_EQ(NumWaiting("foo"), 0); + ASSERT_EQ(NumWaitingTotal(), 0); + AssertNoLeaks(); +} + +} // namespace raylet + +} // namespace ray + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/ray/raylet/tests/local_task_manager_test.cc b/src/ray/raylet/tests/local_lease_manager_test.cc similarity index 75% rename from src/ray/raylet/tests/local_task_manager_test.cc rename to src/ray/raylet/tests/local_lease_manager_test.cc index 0c3dcac90442..d877762a9dc0 100644 --- a/src/ray/raylet/tests/local_task_manager_test.cc +++ b/src/ray/raylet/tests/local_lease_manager_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/raylet/local_task_manager.h" +#include "ray/raylet/local_lease_manager.h" #include #include @@ -28,7 +28,7 @@ #include "mock/ray/gcs/gcs_client/gcs_client.h" #include "mock/ray/object_manager/object_manager.h" #include "ray/common/id.h" -#include "ray/common/task/task.h" +#include "ray/common/lease/lease.h" #include "ray/common/task/task_util.h" #include "ray/common/test_util.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" @@ -42,10 +42,10 @@ class MockWorkerPool : public WorkerPoolInterface { public: MockWorkerPool() : num_pops(0) {} - void PopWorker(const TaskSpecification &task_spec, + void PopWorker(const LeaseSpecification &lease_spec, const PopWorkerCallback &callback) override { num_pops++; - const int runtime_env_hash = task_spec.GetRuntimeEnvHash(); + const int runtime_env_hash = lease_spec.GetRuntimeEnvHash(); callbacks[runtime_env_hash].push_back(callback); } @@ -81,7 +81,7 @@ class MockWorkerPool : public WorkerPoolInterface { RAY_CHECK(status != PopWorkerStatus::OK); for (const auto &pair : callbacks) { for (const auto &callback : pair.second) { - // No task should be dispatched. + // No lease should be granted. ASSERT_FALSE( callback(nullptr, status, @@ -95,23 +95,23 @@ class MockWorkerPool : public WorkerPoolInterface { for (auto it = workers.begin(); it != workers.end();) { std::shared_ptr worker = *it; auto runtime_env_hash = worker->GetRuntimeEnvHash(); - bool dispatched = false; + bool granted = false; auto cb_it = callbacks.find(runtime_env_hash); if (cb_it != callbacks.end()) { auto &list = cb_it->second; RAY_CHECK(!list.empty()); for (auto list_it = list.begin(); list_it != list.end();) { auto &callback = *list_it; - dispatched = callback(worker, PopWorkerStatus::OK, ""); + granted = callback(worker, PopWorkerStatus::OK, ""); list_it = list.erase(list_it); - if (dispatched) { + if (granted) { break; } } if (list.empty()) { callbacks.erase(cb_it); } - if (dispatched) { + if (granted) { it = workers.erase(it); continue; } @@ -208,7 +208,7 @@ class MockWorkerPool : public WorkerPoolInterface { RAY_CHECK(false) << "Not used."; } - void PrestartWorkers(const TaskSpecification &task_spec, + void PrestartWorkers(const LeaseSpecification &lease_spec, int64_t backlog_size) override { RAY_CHECK(false) << "Not used."; } @@ -265,9 +265,9 @@ std::shared_ptr CreateSingleNodeScheduler( return scheduler; } -RayTask CreateTask(const std::unordered_map &required_resources, - const std::string &task_name = "default", - const std::vector> &args = {}) { +RayLease CreateLease(const std::unordered_map &required_resources, + const std::string &task_name = "default", + const std::vector> &args = {}) { TaskSpecBuilder spec_builder; TaskID id = RandomTaskId(); JobID job_id = RandomJobId(); @@ -301,23 +301,26 @@ RayTask CreateTask(const std::unordered_map &required_resou spec_builder.AddArg(*arg); } - return RayTask(std::move(spec_builder).ConsumeAndBuild()); + TaskSpecification spec = std::move(spec_builder).ConsumeAndBuild(); + LeaseSpecification lease_spec(spec.GetMessage()); + lease_spec.GetMutableMessage().set_lease_id(LeaseID::FromRandom().Binary()); + return RayLease(std::move(lease_spec)); } } // namespace -class LocalTaskManagerTest : public ::testing::Test { +class LocalLeaseManagerTest : public ::testing::Test { public: - explicit LocalTaskManagerTest(double num_cpus = 3.0) + explicit LocalLeaseManagerTest(double num_cpus = 3.0) : gcs_client_(std::make_unique()), id_(NodeID::FromRandom()), scheduler_(CreateSingleNodeScheduler(id_.Binary(), num_cpus, *gcs_client_)), object_manager_(), - dependency_manager_(object_manager_), - local_task_manager_(std::make_shared( + lease_dependency_manager_(object_manager_), + local_lease_manager_(std::make_shared( id_, *scheduler_, - dependency_manager_, + lease_dependency_manager_, /* get_node_info= */ [this](const NodeID &node_id) -> const rpc::GcsNodeInfo * { if (node_info_.count(node_id) != 0) { @@ -327,7 +330,7 @@ class LocalTaskManagerTest : public ::testing::Test { }, pool_, leased_workers_, - /* get_task_arguments= */ + /* get_lease_arguments= */ [this](const std::vector &object_ids, std::vector> *results) { for (auto &obj_id : object_ids) { @@ -339,7 +342,7 @@ class LocalTaskManagerTest : public ::testing::Test { } return true; }, - /*max_pinned_task_arguments_bytes=*/1000, + /*max_pinned_lease_arguments_bytes=*/1000, /*get_time=*/[this]() { return current_time_ms_; })) {} void SetUp() override { @@ -370,11 +373,11 @@ class LocalTaskManagerTest : public ::testing::Test { absl::flat_hash_map node_info_; MockObjectManager object_manager_; - DependencyManager dependency_manager_; - std::shared_ptr local_task_manager_; + LeaseDependencyManager lease_dependency_manager_; + std::shared_ptr local_lease_manager_; }; -TEST_F(LocalTaskManagerTest, TestTaskDispatchingOrder) { +TEST_F(LocalLeaseManagerTest, TestLeaseGrantingOrder) { // Initial setup: 3 CPUs available. std::shared_ptr worker1 = std::make_shared(WorkerID::FromRandom(), 0); @@ -386,42 +389,42 @@ TEST_F(LocalTaskManagerTest, TestTaskDispatchingOrder) { pool_.PushWorker(std::static_pointer_cast(worker2)); pool_.PushWorker(std::static_pointer_cast(worker3)); - // First batch of tasks: 2 'f' tasks - auto task_f1 = CreateTask({{ray::kCPU_ResourceLabel, 1}}, "f"); - auto task_f2 = CreateTask({{ray::kCPU_ResourceLabel, 1}}, "f"); + // First batch of leases: [f, f] + auto lease_f1 = CreateLease({{ray::kCPU_ResourceLabel, 1}}, "f"); + auto lease_f2 = CreateLease({{ray::kCPU_ResourceLabel, 1}}, "f"); rpc::RequestWorkerLeaseReply reply; - local_task_manager_->WaitForTaskArgsRequests(std::make_shared( - task_f1, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); - local_task_manager_->ScheduleAndDispatchTasks(); + local_lease_manager_->WaitForLeaseArgsRequests(std::make_shared( + lease_f1, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); + local_lease_manager_->ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - local_task_manager_->WaitForTaskArgsRequests(std::make_shared( - task_f2, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); - local_task_manager_->ScheduleAndDispatchTasks(); + local_lease_manager_->WaitForLeaseArgsRequests(std::make_shared( + lease_f2, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); + local_lease_manager_->ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - // Second batch of tasks: [f, f, f, g] - auto task_f3 = CreateTask({{ray::kCPU_ResourceLabel, 1}}, "f"); - auto task_f4 = CreateTask({{ray::kCPU_ResourceLabel, 1}}, "f"); - auto task_f5 = CreateTask({{ray::kCPU_ResourceLabel, 1}}, "f"); - auto task_g1 = CreateTask({{ray::kCPU_ResourceLabel, 1}}, "g"); - local_task_manager_->WaitForTaskArgsRequests(std::make_shared( - task_f3, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); - local_task_manager_->WaitForTaskArgsRequests(std::make_shared( - task_f4, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); - local_task_manager_->WaitForTaskArgsRequests(std::make_shared( - task_f5, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); - local_task_manager_->WaitForTaskArgsRequests(std::make_shared( - task_g1, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); - local_task_manager_->ScheduleAndDispatchTasks(); + // Second batch of leases: [f, f, f, g] + auto lease_f3 = CreateLease({{ray::kCPU_ResourceLabel, 1}}, "f"); + auto lease_f4 = CreateLease({{ray::kCPU_ResourceLabel, 1}}, "f"); + auto lease_f5 = CreateLease({{ray::kCPU_ResourceLabel, 1}}, "f"); + auto lease_g1 = CreateLease({{ray::kCPU_ResourceLabel, 1}}, "g"); + local_lease_manager_->WaitForLeaseArgsRequests(std::make_shared( + lease_f3, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); + local_lease_manager_->WaitForLeaseArgsRequests(std::make_shared( + lease_f4, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); + local_lease_manager_->WaitForLeaseArgsRequests(std::make_shared( + lease_f5, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); + local_lease_manager_->WaitForLeaseArgsRequests(std::make_shared( + lease_g1, false, false, &reply, [] {}, internal::WorkStatus::WAITING)); + local_lease_manager_->ScheduleAndGrantLeases(); pool_.TriggerCallbacks(); - auto tasks_to_dispatch_ = local_task_manager_->GetTaskToDispatch(); - // Only task f in queue now as g is dispatched. - ASSERT_EQ(tasks_to_dispatch_.size(), 1); + auto leases_to_grant_ = local_lease_manager_->GetLeasesToGrant(); + // Out of the leases in the second batch, only lease g is granted due to fair scheduling + ASSERT_EQ(leases_to_grant_.size(), 1); } -TEST_F(LocalTaskManagerTest, TestNoLeakOnImpossibleInfeasibleTask) { - // Note that ideally it shouldn't be possible for an infeasible task to - // be in the local task manager when ScheduleAndDispatchTasks happens. +TEST_F(LocalLeaseManagerTest, TestNoLeakOnImpossibleInfeasibleLease) { + // Note that ideally it shouldn't be possible for an infeasible lease to + // be in the local lease manager when ScheduleAndGrantLeases happens. // See https://github.com/ray-project/ray/pull/52295 for reasons why added this. std::shared_ptr worker1 = @@ -430,43 +433,43 @@ TEST_F(LocalTaskManagerTest, TestNoLeakOnImpossibleInfeasibleTask) { std::make_shared(WorkerID::FromRandom(), 0); pool_.PushWorker(std::static_pointer_cast(worker1)); - // Create 2 tasks that requires 3 CPU's each and are waiting on an arg. + // Create 2 leases that requires 3 CPU's each and are waiting on an arg. auto arg_id = ObjectID::FromRandom(); std::vector> args; args.push_back( std::make_unique(arg_id, rpc::Address{}, "call_site")); - auto task1 = CreateTask({{kCPU_ResourceLabel, 3}}, "f", args); - auto task2 = CreateTask({{kCPU_ResourceLabel, 3}}, "f2", args); + auto lease1 = CreateLease({{kCPU_ResourceLabel, 3}}, "f", args); + auto lease2 = CreateLease({{kCPU_ResourceLabel, 3}}, "f2", args); EXPECT_CALL(object_manager_, Pull(_, _, _)) .WillOnce(::testing::Return(1)) .WillOnce(::testing::Return(2)); - // Submit the tasks to the local task manager. + // Submit the leases to the local lease manager. int num_callbacks_called = 0; auto callback = [&num_callbacks_called]() { ++num_callbacks_called; }; rpc::RequestWorkerLeaseReply reply1; - local_task_manager_->QueueAndScheduleTask(std::make_shared( - task1, false, false, &reply1, callback, internal::WorkStatus::WAITING)); + local_lease_manager_->QueueAndScheduleLease(std::make_shared( + lease1, false, false, &reply1, callback, internal::WorkStatus::WAITING)); rpc::RequestWorkerLeaseReply reply2; - local_task_manager_->QueueAndScheduleTask(std::make_shared( - task2, false, false, &reply2, callback, internal::WorkStatus::WAITING)); + local_lease_manager_->QueueAndScheduleLease(std::make_shared( + lease2, false, false, &reply2, callback, internal::WorkStatus::WAITING)); // Node no longer has cpu. scheduler_->GetLocalResourceManager().DeleteLocalResource( scheduling::ResourceID::CPU()); // Simulate arg becoming local. - local_task_manager_->TasksUnblocked( - {task1.GetTaskSpecification().TaskId(), task2.GetTaskSpecification().TaskId()}); + local_lease_manager_->LeasesUnblocked({lease1.GetLeaseSpecification().LeaseId(), + lease2.GetLeaseSpecification().LeaseId()}); - // Assert that the the correct rpc replies were sent back and the dispatch map is empty. + // Assert that the the correct rpc replies were sent back and the grant map is empty. ASSERT_EQ(reply1.failure_type(), rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_UNSCHEDULABLE); ASSERT_EQ(reply2.failure_type(), rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_UNSCHEDULABLE); ASSERT_EQ(num_callbacks_called, 2); - ASSERT_EQ(local_task_manager_->GetTaskToDispatch().size(), 0); + ASSERT_EQ(local_lease_manager_->GetLeasesToGrant().size(), 0); } int main(int argc, char **argv) { diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index 2f041bf2825b..e33b9fdfb796 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -29,14 +29,14 @@ #include "mock/ray/object_manager/object_directory.h" #include "mock/ray/object_manager/object_manager.h" #include "mock/ray/object_manager/plasma/client.h" -#include "mock/ray/raylet/local_task_manager.h" +#include "mock/ray/raylet/local_lease_manager.h" #include "mock/ray/raylet/worker_pool.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/buffer.h" #include "ray/common/scheduling/cluster_resource_data.h" #include "ray/object_manager/plasma/client.h" #include "ray/raylet/local_object_manager_interface.h" -#include "ray/raylet/scheduling/cluster_task_manager.h" +#include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/tests/util.h" namespace ray::raylet { @@ -183,7 +183,7 @@ class FakePlasmaClient : public plasma::PlasmaClientInterface { objects_in_plasma_; }; -TaskSpecification BuildTaskSpec( +LeaseSpecification BuildLeaseSpec( const std::unordered_map &resources) { TaskSpecBuilder builder; rpc::Address empty_address; @@ -210,11 +210,11 @@ TaskSpecification BuildTaskSpec( 0, TaskID::Nil(), ""); - return std::move(builder).ConsumeAndBuild(); + return LeaseSpecification(std::move(builder).ConsumeAndBuild().GetMessage()); } -TaskSpecBuilder DetachedActorCreationTaskBuilder(const rpc::Address &owner_address, - const ActorID &actor_id) { +LeaseSpecification DetachedActorCreationLeaseSpec(const rpc::Address &owner_address, + const ActorID &actor_id) { rpc::JobConfig config; const FunctionDescriptor function_descriptor = FunctionDescriptorBuilder::BuildPython("x", "", "", ""); @@ -254,7 +254,7 @@ TaskSpecBuilder DetachedActorCreationTaskBuilder(const rpc::Address &owner_addre /*extension_data=*/"", /*allow_out_of_order_execution=*/false, /*root_detached_actor_id=*/actor_id); - return task_spec_builder; + return LeaseSpecification(std::move(task_spec_builder).ConsumeAndBuild().GetMessage()); } } // namespace @@ -263,7 +263,7 @@ TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog) { { // Worker backlog report from a disconnected worker should be ignored. MockWorkerPool worker_pool; - MockLocalTaskManager local_task_manager; + MockLocalLeaseManager local_lease_manager; WorkerID worker_id = WorkerID::FromRandom(); EXPECT_CALL(worker_pool, GetRegisteredWorker(worker_id)) @@ -272,8 +272,8 @@ TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog) { EXPECT_CALL(worker_pool, GetRegisteredDriver(worker_id)) .Times(1) .WillOnce(Return(nullptr)); - EXPECT_CALL(local_task_manager, ClearWorkerBacklog(_)).Times(0); - EXPECT_CALL(local_task_manager, SetWorkerBacklog(_, _, _)).Times(0); + EXPECT_CALL(local_lease_manager, ClearWorkerBacklog(_)).Times(0); + EXPECT_CALL(local_lease_manager, SetWorkerBacklog(_, _, _)).Times(0); rpc::ReportWorkerBacklogRequest request; request.set_worker_id(worker_id.Binary()); @@ -284,13 +284,13 @@ TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog) { [](Status status, std::function success, std::function failure) { }, worker_pool, - local_task_manager); + local_lease_manager); } { // Worker backlog report from a connected driver should be recorded. MockWorkerPool worker_pool; - MockLocalTaskManager local_task_manager; + MockLocalLeaseManager local_lease_manager; WorkerID worker_id = WorkerID::FromRandom(); std::shared_ptr driver = std::make_shared(worker_id, 10); @@ -298,13 +298,13 @@ TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog) { rpc::ReportWorkerBacklogRequest request; request.set_worker_id(worker_id.Binary()); auto backlog_report_1 = request.add_backlog_reports(); - auto task_spec_1 = BuildTaskSpec({{"CPU", 1}}); - backlog_report_1->mutable_resource_spec()->CopyFrom(task_spec_1.GetMessage()); + auto lease_spec_1 = BuildLeaseSpec({{"CPU", 1}}); + backlog_report_1->mutable_lease_spec()->CopyFrom(lease_spec_1.GetMessage()); backlog_report_1->set_backlog_size(1); auto backlog_report_2 = request.add_backlog_reports(); - auto task_spec_2 = BuildTaskSpec({{"GPU", 2}}); - backlog_report_2->mutable_resource_spec()->CopyFrom(task_spec_2.GetMessage()); + auto lease_spec_2 = BuildLeaseSpec({{"GPU", 2}}); + backlog_report_2->mutable_lease_spec()->CopyFrom(lease_spec_2.GetMessage()); backlog_report_2->set_backlog_size(3); rpc::ReportWorkerBacklogReply reply; @@ -314,12 +314,12 @@ TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog) { EXPECT_CALL(worker_pool, GetRegisteredDriver(worker_id)) .Times(1) .WillOnce(Return(driver)); - EXPECT_CALL(local_task_manager, ClearWorkerBacklog(worker_id)).Times(1); - EXPECT_CALL(local_task_manager, - SetWorkerBacklog(task_spec_1.GetSchedulingClass(), worker_id, 1)) + EXPECT_CALL(local_lease_manager, ClearWorkerBacklog(worker_id)).Times(1); + EXPECT_CALL(local_lease_manager, + SetWorkerBacklog(lease_spec_1.GetSchedulingClass(), worker_id, 1)) .Times(1); - EXPECT_CALL(local_task_manager, - SetWorkerBacklog(task_spec_2.GetSchedulingClass(), worker_id, 3)) + EXPECT_CALL(local_lease_manager, + SetWorkerBacklog(lease_spec_2.GetSchedulingClass(), worker_id, 3)) .Times(1); NodeManager::HandleReportWorkerBacklog( @@ -328,13 +328,13 @@ TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog) { [](Status status, std::function success, std::function failure) { }, worker_pool, - local_task_manager); + local_lease_manager); } { // Worker backlog report from a connected worker should be recorded. MockWorkerPool worker_pool; - MockLocalTaskManager local_task_manager; + MockLocalLeaseManager local_lease_manager; WorkerID worker_id = WorkerID::FromRandom(); std::shared_ptr worker = std::make_shared(worker_id, 10); @@ -342,13 +342,13 @@ TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog) { rpc::ReportWorkerBacklogRequest request; request.set_worker_id(worker_id.Binary()); auto backlog_report_1 = request.add_backlog_reports(); - auto task_spec_1 = BuildTaskSpec({{"CPU", 1}}); - backlog_report_1->mutable_resource_spec()->CopyFrom(task_spec_1.GetMessage()); + auto lease_spec_1 = BuildLeaseSpec({{"CPU", 1}}); + backlog_report_1->mutable_lease_spec()->CopyFrom(lease_spec_1.GetMessage()); backlog_report_1->set_backlog_size(1); auto backlog_report_2 = request.add_backlog_reports(); - auto task_spec_2 = BuildTaskSpec({{"GPU", 2}}); - backlog_report_2->mutable_resource_spec()->CopyFrom(task_spec_2.GetMessage()); + auto lease_spec_2 = BuildLeaseSpec({{"GPU", 2}}); + backlog_report_2->mutable_lease_spec()->CopyFrom(lease_spec_2.GetMessage()); backlog_report_2->set_backlog_size(3); rpc::ReportWorkerBacklogReply reply; @@ -357,12 +357,12 @@ TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog) { .WillOnce(Return(worker)); EXPECT_CALL(worker_pool, GetRegisteredDriver(worker_id)).Times(0); - EXPECT_CALL(local_task_manager, ClearWorkerBacklog(worker_id)).Times(1); - EXPECT_CALL(local_task_manager, - SetWorkerBacklog(task_spec_1.GetSchedulingClass(), worker_id, 1)) + EXPECT_CALL(local_lease_manager, ClearWorkerBacklog(worker_id)).Times(1); + EXPECT_CALL(local_lease_manager, + SetWorkerBacklog(lease_spec_1.GetSchedulingClass(), worker_id, 1)) .Times(1); - EXPECT_CALL(local_task_manager, - SetWorkerBacklog(task_spec_2.GetSchedulingClass(), worker_id, 3)) + EXPECT_CALL(local_lease_manager, + SetWorkerBacklog(lease_spec_2.GetSchedulingClass(), worker_id, 3)) .Times(1); NodeManager::HandleReportWorkerBacklog( @@ -371,7 +371,7 @@ TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog) { [](Status status, std::function success, std::function failure) { }, worker_pool, - local_task_manager); + local_lease_manager); } } @@ -418,7 +418,8 @@ class NodeManagerTest : public ::testing::Test { local_object_manager_ = std::make_unique(objects_pending_deletion_); - dependency_manager_ = std::make_unique(*mock_object_manager_); + lease_dependency_manager_ = + std::make_unique(*mock_object_manager_); cluster_resource_scheduler_ = std::make_unique( io_service_, @@ -458,10 +459,10 @@ class NodeManagerTest : public ::testing::Test { static_cast(mock_object_manager_->GetMemoryCapacity()) * RayConfig::instance().max_task_args_memory_fraction()); - local_task_manager_ = std::make_unique( + local_lease_manager_ = std::make_unique( raylet_node_id_, *cluster_resource_scheduler_, - *dependency_manager_, + *lease_dependency_manager_, get_node_info_func, mock_worker_pool_, leased_workers_, @@ -471,12 +472,12 @@ class NodeManagerTest : public ::testing::Test { }, max_task_args_memory); - cluster_task_manager_ = std::make_unique( + cluster_lease_manager_ = std::make_unique( raylet_node_id_, *cluster_resource_scheduler_, get_node_info_func, - [](const ray::RayTask &task) {}, - *local_task_manager_); + [](const ray::RayLease &lease) {}, + *local_lease_manager_); node_manager_ = std::make_unique(io_service_, raylet_node_id_, @@ -488,12 +489,12 @@ class NodeManagerTest : public ::testing::Test { raylet_client_pool_, *core_worker_subscriber_, *cluster_resource_scheduler_, - *local_task_manager_, - *cluster_task_manager_, + *local_lease_manager_, + *cluster_lease_manager_, *mock_object_directory_, *mock_object_manager_, *local_object_manager_, - *dependency_manager_, + *lease_dependency_manager_, mock_worker_pool_, leased_workers_, *mock_store_client_, @@ -510,10 +511,10 @@ class NodeManagerTest : public ::testing::Test { NodeID raylet_node_id_; std::unique_ptr core_worker_subscriber_; std::unique_ptr cluster_resource_scheduler_; - std::unique_ptr local_task_manager_; - std::unique_ptr cluster_task_manager_; + std::unique_ptr local_lease_manager_; + std::unique_ptr cluster_lease_manager_; std::shared_ptr local_object_manager_; - std::unique_ptr dependency_manager_; + std::unique_ptr lease_dependency_manager_; std::unique_ptr mock_gcs_client_ = std::make_unique(); std::unique_ptr mock_object_directory_; @@ -574,7 +575,7 @@ TEST_F(NodeManagerTest, TestDetachedWorkerIsKilledByFailedWorker) { PopWorkerCallback pop_worker_callback; EXPECT_CALL(mock_worker_pool_, PopWorker(_, _)) .WillOnce( - [&](const TaskSpecification &task_spec, const PopWorkerCallback &callback) { + [&](const LeaseSpecification &lease_spec, const PopWorkerCallback &callback) { pop_worker_callback = callback; }); @@ -600,15 +601,14 @@ TEST_F(NodeManagerTest, TestDetachedWorkerIsKilledByFailedWorker) { owner_address.set_worker_id(owner_worker_id.Binary()); const auto actor_id = ActorID::Of(JobID::FromInt(1), TaskID::FromRandom(JobID::FromInt(1)), 0); - const auto task_spec_builder = - DetachedActorCreationTaskBuilder(owner_address, actor_id); + const auto lease_spec = DetachedActorCreationLeaseSpec(owner_address, actor_id); // Invoke RequestWorkerLease to request a leased worker for the task in the // NodeManager. std::promise promise; rpc::RequestWorkerLeaseReply reply; rpc::RequestWorkerLeaseRequest request; - request.mutable_resource_spec()->CopyFrom(task_spec_builder.GetMessage()); + request.mutable_lease_spec()->CopyFrom(lease_spec.GetMessage()); node_manager_->HandleRequestWorkerLease( request, &reply, @@ -653,7 +653,7 @@ TEST_F(NodeManagerTest, TestDetachedWorkerIsKilledByFailedNode) { PopWorkerCallback pop_worker_callback; EXPECT_CALL(mock_worker_pool_, PopWorker(_, _)) .WillOnce( - [&](const TaskSpecification &task_spec, const PopWorkerCallback &callback) { + [&](const LeaseSpecification &lease_spec, const PopWorkerCallback &callback) { pop_worker_callback = callback; }); @@ -678,15 +678,14 @@ TEST_F(NodeManagerTest, TestDetachedWorkerIsKilledByFailedNode) { owner_address.set_node_id(owner_node_id.Binary()); const auto actor_id = ActorID::Of(JobID::FromInt(1), TaskID::FromRandom(JobID::FromInt(1)), 0); - const auto task_spec_builder = - DetachedActorCreationTaskBuilder(owner_address, actor_id); + const auto lease_spec = DetachedActorCreationLeaseSpec(owner_address, actor_id); // Invoke RequestWorkerLease to request a leased worker for the task in the // NodeManager. std::promise promise; rpc::RequestWorkerLeaseReply reply; rpc::RequestWorkerLeaseRequest request; - request.mutable_resource_spec()->CopyFrom(task_spec_builder.GetMessage()); + request.mutable_lease_spec()->CopyFrom(lease_spec.GetMessage()); node_manager_->HandleRequestWorkerLease( request, &reply, diff --git a/src/ray/raylet/tests/util.h b/src/ray/raylet/tests/util.h index aee501c99870..7f1372bb5ef7 100644 --- a/src/ray/raylet/tests/util.h +++ b/src/ray/raylet/tests/util.h @@ -42,20 +42,22 @@ class MockWorker : public WorkerInterface { void SetOwnerAddress(const rpc::Address &address) override { address_ = address; } - void AssignTaskId(const TaskID &task_id) override { task_id_ = task_id; } - - void SetAssignedTask(const RayTask &assigned_task) override { - task_ = assigned_task; - task_assign_time_ = absl::Now(); - root_detached_actor_id_ = assigned_task.GetTaskSpecification().RootDetachedActorId(); - const auto &task_spec = assigned_task.GetTaskSpecification(); - SetJobId(task_spec.JobId()); - SetBundleId(task_spec.PlacementGroupBundleId()); - SetOwnerAddress(task_spec.CallerAddress()); - AssignTaskId(task_spec.TaskId()); + void GrantLease(const RayLease &granted_lease) override { + lease_ = granted_lease; + lease_grant_time_ = absl::Now(); + root_detached_actor_id_ = granted_lease.GetLeaseSpecification().RootDetachedActorId(); + const auto &lease_spec = granted_lease.GetLeaseSpecification(); + SetJobId(lease_spec.JobId()); + SetBundleId(lease_spec.PlacementGroupBundleId()); + SetOwnerAddress(lease_spec.CallerAddress()); + GrantLeaseId(lease_spec.LeaseId()); }; - absl::Time GetAssignedTaskTime() const override { return task_assign_time_; }; + void GrantLeaseId(const LeaseID &lease_id) override { lease_id_ = lease_id; } + + const RayLease &GetGrantedLease() const override { return lease_; } + + absl::Time GetGrantedLeaseTime() const override { return lease_grant_time_; }; std::optional GetIsGpu() const override { return is_gpu_; } @@ -116,7 +118,7 @@ class MockWorker : public WorkerInterface { return -1; } void SetAssignedPort(int port) override { RAY_CHECK(false) << "Method unused"; } - const TaskID &GetAssignedTaskId() const override { return task_id_; } + const LeaseID &GetGrantedLeaseId() const override { return lease_id_; } const JobID &GetAssignedJobId() const override { return job_id_; } int GetRuntimeEnvHash() const override { return runtime_env_hash_; } void AssignActorId(const ActorID &actor_id) override { @@ -126,13 +128,13 @@ class MockWorker : public WorkerInterface { RAY_CHECK(false) << "Method unused"; return ActorID::Nil(); } - const std::string GetTaskOrActorIdAsDebugString() const override { + const std::string GetLeaseIdAsDebugString() const override { RAY_CHECK(false) << "Method unused"; return ""; } bool IsDetachedActor() const override { - return task_.GetTaskSpecification().IsDetachedActor(); + return lease_.GetLeaseSpecification().IsDetachedActor(); } const std::shared_ptr Connection() const override { @@ -158,7 +160,7 @@ class MockWorker : public WorkerInterface { void SetBundleId(const BundleID &bundle_id) override { bundle_id_ = bundle_id; } - RayTask &GetAssignedTask() override { return task_; } + RayLease &GetGrantedLease() override { return lease_; } bool IsRegistered() override { RAY_CHECK(false) << "Method unused"; @@ -197,10 +199,10 @@ class MockWorker : public WorkerInterface { std::optional is_actor_worker_; BundleID bundle_id_; bool blocked_ = false; - RayTask task_; - absl::Time task_assign_time_; + RayLease lease_; + absl::Time lease_grant_time_; int runtime_env_hash_; - TaskID task_id_; + LeaseID lease_id_; JobID job_id_; ActorID root_detached_actor_id_; Process proc_; diff --git a/src/ray/raylet/tests/worker_killing_policy_group_by_owner_test.cc b/src/ray/raylet/tests/worker_killing_policy_group_by_owner_test.cc index 965712213786..93b9b5f8f718 100644 --- a/src/ray/raylet/tests/worker_killing_policy_group_by_owner_test.cc +++ b/src/ray/raylet/tests/worker_killing_policy_group_by_owner_test.cc @@ -20,7 +20,7 @@ #include #include "gtest/gtest.h" -#include "ray/common/task/task_spec.h" +#include "ray/common/lease/lease_spec.h" #include "ray/raylet/tests/util.h" #include "ray/raylet/worker_killing_policy.h" @@ -41,31 +41,31 @@ class WorkerKillingGroupByOwnerTest : public ::testing::Test { std::shared_ptr CreateActorCreationWorker(TaskID owner_id, int32_t max_restarts) { - rpc::TaskSpec message; - message.set_task_id(TaskID::FromRandom(job_id_).Binary()); + rpc::LeaseSpec message; + message.set_lease_id(LeaseID::FromRandom().Binary()); message.set_parent_task_id(owner_id.Binary()); - message.mutable_actor_creation_task_spec()->set_max_actor_restarts(max_restarts); message.set_type(ray::rpc::TaskType::ACTOR_CREATION_TASK); - TaskSpecification task_spec(message); - RayTask task(task_spec); + message.set_max_actor_restarts(max_restarts); + LeaseSpecification lease_spec(message); + RayLease lease(lease_spec); auto worker = std::make_shared(ray::WorkerID::FromRandom(), port_); - worker->SetAssignedTask(task); - worker->AssignTaskId(task.GetTaskSpecification().TaskId()); + worker->GrantLease(lease); + worker->GrantLeaseId(lease.GetLeaseSpecification().LeaseId()); return worker; } std::shared_ptr CreateTaskWorker(TaskID owner_id, int32_t max_retries) { - rpc::TaskSpec message; - message.set_task_id(TaskID::FromRandom(job_id_).Binary()); + rpc::LeaseSpec message; + message.set_lease_id(LeaseID::FromRandom().Binary()); message.set_parent_task_id(owner_id.Binary()); - message.set_max_retries(max_retries); message.set_type(ray::rpc::TaskType::NORMAL_TASK); - TaskSpecification task_spec(message); - RayTask task(task_spec); + message.set_max_retries(max_retries); + LeaseSpecification lease_spec(message); + RayLease lease(lease_spec); auto worker = std::make_shared(ray::WorkerID::FromRandom(), port_); - worker->SetAssignedTask(task); - worker->AssignTaskId(task.GetTaskSpecification().TaskId()); + worker->GrantLease(lease); + worker->GrantLeaseId(lease.GetLeaseSpecification().LeaseId()); return worker; } }; diff --git a/src/ray/raylet/tests/worker_killing_policy_retriable_fifo_test.cc b/src/ray/raylet/tests/worker_killing_policy_retriable_fifo_test.cc index 27d07b7e5417..9026e26b836a 100644 --- a/src/ray/raylet/tests/worker_killing_policy_retriable_fifo_test.cc +++ b/src/ray/raylet/tests/worker_killing_policy_retriable_fifo_test.cc @@ -18,7 +18,7 @@ #include #include "gtest/gtest.h" -#include "ray/common/task/task_spec.h" +#include "ray/common/lease/lease_spec.h" #include "ray/raylet/tests/util.h" #include "ray/raylet/worker_killing_policy.h" @@ -32,24 +32,24 @@ class WorkerKillerTest : public ::testing::Test { RetriableFIFOWorkerKillingPolicy worker_killing_policy_; std::shared_ptr CreateActorCreationWorker(int32_t max_restarts) { - rpc::TaskSpec message; - message.mutable_actor_creation_task_spec()->set_max_actor_restarts(max_restarts); + rpc::LeaseSpec message; + message.set_max_actor_restarts(max_restarts); message.set_type(ray::rpc::TaskType::ACTOR_CREATION_TASK); - TaskSpecification task_spec(message); - RayTask task(task_spec); + LeaseSpecification lease_spec(message); + RayLease lease(lease_spec); auto worker = std::make_shared(ray::WorkerID::FromRandom(), port_); - worker->SetAssignedTask(task); + worker->GrantLease(lease); return worker; } std::shared_ptr CreateTaskWorker(int32_t max_retries) { - rpc::TaskSpec message; + rpc::LeaseSpec message; message.set_max_retries(max_retries); message.set_type(ray::rpc::TaskType::NORMAL_TASK); - TaskSpecification task_spec(message); - RayTask task(task_spec); + LeaseSpecification lease_spec(message); + RayLease lease(lease_spec); auto worker = std::make_shared(ray::WorkerID::FromRandom(), port_); - worker->SetAssignedTask(task); + worker->GrantLease(lease); return worker; } }; @@ -68,9 +68,9 @@ TEST_F(WorkerKillerTest, auto first_submitted = WorkerKillerTest::CreateActorCreationWorker(0 /* max_restarts */); auto second_submitted = - WorkerKillerTest::CreateActorCreationWorker(5 /* max_restarts */); + WorkerKillerTest::CreateActorCreationWorker(1 /* max_restarts */); auto third_submitted = WorkerKillerTest::CreateTaskWorker(0 /* max_restarts */); - auto fourth_submitted = WorkerKillerTest::CreateTaskWorker(11 /* max_restarts */); + auto fourth_submitted = WorkerKillerTest::CreateTaskWorker(1 /* max_restarts */); workers.push_back(first_submitted); workers.push_back(second_submitted); diff --git a/src/ray/raylet/tests/worker_killing_policy_test.cc b/src/ray/raylet/tests/worker_killing_policy_test.cc index 1026b616c09d..dca60ad6f58c 100644 --- a/src/ray/raylet/tests/worker_killing_policy_test.cc +++ b/src/ray/raylet/tests/worker_killing_policy_test.cc @@ -18,7 +18,7 @@ #include #include "gtest/gtest.h" -#include "ray/common/task/task_spec.h" +#include "ray/common/lease/lease_spec.h" #include "ray/raylet/tests/util.h" namespace ray { @@ -31,36 +31,25 @@ class WorkerKillerTest : public ::testing::Test { int32_t port_ = 2389; RetriableLIFOWorkerKillingPolicy worker_killing_policy_; - std::shared_ptr CreateActorWorker(int32_t max_restarts) { - rpc::TaskSpec message; - message.mutable_actor_creation_task_spec()->set_max_actor_restarts(max_restarts); - message.set_type(ray::rpc::TaskType::ACTOR_TASK); - TaskSpecification task_spec(message); - RayTask task(task_spec); - auto worker = std::make_shared(ray::WorkerID::FromRandom(), port_); - worker->SetAssignedTask(task); - return worker; - } - std::shared_ptr CreateActorCreationWorker(int32_t max_restarts) { - rpc::TaskSpec message; - message.mutable_actor_creation_task_spec()->set_max_actor_restarts(max_restarts); + rpc::LeaseSpec message; + message.set_max_actor_restarts(max_restarts); message.set_type(ray::rpc::TaskType::ACTOR_CREATION_TASK); - TaskSpecification task_spec(message); - RayTask task(task_spec); + LeaseSpecification lease_spec(message); + RayLease lease(lease_spec); auto worker = std::make_shared(ray::WorkerID::FromRandom(), port_); - worker->SetAssignedTask(task); + worker->GrantLease(lease); return worker; } std::shared_ptr CreateTaskWorker(int32_t max_retries) { - rpc::TaskSpec message; + rpc::LeaseSpec message; message.set_max_retries(max_retries); message.set_type(ray::rpc::TaskType::NORMAL_TASK); - TaskSpecification task_spec(message); - RayTask task(task_spec); + LeaseSpecification lease_spec(message); + RayLease lease(lease_spec); auto worker = std::make_shared(ray::WorkerID::FromRandom(), port_); - worker->SetAssignedTask(task); + worker->GrantLease(lease); return worker; } }; @@ -76,14 +65,15 @@ TEST_F(WorkerKillerTest, TestEmptyWorkerPoolSelectsNullWorker) { TEST_F(WorkerKillerTest, TestPreferRetriableOverNonRetriableAndOrderByTimestampDescending) { std::vector> workers; - auto first_submitted = WorkerKillerTest::CreateActorWorker(7 /* max_restarts */); + auto first_submitted = + WorkerKillerTest::CreateActorCreationWorker(false /* max_restarts */); auto second_submitted = - WorkerKillerTest::CreateActorCreationWorker(5 /* max_restarts */); - auto third_submitted = WorkerKillerTest::CreateTaskWorker(0 /* max_restarts */); - auto fourth_submitted = WorkerKillerTest::CreateTaskWorker(11 /* max_restarts */); + WorkerKillerTest::CreateActorCreationWorker(true /* max_restarts */); + auto third_submitted = WorkerKillerTest::CreateTaskWorker(false /* max_retries */); + auto fourth_submitted = WorkerKillerTest::CreateTaskWorker(true /* max_retries */); auto fifth_submitted = - WorkerKillerTest::CreateActorCreationWorker(0 /* max_restarts */); - auto sixth_submitted = WorkerKillerTest::CreateActorWorker(0 /* max_restarts */); + WorkerKillerTest::CreateActorCreationWorker(false /* max_restarts */); + auto sixth_submitted = WorkerKillerTest::CreateTaskWorker(true /* max_retries */); workers.push_back(first_submitted); workers.push_back(second_submitted); @@ -93,9 +83,9 @@ TEST_F(WorkerKillerTest, workers.push_back(sixth_submitted); std::vector> expected_order; + expected_order.push_back(sixth_submitted); expected_order.push_back(fourth_submitted); expected_order.push_back(second_submitted); - expected_order.push_back(sixth_submitted); expected_order.push_back(fifth_submitted); expected_order.push_back(third_submitted); expected_order.push_back(first_submitted); diff --git a/src/ray/raylet/tests/worker_pool_test.cc b/src/ray/raylet/tests/worker_pool_test.cc index 490619ebfa4b..92e0e6556059 100644 --- a/src/ray/raylet/tests/worker_pool_test.cc +++ b/src/ray/raylet/tests/worker_pool_test.cc @@ -31,6 +31,7 @@ #include "ray/common/asio/asio_util.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/constants.h" +#include "ray/common/lease/lease_spec.h" #include "ray/raylet/runtime_env_agent_client.h" #include "ray/util/path_utils.h" #include "ray/util/process.h" @@ -364,14 +365,14 @@ class WorkerPoolMock : public WorkerPool { // \param[in] push_workers If true, tries to push the workers from the started // processes. std::shared_ptr PopWorkerSync( - const TaskSpecification &task_spec, + const LeaseSpecification &lease_spec, bool push_workers = true, PopWorkerStatus *worker_status = nullptr, int timeout_worker_number = 0, std::string *runtime_env_error_msg = nullptr) { std::shared_ptr popped_worker = nullptr; std::promise promise; - this->PopWorker(task_spec, + this->PopWorker(lease_spec, [&popped_worker, worker_status, &promise, runtime_env_error_msg]( const std::shared_ptr worker, PopWorkerStatus status, @@ -387,7 +388,7 @@ class WorkerPoolMock : public WorkerPool { return true; }); if (push_workers) { - PushWorkers(timeout_worker_number, task_spec.JobId()); + PushWorkers(timeout_worker_number, lease_spec.JobId()); } promise.get_future().get(); return popped_worker; @@ -457,7 +458,7 @@ class WorkerPoolTest : public ::testing::Test { const rpc::JobConfig &job_config = rpc::JobConfig()) { auto driver = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, job_id); - driver->AssignTaskId(TaskID::ForDriverTask(job_id)); + driver->GrantLeaseId(LeaseID::FromRandom()); RAY_CHECK_OK(worker_pool_->RegisterDriver(driver, job_config, [](Status, int) {})); return driver; } @@ -529,29 +530,29 @@ static inline rpc::RuntimeEnvInfo ExampleRuntimeEnvInfoFromString( return runtime_env_info; } -static inline TaskSpecification ExampleTaskSpec( +static inline LeaseSpecification ExampleLeaseSpec( const ActorID actor_id = ActorID::Nil(), const Language &language = Language::PYTHON, const JobID &job_id = JOB_ID, const ActorID actor_creation_id = ActorID::Nil(), const std::vector &dynamic_worker_options = {}, - const TaskID &task_id = TaskID::FromRandom(JobID::Nil()), + const LeaseID &lease_id = LeaseID::FromRandom(), const rpc::RuntimeEnvInfo runtime_env_info = rpc::RuntimeEnvInfo(), std::unordered_map resources = {{"CPU", 1}}) { - rpc::TaskSpec message; + rpc::LeaseSpec message; message.set_job_id(job_id.Binary()); message.set_language(language); - // Make sure no reduplicative task id. - RAY_CHECK(!task_id.IsNil()); - message.set_task_id(task_id.Binary()); + // Make sure no reduplicative lease id. + RAY_CHECK(!lease_id.IsNil()); + message.set_lease_id(lease_id.Binary()); if (!actor_id.IsNil()) { message.set_type(TaskType::ACTOR_TASK); - message.mutable_actor_task_spec()->set_actor_id(actor_id.Binary()); + message.set_actor_id(actor_id.Binary()); } else if (!actor_creation_id.IsNil()) { message.set_type(TaskType::ACTOR_CREATION_TASK); - message.mutable_actor_creation_task_spec()->set_actor_id(actor_creation_id.Binary()); + message.set_actor_id(actor_creation_id.Binary()); for (const auto &option : dynamic_worker_options) { - message.mutable_actor_creation_task_spec()->add_dynamic_worker_options(option); + message.add_dynamic_worker_options(option); } } else { message.set_type(TaskType::NORMAL_TASK); @@ -559,7 +560,7 @@ static inline TaskSpecification ExampleTaskSpec( message.mutable_required_resources()->insert(resources.begin(), resources.end()); message.mutable_runtime_env_info()->CopyFrom(runtime_env_info); - return TaskSpecification(std::move(message)); + return LeaseSpecification(std::move(message)); } TEST_F(WorkerPoolDriverRegisteredTest, CompareWorkerProcessObjects) { @@ -650,42 +651,42 @@ TEST_F(WorkerPoolDriverRegisteredTest, InitialWorkerProcessCount) { } TEST_F(WorkerPoolDriverRegisteredTest, TestPrestartingWorkers) { - const auto task_spec = ExampleTaskSpec(); + const auto lease_spec = ExampleLeaseSpec(); // Prestarts 2 workers. - worker_pool_->PrestartWorkers(task_spec, 2); + worker_pool_->PrestartWorkers(lease_spec, 2); ASSERT_EQ(worker_pool_->NumWorkersStarting(), 2); // Prestarts 1 more worker. - worker_pool_->PrestartWorkers(task_spec, 3); + worker_pool_->PrestartWorkers(lease_spec, 3); ASSERT_EQ(worker_pool_->NumWorkersStarting(), 3); // No more needed. - worker_pool_->PrestartWorkers(task_spec, 1); + worker_pool_->PrestartWorkers(lease_spec, 1); ASSERT_EQ(worker_pool_->NumWorkersStarting(), 3); // Capped by soft limit. - worker_pool_->PrestartWorkers(task_spec, 20); + worker_pool_->PrestartWorkers(lease_spec, 20); ASSERT_EQ(worker_pool_->NumWorkersStarting(), POOL_SIZE_SOFT_LIMIT); } TEST_F(WorkerPoolDriverRegisteredTest, TestPrestartingWorkersWithRuntimeEnv) { - auto task_spec = ExampleTaskSpec(); - task_spec.GetMutableMessage().mutable_runtime_env_info()->set_serialized_runtime_env( + auto lease_spec = ExampleLeaseSpec(); + lease_spec.GetMutableMessage().mutable_runtime_env_info()->set_serialized_runtime_env( "{\"env_vars\": {\"FOO\": \"bar\"}}"); // Prestarts 2 workers. - worker_pool_->PrestartWorkers(task_spec, 2); + worker_pool_->PrestartWorkers(lease_spec, 2); ASSERT_EQ(worker_pool_->NumWorkersStarting(), 2); // Prestarts 1 more worker. - worker_pool_->PrestartWorkers(task_spec, 3); + worker_pool_->PrestartWorkers(lease_spec, 3); ASSERT_EQ(worker_pool_->NumWorkersStarting(), 3); // No more needed. - worker_pool_->PrestartWorkers(task_spec, 1); + worker_pool_->PrestartWorkers(lease_spec, 1); ASSERT_EQ(worker_pool_->NumWorkersStarting(), 3); // Capped by soft limit. - worker_pool_->PrestartWorkers(task_spec, 20); + worker_pool_->PrestartWorkers(lease_spec, 20); ASSERT_EQ(worker_pool_->NumWorkersStarting(), POOL_SIZE_SOFT_LIMIT); } TEST_F(WorkerPoolDriverRegisteredTest, HandleWorkerPushPop) { std::shared_ptr popped_worker; - const auto task_spec = ExampleTaskSpec(); + const auto lease_spec = ExampleLeaseSpec(); // Create some workers. std::unordered_set> workers; workers.insert(worker_pool_->CreateWorker(Process::CreateNewDummy())); @@ -695,15 +696,15 @@ TEST_F(WorkerPoolDriverRegisteredTest, HandleWorkerPushPop) { worker_pool_->PushWorker(worker); } // Pop two workers and make sure they're one of the workers we created. - popped_worker = worker_pool_->PopWorkerSync(task_spec); + popped_worker = worker_pool_->PopWorkerSync(lease_spec); ASSERT_NE(popped_worker, nullptr); ASSERT_GT(workers.count(popped_worker), 0); - popped_worker = worker_pool_->PopWorkerSync(task_spec); + popped_worker = worker_pool_->PopWorkerSync(lease_spec); ASSERT_NE(popped_worker, nullptr); ASSERT_GT(workers.count(popped_worker), 0); // Pop a worker from the empty pool and make sure it isn't one of the workers we // created. - popped_worker = worker_pool_->PopWorkerSync(task_spec); + popped_worker = worker_pool_->PopWorkerSync(lease_spec); ASSERT_NE(popped_worker, nullptr); ASSERT_EQ(workers.count(popped_worker), 0); } @@ -713,26 +714,26 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerSyncsOfMultipleLanguages) { auto py_worker = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON); worker_pool_->PushWorker(py_worker); - // Check that the Python worker will not be popped if the given task is a Java task - const auto java_task_spec = ExampleTaskSpec(ActorID::Nil(), Language::JAVA); - ASSERT_NE(worker_pool_->PopWorkerSync(java_task_spec), py_worker); - // Check that the Python worker can be popped if the given task is a Python task - const auto py_task_spec = ExampleTaskSpec(ActorID::Nil(), Language::PYTHON); - ASSERT_EQ(worker_pool_->PopWorkerSync(py_task_spec), py_worker); + // Check that the Python worker will not be popped if the given lease is a Java lease + const auto java_lease_spec = ExampleLeaseSpec(ActorID::Nil(), Language::JAVA); + ASSERT_NE(worker_pool_->PopWorkerSync(java_lease_spec), py_worker); + // Check that the Python worker can be popped if the given lease is a Python lease + const auto py_lease_spec = ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON); + ASSERT_EQ(worker_pool_->PopWorkerSync(py_lease_spec), py_worker); // Create a Java Worker, and add it to the pool auto java_worker = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::JAVA); worker_pool_->PushWorker(java_worker); - // Check that the Java worker will be popped now for Java task - ASSERT_EQ(worker_pool_->PopWorkerSync(java_task_spec), java_worker); + // Check that the Java worker will be popped now for Java lease + ASSERT_EQ(worker_pool_->PopWorkerSync(java_lease_spec), java_worker); } TEST_F(WorkerPoolDriverRegisteredTest, StartWorkerWithNodeIdArg) { - auto task_id = TaskID::FromRandom(JOB_ID); - TaskSpecification task_spec = ExampleTaskSpec( - ActorID::Nil(), Language::PYTHON, JOB_ID, ActorID::Nil(), {}, task_id); - ASSERT_NE(worker_pool_->PopWorkerSync(task_spec), nullptr); + auto lease_id = LeaseID::FromRandom(); + LeaseSpecification lease_spec = ExampleLeaseSpec( + ActorID::Nil(), Language::PYTHON, JOB_ID, ActorID::Nil(), {}, lease_id); + ASSERT_NE(worker_pool_->PopWorkerSync(lease_spec), nullptr); const auto real_command = worker_pool_->GetWorkerCommand(worker_pool_->LastStartedWorkerProcess()); @@ -756,10 +757,13 @@ TEST_F(WorkerPoolDriverRegisteredTest, StartWorkerWithDynamicOptionsCommand) { actor_jvm_options.end(), {"-Dmy-actor.hello=foo", "-Dmy-actor.world=bar", "-Xmx2g", "-Xms1g"}); JobID job_id = JobID::FromInt(12345); - auto task_id = TaskID::ForDriverTask(job_id); - auto actor_id = ActorID::Of(job_id, task_id, 1); - TaskSpecification task_spec = ExampleTaskSpec( - ActorID::Nil(), Language::JAVA, job_id, actor_id, actor_jvm_options, task_id); + auto actor_id = ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 1); + LeaseSpecification lease_spec = ExampleLeaseSpec(ActorID::Nil(), + Language::JAVA, + job_id, + actor_id, + actor_jvm_options, + LeaseID::FromRandom()); rpc::JobConfig job_config = rpc::JobConfig(); job_config.add_code_search_path("/test/code_search_path"); @@ -769,7 +773,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, StartWorkerWithDynamicOptionsCommand) { job_config.add_jvm_options("-Dmy-job.foo=bar"); worker_pool_->HandleJobStarted(job_id, job_config); - ASSERT_NE(worker_pool_->PopWorkerSync(task_spec), nullptr); + ASSERT_NE(worker_pool_->PopWorkerSync(lease_spec), nullptr); const auto real_command = worker_pool_->GetWorkerCommand(worker_pool_->LastStartedWorkerProcess()); @@ -840,7 +844,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestWorkerStartupKeepAliveDuration) { ASSERT_EQ(worker_pool_->GetProcessSize(), POOL_SIZE_SOFT_LIMIT + 2); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); - // The worker registered. There's no pending tasks so it becomes idle. + // The worker registered. There's no pending leases so it becomes idle. worker_pool_->PushWorkers(0, JOB_ID); ASSERT_EQ(worker_pool_->NumWorkersStarting(), 0); ASSERT_EQ(worker_pool_->GetProcessSize(), POOL_SIZE_SOFT_LIMIT + 2); @@ -881,9 +885,9 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerMultiTenancy) { // Make the first worker an actor worker. if (i == 0) { auto actor_creation_id = ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 1); - auto task_spec = ExampleTaskSpec( + auto lease_spec = ExampleLeaseSpec( /*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id, actor_creation_id); - runtime_env_hash = task_spec.GetRuntimeEnvHash(); + runtime_env_hash = lease_spec.GetRuntimeEnvHash(); } auto worker = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, @@ -900,19 +904,19 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerMultiTenancy) { // Pop workers for actor. for (auto job_id : job_ids) { auto actor_creation_id = ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 1); - // Pop workers for actor creation tasks. - auto task_spec = ExampleTaskSpec( + // Pop workers for actor creation leases. + auto lease_spec = ExampleLeaseSpec( /*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id, actor_creation_id); - auto worker = worker_pool_->PopWorkerSync(task_spec); + auto worker = worker_pool_->PopWorkerSync(lease_spec); ASSERT_TRUE(worker); ASSERT_EQ(worker->GetAssignedJobId(), job_id); workers.push_back(worker); } - // Pop workers for normal tasks. + // Pop workers for normal leases. for (auto job_id : job_ids) { - auto task_spec = ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, job_id); - auto worker = worker_pool_->PopWorkerSync(task_spec); + auto lease_spec = ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_id); + auto worker = worker_pool_->PopWorkerSync(lease_spec); ASSERT_TRUE(worker); ASSERT_EQ(worker->GetAssignedJobId(), job_id); workers.push_back(worker); @@ -932,8 +936,8 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerMultiTenancy) { } } -// Tests the worker assignment logic for task specs that have a root detached actor ID. -// These tasks: +// Tests the worker assignment logic for lease specs that have a root detached actor ID. +// These leases: // - Must be matched to workers that have a matching job ID (or no job ID). // - Must be matched to workers that have a matching detached actor ID (or no detached // actor ID). @@ -943,10 +947,11 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerForRequestWithRootDetachedActor) // NOTE: in all test cases the request has job_1_detached_actor_1 as its root detached // actor. - auto detached_actor_id_1_job_1 = ActorID::Of(job_1_id, TaskID::FromRandom(job_1_id), 0); - auto task_spec_job_1_detached_actor_1 = - ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, job_1_id); - task_spec_job_1_detached_actor_1.GetMutableMessage().set_root_detached_actor_id( + auto detached_actor_id_1_job_1 = + ActorID::Of(job_1_id, TaskID::ForDriverTask(job_1_id), 0); + auto lease_spec_job_1_detached_actor_1 = + ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_1_id); + lease_spec_job_1_detached_actor_1.GetMutableMessage().set_root_detached_actor_id( detached_actor_id_1_job_1.Binary()); // Case 1 (match): @@ -955,7 +960,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerForRequestWithRootDetachedActor) Process::CreateNewDummy(), Language::PYTHON, JobID::Nil()); worker_pool_->PushWorker(worker_no_job_no_detached_actor); - ASSERT_EQ(worker_pool_->PopWorkerSync(task_spec_job_1_detached_actor_1), + ASSERT_EQ(worker_pool_->PopWorkerSync(lease_spec_job_1_detached_actor_1), worker_no_job_no_detached_actor); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); @@ -965,7 +970,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerForRequestWithRootDetachedActor) worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, job_1_id); worker_pool_->PushWorker(worker_job_1_no_detached_actor); - ASSERT_EQ(worker_pool_->PopWorkerSync(task_spec_job_1_detached_actor_1), + ASSERT_EQ(worker_pool_->PopWorkerSync(lease_spec_job_1_detached_actor_1), worker_job_1_no_detached_actor); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); @@ -973,12 +978,12 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerForRequestWithRootDetachedActor) // worker has matching root detached actor ID and job ID auto worker_job_1_detached_actor_1 = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, job_1_id); - RayTask job_1_detached_actor_1_task(task_spec_job_1_detached_actor_1); - worker_job_1_detached_actor_1->SetAssignedTask(job_1_detached_actor_1_task); - worker_job_1_detached_actor_1->AssignTaskId(TaskID::Nil()); + RayLease job_1_detached_actor_1_lease(lease_spec_job_1_detached_actor_1); + worker_job_1_detached_actor_1->GrantLease(job_1_detached_actor_1_lease); + worker_job_1_detached_actor_1->GrantLeaseId(LeaseID::Nil()); worker_pool_->PushWorker(worker_job_1_detached_actor_1); - ASSERT_EQ(worker_pool_->PopWorkerSync(task_spec_job_1_detached_actor_1), + ASSERT_EQ(worker_pool_->PopWorkerSync(lease_spec_job_1_detached_actor_1), worker_job_1_detached_actor_1); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); @@ -988,7 +993,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerForRequestWithRootDetachedActor) worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, job_2_id); worker_pool_->PushWorker(worker_job_2_no_detached_actor); - ASSERT_NE(worker_pool_->PopWorkerSync(task_spec_job_1_detached_actor_1), + ASSERT_NE(worker_pool_->PopWorkerSync(lease_spec_job_1_detached_actor_1), worker_job_2_no_detached_actor); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 1); worker_job_2_no_detached_actor->MarkDead(); @@ -999,17 +1004,18 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerForRequestWithRootDetachedActor) // worker has mismatched detached actor ID and mismatched job ID auto worker_job_2_detached_actor_3 = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, job_2_id); - auto detached_actor_3_id_job_2 = ActorID::Of(job_2_id, TaskID::FromRandom(job_2_id), 0); - auto task_spec_job_2_detached_actor_3 = - ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, job_2_id); - task_spec_job_2_detached_actor_3.GetMutableMessage().set_root_detached_actor_id( + auto detached_actor_3_id_job_2 = + ActorID::Of(job_2_id, TaskID::ForDriverTask(job_2_id), 0); + auto lease_spec_job_2_detached_actor_3 = + ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_2_id); + lease_spec_job_2_detached_actor_3.GetMutableMessage().set_root_detached_actor_id( detached_actor_3_id_job_2.Binary()); - RayTask job_2_detached_actor_3_task(task_spec_job_2_detached_actor_3); - worker_job_2_detached_actor_3->SetAssignedTask(job_2_detached_actor_3_task); - worker_job_2_detached_actor_3->AssignTaskId(TaskID::Nil()); + RayLease job_2_detached_actor_3_lease(lease_spec_job_2_detached_actor_3); + worker_job_2_detached_actor_3->GrantLease(job_2_detached_actor_3_lease); + worker_job_2_detached_actor_3->GrantLeaseId(LeaseID::Nil()); worker_pool_->PushWorker(worker_job_2_detached_actor_3); - ASSERT_NE(worker_pool_->PopWorkerSync(task_spec_job_1_detached_actor_1), + ASSERT_NE(worker_pool_->PopWorkerSync(lease_spec_job_1_detached_actor_1), worker_job_2_detached_actor_3); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 1); worker_job_2_detached_actor_3->MarkDead(); @@ -1020,17 +1026,18 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerForRequestWithRootDetachedActor) // worker has mismatched detached actor ID and matching job ID auto worker_job_1_detached_actor_2 = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, job_1_id); - auto detached_actor_id_2_job_1 = ActorID::Of(job_1_id, TaskID::FromRandom(job_1_id), 1); - auto task_spec_job_1_detached_actor_2 = - ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, job_1_id); - task_spec_job_1_detached_actor_2.GetMutableMessage().set_root_detached_actor_id( + auto detached_actor_id_2_job_1 = + ActorID::Of(job_1_id, TaskID::ForDriverTask(job_1_id), 1); + auto lease_spec_job_1_detached_actor_2 = + ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_1_id); + lease_spec_job_1_detached_actor_2.GetMutableMessage().set_root_detached_actor_id( detached_actor_id_2_job_1.Binary()); - RayTask job_1_detached_actor_2_task(task_spec_job_1_detached_actor_2); - worker_job_1_detached_actor_2->SetAssignedTask(job_1_detached_actor_2_task); - worker_job_1_detached_actor_2->AssignTaskId(TaskID::Nil()); + RayLease job_1_detached_actor_2_lease(lease_spec_job_1_detached_actor_2); + worker_job_1_detached_actor_2->GrantLease(job_1_detached_actor_2_lease); + worker_job_1_detached_actor_2->GrantLeaseId(LeaseID::Nil()); worker_pool_->PushWorker(worker_job_1_detached_actor_2); - ASSERT_NE(worker_pool_->PopWorkerSync(task_spec_job_1_detached_actor_1), + ASSERT_NE(worker_pool_->PopWorkerSync(lease_spec_job_1_detached_actor_1), worker_job_1_detached_actor_2); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 1); worker_job_1_detached_actor_2->MarkDead(); @@ -1045,16 +1052,16 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerForRequestWithRootDetachedActor) // Test the worker pool logic regardless for completeness. auto worker_job_2_detached_actor_1 = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, job_2_id); - auto task_spec_job_2_detached_actor_1 = - ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, job_2_id); - task_spec_job_2_detached_actor_1.GetMutableMessage().set_root_detached_actor_id( + auto lease_spec_job_2_detached_actor_1 = + ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_2_id); + lease_spec_job_2_detached_actor_1.GetMutableMessage().set_root_detached_actor_id( detached_actor_id_1_job_1.Binary()); - RayTask job_2_detached_actor_1_task(task_spec_job_2_detached_actor_1); - worker_job_2_detached_actor_1->SetAssignedTask(job_2_detached_actor_1_task); - worker_job_2_detached_actor_1->AssignTaskId(TaskID::Nil()); + RayLease job_2_detached_actor_1_lease(lease_spec_job_2_detached_actor_1); + worker_job_2_detached_actor_1->GrantLease(job_2_detached_actor_1_lease); + worker_job_2_detached_actor_1->GrantLeaseId(LeaseID::Nil()); worker_pool_->PushWorker(worker_job_2_detached_actor_1); - ASSERT_NE(worker_pool_->PopWorkerSync(task_spec_job_1_detached_actor_1), + ASSERT_NE(worker_pool_->PopWorkerSync(lease_spec_job_1_detached_actor_1), worker_job_2_detached_actor_1); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 1); worker_job_2_detached_actor_1->MarkDead(); @@ -1063,7 +1070,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerForRequestWithRootDetachedActor) } // Tests the worker assignment logic for workers that have a root detached actor ID -// but tasks that *don't* have one. +// but leases that *don't* have one. // // Workers with a root detached actor ID can be used so long as their job ID matches // or hasn't been assigned yet. @@ -1074,63 +1081,65 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerWithRootDetachedActorID) { // NOTE: in all test cases the only worker in the pool is worker_job_1_detached_actor_1. auto worker_job_1_detached_actor_1 = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, job_1_id); - auto task_spec_job_1_detached_actor_1 = - ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, job_1_id); - auto detached_actor_id_1_job_1 = ActorID::Of(job_1_id, TaskID::FromRandom(job_1_id), 0); - task_spec_job_1_detached_actor_1.GetMutableMessage().set_root_detached_actor_id( + auto lease_spec_job_1_detached_actor_1 = + ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_1_id); + auto detached_actor_id_1_job_1 = + ActorID::Of(job_1_id, TaskID::ForDriverTask(job_1_id), 0); + lease_spec_job_1_detached_actor_1.GetMutableMessage().set_root_detached_actor_id( detached_actor_id_1_job_1.Binary()); - RayTask job_1_detached_actor_1_task(task_spec_job_1_detached_actor_1); - worker_job_1_detached_actor_1->SetAssignedTask(job_1_detached_actor_1_task); - worker_job_1_detached_actor_1->AssignTaskId(TaskID::Nil()); + RayLease job_1_detached_actor_1_lease(lease_spec_job_1_detached_actor_1); + worker_job_1_detached_actor_1->GrantLease(job_1_detached_actor_1_lease); + worker_job_1_detached_actor_1->GrantLeaseId(LeaseID::Nil()); // Case 1 (match): // request has no root detached actor ID and matching job ID - auto task_spec_job_1_no_detached_actor = - ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, job_1_id); + auto lease_spec_job_1_no_detached_actor = + ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_1_id); worker_pool_->PushWorker(worker_job_1_detached_actor_1); - ASSERT_EQ(worker_pool_->PopWorkerSync(task_spec_job_1_no_detached_actor), + ASSERT_EQ(worker_pool_->PopWorkerSync(lease_spec_job_1_no_detached_actor), worker_job_1_detached_actor_1); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); // Case 2 (match): // request has matching root detached actor ID and matching job ID worker_pool_->PushWorker(worker_job_1_detached_actor_1); - ASSERT_EQ(worker_pool_->PopWorkerSync(task_spec_job_1_detached_actor_1), + ASSERT_EQ(worker_pool_->PopWorkerSync(lease_spec_job_1_detached_actor_1), worker_job_1_detached_actor_1); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); // Case 3 (mismatch): // request has no root detached actor ID and mismatched job ID - auto task_spec_job_2_no_detached_actor = - ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, job_2_id); + auto lease_spec_job_2_no_detached_actor = + ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_2_id); worker_pool_->PushWorker(worker_job_1_detached_actor_1); - ASSERT_NE(worker_pool_->PopWorkerSync(task_spec_job_2_no_detached_actor), + ASSERT_NE(worker_pool_->PopWorkerSync(lease_spec_job_2_no_detached_actor), worker_job_1_detached_actor_1); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 1); // Case 4 (mismatch): // request has mismatched root detached actor ID and mismatched job ID - auto task_spec_job_2_detached_actor_2 = - ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, job_2_id); - auto job_2_detached_actor_2_id = ActorID::Of(job_2_id, TaskID::FromRandom(job_2_id), 0); - task_spec_job_2_detached_actor_2.GetMutableMessage().set_root_detached_actor_id( + auto lease_spec_job_2_detached_actor_2 = + ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_2_id); + auto job_2_detached_actor_2_id = + ActorID::Of(job_2_id, TaskID::ForDriverTask(job_2_id), 0); + lease_spec_job_2_detached_actor_2.GetMutableMessage().set_root_detached_actor_id( job_2_detached_actor_2_id.Binary()); - ASSERT_NE(worker_pool_->PopWorkerSync(task_spec_job_2_detached_actor_2), + ASSERT_NE(worker_pool_->PopWorkerSync(lease_spec_job_2_detached_actor_2), worker_job_1_detached_actor_1); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 1); } TEST_F(WorkerPoolDriverRegisteredTest, MaximumStartupConcurrency) { - auto task_spec = ExampleTaskSpec(); + auto lease_spec = ExampleLeaseSpec(); std::vector started_processes; // Try to pop some workers. Some worker processes will be started. for (int i = 0; i < MAXIMUM_STARTUP_CONCURRENCY; i++) { worker_pool_->PopWorker( - task_spec, + lease_spec, [](const std::shared_ptr worker, PopWorkerStatus status, const std::string &runtime_env_setup_error_message) -> bool { return true; }); @@ -1144,7 +1153,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, MaximumStartupConcurrency) { // Can't start a new worker process at this point. worker_pool_->PopWorker( - task_spec, + lease_spec, [](const std::shared_ptr worker, PopWorkerStatus status, const std::string &runtime_env_setup_error_message) -> bool { return true; }); @@ -1172,7 +1181,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, MaximumStartupConcurrency) { // Can't start a new worker process at this point. ASSERT_EQ(MAXIMUM_STARTUP_CONCURRENCY, worker_pool_->NumWorkersStarting()); worker_pool_->PopWorker( - task_spec, + lease_spec, [](const std::shared_ptr worker, PopWorkerStatus status, const std::string &runtime_env_setup_error_message) -> bool { return true; }); @@ -1192,7 +1201,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, MaximumStartupConcurrency) { // Can't start a new worker process at this point. worker_pool_->PopWorker( - task_spec, + lease_spec, [](const std::shared_ptr worker, PopWorkerStatus status, const std::string &runtime_env_setup_error_message) -> bool { return true; }); @@ -1249,7 +1258,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, HandleIOWorkersPushPop) { spill_workers.insert(CreateSpillWorker(Process())); spill_workers.insert(CreateSpillWorker(Process())); // Add the workers to the pool. - // 2 pending tasks / 2 new idle workers. + // 2 pending leases / 2 new idle workers. for (const auto &worker : spill_workers) { auto status = PopWorkerStatus::OK; auto [proc, token] = worker_pool_->StartWorkerProcess( @@ -1277,7 +1286,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, HandleIOWorkersPushPop) { worker_pool_->OnWorkerStarted(worker); } // Now push back to used workers - // 0 pending task, 3 idle workers. + // 0 pending lease, 3 idle workers. for (const auto &worker : spill_workers) { worker_pool_->PushSpillWorker(worker); } @@ -1498,20 +1507,20 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestWorkerCapping) { /// std::vector> popped_workers; for (int i = 0; i < num_workers; i++) { - // Pop workers for actor creation tasks. - auto task_spec = - ExampleTaskSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); - auto worker = worker_pool_->PopWorkerSync(task_spec, false); - // Simulate running the task and finish. This is to set task_assign_time_. - RayTask task(task_spec); - worker->SetAssignedTask(task); - worker->AssignTaskId(TaskID::Nil()); + // Pop workers for actor creation leases. + auto lease_spec = + ExampleLeaseSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); + auto worker = worker_pool_->PopWorkerSync(lease_spec, false); + // Simulate granting the lease and finish. This is to set lease_grant_time_. + RayLease lease(lease_spec); + worker->GrantLease(lease); + worker->GrantLeaseId(LeaseID::Nil()); popped_workers.push_back(worker); ASSERT_TRUE(worker); ASSERT_EQ(worker->GetAssignedJobId(), job_id); } - // After scheduling an actor and task, there's no more idle worker. + // After granting a lease to each worker, there should be no idle workers. ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); /// @@ -1720,10 +1729,11 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestJobFinishedForPopWorker) { // Finish the job. worker_pool_->HandleJobFinished(job_id); - auto task_spec = ExampleTaskSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); + auto lease_spec = + ExampleLeaseSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); PopWorkerStatus pop_worker_status; // This PopWorker should fail since the job finished. - worker = worker_pool_->PopWorkerSync(task_spec, false, &pop_worker_status); + worker = worker_pool_->PopWorkerSync(lease_spec, false, &pop_worker_status); ASSERT_EQ(pop_worker_status, PopWorkerStatus::JobFinished); ASSERT_FALSE(worker); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 1); @@ -1736,12 +1746,12 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestJobFinishedForPopWorker) { job_id = JOB_ID_2; rpc::JobConfig job_config; RegisterDriver(Language::PYTHON, job_id, job_config); - task_spec = ExampleTaskSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); + lease_spec = ExampleLeaseSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); pop_worker_status = PopWorkerStatus::OK; // This will start a new worker. std::promise promise; worker_pool_->PopWorker( - task_spec, + lease_spec, [&](const std::shared_ptr worker, PopWorkerStatus status, const std::string &runtime_env_setup_error_message) -> bool { @@ -1796,9 +1806,10 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestJobFinishedForceKillIdleWorker) { worker_pool_->PushWorker(worker); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 1); - /// Execute some task with the worker. - auto task_spec = ExampleTaskSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); - worker = worker_pool_->PopWorkerSync(task_spec, false); + /// Grant some lease with the worker. + auto lease_spec = + ExampleLeaseSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); + worker = worker_pool_->PopWorkerSync(lease_spec, false); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); /// Return the worker. @@ -1889,41 +1900,41 @@ TEST_F(WorkerPoolDriverRegisteredTest, TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerWithRuntimeEnv) { ASSERT_EQ(worker_pool_->GetProcessSize(), 0); auto actor_creation_id = ActorID::Of(JOB_ID, TaskID::ForDriverTask(JOB_ID), 1); - const auto actor_creation_task_spec = ExampleTaskSpec(ActorID::Nil(), - Language::PYTHON, - JOB_ID, - actor_creation_id, - {"XXX=YYY"}, - TaskID::FromRandom(JobID::Nil()), - ExampleRuntimeEnvInfo({"XXX"})); - const auto normal_task_spec = ExampleTaskSpec(ActorID::Nil(), - Language::PYTHON, - JOB_ID, - ActorID::Nil(), - {"XXX=YYY"}, - TaskID::FromRandom(JobID::Nil()), - ExampleRuntimeEnvInfo({"XXX"})); - const auto normal_task_spec_without_runtime_env = - ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, JOB_ID, ActorID::Nil(), {}); - // Pop worker for actor creation task again. - auto popped_worker = worker_pool_->PopWorkerSync(actor_creation_task_spec); + const auto actor_creation_lease_spec = ExampleLeaseSpec(ActorID::Nil(), + Language::PYTHON, + JOB_ID, + actor_creation_id, + {"XXX=YYY"}, + LeaseID::FromRandom(), + ExampleRuntimeEnvInfo({"XXX"})); + const auto normal_lease_spec = ExampleLeaseSpec(ActorID::Nil(), + Language::PYTHON, + JOB_ID, + ActorID::Nil(), + {"XXX=YYY"}, + LeaseID::FromRandom(), + ExampleRuntimeEnvInfo({"XXX"})); + const auto normal_lease_spec_without_runtime_env = + ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, JOB_ID, ActorID::Nil(), {}); + // Pop worker for actor creation lease again. + auto popped_worker = worker_pool_->PopWorkerSync(actor_creation_lease_spec); // Got a worker with correct runtime env hash. ASSERT_NE(popped_worker, nullptr); ASSERT_EQ(popped_worker->GetRuntimeEnvHash(), - actor_creation_task_spec.GetRuntimeEnvHash()); + actor_creation_lease_spec.GetRuntimeEnvHash()); ASSERT_EQ(worker_pool_->GetProcessSize(), 1); - // Pop worker for normal task. - popped_worker = worker_pool_->PopWorkerSync(normal_task_spec); + // Pop worker for normal lease. + popped_worker = worker_pool_->PopWorkerSync(normal_lease_spec); // Got a worker with correct runtime env hash. ASSERT_NE(popped_worker, nullptr); - ASSERT_EQ(popped_worker->GetRuntimeEnvHash(), normal_task_spec.GetRuntimeEnvHash()); + ASSERT_EQ(popped_worker->GetRuntimeEnvHash(), normal_lease_spec.GetRuntimeEnvHash()); ASSERT_EQ(worker_pool_->GetProcessSize(), 2); - // Pop worker for normal task without runtime env. - popped_worker = worker_pool_->PopWorkerSync(normal_task_spec_without_runtime_env); + // Pop worker for normal lease without runtime env. + popped_worker = worker_pool_->PopWorkerSync(normal_lease_spec_without_runtime_env); // Got a worker with correct runtime env hash. ASSERT_NE(popped_worker, nullptr); ASSERT_EQ(popped_worker->GetRuntimeEnvHash(), - normal_task_spec_without_runtime_env.GetRuntimeEnvHash()); + normal_lease_spec_without_runtime_env.GetRuntimeEnvHash()); ASSERT_EQ(worker_pool_->GetProcessSize(), 3); } @@ -1972,25 +1983,24 @@ TEST_F(WorkerPoolDriverRegisteredTest, RuntimeEnvUriReferenceWorkerLevel) { ASSERT_EQ(GetReferenceCount(runtime_env_info.serialized_runtime_env()), 1); // Start actor with runtime env. auto actor_creation_id = ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 1); - const auto actor_creation_task_spec = - ExampleTaskSpec(ActorID::Nil(), - Language::PYTHON, - job_id, - actor_creation_id, - {"XXX=YYY"}, - TaskID::FromRandom(JobID::Nil()), - runtime_env_info); - auto popped_actor_worker = worker_pool_->PopWorkerSync(actor_creation_task_spec); + const auto actor_creation_lease_spec = ExampleLeaseSpec(ActorID::Nil(), + Language::PYTHON, + job_id, + actor_creation_id, + {"XXX=YYY"}, + LeaseID::FromRandom(), + runtime_env_info); + auto popped_actor_worker = worker_pool_->PopWorkerSync(actor_creation_lease_spec); ASSERT_EQ(GetReferenceCount(runtime_env_info.serialized_runtime_env()), 2); - // Start task with runtime env. - const auto normal_task_spec = ExampleTaskSpec(ActorID::Nil(), - Language::PYTHON, - job_id, - ActorID::Nil(), - {"XXX=YYY"}, - TaskID::FromRandom(JobID::Nil()), - runtime_env_info); - auto popped_normal_worker = worker_pool_->PopWorkerSync(actor_creation_task_spec); + // Start lease with runtime env. + const auto normal_lease_spec = ExampleLeaseSpec(ActorID::Nil(), + Language::PYTHON, + job_id, + ActorID::Nil(), + {"XXX=YYY"}, + LeaseID::FromRandom(), + runtime_env_info); + auto popped_normal_worker = worker_pool_->PopWorkerSync(actor_creation_lease_spec); ASSERT_EQ(GetReferenceCount(runtime_env_info.serialized_runtime_env()), 3); // Disconnect actor worker. worker_pool_->DisconnectWorker(popped_actor_worker, @@ -2019,18 +2029,17 @@ TEST_F(WorkerPoolDriverRegisteredTest, RuntimeEnvUriReferenceWorkerLevel) { ASSERT_EQ(GetReferenceCount(runtime_env_info.serialized_runtime_env()), 0); // Start actor with runtime env. auto actor_creation_id = ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 2); - const auto actor_creation_task_spec = - ExampleTaskSpec(ActorID::Nil(), - Language::PYTHON, - job_id, - actor_creation_id, - {"XXX=YYY"}, - TaskID::FromRandom(JobID::Nil()), - runtime_env_info); - auto popped_actor_worker = worker_pool_->PopWorkerSync(actor_creation_task_spec); + const auto actor_creation_lease_spec = ExampleLeaseSpec(ActorID::Nil(), + Language::PYTHON, + job_id, + actor_creation_id, + {"XXX=YYY"}, + LeaseID::FromRandom(), + runtime_env_info); + auto popped_actor_worker = worker_pool_->PopWorkerSync(actor_creation_lease_spec); ASSERT_EQ(GetReferenceCount(runtime_env_info.serialized_runtime_env()), 1); - // Start task with runtime env. - auto popped_normal_worker = worker_pool_->PopWorkerSync(actor_creation_task_spec); + // Start lease with runtime env. + auto popped_normal_worker = worker_pool_->PopWorkerSync(actor_creation_lease_spec); ASSERT_EQ(GetReferenceCount(runtime_env_info.serialized_runtime_env()), 2); // Disconnect actor worker. worker_pool_->DisconnectWorker(popped_actor_worker, @@ -2050,36 +2059,36 @@ TEST_F(WorkerPoolDriverRegisteredTest, CacheWorkersByRuntimeEnvHash) { /// /// Check that a worker can be popped only if there is a /// worker available whose runtime env matches the runtime env - /// in the task spec. + /// in the lease spec. /// ASSERT_EQ(worker_pool_->GetProcessSize(), 0); auto actor_creation_id = ActorID::Of(JOB_ID, TaskID::ForDriverTask(JOB_ID), 1); - const auto actor_creation_task_spec_1 = - ExampleTaskSpec(ActorID::Nil(), - Language::PYTHON, - JOB_ID, - actor_creation_id, - /*dynamic_worker_options=*/{}, - TaskID::FromRandom(JobID::Nil()), - ExampleRuntimeEnvInfoFromString("mock_runtime_env_1")); - const auto task_spec_1 = - ExampleTaskSpec(ActorID::Nil(), - Language::PYTHON, - JOB_ID, - ActorID::Nil(), - /*dynamic_worker_options=*/{}, - TaskID::FromRandom(JobID::Nil()), - ExampleRuntimeEnvInfoFromString("mock_runtime_env_1")); - const auto task_spec_2 = - ExampleTaskSpec(ActorID::Nil(), - Language::PYTHON, - JOB_ID, - ActorID::Nil(), - /*dynamic_worker_options=*/{}, - TaskID::FromRandom(JobID::Nil()), - ExampleRuntimeEnvInfoFromString("mock_runtime_env_2")); - - const int runtime_env_hash_1 = actor_creation_task_spec_1.GetRuntimeEnvHash(); + const auto actor_creation_lease_spec_1 = + ExampleLeaseSpec(ActorID::Nil(), + Language::PYTHON, + JOB_ID, + actor_creation_id, + /*dynamic_worker_options=*/{}, + LeaseID::FromRandom(), + ExampleRuntimeEnvInfoFromString("mock_runtime_env_1")); + const auto lease_spec_1 = + ExampleLeaseSpec(ActorID::Nil(), + Language::PYTHON, + JOB_ID, + ActorID::Nil(), + /*dynamic_worker_options=*/{}, + LeaseID::FromRandom(), + ExampleRuntimeEnvInfoFromString("mock_runtime_env_1")); + const auto lease_spec_2 = + ExampleLeaseSpec(ActorID::Nil(), + Language::PYTHON, + JOB_ID, + ActorID::Nil(), + /*dynamic_worker_options=*/{}, + LeaseID::FromRandom(), + ExampleRuntimeEnvInfoFromString("mock_runtime_env_2")); + + const int runtime_env_hash_1 = actor_creation_lease_spec_1.GetRuntimeEnvHash(); // Push worker with runtime env 1. auto worker = worker_pool_->CreateWorker(Process::CreateNewDummy(), @@ -2089,14 +2098,14 @@ TEST_F(WorkerPoolDriverRegisteredTest, CacheWorkersByRuntimeEnvHash) { runtime_env_hash_1); worker_pool_->PushWorker(worker); - // Try to pop worker for task with runtime env 2. - auto popped_worker = worker_pool_->PopWorkerSync(task_spec_2); + // Try to pop worker for lease with runtime env 2. + auto popped_worker = worker_pool_->PopWorkerSync(lease_spec_2); // Check that popped worker isn't the one we pushed. ASSERT_NE(popped_worker, nullptr); ASSERT_NE(popped_worker, worker); - // Try to pop the worker for task with runtime env 1. - popped_worker = worker_pool_->PopWorkerSync(task_spec_1); + // Try to pop the worker for lease with runtime env 1. + popped_worker = worker_pool_->PopWorkerSync(lease_spec_1); ASSERT_EQ(popped_worker, worker); // Push another worker with runtime env 1. @@ -2108,7 +2117,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, CacheWorkersByRuntimeEnvHash) { worker_pool_->PushWorker(worker); // Try to pop the worker for an actor with runtime env 1. - popped_worker = worker_pool_->PopWorkerSync(actor_creation_task_spec_1); + popped_worker = worker_pool_->PopWorkerSync(actor_creation_lease_spec_1); // Check that we got the pushed worker. ASSERT_EQ(popped_worker, worker); worker_pool_->ClearProcesses(); @@ -2116,10 +2125,10 @@ TEST_F(WorkerPoolDriverRegisteredTest, CacheWorkersByRuntimeEnvHash) { TEST_F(WorkerPoolDriverRegisteredTest, WorkerNoLeaks) { std::shared_ptr popped_worker; - const auto task_spec = ExampleTaskSpec(); + const auto lease_spec = ExampleLeaseSpec(); // Pop a worker and don't dispatch. - worker_pool_->PopWorker(task_spec, + worker_pool_->PopWorker(lease_spec, [](const std::shared_ptr worker, PopWorkerStatus status, const std::string &runtime_env_setup_error_message) -> bool { @@ -2131,11 +2140,11 @@ TEST_F(WorkerPoolDriverRegisteredTest, WorkerNoLeaks) { // No idle workers because no workers pushed. ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); // push workers. - worker_pool_->PushWorkers(0, task_spec.JobId()); + worker_pool_->PushWorkers(0, lease_spec.JobId()); // The worker has been pushed but not dispatched. ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 1); // Pop a worker and don't dispatch. - worker_pool_->PopWorker(task_spec, + worker_pool_->PopWorker(lease_spec, [](const std::shared_ptr worker, PopWorkerStatus status, const std::string &runtime_env_setup_error_message) -> bool { @@ -2146,7 +2155,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, WorkerNoLeaks) { ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 1); ASSERT_EQ(worker_pool_->GetProcessSize(), 1); // Pop a worker and dispatch. - worker_pool_->PopWorker(task_spec, + worker_pool_->PopWorker(lease_spec, [](const std::shared_ptr worker, PopWorkerStatus status, const std::string &runtime_env_setup_error_message) -> bool { @@ -2164,56 +2173,56 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerStatus) { PopWorkerStatus status; /* Test PopWorkerStatus JobConfigMissing */ - // Create a task by unregistered job id. + // Create a lease by unregistered job id. auto job_id = JobID::FromInt(123); - auto task_spec = ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, job_id); - popped_worker = worker_pool_->PopWorkerSync(task_spec, true, &status); + auto lease_spec = ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_id); + popped_worker = worker_pool_->PopWorkerSync(lease_spec, true, &status); // PopWorker failed and the status is `JobConfigMissing`. ASSERT_EQ(popped_worker, nullptr); ASSERT_EQ(status, PopWorkerStatus::JobConfigMissing); // Register driver fot the job. RegisterDriver(Language::PYTHON, job_id); - popped_worker = worker_pool_->PopWorkerSync(task_spec, true, &status); + popped_worker = worker_pool_->PopWorkerSync(lease_spec, true, &status); // PopWorker success. ASSERT_NE(popped_worker, nullptr); ASSERT_EQ(status, PopWorkerStatus::OK); /* Test PopWorkerStatus RuntimeEnvCreationFailed */ - // Create a task with bad runtime env. - const auto task_spec_with_bad_runtime_env = - ExampleTaskSpec(ActorID::Nil(), - Language::PYTHON, - job_id, - ActorID::Nil(), - {"XXX=YYY"}, - TaskID::FromRandom(JobID::Nil()), - ExampleRuntimeEnvInfoFromString(std::string(kBadRuntimeEnv))); + // Create a lease with bad runtime env. + const auto lease_spec_with_bad_runtime_env = + ExampleLeaseSpec(ActorID::Nil(), + Language::PYTHON, + job_id, + ActorID::Nil(), + {"XXX=YYY"}, + LeaseID::FromRandom(), + ExampleRuntimeEnvInfoFromString(std::string(kBadRuntimeEnv))); std::string error_msg; popped_worker = worker_pool_->PopWorkerSync( - task_spec_with_bad_runtime_env, true, &status, 0, &error_msg); + lease_spec_with_bad_runtime_env, true, &status, 0, &error_msg); // PopWorker failed and the status is `RuntimeEnvCreationFailed`. ASSERT_EQ(popped_worker, nullptr); ASSERT_EQ(status, PopWorkerStatus::RuntimeEnvCreationFailed); ASSERT_EQ(error_msg, kBadRuntimeEnvErrorMsg); - // Create a task with available runtime env. - const auto task_spec_with_runtime_env = - ExampleTaskSpec(ActorID::Nil(), - Language::PYTHON, - job_id, - ActorID::Nil(), - {"XXX=YYY"}, - TaskID::FromRandom(JobID::Nil()), - ExampleRuntimeEnvInfo({"XXX"})); - popped_worker = worker_pool_->PopWorkerSync(task_spec_with_runtime_env, true, &status); + // Create a lease with available runtime env. + const auto lease_spec_with_runtime_env = + ExampleLeaseSpec(ActorID::Nil(), + Language::PYTHON, + job_id, + ActorID::Nil(), + {"XXX=YYY"}, + LeaseID::FromRandom(), + ExampleRuntimeEnvInfo({"XXX"})); + popped_worker = worker_pool_->PopWorkerSync(lease_spec_with_runtime_env, true, &status); // PopWorker success. ASSERT_NE(popped_worker, nullptr); ASSERT_EQ(status, PopWorkerStatus::OK); /* Test PopWorkerStatus WorkerPendingRegistration */ - // Create a task without push worker. - popped_worker = worker_pool_->PopWorkerSync(task_spec, false, &status); + // Create a lease without push worker. + popped_worker = worker_pool_->PopWorkerSync(lease_spec, false, &status); ASSERT_EQ(popped_worker, nullptr); // PopWorker failed while the timer was triggered and the status is // `WorkerPendingRegistration`. @@ -2224,9 +2233,9 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerStatus) { TEST_F(WorkerPoolDriverRegisteredTest, WorkerPendingRegistrationErasesRequest) { std::shared_ptr popped_worker; PopWorkerStatus status; - auto task_spec = ExampleTaskSpec(); - // Create a task without push worker. It should time out (WorkerPendingRegistration). - popped_worker = worker_pool_->PopWorkerSync(task_spec, false, &status); + auto lease_spec = ExampleLeaseSpec(); + // Create a lease without push worker. It should time out (WorkerPendingRegistration). + popped_worker = worker_pool_->PopWorkerSync(lease_spec, false, &status); ASSERT_EQ(popped_worker, nullptr); ASSERT_EQ(status, PopWorkerStatus::WorkerPendingRegistration); // The request should be erased. @@ -2346,14 +2355,14 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestIOWorkerFailureAndSpawn) { } TEST_F(WorkerPoolDriverRegisteredTest, WorkerReuseForPrestartedWorker) { - const auto task_spec = ExampleTaskSpec(); - worker_pool_->PrestartWorkersInternal(task_spec, /*num_needed=*/1); - worker_pool_->PushWorkers(0, task_spec.JobId()); + const auto lease_spec = ExampleLeaseSpec(); + worker_pool_->PrestartWorkersInternal(lease_spec, /*num_needed=*/1); + worker_pool_->PushWorkers(0, lease_spec.JobId()); // One worker process has been prestarted. ASSERT_EQ(worker_pool_->GetProcessSize(), 1); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 1); // Pop a worker and don't dispatch. - auto popped_worker = worker_pool_->PopWorkerSync(task_spec); + auto popped_worker = worker_pool_->PopWorkerSync(lease_spec); ASSERT_NE(popped_worker, nullptr); // no new worker started since we can reuse the cached worker. ASSERT_EQ(worker_pool_->GetProcessSize(), 1); @@ -2362,17 +2371,17 @@ TEST_F(WorkerPoolDriverRegisteredTest, WorkerReuseForPrestartedWorker) { } TEST_F(WorkerPoolDriverRegisteredTest, WorkerReuseForSameJobId) { - const auto task_spec = ExampleTaskSpec(); + const auto lease_spec = ExampleLeaseSpec(); // start one worker - auto popped_worker = worker_pool_->PopWorkerSync(task_spec); + auto popped_worker = worker_pool_->PopWorkerSync(lease_spec); ASSERT_NE(popped_worker, nullptr); ASSERT_EQ(worker_pool_->GetProcessSize(), 1); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); worker_pool_->PushWorker(popped_worker); // start a new worker withe same job_id resuse the same worker. - auto popped_worker1 = worker_pool_->PopWorkerSync(task_spec); + auto popped_worker1 = worker_pool_->PopWorkerSync(lease_spec); ASSERT_NE(popped_worker1, nullptr); ASSERT_EQ(popped_worker1, popped_worker); ASSERT_EQ(worker_pool_->GetProcessSize(), 1); @@ -2380,11 +2389,11 @@ TEST_F(WorkerPoolDriverRegisteredTest, WorkerReuseForSameJobId) { } TEST_F(WorkerPoolDriverRegisteredTest, WorkerReuseFailureForDifferentJobId) { - const auto task_spec = ExampleTaskSpec(); - const auto task_spec1 = ExampleTaskSpec(ActorID::Nil(), Language::PYTHON, JOB_ID_2); + const auto lease_spec = ExampleLeaseSpec(); + const auto lease_spec1 = ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, JOB_ID_2); // start one worker - auto popped_worker = worker_pool_->PopWorkerSync(task_spec); + auto popped_worker = worker_pool_->PopWorkerSync(lease_spec); ASSERT_NE(popped_worker, nullptr); ASSERT_EQ(worker_pool_->GetProcessSize(), 1); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); @@ -2393,7 +2402,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, WorkerReuseFailureForDifferentJobId) { RegisterDriver(Language::PYTHON, JOB_ID_2); // start a new worker with different job_id requires a new worker. - auto popped_worker1 = worker_pool_->PopWorkerSync(task_spec1); + auto popped_worker1 = worker_pool_->PopWorkerSync(lease_spec1); ASSERT_NE(popped_worker1, nullptr); ASSERT_NE(popped_worker1, popped_worker); ASSERT_EQ(worker_pool_->GetProcessSize(), 2); @@ -2403,7 +2412,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, WorkerReuseFailureForDifferentJobId) { TEST_F(WorkerPoolTest, RegisterFirstPythonDriverWaitForWorkerStart) { auto driver = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, JOB_ID); - driver->AssignTaskId(TaskID::ForDriverTask(JOB_ID)); + driver->GrantLeaseId(LeaseID::FromRandom()); bool callback_called = false; auto callback = [callback_called_ptr = &callback_called](Status, int) mutable { *callback_called_ptr = true; @@ -2415,7 +2424,7 @@ TEST_F(WorkerPoolTest, RegisterFirstPythonDriverWaitForWorkerStart) { TEST_F(WorkerPoolTest, RegisterSecondPythonDriverCallbackImmediately) { auto driver = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, JOB_ID); - driver->AssignTaskId(TaskID::ForDriverTask(JOB_ID)); + driver->GrantLeaseId(LeaseID::FromRandom()); RAY_CHECK_OK( worker_pool_->RegisterDriver(driver, rpc::JobConfig(), [](Status, int) {})); @@ -2425,7 +2434,7 @@ TEST_F(WorkerPoolTest, RegisterSecondPythonDriverCallbackImmediately) { }; auto second_driver = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, JOB_ID); - second_driver->AssignTaskId(TaskID::ForDriverTask(JOB_ID)); + second_driver->GrantLeaseId(LeaseID::FromRandom()); RAY_CHECK_OK(worker_pool_->RegisterDriver(second_driver, rpc::JobConfig(), callback)); ASSERT_TRUE(callback_called); } @@ -2434,7 +2443,7 @@ TEST_F(WorkerPoolTest, RegisterFirstJavaDriverCallbackImmediately) { auto driver = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::JAVA, JOB_ID); - driver->AssignTaskId(TaskID::ForDriverTask(JOB_ID)); + driver->GrantLeaseId(LeaseID::FromRandom()); bool callback_called = false; auto callback = [callback_called_ptr = &callback_called](Status, int) mutable { *callback_called_ptr = true; diff --git a/src/ray/raylet/worker.cc b/src/ray/raylet/worker.cc index fbdf0400a152..abf4a7e9d4d2 100644 --- a/src/ray/raylet/worker.cc +++ b/src/ray/raylet/worker.cc @@ -173,14 +173,14 @@ void Worker::Connect(std::shared_ptr rpc_client) } } -void Worker::AssignTaskId(const TaskID &task_id) { - assigned_task_id_ = task_id; - if (!task_id.IsNil()) { - task_assign_time_ = absl::Now(); +void Worker::GrantLeaseId(const LeaseID &lease_id) { + lease_id_ = lease_id; + if (!lease_id.IsNil()) { + lease_grant_time_ = absl::Now(); } -} +}; -const TaskID &Worker::GetAssignedTaskId() const { return assigned_task_id_; } +const LeaseID &Worker::GetGrantedLeaseId() const { return lease_id_; } const JobID &Worker::GetAssignedJobId() const { return assigned_job_id_; } @@ -199,18 +199,19 @@ void Worker::AssignActorId(const ActorID &actor_id) { const ActorID &Worker::GetActorId() const { return actor_id_; } -const std::string Worker::GetTaskOrActorIdAsDebugString() const { +const RayLease &Worker::GetGrantedLease() const { return granted_lease_; } + +const std::string Worker::GetLeaseIdAsDebugString() const { std::stringstream id_ss; if (GetActorId().IsNil()) { - id_ss << "task ID: " << GetAssignedTaskId(); - } else { id_ss << "actor ID: " << GetActorId(); } + id_ss << "lease ID: " << GetGrantedLeaseId(); return id_ss.str(); } bool Worker::IsDetachedActor() const { - return assigned_task_.GetTaskSpecification().IsDetachedActor(); + return granted_lease_.GetLeaseSpecification().IsDetachedActor(); } const std::shared_ptr Worker::Connection() const { return connection_; } diff --git a/src/ray/raylet/worker.h b/src/ray/raylet/worker.h index 4c61f4860c3d..b4af8ee5cd1b 100644 --- a/src/ray/raylet/worker.h +++ b/src/ray/raylet/worker.h @@ -23,10 +23,9 @@ #include "absl/time/time.h" #include "gtest/gtest_prod.h" #include "ray/common/id.h" +#include "ray/common/lease/lease.h" #include "ray/common/scheduling/resource_set.h" #include "ray/common/scheduling/scheduling_ids.h" -#include "ray/common/task/task.h" -#include "ray/common/task/task_common.h" #include "ray/ipc/client_connection.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/rpc/worker/core_worker_client.h" @@ -67,15 +66,16 @@ class WorkerInterface { virtual int Port() const = 0; virtual int AssignedPort() const = 0; virtual void SetAssignedPort(int port) = 0; - virtual void AssignTaskId(const TaskID &task_id) = 0; - virtual const TaskID &GetAssignedTaskId() const = 0; + virtual void GrantLeaseId(const LeaseID &lease_id) = 0; + virtual const LeaseID &GetGrantedLeaseId() const = 0; virtual const JobID &GetAssignedJobId() const = 0; + virtual const RayLease &GetGrantedLease() const = 0; virtual std::optional GetIsGpu() const = 0; virtual std::optional GetIsActorWorker() const = 0; virtual int GetRuntimeEnvHash() const = 0; virtual void AssignActorId(const ActorID &actor_id) = 0; virtual const ActorID &GetActorId() const = 0; - virtual const std::string GetTaskOrActorIdAsDebugString() const = 0; + virtual const std::string GetLeaseIdAsDebugString() const = 0; virtual bool IsDetachedActor() const = 0; virtual const std::shared_ptr Connection() const = 0; virtual void SetOwnerAddress(const rpc::Address &address) = 0; @@ -100,9 +100,9 @@ class WorkerInterface { virtual void ClearLifetimeAllocatedInstances() = 0; - virtual RayTask &GetAssignedTask() = 0; + virtual RayLease &GetGrantedLease() = 0; - virtual void SetAssignedTask(const RayTask &assigned_task) = 0; + virtual void GrantLease(const RayLease &granted_lease) = 0; virtual bool IsRegistered() = 0; @@ -112,7 +112,7 @@ class WorkerInterface { virtual bool IsAvailableForScheduling() const = 0; /// Time when the last task was assigned to this worker. - virtual absl::Time GetAssignedTaskTime() const = 0; + virtual absl::Time GetGrantedLeaseTime() const = 0; virtual void SetJobId(const JobID &job_id) = 0; @@ -179,17 +179,17 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa int Port() const; int AssignedPort() const; void SetAssignedPort(int port); - void AssignTaskId(const TaskID &task_id); - const TaskID &GetAssignedTaskId() const; + void GrantLeaseId(const LeaseID &lease_id); + const LeaseID &GetGrantedLeaseId() const; const JobID &GetAssignedJobId() const; + const RayLease &GetGrantedLease() const; std::optional GetIsGpu() const; std::optional GetIsActorWorker() const; int GetRuntimeEnvHash() const; void AssignActorId(const ActorID &actor_id); const ActorID &GetActorId() const; - // Creates the debug string for the ID of the task or actor depending on which is - // running. - const std::string GetTaskOrActorIdAsDebugString() const; + // Creates the debug string for the ID of the lease and the actor ID if it exists. + const std::string GetLeaseIdAsDebugString() const; bool IsDetachedActor() const; const std::shared_ptr Connection() const; void SetOwnerAddress(const rpc::Address &address); @@ -225,28 +225,27 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa void ClearLifetimeAllocatedInstances() { lifetime_allocated_instances_ = nullptr; }; - RayTask &GetAssignedTask() { return assigned_task_; }; - - void SetAssignedTask(const RayTask &assigned_task) { - const auto &task_spec = assigned_task.GetTaskSpecification(); - SetJobId(task_spec.JobId()); - SetBundleId(task_spec.PlacementGroupBundleId()); - SetOwnerAddress(task_spec.CallerAddress()); - AssignTaskId(task_spec.TaskId()); - SetIsGpu(task_spec.GetRequiredResources().Get(scheduling::ResourceID::GPU()) > 0); - RAY_CHECK(!task_spec.IsActorTask()); - SetIsActorWorker(task_spec.IsActorCreationTask()); - assigned_task_ = assigned_task; - root_detached_actor_id_ = assigned_task.GetTaskSpecification().RootDetachedActorId(); + RayLease &GetGrantedLease() { return granted_lease_; }; + + void GrantLease(const RayLease &granted_lease) { + const auto &lease_spec = granted_lease.GetLeaseSpecification(); + SetJobId(lease_spec.JobId()); + SetBundleId(lease_spec.PlacementGroupBundleId()); + SetOwnerAddress(lease_spec.CallerAddress()); + GrantLeaseId(lease_spec.LeaseId()); + SetIsGpu(lease_spec.GetRequiredResources().Get(scheduling::ResourceID::GPU()) > 0); + SetIsActorWorker(lease_spec.IsActorCreationTask()); + granted_lease_ = granted_lease; + root_detached_actor_id_ = granted_lease.GetLeaseSpecification().RootDetachedActorId(); } - absl::Time GetAssignedTaskTime() const { return task_assign_time_; }; + absl::Time GetGrantedLeaseTime() const { return lease_grant_time_; }; bool IsRegistered() { return rpc_client_ != nullptr; } bool IsAvailableForScheduling() const { return !IsDead() // Not dead - && GetAssignedTaskId().IsNil() // No assigned task + && GetGrantedLeaseId().IsNil() // No assigned lease && !IsBlocked() // Not blocked && GetActorId().IsNil(); // No assigned actor } @@ -255,7 +254,6 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa RAY_CHECK(IsRegistered()); return rpc_client_.get(); } - void SetJobId(const JobID &job_id); void SetIsGpu(bool is_gpu); void SetIsActorWorker(bool is_actor_worker); @@ -285,9 +283,9 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa int port_; /// Connection state of a worker. std::shared_ptr connection_; - /// The worker's currently assigned task. - TaskID assigned_task_id_; - /// Job ID for the worker's current assigned task. + /// The lease id of the worker's currently assigned lease. + LeaseID lease_id_; + /// Job ID for the worker's current assigned lease. JobID assigned_job_id_; /// The hash of the worker's assigned runtime env. We use this in the worker /// pool to cache and reuse workers with the same runtime env, because @@ -295,7 +293,7 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa const int runtime_env_hash_; /// The worker's actor ID. If this is nil, then the worker is not an actor. ActorID actor_id_; - /// Root detached actor ID for the worker's last assigned task. + /// Root detached actor ID for the worker's last assigned lease. ActorID root_detached_actor_id_; /// The worker's placement group bundle. It is used to detect if the worker is /// associated with a placement group bundle. @@ -314,19 +312,19 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa /// currently holds the lease on this worker, if any. rpc::Address owner_address_; /// The capacity of each resource instance allocated to this worker in order - /// to satisfy the resource requests of the task is currently running. + /// to satisfy the resource requests of the granted lease. std::shared_ptr allocated_instances_; /// The capacity of each resource instance allocated to this worker /// when running as an actor. std::shared_ptr lifetime_allocated_instances_; - /// RayTask being assigned to this worker. - RayTask assigned_task_; - /// Time when the last task was assigned to this worker. - absl::Time task_assign_time_; - /// Whether this worker ever holded a GPU resource. Once it holds a GPU or non-GPU task + /// RayLease being assigned to this worker. + RayLease granted_lease_; + /// Time when the last lease was granted to this worker. + absl::Time lease_grant_time_; + /// Whether this worker ever holded a GPU resource. Once it holds a GPU or non-GPU lease /// it can't switch to the other type. std::optional is_gpu_ = std::nullopt; - /// Whether this worker can hold an actor. Once it holds an actor or a normal task, it + /// Whether this worker can hold an actor. Once it holds an actor or a normal lease, it /// can't switch to the other type. std::optional is_actor_worker_ = std::nullopt; /// If true, a RPC need to be sent to notify the worker about GCS restarting. diff --git a/src/ray/raylet/worker_killing_policy.cc b/src/ray/raylet/worker_killing_policy.cc index ce37f23b298b..e804864f54ac 100644 --- a/src/ray/raylet/worker_killing_policy.cc +++ b/src/ray/raylet/worker_killing_policy.cc @@ -50,13 +50,14 @@ RetriableLIFOWorkerKillingPolicy::SelectWorkerToKill( sorted.end(), [](std::shared_ptr const &left, std::shared_ptr const &right) -> bool { - // First sort by retriable tasks and then by task time in descending order. + // First sort by retriable tasks and then by assigned time in descending + // order. int left_retriable = - left->GetAssignedTask().GetTaskSpecification().IsRetriable() ? 0 : 1; + left->GetGrantedLease().GetLeaseSpecification().IsRetriable() ? 0 : 1; int right_retriable = - right->GetAssignedTask().GetTaskSpecification().IsRetriable() ? 0 : 1; + right->GetGrantedLease().GetLeaseSpecification().IsRetriable() ? 0 : 1; if (left_retriable == right_retriable) { - return left->GetAssignedTaskTime() > right->GetAssignedTaskTime(); + return left->GetGrantedLeaseTime() > right->GetGrantedLeaseTime(); } return left_retriable < right_retriable; }); @@ -84,11 +85,11 @@ std::string WorkerKillingPolicy::WorkersDebugString( RAY_LOG_EVERY_MS(INFO, 60000) << "Can't find memory usage for PID, reporting zero. PID: " << pid; } - result << "Worker " << index << ": task assigned time " - << absl::FormatTime(worker->GetAssignedTaskTime(), absl::UTCTimeZone()) + result << "Worker " << index << ": lease granted time " + << absl::FormatTime(worker->GetGrantedLeaseTime(), absl::UTCTimeZone()) << " worker id " << worker->WorkerId() << " memory used " << used_memory - << " task spec " - << worker->GetAssignedTask().GetTaskSpecification().DebugString() << "\n"; + << " lease spec " + << worker->GetGrantedLease().GetLeaseSpecification().DebugString() << "\n"; index += 1; if (index > num_workers) { diff --git a/src/ray/raylet/worker_killing_policy_group_by_owner.cc b/src/ray/raylet/worker_killing_policy_group_by_owner.cc index dfa2588856a8..97d7010d55c7 100644 --- a/src/ray/raylet/worker_killing_policy_group_by_owner.cc +++ b/src/ray/raylet/worker_killing_policy_group_by_owner.cc @@ -50,9 +50,9 @@ GroupByOwnerIdWorkerKillingPolicy::SelectWorkerToKill( TaskID non_retriable_owner_id = TaskID::Nil(); std::unordered_map group_map; for (auto worker : workers) { - bool retriable = worker->GetAssignedTask().GetTaskSpecification().IsRetriable(); + bool retriable = worker->GetGrantedLease().GetLeaseSpecification().IsRetriable(); TaskID owner_id = - retriable ? worker->GetAssignedTask().GetTaskSpecification().ParentTaskId() + retriable ? worker->GetGrantedLease().GetLeaseSpecification().ParentTaskId() : non_retriable_owner_id; auto it = group_map.find(owner_id); @@ -81,7 +81,7 @@ GroupByOwnerIdWorkerKillingPolicy::SelectWorkerToKill( if (left_retriable == right_retriable) { if (left.GetAllWorkers().size() == right.GetAllWorkers().size()) { - return left.GetAssignedTaskTime() > right.GetAssignedTaskTime(); + return left.GetGrantedLeaseTime() > right.GetGrantedLeaseTime(); } return left.GetAllWorkers().size() > right.GetAllWorkers().size(); } @@ -93,9 +93,9 @@ GroupByOwnerIdWorkerKillingPolicy::SelectWorkerToKill( selected_group.GetAllWorkers().size() > 1 && selected_group.IsRetriable(); auto worker_to_kill = selected_group.SelectWorkerToKill(); - RAY_LOG(INFO) << "Sorted list of tasks based on the policy:\n" + RAY_LOG(INFO) << "Sorted list of leases based on the policy:\n" << PolicyDebugString(sorted, system_memory) - << "\nTask should be retried? " << should_retry; + << "\nLease should be retried? " << should_retry; return std::make_pair(worker_to_kill, should_retry); } @@ -105,9 +105,9 @@ std::string GroupByOwnerIdWorkerKillingPolicy::PolicyDebugString( std::stringstream result; int32_t group_index = 0; for (auto &group : groups) { - result << "Tasks (retriable: " << group.IsRetriable() - << ") (parent task id: " << group.OwnerId() << ") (Earliest assigned time: " - << absl::FormatTime(group.GetAssignedTaskTime(), absl::UTCTimeZone()) + result << "Leases (retriable: " << group.IsRetriable() + << ") (parent task id: " << group.OwnerId() << ") (Earliest granted time: " + << absl::FormatTime(group.GetGrantedLeaseTime(), absl::UTCTimeZone()) << "):\n"; int64_t worker_index = 0; @@ -121,11 +121,11 @@ std::string GroupByOwnerIdWorkerKillingPolicy::PolicyDebugString( RAY_LOG_EVERY_MS(INFO, 60000) << "Can't find memory usage for PID, reporting zero. PID: " << pid; } - result << "Task assigned time " - << absl::FormatTime(worker->GetAssignedTaskTime(), absl::UTCTimeZone()) + result << "Lease granted time " + << absl::FormatTime(worker->GetGrantedLeaseTime(), absl::UTCTimeZone()) << " worker id " << worker->WorkerId() << " memory used " << used_memory - << " task spec " - << worker->GetAssignedTask().GetTaskSpecification().DebugString() << "\n"; + << " lease spec " + << worker->GetGrantedLease().GetLeaseSpecification().DebugString() << "\n"; worker_index += 1; if (worker_index > 10) { @@ -146,13 +146,15 @@ const TaskID &Group::OwnerId() const { return owner_id_; } const bool Group::IsRetriable() const { return retriable_; } -const absl::Time Group::GetAssignedTaskTime() const { return earliest_task_time_; } +const absl::Time Group::GetGrantedLeaseTime() const { + return earliest_granted_lease_time_; +} void Group::AddToGroup(std::shared_ptr worker) { - if (worker->GetAssignedTaskTime() < earliest_task_time_) { - earliest_task_time_ = worker->GetAssignedTaskTime(); + if (worker->GetGrantedLeaseTime() < earliest_granted_lease_time_) { + earliest_granted_lease_time_ = worker->GetGrantedLeaseTime(); } - bool retriable = worker->GetAssignedTask().GetTaskSpecification().IsRetriable(); + bool retriable = worker->GetGrantedLease().GetLeaseSpecification().IsRetriable(); RAY_CHECK_EQ(retriable_, retriable); workers_.push_back(worker); } @@ -165,7 +167,7 @@ const std::shared_ptr Group::SelectWorkerToKill() const { sorted.end(), [](std::shared_ptr const &left, std::shared_ptr const &right) -> bool { - return left->GetAssignedTaskTime() > right->GetAssignedTaskTime(); + return left->GetGrantedLeaseTime() > right->GetGrantedLeaseTime(); }); return sorted.front(); diff --git a/src/ray/raylet/worker_killing_policy_group_by_owner.h b/src/ray/raylet/worker_killing_policy_group_by_owner.h index 2d57b6227662..791126aab92d 100644 --- a/src/ray/raylet/worker_killing_policy_group_by_owner.h +++ b/src/ray/raylet/worker_killing_policy_group_by_owner.h @@ -32,8 +32,8 @@ namespace ray { namespace raylet { -/// Key groups on its owner id. For non-retriable task the owner id is itself, -/// Since non-retriable task forms its own group. +/// Key groups on its owner id. For non-retriable lease the owner id is Nil, +/// Since non-retriable lease forms its own group. struct GroupKey { explicit GroupKey(const TaskID &owner_id) : owner_id_(owner_id) {} const TaskID &owner_id_; @@ -44,46 +44,44 @@ struct Group { Group(const TaskID &owner_id, bool retriable) : owner_id_(owner_id), retriable_(retriable) {} - /// The parent task id of the tasks belonging to this group + /// The parent task id of the leases belonging to this group const TaskID &OwnerId() const; - /// Whether tasks in this group are retriable. + /// Whether leases in this group are retriable. const bool IsRetriable() const; - /// Gets the task time of the earliest task of this group, to be + /// Gets the assigned lease time of the earliest lease of this group, to be /// used for group priority. - const absl::Time GetAssignedTaskTime() const; + const absl::Time GetGrantedLeaseTime() const; /// Returns the worker to be killed in this group, in LIFO order. const std::shared_ptr SelectWorkerToKill() const; - /// Tasks belonging to this group. + /// Leases belonging to this group. const std::vector> GetAllWorkers() const; - /// Adds worker that the task belongs to to the group. + /// Adds worker that the lease belongs to to the group. void AddToGroup(std::shared_ptr worker); private: - /// Tasks belonging to this group. + /// Leases belonging to this group. std::vector> workers_; - /// The earliest creation time of the tasks. - absl::Time earliest_task_time_ = absl::Now(); + /// The earliest creation time of the leases. + absl::Time earliest_granted_lease_time_ = absl::Now(); - /// The owner id shared by tasks of this group. - /// TODO(clarng): make this const and implement move / swap. + /// The owner id shared by leases of this group. TaskID owner_id_; - /// Whether the tasks are retriable. - /// TODO(clarng): make this const and implement move / swap. + /// Whether the leases are retriable. bool retriable_; }; -/// Groups task by its owner id. Non-retriable task (whether it be task or actor) forms -/// its own group. Prioritizes killing groups that are retriable first, else it picks the -/// largest group, else it picks the newest group. The "age" of a group is based on the -/// time of its earliest submitted task. When a group is selected for killing it selects -/// the last submitted task. +/// Groups leases by its owner id. Non-retriable leases (whether it be task or actor) +/// forms its own group. Prioritizes killing groups that are retriable first, else it +/// picks the largest group, else it picks the newest group. The "age" of a group is based +/// on the time of its earliest granted leases. When a group is selected for killing it +/// selects the last submitted task. /// /// When selecting a worker / task to be killed, it will set the task to-be-killed to be /// non-retriable if it is the last member of the group, and is retriable otherwise. diff --git a/src/ray/raylet/worker_killing_policy_retriable_fifo.cc b/src/ray/raylet/worker_killing_policy_retriable_fifo.cc index 571517f4558e..1169caf35370 100644 --- a/src/ray/raylet/worker_killing_policy_retriable_fifo.cc +++ b/src/ray/raylet/worker_killing_policy_retriable_fifo.cc @@ -52,13 +52,13 @@ RetriableFIFOWorkerKillingPolicy::SelectWorkerToKill( sorted.end(), [](std::shared_ptr const &left, std::shared_ptr const &right) -> bool { - // First sort by retriable tasks and then by task time in ascending order. + // First sort by retriable leases and then by lease time in ascending order. int left_retriable = - left->GetAssignedTask().GetTaskSpecification().IsRetriable() ? 0 : 1; + left->GetGrantedLease().GetLeaseSpecification().IsRetriable() ? 0 : 1; int right_retriable = - right->GetAssignedTask().GetTaskSpecification().IsRetriable() ? 0 : 1; + right->GetGrantedLease().GetLeaseSpecification().IsRetriable() ? 0 : 1; if (left_retriable == right_retriable) { - return left->GetAssignedTaskTime() < right->GetAssignedTaskTime(); + return left->GetGrantedLeaseTime() < right->GetGrantedLeaseTime(); } return left_retriable < right_retriable; }); diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index 182070e8777b..342068c7f2e6 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -28,10 +28,10 @@ #include "absl/strings/str_split.h" #include "ray/common/constants.h" +#include "ray/common/lease/lease_spec.h" #include "ray/common/ray_config.h" #include "ray/common/runtime_env_common.h" #include "ray/common/status.h" -#include "ray/common/task/task_spec.h" #include "ray/gcs/pb_util.h" #include "ray/stats/metric_defs.h" #include "ray/util/logging.h" @@ -186,12 +186,12 @@ void WorkerPool::Start() { } if (RayConfig::instance().enable_worker_prestart()) { - rpc::TaskSpec rpc_task_spec; - rpc_task_spec.set_language(Language::PYTHON); - rpc_task_spec.mutable_runtime_env_info()->set_serialized_runtime_env("{}"); + rpc::LeaseSpec rpc_lease_spec; + rpc_lease_spec.set_language(Language::PYTHON); + rpc_lease_spec.mutable_runtime_env_info()->set_serialized_runtime_env("{}"); - TaskSpecification task_spec{std::move(rpc_task_spec)}; - PrestartWorkersInternal(task_spec, num_prestart_python_workers); + LeaseSpecification lease_spec{std::move(rpc_lease_spec)}; + PrestartWorkersInternal(lease_spec, num_prestart_python_workers); } } @@ -481,7 +481,7 @@ std::tuple WorkerPool::StartWorkerProcess( auto it = all_jobs_.find(job_id); if (it == all_jobs_.end()) { RAY_LOG(DEBUG) << "Job config of job " << job_id << " are not local yet."; - // Will reschedule ready tasks in `NodeManager::HandleJobStarted`. + // Will reschedule ready leases in `NodeManager::HandleJobStarted`. *status = PopWorkerStatus::JobConfigMissing; process_failed_job_config_missing_++; return {Process(), (StartupToken)-1}; @@ -631,7 +631,7 @@ void WorkerPool::MonitorPopWorkerRequestForRegistration( auto &requests = state.pending_registration_requests; auto it = std::find(requests.begin(), requests.end(), pop_worker_request); if (it != requests.end()) { - // Pop and fail the task... + // Pop and fail the lease... requests.erase(it); PopWorkerStatus status = PopWorkerStatus::WorkerPendingRegistration; PopWorkerCallbackAsync(pop_worker_request->callback_, nullptr, status); @@ -876,7 +876,7 @@ Status WorkerPool::RegisterDriver(const std::shared_ptr &driver const rpc::JobConfig &job_config, std::function send_reply_callback) { int port; - RAY_CHECK(!driver->GetAssignedTaskId().IsNil()); + RAY_CHECK(!driver->GetGrantedLeaseId().IsNil()); Status status = GetNextFreePort(&port); if (!status.ok()) { send_reply_callback(status, /*port=*/0); @@ -894,12 +894,12 @@ Status WorkerPool::RegisterDriver(const std::shared_ptr &driver if (!first_job_registered_ && RayConfig::instance().prestart_worker_first_driver() && !RayConfig::instance().enable_worker_prestart()) { RAY_LOG(DEBUG) << "PrestartDefaultCpuWorkers " << num_prestart_python_workers; - rpc::TaskSpec rpc_task_spec; - rpc_task_spec.set_language(Language::PYTHON); - rpc_task_spec.mutable_runtime_env_info()->set_serialized_runtime_env("{}"); + rpc::LeaseSpec rpc_lease_spec; + rpc_lease_spec.set_language(Language::PYTHON); + rpc_lease_spec.mutable_runtime_env_info()->set_serialized_runtime_env("{}"); - TaskSpecification task_spec{std::move(rpc_task_spec)}; - PrestartWorkersInternal(task_spec, num_prestart_python_workers); + LeaseSpecification lease_spec{std::move(rpc_lease_spec)}; + PrestartWorkersInternal(lease_spec, num_prestart_python_workers); } // Invoke the `send_reply_callback` later to only finish driver @@ -1049,11 +1049,10 @@ void WorkerPool::PopDeleteWorker( } void WorkerPool::PushWorker(const std::shared_ptr &worker) { - // Since the worker is now idle, unset its assigned task ID. - RAY_CHECK(worker->GetAssignedTaskId().IsNil()) - << "Idle workers cannot have an assigned task ID"; - - // Find a task that this worker can fit. If there's none, put it in the idle pool. + // Since the worker is now idle, verify that it has no assigned lease ID. + RAY_CHECK(worker->GetGrantedLeaseId().IsNil()) + << "Idle workers cannot have an assigned lease ID"; + // Find a lease that this worker can fit. If there's none, put it in the idle pool. // First find in pending_registration_requests, then in pending_start_requests. std::shared_ptr pop_worker_request = nullptr; auto &state = GetStateForLanguage(worker->GetLanguage()); @@ -1062,7 +1061,7 @@ void WorkerPool::PushWorker(const std::shared_ptr &worker) { state.pending_registration_requests.begin(), state.pending_registration_requests.end(), [this, &worker](const std::shared_ptr &request) { - return WorkerFitsForTask(*worker, *request) == WorkerUnfitForTaskReason::NONE; + return WorkerFitForLease(*worker, *request) == WorkerUnfitForLeaseReason::NONE; }); if (it != state.pending_registration_requests.end()) { pop_worker_request = *it; @@ -1074,7 +1073,7 @@ void WorkerPool::PushWorker(const std::shared_ptr &worker) { state.pending_start_requests.begin(), state.pending_start_requests.end(), [this, &worker](const std::shared_ptr &request) { - return WorkerFitsForTask(*worker, *request) == WorkerUnfitForTaskReason::NONE; + return WorkerFitForLease(*worker, *request) == WorkerUnfitForLeaseReason::NONE; }); if (it != state.pending_start_requests.end()) { pop_worker_request = *it; @@ -1085,7 +1084,7 @@ void WorkerPool::PushWorker(const std::shared_ptr &worker) { if (pop_worker_request) { bool used = pop_worker_request->callback_(worker, PopWorkerStatus::OK, ""); if (!used) { - // Retry PushWorker. Maybe it can be used by other tasks. + // Retry PushWorker. Maybe it can be used by other leases. // Can we have tail call optimization for this? :) return PushWorker(worker); } @@ -1097,7 +1096,7 @@ void WorkerPool::PushWorker(const std::shared_ptr &worker) { absl::Time keep_alive_until = now + absl::Milliseconds(RayConfig::instance().idle_worker_killing_time_threshold_ms()); - if (worker->GetAssignedTaskTime() == absl::Time()) { + if (worker->GetGrantedLeaseTime() == absl::Time()) { // Newly registered worker. Respect worker_startup_keep_alive_duration if any. auto it = state.worker_processes.find(worker->GetStartupToken()); if (it != state.worker_processes.end()) { @@ -1107,9 +1106,9 @@ void WorkerPool::PushWorker(const std::shared_ptr &worker) { } } - // If the worker never held any tasks, then we should consider it first when + // If the worker never held any leases, then we should consider it first when // choosing which idle workers to kill because it is not warmed up and is slower - // than those workers who served tasks before. + // than those workers who held leases before. // See https://github.com/ray-project/ray/pull/36766 // // Also, we set keep_alive_until w.r.t. worker_startup_keep_alive_duration. @@ -1158,7 +1157,7 @@ void WorkerPool::TryKillingIdleWorkers() { } // Compute the soft limit for the number of idle workers to keep around. - // This assumes the common case where each task requires 1 CPU. + // This assumes the common case where each lease requires 1 CPU. const auto num_desired_idle_workers = get_num_cpus_available_(); RAY_LOG(DEBUG) << "Idle workers: " << idle_of_all_languages_.size() << ", idle workers that are eligible to kill: " @@ -1237,64 +1236,64 @@ void WorkerPool::KillIdleWorker(const IdleWorkerEntry &entry) { }); } -WorkerUnfitForTaskReason WorkerPool::WorkerFitsForTask( +WorkerUnfitForLeaseReason WorkerPool::WorkerFitForLease( const WorkerInterface &worker, const PopWorkerRequest &pop_worker_request) const { if (worker.IsDead()) { - return WorkerUnfitForTaskReason::OTHERS; + return WorkerUnfitForLeaseReason::OTHERS; } // These workers are exiting. So skip them. if (pending_exit_idle_workers_.contains(worker.WorkerId())) { - return WorkerUnfitForTaskReason::OTHERS; + return WorkerUnfitForLeaseReason::OTHERS; } if (worker.GetLanguage() != pop_worker_request.language_) { - return WorkerUnfitForTaskReason::OTHERS; + return WorkerUnfitForLeaseReason::OTHERS; } if (worker.GetWorkerType() != pop_worker_request.worker_type_) { - return WorkerUnfitForTaskReason::OTHERS; + return WorkerUnfitForLeaseReason::OTHERS; } // For scheduling requests with a root detached actor ID, ensure that either the // worker has _no_ detached actor ID or it matches the request. // NOTE(edoakes): the job ID for a worker with no detached actor ID must still match, - // which is checked below. The pop_worker_request for a task rooted in a detached + // which is checked below. The pop_worker_request for a lease rooted in a detached // actor will have the job ID of the job that created the detached actor. if (!pop_worker_request.root_detached_actor_id_.IsNil() && !worker.GetRootDetachedActorId().IsNil() && pop_worker_request.root_detached_actor_id_ != worker.GetRootDetachedActorId()) { - return WorkerUnfitForTaskReason::ROOT_MISMATCH; + return WorkerUnfitForLeaseReason::ROOT_MISMATCH; } // Only consider workers that haven't been assigned to a job yet or have been assigned // to the requested job. const auto worker_job_id = worker.GetAssignedJobId(); if (!worker_job_id.IsNil() && pop_worker_request.job_id_ != worker_job_id) { - return WorkerUnfitForTaskReason::ROOT_MISMATCH; + return WorkerUnfitForLeaseReason::ROOT_MISMATCH; } // If the request asks for a is_gpu, and the worker is assigned a different is_gpu, // then skip it. if (!OptionalsMatchOrEitherEmpty(pop_worker_request.is_gpu_, worker.GetIsGpu())) { - return WorkerUnfitForTaskReason::OTHERS; + return WorkerUnfitForLeaseReason::OTHERS; } // If the request asks for a is_actor_worker, and the worker is assigned a different // is_actor_worker, then skip it. if (!OptionalsMatchOrEitherEmpty(pop_worker_request.is_actor_worker_, worker.GetIsActorWorker())) { - return WorkerUnfitForTaskReason::OTHERS; + return WorkerUnfitForLeaseReason::OTHERS; } // Skip workers with a mismatched runtime_env. - // Even if the task doesn't have a runtime_env specified, we cannot schedule it to a - // worker with a runtime_env because the task is expected to run in the base + // Even if the lease doesn't have a runtime_env specified, we cannot schedule it to a + // worker with a runtime_env because the lease is expected to run in the base // environment. if (worker.GetRuntimeEnvHash() != pop_worker_request.runtime_env_hash_) { - return WorkerUnfitForTaskReason::RUNTIME_ENV_MISMATCH; + return WorkerUnfitForLeaseReason::RUNTIME_ENV_MISMATCH; } // Skip if the dynamic_options doesn't match. if (LookupWorkerDynamicOptions(worker.GetStartupToken()) != pop_worker_request.dynamic_options_) { - return WorkerUnfitForTaskReason::DYNAMIC_OPTIONS_MISMATCH; + return WorkerUnfitForLeaseReason::DYNAMIC_OPTIONS_MISMATCH; } - return WorkerUnfitForTaskReason::NONE; + return WorkerUnfitForLeaseReason::NONE; } void WorkerPool::StartNewWorker( @@ -1361,32 +1360,27 @@ void WorkerPool::StartNewWorker( } } -void WorkerPool::PopWorker(const TaskSpecification &task_spec, +void WorkerPool::PopWorker(const LeaseSpecification &lease_spec, const PopWorkerCallback &callback) { - RAY_LOG(DEBUG) << "Pop worker for task " << task_spec.TaskId() << " task name " - << task_spec.FunctionDescriptor()->ToString(); - // Code path of actor task. - RAY_CHECK(!task_spec.IsActorTask()) << "Direct call shouldn't reach here."; - auto pop_worker_request = std::make_shared( - task_spec.GetLanguage(), + lease_spec.GetLanguage(), rpc::WorkerType::WORKER, - task_spec.JobId(), - task_spec.RootDetachedActorId(), - /*is_gpu=*/task_spec.GetRequiredResources().Get(scheduling::ResourceID::GPU()) > 0, - /*is_actor_worker=*/task_spec.IsActorCreationTask(), - task_spec.RuntimeEnvInfo(), - task_spec.GetRuntimeEnvHash(), - task_spec.DynamicWorkerOptionsOrEmpty(), + lease_spec.JobId(), + lease_spec.RootDetachedActorId(), + /*is_gpu=*/lease_spec.GetRequiredResources().Get(scheduling::ResourceID::GPU()) > 0, + /*is_actor_worker=*/lease_spec.IsActorCreationTask(), + lease_spec.RuntimeEnvInfo(), + lease_spec.GetRuntimeEnvHash(), + lease_spec.DynamicWorkerOptionsOrEmpty(), /*worker_startup_keep_alive_duration=*/std::nullopt, - [this, task_spec, callback]( + [this, lease_spec, callback]( const std::shared_ptr &worker, PopWorkerStatus status, const std::string &runtime_env_setup_error_message) -> bool { - // We got a worker suitable for the task. Now let's check if the task is still + // We got a worker suitable for the lease. Now let's check if the lease is still // executable. - if (worker && finished_jobs_.contains(task_spec.JobId()) && - task_spec.RootDetachedActorId().IsNil()) { + if (worker && finished_jobs_.contains(lease_spec.JobId()) && + lease_spec.RootDetachedActorId().IsNil()) { // When a job finishes, node manager will kill leased workers one time // and worker pool will kill idle workers periodically. // The current worker is already removed from the idle workers @@ -1407,21 +1401,21 @@ void WorkerPool::PopWorker(const TaskSpecification &task_spec, std::shared_ptr WorkerPool::FindAndPopIdleWorker( const PopWorkerRequest &pop_worker_request) { - absl::flat_hash_map skip_reason_count; + absl::flat_hash_map skip_reason_count; - auto worker_fits_for_task_fn = [this, &pop_worker_request, &skip_reason_count]( + auto worker_fit_for_lease_fn = [this, &pop_worker_request, &skip_reason_count]( const IdleWorkerEntry &entry) -> bool { - WorkerUnfitForTaskReason reason = - WorkerFitsForTask(*entry.worker, pop_worker_request); - if (reason == WorkerUnfitForTaskReason::NONE) { + WorkerUnfitForLeaseReason reason = + WorkerFitForLease(*entry.worker, pop_worker_request); + if (reason == WorkerUnfitForLeaseReason::NONE) { return true; } skip_reason_count[reason]++; - if (reason == WorkerUnfitForTaskReason::DYNAMIC_OPTIONS_MISMATCH) { + if (reason == WorkerUnfitForLeaseReason::DYNAMIC_OPTIONS_MISMATCH) { ray_metric_num_cached_workers_skipped_dynamic_options_mismatch_.Record(1); - } else if (reason == WorkerUnfitForTaskReason::RUNTIME_ENV_MISMATCH) { + } else if (reason == WorkerUnfitForLeaseReason::RUNTIME_ENV_MISMATCH) { ray_metric_num_cached_workers_skipped_runtime_environment_mismatch_.Record(1); - } else if (reason == WorkerUnfitForTaskReason::ROOT_MISMATCH) { + } else if (reason == WorkerUnfitForLeaseReason::ROOT_MISMATCH) { ray_metric_num_cached_workers_skipped_job_mismatch_.Record(1); } return false; @@ -1429,7 +1423,7 @@ std::shared_ptr WorkerPool::FindAndPopIdleWorker( auto &state = GetStateForLanguage(pop_worker_request.language_); auto worker_it = std::find_if(idle_of_all_languages_.rbegin(), idle_of_all_languages_.rend(), - worker_fits_for_task_fn); + worker_fit_for_lease_fn); if (worker_it == idle_of_all_languages_.rend()) { RAY_LOG(DEBUG) << "No cached worker, cached workers skipped due to " << debug_string(skip_reason_count); @@ -1444,7 +1438,7 @@ std::shared_ptr WorkerPool::FindAndPopIdleWorker( idle_of_all_languages_.erase(lit); // Assigned workers should always match the request's job_id - // *except* if the task originates from a detached actor. + // *except* if the lease originates from a detached actor. RAY_CHECK(worker->GetAssignedJobId().IsNil() || worker->GetAssignedJobId() == pop_worker_request.job_id_ || !pop_worker_request.root_detached_actor_id_.IsNil()); @@ -1452,7 +1446,7 @@ std::shared_ptr WorkerPool::FindAndPopIdleWorker( } void WorkerPool::PopWorker(std::shared_ptr pop_worker_request) { - // If there's an idle worker that fits the task, use it. + // If there's an idle worker that fits the lease, use it. // Else, start a new worker. auto worker = FindAndPopIdleWorker(*pop_worker_request); if (worker == nullptr) { @@ -1465,21 +1459,21 @@ void WorkerPool::PopWorker(std::shared_ptr pop_worker_request) PopWorkerCallbackAsync(pop_worker_request->callback_, worker, PopWorkerStatus::OK); } -void WorkerPool::PrestartWorkers(const TaskSpecification &task_spec, +void WorkerPool::PrestartWorkers(const LeaseSpecification &lease_spec, int64_t backlog_size) { int64_t num_available_cpus = get_num_cpus_available_(); - // Code path of task that needs a dedicated worker. + // Code path of lease that needs a dedicated worker. RAY_LOG(DEBUG) << "PrestartWorkers, num_available_cpus " << num_available_cpus - << " backlog_size " << backlog_size << " task spec " - << task_spec.DebugString() << " has runtime env " - << task_spec.HasRuntimeEnv(); - if ((task_spec.IsActorCreationTask() && !task_spec.DynamicWorkerOptions().empty()) || - task_spec.GetLanguage() != ray::Language::PYTHON) { + << " backlog_size " << backlog_size << " lease spec " + << lease_spec.DebugString() << " has runtime env " + << lease_spec.HasRuntimeEnv(); + if (lease_spec.IsActorCreationTask() && lease_spec.DynamicWorkerOptionsSize() > 0 && + lease_spec.GetLanguage() != ray::Language::PYTHON) { return; // Not handled. } - auto &state = GetStateForLanguage(task_spec.GetLanguage()); - // The number of available workers that can be used for this task spec. + auto &state = GetStateForLanguage(lease_spec.GetLanguage()); + // The number of available workers that can be used for this lease spec. int num_usable_workers = state.idle.size(); for (auto &entry : state.worker_processes) { num_usable_workers += entry.second.is_pending_registration ? 1 : 0; @@ -1490,48 +1484,48 @@ void WorkerPool::PrestartWorkers(const TaskSpecification &task_spec, if (num_usable_workers < desired_usable_workers) { // Account for workers that are idle or already starting. int64_t num_needed = desired_usable_workers - num_usable_workers; - RAY_LOG(DEBUG) << "Prestarting " << num_needed << " workers given task backlog size " + RAY_LOG(DEBUG) << "Prestarting " << num_needed << " workers given lease backlog size " << backlog_size << " and available CPUs " << num_available_cpus << " num idle workers " << state.idle.size() << " num registered workers " << state.registered_workers.size(); - PrestartWorkersInternal(task_spec, num_needed); + PrestartWorkersInternal(lease_spec, num_needed); } } -void WorkerPool::PrestartWorkersInternal(const TaskSpecification &task_spec, +void WorkerPool::PrestartWorkersInternal(const LeaseSpecification &lease_spec, int64_t num_needed) { RAY_LOG(DEBUG) << "PrestartWorkers " << num_needed; for (int ii = 0; ii < num_needed; ++ii) { // Prestart worker with no runtime env. - if (IsRuntimeEnvEmpty(task_spec.SerializedRuntimeEnv())) { + if (IsRuntimeEnvEmpty(lease_spec.SerializedRuntimeEnv())) { PopWorkerStatus status; StartWorkerProcess( - task_spec.GetLanguage(), rpc::WorkerType::WORKER, task_spec.JobId(), &status); + lease_spec.GetLanguage(), rpc::WorkerType::WORKER, lease_spec.JobId(), &status); continue; } // Prestart worker with runtime env. GetOrCreateRuntimeEnv( - task_spec.SerializedRuntimeEnv(), - task_spec.RuntimeEnvConfig(), - task_spec.JobId(), - [this, task_spec = task_spec](bool successful, - const std::string &serialized_runtime_env_context, - const std::string &setup_error_message) { + lease_spec.SerializedRuntimeEnv(), + lease_spec.RuntimeEnvConfig(), + lease_spec.JobId(), + [this, lease_spec = lease_spec](bool successful, + const std::string &serialized_runtime_env_context, + const std::string &setup_error_message) { if (!successful) { RAY_LOG(ERROR) << "Fails to create or get runtime env " << setup_error_message; return; } PopWorkerStatus status; - StartWorkerProcess(task_spec.GetLanguage(), + StartWorkerProcess(lease_spec.GetLanguage(), rpc::WorkerType::WORKER, - task_spec.JobId(), + lease_spec.JobId(), &status, /*dynamic_options=*/{}, - task_spec.GetRuntimeEnvHash(), + lease_spec.GetRuntimeEnvHash(), serialized_runtime_env_context, - task_spec.RuntimeEnvInfo()); + lease_spec.RuntimeEnvInfo()); }); } } diff --git a/src/ray/raylet/worker_pool.h b/src/ray/raylet/worker_pool.h index 8da7d3046472..30cdb2dc82d7 100644 --- a/src/ray/raylet/worker_pool.h +++ b/src/ray/raylet/worker_pool.h @@ -34,9 +34,8 @@ #include "absl/time/time.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/asio/periodical_runner.h" +#include "ray/common/lease/lease.h" #include "ray/common/runtime_env_manager.h" -#include "ray/common/task/task.h" -#include "ray/common/task/task_common.h" #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/ipc/client_connection.h" #include "ray/raylet/runtime_env_agent_client.h" @@ -67,7 +66,7 @@ enum PopWorkerStatus { // Any fails of runtime env creation. // A nullptr worker will be returned with callback. RuntimeEnvCreationFailed = 4, - // The task's job has finished. + // The lease's job has finished. // A nullptr worker will be returned with callback. JobFinished = 5, }; @@ -154,7 +153,7 @@ class WorkerPoolInterface : public IOWorkerPoolInterface { /// Pop an idle worker from the pool. The caller is responsible for pushing /// the worker back onto the pool once the worker has completed its work. /// - /// \param task_spec The returned worker must be able to execute this task. + /// \param lease_spec The returned worker must be able to execute this lease. /// \param callback The callback function that executed when gets the result of /// worker popping. /// The callback will be executed with an empty worker in following cases: @@ -168,7 +167,7 @@ class WorkerPoolInterface : public IOWorkerPoolInterface { /// Case 1: An suitable worker was found in idle worker pool. /// Case 2: An suitable worker registered to raylet. /// The corresponding PopWorkerStatus will be passed to the callback. - virtual void PopWorker(const TaskSpecification &task_spec, + virtual void PopWorker(const LeaseSpecification &lease_spec, const PopWorkerCallback &callback) = 0; /// Add an idle worker to the pool. /// @@ -238,7 +237,7 @@ class WorkerPoolInterface : public IOWorkerPoolInterface { virtual void DisconnectDriver(const std::shared_ptr &driver) = 0; - virtual void PrestartWorkers(const TaskSpecification &task_spec, + virtual void PrestartWorkers(const LeaseSpecification &lease_spec, int64_t backlog_size) = 0; virtual void StartNewWorker( @@ -250,14 +249,14 @@ class WorkerPoolInterface : public IOWorkerPoolInterface { class WorkerInterface; class Worker; -enum class WorkerUnfitForTaskReason { +enum class WorkerUnfitForLeaseReason { NONE = 0, // OK ROOT_MISMATCH = 1, // job ID or root detached actor ID mismatch RUNTIME_ENV_MISMATCH = 2, // runtime env hash mismatch DYNAMIC_OPTIONS_MISMATCH = 3, // dynamic options mismatch OTHERS = 4, // reasons we don't do stats for (e.g. language) }; -static constexpr std::string_view kWorkerUnfitForTaskReasonDebugName[] = { +static constexpr std::string_view kWorkerUnfitForLeaseReasonDebugName[] = { "NONE", "ROOT_MISMATCH", "RUNTIME_ENV_MISMATCH", @@ -266,8 +265,8 @@ static constexpr std::string_view kWorkerUnfitForTaskReasonDebugName[] = { }; inline std::ostream &operator<<(std::ostream &os, - const WorkerUnfitForTaskReason &reason) { - os << kWorkerUnfitForTaskReasonDebugName[static_cast(reason)]; + const WorkerUnfitForLeaseReason &reason) { + os << kWorkerUnfitForLeaseReasonDebugName[static_cast(reason)]; return os; } @@ -477,19 +476,20 @@ class WorkerPool : public WorkerPoolInterface { void PushWorker(const std::shared_ptr &worker) override; /// See interface. - void PopWorker(const TaskSpecification &task_spec, + void PopWorker(const LeaseSpecification &lease_spec, const PopWorkerCallback &callback) override; - /// Try to prestart a number of workers suitable the given task spec. Prestarting + /// Try to prestart a number of workers suitable the given lease spec. Prestarting /// is needed since core workers request one lease at a time, if starting is slow, /// then it means it takes a long time to scale up. /// - /// \param task_spec The returned worker must be able to execute this task. - /// \param backlog_size The number of tasks in the client backlog of this shape. + /// \param lease_spec The returned worker must be able to execute this lease. + /// \param backlog_size The number of leases in the client backlog of this shape. /// We aim to prestart 1 worker per CPU, up to the backlog size. - void PrestartWorkers(const TaskSpecification &task_spec, int64_t backlog_size) override; + void PrestartWorkers(const LeaseSpecification &lease_spec, + int64_t backlog_size) override; - void PrestartWorkersInternal(const TaskSpecification &task_spec, int64_t num_needed); + void PrestartWorkersInternal(const LeaseSpecification &lease_spec, int64_t num_needed); /// Return the current size of the worker pool for the requested language. Counts only /// idle workers. @@ -534,7 +534,7 @@ class WorkerPool : public WorkerPoolInterface { /// Internal implementation of PopWorker. void PopWorker(std::shared_ptr pop_worker_request); - // Find an idle worker that can serve the task. If found, pop it out and return it. + // Find an idle worker that can serve the lease. If found, pop it out and return it. // Otherwise, return nullptr. std::shared_ptr FindAndPopIdleWorker( const PopWorkerRequest &pop_worker_request); @@ -570,8 +570,8 @@ class WorkerPool : public WorkerPoolInterface { /// \param serialized_runtime_env_context The context of runtime env. /// \param runtime_env_info The raw runtime env info. /// \param worker_startup_keep_alive_duration If set, the worker will be kept alive for - /// this duration even if it's idle. This is only applicable before a task is assigned - /// to the worker. + /// this duration even if it's idle. This is only applicable before a lease is + /// assigned to the worker. /// \return The process that we started and a token. If the token is less than 0, /// we didn't start a process. std::tuple StartWorkerProcess( @@ -639,7 +639,7 @@ class WorkerPool : public WorkerPoolInterface { rpc::RuntimeEnvInfo runtime_env_info; /// The dynamic_options. std::vector dynamic_options; - /// The duration to keep the newly created worker alive before it's assigned a task. + /// The duration to keep the newly created worker alive before it's assigned a lease. std::optional worker_startup_keep_alive_duration; }; @@ -843,9 +843,9 @@ class WorkerPool : public WorkerPoolInterface { /// /// \param[in] worker The worker. /// \param[in] pop_worker_request The pop worker request. - /// \return WorkerUnfitForTaskReason::NONE if the worker can be used, else a + /// \return WorkerUnfitForLeaseReason::NONE if the worker can be used, else a /// status indicating why it cannot. - WorkerUnfitForTaskReason WorkerFitsForTask( + WorkerUnfitForLeaseReason WorkerFitForLease( const WorkerInterface &worker, const PopWorkerRequest &pop_worker_request) const; /// For Process class for managing subprocesses (e.g. reaping zombies). diff --git a/src/ray/raylet_client/raylet_client.cc b/src/ray/raylet_client/raylet_client.cc index bfb8b0665d0a..8e55cb4adfe7 100644 --- a/src/ray/raylet_client/raylet_client.cc +++ b/src/ray/raylet_client/raylet_client.cc @@ -23,7 +23,6 @@ #include "absl/synchronization/notification.h" #include "ray/common/common_protocol.h" #include "ray/common/ray_config.h" -#include "ray/common/task/task_spec.h" #include "ray/util/logging.h" namespace ray::raylet { @@ -37,7 +36,7 @@ RayletClient::RayletClient(const rpc::Address &address, std::move(raylet_unavailable_timeout_callback)))) {} void RayletClient::RequestWorkerLease( - const rpc::TaskSpec &task_spec, + const rpc::LeaseSpec &lease_spec, bool grant_or_reject, const rpc::ClientCallback &callback, const int64_t backlog_size, @@ -46,11 +45,11 @@ void RayletClient::RequestWorkerLease( auto request = google::protobuf::Arena::CreateMessage(&arena); // The unsafe allocating here is actually safe because the life-cycle of - // task_spec is longer than request. + // lease_spec is longer than request. // Request will be sent before the end of this call, and after that, it won't be // used any more. - request->unsafe_arena_set_allocated_resource_spec( - const_cast(&task_spec)); + request->unsafe_arena_set_allocated_lease_spec( + const_cast(&lease_spec)); request->set_grant_or_reject(grant_or_reject); request->set_backlog_size(backlog_size); request->set_is_selected_based_on_locality(is_selected_based_on_locality); @@ -77,35 +76,35 @@ void RayletClient::ReportWorkerBacklog( request, [](const Status &status, rpc::ReportWorkerBacklogReply &&reply /*unused*/) { RAY_LOG_IF_ERROR(INFO, status) - << "Error reporting task backlog information: " << status; + << "Error reporting lease backlog information: " << status; }); } -Status RayletClient::ReturnWorker(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) { - rpc::ReturnWorkerRequest request; +Status RayletClient::ReturnWorkerLease(int worker_port, + const WorkerID &worker_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) { + rpc::ReturnWorkerLeaseRequest request; request.set_worker_port(worker_port); request.set_worker_id(worker_id.Binary()); request.set_disconnect_worker(disconnect_worker); request.set_disconnect_worker_error_detail(disconnect_worker_error_detail); request.set_worker_exiting(worker_exiting); - grpc_client_->ReturnWorker( - request, [](const Status &status, rpc::ReturnWorkerReply &&reply /*unused*/) { + grpc_client_->ReturnWorkerLease( + request, [](const Status &status, rpc::ReturnWorkerLeaseReply &&reply /*unused*/) { RAY_LOG_IF_ERROR(INFO, status) << "Error returning worker: " << status; }); return Status::OK(); } -void RayletClient::GetTaskFailureCause( - const TaskID &task_id, - const ray::rpc::ClientCallback &callback) { - rpc::GetTaskFailureCauseRequest request; - request.set_task_id(task_id.Binary()); - grpc_client_->GetTaskFailureCause( - request, [callback](const Status &status, rpc::GetTaskFailureCauseReply &&reply) { +void RayletClient::GetWorkerFailureCause( + const LeaseID &lease_id, + const ray::rpc::ClientCallback &callback) { + rpc::GetWorkerFailureCauseRequest request; + request.set_lease_id(lease_id.Binary()); + grpc_client_->GetWorkerFailureCause( + request, [callback](const Status &status, rpc::GetWorkerFailureCauseReply &&reply) { RAY_LOG_IF_ERROR(INFO, status) << "Error getting task result: " << status; callback(status, std::move(reply)); }); @@ -189,10 +188,10 @@ void RayletClient::ReleaseUnusedActorWorkers( } void RayletClient::CancelWorkerLease( - const TaskID &task_id, + const LeaseID &lease_id, const rpc::ClientCallback &callback) { rpc::CancelWorkerLeaseRequest request; - request.set_task_id(task_id.Binary()); + request.set_lease_id(lease_id.Binary()); grpc_client_->CancelWorkerLease(request, callback); } @@ -312,19 +311,19 @@ void RayletClient::GetResourceLoad( grpc_client_->GetResourceLoad(request, callback); } -void RayletClient::CancelTasksWithResourceShapes( +void RayletClient::CancelLeasesWithResourceShapes( const std::vector> &resource_shapes, - const rpc::ClientCallback &callback) { - rpc::CancelTasksWithResourceShapesRequest request; + const rpc::ClientCallback &callback) { + rpc::CancelLeasesWithResourceShapesRequest request; for (const auto &resource_shape : resource_shapes) { - rpc::CancelTasksWithResourceShapesRequest::ResourceShape *resource_shape_proto = + rpc::CancelLeasesWithResourceShapesRequest::ResourceShape *resource_shape_proto = request.add_resource_shapes(); resource_shape_proto->mutable_resource_shape()->insert(resource_shape.begin(), resource_shape.end()); } - grpc_client_->CancelTasksWithResourceShapes(request, callback); + grpc_client_->CancelLeasesWithResourceShapes(request, callback); } void RayletClient::NotifyGCSRestart( diff --git a/src/ray/raylet_client/raylet_client.h b/src/ray/raylet_client/raylet_client.h index cba4feeee6b7..75592cb1a90c 100644 --- a/src/ray/raylet_client/raylet_client.h +++ b/src/ray/raylet_client/raylet_client.h @@ -25,7 +25,6 @@ #include "ray/common/bundle_spec.h" #include "ray/common/status.h" #include "ray/common/status_or.h" -#include "ray/common/task/task_spec.h" #include "ray/ipc/client_connection.h" #include "ray/rpc/node_manager/node_manager_client.h" #include "ray/util/process.h" @@ -57,13 +56,14 @@ class RayletClientInterface { const ray::rpc::ClientCallback &callback) = 0; /// Requests a worker from the raylet. The callback will be sent via gRPC. - /// \param resource_spec Resources that should be allocated for the worker. + /// \param lease_spec Lease that is requested by the owner. /// \param grant_or_reject: True if we we should either grant or reject the request /// but no spillback. /// \param callback: The callback to call when the request finishes. /// \param backlog_size The queue length for the given shape on the CoreWorker. + /// \param lease_id Unique lease ID for this worker lease request. virtual void RequestWorkerLease( - const rpc::TaskSpec &task_spec, + const rpc::LeaseSpec &lease_spec, bool grant_or_reject, const ray::rpc::ClientCallback &callback, const int64_t backlog_size = -1, @@ -75,11 +75,11 @@ class RayletClientInterface { /// \param disconnect_worker Whether the raylet should disconnect the worker. /// \param worker_exiting Whether the worker is exiting and cannot be reused. /// \return ray::Status - virtual ray::Status ReturnWorker(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) = 0; + virtual ray::Status ReturnWorkerLease(int worker_port, + const WorkerID &worker_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) = 0; /// Request the raylet to prestart workers. In `request` we can set the worker's owner, /// runtime env info and number of workers. @@ -97,7 +97,7 @@ class RayletClientInterface { const rpc::ClientCallback &callback) = 0; virtual void CancelWorkerLease( - const TaskID &task_id, + const LeaseID &lease_id, const rpc::ClientCallback &callback) = 0; /// Report the backlog size of a given worker and a given scheduling class to the @@ -108,9 +108,9 @@ class RayletClientInterface { const WorkerID &worker_id, const std::vector &backlog_reports) = 0; - virtual void GetTaskFailureCause( - const TaskID &task_id, - const ray::rpc::ClientCallback &callback) = 0; + virtual void GetWorkerFailureCause( + const LeaseID &lease_id, + const ray::rpc::ClientCallback &callback) = 0; /// Request a raylet to prepare resources of given bundles for atomic placement group /// creation. This is used for the first phase of atomic placement group creation. The @@ -202,9 +202,9 @@ class RayletClientInterface { int64_t deadline_timestamp_ms, const rpc::ClientCallback &callback) = 0; - virtual void CancelTasksWithResourceShapes( + virtual void CancelLeasesWithResourceShapes( const std::vector> &resource_shapes, - const rpc::ClientCallback &callback) = 0; + const rpc::ClientCallback &callback) = 0; virtual void IsLocalWorkerDead( const WorkerID &worker_id, @@ -240,25 +240,25 @@ class RayletClient : public RayletClientInterface { std::shared_ptr GetChannel() const override; void RequestWorkerLease( - const rpc::TaskSpec &resource_spec, + const rpc::LeaseSpec &lease_spec, bool grant_or_reject, const ray::rpc::ClientCallback &callback, const int64_t backlog_size, const bool is_selected_based_on_locality) override; - ray::Status ReturnWorker(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) override; + ray::Status ReturnWorkerLease(int worker_port, + const WorkerID &worker_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) override; void PrestartWorkers( const ray::rpc::PrestartWorkersRequest &request, const ray::rpc::ClientCallback &callback) override; - void GetTaskFailureCause( - const TaskID &task_id, - const ray::rpc::ClientCallback &callback) + void GetWorkerFailureCause( + const LeaseID &lease_id, + const ray::rpc::ClientCallback &callback) override; void RegisterMutableObjectReader( @@ -285,7 +285,7 @@ class RayletClient : public RayletClientInterface { const rpc::ClientCallback &callback) override; void CancelWorkerLease( - const TaskID &task_id, + const LeaseID &lease_id, const rpc::ClientCallback &callback) override; void PrepareBundleResources( @@ -323,9 +323,9 @@ class RayletClient : public RayletClientInterface { int64_t deadline_timestamp_ms, const rpc::ClientCallback &callback) override; - void CancelTasksWithResourceShapes( + void CancelLeasesWithResourceShapes( const std::vector> &resource_shapes, - const rpc::ClientCallback &callback) + const rpc::ClientCallback &callback) override; void IsLocalWorkerDead( diff --git a/src/ray/rpc/node_manager/node_manager_client.h b/src/ray/rpc/node_manager/node_manager_client.h index 7f119f2db8af..7d8d05df8232 100644 --- a/src/ray/rpc/node_manager/node_manager_client.h +++ b/src/ray/rpc/node_manager/node_manager_client.h @@ -81,7 +81,7 @@ class NodeManagerClient { /*method_timeout_ms*/ -1, ) VOID_RPC_CLIENT_METHOD(NodeManagerService, - CancelTasksWithResourceShapes, + CancelLeasesWithResourceShapes, grpc_client_, /*method_timeout_ms*/ -1, ) @@ -106,7 +106,7 @@ class NodeManagerClient { /*method_timeout_ms*/ -1, ) VOID_RPC_CLIENT_METHOD(NodeManagerService, - ReturnWorker, + ReturnWorkerLease, grpc_client_, /*method_timeout_ms*/ -1, ) @@ -177,7 +177,7 @@ class NodeManagerClient { /*method_timeout_ms*/ -1, ) VOID_RPC_CLIENT_METHOD(NodeManagerService, - GetTaskFailureCause, + GetWorkerFailureCause, grpc_client_, /*method_timeout_ms*/ -1, ) diff --git a/src/ray/rpc/node_manager/node_manager_server.h b/src/ray/rpc/node_manager/node_manager_server.h index 7e0a726832e4..d8b2a781a151 100644 --- a/src/ray/rpc/node_manager/node_manager_server.h +++ b/src/ray/rpc/node_manager/node_manager_server.h @@ -31,32 +31,32 @@ namespace rpc { RPC_SERVICE_HANDLER_CUSTOM_AUTH(NodeManagerService, METHOD, -1, AuthType::NO_AUTH) /// NOTE: See src/ray/core_worker/core_worker.h on how to add a new grpc handler. -#define RAY_NODE_MANAGER_RPC_HANDLERS \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GetResourceLoad) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(CancelTasksWithResourceShapes) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(NotifyGCSRestart) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(RequestWorkerLease) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(PrestartWorkers) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ReportWorkerBacklog) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ReturnWorker) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ReleaseUnusedActorWorkers) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(CancelWorkerLease) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(PinObjectIDs) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GetNodeStats) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GlobalGC) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(FormatGlobalMemoryInfo) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(PrepareBundleResources) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(CommitBundleResources) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(CancelResourceReserve) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ResizeLocalResourceInstances) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ReleaseUnusedBundles) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GetSystemConfig) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(IsLocalWorkerDead) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ShutdownRaylet) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(DrainRaylet) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GetObjectsInfo) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GetTaskFailureCause) \ - RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(RegisterMutableObject) \ +#define RAY_NODE_MANAGER_RPC_HANDLERS \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GetResourceLoad) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(CancelLeasesWithResourceShapes) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(NotifyGCSRestart) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(RequestWorkerLease) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(PrestartWorkers) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ReportWorkerBacklog) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ReturnWorkerLease) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ReleaseUnusedActorWorkers) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(CancelWorkerLease) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(PinObjectIDs) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GetNodeStats) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GlobalGC) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(FormatGlobalMemoryInfo) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(PrepareBundleResources) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(CommitBundleResources) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(CancelResourceReserve) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ResizeLocalResourceInstances) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ReleaseUnusedBundles) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GetSystemConfig) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(IsLocalWorkerDead) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(ShutdownRaylet) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(DrainRaylet) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GetObjectsInfo) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(GetWorkerFailureCause) \ + RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(RegisterMutableObject) \ RAY_NODE_MANAGER_RPC_SERVICE_HANDLER(PushMutableObject) /// Interface of the `NodeManagerService`, see `src/ray/protobuf/node_manager.proto`. @@ -77,9 +77,9 @@ class NodeManagerServiceHandler { rpc::GetResourceLoadReply *reply, rpc::SendReplyCallback send_reply_callback) = 0; - virtual void HandleCancelTasksWithResourceShapes( - rpc::CancelTasksWithResourceShapesRequest request, - rpc::CancelTasksWithResourceShapesReply *reply, + virtual void HandleCancelLeasesWithResourceShapes( + rpc::CancelLeasesWithResourceShapesRequest request, + rpc::CancelLeasesWithResourceShapesReply *reply, rpc::SendReplyCallback send_reply_callback) = 0; virtual void HandleNotifyGCSRestart(rpc::NotifyGCSRestartRequest request, @@ -98,9 +98,9 @@ class NodeManagerServiceHandler { ReportWorkerBacklogReply *reply, SendReplyCallback send_reply_callback) = 0; - virtual void HandleReturnWorker(ReturnWorkerRequest request, - ReturnWorkerReply *reply, - SendReplyCallback send_reply_callback) = 0; + virtual void HandleReturnWorkerLease(ReturnWorkerLeaseRequest request, + ReturnWorkerLeaseReply *reply, + SendReplyCallback send_reply_callback) = 0; virtual void HandleReleaseUnusedActorWorkers(ReleaseUnusedActorWorkersRequest request, ReleaseUnusedActorWorkersReply *reply, @@ -170,9 +170,9 @@ class NodeManagerServiceHandler { GetObjectsInfoReply *reply, SendReplyCallback send_reply_callback) = 0; - virtual void HandleGetTaskFailureCause(GetTaskFailureCauseRequest request, - GetTaskFailureCauseReply *reply, - SendReplyCallback send_reply_callback) = 0; + virtual void HandleGetWorkerFailureCause(GetWorkerFailureCauseRequest request, + GetWorkerFailureCauseReply *reply, + SendReplyCallback send_reply_callback) = 0; virtual void HandleRegisterMutableObject(RegisterMutableObjectRequest request, RegisterMutableObjectReply *reply, diff --git a/src/ray/util/logging.h b/src/ray/util/logging.h index 040a609f9722..405d772b57ef 100644 --- a/src/ray/util/logging.h +++ b/src/ray/util/logging.h @@ -104,6 +104,7 @@ inline constexpr std::string_view kLogKeyActorID = "actor_id"; inline constexpr std::string_view kLogKeyTaskID = "task_id"; inline constexpr std::string_view kLogKeyObjectID = "object_id"; inline constexpr std::string_view kLogKeyPlacementGroupID = "placement_group_id"; +inline constexpr std::string_view kLogKeyLeaseID = "lease_id"; // Define your specialization DefaultLogKey::key to get .WithField(t) // See src/ray/common/id.h From b60e1277af7512bef2176d002c183444d20a2330 Mon Sep 17 00:00:00 2001 From: Yicheng-Lu-llll <51814063+Yicheng-Lu-llll@users.noreply.github.com> Date: Thu, 28 Aug 2025 21:19:18 -0700 Subject: [PATCH 360/634] [3/n] IPv6 support: Add Cython bindings for network utility functions (#55443) Signed-off-by: Yicheng-Lu-llll Co-authored-by: Jiajun Yao --- python/ray/_common/network_utils.py | 25 ++------ .../ray/_common/tests/test_network_utils.py | 60 +------------------ python/ray/_raylet.pyx | 32 +++++----- .../modules/reporter/tests/test_reporter.py | 6 ++ python/ray/includes/array.pxd | 6 ++ python/ray/includes/network_util.pxd | 9 +++ python/ray/includes/network_util.pxi | 47 +++++++++++++++ python/ray/serve/_private/controller.py | 3 + 8 files changed, 93 insertions(+), 95 deletions(-) create mode 100644 python/ray/includes/array.pxd create mode 100644 python/ray/includes/network_util.pxd create mode 100644 python/ray/includes/network_util.pxi diff --git a/python/ray/_common/network_utils.py b/python/ray/_common/network_utils.py index 952559142398..9664c53a94a2 100644 --- a/python/ray/_common/network_utils.py +++ b/python/ray/_common/network_utils.py @@ -1,5 +1,8 @@ from typing import Optional, Tuple, Union +from ray._raylet import build_address as _build_address +from ray._raylet import parse_address as _parse_address + def parse_address(address: str) -> Optional[Tuple[str, str]]: """Parse a network address string into host and port. @@ -10,21 +13,7 @@ def parse_address(address: str) -> Optional[Tuple[str, str]]: Returns: Tuple with (host, port) if port found, None if no colon separator. """ - pos = address.rfind(":") - if pos == -1: - return None - - host = address[:pos] - port = address[pos + 1 :] - - if ":" in host: - if host.startswith("[") and host.endswith("]"): - host = host[1:-1] - else: - # Invalid IPv6 (missing brackets) or colon is part of the address, not a host:port split. - return None - - return (host, port) + return _parse_address(address) def build_address(host: str, port: Union[int, str]) -> str: @@ -37,11 +26,7 @@ def build_address(host: str, port: Union[int, str]) -> str: Returns: Formatted address string (e.g., "localhost:8000" or "[::1]:8000"). """ - if host is not None and ":" in host: - # IPv6 address - return f"[{host}]:{port}" - # IPv4 address or hostname - return f"{host}:{port}" + return _build_address(host, port) def is_localhost(host: str) -> bool: diff --git a/python/ray/_common/tests/test_network_utils.py b/python/ray/_common/tests/test_network_utils.py index 98b070cf7b6c..347eed26055a 100644 --- a/python/ray/_common/tests/test_network_utils.py +++ b/python/ray/_common/tests/test_network_utils.py @@ -1,7 +1,7 @@ import pytest import sys -from ray._common.network_utils import parse_address, build_address, is_localhost +from ray._common.network_utils import is_localhost def test_is_localhost(): @@ -12,63 +12,5 @@ def test_is_localhost(): assert not is_localhost("2001:db8::1") -@pytest.mark.parametrize( - "host,port,expected", - [ - # IPv4 - ("192.168.1.1", 8080, "192.168.1.1:8080"), - ("192.168.1.1", "8080", "192.168.1.1:8080"), - # IPv6 - ("::1", 8080, "[::1]:8080"), - ("::1", "8080", "[::1]:8080"), - ("2001:db8::1", 8080, "[2001:db8::1]:8080"), - ("2001:db8::1", "8080", "[2001:db8::1]:8080"), - # Hostname - ("localhost", 9000, "localhost:9000"), - ("localhost", "9000", "localhost:9000"), - ], -) -def test_build_address(host, port, expected): - """Test cases for build_address function, matching C++ tests exactly.""" - result = build_address(host, port) - assert result == expected - - -@pytest.mark.parametrize( - "address,expected", - [ - # IPv4 - ("192.168.1.1:8080", ("192.168.1.1", "8080")), - # IPv6:loopback address - ("[::1]:8080", ("::1", "8080")), - # IPv6 - ("[2001:db8::1]:8080", ("2001:db8::1", "8080")), - # Hostname:Port - ("localhost:9000", ("localhost", "9000")), - ], -) -def test_parse_valid_addresses(address, expected): - """Test cases for parse_address function, matching C++ tests exactly.""" - result = parse_address(address) - assert result == expected - - -@pytest.mark.parametrize( - "address", - [ - # bare IP or hostname - # should return None when no port is found - "::1", - "2001:db8::1", - "192.168.1.1", - "localhost", - ], -) -def test_parse_bare_addresses(address): - """Test parsing bare addresses returns None.""" - result = parse_address(address) - assert result is None - - if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index 4b4ce13e0205..5765d272a115 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -184,6 +184,22 @@ from ray.includes.optional cimport ( optional, nullopt ) +cimport cpython + +include "includes/network_util.pxi" +include "includes/object_ref.pxi" +include "includes/unique_ids.pxi" +include "includes/ray_config.pxi" +include "includes/function_descriptor.pxi" +include "includes/buffer.pxi" +include "includes/common.pxi" +include "includes/gcs_client.pxi" +include "includes/serialization.pxi" +include "includes/libcoreworker.pxi" +include "includes/global_state_accessor.pxi" +include "includes/metric.pxi" +include "includes/setproctitle.pxi" + import ray from ray.exceptions import ( RayActorError, @@ -230,25 +246,9 @@ import ray._private.profiling as profiling from ray._common.utils import decode from ray._private.utils import DeferSigint from ray._private.object_ref_generator import DynamicObjectRefGenerator -from ray._common.network_utils import build_address, parse_address from ray.util.annotations import PublicAPI from ray._private.custom_types import TensorTransportEnum -cimport cpython - -include "includes/object_ref.pxi" -include "includes/unique_ids.pxi" -include "includes/ray_config.pxi" -include "includes/function_descriptor.pxi" -include "includes/buffer.pxi" -include "includes/common.pxi" -include "includes/gcs_client.pxi" -include "includes/serialization.pxi" -include "includes/libcoreworker.pxi" -include "includes/global_state_accessor.pxi" -include "includes/metric.pxi" -include "includes/setproctitle.pxi" - # Expose GCC & Clang macro to report # whether C++ optimizations were enabled during compilation. OPTIMIZED = __OPTIMIZE__ diff --git a/python/ray/dashboard/modules/reporter/tests/test_reporter.py b/python/ray/dashboard/modules/reporter/tests/test_reporter.py index 58fd949584a7..658b23dfa9ee 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_reporter.py +++ b/python/ray/dashboard/modules/reporter/tests/test_reporter.py @@ -313,6 +313,7 @@ def test_worker_stats(): def test_report_stats(): dashboard_agent = MagicMock() + dashboard_agent.gcs_address = build_address("127.0.0.1", 6379) agent = ReporterAgent(dashboard_agent) # Assume it is a head node. agent._is_head_node = True @@ -374,6 +375,7 @@ def test_report_stats(): def test_report_stats_gpu(): dashboard_agent = MagicMock() + dashboard_agent.gcs_address = build_address("127.0.0.1", 6379) agent = ReporterAgent(dashboard_agent) # Assume it is a head node. agent._is_head_node = True @@ -494,6 +496,7 @@ def test_report_stats_gpu(): def test_report_per_component_stats_gpu(): dashboard_agent = MagicMock() + dashboard_agent.gcs_address = build_address("127.0.0.1", 6379) agent = ReporterAgent(dashboard_agent) # Assume it is a head node. agent._is_head_node = True @@ -857,6 +860,7 @@ def create_mock_worker_processes(): def test_get_tpu_usage(): dashboard_agent = MagicMock() + dashboard_agent.gcs_address = build_address("127.0.0.1", 6379) agent = ReporterAgent(dashboard_agent) fake_metrics_content = """ @@ -913,6 +917,7 @@ def test_get_tpu_usage(): def test_report_stats_tpu(): dashboard_agent = MagicMock() + dashboard_agent.gcs_address = build_address("127.0.0.1", 6379) agent = ReporterAgent(dashboard_agent) STATS_TEMPLATE["tpus"] = [ @@ -985,6 +990,7 @@ def test_report_stats_tpu(): def test_report_per_component_stats(): dashboard_agent = MagicMock() + dashboard_agent.gcs_address = build_address("127.0.0.1", 6379) agent = ReporterAgent(dashboard_agent) # Assume it is a head node. agent._is_head_node = True diff --git a/python/ray/includes/array.pxd b/python/ray/includes/array.pxd new file mode 100644 index 000000000000..a6ce5e135a70 --- /dev/null +++ b/python/ray/includes/array.pxd @@ -0,0 +1,6 @@ +from libc.stddef cimport size_t +from libcpp.string cimport string + +cdef extern from "" namespace "std": + cdef cppclass array_string_2 "std::array": + string& operator[](size_t) except + diff --git a/python/ray/includes/network_util.pxd b/python/ray/includes/network_util.pxd new file mode 100644 index 000000000000..df4f8cb9d18d --- /dev/null +++ b/python/ray/includes/network_util.pxd @@ -0,0 +1,9 @@ +from libc.stddef cimport size_t +from libcpp.string cimport string +from ray.includes.array cimport array_string_2 +from ray.includes.optional cimport optional + +cdef extern from "ray/util/network_util.h" namespace "ray": + optional[array_string_2] ParseAddress(const string &address) + string BuildAddress(const string &host, const string &port) + string BuildAddress(const string &host, int port) diff --git a/python/ray/includes/network_util.pxi b/python/ray/includes/network_util.pxi new file mode 100644 index 000000000000..27e330eeace4 --- /dev/null +++ b/python/ray/includes/network_util.pxi @@ -0,0 +1,47 @@ +from ray.includes.network_util cimport ( + BuildAddress, + ParseAddress, + array_string_2, + optional, +) +from libcpp.string cimport string +from typing import Optional, Tuple, Union + +def parse_address(address: str) -> Optional[Tuple[str, str]]: + """Parse a network address string into host and port. + + Args: + address: The address string to parse (e.g., "localhost:8000", "[::1]:8000"). + + Returns: + Tuple with (host, port) if port found, None if no colon separator. + """ + cdef optional[array_string_2] res = ParseAddress(address.encode('utf-8')) + if not res.has_value(): + return None + + cdef array_string_2 ip_port = res.value() + return (ip_port[0].decode('utf-8'), ip_port[1].decode('utf-8')) + + +def build_address(host: str, port: Union[int, str]) -> str: + """Build a network address string from host and port. + + Args: + host: The hostname or IP address. + port: The port number (int or string). + + Returns: + Formatted address string (e.g., "localhost:8000" or "[::1]:8000"). + """ + cdef string host_c = host.encode('utf-8') + cdef string result + cdef string port_c + + if isinstance(port, int): + result = BuildAddress(host_c, port) + else: + port_c = str(port).encode('utf-8') + result = BuildAddress(host_c, port_c) + + return result.decode('utf-8') diff --git a/python/ray/serve/_private/controller.py b/python/ray/serve/_private/controller.py index 3caad988f715..5d53906cecf5 100644 --- a/python/ray/serve/_private/controller.py +++ b/python/ray/serve/_private/controller.py @@ -652,6 +652,9 @@ def get_root_url(self): if SERVE_ROOT_URL_ENV_KEY in os.environ: return os.environ[SERVE_ROOT_URL_ENV_KEY] else: + # HTTP is disabled + if http_config.host is None: + return "" return ( f"http://{build_address(http_config.host, http_config.port)}" f"{http_config.root_path}" From e7889ae542bf0188610bc8b06d274cbf53790cbd Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Thu, 28 Aug 2025 21:24:24 -0700 Subject: [PATCH 361/634] [core] Fix windows build wshadow (#56060) Signed-off-by: dayshah --- bazel/ray.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/bazel/ray.bzl b/bazel/ray.bzl index f2215880338b..0fda67ebd6d0 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -26,6 +26,7 @@ COPTS_TESTS = select({ }) COPTS = COPTS_TESTS + select({ + "@platforms//os:windows": [""], "//conditions:default": ["-Wshadow"], }) From b2355ec50e9c46360866870e3b5a3961d19a5c4a Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Fri, 29 Aug 2025 00:03:33 -0700 Subject: [PATCH 362/634] [core] Fixing build failure due to LeaseID change (#56071) Signed-off-by: joshlee --- src/ray/gcs/gcs_server/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 4ff406b335e5..45b348dba6e8 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -397,7 +397,7 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:task_common", "//src/ray/protobuf:gcs_service_cc_proto", - "//src/ray/raylet/scheduling:cluster_task_manager", + "//src/ray/raylet/scheduling:cluster_lease_manager", "//src/ray/raylet_client:raylet_client_lib", "//src/ray/rpc:core_worker_client", "//src/ray/rpc:node_manager_client", From 24cc5fcb9d952a739f5738757dd11dad65aacaa2 Mon Sep 17 00:00:00 2001 From: Srinath Krishnamachari <68668616+srinathk10@users.noreply.github.com> Date: Fri, 29 Aug 2025 09:06:24 -0700 Subject: [PATCH 363/634] [Data] Fix flakey test_operators (#56028) --- python/ray/data/tests/test_operators.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/ray/data/tests/test_operators.py b/python/ray/data/tests/test_operators.py index 7f46d75d6dc8..92e5f0ab5741 100644 --- a/python/ray/data/tests/test_operators.py +++ b/python/ray/data/tests/test_operators.py @@ -139,7 +139,9 @@ def dummy_all_transform(bundles: List[RefBundle], ctx): # Check we return transformed bundles. assert not op.completed() - assert _take_outputs(op) == [[1, 2], [3, 4]] + outputs = _take_outputs(op) + expected = [[1, 2], [3, 4]] + assert sorted(outputs) == expected, f"Expected {expected}, got {outputs}" stats = op.get_stats() assert "FooStats" in stats assert op.completed() @@ -515,7 +517,9 @@ def test_map_operator_ray_args(shutdown_only, use_actors): run_op_tasks_sync(op) # Check we don't hang and complete with num_gpus=1. - assert _take_outputs(op) == [[i * 2] for i in range(10)] + outputs = _take_outputs(op) + expected = [[i * 2] for i in range(10)] + assert sorted(outputs) == expected, f"Expected {expected}, got {outputs}" assert op.completed() From 2ffcd3d10bb43aeecf2e5787b29c75c2a5eeb46d Mon Sep 17 00:00:00 2001 From: Potato Date: Sat, 30 Aug 2025 01:31:01 +0800 Subject: [PATCH 364/634] [DOC][Cluster] Fix typos and grammar issues in cluster documentation (#56065) Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Jiajun Yao --- doc/source/cluster/configure-manage-dashboard.md | 6 +++--- doc/source/cluster/faq.rst | 6 +++--- doc/source/cluster/kubernetes/examples.md | 2 +- .../examples/distributed-checkpointing-with-gcsfuse.md | 2 +- doc/source/cluster/kubernetes/troubleshooting.md | 2 +- .../cluster/kubernetes/troubleshooting/troubleshooting.md | 4 ++-- doc/source/cluster/package-overview.rst | 2 +- .../cluster/vms/user-guides/launching-clusters/azure.md | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/source/cluster/configure-manage-dashboard.md b/doc/source/cluster/configure-manage-dashboard.md index b14476ab5f3b..1e996e6b0e5a 100644 --- a/doc/source/cluster/configure-manage-dashboard.md +++ b/doc/source/cluster/configure-manage-dashboard.md @@ -2,7 +2,7 @@ # Configuring and Managing Ray Dashboard {ref}`Ray Dashboard` is one of the most important tools to monitor and debug Ray applications and Clusters. This page describes how to configure Ray Dashboard on your Clusters. -Dashboard configurations may differ depending on how you launch Ray Clusters (e.g., local Ray Cluster v.s. KubeRay). Integrations with Prometheus and Grafana are optional for enhanced Dashboard experience. +Dashboard configurations may differ depending on how you launch Ray Clusters (e.g., local Ray Cluster vs. KubeRay). Integrations with Prometheus and Grafana are optional for enhanced Dashboard experience. :::{note} Ray Dashboard is useful for interactive development and debugging because when clusters terminate, the dashboard UI and the underlying data are no longer accessible. For production monitoring and debugging, you should rely on [persisted logs](../cluster/kubernetes/user-guides/persist-kuberay-custom-resource-logs.md), [persisted metrics](./metrics.md), [persisted Ray states](../ray-observability/user-guides/cli-sdk.rst), and other observability tools. @@ -135,7 +135,7 @@ The Ray Dashboard provides read **and write** access to the Ray Cluster. The rev Dashboard is included if you use `ray[default]` or {ref}`other installation commands ` and automatically started. -To disable Dashboard, use the following arguments `--include-dashboard`. +To disable the Dashboard, use the `--include-dashboard` argument. ::::{tab-set} @@ -267,7 +267,7 @@ If you have followed the instructions above to set up everything, run the connec ##### Getting an error that says `RAY_GRAFANA_HOST` is not setup -If you have set up Grafana , check that: +If you have set up Grafana, check that: * You've included the protocol in the URL (e.g., `http://your-grafana-url.com` instead of `your-grafana-url.com`). * The URL doesn't have a trailing slash (e.g., `http://your-grafana-url.com` instead of `http://your-grafana-url.com/`). diff --git a/doc/source/cluster/faq.rst b/doc/source/cluster/faq.rst index 19be814cdd81..64f6526415b3 100644 --- a/doc/source/cluster/faq.rst +++ b/doc/source/cluster/faq.rst @@ -43,7 +43,7 @@ connect. Use this command: .. code:: bash - ray start --head --node-ip-address xx.xx.xx.xx --port nnnn`` + ray start --head --node-ip-address xx.xx.xx.xx --port nnnn Then when starting the worker node, use this command to connect to the head node: @@ -66,8 +66,8 @@ debugging routing issues. You may also see failures in the log like: - This node has an IP address of xx.xx.xx.xx, while we can not found the - matched Raylet address. This maybe come from when you connect the Ray + This node has an IP address of xx.xx.xx.xx, while we cannot find the + matched Raylet address. This may come from when you connect the Ray cluster with a different IP address or connect a container. The cause of this error may be the head node overloading with too many simultaneous diff --git a/doc/source/cluster/kubernetes/examples.md b/doc/source/cluster/kubernetes/examples.md index 4e683a784a1f..167a1362b221 100644 --- a/doc/source/cluster/kubernetes/examples.md +++ b/doc/source/cluster/kubernetes/examples.md @@ -31,7 +31,7 @@ This section presents example Ray workloads to try out on your Kubernetes cluste - {ref}`kuberay-batch-inference-example` - {ref}`kuberay-kueue-priority-scheduling-example` - {ref}`kuberay-kueue-gang-scheduling-example` -- {ref}`kuberay-distributed-checkpointing-gcsefuse` +- {ref}`kuberay-distributed-checkpointing-gcsfuse` - {ref}`kuberay-modin-example` - {ref}`kuberay-rayservice-llm-example` - {ref}`kuberay-rayservice-deepseek-example` diff --git a/doc/source/cluster/kubernetes/examples/distributed-checkpointing-with-gcsfuse.md b/doc/source/cluster/kubernetes/examples/distributed-checkpointing-with-gcsfuse.md index fceabc555b8b..19fe6572df6a 100644 --- a/doc/source/cluster/kubernetes/examples/distributed-checkpointing-with-gcsfuse.md +++ b/doc/source/cluster/kubernetes/examples/distributed-checkpointing-with-gcsfuse.md @@ -1,4 +1,4 @@ -(kuberay-distributed-checkpointing-gcsefuse)= +(kuberay-distributed-checkpointing-gcsfuse)= # Distributed checkpointing with KubeRay and GCSFuse diff --git a/doc/source/cluster/kubernetes/troubleshooting.md b/doc/source/cluster/kubernetes/troubleshooting.md index 5bf2257b44f5..b3525993ee88 100644 --- a/doc/source/cluster/kubernetes/troubleshooting.md +++ b/doc/source/cluster/kubernetes/troubleshooting.md @@ -9,5 +9,5 @@ troubleshooting/troubleshooting troubleshooting/rayservice-troubleshooting ``` -- {ref}`kuberay-troubleshootin-guides` +- {ref}`kuberay-troubleshooting-guides` - {ref}`kuberay-raysvc-troubleshoot` diff --git a/doc/source/cluster/kubernetes/troubleshooting/troubleshooting.md b/doc/source/cluster/kubernetes/troubleshooting/troubleshooting.md index f381efe16567..7ec4279f6f81 100644 --- a/doc/source/cluster/kubernetes/troubleshooting/troubleshooting.md +++ b/doc/source/cluster/kubernetes/troubleshooting/troubleshooting.md @@ -1,4 +1,4 @@ -(kuberay-troubleshootin-guides)= +(kuberay-troubleshooting-guides)= # Troubleshooting guide @@ -29,7 +29,7 @@ When a Ray job is created, the Ray dashboard agent process on the head node gets (docker-image-for-apple-macbooks)= ## Use ARM-based docker images for Apple M1 or M2 MacBooks -Ray builds different images for different platforms. Until Ray moves to building multi-architecture images, [tracked by this Github issue](https://github.com/ray-project/ray/issues/39364), use platform-specific docker images in the head and worker group specs of the [RayCluster config](https://docs.ray.io/en/latest/cluster/kubernetes/user-guides/config.html#image). +Ray builds different images for different platforms. Until Ray moves to building multi-architecture images, [tracked by this GitHub issue](https://github.com/ray-project/ray/issues/39364), use platform-specific docker images in the head and worker group specs of the [RayCluster config](https://docs.ray.io/en/latest/cluster/kubernetes/user-guides/config.html#image). Use an image with the tag `aarch64`, for example, `image: rayproject/ray:2.41.0-aarch64`), if you are running KubeRay on a MacBook M1 or M2. diff --git a/doc/source/cluster/package-overview.rst b/doc/source/cluster/package-overview.rst index 1aef16167d64..b79b5488d242 100644 --- a/doc/source/cluster/package-overview.rst +++ b/doc/source/cluster/package-overview.rst @@ -4,7 +4,7 @@ Ray Cluster Management API ========================== This section contains a reference for the cluster management API. If there is anything missing, please open an issue -on `Github`_. +on `GitHub`_. .. _`GitHub`: https://github.com/ray-project/ray/issues diff --git a/doc/source/cluster/vms/user-guides/launching-clusters/azure.md b/doc/source/cluster/vms/user-guides/launching-clusters/azure.md index 638b66dfb740..37242aeb3bd7 100644 --- a/doc/source/cluster/vms/user-guides/launching-clusters/azure.md +++ b/doc/source/cluster/vms/user-guides/launching-clusters/azure.md @@ -8,7 +8,7 @@ There are two ways to start an Azure Ray cluster. - Deploy a cluster using Azure portal. ```{note} -The Azure integration is community-maintained. Please reach out to the integration maintainers on Github if +The Azure integration is community-maintained. Please reach out to the integration maintainers on GitHub if you run into any problems: gramhagen, eisber, ijrsvt. ``` From fc44f8747c174b39ce623d5dab53c2456bb0ae30 Mon Sep 17 00:00:00 2001 From: Ping Date: Sat, 30 Aug 2025 04:29:52 +0800 Subject: [PATCH 365/634] [Core] store clients to return void instead of Status::OK (#55663) Signed-off-by: 400Ping Signed-off-by: Ping Signed-off-by: Dhyey Shah Co-authored-by: Dhyey Shah --- .../gcs/store_client/in_memory_store_client.h | 20 +- .../ray/gcs/store_client/redis_store_client.h | 22 +- src/mock/ray/gcs/store_client/store_client.h | 20 +- src/ray/gcs/gcs_server/gcs_actor_manager.cc | 49 ++- src/ray/gcs/gcs_server/gcs_actor_scheduler.cc | 23 +- src/ray/gcs/gcs_server/gcs_init_data.cc | 39 +-- src/ray/gcs/gcs_server/gcs_job_manager.cc | 25 +- src/ray/gcs/gcs_server/gcs_node_manager.cc | 22 +- .../gcs_server/gcs_placement_group_manager.cc | 40 +-- .../gcs_placement_group_scheduler.cc | 6 +- src/ray/gcs/gcs_server/gcs_table_storage.cc | 83 +++-- src/ray/gcs/gcs_server/gcs_table_storage.h | 51 ++- src/ray/gcs/gcs_server/gcs_worker_manager.cc | 47 +-- src/ray/gcs/gcs_server/store_client_kv.cc | 53 ++- .../tests/gcs_actor_scheduler_mock_test.cc | 4 +- .../tests/gcs_actor_scheduler_test.cc | 10 +- .../gcs_placement_group_manager_mock_test.cc | 6 +- .../tests/gcs_placement_group_manager_test.cc | 13 +- .../tests/gcs_table_storage_test_base.h | 12 +- .../store_client/in_memory_store_client.cc | 48 +-- .../gcs/store_client/in_memory_store_client.h | 46 +-- .../store_client/observable_store_client.cc | 106 +++--- .../store_client/observable_store_client.h | 46 +-- .../gcs/store_client/redis_store_client.cc | 75 ++-- src/ray/gcs/store_client/redis_store_client.h | 52 +-- src/ray/gcs/store_client/store_client.h | 54 ++- .../tests/redis_store_client_test.cc | 328 ++++++++---------- .../tests/store_client_test_base.h | 36 +- 28 files changed, 606 insertions(+), 730 deletions(-) diff --git a/src/mock/ray/gcs/store_client/in_memory_store_client.h b/src/mock/ray/gcs/store_client/in_memory_store_client.h index 51bebc607e02..16a7a5cab895 100644 --- a/src/mock/ray/gcs/store_client/in_memory_store_client.h +++ b/src/mock/ray/gcs/store_client/in_memory_store_client.h @@ -17,64 +17,64 @@ namespace gcs { class MockInMemoryStoreClient : public InMemoryStoreClient { public: - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncPut, (const std::string &table_name, const std::string &key, - const std::string &data, + std::string data, bool overwrite, Postable callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncGet, (const std::string &table_name, const std::string &key, ToPostable> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncGetAll, (const std::string &table_name, Postable)> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncMultiGet, (const std::string &table_name, const std::vector &keys, Postable)> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncDelete, (const std::string &table_name, const std::string &key, Postable callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncBatchDelete, (const std::string &table_name, const std::vector &keys, Postable callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncGetKeys, (const std::string &table_name, const std::string &prefix, Postable)> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncExists, (const std::string &table_name, const std::string &key, Postable callback), (override)); - MOCK_METHOD(Status, AsyncGetNextJobID, (Postable callback), (override)); + MOCK_METHOD(void, AsyncGetNextJobID, (Postable callback), (override)); }; } // namespace gcs diff --git a/src/mock/ray/gcs/store_client/redis_store_client.h b/src/mock/ray/gcs/store_client/redis_store_client.h index a0fc20272f9c..7a73e5b045dd 100644 --- a/src/mock/ray/gcs/store_client/redis_store_client.h +++ b/src/mock/ray/gcs/store_client/redis_store_client.h @@ -17,52 +17,52 @@ namespace gcs { class MockStoreClient : public StoreClient { public: - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncPut, (const std::string &table_name, const std::string &key, - const std::string &data, + std::string data, bool overwrite, Postable callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncGet, (const std::string &table_name, const std::string &key, ToPostable> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncGetAll, (const std::string &table_name, Postable)> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncMultiGet, (const std::string &table_name, - const std::vector &key, + const std::vector &keys, Postable)> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncDelete, (const std::string &table_name, const std::string &key, Postable callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncBatchDelete, (const std::string &table_name, const std::vector &keys, Postable callback), (override)); - MOCK_METHOD(Status, AsyncGetNextJobID, (Postable callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncGetNextJobID, (Postable callback), (override)); + MOCK_METHOD(void, AsyncGetKeys, (const std::string &table_name, const std::string &prefix, Postable)> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncExists, (const std::string &table_name, const std::string &key, diff --git a/src/mock/ray/gcs/store_client/store_client.h b/src/mock/ray/gcs/store_client/store_client.h index 9094588f5e37..7a73e5b045dd 100644 --- a/src/mock/ray/gcs/store_client/store_client.h +++ b/src/mock/ray/gcs/store_client/store_client.h @@ -17,7 +17,7 @@ namespace gcs { class MockStoreClient : public StoreClient { public: - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncPut, (const std::string &table_name, const std::string &key, @@ -25,44 +25,44 @@ class MockStoreClient : public StoreClient { bool overwrite, Postable callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncGet, (const std::string &table_name, const std::string &key, ToPostable> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncGetAll, (const std::string &table_name, Postable)> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncMultiGet, (const std::string &table_name, - const std::vector &key, + const std::vector &keys, Postable)> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncDelete, (const std::string &table_name, const std::string &key, Postable callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncBatchDelete, (const std::string &table_name, const std::vector &keys, Postable callback), (override)); - MOCK_METHOD(Status, AsyncGetNextJobID, (Postable callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncGetNextJobID, (Postable callback), (override)); + MOCK_METHOD(void, AsyncGetKeys, (const std::string &table_name, const std::string &prefix, Postable)> callback), (override)); - MOCK_METHOD(Status, + MOCK_METHOD(void, AsyncExists, (const std::string &table_name, const std::string &key, diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index e2b9b5cbf064..962ab4b8be02 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -529,7 +529,7 @@ void GcsActorManager::HandleGetAllActorInfo(rpc::GetAllActorInfoRequest request, RAY_CHECK(request.show_dead_jobs()); // We don't maintain an in-memory cache of all actors which belong to dead // jobs, so fetch it from redis. - Status status = gcs_table_storage_->ActorTable().GetAll( + gcs_table_storage_->ActorTable().GetAll( {[reply, send_reply_callback, limit, request = std::move(request), filter_fn]( absl::flat_hash_map &&result) { auto total_actors = result.size(); @@ -559,10 +559,6 @@ void GcsActorManager::HandleGetAllActorInfo(rpc::GetAllActorInfoRequest request, RAY_LOG(DEBUG) << "Finished getting all actor info."; }, io_context_}); - if (!status.ok()) { - // Send the response to unblock the sender and free the request. - GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); - } } void GcsActorManager::HandleGetNamedActorInfo( @@ -722,11 +718,11 @@ Status GcsActorManager::RegisterActor(const ray::rpc::RegisterActorRequest &requ } // The backend storage is supposed to be reliable, so the status must be ok. - RAY_CHECK_OK(gcs_table_storage_->ActorTaskSpecTable().Put( + gcs_table_storage_->ActorTaskSpecTable().Put( actor_id, request.task_spec(), {[this, actor](Status status) { - RAY_CHECK_OK(gcs_table_storage_->ActorTable().Put( + gcs_table_storage_->ActorTable().Put( actor->GetActorID(), *actor->GetMutableActorTableData(), {[this, actor](Status put_status) { @@ -765,9 +761,9 @@ Status GcsActorManager::RegisterActor(const ray::rpc::RegisterActorRequest &requ } actor_to_register_callbacks_.erase(callback_iter); }, - io_context_})); + io_context_}); }, - io_context_})); + io_context_}); return Status::OK(); } @@ -1044,7 +1040,7 @@ void GcsActorManager::DestroyActor(const ActorID &actor_id, auto actor_table_data = std::make_shared(*mutable_actor_table_data); // The backend storage is reliable in the future, so the status must be ok. - RAY_CHECK_OK(gcs_table_storage_->ActorTable().Put( + gcs_table_storage_->ActorTable().Put( actor->GetActorID(), *actor_table_data, {[this, @@ -1059,14 +1055,14 @@ void GcsActorManager::DestroyActor(const ActorID &actor_id, gcs_publisher_->PublishActor(actor_id, GenActorDataOnlyWithStates(*actor_table_data)); if (!is_restartable) { - RAY_CHECK_OK(gcs_table_storage_->ActorTaskSpecTable().Delete( - actor_id, {[](auto) {}, io_context_})); + gcs_table_storage_->ActorTaskSpecTable().Delete(actor_id, + {[](auto) {}, io_context_}); } actor->WriteActorExportEvent(); // Destroy placement group owned by this actor. destroy_owned_placement_group_if_needed_(actor_id); }, - io_context_})); + io_context_}); // Inform all creation callbacks that the actor was cancelled, not created. RunAndClearActorCreationCallbacks( @@ -1288,14 +1284,14 @@ void GcsActorManager::SetPreemptedAndPublish(const NodeID &node_id) { const auto &actor_id = id_iter.second; const auto &actor_table_data = actor_iter->second->GetActorTableData(); - RAY_CHECK_OK(gcs_table_storage_->ActorTable().Put( + gcs_table_storage_->ActorTable().Put( actor_id, actor_table_data, {[this, actor_id, actor_table_data](Status status) { gcs_publisher_->PublishActor(actor_id, GenActorDataOnlyWithStates(actor_table_data)); }, - io_context_})); + io_context_}); } } @@ -1364,7 +1360,7 @@ void GcsActorManager::RestartActor(const ActorID &actor_id, actor->UpdateAddress(rpc::Address()); mutable_actor_table_data->clear_resource_mapping(); // The backend storage is reliable in the future, so the status must be ok. - RAY_CHECK_OK(gcs_table_storage_->ActorTable().Put( + gcs_table_storage_->ActorTable().Put( actor_id, *mutable_actor_table_data, {[this, actor, actor_id, mutable_actor_table_data, done_callback](Status status) { @@ -1375,7 +1371,7 @@ void GcsActorManager::RestartActor(const ActorID &actor_id, actor_id, GenActorDataOnlyWithStates(*mutable_actor_table_data)); actor->WriteActorExportEvent(); }, - io_context_})); + io_context_}); gcs_actor_scheduler_->Schedule(actor); } else { actor->UpdateState(rpc::ActorTableData::DEAD); @@ -1385,7 +1381,7 @@ void GcsActorManager::RestartActor(const ActorID &actor_id, mutable_actor_table_data->set_timestamp(time); // The backend storage is reliable in the future, so the status must be ok. - RAY_CHECK_OK(gcs_table_storage_->ActorTable().Put( + gcs_table_storage_->ActorTable().Put( actor_id, *mutable_actor_table_data, {[this, actor, actor_id, mutable_actor_table_data, death_cause, done_callback]( @@ -1401,11 +1397,11 @@ void GcsActorManager::RestartActor(const ActorID &actor_id, } gcs_publisher_->PublishActor( actor_id, GenActorDataOnlyWithStates(*mutable_actor_table_data)); - RAY_CHECK_OK(gcs_table_storage_->ActorTaskSpecTable().Delete( - actor_id, {[](auto) {}, io_context_})); + gcs_table_storage_->ActorTaskSpecTable().Delete(actor_id, + {[](auto) {}, io_context_}); actor->WriteActorExportEvent(); }, - io_context_})); + io_context_}); // The actor is dead, but we should not remove the entry from the // registered actors yet. If the actor is owned, we will destroy the actor // once the owner fails or notifies us that the actor has no references. @@ -1508,7 +1504,7 @@ void GcsActorManager::OnActorCreationSuccess(const std::shared_ptr &ac auto actor_data_only_with_states = GenActorDataOnlyWithStates(*mutable_actor_table_data); // The backend storage is reliable in the future, so the status must be ok. - RAY_CHECK_OK(gcs_table_storage_->ActorTable().Put( + gcs_table_storage_->ActorTable().Put( actor_id, *mutable_actor_table_data, {[this, @@ -1523,7 +1519,7 @@ void GcsActorManager::OnActorCreationSuccess(const std::shared_ptr &ac // actor_to_create_callbacks_. RunAndClearActorCreationCallbacks(actor, reply, Status::OK()); }, - io_context_})); + io_context_}); } void GcsActorManager::SchedulePendingActors() { @@ -1589,8 +1585,8 @@ void GcsActorManager::Initialize(const GcsInitData &gcs_init_data) { } } if (!dead_actors.empty()) { - RAY_CHECK_OK(gcs_table_storage_->ActorTaskSpecTable().BatchDelete( - dead_actors, {[](auto) {}, io_context_})); + gcs_table_storage_->ActorTaskSpecTable().BatchDelete(dead_actors, + {[](auto) {}, io_context_}); } sorted_destroyed_actor_list_.sort([](const std::pair &left, const std::pair &right) { @@ -1730,8 +1726,7 @@ void GcsActorManager::AddDestroyedActorToCache(const std::shared_ptr & if (destroyed_actors_.size() >= RayConfig::instance().maximum_gcs_destroyed_actor_cached_count()) { const auto &actor_id = sorted_destroyed_actor_list_.front().first; - RAY_CHECK_OK( - gcs_table_storage_->ActorTable().Delete(actor_id, {[](auto) {}, io_context_})); + gcs_table_storage_->ActorTable().Delete(actor_id, {[](auto) {}, io_context_}); destroyed_actors_.erase(actor_id); sorted_destroyed_actor_list_.pop_front(); } diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc index c8e2aff769dc..a607b94f237f 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc @@ -416,18 +416,17 @@ void GcsActorScheduler::HandleWorkerLeaseGrantedReply( // Without this, there could be a possible race condition. Related issues: // https://github.com/ray-project/ray/pull/9215/files#r449469320 worker_client_pool_.GetOrConnect(leased_worker->GetAddress()); - RAY_CHECK_OK(gcs_actor_table_.Put(actor->GetActorID(), - actor->GetActorTableData(), - {[this, actor, leased_worker](Status status) { - RAY_CHECK_OK(status); - if (actor->GetState() == - rpc::ActorTableData::DEAD) { - // Actor has already been killed. - return; - } - CreateActorOnWorker(actor, leased_worker); - }, - io_context_})); + gcs_actor_table_.Put(actor->GetActorID(), + actor->GetActorTableData(), + {[this, actor, leased_worker](Status status) { + RAY_CHECK_OK(status); + if (actor->GetState() == rpc::ActorTableData::DEAD) { + // Actor has already been killed. + return; + } + CreateActorOnWorker(actor, leased_worker); + }, + io_context_}); } } diff --git a/src/ray/gcs/gcs_server/gcs_init_data.cc b/src/ray/gcs/gcs_server/gcs_init_data.cc index 1e33538bf521..1c7ed5389813 100644 --- a/src/ray/gcs/gcs_server/gcs_init_data.cc +++ b/src/ray/gcs/gcs_server/gcs_init_data.cc @@ -44,56 +44,53 @@ void GcsInitData::AsyncLoad(Postable on_done) { void GcsInitData::AsyncLoadJobTableData(Postable on_done) { RAY_LOG(INFO) << "Loading job table data."; - RAY_CHECK_OK(gcs_table_storage_.JobTable().GetAll(std::move(on_done).TransformArg( + gcs_table_storage_.JobTable().GetAll(std::move(on_done).TransformArg( [this](absl::flat_hash_map result) { job_table_data_ = std::move(result); RAY_LOG(INFO) << "Finished loading job table data, size = " << job_table_data_.size(); - }))); + })); } void GcsInitData::AsyncLoadNodeTableData(Postable on_done) { RAY_LOG(INFO) << "Loading node table data."; - RAY_CHECK_OK(gcs_table_storage_.NodeTable().GetAll(std::move(on_done).TransformArg( + gcs_table_storage_.NodeTable().GetAll(std::move(on_done).TransformArg( [this](absl::flat_hash_map result) { node_table_data_ = std::move(result); RAY_LOG(INFO) << "Finished loading node table data, size = " << node_table_data_.size(); - }))); + })); } void GcsInitData::AsyncLoadPlacementGroupTableData(Postable on_done) { RAY_LOG(INFO) << "Loading placement group table data."; - RAY_CHECK_OK( - gcs_table_storage_.PlacementGroupTable().GetAll(std::move(on_done).TransformArg( - [this](absl::flat_hash_map - result) { - placement_group_table_data_ = std::move(result); - RAY_LOG(INFO) << "Finished loading placement group table data, size = " - << placement_group_table_data_.size(); - }))); + gcs_table_storage_.PlacementGroupTable().GetAll(std::move(on_done).TransformArg( + [this](absl::flat_hash_map result) { + placement_group_table_data_ = std::move(result); + RAY_LOG(INFO) << "Finished loading placement group table data, size = " + << placement_group_table_data_.size(); + })); } void GcsInitData::AsyncLoadActorTableData(Postable on_done) { RAY_LOG(INFO) << "Loading actor table data."; - RAY_CHECK_OK(gcs_table_storage_.ActorTable().AsyncRebuildIndexAndGetAll( + gcs_table_storage_.ActorTable().AsyncRebuildIndexAndGetAll( std::move(on_done).TransformArg( [this](absl::flat_hash_map result) { actor_table_data_ = std::move(result); RAY_LOG(INFO) << "Finished loading actor table data, size = " << actor_table_data_.size(); - }))); + })); } void GcsInitData::AsyncLoadActorTaskSpecTableData(Postable on_done) { RAY_LOG(INFO) << "Loading actor task spec table data."; - RAY_CHECK_OK( - gcs_table_storage_.ActorTaskSpecTable().GetAll(std::move(on_done).TransformArg( - [this](absl::flat_hash_map result) -> void { - actor_task_spec_table_data_ = std::move(result); - RAY_LOG(INFO) << "Finished loading actor task spec table data, size = " - << actor_task_spec_table_data_.size(); - }))); + gcs_table_storage_.ActorTaskSpecTable().GetAll(std::move(on_done).TransformArg( + [this](absl::flat_hash_map result) -> void { + actor_task_spec_table_data_ = std::move(result); + RAY_LOG(INFO) << "Finished loading actor task spec table data, size = " + << actor_task_spec_table_data_.size(); + })); } } // namespace gcs diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.cc b/src/ray/gcs/gcs_server/gcs_job_manager.cc index ec0e251d5b20..16115389726e 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_job_manager.cc @@ -130,8 +130,8 @@ void GcsJobManager::HandleAddJob(rpc::AddJobRequest request, GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); }; - RAY_UNUSED(gcs_table_storage_.JobTable().Put( - job_id, mutable_job_table_data, {std::move(on_done), io_context_})); + gcs_table_storage_.JobTable().Put( + job_id, mutable_job_table_data, {std::move(on_done), io_context_}); } void GcsJobManager::MarkJobAsFinished(rpc::JobTableData job_table_data, @@ -173,11 +173,8 @@ void GcsJobManager::MarkJobAsFinished(rpc::JobTableData job_table_data, done_callback(status); }; - Status status = - gcs_table_storage_.JobTable().Put(job_id, job_table_data, {on_done, io_context_}); - if (!status.ok()) { - on_done(status); - } + gcs_table_storage_.JobTable().Put( + job_id, job_table_data, {std::move(on_done), io_context_}); } void GcsJobManager::HandleMarkJobFinished(rpc::MarkJobFinishedRequest request, @@ -190,7 +187,7 @@ void GcsJobManager::HandleMarkJobFinished(rpc::MarkJobFinishedRequest request, GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); }; - Status status = gcs_table_storage_.JobTable().Get( + gcs_table_storage_.JobTable().Get( job_id, {[this, job_id, send_reply](Status get_status, std::optional result) { @@ -211,9 +208,6 @@ void GcsJobManager::HandleMarkJobFinished(rpc::MarkJobFinishedRequest request, send_reply(get_status); }, io_context_}); - if (!status.ok()) { - send_reply(status); - } } void GcsJobManager::ClearJobInfos(const rpc::JobTableData &job_data) { @@ -445,10 +439,7 @@ void GcsJobManager::HandleGetAllJobInfo(rpc::GetAllJobInfoRequest request, "job", job_api_data_keys, {kv_multi_get_callback, io_context_}); } }; - Status status = gcs_table_storage_.JobTable().GetAll({on_done, io_context_}); - if (!status.ok()) { - on_done(absl::flat_hash_map()); - } + gcs_table_storage_.JobTable().GetAll({std::move(on_done), io_context_}); } void GcsJobManager::HandleReportJobError(rpc::ReportJobErrorRequest request, @@ -467,7 +458,7 @@ void GcsJobManager::HandleGetNextJobID(rpc::GetNextJobIDRequest request, reply->set_job_id(job_id); GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); }; - RAY_CHECK_OK(gcs_table_storage_.AsyncGetNextJobID({std::move(callback), io_context_})); + gcs_table_storage_.AsyncGetNextJobID({std::move(callback), io_context_}); } std::shared_ptr GcsJobManager::GetJobConfig(const JobID &job_id) const { @@ -499,7 +490,7 @@ void GcsJobManager::OnNodeDead(const NodeID &node_id) { } }; - RAY_CHECK_OK(gcs_table_storage_.JobTable().GetAll({on_done, io_context_})); + gcs_table_storage_.JobTable().GetAll({std::move(on_done), io_context_}); } void GcsJobManager::RecordMetrics() { diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.cc b/src/ray/gcs/gcs_server/gcs_node_manager.cc index 8a6f6ffbb00c..dbdc5aee78a9 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_node_manager.cc @@ -115,16 +115,16 @@ void GcsNodeManager::HandleRegisterNode(rpc::RegisterNodeRequest request, if (head_nodes.size() == 1) { OnNodeFailure(head_nodes[0], [this, node_id, node_info, on_done = std::move(on_done)]() { - RAY_CHECK_OK(gcs_table_storage_->NodeTable().Put( - node_id, node_info, {on_done, io_context_})); + gcs_table_storage_->NodeTable().Put( + node_id, node_info, {on_done, io_context_}); }); } else { - RAY_CHECK_OK(gcs_table_storage_->NodeTable().Put( - node_id, node_info, {std::move(on_done), io_context_})); + gcs_table_storage_->NodeTable().Put( + node_id, node_info, {std::move(on_done), io_context_}); } } else { - RAY_CHECK_OK(gcs_table_storage_->NodeTable().Put( - node_id, node_info, {std::move(on_done), io_context_})); + gcs_table_storage_->NodeTable().Put( + node_id, node_info, {std::move(on_done), io_context_}); } ++counts_[CountType::REGISTER_NODE_REQUEST]; } @@ -168,8 +168,7 @@ void GcsNodeManager::HandleUnregisterNode(rpc::UnregisterNodeRequest request, gcs_publisher_->PublishNodeInfo(node_id, *node_info_delta); WriteNodeExportEvent(*node); }; - RAY_CHECK_OK( - gcs_table_storage_->NodeTable().Put(node_id, *node, {on_put_done, io_context_})); + gcs_table_storage_->NodeTable().Put(node_id, *node, {on_put_done, io_context_}); GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); } @@ -469,8 +468,8 @@ void GcsNodeManager::OnNodeFailure( } gcs_publisher_->PublishNodeInfo(node_id, std::move(node_info_delta)); }; - RAY_CHECK_OK(gcs_table_storage_->NodeTable().Put( - node_id, *node, {std::move(on_done), io_context_})); + gcs_table_storage_->NodeTable().Put( + node_id, *node, {std::move(on_done), io_context_}); } else if (node_table_updated_callback != nullptr) { node_table_updated_callback(); } @@ -506,8 +505,7 @@ void GcsNodeManager::Initialize(const GcsInitData &gcs_init_data) { void GcsNodeManager::AddDeadNodeToCache(std::shared_ptr node) { if (dead_nodes_.size() >= RayConfig::instance().maximum_gcs_dead_node_cached_count()) { const auto &node_id = sorted_dead_node_list_.front().first; - RAY_CHECK_OK(gcs_table_storage_->NodeTable().Delete( - node_id, {[](const auto &) {}, io_context_})); + gcs_table_storage_->NodeTable().Delete(node_id, {[](const auto &) {}, io_context_}); dead_nodes_.erase(sorted_dead_node_list_.front().first); sorted_dead_node_list_.pop_front(); } diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc b/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc index c213394b923d..19a7fb93d588 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc @@ -133,7 +133,7 @@ void GcsPlacementGroupManager::RegisterPlacementGroup( placement_group); AddToPendingQueue(placement_group); - RAY_CHECK_OK(gcs_table_storage_->PlacementGroupTable().Put( + gcs_table_storage_->PlacementGroupTable().Put( placement_group_id, placement_group->GetPlacementGroupTableData(), {[this, placement_group_id, placement_group](Status status) { @@ -163,7 +163,7 @@ void GcsPlacementGroupManager::RegisterPlacementGroup( return; } }, - io_context_})); + io_context_}); } PlacementGroupID GcsPlacementGroupManager::GetPlacementGroupIDByName( @@ -245,7 +245,7 @@ void GcsPlacementGroupManager::OnPlacementGroupCreationSuccess( // Update states and persists the information. placement_group->UpdateState(rpc::PlacementGroupTableData::CREATED); auto placement_group_id = placement_group->GetPlacementGroupID(); - RAY_CHECK_OK(gcs_table_storage_->PlacementGroupTable().Put( + gcs_table_storage_->PlacementGroupTable().Put( placement_group_id, placement_group->GetPlacementGroupTableData(), {[this, placement_group_id](Status status) { @@ -269,7 +269,7 @@ void GcsPlacementGroupManager::OnPlacementGroupCreationSuccess( placement_group_to_create_callbacks_.erase(pg_to_create_iter); } }, - io_context_})); + io_context_}); lifetime_num_placement_groups_created_++; io_context_.post([this] { SchedulePendingPlacementGroups(); }, "GcsPlacementGroupManager.SchedulePendingPlacementGroups"); @@ -434,7 +434,7 @@ void GcsPlacementGroupManager::RemovePlacementGroup( placement_group->UpdateState(rpc::PlacementGroupTableData::REMOVED); placement_group->GetMutableStats()->set_scheduling_state( rpc::PlacementGroupStats::REMOVED); - RAY_CHECK_OK(gcs_table_storage_->PlacementGroupTable().Put( + gcs_table_storage_->PlacementGroupTable().Put( placement_group->GetPlacementGroupID(), placement_group->GetPlacementGroupTableData(), {[this, on_placement_group_removed, placement_group_id](Status status) { @@ -451,7 +451,7 @@ void GcsPlacementGroupManager::RemovePlacementGroup( } on_placement_group_removed(status); }, - io_context_})); + io_context_}); } void GcsPlacementGroupManager::HandleGetPlacementGroup( @@ -478,11 +478,8 @@ void GcsPlacementGroupManager::HandleGetPlacementGroup( if (it != registered_placement_groups_.end()) { on_done(Status::OK(), it->second->GetPlacementGroupTableData()); } else { - Status status = gcs_table_storage_->PlacementGroupTable().Get( - placement_group_id, {std::move(on_done), io_context_}); - if (!status.ok()) { - on_done(status, std::nullopt); - } + gcs_table_storage_->PlacementGroupTable().Get(placement_group_id, + {std::move(on_done), io_context_}); } ++counts_[CountType::GET_PLACEMENT_GROUP_REQUEST]; } @@ -549,11 +546,7 @@ void GcsPlacementGroupManager::HandleGetAllPlacementGroup( RAY_LOG(DEBUG) << "Finished getting all placement group info."; GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); }; - Status status = - gcs_table_storage_->PlacementGroupTable().GetAll({std::move(on_done), io_context_}); - if (!status.ok()) { - on_done(absl::flat_hash_map()); - } + gcs_table_storage_->PlacementGroupTable().GetAll({std::move(on_done), io_context_}); ++counts_[CountType::GET_ALL_PLACEMENT_GROUP_REQUEST]; } @@ -614,11 +607,8 @@ void GcsPlacementGroupManager::WaitPlacementGroup( } }; - Status status = gcs_table_storage_->PlacementGroupTable().Get( - placement_group_id, {std::move(on_done), io_context_}); - if (!status.ok()) { - on_done(status, std::nullopt); - } + gcs_table_storage_->PlacementGroupTable().Get(placement_group_id, + {std::move(on_done), io_context_}); } else if (iter->second->GetState() == rpc::PlacementGroupTableData::CREATED) { RAY_LOG(DEBUG) << "Placement group is created, placement group id = " << placement_group_id; @@ -710,10 +700,10 @@ void GcsPlacementGroupManager::OnNodeDead(const NodeID &node_id) { iter->second->GetMutableStats()->set_scheduling_state( rpc::PlacementGroupStats::QUEUED); AddToPendingQueue(iter->second, 0); - RAY_CHECK_OK(gcs_table_storage_->PlacementGroupTable().Put( + gcs_table_storage_->PlacementGroupTable().Put( iter->second->GetPlacementGroupID(), iter->second->GetPlacementGroupTableData(), - {[this](Status status) { SchedulePendingPlacementGroups(); }, io_context_})); + {[this](Status status) { SchedulePendingPlacementGroups(); }, io_context_}); } } } @@ -1020,10 +1010,10 @@ bool GcsPlacementGroupManager::RescheduleIfStillHasUnplacedBundles( << placement_group->GetPlacementGroupID(); placement_group->UpdateState(rpc::PlacementGroupTableData::RESCHEDULING); AddToPendingQueue(placement_group, 0); - RAY_CHECK_OK(gcs_table_storage_->PlacementGroupTable().Put( + gcs_table_storage_->PlacementGroupTable().Put( placement_group->GetPlacementGroupID(), placement_group->GetPlacementGroupTableData(), - {[this](Status status) { SchedulePendingPlacementGroups(); }, io_context_})); + {[this](Status status) { SchedulePendingPlacementGroups(); }, io_context_}); return true; } } diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc index 855139ee8e15..68e29aadcc3a 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc @@ -398,16 +398,16 @@ void GcsPlacementGroupScheduler::OnAllBundlePrepareRequestReturned( placement_group->UpdateState(rpc::PlacementGroupTableData::PREPARED); - RAY_CHECK_OK(gcs_table_storage_.PlacementGroupTable().Put( + gcs_table_storage_.PlacementGroupTable().Put( placement_group_id, placement_group->GetPlacementGroupTableData(), {[this, lease_status_tracker, schedule_failure_handler, schedule_success_handler]( - Status status) { + const ray::Status &status) { RAY_CHECK_OK(status); CommitAllBundles( lease_status_tracker, schedule_failure_handler, schedule_success_handler); }, - io_context_})); + io_context_}); } void GcsPlacementGroupScheduler::OnAllBundleCommitRequestReturned( diff --git a/src/ray/gcs/gcs_server/gcs_table_storage.cc b/src/ray/gcs/gcs_server/gcs_table_storage.cc index c3bd5d226ae7..83847884a199 100644 --- a/src/ray/gcs/gcs_server/gcs_table_storage.cc +++ b/src/ray/gcs/gcs_server/gcs_table_storage.cc @@ -37,24 +37,23 @@ Postable JustOk(Postable callback) { } // namespace template -Status GcsTable::Put(const Key &key, - const Data &value, - Postable callback) { - return store_client_->AsyncPut(table_name_, - key.Binary(), - value.SerializeAsString(), - /*overwrite*/ true, - JustOk(std::move(callback))); +void GcsTable::Put(const Key &key, + const Data &value, + Postable callback) { + store_client_->AsyncPut(table_name_, + key.Binary(), + value.SerializeAsString(), + /*overwrite*/ true, + JustOk(std::move(callback))); } template -Status GcsTable::Get(const Key &key, - Postable)> callback) { +void GcsTable::Get(const Key &key, + Postable)> callback) { // We can't use TransformArg here because we need to return 2 arguments. - return store_client_->AsyncGet( - table_name_, key.Binary(), std::move(callback).Rebind([](auto async_get_callback) { - return [cb = std::move(async_get_callback)](Status status, - std::optional result) { + store_client_->AsyncGet( + table_name_, key.Binary(), std::move(callback).Rebind([](auto cb) { + return [cb = std::move(cb)](Status status, std::optional result) { std::optional value; if (result) { Data data; @@ -67,9 +66,9 @@ Status GcsTable::Get(const Key &key, } template -Status GcsTable::GetAll( +void GcsTable::GetAll( Postable)> callback) { - return store_client_->AsyncGetAll( + store_client_->AsyncGetAll( table_name_, std::move(callback).TransformArg( [](absl::flat_hash_map result) { @@ -85,40 +84,40 @@ Status GcsTable::GetAll( } template -Status GcsTable::Delete(const Key &key, Postable callback) { - return store_client_->AsyncDelete( +void GcsTable::Delete(const Key &key, Postable callback) { + store_client_->AsyncDelete( table_name_, key.Binary(), JustOk(std::move(callback))); } template -Status GcsTable::BatchDelete(const std::vector &keys, - Postable callback) { +void GcsTable::BatchDelete(const std::vector &keys, + Postable callback) { std::vector keys_to_delete; keys_to_delete.reserve(keys.size()); for (auto &key : keys) { keys_to_delete.emplace_back(std::move(key.Binary())); } - return this->store_client_->AsyncBatchDelete( + this->store_client_->AsyncBatchDelete( this->table_name_, keys_to_delete, JustOk(std::move(callback))); } template -Status GcsTableWithJobId::Put(const Key &key, - const Data &value, - Postable callback) { +void GcsTableWithJobId::Put(const Key &key, + const Data &value, + Postable callback) { { absl::MutexLock lock(&mutex_); index_[GetJobIdFromKey(key)].insert(key); } - return this->store_client_->AsyncPut(this->table_name_, - key.Binary(), - value.SerializeAsString(), - /*overwrite*/ true, - JustOk(std::move(callback))); + this->store_client_->AsyncPut(this->table_name_, + key.Binary(), + value.SerializeAsString(), + /*overwrite*/ true, + JustOk(std::move(callback))); } template -Status GcsTableWithJobId::GetByJobId( +void GcsTableWithJobId::GetByJobId( const JobID &job_id, Postable)> callback) { std::vector keys; { @@ -128,7 +127,7 @@ Status GcsTableWithJobId::GetByJobId( keys.push_back(key.Binary()); } } - return this->store_client_->AsyncMultiGet( + this->store_client_->AsyncMultiGet( this->table_name_, keys, std::move(callback).TransformArg( @@ -144,8 +143,8 @@ Status GcsTableWithJobId::GetByJobId( } template -Status GcsTableWithJobId::DeleteByJobId(const JobID &job_id, - Postable callback) { +void GcsTableWithJobId::DeleteByJobId(const JobID &job_id, + Postable callback) { std::vector keys; { absl::MutexLock lock(&mutex_); @@ -154,24 +153,24 @@ Status GcsTableWithJobId::DeleteByJobId(const JobID &job_id, keys.push_back(key); } } - return BatchDelete(keys, std::move(callback)); + BatchDelete(keys, std::move(callback)); } template -Status GcsTableWithJobId::Delete(const Key &key, - Postable callback) { - return BatchDelete({key}, std::move(callback)); +void GcsTableWithJobId::Delete(const Key &key, + Postable callback) { + BatchDelete({key}, std::move(callback)); } template -Status GcsTableWithJobId::BatchDelete(const std::vector &keys, - Postable callback) { +void GcsTableWithJobId::BatchDelete(const std::vector &keys, + Postable callback) { std::vector keys_to_delete; keys_to_delete.reserve(keys.size()); for (auto &key : keys) { keys_to_delete.push_back(key.Binary()); } - return this->store_client_->AsyncBatchDelete( + this->store_client_->AsyncBatchDelete( this->table_name_, keys_to_delete, std::move(callback).TransformArg([this, callback, keys](int64_t) { @@ -186,9 +185,9 @@ Status GcsTableWithJobId::BatchDelete(const std::vector &keys, } template -Status GcsTableWithJobId::AsyncRebuildIndexAndGetAll( +void GcsTableWithJobId::AsyncRebuildIndexAndGetAll( Postable)> callback) { - return this->GetAll(std::move(callback).TransformArg( + this->GetAll(std::move(callback).TransformArg( [this](absl::flat_hash_map result) mutable { absl::MutexLock lock(&this->mutex_); this->index_.clear(); diff --git a/src/ray/gcs/gcs_server/gcs_table_storage.h b/src/ray/gcs/gcs_server/gcs_table_storage.h index 92baeb8f38b4..46f1e8b746dc 100644 --- a/src/ray/gcs/gcs_server/gcs_table_storage.h +++ b/src/ray/gcs/gcs_server/gcs_table_storage.h @@ -46,38 +46,33 @@ class GcsTable { /// \param key The key that will be written to the table. /// \param value The value of the key that will be written to the table. /// \param callback Callback that will be called after write finishes. - /// \return Status - virtual Status Put(const Key &key, - const Data &value, - Postable callback); + virtual void Put(const Key &key, + const Data &value, + Postable callback); /// Get data from the table asynchronously. /// /// \param key The key to lookup from the table. /// \param callback Callback that will be called after read finishes. - /// \return Status - Status Get(const Key &key, Postable)> callback); + void Get(const Key &key, Postable)> callback); /// Get all data from the table asynchronously. /// /// \param callback Callback that will be called after data has been received. - /// \return Status - Status GetAll(Postable)> callback); + void GetAll(Postable)> callback); /// Delete data from the table asynchronously. /// /// \param key The key that will be deleted from the table. /// \param callback Callback that will be called after delete finishes. - /// \return Status - virtual Status Delete(const Key &key, Postable callback); + virtual void Delete(const Key &key, Postable callback); /// Delete a batch of data from the table asynchronously. /// /// \param keys The batch key that will be deleted from the table. /// \param callback Callback that will be called after delete finishes. - /// \return Status - virtual Status BatchDelete(const std::vector &keys, - Postable callback); + virtual void BatchDelete(const std::vector &keys, + Postable callback); protected: std::string table_name_; @@ -105,43 +100,39 @@ class GcsTableWithJobId : public GcsTable { /// from the key. /// \param value The value of the key that will be written to the table. /// \param callback Callback that will be called after write finishes, whether it - /// succeeds or not. \return Status for issuing the asynchronous write operation. - Status Put(const Key &key, - const Data &value, - Postable callback) override; + /// succeeds or not. + void Put(const Key &key, + const Data &value, + Postable callback) override; /// Get all the data of the specified job id from the table asynchronously. /// /// \param job_id The key to lookup from the table. /// \param callback Callback that will be called after read finishes. - /// \return Status - Status GetByJobId(const JobID &job_id, - Postable)> callback); + void GetByJobId(const JobID &job_id, + Postable)> callback); /// Delete all the data of the specified job id from the table asynchronously. /// /// \param job_id The key that will be deleted from the table. /// \param callback Callback that will be called after delete finishes. - /// \return Status - Status DeleteByJobId(const JobID &job_id, Postable callback); + void DeleteByJobId(const JobID &job_id, Postable callback); /// Delete data and index from the table asynchronously. /// /// \param key The key that will be deleted from the table. /// \param callback Callback that will be called after delete finishes. - /// \return Status - Status Delete(const Key &key, Postable callback) override; + void Delete(const Key &key, Postable callback) override; /// Delete a batch of data and index from the table asynchronously. /// /// \param keys The batch key that will be deleted from the table. /// \param callback Callback that will be called after delete finishes. - /// \return Status - Status BatchDelete(const std::vector &keys, - Postable callback) override; + void BatchDelete(const std::vector &keys, + Postable callback) override; /// Rebuild the index during startup. - Status AsyncRebuildIndexAndGetAll( + void AsyncRebuildIndexAndGetAll( Postable)> callback); protected: @@ -250,9 +241,9 @@ class GcsTableStorage { return *worker_table_; } - Status AsyncGetNextJobID(Postable callback) { + void AsyncGetNextJobID(Postable callback) { RAY_CHECK(store_client_); - return store_client_->AsyncGetNextJobID(std::move(callback)); + store_client_->AsyncGetNextJobID(std::move(callback)); } protected: diff --git a/src/ray/gcs/gcs_server/gcs_worker_manager.cc b/src/ray/gcs/gcs_server/gcs_worker_manager.cc index b45e47cf080c..aa13eeda4819 100644 --- a/src/ray/gcs/gcs_server/gcs_worker_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_worker_manager.cc @@ -105,8 +105,8 @@ void GcsWorkerManager::HandleReportWorkerFailure( // receives the worker registration information first and then the worker failure // message, so we delete the get operation. Related issues: // https://github.com/ray-project/ray/pull/11599 - RAY_UNUSED(gcs_table_storage_.WorkerTable().Put( - worker_id, *worker_failure_data, {std::move(on_done), io_context_})); + gcs_table_storage_.WorkerTable().Put( + worker_id, *worker_failure_data, {std::move(on_done), io_context_}); if (request.worker_failure().exit_type() == rpc::WorkerExitType::SYSTEM_ERROR || request.worker_failure().exit_type() == @@ -197,10 +197,7 @@ void GcsWorkerManager::HandleGetAllWorkerInfo( RAY_LOG(DEBUG) << "Finished getting all worker info."; GCS_RPC_SEND_REPLY(send_reply_callback, reply, Status::OK()); }; - Status status = gcs_table_storage_.WorkerTable().GetAll({on_done, io_context_}); - if (!status.ok()) { - on_done(absl::flat_hash_map()); - } + gcs_table_storage_.WorkerTable().GetAll({std::move(on_done), io_context_}); } void GcsWorkerManager::HandleAddWorkerInfo(rpc::AddWorkerInfoRequest request, @@ -221,11 +218,7 @@ void GcsWorkerManager::HandleAddWorkerInfo(rpc::AddWorkerInfoRequest request, GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); }; - Status status = gcs_table_storage_.WorkerTable().Put( - worker_id, *worker_data, {on_done, io_context_}); - if (!status.ok()) { - on_done(status); - } + gcs_table_storage_.WorkerTable().Put(worker_id, *worker_data, {on_done, io_context_}); } void GcsWorkerManager::HandleUpdateWorkerDebuggerPort( @@ -259,19 +252,13 @@ void GcsWorkerManager::HandleUpdateWorkerDebuggerPort( auto worker_data = std::make_shared(); worker_data->CopyFrom(*result); worker_data->set_debugger_port(debugger_port); - Status put_status = gcs_table_storage_.WorkerTable().Put( - worker_id, *worker_data, {on_worker_update_done, io_context_}); - if (!put_status.ok()) { - GCS_RPC_SEND_REPLY(send_reply_callback, reply, put_status); - } + gcs_table_storage_.WorkerTable().Put( + worker_id, *worker_data, {std::move(on_worker_update_done), io_context_}); } }; - Status status = - gcs_table_storage_.WorkerTable().Get(worker_id, {on_worker_get_done, io_context_}); - if (!status.ok()) { - GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); - } + gcs_table_storage_.WorkerTable().Get(worker_id, + {std::move(on_worker_get_done), io_context_}); } void GcsWorkerManager::HandleUpdateWorkerNumPausedThreads( @@ -317,19 +304,13 @@ void GcsWorkerManager::HandleUpdateWorkerNumPausedThreads( worker_data->has_num_paused_threads() ? worker_data->num_paused_threads() : 0; worker_data->set_num_paused_threads(current_num_paused_threads + num_paused_threads_delta); - Status put_status = gcs_table_storage_.WorkerTable().Put( - worker_id, *worker_data, {on_worker_update_done, io_context_}); - if (!put_status.ok()) { - GCS_RPC_SEND_REPLY(send_reply_callback, reply, put_status); - } + gcs_table_storage_.WorkerTable().Put( + worker_id, *worker_data, {std::move(on_worker_update_done), io_context_}); } }; - Status status = - gcs_table_storage_.WorkerTable().Get(worker_id, {on_worker_get_done, io_context_}); - if (!status.ok()) { - GCS_RPC_SEND_REPLY(send_reply_callback, reply, status); - } + gcs_table_storage_.WorkerTable().Get(worker_id, + {std::move(on_worker_get_done), io_context_}); } void GcsWorkerManager::AddWorkerDeadListener( @@ -341,7 +322,7 @@ void GcsWorkerManager::AddWorkerDeadListener( void GcsWorkerManager::GetWorkerInfo( const WorkerID &worker_id, Postable)> callback) const { - RAY_CHECK_OK(gcs_table_storage_.WorkerTable().Get( + gcs_table_storage_.WorkerTable().Get( worker_id, std::move(callback).TransformArg( [worker_id](Status status, std::optional data) { @@ -350,7 +331,7 @@ void GcsWorkerManager::GetWorkerInfo( << "Failed to get worker info, status = " << status; } return data; - }))); + })); } } // namespace gcs diff --git a/src/ray/gcs/gcs_server/store_client_kv.cc b/src/ray/gcs/gcs_server/store_client_kv.cc index 6c1fc739073d..1586087246b5 100644 --- a/src/ray/gcs/gcs_server/store_client_kv.cc +++ b/src/ray/gcs/gcs_server/store_client_kv.cc @@ -56,7 +56,7 @@ StoreClientInternalKV::StoreClientInternalKV(std::unique_ptr store_ void StoreClientInternalKV::Get(const std::string &ns, const std::string &key, Postable)> callback) { - RAY_CHECK_OK(delegate_->AsyncGet( + delegate_->AsyncGet( table_name_, MakeKey(ns, key), std::move(callback).TransformArg( @@ -64,7 +64,7 @@ void StoreClientInternalKV::Get(const std::string &ns, std::optional result) -> std::optional { RAY_CHECK(status.ok()) << "Fails to get key from storage " << status; return result; - }))); + })); } void StoreClientInternalKV::MultiGet( @@ -76,20 +76,18 @@ void StoreClientInternalKV::MultiGet( for (const auto &key : keys) { prefixed_keys.emplace_back(MakeKey(ns, key)); } - RAY_CHECK_OK(delegate_->AsyncMultiGet( + delegate_->AsyncMultiGet( table_name_, prefixed_keys, - std::move(callback).TransformArg // < - // absl::flat_hash_map( - // absl::flat_hash_map)> - ([](absl::flat_hash_map before_extract) { - absl::flat_hash_map ret; - ret.reserve(before_extract.size()); - for (auto &&item : std::move(before_extract)) { - ret.emplace(ExtractKey(item.first), std::move(item.second)); - } - return ret; - }))); + std::move(callback).TransformArg( + [](absl::flat_hash_map before_extract) { + absl::flat_hash_map ret; + ret.reserve(before_extract.size()); + for (auto &&item : std::move(before_extract)) { + ret.emplace(ExtractKey(item.first), std::move(item.second)); + } + return ret; + })); } void StoreClientInternalKV::Put(const std::string &ns, @@ -97,8 +95,8 @@ void StoreClientInternalKV::Put(const std::string &ns, std::string value, bool overwrite, Postable callback) { - RAY_CHECK_OK(delegate_->AsyncPut( - table_name_, MakeKey(ns, key), std::move(value), overwrite, std::move(callback))); + delegate_->AsyncPut( + table_name_, MakeKey(ns, key), std::move(value), overwrite, std::move(callback)); } void StoreClientInternalKV::Del(const std::string &ns, @@ -106,17 +104,16 @@ void StoreClientInternalKV::Del(const std::string &ns, bool del_by_prefix, Postable callback) { if (!del_by_prefix) { - RAY_CHECK_OK(delegate_->AsyncDelete( - table_name_, - MakeKey(ns, key), - std::move(callback).TransformArg( - [](bool deleted) -> int64_t { return deleted ? 1 : 0; }))); + delegate_->AsyncDelete(table_name_, + MakeKey(ns, key), + std::move(callback).TransformArg( + [](bool deleted) -> int64_t { return deleted ? 1 : 0; })); return; } instrumented_io_context &io_context = callback.io_context(); - RAY_CHECK_OK(delegate_->AsyncGetKeys( + delegate_->AsyncGetKeys( table_name_, MakeKey(ns, key), {[this, ns, callback = std::move(callback)](auto keys) mutable { @@ -124,23 +121,21 @@ void StoreClientInternalKV::Del(const std::string &ns, std::move(callback).Dispatch("StoreClientInternalKV.Del", 0); return; } - RAY_CHECK_OK( - delegate_->AsyncBatchDelete(table_name_, keys, std::move(callback))); + delegate_->AsyncBatchDelete(table_name_, keys, std::move(callback)); }, - io_context})); + io_context}); } void StoreClientInternalKV::Exists(const std::string &ns, const std::string &key, Postable callback) { - RAY_CHECK_OK( - delegate_->AsyncExists(table_name_, MakeKey(ns, key), std::move(callback))); + delegate_->AsyncExists(table_name_, MakeKey(ns, key), std::move(callback)); } void StoreClientInternalKV::Keys(const std::string &ns, const std::string &prefix, Postable)> callback) { - RAY_CHECK_OK(delegate_->AsyncGetKeys( + delegate_->AsyncGetKeys( table_name_, MakeKey(ns, prefix), std::move(callback).TransformArg([](std::vector keys) { @@ -150,7 +145,7 @@ void StoreClientInternalKV::Keys(const std::string &ns, true_keys.emplace_back(ExtractKey(key)); } return true_keys; - }))); + })); } } // namespace gcs diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc index 70e1003cfeae..50208c9ea62a 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc @@ -159,8 +159,8 @@ TEST_F(GcsActorSchedulerMockTest, KillWorkerLeak2) { std::unique_ptr> async_put_with_index_cb; // Leasing successfully EXPECT_CALL(*store_client, AsyncPut(_, _, _, _, _)) - .WillOnce( - DoAll(SaveArgToUniquePtr<4>(&async_put_with_index_cb), Return(Status::OK()))); + .WillOnce(DoAll(SaveArgToUniquePtr<4>(&async_put_with_index_cb), + InvokeWithoutArgs([]() {}))); actor_scheduler->ScheduleByRaylet(actor); rpc::RequestWorkerLeaseReply reply; reply.mutable_worker_address()->set_node_id(node_id.Binary()); diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc index 049b7faa42c3..89816e4ca1e3 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc @@ -68,12 +68,10 @@ class FakeGcsActorTable : public gcs::GcsActorTable { explicit FakeGcsActorTable(std::shared_ptr store_client) : GcsActorTable(store_client) {} - Status Put(const ActorID &key, - const rpc::ActorTableData &value, - Postable callback) override { - auto status = Status::OK(); - std::move(callback).Post("FakeGcsActorTable.Put", status); - return status; + void Put(const ActorID &key, + const rpc::ActorTableData &value, + Postable callback) override { + std::move(callback).Post("FakeGcsActorTable.Put", Status::OK()); } private: diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc index 07caca6d2a56..4be984192de4 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc @@ -76,7 +76,7 @@ TEST_F(GcsPlacementGroupManagerMockTest, PendingQueuePriorityReschedule) { SchedulePgRequest request; std::unique_ptr> put_cb; EXPECT_CALL(*store_client_, AsyncPut(_, _, _, _, _)) - .WillOnce(DoAll(SaveArgToUniquePtr<4>(&put_cb), Return(Status::OK()))); + .WillOnce(DoAll(SaveArgToUniquePtr<4>(&put_cb))); EXPECT_CALL(*gcs_placement_group_scheduler_, ScheduleUnplacedBundles(_)) .WillOnce(DoAll(SaveArg<0>(&request))); auto now = absl::GetCurrentTimeNanos(); @@ -103,7 +103,7 @@ TEST_F(GcsPlacementGroupManagerMockTest, PendingQueuePriorityFailed) { SchedulePgRequest request; std::unique_ptr> put_cb; EXPECT_CALL(*store_client_, AsyncPut(_, _, _, _, _)) - .WillOnce(DoAll(SaveArgToUniquePtr<4>(&put_cb), Return(Status::OK()))); + .WillOnce(DoAll(SaveArgToUniquePtr<4>(&put_cb))); EXPECT_CALL(*gcs_placement_group_scheduler_, ScheduleUnplacedBundles(_)) .Times(2) .WillRepeatedly(DoAll(SaveArg<0>(&request))); @@ -161,7 +161,7 @@ TEST_F(GcsPlacementGroupManagerMockTest, PendingQueuePriorityOrder) { std::unique_ptr> put_cb; EXPECT_CALL(*store_client_, AsyncPut(_, _, _, _, _)) .Times(2) - .WillRepeatedly(DoAll(SaveArgToUniquePtr<4>(&put_cb), Return(Status::OK()))); + .WillRepeatedly(DoAll(SaveArgToUniquePtr<4>(&put_cb))); EXPECT_CALL(*gcs_placement_group_scheduler_, ScheduleUnplacedBundles(_)) .Times(2) .WillRepeatedly(DoAll(SaveArg<0>(&request))); diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc index 15876316cac2..d6421e6b7758 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc @@ -816,9 +816,9 @@ TEST_F(GcsPlacementGroupManagerTest, TestSchedulerReinitializeAfterGcsRestart) { /* cpu_num */ 1.0, /* job_id */ job_id); auto job_table_data = Mocker::GenJobTableData(job_id); - RAY_CHECK_OK(gcs_table_storage_->JobTable().Put( - job_id, *job_table_data, {[](auto) {}, io_service_})); - std::atomic registered_placement_group_count(0); + gcs_table_storage_->JobTable().Put( + job_id, *job_table_data, {[](auto) {}, io_service_}); + std::atomic registered_placement_group_count{0}; RegisterPlacementGroup(request, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; }); @@ -1244,10 +1244,9 @@ TEST_F(GcsPlacementGroupManagerTest, TestCheckCreatorJobIsDeadWhenGcsRestart) { /* job_id */ job_id); auto job_table_data = Mocker::GenJobTableData(job_id); job_table_data->set_is_dead(true); - RAY_CHECK_OK(gcs_table_storage_->JobTable().Put( - job_id, *job_table_data, {[](auto) {}, io_service_})); - - std::atomic registered_placement_group_count(0); + gcs_table_storage_->JobTable().Put( + job_id, *job_table_data, {[](auto) {}, io_service_}); + std::atomic registered_placement_group_count{0}; RegisterPlacementGroup(request, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; }); diff --git a/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h b/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h index 8e628085e38f..cd66ea7fb0e7 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h +++ b/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h @@ -38,7 +38,7 @@ class GcsTableStorageTestBase : public ::testing::Test { protected: void TestGcsTableApi() { - auto table = gcs_table_storage_->JobTable(); + auto &table = gcs_table_storage_->JobTable(); JobID job1_id = JobID::FromInt(1); JobID job2_id = JobID::FromInt(2); auto job1_table_data = Mocker::GenJobTableData(job1_id); @@ -105,7 +105,7 @@ class GcsTableStorageTestBase : public ::testing::Test { void Put(TABLE &table, const KEY &key, const VALUE &value) { auto on_done = [this](const Status &status) { --pending_count_; }; ++pending_count_; - RAY_CHECK_OK(table.Put(key, value, {on_done, *(io_service_pool_->Get())})); + table.Put(key, value, {on_done, *(io_service_pool_->Get())}); WaitPendingDone(); } @@ -124,7 +124,7 @@ class GcsTableStorageTestBase : public ::testing::Test { --pending_count_; }; ++pending_count_; - RAY_CHECK_OK(table.Get(key, {on_done, *(io_service_pool_->Get())})); + table.Get(key, {on_done, *(io_service_pool_->Get())}); WaitPendingDone(); return values.size(); } @@ -147,7 +147,7 @@ class GcsTableStorageTestBase : public ::testing::Test { --pending_count_; }; ++pending_count_; - RAY_CHECK_OK(table.GetByJobId(job_id, {on_done, *(io_service_pool_->Get())})); + table.GetByJobId(job_id, {on_done, *(io_service_pool_->Get())}); WaitPendingDone(); return values.size(); } @@ -159,7 +159,7 @@ class GcsTableStorageTestBase : public ::testing::Test { --pending_count_; }; ++pending_count_; - RAY_CHECK_OK(table.Delete(key, {on_done, *(io_service_pool_->Get())})); + table.Delete(key, {on_done, *(io_service_pool_->Get())}); WaitPendingDone(); } @@ -170,7 +170,7 @@ class GcsTableStorageTestBase : public ::testing::Test { --pending_count_; }; ++pending_count_; - RAY_CHECK_OK(table.BatchDelete(keys, {on_done, *(io_service_pool_->Get())})); + table.BatchDelete(keys, {on_done, *(io_service_pool_->Get())}); WaitPendingDone(); } diff --git a/src/ray/gcs/store_client/in_memory_store_client.cc b/src/ray/gcs/store_client/in_memory_store_client.cc index d7e64bba9fcd..cea449dd71d5 100644 --- a/src/ray/gcs/store_client/in_memory_store_client.cc +++ b/src/ray/gcs/store_client/in_memory_store_client.cc @@ -20,11 +20,11 @@ namespace ray::gcs { -Status InMemoryStoreClient::AsyncPut(const std::string &table_name, - const std::string &key, - std::string data, - bool overwrite, - Postable callback) { +void InMemoryStoreClient::AsyncPut(const std::string &table_name, + const std::string &key, + std::string data, + bool overwrite, + Postable callback) { auto &table = GetOrCreateMutableTable(table_name); bool inserted = false; if (overwrite) { @@ -33,10 +33,9 @@ Status InMemoryStoreClient::AsyncPut(const std::string &table_name, inserted = table.Emplace(key, std::move(data)); } std::move(callback).Post("GcsInMemoryStore.Put", inserted); - return Status::OK(); } -Status InMemoryStoreClient::AsyncGet( +void InMemoryStoreClient::AsyncGet( const std::string &table_name, const std::string &key, ToPostable> callback) { @@ -46,10 +45,9 @@ Status InMemoryStoreClient::AsyncGet( data = table->Get(key); } std::move(callback).Post("GcsInMemoryStore.Get", Status::OK(), std::move(data)); - return Status::OK(); } -Status InMemoryStoreClient::AsyncGetAll( +void InMemoryStoreClient::AsyncGetAll( const std::string &table_name, Postable)> callback) { auto result = absl::flat_hash_map(); @@ -58,10 +56,9 @@ Status InMemoryStoreClient::AsyncGetAll( result = table->GetMapClone(); } std::move(callback).Post("GcsInMemoryStore.GetAll", std::move(result)); - return Status::OK(); } -Status InMemoryStoreClient::AsyncMultiGet( +void InMemoryStoreClient::AsyncMultiGet( const std::string &table_name, const std::vector &keys, Postable)> callback) { @@ -74,31 +71,27 @@ Status InMemoryStoreClient::AsyncMultiGet( }); } std::move(callback).Post("GcsInMemoryStore.GetAll", std::move(result)); - return Status::OK(); } -Status InMemoryStoreClient::AsyncDelete(const std::string &table_name, - const std::string &key, - Postable callback) { +void InMemoryStoreClient::AsyncDelete(const std::string &table_name, + const std::string &key, + Postable callback) { auto &table = GetOrCreateMutableTable(table_name); auto erased = table.Erase(key); std::move(callback).Post("GcsInMemoryStore.Delete", erased); - return Status::OK(); } -Status InMemoryStoreClient::AsyncBatchDelete(const std::string &table_name, - const std::vector &keys, - Postable callback) { +void InMemoryStoreClient::AsyncBatchDelete(const std::string &table_name, + const std::vector &keys, + Postable callback) { auto &table = GetOrCreateMutableTable(table_name); int64_t num_erased = table.EraseKeys(absl::MakeSpan(keys)); std::move(callback).Post("GcsInMemoryStore.BatchDelete", num_erased); - return Status::OK(); } -Status InMemoryStoreClient::AsyncGetNextJobID(Postable callback) { +void InMemoryStoreClient::AsyncGetNextJobID(Postable callback) { auto job_id = job_id_.fetch_add(1, std::memory_order_acq_rel); std::move(callback).Post("GcsInMemoryStore.GetNextJobID", job_id); - return Status::OK(); } ConcurrentFlatMap &InMemoryStoreClient::GetOrCreateMutableTable( @@ -121,7 +114,7 @@ const ConcurrentFlatMap *InMemoryStoreClient::GetTable return nullptr; } -Status InMemoryStoreClient::AsyncGetKeys( +void InMemoryStoreClient::AsyncGetKeys( const std::string &table_name, const std::string &prefix, Postable)> callback) { @@ -135,20 +128,17 @@ Status InMemoryStoreClient::AsyncGetKeys( }); } std::move(callback).Post("GcsInMemoryStore.Keys", std::move(result)); - - return Status::OK(); } -Status InMemoryStoreClient::AsyncExists(const std::string &table_name, - const std::string &key, - Postable callback) { +void InMemoryStoreClient::AsyncExists(const std::string &table_name, + const std::string &key, + Postable callback) { bool result = false; auto table = GetTable(table_name); if (table != nullptr) { result = table->Contains(key); } std::move(callback).Post("GcsInMemoryStore.Exists", result); - return Status::OK(); } } // namespace ray::gcs diff --git a/src/ray/gcs/store_client/in_memory_store_client.h b/src/ray/gcs/store_client/in_memory_store_client.h index e95754592857..0eb2904735b2 100644 --- a/src/ray/gcs/store_client/in_memory_store_client.h +++ b/src/ray/gcs/store_client/in_memory_store_client.h @@ -34,42 +34,42 @@ class InMemoryStoreClient : public StoreClient { public: explicit InMemoryStoreClient() = default; - Status AsyncPut(const std::string &table_name, - const std::string &key, - std::string data, - bool overwrite, - Postable callback) override; + void AsyncPut(const std::string &table_name, + const std::string &key, + std::string data, + bool overwrite, + Postable callback) override; - Status AsyncGet(const std::string &table_name, - const std::string &key, - ToPostable> callback) override; + void AsyncGet(const std::string &table_name, + const std::string &key, + ToPostable> callback) override; - Status AsyncGetAll( + void AsyncGetAll( const std::string &table_name, Postable)> callback) override; - Status AsyncMultiGet( + void AsyncMultiGet( const std::string &table_name, const std::vector &keys, Postable)> callback) override; - Status AsyncDelete(const std::string &table_name, - const std::string &key, - Postable callback) override; + void AsyncDelete(const std::string &table_name, + const std::string &key, + Postable callback) override; - Status AsyncBatchDelete(const std::string &table_name, - const std::vector &keys, - Postable callback) override; + void AsyncBatchDelete(const std::string &table_name, + const std::vector &keys, + Postable callback) override; - Status AsyncGetNextJobID(Postable callback) override; + void AsyncGetNextJobID(Postable callback) override; - Status AsyncGetKeys(const std::string &table_name, - const std::string &prefix, - Postable)> callback) override; + void AsyncGetKeys(const std::string &table_name, + const std::string &prefix, + Postable)> callback) override; - Status AsyncExists(const std::string &table_name, - const std::string &key, - Postable callback) override; + void AsyncExists(const std::string &table_name, + const std::string &key, + Postable callback) override; private: // The returned reference is valid as long as the InMemoryStoreClient is alive and diff --git a/src/ray/gcs/store_client/observable_store_client.cc b/src/ray/gcs/store_client/observable_store_client.cc index 5243944a9f77..b9c5d84c31dc 100644 --- a/src/ray/gcs/store_client/observable_store_client.cc +++ b/src/ray/gcs/store_client/observable_store_client.cc @@ -24,79 +24,81 @@ namespace ray { namespace gcs { -Status ObservableStoreClient::AsyncPut(const std::string &table_name, - const std::string &key, - std::string data, - bool overwrite, - Postable callback) { +void ObservableStoreClient::AsyncPut(const std::string &table_name, + const std::string &key, + std::string data, + bool overwrite, + Postable callback) { auto start = absl::GetCurrentTimeNanos(); ray::stats::STATS_gcs_storage_operation_count.Record(1, "Put"); - return delegate_->AsyncPut( - table_name, key, data, overwrite, std::move(callback).OnInvocation([start]() { - auto end = absl::GetCurrentTimeNanos(); - ray::stats::STATS_gcs_storage_operation_latency_ms.Record( - absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), "Put"); - })); + delegate_->AsyncPut(table_name, + key, + std::move(data), + overwrite, + std::move(callback).OnInvocation([start]() { + auto end = absl::GetCurrentTimeNanos(); + ray::stats::STATS_gcs_storage_operation_latency_ms.Record( + absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), + "Put"); + })); } -Status ObservableStoreClient::AsyncGet( +void ObservableStoreClient::AsyncGet( const std::string &table_name, const std::string &key, ToPostable> callback) { auto start = absl::GetCurrentTimeNanos(); ray::stats::STATS_gcs_storage_operation_count.Record(1, "Get"); - return delegate_->AsyncGet(table_name, key, std::move(callback).OnInvocation([start]() { + delegate_->AsyncGet(table_name, key, std::move(callback).OnInvocation([start]() { auto end = absl::GetCurrentTimeNanos(); ray::stats::STATS_gcs_storage_operation_latency_ms.Record( absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), "Get"); })); } -Status ObservableStoreClient::AsyncGetAll( +void ObservableStoreClient::AsyncGetAll( const std::string &table_name, Postable)> callback) { auto start = absl::GetCurrentTimeNanos(); ray::stats::STATS_gcs_storage_operation_count.Record(1, "GetAll"); - return delegate_->AsyncGetAll(table_name, std::move(callback).OnInvocation([start]() { + delegate_->AsyncGetAll(table_name, std::move(callback).OnInvocation([start]() { auto end = absl::GetCurrentTimeNanos(); ray::stats::STATS_gcs_storage_operation_latency_ms.Record( absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), "GetAll"); })); } -Status ObservableStoreClient::AsyncMultiGet( +void ObservableStoreClient::AsyncMultiGet( const std::string &table_name, const std::vector &keys, Postable)> callback) { auto start = absl::GetCurrentTimeNanos(); ray::stats::STATS_gcs_storage_operation_count.Record(1, "MultiGet"); - return delegate_->AsyncMultiGet( - table_name, keys, std::move(callback).OnInvocation([start]() { - auto end = absl::GetCurrentTimeNanos(); - ray::stats::STATS_gcs_storage_operation_latency_ms.Record( - absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), "MultiGet"); - })); + delegate_->AsyncMultiGet(table_name, keys, std::move(callback).OnInvocation([start]() { + auto end = absl::GetCurrentTimeNanos(); + ray::stats::STATS_gcs_storage_operation_latency_ms.Record( + absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), "MultiGet"); + })); } -Status ObservableStoreClient::AsyncDelete(const std::string &table_name, - const std::string &key, - Postable callback) { +void ObservableStoreClient::AsyncDelete(const std::string &table_name, + const std::string &key, + Postable callback) { auto start = absl::GetCurrentTimeNanos(); ray::stats::STATS_gcs_storage_operation_count.Record(1, "Delete"); - return delegate_->AsyncDelete( - table_name, key, std::move(callback).OnInvocation([start]() { - auto end = absl::GetCurrentTimeNanos(); - ray::stats::STATS_gcs_storage_operation_latency_ms.Record( - absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), "Delete"); - })); + delegate_->AsyncDelete(table_name, key, std::move(callback).OnInvocation([start]() { + auto end = absl::GetCurrentTimeNanos(); + ray::stats::STATS_gcs_storage_operation_latency_ms.Record( + absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), "Delete"); + })); } -Status ObservableStoreClient::AsyncBatchDelete(const std::string &table_name, - const std::vector &keys, - Postable callback) { +void ObservableStoreClient::AsyncBatchDelete(const std::string &table_name, + const std::vector &keys, + Postable callback) { auto start = absl::GetCurrentTimeNanos(); ray::stats::STATS_gcs_storage_operation_count.Record(1, "BatchDelete"); - return delegate_->AsyncBatchDelete( + delegate_->AsyncBatchDelete( table_name, keys, std::move(callback).OnInvocation([start]() { auto end = absl::GetCurrentTimeNanos(); ray::stats::STATS_gcs_storage_operation_latency_ms.Record( @@ -104,35 +106,33 @@ Status ObservableStoreClient::AsyncBatchDelete(const std::string &table_name, })); } -Status ObservableStoreClient::AsyncGetNextJobID(Postable callback) { - return delegate_->AsyncGetNextJobID(std::move(callback)); +void ObservableStoreClient::AsyncGetNextJobID(Postable callback) { + delegate_->AsyncGetNextJobID(std::move(callback)); } -Status ObservableStoreClient::AsyncGetKeys( +void ObservableStoreClient::AsyncGetKeys( const std::string &table_name, const std::string &prefix, Postable)> callback) { auto start = absl::GetCurrentTimeNanos(); ray::stats::STATS_gcs_storage_operation_count.Record(1, "GetKeys"); - return delegate_->AsyncGetKeys( - table_name, prefix, std::move(callback).OnInvocation([start]() { - auto end = absl::GetCurrentTimeNanos(); - ray::stats::STATS_gcs_storage_operation_latency_ms.Record( - absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), "GetKeys"); - })); + delegate_->AsyncGetKeys(table_name, prefix, std::move(callback).OnInvocation([start]() { + auto end = absl::GetCurrentTimeNanos(); + ray::stats::STATS_gcs_storage_operation_latency_ms.Record( + absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), "GetKeys"); + })); } -Status ObservableStoreClient::AsyncExists(const std::string &table_name, - const std::string &key, - Postable callback) { +void ObservableStoreClient::AsyncExists(const std::string &table_name, + const std::string &key, + Postable callback) { auto start = absl::GetCurrentTimeNanos(); ray::stats::STATS_gcs_storage_operation_count.Record(1, "Exists"); - return delegate_->AsyncExists( - table_name, key, std::move(callback).OnInvocation([start]() { - auto end = absl::GetCurrentTimeNanos(); - ray::stats::STATS_gcs_storage_operation_latency_ms.Record( - absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), "Exists"); - })); + delegate_->AsyncExists(table_name, key, std::move(callback).OnInvocation([start]() { + auto end = absl::GetCurrentTimeNanos(); + ray::stats::STATS_gcs_storage_operation_latency_ms.Record( + absl::ToDoubleMilliseconds(absl::Nanoseconds(end - start)), "Exists"); + })); } } // namespace gcs diff --git a/src/ray/gcs/store_client/observable_store_client.h b/src/ray/gcs/store_client/observable_store_client.h index 1c7bfa9857b6..c483b03b75cd 100644 --- a/src/ray/gcs/store_client/observable_store_client.h +++ b/src/ray/gcs/store_client/observable_store_client.h @@ -31,42 +31,42 @@ class ObservableStoreClient : public StoreClient { explicit ObservableStoreClient(std::unique_ptr delegate) : delegate_(std::move(delegate)) {} - Status AsyncPut(const std::string &table_name, - const std::string &key, - std::string data, - bool overwrite, - Postable callback) override; + void AsyncPut(const std::string &table_name, + const std::string &key, + std::string data, + bool overwrite, + Postable callback) override; - Status AsyncGet(const std::string &table_name, - const std::string &key, - ToPostable> callback) override; + void AsyncGet(const std::string &table_name, + const std::string &key, + ToPostable> callback) override; - Status AsyncGetAll( + void AsyncGetAll( const std::string &table_name, Postable)> callback) override; - Status AsyncMultiGet( + void AsyncMultiGet( const std::string &table_name, const std::vector &keys, Postable)> callback) override; - Status AsyncDelete(const std::string &table_name, - const std::string &key, - Postable callback) override; + void AsyncDelete(const std::string &table_name, + const std::string &key, + Postable callback) override; - Status AsyncBatchDelete(const std::string &table_name, - const std::vector &keys, - Postable callback) override; + void AsyncBatchDelete(const std::string &table_name, + const std::vector &keys, + Postable callback) override; - Status AsyncGetNextJobID(Postable callback) override; + void AsyncGetNextJobID(Postable callback) override; - Status AsyncGetKeys(const std::string &table_name, - const std::string &prefix, - Postable)> callback) override; + void AsyncGetKeys(const std::string &table_name, + const std::string &prefix, + Postable)> callback) override; - Status AsyncExists(const std::string &table_name, - const std::string &key, - Postable callback) override; + void AsyncExists(const std::string &table_name, + const std::string &key, + Postable callback) override; private: std::unique_ptr delegate_; diff --git a/src/ray/gcs/store_client/redis_store_client.cc b/src/ray/gcs/store_client/redis_store_client.cc index f564481ea2cf..c963a693bce0 100644 --- a/src/ray/gcs/store_client/redis_store_client.cc +++ b/src/ray/gcs/store_client/redis_store_client.cc @@ -142,11 +142,11 @@ RedisStoreClient::RedisStoreClient(instrumented_io_context &io_service, << kClusterSeparator << "."; } -Status RedisStoreClient::AsyncPut(const std::string &table_name, - const std::string &key, - std::string data, - bool overwrite, - Postable callback) { +void RedisStoreClient::AsyncPut(const std::string &table_name, + const std::string &key, + std::string data, + bool overwrite, + Postable callback) { RedisCommand command{/*command=*/overwrite ? "HSET" : "HSETNX", RedisKey{external_storage_namespace_, table_name}, /*args=*/{key, std::move(data)}}; @@ -157,13 +157,11 @@ Status RedisStoreClient::AsyncPut(const std::string &table_name, std::move(callback).Dispatch("RedisStoreClient.AsyncPut", added_num != 0); }; SendRedisCmdWithKeys({key}, std::move(command), std::move(write_callback)); - return Status::OK(); } -Status RedisStoreClient::AsyncGet( - const std::string &table_name, - const std::string &key, - ToPostable> callback) { +void RedisStoreClient::AsyncGet(const std::string &table_name, + const std::string &key, + ToPostable> callback) { auto redis_callback = [callback = std::move(callback)]( const std::shared_ptr &reply) mutable { std::optional result; @@ -181,49 +179,45 @@ Status RedisStoreClient::AsyncGet( RedisKey{external_storage_namespace_, table_name}, /*args=*/{key}}; SendRedisCmdArgsAsKeys(std::move(command), std::move(redis_callback)); - return Status::OK(); } -Status RedisStoreClient::AsyncGetAll( +void RedisStoreClient::AsyncGetAll( const std::string &table_name, Postable)> callback) { RedisScanner::ScanKeysAndValues(primary_context_, RedisKey{external_storage_namespace_, table_name}, RedisMatchPattern::Any(), std::move(callback)); - return Status::OK(); } -Status RedisStoreClient::AsyncDelete(const std::string &table_name, - const std::string &key, - Postable callback) { - return AsyncBatchDelete( - table_name, {key}, std::move(callback).TransformArg([](int64_t cnt) { - return cnt > 0; - })); +void RedisStoreClient::AsyncDelete(const std::string &table_name, + const std::string &key, + Postable callback) { + AsyncBatchDelete(table_name, {key}, std::move(callback).TransformArg([](int64_t cnt) { + return cnt > 0; + })); } -Status RedisStoreClient::AsyncBatchDelete(const std::string &table_name, - const std::vector &keys, - Postable callback) { +void RedisStoreClient::AsyncBatchDelete(const std::string &table_name, + const std::vector &keys, + Postable callback) { if (keys.empty()) { std::move(callback).Dispatch("RedisStoreClient.AsyncBatchDelete", 0); - return Status::OK(); + return; } - return DeleteByKeys(table_name, keys, std::move(callback)); + DeleteByKeys(table_name, keys, std::move(callback)); } -Status RedisStoreClient::AsyncMultiGet( +void RedisStoreClient::AsyncMultiGet( const std::string &table_name, const std::vector &keys, Postable)> callback) { if (keys.empty()) { std::move(callback).Dispatch("RedisStoreClient.AsyncMultiGet", absl::flat_hash_map{}); - return Status::OK(); + return; } MGetValues(table_name, keys, std::move(callback)); - return Status::OK(); } size_t RedisStoreClient::PushToSendingQueue(const std::vector &keys, @@ -341,9 +335,9 @@ void RedisStoreClient::SendRedisCmdWithKeys(std::vector keys, } } -Status RedisStoreClient::DeleteByKeys(const std::string &table, - const std::vector &keys, - Postable callback) { +void RedisStoreClient::DeleteByKeys(const std::string &table, + const std::vector &keys, + Postable callback) { auto del_cmds = GenCommandsBatched("HDEL", RedisKey{external_storage_namespace_, table}, keys); auto total_count = del_cmds.size(); @@ -364,7 +358,6 @@ Status RedisStoreClient::DeleteByKeys(const std::string &table, }; SendRedisCmdArgsAsKeys(std::move(command), std::move(delete_callback)); } - return Status::OK(); } RedisStoreClient::RedisScanner::RedisScanner( @@ -459,7 +452,7 @@ void RedisStoreClient::RedisScanner::OnScanCallback( } } -Status RedisStoreClient::AsyncGetNextJobID(Postable callback) { +void RedisStoreClient::AsyncGetNextJobID(Postable callback) { // Note: This is not a HASH! It's a simple key-value pair. // Key: "RAYexternal_storage_namespace@JobCounter" // Value: The next job ID. @@ -473,13 +466,11 @@ Status RedisStoreClient::AsyncGetNextJobID(Postable callback) { auto job_id = static_cast(reply->ReadAsInteger()); std::move(callback).Post("GcsStore.GetNextJobID", job_id); }); - - return Status::OK(); } -Status RedisStoreClient::AsyncGetKeys(const std::string &table_name, - const std::string &prefix, - Postable)> callback) { +void RedisStoreClient::AsyncGetKeys(const std::string &table_name, + const std::string &prefix, + Postable)> callback) { RedisScanner::ScanKeysAndValues( primary_context_, RedisKey{external_storage_namespace_, table_name}, @@ -493,12 +484,11 @@ Status RedisStoreClient::AsyncGetKeys(const std::string &table_name, } return keys; })); - return Status::OK(); } -Status RedisStoreClient::AsyncExists(const std::string &table_name, - const std::string &key, - Postable callback) { +void RedisStoreClient::AsyncExists(const std::string &table_name, + const std::string &key, + Postable callback) { RedisCommand command = { "HEXISTS", RedisKey{external_storage_namespace_, table_name}, {key}}; SendRedisCmdArgsAsKeys( @@ -508,7 +498,6 @@ Status RedisStoreClient::AsyncExists(const std::string &table_name, bool exists = reply->ReadAsInteger() > 0; std::move(callback).Dispatch("RedisStoreClient.AsyncExists", exists); }); - return Status::OK(); } void RedisStoreClient::AsyncCheckHealth(Postable callback) { diff --git a/src/ray/gcs/store_client/redis_store_client.h b/src/ray/gcs/store_client/redis_store_client.h index 6456ee606d86..9d4a6aa6cd62 100644 --- a/src/ray/gcs/store_client/redis_store_client.h +++ b/src/ray/gcs/store_client/redis_store_client.h @@ -135,42 +135,42 @@ class RedisStoreClient : public StoreClient { explicit RedisStoreClient(instrumented_io_context &io_service, const RedisClientOptions &options); - Status AsyncPut(const std::string &table_name, - const std::string &key, - std::string data, - bool overwrite, - Postable callback) override; + void AsyncPut(const std::string &table_name, + const std::string &key, + std::string data, + bool overwrite, + Postable callback) override; - Status AsyncGet(const std::string &table_name, - const std::string &key, - ToPostable> callback) override; + void AsyncGet(const std::string &table_name, + const std::string &key, + ToPostable> callback) override; - Status AsyncGetAll( + void AsyncGetAll( const std::string &table_name, Postable)> callback) override; - Status AsyncMultiGet( + void AsyncMultiGet( const std::string &table_name, const std::vector &keys, Postable)> callback) override; - Status AsyncDelete(const std::string &table_name, - const std::string &key, - Postable callback) override; + void AsyncDelete(const std::string &table_name, + const std::string &key, + Postable callback) override; - Status AsyncBatchDelete(const std::string &table_name, - const std::vector &keys, - Postable callback) override; + void AsyncBatchDelete(const std::string &table_name, + const std::vector &keys, + Postable callback) override; - Status AsyncGetNextJobID(Postable callback) override; + void AsyncGetNextJobID(Postable callback) override; - Status AsyncGetKeys(const std::string &table_name, - const std::string &prefix, - Postable)> callback) override; + void AsyncGetKeys(const std::string &table_name, + const std::string &prefix, + Postable)> callback) override; - Status AsyncExists(const std::string &table_name, - const std::string &key, - Postable callback) override; + void AsyncExists(const std::string &table_name, + const std::string &key, + Postable callback) override; // Check if Redis is available. // @@ -261,9 +261,9 @@ class RedisStoreClient : public StoreClient { std::vector> TakeRequestsFromSendingQueue( const std::vector &keys) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); - Status DeleteByKeys(const std::string &table_name, - const std::vector &keys, - Postable callback); + void DeleteByKeys(const std::string &table_name, + const std::vector &keys, + Postable callback); // Send the redis command to the server. This method will make request to be // serialized for each key in keys. At a given time, only one request for a {table_name, diff --git a/src/ray/gcs/store_client/store_client.h b/src/ray/gcs/store_client/store_client.h index 882a5201a9ee..e2f6a75fb9fc 100644 --- a/src/ray/gcs/store_client/store_client.h +++ b/src/ray/gcs/store_client/store_client.h @@ -43,29 +43,26 @@ class StoreClient { /// will be ignored. /// \param callback WARNING: it returns true if and only if A NEW ENTRY is added. /// Overwritten return false. - /// \return Status - virtual Status AsyncPut(const std::string &table_name, - const std::string &key, - std::string data, - bool overwrite, - Postable callback) = 0; + virtual void AsyncPut(const std::string &table_name, + const std::string &key, + std::string data, + bool overwrite, + Postable callback) = 0; /// Get data from the given table asynchronously. /// /// \param table_name The name of the table to be read. /// \param key The key to lookup from the table. /// \param callback returns the value or null. - /// \return Status - virtual Status AsyncGet(const std::string &table_name, - const std::string &key, - ToPostable> callback) = 0; + virtual void AsyncGet(const std::string &table_name, + const std::string &key, + ToPostable> callback) = 0; /// Get all data from the given table asynchronously. /// /// \param table_name The name of the table to be read. /// \param callback returns the key value pairs in a map. - /// \return Status - virtual Status AsyncGetAll( + virtual void AsyncGetAll( const std::string &table_name, Postable)> callback) = 0; @@ -74,8 +71,7 @@ class StoreClient { /// \param table_name The name of the table to be read. /// \param keys The keys to look up from the table. /// \param callback returns the key value pairs in a map for those keys that exist. - /// \return Status - virtual Status AsyncMultiGet( + virtual void AsyncMultiGet( const std::string &table_name, const std::vector &keys, Postable)> callback) = 0; @@ -85,45 +81,41 @@ class StoreClient { /// \param table_name The name of the table from which data is to be deleted. /// \param key The key that will be deleted from the table. /// \param callback returns true if an entry with matching key is deleted. - /// \return Status - virtual Status AsyncDelete(const std::string &table_name, - const std::string &key, - Postable callback) = 0; + virtual void AsyncDelete(const std::string &table_name, + const std::string &key, + Postable callback) = 0; /// Batch delete data from the given table asynchronously. /// /// \param table_name The name of the table from which data is to be deleted. /// \param keys The keys that will be deleted from the table. /// \param callback returns the number of deleted entries. - /// \return Status - virtual Status AsyncBatchDelete(const std::string &table_name, - const std::vector &keys, - Postable callback) = 0; + virtual void AsyncBatchDelete(const std::string &table_name, + const std::vector &keys, + Postable callback) = 0; /// Get next job id by `INCR` "JobCounter" key asynchronously. /// /// \param callback returns the next job id in integer representation. - /// \return Status - virtual Status AsyncGetNextJobID(Postable callback) = 0; + virtual void AsyncGetNextJobID(Postable callback) = 0; /// Get all the keys match the prefix from the given table asynchronously. /// /// \param table_name The name of the table to be read. /// \param prefix The prefix to be scaned. /// \param callback returns all matching keys in a vector. - /// \return Status - virtual Status AsyncGetKeys(const std::string &table_name, - const std::string &prefix, - Postable)> callback) = 0; + virtual void AsyncGetKeys(const std::string &table_name, + const std::string &prefix, + Postable)> callback) = 0; /// Check whether the key exists in the table. /// /// \param table_name The name of the table to be read. /// \param key The key to be checked. /// \param callback Returns true if such key exists. - virtual Status AsyncExists(const std::string &table_name, - const std::string &key, - Postable callback) = 0; + virtual void AsyncExists(const std::string &table_name, + const std::string &key, + Postable callback) = 0; protected: StoreClient() = default; diff --git a/src/ray/gcs/store_client/tests/redis_store_client_test.cc b/src/ray/gcs/store_client/tests/redis_store_client_test.cc index 1d3cb4c3f7a9..e3d29abaa1cb 100644 --- a/src/ray/gcs/store_client/tests/redis_store_client_test.cc +++ b/src/ray/gcs/store_client/tests/redis_store_client_test.cc @@ -96,31 +96,27 @@ TEST_F(RedisStoreClientTest, BasicSimple) { for (size_t i = 0; i < 100; ++i) { for (size_t j = 0; j < 20; ++j) { ++*cnt; - ASSERT_TRUE(store_client_ - ->AsyncPut("T", - absl::StrCat("A", std::to_string(j)), - std::to_string(i), - true, - {[i, cnt](auto r) { - --*cnt; - ASSERT_TRUE((i == 0 && r) || (i != 0 && !r)); - }, - *io_service_pool_->Get()}) - .ok()); + store_client_->AsyncPut("T", + absl::StrCat("A", std::to_string(j)), + std::to_string(i), + true, + {[i, cnt](auto r) { + --*cnt; + ASSERT_TRUE((i == 0 && r) || (i != 0 && !r)); + }, + *io_service_pool_->Get()}); } } for (size_t j = 0; j < 20; ++j) { ++*cnt; - ASSERT_TRUE(store_client_ - ->AsyncGet("T", - absl::StrCat("A", std::to_string(j)), - {[cnt](auto s, auto r) { - --*cnt; - ASSERT_TRUE(r.has_value()); - ASSERT_EQ(*r, "99"); - }, - *io_service_pool_->Get()}) - .ok()); + store_client_->AsyncGet("T", + absl::StrCat("A", std::to_string(j)), + {[cnt](auto s, auto r) { + --*cnt; + ASSERT_TRUE(r.has_value()); + ASSERT_EQ(*r, "99"); + }, + *io_service_pool_->Get()}); } ASSERT_TRUE(WaitForCondition([cnt]() { return *cnt == 0; }, 5000)); } @@ -135,19 +131,17 @@ TEST_F(RedisStoreClientTest, Complicated) { for (int j = i; j < i + window; ++j) { ++sent; RAY_LOG(INFO) << "S AsyncPut: " << ("P_" + std::to_string(j)); - ASSERT_TRUE(store_client_ - ->AsyncPut("N", - "P_" + std::to_string(j), - std::to_string(j), - true, - {[&finished, j](auto r) mutable { - RAY_LOG(INFO) - << "F AsyncPut: " << ("P_" + std::to_string(j)); - ++finished; - ASSERT_TRUE(r); - }, - *io_service_pool_->Get()}) - .ok()); + store_client_->AsyncPut("N", + "P_" + std::to_string(j), + std::to_string(j), + true, + {[&finished, j](auto r) mutable { + RAY_LOG(INFO) + << "F AsyncPut: " << ("P_" + std::to_string(j)); + ++finished; + ASSERT_TRUE(r); + }, + *io_service_pool_->Get()}); keys.push_back(std::to_string(j)); } @@ -163,89 +157,73 @@ TEST_F(RedisStoreClientTest, Complicated) { ++sent; RAY_LOG(INFO) << "S AsyncMultiGet: " << absl::StrJoin(p_keys, ","); - ASSERT_TRUE( - store_client_ - ->AsyncMultiGet( - "N", - p_keys, - {[&finished, i, keys, window, &sent, p_keys, n_keys, this]( - absl::flat_hash_map m) mutable -> void { - RAY_LOG(INFO) << "F SendAsyncMultiGet: " << absl::StrJoin(p_keys, ","); - ++finished; - ASSERT_EQ(keys.size(), m.size()); - for (auto &key : keys) { - ASSERT_EQ(m["P_" + key], key); - } - - if ((i / window) % 2 == 0) { - // Delete non exist keys - for (size_t jj = 0; jj < keys.size(); ++jj) { - ++sent; - RAY_LOG(INFO) << "S AsyncDelete: " << n_keys[jj]; - ASSERT_TRUE( - store_client_ - ->AsyncDelete("N", - n_keys[jj], - {[&finished, n_keys, jj](auto b) mutable { - RAY_LOG(INFO) - << "F AsyncDelete: " << n_keys[jj]; - ++finished; - ASSERT_FALSE(b); - }, - *this->io_service_pool_->Get()}) - .ok()); - - ++sent; - RAY_LOG(INFO) << "S AsyncExists: " << p_keys[jj]; - ASSERT_TRUE( - store_client_ - ->AsyncExists("N", - p_keys[jj], - {[&finished, p_keys, jj](auto b) mutable { - RAY_LOG(INFO) - << "F AsyncExists: " << p_keys[jj]; - ++finished; - ASSERT_TRUE(b); - }, - *this->io_service_pool_->Get()}) - .ok()); - } - } else { - ++sent; - RAY_LOG(INFO) - << "S AsyncBatchDelete: " << absl::StrJoin(p_keys, ","); - ASSERT_TRUE(store_client_ - ->AsyncBatchDelete( - "N", - p_keys, - {[&finished, p_keys, keys](auto n) mutable { - RAY_LOG(INFO) << "F AsyncBatchDelete: " - << absl::StrJoin(p_keys, ","); - ++finished; - ASSERT_EQ(n, keys.size()); - }, - *this->io_service_pool_->Get()}) - .ok()); - - for (auto p_key : p_keys) { - ++sent; - RAY_LOG(INFO) << "S AsyncExists: " << p_key; - ASSERT_TRUE(store_client_ - ->AsyncExists("N", - p_key, - {[&finished, p_key](auto b) mutable { - RAY_LOG(INFO) - << "F AsyncExists: " << p_key; - ++finished; - ASSERT_FALSE(false); - }, - *this->io_service_pool_->Get()}) - .ok()); - } - } - }, - *io_service_pool_->Get()}) - .ok()); + store_client_->AsyncMultiGet( + "N", + p_keys, + {[&finished, i, keys, window, &sent, p_keys, n_keys, this]( + absl::flat_hash_map m) mutable -> void { + RAY_LOG(INFO) << "F SendAsyncMultiGet: " << absl::StrJoin(p_keys, ","); + ++finished; + ASSERT_EQ(keys.size(), m.size()); + for (auto &key : keys) { + ASSERT_EQ(m["P_" + key], key); + } + + if ((i / window) % 2 == 0) { + // Delete non exist keys + for (size_t jj = 0; jj < keys.size(); ++jj) { + ++sent; + RAY_LOG(INFO) << "S AsyncDelete: " << n_keys[jj]; + store_client_->AsyncDelete("N", + n_keys[jj], + {[&finished, n_keys, jj](auto b) mutable { + RAY_LOG(INFO) + << "F AsyncDelete: " << n_keys[jj]; + ++finished; + ASSERT_FALSE(b); + }, + *this->io_service_pool_->Get()}); + + ++sent; + RAY_LOG(INFO) << "S AsyncExists: " << p_keys[jj]; + store_client_->AsyncExists("N", + p_keys[jj], + {[&finished, p_keys, jj](auto b) mutable { + RAY_LOG(INFO) + << "F AsyncExists: " << p_keys[jj]; + ++finished; + ASSERT_TRUE(b); + }, + *this->io_service_pool_->Get()}); + } + } else { + ++sent; + RAY_LOG(INFO) << "S AsyncBatchDelete: " << absl::StrJoin(p_keys, ","); + store_client_->AsyncBatchDelete( + "N", + p_keys, + {[&finished, p_keys, keys](auto n) mutable { + RAY_LOG(INFO) << "F AsyncBatchDelete: " << absl::StrJoin(p_keys, ","); + ++finished; + ASSERT_EQ(n, keys.size()); + }, + *this->io_service_pool_->Get()}); + + for (auto p_key : p_keys) { + ++sent; + RAY_LOG(INFO) << "S AsyncExists: " << p_key; + store_client_->AsyncExists("N", + p_key, + {[&finished, p_key](auto b) mutable { + RAY_LOG(INFO) << "F AsyncExists: " << p_key; + ++finished; + ASSERT_FALSE(false); + }, + *this->io_service_pool_->Get()}); + } + } + }, + *io_service_pool_->Get()}); } ASSERT_TRUE(WaitForCondition( [&finished, &sent]() { @@ -279,16 +257,15 @@ TEST_F(RedisStoreClientTest, Random) { } RAY_LOG(INFO) << "m_multi_get Sending: " << idx; *counter += 1; - RAY_CHECK_OK(store_client_->AsyncMultiGet("N", - keys, - {[result, idx, counter](auto m) mutable { - RAY_LOG(INFO) - << "m_multi_get Finished: " << idx - << " " << m.size(); - *counter -= 1; - ASSERT_TRUE(m == result); - }, - *io_service_pool_->Get()})); + store_client_->AsyncMultiGet("N", + keys, + {[result, idx, counter](auto m) mutable { + RAY_LOG(INFO) << "m_multi_get Finished: " << idx + << " " << m.size(); + *counter -= 1; + ASSERT_TRUE(m == result); + }, + *io_service_pool_->Get()}); }; auto m_batch_delete = [&, counter, this](size_t idx) mutable { @@ -299,15 +276,15 @@ TEST_F(RedisStoreClientTest, Random) { } RAY_LOG(INFO) << "m_batch_delete Sending: " << idx; *counter += 1; - RAY_CHECK_OK(store_client_->AsyncBatchDelete( - "N", - keys, - {[&counter, deleted_num, idx](auto v) mutable { - RAY_LOG(INFO) << "m_batch_delete Finished: " << idx << " " << v; - *counter -= 1; - ASSERT_EQ(v, deleted_num); - }, - *io_service_pool_->Get()})); + store_client_->AsyncBatchDelete("N", + keys, + {[&counter, deleted_num, idx](auto v) mutable { + RAY_LOG(INFO) << "m_batch_delete Finished: " << idx + << " " << v; + *counter -= 1; + ASSERT_EQ(v, deleted_num); + }, + *io_service_pool_->Get()}); }; auto m_delete = [&, this](size_t idx) mutable { @@ -315,16 +292,15 @@ TEST_F(RedisStoreClientTest, Random) { bool deleted = dict.erase(k) > 0; RAY_LOG(INFO) << "m_delete Sending: " << idx << " " << k; *counter += 1; - RAY_CHECK_OK(store_client_->AsyncDelete("N", - k, - {[counter, k, idx, deleted](auto r) { - RAY_LOG(INFO) - << "m_delete Finished: " << idx << " " - << k << " " << deleted; - *counter -= 1; - ASSERT_EQ(deleted, r); - }, - *io_service_pool_->Get()})); + store_client_->AsyncDelete("N", + k, + {[counter, k, idx, deleted](auto r) { + RAY_LOG(INFO) << "m_delete Finished: " << idx << " " + << k << " " << deleted; + *counter -= 1; + ASSERT_EQ(deleted, r); + }, + *io_service_pool_->Get()}); }; auto m_get = [&, counter, this](size_t idx) { @@ -335,16 +311,15 @@ TEST_F(RedisStoreClientTest, Random) { } RAY_LOG(INFO) << "m_get Sending: " << idx; *counter += 1; - RAY_CHECK_OK(store_client_->AsyncGet("N", - k, - {[counter, idx, v](auto, auto r) { - RAY_LOG(INFO) - << "m_get Finished: " << idx << " " - << (r ? *r : std::string("-")); - *counter -= 1; - ASSERT_EQ(v, r); - }, - *io_service_pool_->Get()})); + store_client_->AsyncGet("N", + k, + {[counter, idx, v](auto, auto r) { + RAY_LOG(INFO) << "m_get Finished: " << idx << " " + << (r ? *r : std::string("-")); + *counter -= 1; + ASSERT_EQ(v, r); + }, + *io_service_pool_->Get()}); }; auto m_exists = [&, counter, this](size_t idx) { @@ -352,15 +327,15 @@ TEST_F(RedisStoreClientTest, Random) { bool existed = dict.count(k); RAY_LOG(INFO) << "m_exists Sending: " << idx; *counter += 1; - RAY_CHECK_OK(store_client_->AsyncExists( - "N", - k, - {[k, existed, counter, idx](auto r) mutable { - RAY_LOG(INFO) << "m_exists Finished: " << idx << " " << k << " " << r; - *counter -= 1; - ASSERT_EQ(existed, r) << " exists check " << k; - }, - *io_service_pool_->Get()})); + store_client_->AsyncExists("N", + k, + {[k, existed, counter, idx](auto r) mutable { + RAY_LOG(INFO) << "m_exists Finished: " << idx << " " + << k << " " << r; + *counter -= 1; + ASSERT_EQ(existed, r) << " exists check " << k; + }, + *io_service_pool_->Get()}); }; auto m_puts = [&, counter, this](size_t idx) mutable { @@ -373,18 +348,17 @@ TEST_F(RedisStoreClientTest, Random) { dict[k] = v; RAY_LOG(INFO) << "m_put Sending: " << idx << " " << k << " " << v; *counter += 1; - RAY_CHECK_OK(store_client_->AsyncPut("N", - k, - v, - true, - {[idx, added, k, counter](bool r) mutable { - RAY_LOG(INFO) - << "m_put Finished: " - << " " << idx << " " << k << " " << r; - *counter -= 1; - ASSERT_EQ(r, added); - }, - *io_service_pool_->Get()})); + store_client_->AsyncPut("N", + k, + v, + true, + {[idx, added, k, counter](bool r) mutable { + RAY_LOG(INFO) + << "m_put Finished: " << idx << " " << k << " " << r; + *counter -= 1; + ASSERT_EQ(r, added); + }, + *io_service_pool_->Get()}); }; std::vector> ops{ diff --git a/src/ray/gcs/store_client/tests/store_client_test_base.h b/src/ray/gcs/store_client/tests/store_client_test_base.h index c687a7d96e53..5bbc27432708 100644 --- a/src/ray/gcs/store_client/tests/store_client_test_base.h +++ b/src/ray/gcs/store_client/tests/store_client_test_base.h @@ -60,11 +60,11 @@ class StoreClientTestBase : public ::testing::Test { auto put_callback = [this](auto) { --pending_count_; }; for (const auto &[key, value] : key_to_value_) { ++pending_count_; - RAY_CHECK_OK(store_client_->AsyncPut(table_name_, - key.Hex(), - value.SerializeAsString(), - true, - {put_callback, *io_service_pool_->Get()})); + store_client_->AsyncPut(table_name_, + key.Hex(), + value.SerializeAsString(), + true, + {put_callback, *io_service_pool_->Get()}); } WaitPendingDone(); } @@ -73,8 +73,8 @@ class StoreClientTestBase : public ::testing::Test { auto delete_callback = [this](auto) { --pending_count_; }; for (const auto &[key, _] : key_to_value_) { ++pending_count_; - RAY_CHECK_OK(store_client_->AsyncDelete( - table_name_, key.Hex(), {delete_callback, *io_service_pool_->Get()})); + store_client_->AsyncDelete( + table_name_, key.Hex(), {delete_callback, *io_service_pool_->Get()}); } WaitPendingDone(); } @@ -93,8 +93,8 @@ class StoreClientTestBase : public ::testing::Test { }; for (const auto &[key, _] : key_to_value_) { ++pending_count_; - RAY_CHECK_OK(store_client_->AsyncGet( - table_name_, key.Hex(), {get_callback, *io_service_pool_->Get()})); + store_client_->AsyncGet( + table_name_, key.Hex(), {get_callback, *io_service_pool_->Get()}); } WaitPendingDone(); } @@ -110,8 +110,7 @@ class StoreClientTestBase : public ::testing::Test { }; ++pending_count_; - RAY_CHECK_OK(store_client_->AsyncGet( - table_name_, key, {get_callback, *io_service_pool_->Get()})); + store_client_->AsyncGet(table_name_, key, {get_callback, *io_service_pool_->Get()}); } WaitPendingDone(); } @@ -135,8 +134,7 @@ class StoreClientTestBase : public ::testing::Test { }; pending_count_ += key_to_value_.size(); - RAY_CHECK_OK(store_client_->AsyncGetAll( - table_name_, {get_all_callback, *io_service_pool_->Get()})); + store_client_->AsyncGetAll(table_name_, {get_all_callback, *io_service_pool_->Get()}); WaitPendingDone(); } @@ -162,8 +160,8 @@ class StoreClientTestBase : public ::testing::Test { pending_count_ += result_set.size(); - RAY_CHECK_OK(store_client_->AsyncGetKeys( - table_name_, prefix, {get_keys_callback, *io_service_pool_->Get()})); + store_client_->AsyncGetKeys( + table_name_, prefix, {get_keys_callback, *io_service_pool_->Get()}); WaitPendingDone(); } } @@ -176,8 +174,8 @@ class StoreClientTestBase : public ::testing::Test { pending_count_ += key_to_value_.size(); for (const auto &item : key_to_value_) { - RAY_CHECK_OK(store_client_->AsyncExists( - table_name_, item.first.Hex(), {exists_callback, *io_service_pool_->Get()})); + store_client_->AsyncExists( + table_name_, item.first.Hex(), {exists_callback, *io_service_pool_->Get()}); } WaitPendingDone(); } @@ -189,8 +187,8 @@ class StoreClientTestBase : public ::testing::Test { for (auto &[key, _] : key_to_value_) { keys.push_back(key.Hex()); } - RAY_CHECK_OK(store_client_->AsyncBatchDelete( - table_name_, keys, {delete_callback, *io_service_pool_->Get()})); + store_client_->AsyncBatchDelete( + table_name_, keys, {delete_callback, *io_service_pool_->Get()}); WaitPendingDone(); } From bf1909dceccb3fda196d80d14d31c0fb4bfe6fc2 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Fri, 29 Aug 2025 14:05:39 -0700 Subject: [PATCH 366/634] [release test] always use images from previous pipeline steps (#56029) never try to build on one's own --------- Signed-off-by: Lonnie Liu --- .buildkite/release/test.rayci.yml | 6 +- release/ray_release/byod/build.py | 111 ++----------------- release/ray_release/tests/test_byod_build.py | 30 ++--- 3 files changed, 23 insertions(+), 124 deletions(-) diff --git a/.buildkite/release/test.rayci.yml b/.buildkite/release/test.rayci.yml index 4faa7cf05fca..dd7d235780b0 100644 --- a/.buildkite/release/test.rayci.yml +++ b/.buildkite/release/test.rayci.yml @@ -8,4 +8,8 @@ steps: commands: - /bin/bash .buildkite/release/test-init.sh mount_buildkite_agent: true - depends_on: forge + depends_on: + - forge + - anyscalebuild + - anyscalellmbuild + - anyscalemlbuild diff --git a/release/ray_release/byod/build.py b/release/ray_release/byod/build.py index 62ae1d62f3b3..882a73c24859 100644 --- a/release/ray_release/byod/build.py +++ b/release/ray_release/byod/build.py @@ -1,11 +1,8 @@ from typing import List, Optional, Dict -import boto3 -import hashlib import os import subprocess import sys -import time from ray_release.config import RELEASE_PACKAGE_DIR from ray_release.logger import logger @@ -15,19 +12,11 @@ bazel_workspace_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", "") -DATAPLANE_S3_BUCKET = "ray-release-automation-results" -DATAPLANE_FILENAME = "dataplane_20250624.tar.gz" -DATAPLANE_DIGEST = "3cffb55f1a56f0bc6256cbf1a38bf1e764e202a647a4272b80531760f1250059" -BASE_IMAGE_WAIT_TIMEOUT = 7200 -BASE_IMAGE_WAIT_DURATION = 30 RELEASE_BYOD_DIR = ( os.path.join(bazel_workspace_dir, "release/ray_release/byod") if bazel_workspace_dir else os.path.join(RELEASE_PACKAGE_DIR, "ray_release/byod") ) -REQUIREMENTS_BYOD = "requirements_byod" -REQUIREMENTS_LLM_BYOD = "requirements_llm_byod" -REQUIREMENTS_ML_BYOD = "requirements_ml_byod" def build_anyscale_custom_byod_image( @@ -60,85 +49,22 @@ def build_anyscale_custom_byod_image( _validate_and_push(image) -def build_anyscale_base_byod_images(tests: List[Test]) -> None: +def build_anyscale_base_byod_images(tests: List[Test]) -> List[str]: """ Builds the Anyscale BYOD images for the given tests. """ - _download_dataplane_build_file() - to_be_built = {} - built = set() + images = set() for test in tests: - to_be_built[test.get_anyscale_base_byod_image()] = test + images.add(test.get_anyscale_base_byod_image()) - env = os.environ.copy() - env["DOCKER_BUILDKIT"] = "1" - start = int(time.time()) - # ray images are built on post-merge, so we can wait for them to be available - while ( - len(built) < len(to_be_built) - and int(time.time()) - start < BASE_IMAGE_WAIT_TIMEOUT - ): - for byod_image, test in to_be_built.items(): - py_version = test.get_python_version() - if test.use_byod_ml_image(): - byod_requirements = f"{REQUIREMENTS_ML_BYOD}_{py_version}.txt" - elif test.use_byod_llm_image(): - byod_requirements = f"{REQUIREMENTS_LLM_BYOD}_{py_version}.txt" - else: - byod_requirements = f"{REQUIREMENTS_BYOD}_{py_version}.txt" - - if _image_exist(byod_image): - logger.info(f"Image {byod_image} already exists") - built.add(byod_image) - continue - ray_image = test.get_ray_image() - if not _image_exist(ray_image): - # TODO(can): instead of waiting for the base image to be built, we can - # build it ourselves - timeout = BASE_IMAGE_WAIT_TIMEOUT - (int(time.time()) - start) - logger.info( - f"Image {ray_image} does not exist yet. " - f"Wait for another {timeout}s..." - ) - time.sleep(BASE_IMAGE_WAIT_DURATION) - continue - logger.info(f"Building {byod_image} from {ray_image}") - with open(DATAPLANE_FILENAME, "rb") as build_file: - subprocess.check_call( - [ - "docker", - "build", - "--progress=plain", - "--build-arg", - f"BASE_IMAGE={ray_image}", - "-t", - byod_image, - "-", - ], - stdin=build_file, - stdout=sys.stderr, - env=env, - ) - subprocess.check_call( - [ - "docker", - "build", - "--progress=plain", - "--build-arg", - f"BASE_IMAGE={byod_image}", - "--build-arg", - f"PIP_REQUIREMENTS={byod_requirements}", - "-t", - byod_image, - "-f", - os.path.join(RELEASE_BYOD_DIR, "byod.Dockerfile"), - RELEASE_BYOD_DIR, - ], - stdout=sys.stderr, - env=env, - ) - _validate_and_push(byod_image) - built.add(byod_image) + image_list = list(images) + image_list.sort() + + for image in image_list: + if not _image_exist(image): + raise RuntimeError(f"Image {image} not found") + + return image_list def _validate_and_push(byod_image: str) -> None: @@ -189,21 +115,6 @@ def _get_ray_commit(envs: Optional[Dict[str, str]] = None) -> str: return "" -def _download_dataplane_build_file() -> None: - """ - Downloads the dataplane build file from S3. - """ - s3 = boto3.client("s3") - s3.download_file( - Bucket=DATAPLANE_S3_BUCKET, - Key=DATAPLANE_FILENAME, - Filename=DATAPLANE_FILENAME, - ) - with open(DATAPLANE_FILENAME, "rb") as build_context: - digest = hashlib.sha256(build_context.read()).hexdigest() - assert digest == DATAPLANE_DIGEST, "Mismatched dataplane digest found!" - - def _image_exist(image: str) -> bool: """ Checks if the given image exists in Docker diff --git a/release/ray_release/tests/test_byod_build.py b/release/ray_release/tests/test_byod_build.py index 8548a309632f..039d302182a0 100644 --- a/release/ray_release/tests/test_byod_build.py +++ b/release/ray_release/tests/test_byod_build.py @@ -10,7 +10,6 @@ from ray_release.byod.build import ( build_anyscale_custom_byod_image, build_anyscale_base_byod_images, - DATAPLANE_FILENAME, _get_ray_commit, ) @@ -42,10 +41,6 @@ def test_get_ray_commit() -> None: init_global_config(bazel_runfile("release/ray_release/configs/oss_config.yaml")) -# Create a mock file to simulate the S3 download -with open(DATAPLANE_FILENAME, "wb") as f: - f.write(b"abc123") - def test_build_anyscale_custom_byod_image() -> None: cmds = [] @@ -83,28 +78,17 @@ def _mock_check_call( def test_build_anyscale_base_byod_images() -> None: - images = [] - - def _mock_validate_and_push(image: str) -> None: - images.append(image) - def _mock_image_exist(image: str) -> bool: - return "rayproject/ray" in image + return True with patch( - "ray_release.byod.build._download_dataplane_build_file", return_value=None - ), patch( "os.environ", { "BUILDKITE_COMMIT": "abc123", "RAYCI_BUILD_ID": "a1b2c3d4", }, - ), patch( - "subprocess.check_call", return_value=None - ), patch( + ), patch("subprocess.check_call", return_value=None), patch( "ray_release.byod.build._image_exist", side_effect=_mock_image_exist - ), patch( - "ray_release.byod.build._validate_and_push", side_effect=_mock_validate_and_push ): tests = [ Test(name="aws", env="aws", cluster={"byod": {}}), @@ -128,18 +112,18 @@ def _mock_image_exist(image: str) -> bool: ), Test(name="gce", env="gce", cluster={"byod": {}}), ] - build_anyscale_base_byod_images(tests) + images = build_anyscale_base_byod_images(tests) global_config = get_global_config() aws_cr = global_config["byod_aws_cr"] gcp_cr = global_config["byod_gcp_cr"] - assert images == [ + assert set(images) == { f"{aws_cr}/anyscale/ray:a1b2c3d4-py39-cpu", - f"{aws_cr}/anyscale/ray-ml:a1b2c3d4-py39-gpu", - f"{aws_cr}/anyscale/ray:a1b2c3d4-py39-cu121", f"{aws_cr}/anyscale/ray:a1b2c3d4-py39-cu116", + f"{aws_cr}/anyscale/ray:a1b2c3d4-py39-cu121", f"{aws_cr}/anyscale/ray:a1b2c3d4-py311-cu118", + f"{aws_cr}/anyscale/ray-ml:a1b2c3d4-py39-gpu", f"{gcp_cr}/anyscale/ray:a1b2c3d4-py39-cpu", - ] + } if __name__ == "__main__": From f949323c67bd0107b9601bdee7becc0431fb12ed Mon Sep 17 00:00:00 2001 From: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Date: Fri, 29 Aug 2025 16:24:39 -0500 Subject: [PATCH 367/634] [Serve.llm][dashboard] Add prefix cache hit rate to Serve LLM dashboard (#55675) Signed-off-by: Seiji Eicher --- .../dashboards/serve_llm_dashboard_panels.py | 76 +++++++++++-------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/python/ray/dashboard/modules/metrics/dashboards/serve_llm_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/serve_llm_dashboard_panels.py index dff60f13a117..cc876bd17771 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/serve_llm_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/serve_llm_dashboard_panels.py @@ -274,6 +274,43 @@ stack=False, grid_pos=GridPos(12, 40, 12, 8), ), + Panel( + id=28, + title="vLLM: Prefix Cache Hit Rate", + description="Percentage of prefix cache queries that resulted in a cache hit (GPU).", + unit="percentunit", + targets=[ + Target( + expr='increase(ray_vllm:gpu_prefix_cache_hits_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", {global_filters}}}[30s]) / increase(ray_vllm:gpu_prefix_cache_queries_total{{model_name=~"$vllm_model_name", WorkerId=~"$workerid", {global_filters}}}[30s])', + legend="GPU: {{model_name}} - {{WorkerId}}", + ), + ], + fill=1, + linewidth=2, + stack=False, + grid_pos=GridPos(0, 48, 12, 8), + ), + Panel( + id=27, + title="Tokens Per Request Per Model Last 7 Days", + description="", + unit="Tokens", + targets=[ + Target( + expr='sum by (model_name) (delta(ray_vllm:prompt_tokens_total{{WorkerId=~"$workerid", {global_filters}}}[1w])) / sum by (model_name) (delta(ray_vllm:request_success_total{{WorkerId=~"$workerid", {global_filters}}}[1w]))', + legend="In: {{ model_name}}", + ), + Target( + expr='sum by (model_name) (delta(ray_vllm:generation_tokens_total{{WorkerId=~"$workerid", {global_filters}}}[1w])) / sum by (model_name) (delta(ray_vllm:request_success_total{{WorkerId=~"$workerid", {global_filters}}}[1w]))', + legend="Out: {{ model_name}}", + ), + ], + fill=1, + linewidth=2, + stack=False, + grid_pos=GridPos(12, 48, 12, 8), + template=PanelTemplate.GAUGE, + ), Panel( id=14, title="Tokens Last 24 Hours", @@ -292,7 +329,7 @@ fill=1, linewidth=2, stack=False, - grid_pos=GridPos(0, 48, 12, 8), + grid_pos=GridPos(0, 56, 12, 8), template=PanelTemplate.STAT, ), Panel( @@ -313,7 +350,7 @@ fill=1, linewidth=2, stack=False, - grid_pos=GridPos(12, 48, 12, 8), + grid_pos=GridPos(12, 56, 12, 8), template=PanelTemplate.STAT, ), Panel( @@ -330,7 +367,7 @@ fill=1, linewidth=2, stack=False, - grid_pos=GridPos(12, 56, 12, 8), + grid_pos=GridPos(0, 64, 12, 8), template=PanelTemplate.PIE_CHART, ), Panel( @@ -347,7 +384,7 @@ fill=1, linewidth=2, stack=False, - grid_pos=GridPos(0, 64, 12, 8), + grid_pos=GridPos(12, 64, 12, 8), template=PanelTemplate.STAT, ), Panel( @@ -364,7 +401,7 @@ fill=1, linewidth=2, stack=False, - grid_pos=GridPos(12, 64, 12, 8), + grid_pos=GridPos(0, 72, 12, 8), template=PanelTemplate.STAT, ), Panel( @@ -381,7 +418,7 @@ fill=1, linewidth=2, stack=False, - grid_pos=GridPos(0, 72, 12, 8), + grid_pos=GridPos(12, 72, 12, 8), template=PanelTemplate.STAT, ), Panel( @@ -398,7 +435,7 @@ fill=1, linewidth=2, stack=False, - grid_pos=GridPos(12, 72, 12, 8), + grid_pos=GridPos(0, 80, 12, 8), template=PanelTemplate.GAUGE, ), Panel( @@ -415,7 +452,7 @@ fill=1, linewidth=2, stack=False, - grid_pos=GridPos(0, 80, 12, 8), + grid_pos=GridPos(12, 80, 12, 8), template=PanelTemplate.GAUGE, ), Panel( @@ -432,7 +469,7 @@ fill=1, linewidth=2, stack=False, - grid_pos=GridPos(12, 80, 12, 8), + grid_pos=GridPos(0, 88, 12, 8), template=PanelTemplate.GAUGE, ), Panel( @@ -453,27 +490,6 @@ fill=1, linewidth=2, stack=False, - grid_pos=GridPos(0, 88, 12, 8), - template=PanelTemplate.GAUGE, - ), - Panel( - id=27, - title="Tokens Per Request Per Model Last 7 Days", - description="", - unit="Tokens", - targets=[ - Target( - expr='sum by (model_name) (delta(ray_vllm:prompt_tokens_total{{WorkerId=~"$workerid", {global_filters}}}[1w])) / sum by (model_name) (delta(ray_vllm:request_success_total{{WorkerId=~"$workerid", {global_filters}}}[1w]))', - legend="In: {{ model_name}}", - ), - Target( - expr='sum by (model_name) (delta(ray_vllm:generation_tokens_total{{WorkerId=~"$workerid", {global_filters}}}[1w])) / sum by (model_name) (delta(ray_vllm:request_success_total{{WorkerId=~"$workerid", {global_filters}}}[1w]))', - legend="Out: {{ model_name}}", - ), - ], - fill=1, - linewidth=2, - stack=False, grid_pos=GridPos(12, 88, 12, 8), template=PanelTemplate.GAUGE, ), From cc77ec1396770520a3009612f23daea3270f58a7 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Sat, 30 Aug 2025 02:59:59 +0530 Subject: [PATCH 368/634] [Core] Enable ruff for python/ray/dag/ (#56075) Signed-off-by: czgdp1807 --- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- python/ray/dag/class_node.py | 13 +-- python/ray/dag/collective_node.py | 6 +- python/ray/dag/compiled_dag_node.py | 106 +++++++++--------- python/ray/dag/conftest.py | 1 + python/ray/dag/context.py | 3 +- python/ray/dag/dag_node.py | 30 +++-- python/ray/dag/dag_node_operation.py | 10 +- python/ray/dag/dag_operation_future.py | 6 +- python/ray/dag/function_node.py | 1 - python/ray/dag/input_node.py | 2 +- python/ray/dag/output_node.py | 4 +- python/ray/dag/py_obj_scanner.py | 4 +- .../ray/dag/tests/experimental/actor_defs.py | 5 +- .../tests/experimental/test_collective_dag.py | 3 +- .../experimental/test_compiled_graphs.py | 15 +-- .../experimental/test_cpu_communicator_dag.py | 8 +- .../experimental/test_dag_error_handling.py | 12 +- .../experimental/test_dag_visualization.py | 9 +- .../experimental/test_execution_schedule.py | 18 +-- .../test_execution_schedule_gpu.py | 8 +- .../experimental/test_mocked_nccl_dag.py | 7 +- .../tests/experimental/test_multi_args_gpu.py | 4 +- .../tests/experimental/test_multi_node_dag.py | 10 +- .../experimental/test_torch_tensor_dag.py | 22 ++-- .../test_torch_tensor_transport.py | 11 +- python/ray/dag/tests/test_input_node.py | 7 +- python/ray/dag/tests/test_output_node.py | 2 +- python/ray/dag/tests/test_plot.py | 3 +- python/ray/dag/tests/test_py_obj_scanner.py | 5 +- python/ray/dag/utils.py | 8 +- python/ray/dag/vis_utils.py | 3 +- 33 files changed, 172 insertions(+), 178 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a19234ed517..97f83258dffb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: args: [ --fix, --exit-non-zero-on-fix ] - id: ruff args: [ --select, "I", --fix, --exit-non-zero-on-fix ] - files: '^python/ray/serve/|^python/ray/train|^python/ray/data|^python/ray/_private/|^python/ray/llm/|^python/ray/tune/' + files: '^python/ray/serve/|^python/ray/train|^python/ray/data|^python/ray/_private/|^python/ray/llm/|^python/ray/tune/|^python/ray/dag/' - repo: https://github.com/jsh9/pydoclint rev: "0.6.6" diff --git a/pyproject.toml b/pyproject.toml index 5d803303c12a..fde27ee07b05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ afterray = ["psutil", "setproctitle"] "python/ray/__init__.py" = ["I"] "python/ray/setup-dev.py" = ["I"] "python/ray/cloudpickle/*" = ["I"] -"python/ray/dag/*.py" = ["I"] +"python/ray/dag/__init__.py" = ["I"] "python/ray/includes/*" = ["I"] "python/ray/internal/*" = ["I"] "python/ray/ray_operator/*" = ["I"] diff --git a/python/ray/dag/class_node.py b/python/ray/dag/class_node.py index 63d29086d34a..1a5b78e8e706 100644 --- a/python/ray/dag/class_node.py +++ b/python/ray/dag/class_node.py @@ -1,19 +1,18 @@ +from typing import Any, Dict, List, Optional, Tuple, Union from weakref import ReferenceType import ray -from ray.dag.dag_node import DAGNode -from ray.dag.input_node import InputNode -from ray.dag.format_utils import get_dag_node_str from ray.dag.constants import ( - PARENT_CLASS_NODE_KEY, - PREV_CLASS_METHOD_CALL_KEY, BIND_INDEX_KEY, IS_CLASS_METHOD_OUTPUT_KEY, + PARENT_CLASS_NODE_KEY, + PREV_CLASS_METHOD_CALL_KEY, ) +from ray.dag.dag_node import DAGNode +from ray.dag.format_utils import get_dag_node_str +from ray.dag.input_node import InputNode from ray.util.annotations import DeveloperAPI -from typing import Any, Dict, List, Union, Tuple, Optional - @DeveloperAPI class ClassNode(DAGNode): diff --git a/python/ray/dag/collective_node.py b/python/ray/dag/collective_node.py index ad55b8c1a08c..03609b20cc2e 100644 --- a/python/ray/dag/collective_node.py +++ b/python/ray/dag/collective_node.py @@ -1,21 +1,21 @@ -from typing import Any, Dict, List, Union, Tuple, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union if TYPE_CHECKING: import torch import ray from ray.dag import ( - DAGNode, ClassMethodNode, + DAGNode, ) from ray.dag.constants import COLLECTIVE_OPERATION_KEY, IS_CLASS_METHOD_OUTPUT_KEY from ray.experimental.channel import ChannelContext from ray.experimental.channel.torch_tensor_type import Communicator, TorchTensorType from ray.experimental.util.types import ( - _CollectiveOp, AllGatherOp, AllReduceOp, ReduceScatterOp, + _CollectiveOp, ) from ray.util.annotations import DeveloperAPI diff --git a/python/ray/dag/compiled_dag_node.py b/python/ray/dag/compiled_dag_node.py index ce59c2c244da..2204cb40ab7b 100644 --- a/python/ray/dag/compiled_dag_node.py +++ b/python/ray/dag/compiled_dag_node.py @@ -1,89 +1,84 @@ -import weakref import asyncio +import logging +import threading +import time +import traceback +import uuid +import weakref from collections import defaultdict from contextlib import nullcontext -from dataclasses import dataclass, asdict +from dataclasses import asdict, dataclass from typing import ( Any, Dict, List, - Tuple, - Union, Optional, Set, + Tuple, + Union, ) -import logging -import threading -import time -import uuid -import traceback -from ray.experimental.channel.auto_transport_type import ( - AutoTransportType, - TypeHintResolver, -) +import ray import ray.exceptions -from ray.dag.dag_operation_future import GPUFuture, DAGOperationFuture, ResolvedFuture -from ray.experimental.channel.cached_channel import CachedChannel -from ray.experimental.channel.communicator import Communicator from ray.dag.constants import ( RAY_CGRAPH_ENABLE_NVTX_PROFILING, RAY_CGRAPH_ENABLE_TORCH_PROFILING, RAY_CGRAPH_VISUALIZE_SCHEDULE, ) -import ray +from ray.dag.dag_node_operation import ( + _build_dag_node_operation_graph, + _DAGNodeOperation, + _DAGNodeOperationType, + _DAGOperationGraphNode, + _extract_execution_schedule, + _generate_actor_to_execution_schedule, + _generate_overlapped_execution_schedule, + _visualize_execution_schedule, +) +from ray.dag.dag_operation_future import DAGOperationFuture, GPUFuture, ResolvedFuture from ray.exceptions import ( RayCgraphCapacityExceeded, - RayTaskError, RayChannelError, RayChannelTimeoutError, -) -from ray.experimental.compiled_dag_ref import ( - CompiledDAGRef, - CompiledDAGFuture, - _process_return_vals, + RayTaskError, ) from ray.experimental.channel import ( + AwaitableBackgroundReader, + AwaitableBackgroundWriter, ChannelContext, ChannelInterface, ChannelOutputType, - ReaderInterface, - SynchronousReader, - WriterInterface, - SynchronousWriter, - AwaitableBackgroundReader, - AwaitableBackgroundWriter, CompiledDAGArgs, CompositeChannel, IntraProcessChannel, + ReaderInterface, + SynchronousReader, + SynchronousWriter, + WriterInterface, ) -from ray.util.annotations import DeveloperAPI - +from ray.experimental.channel.accelerator_context import AcceleratorContext +from ray.experimental.channel.auto_transport_type import ( + AutoTransportType, + TypeHintResolver, +) +from ray.experimental.channel.cached_channel import CachedChannel +from ray.experimental.channel.communicator import Communicator from ray.experimental.channel.shared_memory_channel import ( SharedMemoryType, ) -from ray.experimental.channel.torch_tensor_type import TorchTensorType - from ray.experimental.channel.torch_tensor_accelerator_channel import ( - _init_communicator, _destroy_communicator, + _init_communicator, ) - -from ray.dag.dag_node_operation import ( - _DAGNodeOperation, - _DAGNodeOperationType, - _DAGOperationGraphNode, - _build_dag_node_operation_graph, - _extract_execution_schedule, - _generate_actor_to_execution_schedule, - _generate_overlapped_execution_schedule, - _visualize_execution_schedule, +from ray.experimental.channel.torch_tensor_type import TorchTensorType +from ray.experimental.compiled_dag_ref import ( + CompiledDAGFuture, + CompiledDAGRef, + _process_return_vals, ) - +from ray.util.annotations import DeveloperAPI from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy -from ray.experimental.channel.accelerator_context import AcceleratorContext - logger = logging.getLogger(__name__) # Keep tracking of every compiled dag created during the lifetime of @@ -370,6 +365,7 @@ def _device_context_manager(): return nullcontext() import torch + from ray.experimental.channel.accelerator_context import AcceleratorContext device = ChannelContext.get_current().torch_device @@ -1091,9 +1087,9 @@ def _preprocess(self) -> None: This function is idempotent. """ from ray.dag import ( - DAGNode, ClassMethodNode, CollectiveOutputNode, + DAGNode, FunctionNode, InputAttributeNode, InputNode, @@ -1491,8 +1487,8 @@ def _check_leaf_nodes(self) -> None: Check if there are leaf nodes in the DAG and raise an error if there are. """ from ray.dag import ( - DAGNode, ClassMethodNode, + DAGNode, ) leaf_nodes: List[DAGNode] = [] @@ -1565,11 +1561,11 @@ def _get_or_compile( outputs for the DAG. """ from ray.dag import ( + ClassMethodNode, DAGNode, - InputNode, InputAttributeNode, + InputNode, MultiOutputNode, - ClassMethodNode, ) if self.input_task_idx is None: @@ -2789,11 +2785,11 @@ def _visualize_ascii(self) -> str: """ from ray.dag import ( + ClassMethodNode, + DAGNode, InputAttributeNode, InputNode, MultiOutputNode, - ClassMethodNode, - DAGNode, ) # Check that the DAG has been compiled @@ -3097,11 +3093,11 @@ def visualize( "You can install it by running `pip install graphviz`." ) from ray.dag import ( + ClassMethodNode, + DAGNode, InputAttributeNode, InputNode, MultiOutputNode, - ClassMethodNode, - DAGNode, ) # Check that the DAG has been compiled diff --git a/python/ray/dag/conftest.py b/python/ray/dag/conftest.py index a350eb5be2d7..a6a1a22b89a8 100644 --- a/python/ray/dag/conftest.py +++ b/python/ray/dag/conftest.py @@ -1,4 +1,5 @@ import os + import pytest import ray diff --git a/python/ray/dag/context.py b/python/ray/dag/context.py index 37e29521603c..89fb981eb019 100644 --- a/python/ray/dag/context.py +++ b/python/ray/dag/context.py @@ -1,7 +1,8 @@ -from dataclasses import dataclass import os import threading +from dataclasses import dataclass from typing import Optional + from ray.util.annotations import DeveloperAPI # The context singleton on this process. diff --git a/python/ray/dag/dag_node.py b/python/ray/dag/dag_node.py index 8c43a7bf5f22..c2d0f0579df5 100644 --- a/python/ray/dag/dag_node.py +++ b/python/ray/dag/dag_node.py @@ -1,31 +1,29 @@ +import asyncio import copy -from ray.experimental.channel.auto_transport_type import AutoTransportType -from ray.experimental.channel.torch_tensor_type import TorchTensorType -import ray -from ray.dag.base import DAGNodeBase -from ray.dag.py_obj_scanner import _PyObjScanner -from ray.util.annotations import DeveloperAPI - +import uuid from itertools import chain - from typing import ( - Optional, - Union, - List, - Tuple, - Dict, Any, - TypeVar, Callable, + Dict, + List, Literal, + Optional, + Tuple, + TypeVar, + Union, ) -import uuid -import asyncio +import ray +from ray.dag.base import DAGNodeBase from ray.dag.compiled_dag_node import build_compiled_dag_from_ray_dag +from ray.dag.py_obj_scanner import _PyObjScanner from ray.experimental.channel import ChannelOutputType +from ray.experimental.channel.auto_transport_type import AutoTransportType from ray.experimental.channel.communicator import Communicator +from ray.experimental.channel.torch_tensor_type import TorchTensorType from ray.experimental.util.types import Device +from ray.util.annotations import DeveloperAPI T = TypeVar("T") diff --git a/python/ray/dag/dag_node_operation.py b/python/ray/dag/dag_node_operation.py index 52072eec12e9..5a192e9f5da2 100644 --- a/python/ray/dag/dag_node_operation.py +++ b/python/ray/dag/dag_node_operation.py @@ -1,12 +1,12 @@ -from functools import total_ordering -from enum import Enum -from typing import Set, Tuple, List, Dict, Optional import copy -import logging -import ray import heapq +import logging from collections import defaultdict +from enum import Enum +from functools import total_ordering +from typing import Dict, List, Optional, Set, Tuple +import ray logger = logging.getLogger(__name__) diff --git a/python/ray/dag/dag_operation_future.py b/python/ray/dag/dag_operation_future.py index acfc83d7c1d6..392c86286a99 100644 --- a/python/ray/dag/dag_operation_future.py +++ b/python/ray/dag/dag_operation_future.py @@ -1,8 +1,8 @@ from abc import ABC, abstractmethod -from typing import Any, Generic, TypeVar, Dict -from ray.util.annotations import DeveloperAPI -from ray.experimental.channel.accelerator_context import AcceleratorContext +from typing import Any, Dict, Generic, TypeVar +from ray.experimental.channel.accelerator_context import AcceleratorContext +from ray.util.annotations import DeveloperAPI T = TypeVar("T") diff --git a/python/ray/dag/function_node.py b/python/ray/dag/function_node.py index 4565fcffe8ff..b48c63509f2c 100644 --- a/python/ray/dag/function_node.py +++ b/python/ray/dag/function_node.py @@ -1,6 +1,5 @@ from typing import Any, Dict, List - import ray from ray.dag.dag_node import DAGNode from ray.dag.format_utils import get_dag_node_str diff --git a/python/ray/dag/input_node.py b/python/ray/dag/input_node.py index 83f212e4e58f..0386f84cb999 100644 --- a/python/ray/dag/input_node.py +++ b/python/ray/dag/input_node.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Union, Optional +from typing import Any, Dict, List, Optional, Union from ray.dag import DAGNode from ray.dag.format_utils import get_dag_node_str diff --git a/python/ray/dag/output_node.py b/python/ray/dag/output_node.py index f9abdf1643e0..fc0ec1a10026 100644 --- a/python/ray/dag/output_node.py +++ b/python/ray/dag/output_node.py @@ -1,6 +1,6 @@ -import ray -from typing import Any, Dict, List, Union, Tuple +from typing import Any, Dict, List, Tuple, Union +import ray from ray.dag import DAGNode from ray.dag.format_utils import get_dag_node_str from ray.util.annotations import DeveloperAPI diff --git a/python/ray/dag/py_obj_scanner.py b/python/ray/dag/py_obj_scanner.py index 6bd6b94ab535..d86b982c10e1 100644 --- a/python/ray/dag/py_obj_scanner.py +++ b/python/ray/dag/py_obj_scanner.py @@ -1,12 +1,10 @@ import io -from typing import Any, Dict, Generic, List, Tuple, Type, TypeVar, Union - import pickle # noqa: F401 +from typing import Any, Dict, Generic, List, Tuple, Type, TypeVar, Union import ray from ray.dag.base import DAGNodeBase - # Used in deserialization hooks to reference scanner instances. _instances: Dict[int, "_PyObjScanner"] = {} diff --git a/python/ray/dag/tests/experimental/actor_defs.py b/python/ray/dag/tests/experimental/actor_defs.py index 55603ef64268..a0446746bc78 100644 --- a/python/ray/dag/tests/experimental/actor_defs.py +++ b/python/ray/dag/tests/experimental/actor_defs.py @@ -1,7 +1,8 @@ -import ray import os -import time import random +import time + +import ray @ray.remote diff --git a/python/ray/dag/tests/experimental/test_collective_dag.py b/python/ray/dag/tests/experimental/test_collective_dag.py index cc14a3b36dfb..9c426791ec14 100644 --- a/python/ray/dag/tests/experimental/test_collective_dag.py +++ b/python/ray/dag/tests/experimental/test_collective_dag.py @@ -2,7 +2,8 @@ import logging import os import sys -from typing import Callable, List, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Callable, List, Optional, Tuple + import pytest import ray diff --git a/python/ray/dag/tests/experimental/test_compiled_graphs.py b/python/ray/dag/tests/experimental/test_compiled_graphs.py index 4195eb14a481..85b2a5083920 100644 --- a/python/ray/dag/tests/experimental/test_compiled_graphs.py +++ b/python/ray/dag/tests/experimental/test_compiled_graphs.py @@ -5,25 +5,22 @@ import re import sys import time -import numpy as np -import torch +import numpy as np import pytest +import torch - -from ray._private.test_utils import run_string_as_driver -from ray.exceptions import RayChannelTimeoutError import ray import ray._private import ray.cluster_utils -from ray.dag import DAGContext, InputNode, MultiOutputNode -from ray.tests.conftest import * # noqa from ray._common.utils import ( get_or_create_event_loop, ) - +from ray._private.test_utils import run_string_as_driver +from ray.dag import DAGContext, InputNode, MultiOutputNode from ray.dag.tests.experimental.actor_defs import Actor, Collector - +from ray.exceptions import RayChannelTimeoutError +from ray.tests.conftest import * # noqa logger = logging.getLogger(__name__) diff --git a/python/ray/dag/tests/experimental/test_cpu_communicator_dag.py b/python/ray/dag/tests/experimental/test_cpu_communicator_dag.py index 0ec0a2ebd9e3..64a375985069 100644 --- a/python/ray/dag/tests/experimental/test_cpu_communicator_dag.py +++ b/python/ray/dag/tests/experimental/test_cpu_communicator_dag.py @@ -1,16 +1,16 @@ import os import sys -import torch import pytest +import torch import ray import ray.cluster_utils -from ray.exceptions import RayChannelError, RayTaskError -from ray.experimental.channel.cpu_communicator import CPUCommunicator -from ray.dag import InputNode import ray.experimental.collective as collective +from ray.dag import InputNode from ray.dag.output_node import MultiOutputNode +from ray.exceptions import RayChannelError, RayTaskError +from ray.experimental.channel.cpu_communicator import CPUCommunicator from ray.tests.conftest import * # noqa diff --git a/python/ray/dag/tests/experimental/test_dag_error_handling.py b/python/ray/dag/tests/experimental/test_dag_error_handling.py index bb129a444ca3..e0753e2b4e22 100644 --- a/python/ray/dag/tests/experimental/test_dag_error_handling.py +++ b/python/ray/dag/tests/experimental/test_dag_error_handling.py @@ -3,18 +3,16 @@ import logging import pickle import re +import signal import sys import time import pytest - -from ray.exceptions import ActorDiedError, RayChannelError, RayChannelTimeoutError import ray import ray._private import ray.cluster_utils -from ray.dag import DAGContext, InputNode, MultiOutputNode -from ray.tests.conftest import * # noqa +from ray._common.test_utils import SignalActor from ray._common.utils import ( get_or_create_event_loop, ) @@ -22,10 +20,10 @@ run_string_as_driver_nonblocking, wait_for_pid_to_exit, ) -from ray._common.test_utils import SignalActor -import signal - +from ray.dag import DAGContext, InputNode, MultiOutputNode from ray.dag.tests.experimental.actor_defs import Actor +from ray.exceptions import ActorDiedError, RayChannelError, RayChannelTimeoutError +from ray.tests.conftest import * # noqa logger = logging.getLogger(__name__) diff --git a/python/ray/dag/tests/experimental/test_dag_visualization.py b/python/ray/dag/tests/experimental/test_dag_visualization.py index 4278df31a196..c2908ef63f1e 100644 --- a/python/ray/dag/tests/experimental/test_dag_visualization.py +++ b/python/ray/dag/tests/experimental/test_dag_visualization.py @@ -1,12 +1,13 @@ +import os import sys -import ray + import pydot -import os +import pytest + +import ray from ray.dag import InputNode, MultiOutputNode from ray.tests.conftest import * # noqa -import pytest - @pytest.fixture def cleanup_files(): diff --git a/python/ray/dag/tests/experimental/test_execution_schedule.py b/python/ray/dag/tests/experimental/test_execution_schedule.py index 46bd714d7f47..2c6e2a025dae 100644 --- a/python/ray/dag/tests/experimental/test_execution_schedule.py +++ b/python/ray/dag/tests/experimental/test_execution_schedule.py @@ -1,24 +1,24 @@ # coding: utf-8 import os import sys +from typing import Dict, List, Tuple import pytest -from ray.tests.conftest import * # noqa -from ray.dag import InputNode, MultiOutputNode, ClassMethodNode +from ray.actor import ActorHandle +from ray.dag import ClassMethodNode, InputNode, MultiOutputNode +from ray.dag.compiled_dag_node import CompiledTask from ray.dag.dag_node_operation import ( + _add_edge, + _build_dag_node_operation_graph, + _DAGNodeOperation, _DAGNodeOperationType, _DAGOperationGraphNode, - _DAGNodeOperation, _extract_execution_schedule, - _select_next_nodes, - _build_dag_node_operation_graph, - _add_edge, _generate_actor_to_execution_schedule, + _select_next_nodes, ) -from ray.dag.compiled_dag_node import CompiledTask -from typing import List, Dict, Tuple -from ray.actor import ActorHandle +from ray.tests.conftest import * # noqa if sys.platform != "linux" and sys.platform != "darwin": pytest.skip("Skipping, requires Linux or Mac.", allow_module_level=True) diff --git a/python/ray/dag/tests/experimental/test_execution_schedule_gpu.py b/python/ray/dag/tests/experimental/test_execution_schedule_gpu.py index 8bd3a9dbf751..639db895cff3 100644 --- a/python/ray/dag/tests/experimental/test_execution_schedule_gpu.py +++ b/python/ray/dag/tests/experimental/test_execution_schedule_gpu.py @@ -1,17 +1,17 @@ # coding: utf-8 import os import sys +from typing import Optional import pytest +import torch import ray import ray.cluster_utils -from ray.tests.conftest import * # noqa from ray.dag import InputNode, MultiOutputNode -from ray.dag.dag_node_operation import _DAGNodeOperationType -import torch -from typing import Optional from ray.dag.compiled_dag_node import CompiledDAG +from ray.dag.dag_node_operation import _DAGNodeOperationType +from ray.tests.conftest import * # noqa if sys.platform != "linux" and sys.platform != "darwin": pytest.skip("Skipping, requires Linux or Mac.", allow_module_level=True) diff --git a/python/ray/dag/tests/experimental/test_mocked_nccl_dag.py b/python/ray/dag/tests/experimental/test_mocked_nccl_dag.py index ad10e6d53c5f..0a5dae633792 100644 --- a/python/ray/dag/tests/experimental/test_mocked_nccl_dag.py +++ b/python/ray/dag/tests/experimental/test_mocked_nccl_dag.py @@ -2,20 +2,19 @@ import os import sys -from ray._common.test_utils import wait_for_condition -import torch - import pytest +import torch import ray import ray.cluster_utils +from ray._common.test_utils import wait_for_condition +from ray.dag import InputNode from ray.exceptions import RayChannelError, RayTaskError from ray.experimental.channel.conftest import ( Barrier, start_nccl_mock, ) from ray.tests.conftest import * # noqa -from ray.dag import InputNode def error_logged(capsys, msg): diff --git a/python/ray/dag/tests/experimental/test_multi_args_gpu.py b/python/ray/dag/tests/experimental/test_multi_args_gpu.py index 9a746b8b7f03..d0d88432c099 100644 --- a/python/ray/dag/tests/experimental/test_multi_args_gpu.py +++ b/python/ray/dag/tests/experimental/test_multi_args_gpu.py @@ -3,12 +3,12 @@ import sys import pytest +import torch import ray -from ray.dag import InputNode, MultiOutputNode import ray.cluster_utils +from ray.dag import InputNode, MultiOutputNode from ray.tests.conftest import * # noqa -import torch if sys.platform != "linux" and sys.platform != "darwin": pytest.skip("Skipping, requires Linux or Mac.", allow_module_level=True) diff --git a/python/ray/dag/tests/experimental/test_multi_node_dag.py b/python/ray/dag/tests/experimental/test_multi_node_dag.py index 92c38df66cb5..301f187115c9 100644 --- a/python/ray/dag/tests/experimental/test_multi_node_dag.py +++ b/python/ray/dag/tests/experimental/test_multi_node_dag.py @@ -1,14 +1,16 @@ -import random -import ray import os +import random import sys import time + import pytest + +import ray +import ray.remote_function from ray._common.test_utils import wait_for_condition from ray.dag import InputNode, MultiOutputNode -import ray.remote_function -from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy from ray.tests.conftest import * # noqa +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy if sys.platform != "linux" and sys.platform != "darwin": pytest.skip("Skipping, requires Linux or Mac.", allow_module_level=True) diff --git a/python/ray/dag/tests/experimental/test_torch_tensor_dag.py b/python/ray/dag/tests/experimental/test_torch_tensor_dag.py index 24c4d2524ff8..84ceb2d17f43 100644 --- a/python/ray/dag/tests/experimental/test_torch_tensor_dag.py +++ b/python/ray/dag/tests/experimental/test_torch_tensor_dag.py @@ -3,31 +3,31 @@ import os import socket import sys +import time from typing import List, Optional, Tuple import pytest +import torch + import ray import ray.cluster_utils import ray.experimental.collective as collective -import torch -import time +from ray._private.test_utils import ( + get_log_message, + init_log_pubsub, +) from ray.dag import InputNode -from ray.exceptions import RayChannelError, RayTaskError from ray.dag.output_node import MultiOutputNode +from ray.exceptions import RayChannelError, RayTaskError +from ray.experimental.channel.accelerator_context import AcceleratorContext from ray.experimental.channel.communicator import ( Communicator, TorchTensorAllocator, ) -from ray.experimental.channel.torch_tensor_type import TorchTensorType from ray.experimental.channel.nccl_group import _NcclGroup -from ray._private.test_utils import ( - get_log_message, - init_log_pubsub, -) - -from ray.tests.conftest import * # noqa +from ray.experimental.channel.torch_tensor_type import TorchTensorType from ray.experimental.util.types import ReduceOp -from ray.experimental.channel.accelerator_context import AcceleratorContext +from ray.tests.conftest import * # noqa logger = logging.getLogger(__name__) diff --git a/python/ray/dag/tests/experimental/test_torch_tensor_transport.py b/python/ray/dag/tests/experimental/test_torch_tensor_transport.py index 5ec1f2526e75..84722ef2f7db 100644 --- a/python/ray/dag/tests/experimental/test_torch_tensor_transport.py +++ b/python/ray/dag/tests/experimental/test_torch_tensor_transport.py @@ -1,13 +1,14 @@ -import ray import os import sys -import torch -import pytest from typing import Dict + +import pytest +import torch + +import ray from ray.dag import InputNode -from ray.exceptions import RayTaskError +from ray.exceptions import RaySystemError, RayTaskError from ray.tests.conftest import * # noqa -from ray.exceptions import RaySystemError if sys.platform != "linux" and sys.platform != "darwin": pytest.skip("Skipping, requires Linux or Mac.", allow_module_level=True) diff --git a/python/ray/dag/tests/test_input_node.py b/python/ray/dag/tests/test_input_node.py index 6874ff21cfd1..e5b54e6c60fd 100644 --- a/python/ray/dag/tests/test_input_node.py +++ b/python/ray/dag/tests/test_input_node.py @@ -3,12 +3,13 @@ request, for all DAGNode types. """ -import pytest -from ray.dag.dag_node import DAGNode -from ray.dag.input_node import InputNode from typing import Any, TypeVar +import pytest + import ray +from ray.dag.dag_node import DAGNode +from ray.dag.input_node import InputNode RayHandleLike = TypeVar("RayHandleLike") diff --git a/python/ray/dag/tests/test_output_node.py b/python/ray/dag/tests/test_output_node.py index 95890b5b6b30..795e736cdfa7 100644 --- a/python/ray/dag/tests/test_output_node.py +++ b/python/ray/dag/tests/test_output_node.py @@ -1,10 +1,10 @@ import pytest import ray +from ray._common.test_utils import wait_for_condition from ray.dag.input_node import InputNode from ray.dag.output_node import MultiOutputNode from ray.util.state import list_tasks -from ray._common.test_utils import wait_for_condition def test_output_node(shared_ray_instance): diff --git a/python/ray/dag/tests/test_plot.py b/python/ray/dag/tests/test_plot.py index d6e00f14b3ef..d3d1244e3ecf 100644 --- a/python/ray/dag/tests/test_plot.py +++ b/python/ray/dag/tests/test_plot.py @@ -1,8 +1,9 @@ import os -import pytest import sys import tempfile +import pytest + import ray diff --git a/python/ray/dag/tests/test_py_obj_scanner.py b/python/ray/dag/tests/test_py_obj_scanner.py index c07fdd499e38..104e6dc94d8f 100644 --- a/python/ray/dag/tests/test_py_obj_scanner.py +++ b/python/ray/dag/tests/test_py_obj_scanner.py @@ -1,7 +1,8 @@ -import pytest from typing import Any -from ray.dag.py_obj_scanner import _PyObjScanner, _instances +import pytest + +from ray.dag.py_obj_scanner import _instances, _PyObjScanner class Source: diff --git a/python/ray/dag/utils.py b/python/ray/dag/utils.py index ce96b3c27a8a..2fe1f3adf3d7 100644 --- a/python/ray/dag/utils.py +++ b/python/ray/dag/utils.py @@ -1,12 +1,12 @@ from typing import Dict from ray.dag import ( + ClassMethodNode, + ClassNode, DAGNode, - InputNode, - InputAttributeNode, FunctionNode, - ClassNode, - ClassMethodNode, + InputAttributeNode, + InputNode, MultiOutputNode, ) diff --git a/python/ray/dag/vis_utils.py b/python/ray/dag/vis_utils.py index c5a3b5cbc096..1274a53cc20d 100644 --- a/python/ray/dag/vis_utils.py +++ b/python/ray/dag/vis_utils.py @@ -1,8 +1,7 @@ -from ray.dag import DAGNode - import os import tempfile +from ray.dag import DAGNode from ray.dag.utils import _DAGNodeNameGenerator from ray.util.annotations import DeveloperAPI From 1373fd20d3df920fc237281496c5abe4c452d3e8 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Fri, 29 Aug 2025 15:05:16 -0700 Subject: [PATCH 369/634] [ci] raydepsets: removing duplicate utils (#56095) removing duplicate testing utils file Signed-off-by: elliot-barn --- ci/raydepsets/testing_utils.py | 45 ---------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 ci/raydepsets/testing_utils.py diff --git a/ci/raydepsets/testing_utils.py b/ci/raydepsets/testing_utils.py deleted file mode 100644 index 38f595ce7003..000000000000 --- a/ci/raydepsets/testing_utils.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Shared test utilities for raydepsets tests.""" - -import shutil - -import runfiles - -_REPO_NAME = "io_ray" -_runfiles = runfiles.Create() - - -def copy_data_to_tmpdir(tmpdir): - """Copy test data to a temporary directory.""" - shutil.copytree( - _runfiles.Rlocation(f"{_REPO_NAME}/ci/raydepsets/test_data"), - tmpdir, - dirs_exist_ok=True, - ) - - -def replace_in_file(filepath, old, new): - with open(filepath, "r") as f: - contents = f.read() - - contents = contents.replace(old, new) - - with open(filepath, "w") as f: - f.write(contents) - - -def save_packages_to_file(filepath, packages): - with open(filepath, "w") as f: - for package in packages: - f.write(package + "\n") - - -def save_file_as(input_file, output_file): - with open(input_file, "rb") as f: - contents = f.read() - with open(output_file, "wb") as f: - f.write(contents) - - -def append_to_file(filepath, new): - with open(filepath, "a") as f: - f.write(new + "\n") From 75eb3c25bc20813abe3a0948bca718bef1d7e294 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 29 Aug 2025 18:32:29 -0500 Subject: [PATCH 370/634] [core] Remove `gcs_rpc_server.h` (#56062) No longer used; all targets have been moved to `gcs_server/grpc_service_interfaces.h` --------- Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 3 +- src/ray/gcs/gcs_server/gcs_server.h | 2 +- src/ray/rpc/BUILD.bazel | 20 -------------- src/ray/rpc/gcs/gcs_rpc_server.h | 43 ----------------------------- 4 files changed, 2 insertions(+), 66 deletions(-) delete mode 100644 src/ray/rpc/gcs/gcs_rpc_server.h diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 45b348dba6e8..fd3ad8b2c890 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -108,7 +108,6 @@ ray_cc_library( "//src/ray/protobuf:ray_syncer_cc_proto", "//src/ray/raylet/scheduling:cluster_lease_manager", "//src/ray/raylet/scheduling:cluster_resource_manager", - "//src/ray/rpc:gcs_server", "//src/ray/stats:stats_metric", "//src/ray/util:logging", "@com_google_absl//absl/container:flat_hash_map", @@ -524,7 +523,7 @@ ray_cc_library( "//src/ray/raylet/scheduling:scheduler", "//src/ray/raylet_client:raylet_client_lib", "//src/ray/rpc:core_worker_client", - "//src/ray/rpc:gcs_server", + "//src/ray/rpc:grpc_server", "//src/ray/rpc:metrics_agent_client", "//src/ray/rpc:node_manager_client", "//src/ray/util:counter_map", diff --git a/src/ray/gcs/gcs_server/gcs_server.h b/src/ray/gcs/gcs_server/gcs_server.h index 5a4ece5e4b21..fc4ca3d63239 100644 --- a/src/ray/gcs/gcs_server/gcs_server.h +++ b/src/ray/gcs/gcs_server/gcs_server.h @@ -40,7 +40,7 @@ #include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/rpc/client_call.h" -#include "ray/rpc/gcs/gcs_rpc_server.h" +#include "ray/rpc/grpc_server.h" #include "ray/rpc/metrics_agent_client.h" #include "ray/rpc/node_manager/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client_pool.h" diff --git a/src/ray/rpc/BUILD.bazel b/src/ray/rpc/BUILD.bazel index a1bf901ab98f..4260d0065844 100644 --- a/src/ray/rpc/BUILD.bazel +++ b/src/ray/rpc/BUILD.bazel @@ -181,26 +181,6 @@ ray_cc_library( ], ) -ray_cc_library( - name = "gcs_server", - hdrs = [ - "gcs/gcs_rpc_server.h", - ], - visibility = ["//visibility:public"], - deps = [ - ":grpc_server", - ":server_call", - "//src/ray/common:asio", - "//src/ray/common:id", - "//src/ray/common:ray_config", - "//src/ray/protobuf:autoscaler_cc_grpc", - "//src/ray/protobuf:events_event_aggregator_service_cc_grpc", - "//src/ray/protobuf:gcs_service_cc_grpc", - "@boost//:asio", - "@com_github_grpc_grpc//:grpc++", - ], -) - ray_cc_library( name = "object_manager_client", hdrs = [ diff --git a/src/ray/rpc/gcs/gcs_rpc_server.h b/src/ray/rpc/gcs/gcs_rpc_server.h deleted file mode 100644 index 0e797263efda..000000000000 --- a/src/ray/rpc/gcs/gcs_rpc_server.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/id.h" -#include "ray/rpc/grpc_server.h" -#include "ray/rpc/server_call.h" -#include "src/ray/protobuf/autoscaler.grpc.pb.h" -#include "src/ray/protobuf/events_event_aggregator_service.pb.h" -#include "src/ray/protobuf/gcs_service.grpc.pb.h" - -// Most of our RPC templates, if not all, expect messages in the ray::rpc protobuf -// namespace. Since the following two messages are defined under the rpc::events -// namespace, we treat them as if they were part of ray::rpc for compatibility. -using ray::rpc::events::AddEventsReply; -using ray::rpc::events::AddEventsRequest; - -namespace ray { -namespace rpc { - -#define GCS_RPC_SEND_REPLY(send_reply_callback, reply, status) \ - reply->mutable_status()->set_code(static_cast(status.code())); \ - reply->mutable_status()->set_message(status.message()); \ - send_reply_callback(ray::Status::OK(), nullptr, nullptr) - -} // namespace rpc -} // namespace ray From 5d9589438d5dc8f6a85e5d408eb9babce1ea9158 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Fri, 29 Aug 2025 16:39:13 -0700 Subject: [PATCH 371/634] [ci] build & test placeholder wheel (#55594) - Building and testing placeholder wheel --------- Signed-off-by: elliot-barn --- .buildkite/build.rayci.yml | 25 ++++++++++++++ ci/build/test-linux-placeholder-wheel.sh | 42 ++++++++++++++++++++++++ ci/ray_ci/builder_container.py | 1 + python/setup.py | 14 ++++++-- 4 files changed, 79 insertions(+), 3 deletions(-) create mode 100755 ci/build/test-linux-placeholder-wheel.sh diff --git a/.buildkite/build.rayci.yml b/.buildkite/build.rayci.yml index 01256a3b63c8..3ca111f3ad16 100644 --- a/.buildkite/build.rayci.yml +++ b/.buildkite/build.rayci.yml @@ -19,6 +19,31 @@ steps: - manylinux - forge + - label: ":tapioca: build & test placeholder wheel {{matrix}}" + key: placeholder_wheels + tags: + - python + instance_type: medium + commands: + # validate minimal installation + - python ./ci/env/check_minimal_install.py --expected-python-version {{matrix}} + - python -m pip install --upgrade pip + # build placeholder wheel + - export RAY_DEBUG_BUILD=deps-only + - mkdir -p .whl + - pip wheel python/ --no-deps -w .whl/ --use-pep517 + - ls -a .whl + # test placeholder wheel + - ./ci/build/test-linux-placeholder-wheel.sh {{matrix}} + depends_on: + - minbuild-core + job_env: minbuild-core-py{{matrix}} + matrix: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - label: ":tapioca: build: jar" key: java_wheels tags: diff --git a/ci/build/test-linux-placeholder-wheel.sh b/ci/build/test-linux-placeholder-wheel.sh new file mode 100755 index 000000000000..921f7ee05b6d --- /dev/null +++ b/ci/build/test-linux-placeholder-wheel.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -exuo pipefail + +PYTHON="$1" + +if [[ ! "${OSTYPE}" =~ ^linux ]]; then + echo "ERROR: This wheel test script is only for Linux platforms." >/dev/stderr + exit 1 +fi + +PYTHON_VERSION="${PYTHON//./}" + +which python + +which pip + +RAY_PLACEHOLDER_VERSION="100.0.0-dev" +MINIFORGE_BIN_PATH="/opt/miniforge/bin" +PYTHON_EXE="${MINIFORGE_BIN_PATH}/python" +PIP_CMD="${MINIFORGE_BIN_PATH}/pip" +PIP_COMPILE_CMD="${MINIFORGE_BIN_PATH}/pip-compile" +# Find the appropriate wheel by grepping for the Python version. +PYTHON_WHEEL=$(find ./.whl -maxdepth 1 -type f -name "*${PYTHON_VERSION}*.whl" -print -quit) + +if [[ -z "$PYTHON_WHEEL" ]]; then + echo "No wheel found for pattern *${PYTHON_VERSION}*.whl" >/dev/stderr + exit 1 +fi + +"$PYTHON_EXE" --version + +"$PIP_CMD" install --upgrade pip + +"$PIP_CMD" install pip-tools + +"$PIP_COMPILE_CMD" --version + +echo "ray[all]==${RAY_PLACEHOLDER_VERSION}" > ray-requirement.txt + +"$PIP_COMPILE_CMD" ray-requirement.txt -o /ray.lock --find-links=.whl/ + +echo "✅ Completed ray placeholder wheel test" diff --git a/ci/ray_ci/builder_container.py b/ci/ray_ci/builder_container.py index 84e5c78ed634..f3601d47b779 100644 --- a/ci/ray_ci/builder_container.py +++ b/ci/ray_ci/builder_container.py @@ -62,6 +62,7 @@ def run(self) -> None: f"./ci/build/build-manylinux-wheel.sh {self.bin_path}", "chown -R 2000:100 /artifact-mount", ] + if self.upload: cmds += ["./ci/build/copy_build_artifacts.sh wheel"] self.run_script(cmds) diff --git a/python/setup.py b/python/setup.py index bae70a9a265e..7f65cf5ef2cd 100644 --- a/python/setup.py +++ b/python/setup.py @@ -38,7 +38,7 @@ RUNTIME_ENV_AGENT_THIRDPARTY_SUBDIR = os.path.join( "ray", "_private", "runtime_env", "agent", "thirdparty_files" ) - +DEPS_ONLY_VERSION = "100.0.0-dev" # In automated builds, we do a few adjustments before building. For instance, # the bazel environment is set up slightly differently, and symlinks are # replaced with junctions in Windows. This variable is set in our conda-forge @@ -71,6 +71,7 @@ class BuildType(Enum): DEBUG = 2 ASAN = 3 TSAN = 4 + DEPS_ONLY = 5 class SetupSpec: @@ -87,6 +88,8 @@ def __init__( self.version: str = f"{version}+asan" elif build_type == BuildType.TSAN: self.version: str = f"{version}+tsan" + elif build_type == BuildType.DEPS_ONLY: + self.version: str = DEPS_ONLY_VERSION else: self.version = version self.description: str = description @@ -96,7 +99,7 @@ def __init__( self.extras: dict = {} def get_packages(self): - if self.type == SetupType.RAY: + if self.type == SetupType.RAY and self.build_type != BuildType.DEPS_ONLY: return setuptools.find_packages(exclude=("tests", "*.tests", "*.tests.*")) else: return [] @@ -109,6 +112,8 @@ def get_packages(self): BUILD_TYPE = BuildType.ASAN elif build_type == "tsan": BUILD_TYPE = BuildType.TSAN +elif build_type == "deps-only": + BUILD_TYPE = BuildType.DEPS_ONLY else: BUILD_TYPE = BuildType.DEFAULT @@ -701,12 +706,15 @@ def copy_file(target_dir, filename, rootdir): def pip_run(build_ext): - if SKIP_BAZEL_BUILD: + if SKIP_BAZEL_BUILD or setup_spec.build_type == BuildType.DEPS_ONLY: build(False, False, False) else: build(True, BUILD_JAVA, BUILD_CPP) if setup_spec.type == SetupType.RAY: + if setup_spec.build_type == BuildType.DEPS_ONLY: + setup_spec.files_to_include = [] + return setup_spec.files_to_include += ray_files thirdparty_dir = os.path.join(ROOT_DIR, THIRDPARTY_SUBDIR) From d0e73e88c1ce7a087875544d7a63252bd4cd71a0 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Fri, 29 Aug 2025 18:43:13 -0700 Subject: [PATCH 372/634] [deps] add adlfs[abfs] (#56084) into ray images Signed-off-by: Lonnie Liu --- docker/base-deps/Dockerfile | 1 + docker/base-slim/Dockerfile | 1 + python/deplocks/llm/ray_test_py311_cpu.lock | 20 +++++++++++++++++++ python/deplocks/llm/ray_test_py311_cu121.lock | 20 +++++++++++++++++++ python/deplocks/llm/ray_test_py311_cu128.lock | 20 +++++++++++++++++++ .../deplocks/llm/rayllm_test_py311_cpu.lock | 20 +++++++++++++++++++ .../deplocks/llm/rayllm_test_py311_cu121.lock | 20 +++++++++++++++++++ .../deplocks/llm/rayllm_test_py311_cu128.lock | 20 +++++++++++++++++++ python/requirements/cloud-requirements.txt | 1 + python/requirements_compiled.txt | 15 +++++++++++++- 10 files changed, 137 insertions(+), 1 deletion(-) diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index 2857ec1cb34f..f86e70e5fc66 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -117,6 +117,7 @@ PIP_PKGS=( cryptography google-api-python-client google-oauth + "adlfs[abfs]" ) # Install uv diff --git a/docker/base-slim/Dockerfile b/docker/base-slim/Dockerfile index 259bfba3d19c..2a3e62d2cc9d 100644 --- a/docker/base-slim/Dockerfile +++ b/docker/base-slim/Dockerfile @@ -124,6 +124,7 @@ PIP_PKGS=( anyscale packaging azure-identity + "adlfs[abfs]" boto3 google-cloud-storage jupyterlab diff --git a/python/deplocks/llm/ray_test_py311_cpu.lock b/python/deplocks/llm/ray_test_py311_cpu.lock index a5778d763b58..4c7476294eaf 100644 --- a/python/deplocks/llm/ray_test_py311_cpu.lock +++ b/python/deplocks/llm/ray_test_py311_cpu.lock @@ -3,6 +3,12 @@ --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu +adlfs==2023.8.0 \ + --hash=sha256:07e804f6df4593acfcaf01025b162e30ac13e523d3570279c98b2d91a18026d9 \ + --hash=sha256:3eb248a3c2a30b419f1147bd7676d156b5219f96ef7f11d47166afd2a3bdb07e + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # -r python/requirements/cloud-requirements.txt aiofiles==22.1.0 \ --hash=sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad \ --hash=sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6 @@ -101,6 +107,7 @@ aiohttp==3.11.16 \ # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt + # adlfs # aiohttp-cors # pytest-aiohttp aiohttp-cors==0.7.0 \ @@ -211,20 +218,29 @@ azure-core==1.29.5 \ --hash=sha256:52983c89d394c6f881a121e5101c5fa67278ca3b1f339c8fb2ef39230c70e9ac # via # -c /tmp/ray-deps/requirements_compiled.txt + # adlfs # azure-identity # azure-storage-blob # smart-open +azure-datalake-store==0.0.53 \ + --hash=sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393 \ + --hash=sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # adlfs azure-identity==1.17.1 \ --hash=sha256:32ecc67cc73f4bd0595e4f64b1ca65cd05186f4fe6f98ed2ae9f1aa32646efea \ --hash=sha256:db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt + # adlfs azure-storage-blob==12.22.0 \ --hash=sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e \ --hash=sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8 # via # -c /tmp/ray-deps/requirements_compiled.txt + # adlfs # smart-open babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ @@ -346,6 +362,7 @@ cffi==1.16.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # argon2-cffi-bindings + # azure-datalake-store # cryptography charset-normalizer==3.3.2 \ --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ @@ -860,6 +877,7 @@ fsspec==2023.5.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt + # adlfs gitdb==4.0.11 \ --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b @@ -1593,6 +1611,7 @@ msal==1.28.1 \ --hash=sha256:d72bbfe2d5c2f2555f4bc6205be4450ddfd12976610dd9a16a9ab0f05c68b64d # via # -c /tmp/ray-deps/requirements_compiled.txt + # azure-datalake-store # azure-identity # msal-extensions msal-extensions==1.2.0b1 \ @@ -2734,6 +2753,7 @@ requests==2.32.3 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # azure-core + # azure-datalake-store # google-api-core # google-cloud-storage # jupyterlab-server diff --git a/python/deplocks/llm/ray_test_py311_cu121.lock b/python/deplocks/llm/ray_test_py311_cu121.lock index 9d7dd16bc2b9..92c04a8fb0ea 100644 --- a/python/deplocks/llm/ray_test_py311_cu121.lock +++ b/python/deplocks/llm/ray_test_py311_cu121.lock @@ -3,6 +3,12 @@ --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 +adlfs==2023.8.0 \ + --hash=sha256:07e804f6df4593acfcaf01025b162e30ac13e523d3570279c98b2d91a18026d9 \ + --hash=sha256:3eb248a3c2a30b419f1147bd7676d156b5219f96ef7f11d47166afd2a3bdb07e + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # -r python/requirements/cloud-requirements.txt aiofiles==22.1.0 \ --hash=sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad \ --hash=sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6 @@ -101,6 +107,7 @@ aiohttp==3.11.16 \ # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt + # adlfs # aiohttp-cors # pytest-aiohttp aiohttp-cors==0.7.0 \ @@ -211,20 +218,29 @@ azure-core==1.29.5 \ --hash=sha256:52983c89d394c6f881a121e5101c5fa67278ca3b1f339c8fb2ef39230c70e9ac # via # -c /tmp/ray-deps/requirements_compiled.txt + # adlfs # azure-identity # azure-storage-blob # smart-open +azure-datalake-store==0.0.53 \ + --hash=sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393 \ + --hash=sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # adlfs azure-identity==1.17.1 \ --hash=sha256:32ecc67cc73f4bd0595e4f64b1ca65cd05186f4fe6f98ed2ae9f1aa32646efea \ --hash=sha256:db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt + # adlfs azure-storage-blob==12.22.0 \ --hash=sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e \ --hash=sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8 # via # -c /tmp/ray-deps/requirements_compiled.txt + # adlfs # smart-open babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ @@ -346,6 +362,7 @@ cffi==1.16.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # argon2-cffi-bindings + # azure-datalake-store # cryptography charset-normalizer==3.3.2 \ --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ @@ -860,6 +877,7 @@ fsspec==2023.5.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt + # adlfs gitdb==4.0.11 \ --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b @@ -1593,6 +1611,7 @@ msal==1.28.1 \ --hash=sha256:d72bbfe2d5c2f2555f4bc6205be4450ddfd12976610dd9a16a9ab0f05c68b64d # via # -c /tmp/ray-deps/requirements_compiled.txt + # azure-datalake-store # azure-identity # msal-extensions msal-extensions==1.2.0b1 \ @@ -2734,6 +2753,7 @@ requests==2.32.3 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # azure-core + # azure-datalake-store # google-api-core # google-cloud-storage # jupyterlab-server diff --git a/python/deplocks/llm/ray_test_py311_cu128.lock b/python/deplocks/llm/ray_test_py311_cu128.lock index 8bde8ef1045a..04402a5fd426 100644 --- a/python/deplocks/llm/ray_test_py311_cu128.lock +++ b/python/deplocks/llm/ray_test_py311_cu128.lock @@ -3,6 +3,12 @@ --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 +adlfs==2023.8.0 \ + --hash=sha256:07e804f6df4593acfcaf01025b162e30ac13e523d3570279c98b2d91a18026d9 \ + --hash=sha256:3eb248a3c2a30b419f1147bd7676d156b5219f96ef7f11d47166afd2a3bdb07e + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # -r python/requirements/cloud-requirements.txt aiofiles==22.1.0 \ --hash=sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad \ --hash=sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6 @@ -101,6 +107,7 @@ aiohttp==3.11.16 \ # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt + # adlfs # aiohttp-cors # pytest-aiohttp aiohttp-cors==0.7.0 \ @@ -211,20 +218,29 @@ azure-core==1.29.5 \ --hash=sha256:52983c89d394c6f881a121e5101c5fa67278ca3b1f339c8fb2ef39230c70e9ac # via # -c /tmp/ray-deps/requirements_compiled.txt + # adlfs # azure-identity # azure-storage-blob # smart-open +azure-datalake-store==0.0.53 \ + --hash=sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393 \ + --hash=sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # adlfs azure-identity==1.17.1 \ --hash=sha256:32ecc67cc73f4bd0595e4f64b1ca65cd05186f4fe6f98ed2ae9f1aa32646efea \ --hash=sha256:db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt + # adlfs azure-storage-blob==12.22.0 \ --hash=sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e \ --hash=sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8 # via # -c /tmp/ray-deps/requirements_compiled.txt + # adlfs # smart-open babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ @@ -346,6 +362,7 @@ cffi==1.16.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # argon2-cffi-bindings + # azure-datalake-store # cryptography charset-normalizer==3.3.2 \ --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ @@ -860,6 +877,7 @@ fsspec==2023.5.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt + # adlfs gitdb==4.0.11 \ --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b @@ -1593,6 +1611,7 @@ msal==1.28.1 \ --hash=sha256:d72bbfe2d5c2f2555f4bc6205be4450ddfd12976610dd9a16a9ab0f05c68b64d # via # -c /tmp/ray-deps/requirements_compiled.txt + # azure-datalake-store # azure-identity # msal-extensions msal-extensions==1.2.0b1 \ @@ -2734,6 +2753,7 @@ requests==2.32.3 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # azure-core + # azure-datalake-store # google-api-core # google-cloud-storage # jupyterlab-server diff --git a/python/deplocks/llm/rayllm_test_py311_cpu.lock b/python/deplocks/llm/rayllm_test_py311_cpu.lock index d52e9ae3b25a..04c9ddb2de7b 100644 --- a/python/deplocks/llm/rayllm_test_py311_cpu.lock +++ b/python/deplocks/llm/rayllm_test_py311_cpu.lock @@ -3,6 +3,12 @@ --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu +adlfs==2023.8.0 \ + --hash=sha256:07e804f6df4593acfcaf01025b162e30ac13e523d3570279c98b2d91a18026d9 \ + --hash=sha256:3eb248a3c2a30b419f1147bd7676d156b5219f96ef7f11d47166afd2a3bdb07e + # via + # -c python/deplocks/llm/ray_test_py311_cpu.lock + # -r python/requirements/cloud-requirements.txt aiofiles==22.1.0 \ --hash=sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad \ --hash=sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6 @@ -102,6 +108,7 @@ aiohttp==3.11.16 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements/llm/llm-test-requirements.txt # -r python/requirements.txt + # adlfs # aiohttp-cors # pytest-aiohttp # vllm @@ -223,20 +230,29 @@ azure-core==1.29.5 \ --hash=sha256:52983c89d394c6f881a121e5101c5fa67278ca3b1f339c8fb2ef39230c70e9ac # via # -c python/deplocks/llm/ray_test_py311_cpu.lock + # adlfs # azure-identity # azure-storage-blob # smart-open +azure-datalake-store==0.0.53 \ + --hash=sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393 \ + --hash=sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b + # via + # -c python/deplocks/llm/ray_test_py311_cpu.lock + # adlfs azure-identity==1.17.1 \ --hash=sha256:32ecc67cc73f4bd0595e4f64b1ca65cd05186f4fe6f98ed2ae9f1aa32646efea \ --hash=sha256:db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt + # adlfs azure-storage-blob==12.22.0 \ --hash=sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e \ --hash=sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8 # via # -c python/deplocks/llm/ray_test_py311_cpu.lock + # adlfs # smart-open babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ @@ -499,6 +515,7 @@ cffi==1.16.0 \ # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # argon2-cffi-bindings + # azure-datalake-store # cryptography # soundfile charset-normalizer==3.3.2 \ @@ -1064,6 +1081,7 @@ fsspec==2023.5.0 \ # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt + # adlfs # huggingface-hub # torch gguf==0.16.2 \ @@ -2100,6 +2118,7 @@ msal==1.28.1 \ --hash=sha256:d72bbfe2d5c2f2555f4bc6205be4450ddfd12976610dd9a16a9ab0f05c68b64d # via # -c python/deplocks/llm/ray_test_py311_cpu.lock + # azure-datalake-store # azure-identity # msal-extensions msal-extensions==1.2.0b1 \ @@ -3725,6 +3744,7 @@ requests==2.32.3 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # azure-core + # azure-datalake-store # google-api-core # google-cloud-storage # huggingface-hub diff --git a/python/deplocks/llm/rayllm_test_py311_cu121.lock b/python/deplocks/llm/rayllm_test_py311_cu121.lock index eea2023989b0..18d46b031806 100644 --- a/python/deplocks/llm/rayllm_test_py311_cu121.lock +++ b/python/deplocks/llm/rayllm_test_py311_cu121.lock @@ -3,6 +3,12 @@ --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 +adlfs==2023.8.0 \ + --hash=sha256:07e804f6df4593acfcaf01025b162e30ac13e523d3570279c98b2d91a18026d9 \ + --hash=sha256:3eb248a3c2a30b419f1147bd7676d156b5219f96ef7f11d47166afd2a3bdb07e + # via + # -c python/deplocks/llm/ray_test_py311_cu121.lock + # -r python/requirements/cloud-requirements.txt aiofiles==22.1.0 \ --hash=sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad \ --hash=sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6 @@ -102,6 +108,7 @@ aiohttp==3.11.16 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements/llm/llm-test-requirements.txt # -r python/requirements.txt + # adlfs # aiohttp-cors # pytest-aiohttp # vllm @@ -223,20 +230,29 @@ azure-core==1.29.5 \ --hash=sha256:52983c89d394c6f881a121e5101c5fa67278ca3b1f339c8fb2ef39230c70e9ac # via # -c python/deplocks/llm/ray_test_py311_cu121.lock + # adlfs # azure-identity # azure-storage-blob # smart-open +azure-datalake-store==0.0.53 \ + --hash=sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393 \ + --hash=sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b + # via + # -c python/deplocks/llm/ray_test_py311_cu121.lock + # adlfs azure-identity==1.17.1 \ --hash=sha256:32ecc67cc73f4bd0595e4f64b1ca65cd05186f4fe6f98ed2ae9f1aa32646efea \ --hash=sha256:db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt + # adlfs azure-storage-blob==12.22.0 \ --hash=sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e \ --hash=sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8 # via # -c python/deplocks/llm/ray_test_py311_cu121.lock + # adlfs # smart-open babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ @@ -499,6 +515,7 @@ cffi==1.16.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # argon2-cffi-bindings + # azure-datalake-store # cryptography # soundfile charset-normalizer==3.3.2 \ @@ -1064,6 +1081,7 @@ fsspec==2023.5.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt + # adlfs # huggingface-hub # torch gguf==0.16.2 \ @@ -2100,6 +2118,7 @@ msal==1.28.1 \ --hash=sha256:d72bbfe2d5c2f2555f4bc6205be4450ddfd12976610dd9a16a9ab0f05c68b64d # via # -c python/deplocks/llm/ray_test_py311_cu121.lock + # azure-datalake-store # azure-identity # msal-extensions msal-extensions==1.2.0b1 \ @@ -3816,6 +3835,7 @@ requests==2.32.3 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # azure-core + # azure-datalake-store # google-api-core # google-cloud-storage # huggingface-hub diff --git a/python/deplocks/llm/rayllm_test_py311_cu128.lock b/python/deplocks/llm/rayllm_test_py311_cu128.lock index f750627c7e8a..bea79b3cf3b6 100644 --- a/python/deplocks/llm/rayllm_test_py311_cu128.lock +++ b/python/deplocks/llm/rayllm_test_py311_cu128.lock @@ -3,6 +3,12 @@ --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 +adlfs==2023.8.0 \ + --hash=sha256:07e804f6df4593acfcaf01025b162e30ac13e523d3570279c98b2d91a18026d9 \ + --hash=sha256:3eb248a3c2a30b419f1147bd7676d156b5219f96ef7f11d47166afd2a3bdb07e + # via + # -c python/deplocks/llm/ray_test_py311_cu128.lock + # -r python/requirements/cloud-requirements.txt aiofiles==22.1.0 \ --hash=sha256:1142fa8e80dbae46bb6339573ad4c8c0841358f79c6eb50a493dceca14621bad \ --hash=sha256:9107f1ca0b2a5553987a94a3c9959fe5b491fdf731389aa5b7b1bd0733e32de6 @@ -102,6 +108,7 @@ aiohttp==3.11.16 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements/llm/llm-test-requirements.txt # -r python/requirements.txt + # adlfs # aiohttp-cors # pytest-aiohttp # vllm @@ -223,20 +230,29 @@ azure-core==1.29.5 \ --hash=sha256:52983c89d394c6f881a121e5101c5fa67278ca3b1f339c8fb2ef39230c70e9ac # via # -c python/deplocks/llm/ray_test_py311_cu128.lock + # adlfs # azure-identity # azure-storage-blob # smart-open +azure-datalake-store==0.0.53 \ + --hash=sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393 \ + --hash=sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b + # via + # -c python/deplocks/llm/ray_test_py311_cu128.lock + # adlfs azure-identity==1.17.1 \ --hash=sha256:32ecc67cc73f4bd0595e4f64b1ca65cd05186f4fe6f98ed2ae9f1aa32646efea \ --hash=sha256:db8d59c183b680e763722bfe8ebc45930e6c57df510620985939f7f3191e0382 # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt + # adlfs azure-storage-blob==12.22.0 \ --hash=sha256:b3804bb4fe8ab1c32771fa464053da772a682c2737b19da438a3f4e5e3b3736e \ --hash=sha256:bb7d2d824ce3f11f14a27ee7d9281289f7e072ac8311c52e3652672455b7d5e8 # via # -c python/deplocks/llm/ray_test_py311_cu128.lock + # adlfs # smart-open babel==2.13.1 \ --hash=sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900 \ @@ -499,6 +515,7 @@ cffi==1.16.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # argon2-cffi-bindings + # azure-datalake-store # cryptography # soundfile charset-normalizer==3.3.2 \ @@ -1063,6 +1080,7 @@ fsspec==2023.5.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt + # adlfs # huggingface-hub # torch gguf==0.17.0 \ @@ -2064,6 +2082,7 @@ msal==1.28.1 \ --hash=sha256:d72bbfe2d5c2f2555f4bc6205be4450ddfd12976610dd9a16a9ab0f05c68b64d # via # -c python/deplocks/llm/ray_test_py311_cu128.lock + # azure-datalake-store # azure-identity # msal-extensions msal-extensions==1.2.0b1 \ @@ -3739,6 +3758,7 @@ requests==2.32.3 \ # -r python/requirements/cloud-requirements.txt # -r python/requirements.txt # azure-core + # azure-datalake-store # google-api-core # google-cloud-storage # huggingface-hub diff --git a/python/requirements/cloud-requirements.txt b/python/requirements/cloud-requirements.txt index 2865270b4bb2..6f8677797288 100644 --- a/python/requirements/cloud-requirements.txt +++ b/python/requirements/cloud-requirements.txt @@ -9,6 +9,7 @@ certifi pycurl azure-identity smart_open[s3,gcs,azure,http] +adlfs[abfs] # Anyscale CLI requirements boto3>=1.26.76 diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt index f66c65c7b33f..03fbe19cc80f 100644 --- a/python/requirements_compiled.txt +++ b/python/requirements_compiled.txt @@ -24,6 +24,8 @@ adagio==0.2.4 # qpd adal==1.2.7 # via msrestazure +adlfs==2023.8.0 + # via -r python/requirements/cloud-requirements.txt aim==3.23.0 ; python_version < "3.12" # via -r python/requirements/ml/tune-test-requirements.txt aim-ui==3.23.0 @@ -50,6 +52,7 @@ aiohttp==3.11.16 # -r python/requirements.txt # -r python/requirements/cloud-requirements.txt # -r python/requirements/test-requirements.txt + # adlfs # aiobotocore # aiohttp-cors # delta-sharing @@ -160,15 +163,19 @@ azure-common==1.1.28 # smart-open azure-core==1.29.5 # via + # adlfs # azure-identity # azure-mgmt-core # azure-storage-blob # msrest # smart-open +azure-datalake-store==0.0.53 + # via adlfs azure-identity==1.17.1 # via # -r python/requirements/cloud-requirements.txt # -r python/requirements/test-requirements.txt + # adlfs azure-mgmt-compute==31.0.0 # via -r python/requirements/test-requirements.txt azure-mgmt-core==1.4.0 @@ -182,7 +189,9 @@ azure-mgmt-network==25.4.0 azure-mgmt-resource==23.1.1 # via -r python/requirements/test-requirements.txt azure-storage-blob==12.22.0 - # via smart-open + # via + # adlfs + # smart-open babel==2.13.1 # via # jupyterlab-server @@ -270,6 +279,7 @@ certifi==2025.1.31 cffi==1.16.0 # via # argon2-cffi-bindings + # azure-datalake-store # cryptography # pymunk # pynacl @@ -566,6 +576,7 @@ fs==2.4.16 fsspec==2023.5.0 # via # -r python/requirements.txt + # adlfs # dask # datasets # delta-sharing @@ -1096,6 +1107,7 @@ mpmath==1.3.0 msal==1.28.1 # via # azure-cli-core + # azure-datalake-store # azure-identity # msal-extensions msal-extensions==1.2.0b1 @@ -1861,6 +1873,7 @@ requests==2.32.3 # aim # azure-cli-core # azure-core + # azure-datalake-store # comet-ml # databricks-sdk # datasets From d93eb0f7b01a4a661f7190d9d083ec50258f6851 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Sat, 30 Aug 2025 09:48:27 -0500 Subject: [PATCH 373/634] [core] Remove unnecessary `core_worker_client` dependency (#56117) Signed-off-by: Edward Oakes --- src/ray/gcs/gcs_server/BUILD.bazel | 1 - src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h | 1 - 2 files changed, 2 deletions(-) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index fd3ad8b2c890..2e49691bc518 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -289,7 +289,6 @@ ray_cc_library( "//src/ray/raylet/scheduling:cluster_resource_scheduler", "//src/ray/raylet/scheduling:scheduling_context", "//src/ray/raylet_client:raylet_client_lib", - "//src/ray/rpc:core_worker_client", "//src/ray/rpc:node_manager_client", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h index 61b7fa651a79..ee3b7a44e161 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h @@ -33,7 +33,6 @@ #include "ray/raylet_client/raylet_client.h" #include "ray/rpc/node_manager/node_manager_client.h" #include "ray/rpc/node_manager/raylet_client_pool.h" -#include "ray/rpc/worker/core_worker_client.h" #include "src/ray/protobuf/gcs_service.pb.h" namespace ray { From 073536c7efa25db27467bbef7dd9957452d2f357 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Sat, 30 Aug 2025 13:04:05 -0700 Subject: [PATCH 374/634] [wheel] build wheel with -v (#56119) so that we can see how the bazel action goes and if they are hitting the build cache Signed-off-by: Lonnie Liu --- ci/build/build-manylinux-wheel.sh | 4 ++-- python/build-wheel-macos.sh | 2 +- python/build-wheel-windows.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/build/build-manylinux-wheel.sh b/ci/build/build-manylinux-wheel.sh index a324091b7903..b2b1abdadaa7 100755 --- a/ci/build/build-manylinux-wheel.sh +++ b/ci/build/build-manylinux-wheel.sh @@ -28,13 +28,13 @@ sudo ln -sf "/opt/python/${PYTHON}/bin/python3" /usr/local/bin/python3 # build ray wheel PATH="/opt/python/${PYTHON}/bin:$PATH" RAY_INSTALL_JAVA=0 \ -"/opt/python/${PYTHON}/bin/python" -m pip wheel -q -w dist . --no-deps +"/opt/python/${PYTHON}/bin/python" -m pip wheel -v -w dist . --no-deps if [[ "${RAY_DISABLE_EXTRA_CPP:-}" != 1 ]]; then # build ray-cpp wheel PATH="/opt/python/${PYTHON}/bin:$PATH" RAY_INSTALL_JAVA=0 \ - RAY_INSTALL_CPP=1 "/opt/python/${PYTHON}/bin/python" -m pip wheel -q -w dist . --no-deps + RAY_INSTALL_CPP=1 "/opt/python/${PYTHON}/bin/python" -m pip wheel -v -w dist . --no-deps fi # Rename the wheels so that they can be uploaded to PyPI. TODO(rkn): This is a diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 9837a007f8eb..3b8b0103aefc 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -82,7 +82,7 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do # Add the correct Python to the path and build the wheel. This is only # needed so that the installation finds the cython executable. # build ray wheel - $PIP_CMD wheel -q -w dist . --no-deps + $PIP_CMD wheel -v -w dist . --no-deps # build ray-cpp wheel RAY_INSTALL_CPP=1 $PIP_CMD wheel -q -w dist . --no-deps mv dist/*.whl ../.whl/ diff --git a/python/build-wheel-windows.sh b/python/build-wheel-windows.sh index 3a4b22e8b890..623d4db613a2 100755 --- a/python/build-wheel-windows.sh +++ b/python/build-wheel-windows.sh @@ -130,11 +130,11 @@ build_wheel_windows() { exit 1 fi # build ray wheel - python -m pip wheel -q -w dist . --no-deps + python -m pip wheel -v -w dist . --no-deps # Pack any needed system dlls like msvcp140.dll delvewheel repair dist/ray-*.whl # build ray-cpp wheel - RAY_INSTALL_CPP=1 python -m pip wheel -q -w dist . --no-deps + RAY_INSTALL_CPP=1 python -m pip wheel -v -w dist . --no-deps # No extra dlls are needed, do not call delvewheel uninstall_ray ) From 62b6bb4fa284e5c087a2f6e907143e90e20da17a Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Sun, 31 Aug 2025 16:02:08 -0700 Subject: [PATCH 375/634] [core] Remove Driver Specific LeaseID (#56090) Signed-off-by: joshlee --- src/ray/common/id.cc | 7 -- src/ray/common/id.h | 9 --- .../task_submission/normal_task_submitter.cc | 2 +- src/ray/gcs/gcs_server/gcs_actor_scheduler.cc | 2 +- src/ray/raylet/node_manager.cc | 8 +- src/ray/raylet/tests/worker_pool_test.cc | 75 +++++++------------ src/ray/raylet/worker.cc | 1 + src/ray/raylet/worker.h | 1 + src/ray/raylet/worker_pool.cc | 4 +- 9 files changed, 34 insertions(+), 75 deletions(-) diff --git a/src/ray/common/id.cc b/src/ray/common/id.cc index 65ace9147bed..646fe65abb78 100644 --- a/src/ray/common/id.cc +++ b/src/ray/common/id.cc @@ -322,19 +322,12 @@ LeaseID LeaseID::FromRandom() { } LeaseID LeaseID::FromWorker(const WorkerID &worker_id, uint32_t counter) { - RAY_CHECK_GT(counter, 0u); std::string data(kUniqueBytesLength, 0); std::memcpy(data.data(), &counter, sizeof(counter)); std::copy_n(worker_id.Data(), kUniqueIDSize, std::back_inserter(data)); return LeaseID::FromBinary(data); } -LeaseID LeaseID::DriverLeaseId(const WorkerID &driver_id) { - std::string data(kUniqueBytesLength, 0); - std::copy_n(driver_id.Data(), kUniqueIDSize, std::back_inserter(data)); - return LeaseID::FromBinary(data); -} - WorkerID LeaseID::WorkerId() const { return WorkerID::FromBinary(std::string( reinterpret_cast(id_ + kUniqueBytesLength), kUniqueIDSize)); diff --git a/src/ray/common/id.h b/src/ray/common/id.h index 8a1f7e99d610..e01ee20a2ff1 100644 --- a/src/ray/common/id.h +++ b/src/ray/common/id.h @@ -349,15 +349,6 @@ class LeaseID : public BaseID { /// \return The `LeaseID` for the worker lease. static LeaseID FromWorker(const WorkerID &worker_id, uint32_t counter); - /// Creates a `LeaseID` from a driver ID. The counter bits are nulled out only for - /// driver as we need a predetermined lease value that can be calculated indepently by - /// the raylet without having to send the ID over. - /// - /// \param driver_id The driver ID to which this lease belongs. - /// - /// \return The `LeaseID` for the worker lease. - static LeaseID DriverLeaseId(const WorkerID &driver_id); - /// Creates a random `LeaseID`. /// /// \return A `LeaseID` generated with random bytes diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 0b7ba9e4c781..4796a41dcd39 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -310,7 +310,7 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli return; } // Counter for generating unique lease IDs. - static uint32_t lease_id_counter = 1; + static uint32_t lease_id_counter = 0; const LeaseID lease_id = LeaseID::FromWorker(worker_id_, lease_id_counter++); rpc::LeaseSpec lease_spec_msg = scheduling_key_entry.lease_spec.GetMessage(); lease_spec_msg.set_lease_id(lease_id.Binary()); diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc index a607b94f237f..d471ada70c15 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc @@ -328,7 +328,7 @@ void GcsActorScheduler::LeaseWorkerFromNode(std::shared_ptr actor, // Actor leases should be sent to the raylet immediately, so we should never build up a // backlog in GCS. // Counter for generating unique lease IDs. - static uint32_t lease_id_counter = 1; + static uint32_t lease_id_counter = 0; actor->GetMutableLeaseSpec()->set_lease_id( LeaseID::FromWorker(WorkerID::FromRandom(), lease_id_counter++).Binary()); raylet_client->RequestWorkerLease( diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 6a958c060631..39334551d410 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -1159,13 +1159,6 @@ Status NodeManager::RegisterForNewDriver( const ray::protocol::RegisterClientRequest *message, std::function send_reply_callback) { worker->SetProcess(Process::FromPid(pid)); - // Compute a dummy driver lease id from a given driver. - // The lease id set in the worker here should be consistent with the lease - // id set in the core worker. - // TODO(#56010): We shouldn't need to have a special lease id for the driver, just check - // the worker type instead - const LeaseID driver_lease_id = LeaseID::DriverLeaseId(worker->WorkerId()); - worker->GrantLeaseId(driver_lease_id); rpc::JobConfig job_config; job_config.ParseFromString(message->serialized_job_config()->str()); @@ -1250,6 +1243,7 @@ void NodeManager::SendPortAnnouncementResponse( void NodeManager::HandleWorkerAvailable(const std::shared_ptr &worker) { RAY_CHECK(worker); + RAY_CHECK_NE(worker->GetWorkerType(), rpc::WorkerType::DRIVER); if (worker->GetWorkerType() == rpc::WorkerType::SPILL_WORKER) { // Return the worker to the idle pool. diff --git a/src/ray/raylet/tests/worker_pool_test.cc b/src/ray/raylet/tests/worker_pool_test.cc index 92e0e6556059..41c275d8ba5f 100644 --- a/src/ray/raylet/tests/worker_pool_test.cc +++ b/src/ray/raylet/tests/worker_pool_test.cc @@ -458,7 +458,6 @@ class WorkerPoolTest : public ::testing::Test { const rpc::JobConfig &job_config = rpc::JobConfig()) { auto driver = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, job_id); - driver->GrantLeaseId(LeaseID::FromRandom()); RAY_CHECK_OK(worker_pool_->RegisterDriver(driver, job_config, [](Status, int) {})); return driver; } @@ -531,24 +530,18 @@ static inline rpc::RuntimeEnvInfo ExampleRuntimeEnvInfoFromString( } static inline LeaseSpecification ExampleLeaseSpec( - const ActorID actor_id = ActorID::Nil(), + const ActorID actor_creation_id = ActorID::Nil(), const Language &language = Language::PYTHON, const JobID &job_id = JOB_ID, - const ActorID actor_creation_id = ActorID::Nil(), const std::vector &dynamic_worker_options = {}, - const LeaseID &lease_id = LeaseID::FromRandom(), + const LeaseID &lease_id = LeaseID::Nil(), const rpc::RuntimeEnvInfo runtime_env_info = rpc::RuntimeEnvInfo(), std::unordered_map resources = {{"CPU", 1}}) { rpc::LeaseSpec message; message.set_job_id(job_id.Binary()); message.set_language(language); - // Make sure no reduplicative lease id. - RAY_CHECK(!lease_id.IsNil()); message.set_lease_id(lease_id.Binary()); - if (!actor_id.IsNil()) { - message.set_type(TaskType::ACTOR_TASK); - message.set_actor_id(actor_id.Binary()); - } else if (!actor_creation_id.IsNil()) { + if (!actor_creation_id.IsNil()) { message.set_type(TaskType::ACTOR_CREATION_TASK); message.set_actor_id(actor_creation_id.Binary()); for (const auto &option : dynamic_worker_options) { @@ -651,7 +644,8 @@ TEST_F(WorkerPoolDriverRegisteredTest, InitialWorkerProcessCount) { } TEST_F(WorkerPoolDriverRegisteredTest, TestPrestartingWorkers) { - const auto lease_spec = ExampleLeaseSpec(); + auto lease_spec = ExampleLeaseSpec(); + lease_spec.GetMutableMessage().set_lease_id(LeaseID::FromRandom().Binary()); // Prestarts 2 workers. worker_pool_->PrestartWorkers(lease_spec, 2); ASSERT_EQ(worker_pool_->NumWorkersStarting(), 2); @@ -731,8 +725,8 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerSyncsOfMultipleLanguages) { TEST_F(WorkerPoolDriverRegisteredTest, StartWorkerWithNodeIdArg) { auto lease_id = LeaseID::FromRandom(); - LeaseSpecification lease_spec = ExampleLeaseSpec( - ActorID::Nil(), Language::PYTHON, JOB_ID, ActorID::Nil(), {}, lease_id); + LeaseSpecification lease_spec = + ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, JOB_ID, {}, lease_id); ASSERT_NE(worker_pool_->PopWorkerSync(lease_spec), nullptr); const auto real_command = worker_pool_->GetWorkerCommand(worker_pool_->LastStartedWorkerProcess()); @@ -757,11 +751,10 @@ TEST_F(WorkerPoolDriverRegisteredTest, StartWorkerWithDynamicOptionsCommand) { actor_jvm_options.end(), {"-Dmy-actor.hello=foo", "-Dmy-actor.world=bar", "-Xmx2g", "-Xms1g"}); JobID job_id = JobID::FromInt(12345); - auto actor_id = ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 1); - LeaseSpecification lease_spec = ExampleLeaseSpec(ActorID::Nil(), + auto actor_creation_id = ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 1); + LeaseSpecification lease_spec = ExampleLeaseSpec(actor_creation_id, Language::JAVA, job_id, - actor_id, actor_jvm_options, LeaseID::FromRandom()); @@ -885,8 +878,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerMultiTenancy) { // Make the first worker an actor worker. if (i == 0) { auto actor_creation_id = ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 1); - auto lease_spec = ExampleLeaseSpec( - /*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id, actor_creation_id); + auto lease_spec = ExampleLeaseSpec(actor_creation_id, Language::PYTHON, job_id); runtime_env_hash = lease_spec.GetRuntimeEnvHash(); } auto worker = worker_pool_->CreateWorker(Process::CreateNewDummy(), @@ -905,8 +897,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerMultiTenancy) { for (auto job_id : job_ids) { auto actor_creation_id = ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 1); // Pop workers for actor creation leases. - auto lease_spec = ExampleLeaseSpec( - /*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id, actor_creation_id); + auto lease_spec = ExampleLeaseSpec(actor_creation_id, Language::PYTHON, job_id); auto worker = worker_pool_->PopWorkerSync(lease_spec); ASSERT_TRUE(worker); ASSERT_EQ(worker->GetAssignedJobId(), job_id); @@ -1508,14 +1499,12 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestWorkerCapping) { std::vector> popped_workers; for (int i = 0; i < num_workers; i++) { // Pop workers for actor creation leases. - auto lease_spec = - ExampleLeaseSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); + auto lease_spec = ExampleLeaseSpec( + /*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id, {}, LeaseID::FromRandom()); auto worker = worker_pool_->PopWorkerSync(lease_spec, false); // Simulate granting the lease and finish. This is to set lease_grant_time_. RayLease lease(lease_spec); worker->GrantLease(lease); - worker->GrantLeaseId(LeaseID::Nil()); - popped_workers.push_back(worker); ASSERT_TRUE(worker); ASSERT_EQ(worker->GetAssignedJobId(), job_id); @@ -1528,6 +1517,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestWorkerCapping) { /// // Return all workers. for (const auto &worker : popped_workers) { + worker->GrantLeaseId(LeaseID::Nil()); worker_pool_->PushWorker(worker); } ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), num_workers); @@ -1541,8 +1531,8 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestWorkerCapping) { ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), POOL_SIZE_SOFT_LIMIT); // The first core worker exits, so one of idle workers should've been killed. - // Since the idle workers are killed in FIFO, we can assume the first entry in the idle - // workers will be killed. + // Since the idle workers are killed in FIFO if they've been granted a lease, we can + // assume the first entry in the idle workers will be killed. auto mock_rpc_client_it = mock_worker_rpc_clients_.find(popped_workers[0]->WorkerId()); ASSERT_EQ(mock_rpc_client_it->second->exit_count, 1) << " expected pid " << popped_workers[0]->GetProcess().GetId(); @@ -1730,7 +1720,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestJobFinishedForPopWorker) { worker_pool_->HandleJobFinished(job_id); auto lease_spec = - ExampleLeaseSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); + ExampleLeaseSpec(/*actor_creation_id=*/ActorID::Nil(), Language::PYTHON, job_id); PopWorkerStatus pop_worker_status; // This PopWorker should fail since the job finished. worker = worker_pool_->PopWorkerSync(lease_spec, false, &pop_worker_status); @@ -1746,7 +1736,8 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestJobFinishedForPopWorker) { job_id = JOB_ID_2; rpc::JobConfig job_config; RegisterDriver(Language::PYTHON, job_id, job_config); - lease_spec = ExampleLeaseSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); + lease_spec = + ExampleLeaseSpec(/*actor_creation_id=*/ActorID::Nil(), Language::PYTHON, job_id); pop_worker_status = PopWorkerStatus::OK; // This will start a new worker. std::promise promise; @@ -1808,7 +1799,7 @@ TEST_F(WorkerPoolDriverRegisteredTest, TestJobFinishedForceKillIdleWorker) { /// Grant some lease with the worker. auto lease_spec = - ExampleLeaseSpec(/*actor_id=*/ActorID::Nil(), Language::PYTHON, job_id); + ExampleLeaseSpec(/*actor_creation_id=*/ActorID::Nil(), Language::PYTHON, job_id); worker = worker_pool_->PopWorkerSync(lease_spec, false); ASSERT_EQ(worker_pool_->GetIdleWorkerSize(), 0); @@ -1900,22 +1891,20 @@ TEST_F(WorkerPoolDriverRegisteredTest, TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerWithRuntimeEnv) { ASSERT_EQ(worker_pool_->GetProcessSize(), 0); auto actor_creation_id = ActorID::Of(JOB_ID, TaskID::ForDriverTask(JOB_ID), 1); - const auto actor_creation_lease_spec = ExampleLeaseSpec(ActorID::Nil(), + const auto actor_creation_lease_spec = ExampleLeaseSpec(actor_creation_id, Language::PYTHON, JOB_ID, - actor_creation_id, {"XXX=YYY"}, LeaseID::FromRandom(), ExampleRuntimeEnvInfo({"XXX"})); - const auto normal_lease_spec = ExampleLeaseSpec(ActorID::Nil(), + const auto normal_lease_spec = ExampleLeaseSpec(actor_creation_id, Language::PYTHON, JOB_ID, - ActorID::Nil(), {"XXX=YYY"}, LeaseID::FromRandom(), ExampleRuntimeEnvInfo({"XXX"})); const auto normal_lease_spec_without_runtime_env = - ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, JOB_ID, ActorID::Nil(), {}); + ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, JOB_ID, {}); // Pop worker for actor creation lease again. auto popped_worker = worker_pool_->PopWorkerSync(actor_creation_lease_spec); // Got a worker with correct runtime env hash. @@ -1983,10 +1972,9 @@ TEST_F(WorkerPoolDriverRegisteredTest, RuntimeEnvUriReferenceWorkerLevel) { ASSERT_EQ(GetReferenceCount(runtime_env_info.serialized_runtime_env()), 1); // Start actor with runtime env. auto actor_creation_id = ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 1); - const auto actor_creation_lease_spec = ExampleLeaseSpec(ActorID::Nil(), + const auto actor_creation_lease_spec = ExampleLeaseSpec(actor_creation_id, Language::PYTHON, job_id, - actor_creation_id, {"XXX=YYY"}, LeaseID::FromRandom(), runtime_env_info); @@ -1996,7 +1984,6 @@ TEST_F(WorkerPoolDriverRegisteredTest, RuntimeEnvUriReferenceWorkerLevel) { const auto normal_lease_spec = ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_id, - ActorID::Nil(), {"XXX=YYY"}, LeaseID::FromRandom(), runtime_env_info); @@ -2029,10 +2016,9 @@ TEST_F(WorkerPoolDriverRegisteredTest, RuntimeEnvUriReferenceWorkerLevel) { ASSERT_EQ(GetReferenceCount(runtime_env_info.serialized_runtime_env()), 0); // Start actor with runtime env. auto actor_creation_id = ActorID::Of(job_id, TaskID::ForDriverTask(job_id), 2); - const auto actor_creation_lease_spec = ExampleLeaseSpec(ActorID::Nil(), + const auto actor_creation_lease_spec = ExampleLeaseSpec(actor_creation_id, Language::PYTHON, job_id, - actor_creation_id, {"XXX=YYY"}, LeaseID::FromRandom(), runtime_env_info); @@ -2064,10 +2050,9 @@ TEST_F(WorkerPoolDriverRegisteredTest, CacheWorkersByRuntimeEnvHash) { ASSERT_EQ(worker_pool_->GetProcessSize(), 0); auto actor_creation_id = ActorID::Of(JOB_ID, TaskID::ForDriverTask(JOB_ID), 1); const auto actor_creation_lease_spec_1 = - ExampleLeaseSpec(ActorID::Nil(), + ExampleLeaseSpec(actor_creation_id, Language::PYTHON, JOB_ID, - actor_creation_id, /*dynamic_worker_options=*/{}, LeaseID::FromRandom(), ExampleRuntimeEnvInfoFromString("mock_runtime_env_1")); @@ -2075,7 +2060,6 @@ TEST_F(WorkerPoolDriverRegisteredTest, CacheWorkersByRuntimeEnvHash) { ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, JOB_ID, - ActorID::Nil(), /*dynamic_worker_options=*/{}, LeaseID::FromRandom(), ExampleRuntimeEnvInfoFromString("mock_runtime_env_1")); @@ -2083,7 +2067,6 @@ TEST_F(WorkerPoolDriverRegisteredTest, CacheWorkersByRuntimeEnvHash) { ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, JOB_ID, - ActorID::Nil(), /*dynamic_worker_options=*/{}, LeaseID::FromRandom(), ExampleRuntimeEnvInfoFromString("mock_runtime_env_2")); @@ -2194,7 +2177,6 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerStatus) { ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_id, - ActorID::Nil(), {"XXX=YYY"}, LeaseID::FromRandom(), ExampleRuntimeEnvInfoFromString(std::string(kBadRuntimeEnv))); @@ -2211,7 +2193,6 @@ TEST_F(WorkerPoolDriverRegisteredTest, PopWorkerStatus) { ExampleLeaseSpec(ActorID::Nil(), Language::PYTHON, job_id, - ActorID::Nil(), {"XXX=YYY"}, LeaseID::FromRandom(), ExampleRuntimeEnvInfo({"XXX"})); @@ -2412,7 +2393,6 @@ TEST_F(WorkerPoolDriverRegisteredTest, WorkerReuseFailureForDifferentJobId) { TEST_F(WorkerPoolTest, RegisterFirstPythonDriverWaitForWorkerStart) { auto driver = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, JOB_ID); - driver->GrantLeaseId(LeaseID::FromRandom()); bool callback_called = false; auto callback = [callback_called_ptr = &callback_called](Status, int) mutable { *callback_called_ptr = true; @@ -2424,7 +2404,6 @@ TEST_F(WorkerPoolTest, RegisterFirstPythonDriverWaitForWorkerStart) { TEST_F(WorkerPoolTest, RegisterSecondPythonDriverCallbackImmediately) { auto driver = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, JOB_ID); - driver->GrantLeaseId(LeaseID::FromRandom()); RAY_CHECK_OK( worker_pool_->RegisterDriver(driver, rpc::JobConfig(), [](Status, int) {})); @@ -2434,7 +2413,6 @@ TEST_F(WorkerPoolTest, RegisterSecondPythonDriverCallbackImmediately) { }; auto second_driver = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::PYTHON, JOB_ID); - second_driver->GrantLeaseId(LeaseID::FromRandom()); RAY_CHECK_OK(worker_pool_->RegisterDriver(second_driver, rpc::JobConfig(), callback)); ASSERT_TRUE(callback_called); } @@ -2443,7 +2421,6 @@ TEST_F(WorkerPoolTest, RegisterFirstJavaDriverCallbackImmediately) { auto driver = worker_pool_->CreateWorker(Process::CreateNewDummy(), Language::JAVA, JOB_ID); - driver->GrantLeaseId(LeaseID::FromRandom()); bool callback_called = false; auto callback = [callback_called_ptr = &callback_called](Status, int) mutable { *callback_called_ptr = true; diff --git a/src/ray/raylet/worker.cc b/src/ray/raylet/worker.cc index abf4a7e9d4d2..8f5a59ef399d 100644 --- a/src/ray/raylet/worker.cc +++ b/src/ray/raylet/worker.cc @@ -176,6 +176,7 @@ void Worker::Connect(std::shared_ptr rpc_client) void Worker::GrantLeaseId(const LeaseID &lease_id) { lease_id_ = lease_id; if (!lease_id.IsNil()) { + RAY_CHECK(worker_type_ != rpc::WorkerType::DRIVER); lease_grant_time_ = absl::Now(); } }; diff --git a/src/ray/raylet/worker.h b/src/ray/raylet/worker.h index b4af8ee5cd1b..c88c4b83402e 100644 --- a/src/ray/raylet/worker.h +++ b/src/ray/raylet/worker.h @@ -284,6 +284,7 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa /// Connection state of a worker. std::shared_ptr connection_; /// The lease id of the worker's currently assigned lease. + /// It is always Nil for the driver. LeaseID lease_id_; /// Job ID for the worker's current assigned lease. JobID assigned_job_id_; diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index 342068c7f2e6..ce85798dc24f 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -876,7 +876,7 @@ Status WorkerPool::RegisterDriver(const std::shared_ptr &driver const rpc::JobConfig &job_config, std::function send_reply_callback) { int port; - RAY_CHECK(!driver->GetGrantedLeaseId().IsNil()); + RAY_CHECK(driver->GetGrantedLeaseId().IsNil()); Status status = GetNextFreePort(&port); if (!status.ok()) { send_reply_callback(status, /*port=*/0); @@ -1052,6 +1052,8 @@ void WorkerPool::PushWorker(const std::shared_ptr &worker) { // Since the worker is now idle, verify that it has no assigned lease ID. RAY_CHECK(worker->GetGrantedLeaseId().IsNil()) << "Idle workers cannot have an assigned lease ID"; + RAY_CHECK(worker->GetWorkerType() != rpc::WorkerType::DRIVER) + << "Idle workers cannot be drivers"; // Find a lease that this worker can fit. If there's none, put it in the idle pool. // First find in pending_registration_requests, then in pending_start_requests. std::shared_ptr pop_worker_request = nullptr; From 2149d8300d7a0dfc674456a88c5a467bf4701735 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Sun, 31 Aug 2025 23:13:01 -0700 Subject: [PATCH 376/634] [core] Don't need to return status from submitter cancel + cleanup (#56121) Signed-off-by: dayshah --- src/ray/core_worker/core_worker.cc | 71 +++++++++---------- .../core_worker/task_submission/BUILD.bazel | 2 +- .../task_submission/actor_task_submitter.cc | 17 ++--- .../task_submission/actor_task_submitter.h | 6 +- .../task_submission/normal_task_submitter.cc | 24 +++---- .../task_submission/normal_task_submitter.h | 14 ++-- .../tests/normal_task_submitter_test.cc | 14 ++-- 7 files changed, 64 insertions(+), 84 deletions(-) diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index fcac4106e5dd..4b73866600b0 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -2392,8 +2392,8 @@ Status CoreWorker::CancelTask(const ObjectID &object_id, RAY_LOG(DEBUG).WithField(object_id) << "Request to cancel a task of object to an owner " << obj_addr.SerializeAsString(); - return normal_task_submitter_->CancelRemoteTask( - object_id, obj_addr, force_kill, recursive); + normal_task_submitter_->CancelRemoteTask(object_id, obj_addr, force_kill, recursive); + return Status::OK(); } auto task_spec = task_manager_->GetTaskSpec(object_id.TaskId()); @@ -2414,58 +2414,51 @@ Status CoreWorker::CancelTask(const ObjectID &object_id, return Status::InvalidArgument("force=True is not supported for actor tasks."); } - return actor_task_submitter_->CancelTask(task_spec.value(), recursive); + actor_task_submitter_->CancelTask(task_spec.value(), recursive); } else { - return normal_task_submitter_->CancelTask(task_spec.value(), force_kill, recursive); + normal_task_submitter_->CancelTask(task_spec.value(), force_kill, recursive); } + return Status::OK(); } Status CoreWorker::CancelChildren(const TaskID &task_id, bool force_kill) { - std::vector> recursive_cancellation_status; - bool recursive_success = true; - for (const auto &child_id : task_manager_->GetPendingChildrenTasks(task_id)) { + absl::flat_hash_set unknown_child_task_ids; + auto child_task_ids = task_manager_->GetPendingChildrenTasks(task_id); + for (const auto &child_id : child_task_ids) { auto child_spec = task_manager_->GetTaskSpec(child_id); if (!child_spec.has_value()) { - recursive_success = false; - recursive_cancellation_status.emplace_back( - child_id, - Status::UnknownError( - "Recursive task cancellation failed--check warning logs.")); + unknown_child_task_ids.insert(child_id); } else if (child_spec->IsActorTask()) { - auto result = actor_task_submitter_->CancelTask(child_spec.value(), true); - recursive_cancellation_status.emplace_back(child_id, result); + actor_task_submitter_->CancelTask(std::move(*child_spec), true); } else { - auto result = - normal_task_submitter_->CancelTask(child_spec.value(), force_kill, true); - recursive_cancellation_status.emplace_back(child_id, result); + normal_task_submitter_->CancelTask(std::move(*child_spec), force_kill, true); } } - if (recursive_success) { + if (unknown_child_task_ids.empty()) { return Status::OK(); - } else { - auto kMaxFailedTaskSampleSize = 10; - std::ostringstream ostr; - ostr << "Failed to cancel all the children tasks of " << task_id << " recursively.\n" - << "Here are up to " << kMaxFailedTaskSampleSize - << " samples tasks that failed to be canceled\n"; - auto success = 0; - auto failures = 0; - for (const auto &[child_id, status] : recursive_cancellation_status) { - if (status.ok()) { - success += 1; - } else { - // Only record up to sample sizes. - if (failures < kMaxFailedTaskSampleSize) { - ostr << "\t" << child_id << ", " << status << "\n"; - } - failures += 1; - } + } + + constexpr size_t kMaxFailedTaskSampleSize = 10; + std::ostringstream ostr; + ostr << "Failed to cancel all the children tasks of " << task_id << " recursively.\n" + << "Here are up to " << kMaxFailedTaskSampleSize + << " samples tasks that failed to be canceled\n"; + const auto failure_status_str = + Status::UnknownError("Recursive task cancellation failed--check warning logs.") + .ToString(); + size_t failures = 0; + for (const auto &child_id : unknown_child_task_ids) { + ostr << "\t" << child_id << ", " << failure_status_str << "\n"; + failures += 1; + if (failures >= kMaxFailedTaskSampleSize) { + break; } - ostr << "Total Recursive cancelation success: " << success - << ", failures: " << failures; - return Status::UnknownError(ostr.str()); } + ostr << "Total Recursive cancelation success: " + << (child_task_ids.size() - unknown_child_task_ids.size()) + << ", failures: " << unknown_child_task_ids.size(); + return Status::UnknownError(ostr.str()); } Status CoreWorker::KillActor(const ActorID &actor_id, bool force_kill, bool no_restart) { diff --git a/src/ray/core_worker/task_submission/BUILD.bazel b/src/ray/core_worker/task_submission/BUILD.bazel index 09362fe45c2d..6261dddb4f60 100644 --- a/src/ray/core_worker/task_submission/BUILD.bazel +++ b/src/ray/core_worker/task_submission/BUILD.bazel @@ -91,7 +91,7 @@ ray_cc_library( "//src/ray/common:lease", "//src/ray/core_worker:lease_policy", "//src/ray/core_worker:memory_store", - "//src/ray/core_worker:task_manager", + "//src/ray/core_worker:task_manager_interface", "//src/ray/gcs:gcs_pb_util", "//src/ray/raylet_client:raylet_client_lib", "//src/ray/rpc:core_worker_client", diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.cc b/src/ray/core_worker/task_submission/actor_task_submitter.cc index 33ece8caf203..e34078b5956d 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.cc +++ b/src/ray/core_worker/task_submission/actor_task_submitter.cc @@ -844,12 +844,12 @@ void ActorTaskSubmitter::RetryCancelTask(TaskSpecification task_spec, execute_after( io_service_, [this, task_spec = std::move(task_spec), recursive] { - RAY_UNUSED(CancelTask(task_spec, recursive)); + CancelTask(task_spec, recursive); }, std::chrono::milliseconds(milliseconds)); } -Status ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursive) { +void ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursive) { // We don't support force_kill = true for actor tasks. bool force_kill = false; RAY_LOG(INFO).WithField(task_spec.TaskId()).WithField(task_spec.ActorId()) @@ -871,7 +871,7 @@ Status ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursiv GetTaskManagerWithoutMu().MarkTaskCanceled(task_id); if (!GetTaskManagerWithoutMu().IsTaskPending(task_id)) { RAY_LOG(DEBUG).WithField(task_id) << "Task is already finished or canceled"; - return Status::OK(); + return; } auto task_queued = false; @@ -886,7 +886,7 @@ Status ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursiv // No need to decrement cur_pending_calls because it doesn't matter. RAY_LOG(DEBUG).WithField(task_id) << "Task's actor is already dead. Ignoring the cancel request."; - return Status::OK(); + return; } task_queued = queue->second.actor_submit_queue_->Contains(send_pos); @@ -916,7 +916,7 @@ Status ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursiv error_info.set_error_type(rpc::ErrorType::TASK_CANCELLED); GetTaskManagerWithoutMu().FailOrRetryPendingTask( task_id, rpc::ErrorType::TASK_CANCELLED, /*status*/ nullptr, &error_info); - return Status::OK(); + return; } // At this point, the task is in "sent" state and not finished yet. @@ -934,7 +934,7 @@ Status ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursiv RAY_CHECK(queue != client_queues_.end()); if (!queue->second.rpc_client_) { RetryCancelTask(task_spec, recursive, 1000); - return Status::OK(); + return; } const auto &client = queue->second.rpc_client_; @@ -964,11 +964,6 @@ Status ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursiv } }); } - - // NOTE: Currently, ray.cancel is asynchronous. - // If we want to have a better guarantee in the cancelation result - // we should make it synchronos, but that can regress the performance. - return Status::OK(); } bool ActorTaskSubmitter::QueueGeneratorForResubmit(const TaskSpecification &spec) { diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.h b/src/ray/core_worker/task_submission/actor_task_submitter.h index 160ad7487a6a..8a98b84579b3 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.h +++ b/src/ray/core_worker/task_submission/actor_task_submitter.h @@ -236,11 +236,7 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { /// /// \param task_spec The task spec of a task that will be canceled. /// \param recursive If true, it will cancel all child tasks. - /// \return True if cancel request is not needed or it will be - /// requested. False otherwise. Note that tasks could be "not" - /// canceled although the status is true because it is an - /// asynchronous API. - Status CancelTask(TaskSpecification task_spec, bool recursive); + void CancelTask(TaskSpecification task_spec, bool recursive); /// Retry the CancelTask in milliseconds. void RetryCancelTask(TaskSpecification task_spec, bool recursive, int64_t milliseconds); diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 4796a41dcd39..36a8d8927595 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -678,9 +678,9 @@ bool NormalTaskSubmitter::HandleGetWorkerFailureCause( fail_immediately); } -Status NormalTaskSubmitter::CancelTask(TaskSpecification task_spec, - bool force_kill, - bool recursive) { +void NormalTaskSubmitter::CancelTask(TaskSpecification task_spec, + bool force_kill, + bool recursive) { const auto task_id = task_spec.TaskId(); RAY_LOG(INFO) << "Cancelling a task: " << task_id << " force_kill: " << force_kill << " recursive: " << recursive; @@ -695,13 +695,13 @@ Status NormalTaskSubmitter::CancelTask(TaskSpecification task_spec, // For idempotency. if (cancelled_tasks_.contains(task_id)) { // The task cancel is already in progress. We don't need to do anything. - return Status::OK(); + return; } task_manager_.MarkTaskCanceled(task_id); if (!task_manager_.IsTaskPending(task_id)) { // The task is finished or failed so marking the task as cancelled is sufficient. - return Status::OK(); + return; } auto &scheduling_key_entry = scheduling_key_entries_[scheduling_key]; @@ -714,7 +714,7 @@ Status NormalTaskSubmitter::CancelTask(TaskSpecification task_spec, scheduling_tasks.erase(spec); CancelWorkerLeaseIfNeeded(scheduling_key); task_manager_.FailPendingTask(task_id, rpc::ErrorType::TASK_CANCELLED); - return Status::OK(); + return; } } } @@ -742,7 +742,7 @@ Status NormalTaskSubmitter::CancelTask(TaskSpecification task_spec, // scheduling_key_entries_ hashmap. scheduling_key_entries_.erase(scheduling_key); } - return Status::OK(); + return; } // Looks for an RPC handle for the worker executing the task. client = core_worker_client_pool_->GetOrConnect(rpc_client->second); @@ -792,20 +792,18 @@ Status NormalTaskSubmitter::CancelTask(TaskSpecification task_spec, } } }); - return Status::OK(); } -Status NormalTaskSubmitter::CancelRemoteTask(const ObjectID &object_id, - const rpc::Address &worker_addr, - bool force_kill, - bool recursive) { +void NormalTaskSubmitter::CancelRemoteTask(const ObjectID &object_id, + const rpc::Address &worker_addr, + bool force_kill, + bool recursive) { auto client = core_worker_client_pool_->GetOrConnect(worker_addr); auto request = rpc::RemoteCancelTaskRequest(); request.set_force_kill(force_kill); request.set_recursive(recursive); request.set_remote_object_id(object_id.Binary()); client->RemoteCancelTask(request, nullptr); - return Status::OK(); } bool NormalTaskSubmitter::QueueGeneratorForResubmit(const TaskSpecification &spec) { diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.h b/src/ray/core_worker/task_submission/normal_task_submitter.h index 856c322cfd23..197ed02dd2f9 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.h +++ b/src/ray/core_worker/task_submission/normal_task_submitter.h @@ -25,11 +25,9 @@ #include "absl/base/thread_annotations.h" #include "ray/common/id.h" -#include "ray/core_worker/actor_manager.h" -#include "ray/core_worker/context.h" #include "ray/core_worker/lease_policy.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" -#include "ray/core_worker/task_manager.h" +#include "ray/core_worker/task_manager_interface.h" #include "ray/core_worker/task_submission/dependency_resolver.h" #include "ray/raylet_client/raylet_client.h" #include "ray/rpc/node_manager/raylet_client_pool.h" @@ -123,16 +121,16 @@ class NormalTaskSubmitter { /// /// \param[in] task_spec The task to kill. /// \param[in] force_kill Whether to kill the worker executing the task. - Status CancelTask(TaskSpecification task_spec, bool force_kill, bool recursive); + void CancelTask(TaskSpecification task_spec, bool force_kill, bool recursive); /// Request the owner of the object ID to cancel a request. /// It is used when a object ID is not owned by the current process. /// We cannot cancel the task in this case because we don't have enough /// information to cancel a task. - Status CancelRemoteTask(const ObjectID &object_id, - const rpc::Address &worker_addr, - bool force_kill, - bool recursive); + void CancelRemoteTask(const ObjectID &object_id, + const rpc::Address &worker_addr, + bool force_kill, + bool recursive); /// Queue the streaming generator up for resubmission. /// \return true if the task is still executing and the submitter agrees to resubmit diff --git a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc index a3af7a15b838..a42a54991cf5 100644 --- a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc @@ -690,7 +690,7 @@ TEST_F(NormalTaskSubmitterTest, TestCancellationWhileHandlingTaskFailure) { // GetWorkerFailureCause is called. ASSERT_TRUE(worker_client->ReplyPushTask(Status::IOError("oops"))); // Cancel the task while GetWorkerFailureCause has not been completed. - ASSERT_TRUE(submitter.CancelTask(task, true, false).ok()); + submitter.CancelTask(task, true, false); // Completing the GetWorkerFailureCause call. Check that the reply runs without error // and FailPendingTask is not called. ASSERT_TRUE(raylet_client->ReplyGetWorkerFailureCause()); @@ -1738,7 +1738,7 @@ TEST_F(NormalTaskSubmitterTest, TestKillExecutingTask) { ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); // Try force kill, exiting the worker - ASSERT_TRUE(submitter.CancelTask(task, true, false).ok()); + submitter.CancelTask(task, true, false); ASSERT_EQ(worker_client->kill_requests.front().intended_task_id(), task.TaskIdBinary()); ASSERT_TRUE(worker_client->ReplyPushTask(Status::IOError("workerdying"), true)); ASSERT_TRUE(raylet_client->ReplyGetWorkerFailureCause()); @@ -1755,7 +1755,7 @@ TEST_F(NormalTaskSubmitterTest, TestKillExecutingTask) { ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); // Try non-force kill, worker returns normally - ASSERT_TRUE(submitter.CancelTask(task, false, false).ok()); + submitter.CancelTask(task, false, false); ASSERT_TRUE(worker_client->ReplyPushTask()); ASSERT_EQ(worker_client->kill_requests.front().intended_task_id(), task.TaskIdBinary()); ASSERT_EQ(worker_client->callbacks.size(), 0); @@ -1776,7 +1776,7 @@ TEST_F(NormalTaskSubmitterTest, TestKillPendingTask) { TaskSpecification task = BuildEmptyTaskSpec(); ASSERT_TRUE(submitter.SubmitTask(task).ok()); - ASSERT_TRUE(submitter.CancelTask(task, true, false).ok()); + submitter.CancelTask(task, true, false); ASSERT_EQ(worker_client->kill_requests.size(), 0); ASSERT_EQ(worker_client->callbacks.size(), 0); ASSERT_EQ(raylet_client->num_workers_returned, 0); @@ -1803,7 +1803,7 @@ TEST_F(NormalTaskSubmitterTest, TestKillResolvingTask) { task.GetMutableMessage().add_args()->mutable_object_ref()->set_object_id(obj1.Binary()); ASSERT_TRUE(submitter.SubmitTask(task).ok()); ASSERT_EQ(task_manager->num_inlined_dependencies, 0); - ASSERT_TRUE(submitter.CancelTask(task, true, false).ok()); + submitter.CancelTask(task, true, false); auto data = GenerateRandomObject(); store->Put(*data, obj1); WaitForObjectIdInMemoryStore(*store, obj1); @@ -1841,7 +1841,7 @@ TEST_F(NormalTaskSubmitterTest, TestCancelBeforeAfterQueueGeneratorForResubmit) TaskSpecification task = BuildEmptyTaskSpec(); ASSERT_TRUE(submitter.SubmitTask(task).ok()); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); - ASSERT_TRUE(submitter.CancelTask(task, /*force_kill=*/false, /*recursive=*/true).ok()); + submitter.CancelTask(task, /*force_kill=*/false, /*recursive=*/true); ASSERT_FALSE(submitter.QueueGeneratorForResubmit(task)); worker_client->ReplyCancelTask(); ASSERT_TRUE(submitter.QueueGeneratorForResubmit(task)); @@ -1859,7 +1859,7 @@ TEST_F(NormalTaskSubmitterTest, TestCancelBeforeAfterQueueGeneratorForResubmit) ASSERT_TRUE(submitter.SubmitTask(task2).ok()); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); ASSERT_TRUE(submitter.QueueGeneratorForResubmit(task2)); - ASSERT_TRUE(submitter.CancelTask(task2, /*force_kill=*/false, /*recursive=*/true).ok()); + submitter.CancelTask(task2, /*force_kill=*/false, /*recursive=*/true); ASSERT_TRUE(worker_client->ReplyPushTask()); worker_client->ReplyCancelTask(Status::OK(), /*attempt_succeeded=*/true, From 0b66cc4a79df9a6232e73888f728379c15ea53bb Mon Sep 17 00:00:00 2001 From: Alan Guo Date: Mon, 1 Sep 2025 01:29:49 -0700 Subject: [PATCH 377/634] [JobManager] Make sure JobManager tolerates intermittent failures (#56085) Signed-off-by: Alan Guo --- python/ray/dashboard/BUILD | 12 ++ python/ray/dashboard/modules/job/common.py | 2 + .../ray/dashboard/modules/job/job_manager.py | 153 +++++++++++------- .../modules/job/tests/test_job_manager.py | 97 ++++++++++- 4 files changed, 200 insertions(+), 64 deletions(-) diff --git a/python/ray/dashboard/BUILD b/python/ray/dashboard/BUILD index a443e8f6d8dc..f0e5fc640fb5 100644 --- a/python/ray/dashboard/BUILD +++ b/python/ray/dashboard/BUILD @@ -40,6 +40,7 @@ py_test_run_all_subdirectory( "tests/test_dashboard.py", "tests/test_state_head.py", "modules/serve/tests/**/*.py", + "modules/job/tests/test_job_manager.py", ], extra_srcs = [], tags = [ @@ -60,6 +61,17 @@ py_test( deps = [":conftest"], ) +py_test( + name = "test_job_manager", + size = "large", + srcs = ["modules/job/tests/test_job_manager.py"], + tags = [ + "exclusive", + "team:core", + ], + deps = [":conftest"], +) + py_test( name = "test_http_job_server", size = "large", diff --git a/python/ray/dashboard/modules/job/common.py b/python/ray/dashboard/modules/job/common.py index ee55cb78de13..3f205e6478d2 100644 --- a/python/ray/dashboard/modules/job/common.py +++ b/python/ray/dashboard/modules/job/common.py @@ -80,6 +80,8 @@ class JobErrorType(str, Enum): JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE = "JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE" # Job supervisor actor failed for unknown exception JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE = "JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE" + # Job supervisor actor died + JOB_SUPERVISOR_ACTOR_DIED = "JOB_SUPERVISOR_ACTOR_DIED" # Job driver script failed to start due to exception JOB_ENTRYPOINT_COMMAND_START_ERROR = "JOB_ENTRYPOINT_COMMAND_START_ERROR" # Job driver script failed due to non-zero exit code diff --git a/python/ray/dashboard/modules/job/job_manager.py b/python/ray/dashboard/modules/job/job_manager.py index 8c3a13690768..78aff4850168 100644 --- a/python/ray/dashboard/modules/job/job_manager.py +++ b/python/ray/dashboard/modules/job/job_manager.py @@ -32,7 +32,7 @@ from ray.dashboard.modules.job.job_supervisor import JobSupervisor from ray.dashboard.modules.job.utils import get_head_node_id from ray.dashboard.utils import close_logger_file_descriptor -from ray.exceptions import ActorUnschedulableError, RuntimeEnvSetupError +from ray.exceptions import ActorDiedError, ActorUnschedulableError, RuntimeEnvSetupError from ray.job_submission import JobStatus, JobErrorType from ray.runtime_env import RuntimeEnvConfig from ray.util.scheduling_strategies import ( @@ -43,6 +43,11 @@ logger = logging.getLogger(__name__) +RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES = ray_constants.env_integer( + "RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES", 5 +) + + def generate_job_id() -> str: """Returns a job_id of the form 'raysubmit_XYZ'. @@ -145,6 +150,9 @@ async def _monitor_job( self.monitored_jobs.add(job_id) try: await self._monitor_job_internal(job_id, job_supervisor) + except Exception as e: + logger.error("Unhandled exception in job monitoring!", exc_info=e) + raise e finally: self.monitored_jobs.remove(job_id) @@ -158,10 +166,19 @@ async def _monitor_job_internal( ) ) - is_alive = True + should_monitor = True + num_consecutive_failures = 0 + + job_status = None - while is_alive: + while should_monitor: try: + # NOTE: Job monitoring loop sleeps before proceeding with monitoring + # sequence to consolidate the control-flow of the pacing + # in a single place, rather than having it spread across + # many branches + await asyncio.sleep(self.JOB_MONITOR_LOOP_PERIOD_S) + job_status = await self._job_info_client.get_status(job_id) if job_status == JobStatus.PENDING: # Compare the current time with the job start time. @@ -210,7 +227,7 @@ async def _monitor_job_internal( message=err_msg, error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_TIMEOUT, ) - is_alive = False + should_monitor = False logger.error(err_msg) continue @@ -237,78 +254,102 @@ async def _monitor_job_internal( ), error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_FAILURE, ) - is_alive = False + should_monitor = False continue + # Verify `JobSupervisor` is alive and reachable await job_supervisor.ping.remote() + # Reset consecutive failures counter + num_consecutive_failures = 0 - await asyncio.sleep(self.JOB_MONITOR_LOOP_PERIOD_S) except Exception as e: - is_alive = False - job_status = await self._job_info_client.get_status(job_id) - job_error_message = None - if job_status == JobStatus.FAILED: - job_error_message = ( - "See more details from the dashboard " - "`Job` page or the state API `ray list jobs`." - ) - - job_error_message = "" - if job_status.is_terminal(): + target_job_error_message = "" + target_job_error_type: Optional[JobErrorType] = None + if job_status is not None and job_status.is_terminal(): # If the job is already in a terminal state, then the actor # exiting is expected. + should_monitor = False pass - elif isinstance(e, RuntimeEnvSetupError): - logger.info(f"Failed to set up runtime_env for job {job_id}.") - job_error_message = f"runtime_env setup failed: {e}" - job_status = JobStatus.FAILED - await self._job_info_client.put_status( - job_id, - job_status, - message=job_error_message, - error_type=JobErrorType.RUNTIME_ENV_SETUP_FAILURE, - ) - elif isinstance(e, ActorUnschedulableError): - logger.info( - f"Failed to schedule job {job_id} because the supervisor actor " - f"could not be scheduled: {e}" - ) - job_error_message = ( - f"Job supervisor actor could not be scheduled: {e}" - ) - await self._job_info_client.put_status( - job_id, - JobStatus.FAILED, - message=job_error_message, - error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE, - ) else: - logger.warning( - f"Job supervisor for job {job_id} failed unexpectedly: {e}." - ) - job_error_message = f"Unexpected error occurred: {e}" - job_status = JobStatus.FAILED - await self._job_info_client.put_status( - job_id, - job_status, - message=job_error_message, - error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE, - ) + if isinstance(e, RuntimeEnvSetupError): + logger.error(f"Failed to set up runtime_env for job {job_id}.") + + target_job_error_message = f"runtime_env setup failed: {e}" + target_job_error_type = JobErrorType.RUNTIME_ENV_SETUP_FAILURE + + elif isinstance(e, ActorUnschedulableError): + logger.error( + f"Failed to schedule job {job_id} because the supervisor " + f"actor could not be scheduled: {e}" + ) + + target_job_error_message = ( + f"Job supervisor actor could not be scheduled: {e}" + ) + target_job_error_type = ( + JobErrorType.JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE + ) + + elif isinstance(e, ActorDiedError): + logger.error(f"Job supervisor actor for {job_id} died: {e}") + target_job_error_message = f"Job supervisor actor died: {e}" + target_job_error_type = JobErrorType.JOB_SUPERVISOR_ACTOR_DIED + + else: + logger.warning( + f"Job monitoring for job {job_id} failed " + f"unexpectedly: {e}.", + exc_info=e, + ) + + if ( + num_consecutive_failures + < RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES + ): + num_consecutive_failures += 1 + continue + else: + logger.error( + f"Job monitoring failed more than " + f"{RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES} " + f"times, marking job as failed", + exc_info=e, + ) + + target_job_error_message = f"Unexpected error occurred: {e}" + target_job_error_type = ( + JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE + ) + + # If target job error message is set it entails that the job ought + # to be marked as failed (and terminated) + if target_job_error_message: + # Terminate monitoring loop + should_monitor = False + + job_status = JobStatus.FAILED + await self._job_info_client.put_status( + job_id, + job_status, + message=target_job_error_message, + error_type=target_job_error_type + or JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE, + ) # Log error message to the job driver file for easy access. - if job_error_message: + if target_job_error_message: log_path = self._log_client.get_log_file_path(job_id) os.makedirs(os.path.dirname(log_path), exist_ok=True) with open(log_path, "a") as log_file: - log_file.write(job_error_message) + log_file.write(target_job_error_message) # Log events if self.event_logger: event_log = ( f"Completed a ray job {job_id} with a status {job_status}." ) - if job_error_message: - event_log += f" {job_error_message}" + if target_job_error_message: + event_log += f" {target_job_error_message}" self.event_logger.error(event_log, submission_id=job_id) else: self.event_logger.info(event_log, submission_id=job_id) diff --git a/python/ray/dashboard/modules/job/tests/test_job_manager.py b/python/ray/dashboard/modules/job/tests/test_job_manager.py index d0ed6271a8ff..fb13cbb53df9 100644 --- a/python/ray/dashboard/modules/job/tests/test_job_manager.py +++ b/python/ray/dashboard/modules/job/tests/test_job_manager.py @@ -5,6 +5,7 @@ import tempfile import time import urllib.request +from unittest.mock import AsyncMock from uuid import uuid4 import pytest @@ -27,6 +28,7 @@ ) from ray.dashboard.modules.job.common import JOB_ID_METADATA_KEY, JOB_NAME_METADATA_KEY from ray.dashboard.modules.job.job_manager import ( + RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES, JobLogStorageClient, JobManager, JobSupervisor, @@ -37,6 +39,7 @@ create_job_manager, create_ray_cluster, ) +from ray.exceptions import RpcError from ray.job_submission import JobStatus, JobErrorType from ray.tests.conftest import call_ray_start # noqa: F401 from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy # noqa: F401 @@ -350,14 +353,15 @@ async def test_runtime_env_setup_logged_to_job_driver_logs( assert start_message in logs -@pytest.fixture(scope="module") +@pytest.fixture def shared_ray_instance(): # Remove ray address for test ray cluster in case we have # lingering RAY_ADDRESS="http://127.0.0.1:8265" from previous local job # submissions. old_ray_address = os.environ.pop(RAY_ADDRESS_ENVIRONMENT_VARIABLE, None) - yield create_ray_cluster() + with create_ray_cluster() as cluster: + yield cluster if old_ray_address is not None: os.environ[RAY_ADDRESS_ENVIRONMENT_VARIABLE] = old_ray_address @@ -365,7 +369,10 @@ def shared_ray_instance(): @pytest.fixture def job_manager(shared_ray_instance, tmp_path): - yield create_job_manager(shared_ray_instance, tmp_path) + job_manager = create_job_manager(shared_ray_instance, tmp_path) + job_manager.JOB_MONITOR_LOOP_PERIOD_S = 0.01 + + yield job_manager async def _run_hanging_command(job_manager, tmp_dir, start_signal_actor=None): @@ -400,7 +407,14 @@ async def _run_hanging_command(job_manager, tmp_dir, start_signal_actor=None): async def check_job_succeeded(job_manager, job_id): - data = await job_manager.get_job_info(job_id) + return await _check_job_succeeded( + get_job_info=job_manager.get_job_info, job_id=job_id + ) + + +async def _check_job_succeeded(*, get_job_info, job_id: str): + data = await get_job_info(job_id) + status = data.status if status == JobStatus.FAILED: raise RuntimeError(f"Job failed! {data.message}") @@ -413,7 +427,15 @@ async def check_job_succeeded(job_manager, job_id): async def check_job_failed(job_manager, job_id, expected_error_type=None): - data = await job_manager.get_job_info(job_id) + return await _check_job_failed( + get_job_info=job_manager.get_job_info, + job_id=job_id, + expected_error_type=expected_error_type, + ) + + +async def _check_job_failed(*, get_job_info, job_id: str, expected_error_type=None): + data = await get_job_info(job_id) status = data.status assert status in {JobStatus.PENDING, JobStatus.RUNNING, JobStatus.FAILED} if expected_error_type: @@ -889,7 +911,7 @@ async def test_kill_job_actor_in_before_driver_finish(self, job_manager): check_job_failed, job_manager=job_manager, job_id=job_id, - expected_error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE, + expected_error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_DIED, ) data = await job_manager.get_job_info(job_id) assert data.driver_exit_code is None @@ -946,10 +968,15 @@ async def test_kill_job_actor_in_pending(self, job_manager): check_job_failed, job_manager=job_manager, job_id=job_id, - expected_error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE, + expected_error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_DIED, ) data = await job_manager.get_job_info(job_id) + assert data.driver_exit_code is None + assert data.message.startswith( + "Job supervisor actor died: The actor died unexpectedly before " + "finishing this task" + ) async def test_stop_job_subprocess_cleanup_upon_stop(self, job_manager): """ @@ -1416,7 +1443,6 @@ async def test_actor_creation_error_not_overwritten(shared_ray_instance, tmp_pat assert data.driver_exit_code is None -@pytest.mark.asyncio async def test_no_task_events_exported(shared_ray_instance, tmp_path): """Verify that no task events are exported by the JobSupervisor.""" job_manager = create_job_manager(shared_ray_instance, tmp_path) @@ -1433,5 +1459,60 @@ async def test_no_task_events_exported(shared_ray_instance, tmp_path): assert "JobSupervisor" not in t.name +@pytest.mark.parametrize( + "max_failures,expected_job_status", + [ + (RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES - 1, JobStatus.SUCCEEDED), + (RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES + 1, JobStatus.FAILED), + ], +) +async def test_job_manager_tolerates_gcs_failures( + job_manager, max_failures, expected_job_status +): + """Test driver exit code from finished task that failed""" + + original_get_info = job_manager._job_info_client.get_info + + num_failures = 0 + + async def _failing_get_info(*args, **kwargs): + nonlocal num_failures + + if num_failures < max_failures: + num_failures += 1 + raise RpcError("deadline exceeded") + else: + return await original_get_info(*args, **kwargs) + + # Mock out `JobManager._job_info_client` + job_manager._job_info_client.get_info = AsyncMock(side_effect=_failing_get_info) + + # Override `JobManager`s monitoring frequency to 100ms + job_manager.JOB_MONITOR_LOOP_PERIOD_S = 0.1 + + # Simulate job running for 5 seconds + job_id = await job_manager.submit_job(entrypoint="sleep 3; echo 'hello world'") + + if expected_job_status == JobStatus.FAILED: + expected_job_state_check = _check_job_failed + elif expected_job_status == JobStatus.SUCCEEDED: + expected_job_state_check = _check_job_succeeded + else: + raise NotImplementedError(f"unexpected job status: {expected_job_status}") + + # Wait for the job to reach expected target state + await async_wait_for_condition( + expected_job_state_check, + timeout=10, + get_job_info=original_get_info, + job_id=job_id, + ) + + # Check that the job failed + job_info = await job_manager.get_job_info(job_id) + + assert job_info.status == expected_job_status + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) From dd3ae1f6faa0e15418b5a11a7a6f85bde9591182 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Mon, 1 Sep 2025 01:47:17 -0700 Subject: [PATCH 378/634] [core] Fix named actor test for windows (#56134) Signed-off-by: dayshah --- python/ray/tests/test_actor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/ray/tests/test_actor.py b/python/ray/tests/test_actor.py index 5c738808ad8b..175f75c7c518 100644 --- a/python/ray/tests/test_actor.py +++ b/python/ray/tests/test_actor.py @@ -2,7 +2,6 @@ import random import sys import tempfile -import signal import numpy as np import pytest @@ -1695,7 +1694,7 @@ def get_pid(self): a = Actor.options(name=ACTOR_NAME, max_restarts=0, max_task_retries=-1).remote() pid = ray.get(a.get_pid.remote()) - os.kill(pid, signal.SIGKILL) + psutil.Process(pid).kill() a_actor_id = a._actor_id.hex() wait_for_condition(lambda: ray.state.actors(a_actor_id)["State"] == "DEAD") @@ -1728,7 +1727,7 @@ def wait_new_actor_ready(): _ = ray.get_actor(ACTOR_NAME, namespace=NAMESPACE_NAME) pid = ray.get(c.get_pid.remote()) - os.kill(pid, signal.SIGKILL) + psutil.Process(pid).kill() wait_for_condition(lambda: ray.state.actors(c._actor_id.hex())["State"] == "DEAD") From 89fef22c869a732bd22582a9876616beb5f34a71 Mon Sep 17 00:00:00 2001 From: Hassam Ullah Sheikh Date: Mon, 1 Sep 2025 08:05:21 -0400 Subject: [PATCH 379/634] Fix/metric set state issue (#55892) ## Why are these changes needed? ## Related issue number Closes #55248 --------- Signed-off-by: Hassam Sheikh Co-authored-by: Hassam Sheikh Co-authored-by: Kamil Kaczmarek --- .gitignore | 2 +- rllib/utils/metrics/metrics_logger.py | 2 + rllib/utils/metrics/tests/test_stats.py | 89 +++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e33fde76315d..81d26138d2cf 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,7 @@ python/ray/autoscaler/kuberay/config # Python byte code files *.pyc python/.eggs - +.eggs # Backup files *.bak diff --git a/rllib/utils/metrics/metrics_logger.py b/rllib/utils/metrics/metrics_logger.py index 28432ea7cca3..9d3ba1c4e8f9 100644 --- a/rllib/utils/metrics/metrics_logger.py +++ b/rllib/utils/metrics/metrics_logger.py @@ -1167,6 +1167,8 @@ def set_state(self, state: Dict[str, Any]) -> None: state: The state to set `self` to. """ with self._threading_lock: + # Reset all existing stats to ensure a clean state transition + self.stats = {} for flat_key, stats_state in state["stats"].items(): self._set_key(flat_key.split("--"), Stats.from_state(stats_state)) diff --git a/rllib/utils/metrics/tests/test_stats.py b/rllib/utils/metrics/tests/test_stats.py index b43a0a15559a..e85783a238f1 100644 --- a/rllib/utils/metrics/tests/test_stats.py +++ b/rllib/utils/metrics/tests/test_stats.py @@ -3,6 +3,7 @@ import numpy as np from ray.rllib.utils.metrics.stats import Stats, merge_stats +from ray.rllib.utils.metrics.metrics_logger import MetricsLogger from ray.rllib.utils.test_utils import check # Default values used throughout the tests @@ -1190,6 +1191,94 @@ def test_percentiles(): ) +def test_set_state_complete_replacement(): + """Test that set_state() completely replaces the logger's state. + + This test verifies the fix for the issue where set_state() would only update + keys present in the new state but leave old keys intact, causing stale data + to persist after checkpoint restoration. + """ + # Test case 1: Basic replacement with fewer keys + logger1 = MetricsLogger() + logger1.log_value("solo", 0) + logger1.log_value("duo", 0) + + logger2 = MetricsLogger() + logger2.log_value("duo", 1) + + # Before fix: {'solo': 0, 'duo': 1} - 'solo' would persist + # After fix: {'duo': 1} - only new state keys remain + logger1.set_state(logger2.get_state()) + result = logger1.peek() + expected = {"duo": 1} + + check(result, expected) + + # Test case 2: Complete replacement with different keys + logger3 = MetricsLogger() + logger3.log_value("old_key1", 10) + logger3.log_value("old_key2", 20) + logger3.log_value("shared_key", 30) + + logger4 = MetricsLogger() + logger4.log_value("shared_key", 100) + logger4.log_value("new_key", 200) + + logger3.set_state(logger4.get_state()) + result = logger3.peek() + expected = {"shared_key": 100, "new_key": 200} + + check(result, expected) + + # Test case 3: Setting to empty state + logger5 = MetricsLogger() + logger5.log_value("key1", 1) + logger5.log_value("key2", 2) + + empty_logger = MetricsLogger() + logger5.set_state(empty_logger.get_state()) + result = logger5.peek() + + check(result, {}) + + # Test case 4: Nested keys + logger6 = MetricsLogger() + logger6.log_value(("nested", "old_key"), 1) + logger6.log_value(("nested", "shared_key"), 2) + logger6.log_value("top_level", 3) + + logger7 = MetricsLogger() + logger7.log_value(("nested", "shared_key"), 20) + logger7.log_value(("nested", "new_key"), 30) + + logger6.set_state(logger7.get_state()) + result = logger6.peek() + expected = {"nested": {"shared_key": 20, "new_key": 30}} + + check(result, expected) + + # Test case 5: Multiple set_state calls (simulating multiple restore_from_path calls) + logger8 = MetricsLogger() + logger8.log_value("initial", 0) + + # First set_state + temp1 = MetricsLogger() + temp1.log_value("first", 1) + temp1.log_value("shared", 100) + logger8.set_state(temp1.get_state()) + + # Second set_state - should completely replace first state + temp2 = MetricsLogger() + temp2.log_value("second", 2) + temp2.log_value("shared", 20) + logger8.set_state(temp2.get_state()) + + result = logger8.peek() + expected = {"second": 2, "shared": 20} + + check(result, expected) + + if __name__ == "__main__": import sys From 686cd6e8587f95a1ac403243c52bfcc5302f3342 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Mon, 1 Sep 2025 12:43:23 -0700 Subject: [PATCH 380/634] [core] Cleanup / detangle flatbuffer utilties (#55794) Signed-off-by: dayshah --- .pre-commit-config.yaml | 2 +- src/ray/common/BUILD.bazel | 17 +- src/ray/common/common_protocol.cc | 21 --- src/ray/common/common_protocol.h | 173 ------------------ src/ray/common/constants.h | 3 +- src/ray/common/flatbuf_utils.h | 72 ++++++++ src/ray/common/id.cc | 6 - src/ray/common/id.h | 21 ++- src/ray/common/lease/lease_spec.cc | 1 - src/ray/common/tests/id_test.cc | 3 +- src/ray/core_worker/task_manager.cc | 1 - src/ray/gcs/gcs_client/accessor.cc | 12 +- .../gcs/store_client/in_memory_store_client.h | 1 - src/ray/gcs/store_client/redis_context.h | 3 +- .../gcs/store_client/redis_store_client.cc | 2 +- src/ray/gcs/store_client/redis_store_client.h | 7 +- .../tests/store_client_test_base.h | 1 + src/ray/ipc/BUILD.bazel | 1 + src/ray/ipc/client_connection.h | 3 +- src/ray/ipc/fake_raylet_ipc_client.h | 4 + src/ray/ipc/raylet_ipc_client.cc | 102 ++++++----- src/ray/ipc/raylet_ipc_client.h | 2 - src/ray/object_manager/object_manager.cc | 1 - src/ray/object_manager/pull_manager.cc | 1 - .../object_manager/tests/pull_manager_test.cc | 1 - src/ray/raylet/BUILD.bazel | 1 + src/ray/raylet/lease_dependency_manager.cc | 11 +- src/ray/raylet/lease_dependency_manager.h | 23 +-- src/ray/raylet/main.cc | 8 +- src/ray/raylet/node_manager.cc | 170 ++++++++--------- src/ray/raylet/node_manager.h | 2 - src/ray/raylet_client/raylet_client.cc | 2 - 32 files changed, 277 insertions(+), 401 deletions(-) delete mode 100644 src/ray/common/common_protocol.cc delete mode 100644 src/ray/common/common_protocol.h create mode 100644 src/ray/common/flatbuf_utils.h diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 97f83258dffb..30ef94097deb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -73,7 +73,7 @@ repos: hooks: - id: cpplint args: ["--filter=-whitespace/braces,-whitespace/line_length,-build/c++11,-build/c++14,-build/c++17,-readability/braces,-whitespace/indent_namespace,-runtime/int,-runtime/references,-build/include_order"] - files: ^src/ray/(common/cgroup2|common/scheduling|common/ray_syncer|common/test|util|raylet_client|internal|scheduling|pubsub|object_manager|rpc(?:/.*)?|raylet|core_worker)/.*\.(h|cc)$ + files: ^src/ray/(common/cgroup2|common/scheduling|common/ray_syncer|common/test|util|raylet_client|internal|scheduling|pubsub|object_manager|rpc(?:/.*)?|raylet|core_worker|ipc)/.*\.(h|cc)$ exclude: | (?x)^( src/ray/raylet/scheduling/.*\.(h|cc)$ | diff --git a/src/ray/common/BUILD.bazel b/src/ray/common/BUILD.bazel index 9b8a89329019..e7baf5412576 100644 --- a/src/ray/common/BUILD.bazel +++ b/src/ray/common/BUILD.bazel @@ -119,26 +119,33 @@ ray_cc_library( ray_cc_library( name = "id", srcs = [ - "common_protocol.cc", "id.cc", ], hdrs = [ - "common_protocol.h", "id.h", "id_def.h", ], deps = [ ":constants", - ":status", "//src/ray/protobuf:common_cc_proto", - "//src/ray/protobuf:gcs_cc_proto", "//src/ray/thirdparty:sha256", + "//src/ray/util:logging", "//src/ray/util:random", - "@com_github_google_flatbuffers//:flatbuffers", + "//src/ray/util:visibility", "@msgpack", ], ) +ray_cc_library( + name = "flatbuf_utils", + hdrs = [ + "flatbuf_utils.h", + ], + deps = [ + "@com_github_google_flatbuffers//:flatbuffers", + ], +) + # TODO(#55922): split out the scheduling dependencies into a separate bazel target # and have lease and task bazel targets depend on this diff --git a/src/ray/common/common_protocol.cc b/src/ray/common/common_protocol.cc deleted file mode 100644 index 03043efc2dc0..000000000000 --- a/src/ray/common/common_protocol.cc +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ray/common/common_protocol.h" - -#include "ray/util/logging.h" - -std::string string_from_flatbuf(const flatbuffers::String &string) { - return std::string(string.data(), string.size()); -} diff --git a/src/ray/common/common_protocol.h b/src/ray/common/common_protocol.h deleted file mode 100644 index e5c06e6fc401..000000000000 --- a/src/ray/common/common_protocol.h +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include - -#include "ray/common/id.h" -#include "ray/util/logging.h" -#include "src/ray/protobuf/common.pb.h" -#include "src/ray/protobuf/gcs.pb.h" - -/// Convert an unique ID to a flatbuffer string. -/// -/// @param fbb Reference to the flatbuffer builder. -/// @param id The ID to be converted. -/// @return The flatbuffer string containing the ID. -template -flatbuffers::Offset to_flatbuf(flatbuffers::FlatBufferBuilder &fbb, - ID id); - -/// Convert a flatbuffer string to an unique ID. -/// -/// @param string The flatbuffer string. -/// @return The ID. -template -ID from_flatbuf(const flatbuffers::String &string); - -/// Convert a flatbuffer vector of strings to a vector of unique IDs. -/// -/// @param vector The flatbuffer vector. -/// @return The vector of IDs. -template -const std::vector from_flatbuf( - const flatbuffers::Vector> &vector); - -/// Convert an array of unique IDs to a flatbuffer vector of strings. -/// -/// @param fbb Reference to the flatbuffer builder. -/// @param ids Array of unique IDs. -/// @param num_ids Number of elements in the array. -/// @return Flatbuffer vector of strings. -template -flatbuffers::Offset>> -to_flatbuf(flatbuffers::FlatBufferBuilder &fbb, ID ids[], int64_t num_ids); - -/// Convert a vector of unique IDs to a flatbuffer vector of strings. -/// -/// @param fbb Reference to the flatbuffer builder. -/// @param ids Vector of IDs. -/// @return Flatbuffer vector of strings. -template -flatbuffers::Offset>> -to_flatbuf(flatbuffers::FlatBufferBuilder &fbb, const std::vector &ids); - -/// Convert an unordered_set of unique IDs to a flatbuffer vector of strings. -/// -/// @param fbb Reference to the flatbuffer builder. -/// @param ids Unordered set of IDs. -/// @return Flatbuffer vector of strings. -template -flatbuffers::Offset>> -to_flatbuf(flatbuffers::FlatBufferBuilder &fbb, const std::unordered_set &ids); - -/// Convert a flatbuffer string to a std::string. -/// -/// @param fbb Reference to the flatbuffer builder. -/// @param string A flatbuffers string. -/// @return The std::string version of the flatbuffer string. -std::string string_from_flatbuf(const flatbuffers::String &string); - -template -flatbuffers::Offset to_flatbuf(flatbuffers::FlatBufferBuilder &fbb, - ID id) { - return fbb.CreateString(reinterpret_cast(id.Data()), id.Size()); -} - -template -ID from_flatbuf(const flatbuffers::String &string) { - return ID::FromBinary(string.str()); -} - -template -const std::vector from_flatbuf( - const flatbuffers::Vector> &vector) { - std::vector ids; - for (int64_t i = 0; i < vector.size(); i++) { - ids.push_back(from_flatbuf(*vector.Get(i))); - } - return ids; -} - -template -flatbuffers::Offset>> -to_flatbuf(flatbuffers::FlatBufferBuilder &fbb, ID ids[], int64_t num_ids) { - std::vector> results; - for (int64_t i = 0; i < num_ids; i++) { - results.push_back(to_flatbuf(fbb, ids[i])); - } - return fbb.CreateVector(results); -} - -template -flatbuffers::Offset>> -to_flatbuf(flatbuffers::FlatBufferBuilder &fbb, const std::vector &ids) { - std::vector> results; - for (auto id : ids) { - results.push_back(to_flatbuf(fbb, id)); - } - return fbb.CreateVector(results); -} - -template -flatbuffers::Offset>> -to_flatbuf(flatbuffers::FlatBufferBuilder &fbb, const std::unordered_set &ids) { - std::vector> results; - for (auto id : ids) { - results.push_back(to_flatbuf(fbb, id)); - } - return fbb.CreateVector(results); -} - -static inline ray::rpc::ObjectReference ObjectIdToRef( - const ray::ObjectID &object_id, const ray::rpc::Address owner_address) { - ray::rpc::ObjectReference ref; - ref.set_object_id(object_id.Binary()); - ref.mutable_owner_address()->CopyFrom(owner_address); - return ref; -} - -static inline ray::ObjectID ObjectRefToId(const ray::rpc::ObjectReference &object_ref) { - return ray::ObjectID::FromBinary(object_ref.object_id()); -} - -static inline std::vector ObjectRefsToIds( - const std::vector &object_refs) { - std::vector object_ids; - for (const auto &ref : object_refs) { - object_ids.push_back(ObjectRefToId(ref)); - } - return object_ids; -} - -static inline ray::rpc::ActorTableData::ActorState StringToActorState( - const std::string &actor_state_name) { - if (actor_state_name == "DEPENDENCIES_UNREADY") { - return ray::rpc::ActorTableData::DEPENDENCIES_UNREADY; - } else if (actor_state_name == "PENDING_CREATION") { - return ray::rpc::ActorTableData::PENDING_CREATION; - } else if (actor_state_name == "ALIVE") { - return ray::rpc::ActorTableData::ALIVE; - } else if (actor_state_name == "RESTARTING") { - return ray::rpc::ActorTableData::RESTARTING; - } else if (actor_state_name == "DEAD") { - return ray::rpc::ActorTableData::DEAD; - } else { - RAY_CHECK(false) << "Invalid actor state name:" << actor_state_name; - return {}; - } -} diff --git a/src/ray/common/constants.h b/src/ray/common/constants.h index 9ee40fd92673..aa9d858f2811 100644 --- a/src/ray/common/constants.h +++ b/src/ray/common/constants.h @@ -14,8 +14,7 @@ #pragma once -#include -#include +#include /// Default value for enable_task_events within core. constexpr bool kDefaultTaskEventEnabled = true; diff --git a/src/ray/common/flatbuf_utils.h b/src/ray/common/flatbuf_utils.h new file mode 100644 index 000000000000..7a1d56854a16 --- /dev/null +++ b/src/ray/common/flatbuf_utils.h @@ -0,0 +1,72 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include + +namespace ray { + +namespace flatbuf { + +using flatbuffers::FlatBufferBuilder; +using flatbuffers::Offset; +using flatbuffers::String; +using flatbuffers::uoffset_t; +using flatbuffers::Vector; + +template +Offset to_flatbuf(FlatBufferBuilder &fbb, const ID &id) { + return fbb.CreateString(reinterpret_cast(id.Data()), id.Size()); +} + +template +Offset>> to_flatbuf(FlatBufferBuilder &fbb, + ID ids[], + int64_t num_ids) { + std::vector> results; + results.reserve(num_ids); + for (int64_t i = 0; i < num_ids; i++) { + results.push_back(to_flatbuf(fbb, ids[i])); + } + return fbb.CreateVector(results); +} + +template +Offset>> to_flatbuf(FlatBufferBuilder &fbb, + const std::vector &ids) { + std::vector> results; + results.reserve(ids.size()); + for (const auto &id : ids) { + results.push_back(to_flatbuf(fbb, id)); + } + return fbb.CreateVector(results); +} + +template +Offset>> to_flatbuf(FlatBufferBuilder &fbb, + const std::unordered_set &ids) { + std::vector> results; + results.reserve(ids.size()); + for (const auto &id : ids) { + results.push_back(to_flatbuf(fbb, id)); + } + return fbb.CreateVector(results); +} + +} // namespace flatbuf + +} // namespace ray diff --git a/src/ray/common/id.cc b/src/ray/common/id.cc index 646fe65abb78..9883ef0c26dc 100644 --- a/src/ray/common/id.cc +++ b/src/ray/common/id.cc @@ -14,16 +14,10 @@ #include "ray/common/id.h" -#include - #include -#include -#include -#include #include "absl/time/clock.h" #include "ray/common/constants.h" -#include "ray/common/status.h" #include "ray/util/macros.h" extern "C" { diff --git a/src/ray/common/id.h b/src/ray/common/id.h index e01ee20a2ff1..8e89d7e55cca 100644 --- a/src/ray/common/id.h +++ b/src/ray/common/id.h @@ -14,20 +14,15 @@ #pragma once -#include -#include - -#include #include #include -#include -#include #include #include "ray/common/constants.h" #include "ray/util/logging.h" #include "ray/util/random.h" #include "ray/util/visibility.h" +#include "src/ray/protobuf/common.pb.h" namespace ray { @@ -588,6 +583,20 @@ struct DefaultLogKey { constexpr static std::string_view key = kLogKeyLeaseID; }; +inline ObjectID ObjectRefToId(const rpc::ObjectReference &object_ref) { + return ObjectID::FromBinary(object_ref.object_id()); +} + +inline std::vector ObjectRefsToIds( + const std::vector &object_refs) { + std::vector object_ids; + object_ids.reserve(object_refs.size()); + for (const auto &ref : object_refs) { + object_ids.push_back(ObjectRefToId(ref)); + } + return object_ids; +} + } // namespace ray namespace std { diff --git a/src/ray/common/lease/lease_spec.cc b/src/ray/common/lease/lease_spec.cc index 8dcf2552d472..967ce54b284a 100644 --- a/src/ray/common/lease/lease_spec.cc +++ b/src/ray/common/lease/lease_spec.cc @@ -14,7 +14,6 @@ #include "ray/common/lease/lease_spec.h" -#include "ray/common/common_protocol.h" #include "ray/common/function_descriptor.h" #include "ray/common/runtime_env_common.h" diff --git a/src/ray/common/tests/id_test.cc b/src/ray/common/tests/id_test.cc index eee4b98e5b5b..3cbf782eb5a3 100644 --- a/src/ray/common/tests/id_test.cc +++ b/src/ray/common/tests/id_test.cc @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "ray/common/id.h" + #include #include #include "absl/container/flat_hash_set.h" -#include "ray/common/common_protocol.h" namespace ray { diff --git a/src/ray/core_worker/task_manager.cc b/src/ray/core_worker/task_manager.cc index 56aa59d5b495..8a912f74c585 100644 --- a/src/ray/core_worker/task_manager.cc +++ b/src/ray/core_worker/task_manager.cc @@ -23,7 +23,6 @@ #include "absl/strings/match.h" #include "ray/common/buffer.h" -#include "ray/common/common_protocol.h" #include "ray/core_worker/actor_manager.h" #include "ray/gcs/pb_util.h" #include "ray/util/exponential_backoff.h" diff --git a/src/ray/gcs/gcs_client/accessor.cc b/src/ray/gcs/gcs_client/accessor.cc index 75f97ea938bf..e2a3fbf39387 100644 --- a/src/ray/gcs/gcs_client/accessor.cc +++ b/src/ray/gcs/gcs_client/accessor.cc @@ -21,7 +21,6 @@ #include #include -#include "ray/common/common_protocol.h" #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/util/container_util.h" @@ -200,9 +199,14 @@ void ActorInfoAccessor::AsyncGetAllByFilter( request.mutable_filters()->set_job_id(job_id.value().Binary()); } if (actor_state_name) { - rpc::ActorTableData::ActorState actor_state = - StringToActorState(actor_state_name.value()); - request.mutable_filters()->set_state(actor_state); + static absl::flat_hash_map + actor_state_map = { + {"DEPENDENCIES_UNREADY", rpc::ActorTableData::DEPENDENCIES_UNREADY}, + {"PENDING_CREATION", rpc::ActorTableData::PENDING_CREATION}, + {"ALIVE", rpc::ActorTableData::ALIVE}, + {"RESTARTING", rpc::ActorTableData::RESTARTING}, + {"DEAD", rpc::ActorTableData::DEAD}}; + request.mutable_filters()->set_state(actor_state_map[*actor_state_name]); } client_impl_->GetGcsRpcClient().GetAllActorInfo( diff --git a/src/ray/gcs/store_client/in_memory_store_client.h b/src/ray/gcs/store_client/in_memory_store_client.h index 0eb2904735b2..d956ad40752c 100644 --- a/src/ray/gcs/store_client/in_memory_store_client.h +++ b/src/ray/gcs/store_client/in_memory_store_client.h @@ -22,7 +22,6 @@ #include "absl/synchronization/mutex.h" #include "ray/gcs/store_client/store_client.h" #include "ray/util/concurrent_flat_map.h" -#include "src/ray/protobuf/gcs.pb.h" namespace ray::gcs { diff --git a/src/ray/gcs/store_client/redis_context.h b/src/ray/gcs/store_client/redis_context.h index d66f3a04bf12..343465e70565 100644 --- a/src/ray/gcs/store_client/redis_context.h +++ b/src/ray/gcs/store_client/redis_context.h @@ -27,7 +27,6 @@ #include "ray/stats/metric.h" #include "ray/stats/tag_defs.h" #include "ray/util/exponential_backoff.h" -#include "src/ray/protobuf/gcs.pb.h" extern "C" { #include "hiredis/hiredis.h" @@ -63,7 +62,7 @@ class CallbackReply { const std::string &ReadAsString() const; /// Read this reply data as a string array. - [[nodiscard]] const std::vector> &ReadAsStringArray() const; + const std::vector> &ReadAsStringArray() const; /// Read this reply data as a scan array. /// diff --git a/src/ray/gcs/store_client/redis_store_client.cc b/src/ray/gcs/store_client/redis_store_client.cc index c963a693bce0..2f58eac6afe9 100644 --- a/src/ray/gcs/store_client/redis_store_client.cc +++ b/src/ray/gcs/store_client/redis_store_client.cc @@ -100,7 +100,7 @@ void RedisStoreClient::MGetValues( shared_callback, key_value_map](const std::shared_ptr &reply) { if (!reply->IsNil()) { - auto value = reply->ReadAsStringArray(); + const auto &value = reply->ReadAsStringArray(); for (size_t index = 0; index < value.size(); ++index) { if (value[index].has_value()) { (*key_value_map)[args[index]] = *(value[index]); diff --git a/src/ray/gcs/store_client/redis_store_client.h b/src/ray/gcs/store_client/redis_store_client.h index 9d4a6aa6cd62..6ba9c8b4ea3b 100644 --- a/src/ray/gcs/store_client/redis_store_client.h +++ b/src/ray/gcs/store_client/redis_store_client.h @@ -22,14 +22,11 @@ #include #include -#include "absl/container/flat_hash_set.h" #include "absl/synchronization/mutex.h" -#include "ray/common/asio/asio_util.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/asio/postable.h" #include "ray/gcs/store_client/redis_context.h" #include "ray/gcs/store_client/store_client.h" -#include "src/ray/protobuf/gcs.pb.h" namespace ray { @@ -96,8 +93,8 @@ struct RedisClientOptions { int port; // Redis username and password. - std::string username = ""; - std::string password = ""; + std::string username; + std::string password; // Whether to use TLS/SSL for the connection. bool enable_ssl = false; diff --git a/src/ray/gcs/store_client/tests/store_client_test_base.h b/src/ray/gcs/store_client/tests/store_client_test_base.h index 5bbc27432708..0495fff7e24e 100644 --- a/src/ray/gcs/store_client/tests/store_client_test_base.h +++ b/src/ray/gcs/store_client/tests/store_client_test_base.h @@ -29,6 +29,7 @@ #include "ray/common/test_util.h" #include "ray/gcs/store_client/store_client.h" #include "ray/util/logging.h" +#include "src/ray/protobuf/gcs.pb.h" namespace ray { diff --git a/src/ray/ipc/BUILD.bazel b/src/ray/ipc/BUILD.bazel index 38585313420b..1dcf5778fac3 100644 --- a/src/ray/ipc/BUILD.bazel +++ b/src/ray/ipc/BUILD.bazel @@ -31,6 +31,7 @@ ray_cc_library( ":client_connection", "//src/ray/common:asio", "//src/ray/common:buffer", + "//src/ray/common:flatbuf_utils", "//src/ray/common:id", "//src/ray/common:status", "//src/ray/flatbuffers:node_manager_generated", diff --git a/src/ray/ipc/client_connection.h b/src/ray/ipc/client_connection.h index 86f7788ba73c..530d117f7766 100644 --- a/src/ray/ipc/client_connection.h +++ b/src/ray/ipc/client_connection.h @@ -25,7 +25,6 @@ #include #include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/common_protocol.h" #include "ray/common/id.h" #include "ray/common/status.h" @@ -232,7 +231,7 @@ class ClientConnection : public ServerConnection { /// ProcessClientMessage handler will be called. void ProcessMessages(); - const std::string GetDebugLabel() const { return debug_label_; } + std::string GetDebugLabel() const { return debug_label_; } protected: /// A protected constructor for a node client connection. diff --git a/src/ray/ipc/fake_raylet_ipc_client.h b/src/ray/ipc/fake_raylet_ipc_client.h index 1b8d32c8d227..7f0ba5daac75 100644 --- a/src/ray/ipc/fake_raylet_ipc_client.h +++ b/src/ray/ipc/fake_raylet_ipc_client.h @@ -14,6 +14,10 @@ #pragma once +#include +#include +#include + #include "ray/ipc/raylet_ipc_client_interface.h" namespace ray { diff --git a/src/ray/ipc/raylet_ipc_client.cc b/src/ray/ipc/raylet_ipc_client.cc index 0cb7b229af82..07e5dcc305c7 100644 --- a/src/ray/ipc/raylet_ipc_client.cc +++ b/src/ray/ipc/raylet_ipc_client.cc @@ -15,44 +15,65 @@ #include "ray/ipc/raylet_ipc_client.h" #include -#include #include #include #include #include "absl/container/flat_hash_set.h" -#include "ray/common/common_protocol.h" +#include "ray/common/flatbuf_utils.h" #include "ray/common/ray_config.h" #include "ray/flatbuffers/node_manager_generated.h" #include "ray/ipc/client_connection.h" #include "ray/util/logging.h" #include "ray/util/process.h" +namespace ray::ipc { + namespace { -flatbuffers::Offset to_flatbuf( - flatbuffers::FlatBufferBuilder &fbb, const ray::rpc::Address &address) { - return ray::protocol::CreateAddress(fbb, - fbb.CreateString(address.node_id()), - fbb.CreateString(address.ip_address()), - address.port(), - fbb.CreateString(address.worker_id())); +flatbuffers::Offset AddressToFlatbuffer( + flatbuffers::FlatBufferBuilder &fbb, const rpc::Address &address) { + return protocol::CreateAddress(fbb, + fbb.CreateString(address.node_id()), + fbb.CreateString(address.ip_address()), + address.port(), + fbb.CreateString(address.worker_id())); } -flatbuffers::Offset>> +flatbuffers::Offset>> AddressesToFlatbuffer(flatbuffers::FlatBufferBuilder &fbb, - const std::vector &addresses) { - std::vector> address_vec; + const std::vector &addresses) { + std::vector> address_vec; address_vec.reserve(addresses.size()); for (const auto &addr : addresses) { - address_vec.push_back(to_flatbuf(fbb, addr)); + address_vec.push_back(AddressToFlatbuffer(fbb, addr)); } return fbb.CreateVector(address_vec); } -} // namespace +void ShutdownIfLocalRayletDisconnected(const Status &status) { + // Check if the Raylet process is still alive. + // If we know the Raylet PID, check using that. + // Else, assume the Raylet is our parent process. + bool raylet_alive = true; + auto raylet_pid = RayConfig::instance().RAYLET_PID(); + if (!raylet_pid.empty()) { + if (!IsProcessAlive(static_cast(std::stoi(raylet_pid)))) { + raylet_alive = false; + } + } else if (!IsParentProcessAlive()) { + raylet_alive = false; + } -namespace ray::ipc { + if (!status.ok() && !raylet_alive) { + RAY_LOG(WARNING) << "Exiting because the Raylet IPC connection failed and the local " + "Raylet is dead. Status: " + << status; + QuickExit(); + } +} + +} // namespace RayletIpcClient::RayletIpcClient(instrumented_io_context &io_service, const std::string &address, @@ -81,10 +102,10 @@ ray::Status RayletIpcClient::RegisterClient(const WorkerID &worker_id, auto message = protocol::CreateRegisterClientRequest(fbb, static_cast(worker_type), - to_flatbuf(fbb, worker_id), + flatbuf::to_flatbuf(fbb, worker_id), getpid(), startup_token, - to_flatbuf(fbb, job_id), + flatbuf::to_flatbuf(fbb, job_id), runtime_env_hash, language, fbb.CreateString(ip_address), @@ -99,7 +120,7 @@ ray::Status RayletIpcClient::RegisterClient(const WorkerID &worker_id, auto reply_message = flatbuffers::GetRoot(reply.data()); bool success = reply_message->success(); if (!success) { - return Status::Invalid(string_from_flatbuf(*reply_message->failure_reason())); + return Status::Invalid(reply_message->failure_reason()->str()); } *node_id = NodeID::FromBinary(reply_message->node_id()->str()); @@ -165,7 +186,7 @@ Status RayletIpcClient::AnnounceWorkerPortForDriver(int port, if (reply_message->success()) { return Status::OK(); } - return Status::Invalid(string_from_flatbuf(*reply_message->failure_reason())); + return Status::Invalid(reply_message->failure_reason()->str()); } Status RayletIpcClient::ActorCreationTaskDone() { @@ -177,7 +198,7 @@ Status RayletIpcClient::AsyncGetObjects( const std::vector &owner_addresses) { RAY_CHECK(object_ids.size() == owner_addresses.size()); flatbuffers::FlatBufferBuilder fbb; - auto object_ids_message = to_flatbuf(fbb, object_ids); + auto object_ids_message = flatbuf::to_flatbuf(fbb, object_ids); auto message = protocol::CreateAsyncGetObjectsRequest( fbb, object_ids_message, AddressesToFlatbuffer(fbb, owner_addresses)); fbb.Finish(message); @@ -213,7 +234,7 @@ StatusOr> RayletIpcClient::Wait( // Write request. flatbuffers::FlatBufferBuilder fbb; auto message = protocol::CreateWaitRequest(fbb, - to_flatbuf(fbb, object_ids), + flatbuf::to_flatbuf(fbb, object_ids), AddressesToFlatbuffer(fbb, owner_addresses), num_returns, timeout_milliseconds); @@ -242,7 +263,10 @@ Status RayletIpcClient::WaitForActorCallArgs( owner_addresses.push_back(ref.owner_address()); } auto message = protocol::CreateWaitForActorCallArgsRequest( - fbb, to_flatbuf(fbb, object_ids), AddressesToFlatbuffer(fbb, owner_addresses), tag); + fbb, + flatbuf::to_flatbuf(fbb, object_ids), + AddressesToFlatbuffer(fbb, owner_addresses), + tag); fbb.Finish(message); return WriteMessage(MessageType::WaitForActorCallArgsRequest, &fbb); } @@ -253,7 +277,7 @@ Status RayletIpcClient::PushError(const JobID &job_id, double timestamp) { flatbuffers::FlatBufferBuilder fbb; auto message = protocol::CreatePushErrorRequest(fbb, - to_flatbuf(fbb, job_id), + flatbuf::to_flatbuf(fbb, job_id), fbb.CreateString(type), fbb.CreateString(error_message), timestamp); @@ -264,8 +288,8 @@ Status RayletIpcClient::PushError(const JobID &job_id, Status RayletIpcClient::FreeObjects(const std::vector &object_ids, bool local_only) { flatbuffers::FlatBufferBuilder fbb; - auto message = - protocol::CreateFreeObjectsRequest(fbb, local_only, to_flatbuf(fbb, object_ids)); + auto message = protocol::CreateFreeObjectsRequest( + fbb, local_only, flatbuf::to_flatbuf(fbb, object_ids)); fbb.Finish(message); return WriteMessage(MessageType::FreeObjectsInObjectStoreRequest, &fbb); } @@ -274,39 +298,17 @@ void RayletIpcClient::SubscribePlasmaReady(const ObjectID &object_id, const rpc::Address &owner_address) { flatbuffers::FlatBufferBuilder fbb; auto message = protocol::CreateSubscribePlasmaReady( - fbb, to_flatbuf(fbb, object_id), to_flatbuf(fbb, owner_address)); + fbb, flatbuf::to_flatbuf(fbb, object_id), AddressToFlatbuffer(fbb, owner_address)); fbb.Finish(message); RAY_CHECK_OK(WriteMessage(MessageType::SubscribePlasmaReady, &fbb)); } -void ShutdownIfLocalRayletDisconnected(const Status &status) { - // Check if the Raylet process is still alive. - // If we know the Raylet PID, check using that. - // Else, assume the Raylet is our parent process. - bool raylet_alive = true; - auto raylet_pid = RayConfig::instance().RAYLET_PID(); - if (!raylet_pid.empty()) { - if (!IsProcessAlive(static_cast(std::stoi(raylet_pid)))) { - raylet_alive = false; - } - } else if (!IsParentProcessAlive()) { - raylet_alive = false; - } - - if (!status.ok() && !raylet_alive) { - RAY_LOG(WARNING) << "Exiting because the Raylet IPC connection failed and the local " - "Raylet is dead. Status: " - << status; - QuickExit(); - } -} - Status RayletIpcClient::WriteMessage(MessageType type, flatbuffers::FlatBufferBuilder *fbb) { std::unique_lock guard(write_mutex_); - int64_t length = fbb ? fbb->GetSize() : 0; - uint8_t *bytes = fbb ? fbb->GetBufferPointer() : nullptr; + int64_t length = fbb != nullptr ? fbb->GetSize() : 0; + uint8_t *bytes = fbb != nullptr ? fbb->GetBufferPointer() : nullptr; auto status = conn_->WriteMessage(static_cast(type), length, bytes); ShutdownIfLocalRayletDisconnected(status); return status; diff --git a/src/ray/ipc/raylet_ipc_client.h b/src/ray/ipc/raylet_ipc_client.h index b7faec8e8ca6..f6bf672cf799 100644 --- a/src/ray/ipc/raylet_ipc_client.h +++ b/src/ray/ipc/raylet_ipc_client.h @@ -17,8 +17,6 @@ #include #include #include -#include -#include #include #include "absl/container/flat_hash_set.h" diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index c8c3deb29311..4291637d3fd7 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -21,7 +21,6 @@ #include #include -#include "ray/common/common_protocol.h" #include "ray/object_manager/plasma/store_runner.h" #include "ray/object_manager/spilled_object_reader.h" #include "ray/stats/metric_defs.h" diff --git a/src/ray/object_manager/pull_manager.cc b/src/ray/object_manager/pull_manager.cc index ca616c3b0737..3c9af345c527 100644 --- a/src/ray/object_manager/pull_manager.cc +++ b/src/ray/object_manager/pull_manager.cc @@ -20,7 +20,6 @@ #include #include -#include "ray/common/common_protocol.h" #include "ray/common/ray_config.h" #include "ray/stats/metric_defs.h" diff --git a/src/ray/object_manager/tests/pull_manager_test.cc b/src/ray/object_manager/tests/pull_manager_test.cc index 86a727cb8bdf..f245445e7712 100644 --- a/src/ray/object_manager/tests/pull_manager_test.cc +++ b/src/ray/object_manager/tests/pull_manager_test.cc @@ -22,7 +22,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "ray/common/common_protocol.h" namespace ray { diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index aaabeefac51d..65a002214838 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -225,6 +225,7 @@ ray_cc_library( ":worker_killing_policy", ":worker_pool", "//src/ray/common:buffer", + "//src/ray/common:flatbuf_utils", "//src/ray/common:lease", "//src/ray/common:memory_monitor", "//src/ray/core_worker:experimental_mutable_object_provider", diff --git a/src/ray/raylet/lease_dependency_manager.cc b/src/ray/raylet/lease_dependency_manager.cc index 1cf2b602f381..543866f0cfb4 100644 --- a/src/ray/raylet/lease_dependency_manager.cc +++ b/src/ray/raylet/lease_dependency_manager.cc @@ -24,7 +24,7 @@ namespace ray { namespace raylet { bool LeaseDependencyManager::CheckObjectLocal(const ObjectID &object_id) const { - return local_objects_.count(object_id) == 1; + return local_objects_.contains(object_id); } bool LeaseDependencyManager::GetOwnerAddress(const ObjectID &object_id, @@ -70,7 +70,7 @@ void LeaseDependencyManager::StartOrUpdateWaitRequest( auto &wait_request = wait_requests_[worker_id]; for (const auto &ref : required_objects) { const auto obj_id = ObjectRefToId(ref); - if (local_objects_.count(obj_id)) { + if (local_objects_.contains(obj_id)) { // Object is already local. No need to fetch it. continue; } @@ -133,7 +133,10 @@ void LeaseDependencyManager::StartOrUpdateGetRequest( for (auto &obj_id : get_request.first) { auto it = required_objects_.find(obj_id); RAY_CHECK(it != required_objects_.end()); - refs.push_back(ObjectIdToRef(obj_id, it->second.owner_address)); + ray::rpc::ObjectReference ref; + ref.set_object_id(obj_id.Binary()); + ref.mutable_owner_address()->CopyFrom(it->second.owner_address); + refs.push_back(std::move(ref)); } // Pull the new dependencies before canceling the old request, in case some // of the old dependencies are still being fetched. @@ -198,7 +201,7 @@ bool LeaseDependencyManager::RequestLeaseDependencies( } for (const auto &obj_id : lease_entry->dependencies_) { - if (local_objects_.count(obj_id)) { + if (local_objects_.contains(obj_id)) { lease_entry->DecrementMissingDependencies(); } } diff --git a/src/ray/raylet/lease_dependency_manager.h b/src/ray/raylet/lease_dependency_manager.h index 7c051e4bbbfe..cb72ebd93405 100644 --- a/src/ray/raylet/lease_dependency_manager.h +++ b/src/ray/raylet/lease_dependency_manager.h @@ -22,9 +22,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" -#include "ray/common/common_protocol.h" #include "ray/common/id.h" -#include "ray/common/lease/lease.h" #include "ray/object_manager/object_manager.h" #include "ray/util/counter_map.h" @@ -43,7 +41,7 @@ class LeaseDependencyManagerInterface { virtual void RemoveLeaseDependencies(const LeaseID &lease_id) = 0; virtual bool LeaseDependenciesBlocked(const LeaseID &lease_id) const = 0; virtual bool CheckObjectLocal(const ObjectID &object_id) const = 0; - virtual ~LeaseDependencyManagerInterface(){}; + virtual ~LeaseDependencyManagerInterface() = default; }; /// \class LeaseDependencyManager @@ -94,7 +92,7 @@ class LeaseDependencyManager : public LeaseDependencyManagerInterface { /// /// \param object_id The object to check for. /// \return Whether the object is local. - bool CheckObjectLocal(const ObjectID &object_id) const; + bool CheckObjectLocal(const ObjectID &object_id) const override; /// Get the address of the owner of this object. An address will only be /// returned if the caller previously specified that this object is required @@ -158,7 +156,7 @@ class LeaseDependencyManager : public LeaseDependencyManagerInterface { /// \param required_objects The objects required by the lease. bool RequestLeaseDependencies(const LeaseID &lease_id, const std::vector &required_objects, - const TaskMetricsKey &task_key); + const TaskMetricsKey &task_key) override; /// Cancel a lease's dependencies. We will no longer attempt to fetch any /// remote dependencies, if no other lease or worker requires them. @@ -167,7 +165,7 @@ class LeaseDependencyManager : public LeaseDependencyManagerInterface { /// /// \param lease_id The lease that requires the objects. /// \param required_objects The objects required by the lease. - void RemoveLeaseDependencies(const LeaseID &lease_id); + void RemoveLeaseDependencies(const LeaseID &lease_id) override; /// Handle an object becoming locally available. /// @@ -188,7 +186,7 @@ class LeaseDependencyManager : public LeaseDependencyManagerInterface { /// Check whether a requested lease's dependencies are not being fetched to /// the local node due to lack of memory. - bool LeaseDependenciesBlocked(const LeaseID &lease_id) const; + bool LeaseDependenciesBlocked(const LeaseID &lease_id) const override; /// Returns debug string for class. /// @@ -227,13 +225,13 @@ class LeaseDependencyManager : public LeaseDependencyManagerInterface { /// A struct to represent the object dependencies of a task. struct LeaseDependencies { - LeaseDependencies(const absl::flat_hash_set &deps, + LeaseDependencies(absl::flat_hash_set deps, CounterMap> &counter_map, - const TaskMetricsKey &task_key) + TaskMetricsKey task_key) : dependencies_(std::move(deps)), num_missing_dependencies_(dependencies_.size()), waiting_task_counter_map_(counter_map), - task_key_(task_key) { + task_key_(std::move(task_key)) { if (num_missing_dependencies_ > 0) { waiting_task_counter_map_.Increment(task_key_); } @@ -268,6 +266,9 @@ class LeaseDependencyManager : public LeaseDependencyManagerInterface { } } + LeaseDependencies(const LeaseDependencies &) = delete; + LeaseDependencies &operator=(const LeaseDependencies &) = delete; + ~LeaseDependencies() { if (num_missing_dependencies_ > 0) { waiting_task_counter_map_.Decrement(task_key_); @@ -310,7 +311,7 @@ class LeaseDependencyManager : public LeaseDependencyManagerInterface { /// The set of locally available objects. This is used to determine which /// leases are ready to run and which `ray.wait` requests can be finished. - std::unordered_set local_objects_; + absl::flat_hash_set local_objects_; /// Counts the number of active lease dependency fetches by lease name. The counter /// total will be less than or equal to the size of queued_lease_requests_. diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index bc886a851236..08eaa3ce91d5 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -25,6 +25,7 @@ #include "nlohmann/json.hpp" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/cgroup/cgroup_manager.h" +#include "ray/common/constants.h" #include "ray/common/id.h" #include "ray/common/lease/lease.h" #include "ray/common/ray_config.h" @@ -498,7 +499,12 @@ int main(int argc, char *argv[]) { << ", object_chunk_size = " << object_manager_config.object_chunk_size; RAY_LOG(INFO).WithField(raylet_node_id) << "Setting node ID"; - node_manager_config.AddDefaultLabels(raylet_node_id.Hex()); + std::vector default_keys = {kLabelKeyNodeID}; + for (const auto &key : default_keys) { + RAY_CHECK(!node_manager_config.labels.contains(key)) + << "The label key name " << key << " should never be set by the user."; + } + node_manager_config.labels[kLabelKeyNodeID] = raylet_node_id.Hex(); worker_pool = std::make_unique( main_service, diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 39334551d410..4e55d453d180 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -31,8 +31,8 @@ #include "ray/common/asio/asio_util.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/buffer.h" -#include "ray/common/common_protocol.h" #include "ray/common/constants.h" +#include "ray/common/flatbuf_utils.h" #include "ray/common/grpc_util.h" #include "ray/common/lease/lease.h" #include "ray/common/memory_monitor.h" @@ -41,12 +41,9 @@ #include "ray/flatbuffers/node_manager_generated.h" #include "ray/gcs/pb_util.h" #include "ray/ipc/client_connection.h" -#include "ray/object_manager/ownership_object_directory.h" #include "ray/raylet/local_object_manager_interface.h" -#include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/worker_killing_policy.h" #include "ray/raylet/worker_pool.h" -#include "ray/rpc/node_manager/node_manager_client.h" #include "ray/stats/metric_defs.h" #include "ray/util/cmd_line_utils.h" #include "ray/util/event.h" @@ -54,20 +51,13 @@ #include "ray/util/string_utils.h" #include "ray/util/time.h" -namespace { - -#define RAY_CHECK_ENUM(x, y) \ - static_assert(static_cast(x) == static_cast(y), "protocol mismatch") +namespace ray::raylet { -struct ActorStats { - int live_actors = 0; - int dead_actors = 0; - int restarting_actors = 0; -}; +namespace { -inline ray::rpc::ObjectReference FlatbufferToSingleObjectReference( - const flatbuffers::String &object_id, const ray::protocol::Address &address) { - ray::rpc::ObjectReference ref; +rpc::ObjectReference FlatbufferToSingleObjectReference( + const flatbuffers::String &object_id, const protocol::Address &address) { + rpc::ObjectReference ref; ref.set_object_id(object_id.str()); ref.mutable_owner_address()->set_node_id(address.node_id()->str()); ref.mutable_owner_address()->set_ip_address(address.ip_address()->str()); @@ -76,39 +66,31 @@ inline ray::rpc::ObjectReference FlatbufferToSingleObjectReference( return ref; } -std::vector FlatbufferToObjectReference( +std::vector FlatbufferToObjectReferences( const flatbuffers::Vector> &object_ids, - const flatbuffers::Vector> - &owner_addresses) { + const flatbuffers::Vector> &owner_addresses) { RAY_CHECK(object_ids.size() == owner_addresses.size()); - std::vector refs; + std::vector refs; + refs.reserve(object_ids.size()); for (int64_t i = 0; i < object_ids.size(); i++) { - ray::rpc::ObjectReference ref; - ref.set_object_id(object_ids.Get(i)->str()); - const auto &addr = owner_addresses.Get(i); - ref.mutable_owner_address()->set_node_id(addr->node_id()->str()); - ref.mutable_owner_address()->set_ip_address(addr->ip_address()->str()); - ref.mutable_owner_address()->set_port(addr->port()); - ref.mutable_owner_address()->set_worker_id(addr->worker_id()->str()); - refs.emplace_back(std::move(ref)); + refs.push_back( + FlatbufferToSingleObjectReference(*object_ids.Get(i), *owner_addresses.Get(i))); } return refs; } -} // namespace - -namespace ray::raylet { - -void NodeManagerConfig::AddDefaultLabels(const std::string &self_node_id) { - std::vector default_keys = {kLabelKeyNodeID}; - - for (const auto &key : default_keys) { - RAY_CHECK(!labels.contains(key)) - << "The label key name " << key << " should never be set by the user."; +std::vector FlatbufferToObjectIds( + const flatbuffers::Vector> &vector) { + std::vector ids; + ids.reserve(vector.size()); + for (int64_t i = 0; i < vector.size(); i++) { + ids.push_back(ObjectID::FromBinary(vector.Get(i)->str())); } - labels[kLabelKeyNodeID] = self_node_id; + return ids; } +} // namespace + NodeManager::NodeManager( instrumented_io_context &io_service, const NodeID &self_node_id, @@ -136,7 +118,7 @@ NodeManager::NodeManager( self_node_name_(std::move(self_node_name)), io_service_(io_service), gcs_client_(gcs_client), - shutdown_raylet_gracefully_(shutdown_raylet_gracefully), + shutdown_raylet_gracefully_(std::move(shutdown_raylet_gracefully)), worker_pool_(worker_pool), client_call_manager_(client_call_manager), worker_rpc_pool_(worker_rpc_pool), @@ -156,8 +138,9 @@ NodeManager::NodeManager( }, /*delay_executor*/ [this](std::function fn, int64_t delay_ms) { - RAY_UNUSED(execute_after( - io_service_, fn, std::chrono::milliseconds(delay_ms))); + RAY_UNUSED(execute_after(io_service_, + std::move(fn), + std::chrono::milliseconds(delay_ms))); }), node_manager_server_("NodeManager", config.node_manager_port, @@ -647,7 +630,7 @@ void NodeManager::QueryAllWorkerStates( const std::function &on_all_replied) { auto all_workers = worker_pool_.GetAllRegisteredWorkers(/* filter_dead_worker */ true, /*filter_io_workers*/ true); - for (auto driver : + for (auto &driver : worker_pool_.GetAllRegisteredDrivers(/* filter_dead_driver */ true)) { all_workers.push_back(driver); } @@ -1047,7 +1030,7 @@ void NodeManager::ProcessClientMessage(const std::shared_ptr & } break; case protocol::MessageType::FreeObjectsInObjectStoreRequest: { auto message = flatbuffers::GetRoot(message_data); - std::vector object_ids = from_flatbuf(*message->object_ids()); + auto object_ids = FlatbufferToObjectIds(*message->object_ids()); // Clean up objects from the object store. object_manager_.FreeObjects(object_ids, message->local_only()); } break; @@ -1074,12 +1057,12 @@ Status NodeManager::ProcessRegisterClientRequestMessageImpl( client->Register(); Language language = static_cast(message->language()); - const JobID job_id = from_flatbuf(*message->job_id()); + const JobID job_id = JobID::FromBinary(message->job_id()->str()); const int runtime_env_hash = static_cast(message->runtime_env_hash()); - WorkerID worker_id = from_flatbuf(*message->worker_id()); + WorkerID worker_id = WorkerID::FromBinary(message->worker_id()->str()); pid_t pid = message->worker_pid(); StartupToken worker_startup_token = message->startup_token(); - std::string worker_ip_address = string_from_flatbuf(*message->ip_address()); + std::string worker_ip_address = message->ip_address()->str(); // TODO(suquark): Use `WorkerType` in `common.proto` without type converting. rpc::WorkerType worker_type = static_cast(message->worker_type()); if (worker_type == rpc::WorkerType::DRIVER) { @@ -1107,7 +1090,7 @@ Status NodeManager::ProcessRegisterClientRequestMessageImpl( ray::protocol::CreateRegisterClientReply(fbb, status.ok(), fbb.CreateString(status.ToString()), - to_flatbuf(fbb, self_node_id_), + flatbuf::to_flatbuf(fbb, self_node_id_), assigned_port); fbb.Finish(reply); client->WriteMessageAsync( @@ -1201,13 +1184,12 @@ void NodeManager::ProcessAnnounceWorkerPortMessageImpl( driver_address.set_ip_address(worker->IpAddress()); driver_address.set_port(port); driver_address.set_worker_id(worker->WorkerId().Binary()); - auto job_data_ptr = - gcs::CreateJobTableData(job_id, - /*is_dead=*/false, - driver_address, - worker->GetProcess().GetId(), - string_from_flatbuf(*message->entrypoint()), - *job_config); + auto job_data_ptr = gcs::CreateJobTableData(job_id, + /*is_dead=*/false, + driver_address, + worker->GetProcess().GetId(), + message->entrypoint()->str(), + *job_config); gcs_client_.Jobs().AsyncAdd(job_data_ptr, [this, client](Status status) { SendPortAnnouncementResponse(client, std::move(status)); @@ -1455,7 +1437,7 @@ void NodeManager::HandleAsyncGetObjectsRequest( const std::shared_ptr &client, const uint8_t *message_data) { auto request = flatbuffers::GetRoot(message_data); const auto refs = - FlatbufferToObjectReference(*request->object_ids(), *request->owner_addresses()); + FlatbufferToObjectReferences(*request->object_ids(), *request->owner_addresses()); // Asynchronously pull all requested objects to the local node. AsyncGetOrWait(client, @@ -1467,9 +1449,9 @@ void NodeManager::ProcessWaitRequestMessage( const std::shared_ptr &client, const uint8_t *message_data) { // Read the data. auto message = flatbuffers::GetRoot(message_data); - std::vector object_ids = from_flatbuf(*message->object_ids()); + auto object_ids = FlatbufferToObjectIds(*message->object_ids()); const auto refs = - FlatbufferToObjectReference(*message->object_ids(), *message->owner_addresses()); + FlatbufferToObjectReferences(*message->object_ids(), *message->owner_addresses()); bool all_objects_local = true; for (auto const &object_id : object_ids) { @@ -1490,9 +1472,10 @@ void NodeManager::ProcessWaitRequestMessage( // If we don't need to wait for any, return immediately after making the pull // requests through AsyncGetOrWait above. flatbuffers::FlatBufferBuilder fbb; - auto wait_reply = protocol::CreateWaitReply(fbb, - to_flatbuf(fbb, std::vector{}), - to_flatbuf(fbb, std::vector{})); + auto wait_reply = + protocol::CreateWaitReply(fbb, + flatbuf::to_flatbuf(fbb, std::vector{}), + flatbuf::to_flatbuf(fbb, std::vector{})); fbb.Finish(wait_reply); const auto status = client->WriteMessage(static_cast(protocol::MessageType::WaitReply), @@ -1507,17 +1490,17 @@ void NodeManager::ProcessWaitRequestMessage( } return; } - uint64_t num_required_objects = static_cast(message->num_required_objects()); + wait_manager_.Wait( object_ids, message->timeout(), - num_required_objects, - [this, client, all_objects_local](std::vector ready, - std::vector remaining) { + message->num_required_objects(), + [this, client, all_objects_local](const std::vector &ready, + const std::vector &remaining) { // Write the data. flatbuffers::FlatBufferBuilder fbb; flatbuffers::Offset wait_reply = protocol::CreateWaitReply( - fbb, to_flatbuf(fbb, ready), to_flatbuf(fbb, remaining)); + fbb, flatbuf::to_flatbuf(fbb, ready), flatbuf::to_flatbuf(fbb, remaining)); fbb.Finish(wait_reply); auto status = @@ -1544,39 +1527,39 @@ void NodeManager::ProcessWaitForActorCallArgsRequestMessage( const std::shared_ptr &client, const uint8_t *message_data) { auto message = flatbuffers::GetRoot(message_data); - std::vector object_ids = from_flatbuf(*message->object_ids()); + auto object_ids = FlatbufferToObjectIds(*message->object_ids()); int64_t tag = message->tag(); // Pull any missing objects to the local node. const auto refs = - FlatbufferToObjectReference(*message->object_ids(), *message->owner_addresses()); + FlatbufferToObjectReferences(*message->object_ids(), *message->owner_addresses()); AsyncGetOrWait(client, refs, /*is_get_request=*/false); // De-duplicate the object IDs. absl::flat_hash_set object_id_set(object_ids.begin(), object_ids.end()); object_ids.assign(object_id_set.begin(), object_id_set.end()); - wait_manager_.Wait( - object_ids, - -1, - object_ids.size(), - [this, client, tag](std::vector ready, std::vector remaining) { - RAY_CHECK(remaining.empty()); - std::shared_ptr worker = - worker_pool_.GetRegisteredWorker(client); - if (!worker) { - RAY_LOG(ERROR) << "Lost worker for wait request " << client; - } else { - worker->ActorCallArgWaitComplete(tag); - } - }); + wait_manager_.Wait(object_ids, + -1, + object_ids.size(), + [this, client, tag](const std::vector &ready, + const std::vector &remaining) { + RAY_CHECK(remaining.empty()); + std::shared_ptr worker = + worker_pool_.GetRegisteredWorker(client); + if (!worker) { + RAY_LOG(ERROR) << "Lost worker for wait request " << client; + } else { + worker->ActorCallArgWaitComplete(tag); + } + }); } void NodeManager::ProcessPushErrorRequestMessage(const uint8_t *message_data) { auto message = flatbuffers::GetRoot(message_data); - auto const &type = string_from_flatbuf(*message->type()); - auto const &error_message = string_from_flatbuf(*message->error_message()); + auto const &type = message->type()->str(); + auto const &error_message = message->error_message()->str(); // TODO(hjiang): Figure out what's the unit for `PushErrorRequest`. double timestamp = message->timestamp(); - JobID job_id = from_flatbuf(*message->job_id()); + JobID job_id = JobID::FromBinary(message->job_id()->str()); auto error_data = gcs::CreateErrorTableData( type, error_message, absl::FromUnixMillis(timestamp), job_id); gcs_client_.Errors().AsyncReportJobError(std::move(error_data)); @@ -1599,8 +1582,7 @@ void NodeManager::HandleCancelLeasesWithResourceShapes( const auto &resource_shapes = request.resource_shapes(); std::vector target_resource_shapes; for (const auto &resource_shape : resource_shapes) { - target_resource_shapes.emplace_back( - ResourceSet(MapFromProtobuf(resource_shape.resource_shape()))); + target_resource_shapes.emplace_back(MapFromProtobuf(resource_shape.resource_shape())); } cluster_lease_manager_.CancelLeasesWithResourceShapes(target_resource_shapes); @@ -2034,17 +2016,17 @@ void NodeManager::HandleReleaseUnusedActorWorkers( rpc::ReleaseUnusedActorWorkersRequest request, rpc::ReleaseUnusedActorWorkersReply *reply, rpc::SendReplyCallback send_reply_callback) { - std::unordered_set in_use_worker_ids; - for (int index = 0; index < request.worker_ids_in_use_size(); ++index) { - auto worker_id = WorkerID::FromBinary(request.worker_ids_in_use(index)); - in_use_worker_ids.emplace(worker_id); + absl::flat_hash_set in_use_worker_ids; + in_use_worker_ids.reserve(request.worker_ids_in_use_size()); + for (const auto &worker_id_in_use_binary : request.worker_ids_in_use()) { + in_use_worker_ids.emplace(WorkerID::FromBinary(worker_id_in_use_binary)); } std::vector> unused_actor_workers; for (auto &iter : leased_workers_) { // We only kill *actor* workers. if (!iter.second->GetActorId().IsNil() && - !in_use_worker_ids.count(iter.second->WorkerId())) { + !in_use_worker_ids.contains(iter.second->WorkerId())) { unused_actor_workers.push_back(iter.second); } } @@ -2253,7 +2235,7 @@ void NodeManager::HandleObjectLocal(const ObjectInfo &object_info) { rpc::PlasmaObjectReadyRequest request; request.set_object_id(object_id.Binary()); - for (auto worker : waiting_workers) { + for (const auto &worker : waiting_workers) { worker->rpc_client()->PlasmaObjectReady( request, [](Status status, const rpc::PlasmaObjectReadyReply &reply) { if (!status.ok()) { @@ -2295,7 +2277,7 @@ void NodeManager::ProcessSubscribePlasmaReady( << "No worker exists for CoreWorker with client: " << client->DebugString(); auto message = flatbuffers::GetRoot(message_data); - auto id = from_flatbuf(*message->object_id()); + auto id = ObjectID::FromBinary(message->object_id()->str()); if (lease_dependency_manager_.CheckObjectLocal(id)) { // Object is already local, so we directly fire the callback to tell the core worker diff --git a/src/ray/raylet/node_manager.h b/src/ray/raylet/node_manager.h index 0344677df99e..4631f8bf0b72 100644 --- a/src/ray/raylet/node_manager.h +++ b/src/ray/raylet/node_manager.h @@ -119,8 +119,6 @@ struct NodeManagerConfig { // If true, core worker enables resource isolation by adding itself into appropriate // cgroup. bool enable_resource_isolation = false; - - void AddDefaultLabels(const std::string &self_node_id); }; class NodeManager : public rpc::NodeManagerServiceHandler, diff --git a/src/ray/raylet_client/raylet_client.cc b/src/ray/raylet_client/raylet_client.cc index 8e55cb4adfe7..5981265d503c 100644 --- a/src/ray/raylet_client/raylet_client.cc +++ b/src/ray/raylet_client/raylet_client.cc @@ -20,8 +20,6 @@ #include #include -#include "absl/synchronization/notification.h" -#include "ray/common/common_protocol.h" #include "ray/common/ray_config.h" #include "ray/util/logging.h" From ea27046ad632ac40b015fd6f02f348ef4a91a8e9 Mon Sep 17 00:00:00 2001 From: Timothy Seah Date: Mon, 1 Sep 2025 21:06:47 -0700 Subject: [PATCH 381/634] [train][checkpoint] Add ray.train.get_all_reported_checkpoints method (#54555) # Summary This PR adds a `ray.train.get_all_reported_checkpoints` method that allows users to get all the checkpoints they have reported from within their training function. This is different from [Result](https://docs.ray.io/en/latest/train/user-guides/results.html) in two ways: * It is called from the training function on the training worker instead of from the driver * It can be called while training is still in progress # Implementation Notes The main idea is to use a worker-side counter and controller-side counter as follows: * Train worker: `ray.train.report` increments a `num_reported_checkpoints` counter and puts the training result into its queue * Train controller: polls the training results from all worker, registers the checkpoint, increments `num_reported_checkpoints`, and then creates an asyncio task to notify asyncio Condition. This works because asyncio Ray actors should always have an event loop. * Train worker: `get_all_reported_results` uses an asyncio.Condition to wait until the worker-side `num_reported_checkpoints` counter matches its controller-side counterpart before returning the checkpoints. This ensures that we wait for all pending reports to finish. It has access to the controller actor through `init_train_context`. `get_checkpoint` should be unaffected because it uses the local checkpoint; we can consider changing it to use the "centrally committed" checkpoint in the future. # Testing I ran the [ray train pytorch example](https://docs.ray.io/en/latest/train/getting-started-pytorch.html) and called `ray.train.get_all_reported_checkpoints` at the end of each epoch. The results are as expected; here are a few examples ` epoch 4: [TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-34-52.538994), metrics={'loss': 0.24510294198989868, 'epoch': 0}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-35-07.511694), metrics={'loss': 0.23799467086791992, 'epoch': 1}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-35-24.355974), metrics={'loss': 0.39628422260284424, 'epoch': 2}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-35-40.273211), metrics={'loss': 0.15193207561969757, 'epoch': 3}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-35-56.178119), metrics={'loss': 0.17416314780712128, 'epoch': 4})] ` ` epoch 9: [TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-34-52.538994), metrics={'loss': 0.24510294198989868, 'epoch': 0}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-35-07.511694), metrics={'loss': 0.23799467086791992, 'epoch': 1}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-35-24.355974), metrics={'loss': 0.39628422260284424, 'epoch': 2}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-35-40.273211), metrics={'loss': 0.15193207561969757, 'epoch': 3}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-35-56.178119), metrics={'loss': 0.17416314780712128, 'epoch': 4}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-36-12.547310), metrics={'loss': 0.2924661934375763, 'epoch': 5}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-36-28.538090), metrics={'loss': 0.18640762567520142, 'epoch': 6}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-36-44.583228), metrics={'loss': 0.12567029893398285, 'epoch': 7}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-37-00.540405), metrics={'loss': 0.1620682030916214, 'epoch': 8}), TrainingResult(checkpoint=Checkpoint(filesystem=local, path=/mnt/cluster_storage/my_run_name/checkpoint_2025-08-04_17-37-17.129973), metrics={'loss': 0.07022886723279953, 'epoch': 9})] ` I also modified all the Ray Train v2 unit tests that call `ray.train.report`: * `test_persistence` also verifies that `get_all_reported_checkpoints` works on resumption * `test_data_parallel_trainer` verifies that `get_all_reported_checkpoints` stalls until all workers report. I also verified that `get_all_reported_checkpoints` produced similar output when called from Tune + Train. I tried to test that `get_all_reported_checkpoints` finished even with graceful abort but was unable to create such a scenario since `get_all_reported_checkpoints` returns very quickly and each `report` forms a barrier. --------- Signed-off-by: Timothy Seah --- ci/lint/pydoclint-baseline.txt | 8 --- python/ray/train/__init__.py | 7 +++ .../checkpoint/checkpoint_manager.py | 41 +++++++++++++- .../train/v2/_internal/execution/context.py | 30 ++++++++--- .../execution/controller/controller.py | 18 ++++++- .../v2/_internal/execution/train_fn_utils.py | 18 +++++-- .../execution/worker_group/worker.py | 2 + .../execution/worker_group/worker_group.py | 1 + .../ray/train/v2/api/reported_checkpoint.py | 21 ++++++++ python/ray/train/v2/api/train_fn_utils.py | 54 +++++++++++++++++-- python/ray/train/v2/tests/conftest.py | 24 +++++++++ .../train/v2/tests/test_accelerator_utils.py | 2 +- .../train/v2/tests/test_checkpoint_manager.py | 16 +++--- python/ray/train/v2/tests/test_controller.py | 2 + .../v2/tests/test_data_parallel_trainer.py | 24 +++++++++ python/ray/train/v2/tests/test_persistence.py | 13 +++++ .../ray/train/v2/tests/test_worker_group.py | 2 + 17 files changed, 252 insertions(+), 31 deletions(-) create mode 100644 python/ray/train/v2/api/reported_checkpoint.py diff --git a/ci/lint/pydoclint-baseline.txt b/ci/lint/pydoclint-baseline.txt index 84b97b4f4724..a6ab3cb60a2b 100644 --- a/ci/lint/pydoclint-baseline.txt +++ b/ci/lint/pydoclint-baseline.txt @@ -2006,14 +2006,6 @@ python/ray/train/v2/_internal/callbacks/accelerators.py python/ray/train/v2/_internal/execution/checkpoint/checkpoint_manager.py DOC103: Method `CheckpointManager.register_checkpoint`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [checkpoint_result: _TrainingResult]. Arguments in the docstring but not in the function signature: [checkpoint: ]. -------------------- -python/ray/train/v2/_internal/execution/context.py - DOC101: Method `TrainContext._save_checkpoint`: Docstring contains fewer arguments than in function signature. - DOC103: Method `TrainContext._save_checkpoint`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [checkpoint: Optional[Checkpoint], checkpoint_dir_name: str, metrics: Dict[str, Any]]. --------------------- -python/ray/train/v2/_internal/execution/controller/controller.py - DOC101: Method `TrainController._start_worker_group`: Docstring contains fewer arguments than in function signature. - DOC103: Method `TrainController._start_worker_group`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [num_workers: int, resources_per_worker: dict]. --------------------- python/ray/train/v2/_internal/execution/storage.py DOC101: Method `_ExcludingLocalFilesystem.__init__`: Docstring contains fewer arguments than in function signature. DOC103: Method `_ExcludingLocalFilesystem.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**kwargs: ]. diff --git a/python/ray/train/__init__.py b/python/ray/train/__init__.py index 7713ccb705af..5b72413f79bf 100644 --- a/python/ray/train/__init__.py +++ b/python/ray/train/__init__.py @@ -34,8 +34,10 @@ RunConfig, ScalingConfig, ) + from ray.train.v2.api.reported_checkpoint import ReportedCheckpoint # noqa: F811 from ray.train.v2.api.result import Result # noqa: F811 from ray.train.v2.api.train_fn_utils import ( # noqa: F811 + get_all_reported_checkpoints, get_checkpoint, get_context, get_dataset_shard, @@ -76,9 +78,14 @@ SyncConfig.__module__ = "ray.train" TrainingIterator.__module__ = "ray.train" +# TODO: consider implementing these in v1 and raising ImportError instead. if is_v2_enabled(): __all__.append("UserCallback") UserCallback.__module__ = "ray.train" + __all__.append("get_all_reported_checkpoints") + get_all_reported_checkpoints.__module__ = "ray.train" + __all__.append("ReportedCheckpoint") + ReportedCheckpoint.__module__ = "ray.train" # DO NOT ADD ANYTHING AFTER THIS LINE. diff --git a/python/ray/train/v2/_internal/execution/checkpoint/checkpoint_manager.py b/python/ray/train/v2/_internal/execution/checkpoint/checkpoint_manager.py index a241ce43619d..bb0ed40e5503 100644 --- a/python/ray/train/v2/_internal/execution/checkpoint/checkpoint_manager.py +++ b/python/ray/train/v2/_internal/execution/checkpoint/checkpoint_manager.py @@ -1,3 +1,4 @@ +import asyncio import logging from typing import Any, Dict, List, Optional @@ -16,6 +17,7 @@ from ray.train.v2._internal.execution.context import StorageContext from ray.train.v2._internal.execution.storage import _delete_fs_path, _exists_at_fs_path from ray.train.v2._internal.execution.worker_group import Worker +from ray.train.v2.api.reported_checkpoint import ReportedCheckpoint try: from pydantic import BaseModel @@ -81,6 +83,12 @@ def __init__( ): self._storage_context = storage_context self._checkpoint_config = checkpoint_config + + # This tracks the number of report calls that have been processed + # for the current worker group. + self._num_report_calls = 0 + + self._condition = asyncio.Condition() super().__init__(checkpoint_config) # If the snapshot is found, the checkpoint manager will restore its state. self._maybe_load_state_from_storage() @@ -139,6 +147,14 @@ def register_checkpoint(self, checkpoint_result: _TrainingResult): logger.debug("Deleting checkpoint: ", checkpoint) _delete_fs_path(fs=checkpoint.filesystem, fs_path=checkpoint.path) + self._num_report_calls += 1 + + async def async_notify(): + async with self._condition: + self._condition.notify_all() + + asyncio.create_task(async_notify()) + # -------------------------- # CheckpointManager state # -------------------------- @@ -267,6 +283,7 @@ def after_report( self, metrics: List[Dict[str, Any]], checkpoint: Optional[Checkpoint] ): if not checkpoint: + self._num_report_calls += 1 return rank_0_metrics = metrics[0] @@ -279,9 +296,31 @@ def after_report( # -------------------------- def before_init_train_context(self, workers: List[Worker]) -> Dict[str, List[Any]]: + self._num_report_calls = 0 latest_checkpoint = ( self.latest_checkpoint_result.checkpoint if self.latest_checkpoint_result else None ) - return {"checkpoint": [latest_checkpoint] * len(workers)} + train_context_args = { + "checkpoint": [latest_checkpoint] * len(workers), + } + return train_context_args + + async def get_all_reported_checkpoints( + self, expected_num_report_calls: int + ) -> List[ReportedCheckpoint]: + """Once expected_num_checkpoints are reported, return the ReportedCheckpoints.""" + async with self._condition: + await self._condition.wait_for( + lambda: self._num_report_calls == expected_num_report_calls + ) + # TODO: might be nice for CheckpointManager to manage ReportedCheckpoint + # instead of _TrainingResult but that is a large refactor. + return [ + ReportedCheckpoint( + checkpoint=tr.checkpoint, + metrics=tr.metrics, + ) + for tr in self._checkpoint_results + ] diff --git a/python/ray/train/v2/_internal/execution/context.py b/python/ray/train/v2/_internal/execution/context.py index a19b308b0709..d21678b307b6 100644 --- a/python/ray/train/v2/_internal/execution/context.py +++ b/python/ray/train/v2/_internal/execution/context.py @@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional import ray +from ray.actor import ActorHandle from ray.data import DataIterator, Dataset -from ray.train import BackendConfig, Checkpoint, DataConfig from ray.train._internal import session from ray.train._internal.session import _TrainingResult from ray.train.v2._internal.execution.checkpoint.sync_actor import SynchronizationActor @@ -17,12 +17,14 @@ from ray.train.v2.api.config import RunConfig, ScalingConfig if TYPE_CHECKING: + from ray.train import BackendConfig, Checkpoint, DataConfig from ray.train.v2._internal.data_integration.interfaces import ( DatasetShardMetadata, DatasetShardProvider, ) from ray.train.v2._internal.execution.callback import TrainContextCallback from ray.train.v2._internal.execution.worker_group.thread_runner import ThreadRunner + from ray.train.v2.api.reported_checkpoint import ReportedCheckpoint logger = logging.getLogger(__file__) @@ -45,13 +47,13 @@ class TrainRunContext: scaling_config: ScalingConfig # The configuration for the training backend (e.g., PyTorch, XGBoost). - backend_config: BackendConfig + backend_config: "BackendConfig" # The datasets used in the current training run. datasets: Dict[str, Dataset] # The configuration for dataset ingestion and sharding. - dataset_config: DataConfig + dataset_config: "DataConfig" def get_run_config(self) -> RunConfig: """Returns the run config of the current training run.""" @@ -96,8 +98,11 @@ class TrainContext: distributed_context: DistributedContext execution_context: ExecutionContext storage_context: StorageContext + controller_actor: ActorHandle + dataset_shard_provider: "DatasetShardProvider" - checkpoint: Optional[Checkpoint] = None + checkpoint: Optional["Checkpoint"] = None + num_report_calls: int = 0 @_copy_doc(session.get_experiment_name) def get_experiment_name(self) -> str: @@ -137,6 +142,13 @@ def get_synchronization_actor(self): def get_checkpoint(self): return self.checkpoint + def get_all_reported_checkpoints(self) -> List["ReportedCheckpoint"]: + return ray.get( + self.controller_actor.get_all_reported_checkpoints.remote( + self.num_report_calls + ) + ) + def get_dataset_shard(self, dataset_info: "DatasetShardMetadata") -> DataIterator: """Returns the :class:`ray.data.DataIterator` shard for this worker. @@ -189,10 +201,15 @@ def _save_checkpoint( self, checkpoint_dir_name: str, metrics: Dict[str, Any], - checkpoint: Optional[Checkpoint] = None, + checkpoint: Optional["Checkpoint"] = None, ) -> _TrainingResult: """Save the checkpoint to remote storage. + Args: + checkpoint_dir_name: The checkpoint dir to persist to. + metrics: The metrics to report. + checkpoint: The checkpoint to report. + Returns: The training result object containing the persisted checkpoint. """ @@ -212,7 +229,7 @@ def _save_checkpoint( def report( self, metrics: Dict[str, Any], - checkpoint: Optional[Checkpoint] = None, + checkpoint: Optional["Checkpoint"] = None, checkpoint_dir_name: Optional[str] = None, ) -> None: """ @@ -265,6 +282,7 @@ def report( # TODO (hpguo): Add a metrics to track the blocking time waiting for the # training result to be consumed by the controller. self.get_result_queue().put(training_result) + self.num_report_calls += 1 # The global variable holding the current TrainContext diff --git a/python/ray/train/v2/_internal/execution/controller/controller.py b/python/ray/train/v2/_internal/execution/controller/controller.py index 59d3760bb503..224d8b906362 100644 --- a/python/ray/train/v2/_internal/execution/controller/controller.py +++ b/python/ray/train/v2/_internal/execution/controller/controller.py @@ -3,7 +3,7 @@ import os import uuid from dataclasses import dataclass -from typing import Callable, List, Optional, Union +from typing import TYPE_CHECKING, Callable, List, Optional, Union import pandas as pd @@ -67,6 +67,10 @@ ) from ray.train.v2.api.result import Result +if TYPE_CHECKING: + from ray.train.v2.api.reported_checkpoint import ReportedCheckpoint + + logger = logging.getLogger(__name__) @@ -275,6 +279,10 @@ def _start_worker_group( ) -> Optional[ControllerError]: """Start the worker group and launch the train function. + Args: + num_workers: The number of workers to start. + resources_per_worker: The resources per worker to start. + Returns: None if the worker group was successfully started, ControllerError if the worker group failed to start. @@ -537,7 +545,6 @@ def get_result(self) -> Result: raise ValueError( f"Cannot get result when controller is in state {controller_state}" ) - return self._build_result() def get_training_failed_error(self) -> Optional[TrainingFailedError]: @@ -553,3 +560,10 @@ def get_training_failed_error(self) -> Optional[TrainingFailedError]: return controller_state.training_failed_error return None + + async def get_all_reported_checkpoints( + self, expected_num_report_calls: int + ) -> List["ReportedCheckpoint"]: + return await self._checkpoint_manager.get_all_reported_checkpoints( + expected_num_report_calls + ) diff --git a/python/ray/train/v2/_internal/execution/train_fn_utils.py b/python/ray/train/v2/_internal/execution/train_fn_utils.py index 6038e6655465..3fb791b667d6 100644 --- a/python/ray/train/v2/_internal/execution/train_fn_utils.py +++ b/python/ray/train/v2/_internal/execution/train_fn_utils.py @@ -1,14 +1,17 @@ import threading -from typing import Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional from ray.data import DataIterator -from ray.train import Checkpoint from ray.train.v2._internal.execution import collective_impl from ray.train.v2._internal.execution.context import ( get_train_context as get_internal_train_context, ) from ray.train.v2.api.context import TrainContext as ExternalTrainContext +if TYPE_CHECKING: + from ray.train import Checkpoint + from ray.train.v2.api.reported_checkpoint import ReportedCheckpoint + class TrainFnUtils: """Utility class providing an abstraction layer between user-facing APIs @@ -21,7 +24,7 @@ class TrainFnUtils: def report( self, metrics: Dict[str, Any], - checkpoint: Optional[Checkpoint] = None, + checkpoint: Optional["Checkpoint"] = None, checkpoint_dir_name: Optional[str] = None, ) -> None: """Upload checkpoint to remote storage and put a training result on the result queue. @@ -46,6 +49,15 @@ def get_checkpoint(self): """ return get_internal_train_context().get_checkpoint() + def get_all_reported_checkpoints(self) -> List["ReportedCheckpoint"]: + """Get all the checkpoints reported by the workers. + + Returns: + A list of ReportedCheckpoint objects that represent the checkpoints and + corresponding metrics reported by the workers. + """ + return get_internal_train_context().get_all_reported_checkpoints() + def get_dataset_shard(self, dataset_name: str) -> DataIterator: """Get the dataset shard for this worker. diff --git a/python/ray/train/v2/_internal/execution/worker_group/worker.py b/python/ray/train/v2/_internal/execution/worker_group/worker.py index be2e935ca40e..8afc8a7b681e 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/worker.py +++ b/python/ray/train/v2/_internal/execution/worker_group/worker.py @@ -194,6 +194,7 @@ def init_train_context( synchronization_actor: SynchronizationActor, storage_context: StorageContext, worker_callbacks: List[Union[WorkerCallback, TrainContextCallback]], + controller_actor: ActorHandle, dataset_shard_provider: Optional["DatasetShardProvider"] = None, checkpoint: Optional[Checkpoint] = None, ): @@ -213,6 +214,7 @@ def init_train_context( train_context_callbacks=context_callbacks_to_propagate, ), storage_context=storage_context, + controller_actor=controller_actor, checkpoint=checkpoint, dataset_shard_provider=dataset_shard_provider, ) diff --git a/python/ray/train/v2/_internal/execution/worker_group/worker_group.py b/python/ray/train/v2/_internal/execution/worker_group/worker_group.py index 81087c0f91c7..a3a81fa2021c 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/worker_group.py +++ b/python/ray/train/v2/_internal/execution/worker_group/worker_group.py @@ -437,6 +437,7 @@ def _init_train_context_on_workers( synchronization_actor=sync_actor, storage_context=self._storage_context, worker_callbacks=self._worker_callbacks_to_propagate, + controller_actor=ray.get_runtime_context().current_actor, **{ arg: arg_values[i] for arg, arg_values in train_context_args.items() }, diff --git a/python/ray/train/v2/api/reported_checkpoint.py b/python/ray/train/v2/api/reported_checkpoint.py new file mode 100644 index 000000000000..2224f52280d4 --- /dev/null +++ b/python/ray/train/v2/api/reported_checkpoint.py @@ -0,0 +1,21 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Dict + +from ray.util.annotations import PublicAPI + +if TYPE_CHECKING: + from ray.train import Checkpoint + + +@dataclass +@PublicAPI(stability="alpha") +class ReportedCheckpoint: + """A user-reported checkpoint and its associated metrics. + + Attributes: + checkpoint: The checkpoint reported by the user. + metrics: The metrics associated with that checkpoint. + """ + + checkpoint: "Checkpoint" + metrics: Dict[str, Any] diff --git a/python/ray/train/v2/api/train_fn_utils.py b/python/ray/train/v2/api/train_fn_utils.py index efd3e6575ef4..06603b8c409b 100644 --- a/python/ray/train/v2/api/train_fn_utils.py +++ b/python/ray/train/v2/api/train_fn_utils.py @@ -1,18 +1,19 @@ -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional -from ray.train import Checkpoint from ray.train.v2._internal.execution.train_fn_utils import get_train_fn_utils from ray.train.v2.api.context import TrainContext from ray.util.annotations import PublicAPI if TYPE_CHECKING: from ray.data import DataIterator + from ray.train import Checkpoint + from ray.train.v2.api.reported_checkpoint import ReportedCheckpoint @PublicAPI(stability="stable") def report( metrics: Dict[str, Any], - checkpoint: Optional[Checkpoint] = None, + checkpoint: Optional["Checkpoint"] = None, checkpoint_dir_name: Optional[str] = None, ): """Report metrics and optionally save a checkpoint. @@ -107,7 +108,7 @@ def get_context() -> TrainContext: @PublicAPI(stability="stable") -def get_checkpoint() -> Optional[Checkpoint]: +def get_checkpoint() -> Optional["Checkpoint"]: """Access the latest reported checkpoint to resume from if one exists. Example: @@ -151,6 +152,51 @@ def train_func(config): return get_train_fn_utils().get_checkpoint() +@PublicAPI(stability="alpha") +def get_all_reported_checkpoints() -> List["ReportedCheckpoint"]: + """Get all the reported checkpoints so far. + + Blocks until Ray Train has finished processing every `ray.train.report` call. + + Example: + + .. testcode:: + + import tempfile + + from ray import train + from ray.train import Checkpoint + from ray.train.torch import TorchTrainer + + + def train_func(config): + start_epoch = 0 + + for epoch in range(start_epoch, config.get("num_epochs", 10)): + # Do training... + + metrics = {"loss": ...} + + with tempfile.TemporaryDirectory() as temp_checkpoint_dir: + # Save the checkpoint... + + checkpoint = Checkpoint.from_directory(temp_checkpoint_dir) + train.report(metrics, checkpoint=checkpoint) + + reported_checkpoints = train.get_all_reported_checkpoints() + # Report artifacts/metrics to experiment tracking framework... + + trainer = TorchTrainer( + train_func, scaling_config=train.ScalingConfig(num_workers=2) + ) + + Returns: + List of ReportedCheckpoint objects that represent the checkpoints and + corresponding metrics reported by the workers. + """ + return get_train_fn_utils().get_all_reported_checkpoints() + + @PublicAPI(stability="stable") def get_dataset_shard(dataset_name: Optional[str] = None) -> Optional["DataIterator"]: """Returns the :class:`ray.data.DataIterator` shard for this worker. diff --git a/python/ray/train/v2/tests/conftest.py b/python/ray/train/v2/tests/conftest.py index 6f52bcd6e438..da060d7e8dee 100644 --- a/python/ray/train/v2/tests/conftest.py +++ b/python/ray/train/v2/tests/conftest.py @@ -3,6 +3,7 @@ import pytest import ray +from ray import runtime_context from ray.train.v2._internal.constants import ( ENABLE_STATE_ACTOR_RECONCILIATION_ENV_VAR, ) @@ -34,3 +35,26 @@ def shutdown_only(): def disable_state_actor_polling(monkeypatch): monkeypatch.setenv(ENABLE_STATE_ACTOR_RECONCILIATION_ENV_VAR, "0") yield + + +@pytest.fixture +def mock_runtime_context(monkeypatch): + @ray.remote + class DummyActor: + pass + + # Must return real actor handle so it can get passed to other actors + # Cannot create actor here since ray has not been initialized yet + def mock_current_actor(self): + return DummyActor.remote() + + # In unit tests where the controller is not an actor, current_actor is + # a DummyActor, which is ok because it won't be called in those tests. + # In unit tests where the controller is an actor, current_actor is the + # controller actor because monkeypatch doesn't propagate to the actor + # process. Those tests can successfully test methods on that actor. + monkeypatch.setattr( + runtime_context.RuntimeContext, "current_actor", property(mock_current_actor) + ) + + yield diff --git a/python/ray/train/v2/tests/test_accelerator_utils.py b/python/ray/train/v2/tests/test_accelerator_utils.py index 8c5cff9cd06f..d9e0149bf64c 100644 --- a/python/ray/train/v2/tests/test_accelerator_utils.py +++ b/python/ray/train/v2/tests/test_accelerator_utils.py @@ -98,7 +98,7 @@ def test_missing_accelerator(): ) -def test_accelerator_setup_callback(mock_gpu_cluster): +def test_accelerator_setup_callback(mock_gpu_cluster, mock_runtime_context): """The accelerator setup callback should set the CUDA_VISIBLE_DEVICES on each worker properly.""" diff --git a/python/ray/train/v2/tests/test_checkpoint_manager.py b/python/ray/train/v2/tests/test_checkpoint_manager.py index e1caf79cd8f9..3debda020e7e 100644 --- a/python/ray/train/v2/tests/test_checkpoint_manager.py +++ b/python/ray/train/v2/tests/test_checkpoint_manager.py @@ -89,9 +89,11 @@ def _training_results_equal( ), ], ) -def test_save_load_state_equivalence( +async def test_save_load_state_equivalence( monkeypatch, tmp_path, checkpoint_config: CheckpointConfig ): + # Use async here because register_checkpoint creates an async task + # Mock the delete function as we don't want report checkpoints to be deleted. monkeypatch.setattr( ray.train.v2._internal.execution.checkpoint.checkpoint_manager, @@ -113,8 +115,9 @@ def test_save_load_state_equivalence( ) # Register the training results into checkpoint manager - for tr in training_results: + for i, tr in enumerate(training_results): checkpoint_manager.register_checkpoint(tr) + assert checkpoint_manager._num_report_calls == i + 1 loaded_checkpoint_manager = CheckpointManager( storage_context=storage_context, checkpoint_config=checkpoint_config, @@ -145,7 +148,8 @@ def test_load_state_error(tmp_path, json_state): checkpoint_manager._load_state(json_state) -def test_before_init_train_context(tmp_path): +async def test_before_init_train_context(tmp_path): + storage_context = StorageContext( storage_path=tmp_path, experiment_dir_name="my_experiment_name", @@ -158,14 +162,14 @@ def test_before_init_train_context(tmp_path): # Assert without a checkpoint. assert checkpoint_manager.before_init_train_context(workers) == { - "checkpoint": [None] * 4 + "checkpoint": [None] * 4, } # Assert with a checkpoint latest_checkpoint_result = _create_dummy_training_results(1, storage_context)[0] - checkpoint_manager._latest_checkpoint_result = latest_checkpoint_result + checkpoint_manager.register_checkpoint(latest_checkpoint_result) assert checkpoint_manager.before_init_train_context(workers) == { - "checkpoint": [latest_checkpoint_result.checkpoint] * 4 + "checkpoint": [latest_checkpoint_result.checkpoint] * 4, } diff --git a/python/ray/train/v2/tests/test_controller.py b/python/ray/train/v2/tests/test_controller.py index 3fbddbc2b64e..80a45fba69a3 100644 --- a/python/ray/train/v2/tests/test_controller.py +++ b/python/ray/train/v2/tests/test_controller.py @@ -36,6 +36,8 @@ create_dummy_run_context, ) +pytestmark = pytest.mark.usefixtures("mock_runtime_context") + @pytest.fixture(autouse=True) def patch_worker_group(monkeypatch): diff --git a/python/ray/train/v2/tests/test_data_parallel_trainer.py b/python/ray/train/v2/tests/test_data_parallel_trainer.py index c4ec69739ed3..007bd1226f23 100644 --- a/python/ray/train/v2/tests/test_data_parallel_trainer.py +++ b/python/ray/train/v2/tests/test_data_parallel_trainer.py @@ -141,6 +141,30 @@ def train_fn(): assert tmp_path.joinpath("validate", str(rank)).exists() +def test_report_get_all_reported_checkpoints(): + """Check that get_all_reported_checkpoints returns checkpoints depending on # report calls.""" + + def train_fn(): + if ray.train.get_context().get_world_rank() == 0: + ray.train.report(metrics={}, checkpoint=None) + with create_dict_checkpoint({}) as checkpoint: + ray.train.report(metrics={}, checkpoint=checkpoint) + assert len(ray.train.get_all_reported_checkpoints()) == 1 + with create_dict_checkpoint({}) as checkpoint: + ray.train.report(metrics={}, checkpoint=checkpoint) + else: + ray.train.report(metrics={}, checkpoint=None) + ray.train.report(metrics={}, checkpoint=None) + ray.train.report(metrics={}, checkpoint=None) + assert len(ray.train.get_all_reported_checkpoints()) == 2 + + trainer = DataParallelTrainer( + train_fn, + scaling_config=ScalingConfig(num_workers=2), + ) + trainer.fit() + + def test_error(tmp_path): def _error_func_rank_0(): """An example train_fun that raises an error on rank 0.""" diff --git a/python/ray/train/v2/tests/test_persistence.py b/python/ray/train/v2/tests/test_persistence.py index b1891bbc6e89..2649c5ce2ad7 100644 --- a/python/ray/train/v2/tests/test_persistence.py +++ b/python/ray/train/v2/tests/test_persistence.py @@ -174,6 +174,10 @@ def train_fn(config): print("Loaded back state from checkpoint:", state) start = state["iter"] + 1 + assert len(ray.train.get_all_reported_checkpoints()) == min( + start, config.get("num_to_keep", float("inf")) + ) + for i in range(start, config.get("num_iterations", 5)): time.sleep(config.get("time_per_iter", 0.25)) @@ -215,6 +219,9 @@ def train_fn(config): ray.train.collective.barrier() if i in config.get("fail_iters", []): + assert len(ray.train.get_all_reported_checkpoints()) == min( + i + 1, config.get("num_to_keep", float("inf")) + ) raise RuntimeError(f"Failing on iter={i}!!") @@ -305,6 +312,10 @@ def test_trainer( exp_name = f"trainer_persistence_test-{uuid.uuid4().hex}" no_checkpoint_ranks = [0] + if checkpoint_config.num_to_keep: + num_to_keep = checkpoint_config.num_to_keep + else: + num_to_keep = float("inf") with _resolve_storage_type(storage_path_type, tmp_path) as ( storage_path, @@ -325,6 +336,7 @@ def test_trainer( # Test that global rank 0 is not required to checkpoint. "no_checkpoint_ranks": no_checkpoint_ranks, "time_per_iter": time_between_reports, + "num_to_keep": num_to_keep, }, scaling_config=ScalingConfig(num_workers=TestConstants.NUM_WORKERS), run_config=run_config, @@ -341,6 +353,7 @@ def test_trainer( # Test that global rank 0 is not required to checkpoint. "no_checkpoint_ranks": no_checkpoint_ranks, "time_per_iter": time_between_reports, + "num_to_keep": num_to_keep, }, scaling_config=ScalingConfig(num_workers=TestConstants.NUM_WORKERS), run_config=run_config, diff --git a/python/ray/train/v2/tests/test_worker_group.py b/python/ray/train/v2/tests/test_worker_group.py index 4146cdd763aa..2b796e751c9a 100644 --- a/python/ray/train/v2/tests/test_worker_group.py +++ b/python/ray/train/v2/tests/test_worker_group.py @@ -32,6 +32,8 @@ from ray.train.v2.api.config import RunConfig from ray.train.v2.tests.util import DummyObjectRefWrapper, create_dummy_run_context +pytestmark = pytest.mark.usefixtures("mock_runtime_context") + @pytest.fixture(autouse=True, scope="module") def ray_start_4_cpus(): From d375745517ed8bf6bb63cd6022eb42b9b757e6d9 Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 2 Sep 2025 20:22:53 +0530 Subject: [PATCH 382/634] Enable ruff for several folders including `python/ray/includes/ ` and `python/ray/streaming/` (#56078) Signed-off-by: czgdp1807 --- .pre-commit-config.yaml | 2 +- pyproject.toml | 6 ------ python/ray/scripts/scripts.py | 30 ++++++++++++++--------------- python/ray/scripts/symmetric_run.py | 11 ++++++----- 4 files changed, 22 insertions(+), 27 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 30ef94097deb..28358c2a9218 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: args: [ --fix, --exit-non-zero-on-fix ] - id: ruff args: [ --select, "I", --fix, --exit-non-zero-on-fix ] - files: '^python/ray/serve/|^python/ray/train|^python/ray/data|^python/ray/_private/|^python/ray/llm/|^python/ray/tune/|^python/ray/dag/' + files: '^python/ray/serve/|^python/ray/train|^python/ray/data|^python/ray/_private/|^python/ray/llm/|^python/ray/tune/|^python/ray/includes/|^python/ray/internal/|^python/ray/ray_operator/|^python/ray/scripts/|^python/ray/streaming/|^python/ray/dag/' - repo: https://github.com/jsh9/pydoclint rev: "0.6.6" diff --git a/pyproject.toml b/pyproject.toml index fde27ee07b05..765fd83f3423 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,12 +65,6 @@ afterray = ["psutil", "setproctitle"] "python/ray/setup-dev.py" = ["I"] "python/ray/cloudpickle/*" = ["I"] "python/ray/dag/__init__.py" = ["I"] -"python/ray/includes/*" = ["I"] -"python/ray/internal/*" = ["I"] -"python/ray/ray_operator/*" = ["I"] -"python/ray/scripts/*" = ["I"] -"python/ray/serve/generated/serve_pb2.py" = ["I"] -"python/ray/streaming/*" = ["I"] "python/ray/tests/*" = ["I"] "python/ray/util/*" = ["I"] "python/ray/workers/*" = ["I"] diff --git a/python/ray/scripts/scripts.py b/python/ray/scripts/scripts.py index 9818eb308604..e05657a5d3c2 100644 --- a/python/ray/scripts/scripts.py +++ b/python/ray/scripts/scripts.py @@ -3,6 +3,7 @@ import logging import os import platform +import shutil import signal import subprocess import sys @@ -10,35 +11,32 @@ import urllib import urllib.parse import warnings -import shutil from datetime import datetime -from typing import Optional, Set, List, Tuple -from ray._common.utils import load_class -from ray.dashboard.modules.metrics import install_and_start_prometheus -from ray.util.check_open_ports import check_open_ports -import requests +from typing import List, Optional, Set, Tuple import click import colorama -import psutil +import requests import yaml import ray +import ray._common.usage.usage_constants as usage_constant import ray._private.ray_constants as ray_constants import ray._private.services as services +from ray._common.network_utils import build_address, parse_address +from ray._common.usage import usage_lib +from ray._common.utils import load_class +from ray._private.internal_api import memory_summary from ray._private.label_utils import ( - parse_node_labels_json, parse_node_labels_from_yaml_file, + parse_node_labels_json, parse_node_labels_string, ) +from ray._private.resource_isolation_config import ResourceIsolationConfig from ray._private.utils import ( get_ray_client_dependency_error, parse_resources_json, ) -from ray._common.network_utils import parse_address, build_address -from ray._private.internal_api import memory_summary -from ray._common.usage import usage_lib -import ray._common.usage.usage_constants as usage_constant from ray.autoscaler._private.cli_logger import add_click_logging_options, cf, cli_logger from ray.autoscaler._private.commands import ( RUN_ENV_TYPES, @@ -57,10 +55,12 @@ ) from ray.autoscaler._private.constants import RAY_PROCESSES from ray.autoscaler._private.fake_multi_node.node_provider import FAKE_HEAD_NODE_ID -from ray.util.annotations import PublicAPI from ray.core.generated import autoscaler_pb2 -from ray._private.resource_isolation_config import ResourceIsolationConfig +from ray.dashboard.modules.metrics import install_and_start_prometheus +from ray.util.annotations import PublicAPI +from ray.util.check_open_ports import check_open_ports +import psutil logger = logging.getLogger(__name__) @@ -2725,9 +2725,9 @@ def add_command_alias(command, name, hidden): try: from ray.util.state.state_cli import ( + logs_state_cli_group, ray_get, ray_list, - logs_state_cli_group, summary_state_cli_group, ) diff --git a/python/ray/scripts/symmetric_run.py b/python/ray/scripts/symmetric_run.py index e93087574369..22f43e58ad57 100644 --- a/python/ray/scripts/symmetric_run.py +++ b/python/ray/scripts/symmetric_run.py @@ -1,18 +1,19 @@ """Symmetric Run for Ray.""" -from typing import List - -import click -import ray import socket -import psutil import subprocess import sys import time +from typing import List +import click + +import ray from ray._private.ray_constants import env_integer from ray._raylet import GcsClient +import psutil + CLUSTER_WAIT_TIMEOUT = env_integer("RAY_SYMMETRIC_RUN_CLUSTER_WAIT_TIMEOUT", 30) From e65e8704d248728bfe1471cac6010a25d8872806 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 2 Sep 2025 11:56:12 -0500 Subject: [PATCH 383/634] [core] Move `gcs/callback.h` to `common/gcs_callback.h` (#56161) These callbacks are used across components through pubub & the GCS client. --------- Signed-off-by: Edward Oakes --- src/ray/common/BUILD.bazel | 8 ++++++++ src/ray/{gcs/callback.h => common/gcs_callbacks.h} | 12 ------------ src/ray/gcs/BUILD.bazel | 8 -------- src/ray/gcs/gcs_client/accessor.h | 2 +- src/ray/gcs/gcs_server/BUILD.bazel | 2 -- src/ray/gcs/gcs_server/gcs_init_data.h | 1 - src/ray/gcs/gcs_server/gcs_table_storage.cc | 1 - src/ray/gcs/pubsub/BUILD.bazel | 2 +- src/ray/gcs/pubsub/gcs_pub_sub.h | 2 +- src/ray/gcs/store_client/BUILD.bazel | 5 +---- src/ray/gcs/store_client/store_client.h | 2 +- 11 files changed, 13 insertions(+), 32 deletions(-) rename src/ray/{gcs/callback.h => common/gcs_callbacks.h} (81%) diff --git a/src/ray/common/BUILD.bazel b/src/ray/common/BUILD.bazel index e7baf5412576..9a2434e20b78 100644 --- a/src/ray/common/BUILD.bazel +++ b/src/ray/common/BUILD.bazel @@ -334,3 +334,11 @@ ray_cc_library( srcs = ["source_location.cc"], hdrs = ["source_location.h"], ) + +ray_cc_library( + name = "gcs_callbacks", + hdrs = ["gcs_callbacks.h"], + deps = [ + "//src/ray/common:status", + ], +) diff --git a/src/ray/gcs/callback.h b/src/ray/common/gcs_callbacks.h similarity index 81% rename from src/ray/gcs/callback.h rename to src/ray/common/gcs_callbacks.h index 5d00d80805d4..1d5da52fec9b 100644 --- a/src/ray/gcs/callback.h +++ b/src/ray/common/gcs_callbacks.h @@ -17,16 +17,11 @@ #include #include -#include "absl/container/flat_hash_map.h" #include "ray/common/status.h" namespace ray { - namespace gcs { -/// This callback is used to notify when a operation completes. -using EmptyCallback = std::function; - /// This callback is used to notify when a write/subscribe to GCS completes. /// \param status Status indicates whether the write/subscribe was successful. using StatusCallback = std::function; @@ -35,7 +30,6 @@ using StatusCallback = std::function; /// \param status Status indicates whether the read was successful. /// \param result The item returned by GCS. If the item to read doesn't exist, /// this optional object is empty. -/// TODO(ryw): make an Either union type to avoid the optional. template using OptionalItemCallback = std::function result)>; @@ -57,11 +51,5 @@ using SubscribeCallback = std::function; template using ItemCallback = std::function; -/// This callback is used to receive multiple key-value items from GCS. -/// \param result The key-value items returned by GCS. -template -using MapCallback = std::function &&result)>; - } // namespace gcs - } // namespace ray diff --git a/src/ray/gcs/BUILD.bazel b/src/ray/gcs/BUILD.bazel index eaaae3219fdd..fc039b56ae85 100644 --- a/src/ray/gcs/BUILD.bazel +++ b/src/ray/gcs/BUILD.bazel @@ -14,11 +14,3 @@ ray_cc_library( "//src/ray/util:time", ], ) - -ray_cc_library( - name = "gcs_callback", - hdrs = ["callback.h"], - deps = [ - "//src/ray/common:status", - ], -) diff --git a/src/ray/gcs/gcs_client/accessor.h b/src/ray/gcs/gcs_client/accessor.h index 0545fb2268da..973ca76e8824 100644 --- a/src/ray/gcs/gcs_client/accessor.h +++ b/src/ray/gcs/gcs_client/accessor.h @@ -20,11 +20,11 @@ #include #include "absl/types/optional.h" +#include "ray/common/gcs_callbacks.h" #include "ray/common/id.h" #include "ray/common/placement_group.h" #include "ray/common/status_or.h" #include "ray/common/task/task_spec.h" -#include "ray/gcs/callback.h" #include "ray/rpc/client_call.h" #include "ray/util/sequencer.h" #include "src/ray/protobuf/autoscaler.pb.h" diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 2e49691bc518..b11f99a222ed 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -18,7 +18,6 @@ ray_cc_library( "//src/ray/common:asio", "//src/ray/common:id", "//src/ray/common:status", - "//src/ray/gcs:gcs_callback", "//src/ray/gcs/store_client", "//src/ray/protobuf:gcs_cc_proto", "@com_google_absl//absl/container:flat_hash_map", @@ -34,7 +33,6 @@ ray_cc_library( ":gcs_table_storage", "//src/ray/common:asio", "//src/ray/common:id", - "//src/ray/gcs:gcs_callback", "//src/ray/protobuf:gcs_cc_proto", "@com_google_absl//absl/container:flat_hash_map", ], diff --git a/src/ray/gcs/gcs_server/gcs_init_data.h b/src/ray/gcs/gcs_server/gcs_init_data.h index f6627499cbfd..d5c2c24c8f2c 100644 --- a/src/ray/gcs/gcs_server/gcs_init_data.h +++ b/src/ray/gcs/gcs_server/gcs_init_data.h @@ -17,7 +17,6 @@ #include "absl/container/flat_hash_map.h" #include "ray/common/asio/postable.h" #include "ray/common/id.h" -#include "ray/gcs/callback.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_table_storage.cc b/src/ray/gcs/gcs_server/gcs_table_storage.cc index 83847884a199..203e8c0e4848 100644 --- a/src/ray/gcs/gcs_server/gcs_table_storage.cc +++ b/src/ray/gcs/gcs_server/gcs_table_storage.cc @@ -22,7 +22,6 @@ #include "ray/common/asio/postable.h" #include "ray/common/id.h" #include "ray/common/status.h" -#include "ray/gcs/callback.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/pubsub/BUILD.bazel b/src/ray/gcs/pubsub/BUILD.bazel index 5c61f477186a..5512c0ad67a4 100644 --- a/src/ray/gcs/pubsub/BUILD.bazel +++ b/src/ray/gcs/pubsub/BUILD.bazel @@ -5,8 +5,8 @@ ray_cc_library( srcs = ["gcs_pub_sub.cc"], hdrs = ["gcs_pub_sub.h"], deps = [ + "//src/ray/common:gcs_callbacks", "//src/ray/common:ray_config", - "//src/ray/gcs:gcs_callback", "//src/ray/pubsub:publisher_interface", "//src/ray/pubsub:subscriber_interface", "//src/ray/rpc:gcs_client", diff --git a/src/ray/gcs/pubsub/gcs_pub_sub.h b/src/ray/gcs/pubsub/gcs_pub_sub.h index abc40f733bbc..d6da94965247 100644 --- a/src/ray/gcs/pubsub/gcs_pub_sub.h +++ b/src/ray/gcs/pubsub/gcs_pub_sub.h @@ -21,7 +21,7 @@ #include #include "absl/synchronization/mutex.h" -#include "ray/gcs/callback.h" +#include "ray/common/gcs_callbacks.h" #include "ray/pubsub/publisher_interface.h" #include "ray/pubsub/subscriber_interface.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/store_client/BUILD.bazel b/src/ray/gcs/store_client/BUILD.bazel index 4a2af91411b8..dbf318a7ba6c 100644 --- a/src/ray/gcs/store_client/BUILD.bazel +++ b/src/ray/gcs/store_client/BUILD.bazel @@ -5,9 +5,9 @@ ray_cc_library( hdrs = ["store_client.h"], deps = [ "//src/ray/common:asio", + "//src/ray/common:gcs_callbacks", "//src/ray/common:id", "//src/ray/common:status", - "//src/ray/gcs:gcs_callback", ], ) @@ -29,7 +29,6 @@ ray_cc_library( "//src/ray/common:asio", "//src/ray/common:ray_config", "//src/ray/common:status", - "//src/ray/gcs:gcs_callback", "//src/ray/stats:stats_lib", "//src/ray/util:container_util", "//src/ray/util:exponential_backoff", @@ -47,7 +46,6 @@ ray_cc_library( deps = [ ":store_client", "//src/ray/common:asio", - "//src/ray/gcs:gcs_callback", "//src/ray/util:concurrent_flat_map", "@com_google_absl//absl/container:node_hash_map", ], @@ -59,6 +57,5 @@ ray_cc_library( hdrs = ["observable_store_client.h"], deps = [ ":store_client", - "//src/ray/gcs:gcs_callback", ], ) diff --git a/src/ray/gcs/store_client/store_client.h b/src/ray/gcs/store_client/store_client.h index e2f6a75fb9fc..f9c78b0057af 100644 --- a/src/ray/gcs/store_client/store_client.h +++ b/src/ray/gcs/store_client/store_client.h @@ -20,9 +20,9 @@ #include "ray/common/asio/io_service_pool.h" #include "ray/common/asio/postable.h" +#include "ray/common/gcs_callbacks.h" #include "ray/common/id.h" #include "ray/common/status.h" -#include "ray/gcs/callback.h" namespace ray { From 9f27a3c8d6073c1d72205dee30dd3643c91875c8 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 2 Sep 2025 12:06:56 -0500 Subject: [PATCH 384/634] [core] Remove `gcs_test_utils_lib` (#56162) Used by GCS client & server tests and some other components. Moving the utils to `test_utils.cc`. --------- Signed-off-by: Edward Oakes --- src/ray/common/BUILD.bazel | 10 +- src/ray/common/test_util.cc | 193 ------ src/ray/common/test_util.h | 116 ---- src/ray/common/test_utils.cc | 594 ++++++++++++++++++ src/ray/common/test_utils.h | 232 +++++++ .../task_execution/tests/BUILD.bazel | 6 +- .../tests/concurrency_group_manager_test.cc | 2 +- .../tests/scheduling_queue_test.cc | 2 +- .../tests/task_receiver_test.cc | 2 +- .../task_submission/tests/BUILD.bazel | 6 +- .../tests/actor_task_submitter_test.cc | 2 +- .../tests/dependency_resolver_test.cc | 2 +- .../tests/normal_task_submitter_test.cc | 2 +- src/ray/core_worker/tests/BUILD.bazel | 18 +- .../core_worker/tests/actor_creator_test.cc | 2 +- .../core_worker/tests/actor_manager_test.cc | 2 +- .../core_worker/tests/memory_store_test.cc | 2 +- .../tests/object_recovery_manager_test.cc | 2 +- .../task_event_buffer_export_event_test.cc | 2 +- .../tests/task_event_buffer_test.cc | 2 +- .../core_worker/tests/task_manager_test.cc | 2 +- src/ray/gcs/gcs_client/tests/BUILD.bazel | 8 +- .../tests/gcs_client_reconnection_test.cc | 2 +- .../gcs/gcs_client/tests/gcs_client_test.cc | 50 +- .../tests/global_state_accessor_test.cc | 29 +- src/ray/gcs/gcs_server/tests/BUILD.bazel | 43 +- .../gcs_actor_manager_export_event_test.cc | 11 +- .../gcs_job_manager_export_event_test.cc | 12 +- .../gcs_node_manager_export_event_test.cc | 11 +- .../tests/gcs_actor_manager_test.cc | 118 ++-- .../tests/gcs_actor_scheduler_mock_test.cc | 2 +- .../tests/gcs_actor_scheduler_test.cc | 68 +- .../gcs_autoscaler_state_manager_test.cc | 237 ++++--- .../gcs_server/tests/gcs_job_manager_test.cc | 35 +- .../gcs_server/tests/gcs_kv_manager_test.cc | 2 +- .../gcs_server/tests/gcs_node_manager_test.cc | 8 +- .../gcs_placement_group_manager_mock_test.cc | 14 +- .../tests/gcs_placement_group_manager_test.cc | 70 +-- .../gcs_placement_group_scheduler_test.cc | 167 +++-- .../tests/gcs_resource_manager_test.cc | 16 +- .../gcs_server/tests/gcs_server_rpc_test.cc | 24 +- .../gcs_server/tests/gcs_server_test_util.h | 2 +- .../tests/gcs_table_storage_test_base.h | 13 +- .../gcs_server/tests/gcs_task_manager_test.cc | 65 +- .../tests/gcs_worker_manager_test.cc | 2 +- .../tests/in_memory_gcs_table_storage_test.cc | 2 +- .../tests/redis_gcs_table_storage_test.cc | 2 +- src/ray/gcs/store_client/tests/BUILD.bazel | 4 +- .../tests/redis_async_context_test.cc | 2 +- .../tests/redis_store_client_test.cc | 2 +- .../tests/store_client_test_base.h | 2 +- src/ray/gcs/tests/BUILD.bazel | 14 - src/ray/gcs/tests/gcs_test_util.h | 483 -------------- src/ray/raylet/scheduling/tests/BUILD.bazel | 4 +- .../tests/cluster_lease_manager_test.cc | 2 +- .../tests/cluster_resource_scheduler_test.cc | 2 +- src/ray/raylet/tests/BUILD.bazel | 5 +- .../tests/lease_dependency_manager_test.cc | 2 +- .../raylet/tests/local_lease_manager_test.cc | 2 +- .../placement_group_resource_manager_test.cc | 74 ++- 60 files changed, 1422 insertions(+), 1390 deletions(-) delete mode 100644 src/ray/common/test_util.cc delete mode 100644 src/ray/common/test_util.h create mode 100644 src/ray/common/test_utils.cc create mode 100644 src/ray/common/test_utils.h delete mode 100644 src/ray/gcs/tests/BUILD.bazel delete mode 100644 src/ray/gcs/tests/gcs_test_util.h diff --git a/src/ray/common/BUILD.bazel b/src/ray/common/BUILD.bazel index 9a2434e20b78..9abdd7832b6a 100644 --- a/src/ray/common/BUILD.bazel +++ b/src/ray/common/BUILD.bazel @@ -11,14 +11,18 @@ ray_cc_library( ) ray_cc_library( - name = "test_util", - srcs = ["test_util.cc"], - hdrs = ["test_util.h"], + name = "test_utils", + srcs = ["test_utils.cc"], + hdrs = ["test_utils.h"], deps = [ ":asio", ":id", ":ray_object", + ":task_common", + "//src/ray/protobuf:autoscaler_cc_grpc", "//src/ray/protobuf:common_cc_proto", + "//src/ray/protobuf:gcs_cc_proto", + "//src/ray/protobuf:gcs_service_cc_grpc", "//src/ray/util:cmd_line_utils", "//src/ray/util:network_util", "//src/ray/util:path_utils", diff --git a/src/ray/common/test_util.cc b/src/ray/common/test_util.cc deleted file mode 100644 index 0fed9ae8dc5e..000000000000 --- a/src/ray/common/test_util.cc +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ray/common/test_util.h" - -#include -#include - -#include "absl/strings/escaping.h" -#include "ray/common/buffer.h" -#include "ray/common/ray_config.h" -#include "ray/common/ray_object.h" -#include "ray/common/test_util.h" -#include "ray/util/cmd_line_utils.h" -#include "ray/util/filesystem.h" -#include "ray/util/logging.h" -#include "ray/util/network_util.h" -#include "ray/util/path_utils.h" -#include "ray/util/process.h" -#include "ray/util/time.h" - -namespace ray { - -void TestSetupUtil::StartUpRedisServers(const std::vector &redis_server_ports, - bool save) { - if (redis_server_ports.empty()) { - TEST_REDIS_SERVER_PORTS.push_back(StartUpRedisServer(0, save)); - } else { - for (const auto &port : redis_server_ports) { - TEST_REDIS_SERVER_PORTS.push_back(StartUpRedisServer(port, save)); - } - } -} - -// start a redis server with specified port, use random one when 0 given -int TestSetupUtil::StartUpRedisServer(int port, bool save) { - int actual_port = port; - if (port == 0) { - static std::atomic srand_called(false); - if (!srand_called.exchange(true)) { - srand(current_time_ms() % RAND_MAX); - } - // Use random port (in range [2000, 7000) to avoid port conflicts between UTs. - do { - actual_port = rand() % 5000 + 2000; - } while (!CheckPortFree(actual_port)); - } - - std::string program = TEST_REDIS_SERVER_EXEC_PATH; -#ifdef _WIN32 - std::vector cmdargs({program, "--loglevel", "warning"}); -#else - std::vector cmdargs; - if (!save) { - cmdargs = {program, "--loglevel", "warning", "--save", "", "--appendonly", "no"}; - } else { - cmdargs = {program, "--loglevel", "warning"}; - } -#endif - cmdargs.insert(cmdargs.end(), {"--port", std::to_string(actual_port)}); - RAY_LOG(INFO) << "Start redis command is: " << CreateCommandLine(cmdargs); - RAY_CHECK(!Process::Spawn(cmdargs, true).second); - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - return actual_port; -} - -void TestSetupUtil::ShutDownRedisServers() { - for (const auto &port : TEST_REDIS_SERVER_PORTS) { - ShutDownRedisServer(port); - } - TEST_REDIS_SERVER_PORTS = std::vector(); -} - -void TestSetupUtil::ShutDownRedisServer(int port) { - std::vector cmdargs( - {TEST_REDIS_CLIENT_EXEC_PATH, "-p", std::to_string(port), "shutdown"}); - RAY_LOG(INFO) << "Stop redis command is: " << CreateCommandLine(cmdargs); - if (Process::Call(cmdargs) != std::error_code()) { - RAY_LOG(WARNING) << "Failed to stop redis. The redis process may no longer exist."; - } - std::this_thread::sleep_for(std::chrono::milliseconds(100)); -} - -void TestSetupUtil::FlushAllRedisServers() { - for (const auto &port : TEST_REDIS_SERVER_PORTS) { - FlushRedisServer(port); - } -} - -void TestSetupUtil::ExecuteRedisCmd(int port, std::vector cmd) { - std::vector cmdargs( - {TEST_REDIS_CLIENT_EXEC_PATH, "-p", std::to_string(port)}); - cmdargs.insert(cmdargs.end(), cmd.begin(), cmd.end()); - RAY_LOG(INFO) << "Send command to redis: " << CreateCommandLine(cmdargs); - if (Process::Call(cmdargs)) { - RAY_LOG(WARNING) << "Failed to send request to redis."; - } -} - -void TestSetupUtil::FlushRedisServer(int port) { - std::vector cmdargs( - {TEST_REDIS_CLIENT_EXEC_PATH, "-p", std::to_string(port), "flushall"}); - RAY_LOG(INFO) << "Cleaning up redis with command: " << CreateCommandLine(cmdargs); - if (Process::Call(cmdargs)) { - RAY_LOG(WARNING) << "Failed to flush redis. The redis process may no longer exist."; - } - std::this_thread::sleep_for(std::chrono::milliseconds(100)); -} - -bool WaitReady(std::future future, const std::chrono::milliseconds &timeout_ms) { - auto status = future.wait_for(timeout_ms); - return status == std::future_status::ready && future.get(); -} - -bool WaitForCondition(std::function condition, int timeout_ms) { - int wait_time = 0; - while (true) { - if (condition()) { - return true; - } - - // sleep 10ms. - const int wait_interval_ms = 10; - std::this_thread::sleep_for(std::chrono::milliseconds(wait_interval_ms)); - wait_time += wait_interval_ms; - if (wait_time > timeout_ms) { - break; - } - } - return false; -} - -void WaitForExpectedCount(std::atomic ¤t_count, - int expected_count, - int timeout_ms) { - auto condition = [¤t_count, expected_count]() { - return current_count == expected_count; - }; - EXPECT_TRUE(WaitForCondition(condition, timeout_ms)); -} - -TaskID RandomTaskId() { - std::string data(TaskID::Size(), 0); - FillRandom(&data); - return TaskID::FromBinary(data); -} - -JobID RandomJobId() { - std::string data(JobID::Size(), 0); - FillRandom(&data); - return JobID::FromBinary(data); -} - -std::shared_ptr GenerateRandomBuffer() { - auto seed = std::chrono::high_resolution_clock::now().time_since_epoch().count(); - std::mt19937 gen(seed); - std::uniform_int_distribution<> dis(1, 10); - std::uniform_int_distribution<> value_dis(1, 255); - - std::vector arg1(dis(gen), value_dis(gen)); - return std::make_shared(arg1.data(), arg1.size(), true); -} - -std::shared_ptr GenerateRandomObject( - const std::vector &inlined_ids) { - std::vector refs; - for (const auto &inlined_id : inlined_ids) { - rpc::ObjectReference ref; - ref.set_object_id(inlined_id.Binary()); - refs.push_back(ref); - } - return std::make_shared(GenerateRandomBuffer(), nullptr, refs); -} - -/// Path to redis server executable binary. -std::string TEST_REDIS_SERVER_EXEC_PATH; -/// Path to redis client executable binary. -std::string TEST_REDIS_CLIENT_EXEC_PATH; -/// Ports of redis server. -std::vector TEST_REDIS_SERVER_PORTS; - -} // namespace ray diff --git a/src/ray/common/test_util.h b/src/ray/common/test_util.h deleted file mode 100644 index cdb9ae4dfc88..000000000000 --- a/src/ray/common/test_util.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include - -#include "gtest/gtest.h" -#include "ray/common/asio/asio_util.h" -#include "ray/common/id.h" -#include "src/ray/protobuf/common.pb.h" -namespace ray { - -static inline std::vector ObjectIdsToRefs( - std::vector object_ids) { - std::vector refs; - for (const auto &object_id : object_ids) { - rpc::ObjectReference ref; - ref.set_object_id(object_id.Binary()); - refs.push_back(ref); - } - return refs; -} - -class Buffer; -class RayObject; - -/// Wait until the future is ready, or timeout is reached. -/// -/// \param[in] future The future to wait for. -/// \param[in] timeout_ms Timeout in milliseconds to wait for for. -/// \return Whether the future is ready. -bool WaitReady(std::future future, const std::chrono::milliseconds &timeout_ms); - -/// Wait until the condition is met, or timeout is reached. -/// -/// \param[in] condition The condition to wait for. -/// \param[in] timeout_ms Timeout in milliseconds to wait for for. -/// \return Whether the condition is met. -bool WaitForCondition(std::function condition, int timeout_ms); - -/// Wait until the expected count is met, or timeout is reached. -/// -/// \param[in] current_count The current count. -/// \param[in] expected_count The expected count. -/// \param[in] timeout_ms Timeout in milliseconds to wait for for. -/// \return Whether the expected count is met. -void WaitForExpectedCount(std::atomic ¤t_count, - int expected_count, - int timeout_ms = 60000); - -// A helper function to return a random task id. -TaskID RandomTaskId(); - -// A helper function to return a random job id. -JobID RandomJobId(); - -std::shared_ptr GenerateRandomBuffer(); - -std::shared_ptr GenerateRandomObject( - const std::vector &inlined_ids = {}); - -/// Path to redis server executable binary. -extern std::string TEST_REDIS_SERVER_EXEC_PATH; -/// Path to redis client executable binary. -extern std::string TEST_REDIS_CLIENT_EXEC_PATH; -/// Ports of redis server. -extern std::vector TEST_REDIS_SERVER_PORTS; - -//-------------------------------------------------------------------------------- -// COMPONENT MANAGEMENT CLASSES FOR TEST CASES -//-------------------------------------------------------------------------------- -/// Test cases can use it to start/stop/flush redis server(s). -class TestSetupUtil { - public: - static void StartUpRedisServers(const std::vector &redis_server_ports, - bool save = false); - static void ShutDownRedisServers(); - static void FlushAllRedisServers(); - - static void ExecuteRedisCmd(int port, std::vector cmd); - static int StartUpRedisServer(int port, bool save = false); - static void ShutDownRedisServer(int port); - static void FlushRedisServer(int port); -}; - -template -struct SaveArgToUniquePtrAction { - std::unique_ptr *pointer; - - template - void operator()(const Args &...args) const { - *pointer = std::make_unique(std::get(std::tie(args...))); - } -}; - -// Copies the k-th arg with make_unique(arg) into ptr. -template -SaveArgToUniquePtrAction SaveArgToUniquePtr(std::unique_ptr *ptr) { - return {ptr}; -} - -} // namespace ray diff --git a/src/ray/common/test_utils.cc b/src/ray/common/test_utils.cc new file mode 100644 index 000000000000..31b4530df525 --- /dev/null +++ b/src/ray/common/test_utils.cc @@ -0,0 +1,594 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/common/test_utils.h" + +#include +#include + +#include "absl/strings/escaping.h" +#include "ray/common/buffer.h" +#include "ray/common/ray_object.h" +#include "ray/common/task/task_util.h" +#include "ray/util/cmd_line_utils.h" +#include "ray/util/filesystem.h" +#include "ray/util/logging.h" +#include "ray/util/network_util.h" +#include "ray/util/path_utils.h" +#include "ray/util/process.h" +#include "ray/util/time.h" + +namespace ray { + +void TestSetupUtil::StartUpRedisServers(const std::vector &redis_server_ports, + bool save) { + if (redis_server_ports.empty()) { + TEST_REDIS_SERVER_PORTS.push_back(StartUpRedisServer(0, save)); + } else { + for (const auto &port : redis_server_ports) { + TEST_REDIS_SERVER_PORTS.push_back(StartUpRedisServer(port, save)); + } + } +} + +// start a redis server with specified port, use random one when 0 given +int TestSetupUtil::StartUpRedisServer(int port, bool save) { + int actual_port = port; + if (port == 0) { + static std::atomic srand_called(false); + if (!srand_called.exchange(true)) { + srand(current_time_ms() % RAND_MAX); + } + // Use random port (in range [2000, 7000) to avoid port conflicts between UTs. + do { + actual_port = rand() % 5000 + 2000; + } while (!CheckPortFree(actual_port)); + } + + std::string program = TEST_REDIS_SERVER_EXEC_PATH; +#ifdef _WIN32 + std::vector cmdargs({program, "--loglevel", "warning"}); +#else + std::vector cmdargs; + if (!save) { + cmdargs = {program, "--loglevel", "warning", "--save", "", "--appendonly", "no"}; + } else { + cmdargs = {program, "--loglevel", "warning"}; + } +#endif + cmdargs.insert(cmdargs.end(), {"--port", std::to_string(actual_port)}); + RAY_LOG(INFO) << "Start redis command is: " << CreateCommandLine(cmdargs); + RAY_CHECK(!Process::Spawn(cmdargs, true).second); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + return actual_port; +} + +void TestSetupUtil::ShutDownRedisServers() { + for (const auto &port : TEST_REDIS_SERVER_PORTS) { + ShutDownRedisServer(port); + } + TEST_REDIS_SERVER_PORTS = std::vector(); +} + +void TestSetupUtil::ShutDownRedisServer(int port) { + std::vector cmdargs( + {TEST_REDIS_CLIENT_EXEC_PATH, "-p", std::to_string(port), "shutdown"}); + RAY_LOG(INFO) << "Stop redis command is: " << CreateCommandLine(cmdargs); + if (Process::Call(cmdargs) != std::error_code()) { + RAY_LOG(WARNING) << "Failed to stop redis. The redis process may no longer exist."; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); +} + +void TestSetupUtil::FlushAllRedisServers() { + for (const auto &port : TEST_REDIS_SERVER_PORTS) { + FlushRedisServer(port); + } +} + +void TestSetupUtil::ExecuteRedisCmd(int port, std::vector cmd) { + std::vector cmdargs( + {TEST_REDIS_CLIENT_EXEC_PATH, "-p", std::to_string(port)}); + cmdargs.insert(cmdargs.end(), cmd.begin(), cmd.end()); + RAY_LOG(INFO) << "Send command to redis: " << CreateCommandLine(cmdargs); + if (Process::Call(cmdargs)) { + RAY_LOG(WARNING) << "Failed to send request to redis."; + } +} + +void TestSetupUtil::FlushRedisServer(int port) { + std::vector cmdargs( + {TEST_REDIS_CLIENT_EXEC_PATH, "-p", std::to_string(port), "flushall"}); + RAY_LOG(INFO) << "Cleaning up redis with command: " << CreateCommandLine(cmdargs); + if (Process::Call(cmdargs)) { + RAY_LOG(WARNING) << "Failed to flush redis. The redis process may no longer exist."; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); +} + +bool WaitReady(std::future future, const std::chrono::milliseconds &timeout_ms) { + auto status = future.wait_for(timeout_ms); + return status == std::future_status::ready && future.get(); +} + +bool WaitForCondition(std::function condition, int timeout_ms) { + int wait_time = 0; + while (true) { + if (condition()) { + return true; + } + + // sleep 10ms. + const int wait_interval_ms = 10; + std::this_thread::sleep_for(std::chrono::milliseconds(wait_interval_ms)); + wait_time += wait_interval_ms; + if (wait_time > timeout_ms) { + break; + } + } + return false; +} + +void WaitForExpectedCount(std::atomic ¤t_count, + int expected_count, + int timeout_ms) { + auto condition = [¤t_count, expected_count]() { + return current_count == expected_count; + }; + EXPECT_TRUE(WaitForCondition(condition, timeout_ms)); +} + +TaskID RandomTaskId() { + std::string data(TaskID::Size(), 0); + FillRandom(&data); + return TaskID::FromBinary(data); +} + +JobID RandomJobId() { + std::string data(JobID::Size(), 0); + FillRandom(&data); + return JobID::FromBinary(data); +} + +std::shared_ptr GenerateRandomBuffer() { + auto seed = std::chrono::high_resolution_clock::now().time_since_epoch().count(); + std::mt19937 gen(seed); + std::uniform_int_distribution<> dis(1, 10); + std::uniform_int_distribution<> value_dis(1, 255); + + std::vector arg1(dis(gen), value_dis(gen)); + return std::make_shared(arg1.data(), arg1.size(), true); +} + +std::shared_ptr GenerateRandomObject( + const std::vector &inlined_ids) { + std::vector refs; + for (const auto &inlined_id : inlined_ids) { + rpc::ObjectReference ref; + ref.set_object_id(inlined_id.Binary()); + refs.push_back(ref); + } + return std::make_shared(GenerateRandomBuffer(), nullptr, refs); +} + +TaskSpecification GenActorCreationTask( + const JobID &job_id, + int max_restarts, + bool detached, + const std::string &name, + const std::string &ray_namespace, + const rpc::Address &owner_address, + std::unordered_map required_resources, + std::unordered_map required_placement_resources) { + TaskSpecBuilder builder; + rpc::JobConfig kJobConfig; + auto actor_id = ActorID::Of(job_id, RandomTaskId(), 0); + auto task_id = TaskID::ForActorCreationTask(actor_id); + FunctionDescriptor function_descriptor; + function_descriptor = FunctionDescriptorBuilder::BuildPython("", "", "", ""); + builder.SetCommonTaskSpec(task_id, + name + ":" + function_descriptor->CallString(), + Language::PYTHON, + function_descriptor, + job_id, + kJobConfig, + TaskID::Nil(), + 0, + TaskID::Nil(), + owner_address, + 1, + false, + false, + -1, + required_resources, + required_placement_resources, + "", + 0, + TaskID::Nil(), + ""); + rpc::SchedulingStrategy scheduling_strategy; + scheduling_strategy.mutable_default_scheduling_strategy(); + builder.SetActorCreationTaskSpec(actor_id, + {}, + scheduling_strategy, + max_restarts, + /*max_task_retries=*/0, + {}, + 1, + detached, + name, + ray_namespace); + return std::move(builder).ConsumeAndBuild(); +} + +rpc::CreateActorRequest GenCreateActorRequest(const JobID &job_id, + int max_restarts, + bool detached, + const std::string &name, + const std::string &ray_namespace) { + rpc::Address owner_address; + owner_address.set_node_id(NodeID::FromRandom().Binary()); + owner_address.set_ip_address("1234"); + owner_address.set_port(5678); + owner_address.set_worker_id(WorkerID::FromRandom().Binary()); + auto actor_creation_task_spec = GenActorCreationTask( + job_id, max_restarts, detached, name, ray_namespace, owner_address); + rpc::CreateActorRequest request; + request.mutable_task_spec()->CopyFrom(actor_creation_task_spec.GetMessage()); + return request; +} + +rpc::RegisterActorRequest GenRegisterActorRequest(const JobID &job_id, + int max_restarts, + bool detached, + const std::string &name, + const std::string &ray_namespace) { + rpc::Address owner_address; + owner_address.set_node_id(NodeID::FromRandom().Binary()); + owner_address.set_ip_address("1234"); + owner_address.set_port(5678); + owner_address.set_worker_id(WorkerID::FromRandom().Binary()); + auto actor_creation_task_spec = GenActorCreationTask( + job_id, max_restarts, detached, name, ray_namespace, owner_address); + rpc::RegisterActorRequest request; + request.mutable_task_spec()->CopyFrom(actor_creation_task_spec.GetMessage()); + return request; +} + +PlacementGroupSpecification GenPlacementGroupCreation( + const std::string &name, + std::vector> &bundles, + rpc::PlacementStrategy strategy, + const JobID &job_id, + const ActorID &actor_id) { + PlacementGroupSpecBuilder builder; + + auto placement_group_id = PlacementGroupID::Of(job_id); + builder.SetPlacementGroupSpec(placement_group_id, + name, + bundles, + strategy, + /* is_detached */ false, + /* soft_target_node_id */ NodeID::Nil(), + job_id, + actor_id, + /* is_creator_detached */ false); + return builder.Build(); +} + +rpc::CreatePlacementGroupRequest GenCreatePlacementGroupRequest( + const std::string name, + rpc::PlacementStrategy strategy, + int bundles_count, + double cpu_num, + const JobID job_id, + const ActorID &actor_id) { + rpc::CreatePlacementGroupRequest request; + std::vector> bundles; + std::unordered_map bundle; + bundle["CPU"] = cpu_num; + for (int index = 0; index < bundles_count; ++index) { + bundles.push_back(bundle); + } + auto placement_group_creation_spec = + GenPlacementGroupCreation(name, bundles, strategy, job_id, actor_id); + request.mutable_placement_group_spec()->CopyFrom( + placement_group_creation_spec.GetMessage()); + return request; +} +std::shared_ptr GenNodeInfo(uint16_t port, + const std::string address, + const std::string node_name) { + auto node = std::make_shared(); + node->set_node_id(NodeID::FromRandom().Binary()); + node->set_node_manager_port(port); + node->set_node_manager_address(address); + node->set_node_name(node_name); + node->set_instance_id("instance_x"); + node->set_state(rpc::GcsNodeInfo::ALIVE); + return node; +} + +std::shared_ptr GenJobTableData(JobID job_id) { + auto job_table_data = std::make_shared(); + job_table_data->set_job_id(job_id.Binary()); + job_table_data->set_is_dead(false); + job_table_data->set_timestamp(current_sys_time_ms()); + job_table_data->set_driver_ip_address("127.0.0.1"); + rpc::Address address; + address.set_ip_address("127.0.0.1"); + address.set_port(1234); + address.set_node_id(UniqueID::FromRandom().Binary()); + address.set_worker_id(UniqueID::FromRandom().Binary()); + job_table_data->mutable_driver_address()->CopyFrom(address); + job_table_data->set_driver_pid(5667L); + return job_table_data; +} + +std::shared_ptr GenActorTableData(const JobID &job_id) { + auto actor_table_data = std::make_shared(); + ActorID actor_id = ActorID::Of(job_id, RandomTaskId(), 0); + actor_table_data->set_actor_id(actor_id.Binary()); + actor_table_data->set_job_id(job_id.Binary()); + actor_table_data->set_state(rpc::ActorTableData::ALIVE); + actor_table_data->set_max_restarts(1); + actor_table_data->set_num_restarts(0); + return actor_table_data; +} + +std::shared_ptr GenErrorTableData(const JobID &job_id) { + auto error_table_data = std::make_shared(); + error_table_data->set_job_id(job_id.Binary()); + return error_table_data; +} + +std::shared_ptr GenWorkerTableData() { + auto worker_table_data = std::make_shared(); + worker_table_data->set_timestamp(std::time(nullptr)); + return worker_table_data; +} + +std::shared_ptr GenAddJobRequest( + const JobID &job_id, + const std::string &ray_namespace, + const std::optional &submission_id, + const std::optional &address) { + auto job_config_data = std::make_shared(); + job_config_data->set_ray_namespace(ray_namespace); + + auto job_table_data = std::make_shared(); + job_table_data->set_job_id(job_id.Binary()); + job_table_data->mutable_config()->CopyFrom(*job_config_data); + if (address.has_value()) { + job_table_data->mutable_driver_address()->CopyFrom(address.value()); + } else { + rpc::Address dummy_address; + dummy_address.set_port(1234); + dummy_address.set_node_id(NodeID::FromRandom().Binary()); + dummy_address.set_ip_address("123.456.7.8"); + dummy_address.set_worker_id(WorkerID::FromRandom().Binary()); + job_table_data->mutable_driver_address()->CopyFrom(dummy_address); + } + if (submission_id.has_value()) { + job_table_data->mutable_config()->mutable_metadata()->insert( + {"job_submission_id", submission_id.value()}); + } + + auto add_job_request = std::make_shared(); + add_job_request->mutable_data()->CopyFrom(*job_table_data); + return add_job_request; +} + +rpc::TaskEventData GenTaskEventsData(const std::vector &task_events, + int32_t num_profile_task_events_dropped, + int32_t num_status_task_events_dropped) { + rpc::TaskEventData data; + for (auto &events : task_events) { + auto new_events = data.add_events_by_task(); + new_events->CopyFrom(events); + } + + for (int i = 0; i < num_status_task_events_dropped; ++i) { + rpc::TaskAttempt rpc_task_attempt; + rpc_task_attempt.set_task_id(RandomTaskId().Binary()); + rpc_task_attempt.set_attempt_number(0); + *(data.add_dropped_task_attempts()) = rpc_task_attempt; + } + + data.set_num_profile_events_dropped(num_profile_task_events_dropped); + data.set_job_id(JobID::FromInt(0).Binary()); + + return data; +} + +rpc::events::RayEventsData GenRayEventsData( + const std::vector &task_events, + const std::vector &drop_tasks) { + rpc::events::RayEventsData data; + rpc::events::TaskEventsMetadata metadata; + for (const auto &task_attempt : drop_tasks) { + rpc::TaskAttempt rpc_task_attempt; + rpc_task_attempt.set_task_id(task_attempt.first.Binary()); + rpc_task_attempt.set_attempt_number(task_attempt.second); + *(metadata.add_dropped_task_attempts()) = rpc_task_attempt; + } + data.mutable_task_events_metadata()->CopyFrom(metadata); + for (const auto &task_event : task_events) { + rpc::events::RayEvent ray_event; + rpc::events::TaskDefinitionEvent task_definition_event; + task_definition_event.set_task_id(task_event.task_id()); + task_definition_event.set_task_attempt(task_event.attempt_number()); + task_definition_event.set_job_id(task_event.job_id()); + if (task_event.has_task_info()) { + const auto &task_info = task_event.task_info(); + task_definition_event.set_task_type(task_info.type()); + task_definition_event.set_task_name(task_info.name()); + task_definition_event.set_language(task_info.language()); + } + ray_event.set_event_id(task_event.task_id()); + ray_event.set_event_type(rpc::events::RayEvent::TASK_DEFINITION_EVENT); + ray_event.set_message("test"); + ray_event.mutable_task_definition_event()->CopyFrom(task_definition_event); + *(data.add_events()) = ray_event; + } + + return data; +} + +rpc::TaskEventData GenTaskEventsDataLoss(const std::vector &drop_tasks, + int job_id) { + rpc::TaskEventData data; + for (const auto &task_attempt : drop_tasks) { + rpc::TaskAttempt rpc_task_attempt; + rpc_task_attempt.set_task_id(task_attempt.first.Binary()); + rpc_task_attempt.set_attempt_number(task_attempt.second); + *(data.add_dropped_task_attempts()) = rpc_task_attempt; + } + data.set_job_id(JobID::FromInt(job_id).Binary()); + + return data; +} + +rpc::ResourceDemand GenResourceDemand( + const absl::flat_hash_map &resource_demands, + int64_t num_ready_queued, + int64_t num_infeasible, + int64_t num_backlog, + const std::vector &label_selectors) { + rpc::ResourceDemand resource_demand; + for (const auto &resource : resource_demands) { + (*resource_demand.mutable_shape())[resource.first] = resource.second; + } + resource_demand.set_num_ready_requests_queued(num_ready_queued); + resource_demand.set_num_infeasible_requests_queued(num_infeasible); + resource_demand.set_backlog_size(num_backlog); + for (const auto &selector : label_selectors) { + *resource_demand.add_label_selectors() = selector; + } + return resource_demand; +} + +void FillResourcesData( + rpc::ResourcesData &resources_data, + const NodeID &node_id, + const absl::flat_hash_map &available_resources, + const absl::flat_hash_map &total_resources, + int64_t idle_ms, + bool is_draining, + int64_t draining_deadline_timestamp_ms) { + resources_data.set_node_id(node_id.Binary()); + for (const auto &resource : available_resources) { + (*resources_data.mutable_resources_available())[resource.first] = resource.second; + } + for (const auto &resource : total_resources) { + (*resources_data.mutable_resources_total())[resource.first] = resource.second; + } + resources_data.set_idle_duration_ms(idle_ms); + resources_data.set_is_draining(is_draining); + resources_data.set_draining_deadline_timestamp_ms(draining_deadline_timestamp_ms); +} + +void FillResourcesData(rpc::ResourcesData &data, + const std::string &node_id, + std::vector demands) { + auto load_by_shape = data.mutable_resource_load_by_shape(); + auto agg_load = data.mutable_resource_load(); + for (const auto &demand : demands) { + load_by_shape->add_resource_demands()->CopyFrom(demand); + for (const auto &resource : demand.shape()) { + (*agg_load)[resource.first] += + (resource.second * (demand.num_ready_requests_queued() + + demand.num_infeasible_requests_queued())); + } + } + data.set_node_id(node_id); +} + +std::shared_ptr GenPlacementGroupLoad( + std::vector placement_group_table_data_vec) { + auto placement_group_load = std::make_shared(); + for (auto &placement_group_table_data : placement_group_table_data_vec) { + placement_group_load->add_placement_group_data()->CopyFrom( + placement_group_table_data); + } + return placement_group_load; +} + +rpc::PlacementGroupTableData GenPlacementGroupTableData( + const PlacementGroupID &placement_group_id, + const JobID &job_id, + const std::vector> &bundles, + const std::vector &nodes, + rpc::PlacementStrategy strategy, + const rpc::PlacementGroupTableData::PlacementGroupState state, + const std::string &name, + const ActorID &actor_id) { + rpc::PlacementGroupTableData placement_group_table_data; + placement_group_table_data.set_placement_group_id(placement_group_id.Binary()); + placement_group_table_data.set_state(state); + placement_group_table_data.set_name(name); + placement_group_table_data.set_strategy(strategy); + RAY_CHECK(bundles.size() == nodes.size()); + size_t i = 0; + for (auto &bundle : bundles) { + // Add unit resources + auto bundle_spec = placement_group_table_data.add_bundles(); + for (auto &resource : bundle) { + (*bundle_spec->mutable_unit_resources())[resource.first] = resource.second; + } + + // Add node id + const auto &node = nodes[i]; + if (!node.empty()) { + bundle_spec->set_node_id(node); + } + + i++; + } + return placement_group_table_data; +} +rpc::autoscaler::ClusterResourceConstraint GenClusterResourcesConstraint( + const std::vector> &request_resources, + const std::vector &count_array) { + rpc::autoscaler::ClusterResourceConstraint constraint; + RAY_CHECK(request_resources.size() == count_array.size()); + for (size_t i = 0; i < request_resources.size(); i++) { + auto &resource = request_resources[i]; + auto count = count_array[i]; + auto bundle = constraint.add_resource_requests(); + bundle->set_count(count); + bundle->mutable_request()->mutable_resources_bundle()->insert(resource.begin(), + resource.end()); + } + return constraint; +} +// Read all lines of a file into vector vc +void ReadContentFromFile(std::vector &vc, std::string log_file) { + std::string line; + std::ifstream read_file; + read_file.open(log_file, std::ios::binary); + while (std::getline(read_file, line)) { + vc.push_back(line); + } + read_file.close(); +} + +/// Path to redis server executable binary. +std::string TEST_REDIS_SERVER_EXEC_PATH; +/// Path to redis client executable binary. +std::string TEST_REDIS_CLIENT_EXEC_PATH; +/// Ports of redis server. +std::vector TEST_REDIS_SERVER_PORTS; + +} // namespace ray diff --git a/src/ray/common/test_utils.h b/src/ray/common/test_utils.h new file mode 100644 index 000000000000..70be73269c9c --- /dev/null +++ b/src/ray/common/test_utils.h @@ -0,0 +1,232 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "gtest/gtest.h" +#include "ray/common/asio/asio_util.h" +#include "ray/common/id.h" +#include "ray/common/placement_group.h" +#include "ray/common/task/task_spec.h" +#include "src/ray/protobuf/autoscaler.pb.h" +#include "src/ray/protobuf/common.pb.h" +#include "src/ray/protobuf/gcs.pb.h" +#include "src/ray/protobuf/gcs_service.grpc.pb.h" + +namespace ray { + +static inline std::vector ObjectIdsToRefs( + std::vector object_ids) { + std::vector refs; + for (const auto &object_id : object_ids) { + rpc::ObjectReference ref; + ref.set_object_id(object_id.Binary()); + refs.push_back(ref); + } + return refs; +} + +class Buffer; +class RayObject; + +/// Wait until the future is ready, or timeout is reached. +/// +/// \param[in] future The future to wait for. +/// \param[in] timeout_ms Timeout in milliseconds to wait for for. +/// \return Whether the future is ready. +bool WaitReady(std::future future, const std::chrono::milliseconds &timeout_ms); + +/// Wait until the condition is met, or timeout is reached. +/// +/// \param[in] condition The condition to wait for. +/// \param[in] timeout_ms Timeout in milliseconds to wait for for. +/// \return Whether the condition is met. +bool WaitForCondition(std::function condition, int timeout_ms); + +/// Wait until the expected count is met, or timeout is reached. +/// +/// \param[in] current_count The current count. +/// \param[in] expected_count The expected count. +/// \param[in] timeout_ms Timeout in milliseconds to wait for for. +/// \return Whether the expected count is met. +void WaitForExpectedCount(std::atomic ¤t_count, + int expected_count, + int timeout_ms = 60000); + +// A helper function to return a random task id. +TaskID RandomTaskId(); + +// A helper function to return a random job id. +JobID RandomJobId(); + +std::shared_ptr GenerateRandomBuffer(); + +std::shared_ptr GenerateRandomObject( + const std::vector &inlined_ids = {}); + +/// Path to redis server executable binary. +extern std::string TEST_REDIS_SERVER_EXEC_PATH; +/// Path to redis client executable binary. +extern std::string TEST_REDIS_CLIENT_EXEC_PATH; +/// Ports of redis server. +extern std::vector TEST_REDIS_SERVER_PORTS; + +//-------------------------------------------------------------------------------- +// COMPONENT MANAGEMENT CLASSES FOR TEST CASES +//-------------------------------------------------------------------------------- +/// Test cases can use it to start/stop/flush redis server(s). +class TestSetupUtil { + public: + static void StartUpRedisServers(const std::vector &redis_server_ports, + bool save = false); + static void ShutDownRedisServers(); + static void FlushAllRedisServers(); + + static void ExecuteRedisCmd(int port, std::vector cmd); + static int StartUpRedisServer(int port, bool save = false); + static void ShutDownRedisServer(int port); + static void FlushRedisServer(int port); +}; + +template +struct SaveArgToUniquePtrAction { + std::unique_ptr *pointer; + + template + void operator()(const Args &...args) const { + *pointer = std::make_unique(std::get(std::tie(args...))); + } +}; + +// Copies the k-th arg with make_unique(arg) into ptr. +template +SaveArgToUniquePtrAction SaveArgToUniquePtr(std::unique_ptr *ptr) { + return {ptr}; +} + +TaskSpecification GenActorCreationTask( + const JobID &job_id, + int max_restarts, + bool detached, + const std::string &name, + const std::string &ray_namespace, + const rpc::Address &owner_address, + std::unordered_map required_resources = + std::unordered_map(), + std::unordered_map required_placement_resources = + std::unordered_map()); + +rpc::CreateActorRequest GenCreateActorRequest(const JobID &job_id, + int max_restarts = 0, + bool detached = false, + const std::string &name = "", + const std::string &ray_namespace = ""); + +rpc::RegisterActorRequest GenRegisterActorRequest( + const JobID &job_id, + int max_restarts = 0, + bool detached = false, + const std::string &name = "", + const std::string &ray_namespace = "test"); + +PlacementGroupSpecification GenPlacementGroupCreation( + const std::string &name, + std::vector> &bundles, + rpc::PlacementStrategy strategy, + const JobID &job_id, + const ActorID &actor_id); + +rpc::CreatePlacementGroupRequest GenCreatePlacementGroupRequest( + const std::string name = "", + rpc::PlacementStrategy strategy = rpc::PlacementStrategy::SPREAD, + int bundles_count = 2, + double cpu_num = 1.0, + const JobID job_id = JobID::FromInt(1), + const ActorID &actor_id = ActorID::Nil()); + +std::shared_ptr GenNodeInfo( + uint16_t port = 0, + const std::string address = "127.0.0.1", + const std::string node_name = "Mocker_node"); + +std::shared_ptr GenJobTableData(JobID job_id); + +std::shared_ptr GenActorTableData(const JobID &job_id); + +std::shared_ptr GenErrorTableData(const JobID &job_id); + +std::shared_ptr GenWorkerTableData(); + +std::shared_ptr GenAddJobRequest( + const JobID &job_id, + const std::string &ray_namespace, + const std::optional &submission_id = std::nullopt, + const std::optional &address = std::nullopt); + +rpc::TaskEventData GenTaskEventsData(const std::vector &task_events, + int32_t num_profile_task_events_dropped = 0, + int32_t num_status_task_events_dropped = 0); + +rpc::events::RayEventsData GenRayEventsData( + const std::vector &task_events, + const std::vector &drop_tasks); + +rpc::TaskEventData GenTaskEventsDataLoss(const std::vector &drop_tasks, + int job_id = 0); + +rpc::ResourceDemand GenResourceDemand( + const absl::flat_hash_map &resource_demands, + int64_t num_ready_queued, + int64_t num_infeasible, + int64_t num_backlog, + const std::vector &label_selectors = {}); + +void FillResourcesData( + rpc::ResourcesData &resources_data, + const NodeID &node_id, + const absl::flat_hash_map &available_resources, + const absl::flat_hash_map &total_resources, + int64_t idle_ms = 0, + bool is_draining = false, + int64_t draining_deadline_timestamp_ms = -1); + +void FillResourcesData(rpc::ResourcesData &data, + const std::string &node_id, + std::vector demands); + +std::shared_ptr GenPlacementGroupLoad( + std::vector placement_group_table_data_vec); + +rpc::PlacementGroupTableData GenPlacementGroupTableData( + const PlacementGroupID &placement_group_id, + const JobID &job_id, + const std::vector> &bundles, + const std::vector &nodes, + rpc::PlacementStrategy strategy, + const rpc::PlacementGroupTableData::PlacementGroupState state, + const std::string &name = "", + const ActorID &actor_id = ActorID::Nil()); + +rpc::autoscaler::ClusterResourceConstraint GenClusterResourcesConstraint( + const std::vector> &request_resources, + const std::vector &count_array); + +// Read all lines of a file into vector vc +void ReadContentFromFile(std::vector &vc, std::string log_file); + +} // namespace ray diff --git a/src/ray/core_worker/task_execution/tests/BUILD.bazel b/src/ray/core_worker/task_execution/tests/BUILD.bazel index 48666ca86ce0..b60f96696bd1 100644 --- a/src/ray/core_worker/task_execution/tests/BUILD.bazel +++ b/src/ray/core_worker/task_execution/tests/BUILD.bazel @@ -29,7 +29,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/ray/common:asio", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker/task_execution:concurrency_group_manager", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", @@ -42,7 +42,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/ray/common:asio", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker/task_execution:actor_scheduling_queue", "//src/ray/core_worker/task_execution:normal_scheduling_queue", "//src/ray/core_worker/task_execution:out_of_order_actor_scheduling_queue", @@ -58,7 +58,7 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:asio", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker/task_execution:task_receiver", "//src/ray/rpc:core_worker_client", "@com_google_googletest//:gtest", diff --git a/src/ray/core_worker/task_execution/tests/concurrency_group_manager_test.cc b/src/ray/core_worker/task_execution/tests/concurrency_group_manager_test.cc index b12bb8876535..6d3f95030484 100644 --- a/src/ray/core_worker/task_execution/tests/concurrency_group_manager_test.cc +++ b/src/ray/core_worker/task_execution/tests/concurrency_group_manager_test.cc @@ -19,7 +19,7 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/core_worker/task_execution/fiber.h" #include "ray/core_worker/task_execution/thread_pool.h" diff --git a/src/ray/core_worker/task_execution/tests/scheduling_queue_test.cc b/src/ray/core_worker/task_execution/tests/scheduling_queue_test.cc index 4eff53179edf..5a779ff091b7 100644 --- a/src/ray/core_worker/task_execution/tests/scheduling_queue_test.cc +++ b/src/ray/core_worker/task_execution/tests/scheduling_queue_test.cc @@ -19,7 +19,7 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/core_worker/task_execution/actor_scheduling_queue.h" #include "ray/core_worker/task_execution/normal_scheduling_queue.h" #include "ray/core_worker/task_execution/out_of_order_actor_scheduling_queue.h" diff --git a/src/ray/core_worker/task_execution/tests/task_receiver_test.cc b/src/ray/core_worker/task_execution/tests/task_receiver_test.cc index 647be3564063..cae330fec4e0 100644 --- a/src/ray/core_worker/task_execution/tests/task_receiver_test.cc +++ b/src/ray/core_worker/task_execution/tests/task_receiver_test.cc @@ -21,7 +21,7 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/task/task_spec.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/rpc/worker/core_worker_client.h" namespace ray { diff --git a/src/ray/core_worker/task_submission/tests/BUILD.bazel b/src/ray/core_worker/task_submission/tests/BUILD.bazel index 517b2fb220f3..0647614c0dd7 100644 --- a/src/ray/core_worker/task_submission/tests/BUILD.bazel +++ b/src/ray/core_worker/task_submission/tests/BUILD.bazel @@ -8,7 +8,7 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:task_common", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker/task_submission:dependency_resolver", "@com_google_googletest//:gtest", ], @@ -47,7 +47,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:asio", "//src/ray/common:task_common", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker:actor_creator", "//src/ray/core_worker:reference_count", "//src/ray/core_worker:task_manager", @@ -67,7 +67,7 @@ ray_cc_test( "//:ray_mock", "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/common:task_common", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker:memory_store", "//src/ray/core_worker/task_submission:normal_task_submitter", "//src/ray/raylet_client:raylet_client_lib", diff --git a/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc index 09874520b038..85d058f9007a 100644 --- a/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc @@ -22,7 +22,7 @@ #include "mock/ray/core_worker/actor_creator.h" #include "mock/ray/core_worker/reference_count.h" #include "mock/ray/core_worker/task_manager_interface.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/rpc/worker/core_worker_client.h" namespace ray::core { diff --git a/src/ray/core_worker/task_submission/tests/dependency_resolver_test.cc b/src/ray/core_worker/task_submission/tests/dependency_resolver_test.cc index 011a72e99859..21aad49fa165 100644 --- a/src/ray/core_worker/task_submission/tests/dependency_resolver_test.cc +++ b/src/ray/core_worker/task_submission/tests/dependency_resolver_test.cc @@ -26,7 +26,7 @@ #include "mock/ray/core_worker/task_manager_interface.h" #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc index a42a54991cf5..c9b583fd3e10 100644 --- a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc @@ -28,7 +28,7 @@ #include "mock/ray/core_worker/task_manager_interface.h" #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/raylet_client/raylet_client.h" #include "ray/rpc/worker/core_worker_client.h" diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index 451bff46d50f..205b16d93ae2 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -32,7 +32,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:status", "//src/ray/common:status_or", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker:memory_store", "@com_google_absl//absl/synchronization", "@com_google_googletest//:gtest", @@ -71,7 +71,7 @@ ray_cc_test( "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/common:task_common", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker:memory_store", "//src/ray/core_worker:object_recovery_manager", "//src/ray/object_manager:object_manager_common", @@ -91,7 +91,7 @@ ray_cc_test( "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/common:task_common", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker:memory_store", "//src/ray/core_worker:reference_count", "//src/ray/core_worker:task_event_buffer", @@ -110,7 +110,7 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:task_common", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker:task_event_buffer", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/util:event", @@ -133,7 +133,7 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:task_common", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker:task_event_buffer", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/util:event", @@ -152,7 +152,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker:actor_creator", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/util:path_utils", @@ -169,7 +169,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker:common", "//src/ray/core_worker:generator_waiter", "//src/ray/gcs/gcs_client:gcs_client_lib", @@ -186,7 +186,7 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:task_common", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker:actor_manager", "//src/ray/gcs/gcs_client:gcs_client_lib", "@com_google_googletest//:gtest", @@ -246,7 +246,7 @@ ray_cc_test( "//:ray_fakes", "//:ray_mock", "//src/fakes/ray/rpc/raylet:fake_raylet_client", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/core_worker:core_worker_lib", "//src/ray/core_worker:memory_store", "//src/ray/core_worker:reference_count", diff --git a/src/ray/core_worker/tests/actor_creator_test.cc b/src/ray/core_worker/tests/actor_creator_test.cc index 4bb50d9a6004..7d749d011c47 100644 --- a/src/ray/core_worker/tests/actor_creator_test.cc +++ b/src/ray/core_worker/tests/actor_creator_test.cc @@ -18,7 +18,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "ray/core_worker/actor_creator.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/util/path_utils.h" #include "ray/util/raii.h" #include "mock/ray/gcs/gcs_client/gcs_client.h" diff --git a/src/ray/core_worker/tests/actor_manager_test.cc b/src/ray/core_worker/tests/actor_manager_test.cc index 4856e3b6c214..cd5d28bb51d4 100644 --- a/src/ray/core_worker/tests/actor_manager_test.cc +++ b/src/ray/core_worker/tests/actor_manager_test.cc @@ -22,7 +22,7 @@ #include "gtest/gtest.h" #include "mock/ray/core_worker/reference_count.h" #include "ray/common/task/task_spec.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_client/accessor.h" #include "ray/gcs/gcs_client/gcs_client.h" diff --git a/src/ray/core_worker/tests/memory_store_test.cc b/src/ray/core_worker/tests/memory_store_test.cc index 65da14ceec22..fcee0b090698 100644 --- a/src/ray/core_worker/tests/memory_store_test.cc +++ b/src/ray/core_worker/tests/memory_store_test.cc @@ -26,7 +26,7 @@ #include "mock/ray/core_worker/memory_store.h" #include "ray/common/status.h" #include "ray/common/status_or.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/tests/object_recovery_manager_test.cc b/src/ray/core_worker/tests/object_recovery_manager_test.cc index 99024c599bde..db5ad2b37048 100644 --- a/src/ray/core_worker/tests/object_recovery_manager_test.cc +++ b/src/ray/core_worker/tests/object_recovery_manager_test.cc @@ -28,7 +28,7 @@ #include "mock/ray/pubsub/publisher.h" #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/raylet_client/raylet_client.h" diff --git a/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc b/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc index 9e6345bd9582..066ad5e4c965 100644 --- a/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc +++ b/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc @@ -28,7 +28,7 @@ #include "gtest/gtest.h" #include "mock/ray/gcs/gcs_client/gcs_client.h" #include "ray/common/task/task_spec.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/core_worker/task_event_buffer.h" #include "ray/util/event.h" diff --git a/src/ray/core_worker/tests/task_event_buffer_test.cc b/src/ray/core_worker/tests/task_event_buffer_test.cc index d117bed1560a..061ccdae97bf 100644 --- a/src/ray/core_worker/tests/task_event_buffer_test.cc +++ b/src/ray/core_worker/tests/task_event_buffer_test.cc @@ -34,7 +34,7 @@ #include "mock/ray/gcs/gcs_client/gcs_client.h" #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/util/event.h" using ::testing::_; diff --git a/src/ray/core_worker/tests/task_manager_test.cc b/src/ray/core_worker/tests/task_manager_test.cc index e2704d4a3cb7..519a6020e9ef 100644 --- a/src/ray/core_worker/tests/task_manager_test.cc +++ b/src/ray/core_worker/tests/task_manager_test.cc @@ -27,7 +27,7 @@ #include "mock/ray/pubsub/publisher.h" #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/core_worker/reference_count.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/core_worker/task_event_buffer.h" diff --git a/src/ray/gcs/gcs_client/tests/BUILD.bazel b/src/ray/gcs/gcs_client/tests/BUILD.bazel index 57054315512a..a7b7511a7544 100644 --- a/src/ray/gcs/gcs_client/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_client/tests/BUILD.bazel @@ -8,8 +8,8 @@ ray_cc_test( ], tags = ["team:core"], deps = [ + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_client:gcs_client_lib", - "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -30,10 +30,10 @@ ray_cc_test( ], tags = ["team:core"], deps = [ + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_client:global_state_accessor_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:path_utils", "//src/ray/util:raii", "@com_google_googletest//:gtest_main", @@ -60,9 +60,9 @@ ray_cc_test( "team:core", ], deps = [ + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:network_util", "//src/ray/util:raii", "@com_google_googletest//:gtest_main", @@ -87,9 +87,9 @@ ray_cc_test( "team:core", ], deps = [ + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:network_util", "//src/ray/util:path_utils", "//src/ray/util:raii", diff --git a/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc b/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc index 1669031fab25..1167df2f0c53 100644 --- a/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc +++ b/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc @@ -22,10 +22,10 @@ #include "absl/strings/substitute.h" #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_client/accessor.h" #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/rpc/gcs/gcs_rpc_client.h" #include "ray/util/network_util.h" #include "ray/util/path_utils.h" diff --git a/src/ray/gcs/gcs_client/tests/gcs_client_test.cc b/src/ray/gcs/gcs_client/tests/gcs_client_test.cc index ff78ae00d4ab..bc775916d28d 100644 --- a/src/ray/gcs/gcs_client/tests/gcs_client_test.cc +++ b/src/ray/gcs/gcs_client/tests/gcs_client_test.cc @@ -22,9 +22,9 @@ #include "absl/strings/substitute.h" #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_client/accessor.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/rpc/gcs/gcs_rpc_client.h" #include "ray/util/network_util.h" #include "ray/util/path_utils.h" @@ -421,11 +421,11 @@ class GcsClientTest : public ::testing::TestWithParam { INSTANTIATE_TEST_SUITE_P(RedisMigration, GcsClientTest, testing::Bool()); TEST_P(GcsClientTest, TestCheckAlive) { - auto node_info1 = Mocker::GenNodeInfo(); + auto node_info1 = GenNodeInfo(); node_info1->set_node_manager_address("172.1.2.3"); node_info1->set_node_manager_port(31292); - auto node_info2 = Mocker::GenNodeInfo(); + auto node_info2 = GenNodeInfo(); node_info2->set_node_manager_address("172.1.2.4"); node_info2->set_node_manager_port(31293); @@ -458,11 +458,11 @@ TEST_P(GcsClientTest, TestCheckAlive) { } TEST_P(GcsClientTest, TestGcsClientCheckAlive) { - auto node_info1 = Mocker::GenNodeInfo(); + auto node_info1 = GenNodeInfo(); node_info1->set_node_manager_address("172.1.2.3"); node_info1->set_node_manager_port(31292); - auto node_info2 = Mocker::GenNodeInfo(); + auto node_info2 = GenNodeInfo(); node_info2->set_node_manager_address("172.1.2.4"); node_info2->set_node_manager_port(31293); @@ -491,7 +491,7 @@ TEST_P(GcsClientTest, TestGcsClientCheckAlive) { TEST_P(GcsClientTest, TestJobInfo) { // Create job table data. JobID add_job_id = JobID::FromInt(1); - auto job_table_data = Mocker::GenJobTableData(add_job_id); + auto job_table_data = GenJobTableData(add_job_id); // Subscribe to all jobs. std::atomic job_updates(0); @@ -515,7 +515,7 @@ TEST_P(GcsClientTest, TestActorInfo) { // Create actor table data. JobID job_id = JobID::FromInt(1); AddJob(job_id); - auto actor_table_data = Mocker::GenActorTableData(job_id); + auto actor_table_data = GenActorTableData(job_id); ActorID actor_id = ActorID::FromBinary(actor_table_data->actor_id()); // Subscribe to any update operations of an actor. @@ -533,7 +533,7 @@ TEST_P(GcsClientTest, TestActorInfo) { TEST_P(GcsClientTest, TestNodeInfo) { // Create gcs node info. - auto gcs_node1_info = Mocker::GenNodeInfo(); + auto gcs_node1_info = GenNodeInfo(); NodeID node1_id = NodeID::FromBinary(gcs_node1_info->node_id()); // Subscribe to node addition and removal events from GCS. @@ -557,7 +557,7 @@ TEST_P(GcsClientTest, TestNodeInfo) { EXPECT_EQ(gcs_client_->Nodes().GetSelfInfo().state(), gcs_node1_info->state()); // Register a node to GCS. - auto gcs_node2_info = Mocker::GenNodeInfo(); + auto gcs_node2_info = GenNodeInfo(); NodeID node2_id = NodeID::FromBinary(gcs_node2_info->node_id()); ASSERT_TRUE(RegisterNode(*gcs_node2_info)); WaitForExpectedCount(register_count, 2); @@ -572,7 +572,7 @@ TEST_P(GcsClientTest, TestNodeInfo) { TEST_P(GcsClientTest, TestUnregisterNode) { // Create gcs node info. - auto gcs_node_info = Mocker::GenNodeInfo(); + auto gcs_node_info = GenNodeInfo(); NodeID node_id = NodeID::FromBinary(gcs_node_info->node_id()); // Register local node to GCS. @@ -601,7 +601,7 @@ TEST_P(GcsClientTest, TestUnregisterNode) { TEST_P(GcsClientTest, TestGetAllAvailableResources) { // Register node. - auto node_info = Mocker::GenNodeInfo(); + auto node_info = GenNodeInfo(); node_info->mutable_resources_total()->insert({"CPU", 1.0}); node_info->mutable_resources_total()->insert({"GPU", 10.0}); @@ -634,7 +634,7 @@ TEST_P(GcsClientTest, TestWorkerInfo) { ASSERT_TRUE(SubscribeToWorkerFailures(on_subscribe)); // Report a worker failure to GCS when this worker doesn't exist. - auto worker_data = Mocker::GenWorkerTableData(); + auto worker_data = GenWorkerTableData(); worker_data->mutable_worker_address()->set_worker_id(WorkerID::FromRandom().Binary()); ASSERT_TRUE(ReportWorkerFailure(worker_data)); WaitForExpectedCount(worker_failure_count, 1); @@ -653,7 +653,7 @@ TEST_P(GcsClientTest, TestJobTableResubscribe) { // Test that subscription of the job table can still work when GCS server restarts. JobID job_id = JobID::FromInt(1); - auto job_table_data = Mocker::GenJobTableData(job_id); + auto job_table_data = GenJobTableData(job_id); // Subscribe to all jobs. std::atomic job_update_count(0); @@ -681,7 +681,7 @@ TEST_P(GcsClientTest, TestActorTableResubscribe) { // Test that subscription of the actor table can still work when GCS server restarts. JobID job_id = JobID::FromInt(1); AddJob(job_id); - auto actor_table_data = Mocker::GenActorTableData(job_id); + auto actor_table_data = GenActorTableData(job_id); auto actor_id = ActorID::FromBinary(actor_table_data->actor_id()); // Number of notifications for the following `SubscribeActor` operation. @@ -744,7 +744,7 @@ TEST_P(GcsClientTest, TestNodeTableResubscribe) { }; ASSERT_TRUE(SubscribeToNodeChange(node_subscribe)); - auto node_info = Mocker::GenNodeInfo(1); + auto node_info = GenNodeInfo(1); ASSERT_TRUE(RegisterNode(*node_info)); NodeID node_id = NodeID::FromBinary(node_info->node_id()); std::string key = "CPU"; @@ -753,7 +753,7 @@ TEST_P(GcsClientTest, TestNodeTableResubscribe) { RestartGcsServer(); - node_info = Mocker::GenNodeInfo(1); + node_info = GenNodeInfo(1); ASSERT_TRUE(RegisterNode(*node_info)); node_id = NodeID::FromBinary(node_info->node_id()); gcs_server_->UpdateGcsResourceManagerInTest(node_id, resources); @@ -776,7 +776,7 @@ TEST_P(GcsClientTest, TestWorkerTableResubscribe) { RestartGcsServer(); // Add a worker before report worker failure to GCS. - auto worker_data = Mocker::GenWorkerTableData(); + auto worker_data = GenWorkerTableData(); worker_data->mutable_worker_address()->set_worker_id(WorkerID::FromRandom().Binary()); ASSERT_TRUE(AddWorker(worker_data)); @@ -791,7 +791,7 @@ TEST_P(GcsClientTest, TestGcsTableReload) { return; } // Register node to GCS. - auto node_info = Mocker::GenNodeInfo(); + auto node_info = GenNodeInfo(); ASSERT_TRUE(RegisterNode(*node_info)); // Restart GCS. @@ -858,7 +858,7 @@ TEST_P(GcsClientTest, DISABLED_TestGetActorPerf) { task_spec.add_args()->CopyFrom(task_arg); } for (int index = 0; index < actor_count; ++index) { - auto actor_table_data = Mocker::GenActorTableData(job_id); + auto actor_table_data = GenActorTableData(job_id); RegisterActor(actor_table_data, false, true); } @@ -885,7 +885,7 @@ TEST_P(GcsClientTest, TestEvictExpiredDestroyedActors) { absl::flat_hash_set actor_ids; int actor_count = RayConfig::instance().maximum_gcs_destroyed_actor_cached_count(); for (int index = 0; index < actor_count; ++index) { - auto actor_table_data = Mocker::GenActorTableData(job_id); + auto actor_table_data = GenActorTableData(job_id); RegisterActor(actor_table_data, false); actor_ids.insert(ActorID::FromBinary(actor_table_data->actor_id())); } @@ -895,7 +895,7 @@ TEST_P(GcsClientTest, TestEvictExpiredDestroyedActors) { ReconnectClient(); for (int index = 0; index < actor_count; ++index) { - auto actor_table_data = Mocker::GenActorTableData(job_id); + auto actor_table_data = GenActorTableData(job_id); RegisterActor(actor_table_data, false); actor_ids.insert(ActorID::FromBinary(actor_table_data->actor_id())); } @@ -937,7 +937,7 @@ TEST_P(GcsClientTest, TestGcsAuth) { RayConfig::instance().initialize(R"({"enable_cluster_auth": true})"); // Restart GCS. RestartGcsServer(); - auto node_info = Mocker::GenNodeInfo(); + auto node_info = GenNodeInfo(); if (!no_redis_) { // If we are backed by Redis, we can reuse cluster ID, so the RPC passes. EXPECT_TRUE(RegisterNode(*node_info)); @@ -953,14 +953,14 @@ TEST_P(GcsClientTest, TestGcsAuth) { TEST_P(GcsClientTest, TestRegisterHeadNode) { // Test at most only one head node is alive in GCS server - auto head_node_info = Mocker::GenNodeInfo(1); + auto head_node_info = GenNodeInfo(1); head_node_info->set_is_head_node(true); ASSERT_TRUE(RegisterNode(*head_node_info)); - auto worker_node_info = Mocker::GenNodeInfo(1); + auto worker_node_info = GenNodeInfo(1); ASSERT_TRUE(RegisterNode(*worker_node_info)); - auto head_node_info_2 = Mocker::GenNodeInfo(1); + auto head_node_info_2 = GenNodeInfo(1); head_node_info_2->set_is_head_node(true); ASSERT_TRUE(RegisterNode(*head_node_info_2)); diff --git a/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc b/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc index 3613d177f4dd..baa880a625a9 100644 --- a/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc +++ b/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc @@ -20,8 +20,8 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/rpc/gcs/gcs_rpc_client.h" #include "ray/util/path_utils.h" #include "ray/util/raii.h" @@ -132,7 +132,7 @@ TEST_P(GlobalStateAccessorTest, TestJobTable) { ASSERT_EQ(global_state_->GetAllJobInfo().size(), 0); for (int index = 0; index < job_count; ++index) { auto job_id = JobID::FromInt(index); - auto job_table_data = Mocker::GenJobTableData(job_id); + auto job_table_data = GenJobTableData(job_id); std::promise promise; gcs_client_->Jobs().AsyncAdd( job_table_data, [&promise](Status status) { promise.set_value(status.ok()); }); @@ -148,7 +148,7 @@ TEST_P(GlobalStateAccessorTest, TestJobTableWithSubmissionId) { ASSERT_EQ(global_state_->GetAllJobInfo().size(), 0); for (int index = 0; index < job_count; ++index) { auto job_id = JobID::FromInt(index); - auto job_table_data = Mocker::GenJobTableData(job_id); + auto job_table_data = GenJobTableData(job_id); if (index % 2 == 0) { (*job_table_data->mutable_config()->mutable_metadata())["job_submission_id"] = std::to_string(index); @@ -166,10 +166,9 @@ TEST_P(GlobalStateAccessorTest, TestNodeTable) { ASSERT_EQ(global_state_->GetAllNodeInfo().size(), 0); // It's useful to check if index value will be marked as address suffix. for (int index = 0; index < node_count; ++index) { - auto node_table_data = - Mocker::GenNodeInfo(index, - std::string("127.0.0.") + std::to_string(index), - "Mocker_node_" + std::to_string(index * 10)); + auto node_table_data = GenNodeInfo(index, + std::string("127.0.0.") + std::to_string(index), + "Mocker_node_" + std::to_string(index * 10)); std::promise promise; gcs_client_->Nodes().AsyncRegister( *node_table_data, [&promise](Status status) { promise.set_value(status.ok()); }); @@ -192,7 +191,7 @@ TEST_P(GlobalStateAccessorTest, TestGetAllTotalResources) { ASSERT_EQ(global_state_->GetAllTotalResources().size(), 0); // Register node - auto node_table_data = Mocker::GenNodeInfo(); + auto node_table_data = GenNodeInfo(); node_table_data->mutable_resources_total()->insert({"CPU", 1}); node_table_data->mutable_resources_total()->insert({"GPU", 10}); @@ -222,7 +221,7 @@ TEST_P(GlobalStateAccessorTest, TestGetAllResourceUsage) { resource_usage_batch_data.ParseFromString(*resources.get()); ASSERT_EQ(resource_usage_batch_data.batch_size(), 0); - auto node_table_data = Mocker::GenNodeInfo(); + auto node_table_data = GenNodeInfo(); node_table_data->mutable_resources_total()->insert({"CPU", 1}); std::promise promise; @@ -267,7 +266,7 @@ TEST_P(GlobalStateAccessorTest, TestGetAllResourceUsage) { TEST_P(GlobalStateAccessorTest, TestWorkerTable) { ASSERT_EQ(global_state_->GetAllWorkerInfo().size(), 0); // Add worker info - auto worker_table_data = Mocker::GenWorkerTableData(); + auto worker_table_data = GenWorkerTableData(); worker_table_data->mutable_worker_address()->set_worker_id( WorkerID::FromRandom().Binary()); ASSERT_TRUE(global_state_->AddWorkerInfo(worker_table_data->SerializeAsString())); @@ -277,7 +276,7 @@ TEST_P(GlobalStateAccessorTest, TestWorkerTable) { ASSERT_TRUE(global_state_->GetWorkerInfo(worker_id)); // Add another worker info - auto another_worker_data = Mocker::GenWorkerTableData(); + auto another_worker_data = GenWorkerTableData(); another_worker_data->mutable_worker_address()->set_worker_id( WorkerID::FromRandom().Binary()); ASSERT_TRUE(global_state_->AddWorkerInfo(another_worker_data->SerializeAsString())); @@ -287,7 +286,7 @@ TEST_P(GlobalStateAccessorTest, TestWorkerTable) { TEST_P(GlobalStateAccessorTest, TestUpdateWorkerDebuggerPort) { ASSERT_EQ(global_state_->GetAllWorkerInfo().size(), 0); // Add worker info - auto worker_table_data = Mocker::GenWorkerTableData(); + auto worker_table_data = GenWorkerTableData(); worker_table_data->mutable_worker_address()->set_worker_id( WorkerID::FromRandom().Binary()); ASSERT_TRUE(global_state_->AddWorkerInfo(worker_table_data->SerializeAsString())); @@ -301,7 +300,7 @@ TEST_P(GlobalStateAccessorTest, TestUpdateWorkerDebuggerPort) { ASSERT_TRUE(global_state_->UpdateWorkerDebuggerPort(worker_id, debugger_port)); // Verify the debugger port - auto another_worker_table_data = Mocker::GenWorkerTableData(); + auto another_worker_table_data = GenWorkerTableData(); auto worker_info = global_state_->GetWorkerInfo(worker_id); ASSERT_TRUE(another_worker_table_data->ParseFromString(*worker_info)); ASSERT_EQ(another_worker_table_data->debugger_port(), debugger_port); @@ -310,7 +309,7 @@ TEST_P(GlobalStateAccessorTest, TestUpdateWorkerDebuggerPort) { TEST_P(GlobalStateAccessorTest, TestUpdateWorkerNumPausedThreads) { ASSERT_EQ(global_state_->GetAllWorkerInfo().size(), 0); // Add worker info - auto worker_table_data = Mocker::GenWorkerTableData(); + auto worker_table_data = GenWorkerTableData(); worker_table_data->mutable_worker_address()->set_worker_id( WorkerID::FromRandom().Binary()); ASSERT_TRUE(global_state_->AddWorkerInfo(worker_table_data->SerializeAsString())); @@ -325,7 +324,7 @@ TEST_P(GlobalStateAccessorTest, TestUpdateWorkerNumPausedThreads) { global_state_->UpdateWorkerNumPausedThreads(worker_id, num_paused_threads_delta)); // Verify the num paused threads is equal to num_paused_threads_delta - auto another_worker_table_data = Mocker::GenWorkerTableData(); + auto another_worker_table_data = GenWorkerTableData(); auto worker_info = global_state_->GetWorkerInfo(worker_id); ASSERT_TRUE(another_worker_table_data->ParseFromString(*worker_info)); ASSERT_EQ(another_worker_table_data->num_paused_threads(), num_paused_threads_delta); diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index 2bf6918e5fa0..05534230c9db 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -20,8 +20,8 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_placement_group_manager", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], @@ -47,8 +47,8 @@ ray_cc_test( "team:core", ], deps = [ + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_server_lib", - "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest", ], ) @@ -69,11 +69,11 @@ ray_cc_test( ], tags = ["team:core"], deps = [ + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_kv_manager", "//src/ray/gcs/gcs_server:gcs_store_client_kv", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/store_client:redis_store_client", - "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest", ], ) @@ -108,8 +108,8 @@ ray_cc_test( "//:ray_fakes", "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/mock/ray/pubsub:mock_publisher", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_node_manager", - "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -124,10 +124,10 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_job_manager", "//src/ray/gcs/gcs_server:gcs_kv_manager", "//src/ray/gcs/store_client:in_memory_store_client", - "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -141,8 +141,8 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_task_manager", - "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -160,9 +160,9 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_placement_group_manager", "//src/ray/gcs/store_client:in_memory_store_client", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], @@ -182,13 +182,13 @@ ray_cc_test( "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/fakes/ray/rpc/worker:fake_core_worker_client", "//src/mock/ray/pubsub:mock_publisher", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_node_manager", "//src/ray/gcs/gcs_server:gcs_placement_group", "//src/ray/gcs/gcs_server:gcs_placement_group_scheduler", "//src/ray/gcs/gcs_server:gcs_resource_manager", "//src/ray/gcs/gcs_server:gcs_table_storage", "//src/ray/gcs/store_client:in_memory_store_client", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], @@ -205,12 +205,11 @@ ray_cc_test( "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/fakes/ray/rpc/worker:fake_core_worker_client", "//src/mock/ray/pubsub:mock_publisher", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_actor", "//src/ray/gcs/gcs_server:gcs_actor_scheduler", "//src/ray/gcs/gcs_server:gcs_resource_manager", "//src/ray/gcs/store_client:in_memory_store_client", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], @@ -225,7 +224,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_actor", "//src/ray/gcs/gcs_server:gcs_actor_scheduler", "//src/ray/util:counter_map", @@ -246,12 +245,12 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:asio", "//src/ray/common:runtime_env", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_actor", "//src/ray/gcs/gcs_server:gcs_actor_manager", "//src/ray/gcs/gcs_server:gcs_actor_scheduler", "//src/ray/gcs/gcs_server:gcs_function_manager", "//src/ray/gcs/store_client:in_memory_store_client", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/pubsub:publisher", "@com_google_googletest//:gtest_main", ], @@ -266,10 +265,10 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/mock/ray/pubsub:mock_publisher", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_store_client_kv", "//src/ray/gcs/gcs_server:gcs_worker_manager", "//src/ray/gcs/store_client:in_memory_store_client", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:process", "@com_google_googletest//:gtest_main", ], @@ -302,9 +301,9 @@ ray_cc_test( tags = ["team:core"], deps = [ ":gcs_table_storage_test_lib", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_table_storage", "//src/ray/gcs/store_client/tests:store_client_test_lib", - "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest", ], ) @@ -316,11 +315,10 @@ ray_cc_test( tags = ["team:core"], deps = [ ":gcs_table_storage_test_lib", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_table_storage", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/store_client/tests:store_client_test_lib", - "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -337,11 +335,12 @@ ray_cc_test( "//:ray_mock", "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/common:asio", + "//src/ray/common:test_utils", + "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/gcs_server:gcs_autoscaler_state_manager", "//src/ray/gcs/gcs_server:gcs_init_data", "//src/ray/gcs/gcs_server:gcs_resource_manager", "//src/ray/gcs/gcs_server:gcs_store_client_kv", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/raylet/scheduling:cluster_resource_manager", "@com_google_googletest//:gtest_main", ], @@ -356,9 +355,9 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_node_manager", "//src/ray/gcs/gcs_server:gcs_resource_manager", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/raylet/scheduling:cluster_resource_manager", "@com_google_googletest//:gtest_main", ], @@ -374,8 +373,8 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:asio", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_usage_stats_client", - "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -391,10 +390,10 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_job_manager", "//src/ray/gcs/gcs_server:gcs_kv_manager", "//src/ray/gcs/store_client:in_memory_store_client", - "//src/ray/gcs/tests:gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) @@ -411,12 +410,12 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:asio", "//src/ray/common:runtime_env", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_actor", "//src/ray/gcs/gcs_server:gcs_actor_manager", "//src/ray/gcs/gcs_server:gcs_actor_scheduler", "//src/ray/gcs/gcs_server:gcs_function_manager", "//src/ray/gcs/store_client:in_memory_store_client", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/pubsub:publisher", "//src/ray/rpc:core_worker_client", "//src/ray/util:event", @@ -435,9 +434,9 @@ ray_cc_test( deps = [ "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/mock/ray/pubsub:mock_publisher", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_node_manager", "//src/ray/gcs/store_client:in_memory_store_client", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/util:string_utils", "@com_google_googletest//:gtest", ], diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc index 1dc0fff9e892..104715028a9f 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -25,11 +26,11 @@ #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/runtime_env_manager.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_actor_manager.h" #include "ray/gcs/gcs_server/gcs_function_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/pubsub/publisher.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" @@ -227,8 +228,8 @@ class GcsActorManagerTest : public ::testing::Test { const std::string &name = "", const std::string &ray_namespace = "test") { std::promise> promise; - auto request = Mocker::GenRegisterActorRequest( - job_id, max_restarts, detached, name, ray_namespace); + auto request = + GenRegisterActorRequest(job_id, max_restarts, detached, name, ray_namespace); // `DestroyActor` triggers some asynchronous operations. // If we register an actor after destroying an actor, it may result in multithreading // reading and writing the same variable. In order to avoid the problem of @@ -318,7 +319,7 @@ TEST_F(GcsActorManagerTest, TestBasic) { "DEPENDENCIES_UNREADY", "PENDING_CREATION", "ALIVE", "DEAD"}; std::vector vc; for (int i = 0; i < num_retry; i++) { - Mocker::ReadContentFromFile(vc, log_dir_ + "/export_events/event_EXPORT_ACTOR.log"); + ReadContentFromFile(vc, log_dir_ + "/export_events/event_EXPORT_ACTOR.log"); if (static_cast(vc.size()) == num_export_events) { for (int event_idx = 0; event_idx < num_export_events; event_idx++) { json export_event_as_json = json::parse(vc[event_idx]); @@ -342,7 +343,7 @@ TEST_F(GcsActorManagerTest, TestBasic) { vc.clear(); } } - Mocker::ReadContentFromFile(vc, log_dir_ + "/export_events/event_EXPORT_ACTOR.log"); + ReadContentFromFile(vc, log_dir_ + "/export_events/event_EXPORT_ACTOR.log"); std::ostringstream lines; for (auto line : vc) { lines << line << "\n"; diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc index 7d2581b5cf44..7ce39262c691 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc @@ -14,6 +14,7 @@ #include +#include #include #include #include @@ -21,10 +22,10 @@ #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/pubsub/publisher.h" #include "mock/ray/rpc/worker/core_worker_client.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_job_manager.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/tests/gcs_test_util.h" using json = nlohmann::json; @@ -111,8 +112,7 @@ TEST_F(GcsJobManagerTest, TestExportDriverJobEvents) { auto job_api_job_id = JobID::FromInt(100); std::string submission_id = "submission_id_100"; - auto add_job_request = - Mocker::GenAddJobRequest(job_api_job_id, "namespace_100", submission_id); + auto add_job_request = GenAddJobRequest(job_api_job_id, "namespace_100", submission_id); rpc::AddJobReply empty_reply; std::promise promise; gcs_job_manager.HandleAddJob( @@ -124,8 +124,7 @@ TEST_F(GcsJobManagerTest, TestExportDriverJobEvents) { promise.get_future().get(); std::vector vc; - Mocker::ReadContentFromFile(vc, - log_dir_ + "/export_events/event_EXPORT_DRIVER_JOB.log"); + ReadContentFromFile(vc, log_dir_ + "/export_events/event_EXPORT_DRIVER_JOB.log"); ASSERT_EQ((int)vc.size(), 1); json event_data = json::parse(vc[0])["event_data"].get(); ASSERT_EQ(event_data["is_dead"], false); @@ -144,8 +143,7 @@ TEST_F(GcsJobManagerTest, TestExportDriverJobEvents) { job_finished_promise.get_future().get(); vc.clear(); - Mocker::ReadContentFromFile(vc, - log_dir_ + "/export_events/event_EXPORT_DRIVER_JOB.log"); + ReadContentFromFile(vc, log_dir_ + "/export_events/event_EXPORT_DRIVER_JOB.log"); ASSERT_EQ((int)vc.size(), 2); event_data = json::parse(vc[1])["event_data"].get(); ASSERT_EQ(event_data["is_dead"], true); diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc index 03237fe82167..209411686a7a 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -22,9 +23,9 @@ #include "fakes/ray/rpc/raylet/raylet_client.h" #include "mock/ray/pubsub/publisher.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/util/event.h" #include "ray/util/string_utils.h" @@ -89,7 +90,7 @@ TEST_F(GcsNodeManagerExportAPITest, TestExportEventRegisterNode) { io_service_, client_pool_.get(), ClusterID::Nil()); - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); rpc::RegisterNodeRequest register_request; register_request.mutable_node_info()->CopyFrom(*node); @@ -101,7 +102,7 @@ TEST_F(GcsNodeManagerExportAPITest, TestExportEventRegisterNode) { io_service_.poll(); std::vector vc; - Mocker::ReadContentFromFile(vc, log_dir_ + "/export_events/event_EXPORT_NODE.log"); + ReadContentFromFile(vc, log_dir_ + "/export_events/event_EXPORT_NODE.log"); ASSERT_EQ((int)vc.size(), 1); json event_data = json::parse(vc[0])["event_data"].get(); ASSERT_EQ(event_data["state"], "ALIVE"); @@ -114,7 +115,7 @@ TEST_F(GcsNodeManagerExportAPITest, TestExportEventUnregisterNode) { io_service_, client_pool_.get(), ClusterID::Nil()); - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); node_manager.AddNode(node); @@ -132,7 +133,7 @@ TEST_F(GcsNodeManagerExportAPITest, TestExportEventUnregisterNode) { io_service_.poll(); std::vector vc; - Mocker::ReadContentFromFile(vc, log_dir_ + "/export_events/event_EXPORT_NODE.log"); + ReadContentFromFile(vc, log_dir_ + "/export_events/event_EXPORT_NODE.log"); ASSERT_EQ((int)vc.size(), 1); json event_data = json::parse(vc[0])["event_data"].get(); ASSERT_EQ(event_data["state"], "DEAD"); diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc index 8dca97d5ad0c..836687f36fcc 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc @@ -26,11 +26,11 @@ #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/runtime_env_manager.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_actor_scheduler.h" #include "ray/gcs/gcs_server/gcs_function_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/pubsub/publisher.h" namespace ray { @@ -179,8 +179,8 @@ class GcsActorManagerTest : public ::testing::Test { while (io_service_.poll_one()) { continue; } - auto request = Mocker::GenRegisterActorRequest( - job_id, max_restarts, detached, name, ray_namespace); + auto request = + GenRegisterActorRequest(job_id, max_restarts, detached, name, ray_namespace); auto status = gcs_actor_manager_->RegisterActor(request, [](const Status &) {}); io_service_.run_one(); io_service_.run_one(); @@ -595,10 +595,10 @@ TEST_F(GcsActorManagerTest, TestActorWithEmptyName) { // Gen `CreateActorRequest` with an empty name. // (name,actor_id) => ("", actor_id_1) - auto request1 = Mocker::GenRegisterActorRequest(job_id, - /*max_restarts=*/0, - /*detached=*/true, - /*name=*/""); + auto request1 = GenRegisterActorRequest(job_id, + /*max_restarts=*/0, + /*detached=*/true, + /*name=*/""); Status status = gcs_actor_manager_->RegisterActor(request1, [](const Status &) {}); io_service_.run_one(); @@ -610,10 +610,10 @@ TEST_F(GcsActorManagerTest, TestActorWithEmptyName) { // Gen another `CreateActorRequest` with an empty name. // (name,actor_id) => ("", actor_id_2) - auto request2 = Mocker::GenRegisterActorRequest(job_id, - /*max_restarts=*/0, - /*detached=*/true, - /*name=*/""); + auto request2 = GenRegisterActorRequest(job_id, + /*max_restarts=*/0, + /*detached=*/true, + /*name=*/""); status = gcs_actor_manager_->RegisterActor(request2, [](const Status &) {}); io_service_.run_one(); // Ensure successful registration. @@ -624,22 +624,22 @@ TEST_F(GcsActorManagerTest, TestNamedActors) { auto job_id_1 = JobID::FromInt(1); auto job_id_2 = JobID::FromInt(2); - auto request1 = Mocker::GenRegisterActorRequest(job_id_1, - /*max_restarts=*/0, - /*detached=*/true, - /*name=*/"actor1", - /*ray_namespace=*/"test_named_actor"); + auto request1 = GenRegisterActorRequest(job_id_1, + /*max_restarts=*/0, + /*detached=*/true, + /*name=*/"actor1", + /*ray_namespace=*/"test_named_actor"); Status status = gcs_actor_manager_->RegisterActor(request1, [](const Status &) {}); io_service_.run_one(); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor1", "test_named_actor").Binary(), request1.task_spec().actor_creation_task_spec().actor_id()); - auto request2 = Mocker::GenRegisterActorRequest(job_id_1, - /*max_restarts=*/0, - /*detached=*/true, - /*name=*/"actor2", - /*ray_namesapce=*/"test_named_actor"); + auto request2 = GenRegisterActorRequest(job_id_1, + /*max_restarts=*/0, + /*detached=*/true, + /*name=*/"actor2", + /*ray_namesapce=*/"test_named_actor"); status = gcs_actor_manager_->RegisterActor(request2, [](const Status &) {}); io_service_.run_one(); ASSERT_TRUE(status.ok()); @@ -651,11 +651,11 @@ TEST_F(GcsActorManagerTest, TestNamedActors) { ActorID::Nil()); // Check that naming collisions return Status::AlreadyExists. - auto request3 = Mocker::GenRegisterActorRequest(job_id_1, - /*max_restarts=*/0, - /*detached=*/true, - /*name=*/"actor2", - /*ray_namesapce=*/"test_named_actor"); + auto request3 = GenRegisterActorRequest(job_id_1, + /*max_restarts=*/0, + /*detached=*/true, + /*name=*/"actor2", + /*ray_namesapce=*/"test_named_actor"); status = gcs_actor_manager_->RegisterActor(request3, [](const Status &) {}); io_service_.run_one(); ASSERT_TRUE(status.IsAlreadyExists()); @@ -663,11 +663,11 @@ TEST_F(GcsActorManagerTest, TestNamedActors) { request2.task_spec().actor_creation_task_spec().actor_id()); // Check that naming collisions are enforced across JobIDs. - auto request4 = Mocker::GenRegisterActorRequest(job_id_2, - /*max_restarts=*/0, - /*detached=*/true, - /*name=*/"actor2", - /*ray_namesapce=*/"test_named_actor"); + auto request4 = GenRegisterActorRequest(job_id_2, + /*max_restarts=*/0, + /*detached=*/true, + /*name=*/"actor2", + /*ray_namesapce=*/"test_named_actor"); status = gcs_actor_manager_->RegisterActor(request4, [](const Status &) {}); io_service_.run_one(); ASSERT_TRUE(status.IsAlreadyExists()); @@ -845,10 +845,10 @@ TEST_F(GcsActorManagerTest, TestNamedActorDeletionNotHappendWhenReconstructed) { // It should fail because actor has been reconstructed, and names shouldn't have been // cleaned. const auto job_id_2 = JobID::FromInt(2); - auto request2 = Mocker::GenRegisterActorRequest(job_id_2, - /*max_restarts=*/0, - /*detached=*/true, - /*name=*/"actor"); + auto request2 = GenRegisterActorRequest(job_id_2, + /*max_restarts=*/0, + /*detached=*/true, + /*name=*/"actor"); status = gcs_actor_manager_->RegisterActor(request2, [](const Status &) {}); io_service_.run_one(); ASSERT_TRUE(status.IsAlreadyExists()); @@ -1108,21 +1108,21 @@ TEST_F(GcsActorManagerTest, TestRayNamespace) { std::string second_namespace = "another_namespace"; job_namespace_table_[job_id_2] = second_namespace; - auto request1 = Mocker::GenRegisterActorRequest(job_id_1, - /*max_restarts=*/0, - /*detached=*/true, - /*name=*/"actor"); + auto request1 = GenRegisterActorRequest(job_id_1, + /*max_restarts=*/0, + /*detached=*/true, + /*name=*/"actor"); Status status = gcs_actor_manager_->RegisterActor(request1, [](const Status &) {}); ASSERT_TRUE(status.ok()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor", "test").Binary(), request1.task_spec().actor_creation_task_spec().actor_id()); io_service_.run_one(); - auto request2 = Mocker::GenRegisterActorRequest(job_id_2, - /*max_restarts=*/0, - /*detached=*/true, - /*name=*/"actor", - second_namespace); + auto request2 = GenRegisterActorRequest(job_id_2, + /*max_restarts=*/0, + /*detached=*/true, + /*name=*/"actor", + second_namespace); // Create a second actor of the same name. Its job id belongs to a different // namespace though. status = gcs_actor_manager_->RegisterActor(request2, [](const Status &) {}); @@ -1134,11 +1134,11 @@ TEST_F(GcsActorManagerTest, TestRayNamespace) { request1.task_spec().actor_creation_task_spec().actor_id()); io_service_.run_one(); - auto request3 = Mocker::GenRegisterActorRequest(job_id_3, - /*max_restarts=*/0, - /*detached=*/true, - /*name=*/"actor", - /*ray_namespace=*/"test"); + auto request3 = GenRegisterActorRequest(job_id_3, + /*max_restarts=*/0, + /*detached=*/true, + /*name=*/"actor", + /*ray_namespace=*/"test"); status = gcs_actor_manager_->RegisterActor(request3, [](const Status &) {}); ASSERT_TRUE(status.IsAlreadyExists()); ASSERT_EQ(gcs_actor_manager_->GetActorIDByName("actor", "test").Binary(), @@ -1151,8 +1151,7 @@ TEST_F(GcsActorManagerTest, TestReuseActorNameInNamespace) { std::string ray_namespace = "actor_namespace"; auto job_id_1 = JobID::FromInt(1); - auto request_1 = - Mocker::GenRegisterActorRequest(job_id_1, 0, true, actor_name, ray_namespace); + auto request_1 = GenRegisterActorRequest(job_id_1, 0, true, actor_name, ray_namespace); auto actor_id_1 = ActorID::FromBinary(request_1.task_spec().actor_creation_task_spec().actor_id()); Status status = gcs_actor_manager_->RegisterActor(request_1, [](const Status &) {}); @@ -1170,8 +1169,7 @@ TEST_F(GcsActorManagerTest, TestReuseActorNameInNamespace) { io_service_.run_one(); auto job_id_2 = JobID::FromInt(2); - auto request_2 = - Mocker::GenRegisterActorRequest(job_id_2, 0, true, actor_name, ray_namespace); + auto request_2 = GenRegisterActorRequest(job_id_2, 0, true, actor_name, ray_namespace); auto actor_id_2 = ActorID::FromBinary(request_2.task_spec().actor_creation_task_spec().actor_id()); status = gcs_actor_manager_->RegisterActor(request_2, [](const Status &) {}); @@ -1211,9 +1209,9 @@ TEST_F(GcsActorManagerTest, TestGetAllActorInfoFilters) { auto job_id_other = JobID::FromInt(2); auto num_other_actors = 3; for (int i = 0; i < num_other_actors; i++) { - auto request1 = Mocker::GenRegisterActorRequest(job_id_other, - /*max_restarts=*/0, - /*detached=*/false); + auto request1 = GenRegisterActorRequest(job_id_other, + /*max_restarts=*/0, + /*detached=*/false); Status register_status = gcs_actor_manager_->RegisterActor(request1, [](const Status &) {}); ASSERT_TRUE(register_status.ok()); @@ -1288,9 +1286,9 @@ TEST_F(GcsActorManagerTest, TestGetAllActorInfoLimit) { auto job_id_1 = JobID::FromInt(1); auto num_actors = 3; for (int i = 0; i < num_actors; i++) { - auto request1 = Mocker::GenRegisterActorRequest(job_id_1, - /*max_restarts=*/0, - /*detached=*/false); + auto request1 = GenRegisterActorRequest(job_id_1, + /*max_restarts=*/0, + /*detached=*/false); Status status = gcs_actor_manager_->RegisterActor(request1, [](const Status &) {}); ASSERT_TRUE(status.ok()); io_service_.run_one(); @@ -1638,7 +1636,7 @@ TEST_F(GcsActorManagerTest, TestDestroyActorWhenActorIsCreating) { TEST_F(GcsActorManagerTest, TestDestroyWhileRegistering) { // Register comes in -> Kill comes in -> Run all kv operations and callbacks - auto register_request = Mocker::GenRegisterActorRequest( + auto register_request = GenRegisterActorRequest( JobID::FromInt(1), /*max_restarts=*/0, /*detached=*/false, "", "test"); rpc::RegisterActorReply register_reply; gcs_actor_manager_->HandleRegisterActor( diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc index 50208c9ea62a..4aeb171c36d1 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc @@ -22,7 +22,7 @@ #include "mock/ray/gcs/store_client/store_client.h" #include "mock/ray/raylet_client/raylet_client.h" #include "mock/ray/rpc/worker/core_worker_client.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_actor_scheduler.h" #include "ray/util/counter_map.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc index 89816e4ca1e3..dcbe76cf4955 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc @@ -26,12 +26,11 @@ #include "fakes/ray/rpc/worker/core_worker_client.h" #include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/asio_util.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_actor_scheduler.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/util/counter_map.h" namespace ray { @@ -177,15 +176,14 @@ class GcsActorSchedulerTest : public ::testing::Test { required_resources.insert(required_placement_resources.begin(), required_placement_resources.end()); - auto actor_creating_task_spec = - Mocker::GenActorCreationTask(job_id, - /*max_restarts=*/1, - /*detached=*/true, - /*name=*/"", - "", - owner_address, - required_resources, - required_placement_resources); + auto actor_creating_task_spec = GenActorCreationTask(job_id, + /*max_restarts=*/1, + /*detached=*/true, + /*name=*/"", + "", + owner_address, + required_resources, + required_placement_resources); return std::make_shared(actor_creating_task_spec.GetMessage(), /*ray_namespace=*/"", counter); @@ -193,7 +191,7 @@ class GcsActorSchedulerTest : public ::testing::Test { std::shared_ptr AddNewNode( std::unordered_map node_resources) { - auto node_info = Mocker::GenNodeInfo(); + auto node_info = GenNodeInfo(); node_info->mutable_resources_total()->insert(node_resources.begin(), node_resources.end()); gcs_node_manager_->AddNode(node_info); @@ -230,7 +228,7 @@ TEST_F(GcsActorSchedulerTest, TestScheduleFailedWithZeroNode) { ASSERT_EQ(0, gcs_node_manager_->GetAllAliveNodes().size()); auto job_id = JobID::FromInt(1); - auto create_actor_request = Mocker::GenCreateActorRequest(job_id); + auto create_actor_request = GenCreateActorRequest(job_id); auto actor = std::make_shared(create_actor_request.task_spec(), "", counter); @@ -246,13 +244,13 @@ TEST_F(GcsActorSchedulerTest, TestScheduleFailedWithZeroNode) { } TEST_F(GcsActorSchedulerTest, TestScheduleActorSuccess) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); gcs_node_manager_->AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto job_id = JobID::FromInt(1); - auto create_actor_request = Mocker::GenCreateActorRequest(job_id); + auto create_actor_request = GenCreateActorRequest(job_id); auto actor = std::make_shared(create_actor_request.task_spec(), "", counter); @@ -284,13 +282,13 @@ TEST_F(GcsActorSchedulerTest, TestScheduleActorSuccess) { } TEST_F(GcsActorSchedulerTest, TestScheduleRetryWhenLeasing) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); gcs_node_manager_->AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto job_id = JobID::FromInt(1); - auto create_actor_request = Mocker::GenCreateActorRequest(job_id); + auto create_actor_request = GenCreateActorRequest(job_id); auto actor = std::make_shared(create_actor_request.task_spec(), "", counter); @@ -335,13 +333,13 @@ TEST_F(GcsActorSchedulerTest, TestScheduleRetryWhenLeasing) { } TEST_F(GcsActorSchedulerTest, TestScheduleRetryWhenCreating) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); gcs_node_manager_->AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto job_id = JobID::FromInt(1); - auto create_actor_request = Mocker::GenCreateActorRequest(job_id); + auto create_actor_request = GenCreateActorRequest(job_id); auto actor = std::make_shared(create_actor_request.task_spec(), "", counter); @@ -379,13 +377,13 @@ TEST_F(GcsActorSchedulerTest, TestScheduleRetryWhenCreating) { } TEST_F(GcsActorSchedulerTest, TestNodeFailedWhenLeasing) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); gcs_node_manager_->AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto job_id = JobID::FromInt(1); - auto create_actor_request = Mocker::GenCreateActorRequest(job_id); + auto create_actor_request = GenCreateActorRequest(job_id); auto actor = std::make_shared(create_actor_request.task_spec(), "", counter); @@ -421,13 +419,13 @@ TEST_F(GcsActorSchedulerTest, TestNodeFailedWhenLeasing) { } TEST_F(GcsActorSchedulerTest, TestLeasingCancelledWhenLeasing) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); gcs_node_manager_->AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto job_id = JobID::FromInt(1); - auto create_actor_request = Mocker::GenCreateActorRequest(job_id); + auto create_actor_request = GenCreateActorRequest(job_id); auto actor = std::make_shared(create_actor_request.task_spec(), "", counter); @@ -458,13 +456,13 @@ TEST_F(GcsActorSchedulerTest, TestLeasingCancelledWhenLeasing) { } TEST_F(GcsActorSchedulerTest, TestNodeFailedWhenCreating) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); gcs_node_manager_->AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto job_id = JobID::FromInt(1); - auto create_actor_request = Mocker::GenCreateActorRequest(job_id); + auto create_actor_request = GenCreateActorRequest(job_id); auto actor = std::make_shared(create_actor_request.task_spec(), "", counter); @@ -504,13 +502,13 @@ TEST_F(GcsActorSchedulerTest, TestNodeFailedWhenCreating) { } TEST_F(GcsActorSchedulerTest, TestWorkerFailedWhenCreating) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); gcs_node_manager_->AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto job_id = JobID::FromInt(1); - auto create_actor_request = Mocker::GenCreateActorRequest(job_id); + auto create_actor_request = GenCreateActorRequest(job_id); auto actor = std::make_shared(create_actor_request.task_spec(), "", counter); @@ -546,13 +544,13 @@ TEST_F(GcsActorSchedulerTest, TestWorkerFailedWhenCreating) { } TEST_F(GcsActorSchedulerTest, TestSpillback) { - auto node1 = Mocker::GenNodeInfo(); + auto node1 = GenNodeInfo(); auto node_id_1 = NodeID::FromBinary(node1->node_id()); gcs_node_manager_->AddNode(node1); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); auto job_id = JobID::FromInt(1); - auto create_actor_request = Mocker::GenCreateActorRequest(job_id); + auto create_actor_request = GenCreateActorRequest(job_id); auto actor = std::make_shared(create_actor_request.task_spec(), "", counter); @@ -564,13 +562,13 @@ TEST_F(GcsActorSchedulerTest, TestSpillback) { ASSERT_EQ(0, worker_client_->GetNumCallbacks()); // Add another node. - auto node2 = Mocker::GenNodeInfo(); + auto node2 = GenNodeInfo(); auto node_id_2 = NodeID::FromBinary(node2->node_id()); gcs_node_manager_->AddNode(node2); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); // Grant with an invalid spillback node, and schedule again. - auto invalid_node_id = NodeID::FromBinary(Mocker::GenNodeInfo()->node_id()); + auto invalid_node_id = NodeID::FromBinary(GenNodeInfo()->node_id()); ASSERT_TRUE(raylet_client_->GrantWorkerLease(node2->node_manager_address(), node2->node_manager_port(), WorkerID::Nil(), @@ -613,14 +611,14 @@ TEST_F(GcsActorSchedulerTest, TestSpillback) { } TEST_F(GcsActorSchedulerTest, TestReschedule) { - auto node1 = Mocker::GenNodeInfo(); + auto node1 = GenNodeInfo(); auto node_id_1 = NodeID::FromBinary(node1->node_id()); gcs_node_manager_->AddNode(node1); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); // 1.Actor is already tied to a leased worker. auto job_id = JobID::FromInt(1); - auto create_actor_request = Mocker::GenCreateActorRequest(job_id); + auto create_actor_request = GenCreateActorRequest(job_id); auto actor = std::make_shared(create_actor_request.task_spec(), "", counter); rpc::Address address; @@ -669,7 +667,7 @@ TEST_F(GcsActorSchedulerTest, TestReleaseUnusedActorWorkers) { // if there is still a pending `ReleaseUnusedActorWorkers` request. // Add a node to the cluster. - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); gcs_node_manager_->AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); @@ -687,7 +685,7 @@ TEST_F(GcsActorSchedulerTest, TestReleaseUnusedActorWorkers) { // `GcsActorScheduler` won't send `RequestWorkerLease` request to node immediately. But // instead, it will invoke the `RetryLeasingWorkerFromNode` to retry later. auto job_id = JobID::FromInt(1); - auto request = Mocker::GenCreateActorRequest(job_id); + auto request = GenCreateActorRequest(job_id); auto actor = std::make_shared(request.task_spec(), "", counter); gcs_actor_scheduler_->ScheduleByRaylet(actor); ASSERT_EQ(2, gcs_actor_scheduler_->num_retry_leasing_count_); diff --git a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc index 68ed5f907e8a..d94285270952 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc @@ -32,10 +32,11 @@ #include "mock/ray/gcs/store_client/store_client.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/store_client_kv.h" -#include "ray/gcs/tests/gcs_test_util.h" +#include "ray/gcs/pb_util.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" namespace ray { @@ -201,13 +202,13 @@ class GcsAutoscalerStateManagerTest : public ::testing::Test { bool is_draining = false, int64_t draining_deadline_timestamp_ms = -1) { rpc::ResourcesData resources_data; - Mocker::FillResourcesData(resources_data, - node_id, - available_resources, - total_resources, - idle_ms, - is_draining, - draining_deadline_timestamp_ms); + FillResourcesData(resources_data, + node_id, + available_resources, + total_resources, + idle_ms, + is_draining, + draining_deadline_timestamp_ms); gcs_autoscaler_state_manager_->UpdateResourceLoadAndUsage(resources_data); } @@ -225,7 +226,7 @@ class GcsAutoscalerStateManagerTest : public ::testing::Test { void UpdateResourceLoads(const std::string &node_id, std::vector demands) { rpc::ResourcesData data; - Mocker::FillResourcesData(data, node_id, demands); + FillResourcesData(data, node_id, demands); gcs_autoscaler_state_manager_->UpdateResourceLoadAndUsage(data); } @@ -384,7 +385,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestGenPlacementConstraintForPlacementGrou } TEST_F(GcsAutoscalerStateManagerTest, TestNodeAddUpdateRemove) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); // Adding a node. { @@ -426,7 +427,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestNodeAddUpdateRemove) { } TEST_F(GcsAutoscalerStateManagerTest, TestGetClusterStatusBasic) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); // Test basic cluster resource. { @@ -457,7 +458,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestGetClusterStatusBasic) { TEST_F(GcsAutoscalerStateManagerTest, TestNodeDynamicLabelsWithPG) { /// Check if PGs are created on a node, the node status should include /// the PG labels. - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); // Adding a node. node->mutable_resources_total()->insert({"CPU", 2}); @@ -485,7 +486,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestNodeDynamicLabelsWithPG) { } TEST_F(GcsAutoscalerStateManagerTest, TestBasicResourceRequests) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node->mutable_resources_total()->insert({"CPU", 2}); node->mutable_resources_total()->insert({"GPU", 1}); node->set_instance_id("instance_1"); @@ -501,16 +502,16 @@ TEST_F(GcsAutoscalerStateManagerTest, TestBasicResourceRequests) { // Update resource usages. { UpdateResourceLoads(node->node_id(), - {Mocker::GenResourceDemand({{"CPU", 1}}, - /* nun_ready_queued */ 1, - /* nun_infeasible */ 1, - /* num_backlog */ 0, - /* label_selectors */ {}), - Mocker::GenResourceDemand({{"CPU", 4}, {"GPU", 2}}, - /* num_ready_queued */ 0, - /* num_infeasible */ 1, - /* num_backlog */ 1, - /* label_selectors */ {})}); + {GenResourceDemand({{"CPU", 1}}, + /* nun_ready_queued */ 1, + /* nun_infeasible */ 1, + /* num_backlog */ 0, + /* label_selectors */ {}), + GenResourceDemand({{"CPU", 4}, {"GPU", 2}}, + /* num_ready_queued */ 0, + /* num_infeasible */ 1, + /* num_backlog */ 1, + /* label_selectors */ {})}); const auto &state = GetClusterResourceStateSync(); // Expect each pending resources shape to be num_infeasible + num_backlog. @@ -526,7 +527,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestBasicResourceRequests) { } TEST_F(GcsAutoscalerStateManagerTest, TestGangResourceRequestsBasic) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node->mutable_resources_total()->insert({"CPU", 1}); node->set_instance_id("instance_1"); // Adding a node. @@ -543,14 +544,13 @@ TEST_F(GcsAutoscalerStateManagerTest, TestGangResourceRequestsBasic) { { auto pg = PlacementGroupID::Of(job_id); EXPECT_CALL(*gcs_placement_group_manager_, GetPlacementGroupLoad) - .WillOnce( - Return(Mocker::GenPlacementGroupLoad({Mocker::GenPlacementGroupTableData( - pg, - job_id, - {{{"CPU", 1}}, {{"GPU", 1}}}, - {"", ""}, - rpc::PlacementStrategy::STRICT_SPREAD, - rpc::PlacementGroupTableData::PENDING)}))); + .WillOnce(Return(GenPlacementGroupLoad( + {GenPlacementGroupTableData(pg, + job_id, + {{{"CPU", 1}}, {{"GPU", 1}}}, + {"", ""}, + rpc::PlacementStrategy::STRICT_SPREAD, + rpc::PlacementGroupTableData::PENDING)}))); auto state = GetClusterResourceStateSync(); CheckGangResourceRequests(state, @@ -564,14 +564,13 @@ TEST_F(GcsAutoscalerStateManagerTest, TestGangResourceRequestsBasic) { { auto pg = PlacementGroupID::Of(job_id); EXPECT_CALL(*gcs_placement_group_manager_, GetPlacementGroupLoad) - .WillOnce( - Return(Mocker::GenPlacementGroupLoad({Mocker::GenPlacementGroupTableData( - pg, - job_id, - {{{"CPU", 1}}, {{"GPU", 1}}}, - {"", ""}, - rpc::PlacementStrategy::STRICT_PACK, - rpc::PlacementGroupTableData::PENDING)}))); + .WillOnce(Return(GenPlacementGroupLoad( + {GenPlacementGroupTableData(pg, + job_id, + {{{"CPU", 1}}, {{"GPU", 1}}}, + {"", ""}, + rpc::PlacementStrategy::STRICT_PACK, + rpc::PlacementGroupTableData::PENDING)}))); auto state = GetClusterResourceStateSync(); CheckGangResourceRequests(state, @@ -583,7 +582,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestGangResourceRequestsBasic) { } TEST_F(GcsAutoscalerStateManagerTest, TestGangResourceRequestsNonStrict) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node->set_instance_id("instance_1"); node->mutable_resources_total()->insert({"CPU", 1}); // Adding a node. @@ -597,20 +596,19 @@ TEST_F(GcsAutoscalerStateManagerTest, TestGangResourceRequestsNonStrict) { auto pg1 = PlacementGroupID::Of(job_id1); auto pg2 = PlacementGroupID::Of(job_id2); EXPECT_CALL(*gcs_placement_group_manager_, GetPlacementGroupLoad) - .WillOnce(Return(Mocker::GenPlacementGroupLoad( - {Mocker::GenPlacementGroupTableData(pg1, - job_id1, - {{{"CPU", 1}, {"GPU", 2}}}, - {""}, - rpc::PlacementStrategy::PACK, - rpc::PlacementGroupTableData::PENDING), - Mocker::GenPlacementGroupTableData( - pg2, - job_id2, - {{{"TPU", 1}}}, - {""}, - rpc::PlacementStrategy::SPREAD, - rpc::PlacementGroupTableData::PENDING)}))); + .WillOnce(Return(GenPlacementGroupLoad( + {GenPlacementGroupTableData(pg1, + job_id1, + {{{"CPU", 1}, {"GPU", 2}}}, + {""}, + rpc::PlacementStrategy::PACK, + rpc::PlacementGroupTableData::PENDING), + GenPlacementGroupTableData(pg2, + job_id2, + {{{"TPU", 1}}}, + {""}, + rpc::PlacementStrategy::SPREAD, + rpc::PlacementGroupTableData::PENDING)}))); const auto &state = GetClusterResourceStateSync(); CheckGangResourceRequests(state, @@ -621,7 +619,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestGangResourceRequestsNonStrict) { } TEST_F(GcsAutoscalerStateManagerTest, TestGangResourceRequestsPartialRescheduling) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node->set_instance_id("instance_1"); node->mutable_resources_total()->insert({"CPU", 1}); // Adding a node. @@ -632,14 +630,13 @@ TEST_F(GcsAutoscalerStateManagerTest, TestGangResourceRequestsPartialReschedulin auto pg1 = PlacementGroupID::Of(job_id1); EXPECT_CALL(*gcs_placement_group_manager_, GetPlacementGroupLoad) - .WillOnce( - Return(Mocker::GenPlacementGroupLoad({Mocker::GenPlacementGroupTableData( - pg1, - job_id1, - {{{"CPU_failed_1", 1}}, {{"CPU_success_2", 2}}}, - {"", node->node_id()}, - rpc::PlacementStrategy::STRICT_SPREAD, - rpc::PlacementGroupTableData::RESCHEDULING)}))); + .WillOnce(Return(GenPlacementGroupLoad( + {GenPlacementGroupTableData(pg1, + job_id1, + {{{"CPU_failed_1", 1}}, {{"CPU_success_2", 2}}}, + {"", node->node_id()}, + rpc::PlacementStrategy::STRICT_SPREAD, + rpc::PlacementGroupTableData::RESCHEDULING)}))); const auto &state = GetClusterResourceStateSync(); @@ -662,7 +659,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestClusterResourcesConstraint) { // Generate one constraint. { RequestClusterResourceConstraint( - Mocker::GenClusterResourcesConstraint({{{"CPU", 2}, {"GPU", 1}}}, {1})); + GenClusterResourcesConstraint({{{"CPU", 2}, {"GPU", 1}}}, {1})); const auto &state = GetClusterResourceStateSync(); ASSERT_EQ(state.cluster_resource_constraints_size(), 1); ASSERT_EQ(state.cluster_resource_constraints(0).resource_requests_size(), 1); @@ -673,8 +670,8 @@ TEST_F(GcsAutoscalerStateManagerTest, TestClusterResourcesConstraint) { // Override it { - RequestClusterResourceConstraint(Mocker::GenClusterResourcesConstraint( - {{{"CPU", 4}, {"GPU", 5}, {"TPU", 1}}}, {1})); + RequestClusterResourceConstraint( + GenClusterResourcesConstraint({{{"CPU", 4}, {"GPU", 5}, {"TPU", 1}}}, {1})); const auto &state = GetClusterResourceStateSync(); ASSERT_EQ(state.cluster_resource_constraints_size(), 1); ASSERT_EQ(state.cluster_resource_constraints(0).resource_requests_size(), 1); @@ -726,7 +723,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestReportAutoscalingState) { } TEST_F(GcsAutoscalerStateManagerTest, TestDrainNonAliveNode) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); // Adding a node. node->mutable_resources_total()->insert({"CPU", 2}); @@ -751,7 +748,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestDrainNonAliveNode) { } TEST_F(GcsAutoscalerStateManagerTest, TestDrainingStatus) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); // Adding a node. node->mutable_resources_total()->insert({"CPU", 2}); @@ -786,7 +783,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestDrainingStatus) { } TEST_F(GcsAutoscalerStateManagerTest, TestDrainNodeRaceCondition) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); // Adding a node. node->mutable_resources_total()->insert({"CPU", 2}); @@ -818,7 +815,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestDrainNodeRaceCondition) { } TEST_F(GcsAutoscalerStateManagerTest, TestIdleTime) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); // Adding a node. node->mutable_resources_total()->insert({"CPU", 2}); @@ -879,8 +876,8 @@ TEST_F(GcsAutoscalerStateManagerTest, TestGcsKvManagerInternalConfig) { TEST_F(GcsAutoscalerStateManagerTest, TestGetPerNodeInfeasibleResourceRequests_NoInfeasibleRequests) { // Prepare - auto node_1 = Mocker::GenNodeInfo(); - auto node_2 = Mocker::GenNodeInfo(); + auto node_1 = GenNodeInfo(); + auto node_2 = GenNodeInfo(); // Add nodes { @@ -895,27 +892,27 @@ TEST_F(GcsAutoscalerStateManagerTest, // Update resource usages { UpdateResourceLoads(node_1->node_id(), - {Mocker::GenResourceDemand({{"GPU", 1}}, - /* nun_ready_queued */ 1, - /* nun_infeasible */ 1, - /* num_backlog */ 0, - /* label_selectors */ {}), - Mocker::GenResourceDemand({{"CPU", 1}}, - /* nun_ready_queued */ 1, - /* nun_infeasible */ 0, - /* num_backlog */ 1, - /* label_selectors */ {}), - Mocker::GenResourceDemand({{"CPU", 3}}, - /* num_ready_queued */ 0, - /* num_infeasible */ 1, - /* num_backlog */ 1, - /* label_selectors */ {})}); + {GenResourceDemand({{"GPU", 1}}, + /* nun_ready_queued */ 1, + /* nun_infeasible */ 1, + /* num_backlog */ 0, + /* label_selectors */ {}), + GenResourceDemand({{"CPU", 1}}, + /* nun_ready_queued */ 1, + /* nun_infeasible */ 0, + /* num_backlog */ 1, + /* label_selectors */ {}), + GenResourceDemand({{"CPU", 3}}, + /* num_ready_queued */ 0, + /* num_infeasible */ 1, + /* num_backlog */ 1, + /* label_selectors */ {})}); UpdateResourceLoads(node_2->node_id(), - {Mocker::GenResourceDemand({{"CPU", 2}}, - /* nun_ready_queued */ 1, - /* nun_infeasible */ 0, - /* num_backlog */ 1, - /* label_selectors */ {})}); + {GenResourceDemand({{"CPU", 2}}, + /* nun_ready_queued */ 1, + /* nun_infeasible */ 0, + /* num_backlog */ 1, + /* label_selectors */ {})}); } // Update autoscaling state @@ -942,8 +939,8 @@ TEST_F(GcsAutoscalerStateManagerTest, TEST_F(GcsAutoscalerStateManagerTest, TestGetPerNodeInfeasibleResourceRequests_WithInfeasibleRequests) { // Prepare - auto node_1 = Mocker::GenNodeInfo(); - auto node_2 = Mocker::GenNodeInfo(); + auto node_1 = GenNodeInfo(); + auto node_2 = GenNodeInfo(); // Add nodes { @@ -958,27 +955,27 @@ TEST_F(GcsAutoscalerStateManagerTest, // Update resource usages { UpdateResourceLoads(node_1->node_id(), - {Mocker::GenResourceDemand({{"GPU", 1}}, - /* nun_ready_queued */ 1, - /* nun_infeasible */ 1, - /* num_backlog */ 0), + {GenResourceDemand({{"GPU", 1}}, + /* nun_ready_queued */ 1, + /* nun_infeasible */ 1, + /* num_backlog */ 0), /* label_selectors */ {}, - Mocker::GenResourceDemand({{"CPU", 1}}, - /* nun_ready_queued */ 1, - /* nun_infeasible */ 0, - /* num_backlog */ 1), + GenResourceDemand({{"CPU", 1}}, + /* nun_ready_queued */ 1, + /* nun_infeasible */ 0, + /* num_backlog */ 1), /* label_selectors */ {}, - Mocker::GenResourceDemand({{"CPU", 3}}, - /* num_ready_queued */ 0, - /* num_infeasible */ 1, - /* num_backlog */ 1, - /* label_selectors */ {})}); + GenResourceDemand({{"CPU", 3}}, + /* num_ready_queued */ 0, + /* num_infeasible */ 1, + /* num_backlog */ 1, + /* label_selectors */ {})}); UpdateResourceLoads(node_2->node_id(), - {Mocker::GenResourceDemand({{"CPU", 2}}, - /* nun_ready_queued */ 1, - /* nun_infeasible */ 0, - /* num_backlog */ 1, - /* label_selectors */ {})}); + {GenResourceDemand({{"CPU", 2}}, + /* nun_ready_queued */ 1, + /* nun_infeasible */ 0, + /* num_backlog */ 1, + /* label_selectors */ {})}); } // Update autoscaling state @@ -1021,7 +1018,7 @@ TEST_F(GcsAutoscalerStateManagerTest, } TEST_F(GcsAutoscalerStateManagerTest, TestNodeLabelsAdded) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node->mutable_resources_total()->insert({"CPU", 2}); node->set_instance_id("instance_1"); (*node->mutable_labels())["accelerator-type"] = "TPU"; @@ -1036,7 +1033,7 @@ TEST_F(GcsAutoscalerStateManagerTest, TestNodeLabelsAdded) { } TEST_F(GcsAutoscalerStateManagerTest, TestGetPendingResourceRequestsWithLabelSelectors) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node->mutable_resources_total()->insert({"CPU", 2}); node->set_instance_id("instance_1"); AddNode(node); @@ -1061,11 +1058,11 @@ TEST_F(GcsAutoscalerStateManagerTest, TestGetPendingResourceRequestsWithLabelSel // Simulate an infeasible request with a label selector UpdateResourceLoads(node->node_id(), - {Mocker::GenResourceDemand({{"CPU", 2}}, - /*ready=*/0, - /*infeasible=*/1, - /*backlog=*/0, - {selector})}); + {GenResourceDemand({{"CPU", 2}}, + /*ready=*/0, + /*infeasible=*/1, + /*backlog=*/0, + {selector})}); } // Validate the cluster state includes the generated pending request diff --git a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc index 5fe6fad4aa96..0678826f0ac8 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc @@ -21,9 +21,9 @@ #include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" #include "mock/ray/pubsub/publisher.h" #include "mock/ray/rpc/worker/core_worker_client.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/tests/gcs_test_util.h" namespace ray { @@ -129,7 +129,7 @@ TEST_F(GcsJobManagerTest, TestIsRunningTasks) { address.set_worker_id(WorkerID::FromRandom().Binary()); auto add_job_request = - Mocker::GenAddJobRequest(job_id, std::to_string(i), std::to_string(i), address); + GenAddJobRequest(job_id, std::to_string(i), std::to_string(i), address); rpc::AddJobReply empty_reply; std::promise promise; gcs_job_manager_->HandleAddJob( @@ -171,8 +171,7 @@ TEST_F(GcsJobManagerTest, TestGetAllJobInfo) { // Add 100 jobs. for (int i = 0; i < 100; ++i) { auto job_id = JobID::FromInt(i); - auto add_job_request = - Mocker::GenAddJobRequest(job_id, "namespace_" + std::to_string(i)); + auto add_job_request = GenAddJobRequest(job_id, "namespace_" + std::to_string(i)); rpc::AddJobReply empty_reply; std::promise promise; gcs_job_manager_->HandleAddJob( @@ -203,8 +202,7 @@ TEST_F(GcsJobManagerTest, TestGetAllJobInfo) { // API.") auto job_api_job_id = JobID::FromInt(100); std::string submission_id = "submission_id_100"; - auto add_job_request = - Mocker::GenAddJobRequest(job_api_job_id, "namespace_100", submission_id); + auto add_job_request = GenAddJobRequest(job_api_job_id, "namespace_100", submission_id); rpc::AddJobReply empty_reply; std::promise promise; gcs_job_manager_->HandleAddJob( @@ -307,8 +305,7 @@ TEST_F(GcsJobManagerTest, TestGetAllJobInfo) { // Add another job with the *same* submission ID. This can happen if the entrypoint // script calls ray.init() multiple times. auto job_id2 = JobID::FromInt(2); - auto add_job_request2 = - Mocker::GenAddJobRequest(job_id2, "namespace_100", submission_id); + auto add_job_request2 = GenAddJobRequest(job_id2, "namespace_100", submission_id); std::promise promise4; gcs_job_manager_->HandleAddJob( *add_job_request2, @@ -344,8 +341,7 @@ TEST_F(GcsJobManagerTest, TestGetAllJobInfoWithFilter) { std::promise promise1; std::promise promise2; - auto add_job_request1 = - Mocker::GenAddJobRequest(job_id1, "namespace_1", "submission_1"); + auto add_job_request1 = GenAddJobRequest(job_id1, "namespace_1", "submission_1"); gcs_job_manager_->HandleAddJob( *add_job_request1, &empty_reply, @@ -354,8 +350,7 @@ TEST_F(GcsJobManagerTest, TestGetAllJobInfoWithFilter) { }); promise1.get_future().get(); - auto add_job_request2 = - Mocker::GenAddJobRequest(job_id2, "namespace_2", "submission_2"); + auto add_job_request2 = GenAddJobRequest(job_id2, "namespace_2", "submission_2"); gcs_job_manager_->HandleAddJob( *add_job_request2, &empty_reply, @@ -422,7 +417,7 @@ TEST_F(GcsJobManagerTest, TestGetAllJobInfoWithLimit) { std::promise promise1; std::promise promise2; - auto add_job_request1 = Mocker::GenAddJobRequest(job_id1, "namespace_1"); + auto add_job_request1 = GenAddJobRequest(job_id1, "namespace_1"); gcs_job_manager_->HandleAddJob( *add_job_request1, &empty_reply, @@ -431,7 +426,7 @@ TEST_F(GcsJobManagerTest, TestGetAllJobInfoWithLimit) { }); promise1.get_future().get(); - auto add_job_request2 = Mocker::GenAddJobRequest(job_id2, "namespace_2"); + auto add_job_request2 = GenAddJobRequest(job_id2, "namespace_2"); gcs_job_manager_->HandleAddJob( *add_job_request2, &empty_reply, @@ -518,7 +513,7 @@ TEST_F(GcsJobManagerTest, TestGetJobConfig) { std::promise promise1; std::promise promise2; - auto add_job_request1 = Mocker::GenAddJobRequest(job_id1, "namespace_1"); + auto add_job_request1 = GenAddJobRequest(job_id1, "namespace_1"); gcs_job_manager_->HandleAddJob( *add_job_request1, &empty_reply, @@ -527,7 +522,7 @@ TEST_F(GcsJobManagerTest, TestGetJobConfig) { }); promise1.get_future().get(); - auto add_job_request2 = Mocker::GenAddJobRequest(job_id2, "namespace_2"); + auto add_job_request2 = GenAddJobRequest(job_id2, "namespace_2"); gcs_job_manager_->HandleAddJob( *add_job_request2, &empty_reply, @@ -547,7 +542,7 @@ TEST_F(GcsJobManagerTest, TestPreserveDriverInfo) { auto job_id = JobID::FromInt(1); gcs::GcsInitData gcs_init_data(*gcs_table_storage_); gcs_job_manager_->Initialize(/*init_data=*/gcs_init_data); - auto add_job_request = Mocker::GenAddJobRequest(job_id, "namespace"); + auto add_job_request = GenAddJobRequest(job_id, "namespace"); rpc::Address address; address.set_ip_address("10.0.0.1"); @@ -619,7 +614,7 @@ TEST_F(GcsJobManagerTest, TestMarkJobFinishedIdempotency) { gcs_job_manager.Initialize(/*init_data=*/gcs_init_data); // Add a job first - auto add_job_request = Mocker::GenAddJobRequest(job_id, "namespace"); + auto add_job_request = GenAddJobRequest(job_id, "namespace"); rpc::AddJobReply add_job_reply; std::promise add_promise; gcs_job_manager.HandleAddJob( @@ -703,7 +698,7 @@ TEST_F(GcsJobManagerTest, TestNodeFailure) { std::promise promise1; std::promise promise2; - auto add_job_request1 = Mocker::GenAddJobRequest(job_id1, "namespace_1"); + auto add_job_request1 = GenAddJobRequest(job_id1, "namespace_1"); gcs_job_manager_->HandleAddJob( *add_job_request1, &empty_reply, @@ -712,7 +707,7 @@ TEST_F(GcsJobManagerTest, TestNodeFailure) { }); promise1.get_future().get(); - auto add_job_request2 = Mocker::GenAddJobRequest(job_id2, "namespace_2"); + auto add_job_request2 = GenAddJobRequest(job_id2, "namespace_2"); gcs_job_manager_->HandleAddJob( *add_job_request2, &empty_reply, diff --git a/src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc index 9792fd9e0898..26d9f1a56fe5 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc @@ -21,7 +21,7 @@ #include #include "gtest/gtest.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/store_client_kv.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/store_client/redis_store_client.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc index 7b8472a385bf..7086c31bb025 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc @@ -22,7 +22,7 @@ #include "fakes/ray/rpc/raylet/raylet_client.h" #include "mock/ray/pubsub/publisher.h" -#include "ray/gcs/tests/gcs_test_util.h" +#include "ray/common/test_utils.h" namespace ray { class GcsNodeManagerTest : public ::testing::Test { @@ -52,7 +52,7 @@ TEST_F(GcsNodeManagerTest, TestManagement) { client_pool_.get(), ClusterID::Nil()); // Test Add/Get/Remove functionality. - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); node_manager.AddNode(node); @@ -77,7 +77,7 @@ TEST_F(GcsNodeManagerTest, TestListener) { added_nodes.emplace_back(std::move(node)); }); for (int i = 0; i < node_count; ++i) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node_manager.AddNode(node); } ASSERT_EQ(node_count, added_nodes.size()); @@ -114,7 +114,7 @@ TEST_F(GcsNodeManagerTest, TestUpdateAliveNode) { ClusterID::Nil()); // Create a test node - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); auto node_id = NodeID::FromBinary(node->node_id()); // Add the node to the manager diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc index 4be984192de4..ad8db572aa44 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc @@ -22,8 +22,8 @@ #include "mock/ray/gcs/gcs_server/gcs_placement_group_scheduler.h" #include "mock/ray/gcs/gcs_server/gcs_resource_manager.h" #include "mock/ray/gcs/store_client/store_client.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_placement_group_manager.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" #include "ray/util/counter_map.h" @@ -69,8 +69,7 @@ class GcsPlacementGroupManagerMockTest : public Test { TEST_F(GcsPlacementGroupManagerMockTest, PendingQueuePriorityReschedule) { // Test priority works // When return with reschedule, it should be given with the highest pri - auto req = - Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::SPREAD, 1); + auto req = GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::SPREAD, 1); auto pg = std::make_shared(req, "", counter_); auto cb = [](Status s) {}; SchedulePgRequest request; @@ -96,8 +95,7 @@ TEST_F(GcsPlacementGroupManagerMockTest, PendingQueuePriorityReschedule) { TEST_F(GcsPlacementGroupManagerMockTest, PendingQueuePriorityFailed) { // Test priority works // When return with a failure, exp backoff should work - auto req = - Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::SPREAD, 1); + auto req = GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::SPREAD, 1); auto pg = std::make_shared(req, "", counter_); auto cb = [](Status s) {}; SchedulePgRequest request; @@ -150,11 +148,9 @@ TEST_F(GcsPlacementGroupManagerMockTest, PendingQueuePriorityOrder) { // Test priority works // Add two pgs // Fail one and make sure it's scheduled later - auto req1 = - Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::SPREAD, 1); + auto req1 = GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::SPREAD, 1); auto pg1 = std::make_shared(req1, "", counter_); - auto req2 = - Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::SPREAD, 1); + auto req2 = GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::SPREAD, 1); auto pg2 = std::make_shared(req2, "", counter_); auto cb = [](Status s) {}; SchedulePgRequest request; diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc index d6421e6b7758..25a0986b0a96 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc @@ -23,8 +23,8 @@ #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/test_utils.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" #include "ray/util/counter_map.h" @@ -224,7 +224,7 @@ class GcsPlacementGroupManagerTest : public ::testing::Test { }; TEST_F(GcsPlacementGroupManagerTest, TestPlacementGroupBundleCache) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](const Status &status) { @@ -244,7 +244,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestPlacementGroupBundleCache) { } TEST_F(GcsPlacementGroupManagerTest, TestBasic) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](const Status &status) { @@ -263,7 +263,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestBasic) { } TEST_F(GcsPlacementGroupManagerTest, TestSchedulingFailed) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](const Status &status) { @@ -296,7 +296,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestSchedulingFailed) { } TEST_F(GcsPlacementGroupManagerTest, TestGetPlacementGroupIDByName) { - auto request = Mocker::GenCreatePlacementGroupRequest("test_name"); + auto request = GenCreatePlacementGroupRequest("test_name"); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -315,7 +315,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestGetPlacementGroupIDByName) { } TEST_F(GcsPlacementGroupManagerTest, TestRemoveNamedPlacementGroup) { - auto request = Mocker::GenCreatePlacementGroupRequest("test_name"); + auto request = GenCreatePlacementGroupRequest("test_name"); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](const Status &status) { @@ -339,7 +339,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestRemoveNamedPlacementGroup) { } TEST_F(GcsPlacementGroupManagerTest, TestRemovedPlacementGroupNotReportedAsLoad) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -367,7 +367,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestRemovedPlacementGroupNotReportedAsLoad) } TEST_F(GcsPlacementGroupManagerTest, TestRescheduleWhenNodeAdd) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -389,7 +389,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestRescheduleWhenNodeAdd) { } TEST_F(GcsPlacementGroupManagerTest, TestRemovingPendingPlacementGroup) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -428,7 +428,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestRemovingPendingPlacementGroup) { } TEST_F(GcsPlacementGroupManagerTest, TestRemovingLeasingPlacementGroup) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -464,7 +464,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestRemovingLeasingPlacementGroup) { } TEST_F(GcsPlacementGroupManagerTest, TestRemovingCreatedPlacementGroup) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -508,7 +508,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestReschedulingRetry) { /// pg scheduled -> pg created -> node dead -> /// pg rescheduled -> rescheduling failed -> retry. /// - auto request1 = Mocker::GenCreatePlacementGroupRequest(); + auto request1 = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request1, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -551,7 +551,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestRescheduleWhenNodeDead) { /// Test the basic case. /// pg scheduled -> pg created -> node dead -> pg rescheduled. /// - auto request1 = Mocker::GenCreatePlacementGroupRequest(); + auto request1 = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request1, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -587,7 +587,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestNodeDeadBeforePlacementGroupCreated) { /// Test the case where a node dies before the placement group is created. /// pg scheduled -> node dead -> pg created -> pg rescheduled. /// - auto request1 = Mocker::GenCreatePlacementGroupRequest(); + auto request1 = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request1, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -630,7 +630,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestTwoNodesWithBundlesFromSamePlacementGro /// -> bundles on node1 created -> pg rescheduled (for bundles on node2) -> pg created /// - auto request1 = Mocker::GenCreatePlacementGroupRequest(); + auto request1 = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request1, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -693,7 +693,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestTwoNodesWithBundlesFromSamePlacementGro /// -> pg prepared -> bundles on node1 created -> pg rescheduled (for bundles on node2) /// -> pg created /// - auto request1 = Mocker::GenCreatePlacementGroupRequest(); + auto request1 = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request1, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -755,7 +755,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestTwoNodesWithBundlesFromSamePlacementGro /// pg scheduled -> pg created -> node1 dead -> pg rescheduled -> node2 dead /// -> pg still in rescheduled state -> pg prepared -> pg created /// - auto request1 = Mocker::GenCreatePlacementGroupRequest(); + auto request1 = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request1, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -809,15 +809,14 @@ TEST_F(GcsPlacementGroupManagerTest, TestTwoNodesWithBundlesFromSamePlacementGro TEST_F(GcsPlacementGroupManagerTest, TestSchedulerReinitializeAfterGcsRestart) { // Create a placement group and make sure it has been created successfully. auto job_id = JobID::FromInt(1); - auto request = Mocker::GenCreatePlacementGroupRequest( + auto request = GenCreatePlacementGroupRequest( /* name */ "", rpc::PlacementStrategy::SPREAD, /* bundles_count */ 2, /* cpu_num */ 1.0, /* job_id */ job_id); - auto job_table_data = Mocker::GenJobTableData(job_id); - gcs_table_storage_->JobTable().Put( - job_id, *job_table_data, {[](auto) {}, io_service_}); + auto job_table_data = GenJobTableData(job_id); + gcs_table_storage_->JobTable().Put(job_id, *job_table_data, {[](auto) {}, io_service_}); std::atomic registered_placement_group_count{0}; RegisterPlacementGroup(request, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; @@ -851,7 +850,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestAutomaticCleanupWhenActorDeadAndJobDead // Test the scenario where actor dead -> job dead. const auto job_id = JobID::FromInt(1); const auto actor_id = ActorID::Of(job_id, TaskID::Nil(), 0); - auto request = Mocker::GenCreatePlacementGroupRequest( + auto request = GenCreatePlacementGroupRequest( /* name */ "", rpc::PlacementStrategy::SPREAD, /* bundles_count */ 2, @@ -887,7 +886,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestAutomaticCleanupWhenActorAndJobDead) { // Test the scenario where job dead -> actor dead. const auto job_id = JobID::FromInt(1); const auto actor_id = ActorID::Of(job_id, TaskID::Nil(), 0); - auto request = Mocker::GenCreatePlacementGroupRequest( + auto request = GenCreatePlacementGroupRequest( /* name */ "", rpc::PlacementStrategy::SPREAD, /* bundles_count */ 2, @@ -925,7 +924,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestAutomaticCleanupWhenActorAndJobDead) { TEST_F(GcsPlacementGroupManagerTest, TestAutomaticCleanupWhenOnlyJobDead) { // Test placement group is cleaned when both actor & job are dead. const auto job_id = JobID::FromInt(1); - auto request = Mocker::GenCreatePlacementGroupRequest( + auto request = GenCreatePlacementGroupRequest( /* name */ "", rpc::PlacementStrategy::SPREAD, /* bundles_count */ 2, @@ -959,7 +958,7 @@ TEST_F(GcsPlacementGroupManagerTest, // Test placement group is cleaned when both actor & job are dead. const auto job_id = JobID::FromInt(1); const auto different_job_id = JobID::FromInt(3); - auto request = Mocker::GenCreatePlacementGroupRequest( + auto request = GenCreatePlacementGroupRequest( /* name */ "", rpc::PlacementStrategy::SPREAD, /* bundles_count */ 2, @@ -990,7 +989,7 @@ TEST_F(GcsPlacementGroupManagerTest, } TEST_F(GcsPlacementGroupManagerTest, TestSchedulingCanceledWhenPgIsInfeasible) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](const Status &status) { @@ -1029,11 +1028,11 @@ TEST_F(GcsPlacementGroupManagerTest, TestSchedulingCanceledWhenPgIsInfeasible) { } TEST_F(GcsPlacementGroupManagerTest, TestRayNamespace) { - auto request1 = Mocker::GenCreatePlacementGroupRequest("test_name"); + auto request1 = GenCreatePlacementGroupRequest("test_name"); job_namespace_table_[JobID::FromInt(11)] = "another_namespace"; - auto request2 = Mocker::GenCreatePlacementGroupRequest( + auto request2 = GenCreatePlacementGroupRequest( "test_name", rpc::PlacementStrategy::SPREAD, 2, 1.0, JobID::FromInt(11)); - auto request3 = Mocker::GenCreatePlacementGroupRequest("test_name"); + auto request3 = GenCreatePlacementGroupRequest("test_name"); { // Create a placement group in the empty namespace. std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request1, [®istered_placement_group_count](Status status) { @@ -1092,7 +1091,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestRayNamespace) { } TEST_F(GcsPlacementGroupManagerTest, TestStats) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); RegisterPlacementGroup(request, [®istered_placement_group_count](const Status &status) { @@ -1152,7 +1151,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestStats) { } TEST_F(GcsPlacementGroupManagerTest, TestStatsCreationTime) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); std::atomic registered_placement_group_count(0); auto request_received_ns = absl::GetCurrentTimeNanos(); RegisterPlacementGroup(request, @@ -1197,7 +1196,7 @@ TEST_F(GcsPlacementGroupManagerTest, TestGetAllPlacementGroupInfoLimit) { auto num_pgs = 3; std::atomic registered_placement_group_count(0); for (int i = 0; i < num_pgs; i++) { - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); RegisterPlacementGroup(request, [®istered_placement_group_count](const Status &status) { ++registered_placement_group_count; @@ -1236,16 +1235,15 @@ TEST_F(GcsPlacementGroupManagerTest, TestGetAllPlacementGroupInfoLimit) { TEST_F(GcsPlacementGroupManagerTest, TestCheckCreatorJobIsDeadWhenGcsRestart) { auto job_id = JobID::FromInt(1); - auto request = Mocker::GenCreatePlacementGroupRequest( + auto request = GenCreatePlacementGroupRequest( /* name */ "", rpc::PlacementStrategy::SPREAD, /* bundles_count */ 2, /* cpu_num */ 1.0, /* job_id */ job_id); - auto job_table_data = Mocker::GenJobTableData(job_id); + auto job_table_data = GenJobTableData(job_id); job_table_data->set_is_dead(true); - gcs_table_storage_->JobTable().Put( - job_id, *job_table_data, {[](auto) {}, io_service_}); + gcs_table_storage_->JobTable().Put(job_id, *job_table_data, {[](auto) {}, io_service_}); std::atomic registered_placement_group_count{0}; RegisterPlacementGroup(request, [®istered_placement_group_count](Status status) { ++registered_placement_group_count; diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc index af3db9c5f798..92b8208f410a 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc @@ -25,12 +25,12 @@ #include "fakes/ray/rpc/raylet/raylet_client.h" #include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/gcs_server/gcs_placement_group.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/util/counter_map.h" @@ -145,7 +145,7 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { void ScheduleFailedWithZeroNodeTest(rpc::PlacementStrategy strategy) { ASSERT_EQ(0, gcs_node_manager_->GetAllAliveNodes().size()); - auto request = Mocker::GenCreatePlacementGroupRequest("", strategy); + auto request = GenCreatePlacementGroupRequest("", strategy); auto placement_group = std::make_shared(request, "", counter_); // Schedule the placement_group with zero node. @@ -169,11 +169,11 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { } void SchedulePlacementGroupSuccessTest(rpc::PlacementStrategy strategy) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); - auto request = Mocker::GenCreatePlacementGroupRequest("", strategy); + auto request = GenCreatePlacementGroupRequest("", strategy); auto placement_group = std::make_shared(request, "", counter_); // Schedule the placement_group with 1 available node, and the lease request should be @@ -200,7 +200,7 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { } void ReschedulingWhenNodeAddTest(rpc::PlacementStrategy strategy) { - AddNode(Mocker::GenNodeInfo(0), 1); + AddNode(GenNodeInfo(0), 1); auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); @@ -212,7 +212,7 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { }; // Failed to schedule the placement group, because the node resources is not enough. - auto request = Mocker::GenCreatePlacementGroupRequest("", strategy); + auto request = GenCreatePlacementGroupRequest("", strategy); auto placement_group = std::make_shared(request, "", counter_); scheduler_->ScheduleUnplacedBundles( SchedulePgRequest{placement_group, failure_handler, success_handler}); @@ -220,7 +220,7 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { CheckPlacementGroupSize(0, GcsPlacementGroupStatus::SUCCESS); // A new node is added, and the rescheduling is successful. - AddNode(Mocker::GenNodeInfo(0), 2); + AddNode(GenNodeInfo(0), 2); scheduler_->ScheduleUnplacedBundles( SchedulePgRequest{placement_group, failure_handler, success_handler}); ASSERT_TRUE(raylet_clients_[0]->GrantPrepareBundleResources()); @@ -230,8 +230,8 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { } void AddTwoNodes() { - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); } @@ -335,11 +335,11 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackSchedulePlacementGroupSucce } TEST_F(GcsPlacementGroupSchedulerTest, TestSchedulePlacementGroupReplyFailure) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(request, "", counter_); // Schedule the placement_group with 1 available node, and the lease request should be @@ -367,7 +367,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestSchedulePlacementGroupReplyFailure) { } TEST_F(GcsPlacementGroupSchedulerTest, TestSpreadStrategyResourceCheck) { - auto node = Mocker::GenNodeInfo(0); + auto node = GenNodeInfo(0); AddNode(node, 2); auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { @@ -378,8 +378,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestSpreadStrategyResourceCheck) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - auto request = - Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::SPREAD, 3, 2); + auto request = GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::SPREAD, 3, 2); auto placement_group = std::make_shared(request, "", counter_); scheduler_->ScheduleUnplacedBundles( SchedulePgRequest{placement_group, failure_handler, success_handler}); @@ -395,11 +394,11 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestSpreadStrategyResourceCheck) { } TEST_F(GcsPlacementGroupSchedulerTest, TestSchedulePlacementGroupReturnResource) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); - auto request = Mocker::GenCreatePlacementGroupRequest(); + auto request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(request, "", counter_); // Schedule the placement_group with 1 available node, and the lease request should be @@ -428,8 +427,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestSchedulePlacementGroupReturnResource) } TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyBalancedScheduling) { - AddNode(Mocker::GenNodeInfo(0)); - AddNode(Mocker::GenNodeInfo(1)); + AddNode(GenNodeInfo(0)); + AddNode(GenNodeInfo(1)); auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); @@ -446,7 +445,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyBalancedScheduling) int node_index = 0; for (int index = 0; index < 10; ++index) { auto request = - Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::STRICT_PACK); + GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::STRICT_PACK); auto placement_group = std::make_shared(request, "", counter_); scheduler_->ScheduleUnplacedBundles( SchedulePgRequest{placement_group, failure_handler, success_handler}); @@ -474,7 +473,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyReschedulingWhenNod } TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyResourceCheck) { - auto node0 = Mocker::GenNodeInfo(0); + auto node0 = GenNodeInfo(0); AddNode(node0); auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { @@ -485,8 +484,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyResourceCheck) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - auto request = - Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::STRICT_PACK); + auto request = GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::STRICT_PACK); auto placement_group = std::make_shared(request, "", counter_); scheduler_->ScheduleUnplacedBundles( SchedulePgRequest{placement_group, failure_handler, success_handler}); @@ -497,10 +495,10 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyResourceCheck) { // Node1 has less number of bundles, but it doesn't satisfy the resource // requirement. In this case, the bundles should be scheduled on Node0. - auto node1 = Mocker::GenNodeInfo(1); + auto node1 = GenNodeInfo(1); AddNode(node1, 1); auto create_placement_group_request2 = - Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::STRICT_PACK); + GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::STRICT_PACK); auto placement_group2 = std::make_shared(create_placement_group_request2, "", counter_); scheduler_->ScheduleUnplacedBundles( @@ -512,11 +510,11 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictPackStrategyResourceCheck) { } TEST_F(GcsPlacementGroupSchedulerTest, DestroyPlacementGroup) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -548,13 +546,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, DestroyPlacementGroup) { } TEST_F(GcsPlacementGroupSchedulerTest, DestroyCancelledPlacementGroup) { - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); const auto &placement_group_id = placement_group->GetPlacementGroupID(); @@ -582,13 +580,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, DestroyCancelledPlacementGroup) { } TEST_F(GcsPlacementGroupSchedulerTest, PlacementGroupCancelledDuringCommit) { - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); const auto &placement_group_id = placement_group->GetPlacementGroupID(); @@ -624,13 +622,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, PlacementGroupCancelledDuringPreparedPut) // After a PG is prepared by all nodes, GCS writes to Redis then commit-all. // If a Cancel is happening during prepare, or during the Redis write, i.e. before the // commit-all is called, the PG should be removed and no commits should be sent. - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -675,8 +673,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPackStrategyReschedulingWhenNodeAdd) } TEST_F(GcsPlacementGroupSchedulerTest, TestPackStrategyLargeBundlesScheduling) { - AddNode(Mocker::GenNodeInfo(0)); - AddNode(Mocker::GenNodeInfo(1)); + AddNode(GenNodeInfo(0)); + AddNode(GenNodeInfo(1)); auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { absl::MutexLock lock(&placement_group_requests_mutex_); @@ -689,8 +687,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPackStrategyLargeBundlesScheduling) { // Schedule placement group which has large bundles. // One node does not have enough resources, so we will divide bundles to two nodes. - auto request = - Mocker::GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::PACK, 15); + auto request = GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::PACK, 15); auto placement_group = std::make_shared(request, "", counter_); scheduler_->ScheduleUnplacedBundles( SchedulePgRequest{placement_group, failure_handler, success_handler}); @@ -712,13 +709,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPackStrategyLargeBundlesScheduling) { TEST_F(GcsPlacementGroupSchedulerTest, TestStrictSpreadRescheduleWhenNodeDead) { int node_count = 3; for (int index = 0; index < node_count; ++index) { - auto node = Mocker::GenNodeInfo(index); + auto node = GenNodeInfo(index); AddNode(node); } ASSERT_EQ(3, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest( - "pg1", rpc::PlacementStrategy::STRICT_SPREAD); + auto create_placement_group_request = + GenCreatePlacementGroupRequest("pg1", rpc::PlacementStrategy::STRICT_SPREAD); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -790,7 +787,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictSpreadRescheduleWhenNodeDead) { } TEST_F(GcsPlacementGroupSchedulerTest, TestStrictSpreadStrategyResourceCheck) { - auto node0 = Mocker::GenNodeInfo(0); + auto node0 = GenNodeInfo(0); AddNode(node0); auto failure_handler = [this](std::shared_ptr placement_group, bool is_insfeasble) { @@ -801,8 +798,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictSpreadStrategyResourceCheck) { absl::MutexLock lock(&placement_group_requests_mutex_); success_placement_groups_.emplace_back(std::move(placement_group)); }; - auto request = Mocker::GenCreatePlacementGroupRequest( - "", rpc::PlacementStrategy::STRICT_SPREAD, 2, 2); + auto request = + GenCreatePlacementGroupRequest("", rpc::PlacementStrategy::STRICT_SPREAD, 2, 2); auto placement_group = std::make_shared(request, "", counter_); scheduler_->ScheduleUnplacedBundles( SchedulePgRequest{placement_group, failure_handler, success_handler}); @@ -811,14 +808,14 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestStrictSpreadStrategyResourceCheck) { WaitPlacementGroupPendingDone(1, GcsPlacementGroupStatus::FAILURE); // Node1 resource is insufficient, scheduling failed. - auto node1 = Mocker::GenNodeInfo(1); + auto node1 = GenNodeInfo(1); AddNode(node1, 1); scheduler_->ScheduleUnplacedBundles( SchedulePgRequest{placement_group, failure_handler, success_handler}); WaitPlacementGroupPendingDone(2, GcsPlacementGroupStatus::FAILURE); // The node2 resource is enough and the scheduling is successful. - auto node2 = Mocker::GenNodeInfo(2); + auto node2 = GenNodeInfo(2); AddNode(node2); scheduler_->ScheduleUnplacedBundles( SchedulePgRequest{placement_group, failure_handler, success_handler}); @@ -836,8 +833,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestBundleLocationIndex) { /// Generate data. const auto node1 = NodeID::FromRandom(); const auto node2 = NodeID::FromRandom(); - rpc::CreatePlacementGroupRequest request_pg1 = - Mocker::GenCreatePlacementGroupRequest("pg1"); + rpc::CreatePlacementGroupRequest request_pg1 = GenCreatePlacementGroupRequest("pg1"); const auto pg1_id = PlacementGroupID::FromBinary( request_pg1.placement_group_spec().placement_group_id()); const std::shared_ptr bundle_node1_pg1 = @@ -853,8 +849,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestBundleLocationIndex) { (*bundle_locations_pg1) .emplace(bundle_node2_pg1->BundleId(), std::make_pair(node2, bundle_node2_pg1)); - rpc::CreatePlacementGroupRequest request_pg2 = - Mocker::GenCreatePlacementGroupRequest("pg2"); + rpc::CreatePlacementGroupRequest request_pg2 = GenCreatePlacementGroupRequest("pg2"); const auto pg2_id = PlacementGroupID::FromBinary( request_pg2.placement_group_spec().placement_group_id()); const std::shared_ptr bundle_node1_pg2 = @@ -908,13 +903,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestBundleLocationIndex) { } TEST_F(GcsPlacementGroupSchedulerTest, TestNodeDeadDuringPreparingResources) { - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -946,13 +941,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestNodeDeadDuringPreparingResourcesRaceCondition) { // This covers the scnario where the node is dead right after raylet sends a success // response. - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -988,13 +983,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, } TEST_F(GcsPlacementGroupSchedulerTest, TestNodeDeadBeforeCommittingResources) { - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -1027,13 +1022,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestNodeDeadBeforeCommittingResources) { } TEST_F(GcsPlacementGroupSchedulerTest, TestNodeErrorDuringCommittingResources) { - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -1064,13 +1059,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestNodeErrorDuringCommittingResources) { } TEST_F(GcsPlacementGroupSchedulerTest, TestNodeDeadDuringRescheduling) { - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -1121,13 +1116,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestNodeDeadDuringRescheduling) { } TEST_F(GcsPlacementGroupSchedulerTest, TestPGCancelledDuringReschedulingCommit) { - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -1180,13 +1175,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPGCancelledDuringReschedulingCommit) } TEST_F(GcsPlacementGroupSchedulerTest, TestPGCancelledDuringReschedulingCommitPrepare) { - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -1244,13 +1239,13 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestReleaseUnusedBundles) { } TEST_F(GcsPlacementGroupSchedulerTest, TestInitialize) { - auto node0 = Mocker::GenNodeInfo(0); - auto node1 = Mocker::GenNodeInfo(1); + auto node0 = GenNodeInfo(0); + auto node1 = GenNodeInfo(1); AddNode(node0); AddNode(node1); ASSERT_EQ(2, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); placement_group->GetMutableBundle(0)->set_node_id(node0->node_id()); @@ -1284,8 +1279,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPrepareFromDeadNodes) { ASSERT_TRUE(EnsureClusterResourcesAreNotInUse()); // Create a placement group. - auto placement_group = std::make_shared( - Mocker::GenCreatePlacementGroupRequest(), "", counter_); + auto placement_group = + std::make_shared(GenCreatePlacementGroupRequest(), "", counter_); // Schedule the unplaced bundles of the placement_group. ScheduleUnplacedBundles(placement_group); @@ -1312,8 +1307,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestPrepareFromNodeWithInsufficientResour ASSERT_TRUE(EnsureClusterResourcesAreNotInUse()); // Create a placement group. - auto placement_group = std::make_shared( - Mocker::GenCreatePlacementGroupRequest(), "", counter_); + auto placement_group = + std::make_shared(GenCreatePlacementGroupRequest(), "", counter_); // Schedule the unplaced bundles of the placement_group. ScheduleUnplacedBundles(placement_group); @@ -1340,8 +1335,8 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestCommitToDeadNodes) { ASSERT_TRUE(EnsureClusterResourcesAreNotInUse()); // Create a placement group. - auto placement_group = std::make_shared( - Mocker::GenCreatePlacementGroupRequest(), "", counter_); + auto placement_group = + std::make_shared(GenCreatePlacementGroupRequest(), "", counter_); // Schedule the unplaced bundles of the placement_group. ScheduleUnplacedBundles(placement_group); @@ -1366,7 +1361,7 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestCommitToDeadNodes) { } TEST_F(GcsPlacementGroupSchedulerTest, TestCheckingWildcardResource) { - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest( + auto create_placement_group_request = GenCreatePlacementGroupRequest( /*name=*/"", /*strategy=*/rpc::PlacementStrategy::SPREAD, /*bundles_count=*/1); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -1387,11 +1382,11 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestWaitingRemovedBundles) { // This feature is only required by gcs actor scheduler. RayConfig::instance().initialize(R"({"gcs_actor_scheduling_enabled": true})"); - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); @@ -1456,11 +1451,11 @@ TEST_F(GcsPlacementGroupSchedulerTest, TestWaitingRemovedBundles) { } TEST_F(GcsPlacementGroupSchedulerTest, TestBundlesRemovedWhenNodeDead) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); AddNode(node); ASSERT_EQ(1, gcs_node_manager_->GetAllAliveNodes().size()); - auto create_placement_group_request = Mocker::GenCreatePlacementGroupRequest(); + auto create_placement_group_request = GenCreatePlacementGroupRequest(); auto placement_group = std::make_shared(create_placement_group_request, "", counter_); diff --git a/src/ray/gcs/gcs_server/tests/gcs_resource_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_resource_manager_test.cc index 8a417a77363f..a652ec345edf 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_resource_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_resource_manager_test.cc @@ -21,7 +21,7 @@ #include "gtest/gtest.h" #include "mock/ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/tests/gcs_test_util.h" +#include "ray/common/test_utils.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" namespace ray { @@ -71,7 +71,7 @@ TEST_F(GcsResourceManagerTest, TestBasic) { absl::flat_hash_map resource_map; resource_map[cpu_resource] = 10; - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node->mutable_resources_total()->insert(resource_map.begin(), resource_map.end()); // Add node resources. gcs_resource_manager_->OnNodeAdd(*node); @@ -103,7 +103,7 @@ TEST_F(GcsResourceManagerTest, TestBasic) { } TEST_F(GcsResourceManagerTest, TestResourceUsageAPI) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node->mutable_resources_total()->insert({"CPU", 2}); auto node_id = NodeID::FromBinary(node->node_id()); rpc::GetAllResourceUsageRequest get_all_request; @@ -140,7 +140,7 @@ TEST_F(GcsResourceManagerTest, TestResourceUsageAPI) { } TEST_F(GcsResourceManagerTest, TestResourceUsageFromDifferentSyncMsgs) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node->mutable_resources_total()->insert({"CPU", 10}); gcs_resource_manager_->OnNodeAdd(*node); @@ -188,7 +188,7 @@ TEST_F(GcsResourceManagerTest, TestResourceUsageFromDifferentSyncMsgs) { } TEST_F(GcsResourceManagerTest, TestSetAvailableResourcesWhenNodeDead) { - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node->mutable_resources_total()->insert({"CPU", 10}); gcs_resource_manager_->OnNodeAdd(*node); @@ -212,7 +212,7 @@ TEST_F(GcsResourceManagerTest, TestNodeLabels) { absl::flat_hash_map labels = {{"key", "value"}, {"gpu_type", "a100"}}; - auto node = Mocker::GenNodeInfo(); + auto node = GenNodeInfo(); node->mutable_resources_total()->insert(resource_map.begin(), resource_map.end()); node->mutable_labels()->insert(labels.begin(), labels.end()); // Add node resources. @@ -226,7 +226,7 @@ TEST_F(GcsResourceManagerTest, TestNodeLabels) { } TEST_F(GcsResourceManagerTest, TestGetDrainingNodes) { - auto node1 = Mocker::GenNodeInfo(); + auto node1 = GenNodeInfo(); node1->mutable_resources_total()->insert({"CPU", 10}); gcs_resource_manager_->OnNodeAdd(*node1); UpdateFromResourceViewSync( @@ -237,7 +237,7 @@ TEST_F(GcsResourceManagerTest, TestGetDrainingNodes) { /* is_draining */ true, /* draining_deadline_timestamp_ms */ std::numeric_limits::max()); - auto node2 = Mocker::GenNodeInfo(); + auto node2 = GenNodeInfo(); node2->mutable_resources_total()->insert({"CPU", 1}); gcs_resource_manager_->OnNodeAdd(*node2); UpdateFromResourceViewSync(NodeID::FromBinary(node2->node_id()), diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc b/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc index 2df5c2681201..f9b208f0e33f 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc @@ -19,8 +19,8 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/ray_config.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/rpc/gcs/gcs_rpc_client.h" namespace ray { @@ -230,14 +230,14 @@ class GcsServerTest : public ::testing::Test { TEST_F(GcsServerTest, TestActorInfo) { // Create actor_table_data JobID job_id = JobID::FromInt(1); - auto actor_table_data = Mocker::GenActorTableData(job_id); + auto actor_table_data = GenActorTableData(job_id); // TODO(sand): Add tests that don't require checkponit. } TEST_F(GcsServerTest, TestJobInfo) { // Create job_table_data JobID job_id = JobID::FromInt(1); - auto job_table_data = Mocker::GenJobTableData(job_id); + auto job_table_data = GenJobTableData(job_id); // Add job rpc::AddJobRequest add_job_request; @@ -253,17 +253,17 @@ TEST_F(GcsServerTest, TestJobInfo) { TEST_F(GcsServerTest, TestJobGarbageCollection) { // Create job_table_data JobID job_id = JobID::FromInt(1); - auto job_table_data = Mocker::GenJobTableData(job_id); + auto job_table_data = GenJobTableData(job_id); // Add job rpc::AddJobRequest add_job_request; add_job_request.mutable_data()->CopyFrom(*job_table_data); ASSERT_TRUE(AddJob(add_job_request)); - auto actor_table_data = Mocker::GenActorTableData(job_id); + auto actor_table_data = GenActorTableData(job_id); // Register detached actor for job - auto detached_actor_table_data = Mocker::GenActorTableData(job_id); + auto detached_actor_table_data = GenActorTableData(job_id); detached_actor_table_data->set_is_detached(true); // Mark job finished @@ -279,7 +279,7 @@ TEST_F(GcsServerTest, TestJobGarbageCollection) { TEST_F(GcsServerTest, TestNodeInfo) { // Create gcs node info - auto gcs_node_info = Mocker::GenNodeInfo(); + auto gcs_node_info = GenNodeInfo(); // Register node info rpc::RegisterNodeRequest register_node_info_request; @@ -308,9 +308,9 @@ TEST_F(GcsServerTest, TestNodeInfo) { TEST_F(GcsServerTest, TestNodeInfoFilters) { // Create gcs node info - auto node1 = Mocker::GenNodeInfo(1, "127.0.0.1", "node1"); - auto node2 = Mocker::GenNodeInfo(2, "127.0.0.2", "node2"); - auto node3 = Mocker::GenNodeInfo(3, "127.0.0.3", "node3"); + auto node1 = GenNodeInfo(1, "127.0.0.1", "node1"); + auto node2 = GenNodeInfo(2, "127.0.0.2", "node2"); + auto node3 = GenNodeInfo(3, "127.0.0.3", "node3"); // Register node infos for (auto &node : {node1, node2, node3}) { @@ -442,7 +442,7 @@ TEST_F(GcsServerTest, TestNodeInfoFilters) { TEST_F(GcsServerTest, TestWorkerInfo) { // Report worker failure - auto worker_failure_data = Mocker::GenWorkerTableData(); + auto worker_failure_data = GenWorkerTableData(); worker_failure_data->mutable_worker_address()->set_ip_address("127.0.0.1"); worker_failure_data->mutable_worker_address()->set_port(5566); rpc::ReportWorkerFailureRequest report_worker_failure_request; @@ -452,7 +452,7 @@ TEST_F(GcsServerTest, TestWorkerInfo) { ASSERT_EQ(worker_table_data.size(), 1); // Add worker info - auto worker_data = Mocker::GenWorkerTableData(); + auto worker_data = GenWorkerTableData(); worker_data->mutable_worker_address()->set_worker_id(WorkerID::FromRandom().Binary()); rpc::AddWorkerInfoRequest add_worker_request; add_worker_request.mutable_worker_data()->CopyFrom(*worker_data); diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h index 839bbddd31d3..b90f7b6aefab 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h +++ b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h @@ -26,7 +26,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/lease/lease.h" #include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_actor_manager.h" #include "ray/gcs/gcs_server/gcs_actor_scheduler.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h b/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h index cd66ea7fb0e7..982327dffee1 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h +++ b/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h @@ -19,9 +19,8 @@ #include "gtest/gtest.h" #include "ray/common/id.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/tests/gcs_test_util.h" namespace ray { @@ -41,8 +40,8 @@ class GcsTableStorageTestBase : public ::testing::Test { auto &table = gcs_table_storage_->JobTable(); JobID job1_id = JobID::FromInt(1); JobID job2_id = JobID::FromInt(2); - auto job1_table_data = Mocker::GenJobTableData(job1_id); - auto job2_table_data = Mocker::GenJobTableData(job2_id); + auto job1_table_data = GenJobTableData(job1_id); + auto job2_table_data = GenJobTableData(job2_id); // Put. Put(table, job1_id, *job1_table_data); @@ -65,9 +64,9 @@ class GcsTableStorageTestBase : public ::testing::Test { JobID job_id1 = JobID::FromInt(1); JobID job_id2 = JobID::FromInt(2); JobID job_id3 = JobID::FromInt(3); - auto actor_table_data1 = Mocker::GenActorTableData(job_id1); - auto actor_table_data2 = Mocker::GenActorTableData(job_id2); - auto actor_table_data3 = Mocker::GenActorTableData(job_id3); + auto actor_table_data1 = GenActorTableData(job_id1); + auto actor_table_data2 = GenActorTableData(job_id2); + auto actor_table_data3 = GenActorTableData(job_id3); ActorID actor_id1 = ActorID::FromBinary(actor_table_data1->actor_id()); ActorID actor_id2 = ActorID::FromBinary(actor_table_data2->actor_id()); ActorID actor_id3 = ActorID::FromBinary(actor_table_data3->actor_id()); diff --git a/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc index 099ceadfad98..bd39ef33c655 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc @@ -26,8 +26,8 @@ #include "ray/common/asio/asio_util.h" #include "ray/common/id.h" #include "ray/common/status.h" +#include "ray/common/test_utils.h" #include "ray/gcs/pb_util.h" -#include "ray/gcs/tests/gcs_test_util.h" namespace ray { namespace gcs { @@ -117,7 +117,7 @@ class GcsTaskManagerTest : public ::testing::Test { actor_id.IsNil() ? TaskType::NORMAL_TASK : TaskType::ACTOR_TASK, actor_id), error_info); - auto events_data = Mocker::GenTaskEventsData(events); + auto events_data = GenTaskEventsData(events); SyncAddTaskEventData(events_data); } @@ -431,7 +431,7 @@ TEST_F(GcsTaskManagerTest, TestHandleAddEventBasic) { size_t num_task_events = 100; auto task_ids = GenTaskIDs(num_task_events); auto events = GenTaskEvents(task_ids, 0); - auto events_data = Mocker::GenRayEventsData(events, {}); + auto events_data = GenRayEventsData(events, {}); auto reply = SyncAddEvents(events_data); // Assert on RPC reply. @@ -448,8 +448,8 @@ TEST_F(GcsTaskManagerTest, TestHandleAddTaskEventBasic) { int32_t num_profile_events_dropped = 10; auto task_ids = GenTaskIDs(num_task_events); auto events = GenTaskEvents(task_ids, 0); - auto events_data = Mocker::GenTaskEventsData( - events, num_profile_events_dropped, num_status_events_dropped); + auto events_data = + GenTaskEventsData(events, num_profile_events_dropped, num_status_events_dropped); auto reply = SyncAddTaskEventData(events_data); @@ -483,7 +483,7 @@ TEST_F(GcsTaskManagerTest, TestHandleAddEventsMultiJobGrouping) { dropped_attempts.emplace_back(GenTaskIDForJob(0), 0); dropped_attempts.emplace_back(GenTaskIDForJob(1), 0); - auto ray_events_data = Mocker::GenRayEventsData(all_events, dropped_attempts); + auto ray_events_data = GenRayEventsData(all_events, dropped_attempts); // Send AddEvents once; converter should group by job id and GCS should record all auto reply = SyncAddEvents(ray_events_data); @@ -518,7 +518,7 @@ TEST_F(GcsTaskManagerTest, TestMergeTaskEventsSameTaskAttempt) { for (size_t i = 0; i < num_task_events; ++i) { auto profile_events = GenProfileEvents("event", i, i); auto events = GenTaskEvents(task_ids, attempt_number, 0, profile_events); - auto events_data = Mocker::GenTaskEventsData(events); + auto events_data = GenTaskEventsData(events); auto reply = SyncAddTaskEventData(events_data); EXPECT_EQ(StatusCode(reply.status().code()), StatusCode::OK); @@ -572,14 +572,14 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEvents) { auto all_events = {events_with_profile, events_with_status, events_with_both}; for (auto &events : all_events) { - auto data = Mocker::GenTaskEventsData(events); + auto data = GenTaskEventsData(events); SyncAddTaskEventData(data); } } { // Add drop counter. - auto data = Mocker::GenTaskEventsData( + auto data = GenTaskEventsData( {}, num_profile_task_events_dropped, num_status_task_events_dropped); SyncAddTaskEventData(data); } @@ -591,7 +591,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEvents) { std::vector expected_events = ConcatTaskEvents({events_with_status, events_with_profile, events_with_both}); - auto expected_data = Mocker::GenTaskEventsData(expected_events); + auto expected_data = GenTaskEventsData(expected_events); // Expect match events ExpectTaskEventsEq(expected_data.mutable_events_by_task(), reply.mutable_events_by_task()); @@ -613,7 +613,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEventsWithLimit) { auto profile_events = GenProfileEvents("event", /*start*/ 1, /*end*/ 1); auto status_update = GenStateUpdate(); auto events = GenTaskEvents(task_ids, 0, 0, profile_events, status_update); - auto data = Mocker::GenTaskEventsData(events); + auto data = GenTaskEventsData(events); SyncAddTaskEventData(data); } @@ -661,7 +661,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEventsByTaskIDs) { all_events.push_back(GenTaskEvents({task_id1}, attempt_num)); } auto events_task1 = ConcatTaskEvents(all_events); - events_data_task1 = Mocker::GenTaskEventsData(events_task1); + events_data_task1 = GenTaskEventsData(events_task1); SyncAddTaskEventData(events_data_task1); } @@ -673,7 +673,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEventsByTaskIDs) { all_events.push_back(GenTaskEvents({task_id2}, attempt_num)); } auto events_task2 = ConcatTaskEvents(all_events); - events_data_task2 = Mocker::GenTaskEventsData(events_task2); + events_data_task2 = GenTaskEventsData(events_task2); SyncAddTaskEventData(events_data_task2); } @@ -685,7 +685,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEventsByTaskIDs) { all_events.push_back(GenTaskEvents({task_id3}, attempt_num)); } auto events_task3 = ConcatTaskEvents(all_events); - events_data_task3 = Mocker::GenTaskEventsData(events_task3); + events_data_task3 = GenTaskEventsData(events_task3); SyncAddTaskEventData(events_data_task3); } @@ -783,7 +783,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEventsByJobs) { absl::nullopt, absl::nullopt, task_info); - events_data_job1 = Mocker::GenTaskEventsData(events); + events_data_job1 = GenTaskEventsData(events); SyncAddTaskEventData(events_data_job1); } @@ -798,7 +798,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEventsByJobs) { absl::nullopt, absl::nullopt, task_info); - events_data_job2 = Mocker::GenTaskEventsData(events); + events_data_job2 = GenTaskEventsData(events); SyncAddTaskEventData(events_data_job2); } @@ -813,7 +813,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEventsByJobs) { absl::nullopt, absl::nullopt, task_info); - events_data_job3 = Mocker::GenTaskEventsData(events); + events_data_job3 = GenTaskEventsData(events); SyncAddTaskEventData(events_data_job3); } @@ -921,7 +921,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEventsFilters) { absl::nullopt, absl::nullopt, task_info_actor_id); - event_data_actor_id_job1 = Mocker::GenTaskEventsData(events); + event_data_actor_id_job1 = GenTaskEventsData(events); SyncAddTaskEventData(event_data_actor_id_job1); } @@ -940,7 +940,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEventsFilters) { absl::nullopt, absl::nullopt, task_info_name); - event_data_task_name_job1 = Mocker::GenTaskEventsData(events); + event_data_task_name_job1 = GenTaskEventsData(events); SyncAddTaskEventData(event_data_task_name_job1); } @@ -960,7 +960,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEventsFilters) { GenStateUpdate({{rpc::TaskStatus::PENDING_NODE_ASSIGNMENT, 1}, {task_status, 5}}, WorkerID::Nil()), task_info); - event_data_task_state_job2 = Mocker::GenTaskEventsData(events); + event_data_task_state_job2 = GenTaskEventsData(events); SyncAddTaskEventData(event_data_task_state_job2); } @@ -1424,7 +1424,7 @@ TEST_F(GcsTaskManagerMemoryLimitedTest, TestIndexNoLeak) { GenProfileEvents("event", 1, 1), GenStateUpdate({}, worker_id), GenTaskInfo(job_id)); - auto events_data = Mocker::GenTaskEventsData(events); + auto events_data = GenTaskEventsData(events); SyncAddTaskEventData(events_data); } @@ -1445,7 +1445,7 @@ TEST_F(GcsTaskManagerMemoryLimitedTest, TestIndexNoLeak) { GenProfileEvents("event", 1, 1), GenStateUpdate(), GenTaskInfo(JobID::FromInt(job_id))); - auto events_data = Mocker::GenTaskEventsData(events); + auto events_data = GenTaskEventsData(events); SyncAddTaskEventData(events_data); } } @@ -1479,8 +1479,7 @@ TEST_F(GcsTaskManagerMemoryLimitedTest, TestLimitTaskEvents) { /* attempt_number */ 0, /* job_id */ 0, GenProfileEvents("event", 1, 1)); - auto events_data = - Mocker::GenTaskEventsData(events, num_profile_events_dropped_on_worker); + auto events_data = GenTaskEventsData(events, num_profile_events_dropped_on_worker); SyncAddTaskEventData(events_data); } { @@ -1490,9 +1489,9 @@ TEST_F(GcsTaskManagerMemoryLimitedTest, TestLimitTaskEvents) { /* job_id */ 0, /* profile_events */ absl::nullopt, GenStateUpdate()); - auto events_data = Mocker::GenTaskEventsData(events, - /*num_profile_task_events_dropped*/ 0, - num_status_events_dropped_on_worker); + auto events_data = GenTaskEventsData(events, + /*num_profile_task_events_dropped*/ 0, + num_status_events_dropped_on_worker); SyncAddTaskEventData(events_data); } @@ -1501,7 +1500,7 @@ TEST_F(GcsTaskManagerMemoryLimitedTest, TestLimitTaskEvents) { { // Add new task events to overwrite the existing ones. expected_events = GenTaskEvents(GenTaskIDs(num_batch2), 0); - auto events_data = Mocker::GenTaskEventsData(expected_events); + auto events_data = GenTaskEventsData(expected_events); SyncAddTaskEventData(events_data); } @@ -1542,7 +1541,7 @@ TEST_F(GcsTaskManagerTest, TestGetTaskEventsWithDriver) { /* status_update*/ absl::nullopt, GenTaskInfo( /* job_id */ JobID::FromInt(0), TaskID::Nil(), rpc::TaskType::DRIVER_TASK)); - auto events_data = Mocker::GenTaskEventsData(events); + auto events_data = GenTaskEventsData(events); SyncAddTaskEventData(events_data); } @@ -1583,7 +1582,7 @@ TEST_F(GcsTaskManagerMemoryLimitedTest, TestLimitReturnRecentTasksWhenGetAll) { /* job_id */ 0, /* profile event */ absl::nullopt, GenStateUpdate({{rpc::TaskStatus::RUNNING, 1}}, WorkerID::Nil())); - auto events_data = Mocker::GenTaskEventsData(events); + auto events_data = GenTaskEventsData(events); SyncAddTaskEventData(events_data); } @@ -1616,7 +1615,7 @@ TEST_F(GcsTaskManagerTest, TestTaskDataLossWorker) { EXPECT_EQ(reply.events_by_task_size(), 1); // Report it as data loss. - auto data = Mocker::GenTaskEventsDataLoss({{task_id, 0}}); + auto data = GenTaskEventsDataLoss({{task_id, 0}}); SyncAddTaskEventData(data); // The task attempt should be dropped. @@ -1639,7 +1638,7 @@ TEST_F(GcsTaskManagerTest, TestMultipleJobsDataLoss) { SyncAddTaskEvent({job_task1}, {{rpc::TaskStatus::RUNNING, 1}}, TaskID::Nil(), 1); // Make data loss happens on job 0. - auto data = Mocker::GenTaskEventsDataLoss({{job_task0, 0}}, 0); + auto data = GenTaskEventsDataLoss({{job_task0, 0}}, 0); SyncAddTaskEventData(data); // Job 0 has data loss @@ -1719,7 +1718,7 @@ TEST_F(GcsTaskManagerProfileEventsLimitTest, TestProfileEventsNoLeak) { /* attempt_number */ 0, /* job_id */ 0, GenProfileEvents("event", 1, 1)); - auto events_data = Mocker::GenTaskEventsData(events); + auto events_data = GenTaskEventsData(events); SyncAddTaskEventData(events_data); } diff --git a/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc index def15f75e97d..e23fad95e850 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc @@ -21,9 +21,9 @@ #include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/store_client_kv.h" #include "ray/gcs/store_client/in_memory_store_client.h" -#include "ray/gcs/tests/gcs_test_util.h" #include "ray/util/process.h" #include "src/ray/protobuf/common.pb.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc b/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc index 40c68497f628..4b4ddcbcfa7b 100644 --- a/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc +++ b/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc @@ -16,7 +16,7 @@ #include -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h" #include "ray/gcs/store_client/in_memory_store_client.h" diff --git a/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc b/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc index 9261ca5042d6..f89124b3319e 100644 --- a/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc +++ b/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc @@ -15,7 +15,7 @@ #include #include "gtest/gtest.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h" #include "ray/gcs/store_client/redis_store_client.h" diff --git a/src/ray/gcs/store_client/tests/BUILD.bazel b/src/ray/gcs/store_client/tests/BUILD.bazel index 5e48c8ef39be..b6bbb280fed8 100644 --- a/src/ray/gcs/store_client/tests/BUILD.bazel +++ b/src/ray/gcs/store_client/tests/BUILD.bazel @@ -4,7 +4,7 @@ ray_cc_library( name = "store_client_test_lib", hdrs = ["store_client_test_base.h"], deps = [ - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/gcs/store_client", ], ) @@ -114,7 +114,7 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/gcs/store_client:redis_store_client", "//src/ray/util:raii", "@com_google_googletest//:gtest_main", diff --git a/src/ray/gcs/store_client/tests/redis_async_context_test.cc b/src/ray/gcs/store_client/tests/redis_async_context_test.cc index 0bb967c55c84..605ded810aa9 100644 --- a/src/ray/gcs/store_client/tests/redis_async_context_test.cc +++ b/src/ray/gcs/store_client/tests/redis_async_context_test.cc @@ -20,7 +20,7 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/gcs/store_client/redis_context.h" #include "ray/util/logging.h" #include "ray/util/path_utils.h" diff --git a/src/ray/gcs/store_client/tests/redis_store_client_test.cc b/src/ray/gcs/store_client/tests/redis_store_client_test.cc index e3d29abaa1cb..454a2a7af2dc 100644 --- a/src/ray/gcs/store_client/tests/redis_store_client_test.cc +++ b/src/ray/gcs/store_client/tests/redis_store_client_test.cc @@ -22,7 +22,7 @@ #include #include -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/gcs/store_client/tests/store_client_test_base.h" #include "ray/util/network_util.h" #include "ray/util/path_utils.h" diff --git a/src/ray/gcs/store_client/tests/store_client_test_base.h b/src/ray/gcs/store_client/tests/store_client_test_base.h index 0495fff7e24e..558443e63450 100644 --- a/src/ray/gcs/store_client/tests/store_client_test_base.h +++ b/src/ray/gcs/store_client/tests/store_client_test_base.h @@ -26,7 +26,7 @@ #include "absl/container/flat_hash_map.h" #include "ray/common/asio/io_service_pool.h" #include "ray/common/id.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/gcs/store_client/store_client.h" #include "ray/util/logging.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/tests/BUILD.bazel b/src/ray/gcs/tests/BUILD.bazel deleted file mode 100644 index 8485c8c5a794..000000000000 --- a/src/ray/gcs/tests/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -load("//bazel:ray.bzl", "ray_cc_library") - -ray_cc_library( - name = "gcs_test_util_lib", - hdrs = [ - "gcs_test_util.h", - ], - deps = [ - "//src/ray/common:test_util", - "//src/ray/gcs:gcs_pb_util", - "//src/ray/protobuf:autoscaler_cc_grpc", - "//src/ray/protobuf:gcs_service_cc_grpc", - ], -) diff --git a/src/ray/gcs/tests/gcs_test_util.h b/src/ray/gcs/tests/gcs_test_util.h deleted file mode 100644 index cbe7486a04c2..000000000000 --- a/src/ray/gcs/tests/gcs_test_util.h +++ /dev/null @@ -1,483 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "gmock/gmock.h" -#include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/bundle_spec.h" -#include "ray/common/placement_group.h" -#include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" -#include "ray/gcs/pb_util.h" -#include "src/ray/protobuf/autoscaler.grpc.pb.h" -#include "src/ray/protobuf/gcs_service.grpc.pb.h" - -namespace ray { - -struct Mocker { - static TaskSpecification GenActorCreationTask( - const JobID &job_id, - int max_restarts, - bool detached, - const std::string &name, - const std::string &ray_namespace, - const rpc::Address &owner_address, - std::unordered_map required_resources = - std::unordered_map(), - std::unordered_map required_placement_resources = - std::unordered_map()) { - TaskSpecBuilder builder; - static rpc::JobConfig kJobConfig; - auto actor_id = ActorID::Of(job_id, RandomTaskId(), 0); - auto task_id = TaskID::ForActorCreationTask(actor_id); - FunctionDescriptor function_descriptor; - function_descriptor = FunctionDescriptorBuilder::BuildPython("", "", "", ""); - builder.SetCommonTaskSpec(task_id, - name + ":" + function_descriptor->CallString(), - Language::PYTHON, - function_descriptor, - job_id, - kJobConfig, - TaskID::Nil(), - 0, - TaskID::Nil(), - owner_address, - 1, - false, - false, - -1, - required_resources, - required_placement_resources, - "", - 0, - TaskID::Nil(), - ""); - rpc::SchedulingStrategy scheduling_strategy; - scheduling_strategy.mutable_default_scheduling_strategy(); - builder.SetActorCreationTaskSpec(actor_id, - {}, - scheduling_strategy, - max_restarts, - /*max_task_retries=*/0, - {}, - 1, - detached, - name, - ray_namespace); - return std::move(builder).ConsumeAndBuild(); - } - - static rpc::CreateActorRequest GenCreateActorRequest( - const JobID &job_id, - int max_restarts = 0, - bool detached = false, - const std::string &name = "", - const std::string &ray_namespace = "") { - rpc::Address owner_address; - owner_address.set_node_id(NodeID::FromRandom().Binary()); - owner_address.set_ip_address("1234"); - owner_address.set_port(5678); - owner_address.set_worker_id(WorkerID::FromRandom().Binary()); - auto actor_creation_task_spec = GenActorCreationTask( - job_id, max_restarts, detached, name, ray_namespace, owner_address); - rpc::CreateActorRequest request; - request.mutable_task_spec()->CopyFrom(actor_creation_task_spec.GetMessage()); - return request; - } - - static rpc::RegisterActorRequest GenRegisterActorRequest( - const JobID &job_id, - int max_restarts = 0, - bool detached = false, - const std::string &name = "", - const std::string &ray_namespace = "test") { - rpc::Address owner_address; - owner_address.set_node_id(NodeID::FromRandom().Binary()); - owner_address.set_ip_address("1234"); - owner_address.set_port(5678); - owner_address.set_worker_id(WorkerID::FromRandom().Binary()); - auto actor_creation_task_spec = GenActorCreationTask( - job_id, max_restarts, detached, name, ray_namespace, owner_address); - rpc::RegisterActorRequest request; - request.mutable_task_spec()->CopyFrom(actor_creation_task_spec.GetMessage()); - return request; - } - - static std::vector> GenBundleSpecifications( - const PlacementGroupID &placement_group_id, - absl::flat_hash_map &unit_resource, - int bundles_size = 1) { - std::vector> bundle_specs; - for (int i = 0; i < bundles_size; i++) { - rpc::Bundle bundle; - auto mutable_bundle_id = bundle.mutable_bundle_id(); - // The bundle index is start from 1. - mutable_bundle_id->set_bundle_index(i + 1); - mutable_bundle_id->set_placement_group_id(placement_group_id.Binary()); - auto mutable_unit_resources = bundle.mutable_unit_resources(); - for (auto &resource : unit_resource) { - mutable_unit_resources->insert({resource.first, resource.second}); - } - bundle_specs.emplace_back(std::make_shared(bundle)); - } - return bundle_specs; - } - - // TODO(@clay4444): Remove this once we did the batch rpc request refactor. - static BundleSpecification GenBundleCreation( - const PlacementGroupID &placement_group_id, - const int bundle_index, - absl::flat_hash_map &unit_resource) { - rpc::Bundle bundle; - auto mutable_bundle_id = bundle.mutable_bundle_id(); - mutable_bundle_id->set_bundle_index(bundle_index); - mutable_bundle_id->set_placement_group_id(placement_group_id.Binary()); - auto mutable_unit_resources = bundle.mutable_unit_resources(); - for (auto &resource : unit_resource) { - mutable_unit_resources->insert({resource.first, resource.second}); - } - return BundleSpecification(bundle); - } - - static PlacementGroupSpecification GenPlacementGroupCreation( - const std::string &name, - std::vector> &bundles, - rpc::PlacementStrategy strategy, - const JobID &job_id, - const ActorID &actor_id) { - PlacementGroupSpecBuilder builder; - - auto placement_group_id = PlacementGroupID::Of(job_id); - builder.SetPlacementGroupSpec(placement_group_id, - name, - bundles, - strategy, - /* is_detached */ false, - /* soft_target_node_id */ NodeID::Nil(), - job_id, - actor_id, - /* is_creator_detached */ false); - return builder.Build(); - } - - static rpc::CreatePlacementGroupRequest GenCreatePlacementGroupRequest( - const std::string name = "", - rpc::PlacementStrategy strategy = rpc::PlacementStrategy::SPREAD, - int bundles_count = 2, - double cpu_num = 1.0, - const JobID job_id = JobID::FromInt(1), - const ActorID &actor_id = ActorID::Nil()) { - rpc::CreatePlacementGroupRequest request; - std::vector> bundles; - std::unordered_map bundle; - bundle["CPU"] = cpu_num; - for (int index = 0; index < bundles_count; ++index) { - bundles.push_back(bundle); - } - auto placement_group_creation_spec = - GenPlacementGroupCreation(name, bundles, strategy, job_id, actor_id); - request.mutable_placement_group_spec()->CopyFrom( - placement_group_creation_spec.GetMessage()); - return request; - } - static std::shared_ptr GenNodeInfo( - uint16_t port = 0, - const std::string address = "127.0.0.1", - const std::string node_name = "Mocker_node") { - auto node = std::make_shared(); - node->set_node_id(NodeID::FromRandom().Binary()); - node->set_node_manager_port(port); - node->set_node_manager_address(address); - node->set_node_name(node_name); - node->set_instance_id("instance_x"); - node->set_state(rpc::GcsNodeInfo::ALIVE); - return node; - } - - static std::shared_ptr GenJobTableData(JobID job_id) { - auto job_table_data = std::make_shared(); - job_table_data->set_job_id(job_id.Binary()); - job_table_data->set_is_dead(false); - job_table_data->set_timestamp(current_sys_time_ms()); - job_table_data->set_driver_ip_address("127.0.0.1"); - rpc::Address address; - address.set_ip_address("127.0.0.1"); - address.set_port(1234); - address.set_node_id(UniqueID::FromRandom().Binary()); - address.set_worker_id(UniqueID::FromRandom().Binary()); - job_table_data->mutable_driver_address()->CopyFrom(address); - job_table_data->set_driver_pid(5667L); - return job_table_data; - } - - static std::shared_ptr GenActorTableData(const JobID &job_id) { - auto actor_table_data = std::make_shared(); - ActorID actor_id = ActorID::Of(job_id, RandomTaskId(), 0); - actor_table_data->set_actor_id(actor_id.Binary()); - actor_table_data->set_job_id(job_id.Binary()); - actor_table_data->set_state(rpc::ActorTableData::ALIVE); - actor_table_data->set_max_restarts(1); - actor_table_data->set_num_restarts(0); - return actor_table_data; - } - - static std::shared_ptr GenErrorTableData(const JobID &job_id) { - auto error_table_data = std::make_shared(); - error_table_data->set_job_id(job_id.Binary()); - return error_table_data; - } - - static std::shared_ptr GenWorkerTableData() { - auto worker_table_data = std::make_shared(); - worker_table_data->set_timestamp(std::time(nullptr)); - return worker_table_data; - } - - static std::shared_ptr GenAddJobRequest( - const JobID &job_id, - const std::string &ray_namespace, - const std::optional &submission_id = std::nullopt, - const std::optional &address = std::nullopt) { - auto job_config_data = std::make_shared(); - job_config_data->set_ray_namespace(ray_namespace); - - auto job_table_data = std::make_shared(); - job_table_data->set_job_id(job_id.Binary()); - job_table_data->mutable_config()->CopyFrom(*job_config_data); - if (address.has_value()) { - job_table_data->mutable_driver_address()->CopyFrom(address.value()); - } else { - rpc::Address dummy_address; - dummy_address.set_port(1234); - dummy_address.set_node_id(NodeID::FromRandom().Binary()); - dummy_address.set_ip_address("123.456.7.8"); - dummy_address.set_worker_id(WorkerID::FromRandom().Binary()); - job_table_data->mutable_driver_address()->CopyFrom(dummy_address); - } - if (submission_id.has_value()) { - job_table_data->mutable_config()->mutable_metadata()->insert( - {"job_submission_id", submission_id.value()}); - } - - auto add_job_request = std::make_shared(); - add_job_request->mutable_data()->CopyFrom(*job_table_data); - return add_job_request; - } - - static rpc::TaskEventData GenTaskEventsData( - const std::vector &task_events, - int32_t num_profile_task_events_dropped = 0, - int32_t num_status_task_events_dropped = 0) { - rpc::TaskEventData data; - for (auto &events : task_events) { - auto new_events = data.add_events_by_task(); - new_events->CopyFrom(events); - } - - for (int i = 0; i < num_status_task_events_dropped; ++i) { - rpc::TaskAttempt rpc_task_attempt; - rpc_task_attempt.set_task_id(RandomTaskId().Binary()); - rpc_task_attempt.set_attempt_number(0); - *(data.add_dropped_task_attempts()) = rpc_task_attempt; - } - - data.set_num_profile_events_dropped(num_profile_task_events_dropped); - data.set_job_id(JobID::FromInt(0).Binary()); - - return data; - } - - static rpc::events::RayEventsData GenRayEventsData( - const std::vector &task_events, - const std::vector &drop_tasks) { - rpc::events::RayEventsData data; - rpc::events::TaskEventsMetadata metadata; - for (const auto &task_attempt : drop_tasks) { - rpc::TaskAttempt rpc_task_attempt; - rpc_task_attempt.set_task_id(task_attempt.first.Binary()); - rpc_task_attempt.set_attempt_number(task_attempt.second); - *(metadata.add_dropped_task_attempts()) = rpc_task_attempt; - } - data.mutable_task_events_metadata()->CopyFrom(metadata); - for (const auto &task_event : task_events) { - rpc::events::RayEvent ray_event; - rpc::events::TaskDefinitionEvent task_definition_event; - task_definition_event.set_task_id(task_event.task_id()); - task_definition_event.set_task_attempt(task_event.attempt_number()); - task_definition_event.set_job_id(task_event.job_id()); - if (task_event.has_task_info()) { - const auto &task_info = task_event.task_info(); - task_definition_event.set_task_type(task_info.type()); - task_definition_event.set_task_name(task_info.name()); - task_definition_event.set_language(task_info.language()); - } - ray_event.set_event_id(task_event.task_id()); - ray_event.set_event_type(rpc::events::RayEvent::TASK_DEFINITION_EVENT); - ray_event.set_message("test"); - ray_event.mutable_task_definition_event()->CopyFrom(task_definition_event); - *(data.add_events()) = ray_event; - } - - return data; - } - - static rpc::TaskEventData GenTaskEventsDataLoss( - const std::vector &drop_tasks, int job_id = 0) { - rpc::TaskEventData data; - for (const auto &task_attempt : drop_tasks) { - rpc::TaskAttempt rpc_task_attempt; - rpc_task_attempt.set_task_id(task_attempt.first.Binary()); - rpc_task_attempt.set_attempt_number(task_attempt.second); - *(data.add_dropped_task_attempts()) = rpc_task_attempt; - } - data.set_job_id(JobID::FromInt(job_id).Binary()); - - return data; - } - - static rpc::ResourceDemand GenResourceDemand( - const absl::flat_hash_map &resource_demands, - int64_t num_ready_queued, - int64_t num_infeasible, - int64_t num_backlog, - const std::vector &label_selectors = {}) { - rpc::ResourceDemand resource_demand; - for (const auto &resource : resource_demands) { - (*resource_demand.mutable_shape())[resource.first] = resource.second; - } - resource_demand.set_num_ready_requests_queued(num_ready_queued); - resource_demand.set_num_infeasible_requests_queued(num_infeasible); - resource_demand.set_backlog_size(num_backlog); - for (const auto &selector : label_selectors) { - *resource_demand.add_label_selectors() = selector; - } - return resource_demand; - } - - static void FillResourcesData( - rpc::ResourcesData &resources_data, - const NodeID &node_id, - const absl::flat_hash_map &available_resources, - const absl::flat_hash_map &total_resources, - int64_t idle_ms = 0, - bool is_draining = false, - int64_t draining_deadline_timestamp_ms = -1) { - resources_data.set_node_id(node_id.Binary()); - for (const auto &resource : available_resources) { - (*resources_data.mutable_resources_available())[resource.first] = resource.second; - } - for (const auto &resource : total_resources) { - (*resources_data.mutable_resources_total())[resource.first] = resource.second; - } - resources_data.set_idle_duration_ms(idle_ms); - resources_data.set_is_draining(is_draining); - resources_data.set_draining_deadline_timestamp_ms(draining_deadline_timestamp_ms); - } - - static void FillResourcesData(rpc::ResourcesData &data, - const std::string &node_id, - std::vector demands) { - auto load_by_shape = data.mutable_resource_load_by_shape(); - auto agg_load = data.mutable_resource_load(); - for (const auto &demand : demands) { - load_by_shape->add_resource_demands()->CopyFrom(demand); - for (const auto &resource : demand.shape()) { - (*agg_load)[resource.first] += - (resource.second * (demand.num_ready_requests_queued() + - demand.num_infeasible_requests_queued())); - } - } - data.set_node_id(node_id); - } - - static std::shared_ptr GenPlacementGroupLoad( - std::vector placement_group_table_data_vec) { - auto placement_group_load = std::make_shared(); - for (auto &placement_group_table_data : placement_group_table_data_vec) { - placement_group_load->add_placement_group_data()->CopyFrom( - placement_group_table_data); - } - return placement_group_load; - } - - static rpc::PlacementGroupTableData GenPlacementGroupTableData( - const PlacementGroupID &placement_group_id, - const JobID &job_id, - const std::vector> &bundles, - const std::vector &nodes, - rpc::PlacementStrategy strategy, - const rpc::PlacementGroupTableData::PlacementGroupState state, - const std::string &name = "", - const ActorID &actor_id = ActorID::Nil()) { - rpc::PlacementGroupTableData placement_group_table_data; - placement_group_table_data.set_placement_group_id(placement_group_id.Binary()); - placement_group_table_data.set_state(state); - placement_group_table_data.set_name(name); - placement_group_table_data.set_strategy(strategy); - RAY_CHECK(bundles.size() == nodes.size()); - size_t i = 0; - for (auto &bundle : bundles) { - // Add unit resources - auto bundle_spec = placement_group_table_data.add_bundles(); - for (auto &resource : bundle) { - (*bundle_spec->mutable_unit_resources())[resource.first] = resource.second; - } - - // Add node id - const auto &node = nodes[i]; - if (!node.empty()) { - bundle_spec->set_node_id(node); - } - - i++; - } - return placement_group_table_data; - } - static rpc::autoscaler::ClusterResourceConstraint GenClusterResourcesConstraint( - const std::vector> &request_resources, - const std::vector &count_array) { - rpc::autoscaler::ClusterResourceConstraint constraint; - RAY_CHECK(request_resources.size() == count_array.size()); - for (size_t i = 0; i < request_resources.size(); i++) { - auto &resource = request_resources[i]; - auto count = count_array[i]; - auto bundle = constraint.add_resource_requests(); - bundle->set_count(count); - bundle->mutable_request()->mutable_resources_bundle()->insert(resource.begin(), - resource.end()); - } - return constraint; - } - // Read all lines of a file into vector vc - static void ReadContentFromFile(std::vector &vc, std::string log_file) { - std::string line; - std::ifstream read_file; - read_file.open(log_file, std::ios::binary); - while (std::getline(read_file, line)) { - vc.push_back(line); - } - read_file.close(); - } -}; - -} // namespace ray diff --git a/src/ray/raylet/scheduling/tests/BUILD.bazel b/src/ray/raylet/scheduling/tests/BUILD.bazel index 661e52007f1f..fa88ef1be1cb 100644 --- a/src/ray/raylet/scheduling/tests/BUILD.bazel +++ b/src/ray/raylet/scheduling/tests/BUILD.bazel @@ -11,7 +11,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:lease", "//src/ray/common:ray_config", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "@com_google_googletest//:gtest_main", @@ -58,7 +58,7 @@ ray_cc_test( "//src/ray/common:id", "//src/ray/common:lease", "//src/ray/common:task_common", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/raylet:local_lease_manager", "//src/ray/raylet/scheduling:cluster_lease_manager", "//src/ray/raylet/scheduling:cluster_resource_scheduler", diff --git a/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc b/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc index 8d6c3db5890f..3347785c7407 100644 --- a/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc +++ b/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc @@ -31,7 +31,7 @@ #include "ray/common/scheduling/scheduling_ids.h" #include "ray/common/lease/lease.h" #include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/raylet/local_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/tests/util.h" diff --git a/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc b/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc index f4b0d92bdaed..3237971f00de 100644 --- a/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc +++ b/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc @@ -25,7 +25,7 @@ #include "gtest/gtest.h" #include "ray/common/ray_config.h" #include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/common/scheduling/resource_set.h" #include "ray/common/scheduling/scheduling_ids.h" #include "mock/ray/gcs/gcs_client/gcs_client.h" diff --git a/src/ray/raylet/tests/BUILD.bazel b/src/ray/raylet/tests/BUILD.bazel index f0b0cbc877ff..25321d7d3fab 100644 --- a/src/ray/raylet/tests/BUILD.bazel +++ b/src/ray/raylet/tests/BUILD.bazel @@ -70,7 +70,6 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:id", - "//src/ray/gcs/tests:gcs_test_util_lib", "//src/ray/raylet:placement_group_resource_manager", "@com_google_googletest//:gtest_main", ], @@ -102,7 +101,7 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:lease", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/raylet:lease_dependency_manager", "@com_google_googletest//:gtest_main", ], @@ -119,7 +118,7 @@ ray_cc_test( "//src/ray/common:id", "//src/ray/common:lease", "//src/ray/common:task_common", - "//src/ray/common:test_util", + "//src/ray/common:test_utils", "//src/ray/raylet:local_lease_manager", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "@com_google_googletest//:gtest_main", diff --git a/src/ray/raylet/tests/lease_dependency_manager_test.cc b/src/ray/raylet/tests/lease_dependency_manager_test.cc index d240d45566db..906472c71d46 100644 --- a/src/ray/raylet/tests/lease_dependency_manager_test.cc +++ b/src/ray/raylet/tests/lease_dependency_manager_test.cc @@ -23,7 +23,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "mock/ray/object_manager/object_manager.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" namespace ray { diff --git a/src/ray/raylet/tests/local_lease_manager_test.cc b/src/ray/raylet/tests/local_lease_manager_test.cc index d877762a9dc0..0d4a829785f0 100644 --- a/src/ray/raylet/tests/local_lease_manager_test.cc +++ b/src/ray/raylet/tests/local_lease_manager_test.cc @@ -30,7 +30,7 @@ #include "ray/common/id.h" #include "ray/common/lease/lease.h" #include "ray/common/task/task_util.h" -#include "ray/common/test_util.h" +#include "ray/common/test_utils.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/tests/util.h" diff --git a/src/ray/raylet/tests/placement_group_resource_manager_test.cc b/src/ray/raylet/tests/placement_group_resource_manager_test.cc index 9247c75ffb7c..c37fdc477cc0 100644 --- a/src/ray/raylet/tests/placement_group_resource_manager_test.cc +++ b/src/ray/raylet/tests/placement_group_resource_manager_test.cc @@ -12,24 +12,60 @@ // See the License for the specific language governing permissions and // limitations under the License. -// clang-format off #include "ray/raylet/placement_group_resource_manager.h" #include -#include -#include #include +#include +#include #include "gtest/gtest.h" +#include "mock/ray/gcs/gcs_client/gcs_client.h" #include "ray/common/bundle_spec.h" #include "ray/common/id.h" #include "ray/common/scheduling/resource_set.h" -#include "ray/gcs/tests/gcs_test_util.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" -// clang-format on namespace ray { +namespace { + +BundleSpecification GenBundleCreation( + const PlacementGroupID &placement_group_id, + const int bundle_index, + const absl::flat_hash_map &unit_resource) { + rpc::Bundle bundle; + auto mutable_bundle_id = bundle.mutable_bundle_id(); + mutable_bundle_id->set_bundle_index(bundle_index); + mutable_bundle_id->set_placement_group_id(placement_group_id.Binary()); + auto mutable_unit_resources = bundle.mutable_unit_resources(); + for (auto &resource : unit_resource) { + mutable_unit_resources->insert({resource.first, resource.second}); + } + return BundleSpecification(bundle); +} + +std::vector> GenBundleSpecifications( + const PlacementGroupID &placement_group_id, + const absl::flat_hash_map &unit_resource, + int bundles_size = 1) { + std::vector> bundle_specs; + for (int i = 0; i < bundles_size; i++) { + rpc::Bundle bundle; + auto mutable_bundle_id = bundle.mutable_bundle_id(); + // The bundle index is start from 1. + mutable_bundle_id->set_bundle_index(i + 1); + mutable_bundle_id->set_placement_group_id(placement_group_id.Binary()); + auto mutable_unit_resources = bundle.mutable_unit_resources(); + for (auto &resource : unit_resource) { + mutable_unit_resources->insert({resource.first, resource.second}); + } + bundle_specs.emplace_back(std::make_shared(bundle)); + } + return bundle_specs; +} + +} // namespace + class NewPlacementGroupResourceManagerTest : public ::testing::Test { public: instrumented_io_context io_context; @@ -150,7 +186,7 @@ TEST_F(NewPlacementGroupResourceManagerTest, TestNewPrepareBundleResource) { auto group_id = PlacementGroupID::Of(JobID::FromInt(1)); absl::flat_hash_map unit_resource; unit_resource.insert({"CPU", 1.0}); - auto bundle_specs = Mocker::GenBundleSpecifications(group_id, unit_resource, 1); + auto bundle_specs = GenBundleSpecifications(group_id, unit_resource, 1); /// 2. init local available resource. InitLocalAvailableResource(unit_resource); /// 3. prepare bundle resource. @@ -165,7 +201,7 @@ TEST_F(NewPlacementGroupResourceManagerTest, auto group_id = PlacementGroupID::Of(JobID::FromInt(1)); absl::flat_hash_map unit_resource; unit_resource.insert({"CPU", 2.0}); - auto bundle_specs = Mocker::GenBundleSpecifications(group_id, unit_resource, 1); + auto bundle_specs = GenBundleSpecifications(group_id, unit_resource, 1); /// 2. init local available resource. absl::flat_hash_map init_unit_resource; init_unit_resource.insert({"CPU", 1.0}); @@ -179,9 +215,9 @@ TEST_F(NewPlacementGroupResourceManagerTest, TestNewPrepareBundleDuringDraining) absl::flat_hash_map unit_resource; unit_resource.insert({"CPU", 1.0}); auto group1_id = PlacementGroupID::Of(JobID::FromInt(1)); - auto bundle1_specs = Mocker::GenBundleSpecifications(group1_id, unit_resource, 1); + auto bundle1_specs = GenBundleSpecifications(group1_id, unit_resource, 1); auto group2_id = PlacementGroupID::Of(JobID::FromInt(2)); - auto bundle2_specs = Mocker::GenBundleSpecifications(group2_id, unit_resource, 1); + auto bundle2_specs = GenBundleSpecifications(group2_id, unit_resource, 1); /// 2. init local available resource. absl::flat_hash_map init_unit_resource; init_unit_resource.insert({"CPU", 2.0}); @@ -218,7 +254,7 @@ TEST_F(NewPlacementGroupResourceManagerTest, TestNewCommitBundleResource) { auto group_id = PlacementGroupID::Of(JobID::FromInt(1)); absl::flat_hash_map unit_resource; unit_resource.insert({"CPU", 1.0}); - auto bundle_specs = Mocker::GenBundleSpecifications(group_id, unit_resource, 1); + auto bundle_specs = GenBundleSpecifications(group_id, unit_resource, 1); /// 2. init local available resource. InitLocalAvailableResource(unit_resource); /// 3. prepare and commit bundle resource. @@ -247,7 +283,7 @@ TEST_F(NewPlacementGroupResourceManagerTest, TestNewReturnBundleResource) { auto group_id = PlacementGroupID::Of(JobID::FromInt(1)); absl::flat_hash_map unit_resource; unit_resource.insert({"CPU", 1.0}); - auto bundle_spec = Mocker::GenBundleCreation(group_id, 1, unit_resource); + auto bundle_spec = GenBundleCreation(group_id, 1, unit_resource); /// 2. init local available resource. InitLocalAvailableResource(unit_resource); /// 3. prepare and commit bundle resource. @@ -268,8 +304,8 @@ TEST_F(NewPlacementGroupResourceManagerTest, TestNewMultipleBundlesCommitAndRetu auto group_id = PlacementGroupID::Of(JobID::FromInt(1)); absl::flat_hash_map unit_resource; unit_resource.insert({"CPU", 1.0}); - auto first_bundle_spec = Mocker::GenBundleCreation(group_id, 1, unit_resource); - auto second_bundle_spec = Mocker::GenBundleCreation(group_id, 2, unit_resource); + auto first_bundle_spec = GenBundleCreation(group_id, 1, unit_resource); + auto second_bundle_spec = GenBundleCreation(group_id, 2, unit_resource); /// 2. init local available resource. absl::flat_hash_map init_unit_resource; init_unit_resource.insert({"CPU", 2.0}); @@ -335,7 +371,7 @@ TEST_F(NewPlacementGroupResourceManagerTest, TestNewIdempotencyWithMultiPrepare) auto group_id = PlacementGroupID::Of(JobID::FromInt(1)); absl::flat_hash_map unit_resource; unit_resource.insert({"CPU", 1.0}); - auto bundle_specs = Mocker::GenBundleSpecifications(group_id, unit_resource, 1); + auto bundle_specs = GenBundleSpecifications(group_id, unit_resource, 1); /// 2. init local available resource. absl::flat_hash_map available_resource = { std::make_pair("CPU", 3.0)}; @@ -357,7 +393,7 @@ TEST_F(NewPlacementGroupResourceManagerTest, TestNewIdempotencyWithRandomOrder) auto group_id = PlacementGroupID::Of(JobID::FromInt(1)); absl::flat_hash_map unit_resource; unit_resource.insert({"CPU", 1.0}); - auto bundle_spec = Mocker::GenBundleCreation(group_id, 1, unit_resource); + auto bundle_spec = GenBundleCreation(group_id, 1, unit_resource); /// 2. init local available resource. absl::flat_hash_map available_resource = { std::make_pair("CPU", 3.0)}; @@ -413,7 +449,7 @@ TEST_F(NewPlacementGroupResourceManagerTest, TestPreparedResourceBatched) { auto group_id = PlacementGroupID::Of(JobID::FromInt(1)); absl::flat_hash_map unit_resource; unit_resource.insert({"CPU", 1.0}); - auto bundle_specs = Mocker::GenBundleSpecifications(group_id, unit_resource, 4); + auto bundle_specs = GenBundleSpecifications(group_id, unit_resource, 4); // 2. init local available resource with 3 CPUs. absl::flat_hash_map available_resource = { std::make_pair("CPU", 3.0)}; @@ -472,7 +508,7 @@ TEST_F(NewPlacementGroupResourceManagerTest, TestCommiteResourceBatched) { auto group_id = PlacementGroupID::Of(JobID::FromInt(1)); absl::flat_hash_map unit_resource; unit_resource.insert({"GPU", 2.0}); - auto bundle_specs = Mocker::GenBundleSpecifications(group_id, unit_resource, 4); + auto bundle_specs = GenBundleSpecifications(group_id, unit_resource, 4); // 2. init local available resource with 4 CPUs. absl::flat_hash_map available_resource = { std::make_pair("GPU", 10.0)}; @@ -520,7 +556,7 @@ TEST_F(NewPlacementGroupResourceManagerTest, TestNewReturnBundleFailure) { auto group_id = PlacementGroupID::Of(JobID::FromInt(1)); absl::flat_hash_map unit_resource; unit_resource.insert({"CPU", 1.0}); - auto bundle_spec = Mocker::GenBundleCreation(group_id, 1, unit_resource); + auto bundle_spec = GenBundleCreation(group_id, 1, unit_resource); /// init local available resource. InitLocalAvailableResource(unit_resource); /// prepare and commit bundle resource. From 327bb284d52bd721e226a404a8f363f8fb65d9fc Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 2 Sep 2025 22:47:02 +0530 Subject: [PATCH 385/634] Enable ruff lint for `python/ray/tests/` (#56079) Signed-off-by: czgdp1807 --- .pre-commit-config.yaml | 2 +- pyproject.toml | 1 - python/ray/tests/accelerators/mock_pynvml.py | 3 +- .../tests/accelerators/test_accelerators.py | 1 + python/ray/tests/accelerators/test_amd_gpu.py | 9 ++- python/ray/tests/accelerators/test_hpu.py | 5 +- .../ray/tests/accelerators/test_intel_gpu.py | 11 ++- python/ray/tests/accelerators/test_neuron.py | 5 +- python/ray/tests/accelerators/test_npu.py | 3 +- .../ray/tests/accelerators/test_nvidia_gpu.py | 1 + python/ray/tests/accelerators/test_rbln.py | 7 +- python/ray/tests/accelerators/test_tpu.py | 6 +- python/ray/tests/autoscaler/test_providers.py | 8 +- python/ray/tests/autoscaler/util.py | 1 + python/ray/tests/autoscaler_test_utils.py | 2 +- python/ray/tests/aws/conftest.py | 5 +- .../tests/aws/test_aws_batch_tag_update.py | 3 +- python/ray/tests/aws/utils/constants.py | 4 +- python/ray/tests/aws/utils/helpers.py | 15 ++-- python/ray/tests/aws/utils/stubs.py | 33 ++++---- python/ray/tests/chaos/potato_passer.py | 3 +- python/ray/tests/chaos/streaming_llm.py | 4 +- python/ray/tests/conftest.py | 27 ++++--- python/ray/tests/conftest_docker.py | 10 ++- .../ray/tests/gcp/test_gcp_node_provider.py | 25 +++--- .../tests/gcp/test_gcp_tpu_command_runner.py | 6 +- python/ray/tests/horovod/horovod_example.py | 7 +- python/ray/tests/horovod/test_horovod.py | 7 +- .../scripts/non_terminated_nodes_count.py | 2 +- .../tests/kuberay/scripts/scale_up_custom.py | 3 +- .../tests/kuberay/test_autoscaling_config.py | 12 +-- .../ray/tests/kuberay/test_autoscaling_e2e.py | 8 +- .../kuberay/test_kuberay_node_provider.py | 17 ++-- python/ray/tests/kuberay/utils.py | 4 +- python/ray/tests/ludwig/ludwig_test_utils.py | 6 +- python/ray/tests/ludwig/test_ludwig.py | 29 ++++--- python/ray/tests/mock_s3_server.py | 5 +- python/ray/tests/modin/modin_test_utils.py | 8 +- python/ray/tests/modin/test_modin.py | 9 ++- .../tests/runtime_env_container/test_job.py | 2 +- .../test_log_file_exists.py | 9 ++- .../runtime_env_container/test_put_get.py | 6 +- .../runtime_env_container/test_serve_basic.py | 1 + .../test_serve_telemetry.py | 6 +- .../test_shared_memory.py | 7 +- ...xit_intended_system_exit_and_user_error.py | 6 +- python/ray/tests/spark/test_GPU.py | 17 ++-- python/ray/tests/spark/test_basic.py | 27 +++---- .../ray/tests/spark/test_databricks_hook.py | 10 +-- .../tests/spark/test_multicores_per_task.py | 4 +- python/ray/tests/spark/test_utils.py | 15 ++-- python/ray/tests/test_actor.py | 11 +-- python/ray/tests/test_actor_advanced.py | 6 +- .../ray/tests/test_actor_bounded_threads.py | 10 +-- python/ray/tests/test_actor_cancel.py | 5 +- python/ray/tests/test_actor_failures.py | 10 +-- python/ray/tests/test_actor_lifetime.py | 6 +- .../test_actor_lineage_reconstruction.py | 5 +- python/ray/tests/test_actor_pool.py | 1 + python/ray/tests/test_actor_retry_2.py | 2 +- python/ray/tests/test_actor_state_metrics.py | 5 +- python/ray/tests/test_advanced.py | 6 +- python/ray/tests/test_advanced_2.py | 2 +- python/ray/tests/test_advanced_3.py | 5 +- python/ray/tests/test_advanced_4.py | 2 +- python/ray/tests/test_advanced_6.py | 5 +- python/ray/tests/test_advanced_7.py | 2 +- python/ray/tests/test_advanced_8.py | 5 +- python/ray/tests/test_advanced_9.py | 15 ++-- python/ray/tests/test_annotations.py | 2 +- python/ray/tests/test_async.py | 3 +- python/ray/tests/test_asyncio.py | 2 +- python/ray/tests/test_asyncio_cluster.py | 2 +- python/ray/tests/test_autoscaler.py | 22 +++-- .../tests/test_autoscaler_drain_node_api.py | 4 +- python/ray/tests/test_autoscaler_e2e.py | 6 +- .../tests/test_autoscaler_fake_multinode.py | 5 +- python/ray/tests/test_autoscaler_util.py | 2 +- python/ray/tests/test_autoscaler_yaml.py | 2 +- python/ray/tests/test_autoscaling_policy.py | 41 +++++----- .../ray/tests/test_baseexceptionandgroup.py | 5 +- python/ray/tests/test_basic.py | 3 +- python/ray/tests/test_basic_5.py | 8 +- .../test_batch_node_provider_integration.py | 11 +-- .../tests/test_batch_node_provider_unit.py | 24 +++--- python/ray/tests/test_bounded_unix_sockets.py | 7 +- .../ray/tests/test_bundle_label_selector.py | 5 +- python/ray/tests/test_cancel.py | 13 ++- python/ray/tests/test_channel.py | 8 +- .../ray/tests/test_channel_serialization.py | 5 +- python/ray/tests/test_chaos.py | 18 ++--- python/ray/tests/test_cli.py | 10 +-- python/ray/tests/test_client.py | 4 +- python/ray/tests/test_client_builder.py | 2 +- python/ray/tests/test_client_init.py | 12 ++- python/ray/tests/test_client_metadata.py | 3 +- python/ray/tests/test_client_multi.py | 2 + python/ray/tests/test_client_proxy.py | 6 +- python/ray/tests/test_client_reconnect.py | 16 ++-- python/ray/tests/test_client_warnings.py | 2 +- python/ray/tests/test_command_runner.py | 6 +- python/ray/tests/test_component_failures_2.py | 2 +- python/ray/tests/test_component_failures_3.py | 2 +- python/ray/tests/test_concurrency_group.py | 2 +- python/ray/tests/test_coordinator_server.py | 26 +++--- .../tests/test_core_worker_fault_tolerance.py | 3 +- python/ray/tests/test_cross_language.py | 3 +- python/ray/tests/test_dashboard.py | 5 +- python/ray/tests/test_dashboard_profiler.py | 7 +- python/ray/tests/test_debug_tools.py | 2 +- python/ray/tests/test_distributed_sort.py | 3 +- python/ray/tests/test_draining.py | 7 +- python/ray/tests/test_exit_observability.py | 2 +- .../ray/tests/test_experimental_collective.py | 4 +- python/ray/tests/test_failure.py | 6 +- python/ray/tests/test_failure_2.py | 4 +- python/ray/tests/test_failure_3.py | 15 ++-- python/ray/tests/test_failure_4.py | 11 +-- python/ray/tests/test_gcs_fault_tolerance.py | 23 +++--- python/ray/tests/test_gcs_ha_e2e_2.py | 4 +- python/ray/tests/test_gcs_pubsub.py | 5 +- python/ray/tests/test_gcs_utils.py | 8 +- python/ray/tests/test_generators.py | 15 ++-- python/ray/tests/test_get_or_create_actor.py | 3 +- python/ray/tests/test_global_gc.py | 2 +- python/ray/tests/test_global_state.py | 8 +- python/ray/tests/test_gpu_objects_gloo.py | 15 ++-- python/ray/tests/test_gpu_objects_nccl.py | 4 +- python/ray/tests/test_gpu_objects_nixl.py | 4 +- .../ray/tests/test_grpc_client_credentials.py | 2 +- python/ray/tests/test_healthcheck.py | 3 +- python/ray/tests/test_ids.py | 16 ++-- python/ray/tests/test_iter.py | 11 +-- python/ray/tests/test_job.py | 17 ++-- python/ray/tests/test_joblib.py | 24 +++--- .../ray/tests/test_kill_raylet_signal_log.py | 5 +- python/ray/tests/test_kill_subprocesses.py | 15 ++-- python/ray/tests/test_label_utils.py | 8 +- python/ray/tests/test_list_actors.py | 3 +- python/ray/tests/test_list_actors_2.py | 3 +- python/ray/tests/test_list_actors_3.py | 3 +- python/ray/tests/test_list_actors_4.py | 3 +- python/ray/tests/test_logging.py | 26 +++--- python/ray/tests/test_logging_2.py | 9 ++- python/ray/tests/test_memory_deadlock.py | 5 +- python/ray/tests/test_memory_pressure.py | 11 +-- python/ray/tests/test_memory_scheduling.py | 2 +- python/ray/tests/test_memstat.py | 3 +- python/ray/tests/test_metrics.py | 9 ++- python/ray/tests/test_metrics_agent.py | 42 +++++----- python/ray/tests/test_metrics_agent_2.py | 45 +++++------ python/ray/tests/test_metrics_head.py | 8 +- python/ray/tests/test_minimal_install.py | 7 +- python/ray/tests/test_mpi.py | 11 ++- python/ray/tests/test_multi_node.py | 3 +- python/ray/tests/test_multi_node_3.py | 7 +- python/ray/tests/test_multi_tenancy.py | 2 +- python/ray/tests/test_multinode_failures.py | 2 +- python/ray/tests/test_multiprocessing.py | 5 +- .../tests/test_multiprocessing_standalone.py | 3 +- python/ray/tests/test_nccl_channel.py | 10 +-- python/ray/tests/test_network_failure_e2e.py | 8 +- python/ray/tests/test_node_death.py | 2 +- .../test_node_label_scheduling_strategy.py | 7 +- python/ray/tests/test_node_labels.py | 9 ++- python/ray/tests/test_node_manager.py | 9 +-- ...test_node_provider_availability_tracker.py | 10 +-- python/ray/tests/test_numba.py | 5 +- python/ray/tests/test_object_assign_owner.py | 2 +- python/ray/tests/test_object_spilling.py | 18 ++--- python/ray/tests/test_object_spilling_2.py | 5 +- python/ray/tests/test_object_store_metrics.py | 4 +- .../test_open_telemetry_metric_recorder.py | 4 +- python/ray/tests/test_placement_group.py | 12 +-- python/ray/tests/test_placement_group_2.py | 2 +- python/ray/tests/test_placement_group_3.py | 6 +- python/ray/tests/test_placement_group_4.py | 9 ++- python/ray/tests/test_placement_group_5.py | 16 ++-- .../tests/test_placement_group_failover.py | 10 ++- .../test_placement_group_mini_integration.py | 4 +- python/ray/tests/test_plasma_unlimited.py | 11 +-- .../ray/tests/test_pydantic_serialization.py | 24 +++--- python/ray/tests/test_queue.py | 4 +- python/ray/tests/test_ray_debugger.py | 6 +- python/ray/tests/test_ray_init.py | 14 ++-- python/ray/tests/test_ray_init_2.py | 12 +-- python/ray/tests/test_ray_shutdown.py | 16 ++-- python/ray/tests/test_reconstruction_2.py | 7 +- .../tests/test_reconstruction_stress_spill.py | 2 +- python/ray/tests/test_redis_tls.py | 4 +- python/ray/tests/test_reference_counting.py | 2 +- python/ray/tests/test_reference_counting_2.py | 6 +- .../test_reference_counting_standalone.py | 4 +- .../tests/test_resource_demand_scheduler.py | 20 ++--- .../tests/test_resource_isolation_config.py | 3 +- python/ray/tests/test_resource_metrics.py | 4 +- python/ray/tests/test_response_cache.py | 6 +- python/ray/tests/test_runtime_context.py | 4 +- python/ray/tests/test_runtime_env_agent.py | 8 +- .../ray/tests/test_runtime_env_complicated.py | 19 +++-- .../tests/test_runtime_env_conda_and_pip.py | 28 +++---- .../tests/test_runtime_env_conda_and_pip_2.py | 5 +- .../tests/test_runtime_env_conda_and_pip_3.py | 6 +- .../tests/test_runtime_env_conda_and_pip_4.py | 6 +- .../tests/test_runtime_env_conda_and_pip_5.py | 1 + .../ray/tests/test_runtime_env_container.py | 3 +- python/ray/tests/test_runtime_env_failure.py | 3 +- .../tests/test_runtime_env_fork_process.py | 2 +- .../tests/test_runtime_env_get_wheel_names.py | 3 +- .../ray/tests/test_runtime_env_packaging.py | 10 +-- python/ray/tests/test_runtime_env_profiler.py | 7 +- .../tests/test_runtime_env_py_executable.py | 3 +- .../ray/tests/test_runtime_env_ray_minimal.py | 1 + .../ray/tests/test_runtime_env_setup_func.py | 8 +- .../ray/tests/test_runtime_env_standalone.py | 2 +- .../ray/tests/test_runtime_env_strong_type.py | 7 +- python/ray/tests/test_runtime_env_uv.py | 5 +- python/ray/tests/test_runtime_env_uv_run.py | 3 +- .../ray/tests/test_runtime_env_working_dir.py | 6 +- .../tests/test_runtime_env_working_dir_2.py | 15 ++-- .../tests/test_runtime_env_working_dir_3.py | 8 +- .../tests/test_runtime_env_working_dir_4.py | 4 +- python/ray/tests/test_scheduling.py | 14 ++-- python/ray/tests/test_scheduling_2.py | 6 +- python/ray/tests/test_shuffle.py | 5 +- python/ray/tests/test_state_api.py | 81 +++++++++---------- python/ray/tests/test_state_api_2.py | 15 ++-- python/ray/tests/test_state_api_log.py | 26 +++--- python/ray/tests/test_state_api_summary.py | 41 +++++----- python/ray/tests/test_streaming_generator.py | 12 +-- .../ray/tests/test_streaming_generator_2.py | 11 +-- .../ray/tests/test_streaming_generator_3.py | 6 +- .../ray/tests/test_streaming_generator_4.py | 13 +-- .../test_streaming_generator_backpressure.py | 9 ++- python/ray/tests/test_stress.py | 4 +- python/ray/tests/test_symmetric_run.py | 8 +- python/ray/tests/test_task_events.py | 17 ++-- python/ray/tests/test_task_events_2.py | 18 +++-- python/ray/tests/test_task_events_3.py | 3 +- python/ray/tests/test_task_metrics.py | 1 - .../tests/test_task_metrics_reconstruction.py | 3 +- python/ray/tests/test_tls_auth.py | 2 +- python/ray/tests/test_top_level_api.py | 2 +- python/ray/tests/test_tqdm.py | 2 +- python/ray/tests/test_traceback.py | 2 +- python/ray/tests/test_typing.py | 1 - python/ray/tests/test_unavailable_actors.py | 10 +-- python/ray/tests/test_util_helpers.py | 6 +- python/ray/tests/test_utils.py | 7 +- python/ray/tests/test_wait.py | 8 +- python/ray/tests/test_widgets.py | 2 +- python/ray/tests/test_worker_capping.py | 3 +- .../tests/test_worker_graceful_shutdown.py | 3 +- python/ray/tests/test_worker_state.py | 4 +- .../tests/typing_files/check_typing_good.py | 3 +- .../unit/test_node_affinity_validation.py | 3 +- .../unit/test_resource_and_label_spec.py | 8 +- python/ray/tests/unit/test_runtime_env.py | 13 ++- python/ray/tests/unit/test_runtime_env_uv.py | 7 +- .../tests/unit/test_runtime_env_validation.py | 18 ++--- .../tests/vsphere/test_cluster_operator.py | 11 ++- .../tests/vsphere/test_vmray_node_provider.py | 5 +- 262 files changed, 1122 insertions(+), 1054 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 28358c2a9218..189a39b5d48e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: args: [ --fix, --exit-non-zero-on-fix ] - id: ruff args: [ --select, "I", --fix, --exit-non-zero-on-fix ] - files: '^python/ray/serve/|^python/ray/train|^python/ray/data|^python/ray/_private/|^python/ray/llm/|^python/ray/tune/|^python/ray/includes/|^python/ray/internal/|^python/ray/ray_operator/|^python/ray/scripts/|^python/ray/streaming/|^python/ray/dag/' + files: '^python/ray/serve/|^python/ray/train|^python/ray/data|^python/ray/_private/|^python/ray/llm/|^python/ray/tune/|^python/ray/includes/|^python/ray/internal/|^python/ray/ray_operator/|^python/ray/scripts/|^python/ray/streaming/|^python/ray/dag/|^python/ray/tests/' - repo: https://github.com/jsh9/pydoclint rev: "0.6.6" diff --git a/pyproject.toml b/pyproject.toml index 765fd83f3423..70e1ee4249aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,6 @@ afterray = ["psutil", "setproctitle"] "python/ray/setup-dev.py" = ["I"] "python/ray/cloudpickle/*" = ["I"] "python/ray/dag/__init__.py" = ["I"] -"python/ray/tests/*" = ["I"] "python/ray/util/*" = ["I"] "python/ray/workers/*" = ["I"] "python/ray/workflow/*" = ["I"] diff --git a/python/ray/tests/accelerators/mock_pynvml.py b/python/ray/tests/accelerators/mock_pynvml.py index 6240961aea4b..24079a456d52 100644 --- a/python/ray/tests/accelerators/mock_pynvml.py +++ b/python/ray/tests/accelerators/mock_pynvml.py @@ -1,7 +1,8 @@ -import pytest from typing import List from unittest.mock import patch +import pytest + import ray._private.thirdparty.pynvml as pynvml diff --git a/python/ray/tests/accelerators/test_accelerators.py b/python/ray/tests/accelerators/test_accelerators.py index 80c1ef6ebf57..ac79765e88a7 100644 --- a/python/ray/tests/accelerators/test_accelerators.py +++ b/python/ray/tests/accelerators/test_accelerators.py @@ -1,4 +1,5 @@ import sys + import pytest from ray.util import accelerators diff --git a/python/ray/tests/accelerators/test_amd_gpu.py b/python/ray/tests/accelerators/test_amd_gpu.py index 1449d392b5b3..a1b13e575713 100644 --- a/python/ray/tests/accelerators/test_amd_gpu.py +++ b/python/ray/tests/accelerators/test_amd_gpu.py @@ -1,11 +1,14 @@ import os import sys -import pytest from unittest.mock import patch +import pytest + import ray -from ray._private.accelerators import AMDGPUAcceleratorManager -from ray._private.accelerators import get_accelerator_manager_for_resource +from ray._private.accelerators import ( + AMDGPUAcceleratorManager, + get_accelerator_manager_for_resource, +) @pytest.mark.parametrize( diff --git a/python/ray/tests/accelerators/test_hpu.py b/python/ray/tests/accelerators/test_hpu.py index e1a359051409..f6665c3001ed 100644 --- a/python/ray/tests/accelerators/test_hpu.py +++ b/python/ray/tests/accelerators/test_hpu.py @@ -1,10 +1,11 @@ import os import sys -import pytest from unittest.mock import patch +import pytest + import ray -from ray._private.accelerators import hpu, HPUAcceleratorManager +from ray._private.accelerators import HPUAcceleratorManager, hpu from ray.util.placement_group import placement_group from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy diff --git a/python/ray/tests/accelerators/test_intel_gpu.py b/python/ray/tests/accelerators/test_intel_gpu.py index 93dc8843bbdf..b74dd5296265 100644 --- a/python/ray/tests/accelerators/test_intel_gpu.py +++ b/python/ray/tests/accelerators/test_intel_gpu.py @@ -1,12 +1,15 @@ import os import sys -import pytest from unittest.mock import patch +import pytest + import ray -from ray._private.accelerators import IntelGPUAcceleratorManager as Accelerator -from ray._private.accelerators import get_accelerator_manager_for_resource -from ray.util.accelerators import INTEL_MAX_1550, INTEL_MAX_1100 +from ray._private.accelerators import ( + IntelGPUAcceleratorManager as Accelerator, + get_accelerator_manager_for_resource, +) +from ray.util.accelerators import INTEL_MAX_1100, INTEL_MAX_1550 def test_visible_intel_gpu_ids(shutdown_only): diff --git a/python/ray/tests/accelerators/test_neuron.py b/python/ray/tests/accelerators/test_neuron.py index 75443ec4ae11..19ba76d3d3e3 100644 --- a/python/ray/tests/accelerators/test_neuron.py +++ b/python/ray/tests/accelerators/test_neuron.py @@ -1,8 +1,9 @@ -import sys import subprocess -import pytest +import sys from unittest.mock import patch +import pytest + import ray from ray._private.accelerators import NeuronAcceleratorManager diff --git a/python/ray/tests/accelerators/test_npu.py b/python/ray/tests/accelerators/test_npu.py index 5c79d4d9c185..51cae14422b5 100644 --- a/python/ray/tests/accelerators/test_npu.py +++ b/python/ray/tests/accelerators/test_npu.py @@ -1,8 +1,9 @@ import os import sys -import pytest from unittest.mock import patch +import pytest + import ray from ray._private.accelerators import NPUAcceleratorManager as Accelerator diff --git a/python/ray/tests/accelerators/test_nvidia_gpu.py b/python/ray/tests/accelerators/test_nvidia_gpu.py index 035a866bfcbf..10c2065d3066 100644 --- a/python/ray/tests/accelerators/test_nvidia_gpu.py +++ b/python/ray/tests/accelerators/test_nvidia_gpu.py @@ -1,4 +1,5 @@ import sys + import pytest from ray._private.accelerators import NvidiaGPUAcceleratorManager diff --git a/python/ray/tests/accelerators/test_rbln.py b/python/ray/tests/accelerators/test_rbln.py index fa98927c346f..37865bff1392 100644 --- a/python/ray/tests/accelerators/test_rbln.py +++ b/python/ray/tests/accelerators/test_rbln.py @@ -1,11 +1,12 @@ -import pytest import os import sys +import pytest + from ray._private.accelerators.rbln import ( - RBLNAcceleratorManager, - RBLN_RT_VISIBLE_DEVICES_ENV_VAR, NOSET_RBLN_RT_VISIBLE_DEVICES_ENV_VAR, + RBLN_RT_VISIBLE_DEVICES_ENV_VAR, + RBLNAcceleratorManager, ) diff --git a/python/ray/tests/accelerators/test_tpu.py b/python/ray/tests/accelerators/test_tpu.py index f13f27ffea80..3f2c53286996 100644 --- a/python/ray/tests/accelerators/test_tpu.py +++ b/python/ray/tests/accelerators/test_tpu.py @@ -1,13 +1,13 @@ import os import sys from unittest import mock +from unittest.mock import patch + import pytest import requests -from unittest.mock import patch import ray -from ray._private.accelerators import TPUAcceleratorManager -from ray._private.accelerators import tpu +from ray._private.accelerators import TPUAcceleratorManager, tpu from ray.tests.conftest import _ray_start_cluster diff --git a/python/ray/tests/autoscaler/test_providers.py b/python/ray/tests/autoscaler/test_providers.py index b4e0a8c87676..85dd8b68171a 100644 --- a/python/ray/tests/autoscaler/test_providers.py +++ b/python/ray/tests/autoscaler/test_providers.py @@ -1,10 +1,12 @@ +import unittest + +import yaml + from ray.autoscaler._private.providers import ( + _DEFAULT_CONFIGS, _NODE_PROVIDERS, _PROVIDER_PRETTY_NAMES, - _DEFAULT_CONFIGS, ) -import unittest -import yaml class TestProviders(unittest.TestCase): diff --git a/python/ray/tests/autoscaler/util.py b/python/ray/tests/autoscaler/util.py index 426ff3662402..a6631f3d2a31 100644 --- a/python/ray/tests/autoscaler/util.py +++ b/python/ray/tests/autoscaler/util.py @@ -1,5 +1,6 @@ import unittest from unittest.mock import Mock + from ray.autoscaler._private.util import get_per_node_breakdown_as_dict diff --git a/python/ray/tests/autoscaler_test_utils.py b/python/ray/tests/autoscaler_test_utils.py index 8cbcebd6ac2a..d0a0d39e567a 100644 --- a/python/ray/tests/autoscaler_test_utils.py +++ b/python/ray/tests/autoscaler_test_utils.py @@ -1,8 +1,8 @@ import re import threading - from subprocess import CalledProcessError from typing import Any, Dict, List, Optional + from ray.autoscaler.node_provider import NodeProvider diff --git a/python/ray/tests/aws/conftest.py b/python/ray/tests/aws/conftest.py index ed3a6a4b71ad..8f63e619cea9 100644 --- a/python/ray/tests/aws/conftest.py +++ b/python/ray/tests/aws/conftest.py @@ -1,9 +1,8 @@ import pytest +from botocore.stub import Stubber +from ray.autoscaler._private.aws.utils import client_cache, resource_cache from ray.autoscaler._private.constants import BOTO_MAX_RETRIES -from ray.autoscaler._private.aws.utils import resource_cache, client_cache - -from botocore.stub import Stubber @pytest.fixture() diff --git a/python/ray/tests/aws/test_aws_batch_tag_update.py b/python/ray/tests/aws/test_aws_batch_tag_update.py index a9bcd45ffab5..3bd39c0a4ef0 100644 --- a/python/ray/tests/aws/test_aws_batch_tag_update.py +++ b/python/ray/tests/aws/test_aws_batch_tag_update.py @@ -6,8 +6,7 @@ import pytest -from ray.autoscaler._private.aws.node_provider import AWSNodeProvider -from ray.autoscaler._private.aws.node_provider import TAG_BATCH_DELAY +from ray.autoscaler._private.aws.node_provider import TAG_BATCH_DELAY, AWSNodeProvider def mock_create_tags(provider, batch_updates): diff --git a/python/ray/tests/aws/utils/constants.py b/python/ray/tests/aws/utils/constants.py index b92fc5e45ea0..7b0ee9eca340 100644 --- a/python/ray/tests/aws/utils/constants.py +++ b/python/ray/tests/aws/utils/constants.py @@ -1,11 +1,11 @@ import copy -import ray from datetime import datetime +import ray from ray.autoscaler.tags import ( + NODE_KIND_HEAD, TAG_RAY_LAUNCH_CONFIG, TAG_RAY_NODE_KIND, - NODE_KIND_HEAD, TAG_RAY_USER_NODE_TYPE, ) diff --git a/python/ray/tests/aws/utils/helpers.py b/python/ray/tests/aws/utils/helpers.py index 12476cd6649c..9a3825634896 100644 --- a/python/ray/tests/aws/utils/helpers.py +++ b/python/ray/tests/aws/utils/helpers.py @@ -1,23 +1,24 @@ +import copy import os +from typing import Any, Dict + import yaml -import ray -import copy -from typing import Dict, Any +import ray +from ray.autoscaler._private.aws.cloudwatch.cloudwatch_helper import CloudwatchHelper from ray.autoscaler._private.aws.node_provider import AWSNodeProvider +from ray.autoscaler._private.commands import prepare_config, validate_config from ray.autoscaler.tags import ( - TAG_RAY_NODE_KIND, NODE_KIND_HEAD, NODE_KIND_WORKER, - TAG_RAY_USER_NODE_TYPE, TAG_RAY_CLUSTER_NAME, + TAG_RAY_NODE_KIND, + TAG_RAY_USER_NODE_TYPE, ) -from ray.autoscaler._private.commands import prepare_config, validate_config from ray.tests.aws.utils.constants import ( DEFAULT_CLUSTER_NAME, DEFAULT_NODE_PROVIDER_INSTANCE_TAGS, ) -from ray.autoscaler._private.aws.cloudwatch.cloudwatch_helper import CloudwatchHelper def get_aws_example_config_file_path(file_name): diff --git a/python/ray/tests/aws/utils/stubs.py b/python/ray/tests/aws/utils/stubs.py index a95b65cd9fdb..11c625c5b588 100644 --- a/python/ray/tests/aws/utils/stubs.py +++ b/python/ray/tests/aws/utils/stubs.py @@ -1,33 +1,32 @@ -from typing import Dict, List -import ray import copy import json - +from typing import Dict, List +from unittest import mock from uuid import uuid4 + +from botocore.stub import ANY + +import ray +from ray.autoscaler._private.aws.cloudwatch.cloudwatch_helper import ( + CLOUDWATCH_AGENT_INSTALLED_TAG, + CLOUDWATCH_CONFIG_HASH_TAG_BASE, +) +from ray.autoscaler._private.aws.config import key_pair +from ray.autoscaler.tags import NODE_KIND_HEAD, TAG_RAY_NODE_KIND from ray.tests.aws.utils import helpers from ray.tests.aws.utils.constants import ( + A_THOUSAND_SUBNETS_IN_DIFFERENT_VPCS, + DEFAULT_CLUSTER_NAME, DEFAULT_INSTANCE_PROFILE, DEFAULT_KEY_PAIR, - DEFAULT_SUBNET, - A_THOUSAND_SUBNETS_IN_DIFFERENT_VPCS, DEFAULT_LT, + DEFAULT_SUBNET, TWENTY_SUBNETS_IN_DIFFERENT_AZS, - DEFAULT_CLUSTER_NAME, ) -from ray.autoscaler._private.aws.config import key_pair from ray.tests.aws.utils.helpers import ( - get_cloudwatch_dashboard_config_file_path, get_cloudwatch_alarm_config_file_path, + get_cloudwatch_dashboard_config_file_path, ) -from ray.autoscaler._private.aws.cloudwatch.cloudwatch_helper import ( - CLOUDWATCH_AGENT_INSTALLED_TAG, - CLOUDWATCH_CONFIG_HASH_TAG_BASE, -) -from ray.autoscaler.tags import NODE_KIND_HEAD, TAG_RAY_NODE_KIND - -from unittest import mock - -from botocore.stub import ANY def configure_iam_role_default(iam_client_stub): diff --git a/python/ray/tests/chaos/potato_passer.py b/python/ray/tests/chaos/potato_passer.py index 4f84693fa647..25e5b912ef96 100644 --- a/python/ray/tests/chaos/potato_passer.py +++ b/python/ray/tests/chaos/potato_passer.py @@ -1,5 +1,6 @@ -import asyncio import argparse +import asyncio + import ray ray.init() diff --git a/python/ray/tests/chaos/streaming_llm.py b/python/ray/tests/chaos/streaming_llm.py index bbe5075d8c47..4b0536687dba 100644 --- a/python/ray/tests/chaos/streaming_llm.py +++ b/python/ray/tests/chaos/streaming_llm.py @@ -1,8 +1,8 @@ +import argparse import asyncio import logging -import requests -import argparse +import requests from fastapi import FastAPI from starlette.responses import StreamingResponse diff --git a/python/ray/tests/conftest.py b/python/ray/tests/conftest.py index 7909600440d9..386f9f63baa0 100644 --- a/python/ray/tests/conftest.py +++ b/python/ray/tests/conftest.py @@ -2,6 +2,7 @@ This file defines the common pytest fixtures used in current directory. """ +import copy import json import logging import os @@ -16,35 +17,35 @@ from tempfile import gettempdir from typing import List, Optional from unittest import mock -import psutil + import pytest -import copy import ray -from ray._common.test_utils import wait_for_condition import ray._private.ray_constants as ray_constants -from ray._private.conftest_utils import set_override_dashboard_url # noqa: F401 from ray._common.network_utils import build_address +from ray._common.test_utils import wait_for_condition +from ray._private.conftest_utils import set_override_dashboard_url # noqa: F401 from ray._private.runtime_env import virtualenv_utils - from ray._private.test_utils import ( + RayletKiller, + external_redis_test_enabled, + find_free_port, get_and_run_resource_killer, + get_redis_cli, init_error_pubsub, init_log_pubsub, - setup_tls, - teardown_tls, - external_redis_test_enabled, redis_replicas, - get_redis_cli, - start_redis_instance, - start_redis_sentinel_instance, redis_sentinel_replicas, - find_free_port, reset_autoscaler_v2_enabled_cache, - RayletKiller, + setup_tls, + start_redis_instance, + start_redis_sentinel_instance, + teardown_tls, ) from ray.cluster_utils import AutoscalingCluster, Cluster, cluster_not_supported +import psutil + # TODO (mengjin) Improve the logging in the conftest files so that the logger can log # information in stdout as well as stderr and replace the print statements in the test # files diff --git a/python/ray/tests/conftest_docker.py b/python/ray/tests/conftest_docker.py index 12be797a23c6..02f4598484b1 100644 --- a/python/ray/tests/conftest_docker.py +++ b/python/ray/tests/conftest_docker.py @@ -1,10 +1,12 @@ +import subprocess import time +from typing import List + import pytest -from pytest_docker_tools import container, fetch, network, volume -from pytest_docker_tools import wrappers -import subprocess +from pytest_docker_tools import container, fetch, network, volume, wrappers + import docker -from typing import List + from ray._common.network_utils import build_address # If you need to debug tests using fixtures in this file, diff --git a/python/ray/tests/gcp/test_gcp_node_provider.py b/python/ray/tests/gcp/test_gcp_node_provider.py index 13623ad41e04..1826b6781d6f 100644 --- a/python/ray/tests/gcp/test_gcp_node_provider.py +++ b/python/ray/tests/gcp/test_gcp_node_provider.py @@ -1,33 +1,32 @@ import logging import sys -from typing import Dict from threading import RLock -from unittest.mock import MagicMock, patch, call +from typing import Dict +from unittest.mock import MagicMock, call, patch import pytest +from ray.autoscaler._private.command_runner import DockerCommandRunner, SSHCommandRunner +from ray.autoscaler._private.gcp.config import ( + _get_num_tpu_chips, + _has_tpus_in_node_configs, + _is_single_host_tpu, + get_node_type, + tpu_accelerator_config_to_type, +) from ray.autoscaler._private.gcp.node import ( GCPCompute, GCPNode, GCPNodeType, GCPResource, ) - -from ray.tests.test_autoscaler import MockProcessRunner from ray.autoscaler._private.gcp.node_provider import GCPNodeProvider -from ray.autoscaler._private.gcp.config import ( - get_node_type, - _get_num_tpu_chips, - _is_single_host_tpu, - _has_tpus_in_node_configs, - tpu_accelerator_config_to_type, -) from ray.autoscaler._private.gcp.tpu_command_runner import ( TPUCommandRunner, - TPUVMSSHCommandRunner, TPUVMDockerCommandRunner, + TPUVMSSHCommandRunner, ) -from ray.autoscaler._private.command_runner import SSHCommandRunner, DockerCommandRunner +from ray.tests.test_autoscaler import MockProcessRunner _PROJECT_NAME = "project-one" _AZ = "us-west1-b" diff --git a/python/ray/tests/gcp/test_gcp_tpu_command_runner.py b/python/ray/tests/gcp/test_gcp_tpu_command_runner.py index df908f58cf8d..4c8a88e9149e 100644 --- a/python/ray/tests/gcp/test_gcp_tpu_command_runner.py +++ b/python/ray/tests/gcp/test_gcp_tpu_command_runner.py @@ -6,10 +6,10 @@ import pytest -from ray.tests.test_autoscaler import MockProvider, MockProcessRunner -from ray.autoscaler._private.gcp.tpu_command_runner import TPUCommandRunner -from ray.autoscaler._private.command_runner import SSHCommandRunner from ray._private import ray_constants +from ray.autoscaler._private.command_runner import SSHCommandRunner +from ray.autoscaler._private.gcp.tpu_command_runner import TPUCommandRunner +from ray.tests.test_autoscaler import MockProcessRunner, MockProvider _MOCK_TPU_NAME = "my-tpu" _MOCK_ACCELERATOR_TYPE = "v4-16" diff --git a/python/ray/tests/horovod/horovod_example.py b/python/ray/tests/horovod/horovod_example.py index 92b4cc1a67f6..d53a93868b82 100644 --- a/python/ray/tests/horovod/horovod_example.py +++ b/python/ray/tests/horovod/horovod_example.py @@ -1,16 +1,15 @@ # This file is duplicated in release/ml_user_tests/horovod import argparse import os -from filelock import FileLock +import horovod.torch as hvd import torch.nn as nn import torch.nn.functional as F import torch.optim as optim -from torchvision import datasets, transforms import torch.utils.data.distributed - -import horovod.torch as hvd +from filelock import FileLock from horovod.ray import RayExecutor +from torchvision import datasets, transforms def metric_average(val, name): diff --git a/python/ray/tests/horovod/test_horovod.py b/python/ray/tests/horovod/test_horovod.py index 19103f399a6d..93aceaae278b 100644 --- a/python/ray/tests/horovod/test_horovod.py +++ b/python/ray/tests/horovod/test_horovod.py @@ -9,8 +9,8 @@ pytest.importorskip("horovod") try: - from horovod.ray.runner import RayExecutor from horovod.common.util import gloo_built + from horovod.ray.runner import RayExecutor except ImportError: pass # This shouldn't be reached - the test should be skipped. @@ -30,11 +30,12 @@ def ray_start_4_cpus(request): def _train(batch_size=32, batch_per_iter=10): + import timeit + + import horovod.torch as hvd import torch.nn.functional as F import torch.optim as optim import torch.utils.data.distributed - import horovod.torch as hvd - import timeit hvd.init() diff --git a/python/ray/tests/kuberay/scripts/non_terminated_nodes_count.py b/python/ray/tests/kuberay/scripts/non_terminated_nodes_count.py index 86f55f67a8ff..7d64678cf679 100644 --- a/python/ray/tests/kuberay/scripts/non_terminated_nodes_count.py +++ b/python/ray/tests/kuberay/scripts/non_terminated_nodes_count.py @@ -1,6 +1,6 @@ import ray -from ray.autoscaler._private.providers import _get_node_provider from ray.autoscaler._private.kuberay.autoscaling_config import _generate_provider_config +from ray.autoscaler._private.providers import _get_node_provider @ray.remote diff --git a/python/ray/tests/kuberay/scripts/scale_up_custom.py b/python/ray/tests/kuberay/scripts/scale_up_custom.py index ada4c9eb757e..3810c635e3be 100644 --- a/python/ray/tests/kuberay/scripts/scale_up_custom.py +++ b/python/ray/tests/kuberay/scripts/scale_up_custom.py @@ -1,6 +1,7 @@ -import ray import time +import ray + def main(): """Submits custom resource request. diff --git a/python/ray/tests/kuberay/test_autoscaling_config.py b/python/ray/tests/kuberay/test_autoscaling_config.py index 29597fc86f8b..61a886d96758 100644 --- a/python/ray/tests/kuberay/test_autoscaling_config.py +++ b/python/ray/tests/kuberay/test_autoscaling_config.py @@ -1,23 +1,23 @@ import copy -from pathlib import Path import platform -import requests import sys +from pathlib import Path from typing import Any, Dict, Optional, Type from unittest import mock -import yaml import pytest +import requests +import yaml from ray.autoscaler._private.kuberay.autoscaling_config import ( GKE_TPU_ACCELERATOR_LABEL, GKE_TPU_TOPOLOGY_LABEL, - _derive_autoscaling_config_from_ray_cr, AutoscalingConfigProducer, - _round_up_k8s_quantity, - _get_num_tpus, + _derive_autoscaling_config_from_ray_cr, _get_custom_resources, + _get_num_tpus, _get_ray_resources_from_group_spec, + _round_up_k8s_quantity, ) from ray.autoscaler._private.kuberay.utils import tpu_node_selectors_to_type diff --git a/python/ray/tests/kuberay/test_autoscaling_e2e.py b/python/ray/tests/kuberay/test_autoscaling_e2e.py index 22d8cd811fb1..5ecc1410cb3f 100644 --- a/python/ray/tests/kuberay/test_autoscaling_e2e.py +++ b/python/ray/tests/kuberay/test_autoscaling_e2e.py @@ -7,20 +7,20 @@ import tempfile import unittest from typing import Any, Dict -import yaml import pytest +import yaml from ray.tests.kuberay.utils import ( get_pod, get_pod_names, get_raycluster, - switch_to_ray_parent_dir, + kubectl_delete, kubectl_exec_python_script, kubectl_logs, - kubectl_delete, - wait_for_pods, + switch_to_ray_parent_dir, wait_for_pod_to_start, + wait_for_pods, wait_for_ray_health, ) diff --git a/python/ray/tests/kuberay/test_kuberay_node_provider.py b/python/ray/tests/kuberay/test_kuberay_node_provider.py index 96ca4863e865..189de40b5521 100644 --- a/python/ray/tests/kuberay/test_kuberay_node_provider.py +++ b/python/ray/tests/kuberay/test_kuberay_node_provider.py @@ -1,25 +1,24 @@ import copy -from unittest import mock import sys +from collections import defaultdict +from pathlib import Path +from typing import List, Set +from unittest import mock import jsonpatch import pytest +import yaml -from collections import defaultdict -from ray.autoscaler.batching_node_provider import NodeData from ray.autoscaler._private.kuberay.node_provider import ( + KubeRayNodeProvider, + ScaleRequest, _worker_group_index, _worker_group_max_replicas, _worker_group_replicas, - KubeRayNodeProvider, - ScaleRequest, ) from ray.autoscaler._private.util import NodeID -from pathlib import Path -import yaml - +from ray.autoscaler.batching_node_provider import NodeData from ray.tests.kuberay.test_autoscaling_config import get_basic_ray_cr -from typing import Set, List def _get_basic_ray_cr_workers_to_delete( diff --git a/python/ray/tests/kuberay/utils.py b/python/ray/tests/kuberay/utils.py index f11f0b24b4c4..63ddd9b3578f 100644 --- a/python/ray/tests/kuberay/utils.py +++ b/python/ray/tests/kuberay/utils.py @@ -4,14 +4,14 @@ import atexit import contextlib import logging +import os import pathlib import subprocess import tempfile import time from typing import Any, Dict, Generator, List, Optional -import yaml -import os +import yaml logger = logging.getLogger(__name__) diff --git a/python/ray/tests/ludwig/ludwig_test_utils.py b/python/ray/tests/ludwig/ludwig_test_utils.py index 069d431655ad..3b567bce129b 100644 --- a/python/ray/tests/ludwig/ludwig_test_utils.py +++ b/python/ray/tests/ludwig/ludwig_test_utils.py @@ -30,12 +30,10 @@ import cloudpickle import numpy as np import pandas as pd - from ludwig.api import LudwigModel from ludwig.backend import LocalBackend -from ludwig.constants import VECTOR, COLUMN, NAME, PROC_COLUMN -from ludwig.data.dataset_synthesizer import DATETIME_FORMATS -from ludwig.data.dataset_synthesizer import build_synthetic_dataset +from ludwig.constants import COLUMN, NAME, PROC_COLUMN, VECTOR +from ludwig.data.dataset_synthesizer import DATETIME_FORMATS, build_synthetic_dataset from ludwig.experiment import experiment_cli from ludwig.features.feature_utils import compute_feature_hash from ludwig.utils.data_utils import read_csv, replace_file_extension diff --git a/python/ray/tests/ludwig/test_ludwig.py b/python/ray/tests/ludwig/test_ludwig.py index a19ec33520a4..6978234a394f 100644 --- a/python/ray/tests/ludwig/test_ludwig.py +++ b/python/ray/tests/ludwig/test_ludwig.py @@ -19,8 +19,8 @@ import contextlib import os -import tempfile import sys +import tempfile import pytest @@ -47,18 +47,21 @@ if not skip: from ludwig.backend.ray import RayBackend, get_horovod_kwargs - from ray.tests.ludwig.ludwig_test_utils import create_data_set_to_use, spawn - from ray.tests.ludwig.ludwig_test_utils import bag_feature - from ray.tests.ludwig.ludwig_test_utils import binary_feature - from ray.tests.ludwig.ludwig_test_utils import category_feature - from ray.tests.ludwig.ludwig_test_utils import date_feature - from ray.tests.ludwig.ludwig_test_utils import generate_data - from ray.tests.ludwig.ludwig_test_utils import h3_feature - from ray.tests.ludwig.ludwig_test_utils import numerical_feature - from ray.tests.ludwig.ludwig_test_utils import sequence_feature - from ray.tests.ludwig.ludwig_test_utils import set_feature - from ray.tests.ludwig.ludwig_test_utils import train_with_backend - from ray.tests.ludwig.ludwig_test_utils import vector_feature + from ray.tests.ludwig.ludwig_test_utils import ( + bag_feature, + binary_feature, + category_feature, + create_data_set_to_use, + date_feature, + generate_data, + h3_feature, + numerical_feature, + sequence_feature, + set_feature, + spawn, + train_with_backend, + vector_feature, + ) else: diff --git a/python/ray/tests/mock_s3_server.py b/python/ray/tests/mock_s3_server.py index b935e7865c1a..f5bd792be488 100644 --- a/python/ray/tests/mock_s3_server.py +++ b/python/ray/tests/mock_s3_server.py @@ -1,12 +1,13 @@ # extracted from aioboto3 # https://github.com/terrycain/aioboto3/blob/16a1a1085191ebe6d40ee45d9588b2173738af0c/tests/mock_server.py -import pytest -import requests import shutil import signal import subprocess as sp import time +import pytest +import requests + from ray._common.network_utils import build_address _proxy_bypass = { diff --git a/python/ray/tests/modin/modin_test_utils.py b/python/ray/tests/modin/modin_test_utils.py index 4071b536104f..5c7ec28aaaf9 100644 --- a/python/ray/tests/modin/modin_test_utils.py +++ b/python/ray/tests/modin/modin_test_utils.py @@ -16,16 +16,16 @@ # This file is copied and adapted from # http://github.com/modin-project/modin/master/modin/pandas/test/utils.py -import pandas import modin.pandas as pd +import numpy as np +import pandas from modin.utils import to_pandas from pandas.testing import ( - assert_series_equal, - assert_frame_equal, assert_extension_array_equal, + assert_frame_equal, assert_index_equal, + assert_series_equal, ) -import numpy as np def categories_equals(left, right): diff --git a/python/ray/tests/modin/test_modin.py b/python/ray/tests/modin/test_modin.py index 9f50fe465a2c..e379c2026df7 100644 --- a/python/ray/tests/modin/test_modin.py +++ b/python/ray/tests/modin/test_modin.py @@ -17,10 +17,12 @@ # http://github.com/modin-project/modin/master/modin/pandas/test/test_general.py import sys -import pytest -import pandas + import numpy as np +import pandas +import pytest from numpy.testing import assert_array_equal + from ray.tests.conftest import ray_start_regular_shared # noqa F401 modin_installed = True @@ -36,9 +38,10 @@ pytestmark = pytest.mark.skipif(skip, reason="Outdated or missing Modin dependency") if not skip: - from ray.tests.modin.modin_test_utils import df_equals import modin.pandas as pd + from ray.tests.modin.modin_test_utils import df_equals + @pytest.fixture(autouse=True) def connect_to_ray_cluster(ray_start_regular_shared): # noqa F811 diff --git a/python/ray/tests/runtime_env_container/test_job.py b/python/ray/tests/runtime_env_container/test_job.py index 9d20c29d8e5a..e0a04d169cc7 100644 --- a/python/ray/tests/runtime_env_container/test_job.py +++ b/python/ray/tests/runtime_env_container/test_job.py @@ -1,8 +1,8 @@ import argparse import ray -from ray.job_submission import JobStatus, JobSubmissionClient from ray._common.test_utils import wait_for_condition +from ray.job_submission import JobStatus, JobSubmissionClient parser = argparse.ArgumentParser() parser.add_argument("--image", type=str, help="The docker image to use for Ray worker") diff --git a/python/ray/tests/runtime_env_container/test_log_file_exists.py b/python/ray/tests/runtime_env_container/test_log_file_exists.py index 8d1afff7eefe..a3dcec682c01 100644 --- a/python/ray/tests/runtime_env_container/test_log_file_exists.py +++ b/python/ray/tests/runtime_env_container/test_log_file_exists.py @@ -1,9 +1,10 @@ -import ray -from pathlib import Path +import argparse import re -from ray.util.state import list_tasks +from pathlib import Path + +import ray from ray._common.test_utils import wait_for_condition -import argparse +from ray.util.state import list_tasks parser = argparse.ArgumentParser() parser.add_argument("--image", type=str, help="The docker image to use for Ray worker") diff --git a/python/ray/tests/runtime_env_container/test_put_get.py b/python/ray/tests/runtime_env_container/test_put_get.py index cc79edf58d29..048b3b863804 100644 --- a/python/ray/tests/runtime_env_container/test_put_get.py +++ b/python/ray/tests/runtime_env_container/test_put_get.py @@ -1,7 +1,9 @@ -import ray -import numpy as np import argparse +import numpy as np + +import ray + parser = argparse.ArgumentParser() parser.add_argument("--image", type=str, help="The docker image to use for Ray worker") parser.add_argument( diff --git a/python/ray/tests/runtime_env_container/test_serve_basic.py b/python/ray/tests/runtime_env_container/test_serve_basic.py index c68597600dbd..8175441eebed 100644 --- a/python/ray/tests/runtime_env_container/test_serve_basic.py +++ b/python/ray/tests/runtime_env_container/test_serve_basic.py @@ -1,4 +1,5 @@ import argparse + from ray import serve from ray._common.test_utils import wait_for_condition from ray.serve.handle import DeploymentHandle diff --git a/python/ray/tests/runtime_env_container/test_serve_telemetry.py b/python/ray/tests/runtime_env_container/test_serve_telemetry.py index d20f0d4c48a3..bd24b23318b2 100644 --- a/python/ray/tests/runtime_env_container/test_serve_telemetry.py +++ b/python/ray/tests/runtime_env_container/test_serve_telemetry.py @@ -5,14 +5,14 @@ import ray from ray import serve from ray._common.test_utils import wait_for_condition -from ray.serve._private.usage import ServeUsageTag -from ray.serve.context import _get_global_client -from ray.serve.schema import ServeDeploySchema from ray.serve._private.test_utils import ( TelemetryStorage, check_ray_started, check_ray_stopped, ) +from ray.serve._private.usage import ServeUsageTag +from ray.serve.context import _get_global_client +from ray.serve.schema import ServeDeploySchema parser = argparse.ArgumentParser( description="Example Python script taking command line arguments." diff --git a/python/ray/tests/runtime_env_container/test_shared_memory.py b/python/ray/tests/runtime_env_container/test_shared_memory.py index 622b6813fbb1..d501a41709a9 100644 --- a/python/ray/tests/runtime_env_container/test_shared_memory.py +++ b/python/ray/tests/runtime_env_container/test_shared_memory.py @@ -1,8 +1,9 @@ -import ray -import numpy as np -import sys import argparse +import sys +import numpy as np + +import ray parser = argparse.ArgumentParser() parser.add_argument("--image", type=str, help="The docker image to use for Ray worker") diff --git a/python/ray/tests/runtime_env_container/test_worker_exit_intended_system_exit_and_user_error.py b/python/ray/tests/runtime_env_container/test_worker_exit_intended_system_exit_and_user_error.py index e8280a2b9214..80db3a460459 100644 --- a/python/ray/tests/runtime_env_container/test_worker_exit_intended_system_exit_and_user_error.py +++ b/python/ray/tests/runtime_env_container/test_worker_exit_intended_system_exit_and_user_error.py @@ -1,12 +1,12 @@ +import argparse import asyncio import os -import argparse import ray -from ray._private.state_api_test_utils import verify_failed_task -from ray.util.state import list_workers from ray._common.test_utils import wait_for_condition +from ray._private.state_api_test_utils import verify_failed_task from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy +from ray.util.state import list_workers parser = argparse.ArgumentParser() parser.add_argument("--image", type=str, help="The docker image to use for Ray worker") diff --git a/python/ray/tests/spark/test_GPU.py b/python/ray/tests/spark/test_GPU.py index 0d8641b75f24..d1d65ef345bf 100644 --- a/python/ray/tests/spark/test_GPU.py +++ b/python/ray/tests/spark/test_GPU.py @@ -1,21 +1,22 @@ -import sys -import pytest +import functools import os +import sys import time -import functools from abc import ABC + +import pytest from pyspark.sql import SparkSession + +import ray +from ray._common.test_utils import wait_for_condition from ray.tests.spark.test_basic import ( + _RAY_ON_SPARK_WORKER_PHYSICAL_MEMORY_BYTES, + _RAY_ON_SPARK_WORKER_SHARED_MEMORY_BYTES, RayOnSparkCPUClusterTestBase, _setup_ray_cluster, _setup_ray_on_spark_envs, - _RAY_ON_SPARK_WORKER_PHYSICAL_MEMORY_BYTES, - _RAY_ON_SPARK_WORKER_SHARED_MEMORY_BYTES, ) from ray.util.spark.utils import _calc_mem_per_ray_worker_node -from ray._common.test_utils import wait_for_condition - -import ray pytestmark = [ pytest.mark.skipif( diff --git a/python/ray/tests/spark/test_basic.py b/python/ray/tests/spark/test_basic.py index 3b1003e591df..bce74852f72f 100644 --- a/python/ray/tests/spark/test_basic.py +++ b/python/ray/tests/spark/test_basic.py @@ -1,32 +1,31 @@ +import logging import os +import re import shutil -import tempfile import socket -import threading -import re -import pytest import sys -from unittest import mock +import tempfile +import threading +import time from abc import ABC +from contextlib import contextmanager +from unittest import mock -import ray +import pytest +from pyspark.sql import SparkSession +import ray import ray.util.spark.cluster_init +from ray._common.test_utils import wait_for_condition from ray.util.spark import ( + MAX_NUM_WORKER_NODES, + setup_global_ray_cluster, setup_ray_cluster, shutdown_ray_cluster, - setup_global_ray_cluster, - MAX_NUM_WORKER_NODES, ) from ray.util.spark.utils import ( _calc_mem_per_ray_worker_node, ) -from pyspark.sql import SparkSession -import time -import logging -from contextlib import contextmanager -from ray._common.test_utils import wait_for_condition - pytestmark = [ pytest.mark.skipif( diff --git a/python/ray/tests/spark/test_databricks_hook.py b/python/ray/tests/spark/test_databricks_hook.py index e0e2d5ed12d8..5995e1103a68 100644 --- a/python/ray/tests/spark/test_databricks_hook.py +++ b/python/ray/tests/spark/test_databricks_hook.py @@ -1,14 +1,14 @@ +import os import sys +import time import pytest -import os -import time -import ray from pyspark.sql import SparkSession -from ray.util.spark import setup_ray_cluster + +import ray import ray.util.spark.databricks_hook from ray._common.test_utils import wait_for_condition - +from ray.util.spark import setup_ray_cluster pytestmark = pytest.mark.skipif( not sys.platform.startswith("linux"), diff --git a/python/ray/tests/spark/test_multicores_per_task.py b/python/ray/tests/spark/test_multicores_per_task.py index b34d93ec3616..3fb693d7466d 100644 --- a/python/ray/tests/spark/test_multicores_per_task.py +++ b/python/ray/tests/spark/test_multicores_per_task.py @@ -1,7 +1,9 @@ +import os import sys + import pytest -import os from pyspark.sql import SparkSession + from ray.tests.spark.test_basic import _setup_ray_on_spark_envs from ray.tests.spark.test_GPU import RayOnSparkGPUClusterTestBase diff --git a/python/ray/tests/spark/test_utils.py b/python/ray/tests/spark/test_utils.py index 35a516c9d4e3..a8efd615f81d 100644 --- a/python/ray/tests/spark/test_utils.py +++ b/python/ray/tests/spark/test_utils.py @@ -1,18 +1,19 @@ -from unittest.mock import patch import os import re import sys +from unittest.mock import patch import pytest -from ray.util.spark.utils import ( - get_spark_task_assigned_physical_gpus, - _calc_mem_per_ray_worker_node, - _get_avail_mem_per_ray_worker_node, -) + from ray.util.spark.cluster_init import ( + _append_default_spilling_dir_config, _convert_ray_node_options, _verify_node_options, - _append_default_spilling_dir_config, +) +from ray.util.spark.utils import ( + _calc_mem_per_ray_worker_node, + _get_avail_mem_per_ray_worker_node, + get_spark_task_assigned_physical_gpus, ) pytestmark = pytest.mark.skipif( diff --git a/python/ray/tests/test_actor.py b/python/ray/tests/test_actor.py index 175f75c7c518..7055b78c46e9 100644 --- a/python/ray/tests/test_actor.py +++ b/python/ray/tests/test_actor.py @@ -5,23 +5,24 @@ import numpy as np import pytest -import psutil import ray from ray import cloudpickle as pickle +from ray._common.test_utils import SignalActor, wait_for_condition +from ray._common.utils import hex_to_binary from ray._private import ray_constants +from ray._private.state_api_test_utils import invoke_state_api, invoke_state_api_n from ray._private.test_utils import ( client_test_enabled, wait_for_pid_to_exit, ) from ray.actor import ActorClassInheritanceException -from ray.tests.client_test_utils import create_remote_signal_actor -from ray._common.test_utils import SignalActor, wait_for_condition from ray.core.generated import gcs_pb2 -from ray._common.utils import hex_to_binary -from ray._private.state_api_test_utils import invoke_state_api, invoke_state_api_n +from ray.tests.client_test_utils import create_remote_signal_actor from ray.util.state import list_actors +import psutil + @pytest.mark.parametrize("set_enable_auto_connect", [True, False], indirect=True) def test_caching_actors(shutdown_only, set_enable_auto_connect): diff --git a/python/ray/tests/test_actor_advanced.py b/python/ray/tests/test_actor_advanced.py index 79b4b1c0a541..812cfd7f9557 100644 --- a/python/ray/tests/test_actor_advanced.py +++ b/python/ray/tests/test_actor_advanced.py @@ -1,16 +1,16 @@ import os import sys import time -from typing import Optional from concurrent.futures import ThreadPoolExecutor +from typing import Optional import pytest import ray import ray._private.gcs_utils as gcs_utils -from ray.util.state import list_actors import ray.cluster_utils from ray._common.test_utils import SignalActor, wait_for_condition +from ray._private.ray_constants import gcs_actor_scheduling_enabled from ray._private.test_utils import ( convert_actor_state, kill_actor_and_wait_for_failure, @@ -18,8 +18,8 @@ run_string_as_driver, wait_for_pid_to_exit, ) -from ray._private.ray_constants import gcs_actor_scheduling_enabled from ray.experimental.internal_kv import _internal_kv_get, _internal_kv_put +from ray.util.state import list_actors def test_actors_on_nodes_with_no_cpus(ray_start_no_cpu): diff --git a/python/ray/tests/test_actor_bounded_threads.py b/python/ray/tests/test_actor_bounded_threads.py index ba3d536a5851..f2fc7bf24857 100644 --- a/python/ray/tests/test_actor_bounded_threads.py +++ b/python/ray/tests/test_actor_bounded_threads.py @@ -1,13 +1,13 @@ -import sys -import os - -import ray import logging -from typing import Dict +import os +import sys from collections import Counter +from typing import Dict import pytest +import ray + logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_actor_cancel.py b/python/ray/tests/test_actor_cancel.py index 7f7042a039f2..201ec2a86eb1 100644 --- a/python/ray/tests/test_actor_cancel.py +++ b/python/ray/tests/test_actor_cancel.py @@ -1,14 +1,13 @@ import asyncio +import concurrent.futures import sys import time -import concurrent.futures from collections import defaultdict import pytest import ray -from ray._common.test_utils import SignalActor -from ray._common.test_utils import wait_for_condition +from ray._common.test_utils import SignalActor, wait_for_condition from ray.exceptions import TaskCancelledError from ray.util.state import list_tasks diff --git a/python/ray/tests/test_actor_failures.py b/python/ray/tests/test_actor_failures.py index 6397b1b48b9c..ed5b2b8a8db4 100644 --- a/python/ray/tests/test_actor_failures.py +++ b/python/ray/tests/test_actor_failures.py @@ -1,5 +1,5 @@ -import atexit import asyncio +import atexit import collections import os import signal @@ -8,18 +8,18 @@ import time from typing import Callable, Generator -import pytest import numpy as np +import pytest import ray -from ray.actor import exit_actor -from ray.exceptions import AsyncioActorExit import ray.cluster_utils from ray._common.test_utils import SignalActor, wait_for_condition from ray._private.test_utils import ( - wait_for_pid_to_exit, generate_system_config_map, + wait_for_pid_to_exit, ) +from ray.actor import exit_actor +from ray.exceptions import AsyncioActorExit SIGKILL = signal.SIGKILL if sys.platform != "win32" else signal.SIGTERM diff --git a/python/ray/tests/test_actor_lifetime.py b/python/ray/tests/test_actor_lifetime.py index 09ff2a16a13c..7d4f118f98a7 100644 --- a/python/ray/tests/test_actor_lifetime.py +++ b/python/ray/tests/test_actor_lifetime.py @@ -1,17 +1,17 @@ import os -import time import signal import sys +import time import pytest import ray from ray._common.test_utils import wait_for_condition -from ray.exceptions import RayActorError -from ray.job_config import JobConfig from ray._private.test_utils import ( wait_for_pid_to_exit, ) +from ray.exceptions import RayActorError +from ray.job_config import JobConfig SIGKILL = signal.SIGKILL if sys.platform != "win32" else signal.SIGTERM diff --git a/python/ray/tests/test_actor_lineage_reconstruction.py b/python/ray/tests/test_actor_lineage_reconstruction.py index 3e3fac33a507..eb06d0bd77e3 100644 --- a/python/ray/tests/test_actor_lineage_reconstruction.py +++ b/python/ray/tests/test_actor_lineage_reconstruction.py @@ -1,14 +1,13 @@ import gc import os -import sys import signal +import sys import pytest import ray from ray._common.test_utils import wait_for_condition -from ray.core.generated import gcs_pb2 -from ray.core.generated import common_pb2 +from ray.core.generated import common_pb2, gcs_pb2 @pytest.mark.parametrize("deterministic_failure", ["request", "response"]) diff --git a/python/ray/tests/test_actor_pool.py b/python/ray/tests/test_actor_pool.py index f7677deccfbf..b969933531cd 100644 --- a/python/ray/tests/test_actor_pool.py +++ b/python/ray/tests/test_actor_pool.py @@ -2,6 +2,7 @@ import sys import time from unittest.mock import MagicMock + import pytest import ray diff --git a/python/ray/tests/test_actor_retry_2.py b/python/ray/tests/test_actor_retry_2.py index fb5fde7f503a..0f06ba3d940e 100644 --- a/python/ray/tests/test_actor_retry_2.py +++ b/python/ray/tests/test_actor_retry_2.py @@ -2,11 +2,11 @@ import sys from collections import defaultdict from typing import Optional -from ray._common.test_utils import SignalActor import pytest import ray +from ray._common.test_utils import SignalActor class MyError(Exception): diff --git a/python/ray/tests/test_actor_state_metrics.py b/python/ray/tests/test_actor_state_metrics.py index f14af6fd8cea..c03ffe5560a2 100644 --- a/python/ray/tests/test_actor_state_metrics.py +++ b/python/ray/tests/test_actor_state_metrics.py @@ -1,6 +1,6 @@ import asyncio -import time import sys +import time from collections import defaultdict from typing import Dict @@ -9,13 +9,12 @@ import ray from ray._common.test_utils import wait_for_condition from ray._common.utils import hex_to_binary - -from ray.util.state import list_actors from ray._private.test_utils import ( raw_metrics, run_string_as_driver, ) from ray._private.worker import RayContext +from ray.util.state import list_actors _SYSTEM_CONFIG = { "metrics_report_interval_ms": 200, diff --git a/python/ray/tests/test_advanced.py b/python/ray/tests/test_advanced.py index 047e71d33c1f..ce5bb7f1e08e 100644 --- a/python/ray/tests/test_advanced.py +++ b/python/ray/tests/test_advanced.py @@ -8,18 +8,18 @@ import numpy as np import pytest -from ray._common.test_utils import wait_for_condition import ray._private.profiling as profiling import ray.cluster_utils +from ray._common.test_utils import wait_for_condition from ray._private.internal_api import ( - memory_summary, get_local_ongoing_lineage_reconstruction_tasks, + memory_summary, ) from ray._private.test_utils import ( client_test_enabled, ) -from ray.exceptions import ObjectFreedError from ray.core.generated import common_pb2 +from ray.exceptions import ObjectFreedError if client_test_enabled(): from ray.util.client import ray diff --git a/python/ray/tests/test_advanced_2.py b/python/ray/tests/test_advanced_2.py index 24714270a7fd..20e7715a461a 100644 --- a/python/ray/tests/test_advanced_2.py +++ b/python/ray/tests/test_advanced_2.py @@ -10,8 +10,8 @@ import ray import ray.cluster_utils from ray._common.test_utils import wait_for_condition -from ray.util.placement_group import placement_group from ray.util.accelerators import AWS_NEURON_CORE +from ray.util.placement_group import placement_group from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_advanced_3.py b/python/ray/tests/test_advanced_3.py index 55a5afc79d8e..09d0743b6a0d 100644 --- a/python/ray/tests/test_advanced_3.py +++ b/python/ray/tests/test_advanced_3.py @@ -1,21 +1,22 @@ # coding: utf-8 +import importlib import logging import os import pickle import socket import sys import time -import importlib import numpy as np import pytest -import psutil import ray import ray._private.ray_constants import ray._private.utils from ray._private.test_utils import check_call_ray, wait_for_num_actors +import psutil + logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_advanced_4.py b/python/ray/tests/test_advanced_4.py index f000ad4d7088..699757aa5592 100644 --- a/python/ray/tests/test_advanced_4.py +++ b/python/ray/tests/test_advanced_4.py @@ -6,11 +6,11 @@ import pytest import ray +from ray._common.test_utils import Semaphore, wait_for_condition from ray._private.test_utils import ( client_test_enabled, get_gcs_memory_used, ) -from ray._common.test_utils import Semaphore, wait_for_condition from ray.experimental.internal_kv import _internal_kv_list diff --git a/python/ray/tests/test_advanced_6.py b/python/ray/tests/test_advanced_6.py index 98e6edfec377..3f54d79fa95f 100644 --- a/python/ray/tests/test_advanced_6.py +++ b/python/ray/tests/test_advanced_6.py @@ -6,17 +6,18 @@ import sys import time -import psutil import pytest import ray -from ray._common.test_utils import wait_for_condition import ray.cluster_utils +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( run_string_as_driver_nonblocking, wait_for_pid_to_exit, ) +import psutil + logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_advanced_7.py b/python/ray/tests/test_advanced_7.py index 339b90fc89d3..7eb7e23fd0a9 100644 --- a/python/ray/tests/test_advanced_7.py +++ b/python/ray/tests/test_advanced_7.py @@ -6,8 +6,8 @@ import time from concurrent.futures import ThreadPoolExecutor -import pytest import numpy as np +import pytest import ray.cluster_utils from ray._private.test_utils import client_test_enabled diff --git a/python/ray/tests/test_advanced_8.py b/python/ray/tests/test_advanced_8.py index 6c0c32d4f1c2..0b24bac5b435 100644 --- a/python/ray/tests/test_advanced_8.py +++ b/python/ray/tests/test_advanced_8.py @@ -9,10 +9,8 @@ from unittest import mock import numpy as np -import psutil import pytest -from ray._common.utils import RESOURCE_CONSTRAINT_PREFIX import ray import ray._private.gcs_utils as gcs_utils import ray._private.ray_constants as ray_constants @@ -20,9 +18,12 @@ import ray.cluster_utils import ray.util.accelerators from ray._common.test_utils import wait_for_condition +from ray._common.utils import RESOURCE_CONSTRAINT_PREFIX from ray.dashboard import k8s_utils from ray.runtime_env import RuntimeEnv +import psutil + logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_advanced_9.py b/python/ray/tests/test_advanced_9.py index 39160849b148..dd139ebed0a3 100644 --- a/python/ray/tests/test_advanced_9.py +++ b/python/ray/tests/test_advanced_9.py @@ -1,25 +1,26 @@ import os -import psutil import subprocess import sys import pytest import ray -import ray.util.state import ray._private.ray_constants as ray_constants +import ray.util.state +from ray._common.network_utils import parse_address +from ray._common.test_utils import Semaphore, wait_for_condition from ray._private.test_utils import ( - external_redis_test_enabled, client_test_enabled, - run_string_as_driver, + external_redis_test_enabled, get_gcs_memory_used, + run_string_as_driver, run_string_as_driver_nonblocking, ) -from ray._common.network_utils import parse_address -from ray._common.test_utils import Semaphore, wait_for_condition +from ray._raylet import GCS_PID_KEY, GcsClient from ray.experimental.internal_kv import _internal_kv_list from ray.tests.conftest import call_ray_start -from ray._raylet import GcsClient, GCS_PID_KEY + +import psutil @pytest.fixture diff --git a/python/ray/tests/test_annotations.py b/python/ray/tests/test_annotations.py index 44569cf32f95..cc8a39ba4561 100644 --- a/python/ray/tests/test_annotations.py +++ b/python/ray/tests/test_annotations.py @@ -3,10 +3,10 @@ import pytest -from ray.util.annotations import Deprecated from ray._private.test_utils import ( run_string_as_driver, ) +from ray.util.annotations import Deprecated # Use default filterwarnings behavior for this test diff --git a/python/ray/tests/test_async.py b/python/ray/tests/test_async.py index 0b556a1cd08e..e98b6971fd77 100644 --- a/python/ray/tests/test_async.py +++ b/python/ray/tests/test_async.py @@ -4,14 +4,13 @@ import time import numpy as np - import pytest import ray +from ray._common.test_utils import wait_for_condition from ray._common.utils import ( get_or_create_event_loop, ) -from ray._common.test_utils import wait_for_condition @pytest.fixture diff --git a/python/ray/tests/test_asyncio.py b/python/ray/tests/test_asyncio.py index 904ff1bb403a..5223ffd98701 100644 --- a/python/ray/tests/test_asyncio.py +++ b/python/ray/tests/test_asyncio.py @@ -8,8 +8,8 @@ import pytest import ray -from ray._private.client_mode_hook import client_mode_should_convert from ray._common.test_utils import SignalActor, wait_for_condition +from ray._private.client_mode_hook import client_mode_should_convert from ray._private.test_utils import ( kill_actor_and_wait_for_failure, wait_for_pid_to_exit, diff --git a/python/ray/tests/test_asyncio_cluster.py b/python/ray/tests/test_asyncio_cluster.py index 280bbbae698d..52165bdbb353 100644 --- a/python/ray/tests/test_asyncio_cluster.py +++ b/python/ray/tests/test_asyncio_cluster.py @@ -2,8 +2,8 @@ import asyncio import sys -import pytest import numpy as np +import pytest import ray from ray.cluster_utils import Cluster, cluster_not_supported diff --git a/python/ray/tests/test_autoscaler.py b/python/ray/tests/test_autoscaler.py index a728071528c2..a4a6a0216b71 100644 --- a/python/ray/tests/test_autoscaler.py +++ b/python/ray/tests/test_autoscaler.py @@ -1,10 +1,10 @@ import copy -import logging -import sys import json +import logging import os import re import shutil +import sys import tempfile import time import unittest @@ -20,20 +20,15 @@ from jsonschema.exceptions import ValidationError import ray -from ray.tests.autoscaler_test_utils import ( - MockNode, - MockProcessRunner, - MockProvider, -) from ray.autoscaler._private import commands from ray.autoscaler._private.autoscaler import NonTerminatedNodes, StandardAutoscaler from ray.autoscaler._private.commands import get_or_create_head_node from ray.autoscaler._private.constants import ( + AUTOSCALER_HEARTBEAT_TIMEOUT_S, DISABLE_LAUNCH_CONFIG_CHECK_KEY, DISABLE_NODE_UPDATERS_KEY, FOREGROUND_NODE_LAUNCH_KEY, WORKER_LIVENESS_CHECK_KEY, - AUTOSCALER_HEARTBEAT_TIMEOUT_S, ) from ray.autoscaler._private.load_metrics import LoadMetrics from ray.autoscaler._private.monitor import Monitor @@ -62,13 +57,16 @@ TAG_RAY_NODE_STATUS, TAG_RAY_USER_NODE_TYPE, ) +from ray.core.generated import common_pb2, gcs_pb2 +from ray.exceptions import RpcError +from ray.tests.autoscaler_test_utils import ( + MockNode, + MockProcessRunner, + MockProvider, +) from ray.tests.test_batch_node_provider_unit import ( MockBatchingNodeProvider, ) -from ray.exceptions import RpcError - -from ray.core.generated import gcs_pb2, common_pb2 - WORKER_FILTER = {TAG_RAY_NODE_KIND: NODE_KIND_WORKER} diff --git a/python/ray/tests/test_autoscaler_drain_node_api.py b/python/ray/tests/test_autoscaler_drain_node_api.py index 207d9f4f1dfe..abe5e97d1ce0 100644 --- a/python/ray/tests/test_autoscaler_drain_node_api.py +++ b/python/ray/tests/test_autoscaler_drain_node_api.py @@ -1,13 +1,13 @@ import logging import platform -import time import sys +import time import pytest import ray -from ray._common.test_utils import wait_for_condition import ray._private.ray_constants as ray_constants +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( get_error_message, init_error_pubsub, diff --git a/python/ray/tests/test_autoscaler_e2e.py b/python/ray/tests/test_autoscaler_e2e.py index b491a824a14a..f3ea5dec3195 100644 --- a/python/ray/tests/test_autoscaler_e2e.py +++ b/python/ray/tests/test_autoscaler_e2e.py @@ -4,13 +4,13 @@ import pytest import ray -from ray.autoscaler._private.constants import AUTOSCALER_METRIC_PORT from ray._common.network_utils import build_address +from ray._common.test_utils import SignalActor, wait_for_condition from ray._private.test_utils import ( - get_metric_check_condition, MetricSamplePattern, + get_metric_check_condition, ) -from ray._common.test_utils import SignalActor, wait_for_condition +from ray.autoscaler._private.constants import AUTOSCALER_METRIC_PORT from ray.autoscaler.node_launch_exception import NodeLaunchException diff --git a/python/ray/tests/test_autoscaler_fake_multinode.py b/python/ray/tests/test_autoscaler_fake_multinode.py index 767edb2596a7..a0a1772f447d 100644 --- a/python/ray/tests/test_autoscaler_fake_multinode.py +++ b/python/ray/tests/test_autoscaler_fake_multinode.py @@ -1,7 +1,8 @@ -import time -import pytest import platform import sys +import time + +import pytest import ray from ray.cluster_utils import AutoscalingCluster diff --git a/python/ray/tests/test_autoscaler_util.py b/python/ray/tests/test_autoscaler_util.py index d4b7a1b27e73..eee85e20334c 100644 --- a/python/ray/tests/test_autoscaler_util.py +++ b/python/ray/tests/test_autoscaler_util.py @@ -1,6 +1,6 @@ import sys -import pytest +import pytest from ray.autoscaler._private.util import with_envs, with_head_node_ip diff --git a/python/ray/tests/test_autoscaler_yaml.py b/python/ray/tests/test_autoscaler_yaml.py index 7e302e47a2f8..320461cebc6a 100644 --- a/python/ray/tests/test_autoscaler_yaml.py +++ b/python/ray/tests/test_autoscaler_yaml.py @@ -5,7 +5,7 @@ import tempfile import unittest import urllib -from typing import Dict, Any +from typing import Any, Dict from unittest import mock from unittest.mock import MagicMock, Mock, patch diff --git a/python/ray/tests/test_autoscaling_policy.py b/python/ray/tests/test_autoscaling_policy.py index 32f44e217555..fa7235250f28 100644 --- a/python/ray/tests/test_autoscaling_policy.py +++ b/python/ray/tests/test_autoscaling_policy.py @@ -1,39 +1,40 @@ import collections import copy import logging -import yaml -import tempfile -import sys -from typing import Dict, Callable, List import shutil -from queue import PriorityQueue +import sys +import tempfile import unittest +from queue import PriorityQueue +from typing import Callable, Dict, List + import pytest +import yaml import ray -from ray.tests.test_autoscaler import ( - MockProvider, - MockProcessRunner, - MockGcsClient, - mock_node_id, - MockAutoscaler, -) -from ray.tests.test_resource_demand_scheduler import MULTI_WORKER_CLUSTER +from ray._private.gcs_utils import PlacementGroupTableData +from ray.autoscaler._private.cli_logger import cli_logger +from ray.autoscaler._private.constants import AUTOSCALER_UPDATE_INTERVAL_S +from ray.autoscaler._private.load_metrics import LoadMetrics +from ray.autoscaler._private.node_launcher import NodeLauncher from ray.autoscaler._private.providers import ( _NODE_PROVIDERS, _clear_provider_cache, ) -from ray.autoscaler._private.load_metrics import LoadMetrics -from ray.autoscaler._private.node_launcher import NodeLauncher from ray.autoscaler.tags import ( - TAG_RAY_USER_NODE_TYPE, - TAG_RAY_NODE_KIND, NODE_KIND_HEAD, + TAG_RAY_NODE_KIND, + TAG_RAY_USER_NODE_TYPE, ) -from ray.autoscaler._private.constants import AUTOSCALER_UPDATE_INTERVAL_S -from ray.autoscaler._private.cli_logger import cli_logger from ray.core.generated.common_pb2 import Bundle, PlacementStrategy -from ray._private.gcs_utils import PlacementGroupTableData +from ray.tests.test_autoscaler import ( + MockAutoscaler, + MockGcsClient, + MockProcessRunner, + MockProvider, + mock_node_id, +) +from ray.tests.test_resource_demand_scheduler import MULTI_WORKER_CLUSTER class Task: diff --git a/python/ray/tests/test_baseexceptionandgroup.py b/python/ray/tests/test_baseexceptionandgroup.py index a363bdf84eaf..208a4b673e9e 100644 --- a/python/ray/tests/test_baseexceptionandgroup.py +++ b/python/ray/tests/test_baseexceptionandgroup.py @@ -1,11 +1,12 @@ -import pytest import sys from textwrap import dedent +import pytest + import ray from ray.exceptions import ( - RayTaskError, ActorDiedError, + RayTaskError, TaskCancelledError, WorkerCrashedError, ) diff --git a/python/ray/tests/test_basic.py b/python/ray/tests/test_basic.py index 6300b1e7aa1a..c2bb0c662034 100644 --- a/python/ray/tests/test_basic.py +++ b/python/ray/tests/test_basic.py @@ -10,7 +10,6 @@ import pytest import ray -import psutil import ray.cluster_utils from ray._common.test_utils import SignalActor from ray._private.test_utils import ( @@ -19,6 +18,8 @@ ) from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy +import psutil + logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_basic_5.py b/python/ray/tests/test_basic_5.py index 765bb4d72ee8..2a1568026ebd 100644 --- a/python/ray/tests/test_basic_5.py +++ b/python/ray/tests/test_basic_5.py @@ -2,22 +2,22 @@ import gc import logging import os +import subprocess import sys import time -import subprocess -from unittest.mock import Mock, patch import unittest +from unittest.mock import Mock, patch import pytest import ray import ray.cluster_utils +from ray._common.constants import HEAD_NODE_RESOURCE_NAME from ray._private.test_utils import ( + client_test_enabled, run_string_as_driver, wait_for_pid_to_exit, - client_test_enabled, ) -from ray._common.constants import HEAD_NODE_RESOURCE_NAME logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_batch_node_provider_integration.py b/python/ray/tests/test_batch_node_provider_integration.py index 7a7290a3f021..19423cc2e323 100644 --- a/python/ray/tests/test_batch_node_provider_integration.py +++ b/python/ray/tests/test_batch_node_provider_integration.py @@ -1,21 +1,21 @@ """Integration/e2e test for BatchingNodeProvider. Adapts FakeMultiNodeProvider tests. """ -from copy import deepcopy +import logging import sys +from copy import deepcopy import pytest - import ray from ray._common.test_utils import wait_for_condition +from ray.autoscaler._private.constants import FOREGROUND_NODE_LAUNCH_KEY +from ray.autoscaler._private.fake_multi_node.node_provider import FakeMultiNodeProvider from ray.autoscaler.batching_node_provider import ( BatchingNodeProvider, NodeData, ScaleRequest, ) -from ray.autoscaler._private.fake_multi_node.node_provider import FakeMultiNodeProvider -from ray.autoscaler._private.constants import FOREGROUND_NODE_LAUNCH_KEY from ray.autoscaler.tags import ( NODE_KIND_WORKER, STATUS_UP_TO_DATE, @@ -25,9 +25,6 @@ ) from ray.cluster_utils import AutoscalingCluster - -import logging - logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_batch_node_provider_unit.py b/python/ray/tests/test_batch_node_provider_unit.py index a2adc8a7b0a8..7f8fceb86637 100644 --- a/python/ray/tests/test_batch_node_provider_unit.py +++ b/python/ray/tests/test_batch_node_provider_unit.py @@ -1,34 +1,34 @@ """Unit test for BatchingNodeProvider. Validates BatchingNodeProvider's book-keeping logic. """ -from copy import copy -from uuid import uuid4 import random import sys -from typing import Any, Dict from collections import defaultdict +from copy import copy +from typing import Any, Dict +from uuid import uuid4 import pytest +from ray.autoscaler._private.constants import ( + DISABLE_LAUNCH_CONFIG_CHECK_KEY, + DISABLE_NODE_UPDATERS_KEY, + FOREGROUND_NODE_LAUNCH_KEY, +) +from ray.autoscaler._private.util import NodeID, NodeType from ray.autoscaler.batching_node_provider import ( BatchingNodeProvider, NodeData, ScaleRequest, ) -from ray.autoscaler._private.util import NodeID, NodeType from ray.autoscaler.tags import ( + NODE_KIND_HEAD, + NODE_KIND_WORKER, STATUS_UP_TO_DATE, - TAG_RAY_USER_NODE_TYPE, TAG_RAY_NODE_KIND, TAG_RAY_NODE_STATUS, TAG_RAY_REPLICA_INDEX, - NODE_KIND_HEAD, - NODE_KIND_WORKER, -) -from ray.autoscaler._private.constants import ( - DISABLE_LAUNCH_CONFIG_CHECK_KEY, - DISABLE_NODE_UPDATERS_KEY, - FOREGROUND_NODE_LAUNCH_KEY, + TAG_RAY_USER_NODE_TYPE, ) diff --git a/python/ray/tests/test_bounded_unix_sockets.py b/python/ray/tests/test_bounded_unix_sockets.py index 0f13c2e4a08f..9bbb5e9b09a5 100644 --- a/python/ray/tests/test_bounded_unix_sockets.py +++ b/python/ray/tests/test_bounded_unix_sockets.py @@ -1,10 +1,11 @@ +import logging import sys +import pytest + import ray -import logging -import psutil -import pytest +import psutil logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_bundle_label_selector.py b/python/ray/tests/test_bundle_label_selector.py index 8a029dbf03c8..c27b12eef787 100644 --- a/python/ray/tests/test_bundle_label_selector.py +++ b/python/ray/tests/test_bundle_label_selector.py @@ -1,12 +1,11 @@ -import sys import os +import sys import pytest import ray - -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy from ray._private.test_utils import placement_group_assert_no_leak +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy def test_bundle_label_selector_with_repeated_labels(ray_start_cluster): diff --git a/python/ray/tests/test_cancel.py b/python/ray/tests/test_cancel.py index 5878853fcf08..54c27641247c 100644 --- a/python/ray/tests/test_cancel.py +++ b/python/ray/tests/test_cancel.py @@ -1,25 +1,24 @@ +import _thread import random import signal import sys import threading -import _thread import time -import numpy as np from typing import List +import numpy as np import pytest import ray +from ray._common.test_utils import SignalActor, wait_for_condition +from ray._private.utils import DeferSigint from ray.exceptions import ( - TaskCancelledError, - RayTaskError, GetTimeoutError, + RayTaskError, + TaskCancelledError, WorkerCrashedError, ) from ray.types import ObjectRef -from ray._private.utils import DeferSigint -from ray._common.test_utils import SignalActor -from ray._common.test_utils import wait_for_condition from ray.util.state import list_tasks diff --git a/python/ray/tests/test_channel.py b/python/ray/tests/test_channel.py index f2c692b1d114..ad08512276fa 100644 --- a/python/ray/tests/test_channel.py +++ b/python/ray/tests/test_channel.py @@ -1,6 +1,6 @@ # coding: utf-8 -import pickle import logging +import pickle import sys import time import traceback @@ -13,11 +13,11 @@ import ray.cluster_utils import ray.exceptions import ray.experimental.channel as ray_channel -from ray.experimental.channel.torch_tensor_type import TorchTensorType +from ray._private.test_utils import get_actor_node_id +from ray.dag.compiled_dag_node import CompiledDAG from ray.exceptions import RayChannelError, RayChannelTimeoutError +from ray.experimental.channel.torch_tensor_type import TorchTensorType from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy -from ray.dag.compiled_dag_node import CompiledDAG -from ray._private.test_utils import get_actor_node_id logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_channel_serialization.py b/python/ray/tests/test_channel_serialization.py index 36ef7eebacca..732681feb1fc 100644 --- a/python/ray/tests/test_channel_serialization.py +++ b/python/ray/tests/test_channel_serialization.py @@ -2,11 +2,12 @@ import logging import os import sys + import pytest -from ray.experimental.util.types import Device -from ray.experimental.channel.serialization_context import _SerializationContext import torch +from ray.experimental.channel.serialization_context import _SerializationContext +from ray.experimental.util.types import Device logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_chaos.py b/python/ray/tests/test_chaos.py index 4805e001e2ec..19458773c2f4 100644 --- a/python/ray/tests/test_chaos.py +++ b/python/ray/tests/test_chaos.py @@ -1,24 +1,24 @@ +import random import sys import time -import random import pytest import ray from ray._common.test_utils import wait_for_condition -from ray.experimental import shuffle -from ray.tests.conftest import _ray_start_chaos_cluster -from ray.util.placement_group import placement_group from ray._private.test_utils import ( RayletKiller, - get_log_message, - get_and_run_resource_killer, WorkerKillerActor, + get_and_run_resource_killer, + get_log_message, ) -from ray.exceptions import RayTaskError, ObjectLostError -from ray.util.state.common import ListApiOptions, StateResource -from ray.util.state.api import StateApiClient, list_nodes from ray.cluster_utils import AutoscalingCluster +from ray.exceptions import ObjectLostError, RayTaskError +from ray.experimental import shuffle +from ray.tests.conftest import _ray_start_chaos_cluster +from ray.util.placement_group import placement_group +from ray.util.state.api import StateApiClient, list_nodes +from ray.util.state.common import ListApiOptions, StateResource def assert_no_system_failure(p, timeout): diff --git a/python/ray/tests/test_cli.py b/python/ray/tests/test_cli.py index 782d8792b879..577c54308015 100644 --- a/python/ray/tests/test_cli.py +++ b/python/ray/tests/test_cli.py @@ -18,6 +18,7 @@ randomized each time. """ import glob +import json import multiprocessing as mp import multiprocessing.connection import os @@ -25,10 +26,10 @@ import sys import tempfile import threading -import json import time import uuid from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path from typing import Optional from unittest import mock @@ -42,17 +43,16 @@ from testfixtures.popen import MockPopen, PopenBehaviour import ray +import ray._private.ray_constants as ray_constants +import ray._private.utils as utils import ray.autoscaler._private.aws.config as aws_config import ray.autoscaler._private.constants as autoscaler_constants -import ray._private.ray_constants as ray_constants import ray.scripts.scripts as scripts -import ray._private.utils as utils -from ray.util.check_open_ports import check_open_ports from ray._common.network_utils import build_address, parse_address from ray._common.test_utils import wait_for_condition from ray.cluster_utils import cluster_not_supported +from ray.util.check_open_ports import check_open_ports from ray.util.state import list_nodes -from http.server import BaseHTTPRequestHandler, HTTPServer import psutil diff --git a/python/ray/tests/test_client.py b/python/ray/tests/test_client.py index d9c52088bd97..4ff31cdfba05 100644 --- a/python/ray/tests/test_client.py +++ b/python/ray/tests/test_client.py @@ -6,8 +6,8 @@ import sys import threading import time -from unittest.mock import Mock from typing import Type +from unittest.mock import Mock import numpy as np import pytest @@ -17,12 +17,12 @@ import ray import ray.cloudpickle as cloudpickle import ray.util.client.server.server as ray_client_server +from ray._common.network_utils import build_address from ray._private.client_mode_hook import ( client_mode_should_convert, disable_client_hook, enable_client_mode, ) -from ray._common.network_utils import build_address from ray._private.test_utils import run_string_as_driver from ray.tests.client_test_utils import ( create_remote_signal_actor, diff --git a/python/ray/tests/test_client_builder.py b/python/ray/tests/test_client_builder.py index 5bcad34ca8a5..b98a565f9121 100644 --- a/python/ray/tests/test_client_builder.py +++ b/python/ray/tests/test_client_builder.py @@ -7,9 +7,9 @@ import pytest import ray -from ray._common.test_utils import wait_for_condition import ray.client_builder as client_builder import ray.util.client.server.server as ray_client_server +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( run_string_as_driver, run_string_as_driver_nonblocking, diff --git a/python/ray/tests/test_client_init.py b/python/ray/tests/test_client_init.py index a217245eed28..bcba6a5d5ce6 100644 --- a/python/ray/tests/test_client_init.py +++ b/python/ray/tests/test_client_init.py @@ -1,20 +1,18 @@ """Client tests that run their own init (as with init_and_serve) live here""" -import time import random -import sys import subprocess +import sys +import time from unittest.mock import patch import pytest -import ray.util.client.server.server as ray_client_server +import ray import ray.core.generated.ray_client_pb2 as ray_client_pb2 - -from ray.util.client import _ClientContext +import ray.util.client.server.server as ray_client_server from ray.cluster_utils import cluster_not_supported - -import ray +from ray.util.client import _ClientContext @ray.remote diff --git a/python/ray/tests/test_client_metadata.py b/python/ray/tests/test_client_metadata.py index 7b96588cad29..fa88466a9fab 100644 --- a/python/ray/tests/test_client_metadata.py +++ b/python/ray/tests/test_client_metadata.py @@ -2,10 +2,9 @@ import pytest -from ray.util.client.ray_client_helpers import ray_start_client_server from ray._raylet import NodeID - from ray.runtime_context import RuntimeContext +from ray.util.client.ray_client_helpers import ray_start_client_server def test_get_ray_metadata(ray_start_regular_shared): diff --git a/python/ray/tests/test_client_multi.py b/python/ray/tests/test_client_multi.py index e6c4f8ec6ca4..69e8b7254691 100644 --- a/python/ray/tests/test_client_multi.py +++ b/python/ray/tests/test_client_multi.py @@ -1,5 +1,7 @@ import sys + import pytest + import ray diff --git a/python/ray/tests/test_client_proxy.py b/python/ray/tests/test_client_proxy.py index a653db604bb7..8501cccf2c1e 100644 --- a/python/ray/tests/test_client_proxy.py +++ b/python/ray/tests/test_client_proxy.py @@ -4,17 +4,17 @@ import sys import time from glob import glob -from unittest.mock import patch, MagicMock from itertools import chain +from unittest.mock import MagicMock, patch import grpc import pytest import ray -from ray._common.test_utils import wait_for_condition import ray.core.generated.ray_client_pb2 as ray_client_pb2 -from ray._common.network_utils import parse_address import ray.util.client.server.proxier as proxier +from ray._common.network_utils import parse_address +from ray._common.test_utils import wait_for_condition from ray._private.ray_constants import REDIS_DEFAULT_PASSWORD from ray._private.test_utils import run_string_as_driver from ray.cloudpickle.compat import pickle diff --git a/python/ray/tests/test_client_reconnect.py b/python/ray/tests/test_client_reconnect.py index 47ec8aba8812..28361abbc9a1 100644 --- a/python/ray/tests/test_client_reconnect.py +++ b/python/ray/tests/test_client_reconnect.py @@ -1,21 +1,21 @@ -from concurrent import futures import contextlib import os -import threading +import random import sys +import threading +import time +from concurrent import futures +from typing import Any, Callable, Optional +from unittest.mock import Mock, patch + import grpc import numpy as np - -import time -import random import pytest -from typing import Any, Callable, Optional -from unittest.mock import patch, Mock import ray -from ray._common.utils import get_or_create_event_loop import ray.core.generated.ray_client_pb2 as ray_client_pb2 import ray.core.generated.ray_client_pb2_grpc as ray_client_pb2_grpc +from ray._common.utils import get_or_create_event_loop from ray.tests.conftest import call_ray_start_context from ray.util.client.common import CLIENT_SERVER_MAX_THREADS, GRPC_OPTIONS diff --git a/python/ray/tests/test_client_warnings.py b/python/ray/tests/test_client_warnings.py index cf463eae1289..7e276c8bfe2c 100644 --- a/python/ray/tests/test_client_warnings.py +++ b/python/ray/tests/test_client_warnings.py @@ -1,8 +1,8 @@ import sys import unittest -import pytest import numpy as np +import pytest from ray.util.client.ray_client_helpers import ray_start_client_server from ray.util.debug import _logged diff --git a/python/ray/tests/test_command_runner.py b/python/ray/tests/test_command_runner.py index af6c609cd502..9832698b0d3c 100644 --- a/python/ray/tests/test_command_runner.py +++ b/python/ray/tests/test_command_runner.py @@ -4,14 +4,14 @@ import pytest -from ray.tests.test_autoscaler import MockProvider, MockProcessRunner -from ray.autoscaler.command_runner import CommandRunnerInterface from ray.autoscaler._private.command_runner import ( - SSHCommandRunner, DockerCommandRunner, + SSHCommandRunner, _with_environment_variables, ) +from ray.autoscaler.command_runner import CommandRunnerInterface from ray.autoscaler.sdk import get_docker_host_mount_location +from ray.tests.test_autoscaler import MockProcessRunner, MockProvider auth_config = { "ssh_user": "ray", diff --git a/python/ray/tests/test_component_failures_2.py b/python/ray/tests/test_component_failures_2.py index 59b82315f919..6ea489606386 100644 --- a/python/ray/tests/test_component_failures_2.py +++ b/python/ray/tests/test_component_failures_2.py @@ -5,8 +5,8 @@ import pytest import ray -from ray._common.test_utils import wait_for_condition import ray._private.ray_constants as ray_constants +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import get_other_nodes from ray.cluster_utils import Cluster, cluster_not_supported diff --git a/python/ray/tests/test_component_failures_3.py b/python/ray/tests/test_component_failures_3.py index d8f1cf737b9e..2a3c10d3a048 100644 --- a/python/ray/tests/test_component_failures_3.py +++ b/python/ray/tests/test_component_failures_3.py @@ -1,8 +1,8 @@ import sys import time -import pytest import numpy as np +import pytest import ray import ray._private.ray_constants as ray_constants diff --git a/python/ray/tests/test_concurrency_group.py b/python/ray/tests/test_concurrency_group.py index 2b31d5b0d0f2..4ae7eff96478 100644 --- a/python/ray/tests/test_concurrency_group.py +++ b/python/ray/tests/test_concurrency_group.py @@ -8,9 +8,9 @@ import pytest import ray +from ray._common.test_utils import SignalActor from ray._common.utils import get_or_create_event_loop from ray._private.test_utils import run_string_as_driver -from ray._common.test_utils import SignalActor # This tests the methods are executed in the correct eventloop. diff --git a/python/ray/tests/test_coordinator_server.py b/python/ray/tests/test_coordinator_server.py index 645b2d91c4e0..6d1fae41bc81 100644 --- a/python/ray/tests/test_coordinator_server.py +++ b/python/ray/tests/test_coordinator_server.py @@ -7,28 +7,28 @@ import pytest -from ray.autoscaler.local.coordinator_server import OnPremCoordinatorServer -from ray.autoscaler._private.providers import _NODE_PROVIDERS, _get_node_provider +from ray._common.network_utils import build_address +from ray._common.utils import get_ray_temp_dir from ray.autoscaler._private.local import config as local_config -from ray.autoscaler._private.local.node_provider import LocalNodeProvider -from ray.autoscaler._private.local.node_provider import ( - record_local_head_state_if_needed, -) from ray.autoscaler._private.local.coordinator_node_provider import ( CoordinatorSenderNodeProvider, ) -from ray._common.network_utils import build_address +from ray.autoscaler._private.local.node_provider import ( + LocalNodeProvider, + record_local_head_state_if_needed, +) +from ray.autoscaler._private.providers import _NODE_PROVIDERS, _get_node_provider +from ray.autoscaler.local.coordinator_server import OnPremCoordinatorServer from ray.autoscaler.tags import ( - TAG_RAY_NODE_KIND, + NODE_KIND_HEAD, + NODE_KIND_WORKER, + STATUS_UP_TO_DATE, TAG_RAY_CLUSTER_NAME, + TAG_RAY_NODE_KIND, TAG_RAY_NODE_NAME, - NODE_KIND_WORKER, - NODE_KIND_HEAD, - TAG_RAY_USER_NODE_TYPE, TAG_RAY_NODE_STATUS, - STATUS_UP_TO_DATE, + TAG_RAY_USER_NODE_TYPE, ) -from ray._common.utils import get_ray_temp_dir class OnPremCoordinatorServerTest(unittest.TestCase): diff --git a/python/ray/tests/test_core_worker_fault_tolerance.py b/python/ray/tests/test_core_worker_fault_tolerance.py index 578feab855b6..e66d90b2180a 100644 --- a/python/ray/tests/test_core_worker_fault_tolerance.py +++ b/python/ray/tests/test_core_worker_fault_tolerance.py @@ -1,6 +1,7 @@ -import ray import pytest +import ray + @pytest.mark.parametrize("deterministic_failure", ["request", "response"]) def test_get_object_status_rpc_retry_and_idempotency( diff --git a/python/ray/tests/test_cross_language.py b/python/ray/tests/test_cross_language.py index ca87cca5b3de..99cbf7dec7ce 100644 --- a/python/ray/tests/test_cross_language.py +++ b/python/ray/tests/test_cross_language.py @@ -1,6 +1,7 @@ -import pytest import sys +import pytest + import ray import ray.cluster_utils diff --git a/python/ray/tests/test_dashboard.py b/python/ray/tests/test_dashboard.py index f7d23799304e..4f6d0e958a29 100644 --- a/python/ray/tests/test_dashboard.py +++ b/python/ray/tests/test_dashboard.py @@ -4,15 +4,16 @@ import sys import time -import psutil import pytest -from ray._common.test_utils import wait_for_condition import requests import ray +from ray._common.test_utils import wait_for_condition from ray._private import ray_constants from ray._private.test_utils import run_string_as_driver +import psutil + def search_agents(cluster): all_processes = cluster.head_node.all_processes diff --git a/python/ray/tests/test_dashboard_profiler.py b/python/ray/tests/test_dashboard_profiler.py index 0a261e5f5951..7beee86cfec2 100644 --- a/python/ray/tests/test_dashboard_profiler.py +++ b/python/ray/tests/test_dashboard_profiler.py @@ -1,9 +1,10 @@ -import pytest -import subprocess import os -import requests +import subprocess import sys +import pytest +import requests + import ray from ray._private.test_utils import ( format_web_url, diff --git a/python/ray/tests/test_debug_tools.py b/python/ray/tests/test_debug_tools.py index 6117d07a059d..4a3a60cbf46c 100644 --- a/python/ray/tests/test_debug_tools.py +++ b/python/ray/tests/test_debug_tools.py @@ -6,8 +6,8 @@ import pytest import ray -import ray._private.services as services import ray._private.ray_constants as ray_constants +import ray._private.services as services from ray._common.test_utils import wait_for_condition diff --git a/python/ray/tests/test_distributed_sort.py b/python/ray/tests/test_distributed_sort.py index 036970f39179..6138e469ca2e 100644 --- a/python/ray/tests/test_distributed_sort.py +++ b/python/ray/tests/test_distributed_sort.py @@ -1,6 +1,7 @@ -import pytest import sys +import pytest + from ray.experimental.raysort import main diff --git a/python/ray/tests/test_draining.py b/python/ray/tests/test_draining.py index c94eb8b5a92c..373fca4a8824 100644 --- a/python/ray/tests/test_draining.py +++ b/python/ray/tests/test_draining.py @@ -1,12 +1,13 @@ import sys +import time +from collections import Counter + import pytest import ray -import time -from collections import Counter +from ray._common.test_utils import SignalActor, wait_for_condition from ray._raylet import GcsClient from ray.core.generated import autoscaler_pb2, common_pb2 -from ray._common.test_utils import wait_for_condition, SignalActor from ray.util.scheduling_strategies import ( NodeAffinitySchedulingStrategy, PlacementGroupSchedulingStrategy, diff --git a/python/ray/tests/test_exit_observability.py b/python/ray/tests/test_exit_observability.py index 715b8b9ed2f3..1b74bed2035f 100644 --- a/python/ray/tests/test_exit_observability.py +++ b/python/ray/tests/test_exit_observability.py @@ -9,8 +9,8 @@ from ray._common.test_utils import wait_for_condition from ray._private.state_api_test_utils import verify_failed_task from ray._private.test_utils import run_string_as_driver -from ray.util.state import list_workers, list_nodes, list_tasks from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy +from ray.util.state import list_nodes, list_tasks, list_workers def get_worker_by_pid(pid, detail=True): diff --git a/python/ray/tests/test_experimental_collective.py b/python/ray/tests/test_experimental_collective.py index 2944be7b76c9..4a13813301e1 100644 --- a/python/ray/tests/test_experimental_collective.py +++ b/python/ray/tests/test_experimental_collective.py @@ -1,11 +1,11 @@ -import pytest import sys + +import pytest import torch import ray import ray.experimental.collective - SHAPE = (2, 2) DTYPE = torch.float16 diff --git a/python/ray/tests/test_failure.py b/python/ray/tests/test_failure.py index aa77f870e193..a63553b4539f 100644 --- a/python/ray/tests/test_failure.py +++ b/python/ray/tests/test_failure.py @@ -1,9 +1,9 @@ +import logging import os import signal import sys -import time -import logging import threading +import time import numpy as np import pytest @@ -18,7 +18,7 @@ get_error_message, init_error_pubsub, ) -from ray.exceptions import GetTimeoutError, RayActorError, RayTaskError, ActorDiedError +from ray.exceptions import ActorDiedError, GetTimeoutError, RayActorError, RayTaskError from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy diff --git a/python/ray/tests/test_failure_2.py b/python/ray/tests/test_failure_2.py index 83231a5bd0e4..2543e56b9602 100644 --- a/python/ray/tests/test_failure_2.py +++ b/python/ray/tests/test_failure_2.py @@ -10,15 +10,15 @@ import ray import ray._private.ray_constants as ray_constants import ray._private.utils -from ray._private.ray_constants import DEBUG_AUTOSCALING_ERROR from ray._common.network_utils import parse_address +from ray._common.test_utils import Semaphore, wait_for_condition +from ray._private.ray_constants import DEBUG_AUTOSCALING_ERROR from ray._private.test_utils import ( get_error_message, get_log_batch, init_error_pubsub, run_string_as_driver_nonblocking, ) -from ray._common.test_utils import Semaphore, wait_for_condition from ray.cluster_utils import cluster_not_supported from ray.experimental.internal_kv import _internal_kv_get diff --git a/python/ray/tests/test_failure_3.py b/python/ray/tests/test_failure_3.py index 759c8c38ada6..13facd165098 100644 --- a/python/ray/tests/test_failure_3.py +++ b/python/ray/tests/test_failure_3.py @@ -1,21 +1,22 @@ +import json import os -import sys import signal -import time +import sys import threading -import json +import time from pathlib import Path -import ray import numpy as np import pytest -import psutil +import ray +from ray._common.test_utils import SignalActor, wait_for_condition from ray._private.test_utils import ( - wait_for_pid_to_exit, run_string_as_driver_nonblocking, + wait_for_pid_to_exit, ) -from ray._common.test_utils import SignalActor, wait_for_condition + +import psutil SIGKILL = signal.SIGKILL if sys.platform != "win32" else signal.SIGTERM diff --git a/python/ray/tests/test_failure_4.py b/python/ray/tests/test_failure_4.py index 0c820b623751..7c3dd9552caa 100644 --- a/python/ray/tests/test_failure_4.py +++ b/python/ray/tests/test_failure_4.py @@ -4,24 +4,22 @@ import grpc import numpy as np -import psutil import pytest from grpc._channel import _InactiveRpcError -from ray.util.state import list_tasks -from ray._private.state_api_test_utils import verify_failed_task import ray import ray._private.ray_constants as ray_constants import ray.experimental.internal_kv as internal_kv from ray import NodeID +from ray._common.network_utils import build_address from ray._common.test_utils import SignalActor, wait_for_condition +from ray._private.state_api_test_utils import verify_failed_task from ray._private.test_utils import ( get_error_message, init_error_pubsub, - run_string_as_driver, kill_raylet, + run_string_as_driver, ) -from ray._common.network_utils import build_address from ray.cluster_utils import Cluster, cluster_not_supported from ray.core.generated import ( gcs_service_pb2, @@ -30,6 +28,9 @@ node_manager_pb2_grpc, ) from ray.exceptions import LocalRayletDiedError +from ray.util.state import list_tasks + +import psutil def search_raylet(cluster): diff --git a/python/ray/tests/test_gcs_fault_tolerance.py b/python/ray/tests/test_gcs_fault_tolerance.py index af14adf9a2d9..697bb51fed41 100644 --- a/python/ray/tests/test_gcs_fault_tolerance.py +++ b/python/ray/tests/test_gcs_fault_tolerance.py @@ -3,33 +3,33 @@ import signal import subprocess import sys -import time import tempfile +import time from concurrent.futures import ThreadPoolExecutor from typing import Any -from filelock import FileLock import pytest +from filelock import FileLock import ray -from ray._common.test_utils import wait_for_condition -from ray.autoscaler.v2.sdk import get_cluster_status -from ray.util.placement_group import placement_group -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy import ray._private.gcs_utils as gcs_utils -from ray._private import ray_constants from ray._common.network_utils import parse_address +from ray._common.test_utils import wait_for_condition +from ray._private import ray_constants +from ray._private.runtime_env.plugin import RuntimeEnvPlugin from ray._private.test_utils import ( convert_actor_state, external_redis_test_enabled, generate_system_config_map, - wait_for_pid_to_exit, - run_string_as_driver, redis_sentinel_replicas, + run_string_as_driver, + wait_for_pid_to_exit, ) -from ray.job_submission import JobSubmissionClient, JobStatus from ray._raylet import GcsClient -from ray._private.runtime_env.plugin import RuntimeEnvPlugin +from ray.autoscaler.v2.sdk import get_cluster_status +from ray.job_submission import JobStatus, JobSubmissionClient +from ray.util.placement_group import placement_group +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy from ray.util.state import list_placement_groups import psutil @@ -1008,6 +1008,7 @@ def test_redis_logs(external_redis): # assert "redis_context.cc" not in result.output finally: from click.testing import CliRunner + import ray.scripts.scripts as scripts runner = CliRunner(env={"RAY_USAGE_STATS_PROMPT_ENABLED": "0"}) diff --git a/python/ray/tests/test_gcs_ha_e2e_2.py b/python/ray/tests/test_gcs_ha_e2e_2.py index 2f8896b81dc7..e2b667913efd 100644 --- a/python/ray/tests/test_gcs_ha_e2e_2.py +++ b/python/ray/tests/test_gcs_ha_e2e_2.py @@ -1,6 +1,8 @@ -import pytest import sys from time import sleep + +import pytest + from ray._common.test_utils import wait_for_condition from ray.tests.conftest_docker import * # noqa diff --git a/python/ray/tests/test_gcs_pubsub.py b/python/ray/tests/test_gcs_pubsub.py index 4db3789cd81b..b769cb2a5da9 100644 --- a/python/ray/tests/test_gcs_pubsub.py +++ b/python/ray/tests/test_gcs_pubsub.py @@ -1,13 +1,14 @@ import asyncio +import re import sys import threading -import re + +import pytest import ray from ray._private.gcs_pubsub import ( GcsAioResourceUsageSubscriber, ) -import pytest def test_publish_and_subscribe_error_info(ray_start_regular): diff --git a/python/ray/tests/test_gcs_utils.py b/python/ray/tests/test_gcs_utils.py index 6ada3e99c4ad..7e102c42efca 100644 --- a/python/ray/tests/test_gcs_utils.py +++ b/python/ray/tests/test_gcs_utils.py @@ -6,19 +6,19 @@ import time import pytest -from ray._common.test_utils import async_wait_for_condition import redis import ray -from ray._raylet import GcsClient, NodeID import ray._private.gcs_utils as gcs_utils +import ray._private.ray_constants as ray_constants +from ray._common.network_utils import parse_address +from ray._common.test_utils import async_wait_for_condition from ray._private.test_utils import ( external_redis_test_enabled, find_free_port, generate_system_config_map, ) -from ray._common.network_utils import parse_address -import ray._private.ray_constants as ray_constants +from ray._raylet import GcsClient, NodeID # Import asyncio timeout depends on python version if sys.version_info >= (3, 11): diff --git a/python/ray/tests/test_generators.py b/python/ray/tests/test_generators.py index 417a491c8f30..74f14807cf43 100644 --- a/python/ray/tests/test_generators.py +++ b/python/ray/tests/test_generators.py @@ -1,18 +1,19 @@ -import pytest -import numpy as np +import gc import sys import time -import gc from unittest.mock import Mock +import numpy as np +import pytest + import ray -from ray.util.client.ray_client_helpers import ( - ray_start_client_server_for_address, +from ray._common.test_utils import ( + wait_for_condition, ) from ray._private.client_mode_hook import enable_client_mode from ray.tests.conftest import call_ray_start_context -from ray._common.test_utils import ( - wait_for_condition, +from ray.util.client.ray_client_helpers import ( + ray_start_client_server_for_address, ) diff --git a/python/ray/tests/test_get_or_create_actor.py b/python/ray/tests/test_get_or_create_actor.py index 490d9bc7a9a4..f30d66d654bc 100644 --- a/python/ray/tests/test_get_or_create_actor.py +++ b/python/ray/tests/test_get_or_create_actor.py @@ -1,5 +1,6 @@ -import sys import os +import sys + import pytest import ray diff --git a/python/ray/tests/test_global_gc.py b/python/ray/tests/test_global_gc.py index 204ecf56c601..ad3aca696ebb 100644 --- a/python/ray/tests/test_global_gc.py +++ b/python/ray/tests/test_global_gc.py @@ -9,8 +9,8 @@ import ray import ray.cluster_utils -from ray._private.internal_api import global_gc from ray._common.test_utils import wait_for_condition +from ray._private.internal_api import global_gc logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_global_state.py b/python/ray/tests/test_global_state.py index d786877fb2e4..dcf9654deea5 100644 --- a/python/ray/tests/test_global_state.py +++ b/python/ray/tests/test_global_state.py @@ -1,20 +1,20 @@ import os import sys import time -from typing import Optional, Dict +from typing import Dict, Optional import pytest import ray -from ray._common.test_utils import wait_for_condition import ray._private.gcs_utils as gcs_utils import ray._private.ray_constants -from ray._raylet import GcsClient -from ray.core.generated import autoscaler_pb2 +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( convert_actor_state, make_global_state_accessor, ) +from ray._raylet import GcsClient +from ray.core.generated import autoscaler_pb2 def test_replenish_resources(ray_start_regular): diff --git a/python/ray/tests/test_gpu_objects_gloo.py b/python/ray/tests/test_gpu_objects_gloo.py index 6298dbcac4b8..1349dc09f4da 100644 --- a/python/ray/tests/test_gpu_objects_gloo.py +++ b/python/ray/tests/test_gpu_objects_gloo.py @@ -1,14 +1,15 @@ -import sys import random -import torch -import pytest +import sys import threading -import ray import time -from ray.experimental.collective import create_collective_group + +import pytest +import torch + +import ray +from ray._common.test_utils import SignalActor, wait_for_condition from ray._private.custom_types import TensorTransportEnum -from ray._common.test_utils import wait_for_condition -from ray._common.test_utils import SignalActor +from ray.experimental.collective import create_collective_group # tensordict is not supported on macos ci, so we skip the tests support_tensordict = sys.platform != "darwin" diff --git a/python/ray/tests/test_gpu_objects_nccl.py b/python/ray/tests/test_gpu_objects_nccl.py index f3e8bc409f20..7c2307871733 100644 --- a/python/ray/tests/test_gpu_objects_nccl.py +++ b/python/ray/tests/test_gpu_objects_nccl.py @@ -1,6 +1,8 @@ import sys -import torch + import pytest +import torch + import ray from ray.experimental.collective import create_collective_group diff --git a/python/ray/tests/test_gpu_objects_nixl.py b/python/ray/tests/test_gpu_objects_nixl.py index 3a65e8ea7eb9..1b429ea05fc9 100644 --- a/python/ray/tests/test_gpu_objects_nixl.py +++ b/python/ray/tests/test_gpu_objects_nixl.py @@ -1,6 +1,8 @@ import sys -import torch + import pytest +import torch + import ray diff --git a/python/ray/tests/test_grpc_client_credentials.py b/python/ray/tests/test_grpc_client_credentials.py index 7109ac9fe0dd..910ca0e8a84d 100644 --- a/python/ray/tests/test_grpc_client_credentials.py +++ b/python/ray/tests/test_grpc_client_credentials.py @@ -1,7 +1,7 @@ import sys -import pytest import grpc +import pytest from ray.util.client.worker import Worker diff --git a/python/ray/tests/test_healthcheck.py b/python/ray/tests/test_healthcheck.py index 86e83c40674d..f9eac95143e8 100644 --- a/python/ray/tests/test_healthcheck.py +++ b/python/ray/tests/test_healthcheck.py @@ -5,12 +5,13 @@ import sys import time -import psutil import pytest import ray from ray._private import ray_constants +import psutil + logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_ids.py b/python/ray/tests/test_ids.py index a33407e20c95..024d5677d66c 100644 --- a/python/ray/tests/test_ids.py +++ b/python/ray/tests/test_ids.py @@ -1,17 +1,19 @@ -import sys import os +import sys + +import pytest + from ray import ( + ActorClassID, ActorID, + ClusterID, + FunctionID, JobID, - TaskID, NodeID, - WorkerID, - FunctionID, - ActorClassID, - ClusterID, PlacementGroupID, + TaskID, + WorkerID, ) -import pytest @pytest.mark.parametrize( diff --git a/python/ray/tests/test_iter.py b/python/ray/tests/test_iter.py index 979be1d744ae..3c8f011baf18 100644 --- a/python/ray/tests/test_iter.py +++ b/python/ray/tests/test_iter.py @@ -1,19 +1,20 @@ +import collections import sys import time -import collections from collections import Counter + import pytest import ray +from ray._common.test_utils import Semaphore from ray.util.iter import ( + LocalIterator, + ParallelIteratorWorker, + from_actors, from_items, from_iterators, from_range, - from_actors, - ParallelIteratorWorker, - LocalIterator, ) -from ray._common.test_utils import Semaphore def test_select_shards(ray_start_regular_shared): diff --git a/python/ray/tests/test_job.py b/python/ray/tests/test_job.py index 16e5071381c9..4f2efb9494db 100644 --- a/python/ray/tests/test_job.py +++ b/python/ray/tests/test_job.py @@ -1,28 +1,27 @@ +import json import os +import re import subprocess import sys import tempfile import time -import re -import json - -from subprocess import Popen, PIPE, STDOUT, list2cmdline +from subprocess import PIPE, STDOUT, Popen, list2cmdline from typing import List -import pytest -from ray._common.test_utils import wait_for_condition -import ray.cloudpickle as pickle +import pytest import ray +import ray.cloudpickle as pickle +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( + format_web_url, run_string_as_driver, run_string_as_driver_nonblocking, - format_web_url, wait_for_pid_to_exit, ) +from ray.dashboard.modules.job.pydantic_models import JobDetails from ray.job_config import JobConfig, LoggingConfig from ray.job_submission import JobStatus, JobSubmissionClient -from ray.dashboard.modules.job.pydantic_models import JobDetails def execute_driver(commands: List[str], input: bytes = None): diff --git a/python/ray/tests/test_joblib.py b/python/ray/tests/test_joblib.py index 48b7fffa93d6..8d35e148ce76 100644 --- a/python/ray/tests/test_joblib.py +++ b/python/ray/tests/test_joblib.py @@ -1,30 +1,26 @@ +import os +import pickle import sys import time -import os from unittest import mock import joblib -import pickle -import pytest import numpy as np - +import pytest from sklearn.datasets import load_digits, load_iris -from sklearn.model_selection import RandomizedSearchCV -from sklearn.ensemble import ExtraTreesClassifier -from sklearn.ensemble import RandomForestClassifier -from sklearn.kernel_approximation import Nystroem -from sklearn.kernel_approximation import RBFSampler -from sklearn.pipeline import make_pipeline -from sklearn.svm import LinearSVC, SVC -from sklearn.tree import DecisionTreeClassifier +from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier +from sklearn.kernel_approximation import Nystroem, RBFSampler from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import RandomizedSearchCV, cross_val_score from sklearn.neural_network import MLPClassifier -from sklearn.model_selection import cross_val_score +from sklearn.pipeline import make_pipeline +from sklearn.svm import SVC, LinearSVC +from sklearn.tree import DecisionTreeClassifier import ray +from ray._common.test_utils import wait_for_condition from ray.util.joblib import register_ray from ray.util.joblib.ray_backend import RayBackend -from ray._common.test_utils import wait_for_condition def test_register_ray(): diff --git a/python/ray/tests/test_kill_raylet_signal_log.py b/python/ray/tests/test_kill_raylet_signal_log.py index 28670507ada8..651caa5bb44a 100644 --- a/python/ray/tests/test_kill_raylet_signal_log.py +++ b/python/ray/tests/test_kill_raylet_signal_log.py @@ -1,13 +1,14 @@ import signal import sys -# Import psutil after ray so the packaged version is used. -import psutil import pytest import ray from ray._common.test_utils import wait_for_condition +# Import psutil after ray so the packaged version is used. +import psutil + def get_pid(name): pids = psutil.process_iter() diff --git a/python/ray/tests/test_kill_subprocesses.py b/python/ray/tests/test_kill_subprocesses.py index 9defa7fc5108..e83db14f4780 100644 --- a/python/ray/tests/test_kill_subprocesses.py +++ b/python/ray/tests/test_kill_subprocesses.py @@ -1,14 +1,17 @@ -import ray -import pytest -import multiprocessing -import subprocess -import time -import psutil import logging +import multiprocessing import os +import subprocess import sys +import time + +import pytest + +import ray from ray._common.test_utils import wait_for_condition +import psutil + logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_label_utils.py b/python/ray/tests/test_label_utils.py index 8d6c3226cd22..24c78685e6e2 100644 --- a/python/ray/tests/test_label_utils.py +++ b/python/ray/tests/test_label_utils.py @@ -1,22 +1,22 @@ -from contextlib import contextmanager import json import os import sys import tempfile +from contextlib import contextmanager from typing import ContextManager, Dict, Optional, Union import pytest from ray._private.label_utils import ( + parse_node_labels_from_yaml_file, parse_node_labels_json, parse_node_labels_string, - parse_node_labels_from_yaml_file, - validate_node_labels, validate_label_key, - validate_label_value, validate_label_selector, validate_label_selector_value, + validate_label_value, validate_node_label_syntax, + validate_node_labels, ) diff --git a/python/ray/tests/test_list_actors.py b/python/ray/tests/test_list_actors.py index d937af64fcd2..305749d703c9 100644 --- a/python/ray/tests/test_list_actors.py +++ b/python/ray/tests/test_list_actors.py @@ -1,6 +1,7 @@ -import pytest import sys +import pytest + import ray from ray._common.test_utils import wait_for_condition diff --git a/python/ray/tests/test_list_actors_2.py b/python/ray/tests/test_list_actors_2.py index dca2e27e9685..cc303ff7089a 100644 --- a/python/ray/tests/test_list_actors_2.py +++ b/python/ray/tests/test_list_actors_2.py @@ -1,7 +1,8 @@ import os -import pytest import sys +import pytest + import ray from ray._common.test_utils import wait_for_condition diff --git a/python/ray/tests/test_list_actors_3.py b/python/ray/tests/test_list_actors_3.py index dd3a416459d1..4e1484512882 100644 --- a/python/ray/tests/test_list_actors_3.py +++ b/python/ray/tests/test_list_actors_3.py @@ -1,6 +1,7 @@ -import pytest import sys +import pytest + import ray from ray._private.test_utils import run_string_as_driver diff --git a/python/ray/tests/test_list_actors_4.py b/python/ray/tests/test_list_actors_4.py index 0527de580b2d..e8dc604ac6c1 100644 --- a/python/ray/tests/test_list_actors_4.py +++ b/python/ray/tests/test_list_actors_4.py @@ -1,8 +1,9 @@ import asyncio -import pytest import sys import time +import pytest + import ray from ray._private.test_utils import run_string_as_driver diff --git a/python/ray/tests/test_logging.py b/python/ray/tests/test_logging.py index e3cd9160fccc..837ebd9b8e95 100644 --- a/python/ray/tests/test_logging.py +++ b/python/ray/tests/test_logging.py @@ -1,16 +1,16 @@ import io +import logging import os import re import subprocess import sys import tempfile import time -import logging from collections import Counter, defaultdict from contextlib import redirect_stderr, redirect_stdout from pathlib import Path from typing import Dict, List, Tuple -from unittest.mock import Mock, MagicMock, patch +from unittest.mock import MagicMock, Mock, patch import colorama import pytest @@ -18,6 +18,13 @@ import ray from ray._common.test_utils import wait_for_condition from ray._private import ray_constants +from ray._private.log_monitor import ( + LOG_NAME_UPDATE_INTERVAL_S, + RAY_LOG_MONITOR_MANY_FILES_THRESHOLD, + LogFileInfo, + LogMonitor, + is_proc_alive, +) from ray._private.ray_constants import ( PROCESS_TYPE_DASHBOARD, PROCESS_TYPE_DASHBOARD_AGENT, @@ -26,30 +33,23 @@ PROCESS_TYPE_MONITOR, PROCESS_TYPE_PYTHON_CORE_WORKER, PROCESS_TYPE_PYTHON_CORE_WORKER_DRIVER, - PROCESS_TYPE_RAYLET, PROCESS_TYPE_RAY_CLIENT_SERVER, + PROCESS_TYPE_RAYLET, PROCESS_TYPE_REAPER, PROCESS_TYPE_REDIS_SERVER, PROCESS_TYPE_RUNTIME_ENV_AGENT, PROCESS_TYPE_WORKER, ) -from ray._private.log_monitor import ( - LOG_NAME_UPDATE_INTERVAL_S, - RAY_LOG_MONITOR_MANY_FILES_THRESHOLD, - LogFileInfo, - LogMonitor, - is_proc_alive, -) from ray._private.test_utils import ( get_log_batch, - get_log_message, get_log_data, + get_log_message, init_log_pubsub, run_string_as_driver, ) -from ray.cross_language import java_actor_class -from ray.autoscaler._private.cli_logger import cli_logger from ray._private.worker import print_worker_logs +from ray.autoscaler._private.cli_logger import cli_logger +from ray.cross_language import java_actor_class def set_logging_config(monkeypatch, max_bytes, backup_count): diff --git a/python/ray/tests/test_logging_2.py b/python/ray/tests/test_logging_2.py index 69bc24c8bdd1..11cca61fb1cf 100644 --- a/python/ray/tests/test_logging_2.py +++ b/python/ray/tests/test_logging_2.py @@ -1,10 +1,11 @@ -import logging.config -import pytest -import ray +import json import logging +import logging.config import sys -import json +import pytest + +import ray from ray._private.ray_logging.filters import CoreContextFilter from ray._private.ray_logging.formatters import JSONFormatter, TextFormatter from ray._private.ray_logging.logging_config import LoggingConfig diff --git a/python/ray/tests/test_memory_deadlock.py b/python/ray/tests/test_memory_deadlock.py index 132455b63a34..fc94f19d32f8 100644 --- a/python/ray/tests/test_memory_deadlock.py +++ b/python/ray/tests/test_memory_deadlock.py @@ -4,13 +4,12 @@ import pytest import ray - from ray.tests.test_memory_pressure import ( - allocate_memory, Leaker, + allocate_memory, get_additional_bytes_to_reach_memory_usage_pct, - memory_usage_threshold, memory_monitor_refresh_ms, + memory_usage_threshold, ) diff --git a/python/ray/tests/test_memory_pressure.py b/python/ray/tests/test_memory_pressure.py index 70264a1ca097..864ba040a673 100644 --- a/python/ray/tests/test_memory_pressure.py +++ b/python/ray/tests/test_memory_pressure.py @@ -1,24 +1,21 @@ -from math import ceil import sys import time +from math import ceil +import numpy as np import pytest import ray from ray._common.test_utils import wait_for_condition +from ray._common.utils import get_system_memory from ray._private import ( ray_constants, ) +from ray._private.state_api_test_utils import verify_failed_task from ray._private.test_utils import raw_metrics - -import numpy as np -from ray._common.utils import get_system_memory from ray._private.utils import get_used_memory -from ray._private.state_api_test_utils import verify_failed_task - from ray.util.state.state_manager import StateDataSourceClient - memory_usage_threshold = 0.5 task_oom_retries = 1 memory_monitor_refresh_ms = 100 diff --git a/python/ray/tests/test_memory_scheduling.py b/python/ray/tests/test_memory_scheduling.py index 73876a902705..cc9eb979ca80 100644 --- a/python/ray/tests/test_memory_scheduling.py +++ b/python/ray/tests/test_memory_scheduling.py @@ -1,8 +1,8 @@ import sys import time -import pytest import numpy as np +import pytest import ray from ray._common.test_utils import wait_for_condition diff --git a/python/ray/tests/test_memstat.py b/python/ray/tests/test_memstat.py index 60ac96d71443..9d701441c887 100644 --- a/python/ray/tests/test_memstat.py +++ b/python/ray/tests/test_memstat.py @@ -6,9 +6,8 @@ import pytest import ray +from ray._common.test_utils import Semaphore, wait_for_condition from ray._private.internal_api import memory_summary -from ray._common.test_utils import wait_for_condition -from ray._common.test_utils import Semaphore from ray.cluster_utils import Cluster, cluster_not_supported # RayConfig to enable recording call sites during ObjectRej creations. diff --git a/python/ray/tests/test_metrics.py b/python/ray/tests/test_metrics.py index ecc937fa9453..870b24d86e85 100644 --- a/python/ray/tests/test_metrics.py +++ b/python/ray/tests/test_metrics.py @@ -2,19 +2,20 @@ import platform import sys -import psutil import pytest -from ray._common.test_utils import wait_for_condition import requests import ray +from ray._common.network_utils import build_address +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( - wait_until_succeeded_without_exception, get_node_stats, + wait_until_succeeded_without_exception, ) -from ray._common.network_utils import build_address from ray.core.generated import common_pb2 +import psutil + _WIN32 = os.name == "nt" diff --git a/python/ray/tests/test_metrics_agent.py b/python/ray/tests/test_metrics_agent.py index 9767ba79091d..253386c5ac94 100644 --- a/python/ray/tests/test_metrics_agent.py +++ b/python/ray/tests/test_metrics_agent.py @@ -1,11 +1,10 @@ -import time -import signal import json import os import pathlib -import sys import re -import requests +import signal +import sys +import time import warnings from collections import defaultdict from pprint import pformat @@ -13,35 +12,38 @@ import numpy as np import pytest - +import requests from google.protobuf.timestamp_pb2 import Timestamp + import ray -from ray.dashboard.modules.aggregator.tests.test_aggregator_agent import ( - get_event_aggregator_grpc_stub, -) -from ray.core.generated.common_pb2 import TaskAttempt -from ray.core.generated.events_base_event_pb2 import RayEvent -from ray.core.generated.events_event_aggregator_service_pb2 import ( - AddEventsRequest, - RayEventsData, - TaskEventsMetadata, +from ray._common.network_utils import build_address +from ray._common.test_utils import SignalActor, wait_for_condition +from ray._private.metrics_agent import ( + Gauge as MetricsAgentGauge, + PrometheusServiceDiscoveryWriter, ) -from ray.util.state import list_nodes -from ray._private.metrics_agent import PrometheusServiceDiscoveryWriter -from ray._private.metrics_agent import Gauge as MetricsAgentGauge from ray._private.ray_constants import PROMETHEUS_SERVICE_DISCOVERY_FILE -from ray._common.test_utils import SignalActor, wait_for_condition from ray._private.test_utils import ( fetch_prometheus, fetch_prometheus_metrics, + find_free_port, get_log_batch, raw_metrics, - find_free_port, ) -from ray._common.network_utils import build_address from ray.autoscaler._private.constants import AUTOSCALER_METRIC_PORT +from ray.core.generated.common_pb2 import TaskAttempt +from ray.core.generated.events_base_event_pb2 import RayEvent +from ray.core.generated.events_event_aggregator_service_pb2 import ( + AddEventsRequest, + RayEventsData, + TaskEventsMetadata, +) from ray.dashboard.consts import DASHBOARD_METRIC_PORT +from ray.dashboard.modules.aggregator.tests.test_aggregator_agent import ( + get_event_aggregator_grpc_stub, +) from ray.util.metrics import Counter, Gauge, Histogram, Metric +from ray.util.state import list_nodes os.environ["RAY_event_stats"] = "1" diff --git a/python/ray/tests/test_metrics_agent_2.py b/python/ray/tests/test_metrics_agent_2.py index 7da1ded7a348..cbe859930e5c 100644 --- a/python/ray/tests/test_metrics_agent_2.py +++ b/python/ray/tests/test_metrics_agent_2.py @@ -1,46 +1,45 @@ import random import sys import time - -import pytest - -from ray._common.test_utils import wait_for_condition -import ray._private.prometheus_exporter as prometheus_exporter - from typing import List +import pytest from opencensus.metrics.export.metric_descriptor import MetricDescriptorType -from opencensus.stats.view_manager import ViewManager -from opencensus.stats.stats_recorder import StatsRecorder +from opencensus.metrics.export.value import ValueDouble from opencensus.stats import execution_context -from prometheus_client.core import REGISTRY - - -from ray._private.metrics_agent import Gauge, MetricsAgent, Record, RAY_WORKER_TIMEOUT_S from opencensus.stats.aggregation_data import ( - LastValueAggregationData, - SumAggregationData, CountAggregationData, DistributionAggregationData, + LastValueAggregationData, + SumAggregationData, ) -from opencensus.metrics.export.value import ValueDouble -from ray._private.telemetry.metric_cardinality import WORKER_ID_TAG_KEY +from opencensus.stats.stats_recorder import StatsRecorder +from opencensus.stats.view_manager import ViewManager +from prometheus_client.core import REGISTRY + +import ray._private.prometheus_exporter as prometheus_exporter +from ray._common.test_utils import wait_for_condition from ray._private.metrics_agent import ( + RAY_WORKER_TIMEOUT_S, + Gauge, + MetricsAgent, OpenCensusProxyCollector, OpencensusProxyMetric, + Record, +) +from ray._private.telemetry.metric_cardinality import WORKER_ID_TAG_KEY +from ray._private.test_utils import ( + fetch_prometheus_metrics, + fetch_raw_prometheus, ) +from ray._raylet import WorkerID from ray.core.generated.metrics_pb2 import ( + LabelKey, + LabelValue, Metric, MetricDescriptor, Point, - LabelKey, TimeSeries, - LabelValue, -) -from ray._raylet import WorkerID -from ray._private.test_utils import ( - fetch_prometheus_metrics, - fetch_raw_prometheus, ) diff --git a/python/ray/tests/test_metrics_head.py b/python/ray/tests/test_metrics_head.py index 0a7f6133ab99..88bd84b94204 100644 --- a/python/ray/tests/test_metrics_head.py +++ b/python/ray/tests/test_metrics_head.py @@ -2,10 +2,13 @@ import json import logging import os -import pytest import sys import tempfile +import pytest + +from ray._common.utils import get_ray_temp_dir +from ray._private.ray_constants import SESSION_LATEST from ray.dashboard.modules.metrics.dashboards.default_dashboard_panels import ( DEFAULT_GRAFANA_ROWS, ) @@ -13,9 +16,6 @@ SERVE_GRAFANA_PANELS, ) from ray.tests.conftest import _ray_start -from ray._private.ray_constants import SESSION_LATEST -from ray._common.utils import get_ray_temp_dir - logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_minimal_install.py b/python/ray/tests/test_minimal_install.py index a1da0d51648b..50940b817ffa 100644 --- a/python/ray/tests/test_minimal_install.py +++ b/python/ray/tests/test_minimal_install.py @@ -3,13 +3,13 @@ Tests that are specific to minimal installations. """ -import unittest.mock as mock import itertools -import packaging import os import sys +import unittest.mock as mock from typing import Dict +import packaging import pytest @@ -94,8 +94,7 @@ def test_module_import_with_various_non_minimal_deps(pydantic_version: str): mock_modules[mod] = mock.MagicMock() with mock.patch.dict("sys.modules", mock_modules): - from ray.dashboard.utils import get_all_modules - from ray.dashboard.utils import DashboardHeadModule + from ray.dashboard.utils import DashboardHeadModule, get_all_modules get_all_modules(DashboardHeadModule) diff --git a/python/ray/tests/test_mpi.py b/python/ray/tests/test_mpi.py index e806b3f3e1c5..e204da4191d4 100644 --- a/python/ray/tests/test_mpi.py +++ b/python/ray/tests/test_mpi.py @@ -1,8 +1,10 @@ -import pytest -import ray -import sys import os +import sys + import numpy +import pytest + +import ray from ray.runtime_env import mpi_init @@ -91,9 +93,10 @@ def calc_pi(self): def check_gpu_setup(): - from mpi4py import MPI import os + from mpi4py import MPI + mpi_init() comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/python/ray/tests/test_multi_node.py b/python/ray/tests/test_multi_node.py index 52acfa85bad5..a687c1f2290a 100644 --- a/python/ray/tests/test_multi_node.py +++ b/python/ray/tests/test_multi_node.py @@ -2,7 +2,6 @@ import sys import time -import psutil import pytest import ray @@ -16,6 +15,8 @@ run_string_as_driver_nonblocking, ) +import psutil + @pytest.mark.parametrize( "call_ray_start", diff --git a/python/ray/tests/test_multi_node_3.py b/python/ray/tests/test_multi_node_3.py index f741baa7bef0..cc10937d17e8 100644 --- a/python/ray/tests/test_multi_node_3.py +++ b/python/ray/tests/test_multi_node_3.py @@ -4,24 +4,25 @@ import sys from pathlib import Path -import psutil import pytest import ray import ray._private.ray_constants as ray_constants +from ray._common.test_utils import Semaphore from ray._private.test_utils import ( check_call_ray, check_call_subprocess, kill_process_by_name, - start_redis_instance, run_string_as_driver, run_string_as_driver_nonblocking, + start_redis_instance, wait_for_children_of_pid, wait_for_children_of_pid_to_exit, ) -from ray._common.test_utils import Semaphore from ray._private.utils import detect_fate_sharing_support +import psutil + def test_calling_start_ray_head(call_ray_stop_only): # Test that we can call ray start with various command line diff --git a/python/ray/tests/test_multi_tenancy.py b/python/ray/tests/test_multi_tenancy.py index b4458f8f2d59..608fd5078184 100644 --- a/python/ray/tests/test_multi_tenancy.py +++ b/python/ray/tests/test_multi_tenancy.py @@ -6,8 +6,8 @@ import time from typing import List -import pytest import numpy as np +import pytest import ray from ray._common.test_utils import wait_for_condition diff --git a/python/ray/tests/test_multinode_failures.py b/python/ray/tests/test_multinode_failures.py index 8faa98513a50..30824cae483b 100644 --- a/python/ray/tests/test_multinode_failures.py +++ b/python/ray/tests/test_multinode_failures.py @@ -7,8 +7,8 @@ import ray import ray._private.ray_constants as ray_constants -from ray._private.test_utils import get_other_nodes from ray._common.test_utils import Semaphore +from ray._private.test_utils import get_other_nodes from ray.cluster_utils import Cluster, cluster_not_supported SIGKILL = signal.SIGKILL if sys.platform != "win32" else signal.SIGTERM diff --git a/python/ray/tests/test_multiprocessing.py b/python/ray/tests/test_multiprocessing.py index 947cb173c230..5d1c89ef0d70 100644 --- a/python/ray/tests/test_multiprocessing.py +++ b/python/ray/tests/test_multiprocessing.py @@ -3,6 +3,7 @@ Tests that require a standalone Ray cluster (for example, testing ray.init or shutdown behavior) should go in test_multiprocessing_standalone.py. """ +import multiprocessing as mp import os import platform import queue @@ -10,15 +11,13 @@ import sys import tempfile import time -import multiprocessing as mp from collections import defaultdict import pytest - import ray from ray._common.test_utils import SignalActor -from ray.util.multiprocessing import Pool, TimeoutError, JoinableQueue +from ray.util.multiprocessing import JoinableQueue, Pool, TimeoutError @pytest.fixture(scope="module") diff --git a/python/ray/tests/test_multiprocessing_standalone.py b/python/ray/tests/test_multiprocessing_standalone.py index dec1d956b8f4..1b41f1c08991 100644 --- a/python/ray/tests/test_multiprocessing_standalone.py +++ b/python/ray/tests/test_multiprocessing_standalone.py @@ -3,13 +3,12 @@ Tests that can run on a shared Ray cluster fixture should go in test_multiprocessing.py """ import math +import multiprocessing as mp import os import sys -import multiprocessing as mp import pytest - import ray from ray._private.test_utils import external_redis_test_enabled from ray.util.multiprocessing import Pool diff --git a/python/ray/tests/test_nccl_channel.py b/python/ray/tests/test_nccl_channel.py index 786960999491..555d06cb83a6 100644 --- a/python/ray/tests/test_nccl_channel.py +++ b/python/ray/tests/test_nccl_channel.py @@ -1,23 +1,23 @@ # coding: utf-8 import logging import sys -import torch -from typing import List, Dict, Tuple +from typing import Dict, List, Tuple import pytest +import torch import ray import ray.cluster_utils +from ray._private.test_utils import get_actor_node_id from ray.experimental.channel.conftest import ( Barrier, - start_nccl_mock, TracedChannel, + start_nccl_mock, ) -from ray.experimental.channel.torch_tensor_type import TorchTensorType from ray.experimental.channel.torch_tensor_accelerator_channel import ( _init_communicator, ) -from ray._private.test_utils import get_actor_node_id +from ray.experimental.channel.torch_tensor_type import TorchTensorType logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_network_failure_e2e.py b/python/ray/tests/test_network_failure_e2e.py index 59b1403f8aeb..39c56e4bd03d 100644 --- a/python/ray/tests/test_network_failure_e2e.py +++ b/python/ray/tests/test_network_failure_e2e.py @@ -1,14 +1,14 @@ -import sys import json - +import sys +import threading from time import sleep + import pytest -import threading + from ray._common.test_utils import wait_for_condition from ray.tests.conftest_docker import * # noqa from ray.tests.conftest_docker import gen_head_node, gen_worker_node - SLEEP_TASK_SCRIPTS = """ import ray ray.init() diff --git a/python/ray/tests/test_node_death.py b/python/ray/tests/test_node_death.py index 66d3cad6ed1f..f12abc31e83c 100644 --- a/python/ray/tests/test_node_death.py +++ b/python/ray/tests/test_node_death.py @@ -1,8 +1,8 @@ import sys + import pytest import ray - from ray._common.test_utils import wait_for_condition from ray.core.generated import common_pb2 diff --git a/python/ray/tests/test_node_label_scheduling_strategy.py b/python/ray/tests/test_node_label_scheduling_strategy.py index d13fc587ac08..4b3f0dadb272 100644 --- a/python/ray/tests/test_node_label_scheduling_strategy.py +++ b/python/ray/tests/test_node_label_scheduling_strategy.py @@ -1,13 +1,14 @@ import sys + import pytest import ray from ray.util.scheduling_strategies import ( - In, - NotIn, - Exists, DoesNotExist, + Exists, + In, NodeLabelSchedulingStrategy, + NotIn, ) diff --git a/python/ray/tests/test_node_labels.py b/python/ray/tests/test_node_labels.py index 02e2b9630b85..ae94aadef6d2 100644 --- a/python/ray/tests/test_node_labels.py +++ b/python/ray/tests/test_node_labels.py @@ -1,14 +1,15 @@ import os -import sys -import pytest import subprocess +import sys import tempfile from unittest.mock import patch -from ray._private.accelerators.tpu import TPUAcceleratorManager + +import pytest import ray -from ray.cluster_utils import AutoscalingCluster from ray._common.test_utils import wait_for_condition +from ray._private.accelerators.tpu import TPUAcceleratorManager +from ray.cluster_utils import AutoscalingCluster def check_cmd_stderr(cmd): diff --git a/python/ray/tests/test_node_manager.py b/python/ray/tests/test_node_manager.py index d859ae135286..75fec392d2c0 100644 --- a/python/ray/tests/test_node_manager.py +++ b/python/ray/tests/test_node_manager.py @@ -12,17 +12,16 @@ import ray from ray._common.test_utils import wait_for_condition -from ray.util.state import list_workers +from ray._private.runtime_env.context import RuntimeEnvContext +from ray._private.runtime_env.plugin import RuntimeEnvPlugin from ray._private.test_utils import ( get_load_metrics_report, + get_resource_usage, run_string_as_driver, run_string_as_driver_nonblocking, - get_resource_usage, ) -from ray.util.state import list_objects from ray._private.utils import get_num_cpus -from ray._private.runtime_env.context import RuntimeEnvContext -from ray._private.runtime_env.plugin import RuntimeEnvPlugin +from ray.util.state import list_objects, list_workers # This tests the queue transitions for infeasible tasks. This has been an issue diff --git a/python/ray/tests/test_node_provider_availability_tracker.py b/python/ray/tests/test_node_provider_availability_tracker.py index 448c7500cd2f..9512bc7b2010 100644 --- a/python/ray/tests/test_node_provider_availability_tracker.py +++ b/python/ray/tests/test_node_provider_availability_tracker.py @@ -1,16 +1,16 @@ -import datetime import dataclasses +import datetime import sys + import pytest -from ray.autoscaler.node_launch_exception import NodeLaunchException from ray.autoscaler._private.node_provider_availability_tracker import ( - NodeProviderAvailabilityTracker, - NodeAvailabilitySummary, NodeAvailabilityRecord, + NodeAvailabilitySummary, + NodeProviderAvailabilityTracker, UnavailableNodeInformation, ) - +from ray.autoscaler.node_launch_exception import NodeLaunchException cur_time = float(0) diff --git a/python/ray/tests/test_numba.py b/python/ray/tests/test_numba.py index 182684f1f212..7f2ab2800640 100644 --- a/python/ray/tests/test_numba.py +++ b/python/ray/tests/test_numba.py @@ -1,10 +1,9 @@ -import pytest import sys import unittest - -from numba import njit import numpy as np +import pytest +from numba import njit import ray diff --git a/python/ray/tests/test_object_assign_owner.py b/python/ray/tests/test_object_assign_owner.py index de7a4ca76759..af39e5e65ae4 100644 --- a/python/ray/tests/test_object_assign_owner.py +++ b/python/ray/tests/test_object_assign_owner.py @@ -1,8 +1,8 @@ import sys import time -import pytest import numpy as np +import pytest import ray from ray.exceptions import OwnerDiedError diff --git a/python/ray/tests/test_object_spilling.py b/python/ray/tests/test_object_spilling.py index e219a564cc37..f1a2c9741d99 100644 --- a/python/ray/tests/test_object_spilling.py +++ b/python/ray/tests/test_object_spilling.py @@ -1,35 +1,35 @@ import copy import json +import os import platform import random import sys from datetime import datetime, timedelta -from unittest.mock import patch from pathlib import Path -import os +from unittest.mock import patch -import psutil import numpy as np import pytest - import ray +import ray.remote_function +from ray._common.test_utils import wait_for_condition from ray._private.external_storage import ( + ExternalStorageSmartOpenImpl, + FileSystemStorage, + _get_unique_spill_filename, create_url_with_offset, parse_url_with_offset, - _get_unique_spill_filename, - FileSystemStorage, - ExternalStorageSmartOpenImpl, ) from ray._private.internal_api import memory_summary -from ray._common.test_utils import wait_for_condition -import ray.remote_function from ray.tests.conftest import ( buffer_object_spilling_config, file_system_object_spilling_config, mock_distributed_fs_object_spilling_config, ) +import psutil + # Note: Disk write speed can be as low as 6 MiB/s in AWS Mac instances, so we have to # increase the timeout. pytestmark = [pytest.mark.timeout(900 if platform.system() == "Darwin" else 180)] diff --git a/python/ray/tests/test_object_spilling_2.py b/python/ray/tests/test_object_spilling_2.py index 7828edb96e51..c44404f03e5e 100644 --- a/python/ray/tests/test_object_spilling_2.py +++ b/python/ray/tests/test_object_spilling_2.py @@ -10,12 +10,11 @@ import ray from ray._common.test_utils import wait_for_condition -from ray._private.test_utils import run_string_as_driver -from ray.tests.test_object_spilling import is_dir_empty from ray._private.external_storage import ( FileSystemStorage, ) - +from ray._private.test_utils import run_string_as_driver +from ray.tests.test_object_spilling import is_dir_empty # Note: Disk write speed can be as low as 6 MiB/s in AWS Mac instances, so we have to # increase the timeout. diff --git a/python/ray/tests/test_object_store_metrics.py b/python/ray/tests/test_object_store_metrics.py index c1c5c4eebff2..a2070c4665c2 100644 --- a/python/ray/tests/test_object_store_metrics.py +++ b/python/ray/tests/test_object_store_metrics.py @@ -2,12 +2,12 @@ from collections import defaultdict from typing import Dict +import numpy as np import pytest -from ray._common.test_utils import wait_for_condition import requests -import numpy as np import ray +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( raw_metrics, ) diff --git a/python/ray/tests/test_open_telemetry_metric_recorder.py b/python/ray/tests/test_open_telemetry_metric_recorder.py index e20b24563a6a..81095041a42c 100644 --- a/python/ray/tests/test_open_telemetry_metric_recorder.py +++ b/python/ray/tests/test_open_telemetry_metric_recorder.py @@ -2,12 +2,12 @@ from unittest.mock import MagicMock, patch import pytest -from opentelemetry.metrics import NoOpCounter, NoOpUpDownCounter, NoOpHistogram +from opentelemetry.metrics import NoOpCounter, NoOpHistogram, NoOpUpDownCounter +from ray._private.metrics_agent import Gauge, Record from ray._private.telemetry.open_telemetry_metric_recorder import ( OpenTelemetryMetricRecorder, ) -from ray._private.metrics_agent import Record, Gauge @patch("opentelemetry.metrics.set_meter_provider") diff --git a/python/ray/tests/test_placement_group.py b/python/ray/tests/test_placement_group.py index 8dca2c15f135..6cbc51ca12be 100644 --- a/python/ray/tests/test_placement_group.py +++ b/python/ray/tests/test_placement_group.py @@ -1,19 +1,19 @@ +import os import sys import warnings -import os import pytest import ray -from ray._private.utils import get_ray_doc_version from ray._private.test_utils import placement_group_assert_no_leak -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy +from ray._private.utils import get_ray_doc_version from ray.util.placement_group import ( - validate_placement_group, - _validate_bundles, - _validate_bundle_label_selector, VALID_PLACEMENT_GROUP_STRATEGIES, + _validate_bundle_label_selector, + _validate_bundles, + validate_placement_group, ) +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy def are_pairwise_unique(g): diff --git a/python/ray/tests/test_placement_group_2.py b/python/ray/tests/test_placement_group_2.py index e71d2968c628..a23a3bcba118 100644 --- a/python/ray/tests/test_placement_group_2.py +++ b/python/ray/tests/test_placement_group_2.py @@ -4,9 +4,9 @@ import pytest import ray -from ray._common.test_utils import wait_for_condition import ray._private.gcs_utils as gcs_utils import ray.cluster_utils +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( convert_actor_state, generate_system_config_map, diff --git a/python/ray/tests/test_placement_group_3.py b/python/ray/tests/test_placement_group_3.py index 1b1a234eccce..a4a199a740d5 100644 --- a/python/ray/tests/test_placement_group_3.py +++ b/python/ray/tests/test_placement_group_3.py @@ -6,16 +6,15 @@ import pytest import ray -from ray import ObjectRef -from ray._common.test_utils import wait_for_condition import ray._private.gcs_utils as gcs_utils import ray.cluster_utils import ray.experimental.internal_kv as internal_kv +from ray import ObjectRef +from ray._common.test_utils import wait_for_condition from ray._private.ray_constants import ( DEBUG_AUTOSCALING_ERROR, DEBUG_AUTOSCALING_STATUS, ) -from ray.autoscaler._private.constants import AUTOSCALER_UPDATE_INTERVAL_S from ray._private.test_utils import ( convert_actor_state, generate_system_config_map, @@ -25,6 +24,7 @@ run_string_as_driver, ) from ray.autoscaler._private.commands import debug_status +from ray.autoscaler._private.constants import AUTOSCALER_UPDATE_INTERVAL_S from ray.exceptions import RaySystemError from ray.util.placement_group import placement_group, remove_placement_group from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy diff --git a/python/ray/tests/test_placement_group_4.py b/python/ray/tests/test_placement_group_4.py index dd0b5cbc0c12..9e9364b51056 100644 --- a/python/ray/tests/test_placement_group_4.py +++ b/python/ray/tests/test_placement_group_4.py @@ -1,11 +1,14 @@ -import pytest import os import sys import time +import pytest + import ray -from ray._common.test_utils import wait_for_condition import ray.cluster_utils +from ray._common.test_utils import wait_for_condition +from ray._private.runtime_env.context import RuntimeEnvContext +from ray._private.runtime_env.plugin import RuntimeEnvPlugin from ray._private.test_utils import ( get_other_nodes, is_placement_group_removed, @@ -14,8 +17,6 @@ from ray._raylet import PlacementGroupID from ray.util.placement_group import PlacementGroup from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy -from ray._private.runtime_env.context import RuntimeEnvContext -from ray._private.runtime_env.plugin import RuntimeEnvPlugin MOCK_WORKER_STARTUP_SLOWLY_PLUGIN_CLASS_PATH = ( "ray.tests.test_placement_group_4.MockWorkerStartupSlowlyPlugin" # noqa diff --git a/python/ray/tests/test_placement_group_5.py b/python/ray/tests/test_placement_group_5.py index 7caf7a77c61f..04702b2e7426 100644 --- a/python/ray/tests/test_placement_group_5.py +++ b/python/ray/tests/test_placement_group_5.py @@ -4,19 +4,21 @@ from functools import reduce from itertools import chain -from click.testing import CliRunner import pytest +from click.testing import CliRunner import ray +import ray.scripts.scripts as scripts +from ray._common.network_utils import build_address from ray._common.test_utils import wait_for_condition -from ray._private.test_utils import placement_group_assert_no_leak +from ray._private.runtime_env.plugin import RuntimeEnvPlugin +from ray._private.test_utils import ( + fetch_prometheus_metrics, + placement_group_assert_no_leak, +) from ray.tests.test_placement_group import are_pairwise_unique -from ray.util.state import list_actors, list_placement_groups from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy -from ray._private.runtime_env.plugin import RuntimeEnvPlugin -from ray._private.test_utils import fetch_prometheus_metrics -from ray._common.network_utils import build_address -import ray.scripts.scripts as scripts +from ray.util.state import list_actors, list_placement_groups def test_placement_group_no_resource(ray_start_cluster): diff --git a/python/ray/tests/test_placement_group_failover.py b/python/ray/tests/test_placement_group_failover.py index f50cc99dc7c8..886189f8e1bb 100755 --- a/python/ray/tests/test_placement_group_failover.py +++ b/python/ray/tests/test_placement_group_failover.py @@ -1,12 +1,14 @@ -import pytest import sys -import ray import time -from ray._common.test_utils import wait_for_condition + +import pytest + +import ray import ray.cluster_utils +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import get_other_nodes -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy from ray.util import placement_group_table +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy MB = 1024 * 1024 diff --git a/python/ray/tests/test_placement_group_mini_integration.py b/python/ray/tests/test_placement_group_mini_integration.py index 6c6dc7f9e3e3..e89e6e988eed 100644 --- a/python/ray/tests/test_placement_group_mini_integration.py +++ b/python/ray/tests/test_placement_group_mini_integration.py @@ -1,9 +1,9 @@ -import pytest import sys import time - from random import random +import pytest + import ray import ray.cluster_utils from ray._common.test_utils import wait_for_condition diff --git a/python/ray/tests/test_plasma_unlimited.py b/python/ray/tests/test_plasma_unlimited.py index 934f2e6eaf42..d5e0186f07eb 100644 --- a/python/ray/tests/test_plasma_unlimited.py +++ b/python/ray/tests/test_plasma_unlimited.py @@ -1,21 +1,22 @@ -import numpy as np import json -import random import os +import platform +import random import shutil import sys -import platform -import psutil +import numpy as np import pytest import ray +from ray._common.network_utils import build_address from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( check_spilled_mb, fetch_prometheus, ) -from ray._common.network_utils import build_address + +import psutil MB = 1024 * 1024 diff --git a/python/ray/tests/test_pydantic_serialization.py b/python/ray/tests/test_pydantic_serialization.py index 63310e3f14a9..ef81b8b2c510 100644 --- a/python/ray/tests/test_pydantic_serialization.py +++ b/python/ray/tests/test_pydantic_serialization.py @@ -1,20 +1,20 @@ -from dataclasses import dataclass import logging -from typing import Any, Dict, List, Optional, Type, Tuple import sys -from packaging import version +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Type +import pydantic import pytest from fastapi import FastAPI -import pydantic +from packaging import version try: # Testing with Pydantic 2 - from pydantic import BaseModel as BaseModelV2 - from pydantic.v1 import BaseModel as BaseModelV1 - - from pydantic import ValidationError as ValidationErrorV2 - from pydantic.v1 import ValidationError as ValidationErrorV1 + from pydantic import BaseModel as BaseModelV2, ValidationError as ValidationErrorV2 + from pydantic.v1 import ( + BaseModel as BaseModelV1, + ValidationError as ValidationErrorV1, + ) BASE_MODELS = [BaseModelV1, BaseModelV2] BASE_MODEL_AND_ERRORS = [ @@ -23,16 +23,14 @@ ] except ImportError: # Testing with Pydantic 1 - from pydantic import BaseModel as BaseModelV1 - from pydantic import ValidationError as ValidationErrorV1 + from pydantic import BaseModel as BaseModelV1, ValidationError as ValidationErrorV1 BaseModelV2 = None BASE_MODELS = [BaseModelV1] BASE_MODEL_AND_ERRORS = [(BaseModelV1, ValidationErrorV1)] import ray - -from ray.tests.pydantic_module import User, app, user, closure +from ray.tests.pydantic_module import User, app, closure, user @pytest.fixture(scope="session") diff --git a/python/ray/tests/test_queue.py b/python/ray/tests/test_queue.py index 7f6b2fd95c24..93f0b8ebfd86 100644 --- a/python/ray/tests/test_queue.py +++ b/python/ray/tests/test_queue.py @@ -1,13 +1,13 @@ -import time import sys +import time import pytest import ray from ray._common.test_utils import wait_for_condition +from ray._private.test_utils import BatchQueue from ray.exceptions import GetTimeoutError, RayActorError from ray.util.queue import Empty, Full, Queue -from ray._private.test_utils import BatchQueue # Remote helper functions for testing concurrency diff --git a/python/ray/tests/test_ray_debugger.py b/python/ray/tests/test_ray_debugger.py index 6cf1f164e71d..41eb0ed65f01 100644 --- a/python/ray/tests/test_ray_debugger.py +++ b/python/ray/tests/test_ray_debugger.py @@ -3,17 +3,17 @@ import subprocess import sys import unittest -import pexpect -from pexpect.popen_spawn import PopenSpawn from telnetlib import Telnet from typing import Union +import pexpect import pytest +from pexpect.popen_spawn import PopenSpawn import ray +from ray._common.network_utils import parse_address from ray._common.test_utils import wait_for_condition from ray._private import ray_constants, services -from ray._common.network_utils import parse_address from ray._private.test_utils import run_string_as_driver from ray.cluster_utils import Cluster, cluster_not_supported diff --git a/python/ray/tests/test_ray_init.py b/python/ray/tests/test_ray_init.py index 1d6330bedc03..73e76cb95861 100644 --- a/python/ray/tests/test_ray_init.py +++ b/python/ray/tests/test_ray_init.py @@ -1,13 +1,12 @@ -from concurrent.futures import ThreadPoolExecutor import json import os -import sys -import unittest.mock import signal import subprocess +import sys import tempfile +import unittest.mock +from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from ray._common.network_utils import parse_address, build_address import grpc import pytest @@ -15,14 +14,15 @@ import ray import ray._private.services import ray._private.utils as utils +from ray._common.network_utils import build_address, parse_address +from ray._private import ray_constants +from ray._private.test_utils import external_redis_test_enabled from ray.client_builder import ClientContext from ray.cluster_utils import Cluster +from ray.runtime_env.runtime_env import RuntimeEnv from ray.util.client.common import ClientObjectRef from ray.util.client.ray_client_helpers import ray_start_client_server from ray.util.client.worker import Worker -from ray._private.test_utils import external_redis_test_enabled -from ray._private import ray_constants -from ray.runtime_env.runtime_env import RuntimeEnv @pytest.mark.skipif( diff --git a/python/ray/tests/test_ray_init_2.py b/python/ray/tests/test_ray_init_2.py index 04bc16cdfd16..9fdf2b71eda6 100644 --- a/python/ray/tests/test_ray_init_2.py +++ b/python/ray/tests/test_ray_init_2.py @@ -1,24 +1,24 @@ import logging import os +import shutil import sys -import unittest.mock import tempfile -import shutil +import unittest.mock from unittest.mock import patch import pytest import ray -from ray._common.test_utils import wait_for_condition -from ray._private.ray_constants import RAY_OVERRIDE_DASHBOARD_URL, DEFAULT_RESOURCES import ray._private.services -from ray._private.services import get_node_ip_address from ray._common.network_utils import parse_address -from ray.dashboard.utils import ray_address_to_api_server_url +from ray._common.test_utils import wait_for_condition +from ray._private.ray_constants import DEFAULT_RESOURCES, RAY_OVERRIDE_DASHBOARD_URL +from ray._private.services import get_node_ip_address from ray._private.test_utils import ( get_current_unused_port, run_string_as_driver, ) +from ray.dashboard.utils import ray_address_to_api_server_url from ray.util.client.ray_client_helpers import ray_start_client_server diff --git a/python/ray/tests/test_ray_shutdown.py b/python/ray/tests/test_ray_shutdown.py index a39f6f38eb17..8f2f4298db15 100644 --- a/python/ray/tests/test_ray_shutdown.py +++ b/python/ray/tests/test_ray_shutdown.py @@ -1,21 +1,21 @@ -import sys -import time -import platform +import multiprocessing import os +import platform import signal -import multiprocessing +import sys +import time import pytest -import ray - -import psutil # We must import psutil after ray because we bundle it with ray. +import ray from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( run_string_as_driver_nonblocking, ) -from ray.util.state import get_worker, list_tasks from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy +from ray.util.state import get_worker, list_tasks + +import psutil # We must import psutil after ray because we bundle it with ray. WAIT_TIMEOUT = 20 diff --git a/python/ray/tests/test_reconstruction_2.py b/python/ray/tests/test_reconstruction_2.py index b7c821356ace..42ac99a8baef 100644 --- a/python/ray/tests/test_reconstruction_2.py +++ b/python/ray/tests/test_reconstruction_2.py @@ -2,15 +2,14 @@ import sys import time -import pytest import numpy as np +import pytest import ray import ray._private.ray_constants as ray_constants -from ray._private.internal_api import memory_summary -from ray._common.test_utils import Semaphore, SignalActor -from ray._common.test_utils import wait_for_condition import ray.exceptions +from ray._common.test_utils import Semaphore, SignalActor, wait_for_condition +from ray._private.internal_api import memory_summary from ray.util.state import list_tasks # Task status. diff --git a/python/ray/tests/test_reconstruction_stress_spill.py b/python/ray/tests/test_reconstruction_stress_spill.py index 72fc6307b47c..434553fe9e84 100644 --- a/python/ray/tests/test_reconstruction_stress_spill.py +++ b/python/ray/tests/test_reconstruction_stress_spill.py @@ -1,8 +1,8 @@ import signal import sys -import pytest import numpy as np +import pytest import ray diff --git a/python/ray/tests/test_redis_tls.py b/python/ray/tests/test_redis_tls.py index 1f9ab273b324..c5990981530f 100644 --- a/python/ray/tests/test_redis_tls.py +++ b/python/ray/tests/test_redis_tls.py @@ -1,5 +1,7 @@ -import pytest import sys + +import pytest + import ray from ray._private.test_utils import external_redis_test_enabled diff --git a/python/ray/tests/test_reference_counting.py b/python/ray/tests/test_reference_counting.py index 5999156204d9..031c42d51e73 100644 --- a/python/ray/tests/test_reference_counting.py +++ b/python/ray/tests/test_reference_counting.py @@ -5,10 +5,10 @@ """ # coding: utf-8 import copy +import gc import logging import os import sys -import gc import time import numpy as np diff --git a/python/ray/tests/test_reference_counting_2.py b/python/ray/tests/test_reference_counting_2.py index 4a2b55a6804e..e3658c8d488d 100644 --- a/python/ray/tests/test_reference_counting_2.py +++ b/python/ray/tests/test_reference_counting_2.py @@ -4,9 +4,9 @@ put the test in `test_reference_counting_standalone.py`. """ # coding: utf-8 +import copy import logging import os -import copy import pickle import signal import sys @@ -17,14 +17,14 @@ import pytest import ray +import ray._private.gcs_utils as gcs_utils import ray.cluster_utils -from ray._private.internal_api import memory_summary from ray._common.test_utils import SignalActor, wait_for_condition +from ray._private.internal_api import memory_summary from ray._private.test_utils import ( put_object, wait_for_num_actors, ) -import ray._private.gcs_utils as gcs_utils SIGKILL = signal.SIGKILL if sys.platform != "win32" else signal.SIGTERM diff --git a/python/ray/tests/test_reference_counting_standalone.py b/python/ray/tests/test_reference_counting_standalone.py index e1ab8132dc50..cb3ced7bb9e2 100644 --- a/python/ray/tests/test_reference_counting_standalone.py +++ b/python/ray/tests/test_reference_counting_standalone.py @@ -15,11 +15,11 @@ import ray import ray.cluster_utils -from ray._private.internal_api import memory_summary -from ray._common.test_utils import SignalActor from ray._common.test_utils import ( + SignalActor, wait_for_condition, ) +from ray._private.internal_api import memory_summary logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_resource_demand_scheduler.py b/python/ray/tests/test_resource_demand_scheduler.py index 99f77f2a812b..2dbd2e459a14 100644 --- a/python/ray/tests/test_resource_demand_scheduler.py +++ b/python/ray/tests/test_resource_demand_scheduler.py @@ -1,6 +1,6 @@ import copy -import os import json +import os import shutil import sys import tempfile @@ -8,21 +8,17 @@ import unittest from dataclasses import asdict from datetime import datetime +from functools import partial from time import sleep from unittest import mock -import yaml import pytest +import yaml import ray import ray._private.ray_constants from ray._private.gcs_utils import PlacementGroupTableData from ray._private.test_utils import same_elements -from ray.autoscaler._private.node_provider_availability_tracker import ( - NodeAvailabilityRecord, - NodeAvailabilitySummary, - UnavailableNodeInformation, -) from ray.autoscaler._private.autoscaler import AutoscalerSummary from ray.autoscaler._private.commands import get_or_create_head_node from ray.autoscaler._private.constants import ( @@ -30,15 +26,20 @@ AUTOSCALER_UTILIZATION_SCORER_KEY, ) from ray.autoscaler._private.load_metrics import LoadMetrics +from ray.autoscaler._private.node_provider_availability_tracker import ( + NodeAvailabilityRecord, + NodeAvailabilitySummary, + UnavailableNodeInformation, +) from ray.autoscaler._private.providers import _NODE_PROVIDERS, _clear_provider_cache from ray.autoscaler._private.resource_demand_scheduler import ( ResourceDemandScheduler, _add_min_workers_nodes, - _resource_based_utilization_scorer, _default_utilization_scorer, + _resource_based_utilization_scorer, get_bin_pack_residual, + get_nodes_for as _get, ) -from ray.autoscaler._private.resource_demand_scheduler import get_nodes_for as _get from ray.autoscaler._private.util import ( LoadMetricsSummary, format_info_string, @@ -65,7 +66,6 @@ fill_in_node_ids, mock_node_id, ) -from functools import partial GET_DEFAULT_METHOD = "ray.autoscaler._private.util._get_default_config" diff --git a/python/ray/tests/test_resource_isolation_config.py b/python/ray/tests/test_resource_isolation_config.py index 0f7c49c6abae..f08b4d694b8d 100644 --- a/python/ray/tests/test_resource_isolation_config.py +++ b/python/ray/tests/test_resource_isolation_config.py @@ -1,6 +1,7 @@ -import pytest import sys +import pytest + from ray._private import utils from ray._private.resource_isolation_config import ResourceIsolationConfig diff --git a/python/ray/tests/test_resource_metrics.py b/python/ray/tests/test_resource_metrics.py index 8e5a003cc176..a3e7377229a5 100644 --- a/python/ray/tests/test_resource_metrics.py +++ b/python/ray/tests/test_resource_metrics.py @@ -4,14 +4,12 @@ import pytest import ray - +from ray._common.network_utils import build_address from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( fetch_prometheus_metrics, run_string_as_driver_nonblocking, ) -from ray._common.network_utils import build_address - METRIC_CONFIG = { "_system_config": { diff --git a/python/ray/tests/test_response_cache.py b/python/ray/tests/test_response_cache.py index 21c135af8e48..1a1e9d11f6f2 100644 --- a/python/ray/tests/test_response_cache.py +++ b/python/ray/tests/test_response_cache.py @@ -5,10 +5,10 @@ import pytest from ray.util.client.common import ( - _id_is_newer, - ResponseCache, - OrderedResponseCache, INT32_MAX, + OrderedResponseCache, + ResponseCache, + _id_is_newer, ) diff --git a/python/ray/tests/test_runtime_context.py b/python/ray/tests/test_runtime_context.py index 0d44d1925784..7e358798874e 100644 --- a/python/ray/tests/test_runtime_context.py +++ b/python/ray/tests/test_runtime_context.py @@ -1,15 +1,15 @@ import os import signal -import time import sys +import time import warnings import pytest import ray +from ray._common.test_utils import wait_for_condition from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy from ray.util.state import list_tasks -from ray._common.test_utils import wait_for_condition @pytest.mark.skipif(sys.platform == "win32", reason="Fails on windows") diff --git a/python/ray/tests/test_runtime_env_agent.py b/python/ray/tests/test_runtime_env_agent.py index a3f9c6b27a57..bbd877a417ff 100644 --- a/python/ray/tests/test_runtime_env_agent.py +++ b/python/ray/tests/test_runtime_env_agent.py @@ -1,20 +1,22 @@ -import sys -import pytest import logging import os +import sys import time from typing import List, Tuple +import pytest + import ray from ray._common.test_utils import wait_for_condition -from ray._private.runtime_env.agent.runtime_env_agent import UriType, ReferenceTable from ray._private import ray_constants +from ray._private.runtime_env.agent.runtime_env_agent import ReferenceTable, UriType from ray._private.test_utils import ( get_error_message, init_error_pubsub, ) from ray.core.generated import common_pb2 from ray.runtime_env import RuntimeEnv + import psutil logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_runtime_env_complicated.py b/python/ray/tests/test_runtime_env_complicated.py index 2113dbeb9165..4754ef102a71 100644 --- a/python/ray/tests/test_runtime_env_complicated.py +++ b/python/ray/tests/test_runtime_env_complicated.py @@ -4,38 +4,37 @@ import sys import tempfile import time -from ray._common.test_utils import wait_for_condition -import yaml from pathlib import Path from typing import List from unittest import mock import pytest +import yaml -from ray._common.utils import try_to_create_directory import ray -from ray.runtime_env import RuntimeEnv +from ray._common.test_utils import wait_for_condition +from ray._common.utils import try_to_create_directory from ray._private.runtime_env.conda import ( - inject_dependencies, + _current_py_version, _inject_ray_to_conda_site, _resolve_install_from_source_ray_dependencies, - _current_py_version, + inject_dependencies, ) - from ray._private.runtime_env.conda_utils import ( get_conda_env_list, - get_conda_info_json, get_conda_envs, + get_conda_info_json, ) from ray._private.test_utils import ( + chdir, run_string_as_driver, run_string_as_driver_nonblocking, - chdir, ) from ray._private.utils import ( - get_conda_env_dir, get_conda_bin_executable, + get_conda_env_dir, ) +from ray.runtime_env import RuntimeEnv if not os.environ.get("CI"): # This flags turns on the local development that link against current ray diff --git a/python/ray/tests/test_runtime_env_conda_and_pip.py b/python/ray/tests/test_runtime_env_conda_and_pip.py index 027a582cf833..f656a0a5366a 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip.py @@ -1,29 +1,29 @@ import os -import pytest -import sys import platform +import subprocess +import sys +import tempfile +from pathlib import Path + +import pytest +import yaml + +import ray from ray._common.test_utils import wait_for_condition -from ray._private.test_utils import ( - chdir, - check_local_files_gced, - generate_runtime_env_dict, -) from ray._private.runtime_env import dependency_utils from ray._private.runtime_env.conda import _get_conda_dict_with_ray_inserted from ray._private.runtime_env.dependency_utils import ( INTERNAL_PIP_FILENAME, MAX_INTERNAL_PIP_FILENAME_TRIES, ) +from ray._private.test_utils import ( + chdir, + check_local_files_gced, + generate_runtime_env_dict, +) from ray.runtime_env import RuntimeEnv from ray.util.state import list_tasks -import yaml -import tempfile -from pathlib import Path -import subprocess - -import ray - if not os.environ.get("CI"): # This flags turns on the local development that link against current ray # packages and fall back all the dependencies to current python's site. diff --git a/python/ray/tests/test_runtime_env_conda_and_pip_2.py b/python/ray/tests/test_runtime_env_conda_and_pip_2.py index 0674a3a65a8d..1c22681d7eb9 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip_2.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip_2.py @@ -1,11 +1,12 @@ import os -import pytest import sys from unittest import mock +import pytest + import ray -from ray.exceptions import RuntimeEnvSetupError from ray._private.test_utils import generate_runtime_env_dict +from ray.exceptions import RuntimeEnvSetupError if not os.environ.get("CI"): # This flags turns on the local development that link against current ray diff --git a/python/ray/tests/test_runtime_env_conda_and_pip_3.py b/python/ray/tests/test_runtime_env_conda_and_pip_3.py index 9dee12f6bacb..d330072e3984 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip_3.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip_3.py @@ -1,14 +1,14 @@ import os -import pytest import sys +import pytest + +import ray from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( check_local_files_gced, generate_runtime_env_dict, ) -import ray - if not os.environ.get("CI"): # This flags turns on the local development that link against current ray diff --git a/python/ray/tests/test_runtime_env_conda_and_pip_4.py b/python/ray/tests/test_runtime_env_conda_and_pip_4.py index 69a446cf9de1..08cb20ac56f2 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip_4.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip_4.py @@ -1,10 +1,10 @@ import os -import pytest import sys -from ray._private.runtime_env import virtualenv_utils -import ray +import pytest +import ray +from ray._private.runtime_env import virtualenv_utils if not os.environ.get("CI"): # This flags turns on the local development that link against current ray diff --git a/python/ray/tests/test_runtime_env_conda_and_pip_5.py b/python/ray/tests/test_runtime_env_conda_and_pip_5.py index f5d23143c4fb..889cf9922f8d 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip_5.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip_5.py @@ -1,4 +1,5 @@ import sys + import pytest from packaging.version import parse diff --git a/python/ray/tests/test_runtime_env_container.py b/python/ray/tests/test_runtime_env_container.py index 933a88e2db6a..7b1ad02bbeb1 100644 --- a/python/ray/tests/test_runtime_env_container.py +++ b/python/ray/tests/test_runtime_env_container.py @@ -5,8 +5,7 @@ import ray from ray.tests.conftest import * # noqa from ray.tests.conftest_docker import * # noqa -from ray.tests.conftest_docker import run_in_container, NESTED_IMAGE_NAME - +from ray.tests.conftest_docker import NESTED_IMAGE_NAME, run_in_container # NOTE(zcin): The actual test code are in python scripts under # python/ray/tests/runtime_env_container. The scripts are copied over to diff --git a/python/ray/tests/test_runtime_env_failure.py b/python/ray/tests/test_runtime_env_failure.py index 7a1f904385ea..3df849d7a802 100644 --- a/python/ray/tests/test_runtime_env_failure.py +++ b/python/ray/tests/test_runtime_env_failure.py @@ -3,12 +3,13 @@ from unittest import mock import pytest + +import ray from ray._private.ray_constants import RAY_RUNTIME_ENV_URI_PIN_EXPIRATION_S_DEFAULT from ray._private.runtime_env.packaging import ( RAY_RUNTIME_ENV_FAIL_DOWNLOAD_FOR_TESTING_ENV_VAR, RAY_RUNTIME_ENV_FAIL_UPLOAD_FOR_TESTING_ENV_VAR, ) -import ray from ray.exceptions import RuntimeEnvSetupError diff --git a/python/ray/tests/test_runtime_env_fork_process.py b/python/ray/tests/test_runtime_env_fork_process.py index e7e31b2768c2..35462c7fe70c 100644 --- a/python/ray/tests/test_runtime_env_fork_process.py +++ b/python/ray/tests/test_runtime_env_fork_process.py @@ -1,7 +1,7 @@ # coding: utf-8 +import json import os import sys -import json import pytest diff --git a/python/ray/tests/test_runtime_env_get_wheel_names.py b/python/ray/tests/test_runtime_env_get_wheel_names.py index e124bc9ae676..fbbc735c461c 100644 --- a/python/ray/tests/test_runtime_env_get_wheel_names.py +++ b/python/ray/tests/test_runtime_env_get_wheel_names.py @@ -3,14 +3,13 @@ import pytest import requests +import ray._private.ray_constants as ray_constants from ray._private.utils import ( get_master_wheel_url, get_release_wheel_url, get_wheel_filename, ) -import ray._private.ray_constants as ray_constants - def test_get_wheel_filename(): """Test the code that generates the filenames of the `latest` wheels.""" diff --git a/python/ray/tests/test_runtime_env_packaging.py b/python/ray/tests/test_runtime_env_packaging.py index 4b56a71c9d03..fdaf2450df92 100644 --- a/python/ray/tests/test_runtime_env_packaging.py +++ b/python/ray/tests/test_runtime_env_packaging.py @@ -6,14 +6,14 @@ import sys import tempfile import uuid +import zipfile from filecmp import dircmp from pathlib import Path from shutil import copytree, make_archive, rmtree -import zipfile -import ray import pytest +import ray from ray._private.ray_constants import ( KV_NAMESPACE_PACKAGE, RAY_RUNTIME_ENV_IGNORE_GITIGNORE, @@ -24,12 +24,13 @@ Protocol, _dir_travel, _get_excludes, + _get_gitignore, _store_package_in_gcs, download_and_unpack_package, get_local_dir_from_uri, get_top_level_dir_from_compressed_package, - get_uri_for_file, get_uri_for_directory, + get_uri_for_file, get_uri_for_package, is_whl_uri, is_zip_uri, @@ -37,7 +38,6 @@ remove_dir_from_filepaths, unzip_package, upload_package_if_needed, - _get_gitignore, upload_package_to_gcs, ) from ray.experimental.internal_kv import ( @@ -713,8 +713,8 @@ async def test_download_and_unpack_package_with_file_uri(self): # Add a file to the zip file so we can verify the file was extracted. zip.writestr("file.txt", "Hello, world!") - from urllib.request import pathname2url from urllib.parse import urljoin + from urllib.request import pathname2url # in windows, file_path = ///C:/Users/... # in linux, file_path = /tmp/... diff --git a/python/ray/tests/test_runtime_env_profiler.py b/python/ray/tests/test_runtime_env_profiler.py index ce3e12735e22..39fa8514677f 100644 --- a/python/ray/tests/test_runtime_env_profiler.py +++ b/python/ray/tests/test_runtime_env_profiler.py @@ -1,13 +1,14 @@ -import os import glob +import os +import subprocess import sys from pathlib import Path + import pytest -import subprocess import ray -from ray._private.runtime_env.nsight import parse_nsight_config from ray._common.test_utils import wait_for_condition +from ray._private.runtime_env.nsight import parse_nsight_config from ray.exceptions import RuntimeEnvSetupError diff --git a/python/ray/tests/test_runtime_env_py_executable.py b/python/ray/tests/test_runtime_env_py_executable.py index b9aef67b5999..daf9445e404d 100644 --- a/python/ray/tests/test_runtime_env_py_executable.py +++ b/python/ray/tests/test_runtime_env_py_executable.py @@ -1,9 +1,10 @@ import os -import pytest import sys import tempfile from pathlib import Path +import pytest + import ray diff --git a/python/ray/tests/test_runtime_env_ray_minimal.py b/python/ray/tests/test_runtime_env_ray_minimal.py index d524ecee30a9..64687d87d5ad 100644 --- a/python/ray/tests/test_runtime_env_ray_minimal.py +++ b/python/ray/tests/test_runtime_env_ray_minimal.py @@ -11,6 +11,7 @@ import os import sys + import pytest import ray diff --git a/python/ray/tests/test_runtime_env_setup_func.py b/python/ray/tests/test_runtime_env_setup_func.py index b66e13a47973..218478ee11c8 100644 --- a/python/ray/tests/test_runtime_env_setup_func.py +++ b/python/ray/tests/test_runtime_env_setup_func.py @@ -1,16 +1,16 @@ -import threading +import logging import os +import platform import sys -import logging import tempfile -import platform +import threading import pytest import ray from ray._common.test_utils import wait_for_condition -from ray.job_submission import JobSubmissionClient, JobStatus from ray._private.test_utils import format_web_url +from ray.job_submission import JobStatus, JobSubmissionClient def _hook(): diff --git a/python/ray/tests/test_runtime_env_standalone.py b/python/ray/tests/test_runtime_env_standalone.py index 4cc1dee743c8..41ba317888d6 100644 --- a/python/ray/tests/test_runtime_env_standalone.py +++ b/python/ray/tests/test_runtime_env_standalone.py @@ -22,8 +22,8 @@ get_log_sources, ) from ray.exceptions import RuntimeEnvSetupError -from ray.runtime_env import RuntimeEnv from ray.job_submission import JobStatus, JobSubmissionClient +from ray.runtime_env import RuntimeEnv @pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.") diff --git a/python/ray/tests/test_runtime_env_strong_type.py b/python/ray/tests/test_runtime_env_strong_type.py index 4d1f77bf40da..bc61df4e1988 100644 --- a/python/ray/tests/test_runtime_env_strong_type.py +++ b/python/ray/tests/test_runtime_env_strong_type.py @@ -1,11 +1,12 @@ import sys +from dataclasses import dataclass +from typing import List + import pytest -import ray -from typing import List +import ray from ray.runtime_env import RuntimeEnv from ray.runtime_env.types.pip import Pip -from dataclasses import dataclass @dataclass diff --git a/python/ray/tests/test_runtime_env_uv.py b/python/ray/tests/test_runtime_env_uv.py index 7b3e083a4ea4..74fec59f029e 100644 --- a/python/ray/tests/test_runtime_env_uv.py +++ b/python/ray/tests/test_runtime_env_uv.py @@ -3,13 +3,14 @@ # 2. Options for `uv install`. import os -import pytest import sys import tempfile from pathlib import Path -from ray._private.runtime_env import virtualenv_utils +import pytest + import ray +from ray._private.runtime_env import virtualenv_utils @pytest.fixture(scope="function") diff --git a/python/ray/tests/test_runtime_env_uv_run.py b/python/ray/tests/test_runtime_env_uv_run.py index 1db6424432d3..401d822ad058 100644 --- a/python/ray/tests/test_runtime_env_uv_run.py +++ b/python/ray/tests/test_runtime_env_uv_run.py @@ -1,9 +1,9 @@ import json import os -from pathlib import Path import subprocess import sys import tempfile +from pathlib import Path import pytest from uv import find_uv_bin @@ -14,7 +14,6 @@ wait_until_server_available, ) - PYPROJECT_TOML = """ [project] name = "test" diff --git a/python/ray/tests/test_runtime_env_working_dir.py b/python/ray/tests/test_runtime_env_working_dir.py index 00e799ba0d53..cd7a1e8f8684 100644 --- a/python/ray/tests/test_runtime_env_working_dir.py +++ b/python/ray/tests/test_runtime_env_working_dir.py @@ -193,8 +193,8 @@ def reinit(): @ray.remote def test_import(): - import test_module import file_module + import test_module assert TEST_IMPORT_DIR in os.environ.get("PYTHONPATH", "") return test_module.one(), file_module.hello() @@ -236,8 +236,8 @@ def test_read(): @ray.remote class Actor: def test_import(self): - import test_module import file_module + import test_module assert TEST_IMPORT_DIR in os.environ.get("PYTHONPATH", "") return test_module.one(), file_module.hello() @@ -297,8 +297,8 @@ def reinit(): # Import in the driver. sys.path.insert(0, tmp_working_dir) - import test_module import file_module + import test_module @ray.remote def test_import(): diff --git a/python/ray/tests/test_runtime_env_working_dir_2.py b/python/ray/tests/test_runtime_env_working_dir_2.py index a77e8353d0ac..b9addec4c1da 100644 --- a/python/ray/tests/test_runtime_env_working_dir_2.py +++ b/python/ray/tests/test_runtime_env_working_dir_2.py @@ -1,23 +1,22 @@ import os -from pathlib import Path import sys import tempfile +from pathlib import Path import pytest -from ray._private.test_utils import ( - chdir, - run_string_as_driver, -) - import ray -from ray._private.runtime_env.packaging import GCS_STORAGE_MAX_SIZE -from ray.exceptions import RuntimeEnvSetupError from ray._private.runtime_env.packaging import ( + GCS_STORAGE_MAX_SIZE, get_uri_for_directory, upload_package_if_needed, ) +from ray._private.test_utils import ( + chdir, + run_string_as_driver, +) from ray._private.utils import get_directory_size_bytes +from ray.exceptions import RuntimeEnvSetupError # This test requires you have AWS credentials set up (any AWS credentials will # do, this test only accesses a public bucket). diff --git a/python/ray/tests/test_runtime_env_working_dir_3.py b/python/ray/tests/test_runtime_env_working_dir_3.py index d90fff342a37..a53f7f015010 100644 --- a/python/ray/tests/test_runtime_env_working_dir_3.py +++ b/python/ray/tests/test_runtime_env_working_dir_3.py @@ -8,15 +8,15 @@ import pytest import ray -from ray._common.test_utils import wait_for_condition import ray.experimental.internal_kv as kv +from ray._common.test_utils import wait_for_condition from ray._private.ray_constants import RAY_RUNTIME_ENV_URI_PIN_EXPIRATION_S_ENV_VAR -from ray._private.utils import get_directory_size_bytes from ray._private.test_utils import ( chdir, check_local_files_gced, find_free_port, ) +from ray._private.utils import get_directory_size_bytes # This test requires you have AWS credentials set up (any AWS credentials will # do, this test only accesses a public bucket). @@ -113,8 +113,8 @@ def test_job_level_gc( @ray.remote(num_cpus=1) class A: def test_import(self): - import test_module import pip_install_test # noqa: F401 + import test_module test_module.one() @@ -239,8 +239,8 @@ def test_detached_actor_gc( @ray.remote class A: def test_import(self): - import test_module import pip_install_test # noqa: F401 + import test_module test_module.one() diff --git a/python/ray/tests/test_runtime_env_working_dir_4.py b/python/ray/tests/test_runtime_env_working_dir_4.py index 77e2e81d6bef..f2aa7d9f04bc 100644 --- a/python/ray/tests/test_runtime_env_working_dir_4.py +++ b/python/ray/tests/test_runtime_env_working_dir_4.py @@ -1,12 +1,12 @@ import os -from pathlib import Path import sys +from pathlib import Path import pytest -from ray._common.test_utils import wait_for_condition from pytest_lazy_fixtures import lf as lazy_fixture import ray +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( check_local_files_gced, run_string_as_driver_nonblocking, diff --git a/python/ray/tests/test_scheduling.py b/python/ray/tests/test_scheduling.py index ab27dfe31e6b..786006b06139 100644 --- a/python/ray/tests/test_scheduling.py +++ b/python/ray/tests/test_scheduling.py @@ -12,16 +12,16 @@ import ray import ray.cluster_utils import ray.util.accelerators -from ray._private.internal_api import memory_summary -from ray.util.scheduling_strategies import ( - PlacementGroupSchedulingStrategy, - NodeAffinitySchedulingStrategy, -) from ray._common.test_utils import SignalActor, wait_for_condition +from ray._private.internal_api import memory_summary from ray._private.test_utils import ( - object_memory_usage, - get_metric_check_condition, MetricSamplePattern, + get_metric_check_condition, + object_memory_usage, +) +from ray.util.scheduling_strategies import ( + NodeAffinitySchedulingStrategy, + PlacementGroupSchedulingStrategy, ) logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_scheduling_2.py b/python/ray/tests/test_scheduling_2.py index 64f237c50892..77ee41286be4 100644 --- a/python/ray/tests/test_scheduling_2.py +++ b/python/ray/tests/test_scheduling_2.py @@ -9,17 +9,17 @@ import ray import ray._private.gcs_utils as gcs_utils import ray.experimental.internal_kv as internal_kv +from ray._common.test_utils import SignalActor, wait_for_condition from ray._private.test_utils import ( - make_global_state_accessor, - get_metric_check_condition, MetricSamplePattern, + get_metric_check_condition, + make_global_state_accessor, ) from ray.util.placement_group import placement_group from ray.util.scheduling_strategies import ( NodeAffinitySchedulingStrategy, PlacementGroupSchedulingStrategy, ) -from ray._common.test_utils import SignalActor, wait_for_condition from ray.util.state import list_tasks diff --git a/python/ray/tests/test_shuffle.py b/python/ray/tests/test_shuffle.py index 0a8bbd5be1dc..520024959d32 100644 --- a/python/ray/tests/test_shuffle.py +++ b/python/ray/tests/test_shuffle.py @@ -1,7 +1,8 @@ -import ray -import pytest import sys +import pytest + +import ray from ray.experimental import shuffle diff --git a/python/ray/tests/test_state_api.py b/python/ray/tests/test_state_api.py index 8c451f72b587..56231924e5f9 100644 --- a/python/ray/tests/test_state_api.py +++ b/python/ray/tests/test_state_api.py @@ -1,91 +1,94 @@ -import os -import time import json -import sys +import os import signal +import sys +import time from collections import Counter from concurrent.futures import ThreadPoolExecutor from typing import List -from unittest.mock import MagicMock, AsyncMock, patch -import yaml +from unittest.mock import AsyncMock, MagicMock, patch -from click.testing import CliRunner import pytest import pytest_asyncio -from ray._private.state_api_test_utils import ( - get_state_api_manager, - create_api_options, - verify_schema, -) -from ray.util.state import get_job -from ray.dashboard.modules.job.pydantic_models import JobDetails -from ray.util.state.common import Humanify +import yaml +from click.testing import CliRunner -from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy import ray -import ray.dashboard.consts as dashboard_consts -import ray._private.state as global_state import ray._private.ray_constants as ray_constants -from ray._raylet import GcsClient, ActorID, JobID, TaskID +import ray._private.state as global_state +import ray.dashboard.consts as dashboard_consts from ray._common.network_utils import parse_address -from ray._private.test_utils import ( - run_string_as_driver, - find_free_port, -) from ray._common.test_utils import ( SignalActor, async_wait_for_condition, wait_for_condition, ) +from ray._private.state_api_test_utils import ( + create_api_options, + get_state_api_manager, + verify_schema, +) +from ray._private.test_utils import ( + find_free_port, + run_string_as_driver, +) +from ray._raylet import ActorID, GcsClient, JobID, NodeID, TaskID from ray.cluster_utils import cluster_not_supported -from ray._raylet import NodeID from ray.core.generated.common_pb2 import ( Address, CoreWorkerStats, ObjectRefInfo, TaskInfoEntry, TaskStatus, - WorkerType, TaskType, + WorkerType, ) -from ray.core.generated.gcs_service_pb2_grpc import TaskInfoGcsServiceStub from ray.core.generated.gcs_pb2 import ( - TaskEvents, - TaskStateUpdate, ActorTableData, GcsNodeInfo, PlacementGroupTableData, + TaskEvents, + TaskStateUpdate, WorkerTableData, ) from ray.core.generated.gcs_service_pb2 import ( FilterPredicate, GcsStatus, - GetTaskEventsReply, GetAllActorInfoReply, GetAllNodeInfoReply, GetAllPlacementGroupReply, GetAllWorkerInfoReply, + GetTaskEventsReply, ) +from ray.core.generated.gcs_service_pb2_grpc import TaskInfoGcsServiceStub from ray.core.generated.node_manager_pb2 import GetObjectsInfoReply from ray.core.generated.reporter_pb2 import ListLogsReply, StreamLogReply from ray.core.generated.runtime_env_agent_pb2 import GetRuntimeEnvsInfoReply from ray.core.generated.runtime_env_common_pb2 import ( RuntimeEnvState as RuntimeEnvStateProto, ) +from ray.dashboard.modules.job.pydantic_models import JobDetails from ray.dashboard.state_aggregator import ( GCS_QUERY_FAILURE_WARNING, NODE_QUERY_FAILURE_WARNING, StateAPIManager, ) from ray.dashboard.state_api_utils import convert_filters_type +from ray.dashboard.utils import ray_address_to_api_server_url +from ray.job_submission import JobSubmissionClient +from ray.runtime_env import RuntimeEnv +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy from ray.util.state import ( + StateApiClient, get_actor, + get_job, get_node, get_objects, get_placement_group, get_task, get_worker, list_actors, + list_cluster_events, list_jobs, list_nodes, list_objects, @@ -96,37 +99,33 @@ summarize_actors, summarize_objects, summarize_tasks, - list_cluster_events, - StateApiClient, ) from ray.util.state.common import ( DEFAULT_RPC_TIMEOUT, ActorState, + GetApiOptions, + Humanify, ListApiOptions, - SummaryApiOptions, NodeState, ObjectState, PlacementGroupState, RuntimeEnvState, + StateSchema, + SummaryApiOptions, TaskState, WorkerState, - StateSchema, state_column, - GetApiOptions, ) -from ray.dashboard.utils import ray_address_to_api_server_url from ray.util.state.exception import DataSourceUnavailable, RayStateApiException from ray.util.state.state_cli import ( AvailableFormat, - format_list_api_output, _parse_filter, + format_list_api_output, + ray_get, + ray_list, summary_state_cli_group, ) -from ray.util.state.state_cli import ray_get -from ray.util.state.state_cli import ray_list from ray.util.state.state_manager import StateDataSourceClient -from ray.job_submission import JobSubmissionClient -from ray.runtime_env import RuntimeEnv """ Unit tests @@ -3460,9 +3459,9 @@ def verify(): def test_state_api_server_enforce_concurrent_http_requests( api_func, monkeypatch, shutdown_only ): - import time - import threading import queue + import threading + import time # Set environment with monkeypatch.context() as m: diff --git a/python/ray/tests/test_state_api_2.py b/python/ray/tests/test_state_api_2.py index c8aa4095d458..2659e6009c07 100644 --- a/python/ray/tests/test_state_api_2.py +++ b/python/ray/tests/test_state_api_2.py @@ -2,25 +2,24 @@ import json import os import sys -from pathlib import Path import tempfile - from collections import defaultdict -from ray._private.test_utils import check_call_subprocess +from pathlib import Path -import ray -import requests import pytest +import requests +import ray +from ray._common.test_utils import wait_for_condition from ray._private.profiling import chrome_tracing_dump +from ray._private.test_utils import check_call_subprocess from ray.util.state import ( get_actor, - list_tasks, list_actors, - list_workers, list_nodes, + list_tasks, + list_workers, ) -from ray._common.test_utils import wait_for_condition def test_timeline(shutdown_only): diff --git a/python/ray/tests/test_state_api_log.py b/python/ray/tests/test_state_api_log.py index 61554b3b90a3..69a6835101cc 100644 --- a/python/ray/tests/test_state_api_log.py +++ b/python/ray/tests/test_state_api_log.py @@ -2,48 +2,46 @@ import json import os import sys +import urllib from pathlib import Path from typing import List -from unittest.mock import MagicMock, AsyncMock +from unittest.mock import AsyncMock, MagicMock import grpc -from ray._common.test_utils import wait_for_condition -import requests import pytest -import urllib +import requests from click.testing import CliRunner import ray +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( format_web_url, wait_until_server_available, ) -from ray.util.state.state_cli import logs_state_cli_group -from ray.util.state import list_jobs - from ray._raylet import ActorID, NodeID, TaskID, WorkerID from ray.core.generated.common_pb2 import Address -from ray.core.generated.gcs_service_pb2 import GetTaskEventsReply -from ray.core.generated.reporter_pb2 import ListLogsReply, StreamLogReply from ray.core.generated.gcs_pb2 import ( ActorTableData, TaskEvents, - TaskStateUpdate, TaskLogInfo, + TaskStateUpdate, ) +from ray.core.generated.gcs_service_pb2 import GetTaskEventsReply +from ray.core.generated.reporter_pb2 import ListLogsReply, StreamLogReply from ray.dashboard.modules.log.log_agent import ( - find_offset_of_content_in_file, + LogAgentV1Grpc, + _stream_log_in_chunk, find_end_offset_file, find_end_offset_next_n_lines_from_offset, + find_offset_of_content_in_file, find_start_offset_last_n_lines_from_offset, - LogAgentV1Grpc, ) -from ray.dashboard.modules.log.log_agent import _stream_log_in_chunk from ray.dashboard.modules.log.log_manager import LogsManager from ray.dashboard.tests.conftest import * # noqa -from ray.util.state import get_log, list_logs, list_nodes, list_workers +from ray.util.state import get_log, list_jobs, list_logs, list_nodes, list_workers from ray.util.state.common import GetLogOptions from ray.util.state.exception import RayStateApiException +from ray.util.state.state_cli import logs_state_cli_group from ray.util.state.state_manager import StateDataSourceClient diff --git a/python/ray/tests/test_state_api_summary.py b/python/ray/tests/test_state_api_summary.py index 8ad65e8021c2..59303828e276 100644 --- a/python/ray/tests/test_state_api_summary.py +++ b/python/ray/tests/test_state_api_summary.py @@ -1,43 +1,42 @@ -import time import json -import pytest -import ray -from unittest.mock import AsyncMock import random import sys -from dataclasses import asdict +import time from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict +from unittest.mock import AsyncMock -from ray.util.state import ( - summarize_tasks, - summarize_actors, - summarize_objects, -) -from ray._common.test_utils import wait_for_condition -from ray._raylet import ActorID, TaskID, ObjectID +import pytest +from click.testing import CliRunner +import ray +from ray._common.test_utils import wait_for_condition +from ray._raylet import ActorID, ObjectID, TaskID from ray.core.generated.common_pb2 import TaskStatus, TaskType, WorkerType +from ray.core.generated.gcs_pb2 import ActorTableData, GcsNodeInfo +from ray.core.generated.gcs_service_pb2 import GetAllActorInfoReply, GetAllNodeInfoReply from ray.core.generated.node_manager_pb2 import GetObjectsInfoReply -from ray.core.generated.gcs_pb2 import GcsNodeInfo +from ray.dashboard.state_aggregator import StateAPIManager from ray.tests.test_state_api import ( - generate_task_data, - generate_task_event, generate_actor_data, generate_object_info, + generate_task_data, + generate_task_event, +) +from ray.util.state import ( + summarize_actors, + summarize_objects, + summarize_tasks, ) from ray.util.state.common import ( DEFAULT_RPC_TIMEOUT, - SummaryApiOptions, + DRIVER_TASK_ID_PREFIX, Link, NestedTaskSummary, + SummaryApiOptions, TaskSummaries, - DRIVER_TASK_ID_PREFIX, ) -from ray.core.generated.gcs_service_pb2 import GetAllActorInfoReply, GetAllNodeInfoReply -from ray.core.generated.gcs_pb2 import ActorTableData -from click.testing import CliRunner from ray.util.state.state_cli import summary_state_cli_group -from ray.dashboard.state_aggregator import StateAPIManager from ray.util.state.state_manager import StateDataSourceClient diff --git a/python/ray/tests/test_streaming_generator.py b/python/ray/tests/test_streaming_generator.py index 2e166e190ba5..d3d85fec0559 100644 --- a/python/ray/tests/test_streaming_generator.py +++ b/python/ray/tests/test_streaming_generator.py @@ -1,19 +1,19 @@ import asyncio -import pytest -import numpy as np +import gc import sys -import time import threading -import gc +import time +from unittest.mock import Mock, patch -from unittest.mock import patch, Mock +import numpy as np +import pytest import ray from ray._common.test_utils import wait_for_condition -from ray.experimental.state.api import list_objects from ray._raylet import ObjectRefGenerator, ObjectRefStreamEndOfStreamError from ray.cloudpickle import dumps from ray.exceptions import WorkerCrashedError +from ray.experimental.state.api import list_objects class MockedWorker: diff --git a/python/ray/tests/test_streaming_generator_2.py b/python/ray/tests/test_streaming_generator_2.py index f7cdc4d5d704..517c6f744c78 100644 --- a/python/ray/tests/test_streaming_generator_2.py +++ b/python/ray/tests/test_streaming_generator_2.py @@ -1,16 +1,17 @@ import asyncio -import pytest -import numpy as np +import gc import sys import time -import gc + +import numpy as np +import pytest import ray -from ray.experimental.state.api import list_actors -from ray._common.test_utils import SignalActor from ray._common.test_utils import ( + SignalActor, wait_for_condition, ) +from ray.experimental.state.api import list_actors RECONSTRUCTION_CONFIG = { "health_check_failure_threshold": 10, diff --git a/python/ray/tests/test_streaming_generator_3.py b/python/ray/tests/test_streaming_generator_3.py index c16d03c0b9c3..14c66f286f8f 100644 --- a/python/ray/tests/test_streaming_generator_3.py +++ b/python/ray/tests/test_streaming_generator_3.py @@ -1,11 +1,11 @@ import asyncio -import pytest -import numpy as np import sys import time - from collections import Counter +import numpy as np +import pytest + import ray from ray._raylet import ObjectRefGenerator from ray.exceptions import TaskCancelledError diff --git a/python/ray/tests/test_streaming_generator_4.py b/python/ray/tests/test_streaming_generator_4.py index f9b378e00a6b..45c31c525e73 100644 --- a/python/ray/tests/test_streaming_generator_4.py +++ b/python/ray/tests/test_streaming_generator_4.py @@ -1,13 +1,14 @@ -import pytest -import numpy as np -import sys -import time +import asyncio import gc import os -import signal import random -import asyncio +import signal +import sys +import time from typing import Optional + +import numpy as np +import pytest from pydantic import BaseModel import ray diff --git a/python/ray/tests/test_streaming_generator_backpressure.py b/python/ray/tests/test_streaming_generator_backpressure.py index 0dc220913b82..6d2a3b3d76e5 100644 --- a/python/ray/tests/test_streaming_generator_backpressure.py +++ b/python/ray/tests/test_streaming_generator_backpressure.py @@ -1,10 +1,11 @@ import asyncio -import pytest -import numpy as np +import os +import signal import sys import time -import signal -import os + +import numpy as np +import pytest import ray from ray._common.test_utils import wait_for_condition diff --git a/python/ray/tests/test_stress.py b/python/ray/tests/test_stress.py index 0ff6559094b5..afbc6f40cbba 100644 --- a/python/ray/tests/test_stress.py +++ b/python/ray/tests/test_stress.py @@ -1,8 +1,8 @@ -import time import sys +import time -import pytest import numpy as np +import pytest import ray from ray.cluster_utils import Cluster, cluster_not_supported diff --git a/python/ray/tests/test_symmetric_run.py b/python/ray/tests/test_symmetric_run.py index 524e4c52b6a1..dfbaac359eb4 100644 --- a/python/ray/tests/test_symmetric_run.py +++ b/python/ray/tests/test_symmetric_run.py @@ -1,9 +1,11 @@ -import ray import sys -import pytest from contextlib import contextmanager -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch + +import pytest from click.testing import CliRunner + +import ray import ray.scripts.scripts as scripts diff --git a/python/ray/tests/test_task_events.py b/python/ray/tests/test_task_events.py index b2372db6117d..943bffecf264 100644 --- a/python/ray/tests/test_task_events.py +++ b/python/ray/tests/test_task_events.py @@ -1,25 +1,24 @@ +import sys +import threading +import time from collections import defaultdict from typing import Dict import pytest -import sys -import threading -import time + +import ray from ray._common.test_utils import wait_for_condition from ray._private.state_api_test_utils import ( verify_failed_task, ) -from ray.exceptions import RuntimeEnvSetupError -from ray.runtime_env import RuntimeEnv - -import ray from ray._private.test_utils import ( raw_metrics, run_string_as_driver_nonblocking, ) -from ray.util.state import list_tasks - from ray._private.worker import RayContext +from ray.exceptions import RuntimeEnvSetupError +from ray.runtime_env import RuntimeEnv +from ray.util.state import list_tasks _SYSTEM_CONFIG = { "task_events_report_interval_ms": 100, diff --git a/python/ray/tests/test_task_events_2.py b/python/ray/tests/test_task_events_2.py index 4bc5fefd57b2..5320aeef1d5a 100644 --- a/python/ray/tests/test_task_events_2.py +++ b/python/ray/tests/test_task_events_2.py @@ -1,23 +1,23 @@ import asyncio -from collections import defaultdict import os -from typing import Dict -import pytest import sys import time -from ray._common.test_utils import async_wait_for_condition, wait_for_condition -from ray._private import ray_constants +from collections import defaultdict from functools import reduce +from typing import Dict + +import pytest import ray +from ray._common.test_utils import async_wait_for_condition, wait_for_condition +from ray._private import ray_constants from ray._private.state_api_test_utils import ( PidActor, + _is_actor_task_running, get_state_api_manager, - verify_tasks_running_or_terminated, verify_failed_task, - _is_actor_task_running, + verify_tasks_running_or_terminated, ) -from ray.util.state.common import ListApiOptions, StateResource from ray._private.test_utils import ( run_string_as_driver, run_string_as_driver_nonblocking, @@ -28,6 +28,8 @@ list_jobs, list_tasks, ) +from ray.util.state.common import ListApiOptions, StateResource + import psutil _SYSTEM_CONFIG = { diff --git a/python/ray/tests/test_task_events_3.py b/python/ray/tests/test_task_events_3.py index 19ca96468585..0f6d3b100043 100644 --- a/python/ray/tests/test_task_events_3.py +++ b/python/ray/tests/test_task_events_3.py @@ -1,6 +1,7 @@ -import pytest import sys +import pytest + import ray from ray._common.test_utils import ( wait_for_condition, diff --git a/python/ray/tests/test_task_metrics.py b/python/ray/tests/test_task_metrics.py index 05da74f917ae..c57525914fe9 100644 --- a/python/ray/tests/test_task_metrics.py +++ b/python/ray/tests/test_task_metrics.py @@ -15,7 +15,6 @@ wait_for_assertion, ) - METRIC_CONFIG = { "_system_config": { "metrics_report_interval_ms": 100, diff --git a/python/ray/tests/test_task_metrics_reconstruction.py b/python/ray/tests/test_task_metrics_reconstruction.py index cb219075a1f3..b066f5b9a498 100644 --- a/python/ray/tests/test_task_metrics_reconstruction.py +++ b/python/ray/tests/test_task_metrics_reconstruction.py @@ -4,11 +4,10 @@ import pytest import ray - -from ray.tests.test_task_metrics import tasks_by_all, METRIC_CONFIG from ray._common.test_utils import ( wait_for_condition, ) +from ray.tests.test_task_metrics import METRIC_CONFIG, tasks_by_all # Copied from similar test in test_reconstruction_2.py. diff --git a/python/ray/tests/test_tls_auth.py b/python/ray/tests/test_tls_auth.py index 0ed5a1e622b6..21c230c278a4 100644 --- a/python/ray/tests/test_tls_auth.py +++ b/python/ray/tests/test_tls_auth.py @@ -1,8 +1,8 @@ # coding: utf-8 import logging import os -import sys import subprocess +import sys import pytest diff --git a/python/ray/tests/test_top_level_api.py b/python/ray/tests/test_top_level_api.py index 1b30a9f698d2..9858fcbe4f9c 100644 --- a/python/ray/tests/test_top_level_api.py +++ b/python/ray/tests/test_top_level_api.py @@ -1,5 +1,5 @@ -from inspect import getmembers, isfunction, ismodule import sys +from inspect import getmembers, isfunction, ismodule import pytest diff --git a/python/ray/tests/test_tqdm.py b/python/ray/tests/test_tqdm.py index 9e7cc5639e0a..7b0bdbf4ec5f 100644 --- a/python/ray/tests/test_tqdm.py +++ b/python/ray/tests/test_tqdm.py @@ -4,8 +4,8 @@ import pytest import ray -from ray.experimental import tqdm_ray from ray._common.test_utils import wait_for_condition +from ray.experimental import tqdm_ray def test_distributed_tqdm_remote(): diff --git a/python/ray/tests/test_traceback.py b/python/ray/tests/test_traceback.py index 0d3de67cda18..4db33239b4b4 100644 --- a/python/ray/tests/test_traceback.py +++ b/python/ray/tests/test_traceback.py @@ -5,7 +5,7 @@ import pytest import ray -from ray.exceptions import RayTaskError, RayActorError +from ray.exceptions import RayActorError, RayTaskError """This module tests stacktrace of Ray. diff --git a/python/ray/tests/test_typing.py b/python/ray/tests/test_typing.py index 46d5726b32f5..b049fdd77532 100644 --- a/python/ray/tests/test_typing.py +++ b/python/ray/tests/test_typing.py @@ -6,7 +6,6 @@ import mypy.api as mypy_api import pytest - # Paths are relative to the directory where Bazel is run in the CI TYPING_GOOD_PATH = "python/ray/tests/typing_files/check_typing_good.py" TYPING_BAD_PATH = "python/ray/tests/typing_files/check_typing_bad.py" diff --git a/python/ray/tests/test_unavailable_actors.py b/python/ray/tests/test_unavailable_actors.py index 3c19a4028c46..a2e4ebde25a4 100644 --- a/python/ray/tests/test_unavailable_actors.py +++ b/python/ray/tests/test_unavailable_actors.py @@ -1,13 +1,13 @@ import os -import pytest -import sys import signal +import sys from typing import Optional, Tuple +import pytest + import ray -from ray.exceptions import ActorUnavailableError, ActorDiedError -from ray._common.test_utils import SignalActor -from ray._common.test_utils import wait_for_condition +from ray._common.test_utils import SignalActor, wait_for_condition +from ray.exceptions import ActorDiedError, ActorUnavailableError import psutil # We must import psutil after ray because we bundle it with ray. diff --git a/python/ray/tests/test_util_helpers.py b/python/ray/tests/test_util_helpers.py index 265b6931f528..5d35fc9bc160 100644 --- a/python/ray/tests/test_util_helpers.py +++ b/python/ray/tests/test_util_helpers.py @@ -1,8 +1,10 @@ -import pytest import sys + +import pytest + import ray -from ray.util import as_completed, map_unordered from ray._common.test_utils import SignalActor +from ray.util import as_completed, map_unordered @pytest.fixture(scope="module") diff --git a/python/ray/tests/test_utils.py b/python/ray/tests/test_utils.py index 543a53f9c3e6..7ea8d2c02126 100644 --- a/python/ray/tests/test_utils.py +++ b/python/ray/tests/test_utils.py @@ -5,14 +5,15 @@ This currently expects to work for minimal installs. """ import logging -import pytest import sys -from unittest.mock import patch, mock_open +from unittest.mock import mock_open, patch + +import pytest from ray._private.utils import ( + get_current_node_cpu_model_name, parse_pg_formatted_resources_to_original, try_import_each_module, - get_current_node_cpu_model_name, ) logger = logging.getLogger(__name__) diff --git a/python/ray/tests/test_wait.py b/python/ray/tests/test_wait.py index 6040bae86841..9a37b9fdb40d 100644 --- a/python/ray/tests/test_wait.py +++ b/python/ray/tests/test_wait.py @@ -1,13 +1,13 @@ # coding: utf-8 -import pytest -import numpy as np -import time import logging import sys +import time -from ray._private.test_utils import client_test_enabled +import numpy as np +import pytest +from ray._private.test_utils import client_test_enabled if client_test_enabled(): from ray.util.client import ray diff --git a/python/ray/tests/test_widgets.py b/python/ray/tests/test_widgets.py index 1c5273aa4a32..226568f944b9 100644 --- a/python/ray/tests/test_widgets.py +++ b/python/ray/tests/test_widgets.py @@ -6,7 +6,7 @@ import pytest import ray -from ray.widgets.util import repr_with_fallback, _can_display_ipywidgets +from ray.widgets.util import _can_display_ipywidgets, repr_with_fallback @pytest.fixture diff --git a/python/ray/tests/test_worker_capping.py b/python/ray/tests/test_worker_capping.py index 58b499ee7608..8861c7204176 100644 --- a/python/ray/tests/test_worker_capping.py +++ b/python/ray/tests/test_worker_capping.py @@ -1,10 +1,11 @@ import asyncio import os -import pytest import sys import tempfile import time +import pytest + import ray from ray._common.test_utils import Semaphore diff --git a/python/ray/tests/test_worker_graceful_shutdown.py b/python/ray/tests/test_worker_graceful_shutdown.py index 7ac893beb06f..56eb5ab4d0c4 100644 --- a/python/ray/tests/test_worker_graceful_shutdown.py +++ b/python/ray/tests/test_worker_graceful_shutdown.py @@ -6,8 +6,7 @@ import pytest import ray -from ray._common.test_utils import SignalActor -from ray._common.test_utils import wait_for_condition +from ray._common.test_utils import SignalActor, wait_for_condition @pytest.mark.skipif( diff --git a/python/ray/tests/test_worker_state.py b/python/ray/tests/test_worker_state.py index 5a93aefb5c84..4fa4ab6d38da 100644 --- a/python/ray/tests/test_worker_state.py +++ b/python/ray/tests/test_worker_state.py @@ -1,14 +1,14 @@ -import pytest import sys import threading +import pytest + import ray from ray._common.test_utils import ( wait_for_condition, ) from ray.util.state import list_workers - _SYSTEM_CONFIG = { "task_events_report_interval_ms": 100, "metrics_report_interval_ms": 200, diff --git a/python/ray/tests/typing_files/check_typing_good.py b/python/ray/tests/typing_files/check_typing_good.py index 97d4ed116c34..3e1e96190d90 100644 --- a/python/ray/tests/typing_files/check_typing_good.py +++ b/python/ray/tests/typing_files/check_typing_good.py @@ -1,5 +1,6 @@ -import ray from typing import Generator + +import ray from ray import ObjectRef ray.init() diff --git a/python/ray/tests/unit/test_node_affinity_validation.py b/python/ray/tests/unit/test_node_affinity_validation.py index 88d93c3c6ba4..d5cbc892f96d 100644 --- a/python/ray/tests/unit/test_node_affinity_validation.py +++ b/python/ray/tests/unit/test_node_affinity_validation.py @@ -1,6 +1,7 @@ -import pytest import sys +import pytest + from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy diff --git a/python/ray/tests/unit/test_resource_and_label_spec.py b/python/ray/tests/unit/test_resource_and_label_spec.py index 53fed3ef9cc1..b1cbf69a5603 100644 --- a/python/ray/tests/unit/test_resource_and_label_spec.py +++ b/python/ray/tests/unit/test_resource_and_label_spec.py @@ -1,9 +1,11 @@ -import sys import json -import pytest +import sys from unittest.mock import patch -from ray._common.constants import HEAD_NODE_RESOURCE_NAME, NODE_ID_PREFIX + +import pytest + import ray._private.ray_constants as ray_constants +from ray._common.constants import HEAD_NODE_RESOURCE_NAME, NODE_ID_PREFIX from ray._private.accelerators import AcceleratorManager from ray._private.resource_and_label_spec import ResourceAndLabelSpec diff --git a/python/ray/tests/unit/test_runtime_env.py b/python/ray/tests/unit/test_runtime_env.py index adb8b17e6bfc..494ebb4ca255 100644 --- a/python/ray/tests/unit/test_runtime_env.py +++ b/python/ray/tests/unit/test_runtime_env.py @@ -1,20 +1,17 @@ -from dataclasses import dataclass import dataclasses import json import os import subprocess import sys import tempfile +from dataclasses import dataclass from typing import Any, Dict from unittest import mock import pytest -from ray.runtime_env.runtime_env import ( - RuntimeEnvConfig, - _merge_runtime_env, -) import ray +import ray._private.ray_constants as ray_constants from ray._private.runtime_env.uri_cache import URICache from ray._private.runtime_env.utils import ( SubprocessCalledProcessError, @@ -24,8 +21,10 @@ chdir, ) from ray.runtime_env import RuntimeEnv - -import ray._private.ray_constants as ray_constants +from ray.runtime_env.runtime_env import ( + RuntimeEnvConfig, + _merge_runtime_env, +) def test_runtime_env_merge(): diff --git a/python/ray/tests/unit/test_runtime_env_uv.py b/python/ray/tests/unit/test_runtime_env_uv.py index b4e210049003..2d8aeec68d5a 100644 --- a/python/ray/tests/unit/test_runtime_env_uv.py +++ b/python/ray/tests/unit/test_runtime_env_uv.py @@ -1,9 +1,10 @@ -from ray._private.runtime_env import uv - -import pytest import sys from unittest.mock import patch +import pytest + +from ray._private.runtime_env import uv + class TestRuntimeEnv: def uv_config(self): diff --git a/python/ray/tests/unit/test_runtime_env_validation.py b/python/ray/tests/unit/test_runtime_env_validation.py index df848c8d4c3f..70d722754300 100644 --- a/python/ray/tests/unit/test_runtime_env_validation.py +++ b/python/ray/tests/unit/test_runtime_env_validation.py @@ -1,25 +1,25 @@ import os -from pathlib import Path import sys import tempfile -import yaml +from pathlib import Path import jsonschema import pytest +import yaml from ray import job_config from ray._private.runtime_env import validation -from ray.runtime_env import RuntimeEnv -from ray.runtime_env.runtime_env import ( - _validate_no_local_paths, -) +from ray._private.runtime_env.plugin_schema_manager import RuntimeEnvPluginSchemaManager from ray._private.runtime_env.validation import ( - parse_and_validate_excludes, - parse_and_validate_working_dir, parse_and_validate_conda, + parse_and_validate_excludes, parse_and_validate_py_modules, + parse_and_validate_working_dir, +) +from ray.runtime_env import RuntimeEnv +from ray.runtime_env.runtime_env import ( + _validate_no_local_paths, ) -from ray._private.runtime_env.plugin_schema_manager import RuntimeEnvPluginSchemaManager _CONDA_DICT = {"dependencies": ["pip", {"pip": ["pip-install-test==0.5"]}]} _PIP_LIST = ["requests==1.0.0", "pip-install-test"] diff --git a/python/ray/tests/vsphere/test_cluster_operator.py b/python/ray/tests/vsphere/test_cluster_operator.py index 7ca46872257a..cecd009d8abe 100644 --- a/python/ray/tests/vsphere/test_cluster_operator.py +++ b/python/ray/tests/vsphere/test_cluster_operator.py @@ -8,6 +8,10 @@ import pytest +from ray.autoscaler._private.vsphere.cluster_operator_client import ( + ClusterOperatorClient, + VMNodeStatus, +) from ray.autoscaler.tags import ( NODE_KIND_HEAD, NODE_KIND_WORKER, @@ -20,11 +24,6 @@ TAG_RAY_NODE_STATUS, TAG_RAY_USER_NODE_TYPE, ) -from ray.autoscaler._private.vsphere.cluster_operator_client import ( - ClusterOperatorClient, - VMNodeStatus, -) - _CLUSTER_NAME = "ray-cluster" _PROVIDER_CONFIG = { @@ -146,8 +145,8 @@ def create_random_pvt_key(): - from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import rsa private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) pem_private_key = private_key.private_bytes( diff --git a/python/ray/tests/vsphere/test_vmray_node_provider.py b/python/ray/tests/vsphere/test_vmray_node_provider.py index cceb7a0d1eba..cd1e9889e6a7 100644 --- a/python/ray/tests/vsphere/test_vmray_node_provider.py +++ b/python/ray/tests/vsphere/test_vmray_node_provider.py @@ -5,14 +5,13 @@ import pytest +from ray.autoscaler._private.vsphere.node_provider import VsphereWcpNodeProvider from ray.autoscaler.tags import ( + STATUS_SETTING_UP, TAG_RAY_CLUSTER_NAME, TAG_RAY_NODE_NAME, TAG_RAY_NODE_STATUS, ) -from ray.autoscaler._private.vsphere.node_provider import VsphereWcpNodeProvider - -from ray.autoscaler.tags import STATUS_SETTING_UP _CLUSTER_NAME = "test" _PROVIDER_CONFIG = { From c5c87ba64f37b15b7712cd5af5d358fcbe8af319 Mon Sep 17 00:00:00 2001 From: Potato Date: Wed, 3 Sep 2025 01:18:30 +0800 Subject: [PATCH 386/634] [DOC] Fix documentation typos, grammar, and terminology inconsistencies in Ray Serve docs (#56131) Signed-off-by: Potato Signed-off-by: Jiajun Yao Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Jiajun Yao Co-authored-by: Douglas Strodtman --- doc/source/serve/advanced-guides/app-builder-guide.md | 2 +- doc/source/serve/advanced-guides/dev-workflow.md | 4 ++-- doc/source/serve/advanced-guides/performance.md | 4 ++-- doc/source/serve/architecture.md | 10 +++++----- doc/source/serve/develop-and-deploy.md | 4 ++-- doc/source/serve/getting_started.md | 8 ++++---- doc/source/serve/http-guide.md | 2 +- doc/source/serve/index.md | 4 ++-- doc/source/serve/model-multiplexing.md | 9 ++++----- doc/source/serve/monitoring.md | 2 +- doc/source/serve/multi-app.md | 2 +- doc/source/serve/resource-allocation.md | 2 -- doc/source/serve/tutorials/java.md | 4 ++-- 13 files changed, 27 insertions(+), 30 deletions(-) diff --git a/doc/source/serve/advanced-guides/app-builder-guide.md b/doc/source/serve/advanced-guides/app-builder-guide.md index aecb1c761f75..a306ce3eb2db 100644 --- a/doc/source/serve/advanced-guides/app-builder-guide.md +++ b/doc/source/serve/advanced-guides/app-builder-guide.md @@ -8,7 +8,7 @@ This section describes how to pass arguments to your applications using an appli When writing an application, there are often parameters that you want to be able to easily change in development or production. For example, you might have a path to trained model weights and want to test out a newly trained model. In Ray Serve, these parameters are typically passed to the constructor of your deployments using `.bind()`. -This pattern allows you to be configure deployments using ordinary Python code but it requires modifying the code anytime one of the parameters needs to change. +This pattern allows you to configure deployments using ordinary Python code, but it requires modifying the code whenever one of the parameters needs to change. To pass arguments without changing the code, define an "application builder" function that takes an arguments dictionary (or [Pydantic object](typed-app-builders)) and returns the built application to be run. diff --git a/doc/source/serve/advanced-guides/dev-workflow.md b/doc/source/serve/advanced-guides/dev-workflow.md index 9785c7a68643..b16cc120de6b 100644 --- a/doc/source/serve/advanced-guides/dev-workflow.md +++ b/doc/source/serve/advanced-guides/dev-workflow.md @@ -97,7 +97,7 @@ This mode runs each deployment in a background thread and supports most of the s ## Testing on a remote cluster -To test on a remote cluster, use `serve run` again, but this time, pass in an `--address` argument to specify the address of the Ray cluster to connect to. For remote clusters, this address has the form `ray://:10001`; see [Ray Client](ray-client-ref) for more information. +To test on a remote cluster, use `serve run` again, but this time, pass in an `--address` argument to specify the address of the Ray cluster to connect to. For remote clusters, this address has the form `ray://:10001`; see [Ray Client](ray-client-ref) for more information. When making the transition from your local machine to a remote cluster, you'll need to make sure your cluster has a similar environment to your local machine--files, environment variables, and Python packages, for example. @@ -107,7 +107,7 @@ Let's see a simple example that just packages the code. Run the following comman serve run --address=ray://:10001 --working-dir="./project/src" local_dev:app ``` -This connects to the remote cluster with the Ray Client, uploads the `working_dir` directory, and runs your Serve application. Here, the local directory specified by `working_dir` must contain `local_dev.py` so that it can be uploaded to the cluster and imported by Ray Serve. +This connects to the remote cluster with the Ray Client, uploads the `working_dir` directory, and runs your Serve application. Here, the local directory specified by `working_dir` must contain `local_dev.py` so that it can be uploaded to the cluster and imported by Ray Serve. Once this is up and running, we can send requests to the application: diff --git a/doc/source/serve/advanced-guides/performance.md b/doc/source/serve/advanced-guides/performance.md index 8720d032a519..634847dc5bc2 100644 --- a/doc/source/serve/advanced-guides/performance.md +++ b/doc/source/serve/advanced-guides/performance.md @@ -46,8 +46,8 @@ According to the [FastAPI documentation](https://fastapi.tiangolo.com/async/#ver Are you using `async def` in your callable? If you are using `asyncio` and hitting the same queuing issue mentioned above, you might want to increase -`max_ongoing_requests`. Serve sets a low number (100) by default so the client gets -proper backpressure. You can increase the value in the deployment decorator; e.g., +`max_ongoing_requests`. By default, Serve sets this to a low value (5) to ensure clients receive proper backpressure. +You can increase the value in the deployment decorator; for example, `@serve.deployment(max_ongoing_requests=1000)`. (serve-performance-e2e-timeout)= diff --git a/doc/source/serve/architecture.md b/doc/source/serve/architecture.md index 9aceefa74bda..fa838b0cafc4 100644 --- a/doc/source/serve/architecture.md +++ b/doc/source/serve/architecture.md @@ -29,8 +29,8 @@ There are three kinds of actors that are created to make up a Serve instance: responds once they are completed. For scalability and high availability, you can also run a proxy on each node in the cluster via the `proxy_location` field inside [`serve.start()`](core-apis) or [the config file](serve-in-production-config-file). - **gRPC Proxy**: If Serve is started with valid `port` and `grpc_servicer_functions`, - then the gRPC proxy is started alongside with the HTTP proxy. This Actor runs a - [grpcio](https://grpc.github.io/grpc/python/) server. The gRPC server that accepts + then the gRPC proxy is started alongside the HTTP proxy. This Actor runs a + [grpcio](https://grpc.github.io/grpc/python/) server. The gRPC server accepts incoming requests, forwards them to replicas, and responds once they are completed. - **Replicas**: Actors that actually execute the code in response to a request. For example, they may contain an instantiation of an ML model. Each @@ -51,7 +51,7 @@ When an HTTP or gRPC request is sent to the corresponding HTTP or gRPC proxy, th Each replica maintains a queue of requests and executes requests one at a time, possibly using `asyncio` to process them concurrently. If the handler (the deployment function or the `__call__` method of the deployment class) is declared with `async def`, the replica will not wait for the -handler to run. Otherwise, the replica blocks until the handler returns. +handler to run. Otherwise, the replica blocks until the handler returns. When making a request via a [DeploymentHandle](serve-key-concepts-deployment-handle) instead of HTTP or gRPC for [model composition](serve-model-composition), the request is placed on a queue in the `DeploymentHandle`, and we skip to step 3 above. @@ -88,7 +88,7 @@ Ray Serve's autoscaling feature automatically increases or decreases a deploymen - The Serve Autoscaler runs in the Serve Controller actor. - Each `DeploymentHandle` and each replica periodically pushes its metrics to the autoscaler. - For each deployment, the autoscaler periodically checks `DeploymentHandle` queues and in-flight queries on replicas to decide whether or not to scale the number of replicas. -- Each `DeploymentHandle` continuously polls the controller to check for new deployment replicas. Whenever new replicas are discovered, it sends any buffered or new queries to the replica until `max_ongoing_requests` is reached. Queries are sent to replicas in round-robin fashion, subject to the constraint that no replica is handling more than `max_ongoing_requests` requests at a time. +- Each `DeploymentHandle` continuously polls the controller to check for new deployment replicas. Whenever new replicas are discovered, it sends any buffered or new queries to the replica until `max_ongoing_requests` is reached. Queries are sent to replicas in round-robin fashion, subject to the constraint that no replica is handling more than `max_ongoing_requests` requests at a time. :::{note} When the controller dies, requests can still be sent via HTTP, gRPC and `DeploymentHandle`, but autoscaling is paused. When the controller recovers, the autoscaling resumes, but all previous metrics collected are lost. @@ -105,7 +105,7 @@ Each node in your Ray cluster provides a Serve REST API server that can connect You can configure Serve to start one proxy Actor per node with the `proxy_location` field inside [`serve.start()`](core-apis) or [the config file](serve-in-production-config-file). Each proxy binds to the same port. You should be able to reach Serve and send requests to any models with any of the -servers. You can use your own load balancer on top of Ray Serve. +servers. You can use your own load balancer on top of Ray Serve. This architecture ensures horizontal scalability for Serve. You can scale your HTTP and gRPC ingress by adding more nodes. You can also scale your model inference by increasing the number of replicas via the `num_replicas` option of your deployment. diff --git a/doc/source/serve/develop-and-deploy.md b/doc/source/serve/develop-and-deploy.md index 2e2bf9d2541d..616184deaedd 100644 --- a/doc/source/serve/develop-and-deploy.md +++ b/doc/source/serve/develop-and-deploy.md @@ -38,7 +38,7 @@ Bonjour Monde! ``` Converting this model into a Ray Serve application with FastAPI requires three changes: -1. Import Ray Serve and Fast API dependencies +1. Import Ray Serve and FastAPI dependencies 2. Add decorators for Serve deployment with FastAPI: `@serve.deployment` and `@serve.ingress(app)` 3. `bind` the `Translator` deployment to the arguments that are passed into its constructor @@ -60,7 +60,7 @@ To test locally, run the script with the `serve run` CLI command. This command t $ serve run model:translator_app ``` -This command runs the `translator_app` application and then blocks streaming logs to the console. You can kill it with `Ctrl-C`, which tears down the application. +This command runs the `translator_app` application and then blocks, streaming logs to the console. You can kill it with `Ctrl-C`, which tears down the application. Now test the model over HTTP. Reach it at the following default URL: diff --git a/doc/source/serve/getting_started.md b/doc/source/serve/getting_started.md index 0bbe4084f3e5..23d5171523a5 100644 --- a/doc/source/serve/getting_started.md +++ b/doc/source/serve/getting_started.md @@ -6,7 +6,7 @@ This tutorial will walk you through the process of writing and testing a Ray Ser * convert a machine learning model to a Ray Serve deployment * test a Ray Serve application locally over HTTP -* compose multiple-model machine learning models together into a single application +* compose multi-model machine learning models together into a single application We'll use two models in this tutorial: @@ -101,7 +101,7 @@ parameters in the `@serve.deployment` decorator. The example configures a few co * `ray_actor_options`: a dictionary containing configuration options for each replica. * `num_cpus`: a float representing the logical number of CPUs each replica should reserve. You can make this a fraction to pack multiple replicas together on a machine with fewer CPUs than replicas. * `num_gpus`: a float representing the logical number of GPUs each replica should reserve. You can make this a fraction to pack multiple replicas together on a machine with fewer GPUs than replicas. - * `resources`: a dictionary containing other resource requirements for the replicate, such as non-GPU accelerators like HPUs or TPUs. + * `resources`: a dictionary containing other resource requirements for the replica, such as non-GPU accelerators like HPUs or TPUs. All these parameters are optional, so feel free to omit them: @@ -193,12 +193,12 @@ For example, let's deploy a machine learning pipeline with two steps: :language: python ``` -You can copy-paste this script and run it locally. It summarizes the snippet from _A Tale of Two Cities_ to `it was the best of times, it was worst of times .` +You can copy-paste this script and run it locally. It summarizes the snippet from _A Tale of Two Cities_ to `it was the best of times, it was the worst of times .` ```console $ python summary_model.py -it was the best of times, it was worst of times . +it was the best of times, it was the worst of times . ``` Here's an application that chains the two models together. The graph takes English text, summarizes it, and then translates it: diff --git a/doc/source/serve/http-guide.md b/doc/source/serve/http-guide.md index 054ac9ff2145..a4b14a07f68d 100644 --- a/doc/source/serve/http-guide.md +++ b/doc/source/serve/http-guide.md @@ -63,7 +63,7 @@ When the request is cancelled, a cancellation error is raised inside the `Snorin If you want to define more complex HTTP handling logic, Serve integrates with [FastAPI](https://fastapi.tiangolo.com/). This allows you to define a Serve deployment using the {mod}`@serve.ingress ` decorator that wraps a FastAPI app with its full range of features. The most basic example of this is shown below, but for more details on all that FastAPI has to offer such as variable routes, automatic type validation, dependency injection (e.g., for database connections), and more, please check out [their documentation](https://fastapi.tiangolo.com/). :::{note} -A Serve application that's integrated with FastAPI still respects the `route_prefix` set through Serve. The routes are that registered through the FastAPI `app` object are layered on top of the route prefix. For instance, if your Serve application has `route_prefix = /my_app` and you decorate a method with `@app.get("/fetch_data")`, then you can call that method by sending a GET request to the path `/my_app/fetch_data`. +A Serve application that's integrated with FastAPI still respects the `route_prefix` set through Serve. The routes that are registered through the FastAPI `app` object are layered on top of the route prefix. For instance, if your Serve application has `route_prefix = /my_app` and you decorate a method with `@app.get("/fetch_data")`, then you can call that method by sending a GET request to the path `/my_app/fetch_data`. ::: ```{literalinclude} doc_code/http_guide/http_guide.py :start-after: __begin_fastapi__ diff --git a/doc/source/serve/index.md b/doc/source/serve/index.md index 5cc34ad54f61..498337140771 100644 --- a/doc/source/serve/index.md +++ b/doc/source/serve/index.md @@ -35,7 +35,7 @@ api/index Ray Serve is a scalable model serving library for building online inference APIs. Serve is framework-agnostic, so you can use a single toolkit to serve everything from deep learning models built with frameworks like PyTorch, TensorFlow, and Keras, to Scikit-Learn models, to arbitrary Python business logic. It has several features and performance optimizations for serving Large Language Models such as response streaming, dynamic request batching, multi-node/multi-GPU serving, etc. -Ray Serve is particularly well suited for [model composition](serve-model-composition) and many model serving, enabling you to build a complex inference service consisting of multiple ML models and business logic all in Python code. +Ray Serve is particularly well suited for [model composition](serve-model-composition) and multi-model serving, enabling you to build a complex inference service consisting of multiple ML models and business logic all in Python code. Ray Serve is built on top of Ray, so it easily scales to many machines and offers flexible scheduling support such as fractional GPUs so you can share resources and serve many machine learning models at low cost. @@ -244,7 +244,7 @@ or head over to the {doc}`examples` to get started building your Ray Serve appli **Getting Started** ^^^ - Start with our quick start tutorials for :ref:`deploying a single model locally ` and how to :ref:`convert an existing model into a Ray Serve deployment ` . + Start with our quick start tutorials for :ref:`deploying a single model locally ` and how to :ref:`convert an existing model into a Ray Serve deployment `. +++ .. button-ref:: serve-getting-started diff --git a/doc/source/serve/model-multiplexing.md b/doc/source/serve/model-multiplexing.md index b0bce6a68cb3..4400aac8a6d9 100644 --- a/doc/source/serve/model-multiplexing.md +++ b/doc/source/serve/model-multiplexing.md @@ -13,7 +13,7 @@ model multiplexing optimizes cost and load balances the traffic. This is useful To write a multiplexed deployment, use the `serve.multiplexed` and `serve.get_multiplexed_model_id` APIs. -Assuming you have multiple Torch models inside an aws s3 bucket with the following structure: +Assuming you have multiple PyTorch models inside an AWS S3 bucket with the following structure: ``` s3://my_bucket/1/model.pt s3://my_bucket/2/model.pt @@ -34,15 +34,14 @@ The `serve.multiplexed` API also has a `max_num_models_per_replica` parameter. U ::: :::{tip} -This code example uses the Pytorch Model object. You can also define your own model class and use it here. To release resources when the model is evicted, implement the `__del__` method. Ray Serve internally calls the `__del__` method to release resources when the model is evicted. +This code example uses the PyTorch Model object. You can also define your own model class and use it here. To release resources when the model is evicted, implement the `__del__` method. Ray Serve internally calls the `__del__` method to release resources when the model is evicted. ::: -`serve.get_multiplexed_model_id` is used to retrieve the model id from the request header, and the model_id is then passed into the `get_model` function. If the model id is not found in the replica, Serve will load the model from the s3 bucket and cache it in the replica. If the model id is found in the replica, Serve will return the cached model. +`serve.get_multiplexed_model_id` retrieves the model ID from the request header. This ID is then passed to the `get_model` function. If the model is not already cached in the replica, Serve loads it from the S3 bucket. Otherwise, the cached model is returned. :::{note} -Internally, serve router will route the traffic to the corresponding replica based on the model id in the request header. -If all replicas holding the model are over-subscribed, ray serve sends the request to a new replica that doesn't have the model loaded. The replica will load the model from the s3 bucket and cache it. +Internally, the Serve router uses the model ID in the request header to route traffic to a corresponding replica. If all replicas that have the model are over-subscribed, Ray Serve routes the request to a new replica, which then loads and caches the model from the S3 bucket. ::: To send a request to a specific model, include the `serve_multiplexed_model_id` field in the request header, and set the value to the model ID to which you want to send the request. diff --git a/doc/source/serve/monitoring.md b/doc/source/serve/monitoring.md index 8ed8c706f0f7..9437ba81c07c 100644 --- a/doc/source/serve/monitoring.md +++ b/doc/source/serve/monitoring.md @@ -54,7 +54,7 @@ For a detailed overview of the Ray dashboard, see the [dashboard documentation]( Two Serve CLI commands help you inspect a Serve application in production: `serve config` and `serve status`. If you have a remote cluster, `serve config` and `serve status` also has an `--address/-a` argument to access the cluster. See [VM deployment](serve-in-production-remote-cluster) for more information on this argument. -`serve config` gets the latest config file that the Ray Cluster received. This config file represents the Serve application's goal state. The Ray Cluster constantly strives to reach and maintain this state by deploying deployments, and recovering failed replicas, and performing other relevant actions. +`serve config` gets the latest config file that the Ray Cluster received. This config file represents the Serve application's goal state. The Ray Cluster constantly strives to reach and maintain this state by deploying deployments, recovering failed replicas, and performing other relevant actions. Using the `serve_config.yaml` example from [the production guide](production-config-yaml): diff --git a/doc/source/serve/multi-app.md b/doc/source/serve/multi-app.md index 027281566adc..abb36cd1946a 100644 --- a/doc/source/serve/multi-app.md +++ b/doc/source/serve/multi-app.md @@ -7,7 +7,7 @@ Serve supports deploying multiple independent Serve applications. This user guid ### Background With the introduction of multi-application Serve, we walk you through the new concept of applications and when you should choose to deploy a single application versus multiple applications per cluster. -An application consists of one or more deployments. The deployments in an application are tied into a direct acyclic graph through [model composition](serve-model-composition). An application can be called via HTTP at the specified route prefix, and the ingress deployment handles all such inbound traffic. Due to the dependence between deployments in an application, one application is a unit of upgrade. +An application consists of one or more deployments. The deployments in an application are tied into a directed acyclic graph through [model composition](serve-model-composition). An application can be called via HTTP at the specified route prefix, and the ingress deployment handles all such inbound traffic. Due to the dependence between deployments in an application, one application is a unit of upgrade. ### When to use multiple applications You can solve many use cases by using either model composition or multi-application. However, both have their own individual benefits and can be used together. diff --git a/doc/source/serve/resource-allocation.md b/doc/source/serve/resource-allocation.md index 18df5a8181a4..04dff0c9cc5c 100644 --- a/doc/source/serve/resource-allocation.md +++ b/doc/source/serve/resource-allocation.md @@ -39,8 +39,6 @@ def func(*args): ### Fractional CPUs and fractional GPUs -Suppose you have two models and each doesn't fully saturate a GPU. You might want to have them share a GPU by allocating 0.5 GPUs each. - To do this, the resources specified in `ray_actor_options` can be *fractional*. For example, if you have two models and each doesn't fully saturate a GPU, you might want to have them share a GPU by allocating 0.5 GPUs each. diff --git a/doc/source/serve/tutorials/java.md b/doc/source/serve/tutorials/java.md index 6a57addc34aa..743fff19af24 100644 --- a/doc/source/serve/tutorials/java.md +++ b/doc/source/serve/tutorials/java.md @@ -20,7 +20,7 @@ To use Java Ray Serve, you need the following dependency in your pom.xml. ## Example model -This example use case is a production workflow of a financial application. The application needs to compute the best strategy to interact with different banks for a single task. +This example use case is a production workflow for a financial application. The application needs to compute the best strategy to interact with different banks for a single task. ```{literalinclude} ../../../../java/serve/src/test/java/io/ray/serve/docdemo/Strategy.java :end-before: docs-strategy-end @@ -43,7 +43,7 @@ This code uses the `Strategy` class: :start-after: docs-strategy-calc-start ``` -When the scale of banks and indicators expands, the three-tier `for` loop slows down the calculation. Even if you use the thread pool to calculate each indicator in parallel, you may encounter a single machine performance bottleneck. Moreover, you can't use this `Strategy` object as a resident service. +When the scale of banks and indicators expands, the three-tier `for` loop slows down the calculation. Even if you use the thread pool to calculate each indicator in parallel, you may encounter a single machine performance bottleneck. Moreover, you can't use this `Strategy` object as a resident service. ## Converting to a Ray Serve Deployment From de87f6b80bc00d48303406cd35ab35b6d71a8a4b Mon Sep 17 00:00:00 2001 From: Rueian Date: Tue, 2 Sep 2025 10:22:54 -0700 Subject: [PATCH 387/634] [core][autoscaler] fix races when waiting for stopping aws nodes to be reused by cache_stopped_nodes (#56007) Signed-off-by: Rueian --- .../autoscaler/_private/aws/node_provider.py | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/python/ray/autoscaler/_private/aws/node_provider.py b/python/ray/autoscaler/_private/aws/node_provider.py index f11e8a6dbcdb..3b8673dbb730 100644 --- a/python/ray/autoscaler/_private/aws/node_provider.py +++ b/python/ray/autoscaler/_private/aws/node_provider.py @@ -127,6 +127,8 @@ def __init__(self, provider_config, cluster_name): self.ready_for_new_batch.set() self.tag_cache_lock = threading.Lock() self.count_lock = threading.Lock() + # Prevent concurrent create_node calls to get the same stopped/stopping node to reuse. + self._reuse_node_lock = threading.Lock() # Cache of node objects from the last nodes() call. This avoids # excessive DescribeInstances requests. @@ -290,32 +292,35 @@ def create_node(self, node_config, tags, count) -> Dict[str, Any]: } ) - reuse_nodes = list(self.ec2.instances.filter(Filters=filters))[:count] - reuse_node_ids = [n.id for n in reuse_nodes] - reused_nodes_dict = {n.id: n for n in reuse_nodes} - if reuse_nodes: - cli_logger.print( - # todo: handle plural vs singular? - "Reusing nodes {}. " - "To disable reuse, set `cache_stopped_nodes: False` " - "under `provider` in the cluster configuration.", - cli_logger.render_list(reuse_node_ids), - ) + with self._reuse_node_lock: + reuse_nodes = list(self.ec2.instances.filter(Filters=filters))[:count] + reuse_node_ids = [n.id for n in reuse_nodes] + reused_nodes_dict = {n.id: n for n in reuse_nodes} + if reuse_nodes: + cli_logger.print( + # todo: handle plural vs singular? + "Reusing nodes {}. " + "To disable reuse, set `cache_stopped_nodes: False` " + "under `provider` in the cluster configuration.", + cli_logger.render_list(reuse_node_ids), + ) - # todo: timed? - with cli_logger.group("Stopping instances to reuse"): - for node in reuse_nodes: - self.tag_cache[node.id] = from_aws_format( - {x["Key"]: x["Value"] for x in node.tags} - ) - if node.state["Name"] == "stopping": - cli_logger.print("Waiting for instance {} to stop", node.id) - node.wait_until_stopped() - - self.ec2.meta.client.start_instances(InstanceIds=reuse_node_ids) - for node_id in reuse_node_ids: - self.set_node_tags(node_id, tags) - count -= len(reuse_node_ids) + # todo: timed? + with cli_logger.group("Stopping instances to reuse"): + for node in reuse_nodes: + self.tag_cache[node.id] = from_aws_format( + {x["Key"]: x["Value"] for x in node.tags} + ) + if node.state["Name"] == "stopping": + cli_logger.print( + "Waiting for instance {} to stop", node.id + ) + node.wait_until_stopped() + + self.ec2.meta.client.start_instances(InstanceIds=reuse_node_ids) + for node_id in reuse_node_ids: + self.set_node_tags(node_id, tags) + count -= len(reuse_node_ids) created_nodes_dict = {} if count: From 12b4dca0f3f3289fcb9f532dd785fd676c60a2e4 Mon Sep 17 00:00:00 2001 From: Praveen Date: Tue, 2 Sep 2025 10:34:34 -0700 Subject: [PATCH 388/634] [data] Do not log the content of data block that failed (#56133) ## Why are these changes needed? This reverts PR #52380. When working with large data blocks, this log can dump entire bock to terminal and can be spammy and insecure. ## Related issue number Fixes #56092 ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Praveen Gorthy Signed-off-by: Praveen Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../data/_internal/planner/plan_udf_map_op.py | 15 ++++++--------- python/ray/data/tests/test_exceptions.py | 16 ---------------- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/python/ray/data/_internal/planner/plan_udf_map_op.py b/python/ray/data/_internal/planner/plan_udf_map_op.py index 940773dc0b43..a27a9a2ec9fb 100644 --- a/python/ray/data/_internal/planner/plan_udf_map_op.py +++ b/python/ray/data/_internal/planner/plan_udf_map_op.py @@ -154,7 +154,7 @@ def fn(block: Block) -> Block: return block except Exception as e: - _try_wrap_udf_exception(e, block) + _try_wrap_udf_exception(e) compute = get_compute(op._compute) transform_fn = _generate_transform_fn_for_map_block(fn) @@ -213,7 +213,7 @@ def filter_batch_fn(block: "pa.Table") -> "pa.Table": try: return block.filter(expression) except Exception as e: - _try_wrap_udf_exception(e, block) + _try_wrap_udf_exception(e) transform_fn = _generate_transform_fn_for_map_batches(filter_batch_fn) map_transformer = _create_map_transformer_for_map_batches_op( @@ -353,7 +353,7 @@ async def _wrapped_udf_map_fn(item: Any) -> Any: **fn_kwargs, ) except Exception as e: - _try_wrap_udf_exception(e, item) + _try_wrap_udf_exception(e) elif inspect.isasyncgenfunction(udf.__call__): @@ -388,7 +388,7 @@ def _wrapped_udf_map_fn(item: Any) -> Any: **fn_kwargs, ) except Exception as e: - _try_wrap_udf_exception(e, item) + _try_wrap_udf_exception(e) else: @@ -396,7 +396,7 @@ def _wrapped_udf_map_fn(item: Any) -> Any: try: return udf(item, *fn_args, **fn_kwargs) except Exception as e: - _try_wrap_udf_exception(e, item) + _try_wrap_udf_exception(e) def init_fn(): pass @@ -408,14 +408,11 @@ def _try_wrap_udf_exception(e: Exception, item: Any = None): """If the Ray Debugger is enabled, keep the full stack trace unmodified so that the debugger can stop at the initial unhandled exception. Otherwise, clear the stack trace to omit noisy internal code path.""" - error_message = f"Failed to process the following data block: {item}" - ctx = ray.data.DataContext.get_current() if _is_ray_debugger_post_mortem_enabled() or ctx.raise_original_map_exception: - logger.error(error_message) raise e else: - raise UserCodeException(error_message) from e + raise UserCodeException("UDF failed to process a data block.") from e # Following are util functions for converting UDFs to `MapTransformCallable`s. diff --git a/python/ray/data/tests/test_exceptions.py b/python/ray/data/tests/test_exceptions.py index 7d57586ca1d2..9818c3f379e0 100644 --- a/python/ray/data/tests/test_exceptions.py +++ b/python/ray/data/tests/test_exceptions.py @@ -9,22 +9,6 @@ from ray.tests.conftest import * # noqa -def test_handle_debugger_exception(ray_start_regular_shared): - def _bad(batch): - if batch["id"][0] == 5: - raise Exception("Test exception") - - return batch - - dataset = ray.data.range(8, override_num_blocks=8).map_batches(_bad) - - with pytest.raises( - UserCodeException, - match=r"Failed to process the following data block: \{'id': array\(\[5\]\)\}", - ): - dataset.materialize() - - @pytest.mark.parametrize("log_internal_stack_trace_to_stdout", [True, False]) def test_user_exception( log_internal_stack_trace_to_stdout, From 7cd7ba7628216239601a86b804b0f95253206735 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 2 Sep 2025 11:50:16 -0700 Subject: [PATCH 389/634] [release test] split images into its own group (#56122) and add skip-on-release-tests tag for skipping steps to run on release tests Signed-off-by: Lonnie Liu --- .buildkite/_forge.rayci.yml | 223 -------------------------- .buildkite/_images.rayci.yml | 227 +++++++++++++++++++++++++++ .buildkite/release/_images.rayci.yml | 1 + .buildkite/release/config.yml | 3 + 4 files changed, 231 insertions(+), 223 deletions(-) create mode 100644 .buildkite/_images.rayci.yml create mode 120000 .buildkite/release/_images.rayci.yml diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml index 220f556651f7..69c066c7bca2 100644 --- a/.buildkite/_forge.rayci.yml +++ b/.buildkite/_forge.rayci.yml @@ -6,226 +6,3 @@ steps: - name: manylinux wanda: ci/docker/manylinux.wanda.yaml - - - name: raycpubase - label: "wanda: ray.py{{matrix}}.cpu.base" - tags: - - python_dependencies - - docker - wanda: docker/base-deps/cpu.wanda.yaml - matrix: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - env: - PYTHON_VERSION: "{{matrix}}" - ARCH_SUFFIX: "" - - - name: raycpubaseextra - label: "wanda: ray.py{{matrix}}.cpu.base-extra" - wanda: docker/base-extra/cpu.wanda.yaml - matrix: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - env: - PYTHON_VERSION: "{{matrix}}" - IMAGE_TYPE: "ray" - ARCH_SUFFIX: "" - depends_on: raycpubase - - - name: raycudabase - label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base" - tags: - - python_dependencies - - docker - wanda: docker/base-deps/cuda.wanda.yaml - matrix: - setup: - python: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - cuda: - - "11.7.1-cudnn8" - - "11.8.0-cudnn8" - - "12.1.1-cudnn8" - - "12.3.2-cudnn9" - - "12.4.1-cudnn" - - "12.5.1-cudnn" - - "12.6.3-cudnn" - - "12.8.1-cudnn" - env: - PYTHON_VERSION: "{{matrix.python}}" - CUDA_VERSION: "{{matrix.cuda}}" - ARCH_SUFFIX: "" - - - name: raycudabaseextra - label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" - wanda: docker/base-extra/cuda.wanda.yaml - matrix: - setup: - python: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - cuda: - - "11.7.1-cudnn8" - - "11.8.0-cudnn8" - - "12.1.1-cudnn8" - - "12.3.2-cudnn9" - - "12.4.1-cudnn" - - "12.5.1-cudnn" - - "12.6.3-cudnn" - - "12.8.1-cudnn" - env: - PYTHON_VERSION: "{{matrix.python}}" - CUDA_VERSION: "{{matrix.cuda}}" - IMAGE_TYPE: "ray" - ARCH_SUFFIX: "" - depends_on: raycudabase - - - name: ray-llmbase - label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base" - tags: - - python_dependencies - - docker - wanda: docker/ray-llm/cuda.wanda.yaml - depends_on: raycudabase - matrix: - setup: - python: - - "3.11" - cuda: - - "12.8.1-cudnn" - env: - PYTHON_VERSION: "{{matrix.python}}" - CUDA_VERSION: "{{matrix.cuda}}" - - - name: ray-llmbaseextra - label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" - wanda: docker/base-extra/cuda.wanda.yaml - matrix: - setup: - python: - - "3.11" - cuda: - - "12.8.1-cudnn" - env: - PYTHON_VERSION: "{{matrix.python}}" - CUDA_VERSION: "{{matrix.cuda}}" - IMAGE_TYPE: "ray-llm" - ARCH_SUFFIX: "" - depends_on: ray-llmbase - - - name: ray-mlcudabase - label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base" - tags: - - python_dependencies - - docker - wanda: docker/ray-ml/cuda.wanda.yaml - depends_on: raycudabase - matrix: - setup: - python: - - "3.9" - - "3.10" - - "3.11" - cuda: - - "12.1.1-cudnn8" - env: - PYTHON_VERSION: "{{matrix.python}}" - CUDA_VERSION: "{{matrix.cuda}}" - - - name: ray-mlcudabaseextra - label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" - wanda: docker/base-extra/cuda.wanda.yaml - matrix: - setup: - python: - - "3.9" - - "3.10" - - "3.11" - cuda: - - "12.1.1-cudnn8" - env: - PYTHON_VERSION: "{{matrix.python}}" - CUDA_VERSION: "{{matrix.cuda}}" - IMAGE_TYPE: "ray-ml" - ARCH_SUFFIX: "" - depends_on: ray-mlcudabase - - - name: ray-mlcpubase - label: "wanda: ray-ml.py{{matrix}}.cpu.base" - tags: - - python_dependencies - - docker - wanda: docker/ray-ml/cpu.wanda.yaml - depends_on: raycpubase - matrix: - - "3.9" - - "3.10" - - "3.11" - env: - PYTHON_VERSION: "{{matrix}}" - - - name: ray-mlcpubaseextra - label: "wanda: ray-ml.py{{matrix}}.cpu.base-extra" - wanda: docker/base-extra/cpu.wanda.yaml - matrix: - - "3.9" - - "3.10" - - "3.11" - env: - PYTHON_VERSION: "{{matrix}}" - IMAGE_TYPE: "ray-ml" - ARCH_SUFFIX: "" - depends_on: ray-mlcpubase - - - name: ray-slimcpubase - label: "wanda: ray-slim.py{{matrix}}.cpu.base" - tags: - - python_dependencies - - docker - wanda: docker/base-slim/cpu.wanda.yaml - depends_on: raycpubase - matrix: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - env: - PYTHON_VERSION: "{{matrix}}" - ARCH_SUFFIX: "" - - - name: ray-slimcudabase - label: "wanda: ray-slim.py{{matrix.python}}.cu{{matrix.cuda}}.base" - tags: - - python_dependencies - - docker - wanda: docker/base-slim/cuda.wanda.yaml - depends_on: raycudabase - matrix: - setup: - python: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - cuda: - - "11.7.1" - - "11.8.0" - - "12.1.1" - - "12.3.2" - - "12.4.1" - - "12.5.1" - - "12.6.3" - - "12.8.1" - env: - PYTHON_VERSION: "{{matrix.python}}" - CUDA_VERSION: "{{matrix.cuda}}" - ARCH_SUFFIX: "" diff --git a/.buildkite/_images.rayci.yml b/.buildkite/_images.rayci.yml new file mode 100644 index 000000000000..74413964085b --- /dev/null +++ b/.buildkite/_images.rayci.yml @@ -0,0 +1,227 @@ +group: images +sort_key: "_images" +steps: + - name: raycpubase + label: "wanda: ray.py{{matrix}}.cpu.base" + tags: + - python_dependencies + - docker + wanda: docker/base-deps/cpu.wanda.yaml + matrix: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + env: + PYTHON_VERSION: "{{matrix}}" + ARCH_SUFFIX: "" + + - name: raycpubaseextra + label: "wanda: ray.py{{matrix}}.cpu.base-extra" + wanda: docker/base-extra/cpu.wanda.yaml + matrix: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + env: + PYTHON_VERSION: "{{matrix}}" + IMAGE_TYPE: "ray" + ARCH_SUFFIX: "" + depends_on: raycpubase + + - name: raycudabase + label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base" + tags: + - python_dependencies + - docker + wanda: docker/base-deps/cuda.wanda.yaml + matrix: + setup: + python: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + cuda: + - "11.7.1-cudnn8" + - "11.8.0-cudnn8" + - "12.1.1-cudnn8" + - "12.3.2-cudnn9" + - "12.4.1-cudnn" + - "12.5.1-cudnn" + - "12.6.3-cudnn" + - "12.8.1-cudnn" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + ARCH_SUFFIX: "" + + - name: raycudabaseextra + label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" + wanda: docker/base-extra/cuda.wanda.yaml + matrix: + setup: + python: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + cuda: + - "11.7.1-cudnn8" + - "11.8.0-cudnn8" + - "12.1.1-cudnn8" + - "12.3.2-cudnn9" + - "12.4.1-cudnn" + - "12.5.1-cudnn" + - "12.6.3-cudnn" + - "12.8.1-cudnn" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + IMAGE_TYPE: "ray" + ARCH_SUFFIX: "" + depends_on: raycudabase + + - name: ray-llmbase + label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base" + tags: + - python_dependencies + - docker + wanda: docker/ray-llm/cuda.wanda.yaml + depends_on: raycudabase + matrix: + setup: + python: + - "3.11" + cuda: + - "12.8.1-cudnn" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + + - name: ray-llmbaseextra + label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" + wanda: docker/base-extra/cuda.wanda.yaml + matrix: + setup: + python: + - "3.11" + cuda: + - "12.8.1-cudnn" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + IMAGE_TYPE: "ray-llm" + ARCH_SUFFIX: "" + depends_on: ray-llmbase + + - name: ray-mlcpubase + label: "wanda: ray-ml.py{{matrix}}.cpu.base" + tags: + - python_dependencies + - docker + wanda: docker/ray-ml/cpu.wanda.yaml + depends_on: raycpubase + matrix: + - "3.9" + - "3.10" + - "3.11" + env: + PYTHON_VERSION: "{{matrix}}" + + - name: ray-mlcpubaseextra + label: "wanda: ray-ml.py{{matrix}}.cpu.base-extra" + wanda: docker/base-extra/cpu.wanda.yaml + matrix: + - "3.9" + - "3.10" + - "3.11" + env: + PYTHON_VERSION: "{{matrix}}" + IMAGE_TYPE: "ray-ml" + ARCH_SUFFIX: "" + depends_on: ray-mlcpubase + + - name: ray-mlcudabase + label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base" + tags: + - python_dependencies + - docker + wanda: docker/ray-ml/cuda.wanda.yaml + depends_on: raycudabase + matrix: + setup: + python: + - "3.9" + - "3.10" + - "3.11" + cuda: + - "12.1.1-cudnn8" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + + - name: ray-mlcudabaseextra + label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" + wanda: docker/base-extra/cuda.wanda.yaml + matrix: + setup: + python: + - "3.9" + - "3.10" + - "3.11" + cuda: + - "12.1.1-cudnn8" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + IMAGE_TYPE: "ray-ml" + ARCH_SUFFIX: "" + depends_on: ray-mlcudabase + + - name: ray-slimcpubase + label: "wanda: ray-slim.py{{matrix}}.cpu.base" + tags: + - python_dependencies + - docker + - skip-on-release-tests + wanda: docker/base-slim/cpu.wanda.yaml + depends_on: raycpubase + matrix: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + env: + PYTHON_VERSION: "{{matrix}}" + ARCH_SUFFIX: "" + + - name: ray-slimcudabase + label: "wanda: ray-slim.py{{matrix.python}}.cu{{matrix.cuda}}.base" + tags: + - python_dependencies + - docker + - skip-on-release-tests + wanda: docker/base-slim/cuda.wanda.yaml + depends_on: raycudabase + matrix: + setup: + python: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + cuda: + - "11.7.1" + - "11.8.0" + - "12.1.1" + - "12.3.2" + - "12.4.1" + - "12.5.1" + - "12.6.3" + - "12.8.1" + env: + PYTHON_VERSION: "{{matrix.python}}" + CUDA_VERSION: "{{matrix.cuda}}" + ARCH_SUFFIX: "" diff --git a/.buildkite/release/_images.rayci.yml b/.buildkite/release/_images.rayci.yml new file mode 120000 index 000000000000..67fd8382b173 --- /dev/null +++ b/.buildkite/release/_images.rayci.yml @@ -0,0 +1 @@ +../_images.rayci.yml \ No newline at end of file diff --git a/.buildkite/release/config.yml b/.buildkite/release/config.yml index 7917e6356526..30ac2983d3b0 100644 --- a/.buildkite/release/config.yml +++ b/.buildkite/release/config.yml @@ -15,6 +15,9 @@ env: RAYCI_SKIP_UPLOAD: "true" hook_env_keys: - RAYCI_CHECKOUT_DIR +skip_tags: + - disabled + - skip-on-release-tests build_env_keys: - AUTOMATIC - RELEASE_FREQUENCY From df08ab7f5c8426998e9a857c00488cf2344631eb Mon Sep 17 00:00:00 2001 From: weiliango Date: Tue, 2 Sep 2025 11:57:03 -0700 Subject: [PATCH 390/634] [RLlib] Update rllib-env.rst - typo (#56140) minor typo ## Why are these changes needed? ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: weiliango Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- doc/source/rllib/rllib-env.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/rllib/rllib-env.rst b/doc/source/rllib/rllib-env.rst index cd97c0259ba6..321ce169f06f 100644 --- a/doc/source/rllib/rllib-env.rst +++ b/doc/source/rllib/rllib-env.rst @@ -289,7 +289,7 @@ in combination. controlled through your :py:class:`~ray.rllib.algorithms.algorithm_config.AlgorithmConfig`: ``config.env_runners(num_env_runners=..)``. -1. **Vectorization within a single process:** Many environments achieve high +#. **Vectorization within a single process:** Many environments achieve high frame rates per core but are limited by policy inference latency. To address this limitation, create multiple environments per process to batch the policy forward pass across these vectorized environments. Set ``config.env_runners(num_envs_per_env_runner=..)`` From 629e4d6d928ecab10e501c14b5ac66631cecbea3 Mon Sep 17 00:00:00 2001 From: Jack Gammack <49536617+JackGammack@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:06:19 -0400 Subject: [PATCH 391/634] [docs] Update performance tips documentation for new execution options resource limits assignment (#56051) ## Why are these changes needed? The performance tips documentation for setting resource limits in ExecutionOptions is no longer correct and gives an error when directly setting them in 2.49 after #54694. Update the documentation to show how to correctly set them. ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Jack Gammack <49536617+JackGammack@users.noreply.github.com> --- doc/source/data/performance-tips.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/source/data/performance-tips.rst b/doc/source/data/performance-tips.rst index 1155663b22f4..0402842c4795 100644 --- a/doc/source/data/performance-tips.rst +++ b/doc/source/data/performance-tips.rst @@ -417,10 +417,12 @@ You can configure execution options with the global DataContext. The options are .. code-block:: - ctx = ray.data.DataContext.get_current() - ctx.execution_options.resource_limits.cpu = 10 - ctx.execution_options.resource_limits.gpu = 5 - ctx.execution_options.resource_limits.object_store_memory = 10e9 + ctx = ray.data.DataContext.get_current() + ctx.execution_options.resource_limits = ctx.execution_options.resource_limits.copy( + cpu=10, + gpu=5, + object_store_memory=10e9, + ) .. note:: It's **not** recommended to modify the Ray Core object store memory limit, as this can reduce available memory for task execution. The one exception to this is if you are using machines with a very large amount of RAM (1 TB or more each); then it's recommended to set the object store to ~30-40%. From cff14b1c92eba43608e3bc215ea92c88ccc312fa Mon Sep 17 00:00:00 2001 From: Potato Date: Wed, 3 Sep 2025 03:07:00 +0800 Subject: [PATCH 392/634] [DOC][Data] Fix grammar and formatting issues in Ray Data documentation (#56066) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR addresses various grammar, punctuation, and formatting issues throughout the Ray Data documentation in `doc/source/data/` to improve clarity and readability. ## Changes Made **Grammar Fixes:** - Fixed verb agreement errors in `key-concepts.rst` ("define" → "defines", "translate" → "translates") - Corrected missing articles and prepositions ("sharding your dataset" → "sharding of your dataset") - Fixed awkward phrasing in `saving-data.rst` ("have the control" → "have control") - Improved sentence flow in multiple files ("like following" → "as follows") **Formatting Improvements:** - Restructured bullet list formatting in `aggregations.rst` for better readability - Added missing punctuation and commas for proper sentence structure - Improved note formatting and punctuation consistency **Files Modified:** - `doc/source/data/key-concepts.rst` - 3 grammar corrections - `doc/source/data/user-guide.rst` - 1 verb form correction - `doc/source/data/aggregations.rst` - Bullet list formatting improvement - `doc/source/data/joining-data.rst` - 2 grammar and punctuation fixes - `doc/source/data/comparisons.rst` - 1 preposition correction - `doc/source/data/data-internals.rst` - 1 punctuation fix - `doc/source/data/saving-data.rst` - 1 phrasing improvement ## Review Methodology The review was conducted manually across all 45 files in the `doc/source/data/` directory, focusing specifically on: - Typos and spelling errors - Grammar and syntax issues - RST formatting consistency - Punctuation and capitalization The approach was conservative, making only clear corrections without rewriting content for style, preserving the technical accuracy and existing tone of the documentation. ## Impact These changes improve the overall quality and professionalism of the Ray Data documentation while maintaining all technical content and existing structure. The fixes address common grammatical issues that could distract readers from the technical content. --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- doc/source/data/aggregations.rst | 11 ++++++++--- doc/source/data/comparisons.rst | 2 +- doc/source/data/data-internals.rst | 2 +- doc/source/data/joining-data.rst | 8 ++++---- doc/source/data/key-concepts.rst | 6 +++--- doc/source/data/saving-data.rst | 2 +- doc/source/data/user-guide.rst | 2 +- 7 files changed, 19 insertions(+), 14 deletions(-) diff --git a/doc/source/data/aggregations.rst b/doc/source/data/aggregations.rst index 3ab3da449acd..443af7cc1878 100644 --- a/doc/source/data/aggregations.rst +++ b/doc/source/data/aggregations.rst @@ -8,9 +8,14 @@ Ray Data provides a flexible and performant API for performing aggregations on : Basic Aggregations ------------------ -Ray Data provides several built-in aggregation functions like -* :class:`~ray.data.aggregate.Count`, * :class:`~ray.data.aggregate.Sum`, * :class:`~ray.data.aggregate.Mean`, -* :class:`~ray.data.aggregate.Min`, * :class:`~ray.data.aggregate.Max`, * :class:`~ray.data.aggregate.Std`, +Ray Data provides several built-in aggregation functions like: + +* :class:`~ray.data.aggregate.Count` +* :class:`~ray.data.aggregate.Sum` +* :class:`~ray.data.aggregate.Mean` +* :class:`~ray.data.aggregate.Min` +* :class:`~ray.data.aggregate.Max` +* :class:`~ray.data.aggregate.Std` * :class:`~ray.data.aggregate.Quantile` These can be used directly with datasets like shown below: diff --git a/doc/source/data/comparisons.rst b/doc/source/data/comparisons.rst index 0b6b23b42a78..dfa5d772c3c8 100644 --- a/doc/source/data/comparisons.rst +++ b/doc/source/data/comparisons.rst @@ -8,7 +8,7 @@ How does Ray Data compare to other solutions for offline inference? Cloud providers such as AWS, GCP, and Azure provide batch services to manage compute infrastructure for you. Each service uses the same process: you provide the code, and the service runs your code on each node in a cluster. However, while infrastructure management is necessary, it is often not enough. These services have limitations, such as a lack of software libraries to address optimized parallelization, efficient data transfer, and easy debugging. These solutions are suitable only for experienced users who can write their own optimized batch inference code. - Ray Data abstracts away not only the infrastructure management, but also the sharding your dataset, the parallelization of the inference over these shards, and the transfer of data from storage to CPU to GPU. + Ray Data abstracts away not only the infrastructure management, but also the sharding of your dataset, the parallelization of the inference over these shards, and the transfer of data from storage to CPU to GPU. .. dropdown:: Online inference solutions: Bento ML, Sagemaker Batch Transform diff --git a/doc/source/data/data-internals.rst b/doc/source/data/data-internals.rst index d4a449939b14..de084bb68995 100644 --- a/doc/source/data/data-internals.rst +++ b/doc/source/data/data-internals.rst @@ -96,7 +96,7 @@ images), then Ray Data can’t bound the block size. Shuffle Algorithms ------------------ -In data processing shuffling refers to the process of redistributing individual dataset's partitions (that in Ray Data are +In data processing, shuffling refers to the process of redistributing individual dataset's partitions (that in Ray Data are called :ref:`blocks `). Ray Data implements two main shuffle algorithms: diff --git a/doc/source/data/joining-data.rst b/doc/source/data/joining-data.rst index f9222ae74173..eeac75e1bc9e 100644 --- a/doc/source/data/joining-data.rst +++ b/doc/source/data/joining-data.rst @@ -4,9 +4,9 @@ Joining datasets ================ -.. note:: This is a new feature released in Ray 2.46. Note, this is an experimental feature and some things might not work as expected. +.. note:: This is a new feature released in Ray 2.46. Note that this is an experimental feature and some things might not work as expected. -Ray Data allows multiple :class:`~ray.data.dataset.Dataset` instances to be joined using different join types (inner, outer, semi, anti) based on the provided key columns like following: +Ray Data allows multiple :class:`~ray.data.dataset.Dataset` instances to be joined using different join types (inner, outer, semi, anti) based on the provided key columns as follows: .. testcode:: @@ -27,7 +27,7 @@ Ray Data allows multiple :class:`~ray.data.dataset.Dataset` instances to be join on=("id",), ) -Ray Data supports following join types (check out `Dataset.join` docs for up-to-date list): +Ray Data supports the following join types (check out `Dataset.join` docs for up-to-date list): **Inner/Outer Joins:** - Inner, Left Outer, Right Outer, Full Outer @@ -47,7 +47,7 @@ Configuring Joins Joins are generally memory-intensive operations that require accurate memory accounting and projection and hence are sensitive to skews and imbalances in the dataset. -Ray Data provides following levers to allow to tune up performance of joins for your workload: +Ray Data provides the following levers to allow tuning the performance of joins for your workload: - `num_partitions`: (required) specifies number of partitions both incoming datasets will be hash-partitioned into. Check out :ref:`configuring number of partitions ` section for guidance on how to tune this up. - `partition_size_hint`: (optional) Hint to joining operator about the estimated avg expected size of the individual partition (in bytes). If not specified, defaults to DataContext.target_max_block_size (128Mb by default). diff --git a/doc/source/data/key-concepts.rst b/doc/source/data/key-concepts.rst index 3be2437b261a..a1a979402ab6 100644 --- a/doc/source/data/key-concepts.rst +++ b/doc/source/data/key-concepts.rst @@ -12,7 +12,7 @@ There are two main concepts in Ray Data: * Datasets * Blocks -`Dataset` is the main user-facing Python API. It represents a distributed data collection and define data loading and processing operations. Users typically use the API by: +`Dataset` is the main user-facing Python API. It represents a distributed data collection and defines data loading and processing operations. Users typically use the API by: 1. Create a :class:`Dataset ` from external storage or in-memory data. 2. Apply transformations to the data. @@ -22,7 +22,7 @@ The Dataset API is lazy, meaning that operations aren't executed until you mater like :meth:`~ray.data.Dataset.show`. This allows Ray Data to optimize the execution plan and execute operations in a pipelined, streaming fashion. -*Block* is a set of rows representing single partition of the dataset. Blocks, as collection of rows represented by columnar formats (like Arrow) +*Block* is a set of rows representing single partition of the dataset. Blocks, as a collection of rows represented by columnar formats (like Arrow) are the basic unit of data processing in Ray Data: 1. Every dataset is partitioned into a number of blocks, then @@ -75,7 +75,7 @@ You can inspect the resulting logical plan by printing the dataset: +- MapBatches(add_column) +- Dataset(schema={...}) -When execution begins, Ray Data optimizes the logical plan, then translate it into a physical plan - a series of operators that implement the actual data transformations. During this translation: +When execution begins, Ray Data optimizes the logical plan, then translates it into a physical plan - a series of operators that implement the actual data transformations. During this translation: 1. A single logical operator may become multiple physical operators. For example, ``ReadOp`` becomes both ``InputDataBuffer`` and ``TaskPoolMapOperator``. 2. Both logical and physical plans go through optimization passes. For example, ``OperatorFusionRule`` combines map operators to reduce serialization overhead. diff --git a/doc/source/data/saving-data.rst b/doc/source/data/saving-data.rst index 7347f0bc4c5b..c6c814d04398 100644 --- a/doc/source/data/saving-data.rst +++ b/doc/source/data/saving-data.rst @@ -173,7 +173,7 @@ Writing into Partitioned Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When writing partitioned dataset (using Hive-style, folder-based partitioning) it's recommended to repartition the dataset by the partition columns prior to writing into it. -This allows you to *have the control over the file-sizes and their number*. When the dataset is repartitioned by the partition columns every block should contain all of the rows corresponding to particular partition, +This allows you to *have control over the file sizes and their number*. When the dataset is repartitioned by the partition columns every block should contain all of the rows corresponding to particular partition, meaning that the number of files created should be controlled based on the configuration provided to, for example, `write_parquet` method (such as `min_rows_per_file`, `max_rows_per_file`). Since every block is written out independently, when writing the dataset without prior diff --git a/doc/source/data/user-guide.rst b/doc/source/data/user-guide.rst index a1f450d17282..a83f34a7ab69 100644 --- a/doc/source/data/user-guide.rst +++ b/doc/source/data/user-guide.rst @@ -7,7 +7,7 @@ User Guides If you’re new to Ray Data, start with the :ref:`Ray Data Quickstart `. This user guide helps you navigate the Ray Data project and -show you how achieve several tasks. +shows you how to achieve several tasks. .. toctree:: :maxdepth: 2 From aacea0660c5ece357a0c628987d50ed91554df1c Mon Sep 17 00:00:00 2001 From: Ping Dai Date: Wed, 3 Sep 2025 03:10:34 +0800 Subject: [PATCH 393/634] [hotfix][data]Fix some Python code defects in ray data scanned by PyLint tool (#55417) ## Overview We scanned the ray data code using the PyLint tool and found some defects. Here are some scan results based on ray 2.46 version: "ray/python/ray/data/read_api.py:3214:4: R1705: Unnecessary "elif" after "return", remove the leading "el" from "elif" (no-else-return) ray/python/ray/data/datasource/file_based_datasource.py:276:20: R1730: Consider using 'num_threads = min(num_threads, len(read_paths))' instead of unnecessary if block (consider-using-min-builtin) R1705: Unnecessary "else" after "return", remove the "else" and de-indent the code inside it (no-else-return)" Scanning the latest branch of the master will also yield similar results ## Why are these changes needed? The modifications in PR do not affect the code logic and functionality, nor do they affect existing unit test cases. The aim is to reduce code complexity and redundant code without changing the code logic, and enhance the readability of ray code. ## Related issue number Closes #53881 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [x] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: daiping8 --- python/ray/data/dataset.py | 9 ++------- python/ray/data/datasource/file_based_datasource.py | 3 +-- python/ray/data/read_api.py | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 079ac1f0fcf7..0fd023adb610 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -923,11 +923,7 @@ def add_column(batch: DataBatch) -> DataBatch: # The index of the column must be set # to align with the index of the batch. - if ( - isinstance(column, pd.Series) - or isinstance(column, pd.DataFrame) - or isinstance(column, pd.Index) - ): + if isinstance(column, (pd.DataFrame, pd.Index, pd.Series)): column.index = batch.index batch.loc[:, col] = column return batch @@ -948,8 +944,7 @@ def add_column(batch: DataBatch) -> DataBatch: column_idx = batch.schema.get_field_index(col) if column_idx == -1: return batch.append_column(col, column) - else: - return batch.set_column(column_idx, col, column) + return batch.set_column(column_idx, col, column) else: # batch format is assumed to be numpy since we checked at the diff --git a/python/ray/data/datasource/file_based_datasource.py b/python/ray/data/datasource/file_based_datasource.py index 521872ac888b..8e66d5f185ca 100644 --- a/python/ray/data/datasource/file_based_datasource.py +++ b/python/ray/data/datasource/file_based_datasource.py @@ -273,8 +273,7 @@ def read_task_fn(): num_threads = 0 if num_threads > 0: - if len(read_paths) < num_threads: - num_threads = len(read_paths) + num_threads = min(num_threads, len(read_paths)) logger.debug( f"Reading {len(read_paths)} files with {num_threads} threads." diff --git a/python/ray/data/read_api.py b/python/ray/data/read_api.py index 227a1378f68f..80ae80d4ff04 100644 --- a/python/ray/data/read_api.py +++ b/python/ray/data/read_api.py @@ -3428,7 +3428,7 @@ def from_huggingface( hf_ds_arrow = dataset.with_format("arrow") ray_ds = from_arrow(hf_ds_arrow[:], override_num_blocks=override_num_blocks) return ray_ds - elif isinstance(dataset, (datasets.DatasetDict, datasets.IterableDatasetDict)): + if isinstance(dataset, (datasets.DatasetDict, datasets.IterableDatasetDict)): available_keys = list(dataset.keys()) raise DeprecationWarning( "You provided a Hugging Face DatasetDict or IterableDatasetDict, " From eb50be6c6dea82fc5ba4b0807bbcb90b4de28dff Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Tue, 2 Sep 2025 12:51:18 -0700 Subject: [PATCH 394/634] [Core] Change the size of `test_gpu_objects_gloo.py` to large (#56168) --- python/ray/tests/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD index 1040d7411cc6..2181d005e59d 100644 --- a/python/ray/tests/BUILD +++ b/python/ray/tests/BUILD @@ -609,7 +609,7 @@ py_test_module_list( ) py_test_module_list( - size = "medium", + size = "large", files = [ "test_gpu_objects_gloo.py", ], From 1d2c95f8e0cee754d20c0df4c598221216162699 Mon Sep 17 00:00:00 2001 From: Rueian Date: Tue, 2 Sep 2025 13:05:00 -0700 Subject: [PATCH 395/634] [core] move the `RayletClientInterface` to its own build target (#56101) ## Why are these changes needed? 1. A new `//src/ray/raylet_client:raylet_client_interface` target containing only the `RayletClientInterface`. 2. A new `//src/ray/raylet_client:raylet_client_pool` target moved from the node_manager. 3. A new `//src/ray/raylet_client:node_manager_client` target moved from the node_manager. 4. Remove `using` statements in the `raylet_client.h` that allow others to omit `ray::` implicitly. There are no behavioral changes. ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Rueian --- BUILD.bazel | 2 +- python/ray/includes/common.pxd | 8 +- src/fakes/ray/rpc/raylet/BUILD.bazel | 2 +- src/fakes/ray/rpc/raylet/raylet_client.h | 3 +- src/ray/core_worker/BUILD.bazel | 4 +- src/ray/core_worker/core_worker.h | 2 +- src/ray/core_worker/core_worker_process.cc | 1 + .../experimental_mutable_object_provider.h | 2 +- .../io_ray_runtime_task_NativeTaskExecutor.cc | 2 +- src/ray/core_worker/object_recovery_manager.h | 4 +- .../core_worker/task_submission/BUILD.bazel | 2 +- .../task_submission/normal_task_submitter.h | 4 +- .../task_submission/tests/BUILD.bazel | 2 +- .../tests/normal_task_submitter_test.cc | 2 +- src/ray/core_worker/tests/BUILD.bazel | 2 +- .../tests/object_recovery_manager_test.cc | 2 +- src/ray/gcs/gcs_server/BUILD.bazel | 10 +- src/ray/gcs/gcs_server/gcs_actor_manager.cc | 4 +- src/ray/gcs/gcs_server/gcs_actor_scheduler.h | 5 +- .../gcs_server/gcs_autoscaler_state_manager.h | 2 +- src/ray/gcs/gcs_server/gcs_node_manager.h | 2 +- .../gcs_placement_group_scheduler.h | 5 +- src/ray/gcs/gcs_server/gcs_server.cc | 1 + src/ray/gcs/gcs_server/gcs_server.h | 2 +- src/ray/gcs/gcs_server/gcs_server_main.cc | 2 +- src/ray/raylet/BUILD.bazel | 3 +- src/ray/raylet/main.cc | 35 +-- src/ray/raylet/node_manager.h | 3 +- src/ray/raylet_client/BUILD.bazel | 50 +++- .../node_manager_client.h | 0 src/ray/raylet_client/raylet_client.cc | 2 + src/ray/raylet_client/raylet_client.h | 198 +--------------- .../raylet_client/raylet_client_interface.h | 215 ++++++++++++++++++ .../raylet_client_pool.cc | 2 +- .../raylet_client_pool.h | 3 +- .../tests/BUILD.bazel | 3 +- .../tests/raylet_client_pool_test.cc | 2 +- src/ray/rpc/BUILD.bazel | 30 +-- .../rpc/node_manager/node_manager_server.h | 2 +- src/ray/rpc/worker/core_worker_client_pool.h | 4 +- 40 files changed, 335 insertions(+), 294 deletions(-) rename src/ray/{rpc/node_manager => raylet_client}/node_manager_client.h (100%) create mode 100644 src/ray/raylet_client/raylet_client_interface.h rename src/ray/{rpc/node_manager => raylet_client}/raylet_client_pool.cc (98%) rename src/ray/{rpc/node_manager => raylet_client}/raylet_client_pool.h (96%) rename src/ray/{rpc/node_manager => raylet_client}/tests/BUILD.bazel (78%) rename src/ray/{rpc/node_manager => raylet_client}/tests/raylet_client_pool_test.cc (99%) diff --git a/BUILD.bazel b/BUILD.bazel index 8fc6429846d3..24bfe58c2f90 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -130,7 +130,7 @@ ray_cc_library( ), deps = [ "//src/ray/common:asio", - "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/raylet_client:raylet_client_interface", ], ) diff --git a/python/ray/includes/common.pxd b/python/ray/includes/common.pxd index d15612e6dd20..d15746effdde 100644 --- a/python/ray/includes/common.pxd +++ b/python/ray/includes/common.pxd @@ -158,7 +158,7 @@ cdef extern from "ray/common/id.h" namespace "ray" nogil: cdef extern from "src/ray/protobuf/common.pb.h" nogil: - cdef cppclass CLanguage "Language": + cdef cppclass CLanguage "ray::Language": pass cdef cppclass CWorkerType "ray::core::WorkerType": pass @@ -243,9 +243,9 @@ cdef extern from "src/ray/protobuf/common.pb.h" nogil: # This is a workaround for C++ enum class since Cython has no corresponding # representation. cdef extern from "src/ray/protobuf/common.pb.h" nogil: - cdef CLanguage LANGUAGE_PYTHON "Language::PYTHON" - cdef CLanguage LANGUAGE_CPP "Language::CPP" - cdef CLanguage LANGUAGE_JAVA "Language::JAVA" + cdef CLanguage LANGUAGE_PYTHON "ray::Language::PYTHON" + cdef CLanguage LANGUAGE_CPP "ray::Language::CPP" + cdef CLanguage LANGUAGE_JAVA "ray::Language::JAVA" cdef extern from "src/ray/protobuf/common.pb.h" nogil: cdef CWorkerType WORKER_TYPE_WORKER "ray::core::WorkerType::WORKER" diff --git a/src/fakes/ray/rpc/raylet/BUILD.bazel b/src/fakes/ray/rpc/raylet/BUILD.bazel index 6e2730394697..f6ced3e4fc2e 100644 --- a/src/fakes/ray/rpc/raylet/BUILD.bazel +++ b/src/fakes/ray/rpc/raylet/BUILD.bazel @@ -4,6 +4,6 @@ ray_cc_library( name = "fake_raylet_client", hdrs = ["raylet_client.h"], deps = [ - "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/raylet_client:raylet_client_interface", ], ) diff --git a/src/fakes/ray/rpc/raylet/raylet_client.h b/src/fakes/ray/rpc/raylet/raylet_client.h index 1a434abc20a6..fa4ef1436b6b 100644 --- a/src/fakes/ray/rpc/raylet/raylet_client.h +++ b/src/fakes/ray/rpc/raylet/raylet_client.h @@ -14,7 +14,8 @@ #pragma once -#include "ray/raylet_client/raylet_client.h" +#include "ray/common/scheduling/scheduling_ids.h" +#include "ray/raylet_client/raylet_client_interface.h" namespace ray { diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 5a6c7c1963c3..67a2cb237046 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -312,7 +312,7 @@ ray_cc_library( hdrs = ["experimental_mutable_object_provider.h"], deps = [ ":experimental_mutable_object_manager", - "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/raylet_client:raylet_client_interface", "//src/ray/rpc:client_call", ], ) @@ -336,7 +336,7 @@ ray_cc_library( ":reference_count", ":task_manager", "//src/ray/common:id", - "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/raylet_client:raylet_client_pool", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/synchronization", ], diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index b44df2b5ee05..480f11752972 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -52,7 +52,7 @@ #include "ray/ipc/raylet_ipc_client_interface.h" #include "ray/pubsub/publisher.h" #include "ray/pubsub/subscriber.h" -#include "ray/raylet_client/raylet_client.h" +#include "ray/raylet_client/raylet_client_interface.h" #include "ray/rpc/worker/core_worker_server.h" #include "ray/util/process.h" #include "ray/util/shared_lru.h" diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index c870835c47bd..1d8fed4ed879 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -35,6 +35,7 @@ #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/gcs/pb_util.h" #include "ray/ipc/raylet_ipc_client.h" +#include "ray/raylet_client/raylet_client.h" #include "ray/stats/stats.h" #include "ray/util/container_util.h" #include "ray/util/env.h" diff --git a/src/ray/core_worker/experimental_mutable_object_provider.h b/src/ray/core_worker/experimental_mutable_object_provider.h index 364d1f7d3aa1..16a9c11e2149 100644 --- a/src/ray/core_worker/experimental_mutable_object_provider.h +++ b/src/ray/core_worker/experimental_mutable_object_provider.h @@ -18,7 +18,7 @@ #include #include "ray/core_worker/experimental_mutable_object_manager.h" -#include "ray/raylet_client/raylet_client.h" +#include "ray/raylet_client/raylet_client_interface.h" #include "ray/rpc/client_call.h" namespace ray { diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc index 9c0adc401893..55ed42f580a5 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc @@ -20,7 +20,7 @@ #include "ray/common/id.h" #include "ray/core_worker/common.h" #include "ray/core_worker/core_worker.h" -#include "ray/raylet_client/raylet_client.h" +#include "ray/raylet_client/raylet_client_interface.h" #ifdef __cplusplus extern "C" { diff --git a/src/ray/core_worker/object_recovery_manager.h b/src/ray/core_worker/object_recovery_manager.h index ede91805c789..0e29b5d6ab7b 100644 --- a/src/ray/core_worker/object_recovery_manager.h +++ b/src/ray/core_worker/object_recovery_manager.h @@ -25,8 +25,8 @@ #include "ray/core_worker/reference_count.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/core_worker/task_manager.h" -#include "ray/raylet_client/raylet_client.h" -#include "ray/rpc/node_manager/raylet_client_pool.h" +#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/raylet_client/raylet_client_pool.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/task_submission/BUILD.bazel b/src/ray/core_worker/task_submission/BUILD.bazel index 6261dddb4f60..4e00ea8ae4eb 100644 --- a/src/ray/core_worker/task_submission/BUILD.bazel +++ b/src/ray/core_worker/task_submission/BUILD.bazel @@ -93,7 +93,7 @@ ray_cc_library( "//src/ray/core_worker:memory_store", "//src/ray/core_worker:task_manager_interface", "//src/ray/gcs:gcs_pb_util", - "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/raylet_client:raylet_client_interface", "//src/ray/rpc:core_worker_client", "@com_google_absl//absl/base:core_headers", ], diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.h b/src/ray/core_worker/task_submission/normal_task_submitter.h index 197ed02dd2f9..423b3d479500 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.h +++ b/src/ray/core_worker/task_submission/normal_task_submitter.h @@ -29,8 +29,8 @@ #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/core_worker/task_manager_interface.h" #include "ray/core_worker/task_submission/dependency_resolver.h" -#include "ray/raylet_client/raylet_client.h" -#include "ray/rpc/node_manager/raylet_client_pool.h" +#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/raylet_client/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" diff --git a/src/ray/core_worker/task_submission/tests/BUILD.bazel b/src/ray/core_worker/task_submission/tests/BUILD.bazel index 0647614c0dd7..5687a2de0698 100644 --- a/src/ray/core_worker/task_submission/tests/BUILD.bazel +++ b/src/ray/core_worker/task_submission/tests/BUILD.bazel @@ -70,7 +70,7 @@ ray_cc_test( "//src/ray/common:test_utils", "//src/ray/core_worker:memory_store", "//src/ray/core_worker/task_submission:normal_task_submitter", - "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/raylet_client:raylet_client_interface", "//src/ray/rpc:core_worker_client", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", diff --git a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc index c9b583fd3e10..707770c8b2a8 100644 --- a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc @@ -30,7 +30,7 @@ #include "ray/common/task/task_util.h" #include "ray/common/test_utils.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" -#include "ray/raylet_client/raylet_client.h" +#include "ray/raylet_client/raylet_client_interface.h" #include "ray/rpc/worker/core_worker_client.h" namespace ray { diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index 205b16d93ae2..c6f99e1103d9 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -75,7 +75,7 @@ ray_cc_test( "//src/ray/core_worker:memory_store", "//src/ray/core_worker:object_recovery_manager", "//src/ray/object_manager:object_manager_common", - "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/raylet_client:raylet_client_interface", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/core_worker/tests/object_recovery_manager_test.cc b/src/ray/core_worker/tests/object_recovery_manager_test.cc index db5ad2b37048..370aefebea00 100644 --- a/src/ray/core_worker/tests/object_recovery_manager_test.cc +++ b/src/ray/core_worker/tests/object_recovery_manager_test.cc @@ -30,7 +30,7 @@ #include "ray/common/task/task_util.h" #include "ray/common/test_utils.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" -#include "ray/raylet_client/raylet_client.h" +#include "ray/raylet_client/raylet_client_interface.h" namespace ray { namespace core { diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index b11f99a222ed..846b509d5d29 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -78,7 +78,7 @@ ray_cc_library( "//src/ray/protobuf:autoscaler_cc_proto", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/protobuf:ray_syncer_cc_proto", - "//src/ray/rpc:node_manager_client", + "//src/ray/raylet_client:raylet_client_pool", "//src/ray/stats:stats_metric", "//src/ray/util:event", "//src/ray/util:logging", @@ -286,8 +286,7 @@ ray_cc_library( "//src/ray/common:task_common", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "//src/ray/raylet/scheduling:scheduling_context", - "//src/ray/raylet_client:raylet_client_lib", - "//src/ray/rpc:node_manager_client", + "//src/ray/raylet_client:raylet_client_interface", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", ], @@ -394,9 +393,8 @@ ray_cc_library( "//src/ray/common:task_common", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/raylet/scheduling:cluster_lease_manager", - "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/raylet_client:raylet_client_interface", "//src/ray/rpc:core_worker_client", - "//src/ray/rpc:node_manager_client", "//src/ray/util:logging", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -519,10 +517,10 @@ ray_cc_library( "//src/ray/pubsub:publisher", "//src/ray/raylet/scheduling:scheduler", "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/raylet_client:raylet_client_pool", "//src/ray/rpc:core_worker_client", "//src/ray/rpc:grpc_server", "//src/ray/rpc:metrics_agent_client", - "//src/ray/rpc:node_manager_client", "//src/ray/util:counter_map", "//src/ray/util:exponential_backoff", "//src/ray/util:network_util", diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index 962ab4b8be02..061fc36fce81 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -80,7 +80,7 @@ const ray::rpc::ActorDeathCause GenWorkerDiedCause( const ray::rpc::ActorDeathCause GenOwnerDiedCause( const ray::gcs::GcsActor *actor, - const WorkerID &owner_id, + const ray::WorkerID &owner_id, const ray::rpc::WorkerExitType disconnect_type, const std::string &disconnect_detail, const std::string &owner_ip_address) { @@ -153,7 +153,7 @@ bool OnInitializeActorShouldLoad(const ray::gcs::GcsInitData &gcs_init_data, } const auto &actor_task_spec = ray::map_find_or_die(actor_task_specs, actor_id); - ActorID root_detached_actor_id = + ray::ActorID root_detached_actor_id = ray::TaskSpecification(actor_task_spec).RootDetachedActorId(); if (root_detached_actor_id.IsNil()) { // owner is job, NOT detached actor, should die with job diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h index 582957e99445..e05fc10e46eb 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h @@ -31,9 +31,8 @@ #include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" -#include "ray/raylet_client/raylet_client.h" -#include "ray/rpc/node_manager/node_manager_client.h" -#include "ray/rpc/node_manager/raylet_client_pool.h" +#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/raylet_client/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "src/ray/protobuf/gcs_service.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h index 2dae21963fa2..d262191611fb 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h @@ -31,7 +31,7 @@ #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/state_util.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" -#include "ray/rpc/node_manager/raylet_client_pool.h" +#include "ray/raylet_client/raylet_client_pool.h" #include "ray/util/thread_checker.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.h b/src/ray/gcs/gcs_server/gcs_node_manager.h index 710899260779..b032d2b5662d 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/ray/gcs/gcs_server/gcs_node_manager.h @@ -27,7 +27,7 @@ #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" -#include "ray/rpc/node_manager/raylet_client_pool.h" +#include "ray/raylet_client/raylet_client_pool.h" #include "ray/stats/metric_defs.h" #include "ray/util/event.h" #include "src/ray/protobuf/autoscaler.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h index ee3b7a44e161..778a58f529dd 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h @@ -30,9 +30,8 @@ #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/scheduling/policy/scheduling_context.h" -#include "ray/raylet_client/raylet_client.h" -#include "ray/rpc/node_manager/node_manager_client.h" -#include "ray/rpc/node_manager/raylet_client_pool.h" +#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/raylet_client/raylet_client_pool.h" #include "src/ray/protobuf/gcs_service.pb.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 3720f93847ff..6ac9cb8492a5 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -36,6 +36,7 @@ #include "ray/gcs/store_client/redis_store_client.h" #include "ray/gcs/store_client/store_client.h" #include "ray/pubsub/publisher.h" +#include "ray/raylet_client/raylet_client.h" #include "ray/stats/stats.h" #include "ray/util/network_util.h" diff --git a/src/ray/gcs/gcs_server/gcs_server.h b/src/ray/gcs/gcs_server/gcs_server.h index fc4ca3d63239..6a869ef9369e 100644 --- a/src/ray/gcs/gcs_server/gcs_server.h +++ b/src/ray/gcs/gcs_server/gcs_server.h @@ -39,10 +39,10 @@ #include "ray/gcs/store_client/redis_store_client.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" +#include "ray/raylet_client/raylet_client_pool.h" #include "ray/rpc/client_call.h" #include "ray/rpc/grpc_server.h" #include "ray/rpc/metrics_agent_client.h" -#include "ray/rpc/node_manager/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/throttler.h" diff --git a/src/ray/gcs/gcs_server/gcs_server_main.cc b/src/ray/gcs/gcs_server/gcs_server_main.cc index 7d7fa32f635e..04a277924503 100644 --- a/src/ray/gcs/gcs_server/gcs_server_main.cc +++ b/src/ray/gcs/gcs_server/gcs_server_main.cc @@ -123,7 +123,7 @@ int main(int argc, char *argv[]) { {ray::stats::VersionKey, kRayVersion}, {ray::stats::NodeAddressKey, node_ip_address}, {ray::stats::SessionNameKey, session_name}}; - ray::stats::Init(global_tags, metrics_agent_port, WorkerID::Nil()); + ray::stats::Init(global_tags, metrics_agent_port, ray::WorkerID::Nil()); // Initialize event framework. if (RayConfig::instance().event_log_reporter_enabled() && !log_dir.empty()) { diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index 65a002214838..48d6c5081e86 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -237,7 +237,6 @@ ray_cc_library( "//src/ray/pubsub:subscriber", "//src/ray/raylet/scheduling:scheduler", "//src/ray/rpc:core_worker_client", - "//src/ray/rpc:node_manager_client", "//src/ray/rpc:node_manager_server", "//src/ray/stats:stats_lib", "//src/ray/util:cmd_line_utils", @@ -287,6 +286,8 @@ ray_cc_binary( "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/object_manager:ownership_object_directory", "//src/ray/raylet/scheduling:cluster_lease_manager", + "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/raylet_client:raylet_client_pool", "//src/ray/rpc:metrics_agent_client", "//src/ray/stats:stats_lib", "//src/ray/util:cmd_line_utils", diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 08eaa3ce91d5..edc2fdba7660 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -35,6 +35,7 @@ #include "ray/raylet/local_object_manager.h" #include "ray/raylet/local_object_manager_interface.h" #include "ray/raylet/raylet.h" +#include "ray/raylet_client/raylet_client.h" #include "ray/stats/stats.h" #include "ray/util/cmd_line_utils.h" #include "ray/util/event.h" @@ -585,10 +586,10 @@ int main(int argc, char *argv[]) { *gcs_client, core_worker_subscriber.get(), worker_rpc_pool.get(), - [&](const ObjectID &obj_id, const ray::rpc::ErrorType &error_type) { + [&](const ray::ObjectID &obj_id, const ray::rpc::ErrorType &error_type) { ray::rpc::ObjectReference ref; ref.set_object_id(obj_id.Binary()); - node_manager->MarkObjectsAsFailed(error_type, {ref}, JobID::Nil()); + node_manager->MarkObjectsAsFailed(error_type, {ref}, ray::JobID::Nil()); }); object_manager = std::make_unique( @@ -598,7 +599,7 @@ int main(int argc, char *argv[]) { *gcs_client, object_directory.get(), /*restore_spilled_object=*/ - [&](const ObjectID &object_id, + [&](const ray::ObjectID &object_id, int64_t object_size, const std::string &object_url, std::function callback) { @@ -606,7 +607,7 @@ int main(int argc, char *argv[]) { object_id, object_size, object_url, std::move(callback)); }, /*get_spilled_object_url=*/ - [&](const ObjectID &object_id) { + [&](const ray::ObjectID &object_id) { return local_object_manager->GetLocalSpilledObjectURL(object_id); }, /*spill_objects_callback=*/ @@ -631,10 +632,12 @@ int main(int argc, char *argv[]) { node_manager->HandleObjectLocal(object_info); }, /*delete_object_callback=*/ - [&](const ObjectID &object_id) { node_manager->HandleObjectMissing(object_id); }, + [&](const ray::ObjectID &object_id) { + node_manager->HandleObjectMissing(object_id); + }, /*pin_object=*/ - [&](const ObjectID &object_id) { - std::vector object_ids = {object_id}; + [&](const ray::ObjectID &object_id) { + std::vector object_ids = {object_id}; std::vector> results; std::unique_ptr result; if (node_manager->GetObjectsFromPlasma(object_ids, &results) && @@ -644,10 +647,10 @@ int main(int argc, char *argv[]) { return result; }, /*fail_pull_request=*/ - [&](const ObjectID &object_id, ray::rpc::ErrorType error_type) { + [&](const ray::ObjectID &object_id, ray::rpc::ErrorType error_type) { ray::rpc::ObjectReference ref; ref.set_object_id(object_id.Binary()); - node_manager->MarkObjectsAsFailed(error_type, {ref}, JobID::Nil()); + node_manager->MarkObjectsAsFailed(error_type, {ref}, ray::JobID::Nil()); }); local_object_manager = std::make_unique( @@ -664,12 +667,12 @@ int main(int argc, char *argv[]) { RayConfig::instance().is_external_storage_type_fs(), /*max_fused_object_count*/ RayConfig::instance().max_fused_object_count(), /*on_objects_freed*/ - [&](const std::vector &object_ids) { + [&](const std::vector &object_ids) { object_manager->FreeObjects(object_ids, /*local_only=*/false); }, /*is_plasma_object_spillable*/ - [&](const ObjectID &object_id) { + [&](const ray::ObjectID &object_id) { return object_manager->IsPlasmaObjectSpillable(object_id); }, /*core_worker_subscriber_=*/core_worker_subscriber.get(), @@ -684,7 +687,7 @@ int main(int argc, char *argv[]) { node_manager_config.resource_config.GetResourceMap(), /*is_node_available_fn*/ [&](ray::scheduling::NodeID id) { - return gcs_client->Nodes().Get(NodeID::FromBinary(id.Binary())) != nullptr; + return gcs_client->Nodes().Get(ray::NodeID::FromBinary(id.Binary())) != nullptr; }, /*get_used_object_store_memory*/ [&]() { @@ -707,7 +710,7 @@ int main(int argc, char *argv[]) { /*labels*/ node_manager_config.labels); - auto get_node_info_func = [&](const NodeID &id) { + auto get_node_info_func = [&](const ray::NodeID &id) { return gcs_client->Nodes().Get(id); }; auto announce_infeasible_lease = [](const ray::RayLease &lease) { @@ -764,7 +767,7 @@ int main(int argc, char *argv[]) { get_node_info_func, *worker_pool, leased_workers, - [&](const std::vector &object_ids, + [&](const std::vector &object_ids, std::vector> *results) { return node_manager->GetObjectsFromPlasma(object_ids, results); }, @@ -777,7 +780,7 @@ int main(int argc, char *argv[]) { announce_infeasible_lease, *local_lease_manager); - auto raylet_client_factory = [&](const NodeID &id) { + auto raylet_client_factory = [&](const ray::NodeID &id) { const ray::rpc::GcsNodeInfo *node_info = gcs_client->Nodes().Get(id); RAY_CHECK(node_info) << "No GCS info for node " << id; auto addr = ray::rpc::RayletClientPool::GenerateRayletAddress( @@ -834,7 +837,7 @@ int main(int argc, char *argv[]) { {ray::stats::VersionKey, kRayVersion}, {ray::stats::NodeAddressKey, node_ip_address}, {ray::stats::SessionNameKey, session_name}}; - ray::stats::Init(global_tags, metrics_agent_port, WorkerID::Nil()); + ray::stats::Init(global_tags, metrics_agent_port, ray::WorkerID::Nil()); metrics_agent_client = std::make_unique( "127.0.0.1", metrics_agent_port, main_service, *client_call_manager); metrics_agent_client->WaitForServerReady( diff --git a/src/ray/raylet/node_manager.h b/src/ray/raylet/node_manager.h index 4631f8bf0b72..4feee4e3fabf 100644 --- a/src/ray/raylet/node_manager.h +++ b/src/ray/raylet/node_manager.h @@ -48,9 +48,8 @@ #include "ray/raylet/wait_manager.h" #include "ray/raylet/worker_killing_policy.h" #include "ray/raylet/worker_pool.h" -#include "ray/raylet_client/raylet_client.h" +#include "ray/raylet_client/raylet_client_pool.h" #include "ray/rpc/node_manager/node_manager_server.h" -#include "ray/rpc/node_manager/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/throttler.h" diff --git a/src/ray/raylet_client/BUILD.bazel b/src/ray/raylet_client/BUILD.bazel index 9dc98c643617..e8f6d4f05244 100644 --- a/src/ray/raylet_client/BUILD.bazel +++ b/src/ray/raylet_client/BUILD.bazel @@ -1,20 +1,56 @@ load("//bazel:ray.bzl", "ray_cc_library") -exports_files([ - "raylet_client.h", -]) +ray_cc_library( + name = "raylet_client_interface", + hdrs = [ + "raylet_client_interface.h", + ], + deps = [ + "//src/ray/protobuf:autoscaler_cc_proto", + "//src/ray/protobuf:common_cc_proto", + "//src/ray/protobuf:node_manager_cc_proto", + "//src/ray/rpc:client_call", + ], +) + +ray_cc_library( + name = "raylet_client_pool", + srcs = ["raylet_client_pool.cc"], + hdrs = [ + "raylet_client_pool.h", + ], + deps = [ + ":raylet_client_interface", + "//src/ray/gcs/gcs_client:gcs_client_lib", + ], +) + +ray_cc_library( + name = "node_manager_client", + hdrs = [ + "node_manager_client.h", + ], + visibility = [":__subpackages__"], + deps = [ + ":raylet_client_interface", + "//src/ray/common:id", + "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/protobuf:node_manager_cc_grpc", + "//src/ray/rpc:client_call", + "//src/ray/rpc:grpc_client", + "//src/ray/util:network_util", + ], +) ray_cc_library( name = "raylet_client_lib", srcs = ["raylet_client.cc"], hdrs = ["raylet_client.h"], deps = [ - "//src/ray/common:id", - "//src/ray/common:status", - "//src/ray/common:task_common", + ":node_manager_client", + ":raylet_client_interface", "//src/ray/flatbuffers:node_manager_generated", "//src/ray/protobuf:common_cc_proto", - "//src/ray/rpc:node_manager_client", "//src/ray/util:logging", ], ) diff --git a/src/ray/rpc/node_manager/node_manager_client.h b/src/ray/raylet_client/node_manager_client.h similarity index 100% rename from src/ray/rpc/node_manager/node_manager_client.h rename to src/ray/raylet_client/node_manager_client.h diff --git a/src/ray/raylet_client/raylet_client.cc b/src/ray/raylet_client/raylet_client.cc index 5981265d503c..87d91f089ee8 100644 --- a/src/ray/raylet_client/raylet_client.cc +++ b/src/ray/raylet_client/raylet_client.cc @@ -20,7 +20,9 @@ #include #include +#include "ray/common/bundle_spec.h" #include "ray/common/ray_config.h" +#include "ray/raylet_client/node_manager_client.h" #include "ray/util/logging.h" namespace ray::raylet { diff --git a/src/ray/raylet_client/raylet_client.h b/src/ray/raylet_client/raylet_client.h index 75592cb1a90c..c2c9c6263d7b 100644 --- a/src/ray/raylet_client/raylet_client.h +++ b/src/ray/raylet_client/raylet_client.h @@ -20,25 +20,7 @@ #include #include -#include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/buffer.h" -#include "ray/common/bundle_spec.h" -#include "ray/common/status.h" -#include "ray/common/status_or.h" -#include "ray/ipc/client_connection.h" -#include "ray/rpc/node_manager/node_manager_client.h" -#include "ray/util/process.h" -#include "src/ray/protobuf/common.pb.h" -#include "src/ray/protobuf/gcs.pb.h" - -using ray::ActorID; -using ray::JobID; -using ray::NodeID; -using ray::ObjectID; -using ray::TaskID; -using ray::WorkerID; - -using ray::Language; +#include "ray/raylet_client/raylet_client_interface.h" // Maps from resource name to its allocation. using ResourceMappingType = @@ -46,180 +28,10 @@ using ResourceMappingType = namespace ray { -class RayletClientInterface { - public: - /// Request to a raylet to pin a plasma object. The callback will be sent via gRPC. - virtual void PinObjectIDs( - const rpc::Address &caller_address, - const std::vector &object_ids, - const ObjectID &generator_id, - const ray::rpc::ClientCallback &callback) = 0; - - /// Requests a worker from the raylet. The callback will be sent via gRPC. - /// \param lease_spec Lease that is requested by the owner. - /// \param grant_or_reject: True if we we should either grant or reject the request - /// but no spillback. - /// \param callback: The callback to call when the request finishes. - /// \param backlog_size The queue length for the given shape on the CoreWorker. - /// \param lease_id Unique lease ID for this worker lease request. - virtual void RequestWorkerLease( - const rpc::LeaseSpec &lease_spec, - bool grant_or_reject, - const ray::rpc::ClientCallback &callback, - const int64_t backlog_size = -1, - const bool is_selected_based_on_locality = false) = 0; - - /// Returns a worker to the raylet. - /// \param worker_port The local port of the worker on the raylet node. - /// \param worker_id The unique worker id of the worker on the raylet node. - /// \param disconnect_worker Whether the raylet should disconnect the worker. - /// \param worker_exiting Whether the worker is exiting and cannot be reused. - /// \return ray::Status - virtual ray::Status ReturnWorkerLease(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) = 0; - - /// Request the raylet to prestart workers. In `request` we can set the worker's owner, - /// runtime env info and number of workers. - /// - virtual void PrestartWorkers( - const rpc::PrestartWorkersRequest &request, - const rpc::ClientCallback &callback) = 0; - - /// Notify raylets to release unused workers. - /// \param workers_in_use Workers currently in use. - /// \param callback Callback that will be called after raylet completes the release of - /// unused workers. \return ray::Status - virtual void ReleaseUnusedActorWorkers( - const std::vector &workers_in_use, - const rpc::ClientCallback &callback) = 0; - - virtual void CancelWorkerLease( - const LeaseID &lease_id, - const rpc::ClientCallback &callback) = 0; - - /// Report the backlog size of a given worker and a given scheduling class to the - /// raylet. - /// \param worker_id The ID of the worker that reports the backlog size. - /// \param backlog_reports The backlog report for each scheduling class - virtual void ReportWorkerBacklog( - const WorkerID &worker_id, - const std::vector &backlog_reports) = 0; - - virtual void GetWorkerFailureCause( - const LeaseID &lease_id, - const ray::rpc::ClientCallback &callback) = 0; - - /// Request a raylet to prepare resources of given bundles for atomic placement group - /// creation. This is used for the first phase of atomic placement group creation. The - /// callback will be sent via gRPC. - /// \param bundle_specs Bundles to be scheduled at this raylet. - /// \return ray::Status - virtual void PrepareBundleResources( - const std::vector> &bundle_specs, - const ray::rpc::ClientCallback - &callback) = 0; - - /// Request a raylet to commit resources of given bundles for atomic placement group - /// creation. This is used for the second phase of atomic placement group creation. The - /// callback will be sent via gRPC. - /// \param bundle_specs Bundles to be scheduled at this raylet. - /// \return ray::Status - virtual void CommitBundleResources( - const std::vector> &bundle_specs, - const ray::rpc::ClientCallback &callback) = 0; - - virtual void CancelResourceReserve( - const BundleSpecification &bundle_spec, - const ray::rpc::ClientCallback &callback) = 0; - - virtual void ReleaseUnusedBundles( - const std::vector &bundles_in_use, - const rpc::ClientCallback &callback) = 0; - - virtual void GetResourceLoad( - const rpc::ClientCallback &callback) = 0; - /// Registers a mutable object on this node so that it can be read. Writes are performed - /// on a remote node. This local node creates a mapping from `object_id` -> - /// `reader_ref`. - /// - /// \param writer_object_id The object ID of the mutable object on the remote node that - /// is written to. - /// \param num_readers The number of readers that will read the object on this local - /// node. - /// \param reader_object_id The object ID of the mutable object that is read on this - /// local node. - /// \param callback This callback is executed to send a reply to the remote - /// node once the mutable object is registered. - virtual void RegisterMutableObjectReader( - const ObjectID &writer_object_id, - int64_t num_readers, - const ObjectID &reader_object_id, - const rpc::ClientCallback &callback) = 0; - - /// Handles a mutable object write that was performed on a remote node and is being - /// transferred to this node so that it can be read. - /// - /// \param writer_object_id The object ID of the mutable object on the remote node that - /// is written to. This is *not* the object ID of the corresponding mutable object on - /// this local node. - /// \param data_size The size of the data to write to the mutable object on this local - /// node. - /// \param metadata_size The size of the metadata to write to the mutable object on this - /// local node. - /// \param data The data to write to the mutable object on this local node. - /// \param metadata The metadata to write to the mutable object on this local node. - /// \param callback This callback is executed to send a reply to the remote node once - /// the mutable object is transferred. - virtual void PushMutableObject( - const ObjectID &writer_object_id, - uint64_t data_size, - uint64_t metadata_size, - void *data, - void *metadata, - const rpc::ClientCallback &callback) = 0; - - /// Get the system config from Raylet. - /// \param callback Callback that will be called after raylet replied the system config. - virtual void GetSystemConfig( - const rpc::ClientCallback &callback) = 0; - - virtual void GlobalGC(const rpc::ClientCallback &callback) = 0; - - virtual void NotifyGCSRestart( - const rpc::ClientCallback &callback) = 0; - - virtual void ShutdownRaylet( - const NodeID &node_id, - bool graceful, - const rpc::ClientCallback &callback) = 0; - - virtual void DrainRaylet( - const rpc::autoscaler::DrainNodeReason &reason, - const std::string &reason_message, - int64_t deadline_timestamp_ms, - const rpc::ClientCallback &callback) = 0; - - virtual void CancelLeasesWithResourceShapes( - const std::vector> &resource_shapes, - const rpc::ClientCallback &callback) = 0; - - virtual void IsLocalWorkerDead( - const WorkerID &worker_id, - const rpc::ClientCallback &callback) = 0; - - virtual std::shared_ptr GetChannel() const = 0; - - virtual void GetNodeStats( - const rpc::GetNodeStatsRequest &request, - const rpc::ClientCallback &callback) = 0; - - virtual int64_t GetPinsInFlight() const = 0; - - virtual ~RayletClientInterface() = default; -}; +// Forward declaration. +namespace rpc { +class NodeManagerClient; +} namespace raylet { diff --git a/src/ray/raylet_client/raylet_client_interface.h b/src/ray/raylet_client/raylet_client_interface.h new file mode 100644 index 000000000000..621acb65c415 --- /dev/null +++ b/src/ray/raylet_client/raylet_client_interface.h @@ -0,0 +1,215 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "ray/rpc/client_call.h" +#include "src/ray/protobuf/autoscaler.pb.h" +#include "src/ray/protobuf/common.pb.h" +#include "src/ray/protobuf/node_manager.pb.h" + +namespace grpc { +class Channel; +} + +namespace ray { + +// Forward declarations. +class Status; +class WorkerID; +class ObjectID; +class LeaseID; +class NodeID; +class BundleSpecification; + +class RayletClientInterface { + public: + /// Request to a raylet to pin a plasma object. The callback will be sent via gRPC. + virtual void PinObjectIDs( + const rpc::Address &caller_address, + const std::vector &object_ids, + const ObjectID &generator_id, + const ray::rpc::ClientCallback &callback) = 0; + + /// Requests a worker from the raylet. The callback will be sent via gRPC. + /// \param lease_spec Lease that is requested by the owner. + /// \param grant_or_reject: True if we we should either grant or reject the request + /// but no spillback. + /// \param callback: The callback to call when the request finishes. + /// \param backlog_size The queue length for the given shape on the CoreWorker. + /// \param lease_id Unique lease ID for this worker lease request. + virtual void RequestWorkerLease( + const rpc::LeaseSpec &lease_spec, + bool grant_or_reject, + const ray::rpc::ClientCallback &callback, + const int64_t backlog_size = -1, + const bool is_selected_based_on_locality = false) = 0; + + /// Returns a worker to the raylet. + /// \param worker_port The local port of the worker on the raylet node. + /// \param worker_id The unique worker id of the worker on the raylet node. + /// \param disconnect_worker Whether the raylet should disconnect the worker. + /// \param worker_exiting Whether the worker is exiting and cannot be reused. + /// \return ray::Status + virtual ray::Status ReturnWorkerLease(int worker_port, + const WorkerID &worker_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) = 0; + + /// Request the raylet to prestart workers. In `request` we can set the worker's owner, + /// runtime env info and number of workers. + /// + virtual void PrestartWorkers( + const rpc::PrestartWorkersRequest &request, + const rpc::ClientCallback &callback) = 0; + + /// Notify raylets to release unused workers. + /// \param workers_in_use Workers currently in use. + /// \param callback Callback that will be called after raylet completes the release of + /// unused workers. \return ray::Status + virtual void ReleaseUnusedActorWorkers( + const std::vector &workers_in_use, + const rpc::ClientCallback &callback) = 0; + + virtual void CancelWorkerLease( + const LeaseID &lease_id, + const rpc::ClientCallback &callback) = 0; + + /// Report the backlog size of a given worker and a given scheduling class to the + /// raylet. + /// \param worker_id The ID of the worker that reports the backlog size. + /// \param backlog_reports The backlog report for each scheduling class + virtual void ReportWorkerBacklog( + const WorkerID &worker_id, + const std::vector &backlog_reports) = 0; + + virtual void GetWorkerFailureCause( + const LeaseID &lease_id, + const ray::rpc::ClientCallback &callback) = 0; + + /// Request a raylet to prepare resources of given bundles for atomic placement group + /// creation. This is used for the first phase of atomic placement group creation. The + /// callback will be sent via gRPC. + /// \param bundle_specs Bundles to be scheduled at this raylet. + /// \return ray::Status + virtual void PrepareBundleResources( + const std::vector> &bundle_specs, + const ray::rpc::ClientCallback + &callback) = 0; + + /// Request a raylet to commit resources of given bundles for atomic placement group + /// creation. This is used for the second phase of atomic placement group creation. The + /// callback will be sent via gRPC. + /// \param bundle_specs Bundles to be scheduled at this raylet. + /// \return ray::Status + virtual void CommitBundleResources( + const std::vector> &bundle_specs, + const ray::rpc::ClientCallback &callback) = 0; + + virtual void CancelResourceReserve( + const BundleSpecification &bundle_spec, + const ray::rpc::ClientCallback &callback) = 0; + + virtual void ReleaseUnusedBundles( + const std::vector &bundles_in_use, + const rpc::ClientCallback &callback) = 0; + + virtual void GetResourceLoad( + const rpc::ClientCallback &callback) = 0; + /// Registers a mutable object on this node so that it can be read. Writes are performed + /// on a remote node. This local node creates a mapping from `object_id` -> + /// `reader_ref`. + /// + /// \param writer_object_id The object ID of the mutable object on the remote node that + /// is written to. + /// \param num_readers The number of readers that will read the object on this local + /// node. + /// \param reader_object_id The object ID of the mutable object that is read on this + /// local node. + /// \param callback This callback is executed to send a reply to the remote + /// node once the mutable object is registered. + virtual void RegisterMutableObjectReader( + const ObjectID &writer_object_id, + int64_t num_readers, + const ObjectID &reader_object_id, + const rpc::ClientCallback &callback) = 0; + + /// Handles a mutable object write that was performed on a remote node and is being + /// transferred to this node so that it can be read. + /// + /// \param writer_object_id The object ID of the mutable object on the remote node that + /// is written to. This is *not* the object ID of the corresponding mutable object on + /// this local node. + /// \param data_size The size of the data to write to the mutable object on this local + /// node. + /// \param metadata_size The size of the metadata to write to the mutable object on this + /// local node. + /// \param data The data to write to the mutable object on this local node. + /// \param metadata The metadata to write to the mutable object on this local node. + /// \param callback This callback is executed to send a reply to the remote node once + /// the mutable object is transferred. + virtual void PushMutableObject( + const ObjectID &writer_object_id, + uint64_t data_size, + uint64_t metadata_size, + void *data, + void *metadata, + const rpc::ClientCallback &callback) = 0; + + /// Get the system config from Raylet. + /// \param callback Callback that will be called after raylet replied the system config. + virtual void GetSystemConfig( + const rpc::ClientCallback &callback) = 0; + + virtual void GlobalGC(const rpc::ClientCallback &callback) = 0; + + virtual void NotifyGCSRestart( + const rpc::ClientCallback &callback) = 0; + + virtual void ShutdownRaylet( + const NodeID &node_id, + bool graceful, + const rpc::ClientCallback &callback) = 0; + + virtual void DrainRaylet( + const rpc::autoscaler::DrainNodeReason &reason, + const std::string &reason_message, + int64_t deadline_timestamp_ms, + const rpc::ClientCallback &callback) = 0; + + virtual void CancelLeasesWithResourceShapes( + const std::vector> &resource_shapes, + const rpc::ClientCallback &callback) = 0; + + virtual void IsLocalWorkerDead( + const WorkerID &worker_id, + const rpc::ClientCallback &callback) = 0; + + virtual std::shared_ptr GetChannel() const = 0; + + virtual void GetNodeStats( + const rpc::GetNodeStatsRequest &request, + const rpc::ClientCallback &callback) = 0; + + virtual int64_t GetPinsInFlight() const = 0; + + virtual ~RayletClientInterface() = default; +}; + +} // namespace ray diff --git a/src/ray/rpc/node_manager/raylet_client_pool.cc b/src/ray/raylet_client/raylet_client_pool.cc similarity index 98% rename from src/ray/rpc/node_manager/raylet_client_pool.cc rename to src/ray/raylet_client/raylet_client_pool.cc index 8c85a8457bb9..f955d37a5a46 100644 --- a/src/ray/rpc/node_manager/raylet_client_pool.cc +++ b/src/ray/raylet_client/raylet_client_pool.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/rpc/node_manager/raylet_client_pool.h" +#include "ray/raylet_client/raylet_client_pool.h" #include #include diff --git a/src/ray/rpc/node_manager/raylet_client_pool.h b/src/ray/raylet_client/raylet_client_pool.h similarity index 96% rename from src/ray/rpc/node_manager/raylet_client_pool.h rename to src/ray/raylet_client/raylet_client_pool.h index a1d686fd4e25..2440ee543e45 100644 --- a/src/ray/rpc/node_manager/raylet_client_pool.h +++ b/src/ray/raylet_client/raylet_client_pool.h @@ -24,8 +24,7 @@ #include "absl/synchronization/mutex.h" #include "ray/common/id.h" #include "ray/gcs/gcs_client/gcs_client.h" -#include "ray/raylet_client/raylet_client.h" -#include "ray/rpc/node_manager/node_manager_client.h" +#include "ray/raylet_client/raylet_client_interface.h" namespace ray { namespace rpc { diff --git a/src/ray/rpc/node_manager/tests/BUILD.bazel b/src/ray/raylet_client/tests/BUILD.bazel similarity index 78% rename from src/ray/rpc/node_manager/tests/BUILD.bazel rename to src/ray/raylet_client/tests/BUILD.bazel index d26b0fdb0a0d..775291eeb00d 100644 --- a/src/ray/rpc/node_manager/tests/BUILD.bazel +++ b/src/ray/raylet_client/tests/BUILD.bazel @@ -8,7 +8,8 @@ ray_cc_test( deps = [ "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/gcs/gcs_client:gcs_client_lib", - "//src/ray/rpc:node_manager_client", + "//src/ray/raylet_client:node_manager_client", + "//src/ray/raylet_client:raylet_client_pool", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/rpc/node_manager/tests/raylet_client_pool_test.cc b/src/ray/raylet_client/tests/raylet_client_pool_test.cc similarity index 99% rename from src/ray/rpc/node_manager/tests/raylet_client_pool_test.cc rename to src/ray/raylet_client/tests/raylet_client_pool_test.cc index 6b031229354b..7c8a85855f85 100644 --- a/src/ray/rpc/node_manager/tests/raylet_client_pool_test.cc +++ b/src/ray/raylet_client/tests/raylet_client_pool_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/rpc/node_manager/raylet_client_pool.h" +#include "ray/raylet_client/raylet_client_pool.h" #include diff --git a/src/ray/rpc/BUILD.bazel b/src/ray/rpc/BUILD.bazel index 4260d0065844..295f39aa0d05 100644 --- a/src/ray/rpc/BUILD.bazel +++ b/src/ray/rpc/BUILD.bazel @@ -123,33 +123,6 @@ ray_cc_library( ], ) -ray_cc_library( - name = "node_manager_client", - srcs = ["node_manager/raylet_client_pool.cc"], - hdrs = [ - "node_manager/node_manager_client.h", - "node_manager/raylet_client_pool.h", - ] + [ - # TODO(eoakes): these are needed due to a circular dependency: - # raylet_client_pool.cc -> raylet_client.h -> node_manager_client.h - "//src/ray/raylet_client:raylet_client.h", - ], - visibility = ["//visibility:public"], - deps = [ - ":client_call", - ":grpc_client", - "//src/ray/common:id", - "//src/ray/gcs/gcs_client:gcs_client_lib", - "//src/ray/protobuf:node_manager_cc_grpc", - "//src/ray/util:network_util", - ] + [ - # TODO(eoakes): these three come from raylet_client.h, remove after breaking the circular dependency. - "//src/ray/ipc:client_connection", - "//src/ray/common:ray_object", - "//src/ray/common:task_common", - ], -) - ray_cc_library( name = "node_manager_server", hdrs = [ @@ -228,7 +201,8 @@ ray_cc_library( "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/protobuf:core_worker_cc_grpc", "//src/ray/pubsub:subscriber", - "//src/ray/raylet_client:raylet_client_lib", + "//src/ray/raylet_client:raylet_client_interface", + "//src/ray/raylet_client:raylet_client_pool", "//src/ray/util:logging", "//src/ray/util:network_util", "@com_github_grpc_grpc//:grpc++", diff --git a/src/ray/rpc/node_manager/node_manager_server.h b/src/ray/rpc/node_manager/node_manager_server.h index d8b2a781a151..315507085459 100644 --- a/src/ray/rpc/node_manager/node_manager_server.h +++ b/src/ray/rpc/node_manager/node_manager_server.h @@ -65,7 +65,7 @@ class NodeManagerServiceHandler { /// Handlers. For all of the following handlers, the implementations can /// handle the request asynchronously. When handling is done, the /// `send_reply_callback` should be called. See - /// src/ray/rpc/node_manager/node_manager_client.h and + /// src/ray/raylet_client/node_manager_client.h and /// src/ray/protobuf/node_manager.proto for a description of the /// functionality of each handler. /// diff --git a/src/ray/rpc/worker/core_worker_client_pool.h b/src/ray/rpc/worker/core_worker_client_pool.h index 370ac45cf6fd..cbbf4cc3d07c 100644 --- a/src/ray/rpc/worker/core_worker_client_pool.h +++ b/src/ray/rpc/worker/core_worker_client_pool.h @@ -24,8 +24,8 @@ #include "absl/synchronization/mutex.h" #include "ray/common/id.h" #include "ray/gcs/gcs_client/gcs_client.h" -#include "ray/raylet_client/raylet_client.h" -#include "ray/rpc/node_manager/raylet_client_pool.h" +#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/raylet_client/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client.h" namespace ray { From abeabdf0efbdbdee5d4481216c77088f06b76284 Mon Sep 17 00:00:00 2001 From: Anmol Singh Date: Wed, 3 Sep 2025 01:37:54 +0530 Subject: [PATCH 396/634] Add metadata to indicate full dashboard embedding is supported (#56077) Since we introduced panel groups to Default (https://github.com/ray-project/ray/pull/55620) & Data (https://github.com/ray-project/ray/pull/55495) dashboards, applications consuming Grafana dashboards can comfortably embed the full dashboard on any UI now (and the other dashboards are pretty usable even without them). Added a `"supportsFullGrafanaView"` tag to the `rayMeta` list in Default Dashboard to indicate to consumers that we support full Grafana dashboard embedding from now on. --------- Signed-off-by: anmol Co-authored-by: anmol --- .../metrics/dashboards/default_grafana_dashboard_base.json | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ray/dashboard/modules/metrics/dashboards/default_grafana_dashboard_base.json b/python/ray/dashboard/modules/metrics/dashboards/default_grafana_dashboard_base.json index 0ad52d3fb128..3771296cd2a1 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/default_grafana_dashboard_base.json +++ b/python/ray/dashboard/modules/metrics/dashboards/default_grafana_dashboard_base.json @@ -163,6 +163,7 @@ } ] }, + "rayMeta": ["supportsFullGrafanaView"], "time": { "from": "now-30m", "to": "now" From d6507ad60db5aca747e9f84519e013479cd3ded9 Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Wed, 3 Sep 2025 01:52:15 +0530 Subject: [PATCH 397/634] [core] Make test-only transition methods private in ShutdownCoordinator (#56050) ## Why are these changes needed? This is a followup from https://github.com/ray-project/ray/pull/54244 - Restrict `TryInitiateShutdown`, `TryTransitionToDisconnecting`, and `TryTransitionToShutdown` to private once all production code calls `RequestShutdown`. - Minimize API surface and prevent misuse; with a single entry point, internal transitions need not be externally callable. - Update tests to exercise only `RequestShutdown` ## Related issue number Closes #55739 --------- Signed-off-by: Sagar Sumit --- .../core_worker_shutdown_executor.cc | 2 - src/ray/core_worker/shutdown_coordinator.cc | 5 -- src/ray/core_worker/shutdown_coordinator.h | 46 +++++-------------- .../tests/shutdown_coordinator_test.cc | 29 ++++-------- 4 files changed, 20 insertions(+), 62 deletions(-) diff --git a/src/ray/core_worker/core_worker_shutdown_executor.cc b/src/ray/core_worker/core_worker_shutdown_executor.cc index f8680d709219..6b1d66ddc836 100644 --- a/src/ray/core_worker/core_worker_shutdown_executor.cc +++ b/src/ray/core_worker/core_worker_shutdown_executor.cc @@ -123,8 +123,6 @@ void CoreWorkerShutdownExecutor::ExecuteExit( [this, exit_type, detail, creation_task_exception_pb_bytes]() { rpc::DrainServerCallExecutor(); KillChildProcessesImmediately(); - // Disconnect should be put close to Shutdown - // https://github.com/ray-project/ray/pull/34883 DisconnectServices(exit_type, detail, creation_task_exception_pb_bytes); ExecuteGracefulShutdown( exit_type, "Post-exit graceful shutdown", std::chrono::milliseconds{30000}); diff --git a/src/ray/core_worker/shutdown_coordinator.cc b/src/ray/core_worker/shutdown_coordinator.cc index 93e5489b4144..a08cb8ec0f88 100644 --- a/src/ray/core_worker/shutdown_coordinator.cc +++ b/src/ray/core_worker/shutdown_coordinator.cc @@ -82,11 +82,6 @@ bool ShutdownCoordinator::RequestShutdown( return true; } -bool ShutdownCoordinator::TryInitiateShutdown(ShutdownReason reason) { - // Legacy compatibility - delegate to graceful shutdown by default - return RequestShutdown(false, reason, "", kInfiniteTimeout, nullptr); -} - bool ShutdownCoordinator::TryTransitionToDisconnecting() { std::lock_guard lock(mu_); if (state_ != ShutdownState::kShuttingDown) { diff --git a/src/ray/core_worker/shutdown_coordinator.h b/src/ray/core_worker/shutdown_coordinator.h index f685261a5989..2692cb72a28d 100644 --- a/src/ray/core_worker/shutdown_coordinator.h +++ b/src/ray/core_worker/shutdown_coordinator.h @@ -165,40 +165,6 @@ class ShutdownCoordinator { const std::shared_ptr<::ray::LocalMemoryBuffer> &creation_task_exception_pb_bytes = nullptr); - /// Legacy method for compatibility - delegates to RequestShutdown - /// TODO (codope): This is public for now to ease incremental migration and testing. - /// Consider removing or making private once all call sites are wired to - /// RequestShutdown directly. - /// \param reason The reason for shutdown initiation - /// \return true if this call initiated shutdown, false if already shutting down - bool TryInitiateShutdown(ShutdownReason reason); - - /// Attempt to transition to disconnecting state. - /// - /// Begins the disconnection/cleanup phase (e.g., GCS/raylet disconnect). Only - /// valid from kShuttingDown. - /// - /// \return true if transition succeeded, false if invalid state - /// TODO (codope): Public-for-now to support targeted tests; make private when tests - /// drive behavior exclusively via RequestShutdown. - /// TODO (codope): Once private, we can consider removing the internal mutex acquisition - /// here and in TryTransitionToShutdown(), since RequestShutdown serializes the - /// execution path and only a single thread invokes transitions. - bool TryTransitionToDisconnecting(); - - /// Attempt to transition to final shutdown state. - /// - /// Finalizes shutdown. Allowed from kDisconnecting (normal) or kShuttingDown - /// (force path). - /// - /// \return true if transition succeeded, false if invalid state - /// TODO (codope): Public-for-now to support targeted tests; make private when tests - /// drive behavior exclusively via RequestShutdown. - /// TODO (codope): Once private, we can consider removing the internal mutex acquisition - /// here and in TryTransitionToDisconnecting(), since RequestShutdown serializes the - /// execution path and only a single thread invokes transitions. - bool TryTransitionToShutdown(); - /// Get the current shutdown state (mutex-protected, fast path safe). /// /// \return Current shutdown state @@ -248,6 +214,18 @@ class ShutdownCoordinator { std::string GetReasonString() const; private: + /// Attempt to transition to disconnecting state. + /// Begins the disconnection/cleanup phase (e.g., GCS/raylet disconnect). Only + /// valid from kShuttingDown. + /// \return true if transition succeeded, false if invalid state + bool TryTransitionToDisconnecting(); + + /// Attempt to transition to final shutdown state. + /// Finalizes shutdown. Allowed from kDisconnecting (normal) or kShuttingDown + /// (force path). + /// \return true if transition succeeded, false if invalid state + bool TryTransitionToShutdown(); + /// Execute shutdown sequence based on worker type and mode void ExecuteShutdownSequence( bool force_shutdown, diff --git a/src/ray/core_worker/tests/shutdown_coordinator_test.cc b/src/ray/core_worker/tests/shutdown_coordinator_test.cc index f01b80dba9d7..8752a8d8c966 100644 --- a/src/ray/core_worker/tests/shutdown_coordinator_test.cc +++ b/src/ray/core_worker/tests/shutdown_coordinator_test.cc @@ -148,18 +148,17 @@ TEST_F(ShutdownCoordinatorTest, RequestShutdown_IdempotentBehavior) { EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kForcedExit); // Reason is updated } -TEST_F(ShutdownCoordinatorTest, - TryInitiateShutdown_DelegatesToGraceful_OnlyFirstSucceeds) { +TEST_F(ShutdownCoordinatorTest, RequestShutdown_DelegatesToGraceful_OnlyFirstSucceeds) { auto coordinator = CreateCoordinator(); - EXPECT_TRUE(coordinator->TryInitiateShutdown(ShutdownReason::kUserError)); + EXPECT_TRUE(coordinator->RequestShutdown(false, ShutdownReason::kUserError)); const auto state = coordinator->GetState(); EXPECT_TRUE(state == ShutdownState::kShuttingDown || state == ShutdownState::kDisconnecting); EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kUserError); // Second call should fail - EXPECT_FALSE(coordinator->TryInitiateShutdown(ShutdownReason::kForcedExit)); + EXPECT_FALSE(coordinator->RequestShutdown(false, ShutdownReason::kForcedExit)); EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kUserError); // unchanged } @@ -177,24 +176,12 @@ TEST_F(ShutdownCoordinatorTest, EXPECT_EQ(coordinator->GetReason(), ShutdownReason::kGracefulExit); // Disconnecting -> Shutdown - EXPECT_TRUE(coordinator->TryTransitionToShutdown()); + EXPECT_TRUE(coordinator->RequestShutdown(true, ShutdownReason::kForcedExit)); EXPECT_EQ(coordinator->GetState(), ShutdownState::kShutdown); // Further transitions are no-ops. - EXPECT_FALSE(coordinator->TryTransitionToDisconnecting()); - EXPECT_FALSE(coordinator->TryTransitionToShutdown()); -} - -TEST_F(ShutdownCoordinatorTest, InvalidTransitions_FromRunning_Fail) { - auto coordinator = CreateCoordinator(); - - // Cannot transition to disconnecting from running - EXPECT_FALSE(coordinator->TryTransitionToDisconnecting()); - EXPECT_EQ(coordinator->GetState(), ShutdownState::kRunning); - - // Cannot transition to shutdown from running - EXPECT_FALSE(coordinator->TryTransitionToShutdown()); - EXPECT_EQ(coordinator->GetState(), ShutdownState::kRunning); + EXPECT_FALSE(coordinator->RequestShutdown(false, ShutdownReason::kGracefulExit)); + EXPECT_FALSE(coordinator->RequestShutdown(true, ShutdownReason::kForcedExit)); } TEST_F(ShutdownCoordinatorTest, ForceShutdown_TransitionsDirectlyToShutdown) { @@ -205,7 +192,7 @@ TEST_F(ShutdownCoordinatorTest, ForceShutdown_TransitionsDirectlyToShutdown) { ShutdownReason::kForcedExit)); // Already in shutdown state, manual transition should fail - EXPECT_FALSE(coordinator->TryTransitionToShutdown()); + EXPECT_FALSE(coordinator->RequestShutdown(true, ShutdownReason::kForcedExit)); EXPECT_EQ(coordinator->GetState(), ShutdownState::kShutdown); } @@ -291,7 +278,7 @@ TEST_F(ShutdownCoordinatorTest, StringRepresentations_StateAndReason_AreReadable EXPECT_EQ(coordinator->GetStateString(), "Disconnecting"); EXPECT_EQ(coordinator->GetReasonString(), "GracefulExit"); - coordinator->TryTransitionToShutdown(); + coordinator->RequestShutdown(true, ShutdownReason::kForcedExit); EXPECT_EQ(coordinator->GetStateString(), "Shutdown"); } From 01a543510a2a9bc2bdbf0511282d8a58cfbd156e Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 2 Sep 2025 13:23:45 -0700 Subject: [PATCH 398/634] [ci] declare test rule tags in test_rules file (#56127) and perform more aggresive checks, so that people do not forget to declaration when adding new tags. Signed-off-by: Lonnie Liu --- ci/pipeline/determine_tests_to_run.py | 57 ++++++++++++++++--------- ci/pipeline/test_conditional_testing.py | 22 +++++++++- ci/pipeline/test_rules.txt | 12 +++++- 3 files changed, 69 insertions(+), 22 deletions(-) diff --git a/ci/pipeline/determine_tests_to_run.py b/ci/pipeline/determine_tests_to_run.py index e32d548668f6..c29815199cf4 100644 --- a/ci/pipeline/determine_tests_to_run.py +++ b/ci/pipeline/determine_tests_to_run.py @@ -8,16 +8,6 @@ from pprint import pformat from typing import List, Optional, Set, Tuple -_ALL_TAGS = set( - """ - always - lint python cpp core_cpp java workflow compiled_graphs dashboard ray_client - data dask serve ml tune train llm rllib rllib_gpu rllib_directly - linux_wheels macos_wheels docker doc python_dependencies tools - release_tests compiled_python spark_on_ray runtime_env_container - """.split() -) - def _list_changed_files(commit_range): """Returns a list of names of files changed in the given commit range. @@ -63,11 +53,13 @@ class TagRule: def __init__( self, tags: List[str], + lineno: int, dirs: Optional[List[str]] = None, files: Optional[List[str]] = None, patterns: Optional[List[str]] = None, ): self.tags = set(tags) + self.lineno = lineno self.dirs = dirs or [] self.patterns = patterns or [] self.files = files or [] @@ -90,7 +82,7 @@ def match_tags(self, changed_file: str) -> Tuple[Set[str], bool]: return set(), False -def _parse_rules(rule_content: str) -> List[TagRule]: +def _parse_rules(rule_content: str) -> Tuple[Set[str], List[TagRule]]: """ Parse the rule config content into a list ot TagRule's. @@ -112,6 +104,9 @@ def _parse_rules(rule_content: str) -> List[TagRule]: """ rules: List[TagRule] = [] + tag_defs: Set[str] = set() + tag_defs_ended: bool = False + tags: Set[str] = set() dirs: List[str] = [] files: List[str] = [] @@ -129,13 +124,22 @@ def _parse_rules(rule_content: str) -> List[TagRule]: if comment_index != -1: line = line[:comment_index].strip() # Remove comments. + if line.startswith("!"): + if tag_defs_ended: + raise ValueError("Tag must be declared at file start.") + tag_defs.update(line[1:].split()) + continue + + if not tag_defs_ended: + tag_defs_ended = True + if line.startswith("@"): # tags. # Strip the leading '@' and split into tags. tags.update(line[1:].split()) elif line.startswith(";"): # End of a rule. if line != ";": raise ValueError(f"Unexpected tokens after semicolon on line {lineno}.") - rules.append(TagRule(tags, dirs, files, patterns)) + rules.append(TagRule(tags, lineno, dirs, files, patterns)) tags, dirs, files, patterns = set(), [], [], [] else: if line.find("*") != -1: # Patterns. @@ -147,20 +151,33 @@ def _parse_rules(rule_content: str) -> List[TagRule]: # Append the last rule if not empty. if tags or dirs or files or patterns: - rules.append(TagRule(tags, dirs, files, patterns)) + rules.append(TagRule(tags, lineno, dirs, files, patterns)) - return rules + return tag_defs, rules class TagRuleSet: def __init__(self, content: Optional[str] = None): + self.tag_defs = set() + self.rules = [] + if content is not None: - self.rules = _parse_rules(content) - else: - self.rules = [] + self.add_rules(content) def add_rules(self, content: str): - self.rules.extend(_parse_rules(content)) + tag_defs, rules = _parse_rules(content) + self.tag_defs.update(tag_defs) + self.rules.extend(rules) + + def check_rules(self): + for rule in self.rules: + if not rule.tags: + continue + for tag in rule.tags: + if tag not in self.tag_defs: + raise ValueError( + f"Tag {tag} not declared, used in rule at line {rule.lineno}." + ) def match_tags(self, changed_file: str) -> Tuple[Set[str], bool]: for rule in self.rules: @@ -187,6 +204,8 @@ def match_tags(self, changed_file: str) -> Tuple[Set[str], bool]: with open(config) as f: rules.add_rules(f.read()) + rules.check_rules() + tags: Set[str] = set() tags.add("always") @@ -220,7 +239,7 @@ def _emit(line: str): # Log the modified environment variables visible in console. output_string = " ".join(list(tags)) for tag in tags: - assert tag in _ALL_TAGS, f"Unknown tag {tag}" + assert tag in rules.tag_defs, f"Unknown tag {tag}" print(output_string, file=sys.stderr) # Debug purpose print(output_string) diff --git a/ci/pipeline/test_conditional_testing.py b/ci/pipeline/test_conditional_testing.py index 91e59c462a94..9dfbb68292d1 100644 --- a/ci/pipeline/test_conditional_testing.py +++ b/ci/pipeline/test_conditional_testing.py @@ -166,6 +166,7 @@ def __init__(self, file: str, tags: Set[str]): def test_tag_rule(): rule = TagRule( tags=["hit"], + lineno=1, dirs=["fancy"], files=["file.txt"], patterns=["python/*.py"], @@ -182,7 +183,7 @@ def test_tag_rule(): assert rule.match_tags("fancy") == ({"hit"}, True) assert rule.match_tags("not_match") == (set(), False) - skip_rule = TagRule(tags=[], files=["skip.txt"]) + skip_rule = TagRule(tags=[], lineno=1, files=["skip.txt"]) assert skip_rule.match("skip.txt") assert skip_rule.match_tags("skip.txt") == (set(), True) assert skip_rule.match_tags("not_match") == (set(), False) @@ -193,8 +194,19 @@ def test_tag_rule_set(): assert rule_set.match_tags("fancy/file.txt") == ({"fancy"}, True) rule_set = TagRuleSet( - "\n".join(["fancy/ #dir", "@fancy", ";", "\t\t ", "foobar.txt", "@foobar"]) + "\n".join( + [ + "!fancy foobar", + "fancy/ #dir", + "@fancy", + ";", + "\t\t ", + "foobar.txt", + "@foobar", + ] + ) ) + rule_set.check_rules() assert rule_set.match_tags("fancy/file.txt") == ({"fancy"}, True) assert rule_set.match_tags("foobar.txt") == ({"foobar"}, True) assert rule_set.match_tags("not_a_match") == (set(), False) @@ -203,5 +215,11 @@ def test_tag_rule_set(): assert rule_set.match_tags("anything") == (set(), False) +def test_tag_rule_set_check_rules(): + rule_set = TagRuleSet("\n".join(["!foobar", "fancy/ #dir", "@fancy"])) + with pytest.raises(ValueError): + rule_set.check_rules() + + if __name__ == "__main__": sys.exit(pytest.main(["-vv", __file__])) diff --git a/ci/pipeline/test_rules.txt b/ci/pipeline/test_rules.txt index f19d274cfee0..a3aebf566d7e 100644 --- a/ci/pipeline/test_rules.txt +++ b/ci/pipeline/test_rules.txt @@ -3,13 +3,23 @@ # Comment content, after '#', will be ignored. # Empty lines will be ignored too. # +# ! tag1 tag2 tag3 # Declares a tag. A tag must be declared first to be used. +# # Tags must be declared at the beginning. +# # dir/ # Directory to match # file # File to match # dir/*.py # Pattern to match, using fnmatch, matches dir/a.py dir/dir/b.py or dir/.py -# @ tag1 tag2 tag3 # Tags to emit for a rule. A rule without tags is a skipping rule. +# @ tag1 tag2 tag3 # Tags to emit for a rule. A rule without tags is a skipping rule. # # ; # Semicolon to separate rules +! always lint +! python cpp core_cpp java workflow compiled_graphs dashboard +! ray_client runtime_env_container +! data dask serve ml tune train llm rllib rllib_gpu rllib_directly +! linux_wheels macos_wheels docker doc python_dependencies tools +! release_tests spark_on_ray + python/ray/air/ @ ml train tune data linux_wheels ; From cc08726a51d2d357b1c79def568c64db17d1e573 Mon Sep 17 00:00:00 2001 From: Alexey Kudinkin Date: Tue, 2 Sep 2025 17:46:04 -0400 Subject: [PATCH 399/634] [Data] Removing Parquet metadata fetching in `ParquetDatasource` (#56105) Historically, `ParquetDatasource` have been fetching individual files parquet footer metadata to obtain granular metadata about the dataset. While laudable in principle, it's really inefficient in practice and manifests itself in extremely poor performance on very large datasets (10s to 100s Ks of files). This change revisits this approach by - Removing metadata fetching as a step (and deprecating involved components) - Cleaning up and streamlining some of the other utilities further optimizing performance ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Alexey Kudinkin --- .../datasource/parquet_datasource.py | 113 ++++++------- .../logical/operators/count_operator.py | 6 +- python/ray/data/dataset.py | 5 +- .../ray/data/datasource/file_meta_provider.py | 43 ++++- .../data/datasource/parquet_meta_provider.py | 155 +++++++----------- python/ray/data/datasource/partitioning.py | 7 +- python/ray/data/datasource/path_util.py | 4 +- python/ray/data/tests/test_consumption.py | 4 +- .../ray/data/tests/test_metadata_provider.py | 14 +- python/ray/data/tests/test_parquet.py | 98 +++-------- 10 files changed, 194 insertions(+), 255 deletions(-) diff --git a/python/ray/data/_internal/datasource/parquet_datasource.py b/python/ray/data/_internal/datasource/parquet_datasource.py index 44ccbd1e55b2..9589036aefd4 100644 --- a/python/ray/data/_internal/datasource/parquet_datasource.py +++ b/python/ray/data/_internal/datasource/parquet_datasource.py @@ -24,7 +24,6 @@ RetryingPyFileSystem, _check_pyarrow_version, _is_local_scheme, - call_with_retry, iterate_with_retry, ) from ray.data.block import Block, BlockAccessor @@ -33,10 +32,13 @@ from ray.data.datasource.datasource import ReadTask from ray.data.datasource.file_based_datasource import FileShuffleConfig from ray.data.datasource.file_meta_provider import ( - DefaultFileMetadataProvider, _handle_read_os_error, + _list_files, +) +from ray.data.datasource.parquet_meta_provider import ( + ParquetFileMetadata, + ParquetMetadataProvider, ) -from ray.data.datasource.parquet_meta_provider import ParquetMetadataProvider from ray.data.datasource.partitioning import ( PartitionDataType, Partitioning, @@ -101,12 +103,19 @@ class _SampleInfo: estimated_bytes_per_row: Optional[int] -# TODO(ekl) this is a workaround for a pyarrow serialization bug, where serializing a -# raw pyarrow file fragment causes S3 network calls. -class SerializedFragment: - def __init__(self, frag: "ParquetFileFragment"): - self._data = cloudpickle.dumps( - (frag.format, frag.path, frag.filesystem, frag.partition_expression) +class _NoIOSerializableFragmentWrapper: + """This is a workaround to avoid utilizing `ParquetFileFragment` original + serialization protocol that actually does network RPCs during serialization + (to fetch metadata)""" + + def __init__(self, f: "ParquetFileFragment"): + self._fragment = f + + def __reduce__(self): + return self._fragment.format.make_fragment, ( + self._fragment.path, + self._fragment.filesystem, + self._fragment.partition_expression, ) def deserialize(self) -> "ParquetFileFragment": @@ -122,7 +131,7 @@ def deserialize(self) -> "ParquetFileFragment": # Visible for test mocking. def _deserialize_fragments( - serialized_fragments: List[SerializedFragment], + serialized_fragments: List[_NoIOSerializableFragmentWrapper], ) -> List["pyarrow._dataset.ParquetFileFragment"]: return [p.deserialize() for p in serialized_fragments] @@ -204,28 +213,24 @@ def __init__( retryable_errors=DataContext.get_current().retried_io_errors, ) - # HACK: PyArrow's `ParquetDataset` errors if input paths contain non-parquet - # files. To avoid this, we expand the input paths with the default metadata - # provider and then apply the partition filter or file extensions. - if partition_filter is not None or file_extensions is not None: - default_meta_provider = DefaultFileMetadataProvider() - expanded_paths, _ = map( - list, zip(*default_meta_provider.expand_paths(paths, filesystem)) - ) - - paths = list(expanded_paths) - if partition_filter is not None: - paths = partition_filter(paths) - if file_extensions is not None: - paths = [ - path for path in paths if _has_file_extension(path, file_extensions) - ] + listed_files = _list_files( + paths, + filesystem, + partition_filter=partition_filter, + file_extensions=file_extensions, + ) - filtered_paths = set(expanded_paths) - set(paths) - if filtered_paths: - logger.info(f"Filtered out {len(filtered_paths)} paths") + if listed_files: + paths, file_sizes = zip(*listed_files) + else: + paths, file_sizes = [], [] - if dataset_kwargs is None: + if dataset_kwargs is not None: + logger.warning( + "Please note that `ParquetDatasource.__init__`s `dataset_kwargs` " + "is a deprecated parameter and will be removed in the future." + ) + else: dataset_kwargs = {} if "partitioning" in dataset_kwargs: @@ -238,7 +243,9 @@ def __init__( # duplicating the partition data, we disable PyArrow's partitioning. dataset_kwargs["partitioning"] = None - pq_ds = get_parquet_dataset(paths, filesystem, dataset_kwargs) + # NOTE: ParquetDataset only accepts list of paths, hence we need to convert + # it to a list + pq_ds = get_parquet_dataset(list(paths), filesystem, dataset_kwargs) # `read_schema` is the schema object that will be used to perform # read operations. @@ -277,12 +284,13 @@ def __init__( "scheduling_strategy" ] = DataContext.get_current().scheduling_strategy - self._metadata = ( - meta_provider.prefetch_file_metadata( - pq_ds.fragments, **prefetch_remote_args + self._metadata = [ + ParquetFileMetadata( + num_bytes=num_bytes, ) - or [] - ) + for num_bytes in file_sizes + ] + except OSError as e: _handle_read_os_error(e, paths) @@ -292,7 +300,9 @@ def __init__( # NOTE: Store the custom serialized `ParquetFileFragment` to avoid unexpected # network calls when `_ParquetDatasourceReader` is serialized. See # `_SerializedFragment()` implementation for more details. - self._pq_fragments = [SerializedFragment(p) for p in pq_ds.fragments] + self._pq_fragments = [ + _NoIOSerializableFragmentWrapper(p) for p in pq_ds.fragments + ] self._pq_paths = [p.path for p in pq_ds.fragments] self._meta_provider = meta_provider self._block_udf = _block_udf @@ -332,7 +342,7 @@ def __init__( def estimate_inmemory_data_size(self) -> Optional[int]: total_size = 0 for file_metadata in self._metadata: - total_size += file_metadata.total_byte_size + total_size += file_metadata.num_bytes return total_size * self._encoding_ratio def get_read_tasks(self, parallelism: int) -> List[ReadTask]: @@ -443,18 +453,13 @@ def read_fragments( data_columns, partition_columns, schema, - serialized_fragments: List[SerializedFragment], + fragments: List["ParquetFileFragment"], include_paths: bool, partitioning: Partitioning, ) -> Iterator["pyarrow.Table"]: # This import is necessary to load the tensor extension type. from ray.data.extensions.tensor_extension import ArrowTensorType # noqa - # Deserialize after loading the filesystem class. - fragments: List[ - "pyarrow._dataset.ParquetFileFragment" - ] = _deserialize_fragments_with_retry(serialized_fragments) - # Ensure that we're reading at least one dataset fragment. assert len(fragments) > 0 @@ -510,30 +515,12 @@ def get_batch_iterable(): yield table -def _deserialize_fragments_with_retry(fragments): - # The deserialization retry helps when the upstream datasource is not able to - # handle overloaded read request or failed with some retriable failures. - # For example when reading data from HA hdfs service, hdfs might - # lose connection for some unknown reason expecially when - # simutaneously running many hyper parameter tuning jobs - # with ray.data parallelism setting at high value like the default 200 - # Such connection failure can be restored with some waiting and retry. - return call_with_retry( - lambda: _deserialize_fragments(fragments), - description="deserialize fragments", - max_attempts=FILE_READING_RETRY, - ) - - def _sample_fragment( to_batches_kwargs, columns, schema, - file_fragment: SerializedFragment, + fragment: "ParquetFileFragment", ) -> _SampleInfo: - # Sample the first rows batch from file fragment `serialized_fragment`. - fragment = _deserialize_fragments_with_retry([file_fragment])[0] - # If the fragment has no row groups, it's an empty or metadata-only file. # Skip it by returning empty sample info. if fragment.metadata.num_row_groups == 0: diff --git a/python/ray/data/_internal/logical/operators/count_operator.py b/python/ray/data/_internal/logical/operators/count_operator.py index 409c99e3c000..39ec706f7e50 100644 --- a/python/ray/data/_internal/logical/operators/count_operator.py +++ b/python/ray/data/_internal/logical/operators/count_operator.py @@ -1,5 +1,3 @@ -from typing import List - from ray.data._internal.logical.interfaces import LogicalOperator @@ -15,6 +13,6 @@ class Count(LogicalOperator): def __init__( self, - input_dependencies: List["LogicalOperator"], + input_op: LogicalOperator, ): - super().__init__("Count", input_dependencies) + super().__init__("Count", [input_op]) diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 0fd023adb610..f2cef1c44761 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -3391,7 +3391,10 @@ def count(self) -> int: return meta_count plan = self._plan.copy() - count_op = Count([self._logical_plan.dag]) + + # NOTE: Project the dataset to avoid the need to carrying actual + # data when we're only interested in the total count + count_op = Count(Project(self._logical_plan.dag, cols=[])) logical_plan = LogicalPlan(count_op, self.context) count_ds = Dataset(plan, logical_plan) diff --git a/python/ray/data/datasource/file_meta_provider.py b/python/ray/data/datasource/file_meta_provider.py index ed5ae26f903c..354f761d9651 100644 --- a/python/ray/data/datasource/file_meta_provider.py +++ b/python/ray/data/datasource/file_meta_provider.py @@ -20,7 +20,8 @@ from ray.data._internal.remote_fn import cached_remote_fn from ray.data._internal.util import RetryingPyFileSystem from ray.data.block import BlockMetadata -from ray.data.datasource.partitioning import Partitioning +from ray.data.datasource.partitioning import Partitioning, PathPartitionFilter +from ray.data.datasource.path_util import _has_file_extension from ray.util.annotations import DeveloperAPI if TYPE_CHECKING: @@ -243,6 +244,46 @@ def _handle_read_os_error(error: OSError, paths: Union[str, List[str]]) -> str: raise error +def _list_files( + paths: List[str], + filesystem: "RetryingPyFileSystem", + *, + partition_filter: Optional[PathPartitionFilter], + file_extensions: Optional[List[str]], +) -> List[Tuple[str, int]]: + return list( + _list_files_internal( + paths, + filesystem, + partition_filter=partition_filter, + file_extensions=file_extensions, + ) + ) + + +def _list_files_internal( + paths: List[str], + filesystem: "RetryingPyFileSystem", + *, + partition_filter: Optional[PathPartitionFilter], + file_extensions: Optional[List[str]], +) -> Iterator[Tuple[str, int]]: + default_meta_provider = DefaultFileMetadataProvider() + + for path, file_size in default_meta_provider.expand_paths(paths, filesystem): + # HACK: PyArrow's `ParquetDataset` errors if input paths contain non-parquet + # files. To avoid this, we expand the input paths with the default metadata + # provider and then apply the partition filter or file extensions. + if ( + partition_filter + and not partition_filter.apply(path) + or not _has_file_extension(path, file_extensions) + ): + continue + + yield path, file_size + + def _expand_paths( paths: List[str], filesystem: "RetryingPyFileSystem", diff --git a/python/ray/data/datasource/parquet_meta_provider.py b/python/ray/data/datasource/parquet_meta_provider.py index 73a3c41ef6e2..25b33f27411e 100644 --- a/python/ray/data/datasource/parquet_meta_provider.py +++ b/python/ray/data/datasource/parquet_meta_provider.py @@ -1,6 +1,7 @@ -from typing import TYPE_CHECKING, List, Optional +import logging +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, List, Optional, Tuple -import ray.cloudpickle as cloudpickle from ray.data._internal.util import call_with_retry from ray.data.block import BlockMetadata from ray.data.datasource.file_meta_provider import ( @@ -11,8 +12,7 @@ if TYPE_CHECKING: import pyarrow - - from ray.data._internal.datasource.parquet_datasource import SerializedFragment + from pyarrow.dataset import ParquetFileFragment FRAGMENTS_PER_META_FETCH = 6 @@ -28,35 +28,21 @@ RETRY_MAX_BACKOFF_S_FOR_META_FETCH_TASK = 64 -class _ParquetFileFragmentMetaData: - """Class to store metadata of a Parquet file fragment. This includes - all attributes from `pyarrow.parquet.FileMetaData` except for `schema`, - which is stored in `self.schema_pickled` as a pickled object from - `cloudpickle.loads()`, used in deduplicating schemas across multiple fragments.""" - - def __init__(self, fragment_metadata: "pyarrow.parquet.FileMetaData"): - self.created_by = fragment_metadata.created_by - self.format_version = fragment_metadata.format_version - self.num_columns = fragment_metadata.num_columns - self.num_row_groups = fragment_metadata.num_row_groups - self.num_rows = fragment_metadata.num_rows - self.serialized_size = fragment_metadata.serialized_size +logger = logging.getLogger(__name__) - # Serialize the schema directly in the constructor - schema_ser = cloudpickle.dumps(fragment_metadata.schema.to_arrow_schema()) - self.schema_pickled = schema_ser - # Calculate the total byte size of the file fragment using the original - # object, as it is not possible to access row groups from this class. - self.total_byte_size = 0 - for row_group_idx in range(fragment_metadata.num_row_groups): - row_group_metadata = fragment_metadata.row_group(row_group_idx) - self.total_byte_size += row_group_metadata.total_byte_size +@DeveloperAPI(stability="alpha") +@dataclass +class ParquetFileMetadata: + num_bytes: int + num_rows: Optional[int] = field(default=None) - def set_schema_pickled(self, schema_pickled: bytes): - """Note: to get the underlying schema, use - `cloudpickle.loads(self.schema_pickled)`.""" - self.schema_pickled = schema_pickled + @classmethod + def from_(cls, pqm: "pyarrow.parquet.FileMetaData"): + return ParquetFileMetadata( + num_rows=pqm.num_rows, + num_bytes=_get_total_bytes(pqm), + ) @DeveloperAPI @@ -68,7 +54,7 @@ def _get_block_metadata( paths: List[str], *, num_fragments: int, - prefetched_metadata: Optional[List["_ParquetFileFragmentMetaData"]], + prefetched_metadata: Optional[List["ParquetFileMetadata"]], ) -> BlockMetadata: """Resolves and returns block metadata for files of a single dataset block. @@ -88,11 +74,13 @@ def _get_block_metadata( and len(prefetched_metadata) == num_fragments and all(m is not None for m in prefetched_metadata) ): + total_bytes, total_rows = self._derive_totals(prefetched_metadata) + # Fragment metadata was available, construct a normal # BlockMetadata. block_metadata = BlockMetadata( - num_rows=sum(m.num_rows for m in prefetched_metadata), - size_bytes=sum(m.total_byte_size for m in prefetched_metadata), + num_rows=total_rows, + size_bytes=total_bytes, input_files=paths, exec_stats=None, ) # Exec stats filled in later. @@ -107,11 +95,29 @@ def _get_block_metadata( ) return block_metadata + @staticmethod + def _derive_totals( + prefetched_metadata: List["ParquetFileMetadata"], + ) -> Tuple[int, int]: + total_bytes = 0 + total_rows = 0 + + for m in prefetched_metadata: + total_bytes += m.num_bytes + + if total_rows is not None: + if m.num_rows is not None: + total_rows += m.num_rows + else: + total_rows = None + + return total_bytes, total_rows + def prefetch_file_metadata( self, fragments: List["pyarrow.dataset.ParquetFileFragment"], **ray_remote_args, - ) -> Optional[List[_ParquetFileFragmentMetaData]]: + ) -> Optional[List[ParquetFileMetadata]]: """Pre-fetches file metadata for all Parquet file fragments in a single batch. Subsets of the metadata returned will be provided as input to subsequent calls @@ -126,15 +132,18 @@ def prefetch_file_metadata( must be returned in the same order as all input file fragments, such that `metadata[i]` always contains the metadata for `fragments[i]`. """ - from ray.data._internal.datasource.parquet_datasource import SerializedFragment + from ray.data._internal.datasource.parquet_datasource import ( + _NoIOSerializableFragmentWrapper, + ) if len(fragments) > PARALLELIZE_META_FETCH_THRESHOLD: # Wrap Parquet fragments in serialization workaround. - fragments = [SerializedFragment(fragment) for fragment in fragments] + fragments = [ + _NoIOSerializableFragmentWrapper(fragment) for fragment in fragments + ] # Fetch Parquet metadata in parallel using Ray tasks. - - def fetch_func(fragments): - return _fetch_metadata_serialization_wrapper( + def _remote_fetch(fragments: List["ParquetFileFragment"]): + return _fetch_metadata_with_retry( fragments, # Ensure that retry settings are propagated to remote tasks. retry_match=RETRY_EXCEPTIONS_FOR_META_FETCH_TASK, @@ -145,13 +154,13 @@ def fetch_func(fragments): raw_metadata = list( _fetch_metadata_parallel( fragments, - fetch_func, + _remote_fetch, FRAGMENTS_PER_META_FETCH, **ray_remote_args, ) ) - return _dedupe_schemas(raw_metadata) + return raw_metadata else: # We don't deduplicate schemas in this branch because they're already @@ -162,20 +171,15 @@ def fetch_func(fragments): return raw_metadata -def _fetch_metadata_serialization_wrapper( - fragments: List["SerializedFragment"], +def _fetch_metadata_with_retry( + fragments: List["ParquetFileFragment"], retry_match: Optional[List[str]], retry_max_attempts: int, retry_max_interval: int, -) -> List["_ParquetFileFragmentMetaData"]: - from ray.data._internal.datasource.parquet_datasource import ( - _deserialize_fragments_with_retry, - ) - - deserialized_fragments = _deserialize_fragments_with_retry(fragments) +) -> List["ParquetFileMetadata"]: try: metadata = call_with_retry( - lambda: _fetch_metadata(deserialized_fragments), + lambda: _fetch_metadata(fragments), description="fetch metdata", match=retry_match, max_attempts=retry_max_attempts, @@ -215,53 +219,18 @@ def _fetch_metadata_serialization_wrapper( def _fetch_metadata( fragments: List["pyarrow.dataset.ParquetFileFragment"], -) -> List[_ParquetFileFragmentMetaData]: +) -> List["ParquetFileMetadata"]: fragment_metadatas = [] for f in fragments: try: # Convert directly to _ParquetFileFragmentMetaData - fragment_metadatas.append(_ParquetFileFragmentMetaData(f.metadata)) - except AttributeError: + fragment_metadatas.append(ParquetFileMetadata.from_(f.metadata)) + except AttributeError as ae: + logger.warning(f"Failed to extract metadata from parquet file: {ae}") break # Deduplicate schemas to reduce memory usage - return _dedupe_schemas(fragment_metadatas) - - -def _dedupe_schemas( - metadatas: List[_ParquetFileFragmentMetaData], -) -> List[_ParquetFileFragmentMetaData]: - """Deduplicates schema objects across existing _ParquetFileFragmentMetaData objects. - - For datasets with a large number of columns, the pickled schema can be very large. - This function reduces memory usage by ensuring that identical schemas across multiple - fragment metadata objects reference the same underlying pickled schema object, - rather than each fragment maintaining its own copy. - - Args: - metadatas: List of _ParquetFileFragmentMetaData objects that already have - pickled schemas set. - - Returns: - The same list of _ParquetFileFragmentMetaData objects, but with duplicate - schemas deduplicated to reference the same object in memory. - """ - schema_to_id = {} # schema_ser -> schema_id - id_to_schema = {} # schema_id -> schema_ser - - for metadata in metadatas: - # Get the current schema serialization - schema_ser = metadata.schema_pickled - - if schema_ser not in schema_to_id: - # This is a new unique schema - schema_id = len(schema_to_id) - schema_to_id[schema_ser] = schema_id - id_to_schema[schema_id] = schema_ser - # No need to set schema_pickled - it already has the correct value - else: - # This schema already exists, reuse the existing one - schema_id = schema_to_id[schema_ser] - existing_schema_ser = id_to_schema[schema_id] - metadata.set_schema_pickled(existing_schema_ser) + return fragment_metadatas + - return metadatas +def _get_total_bytes(pqm: "pyarrow.parquet.FileMetaData") -> int: + return sum(pqm.row_group(i).total_byte_size for i in range(pqm.num_row_groups)) diff --git a/python/ray/data/datasource/partitioning.py b/python/ray/data/datasource/partitioning.py index 2d83fe6b67de..20a626b09bce 100644 --- a/python/ray/data/datasource/partitioning.py +++ b/python/ray/data/datasource/partitioning.py @@ -434,11 +434,12 @@ def __call__(self, paths: List[str]) -> List[str]: """ filtered_paths = paths if self._filter_fn is not None: - filtered_paths = [ - path for path in paths if self._filter_fn(self._parser(path)) - ] + filtered_paths = [path for path in paths if self.apply(path)] return filtered_paths + def apply(self, path: str) -> bool: + return self._filter_fn(self._parser(path)) + @property def parser(self) -> PathPartitionParser: """Returns the path partition parser for this filter.""" diff --git a/python/ray/data/datasource/path_util.py b/python/ray/data/datasource/path_util.py index 6498300caa9f..5d9527243f36 100644 --- a/python/ray/data/datasource/path_util.py +++ b/python/ray/data/datasource/path_util.py @@ -39,7 +39,7 @@ def _has_file_extension(path: str, extensions: Optional[List[str]]) -> bool: def _resolve_paths_and_filesystem( paths: Union[str, List[str]], - filesystem: "pyarrow.fs.FileSystem" = None, + filesystem: Optional["pyarrow.fs.FileSystem"] = None, ) -> Tuple[List[str], "pyarrow.fs.FileSystem"]: """ Resolves and normalizes all provided paths, infers a filesystem from the @@ -69,7 +69,7 @@ def _resolve_paths_and_filesystem( elif not isinstance(paths, list) or any(not isinstance(p, str) for p in paths): raise ValueError( "Expected `paths` to be a `str`, `pathlib.Path`, or `list[str]`, but got " - f"`{paths}`." + f"`{paths}`" ) elif len(paths) == 0: raise ValueError("Must provide at least one path.") diff --git a/python/ray/data/tests/test_consumption.py b/python/ray/data/tests/test_consumption.py index c5b4fd3f5b65..c8437d5aaa22 100644 --- a/python/ray/data/tests/test_consumption.py +++ b/python/ray/data/tests/test_consumption.py @@ -1863,7 +1863,7 @@ def test_dataset_plan_as_string(ray_start_cluster): ds = ray.data.read_parquet("example://iris.parquet", override_num_blocks=8) assert ds._plan.get_plan_as_string(type(ds)) == ( "Dataset(\n" - " num_rows=150,\n" + " num_rows=?,\n" " schema={\n" " sepal.length: double,\n" " sepal.width: double,\n" @@ -1882,7 +1882,7 @@ def test_dataset_plan_as_string(ray_start_cluster): " +- MapBatches()\n" " +- MapBatches()\n" " +- Dataset(\n" - " num_rows=150,\n" + " num_rows=?,\n" " schema={\n" " sepal.length: double,\n" " sepal.width: double,\n" diff --git a/python/ray/data/tests/test_metadata_provider.py b/python/ray/data/tests/test_metadata_provider.py index ef4899abd085..c67e0890d506 100644 --- a/python/ray/data/tests/test_metadata_provider.py +++ b/python/ray/data/tests/test_metadata_provider.py @@ -27,6 +27,7 @@ _get_file_infos_parallel, _get_file_infos_serial, ) +from ray.data.datasource.parquet_meta_provider import _get_total_bytes from ray.data.datasource.path_util import ( _resolve_paths_and_filesystem, _unwrap_protocol, @@ -40,13 +41,6 @@ def df_to_csv(dataframe, path, **kwargs): dataframe.to_csv(path, **kwargs) -def _get_parquet_file_meta_size_bytes(file_metas): - return sum( - sum(m.row_group(i).total_byte_size for i in range(m.num_row_groups)) - for m in file_metas - ) - - def _get_file_sizes_bytes(paths, fs): from pyarrow.fs import FileType @@ -111,9 +105,11 @@ def test_default_parquet_metadata_provider(fs, data_path): num_fragments=len(pq_ds.fragments), prefetched_metadata=fragment_file_metas, ) - expected_meta_size_bytes = _get_parquet_file_meta_size_bytes( - [f.metadata for f in pq_ds.fragments] + + expected_meta_size_bytes = sum( + [_get_total_bytes(f.metadata) for f in pq_ds.fragments] ) + assert meta.size_bytes == expected_meta_size_bytes assert meta.num_rows == 6 assert len(paths) == 2 diff --git a/python/ray/data/tests/test_parquet.py b/python/ray/data/tests/test_parquet.py index bacf2c649941..67a517cfc002 100644 --- a/python/ray/data/tests/test_parquet.py +++ b/python/ray/data/tests/test_parquet.py @@ -2,7 +2,7 @@ import shutil import time from dataclasses import dataclass -from typing import Any, Optional +from typing import Optional import numpy as np import pandas as pd @@ -23,8 +23,6 @@ from ray.data._internal.datasource.parquet_datasource import ( NUM_CPUS_FOR_META_FETCH_TASK, ParquetDatasource, - SerializedFragment, - _deserialize_fragments_with_retry, ) from ray.data._internal.execution.interfaces.ref_bundle import ( _ref_bundles_iterator_to_block_refs_list, @@ -114,70 +112,6 @@ def test_include_paths( assert paths == [path, path] -@pytest.mark.parametrize( - "fs,data_path", - [ - (lazy_fixture("local_fs"), lazy_fixture("local_path")), - ], -) -def test_parquet_deserialize_fragments_with_retry( - ray_start_regular_shared, fs, data_path, monkeypatch -): - setup_data_path = _unwrap_protocol(data_path) - df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - table = pa.Table.from_pandas(df1) - path1 = os.path.join(setup_data_path, "test1.parquet") - pq.write_table(table, path1, filesystem=fs) - df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) - table = pa.Table.from_pandas(df2) - path2 = os.path.join(setup_data_path, "test2.parquet") - pq.write_table(table, path2, filesystem=fs) - - dataset_kwargs = {} - pq_ds = pq.ParquetDataset( - data_path, - **dataset_kwargs, - filesystem=fs, - ) - serialized_fragments = [SerializedFragment(p) for p in pq_ds.fragments] - - # test 1st attempt succeed - fragments = _deserialize_fragments_with_retry(serialized_fragments) - assert "test1.parquet" in fragments[0].path - assert "test2.parquet" in fragments[1].path - - # test the 3rd attempt succeed with a mock function constructed - # to throw in the first two attempts - class MockDeserializer: - def __init__(self, planned_exp_or_return): - self.planned_exp_or_return = planned_exp_or_return - self.cur_index = 0 - - def __call__(self, *args: Any, **kwds: Any) -> Any: - exp_or_ret = self.planned_exp_or_return[self.cur_index] - self.cur_index += 1 - if isinstance(exp_or_ret, Exception): - raise exp_or_ret - else: - return exp_or_ret - - mock_deserializer = MockDeserializer( - [ - Exception("1st mock failed attempt"), - Exception("2nd mock failed attempt"), - fragments, - ] - ) - monkeypatch.setattr( - ray.data._internal.datasource.parquet_datasource, - "_deserialize_fragments", - mock_deserializer, - ) - retried_fragments = _deserialize_fragments_with_retry(serialized_fragments) - assert "test1.parquet" in retried_fragments[0].path - assert "test2.parquet" in retried_fragments[1].path - - @pytest.mark.parametrize( "fs,data_path", [ @@ -267,7 +201,10 @@ def test_parquet_read_meta_provider(ray_start_regular_shared, fs, data_path): pq.write_table(table, path2, filesystem=fs) expected_num_rows = len(df1) + len(df2) - expected_byte_size = 787500 + # NOTE: Since we're testing against various Pyarrow versions size + # on disk could be varying slightly as it on top of data it also + # includes metadata + expected_byte_size = pytest.approx(463500, abs=500) # # Case 1: Test metadata fetching happy path (obtaining, caching and propagating @@ -290,14 +227,19 @@ def prefetch_file_metadata(self, fragments, **ray_remote_args): ) # Expect precomputed row counts and block sizes to be missing. - assert ds._meta_count() == expected_num_rows + assert ds._meta_count() is None read_op = ds._plan._logical_plan.dag # Assert Read op metadata propagation - assert read_op.infer_metadata() == BlockMetadata( - num_rows=expected_num_rows, - size_bytes=expected_byte_size, + metadata = read_op.infer_metadata() + # NOTE: We assert on byte size separately, since we're using `pytest.approx` + # object for it + assert metadata.size_bytes == expected_byte_size + + assert metadata == BlockMetadata( + num_rows=None, + size_bytes=metadata.size_bytes, exec_stats=None, input_files=[path1, path2], ) @@ -371,8 +313,6 @@ def prefetch_file_metadata(self, fragments, **ray_remote_args): assert ds.schema() == Schema(expected_schema) assert set(ds.input_files()) == {path1, path2} - assert ds._plan.has_computed_output() - @pytest.mark.parametrize( "fs,data_path", @@ -899,7 +839,11 @@ def test_parquet_reader_estimate_data_size(shutdown_only, tmp_path): ctx.decoding_size_estimation = True try: tensor_output_path = os.path.join(tmp_path, "tensor") - ray.data.range_tensor(1000, shape=(1000,)).write_parquet(tensor_output_path) + # NOTE: It's crucial to override # of blocks to get stable # of files + # produced and make sure data size estimates are stable + ray.data.range_tensor( + 1000, shape=(1000,), override_num_blocks=10 + ).write_parquet(tensor_output_path) ds = ray.data.read_parquet( tensor_output_path, meta_provider=ParquetMetadataProvider() ) @@ -940,7 +884,7 @@ def test_parquet_reader_estimate_data_size(shutdown_only, tmp_path): assert ds._plan.initial_num_blocks() > 1 data_size = ds.size_bytes() assert ( - data_size >= 800_000 and data_size <= 2_000_000 + data_size >= 800_000 and data_size <= 2_200_000 ), "estimated data size is out of expected bound" data_size = ds.materialize().size_bytes() assert ( @@ -955,7 +899,7 @@ def test_parquet_reader_estimate_data_size(shutdown_only, tmp_path): ), "encoding ratio is out of expected bound" data_size = datasource.estimate_inmemory_data_size() assert ( - data_size >= 800_000 and data_size <= 2_000_000 + data_size >= 800_000 and data_size <= 2_200_000 ), "estimated data size is out of expected bound" assert ( data_size From f465fc3feed7d743e8fb414d73eac3d7e63484b9 Mon Sep 17 00:00:00 2001 From: Abrar Sheikh Date: Tue, 2 Sep 2025 14:56:53 -0700 Subject: [PATCH 400/634] Fix non thread safe asyncio task creation in router (#56124) Closes https://github.com/ray-project/ray/issues/55142 Creating asyncio tasks from one thread on to event loop on another thread is not thread safe (but permissible) in asyncio. Currently this happens in two places in router 1. during handling assign_request 2. during creation of `_report_cached_metrics_forever` This PR fixes that, so that task creation happens in thread safe manner. I validated that this does not break bulk task cancellation, by rerunning the repro script from https://github.com/ray-project/ray/pull/52591 --------- Signed-off-by: abrar --- python/ray/serve/_private/router.py | 49 ++++++++---- python/ray/serve/_private/utils.py | 33 -------- .../unit/test_run_coroutine_threadsafe.py | 78 ------------------- 3 files changed, 36 insertions(+), 124 deletions(-) delete mode 100644 python/ray/serve/tests/unit/test_run_coroutine_threadsafe.py diff --git a/python/ray/serve/_private/router.py b/python/ray/serve/_private/router.py index 29658716439d..52e2335df4c0 100644 --- a/python/ray/serve/_private/router.py +++ b/python/ray/serve/_private/router.py @@ -5,7 +5,7 @@ import time import weakref from abc import ABC, abstractmethod -from asyncio import AbstractEventLoop +from asyncio import AbstractEventLoop, ensure_future, futures from collections import defaultdict from collections.abc import MutableMapping from contextlib import contextmanager @@ -53,7 +53,6 @@ from ray.serve._private.utils import ( generate_request_id, resolve_deployment_response, - run_coroutine_or_future_threadsafe, ) from ray.serve.config import AutoscalingConfig from ray.serve.exceptions import BackPressureError, DeploymentUnavailableError @@ -147,7 +146,14 @@ def __init__( if self._cached_metrics_enabled: self._cached_num_router_requests = defaultdict(int) - event_loop.create_task(self._report_cached_metrics_forever()) + + def create_metrics_task(): + event_loop.create_task(self._report_cached_metrics_forever()) + + # the constructor is called in the user thread, but its trying to create a task on the event loop + # which is running in the router thread. This is not thread safe, so we need to use call_soon_threadsafe + # to create the task on the event loop thread safely. + event_loop.call_soon_threadsafe(create_metrics_task) @contextmanager def wrap_request_assignment(self, request_meta: RequestMetadata): @@ -949,17 +955,34 @@ def asyncio_future_callback( ) result.cancel() - task = self._asyncio_loop.create_task( - self._asyncio_router.assign_request( - request_meta, *request_args, **request_kwargs + concurrent_future = concurrent.futures.Future() + + def create_task_and_setup(): + task = self._asyncio_loop.create_task( + self._asyncio_router.assign_request( + request_meta, *request_args, **request_kwargs + ) ) - ) - # Route the actual request assignment coroutine on the asyncio loop thread. - concurrent_future = run_coroutine_or_future_threadsafe( - task, - loop=self._asyncio_loop, - ) - task.add_done_callback(lambda _: asyncio_future_callback(_, concurrent_future)) + + # Set up your cancellation callback + task.add_done_callback( + lambda _: asyncio_future_callback(_, concurrent_future) + ) + + try: + # chain the two futures to handle direction channel of cancellation + futures._chain_future( + ensure_future(task, loop=self._asyncio_loop), concurrent_future + ) + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + if concurrent_future.set_running_or_notify_cancel(): + concurrent_future.set_exception(exc) + raise + + # Schedule on the event loop thread + self._asyncio_loop.call_soon_threadsafe(create_task_and_setup) return concurrent_future def shutdown(self) -> concurrent.futures.Future: diff --git a/python/ray/serve/_private/utils.py b/python/ray/serve/_private/utils.py index 924702c704da..065f6a8a301d 100644 --- a/python/ray/serve/_private/utils.py +++ b/python/ray/serve/_private/utils.py @@ -1,6 +1,5 @@ import asyncio import collections -import concurrent.futures import copy import importlib import inspect @@ -10,7 +9,6 @@ import time import uuid from abc import ABC, abstractmethod -from asyncio import coroutines, ensure_future, futures from decimal import ROUND_HALF_UP, Decimal from enum import Enum from functools import wraps @@ -636,37 +634,6 @@ def is_grpc_enabled(grpc_config) -> bool: return grpc_config.port > 0 and len(grpc_config.grpc_servicer_functions) > 0 -def run_coroutine_or_future_threadsafe(coro_or_future, loop): - """Submit a coroutine object or future to a given event loop. - - Ref: https://github.com/python/cpython/blob/eef49c359505eaf109d519d39e53dfd3c78d066a/Lib/asyncio/tasks.py#L991 - - Return a concurrent.futures.Future to access the result. - """ - if not coroutines.iscoroutine(coro_or_future) and not futures.isfuture( - coro_or_future - ): - raise TypeError("A coroutine object or future is required") - - if futures.isfuture(coro_or_future): - assert loop == coro_or_future.get_loop() - - future = concurrent.futures.Future() - - def callback(): - try: - futures._chain_future(ensure_future(coro_or_future, loop=loop), future) - except (SystemExit, KeyboardInterrupt): - raise - except BaseException as exc: - if future.set_running_or_notify_cancel(): - future.set_exception(exc) - raise - - loop.call_soon_threadsafe(callback) - return future - - class Semaphore: """Based on asyncio.Semaphore. diff --git a/python/ray/serve/tests/unit/test_run_coroutine_threadsafe.py b/python/ray/serve/tests/unit/test_run_coroutine_threadsafe.py deleted file mode 100644 index ffc8a0fe7c01..000000000000 --- a/python/ray/serve/tests/unit/test_run_coroutine_threadsafe.py +++ /dev/null @@ -1,78 +0,0 @@ -import asyncio -import concurrent.futures -import sys -import threading - -import pytest - -from ray._common.test_utils import wait_for_condition -from ray.serve._private.utils import run_coroutine_or_future_threadsafe - - -@pytest.fixture -def separate_loop(): - loop = asyncio.new_event_loop() - thread = threading.Thread(target=loop.run_forever) - thread.start() - yield loop - loop.call_soon_threadsafe(loop.stop) - thread.join() - loop.close() - - -@pytest.mark.asyncio -async def test_run_coroutine_threadsafe_with_basic_coroutine(separate_loop): - async def sample_coro(): - await asyncio.sleep(0.01) - return "ok" - - future = run_coroutine_or_future_threadsafe(sample_coro(), separate_loop) - result = future.result(timeout=1) - - assert isinstance(future, concurrent.futures.Future) - assert result == "ok" - - -@pytest.mark.asyncio -async def test_run_coroutine_threadsafe_with_future(separate_loop): - async_future = asyncio.Future(loop=separate_loop) - async_future.set_result("ok2") - future = run_coroutine_or_future_threadsafe(async_future, separate_loop) - result = future.result(timeout=1) - assert result == "ok2" - - -@pytest.mark.asyncio -async def test_run_coroutine_threadsafe_with_task(separate_loop): - async def sample_coro(): - await asyncio.sleep(0.01) - return "ok" - - async_future = separate_loop.create_task(sample_coro()) - future = run_coroutine_or_future_threadsafe(async_future, separate_loop) - result = future.result(timeout=1) - assert result == "ok" - - -@pytest.mark.asyncio -async def test_run_coroutine_threadsafe_cancellation(separate_loop): - async def cancelled_coro(): - await asyncio.sleep(5) - - async_future = separate_loop.create_task(cancelled_coro()) - future = run_coroutine_or_future_threadsafe(async_future, separate_loop) - future.cancel() - assert future.cancelled() - wait_for_condition(lambda: async_future.cancelled()) - - -@pytest.mark.asyncio -async def test_run_coroutine_threadsafe_with_future_from_other_loop(separate_loop): - future = asyncio.Future(loop=asyncio.get_running_loop()) - future.set_result("ok") - with pytest.raises(AssertionError): - run_coroutine_or_future_threadsafe(future, separate_loop) - - -if __name__ == "__main__": - sys.exit(pytest.main(["-v", "-s", __file__])) From 44f1c7f22fe6b1fcc68475c2a3849a9e67e67398 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Tue, 2 Sep 2025 15:20:06 -0700 Subject: [PATCH 401/634] [core] Revert granted check change from #55806 (#56155) Signed-off-by: dayshah --- src/ray/raylet/worker.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ray/raylet/worker.h b/src/ray/raylet/worker.h index c88c4b83402e..115ff53ce11d 100644 --- a/src/ray/raylet/worker.h +++ b/src/ray/raylet/worker.h @@ -244,10 +244,10 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa bool IsRegistered() { return rpc_client_ != nullptr; } bool IsAvailableForScheduling() const { - return !IsDead() // Not dead - && GetGrantedLeaseId().IsNil() // No assigned lease - && !IsBlocked() // Not blocked - && GetActorId().IsNil(); // No assigned actor + return !IsDead() // Not dead + && !GetGrantedLeaseId().IsNil() // Has assigned lease + && !IsBlocked() // Not blocked + && GetActorId().IsNil(); // No assigned actor } rpc::CoreWorkerClientInterface *rpc_client() { From 0c1ae99eeab7efcd583108ba0d4b9f5d1eb2ec83 Mon Sep 17 00:00:00 2001 From: ahao-anyscale Date: Tue, 2 Sep 2025 15:33:10 -0700 Subject: [PATCH 402/634] [Serve.llm][PD] changed LMCache dependency to use 0.3.3 to avoid regressions in the release test (#56104) --- release/ray_release/byod/byod_llm_lmcache_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/ray_release/byod/byod_llm_lmcache_test.sh b/release/ray_release/byod/byod_llm_lmcache_test.sh index f3a1a7d77497..413a7409f17a 100755 --- a/release/ray_release/byod/byod_llm_lmcache_test.sh +++ b/release/ray_release/byod/byod_llm_lmcache_test.sh @@ -4,4 +4,4 @@ set -exo pipefail -pip3 install "lmcache>=0.3.2" +pip3 install "lmcache==0.3.3" From 7e48cc4059c2de42a9122752b19dee3d2ee55631 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Tue, 2 Sep 2025 15:35:23 -0700 Subject: [PATCH 403/634] [core] Fix test_kill_raylet_signal_log on mac (#56151) Signed-off-by: dayshah --- .../ray/tests/test_kill_raylet_signal_log.py | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/python/ray/tests/test_kill_raylet_signal_log.py b/python/ray/tests/test_kill_raylet_signal_log.py index 651caa5bb44a..abc01680cea8 100644 --- a/python/ray/tests/test_kill_raylet_signal_log.py +++ b/python/ray/tests/test_kill_raylet_signal_log.py @@ -6,7 +6,6 @@ import ray from ray._common.test_utils import wait_for_condition -# Import psutil after ray so the packaged version is used. import psutil @@ -15,31 +14,25 @@ def get_pid(name): for pid in pids: if name in pid.name(): return pid.pid - return -1 -def check_result(filename, num_signal, check_key): - ray.init(num_cpus=1) +@pytest.mark.skipif(sys.platform == "win32", reason="Not support on Windows.") +def test_kill_raylet_signal_log(ray_start_regular): session_dir = ray._private.worker._global_node.get_session_dir_path() - raylet_out_path = filename.format(session_dir) + raylet_out_path = "{}/logs/raylet.err".format(session_dir) pid = get_pid("raylet") assert pid > 0 p = psutil.Process(pid) - p.send_signal(num_signal) + p.send_signal(signal.SIGABRT) p.wait(timeout=15) - def check_file(): + def check_for_sigabrt_in_log(): with open(raylet_out_path) as f: s = f.read() - return check_key in s - - wait_for_condition(check_file) + return "SIGABRT" in s - -@pytest.mark.skipif(sys.platform == "win32", reason="Not support on Windows.") -def test_kill_raylet_signal_log(shutdown_only): - check_result("{}/logs/raylet.err", signal.SIGABRT, "SIGABRT") + wait_for_condition(check_for_sigabrt_in_log) if __name__ == "__main__": From acea03674448e5e7a35f746466a12a6102b79957 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 2 Sep 2025 17:52:20 -0500 Subject: [PATCH 404/634] [core] Move `gcs_pb_utils` to common (#56160) Used across GCS, Raylet, worker, so should be in `common/`. Also moved implementations to `.cc` file (aside from two single-line functions). --------- Signed-off-by: Edward Oakes --- src/ray/common/BUILD.bazel | 17 ++ .../pb_util.h => common/protobuf_utils.cc} | 167 +++++----------- src/ray/common/protobuf_utils.h | 178 ++++++++++++++++++ src/ray/core_worker/BUILD.bazel | 11 +- src/ray/core_worker/actor_manager.cc | 2 +- src/ray/core_worker/core_worker.cc | 3 +- src/ray/core_worker/core_worker_process.cc | 2 +- src/ray/core_worker/lib/java/BUILD.bazel | 1 + .../java/io_ray_runtime_RayNativeRuntime.cc | 1 + src/ray/core_worker/task_event_buffer.h | 2 +- .../task_execution/tests/BUILD.bazel | 1 + .../tests/task_receiver_test.cc | 1 + src/ray/core_worker/task_manager.cc | 2 +- .../core_worker/task_submission/BUILD.bazel | 10 +- .../task_submission/actor_task_submitter.cc | 3 +- .../task_submission/normal_task_submitter.cc | 3 +- src/ray/gcs/BUILD.bazel | 16 -- src/ray/gcs/gcs_client/BUILD.bazel | 2 +- src/ray/gcs/gcs_client/tests/BUILD.bazel | 1 + .../gcs/gcs_client/tests/gcs_client_test.cc | 1 + src/ray/gcs/gcs_server/BUILD.bazel | 10 +- src/ray/gcs/gcs_server/gcs_actor_manager.cc | 2 +- .../gcs_autoscaler_state_manager.cc | 2 +- src/ray/gcs/gcs_server/gcs_job_manager.cc | 2 +- src/ray/gcs/gcs_server/gcs_node_manager.cc | 2 +- .../gcs_server/gcs_placement_group_manager.cc | 2 +- src/ray/gcs/gcs_server/gcs_task_manager.h | 2 +- src/ray/gcs/gcs_server/tests/BUILD.bazel | 3 +- .../gcs_autoscaler_state_manager_test.cc | 2 +- .../gcs_server/tests/gcs_task_manager_test.cc | 2 +- src/ray/gcs/pb_utils.cc | 48 ----- src/ray/raylet/BUILD.bazel | 2 +- src/ray/raylet/node_manager.cc | 2 +- src/ray/raylet/worker_pool.cc | 2 +- 34 files changed, 295 insertions(+), 212 deletions(-) rename src/ray/{gcs/pb_util.h => common/protobuf_utils.cc} (72%) create mode 100644 src/ray/common/protobuf_utils.h delete mode 100644 src/ray/gcs/BUILD.bazel delete mode 100644 src/ray/gcs/pb_utils.cc diff --git a/src/ray/common/BUILD.bazel b/src/ray/common/BUILD.bazel index 9abdd7832b6a..63c10cc49e43 100644 --- a/src/ray/common/BUILD.bazel +++ b/src/ray/common/BUILD.bazel @@ -339,6 +339,23 @@ ray_cc_library( hdrs = ["source_location.h"], ) +ray_cc_library( + name = "protobuf_utils", + srcs = ["protobuf_utils.cc"], + hdrs = ["protobuf_utils.h"], + deps = [ + ":constants", + ":id", + ":ray_config", + ":task_common", + "//src/ray/protobuf:autoscaler_cc_proto", + "//src/ray/protobuf:export_task_event_cc_proto", + "//src/ray/protobuf:gcs_cc_proto", + "//src/ray/util:time", + "@com_google_absl//absl/time", + ], +) + ray_cc_library( name = "gcs_callbacks", hdrs = ["gcs_callbacks.h"], diff --git a/src/ray/gcs/pb_util.h b/src/ray/common/protobuf_utils.cc similarity index 72% rename from src/ray/gcs/pb_util.h rename to src/ray/common/protobuf_utils.cc index f934ea51309d..29a57343a44c 100644 --- a/src/ray/gcs/pb_util.h +++ b/src/ray/common/protobuf_utils.cc @@ -1,4 +1,4 @@ -// Copyright 2017 The Ray Authors. +// Copyright 2025 The Ray Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,47 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -#pragma once +#include "ray/common/protobuf_utils.h" #include #include +#include #include -#include "absl/time/time.h" -#include "ray/common/constants.h" -#include "ray/common/id.h" #include "ray/common/ray_config.h" -#include "ray/common/task/task_spec.h" #include "ray/util/time.h" -#include "src/ray/protobuf/autoscaler.pb.h" -#include "src/ray/protobuf/export_task_event.pb.h" -#include "src/ray/protobuf/gcs.pb.h" namespace ray { - namespace gcs { -using ContextCase = rpc::ActorDeathCause::ContextCase; -// Forward declaration. -std::string GenErrorMessageFromDeathCause(const rpc::ActorDeathCause &death_cause); - -/// Helper function to produce job table data (for newly created job or updated job). -/// -/// \param job_id The ID of job that needs to be registered or updated. -/// \param is_dead Whether the driver of this job is dead. -/// \param timestamp The UNIX timestamp corresponding to this event. -/// \param driver_address Address of the driver that started this job. -/// \param driver_pid Process ID of the driver running this job. -/// \param entrypoint The entrypoint name of the job. -/// \param job_config The config of this job. -/// \return The job table data created by this method. -inline std::shared_ptr CreateJobTableData( +std::shared_ptr CreateJobTableData( const ray::JobID &job_id, bool is_dead, const ray::rpc::Address &driver_address, int64_t driver_pid, const std::string &entrypoint, - const ray::rpc::JobConfig &job_config = {}) { + const ray::rpc::JobConfig &job_config) { auto job_info_ptr = std::make_shared(); job_info_ptr->set_job_id(job_id.Binary()); job_info_ptr->set_is_dead(is_dead); @@ -64,14 +43,29 @@ inline std::shared_ptr CreateJobTableData( return job_info_ptr; } -/// Helper function to produce error table data. rpc::ErrorTableData CreateErrorTableData(const std::string &error_type, const std::string &error_msg, absl::Time timestamp, - const JobID &job_id = JobID::Nil()); + const JobID &job_id) { + uint32_t max_error_msg_size_bytes = RayConfig::instance().max_error_msg_size_bytes(); + rpc::ErrorTableData error_info; + error_info.set_type(error_type); + if (error_msg.length() > max_error_msg_size_bytes) { + std::string formatted_error_message = absl::StrFormat( + "The message size exceeds %d bytes. Find the full log from the log files. Here " + "is abstract: %s", + max_error_msg_size_bytes, + std::string_view{error_msg}.substr(0, max_error_msg_size_bytes)); + error_info.set_error_message(std::move(formatted_error_message)); + } else { + error_info.set_error_message(error_msg); + } + error_info.set_timestamp(absl::ToUnixMillis(timestamp)); + error_info.set_job_id(job_id.Binary()); + return error_info; +} -/// Helper function to produce worker failure data. -inline std::shared_ptr CreateWorkerFailureData( +std::shared_ptr CreateWorkerFailureData( const WorkerID &worker_id, const NodeID &node_id, const std::string &ip_address, @@ -79,7 +73,7 @@ inline std::shared_ptr CreateWorkerFailureData( rpc::WorkerExitType disconnect_type, const std::string &disconnect_detail, int pid, - const rpc::RayException *creation_task_exception = nullptr) { + const rpc::RayException *creation_task_exception) { auto worker_failure_info_ptr = std::make_shared(); // Only report the worker id + delta (new data upon worker failures). // GCS will merge the data with original worker data. @@ -98,9 +92,7 @@ inline std::shared_ptr CreateWorkerFailureData( return worker_failure_info_ptr; } -/// Get actor creation task exception from ActorDeathCause. -/// Returns nullptr if actor isn't dead due to creation task failure. -inline const rpc::RayException *GetCreationTaskExceptionFromDeathCause( +const rpc::RayException *GetCreationTaskExceptionFromDeathCause( const rpc::ActorDeathCause *death_cause) { if (death_cause == nullptr || death_cause->context_case() != ContextCase::kCreationTaskFailureContext) { @@ -109,8 +101,7 @@ inline const rpc::RayException *GetCreationTaskExceptionFromDeathCause( return &(death_cause->creation_task_failure_context()); } -inline const std::string &GetActorDeathCauseString( - const rpc::ActorDeathCause &death_cause) { +const std::string &GetActorDeathCauseString(const rpc::ActorDeathCause &death_cause) { static absl::flat_hash_map death_cause_string{ {ContextCase::CONTEXT_NOT_SET, "CONTEXT_NOT_SET"}, {ContextCase::kRuntimeEnvFailedContext, "RuntimeEnvFailedContext"}, @@ -124,11 +115,7 @@ inline const std::string &GetActorDeathCauseString( return it->second; } -/// Get the error information from the actor death cause. -/// -/// \param[in] death_cause The rpc message that contains the actos death information. -/// \return RayErrorInfo that has propagated death cause. -inline rpc::RayErrorInfo GetErrorInfoFromActorDeathCause( +rpc::RayErrorInfo GetErrorInfoFromActorDeathCause( const rpc::ActorDeathCause &death_cause) { rpc::RayErrorInfo error_info; switch (death_cause.context_case()) { @@ -157,9 +144,7 @@ inline rpc::RayErrorInfo GetErrorInfoFromActorDeathCause( return error_info; } -/// Generate object error type from ActorDeathCause. -inline std::string GenErrorMessageFromDeathCause( - const rpc::ActorDeathCause &death_cause) { +std::string GenErrorMessageFromDeathCause(const rpc::ActorDeathCause &death_cause) { if (death_cause.context_case() == ContextCase::kCreationTaskFailureContext) { return death_cause.creation_task_failure_context().formatted_exception_string(); } else if (death_cause.context_case() == ContextCase::kRuntimeEnvFailedContext) { @@ -176,7 +161,7 @@ inline std::string GenErrorMessageFromDeathCause( } } -inline bool IsActorRestartable(const rpc::ActorTableData &actor) { +bool IsActorRestartable(const rpc::ActorTableData &actor) { RAY_CHECK_EQ(actor.state(), rpc::ActorTableData::DEAD); return actor.death_cause().context_case() == ContextCase::kActorDiedErrorContext && actor.death_cause().actor_died_error_context().reason() == @@ -189,27 +174,21 @@ inline bool IsActorRestartable(const rpc::ActorTableData &actor) { actor.max_restarts())); } -inline std::string RayErrorInfoToString(const ray::rpc::RayErrorInfo &error_info) { +std::string RayErrorInfoToString(const ray::rpc::RayErrorInfo &error_info) { std::stringstream ss; ss << "Error type " << error_info.error_type() << " exception string " << error_info.error_message(); return ss.str(); } -/// Get the parent task id from the task event. -/// -/// \param task_event Task event. -/// \return TaskID::Nil() if parent task id info not available, else the parent task id -/// for the task. -inline TaskID GetParentTaskId(const rpc::TaskEvents &task_event) { +TaskID GetParentTaskId(const rpc::TaskEvents &task_event) { if (task_event.has_task_info()) { return TaskID::FromBinary(task_event.task_info().parent_task_id()); } return TaskID::Nil(); } -inline void FillTaskInfo(rpc::TaskInfoEntry *task_info, - const TaskSpecification &task_spec) { +void FillTaskInfo(rpc::TaskInfoEntry *task_info, const TaskSpecification &task_spec) { rpc::TaskType type; if (task_spec.IsNormalTask()) { type = rpc::TaskType::NORMAL_TASK; @@ -256,9 +235,8 @@ inline void FillTaskInfo(rpc::TaskInfoEntry *task_info, } } -// Fill task_info for the export API with task specification from task_spec -inline void FillExportTaskInfo(rpc::ExportTaskEventData::TaskInfoEntry *task_info, - const TaskSpecification &task_spec) { +void FillExportTaskInfo(rpc::ExportTaskEventData::TaskInfoEntry *task_info, + const TaskSpecification &task_spec) { rpc::TaskType type; if (task_spec.IsNormalTask()) { type = rpc::TaskType::NORMAL_TASK; @@ -316,31 +294,22 @@ inline void FillExportTaskInfo(rpc::ExportTaskEventData::TaskInfoEntry *task_inf } } -/// Generate a RayErrorInfo from ErrorType -inline rpc::RayErrorInfo GetRayErrorInfo(const rpc::ErrorType &error_type, - const std::string &error_msg = "") { +rpc::RayErrorInfo GetRayErrorInfo(const rpc::ErrorType &error_type, + const std::string &error_msg) { rpc::RayErrorInfo error_info; error_info.set_error_type(error_type); error_info.set_error_message(error_msg); return error_info; } -/// Get the worker id from the task event. -/// -/// \param task_event Task event. -/// \return WorkerID::Nil() if worker id info not available, else the worker id. -inline WorkerID GetWorkerID(const rpc::TaskEvents &task_event) { +WorkerID GetWorkerID(const rpc::TaskEvents &task_event) { if (task_event.has_state_updates() && task_event.state_updates().has_worker_id()) { return WorkerID::FromBinary(task_event.state_updates().worker_id()); } return WorkerID::Nil(); } -/// Return if the task has already terminated (finished or failed) -/// -/// \param task_event Task event. -/// \return True if the task has already terminated, false otherwise. -inline bool IsTaskTerminated(const rpc::TaskEvents &task_event) { +bool IsTaskTerminated(const rpc::TaskEvents &task_event) { if (!task_event.has_state_updates()) { return false; } @@ -350,19 +319,19 @@ inline bool IsTaskTerminated(const rpc::TaskEvents &task_event) { state_updates.state_ts_ns().contains(rpc::TaskStatus::FAILED); } -inline size_t NumProfileEvents(const rpc::TaskEvents &task_event) { +size_t NumProfileEvents(const rpc::TaskEvents &task_event) { if (!task_event.has_profile_events()) { return 0; } return static_cast(task_event.profile_events().events_size()); } -inline TaskAttempt GetTaskAttempt(const rpc::TaskEvents &task_event) { +TaskAttempt GetTaskAttempt(const rpc::TaskEvents &task_event) { return std::make_pair(TaskID::FromBinary(task_event.task_id()), task_event.attempt_number()); } -inline bool IsActorTask(const rpc::TaskEvents &task_event) { +bool IsActorTask(const rpc::TaskEvents &task_event) { if (!task_event.has_task_info()) { return false; } @@ -372,7 +341,7 @@ inline bool IsActorTask(const rpc::TaskEvents &task_event) { task_info.type() == rpc::TaskType::ACTOR_CREATION_TASK; } -inline bool IsTaskFinished(const rpc::TaskEvents &task_event) { +bool IsTaskFinished(const rpc::TaskEvents &task_event) { if (!task_event.has_state_updates()) { return false; } @@ -381,14 +350,9 @@ inline bool IsTaskFinished(const rpc::TaskEvents &task_event) { return state_updates.state_ts_ns().contains(rpc::TaskStatus::FINISHED); } -/// Fill the rpc::TaskStateUpdate with the timestamps according to the status change. -/// -/// \param task_status The task status. -/// \param timestamp The timestamp. -/// \param[out] state_updates The state updates with timestamp to be updated. -inline void FillTaskStatusUpdateTime(const ray::rpc::TaskStatus &task_status, - int64_t timestamp, - ray::rpc::TaskStateUpdate *state_updates) { +void FillTaskStatusUpdateTime(const ray::rpc::TaskStatus &task_status, + int64_t timestamp, + ray::rpc::TaskStateUpdate *state_updates) { if (task_status == rpc::TaskStatus::NIL) { // Not status change. return; @@ -396,13 +360,7 @@ inline void FillTaskStatusUpdateTime(const ray::rpc::TaskStatus &task_status, (*state_updates->mutable_state_ts_ns())[task_status] = timestamp; } -/// Fill the rpc::ExportTaskEventData::TaskStateUpdate with the timestamps -/// according to the status change. -/// -/// \param task_status The task status. -/// \param timestamp The timestamp. -/// \param[out] state_updates The state updates with timestamp to be updated. -inline void FillExportTaskStatusUpdateTime( +void FillExportTaskStatusUpdateTime( const ray::rpc::TaskStatus &task_status, int64_t timestamp, rpc::ExportTaskEventData::TaskStateUpdate *state_updates) { @@ -413,9 +371,8 @@ inline void FillExportTaskStatusUpdateTime( (*state_updates->mutable_state_ts_ns())[task_status] = timestamp; } -/// Convert rpc::TaskLogInfo to rpc::ExportTaskEventData::TaskLogInfo -inline void TaskLogInfoToExport(const rpc::TaskLogInfo &src, - rpc::ExportTaskEventData::TaskLogInfo *dest) { +void TaskLogInfoToExport(const rpc::TaskLogInfo &src, + rpc::ExportTaskEventData::TaskLogInfo *dest) { dest->set_stdout_file(src.stdout_file()); dest->set_stderr_file(src.stderr_file()); dest->set_stdout_start(src.stdout_start()); @@ -424,30 +381,7 @@ inline void TaskLogInfoToExport(const rpc::TaskLogInfo &src, dest->set_stderr_end(src.stderr_end()); } -inline std::string FormatPlacementGroupLabelName(const std::string &pg_id) { - return kPlacementGroupConstraintKeyPrefix + pg_id; -} - -/// \brief Format placement group details. -/// Format: -/// :: -/// -/// \param pg_data -/// \return -inline std::string FormatPlacementGroupDetails( - const rpc::PlacementGroupTableData &pg_data) { - return PlacementGroupID::FromBinary(pg_data.placement_group_id()).Hex() + ":" + - rpc::PlacementStrategy_Name(pg_data.strategy()) + "|" + - rpc::PlacementGroupTableData::PlacementGroupState_Name(pg_data.state()); -} - -/// Generate a placement constraint for placement group. -/// -/// \param pg_id The ID of placement group. -/// \param strategy The placement strategy of placement group. -/// \return The placement constraint for placement group if it's not a strict -/// strategy, else absl::nullopt. -inline std::optional +std::optional GenPlacementConstraintForPlacementGroup(const std::string &pg_id, rpc::PlacementStrategy strategy) { rpc::autoscaler::PlacementConstraint pg_constraint; @@ -480,5 +414,4 @@ GenPlacementConstraintForPlacementGroup(const std::string &pg_id, } } // namespace gcs - } // namespace ray diff --git a/src/ray/common/protobuf_utils.h b/src/ray/common/protobuf_utils.h new file mode 100644 index 000000000000..2017107c3db8 --- /dev/null +++ b/src/ray/common/protobuf_utils.h @@ -0,0 +1,178 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "absl/time/time.h" +#include "ray/common/id.h" +#include "ray/common/task/task_spec.h" +#include "src/ray/protobuf/autoscaler.pb.h" +#include "src/ray/protobuf/export_task_event.pb.h" +#include "src/ray/protobuf/gcs.pb.h" + +namespace ray { +namespace gcs { + +using ContextCase = rpc::ActorDeathCause::ContextCase; + +/// Helper function to produce job table data (for newly created job or updated job). +/// +/// \param job_id The ID of job that needs to be registered or updated. +/// \param is_dead Whether the driver of this job is dead. +/// \param timestamp The UNIX timestamp corresponding to this event. +/// \param driver_address Address of the driver that started this job. +/// \param driver_pid Process ID of the driver running this job. +/// \param entrypoint The entrypoint name of the job. +/// \param job_config The config of this job. +/// \return The job table data created by this method. +std::shared_ptr CreateJobTableData( + const ray::JobID &job_id, + bool is_dead, + const ray::rpc::Address &driver_address, + int64_t driver_pid, + const std::string &entrypoint, + const ray::rpc::JobConfig &job_config = {}); + +/// Helper function to produce error table data. +rpc::ErrorTableData CreateErrorTableData(const std::string &error_type, + const std::string &error_msg, + absl::Time timestamp, + const JobID &job_id = JobID::Nil()); + +/// Helper function to produce worker failure data. +std::shared_ptr CreateWorkerFailureData( + const WorkerID &worker_id, + const NodeID &node_id, + const std::string &ip_address, + int64_t timestamp, + rpc::WorkerExitType disconnect_type, + const std::string &disconnect_detail, + int pid, + const rpc::RayException *creation_task_exception = nullptr); + +/// Get actor creation task exception from ActorDeathCause. +/// Returns nullptr if actor isn't dead due to creation task failure. +const rpc::RayException *GetCreationTaskExceptionFromDeathCause( + const rpc::ActorDeathCause *death_cause); + +const std::string &GetActorDeathCauseString(const rpc::ActorDeathCause &death_cause); + +/// Get the error information from the actor death cause. +/// +/// \param[in] death_cause The rpc message that contains the actos death information. +/// \return RayErrorInfo that has propagated death cause. +rpc::RayErrorInfo GetErrorInfoFromActorDeathCause( + const rpc::ActorDeathCause &death_cause); + +/// Generate object error type from ActorDeathCause. +std::string GenErrorMessageFromDeathCause(const rpc::ActorDeathCause &death_cause); + +bool IsActorRestartable(const rpc::ActorTableData &actor); + +std::string RayErrorInfoToString(const ray::rpc::RayErrorInfo &error_info); + +/// Get the parent task id from the task event. +/// +/// \param task_event Task event. +/// \return TaskID::Nil() if parent task id info not available, else the parent task id +/// for the task. +TaskID GetParentTaskId(const rpc::TaskEvents &task_event); + +void FillTaskInfo(rpc::TaskInfoEntry *task_info, const TaskSpecification &task_spec); + +// Fill task_info for the export API with task specification from task_spec +void FillExportTaskInfo(rpc::ExportTaskEventData::TaskInfoEntry *task_info, + const TaskSpecification &task_spec); + +/// Generate a RayErrorInfo from ErrorType +rpc::RayErrorInfo GetRayErrorInfo(const rpc::ErrorType &error_type, + const std::string &error_msg = ""); + +/// Get the worker id from the task event. +/// +/// \param task_event Task event. +/// \return WorkerID::Nil() if worker id info not available, else the worker id. +WorkerID GetWorkerID(const rpc::TaskEvents &task_event); + +/// Return if the task has already terminated (finished or failed) +/// +/// \param task_event Task event. +/// \return True if the task has already terminated, false otherwise. +bool IsTaskTerminated(const rpc::TaskEvents &task_event); + +size_t NumProfileEvents(const rpc::TaskEvents &task_event); + +TaskAttempt GetTaskAttempt(const rpc::TaskEvents &task_event); + +bool IsActorTask(const rpc::TaskEvents &task_event); + +bool IsTaskFinished(const rpc::TaskEvents &task_event); + +/// Fill the rpc::TaskStateUpdate with the timestamps according to the status change. +/// +/// \param task_status The task status. +/// \param timestamp The timestamp. +/// \param[out] state_updates The state updates with timestamp to be updated. +void FillTaskStatusUpdateTime(const ray::rpc::TaskStatus &task_status, + int64_t timestamp, + ray::rpc::TaskStateUpdate *state_updates); + +/// Fill the rpc::ExportTaskEventData::TaskStateUpdate with the timestamps +/// according to the status change. +/// +/// \param task_status The task status. +/// \param timestamp The timestamp. +/// \param[out] state_updates The state updates with timestamp to be updated. +void FillExportTaskStatusUpdateTime( + const ray::rpc::TaskStatus &task_status, + int64_t timestamp, + rpc::ExportTaskEventData::TaskStateUpdate *state_updates); + +/// Convert rpc::TaskLogInfo to rpc::ExportTaskEventData::TaskLogInfo +void TaskLogInfoToExport(const rpc::TaskLogInfo &src, + rpc::ExportTaskEventData::TaskLogInfo *dest); + +inline std::string FormatPlacementGroupLabelName(const std::string &pg_id) { + return kPlacementGroupConstraintKeyPrefix + pg_id; +} + +/// \brief Format placement group details. +/// Format: +/// :: +/// +/// \param pg_data +/// \return +inline std::string FormatPlacementGroupDetails( + const rpc::PlacementGroupTableData &pg_data) { + return PlacementGroupID::FromBinary(pg_data.placement_group_id()).Hex() + ":" + + rpc::PlacementStrategy_Name(pg_data.strategy()) + "|" + + rpc::PlacementGroupTableData::PlacementGroupState_Name(pg_data.state()); +} + +/// Generate a placement constraint for placement group. +/// +/// \param pg_id The ID of placement group. +/// \param strategy The placement strategy of placement group. +/// \return The placement constraint for placement group if it's not a strict +/// strategy, else absl::nullopt. +std::optional +GenPlacementConstraintForPlacementGroup(const std::string &pg_id, + rpc::PlacementStrategy strategy); + +} // namespace gcs +} // namespace ray diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 67a2cb237046..3e10d3417158 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -13,6 +13,9 @@ ray_cc_library( "core_worker_rpc_proxy.h", "core_worker_shutdown_executor.h", ], + implementation_deps = [ + "//src/ray/util:time", + ], deps = [ ":actor_handle", ":actor_manager", @@ -30,12 +33,12 @@ ray_cc_library( ":reference_count", ":shutdown_coordinator", ":task_event_buffer", + "//src/ray/common:protobuf_utils", "//src/ray/common/cgroup:cgroup_context", "//src/ray/common/cgroup:cgroup_manager", "//src/ray/common/cgroup:constants", "//src/ray/core_worker/task_execution:task_receiver", "//src/ray/core_worker/task_submission:normal_task_submitter", - "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/ipc:raylet_ipc_client", "//src/ray/protobuf:pubsub_cc_proto", @@ -154,9 +157,9 @@ ray_cc_library( ":core_worker_context", ":reference_count", "//src/ray/common:id", + "//src/ray/common:protobuf_utils", "//src/ray/common:task_common", "//src/ray/core_worker/task_submission:actor_task_submitter", - "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/protobuf:core_worker_cc_proto", "@com_google_absl//absl/container:flat_hash_map", @@ -205,8 +208,8 @@ ray_cc_library( deps = [ "//src/ray/common:asio", "//src/ray/common:id", + "//src/ray/common:protobuf_utils", "//src/ray/common:task_common", - "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/protobuf:export_task_event_cc_proto", "//src/ray/protobuf:gcs_cc_proto", @@ -262,7 +265,7 @@ ray_cc_library( ":task_manager_interface", "//src/ray/common:buffer", "//src/ray/common:id", - "//src/ray/gcs:gcs_pb_util", + "//src/ray/common:protobuf_utils", "//src/ray/protobuf:common_cc_proto", "//src/ray/protobuf:core_worker_cc_proto", "//src/ray/stats:stats_metric", diff --git a/src/ray/core_worker/actor_manager.cc b/src/ray/core_worker/actor_manager.cc index 418539515a25..767525be2937 100644 --- a/src/ray/core_worker/actor_manager.cc +++ b/src/ray/core_worker/actor_manager.cc @@ -19,7 +19,7 @@ #include #include -#include "ray/gcs/pb_util.h" +#include "ray/common/protobuf_utils.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 4b73866600b0..a45b65dbe3fe 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -38,15 +38,16 @@ #include "ray/common/cgroup/cgroup_context.h" #include "ray/common/cgroup/cgroup_manager.h" #include "ray/common/cgroup/constants.h" +#include "ray/common/protobuf_utils.h" #include "ray/common/ray_config.h" #include "ray/common/runtime_env_common.h" #include "ray/common/task/task_util.h" #include "ray/gcs/gcs_client/gcs_client.h" -#include "ray/gcs/pb_util.h" #include "ray/rpc/event_aggregator_client.h" #include "ray/util/container_util.h" #include "ray/util/event.h" #include "ray/util/subreaper.h" +#include "ray/util/time.h" using json = nlohmann::json; using MessageType = ray::protocol::MessageType; diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index 1d8fed4ed879..844df982b32a 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -27,13 +27,13 @@ #include "ray/common/cgroup/cgroup_context.h" #include "ray/common/cgroup/cgroup_manager.h" #include "ray/common/cgroup/constants.h" +#include "ray/common/protobuf_utils.h" #include "ray/common/ray_config.h" #include "ray/common/runtime_env_common.h" #include "ray/common/task/task_util.h" #include "ray/core_worker/core_worker.h" #include "ray/core_worker/core_worker_rpc_proxy.h" #include "ray/gcs/gcs_client/gcs_client.h" -#include "ray/gcs/pb_util.h" #include "ray/ipc/raylet_ipc_client.h" #include "ray/raylet_client/raylet_client.h" #include "ray/stats/stats.h" diff --git a/src/ray/core_worker/lib/java/BUILD.bazel b/src/ray/core_worker/lib/java/BUILD.bazel index bce303e200b8..4e35f82a0490 100644 --- a/src/ray/core_worker/lib/java/BUILD.bazel +++ b/src/ray/core_worker/lib/java/BUILD.bazel @@ -26,6 +26,7 @@ ray_cc_binary( "//src/ray/core_worker:core_worker_lib", "//src/ray/gcs/gcs_client:global_state_accessor_lib", "//src/ray/stats:stats_lib", + "//src/ray/util:time", "@bazel_tools//tools/jdk:jni", ], ) diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc b/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc index 7584536ead71..1b8b72cbb8e6 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_RayNativeRuntime.cc @@ -28,6 +28,7 @@ #include "ray/common/ray_config.h" #include "ray/core_worker/actor_handle.h" #include "ray/core_worker/core_worker.h" +#include "ray/util/time.h" thread_local JNIEnv *local_env = nullptr; jobject java_task_executor = nullptr; diff --git a/src/ray/core_worker/task_event_buffer.h b/src/ray/core_worker/task_event_buffer.h index eaf6511b3b33..1f3d11283612 100644 --- a/src/ray/core_worker/task_event_buffer.h +++ b/src/ray/core_worker/task_event_buffer.h @@ -26,9 +26,9 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/asio/periodical_runner.h" #include "ray/common/id.h" +#include "ray/common/protobuf_utils.h" #include "ray/common/task/task_spec.h" #include "ray/gcs/gcs_client/gcs_client.h" -#include "ray/gcs/pb_util.h" #include "ray/rpc/event_aggregator_client.h" #include "ray/util/counter_map.h" #include "ray/util/event.h" diff --git a/src/ray/core_worker/task_execution/tests/BUILD.bazel b/src/ray/core_worker/task_execution/tests/BUILD.bazel index b60f96696bd1..0f7c8e4a5cc2 100644 --- a/src/ray/core_worker/task_execution/tests/BUILD.bazel +++ b/src/ray/core_worker/task_execution/tests/BUILD.bazel @@ -61,6 +61,7 @@ ray_cc_test( "//src/ray/common:test_utils", "//src/ray/core_worker/task_execution:task_receiver", "//src/ray/rpc:core_worker_client", + "//src/ray/util:time", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/core_worker/task_execution/tests/task_receiver_test.cc b/src/ray/core_worker/task_execution/tests/task_receiver_test.cc index cae330fec4e0..94b143bc9cb4 100644 --- a/src/ray/core_worker/task_execution/tests/task_receiver_test.cc +++ b/src/ray/core_worker/task_execution/tests/task_receiver_test.cc @@ -23,6 +23,7 @@ #include "ray/common/task/task_spec.h" #include "ray/common/test_utils.h" #include "ray/rpc/worker/core_worker_client.h" +#include "ray/util/time.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/task_manager.cc b/src/ray/core_worker/task_manager.cc index 8a912f74c585..c058ec121bbd 100644 --- a/src/ray/core_worker/task_manager.cc +++ b/src/ray/core_worker/task_manager.cc @@ -23,8 +23,8 @@ #include "absl/strings/match.h" #include "ray/common/buffer.h" +#include "ray/common/protobuf_utils.h" #include "ray/core_worker/actor_manager.h" -#include "ray/gcs/pb_util.h" #include "ray/util/exponential_backoff.h" #include "ray/util/time.h" #include "src/ray/protobuf/common.pb.h" diff --git a/src/ray/core_worker/task_submission/BUILD.bazel b/src/ray/core_worker/task_submission/BUILD.bazel index 4e00ea8ae4eb..ce90126245e3 100644 --- a/src/ray/core_worker/task_submission/BUILD.bazel +++ b/src/ray/core_worker/task_submission/BUILD.bazel @@ -57,6 +57,9 @@ ray_cc_library( name = "actor_task_submitter", srcs = ["actor_task_submitter.cc"], hdrs = ["actor_task_submitter.h"], + implementation_deps = [ + "//src/ray/util:time", + ], visibility = [ ":__subpackages__", "//src/ray/core_worker:__pkg__", @@ -68,8 +71,8 @@ ray_cc_library( ":sequential_actor_submit_queue", "//src/ray/common:asio", "//src/ray/common:id", + "//src/ray/common:protobuf_utils", "//src/ray/core_worker:actor_creator", - "//src/ray/gcs:gcs_pb_util", "//src/ray/rpc:core_worker_client", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", @@ -81,6 +84,9 @@ ray_cc_library( name = "normal_task_submitter", srcs = ["normal_task_submitter.cc"], hdrs = ["normal_task_submitter.h"], + implementation_deps = [ + "//src/ray/util:time", + ], visibility = [ ":__subpackages__", "//src/ray/core_worker:__pkg__", @@ -89,10 +95,10 @@ ray_cc_library( ":dependency_resolver", "//src/ray/common:id", "//src/ray/common:lease", + "//src/ray/common:protobuf_utils", "//src/ray/core_worker:lease_policy", "//src/ray/core_worker:memory_store", "//src/ray/core_worker:task_manager_interface", - "//src/ray/gcs:gcs_pb_util", "//src/ray/raylet_client:raylet_client_interface", "//src/ray/rpc:core_worker_client", "@com_google_absl//absl/base:core_headers", diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.cc b/src/ray/core_worker/task_submission/actor_task_submitter.cc index e34078b5956d..d0f6063e4865 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.cc +++ b/src/ray/core_worker/task_submission/actor_task_submitter.cc @@ -20,7 +20,8 @@ #include #include -#include "ray/gcs/pb_util.h" +#include "ray/common/protobuf_utils.h" +#include "ray/util/time.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 36a8d8927595..275760546c88 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -22,7 +22,8 @@ #include #include "ray/common/lease/lease_spec.h" -#include "ray/gcs/pb_util.h" +#include "ray/common/protobuf_utils.h" +#include "ray/util/time.h" namespace ray { namespace core { diff --git a/src/ray/gcs/BUILD.bazel b/src/ray/gcs/BUILD.bazel deleted file mode 100644 index fc039b56ae85..000000000000 --- a/src/ray/gcs/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -load("//bazel:ray.bzl", "ray_cc_library") - -ray_cc_library( - name = "gcs_pb_util", - srcs = ["pb_utils.cc"], - hdrs = ["pb_util.h"], - deps = [ - "//src/ray/common:constants", - "//src/ray/common:id", - "//src/ray/common:ray_config", - "//src/ray/common:task_common", - "//src/ray/protobuf:autoscaler_cc_proto", - "//src/ray/protobuf:export_task_event_cc_proto", - "//src/ray/util:time", - ], -) diff --git a/src/ray/gcs/gcs_client/BUILD.bazel b/src/ray/gcs/gcs_client/BUILD.bazel index 52870712ee48..b4b9ec43ade4 100644 --- a/src/ray/gcs/gcs_client/BUILD.bazel +++ b/src/ray/gcs/gcs_client/BUILD.bazel @@ -13,7 +13,7 @@ ray_cc_library( deps = [ "//src/ray/common:asio", "//src/ray/common:id", - "//src/ray/gcs:gcs_pb_util", + "//src/ray/common:protobuf_utils", "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/gcs/store_client:redis_store_client", "//src/ray/protobuf:usage_cc_proto", diff --git a/src/ray/gcs/gcs_client/tests/BUILD.bazel b/src/ray/gcs/gcs_client/tests/BUILD.bazel index a7b7511a7544..d94a8b143f23 100644 --- a/src/ray/gcs/gcs_client/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_client/tests/BUILD.bazel @@ -65,6 +65,7 @@ ray_cc_test( "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/util:network_util", "//src/ray/util:raii", + "//src/ray/util:time", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/gcs/gcs_client/tests/gcs_client_test.cc b/src/ray/gcs/gcs_client/tests/gcs_client_test.cc index bc775916d28d..12fde5c25d50 100644 --- a/src/ray/gcs/gcs_client/tests/gcs_client_test.cc +++ b/src/ray/gcs/gcs_client/tests/gcs_client_test.cc @@ -29,6 +29,7 @@ #include "ray/util/network_util.h" #include "ray/util/path_utils.h" #include "ray/util/raii.h" +#include "ray/util/time.h" using namespace std::chrono_literals; // NOLINT diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 846b509d5d29..b3c6248912a9 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -72,8 +72,8 @@ ray_cc_library( ":grpc_service_interfaces", "//src/ray/common:asio", "//src/ray/common:id", + "//src/ray/common:protobuf_utils", "//src/ray/common:ray_config", - "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/protobuf:autoscaler_cc_proto", "//src/ray/protobuf:gcs_service_cc_proto", @@ -213,9 +213,9 @@ ray_cc_library( ":grpc_service_interfaces", "//src/ray/common:asio", "//src/ray/common:id", + "//src/ray/common:protobuf_utils", "//src/ray/common:ray_config", "//src/ray/common:status", - "//src/ray/gcs:gcs_pb_util", "//src/ray/protobuf:events_event_aggregator_service_cc_proto", "//src/ray/protobuf:gcs_cc_proto", "//src/ray/stats:stats_metric", @@ -248,8 +248,8 @@ ray_cc_library( ":gcs_init_data", ":gcs_table_storage", ":grpc_service_interfaces", + "//src/ray/common:protobuf_utils", "//src/ray/common:runtime_env", - "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/rpc:core_worker_client", "//src/ray/stats:stats_metric", @@ -411,9 +411,9 @@ ray_cc_library( "gcs_actor_manager.h", ], implementation_deps = [ + "//src/ray/common:protobuf_utils", "//src/ray/common:ray_config", "//src/ray/common:task_common", - "//src/ray/gcs:gcs_pb_util", "//src/ray/stats:stats_lib", "//src/ray/util:logging", "//src/ray/util:time", @@ -449,8 +449,8 @@ ray_cc_library( "gcs_autoscaler_state_manager.h", ], implementation_deps = [ + "//src/ray/common:protobuf_utils", "//src/ray/common:ray_config", - "//src/ray/gcs:gcs_pb_util", "//src/ray/util:logging", "//src/ray/util:string_utils", "//src/ray/util:time", diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index 061fc36fce81..6cd3d33d8ca0 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -22,9 +22,9 @@ #include #include +#include "ray/common/protobuf_utils.h" #include "ray/common/ray_config.h" #include "ray/common/task/task_spec.h" -#include "ray/gcs/pb_util.h" #include "ray/stats/metric_defs.h" #include "ray/util/logging.h" #include "ray/util/time.h" diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc index 5ad9d09c719d..f356930f3fc3 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc @@ -19,7 +19,7 @@ #include #include -#include "ray/gcs/pb_util.h" +#include "ray/common/protobuf_utils.h" #include "ray/util/string_utils.h" #include "ray/util/time.h" diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.cc b/src/ray/gcs/gcs_server/gcs_job_manager.cc index 16115389726e..2d4e93495b85 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_job_manager.cc @@ -22,7 +22,7 @@ #include #include "absl/strings/match.h" -#include "ray/gcs/pb_util.h" +#include "ray/common/protobuf_utils.h" #include "ray/gcs/pubsub/gcs_pub_sub.h" #include "ray/stats/metric.h" #include "ray/util/time.h" diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.cc b/src/ray/gcs/gcs_server/gcs_node_manager.cc index dbdc5aee78a9..eef85fc4b08b 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_node_manager.cc @@ -22,7 +22,7 @@ #include #include "absl/container/flat_hash_set.h" -#include "ray/gcs/pb_util.h" +#include "ray/common/protobuf_utils.h" #include "ray/util/logging.h" #include "ray/util/time.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc b/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc index 19a7fb93d588..6580f5872327 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc @@ -21,8 +21,8 @@ #include "ray/common/asio/asio_util.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/protobuf_utils.h" #include "ray/common/ray_config.h" -#include "ray/gcs/pb_util.h" #include "ray/stats/metric_defs.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_task_manager.h b/src/ray/gcs/gcs_server/gcs_task_manager.h index ca1edcc34ed0..2e9b6f3dc877 100644 --- a/src/ray/gcs/gcs_server/gcs_task_manager.h +++ b/src/ray/gcs/gcs_server/gcs_task_manager.h @@ -24,10 +24,10 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/synchronization/mutex.h" +#include "ray/common/protobuf_utils.h" #include "ray/gcs/gcs_server/gcs_ray_event_converter.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/usage_stats_client.h" -#include "ray/gcs/pb_util.h" #include "ray/stats/metric_defs.h" #include "ray/util/counter_map.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index 05534230c9db..7d5ecaf74534 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -141,6 +141,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", + "//src/ray/common:protobuf_utils", "//src/ray/common:test_utils", "//src/ray/gcs/gcs_server:gcs_task_manager", "@com_google_googletest//:gtest_main", @@ -335,8 +336,8 @@ ray_cc_test( "//:ray_mock", "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/common:asio", + "//src/ray/common:protobuf_utils", "//src/ray/common:test_utils", - "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/gcs_server:gcs_autoscaler_state_manager", "//src/ray/gcs/gcs_server:gcs_init_data", "//src/ray/gcs/gcs_server:gcs_resource_manager", diff --git a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc index d94285270952..52d244ae3a35 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc @@ -32,11 +32,11 @@ #include "mock/ray/gcs/store_client/store_client.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/protobuf_utils.h" #include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" #include "ray/gcs/gcs_server/store_client_kv.h" -#include "ray/gcs/pb_util.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc index bd39ef33c655..b52232537d48 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc @@ -25,9 +25,9 @@ #include "gtest/gtest.h" #include "ray/common/asio/asio_util.h" #include "ray/common/id.h" +#include "ray/common/protobuf_utils.h" #include "ray/common/status.h" #include "ray/common/test_utils.h" -#include "ray/gcs/pb_util.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/pb_utils.cc b/src/ray/gcs/pb_utils.cc deleted file mode 100644 index c75e78a12167..000000000000 --- a/src/ray/gcs/pb_utils.cc +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2024 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// TODO(hjiang): Move all functions from `pb_utils.h` to this implementation file. -#include -#include -#include -#include - -#include "absl/strings/str_format.h" -#include "ray/gcs/pb_util.h" - -namespace ray::gcs { - -rpc::ErrorTableData CreateErrorTableData(const std::string &error_type, - const std::string &error_msg, - absl::Time timestamp, - const JobID &job_id) { - uint32_t max_error_msg_size_bytes = RayConfig::instance().max_error_msg_size_bytes(); - rpc::ErrorTableData error_info; - error_info.set_type(error_type); - if (error_msg.length() > max_error_msg_size_bytes) { - std::string formatted_error_message = absl::StrFormat( - "The message size exceeds %d bytes. Find the full log from the log files. Here " - "is abstract: %s", - max_error_msg_size_bytes, - std::string_view{error_msg}.substr(0, max_error_msg_size_bytes)); - error_info.set_error_message(std::move(formatted_error_message)); - } else { - error_info.set_error_message(error_msg); - } - error_info.set_timestamp(absl::ToUnixMillis(timestamp)); - error_info.set_job_id(job_id.Binary()); - return error_info; -} - -} // namespace ray::gcs diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index 48d6c5081e86..e4c6ca19b987 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -107,10 +107,10 @@ ray_cc_library( ":worker", "//src/ray/common:constants", "//src/ray/common:lease", + "//src/ray/common:protobuf_utils", "//src/ray/common:ray_config", "//src/ray/common:runtime_env", "//src/ray/common:status", - "//src/ray/gcs:gcs_pb_util", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/ipc:client_connection", "//src/ray/util:network_util", diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 4e55d453d180..1ce39809764a 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -36,10 +36,10 @@ #include "ray/common/grpc_util.h" #include "ray/common/lease/lease.h" #include "ray/common/memory_monitor.h" +#include "ray/common/protobuf_utils.h" #include "ray/common/scheduling/scheduling_ids.h" #include "ray/common/status.h" #include "ray/flatbuffers/node_manager_generated.h" -#include "ray/gcs/pb_util.h" #include "ray/ipc/client_connection.h" #include "ray/raylet/local_object_manager_interface.h" #include "ray/raylet/worker_killing_policy.h" diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index ce85798dc24f..5f5719d206f1 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -29,10 +29,10 @@ #include "absl/strings/str_split.h" #include "ray/common/constants.h" #include "ray/common/lease/lease_spec.h" +#include "ray/common/protobuf_utils.h" #include "ray/common/ray_config.h" #include "ray/common/runtime_env_common.h" #include "ray/common/status.h" -#include "ray/gcs/pb_util.h" #include "ray/stats/metric_defs.h" #include "ray/util/logging.h" #include "ray/util/network_util.h" From e2bf142ab0e346248297d473e069f0c98841e133 Mon Sep 17 00:00:00 2001 From: Omkar Kulkarni Date: Tue, 2 Sep 2025 16:08:18 -0700 Subject: [PATCH 405/634] [SERVE] E2E Fix: test_standalone_3.py (#55685) ## Why are these changes needed? Current tests are setup only to test the code when `DeploymentMode == EveryNode`. In this case, we have proxies on each node. When the mode is overwritten with `HeadOnly` for any reason whatsoever, the test suite fails. This change enables assertions in both deployment modes. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: omkar Signed-off-by: Omkar Kulkarni Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cindy Zhang --- python/ray/serve/tests/test_proxy.py | 93 ++++++++++++++++++ python/ray/serve/tests/test_standalone_3.py | 102 ++++---------------- 2 files changed, 113 insertions(+), 82 deletions(-) diff --git a/python/ray/serve/tests/test_proxy.py b/python/ray/serve/tests/test_proxy.py index fb7e38c1a743..28337cfbda96 100644 --- a/python/ray/serve/tests/test_proxy.py +++ b/python/ray/serve/tests/test_proxy.py @@ -9,6 +9,7 @@ from ray._common.network_utils import build_address from ray._common.test_utils import wait_for_condition from ray.actor import ActorHandle +from ray.cluster_utils import Cluster from ray.serve._private.constants import ( DEFAULT_UVICORN_KEEP_ALIVE_TIMEOUT_S, SERVE_NAMESPACE, @@ -19,10 +20,22 @@ request_with_retries, ) from ray.serve.config import gRPCOptions +from ray.serve.context import _get_global_client from ray.serve.generated import serve_pb2 +from ray.serve.schema import ProxyStatus, ServeInstanceDetails +from ray.tests.conftest import call_ray_stop_only # noqa: F401 from ray.util.state import list_actors +@pytest.fixture +def shutdown_ray(): + if ray.is_initialized(): + ray.shutdown() + yield + if ray.is_initialized(): + ray.shutdown() + + class TestTimeoutKeepAliveConfig: """Test setting keep_alive_timeout_s in config and env.""" @@ -225,6 +238,86 @@ def check_replicas_on_worker_nodes(): ping_grpc_healthz(worker_node_channel, test_draining=True) +def test_drain_and_undrain_http_proxy_actors( + monkeypatch, shutdown_ray, call_ray_stop_only # noqa: F811 +): + """Test the state transtion of the proxy actor between + HEALTHY, DRAINING and DRAINED + """ + monkeypatch.setenv("RAY_SERVE_PROXY_MIN_DRAINING_PERIOD_S", "10") + + cluster = Cluster() + head_node = cluster.add_node(num_cpus=0) + cluster.add_node(num_cpus=1) + cluster.add_node(num_cpus=1) + cluster.wait_for_nodes() + ray.init(address=head_node.address) + serve.start(http_options={"location": "EveryNode"}) + + @serve.deployment + class HelloModel: + def __call__(self): + return "hello" + + serve.run(HelloModel.options(num_replicas=2).bind()) + + # 3 proxies, 1 controller, 2 replicas. + wait_for_condition(lambda: len(list_actors()) == 6) + assert len(ray.nodes()) == 3 + + client = _get_global_client() + serve_details = ServeInstanceDetails( + **ray.get(client._controller.get_serve_instance_details.remote()) + ) + proxy_actor_ids = {proxy.actor_id for _, proxy in serve_details.proxies.items()} + + assert len(proxy_actor_ids) == 3 + + serve.run(HelloModel.options(num_replicas=1).bind()) + # 1 proxy should be draining + + def check_proxy_status(proxy_status_to_count): + serve_details = ServeInstanceDetails( + **ray.get(client._controller.get_serve_instance_details.remote()) + ) + proxy_status_list = [proxy.status for _, proxy in serve_details.proxies.items()] + print("all proxies!!!", [proxy for _, proxy in serve_details.proxies.items()]) + current_status = { + status: proxy_status_list.count(status) for status in proxy_status_list + } + return current_status == proxy_status_to_count, current_status + + wait_for_condition( + condition_predictor=check_proxy_status, + proxy_status_to_count={ProxyStatus.HEALTHY: 2, ProxyStatus.DRAINING: 1}, + ) + + serve.run(HelloModel.options(num_replicas=2).bind()) + # The draining proxy should become healthy. + wait_for_condition( + condition_predictor=check_proxy_status, + proxy_status_to_count={ProxyStatus.HEALTHY: 3}, + ) + serve_details = ServeInstanceDetails( + **ray.get(client._controller.get_serve_instance_details.remote()) + ) + + assert { + proxy.actor_id for _, proxy in serve_details.proxies.items() + } == proxy_actor_ids + + serve.run(HelloModel.options(num_replicas=1).bind()) + # 1 proxy should be draining and eventually be drained. + wait_for_condition( + condition_predictor=check_proxy_status, + timeout=40, + proxy_status_to_count={ProxyStatus.HEALTHY: 2}, + ) + + # Clean up serve. + serve.shutdown() + + def _kill_http_proxies(): http_proxies = ray.get( serve.context._global_client._controller.get_proxies.remote() diff --git a/python/ray/serve/tests/test_standalone_3.py b/python/ray/serve/tests/test_standalone_3.py index cbf5686dc5bc..da3372f8c810 100644 --- a/python/ray/serve/tests/test_standalone_3.py +++ b/python/ray/serve/tests/test_standalone_3.py @@ -22,6 +22,13 @@ from ray.util.state import list_actors +# Some tests are not possible to run if proxy is not available on every node. +# We skip them if proxy is not available. +def is_proxy_on_every_node() -> bool: + client = _get_global_client() + return client._http_config.location == "EveryNode" + + @pytest.fixture def shutdown_ray(): if ray.is_initialized(): @@ -286,8 +293,10 @@ def __call__(self, *args): serve.run(A.bind(), name="app_f") - # 2 proxies, 1 controller, 2 replicas. - wait_for_condition(lambda: len(list_actors()) == 5) + # If proxy is on every node, total actors are 2 proxies, 1 controller, 2 replicas. + # Otherwise, total actors are 1 proxy, 1 controller, 2 replicas. + expected_actors = 5 if is_proxy_on_every_node() else 4 + wait_for_condition(lambda: len(list_actors()) == expected_actors) assert len(ray.nodes()) == 2 # Stop all deployment replicas. @@ -324,82 +333,6 @@ def serve_details_proxy_count(): ray.shutdown() -def test_drain_and_undrain_http_proxy_actors( - monkeypatch, shutdown_ray, call_ray_stop_only # noqa: F811 -): - """Test the state transtion of the proxy actor between - HEALTHY, DRAINING and DRAINED - """ - monkeypatch.setenv("RAY_SERVE_PROXY_MIN_DRAINING_PERIOD_S", "10") - - cluster = Cluster() - head_node = cluster.add_node(num_cpus=0) - cluster.add_node(num_cpus=1) - cluster.add_node(num_cpus=1) - cluster.wait_for_nodes() - ray.init(address=head_node.address) - serve.start(http_options={"location": "EveryNode"}) - - @serve.deployment - class HelloModel: - def __call__(self): - return "hello" - - serve.run(HelloModel.options(num_replicas=2).bind()) - - # 3 proxies, 1 controller, 2 replicas. - wait_for_condition(lambda: len(list_actors()) == 6) - assert len(ray.nodes()) == 3 - - client = _get_global_client() - serve_details = ServeInstanceDetails( - **ray.get(client._controller.get_serve_instance_details.remote()) - ) - proxy_actor_ids = {proxy.actor_id for _, proxy in serve_details.proxies.items()} - assert len(proxy_actor_ids) == 3 - - serve.run(HelloModel.options(num_replicas=1).bind()) - # 1 proxy should be draining - - def check_proxy_status(proxy_status_to_count): - serve_details = ServeInstanceDetails( - **ray.get(client._controller.get_serve_instance_details.remote()) - ) - proxy_status_list = [proxy.status for _, proxy in serve_details.proxies.items()] - print("all proxies!!!", [proxy for _, proxy in serve_details.proxies.items()]) - current_status = { - status: proxy_status_list.count(status) for status in proxy_status_list - } - return current_status == proxy_status_to_count, current_status - - wait_for_condition( - condition_predictor=check_proxy_status, - proxy_status_to_count={ProxyStatus.HEALTHY: 2, ProxyStatus.DRAINING: 1}, - ) - - serve.run(HelloModel.options(num_replicas=2).bind()) - # The draining proxy should become healthy. - wait_for_condition( - condition_predictor=check_proxy_status, - proxy_status_to_count={ProxyStatus.HEALTHY: 3}, - ) - serve_details = ServeInstanceDetails( - **ray.get(client._controller.get_serve_instance_details.remote()) - ) - {proxy.actor_id for _, proxy in serve_details.proxies.items()} == proxy_actor_ids - - serve.run(HelloModel.options(num_replicas=1).bind()) - # 1 proxy should be draining and eventually be drained. - wait_for_condition( - condition_predictor=check_proxy_status, - timeout=40, - proxy_status_to_count={ProxyStatus.HEALTHY: 2}, - ) - - # Clean up serve. - serve.shutdown() - - @pytest.mark.parametrize("wait_for_controller_shutdown", (True, False)) def test_controller_shutdown_gracefully( shutdown_ray, call_ray_stop_only, wait_for_controller_shutdown # noqa: F811 @@ -426,8 +359,10 @@ def __call__(self): model = HelloModel.bind() serve.run(target=model) - # Ensure total actors of 2 proxies, 1 controller, and 2 replicas - wait_for_condition(lambda: len(list_actors()) == 5) + # If proxy is on every node, total actors are 2 proxies, 1 controller, and 2 replicas + # Otherwise, total actors are 1 proxy, 1 controller, and 2 replicas + expected_actors = 5 if is_proxy_on_every_node() else 4 + wait_for_condition(lambda: len(list_actors()) == expected_actors) assert len(ray.nodes()) == 2 # Call `graceful_shutdown()` on the controller, so it will start shutdown. @@ -485,8 +420,11 @@ def __call__(self): model = HelloModel.bind() serve.run(target=model) - # Ensure total actors of 2 proxies, 1 controller, and 2 replicas - wait_for_condition(lambda: len(list_actors()) == 5) + # Check expected actors based on mode + # If proxy is on every node, total actors are 2 proxies, 1 controller, and 2 replicas + # Otherwise, total actors are 1 proxy, 1 controller, and 2 replicas + expected_actors = 5 if is_proxy_on_every_node() else 4 + wait_for_condition(lambda: len(list_actors()) == expected_actors) assert len(ray.nodes()) == 2 # Ensure client times out if the controller does not shutdown within timeout. From 822b43007b40d52fc424ebca5a640056ccd7eeed Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Tue, 2 Sep 2025 16:29:34 -0700 Subject: [PATCH 406/634] [core][proto] add ownership and readme for public proto (#56023) Add ownership and README for how to modify the proto files in the public directory. This is related to a recent work to define proto exposure via directory structure and set expectations for maintainer/users of these proto. Test: - CI Signed-off-by: Cuong Nguyen --- .github/CODEOWNERS | 3 +++ src/ray/protobuf/public/README | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 src/ray/protobuf/public/README diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6d12a001d401..38e78b8a4cad 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -29,6 +29,9 @@ /doc/source/cluster/ @ray-project/ray-core @ray-project/ray-docs /doc/source/ray-core/ @ray-project/ray-core @ray-project/ray-docs +# Public protobuf files. +/src/ray/protobuf/public/ @edoakes @jjyao + # ==== Libraries and frameworks ==== # Dependencies diff --git a/src/ray/protobuf/public/README b/src/ray/protobuf/public/README new file mode 100644 index 000000000000..9e8a687b9ce8 --- /dev/null +++ b/src/ray/protobuf/public/README @@ -0,0 +1,17 @@ +All proto files in this directory are part of public APIs. Therefore, please keep the +following guidelines in mind when modifying any of these files: + +Do NOT include private protos in these files. If you need to, either (i) obtain approval +from the core team to make the previously private proto public, or (ii) split the proto +into private and public parts, and move only the public part here. + +Do NOT delete existing fields in any proto messages. If renaming is necessary, add a new +field with the new name, and mark the old field as deprecated. + +For consumers of these proto files (end users): you can rely on field names continuing +to exist, ensuring that applications built on top of these protos do not break +unexpectedly. However, always design applications with the assumption that fields are +always optional, and handle missing or deprecated field contents gracefully. While a +field name may remain, its content could eventually be deprecated and moved to a new +field. This provides a path for us to deprecate emitting logic without breaking your +application. From d13aa963f94df3d1e0424009d638e3aa46441be5 Mon Sep 17 00:00:00 2001 From: Zac Policzer Date: Tue, 2 Sep 2025 16:36:40 -0700 Subject: [PATCH 407/634] [core][raylet][gcs] Add io_context metrics to gcs and raylet (#55762) The purpose of this change is to add metrics for monitoring the state of gcs and raylet. This change goes about that by repurposing the usage of RayConfig::instance().event_stats_metrics(). This OS environment variable previously enabled metric emission from all services and all instances of classes which wrapped calls to EventStats. This included things like instrumented_io_context which is a fairly prolific class through the code base. So the main thing we do in this change is change the name of RAY_event_stats_metrics to RAY_ emit_main_service_metrics and bring it's usage back up to the main classes of GCS and Raylet, which then pass this config into the main io_contexts used by those services. We then move usage of EventStats to be opt in, defaulting false for all other code paths. As time goes on and as we identify paths we want more monitoring on, we can opt in those code paths, and as we find cases where we don't care to have this kind of monitoring, move them off usage of things like instrumented_io_context or event_stats completely (since we're not really using the overhead for anything particularly useful). This PR also includes some clean up here and there and some metric type changes to make more sense with what they seem to intend to do. Specifically, operation_run_time_ms and operation_queue_time have been updated to be HISTOGRAM instead of GAUGE. The reason for this is that knowing the run time or queue time of the last event isn't quite as useful as knowing the histogram view which would give proper distributions on QoS. GAUGE does make sense for values which are absolute at a certain time (like queue length or CPU utilization). --------- Signed-off-by: zac --- python/ray/tests/test_metrics_agent.py | 163 ++++++------------ src/ray/common/asio/asio_util.h | 4 +- .../common/asio/instrumented_io_context.cc | 50 ++++-- src/ray/common/asio/instrumented_io_context.h | 20 ++- src/ray/common/asio/io_service_pool.cc | 2 +- src/ray/common/asio/periodical_runner.cc | 3 +- src/ray/common/event_stats.cc | 43 +++-- src/ray/common/event_stats.h | 28 ++- src/ray/common/file_system_monitor.h | 2 +- src/ray/common/ray_config_def.h | 8 +- src/ray/gcs/gcs_server/gcs_server_main.cc | 8 +- src/ray/raylet/main.cc | 6 +- src/ray/rpc/client_call.h | 4 + src/ray/stats/metric_defs.cc | 21 ++- 14 files changed, 174 insertions(+), 188 deletions(-) diff --git a/python/ray/tests/test_metrics_agent.py b/python/ray/tests/test_metrics_agent.py index 253386c5ac94..3dadc7015666 100644 --- a/python/ray/tests/test_metrics_agent.py +++ b/python/ray/tests/test_metrics_agent.py @@ -581,133 +581,68 @@ def test_case_value_correct(): def test_operation_stats(monkeypatch, shutdown_only): # Test operation stats are available when flag is on. operation_metrics = [ - "ray_operation_count", - "ray_operation_run_time_ms", - "ray_operation_queue_time_ms", + "ray_operation_count_total", + "ray_operation_run_time_ms_bucket", + "ray_operation_queue_time_ms_bucket", "ray_operation_active_count", ] - with monkeypatch.context() as m: - m.setenv("RAY_event_stats_metrics", "1") - addr = ray.init() - - signal = SignalActor.remote() - - @ray.remote - class Actor: - def __init__(self, signal): - self.signal = signal - - def get_worker_id(self): - return ray.get_runtime_context().get_worker_id() - - def wait(self): - ray.get(self.signal.wait.remote()) - - actor = Actor.remote(signal) - worker_id = ray.get(actor.get_worker_id.remote()) - obj_ref = actor.wait.remote() - - def verify(): - metrics = raw_metrics(addr) - samples = metrics["ray_operation_count"] - found = False - for sample in samples: - if ( - sample.labels["Method"] == "CoreWorkerService.grpc_client.PushTask" - and sample.labels["Component"] == "core_worker" - and sample.labels["WorkerId"] == worker_id - ): - found = True - assert sample.value == 1 - if not found: - return False + addr = ray.init() + remote_signal = SignalActor.remote() - samples = metrics["ray_operation_active_count"] - found = False - for sample in samples: - if ( - sample.labels["Method"] == "CoreWorkerService.grpc_client.PushTask" - and sample.labels["Component"] == "core_worker" - and sample.labels["WorkerId"] == worker_id - ): - found = True - assert sample.value == 1 - if not found: - return False + @ray.remote + class Actor: + def __init__(self, signal): + self.signal = signal - return True + def get_worker_id(self): + return ray.get_runtime_context().get_worker_id() - wait_for_condition(verify, timeout=60) + def wait(self): + ray.get(self.signal.wait.remote()) - ray.get(signal.send.remote()) - ray.get(obj_ref) + actor = Actor.remote(remote_signal) + ray.get(actor.get_worker_id.remote()) + obj_ref = actor.wait.remote() - def verify(): - metrics = raw_metrics(addr) + ray.get(remote_signal.send.remote()) + ray.get(obj_ref) - samples = metrics["ray_operation_count"] - found = False - for sample in samples: - if ( - sample.labels["Method"] == "CoreWorkerService.grpc_client.PushTask" - and sample.labels["Component"] == "core_worker" - and sample.labels["WorkerId"] == worker_id - ): - found = True - assert sample.value == 1 - if not found: - return False + def verify(): + metrics = raw_metrics(addr) - found = False - for sample in samples: - if ( - sample.labels["Method"] - == "CoreWorkerService.grpc_client.PushTask.OnReplyReceived" - and sample.labels["Component"] == "core_worker" - and sample.labels["WorkerId"] == worker_id - ): - found = True - assert sample.value == 1 - if not found: - return False + samples = metrics["ray_operation_active_count"] + found = False + for sample in samples: + if ( + sample.labels["Name"] == "gcs_server_main_io_context" + and sample.labels["Component"] == "gcs_server" + ): + found = True + if not found: + return False - samples = metrics["ray_operation_active_count"] - found = False - for sample in samples: - if ( - sample.labels["Method"] == "CoreWorkerService.grpc_client.PushTask" - and sample.labels["Component"] == "core_worker" - and sample.labels["WorkerId"] == worker_id - ): - found = True - assert sample.value == 0 - if not found: - return False + found = False + for sample in samples: + if ( + sample.labels["Name"] == "raylet_main_io_context" + and sample.labels["Component"] == "raylet" + ): + found = True + if not found: + return False - found = False + metric_names = set(metrics.keys()) + for op_metric in operation_metrics: + assert op_metric in metric_names + samples = metrics[op_metric] + components = set() + print(components) for sample in samples: - if ( - sample.labels["Method"] - == "CoreWorkerService.grpc_client.PushTask.OnReplyReceived" - and sample.labels["Component"] == "core_worker" - and sample.labels["WorkerId"] == worker_id - ): - found = True - assert sample.value == 0 - if not found: - return False - - metric_names = set(metrics.keys()) - for op_metric in operation_metrics: - assert op_metric in metric_names - samples = metrics[op_metric] - components = set() - for sample in samples: - components.add(sample.labels["Component"]) - assert {"raylet", "gcs_server", "core_worker"} == components - return True + components.add(sample.labels["Component"]) + assert {"raylet", "gcs_server"} == components + return True - wait_for_condition(verify, timeout=60) + wait_for_condition(verify, timeout=30) @pytest.mark.skipif(prometheus_client is None, reason="Prometheus not installed") diff --git a/src/ray/common/asio/asio_util.h b/src/ray/common/asio/asio_util.h index 793729fb5d89..f360b058aa8d 100644 --- a/src/ray/common/asio/asio_util.h +++ b/src/ray/common/asio/asio_util.h @@ -60,7 +60,7 @@ class InstrumentedIOContextWithThread { */ explicit InstrumentedIOContextWithThread(const std::string &thread_name, bool enable_lag_probe = false) - : io_service_(enable_lag_probe, /*running_on_single_thread=*/true), + : io_service_(enable_lag_probe, /*running_on_single_thread=*/true, thread_name), work_(io_service_.get_executor()), thread_name_(thread_name) { io_thread_ = std::thread([this] { @@ -90,7 +90,7 @@ class InstrumentedIOContextWithThread { } private: - instrumented_io_context io_service_{/*enable_lag_probe=*/false, + instrumented_io_context io_service_{/*enable_metrics=*/false, /*running_on_single_thread=*/true}; boost::asio::executor_work_guard work_; // to keep io_service_ running diff --git a/src/ray/common/asio/instrumented_io_context.cc b/src/ray/common/asio/instrumented_io_context.cc index 05a398a131db..9147452b6c7f 100644 --- a/src/ray/common/asio/instrumented_io_context.cc +++ b/src/ray/common/asio/instrumented_io_context.cc @@ -26,17 +26,19 @@ namespace { // Post a probe. Records the lag and schedule another probe. // Requires: `interval_ms` > 0. -void LagProbeLoop(instrumented_io_context &io_context, int64_t interval_ms) { +void LagProbeLoop(instrumented_io_context &io_context, + int64_t interval_ms, + const std::optional &context_name) { auto begin = std::chrono::steady_clock::now(); io_context.post( - [&io_context, begin, interval_ms]() { + [&io_context, begin, interval_ms, context_name]() { auto end = std::chrono::steady_clock::now(); auto duration = std::chrono::duration_cast(end - begin); ray::stats::STATS_io_context_event_loop_lag_ms.Record( duration.count(), { - {"Name", GetThreadName()}, + {"Name", context_name.value_or(GetThreadName())}, }); // Schedule the next probe. If `duration` is larger than `interval_ms`, we @@ -44,42 +46,50 @@ void LagProbeLoop(instrumented_io_context &io_context, int64_t interval_ms) { // for `interval_ms - duration`. auto delay = interval_ms - duration.count(); if (delay <= 0) { - LagProbeLoop(io_context, interval_ms); + LagProbeLoop(io_context, interval_ms, context_name); } else { execute_after( io_context, - [&io_context, interval_ms]() { LagProbeLoop(io_context, interval_ms); }, + [&io_context, interval_ms, context_name]() { + LagProbeLoop(io_context, interval_ms, context_name); + }, std::chrono::milliseconds(delay)); } }, "event_loop_lag_probe"); } -void ScheduleLagProbe(instrumented_io_context &io_context) { - if (!RayConfig::instance().enable_metrics_collection()) { - return; - } +void ScheduleLagProbe(instrumented_io_context &io_context, + const std::optional &context_name) { auto interval = RayConfig::instance().io_context_event_loop_lag_collection_interval_ms(); if (interval <= 0) { return; } RAY_LOG(DEBUG) << "Scheduling lag probe for the io_context on thread " - << GetThreadName() << " every " << interval << "ms"; + << context_name.value_or(GetThreadName()) << " every " << interval + << "ms"; // At this time, the `io_context` may not be running yet, so we need to post the // first probe. - io_context.post([&io_context, interval]() { LagProbeLoop(io_context, interval); }, - "event_loop_lag_probe"); + io_context.post( + [&io_context, interval, context_name]() { + LagProbeLoop(io_context, interval, context_name); + }, + "event_loop_lag_probe"); } } // namespace -instrumented_io_context::instrumented_io_context(bool enable_lag_probe, - bool running_on_single_thread) +instrumented_io_context::instrumented_io_context( + const bool emit_metrics, + const bool running_on_single_thread, + const std::optional context_name) : boost::asio::io_context( running_on_single_thread ? 1 : BOOST_ASIO_CONCURRENCY_HINT_DEFAULT), - event_stats_(std::make_shared()) { - if (enable_lag_probe) { - ScheduleLagProbe(*this); + event_stats_(std::make_shared()), + emit_metrics_(emit_metrics), + context_name_(context_name) { + if (emit_metrics) { + ScheduleLagProbe(*this, context_name_); } } @@ -93,7 +103,8 @@ void instrumented_io_context::post(std::function handler, // GuardedHandlerStats synchronizes internal access, we can concurrently write to the // handler stats it->second from multiple threads without acquiring a table-level // readers lock in the callback. - auto stats_handle = event_stats_->RecordStart(std::move(name)); + auto stats_handle = + event_stats_->RecordStart(std::move(name), emit_metrics_, 0, context_name_); handler = [handler = std::move(handler), stats_handle = std::move(stats_handle)]() mutable { EventTracker::RecordExecution(handler, std::move(stats_handle)); @@ -111,7 +122,8 @@ void instrumented_io_context::dispatch(std::function handler, std::strin if (!RayConfig::instance().event_stats()) { return boost::asio::post(*this, std::move(handler)); } - auto stats_handle = event_stats_->RecordStart(std::move(name)); + auto stats_handle = + event_stats_->RecordStart(std::move(name), emit_metrics_, 0, context_name_); // References are only invalidated upon deletion of the corresponding item from the // table, which we won't do until this io_context is deleted. Provided that // GuardedHandlerStats synchronizes internal access, we can concurrently write to the diff --git a/src/ray/common/asio/instrumented_io_context.h b/src/ray/common/asio/instrumented_io_context.h index 120023233a0b..33778bffc80a 100644 --- a/src/ray/common/asio/instrumented_io_context.h +++ b/src/ray/common/asio/instrumented_io_context.h @@ -15,12 +15,9 @@ #pragma once #include -#include #include #include -#include "absl/container/flat_hash_map.h" -#include "absl/synchronization/mutex.h" #include "ray/common/event_stats.h" #include "ray/common/ray_config.h" #include "ray/util/logging.h" @@ -31,11 +28,16 @@ class instrumented_io_context : public boost::asio::io_context { /// Initializes the global stats struct after calling the base contructor. /// TODO(ekl) allow taking an externally defined event tracker. /// - /// \param enable_lag_probe If true, and if related Ray configs are set, schedule a - /// probe to measure the event loop lag. After a probe is done, it schedules another one - /// so a io_context.run() call will never return. - explicit instrumented_io_context(bool enable_lag_probe = false, - bool running_on_single_thread = false); + /// \param emit_metrics enables or disables metric emission on this io_context + /// \param running_on_single_thread hints to the underlying io_context if locking should + /// be enabled or not (that is, if running on multiple threads is true, then concurrency + /// controls will engage) + /// \param context_name optional name assigned to this io_context used for metric + /// emission + explicit instrumented_io_context( + bool emit_metrics = false, + bool running_on_single_thread = false, + std::optional context_name = std::nullopt); /// A proxy post function that collects count, queueing, and execution statistics for /// the given handler. @@ -59,4 +61,6 @@ class instrumented_io_context : public boost::asio::io_context { private: /// The event stats tracker to use to record asio handler stats to. std::shared_ptr event_stats_; + bool emit_metrics_; + std::optional context_name_; }; diff --git a/src/ray/common/asio/io_service_pool.cc b/src/ray/common/asio/io_service_pool.cc index 4603266ed64c..9f3c9f8d2a1e 100644 --- a/src/ray/common/asio/io_service_pool.cc +++ b/src/ray/common/asio/io_service_pool.cc @@ -25,7 +25,7 @@ IOServicePool::~IOServicePool() {} void IOServicePool::Run() { for (size_t i = 0; i < io_service_num_; ++i) { io_services_.emplace_back(std::make_unique( - /*enable_lag_probe=*/false, /*running_on_single_thread=*/true)); + /*enable_metrics=*/false, /*running_on_single_thread=*/true)); instrumented_io_context &io_service = *io_services_[i]; threads_.emplace_back([&io_service] { boost::asio::executor_work_guard work( diff --git a/src/ray/common/asio/periodical_runner.cc b/src/ray/common/asio/periodical_runner.cc index b4f7307c7101..9da73cc39596 100644 --- a/src/ray/common/asio/periodical_runner.cc +++ b/src/ray/common/asio/periodical_runner.cc @@ -106,7 +106,8 @@ void PeriodicalRunner::DoRunFnPeriodicallyInstrumented( // NOTE: We add the timer period to the enqueue time in order only measure the time in // which the handler was elgible to execute on the event loop but was queued by the // event loop. - auto stats_handle = io_service_.stats().RecordStart(name, period.total_nanoseconds()); + auto stats_handle = + io_service_.stats().RecordStart(name, false, period.total_nanoseconds()); timer->async_wait( [weak_self = weak_from_this(), fn = std::move(fn), diff --git a/src/ray/common/event_stats.cc b/src/ray/common/event_stats.cc index 6e4f3a8b1800..b18cfd442374 100644 --- a/src/ray/common/event_stats.cc +++ b/src/ray/common/event_stats.cc @@ -60,26 +60,31 @@ std::string to_human_readable(int64_t duration) { } // namespace std::shared_ptr EventTracker::RecordStart( - std::string name, int64_t expected_queueing_delay_ns) { + std::string name, + bool emit_metrics, + const int64_t expected_queueing_delay_ns, + const std::optional &event_context_name) { auto stats = GetOrCreate(name); - int64_t cum_count = 0; int64_t curr_count = 0; { absl::MutexLock lock(&(stats->mutex)); - cum_count = ++stats->stats.cum_count; + ++stats->stats.cum_count; curr_count = ++stats->stats.curr_count; } - if (RayConfig::instance().event_stats_metrics()) { - ray::stats::STATS_operation_count.Record(cum_count, name); - ray::stats::STATS_operation_active_count.Record(curr_count, name); + if (emit_metrics) { + ray::stats::STATS_operation_count.Record(1, event_context_name.value_or(name)); + ray::stats::STATS_operation_active_count.Record(curr_count, + event_context_name.value_or(name)); } return std::make_shared( std::move(name), absl::GetCurrentTimeNanos() + expected_queueing_delay_ns, std::move(stats), - global_stats_); + global_stats_, + emit_metrics, + event_context_name); } void EventTracker::RecordEnd(std::shared_ptr handle) { @@ -89,11 +94,12 @@ void EventTracker::RecordEnd(std::shared_ptr handle) { const auto execution_time_ns = absl::GetCurrentTimeNanos() - handle->start_time; handle->handler_stats->stats.cum_execution_time += execution_time_ns; - if (RayConfig::instance().event_stats_metrics()) { + if (handle->emit_stats) { // Update event-specific stats. - ray::stats::STATS_operation_run_time_ms.Record(execution_time_ns / 1000000, - handle->event_name); - ray::stats::STATS_operation_active_count.Record(curr_count, handle->event_name); + ray::stats::STATS_operation_run_time_ms.Record( + execution_time_ns / 1000000, handle->context_name.value_or(handle->event_name)); + ray::stats::STATS_operation_active_count.Record( + curr_count, handle->context_name.value_or(handle->event_name)); } handle->end_or_execution_recorded = true; @@ -134,14 +140,15 @@ void EventTracker::RecordExecution(const std::function &fn, stats->stats.running_count--; } - if (RayConfig::instance().event_stats_metrics()) { + if (handle->emit_stats) { // Update event-specific stats. - ray::stats::STATS_operation_run_time_ms.Record(execution_time_ns / 1000000, - handle->event_name); - ray::stats::STATS_operation_active_count.Record(curr_count, handle->event_name); + ray::stats::STATS_operation_run_time_ms.Record( + execution_time_ns / 1000000, handle->context_name.value_or(handle->event_name)); + ray::stats::STATS_operation_active_count.Record( + curr_count, handle->context_name.value_or(handle->event_name)); // Update global stats. - ray::stats::STATS_operation_queue_time_ms.Record(queue_time_ns / 1000000, - handle->event_name); + ray::stats::STATS_operation_queue_time_ms.Record( + queue_time_ns / 1000000, handle->context_name.value_or(handle->event_name)); } { @@ -186,6 +193,7 @@ GlobalStats EventTracker::get_global_stats() const { return to_global_stats_view(global_stats_); } +// Testing only method std::optional EventTracker::get_event_stats( const std::string &event_name) const { absl::ReaderMutexLock lock(&mutex_); @@ -196,6 +204,7 @@ std::optional EventTracker::get_event_stats( return to_event_stats_view(it->second); } +// Logging only method std::vector> EventTracker::get_event_stats() const { // We lock the stats table while copying the table into a vector. absl::ReaderMutexLock lock(&mutex_); diff --git a/src/ray/common/event_stats.h b/src/ray/common/event_stats.h index 1650733e7770..d687d06de141 100644 --- a/src/ray/common/event_stats.h +++ b/src/ray/common/event_stats.h @@ -73,16 +73,23 @@ struct StatsHandle { const std::shared_ptr global_stats; // Whether RecordEnd or RecordExecution is called. std::atomic end_or_execution_recorded; + // Metric emission specific configurations + const bool emit_stats; + const std::optional context_name; StatsHandle(std::string event_name_, - int64_t start_time_, + const int64_t start_time_, std::shared_ptr handler_stats_, - std::shared_ptr global_stats_) + std::shared_ptr global_stats_, + const bool emit_stats_, + const std::optional &context_name_) : event_name(std::move(event_name_)), start_time(start_time_), handler_stats(std::move(handler_stats_)), global_stats(std::move(global_stats_)), - end_or_execution_recorded(false) {} + end_or_execution_recorded(false), + emit_stats(emit_stats_), + context_name(context_name_) {} ~StatsHandle() { if (!end_or_execution_recorded) { @@ -106,12 +113,19 @@ class EventTracker { /// The returned opaque stats handle MUST be given to a subsequent /// RecordExecution() or RecordEnd() call. /// - /// \param name A human-readable name to which collected stats will be associated. - /// \param expected_queueing_delay_ns How much to pad the observed queueing start time, + /// \param name A human-readable name to which collected stats will be associated for + /// logging. \param expected_queueing_delay_ns How much to pad the observed queueing + /// start time, /// in nanoseconds. + /// \param emit_metrics Emit the underlying stat as a service metric + /// \param event_context_name A human-readable name to which collected stats will be + /// associated for metrics. /// \return An opaque stats handle, to be given to RecordExecution() or RecordEnd(). - std::shared_ptr RecordStart(std::string name, - int64_t expected_queueing_delay_ns = 0); + std::shared_ptr RecordStart( + std::string name, + bool emit_metrics = false, + int64_t expected_queueing_delay_ns = 0, + const std::optional &event_context_name = std::nullopt); /// Records stats about the provided function's execution. This is used in conjunction /// with RecordStart() to manually instrument an event loop handler that calls .post(). diff --git a/src/ray/common/file_system_monitor.h b/src/ray/common/file_system_monitor.h index ccba1d5e8696..eae48ae93e3f 100644 --- a/src/ray/common/file_system_monitor.h +++ b/src/ray/common/file_system_monitor.h @@ -67,7 +67,7 @@ class FileSystemMonitor { const std::vector paths_; const double capacity_threshold_; std::atomic over_capacity_; - instrumented_io_context io_context_{/*enable_lag_probe=*/false, + instrumented_io_context io_context_{/*enable_metrics=*/false, /*running_on_single_thread=*/true}; std::thread monitor_thread_; std::shared_ptr runner_; diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index 85adf608b1cc..6e89e27d4b25 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -24,10 +24,10 @@ RAY_CONFIG(uint64_t, debug_dump_period_milliseconds, 10000) /// Whether to enable Ray event stats collection. RAY_CONFIG(bool, event_stats, true) -/// Whether to enable Ray event stats metrics export. -/// Note that enabling this adds high overhead to -/// Ray metrics agent. -RAY_CONFIG(bool, event_stats_metrics, false) +/// Whether to enable Ray event stats metrics for main services +/// such as gcs and raylet (which today are the sole consumers of +/// this config) +RAY_CONFIG(bool, emit_main_service_metrics, true) /// Whether to enable cluster authentication. RAY_CONFIG(bool, enable_cluster_auth, true) diff --git a/src/ray/gcs/gcs_server/gcs_server_main.cc b/src/ray/gcs/gcs_server/gcs_server_main.cc index 04a277924503..9cf428dbbae7 100644 --- a/src/ray/gcs/gcs_server/gcs_server_main.cc +++ b/src/ray/gcs/gcs_server/gcs_server_main.cc @@ -37,7 +37,7 @@ DEFINE_string(stdout_filepath, "", "The filepath to dump gcs server stdout."); DEFINE_string(stderr_filepath, "", "The filepath to dump gcs server stderr."); DEFINE_int32(gcs_server_port, 0, "The port of gcs server."); DEFINE_int32(metrics_agent_port, -1, "The port of metrics agent."); -DEFINE_string(config_list, "", "The config list of raylet."); +DEFINE_string(config_list, "", "The config list of gcs."); DEFINE_string(redis_username, "", "The username of Redis."); DEFINE_string(redis_password, "", "The password of Redis."); DEFINE_bool(retry_redis, false, "Whether to retry to connect to Redis."); @@ -109,8 +109,10 @@ int main(int argc, char *argv[]) { // IO Service for main loop. SetThreadName("gcs_server"); - instrumented_io_context main_service(/*enable_lag_probe=*/true, - /*running_on_single_thread=*/true); + instrumented_io_context main_service( + /*enable_metrics=*/RayConfig::instance().emit_main_service_metrics(), + /*running_on_single_thread=*/true, + "gcs_server_main_io_context"); // Ensure that the IO service keeps running. Without this, the main_service will exit // as soon as there is no more work to be processed. boost::asio::executor_work_guard work( diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index edc2fdba7660..d2a59645651e 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -241,8 +241,10 @@ int main(int argc, char *argv[]) { SetThreadName("raylet"); // IO Service for node manager. - instrumented_io_context main_service{/*enable_lag_probe=*/false, - /*running_on_single_thread=*/true}; + instrumented_io_context main_service{ + /*emit_metrics=*/RayConfig::instance().emit_main_service_metrics(), + /*running_on_single_thread=*/true, + "raylet_main_io_context"}; // Ensure that the IO service keeps running. Without this, the service will exit as soon // as there is no more work to be processed. diff --git a/src/ray/rpc/client_call.h b/src/ray/rpc/client_call.h index e9197e6466d3..dd5c794764c9 100644 --- a/src/ray/rpc/client_call.h +++ b/src/ray/rpc/client_call.h @@ -206,6 +206,10 @@ class ClientCallManager { /// /// \param[in] main_service The main event loop, to which the callback functions will be /// posted. + /// \param record_stats Whether to record stats for calls made with this client + /// \param cluster_id UUID of the destination cluster + /// \param num_threads The number of threads used for polling for completion events + /// \param call_timeout_ms Set's the default call timeout for requests on this client /// explicit ClientCallManager(instrumented_io_context &main_service, bool record_stats, diff --git a/src/ray/stats/metric_defs.cc b/src/ray/stats/metric_defs.cc index 0edf7668e4a7..d3dc780a84de 100644 --- a/src/ray/stats/metric_defs.cc +++ b/src/ray/stats/metric_defs.cc @@ -159,16 +159,19 @@ DEFINE_stats(io_context_event_loop_lag_ms, ray::stats::GAUGE); /// Event stats -DEFINE_stats(operation_count, "operation count", ("Method"), (), ray::stats::GAUGE); -DEFINE_stats( - operation_run_time_ms, "operation execution time", ("Method"), (), ray::stats::GAUGE); +DEFINE_stats(operation_count, "operation count", ("Name"), (), ray::stats::COUNT); +DEFINE_stats(operation_run_time_ms, + "operation execution time", + ("Name"), + ({1, 10, 100, 1000, 10000}), + ray::stats::HISTOGRAM); +DEFINE_stats(operation_queue_time_ms, + "operation queuing time", + ("Name"), + ({1, 10, 100, 1000, 10000}), + ray::stats::HISTOGRAM); DEFINE_stats( - operation_queue_time_ms, "operation queuing time", ("Method"), (), ray::stats::GAUGE); -DEFINE_stats(operation_active_count, - "activate operation number", - ("Method"), - (), - ray::stats::GAUGE); + operation_active_count, "active operation number", ("Name"), (), ray::stats::GAUGE); /// GRPC server DEFINE_stats(grpc_server_req_process_time_ms, From 790cfdef057e9852e064156f08592b302e093d61 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Tue, 2 Sep 2025 19:08:09 -0700 Subject: [PATCH 408/634] [release] Use bazel workspace directory when referencing path from repo root (#55989) Currently, this uses Bazel runfiles which causes a problem when `run_release_test` is called as a binary with Bazel, some files in the working directory not included in Bazel binary data don't get packaged into the zip file when submitting as Anyscale job. This switches to use a path with Bazel workspace directory which points to the source code and doesn't have issues of missing files in the zip file. --------- Signed-off-by: kevin --- release/BUILD.bazel | 14 ++++++ release/ray_release/template.py | 24 ++++++++-- release/ray_release/tests/test_template.py | 56 ++++++++++++++++++++++ 3 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 release/ray_release/tests/test_template.py diff --git a/release/BUILD.bazel b/release/BUILD.bazel index a726911feac2..374c5ac8a121 100644 --- a/release/BUILD.bazel +++ b/release/BUILD.bazel @@ -692,6 +692,20 @@ py_test( ], ) +py_test( + name = "test_template", + srcs = ["ray_release/tests/test_template.py"], + exec_compatible_with = ["//:hermetic_python"], + tags = [ + "release_unit", + "team:ci", + ], + deps = [ + ":ray_release", + bk_require("pytest"), + ], +) + py_binary( name = "build_pipeline", srcs = ["ray_release/scripts/build_pipeline.py"], diff --git a/release/ray_release/template.py b/release/ray_release/template.py index da060da25f93..31cca890db37 100644 --- a/release/ray_release/template.py +++ b/release/ray_release/template.py @@ -25,6 +25,7 @@ class TestEnvironment(dict): _test_env = None +_bazel_workspace_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", "") def get_test_environment(): @@ -72,13 +73,28 @@ def render_yaml_template(template: str, env: Optional[Dict] = None): ) from e -def get_working_dir(test: "Test", test_definition_root: Optional[str] = None) -> str: +def get_working_dir( + test: "Test", + test_definition_root: Optional[str] = None, + bazel_workspace_dir: Optional[str] = None, +) -> str: + if not bazel_workspace_dir: + bazel_workspace_dir = _bazel_workspace_dir + if bazel_workspace_dir and test_definition_root: + raise ReleaseTestConfigError( + "test_definition_root should not be specified when running with Bazel." + ) working_dir = test.get("working_dir", "") - if working_dir.startswith("//"): - return bazel_runfile(working_dir.lstrip("//")) if test_definition_root: return os.path.join(test_definition_root, working_dir) - return bazel_runfile("release", working_dir) + if working_dir.startswith("//"): + working_dir = working_dir.lstrip("//") + else: + working_dir = os.path.join("release", working_dir) + if bazel_workspace_dir: + return os.path.join(bazel_workspace_dir, working_dir) + else: + return bazel_runfile(working_dir) def load_test_cluster_compute( diff --git a/release/ray_release/tests/test_template.py b/release/ray_release/tests/test_template.py new file mode 100644 index 000000000000..4f5ef46d6426 --- /dev/null +++ b/release/ray_release/tests/test_template.py @@ -0,0 +1,56 @@ +import sys + +import pytest + +from ray_release.test import Test +from ray_release.template import get_working_dir, bazel_runfile +from ray_release.exception import ReleaseTestConfigError + + +def test_get_working_dir_with_path_from_root(): + test_with_path_from_root = Test( + { + "name": "test", + "working_dir": "//ray_testing/ray_release/tests", + } + ) + assert ( + get_working_dir(test_with_path_from_root, None, "/tmp/bazel_workspace") + == "/tmp/bazel_workspace/ray_testing/ray_release/tests" + ) + assert get_working_dir(test_with_path_from_root, None, None) == bazel_runfile( + "ray_testing/ray_release/tests" + ) + + +def test_get_working_dir_with_relative_path(): + test_with_relative_path = Test( + { + "name": "test", + "working_dir": "ray_release/tests", + } + ) + assert ( + get_working_dir(test_with_relative_path, None, "/tmp/bazel_workspace") + == "/tmp/bazel_workspace/release/ray_release/tests" + ) + assert get_working_dir(test_with_relative_path, None, None) == bazel_runfile( + "release/ray_release/tests" + ) + + +def test_get_working_dir_fail(): + test_with_path_from_root = Test( + { + "name": "test", + "working_dir": "//ray_testing/ray_release/tests", + } + ) + with pytest.raises(ReleaseTestConfigError): + get_working_dir( + test_with_path_from_root, "/tmp/test_definition_root", "tmp/bazel_workspace" + ) + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", __file__])) From f4a5de2530644b924d39bda0a095431809bf88d2 Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Tue, 2 Sep 2025 19:34:22 -0700 Subject: [PATCH 409/634] [core] Making ReturnWorkerLease Idempotent (#56073) Making ReturnWorkerLease RPC idempotent and fault tolerant. Added corresponding cpp + python integration tests. This solves the issue mentioned in #55469 as we now use leaseID and not workerID to track granted leases on the raylet side. Hence, the retry for ReturnWorkerLease will not cause a pre-emptive return of an ongoing lease on the same worker since the lease ids for the retry vs current lease request will contain different lease IDs, thus the retry can just be discarded. Signed-off-by: joshlee --- src/fakes/ray/rpc/raylet/raylet_client.h | 11 ++- src/mock/ray/raylet_client/raylet_client.h | 4 +- .../task_submission/normal_task_submitter.cc | 11 +-- .../tests/normal_task_submitter_test.cc | 11 ++- .../gcs_server/tests/gcs_server_test_util.h | 11 ++- src/ray/protobuf/node_manager.proto | 8 +-- src/ray/raylet/local_lease_manager.cc | 8 +-- src/ray/raylet/local_lease_manager.h | 8 +-- src/ray/raylet/main.cc | 2 +- src/ray/raylet/node_manager.cc | 67 ++++++++++--------- src/ray/raylet/node_manager.h | 20 +++--- .../tests/cluster_lease_manager_test.cc | 6 +- .../raylet/tests/local_lease_manager_test.cc | 2 +- src/ray/raylet/tests/node_manager_test.cc | 53 ++++++++++++++- src/ray/raylet/tests/util.h | 5 +- src/ray/raylet_client/node_manager_client.h | 9 +-- src/ray/raylet_client/raylet_client.cc | 15 ++--- src/ray/raylet_client/raylet_client.h | 10 +-- .../raylet_client/raylet_client_interface.h | 13 ++-- 19 files changed, 157 insertions(+), 117 deletions(-) diff --git a/src/fakes/ray/rpc/raylet/raylet_client.h b/src/fakes/ray/rpc/raylet/raylet_client.h index fa4ef1436b6b..044351f595f4 100644 --- a/src/fakes/ray/rpc/raylet/raylet_client.h +++ b/src/fakes/ray/rpc/raylet/raylet_client.h @@ -37,17 +37,16 @@ class FakeRayletClient : public RayletClientInterface { callbacks.push_back(callback); } - ray::Status ReturnWorkerLease(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) override { + void ReturnWorkerLease(int worker_port, + const LeaseID &lease_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) override { if (disconnect_worker) { num_workers_disconnected++; } else { num_workers_returned++; } - return Status::OK(); } void PrestartWorkers( diff --git a/src/mock/ray/raylet_client/raylet_client.h b/src/mock/ray/raylet_client/raylet_client.h index dab5fb18dc33..9a2c2d06b8b9 100644 --- a/src/mock/ray/raylet_client/raylet_client.h +++ b/src/mock/ray/raylet_client/raylet_client.h @@ -31,10 +31,10 @@ class MockRayletClientInterface : public RayletClientInterface { const int64_t backlog_size, const bool is_selected_based_on_locality), (override)); - MOCK_METHOD(ray::Status, + MOCK_METHOD(void, ReturnWorkerLease, (int worker_port, - const WorkerID &worker_id, + const LeaseID &lease_id, bool disconnect_worker, const std::string &disconnect_worker_error_detail, bool worker_exiting), diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 275760546c88..0045f66876d2 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -136,15 +136,8 @@ void NormalTaskSubmitter::ReturnWorkerLease(const rpc::Address &addr, scheduling_key_entries_.erase(scheduling_key); } - auto status = - lease_entry.raylet_client->ReturnWorkerLease(addr.port(), - WorkerID::FromBinary(addr.worker_id()), - was_error, - error_detail, - worker_exiting); - if (!status.ok()) { - RAY_LOG(ERROR) << "Error returning worker to raylet: " << status.ToString(); - } + lease_entry.raylet_client->ReturnWorkerLease( + addr.port(), lease_entry.lease_id, was_error, error_detail, worker_exiting); worker_to_lease_entry_.erase(addr); } diff --git a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc index 707770c8b2a8..97d8b6bd48fa 100644 --- a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc @@ -223,11 +223,11 @@ class MockTaskManager : public MockTaskManagerInterface { class MockRayletClient : public FakeRayletClient { public: - Status ReturnWorkerLease(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) override { + void ReturnWorkerLease(int worker_port, + const LeaseID &lease_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) override { std::lock_guard lock(mu_); if (disconnect_worker) { num_workers_disconnected++; @@ -237,7 +237,6 @@ class MockRayletClient : public FakeRayletClient { num_workers_returned_exiting++; } } - return Status::OK(); } void GetWorkerFailureCause( diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h index b90f7b6aefab..b52445c5c4f5 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h +++ b/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h @@ -77,17 +77,16 @@ struct GcsServerMocker { class MockRayletClient : public FakeRayletClient { public: - ray::Status ReturnWorkerLease(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) override { + void ReturnWorkerLease(int worker_port, + const LeaseID &lease_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) override { if (disconnect_worker) { num_workers_disconnected++; } else { num_workers_returned++; } - return Status::OK(); } void GetWorkerFailureCause( diff --git a/src/ray/protobuf/node_manager.proto b/src/ray/protobuf/node_manager.proto index 624edac21496..b8e2f9f1c0a9 100644 --- a/src/ray/protobuf/node_manager.proto +++ b/src/ray/protobuf/node_manager.proto @@ -147,8 +147,8 @@ message ResizeLocalResourceInstancesReply { message ReturnWorkerLeaseRequest { // Port of the leased worker that we are now returning. int32 worker_port = 1; - // The worker id of the lease we are now returning. - bytes worker_id = 2; + // The lease id of the lease we are now returning. + bytes lease_id = 2; // If true, there was some unrecoverable error and the raylet should // disconnect the worker. bool disconnect_worker = 3; @@ -428,8 +428,8 @@ service NodeManagerService { // Failure: Doesn't need to be retried since it will keep getting periodically called, // and is not critical. rpc ReportWorkerBacklog(ReportWorkerBacklogRequest) returns (ReportWorkerBacklogReply); - // Release a worker back to its raylet. - // Failure: TODO: Failure behavior needs to be fixed. + // Return a worker lease back to its raylet. + // Failure: Retries, it's idempotent. rpc ReturnWorkerLease(ReturnWorkerLeaseRequest) returns (ReturnWorkerLeaseReply); // This method is only used by GCS, and the purpose is to release leased workers // that may be leaked. When GCS restarts, it doesn't know which workers it has leased diff --git a/src/ray/raylet/local_lease_manager.cc b/src/ray/raylet/local_lease_manager.cc index f1d516d3a642..7a497a04993f 100644 --- a/src/ray/raylet/local_lease_manager.cc +++ b/src/ray/raylet/local_lease_manager.cc @@ -37,7 +37,7 @@ LocalLeaseManager::LocalLeaseManager( LeaseDependencyManagerInterface &lease_dependency_manager, internal::NodeInfoGetter get_node_info, WorkerPoolInterface &worker_pool, - absl::flat_hash_map> &leased_workers, + absl::flat_hash_map> &leased_workers, std::function &object_ids, std::vector> *results)> get_lease_arguments, @@ -957,7 +957,7 @@ const RayLease *LocalLeaseManager::AnyPendingLeasesForResourceAcquisition( void LocalLeaseManager::Grant( std::shared_ptr worker, - absl::flat_hash_map> &leased_workers, + absl::flat_hash_map> &leased_workers, const std::shared_ptr &allocated_instances, const RayLease &lease, rpc::RequestWorkerLeaseReply *reply, @@ -979,8 +979,8 @@ void LocalLeaseManager::Grant( reply->mutable_worker_address()->set_worker_id(worker->WorkerId().Binary()); reply->mutable_worker_address()->set_node_id(self_node_id_.Binary()); - RAY_CHECK(leased_workers.find(worker->WorkerId()) == leased_workers.end()); - leased_workers[worker->WorkerId()] = worker; + RAY_CHECK(!leased_workers.contains(lease_spec.LeaseId())); + leased_workers[lease_spec.LeaseId()] = worker; cluster_resource_scheduler_.GetLocalResourceManager().SetBusyFootprint( WorkFootprint::NODE_WORKERS); diff --git a/src/ray/raylet/local_lease_manager.h b/src/ray/raylet/local_lease_manager.h index 65611c2c338e..595ce7d00786 100644 --- a/src/ray/raylet/local_lease_manager.h +++ b/src/ray/raylet/local_lease_manager.h @@ -81,7 +81,7 @@ class LocalLeaseManager : public LocalLeaseManagerInterface { LeaseDependencyManagerInterface &lease_dependency_manager, internal::NodeInfoGetter get_node_info, WorkerPoolInterface &worker_pool, - absl::flat_hash_map> &leased_workers, + absl::flat_hash_map> &leased_workers, std::function &object_ids, std::vector> *results)> get_lease_arguments, @@ -246,7 +246,7 @@ class LocalLeaseManager : public LocalLeaseManagerInterface { void Grant( std::shared_ptr worker, - absl::flat_hash_map> &leased_workers_, + absl::flat_hash_map> &leased_workers_, const std::shared_ptr &allocated_instances, const RayLease &lease, rpc::RequestWorkerLeaseReply *reply, @@ -341,10 +341,8 @@ class LocalLeaseManager : public LocalLeaseManagerInterface { absl::flat_hash_map> backlog_tracker_; - /// TODO(Shanly): Remove `worker_pool_` and `leased_workers_` and make them as - /// parameters of methods if necessary once we remove the legacy scheduler. WorkerPoolInterface &worker_pool_; - absl::flat_hash_map> &leased_workers_; + absl::flat_hash_map> &leased_workers_; /// Callback to get references to lease arguments. These will be pinned while /// the lease is granted. diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index d2a59645651e..85f29663e916 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -292,7 +292,7 @@ int main(int argc, char *argv[]) { /// The client to export metrics to the metrics agent. std::unique_ptr metrics_agent_client; /// Map of workers leased out to clients. - absl::flat_hash_map> + absl::flat_hash_map> leased_workers; // Enable subreaper. This is called in `AsyncGetInternalConfig` below, but MSVC does diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 1ce39809764a..83585d046e38 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -109,7 +109,7 @@ NodeManager::NodeManager( LocalObjectManagerInterface &local_object_manager, LeaseDependencyManager &lease_dependency_manager, WorkerPoolInterface &worker_pool, - absl::flat_hash_map> &leased_workers, + absl::flat_hash_map> &leased_workers, plasma::PlasmaClientInterface &store_client, std::unique_ptr mutable_object_provider, @@ -1307,7 +1307,9 @@ void NodeManager::DisconnectClient(const std::shared_ptr &clie lease_dependency_manager_.CancelWaitRequest(worker->WorkerId()); // Erase any lease metadata. - ReleaseWorker(worker->WorkerId()); + if (leased_workers_.contains(worker->GetGrantedLeaseId())) { + ReleaseWorker(worker->GetGrantedLeaseId()); + } if (creation_task_exception != nullptr) { RAY_LOG(INFO).WithField(worker->WorkerId()) @@ -1911,38 +1913,41 @@ void NodeManager::HandleReturnWorkerLease(rpc::ReturnWorkerLeaseRequest request, rpc::ReturnWorkerLeaseReply *reply, rpc::SendReplyCallback send_reply_callback) { // Read the resource spec submitted by the client. - auto worker_id = WorkerID::FromBinary(request.worker_id()); - std::shared_ptr worker = leased_workers_[worker_id]; + auto lease_id = LeaseID::FromBinary(request.lease_id()); - Status status; - ReleaseWorker(worker_id); - if (worker) { - if (request.disconnect_worker()) { - // The worker should be destroyed. - DisconnectClient( - worker->Connection(), - /*graceful=*/false, - rpc::WorkerExitType::SYSTEM_ERROR, - absl::StrCat("The leased worker has unrecoverable failure. Worker is requested " - "to be destroyed when it is returned. ", - request.disconnect_worker_error_detail())); - } else { - if (worker->IsBlocked()) { - // Handle the edge case where the worker was returned before we got the - // unblock RPC by unblocking it immediately (unblock is idempotent). - HandleDirectCallTaskUnblocked(worker); - } - local_lease_manager_.ReleaseWorkerResources(worker); - // If the worker is exiting, don't add it to our pool. The worker will cleanup - // and terminate itself. - if (!request.worker_exiting()) { - HandleWorkerAvailable(worker); - } - } + // Check if this message is a retry + if (!leased_workers_.contains(lease_id)) { + send_reply_callback(Status::OK(), nullptr, nullptr); + return; + } + + std::shared_ptr worker = leased_workers_[lease_id]; + ReleaseWorker(lease_id); + + if (request.disconnect_worker()) { + // The worker should be destroyed. + DisconnectClient( + worker->Connection(), + /*graceful=*/false, + rpc::WorkerExitType::SYSTEM_ERROR, + absl::StrCat("The leased worker has unrecoverable failure. Worker is requested " + "to be destroyed when it is returned. ", + request.disconnect_worker_error_detail())); } else { - status = Status::Invalid("Returned worker does not exist any more"); + if (worker->IsBlocked()) { + // Handle the edge case where the worker was returned before we got the + // unblock RPC by unblocking it immediately (unblock is idempotent). + HandleDirectCallTaskUnblocked(worker); + } + local_lease_manager_.ReleaseWorkerResources(worker); + // If the worker is exiting, don't add it to our pool. The worker will cleanup + // and terminate itself. + if (!request.worker_exiting()) { + HandleWorkerAvailable(worker); + } } - send_reply_callback(status, nullptr, nullptr); + + send_reply_callback(Status::OK(), nullptr, nullptr); } void NodeManager::HandleIsLocalWorkerDead(rpc::IsLocalWorkerDeadRequest request, diff --git a/src/ray/raylet/node_manager.h b/src/ray/raylet/node_manager.h index 4feee4e3fabf..f9865e49735a 100644 --- a/src/ray/raylet/node_manager.h +++ b/src/ray/raylet/node_manager.h @@ -147,7 +147,7 @@ class NodeManager : public rpc::NodeManagerServiceHandler, LocalObjectManagerInterface &local_object_manager, LeaseDependencyManager &lease_dependency_manager, WorkerPoolInterface &worker_pool, - absl::flat_hash_map> &leased_workers, + absl::flat_hash_map> &leased_workers, plasma::PlasmaClientInterface &store_client, std::unique_ptr mutable_object_provider, @@ -282,6 +282,10 @@ class NodeManager : public rpc::NodeManagerServiceHandler, rpc::ResizeLocalResourceInstancesReply *reply, rpc::SendReplyCallback send_reply_callback) override; + void HandleReturnWorkerLease(rpc::ReturnWorkerLeaseRequest request, + rpc::ReturnWorkerLeaseReply *reply, + rpc::SendReplyCallback send_reply_callback) override; + private: FRIEND_TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog); @@ -289,8 +293,9 @@ class NodeManager : public rpc::NodeManagerServiceHandler, // Warning: this does NOT release the worker's resources, or put the leased worker // back to the worker pool, or destroy the worker. The caller must handle the worker's // resources well. - void ReleaseWorker(const WorkerID &worker_id) { - leased_workers_.erase(worker_id); + void ReleaseWorker(const LeaseID &lease_id) { + RAY_CHECK(leased_workers_.contains(lease_id)); + leased_workers_.erase(lease_id); SetIdleIfLeaseEmpty(); } @@ -554,11 +559,6 @@ class NodeManager : public rpc::NodeManagerServiceHandler, WorkerPoolInterface &worker_pool, LocalLeaseManagerInterface &local_lease_manager); - /// Handle a `ReturnWorkerLease` request. - void HandleReturnWorkerLease(rpc::ReturnWorkerLeaseRequest request, - rpc::ReturnWorkerLeaseReply *reply, - rpc::SendReplyCallback send_reply_callback) override; - /// Handle a `ReleaseUnusedActorWorkers` request. // On GCS restart, there's a pruning effort. GCS sends raylet a list of actor workers it // still wants (that it keeps tracks of); and the raylet destroys all other actor @@ -792,8 +792,8 @@ class NodeManager : public rpc::NodeManagerServiceHandler, absl::flat_hash_map> remote_node_manager_addresses_; - /// Map of workers to their worker ids. - absl::flat_hash_map> &leased_workers_; + /// Map of leased workers to their lease ids. + absl::flat_hash_map> &leased_workers_; /// Optional extra information about why the worker failed. absl::flat_hash_map worker_failure_reasons_; diff --git a/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc b/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc index 3347785c7407..40eeee6f1cf5 100644 --- a/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc +++ b/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc @@ -505,7 +505,7 @@ class ClusterLeaseManagerTest : public ::testing::Test { NodeID id_; std::shared_ptr scheduler_; MockWorkerPool pool_; - absl::flat_hash_map> leased_workers_; + absl::flat_hash_map> leased_workers_; std::unordered_set missing_objects_; int default_arg_size_ = 10; @@ -762,8 +762,8 @@ TEST_F(ClusterLeaseManagerTest, BlockedWorkerDiesTest) { RayLease finished_lease1; RayLease finished_lease2; // If a resource was double-freed, we will crash in this call. - local_lease_manager_->CleanupLease(leased_workers_[worker_id1], &finished_lease1); - local_lease_manager_->CleanupLease(leased_workers_[worker_id2], &finished_lease2); + local_lease_manager_->CleanupLease(leased_workers_[lease_id1], &finished_lease1); + local_lease_manager_->CleanupLease(leased_workers_[lease_id2], &finished_lease2); ASSERT_EQ(finished_lease1.GetLeaseSpecification().LeaseId(), lease1.GetLeaseSpecification().LeaseId()); ASSERT_EQ(finished_lease2.GetLeaseSpecification().LeaseId(), diff --git a/src/ray/raylet/tests/local_lease_manager_test.cc b/src/ray/raylet/tests/local_lease_manager_test.cc index 0d4a829785f0..faae316bfba9 100644 --- a/src/ray/raylet/tests/local_lease_manager_test.cc +++ b/src/ray/raylet/tests/local_lease_manager_test.cc @@ -364,7 +364,7 @@ class LocalLeaseManagerTest : public ::testing::Test { NodeID id_; std::shared_ptr scheduler_; MockWorkerPool pool_; - absl::flat_hash_map> leased_workers_; + absl::flat_hash_map> leased_workers_; std::unordered_set missing_objects_; int default_arg_size_ = 10; diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index e33b9fdfb796..2a35f075c423 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -525,7 +525,7 @@ class NodeManagerTest : public ::testing::Test { std::unique_ptr node_manager_; MockWorkerPool mock_worker_pool_; - absl::flat_hash_map> leased_workers_; + absl::flat_hash_map> leased_workers_; std::shared_ptr> objects_pending_deletion_; }; @@ -948,6 +948,57 @@ TEST_F(NodeManagerTest, TestResizeLocalResourceInstancesClamps) { EXPECT_EQ(reply.total_resources().at("CPU"), 6.0); } +class NodeManagerReturnWorkerLeaseIdempotentTest + : public NodeManagerTest, + public testing::WithParamInterface> {}; + +TEST_P(NodeManagerReturnWorkerLeaseIdempotentTest, TestDifferentRequestArgs) { + const auto ¶ms = GetParam(); + bool disconnect_worker = std::get<0>(params); + bool worker_exiting = std::get<1>(params); + + LeaseID lease_id = LeaseID::FromRandom(); + leased_workers_[lease_id] = std::make_shared(WorkerID::FromRandom(), 10); + rpc::ReturnWorkerLeaseRequest request; + rpc::ReturnWorkerLeaseReply reply1; + rpc::ReturnWorkerLeaseReply reply2; + request.set_lease_id(lease_id.Binary()); + request.set_disconnect_worker(disconnect_worker); + request.set_disconnect_worker_error_detail("test"); + request.set_worker_exiting(worker_exiting); + + if (disconnect_worker) { + EXPECT_CALL( + mock_worker_pool_, + GetRegisteredWorker(testing::A &>())) + .Times(1) + .WillOnce(Return(nullptr)); + EXPECT_CALL( + mock_worker_pool_, + GetRegisteredDriver(testing::A &>())) + .Times(1) + .WillOnce(Return(nullptr)); + } + node_manager_->HandleReturnWorkerLease( + request, + &reply1, + [](Status s, std::function success, std::function failure) { + ASSERT_TRUE(s.ok()); + }); + ASSERT_EQ(leased_workers_.size(), 0); + node_manager_->HandleReturnWorkerLease( + request, + &reply2, + [](Status s, std::function success, std::function failure) { + ASSERT_TRUE(s.ok()); + }); + ASSERT_EQ(leased_workers_.size(), 0); +} + +INSTANTIATE_TEST_SUITE_P(NodeManagerReturnWorkerLeaseIdempotentVariations, + NodeManagerReturnWorkerLeaseIdempotentTest, + testing::Combine(testing::Bool(), testing::Bool())); + } // namespace ray::raylet int main(int argc, char **argv) { diff --git a/src/ray/raylet/tests/util.h b/src/ray/raylet/tests/util.h index 7f1372bb5ef7..467ff373bbc9 100644 --- a/src/ray/raylet/tests/util.h +++ b/src/ray/raylet/tests/util.h @@ -137,10 +137,7 @@ class MockWorker : public WorkerInterface { return lease_.GetLeaseSpecification().IsDetachedActor(); } - const std::shared_ptr Connection() const override { - RAY_CHECK(false) << "Method unused"; - return nullptr; - } + const std::shared_ptr Connection() const override { return nullptr; } const rpc::Address &GetOwnerAddress() const override { return address_; } void ActorCallArgWaitComplete(int64_t tag) override { diff --git a/src/ray/raylet_client/node_manager_client.h b/src/ray/raylet_client/node_manager_client.h index 7d8d05df8232..6887a4a55ad8 100644 --- a/src/ray/raylet_client/node_manager_client.h +++ b/src/ray/raylet_client/node_manager_client.h @@ -105,10 +105,11 @@ class NodeManagerClient { grpc_client_, /*method_timeout_ms*/ -1, ) - VOID_RPC_CLIENT_METHOD(NodeManagerService, - ReturnWorkerLease, - grpc_client_, - /*method_timeout_ms*/ -1, ) + VOID_RETRYABLE_RPC_CLIENT_METHOD(retryable_grpc_client_, + NodeManagerService, + ReturnWorkerLease, + grpc_client_, + /*method_timeout_ms*/ -1, ) VOID_RPC_CLIENT_METHOD(NodeManagerService, ReleaseUnusedActorWorkers, diff --git a/src/ray/raylet_client/raylet_client.cc b/src/ray/raylet_client/raylet_client.cc index 87d91f089ee8..79c5496f4794 100644 --- a/src/ray/raylet_client/raylet_client.cc +++ b/src/ray/raylet_client/raylet_client.cc @@ -80,22 +80,21 @@ void RayletClient::ReportWorkerBacklog( }); } -Status RayletClient::ReturnWorkerLease(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) { +void RayletClient::ReturnWorkerLease(int worker_port, + const LeaseID &lease_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) { rpc::ReturnWorkerLeaseRequest request; request.set_worker_port(worker_port); - request.set_worker_id(worker_id.Binary()); + request.set_lease_id(lease_id.Binary()); request.set_disconnect_worker(disconnect_worker); request.set_disconnect_worker_error_detail(disconnect_worker_error_detail); request.set_worker_exiting(worker_exiting); grpc_client_->ReturnWorkerLease( - request, [](const Status &status, rpc::ReturnWorkerLeaseReply &&reply /*unused*/) { + std::move(request), [](const Status &status, rpc::ReturnWorkerLeaseReply &&) { RAY_LOG_IF_ERROR(INFO, status) << "Error returning worker: " << status; }); - return Status::OK(); } void RayletClient::GetWorkerFailureCause( diff --git a/src/ray/raylet_client/raylet_client.h b/src/ray/raylet_client/raylet_client.h index c2c9c6263d7b..5c7eedcfa194 100644 --- a/src/ray/raylet_client/raylet_client.h +++ b/src/ray/raylet_client/raylet_client.h @@ -58,11 +58,11 @@ class RayletClient : public RayletClientInterface { const int64_t backlog_size, const bool is_selected_based_on_locality) override; - ray::Status ReturnWorkerLease(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) override; + void ReturnWorkerLease(int worker_port, + const LeaseID &lease_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) override; void PrestartWorkers( const ray::rpc::PrestartWorkersRequest &request, diff --git a/src/ray/raylet_client/raylet_client_interface.h b/src/ray/raylet_client/raylet_client_interface.h index 621acb65c415..3480f86be848 100644 --- a/src/ray/raylet_client/raylet_client_interface.h +++ b/src/ray/raylet_client/raylet_client_interface.h @@ -62,15 +62,14 @@ class RayletClientInterface { /// Returns a worker to the raylet. /// \param worker_port The local port of the worker on the raylet node. - /// \param worker_id The unique worker id of the worker on the raylet node. + /// \param lease_id The unique lease id of the worker on the raylet node. /// \param disconnect_worker Whether the raylet should disconnect the worker. /// \param worker_exiting Whether the worker is exiting and cannot be reused. - /// \return ray::Status - virtual ray::Status ReturnWorkerLease(int worker_port, - const WorkerID &worker_id, - bool disconnect_worker, - const std::string &disconnect_worker_error_detail, - bool worker_exiting) = 0; + virtual void ReturnWorkerLease(int worker_port, + const LeaseID &lease_id, + bool disconnect_worker, + const std::string &disconnect_worker_error_detail, + bool worker_exiting) = 0; /// Request the raylet to prestart workers. In `request` we can set the worker's owner, /// runtime env info and number of workers. From 6cb3e6034ef2c25ea66c04cc920df0f5d4fe7380 Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Tue, 2 Sep 2025 21:44:16 -0700 Subject: [PATCH 410/634] [core] Remove _ID_TYPES (#56184) ## Why are these changes needed? It is not used any more. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Philipp Moritz --- python/ray/includes/unique_ids.pxi | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/python/ray/includes/unique_ids.pxi b/python/ray/includes/unique_ids.pxi index 9e2adad94825..3c387833dc29 100644 --- a/python/ray/includes/unique_ids.pxi +++ b/python/ray/includes/unique_ids.pxi @@ -4,9 +4,6 @@ We define different types for different IDs for type safety. See https://github.com/ray-project/ray/issues/3721. """ -# WARNING: Any additional ID types defined in this file must be added to the -# _ID_TYPES list at the bottom of this file. - import logging import os @@ -430,17 +427,3 @@ cdef class PlacementGroupID(BaseID): cdef size_t hash(self): return self.data.Hash() - -_ID_TYPES = [ - ActorClassID, - ActorID, - NodeID, - JobID, - WorkerID, - FunctionID, - ObjectID, - TaskID, - UniqueID, - PlacementGroupID, - ClusterID, -] From bd684c7b52ba592b637591be10113a2c3238eefb Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Wed, 3 Sep 2025 10:27:30 +0530 Subject: [PATCH 411/634] [core] Fix flaky `ShutdownCoordinator` test under tsan (#56152) Signed-off-by: Sagar Sumit --- src/ray/core_worker/tests/BUILD.bazel | 1 + .../tests/shutdown_coordinator_test.cc | 50 +++++++++++++++---- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index c6f99e1103d9..de46fb466ffa 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -18,6 +18,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/ray/core_worker:shutdown_coordinator", + "@com_google_absl//absl/synchronization", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/core_worker/tests/shutdown_coordinator_test.cc b/src/ray/core_worker/tests/shutdown_coordinator_test.cc index 8752a8d8c966..9d18f8f7030d 100644 --- a/src/ray/core_worker/tests/shutdown_coordinator_test.cc +++ b/src/ray/core_worker/tests/shutdown_coordinator_test.cc @@ -24,6 +24,7 @@ #include #include +#include "absl/synchronization/mutex.h" #include "ray/common/buffer.h" #include "src/ray/protobuf/common.pb.h" @@ -41,26 +42,46 @@ class FakeShutdownExecutor : public ShutdownExecutorInterface { std::string last_exit_type; std::string last_detail; + mutable absl::Mutex mu_; + + std::string GetLastExitType() const { + absl::MutexLock lk(&mu_); + return last_exit_type; + } + + std::string GetLastDetail() const { + absl::MutexLock lk(&mu_); + return last_detail; + } void ExecuteGracefulShutdown(std::string_view exit_type, std::string_view detail, std::chrono::milliseconds timeout_ms) override { graceful_calls++; - last_exit_type = std::string(exit_type); - last_detail = std::string(detail); + { + absl::MutexLock lk(&mu_); + last_exit_type = std::string(exit_type); + last_detail = std::string(detail); + } } void ExecuteForceShutdown(std::string_view exit_type, std::string_view detail) override { force_calls++; - last_exit_type = std::string(exit_type); - last_detail = std::string(detail); + { + absl::MutexLock lk(&mu_); + last_exit_type = std::string(exit_type); + last_detail = std::string(detail); + } } void ExecuteWorkerExit(std::string_view exit_type, std::string_view detail, std::chrono::milliseconds timeout_ms) override { worker_exit_calls++; - last_exit_type = std::string(exit_type); - last_detail = std::string(detail); + { + absl::MutexLock lk(&mu_); + last_exit_type = std::string(exit_type); + last_detail = std::string(detail); + } } void ExecuteExit(std::string_view exit_type, std::string_view detail, @@ -68,15 +89,21 @@ class FakeShutdownExecutor : public ShutdownExecutorInterface { const std::shared_ptr<::ray::LocalMemoryBuffer> &creation_task_exception_pb_bytes) override { worker_exit_calls++; - last_exit_type = std::string(exit_type); - last_detail = std::string(detail); + { + absl::MutexLock lk(&mu_); + last_exit_type = std::string(exit_type); + last_detail = std::string(detail); + } } void ExecuteHandleExit(std::string_view exit_type, std::string_view detail, std::chrono::milliseconds timeout_ms) override { handle_exit_calls++; - last_exit_type = std::string(exit_type); - last_detail = std::string(detail); + { + absl::MutexLock lk(&mu_); + last_exit_type = std::string(exit_type); + last_detail = std::string(detail); + } } void KillChildProcessesImmediately() override {} bool ShouldWorkerIdleExit() const override { return idle_exit_allowed.load(); } @@ -366,7 +393,8 @@ TEST_F(ShutdownCoordinatorTest, Concurrent_DoubleForce_ForceExecutesOnce) { // Verify that only one forced shutdown was called EXPECT_EQ(fake_ptr->force_calls.load(), 1); EXPECT_EQ(fake_ptr->graceful_calls.load(), 0); - EXPECT_TRUE(fake_ptr->last_detail == "force1" || fake_ptr->last_detail == "force2"); + EXPECT_TRUE(fake_ptr->GetLastDetail() == "force1" || + fake_ptr->GetLastDetail() == "force2"); } } // namespace core From c56394682d1d72d8e4a2803c60bfca51668a782e Mon Sep 17 00:00:00 2001 From: Potato Date: Wed, 3 Sep 2025 13:59:15 +0800 Subject: [PATCH 412/634] [DOC] Fix documentation errors in ray-references and ray-security directories (#56128) Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: angelinalg <122562471+angelinalg@users.noreply.github.com> --- doc/source/ray-references/faq.rst | 2 +- doc/source/ray-references/glossary.rst | 16 ++++++++-------- doc/source/ray-security/index.md | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/source/ray-references/faq.rst b/doc/source/ray-references/faq.rst index 9fbf54fc9c60..9b18ef07bded 100644 --- a/doc/source/ray-references/faq.rst +++ b/doc/source/ray-references/faq.rst @@ -11,6 +11,6 @@ FAQ Further Questions or Issues? ----------------------------- +----------------------------- .. include:: /_includes/_help.rst diff --git a/doc/source/ray-references/glossary.rst b/doc/source/ray-references/glossary.rst index 265efc260c9b..d43fdeef19af 100644 --- a/doc/source/ray-references/glossary.rst +++ b/doc/source/ray-references/glossary.rst @@ -23,7 +23,7 @@ documentation, sorted alphabetically. essentially a stateful service. :ref:`Learn more about Ray actors`. Actor task - An invocation of an Ray actor method. Sometimes we just call it a task. + An invocation of a Ray actor method. Sometimes we just call it a task. Ray Agent Daemon process running on each Ray node. It has several functionalities like @@ -38,7 +38,7 @@ documentation, sorted alphabetically. Algorithm A class that holds the who/when/where/how for training one or more RL agent(s). The user interacts with an Algorithm instance directly to train their agents - (it is the top-most user facing API or RLlib). + (it is the top-most user facing API of RLlib). Asynchronous execution An execution model where a later task can begin executing in parallel, @@ -66,7 +66,7 @@ documentation, sorted alphabetically. Backend A class containing the initialization and teardown logic for a specific deep - learning framework (eg. Torch, TensorFlow), used to set up distributed + learning framework (e.g., Torch, TensorFlow), used to set up distributed data-parallel training for :ref:`Ray Train’s built-in trainers`. Batch format @@ -116,7 +116,7 @@ documentation, sorted alphabetically. different Ray components and libraries. A Checkpoint can have its data represented as a directory on local (on-disk) storage, as a directory on an external storage (e.g., cloud storage), and as an in-memory dictionary. - :class:`Learn more `, + :class:`Learn more `. .. TODO: How does this relate to RLlib checkpoints etc.? Be clear here @@ -197,7 +197,7 @@ documentation, sorted alphabetically. Environment The world or simulation, in which one or more reinforcement learning agents - have to learn to behave optimally in wrt. a given reward function. An + have to learn to behave optimally with respect to a given reward function. An environment consists of an observation space, a reward function, an action space, a state transition function, and a distribution over initial states (after a reset). @@ -219,7 +219,7 @@ documentation, sorted alphabetically. Trial Executor An internal :ref:`Ray Tune component` that manages the resource management and execution of each trial’s corresponding remote - Trainable actor. The trial executor’s responsibilities include launching + Trainable actor. The trial executor’s responsibilities include launching training, checkpointing, and restoring remote tasks. Experiment @@ -266,7 +266,7 @@ documentation, sorted alphabetically. .. TODO: Inference Job - A ray job is a packaged ray application that can be executed on a + A Ray job is a packaged Ray application that can be executed on a (remote) Ray cluster. :ref:`Learn more`. Lineage @@ -375,7 +375,7 @@ documentation, sorted alphabetically. On-Policy A type of RL Algorithm. In an on-policy algorithm, the policy used to compute the actions inside an RL environment (to generate the training data) must be the - exact same (matching NN weights at all times) than the one that is being + exact same (matching NN weights at all times) as the one that's being optimized. Examples for on-policy Algorithms are PPO, APPO, and IMPALA. OOM (Out of Memory) diff --git a/doc/source/ray-security/index.md b/doc/source/ray-security/index.md index 8a2d87acedde..f7a4a707e1d4 100644 --- a/doc/source/ray-security/index.md +++ b/doc/source/ray-security/index.md @@ -1,6 +1,6 @@ (security)= -# Security +# Security Ray is an easy-to-use framework to run arbitrary code across one or more nodes in a Ray Cluster. Ray provides fault-tolerance, optimized scheduling, task orchestration, and auto-scaling to run a given workload. @@ -15,7 +15,7 @@ If you expose these services (Ray Dashboard, Ray Jobs, Ray Client), anybody who can access the associated ports can execute arbitrary code on your Ray Cluster. This can happen: * Explicitly: By submitting a Ray Job, or using the Ray Client * Indirectly: By calling the Dashboard REST APIs of these services -* Implicitly: Ray extensively uses cloudpickle for serialization of arbitrary python objects. See [the pickle documentation](https://docs.python.org/3/library/pickle.html) for more details on Pickle's security model. +* Implicitly: Ray extensively uses cloudpickle for serialization of arbitrary Python objects. See [the pickle documentation](https://docs.python.org/3/library/pickle.html) for more details on Pickle's security model. The Ray Dashboard, Ray Jobs and Ray Client are developer tools that you should only use with the necessary access controls in place to restrict access to trusted parties only. From b9df8e62fd6af43162ee2240f03798483e4b7c2e Mon Sep 17 00:00:00 2001 From: Potato Date: Wed, 3 Sep 2025 13:59:43 +0800 Subject: [PATCH 413/634] [DOC] Fix documentation issues in ray-overview directory (#56129) Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../examples/object-detection/README.md | 34 ------------------- doc/source/ray-overview/getting-started.md | 2 +- doc/source/ray-overview/index.md | 12 +++---- doc/source/ray-overview/installation.rst | 2 +- doc/source/ray-overview/use-cases.rst | 2 +- 5 files changed, 9 insertions(+), 43 deletions(-) diff --git a/doc/source/ray-overview/examples/object-detection/README.md b/doc/source/ray-overview/examples/object-detection/README.md index bd4ae7a8b95f..4a37b2d5a8cb 100644 --- a/doc/source/ray-overview/examples/object-detection/README.md +++ b/doc/source/ray-overview/examples/object-detection/README.md @@ -1,39 +1,5 @@ # Scalable video processing - - -This tutorial builds an end-to-end face mask detection pipeline that leverages distributed fine-tuning, large-scale batch inference, video analytics, and scalable serving: - -[1.object_detection_train.ipynb](1.object_detection_train.ipynb) -Fine-tune a pre-trained Faster R-CNN model on a face mask dataset in Pascal Visual Object Classes (VOC) format using Ray Train. Parse XML annotations with Ray Data, retrieve images from S3, run a distributed training loop, checkpoint the model, and visualize inference results. -Object Detection Training Pipeline - -[2.object_detection_batch_inference_eval.ipynb](2.object_detection_batch_inference_eval.ipynb) -Load a fine-tuned model from S3 into Anyscale cluster storage, perform GPU-accelerated batch inference on a test set with Ray Data, and calculate object detection metrics (mAP, IoU, recall) using TorchMetrics for comprehensive model evaluation. -Metrics Calculation Pipeline - -[3.video_processing_batch_inference.ipynb](3.video_processing_batch_inference.ipynb) -Demonstrate a real-world video analytics workflow: read a video from S3, split it into frames, apply the detection model in parallel using Ray Data batch inference, draw bounding boxes and labels on each frame, and regenerate an annotated video for downstream consumption. -Video Processing Pipeline - -[4.object_detection_serve.ipynb](4.object_detection_serve.ipynb) -Deploy the trained Faster R-CNN mask detector as a production-ready microservice using Ray Serve and FastAPI. Set up ingress, configure autoscaling and fractional GPU allocation, test the HTTP endpoint, and manage the service lifecycle both locally and through Anyscale Services. - - -# Face mask detection pipeline - This tutorial builds an end-to-end face mask detection pipeline that leverages distributed fine-tuning, large-scale batch inference, video analytics, and scalable serving: [1.object_detection_train.ipynb](1.object_detection_train.ipynb) diff --git a/doc/source/ray-overview/getting-started.md b/doc/source/ray-overview/getting-started.md index 958134f20619..785c7976393a 100644 --- a/doc/source/ray-overview/getting-started.md +++ b/doc/source/ray-overview/getting-started.md @@ -34,7 +34,7 @@ Use individual libraries for ML workloads. Each library specializes in a specifi [Ray Data](data_quickstart) provides distributed data processing optimized for machine learning and AI workloads. It efficiently streams data through data pipelines. -Here's an example on how to scale offline inference and training ingest with Ray Data. +Here's an example of how to scale offline inference and training ingest with Ray Data. ````{note} To run this example, install Ray Data: diff --git a/doc/source/ray-overview/index.md b/doc/source/ray-overview/index.md index 99303fc819f7..0eb732ed02df 100644 --- a/doc/source/ray-overview/index.md +++ b/doc/source/ray-overview/index.md @@ -1,7 +1,7 @@ (overview-overview)= # Overview -Ray is an open-source unified framework for scaling AI and Python applications like machine learning. It provides the compute layer for parallel processing so that you don’t need to be a distributed systems expert. Ray minimizes the complexity of running your distributed individual and end-to-end machine learning workflows with these components: +Ray is an open-source unified framework for scaling AI and Python applications like machine learning. It provides the compute layer for parallel processing so that you don’t need to be a distributed systems expert. Ray minimizes the complexity of running your distributed individual workflows and end-to-end machine learning workflows with these components: * Scalable libraries for common machine learning tasks such as data preprocessing, distributed training, hyperparameter tuning, reinforcement learning, and model serving. * Pythonic distributed computing primitives for parallelizing and scaling Python applications. * Integrations and utilities for integrating and deploying a Ray cluster with existing tools and infrastructure such as Kubernetes, AWS, GCP, and Azure. @@ -16,10 +16,10 @@ For ML platform builders and ML engineers, Ray: * Reduces friction between development and production by enabling the same Python code to scale seamlessly from a laptop to a large cluster. For distributed systems engineers, Ray automatically handles key processes: -* Orchestration--Managing the various components of a distributed system. -* Scheduling--Coordinating when and where tasks are executed. -* Fault tolerance--Ensuring tasks complete regardless of inevitable points of failure. -* Auto-scaling--Adjusting the number of resources allocated to dynamic demand. +* Orchestration: Managing the various components of a distributed system. +* Scheduling: Coordinating when and where tasks are executed. +* Fault tolerance: Ensuring tasks complete regardless of inevitable points of failure. +* Auto-scaling: Adjusting the number of resources allocated to dynamic demand. ## What you can do with Ray @@ -110,7 +110,7 @@ Each of [Ray's](../ray-air/getting-started) five native libraries distributes a - [Serve](../serve/index): Scalable and programmable serving to deploy models for online inference, with optional microbatching to improve performance. - [RLlib](../rllib/index): Scalable distributed reinforcement learning workloads. -Ray's libraries are for both data scientists and ML engineers alike. For data scientists, these libraries can be used to scale individual workloads, and also end-to-end ML applications. For ML Engineers, these libraries provides scalable platform abstractions that can be used to easily onboard and integrate tooling from the broader ML ecosystem. +Ray's libraries are for both data scientists and ML engineers. For data scientists, these libraries can be used to scale individual workloads and end-to-end ML applications. For ML engineers, these libraries provide scalable platform abstractions that can be used to easily onboard and integrate tooling from the broader ML ecosystem. For custom applications, the [Ray Core](../ray-core/walkthrough) library enables Python developers to easily build scalable, distributed systems that can run on a laptop, cluster, cloud, or Kubernetes. It's the foundation that Ray AI libraries and third-party integrations (Ray ecosystem) are built on. diff --git a/doc/source/ray-overview/installation.rst b/doc/source/ray-overview/installation.rst index 1bb7e8fcc552..7ba1db6424b3 100644 --- a/doc/source/ray-overview/installation.rst +++ b/doc/source/ray-overview/installation.rst @@ -196,7 +196,7 @@ Here's a summary of the variations: * For MacOS x86_64, commits predating August 7, 2021 will have ``macosx_10_13`` in the filename instead of ``macosx_10_15``. * For MacOS x86_64, commits predating June 1, 2025 will have ``macosx_10_15`` in the filename instead of ``macosx_12_0``. -.. _apple-silcon-supprt: +.. _apple-silicon-support: M1 Mac (Apple Silicon) Support ------------------------------ diff --git a/doc/source/ray-overview/use-cases.rst b/doc/source/ray-overview/use-cases.rst index 9a05bda6a96b..b4d74d51f0d7 100644 --- a/doc/source/ray-overview/use-cases.rst +++ b/doc/source/ray-overview/use-cases.rst @@ -137,7 +137,7 @@ RLlib is an open-source library for reinforcement learning (RL), offering suppor .. figure:: /images/rllib_use_case.png - Decentralized distributed proximal polixy optimiation (DD-PPO) architecture. + Decentralized distributed proximal policy optimization (DD-PPO) architecture. Learn more about reinforcement learning with the following resources. From 252403f34b66318d6b6fd6371ba81b76bbebd950 Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Tue, 2 Sep 2025 23:00:20 -0700 Subject: [PATCH 414/634] [core] Revert lease spec optimization from #55806 (#56179) Signed-off-by: joshlee --- .../task_submission/normal_task_submitter.cc | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 0045f66876d2..9c2bf707247e 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -64,14 +64,9 @@ Status NormalTaskSubmitter::SubmitTask(TaskSpecification task_spec) { const SchedulingKey scheduling_key(task_spec.GetSchedulingClass(), task_spec.GetDependencyIds(), task_spec.GetRuntimeEnvHash()); - auto [scheduler_key_entry_iter, new_scheduling_key_entry] = - scheduling_key_entries_.try_emplace(scheduling_key, SchedulingKeyEntry{}); - auto &scheduling_key_entry = scheduler_key_entry_iter->second; - - // Only set lease_spec if this is a new scheduling key entry - if (new_scheduling_key_entry) { - scheduling_key_entry.lease_spec = LeaseSpecification(task_spec.GetMessage()); - } + // TODO(#56107): Only create the lease spec if this is a new scheduling key entry + auto &scheduling_key_entry = scheduling_key_entries_[scheduling_key]; + scheduling_key_entry.lease_spec = LeaseSpecification(task_spec.GetMessage()); scheduling_key_entry.task_queue.push_back(std::move(task_spec)); if (!scheduling_key_entry.AllWorkersBusy()) { From a9a11440aa2182f5d77bf87dde1d126743f7ff5b Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Wed, 3 Sep 2025 00:38:49 -0700 Subject: [PATCH 415/634] [core][gpu-objects] CI for gpu objects (#56020) Signed-off-by: dayshah --- .buildkite/core.rayci.yml | 18 +++--------------- ci/pipeline/test_conditional_testing.py | 4 +++- ci/pipeline/test_rules.txt | 9 ++++++--- python/ray/tests/BUILD | 6 +++--- .../{ => gpu_objects}/test_gpu_objects_gloo.py | 0 .../{ => gpu_objects}/test_gpu_objects_nccl.py | 0 .../{ => gpu_objects}/test_gpu_objects_nixl.py | 4 ++-- 7 files changed, 17 insertions(+), 24 deletions(-) rename python/ray/tests/{ => gpu_objects}/test_gpu_objects_gloo.py (100%) rename python/ray/tests/{ => gpu_objects}/test_gpu_objects_nccl.py (100%) rename python/ray/tests/{ => gpu_objects}/test_gpu_objects_nixl.py (94%) diff --git a/.buildkite/core.rayci.yml b/.buildkite/core.rayci.yml index e5e1afca4e0d..e57b4d61e8b8 100644 --- a/.buildkite/core.rayci.yml +++ b/.buildkite/core.rayci.yml @@ -60,19 +60,7 @@ steps: - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/_common/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core --install-mask all-ray-libraries --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3 - --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,runtime_env_container,manual,multi_gpu,spark_on_ray,ray_client,compiled_graphs,dask - --install-mask all-ray-libraries - - - label: ":ray: core: cgraph python tests" - tags: - - compiled_graphs - instance_type: large - commands: - - bazel run //ci/ray_ci:test_in_docker -- //python/ray/dag/... core - --install-mask all-ray-libraries - --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3 - --only-tags compiled_graphs - --except-tags multi_gpu + --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,runtime_env_container,manual,multi_gpu,spark_on_ray,ray_client,dask --install-mask all-ray-libraries - label: ":ray: core: python {{matrix.python}} tests ({{matrix.worker_id}})" @@ -469,7 +457,7 @@ steps: depends_on: - corebuild - # block on premerge and microcheck + # block gpu tests on premerge and microcheck - block: "run multi gpu tests" if: build.env("BUILDKITE_PIPELINE_ID") == "0189942e-0876-4b8f-80a4-617f988ec59b" || build.env("BUILDKITE_PIPELINE_ID") == "018f4f1e-1b73-4906-9802-92422e3badaa" key: block-core-gpu-tests @@ -478,7 +466,7 @@ steps: - label: ":ray: core: multi gpu tests" key: core-multi-gpu-tests tags: - - compiled_graphs + - cgraphs_direct_transport - gpu instance_type: gpu-large # we're running some cgraph doc tests here as well since they need gpus diff --git a/ci/pipeline/test_conditional_testing.py b/ci/pipeline/test_conditional_testing.py index 9dfbb68292d1..d42e44973d5b 100644 --- a/ci/pipeline/test_conditional_testing.py +++ b/ci/pipeline/test_conditional_testing.py @@ -43,7 +43,9 @@ - lint ml tune train data - python dashboard linux_wheels macos_wheels java python/ray/dag/dag.py: - - lint python compiled_graphs + - lint python cgraphs_direct_transport +python/ray/experimental/gpu_object_manager/gpu_object_manager.py: + - lint python cgraphs_direct_transport .buildkite/core.rayci.yml: lint python core_cpp java/ray.java: lint java diff --git a/ci/pipeline/test_rules.txt b/ci/pipeline/test_rules.txt index a3aebf566d7e..9bb208c2a0fb 100644 --- a/ci/pipeline/test_rules.txt +++ b/ci/pipeline/test_rules.txt @@ -14,7 +14,7 @@ # ; # Semicolon to separate rules ! always lint -! python cpp core_cpp java workflow compiled_graphs dashboard +! python cpp core_cpp java workflow cgraphs_direct_transport dashboard ! ray_client runtime_env_container ! data dask serve ml tune train llm rllib rllib_gpu rllib_directly ! linux_wheels macos_wheels docker doc python_dependencies tools @@ -94,7 +94,10 @@ python/requirements/ python/ray/dag/ python/ray/experimental/channel/ -@ python compiled_graphs +python/ray/experimental/gpu_object_manager/ +python/ray/experimental/collective/ +python/ray/tests/gpu_objects/ +@ python cgraphs_direct_transport ; python/ray/util/client/ @@ -232,7 +235,7 @@ src/ src/ray/core_worker/experimental*.h src/ray/core_worker/experimental*.cc -@ compiled_graphs +@ cgraphs_direct_transport ; .github/ diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD index 2181d005e59d..96da51621016 100644 --- a/python/ray/tests/BUILD +++ b/python/ray/tests/BUILD @@ -611,7 +611,7 @@ py_test_module_list( py_test_module_list( size = "large", files = [ - "test_gpu_objects_gloo.py", + "gpu_objects/test_gpu_objects_gloo.py", ], tags = [ "exclusive", @@ -629,8 +629,8 @@ py_test_module_list( size = "medium", env = {"RAY_PYTEST_USE_GPU": "1"}, files = [ - "test_gpu_objects_nccl.py", - "test_gpu_objects_nixl.py", + "gpu_objects/test_gpu_objects_nccl.py", + "gpu_objects/test_gpu_objects_nixl.py", ], tags = [ "exclusive", diff --git a/python/ray/tests/test_gpu_objects_gloo.py b/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py similarity index 100% rename from python/ray/tests/test_gpu_objects_gloo.py rename to python/ray/tests/gpu_objects/test_gpu_objects_gloo.py diff --git a/python/ray/tests/test_gpu_objects_nccl.py b/python/ray/tests/gpu_objects/test_gpu_objects_nccl.py similarity index 100% rename from python/ray/tests/test_gpu_objects_nccl.py rename to python/ray/tests/gpu_objects/test_gpu_objects_nccl.py diff --git a/python/ray/tests/test_gpu_objects_nixl.py b/python/ray/tests/gpu_objects/test_gpu_objects_nixl.py similarity index 94% rename from python/ray/tests/test_gpu_objects_nixl.py rename to python/ray/tests/gpu_objects/test_gpu_objects_nixl.py index 1b429ea05fc9..835a0ef4c059 100644 --- a/python/ray/tests/test_gpu_objects_nixl.py +++ b/python/ray/tests/gpu_objects/test_gpu_objects_nixl.py @@ -19,8 +19,8 @@ def sum(self, data, device): @pytest.mark.parametrize("ray_start_regular", [{"num_gpus": 2}], indirect=True) def test_p2p(ray_start_regular): - world_size = 2 - actors = [GPUTestActor.remote() for _ in range(world_size)] + num_actors = 2 + actors = [GPUTestActor.remote() for _ in range(num_actors)] src_actor, dst_actor = actors[0], actors[1] From 55190d3214fca1106b6da2900512e30650564761 Mon Sep 17 00:00:00 2001 From: Zac Policzer Date: Wed, 3 Sep 2025 04:47:19 -0700 Subject: [PATCH 416/634] [core] Remove redundant psutil get_cpu_utilzation call from (#56181) The first call to 'get_cpu_utilization' if you don't pass an interval will always return 0 on the first call. Subsequent calls will have a nonzero number because the sample interval is just assumed to be between successive calls. There are three separate functions for agent/worker/raylet/gcs. They all do the exact same thing but differ slightly in terms of how they're written. Let's look at two. ``` def _get_gcs(self): if self._gcs_pid: gcs_proc = psutil.Process(self._gcs_pid) if gcs_proc: dictionary = gcs_proc.as_dict(attrs=PSUTIL_PROCESS_ATTRS) # dictionary["cpu_percent"] = gcs_proc.cpu_percent(interval=1) return dictionary return {} ``` The above is for gcs ``` def _get_agent(self): # Current proc == agent proc if not self._agent_proc: self._agent_proc = psutil.Process() return self._agent_proc.as_dict(attrs=PSUTIL_PROCESS_ATTRS) ``` The above is for agent. The important thing to notice is these two lines: `gcs_proc = psutil.Process(self._gcs_pid)` and ``` if not self._agent_proc: self._agent_proc = psutil.Process() ``` The agent function uses a cached Process object while the gcs one makes a new one on each call to get_gcs which means that every call to get_gcs is the first call to this object and as a result will always return zero. Meanwhile, since the object is cached for the agent version of this function, only the first call will return non zero. The earlier fix here was just to call it again using the api which allowed one to pass an interval. This would return non-zero and at least fixed the metric. However, it's a bit redundant and has the overhead of having to wait for the interval to elapse. This change removes the extra call and the wait. ## Why are these changes needed? ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: zac --- python/ray/dashboard/modules/reporter/reporter_agent.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/ray/dashboard/modules/reporter/reporter_agent.py b/python/ray/dashboard/modules/reporter/reporter_agent.py index 7a7b1ec83d07..8430bff607b6 100644 --- a/python/ray/dashboard/modules/reporter/reporter_agent.py +++ b/python/ray/dashboard/modules/reporter/reporter_agent.py @@ -474,6 +474,7 @@ def __init__(self, dashboard_agent): thread_name_prefix="reporter_agent_executor", ) self._gcs_pid = None + self._gcs_proc = None self._gpu_profiling_manager = GpuProfilingManager( profile_dir_path=self._log_dir, ip_address=self._ip @@ -1004,10 +1005,10 @@ def _get_raylet_proc(self): def _get_gcs(self): if self._gcs_pid: - gcs_proc = psutil.Process(self._gcs_pid) - if gcs_proc: - dictionary = gcs_proc.as_dict(attrs=PSUTIL_PROCESS_ATTRS) - dictionary["cpu_percent"] = gcs_proc.cpu_percent(interval=1) + if not self._gcs_proc or self._gcs_pid != self._gcs_proc.pid: + self._gcs_proc = psutil.Process(self._gcs_pid) + if self._gcs_proc: + dictionary = self._gcs_proc.as_dict(attrs=PSUTIL_PROCESS_ATTRS) return dictionary return {} From 851db675b33d5e5cef158c92f2578655a032569f Mon Sep 17 00:00:00 2001 From: simonsays1980 Date: Wed, 3 Sep 2025 14:29:20 +0200 Subject: [PATCH 417/634] [RLlib - Offline RL] Add spaces in case only offline data is used (#56141) ## Why are these changes needed? The `spaces` are used to build an `RLModule`. This is in online RL handled by the `EnvRunner`s that check the `EnvToModule` and `ModuleToEnv` spaces. In case of offline RL this is not handled and as soon as a user does use a custom connector that changes observations or actions the `RLModule` cannot be built on the `Learner` anymore. This PR handles this case and adds spaces - deduced from a local learner connector such that users can use space transformations in custom connectors. ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [x] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: simonsays1980 --- rllib/algorithms/algorithm.py | 37 +++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/rllib/algorithms/algorithm.py b/rllib/algorithms/algorithm.py index 6d455da75c55..4a01276ea040 100644 --- a/rllib/algorithms/algorithm.py +++ b/rllib/algorithms/algorithm.py @@ -769,14 +769,35 @@ def setup(self, config: AlgorithmConfig) -> None: elif self.eval_env_runner_group: spaces.update(self.eval_env_runner_group.get_spaces()) else: - spaces.update( - { - DEFAULT_MODULE_ID: ( - self.config.observation_space, - self.config.action_space, - ), - } - ) + # If the algorithm is online we use the spaces from as they are + # provided. + if self.config.is_online: + spaces.update( + { + DEFAULT_MODULE_ID: ( + self.config.observation_space, + self.config.action_space, + ), + } + ) + # Otherwise, when we are offline we need to check, if the learner connector + # is transforming the spaces. + elif self.config.is_offline: + # Build the learner connector with the input spaces from the environment. + learner_connector = self.config.build_learner_connector( + input_observation_space=spaces[INPUT_ENV_SPACES][0], + input_action_space=spaces[INPUT_ENV_SPACES][1], + ) + # Update the `spaces` dictionary by using the output spaces of the learner + # connector pipeline. + spaces.update( + { + DEFAULT_MODULE_ID: ( + learner_connector.observation_space, + learner_connector.action_space, + ), + } + ) module_spec: MultiRLModuleSpec = self.config.get_multi_rl_module_spec( spaces=spaces, From 08e6bfd9e370be3fc724a9314f1994a55ccab56a Mon Sep 17 00:00:00 2001 From: Sampan S Nayak Date: Wed, 3 Sep 2025 18:39:38 +0530 Subject: [PATCH 418/634] [core] Introduce new exception type for un-pickleable exceptions (#55878) This PR introduces a new exception type `UnpickleableException` for cases where the RayTaskException is not unpickleable. previously we threw a `RaySystemError` with the original stacktrace (string representation), but this is not right as the exception is not really due to an issue internal to Ray. sample code: ```python import os os.environ["RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO"] = "0" import openai import ray from openai import AuthenticationError def call_openai_and_error_out(): client = openai.OpenAI( base_url="https://api.endpoints.anyscale.com/v1", api_key="test", ) try: client.chat.completions.create( model="gpt-3.5-turbo", messages=[ {"role": "system", "content": "You are a chatbot."}, {"role": "user", "content": "What is the capital of France?"}, ], ) except AuthenticationError as e: print("Errored as expected given API key is invalid.") raise e remote_fn = ray.remote(call_openai_and_error_out) ray.get(remote_fn.remote()) ``` exception message: ```python (base) ubuntu@devbox:~/clone/ray$ python test.py 2025-08-25 08:24:40,371 INFO worker.py:1939 -- Started a local Ray instance. View the dashboard at 127.0.0.1:8265 Traceback (most recent call last): File "/home/ubuntu/clone/ray/test.py", line 28, in ray.get(remote_fn.remote()) File "/home/ubuntu/clone/ray/python/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper return fn(*args, **kwargs) File "/home/ubuntu/clone/ray/python/ray/_private/client_mode_hook.py", line 104, in wrapper return func(*args, **kwargs) File "/home/ubuntu/clone/ray/python/ray/_private/worker.py", line 2894, in get values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout) File "/home/ubuntu/clone/ray/python/ray/_private/worker.py", line 967, in get_objects raise value ray.exceptions.UnserializableException: Failed to deserialize exception. This typically occurs when the original exception class is not available, there are version incompatibilities, or the serialized data is corrupted. Original exception: ray.exceptions.RayTaskError: ray::call_openai_and_error_out() (pid=1996227, ip=172.31.5.49) File "/home/ubuntu/clone/ray/test.py", line 15, in call_openai_and_error_out client.chat.completions.create( File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_utils/_utils.py", line 287, in wrapper return func(*args, **kwargs) File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/resources/chat/completions/completions.py", line 925, in create return self._post( File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_base_client.py", line 1249, in post return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)) File "/home/ubuntu/.local/lib/python3.9/site-packages/openai/_base_client.py", line 1037, in request raise self._make_status_error_from_response(err.response) from None openai.NotFoundError:

Thank you for using Anyscale's Public Endpoints API.

Effective August 1, 2024 Anyscale Endpoints API is available exclusively through the fully Hosted Anyscale Platform. Multi-tenant access to LLM models has been removed.

With the Hosted Anyscale Platform, you can access the latest GPUs billed by the second, and deploy models on your own dedicated instances. Enjoy full customization to build your end-to-end applications with Anyscale. Get started today.

``` --------- Signed-off-by: sampan Co-authored-by: Edward Oakes --- doc/source/ray-core/api/exceptions.rst | 1 + python/ray/exceptions.py | 39 +++++++++++++++++++----- python/ray/tests/test_traceback.py | 41 ++++---------------------- 3 files changed, 39 insertions(+), 42 deletions(-) diff --git a/doc/source/ray-core/api/exceptions.rst b/doc/source/ray-core/api/exceptions.rst index 104c1c361008..fec840b82eab 100644 --- a/doc/source/ray-core/api/exceptions.rst +++ b/doc/source/ray-core/api/exceptions.rst @@ -39,3 +39,4 @@ Exceptions ray.exceptions.CrossLanguageError ray.exceptions.RaySystemError ray.exceptions.NodeDiedError + ray.exceptions.UnserializableException diff --git a/python/ray/exceptions.py b/python/ray/exceptions.py index 451deb295955..f628ed6549bd 100644 --- a/python/ray/exceptions.py +++ b/python/ray/exceptions.py @@ -48,19 +48,16 @@ def from_ray_exception(ray_exception): if ray_exception.language == PYTHON: try: return pickle.loads(ray_exception.serialized_exception) - except Exception as e: - msg = "Failed to unpickle serialized exception" - # Include a fallback string/stacktrace to aid debugging. - # formatted_exception_string is set in to_bytes() above by calling + except Exception: + # formatted_exception_string is set in to_bytes() above by calling # traceback.format_exception() on the original exception. It contains # the string representation and stack trace of the original error. - formatted = getattr( + original_stacktrace = getattr( ray_exception, "formatted_exception_string", "No formatted exception string available.", ) - msg += f"\nOriginal exception (string repr):\n{formatted}" - raise RuntimeError(msg) from e + return UnserializableException(original_stacktrace) else: return CrossLanguageError(ray_exception) @@ -910,6 +907,33 @@ class RayCgraphCapacityExceeded(RaySystemError): pass +@PublicAPI(stability="alpha") +class UnserializableException(RayError): + """Raised when there is an error deserializing a serialized exception. + + This occurs when deserializing (unpickling) a previously serialized exception + fails. In this case, we fall back to raising the string representation of + the original exception along with its stack trace that was captured at the + time of serialization. + + reference for more details: https://docs.ray.io/en/latest/ray-core/objects/serialization.html + + Args: + original_stack_trace: The string representation and stack trace of the + original exception that was captured during serialization. + """ + + def __init__(self, original_stack_trace: str): + self._original_stack_trace = original_stack_trace + + def __str__(self): + return ( + "Failed to deserialize exception. Refer to https://docs.ray.io/en/latest/ray-core/objects/serialization.html#troubleshooting to troubleshoot.\n" + "Original exception:\n" + f"{self._original_stack_trace}" + ) + + RAY_EXCEPTION_TYPES = [ PlasmaObjectNotAvailable, RayError, @@ -939,4 +963,5 @@ class RayCgraphCapacityExceeded(RaySystemError): RayChannelTimeoutError, OufOfBandObjectRefSerializationException, RayCgraphCapacityExceeded, + UnserializableException, ] diff --git a/python/ray/tests/test_traceback.py b/python/ray/tests/test_traceback.py index 4db33239b4b4..4921181d24d3 100644 --- a/python/ray/tests/test_traceback.py +++ b/python/ray/tests/test_traceback.py @@ -5,7 +5,7 @@ import pytest import ray -from ray.exceptions import RayActorError, RayTaskError +from ray.exceptions import RayActorError, RayTaskError, UnserializableException """This module tests stacktrace of Ray. @@ -301,33 +301,8 @@ def __repr__(self): def test_unpickleable_stacktrace(shutdown_only): - expected_output = """System error: Failed to unpickle serialized exception -Original exception (string repr): -ray.exceptions.RayTaskError: ray::f() (pid=XXX, ip=YYY) - File "FILE", line ZZ, in f - return g(c) - File "FILE", line ZZ, in g - raise NoPickleError("FILE") -test_traceback.NoPickleError - -traceback: Traceback (most recent call last): - File "FILE", line ZZ, in from_ray_exception - return pickle.loads(ray_exception.serialized_exception) -TypeError: __init__() missing 1 required positional argument: 'arg' - -The above exception was the direct cause of the following exception: - -Traceback (most recent call last): - File "FILE", line ZZ, in deserialize_objects - obj = self._deserialize_object( - File "FILE", line ZZ, in _deserialize_object - return RayError.from_bytes(obj) - File "FILE", line ZZ, in from_bytes - return RayError.from_ray_exception(ray_exception) - File "FILE", line ZZ, in from_ray_exception - raise RuntimeError(msg) from e -RuntimeError: Failed to unpickle serialized exception -Original exception (string repr): + expected_output = """Failed to deserialize exception. Refer to https://docs.ray.io/en/latest/ray-core/objects/serialization.html#troubleshooting to troubleshoot. +Original exception: ray.exceptions.RayTaskError: ray::f() (pid=XXX, ip=YYY) File "FILE", line ZZ, in f return g(c) @@ -349,14 +324,10 @@ def f(): c = a + b return g(c) - try: + with pytest.raises(UnserializableException) as excinfo: ray.get(f.remote()) - except Exception as ex: - python310_extra_exc_msg = "test_unpickleable_stacktrace..NoPickleError." - cleaned = scrub_traceback(str(ex)).replace( - f"TypeError: {python310_extra_exc_msg}", "TypeError: " - ) - assert clean_noqa(expected_output) == cleaned + + assert clean_noqa(expected_output) == scrub_traceback(str(excinfo.value)) def test_serialization_error_message(shutdown_only): From 7d9f77da5eba2bd0bfc432d9b50789a5c8741844 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Wed, 3 Sep 2025 09:42:13 -0500 Subject: [PATCH 419/634] [core] Split `gcs_pubsub_lib` targets and move to `pubsub/` (#56167) - Avoid coupling publisher & subscriber targets. - Isolate `PythonGcsSubscriber` to only where it needs to be included (it includes heavyweight gRPC headers). - Move to `pubsub/` dir instead of `gcs/` dir. --------- Signed-off-by: Edward Oakes --- BUILD.bazel | 1 + python/ray/_raylet.pyx | 2 +- python/ray/includes/common.pxd | 6 +- src/ray/gcs/gcs_client/BUILD.bazel | 2 +- src/ray/gcs/gcs_client/gcs_client.cc | 3 +- src/ray/gcs/gcs_client/gcs_client.h | 6 +- .../gcs/gcs_client/global_state_accessor.h | 1 - src/ray/gcs/gcs_server/BUILD.bazel | 16 +- src/ray/gcs/gcs_server/gcs_actor_manager.cc | 2 +- src/ray/gcs/gcs_server/gcs_actor_manager.h | 6 +- .../gcs_autoscaler_state_manager.cc | 2 +- .../gcs_server/gcs_autoscaler_state_manager.h | 6 +- src/ray/gcs/gcs_server/gcs_job_manager.cc | 1 - src/ray/gcs/gcs_server/gcs_job_manager.h | 8 +- src/ray/gcs/gcs_server/gcs_node_manager.cc | 3 +- src/ray/gcs/gcs_server/gcs_node_manager.h | 6 +- src/ray/gcs/gcs_server/gcs_server.cc | 8 +- src/ray/gcs/gcs_server/gcs_server.h | 4 +- .../gcs_server/gcs_server_io_context_policy.h | 4 +- src/ray/gcs/gcs_server/gcs_worker_manager.h | 6 +- src/ray/gcs/gcs_server/pubsub_handler.cc | 2 +- src/ray/gcs/gcs_server/pubsub_handler.h | 6 +- .../gcs_actor_manager_export_event_test.cc | 4 +- .../gcs_job_manager_export_event_test.cc | 4 +- .../gcs_node_manager_export_event_test.cc | 4 +- .../tests/gcs_actor_manager_test.cc | 4 +- .../tests/gcs_actor_scheduler_test.cc | 4 +- .../gcs_server/tests/gcs_job_manager_test.cc | 4 +- .../gcs_server/tests/gcs_node_manager_test.cc | 4 +- .../tests/gcs_placement_group_manager_test.cc | 6 +- .../gcs_placement_group_scheduler_test.cc | 6 +- .../tests/gcs_worker_manager_test.cc | 13 +- src/ray/gcs/pubsub/BUILD.bazel | 14 -- src/ray/gcs/pubsub/gcs_pub_sub.h | 169 ----------------- src/ray/pubsub/BUILD.bazel | 38 ++++ src/ray/pubsub/gcs_publisher.cc | 67 +++++++ src/ray/pubsub/gcs_publisher.h | 72 ++++++++ src/ray/pubsub/gcs_subscriber.cc | 144 +++++++++++++++ src/ray/pubsub/gcs_subscriber.h | 73 ++++++++ .../python_gcs_subscriber.cc} | 173 +----------------- src/ray/pubsub/python_gcs_subscriber.h | 90 +++++++++ 41 files changed, 566 insertions(+), 428 deletions(-) delete mode 100644 src/ray/gcs/pubsub/BUILD.bazel delete mode 100644 src/ray/gcs/pubsub/gcs_pub_sub.h create mode 100644 src/ray/pubsub/gcs_publisher.cc create mode 100644 src/ray/pubsub/gcs_publisher.h create mode 100644 src/ray/pubsub/gcs_subscriber.cc create mode 100644 src/ray/pubsub/gcs_subscriber.h rename src/ray/{gcs/pubsub/gcs_pub_sub.cc => pubsub/python_gcs_subscriber.cc} (51%) create mode 100644 src/ray/pubsub/python_gcs_subscriber.h diff --git a/BUILD.bazel b/BUILD.bazel index 24bfe58c2f90..93a988b07b9d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -251,6 +251,7 @@ pyx_library( "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/store_client:redis_store_client", "//src/ray/protobuf:serialization_cc_proto", + "//src/ray/pubsub:python_gcs_subscriber", "//src/ray/thirdparty/setproctitle", "//src/ray/util:memory", "//src/ray/util:raii", diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index 5765d272a115..a8c02bc4848d 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -2844,7 +2844,7 @@ cdef class GcsClient: cdef class _GcsSubscriber: - """Cython wrapper class of C++ `ray::gcs::PythonGcsSubscriber`.""" + """Cython wrapper class of C++ `ray::pubsub::PythonGcsSubscriber`.""" cdef: shared_ptr[CPythonGcsSubscriber] inner diff --git a/python/ray/includes/common.pxd b/python/ray/includes/common.pxd index d15746effdde..95ca2488e59a 100644 --- a/python/ray/includes/common.pxd +++ b/python/ray/includes/common.pxd @@ -649,8 +649,8 @@ cdef extern from "ray/gcs/gcs_client/gcs_client.h" namespace "ray::gcs" nogil: unordered_map[c_string, double] PythonGetResourcesTotal( const CGcsNodeInfo& node_info) -cdef extern from "ray/gcs/pubsub/gcs_pub_sub.h" nogil: - cdef cppclass CPythonGcsSubscriber "ray::gcs::PythonGcsSubscriber": +cdef extern from "ray/pubsub/python_gcs_subscriber.h" nogil: + cdef cppclass CPythonGcsSubscriber "ray::pubsub::PythonGcsSubscriber": CPythonGcsSubscriber( const c_string& gcs_address, int gcs_port, CChannelType channel_type, @@ -668,7 +668,7 @@ cdef extern from "ray/gcs/pubsub/gcs_pub_sub.h" nogil: CRayStatus Close() -cdef extern from "ray/gcs/pubsub/gcs_pub_sub.h" namespace "ray::gcs" nogil: +cdef extern from "ray/pubsub/python_gcs_subscriber.h" namespace "ray::pubsub" nogil: c_vector[c_string] PythonGetLogBatchLines(CLogBatch log_batch) cdef extern from "ray/gcs/gcs_client/gcs_client.h" namespace "ray::gcs" nogil: diff --git a/src/ray/gcs/gcs_client/BUILD.bazel b/src/ray/gcs/gcs_client/BUILD.bazel index b4b9ec43ade4..bd6dabcc8ae7 100644 --- a/src/ray/gcs/gcs_client/BUILD.bazel +++ b/src/ray/gcs/gcs_client/BUILD.bazel @@ -14,9 +14,9 @@ ray_cc_library( "//src/ray/common:asio", "//src/ray/common:id", "//src/ray/common:protobuf_utils", - "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/gcs/store_client:redis_store_client", "//src/ray/protobuf:usage_cc_proto", + "//src/ray/pubsub:gcs_subscriber", "//src/ray/pubsub:subscriber", "//src/ray/rpc:gcs_client", "//src/ray/util:container_util", diff --git a/src/ray/gcs/gcs_client/gcs_client.cc b/src/ray/gcs/gcs_client/gcs_client.cc index 43d960e0bdf9..f3974a75e5d9 100644 --- a/src/ray/gcs/gcs_client/gcs_client.cc +++ b/src/ray/gcs/gcs_client/gcs_client.cc @@ -146,7 +146,8 @@ Status GcsClient::Connect(instrumented_io_context &io_service, int64_t timeout_m /*callback_service*/ &io_service); // Init GCS subscriber instance. - gcs_subscriber_ = std::make_unique(gcs_address, std::move(subscriber)); + gcs_subscriber_ = + std::make_unique(gcs_address, std::move(subscriber)); job_accessor_ = std::make_unique(this); actor_accessor_ = std::make_unique(this); diff --git a/src/ray/gcs/gcs_client/gcs_client.h b/src/ray/gcs/gcs_client/gcs_client.h index 3c339bc8ff50..913f41415908 100644 --- a/src/ray/gcs/gcs_client/gcs_client.h +++ b/src/ray/gcs/gcs_client/gcs_client.h @@ -29,7 +29,7 @@ #include "ray/common/id.h" #include "ray/common/status.h" #include "ray/gcs/gcs_client/accessor.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" +#include "ray/pubsub/gcs_subscriber.h" #include "ray/rpc/gcs/gcs_rpc_client.h" #include "ray/util/logging.h" #include "ray/util/network_util.h" @@ -223,7 +223,7 @@ class RAY_EXPORT GcsClient : public std::enable_shared_from_this { /// This function is thread safe. virtual InternalKVAccessor &InternalKV() { return *internal_kv_accessor_; } - virtual GcsSubscriber &GetGcsSubscriber() { return *gcs_subscriber_; } + virtual pubsub::GcsSubscriber &GetGcsSubscriber() { return *gcs_subscriber_; } virtual rpc::GcsRpcClient &GetGcsRpcClient() { return *gcs_rpc_client_; } @@ -250,7 +250,7 @@ class RAY_EXPORT GcsClient : public std::enable_shared_from_this { const UniqueID gcs_client_id_ = UniqueID::FromRandom(); - std::unique_ptr gcs_subscriber_; + std::unique_ptr gcs_subscriber_; // Gcs rpc client std::shared_ptr gcs_rpc_client_; diff --git a/src/ray/gcs/gcs_client/global_state_accessor.h b/src/ray/gcs/gcs_client/global_state_accessor.h index 8ad2af80cb2f..7c2266a53b15 100644 --- a/src/ray/gcs/gcs_client/global_state_accessor.h +++ b/src/ray/gcs/gcs_client/global_state_accessor.h @@ -24,7 +24,6 @@ #include "absl/synchronization/mutex.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/gcs/gcs_client/gcs_client.h" -#include "ray/rpc/server_call.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index b3c6248912a9..35ced96675f6 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -74,10 +74,10 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:protobuf_utils", "//src/ray/common:ray_config", - "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/protobuf:autoscaler_cc_proto", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/protobuf:ray_syncer_cc_proto", + "//src/ray/pubsub:gcs_publisher", "//src/ray/raylet_client:raylet_client_pool", "//src/ray/stats:stats_metric", "//src/ray/util:event", @@ -139,8 +139,8 @@ ray_cc_library( hdrs = ["pubsub_handler.h"], deps = [ "//src/ray/gcs/gcs_server:grpc_service_interfaces", - "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/protobuf:gcs_service_cc_proto", + "//src/ray/pubsub:gcs_publisher", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", ], @@ -169,7 +169,7 @@ ray_cc_library( ":gcs_table_storage", ":gcs_usage_stats_client", ":grpc_service_interfaces", - "//src/ray/gcs/pubsub:gcs_pub_sub_lib", + "//src/ray/pubsub:gcs_publisher", "//src/ray/stats:stats_metric", ], ) @@ -233,7 +233,7 @@ ray_cc_library( deps = [ ":gcs_task_manager", "//src/ray/common:ray_syncer", - "//src/ray/gcs/pubsub:gcs_pub_sub_lib", + "//src/ray/pubsub:gcs_publisher", "//src/ray/util:array", "//src/ray/util:type_traits", ], @@ -250,7 +250,7 @@ ray_cc_library( ":grpc_service_interfaces", "//src/ray/common:protobuf_utils", "//src/ray/common:runtime_env", - "//src/ray/gcs/pubsub:gcs_pub_sub_lib", + "//src/ray/pubsub:gcs_publisher", "//src/ray/rpc:core_worker_client", "//src/ray/stats:stats_metric", "//src/ray/util:event", @@ -428,8 +428,8 @@ ray_cc_library( ":grpc_service_interfaces", "//src/ray/common:asio", "//src/ray/common:id", - "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/protobuf:gcs_service_cc_proto", + "//src/ray/pubsub:gcs_publisher", "//src/ray/rpc:core_worker_client", "//src/ray/util:counter_map", "//src/ray/util:logging", @@ -465,8 +465,8 @@ ray_cc_library( ":grpc_service_interfaces", "//src/ray/common:asio", "//src/ray/common:id", - "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/protobuf:gcs_cc_proto", + "//src/ray/pubsub:gcs_publisher", "//src/ray/util:thread_checker", "@com_google_absl//absl/container:flat_hash_map", "@com_google_googletest//:gtest", @@ -507,13 +507,13 @@ ray_cc_library( ":gcs_worker_manager", ":grpc_service_interfaces", ":grpc_services", - "//src/ray/gcs/pubsub:gcs_pub_sub_lib", "//src/ray/gcs/store_client", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/store_client:observable_store_client", "//src/ray/gcs/store_client:redis_store_client", "//src/ray/protobuf:autoscaler_cc_grpc", "//src/ray/protobuf:gcs_service_cc_grpc", + "//src/ray/pubsub:gcs_publisher", "//src/ray/pubsub:publisher", "//src/ray/raylet/scheduling:scheduler", "//src/ray/raylet_client:raylet_client_lib", diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_server/gcs_actor_manager.cc index 6cd3d33d8ca0..dde37de4b3e0 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.cc @@ -222,7 +222,7 @@ GcsActorManager::GcsActorManager( std::unique_ptr scheduler, GcsTableStorage *gcs_table_storage, instrumented_io_context &io_context, - GcsPublisher *gcs_publisher, + pubsub::GcsPublisher *gcs_publisher, RuntimeEnvManager &runtime_env_manager, GCSFunctionManager &function_manager, std::function destroy_owned_placement_group_if_needed, diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.h b/src/ray/gcs/gcs_server/gcs_actor_manager.h index 479d3818100b..1f95b8ccf31f 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.h +++ b/src/ray/gcs/gcs_server/gcs_actor_manager.h @@ -34,7 +34,7 @@ #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/usage_stats_client.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" +#include "ray/pubsub/gcs_publisher.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/counter_map.h" @@ -100,7 +100,7 @@ class GcsActorManager : public rpc::ActorInfoGcsServiceHandler { std::unique_ptr scheduler, GcsTableStorage *gcs_table_storage, instrumented_io_context &io_context, - GcsPublisher *gcs_publisher, + pubsub::GcsPublisher *gcs_publisher, RuntimeEnvManager &runtime_env_manager, GCSFunctionManager &function_manager, std::function destroy_owned_placement_group_if_needed, @@ -474,7 +474,7 @@ class GcsActorManager : public rpc::ActorInfoGcsServiceHandler { GcsTableStorage *gcs_table_storage_; instrumented_io_context &io_context_; /// A publisher for publishing gcs messages. - GcsPublisher *gcs_publisher_; + pubsub::GcsPublisher *gcs_publisher_; /// This is used to communicate with actors and their owners. rpc::CoreWorkerClientPool &worker_client_pool_; /// A callback that is used to destroy placemenet group owned by the actor. diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc index f356930f3fc3..85ff43477a9d 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc @@ -34,7 +34,7 @@ GcsAutoscalerStateManager::GcsAutoscalerStateManager( rpc::RayletClientPool &raylet_client_pool, InternalKVInterface &kv, instrumented_io_context &io_context, - GcsPublisher *gcs_publisher) + pubsub::GcsPublisher *gcs_publisher) : session_name_(std::move(session_name)), gcs_node_manager_(gcs_node_manager), gcs_actor_manager_(gcs_actor_manager), diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h index d262191611fb..61246ad0a6c9 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h @@ -30,7 +30,7 @@ #include "ray/gcs/gcs_server/gcs_placement_group_manager.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/state_util.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" +#include "ray/pubsub/gcs_publisher.h" #include "ray/raylet_client/raylet_client_pool.h" #include "ray/util/thread_checker.h" #include "src/ray/protobuf/gcs.pb.h" @@ -47,7 +47,7 @@ class GcsAutoscalerStateManager : public rpc::autoscaler::AutoscalerStateService rpc::RayletClientPool &raylet_client_pool, InternalKVInterface &kv, instrumented_io_context &io_context, - GcsPublisher *gcs_publisher); + pubsub::GcsPublisher *gcs_publisher); void HandleGetClusterResourceState( rpc::autoscaler::GetClusterResourceStateRequest request, @@ -194,7 +194,7 @@ class GcsAutoscalerStateManager : public rpc::autoscaler::AutoscalerStateService instrumented_io_context &io_context_; // A publisher for publishing gcs messages. - GcsPublisher *gcs_publisher_; + pubsub::GcsPublisher *gcs_publisher_; // The default value of the last seen version for the request is 0, which indicates // no version has been reported. So the first reported version should be 1. diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.cc b/src/ray/gcs/gcs_server/gcs_job_manager.cc index 2d4e93495b85..0ea008ad7ee9 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_job_manager.cc @@ -23,7 +23,6 @@ #include "absl/strings/match.h" #include "ray/common/protobuf_utils.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" #include "ray/stats/metric.h" #include "ray/util/time.h" diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.h b/src/ray/gcs/gcs_server/gcs_job_manager.h index 00b21286a372..095393c960c0 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.h +++ b/src/ray/gcs/gcs_server/gcs_job_manager.h @@ -27,7 +27,7 @@ #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" +#include "ray/pubsub/gcs_publisher.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/event.h" @@ -49,12 +49,10 @@ inline std::string JobDataKey(const std::string &submission_id) { using JobFinishListenerCallback = rpc::JobInfoGcsServiceHandler::JobFinishListenerCallback; -class GcsPublisher; - class GcsJobManager : public rpc::JobInfoGcsServiceHandler { public: explicit GcsJobManager(GcsTableStorage &gcs_table_storage, - GcsPublisher &gcs_publisher, + pubsub::GcsPublisher &gcs_publisher, RuntimeEnvManager &runtime_env_manager, GCSFunctionManager &function_manager, InternalKVInterface &internal_kv, @@ -134,7 +132,7 @@ class GcsJobManager : public rpc::JobInfoGcsServiceHandler { int64_t finished_jobs_count_ = 0; GcsTableStorage &gcs_table_storage_; - GcsPublisher &gcs_publisher_; + pubsub::GcsPublisher &gcs_publisher_; /// Listeners which monitors the finish of jobs. std::vector job_finished_listeners_; diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.cc b/src/ray/gcs/gcs_server/gcs_node_manager.cc index eef85fc4b08b..116c664c017c 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_node_manager.cc @@ -30,8 +30,7 @@ namespace ray { namespace gcs { -////////////////////////////////////////////////////////////////////////////////////////// -GcsNodeManager::GcsNodeManager(GcsPublisher *gcs_publisher, +GcsNodeManager::GcsNodeManager(pubsub::GcsPublisher *gcs_publisher, gcs::GcsTableStorage *gcs_table_storage, instrumented_io_context &io_context, rpc::RayletClientPool *raylet_client_pool, diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.h b/src/ray/gcs/gcs_server/gcs_node_manager.h index b032d2b5662d..1189fd9ce521 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/ray/gcs/gcs_server/gcs_node_manager.h @@ -26,7 +26,7 @@ #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" +#include "ray/pubsub/gcs_publisher.h" #include "ray/raylet_client/raylet_client_pool.h" #include "ray/stats/metric_defs.h" #include "ray/util/event.h" @@ -49,7 +49,7 @@ class GcsNodeManager : public rpc::NodeInfoGcsServiceHandler { /// /// \param gcs_publisher GCS message publisher. /// \param gcs_table_storage GCS table external storage accessor. - GcsNodeManager(GcsPublisher *gcs_publisher, + GcsNodeManager(pubsub::GcsPublisher *gcs_publisher, GcsTableStorage *gcs_table_storage, instrumented_io_context &io_context, rpc::RayletClientPool *raylet_client_pool, @@ -262,7 +262,7 @@ class GcsNodeManager : public rpc::NodeInfoGcsServiceHandler { std::vector)>> node_removed_listeners_; /// A publisher for publishing gcs messages. - GcsPublisher *gcs_publisher_; + pubsub::GcsPublisher *gcs_publisher_; /// Storage for GCS tables. GcsTableStorage *gcs_table_storage_; instrumented_io_context &io_context_; diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 6ac9cb8492a5..ee3e1a478acb 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -113,8 +113,8 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, }); }); }), - pubsub_periodical_runner_( - PeriodicalRunner::Create(io_context_provider_.GetIOContext())), + pubsub_periodical_runner_(PeriodicalRunner::Create( + io_context_provider_.GetIOContext())), periodical_runner_( PeriodicalRunner::Create(io_context_provider_.GetDefaultIOContext())), is_started_(false), @@ -171,7 +171,7 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, /*publish_batch_size_=*/RayConfig::instance().publish_batch_size(), /*publisher_id=*/NodeID::FromRandom()); - gcs_publisher_ = std::make_unique(std::move(inner_publisher)); + gcs_publisher_ = std::make_unique(std::move(inner_publisher)); metrics_agent_client_ = std::make_unique( "127.0.0.1", config_.metrics_agent_port, @@ -626,7 +626,7 @@ void GcsServer::InitKVService() { } void GcsServer::InitPubSubHandler() { - auto &io_context = io_context_provider_.GetIOContext(); + auto &io_context = io_context_provider_.GetIOContext(); pubsub_handler_ = std::make_unique(io_context, *gcs_publisher_); // This service is used to handle long poll requests, so we don't limit active RPCs. diff --git a/src/ray/gcs/gcs_server/gcs_server.h b/src/ray/gcs/gcs_server/gcs_server.h index 6a869ef9369e..9db51a1e976d 100644 --- a/src/ray/gcs/gcs_server/gcs_server.h +++ b/src/ray/gcs/gcs_server/gcs_server.h @@ -33,10 +33,10 @@ #include "ray/gcs/gcs_server/pubsub_handler.h" #include "ray/gcs/gcs_server/runtime_env_handler.h" #include "ray/gcs/gcs_server/usage_stats_client.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/store_client/observable_store_client.h" #include "ray/gcs/store_client/redis_store_client.h" +#include "ray/pubsub/gcs_publisher.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet_client/raylet_client_pool.h" @@ -247,7 +247,7 @@ class GcsServer { /// The autoscaler state manager. std::unique_ptr gcs_autoscaler_state_manager_; /// A publisher for publishing gcs messages. - std::unique_ptr gcs_publisher_; + std::unique_ptr gcs_publisher_; /// The gcs node manager. std::unique_ptr gcs_node_manager_; /// The health check manager. diff --git a/src/ray/gcs/gcs_server/gcs_server_io_context_policy.h b/src/ray/gcs/gcs_server/gcs_server_io_context_policy.h index 5fcc02400a1a..d43dc1086657 100644 --- a/src/ray/gcs/gcs_server/gcs_server_io_context_policy.h +++ b/src/ray/gcs/gcs_server/gcs_server_io_context_policy.h @@ -20,7 +20,7 @@ #include "ray/common/ray_syncer/ray_syncer.h" #include "ray/gcs/gcs_server/gcs_task_manager.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" +#include "ray/pubsub/gcs_publisher.h" #include "ray/util/array.h" #include "ray/util/type_traits.h" @@ -37,7 +37,7 @@ struct GcsServerIOContextPolicy { static constexpr int GetDedicatedIOContextIndex() { if constexpr (std::is_same_v) { return IndexOf("task_io_context"); - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return IndexOf("pubsub_io_context"); } else if constexpr (std::is_same_v) { return IndexOf("ray_syncer_io_context"); diff --git a/src/ray/gcs/gcs_server/gcs_worker_manager.h b/src/ray/gcs/gcs_server/gcs_worker_manager.h index 0912e625d1e6..a5823283d9f7 100644 --- a/src/ray/gcs/gcs_server/gcs_worker_manager.h +++ b/src/ray/gcs/gcs_server/gcs_worker_manager.h @@ -20,7 +20,7 @@ #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/usage_stats_client.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" +#include "ray/pubsub/gcs_publisher.h" namespace ray { namespace gcs { @@ -29,7 +29,7 @@ class GcsWorkerManager : public rpc::WorkerInfoGcsServiceHandler { public: GcsWorkerManager(gcs::GcsTableStorage &gcs_table_storage, instrumented_io_context &io_context, - GcsPublisher &gcs_publisher) + pubsub::GcsPublisher &gcs_publisher) : gcs_table_storage_(gcs_table_storage), io_context_(io_context), gcs_publisher_(gcs_publisher) {} @@ -73,7 +73,7 @@ class GcsWorkerManager : public rpc::WorkerInfoGcsServiceHandler { gcs::GcsTableStorage &gcs_table_storage_; instrumented_io_context &io_context_; - GcsPublisher &gcs_publisher_; + pubsub::GcsPublisher &gcs_publisher_; UsageStatsClient *usage_stats_client_; /// Only listens for unexpected worker deaths not expected like node death. diff --git a/src/ray/gcs/gcs_server/pubsub_handler.cc b/src/ray/gcs/gcs_server/pubsub_handler.cc index 51026d4dc0f0..14b0291c6126 100644 --- a/src/ray/gcs/gcs_server/pubsub_handler.cc +++ b/src/ray/gcs/gcs_server/pubsub_handler.cc @@ -21,7 +21,7 @@ namespace ray { namespace gcs { InternalPubSubHandler::InternalPubSubHandler(instrumented_io_context &io_service, - gcs::GcsPublisher &gcs_publisher) + pubsub::GcsPublisher &gcs_publisher) : io_service_(io_service), gcs_publisher_(gcs_publisher) {} void InternalPubSubHandler::HandleGcsPublish(rpc::GcsPublishRequest request, diff --git a/src/ray/gcs/gcs_server/pubsub_handler.h b/src/ray/gcs/gcs_server/pubsub_handler.h index 04a935fd949a..6808c254ef2c 100644 --- a/src/ray/gcs/gcs_server/pubsub_handler.h +++ b/src/ray/gcs/gcs_server/pubsub_handler.h @@ -19,7 +19,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" -#include "ray/gcs/pubsub/gcs_pub_sub.h" +#include "ray/pubsub/gcs_publisher.h" namespace ray { namespace gcs { @@ -30,7 +30,7 @@ namespace gcs { class InternalPubSubHandler : public rpc::InternalPubSubGcsServiceHandler { public: InternalPubSubHandler(instrumented_io_context &io_service, - gcs::GcsPublisher &gcs_publisher); + pubsub::GcsPublisher &gcs_publisher); void HandleGcsPublish(rpc::GcsPublishRequest request, rpc::GcsPublishReply *reply, @@ -51,7 +51,7 @@ class InternalPubSubHandler : public rpc::InternalPubSubGcsServiceHandler { private: /// Not owning the io service, to allow sharing it with pubsub::Publisher. instrumented_io_context &io_service_; - gcs::GcsPublisher &gcs_publisher_; + pubsub::GcsPublisher &gcs_publisher_; absl::flat_hash_map> sender_to_subscribers_; }; diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc index 104715028a9f..4622270460a2 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc @@ -155,7 +155,7 @@ class GcsActorManagerTest : public ::testing::Test { /*subscriber_timeout_ms=*/absl::ToInt64Microseconds(absl::Seconds(30)), /*batch_size=*/100); - gcs_publisher_ = std::make_unique(std::move(publisher)); + gcs_publisher_ = std::make_unique(std::move(publisher)); gcs_table_storage_ = std::make_unique(std::make_unique()); kv_ = std::make_unique(); @@ -262,7 +262,7 @@ class GcsActorManagerTest : public ::testing::Test { std::unique_ptr worker_client_pool_; absl::flat_hash_map job_namespace_table_; std::unique_ptr gcs_actor_manager_; - std::shared_ptr gcs_publisher_; + std::shared_ptr gcs_publisher_; std::unique_ptr runtime_env_mgr_; const std::chrono::milliseconds timeout_ms_{2000}; absl::Mutex mutex_; diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc index 7ce39262c691..498ba1754e4c 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc @@ -43,7 +43,7 @@ class GcsJobManagerTest : public ::testing::Test { }); promise.get_future().get(); - gcs_publisher_ = std::make_shared( + gcs_publisher_ = std::make_shared( std::make_unique()); store_client_ = std::make_shared(); gcs_table_storage_ = std::make_shared(store_client_); @@ -73,7 +73,7 @@ class GcsJobManagerTest : public ::testing::Test { std::unique_ptr thread_io_service_; std::shared_ptr store_client_; std::shared_ptr gcs_table_storage_; - std::shared_ptr gcs_publisher_; + std::shared_ptr gcs_publisher_; std::unique_ptr function_manager_; std::unique_ptr kv_; std::unique_ptr fake_kv_; diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc index 209411686a7a..aa9731d2823f 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc @@ -48,7 +48,7 @@ class GcsNodeManagerExportAPITest : public ::testing::Test { [raylet_client = std::move(raylet_client)](const rpc::Address &) { return raylet_client; }); - gcs_publisher_ = std::make_unique( + gcs_publisher_ = std::make_unique( std::make_unique()); gcs_table_storage_ = std::make_unique( std::make_unique()); @@ -78,7 +78,7 @@ class GcsNodeManagerExportAPITest : public ::testing::Test { protected: std::unique_ptr gcs_table_storage_; std::unique_ptr client_pool_; - std::shared_ptr gcs_publisher_; + std::shared_ptr gcs_publisher_; instrumented_io_context io_service_; std::string log_dir_; }; diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc index 836687f36fcc..1c4a60bff97c 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc @@ -130,7 +130,7 @@ class GcsActorManagerTest : public ::testing::Test { /*subscriber_timeout_ms=*/absl::ToInt64Microseconds(absl::Seconds(30)), /*batch_size=*/100); - gcs_publisher_ = std::make_unique(std::move(publisher)); + gcs_publisher_ = std::make_unique(std::move(publisher)); store_client_ = std::make_shared(); gcs_table_storage_ = std::make_unique(std::make_unique()); @@ -224,7 +224,7 @@ class GcsActorManagerTest : public ::testing::Test { std::unique_ptr worker_client_pool_; absl::flat_hash_map job_namespace_table_; std::unique_ptr gcs_actor_manager_; - std::shared_ptr gcs_publisher_; + std::shared_ptr gcs_publisher_; std::unique_ptr runtime_env_mgr_; const std::chrono::milliseconds timeout_ms_{2000}; absl::Mutex mutex_; diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc index dcbe76cf4955..d891ce55caba 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc @@ -87,7 +87,7 @@ class GcsActorSchedulerTest : public ::testing::Test { raylet_client_pool_ = std::make_shared( [this](const rpc::Address &addr) { return raylet_client_; }); worker_client_ = std::make_shared(); - gcs_publisher_ = std::make_shared( + gcs_publisher_ = std::make_shared( std::make_unique()); store_client_ = std::make_shared(); gcs_table_storage_ = @@ -214,7 +214,7 @@ class GcsActorSchedulerTest : public ::testing::Test { counter; std::vector> failure_actors_; std::vector> success_actors_; - std::shared_ptr gcs_publisher_; + std::shared_ptr gcs_publisher_; std::shared_ptr gcs_table_storage_; std::shared_ptr raylet_client_pool_; NodeID local_node_id_; diff --git a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc index 0678826f0ac8..ee9f6a320e9a 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc @@ -39,7 +39,7 @@ class GcsJobManagerTest : public ::testing::Test { }); promise.get_future().get(); - gcs_publisher_ = std::make_unique( + gcs_publisher_ = std::make_unique( std::make_unique()); store_client_ = std::make_shared(); gcs_table_storage_ = std::make_shared(store_client_); @@ -74,7 +74,7 @@ class GcsJobManagerTest : public ::testing::Test { std::unique_ptr thread_io_service_; std::shared_ptr store_client_; std::shared_ptr gcs_table_storage_; - std::shared_ptr gcs_publisher_; + std::shared_ptr gcs_publisher_; std::unique_ptr function_manager_; std::unique_ptr kv_; std::unique_ptr fake_kv_; diff --git a/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc index 7086c31bb025..e1ec7cf7398a 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc @@ -33,7 +33,7 @@ class GcsNodeManagerTest : public ::testing::Test { [raylet_client = std::move(raylet_client)](const rpc::Address &) { return raylet_client; }); - gcs_publisher_ = std::make_unique( + gcs_publisher_ = std::make_unique( std::make_unique()); io_context_ = std::make_unique("GcsNodeManagerTest"); } @@ -41,7 +41,7 @@ class GcsNodeManagerTest : public ::testing::Test { protected: std::unique_ptr gcs_table_storage_; std::unique_ptr client_pool_; - std::unique_ptr gcs_publisher_; + std::unique_ptr gcs_publisher_; std::unique_ptr io_context_; }; diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc index 25a0986b0a96..fc0a0dd953fa 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc @@ -82,8 +82,8 @@ class GcsPlacementGroupManagerTest : public ::testing::Test { GcsPlacementGroupManagerTest() : mock_placement_group_scheduler_(new MockPlacementGroupScheduler()), cluster_resource_manager_(io_service_) { - gcs_publisher_ = - std::make_shared(std::make_unique()); + gcs_publisher_ = std::make_shared( + std::make_unique()); gcs_table_storage_ = std::make_unique(std::make_unique()); gcs_node_manager_ = std::make_shared(); @@ -220,7 +220,7 @@ class GcsPlacementGroupManagerTest : public ::testing::Test { ClusterResourceManager cluster_resource_manager_; std::shared_ptr gcs_node_manager_; std::shared_ptr gcs_resource_manager_; - std::shared_ptr gcs_publisher_; + std::shared_ptr gcs_publisher_; }; TEST_F(GcsPlacementGroupManagerTest, TestPlacementGroupBundleCache) { diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc b/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc index 92b8208f410a..f5d9d64cd84c 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc @@ -55,8 +55,8 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { } gcs_table_storage_ = std::make_unique(std::make_unique()); - gcs_publisher_ = - std::make_shared(std::make_unique()); + gcs_publisher_ = std::make_shared( + std::make_unique()); auto local_node_id = NodeID::FromRandom(); cluster_resource_scheduler_ = std::make_shared( io_service_, @@ -300,7 +300,7 @@ class GcsPlacementGroupSchedulerTest : public ::testing::Test { ABSL_GUARDED_BY(placement_group_requests_mutex_); std::vector> failure_placement_groups_ ABSL_GUARDED_BY(placement_group_requests_mutex_); - std::shared_ptr gcs_publisher_; + std::shared_ptr gcs_publisher_; std::shared_ptr gcs_table_storage_; std::unique_ptr raylet_client_pool_; std::shared_ptr> counter_; diff --git a/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc index e23fad95e850..4adaa81169b8 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc @@ -28,15 +28,16 @@ #include "src/ray/protobuf/common.pb.h" #include "src/ray/protobuf/gcs.pb.h" -using namespace ::testing; // NOLINT -using namespace ray::gcs; // NOLINT -using namespace ray; // NOLINT +using namespace ::testing; // NOLINT +using namespace ray::gcs; // NOLINT +using namespace ray::pubsub; // NOLINT +using namespace ray; // NOLINT class GcsWorkerManagerTest : public Test { public: GcsWorkerManagerTest() { - gcs_publisher_ = - std::make_shared(std::make_unique()); + gcs_publisher_ = std::make_shared( + std::make_unique()); gcs_table_storage_ = std::make_unique(std::make_unique()); } @@ -74,7 +75,7 @@ class GcsWorkerManagerTest : public Test { std::unique_ptr thread_io_service_; instrumented_io_context io_service_; std::shared_ptr gcs_table_storage_; - std::shared_ptr gcs_publisher_; + std::shared_ptr gcs_publisher_; std::shared_ptr worker_manager_; }; diff --git a/src/ray/gcs/pubsub/BUILD.bazel b/src/ray/gcs/pubsub/BUILD.bazel deleted file mode 100644 index 5512c0ad67a4..000000000000 --- a/src/ray/gcs/pubsub/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -load("//bazel:ray.bzl", "ray_cc_library") - -ray_cc_library( - name = "gcs_pub_sub_lib", - srcs = ["gcs_pub_sub.cc"], - hdrs = ["gcs_pub_sub.h"], - deps = [ - "//src/ray/common:gcs_callbacks", - "//src/ray/common:ray_config", - "//src/ray/pubsub:publisher_interface", - "//src/ray/pubsub:subscriber_interface", - "//src/ray/rpc:gcs_client", - ], -) diff --git a/src/ray/gcs/pubsub/gcs_pub_sub.h b/src/ray/gcs/pubsub/gcs_pub_sub.h deleted file mode 100644 index d6da94965247..000000000000 --- a/src/ray/gcs/pubsub/gcs_pub_sub.h +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "absl/synchronization/mutex.h" -#include "ray/common/gcs_callbacks.h" -#include "ray/pubsub/publisher_interface.h" -#include "ray/pubsub/subscriber_interface.h" -#include "src/ray/protobuf/gcs.pb.h" -#include "src/ray/protobuf/gcs_service.grpc.pb.h" - -namespace ray { -namespace gcs { - -/// \class GcsPublisher -/// -/// Supports publishing per-entity data and errors from GCS. Thread safe. -class GcsPublisher { - public: - /// Initializes GcsPublisher with GCS based publishers. - /// Publish*() member functions below would be incrementally converted to use the GCS - /// based publisher, if available. - explicit GcsPublisher(std::unique_ptr publisher) - : publisher_(std::move(publisher)) { - RAY_CHECK(publisher_); - } - - /// Returns the underlying pubsub::Publisher. Caller does not take ownership. - pubsub::PublisherInterface &GetPublisher() const { return *publisher_; } - - /// Each publishing method below publishes to a different "channel". - /// ID is the entity which the message is associated with, e.g. ActorID for Actor data. - /// Subscribers receive typed messages for the ID that they subscribe to. - /// - /// The full stream of NodeResource and Error channels are needed by its subscribers. - /// But for other channels, subscribers should only need the latest data. - /// - /// TODO: Verify GCS pubsub satisfies the streaming semantics. - /// TODO: Implement optimization for channels where only latest data per ID is useful. - - void PublishActor(const ActorID &id, rpc::ActorTableData message); - - void PublishJob(const JobID &id, rpc::JobTableData message); - - void PublishNodeInfo(const NodeID &id, rpc::GcsNodeInfo message); - - /// Actually rpc::WorkerDeltaData is not a delta message. - void PublishWorkerFailure(const WorkerID &id, rpc::WorkerDeltaData message); - - void PublishError(std::string id, rpc::ErrorTableData message); - - /// Prints debugging info for the publisher. - std::string DebugString() const; - - private: - const std::unique_ptr publisher_; -}; - -/// \class GcsSubscriber -/// -/// Supports subscribing to an entity or a channel from GCS. Thread safe. -class GcsSubscriber { - public: - /// Initializes GcsSubscriber with GCS based GcsSubscribers. - // TODO(mwtian): Support restarted GCS publisher, at the same or a different address. - GcsSubscriber(rpc::Address gcs_address, - std::unique_ptr subscriber) - : gcs_address_(std::move(gcs_address)), subscriber_(std::move(subscriber)) {} - - /// Subscribe*() member functions below would be incrementally converted to use the GCS - /// based subscriber, if available. - /// The `subscribe` callbacks must not be empty. The `done` callbacks can optionally be - /// empty. - - /// Uses GCS pubsub when created with `subscriber`. - Status SubscribeActor(const ActorID &id, - const SubscribeCallback &subscribe, - const StatusCallback &done); - Status UnsubscribeActor(const ActorID &id); - - bool IsActorUnsubscribed(const ActorID &id); - - Status SubscribeAllJobs(const SubscribeCallback &subscribe, - const StatusCallback &done); - - void SubscribeAllNodeInfo(const ItemCallback &subscribe, - const StatusCallback &done); - - Status SubscribeAllWorkerFailures(const ItemCallback &subscribe, - const StatusCallback &done); - - /// Prints debugging info for the subscriber. - std::string DebugString() const; - - private: - const rpc::Address gcs_address_; - const std::unique_ptr subscriber_; -}; - -// This client is only supposed to be used from Cython / Python -class RAY_EXPORT PythonGcsSubscriber { - public: - PythonGcsSubscriber(const std::string &gcs_address, - int gcs_port, - rpc::ChannelType channel_type, - std::string subscriber_id, - std::string worker_id); - - /// Register a subscription for the subscriber's channel type. - /// - /// Before the registration, published messages in the channel - /// will not be saved for the subscriber. - Status Subscribe(); - - /// Polls for new error message. - /// Both key_id and data are out parameters. - Status PollError(std::string *key_id, int64_t timeout_ms, rpc::ErrorTableData *data); - - /// Polls for new log messages. - Status PollLogs(std::string *key_id, int64_t timeout_ms, rpc::LogBatch *data); - - /// Closes the subscriber and its active subscription. - Status Close(); - - int64_t last_batch_size(); - - private: - Status DoPoll(int64_t timeout_ms, rpc::PubMessage *message); - - mutable absl::Mutex mu_; - - std::shared_ptr channel_; - std::unique_ptr pubsub_stub_; - - const rpc::ChannelType channel_type_; - const std::string subscriber_id_; - std::string publisher_id_; - const std::string worker_id_; - int64_t max_processed_sequence_id_ ABSL_GUARDED_BY(mu_) = 0; - int64_t last_batch_size_ ABSL_GUARDED_BY(mu_) = 0; - std::deque queue_ ABSL_GUARDED_BY(mu_); - bool closed_ ABSL_GUARDED_BY(mu_) = false; - std::shared_ptr current_polling_context_ ABSL_GUARDED_BY(mu_); -}; - -/// Get the .lines() attribute of a LogBatch as a std::vector -/// (this is needed so it can be wrapped in Cython) -std::vector PythonGetLogBatchLines(rpc::LogBatch log_batch); - -} // namespace gcs -} // namespace ray diff --git a/src/ray/pubsub/BUILD.bazel b/src/ray/pubsub/BUILD.bazel index b2c5d89e0efb..2fce880c573b 100644 --- a/src/ray/pubsub/BUILD.bazel +++ b/src/ray/pubsub/BUILD.bazel @@ -54,3 +54,41 @@ ray_cc_library( "@com_google_absl//absl/synchronization", ], ) + +ray_cc_library( + name = "gcs_publisher", + srcs = ["gcs_publisher.cc"], + hdrs = ["gcs_publisher.h"], + deps = [ + ":publisher_interface", + "//src/ray/protobuf:gcs_cc_proto", + ], +) + +ray_cc_library( + name = "gcs_subscriber", + srcs = ["gcs_subscriber.cc"], + hdrs = ["gcs_subscriber.h"], + deps = [ + ":subscriber_interface", + "//src/ray/common:gcs_callbacks", + "//src/ray/protobuf:gcs_cc_proto", + ], +) + +ray_cc_library( + name = "python_gcs_subscriber", + srcs = ["python_gcs_subscriber.cc"], + hdrs = ["python_gcs_subscriber.h"], + implementation_deps = [ + "//src/ray/rpc:gcs_client", + "@com_github_grpc_grpc//:grpc++", + ], + deps = [ + "//src/ray/common:status", + "//src/ray/protobuf:gcs_service_cc_proto", + "//src/ray/protobuf:pubsub_cc_proto", + "//src/ray/util:visibility", + "@com_google_absl//absl/synchronization", + ], +) diff --git a/src/ray/pubsub/gcs_publisher.cc b/src/ray/pubsub/gcs_publisher.cc new file mode 100644 index 000000000000..5cd79bdbe5ca --- /dev/null +++ b/src/ray/pubsub/gcs_publisher.cc @@ -0,0 +1,67 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/pubsub/gcs_publisher.h" + +#include +#include + +namespace ray { +namespace pubsub { + +void GcsPublisher::PublishActor(const ActorID &id, rpc::ActorTableData message) { + rpc::PubMessage msg; + msg.set_channel_type(rpc::ChannelType::GCS_ACTOR_CHANNEL); + msg.set_key_id(id.Binary()); + *msg.mutable_actor_message() = std::move(message); + publisher_->Publish(std::move(msg)); +} + +void GcsPublisher::PublishJob(const JobID &id, rpc::JobTableData message) { + rpc::PubMessage msg; + msg.set_channel_type(rpc::ChannelType::GCS_JOB_CHANNEL); + msg.set_key_id(id.Binary()); + *msg.mutable_job_message() = std::move(message); + publisher_->Publish(std::move(msg)); +} + +void GcsPublisher::PublishNodeInfo(const NodeID &id, rpc::GcsNodeInfo message) { + rpc::PubMessage msg; + msg.set_channel_type(rpc::ChannelType::GCS_NODE_INFO_CHANNEL); + msg.set_key_id(id.Binary()); + *msg.mutable_node_info_message() = std::move(message); + publisher_->Publish(std::move(msg)); +} + +void GcsPublisher::PublishWorkerFailure(const WorkerID &id, + rpc::WorkerDeltaData message) { + rpc::PubMessage msg; + msg.set_channel_type(rpc::ChannelType::GCS_WORKER_DELTA_CHANNEL); + msg.set_key_id(id.Binary()); + *msg.mutable_worker_delta_message() = std::move(message); + publisher_->Publish(std::move(msg)); +} + +void GcsPublisher::PublishError(std::string id, rpc::ErrorTableData message) { + rpc::PubMessage msg; + msg.set_channel_type(rpc::ChannelType::RAY_ERROR_INFO_CHANNEL); + msg.set_key_id(std::move(id)); + *msg.mutable_error_info_message() = std::move(message); + publisher_->Publish(std::move(msg)); +} + +std::string GcsPublisher::DebugString() const { return publisher_->DebugString(); } + +} // namespace pubsub +} // namespace ray diff --git a/src/ray/pubsub/gcs_publisher.h b/src/ray/pubsub/gcs_publisher.h new file mode 100644 index 000000000000..3f1eb8a4b2d5 --- /dev/null +++ b/src/ray/pubsub/gcs_publisher.h @@ -0,0 +1,72 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "ray/pubsub/publisher_interface.h" +#include "src/ray/protobuf/gcs.pb.h" + +namespace ray { +namespace pubsub { + +/// \class GcsPublisher +/// +/// Supports publishing per-entity data and errors from GCS. Thread safe. +class GcsPublisher { + public: + /// Initializes GcsPublisher with GCS based publishers. + /// Publish*() member functions below would be incrementally converted to use the GCS + /// based publisher, if available. + explicit GcsPublisher(std::unique_ptr publisher) + : publisher_(std::move(publisher)) { + RAY_CHECK(publisher_); + } + + /// Returns the underlying pubsub::Publisher. Caller does not take ownership. + pubsub::PublisherInterface &GetPublisher() const { return *publisher_; } + + /// Each publishing method below publishes to a different "channel". + /// ID is the entity which the message is associated with, e.g. ActorID for Actor data. + /// Subscribers receive typed messages for the ID that they subscribe to. + /// + /// The full stream of NodeResource and Error channels are needed by its subscribers. + /// But for other channels, subscribers should only need the latest data. + /// + /// TODO: Verify GCS pubsub satisfies the streaming semantics. + /// TODO: Implement optimization for channels where only latest data per ID is useful. + + void PublishActor(const ActorID &id, rpc::ActorTableData message); + + void PublishJob(const JobID &id, rpc::JobTableData message); + + void PublishNodeInfo(const NodeID &id, rpc::GcsNodeInfo message); + + /// Actually rpc::WorkerDeltaData is not a delta message. + void PublishWorkerFailure(const WorkerID &id, rpc::WorkerDeltaData message); + + void PublishError(std::string id, rpc::ErrorTableData message); + + /// Prints debugging info for the publisher. + std::string DebugString() const; + + private: + const std::unique_ptr publisher_; +}; + +} // namespace pubsub +} // namespace ray diff --git a/src/ray/pubsub/gcs_subscriber.cc b/src/ray/pubsub/gcs_subscriber.cc new file mode 100644 index 000000000000..f02e5e66e9ea --- /dev/null +++ b/src/ray/pubsub/gcs_subscriber.cc @@ -0,0 +1,144 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/pubsub/gcs_subscriber.h" + +#include +#include +#include + +namespace ray { +namespace pubsub { + +Status GcsSubscriber::SubscribeAllJobs( + const gcs::SubscribeCallback &subscribe, + const gcs::StatusCallback &done) { + auto subscribe_item_callback = [subscribe](rpc::PubMessage &&msg) { + RAY_CHECK(msg.channel_type() == rpc::ChannelType::GCS_JOB_CHANNEL); + const JobID id = JobID::FromBinary(msg.key_id()); + subscribe(id, std::move(*msg.mutable_job_message())); + }; + auto subscription_failure_callback = [](const std::string &, const Status &status) { + RAY_LOG(WARNING) << "Subscription to Job channel failed: " << status.ToString(); + }; + subscriber_->Subscribe( + std::make_unique(), + rpc::ChannelType::GCS_JOB_CHANNEL, + gcs_address_, + /*key_id=*/std::nullopt, + [done](const Status &status) { + if (done != nullptr) { + done(status); + } + }, + std::move(subscribe_item_callback), + std::move(subscription_failure_callback)); + return Status::OK(); +} + +Status GcsSubscriber::SubscribeActor( + const ActorID &id, + const gcs::SubscribeCallback &subscribe, + const gcs::StatusCallback &done) { + auto subscription_callback = [id, subscribe](rpc::PubMessage &&msg) { + RAY_CHECK(msg.channel_type() == rpc::ChannelType::GCS_ACTOR_CHANNEL); + RAY_CHECK(msg.key_id() == id.Binary()); + subscribe(id, std::move(*msg.mutable_actor_message())); + }; + auto subscription_failure_callback = [id](const std::string &failed_id, + const Status &status) { + RAY_CHECK(failed_id == id.Binary()); + RAY_LOG(WARNING) << "Subscription to Actor " << id.Hex() + << " failed: " << status.ToString(); + }; + subscriber_->Subscribe( + std::make_unique(), + rpc::ChannelType::GCS_ACTOR_CHANNEL, + gcs_address_, + /*key_id=*/id.Binary(), + [done](const Status &status) { + if (done != nullptr) { + done(status); + } + }, + std::move(subscription_callback), + std::move(subscription_failure_callback)); + return Status::OK(); +} + +Status GcsSubscriber::UnsubscribeActor(const ActorID &id) { + subscriber_->Unsubscribe( + rpc::ChannelType::GCS_ACTOR_CHANNEL, gcs_address_, id.Binary()); + return Status::OK(); +} + +bool GcsSubscriber::IsActorUnsubscribed(const ActorID &id) { + return !subscriber_->IsSubscribed( + rpc::ChannelType::GCS_ACTOR_CHANNEL, gcs_address_, id.Binary()); +} + +void GcsSubscriber::SubscribeAllNodeInfo( + const gcs::ItemCallback &subscribe, + const gcs::StatusCallback &done) { + auto subscribe_item_callback = [subscribe](rpc::PubMessage &&msg) { + RAY_CHECK(msg.channel_type() == rpc::ChannelType::GCS_NODE_INFO_CHANNEL); + subscribe(std::move(*msg.mutable_node_info_message())); + }; + auto subscription_failure_callback = [](const std::string &, const Status &status) { + RAY_LOG(WARNING) << "Subscription to NodeInfo channel failed: " << status.ToString(); + }; + subscriber_->Subscribe( + std::make_unique(), + rpc::ChannelType::GCS_NODE_INFO_CHANNEL, + gcs_address_, + /*key_id=*/std::nullopt, + [done](const Status &status) { + if (done != nullptr) { + done(status); + } + }, + std::move(subscribe_item_callback), + std::move(subscription_failure_callback)); +} + +Status GcsSubscriber::SubscribeAllWorkerFailures( + const gcs::ItemCallback &subscribe, + const gcs::StatusCallback &done) { + auto subscribe_item_callback = [subscribe](rpc::PubMessage &&msg) { + RAY_CHECK(msg.channel_type() == rpc::ChannelType::GCS_WORKER_DELTA_CHANNEL); + subscribe(std::move(*msg.mutable_worker_delta_message())); + }; + auto subscription_failure_callback = [](const std::string &, const Status &status) { + RAY_LOG(WARNING) << "Subscription to WorkerDelta channel failed: " + << status.ToString(); + }; + // Ignore if the subscription already exists, because the resubscription is intentional. + subscriber_->Subscribe( + std::make_unique(), + rpc::ChannelType::GCS_WORKER_DELTA_CHANNEL, + gcs_address_, + /*key_id=*/std::nullopt, + /*subscribe_done_callback=*/ + [done](const Status &status) { + if (done != nullptr) { + done(status); + } + }, + std::move(subscribe_item_callback), + std::move(subscription_failure_callback)); + return Status::OK(); +} + +} // namespace pubsub +} // namespace ray diff --git a/src/ray/pubsub/gcs_subscriber.h b/src/ray/pubsub/gcs_subscriber.h new file mode 100644 index 000000000000..31dd9a733bd0 --- /dev/null +++ b/src/ray/pubsub/gcs_subscriber.h @@ -0,0 +1,73 @@ +// Copyright 2017 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "ray/common/gcs_callbacks.h" +#include "ray/pubsub/subscriber_interface.h" +#include "src/ray/protobuf/gcs.pb.h" + +namespace ray { +namespace pubsub { + +/// \class GcsSubscriber +/// +/// Supports subscribing to an entity or a channel from GCS. Thread safe. +class GcsSubscriber { + public: + /// Initializes GcsSubscriber with GCS based GcsSubscribers. + // TODO(mwtian): Support restarted GCS publisher, at the same or a different address. + GcsSubscriber(rpc::Address gcs_address, + std::unique_ptr subscriber) + : gcs_address_(std::move(gcs_address)), subscriber_(std::move(subscriber)) {} + + /// Subscribe*() member functions below would be incrementally converted to use the GCS + /// based subscriber, if available. + /// The `subscribe` callbacks must not be empty. The `done` callbacks can optionally be + /// empty. + + /// Uses GCS pubsub when created with `subscriber`. + Status SubscribeActor( + const ActorID &id, + const gcs::SubscribeCallback &subscribe, + const gcs::StatusCallback &done); + Status UnsubscribeActor(const ActorID &id); + + bool IsActorUnsubscribed(const ActorID &id); + + Status SubscribeAllJobs( + const gcs::SubscribeCallback &subscribe, + const gcs::StatusCallback &done); + + void SubscribeAllNodeInfo(const gcs::ItemCallback &subscribe, + const gcs::StatusCallback &done); + + Status SubscribeAllWorkerFailures( + const gcs::ItemCallback &subscribe, + const gcs::StatusCallback &done); + + /// Prints debugging info for the subscriber. + std::string DebugString() const; + + private: + const rpc::Address gcs_address_; + const std::unique_ptr subscriber_; +}; + +} // namespace pubsub +} // namespace ray diff --git a/src/ray/gcs/pubsub/gcs_pub_sub.cc b/src/ray/pubsub/python_gcs_subscriber.cc similarity index 51% rename from src/ray/gcs/pubsub/gcs_pub_sub.cc rename to src/ray/pubsub/python_gcs_subscriber.cc index 12fd6a767f09..d50628b4713d 100644 --- a/src/ray/gcs/pubsub/gcs_pub_sub.cc +++ b/src/ray/pubsub/python_gcs_subscriber.cc @@ -1,4 +1,4 @@ -// Copyright 2017 The Ray Authors. +// Copyright 2025 The Ray Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/pubsub/gcs_pub_sub.h" +#include "ray/pubsub/python_gcs_subscriber.h" + +#include #include #include @@ -22,170 +24,7 @@ #include "ray/rpc/gcs/gcs_rpc_client.h" namespace ray { -namespace gcs { - -void GcsPublisher::PublishActor(const ActorID &id, rpc::ActorTableData message) { - rpc::PubMessage msg; - msg.set_channel_type(rpc::ChannelType::GCS_ACTOR_CHANNEL); - msg.set_key_id(id.Binary()); - *msg.mutable_actor_message() = std::move(message); - publisher_->Publish(std::move(msg)); -} - -void GcsPublisher::PublishJob(const JobID &id, rpc::JobTableData message) { - rpc::PubMessage msg; - msg.set_channel_type(rpc::ChannelType::GCS_JOB_CHANNEL); - msg.set_key_id(id.Binary()); - *msg.mutable_job_message() = std::move(message); - publisher_->Publish(std::move(msg)); -} - -void GcsPublisher::PublishNodeInfo(const NodeID &id, rpc::GcsNodeInfo message) { - rpc::PubMessage msg; - msg.set_channel_type(rpc::ChannelType::GCS_NODE_INFO_CHANNEL); - msg.set_key_id(id.Binary()); - *msg.mutable_node_info_message() = std::move(message); - publisher_->Publish(std::move(msg)); -} - -void GcsPublisher::PublishWorkerFailure(const WorkerID &id, - rpc::WorkerDeltaData message) { - rpc::PubMessage msg; - msg.set_channel_type(rpc::ChannelType::GCS_WORKER_DELTA_CHANNEL); - msg.set_key_id(id.Binary()); - *msg.mutable_worker_delta_message() = std::move(message); - publisher_->Publish(std::move(msg)); -} - -void GcsPublisher::PublishError(std::string id, rpc::ErrorTableData message) { - rpc::PubMessage msg; - msg.set_channel_type(rpc::ChannelType::RAY_ERROR_INFO_CHANNEL); - msg.set_key_id(std::move(id)); - *msg.mutable_error_info_message() = std::move(message); - publisher_->Publish(std::move(msg)); -} - -std::string GcsPublisher::DebugString() const { return publisher_->DebugString(); } - -Status GcsSubscriber::SubscribeAllJobs( - const SubscribeCallback &subscribe, - const StatusCallback &done) { - // GCS subscriber. - auto subscribe_item_callback = [subscribe](rpc::PubMessage &&msg) { - RAY_CHECK(msg.channel_type() == rpc::ChannelType::GCS_JOB_CHANNEL); - const JobID id = JobID::FromBinary(msg.key_id()); - subscribe(id, std::move(*msg.mutable_job_message())); - }; - auto subscription_failure_callback = [](const std::string &, const Status &status) { - RAY_LOG(WARNING) << "Subscription to Job channel failed: " << status.ToString(); - }; - subscriber_->Subscribe( - std::make_unique(), - rpc::ChannelType::GCS_JOB_CHANNEL, - gcs_address_, - /*key_id=*/std::nullopt, - [done](const Status &status) { - if (done != nullptr) { - done(status); - } - }, - std::move(subscribe_item_callback), - std::move(subscription_failure_callback)); - return Status::OK(); -} - -Status GcsSubscriber::SubscribeActor( - const ActorID &id, - const SubscribeCallback &subscribe, - const StatusCallback &done) { - // GCS subscriber. - auto subscription_callback = [id, subscribe](rpc::PubMessage &&msg) { - RAY_CHECK(msg.channel_type() == rpc::ChannelType::GCS_ACTOR_CHANNEL); - RAY_CHECK(msg.key_id() == id.Binary()); - subscribe(id, std::move(*msg.mutable_actor_message())); - }; - auto subscription_failure_callback = [id](const std::string &failed_id, - const Status &status) { - RAY_CHECK(failed_id == id.Binary()); - RAY_LOG(WARNING) << "Subscription to Actor " << id.Hex() - << " failed: " << status.ToString(); - }; - subscriber_->Subscribe( - std::make_unique(), - rpc::ChannelType::GCS_ACTOR_CHANNEL, - gcs_address_, - /*key_id=*/id.Binary(), - [done](const Status &status) { - if (done != nullptr) { - done(status); - } - }, - std::move(subscription_callback), - std::move(subscription_failure_callback)); - return Status::OK(); -} - -Status GcsSubscriber::UnsubscribeActor(const ActorID &id) { - subscriber_->Unsubscribe( - rpc::ChannelType::GCS_ACTOR_CHANNEL, gcs_address_, id.Binary()); - return Status::OK(); -} - -bool GcsSubscriber::IsActorUnsubscribed(const ActorID &id) { - return !subscriber_->IsSubscribed( - rpc::ChannelType::GCS_ACTOR_CHANNEL, gcs_address_, id.Binary()); -} - -void GcsSubscriber::SubscribeAllNodeInfo(const ItemCallback &subscribe, - const StatusCallback &done) { - // GCS subscriber. - auto subscribe_item_callback = [subscribe](rpc::PubMessage &&msg) { - RAY_CHECK(msg.channel_type() == rpc::ChannelType::GCS_NODE_INFO_CHANNEL); - subscribe(std::move(*msg.mutable_node_info_message())); - }; - auto subscription_failure_callback = [](const std::string &, const Status &status) { - RAY_LOG(WARNING) << "Subscription to NodeInfo channel failed: " << status.ToString(); - }; - subscriber_->Subscribe( - std::make_unique(), - rpc::ChannelType::GCS_NODE_INFO_CHANNEL, - gcs_address_, - /*key_id=*/std::nullopt, - [done](const Status &status) { - if (done != nullptr) { - done(status); - } - }, - std::move(subscribe_item_callback), - std::move(subscription_failure_callback)); -} - -Status GcsSubscriber::SubscribeAllWorkerFailures( - const ItemCallback &subscribe, const StatusCallback &done) { - auto subscribe_item_callback = [subscribe](rpc::PubMessage &&msg) { - RAY_CHECK(msg.channel_type() == rpc::ChannelType::GCS_WORKER_DELTA_CHANNEL); - subscribe(std::move(*msg.mutable_worker_delta_message())); - }; - auto subscription_failure_callback = [](const std::string &, const Status &status) { - RAY_LOG(WARNING) << "Subscription to WorkerDelta channel failed: " - << status.ToString(); - }; - // Ignore if the subscription already exists, because the resubscription is intentional. - subscriber_->Subscribe( - std::make_unique(), - rpc::ChannelType::GCS_WORKER_DELTA_CHANNEL, - gcs_address_, - /*key_id=*/std::nullopt, - /*subscribe_done_callback=*/ - [done](const Status &status) { - if (done != nullptr) { - done(status); - } - }, - std::move(subscribe_item_callback), - std::move(subscription_failure_callback)); - return Status::OK(); -} +namespace pubsub { std::vector PythonGetLogBatchLines(rpc::LogBatch log_batch) { return std::vector( @@ -356,5 +195,5 @@ int64_t PythonGcsSubscriber::last_batch_size() { return last_batch_size_; } -} // namespace gcs +} // namespace pubsub } // namespace ray diff --git a/src/ray/pubsub/python_gcs_subscriber.h b/src/ray/pubsub/python_gcs_subscriber.h new file mode 100644 index 000000000000..5fe4eda29812 --- /dev/null +++ b/src/ray/pubsub/python_gcs_subscriber.h @@ -0,0 +1,90 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "absl/synchronization/mutex.h" +#include "ray/common/status.h" +#include "ray/util/visibility.h" +#include "src/ray/protobuf/gcs_service.grpc.pb.h" +#include "src/ray/protobuf/pubsub.pb.h" + +// Use forward declarations to avoid exposing heavyweight gRPC headers. +namespace grpc { + +class Channel; +class ClientContext; + +} // namespace grpc + +namespace ray { +namespace pubsub { + +// This client is only supposed to be used from Cython / Python +class RAY_EXPORT PythonGcsSubscriber { + public: + PythonGcsSubscriber(const std::string &gcs_address, + int gcs_port, + rpc::ChannelType channel_type, + std::string subscriber_id, + std::string worker_id); + + /// Register a subscription for the subscriber's channel type. + /// + /// Before the registration, published messages in the channel + /// will not be saved for the subscriber. + Status Subscribe(); + + /// Polls for new error message. + /// Both key_id and data are out parameters. + Status PollError(std::string *key_id, int64_t timeout_ms, rpc::ErrorTableData *data); + + /// Polls for new log messages. + Status PollLogs(std::string *key_id, int64_t timeout_ms, rpc::LogBatch *data); + + /// Closes the subscriber and its active subscription. + Status Close(); + + int64_t last_batch_size(); + + private: + Status DoPoll(int64_t timeout_ms, rpc::PubMessage *message); + + mutable absl::Mutex mu_; + + std::shared_ptr channel_; + std::unique_ptr pubsub_stub_; + + const rpc::ChannelType channel_type_; + const std::string subscriber_id_; + std::string publisher_id_; + const std::string worker_id_; + int64_t max_processed_sequence_id_ ABSL_GUARDED_BY(mu_) = 0; + int64_t last_batch_size_ ABSL_GUARDED_BY(mu_) = 0; + std::deque queue_ ABSL_GUARDED_BY(mu_); + bool closed_ ABSL_GUARDED_BY(mu_) = false; + std::shared_ptr current_polling_context_ ABSL_GUARDED_BY(mu_); +}; + +/// Get the .lines() attribute of a LogBatch as a std::vector +/// (this is needed so it can be wrapped in Cython) +std::vector PythonGetLogBatchLines(rpc::LogBatch log_batch); + +} // namespace pubsub +} // namespace ray From af858cfe836b53945f7efb1eca775d4368a46ce5 Mon Sep 17 00:00:00 2001 From: Potato Date: Thu, 4 Sep 2025 01:05:56 +0800 Subject: [PATCH 420/634] [DOC] docs: Fix typos, grammar, and syntax issues in Ray Tune documentation (#56132) This PR addresses various documentation issues found across 56 Ray Tune documentation files, including typos, grammatical errors, syntax problems, and inconsistencies. The changes focus on improving clarity and correctness while maintaining the original structure and meaning of the content. --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- doc/source/tune/api/api.rst | 2 +- doc/source/tune/api/env.rst | 4 ++-- doc/source/tune/api/logging.rst | 2 +- doc/source/tune/api/schedulers.rst | 2 +- doc/source/tune/examples/index.rst | 2 +- doc/source/tune/faq.rst | 8 ++++---- doc/source/tune/getting-started.rst | 10 +++++----- doc/source/tune/index.rst | 4 ++-- doc/source/tune/key-concepts.rst | 6 +++--- doc/source/tune/tutorials/tune-search-spaces.rst | 2 +- doc/source/tune/tutorials/tune_get_data_in_and_out.md | 4 ++-- 11 files changed, 23 insertions(+), 23 deletions(-) diff --git a/doc/source/tune/api/api.rst b/doc/source/tune/api/api.rst index 2a352e01d37d..3c446bca6fc3 100644 --- a/doc/source/tune/api/api.rst +++ b/doc/source/tune/api/api.rst @@ -6,7 +6,7 @@ Ray Tune API .. tip:: We'd love to hear your feedback on using Tune - `get in touch `_! This section contains a reference for the Tune API. If there is anything missing, please open an issue -on `Github`_. +on `GitHub`_. .. _`GitHub`: https://github.com/ray-project/ray/issues diff --git a/doc/source/tune/api/env.rst b/doc/source/tune/api/env.rst index 513044fc1371..a26e691df1f2 100644 --- a/doc/source/tune/api/env.rst +++ b/doc/source/tune/api/env.rst @@ -63,10 +63,10 @@ These are the environment variables Ray Tune currently considers: but never longer than this value. Defaults to 100 (seconds). * **TUNE_RESULT_BUFFER_MIN_TIME_S**: Additionally, you can specify a minimum time to buffer results. Defaults to 0. * **TUNE_WARN_THRESHOLD_S**: Threshold for logging if an Tune event loop operation takes too long. Defaults to 0.5 (seconds). -* **TUNE_WARN_INSUFFICENT_RESOURCE_THRESHOLD_S**: Threshold for throwing a warning if no active trials are in ``RUNNING`` state +* **TUNE_WARN_INSUFFICIENT_RESOURCE_THRESHOLD_S**: Threshold for throwing a warning if no active trials are in ``RUNNING`` state for this amount of seconds. If the Ray Tune job is stuck in this state (most likely due to insufficient resources), the warning message is printed repeatedly every this amount of seconds. Defaults to 60 (seconds). -* **TUNE_WARN_INSUFFICENT_RESOURCE_THRESHOLD_S_AUTOSCALER**: Threshold for throwing a warning when the autoscaler is enabled and +* **TUNE_WARN_INSUFFICIENT_RESOURCE_THRESHOLD_S_AUTOSCALER**: Threshold for throwing a warning when the autoscaler is enabled and if no active trials are in ``RUNNING`` state for this amount of seconds. If the Ray Tune job is stuck in this state (most likely due to insufficient resources), the warning message is printed repeatedly every this amount of seconds. Defaults to 60 (seconds). diff --git a/doc/source/tune/api/logging.rst b/doc/source/tune/api/logging.rst index f8692a19b2d0..2ef841929056 100644 --- a/doc/source/tune/api/logging.rst +++ b/doc/source/tune/api/logging.rst @@ -97,7 +97,7 @@ Aim Integration Tune also provides a logger for the `Aim `_ experiment tracker. You can install Aim via ``pip install aim``. -See the :doc:`tutorial here ` +See the :doc:`tutorial here `. .. autosummary:: :nosignatures: diff --git a/doc/source/tune/api/schedulers.rst b/doc/source/tune/api/schedulers.rst index 74a44fa826d8..d451d4591b0c 100644 --- a/doc/source/tune/api/schedulers.rst +++ b/doc/source/tune/api/schedulers.rst @@ -44,7 +44,7 @@ setting the ``scheduler`` parameter of ``tune.TuneConfig``, which is taken in by .. code-block:: python from ray import tune - from tune.schedulers import ASHAScheduler + from ray.tune.schedulers import ASHAScheduler asha_scheduler = ASHAScheduler( time_attr='training_iteration', diff --git a/doc/source/tune/examples/index.rst b/doc/source/tune/examples/index.rst index 121e3e47d77e..0de91b153247 100644 --- a/doc/source/tune/examples/index.rst +++ b/doc/source/tune/examples/index.rst @@ -9,7 +9,7 @@ Ray Tune Examples See :ref:`tune-main` to learn more about Tune features. -Below are examples for using Ray Tune for a variety use cases and sorted by categories: +Below are examples for using Ray Tune for a variety of use cases and sorted by categories: * `ML frameworks`_ * `Experiment tracking tools`_ diff --git a/doc/source/tune/faq.rst b/doc/source/tune/faq.rst index 7e3ec9b8bc44..0268caa2f6d8 100644 --- a/doc/source/tune/faq.rst +++ b/doc/source/tune/faq.rst @@ -116,7 +116,7 @@ For **layer sizes** we also suggest trying **powers of 2**. For small problems For **discount factors** in reinforcement learning we suggest sampling uniformly between 0.9 and 1.0. Depending on the problem, a much stricter range above 0.97 -or oeven above 0.99 can make sense (e.g. for Atari). +or even above 0.99 can make sense (e.g. for Atari). How can I use nested/conditional search spaces? @@ -295,7 +295,7 @@ Why is my training stuck and Ray reporting that pending actor or tasks cannot be This is usually caused by Ray actors or tasks being started by the trainable without the trainable resources accounting for them, leading to a deadlock. -This can also be "stealthly" caused by using other libraries in the trainable that are +This can also be "stealthily" caused by using other libraries in the trainable that are based on Ray, such as Modin. In order to fix the issue, request additional resources for the trial using :ref:`placement groups `, as outlined in the section above. @@ -490,7 +490,7 @@ on your machine first to avoid any obvious mistakes. How can I get started contributing to Tune? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -We use Github to track issues, feature requests, and bugs. Take a look at the +We use GitHub to track issues, feature requests, and bugs. Take a look at the ones labeled `"good first issue" `__ and `"help wanted" `__ for a place to start. Look for issues with "[tune]" in the title. @@ -674,7 +674,7 @@ running at a time. A symptom was when trials from job A used parameters specifie leading to unexpected results. Please refer to -[this github issue](https://github.com/ray-project/ray/issues/30091#issuecomment-1431676976) +`this GitHub issue `__ for more context and a workaround if you run into this issue. .. _tune-iterative-experimentation: diff --git a/doc/source/tune/getting-started.rst b/doc/source/tune/getting-started.rst index b5321ffbe3bc..9264662589d3 100644 --- a/doc/source/tune/getting-started.rst +++ b/doc/source/tune/getting-started.rst @@ -19,7 +19,7 @@ To run this example, you will need to install the following: $ pip install "ray[tune]" torch torchvision -Setting Up a Pytorch Model to Tune +Setting Up a PyTorch Model to Tune ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To start off, let's first import some dependencies. @@ -44,7 +44,7 @@ connected layer, and a softmax function. :start-after: __model_def_begin__ :end-before: __model_def_end__ -Below, we have implemented functions for training and evaluating your Pytorch model. +Below, we have implemented functions for training and evaluating your PyTorch model. We define a ``train`` and a ``test`` function for that purpose. If you know how to do this, skip ahead to the next section. @@ -60,7 +60,7 @@ If you know how to do this, skip ahead to the next section. Setting up a ``Tuner`` for a Training Run with Tune ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Below, we define a function that trains the Pytorch model for multiple epochs. +Below, we define a function that trains the PyTorch model for multiple epochs. This function will be executed on a separate :ref:`Ray Actor (process) ` underneath the hood, so we need to communicate the performance of the model back to Tune (which is on the main Python process). @@ -150,7 +150,7 @@ Note that each library has a specific way of defining the search space. Evaluating Your Model after Tuning ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can evaluate best trained model using the :ref:`ExperimentAnalysis object ` to retrieve the best model: +You can evaluate the best trained model using the :ref:`ExperimentAnalysis object ` to retrieve the best model: .. literalinclude:: /../../python/ray/tune/tests/tutorial.py :language: python @@ -163,5 +163,5 @@ Next Steps * Check out the :ref:`Tune tutorials ` for guides on using Tune with your preferred machine learning library. * Browse our :ref:`gallery of examples ` to see how to use Tune with PyTorch, XGBoost, Tensorflow, etc. -* `Let us know `__ if you ran into issues or have any questions by opening an issue on our Github. +* `Let us know `__ if you ran into issues or have any questions by opening an issue on our GitHub. * To check how your application is doing, you can use the :ref:`Ray dashboard `. diff --git a/doc/source/tune/index.rst b/doc/source/tune/index.rst index 56537bcf4725..264a402cb54e 100644 --- a/doc/source/tune/index.rst +++ b/doc/source/tune/index.rst @@ -31,7 +31,7 @@ Tune further integrates with a wide range of additional hyperparameter optimizat In this quick-start example you `minimize` a simple function of the form ``f(x) = a**2 + b``, our `objective` function. The closer ``a`` is to zero and the smaller ``b`` is, the smaller the total value of ``f(x)``. - We will define a so-called `search space` for ``a`` and ``b`` and let Ray Tune explore the space for good values. + We will define a so-called `search space` for ``a`` and ``b`` and let Ray Tune explore the space for good values. .. callout:: @@ -261,7 +261,7 @@ Feel free to submit a pull-request adding (or requesting a removal!) of a listed - `Softlearning `_: Softlearning is a reinforcement learning framework for training maximum entropy policies in continuous domains. Includes the official implementation of the Soft Actor-Critic algorithm. - `Flambe `_: An ML framework to accelerate research and its path to production. See `flambe.ai `_. -- `Population Based Augmentation `_: Population Based Augmentation (PBA) is a algorithm that quickly and efficiently learns data augmentation functions for neural network training. PBA matches state-of-the-art results on CIFAR with one thousand times less compute. +- `Population Based Augmentation `_: Population Based Augmentation (PBA) is an algorithm that quickly and efficiently learns data augmentation functions for neural network training. PBA matches state-of-the-art results on CIFAR with one thousand times less compute. - `Fast AutoAugment by Kakao `_: Fast AutoAugment (Accepted at NeurIPS 2019) learns augmentation policies using a more efficient search strategy based on density matching. - `Allentune `_: Hyperparameter Search for AllenNLP from AllenAI. - `machinable `_: A modular configuration system for machine learning research. See `machinable.org `_. diff --git a/doc/source/tune/key-concepts.rst b/doc/source/tune/key-concepts.rst index 1bd43ae963cb..76d5802d124b 100644 --- a/doc/source/tune/key-concepts.rst +++ b/doc/source/tune/key-concepts.rst @@ -42,7 +42,7 @@ Given concrete choices for ``a``, ``b`` and ``x`` we can evaluate the objective .. tab-item:: Function API - With the :ref:`the function-based API ` you create a function (here called ``trainable``) that + With the :ref:`function-based API ` you create a function (here called ``trainable``) that takes in a dictionary of hyperparameters. This function computes a ``score`` in a "training loop" and `reports` this score back to Tune: @@ -238,7 +238,7 @@ Tune also provides helpful utilities to use with Search Algorithms: * :ref:`limiter`: Limits the amount of concurrent trials when running optimization. * :ref:`shim`: Allows creation of the search algorithm object given a string. -Note that in the example above we tell Tune to ``stop`` after ``20`` training iterations. +Note that in the example above we tell Tune to ``stop`` after ``20`` training iterations. This way of stopping trials with explicit rules is useful, but in many cases we can do even better with `schedulers`. @@ -256,7 +256,7 @@ passes through the trials selected by your search algorithm in the order they we In short, schedulers can stop, pause, or tweak the hyperparameters of running trials, potentially making your hyperparameter tuning process much faster. -Unlike search algorithms, :ref:`Trial Scheduler ` do not select which hyperparameter +Unlike search algorithms, :ref:`Trial Schedulers ` do not select which hyperparameter configurations to evaluate. Here's a quick example of using the so-called ``HyperBand`` scheduler to tune an experiment. diff --git a/doc/source/tune/tutorials/tune-search-spaces.rst b/doc/source/tune/tutorials/tune-search-spaces.rst index 3a8eba780c0c..f3707fa80ac1 100644 --- a/doc/source/tune/tutorials/tune-search-spaces.rst +++ b/doc/source/tune/tutorials/tune-search-spaces.rst @@ -59,7 +59,7 @@ If ``grid_search`` is provided as an argument, the *same* grid will be repeated tuner.fit() # 3 different configs. - tuner = tune.Tuner(trainable, tune_config=tune.TuneConfig(num_samples=1), param_space={"x": grid_search([1, 2, 3])}) + tuner = tune.Tuner(trainable, tune_config=tune.TuneConfig(num_samples=1), param_space={"x": tune.grid_search([1, 2, 3])}) tuner.fit() # 6 different configs. diff --git a/doc/source/tune/tutorials/tune_get_data_in_and_out.md b/doc/source/tune/tutorials/tune_get_data_in_and_out.md index ef7fc2d15c38..b255c29c581d 100644 --- a/doc/source/tune/tutorials/tune_get_data_in_and_out.md +++ b/doc/source/tune/tutorials/tune_get_data_in_and_out.md @@ -318,7 +318,7 @@ def training_function(config, data): start_epoch = 0 if checkpoint: with checkpoint.as_directory() as checkpoint_dir: - with open(os.path.join(checkpoint_dir, "model.pkl"), "w") as f: + with open(os.path.join(checkpoint_dir, "model.pkl"), "rb") as f: checkpoint_dict = pickle.load(f) start_epoch = checkpoint_dict["epoch"] + 1 model = checkpoint_dict["state"] @@ -335,7 +335,7 @@ def training_function(config, data): # Create the checkpoint. with tempfile.TemporaryDirectory() as temp_checkpoint_dir: - with open(os.path.join(temp_checkpoint_dir, "model.pkl"), "w") as f: + with open(os.path.join(temp_checkpoint_dir, "model.pkl"), "wb") as f: pickle.dump(checkpoint_dict, f) tune.report( {"metric": metric}, From aac861ad28548fbd957abcda50c47746943f073f Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Wed, 3 Sep 2025 12:49:25 -0500 Subject: [PATCH 421/634] [core] Move core worker gRPC service definition to `core_worker/` (#56197) - Moves the service definition to `core_worker/`. No need for this to be in a common directory because core worker is the only implementer. - Matches the format of the GCS service definitions. - Removed unnecessary macro indirection and split `.h` & `.cc` files. --------- Signed-off-by: Edward Oakes --- src/ray/core_worker/BUILD.bazel | 20 +- src/ray/core_worker/core_worker.h | 11 -- src/ray/core_worker/core_worker_process.cc | 3 +- src/ray/core_worker/core_worker_process.h | 1 + src/ray/core_worker/grpc_service.cc | 118 ++++++++++++ src/ray/core_worker/grpc_service.h | 174 ++++++++++++++++++ src/ray/core_worker/tests/BUILD.bazel | 1 + src/ray/core_worker/tests/core_worker_test.cc | 4 +- src/ray/rpc/BUILD.bazel | 13 -- src/ray/rpc/grpc_server.h | 6 - src/ray/rpc/worker/core_worker_server.h | 141 -------------- 11 files changed, 318 insertions(+), 174 deletions(-) create mode 100644 src/ray/core_worker/grpc_service.cc create mode 100644 src/ray/core_worker/grpc_service.h delete mode 100644 src/ray/rpc/worker/core_worker_server.h diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 3e10d3417158..0c72ff982e03 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -26,6 +26,7 @@ ray_cc_library( ":experimental_mutable_object_provider", ":future_resolver", ":generator_waiter", + ":grpc_service", ":memory_store", ":object_recovery_manager", ":plasma_store_provider", @@ -46,7 +47,6 @@ ray_cc_library( "//src/ray/pubsub:subscriber", "//src/ray/raylet_client:raylet_client_lib", "//src/ray/rpc:core_worker_client", - "//src/ray/rpc:core_worker_server", "//src/ray/rpc:metrics_agent_client", "//src/ray/stats:stats_lib", "//src/ray/util:container_util", @@ -65,6 +65,24 @@ ray_cc_library( ], ) +ray_cc_library( + name = "grpc_service", + srcs = [ + "grpc_service.cc", + ], + hdrs = [ + "grpc_service.h", + ], + visibility = [":__subpackages__"], + deps = [ + "//src/ray/common:asio", + "//src/ray/protobuf:core_worker_cc_grpc", + "//src/ray/protobuf:core_worker_cc_proto", + "//src/ray/rpc:grpc_server", + "//src/ray/rpc:server_call", + ], +) + ray_cc_library( name = "shutdown_coordinator", srcs = [ diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index 480f11752972..b5f07f4e8505 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -53,21 +53,10 @@ #include "ray/pubsub/publisher.h" #include "ray/pubsub/subscriber.h" #include "ray/raylet_client/raylet_client_interface.h" -#include "ray/rpc/worker/core_worker_server.h" #include "ray/util/process.h" #include "ray/util/shared_lru.h" #include "src/ray/protobuf/pubsub.pb.h" -/// The set of gRPC handlers and their associated level of concurrency. If you want to -/// add a new call to the worker gRPC server, do the following: -/// 1) Add the rpc to the CoreWorkerService in core_worker.proto, e.g., "ExampleCall" -/// 2) Add a new macro to RAY_CORE_WORKER_DECLARE_RPC_HANDLERS -/// in core_worker_server.h, -// e.g. "DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(ExampleCall)" -/// 3) Add a new macro to RAY_CORE_WORKER_RPC_HANDLERS in core_worker_server.h, e.g. -/// "RPC_SERVICE_HANDLER(CoreWorkerService, ExampleCall, 1)" -/// 4) Add a method to the CoreWorker class below: "CoreWorker::HandleExampleCall" - namespace ray::core { JobID GetProcessJobID(const CoreWorkerOptions &options); diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index 844df982b32a..57c4e217a1a9 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -255,7 +255,8 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( // Start RPC server after all the task receivers are properly initialized and we have // our assigned port from the raylet. core_worker_server->RegisterService( - std::make_unique(io_service_, *service_handler_), + std::make_unique( + io_service_, *service_handler_, /*max_active_rpcs_per_handler_=*/-1), false /* token_auth */); core_worker_server->Run(); diff --git a/src/ray/core_worker/core_worker_process.h b/src/ray/core_worker/core_worker_process.h index f516315e3aa2..dd8536e3963b 100644 --- a/src/ray/core_worker/core_worker_process.h +++ b/src/ray/core_worker/core_worker_process.h @@ -19,6 +19,7 @@ #include #include "ray/core_worker/core_worker_options.h" +#include "ray/core_worker/grpc_service.h" #include "ray/rpc/metrics_agent_client.h" #include "ray/util/mutex_protected.h" diff --git a/src/ray/core_worker/grpc_service.cc b/src/ray/core_worker/grpc_service.cc new file mode 100644 index 000000000000..128fb3cd72c9 --- /dev/null +++ b/src/ray/core_worker/grpc_service.cc @@ -0,0 +1,118 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/core_worker/grpc_service.h" + +#include +#include + +namespace ray { +namespace rpc { + +void CoreWorkerGrpcService::InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) { + /// TODO(vitsai): Remove this when auth is implemented for node manager. + /// Disable gRPC server metrics since it incurs too high cardinality. + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED( + CoreWorkerService, PushTask, max_active_rpcs_per_handler_, AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + ActorCallArgWaitComplete, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + RayletNotifyGCSRestart, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + GetObjectStatus, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + WaitForActorRefDeleted, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + PubsubLongPolling, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + PubsubCommandBatch, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + UpdateObjectLocationBatch, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + GetObjectLocationsOwner, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + ReportGeneratorItemReturns, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED( + CoreWorkerService, KillActor, max_active_rpcs_per_handler_, AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED( + CoreWorkerService, CancelTask, max_active_rpcs_per_handler_, AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + RemoteCancelTask, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + RegisterMutableObjectReader, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + GetCoreWorkerStats, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED( + CoreWorkerService, LocalGC, max_active_rpcs_per_handler_, AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED( + CoreWorkerService, DeleteObjects, max_active_rpcs_per_handler_, AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED( + CoreWorkerService, SpillObjects, max_active_rpcs_per_handler_, AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + RestoreSpilledObjects, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + DeleteSpilledObjects, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + PlasmaObjectReady, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED( + CoreWorkerService, Exit, max_active_rpcs_per_handler_, AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + AssignObjectOwner, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + NumPendingTasks, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); + RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED(CoreWorkerService, + FreeActorObject, + max_active_rpcs_per_handler_, + AuthType::NO_AUTH); +} + +} // namespace rpc +} // namespace ray diff --git a/src/ray/core_worker/grpc_service.h b/src/ray/core_worker/grpc_service.h new file mode 100644 index 000000000000..fdb2b09fe2c7 --- /dev/null +++ b/src/ray/core_worker/grpc_service.h @@ -0,0 +1,174 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + * This file defines the gRPC service handlers for the core worker server. + * + * core_worker_process should be the only user of this target. If other classes need the + * CoreWorkerInterface in the future, split it into its own target that does not include + * the heavyweight gRPC headers.. + * + * To add a new RPC handler: + * - Update core_worker.proto. + * - Add a virtual method to CoreWorkerService. + * - Initialize the handler for the method in InitServerCallFactories. + * - Implement the method in core_worker. + */ + +#pragma once + +#include +#include + +#include "ray/common/asio/instrumented_io_context.h" +#include "ray/rpc/grpc_server.h" +#include "ray/rpc/server_call.h" +#include "src/ray/protobuf/core_worker.grpc.pb.h" +#include "src/ray/protobuf/core_worker.pb.h" + +namespace ray { +namespace rpc { + +class CoreWorkerServiceHandler : public DelayedServiceHandler { + public: + /// Blocks until the service is ready to serve RPCs. + virtual void WaitUntilInitialized() = 0; + + virtual void HandlePushTask(PushTaskRequest request, + PushTaskReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleActorCallArgWaitComplete(ActorCallArgWaitCompleteRequest request, + ActorCallArgWaitCompleteReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleRayletNotifyGCSRestart(RayletNotifyGCSRestartRequest request, + RayletNotifyGCSRestartReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetObjectStatus(GetObjectStatusRequest request, + GetObjectStatusReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleWaitForActorRefDeleted(WaitForActorRefDeletedRequest request, + WaitForActorRefDeletedReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandlePubsubLongPolling(PubsubLongPollingRequest request, + PubsubLongPollingReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandlePubsubCommandBatch(PubsubCommandBatchRequest request, + PubsubCommandBatchReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleUpdateObjectLocationBatch(UpdateObjectLocationBatchRequest request, + UpdateObjectLocationBatchReply *reply, + SendReplyCallback send_reply_callback) = 0; + virtual void HandleGetObjectLocationsOwner(GetObjectLocationsOwnerRequest request, + GetObjectLocationsOwnerReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleReportGeneratorItemReturns( + ReportGeneratorItemReturnsRequest request, + ReportGeneratorItemReturnsReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleKillActor(KillActorRequest request, + KillActorReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleCancelTask(CancelTaskRequest request, + CancelTaskReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleRemoteCancelTask(RemoteCancelTaskRequest request, + RemoteCancelTaskReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleRegisterMutableObjectReader( + RegisterMutableObjectReaderRequest request, + RegisterMutableObjectReaderReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleGetCoreWorkerStats(GetCoreWorkerStatsRequest request, + GetCoreWorkerStatsReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleLocalGC(LocalGCRequest request, + LocalGCReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleDeleteObjects(DeleteObjectsRequest request, + DeleteObjectsReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleSpillObjects(SpillObjectsRequest request, + SpillObjectsReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleRestoreSpilledObjects(RestoreSpilledObjectsRequest request, + RestoreSpilledObjectsReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleDeleteSpilledObjects(DeleteSpilledObjectsRequest request, + DeleteSpilledObjectsReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandlePlasmaObjectReady(PlasmaObjectReadyRequest request, + PlasmaObjectReadyReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleExit(ExitRequest request, + ExitReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleAssignObjectOwner(AssignObjectOwnerRequest request, + AssignObjectOwnerReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleNumPendingTasks(NumPendingTasksRequest request, + NumPendingTasksReply *reply, + SendReplyCallback send_reply_callback) = 0; + + virtual void HandleFreeActorObject(FreeActorObjectRequest request, + FreeActorObjectReply *reply, + SendReplyCallback send_reply_callback) = 0; +}; + +class CoreWorkerGrpcService : public GrpcService { + public: + CoreWorkerGrpcService(instrumented_io_context &main_service, + CoreWorkerServiceHandler &service_handler, + int64_t max_active_rpcs_per_handler) + : GrpcService(main_service), + service_handler_(service_handler), + max_active_rpcs_per_handler_(max_active_rpcs_per_handler) {} + + protected: + grpc::Service &GetGrpcService() override { return service_; } + + void InitServerCallFactories( + const std::unique_ptr &cq, + std::vector> *server_call_factories, + const ClusterID &cluster_id) override; + + private: + CoreWorkerService::AsyncService service_; + CoreWorkerServiceHandler &service_handler_; + int64_t max_active_rpcs_per_handler_; +}; + +} // namespace rpc +} // namespace ray diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index de46fb466ffa..f8f357699e07 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -249,6 +249,7 @@ ray_cc_test( "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/common:test_utils", "//src/ray/core_worker:core_worker_lib", + "//src/ray/core_worker:grpc_service", "//src/ray/core_worker:memory_store", "//src/ray/core_worker:reference_count", "//src/ray/ipc:fake_raylet_ipc_client", diff --git a/src/ray/core_worker/tests/core_worker_test.cc b/src/ray/core_worker/tests/core_worker_test.cc index 0fa7170c1838..fe05b3f8d39e 100644 --- a/src/ray/core_worker/tests/core_worker_test.cc +++ b/src/ray/core_worker/tests/core_worker_test.cc @@ -34,6 +34,7 @@ #include "ray/core_worker/context.h" #include "ray/core_worker/core_worker_rpc_proxy.h" #include "ray/core_worker/future_resolver.h" +#include "ray/core_worker/grpc_service.h" #include "ray/core_worker/object_recovery_manager.h" #include "ray/core_worker/reference_count.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" @@ -107,7 +108,8 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { auto core_worker_server = std::make_unique(WorkerTypeString(options.worker_type), 0, true); core_worker_server->RegisterService( - std::make_unique(io_service_, *service_handler), + std::make_unique( + io_service_, *service_handler, /*max_active_rpcs_per_handler_=*/-1), false /* token_auth */); core_worker_server->Run(); diff --git a/src/ray/rpc/BUILD.bazel b/src/ray/rpc/BUILD.bazel index 295f39aa0d05..439e72666618 100644 --- a/src/ray/rpc/BUILD.bazel +++ b/src/ray/rpc/BUILD.bazel @@ -209,16 +209,3 @@ ray_cc_library( "@com_google_absl//absl/synchronization", ], ) - -ray_cc_library( - name = "core_worker_server", - hdrs = [ - "worker/core_worker_server.h", - ], - deps = [ - ":grpc_server", - ":server_call", - "//src/ray/common:asio", - "//src/ray/protobuf:core_worker_cc_grpc", - ], -) diff --git a/src/ray/rpc/grpc_server.h b/src/ray/rpc/grpc_server.h index 93c16457f9cb..db63eed08020 100644 --- a/src/ray/rpc/grpc_server.h +++ b/src/ray/rpc/grpc_server.h @@ -68,12 +68,6 @@ namespace rpc { SERVICE, HANDLER, MAX_ACTIVE_RPCS, AUTH_TYPE) \ _RPC_SERVICE_HANDLER(SERVICE, HANDLER, MAX_ACTIVE_RPCS, AUTH_TYPE, false) -// Define a void RPC client method. -#define DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(METHOD) \ - virtual void Handle##METHOD(::ray::rpc::METHOD##Request request, \ - ::ray::rpc::METHOD##Reply *reply, \ - ::ray::rpc::SendReplyCallback send_reply_callback) = 0; - class GrpcService; /// Class that represents an gRPC server. diff --git a/src/ray/rpc/worker/core_worker_server.h b/src/ray/rpc/worker/core_worker_server.h deleted file mode 100644 index 39a6a918414e..000000000000 --- a/src/ray/rpc/worker/core_worker_server.h +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include "ray/common/asio/instrumented_io_context.h" -#include "ray/rpc/grpc_server.h" -#include "ray/rpc/server_call.h" -#include "src/ray/protobuf/core_worker.grpc.pb.h" -#include "src/ray/protobuf/core_worker.pb.h" - -namespace ray { - -class CoreWorker; - -namespace rpc { -/// TODO(vitsai): Remove this when auth is implemented for node manager -#define RAY_CORE_WORKER_RPC_SERVICE_HANDLER(METHOD) \ - RPC_SERVICE_HANDLER_CUSTOM_AUTH_SERVER_METRICS_DISABLED( \ - CoreWorkerService, METHOD, -1, AuthType::NO_AUTH) - -/// NOTE: See src/ray/core_worker/core_worker.h on how to add a new grpc handler. -/// Disable gRPC server metrics since it incurs too high cardinality. -#define RAY_CORE_WORKER_RPC_HANDLERS \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(PushTask) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(ActorCallArgWaitComplete) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(RayletNotifyGCSRestart) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(GetObjectStatus) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(WaitForActorRefDeleted) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(PubsubLongPolling) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(PubsubCommandBatch) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(UpdateObjectLocationBatch) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(GetObjectLocationsOwner) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(ReportGeneratorItemReturns) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(KillActor) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(CancelTask) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(RemoteCancelTask) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(RegisterMutableObjectReader) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(GetCoreWorkerStats) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(LocalGC) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(DeleteObjects) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(SpillObjects) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(RestoreSpilledObjects) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(DeleteSpilledObjects) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(PlasmaObjectReady) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(Exit) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(AssignObjectOwner) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(NumPendingTasks) \ - RAY_CORE_WORKER_RPC_SERVICE_HANDLER(FreeActorObject) - -#define RAY_CORE_WORKER_DECLARE_RPC_HANDLERS \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(PushTask) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(ActorCallArgWaitComplete) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(RayletNotifyGCSRestart) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(GetObjectStatus) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(WaitForActorRefDeleted) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(PubsubLongPolling) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(PubsubCommandBatch) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(UpdateObjectLocationBatch) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(GetObjectLocationsOwner) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(ReportGeneratorItemReturns) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(KillActor) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(CancelTask) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(RemoteCancelTask) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(RegisterMutableObjectReader) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(GetCoreWorkerStats) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(LocalGC) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(DeleteObjects) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(SpillObjects) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(RestoreSpilledObjects) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(DeleteSpilledObjects) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(PlasmaObjectReady) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(Exit) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(AssignObjectOwner) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(NumPendingTasks) \ - DECLARE_VOID_RPC_SERVICE_HANDLER_METHOD(FreeActorObject) - -/// Interface of the `CoreWorkerServiceHandler`, see `src/ray/protobuf/core_worker.proto`. -class CoreWorkerServiceHandler : public DelayedServiceHandler { - public: - /// Blocks until the service is ready to serve RPCs. - virtual void WaitUntilInitialized() = 0; - - /// Handlers. For all of the following handlers, the implementations can - /// handle the request asynchronously. When handling is done, the - /// `send_reply_callback` should be called. See - /// src/ray/rpc/node_manager/node_manager_client.h and - /// src/ray/protobuf/node_manager.proto for a description of the - /// functionality of each handler. - /// - /// \param[in] request The request message. - /// \param[out] reply The reply message. - /// \param[in] send_reply_callback The callback to be called when the request is done. - RAY_CORE_WORKER_DECLARE_RPC_HANDLERS -}; - -/// The `GrpcServer` for `CoreWorkerService`. -class CoreWorkerGrpcService : public GrpcService { - public: - /// Constructor. - /// - /// \param[in] main_service See super class. - /// \param[in] handler The service handler that actually handle the requests. - CoreWorkerGrpcService(instrumented_io_context &main_service, - CoreWorkerServiceHandler &service_handler) - : GrpcService(main_service), service_handler_(service_handler) {} - - protected: - grpc::Service &GetGrpcService() override { return service_; } - - void InitServerCallFactories( - const std::unique_ptr &cq, - std::vector> *server_call_factories, - const ClusterID &cluster_id) override { - RAY_CORE_WORKER_RPC_HANDLERS - } - - private: - /// The grpc async service object. - CoreWorkerService::AsyncService service_; - - /// The service handler that actually handles the requests. - CoreWorkerServiceHandler &service_handler_; -}; - -} // namespace rpc -} // namespace ray From b7387552bb4ecd78dc5d67779f5c70402a5a63c4 Mon Sep 17 00:00:00 2001 From: Timothy Seah Date: Wed, 3 Sep 2025 10:58:09 -0700 Subject: [PATCH 422/634] [train] Add Torch process group shutdown timeout (#56182) Shutting down a healthy torch process group, which we may want to do for reasons like restarting a group of workers if an async checkpoint upload fails, can hang. This is a workaround until we figure out how to avoid this hang. When this happens, `before_worker_group_shutdown` finishes after the timeout and then workers get killed by `ray.kill`: https://github.com/ray-project/ray/blob/master/python/ray/train/v2/_internal/execution/worker_group/state.py#L127. --------- Signed-off-by: Timothy Seah --- python/ray/train/constants.py | 7 +++++++ python/ray/train/tests/test_backend.py | 19 +++++++++++++++++++ python/ray/train/torch/config.py | 20 ++++++++++++++++++-- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/python/ray/train/constants.py b/python/ray/train/constants.py index 459c33a7a8f4..8e2294827dcd 100644 --- a/python/ray/train/constants.py +++ b/python/ray/train/constants.py @@ -125,6 +125,12 @@ def _v2_migration_warnings_enabled() -> bool: "TUNE_ONLY_STORE_CHECKPOINT_SCORE_ATTRIBUTE" ) +# Seconds to wait for torch process group to shut down. +# Shutting down a healthy torch process group, which we may want to do for reasons +# like restarting a group of workers if an async checkpoint upload fails, can hang. +# This is a workaround until we figure out how to avoid this hang. +TORCH_PROCESS_GROUP_SHUTDOWN_TIMEOUT_S = "TORCH_PROCESS_GROUP_SHUTDOWN_TIMEOUT_S" +DEFAULT_TORCH_PROCESS_GROUP_SHUTDOWN_TIMEOUT_S = 30 # NOTE: When adding a new environment variable, please track it in this list. TRAIN_ENV_VARS = { @@ -137,6 +143,7 @@ def _v2_migration_warnings_enabled() -> bool: RAY_TRAIN_COUNT_PREEMPTION_AS_FAILURE, RAY_TRAIN_ENABLE_STATE_TRACKING, TUNE_ONLY_STORE_CHECKPOINT_SCORE_ATTRIBUTE, + TORCH_PROCESS_GROUP_SHUTDOWN_TIMEOUT_S, } # Key for AIR Checkpoint metadata in TrainingResult metadata diff --git a/python/ray/train/tests/test_backend.py b/python/ray/train/tests/test_backend.py index 3bf9e45a5449..7ea35b11e6bc 100644 --- a/python/ray/train/tests/test_backend.py +++ b/python/ray/train/tests/test_backend.py @@ -28,6 +28,7 @@ from ray.train.constants import ( ENABLE_SHARE_CUDA_VISIBLE_DEVICES_ENV, ENABLE_SHARE_NEURON_CORES_ACCELERATOR_ENV, + TORCH_PROCESS_GROUP_SHUTDOWN_TIMEOUT_S, TRAIN_ENABLE_WORKER_SPREAD_ENV, ) from ray.train.torch import TorchConfig @@ -364,6 +365,24 @@ def check_process_group(): assert not any(e.finish_training()) +@pytest.mark.parametrize( + "init_method, timeout_s", [("env", 5), ("tcp", 5), ("env", 0), ("tcp", 0)] +) +def test_torch_process_group_shutdown_timeout( + ray_start_2_cpus, monkeypatch, init_method, timeout_s +): + monkeypatch.setenv(TORCH_PROCESS_GROUP_SHUTDOWN_TIMEOUT_S, timeout_s) + torch_config = TorchConfig(backend="gloo", init_method=init_method) + e = BackendExecutor(torch_config, num_workers=2) + e.start() + + _start_training(e, lambda: 1) + assert e.finish_training() == [1, 1] + + # Verify that we do not raise an exception even if we time out + e._backend.on_shutdown(e.worker_group, e._backend_config) + + @pytest.mark.parametrize( "worker_results", [ diff --git a/python/ray/train/torch/config.py b/python/ray/train/torch/config.py index 9acc0774d5a5..3e9e41c81ac9 100644 --- a/python/ray/train/torch/config.py +++ b/python/ray/train/torch/config.py @@ -10,10 +10,16 @@ import ray from ray._common.network_utils import build_address +from ray._private import ray_constants from ray.air._internal.device_manager import register_custom_torch_dist_backend +from ray.exceptions import GetTimeoutError from ray.train._internal.utils import get_address_and_port from ray.train._internal.worker_group import WorkerGroup from ray.train.backend import Backend, BackendConfig +from ray.train.constants import ( + DEFAULT_TORCH_PROCESS_GROUP_SHUTDOWN_TIMEOUT_S, + TORCH_PROCESS_GROUP_SHUTDOWN_TIMEOUT_S, +) from ray.util import PublicAPI logger = logging.getLogger(__name__) @@ -202,11 +208,21 @@ def set_env_vars(addr, port): else: raise RuntimeError("Distributed torch is not available.") - def on_shutdown(self, worker_group: WorkerGroup, backend_config: TorchConfig): - worker_group.execute( + def on_shutdown(self, worker_group, backend_config): + futures = worker_group.execute_async( _shutdown_torch, destroy_process_group=len(worker_group) > 1, ) + timeout_s = ray_constants.env_integer( + TORCH_PROCESS_GROUP_SHUTDOWN_TIMEOUT_S, + DEFAULT_TORCH_PROCESS_GROUP_SHUTDOWN_TIMEOUT_S, + ) + try: + ray.get(futures, timeout=timeout_s) + except GetTimeoutError: + logger.warning( + f"Torch process group shutdown timed out after {timeout_s} seconds" + ) def on_training_start( self, worker_group: WorkerGroup, backend_config: BackendConfig From dbdfc779ae0d1a1c09c98b2f1e66d744e70dc4e9 Mon Sep 17 00:00:00 2001 From: Lehui Liu Date: Wed, 3 Sep 2025 11:45:34 -0700 Subject: [PATCH 423/634] [release test] Upgrade Ray Train tests on g3 machine to g4dn (#56175) 1. g3 instance is relatively old and have low availability on AWS. 2. upgrade to g4dn machine that have better availability to reduce test flakiness. 3. one successful release test for the [golden_notebook_torch_tune_serve_test](https://buildkite.com/ray-project/release/builds/56322#_).aws Signed-off-by: Lehui Liu --- release/air_tests/air_benchmarks/compute_gpu_1_aws.yaml | 2 +- release/air_tests/air_benchmarks/compute_gpu_2x2_aws.yaml | 4 ++-- release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml | 4 ++-- release/air_tests/horovod/compute_tpl_aws.yaml | 4 ++-- release/golden_notebook_tests/gpu_tpl_aws.yaml | 2 +- release/ml_user_tests/tune_rllib/compute_tpl_aws.yaml | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/release/air_tests/air_benchmarks/compute_gpu_1_aws.yaml b/release/air_tests/air_benchmarks/compute_gpu_1_aws.yaml index 150990710680..c9e54e107da7 100644 --- a/release/air_tests/air_benchmarks/compute_gpu_1_aws.yaml +++ b/release/air_tests/air_benchmarks/compute_gpu_1_aws.yaml @@ -5,7 +5,7 @@ max_workers: 0 head_node_type: name: head_node - instance_type: g3.8xlarge + instance_type: g4dn.8xlarge worker_node_types: [] diff --git a/release/air_tests/air_benchmarks/compute_gpu_2x2_aws.yaml b/release/air_tests/air_benchmarks/compute_gpu_2x2_aws.yaml index 20791f9e4d9d..c81d613a6f89 100644 --- a/release/air_tests/air_benchmarks/compute_gpu_2x2_aws.yaml +++ b/release/air_tests/air_benchmarks/compute_gpu_2x2_aws.yaml @@ -5,11 +5,11 @@ max_workers: 1 head_node_type: name: head_node - instance_type: g3.8xlarge + instance_type: g4dn.8xlarge worker_node_types: - name: worker_node - instance_type: g3.8xlarge + instance_type: g4dn.8xlarge max_workers: 1 min_workers: 1 use_spot: false diff --git a/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml b/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml index e56edf8bbf28..897fb4ed728e 100644 --- a/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml +++ b/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml @@ -5,11 +5,11 @@ max_workers: 3 head_node_type: name: head_node - instance_type: g4dn.12xlarge + instance_type: g4dn.8xlarge worker_node_types: - name: worker_node - instance_type: g4dn.12xlarge + instance_type: g4dn.8xlarge max_workers: 3 min_workers: 3 use_spot: false diff --git a/release/air_tests/horovod/compute_tpl_aws.yaml b/release/air_tests/horovod/compute_tpl_aws.yaml index 2ef09f059167..f94535b53bef 100644 --- a/release/air_tests/horovod/compute_tpl_aws.yaml +++ b/release/air_tests/horovod/compute_tpl_aws.yaml @@ -6,11 +6,11 @@ max_workers: 1 head_node_type: name: head_node - instance_type: g3.8xlarge + instance_type: g4dn.8xlarge worker_node_types: - name: worker_node - instance_type: g3.8xlarge + instance_type: g4dn.8xlarge max_workers: 1 min_workers: 1 use_spot: false diff --git a/release/golden_notebook_tests/gpu_tpl_aws.yaml b/release/golden_notebook_tests/gpu_tpl_aws.yaml index 12d5f1a9d9bb..b01340ffea6e 100644 --- a/release/golden_notebook_tests/gpu_tpl_aws.yaml +++ b/release/golden_notebook_tests/gpu_tpl_aws.yaml @@ -9,7 +9,7 @@ head_node_type: worker_node_types: - name: worker_node - instance_type: g3.8xlarge + instance_type: g4dn.12xlarge min_workers: 2 max_workers: 2 use_spot: true diff --git a/release/ml_user_tests/tune_rllib/compute_tpl_aws.yaml b/release/ml_user_tests/tune_rllib/compute_tpl_aws.yaml index 376fd90539c7..1290ca8b6900 100644 --- a/release/ml_user_tests/tune_rllib/compute_tpl_aws.yaml +++ b/release/ml_user_tests/tune_rllib/compute_tpl_aws.yaml @@ -15,7 +15,7 @@ worker_node_types: max_workers: 6 use_spot: false - name: worker_node_gpu - instance_type: g3.4xlarge # 1 GPU and 16 CPU + instance_type: g4dn.4xlarge # 1 GPU and 16 CPU min_workers: 2 max_workers: 2 use_spot: false From 39add1e945ef2e1443f538521fd2b4b7810508bf Mon Sep 17 00:00:00 2001 From: ahao-anyscale Date: Wed, 3 Sep 2025 11:48:44 -0700 Subject: [PATCH 424/634] [serve.llm][bug] Copied Deprecation utility from RLlib to avoid unnecessary imports (#56190) Signed-off-by: ahao-anyscale --- .../llm/_internal/common/utils/deprecation.py | 136 ++++++++++++++++++ python/ray/serve/llm/__init__.py | 6 +- 2 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 python/ray/llm/_internal/common/utils/deprecation.py diff --git a/python/ray/llm/_internal/common/utils/deprecation.py b/python/ray/llm/_internal/common/utils/deprecation.py new file mode 100644 index 000000000000..f0a4cfbae186 --- /dev/null +++ b/python/ray/llm/_internal/common/utils/deprecation.py @@ -0,0 +1,136 @@ +# Using Deprecated copied from ray.rllib.utils.deprecation since they are returning better messages. + +import inspect +import logging +from typing import Optional, Union + +from ray.util import log_once +from ray.util.annotations import _mark_annotated + +logger = logging.getLogger(__name__) + +# A constant to use for any configuration that should be deprecated +# (to check, whether this config has actually been assigned a proper value or +# not). +DEPRECATED_VALUE = -1 + + +def deprecation_warning( + old: str, + new: Optional[str] = None, + *, + help: Optional[str] = None, + error: Optional[Union[bool, Exception]] = None, +) -> None: + """Warns (via the `logger` object) or throws a deprecation warning/error. + + Args: + old: A description of the "thing" that is to be deprecated. + new: A description of the new "thing" that replaces it. + help: An optional help text to tell the user, what to + do instead of using `old`. + error: Whether or which exception to raise. If True, raise ValueError. + If False, just warn. If `error` is-a subclass of Exception, + raise that Exception. + + Raises: + ValueError: If `error=True`. + Exception: Of type `error`, iff `error` is a sub-class of `Exception`. + """ + msg = "`{}` has been deprecated.{}".format( + old, (" Use `{}` instead.".format(new) if new else f" {help}" if help else "") + ) + + if error: + if not isinstance(error, bool) and issubclass(error, Exception): + # error is an Exception + raise error(msg) + else: + # error is a boolean, construct ValueError ourselves + raise ValueError(msg) + else: + logger.warning( + "DeprecationWarning: " + msg + " This will raise an error in the future!" + ) + + +def Deprecated(old=None, *, new=None, help=None, error): + """Decorator for documenting a deprecated class, method, or function. + + Automatically adds a `deprecation.deprecation_warning(old=..., + error=False)` to not break existing code at this point to the decorated + class' constructor, method, or function. + + In a next major release, this warning should then be made an error + (by setting error=True), which means at this point that the + class/method/function is no longer supported, but will still inform + the user about the deprecation event. + + In a further major release, the class, method, function should be erased + entirely from the codebase. + + + .. testcode:: + :skipif: True + + from ray.rllib.utils.deprecation import Deprecated + # Deprecated class: Patches the constructor to warn if the class is + # used. + @Deprecated(new="NewAndMuchCoolerClass", error=False) + class OldAndUncoolClass: + ... + + # Deprecated class method: Patches the method to warn if called. + class StillCoolClass: + ... + @Deprecated(new="StillCoolClass.new_and_much_cooler_method()", + error=False) + def old_and_uncool_method(self, uncool_arg): + ... + + # Deprecated function: Patches the function to warn if called. + @Deprecated(new="new_and_much_cooler_function", error=False) + def old_and_uncool_function(*uncool_args): + ... + """ + + def _inner(obj): + # A deprecated class. + if inspect.isclass(obj): + # Patch the class' init method to raise the warning/error. + obj_init = obj.__init__ + + def patched_init(*args, **kwargs): + if log_once(old or obj.__name__): + deprecation_warning( + old=old or obj.__name__, + new=new, + help=help, + error=error, + ) + return obj_init(*args, **kwargs) + + obj.__init__ = patched_init + _mark_annotated(obj) + # Return the patched class (with the warning/error when + # instantiated). + return obj + + # A deprecated class method or function. + # Patch with the warning/error at the beginning. + def _ctor(*args, **kwargs): + if log_once(old or obj.__name__): + deprecation_warning( + old=old or obj.__name__, + new=new, + help=help, + error=error, + ) + # Call the deprecated method/function. + return obj(*args, **kwargs) + + # Return the patched class method/function. + return _ctor + + # Return the prepared decorator. + return _inner diff --git a/python/ray/serve/llm/__init__.py b/python/ray/serve/llm/__init__.py index fea4ce70c2f2..02db3545bd70 100644 --- a/python/ray/serve/llm/__init__.py +++ b/python/ray/serve/llm/__init__.py @@ -1,5 +1,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union +# TODO (ahao): Ray core should inherit deprecation utility. +from ray.llm._internal.common.utils.deprecation import Deprecated from ray.llm._internal.serve.configs.server_models import ( CloudMirrorConfig as _CloudMirrorConfig, LLMConfig as _LLMConfig, @@ -15,10 +17,6 @@ from ray.llm._internal.serve.deployments.routers.router import ( LLMRouter as _LLMRouter, ) - -# Using Deprecated from rllib since they are retuning better messages. -# TODO: Ray core should inherit that. -from ray.rllib.utils.deprecation import Deprecated from ray.util.annotations import PublicAPI if TYPE_CHECKING: From a040e6a0c362288e939448448967543b1efe9e50 Mon Sep 17 00:00:00 2001 From: Jason Li <57246540+JasonLi1909@users.noreply.github.com> Date: Wed, 3 Sep 2025 11:51:57 -0700 Subject: [PATCH 425/634] Remove Placement Group on Train Run Abort (#56011) This PR addresses a bug that occurs when users abort a Train Run from within a Python notebook. When a train run is aborted by stopping a cell execution, the associated placement groups are not removed. And because the train job persists while the notebook kernel is still running, it is never cleaned- preventing the subsequent train run from progressing. This fix manually shuts down the worker group state, which includes the placement group, upon abort- allowing the user to immediately kickoff another train run without having to restart the notebook. --------- Signed-off-by: JasonLi1909 Signed-off-by: Jason Li <57246540+JasonLi1909@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python/ray/train/_internal/worker_group.py | 2 + .../execution/worker_group/worker_group.py | 8 ++-- .../ray/train/v2/tests/test_worker_group.py | 46 +++++++++++++------ 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/python/ray/train/_internal/worker_group.py b/python/ray/train/_internal/worker_group.py index 853502b3512f..e64de700da39 100644 --- a/python/ray/train/_internal/worker_group.py +++ b/python/ray/train/_internal/worker_group.py @@ -269,6 +269,8 @@ def execute(self, func: Callable[..., T], *args, **kwargs) -> List[T]: worker. The order is the same as ``self.workers``. """ + # TODO: Add a timeout in the case of a hang, particularly + # relevant when func is TorchConfig.on_shutdown return ray.get(self.execute_async(func, *args, **kwargs)) def execute_single_async( diff --git a/python/ray/train/v2/_internal/execution/worker_group/worker_group.py b/python/ray/train/v2/_internal/execution/worker_group/worker_group.py index a3a81fa2021c..d48772c4e30d 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/worker_group.py +++ b/python/ray/train/v2/_internal/execution/worker_group/worker_group.py @@ -475,13 +475,15 @@ def _clear_state(self): def abort(self): """Abort the worker group.""" - # TODO: consider shutting down the workers in the future. - # We don't do this for now due to this risk of hanging e.g. when calling - # `destroy_process_group` on an active group. self._assert_active() for callback in self._callbacks: callback.before_worker_group_abort(self._worker_group_context) + # TODO: Add shutdown callback hooks + + self._worker_group_state.shutdown() + self._clear_state() + ##################################################################################### # Polling Worker Group ##################################################################################### diff --git a/python/ray/train/v2/tests/test_worker_group.py b/python/ray/train/v2/tests/test_worker_group.py index 2b796e751c9a..8bf83334fcd7 100644 --- a/python/ray/train/v2/tests/test_worker_group.py +++ b/python/ray/train/v2/tests/test_worker_group.py @@ -28,6 +28,7 @@ Worker, WorkerGroup, WorkerGroupContext, + WorkerGroupState, ) from ray.train.v2.api.config import RunConfig from ray.train.v2.tests.util import DummyObjectRefWrapper, create_dummy_run_context @@ -497,7 +498,21 @@ def after_worker_group_poll_status(self, worker_group_status): assert hooks.shutdown_hook_called -def test_worker_group_abort(): +def test_worker_log_file_paths(): + """Test that log file paths are correctly assigned to workers.""" + wg = _default_inactive_worker_group() + wg._start() + + # Check that all workers have log file paths assigned + workers = wg.get_workers() + for worker in workers: + assert worker.log_file_path is not None + assert "ray-train-app-worker" in worker.log_file_path + + wg.shutdown() + + +def test_worker_group_abort(monkeypatch): class AssertCallback(WorkerGroupCallback): def __init__(self): self.abort_hook_called = False @@ -509,21 +524,26 @@ def before_worker_group_abort(self, worker_group_context): wg = _default_inactive_worker_group(callbacks=[hooks]) wg._start() - wg.abort() - assert hooks.abort_hook_called - wg.shutdown() + # Track shutdown calls without preventing actual cleanup + shutdown_call_count = 0 + original_shutdown = WorkerGroupState.shutdown -def test_worker_log_file_paths(): - """Test that log file paths are correctly assigned to workers.""" - wg = _default_inactive_worker_group() - wg._start() + def track_shutdown_calls(self): + nonlocal shutdown_call_count + shutdown_call_count += 1 + return original_shutdown(self) - # Check that all workers have log file paths assigned - workers = wg.get_workers() - for worker in workers: - assert worker.log_file_path is not None - assert "ray-train-app-worker" in worker.log_file_path + monkeypatch.setattr(WorkerGroupState, "shutdown", track_shutdown_calls) + + wg.abort() + assert ( + shutdown_call_count == 1 + ), f"Expected shutdown to be called once, but was called {shutdown_call_count} times" + assert hooks.abort_hook_called + + # Bypass _assert_active method, allowing for shutdown + monkeypatch.setattr(wg, "_assert_active", lambda: None) wg.shutdown() From 9ab68b04a99c9f125f45998976818a8885df6d4f Mon Sep 17 00:00:00 2001 From: Dongjun Na Date: Thu, 4 Sep 2025 04:51:51 +0900 Subject: [PATCH 426/634] [Serve][1/N] Add autoscaler observability core API schema (#55919) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? This PR adds the Serve Autoscaler Observability schema. The schema defines structured models in `schema.py`, allowing `serve status -v` to return detailed observability data for both deployments and applications. With these models, the examples in the design spec can now be expressed as structured Pydantic objects. This lays the groundwork for integrating the schema into controller logic and CLI output in follow-up PRs. ### Example: Deployment snapshot ```sh ======== Serve Autoscaler status: 2025-08-19T15:05:30Z ======== Deployment status --------------------------------------------------------------- deployment_default_policy: Current replicas: 3 Target replicas: 5 Replicas allowed: min=1, max=10 Scaling status: scaling up Scaling decisions: 2025-08-19T14:00:00Z - scaled down from 5 -> 3 (low traffic) 2025-08-19T15:05:00Z - scaled up from 3 -> 5 (12 requests queued) Policy: Default (queue-length based) Metrics (look_back_period_s=30): queued_requests: 12 Metric collection: delayed (last update 30s ago) Errors: (none) ``` | Deployment spec requirement | Schema / fields | |-----------------------------------------------|---------------------------------------------------------------------------------------------------| | Current / Target replicas / Replicas allowed | `DeploymentAutoscalerView.current_replicas`, `target_replicas`, `min_replicas`, `max_replicas` | | Scaling status (up/down/stable) | `DeploymentAutoscalerView.scaling_status` (`ScalingStatus`) | | Scaling decisions (timestamp, from→to, reason)| `ScalingDecision.timestamp_s`, `from_replicas`, `to_replicas`, `reason`, `source`, `policy`, `metrics` | | Policy | `ScalingDecision.policy` | | Metrics (lookback, queued_requests, etc.) | `DeploymentAutoscalerView.metrics`, `lookback_period_s` | | Metric collection state | `DeploymentAutoscalerView.metrics_health` (`MetricsHealth`) | | Errors | `DeploymentAutoscalerView.errors` | | Webhook history | `ExternalScalerView.webhook_history[]` (`WebhookEvent`) | --- ### Example: Application snapshot ```sh ======== Serve Autoscaler status: 2025-08-20T10:00:00Z ======== Application status --------------------------------------------------------------- application_default_policy: Scaling status: scaling up Policy: Custom (example_application_policy) Scaling decisions: 2025-08-20T09:55:00Z - scaled up frontend: 2 -> 4, backend: 4 -> 6 (total_requests=200) Metrics (look_back_period_s=45): total_requests: 200 Errors: (none) Deployments: frontend: Current replicas: 4 Target replicas: 4 Replicas allowed: min=1, max=10 backend: Current replicas: 6 Target replicas: 6 Replicas allowed: min=2, max=20 ``` | Application spec requirement | Schema / fields | |-----------------------------------------------|-------------------------------------------------------------| | Application name | `ApplicationAutoscalerView.application` | | Application scaling status | `ApplicationAutoscalerView.scaling_status` | | Application policy | `ApplicationAutoscalerView.policy` | | Scaling decisions | `ApplicationAutoscalerView.decisions[]` | | Metrics / lookback | `ApplicationAutoscalerView.metrics`, `lookback_period_s` | | Errors | `ApplicationAutoscalerView.errors` | | Deployment summaries (current/target/min/max) | `ApplicationAutoscalerView.deployments` (`{dep: {current, target}}`) | --- ### Snapshot envelope At the top-level, every snapshot is wrapped in: | Snapshot field | Schema / fields | |----------------------|--------------------------------------------------| | Timestamp | `ServeAutoscalerObservability.timestamp_s` | | Version | `ServeAutoscalerObservability.version` | | Deployment list | `ServeAutoscalerObservability.deployments[]` | | Application list | `ServeAutoscalerObservability.applications[]` | | External scaler list | `ServeAutoscalerObservability.external_scalers[]`| ## Related issue number #55834 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Dongjun Na --- doc/source/serve/api/index.md | 4 +++ python/ray/serve/schema.py | 62 +++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/doc/source/serve/api/index.md b/doc/source/serve/api/index.md index 5b5620c86022..d8d251018c70 100644 --- a/doc/source/serve/api/index.md +++ b/doc/source/serve/api/index.md @@ -102,6 +102,10 @@ See the [model composition guide](serve-model-composition) for how to update cod serve.schema.ServeStatus serve.schema.DeploymentStatusOverview serve.schema.EncodingType + serve.schema.AutoscalingMetricsHealth + serve.schema.AutoscalingStatus + serve.schema.ScalingDecision + serve.schema.DeploymentAutoscalingDetail ``` ### Request Router diff --git a/python/ray/serve/schema.py b/python/ray/serve/schema.py index f9d49d079b07..455ba06a9904 100644 --- a/python/ray/serve/schema.py +++ b/python/ray/serve/schema.py @@ -978,6 +978,63 @@ class ReplicaDetails(ServeActorDetails, frozen=True): ) +@PublicAPI(stability="alpha") +class AutoscalingMetricsHealth(str, Enum): + HEALTHY = "healthy" + DELAYED = "delayed" + UNAVAILABLE = "unavailable" + + +@PublicAPI(stability="alpha") +class AutoscalingStatus(str, Enum): + UPSCALING = "UPSCALING" + DOWNSCALING = "DOWNSCALING" + STABLE = "STABLE" + + +@PublicAPI(stability="alpha") +class ScalingDecision(BaseModel): + """One autoscaling decision with minimal provenance.""" + + timestamp_s: float = Field( + ..., description="Unix time (seconds) when the decision was made." + ) + reason: str = Field( + ..., description="Short, human-readable reason for the decision." + ) + prev_num_replicas: int = Field( + ..., ge=0, description="Replica count before the decision." + ) + curr_num_replicas: int = Field( + ..., ge=0, description="Replica count after the decision." + ) + policy: Optional[str] = Field( + None, description="Policy name or identifier (if applicable)." + ) + + +@PublicAPI(stability="alpha") +class DeploymentAutoscalingDetail(BaseModel): + """Deployment-level autoscaler observability.""" + + scaling_status: AutoscalingStatus = Field( + ..., description="Current scaling direction or stability." + ) + decisions: List[ScalingDecision] = Field( + default_factory=list, description="Recent scaling decisions." + ) + metrics: Optional[Dict[str, Any]] = Field( + None, description="Aggregated metrics for this deployment." + ) + metrics_health: AutoscalingMetricsHealth = Field( + AutoscalingMetricsHealth.HEALTHY, + description="Health of metrics collection pipeline.", + ) + errors: List[str] = Field( + default_factory=list, description="Recent errors/abnormal events." + ) + + @PublicAPI(stability="stable") class DeploymentDetails(BaseModel, extra=Extra.forbid, frozen=True): """ @@ -1018,6 +1075,11 @@ class DeploymentDetails(BaseModel, extra=Extra.forbid, frozen=True): description="Details about the live replicas of this deployment." ) + autoscaling_detail: Optional[DeploymentAutoscalingDetail] = Field( + default=None, + description="[EXPERIMENTAL] Deployment-level autoscaler observability for this deployment.", + ) + @PublicAPI(stability="alpha") class APIType(str, Enum): From a25068ae2ba0c8998d5c30e19a690ec19813e046 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:12:12 -0700 Subject: [PATCH 427/634] [image] unify label and tag conventions (#56189) use the same tagging scheme, with dashes separating fields. Signed-off-by: Lonnie Liu --- .buildkite/_images.rayci.yml | 24 ++++++++++++------------ docker/base-slim/cpu.wanda.yaml | 4 ++-- docker/base-slim/cuda.wanda.yaml | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.buildkite/_images.rayci.yml b/.buildkite/_images.rayci.yml index 74413964085b..caa0b158f52d 100644 --- a/.buildkite/_images.rayci.yml +++ b/.buildkite/_images.rayci.yml @@ -2,7 +2,7 @@ group: images sort_key: "_images" steps: - name: raycpubase - label: "wanda: ray.py{{matrix}}.cpu.base" + label: "wanda: ray-py{{matrix}}-cpu-base" tags: - python_dependencies - docker @@ -17,7 +17,7 @@ steps: ARCH_SUFFIX: "" - name: raycpubaseextra - label: "wanda: ray.py{{matrix}}.cpu.base-extra" + label: "wanda: ray-py{{matrix}}-cpu-base-extra" wanda: docker/base-extra/cpu.wanda.yaml matrix: - "3.9" @@ -31,7 +31,7 @@ steps: depends_on: raycpubase - name: raycudabase - label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base" + label: "wanda: ray-py{{matrix.python}}-cu{{matrix.cuda}}-base" tags: - python_dependencies - docker @@ -58,7 +58,7 @@ steps: ARCH_SUFFIX: "" - name: raycudabaseextra - label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" + label: "wanda: ray-py{{matrix.python}}-cu{{matrix.cuda}}-base-extra" wanda: docker/base-extra/cuda.wanda.yaml matrix: setup: @@ -84,7 +84,7 @@ steps: depends_on: raycudabase - name: ray-llmbase - label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base" + label: "wanda: ray-llm-py{{matrix.python}}-cu{{matrix.cuda}}-base" tags: - python_dependencies - docker @@ -101,7 +101,7 @@ steps: CUDA_VERSION: "{{matrix.cuda}}" - name: ray-llmbaseextra - label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" + label: "wanda: ray-llm-py{{matrix.python}}-cu{{matrix.cuda}}-base-extra" wanda: docker/base-extra/cuda.wanda.yaml matrix: setup: @@ -117,7 +117,7 @@ steps: depends_on: ray-llmbase - name: ray-mlcpubase - label: "wanda: ray-ml.py{{matrix}}.cpu.base" + label: "wanda: ray-ml-py{{matrix}}-cpu-base" tags: - python_dependencies - docker @@ -131,7 +131,7 @@ steps: PYTHON_VERSION: "{{matrix}}" - name: ray-mlcpubaseextra - label: "wanda: ray-ml.py{{matrix}}.cpu.base-extra" + label: "wanda: ray-ml-py{{matrix}}-cpu-base-extra" wanda: docker/base-extra/cpu.wanda.yaml matrix: - "3.9" @@ -144,7 +144,7 @@ steps: depends_on: ray-mlcpubase - name: ray-mlcudabase - label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base" + label: "wanda: ray-ml-py{{matrix.python}}-cu{{matrix.cuda}}-base" tags: - python_dependencies - docker @@ -163,7 +163,7 @@ steps: CUDA_VERSION: "{{matrix.cuda}}" - name: ray-mlcudabaseextra - label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra" + label: "wanda: ray-ml-py{{matrix.python}}-cu{{matrix.cuda}}-base-extra" wanda: docker/base-extra/cuda.wanda.yaml matrix: setup: @@ -181,7 +181,7 @@ steps: depends_on: ray-mlcudabase - name: ray-slimcpubase - label: "wanda: ray-slim.py{{matrix}}.cpu.base" + label: "wanda: ray-slim-py{{matrix}}-cpu-base" tags: - python_dependencies - docker @@ -198,7 +198,7 @@ steps: ARCH_SUFFIX: "" - name: ray-slimcudabase - label: "wanda: ray-slim.py{{matrix.python}}.cu{{matrix.cuda}}.base" + label: "wanda: ray-slim-py{{matrix.python}}-cu{{matrix.cuda}}-base" tags: - python_dependencies - docker diff --git a/docker/base-slim/cpu.wanda.yaml b/docker/base-slim/cpu.wanda.yaml index b6d80f13992a..4276c103336e 100644 --- a/docker/base-slim/cpu.wanda.yaml +++ b/docker/base-slim/cpu.wanda.yaml @@ -1,4 +1,4 @@ -name: "ray-slim.py$PYTHON_VERSION.cpu.base$ARCH_SUFFIX" +name: "ray-slim-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX" froms: ["ubuntu:22.04"] dockerfile: docker/base-slim/Dockerfile srcs: @@ -7,4 +7,4 @@ build_args: - PYTHON_VERSION - BASE_IMAGE=ubuntu:22.04 tags: - - cr.ray.io/rayproject/ray-slim.py$PYTHON_VERSION.cpu.base$ARCH_SUFFIX + - cr.ray.io/rayproject/ray-slim-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX diff --git a/docker/base-slim/cuda.wanda.yaml b/docker/base-slim/cuda.wanda.yaml index 8ae98a48d11a..0c78b98f5c8b 100644 --- a/docker/base-slim/cuda.wanda.yaml +++ b/docker/base-slim/cuda.wanda.yaml @@ -1,4 +1,4 @@ -name: "ray-slim.py$PYTHON_VERSION.cu$CUDA_VERSION.base$ARCH_SUFFIX" +name: "ray-slim-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX" froms: ["nvidia/cuda:$CUDA_VERSION-runtime-ubuntu22.04"] dockerfile: docker/base-slim/Dockerfile srcs: @@ -7,4 +7,4 @@ build_args: - PYTHON_VERSION - BASE_IMAGE=nvidia/cuda:$CUDA_VERSION-runtime-ubuntu22.04 tags: - - cr.ray.io/rayproject/ray-slim.py$PYTHON_VERSION.cu$CUDA_VERSION.base$ARCH_SUFFIX + - cr.ray.io/rayproject/ray-slim-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX From 1d76e55d458fad81a9da5bdb363a34d9527c92d8 Mon Sep 17 00:00:00 2001 From: Timothy Seah Date: Wed, 3 Sep 2025 13:34:20 -0700 Subject: [PATCH 428/634] [train][doc] Add get_all_reported_checkpoints and ReportedCheckpoint to API docs (#56174) Signed-off-by: Timothy Seah --- doc/source/train/api/api.rst | 2 ++ python/ray/train/v2/api/train_fn_utils.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/train/api/api.rst b/doc/source/train/api/api.rst index 41e10d7c4a65..344682bd66f5 100644 --- a/doc/source/train/api/api.rst +++ b/doc/source/train/api/api.rst @@ -143,6 +143,7 @@ Ray Train Utilities :nosignatures: :toctree: doc/ + ~train.get_all_reported_checkpoints ~train.get_checkpoint ~train.get_context ~train.get_dataset_shard @@ -165,6 +166,7 @@ Ray Train Output :template: autosummary/class_without_autosummary.rst :toctree: doc/ + ~train.ReportedCheckpoint ~train.Result Ray Train Errors diff --git a/python/ray/train/v2/api/train_fn_utils.py b/python/ray/train/v2/api/train_fn_utils.py index 06603b8c409b..740206e190e2 100644 --- a/python/ray/train/v2/api/train_fn_utils.py +++ b/python/ray/train/v2/api/train_fn_utils.py @@ -192,7 +192,7 @@ def train_func(config): Returns: List of ReportedCheckpoint objects that represent the checkpoints and - corresponding metrics reported by the workers. + corresponding metrics reported by the workers. """ return get_train_fn_utils().get_all_reported_checkpoints() From ecb0d62662d9adab258981f420fabf176139e283 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Wed, 3 Sep 2025 13:37:55 -0700 Subject: [PATCH 429/634] Add perf metrics for 2.49.1 (#56183) ``` REGRESSION 29.81%: multi_client_put_calls_Plasma_Store (THROUGHPUT) regresses from 15560.620089508539 to 10922.349171697762 in microbenchmark.json REGRESSION 26.80%: client__1_1_actor_calls_async (THROUGHPUT) regresses from 1156.2740440851749 to 846.4118553774217 in microbenchmark.json REGRESSION 26.35%: multi_client_put_gigabytes (THROUGHPUT) regresses from 37.43614465888436 to 27.572292929404366 in microbenchmark.json REGRESSION 25.65%: client__tasks_and_put_batch (THROUGHPUT) regresses from 13581.806864962535 to 10098.586104880465 in microbenchmark.json REGRESSION 24.62%: client__1_1_actor_calls_concurrent (THROUGHPUT) regresses from 1144.7059953730677 to 862.8335019710298 in microbenchmark.json REGRESSION 22.61%: client__tasks_and_get_batch (THROUGHPUT) regresses from 1.040122205853533 to 0.8049748278618892 in microbenchmark.json REGRESSION 20.06%: multi_client_tasks_async (THROUGHPUT) regresses from 24135.499735285084 to 19294.747670848352 in microbenchmark.json REGRESSION 14.64%: 1_1_async_actor_calls_with_args_async (THROUGHPUT) regresses from 2842.6446804977995 to 2426.398157992012 in microbenchmark.json REGRESSION 14.49%: tasks_per_second (THROUGHPUT) regresses from 210.13682904062856 to 179.67755143550417 in benchmarks/many_nodes.json REGRESSION 14.46%: 1_1_async_actor_calls_async (THROUGHPUT) regresses from 4261.744798110754 to 3645.3291604906276 in microbenchmark.json REGRESSION 13.55%: client__1_1_actor_calls_sync (THROUGHPUT) regresses from 564.4854333177283 to 488.0060975075199 in microbenchmark.json REGRESSION 12.68%: 1_1_actor_calls_sync (THROUGHPUT) regresses from 2091.781722688228 to 1826.440590474467 in microbenchmark.json REGRESSION 11.51%: client__put_calls (THROUGHPUT) regresses from 869.4133022105747 to 769.3648317551028 in microbenchmark.json REGRESSION 9.32%: single_client_get_calls_Plasma_Store (THROUGHPUT) regresses from 10119.7301338237 to 9176.686326011131 in microbenchmark.json REGRESSION 9.17%: 1_1_actor_calls_concurrent (THROUGHPUT) regresses from 5185.592351945926 to 4710.115509639389 in microbenchmark.json REGRESSION 9.15%: single_client_put_calls_Plasma_Store (THROUGHPUT) regresses from 5278.091294531883 to 4795.051007052156 in microbenchmark.json REGRESSION 8.45%: 1_n_async_actor_calls_async (THROUGHPUT) regresses from 7607.418617152194 to 6964.257909926722 in microbenchmark.json REGRESSION 7.73%: n_n_async_actor_calls_async (THROUGHPUT) regresses from 23412.17782093146 to 21602.16598513169 in microbenchmark.json REGRESSION 6.78%: single_client_tasks_async (THROUGHPUT) regresses from 7958.403181954658 to 7418.67591750316 in microbenchmark.json REGRESSION 6.17%: n_n_actor_calls_async (THROUGHPUT) regresses from 26441.297568592032 to 24808.730524179864 in microbenchmark.json REGRESSION 5.46%: 1_1_actor_calls_async (THROUGHPUT) regresses from 8382.98999101571 to 7925.658042658907 in microbenchmark.json REGRESSION 5.16%: n_n_actor_calls_with_arg_async (THROUGHPUT) regresses from 2704.110675027102 to 2564.489147392739 in microbenchmark.json REGRESSION 4.76%: single_client_tasks_sync (THROUGHPUT) regresses from 946.027654634476 to 900.96738867954 in microbenchmark.json REGRESSION 4.58%: 1_1_async_actor_calls_sync (THROUGHPUT) regresses from 1440.0280310845594 to 1374.047824125402 in microbenchmark.json REGRESSION 4.10%: client__get_calls (THROUGHPUT) regresses from 1025.614536261544 to 983.5607099398597 in microbenchmark.json REGRESSION 2.83%: single_client_wait_1k_refs (THROUGHPUT) regresses from 4.952999927332959 to 4.8129125825624035 in microbenchmark.json REGRESSION 2.45%: 1_n_actor_calls_async (THROUGHPUT) regresses from 7753.148948018643 to 7563.474741840271 in microbenchmark.json REGRESSION 2.24%: single_client_tasks_and_get_batch (THROUGHPUT) regresses from 5.381854147597904 to 5.261194854317881 in microbenchmark.json REGRESSION 0.93%: single_client_get_object_containing_10k_refs (THROUGHPUT) regresses from 13.265651211414822 to 13.142098493341212 in microbenchmark.json REGRESSION 0.65%: placement_group_create/removal (THROUGHPUT) regresses from 755.9481741578835 to 751.064903521573 in microbenchmark.json REGRESSION 20.25%: dashboard_p95_latency_ms (LATENCY) regresses from 542.827 to 652.763 in benchmarks/many_tasks.json REGRESSION 19.04%: stage_4_spread (LATENCY) regresses from 0.4687484200099014 to 0.5580154959703073 in stress_tests/stress_test_many_tasks.json REGRESSION 12.64%: stage_0_time (LATENCY) regresses from 6.86789870262146 to 7.735846281051636 in stress_tests/stress_test_many_tasks.json REGRESSION 12.06%: dashboard_p50_latency_ms (LATENCY) regresses from 9.667 to 10.833 in benchmarks/many_actors.json REGRESSION 8.83%: avg_iteration_time (LATENCY) regresses from 1.1919621157646179 to 1.2971700072288512 in stress_tests/stress_test_dead_actors.json REGRESSION 8.70%: dashboard_p50_latency_ms (LATENCY) regresses from 6.38 to 6.935 in benchmarks/many_nodes.json REGRESSION 7.86%: dashboard_p95_latency_ms (LATENCY) regresses from 10.012 to 10.799 in benchmarks/many_pgs.json REGRESSION 5.87%: dashboard_p99_latency_ms (LATENCY) regresses from 736.395 to 779.606 in benchmarks/many_tasks.json REGRESSION 5.22%: avg_pg_remove_time_ms (LATENCY) regresses from 1.3276310030028873 to 1.396923734234321 in stress_tests/stress_test_placement_group.json REGRESSION 2.17%: dashboard_p95_latency_ms (LATENCY) regresses from 13.055 to 13.338 in benchmarks/many_nodes.json REGRESSION 1.58%: time_to_broadcast_1073741824_bytes_to_50_nodes (LATENCY) regresses from 13.201167912000003 to 13.41017694899999 in scalability/object_store.json REGRESSION 1.23%: avg_pg_create_time_ms (LATENCY) regresses from 1.544663453452921 to 1.5636188018035782 in stress_tests/stress_test_placement_group.json REGRESSION 1.02%: dashboard_p99_latency_ms (LATENCY) regresses from 186.207 to 188.103 in benchmarks/many_pgs.json REGRESSION 0.38%: stage_3_time (LATENCY) regresses from 1814.6457602977753 to 1821.4706330299377 in stress_tests/stress_test_many_tasks.json REGRESSION 0.25%: 107374182400_large_object_time (LATENCY) regresses from 31.283377702999985 to 31.36117459099995 in scalability/single_node.json REGRESSION 0.23%: 10000_get_time (LATENCY) regresses from 23.946038821000002 to 24.000713915999995 in scalability/single_node.json ``` Signed-off-by: Lonnie Liu Co-authored-by: Lonnie Liu --- .../perf_metrics/benchmarks/many_actors.json | 16 +- .../perf_metrics/benchmarks/many_nodes.json | 18 +- release/perf_metrics/benchmarks/many_pgs.json | 18 +- .../perf_metrics/benchmarks/many_tasks.json | 16 +- release/perf_metrics/metadata.json | 2 +- release/perf_metrics/microbenchmark.json | 186 +++++++++--------- .../scalability/object_store.json | 4 +- .../perf_metrics/scalability/single_node.json | 20 +- .../stress_tests/stress_test_dead_actors.json | 10 +- .../stress_tests/stress_test_many_tasks.json | 36 ++-- .../stress_test_placement_group.json | 8 +- 11 files changed, 167 insertions(+), 167 deletions(-) diff --git a/release/perf_metrics/benchmarks/many_actors.json b/release/perf_metrics/benchmarks/many_actors.json index cae597479e54..d41c48a42071 100644 --- a/release/perf_metrics/benchmarks/many_actors.json +++ b/release/perf_metrics/benchmarks/many_actors.json @@ -1,32 +1,32 @@ { - "_dashboard_memory_usage_mb": 102.928384, + "_dashboard_memory_usage_mb": 116.87936, "_dashboard_test_success": true, "_peak_memory": 4.56, - "_peak_process_memory": "PID\tMEM\tCOMMAND\n1147\t7.55GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3549\t2.03GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n4987\t0.83GiB\tpython distributed/test_many_actors.py\n3056\t0.39GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3751\t0.27GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n584\t0.19GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n4235\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3664\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n2979\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n4237\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru", - "actors_per_second": 526.6643292032776, + "_peak_process_memory": "PID\tMEM\tCOMMAND\n3537\t2.03GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n5556\t0.87GiB\tpython distributed/test_many_actors.py\n2945\t0.39GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3735\t0.23GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n582\t0.19GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n1143\t0.11GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3652\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n4226\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3108\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n4228\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru", + "actors_per_second": 566.4200586217125, "num_actors": 10000, "perf_metrics": [ { "perf_metric_name": "actors_per_second", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 526.6643292032776 + "perf_metric_value": 566.4200586217125 }, { "perf_metric_name": "dashboard_p50_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 9.667 + "perf_metric_value": 10.833 }, { "perf_metric_name": "dashboard_p95_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 2841.741 + "perf_metric_value": 2612.102 }, { "perf_metric_name": "dashboard_p99_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 3497.265 + "perf_metric_value": 3446.344 } ], "success": "1", - "time": 18.987426042556763 + "time": 17.654742002487183 } diff --git a/release/perf_metrics/benchmarks/many_nodes.json b/release/perf_metrics/benchmarks/many_nodes.json index 0b2e8ccb2f6a..a49b33405678 100644 --- a/release/perf_metrics/benchmarks/many_nodes.json +++ b/release/perf_metrics/benchmarks/many_nodes.json @@ -1,14 +1,14 @@ { - "_dashboard_memory_usage_mb": 95.510528, + "_dashboard_memory_usage_mb": 96.198656, "_dashboard_test_success": true, - "_peak_memory": 2.29, - "_peak_process_memory": "PID\tMEM\tCOMMAND\n3521\t0.51GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n3063\t0.26GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n1069\t0.19GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n5012\t0.17GiB\tpython distributed/test_many_tasks.py --num-tasks=1000\n3719\t0.14GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n4200\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3637\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n3019\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n5257\t0.09GiB\tray::StateAPIGeneratorActor.start\n4202\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru", + "_peak_memory": 2.28, + "_peak_process_memory": "PID\tMEM\tCOMMAND\n3348\t0.51GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2901\t0.27GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n4907\t0.17GiB\tpython distributed/test_many_tasks.py --num-tasks=1000\n3546\t0.13GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n1091\t0.13GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n4032\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n2825\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n5133\t0.09GiB\tray::StateAPIGeneratorActor.start\n3464\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n3549\t0.08GiB\tray-dashboard-StateHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import s", "num_tasks": 1000, "perf_metrics": [ { "perf_metric_name": "tasks_per_second", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 210.13682904062856 + "perf_metric_value": 179.67755143550417 }, { "perf_metric_name": "used_cpus_by_deadline", @@ -18,21 +18,21 @@ { "perf_metric_name": "dashboard_p50_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 6.38 + "perf_metric_value": 6.935 }, { "perf_metric_name": "dashboard_p95_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 13.055 + "perf_metric_value": 13.338 }, { "perf_metric_name": "dashboard_p99_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 38.335 + "perf_metric_value": 35.162 } ], "success": "1", - "tasks_per_second": 210.13682904062856, - "time": 304.7588040828705, + "tasks_per_second": 179.67755143550417, + "time": 305.5655255317688, "used_cpus": 250.0 } diff --git a/release/perf_metrics/benchmarks/many_pgs.json b/release/perf_metrics/benchmarks/many_pgs.json index 66b733a787ba..2df384ac4bda 100644 --- a/release/perf_metrics/benchmarks/many_pgs.json +++ b/release/perf_metrics/benchmarks/many_pgs.json @@ -1,32 +1,32 @@ { - "_dashboard_memory_usage_mb": 90.968064, + "_dashboard_memory_usage_mb": 98.287616, "_dashboard_test_success": true, - "_peak_memory": 2.88, - "_peak_process_memory": "PID\tMEM\tCOMMAND\n1111\t8.31GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3486\t0.98GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2906\t0.4GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n5166\t0.39GiB\tpython distributed/test_many_pgs.py\n581\t0.18GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n3690\t0.14GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n2765\t0.11GiB\t/app/go/infra/activityprobe/activityprobe ray --port=5903 --metrics_server_port=9092 --raylet_addr=l\n4176\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3039\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n4178\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru", + "_peak_memory": 2.78, + "_peak_process_memory": "PID\tMEM\tCOMMAND\n1146\t7.94GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n3546\t0.92GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n3049\t0.47GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n5004\t0.37GiB\tpython distributed/test_many_pgs.py\n581\t0.19GiB\t/app/go/infra/anyscaled/anyscaled_/anyscaled_shim --cloud_provider=aws\n3758\t0.13GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n2816\t0.11GiB\t/app/go/infra/activityprobe/activityprobe ray --port=5903 --metrics_server_port=9092 --raylet_addr=l\n4241\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3665\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n2941\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/", "num_pgs": 1000, "perf_metrics": [ { "perf_metric_name": "pgs_per_second", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 12.799625675009633 + "perf_metric_value": 13.028153672527967 }, { "perf_metric_name": "dashboard_p50_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 4.562 + "perf_metric_value": 4.26 }, { "perf_metric_name": "dashboard_p95_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 10.012 + "perf_metric_value": 10.799 }, { "perf_metric_name": "dashboard_p99_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 186.207 + "perf_metric_value": 188.103 } ], - "pgs_per_second": 12.799625675009633, + "pgs_per_second": 13.028153672527967, "success": "1", - "time": 78.12728476524353 + "time": 76.75684714317322 } diff --git a/release/perf_metrics/benchmarks/many_tasks.json b/release/perf_metrics/benchmarks/many_tasks.json index b16a12b02b26..cfaad9568f2d 100644 --- a/release/perf_metrics/benchmarks/many_tasks.json +++ b/release/perf_metrics/benchmarks/many_tasks.json @@ -1,14 +1,14 @@ { - "_dashboard_memory_usage_mb": 104.685568, + "_dashboard_memory_usage_mb": 104.443904, "_dashboard_test_success": true, "_peak_memory": 3.96, - "_peak_process_memory": "PID\tMEM\tCOMMAND\n3526\t1.11GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n8718\t0.76GiB\tpython distributed/test_many_tasks.py --num-tasks=10000\n3726\t0.45GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n3040\t0.28GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3729\t0.15GiB\tray-dashboard-StateHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import s\n1122\t0.12GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n4209\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3641\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n2964\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n9009\t0.09GiB\tray::StateAPIGeneratorActor.start", + "_peak_process_memory": "PID\tMEM\tCOMMAND\n3538\t1.1GiB\t/home/ray/anaconda3/lib/python3.9/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n5120\t0.76GiB\tpython distributed/test_many_tasks.py --num-tasks=10000\n3748\t0.46GiB\tray-dashboard-NodeHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import sp\n3008\t0.27GiB\tvector --watch-config --log-format json --config-yaml /etc/vector/vector.yaml\n3751\t0.19GiB\tray-dashboard-StateHead-0 (/home/ray/anaconda3/bin/python3.9 -c \"from multiprocessing.spawn import s\n1134\t0.11GiB\t/app/product/go/infra/anyscaled/anyscaled_/anyscaled startv2 --control_plane_url=https://console.any\n4231\t0.11GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/a\n3653\t0.1GiB\t/home/ray/anaconda3/bin/python3.9 /home/ray/anaconda3/lib/python3.9/site-packages/ray/dashboard/dash\n3029\t0.09GiB\t/usr/bin/python3 /app/infra/dataplane/webterminal/webterminal_sidecar_image.binary.runfiles/product/\n4233\t0.09GiB\t/home/ray/anaconda3/bin/python3.9 -u /home/ray/anaconda3/lib/python3.9/site-packages/ray/_private/ru", "num_tasks": 10000, "perf_metrics": [ { "perf_metric_name": "tasks_per_second", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 347.0074587793457 + "perf_metric_value": 388.36439061844453 }, { "perf_metric_name": "used_cpus_by_deadline", @@ -18,21 +18,21 @@ { "perf_metric_name": "dashboard_p50_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 7.278 + "perf_metric_value": 5.544 }, { "perf_metric_name": "dashboard_p95_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 542.827 + "perf_metric_value": 652.763 }, { "perf_metric_name": "dashboard_p99_latency_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 736.395 + "perf_metric_value": 779.606 } ], "success": "1", - "tasks_per_second": 347.0074587793457, - "time": 328.8178243637085, + "tasks_per_second": 388.36439061844453, + "time": 325.74901366233826, "used_cpus": 2500.0 } diff --git a/release/perf_metrics/metadata.json b/release/perf_metrics/metadata.json index aa69a1c1e5cc..bae7d94a16a3 100644 --- a/release/perf_metrics/metadata.json +++ b/release/perf_metrics/metadata.json @@ -1 +1 @@ -{"release_version": "2.49.0"} +{"release_version": "2.49.1"} diff --git a/release/perf_metrics/microbenchmark.json b/release/perf_metrics/microbenchmark.json index a58f3baee28e..30eb94e6f8a5 100644 --- a/release/perf_metrics/microbenchmark.json +++ b/release/perf_metrics/microbenchmark.json @@ -1,283 +1,283 @@ { "1_1_actor_calls_async": [ - 8382.98999101571, - 117.79830230521418 + 7925.658042658907, + 333.79803776770194 ], "1_1_actor_calls_concurrent": [ - 5185.592351945926, - 90.30753238314676 + 4710.115509639389, + 60.79075536787328 ], "1_1_actor_calls_sync": [ - 2091.781722688228, - 16.9518585087781 + 1826.440590474467, + 30.44826694257455 ], "1_1_async_actor_calls_async": [ - 4261.744798110754, - 180.0414155637879 + 3645.3291604906276, + 145.09649825222274 ], "1_1_async_actor_calls_sync": [ - 1440.0280310845594, - 32.16094111992805 + 1374.047824125402, + 35.86321385785778 ], "1_1_async_actor_calls_with_args_async": [ - 2842.6446804977995, - 86.87223380532768 + 2426.398157992012, + 38.78002524735766 ], "1_n_actor_calls_async": [ - 7753.148948018643, - 157.64457409741257 + 7563.474741840271, + 160.3419047539893 ], "1_n_async_actor_calls_async": [ - 7607.418617152194, - 89.2638829207031 + 6964.257909926722, + 53.3826400982145 ], "client__1_1_actor_calls_async": [ - 1156.2740440851749, - 16.48223271213866 + 846.4118553774217, + 21.74145942353796 ], "client__1_1_actor_calls_concurrent": [ - 1144.7059953730677, - 4.145550595536418 + 862.8335019710298, + 5.40969189845287 ], "client__1_1_actor_calls_sync": [ - 564.4854333177283, - 3.857233665824203 + 488.0060975075199, + 18.43611203884743 ], "client__get_calls": [ - 1025.614536261544, - 21.381968599178617 + 983.5607099398597, + 44.68614802894011 ], "client__put_calls": [ - 869.4133022105747, - 28.436164429083114 + 769.3648317551028, + 25.86986897843832 ], "client__put_gigabytes": [ - 0.1011154460629056, - 0.0022480613263383856 + 0.10294244610916167, + 0.00021781279103519403 ], "client__tasks_and_get_batch": [ - 1.040122205853533, - 0.03858532336305511 + 0.8049748278618892, + 0.0384792096927115 ], "client__tasks_and_put_batch": [ - 13581.806864962535, - 520.4510627044893 + 10098.586104880465, + 158.55761529403424 ], "multi_client_put_calls_Plasma_Store": [ - 15560.620089508539, - 96.2603254838798 + 10922.349171697762, + 411.8369713180647 ], "multi_client_put_gigabytes": [ - 37.43614465888436, - 2.1762237421647583 + 27.572292929404366, + 0.301414736739597 ], "multi_client_tasks_async": [ - 24135.499735285084, - 1599.8731087846127 + 19294.747670848352, + 1531.838851224768 ], "n_n_actor_calls_async": [ - 26441.297568592032, - 1050.529175955539 + 24808.730524179864, + 580.5120779930962 ], "n_n_actor_calls_with_arg_async": [ - 2704.110675027102, - 28.368337606599287 + 2564.489147392739, + 58.242925806948335 ], "n_n_async_actor_calls_async": [ - 23412.17782093146, - 617.218384004416 + 21602.16598513169, + 648.5971305332962 ], "perf_metrics": [ { "perf_metric_name": "single_client_get_calls_Plasma_Store", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 10119.7301338237 + "perf_metric_value": 9176.686326011131 }, { "perf_metric_name": "single_client_put_calls_Plasma_Store", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 5278.091294531883 + "perf_metric_value": 4795.051007052156 }, { "perf_metric_name": "multi_client_put_calls_Plasma_Store", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 15560.620089508539 + "perf_metric_value": 10922.349171697762 }, { "perf_metric_name": "single_client_put_gigabytes", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 18.71278038275444 + "perf_metric_value": 20.350152593657818 }, { "perf_metric_name": "single_client_tasks_and_get_batch", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 5.381854147597904 + "perf_metric_value": 5.261194854317881 }, { "perf_metric_name": "multi_client_put_gigabytes", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 37.43614465888436 + "perf_metric_value": 27.572292929404366 }, { "perf_metric_name": "single_client_get_object_containing_10k_refs", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 13.265651211414822 + "perf_metric_value": 13.142098493341212 }, { "perf_metric_name": "single_client_wait_1k_refs", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 4.952999927332959 + "perf_metric_value": 4.8129125825624035 }, { "perf_metric_name": "single_client_tasks_sync", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 946.027654634476 + "perf_metric_value": 900.96738867954 }, { "perf_metric_name": "single_client_tasks_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 7958.403181954658 + "perf_metric_value": 7418.67591750316 }, { "perf_metric_name": "multi_client_tasks_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 24135.499735285084 + "perf_metric_value": 19294.747670848352 }, { "perf_metric_name": "1_1_actor_calls_sync", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 2091.781722688228 + "perf_metric_value": 1826.440590474467 }, { "perf_metric_name": "1_1_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 8382.98999101571 + "perf_metric_value": 7925.658042658907 }, { "perf_metric_name": "1_1_actor_calls_concurrent", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 5185.592351945926 + "perf_metric_value": 4710.115509639389 }, { "perf_metric_name": "1_n_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 7753.148948018643 + "perf_metric_value": 7563.474741840271 }, { "perf_metric_name": "n_n_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 26441.297568592032 + "perf_metric_value": 24808.730524179864 }, { "perf_metric_name": "n_n_actor_calls_with_arg_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 2704.110675027102 + "perf_metric_value": 2564.489147392739 }, { "perf_metric_name": "1_1_async_actor_calls_sync", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 1440.0280310845594 + "perf_metric_value": 1374.047824125402 }, { "perf_metric_name": "1_1_async_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 4261.744798110754 + "perf_metric_value": 3645.3291604906276 }, { "perf_metric_name": "1_1_async_actor_calls_with_args_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 2842.6446804977995 + "perf_metric_value": 2426.398157992012 }, { "perf_metric_name": "1_n_async_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 7607.418617152194 + "perf_metric_value": 6964.257909926722 }, { "perf_metric_name": "n_n_async_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 23412.17782093146 + "perf_metric_value": 21602.16598513169 }, { "perf_metric_name": "placement_group_create/removal", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 755.9481741578835 + "perf_metric_value": 751.064903521573 }, { "perf_metric_name": "client__get_calls", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 1025.614536261544 + "perf_metric_value": 983.5607099398597 }, { "perf_metric_name": "client__put_calls", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 869.4133022105747 + "perf_metric_value": 769.3648317551028 }, { "perf_metric_name": "client__put_gigabytes", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 0.1011154460629056 + "perf_metric_value": 0.10294244610916167 }, { "perf_metric_name": "client__tasks_and_put_batch", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 13581.806864962535 + "perf_metric_value": 10098.586104880465 }, { "perf_metric_name": "client__1_1_actor_calls_sync", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 564.4854333177283 + "perf_metric_value": 488.0060975075199 }, { "perf_metric_name": "client__1_1_actor_calls_async", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 1156.2740440851749 + "perf_metric_value": 846.4118553774217 }, { "perf_metric_name": "client__1_1_actor_calls_concurrent", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 1144.7059953730677 + "perf_metric_value": 862.8335019710298 }, { "perf_metric_name": "client__tasks_and_get_batch", "perf_metric_type": "THROUGHPUT", - "perf_metric_value": 1.040122205853533 + "perf_metric_value": 0.8049748278618892 } ], "placement_group_create/removal": [ - 755.9481741578835, - 12.306329174758009 + 751.064903521573, + 5.332518268184338 ], "single_client_get_calls_Plasma_Store": [ - 10119.7301338237, - 102.04801817695024 + 9176.686326011131, + 202.59360315795405 ], "single_client_get_object_containing_10k_refs": [ - 13.265651211414822, - 0.01756980511554709 + 13.142098493341212, + 0.280827763090365 ], "single_client_put_calls_Plasma_Store": [ - 5278.091294531883, - 27.18593982260196 + 4795.051007052156, + 55.29886971022227 ], "single_client_put_gigabytes": [ - 18.71278038275444, - 8.13387701917967 + 20.350152593657818, + 6.284060239581299 ], "single_client_tasks_and_get_batch": [ - 5.381854147597904, - 3.1676227294432446 + 5.261194854317881, + 2.8864991514393927 ], "single_client_tasks_async": [ - 7958.403181954658, - 437.13498052024147 + 7418.67591750316, + 224.65732622349898 ], "single_client_tasks_sync": [ - 946.027654634476, - 12.53937510184865 + 900.96738867954, + 14.441231923805944 ], "single_client_wait_1k_refs": [ - 4.952999927332959, - 0.036988477470103795 + 4.8129125825624035, + 0.007111082814526685 ] } diff --git a/release/perf_metrics/scalability/object_store.json b/release/perf_metrics/scalability/object_store.json index a2e424a23abd..7d152e5a6d66 100644 --- a/release/perf_metrics/scalability/object_store.json +++ b/release/perf_metrics/scalability/object_store.json @@ -1,12 +1,12 @@ { - "broadcast_time": 13.201167912000003, + "broadcast_time": 13.41017694899999, "num_nodes": 50, "object_size": 1073741824, "perf_metrics": [ { "perf_metric_name": "time_to_broadcast_1073741824_bytes_to_50_nodes", "perf_metric_type": "LATENCY", - "perf_metric_value": 13.201167912000003 + "perf_metric_value": 13.41017694899999 } ], "success": "1" diff --git a/release/perf_metrics/scalability/single_node.json b/release/perf_metrics/scalability/single_node.json index be655d551555..57cdf632f646 100644 --- a/release/perf_metrics/scalability/single_node.json +++ b/release/perf_metrics/scalability/single_node.json @@ -1,8 +1,8 @@ { - "args_time": 19.17184469900002, - "get_time": 23.946038821000002, + "args_time": 19.077259766999987, + "get_time": 24.000713915999995, "large_object_size": 107374182400, - "large_object_time": 31.283377702999985, + "large_object_time": 31.36117459099995, "num_args": 10000, "num_get_args": 10000, "num_queued": 1000000, @@ -11,30 +11,30 @@ { "perf_metric_name": "10000_args_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 19.17184469900002 + "perf_metric_value": 19.077259766999987 }, { "perf_metric_name": "3000_returns_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 5.863585175999987 + "perf_metric_value": 5.790547841000006 }, { "perf_metric_name": "10000_get_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 23.946038821000002 + "perf_metric_value": 24.000713915999995 }, { "perf_metric_name": "1000000_queued_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 190.35062810999997 + "perf_metric_value": 179.146127773 }, { "perf_metric_name": "107374182400_large_object_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 31.283377702999985 + "perf_metric_value": 31.36117459099995 } ], - "queued_time": 190.35062810999997, - "returns_time": 5.863585175999987, + "queued_time": 179.146127773, + "returns_time": 5.790547841000006, "success": "1" } diff --git a/release/perf_metrics/stress_tests/stress_test_dead_actors.json b/release/perf_metrics/stress_tests/stress_test_dead_actors.json index 98efeb61e846..991e91d96aec 100644 --- a/release/perf_metrics/stress_tests/stress_test_dead_actors.json +++ b/release/perf_metrics/stress_tests/stress_test_dead_actors.json @@ -1,14 +1,14 @@ { - "avg_iteration_time": 1.1919621157646179, - "max_iteration_time": 3.34515118598938, - "min_iteration_time": 0.6364881992340088, + "avg_iteration_time": 1.2971700072288512, + "max_iteration_time": 5.189502000808716, + "min_iteration_time": 0.06091117858886719, "perf_metrics": [ { "perf_metric_name": "avg_iteration_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 1.1919621157646179 + "perf_metric_value": 1.2971700072288512 } ], "success": 1, - "total_time": 119.196359872818 + "total_time": 129.71714234352112 } diff --git a/release/perf_metrics/stress_tests/stress_test_many_tasks.json b/release/perf_metrics/stress_tests/stress_test_many_tasks.json index 1951e0e15b5e..bee0d6200cac 100644 --- a/release/perf_metrics/stress_tests/stress_test_many_tasks.json +++ b/release/perf_metrics/stress_tests/stress_test_many_tasks.json @@ -3,45 +3,45 @@ { "perf_metric_name": "stage_0_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 6.86789870262146 + "perf_metric_value": 7.735846281051636 }, { "perf_metric_name": "stage_1_avg_iteration_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 13.265494346618652 + "perf_metric_value": 12.93162693977356 }, { "perf_metric_name": "stage_2_avg_iteration_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 34.152964401245114 + "perf_metric_value": 33.983641386032104 }, { "perf_metric_name": "stage_3_creation_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 2.032559394836426 + "perf_metric_value": 1.8725192546844482 }, { "perf_metric_name": "stage_3_time", "perf_metric_type": "LATENCY", - "perf_metric_value": 1814.6457602977753 + "perf_metric_value": 1821.4706330299377 }, { "perf_metric_name": "stage_4_spread", "perf_metric_type": "LATENCY", - "perf_metric_value": 0.4687484200099014 + "perf_metric_value": 0.5580154959703073 } ], - "stage_0_time": 6.86789870262146, - "stage_1_avg_iteration_time": 13.265494346618652, - "stage_1_max_iteration_time": 13.843246221542358, - "stage_1_min_iteration_time": 11.710993766784668, - "stage_1_time": 132.65499782562256, - "stage_2_avg_iteration_time": 34.152964401245114, - "stage_2_max_iteration_time": 34.738978147506714, - "stage_2_min_iteration_time": 33.831342458724976, - "stage_2_time": 170.76539039611816, - "stage_3_creation_time": 2.032559394836426, - "stage_3_time": 1814.6457602977753, - "stage_4_spread": 0.4687484200099014, + "stage_0_time": 7.735846281051636, + "stage_1_avg_iteration_time": 12.93162693977356, + "stage_1_max_iteration_time": 13.44619870185852, + "stage_1_min_iteration_time": 11.569173812866211, + "stage_1_time": 129.31632256507874, + "stage_2_avg_iteration_time": 33.983641386032104, + "stage_2_max_iteration_time": 34.43809151649475, + "stage_2_min_iteration_time": 33.45232319831848, + "stage_2_time": 169.91874861717224, + "stage_3_creation_time": 1.8725192546844482, + "stage_3_time": 1821.4706330299377, + "stage_4_spread": 0.5580154959703073, "success": 1 } diff --git a/release/perf_metrics/stress_tests/stress_test_placement_group.json b/release/perf_metrics/stress_tests/stress_test_placement_group.json index 49a763bceb42..d70d74c39e18 100644 --- a/release/perf_metrics/stress_tests/stress_test_placement_group.json +++ b/release/perf_metrics/stress_tests/stress_test_placement_group.json @@ -1,16 +1,16 @@ { - "avg_pg_create_time_ms": 1.544663453452921, - "avg_pg_remove_time_ms": 1.3276310030028873, + "avg_pg_create_time_ms": 1.5636188018035782, + "avg_pg_remove_time_ms": 1.396923734234321, "perf_metrics": [ { "perf_metric_name": "avg_pg_create_time_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 1.544663453452921 + "perf_metric_value": 1.5636188018035782 }, { "perf_metric_name": "avg_pg_remove_time_ms", "perf_metric_type": "LATENCY", - "perf_metric_value": 1.3276310030028873 + "perf_metric_value": 1.396923734234321 } ], "success": 1 From 0bda55aa3ff3e7ef03024cb3255eb518ae8e2c2a Mon Sep 17 00:00:00 2001 From: goutamvenkat-anyscale Date: Wed, 3 Sep 2025 13:41:02 -0700 Subject: [PATCH 430/634] [Data] - Evaluate Expressions sequentially and upsert columns to block (#56193) ## Why are these changes needed? Previously, the columns were updated after all expressions were evaluated, but this required additional handling in the planner and optimization to handle dependency chaining between columns. In this implementation, each expression will be evaluated and the column will be upserted to the block within the provided Project operator. ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Goutam V --- python/ray/data/_internal/arrow_block.py | 13 +++++++++++++ python/ray/data/_internal/pandas_block.py | 8 ++++++++ .../ray/data/_internal/planner/plan_udf_map_op.py | 15 +++++---------- python/ray/data/block.py | 13 +++++++++++++ 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/python/ray/data/_internal/arrow_block.py b/python/ray/data/_internal/arrow_block.py index 8f8058cc58c0..9714ecaab709 100644 --- a/python/ray/data/_internal/arrow_block.py +++ b/python/ray/data/_internal/arrow_block.py @@ -335,6 +335,19 @@ def _zip(self, acc: BlockAccessor) -> "Block": r = r.append_column(col_name, col) return r + def upsert_column( + self, column_name: str, column_data: BlockColumn + ) -> "pyarrow.Table": + assert isinstance( + column_data, (pyarrow.Array, pyarrow.ChunkedArray) + ), f"Expected either a pyarrow.Array or pyarrow.ChunkedArray, got: {type(column_data)}" + + column_idx = self._table.schema.get_field_index(column_name) + if column_idx == -1: + return self._table.append_column(column_name, column_data) + else: + return self._table.set_column(column_idx, column_name, column_data) + @staticmethod def builder() -> ArrowBlockBuilder: return ArrowBlockBuilder() diff --git a/python/ray/data/_internal/pandas_block.py b/python/ray/data/_internal/pandas_block.py index a95ba56691e1..1c82b10cc5c5 100644 --- a/python/ray/data/_internal/pandas_block.py +++ b/python/ray/data/_internal/pandas_block.py @@ -317,6 +317,14 @@ def select(self, columns: List[str]) -> "pandas.DataFrame": def rename_columns(self, columns_rename: Dict[str, str]) -> "pandas.DataFrame": return self._table.rename(columns=columns_rename, inplace=False, copy=False) + def upsert_column( + self, column_name: str, column_data: BlockColumn + ) -> "pandas.DataFrame": + if isinstance(column_data, (pyarrow.Array, pyarrow.ChunkedArray)): + column_data = column_data.to_pandas() + + return self._table.assign(**{column_name: column_data}) + def random_shuffle(self, random_seed: Optional[int]) -> "pandas.DataFrame": table = self._table.sample(frac=1, random_state=random_seed) table.reset_index(drop=True, inplace=True) diff --git a/python/ray/data/_internal/planner/plan_udf_map_op.py b/python/ray/data/_internal/planner/plan_udf_map_op.py index a27a9a2ec9fb..3809bcafd8a5 100644 --- a/python/ray/data/_internal/planner/plan_udf_map_op.py +++ b/python/ray/data/_internal/planner/plan_udf_map_op.py @@ -128,19 +128,14 @@ def fn(block: Block) -> Block: # 1. evaluate / add expressions if exprs: block_accessor = BlockAccessor.for_block(block) - new_columns = {} - for col_name in block_accessor.column_names(): - # For Arrow blocks, block[col_name] gives us a ChunkedArray - # For Pandas blocks, block[col_name] gives us a Series - new_columns[col_name] = block[col_name] - # Add/update with expression results + result_block = block for name, expr in exprs.items(): - result = eval_expr(expr, block) - new_columns[name] = result + result = eval_expr(expr, result_block) + result_block_accessor = BlockAccessor.for_block(result_block) + result_block = result_block_accessor.upsert_column(name, result) - # Create a new block from the combined columns and add it - block = BlockAccessor.batch_to_block(new_columns) + block = result_block # 2. (optional) column projection if columns: diff --git a/python/ray/data/block.py b/python/ray/data/block.py index 073033d1ef1e..6cbeeeae387b 100644 --- a/python/ray/data/block.py +++ b/python/ray/data/block.py @@ -343,6 +343,19 @@ def rename_columns(self, columns_rename: Dict[str, str]) -> Block: """Return the block reflecting the renamed columns.""" raise NotImplementedError + def upsert_column(self, column_name: str, column_data: BlockColumn) -> Block: + """ + Upserts a column into the block. If the column already exists, it will be replaced. + + Args: + column_name: The name of the column to upsert. + column_data: The data to upsert into the column. (Arrow Array/ChunkedArray for Arrow blocks, Series or array-like for Pandas blocks) + + Returns: + The updated block. + """ + raise NotImplementedError() + def random_shuffle(self, random_seed: Optional[int]) -> Block: """Randomly shuffle this block.""" raise NotImplementedError From d9959401c0f93b45da5ec9dc4f9a9b416b409193 Mon Sep 17 00:00:00 2001 From: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Date: Wed, 3 Sep 2025 14:06:56 -0700 Subject: [PATCH 431/634] [serve.llm] Remove upstreamed workarounds 1/3 (#54512) Signed-off-by: Seiji Eicher Signed-off-by: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> --- .../llm/_internal/serve/deployments/llm/vllm/vllm_engine.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py index 9df60582bf2a..b1e9457c8e74 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py @@ -4,7 +4,6 @@ from starlette.datastructures import State from starlette.requests import Request -from transformers.dynamic_module_utils import init_hf_modules from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.openai.cli_args import FrontendArgs from vllm.entrypoints.openai.protocol import ErrorResponse as VLLMErrorResponse @@ -110,10 +109,6 @@ def __init__( """ super().__init__(llm_config) - # Ensure transformers_modules is initialized early in worker processes. - # This is critical for models with trust_remote_code=True to avoid pickle errors. - init_hf_modules() - self.llm_config = llm_config if vllm is None: From 0fc0ed105c1dfd648d2bad7b5ea11bddf4b72304 Mon Sep 17 00:00:00 2001 From: Hassam Ullah Sheikh Date: Wed, 3 Sep 2025 17:48:19 -0400 Subject: [PATCH 432/634] [docs] Fix/docs key concepts (#56083) --- doc/source/_static/css/custom.css | 14 ++++++++++++++ doc/source/rllib/key-concepts.rst | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/doc/source/_static/css/custom.css b/doc/source/_static/css/custom.css index 24ed4992b2ba..4d685fdb4dea 100644 --- a/doc/source/_static/css/custom.css +++ b/doc/source/_static/css/custom.css @@ -408,4 +408,18 @@ readthedocs-flyout { /* Styling the experimental Anyscale upsell CTA */ .anyscale-cta { margin-bottom: 16px; +} + + +/* Prevent text wrapping around left-aligned images on ultra-wide screens */ +@media (min-width: 1600px) { + .bd-content .align-left, + .bd-content .figure.align-left, + .bd-content img.align-left { + float: none !important; + display: block; + clear: both; + margin-left: 0 !important; + margin-right: 0 !important; + } } \ No newline at end of file diff --git a/doc/source/rllib/key-concepts.rst b/doc/source/rllib/key-concepts.rst index 18bbb82f91fa..6d89d5ea9154 100644 --- a/doc/source/rllib/key-concepts.rst +++ b/doc/source/rllib/key-concepts.rst @@ -142,7 +142,7 @@ and the rules that govern environment transitions when applying actions. A simple **RL environment** where an agent starts with an initial observation returned by the ``reset()`` method. The agent, possibly controlled by a neural network policy, sends actions, like ``right`` or ``jump``, - to the environmant's ``step()`` method, which returns a reward. Here, the reward values are +5 for reaching the goal + to the environment's ``step()`` method, which returns a reward. Here, the reward values are +5 for reaching the goal and 0 otherwise. The environment also returns a boolean flag indicating whether the episode is complete. Environments may vary in complexity, from simple tasks, like navigating a grid world, to highly intricate systems, like autonomous @@ -184,7 +184,7 @@ network models and defines how to use them during the three phases of its RL lif **Exploration**, for collecting training data, **inference** when computing actions for evaluation or in production, and **training** for computing the loss function inputs. -You can chose to use :ref:`RLlib's built-in default models and configure these ` as needed, +You can choose to use :ref:`RLlib's built-in default models and configure these ` as needed, for example for changing the number of layers or the activation functions, or :ref:`write your own custom models in PyTorch `, allowing you to implement any architecture and computation logic. From 4b9e5575284888b00e865a6f79f2b506521b55d8 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Wed, 3 Sep 2025 14:54:48 -0700 Subject: [PATCH 433/634] [ci] Raydepsets: Enable defining packages in depset configs (#56169) Enable defining explicit packages in depset config - Added unit tests - Made requirements optional (relying on uv to throw errors if requirements / packages are not present in a depset) - added stdin var to pass packages as stdin to the uv command These changes allow a user to define ray[all]==100.0.0-dev as a requirement in the depset config which removes the need to create temporary requirements files and echoing packages into them --------- Signed-off-by: elliot-barn --- ci/raydepsets/cli.py | 21 ++++++++--- ci/raydepsets/tests/test_cli.py | 62 +++++++++++++++++++++++++-------- ci/raydepsets/workspace.py | 10 +++--- 3 files changed, 71 insertions(+), 22 deletions(-) diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index 70d5287ab680..f7aa52c316f5 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -105,10 +105,12 @@ def execute(self): depset = self.build_graph.nodes[node]["depset"] self.execute_single(depset) - def exec_uv_cmd(self, cmd: str, args: List[str]) -> str: + def exec_uv_cmd( + self, cmd: str, args: List[str], stdin: Optional[bytes] = None + ) -> str: cmd = [self._uv_binary, "pip", cmd, *args] click.echo(f"Executing command: {cmd}") - status = subprocess.run(cmd, cwd=self.workspace.dir) + status = subprocess.run(cmd, cwd=self.workspace.dir, input=stdin) if status.returncode != 0: raise RuntimeError(f"Failed to execute command: {cmd}") return status.stdout @@ -122,6 +124,7 @@ def execute_single(self, depset: Depset): output=depset.output, append_flags=depset.append_flags, override_flags=depset.override_flags, + packages=depset.packages, ) elif depset.operation == "subset": self.subset( @@ -147,14 +150,16 @@ def execute_single(self, depset: Depset): def compile( self, constraints: List[str], - requirements: List[str], name: str, output: str, append_flags: Optional[List[str]] = None, override_flags: Optional[List[str]] = None, + packages: Optional[List[str]] = None, + requirements: Optional[List[str]] = None, ): """Compile a dependency set.""" args = DEFAULT_UV_FLAGS.copy() + stdin = None if self._uv_cache_dir: args.extend(["--cache-dir", self._uv_cache_dir]) if override_flags: @@ -167,9 +172,13 @@ def compile( if requirements: for requirement in requirements: args.extend([requirement]) + if packages: + # need to add a dash to process stdin + args.append("-") + stdin = _get_bytes(packages) if output: args.extend(["-o", output]) - self.exec_uv_cmd("compile", args) + self.exec_uv_cmd("compile", args, stdin) def subset( self, @@ -230,6 +239,10 @@ def check_subset_exists(self, source_depset: Depset, requirements: List[str]): ) +def _get_bytes(packages: List[str]) -> bytes: + return ("\n".join(packages) + "\n").encode("utf-8") + + def _get_depset(depsets: List[Depset], name: str) -> Depset: for depset in depsets: if depset.name == name: diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 052a34e0de40..dc57eda57e52 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -1,4 +1,3 @@ -import shutil import subprocess import sys import tempfile @@ -97,20 +96,12 @@ def test_uv_version(self): assert result.stderr.decode("utf-8") == "" def test_compile(self): - compiled_file = Path( - _runfiles.Rlocation( - f"{_REPO_NAME}/ci/raydepsets/tests/test_data/requirements_compiled_test.txt" - ) - ) - output_file = Path( - _runfiles.Rlocation( - f"{_REPO_NAME}/ci/raydepsets/tests/test_data/requirements_compiled.txt" - ) - ) - shutil.copy(compiled_file, output_file) - with tempfile.TemporaryDirectory() as tmpdir: copy_data_to_tmpdir(tmpdir) + save_file_as( + Path(tmpdir) / "requirements_compiled_test.txt", + Path(tmpdir) / "requirements_compiled.txt", + ) manager = _create_test_manager(tmpdir) manager.compile( constraints=["requirement_constraints_test.txt"], @@ -135,7 +126,7 @@ def test_compile_update_package(self): output_file = Path( _runfiles.Rlocation(f"{tmpdir}/requirements_compiled.txt") ) - shutil.copy(compiled_file, output_file) + save_file_as(compiled_file, output_file) manager = _create_test_manager(tmpdir) manager.compile( constraints=["requirement_constraints_test.txt"], @@ -559,6 +550,49 @@ def test_get_depset_with_build_arg_set_and_no_build_arg_set_provided(self): with self.assertRaises(KeyError): _get_depset(manager.config.depsets, "build_args_test_depset_py311") + def test_compile_with_packages(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + save_file_as( + Path(tmpdir) / "requirements_compiled_test.txt", + Path(tmpdir) / "requirements_compiled_test_packages.txt", + ) + manager = _create_test_manager(tmpdir) + manager.compile( + constraints=["requirement_constraints_test.txt"], + packages=["emoji==2.9.0", "pyperclip==1.6.0"], + append_flags=["--no-annotate", "--no-header"], + name="packages_test_depset", + output="requirements_compiled_test_packages.txt", + ) + output_file = Path(tmpdir) / "requirements_compiled_test_packages.txt" + output_text = output_file.read_text() + output_file_valid = Path(tmpdir) / "requirements_compiled_test.txt" + output_text_valid = output_file_valid.read_text() + assert output_text == output_text_valid + + def test_compile_with_packages_and_requirements(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + save_file_as( + Path(tmpdir) / "requirements_compiled_test.txt", + Path(tmpdir) / "requirements_compiled_test_packages.txt", + ) + manager = _create_test_manager(tmpdir) + manager.compile( + constraints=["requirement_constraints_test.txt"], + packages=["emoji==2.9.0", "pyperclip==1.6.0"], + requirements=["requirements_test.txt"], + append_flags=["--no-annotate", "--no-header"], + name="packages_test_depset", + output="requirements_compiled_test_packages.txt", + ) + output_file = Path(tmpdir) / "requirements_compiled_test_packages.txt" + output_text = output_file.read_text() + output_file_valid = Path(tmpdir) / "requirements_compiled_test.txt" + output_text_valid = output_file_valid.read_text() + assert output_text == output_text_valid + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/ci/raydepsets/workspace.py b/ci/raydepsets/workspace.py index 68d0f9f9bcf6..46d9f74223f0 100644 --- a/ci/raydepsets/workspace.py +++ b/ci/raydepsets/workspace.py @@ -15,11 +15,12 @@ class BuildArgSet: class Depset: name: str operation: str - requirements: List[str] - constraints: List[str] output: str - override_flags: List[str] - append_flags: List[str] + constraints: Optional[List[str]] = None + override_flags: Optional[List[str]] = None + append_flags: Optional[List[str]] = None + requirements: Optional[List[str]] = None + packages: Optional[List[str]] = None source_depset: Optional[str] = None depsets: Optional[List[str]] = None @@ -49,6 +50,7 @@ def _dict_to_depset(depset: dict) -> Depset: depsets=depset.get("depsets", []), override_flags=depset.get("override_flags", []), append_flags=depset.get("append_flags", []), + packages=depset.get("packages", []), ) From 31430716de733de53fb00cceeabe810ccfeaac07 Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Wed, 3 Sep 2025 15:46:54 -0700 Subject: [PATCH 434/634] [data] remove metadata for hashing + truncate warning logs (#56093) ## Why are these changes needed? We need schemas to be hashable for schema deduplication. We previously removed metadata in Refbundle Creation, however, it can be called without a refbundle. For example, it can happen in delegating block builder ``` def concat( blocks: List["pyarrow.Table"], *, promote_types: bool = False ) -> "pyarrow.Table": ``` or implicity called when calling `count` on a dataset ``` def _cached_output_metadata # will grab all the metadata(including schema), not just rows ``` or in `BlockOutputBuffer` - This PR also reduces the log warning to truncate too. To centralize, added it in unify_schemas ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- python/ray/data/_internal/arrow_ops/transform_pyarrow.py | 5 ++++- .../ray/data/_internal/execution/interfaces/ref_bundle.py | 7 ------- .../data/_internal/execution/streaming_executor_state.py | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py index 04f8b5468f3f..ae0c72389c07 100644 --- a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py +++ b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py @@ -179,13 +179,16 @@ def unify_schemas( ArrowVariableShapedTensorType, ) + # The schema metadata might be unhashable. + # We need schemas to be hashable for unification + schemas = [schema.remove_metadata() for schema in schemas] try: if len(set(schemas)) == 1: # Early exit because unifying can be expensive return schemas.pop() except Exception as e: # Unsure if there are cases where schemas are NOT hashable - logger.warning(f"Failed to hash the schemas (for deduplication): {e}") + logger.debug(f"Failed to hash the schemas (for deduplication): {e}") schemas_to_unify = [] schema_field_overrides = {} diff --git a/python/ray/data/_internal/execution/interfaces/ref_bundle.py b/python/ray/data/_internal/execution/interfaces/ref_bundle.py index 310acf160b07..50c905803d2a 100644 --- a/python/ray/data/_internal/execution/interfaces/ref_bundle.py +++ b/python/ray/data/_internal/execution/interfaces/ref_bundle.py @@ -63,13 +63,6 @@ def __post_init__(self): "The size in bytes of the block must be known: {}".format(b) ) - import pyarrow as pa - - # The schema metadata might be unhashable. - # We need schemas to be hashable for unification - if isinstance(self.schema, pa.lib.Schema): - self.schema = self.schema.remove_metadata() - def __setattr__(self, key, value): if hasattr(self, key) and key in ["blocks", "owns_blocks"]: raise ValueError(f"The `{key}` field of RefBundle cannot be updated.") diff --git a/python/ray/data/_internal/execution/streaming_executor_state.py b/python/ray/data/_internal/execution/streaming_executor_state.py index bd4634900651..c3d7ae8e95dc 100644 --- a/python/ray/data/_internal/execution/streaming_executor_state.py +++ b/python/ray/data/_internal/execution/streaming_executor_state.py @@ -790,7 +790,7 @@ def dedupe_schemas_with_validation( return bundle, diverged diverged = True - if warn: + if warn and enforce_schemas: logger.warning( f"Operator produced a RefBundle with a different schema " f"than the previous one. Previous schema: {old_schema}, " From 9dbb7a3ddca6b3b68b9630f5a7811dd02dc4b297 Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Wed, 3 Sep 2025 16:33:44 -0700 Subject: [PATCH 435/634] [Core] Fix enable_tensor_transport in gpu microbenchmark (#56210) --- .../microbenchmark/experimental/gpu_object_microbenchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/microbenchmark/experimental/gpu_object_microbenchmark.py b/release/microbenchmark/experimental/gpu_object_microbenchmark.py index 67deb9cb05c7..bded6c4acc76 100644 --- a/release/microbenchmark/experimental/gpu_object_microbenchmark.py +++ b/release/microbenchmark/experimental/gpu_object_microbenchmark.py @@ -27,7 +27,7 @@ class BackendConfig: BACKEND_CONFIG = { "gloo": BackendConfig( - init_actor_kwargs={"enable_tensor_transport": True}, + init_actor_kwargs={}, send_method_kwargs={"tensor_transport": "gloo"}, device=torch.device("cpu"), collective_group_backend="torch_gloo", @@ -51,7 +51,7 @@ class BackendConfig: } -@ray.remote +@ray.remote(enable_tensor_transport=True) class Actor: def __init__( self, From 729668b6c939493c047ae2c2e345686e64561705 Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Wed, 3 Sep 2025 16:34:03 -0700 Subject: [PATCH 436/634] [serve] Add accessor for `DeploymentResponse`'s `_by_reference` value (#56199) ## Why are these changes needed? Add accessor for `DeploymentResponse`'s `_by_reference` value ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: akyang-anyscale --- python/ray/serve/handle.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/ray/serve/handle.py b/python/ray/serve/handle.py index 929cb6682c27..f5ecf3661d83 100644 --- a/python/ray/serve/handle.py +++ b/python/ray/serve/handle.py @@ -270,6 +270,10 @@ def __init__( def request_id(self) -> str: return self._request_metadata.request_id + @property + def by_reference(self) -> bool: + return self._request_metadata._by_reference + def _fetch_future_result_sync( self, _timeout_s: Optional[float] = None ) -> ReplicaResult: From 117a6421278cbbdf54d63de647df6667bb6e46f0 Mon Sep 17 00:00:00 2001 From: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Date: Wed, 3 Sep 2025 16:59:10 -0700 Subject: [PATCH 437/634] [Train] Add local mode support to Ray Train v2 (num_workers=0) (#55487) This PR implements local mode support for Ray Train v2, allowing training functions to run in the same process when `num_workers=0` is specified in the `ScalingConfig`. This feature provides a lightweight alternative for development, debugging, and single-node training scenarios. ### Key Changes: #### Core Infrastructure - **Abstract TrainContext**: Refactored `TrainContext` to be an abstract base class with concrete implementations for distributed and local modes - Created `LocalTrainContext` and `DistributedTrainContext` implementations - **TrainFnUtils Implementations**: - `DistributedTrainFnUtils`: Handles distributed training scenarios - `LocalTrainFnUtils`: Provides local mode implementation with simulated distributed training APIs - **LocalController**: New controller class for managing local mode training execution #### API Enhancements - **ScalingConfig**: Added support for `num_workers=0` with informational logging - **DataParallelTrainer**: Enhanced to detect local mode and route to appropriate controller #### Framework Integration - **PyTorch**: Updated device detection to work correctly in local mode - **Other Frameworks**: Full support for TensorFlow, Lightning, LightGBM, JAX, Xgboost, and HuggingFace Transformers #### Testing - **Comprehensive Test Suite**: Added extensive tests covering all supported frameworks in local mode - **Edge Cases**: Validation of metrics reporting, checkpointing, and dataset handling ### Usage Example: ```python from ray.train import ScalingConfig from ray.train.v2.api.data_parallel_trainer import DataParallelTrainer def train_fn(): # Your training code here ray.train.report({"accuracy": 0.95}) trainer = DataParallelTrainer( train_fn, scaling_config=ScalingConfig(num_workers=0) # Local mode ) result = trainer.fit() ``` --------- Signed-off-by: xgui Signed-off-by: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Signed-off-by: matthewdeng Co-authored-by: matthewdeng --- .gitignore | 4 + python/ray/train/torch/__init__.py | 2 + python/ray/train/v2/BUILD | 16 + .../_internal/execution/local_mode_utils.py | 40 ++ .../v2/_internal/execution/train_fn_utils.py | 150 ++++- .../execution/worker_group/worker.py | 4 +- python/ray/train/v2/api/config.py | 11 +- python/ray/train/v2/api/context.py | 81 ++- .../ray/train/v2/api/data_parallel_trainer.py | 66 ++- python/ray/train/v2/tests/test_local_mode.py | 522 ++++++++++++++++++ python/ray/train/v2/torch/train_loop_utils.py | 24 +- python/ray/train/v2/xgboost/__init__.py | 2 + python/ray/train/v2/xgboost/config.py | 21 + python/ray/train/xgboost/__init__.py | 1 + 14 files changed, 889 insertions(+), 55 deletions(-) create mode 100644 python/ray/train/v2/_internal/execution/local_mode_utils.py create mode 100644 python/ray/train/v2/tests/test_local_mode.py create mode 100644 python/ray/train/v2/xgboost/config.py diff --git a/.gitignore b/.gitignore index 81d26138d2cf..5e7bbfa27cfa 100644 --- a/.gitignore +++ b/.gitignore @@ -154,6 +154,10 @@ scripts/nodes.txt .benchmarks python-driver-* +# Ray Train unit test artifacts +lightning_logs/ +hf-internal-testing/ + # Vscode .vscode/ diff --git a/python/ray/train/torch/__init__.py b/python/ray/train/torch/__init__.py index db989336afd1..1774b98cb18a 100644 --- a/python/ray/train/torch/__init__.py +++ b/python/ray/train/torch/__init__.py @@ -30,6 +30,8 @@ accelerate, backward, enable_reproducibility, + get_device, + get_devices, prepare_data_loader, prepare_model, prepare_optimizer, diff --git a/python/ray/train/v2/BUILD b/python/ray/train/v2/BUILD index de2284e3a96e..7ec0bed9018e 100644 --- a/python/ray/train/v2/BUILD +++ b/python/ray/train/v2/BUILD @@ -516,3 +516,19 @@ py_test( "//:ray_lib", ], ) + +py_test( + name = "test_local_mode", + size = "medium", + srcs = ["tests/test_local_mode.py"], + env = {"RAY_TRAIN_V2_ENABLED": "1"}, + tags = [ + "exclusive", + "team:ml", + "train_v2", + ], + deps = [ + ":conftest", + "//:ray_lib", + ], +) diff --git a/python/ray/train/v2/_internal/execution/local_mode_utils.py b/python/ray/train/v2/_internal/execution/local_mode_utils.py new file mode 100644 index 000000000000..06a1d12627ac --- /dev/null +++ b/python/ray/train/v2/_internal/execution/local_mode_utils.py @@ -0,0 +1,40 @@ +import logging +from typing import Callable, Dict, Optional + +from ray.train import Result +from ray.train.trainer import GenDataset +from ray.train.v2._internal.execution.train_fn_utils import ( + LocalTrainFnUtils, + get_train_fn_utils, + set_train_fn_utils, +) + +logger = logging.getLogger(__name__) + + +class LocalController: + def __init__( + self, experiment_name: str, datasets: Optional[Dict[str, GenDataset]] = None + ): + if datasets is not None: + datasets = {k: v() if callable(v) else v for k, v in datasets.items()} + + self.datasets = datasets + self.experiment_name = experiment_name + + def run(self, train_func: Callable[[], None]) -> Result: + set_train_fn_utils( + LocalTrainFnUtils( + experiment_name=self.experiment_name, + dataset_shards=self.datasets, + ) + ) + train_func() + train_fn_utils = get_train_fn_utils() + assert isinstance(train_fn_utils, LocalTrainFnUtils) + return Result( + metrics=train_fn_utils._get_last_metrics(), + checkpoint=train_fn_utils.get_checkpoint(), + path=None, + error=None, + ) diff --git a/python/ray/train/v2/_internal/execution/train_fn_utils.py b/python/ray/train/v2/_internal/execution/train_fn_utils.py index 3fb791b667d6..a5caad707c15 100644 --- a/python/ray/train/v2/_internal/execution/train_fn_utils.py +++ b/python/ray/train/v2/_internal/execution/train_fn_utils.py @@ -1,4 +1,6 @@ +import logging import threading +from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, Dict, List, Optional from ray.data import DataIterator @@ -6,21 +8,28 @@ from ray.train.v2._internal.execution.context import ( get_train_context as get_internal_train_context, ) -from ray.train.v2.api.context import TrainContext as ExternalTrainContext +from ray.train.v2.api.context import ( + DistributedTrainContext, + LocalTrainContext, + TrainContext as ExternalTrainContext, +) + +logger = logging.getLogger(__name__) if TYPE_CHECKING: from ray.train import Checkpoint from ray.train.v2.api.reported_checkpoint import ReportedCheckpoint -class TrainFnUtils: +class TrainFnUtils(ABC): """Utility class providing an abstraction layer between user-facing APIs - and :class:`~ray.train.v2._internal.execution.context.TrainContext`. + and :class:`~ray.train.v2.api.context.TrainContext`. - It should be set before the users' training function is called, like training workers initialization. + It should be set before the users' training function is called. This class can be patched if new user APIs behaviors is wanted. """ + @abstractmethod def report( self, metrics: Dict[str, Any], @@ -37,18 +46,18 @@ def report( be stored in the default storage path. If set, make sure this value is unique for each iteration. """ - return get_internal_train_context().report( - metrics, checkpoint, checkpoint_dir_name - ) + pass - def get_checkpoint(self): + @abstractmethod + def get_checkpoint(self) -> Optional["Checkpoint"]: """Get the latest checkpoint to resume training from. Returns: The latest checkpoint if available, None otherwise. """ - return get_internal_train_context().get_checkpoint() + pass + @abstractmethod def get_all_reported_checkpoints(self) -> List["ReportedCheckpoint"]: """Get all the checkpoints reported by the workers. @@ -56,10 +65,11 @@ def get_all_reported_checkpoints(self) -> List["ReportedCheckpoint"]: A list of ReportedCheckpoint objects that represent the checkpoints and corresponding metrics reported by the workers. """ - return get_internal_train_context().get_all_reported_checkpoints() + pass + @abstractmethod def get_dataset_shard(self, dataset_name: str) -> DataIterator: - """Get the dataset shard for this worker. + """Get the dataset shard for this training process. This method is used by the public API function :func:`ray.train.get_dataset_shard`. Users should typically call ``ray.train.get_dataset_shard()`` instead of calling this method directly. @@ -70,17 +80,23 @@ def get_dataset_shard(self, dataset_name: str) -> DataIterator: Returns: The DataIterator shard for this worker. """ - from ray.train.v2._internal.data_integration.interfaces import ( - DatasetShardMetadata, - ) - - return get_internal_train_context().get_dataset_shard( - DatasetShardMetadata(dataset_name=dataset_name) - ) + pass + @abstractmethod def get_context(self) -> ExternalTrainContext: - return ExternalTrainContext() + """Get the TrainContext for this training process. + The specific type of TrainContext returned depends on the implementation of TrainFnUtils. + Returns: + The train context for this training process. + """ + pass + + @abstractmethod + def is_distributed(self) -> bool: + pass + + @abstractmethod def barrier(self) -> None: """Create a barrier across all workers. @@ -89,16 +105,110 @@ def barrier(self) -> None: This method is used by the public API function :func:`ray.train.collective.barrier`. Users should typically call ``ray.train.collective.barrier()`` instead of calling this method directly. """ - return collective_impl.barrier() + pass + @abstractmethod def broadcast_from_rank_zero(self, data: Any) -> Any: """Broadcast data from the rank 0 worker to all other workers. This method is used by the public API function :func:`ray.train.collective.broadcast_from_rank_zero`. Users should typically call ``ray.train.collective.broadcast_from_rank_zero()`` instead of calling this method directly. """ + pass + + +class DistributedTrainFnUtils(TrainFnUtils): + def report( + self, + metrics: Dict[str, Any], + checkpoint: Optional["Checkpoint"] = None, + checkpoint_dir_name: Optional[str] = None, + ) -> None: + return get_internal_train_context().report( + metrics, checkpoint, checkpoint_dir_name + ) + + def get_checkpoint(self): + return get_internal_train_context().get_checkpoint() + + def get_dataset_shard(self, dataset_name: str) -> DataIterator: + from ray.train.v2._internal.data_integration.interfaces import ( + DatasetShardMetadata, + ) + + return get_internal_train_context().get_dataset_shard( + DatasetShardMetadata(dataset_name=dataset_name) + ) + + def get_context(self) -> DistributedTrainContext: + return DistributedTrainContext() + + def is_distributed(self) -> bool: + return True + + def barrier(self) -> None: + return collective_impl.barrier() + + def broadcast_from_rank_zero(self, data: Any) -> Any: return collective_impl.broadcast_from_rank_zero(data) + def get_all_reported_checkpoints(self) -> List["ReportedCheckpoint"]: + return get_internal_train_context().get_all_reported_checkpoints() + + +class LocalTrainFnUtils(TrainFnUtils): + def __init__( + self, + experiment_name: str, + dataset_shards: Optional[Dict[str, DataIterator]] = None, + ): + self._context = LocalTrainContext( + experiment_name=experiment_name, + ) + self._dataset_shards = dataset_shards + self._last_metrics = None + self._last_checkpoint = None + + def report( + self, + metrics: Dict[str, Any], + checkpoint: Optional["Checkpoint"] = None, + checkpoint_dir_name: Optional[str] = None, + ) -> None: + self._last_metrics = metrics + self._last_checkpoint = checkpoint + logger.info(f"Reported metrics: {metrics}") + + def get_checkpoint(self) -> Optional["Checkpoint"]: + return self._last_checkpoint + + def get_dataset_shard(self, dataset_name: str) -> DataIterator: + assert ( + self._dataset_shards is not None and dataset_name in self._dataset_shards + ), f"Dataset shard {dataset_name} not found." + return self._dataset_shards[dataset_name] + + def get_context(self) -> LocalTrainContext: + return self._context + + def is_distributed(self) -> bool: + return False + + def barrier(self) -> None: + pass + + def broadcast_from_rank_zero(self, data: Any) -> Any: + return data + + def _get_last_metrics(self) -> Optional[Dict[str, Any]]: + """Return the last metrics reported by the training function. + This function should only be called by LocalController + """ + return self._last_metrics + + def get_all_reported_checkpoints(self) -> List["ReportedCheckpoint"]: + return [] + _train_fn_utils: Optional[TrainFnUtils] = None _train_fn_utils_lock = threading.Lock() diff --git a/python/ray/train/v2/_internal/execution/worker_group/worker.py b/python/ray/train/v2/_internal/execution/worker_group/worker.py index 8afc8a7b681e..2735a3991bbd 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/worker.py +++ b/python/ray/train/v2/_internal/execution/worker_group/worker.py @@ -30,7 +30,7 @@ ) from ray.train.v2._internal.execution.storage import StorageContext from ray.train.v2._internal.execution.train_fn_utils import ( - TrainFnUtils, + DistributedTrainFnUtils, set_train_fn_utils, ) from ray.train.v2._internal.execution.worker_group.poll import WorkerStatus @@ -228,7 +228,7 @@ def init_train_context( set_train_context(context) # user facing train fn utils - set_train_fn_utils(TrainFnUtils()) + set_train_fn_utils(DistributedTrainFnUtils()) for callback in self._callbacks: callback.after_init_train_context() diff --git a/python/ray/train/v2/api/config.py b/python/ray/train/v2/api/config.py index 665b3998cf70..7935c3ba6dcb 100644 --- a/python/ray/train/v2/api/config.py +++ b/python/ray/train/v2/api/config.py @@ -33,7 +33,9 @@ class ScalingConfig(ScalingConfigV1): num_workers: The number of workers (Ray actors) to launch. Each worker will reserve 1 CPU by default. The number of CPUs reserved by each worker can be overridden with the - ``resources_per_worker`` argument. + ``resources_per_worker`` argument. If the number of workers is 0, + the training function will run in local mode, meaning the training + function runs in the same process. use_gpu: If True, training will be done on GPUs (1 per worker). Defaults to False. The number of GPUs reserved by each worker can be overridden with the ``resources_per_worker`` @@ -119,6 +121,13 @@ def __post_init__(self): "`use_tpu=True` and `num_workers` > 1." ) + if self.num_workers == 0: + logger.info( + "Running in local mode. The training function will run in the same process. " + "If you are using it and running into issues please file a report at " + "https://github.com/ray-project/ray/issues." + ) + super().__post_init__() @property diff --git a/python/ray/train/v2/api/context.py b/python/ray/train/v2/api/context.py index c2fce63e430f..6d8896a1364a 100644 --- a/python/ray/train/v2/api/context.py +++ b/python/ray/train/v2/api/context.py @@ -1,3 +1,4 @@ +from abc import ABC, abstractmethod from typing import Any, Dict from ray.train.v2._internal.execution.context import ( @@ -7,7 +8,9 @@ @PublicAPI(stability="stable") -class TrainContext: +class TrainContext(ABC): + """Abstract interface for training context.""" + @Deprecated def get_metadata(self) -> Dict[str, Any]: """[Deprecated] User metadata dict passed to the Trainer constructor.""" @@ -55,10 +58,12 @@ def get_trial_dir(self) -> str: _TUNE_SPECIFIC_CONTEXT_DEPRECATION_MESSAGE.format("get_trial_dir") ) + @abstractmethod def get_experiment_name(self) -> str: """Experiment name for the corresponding trial.""" - return get_internal_train_context().get_experiment_name() + pass + @abstractmethod def get_world_size(self) -> int: """Get the current world size (i.e. total number of workers) for this run. @@ -85,8 +90,9 @@ def train_loop_per_worker(config): ... """ - return get_internal_train_context().get_world_size() + pass + @abstractmethod def get_world_rank(self) -> int: """Get the world rank of this worker. @@ -112,8 +118,9 @@ def train_loop_per_worker(config): ... """ - return get_internal_train_context().get_world_rank() + pass + @abstractmethod def get_local_rank(self) -> int: """Get the local rank of this worker (rank of the worker on its node). @@ -142,8 +149,9 @@ def train_loop_per_worker(config): ... """ - return get_internal_train_context().get_local_rank() + pass + @abstractmethod def get_local_world_size(self) -> int: """Get the local world size of this node (i.e. number of workers on this node). @@ -170,8 +178,9 @@ def train_loop_per_worker(): ... """ - return get_internal_train_context().get_local_world_size() + pass + @abstractmethod def get_node_rank(self) -> int: """Get the rank of this node. @@ -198,9 +207,10 @@ def train_loop_per_worker(): ... """ - return get_internal_train_context().get_node_rank() + pass @DeveloperAPI + @abstractmethod def get_storage(self): """Returns the :class:`~ray.train._internal.storage.StorageContext` storage context which gives advanced access to the filesystem and paths @@ -209,4 +219,61 @@ def get_storage(self): NOTE: This is a developer API, and the `StorageContext` interface may change without notice between minor versions. """ + pass + + +class DistributedTrainContext(TrainContext): + """Implementation of TrainContext for distributed mode.""" + + def get_experiment_name(self) -> str: + return get_internal_train_context().get_experiment_name() + + def get_world_size(self) -> int: + return get_internal_train_context().get_world_size() + + def get_world_rank(self) -> int: + return get_internal_train_context().get_world_rank() + + def get_local_rank(self) -> int: + return get_internal_train_context().get_local_rank() + + def get_local_world_size(self) -> int: + return get_internal_train_context().get_local_world_size() + + def get_node_rank(self) -> int: + return get_internal_train_context().get_node_rank() + + def get_storage(self): return get_internal_train_context().get_storage() + + +class LocalTrainContext(TrainContext): + """Implementation of TrainContext for local mode.""" + + def __init__( + self, + experiment_name: str, + ): + self.experiment_name = experiment_name + + def get_experiment_name(self) -> str: + return self.experiment_name + + def get_world_size(self) -> int: + return 1 + + def get_world_rank(self) -> int: + return 0 + + def get_local_rank(self) -> int: + return 0 + + def get_local_world_size(self) -> int: + return 1 + + def get_node_rank(self) -> int: + """For local mode, we only use one node.""" + return 0 + + def get_storage(self): + raise NotImplementedError("Local storage context not yet implemented. ") diff --git a/python/ray/train/v2/api/data_parallel_trainer.py b/python/ray/train/v2/api/data_parallel_trainer.py index 6eef7c9bb247..079e6bfe3038 100644 --- a/python/ray/train/v2/api/data_parallel_trainer.py +++ b/python/ray/train/v2/api/data_parallel_trainer.py @@ -46,6 +46,7 @@ from ray.train.v2._internal.execution.context import TrainRunContext from ray.train.v2._internal.execution.controller import TrainController from ray.train.v2._internal.execution.failure_handling import create_failure_policy +from ray.train.v2._internal.execution.local_mode_utils import LocalController from ray.train.v2._internal.execution.scaling_policy import create_scaling_policy from ray.train.v2._internal.util import ObjectRefWrapper, construct_train_func from ray.train.v2.api.callback import UserCallback @@ -86,6 +87,8 @@ def __init__( self.datasets = datasets or {} self.data_config = dataset_config or DataConfig() + self.running_in_local_mode = self.scaling_config.num_workers == 0 + self.train_run_context = TrainRunContext( run_config=self.run_config, train_loop_config=self.train_loop_config, @@ -104,6 +107,14 @@ def __init__( usage_lib.record_library_usage("train") tag_train_v2_trainer(self) + def _get_train_func(self) -> Callable[[], None]: + return construct_train_func( + self.train_loop_per_worker, + config=self.train_loop_config, + train_func_context=self.backend_config.train_func_context, + fn_arg_name="train_loop_per_worker", + ) + def fit(self) -> Result: """Launches the Ray Train controller to run training on workers. @@ -114,31 +125,35 @@ def fit(self) -> Result: ray.train.v2.api.exceptions.ControllerError: If a non-retryable error occurs in the Ray Train controller itself, or if the number of retries configured in `FailureConfig` is exhausted. ray.train.v2.api.exceptions.WorkerGroupError: If one or more workers fail during training and the number of retries configured in `FailureConfig` is exhausted. """ - train_fn = construct_train_func( - self.train_loop_per_worker, - config=self.train_loop_config, - train_func_context=self.backend_config.train_func_context, - fn_arg_name="train_loop_per_worker", - ) - train_fn_ref = ObjectRefWrapper(train_fn) - - result = self._initialize_and_run_controller( - train_fn_ref=train_fn_ref, - scaling_policy=create_scaling_policy(self.scaling_config), - failure_policy=create_failure_policy(self.run_config.failure_config), - train_run_context=self.train_run_context, - callbacks=self._create_default_callbacks(), - ) + train_fn = self._get_train_func() + if self.running_in_local_mode: + return self._initialize_and_run_local_controller(train_fn) + else: + train_fn_ref = ObjectRefWrapper(train_fn) + + result = self._initialize_and_run_controller( + train_fn_ref=train_fn_ref, + scaling_policy=create_scaling_policy(self.scaling_config), + failure_policy=create_failure_policy(self.run_config.failure_config), + train_run_context=self.train_run_context, + callbacks=self._create_default_callbacks(), + ) - if result.error: - # NOTE: If the training run errored out, raise an error back to the - # user's driver script. - # For example, if the Train `FailurePolicy` runs out of retries, - # and one of the workers errors. The controller will exit, and - # the error will be raised here. - raise result.error + if result.error: + # NOTE: If the training run errored out, raise an error back to the + # user's driver script. + # For example, if the Train `FailurePolicy` runs out of retries, + # and one of the workers errors. The controller will exit, and + # the error will be raised here. + raise result.error - return result + return result + + def _get_local_controller(self) -> LocalController: + return LocalController( + experiment_name=self.run_config.name, + datasets=self.datasets, + ) def _create_default_callbacks(self) -> List[RayTrainCallback]: # Initialize callbacks from environment variable @@ -194,6 +209,11 @@ def _create_default_callbacks(self) -> List[RayTrainCallback]: ) return callbacks + def _initialize_and_run_local_controller( + self, train_func: Callable[[], None] + ) -> Result: + return self._get_local_controller().run(train_func) + def _initialize_and_run_controller(self, **controller_init_kwargs) -> Result: # Attach the controller to the node running the driver script. controller_actor_cls = ray.remote( diff --git a/python/ray/train/v2/tests/test_local_mode.py b/python/ray/train/v2/tests/test_local_mode.py new file mode 100644 index 000000000000..3f690c8ee09f --- /dev/null +++ b/python/ray/train/v2/tests/test_local_mode.py @@ -0,0 +1,522 @@ +import math +import sys +from unittest.mock import MagicMock + +import lightgbm +import pandas as pd +import pytest +import xgboost +from datasets import Dataset +from sklearn.datasets import load_breast_cancer +from sklearn.model_selection import train_test_split +from transformers import AutoConfig, AutoModelForCausalLM, Trainer, TrainingArguments + +import ray +from ray.data.preprocessors import Concatenator +from ray.tests.conftest import _ray_start_cluster +from ray.train import ScalingConfig +from ray.train.constants import TRAIN_DATASET_KEY +from ray.train.examples.pytorch.torch_linear_example import ( + train_func as linear_train_func, +) +from ray.train.huggingface.transformers import ( + RayTrainReportCallback as HuggingFaceRayTrainReportCallback, + prepare_trainer, +) +from ray.train.lightgbm import ( + LightGBMTrainer, + RayTrainReportCallback as LightGBMRayTrainReportCallback, +) +from ray.train.lightning import ( + RayDDPStrategy, + RayFSDPStrategy, + RayLightningEnvironment, + RayTrainReportCallback as LightningRayTrainReportCallback, +) +from ray.train.lightning._lightning_utils import import_lightning +from ray.train.tests._huggingface_data import train_data, validation_data +from ray.train.tests.lightning_test_utils import DummyDataModule, LinearModule +from ray.train.tests.util import create_dict_checkpoint +from ray.train.torch import TorchTrainer +from ray.train.v2.api.data_parallel_trainer import DataParallelTrainer +from ray.train.v2.jax import JaxTrainer +from ray.train.xgboost import ( + RayTrainReportCallback as XGBoostRayTrainReportCallback, + XGBoostTrainer, +) + +if sys.version_info >= (3, 12): + # Tensorflow is not installed for Python 3.12 because of keras compatibility. + pass +else: + from ray.train.examples.tf.tensorflow_regression_example import ( + train_func as tensorflow_linear_train_func, + ) + from ray.train.tensorflow import TensorflowTrainer + +pl = import_lightning() + + +@pytest.fixture +def ray_start_6_cpus(): + address_info = ray.init(num_cpus=6) + yield address_info + # The code after the yield will run as teardown code. + ray.shutdown() + + +@pytest.fixture +def ray_tpu_single_host(monkeypatch): + """Start a mock single-host TPU Ray cluster with 2x4 v6e (8 chips per host).""" + with _ray_start_cluster() as cluster: + monkeypatch.setenv("TPU_ACCELERATOR_TYPE", "v6e-8") + + # Simulate one node with 8 TPU chips. + cluster.add_node( + num_cpus=4, + resources={"TPU": 8}, + ) + + ray.init(address=cluster.address) + + yield cluster + ray.shutdown() + + +def test_data_parallel_trainer_local_mode(): + def train_fn(): + with create_dict_checkpoint({}) as checkpoint: + ray.train.report(metrics={"test": 1}, checkpoint=checkpoint) + + trainer = DataParallelTrainer(train_fn, scaling_config=ScalingConfig(num_workers=0)) + result = trainer.fit() + assert result.metrics == {"test": 1} + assert result.checkpoint + + +def test_jax_trainer_local_mode(ray_tpu_single_host, monkeypatch): + def jax_train_func(): + import jax + + devices = jax.devices() + print(f"Devices on this worker: {devices}") + ray.train.report({"result": [str(d) for d in devices]}) + + mock_jax = MagicMock() + mock_jax.devices.return_value = ["TPU:0"] + monkeypatch.setitem(sys.modules, "jax", mock_jax) + + trainer = JaxTrainer( + train_loop_per_worker=jax_train_func, + scaling_config=ScalingConfig( + num_workers=0, + ), + ) + result = trainer.fit() + assert result.error is None + assert result.metrics == {"result": ["TPU:0"]} + + +def test_lightgbm_trainer_local_mode(ray_start_6_cpus): + def lightgbm_train_fn_per_worker( + config: dict, + label_column: str, + dataset_keys: set, + num_boost_round: int = 10, + ): + remaining_iters = num_boost_round + train_ds_iter = ray.train.get_dataset_shard(TRAIN_DATASET_KEY) + train_df = train_ds_iter.materialize().to_pandas() + + eval_ds_iters = { + k: ray.train.get_dataset_shard(k) + for k in dataset_keys + if k != TRAIN_DATASET_KEY + } + eval_dfs = {k: d.materialize().to_pandas() for k, d in eval_ds_iters.items()} + + train_X, train_y = train_df.drop(label_column, axis=1), train_df[label_column] + train_set = lightgbm.Dataset(train_X, label=train_y) + + # NOTE: Include the training dataset in the evaluation datasets. + # This allows `train-*` metrics to be calculated and reported. + valid_sets = [train_set] + valid_names = [TRAIN_DATASET_KEY] + + for eval_name, eval_df in eval_dfs.items(): + eval_X, eval_y = eval_df.drop(label_column, axis=1), eval_df[label_column] + valid_sets.append(lightgbm.Dataset(eval_X, label=eval_y)) + valid_names.append(eval_name) + + # Add network params of the worker group to enable distributed training. + config.update(ray.train.lightgbm.get_network_params()) + + lightgbm.train( + params=config, + train_set=train_set, + num_boost_round=remaining_iters, + valid_sets=valid_sets, + valid_names=valid_names, + init_model=None, + callbacks=[LightGBMRayTrainReportCallback()], + ) + + data_raw = load_breast_cancer() + dataset_df = pd.DataFrame(data_raw["data"], columns=data_raw["feature_names"]) + dataset_df["target"] = data_raw["target"] + train_df, test_df = train_test_split(dataset_df, test_size=0.3) + + train_df_with_cat = train_df.copy() + test_df_with_cat = test_df.copy() + dataset_shard_size = 1 + train_df_with_cat["categorical_column"] = pd.Series( + (["A", "B"] * math.ceil(len(train_df_with_cat) / dataset_shard_size))[ + : len(train_df_with_cat) + ] + ).astype("category") + test_df_with_cat["categorical_column"] = pd.Series( + (["A", "B"] * math.ceil(len(test_df_with_cat) / dataset_shard_size))[ + : len(test_df_with_cat) + ] + ).astype("category") + + scale_config = ScalingConfig(num_workers=0) + train_dataset = ray.data.from_pandas(train_df_with_cat) + valid_dataset = ray.data.from_pandas(test_df_with_cat) + trainer = LightGBMTrainer( + train_loop_per_worker=lambda: lightgbm_train_fn_per_worker( + config={}, + label_column="target", + dataset_keys={TRAIN_DATASET_KEY, "valid"}, + ), + train_loop_config={ + "objective": "binary", + "metric": ["binary_logloss", "binary_error"], + }, + scaling_config=scale_config, + datasets={TRAIN_DATASET_KEY: train_dataset, "valid": valid_dataset}, + ) + result = trainer.fit() + checkpoint = result.checkpoint + assert checkpoint is not None + + +@pytest.mark.parametrize("datasource", ["dataloader", "datamodule"]) +def test_lightning_trainer_local_mode(ray_start_6_cpus, datasource): + + num_epochs = 1 + batch_size = 8 + dataset_size = 256 + dataset_shard_size = 1 + strategy_name = "ddp" + accelerator = "cpu" + + strategy_map = {"ddp": RayDDPStrategy(), "fsdp": RayFSDPStrategy()} + + def train_loop(): + model = LinearModule(input_dim=32, output_dim=4, strategy=strategy_name) + + strategy = strategy_map[strategy_name] + + trainer = pl.Trainer( + max_epochs=num_epochs, + devices="auto", + accelerator=accelerator, + strategy=strategy, + plugins=[RayLightningEnvironment()], + callbacks=[LightningRayTrainReportCallback()], + ) + + datamodule = DummyDataModule(batch_size, dataset_size) + + if datasource == "dataloader": + trainer.fit( + model, + train_dataloaders=datamodule.train_dataloader(), + val_dataloaders=datamodule.val_dataloader(), + ) + if datasource == "datamodule": + trainer.fit(model, datamodule=datamodule) + + trainer = TorchTrainer( + train_loop_per_worker=train_loop, + scaling_config=ScalingConfig(num_workers=0, use_gpu=(accelerator == "gpu")), + ) + + results = trainer.fit() + assert results.metrics["epoch"] == num_epochs - 1 + assert ( + results.metrics["step"] + == num_epochs * dataset_size / dataset_shard_size / batch_size + ) + assert "loss" in results.metrics + assert "val_loss" in results.metrics + + +def test_tensorflow_linear_local_mode(ray_start_4_cpus): + """Also tests air Keras callback.""" + epochs = 1 + + def train_func(config): + result = tensorflow_linear_train_func(config) + assert len(result) == epochs + + train_loop_config = { + "lr": 1e-3, + "batch_size": 32, + "epochs": epochs, + } + scaling_config = ScalingConfig(num_workers=0) + dataset = ray.data.read_csv("s3://anonymous@air-example-data/regression.csv") + columns_to_concatenate = [f"x{i:03}" for i in range(100)] + preprocessor = Concatenator(columns=columns_to_concatenate, output_column_name="x") + dataset = preprocessor.transform(dataset) + + trainer = TensorflowTrainer( + train_loop_per_worker=train_func, + train_loop_config=train_loop_config, + scaling_config=scaling_config, + datasets={TRAIN_DATASET_KEY: dataset}, + ) + result = trainer.fit() + assert not result.error + assert result.checkpoint + + +def test_torch_trainer_local_mode(ray_start_6_cpus): + def train_func(config): + result = linear_train_func(config) + assert len(result) == epochs + assert result[-1]["loss"] < result[0]["loss"] + + epochs = 3 + scaling_config = ScalingConfig(num_workers=0) + config = {"lr": 1e-2, "hidden_size": 1, "batch_size": 4, "epochs": epochs} + trainer = TorchTrainer( + train_loop_per_worker=train_func, + train_loop_config=config, + scaling_config=scaling_config, + ) + result = trainer.fit() + assert result.error is None + assert result.metrics is not None + assert result.metrics["loss"] is not None + assert result.checkpoint + + +HF_BATCH_SIZE_PER_WORKER = 2 +HF_MODEL_NAME = "hf-internal-testing/tiny-random-BloomForCausalLM" +HF_MAX_EPOCHS = 1 +HF_TRAIN_DATASET_SIZE = 16 + + +@pytest.mark.parametrize("use_ray_data", [False, True]) +def test_e2e_hf_local_mode(ray_start_4_cpus, use_ray_data): + def get_transformers_configurations(): + """Get configurations with dynamic step calculations based on number of workers.""" + steps_per_epoch = HF_TRAIN_DATASET_SIZE // HF_BATCH_SIZE_PER_WORKER + return { + "epoch_gpu": { + "evaluation_strategy": "epoch", + "save_strategy": "epoch", + "logging_strategy": "epoch", + "eval_steps": None, + "save_steps": None, + "logging_steps": None, + "no_cuda": False, + }, + "steps_gpu": { + "evaluation_strategy": "steps", + "save_strategy": "steps", + "logging_strategy": "steps", + "eval_steps": steps_per_epoch, + "save_steps": steps_per_epoch * 2, + "logging_steps": 1, + "no_cuda": False, + }, + "steps_cpu": { + "evaluation_strategy": "steps", + "save_strategy": "steps", + "logging_strategy": "steps", + "eval_steps": steps_per_epoch, + "save_steps": steps_per_epoch, + "logging_steps": 1, + "no_cuda": True, + }, + "steps_cpu_local": { + "evaluation_strategy": "steps", + "save_strategy": "steps", + "logging_strategy": "steps", + "eval_steps": steps_per_epoch, + "save_steps": steps_per_epoch, + "logging_steps": 1, + "no_cuda": True, + }, + } + + config_id = "steps_cpu_local" + num_workers = 0 + + def train_func(config): + # Datasets + if config["use_ray_data"]: + train_ds_shard = ray.train.get_dataset_shard("train") + eval_ds_shard = ray.train.get_dataset_shard("eval") + + train_dataset = train_ds_shard.iter_torch_batches( + batch_size=HF_BATCH_SIZE_PER_WORKER + ) + eval_dataset = eval_ds_shard.iter_torch_batches( + batch_size=HF_BATCH_SIZE_PER_WORKER + ) + else: + train_df = pd.read_json(train_data) + validation_df = pd.read_json(validation_data) + + train_dataset = Dataset.from_pandas(train_df) + eval_dataset = Dataset.from_pandas(validation_df) + + # Model + model_config = AutoConfig.from_pretrained(HF_MODEL_NAME) + model = AutoModelForCausalLM.from_config(model_config) + + # HF Transformers Trainer + training_args = TrainingArguments( + f"{HF_MODEL_NAME}-wikitext2", + evaluation_strategy=config["evaluation_strategy"], + logging_strategy=config["logging_strategy"], + save_strategy=config["save_strategy"], + eval_steps=config["eval_steps"], + save_steps=config["save_steps"], + logging_steps=config["logging_steps"], + num_train_epochs=config.get("num_train_epochs", HF_MAX_EPOCHS), + max_steps=config.get("max_steps", -1), + learning_rate=config.get("learning_rate", 2e-5), + per_device_train_batch_size=HF_BATCH_SIZE_PER_WORKER, + per_device_eval_batch_size=HF_BATCH_SIZE_PER_WORKER, + weight_decay=0.01, + disable_tqdm=True, + no_cuda=config["no_cuda"], + report_to="none", + ) + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + ) + + # Report to Ray Train + trainer.add_callback(HuggingFaceRayTrainReportCallback()) + trainer = prepare_trainer(trainer) + + # Start Training + trainer.train() + + configurations = get_transformers_configurations() + train_loop_config = configurations[config_id] + + # Calculate the num of Ray training iterations + max_steps = HF_MAX_EPOCHS * HF_TRAIN_DATASET_SIZE // HF_BATCH_SIZE_PER_WORKER + + train_loop_config["use_ray_data"] = use_ray_data + + datasets = None + if use_ray_data: + # Must specify `max_steps` for Iterable Dataset + train_loop_config["max_steps"] = max_steps + + train_df = pd.read_json(train_data) + validation_df = pd.read_json(validation_data) + + ray_train_ds = ray.data.from_pandas(train_df) + ray_eval_ds = ray.data.from_pandas(validation_df) + datasets = {"train": ray_train_ds, "eval": ray_eval_ds} + else: + # Specify `num_train_epochs` for Map-style Dataset + train_loop_config["num_train_epochs"] = HF_MAX_EPOCHS + + use_gpu = not train_loop_config["no_cuda"] + + trainer = TorchTrainer( + train_func, + train_loop_config=train_loop_config, + scaling_config=ScalingConfig(num_workers=num_workers, use_gpu=use_gpu), + datasets=datasets, + ) + result = trainer.fit() + + assert result.metrics["step"] == max_steps + assert "eval_loss" in result.metrics + if not use_ray_data: + assert result.metrics["epoch"] == HF_MAX_EPOCHS + + +def test_xgboost_trainer_local_mode(ray_start_4_cpus): + def xgboost_train_fn_per_worker(): + label_column = "target" + dataset_keys = {TRAIN_DATASET_KEY, "valid"} + checkpoint = ray.train.get_checkpoint() + starting_model = None + remaining_iters = 10 + if checkpoint: + starting_model = XGBoostRayTrainReportCallback.get_model(checkpoint) + starting_iter = starting_model.num_boosted_rounds() + remaining_iters = remaining_iters - starting_iter + + train_ds_iter = ray.train.get_dataset_shard(TRAIN_DATASET_KEY) + train_df = train_ds_iter.materialize().to_pandas() + + eval_ds_iters = { + k: ray.train.get_dataset_shard(k) + for k in dataset_keys + if k != TRAIN_DATASET_KEY + } + eval_dfs = {k: d.materialize().to_pandas() for k, d in eval_ds_iters.items()} + + train_X, train_y = train_df.drop(label_column, axis=1), train_df[label_column] + dtrain = xgboost.DMatrix(train_X, label=train_y) + + # NOTE: Include the training dataset in the evaluation datasets. + # This allows `train-*` metrics to be calculated and reported. + evals = [(dtrain, TRAIN_DATASET_KEY)] + + for eval_name, eval_df in eval_dfs.items(): + eval_X, eval_y = eval_df.drop(label_column, axis=1), eval_df[label_column] + evals.append((xgboost.DMatrix(eval_X, label=eval_y), eval_name)) + + evals_result = {} + xgboost.train( + {}, + dtrain=dtrain, + evals=evals, + evals_result=evals_result, + num_boost_round=remaining_iters, + xgb_model=starting_model, + ) + + data_raw = load_breast_cancer() + dataset_df = pd.DataFrame(data_raw["data"], columns=data_raw["feature_names"]) + dataset_df["target"] = data_raw["target"] + train_df, test_df = train_test_split(dataset_df, test_size=0.3) + + train_dataset = ray.data.from_pandas(train_df) + valid_dataset = ray.data.from_pandas(test_df) + scale_config = ScalingConfig(num_workers=0) + trainer = XGBoostTrainer( + train_loop_per_worker=xgboost_train_fn_per_worker, + train_loop_config={ + "tree_method": "approx", + "objective": "binary:logistic", + "eval_metric": ["logloss", "error"], + }, + scaling_config=scale_config, + datasets={TRAIN_DATASET_KEY: train_dataset, "valid": valid_dataset}, + ) + result = trainer.fit() + with pytest.raises(DeprecationWarning): + XGBoostTrainer.get_model(result.checkpoint) + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/train/v2/torch/train_loop_utils.py b/python/ray/train/v2/torch/train_loop_utils.py index af546f83014f..dfb618dc6b6a 100644 --- a/python/ray/train/v2/torch/train_loop_utils.py +++ b/python/ray/train/v2/torch/train_loop_utils.py @@ -1,7 +1,7 @@ import logging import os import random -from typing import Any, Callable, Dict, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Union import numpy as np import torch @@ -17,7 +17,11 @@ import ray.train.torch from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag -from ray.train.torch.train_loop_utils import _WrappedDataLoader +from ray.train.torch.train_loop_utils import ( + _WrappedDataLoader, + get_devices as get_devices_distributed, +) +from ray.train.v2._internal.execution.train_fn_utils import get_train_fn_utils from ray.util.annotations import Deprecated, PublicAPI logger = logging.getLogger(__name__) @@ -34,6 +38,22 @@ ) +def get_device() -> torch.device: + return get_devices()[0] + + +def get_devices() -> List[torch.device]: + if get_train_fn_utils().is_distributed(): + return get_devices_distributed() + else: + # Local mode, we defer to torch.cuda + # TODO(xgui): Use `ScalingConfig.use_gpu` instead + if torch.cuda.is_available(): + return [torch.device(f"cuda:{torch.cuda.current_device()}")] + else: + return [torch.device("cpu")] + + def prepare_model( model: torch.nn.Module, move_to_device: Union[bool, torch.device] = True, diff --git a/python/ray/train/v2/xgboost/__init__.py b/python/ray/train/v2/xgboost/__init__.py index e69de29bb2d1..b4e10280aceb 100644 --- a/python/ray/train/v2/xgboost/__init__.py +++ b/python/ray/train/v2/xgboost/__init__.py @@ -0,0 +1,2 @@ +# This is a workaround to avoid a circular import. +import ray.train.xgboost as ray_train_xgboost # noqa: F401 diff --git a/python/ray/train/v2/xgboost/config.py b/python/ray/train/v2/xgboost/config.py new file mode 100644 index 000000000000..d2c04c99c137 --- /dev/null +++ b/python/ray/train/v2/xgboost/config.py @@ -0,0 +1,21 @@ +from contextlib import contextmanager + +from ray.train.v2._internal.execution.train_fn_utils import get_train_fn_utils +from ray.train.xgboost.config import XGBoostConfig as XGBoostConfigV1 + + +class XGBoostConfig(XGBoostConfigV1): + @property + def train_func_context(self): + distributed_context = super(XGBoostConfig, self).train_func_context + + @contextmanager + def collective_communication_context(): + # The distributed_context is only needed in distributed mode + if get_train_fn_utils().is_distributed(): + with distributed_context(): + yield + else: + yield + + return collective_communication_context diff --git a/python/ray/train/xgboost/__init__.py b/python/ray/train/xgboost/__init__.py index aa2d1c88d11b..447515b95b44 100644 --- a/python/ray/train/xgboost/__init__.py +++ b/python/ray/train/xgboost/__init__.py @@ -6,6 +6,7 @@ from ray.train.xgboost.xgboost_trainer import XGBoostTrainer if is_v2_enabled(): + from ray.train.v2.xgboost.config import XGBoostConfig # noqa: F811 from ray.train.v2.xgboost.xgboost_trainer import XGBoostTrainer # noqa: F811 __all__ = [ From 2cbf1e280c2ecb6ddbf39f8def3e4e240f8b2f84 Mon Sep 17 00:00:00 2001 From: Kyle Robinson Date: Thu, 4 Sep 2025 10:30:00 +0900 Subject: [PATCH 438/634] Update metrics_utils for future global metrics aggregation in controller. (#55568) ## Why are these changes needed? These changes modify the autoscaler metrics collection and aggregation functions in preparation for global aggregation in the controller. ## Related issue number Partial for #46497 Required for #41135 #51905 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Kyle Robinson Signed-off-by: Kyle Robinson Signed-off-by: abrar Co-authored-by: Abrar Sheikh Co-authored-by: Cindy Zhang Co-authored-by: abrar --- python/ray/serve/_private/metrics_utils.py | 240 ++++++++-- python/ray/serve/_private/replica.py | 5 +- python/ray/serve/_private/router.py | 16 +- .../serve/tests/unit/test_metrics_utils.py | 449 ++++++++++++++++-- 4 files changed, 623 insertions(+), 87 deletions(-) diff --git a/python/ray/serve/_private/metrics_utils.py b/python/ray/serve/_private/metrics_utils.py index 14efb553ca09..509ed48ed65c 100644 --- a/python/ray/serve/_private/metrics_utils.py +++ b/python/ray/serve/_private/metrics_utils.py @@ -1,15 +1,28 @@ import asyncio import bisect import logging +import statistics from collections import defaultdict from dataclasses import dataclass, field -from typing import Callable, DefaultDict, Dict, Hashable, List, Optional +from itertools import chain +from typing import ( + Callable, + DefaultDict, + Dict, + Hashable, + Iterable, + List, + Optional, + Tuple, +) from ray.serve._private.constants import ( METRICS_PUSHER_GRACEFUL_SHUTDOWN_TIMEOUT_S, SERVE_LOGGER_NAME, ) +QUEUED_REQUESTS_KEY = "queued" + logger = logging.getLogger(SERVE_LOGGER_NAME) @@ -152,7 +165,7 @@ def prune_keys_and_compact_data(self, start_timestamp_s: float): def _get_datapoints( self, key: Hashable, window_start_timestamp_s: float - ) -> List[float]: + ) -> List[TimeStampedValue]: """Get all data points given key after window_start_timestamp_s""" datapoints = self.data[key] @@ -165,52 +178,205 @@ def _get_datapoints( ) return datapoints[idx:] - def window_average( - self, key: Hashable, window_start_timestamp_s: float, do_compact: bool = True + def _aggregate_reduce( + self, + keys: Iterable[Hashable], + aggregate_fn: Callable[[Iterable[float]], float], + ) -> Tuple[Optional[float], int]: + """Reduce the entire set of timeseries values across the specified keys. + + Args: + keys: Iterable of keys to aggregate across. + aggregate_fn: Function to apply across all float values, e.g., sum, max. + + Returns: + A tuple of (float, int) where the first element is the aggregated value + and the second element is the number of valid keys used. + Returns (None, 0) if no valid keys have data. + + Example: + Suppose the store contains: + >>> store = InMemoryMetricsStore() + >>> store.data.update({ + ... "a": [TimeStampedValue(0, 1.0), TimeStampedValue(1, 2.0)], + ... "b": [], + ... "c": [TimeStampedValue(0, 10.0)], + ... }) + + Using sum across keys: + + >>> store._aggregate_reduce(keys=["a", "b", "c"], aggregate_fn=sum) + (13.0, 2) + + Here: + - The aggregated value is 1.0 + 2.0 + 10.0 = 13.0 + - Only keys "a" and "c" contribute values, so report_count = 2 + """ + valid_key_count = 0 + + def _values_generator(): + """Generator that yields values from valid keys without storing them all in memory.""" + nonlocal valid_key_count + for key in keys: + series = self.data.get(key, []) + if not series: + continue + + valid_key_count += 1 + for timestamp_value in series: + yield timestamp_value.value + + # Create the generator and check if it has any values + values_gen = _values_generator() + try: + first_value = next(values_gen) + except StopIteration: + # No valid data found + return None, 0 + + # Apply aggregation to the generator (memory efficient) + aggregated_result = aggregate_fn(chain([first_value], values_gen)) + return aggregated_result, valid_key_count + + def get_latest( + self, + key: Hashable, ) -> Optional[float]: - """Perform a window average operation for metric `key` + """Get the latest value for a given key.""" + if not self.data.get(key, None): + return None + return self.data[key][-1].value + + def aggregate_min( + self, + keys: Iterable[Hashable], + ) -> Tuple[Optional[float], int]: + """Find the min value across all timeseries values at the specified keys. Args: - key: the metric name. - window_start_timestamp_s: the unix epoch timestamp for the - start of the window. The computed average will use all datapoints - from this timestamp until now. - do_compact: whether or not to delete the datapoints that's - before `window_start_timestamp_s` to save memory. Default is - true. + keys: Iterable of keys to aggregate across. Returns: - The average of all the datapoints for the key on and after time - window_start_timestamp_s, or None if there are no such points. + A tuple of (float, int) where the first element is the min across + all values found at `keys`, and the second is the number of valid + keys used to compute the min. + Returns (None, 0) if no valid keys have data. """ - points_after_idx = self._get_datapoints(key, window_start_timestamp_s) + return self._aggregate_reduce(keys, min) - if do_compact: - self.data[key] = points_after_idx + def aggregate_max( + self, + keys: Iterable[Hashable], + ) -> Tuple[Optional[float], int]: + """Find the max value across all timeseries values at the specified keys. - if len(points_after_idx) == 0: - return - return sum(point.value for point in points_after_idx) / len(points_after_idx) + Args: + keys: Iterable of keys to aggregate across. + Returns: + A tuple of (float, int) where the first element is the max across + all values found at `keys`, and the second is the number of valid + keys used to compute the max. + Returns (None, 0) if no valid keys have data. + """ + return self._aggregate_reduce(keys, max) - def max( - self, key: Hashable, window_start_timestamp_s: float, do_compact: bool = True - ): - """Perform a max operation for metric `key`. + def aggregate_sum( + self, + keys: Iterable[Hashable], + ) -> Tuple[Optional[float], int]: + """Sum the entire set of timeseries values across the specified keys. Args: - key: the metric name. - window_start_timestamp_s: the unix epoch timestamp for the - start of the window. The computed average will use all datapoints - from this timestamp until now. - do_compact: whether or not to delete the datapoints that's - before `window_start_timestamp_s` to save memory. Default is - true. + keys: Iterable of keys to aggregate across. Returns: - Max value of the data points for the key on and after time - window_start_timestamp_s, or None if there are no such points. + A tuple of (float, int) where the first element is the sum across + all values found at `keys`, and the second is the number of valid + keys used to compute the sum. + Returns (None, 0) if no valid keys have data. """ - points_after_idx = self._get_datapoints(key, window_start_timestamp_s) + return self._aggregate_reduce(keys, sum) - if do_compact: - self.data[key] = points_after_idx + def aggregate_avg( + self, + keys: Iterable[Hashable], + ) -> Tuple[Optional[float], int]: + """Average the entire set of timeseries values across the specified keys. - return max((point.value for point in points_after_idx), default=None) + Args: + keys: Iterable of keys to aggregate across. + Returns: + A tuple of (float, int) where the first element is the mean across + all values found at `keys`, and the second is the number of valid + keys used to compute the mean. + Returns (None, 0) if no valid keys have data. + """ + return self._aggregate_reduce(keys, statistics.mean) + + +def _bucket_latest_by_window( + series: List[TimeStampedValue], + start: float, + window_s: float, +) -> Dict[int, float]: + """ + Map each window index -> latest value seen in that window. + Assumes series is sorted by timestamp ascending. + """ + buckets: Dict[int, float] = {} + for p in series: + w = int((p.timestamp - start) // window_s) + buckets[w] = p.value # overwrite keeps the latest within the window + return buckets + + +def _merge_two_timeseries( + t1: List[TimeStampedValue], t2: List[TimeStampedValue], window_s: float +) -> List[TimeStampedValue]: + """ + Merge two ascending time series by summing values within a specified time window. + If multiple values fall within the same window in a series, the latest value is used. + The output contains one point per window that had at least one value, timestamped + at the window center. + """ + if window_s <= 0: + raise ValueError(f"window_s must be positive, got {window_s}") + + if not t1 and not t2: + return [] + + # Align windows so each output timestamp sits at the start of its window. + # start is snapped to window_s boundary for binning stability + earliest = min(x[0].timestamp for x in (t1, t2) if x) + start = earliest // window_s * window_s + + b1 = _bucket_latest_by_window(t1, start, window_s) + b2 = _bucket_latest_by_window(t2, start, window_s) + + windows = sorted(set(b1.keys()) | set(b2.keys())) + + merged: List[TimeStampedValue] = [] + for w in windows: + v = b1.get(w, 0.0) + b2.get(w, 0.0) + ts_start = start + w * window_s + merged.append(TimeStampedValue(timestamp=ts_start, value=v)) + return merged + + +def merge_timeseries_dicts( + *timeseries_dicts: DefaultDict[Hashable, List[TimeStampedValue]], + window_s: float, +) -> DefaultDict[Hashable, List[TimeStampedValue]]: + """ + Merge multiple time-series dictionaries, typically contained within + InMemoryMetricsStore().data. For the same key across stores, time series + are merged with a windowed sum, where each series keeps only its latest + value per window before summing. + """ + merged: DefaultDict[Hashable, List[TimeStampedValue]] = defaultdict(list) + for timeseries_dict in timeseries_dicts: + for key, ts in timeseries_dict.items(): + if key in merged: + merged[key] = _merge_two_timeseries(merged[key], ts, window_s) + else: + # Window the data, even if the key is unique. + merged[key] = _merge_two_timeseries(ts, [], window_s) + return merged diff --git a/python/ray/serve/_private/replica.py b/python/ray/serve/_private/replica.py index 6ddfe873d97b..1ca38d1a37f2 100644 --- a/python/ray/serve/_private/replica.py +++ b/python/ray/serve/_private/replica.py @@ -328,11 +328,10 @@ def record_request_metrics(self, *, route: str, latency_ms: float, was_error: bo def _push_autoscaling_metrics(self) -> Dict[str, Any]: look_back_period = self._autoscaling_config.look_back_period_s + self._metrics_store.prune_keys_and_compact_data(time.time() - look_back_period) self._controller_handle.record_autoscaling_metrics.remote( replica_id=self._replica_id, - window_avg=self._metrics_store.window_average( - self._replica_id, time.time() - look_back_period - ), + window_avg=self._metrics_store.aggregate_avg([self._replica_id])[0], send_timestamp=time.time(), ) diff --git a/python/ray/serve/_private/router.py b/python/ray/serve/_private/router.py index 52e2335df4c0..7acc79f17fac 100644 --- a/python/ray/serve/_private/router.py +++ b/python/ray/serve/_private/router.py @@ -42,7 +42,11 @@ SERVE_LOGGER_NAME, ) from ray.serve._private.long_poll import LongPollClient, LongPollNamespace -from ray.serve._private.metrics_utils import InMemoryMetricsStore, MetricsPusher +from ray.serve._private.metrics_utils import ( + QUEUED_REQUESTS_KEY, + InMemoryMetricsStore, + MetricsPusher, +) from ray.serve._private.replica_result import ReplicaResult from ray.serve._private.request_router import PendingRequest, RequestRouter from ray.serve._private.request_router.pow_2_router import ( @@ -61,9 +65,6 @@ logger = logging.getLogger(SERVE_LOGGER_NAME) -QUEUED_REQUESTS_KEY = "queued" - - class RouterMetricsManager: """Manages metrics for the router.""" @@ -392,10 +393,11 @@ def _get_aggregated_requests(self): running_requests = dict() if RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE and self.autoscaling_config: look_back_period = self.autoscaling_config.look_back_period_s + self.metrics_store.prune_keys_and_compact_data( + time.time() - look_back_period + ) running_requests = { - replica_id: self.metrics_store.window_average( - replica_id, time.time() - look_back_period - ) + replica_id: self.metrics_store.aggregate_avg([replica_id])[0] # If data hasn't been recorded yet, return current # number of queued and ongoing requests. or num_requests diff --git a/python/ray/serve/tests/unit/test_metrics_utils.py b/python/ray/serve/tests/unit/test_metrics_utils.py index 52f8c84166e2..0e37d6b26d15 100644 --- a/python/ray/serve/tests/unit/test_metrics_utils.py +++ b/python/ray/serve/tests/unit/test_metrics_utils.py @@ -2,9 +2,18 @@ import sys import pytest +from toolz.tests.test_dicttoolz import defaultdict from ray._common.test_utils import async_wait_for_condition -from ray.serve._private.metrics_utils import InMemoryMetricsStore, MetricsPusher +from ray.serve._private.metrics_utils import ( + QUEUED_REQUESTS_KEY, + InMemoryMetricsStore, + MetricsPusher, + TimeStampedValue, + _bucket_latest_by_window, + _merge_two_timeseries, + merge_timeseries_dicts, +) from ray.serve._private.test_utils import MockAsyncTimer @@ -136,13 +145,26 @@ def new_f(s): await metrics_pusher.graceful_shutdown() +def assert_timeseries_equal(actual, expected): + assert len(actual) == len( + expected + ), f"Length mismatch: {len(actual)} vs {len(expected)}" + for i, (a, e) in enumerate(zip(actual, expected)): + assert ( + a.timestamp == e.timestamp + ), f"Timestamp mismatch at {i}: {a.timestamp} vs {e.timestamp}" + assert a.value == e.value, f"Value mismatch at {i}: {a.value} vs {e.value}" + + class TestInMemoryMetricsStore: def test_basics(self): s = InMemoryMetricsStore() s.add_metrics_point({"m1": 1}, timestamp=1) s.add_metrics_point({"m1": 2}, timestamp=2) - assert s.window_average("m1", window_start_timestamp_s=0) == 1.5 - assert s.max("m1", window_start_timestamp_s=0) == 2 + assert s.aggregate_avg(["m1"]) == (1.5, 1) + assert s.aggregate_max(["m1"]) == (2, 1) + assert s.aggregate_min(["m1"]) == (1, 1) + assert s.get_latest("m1") == 2 def test_out_of_order_insert(self): s = InMemoryMetricsStore() @@ -151,53 +173,42 @@ def test_out_of_order_insert(self): s.add_metrics_point({"m1": 3}, timestamp=3) s.add_metrics_point({"m1": 2}, timestamp=2) s.add_metrics_point({"m1": 4}, timestamp=4) - assert s.window_average("m1", window_start_timestamp_s=0) == 3 - assert s.max("m1", window_start_timestamp_s=0) == 5 + assert s.aggregate_avg(["m1"]) == (3, 1) + assert s.aggregate_max(["m1"]) == (5, 1) + assert s.aggregate_min(["m1"]) == (1, 1) def test_window_start_timestamp(self): s = InMemoryMetricsStore() - assert s.window_average("m1", window_start_timestamp_s=0) is None - assert s.max("m1", window_start_timestamp_s=0) is None + assert s.aggregate_avg(["m1"]) == (None, 0) + assert s.aggregate_max(["m1"]) == (None, 0) + assert s.aggregate_min(["m1"]) == (None, 0) s.add_metrics_point({"m1": 1}, timestamp=2) - assert s.window_average("m1", window_start_timestamp_s=0) == 1 - assert ( - s.window_average("m1", window_start_timestamp_s=10, do_compact=False) - is None - ) - - def test_compaction_window(self): - s = InMemoryMetricsStore() - - s.add_metrics_point({"m1": 1}, timestamp=1) - s.add_metrics_point({"m1": 2}, timestamp=2) - - assert ( - s.window_average("m1", window_start_timestamp_s=0, do_compact=False) == 1.5 - ) - s.window_average("m1", window_start_timestamp_s=1.1, do_compact=True) - # First record should be removed. - assert s.window_average("m1", window_start_timestamp_s=0, do_compact=False) == 2 - - def test_compaction_max(self): - s = InMemoryMetricsStore() - - s.add_metrics_point({"m1": 1}, timestamp=2) - s.add_metrics_point({"m1": 2}, timestamp=1) - - assert s.max("m1", window_start_timestamp_s=0, do_compact=False) == 2 - - s.window_average("m1", window_start_timestamp_s=1.1, do_compact=True) - - assert s.window_average("m1", window_start_timestamp_s=0, do_compact=False) == 1 + assert s.aggregate_avg(["m1"]) == (1, 1) + s.prune_keys_and_compact_data(10) + assert s.aggregate_avg(["m1"]) == (None, 0) def test_multiple_metrics(self): s = InMemoryMetricsStore() s.add_metrics_point({"m1": 1, "m2": -1}, timestamp=1) s.add_metrics_point({"m1": 2, "m2": -2}, timestamp=2) - assert s.window_average("m1", window_start_timestamp_s=0) == 1.5 - assert s.max("m1", window_start_timestamp_s=0) == 2 - assert s.max("m2", window_start_timestamp_s=0) == -1 + assert s.aggregate_avg(["m1"]) == (1.5, 1) + assert s.aggregate_avg(["m2"]) == (-1.5, 1) + assert s.aggregate_avg(["m1", "m2"]) == (0, 2) + assert s.aggregate_max(["m1"]) == (2, 1) + assert s.aggregate_max(["m2"]) == (-1, 1) + assert s.aggregate_max(["m1", "m2"]) == (2, 2) + assert s.aggregate_min(["m1"]) == (1, 1) + assert s.aggregate_min(["m2"]) == (-2, 1) + assert s.aggregate_min(["m1", "m2"]) == (-2, 2) + + def test_empty_key_mix(self): + s = InMemoryMetricsStore() + s.add_metrics_point({"m1": 1}, timestamp=1) + assert s.aggregate_avg(["m1", "m2"]) == (1, 1) + assert s.aggregate_max(["m1", "m2"]) == (1, 1) + assert s.aggregate_min(["m1", "m2"]) == (1, 1) + assert s.aggregate_avg(["m2"]) == (None, 0) def test_prune_keys_and_compact_data(self): s = InMemoryMetricsStore() @@ -210,6 +221,364 @@ def test_prune_keys_and_compact_data(self): assert len(s.data["m2"]) == 2 and s.data["m2"] == s._get_datapoints("m2", 1.1) assert len(s.data["m3"]) == 1 and s.data["m3"] == s._get_datapoints("m3", 1.1) + def test_merge_metrics_stores(self): + s1 = InMemoryMetricsStore() + s2 = InMemoryMetricsStore() + s3 = InMemoryMetricsStore() + s1.add_metrics_point( + {"m1": 1, "m2": 2, "m3": 3, QUEUED_REQUESTS_KEY: 1}, timestamp=1 + ) + s2.add_metrics_point({"m1": 2, "m2": 2, QUEUED_REQUESTS_KEY: 1}, timestamp=2) + s3.add_metrics_point({"m2": 10, QUEUED_REQUESTS_KEY: 10}, timestamp=2) + merged = merge_timeseries_dicts(s1.data, s2.data, s3.data, window_s=1) + + assert_timeseries_equal( + merged["m1"], [TimeStampedValue(1, 1), TimeStampedValue(2, 2)] + ) + assert_timeseries_equal( + merged["m2"], [TimeStampedValue(1, 2), TimeStampedValue(2, 12)] + ) + assert_timeseries_equal(merged["m3"], [TimeStampedValue(1, 3)]) + assert_timeseries_equal( + merged[QUEUED_REQUESTS_KEY], + [TimeStampedValue(1, 1), TimeStampedValue(2, 11)], + ) + + s4 = InMemoryMetricsStore() + s4.add_metrics_point( + {"m1": 100, "m2": 100, "m3": 100, QUEUED_REQUESTS_KEY: 10}, timestamp=0 + ) + + merged = merge_timeseries_dicts(s1.data, s2.data, s3.data, s4.data, window_s=2) + + # With window_s=2 and window start alignment: + # Window boundaries: [0,2), [2,4), etc. + # timestamp=0 (s4) and timestamp=1 (s1) -> window 0 + # timestamp=2 (s2, s3) -> window 1 + assert_timeseries_equal( + merged["m1"], + [TimeStampedValue(0, 101), TimeStampedValue(2, 2)], # 100+1=101, then 2 + ) + assert_timeseries_equal( + merged["m2"], + [ + TimeStampedValue(0, 102), + TimeStampedValue(2, 12), + ], # 100+2=102, then 2+10=12 + ) + assert_timeseries_equal( + merged["m3"], [TimeStampedValue(0, 103)] # 100+3=103, no data in window 1 + ) + assert_timeseries_equal( + merged[QUEUED_REQUESTS_KEY], + [TimeStampedValue(0, 11), TimeStampedValue(2, 11)], # 10+1=11, then 1+10=11 + ) + + s1_s2 = merge_timeseries_dicts(s1.data, s2.data, window_s=1) + s2_s1 = merge_timeseries_dicts(s2.data, s1.data, window_s=1) + s1_s2_s3_s4 = merge_timeseries_dicts( + s1.data, s2.data, s3.data, s4.data, window_s=1 + ) + s4_s1_s3_s2 = merge_timeseries_dicts( + s4.data, s1.data, s3.data, s2.data, window_s=1 + ) + + # dict equality -> compare per-key time series + for k in s1_s2: + assert_timeseries_equal(s1_s2[k], s2_s1[k]) + for k in s1_s2_s3_s4: + assert_timeseries_equal(s1_s2_s3_s4[k], s4_s1_s3_s2[k]) + + a1_none = merge_timeseries_dicts(s1.data, defaultdict(list), window_s=1) + for k in a1_none: + assert_timeseries_equal(a1_none[k], s1.data[k]) + + def test_bucket_latest_by_window_basic(self): + """Test basic functionality of _bucket_latest_by_window.""" + series = [ + TimeStampedValue(1.0, 10.0), + TimeStampedValue(1.5, 15.0), # Same window as 1.0, should overwrite + TimeStampedValue(3.0, 30.0), + ] + + # With window_s=1.0, start=0.0 + buckets = _bucket_latest_by_window(series, start=0.0, window_s=1.0) + + # Window 1: timestamps 1.0-2.0, latest value should be 15.0 + # Window 3: timestamp 3.0-4.0, value should be 30.0 + expected = {1: 15.0, 3: 30.0} + assert buckets == expected + + def test_bucket_latest_by_window_empty(self): + """Test _bucket_latest_by_window with empty series.""" + buckets = _bucket_latest_by_window([], start=0.0, window_s=1.0) + assert buckets == {} + + def test_bucket_latest_by_window_single_value(self): + """Test _bucket_latest_by_window with single value.""" + series = [TimeStampedValue(2.5, 25.0)] + buckets = _bucket_latest_by_window(series, start=0.0, window_s=1.0) + assert buckets == {2: 25.0} + + def test_bucket_latest_by_window_negative_timestamps(self): + """Test _bucket_latest_by_window with negative timestamps.""" + series = [ + TimeStampedValue(-1.5, 10.0), + TimeStampedValue(-0.5, 20.0), + TimeStampedValue(0.5, 30.0), + ] + buckets = _bucket_latest_by_window(series, start=-2.0, window_s=1.0) + # Window 0: -1.5 (index = (-1.5 - (-2.0)) // 1.0 = 0.5 // 1.0 = 0) + # Window 1: -0.5 (index = (-0.5 - (-2.0)) // 1.0 = 1.5 // 1.0 = 1) + # Window 2: 0.5 (index = (0.5 - (-2.0)) // 1.0 = 2.5 // 1.0 = 2) + expected = {0: 10.0, 1: 20.0, 2: 30.0} + assert buckets == expected + + def test_bucket_latest_by_window_very_small_window(self): + """Test _bucket_latest_by_window with very small windows.""" + series = [ + TimeStampedValue(1.001, 10.0), + TimeStampedValue(1.002, 20.0), # Different window + ] + buckets = _bucket_latest_by_window(series, start=1.0, window_s=0.001) + # With window_s=0.001: + # 1.001: (1.001 - 1.0) // 0.001 = 1.0 => window 1, but floor division gives 0 + # 1.002: (1.002 - 1.0) // 0.001 = 2.0 => window 2 + expected = { + 0: 10.0, + 2: 20.0, + } # Corrected based on actual floor division behavior + assert buckets == expected + + def test_merge_two_timeseries_both_empty(self): + """Test _merge_two_timeseries with both series empty.""" + result = _merge_two_timeseries([], [], window_s=1.0) + assert result == [] + + def test_merge_two_timeseries_one_empty(self): + """Test _merge_two_timeseries with one series empty.""" + t1 = [TimeStampedValue(1.0, 10.0), TimeStampedValue(2.0, 20.0)] + + result1 = _merge_two_timeseries(t1, [], window_s=1.0) + result2 = _merge_two_timeseries([], t1, window_s=1.0) + + # Results should be the same regardless of order + assert len(result1) == len(result2) == 2 + assert_timeseries_equal(result1, result2) + + def test_merge_two_timeseries_overlapping_windows(self): + """Test _merge_two_timeseries with values in overlapping time windows.""" + t1 = [TimeStampedValue(1.0, 10.0), TimeStampedValue(1.5, 15.0)] + t2 = [TimeStampedValue(1.3, 13.0), TimeStampedValue(1.8, 18.0)] + + result = _merge_two_timeseries(t1, t2, window_s=1.0) + + # With window_s=1.0 and earliest=1.0: + # start = 1.0 // 1.0 * 1.0 = 1.0 + # Window boundaries are [1.0, 2.0), [2.0, 3.0), etc. + # All values (1.0, 1.3, 1.5, 1.8) fall in window [1.0, 2.0) + # So we get 1 window + assert len(result) == 1 + + # Window 0: latest from t1 is 15.0 (1.5 > 1.0), latest from t2 is 18.0 (1.8 > 1.3), sum: 33.0 + assert result[0].value == 33.0 + + def test_merge_two_timeseries_zero_window(self): + """Test _merge_two_timeseries with zero window size.""" + t1 = [TimeStampedValue(1.0, 10.0)] + t2 = [TimeStampedValue(1.0, 20.0)] + + # Zero window should raise ValueError + with pytest.raises(ValueError, match="window_s must be positive, got 0"): + _merge_two_timeseries(t1, t2, window_s=0.0) + + def test_merge_two_timeseries_negative_window(self): + """Test _merge_two_timeseries with negative window size.""" + t1 = [TimeStampedValue(1.0, 10.0)] + t2 = [TimeStampedValue(1.0, 20.0)] + + # Negative window should raise ValueError + with pytest.raises(ValueError, match="window_s must be positive, got -1"): + _merge_two_timeseries(t1, t2, window_s=-1.0) + + def test_merge_two_timeseries_very_small_window(self): + """Test _merge_two_timeseries with very small window.""" + t1 = [TimeStampedValue(1.0, 10.0)] + t2 = [TimeStampedValue(1.0001, 20.0)] + + result = _merge_two_timeseries(t1, t2, window_s=0.0001) + + # With very small window, these should be in different buckets + assert len(result) == 2 + + def test_merge_two_timeseries_large_window(self): + """Test _merge_two_timeseries with very large window.""" + t1 = [TimeStampedValue(1.0, 10.0), TimeStampedValue(100.0, 15.0)] + t2 = [TimeStampedValue(50.0, 20.0), TimeStampedValue(200.0, 25.0)] + + result = _merge_two_timeseries(t1, t2, window_s=1000.0) + + # All values should be in the same window + assert len(result) == 1 + # Latest from t1: 15.0, latest from t2: 25.0, sum: 40.0 + assert result[0].value == 40.0 + + def test_merge_two_timeseries_duplicate_timestamps(self): + """Test _merge_two_timeseries with duplicate timestamps in same series.""" + t1 = [ + TimeStampedValue(1.0, 10.0), + TimeStampedValue(1.0, 15.0), # Duplicate timestamp + ] + t2 = [TimeStampedValue(1.0, 20.0)] + + result = _merge_two_timeseries(t1, t2, window_s=1.0) + + # Latest from t1 should be 15.0, t2 should be 20.0, sum: 35.0 + assert len(result) == 1 + assert result[0].value == 35.0 + + def test_merge_two_timeseries_floating_point_precision(self): + """Test _merge_two_timeseries with floating point precision edge cases.""" + # Test with timestamps that might have precision issues + t1 = [TimeStampedValue(0.1 + 0.2, 10.0)] # 0.30000000000000004 + t2 = [TimeStampedValue(0.3, 20.0)] + + result = _merge_two_timeseries(t1, t2, window_s=0.01) + + # These should be in the same window due to floating point precision + # but let's verify the behavior + assert len(result) >= 1 + + def test_merge_timeseries_dicts_empty_dicts(self): + """Test merge_timeseries_dicts with empty dictionaries.""" + result = merge_timeseries_dicts( + defaultdict(list), defaultdict(list), window_s=1.0 + ) + assert dict(result) == {} + + def test_merge_timeseries_dicts_single_dict(self): + """Test merge_timeseries_dicts with single dictionary.""" + data = defaultdict(list) + data["key1"] = [TimeStampedValue(1.0, 10.0)] + + result = merge_timeseries_dicts(data, window_s=1.0) + # With windowing applied, the result should have the same values but potentially different timestamps + expected = defaultdict(list) + expected["key1"] = [TimeStampedValue(1.0, 10.0)] # Window [1,2) starts at 1.0 + assert_timeseries_equal(result["key1"], expected["key1"]) + + def test_merge_timeseries_dicts_no_common_keys(self): + """Test merge_timeseries_dicts with dictionaries having no common keys.""" + d1 = defaultdict(list) + d1["key1"] = [TimeStampedValue(1.0, 10.0)] + + d2 = defaultdict(list) + d2["key2"] = [TimeStampedValue(2.0, 20.0)] + + result = merge_timeseries_dicts(d1, d2, window_s=1.0) + + assert "key1" in result + assert "key2" in result + assert len(result["key1"]) == 1 + assert len(result["key2"]) == 1 + + def test_merge_timeseries_dicts_many_stores(self): + """Test merge_timeseries_dicts with many stores.""" + stores = [] + for i in range(10): + store = defaultdict(list) + store["common_key"] = [TimeStampedValue(float(i), float(i * 10))] + stores.append(store) + + result = merge_timeseries_dicts(*stores, window_s=1.0) + + # Each value should be in its own window, sum should be 0+10+20+...+90 = 450 + assert "common_key" in result + total_value = sum(point.value for point in result["common_key"]) + assert total_value == 450.0 + + def test_merge_timeseries_dicts_zero_window(self): + """Test merge_timeseries_dicts with zero window size.""" + d1 = defaultdict(list) + d1["key1"] = [TimeStampedValue(1.0, 10.0)] + + d2 = defaultdict(list) + d2["key1"] = [TimeStampedValue(1.0, 20.0)] + + # Zero window should raise ValueError + with pytest.raises(ValueError, match="window_s must be positive, got 0"): + merge_timeseries_dicts(d1, d2, window_s=0.0) + + def test_merge_timeseries_dicts_negative_window(self): + """Test merge_timeseries_dicts with negative window size.""" + d1 = defaultdict(list) + d1["key1"] = [TimeStampedValue(1.0, 10.0)] + + # Negative window should raise ValueError + with pytest.raises(ValueError, match="window_s must be positive, got -1"): + merge_timeseries_dicts(d1, window_s=-1.0) + + def test_merge_timeseries_dicts_window_alignment_consistency(self): + """Test that window alignment is consistent regardless of input order.""" + # Create data that might expose window alignment issues + d1 = defaultdict(list) + d1["key1"] = [TimeStampedValue(1.1, 10.0)] + + d2 = defaultdict(list) + d2["key1"] = [TimeStampedValue(1.9, 20.0)] + + d3 = defaultdict(list) + d3["key1"] = [TimeStampedValue(2.1, 30.0)] + + # Test different orderings + result1 = merge_timeseries_dicts(d1, d2, d3, window_s=1.0) + result2 = merge_timeseries_dicts(d3, d1, d2, window_s=1.0) + result3 = merge_timeseries_dicts(d2, d3, d1, window_s=1.0) + + # Results should be the same regardless of order + assert_timeseries_equal(result1["key1"], result2["key1"]) + assert_timeseries_equal(result1["key1"], result3["key1"]) + + def test_merge_stores_bug_fix_window_center_calculation(self): + """Test for potential bug in window center calculation.""" + # This test checks if the window center calculation is correct + d1 = defaultdict(list) + d1["key1"] = [ + TimeStampedValue(0.0, 10.0), + TimeStampedValue(1.0, 15.0), + TimeStampedValue(2.0, 20.0), + TimeStampedValue(4.0, 30.0), + TimeStampedValue(5.0, 40.0), + ] + + result = merge_timeseries_dicts(d1, window_s=2.0) + + # With window_s=2.0 and window start alignment: + # Window [0,2): timestamps 0.0, 1.0 -> latest value 15.0 at window start 0.0 + # Window [2,4): timestamp 2.0 -> value 20.0 at window start 2.0 + # Window [4,6): timestamps 4.0, 5.0 -> latest value 40.0 at window start 4.0 + assert len(result["key1"]) == 3 + expected = [ + TimeStampedValue(timestamp=0.0, value=15.0), # Latest in window [0,2) + TimeStampedValue(timestamp=2.0, value=20.0), # Value in window [2,4) + TimeStampedValue(timestamp=4.0, value=40.0), # Latest in window [4,6) + ] + assert_timeseries_equal(result["key1"], expected) + + def test_merge_stores_preserves_value_precision(self): + """Test that merging preserves floating point precision of values.""" + d1 = defaultdict(list) + d1["key1"] = [TimeStampedValue(1.0, 0.1)] + + d2 = defaultdict(list) + d2["key1"] = [TimeStampedValue(1.0, 0.2)] + + result = merge_timeseries_dicts(d1, d2, window_s=1.0) + + # 0.1 + 0.2 should equal 0.3 exactly + assert len(result["key1"]) == 1 + assert abs(result["key1"][0].value - 0.3) < 1e-10 + if __name__ == "__main__": sys.exit(pytest.main(["-v", "-s", __file__])) From 12348ce03b8d26175d1165748ab8f6ef7b71d142 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Wed, 3 Sep 2025 19:04:47 -0700 Subject: [PATCH 439/634] [release] Init step for custom BYOD image build (#55398) - Add `custom_byod_build_init` to scan the tests and list out all custom images to build, then create a yaml file (that will be included when rayci runs) to launch the build jobs. - Add `custom_byod_build` as a python script that the jobs can call to build & push the images This needs to be merged after https://github.com/ray-project/ray/pull/55397 --------- Signed-off-by: kevin --- ci/ray_ci/oss_config.yaml | 1 + release/BUILD.bazel | 27 +++++ release/ray_release/configs/global_config.py | 5 + .../custom_byod_build_init_helper.py | 67 +++++++++++ .../scripts/custom_byod_build_init.py | 112 ++++++++++++++++++ .../test_custom_byod_build_init_helper.py | 77 ++++++++++++ 6 files changed, 289 insertions(+) create mode 100644 release/ray_release/custom_byod_build_init_helper.py create mode 100644 release/ray_release/scripts/custom_byod_build_init.py create mode 100644 release/ray_release/tests/test_custom_byod_build_init_helper.py diff --git a/ci/ray_ci/oss_config.yaml b/ci/ray_ci/oss_config.yaml index cf2b64aa1cc8..1eb2fa0d0a38 100644 --- a/ci/ray_ci/oss_config.yaml +++ b/ci/ray_ci/oss_config.yaml @@ -4,6 +4,7 @@ release_byod: ray_ml_cr_repo: ray-ml ray_llm_cr_repo: ray-llm byod_ecr: 029272617770.dkr.ecr.us-west-2.amazonaws.com + byod_ecr_region: us-west-2 aws_cr: 029272617770.dkr.ecr.us-west-2.amazonaws.com gcp_cr: us-west1-docker.pkg.dev/anyscale-oss-ci aws2gce_credentials: release/aws2gce_iam.json diff --git a/release/BUILD.bazel b/release/BUILD.bazel index 374c5ac8a121..79c3720d6e9b 100644 --- a/release/BUILD.bazel +++ b/release/BUILD.bazel @@ -456,6 +456,24 @@ py_test( ], ) +py_test( + name = "test_custom_byod_build_init_helper", + size = "small", + srcs = ["ray_release/tests/test_custom_byod_build_init_helper.py"], + data = [ + "ray_release/configs/oss_config.yaml", + ], + exec_compatible_with = ["//:hermetic_python"], + tags = [ + "release_unit", + "team:ci", + ], + deps = [ + ":ray_release", + bk_require("pytest"), + ], +) + py_test( name = "test_cluster_manager", size = "small", @@ -729,3 +747,12 @@ py_binary( ":ray_release", ], ) + +py_binary( + name = "custom_byod_build_init", + srcs = ["ray_release/scripts/custom_byod_build_init.py"], + exec_compatible_with = ["//:hermetic_python"], + deps = [ + ":ray_release", + ], +) diff --git a/release/ray_release/configs/global_config.py b/release/ray_release/configs/global_config.py index 9de06e104318..c3a07375a2d2 100644 --- a/release/ray_release/configs/global_config.py +++ b/release/ray_release/configs/global_config.py @@ -11,6 +11,7 @@ class GlobalConfig(TypedDict): byod_ray_ml_cr_repo: str byod_ray_llm_cr_repo: str byod_ecr: str + byod_ecr_region: str byod_aws_cr: str byod_gcp_cr: str state_machine_pr_aws_bucket: str @@ -67,6 +68,10 @@ def _init_global_config(config_file: str): config_content.get("byod", {}).get("byod_ecr") or config_content.get("release_byod", {}).get("byod_ecr") ), + byod_ecr_region=( + config_content.get("byod", {}).get("byod_ecr_region") + or config_content.get("release_byod", {}).get("byod_ecr_region") + ), byod_aws_cr=( config_content.get("byod", {}).get("aws_cr") or config_content.get("release_byod", {}).get("aws_cr") diff --git a/release/ray_release/custom_byod_build_init_helper.py b/release/ray_release/custom_byod_build_init_helper.py new file mode 100644 index 000000000000..cbdd90b55926 --- /dev/null +++ b/release/ray_release/custom_byod_build_init_helper.py @@ -0,0 +1,67 @@ +from typing import List, Tuple +import yaml +from ray_release.configs.global_config import get_global_config +from ray_release.logger import logger +from ray_release.test import Test + + +def _generate_custom_build_step_key(image: str) -> str: + # Buildkite step key cannot contain special characters, so they need to be replaced. + # Buildkite also limits step key length to 80 characters. + return ( + "custom_build_" + + image.replace("/", "_") + .replace(":", "_") + .replace(".", "_") + .replace("-", "_")[-40:] + ) + + +def get_images_from_tests(tests: List[Test]) -> List[Tuple[str, str, str]]: + """Get a list of custom BYOD images to build from a list of tests.""" + custom_byod_images = set() + for test in tests: + if not test.require_custom_byod_image(): + continue + custom_byod_image_build = ( + test.get_anyscale_byod_image(), + test.get_anyscale_base_byod_image(), + test.get_byod_post_build_script(), + ) + logger.info(f"To be built: {custom_byod_image_build[0]}") + custom_byod_images.add(custom_byod_image_build) + return list(custom_byod_images) + + +def create_custom_build_yaml(destination_file: str, tests: List[Test]) -> None: + config = get_global_config() + if not config or not config.get("byod_ecr_region") or not config.get("byod_ecr"): + raise ValueError("byod_ecr_region and byod_ecr must be set in the config") + """Create a yaml file for building custom BYOD images""" + custom_byod_images = get_images_from_tests(tests) + if not custom_byod_images: + return + build_config = {"group": "Custom images build", "steps": []} + + for image, base_image, post_build_script in custom_byod_images: + if not post_build_script: + continue + step = { + "label": f":tapioca: build custom: {image}", + "key": _generate_custom_build_step_key(image), + "instance_type": "release-medium", + "commands": [ + f"aws ecr get-login-password --region {config['byod_ecr_region']} | docker login --username AWS --password-stdin {config['byod_ecr']}", + f"bazelisk run //release:custom_byod_build -- --image-name {image} --base-image {base_image} --post-build-script {post_build_script}", + ], + } + if "ray-ml" in image: + step["depends_on"] = "anyscalemlbuild" + elif "ray-llm" in image: + step["depends_on"] = "anyscalellmbuild" + else: + step["depends_on"] = "anyscalebuild" + build_config["steps"].append(step) + + with open(destination_file, "w") as f: + yaml.dump(build_config, f, default_flow_style=False, sort_keys=False) diff --git a/release/ray_release/scripts/custom_byod_build_init.py b/release/ray_release/scripts/custom_byod_build_init.py new file mode 100644 index 000000000000..ba0df452e2ce --- /dev/null +++ b/release/ray_release/scripts/custom_byod_build_init.py @@ -0,0 +1,112 @@ +import os +from typing import Tuple +from pathlib import Path +import sys + +import click + +from ray_release.buildkite.filter import filter_tests +from ray_release.buildkite.settings import get_pipeline_settings +from ray_release.config import ( + read_and_validate_release_test_collection, + RELEASE_TEST_CONFIG_FILES, +) +from ray_release.configs.global_config import init_global_config +from ray_release.exception import ReleaseTestConfigError, ReleaseTestCLIError +from ray_release.logger import logger +from ray_release.custom_byod_build_init_helper import create_custom_build_yaml + + +@click.command( + help="Create a rayci yaml file for building custom BYOD images based on tests." +) +@click.option( + "--test-collection-file", + type=str, + multiple=True, + help="Test collection file, relative path to ray repo.", +) +@click.option( + "--run-jailed-tests", + is_flag=True, + show_default=True, + default=False, + help=("Will run jailed tests."), +) +@click.option( + "--run-unstable-tests", + is_flag=True, + show_default=True, + default=False, + help=("Will run unstable tests."), +) +@click.option( + "--global-config", + default="oss_config.yaml", + type=click.Choice( + [x.name for x in (Path(__file__).parent.parent / "configs").glob("*.yaml")] + ), + help="Global config to use for test execution.", +) +@click.option( + "--frequency", + default=None, + type=click.Choice(["manual", "nightly", "nightly-3x", "weekly"]), + help="Run frequency of the test", +) +@click.option( + "--test-filters", + default=None, + type=str, + help="Test filters by prefix/regex.", +) +def main( + test_collection_file: Tuple[str], + run_jailed_tests: bool = False, + run_unstable_tests: bool = False, + global_config: str = "oss_config.yaml", + frequency: str = None, + test_filters: str = None, +): + global_config_file = os.path.join( + os.path.dirname(__file__), "..", "configs", global_config + ) + init_global_config(global_config_file) + settings = get_pipeline_settings() + + frequency = frequency or settings["frequency"] + prefer_smoke_tests = settings["prefer_smoke_tests"] + test_filters = test_filters or settings["test_filters"] + + try: + test_collection = read_and_validate_release_test_collection( + test_collection_file or RELEASE_TEST_CONFIG_FILES + ) + except ReleaseTestConfigError as e: + raise ReleaseTestConfigError( + "Cannot load test yaml file.\nHINT: If you're kicking off tests for a " + "specific commit on Buildkite to test Ray wheels, after clicking " + "'New build', leave the commit at HEAD, and only specify the commit " + "in the dialog that asks for the Ray wheels." + ) from e + + filtered_tests = filter_tests( + test_collection, + frequency=frequency, + test_filters=test_filters, + prefer_smoke_tests=prefer_smoke_tests, + run_jailed_tests=run_jailed_tests, + run_unstable_tests=run_unstable_tests, + ) + logger.info(f"Found {len(filtered_tests)} tests to run.") + if len(filtered_tests) == 0: + raise ReleaseTestCLIError( + "Empty test collection. The selected frequency or filter did " + "not return any tests to run. Adjust your filters." + ) + tests = [test for test, _ in filtered_tests] + create_custom_build_yaml(".buildkite/release/custom_byod_build.rayci.yml", tests) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/release/ray_release/tests/test_custom_byod_build_init_helper.py b/release/ray_release/tests/test_custom_byod_build_init_helper.py new file mode 100644 index 000000000000..895f192d3278 --- /dev/null +++ b/release/ray_release/tests/test_custom_byod_build_init_helper.py @@ -0,0 +1,77 @@ +import os +import tempfile +import sys +import pytest +from unittest import mock +import yaml + +from ray_release.custom_byod_build_init_helper import create_custom_build_yaml +from ray_release.configs.global_config import init_global_config +from ray_release.bazel import bazel_runfile +from ray_release.test import Test +from ray_release.configs.global_config import get_global_config + + +init_global_config(bazel_runfile("release/ray_release/configs/oss_config.yaml")) + + +@mock.patch("ray_release.custom_byod_build_init_helper.get_images_from_tests") +def test_create_custom_build_yaml(mock_get_images_from_tests): + config = get_global_config() + custom_byod_images = [ + ( + "ray-project/ray-ml:abc123-custom", + "ray-project/ray-ml:abc123-base", + "custom_script.sh", + ), + ("ray-project/ray-ml:abc123-custom", "ray-project/ray-ml:abc123-base", ""), + ( + "ray-project/ray-ml:nightly-py37-cpu-custom-abcdef123456789abc123456789", + "ray-project/ray-ml:nightly-py37-cpu-base", + "custom_script.sh", + ), # longer than 40 chars + ] + mock_get_images_from_tests.return_value = custom_byod_images + + # List of dummy tests + tests = [ + Test( + name="test_1", + frequency="manual", + group="test_group", + team="test_team", + working_dir="test_working_dir", + ), + Test( + name="test_2", + frequency="manual", + group="test_group", + team="test_team", + working_dir="test_working_dir", + ), + ] + with tempfile.TemporaryDirectory() as tmpdir: + create_custom_build_yaml( + os.path.join(tmpdir, "custom_byod_build.rayci.yml"), tests + ) + with open(os.path.join(tmpdir, "custom_byod_build.rayci.yml"), "r") as f: + content = yaml.safe_load(f) + assert content["group"] == "Custom images build" + assert len(content["steps"]) == 2 + assert ( + f"--region {config['byod_ecr_region']}" + in content["steps"][0]["commands"][0] + ) + assert f"{config['byod_ecr']}" in content["steps"][0]["commands"][0] + assert ( + f"--image-name {custom_byod_images[0][0]}" + in content["steps"][0]["commands"][1] + ) + assert ( + f"--image-name {custom_byod_images[2][0]}" + in content["steps"][1]["commands"][1] + ) + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", __file__])) From 5e4f0653fd56d7d4bab7b3dffce34b7169edcde6 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Wed, 3 Sep 2025 20:10:57 -0700 Subject: [PATCH 440/634] [release] Baseline release test (#56017) Basic hello world release test with minimal cluster compute to smoke test release test pipeline. --------- Signed-off-by: kevin --- release/hello_world_tests/hello_world.py | 14 +++++++++++++ .../hello_world_compute_config.yaml | 8 ++++++++ release/release_tests.yaml | 20 +++++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 release/hello_world_tests/hello_world.py create mode 100644 release/hello_world_tests/hello_world_compute_config.yaml diff --git a/release/hello_world_tests/hello_world.py b/release/hello_world_tests/hello_world.py new file mode 100644 index 000000000000..84756ed0a2f4 --- /dev/null +++ b/release/hello_world_tests/hello_world.py @@ -0,0 +1,14 @@ +import ray + + +@ray.remote +def hello_world(): + return "Hello, world!" + + +def main(): + print(ray.get(hello_world.remote())) + + +if __name__ == "__main__": + main() diff --git a/release/hello_world_tests/hello_world_compute_config.yaml b/release/hello_world_tests/hello_world_compute_config.yaml new file mode 100644 index 000000000000..ca578bf09d6b --- /dev/null +++ b/release/hello_world_tests/hello_world_compute_config.yaml @@ -0,0 +1,8 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +head_node_type: + name: head_node + instance_type: m5.xlarge + +worker_node_types: [] diff --git a/release/release_tests.yaml b/release/release_tests.yaml index 835ceacbe3f6..b124ef316c81 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -76,6 +76,26 @@ # # It can then let the test fail, e.g. if a metric regression is observed. # alert: default +####################### +# Baseline test +####################### +- name: hello_world + team: reef + group: hello_world + frequency: nightly + working_dir: hello_world_tests + + cluster: + byod: {} + cluster_compute: hello_world_compute_config.yaml + + run: + timeout: 1800 + script: python hello_world.py + + variations: + - __suffix__: aws + ####################### # Cluster scaling tests ####################### From edbbc9467f0239f744f6cee7764148c16d45006a Mon Sep 17 00:00:00 2001 From: Ibrahim Rabbani Date: Wed, 3 Sep 2025 21:54:35 -0700 Subject: [PATCH 441/634] [core] (cgroups 2/n) adding integration tests for the cgroup sysfs driver. (#55063) Signed-off-by: irabbani Signed-off-by: Ibrahim Rabbani Signed-off-by: Ibrahim Rabbani Co-authored-by: Edward Oakes --- .buildkite/core.rayci.yml | 9 +- src/ray/common/cgroup2/BUILD.bazel | 23 +- src/ray/common/cgroup2/cgroup_test_utils.cc | 286 +++++++++ src/ray/common/cgroup2/cgroup_test_utils.h | 134 +++++ .../cgroup2/integration_tests/BUILD.bazel | 24 + .../sysfs_cgroup_driver_integration_test.cc | 558 ++++++++++++++++++ ...roup_driver_integration_test_entrypoint.sh | 116 ++++ ..._cgroup_driver_integration_test_fixture.sh | 168 ++++++ src/ray/common/cgroup2/sysfs_cgroup_driver.cc | 4 +- src/ray/common/cgroup2/tests/BUILD.bazel | 23 +- .../common/cgroup2/tests/cgroup_test_utils.cc | 107 ---- .../common/cgroup2/tests/cgroup_test_utils.h | 71 --- .../cgroup2/tests/sysfs_cgroup_driver_test.cc | 4 +- 13 files changed, 1319 insertions(+), 208 deletions(-) create mode 100644 src/ray/common/cgroup2/cgroup_test_utils.cc create mode 100644 src/ray/common/cgroup2/cgroup_test_utils.h create mode 100644 src/ray/common/cgroup2/integration_tests/BUILD.bazel create mode 100644 src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc create mode 100755 src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test_entrypoint.sh create mode 100755 src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test_fixture.sh delete mode 100644 src/ray/common/cgroup2/tests/cgroup_test_utils.cc delete mode 100644 src/ray/common/cgroup2/tests/cgroup_test_utils.h diff --git a/.buildkite/core.rayci.yml b/.buildkite/core.rayci.yml index e57b4d61e8b8..c1291858d276 100644 --- a/.buildkite/core.rayci.yml +++ b/.buildkite/core.rayci.yml @@ -291,15 +291,14 @@ steps: - "3.11" - "3.12" - "3.13" - - # cpp tests - label: ":ray: core: cgroup tests" tags: core_cpp instance_type: medium commands: - - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --only-tags=cgroup --build-type cgroup - --privileged --cache-test-results - + - bazel run //ci/ray_ci:test_in_docker -- //:all //src/ray/common/cgroup2/tests/... core --build-type clang --cache-test-results + - docker run --privileged -i --rm --volume /tmp/artifacts:/artifact-mount --shm-size=2.5gb + "$${RAYCI_WORK_REPO}":"$${RAYCI_BUILD_ID}"-corebuild /bin/bash + "./src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test_entrypoint.sh" - label: ":ray: core: cpp tests" tags: core_cpp instance_type: medium diff --git a/src/ray/common/cgroup2/BUILD.bazel b/src/ray/common/cgroup2/BUILD.bazel index b74e0505428c..2822450214c2 100644 --- a/src/ray/common/cgroup2/BUILD.bazel +++ b/src/ray/common/cgroup2/BUILD.bazel @@ -5,8 +5,8 @@ ray_cc_library( hdrs = [ "cgroup_driver_interface.h", ], - tags = [ - "no_windows", + target_compatible_with = [ + "@platforms//os:linux", ], deps = [ "//src/ray/common:status", @@ -20,8 +20,8 @@ ray_cc_library( hdrs = [ "sysfs_cgroup_driver.h", ], - tags = [ - "no_windows", + target_compatible_with = [ + "@platforms//os:linux", ], deps = [ ":cgroup_driver_interface", @@ -31,3 +31,18 @@ ray_cc_library( "@com_google_absl//absl/strings", ], ) + +ray_cc_library( + name = "cgroup_test_utils", + srcs = ["cgroup_test_utils.cc"], + hdrs = ["cgroup_test_utils.h"], + target_compatible_with = [ + "@platforms//os:linux", + ], + deps = [ + "//src/ray/common:id", + "//src/ray/common:status", + "//src/ray/common:status_or", + "@com_google_absl//absl/strings:str_format", + ], +) diff --git a/src/ray/common/cgroup2/cgroup_test_utils.cc b/src/ray/common/cgroup2/cgroup_test_utils.cc new file mode 100644 index 000000000000..82452818eefc --- /dev/null +++ b/src/ray/common/cgroup2/cgroup_test_utils.cc @@ -0,0 +1,286 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/common/cgroup2/cgroup_test_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/strings/str_format.h" +#include "ray/common/id.h" +#include "ray/common/status.h" +#include "ray/common/status_or.h" +#include "ray/util/logging.h" + +ray::StatusOr> TempCgroupDirectory::Create( + const std::string &base_path, mode_t mode) { + std::string random_name = ray::UniqueID::FromRandom().Hex(); + std::string name = random_name.substr(0, std::min(6, random_name.size())); + std::string path = base_path + std::filesystem::path::preferred_separator + name; + if (mkdir(path.c_str(), mode) == -1) { + return ray::Status::IOError( + absl::StrFormat("Failed to create cgroup directory at path %s.\n" + "Cgroup tests expect tmpfs and cgroupv2 to be mounted " + "and only run on Linux.\n" + "Error: %s", + path, + strerror(errno))); + } + auto output = std::make_unique(std::move(name), std::move(path)); + return output; +} + +TempCgroupDirectory::~TempCgroupDirectory() noexcept(false) { + RAY_CHECK(rmdir(path_.c_str()) != -1) << absl::StrFormat( + "Failed to delete a cgroup directory at %s with error %s. Please manually " + "delete it with rmdir.", + path_, + strerror(errno)); +} + +ray::StatusOr> TempDirectory::Create() { + std::string path = "/tmp/XXXXXX"; + char *ret = mkdtemp(path.data()); + if (ret == nullptr) { + return ray::Status::Invalid( + absl::StrFormat("Failed to create a temp directory on tmpfs with error %s." + "Cgroup tests expect tmpfs to be mounted and only run on Linux.", + strerror(errno))); + } + std::unique_ptr temp_dir = + std::make_unique(std::move(path)); + return ray::StatusOr>(std::move(temp_dir)); +} + +TempDirectory::~TempDirectory() { + std::error_code error_code; + RAY_CHECK(std::filesystem::remove_all(path_, error_code)) << absl::StrFormat( + "Failed to delete temp directory at %s with error %s. Please manually " + "delete it with rmdir.", + path_, + error_code.message()); +} + +/** + Note: clone3 supports creating a process inside a cgroup instead of creating + and then moving. However, clone3 does not have a glibc wrapper and + must be called directly using syscall syscall (see man 2 syscall). + This function needs linux kernel >= 5.7 to use the CLONE_INTO_CGROUP flag. +*/ +#ifdef CLONE_INTO_CGROUP +ray::StatusOr> StartChildProcessInCgroup( + const std::string &cgroup_path) { + int cgroup_fd = open(cgroup_path.c_str(), O_RDONLY); + if (cgroup_fd == -1) { + return ray::Status::InvalidArgument( + absl::StrFormat("Unable to open fd for cgroup at %s with error %s.", + cgroup_path, + strerror(errno))); + } + + // Will be set by clone3 if a child process is successfully created. + pid_t child_pidfd = -1; + + clone_args cl_args = {}; + cl_args.flags = CLONE_PIDFD | CLONE_INTO_CGROUP; + cl_args.cgroup = cgroup_fd; + + // Can be used both as a pid and as a fd. + cl_args.pidfd = ((__u64)((uintptr_t)(&child_pidfd))); + + int child_pid = -1; + + if ((child_pid = syscall(__NR_clone3, &cl_args, sizeof(struct clone_args))) == -1) { + close(cgroup_fd); + return ray::Status::Invalid( + absl::StrFormat("Failed to clone process into cgroup %s with error %s.", + cgroup_path, + strerror(errno))); + } + + if (child_pid == 0) { + // Child process will wait for parent to unblock it. + pause(); + _exit(0); + } + + // Parent process will continue here. + close(cgroup_fd); + return std::make_pair(child_pid, static_cast(child_pidfd)); +} +#else +// Fallback for older kernels. Uses fork/exec instead. +ray::StatusOr> StartChildProcessInCgroup( + const std::string &cgroup_path) { + int new_pid = fork(); + if (new_pid == -1) { + return ray::Status::Invalid( + absl::StrFormat("Failed to fork process with error %s.", strerror(errno))); + } + + if (new_pid == 0) { + // Child process will pause and wait for parent to terminate and reap it. + pause(); + _exit(0); + } + + std::string cgroup_proc_file_path = cgroup_path + "/cgroup.procs"; + + // Parent process has to move the process into a cgroup. + int cgroup_fd = open(cgroup_proc_file_path.c_str(), O_RDWR); + + if (cgroup_fd == -1) { + return ray::Status::Invalid( + absl::StrFormat("Failed to open cgroup procs file at path %s with error %s.", + cgroup_proc_file_path, + strerror(errno))); + } + + std::string pid_to_write = std::to_string(new_pid); + + if (write(cgroup_fd, pid_to_write.c_str(), pid_to_write.size()) == -1) { + // Best effort killing of the child process because we couldn't move it + // into the cgroup. + kill(SIGKILL, new_pid); + close(cgroup_fd); + return ray::Status::Invalid( + absl::StrFormat("Failed to write pid %i to cgroup procs file %s with error %s.", + new_pid, + cgroup_proc_file_path, + strerror(errno))); + } + + close(cgroup_fd); + + int child_pidfd = static_cast(syscall(SYS_pidfd_open, new_pid, 0)); + if (child_pidfd == -1) { + // Best effort killing of the child process because we couldn't create + // a pidfd from the process. + kill(SIGKILL, new_pid); + close(cgroup_fd); + return ray::Status::Invalid( + absl::StrFormat("Failed to create process fd for pid %i with error %s.", + new_pid, + strerror(errno))); + } + return std::make_pair(new_pid, child_pidfd); +} +#endif + +ray::Status TerminateChildProcessAndWaitForTimeout(pid_t pid, int fd, int timeout_ms) { + if (kill(pid, SIGKILL) == -1) { + return ray::Status::InvalidArgument(absl::StrFormat( + "Failed to send SIGTERM to pid: %i with error %s.", pid, strerror(errno))); + } + struct pollfd poll_fd = { + .fd = fd, + .events = POLLIN, + }; + + int poll_status = poll(&poll_fd, 1, timeout_ms); + if (poll_status == -1) { + return ray::Status::InvalidArgument( + absl::StrFormat("Failed to poll process pid: %i, fd: %i with error %s. Process " + "was not killed. Kill it manually to prevent a leak.", + pid, + fd, + strerror(errno))); + } + if (poll_status == 0) { + return ray::Status::Invalid( + absl::StrFormat("Process pid: %i, fd: %i was not killed within the timeout of " + "%ims. Kill it manually to prevent a leak.", + pid, + fd, + timeout_ms)); + } + siginfo_t dummy = {0}; + int wait_id_status = waitid(P_PID, static_cast(fd), &dummy, WEXITED); + if (wait_id_status == -1) { + if (errno != ECHILD) + return ray::Status::Invalid( + absl::StrFormat("Failed to wait for process pid: %i, fd: %i with error %s. " + "Process was not reaped, but " + "it will be reaped by init after program exits.", + pid, + fd, + strerror(errno))); + }; + return ray::Status::OK(); +} + +TempFile::TempFile(std::string path) { + path_ = path; + fd_ = open(path_.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); // NOLINT + RAY_CHECK(fd_ != -1) << absl::StrFormat( + "Failed to create a temp file at path %s with error %s. Cgroup tests expect " + "tmpfs to be mounted and only run on Linux.", + path_, + strerror(errno)); + file_output_stream_ = std::ofstream(path_, std::ios::trunc); + RAY_CHECK(file_output_stream_.is_open()) << absl::StrFormat( + "Failed to open file %s on tmpfs with error %s", path_, strerror(errno)); +} + +TempFile::TempFile() { + fd_ = mkstemp(path_.data()); // NOLINT + if (fd_ == -1) { + throw std::runtime_error( + "Failed to create a temp file. Cgroup tests expect tmpfs to be " + "mounted " + "and only run on Linux"); + } + file_output_stream_ = std::ofstream(path_, std::ios::trunc); + RAY_CHECK(file_output_stream_.is_open()) + << absl::StrFormat("Could not open temporary file at path %s.", path_); +} + +TempFile::~TempFile() { + RAY_CHECK(close(fd_) != -1) << absl::StrFormat( + "Failed to close file descriptor with error %s.", strerror(errno)); + file_output_stream_.close(); + RAY_CHECK(unlink(path_.c_str()) != -1) + << absl::StrFormat("Failed to unlink temporary file at path %s with error %s.", + path_, + strerror(errno)); +} + +void TempFile::AppendLine(const std::string &line) { + file_output_stream_ << line; + file_output_stream_.flush(); + // All current callers treat this is as a fatal error so this is a RAY_CHECK + // instead of returning a Status. + RAY_CHECK(file_output_stream_.good()) + << absl::StrFormat("Failed to write to temporary file at path %s.", path_); +} diff --git a/src/ray/common/cgroup2/cgroup_test_utils.h b/src/ray/common/cgroup2/cgroup_test_utils.h new file mode 100644 index 000000000000..29cb8364eb84 --- /dev/null +++ b/src/ray/common/cgroup2/cgroup_test_utils.h @@ -0,0 +1,134 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include "ray/common/status.h" +#include "ray/common/status_or.h" + +class TempCgroupDirectory { + public: + static ray::StatusOr> Create( + const std::string &base_path, mode_t mode = 0777); + + TempCgroupDirectory() = default; + explicit TempCgroupDirectory(std::string &&name, std::string &&path) + : name_(name), path_(path) {} + + TempCgroupDirectory(const TempCgroupDirectory &) = delete; + TempCgroupDirectory(TempCgroupDirectory &&) = delete; + TempCgroupDirectory &operator=(const TempCgroupDirectory &) = delete; + TempCgroupDirectory &operator=(TempCgroupDirectory &&) = delete; + + const std::string &GetPath() const { return path_; } + const std::string &GetName() const { return name_; } + + ~TempCgroupDirectory() noexcept(false); + + private: + std::string name_; + std::string path_; +}; + +class TempDirectory { + public: + static ray::StatusOr> Create(); + explicit TempDirectory(std::string &&path) : path_(path) {} + + TempDirectory(const TempDirectory &) = delete; + TempDirectory(TempDirectory &&) = delete; + TempDirectory &operator=(const TempDirectory &) = delete; + TempDirectory &operator=(TempDirectory &&) = delete; + + const std::string &GetPath() const { return path_; } + + ~TempDirectory(); + + private: + const std::string path_; +}; + +class TempFile { + public: + explicit TempFile(std::string path); + TempFile(); + + TempFile(TempFile &other) = delete; + TempFile(TempFile &&other) = delete; + TempFile operator=(TempFile &other) = delete; + TempFile &operator=(TempFile &&other) = delete; + + ~TempFile(); + void AppendLine(const std::string &line); + + const std::string &GetPath() const { return path_; } + + private: + std::string path_ = "/tmp/XXXXXX"; + std::ofstream file_output_stream_; + int fd_; +}; + +/** + Starts a process in the given cgroup. Assumes the cgroup already exists and + that the caller has read-write the lowest-common ancestor of the cgroup + the current process is running in and the target cgroup. + + The spawned process will wait forever for the parent to unblock it and then + reap it. + + @param target_cgroup_path target cgroup to create a process in. + @return Status::OK with a pair of the processfd and pid if successful + @return Status::InvalidArgument if target cgroup does exist or current process + has insufficient permissions. + @return Status::Invalid if process cannot be forked/cloned or processfd cannot + be obtained. +*/ +ray::StatusOr> StartChildProcessInCgroup( + const std::string &target_cgroup_path); + +/** + Kills the specified process and polls its processfd to reap it with a timeout. + + @param pid + @param process_fd can be used as a fd and as a pid. It can be created using + clone or pidfd_open or clone. + @param timeout_ms + + @return Status::OK if successfully terminated the process and reaped it. + @return Status::InvalidArgument if could not send SIGKILL to the process or poll its fd. + @return Status::Invalid if could not reap the process within the timeout. +*/ +ray::Status TerminateChildProcessAndWaitForTimeout(pid_t pid, int fd, int timeout_ms); + +// Convenience methods so you can print the TempCgroupDirectory's path directly +// instead of calling temp_cgroup_dir.GetPath() everytime. +std::ostream &operator<<(std::ostream &os, const TempCgroupDirectory &temp_cgroup_dir) { + return os << temp_cgroup_dir.GetPath(); +} + +std::ostream &operator<<(std::ostream &os, + const std::unique_ptr &ptr) { + if (ptr == nullptr) { + return os << ""; + } + return os << *ptr; +} diff --git a/src/ray/common/cgroup2/integration_tests/BUILD.bazel b/src/ray/common/cgroup2/integration_tests/BUILD.bazel new file mode 100644 index 000000000000..fda28fe9a638 --- /dev/null +++ b/src/ray/common/cgroup2/integration_tests/BUILD.bazel @@ -0,0 +1,24 @@ +load("//bazel:ray.bzl", "ray_cc_test") + +# This test is run through sysfs_cgroup_driver_integration_test_entrypoint.sh +# See sysfs_cgroup_driver_integration_test_entrypoint.sh for instructions +# for how to run locally. +ray_cc_test( + name = "sysfs_cgroup_driver_integration_test", + srcs = ["sysfs_cgroup_driver_integration_test.cc"], + tags = [ + "cgroup", + "team:core", + ], + target_compatible_with = [ + "@platforms//os:linux", + ], + deps = [ + "//src/ray/common:status", + "//src/ray/common:status_or", + "//src/ray/common/cgroup2:cgroup_test_utils", + "//src/ray/common/cgroup2:sysfs_cgroup_driver", + "@com_google_absl//absl/strings:str_format", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc b/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc new file mode 100644 index 000000000000..52dbf51eb72e --- /dev/null +++ b/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc @@ -0,0 +1,558 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "ray/common/cgroup2/cgroup_test_utils.h" +#include "ray/common/cgroup2/sysfs_cgroup_driver.h" +#include "ray/common/status.h" + +constexpr const char *ENV_VAR_TEST_CGROUP_PATH = "CGROUP_PATH"; + +namespace ray { + +class SysFsCgroupDriverIntegrationTest : public ::testing::Test { + protected: + static void SetUpTestSuite() { + const char *cgroup_env = std::getenv(ENV_VAR_TEST_CGROUP_PATH); + if (!cgroup_env || std::string(cgroup_env).empty()) { + throw std::runtime_error("Environment variable CGROUP_PATH not set or empty"); + } + test_cgroup_path_ = cgroup_env; + } + + static const std::string &GetTestCgroupPath() { return test_cgroup_path_; } + + inline static std::string test_cgroup_path_; +}; + +TEST_F(SysFsCgroupDriverIntegrationTest, + SysFsCgroupDriverIntegrationTestFailsIfNoCgroupTestPathSpecified) { + ASSERT_FALSE(test_cgroup_path_.empty()) + << "These integration tests cannot be run without the " + "environment variable CGROUP_TEST_PATH"; +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + CheckCgroupFailsIfCgroupv2PathButNoReadPermissions) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, 0000); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.CheckCgroup(cgroup_dir->GetPath()); + EXPECT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + CheckCgroupFailsIfCgroupv2PathButNoWritePermissions) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.CheckCgroup(cgroup_dir->GetPath()); + EXPECT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + CheckCgroupFailsIfCgroupv2PathButNoExecPermissions) { + auto cgroup_dir_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR | S_IWUSR); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.CheckCgroup(cgroup_dir->GetPath()); + EXPECT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + CheckCgroupSucceedsIfCgroupv2PathAndReadWriteExecPermissions) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.CheckCgroup(cgroup_dir->GetPath()); + EXPECT_TRUE(s.ok()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, CreateCgroupFailsIfAlreadyExists) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.CreateCgroup(cgroup_dir->GetPath()); + ASSERT_TRUE(s.IsAlreadyExists()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, CreateCgroupFailsIfAncestorCgroupDoesNotExist) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + std::string non_existent_path = cgroup_dir->GetPath() + + std::filesystem::path::preferred_separator + "no" + + std::filesystem::path::preferred_separator + "bueno"; + Status s = driver.CreateCgroup(non_existent_path); + EXPECT_TRUE(s.IsNotFound()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, CreateCgroupFailsIfOnlyReadPermissions) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + std::string child_cgroup_path = + cgroup_dir->GetPath() + std::filesystem::path::preferred_separator + "child"; + Status s = driver.CreateCgroup(child_cgroup_path); + EXPECT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, CreateCgroupFailsIfOnlyReadWritePermissions) { + auto cgroup_dir_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR | S_IWUSR); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + std::string child_cgroup_path = + cgroup_dir->GetPath() + std::filesystem::path::preferred_separator + "child"; + Status s = driver.CreateCgroup(child_cgroup_path); + EXPECT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + CreateCgroupSucceedsIfParentExistsAndReadWriteExecPermissions) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + std::string child_cgroup_path = + cgroup_dir->GetPath() + std::filesystem::path::preferred_separator + "child"; + Status s = driver.CreateCgroup(child_cgroup_path); + EXPECT_TRUE(s.ok()) << s.ToString(); + Status check_status = driver.CheckCgroup(child_cgroup_path); + EXPECT_TRUE(check_status.ok()) << check_status.ToString(); + ASSERT_EQ(rmdir(child_cgroup_path.c_str()), 0) + << "Failed to cleanup test cgroup at path " << child_cgroup_path << ".\n" + << "Error: " << strerror(errno); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + GetAvailableControllersFailsIfCgroupDoesNotExist) { + std::string non_existent_path = test_cgroup_path_ + + std::filesystem::path::preferred_separator + "no" + + std::filesystem::path::preferred_separator + "bueno"; + SysFsCgroupDriver driver; + StatusOr> s = + driver.GetAvailableControllers(non_existent_path); + EXPECT_TRUE(s.IsNotFound()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + GetAvailableControllersFailsIfReadWriteButNotExecutePermissions) { + auto cgroup_dir_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR | S_IWUSR); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + std::unique_ptr cgroup_dir = + std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + StatusOr> s = + driver.GetAvailableControllers(cgroup_dir->GetPath()); + EXPECT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + GetAvailableControllersSucceedsWithCPUAndMemoryControllersOnBaseCgroup) { + SysFsCgroupDriver driver; + StatusOr> s = + driver.GetAvailableControllers(test_cgroup_path_); + EXPECT_TRUE(s.ok()) << s.ToString(); + std::unordered_set controllers = std::move(s.value()); + EXPECT_TRUE(controllers.find("cpu") != controllers.end()) + << "Cgroup integration tests expect the base cgroup at " << test_cgroup_path_ + << " has the cpu controller available"; +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + GetAvailableControllersSucceedsWithNoAvailableControllers) { + auto parent_cgroup_dir_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(parent_cgroup_dir_or_status.ok()) << parent_cgroup_dir_or_status.ToString(); + std::unique_ptr parent_cgroup = + std::move(parent_cgroup_dir_or_status.value()); + auto child_cgroup_dir_or_status = + TempCgroupDirectory::Create(parent_cgroup->GetPath(), S_IRWXU); + ASSERT_TRUE(child_cgroup_dir_or_status.ok()) << child_cgroup_dir_or_status.ToString(); + std::unique_ptr child_cgroup = + std::move(child_cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + StatusOr> s = + driver.GetAvailableControllers(child_cgroup->GetPath()); + EXPECT_TRUE(s.ok()) << s.ToString(); + std::unordered_set controllers = std::move(s.value()); + EXPECT_EQ(controllers.size(), 0); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, MoveAllProcessesFailsIfSourceDoesntExist) { + auto ancestor_cgroup_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(ancestor_cgroup_or_status.ok()) << ancestor_cgroup_or_status.ToString(); + auto ancestor_cgroup = std::move(ancestor_cgroup_or_status.value()); + auto dest_cgroup_or_status = + TempCgroupDirectory::Create(ancestor_cgroup->GetPath(), S_IRWXU); + ASSERT_TRUE(dest_cgroup_or_status.ok()) << dest_cgroup_or_status.ToString(); + auto dest_cgroup = std::move(dest_cgroup_or_status.value()); + // Do not create the source cgroup + std::string non_existent_path = + ancestor_cgroup->GetPath() + std::filesystem::path::preferred_separator + "nope"; + SysFsCgroupDriver driver; + Status s = driver.MoveAllProcesses(non_existent_path, dest_cgroup->GetPath()); + EXPECT_TRUE(s.IsNotFound()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, MoveAllProcessesFailsIfDestDoesntExist) { + auto ancestor_cgroup_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(ancestor_cgroup_or_status.ok()) << ancestor_cgroup_or_status.ToString(); + auto ancestor_cgroup = std::move(ancestor_cgroup_or_status.value()); + auto source_cgroup_or_status = + TempCgroupDirectory::Create(ancestor_cgroup->GetPath(), S_IRWXU); + ASSERT_TRUE(source_cgroup_or_status.ok()) << source_cgroup_or_status.ToString(); + auto source_cgroup = std::move(source_cgroup_or_status.value()); + // Do not create the dest cgroup. + std::string non_existent_path = + ancestor_cgroup->GetPath() + std::filesystem::path::preferred_separator + "nope"; + SysFsCgroupDriver driver; + Status s = driver.MoveAllProcesses(source_cgroup->GetPath(), non_existent_path); + EXPECT_TRUE(s.IsNotFound()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + MoveAllProcessesFailsIfNotReadWriteExecPermissionsForSource) { + auto ancestor_cgroup_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(ancestor_cgroup_or_status.ok()) << ancestor_cgroup_or_status.ToString(); + auto ancestor_cgroup = std::move(ancestor_cgroup_or_status.value()); + auto source_cgroup_or_status = + TempCgroupDirectory::Create(ancestor_cgroup->GetPath(), S_IRUSR | S_IWUSR); + ASSERT_TRUE(source_cgroup_or_status.ok()) << source_cgroup_or_status.ToString(); + auto source_cgroup = std::move(source_cgroup_or_status.value()); + auto dest_cgroup_or_status = + TempCgroupDirectory::Create(ancestor_cgroup->GetPath(), S_IRWXU); + ASSERT_TRUE(dest_cgroup_or_status.ok()) << dest_cgroup_or_status.ToString(); + auto dest_cgroup = std::move(dest_cgroup_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.MoveAllProcesses(source_cgroup->GetPath(), dest_cgroup->GetPath()); + EXPECT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + MoveAllProcessesFailsIfNotReadWriteExecPermissionsForDest) { + auto ancestor_cgroup_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(ancestor_cgroup_or_status.ok()) << ancestor_cgroup_or_status.ToString(); + auto ancestor_cgroup = std::move(ancestor_cgroup_or_status.value()); + auto source_cgroup_or_status = + TempCgroupDirectory::Create(ancestor_cgroup->GetPath(), S_IRWXU); + ASSERT_TRUE(source_cgroup_or_status.ok()) << source_cgroup_or_status.ToString(); + auto source_cgroup = std::move(source_cgroup_or_status.value()); + auto dest_cgroup_or_status = + TempCgroupDirectory::Create(ancestor_cgroup->GetPath(), S_IRUSR | S_IWUSR); + ASSERT_TRUE(dest_cgroup_or_status.ok()) << dest_cgroup_or_status.ToString(); + auto dest_cgroup = std::move(dest_cgroup_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.MoveAllProcesses(source_cgroup->GetPath(), dest_cgroup->GetPath()); + EXPECT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + MoveAllProcessesFailsIfNotReadWriteExecPermissionsForAncestor) { + auto ancestor_cgroup_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(ancestor_cgroup_or_status.ok()) << ancestor_cgroup_or_status.ToString(); + auto ancestor_cgroup = std::move(ancestor_cgroup_or_status.value()); + auto source_cgroup_or_status = + TempCgroupDirectory::Create(ancestor_cgroup->GetPath(), S_IRWXU); + ASSERT_TRUE(source_cgroup_or_status.ok()) << source_cgroup_or_status.ToString(); + auto source_cgroup = std::move(source_cgroup_or_status.value()); + auto dest_cgroup_or_status = + TempCgroupDirectory::Create(ancestor_cgroup->GetPath(), S_IRWXU); + ASSERT_TRUE(dest_cgroup_or_status.ok()) << dest_cgroup_or_status.ToString(); + auto dest_cgroup = std::move(dest_cgroup_or_status.value()); + ASSERT_EQ(chmod(ancestor_cgroup->GetPath().c_str(), S_IRUSR), 0) + << "Failed to chmod cgroup directory " << ancestor_cgroup->GetPath() + << "\n Error: " << strerror(errno); + SysFsCgroupDriver driver; + Status s = driver.MoveAllProcesses(source_cgroup->GetPath(), dest_cgroup->GetPath()); + EXPECT_TRUE(s.IsPermissionDenied()) << s.ToString(); + // Change the permissions back read, write, and execute so cgroup can be deleted. + ASSERT_EQ(chmod(ancestor_cgroup->GetPath().c_str(), S_IRWXU), 0) + << "Failed to chmod cgroup directory " << ancestor_cgroup->GetPath() + << "\n Error: " << strerror(errno); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + MoveAllProcessesSucceedsWithCorrectPermissionsAndValidCgroups) { + auto source_cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(source_cgroup_or_status.ok()) << source_cgroup_or_status.ToString(); + auto source_cgroup = std::move(source_cgroup_or_status.value()); + auto dest_cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(dest_cgroup_or_status.ok()) << dest_cgroup_or_status.ToString(); + auto dest_cgroup = std::move(dest_cgroup_or_status.value()); + StatusOr> child_process_s = + StartChildProcessInCgroup(source_cgroup->GetPath()); + ASSERT_TRUE(child_process_s.ok()) << child_process_s.ToString(); + auto [child_pid, child_pidfd] = child_process_s.value(); + SysFsCgroupDriver driver; + Status s = driver.MoveAllProcesses(source_cgroup->GetPath(), dest_cgroup->GetPath()); + ASSERT_TRUE(s.ok()) << s.ToString(); + // Assert that the child's pid is actually in the new file. + std::string dest_cgroup_procs_file_path = dest_cgroup->GetPath() + + std::filesystem::path::preferred_separator + + "cgroup.procs"; + std::ifstream dest_cgroup_procs_file(dest_cgroup_procs_file_path); + ASSERT_TRUE(dest_cgroup_procs_file.is_open()) + << "Could not open file " << dest_cgroup_procs_file_path << "."; + std::unordered_set dest_cgroup_pids; + int pid = -1; + while (dest_cgroup_procs_file >> pid) { + ASSERT_FALSE(dest_cgroup_procs_file.fail()) + << "Unable to read pid from file " << dest_cgroup_procs_file_path; + dest_cgroup_pids.emplace(pid); + } + EXPECT_EQ(dest_cgroup_pids.size(), 1); + EXPECT_TRUE(dest_cgroup_pids.find(child_pid) != dest_cgroup_pids.end()); + Status terminate_s = + TerminateChildProcessAndWaitForTimeout(child_pid, child_pidfd, 5000); + ASSERT_TRUE(terminate_s.ok()) << terminate_s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + EnableControllerFailsIfReadOnlyPermissionsForCgroup) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.EnableController(cgroup_dir->GetPath(), "memory"); + ASSERT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + EnableControllerFailsIfReadWriteOnlyPermissionsForCgroup) { + auto cgroup_dir_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR | S_IWUSR); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.EnableController(cgroup_dir->GetPath(), "memory"); + ASSERT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, EnableControllerFailsIfCgroupDoesNotExist) { + std::string non_existent_path = + test_cgroup_path_ + std::filesystem::path::preferred_separator + "nope"; + SysFsCgroupDriver driver; + Status s = driver.EnableController(non_existent_path, "memory"); + ASSERT_TRUE(s.IsNotFound()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + EnableControllerFailsIfControllerNotAvailableForCgroup) { + // This will inherit controllers available because testing_cgroup_ has + // CPU and Memory controllers available. + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + auto nested_cgroup_dir_or_status = + TempCgroupDirectory::Create(cgroup_dir->GetPath(), S_IRWXU); + ASSERT_TRUE(nested_cgroup_dir_or_status.ok()) << nested_cgroup_dir_or_status.ToString(); + auto nested_cgroup_dir = std::move(nested_cgroup_dir_or_status.value()); + // Make sure that the cgroup has 0 available controllers. + SysFsCgroupDriver driver; + auto available_controllers_s = + driver.GetAvailableControllers(nested_cgroup_dir->GetPath()); + ASSERT_TRUE(available_controllers_s.ok()) << available_controllers_s.ToString(); + auto available_controllers = std::move(available_controllers_s.value()); + ASSERT_EQ(available_controllers.size(), 0); + Status s = driver.EnableController(nested_cgroup_dir->GetPath(), "memory"); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, DisableControllerFailsIfControllerNotEnabled) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + auto enabled_controllers_s = driver.GetEnabledControllers(cgroup_dir->GetPath()); + ASSERT_TRUE(enabled_controllers_s.ok()) << enabled_controllers_s.ToString(); + auto enabled_controllers = std::move(enabled_controllers_s.value()); + ASSERT_EQ(enabled_controllers.size(), 0); + Status s = driver.DisableController(cgroup_dir->GetPath(), "memory"); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + DisableControllerFailsIfReadOnlyPermissionsForCgroup) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.DisableController(cgroup_dir->GetPath(), "memory"); + ASSERT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + DisableControllerFailsIfReadWriteOnlyPermissionsForCgroup) { + auto cgroup_dir_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR | S_IWUSR); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.DisableController(cgroup_dir->GetPath(), "memory"); + ASSERT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, DisableControllerFailsIfCgroupDoesNotExist) { + std::string non_existent_path = + test_cgroup_path_ + std::filesystem::path::preferred_separator + "nope"; + SysFsCgroupDriver driver; + Status s = driver.DisableController(non_existent_path, "memory"); + ASSERT_TRUE(s.IsNotFound()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + EnableAndDisableControllerSucceedWithCorrectInputAndPermissions) { + auto parent_cgroup_dir_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(parent_cgroup_dir_or_status.ok()) << parent_cgroup_dir_or_status.ToString(); + auto parent_cgroup_dir = std::move(parent_cgroup_dir_or_status.value()); + auto child_cgroup_dir_or_status = + TempCgroupDirectory::Create(parent_cgroup_dir->GetPath(), S_IRWXU); + ASSERT_TRUE(child_cgroup_dir_or_status.ok()) << child_cgroup_dir_or_status.ToString(); + auto child_cgroup_dir = std::move(child_cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + + // There should be no enabled controllers on the parent cgroup so enabling the memory + // controller should fail. + Status invalid_argument_s = driver.EnableController(child_cgroup_dir->GetPath(), "cpu"); + ASSERT_TRUE(invalid_argument_s.IsInvalidArgument()) << invalid_argument_s.ToString(); + + // Enable the controller on the parent cgroup to make it available on the child + Status enable_parent_s = driver.EnableController(parent_cgroup_dir->GetPath(), "cpu"); + ASSERT_TRUE(enable_parent_s.ok()) << enable_parent_s.ToString(); + + // Enable the controller on the child cgroup. + Status enable_child_s = driver.EnableController(child_cgroup_dir->GetPath(), "cpu"); + ASSERT_TRUE(enable_child_s.ok()) << enable_child_s.ToString(); + + // Cannot disable the controller on the parent cgroup while the child cgroup + // still has it enabled. + Status disable_parent_failure_s = + driver.DisableController(parent_cgroup_dir->GetPath(), "cpu"); + ASSERT_FALSE(disable_parent_failure_s.ok()) << enable_parent_s.ToString(); + // Disable the controller on the child cgroup. + Status disable_child_s = driver.DisableController(child_cgroup_dir->GetPath(), "cpu"); + ASSERT_TRUE(disable_child_s.ok()) << disable_child_s.ToString(); + // Can now disable the controller on the parent cgroup. + Status disable_parent_success_s = + driver.DisableController(parent_cgroup_dir->GetPath(), "cpu"); + ASSERT_TRUE(disable_parent_success_s.ok()) << disable_parent_success_s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, AddResourceConstraintFailsIfCgroupDoesntExist) { + std::string non_existent_path = + test_cgroup_path_ + std::filesystem::path::preferred_separator + "nope"; + SysFsCgroupDriver driver; + Status s = driver.AddConstraint(non_existent_path, "memory.min", "1"); + ASSERT_TRUE(s.IsNotFound()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + AddResourceConstraintFailsIfReadOnlyPermissions) { + auto cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR); + ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); + auto cgroup = std::move(cgroup_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.AddConstraint(cgroup->GetPath(), "memory.min", "1"); + ASSERT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + AddResourceConstraintFailsIfReadWriteOnlyPermissions) { + auto cgroup_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR | S_IWUSR); + ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); + auto cgroup = std::move(cgroup_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.AddConstraint(cgroup->GetPath(), "memory.min", "1"); + ASSERT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + AddResourceConstraintFailsIfConstraintNotSupported) { + auto cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); + auto cgroup = std::move(cgroup_or_status.value()); + SysFsCgroupDriver driver; + // "memory.max" is not supported. + Status s = driver.AddConstraint(cgroup->GetPath(), "memory.max", "1"); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); +} +TEST_F(SysFsCgroupDriverIntegrationTest, + AddResourceConstraintFailsIfControllerNotEnabled) { + auto cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); + auto cgroup = std::move(cgroup_or_status.value()); + SysFsCgroupDriver driver; + // Memory controller is not enabled. + Status s = driver.AddConstraint(cgroup->GetPath(), "memory.min", "1"); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); +} +TEST_F(SysFsCgroupDriverIntegrationTest, + AddResourceConstraintFailsIfInvalidConstraintValue) { + auto cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); + auto cgroup = std::move(cgroup_or_status.value()); + SysFsCgroupDriver driver; + // Enable the cpu controller first. + Status enable_controller_s = driver.EnableController(cgroup->GetPath(), "cpu"); + ASSERT_TRUE(enable_controller_s.ok()) << enable_controller_s.ToString(); + // cpu.weight must be between [1,10000] + Status s_too_low = driver.AddConstraint(cgroup->GetPath(), "cpu.weight", "0"); + ASSERT_TRUE(s_too_low.IsInvalidArgument()) << s_too_low.ToString(); + Status s_too_high = driver.AddConstraint(cgroup->GetPath(), "cpu.weight", "10001"); + ASSERT_TRUE(s_too_high.IsInvalidArgument()) << s_too_high.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, AddResourceConstraintSucceeds) { + auto cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); + auto cgroup = std::move(cgroup_or_status.value()); + SysFsCgroupDriver driver; + // Enable the cpu controller first. + Status enable_controller_s = driver.EnableController(cgroup->GetPath(), "cpu"); + ASSERT_TRUE(enable_controller_s.ok()) << enable_controller_s.ToString(); + // cpu.weight must be between [1,10000] + Status s = driver.AddConstraint(cgroup->GetPath(), "cpu.weight", "500"); + ASSERT_TRUE(s.ok()) << s.ToString(); +} +} // namespace ray diff --git a/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test_entrypoint.sh b/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test_entrypoint.sh new file mode 100755 index 000000000000..ee4f8d3fa3de --- /dev/null +++ b/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test_entrypoint.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +set -euo pipefail + +# To run this test locally, you will need to run it as the root user to be able +# to create cgroups, add users etc. It is recommended to first create a cgroup for testing +# so the tests do not interfere with your root cgroup. +# +# 1) Create a cgroup +# sudo mkdir -p /sys/fs/cgroup/testing +# +# 2) Enable rwx permissions for files in the cgroup +# sudo chmod u+rwx /sys/fs/cgroup/testing +# +# 2) Move the current process into the cgroup +# echo $$ | sudo tee /sys/fs/cgroup/testing/cgroup.procs +# +# 3) Execute the tests with sudo passing your ROOT_CGROUP +# NOTE: the "env PATH=${PATH}" is for the root user to find the bazel executable +# since it may not already be in its path. +# sudo env PATH="${PATH}" ./sysfs_cgroup_driver_integration_test_entrypoint.sh /sys/fs/cgroup/testing +# +# If cleanup fails during local testing, you can run to remove all created cgroups. +# sudo find /sys/fs/cgroup/testing -type d -depth 10 -exec rmdir {} + +if [[ "$(uname -s)" != "Linux" ]]; then + echo "ERROR: Cgroup integration tests can only be run on Linux." + echo " The current OS is $(uname)" + exit 0 +fi + +BAZEL=$(which bazel) +# Defaults to /sys/fs/cgroup if not passed in as an argument. +ROOT_CGROUP="${1:-/sys/fs/cgroup}" +CURR_USER=$(whoami) + +echo "Starting Cgroupv2 Integration Tests as user ${CURR_USER}" +echo "ROOT_CGROUP is ${ROOT_CGROUP}." + +if ! grep -qE 'cgroup2\srw' /etc/mtab; then + echo "Failed because cgroupv2 is not mounted on the system in read-write mode." + echo "See the following documentation for how to enable cgroupv2 properly:" + echo "https://kubernetes.io/docs/concepts/architecture/cgroups/#linux-distribution-cgroup-v2-support" + exit 1 +fi +if grep -qE "cgroup\sr" /etc/mtab; then + echo "Failed because cgroupv2 and cgroupv1 is mounted on this system." + echo "See the following documentation for how to enable cgroupv2 in properly in unified mode:" + echo "https://kubernetes.io/docs/concepts/architecture/cgroups/#linux-distribution-cgroup-v2-support" + exit 1 +fi +if [[ ! -w ${ROOT_CGROUP} ]]; then + echo "$(whoami) needs read and write access to ${ROOT_CGROUP} to run integration tests." + echo "Run 'sudo chown -R ${CURR_USER} ${ROOT_CGROUP}' to fix this." + exit 1 +fi +if ! grep -qE '\scpu\s' "${ROOT_CGROUP}"/cgroup.controllers; then + echo "Failed because the cpu controller is not available in the ${ROOT_CGROUP}/cgroup.controllers." + echo "To enable the cpu controller, you need to add it to the parent cgroup of ${ROOT_CGROUP}." + echo "See: https://docs.kernel.org/admin-guide/cgroup-v2.html#enabling-and-disabling." + exit 1 +fi +if ! grep -qE '\smemory\s' "${ROOT_CGROUP}"/cgroup.controllers; then + echo "Failed because the memory controller is not available in the ${ROOT_CGROUP}/cgroup.controllers." + echo "To enable the memory controller, you need to add it to the parent cgroup of ${ROOT_CGROUP}." + echo "See: https://docs.kernel.org/admin-guide/cgroup-v2.html#enabling-and-disabling." + exit 1 +fi + + +TEST_FIXTURE_SCRIPT=src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test_fixture.sh +BASE_CGROUP="$(mktemp -d -p "${ROOT_CGROUP}" testing.XXXXX)" +TEST_CGROUP=${BASE_CGROUP}/test +LEAF_CGROUP=${BASE_CGROUP}/leaf +UNPRIV_USER=cgroup-tester + +trap 'echo "ERROR on line ${LINENO}"; cleanup' ERR INT TERM + +cleanup() { + echo "Running teardown because of an error." + "${TEST_FIXTURE_SCRIPT}" teardown "${ROOT_CGROUP}" "${BASE_CGROUP}" "${UNPRIV_USER}" +} + +# The integration tests assume that the ROOT_CGROUP exists and has read and write access. +# +# This test suite will create the following cgroup hierarchy for the tests +# starting with BASE_CGROUP. +# +# ROOT_CGROUP +# | +# BASE_CGROUP +# / \ +# TEST_CGROUP LEAF_CGROUP +# +# NOTE: The test suite does not assume that ROOT_CGROUP is an actual ROOT_CGROUP. Therefore, +# 1. setup will migrate all processes from the ROOT_CGROUP -> LEAF_CGROUP +# 2. teardown will migrate all processes from the LEAF_CGROUP -> ROOT_CGROUP +# +# NOTE: BASE_CGROUP will have a randomly generated name to isolate tests from each other. +# +# The test suite assumes that +# 1. cpu, memory controllers are available on ROOT_CGROUP i.e. in the ROOT_CGROUP/cgroup.controllers file. +# 2. All processes inside the base_cgroup can be migrated into the leaf_cgroup to avoid not violating +# the no internal processes contstraint. +# +# All C++ tests should only have access to the TEST_CGROUP and nothing outside of it. +# The C++ tests will be executed as a non-root user. Setup/teardown will need root permissions. +echo "ROOT_CGROUP is ${ROOT_CGROUP}." +echo "BASE_CGROUP for the test suite is ${BASE_CGROUP}." +echo "TEST_CGROUP for the test suite is ${TEST_CGROUP}." +echo "LEAF_CGROUP for the test suite is ${LEAF_CGROUP}." + +"${TEST_FIXTURE_SCRIPT}" setup "${ROOT_CGROUP}" "${BASE_CGROUP}" "${UNPRIV_USER}" + +sudo -u "${UNPRIV_USER}" CGROUP_PATH="${TEST_CGROUP}" \ + "${BAZEL}" run //src/ray/common/cgroup2/integration_tests:sysfs_cgroup_driver_integration_test + +"${TEST_FIXTURE_SCRIPT}" teardown "${ROOT_CGROUP}" "${BASE_CGROUP}" "${UNPRIV_USER}" diff --git a/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test_fixture.sh b/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test_fixture.sh new file mode 100755 index 000000000000..b2f65ef36b0a --- /dev/null +++ b/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test_fixture.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "Usage: $0 " + echo " ACTION - One of {setup, teardown}." + echo " ROOT_CGROUP - The root cgroup path. Assumes the cgroup already exists." + echo " BASE_CGROUP - The base cgroup path. Assumes the cgroup already exists." + echo " UNPRIV_USER - The name of the unprivileged user. Will create if doesn't exist." + exit 1 +} + +ACTION=${1:-} +ROOT_CGROUP=${2:-} +BASE_CGROUP=${3:-} +UNPRIV_USER=${4:-} + +validate_args() { + if [[ -z "$ACTION" || -z "$ROOT_CGROUP" || -z "$BASE_CGROUP" || -z "$UNPRIV_USER" ]]; then + echo "ERROR: Missing arguments." + usage + fi +} + +# Helper function to move all processes from the src cgroup +# into the dest cgroup. +move_all_processes() { + # Errexit is disabled because pids can be transient i.e. + # you can fail to move a pid that existed when you read the file + # but exited by the time you tried to move it. + set +e + local src="$1" dst="$2" + local count=0 + while IFS= read -r pid + do + if echo "${pid}" > "${dst}" 2>/dev/null; then + ((count++)) + fi + done < <(grep -v '^ *#' "${src}") + echo "Moved ${count} procs from ${src} to ${dst}." + set -e +} + +update_controllers() { + local CONTROLLER_FILE=$1 + local UPDATE=$2 + if echo "${UPDATE}" > "${CONTROLLER_FILE}"; then + echo "Updated ${UPDATE} controllers for ${CONTROLLER_FILE}" + else + echo "ERROR: Failed to update controllers ${UPDATE} for ${CONTROLLER_FILE}" >&2 + exit 1 + fi + +} + +# Setup involves the following steps: +# +# 1. Create the LEAF_CGROUP and TEST_CGROUP. +# 2. Move all processes from the ROOT_CGROUP into the LEAF_CGROUP. +# 3. Enable cpu, memory controllers on the ROOT, BASE, and TEST cgroups. +# 4. Create the UNPRIV_USER to run the tests as a non-root user. +# 5. Make UNPRIV_USER owner of the cgroup subtree starting at BASE_CGROUP. +# +# NOTE: The tests need to be run as a separate user because access control +# checks will always pass for the root user so they cannot be tested properly +# without creating an unprivileged user. +setup() { + +mkdir -p "${LEAF_CGROUP}" +mkdir -p "${TEST_CGROUP}" + +echo "Created LEAF_CGROUP at ${LEAF_CGROUP}." +echo "Created TEST_CGROUP at ${TEST_CGROUP}." + +move_all_processes "${ROOT_CGROUP_PROCS}" "${LEAF_CGROUP_PROCS}" + +if [[ -s "${ROOT_CGROUP_PROCS}" ]]; then + echo "ERROR: Failed to move all processes out of ${ROOT_CGROUP_PROCS}." + echo " Expected cgroup.procs to be empty, but it's not:" + cat "${ROOT_CGROUP_PROCS}" + exit 1 +fi + +update_controllers "${ROOT_CGROUP}/cgroup.subtree_control" "+cpu +memory" +update_controllers "${BASE_CGROUP}/cgroup.subtree_control" "+cpu +memory" +update_controllers "${TEST_CGROUP}/cgroup.subtree_control" "+cpu +memory" + +if ! id -u "${UNPRIV_USER}" >/dev/null 2>&1; then + sudo useradd -m -s /usr/sbin/nologin "${UNPRIV_USER}" + echo "Created unprivilged user ${UNPRIV_USER}." +fi + +sudo chown -R "${UNPRIV_USER}":"${UNPRIV_USER}" "${BASE_CGROUP}" +sudo chmod -R u+rwx "${BASE_CGROUP}" +echo "${UNPRIV_USER} is the owner the cgroup subtree starting at ${BASE_CGROUP}" + +} + +# Cleanup is the reverse of setup +# 1) Delete the user we created. +# 2) Disable controllers throughout heirarchy. +# 3) Migrate all processes back into the ROOT_CGROUP. +# 4) Recursively delete all created subcgroups. +# +# This is best effort. There can be leaks. The recommended thing +# to do is to run these tests inside a container. +# Setup involves the following steps: +# +# 1. Delete the UNPRIV_USER. +# 2. Disable cpu, memory controllers on the ROOT, BASE, and TEST cgroups. +# 3. Move all processes from the LEAF_CGROUP into the ROOT_CGROUP. +# 4. Delete the TEST, LEAF, and BASE cgroups in that order. +# +# NOTE: This assumes that all C++ tests will clean up their own cgroups. +# If they do not, teardown will fail. +teardown() { + +# Delete the user we created +if id -u "${UNPRIV_USER}" >/dev/null 2>&1; then + pkill -KILL -u "${UNPRIV_USER}" 2>/dev/null || true + deluser -f "${UNPRIV_USER}" --remove-home 2>/dev/null || true + echo "Deleted unprivilged user ${UNPRIV_USER}." +fi + +update_controllers "${TEST_CGROUP}/cgroup.subtree_control" "-cpu -memory" +update_controllers "${BASE_CGROUP}/cgroup.subtree_control" "-cpu -memory" +update_controllers "${ROOT_CGROUP}/cgroup.subtree_control" "-cpu -memory" + +move_all_processes "${LEAF_CGROUP_PROCS}" "${ROOT_CGROUP_PROCS}" + +rmdir "${TEST_CGROUP}" +echo "Deleted ${TEST_CGROUP}" +rmdir "${LEAF_CGROUP}" +echo "Deleted ${LEAF_CGROUP}" +rmdir "${BASE_CGROUP}" +echo "Deleted ${BASE_CGROUP}" + +echo "Teardown successful." + +} + +validate_args + +LEAF_CGROUP="${BASE_CGROUP}/leaf" +TEST_CGROUP="${BASE_CGROUP}/test" +ROOT_CGROUP_PROCS="${ROOT_CGROUP}/cgroup.procs" +LEAF_CGROUP_PROCS="${LEAF_CGROUP}/cgroup.procs" + +echo "Starting integration test fixture with:" +echo " ACTION=${ACTION}" +echo " ROOT_CGROUP=${ROOT_CGROUP}" +echo " BASE_CGROUP=${BASE_CGROUP}" +echo " TEST_CGROUP=${TEST_CGROUP}" +echo " UNPRIV_USER=${UNPRIV_USER}" + +SETUP_ACTION=setup +TEARDOWN_ACTION=teardown + +if [[ "${ACTION}" == "${SETUP_ACTION}" ]]; then + echo "Running ACTION: ${SETUP_ACTION}" + setup +elif [[ "${ACTION}" == "${TEARDOWN_ACTION}" ]]; then + echo "Running ACTION: ${TEARDOWN_ACTION}" + teardown +else + echo "[ERROR]: Unknown action ${ACTION}." + usage +fi diff --git a/src/ray/common/cgroup2/sysfs_cgroup_driver.cc b/src/ray/common/cgroup2/sysfs_cgroup_driver.cc index fc564dfb7fd9..88a41d74af8a 100644 --- a/src/ray/common/cgroup2/sysfs_cgroup_driver.cc +++ b/src/ray/common/cgroup2/sysfs_cgroup_driver.cc @@ -56,8 +56,8 @@ Status SysFsCgroupDriver::CheckCgroupv2Enabled() { struct mntent *mnt; while ((mnt = getmntent(fp)) != nullptr) { - found_cgroupv1 = found_cgroupv1 || strcmp(mnt->mnt_fsname, "cgroup") == 0; - found_cgroupv2 = found_cgroupv2 || strcmp(mnt->mnt_fsname, "cgroup2") == 0; + found_cgroupv1 = found_cgroupv1 || strcmp(mnt->mnt_type, "cgroup") == 0; + found_cgroupv2 = found_cgroupv2 || strcmp(mnt->mnt_type, "cgroup2") == 0; } // After parsing the mount file, the file should be at the EOF position. diff --git a/src/ray/common/cgroup2/tests/BUILD.bazel b/src/ray/common/cgroup2/tests/BUILD.bazel index c31181a7ec2d..e9d165d5d49c 100644 --- a/src/ray/common/cgroup2/tests/BUILD.bazel +++ b/src/ray/common/cgroup2/tests/BUILD.bazel @@ -1,30 +1,19 @@ -load("//bazel:ray.bzl", "ray_cc_library", "ray_cc_test") - -ray_cc_library( - name = "cgroup_test_utils", - srcs = ["cgroup_test_utils.cc"], - hdrs = ["cgroup_test_utils.h"], - tags = [ - "no_windows", - ], - deps = [ - "//src/ray/common:status", - "//src/ray/common:status_or", - "@com_google_absl//absl/strings:str_format", - ], -) +load("//bazel:ray.bzl", "ray_cc_test") ray_cc_test( name = "sysfs_cgroup_driver_test", srcs = ["sysfs_cgroup_driver_test.cc"], tags = [ - "no_windows", + "cgroup", "team:core", ], + target_compatible_with = [ + "@platforms//os:linux", + ], deps = [ - ":cgroup_test_utils", "//src/ray/common:status", "//src/ray/common:status_or", + "//src/ray/common/cgroup2:cgroup_test_utils", "//src/ray/common/cgroup2:sysfs_cgroup_driver", "//src/ray/common/tests:testing", "@com_google_absl//absl/strings:str_format", diff --git a/src/ray/common/cgroup2/tests/cgroup_test_utils.cc b/src/ray/common/cgroup2/tests/cgroup_test_utils.cc deleted file mode 100644 index c31109dddb54..000000000000 --- a/src/ray/common/cgroup2/tests/cgroup_test_utils.cc +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2025 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "ray/common/cgroup2/tests/cgroup_test_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "absl/strings/str_format.h" -#include "ray/common/status.h" -#include "ray/common/status_or.h" -#include "ray/util/logging.h" - -ray::StatusOr> TempDirectory::Create() { - std::string path = "/tmp/XXXXXX"; - char *ret = mkdtemp(path.data()); - if (ret == nullptr) { - return ray::Status::UnknownError( - absl::StrFormat("Failed to create a temp directory. " - "Cgroup tests expect tmpfs to be mounted and only run on Linux.\n" - "Error: %s", - strerror(errno))); - } - std::unique_ptr temp_dir = - std::make_unique(std::move(path)); - return ray::StatusOr>(std::move(temp_dir)); -} - -TempDirectory::~TempDirectory() { std::filesystem::remove_all(path_); } - -TempFile::TempFile(std::string path) { - path_ = path; - fd_ = open(path_.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); // NOLINT - if (fd_ == -1) { - throw std::runtime_error( - absl::StrFormat("Failed to create a temp file. Cgroup tests expect " - "tmpfs to be mounted " - "and only run on Linux. Error: %s", - strerror(errno))); - } - file_output_stream_ = std::ofstream(path_, std::ios::trunc); - if (!file_output_stream_.is_open()) { - throw std::runtime_error("Could not open file on tmpfs."); - } -} - -TempFile::TempFile() { - fd_ = mkstemp(path_.data()); // NOLINT - if (fd_ == -1) { - throw std::runtime_error( - "Failed to create a temp file. Cgroup tests expect tmpfs to be " - "mounted " - "and only run on Linux"); - } - if (unlink(path_.c_str()) == -1) { - close(fd_); - throw std::runtime_error("Failed to unlink temporary file."); - } - file_output_stream_ = std::ofstream(path_, std::ios::trunc); - if (!file_output_stream_.is_open()) { - throw std::runtime_error("Could not open mount file on tmpfs."); - } -} - -TempFile::~TempFile() { - close(fd_); - file_output_stream_.close(); -} - -void TempFile::AppendLine(const std::string &line) { - file_output_stream_ << line; - file_output_stream_.flush(); - if (file_output_stream_.fail()) { - throw std::runtime_error("Could not write to mount file on tmpfs"); - } -} diff --git a/src/ray/common/cgroup2/tests/cgroup_test_utils.h b/src/ray/common/cgroup2/tests/cgroup_test_utils.h deleted file mode 100644 index f1622d413573..000000000000 --- a/src/ray/common/cgroup2/tests/cgroup_test_utils.h +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2025 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#pragma once - -#include - -#include -#include -#include -#include - -#include "ray/common/status.h" -#include "ray/common/status_or.h" - -/** - RAII style class for creating and destroying temporary directory for testing. - TODO(irabbani): add full documentation once complete. - */ -class TempDirectory { - public: - static ray::StatusOr> Create(); - explicit TempDirectory(std::string &&path) : path_(path) {} - - TempDirectory(const TempDirectory &) = delete; - TempDirectory(TempDirectory &&) = delete; - TempDirectory &operator=(const TempDirectory &) = delete; - TempDirectory &operator=(TempDirectory &&) = delete; - - const std::string &GetPath() const { return path_; } - - ~TempDirectory(); - - private: - const std::string path_; -}; - -/** - RAII wrapper that creates a file that can be written to. - TODO(irabbani): Add full documentation once the API is complete. -*/ -class TempFile { - public: - explicit TempFile(std::string path); - TempFile(); - - TempFile(TempFile &other) = delete; - TempFile(TempFile &&other) = delete; - TempFile operator=(TempFile &other) = delete; - TempFile &operator=(TempFile &&other) = delete; - - ~TempFile(); - void AppendLine(const std::string &line); - - const std::string &GetPath() const { return path_; } - - private: - std::string path_ = "/tmp/XXXXXX"; - std::ofstream file_output_stream_; - int fd_; -}; diff --git a/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc b/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc index a349be577616..a381c81f97de 100644 --- a/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc +++ b/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc @@ -20,7 +20,7 @@ #include #include "gtest/gtest.h" -#include "ray/common/cgroup2/tests/cgroup_test_utils.h" +#include "ray/common/cgroup2/cgroup_test_utils.h" #include "ray/common/status.h" #include "ray/common/status_or.h" @@ -64,7 +64,7 @@ TEST(SysFsCgroupDriverTest, TEST(SysFsCgroupDriverTest, CheckCgroupv2EnabledSucceedsIfOnlyCgroupv2Mounted) { TempFile temp_mount_file; - temp_mount_file.AppendLine("cgroup2 /sys/fs/cgroup rw 0 0\n"); + temp_mount_file.AppendLine("cgroup2 /sys/fs/cgroup cgroup2 rw 0 0\n"); SysFsCgroupDriver driver(temp_mount_file.GetPath()); Status s = driver.CheckCgroupv2Enabled(); EXPECT_TRUE(s.ok()) << s.ToString(); From f03855148501fa2308d0de0036941274ddd3f7a9 Mon Sep 17 00:00:00 2001 From: Ping Dai Date: Thu, 4 Sep 2025 15:06:44 +0800 Subject: [PATCH 442/634] [Dashboard] Fix typo in memory_utils and adjust display formatting for clarity (#56217) Signed-off-by: daiping8 --- python/ray/dashboard/memory_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ray/dashboard/memory_utils.py b/python/ray/dashboard/memory_utils.py index ef2d51a1de1c..e81532f5af9e 100644 --- a/python/ray/dashboard/memory_utils.py +++ b/python/ray/dashboard/memory_utils.py @@ -430,7 +430,7 @@ def memory_summary( "Type", "Call Site", "Status", - "Attampt", + "Attempt", "Size", "Reference Type", "Object Ref", @@ -444,7 +444,7 @@ def memory_summary( mem += f"Grouping by {group_by}...\ Sorting by {sort_by}...\ - Display {num_entries if num_entries is not None else 'all'}\ + Display {num_entries if num_entries is not None else 'all'} \ entries per group...\n\n\n" for key, group in memory_table["group"].items(): From 6f3689a909d85b431983ad68e006fb4f59259233 Mon Sep 17 00:00:00 2001 From: Justin Yu Date: Thu, 4 Sep 2025 02:55:06 -0700 Subject: [PATCH 443/634] [data][train] Create a deepcopy of the data context on the split coordinator process (#56211) The main change of this PR is to create a deepcopy of the base dataset's context before setting the process-global context. Otherwise, mutations to the base dataset's context during the planning phase are also propagated to the global context, which can affect future dataset executions launched from the same process. Misc. drive-by changes: * Utility to create a `StorageContext` from the `RunConfig` directly * Pipe the `DatasetShardMetadata` from the outermost level among other changes, for easier patching --------- Signed-off-by: Justin Yu --- .../iterator/stream_split_iterator.py | 6 ++++- python/ray/data/iterator.py | 7 +++++- .../train/v2/_internal/callbacks/datasets.py | 15 +++++-------- .../execution/controller/controller.py | 7 +----- .../v2/_internal/execution/train_fn_utils.py | 21 ++++++------------ .../execution/worker_group/worker_group.py | 7 +----- python/ray/train/v2/api/config.py | 10 +++++++++ .../ray/train/v2/api/data_parallel_trainer.py | 6 ++--- python/ray/train/v2/api/train_fn_utils.py | 5 ++++- .../train/v2/tests/test_data_integration.py | 22 ++++++++++--------- 10 files changed, 53 insertions(+), 53 deletions(-) diff --git a/python/ray/data/_internal/iterator/stream_split_iterator.py b/python/ray/data/_internal/iterator/stream_split_iterator.py index ab804886a49f..93f041994872 100644 --- a/python/ray/data/_internal/iterator/stream_split_iterator.py +++ b/python/ray/data/_internal/iterator/stream_split_iterator.py @@ -139,9 +139,13 @@ def __init__( locality_hints: Optional[List[NodeIdStr]], ): dataset = dataset_wrapper._dataset + # Set current DataContext. - self._data_context = dataset.context + # This needs to be a deep copy so that updates to the base dataset's + # context does not affect this process's global DataContext. + self._data_context = dataset.context.copy() ray.data.DataContext._set_current(self._data_context) + if self._data_context.execution_options.locality_with_output is True: self._data_context.execution_options.locality_with_output = locality_hints logger.info(f"Auto configuring locality_with_output={locality_hints}") diff --git a/python/ray/data/iterator.py b/python/ray/data/iterator.py index 70144586a80b..7ed94eef077e 100644 --- a/python/ray/data/iterator.py +++ b/python/ray/data/iterator.py @@ -158,6 +158,11 @@ def iter_batches( local_shuffle_seed=local_shuffle_seed, ) + def _create_batch_iterator( + self, ref_bundles_iter: Iterator[RefBundle], **kwargs + ) -> BatchIterator: + return BatchIterator(ref_bundles_iter, **kwargs) + def _iter_batches( self, *, @@ -186,7 +191,7 @@ def _create_iterator() -> Iterator[DataBatch]: dataset_tag = self._get_dataset_tag() - batch_iterator = BatchIterator( + batch_iterator = self._create_batch_iterator( ref_bundles_iterator, stats=stats, dataset_tag=dataset_tag, diff --git a/python/ray/train/v2/_internal/callbacks/datasets.py b/python/ray/train/v2/_internal/callbacks/datasets.py index 9a6ce9d76cab..908411cab443 100644 --- a/python/ray/train/v2/_internal/callbacks/datasets.py +++ b/python/ray/train/v2/_internal/callbacks/datasets.py @@ -7,9 +7,9 @@ from ray.train.v2._internal.data_integration.interfaces import ( DatasetShardMetadata, DatasetShardProvider, - GenDataset, ) from ray.train.v2._internal.execution.callback import WorkerGroupCallback +from ray.train.v2._internal.execution.context import TrainRunContext from ray.train.v2._internal.execution.worker_group.worker_group import ( Worker, WorkerGroup, @@ -37,15 +37,10 @@ def get_dataset_shard(self, dataset_info: DatasetShardMetadata) -> DataIterator: class DatasetsSetupCallback(WorkerGroupCallback): """The callback to setup Ray Datasets for the worker group.""" - def __init__( - self, - datasets: Dict[str, GenDataset], - data_config: ray.train.DataConfig, - scaling_config: ray.train.ScalingConfig, - ): - self._datasets = datasets - self._data_config = data_config - self._scaling_config = scaling_config + def __init__(self, train_run_context: TrainRunContext): + self._datasets = train_run_context.datasets + self._data_config = copy.deepcopy(train_run_context.dataset_config) + self._scaling_config = train_run_context.scaling_config # Capture the current DataContext to propagate it to # the Train workers later. diff --git a/python/ray/train/v2/_internal/execution/controller/controller.py b/python/ray/train/v2/_internal/execution/controller/controller.py index 224d8b906362..53074db40858 100644 --- a/python/ray/train/v2/_internal/execution/controller/controller.py +++ b/python/ray/train/v2/_internal/execution/controller/controller.py @@ -50,7 +50,6 @@ ResizeDecision, ScalingPolicy, ) -from ray.train.v2._internal.execution.storage import StorageContext from ray.train.v2._internal.execution.worker_group import ( WorkerGroup, WorkerGroupPollStatus, @@ -126,11 +125,7 @@ def __init__( self._failure_policy = failure_policy self._run_config = self._train_run_context.run_config self._callbacks = callbacks or [] - self._storage_context = StorageContext( - storage_path=self._run_config.storage_path, - experiment_dir_name=self._run_config.name, - storage_filesystem=self._run_config.storage_filesystem, - ) + self._storage_context = self._train_run_context.run_config.storage_context self._checkpoint_manager = CheckpointManager( checkpoint_config=self._run_config.checkpoint_config, diff --git a/python/ray/train/v2/_internal/execution/train_fn_utils.py b/python/ray/train/v2/_internal/execution/train_fn_utils.py index a5caad707c15..b05f7dae0631 100644 --- a/python/ray/train/v2/_internal/execution/train_fn_utils.py +++ b/python/ray/train/v2/_internal/execution/train_fn_utils.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional from ray.data import DataIterator +from ray.train.v2._internal.data_integration.interfaces import DatasetShardMetadata from ray.train.v2._internal.execution import collective_impl from ray.train.v2._internal.execution.context import ( get_train_context as get_internal_train_context, @@ -68,14 +69,11 @@ def get_all_reported_checkpoints(self) -> List["ReportedCheckpoint"]: pass @abstractmethod - def get_dataset_shard(self, dataset_name: str) -> DataIterator: + def get_dataset_shard(self, dataset_info: DatasetShardMetadata) -> DataIterator: """Get the dataset shard for this training process. - This method is used by the public API function :func:`ray.train.get_dataset_shard`. - Users should typically call ``ray.train.get_dataset_shard()`` instead of calling this method directly. - Args: - dataset_name: The name of the dataset to get the shard for. + dataset_info: The metadata of the dataset to get the shard for. Returns: The DataIterator shard for this worker. @@ -131,14 +129,8 @@ def report( def get_checkpoint(self): return get_internal_train_context().get_checkpoint() - def get_dataset_shard(self, dataset_name: str) -> DataIterator: - from ray.train.v2._internal.data_integration.interfaces import ( - DatasetShardMetadata, - ) - - return get_internal_train_context().get_dataset_shard( - DatasetShardMetadata(dataset_name=dataset_name) - ) + def get_dataset_shard(self, dataset_info: DatasetShardMetadata) -> DataIterator: + return get_internal_train_context().get_dataset_shard(dataset_info) def get_context(self) -> DistributedTrainContext: return DistributedTrainContext() @@ -182,7 +174,8 @@ def report( def get_checkpoint(self) -> Optional["Checkpoint"]: return self._last_checkpoint - def get_dataset_shard(self, dataset_name: str) -> DataIterator: + def get_dataset_shard(self, dataset_info: DatasetShardMetadata) -> DataIterator: + dataset_name = dataset_info.dataset_name assert ( self._dataset_shards is not None and dataset_name in self._dataset_shards ), f"Dataset shard {dataset_name} not found." diff --git a/python/ray/train/v2/_internal/execution/worker_group/worker_group.py b/python/ray/train/v2/_internal/execution/worker_group/worker_group.py index d48772c4e30d..b103bceac1b7 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/worker_group.py +++ b/python/ray/train/v2/_internal/execution/worker_group/worker_group.py @@ -37,7 +37,6 @@ from ray.train.v2._internal.execution.checkpoint.sync_actor import SynchronizationActor from ray.train.v2._internal.execution.context import ( DistributedContext, - StorageContext, TrainRunContext, ) from ray.train.v2._internal.execution.worker_group.poll import ( @@ -145,11 +144,7 @@ def __init__( """ self._train_run_context = train_run_context run_config = self._train_run_context.run_config - self._storage_context = StorageContext( - storage_path=run_config.storage_path, - experiment_dir_name=run_config.name, - storage_filesystem=run_config.storage_filesystem, - ) + self._storage_context = run_config.storage_context self._worker_group_context: WorkerGroupContext = worker_group_context diff --git a/python/ray/train/v2/api/config.py b/python/ray/train/v2/api/config.py index 7935c3ba6dcb..a640143d6efe 100644 --- a/python/ray/train/v2/api/config.py +++ b/python/ray/train/v2/api/config.py @@ -1,5 +1,6 @@ import logging from dataclasses import dataclass +from functools import cached_property from pathlib import Path from typing import TYPE_CHECKING, List, Optional, Union @@ -12,6 +13,7 @@ ) from ray.runtime_env import RuntimeEnv from ray.train.v2._internal.constants import _DEPRECATED +from ray.train.v2._internal.execution.storage import StorageContext from ray.train.v2._internal.migration_utils import ( FAIL_FAST_DEPRECATION_MESSAGE, TRAINER_RESOURCES_DEPRECATION_MESSAGE, @@ -261,3 +263,11 @@ def __post_init__(self): "See this issue for more context: " "https://github.com/ray-project/ray/issues/49454" ) + + @cached_property + def storage_context(self) -> StorageContext: + return StorageContext( + storage_path=self.storage_path, + experiment_dir_name=self.name, + storage_filesystem=self.storage_filesystem, + ) diff --git a/python/ray/train/v2/api/data_parallel_trainer.py b/python/ray/train/v2/api/data_parallel_trainer.py index 079e6bfe3038..b9f607f3422b 100644 --- a/python/ray/train/v2/api/data_parallel_trainer.py +++ b/python/ray/train/v2/api/data_parallel_trainer.py @@ -30,7 +30,6 @@ TPUReservationCallback, WorkingDirectorySetupCallback, ) -from ray.train.v2._internal.callbacks.datasets import GenDataset from ray.train.v2._internal.callbacks.env_callback import _initialize_env_callbacks from ray.train.v2._internal.callbacks.metrics import ( ControllerMetricsCallback, @@ -42,6 +41,7 @@ METRICS_ENABLED_ENV_VAR, get_env_vars_to_propagate, ) +from ray.train.v2._internal.data_integration.interfaces import GenDataset from ray.train.v2._internal.execution.callback import RayTrainCallback from ray.train.v2._internal.execution.context import TrainRunContext from ray.train.v2._internal.execution.controller import TrainController @@ -164,9 +164,7 @@ def _create_default_callbacks(self) -> List[RayTrainCallback]: ) backend_setup_callback = BackendSetupCallback(self.backend_config) datasets_setup_callback = DatasetsSetupCallback( - datasets=self.datasets, - data_config=self.data_config, - scaling_config=self.scaling_config, + train_run_context=self.train_run_context ) tpu_reservation_setup_callback = TPUReservationCallback() callbacks.extend( diff --git a/python/ray/train/v2/api/train_fn_utils.py b/python/ray/train/v2/api/train_fn_utils.py index 740206e190e2..639867e29257 100644 --- a/python/ray/train/v2/api/train_fn_utils.py +++ b/python/ray/train/v2/api/train_fn_utils.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional +from ray.train.v2._internal.data_integration.interfaces import DatasetShardMetadata from ray.train.v2._internal.execution.train_fn_utils import get_train_fn_utils from ray.train.v2.api.context import TrainContext from ray.util.annotations import PublicAPI @@ -241,4 +242,6 @@ def train_loop_per_worker(config): The ``DataIterator`` shard to use for this worker. If no dataset is passed into Trainer, then return None. """ - return get_train_fn_utils().get_dataset_shard(dataset_name) + return get_train_fn_utils().get_dataset_shard( + DatasetShardMetadata(dataset_name=dataset_name) + ) diff --git a/python/ray/train/v2/tests/test_data_integration.py b/python/ray/train/v2/tests/test_data_integration.py index bf7b0f4e694b..a542520f0001 100644 --- a/python/ray/train/v2/tests/test_data_integration.py +++ b/python/ray/train/v2/tests/test_data_integration.py @@ -1,5 +1,3 @@ -from unittest.mock import MagicMock - import pytest import ray.data @@ -9,12 +7,15 @@ from ray.data.tests.conftest import restore_data_context # noqa: F401 from ray.train.v2._internal.callbacks.datasets import DatasetsSetupCallback from ray.train.v2._internal.data_integration.interfaces import DatasetShardMetadata -from ray.train.v2._internal.execution.context import TrainRunContext from ray.train.v2._internal.execution.worker_group.worker_group import ( WorkerGroupContext, ) from ray.train.v2.api.data_parallel_trainer import DataParallelTrainer -from ray.train.v2.tests.util import DummyObjectRefWrapper, DummyWorkerGroup +from ray.train.v2.tests.util import ( + DummyObjectRefWrapper, + DummyWorkerGroup, + create_dummy_run_context, +) # TODO(justinvyu): Bring over more tests from ray/air/tests/test_new_dataset_config.py @@ -77,17 +78,18 @@ def test_dataset_setup_callback(ray_start_4_cpus): num_workers=scaling_config.num_workers, resources_per_worker=scaling_config.resources_per_worker, ) + train_run_context = create_dummy_run_context( + datasets={"train": train_ds, "valid": valid_ds}, + dataset_config=data_config, + scaling_config=scaling_config, + ) worker_group = DummyWorkerGroup( - train_run_context=MagicMock(spec=TrainRunContext), + train_run_context=train_run_context, worker_group_context=worker_group_context, ) worker_group._start() - callback = DatasetsSetupCallback( - datasets={"train": train_ds, "valid": valid_ds}, - data_config=data_config, - scaling_config=scaling_config, - ) + callback = DatasetsSetupCallback(train_run_context) dataset_manager_for_each_worker = callback.before_init_train_context( worker_group.get_workers() )["dataset_shard_provider"] From 339848e6d3afa51619838d9c76fae5e940fd8f01 Mon Sep 17 00:00:00 2001 From: Daniel Sperber Date: Thu, 4 Sep 2025 19:18:04 +0200 Subject: [PATCH 444/634] [doc] fix: Typo and missing import in doc (#48311) ## Why are these changes needed? I realized that the code samples of [`RLModule`](https://docs.ray.io/en/latest/rllib/package_ref/doc/ray.rllib.core.rl_module.rl_module.RLModule.html#ray.rllib.core.rl_module.rl_module.RLModule) were incomplete and missed some imports. Another code sample lacked a comma for correct syntax. ## Related issue number NA/ ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [x] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Daraan Signed-off-by: Daniel --- doc/source/rllib/rl-modules.rst | 2 +- rllib/core/rl_module/rl_module.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/doc/source/rllib/rl-modules.rst b/doc/source/rllib/rl-modules.rst index c8e27ee7c21f..32efc840f599 100644 --- a/doc/source/rllib/rl-modules.rst +++ b/doc/source/rllib/rl-modules.rst @@ -569,7 +569,7 @@ If you don't return the ``actions`` key from your forward method: def _forward_exploration(self, batch): ... return { - Columns.ACTIONS: ... # RLlib uses these actions as-is (no sampling step!) + Columns.ACTIONS: ..., # RLlib uses these actions as-is (no sampling step!) Columns.ACTION_DIST_INPUTS: ... # If provided, RLlib uses these dist inputs to compute probs and logp. } diff --git a/rllib/core/rl_module/rl_module.py b/rllib/core/rl_module/rl_module.py index 644ecae02a46..247fa0b79160 100644 --- a/rllib/core/rl_module/rl_module.py +++ b/rllib/core/rl_module/rl_module.py @@ -269,6 +269,8 @@ class RLModule(Checkpointable, abc.ABC): DefaultPPOTorchRLModule ) from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog + from ray.rllib.core.rl_module.rl_module import RLModuleSpec + from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig import gymnasium as gym import torch @@ -300,6 +302,12 @@ class RLModule(Checkpointable, abc.ABC): .. testcode:: + from ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module import ( + PPOTorchRLModule + ) + from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog + from ray.rllib.core.rl_module.rl_module import RLModuleSpec + from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig import gymnasium as gym import torch @@ -327,6 +335,12 @@ class RLModule(Checkpointable, abc.ABC): .. testcode:: + from ray.rllib.algorithms.ppo.torch.ppo_torch_rl_module import ( + PPOTorchRLModule + ) + from ray.rllib.algorithms.ppo.ppo_catalog import PPOCatalog + from ray.rllib.core.rl_module.rl_module import RLModuleSpec + from ray.rllib.core.rl_module.default_model_config import DefaultModelConfig import gymnasium as gym import torch From 948b0af30f0dd4f664c3748b0df84a0ecf72ade4 Mon Sep 17 00:00:00 2001 From: Daniel Sperber Date: Thu, 4 Sep 2025 19:22:09 +0200 Subject: [PATCH 445/634] [test] fix test not ending cluster; spelling mistake: tearDow -> tearDown (#54171) ## Why are these changes needed? One of the tests had a spelling mistake `tearDowClass` instead of `tearDownClass` not calling `ray.shudown()` ## Related issue number NA ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests Signed-off-by: Daraan --- .../tests/test_algorithm_save_load_checkpoint_learner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rllib/algorithms/tests/test_algorithm_save_load_checkpoint_learner.py b/rllib/algorithms/tests/test_algorithm_save_load_checkpoint_learner.py index 5cb37f805e35..78840a3fe4be 100644 --- a/rllib/algorithms/tests/test_algorithm_save_load_checkpoint_learner.py +++ b/rllib/algorithms/tests/test_algorithm_save_load_checkpoint_learner.py @@ -95,7 +95,7 @@ def setUpClass(cls) -> None: ray.init() @classmethod - def tearDowClass(cls) -> None: + def tearDownClass(cls) -> None: ray.shutdown() def test_save_and_restore(self): From 83fb5621b26939db280dcc9202e795c2cab50129 Mon Sep 17 00:00:00 2001 From: Jack Gammack <49536617+JackGammack@users.noreply.github.com> Date: Thu, 4 Sep 2025 13:24:53 -0400 Subject: [PATCH 446/634] [data] Fix high memory usage with FileBasedDatasource & ParquetDatasource when using a large number of files (#55978) ## Why are these changes needed? Using `FileBasedDatasource` or `ParquetDatasource` with a very large number of files causes OOM when creating read tasks. The full list of file paths is stored in `self`, causing it to persist to every read task, leading to this warning: ``` The serialized size of your read function named 'read_task_fn' is 49.8MB. This size relatively large. As a result, Ray might excessively spill objects during execution. To fix this issue, avoid accessing `self` or other large objects in 'read_task_fn'. ``` When using a small number of blocks, OOM does not occur because the large file list is not repeated so many times. But when setting high parallelism with `override_num_blocks`, OOM occurs. This is because the full list of paths is added to `self._unresolved_paths`. This attribute isn't currently used anywhere in ray. This PR removes `self._unresolved_paths` to alleviate unexpected high memory usage with very large numbers of files. ## Related issue number Similar to this issue with Iceberg: #49054 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [x] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Jack Gammack --- python/ray/data/_internal/datasource/parquet_datasource.py | 1 - python/ray/data/_internal/planner/plan_read_op.py | 2 +- python/ray/data/datasource/file_based_datasource.py | 1 - python/ray/data/tests/test_streaming_executor.py | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/python/ray/data/_internal/datasource/parquet_datasource.py b/python/ray/data/_internal/datasource/parquet_datasource.py index 9589036aefd4..e893365a7e71 100644 --- a/python/ray/data/_internal/datasource/parquet_datasource.py +++ b/python/ray/data/_internal/datasource/parquet_datasource.py @@ -206,7 +206,6 @@ def __init__( ray.get_runtime_context().get_node_id(), soft=False ) - self._unresolved_paths = paths paths, self._filesystem = _resolve_paths_and_filesystem(paths, filesystem) filesystem = RetryingPyFileSystem.wrap( self._filesystem, diff --git a/python/ray/data/_internal/planner/plan_read_op.py b/python/ray/data/_internal/planner/plan_read_op.py index 360dce30cea8..34bb16e49f67 100644 --- a/python/ray/data/_internal/planner/plan_read_op.py +++ b/python/ray/data/_internal/planner/plan_read_op.py @@ -42,7 +42,7 @@ def _derive_metadata(read_task: ReadTask, read_task_ref: ObjectRef) -> BlockMeta warnings.warn( "The serialized size of your read function named " f"'{read_task.read_fn.__name__}' is {memory_string(task_size)}. This size " - "relatively large. As a result, Ray might excessively " + "is relatively large. As a result, Ray might excessively " "spill objects during execution. To fix this issue, avoid accessing " f"`self` or other large objects in '{read_task.read_fn.__name__}'." ) diff --git a/python/ray/data/datasource/file_based_datasource.py b/python/ray/data/datasource/file_based_datasource.py index 8e66d5f185ca..d65fe8d98293 100644 --- a/python/ray/data/datasource/file_based_datasource.py +++ b/python/ray/data/datasource/file_based_datasource.py @@ -140,7 +140,6 @@ def __init__( self._partitioning = partitioning self._ignore_missing_paths = ignore_missing_paths self._include_paths = include_paths - self._unresolved_paths = paths paths, self._filesystem = _resolve_paths_and_filesystem(paths, filesystem) self._filesystem = RetryingPyFileSystem.wrap( self._filesystem, retryable_errors=self._data_context.retried_io_errors diff --git a/python/ray/data/tests/test_streaming_executor.py b/python/ray/data/tests/test_streaming_executor.py index a8989c2bd409..4bb996a280cf 100644 --- a/python/ray/data/tests/test_streaming_executor.py +++ b/python/ray/data/tests/test_streaming_executor.py @@ -798,7 +798,6 @@ def udf(row): assert isinstance(logical_ops[0], Read) datasource = logical_ops[0]._datasource assert isinstance(datasource, ParquetDatasource) - assert datasource._unresolved_paths == input_path assert isinstance(logical_ops[1], MapRows) assert logical_ops[1]._fn == udf From 7e59dd8f108958f1ff24c6c47cd3950a708c47ac Mon Sep 17 00:00:00 2001 From: Aydin Abiar <62435714+Aydin-ab@users.noreply.github.com> Date: Thu, 4 Sep 2025 10:35:15 -0700 Subject: [PATCH 447/634] [docs] Example serve llm deployment (#55819) --- .../config/vocabularies/General/accept.txt | 10 +- doc/BUILD.bazel | 6 + doc/source/conf.py | 2 + doc/source/serve/examples.yml | 48 ++ doc/source/serve/tutorials/BUILD.bazel | 5 - .../deployment-serve-llm/README.ipynb | 58 +++ .../tutorials/deployment-serve-llm/README.md | 41 ++ .../deployment-serve-llm/ci/aws.yaml | 14 + .../deployment-serve-llm/ci/build.sh | 3 + .../deployment-serve-llm/ci/gce.yaml | 14 + .../deployment-serve-llm/ci/nb2py.py | 85 ++++ .../deployment-serve-llm/ci/tests.sh | 20 + .../deployment-serve-llm/configs/aws.yaml | 7 + .../deployment-serve-llm/configs/gce.yaml | 7 + .../hybrid-reasoning-llm/README.md | 369 ++++++++++++++ .../hybrid-reasoning-llm/client_streaming.py | 34 ++ .../client_thinking_disabled.py | 18 + .../client_thinking_enabled.py | 18 + .../hybrid-reasoning-llm/notebook.ipynb | 470 ++++++++++++++++++ .../hybrid-reasoning-llm/serve_qwen_3_32b.py | 23 + .../large-size-llm/Dockerfile | 8 + .../large-size-llm/README.md | 358 +++++++++++++ .../large-size-llm/client.py | 27 + .../large-size-llm/notebook.ipynb | 450 +++++++++++++++++ .../large-size-llm/serve_deepseek_r1.py | 27 + .../large-size-llm/service.yaml | 29 ++ .../medium-size-llm/Dockerfile | 8 + .../medium-size-llm/README.md | 315 ++++++++++++ .../medium-size-llm/client.py | 19 + .../medium-size-llm/notebook.ipynb | 407 +++++++++++++++ .../medium-size-llm/serve_llama_3_1_70b.py | 29 ++ .../medium-size-llm/service.yaml | 10 + .../reasoning-llm/README.md | 269 ++++++++++ .../reasoning-llm/client.py | 21 + .../reasoning-llm/client_streaming.py | 33 ++ .../reasoning-llm/notebook.ipynb | 348 +++++++++++++ .../reasoning-llm/serve_qwq_32b.py | 24 + .../small-size-llm/Dockerfile | 8 + .../small-size-llm/README.md | 315 ++++++++++++ .../small-size-llm/client.py | 18 + .../small-size-llm/notebook.ipynb | 407 +++++++++++++++ .../small-size-llm/serve_llama_3_1_8b.py | 24 + .../small-size-llm/service.yaml | 10 + .../deployment-serve-llm/vision-llm/README.md | 232 +++++++++ .../vision-llm/client_local_image.py | 37 ++ .../vision-llm/client_url_image.py | 33 ++ .../vision-llm/notebook.ipynb | 311 ++++++++++++ .../vision-llm/serve_qwen_VL.py | 22 + release/BUILD.bazel | 1 + .../byod/byod_deployment_serve_llm.sh | 3 + release/release_tests.yaml | 27 + 51 files changed, 5076 insertions(+), 6 deletions(-) delete mode 100644 doc/source/serve/tutorials/BUILD.bazel create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/README.ipynb create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/README.md create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/ci/aws.yaml create mode 100755 doc/source/serve/tutorials/deployment-serve-llm/ci/build.sh create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/ci/gce.yaml create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py create mode 100755 doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/configs/aws.yaml create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/configs/gce.yaml create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_streaming.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_disabled.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_enabled.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/serve_qwen_3_32b.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/Dockerfile create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/client.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/serve_deepseek_r1.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/service.yaml create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/Dockerfile create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/client.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/serve_llama_3_1_70b.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/service.yaml create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client_streaming.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/serve_qwq_32b.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/Dockerfile create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/client.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/serve_llama_3_1_8b.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/service.yaml create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_url_image.py create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/vision-llm/serve_qwen_VL.py create mode 100755 release/ray_release/byod/byod_deployment_serve_llm.sh diff --git a/.vale/styles/config/vocabularies/General/accept.txt b/.vale/styles/config/vocabularies/General/accept.txt index 886a23d9b920..dfd100659547 100644 --- a/.vale/styles/config/vocabularies/General/accept.txt +++ b/.vale/styles/config/vocabularies/General/accept.txt @@ -33,6 +33,7 @@ autoscales bool breakpoint BTS +bursty chatbot CLI configs @@ -45,8 +46,10 @@ deserialize deserializes dev dev to prod -disable +[d|D]isable[d] +[d|D]isable DLinear +Dockerfile DPO EKS ETDataset @@ -69,6 +72,7 @@ LMs LSH MCP Megatron +Mixtral MLflow MLOps namespace @@ -76,6 +80,7 @@ NER Nsight NumPy NVIDIA +NVLink OOM open-source PACK @@ -86,6 +91,8 @@ pretraining productionize Pythonic QPS +Qwen +Quantizing retrigger RISECamp RLHF @@ -104,6 +111,7 @@ teardown uncaptured URI(s)? UUID +USD uv verl VM(s)? diff --git a/doc/BUILD.bazel b/doc/BUILD.bazel index 617dd3bac2a3..018aa5185495 100644 --- a/doc/BUILD.bazel +++ b/doc/BUILD.bazel @@ -599,3 +599,9 @@ filegroup( srcs = glob(["source/ray-overview/examples/**/*.yaml"]), visibility = ["//release:__pkg__"], ) + +filegroup( + name = "deployment_serve_llm_example_configs", + srcs = glob(["source/serve/tutorials/deployment-serve-llm/**/*.yaml"]), + visibility = ["//release:__pkg__"], +) diff --git a/doc/source/conf.py b/doc/source/conf.py index d72a24fb14eb..02db9e3b801a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -228,6 +228,8 @@ def __init__(self, version: str): "data/api/ray.data.*.rst", "ray-overview/examples/**/README.md", # Exclude .md files in examples subfolders "train/examples/**/README.md", + "serve/tutorials/deployment-serve-llm/README.*", + "serve/tutorials/deployment-serve-llm/*/notebook.ipynb", ] + autogen_files # If "DOC_LIB" is found, only build that top-level navigation item. diff --git a/doc/source/serve/examples.yml b/doc/source/serve/examples.yml index bd830b01e8d8..e1077becdf6d 100644 --- a/doc/source/serve/examples.yml +++ b/doc/source/serve/examples.yml @@ -74,6 +74,54 @@ examples: - natural language processing link: tutorials/serve-deepseek related_technology: llm applications + - title: Deploying a small-sized LLM + skill_level: beginner + use_cases: + - generative ai + - large language models + - natural language processing + link: tutorials/deployment-serve-llm/small-size-llm/README + related_technology: llm applications + - title: Deploying a medium-sized LLM + skill_level: beginner + use_cases: + - generative ai + - large language models + - natural language processing + link: tutorials/deployment-serve-llm/medium-size-llm/README + related_technology: llm applications + - title: Deploying a large-sized LLM + skill_level: beginner + use_cases: + - generative ai + - large language models + - natural language processing + link: tutorials/deployment-serve-llm/large-size-llm/README + related_technology: llm applications + - title: Deploying a vision LLM + skill_level: beginner + use_cases: + - generative ai + - large language models + - natural language processing + link: tutorials/deployment-serve-llm/vision-llm/README + related_technology: llm applications + - title: Deploying a reasoning LLM + skill_level: beginner + use_cases: + - generative ai + - large language models + - natural language processing + link: tutorials/deployment-serve-llm/reasoning-llm/README + related_technology: llm applications + - title: Deploying a hybrid reasoning LLM + skill_level: beginner + use_cases: + - generative ai + - large language models + - natural language processing + link: tutorials/deployment-serve-llm/hybrid-reasoning-llm/README + related_technology: llm applications - title: Serve a Chatbot with Request and Response Streaming skill_level: intermediate use_cases: diff --git a/doc/source/serve/tutorials/BUILD.bazel b/doc/source/serve/tutorials/BUILD.bazel deleted file mode 100644 index beb03dfbbaa8..000000000000 --- a/doc/source/serve/tutorials/BUILD.bazel +++ /dev/null @@ -1,5 +0,0 @@ -filegroup( - name = "markdowns", - srcs = glob(["*.md"]), - visibility = ["//python/ray/serve:__subpackages__"], -) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb new file mode 100644 index 000000000000..d7f694270177 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb @@ -0,0 +1,58 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "bc12c0d2", + "metadata": {}, + "source": [ + "# Quickstarts for LLM serving\n", + "\n", + "These guides provide a fast path to serving LLMs using Ray Serve on Anyscale, with focused tutorials for different deployment scales, from single-GPU setups to multi-node clusters.\n", + "\n", + "Each tutorial includes development and production setups, tips for configuring your cluster, and guidance on monitoring and scaling with Ray Serve.\n", + "\n", + "## Tutorial categories\n", + "\n", + "**[Small-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/small-size-llm/README.html)** \n", + "Deploy small-sized models on a single GPU, such as Llama 3 8 B, Mistral 7 B, or Phi-2. \n", + "\n", + "---\n", + "\n", + "**[Medium-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/medium-size-llm/README.html)** \n", + "Deploy medium-sized models using tensor parallelism across 4—8 GPUs on a single node, such as Llama 3 70 B, Qwen 14 B, Mixtral 8x7 B. \n", + "\n", + "---\n", + "\n", + "**[Large-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/large-size-llm/README.html)** \n", + "Deploy massive models using pipeline parallelism across a multi-node cluster, such as Deepseek-R1 or Llama-Nemotron-253 B. \n", + "\n", + "---\n", + "\n", + "**[Vision LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/vision-llm/README.html)** \n", + "Deploy models with image and text input such as Qwen 2.5-VL-7 B-Instruct, MiniGPT-4, or Pixtral-12 B. \n", + "\n", + "---\n", + "\n", + "**[Reasoning LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/reasoning-llm/README.html)** \n", + "Deploy models with reasoning capabilities designed for long-context tasks, coding, or tool use, such as QwQ-32 B. \n", + "\n", + "---\n", + "\n", + "**[Hybrid thinking LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/hybrid-reasoning-llm/README.html)** \n", + "Deploy models that can switch between reasoning and non-reasoning modes for flexible usage, such as Qwen-3." + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + }, + "myst": { + "front_matter": { + "orphan": true + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/source/serve/tutorials/deployment-serve-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/README.md new file mode 100644 index 000000000000..e4372a2a92b7 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/README.md @@ -0,0 +1,41 @@ + + +# Quickstarts for LLM serving + +These guides provide a fast path to serving LLMs using Ray Serve on Anyscale, with focused tutorials for different deployment scales, from single-GPU setups to multi-node clusters. + +Each tutorial includes development and production setups, tips for configuring your cluster, and guidance on monitoring and scaling with Ray Serve. + +## Tutorial categories + +**[Small-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/small-size-llm/README.html)** +Deploy small-sized models on a single GPU, such as Llama 3 8 B, Mistral 7 B, or Phi-2. + +--- + +**[Medium-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/medium-size-llm/README.html)** +Deploy medium-sized models using tensor parallelism across 4—8 GPUs on a single node, such as Llama 3 70 B, Qwen 14 B, Mixtral 8x7 B. + +--- + +**[Large-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/large-size-llm/README.html)** +Deploy massive models using pipeline parallelism across a multi-node cluster, such as Deepseek-R1 or Llama-Nemotron-253 B. + +--- + +**[Vision LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/vision-llm/README.html)** +Deploy models with image and text input such as Qwen 2.5-VL-7 B-Instruct, MiniGPT-4, or Pixtral-12 B. + +--- + +**[Reasoning LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/reasoning-llm/README.html)** +Deploy models with reasoning capabilities designed for long-context tasks, coding, or tool use, such as QwQ-32 B. + +--- + +**[Hybrid thinking LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/hybrid-reasoning-llm/README.html)** +Deploy models that can switch between reasoning and non-reasoning modes for flexible usage, such as Qwen-3. diff --git a/doc/source/serve/tutorials/deployment-serve-llm/ci/aws.yaml b/doc/source/serve/tutorials/deployment-serve-llm/ci/aws.yaml new file mode 100644 index 000000000000..beb4314156b7 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/ci/aws.yaml @@ -0,0 +1,14 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +# Head node +head_node_type: + name: head + instance_type: m5.2xlarge + resources: + cpu: 8 + +# Worker nodes +auto_select_worker_config: true +flags: + allow-cross-zone-autoscaling: true diff --git a/doc/source/serve/tutorials/deployment-serve-llm/ci/build.sh b/doc/source/serve/tutorials/deployment-serve-llm/ci/build.sh new file mode 100755 index 000000000000..ef7e19de90b6 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/ci/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +set -exo pipefail diff --git a/doc/source/serve/tutorials/deployment-serve-llm/ci/gce.yaml b/doc/source/serve/tutorials/deployment-serve-llm/ci/gce.yaml new file mode 100644 index 000000000000..9c3790622d03 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/ci/gce.yaml @@ -0,0 +1,14 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-central1 + +# Head node +head_node_type: + name: head + instance_type: n2-standard-8 + resources: + cpu: 8 + +# Worker nodes +auto_select_worker_config: true +flags: + allow-cross-zone-autoscaling: true diff --git a/doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py b/doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py new file mode 100644 index 000000000000..2c94f8270b9e --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/ci/nb2py.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +import argparse +import nbformat + + +def convert_notebook( + input_path: str, output_path: str, ignore_cmds: bool = False +) -> None: + """ + Read a Jupyter notebook and write a Python script, converting all %%bash + cells and IPython "!" commands into subprocess.run calls that raise on error. + Cells that load or autoreload extensions are ignored. + """ + nb = nbformat.read(input_path, as_version=4) + with open(output_path, "w") as out: + for cell in nb.cells: + # Only process code cells + if cell.cell_type != "code": + continue + + lines = cell.source.splitlines() + # Skip cells that load or autoreload extensions + if any( + l.strip().startswith("%load_ext autoreload") + or l.strip().startswith("%autoreload all") + for l in lines + ): + continue + + # Detect a %%bash cell + if lines and lines[0].strip().startswith("%%bash"): + if ignore_cmds: + continue + bash_script = "\n".join(lines[1:]).rstrip() + out.write("import subprocess\n") + out.write( + f"subprocess.run(r'''{bash_script}''',\n" + " shell=True,\n" + " check=True,\n" + " executable='/bin/bash')\n\n" + ) + else: + # Detect any IPython '!' shell commands in code lines + has_bang = any(line.lstrip().startswith("!") for line in lines) + if has_bang: + if ignore_cmds: + continue + out.write("import subprocess\n") + for line in lines: + stripped = line.lstrip() + if stripped.startswith("!"): + cmd = stripped[1:].lstrip() + out.write( + f"subprocess.run(r'''{cmd}''',\n" + " shell=True,\n" + " check=True,\n" + " executable='/bin/bash')\n" + ) + else: + out.write(line.rstrip() + "\n") + out.write("\n") + else: + # Regular Python cell: + code = cell.source.rstrip() + if "client.chat.completions.create" in code: + continue # Model isn't deployed in CI so skip cells calling the service + # else, dump as-is + out.write(cell.source.rstrip() + "\n\n") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Convert a Jupyter notebook to a Python script, preserving bash cells and '!' commands as subprocess calls unless ignored with --ignore-cmds." + ) + parser.add_argument("input_nb", help="Path to the input .ipynb file") + parser.add_argument("output_py", help="Path for the output .py script") + parser.add_argument( + "--ignore-cmds", action="store_true", help="Ignore bash cells and '!' commands" + ) + args = parser.parse_args() + convert_notebook(args.input_nb, args.output_py, ignore_cmds=args.ignore_cmds) + + +if __name__ == "__main__": + main() diff --git a/doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh b/doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh new file mode 100755 index 000000000000..79e90ee0905e --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/ci/tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Don't use nbconvert or jupytext unless you're willing +# to check each subprocess unit and validate that errors +# aren't being consumed/hidden + +set -exo pipefail + +for nb in \ + "small-size-llm/notebook" \ + "medium-size-llm/notebook" \ + "large-size-llm/notebook" \ + "vision-llm/notebook" \ + "reasoning-llm/notebook" \ + "hybrid-reasoning-llm/notebook" +do + python ci/nb2py.py "${nb}.ipynb" "${nb}.py" --ignore-cmds + python "${nb}.py" + rm "${nb}.py" +done diff --git a/doc/source/serve/tutorials/deployment-serve-llm/configs/aws.yaml b/doc/source/serve/tutorials/deployment-serve-llm/configs/aws.yaml new file mode 100644 index 000000000000..823b7cf2d786 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/configs/aws.yaml @@ -0,0 +1,7 @@ +head_node_type: + name: head + instance_type: m5.2xlarge +worker_node_types: [] +auto_select_worker_config: true +flags: + allow-cross-zone-autoscaling: true diff --git a/doc/source/serve/tutorials/deployment-serve-llm/configs/gce.yaml b/doc/source/serve/tutorials/deployment-serve-llm/configs/gce.yaml new file mode 100644 index 000000000000..455977d495e0 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/configs/gce.yaml @@ -0,0 +1,7 @@ +head_node_type: + name: head + instance_type: n1-standard-8 +worker_node_types: [] +auto_select_worker_config: true +flags: + allow-cross-zone-autoscaling: true diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md new file mode 100644 index 000000000000..6fdd089399d8 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md @@ -0,0 +1,369 @@ +--- +orphan: true +--- + + + +# Deploy a hybrid reasoning LLM + +A hybrid reasoning model provides flexibility by allowing you to enable or disable reasoning as needed. You can use structured, step-by-step thinking for complex queries while skipping it for simpler ones, balancing accuracy with efficiency depending on the task. + +This tutorial deploys a hybrid reasoning LLM using Ray Serve LLM. + +--- + +## Distinction with purely reasoning models + +*Hybrid reasoning models* are reasoning-capable models that allow you to toggle the thinking process on and off. You can enable structured, step-by-step reasoning when needed but skip it for simpler queries to reduce latency. Purely reasoning models always apply their reasoning behavior, while hybrid models give you fine-grained control over when to use reasoning. + +| **Mode** | **Core behavior** | **Use case examples** | **Limitation** | +| ---------------- | -------------------------------------------- | ------------------------------------------------------------------- | ------------------------------------------------- | +| **Thinking ON** | Explicit multi-step thinking process | Math, coding, logic puzzles, multi-hop QA, CoT prompting | Slower response time, more tokens used. | +| **Thinking OFF** | Direct answer generation | Casual queries, short instructions, single-step answers | May struggle with complex reasoning or interpretability. | + +**Note:** Reasoning often benefits from long context windows (32K up to +1M tokens), high token throughput, low-temperature decoding (greedy sampling), and strong instruction tuning or scratchpad-style reasoning. + +To see an example of deploying a purely reasoning model like *QwQ-32 B*, see [Deploying a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.html). + +--- + +## Enable or disable thinking + +Some hybrid reasoning models let you toggle their "thinking" mode on or off. This section explains when to use thinking mode versus skipping it, and shows how to control the setting in practice. + +--- + +### When to enable or disable thinking mode + +**Enable thinking mode for:** +- Complex, multi-step tasks that require reasoning, such as math, physics, or logic problems. +- Ambiguous queries or situations with incomplete information. +- Planning, workflow orchestration, or when the model needs to act as an "agent" coordinating other tools or models. +- Analyzing intricate data, images, or charts. +- In-depth code reviews or evaluating outputs from other AI systems (LLM as Judge approach). + +**Disable thinking mode for:** +- Simple, well-defined, or routine tasks. +- Low latency and fast responses as the priority. +- Repetitive, straightforward steps within a larger automated workflow. + +--- + +### How to enable or disable thinking mode + +Toggle thinking mode varies by model and framework. Consult the documentation for the model to see how it structures and controls thinking. + +For example, to [control reasoning in Qwen-3](https://huggingface.co/Qwen/Qwen3-32B#switching-between-thinking-and-non-thinking-mode), you can: +* Add `"/think"` or `"/no_think"` in the prompt. +* Set `enable_thinking` in the request: + `extra_body={"chat_template_kwargs": {"enable_thinking": ...}}`. + +See [Send request with thinking enabled](#send-request-with-thinking-enabled) or [Send request with thinking disabled](#send-request-with-thinking-disabled) for practical examples. + +--- + +## Parse reasoning outputs + +In thinking mode, hybrid models often separate _reasoning_ from the _final answer_ using tags like `...`. Without a proper parser, this reasoning may end up in the `content` field instead of the dedicated `reasoning_content` field. + +To ensure that Ray Serve LLM correctly parses the reasoning output, configure a `reasoning_parser` in your Ray Serve LLM deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output. +**Note:** For example, *Qwen-3* uses the `qwen3` parser. See the [vLLM docs](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#supported-models) or your model's documentation to find a supported parser, or [build your own](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#how-to-support-a-new-reasoning-model) if needed. + +```yaml +applications: +- ... + args: + llm_configs: + - model_loading_config: + model_id: my-qwen-3-32b + model_source: Qwen/Qwen3-32B + ... + engine_kwargs: + ... + reasoning_parser: qwen3 # <-- for Qwen-3 models +``` + +See [Configure Ray Serve LLM](#configure-ray-serve-llm) for a complete example. + +**Example response** +When using a reasoning parser, the response is typically structured like this: + +```python +ChatCompletionMessage( + content="The temperature is...", + ..., + reasoning_content="Okay, the user is asking for the temperature today and tomorrow..." +) +``` +And you can extract the content and reasoning like this +```python +response = client.chat.completions.create( + ... +) + +print(f"Content: {response.choices[0].message.content}") +print(f"Reasoning: {response.choices[0].message.reasoning_content}") +``` + +--- + +## Configure Ray Serve LLM + +Set your Hugging Face token in the config file to access gated models. + +Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object. + +Set `tensor_parallel_size` to distribute the model's weights among 8 GPUs in the node. + + +```python +# serve_qwen_3_32b.py +from ray.serve.llm import LLMConfig, build_openai_app +import os + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-qwen-3-32b", + model_source="Qwen/Qwen3-32B", + ), + accelerator_type="A100-40G", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=2, + ) + ), + ### Uncomment if your model is gated and needs your Hugging Face token to access it. + # runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict( + tensor_parallel_size=8, max_model_len=32768, reasoning_parser="qwen3" + ), +) +app = build_openai_app({"llm_configs": [llm_config]}) + +``` + +**Note:** Before moving to a production setup, migrate your settings to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. + +--- + +## Deploy locally + +**Prerequisites** + +* Access to GPU compute. +* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`. + +**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks. + +**Dependencies:** +```bash +pip install "ray[serve,llm]" +``` + +--- + +### Launch + +Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_qwen_3_32b.py`. + +In a terminal, run: + + +```bash +%%bash +serve run serve_qwen_3_32b:app --non-blocking +``` + +Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. + +Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `"FAKE_KEY"` + +Use the `model_id` defined in your config (here, `my-qwen-3-32b`) to query your model. Below are some examples on how to send a request to a Qwen-3 deployment with thinking enabled or disabled. + +--- + +### Send request with thinking disabled + +You can disable thinking in Qwen-3 by either adding a `/no_think` tag in the prompt or by forwarding `enable_thinking: False` to the vLLM inference engine. + +Example curl with `/no_think` + + +```bash +%%bash +curl -X POST http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer FAKE_KEY" \ + -d '{ \ + "model": "my-qwen-3-32b", \ + "messages": [{"role": "user", "content": "What is greater between 7.8 and 7.11 ? /no_think"}] \ + }' +``` + +Example Python with `enable_thinking: False`: + + +```python +#client_thinking_disabled.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +# Example: Complex query with thinking process +response = client.chat.completions.create( + model="my-qwen-3-32b", + messages=[ + {"role": "user", "content": "What's the capital of France ?"} + ], + extra_body={"chat_template_kwargs": {"enable_thinking": False}} +) + +print(f"Reasoning: \n{response.choices[0].message.reasoning_content}\n\n") +print(f"Answer: \n {response.choices[0].message.content}") +``` + +Notice the `reasoning_content` is empty here. +**Note:** Depending on your model's documentation, empty could mean `None`, an empty string or even empty tags `""`. + +--- + +### Send request with thinking enabled + +You can enable thinking in Qwen-3 by either adding a `/think` tag in the prompt or by forwarding `enable_thinking: True` to the vLLM inference engine. + +Example curl with `/think` + + +```bash +%%bash +curl -X POST http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer FAKE_KEY" \ + -d '{ \ + "model": "my-qwen-3-32b", \ + "messages": [{"role": "user", "content": "What is greater between 7.8 and 7.11 ? /think"}] \ + }' +``` + + Example Python with `enable_thinking: True`: + + +```python +#client_thinking_enabled.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +# Example: Complex query with thinking process +response = client.chat.completions.create( + model="my-qwen-3-32b", + messages=[ + {"role": "user", "content": "What's the capital of France ?"} + ], + extra_body={"chat_template_kwargs": {"enable_thinking": True}} +) + +print(f"Reasoning: \n{response.choices[0].message.reasoning_content}\n\n") +print(f"Answer: \n {response.choices[0].message.content}") +``` + +If you configure a valid reasoning parser, the reasoning output should appear in the `reasoning_content` field of the response message. Otherwise, it may be included in the main `content` field, typically wrapped in `...` tags. See [Parse reasoning outputs](#parse-reasoning-outputs) for more information. + +--- + +### Shutdown + +Shutdown your LLM service: + + +```bash +%%bash +serve shutdown -y +``` + + +--- + +## Deploy to production with Anyscale services + +For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without any code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#production-deployment-with-anyscale-service) for an example with a medium-sized model like the *Qwen-32b* from this tutorial. + +--- + +## Stream reasoning content + +In thinking mode, hybrid reasoning models may take longer to begin generating the main content. You can stream intermediate reasoning output in the same way as the main content. + + +```python +#client_streaming.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +# Example: Complex query with thinking process +response = client.chat.completions.create( + model="my-qwen-3-32b", + messages=[ + {"role": "user", "content": "What's the capital of France ?"} + ], + extra_body={"chat_template_kwargs": {"enable_thinking": True}} +) + +print(f"Reasoning: \n{response.choices[0].message.reasoning_content}\n\n") +print(f"Answer: \n {response.choices[0].message.content}") +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +# Example: Complex query with thinking process +response = client.chat.completions.create( + model="my-qwen-3-32b", + messages=[ + {"role": "user", "content": "I need to plan a trip to Paris from Seattle. Can you help me research flight costs, create an itinerary for 3 days, and suggest restaurants based on my dietary restrictions (vegetarian)?"} + ], + extra_body={"chat_template_kwargs": {"enable_thinking": True}}, + stream=True +) + +# Stream +for chunk in response: + # Stream reasoning content + if hasattr(chunk.choices[0].delta, "reasoning_content"): + data_reasoning = chunk.choices[0].delta.reasoning_content + if data_reasoning: + print(data_reasoning, end="", flush=True) + # Later, stream the final answer + if hasattr(chunk.choices[0].delta, "content"): + data_content = chunk.choices[0].delta.content + if data_content: + print(data_content, end="", flush=True) +``` + + +--- + +## Summary + +In this tutorial, you deployed a hybrid reasoning LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM with the right reasoning parser, deploy your service on your Ray cluster, send requests, and parse reasoning outputs in the response. diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_streaming.py b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_streaming.py new file mode 100644 index 000000000000..7e51b794445c --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_streaming.py @@ -0,0 +1,34 @@ +# client_streaming.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +# Example: Complex query with thinking process +response = client.chat.completions.create( + model="my-qwen-3-32b", + messages=[ + { + "role": "user", + "content": "I need to plan a trip to Paris from Seattle. Can you help me research flight costs, create an itinerary for 3 days, and suggest restaurants based on my dietary restrictions (vegetarian)?", + } + ], + extra_body={"chat_template_kwargs": {"enable_thinking": True}}, + stream=True, +) + +# Stream +for chunk in response: + # Stream reasoning content + if hasattr(chunk.choices[0].delta, "reasoning_content"): + data_reasoning = chunk.choices[0].delta.reasoning_content + if data_reasoning: + print(data_reasoning, end="", flush=True) + # Later, stream the final answer + if hasattr(chunk.choices[0].delta, "content"): + data_content = chunk.choices[0].delta.content + if data_content: + print(data_content, end="", flush=True) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_disabled.py b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_disabled.py new file mode 100644 index 000000000000..46aa914441ea --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_disabled.py @@ -0,0 +1,18 @@ +# client.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +# Example: Complex query with thinking process +response = client.chat.completions.create( + model="my-qwen-3-32b", + messages=[{"role": "user", "content": "What's the capital of France ?"}], + extra_body={"chat_template_kwargs": {"enable_thinking": False}}, +) + +print(f"Reasoning: \n{response.choices[0].message.reasoning_content}\n\n") +print(f"Answer: \n {response.choices[0].message.content}") diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_enabled.py b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_enabled.py new file mode 100644 index 000000000000..f09b5868648d --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_enabled.py @@ -0,0 +1,18 @@ +# client_thinking_enabled.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +# Example: Complex query with thinking process +response = client.chat.completions.create( + model="my-qwen-3-32b", + messages=[{"role": "user", "content": "What's the capital of France ?"}], + extra_body={"chat_template_kwargs": {"enable_thinking": True}}, +) + +print(f"Reasoning: \n{response.choices[0].message.reasoning_content}\n\n") +print(f"Answer: \n {response.choices[0].message.content}") diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb new file mode 100644 index 000000000000..08d565da1443 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb @@ -0,0 +1,470 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e926219a", + "metadata": {}, + "source": [ + "# Deploy a hybrid reasoning LLM\n", + "\n", + "A hybrid reasoning model provides flexibility by allowing you to enable or disable reasoning as needed. You can use structured, step-by-step thinking for complex queries while skipping it for simpler ones, balancing accuracy with efficiency depending on the task.\n", + "\n", + "This tutorial deploys a hybrid reasoning LLM using Ray Serve LLM. \n", + "\n", + "---\n", + "\n", + "## Distinction with purely reasoning models\n", + "\n", + "*Hybrid reasoning models* are reasoning-capable models that allow you to toggle the thinking process on and off. You can enable structured, step-by-step reasoning when needed but skip it for simpler queries to reduce latency. Purely reasoning models always apply their reasoning behavior, while hybrid models give you fine-grained control over when to use reasoning.\n", + "\n", + "| **Mode** | **Core behavior** | **Use case examples** | **Limitation** |\n", + "| ---------------- | -------------------------------------------- | ------------------------------------------------------------------- | ------------------------------------------------- |\n", + "| **Thinking ON** | Explicit multi-step thinking process | Math, coding, logic puzzles, multi-hop QA, CoT prompting | Slower response time, more tokens used. |\n", + "| **Thinking OFF** | Direct answer generation | Casual queries, short instructions, single-step answers | May struggle with complex reasoning or interpretability. |\n", + "\n", + "**Note:** Reasoning often benefits from long context windows (32K up to +1M tokens), high token throughput, low-temperature decoding (greedy sampling), and strong instruction tuning or scratchpad-style reasoning.\n", + "\n", + "To see an example of deploying a purely reasoning model like *QwQ-32 B*, see [Deploying a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.html).\n", + "\n", + "---\n", + "\n", + "## Enable or disable thinking\n", + "\n", + "Some hybrid reasoning models let you toggle their \"thinking\" mode on or off. This section explains when to use thinking mode versus skipping it, and shows how to control the setting in practice.\n", + "\n", + "---\n", + "\n", + "### When to enable or disable thinking mode\n", + "\n", + "**Enable thinking mode for:**\n", + "- Complex, multi-step tasks that require reasoning, such as math, physics, or logic problems.\n", + "- Ambiguous queries or situations with incomplete information.\n", + "- Planning, workflow orchestration, or when the model needs to act as an \"agent\" coordinating other tools or models.\n", + "- Analyzing intricate data, images, or charts.\n", + "- In-depth code reviews or evaluating outputs from other AI systems (LLM as Judge approach).\n", + "\n", + "**Disable thinking mode for:**\n", + "- Simple, well-defined, or routine tasks.\n", + "- Low latency and fast responses as the priority.\n", + "- Repetitive, straightforward steps within a larger automated workflow.\n", + "\n", + "---\n", + "\n", + "### How to enable or disable thinking mode\n", + "\n", + "Toggle thinking mode varies by model and framework. Consult the documentation for the model to see how it structures and controls thinking.\n", + "\n", + "For example, to [control reasoning in Qwen-3](https://huggingface.co/Qwen/Qwen3-32B#switching-between-thinking-and-non-thinking-mode), you can:\n", + "* Add `\"/think\"` or `\"/no_think\"` in the prompt.\n", + "* Set `enable_thinking` in the request:\n", + " `extra_body={\"chat_template_kwargs\": {\"enable_thinking\": ...}}`.\n", + "\n", + "See [Send request with thinking enabled](#send-request-with-thinking-enabled) or [Send request with thinking disabled](#send-request-with-thinking-disabled) for practical examples.\n", + "\n", + "---\n", + "\n", + "## Parse reasoning outputs\n", + "\n", + "In thinking mode, hybrid models often separate _reasoning_ from the _final answer_ using tags like `...`. Without a proper parser, this reasoning may end up in the `content` field instead of the dedicated `reasoning_content` field. \n", + "\n", + "To ensure that Ray Serve LLM correctly parses the reasoning output, configure a `reasoning_parser` in your Ray Serve LLM deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output.\n", + "**Note:** For example, *Qwen-3* uses the `qwen3` parser. See the [vLLM docs](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#supported-models) or your model's documentation to find a supported parser, or [build your own](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#how-to-support-a-new-reasoning-model) if needed.\n", + "\n", + "```yaml\n", + "applications:\n", + "- ...\n", + " args:\n", + " llm_configs:\n", + " - model_loading_config:\n", + " model_id: my-qwen-3-32b\n", + " model_source: Qwen/Qwen3-32B\n", + " ...\n", + " engine_kwargs:\n", + " ...\n", + " reasoning_parser: qwen3 # <-- for Qwen-3 models\n", + "```\n", + "\n", + "See [Configure Ray Serve LLM](#configure-ray-serve-llm) for a complete example.\n", + "\n", + "**Example response** \n", + "When using a reasoning parser, the response is typically structured like this:\n", + "\n", + "```python\n", + "ChatCompletionMessage(\n", + " content=\"The temperature is...\",\n", + " ...,\n", + " reasoning_content=\"Okay, the user is asking for the temperature today and tomorrow...\"\n", + ")\n", + "```\n", + "And you can extract the content and reasoning like this\n", + "```python\n", + "response = client.chat.completions.create(\n", + " ...\n", + ")\n", + "\n", + "print(f\"Content: {response.choices[0].message.content}\")\n", + "print(f\"Reasoning: {response.choices[0].message.reasoning_content}\")\n", + "```\n", + "\n", + "---\n", + "\n", + "## Configure Ray Serve LLM\n", + "\n", + "Set your Hugging Face token in the config file to access gated models.\n", + "\n", + "Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object.\n", + "\n", + "Set `tensor_parallel_size` to distribute the model's weights among 8 GPUs in the node. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1daf892", + "metadata": {}, + "outputs": [], + "source": [ + "# serve_qwen_3_32b.py\n", + "from ray.serve.llm import LLMConfig, build_openai_app\n", + "import os\n", + "\n", + "llm_config = LLMConfig(\n", + " model_loading_config=dict(\n", + " model_id=\"my-qwen-3-32b\",\n", + " model_source=\"Qwen/Qwen3-32B\",\n", + " ),\n", + " accelerator_type=\"A100-40G\",\n", + " deployment_config=dict(\n", + " autoscaling_config=dict(\n", + " min_replicas=1,\n", + " max_replicas=2,\n", + " )\n", + " ),\n", + " ### Uncomment if your model is gated and needs your Hugging Face token to access it.\n", + " # runtime_env=dict(env_vars={\"HF_TOKEN\": os.environ.get(\"HF_TOKEN\")}),\n", + " engine_kwargs=dict(\n", + " tensor_parallel_size=8, max_model_len=32768, reasoning_parser=\"qwen3\"\n", + " ),\n", + ")\n", + "app = build_openai_app({\"llm_configs\": [llm_config]})\n" + ] + }, + { + "cell_type": "markdown", + "id": "32272280", + "metadata": {}, + "source": [ + "**Note:** Before moving to a production setup, migrate your settings to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "\n", + "---\n", + "\n", + "## Deploy locally\n", + "\n", + "**Prerequisites**\n", + "\n", + "* Access to GPU compute.\n", + "* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`.\n", + "\n", + "**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks.\n", + "\n", + "**Dependencies:** \n", + "```bash\n", + "pip install \"ray[serve,llm]\"\n", + "```\n", + "\n", + "---\n", + "\n", + "### Launch\n", + "\n", + "Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_qwen_3_32b.py`. \n", + "\n", + "In a terminal, run: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a8f1b58", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve run serve_qwen_3_32b:app --non-blocking" + ] + }, + { + "cell_type": "markdown", + "id": "a24501f5", + "metadata": {}, + "source": [ + "Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. \n", + "\n", + "Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `\"FAKE_KEY\"`\n", + "\n", + "Use the `model_id` defined in your config (here, `my-qwen-3-32b`) to query your model. Below are some examples on how to send a request to a Qwen-3 deployment with thinking enabled or disabled. \n", + "\n", + "---\n", + "\n", + "### Send request with thinking disabled\n", + "\n", + "You can disable thinking in Qwen-3 by either adding a `/no_think` tag in the prompt or by forwarding `enable_thinking: False` to the vLLM inference engine. \n", + "\n", + "Example curl with `/no_think`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d77d2201", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "curl -X POST http://localhost:8000/v1/chat/completions \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -H \"Authorization: Bearer FAKE_KEY\" \\\n", + " -d '{ \\\n", + " \"model\": \"my-qwen-3-32b\", \\\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"What is greater between 7.8 and 7.11 ? /no_think\"}] \\\n", + " }'" + ] + }, + { + "cell_type": "markdown", + "id": "a127ea5f", + "metadata": {}, + "source": [ + "Example Python with `enable_thinking: False`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e51e9d85", + "metadata": {}, + "outputs": [], + "source": [ + "#client_thinking_disabled.py\n", + "from urllib.parse import urljoin\n", + "from openai import OpenAI\n", + "\n", + "api_key = \"FAKE_KEY\"\n", + "base_url = \"http://localhost:8000\"\n", + "\n", + "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "\n", + "# Example: Complex query with thinking process\n", + "response = client.chat.completions.create(\n", + " model=\"my-qwen-3-32b\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"What's the capital of France ?\"}\n", + " ],\n", + " extra_body={\"chat_template_kwargs\": {\"enable_thinking\": False}}\n", + ")\n", + "\n", + "print(f\"Reasoning: \\n{response.choices[0].message.reasoning_content}\\n\\n\")\n", + "print(f\"Answer: \\n {response.choices[0].message.content}\")" + ] + }, + { + "cell_type": "markdown", + "id": "9765b3f8", + "metadata": {}, + "source": [ + "Notice the `reasoning_content` is empty here. \n", + "**Note:** Depending on your model's documentation, empty could mean `None`, an empty string or even empty tags `\"\"`.\n", + "\n", + "---\n", + "\n", + "### Send request with thinking enabled\n", + " \n", + "You can enable thinking in Qwen-3 by either adding a `/think` tag in the prompt or by forwarding `enable_thinking: True` to the vLLM inference engine. \n", + "\n", + "Example curl with `/think`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8702258c", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "curl -X POST http://localhost:8000/v1/chat/completions \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -H \"Authorization: Bearer FAKE_KEY\" \\\n", + " -d '{ \\\n", + " \"model\": \"my-qwen-3-32b\", \\\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"What is greater between 7.8 and 7.11 ? /think\"}] \\\n", + " }'" + ] + }, + { + "cell_type": "markdown", + "id": "c0bad31b", + "metadata": {}, + "source": [ + " Example Python with `enable_thinking: True`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a52eb68", + "metadata": {}, + "outputs": [], + "source": [ + "#client_thinking_enabled.py\n", + "from urllib.parse import urljoin\n", + "from openai import OpenAI\n", + "\n", + "api_key = \"FAKE_KEY\"\n", + "base_url = \"http://localhost:8000\"\n", + "\n", + "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "\n", + "# Example: Complex query with thinking process\n", + "response = client.chat.completions.create(\n", + " model=\"my-qwen-3-32b\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"What's the capital of France ?\"}\n", + " ],\n", + " extra_body={\"chat_template_kwargs\": {\"enable_thinking\": True}}\n", + ")\n", + "\n", + "print(f\"Reasoning: \\n{response.choices[0].message.reasoning_content}\\n\\n\")\n", + "print(f\"Answer: \\n {response.choices[0].message.content}\")" + ] + }, + { + "cell_type": "markdown", + "id": "1f36ba3d", + "metadata": {}, + "source": [ + "If you configure a valid reasoning parser, the reasoning output should appear in the `reasoning_content` field of the response message. Otherwise, it may be included in the main `content` field, typically wrapped in `...` tags. See [Parse reasoning outputs](#parse-reasoning-outputs) for more information.\n", + "\n", + "---\n", + "\n", + "### Shutdown \n", + "\n", + "Shutdown your LLM service:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2cc5cc23", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve shutdown -y" + ] + }, + { + "cell_type": "markdown", + "id": "8009515b", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "## Deploy to production with Anyscale services\n", + "\n", + "For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without any code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#production-deployment-with-anyscale-service) for an example with a medium-sized model like the *Qwen-32b* from this tutorial.\n", + "\n", + "---\n", + "\n", + "## Stream reasoning content\n", + "\n", + "In thinking mode, hybrid reasoning models may take longer to begin generating the main content. You can stream intermediate reasoning output in the same way as the main content. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5f5a877", + "metadata": {}, + "outputs": [], + "source": [ + "#client_streaming.py\n", + "from urllib.parse import urljoin\n", + "from openai import OpenAI\n", + "\n", + "api_key = \"FAKE_KEY\"\n", + "base_url = \"http://localhost:8000\"\n", + "\n", + "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "\n", + "# Example: Complex query with thinking process\n", + "response = client.chat.completions.create(\n", + " model=\"my-qwen-3-32b\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"What's the capital of France ?\"}\n", + " ],\n", + " extra_body={\"chat_template_kwargs\": {\"enable_thinking\": True}}\n", + ")\n", + "\n", + "print(f\"Reasoning: \\n{response.choices[0].message.reasoning_content}\\n\\n\")\n", + "print(f\"Answer: \\n {response.choices[0].message.content}\")\n", + "from urllib.parse import urljoin\n", + "from openai import OpenAI\n", + "\n", + "api_key = \"FAKE_KEY\"\n", + "base_url = \"http://localhost:8000\"\n", + "\n", + "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "\n", + "# Example: Complex query with thinking process\n", + "response = client.chat.completions.create(\n", + " model=\"my-qwen-3-32b\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"I need to plan a trip to Paris from Seattle. Can you help me research flight costs, create an itinerary for 3 days, and suggest restaurants based on my dietary restrictions (vegetarian)?\"}\n", + " ],\n", + " extra_body={\"chat_template_kwargs\": {\"enable_thinking\": True}},\n", + " stream=True\n", + ")\n", + "\n", + "# Stream \n", + "for chunk in response:\n", + " # Stream reasoning content\n", + " if hasattr(chunk.choices[0].delta, \"reasoning_content\"):\n", + " data_reasoning = chunk.choices[0].delta.reasoning_content\n", + " if data_reasoning:\n", + " print(data_reasoning, end=\"\", flush=True)\n", + " # Later, stream the final answer\n", + " if hasattr(chunk.choices[0].delta, \"content\"):\n", + " data_content = chunk.choices[0].delta.content\n", + " if data_content:\n", + " print(data_content, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "id": "d6357c06", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "## Summary\n", + "\n", + "In this tutorial, you deployed a hybrid reasoning LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM with the right reasoning parser, deploy your service on your Ray cluster, send requests, and parse reasoning outputs in the response." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "repo_ray_docs", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/serve_qwen_3_32b.py b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/serve_qwen_3_32b.py new file mode 100644 index 000000000000..e53f28ac6a90 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/serve_qwen_3_32b.py @@ -0,0 +1,23 @@ +# serve_qwen_3_32b.py +from ray.serve.llm import LLMConfig, build_openai_app +import os + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-qwen-3-32b", + model_source="Qwen/Qwen3-32B", + ), + accelerator_type="A100-40G", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=2, + ) + ), + ### Uncomment if your model is gated and needs your Hugging Face token to access it. + # runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict( + tensor_parallel_size=8, max_model_len=32768, reasoning_parser="qwen3" + ), +) +app = build_openai_app({"llm_configs": [llm_config]}) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/Dockerfile b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/Dockerfile new file mode 100644 index 000000000000..a2412390df61 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/Dockerfile @@ -0,0 +1,8 @@ +FROM anyscale/ray:2.49.0-slim-py312-cu128 + +# C compiler for Triton’s runtime build step (vLLM V1 engine) +# https://github.com/vllm-project/vllm/issues/2997 +RUN sudo apt-get update && \ + sudo apt-get install -y --no-install-recommends build-essential + +RUN pip install vllm==0.10.0 \ No newline at end of file diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md new file mode 100644 index 000000000000..f269a9ea8db5 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md @@ -0,0 +1,358 @@ +--- +orphan: true +--- + + + +# Deploy a large size LLM + +A large LLM typically runs on multiple nodes with multiple GPUs, prioritizing peak quality and capability: stronger reasoning, broader knowledge, longer context windows, more robust generalization. When higher latency, complexity, and cost are acceptable trade-offs because you require state-of-the-art results. + +This tutorial deploys DeepSeek-R1, a large LLM with 685 B parameters, using Ray Serve LLM. For smaller models, see [Deploying a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html) or [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html). + +--- + +## Challenges of large-scale deployments + +Deploying a 685 B-parameter model like DeepSeek-R1 presents significant technical challenges. At this scale, the model can't fit on a single GPU or even a single node. You must distribute it across multiple GPUs and nodes using *tensor parallelism* (splitting tensors within each layer) and *pipeline parallelism* (spreading layers across devices). + +Deploying a model of this scale normally requires you to manually launch and coordinate multiple nodes, unless you use a managed platform like [Anyscale](https://www.anyscale.com/), which automates cluster scaling and node orchestration. See [Deploy to production with Anyscale Services](#deploy-to-production-with-anyscale-services) for more details. + +--- + +## Configure Ray Serve LLM + +A large-sized LLM is typically deployed across multiple nodes with multiple GPUs. To fully utilize the hardware, set `pipeline_parallel_size` to the number of nodes and `tensor_parallel_size` to the number of GPUs per node, which distributes the model’s weights evenly. + +Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object. + +**Optional:** Because Deepseek-R1 is a reasoning model, this tutorial uses vLLM’s built-in reasoning parser to correctly separate its reasoning content from the final response. See [Deploying a reasoning LLM: Parse reasoning outputs](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html#parse-reasoning-outputs). + + +```python +# serve_deepseek_r1.py +from ray.serve.llm import LLMConfig, build_openai_app + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-deepseek-r1", + model_source="deepseek-ai/DeepSeek-R1", + ), + accelerator_type="H100", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=1, + ) + ), + ### Uncomment if your model is gated and needs your Hugging Face token to access it. + # runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict( + max_model_len=16384, + # Split weights among 8 GPUs in the node + tensor_parallel_size=8, + pipeline_parallel_size=2, + reasoning_parser="deepseek_r1", # Optional: separate reasoning content from the final answer + ), +) + +app = build_openai_app({"llm_configs": [llm_config]}) + +``` + +**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. + +--- + +## Deploy locally + +**Prerequisites** + +* Access to GPU compute. +* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`. + +**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks. + +**Dependencies:** +```bash +pip install "ray[serve,llm]" +``` + +**Beware**: this is an expensive deployment. + +--- + +### Launch + +Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_deepseek_r1.py`. + +In a terminal, run: + + +```bash +%%bash +serve run serve_deepseek_r1:app --non-blocking +``` + +Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. + +--- + +### Send requests + +Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `"FAKE_KEY"`. + +Example curl: + + +```bash +%%bash +curl -X POST http://localhost:8000/v1/chat/completions \ + -H "Authorization: Bearer FAKE_KEY" \ + -H "Content-Type: application/json" \ + -d '{ \ + "model": "my-deepseek-r1", \ + "messages": [{"role": "user", "content": "What is 2 + 2?"}] \ + }' +``` + +Example Python: + + +```python +#client.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +response = client.chat.completions.create( + model="my-deepseek-r1", + messages=[{"role": "user", "content": "Tell me a joke"}], + stream=True, +) + +# Stream and print JSON +for chunk in response: + # Stream reasoning content first + if hasattr(chunk.choices[0].delta, "reasoning_content"): + data_reasoning = chunk.choices[0].delta.reasoning_content + if data_reasoning: + print(data_reasoning, end="", flush=True) + # Later, stream the final answer + if hasattr(chunk.choices[0].delta, "content"): + data_content = chunk.choices[0].delta.content + if data_content: + print(data_content, end="", flush=True) +``` + + +--- + +### Shutdown + +Shutdown your LLM service: + + +```bash +%%bash +serve shutdown -y +``` + + +--- + +## Deploy to production with Anyscale services + +For production deployment, use Anyscale services to deploy the Ray Serve app to a dedicated cluster without modifying the code. Anyscale provides scalability, fault tolerance, and load balancing, keeping the service resilient against node failures, high traffic, and rolling updates, while also automating multi-node setup and autoscaling for large models like DeepSeek-R1. + +**Beware**: this is an expensive deployment. At the time of writing, the deployment cost is around \$110 USD per hour in the `us-west-2` AWS region using on-demand instances. Because this node has a high amount of inter-node traffic, and cross-zone traffic is expensive (around \$0.02 per GB), it's recommended to *disable cross-zone autoscaling*. This demo is pre-configured with cross-zone autoscaling disabled for your convenience. + +### Prerequisites + +The following template runs only on H100 GPUs in your self-hosted Anyscale cloud, as H100s aren't available in Anyscale’s public cloud. This example uses two nodes of type *8xH100-80 GB:208CPU-1830 GB* on an AWS cloud. + +To provision nodes with 1000 GB of disk capacity, see [Changing the default disk size for GCP clusters](https://docs.anyscale.com/configuration/compute/gcp/#changing-the-default-disk-size) for Google Cloud Platform (GCP) or [Changing the default disk size for AWS clusters](https://docs.anyscale.com/configuration/compute/aws/#changing-the-default-disk-size) for Amazon Web Services (AWS). + +--- + +### Launch the service + +Anyscale provides out-of-the-box images (`anyscale/ray-llm`), which come pre-loaded with Ray Serve LLM, vLLM, and all required GPU/runtime dependencies. This makes it easy to get started without building a custom image. + +Create your Anyscale service configuration in a new `service.yaml` file: +```yaml +#service.yaml +name: deploy-deepseek-r1 +image_uri: anyscale/ray-llm:2.49.0-py311-cu128 +compute_config: + auto_select_worker_config: true + # Change default disk size to 1000GB + advanced_instance_config: + ## AWS ## + BlockDeviceMappings: + - Ebs: + - VolumeSize: 1000 + VolumeType: gp3 + DeleteOnTermination: true + DeviceName: "/dev/sda1" + ######### + ## GCP ## + #instanceProperties: + # disks: + # - boot: true + # auto_delete: true + # initialize_params: + # - disk_size_gb: 1000 + ######### + +working_dir: . +cloud: +applications: +# Point to your app in your Python module +- import_path: serve_deepseek_r1:app +``` + +Deploy your service + + +```bash +%%bash +anyscale service deploy -f service.yaml +``` + +**Note:** If your model is gated, make sure to pass your Hugging Face token to the service with `--env HF_TOKEN=` + +**Custom Dockerfile** +You can customize the container by building your own Dockerfile. In your Anyscale Service config, reference the Dockerfile with `containerfile` (instead of `image_uri`): + +```yaml +# service.yaml +# Replace: +# image_uri: anyscale/ray-llm:2.49.0-py311-cu128 + +# with: +containerfile: ./Dockerfile +``` + +See the [Anyscale base images](https://docs.anyscale.com/reference/base-images) for details on what each image includes. + +--- + +### Send requests + +The `anyscale service deploy` command output shows both the endpoint and authentication token: +```console +(anyscale +3.9s) curl -H "Authorization: Bearer " +``` +You can also retrieve both from the service page in the Anyscale console. Click the **Query** button at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token. + +--- + +### Access the Serve LLM dashboard + +See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling LLM-specific logging. To open the Ray Serve LLM dashboard from an Anyscale service: +1. In the Anyscale console, go to your **Service** or **Workspace** +2. Navigate to the **Metrics** tab +3. Click **View in Grafana** and click **Serve LLM Dashboard** + +--- + +### Shutdown + +Shutdown your Anyscale service: + + +```bash +%%bash +anyscale service terminate -n deploy-deepseek-r1 +``` + + +--- + +## Enable LLM monitoring + +The *Serve LLM dashboard* offers deep visibility into model performance, latency, and system behavior, including: + +* Token throughput (tokens/sec) +* Latency metrics: Time To First Token (TTFT), Time Per Output Token (TPOT) +* KV cache utilization + +To enable these metrics, go to your LLM config and set `log_engine_metrics: true`. Ensure vLLM V1 is active with `VLLM_USE_V1: "1"`. +**Note:** `VLLM_USE_V1: "1"` is the default value with `ray >= 2.48.0` and can be omitted. +```yaml +applications: +- ... + args: + llm_configs: + - ... + runtime_env: + env_vars: + VLLM_USE_V1: "1" + ... + log_engine_metrics: true +``` + +--- + +## Improve concurrency + +Ray Serve LLM uses [vLLM](https://docs.vllm.ai/en/stable/) as its backend engine, which logs the *maximum concurrency* it can support based on your configuration. + +Example log: +```console +INFO 07-30 11:56:04 [kv_cache_utils.py:637] Maximum concurrency for 32,768 tokens per request: 29.06x +``` + +The following are a few ways to improve concurrency depending on your model and hardware: + +**Reduce `max_model_len`** +Lowering `max_model_len` reduces the memory needed for KV cache. + +**Example**: Running DeepSeek-R1 on 2 nodes with 8xH100-80 GB GPUs each: +* `max_model_len = 32,768` → concurrency ≈ 29 +* `max_model_len = 16,384` → concurrency ≈ 58 + +**Use distilled or quantized models** +Quantizing or distilling your model reduces its memory footprint, freeing up space for more KV cache and enabling more concurrent requests. For example, see [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B`](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B) for a distilled version of DeepSeek-R1. + + +**Upgrade to GPUs with more memory** +Some GPUs provide significantly more room for KV cache and allow for higher concurrency out of the box. + +**Scale with more replicas** +In addition to tuning per-GPU concurrency, you can scale *horizontally* by increasing the number of replicas in your config. +Each replica runs on its own GPU, so raising the replica count increases the total number of concurrent requests your service can handle, especially under sustained or bursty traffic. +```yaml +deployment_config: + autoscaling_config: + min_replicas: 1 + max_replicas: 4 +``` + +*For more details on tuning strategies, hardware guidance, and serving configurations, see [Choose a GPU for LLM serving](https://docs.anyscale.com/llm/serving/gpu-guidance) and [Tune parameters for LLMs on Anyscale services](https://docs.anyscale.com/llm/serving/parameter-tuning).* + +--- + +## Troubleshooting + +**Hugging Face auth errors** +Some models, such as Llama-3.1, are gated and require prior authorization from the organization. See your model’s documentation for instructions on obtaining access. + +**Out-Of-Memory errors** +Out‑of‑memory (OOM) errors are one of the most common failure modes when deploying LLMs, especially as model sizes, and context length increase. +See [Troubleshooting Guide](https://docs.anyscale.com/overview) for common errors and how to fix them. + +--- + +## Summary + +In this tutorial, you deployed a large-sized LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and how to send requests. You also learned how to monitor your app and troubleshoot common issues. diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/client.py b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/client.py new file mode 100644 index 000000000000..0f76eb43e6b9 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/client.py @@ -0,0 +1,27 @@ +# client.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +response = client.chat.completions.create( + model="my-deepseek-r1", + messages=[{"role": "user", "content": "Tell me a joke"}], + stream=True, +) + +# Stream and print JSON +for chunk in response: + # Stream reasoning content first + if hasattr(chunk.choices[0].delta, "reasoning_content"): + data_reasoning = chunk.choices[0].delta.reasoning_content + if data_reasoning: + print(data_reasoning, end="", flush=True) + # Later, stream the final answer + if hasattr(chunk.choices[0].delta, "content"): + data_content = chunk.choices[0].delta.content + if data_content: + print(data_content, end="", flush=True) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb new file mode 100644 index 000000000000..e55d72774072 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb @@ -0,0 +1,450 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f8f6fcbd", + "metadata": {}, + "source": [ + "# Deploy a large size LLM\n", + "\n", + "A large LLM typically runs on multiple nodes with multiple GPUs, prioritizing peak quality and capability: stronger reasoning, broader knowledge, longer context windows, more robust generalization. When higher latency, complexity, and cost are acceptable trade-offs because you require state-of-the-art results.\n", + "\n", + "This tutorial deploys DeepSeek-R1, a large LLM with 685 B parameters, using Ray Serve LLM. For smaller models, see [Deploying a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html) or [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html).\n", + "\n", + "---\n", + "\n", + "## Challenges of large-scale deployments\n", + "\n", + "Deploying a 685 B-parameter model like DeepSeek-R1 presents significant technical challenges. At this scale, the model can't fit on a single GPU or even a single node. You must distribute it across multiple GPUs and nodes using *tensor parallelism* (splitting tensors within each layer) and *pipeline parallelism* (spreading layers across devices). \n", + "\n", + "Deploying a model of this scale normally requires you to manually launch and coordinate multiple nodes, unless you use a managed platform like [Anyscale](https://www.anyscale.com/), which automates cluster scaling and node orchestration. See [Deploy to production with Anyscale Services](#deploy-to-production-with-anyscale-services) for more details.\n", + "\n", + "---\n", + "\n", + "## Configure Ray Serve LLM\n", + "\n", + "A large-sized LLM is typically deployed across multiple nodes with multiple GPUs. To fully utilize the hardware, set `pipeline_parallel_size` to the number of nodes and `tensor_parallel_size` to the number of GPUs per node, which distributes the model’s weights evenly.\n", + "\n", + "Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object.\n", + "\n", + "**Optional:** Because Deepseek-R1 is a reasoning model, this tutorial uses vLLM’s built-in reasoning parser to correctly separate its reasoning content from the final response. See [Deploying a reasoning LLM: Parse reasoning outputs](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html#parse-reasoning-outputs)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d185d580", + "metadata": {}, + "outputs": [], + "source": [ + "# serve_deepseek_r1.py\n", + "from ray.serve.llm import LLMConfig, build_openai_app\n", + "\n", + "llm_config = LLMConfig(\n", + " model_loading_config=dict(\n", + " model_id=\"my-deepseek-r1\",\n", + " model_source=\"deepseek-ai/DeepSeek-R1\",\n", + " ),\n", + " accelerator_type=\"H100\",\n", + " deployment_config=dict(\n", + " autoscaling_config=dict(\n", + " min_replicas=1,\n", + " max_replicas=1,\n", + " )\n", + " ),\n", + " ### Uncomment if your model is gated and needs your Hugging Face token to access it.\n", + " # runtime_env=dict(env_vars={\"HF_TOKEN\": os.environ.get(\"HF_TOKEN\")}),\n", + " engine_kwargs=dict(\n", + " max_model_len=16384,\n", + " # Split weights among 8 GPUs in the node\n", + " tensor_parallel_size=8,\n", + " pipeline_parallel_size=2,\n", + " reasoning_parser=\"deepseek_r1\", # Optional: separate reasoning content from the final answer\n", + " ),\n", + ")\n", + "\n", + "app = build_openai_app({\"llm_configs\": [llm_config]})\n" + ] + }, + { + "cell_type": "markdown", + "id": "6b2231a5", + "metadata": {}, + "source": [ + "**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "\n", + "---\n", + "\n", + "## Deploy locally\n", + "\n", + "**Prerequisites**\n", + "\n", + "* Access to GPU compute.\n", + "* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`.\n", + "\n", + "**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks.\n", + "\n", + "**Dependencies:** \n", + "```bash\n", + "pip install \"ray[serve,llm]\"\n", + "```\n", + "\n", + "**Beware**: this is an expensive deployment.\n", + "\n", + "---\n", + "\n", + "### Launch\n", + "\n", + "Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_deepseek_r1.py`. \n", + "\n", + "In a terminal, run: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae9da12c", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve run serve_deepseek_r1:app --non-blocking" + ] + }, + { + "cell_type": "markdown", + "id": "96d18e22", + "metadata": {}, + "source": [ + "Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. \n", + "\n", + "---\n", + "\n", + "### Send requests\n", + "\n", + "Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `\"FAKE_KEY\"`.\n", + "\n", + "Example curl:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1dd345c", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "curl -X POST http://localhost:8000/v1/chat/completions \\\n", + " -H \"Authorization: Bearer FAKE_KEY\" \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{ \\\n", + " \"model\": \"my-deepseek-r1\", \\\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"What is 2 + 2?\"}] \\\n", + " }'" + ] + }, + { + "cell_type": "markdown", + "id": "dca5e4fd", + "metadata": {}, + "source": [ + "Example Python:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "584f01f7", + "metadata": {}, + "outputs": [], + "source": [ + "#client.py\n", + "from urllib.parse import urljoin\n", + "from openai import OpenAI\n", + "\n", + "api_key = \"FAKE_KEY\"\n", + "base_url = \"http://localhost:8000\"\n", + "\n", + "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"my-deepseek-r1\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Tell me a joke\"}],\n", + " stream=True,\n", + ")\n", + "\n", + "# Stream and print JSON\n", + "for chunk in response:\n", + " # Stream reasoning content first\n", + " if hasattr(chunk.choices[0].delta, \"reasoning_content\"):\n", + " data_reasoning = chunk.choices[0].delta.reasoning_content\n", + " if data_reasoning:\n", + " print(data_reasoning, end=\"\", flush=True)\n", + " # Later, stream the final answer\n", + " if hasattr(chunk.choices[0].delta, \"content\"):\n", + " data_content = chunk.choices[0].delta.content\n", + " if data_content:\n", + " print(data_content, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "id": "1a5fd1fb", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "### Shutdown\n", + "\n", + "Shutdown your LLM service: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c03cdb9", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve shutdown -y" + ] + }, + { + "cell_type": "markdown", + "id": "dc223463", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "## Deploy to production with Anyscale services\n", + "\n", + "For production deployment, use Anyscale services to deploy the Ray Serve app to a dedicated cluster without modifying the code. Anyscale provides scalability, fault tolerance, and load balancing, keeping the service resilient against node failures, high traffic, and rolling updates, while also automating multi-node setup and autoscaling for large models like DeepSeek-R1.\n", + "\n", + "**Beware**: this is an expensive deployment. At the time of writing, the deployment cost is around \\$110 USD per hour in the `us-west-2` AWS region using on-demand instances. Because this node has a high amount of inter-node traffic, and cross-zone traffic is expensive (around \\$0.02 per GB), it's recommended to *disable cross-zone autoscaling*. This demo is pre-configured with cross-zone autoscaling disabled for your convenience.\n", + "\n", + "### Prerequisites\n", + "\n", + "The following template runs only on H100 GPUs in your self-hosted Anyscale cloud, as H100s aren't available in Anyscale’s public cloud. This example uses two nodes of type *8xH100-80 GB:208CPU-1830 GB* on an AWS cloud.\n", + "\n", + "To provision nodes with 1000 GB of disk capacity, see [Changing the default disk size for GCP clusters](https://docs.anyscale.com/configuration/compute/gcp/#changing-the-default-disk-size) for Google Cloud Platform (GCP) or [Changing the default disk size for AWS clusters](https://docs.anyscale.com/configuration/compute/aws/#changing-the-default-disk-size) for Amazon Web Services (AWS). \n", + "\n", + "---\n", + "\n", + "### Launch the service\n", + "\n", + "Anyscale provides out-of-the-box images (`anyscale/ray-llm`), which come pre-loaded with Ray Serve LLM, vLLM, and all required GPU/runtime dependencies. This makes it easy to get started without building a custom image.\n", + "\n", + "Create your Anyscale service configuration in a new `service.yaml` file:\n", + "```yaml\n", + "#service.yaml\n", + "name: deploy-deepseek-r1\n", + "image_uri: anyscale/ray-llm:2.49.0-py311-cu128\n", + "compute_config:\n", + " auto_select_worker_config: true \n", + " # Change default disk size to 1000GB\n", + " advanced_instance_config:\n", + " ## AWS ##\n", + " BlockDeviceMappings:\n", + " - Ebs:\n", + " - VolumeSize: 1000\n", + " VolumeType: gp3\n", + " DeleteOnTermination: true\n", + " DeviceName: \"/dev/sda1\"\n", + " #########\n", + " ## GCP ##\n", + " #instanceProperties:\n", + " # disks:\n", + " # - boot: true\n", + " # auto_delete: true\n", + " # initialize_params:\n", + " # - disk_size_gb: 1000\n", + " #########\n", + " \n", + "working_dir: .\n", + "cloud:\n", + "applications:\n", + "# Point to your app in your Python module\n", + "- import_path: serve_deepseek_r1:app\n", + "```\n", + "\n", + "Deploy your service" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa1c6108", + "metadata": { + "pygments_lexer": "bash" + }, + "outputs": [], + "source": [ + "%%bash\n", + "anyscale service deploy -f service.yaml" + ] + }, + { + "cell_type": "markdown", + "id": "18226fd7", + "metadata": {}, + "source": [ + "**Note:** If your model is gated, make sure to pass your Hugging Face token to the service with `--env HF_TOKEN=`\n", + "\n", + "**Custom Dockerfile** \n", + "You can customize the container by building your own Dockerfile. In your Anyscale Service config, reference the Dockerfile with `containerfile` (instead of `image_uri`):\n", + "\n", + "```yaml\n", + "# service.yaml\n", + "# Replace:\n", + "# image_uri: anyscale/ray-llm:2.49.0-py311-cu128\n", + "\n", + "# with:\n", + "containerfile: ./Dockerfile\n", + "```\n", + "\n", + "See the [Anyscale base images](https://docs.anyscale.com/reference/base-images) for details on what each image includes.\n", + "\n", + "---\n", + "\n", + "### Send requests \n", + "\n", + "The `anyscale service deploy` command output shows both the endpoint and authentication token:\n", + "```console\n", + "(anyscale +3.9s) curl -H \"Authorization: Bearer \" \n", + "```\n", + "You can also retrieve both from the service page in the Anyscale console. Click the **Query** button at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token. \n", + "\n", + "---\n", + "\n", + "### Access the Serve LLM dashboard\n", + "\n", + "See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling LLM-specific logging. To open the Ray Serve LLM dashboard from an Anyscale service:\n", + "1. In the Anyscale console, go to your **Service** or **Workspace**\n", + "2. Navigate to the **Metrics** tab\n", + "3. Click **View in Grafana** and click **Serve LLM Dashboard**\n", + "\n", + "---\n", + "\n", + "### Shutdown \n", + " \n", + "Shutdown your Anyscale service:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "211d5baf", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "anyscale service terminate -n deploy-deepseek-r1" + ] + }, + { + "cell_type": "markdown", + "id": "1d8fba49", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "## Enable LLM monitoring\n", + "\n", + "The *Serve LLM dashboard* offers deep visibility into model performance, latency, and system behavior, including:\n", + "\n", + "* Token throughput (tokens/sec)\n", + "* Latency metrics: Time To First Token (TTFT), Time Per Output Token (TPOT)\n", + "* KV cache utilization\n", + "\n", + "To enable these metrics, go to your LLM config and set `log_engine_metrics: true`. Ensure vLLM V1 is active with `VLLM_USE_V1: \"1\"`. \n", + "**Note:** `VLLM_USE_V1: \"1\"` is the default value with `ray >= 2.48.0` and can be omitted.\n", + "```yaml\n", + "applications:\n", + "- ...\n", + " args:\n", + " llm_configs:\n", + " - ...\n", + " runtime_env:\n", + " env_vars:\n", + " VLLM_USE_V1: \"1\"\n", + " ...\n", + " log_engine_metrics: true\n", + "```\n", + "\n", + "---\n", + "\n", + "## Improve concurrency\n", + "\n", + "Ray Serve LLM uses [vLLM](https://docs.vllm.ai/en/stable/) as its backend engine, which logs the *maximum concurrency* it can support based on your configuration. \n", + "\n", + "Example log:\n", + "```console\n", + "INFO 07-30 11:56:04 [kv_cache_utils.py:637] Maximum concurrency for 32,768 tokens per request: 29.06x\n", + "```\n", + "\n", + "The following are a few ways to improve concurrency depending on your model and hardware: \n", + "\n", + "**Reduce `max_model_len`** \n", + "Lowering `max_model_len` reduces the memory needed for KV cache.\n", + "\n", + "**Example**: Running DeepSeek-R1 on 2 nodes with 8xH100-80 GB GPUs each:\n", + "* `max_model_len = 32,768` → concurrency ≈ 29\n", + "* `max_model_len = 16,384` → concurrency ≈ 58\n", + "\n", + "**Use distilled or quantized models** \n", + "Quantizing or distilling your model reduces its memory footprint, freeing up space for more KV cache and enabling more concurrent requests. For example, see [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B`](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B) for a distilled version of DeepSeek-R1.\n", + "\n", + "\n", + "**Upgrade to GPUs with more memory** \n", + "Some GPUs provide significantly more room for KV cache and allow for higher concurrency out of the box.\n", + "\n", + "**Scale with more replicas** \n", + "In addition to tuning per-GPU concurrency, you can scale *horizontally* by increasing the number of replicas in your config. \n", + "Each replica runs on its own GPU, so raising the replica count increases the total number of concurrent requests your service can handle, especially under sustained or bursty traffic.\n", + "```yaml\n", + "deployment_config:\n", + " autoscaling_config:\n", + " min_replicas: 1\n", + " max_replicas: 4\n", + "```\n", + "\n", + "*For more details on tuning strategies, hardware guidance, and serving configurations, see [Choose a GPU for LLM serving](https://docs.anyscale.com/llm/serving/gpu-guidance) and [Tune parameters for LLMs on Anyscale services](https://docs.anyscale.com/llm/serving/parameter-tuning).*\n", + "\n", + "---\n", + "\n", + "## Troubleshooting\n", + "\n", + "**Hugging Face auth errors** \n", + "Some models, such as Llama-3.1, are gated and require prior authorization from the organization. See your model’s documentation for instructions on obtaining access.\n", + "\n", + "**Out-Of-Memory errors** \n", + "Out‑of‑memory (OOM) errors are one of the most common failure modes when deploying LLMs, especially as model sizes, and context length increase. \n", + "See [Troubleshooting Guide](https://docs.anyscale.com/overview) for common errors and how to fix them.\n", + "\n", + "---\n", + "\n", + "## Summary\n", + "\n", + "In this tutorial, you deployed a large-sized LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and how to send requests. You also learned how to monitor your app and troubleshoot common issues." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "repo_ray_docs", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/serve_deepseek_r1.py b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/serve_deepseek_r1.py new file mode 100644 index 000000000000..4f95a2e6d8aa --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/serve_deepseek_r1.py @@ -0,0 +1,27 @@ +# serve_deepseek_r1.py +from ray.serve.llm import LLMConfig, build_openai_app + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-deepseek-r1", + model_source="deepseek-ai/DeepSeek-R1", + ), + accelerator_type="H100", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=1, + ) + ), + ### Uncomment if your model is gated and needs your Hugging Face token to access it. + # runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict( + max_model_len=16384, + # Split weights among 8 GPUs in the node + tensor_parallel_size=8, + pipeline_parallel_size=2, + reasoning_parser="deepseek_r1", # Optional: separate reasoning content from the final answer + ), +) + +app = build_openai_app({"llm_configs": [llm_config]}) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/service.yaml b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/service.yaml new file mode 100644 index 000000000000..fb53624fe496 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/service.yaml @@ -0,0 +1,29 @@ +#service.yaml +name: deploy-deepseek-r1 +image_uri: anyscale/ray-llm:2.48.0-py311-cu128 +compute_config: + auto_select_worker_config: true + # Change default disk size to 1000GB + advanced_instance_config: + ## AWS ## + BlockDeviceMappings: + - Ebs: + - VolumeSize: 1000 + VolumeType: gp3 + DeleteOnTermination: true + DeviceName: "/dev/sda1" + ######### + ## GCP ## + #instanceProperties: + # disks: + # - boot: true + # auto_delete: true + # initialize_params: + # - disk_size_gb: 1000 + ######### + +working_dir: . +cloud: +applications: +# Point to your app in your Python module +- import_path: serve_deepseek_r1:app \ No newline at end of file diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/Dockerfile b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/Dockerfile new file mode 100644 index 000000000000..a2412390df61 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/Dockerfile @@ -0,0 +1,8 @@ +FROM anyscale/ray:2.49.0-slim-py312-cu128 + +# C compiler for Triton’s runtime build step (vLLM V1 engine) +# https://github.com/vllm-project/vllm/issues/2997 +RUN sudo apt-get update && \ + sudo apt-get install -y --no-install-recommends build-essential + +RUN pip install vllm==0.10.0 \ No newline at end of file diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md new file mode 100644 index 000000000000..b93843b2cc90 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md @@ -0,0 +1,315 @@ +--- +orphan: true +--- + + + +# Deploying a medium size LLM + +A medium LLM typically runs on a single node with 4-8 GPUs. It offers a balance between performance and efficiency. These models provide stronger accuracy and reasoning than small models while remaining more affordable and resource-friendly than very large ones. This makes them a solid choice for production workloads that need good quality at lower cost. They're also ideal for scaling applications where large models would be too slow or expensive. + +This tutorial deploys a medium-sized LLM using Ray Serve LLM. For smaller models, see [Deploying a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html), and for larger models, see [Deploying a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html). + +--- + +## Configure Ray Serve LLM + +You can deploy a medium-sized LLM on a single node with multiple GPUs. To leverage all available GPUs, set `tensor_parallel_size` to the number of GPUs on the node, which distributes the model’s weights evenly across them. + +Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object. + +Set your Hugging Face token in the config file to access gated models like `Llama-3.1`. + + +```python +# serve_llama_3_1_70b.py +from ray.serve.llm import LLMConfig, build_openai_app +import os + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-llama-3.1-70b", + # Or unsloth/Meta-Llama-3.1-70B-Instruct for an ungated model + model_source="meta-llama/Llama-3.1-70B-Instruct", + ), + accelerator_type="A100-40G", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=4, + ) + ), + ### If your model is not gated, you can skip `hf_token` + # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3. + # Type `export HF_TOKEN=` in a terminal + runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict( + max_model_len=32768, + # Split weights among 8 GPUs in the node + tensor_parallel_size=8, + ), +) + +app = build_openai_app({"llm_configs": [llm_config]}) + +``` + +**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. + +--- + +## Deploy locally + +**Prerequisites** + +* Access to GPU compute. +* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`. + +**Note: **Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama model approval can take anywhere from a few hours to several weeks. + +**Dependencies:** +```bash +pip install "ray[serve,llm]" +``` + +--- + +### Launch + +Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_llama_3_1_70b.py`. + +In a terminal, run: + + +```bash +%%bash +serve run serve_llama_3_1_70b:app --non-blocking +``` + +Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. + +--- + +### Send requests + +Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `"FAKE_KEY"`. + +Example curl: + + +```bash +%%bash +curl -X POST http://localhost:8000/v1/chat/completions \ + -H "Authorization: Bearer FAKE_KEY" \ + -H "Content-Type: application/json" \ + -d '{ \ + "model": "my-llama-3.1-70b", \ + "messages": [{"role": "user", "content": "What is 2 + 2?"}] \ + }' +``` + +Example Python: + + +```python +#client.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +response = client.chat.completions.create( + model="my-llama-3.1-70b", + messages=[{"role": "user", "content": "Tell me a joke"}], + stream=True +) + +for chunk in response: + content = chunk.choices[0].delta.content + if content: + print(content, end="", flush=True) +``` + + +--- + +### Shutdown + +Shutdown your LLM service: + + +```bash +%%bash +serve shutdown -y +``` + + +--- + +## Deploy to production with Anyscale services + +For production deployment, use Anyscale services to deploy the Ray Serve app to a dedicated cluster without modifying the code. Anyscale ensures scalability, fault tolerance, and load balancing, keeping the service resilient against node failures, high traffic, and rolling updates. + +--- + +### Launch the service + +Anyscale provides out-of-the-box images (`anyscale/ray-llm`), which come pre-loaded with Ray Serve LLM, vLLM, and all required GPU/runtime dependencies. This makes it easy to get started without building a custom image. + +Create your Anyscale service configuration in a new `service.yaml` file: +```yaml +#service.yaml +name: deploy-llama-3-70b +image_uri: anyscale/ray-llm:2.49.0-py311-cu128 +compute_config: + auto_select_worker_config: true +working_dir: . +cloud: +applications: +# Point to your app in your Python module +- import_path: serve_llama_3_1_70b:app +``` + +Deploy your service. Make sure you forward your Hugging Face token to the command. + + +```bash +%%bash +anyscale service deploy -f service.yaml --env HF_TOKEN= +``` + +**Custom Dockerfile** +You can customize the container by building your own Dockerfile. In your Anyscale Service config, reference the Dockerfile with `containerfile` (instead of `image_uri`): + +```yaml +# service.yaml +# Replace: +# image_uri: anyscale/ray-llm:2.49.0-py311-cu128 + +# with: +containerfile: ./Dockerfile +``` + +See the [Anyscale base images](https://docs.anyscale.com/reference/base-images) for details on what each image includes. + +--- + +### Send requests + +The `anyscale service deploy` command output shows both the endpoint and authentication token: +```console +(anyscale +3.9s) curl -H "Authorization: Bearer " +``` +You can also retrieve both from the service page in the Anyscale console. Click the **Query** button at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token. + +--- + +### Access the Serve LLM dashboard + +See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling LLM-specific logging. To open the Ray Serve LLM dashboard from an Anyscale service: +1. In the Anyscale console, go to your **Service** or **Workspace** +2. Navigate to the **Metrics** tab +3. Click **View in Grafana** and click **Serve LLM Dashboard** + +--- + +### Shutdown + +Shutdown your Anyscale service: + + +```bash +%%bash +anyscale service terminate -n deploy-llama-3-70b +``` + + +--- + +## Enable LLM monitoring + +The *Serve LLM Dashboard* offers deep visibility into model performance, latency, and system behavior, including: + +* Token throughput (tokens/sec). +* Latency metrics: Time To First Token (TTFT), Time Per Output Token (TPOT). +* KV cache utilization. + +To enable these metrics, go to your LLM config and set `log_engine_metrics: true`. Ensure vLLM V1 is active with `VLLM_USE_V1: "1"`. +**Note:** `VLLM_USE_V1: "1"` is the default value with `ray >= 2.48.0` and can be omitted. +```yaml +applications: +- ... + args: + llm_configs: + - ... + runtime_env: + env_vars: + VLLM_USE_V1: "1" + ... + log_engine_metrics: true +``` + +--- + +## Improve concurrency + +Ray Serve LLM uses [vLLM](https://docs.vllm.ai/en/latest/) as its backend engine, which logs the *maximum concurrency* it can support based on your configuration. + +Example log: +```console +INFO 08-19 20:57:37 [kv_cache_utils.py:837] Maximum concurrency for 32,768 tokens per request: 13.02x +``` + +The following are a few ways to improve concurrency depending on your model and hardware: + +**Reduce `max_model_len`** +Lowering `max_model_len` reduces the memory needed for KV cache. + +**Example:** Running Llama-3.1-70 B on an A100-40G: +* `max_model_len = 32,768` → concurrency ≈ 13 +* `max_model_len = 16,384` → concurrency ≈ 26 + +**Use Quantized models** +Quantizing your model (for example, to FP8) reduces the model's memory footprint, freeing up memory for more KV cache and enabling more concurrent requests. + +**Use pipeline parallelism** +If a single node isn't enough to handle your workload, consider distributing the model's layers across multiple nodes with `pipeline_parallel_size > 1`. + +**Upgrade to GPUs with more memory** +Some GPUs provide significantly more room for KV cache and allow for higher concurrency out of the box. + +**Scale with more replicas** +In addition to tuning per-GPU concurrency, you can scale *horizontally* by increasing the number of replicas in your config. +Each replica runs on its own GPU, so raising the replica count increases the total number of concurrent requests your service can handle, especially under sustained or bursty traffic. +```yaml +deployment_config: + autoscaling_config: + min_replicas: 1 + max_replicas: 4 +``` + +*For more details on tuning strategies, hardware guidance, and serving configurations, see [Choose a GPU for LLM serving](https://docs.anyscale.com/llm/serving/gpu-guidance) and [Tune parameters for LLMs on Anyscale services](https://docs.anyscale.com/llm/serving/parameter-tuning).* + +--- + +## Troubleshooting + +**Hugging Face auth errors** +Some models, such as Llama-3.1, are gated and require prior authorization from the organization. See your model’s documentation for instructions on obtaining access. + +**Out-of-memory errors** +Out-of-memory (OOM) errors are one of the most common failure modes when deploying LLMs, especially as model sizes and context length increase. +See this [Troubleshooting Guide](https://docs.anyscale.com/overview) for common errors and how to fix them. + +--- + +## Summary + +In this tutorial, you deployed a medium-sized LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and send requests. You also learned how to monitor your app and troubleshoot common issues. diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/client.py b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/client.py new file mode 100644 index 000000000000..4f51d2f67a46 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/client.py @@ -0,0 +1,19 @@ +# client.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +response = client.chat.completions.create( + model="my-llama-3.1-70b", + messages=[{"role": "user", "content": "Tell me a joke"}], + stream=True, +) + +for chunk in response: + content = chunk.choices[0].delta.content + if content: + print(content, end="", flush=True) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb new file mode 100644 index 000000000000..40d49aa0a749 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb @@ -0,0 +1,407 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f8f6fcbd", + "metadata": {}, + "source": [ + "# Deploying a medium size LLM\n", + "\n", + "A medium LLM typically runs on a single node with 4-8 GPUs. It offers a balance between performance and efficiency. These models provide stronger accuracy and reasoning than small models while remaining more affordable and resource-friendly than very large ones. This makes them a solid choice for production workloads that need good quality at lower cost. They're also ideal for scaling applications where large models would be too slow or expensive.\n", + "\n", + "This tutorial deploys a medium-sized LLM using Ray Serve LLM. For smaller models, see [Deploying a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html), and for larger models, see [Deploying a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html).\n", + "\n", + "---\n", + "\n", + "## Configure Ray Serve LLM\n", + "\n", + "You can deploy a medium-sized LLM on a single node with multiple GPUs. To leverage all available GPUs, set `tensor_parallel_size` to the number of GPUs on the node, which distributes the model’s weights evenly across them.\n", + "\n", + "Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object.\n", + "\n", + "Set your Hugging Face token in the config file to access gated models like `Llama-3.1`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d185d580", + "metadata": {}, + "outputs": [], + "source": [ + "# serve_llama_3_1_70b.py\n", + "from ray.serve.llm import LLMConfig, build_openai_app\n", + "import os\n", + "\n", + "llm_config = LLMConfig(\n", + " model_loading_config=dict(\n", + " model_id=\"my-llama-3.1-70b\",\n", + " # Or unsloth/Meta-Llama-3.1-70B-Instruct for an ungated model\n", + " model_source=\"meta-llama/Llama-3.1-70B-Instruct\",\n", + " ),\n", + " accelerator_type=\"A100-40G\",\n", + " deployment_config=dict(\n", + " autoscaling_config=dict(\n", + " min_replicas=1,\n", + " max_replicas=4,\n", + " )\n", + " ),\n", + " ### If your model is not gated, you can skip `hf_token`\n", + " # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3.\n", + " # Type `export HF_TOKEN=` in a terminal\n", + " runtime_env=dict(env_vars={\"HF_TOKEN\": os.environ.get(\"HF_TOKEN\")}),\n", + " engine_kwargs=dict(\n", + " max_model_len=32768,\n", + " # Split weights among 8 GPUs in the node\n", + " tensor_parallel_size=8,\n", + " ),\n", + ")\n", + "\n", + "app = build_openai_app({\"llm_configs\": [llm_config]})\n" + ] + }, + { + "cell_type": "markdown", + "id": "6b2231a5", + "metadata": {}, + "source": [ + "**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "\n", + "---\n", + "\n", + "## Deploy locally\n", + "\n", + "**Prerequisites**\n", + "\n", + "* Access to GPU compute.\n", + "* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`.\n", + "\n", + "**Note: **Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama model approval can take anywhere from a few hours to several weeks.\n", + "\n", + "**Dependencies:** \n", + "```bash\n", + "pip install \"ray[serve,llm]\"\n", + "```\n", + "\n", + "---\n", + "\n", + "### Launch\n", + "\n", + "Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_llama_3_1_70b.py`. \n", + "\n", + "In a terminal, run: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae9da12c", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve run serve_llama_3_1_70b:app --non-blocking" + ] + }, + { + "cell_type": "markdown", + "id": "96d18e22", + "metadata": {}, + "source": [ + "Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. \n", + "\n", + "---\n", + "\n", + "### Send requests\n", + "\n", + "Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `\"FAKE_KEY\"`.\n", + "\n", + "Example curl:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1dd345c", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "curl -X POST http://localhost:8000/v1/chat/completions \\\n", + " -H \"Authorization: Bearer FAKE_KEY\" \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{ \\\n", + " \"model\": \"my-llama-3.1-70b\", \\\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"What is 2 + 2?\"}] \\\n", + " }'" + ] + }, + { + "cell_type": "markdown", + "id": "dca5e4fd", + "metadata": {}, + "source": [ + "Example Python:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "584f01f7", + "metadata": {}, + "outputs": [], + "source": [ + "#client.py\n", + "from urllib.parse import urljoin\n", + "from openai import OpenAI\n", + "\n", + "api_key = \"FAKE_KEY\"\n", + "base_url = \"http://localhost:8000\"\n", + "\n", + "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"my-llama-3.1-70b\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Tell me a joke\"}],\n", + " stream=True\n", + ")\n", + "\n", + "for chunk in response:\n", + " content = chunk.choices[0].delta.content\n", + " if content:\n", + " print(content, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "id": "1a5fd1fb", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "### Shutdown\n", + "\n", + "Shutdown your LLM service: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c03cdb9", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve shutdown -y" + ] + }, + { + "cell_type": "markdown", + "id": "dc223463", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "## Deploy to production with Anyscale services\n", + "\n", + "For production deployment, use Anyscale services to deploy the Ray Serve app to a dedicated cluster without modifying the code. Anyscale ensures scalability, fault tolerance, and load balancing, keeping the service resilient against node failures, high traffic, and rolling updates.\n", + "\n", + "---\n", + "\n", + "### Launch the service\n", + "\n", + "Anyscale provides out-of-the-box images (`anyscale/ray-llm`), which come pre-loaded with Ray Serve LLM, vLLM, and all required GPU/runtime dependencies. This makes it easy to get started without building a custom image.\n", + "\n", + "Create your Anyscale service configuration in a new `service.yaml` file:\n", + "```yaml\n", + "#service.yaml\n", + "name: deploy-llama-3-70b\n", + "image_uri: anyscale/ray-llm:2.49.0-py311-cu128\n", + "compute_config:\n", + " auto_select_worker_config: true \n", + "working_dir: .\n", + "cloud:\n", + "applications:\n", + "# Point to your app in your Python module\n", + "- import_path: serve_llama_3_1_70b:app\n", + "```\n", + "\n", + "Deploy your service. Make sure you forward your Hugging Face token to the command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa1c6108", + "metadata": { + "pygments_lexer": "bash" + }, + "outputs": [], + "source": [ + "%%bash\n", + "anyscale service deploy -f service.yaml --env HF_TOKEN=" + ] + }, + { + "cell_type": "markdown", + "id": "18226fd7", + "metadata": {}, + "source": [ + "**Custom Dockerfile** \n", + "You can customize the container by building your own Dockerfile. In your Anyscale Service config, reference the Dockerfile with `containerfile` (instead of `image_uri`):\n", + "\n", + "```yaml\n", + "# service.yaml\n", + "# Replace:\n", + "# image_uri: anyscale/ray-llm:2.49.0-py311-cu128\n", + "\n", + "# with:\n", + "containerfile: ./Dockerfile\n", + "```\n", + "\n", + "See the [Anyscale base images](https://docs.anyscale.com/reference/base-images) for details on what each image includes.\n", + "\n", + "---\n", + "\n", + "### Send requests \n", + "\n", + "The `anyscale service deploy` command output shows both the endpoint and authentication token:\n", + "```console\n", + "(anyscale +3.9s) curl -H \"Authorization: Bearer \" \n", + "```\n", + "You can also retrieve both from the service page in the Anyscale console. Click the **Query** button at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token. \n", + "\n", + "---\n", + "\n", + "### Access the Serve LLM dashboard\n", + "\n", + "See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling LLM-specific logging. To open the Ray Serve LLM dashboard from an Anyscale service:\n", + "1. In the Anyscale console, go to your **Service** or **Workspace**\n", + "2. Navigate to the **Metrics** tab\n", + "3. Click **View in Grafana** and click **Serve LLM Dashboard**\n", + "\n", + "---\n", + "\n", + "### Shutdown \n", + " \n", + "Shutdown your Anyscale service:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "211d5baf", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "anyscale service terminate -n deploy-llama-3-70b" + ] + }, + { + "cell_type": "markdown", + "id": "1d8fba49", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "## Enable LLM monitoring\n", + "\n", + "The *Serve LLM Dashboard* offers deep visibility into model performance, latency, and system behavior, including:\n", + "\n", + "* Token throughput (tokens/sec).\n", + "* Latency metrics: Time To First Token (TTFT), Time Per Output Token (TPOT).\n", + "* KV cache utilization.\n", + "\n", + "To enable these metrics, go to your LLM config and set `log_engine_metrics: true`. Ensure vLLM V1 is active with `VLLM_USE_V1: \"1\"`. \n", + "**Note:** `VLLM_USE_V1: \"1\"` is the default value with `ray >= 2.48.0` and can be omitted.\n", + "```yaml\n", + "applications:\n", + "- ...\n", + " args:\n", + " llm_configs:\n", + " - ...\n", + " runtime_env:\n", + " env_vars:\n", + " VLLM_USE_V1: \"1\"\n", + " ...\n", + " log_engine_metrics: true\n", + "```\n", + "\n", + "---\n", + "\n", + "## Improve concurrency\n", + "\n", + "Ray Serve LLM uses [vLLM](https://docs.vllm.ai/en/latest/) as its backend engine, which logs the *maximum concurrency* it can support based on your configuration. \n", + "\n", + "Example log:\n", + "```console\n", + "INFO 08-19 20:57:37 [kv_cache_utils.py:837] Maximum concurrency for 32,768 tokens per request: 13.02x\n", + "```\n", + "\n", + "The following are a few ways to improve concurrency depending on your model and hardware: \n", + "\n", + "**Reduce `max_model_len`** \n", + "Lowering `max_model_len` reduces the memory needed for KV cache.\n", + "\n", + "**Example:** Running Llama-3.1-70 B on an A100-40G:\n", + "* `max_model_len = 32,768` → concurrency ≈ 13\n", + "* `max_model_len = 16,384` → concurrency ≈ 26\n", + "\n", + "**Use Quantized models** \n", + "Quantizing your model (for example, to FP8) reduces the model's memory footprint, freeing up memory for more KV cache and enabling more concurrent requests.\n", + "\n", + "**Use pipeline parallelism** \n", + "If a single node isn't enough to handle your workload, consider distributing the model's layers across multiple nodes with `pipeline_parallel_size > 1`.\n", + "\n", + "**Upgrade to GPUs with more memory** \n", + "Some GPUs provide significantly more room for KV cache and allow for higher concurrency out of the box.\n", + "\n", + "**Scale with more replicas** \n", + "In addition to tuning per-GPU concurrency, you can scale *horizontally* by increasing the number of replicas in your config. \n", + "Each replica runs on its own GPU, so raising the replica count increases the total number of concurrent requests your service can handle, especially under sustained or bursty traffic.\n", + "```yaml\n", + "deployment_config:\n", + " autoscaling_config:\n", + " min_replicas: 1\n", + " max_replicas: 4\n", + "```\n", + "\n", + "*For more details on tuning strategies, hardware guidance, and serving configurations, see [Choose a GPU for LLM serving](https://docs.anyscale.com/llm/serving/gpu-guidance) and [Tune parameters for LLMs on Anyscale services](https://docs.anyscale.com/llm/serving/parameter-tuning).*\n", + "\n", + "---\n", + "\n", + "## Troubleshooting\n", + "\n", + "**Hugging Face auth errors** \n", + "Some models, such as Llama-3.1, are gated and require prior authorization from the organization. See your model’s documentation for instructions on obtaining access.\n", + "\n", + "**Out-of-memory errors** \n", + "Out-of-memory (OOM) errors are one of the most common failure modes when deploying LLMs, especially as model sizes and context length increase. \n", + "See this [Troubleshooting Guide](https://docs.anyscale.com/overview) for common errors and how to fix them.\n", + "\n", + "---\n", + "\n", + "## Summary\n", + "\n", + "In this tutorial, you deployed a medium-sized LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and send requests. You also learned how to monitor your app and troubleshoot common issues." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "repo_ray_docs", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/serve_llama_3_1_70b.py b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/serve_llama_3_1_70b.py new file mode 100644 index 000000000000..650b4e2d6574 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/serve_llama_3_1_70b.py @@ -0,0 +1,29 @@ +# serve_llama_3_1_70b.py +from ray.serve.llm import LLMConfig, build_openai_app +import os + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-llama-3.1-70b", + # Or unsloth/Meta-Llama-3.1-70B-Instruct for an ungated model + model_source="meta-llama/Llama-3.1-70B-Instruct", + ), + accelerator_type="A100-40G", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=4, + ) + ), + ### If your model is not gated, you can skip `hf_token` + # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3. + # Type `export HF_TOKEN=` in a terminal + runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict( + max_model_len=32768, + # Split weights among 8 GPUs in the node + tensor_parallel_size=8, + ), +) + +app = build_openai_app({"llm_configs": [llm_config]}) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/service.yaml b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/service.yaml new file mode 100644 index 000000000000..c3828c619110 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/service.yaml @@ -0,0 +1,10 @@ +#service.yaml +name: deploy-llama-3-70b +containerfile: ./Dockerfile +compute_config: + auto_select_worker_config: true +working_dir: . +cloud: +applications: +# Point to your app in your Python module +- import_path: serve_llama_3_1_70b:app \ No newline at end of file diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md new file mode 100644 index 000000000000..dcecf2ae8d0e --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md @@ -0,0 +1,269 @@ +--- +orphan: true +--- + + + +# Deploy a reasoning LLM + +A reasoning LLM handles tasks that require deeper analysis or step-by-step thought. It generates intermediate reasoning before arriving at a final answer, making it better suited for situations where careful logic or structured problem-solving is more important than speed or efficiency. + +This tutorial deploys a reasoning LLM using Ray Serve LLM. + +--- + +## Compare reasoning and non-reasoning models + +Reasoning models simulate step-by-step, structured thought processes to solve complex tasks like math, multi-hop QA, or code generation. In contrast, non-reasoning models provide fast, direct responses and focus on fluency or instruction following without explicit intermediate reasoning. The key distinction lies in whether the model attempts to "think through" the problem before answering. + +| **Model type** | **Core behavior** | **Use case examples** | **Limitation** | +| ----------------------- | ------------------------------------ | -------------------------------------------------------- | ----------------------------------------------------- | +| **Reasoning model** | Explicit multi-step thinking process | Math, coding, logic puzzles, multi-hop QA, CoT prompting | Slower response time, more tokens used. | +| **Non-reasoning model** | Direct answer generation | Casual queries, short instructions, single-step answers | May struggle with complex reasoning or interpretability. | + +Many reasoning-capable models structure their outputs with special markers such as `` tags, or expose reasoning traces inside dedicated fields like `reasoning_content` in the OpenAI API response. Always check the model's documentation for how to structure and control thinking. + +**Note:** Reasoning LLMs often benefit from long context windows (32K up to +1M tokens), high token throughput, low-temperature decoding (greedy sampling), and strong instruction tuning or scratchpad-style reasoning. + +--- + +### Choose when to use reasoning models + +Whether you should use a reasoning model depends on how much information your prompt already provides. + +If your input is clear and complete, a standard model is usually faster and more efficient. If your input is ambiguous or complex, a reasoning model works better because it can work through the problem step by step and fill in gaps through intermediate reasoning. + +--- + +## Parse reasoning outputs + +Reasoning models often separate *reasoning* from the *final answer* using tags like `...`. Without a proper parser, this reasoning may end up in the `content` field instead of the dedicated `reasoning_content` field. + +To extract reasoning correctly, configure a `reasoning_parser` in your Ray Serve deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output. +**Note:** For example, *QwQ* uses the `deepseek-r1` parser. Other models may require different parsers. See the [vLLM docs](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#supported-models) or your model's documentation to find a supported parser, or [build your own](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#how-to-support-a-new-reasoning-model) if needed. + +```yaml +applications: +- name: reasoning-llm-app + ... + args: + llm_configs: + - model_loading_config: + model_id: my-qwq-32B + model_source: Qwen/QwQ-32B + ... + engine_kwargs: + ... + reasoning_parser: deepseek_r1 # <-- for QwQ models +``` + +See [Configure Ray Serve LLM](#configure-ray-serve-llm) for a complete example. + +**Example response** +When using a reasoning parser, the response is typically structured like this: + +```python +ChatCompletionMessage( + content="The temperature is...", + ..., + reasoning_content="Okay, the user is asking for the temperature today and tomorrow..." +) +``` +And you can extract the content and reasoning as follows: +```python +response = client.chat.completions.create( + ... +) + +print(f"Content: {response.choices[0].message.content}") +print(f"Reasoning: {response.choices[0].message.reasoning_content}") +``` + +--- + +## Configure Ray Serve LLM + +Set your Hugging Face token in the config file to access gated models. + +Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object. + +Set `tensor_parallel_size=8` to distribute the model's weights among 8 GPUs in the node. + + +```python +# serve_qwq_32b.py +from ray.serve.llm import LLMConfig, build_openai_app +import os + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-qwq-32B", + model_source="Qwen/QwQ-32B", + ), + accelerator_type="A100-40G", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=2, + ) + ), + ### Uncomment if your model is gated and needs your Hugging Face token to access it + # runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict( + tensor_parallel_size=8, max_model_len=32768, reasoning_parser="deepseek_r1" + ), +) + +app = build_openai_app({"llm_configs": [llm_config]}) + +``` + +**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. + +--- + +## Deploy locally + +**Prerequisites** + +* Access to GPU compute. +* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=` + +**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks. + +**Dependencies:** +```bash +pip install "ray[serve,llm]" +``` + +--- + +### Launch the service + +Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_qwq_32b.py`. + +In a terminal, run: + + +```bash +%%bash +serve run serve_qwq_32b:app --non-blocking +``` + +Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. + +--- + +### Send requests + +Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `"FAKE_KEY"`. + +Example curl: + + +```bash +%%bash +curl -X POST http://localhost:8000/v1/chat/completions \ + -H "Authorization: Bearer FAKE_KEY" \ + -H "Content-Type: application/json" \ + -d '{ \ + "model": "my-qwq-32B", \ + "messages": [{"role": "user", "content": "Pick three random words with 3 syllables each and count the number of R'\''s in each of them"}] \ + }' +``` + +Example python: + + +```python +#client.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +response = client.chat.completions.create( + model="my-qwq-32B", + messages=[ + {"role": "user", "content": "What is the sum of all even numbers between 1 and 100?"} + ] +) + +print(f"Reasoning: \n{response.choices[0].message.reasoning_content}\n\n") +print(f"Answer: \n {response.choices[0].message.content}") +``` + +If you configure a valid reasoning parser, the reasoning output should appear in the `reasoning_content` field of the response message. Otherwise, it may be included in the main `content` field, typically wrapped in `...` tags. See [Parse reasoning outputs](#parse-reasoning-outputs) for more information. + +--- + +### Shutdown + +Shutdown your LLM service: + + +```bash +%%bash +serve shutdown -y +``` + + +--- + +## Deploy to production with Anyscale services + +For production, use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium size LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *QwQ-32 B* used here. + +--- + +## Stream reasoning content + +Reasoning models may take longer to begin generating the main content. You can stream their intermediate reasoning output in the same way as the main content. + + +```python +#client_streaming.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = +base_url = + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +# Example: Complex query with thinking process +response = client.chat.completions.create( + model="my-qwq-32B", + messages=[ + {"role": "user", "content": "I need to plan a trip to Paris from Seattle. Can you help me research flight costs, create an itinerary for 3 days, and suggest restaurants based on my dietary restrictions (vegetarian)?"} + ], + stream=True +) + +# Stream +for chunk in response: + # Stream reasoning content + if hasattr(chunk.choices[0].delta, "reasoning_content"): + data_reasoning = chunk.choices[0].delta.reasoning_content + if data_reasoning: + print(data_reasoning, end="", flush=True) + # Later, stream the final answer + if hasattr(chunk.choices[0].delta, "content"): + data_content = chunk.choices[0].delta.content + if data_content: + print(data_content, end="", flush=True) +``` + + +--- + +## Summary + +In this tutorial, you deployed a reasoning LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM with the right reasoning parser, deploy your service on your Ray cluster, send requests, and parse reasoning outputs in the response. diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client.py b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client.py new file mode 100644 index 000000000000..9b5d768acc65 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client.py @@ -0,0 +1,21 @@ +# client.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +response = client.chat.completions.create( + model="my-qwq-32B", + messages=[ + { + "role": "user", + "content": "What is the sum of all even numbers between 1 and 100?", + } + ], +) + +print(f"Reasoning: \n{response.choices[0].message.reasoning_content}\n\n") +print(f"Answer: \n {response.choices[0].message.content}") diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client_streaming.py b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client_streaming.py new file mode 100644 index 000000000000..f5c896593a3c --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client_streaming.py @@ -0,0 +1,33 @@ +# client_streaming.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +# Example: Complex query with thinking process +response = client.chat.completions.create( + model="my-qwq-32B", + messages=[ + { + "role": "user", + "content": "I need to plan a trip to Paris from Seattle. Can you help me research flight costs, create an itinerary for 3 days, and suggest restaurants based on my dietary restrictions (vegetarian)?", + } + ], + stream=True, +) + +# Stream +for chunk in response: + # Stream reasoning content + if hasattr(chunk.choices[0].delta, "reasoning_content"): + data_reasoning = chunk.choices[0].delta.reasoning_content + if data_reasoning: + print(data_reasoning, end="", flush=True) + # Later, stream the final answer + if hasattr(chunk.choices[0].delta, "content"): + data_content = chunk.choices[0].delta.content + if data_content: + print(data_content, end="", flush=True) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb new file mode 100644 index 000000000000..938268090be5 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb @@ -0,0 +1,348 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c105c497", + "metadata": {}, + "source": [ + "# Deploy a reasoning LLM\n", + "\n", + "A reasoning LLM handles tasks that require deeper analysis or step-by-step thought. It generates intermediate reasoning before arriving at a final answer, making it better suited for situations where careful logic or structured problem-solving is more important than speed or efficiency.\n", + "\n", + "This tutorial deploys a reasoning LLM using Ray Serve LLM. \n", + "\n", + "---\n", + "\n", + "## Compare reasoning and non-reasoning models\n", + "\n", + "Reasoning models simulate step-by-step, structured thought processes to solve complex tasks like math, multi-hop QA, or code generation. In contrast, non-reasoning models provide fast, direct responses and focus on fluency or instruction following without explicit intermediate reasoning. The key distinction lies in whether the model attempts to \"think through\" the problem before answering.\n", + "\n", + "| **Model type** | **Core behavior** | **Use case examples** | **Limitation** |\n", + "| ----------------------- | ------------------------------------ | -------------------------------------------------------- | ----------------------------------------------------- |\n", + "| **Reasoning model** | Explicit multi-step thinking process | Math, coding, logic puzzles, multi-hop QA, CoT prompting | Slower response time, more tokens used. |\n", + "| **Non-reasoning model** | Direct answer generation | Casual queries, short instructions, single-step answers | May struggle with complex reasoning or interpretability. |\n", + "\n", + "Many reasoning-capable models structure their outputs with special markers such as `` tags, or expose reasoning traces inside dedicated fields like `reasoning_content` in the OpenAI API response. Always check the model's documentation for how to structure and control thinking.\n", + "\n", + "**Note:** Reasoning LLMs often benefit from long context windows (32K up to +1M tokens), high token throughput, low-temperature decoding (greedy sampling), and strong instruction tuning or scratchpad-style reasoning.\n", + "\n", + "---\n", + "\n", + "### Choose when to use reasoning models\n", + "\n", + "Whether you should use a reasoning model depends on how much information your prompt already provides.\n", + "\n", + "If your input is clear and complete, a standard model is usually faster and more efficient. If your input is ambiguous or complex, a reasoning model works better because it can work through the problem step by step and fill in gaps through intermediate reasoning.\n", + "\n", + "---\n", + "\n", + "## Parse reasoning outputs\n", + "\n", + "Reasoning models often separate *reasoning* from the *final answer* using tags like `...`. Without a proper parser, this reasoning may end up in the `content` field instead of the dedicated `reasoning_content` field.\n", + "\n", + "To extract reasoning correctly, configure a `reasoning_parser` in your Ray Serve deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output.\n", + "**Note:** For example, *QwQ* uses the `deepseek-r1` parser. Other models may require different parsers. See the [vLLM docs](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#supported-models) or your model's documentation to find a supported parser, or [build your own](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#how-to-support-a-new-reasoning-model) if needed.\n", + "\n", + "```yaml\n", + "applications:\n", + "- name: reasoning-llm-app\n", + " ...\n", + " args:\n", + " llm_configs:\n", + " - model_loading_config:\n", + " model_id: my-qwq-32B\n", + " model_source: Qwen/QwQ-32B\n", + " ...\n", + " engine_kwargs:\n", + " ...\n", + " reasoning_parser: deepseek_r1 # <-- for QwQ models\n", + "```\n", + "\n", + "See [Configure Ray Serve LLM](#configure-ray-serve-llm) for a complete example.\n", + "\n", + "**Example response** \n", + "When using a reasoning parser, the response is typically structured like this:\n", + "\n", + "```python\n", + "ChatCompletionMessage(\n", + " content=\"The temperature is...\",\n", + " ...,\n", + " reasoning_content=\"Okay, the user is asking for the temperature today and tomorrow...\"\n", + ")\n", + "```\n", + "And you can extract the content and reasoning as follows:\n", + "```python\n", + "response = client.chat.completions.create(\n", + " ...\n", + ")\n", + "\n", + "print(f\"Content: {response.choices[0].message.content}\")\n", + "print(f\"Reasoning: {response.choices[0].message.reasoning_content}\")\n", + "```\n", + "\n", + "---\n", + "\n", + "## Configure Ray Serve LLM\n", + "\n", + "Set your Hugging Face token in the config file to access gated models.\n", + "\n", + "Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object.\n", + "\n", + "Set `tensor_parallel_size=8` to distribute the model's weights among 8 GPUs in the node. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99ae0ed2", + "metadata": {}, + "outputs": [], + "source": [ + "# serve_qwq_32b.py\n", + "from ray.serve.llm import LLMConfig, build_openai_app\n", + "import os\n", + "\n", + "llm_config = LLMConfig(\n", + " model_loading_config=dict(\n", + " model_id=\"my-qwq-32B\",\n", + " model_source=\"Qwen/QwQ-32B\",\n", + " ),\n", + " accelerator_type=\"A100-40G\",\n", + " deployment_config=dict(\n", + " autoscaling_config=dict(\n", + " min_replicas=1,\n", + " max_replicas=2,\n", + " )\n", + " ),\n", + " ### Uncomment if your model is gated and needs your Hugging Face token to access it\n", + " # runtime_env=dict(env_vars={\"HF_TOKEN\": os.environ.get(\"HF_TOKEN\")}),\n", + " engine_kwargs=dict(\n", + " tensor_parallel_size=8, max_model_len=32768, reasoning_parser=\"deepseek_r1\"\n", + " ),\n", + ")\n", + "\n", + "app = build_openai_app({\"llm_configs\": [llm_config]})\n" + ] + }, + { + "cell_type": "markdown", + "id": "d515e268", + "metadata": {}, + "source": [ + "**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "\n", + "---\n", + "\n", + "## Deploy locally\n", + "\n", + "**Prerequisites**\n", + "\n", + "* Access to GPU compute.\n", + "* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`\n", + "\n", + "**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks.\n", + "\n", + "**Dependencies:** \n", + "```bash\n", + "pip install \"ray[serve,llm]\"\n", + "```\n", + "\n", + "---\n", + "\n", + "### Launch the service\n", + "\n", + "Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_qwq_32b.py`. \n", + "\n", + "In a terminal, run: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6d6a307", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve run serve_qwq_32b:app --non-blocking" + ] + }, + { + "cell_type": "markdown", + "id": "646f1272", + "metadata": {}, + "source": [ + "Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. \n", + "\n", + "---\n", + "\n", + "### Send requests\n", + "\n", + "Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `\"FAKE_KEY\"`.\n", + "\n", + "Example curl:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56a53387", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "curl -X POST http://localhost:8000/v1/chat/completions \\\n", + " -H \"Authorization: Bearer FAKE_KEY\" \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{ \\\n", + " \"model\": \"my-qwq-32B\", \\\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"Pick three random words with 3 syllables each and count the number of R'\\''s in each of them\"}] \\\n", + " }'" + ] + }, + { + "cell_type": "markdown", + "id": "942e675c", + "metadata": {}, + "source": [ + "Example python:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5005dde7", + "metadata": {}, + "outputs": [], + "source": [ + "#client.py\n", + "from urllib.parse import urljoin\n", + "from openai import OpenAI\n", + "\n", + "api_key = \"FAKE_KEY\"\n", + "base_url = \"http://localhost:8000\"\n", + "\n", + "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"my-qwq-32B\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"What is the sum of all even numbers between 1 and 100?\"}\n", + " ]\n", + ")\n", + "\n", + "print(f\"Reasoning: \\n{response.choices[0].message.reasoning_content}\\n\\n\")\n", + "print(f\"Answer: \\n {response.choices[0].message.content}\")" + ] + }, + { + "cell_type": "markdown", + "id": "5e04db4e", + "metadata": {}, + "source": [ + "If you configure a valid reasoning parser, the reasoning output should appear in the `reasoning_content` field of the response message. Otherwise, it may be included in the main `content` field, typically wrapped in `...` tags. See [Parse reasoning outputs](#parse-reasoning-outputs) for more information.\n", + "\n", + "---\n", + "\n", + "### Shutdown\n", + "\n", + "Shutdown your LLM service:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac1f3edd", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve shutdown -y" + ] + }, + { + "cell_type": "markdown", + "id": "fdc9e8eb", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "## Deploy to production with Anyscale services\n", + "\n", + "For production, use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium size LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *QwQ-32 B* used here.\n", + "\n", + "---\n", + "\n", + "## Stream reasoning content\n", + "\n", + "Reasoning models may take longer to begin generating the main content. You can stream their intermediate reasoning output in the same way as the main content. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02472f7c", + "metadata": {}, + "outputs": [], + "source": [ + "#client_streaming.py\n", + "from urllib.parse import urljoin\n", + "from openai import OpenAI\n", + "\n", + "api_key = \n", + "base_url = \n", + "\n", + "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "\n", + "# Example: Complex query with thinking process\n", + "response = client.chat.completions.create(\n", + " model=\"my-qwq-32B\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"I need to plan a trip to Paris from Seattle. Can you help me research flight costs, create an itinerary for 3 days, and suggest restaurants based on my dietary restrictions (vegetarian)?\"}\n", + " ],\n", + " stream=True\n", + ")\n", + "\n", + "# Stream\n", + "for chunk in response:\n", + " # Stream reasoning content\n", + " if hasattr(chunk.choices[0].delta, \"reasoning_content\"):\n", + " data_reasoning = chunk.choices[0].delta.reasoning_content\n", + " if data_reasoning:\n", + " print(data_reasoning, end=\"\", flush=True)\n", + " # Later, stream the final answer\n", + " if hasattr(chunk.choices[0].delta, \"content\"):\n", + " data_content = chunk.choices[0].delta.content\n", + " if data_content:\n", + " print(data_content, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "id": "70455ea2", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "## Summary\n", + "\n", + "In this tutorial, you deployed a reasoning LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM with the right reasoning parser, deploy your service on your Ray cluster, send requests, and parse reasoning outputs in the response." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "repo_ray_docs", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/serve_qwq_32b.py b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/serve_qwq_32b.py new file mode 100644 index 000000000000..8c8cda59f8a5 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/serve_qwq_32b.py @@ -0,0 +1,24 @@ +# serve_qwq_32b.py +from ray.serve.llm import LLMConfig, build_openai_app +import os + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-qwq-32B", + model_source="Qwen/QwQ-32B", + ), + accelerator_type="A100-40G", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=2, + ) + ), + ### Uncomment if your model is gated and needs your Hugging Face token to access it + # runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict( + tensor_parallel_size=8, max_model_len=32768, reasoning_parser="deepseek_r1" + ), +) + +app = build_openai_app({"llm_configs": [llm_config]}) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/Dockerfile b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/Dockerfile new file mode 100644 index 000000000000..a2412390df61 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/Dockerfile @@ -0,0 +1,8 @@ +FROM anyscale/ray:2.49.0-slim-py312-cu128 + +# C compiler for Triton’s runtime build step (vLLM V1 engine) +# https://github.com/vllm-project/vllm/issues/2997 +RUN sudo apt-get update && \ + sudo apt-get install -y --no-install-recommends build-essential + +RUN pip install vllm==0.10.0 \ No newline at end of file diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md new file mode 100644 index 000000000000..503abeb49e94 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md @@ -0,0 +1,315 @@ +--- +orphan: true +--- + + + +# Deploy a small size LLM + +A small LLM runs on a single node with 1–2 GPUs, making it fast, inexpensive, and simple to use. It’s ideal for prototyping, lightweight applications, latency-critical use cases, cost-sensitive deployments, and environments with limited resources where efficiency matters more than peak accuracy. + + +For larger models, see [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html) or [Deploying a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html). + +--- + +## Configure Ray Serve LLM + +Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object. + +Set your Hugging Face token in the config file to access gated models like `Llama-3.1`. + + +```python +# serve_llama_3_1_8b.py +from ray.serve.llm import LLMConfig, build_openai_app +import os + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-llama-3.1-8b", + # Or unsloth/Meta-Llama-3.1-8B-Instruct for an ungated model + model_source="meta-llama/Llama-3.1-8B-Instruct", + ), + accelerator_type="L4", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=2, + ) + ), + ### If your model isn't gated, you can skip `hf_token` + # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3 + # Type `export HF_TOKEN=` in a terminal + runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict(max_model_len=8192), +) +app = build_openai_app({"llm_configs": [llm_config]}) + +``` + +**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. + +--- + +## Deploy locally + + +**Prerequisites** + +* Access to GPU compute. +* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`. + +**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks. + +**Dependencies:** +```bash +pip install "ray[serve,llm]" +``` + +--- + +### Launch + +Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_llama_3_1_8b.py`. + +In a terminal, run: + + +```bash +%%bash +serve run serve_llama_3_1_8b:app --non-blocking +``` + +Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. + +--- + +### Send requests + +Your endpoint is available locally at `http://localhost:8000`. You can use a placeholder authentication token for the OpenAI client, for example `"FAKE_KEY"`. + +**Example curl:** + + +```bash +%%bash +curl -X POST http://localhost:8000/v1/chat/completions \ + -H "Authorization: Bearer FAKE_KEY" \ + -H "Content-Type: application/json" \ + -d '{ \ + "model": "my-llama-3.1-8b", \ + "messages": [{"role": "user", "content": "What is 2 + 2?"}] \ + }' +``` + +**Example Python:** + + +```python +#client.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +response = client.chat.completions.create( + model="my-llama-3.1-8b", + messages=[{"role": "user", "content": "Tell me a joke"}], + stream=True +) + +for chunk in response: + content = chunk.choices[0].delta.content + if content: + print(content, end="", flush=True) +``` + + +--- + +### Shutdown + +Shutdown your LLM service: + + +```bash +%%bash +serve shutdown -y +``` + + +--- + +## Deploy to production with Anyscale Services + +For production deployment, use Anyscale Services to deploy the Ray Serve app to a dedicated cluster without modifying the code. Anyscale ensures scalability, fault tolerance, and load balancing, keeping the service resilient against node failures, high traffic, and rolling updates. + +--- + +### Launch the service + +Anyscale provides out-of-the-box images (`anyscale/ray-llm`) which come pre-loaded with Ray Serve LLM, vLLM, and all required GPU/runtime dependencies. This makes it easy to get started without building a custom image. + +Create your Anyscale Service configuration in a new `service.yaml` file: + +```yaml +# service.yaml +name: deploy-llama-3-8b +image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image +compute_config: + auto_select_worker_config: true +working_dir: . +cloud: +applications: + # Point to your app in your Python module + - import_path: serve_llama_3_1_8b:app +``` + + +Deploy your service with the following command. Make sure to forward your Hugging Face token: + + +```bash +%%bash +anyscale service deploy -f service.yaml --env HF_TOKEN= +``` + +**Custom Dockerfile** +You can customize the container by building your own Dockerfile. In your Anyscale Service config, reference the Dockerfile with `containerfile` (instead of `image_uri`): + +```yaml +# service.yaml +# Replace: +# image_uri: anyscale/ray-llm:2.49.0-py311-cu128 + +# with: +containerfile: ./Dockerfile +``` + +See the [Anyscale base images](https://docs.anyscale.com/reference/base-images) for details on what each image includes. + +--- + +### Send requests + +The `anyscale service deploy` command output shows both the endpoint and authentication token: +```console +(anyscale +3.9s) curl -H "Authorization: Bearer " +``` +You can also retrieve both from the service page in the Anyscale Console. Click the **Query** button at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token. + +--- + +### Access the Serve LLM dashboard + +See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling LLM-specific logging. To open the Ray Serve LLM Dashboard from an Anyscale Service: +1. In the Anyscale console, go to your **Service** or **Workspace**. +2. Navigate to the **Metrics** tab. +3. Expand **View in Grafana** and click **Serve LLM Dashboard**. + +--- + +### Shutdown + +Shutdown your Anyscale Service: + + +```bash +%%bash +anyscale service terminate -n deploy-llama-3-8b +``` + + +--- + +## Enable LLM monitoring + +The *Serve LLM Dashboard* offers deep visibility into model performance, latency, and system behavior, including: + +- Token throughput (tokens/sec). +- Latency metrics: Time To First Token (TTFT), Time Per Output Token (TPOT). +- KV cache utilization. + +To enable these metrics, go to your LLM config and set `log_engine_metrics: true`. Ensure vLLM V1 is active with `VLLM_USE_V1: "1"`. + +**Note:** `VLLM_USE_V1: "1"` is the default value with `ray >= 2.48.0` and can be omitted. +```yaml +applications: +- ... + args: + llm_configs: + - ... + runtime_env: + env_vars: + VLLM_USE_V1: "1" + ... + log_engine_metrics: true +``` + +--- + +## Improve concurrency + +Ray Serve LLM uses [vLLM](https://docs.vllm.ai/en/stable/) as its backend engine, which logs the *maximum concurrency* it can support based on your configuration. + +Example log: +```console +INFO 08-06 20:15:53 [executor_base.py:118] Maximum concurrency for 8192 tokens per request: 3.53x +``` + +You can improve concurrency depending on your model and hardware in several ways: + +**Reduce `max_model_len`** +Lowering `max_model_len` reduces the memory needed for KV cache. + +**Example:** Running *llama-3.1-8 B* on an A10G or L4 GPU: +- `max_model_len = 8192` → concurrency ≈ 3.5 +- `max_model_len = 4096` → concurrency ≈ 7 + +**Use Quantized Models** +Quantizing your model (for example, to FP8) reduces the model's memory footprint, freeing up memory for more KV cache and enabling more concurrent requests. + +**Use Tensor Parallelism** +Distribute the model across multiple GPUs with `tensor_parallel_size > 1`. + +**Note:** Latency may rise if GPUs don’t have strong GPU interconnect like NVLink. + +**Upgrade to GPUs with more memory** +Some GPUs provide significantly more room for KV cache and allow for higher concurrency out of the box. + +**Scale with more Replicas** +In addition to tuning per-GPU concurrency, you can scale *horizontally* by increasing the number of replicas in your config. +Each replica runs on its own GPU, so raising the replica count increases the total number of concurrent requests your service can handle, especially under sustained or bursty traffic. +```yaml +deployment_config: + autoscaling_config: + min_replicas: 1 + max_replicas: 4 +``` + +*For more details on tuning strategies, hardware guidance, and serving configurations, see [Choose a GPU for LLM serving](https://docs.anyscale.com/llm/serving/gpu-guidance) and [Tune parameters for LLMs on Anyscale services](https://docs.anyscale.com/llm/serving/parameter-tuning).* + +--- + +## Troubleshooting + +**Hugging Face authentication errors** +Some models, such as Llama-3.1, are gated and require prior authorization from the organization. See your model’s documentation for instructions on obtaining access. + +**Out-of-memory errors** +Out-of-memory (OOM) errors are one of the most common failure modes when deploying LLMs, especially as model sizes and context length increase. +See this [Troubleshooting Guide](https://docs.anyscale.com/overview) for common errors and how to fix them. + +--- + +## Summary + +In this tutorial, you deployed a small size LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and how to send requests. You also learned how to monitor your app and common troubleshooting issues. diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/client.py b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/client.py new file mode 100644 index 000000000000..25025a73dbcf --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/client.py @@ -0,0 +1,18 @@ +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +response = client.chat.completions.create( + model="my-llama-3.1-8b", + messages=[{"role": "user", "content": "Tell me a joke"}], + stream=True, +) + +for chunk in response: + content = chunk.choices[0].delta.content + if content: + print(content, end="", flush=True) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb new file mode 100644 index 000000000000..09dfb2206b82 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb @@ -0,0 +1,407 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6a51548b", + "metadata": {}, + "source": [ + "# Deploy a small size LLM\n", + "\n", + "A small LLM runs on a single node with 1–2 GPUs, making it fast, inexpensive, and simple to use. It’s ideal for prototyping, lightweight applications, latency-critical use cases, cost-sensitive deployments, and environments with limited resources where efficiency matters more than peak accuracy.\n", + "\n", + "\n", + "For larger models, see [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html) or [Deploying a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html).\n", + "\n", + "---\n", + "\n", + "## Configure Ray Serve LLM\n", + "\n", + "Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object.\n", + "\n", + "Set your Hugging Face token in the config file to access gated models like `Llama-3.1`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e555ca3f", + "metadata": {}, + "outputs": [], + "source": [ + "# serve_llama_3_1_8b.py\n", + "from ray.serve.llm import LLMConfig, build_openai_app\n", + "import os\n", + "\n", + "llm_config = LLMConfig(\n", + " model_loading_config=dict(\n", + " model_id=\"my-llama-3.1-8b\",\n", + " # Or unsloth/Meta-Llama-3.1-8B-Instruct for an ungated model\n", + " model_source=\"meta-llama/Llama-3.1-8B-Instruct\",\n", + " ),\n", + " accelerator_type=\"L4\",\n", + " deployment_config=dict(\n", + " autoscaling_config=dict(\n", + " min_replicas=1,\n", + " max_replicas=2,\n", + " )\n", + " ),\n", + " ### If your model isn't gated, you can skip `hf_token`\n", + " # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3\n", + " # Type `export HF_TOKEN=` in a terminal\n", + " runtime_env=dict(env_vars={\"HF_TOKEN\": os.environ.get(\"HF_TOKEN\")}),\n", + " engine_kwargs=dict(max_model_len=8192),\n", + ")\n", + "app = build_openai_app({\"llm_configs\": [llm_config]})\n" + ] + }, + { + "cell_type": "markdown", + "id": "b17a7140", + "metadata": {}, + "source": [ + "**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "\n", + "---\n", + "\n", + "## Deploy locally\n", + "\n", + "\n", + "**Prerequisites**\n", + "\n", + "* Access to GPU compute.\n", + "* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`.\n", + "\n", + "**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks.\n", + "\n", + "**Dependencies:** \n", + "```bash\n", + "pip install \"ray[serve,llm]\"\n", + "```\n", + "\n", + "---\n", + "\n", + "### Launch\n", + "\n", + "Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_llama_3_1_8b.py`. \n", + "\n", + "In a terminal, run: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dbdb0921", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve run serve_llama_3_1_8b:app --non-blocking" + ] + }, + { + "cell_type": "markdown", + "id": "df944967", + "metadata": {}, + "source": [ + "Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. \n", + "\n", + "---\n", + "\n", + "### Send requests\n", + "\n", + "Your endpoint is available locally at `http://localhost:8000`. You can use a placeholder authentication token for the OpenAI client, for example `\"FAKE_KEY\"`.\n", + "\n", + "**Example curl:**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5309437", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "curl -X POST http://localhost:8000/v1/chat/completions \\\n", + " -H \"Authorization: Bearer FAKE_KEY\" \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{ \\\n", + " \"model\": \"my-llama-3.1-8b\", \\\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"What is 2 + 2?\"}] \\\n", + " }'" + ] + }, + { + "cell_type": "markdown", + "id": "d623a30f", + "metadata": {}, + "source": [ + "**Example Python:**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "75bedc22", + "metadata": {}, + "outputs": [], + "source": [ + "#client.py\n", + "from urllib.parse import urljoin\n", + "from openai import OpenAI\n", + "\n", + "api_key = \"FAKE_KEY\"\n", + "base_url = \"http://localhost:8000\"\n", + "\n", + "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"my-llama-3.1-8b\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Tell me a joke\"}],\n", + " stream=True\n", + ")\n", + "\n", + "for chunk in response:\n", + " content = chunk.choices[0].delta.content\n", + " if content:\n", + " print(content, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "id": "b095ebf3", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "### Shutdown\n", + "\n", + "Shutdown your LLM service: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4fd3dacf", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve shutdown -y" + ] + }, + { + "cell_type": "markdown", + "id": "fb81fa41", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "## Deploy to production with Anyscale Services\n", + "\n", + "For production deployment, use Anyscale Services to deploy the Ray Serve app to a dedicated cluster without modifying the code. Anyscale ensures scalability, fault tolerance, and load balancing, keeping the service resilient against node failures, high traffic, and rolling updates.\n", + "\n", + "---\n", + "\n", + "### Launch the service\n", + "\n", + "Anyscale provides out-of-the-box images (`anyscale/ray-llm`) which come pre-loaded with Ray Serve LLM, vLLM, and all required GPU/runtime dependencies. This makes it easy to get started without building a custom image.\n", + "\n", + "Create your Anyscale Service configuration in a new `service.yaml` file:\n", + "\n", + "```yaml\n", + "# service.yaml\n", + "name: deploy-llama-3-8b\n", + "image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image\n", + "compute_config:\n", + " auto_select_worker_config: true \n", + "working_dir: .\n", + "cloud:\n", + "applications:\n", + " # Point to your app in your Python module\n", + " - import_path: serve_llama_3_1_8b:app\n", + "```\n", + "\n", + "\n", + "Deploy your service with the following command. Make sure to forward your Hugging Face token:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "66b3b53a", + "metadata": { + "pygments_lexer": "bash" + }, + "outputs": [], + "source": [ + "%%bash\n", + "anyscale service deploy -f service.yaml --env HF_TOKEN=" + ] + }, + { + "cell_type": "markdown", + "id": "7e6de36c", + "metadata": {}, + "source": [ + "**Custom Dockerfile** \n", + "You can customize the container by building your own Dockerfile. In your Anyscale Service config, reference the Dockerfile with `containerfile` (instead of `image_uri`):\n", + "\n", + "```yaml\n", + "# service.yaml\n", + "# Replace:\n", + "# image_uri: anyscale/ray-llm:2.49.0-py311-cu128\n", + "\n", + "# with:\n", + "containerfile: ./Dockerfile\n", + "```\n", + "\n", + "See the [Anyscale base images](https://docs.anyscale.com/reference/base-images) for details on what each image includes.\n", + "\n", + "---\n", + "\n", + "### Send requests \n", + "\n", + "The `anyscale service deploy` command output shows both the endpoint and authentication token:\n", + "```console\n", + "(anyscale +3.9s) curl -H \"Authorization: Bearer \" \n", + "```\n", + "You can also retrieve both from the service page in the Anyscale Console. Click the **Query** button at the top. See [Send requests](#send-requests) for example requests, but make sure to use the correct endpoint and authentication token. \n", + "\n", + "---\n", + "\n", + "### Access the Serve LLM dashboard\n", + "\n", + "See [Enable LLM monitoring](#enable-llm-monitoring) for instructions on enabling LLM-specific logging. To open the Ray Serve LLM Dashboard from an Anyscale Service:\n", + "1. In the Anyscale console, go to your **Service** or **Workspace**.\n", + "2. Navigate to the **Metrics** tab.\n", + "3. Expand **View in Grafana** and click **Serve LLM Dashboard**.\n", + "\n", + "---\n", + "\n", + "### Shutdown\n", + "\n", + "Shutdown your Anyscale Service:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "474b2764", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "anyscale service terminate -n deploy-llama-3-8b" + ] + }, + { + "cell_type": "markdown", + "id": "49f67c39", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "## Enable LLM monitoring\n", + "\n", + "The *Serve LLM Dashboard* offers deep visibility into model performance, latency, and system behavior, including:\n", + "\n", + "- Token throughput (tokens/sec).\n", + "- Latency metrics: Time To First Token (TTFT), Time Per Output Token (TPOT).\n", + "- KV cache utilization.\n", + "\n", + "To enable these metrics, go to your LLM config and set `log_engine_metrics: true`. Ensure vLLM V1 is active with `VLLM_USE_V1: \"1\"`.\n", + "\n", + "**Note:** `VLLM_USE_V1: \"1\"` is the default value with `ray >= 2.48.0` and can be omitted.\n", + "```yaml\n", + "applications:\n", + "- ...\n", + " args:\n", + " llm_configs:\n", + " - ...\n", + " runtime_env:\n", + " env_vars:\n", + " VLLM_USE_V1: \"1\"\n", + " ...\n", + " log_engine_metrics: true\n", + "```\n", + "\n", + "---\n", + "\n", + "## Improve concurrency\n", + "\n", + "Ray Serve LLM uses [vLLM](https://docs.vllm.ai/en/stable/) as its backend engine, which logs the *maximum concurrency* it can support based on your configuration.\n", + "\n", + "Example log:\n", + "```console\n", + "INFO 08-06 20:15:53 [executor_base.py:118] Maximum concurrency for 8192 tokens per request: 3.53x\n", + "```\n", + "\n", + "You can improve concurrency depending on your model and hardware in several ways: \n", + "\n", + "**Reduce `max_model_len`** \n", + "Lowering `max_model_len` reduces the memory needed for KV cache.\n", + "\n", + "**Example:** Running *llama-3.1-8 B* on an A10G or L4 GPU:\n", + "- `max_model_len = 8192` → concurrency ≈ 3.5\n", + "- `max_model_len = 4096` → concurrency ≈ 7\n", + "\n", + "**Use Quantized Models** \n", + "Quantizing your model (for example, to FP8) reduces the model's memory footprint, freeing up memory for more KV cache and enabling more concurrent requests.\n", + "\n", + "**Use Tensor Parallelism** \n", + "Distribute the model across multiple GPUs with `tensor_parallel_size > 1`.\n", + "\n", + "**Note:** Latency may rise if GPUs don’t have strong GPU interconnect like NVLink.\n", + "\n", + "**Upgrade to GPUs with more memory** \n", + "Some GPUs provide significantly more room for KV cache and allow for higher concurrency out of the box.\n", + "\n", + "**Scale with more Replicas** \n", + "In addition to tuning per-GPU concurrency, you can scale *horizontally* by increasing the number of replicas in your config. \n", + "Each replica runs on its own GPU, so raising the replica count increases the total number of concurrent requests your service can handle, especially under sustained or bursty traffic.\n", + "```yaml\n", + "deployment_config:\n", + " autoscaling_config:\n", + " min_replicas: 1\n", + " max_replicas: 4\n", + "```\n", + "\n", + "*For more details on tuning strategies, hardware guidance, and serving configurations, see [Choose a GPU for LLM serving](https://docs.anyscale.com/llm/serving/gpu-guidance) and [Tune parameters for LLMs on Anyscale services](https://docs.anyscale.com/llm/serving/parameter-tuning).*\n", + "\n", + "---\n", + "\n", + "## Troubleshooting\n", + "\n", + "**Hugging Face authentication errors** \n", + "Some models, such as Llama-3.1, are gated and require prior authorization from the organization. See your model’s documentation for instructions on obtaining access.\n", + "\n", + "**Out-of-memory errors** \n", + "Out-of-memory (OOM) errors are one of the most common failure modes when deploying LLMs, especially as model sizes and context length increase. \n", + "See this [Troubleshooting Guide](https://docs.anyscale.com/overview) for common errors and how to fix them.\n", + "\n", + "---\n", + "\n", + "## Summary\n", + "\n", + "In this tutorial, you deployed a small size LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and how to send requests. You also learned how to monitor your app and common troubleshooting issues." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "repo_ray_docs", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/serve_llama_3_1_8b.py b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/serve_llama_3_1_8b.py new file mode 100644 index 000000000000..c5ed01f7b304 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/serve_llama_3_1_8b.py @@ -0,0 +1,24 @@ +# serve_llama_3_1_8b.py +from ray.serve.llm import LLMConfig, build_openai_app +import os + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-llama-3.1-8b", + # Or unsloth/Meta-Llama-3.1-8B-Instruct for an ungated model + model_source="meta-llama/Llama-3.1-8B-Instruct", + ), + accelerator_type="L4", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=2, + ) + ), + ### If your model isn't gated, you can skip `hf_token` + # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3 + # Type `export HF_TOKEN=` in a terminal + runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict(max_model_len=8192), +) +app = build_openai_app({"llm_configs": [llm_config]}) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/service.yaml b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/service.yaml new file mode 100644 index 000000000000..98d3ed37539b --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/service.yaml @@ -0,0 +1,10 @@ +#service.yaml +name: deploy-llama-3-8b +containerfile: ./Dockerfile +compute_config: + auto_select_worker_config: true +working_dir: . +cloud: +applications: +# Point to your app in your Python module +- import_path: serve_llama_3_1_8b:app \ No newline at end of file diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md new file mode 100644 index 000000000000..050f7e32904b --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md @@ -0,0 +1,232 @@ +--- +orphan: true +--- + + + +# Deploying a vision LLM + +A vision LLM can interpret images as well as text, enabling tasks like answering questions about charts, analyzing photos, or combining visuals with instructions. It extends LLMs beyond language to support multimodal reasoning and richer applications. + +This tutorial deploys a vision LLM using Ray Serve LLM. + +--- + +## Configure Ray Serve LLM + +Make sure to set your Hugging Face token in the config file to access gated models. + +Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object. + + +```python +# serve_qwen_VL.py +from ray.serve.llm import LLMConfig, build_openai_app +import os + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-qwen-VL", + model_source="qwen/Qwen2.5-VL-7B-Instruct", + ), + accelerator_type="L40S", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=2, + ) + ), + ### Uncomment if your model is gated and needs your Hugging Face token to access it. + # runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict(max_model_len=8192), +) + +app = build_openai_app({"llm_configs": [llm_config]}) + +``` + +**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. + +--- + +## Deploy locally + +**Prerequisites** + +* Access to GPU compute. +* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=` + +**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks. + +**Dependencies:** +```bash +pip install "ray[serve,llm]" +``` + +--- + +### Launch + +Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_qwen_VL.py`. + +In a terminal, run: + + +```bash +%%bash +serve run serve_qwen_VL:app --non-blocking +``` + +Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. + +--- + +### Sending requests with images + +Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `"FAKE_KEY"`. + +Example cURL with image URL + + +```bash +%%bash +curl -X POST http://localhost:8000/v1/chat/completions \ + -H "Authorization: Bearer FAKE_KEY" \ + -H "Content-Type: application/json" \ + -d '{ \ + "model": "my-qwen-VL", \ + "messages": [ \ + { \ + "role": "user", \ + "content": [ \ + {"type": "text", "text": "What do you see in this image?"}, \ + {"type": "image_url", "image_url": { \ + "url": "http://images.cocodataset.org/val2017/000000039769.jpg" \ + }} \ + ] \ + } \ + ] \ + }' +``` + +Example Python with image URL: + + +```python +#client_url_image.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +response = client.chat.completions.create( + model="my-qwen-VL", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"url": "http://images.cocodataset.org/val2017/000000039769.jpg"}} + ] + } + ], + temperature=0.5, + stream=True +) + +for chunk in response: + content = chunk.choices[0].delta.content + if content: + print(content, end="", flush=True) +``` + +Example Python with local image: + + +```python +#client_local_image.py +from urllib.parse import urljoin +import base64 +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +### From an image locally saved as `example.jpg` +# Load and encode image as base64 +with open("example.jpg", "rb") as f: + img_base64 = base64.b64encode(f.read()).decode() + +response = client.chat.completions.create( + model="my-qwen-VL", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_base64}"}} + ] + } + ], + temperature=0.5, + stream=True +) + +for chunk in response: + content = chunk.choices[0].delta.content + if content: + print(content, end="", flush=True) +``` + + +--- + +### Shutdown + +Shutdown your LLM service: + + +```bash +%%bash +serve shutdown -y +``` + + +--- + +## Deploy to production with Anyscale services + +For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a small-sized model like the *Qwen2.5-VL-7 B-Instruct* used in this tutorial. + +--- + +## Limiting images per prompt + +Ray Serve LLM uses [vLLM](https://docs.vllm.ai/en/stable/) as its backend engine. You can configure vLLM by passing parameters through the `engine_kwargs` section of your Serve LLM configuration. For a full list of supported options, see the [vLLM documentation](https://docs.vllm.ai/en/stable/configuration/engine_args.html#multimodalconfig). + +In particular, you can limit the number of images per request by setting `limit_mm_per_prompt` in your configuration. +```yaml +applications: +- ... + args: + llm_configs: + ... + engine_kwargs: + ... + limit_mm_per_prompt: {"image": 3} +``` + +--- + +## Summary + +In this tutorial, you deployed a vision LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and send requests with images. diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py new file mode 100644 index 000000000000..14cfcb2c9f0a --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py @@ -0,0 +1,37 @@ +# client_local_image.py +from urllib.parse import urljoin +import base64 +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +### From an image locally saved as `example.jpg` +# Load and encode image as base64 +with open("example.jpg", "rb") as f: + img_base64 = base64.b64encode(f.read()).decode() + +response = client.chat.completions.create( + model="my-qwen-VL", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{img_base64}"}, + }, + ], + } + ], + temperature=0.5, + stream=True, +) + +for chunk in response: + content = chunk.choices[0].delta.content + if content: + print(content, end="", flush=True) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_url_image.py b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_url_image.py new file mode 100644 index 000000000000..c976e460f2dc --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_url_image.py @@ -0,0 +1,33 @@ +# client_url_image.py +from urllib.parse import urljoin +from openai import OpenAI + +api_key = "FAKE_KEY" +base_url = "http://localhost:8000" + +client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) + +response = client.chat.completions.create( + model="my-qwen-VL", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "http://images.cocodataset.org/val2017/000000039769.jpg" + }, + }, + ], + } + ], + temperature=0.5, + stream=True, +) + +for chunk in response: + content = chunk.choices[0].delta.content + if content: + print(content, end="", flush=True) diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb new file mode 100644 index 000000000000..e978985f0aa0 --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb @@ -0,0 +1,311 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "23243c2e", + "metadata": {}, + "source": [ + "# Deploying a vision LLM\n", + "\n", + "A vision LLM can interpret images as well as text, enabling tasks like answering questions about charts, analyzing photos, or combining visuals with instructions. It extends LLMs beyond language to support multimodal reasoning and richer applications. \n", + "\n", + "This tutorial deploys a vision LLM using Ray Serve LLM. \n", + "\n", + "---\n", + "\n", + "## Configure Ray Serve LLM\n", + "\n", + "Make sure to set your Hugging Face token in the config file to access gated models.\n", + "\n", + "Ray Serve LLM provides multiple [Python APIs](https://docs.ray.io/en/latest/serve/api/index.html#llm-api) for defining your application. Use [`build_openai_app`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.build_openai_app.html#ray.serve.llm.build_openai_app) to build a full application from your [`LLMConfig`](https://docs.ray.io/en/latest/serve/api/doc/ray.serve.llm.LLMConfig.html#ray.serve.llm.LLMConfig) object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ebc41d60", + "metadata": {}, + "outputs": [], + "source": [ + "# serve_qwen_VL.py\n", + "from ray.serve.llm import LLMConfig, build_openai_app\n", + "import os\n", + "\n", + "llm_config = LLMConfig(\n", + " model_loading_config=dict(\n", + " model_id=\"my-qwen-VL\",\n", + " model_source=\"qwen/Qwen2.5-VL-7B-Instruct\",\n", + " ),\n", + " accelerator_type=\"L40S\",\n", + " deployment_config=dict(\n", + " autoscaling_config=dict(\n", + " min_replicas=1,\n", + " max_replicas=2,\n", + " )\n", + " ),\n", + " ### Uncomment if your model is gated and needs your Hugging Face token to access it.\n", + " # runtime_env=dict(env_vars={\"HF_TOKEN\": os.environ.get(\"HF_TOKEN\")}),\n", + " engine_kwargs=dict(max_model_len=8192),\n", + ")\n", + "\n", + "app = build_openai_app({\"llm_configs\": [llm_config]})\n" + ] + }, + { + "cell_type": "markdown", + "id": "c76a6362", + "metadata": {}, + "source": [ + "**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "\n", + "---\n", + "\n", + "## Deploy locally\n", + "\n", + "**Prerequisites**\n", + "\n", + "* Access to GPU compute.\n", + "* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`\n", + "\n", + "**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks.\n", + "\n", + "**Dependencies:** \n", + "```bash\n", + "pip install \"ray[serve,llm]\"\n", + "```\n", + "\n", + "---\n", + "\n", + "### Launch\n", + "\n", + "Follow the instructions at [Configure Ray Serve LLM](#configure-ray-serve-llm) to define your app in a Python module `serve_qwen_VL.py`. \n", + "\n", + "In a terminal, run: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7eb8734c", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve run serve_qwen_VL:app --non-blocking" + ] + }, + { + "cell_type": "markdown", + "id": "d36f41d1", + "metadata": {}, + "source": [ + "Deployment typically takes a few minutes as the cluster is provisioned, the vLLM server starts, and the model is downloaded. \n", + "\n", + "---\n", + "\n", + "### Sending requests with images\n", + "\n", + "Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `\"FAKE_KEY\"`.\n", + "\n", + "Example cURL with image URL" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "400e7790", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "curl -X POST http://localhost:8000/v1/chat/completions \\\n", + " -H \"Authorization: Bearer FAKE_KEY\" \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{ \\\n", + " \"model\": \"my-qwen-VL\", \\\n", + " \"messages\": [ \\\n", + " { \\\n", + " \"role\": \"user\", \\\n", + " \"content\": [ \\\n", + " {\"type\": \"text\", \"text\": \"What do you see in this image?\"}, \\\n", + " {\"type\": \"image_url\", \"image_url\": { \\\n", + " \"url\": \"http://images.cocodataset.org/val2017/000000039769.jpg\" \\\n", + " }} \\\n", + " ] \\\n", + " } \\\n", + " ] \\\n", + " }'" + ] + }, + { + "cell_type": "markdown", + "id": "291743a5", + "metadata": {}, + "source": [ + "Example Python with image URL:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b447094", + "metadata": {}, + "outputs": [], + "source": [ + "#client_url_image.py\n", + "from urllib.parse import urljoin\n", + "from openai import OpenAI\n", + "\n", + "api_key = \"FAKE_KEY\"\n", + "base_url = \"http://localhost:8000\"\n", + "\n", + "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"my-qwen-VL\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": \"What is in this image?\"},\n", + " {\"type\": \"image_url\", \"image_url\": {\"url\": \"http://images.cocodataset.org/val2017/000000039769.jpg\"}}\n", + " ]\n", + " }\n", + " ],\n", + " temperature=0.5,\n", + " stream=True\n", + ")\n", + "\n", + "for chunk in response:\n", + " content = chunk.choices[0].delta.content\n", + " if content:\n", + " print(content, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "id": "811f1d41", + "metadata": {}, + "source": [ + "Example Python with local image:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8296023b", + "metadata": {}, + "outputs": [], + "source": [ + "#client_local_image.py\n", + "from urllib.parse import urljoin\n", + "import base64\n", + "from openai import OpenAI\n", + "\n", + "api_key = \"FAKE_KEY\"\n", + "base_url = \"http://localhost:8000\"\n", + "\n", + "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "\n", + "### From an image locally saved as `example.jpg`\n", + "# Load and encode image as base64\n", + "with open(\"example.jpg\", \"rb\") as f:\n", + " img_base64 = base64.b64encode(f.read()).decode()\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"my-qwen-VL\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": \"What is in this image?\"},\n", + " {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:image/jpeg;base64,{img_base64}\"}}\n", + " ]\n", + " }\n", + " ],\n", + " temperature=0.5,\n", + " stream=True\n", + ")\n", + "\n", + "for chunk in response:\n", + " content = chunk.choices[0].delta.content\n", + " if content:\n", + " print(content, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "id": "ccc60c1f", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "### Shutdown \n", + "\n", + "Shutdown your LLM service:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ee4b879", + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "serve shutdown -y" + ] + }, + { + "cell_type": "markdown", + "id": "a94c0307", + "metadata": {}, + "source": [ + "\n", + "---\n", + "\n", + "## Deploy to production with Anyscale services\n", + "\n", + "For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a small-sized model like the *Qwen2.5-VL-7 B-Instruct* used in this tutorial.\n", + "\n", + "---\n", + "\n", + "## Limiting images per prompt\n", + "\n", + "Ray Serve LLM uses [vLLM](https://docs.vllm.ai/en/stable/) as its backend engine. You can configure vLLM by passing parameters through the `engine_kwargs` section of your Serve LLM configuration. For a full list of supported options, see the [vLLM documentation](https://docs.vllm.ai/en/stable/configuration/engine_args.html#multimodalconfig). \n", + "\n", + "In particular, you can limit the number of images per request by setting `limit_mm_per_prompt` in your configuration. \n", + "```yaml\n", + "applications:\n", + "- ...\n", + " args:\n", + " llm_configs:\n", + " ...\n", + " engine_kwargs:\n", + " ...\n", + " limit_mm_per_prompt: {\"image\": 3}\n", + "```\n", + "\n", + "---\n", + "\n", + "## Summary\n", + "\n", + "In this tutorial, you deployed a vision LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and send requests with images." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "repo_ray_docs", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/serve_qwen_VL.py b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/serve_qwen_VL.py new file mode 100644 index 000000000000..d1239439700a --- /dev/null +++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/serve_qwen_VL.py @@ -0,0 +1,22 @@ +# serve_qwen_VL.py +from ray.serve.llm import LLMConfig, build_openai_app +import os + +llm_config = LLMConfig( + model_loading_config=dict( + model_id="my-qwen-VL", + model_source="qwen/Qwen2.5-VL-7B-Instruct", + ), + accelerator_type="L40S", + deployment_config=dict( + autoscaling_config=dict( + min_replicas=1, + max_replicas=2, + ) + ), + ### Uncomment if your model is gated and needs your Hugging Face token to access it. + # runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), + engine_kwargs=dict(max_model_len=8192), +) + +app = build_openai_app({"llm_configs": [llm_config]}) diff --git a/release/BUILD.bazel b/release/BUILD.bazel index 79c3720d6e9b..705d230a9c32 100644 --- a/release/BUILD.bazel +++ b/release/BUILD.bazel @@ -309,6 +309,7 @@ py_library( ]) + [ "ray_release/buildkite/aws_instance_types.csv", "ray_release/schema.json", + "//doc:deployment_serve_llm_example_configs", "//doc:example_configs", "//doc/source/train/examples/pytorch/distributing-pytorch/ci:ci_yamls", ], diff --git a/release/ray_release/byod/byod_deployment_serve_llm.sh b/release/ray_release/byod/byod_deployment_serve_llm.sh new file mode 100755 index 000000000000..ef7e19de90b6 --- /dev/null +++ b/release/ray_release/byod/byod_deployment_serve_llm.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +set -exo pipefail diff --git a/release/release_tests.yaml b/release/release_tests.yaml index b124ef316c81..1c7932f46d7d 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -4791,6 +4791,33 @@ cluster: cluster_compute: ci/gce.yaml # relative to working_dir + +- name: deployment_serve_llm # do not use dashes (regex sensitive) + frequency: weekly + python: "3.11" + group: ray-examples + team: ml + working_dir: //doc/source/serve/tutorials/deployment-serve-llm # use // to access from repo's root + + cluster: + byod: + type: llm-cu128 # anyscale/ray-llm:-py311-cu128 + post_build_script: byod_deployment_serve_llm.sh # release/ray_release/byod/ + cluster_compute: ci/aws.yaml # relative to working_dir + + run: + timeout: 3600 + script: bash ci/tests.sh # relative to working_dir + + variations: + - __suffix__: aws # uses default specs above + - __suffix__: gce + env: gce + frequency: manual + cluster: + cluster_compute: ci/gce.yaml # relative to working_dir + + - name: distributing_pytorch # do not use dashes (regex sensitive) frequency: weekly group: ray-examples From 2cdbf63b7f8f84da48d99731b475c37d87e6e862 Mon Sep 17 00:00:00 2001 From: Douglas Strodtman Date: Thu, 4 Sep 2025 13:43:41 -0400 Subject: [PATCH 448/634] stable links for Ray serve (#56241) --- doc/source/serve/production-guide/config.md | 26 ++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/source/serve/production-guide/config.md b/doc/source/serve/production-guide/config.md index 8906d39894f0..fa5eff346fe5 100644 --- a/doc/source/serve/production-guide/config.md +++ b/doc/source/serve/production-guide/config.md @@ -51,11 +51,19 @@ applications: The file contains `proxy_location`, `http_options`, `grpc_options`, `logging_config` and `applications`. +(proxy-config)= + +## Proxy config + The `proxy_location` field configures where to run proxies to handle traffic to the cluster. You can set `proxy_location` to the following values: - EveryNode (default): Run a proxy on every node in the cluster that has at least one replica actor. - HeadOnly: Only run a single proxy on the head node. - Disabled: Don't run proxies at all. Set this value if you are only making calls to your applications using deployment handles. +(http-config)= + +## HTTP config + The `http_options` are as follows. Note that the HTTP config is global to your Ray cluster, and you can't update it during runtime. - **`host`**: The host IP address for Serve's HTTP proxies. This is optional and can be omitted. By default, the `host` is set to `0.0.0.0` to expose your deployments publicly. If you're using Kubernetes, you must set `host` to `0.0.0.0` to expose your deployments outside the cluster. @@ -63,15 +71,29 @@ The `http_options` are as follows. Note that the HTTP config is global to your R - **`request_timeout_s`**: Allows you to set the end-to-end timeout for a request before terminating and retrying at another replica. By default, there is no request timeout. - **`keep_alive_timeout_s`**: Allows you to set the keep alive timeout for the HTTP proxy. For more details, see [here](serve-http-guide-keep-alive-timeout) +(grpc-config)= + +## gRPC config + The `grpc_options` are as follows. Note that the gRPC config is global to your Ray cluster, and you can't update it during runtime. - **`port`**: The port that the gRPC proxies listen on. These are optional settings and can be omitted. By default, the port is set to `9000`. - **`grpc_servicer_functions`**: List of import paths for gRPC `add_servicer_to_server` functions to add to Serve's gRPC proxy. The servicer functions need to be importable from the context of where Serve is running. This defaults to an empty list, which means the gRPC server isn't started. - **`request_timeout_s`**: Allows you to set the end-to-end timeout for a request before terminating and retrying at another replica. By default, there is no request timeout. +(logging-config)= + +## Logging config + The `logging_config` is global config, you can configure controller & proxy & replica logs. Note that you can also set application and deployment level logging config, which will take precedence over the global config. See logging config API [here](../../serve/api/doc/ray.serve.schema.LoggingConfig.rst) for more details. -These are the fields per application: +(application-config)= + +## Application config + +You configure one or more deployments as part of your Serve application. See [deployment config](serve-configure-deployment). + +These are the fields per `application`: - **`name`**: The names for each application that are auto-generated by `serve build`. The name of each application must be unique. - **`route_prefix`**: An application can be called via HTTP at the specified route prefix. It defaults to `/`. The route prefix for each application must be unique. @@ -80,6 +102,8 @@ These are the fields per application: - **`deployments (optional)`**: A list of deployment options that allows you to override the `@serve.deployment` settings specified in the deployment graph code. Each entry in this list must include the deployment `name`, which must match one in the code. If this section is omitted, Serve launches all deployments in the graph with the parameters specified in the code. See how to [configure serve deployment options](serve-configure-deployment). - **`args`**: Arguments that are passed to the [application builder](serve-app-builder-guide). +## Example config + Below is a config for the [`Text ML Model` example](serve-in-production-example) that follows the format explained above: ```yaml From 016ebd8a6f27d86c4ab9019445b91a5bc6abd7f9 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Thu, 4 Sep 2025 11:44:59 -0700 Subject: [PATCH 449/634] [core][proto] move events_base_event_proto to public (#56212) Move events_base_event_proto to the public proto directory. Will merge and update doc after https://github.com/ray-project/ray/pull/56203. Test: - CI Signed-off-by: Cuong Nguyen --- .../modules/aggregator/aggregator_agent.py | 2 +- .../tests/gcs_ray_event_converter_test.cc | 2 +- src/ray/protobuf/BUILD.bazel | 21 +------------------ .../events_event_aggregator_service.proto | 2 +- src/ray/protobuf/public/BUILD.bazel | 19 +++++++++++++++++ .../{ => public}/events_base_event.proto | 0 6 files changed, 23 insertions(+), 23 deletions(-) rename src/ray/protobuf/{ => public}/events_base_event.proto (100%) diff --git a/python/ray/dashboard/modules/aggregator/aggregator_agent.py b/python/ray/dashboard/modules/aggregator/aggregator_agent.py index 228685b90fbb..d7782436ac22 100644 --- a/python/ray/dashboard/modules/aggregator/aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/aggregator_agent.py @@ -74,7 +74,7 @@ # Event filtering configurations # Comma-separated list of event types that are allowed to be exposed to external services # Valid values: TASK_DEFINITION_EVENT, TASK_EXECUTION_EVENT, ACTOR_TASK_DEFINITION_EVENT, ACTOR_TASK_EXECUTION_EVENT -# The list of all supported event types can be found in src/ray/protobuf/events_base_event.proto (EventType enum) +# The list of all supported event types can be found in src/ray/protobuf/public/events_base_event.proto (EventType enum) # By default TASK_PROFILE_EVENT is not exposed to external services DEFAULT_EXPOSABLE_EVENT_TYPES = ( "TASK_DEFINITION_EVENT,TASK_EXECUTION_EVENT," diff --git a/src/ray/gcs/gcs_server/tests/gcs_ray_event_converter_test.cc b/src/ray/gcs/gcs_server/tests/gcs_ray_event_converter_test.cc index 02738c4cce4c..ffffa9ddea0b 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_ray_event_converter_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_ray_event_converter_test.cc @@ -19,9 +19,9 @@ #include "gtest/gtest.h" #include "ray/common/id.h" #include "src/ray/protobuf/common.pb.h" -#include "src/ray/protobuf/events_base_event.pb.h" #include "src/ray/protobuf/events_event_aggregator_service.pb.h" #include "src/ray/protobuf/gcs_service.pb.h" +#include "src/ray/protobuf/public/events_base_event.pb.h" namespace ray { namespace gcs { diff --git a/src/ray/protobuf/BUILD.bazel b/src/ray/protobuf/BUILD.bazel index 7e89c341b679..5448fa1d071f 100644 --- a/src/ray/protobuf/BUILD.bazel +++ b/src/ray/protobuf/BUILD.bazel @@ -456,31 +456,12 @@ cc_proto_library( deps = [":events_task_profile_events_proto"], ) -proto_library( - name = "events_base_event_proto", - srcs = ["events_base_event.proto"], - deps = [ - ":events_task_profile_events_proto", - "//src/ray/protobuf/public:events_actor_task_definition_event_proto", - "//src/ray/protobuf/public:events_driver_job_definition_event_proto", - "//src/ray/protobuf/public:events_driver_job_execution_event_proto", - "//src/ray/protobuf/public:events_task_definition_event_proto", - "//src/ray/protobuf/public:events_task_execution_event_proto", - "@com_google_protobuf//:timestamp_proto", - ], -) - -cc_proto_library( - name = "events_base_event_cc_proto", - deps = [":events_base_event_proto"], -) - proto_library( name = "events_event_aggregator_service_proto", srcs = ["events_event_aggregator_service.proto"], deps = [ ":common_proto", - ":events_base_event_proto", + "//src/ray/protobuf/public:events_base_event_proto", ], ) diff --git a/src/ray/protobuf/events_event_aggregator_service.proto b/src/ray/protobuf/events_event_aggregator_service.proto index 896a9d9be350..9c1557ca7563 100644 --- a/src/ray/protobuf/events_event_aggregator_service.proto +++ b/src/ray/protobuf/events_event_aggregator_service.proto @@ -15,7 +15,7 @@ syntax = "proto3"; import "src/ray/protobuf/common.proto"; -import "src/ray/protobuf/events_base_event.proto"; +import "src/ray/protobuf/public/events_base_event.proto"; package ray.rpc.events; diff --git a/src/ray/protobuf/public/BUILD.bazel b/src/ray/protobuf/public/BUILD.bazel index 683e604ffffa..03a9f4b1c5f4 100644 --- a/src/ray/protobuf/public/BUILD.bazel +++ b/src/ray/protobuf/public/BUILD.bazel @@ -3,6 +3,25 @@ load("@rules_proto//proto:defs.bzl", "proto_library") package(default_visibility = ["//visibility:public"]) +proto_library( + name = "events_base_event_proto", + srcs = ["events_base_event.proto"], + deps = [ + ":events_actor_task_definition_event_proto", + ":events_driver_job_definition_event_proto", + ":events_driver_job_execution_event_proto", + ":events_task_definition_event_proto", + ":events_task_execution_event_proto", + "//src/ray/protobuf:events_task_profile_events_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + +cc_proto_library( + name = "events_base_event_cc_proto", + deps = [":events_base_event_proto"], +) + proto_library( name = "events_actor_task_definition_event_proto", srcs = ["events_actor_task_definition_event.proto"], diff --git a/src/ray/protobuf/events_base_event.proto b/src/ray/protobuf/public/events_base_event.proto similarity index 100% rename from src/ray/protobuf/events_base_event.proto rename to src/ray/protobuf/public/events_base_event.proto From 4f703d4efe1bf08f2a414119acf9316750fa1a63 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Thu, 4 Sep 2025 12:05:47 -0700 Subject: [PATCH 450/634] [core] Task submitters don't need to return a status (#56220) Signed-off-by: dayshah --- src/ray/core_worker/core_worker.cc | 18 +-- .../task_submission/actor_task_submitter.cc | 10 +- .../task_submission/actor_task_submitter.h | 8 +- .../task_submission/normal_task_submitter.cc | 3 +- .../task_submission/normal_task_submitter.h | 4 +- .../tests/actor_task_submitter_test.cc | 80 ++++++------- .../tests/direct_actor_transport_test.cc | 17 ++- .../tests/normal_task_submitter_test.cc | 112 +++++++++--------- 8 files changed, 118 insertions(+), 134 deletions(-) diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index a45b65dbe3fe..0dc1ea211221 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -778,11 +778,11 @@ void CoreWorker::InternalHeartbeat() { if (spec.IsActorTask()) { auto actor_handle = actor_manager_->GetActorHandle(spec.ActorId()); actor_handle->SetResubmittedActorTaskSpec(spec); - RAY_CHECK_OK(actor_task_submitter_->SubmitTask(spec)); + actor_task_submitter_->SubmitTask(spec); } else if (spec.IsActorCreationTask()) { - RAY_CHECK_OK(actor_task_submitter_->SubmitActorCreationTask(spec)); + actor_task_submitter_->SubmitActorCreationTask(spec); } else { - RAY_CHECK_OK(normal_task_submitter_->SubmitTask(spec)); + normal_task_submitter_->SubmitTask(spec); } } @@ -1996,7 +1996,7 @@ std::vector CoreWorker::SubmitTask( io_service_.post( [this, task_spec = std::move(task_spec)]() mutable { - RAY_UNUSED(normal_task_submitter_->SubmitTask(std::move(task_spec))); + normal_task_submitter_->SubmitTask(std::move(task_spec)); }, "CoreWorker.SubmitTask"); } @@ -2176,7 +2176,7 @@ Status CoreWorker::CreateActor(const RayFunction &function, task_manager_->FailPendingTask( task_spec.TaskId(), rpc::ErrorType::ACTOR_CREATION_FAILED, &status); } else { - RAY_UNUSED(actor_task_submitter_->SubmitActorCreationTask(task_spec)); + actor_task_submitter_->SubmitActorCreationTask(task_spec); } }); }, @@ -2191,7 +2191,7 @@ Status CoreWorker::CreateActor(const RayFunction &function, } io_service_.post( [this, task_spec = std::move(task_spec)]() { - RAY_UNUSED(actor_task_submitter_->SubmitActorCreationTask(task_spec)); + actor_task_submitter_->SubmitActorCreationTask(task_spec); }, "CoreWorker.SubmitTask"); } @@ -2373,7 +2373,7 @@ Status CoreWorker::SubmitActorTask( returned_refs = task_manager_->AddPendingTask( rpc_address_, task_spec, CurrentCallSite(), max_retries); - RAY_CHECK_OK(actor_task_submitter_->SubmitTask(task_spec)); + actor_task_submitter_->SubmitTask(task_spec); } task_returns = std::move(returned_refs); return Status::OK(); @@ -4560,10 +4560,10 @@ void CoreWorker::TaskManagerRetryTask(TaskSpecification &spec, if (spec.IsActorTask()) { auto actor_handle = actor_manager_->GetActorHandle(spec.ActorId()); actor_handle->SetResubmittedActorTaskSpec(spec); - RAY_CHECK_OK(actor_task_submitter_->SubmitTask(spec)); + actor_task_submitter_->SubmitTask(spec); } else { RAY_CHECK(spec.IsNormalTask()); - RAY_CHECK_OK(normal_task_submitter_->SubmitTask(spec)); + normal_task_submitter_->SubmitTask(spec); } } } diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.cc b/src/ray/core_worker/task_submission/actor_task_submitter.cc index d0f6063e4865..765697d75a1a 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.cc +++ b/src/ray/core_worker/task_submission/actor_task_submitter.cc @@ -89,7 +89,7 @@ void ActorTaskSubmitter::AddActorQueueIfNotExists(const ActorID &actor_id, } } -Status ActorTaskSubmitter::SubmitActorCreationTask(TaskSpecification task_spec) { +void ActorTaskSubmitter::SubmitActorCreationTask(TaskSpecification task_spec) { RAY_CHECK(task_spec.IsActorCreationTask()); RAY_LOG(DEBUG).WithField(task_spec.ActorCreationId()).WithField(task_spec.TaskId()) << "Submitting actor creation task"; @@ -162,11 +162,9 @@ Status ActorTaskSubmitter::SubmitActorCreationTask(TaskSpecification task_spec) } }); }); - - return Status::OK(); } -Status ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { +void ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { auto task_id = task_spec.TaskId(); auto actor_id = task_spec.ActorId(); RAY_LOG(DEBUG).WithField(task_id) << "Submitting task"; @@ -254,10 +252,6 @@ Status ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { /*mark_task_object_failed*/ true, fail_immediately); } - - // If the task submission subsequently fails, then the client will receive - // the error in a callback. - return Status::OK(); } void ActorTaskSubmitter::DisconnectRpcClient(ClientQueue &queue) { diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.h b/src/ray/core_worker/task_submission/actor_task_submitter.h index 8a98b84579b3..64b809da29b9 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.h +++ b/src/ray/core_worker/task_submission/actor_task_submitter.h @@ -113,14 +113,10 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { bool owned); /// Submit a task to an actor for execution. - /// - /// \param[in] task_spec The task spec to submit. - /// - /// \return Status::Invalid if the task is not yet supported. - Status SubmitTask(TaskSpecification task_spec); + void SubmitTask(TaskSpecification task_spec); /// Submit an actor creation task to an actor via GCS. - Status SubmitActorCreationTask(TaskSpecification task_spec); + void SubmitActorCreationTask(TaskSpecification task_spec); /// Create connection to actor and send all pending tasks. /// diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 9c2bf707247e..00a9a71ec4ac 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -28,7 +28,7 @@ namespace ray { namespace core { -Status NormalTaskSubmitter::SubmitTask(TaskSpecification task_spec) { +void NormalTaskSubmitter::SubmitTask(TaskSpecification task_spec) { RAY_CHECK(task_spec.IsNormalTask()); RAY_LOG(DEBUG) << "Submit task " << task_spec.TaskId(); @@ -89,7 +89,6 @@ Status NormalTaskSubmitter::SubmitTask(TaskSpecification task_spec) { } RequestNewWorkerIfNeeded(scheduling_key); }); - return Status::OK(); } void NormalTaskSubmitter::AddWorkerLeaseClient( diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.h b/src/ray/core_worker/task_submission/normal_task_submitter.h index 423b3d479500..1bd4b75f67ad 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.h +++ b/src/ray/core_worker/task_submission/normal_task_submitter.h @@ -113,9 +113,7 @@ class NormalTaskSubmitter { cancel_retry_timer_(std::move(cancel_timer)) {} /// Schedule a task for direct submission to a worker. - /// - /// \param[in] task_spec The task to schedule. - Status SubmitTask(TaskSpecification task_spec); + void SubmitTask(TaskSpecification task_spec); /// Either remove a pending task or send an RPC to kill a running task /// diff --git a/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc index 85d058f9007a..96044df6722e 100644 --- a/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc @@ -135,7 +135,7 @@ TEST_P(ActorTaskSubmitterTest, TestSubmitTask) { /*owned*/ false); auto task1 = CreateActorTaskHelper(actor_id, worker_id, 0); - ASSERT_TRUE(submitter_.SubmitTask(task1).ok()); + submitter_.SubmitTask(task1); ASSERT_EQ(io_context.poll_one(), 1); ASSERT_EQ(worker_client_->callbacks.size(), 0); @@ -143,7 +143,7 @@ TEST_P(ActorTaskSubmitterTest, TestSubmitTask) { ASSERT_EQ(worker_client_->callbacks.size(), 1); auto task2 = CreateActorTaskHelper(actor_id, worker_id, 1); - ASSERT_TRUE(submitter_.SubmitTask(task2).ok()); + submitter_.SubmitTask(task2); ASSERT_EQ(io_context.poll_one(), 1); ASSERT_EQ(worker_client_->callbacks.size(), 2); @@ -176,7 +176,7 @@ TEST_P(ActorTaskSubmitterTest, TestQueueingWarning) { for (int i = 0; i < 7500; i++) { auto task = CreateActorTaskHelper(actor_id, worker_id, i); - ASSERT_TRUE(submitter_.SubmitTask(task).ok()); + submitter_.SubmitTask(task); ASSERT_EQ(io_context.poll_one(), 1); ASSERT_TRUE(worker_client_->ReplyPushTask(task.GetTaskAttempt(), Status::OK())); } @@ -184,7 +184,7 @@ TEST_P(ActorTaskSubmitterTest, TestQueueingWarning) { for (int i = 7500; i < 15000; i++) { auto task = CreateActorTaskHelper(actor_id, worker_id, i); - ASSERT_TRUE(submitter_.SubmitTask(task).ok()); + submitter_.SubmitTask(task); ASSERT_EQ(io_context.poll_one(), 1); /* no ack */ } @@ -192,7 +192,7 @@ TEST_P(ActorTaskSubmitterTest, TestQueueingWarning) { for (int i = 15000; i < 35000; i++) { auto task = CreateActorTaskHelper(actor_id, worker_id, i); - ASSERT_TRUE(submitter_.SubmitTask(task).ok()); + submitter_.SubmitTask(task); ASSERT_EQ(io_context.poll_one(), 1); /* no ack */ } @@ -225,9 +225,9 @@ TEST_P(ActorTaskSubmitterTest, TestDependencies) { // Neither task can be submitted yet because they are still waiting on // dependencies. - ASSERT_TRUE(submitter_.SubmitTask(task1).ok()); + submitter_.SubmitTask(task1); ASSERT_EQ(io_context.poll_one(), 1); - ASSERT_TRUE(submitter_.SubmitTask(task2).ok()); + submitter_.SubmitTask(task2); ASSERT_EQ(io_context.poll_one(), 1); ASSERT_EQ(worker_client_->callbacks.size(), 0); @@ -272,9 +272,9 @@ TEST_P(ActorTaskSubmitterTest, TestOutOfOrderDependencies) { // Neither task can be submitted yet because they are still waiting on // dependencies. - ASSERT_TRUE(submitter_.SubmitTask(task1).ok()); + submitter_.SubmitTask(task1); ASSERT_EQ(io_context.poll_one(), 1); - ASSERT_TRUE(submitter_.SubmitTask(task2).ok()); + submitter_.SubmitTask(task2); ASSERT_EQ(io_context.poll_one(), 1); ASSERT_EQ(worker_client_->callbacks.size(), 0); @@ -325,9 +325,9 @@ TEST_P(ActorTaskSubmitterTest, TestActorDead) { ObjectID obj = ObjectID::FromRandom(); auto task2 = CreateActorTaskHelper(actor_id, worker_id, 1); task2.GetMutableMessage().add_args()->mutable_object_ref()->set_object_id(obj.Binary()); - ASSERT_TRUE(submitter_.SubmitTask(task1).ok()); + submitter_.SubmitTask(task1); ASSERT_EQ(io_context.poll_one(), 1); - ASSERT_TRUE(submitter_.SubmitTask(task2).ok()); + submitter_.SubmitTask(task2); ASSERT_EQ(io_context.poll_one(), 1); ASSERT_EQ(worker_client_->callbacks.size(), 1); @@ -369,11 +369,11 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartNoRetry) { auto task3 = CreateActorTaskHelper(actor_id, worker_id, 2); auto task4 = CreateActorTaskHelper(actor_id, worker_id, 3); // Submit three tasks. - ASSERT_TRUE(submitter_.SubmitTask(task1).ok()); + submitter_.SubmitTask(task1); ASSERT_EQ(io_context.poll_one(), 1); - ASSERT_TRUE(submitter_.SubmitTask(task2).ok()); + submitter_.SubmitTask(task2); ASSERT_EQ(io_context.poll_one(), 1); - ASSERT_TRUE(submitter_.SubmitTask(task3).ok()); + submitter_.SubmitTask(task3); ASSERT_EQ(io_context.poll_one(), 1); EXPECT_CALL(*task_manager_, CompletePendingTask(task1.TaskId(), _, _, _)).Times(1); @@ -397,7 +397,7 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartNoRetry) { // Actor gets restarted. addr.set_port(1); submitter_.ConnectActor(actor_id, addr, 1); - ASSERT_TRUE(submitter_.SubmitTask(task4).ok()); + submitter_.SubmitTask(task4); ASSERT_EQ(io_context.poll_one(), 1); ASSERT_TRUE(worker_client_->ReplyPushTask(task4.GetTaskAttempt(), Status::OK())); ASSERT_TRUE(worker_client_->callbacks.empty()); @@ -426,11 +426,11 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartRetry) { auto task3 = CreateActorTaskHelper(actor_id, worker_id, 2); auto task4 = CreateActorTaskHelper(actor_id, worker_id, 3); // Submit three tasks. - ASSERT_TRUE(submitter_.SubmitTask(task1).ok()); + submitter_.SubmitTask(task1); ASSERT_EQ(io_context.poll_one(), 1); - ASSERT_TRUE(submitter_.SubmitTask(task2).ok()); + submitter_.SubmitTask(task2); ASSERT_EQ(io_context.poll_one(), 1); - ASSERT_TRUE(submitter_.SubmitTask(task3).ok()); + submitter_.SubmitTask(task3); ASSERT_EQ(io_context.poll_one(), 1); // All tasks will eventually finish. @@ -457,17 +457,17 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartRetry) { addr.set_port(1); submitter_.ConnectActor(actor_id, addr, 1); // A new task is submitted. - ASSERT_TRUE(submitter_.SubmitTask(task4).ok()); + submitter_.SubmitTask(task4); ASSERT_EQ(io_context.poll_one(), 1); // Tasks 2 and 3 get retried. In the real world, the seq_no of these two tasks should be // updated to 4 and 5 by `CoreWorker::InternalHeartbeat`. task2.GetMutableMessage().set_attempt_number(task2.AttemptNumber() + 1); task2.GetMutableMessage().mutable_actor_task_spec()->set_sequence_number(4); - ASSERT_TRUE(submitter_.SubmitTask(task2).ok()); + submitter_.SubmitTask(task2); ASSERT_EQ(io_context.poll_one(), 1); task3.GetMutableMessage().set_attempt_number(task2.AttemptNumber() + 1); task3.GetMutableMessage().mutable_actor_task_spec()->set_sequence_number(5); - ASSERT_TRUE(submitter_.SubmitTask(task3).ok()); + submitter_.SubmitTask(task3); ASSERT_EQ(io_context.poll_one(), 1); ASSERT_TRUE(worker_client_->ReplyPushTask(task4.GetTaskAttempt(), Status::OK())); ASSERT_TRUE(worker_client_->ReplyPushTask(task2.GetTaskAttempt(), Status::OK())); @@ -496,11 +496,11 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderRetry) { auto task2 = CreateActorTaskHelper(actor_id, worker_id, 1); auto task3 = CreateActorTaskHelper(actor_id, worker_id, 2); // Submit three tasks. - ASSERT_TRUE(submitter_.SubmitTask(task1).ok()); + submitter_.SubmitTask(task1); ASSERT_EQ(io_context.poll_one(), 1); - ASSERT_TRUE(submitter_.SubmitTask(task2).ok()); + submitter_.SubmitTask(task2); ASSERT_EQ(io_context.poll_one(), 1); - ASSERT_TRUE(submitter_.SubmitTask(task3).ok()); + submitter_.SubmitTask(task3); ASSERT_EQ(io_context.poll_one(), 1); // All tasks will eventually finish. EXPECT_CALL(*task_manager_, CompletePendingTask(_, _, _, _)).Times(3); @@ -526,7 +526,7 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderRetry) { // Retry task 2 manually (simulating task_manager and SendPendingTask's behavior) task2.GetMutableMessage().set_attempt_number(task2.AttemptNumber() + 1); task2.GetMutableMessage().mutable_actor_task_spec()->set_sequence_number(3); - ASSERT_TRUE(submitter_.SubmitTask(task2).ok()); + submitter_.SubmitTask(task2); ASSERT_EQ(io_context.poll_one(), 1); // Only task2 should be submitted. task 3 (completed) should not be retried. @@ -553,7 +553,7 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderGcs) { // Create four tasks for the actor. auto task1 = CreateActorTaskHelper(actor_id, worker_id, 0); // Submit a task. - ASSERT_TRUE(submitter_.SubmitTask(task1).ok()); + submitter_.SubmitTask(task1); ASSERT_EQ(io_context.poll_one(), 1); EXPECT_CALL(*task_manager_, CompletePendingTask(task1.TaskId(), _, _, _)).Times(1); ASSERT_TRUE(worker_client_->ReplyPushTask(task1.GetTaskAttempt(), Status::OK())); @@ -564,7 +564,7 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderGcs) { ASSERT_EQ(num_clients_connected_, 2); // Submit a task. auto task2 = CreateActorTaskHelper(actor_id, worker_id, 1); - ASSERT_TRUE(submitter_.SubmitTask(task2).ok()); + submitter_.SubmitTask(task2); ASSERT_EQ(io_context.poll_one(), 1); EXPECT_CALL(*task_manager_, CompletePendingTask(task2.TaskId(), _, _, _)).Times(1); ASSERT_TRUE(worker_client_->ReplyPushTask(task2.GetTaskAttempt(), Status::OK())); @@ -576,7 +576,7 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderGcs) { ASSERT_EQ(num_clients_connected_, 2); // Submit a task. auto task3 = CreateActorTaskHelper(actor_id, worker_id, 2); - ASSERT_TRUE(submitter_.SubmitTask(task3).ok()); + submitter_.SubmitTask(task3); ASSERT_EQ(io_context.poll_one(), 1); EXPECT_CALL(*task_manager_, CompletePendingTask(task3.TaskId(), _, _, _)).Times(1); ASSERT_TRUE(worker_client_->ReplyPushTask(task3.GetTaskAttempt(), Status::OK())); @@ -587,7 +587,7 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderGcs) { ASSERT_EQ(num_clients_connected_, 2); // Submit a task. auto task4 = CreateActorTaskHelper(actor_id, worker_id, 3); - ASSERT_TRUE(submitter_.SubmitTask(task4).ok()); + submitter_.SubmitTask(task4); ASSERT_EQ(io_context.poll_one(), 1); // Tasks submitted when the actor is in RESTARTING state will fail immediately. // This happens in an io_service.post. Search `SendPendingTasks_ForceFail` to locate @@ -618,7 +618,7 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderGcs) { auto task5 = CreateActorTaskHelper(actor_id, worker_id, 4); EXPECT_CALL(*task_manager_, FailOrRetryPendingTask(task5.TaskId(), _, _, _, _, _)) .Times(1); - ASSERT_TRUE(submitter_.SubmitTask(task5).ok()); + submitter_.SubmitTask(task5); ASSERT_EQ(io_context.poll_one(), 0); } @@ -642,7 +642,7 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartFailInflightTasks) { auto task2_first_attempt = CreateActorTaskHelper(actor_id, caller_worker_id, 1); auto task3_first_attempt = CreateActorTaskHelper(actor_id, caller_worker_id, 2); // Submit a task. - ASSERT_TRUE(submitter_.SubmitTask(task1_first_attempt).ok()); + submitter_.SubmitTask(task1_first_attempt); ASSERT_EQ(io_context.poll_one(), 1); EXPECT_CALL(*task_manager_, CompletePendingTask(task1_first_attempt.TaskId(), _, _, _)) .Times(1); @@ -651,9 +651,9 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartFailInflightTasks) { ASSERT_EQ(worker_client_->callbacks.size(), 0); // Submit 2 tasks. - ASSERT_TRUE(submitter_.SubmitTask(task2_first_attempt).ok()); + submitter_.SubmitTask(task2_first_attempt); ASSERT_EQ(io_context.poll_one(), 1); - ASSERT_TRUE(submitter_.SubmitTask(task3_first_attempt).ok()); + submitter_.SubmitTask(task3_first_attempt); ASSERT_EQ(io_context.poll_one(), 1); // Actor failed, but the task replies are delayed (or in some scenarios, lost). // We should still be able to fail the inflight tasks. @@ -681,9 +681,9 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartFailInflightTasks) { task3_first_attempt.TaskIdBinary()); task3_second_attempt.GetMutableMessage().set_attempt_number( task3_first_attempt.AttemptNumber() + 1); - ASSERT_TRUE(submitter_.SubmitTask(task2_second_attempt).ok()); + submitter_.SubmitTask(task2_second_attempt); ASSERT_EQ(io_context.poll_one(), 1); - ASSERT_TRUE(submitter_.SubmitTask(task3_second_attempt).ok()); + submitter_.SubmitTask(task3_second_attempt); ASSERT_EQ(io_context.poll_one(), 1); // Restart the actor. @@ -752,7 +752,7 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartFastFail) { auto task1 = CreateActorTaskHelper(actor_id, worker_id, 0); // Submit a task. - ASSERT_TRUE(submitter_.SubmitTask(task1).ok()); + submitter_.SubmitTask(task1); ASSERT_EQ(io_context.poll_one(), 1); EXPECT_CALL(*task_manager_, CompletePendingTask(task1.TaskId(), _, _, _)).Times(1); ASSERT_TRUE(worker_client_->ReplyPushTask(task1.GetTaskAttempt(), Status::OK())); @@ -764,7 +764,7 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartFastFail) { // Submit a new task. This task should fail immediately because "max_task_retries" is 0. auto task2 = CreateActorTaskHelper(actor_id, worker_id, 1); - ASSERT_TRUE(submitter_.SubmitTask(task2).ok()); + submitter_.SubmitTask(task2); ASSERT_EQ(io_context.poll_one(), 1); EXPECT_CALL(*task_manager_, CompletePendingTask(task2.TaskId(), _, _, _)).Times(0); EXPECT_CALL(*task_manager_, FailOrRetryPendingTask(task2.TaskId(), _, _, _, _, _)) @@ -792,7 +792,7 @@ TEST_P(ActorTaskSubmitterTest, TestPendingTasks) { ASSERT_FALSE(submitter_.PendingTasksFull(actor_id)); auto task = CreateActorTaskHelper(actor_id, worker_id, i); tasks.push_back(task); - ASSERT_TRUE(submitter_.SubmitTask(task).ok()); + submitter_.SubmitTask(task); ASSERT_EQ(io_context.poll_one(), 1); } @@ -811,7 +811,7 @@ TEST_P(ActorTaskSubmitterTest, TestPendingTasks) { // We can submit task 10, but after that the queue is full. auto task = CreateActorTaskHelper(actor_id, worker_id, 10); tasks.push_back(task); - ASSERT_TRUE(submitter_.SubmitTask(task).ok()); + submitter_.SubmitTask(task); ASSERT_EQ(io_context.poll_one(), 1); ASSERT_TRUE(submitter_.PendingTasksFull(actor_id)); @@ -837,7 +837,7 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartResubmit) { // Generator is pushed to worker -> generator queued for resubmit -> comes back from // worker -> resubmit happens. auto task1 = CreateActorTaskHelper(actor_id, worker_id, 0); - ASSERT_TRUE(submitter_.SubmitTask(task1).ok()); + submitter_.SubmitTask(task1); io_context.run_one(); submitter_.ConnectActor(actor_id, addr, 0); ASSERT_EQ(worker_client_->callbacks.size(), 1); diff --git a/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc b/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc index b5a7bf44bb77..eb52aa92bef5 100644 --- a/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc +++ b/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc @@ -11,20 +11,17 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/core_worker/task_submission/actor_task_submitter.h" -// clang-format off #include #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "ray/core_worker/actor_creator.h" +#include "mock/ray/core_worker/memory_store.h" +#include "mock/ray/core_worker/reference_count.h" #include "mock/ray/core_worker/task_manager_interface.h" #include "mock/ray/gcs/gcs_client/gcs_client.h" -#include "mock/ray/core_worker/reference_count.h" -#include "mock/ray/core_worker/memory_store.h" - -// clang-format on +#include "ray/core_worker/actor_creator.h" +#include "ray/core_worker/task_submission/actor_task_submitter.h" namespace ray { namespace core { @@ -75,7 +72,7 @@ class DirectTaskTransportTest : public ::testing::Test { protected: bool CheckSubmitTask(TaskSpecification task) { - EXPECT_TRUE(actor_task_submitter->SubmitTask(task).ok()); + actor_task_submitter->SubmitTask(task); return 1 == io_context.poll_one(); } @@ -99,7 +96,7 @@ TEST_F(DirectTaskTransportTest, ActorCreationOk) { EXPECT_CALL(*gcs_client->mock_actor_accessor, AsyncCreateActor(creation_task_spec, ::testing::_)) .WillOnce(::testing::DoAll(::testing::SaveArg<1>(&create_cb))); - ASSERT_TRUE(actor_task_submitter->SubmitActorCreationTask(creation_task_spec).ok()); + actor_task_submitter->SubmitActorCreationTask(creation_task_spec); create_cb(Status::OK(), rpc::CreateActorReply()); } @@ -115,7 +112,7 @@ TEST_F(DirectTaskTransportTest, ActorCreationFail) { EXPECT_CALL(*gcs_client->mock_actor_accessor, AsyncCreateActor(creation_task_spec, ::testing::_)) .WillOnce(::testing::DoAll(::testing::SaveArg<1>(&create_cb))); - ASSERT_TRUE(actor_task_submitter->SubmitActorCreationTask(creation_task_spec).ok()); + actor_task_submitter->SubmitActorCreationTask(creation_task_spec); create_cb(Status::IOError(""), rpc::CreateActorReply()); } diff --git a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc index 97d8b6bd48fa..fc2d400714c5 100644 --- a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc @@ -556,7 +556,7 @@ TEST_F(NormalTaskSubmitterTest, TestLocalityAwareSubmitOneTask) { TaskSpecification task = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 1); ASSERT_EQ(raylet_client->num_is_selected_based_on_locality_leases_requested, 1); ASSERT_EQ(raylet_client->num_workers_requested, 1); @@ -587,7 +587,7 @@ TEST_F(NormalTaskSubmitterTest, TestSubmitOneTask) { CreateNormalTaskSubmitter(std::make_shared(1)); TaskSpecification task = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 1); ASSERT_EQ(raylet_client->num_is_selected_based_on_locality_leases_requested, 0); ASSERT_EQ(raylet_client->num_workers_requested, 1); @@ -619,7 +619,7 @@ TEST_F(NormalTaskSubmitterTest, TestRetryTaskApplicationLevelError) { TaskSpecification task = BuildEmptyTaskSpec(); task.GetMutableMessage().set_retry_exceptions(true); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); // Simulate an application-level error. ASSERT_TRUE(worker_client->ReplyPushTask(Status::OK(), false, true)); @@ -633,7 +633,7 @@ TEST_F(NormalTaskSubmitterTest, TestRetryTaskApplicationLevelError) { task.GetMutableMessage().set_retry_exceptions(false); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); // Simulate an application-level error. ASSERT_TRUE(worker_client->ReplyPushTask(Status::OK(), false, true)); @@ -655,7 +655,7 @@ TEST_F(NormalTaskSubmitterTest, TestHandleTaskFailure) { CreateNormalTaskSubmitter(std::make_shared(1)); TaskSpecification task = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); // Simulate a system failure, i.e., worker died unexpectedly. ASSERT_TRUE(worker_client->ReplyPushTask(Status::IOError("oops"))); @@ -683,7 +683,7 @@ TEST_F(NormalTaskSubmitterTest, TestCancellationWhileHandlingTaskFailure) { CreateNormalTaskSubmitter(std::make_shared(1)); TaskSpecification task = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); // Simulate a system failure, i.e., worker died unexpectedly so that // GetWorkerFailureCause is called. @@ -703,9 +703,9 @@ TEST_F(NormalTaskSubmitterTest, TestHandleUnschedulableTask) { TaskSpecification task2 = BuildEmptyTaskSpec(); TaskSpecification task3 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); - ASSERT_TRUE(submitter.SubmitTask(task2).ok()); - ASSERT_TRUE(submitter.SubmitTask(task3).ok()); + submitter.SubmitTask(task1); + submitter.SubmitTask(task2); + submitter.SubmitTask(task3); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2); ASSERT_EQ(raylet_client->num_workers_requested, 2); ASSERT_EQ(raylet_client->num_workers_returned, 0); @@ -753,9 +753,9 @@ TEST_F(NormalTaskSubmitterTest, TestHandleRuntimeEnvSetupFailed) { TaskSpecification task2 = BuildEmptyTaskSpec(); TaskSpecification task3 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); - ASSERT_TRUE(submitter.SubmitTask(task2).ok()); - ASSERT_TRUE(submitter.SubmitTask(task3).ok()); + submitter.SubmitTask(task1); + submitter.SubmitTask(task2); + submitter.SubmitTask(task3); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2); ASSERT_EQ(raylet_client->num_workers_requested, 2); ASSERT_EQ(raylet_client->num_workers_returned, 0); @@ -800,7 +800,7 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerHandleLocalRayletDied) { CreateNormalTaskSubmitter(std::make_shared(2)); TaskSpecification task1 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); + submitter.SubmitTask(task1); ASSERT_DEATH(raylet_client->FailWorkerLeaseDueToGrpcUnavailable(), ""); } @@ -812,9 +812,9 @@ TEST_F(NormalTaskSubmitterTest, TestDriverHandleLocalRayletDied) { TaskSpecification task2 = BuildEmptyTaskSpec(); TaskSpecification task3 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); - ASSERT_TRUE(submitter.SubmitTask(task2).ok()); - ASSERT_TRUE(submitter.SubmitTask(task3).ok()); + submitter.SubmitTask(task1); + submitter.SubmitTask(task2); + submitter.SubmitTask(task3); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2); ASSERT_EQ(raylet_client->num_workers_requested, 2); ASSERT_EQ(raylet_client->num_workers_returned, 0); @@ -847,7 +847,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeases) { for (int i = 0; i < 2 * concurrency; i++) { auto task = BuildEmptyTaskSpec(); tasks.push_back(task); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); } ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, concurrency); @@ -903,7 +903,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamic) { for (int i = 0; i < 2 * concurrency; i++) { auto task = BuildEmptyTaskSpec(); tasks.push_back(task); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); } ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 1); @@ -988,7 +988,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamicWithSpillback) for (int i = 0; i < 2 * concurrency; i++) { auto task = BuildEmptyTaskSpec(); tasks.push_back(task); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); } ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 1); @@ -1071,9 +1071,9 @@ TEST_F(NormalTaskSubmitterTest, TestSubmitMultipleTasks) { TaskSpecification task2 = BuildEmptyTaskSpec(); TaskSpecification task3 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); - ASSERT_TRUE(submitter.SubmitTask(task2).ok()); - ASSERT_TRUE(submitter.SubmitTask(task3).ok()); + submitter.SubmitTask(task1); + submitter.SubmitTask(task2); + submitter.SubmitTask(task3); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 1); ASSERT_EQ(raylet_client->num_workers_requested, 1); ASSERT_EQ(raylet_client->reported_backlog_size, 0); @@ -1122,9 +1122,9 @@ TEST_F(NormalTaskSubmitterTest, TestReuseWorkerLease) { TaskSpecification task2 = BuildEmptyTaskSpec(); TaskSpecification task3 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); - ASSERT_TRUE(submitter.SubmitTask(task2).ok()); - ASSERT_TRUE(submitter.SubmitTask(task3).ok()); + submitter.SubmitTask(task1); + submitter.SubmitTask(task2); + submitter.SubmitTask(task3); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 1); ASSERT_EQ(raylet_client->num_workers_requested, 1); @@ -1174,9 +1174,9 @@ TEST_F(NormalTaskSubmitterTest, TestRetryLeaseCancellation) { TaskSpecification task2 = BuildEmptyTaskSpec(); TaskSpecification task3 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); - ASSERT_TRUE(submitter.SubmitTask(task2).ok()); - ASSERT_TRUE(submitter.SubmitTask(task3).ok()); + submitter.SubmitTask(task1); + submitter.SubmitTask(task2); + submitter.SubmitTask(task3); ASSERT_EQ(raylet_client->num_workers_requested, 1); // Task 1 is pushed. @@ -1222,8 +1222,8 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentCancellationAndSubmission) { TaskSpecification task2 = BuildEmptyTaskSpec(); TaskSpecification task3 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); - ASSERT_TRUE(submitter.SubmitTask(task2).ok()); + submitter.SubmitTask(task1); + submitter.SubmitTask(task2); // Task 1 is pushed. ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); @@ -1239,7 +1239,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentCancellationAndSubmission) { ASSERT_EQ(raylet_client->num_workers_returned, 1); // Another task is submitted while task 2's lease request is being canceled. - ASSERT_TRUE(submitter.SubmitTask(task3).ok()); + submitter.SubmitTask(task3); ASSERT_EQ(raylet_client->num_workers_requested, 2); // Task 2's lease request is canceled, a new worker is requested for task 3. @@ -1266,8 +1266,8 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerNotReusedOnError) { TaskSpecification task1 = BuildEmptyTaskSpec(); TaskSpecification task2 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); - ASSERT_TRUE(submitter.SubmitTask(task2).ok()); + submitter.SubmitTask(task1); + submitter.SubmitTask(task2); ASSERT_EQ(raylet_client->num_workers_requested, 1); // Task 1 is pushed. @@ -1302,7 +1302,7 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerNotReturnedOnExit) { CreateNormalTaskSubmitter(std::make_shared(1)); TaskSpecification task1 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); + submitter.SubmitTask(task1); ASSERT_EQ(raylet_client->num_workers_requested, 1); // Task 1 is pushed. @@ -1338,7 +1338,7 @@ TEST_F(NormalTaskSubmitterTest, TestSpillback) { raylet_client_factory); TaskSpecification task = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 1); ASSERT_EQ(raylet_client->num_workers_requested, 1); ASSERT_EQ(raylet_client->num_workers_returned, 0); @@ -1395,7 +1395,7 @@ TEST_F(NormalTaskSubmitterTest, TestSpillbackRoundTrip) { kLongTimeout); TaskSpecification task = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_EQ(raylet_client->num_grant_or_reject_leases_requested, 0); ASSERT_EQ(raylet_client->num_workers_requested, 1); ASSERT_EQ(raylet_client->num_workers_returned, 0); @@ -1479,9 +1479,9 @@ void TestSchedulingKey(const std::shared_ptr store, [](const ObjectID &object_id) { return rpc::TensorTransport::OBJECT_STORE; }, boost::asio::steady_timer(io_context)); - ASSERT_TRUE(submitter.SubmitTask(same1).ok()); - ASSERT_TRUE(submitter.SubmitTask(same2).ok()); - ASSERT_TRUE(submitter.SubmitTask(different).ok()); + submitter.SubmitTask(same1); + submitter.SubmitTask(same2); + submitter.SubmitTask(different); WaitForCondition( [&raylet_client]() { return raylet_client->num_workers_returned == 2; }, @@ -1652,14 +1652,14 @@ TEST_F(NormalTaskSubmitterTest, TestBacklogReport) { TaskSpecification task4 = BuildTaskSpec(resources2, descriptor2); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); + submitter.SubmitTask(task1); // One is requested and one is in the backlog for each SchedulingKey - ASSERT_TRUE(submitter.SubmitTask(WithRandomTaskId(task2)).ok()); - ASSERT_TRUE(submitter.SubmitTask(WithRandomTaskId(task2)).ok()); - ASSERT_TRUE(submitter.SubmitTask(WithRandomTaskId(task3)).ok()); - ASSERT_TRUE(submitter.SubmitTask(WithRandomTaskId(task3)).ok()); - ASSERT_TRUE(submitter.SubmitTask(WithRandomTaskId(task4)).ok()); - ASSERT_TRUE(submitter.SubmitTask(WithRandomTaskId(task4)).ok()); + submitter.SubmitTask(WithRandomTaskId(task2)); + submitter.SubmitTask(WithRandomTaskId(task2)); + submitter.SubmitTask(WithRandomTaskId(task3)); + submitter.SubmitTask(WithRandomTaskId(task3)); + submitter.SubmitTask(WithRandomTaskId(task4)); + submitter.SubmitTask(WithRandomTaskId(task4)); // Waits for the async callbacks in submitter.SubmitTask to finish, before we call // ReportWorkerBacklog. @@ -1688,9 +1688,9 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerLeaseTimeout) { TaskSpecification task2 = BuildEmptyTaskSpec(); TaskSpecification task3 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task1).ok()); - ASSERT_TRUE(submitter.SubmitTask(task2).ok()); - ASSERT_TRUE(submitter.SubmitTask(task3).ok()); + submitter.SubmitTask(task1); + submitter.SubmitTask(task2); + submitter.SubmitTask(task3); ASSERT_EQ(raylet_client->num_workers_requested, 1); // Task 1 is pushed. @@ -1733,7 +1733,7 @@ TEST_F(NormalTaskSubmitterTest, TestKillExecutingTask) { CreateNormalTaskSubmitter(std::make_shared(1)); TaskSpecification task = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); // Try force kill, exiting the worker @@ -1750,7 +1750,7 @@ TEST_F(NormalTaskSubmitterTest, TestKillExecutingTask) { task.GetMutableMessage().set_task_id( TaskID::ForNormalTask(JobID::Nil(), TaskID::Nil(), 1).Binary()); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); // Try non-force kill, worker returns normally @@ -1774,7 +1774,7 @@ TEST_F(NormalTaskSubmitterTest, TestKillPendingTask) { CreateNormalTaskSubmitter(std::make_shared(1)); TaskSpecification task = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); submitter.CancelTask(task, true, false); ASSERT_EQ(worker_client->kill_requests.size(), 0); ASSERT_EQ(worker_client->callbacks.size(), 0); @@ -1800,7 +1800,7 @@ TEST_F(NormalTaskSubmitterTest, TestKillResolvingTask) { TaskSpecification task = BuildEmptyTaskSpec(); ObjectID obj1 = ObjectID::FromRandom(); task.GetMutableMessage().add_args()->mutable_object_ref()->set_object_id(obj1.Binary()); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_EQ(task_manager->num_inlined_dependencies, 0); submitter.CancelTask(task, true, false); auto data = GenerateRandomObject(); @@ -1823,7 +1823,7 @@ TEST_F(NormalTaskSubmitterTest, TestQueueGeneratorForResubmit) { auto submitter = CreateNormalTaskSubmitter(std::make_shared(1)); TaskSpecification task = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); ASSERT_TRUE(submitter.QueueGeneratorForResubmit(task)); ASSERT_TRUE(worker_client->ReplyPushTask()); @@ -1838,7 +1838,7 @@ TEST_F(NormalTaskSubmitterTest, TestCancelBeforeAfterQueueGeneratorForResubmit) auto submitter = CreateNormalTaskSubmitter(std::make_shared(1)); TaskSpecification task = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task).ok()); + submitter.SubmitTask(task); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); submitter.CancelTask(task, /*force_kill=*/false, /*recursive=*/true); ASSERT_FALSE(submitter.QueueGeneratorForResubmit(task)); @@ -1855,7 +1855,7 @@ TEST_F(NormalTaskSubmitterTest, TestCancelBeforeAfterQueueGeneratorForResubmit) // Succesful queue generator for resubmit -> cancel -> successful execution -> no // resubmit. TaskSpecification task2 = BuildEmptyTaskSpec(); - ASSERT_TRUE(submitter.SubmitTask(task2).ok()); + submitter.SubmitTask(task2); ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); ASSERT_TRUE(submitter.QueueGeneratorForResubmit(task2)); submitter.CancelTask(task2, /*force_kill=*/false, /*recursive=*/true); From 5022884ec084449f048f2c2ed008a11203e278f3 Mon Sep 17 00:00:00 2001 From: goutamvenkat-anyscale Date: Thu, 4 Sep 2025 12:15:15 -0700 Subject: [PATCH 451/634] [Data] - Add DataType to Expression class (#55915) ## Why are these changes needed? Add return_dtype to all Expr types. ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Goutam V Signed-off-by: Goutam V. --- doc/source/data/api/api.rst | 1 + doc/source/data/api/datatype.rst | 12 + python/ray/data/BUILD | 10 + python/ray/data/dataset.py | 3 +- python/ray/data/datatype.py | 255 ++++++++++++++++ python/ray/data/expressions.py | 37 ++- python/ray/data/tests/test_datatype.py | 392 +++++++++++++++++++++++++ python/ray/data/tests/test_map.py | 39 +-- 8 files changed, 722 insertions(+), 27 deletions(-) create mode 100644 doc/source/data/api/datatype.rst create mode 100644 python/ray/data/datatype.py create mode 100644 python/ray/data/tests/test_datatype.py diff --git a/doc/source/data/api/api.rst b/doc/source/data/api/api.rst index 2926be209658..009eafbdc950 100644 --- a/doc/source/data/api/api.rst +++ b/doc/source/data/api/api.rst @@ -13,6 +13,7 @@ Ray Data API aggregate.rst grouped_data.rst expressions.rst + datatype.rst data_context.rst preprocessor.rst llm.rst diff --git a/doc/source/data/api/datatype.rst b/doc/source/data/api/datatype.rst new file mode 100644 index 000000000000..4e39831b50a9 --- /dev/null +++ b/doc/source/data/api/datatype.rst @@ -0,0 +1,12 @@ +.. _datatype-api: + +Data types +========== + +.. currentmodule:: ray.data.datatype + +Class +----- + +.. autoclass:: DataType + :members: diff --git a/python/ray/data/BUILD b/python/ray/data/BUILD index c9273b2aec72..396cb8501b2a 100644 --- a/python/ray/data/BUILD +++ b/python/ray/data/BUILD @@ -111,6 +111,16 @@ py_test( ], ) +py_test( + name = "test_datatype", + size = "small", + srcs = ["tests/test_datatype.py"], + tags = [ + "exclusive", + "team:data", + ], +) + py_test( name = "test_sql", size = "small", diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index f2cef1c44761..33a628977bae 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -806,10 +806,11 @@ def with_column( {'id': 1, 'id_2': 2} >>> # Using a UDF with with_column + >>> from ray.data.datatype import DataType >>> from ray.data.expressions import udf >>> import pyarrow.compute as pc >>> - >>> @udf() + >>> @udf(return_dtype=DataType.int32()) ... def add_one(column): ... return pc.add(column, 1) >>> diff --git a/python/ray/data/datatype.py b/python/ray/data/datatype.py new file mode 100644 index 000000000000..4c9fb79defce --- /dev/null +++ b/python/ray/data/datatype.py @@ -0,0 +1,255 @@ +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import pyarrow as pa + +from ray.air.util.tensor_extensions.arrow import ( + _infer_pyarrow_type, +) +from ray.util.annotations import PublicAPI + +PYARROW_TYPE_DEFINITIONS: Dict[str, Tuple[callable, str]] = { + "int8": (pa.int8, "an 8-bit signed integer"), + "int16": (pa.int16, "a 16-bit signed integer"), + "int32": (pa.int32, "a 32-bit signed integer"), + "int64": (pa.int64, "a 64-bit signed integer"), + "uint8": (pa.uint8, "an 8-bit unsigned integer"), + "uint16": (pa.uint16, "a 16-bit unsigned integer"), + "uint32": (pa.uint32, "a 32-bit unsigned integer"), + "uint64": (pa.uint64, "a 64-bit unsigned integer"), + "float32": (pa.float32, "a 32-bit floating point number"), + "float64": (pa.float64, "a 64-bit floating point number"), + "string": (pa.string, "a variable-length string"), + "bool": (pa.bool_, "a boolean value"), + "binary": (pa.binary, "variable-length binary data"), +} + + +def _factory_methods(cls: type): + """Metaprogramming: Class decorator to generate factory methods for PyArrow types using from_arrow. + + This decorator automatically creates class methods for common PyArrow data types. + Each generated method is a convenient factory that calls cls.from_arrow(pa.type()). + + Generated methods include: + - Signed integers: int8, int16, int32, int64 + - Unsigned integers: uint8, uint16, uint32, uint64 + - Floating point: float32, float64 + - Other types: string, bool, binary + + Examples of generated methods:: + + @classmethod + def int32(cls): + \"\"\"Create a DataType representing a 32-bit signed integer. + + Returns: + DataType: A DataType with PyArrow int32 type + \"\"\" + return cls.from_arrow(pa.int32()) + + @classmethod + def string(cls): + \"\"\"Create a DataType representing a variable-length string. + + Returns: + DataType: A DataType with PyArrow string type + \"\"\" + return cls.from_arrow(pa.string()) + + Usage: + Instead of DataType.from_arrow(pa.int32()), you can use DataType.int32() + """ + + for method_name, (pa_func, description) in PYARROW_TYPE_DEFINITIONS.items(): + + def create_method(name, func, desc): + def factory_method(cls): + return cls.from_arrow(func()) + + factory_method.__doc__ = f"""Create a DataType representing {desc}. + + Returns: + DataType: A DataType with PyArrow {name} type + """ + factory_method.__name__ = name + factory_method.__qualname__ = f"{cls.__name__}.{name}" + return classmethod(factory_method) + + setattr(cls, method_name, create_method(method_name, pa_func, description)) + + return cls + + +@PublicAPI(stability="alpha") +@dataclass +@_factory_methods +class DataType: + """A simplified Ray Data DataType supporting Arrow, NumPy, and Python types.""" + + _internal_type: Union[pa.DataType, np.dtype, type] + + def __post_init__(self): + """Validate the _internal_type after initialization.""" + # TODO: Support Pandas extension types + if not isinstance( + self._internal_type, + (pa.DataType, np.dtype, type), + ): + raise TypeError( + f"DataType supports only PyArrow DataType, NumPy dtype, or Python type, but was given type {type(self._internal_type)}." + ) + + # Type checking methods + def is_arrow_type(self) -> bool: + return isinstance(self._internal_type, pa.DataType) + + def is_numpy_type(self) -> bool: + return isinstance(self._internal_type, np.dtype) + + def is_python_type(self) -> bool: + return isinstance(self._internal_type, type) + + # Conversion methods + def to_arrow_dtype(self, values: Optional[List[Any]] = None) -> pa.DataType: + """ + Convert the DataType to a PyArrow DataType. + + Args: + values: Optional list of values to infer the Arrow type from. Required if the DataType is a Python type. + + Returns: + A PyArrow DataType + """ + if self.is_arrow_type(): + return self._internal_type + else: + if isinstance(self._internal_type, np.dtype): + return pa.from_numpy_dtype(self._internal_type) + else: + assert ( + values is not None and len(values) > 0 + ), "Values are required to infer Arrow type if the provided type is a Python type" + return _infer_pyarrow_type(values) + + def to_numpy_dtype(self) -> np.dtype: + if self.is_numpy_type(): + return self._internal_type + elif self.is_arrow_type(): + try: + # For most basic arrow types, this will work + pandas_dtype = self._internal_type.to_pandas_dtype() + if isinstance(pandas_dtype, np.dtype): + return pandas_dtype + else: + # If pandas returns an extension dtype, fall back to object + return np.dtype("object") + except (TypeError, NotImplementedError, pa.ArrowNotImplementedError): + return np.dtype("object") + else: + return np.dtype("object") + + def to_python_type(self) -> type: + if self.is_python_type(): + return self._internal_type + else: + raise ValueError(f"DataType {self} is not a Python type") + + # Factory methods from external systems + @classmethod + def from_arrow(cls, arrow_type: pa.DataType) -> "DataType": + """Create a DataType from a PyArrow DataType. + + Args: + arrow_type: A PyArrow DataType to wrap + + Returns: + DataType: A DataType wrapping the given PyArrow type + + Examples: + >>> import pyarrow as pa + >>> from ray.data.datatype import DataType + >>> DataType.from_arrow(pa.timestamp('s')) + DataType(arrow:timestamp[s]) + >>> DataType.from_arrow(pa.int64()) + DataType(arrow:int64) + """ + return cls(_internal_type=arrow_type) + + @classmethod + def from_numpy(cls, numpy_dtype: Union[np.dtype, str]) -> "DataType": + """Create a DataType from a NumPy dtype. + + Args: + numpy_dtype: A NumPy dtype object or string representation + + Returns: + DataType: A DataType wrapping the given NumPy dtype + + Examples: + >>> import numpy as np + >>> from ray.data.datatype import DataType + >>> DataType.from_numpy(np.dtype('int32')) + DataType(numpy:int32) + >>> DataType.from_numpy('float64') + DataType(numpy:float64) + """ + if isinstance(numpy_dtype, str): + numpy_dtype = np.dtype(numpy_dtype) + return cls(_internal_type=numpy_dtype) + + @classmethod + def infer_dtype(cls, value: Any) -> "DataType": + """Infer DataType from a Python value, handling numpy, Arrow, and Python types. + + Args: + value: Any Python value to infer the type from + + Returns: + DataType: The inferred data type + + Examples: + >>> import numpy as np + >>> from ray.data.datatype import DataType + >>> DataType.infer_dtype(5) + DataType(arrow:int64) + >>> DataType.infer_dtype("hello") + DataType(arrow:string) + >>> DataType.infer_dtype(np.int32(42)) + DataType(numpy:int32) + """ + # 1. Handle numpy arrays and scalars + if isinstance(value, (np.ndarray, np.generic)): + return cls.from_numpy(value.dtype) + # 3. Try PyArrow type inference for regular Python values + try: + inferred_arrow_type = _infer_pyarrow_type([value]) + if inferred_arrow_type is not None: + return cls.from_arrow(inferred_arrow_type) + except Exception: + return cls(type(value)) + + def __repr__(self) -> str: + if self.is_arrow_type(): + return f"DataType(arrow:{self._internal_type})" + elif self.is_numpy_type(): + return f"DataType(numpy:{self._internal_type})" + else: + return f"DataType(python:{self._internal_type.__name__})" + + def __eq__(self, other) -> bool: + if not isinstance(other, DataType): + return False + + # Ensure they're from the same type system by checking the actual type + # of the internal type object, not just the value + if type(self._internal_type) is not type(other._internal_type): + return False + + return self._internal_type == other._internal_type + + def __hash__(self) -> int: + # Include the type of the internal type in the hash to ensure + # different type systems don't collide + return hash((type(self._internal_type), self._internal_type)) diff --git a/python/ray/data/expressions.py b/python/ray/data/expressions.py index fa99ae638eab..94a799802068 100644 --- a/python/ray/data/expressions.py +++ b/python/ray/data/expressions.py @@ -2,11 +2,12 @@ import functools from abc import ABC, abstractmethod -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum from typing import Any, Callable, Dict, List from ray.data.block import BatchColumn +from ray.data.datatype import DataType from ray.util.annotations import DeveloperAPI, PublicAPI @@ -69,6 +70,8 @@ class Expr(ABC): subclasses like ColumnExpr, LiteralExpr, etc. """ + data_type: DataType + @abstractmethod def structurally_equals(self, other: Any) -> bool: """Compare two expression ASTs for structural equality.""" @@ -174,6 +177,7 @@ class ColumnExpr(Expr): """ name: str + data_type: DataType = field(default_factory=lambda: DataType(object), init=False) def structurally_equals(self, other: Any) -> bool: return isinstance(other, ColumnExpr) and self.name == other.name @@ -192,12 +196,22 @@ class LiteralExpr(Expr): Example: >>> from ray.data.expressions import lit + >>> import numpy as np >>> # Create a literal value >>> five = lit(5) # Creates LiteralExpr(value=5) >>> name = lit("John") # Creates LiteralExpr(value="John") + >>> numpy_val = lit(np.int32(42)) # Creates LiteralExpr with numpy type """ value: Any + data_type: DataType = field(init=False) + + def __post_init__(self): + # Infer the type from the value using DataType.infer_dtype + inferred_dtype = DataType.infer_dtype(self.value) + + # Use object.__setattr__ since the dataclass is frozen + object.__setattr__(self, "data_type", inferred_dtype) def structurally_equals(self, other: Any) -> bool: return ( @@ -232,6 +246,8 @@ class BinaryExpr(Expr): left: Expr right: Expr + data_type: DataType = field(default_factory=lambda: DataType(object), init=False) + def structurally_equals(self, other: Any) -> bool: return ( isinstance(other, BinaryExpr) @@ -263,7 +279,7 @@ class UDFExpr(Expr): >>> import pyarrow as pa >>> import pyarrow.compute as pc >>> - >>> @udf() + >>> @udf(return_dtype=DataType.int32()) ... def add_one(x: pa.Array) -> pa.Array: ... return pc.add(x, 1) >>> @@ -289,7 +305,9 @@ def structurally_equals(self, other: Any) -> bool: ) -def _create_udf_callable(fn: Callable[..., BatchColumn]) -> Callable[..., UDFExpr]: +def _create_udf_callable( + fn: Callable[..., BatchColumn], return_dtype: DataType +) -> Callable[..., UDFExpr]: """Create a callable that generates UDFExpr when called with expressions.""" def udf_callable(*args, **kwargs) -> UDFExpr: @@ -312,6 +330,7 @@ def udf_callable(*args, **kwargs) -> UDFExpr: fn=fn, args=expr_args, kwargs=expr_kwargs, + data_type=return_dtype, ) # Preserve original function metadata @@ -324,7 +343,7 @@ def udf_callable(*args, **kwargs) -> UDFExpr: @PublicAPI(stability="alpha") -def udf() -> Callable[..., UDFExpr]: +def udf(return_dtype: DataType) -> Callable[..., UDFExpr]: """ Decorator to convert a UDF into an expression-compatible function. @@ -336,6 +355,9 @@ def udf() -> Callable[..., UDFExpr]: multiple values from that column across the batch. Under the hood, when working with multiple columns, they get translated to PyArrow arrays (one array per column). + Args: + return_dtype: The data type of the return value of the UDF + Returns: A callable that creates UDFExpr instances when called with expressions @@ -346,12 +368,12 @@ def udf() -> Callable[..., UDFExpr]: >>> import ray >>> >>> # UDF that operates on a batch of values (PyArrow Array) - >>> @udf() + >>> @udf(return_dtype=DataType.int32()) ... def add_one(x: pa.Array) -> pa.Array: ... return pc.add(x, 1) # Vectorized operation on the entire Array >>> >>> # UDF that combines multiple columns (each as a PyArrow Array) - >>> @udf() + >>> @udf(return_dtype=DataType.string()) ... def format_name(first: pa.Array, last: pa.Array) -> pa.Array: ... return pc.binary_join_element_wise(first, last, " ") # Vectorized string concatenation >>> @@ -372,7 +394,7 @@ def udf() -> Callable[..., UDFExpr]: """ def decorator(func: Callable[..., BatchColumn]) -> Callable[..., UDFExpr]: - return _create_udf_callable(func) + return _create_udf_callable(func, return_dtype) return decorator @@ -383,6 +405,7 @@ class DownloadExpr(Expr): """Expression that represents a download operation.""" uri_column_name: str + data_type: DataType = field(default_factory=lambda: DataType.binary(), init=False) def structurally_equals(self, other: Any) -> bool: return ( diff --git a/python/ray/data/tests/test_datatype.py b/python/ray/data/tests/test_datatype.py new file mode 100644 index 000000000000..ceb3a2650941 --- /dev/null +++ b/python/ray/data/tests/test_datatype.py @@ -0,0 +1,392 @@ +import numpy as np +import pyarrow as pa +import pytest + +from ray.data.datatype import DataType + + +class TestDataTypeFactoryMethods: + """Test the generated factory methods.""" + + @pytest.mark.parametrize( + "method_name,pa_type,description", + [ + ("int8", pa.int8(), "8-bit signed integer"), + ("int16", pa.int16(), "16-bit signed integer"), + ("int32", pa.int32(), "32-bit signed integer"), + ("int64", pa.int64(), "64-bit signed integer"), + ("uint8", pa.uint8(), "8-bit unsigned integer"), + ("uint16", pa.uint16(), "16-bit unsigned integer"), + ("uint32", pa.uint32(), "32-bit unsigned integer"), + ("uint64", pa.uint64(), "64-bit unsigned integer"), + ("float32", pa.float32(), "32-bit floating point number"), + ("float64", pa.float64(), "64-bit floating point number"), + ("string", pa.string(), "variable-length string"), + ("bool", pa.bool_(), "boolean value"), + ("binary", pa.binary(), "variable-length binary data"), + ], + ) + def test_factory_method_creates_correct_type( + self, method_name, pa_type, description + ): + """Test that factory methods create DataType with correct PyArrow type.""" + factory_method = getattr(DataType, method_name) + result = factory_method() + + assert isinstance(result, DataType) + assert result.is_arrow_type() + assert result._internal_type == pa_type + + @pytest.mark.parametrize( + "method_name", + [ + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", + "float32", + "float64", + "string", + "bool", + "binary", + ], + ) + def test_factory_method_has_proper_docstring(self, method_name): + """Test that generated factory methods have proper docstrings.""" + factory_method = getattr(DataType, method_name) + doc = factory_method.__doc__ + + assert "Create a DataType representing" in doc + assert "Returns:" in doc + assert f"DataType with PyArrow {method_name} type" in doc + + +class TestDataTypeValidation: + """Test DataType validation and initialization.""" + + @pytest.mark.parametrize( + "valid_type", + [ + pa.int64(), + pa.string(), + pa.timestamp("s"), + np.dtype("int32"), + np.dtype("float64"), + int, + str, + float, + ], + ) + def test_post_init_accepts_valid_types(self, valid_type): + """Test that __post_init__ accepts valid type objects.""" + # Should not raise + dt = DataType(valid_type) + assert dt._internal_type == valid_type + + @pytest.mark.parametrize( + "invalid_type", + [ + "string", + 123, + [1, 2, 3], + {"key": "value"}, + None, + object(), + ], + ) + def test_post_init_rejects_invalid_types(self, invalid_type): + """Test that __post_init__ rejects invalid type objects.""" + with pytest.raises( + TypeError, + match="DataType supports only PyArrow DataType, NumPy dtype, or Python type", + ): + DataType(invalid_type) + + +class TestDataTypeCheckers: + """Test type checking methods.""" + + @pytest.mark.parametrize( + "datatype,is_arrow,is_numpy,is_python", + [ + (DataType.from_arrow(pa.int64()), True, False, False), + (DataType.from_arrow(pa.string()), True, False, False), + (DataType.from_numpy(np.dtype("int32")), False, True, False), + (DataType.from_numpy(np.dtype("float64")), False, True, False), + (DataType(int), False, False, True), + (DataType(str), False, False, True), + ], + ) + def test_type_checkers(self, datatype, is_arrow, is_numpy, is_python): + """Test is_arrow_type, is_numpy_type, and is_python_type methods.""" + assert datatype.is_arrow_type() == is_arrow + assert datatype.is_numpy_type() == is_numpy + assert datatype.is_python_type() == is_python + + +class TestDataTypeFactories: + """Test factory methods from external systems.""" + + @pytest.mark.parametrize( + "pa_type", + [ + pa.int32(), + pa.string(), + pa.timestamp("s"), + pa.list_(pa.int32()), + pa.decimal128(10, 2), + ], + ) + def test_from_arrow(self, pa_type): + """Test from_arrow factory method.""" + dt = DataType.from_arrow(pa_type) + + assert isinstance(dt, DataType) + assert dt.is_arrow_type() + assert dt._internal_type == pa_type + + @pytest.mark.parametrize( + "numpy_input,expected_dtype", + [ + (np.dtype("int32"), np.dtype("int32")), + (np.dtype("float64"), np.dtype("float64")), + ("int64", np.dtype("int64")), + ("float32", np.dtype("float32")), + ], + ) + def test_from_numpy(self, numpy_input, expected_dtype): + """Test from_numpy factory method.""" + dt = DataType.from_numpy(numpy_input) + + assert isinstance(dt, DataType) + assert dt.is_numpy_type() + assert dt._internal_type == expected_dtype + + +class TestDataTypeConversions: + """Test type conversion methods.""" + + def test_to_arrow_dtype_arrow_passthrough(self): + """Test that Arrow types return themselves.""" + dt = DataType.from_arrow(pa.int64()) + result = dt.to_arrow_dtype() + assert result == pa.int64() + + def test_to_arrow_dtype_numpy_conversion(self): + """Test conversion from NumPy to Arrow types.""" + dt = DataType.from_numpy(np.dtype("int32")) + result = dt.to_arrow_dtype() + assert result == pa.int32() + + def test_to_arrow_dtype_python_conversion(self): + """Test conversion from Python to Arrow types.""" + dt = DataType(int) + result = dt.to_arrow_dtype([1]) + # Python int should map to int64 in Arrow + assert result == pa.int64() + + @pytest.mark.parametrize( + "source_dt,expected_result", + [ + # NumPy types should return themselves + (DataType.from_numpy(np.dtype("int32")), np.dtype("int32")), + (DataType.from_numpy(np.dtype("float64")), np.dtype("float64")), + # Python types should fall back to object + (DataType(str), np.dtype("object")), + (DataType(list), np.dtype("object")), + ], + ) + def test_to_numpy_dtype(self, source_dt, expected_result): + """Test to_numpy_dtype conversion.""" + result = source_dt.to_numpy_dtype() + assert result == expected_result + + def test_to_numpy_dtype_arrow_basic_types(self): + """Test Arrow to NumPy conversion for types that should work.""" + # Test basic types that should convert properly + test_cases = [ + (pa.int32(), np.dtype("int32")), + (pa.float64(), np.dtype("float64")), + (pa.bool_(), np.dtype("bool")), + ] + + for pa_type, expected_np_dtype in test_cases: + dt = DataType.from_arrow(pa_type) + result = dt.to_numpy_dtype() + # Some Arrow types may not convert exactly as expected, + # so let's just verify the result is a valid numpy dtype + assert isinstance(result, np.dtype) + + def test_to_numpy_dtype_complex_arrow_fallback(self): + """Test that complex Arrow types fall back to object dtype.""" + complex_dt = DataType.from_arrow(pa.list_(pa.int32())) + result = complex_dt.to_numpy_dtype() + assert result == np.dtype("object") + + @pytest.mark.parametrize("python_type", [int, str, float, bool, list]) + def test_to_python_type_success(self, python_type): + """Test to_python_type returns the original Python type.""" + dt = DataType(python_type) + result = dt.to_python_type() + assert result == python_type + + @pytest.mark.parametrize( + "non_python_dt", + [ + DataType.from_arrow(pa.int64()), + DataType.from_numpy(np.dtype("float32")), + ], + ) + def test_to_python_type_failure(self, non_python_dt): + """Test to_python_type raises ValueError for non-Python types.""" + with pytest.raises(ValueError, match="is not a Python type"): + non_python_dt.to_python_type() + + +class TestDataTypeInference: + """Test type inference from values.""" + + @pytest.mark.parametrize( + "numpy_value,expected_dtype", + [ + (np.array([1, 2, 3], dtype="int32"), np.dtype("int32")), + (np.array([1.0, 2.0], dtype="float64"), np.dtype("float64")), + (np.int64(42), np.dtype("int64")), + (np.float32(3.14), np.dtype("float32")), + ], + ) + def test_infer_dtype_numpy_values(self, numpy_value, expected_dtype): + """Test inference of NumPy arrays and scalars.""" + dt = DataType.infer_dtype(numpy_value) + + assert dt.is_numpy_type() + assert dt._internal_type == expected_dtype + + # Removed test_infer_dtype_pyarrow_scalar - no longer works with current implementation + + @pytest.mark.parametrize( + "python_value", + [ + 42, # int + 3.14, # float + "hello", # str + True, # bool + [1, 2, 3], # list + ], + ) + def test_infer_dtype_python_values_arrow_success(self, python_value): + """Test inference of Python values that Arrow can handle.""" + dt = DataType.infer_dtype(python_value) + + # Should infer to Arrow type for basic Python values + assert dt.is_arrow_type() + + # Removed test_infer_dtype_fallback_to_python_type - no longer supported + + +class TestDataTypeStringRepresentation: + """Test string representation methods.""" + + @pytest.mark.parametrize( + "datatype,expected_repr", + [ + (DataType.from_arrow(pa.int64()), "DataType(arrow:int64)"), + (DataType.from_arrow(pa.string()), "DataType(arrow:string)"), + (DataType.from_numpy(np.dtype("float32")), "DataType(numpy:float32)"), + (DataType.from_numpy(np.dtype("int64")), "DataType(numpy:int64)"), + (DataType(str), "DataType(python:str)"), + (DataType(int), "DataType(python:int)"), + ], + ) + def test_repr(self, datatype, expected_repr): + """Test __repr__ method for different type categories.""" + assert repr(datatype) == expected_repr + + +class TestDataTypeEqualityAndHashing: + """Test equality and hashing behavior.""" + + @pytest.mark.parametrize( + "dt1,dt2,should_be_equal", + [ + # Same types should be equal + (DataType.from_arrow(pa.int64()), DataType.from_arrow(pa.int64()), True), + ( + DataType.from_numpy(np.dtype("float32")), + DataType.from_numpy(np.dtype("float32")), + True, + ), + (DataType(str), DataType(str), True), + # Different Arrow types should not be equal + (DataType.from_arrow(pa.int64()), DataType.from_arrow(pa.int32()), False), + # Same conceptual type but different systems should not be equal + ( + DataType.from_arrow(pa.int64()), + DataType.from_numpy(np.dtype("int64")), + False, + ), + ], + ) + def test_equality(self, dt1, dt2, should_be_equal): + """Test __eq__ method.""" + if should_be_equal: + assert dt1 == dt2 + assert hash(dt1) == hash(dt2) + else: + assert dt1 != dt2 + + def test_numpy_vs_python_inequality(self): + """Test that numpy int64 and python int are not equal.""" + numpy_dt = DataType.from_numpy(np.dtype("int64")) + python_dt = DataType(int) + + # These represent the same conceptual type but with different systems + # so they should not be equal + + # First verify they have different internal types + assert type(numpy_dt._internal_type) is not type(python_dt._internal_type) + assert numpy_dt._internal_type is not python_dt._internal_type + + # Test the type checkers return different results + assert numpy_dt.is_numpy_type() and not python_dt.is_numpy_type() + assert python_dt.is_python_type() and not numpy_dt.is_python_type() + + # They should not be equal + assert numpy_dt != python_dt + + @pytest.mark.parametrize( + "non_datatype_value", + [ + "not_a_datatype", + 42, + [1, 2, 3], + {"key": "value"}, + None, + ], + ) + def test_inequality_with_non_datatype(self, non_datatype_value): + """Test that DataType is not equal to non-DataType objects.""" + dt = DataType.from_arrow(pa.int64()) + assert dt != non_datatype_value + + def test_hashability(self): + """Test that DataType objects can be used in sets and as dict keys.""" + dt1 = DataType.from_arrow(pa.int64()) + dt2 = DataType.from_arrow(pa.int64()) # Same as dt1 + dt3 = DataType.from_arrow(pa.int32()) # Different + + # Test in set + dt_set = {dt1, dt2, dt3} + assert len(dt_set) == 2 # dt1 and dt2 are the same + + # Test as dict keys + dt_dict = {dt1: "first", dt3: "second"} + assert dt_dict[dt2] == "first" # dt2 should match dt1 + + +if __name__ == "__main__": + pytest.main(["-v", __file__]) diff --git a/python/ray/data/tests/test_map.py b/python/ray/data/tests/test_map.py index 6b72f2e6b972..4bf7550d7a1d 100644 --- a/python/ray/data/tests/test_map.py +++ b/python/ray/data/tests/test_map.py @@ -34,6 +34,7 @@ _MapActorContext, ) from ray.data.context import DataContext +from ray.data.datatype import DataType from ray.data.exceptions import UserCodeException from ray.data.expressions import col, lit, udf from ray.data.tests.conftest import * # noqa @@ -2377,35 +2378,35 @@ def test_with_column_multiple_expressions( [ # Single column UDF - add one to each value pytest.param( - lambda: udf()(lambda x: pc.add(x, 1)), + lambda: udf(DataType.int64())(lambda x: pc.add(x, 1)), "add_one", 1, # 0 + 1 = 1 id="single_column_add_one", ), # Single column UDF - multiply by 2 pytest.param( - lambda: udf()(lambda x: pc.multiply(x, 2)), + lambda: udf(DataType.int64())(lambda x: pc.multiply(x, 2)), "times_two", 0, # 0 * 2 = 0 id="single_column_multiply", ), # Single column UDF - square the value pytest.param( - lambda: udf()(lambda x: pc.multiply(x, x)), + lambda: udf(DataType.int64())(lambda x: pc.multiply(x, x)), "squared", 0, # 0 * 0 = 0 id="single_column_square", ), # Single column UDF with string return type pytest.param( - lambda: udf()(lambda x: pc.cast(x, pa.string())), + lambda: udf(DataType.string())(lambda x: pc.cast(x, pa.string())), "id_str", "0", # Convert 0 to "0" id="single_column_to_string", ), # Single column UDF with float return type pytest.param( - lambda: udf()(lambda x: pc.divide(x, 2.0)), + lambda: udf(DataType.float64())(lambda x: pc.divide(x, 2.0)), "half", 0.0, # 0 / 2.0 = 0.0 id="single_column_divide_float", @@ -2442,7 +2443,7 @@ def test_with_column_udf_single_column( pytest.param( { "data": [{"a": 1, "b": 2}, {"a": 3, "b": 4}], - "udf": lambda: udf()(lambda x, y: pc.add(x, y)), + "udf": lambda: udf(DataType.int64())(lambda x, y: pc.add(x, y)), "column_name": "sum_ab", "expected_first": 3, # 1 + 2 = 3 "expected_second": 7, # 3 + 4 = 7 @@ -2453,7 +2454,7 @@ def test_with_column_udf_single_column( pytest.param( { "data": [{"x": 2, "y": 3}, {"x": 4, "y": 5}], - "udf": lambda: udf()(lambda x, y: pc.multiply(x, y)), + "udf": lambda: udf(DataType.int64())(lambda x, y: pc.multiply(x, y)), "column_name": "product_xy", "expected_first": 6, # 2 * 3 = 6 "expected_second": 20, # 4 * 5 = 20 @@ -2467,7 +2468,7 @@ def test_with_column_udf_single_column( {"first": "John", "last": "Doe"}, {"first": "Jane", "last": "Smith"}, ], - "udf": lambda: udf()( + "udf": lambda: udf(DataType.string())( lambda first, last: pc.binary_join_element_wise(first, last, " ") ), "column_name": "full_name", @@ -2560,7 +2561,7 @@ def test_with_column_udf_in_complex_expressions( ds = ray.data.range(5) # Create a simple add_one UDF for use in expressions - @udf() + @udf(DataType.int64()) def add_one(x: pa.Array) -> pa.Array: return pc.add(x, 1) @@ -2586,15 +2587,15 @@ def test_with_column_udf_multiple_udfs( ds = ray.data.range(5) # Define multiple UDFs - @udf() + @udf(DataType.int64()) def add_one(x: pa.Array) -> pa.Array: return pc.add(x, 1) - @udf() + @udf(DataType.int64()) def multiply_by_two(x: pa.Array) -> pa.Array: return pc.multiply(x, 2) - @udf() + @udf(DataType.float64()) def divide_by_three(x: pa.Array) -> pa.Array: return pc.divide(x, 3.0) @@ -2635,7 +2636,7 @@ def test_with_column_mixed_udf_and_regular_expressions( ds = ray.data.range(5) # Define a UDF for testing - @udf() + @udf(DataType.int64()) def multiply_by_three(x: pa.Array) -> pa.Array: return pc.multiply(x, 3) @@ -2677,18 +2678,18 @@ def test_with_column_udf_invalid_return_type_validation( """Test that UDFs returning invalid types raise TypeError with clear message.""" ds = ray.data.range(3) - # Test UDF returning invalid type (dict) - @udf() + # Test UDF returning invalid type (dict) - expecting string but returning dict + @udf(DataType.string()) def invalid_dict_return(x: pa.Array) -> dict: return {"invalid": "return_type"} - # Test UDF returning invalid type (str) - @udf() + # Test UDF returning invalid type (str) - expecting string but returning plain str + @udf(DataType.string()) def invalid_str_return(x: pa.Array) -> str: return "invalid_string" - # Test UDF returning invalid type (int) - @udf() + # Test UDF returning invalid type (int) - expecting int64 but returning plain int + @udf(DataType.int64()) def invalid_int_return(x: pa.Array) -> int: return 42 From 1c74f5f9f906df13bd53990895ca0c6f9238426f Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Thu, 4 Sep 2025 12:49:53 -0700 Subject: [PATCH 452/634] [core] Making CancelWorkerLease RPC Fault Tolerant (#56195) Signed-off-by: joshlee --- .../task_submission/normal_task_submitter.cc | 13 ++-- src/ray/raylet/local_lease_manager.h | 1 + src/ray/raylet/node_manager.cc | 6 +- src/ray/raylet/node_manager.h | 9 +-- src/ray/raylet/tests/node_manager_test.cc | 74 +++++++++++++++++++ src/ray/raylet_client/node_manager_client.h | 9 ++- src/ray/raylet_client/raylet_client.cc | 2 +- 7 files changed, 96 insertions(+), 18 deletions(-) diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 00a9a71ec4ac..826e19904139 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -213,11 +213,14 @@ void NormalTaskSubmitter::CancelWorkerLeaseIfNeeded(const SchedulingKey &schedul // The cancellation request can fail if the raylet does not have // the request queued. This can happen if: a) due to message // reordering, the raylet has not yet received the worker lease - // request, or b) we have already returned the worker lease - // request. In the former case, we should try the cancellation - // request again. In the latter case, the in-flight lease request - // should already have been removed from our local state, so we no - // longer need to cancel. + // request, b) we have already returned the worker lease + // request, or c) the current request is a retry and the server response to + // the initial request was lost after cancelling the lease. In case a), we + // should try the cancellation request again. In case b), the in-flight lease + // request should already have been removed from our local state, so we no + // longer need to cancel. In case c), the response for ReturnWorkerLease + // should have already been triggered and the pending lease request will be + // cleaned up. CancelWorkerLeaseIfNeeded(scheduling_key); } }); diff --git a/src/ray/raylet/local_lease_manager.h b/src/ray/raylet/local_lease_manager.h index 595ce7d00786..e3ce995650d0 100644 --- a/src/ray/raylet/local_lease_manager.h +++ b/src/ray/raylet/local_lease_manager.h @@ -389,6 +389,7 @@ class LocalLeaseManager : public LocalLeaseManagerInterface { friend class LocalLeaseManagerTest; FRIEND_TEST(ClusterLeaseManagerTest, FeasibleToNonFeasible); FRIEND_TEST(LocalLeaseManagerTest, TestLeaseGrantingOrder); + friend size_t GetPendingLeaseWorkerCount(const LocalLeaseManager &local_lease_manager); }; } // namespace raylet } // namespace ray diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 83585d046e38..3503f3d3d7f4 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -2052,9 +2052,9 @@ void NodeManager::HandleCancelWorkerLease(rpc::CancelWorkerLeaseRequest request, rpc::SendReplyCallback send_reply_callback) { const LeaseID lease_id = LeaseID::FromBinary(request.lease_id()); bool canceled = cluster_lease_manager_.CancelLease(lease_id); - // The task cancellation failed if we did not have the task queued, since - // this means that we may not have received the task request yet. It is - // successful if we did have the task queued, since we have now replied to + // The lease cancellation failed if we did not have the lease queued, since + // this means that we may not have received the lease request yet. It is + // successful if we did have the lease queued, since we have now replied to // the client that requested the lease. reply->set_success(canceled); send_reply_callback(Status::OK(), nullptr, nullptr); diff --git a/src/ray/raylet/node_manager.h b/src/ray/raylet/node_manager.h index f9865e49735a..2cfae2004fa5 100644 --- a/src/ray/raylet/node_manager.h +++ b/src/ray/raylet/node_manager.h @@ -286,6 +286,10 @@ class NodeManager : public rpc::NodeManagerServiceHandler, rpc::ReturnWorkerLeaseReply *reply, rpc::SendReplyCallback send_reply_callback) override; + void HandleCancelWorkerLease(rpc::CancelWorkerLeaseRequest request, + rpc::CancelWorkerLeaseReply *reply, + rpc::SendReplyCallback send_reply_callback) override; + private: FRIEND_TEST(NodeManagerStaticTest, TestHandleReportWorkerBacklog); @@ -582,11 +586,6 @@ class NodeManager : public rpc::NodeManagerServiceHandler, rpc::IsLocalWorkerDeadReply *reply, rpc::SendReplyCallback send_reply_callback) override; - /// Handle a `CancelWorkerLease` request. - void HandleCancelWorkerLease(rpc::CancelWorkerLeaseRequest request, - rpc::CancelWorkerLeaseReply *reply, - rpc::SendReplyCallback send_reply_callback) override; - /// Handle a `NodeStats` request. void HandleGetNodeStats(rpc::GetNodeStatsRequest request, rpc::GetNodeStatsReply *reply, diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index 2a35f075c423..5cda36052449 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -999,6 +999,80 @@ INSTANTIATE_TEST_SUITE_P(NodeManagerReturnWorkerLeaseIdempotentVariations, NodeManagerReturnWorkerLeaseIdempotentTest, testing::Combine(testing::Bool(), testing::Bool())); +size_t GetPendingLeaseWorkerCount(const LocalLeaseManager &local_lease_manager) { + return local_lease_manager.waiting_lease_queue_.size() + + local_lease_manager.leases_to_grant_.size(); +} + +TEST_F(NodeManagerTest, RetryHandleCancelWorkerLeaseWhenHasLeaseRequest) { + auto lease_spec = BuildLeaseSpec({}); + rpc::RequestWorkerLeaseRequest request_worker_lease_request; + rpc::RequestWorkerLeaseReply request_worker_lease_reply; + LeaseID lease_id = LeaseID::FromRandom(); + lease_spec.GetMutableMessage().set_lease_id(lease_id.Binary()); + request_worker_lease_request.mutable_lease_spec()->CopyFrom(lease_spec.GetMessage()); + request_worker_lease_request.set_backlog_size(1); + request_worker_lease_request.set_grant_or_reject(true); + request_worker_lease_request.set_is_selected_based_on_locality(true); + node_manager_->HandleRequestWorkerLease( + request_worker_lease_request, + &request_worker_lease_reply, + [](Status s, std::function success, std::function failure) { + ASSERT_TRUE(s.ok()); + }); + ASSERT_EQ(GetPendingLeaseWorkerCount(*local_lease_manager_), 1); + rpc::CancelWorkerLeaseRequest cancel_worker_lease_request; + cancel_worker_lease_request.set_lease_id(lease_id.Binary()); + rpc::CancelWorkerLeaseReply cancel_worker_lease_reply1; + rpc::CancelWorkerLeaseReply cancel_worker_lease_reply2; + node_manager_->HandleCancelWorkerLease( + cancel_worker_lease_request, + &cancel_worker_lease_reply1, + [](Status s, std::function success, std::function failure) { + ASSERT_TRUE(s.ok()); + }); + ASSERT_EQ(GetPendingLeaseWorkerCount(*local_lease_manager_), 0); + node_manager_->HandleCancelWorkerLease( + cancel_worker_lease_request, + &cancel_worker_lease_reply2, + [](Status s, std::function success, std::function failure) { + ASSERT_TRUE(s.ok()); + }); + ASSERT_EQ(GetPendingLeaseWorkerCount(*local_lease_manager_), 0); + ASSERT_EQ(cancel_worker_lease_reply1.success(), true); + // Due to the message reordering case where the cancel worker lease request + // arrives at the raylet before the worker lease request has been received, we + // cannot return true on the retry since from the raylet perspective both situations are + // equivalent. Even if this returns false, the first request to HandleCancelWorkerLease + // will trigger the callback for HandleRequestWorkerLease and remove the pending lease + // request which prevents the CancelWorkerLease loop. + ASSERT_EQ(cancel_worker_lease_reply2.success(), false); +} + +TEST_F(NodeManagerTest, TestHandleCancelWorkerLeaseNoLeaseIdempotent) { + LeaseID lease_id = LeaseID::FromRandom(); + rpc::CancelWorkerLeaseRequest request; + request.set_lease_id(lease_id.Binary()); + rpc::CancelWorkerLeaseReply reply1; + rpc::CancelWorkerLeaseReply reply2; + node_manager_->HandleCancelWorkerLease( + request, + &reply1, + [](Status s, std::function success, std::function failure) { + ASSERT_TRUE(s.ok()); + }); + ASSERT_EQ(GetPendingLeaseWorkerCount(*local_lease_manager_), 0); + node_manager_->HandleCancelWorkerLease( + request, + &reply2, + [](Status s, std::function success, std::function failure) { + ASSERT_TRUE(s.ok()); + }); + ASSERT_EQ(GetPendingLeaseWorkerCount(*local_lease_manager_), 0); + ASSERT_EQ(reply1.success(), false); + ASSERT_EQ(reply2.success(), false); +} + } // namespace ray::raylet int main(int argc, char **argv) { diff --git a/src/ray/raylet_client/node_manager_client.h b/src/ray/raylet_client/node_manager_client.h index 6887a4a55ad8..60558226555b 100644 --- a/src/ray/raylet_client/node_manager_client.h +++ b/src/ray/raylet_client/node_manager_client.h @@ -131,10 +131,11 @@ class NodeManagerClient { grpc_client_, /*method_timeout_ms*/ -1, ) - VOID_RPC_CLIENT_METHOD(NodeManagerService, - CancelWorkerLease, - grpc_client_, - /*method_timeout_ms*/ -1, ) + VOID_RETRYABLE_RPC_CLIENT_METHOD(retryable_grpc_client_, + NodeManagerService, + CancelWorkerLease, + grpc_client_, + /*method_timeout_ms*/ -1, ) VOID_RPC_CLIENT_METHOD(NodeManagerService, PrepareBundleResources, diff --git a/src/ray/raylet_client/raylet_client.cc b/src/ray/raylet_client/raylet_client.cc index 79c5496f4794..a4b825f7ee7f 100644 --- a/src/ray/raylet_client/raylet_client.cc +++ b/src/ray/raylet_client/raylet_client.cc @@ -191,7 +191,7 @@ void RayletClient::CancelWorkerLease( const rpc::ClientCallback &callback) { rpc::CancelWorkerLeaseRequest request; request.set_lease_id(lease_id.Binary()); - grpc_client_->CancelWorkerLease(request, callback); + grpc_client_->CancelWorkerLease(std::move(request), callback); } void RayletClient::PrepareBundleResources( From dbbfc69e3af0630ce30f97b5be45e6eab4c25409 Mon Sep 17 00:00:00 2001 From: Yun Tang Date: Fri, 5 Sep 2025 04:50:20 +0800 Subject: [PATCH 453/634] [dashboard][train] Catch OSError when detecting the GPU (#56158) GPU profiling failed with OSError when attempting `nvidia-smi` to detect whether there are GPUs available. This crashes the dashboard agent which prevents Ray from starting successfully. This PR catches all exceptions so that this optional GPU profiling feature doesn't prevent Ray from starting. --------- Signed-off-by: Yun Tang Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Jiajun Yao --- python/ray/dashboard/modules/reporter/gpu_profile_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/dashboard/modules/reporter/gpu_profile_manager.py b/python/ray/dashboard/modules/reporter/gpu_profile_manager.py index de66113b86dd..3c0fe01b7402 100644 --- a/python/ray/dashboard/modules/reporter/gpu_profile_manager.py +++ b/python/ray/dashboard/modules/reporter/gpu_profile_manager.py @@ -106,7 +106,7 @@ def node_has_gpus(cls) -> bool: try: subprocess.check_output(["nvidia-smi"], stderr=subprocess.DEVNULL) return True - except (subprocess.CalledProcessError, FileNotFoundError): + except Exception: return False @classmethod From 3346d67c35e4af9b5662d428f7ed1ed10a441ec3 Mon Sep 17 00:00:00 2001 From: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Date: Thu, 4 Sep 2025 13:58:19 -0700 Subject: [PATCH 454/634] [Train] ignore tensorflow test for py312 (#56244) Signed-off-by: xgui --- python/ray/train/v2/tests/test_local_mode.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/ray/train/v2/tests/test_local_mode.py b/python/ray/train/v2/tests/test_local_mode.py index 3f690c8ee09f..9b22a8e6daef 100644 --- a/python/ray/train/v2/tests/test_local_mode.py +++ b/python/ray/train/v2/tests/test_local_mode.py @@ -253,6 +253,10 @@ def train_loop(): assert "val_loss" in results.metrics +@pytest.mark.skipif( + sys.version_info >= (3, 12), + reason="Tensorflow is not installed for Python 3.12 because of keras compatibility.", +) def test_tensorflow_linear_local_mode(ray_start_4_cpus): """Also tests air Keras callback.""" epochs = 1 From fd63d2036d5d9bbd4835d6e053a30d017358adeb Mon Sep 17 00:00:00 2001 From: Lehui Liu Date: Thu, 4 Sep 2025 14:10:40 -0700 Subject: [PATCH 455/634] [Tune] Increase tune checkpoint test latency threshold (#56251) Bumping it to 5s to de-flake. Signed-off-by: Lehui Liu --- python/ray/tune/tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/tune/tests/test_commands.py b/python/ray/tune/tests/test_commands.py index f6615454cb07..b649c65ed59c 100644 --- a/python/ray/tune/tests/test_commands.py +++ b/python/ray/tune/tests/test_commands.py @@ -73,7 +73,7 @@ def train_fn(config): times += [time.time() - start] print("Average CLI time: ", sum(times) / len(times)) - assert sum(times) / len(times) < 2, "CLI is taking too long!" + assert sum(times) / len(times) < 5, "CLI is taking too long!" @mock.patch( From 86ba5d55fa7c9bd92befe1597b74a6eb12398d72 Mon Sep 17 00:00:00 2001 From: Kamil Kaczmarek Date: Thu, 4 Sep 2025 14:37:09 -0700 Subject: [PATCH 456/634] [RLlib] Add tests for the Footsies environment (#55041) ## Why are these changes needed? * RLlib tests for Footsies: multi-agent / self-play reinforcement learning environment (for two-players). ## Related issue number n.a. ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [x] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Kamil Kaczmarek Signed-off-by: Lonnie Liu Signed-off-by: avibasnet31 Signed-off-by: sampan Signed-off-by: harshit Signed-off-by: dragongu Signed-off-by: Balaji Veeramani Signed-off-by: Yiwen Xiang Signed-off-by: Yevet Signed-off-by: Linkun Chen Signed-off-by: lkchen Signed-off-by: Ryan O'Leary Signed-off-by: Ryan O'Leary <113500783+ryanaoleary@users.noreply.github.com> Signed-off-by: Potato Signed-off-by: cong.qian Signed-off-by: joshlee Signed-off-by: elliot-barn Signed-off-by: Stephanie wang Signed-off-by: Stephanie Wang Signed-off-by: Mengjin Yan Signed-off-by: Goku Mohandas Signed-off-by: doyoung Signed-off-by: dayshah Signed-off-by: Daraan Signed-off-by: Goutam V Signed-off-by: Rui Qiao Signed-off-by: Cuong Nguyen Signed-off-by: omkar Signed-off-by: Seiji Eicher Signed-off-by: lmsh7 Signed-off-by: lmsh7 <36391487+lmsh7@users.noreply.github.com> Signed-off-by: irabbani Signed-off-by: Ibrahim Rabbani Signed-off-by: Kamil Kaczmarek Signed-off-by: Matthew Deng Signed-off-by: xgui Signed-off-by: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Signed-off-by: Emanuele Petriglia Signed-off-by: Sampan S Nayak Signed-off-by: Chi-Sheng Liu Signed-off-by: Rueian Signed-off-by: Rueian Signed-off-by: zhilong Signed-off-by: zhaoch23 Signed-off-by: zhilong <121425509+Bye-legumes@users.noreply.github.com> Signed-off-by: Ibrahim Rabbani Signed-off-by: Krishna Kalyan Signed-off-by: myan Signed-off-by: abrar Signed-off-by: iamjustinhsu Signed-off-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Signed-off-by: Edward Oakes Signed-off-by: avigyabb Signed-off-by: akyang-anyscale Signed-off-by: Lehui Liu Signed-off-by: fscnick Signed-off-by: Alexey Kudinkin Signed-off-by: Ricardo Decal Signed-off-by: Matthew Owen Signed-off-by: JasonLi1909 Signed-off-by: Jason Li <57246540+JasonLi1909@users.noreply.github.com> Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Co-authored-by: avibasnet31 Co-authored-by: Dhyey Shah Co-authored-by: Sampan S Nayak Co-authored-by: sampan Co-authored-by: harshit-anyscale Co-authored-by: dragongu <38997200+dragongu@users.noreply.github.com> Co-authored-by: Balaji Veeramani Co-authored-by: Yevet Co-authored-by: Kai-Hsun Chen Co-authored-by: lkchen Co-authored-by: Ryan O'Leary <113500783+ryanaoleary@users.noreply.github.com> Co-authored-by: Jiajun Yao Co-authored-by: Mengjin Yan Co-authored-by: Potato Co-authored-by: coqian <1136656767@qq.com> Co-authored-by: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Co-authored-by: Elliot Barnwell Co-authored-by: Stephanie Wang Co-authored-by: Edward Oakes Co-authored-by: Goku Mohandas Co-authored-by: angelinalg <122562471+angelinalg@users.noreply.github.com> Co-authored-by: Masoud Co-authored-by: Rueian Co-authored-by: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Co-authored-by: Daniel Sperber Co-authored-by: Sven Mika Co-authored-by: goutamvenkat-anyscale Co-authored-by: Rui Qiao <161574667+ruisearch42@users.noreply.github.com> Co-authored-by: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Co-authored-by: Omkar Kulkarni Co-authored-by: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Co-authored-by: lmsh7 <36391487+lmsh7@users.noreply.github.com> Co-authored-by: Ibrahim Rabbani Co-authored-by: matthewdeng Co-authored-by: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Co-authored-by: Justin Yu Co-authored-by: Emanuele Petriglia Co-authored-by: Chi-Sheng Liu Co-authored-by: Rueian Co-authored-by: zhilong <121425509+Bye-legumes@users.noreply.github.com> Co-authored-by: zhaoch23 Co-authored-by: Krishna Kalyan Co-authored-by: Nikhil G Co-authored-by: Qiaolin Yu Co-authored-by: Abrar Sheikh Co-authored-by: Saihajpreet Singh Co-authored-by: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Co-authored-by: Seiji Eicher Co-authored-by: Balaji Veeramani Co-authored-by: avigyabb <98926738+avigyabb@users.noreply.github.com> Co-authored-by: akyang-anyscale Co-authored-by: Lehui Liu Co-authored-by: fscnick <6858627+fscnick@users.noreply.github.com> Co-authored-by: Alexey Kudinkin Co-authored-by: Ricardo Decal Co-authored-by: Matthew Owen Co-authored-by: iamjustinhsu Co-authored-by: Jason Li <57246540+JasonLi1909@users.noreply.github.com> Co-authored-by: simonsays1980 --- doc/source/rllib/rllib-examples.rst | 5 + pyproject.toml | 2 + rllib/BUILD | 137 ++++++-- rllib/algorithms/algorithm.py | 8 +- rllib/algorithms/algorithm_config.py | 1 + .../classes/multi_agent/footsies/README.md | 10 + .../classes/multi_agent/footsies/__init__.py | 0 .../classes/multi_agent/footsies/encoder.py | 225 ++++++++++++ .../multi_agent/footsies/fixed_rlmodules.py | 39 +++ .../multi_agent/footsies/footsies_env.py | 282 +++++++++++++++ .../multi_agent/footsies/game/__init__.py | 0 .../multi_agent/footsies/game/constants.py | 151 ++++++++ .../footsies/game/footsies_binary.py | 195 +++++++++++ .../footsies/game/footsies_game.py | 121 +++++++ .../game/proto/footsies_service.proto | 63 ++++ .../game/proto/footsies_service_pb2.py | 38 ++ .../game/proto/footsies_service_pb2_grpc.py | 307 ++++++++++++++++ .../classes/multi_agent/footsies/utils.py | 331 ++++++++++++++++++ .../multi_agent/self_play_footsies.py | 112 ++++++ .../ppo/multi_agent_footsies_ppo.py | 259 ++++++++++++++ 20 files changed, 2262 insertions(+), 24 deletions(-) create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/README.md create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/__init__.py create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/encoder.py create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/fixed_rlmodules.py create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/footsies_env.py create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/game/__init__.py create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/game/constants.py create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/game/footsies_binary.py create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/game/footsies_game.py create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service.proto create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service_pb2.py create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service_pb2_grpc.py create mode 100644 rllib/examples/envs/classes/multi_agent/footsies/utils.py create mode 100644 rllib/examples/multi_agent/self_play_footsies.py create mode 100644 rllib/tuned_examples/ppo/multi_agent_footsies_ppo.py diff --git a/doc/source/rllib/rllib-examples.rst b/doc/source/rllib/rllib-examples.rst index 50a894e33b2d..c2d15e49afa3 100644 --- a/doc/source/rllib/rllib-examples.rst +++ b/doc/source/rllib/rllib-examples.rst @@ -363,6 +363,11 @@ Multi-agent RL Uses OpenSpiel to demonstrate league-based self-play, where agents play against various versions of themselves, frozen or in-training, to improve through competitive interaction. +- `Self-play with Footsies and PPO algorithm `__: + Implements self-play with the Footsies environment (two player zero-sum game). + This example highlights RLlib's capabilities in connecting to the external binaries running the game engine, as well as + setting up a multi-agent self-play training scenario. + - `Self-play with OpenSpiel `__: Similar to the league-based self-play, but simpler. This script leverages OpenSpiel for two-player games, allowing agents to improve through direct self-play without building a complex, structured league. diff --git a/pyproject.toml b/pyproject.toml index 70e1ee4249aa..63ab2c1660d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,8 @@ extend-exclude = [ "python/build/", "python/ray/workflow/tests/mock_server.py", "python/ray/serve/tests/test_config_files/syntax_error.py", + "rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service_pb2.py", + "rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service_pb2_grpc.py", ] [tool.ruff.lint] diff --git a/rllib/BUILD b/rllib/BUILD index b35b343ef212..4dbfccb6c865 100644 --- a/rllib/BUILD +++ b/rllib/BUILD @@ -1538,6 +1538,86 @@ py_test( ], ) +# Footsies +py_test( + name = "learning_tests_multi_agent_footsies_ppo", + size = "large", + srcs = ["tuned_examples/ppo/multi_agent_footsies_ppo.py"], + args = [ + "--as-test", + "--num-env-runners=6", + "--evaluation-num-env-runners=2", + ], + main = "tuned_examples/ppo/multi_agent_footsies_ppo.py", + tags = [ + "exclusive", + "learning_tests", + "learning_tests_discrete", + "team:rllib", + ], +) + +py_test( + name = "learning_tests_multi_agent_footsies_ppo_gpu", + size = "large", + srcs = ["tuned_examples/ppo/multi_agent_footsies_ppo.py"], + args = [ + "--as-test", + "--num-env-runners=20", + "--evaluation-num-env-runners=3", + "--num-learners=1", + "--num-gpus-per-learner=1", + ], + main = "tuned_examples/ppo/multi_agent_footsies_ppo.py", + tags = [ + "exclusive", + "learning_tests", + "learning_tests_discrete", + "multi_gpu", + "team:rllib", + ], +) + +py_test( + name = "learning_tests_multi_agent_footsies_ppo_multi_cpu", + size = "large", + srcs = ["tuned_examples/ppo/multi_agent_footsies_ppo.py"], + args = [ + "--as-test", + "--num-env-runners=6", + "--evaluation-num-env-runners=2", + "--num-learners=2", + ], + main = "tuned_examples/ppo/multi_agent_footsies_ppo.py", + tags = [ + "exclusive", + "learning_tests", + "learning_tests_discrete", + "team:rllib", + ], +) + +py_test( + name = "learning_tests_multi_agent_footsies_ppo_multi_gpu", + size = "large", + srcs = ["tuned_examples/ppo/multi_agent_footsies_ppo.py"], + args = [ + "--as-test", + "--num-env-runners=20", + "--evaluation-num-env-runners=3", + "--num-learners=2", + "--num-gpus-per-learner=1", + ], + main = "tuned_examples/ppo/multi_agent_footsies_ppo.py", + tags = [ + "exclusive", + "learning_tests", + "learning_tests_discrete", + "multi_gpu", + "team:rllib", + ], +) + # Pendulum py_test( name = "learning_tests_pendulum_ppo", @@ -4084,14 +4164,14 @@ py_test( # subdirectory: envs/ # .................................... py_test( - name = "examples/envs/agents_act_simultaneously", + name = "examples/envs/agents_act_in_sequence", size = "medium", - srcs = ["examples/envs/agents_act_simultaneously.py"], + srcs = ["examples/envs/agents_act_in_sequence.py"], args = [ "--num-agents=2", "--stop-iters=3", ], - main = "examples/envs/agents_act_simultaneously.py", + main = "examples/envs/agents_act_in_sequence.py", tags = [ "examples", "exclusive", @@ -4100,14 +4180,14 @@ py_test( ) py_test( - name = "examples/envs/agents_act_in_sequence", + name = "examples/envs/agents_act_simultaneously", size = "medium", - srcs = ["examples/envs/agents_act_in_sequence.py"], + srcs = ["examples/envs/agents_act_simultaneously.py"], args = [ "--num-agents=2", "--stop-iters=3", ], - main = "examples/envs/agents_act_in_sequence.py", + main = "examples/envs/agents_act_simultaneously.py", tags = [ "examples", "exclusive", @@ -5014,13 +5094,34 @@ py_test( ) py_test( - name = "examples/multi_agent/shared_encoder_cartpole", - size = "medium", - srcs = ["examples/multi_agent/shared_encoder_cartpole.py"], + name = "examples/multi_agent/self_play_footsies", + size = "large", + srcs = ["examples/multi_agent/self_play_footsies.py"], args = [ - "--stop-iter=10", + "--as-test", + "--num-cpus=4", ], - main = "examples/multi_agent/shared_encoder_cartpole.py", + main = "examples/multi_agent/self_play_footsies.py", + tags = [ + "examples", + "examples_use_all_core", + "exclusive", + "team:rllib", + ], +) + +py_test( + name = "examples/multi_agent/self_play_league_based_with_open_spiel_connect_4_ppo_torch", + size = "large", + srcs = ["examples/multi_agent/self_play_league_based_with_open_spiel.py"], + args = [ + "--framework=torch", + "--env=connect_four", + "--win-rate-threshold=0.8", + "--num-episodes-human-play=0", + "--min-league-size=8", + ], + main = "examples/multi_agent/self_play_league_based_with_open_spiel.py", tags = [ "examples", "exclusive", @@ -5090,17 +5191,13 @@ py_test( ) py_test( - name = "examples/multi_agent/self_play_league_based_with_open_spiel_connect_4_ppo_torch", - size = "large", - srcs = ["examples/multi_agent/self_play_league_based_with_open_spiel.py"], + name = "examples/multi_agent/shared_encoder_cartpole", + size = "medium", + srcs = ["examples/multi_agent/shared_encoder_cartpole.py"], args = [ - "--framework=torch", - "--env=connect_four", - "--win-rate-threshold=0.8", - "--num-episodes-human-play=0", - "--min-league-size=8", + "--stop-iter=10", ], - main = "examples/multi_agent/self_play_league_based_with_open_spiel.py", + main = "examples/multi_agent/shared_encoder_cartpole.py", tags = [ "examples", "exclusive", diff --git a/rllib/algorithms/algorithm.py b/rllib/algorithms/algorithm.py index 4a01276ea040..d0ac2675ae11 100644 --- a/rllib/algorithms/algorithm.py +++ b/rllib/algorithms/algorithm.py @@ -2211,11 +2211,11 @@ def add_module( EnvRunnerGroup (with its o EnvRunners plus the local one). Returns: - The new MultiAgentRLModuleSpec (after the RLModule has been added). + The new MultiRLModuleSpec (after the RLModule has been added). """ validate_module_id(module_id, error=True) - # The to-be-returned new MultiAgentRLModuleSpec. + # The to-be-returned new MultiRLModuleSpec. multi_rl_module_spec = None if not self.config.is_multi_agent: @@ -2337,9 +2337,9 @@ def remove_module( EnvRunnerGroup (with its o EnvRunners plus the local one). Returns: - The new MultiAgentRLModuleSpec (after the RLModule has been removed). + The new MultiRLModuleSpec (after the RLModule has been removed). """ - # The to-be-returned new MultiAgentRLModuleSpec. + # The to-be-returned new MultiRLModuleSpec. multi_rl_module_spec = None # Remove RLModule from the LearnerGroup. diff --git a/rllib/algorithms/algorithm_config.py b/rllib/algorithms/algorithm_config.py index 018554b8540d..26067dc840ce 100644 --- a/rllib/algorithms/algorithm_config.py +++ b/rllib/algorithms/algorithm_config.py @@ -143,6 +143,7 @@ def DEFAULT_AGENT_TO_MODULE_MAPPING_FN(agent_id, episode): # Map any agent ID to "default_policy". return DEFAULT_MODULE_ID + # @OldAPIStack # TODO (sven): Deprecate in new API stack. @staticmethod def DEFAULT_POLICY_MAPPING_FN(aid, episode, worker, **kwargs): diff --git a/rllib/examples/envs/classes/multi_agent/footsies/README.md b/rllib/examples/envs/classes/multi_agent/footsies/README.md new file mode 100644 index 000000000000..6c9bec11c453 --- /dev/null +++ b/rllib/examples/envs/classes/multi_agent/footsies/README.md @@ -0,0 +1,10 @@ +# Footsies Environment + +This environment implementation is based on the [FootsiesGym project](https://github.com/chasemcd/FootsiesGym), +specifically the version as of **July 28, 2025**. + +## Notes + +All examples in the RLlib documentation that use the Footsies environment are self-contained. +This means that you do not need to install anything from the FootsiesGym repository or other places. +Examples handle binary automatically (downloading, extracting, starting, stopping, etc.). diff --git a/rllib/examples/envs/classes/multi_agent/footsies/__init__.py b/rllib/examples/envs/classes/multi_agent/footsies/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/rllib/examples/envs/classes/multi_agent/footsies/encoder.py b/rllib/examples/envs/classes/multi_agent/footsies/encoder.py new file mode 100644 index 000000000000..475e1574891e --- /dev/null +++ b/rllib/examples/envs/classes/multi_agent/footsies/encoder.py @@ -0,0 +1,225 @@ +import collections +import copy +from typing import Any, Optional, Union + +import numpy as np +from ray.rllib.examples.envs.classes.multi_agent.footsies.game import constants +from ray.rllib.examples.envs.classes.multi_agent.footsies.game.proto import ( + footsies_service_pb2 as footsies_pb2, +) + + +class FootsiesEncoder: + """Encoder class to generate observations from the game state""" + + def __init__(self, observation_delay: int): + self._encoding_history = { + agent_id: collections.deque(maxlen=int(observation_delay)) + for agent_id in ["p1", "p2"] + } + self.observation_delay = observation_delay + self._last_common_state: Optional[np.ndarray] = None + self._action_id_values = list(constants.FOOTSIES_ACTION_IDS.values()) + + @staticmethod + def encode_common_state(game_state: footsies_pb2.GameState) -> np.ndarray: + p1_state, p2_state = game_state.player1, game_state.player2 + + dist_x = np.abs(p1_state.player_position_x - p2_state.player_position_x) / 8.0 + + return np.array( + [ + dist_x, + ], + dtype=np.float32, + ) + + @staticmethod + def _encode_input_buffer( + input_buffer: list[int], last_n: Optional[int] = None + ) -> np.ndarray: + """Encodes the input buffer into a one-hot vector. + + :param input_buffer: The input buffer to encode + :type input_buffer: list[int] + :return: The encoded one-hot vector + :rtype: np.ndarray + """ + + if last_n is not None: + input_buffer = input_buffer[last_n:] + + ib_encoding = [] + for action_id in input_buffer: + arr = [0] * (len(constants.ACTION_TO_BITS) + 1) + arr[action_id] = 1 + ib_encoding.extend(arr) + + input_buffer_vector = np.asarray(ib_encoding, dtype=np.float32) + + return input_buffer_vector + + def encode( + self, + game_state: footsies_pb2.GameState, + ) -> dict[str, Any]: + """Encodes the game state into observations for all agents. + + :param game_state: The game state to encode + :type game_state: footsies_pb2.GameState + :return: The encoded observations for all agents. + :rtype: dict[str, Any] + """ + common_state = self.encode_common_state(game_state) + p1_encoding = self.encode_player_state(game_state.player1) + p2_encoding = self.encode_player_state(game_state.player2) + + observation_delay = min( + self.observation_delay, len(self._encoding_history["p1"]) + ) + + if observation_delay > 0: + p1_delayed_encoding = self._encoding_history["p1"][-observation_delay] + p2_delayed_encoding = self._encoding_history["p2"][-observation_delay] + else: + p1_delayed_encoding = copy.deepcopy(p1_encoding) + p2_delayed_encoding = copy.deepcopy(p2_encoding) + + self._encoding_history["p1"].append(p1_encoding) + self._encoding_history["p2"].append(p2_encoding) + self._last_common_state = common_state + + # Create features dictionary + features = {} + current_index = 0 + + # Common state + features["common_state"] = { + "start": current_index, + "length": len(common_state), + } + current_index += len(common_state) + + # Concatenate the observations for the undelayed encoding + p1_encoding = np.hstack(list(p1_encoding.values()), dtype=np.float32) + p2_encoding = np.hstack(list(p2_encoding.values()), dtype=np.float32) + + # Concatenate the observations for the delayed encoding + p1_delayed_encoding = np.hstack( + list(p1_delayed_encoding.values()), dtype=np.float32 + ) + p2_delayed_encoding = np.hstack( + list(p2_delayed_encoding.values()), dtype=np.float32 + ) + + p1_centric_observation = np.hstack( + [common_state, p1_encoding, p2_delayed_encoding] + ) + + p2_centric_observation = np.hstack( + [common_state, p2_encoding, p1_delayed_encoding] + ) + + return {"p1": p1_centric_observation, "p2": p2_centric_observation} + + def encode_player_state( + self, + player_state: footsies_pb2.PlayerState, + ) -> dict[str, Union[int, float, list, np.ndarray]]: + """Encodes the player state into observations. + + :param player_state: The player state to encode + :type player_state: footsies_pb2.PlayerState + :return: The encoded observations for the player + :rtype: dict[str, Any] + """ + feature_dict = { + "player_position_x": player_state.player_position_x + / constants.FeatureDictNormalizers.PLAYER_POSITION_X, + "velocity_x": player_state.velocity_x + / constants.FeatureDictNormalizers.VELOCITY_X, + "is_dead": int(player_state.is_dead), + "vital_health": player_state.vital_health, + "guard_health": one_hot_encoder(player_state.guard_health, [0, 1, 2, 3]), + "current_action_id": self._encode_action_id(player_state.current_action_id), + "current_action_frame": player_state.current_action_frame + / constants.FeatureDictNormalizers.CURRENT_ACTION_FRAME, + "current_action_frame_count": player_state.current_action_frame_count + / constants.FeatureDictNormalizers.CURRENT_ACTION_FRAME_COUNT, + "current_action_remaining_frames": ( + player_state.current_action_frame_count + - player_state.current_action_frame + ) + / constants.FeatureDictNormalizers.CURRENT_ACTION_REMAINING_FRAMES, + "is_action_end": int(player_state.is_action_end), + "is_always_cancelable": int(player_state.is_always_cancelable), + "current_action_hit_count": player_state.current_action_hit_count, + "current_hit_stun_frame": player_state.current_hit_stun_frame + / constants.FeatureDictNormalizers.CURRENT_HIT_STUN_FRAME, + "is_in_hit_stun": int(player_state.is_in_hit_stun), + "sprite_shake_position": player_state.sprite_shake_position, + "max_sprite_shake_frame": player_state.max_sprite_shake_frame + / constants.FeatureDictNormalizers.MAX_SPRITE_SHAKE_FRAME, + "is_face_right": int(player_state.is_face_right), + "current_frame_advantage": player_state.current_frame_advantage + / constants.FeatureDictNormalizers.CURRENT_FRAME_ADVANTAGE, + # The below features leak some information about the opponent! + "would_next_forward_input_dash": int( + player_state.would_next_forward_input_dash + ), + "would_next_backward_input_dash": int( + player_state.would_next_backward_input_dash + ), + "special_attack_progress": min(player_state.special_attack_progress, 1.0), + } + + return feature_dict + + def get_last_encoding(self) -> Optional[dict[str, np.ndarray]]: + if self._last_common_state is None: + return None + + return { + "common_state": self._last_common_state.reshape(-1), + "p1": np.hstack( + list(self._encoding_history["p1"][-1].values()), + dtype=np.float32, + ), + "p2": np.hstack( + list(self._encoding_history["p2"][-1].values()), + dtype=np.float32, + ), + } + + def reset(self): + self._encoding_history = { + agent_id: collections.deque(maxlen=int(self.observation_delay)) + for agent_id in ["p1", "p2"] + } + + def _encode_action_id(self, action_id: int) -> np.ndarray: + """Encodes the action id into a one-hot vector. + + :param action_id: The action id to encode + :type action_id: int + :return: The encoded one-hot vector + :rtype: np.ndarray + """ + + action_vector = np.zeros(len(self._action_id_values), dtype=np.float32) + + # Get the index of the action id in constants.ActionID + action_index = self._action_id_values.index(action_id) + action_vector[action_index] = 1 + + assert action_vector.max() == 1 and action_vector.min() == 0 + + return action_vector + + +def one_hot_encoder( + value: Union[int, float, str], collection: list[Union[int, float, str]] +) -> np.ndarray: + vector = np.zeros(len(collection), dtype=np.float32) + vector[collection.index(value)] = 1 + return vector diff --git a/rllib/examples/envs/classes/multi_agent/footsies/fixed_rlmodules.py b/rllib/examples/envs/classes/multi_agent/footsies/fixed_rlmodules.py new file mode 100644 index 000000000000..cf9030a96ad8 --- /dev/null +++ b/rllib/examples/envs/classes/multi_agent/footsies/fixed_rlmodules.py @@ -0,0 +1,39 @@ +import tree # pip install dm_tree +from ray.rllib.core.rl_module import RLModule +from ray.rllib.examples.envs.classes.multi_agent.footsies.game import constants +from ray.rllib.policy import sample_batch +from ray.rllib.utils.spaces.space_utils import batch as batch_func + + +class FixedRLModule(RLModule): + def _forward_inference(self, batch, **kwargs): + return self._fixed_forward(batch, **kwargs) + + def _forward_exploration(self, batch, **kwargs): + return self._fixed_forward(batch, **kwargs) + + def _forward_train(self, *args, **kwargs): + raise NotImplementedError( + f"RLlib: {self.__class__.__name__} should not be trained. " + f"It is a fixed RLModule, returning a fixed action for all observations." + ) + + def _fixed_forward(self, batch, **kwargs): + """Implements a fixed that always returns the same action.""" + raise NotImplementedError( + "FixedRLModule: This method should be overridden by subclasses to implement a specific action." + ) + + +class NoopFixedRLModule(FixedRLModule): + def _fixed_forward(self, batch, **kwargs): + obs_batch_size = len(tree.flatten(batch[sample_batch.SampleBatch.OBS])[0]) + actions = batch_func([constants.EnvActions.NONE for _ in range(obs_batch_size)]) + return {sample_batch.SampleBatch.ACTIONS: actions} + + +class BackFixedRLModule(FixedRLModule): + def _fixed_forward(self, batch, **kwargs): + obs_batch_size = len(tree.flatten(batch[sample_batch.SampleBatch.OBS])[0]) + actions = batch_func([constants.EnvActions.BACK for _ in range(obs_batch_size)]) + return {sample_batch.SampleBatch.ACTIONS: actions} diff --git a/rllib/examples/envs/classes/multi_agent/footsies/footsies_env.py b/rllib/examples/envs/classes/multi_agent/footsies/footsies_env.py new file mode 100644 index 000000000000..c08c0bce570e --- /dev/null +++ b/rllib/examples/envs/classes/multi_agent/footsies/footsies_env.py @@ -0,0 +1,282 @@ +import logging +from typing import Any, Optional + +import numpy as np +import psutil +from gymnasium import spaces +from pettingzoo.utils.env import ( + AgentID, + ActionType, + ObsType, +) +from ray.rllib.env import EnvContext +from ray.rllib.env.multi_agent_env import MultiAgentEnv +from ray.rllib.examples.envs.classes.multi_agent.footsies.encoder import FootsiesEncoder +from ray.rllib.examples.envs.classes.multi_agent.footsies.game import constants +from ray.rllib.examples.envs.classes.multi_agent.footsies.game.footsies_binary import ( + FootsiesBinary, +) +from ray.rllib.examples.envs.classes.multi_agent.footsies.game.footsies_game import ( + FootsiesGame, +) + +logger = logging.getLogger("ray.rllib") + + +class FootsiesEnv(MultiAgentEnv): + metadata = {"render.modes": ["human"]} + SPECIAL_CHARGE_FRAMES = 60 + GUARD_BREAK_REWARD = 0.3 + + observation_space = spaces.Dict( + { + agent: spaces.Box( + low=-np.inf, + high=np.inf, + shape=(constants.OBSERVATION_SPACE_SIZE,), + ) + for agent in ["p1", "p2"] + } + ) + + action_space = spaces.Dict( + { + agent: spaces.Discrete( + len( + [ + constants.EnvActions.NONE, + constants.EnvActions.BACK, + constants.EnvActions.FORWARD, + constants.EnvActions.ATTACK, + constants.EnvActions.BACK_ATTACK, + constants.EnvActions.FORWARD_ATTACK, + # This is a special input that holds down + # attack for 60 frames. It's just too long of a sequence + # to easily learn by holding ATTACK for so long. + constants.EnvActions.SPECIAL_CHARGE, + ] + ) + ) + for agent in ["p1", "p2"] + } + ) + + def __init__(self, config: EnvContext, port: int): + super().__init__() + + if config is None: + config = {} + self.config = config + self.port = port + self.footsies_process_pid = ( + None # Store PID of the running footsies process (we assume one per env) + ) + self.agents: list[AgentID] = ["p1", "p2"] + self.possible_agents: list[AgentID] = self.agents.copy() + self._agent_ids: set[AgentID] = set(self.agents) + + self.t: int = 0 + self.max_t: int = config.get("max_t", 1000) + self.frame_skip = config.get("frame_skip", 4) + observation_delay = config.get("observation_delay", 16) + + assert ( + observation_delay % self.frame_skip == 0 + ), "observation_delay must be divisible by frame_skip" + + self.encoder = FootsiesEncoder( + observation_delay=observation_delay // self.frame_skip + ) + + # start the game server before initializing the communication between the + # game server and the Python harness via gRPC + self._prepare_and_start_game_server() + self.game = FootsiesGame( + host=config["host"], + port=self.port, + ) + + self.last_game_state = None + self.special_charge_queue = { + "p1": -1, + "p2": -1, + } + + @staticmethod + def _convert_to_charge_action(action: int) -> int: + if action == constants.EnvActions.BACK: + return constants.EnvActions.BACK_ATTACK + elif action == constants.EnvActions.FORWARD: + return constants.EnvActions.FORWARD_ATTACK + else: + return constants.EnvActions.ATTACK + + def close(self): + """Terminate Footsies game server process. + + Run to ensure no game servers are left running. + """ + timeout = 2 + try: + logger.info( + f"RLlib {self.__class__.__name__}: Terminating Footsies " + f"game server process with PID: {self.footsies_process_pid}..." + ) + p = psutil.Process(self.footsies_process_pid) + p.terminate() + p.wait(timeout=timeout) + except psutil.NoSuchProcess: + logger.info( + f"RLlib {self.__class__.__name__}: Process with PID {self.footsies_process_pid} not found, " + f"it might have been already terminated." + ) + except psutil.TimeoutExpired: + logger.warning( + f"RLlib {self.__class__.__name__}: Process with PID {self.footsies_process_pid} did not terminate " + f"within {timeout} seconds. " + f"Sending SIGKILL signal instead.", + ) + p.kill() + p.wait(timeout=timeout) + + def get_infos(self): + return {agent: {} for agent in self.agents} + + def get_obs(self, game_state): + return self.encoder.encode(game_state) + + def reset( + self, + *, + seed: Optional[int] = None, + options: Optional[dict] = None, + ) -> tuple[dict[AgentID, ObsType], dict[AgentID, Any]]: + """Resets the environment to the starting state + and returns the initial observations for all agents. + + :return: Tuple of observations and infos for each agent. + :rtype: tuple[dict[AgentID, ObsType], dict[AgentID, Any]] + """ + self.t = 0 + self.game.reset_game() + self.game.start_game() + + self.encoder.reset() + self.last_game_state = self.game.get_state() + + observations = self.get_obs(self.last_game_state) + + return observations, {agent: {} for agent in self.agents} + + def step( + self, actions: dict[AgentID, ActionType] + ) -> tuple[ + dict[AgentID, ObsType], + dict[AgentID, float], + dict[AgentID, bool], + dict[AgentID, bool], + dict[AgentID, dict[str, Any]], + ]: + """Step the environment with the provided actions for all agents. + + :param actions: Dictionary mapping agent ids to their actions for this step. + :type actions: dict[AgentID, ActionType] + :return: Tuple of observations, rewards, terminates, truncateds and infos for all agents. + :rtype: tuple[ dict[AgentID, ObsType], dict[AgentID, float], dict[AgentID, bool], dict[AgentID, bool], dict[AgentID, dict[str, Any]], ] + """ + self.t += 1 + + for agent_id in self.agents: + empty_queue = self.special_charge_queue[agent_id] < 0 + action_is_special_charge = ( + actions[agent_id] == constants.EnvActions.SPECIAL_CHARGE + ) + + # Refill the charge queue only if we're not already in a special charge. + if action_is_special_charge and empty_queue: + self.special_charge_queue[ + agent_id + ] = self._build_charged_special_queue() + + if self.special_charge_queue[agent_id] >= 0: + self.special_charge_queue[agent_id] -= 1 + actions[agent_id] = self._convert_to_charge_action(actions[agent_id]) + + p1_action = self.game.action_to_bits(actions["p1"], is_player_1=True) + p2_action = self.game.action_to_bits(actions["p2"], is_player_1=False) + + game_state = self.game.step_n_frames( + p1_action=p1_action, p2_action=p2_action, n_frames=self.frame_skip + ) + observations = self.get_obs(game_state) + + terminated = game_state.player1.is_dead or game_state.player2.is_dead + + # Zero-sum game: 1 if other player is dead, -1 if you're dead: + rewards = { + "p1": int(game_state.player2.is_dead) - int(game_state.player1.is_dead), + "p2": int(game_state.player1.is_dead) - int(game_state.player2.is_dead), + } + + if self.config.get("reward_guard_break", False): + p1_prev_guard_health = self.last_game_state.player1.guard_health + p2_prev_guard_health = self.last_game_state.player2.guard_health + p1_guard_health = game_state.player1.guard_health + p2_guard_health = game_state.player2.guard_health + + if p2_guard_health < p2_prev_guard_health: + rewards["p1"] += self.GUARD_BREAK_REWARD + rewards["p2"] -= self.GUARD_BREAK_REWARD + if p1_guard_health < p1_prev_guard_health: + rewards["p2"] += self.GUARD_BREAK_REWARD + rewards["p1"] -= self.GUARD_BREAK_REWARD + + terminateds = { + "p1": terminated, + "p2": terminated, + "__all__": terminated, + } + + truncated = self.t >= self.max_t + truncateds = { + "p1": truncated, + "p2": truncated, + "__all__": truncated, + } + + self.last_game_state = game_state + + return observations, rewards, terminateds, truncateds, self.get_infos() + + def _build_charged_special_queue(self): + assert self.SPECIAL_CHARGE_FRAMES % self.frame_skip == 0 + steps_to_apply_attack = int(self.SPECIAL_CHARGE_FRAMES // self.frame_skip) + return steps_to_apply_attack + + def _prepare_and_start_game_server(self): + fb = FootsiesBinary(config=self.config, port=self.port) + self.footsies_process_pid = fb.start_game_server() + + +def env_creator(env_config: EnvContext) -> FootsiesEnv: + """Creates the Footsies environment + + Ensure that each game server runs on a unique port. Training and evaluation env runners have separate port ranges. + + Helper function to create the FootsiesEnv with a unique port based on the worker index and vector index. + It's usually passed to the `register_env()`, like this: register_env(name="FootsiesEnv", env_creator=env_creator). + """ + if env_config.get("env-for-evaluation", False): + port = ( + env_config["eval_start_port"] + - 1 # "-1" to start with eval_start_port as the first port (eval worker index starts at 1) + + int(env_config.worker_index) * env_config.get("num_envs_per_worker", 1) + + env_config.get("vector_index", 0) + ) + else: + port = ( + env_config["train_start_port"] + + int(env_config.worker_index) * env_config.get("num_envs_per_worker", 1) + + env_config.get("vector_index", 0) + ) + return FootsiesEnv(config=env_config, port=port) diff --git a/rllib/examples/envs/classes/multi_agent/footsies/game/__init__.py b/rllib/examples/envs/classes/multi_agent/footsies/game/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/rllib/examples/envs/classes/multi_agent/footsies/game/constants.py b/rllib/examples/envs/classes/multi_agent/footsies/game/constants.py new file mode 100644 index 000000000000..9a5c86065128 --- /dev/null +++ b/rllib/examples/envs/classes/multi_agent/footsies/game/constants.py @@ -0,0 +1,151 @@ +from dataclasses import dataclass + +OBSERVATION_SPACE_SIZE: int = 81 + + +@dataclass +class EnvActions: + NONE = 0 + BACK = 1 + FORWARD = 2 + ATTACK = 3 + BACK_ATTACK = 4 + FORWARD_ATTACK = 5 + SPECIAL_CHARGE = 6 + + +@dataclass +class GameActions: + NONE = 0 + LEFT = 1 + RIGHT = 2 + ATTACK = 3 + LEFT_ATTACK = 4 + RIGHT_ATTACK = 5 + + +@dataclass +class ActionBits: + NONE: int = 0 + LEFT: int = 1 << 0 + RIGHT: int = 1 << 1 + ATTACK: int = 1 << 2 + LEFT_ATTACK: int = LEFT | ATTACK + RIGHT_ATTACK: int = RIGHT | ATTACK + + +@dataclass +class ActionID: + STAND = 0 + FORWARD = 1 + BACKWARD = 2 + DASH_FORWARD = 10 + DASH_BACKWARD = 11 + N_ATTACK = 100 + B_ATTACK = 105 + N_SPECIAL = 110 + B_SPECIAL = 115 + DAMAGE = 200 + GUARD_M = 301 + GUARD_STAND = 305 + GUARD_CROUCH = 306 + GUARD_BREAK = 310 + GUARD_PROXIMITY = 350 + DEAD = 500 + WIN = 510 + + +@dataclass +class FeatureDictNormalizers: + PLAYER_POSITION_X = 4.0 + VELOCITY_X = 5.0 + CURRENT_ACTION_FRAME = 25 + CURRENT_ACTION_FRAME_COUNT = 25 + CURRENT_ACTION_REMAINING_FRAMES = 25 + CURRENT_HIT_STUN_FRAME = 10 + MAX_SPRITE_SHAKE_FRAME = 10 + CURRENT_FRAME_ADVANTAGE = 10 + + +ACTION_TO_BITS = { + GameActions.NONE: ActionBits.NONE, + GameActions.LEFT: ActionBits.LEFT, + GameActions.RIGHT: ActionBits.RIGHT, + GameActions.ATTACK: ActionBits.ATTACK, + GameActions.LEFT_ATTACK: ActionBits.LEFT_ATTACK, + GameActions.RIGHT_ATTACK: ActionBits.RIGHT_ATTACK, +} + +FOOTSIES_ACTION_IDS = { + "STAND": ActionID.STAND, + "FORWARD": ActionID.FORWARD, + "BACKWARD": ActionID.BACKWARD, + "DASH_FORWARD": ActionID.DASH_FORWARD, + "DASH_BACKWARD": ActionID.DASH_BACKWARD, + "N_ATTACK": ActionID.N_ATTACK, + "B_ATTACK": ActionID.B_ATTACK, + "N_SPECIAL": ActionID.N_SPECIAL, + "B_SPECIAL": ActionID.B_SPECIAL, + "DAMAGE": ActionID.DAMAGE, + "GUARD_M": ActionID.GUARD_M, + "GUARD_STAND": ActionID.GUARD_STAND, + "GUARD_CROUCH": ActionID.GUARD_CROUCH, + "GUARD_BREAK": ActionID.GUARD_BREAK, + "GUARD_PROXIMITY": ActionID.GUARD_PROXIMITY, + "DEAD": ActionID.DEAD, + "WIN": ActionID.WIN, +} + +# backup file location (uploaded July 29th, 2025): +# https://ray-example-data.s3.us-west-2.amazonaws.com/rllib/env-footsies/feature_indices.json +# Dictionary mapping feature names to their index ranges within a flat observation vector. +# Each key is a feature name, and its value is a dictionary with keys: +# "start": the starting index in the observation array. +# "length": it's length in bytes +feature_indices = { + "common_state": {"start": 0, "length": 1}, + "frame_count": {"start": 1, "length": 1}, + "player_position_x": {"start": 2, "length": 1}, + "velocity_x": {"start": 3, "length": 1}, + "is_dead": {"start": 4, "length": 1}, + "vital_health": {"start": 5, "length": 1}, + "guard_health": {"start": 6, "length": 4}, + "current_action_id": {"start": 10, "length": 17}, + "current_action_frame": {"start": 27, "length": 1}, + "current_action_frame_count": {"start": 28, "length": 1}, + "current_action_remaining_frames": {"start": 29, "length": 1}, + "is_action_end": {"start": 30, "length": 1}, + "is_always_cancelable": {"start": 31, "length": 1}, + "current_action_hit_count": {"start": 32, "length": 1}, + "current_hit_stun_frame": {"start": 33, "length": 1}, + "is_in_hit_stun": {"start": 34, "length": 1}, + "sprite_shake_position": {"start": 35, "length": 1}, + "max_sprite_shake_frame": {"start": 36, "length": 1}, + "is_face_right": {"start": 37, "length": 1}, + "current_frame_advantage": {"start": 38, "length": 1}, + "would_next_forward_input_dash": {"start": 39, "length": 1}, + "would_next_backward_input_dash": {"start": 40, "length": 1}, + "special_attack_progress": {"start": 41, "length": 1}, + "opponent_frame_count": {"start": 42, "length": 1}, + "opponent_player_position_x": {"start": 43, "length": 1}, + "opponent_velocity_x": {"start": 44, "length": 1}, + "opponent_is_dead": {"start": 45, "length": 1}, + "opponent_vital_health": {"start": 46, "length": 1}, + "opponent_guard_health": {"start": 47, "length": 4}, + "opponent_current_action_id": {"start": 51, "length": 17}, + "opponent_current_action_frame": {"start": 68, "length": 1}, + "opponent_current_action_frame_count": {"start": 69, "length": 1}, + "opponent_current_action_remaining_frames": {"start": 70, "length": 1}, + "opponent_is_action_end": {"start": 71, "length": 1}, + "opponent_is_always_cancelable": {"start": 72, "length": 1}, + "opponent_current_action_hit_count": {"start": 73, "length": 1}, + "opponent_current_hit_stun_frame": {"start": 74, "length": 1}, + "opponent_is_in_hit_stun": {"start": 75, "length": 1}, + "opponent_sprite_shake_position": {"start": 76, "length": 1}, + "opponent_max_sprite_shake_frame": {"start": 77, "length": 1}, + "opponent_is_face_right": {"start": 78, "length": 1}, + "opponent_current_frame_advantage": {"start": 79, "length": 1}, + "opponent_would_next_forward_input_dash": {"start": 80, "length": 1}, + "opponent_would_next_backward_input_dash": {"start": 81, "length": 1}, + "opponent_special_attack_progress": {"start": 82, "length": 1}, +} diff --git a/rllib/examples/envs/classes/multi_agent/footsies/game/footsies_binary.py b/rllib/examples/envs/classes/multi_agent/footsies/game/footsies_binary.py new file mode 100644 index 000000000000..2b9e3bcbc5b9 --- /dev/null +++ b/rllib/examples/envs/classes/multi_agent/footsies/game/footsies_binary.py @@ -0,0 +1,195 @@ +import logging +import os +import stat +import subprocess +import time +import zipfile +from dataclasses import dataclass +from pathlib import Path + +import grpc +import requests +from ray.rllib.env import EnvContext +from ray.rllib.examples.envs.classes.multi_agent.footsies.game.proto import ( + footsies_service_pb2 as footsies_pb2, +) +from ray.rllib.examples.envs.classes.multi_agent.footsies.game.proto import ( + footsies_service_pb2_grpc as footsies_pb2_grpc, +) + +logger = logging.getLogger(__name__) + + +@dataclass +class BinaryUrls: + # Uploaded 07.28.2025 + S3_ROOT = "https://ray-example-data.s3.us-west-2.amazonaws.com/rllib/env-footsies/binaries/" + + # Zip file names + ZIP_LINUX_SERVER = "footsies_linux_server_021725.zip" + ZIP_LINUX_WINDOWED = "footsies_linux_windowed_021725.zip" + ZIP_MAC_HEADLESS = "footsies_mac_headless_5709b6d.zip" + ZIP_MAC_WINDOWED = "footsies_mac_windowed_5709b6d.zip" + + # Full URLs + URL_LINUX_SERVER_BINARIES = S3_ROOT + ZIP_LINUX_SERVER + URL_LINUX_WINDOWED_BINARIES = S3_ROOT + ZIP_LINUX_WINDOWED + URL_MAC_HEADLESS_BINARIES = S3_ROOT + ZIP_MAC_HEADLESS + URL_MAC_WINDOWED_BINARIES = S3_ROOT + ZIP_MAC_WINDOWED + + +class FootsiesBinary: + def __init__(self, config: EnvContext, port: int): + self._urls = BinaryUrls() + self.config = config + self.port = port + self.binary_to_download = config["binary_to_download"] + if self.binary_to_download == "linux_server": + self.url = self._urls.URL_LINUX_SERVER_BINARIES + elif self.binary_to_download == "linux_windowed": + self.url = self._urls.URL_LINUX_WINDOWED_BINARIES + elif self.binary_to_download == "mac_headless": + self.url = self._urls.URL_MAC_HEADLESS_BINARIES + elif self.binary_to_download == "mac_windowed": + self.url = self._urls.URL_MAC_WINDOWED_BINARIES + else: + raise ValueError(f"Invalid target binary: {self.binary_to_download}") + + self.full_download_dir = Path(config["binary_download_dir"]).resolve() + self.full_download_path = ( + self.full_download_dir / str.split(self.url, sep="/")[-1] + ) + self.full_extract_dir = Path(config["binary_extract_dir"]).resolve() + self.renamed_path = self.full_extract_dir / "footsies_binaries" + + @staticmethod + def _add_executable_permission(binary_path: Path) -> None: + binary_path.chmod(binary_path.stat().st_mode | stat.S_IXUSR) + + def start_game_server(self) -> int: + """Downloads, unzips, and starts the Footsies game server binary. + + Returns footsies process PID. + """ + self._download_game_binary() + self._unzip_game_binary() + + if self.binary_to_download == "mac_windowed": + game_binary_path = ( + Path(self.renamed_path) / "Contents" / "MacOS" / "FOOTSIES" + ) + elif self.binary_to_download == "mac_headless": + game_binary_path = Path(self.renamed_path) / "FOOTSIES" + else: + game_binary_path = Path(self.renamed_path) / "footsies.x86_64" + + if os.access(game_binary_path, os.X_OK): + logger.info( + f"Game binary has an 'executable' permission: {game_binary_path}" + ) + else: + self._add_executable_permission(game_binary_path) + logger.info(f"Game binary path: {game_binary_path}") + + if ( + self.binary_to_download == "linux_server" + or self.binary_to_download == "linux_windowed" + ): + process = subprocess.Popen([game_binary_path, "--port", str(self.port)]) + else: + process = subprocess.Popen( + [ + "arch", + "-x86_64", + game_binary_path, + "--port", + str(self.port), + ], + ) + + # check if the game server is running correctly + timeout = 2 + channel = grpc.insecure_channel(f"localhost:{self.port}") + stub = footsies_pb2_grpc.FootsiesGameServiceStub(channel) + + # step 1: try to start the game + while True: + try: + stub.StartGame(footsies_pb2.Empty()) + logger.info("Game ready!") + break + except grpc.RpcError as e: + code = e.code() + if code in ( + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.DEADLINE_EXCEEDED, + ): + logger.info(f"RLlib {self.__class__.__name__}: Game not ready...") + time.sleep(timeout) + continue + raise + + # step 2: check if the game is ready + ready = False + while not ready: + try: + ready = stub.IsReady(footsies_pb2.Empty()).value + if not ready: + logger.info(f"RLlib {self.__class__.__name__}: Game not ready...") + time.sleep(timeout) + continue + else: + logger.info("Game ready!") + break + except grpc.RpcError as e: + if e.code() in ( + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.DEADLINE_EXCEEDED, + ): + time.sleep(timeout) + logger.info(f"RLlib {self.__class__.__name__}: Game not ready...") + continue + raise + + channel.close() + return process.pid + + def _download_game_binary(self): + chunk_size = 1024 * 1024 # 1MB + + if Path(self.full_download_path).exists(): + logger.info( + f"Game binary already exists at {self.full_download_path}, skipping download." + ) + + else: + try: + with requests.get(self.url, stream=True) as response: + response.raise_for_status() + self.full_download_dir.mkdir(parents=True, exist_ok=True) + with open(self.full_download_path, "wb") as f: + for chunk in response.iter_content(chunk_size=chunk_size): + if chunk: + f.write(chunk) + logger.info( + f"Downloaded game binary to {self.full_download_path}\n" + f"Binary size: {self.full_download_path.stat().st_size / 1024 / 1024:.1f} MB\n" + ) + except requests.exceptions.RequestException as e: + logger.error(f"Failed to download binary from {self.url}: {e}") + + def _unzip_game_binary(self): + if Path(self.renamed_path).exists(): + logger.info( + f"Game binary already extracted at {self.renamed_path}, skipping extraction." + ) + else: + self.full_extract_dir.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(self.full_download_path, mode="r") as zip_ref: + zip_ref.extractall(self.full_extract_dir) + + if self.binary_to_download == "mac_windowed": + self.full_download_path.with_suffix(".app").rename(self.renamed_path) + else: + self.full_download_path.with_suffix("").rename(self.renamed_path) + logger.info(f"Extracted game binary to {self.renamed_path}") diff --git a/rllib/examples/envs/classes/multi_agent/footsies/game/footsies_game.py b/rllib/examples/envs/classes/multi_agent/footsies/game/footsies_game.py new file mode 100644 index 000000000000..5f4252412958 --- /dev/null +++ b/rllib/examples/envs/classes/multi_agent/footsies/game/footsies_game.py @@ -0,0 +1,121 @@ +import logging +import time + +import grpc +import numpy as np + +import ray.rllib.examples.envs.classes.multi_agent.footsies.game.proto.footsies_service_pb2 as footsies_pb2 +import ray.rllib.examples.envs.classes.multi_agent.footsies.game.proto.footsies_service_pb2_grpc as footsies_pb2_grpc +from ray.rllib.examples.envs.classes.multi_agent.footsies.game import constants + +logger = logging.getLogger(__name__) + + +class FootsiesGame: + """Handles gRPC communication with game the server. + + This class establishes communication between the + game server and the Python harness via gRPC. It provides methods + to start the game, reset it, get the current state, and step the + game by a certain number of frames. + """ + + def __init__(self, host: str, port: int): + self.host = host + self.port = port + self.stub = self._initialize_stub() + + @staticmethod + def action_to_bits(action: int, is_player_1: bool) -> int: + """Converts an action to its corresponding bit representation.""" + + if isinstance(action, np.ndarray): + action = action.item() + + if is_player_1: + if action == constants.EnvActions.BACK: + action = constants.GameActions.LEFT + elif action == constants.EnvActions.FORWARD: + action = constants.GameActions.RIGHT + elif action == constants.EnvActions.BACK_ATTACK: + action = constants.GameActions.LEFT_ATTACK + elif action == constants.EnvActions.FORWARD_ATTACK: + action = constants.GameActions.RIGHT_ATTACK + else: + if action == constants.EnvActions.BACK: + action = constants.GameActions.RIGHT + elif action == constants.EnvActions.FORWARD: + action = constants.GameActions.LEFT + elif action == constants.EnvActions.BACK_ATTACK: + action = constants.GameActions.RIGHT_ATTACK + elif action == constants.EnvActions.FORWARD_ATTACK: + action = constants.GameActions.LEFT_ATTACK + + return constants.ACTION_TO_BITS[action] + + def get_encoded_state(self) -> footsies_pb2.EncodedGameState: + """Gets the current encoded game state by calling the GetEncodedState RPC.""" + try: + return self.stub.GetEncodedState(footsies_pb2.Empty()) + except Exception as e: + logger.error(f"Error calling GetEncodedState with exception: {e}") + raise e + + def get_state(self) -> footsies_pb2.GameState: + """Gets the current game state by calling the GetState RPC.""" + try: + return self.stub.GetState(footsies_pb2.Empty()) + except Exception as e: + logger.error(f"Error calling GetState with exception: {e}") + raise e + + def is_ready(self) -> bool: + """Checks if the game is ready by calling the IsReady RPC.""" + try: + return self.stub.IsReady(footsies_pb2.Empty()).value + except Exception as e: + logger.error(f"Error calling IsReady with exception: {e}") + raise e + + def reset_game(self) -> None: + """Resets the game by calling the ResetGame RPC.""" + try: + self.stub.ResetGame(footsies_pb2.Empty()) + except Exception as e: + logger.error(f"Error calling ResetGame with exception: {e}") + raise e + + def start_game(self) -> None: + """Starts the game by calling the StartGame RPC.""" + try: + self.stub.StartGame(footsies_pb2.Empty()) + + while not self.is_ready(): + logger.info("Game not ready...") + time.sleep(0.5) + logger.info("StartGame called successfully") + + except Exception as e: + logger.error(f"Error calling StartGame with exception: {e}") + raise e + + def step_n_frames( + self, p1_action: int, p2_action: int, n_frames: int + ) -> footsies_pb2.GameState: + """Steps the game by n_frames with the given player actions. The provided actions will be repeated for all n_frames.""" + try: + step_input = footsies_pb2.StepInput( + p1_action=p1_action, p2_action=p2_action, nFrames=n_frames + ) + return self.stub.StepNFrames(step_input) + except Exception as e: + logger.error(f"Error calling StepNFrames with exception: {e}") + raise e + + def _initialize_stub(self) -> footsies_pb2_grpc.FootsiesGameServiceStub: + try: + channel = grpc.insecure_channel(f"{self.host}:{self.port}") + return footsies_pb2_grpc.FootsiesGameServiceStub(channel) + except grpc.RpcError as e: + logger.error(f"Error connecting to gRPC stub with exception: {e}") + raise e diff --git a/rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service.proto b/rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service.proto new file mode 100644 index 000000000000..5edbd7bda692 --- /dev/null +++ b/rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service.proto @@ -0,0 +1,63 @@ +syntax = "proto3"; + + +service FootsiesGameService { + rpc StartGame(Empty) returns (Empty) {} + rpc ResetGame(Empty) returns (Empty) {} + rpc StepNFrames(StepInput) returns (GameState) {} + rpc GetState(Empty) returns (GameState) {} + rpc GetEncodedState(Empty) returns (EncodedGameState) {} + rpc IsReady(Empty) returns (BoolValue) {} +} + + +message StepInput { + int64 p1_action = 1; + int64 p2_action = 2; + int64 nFrames = 3; +} + +message PlayerState { + float player_position_x = 1; + bool is_dead = 2; + int64 vital_health = 3; + int64 guard_health = 4; + int64 current_action_id = 5; + int64 current_action_frame = 6; + int64 current_action_frame_count = 7; + bool is_action_end = 8; + bool is_always_cancelable = 9; + int64 current_action_hit_count = 10; + int64 current_hit_stun_frame = 11; + bool is_in_hit_stun = 12; + int64 sprite_shake_position = 13; + int64 max_sprite_shake_frame = 14; + float velocity_x = 15; + bool is_face_right = 16; + repeated int64 input_buffer = 17; + int64 current_frame_advantage = 18; + bool would_next_forward_input_dash = 19; + bool would_next_backward_input_dash = 20; + float special_attack_progress = 21; +} + +message GameState { + PlayerState player1 = 1; + PlayerState player2 = 2; + int64 round_state = 3; + int64 frame_count = 4; +} + +message EncodedGameState { + repeated float player1_encoding = 1; + repeated float player2_encoding = 2; +} + +message BoolValue { + bool value = 1; +} + + + + +message Empty {} diff --git a/rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service_pb2.py b/rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service_pb2.py new file mode 100644 index 000000000000..8dc26277dff8 --- /dev/null +++ b/rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service_pb2.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: footsies_service.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x16\x66ootsies_service.proto"B\n\tStepInput\x12\x11\n\tp1_action\x18\x01 \x01(\x03\x12\x11\n\tp2_action\x18\x02 \x01(\x03\x12\x0f\n\x07nFrames\x18\x03 \x01(\x03"\xe2\x04\n\x0bPlayerState\x12\x19\n\x11player_position_x\x18\x01 \x01(\x02\x12\x0f\n\x07is_dead\x18\x02 \x01(\x08\x12\x14\n\x0cvital_health\x18\x03 \x01(\x03\x12\x14\n\x0cguard_health\x18\x04 \x01(\x03\x12\x19\n\x11\x63urrent_action_id\x18\x05 \x01(\x03\x12\x1c\n\x14\x63urrent_action_frame\x18\x06 \x01(\x03\x12"\n\x1a\x63urrent_action_frame_count\x18\x07 \x01(\x03\x12\x15\n\ris_action_end\x18\x08 \x01(\x08\x12\x1c\n\x14is_always_cancelable\x18\t \x01(\x08\x12 \n\x18\x63urrent_action_hit_count\x18\n \x01(\x03\x12\x1e\n\x16\x63urrent_hit_stun_frame\x18\x0b \x01(\x03\x12\x16\n\x0eis_in_hit_stun\x18\x0c \x01(\x08\x12\x1d\n\x15sprite_shake_position\x18\r \x01(\x03\x12\x1e\n\x16max_sprite_shake_frame\x18\x0e \x01(\x03\x12\x12\n\nvelocity_x\x18\x0f \x01(\x02\x12\x15\n\ris_face_right\x18\x10 \x01(\x08\x12\x14\n\x0cinput_buffer\x18\x11 \x03(\x03\x12\x1f\n\x17\x63urrent_frame_advantage\x18\x12 \x01(\x03\x12%\n\x1dwould_next_forward_input_dash\x18\x13 \x01(\x08\x12&\n\x1ewould_next_backward_input_dash\x18\x14 \x01(\x08\x12\x1f\n\x17special_attack_progress\x18\x15 \x01(\x02"s\n\tGameState\x12\x1d\n\x07player1\x18\x01 \x01(\x0b\x32\x0c.PlayerState\x12\x1d\n\x07player2\x18\x02 \x01(\x0b\x32\x0c.PlayerState\x12\x13\n\x0bround_state\x18\x03 \x01(\x03\x12\x13\n\x0b\x66rame_count\x18\x04 \x01(\x03"F\n\x10\x45ncodedGameState\x12\x18\n\x10player1_encoding\x18\x01 \x03(\x02\x12\x18\n\x10player2_encoding\x18\x02 \x03(\x02"\x1a\n\tBoolValue\x12\r\n\x05value\x18\x01 \x01(\x08"\x07\n\x05\x45mpty2\xef\x01\n\x13\x46ootsiesGameService\x12\x1d\n\tStartGame\x12\x06.Empty\x1a\x06.Empty"\x00\x12\x1d\n\tResetGame\x12\x06.Empty\x1a\x06.Empty"\x00\x12\'\n\x0bStepNFrames\x12\n.StepInput\x1a\n.GameState"\x00\x12 \n\x08GetState\x12\x06.Empty\x1a\n.GameState"\x00\x12.\n\x0fGetEncodedState\x12\x06.Empty\x1a\x11.EncodedGameState"\x00\x12\x1f\n\x07IsReady\x12\x06.Empty\x1a\n.BoolValue"\x00\x62\x06proto3' +) + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "footsies_service_pb2", globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _STEPINPUT._serialized_start = 26 + _STEPINPUT._serialized_end = 92 + _PLAYERSTATE._serialized_start = 95 + _PLAYERSTATE._serialized_end = 705 + _GAMESTATE._serialized_start = 707 + _GAMESTATE._serialized_end = 822 + _ENCODEDGAMESTATE._serialized_start = 824 + _ENCODEDGAMESTATE._serialized_end = 894 + _BOOLVALUE._serialized_start = 896 + _BOOLVALUE._serialized_end = 922 + _EMPTY._serialized_start = 924 + _EMPTY._serialized_end = 931 + _FOOTSIESGAMESERVICE._serialized_start = 934 + _FOOTSIESGAMESERVICE._serialized_end = 1173 +# @@protoc_insertion_point(module_scope) diff --git a/rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service_pb2_grpc.py b/rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service_pb2_grpc.py new file mode 100644 index 000000000000..b39a76d7bf5a --- /dev/null +++ b/rllib/examples/envs/classes/multi_agent/footsies/game/proto/footsies_service_pb2_grpc.py @@ -0,0 +1,307 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import ray.rllib.examples.envs.classes.multi_agent.footsies.game.proto.footsies_service_pb2 as footsies__service__pb2 + + +# import footsies_service_pb2 as footsies__service__pb2 + + +class FootsiesGameServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.StartGame = channel.unary_unary( + "/FootsiesGameService/StartGame", + request_serializer=footsies__service__pb2.Empty.SerializeToString, + response_deserializer=footsies__service__pb2.Empty.FromString, + ) + self.ResetGame = channel.unary_unary( + "/FootsiesGameService/ResetGame", + request_serializer=footsies__service__pb2.Empty.SerializeToString, + response_deserializer=footsies__service__pb2.Empty.FromString, + ) + self.StepNFrames = channel.unary_unary( + "/FootsiesGameService/StepNFrames", + request_serializer=footsies__service__pb2.StepInput.SerializeToString, + response_deserializer=footsies__service__pb2.GameState.FromString, + ) + self.GetState = channel.unary_unary( + "/FootsiesGameService/GetState", + request_serializer=footsies__service__pb2.Empty.SerializeToString, + response_deserializer=footsies__service__pb2.GameState.FromString, + ) + self.GetEncodedState = channel.unary_unary( + "/FootsiesGameService/GetEncodedState", + request_serializer=footsies__service__pb2.Empty.SerializeToString, + response_deserializer=footsies__service__pb2.EncodedGameState.FromString, + ) + self.IsReady = channel.unary_unary( + "/FootsiesGameService/IsReady", + request_serializer=footsies__service__pb2.Empty.SerializeToString, + response_deserializer=footsies__service__pb2.BoolValue.FromString, + ) + + +class FootsiesGameServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def StartGame(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def ResetGame(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def StepNFrames(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def GetState(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def GetEncodedState(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def IsReady(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + +def add_FootsiesGameServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + "StartGame": grpc.unary_unary_rpc_method_handler( + servicer.StartGame, + request_deserializer=footsies__service__pb2.Empty.FromString, + response_serializer=footsies__service__pb2.Empty.SerializeToString, + ), + "ResetGame": grpc.unary_unary_rpc_method_handler( + servicer.ResetGame, + request_deserializer=footsies__service__pb2.Empty.FromString, + response_serializer=footsies__service__pb2.Empty.SerializeToString, + ), + "StepNFrames": grpc.unary_unary_rpc_method_handler( + servicer.StepNFrames, + request_deserializer=footsies__service__pb2.StepInput.FromString, + response_serializer=footsies__service__pb2.GameState.SerializeToString, + ), + "GetState": grpc.unary_unary_rpc_method_handler( + servicer.GetState, + request_deserializer=footsies__service__pb2.Empty.FromString, + response_serializer=footsies__service__pb2.GameState.SerializeToString, + ), + "GetEncodedState": grpc.unary_unary_rpc_method_handler( + servicer.GetEncodedState, + request_deserializer=footsies__service__pb2.Empty.FromString, + response_serializer=footsies__service__pb2.EncodedGameState.SerializeToString, + ), + "IsReady": grpc.unary_unary_rpc_method_handler( + servicer.IsReady, + request_deserializer=footsies__service__pb2.Empty.FromString, + response_serializer=footsies__service__pb2.BoolValue.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + "FootsiesGameService", rpc_method_handlers + ) + server.add_generic_rpc_handlers((generic_handler,)) + + +# This class is part of an EXPERIMENTAL API. +class FootsiesGameService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def StartGame( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/FootsiesGameService/StartGame", + footsies__service__pb2.Empty.SerializeToString, + footsies__service__pb2.Empty.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def ResetGame( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/FootsiesGameService/ResetGame", + footsies__service__pb2.Empty.SerializeToString, + footsies__service__pb2.Empty.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def StepNFrames( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/FootsiesGameService/StepNFrames", + footsies__service__pb2.StepInput.SerializeToString, + footsies__service__pb2.GameState.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def GetState( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/FootsiesGameService/GetState", + footsies__service__pb2.Empty.SerializeToString, + footsies__service__pb2.GameState.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def GetEncodedState( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/FootsiesGameService/GetEncodedState", + footsies__service__pb2.Empty.SerializeToString, + footsies__service__pb2.EncodedGameState.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def IsReady( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/FootsiesGameService/IsReady", + footsies__service__pb2.Empty.SerializeToString, + footsies__service__pb2.BoolValue.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) diff --git a/rllib/examples/envs/classes/multi_agent/footsies/utils.py b/rllib/examples/envs/classes/multi_agent/footsies/utils.py new file mode 100644 index 000000000000..3321f32058c6 --- /dev/null +++ b/rllib/examples/envs/classes/multi_agent/footsies/utils.py @@ -0,0 +1,331 @@ +import collections +import logging +from dataclasses import dataclass +from typing import Dict, Optional + +import gymnasium as gym +import numpy as np +from ray.rllib.algorithms.algorithm import Algorithm +from ray.rllib.algorithms.callbacks import RLlibCallback +from ray.rllib.core.rl_module import RLModuleSpec +from ray.rllib.env.env_runner import EnvRunner +from ray.rllib.env.multi_agent_episode import MultiAgentEpisode +from ray.rllib.examples.envs.classes.multi_agent.footsies.game.constants import ( + FOOTSIES_ACTION_IDS, +) +from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS +from ray.rllib.utils.metrics.metrics_logger import MetricsLogger +from ray.rllib.utils.typing import EpisodeType + +logger = logging.getLogger("ray.rllib") + + +@dataclass +class Matchup: + p1: str + p2: str + prob: float + + +class Matchmaker: + def __init__(self, matchups: list[Matchup]): + self.matchups = matchups + self.probs = [matchup.prob for matchup in matchups] + self.current_matchups = collections.defaultdict(dict) + + def agent_to_module_mapping_fn( + self, agent_id: str, episode: EpisodeType, **kwargs + ) -> str: + """Mapping function that retrieves policy_id from the sampled matchup""" + id_ = episode.id_ + if self.current_matchups.get(id_) is None: + # step 1: sample a matchup according to the specified probabilities + sampled_matchup = np.random.choice(a=self.matchups, p=self.probs) + + # step 2: Randomize who is player 1 and player 2 + policies = [sampled_matchup.p1, sampled_matchup.p2] + p1, p2 = np.random.choice(policies, size=2, replace=False) + + # step 3: Set as the current matchup for the episode in question (id_) + self.current_matchups[id_]["p1"] = p1 + self.current_matchups[id_]["p2"] = p2 + + policy_id = self.current_matchups[id_].pop(agent_id) + + # remove (an empty dict) for the current episode with id_ + if not self.current_matchups[id_]: + del self.current_matchups[id_] + + return policy_id + + +class MetricsLoggerCallback(RLlibCallback): + def __init__(self, main_policy: str) -> None: + """Log experiment metrics + + Logs metrics after each episode step and at the end of each (train or eval) episode. + Metrics logged at the end of each episode will be later used by MixManagerCallback + to decide whether to add a new opponent to the mix. + """ + super().__init__() + self.main_policy = main_policy + self.action_id_to_str = { + action_id: action_str + for action_str, action_id in FOOTSIES_ACTION_IDS.items() + } + + def on_episode_step( + self, + *, + episode: MultiAgentEpisode, + env_runner: Optional[EnvRunner] = None, + metrics_logger: Optional[MetricsLogger] = None, + env: Optional[gym.Env] = None, + env_index: int, + **kwargs, + ) -> None: + """Log action usage frequency + + Log actions performed by both players at each step of the (training or evaluation) episode. + """ + stage = "eval" if env_runner.config.in_evaluation else "train" + + # get the ModuleID for each agent + p1_module = episode.module_for("p1") + p2_module = episode.module_for("p2") + + # get action string for each agent + p1_action_id = env.envs[ + env_index + ].unwrapped.last_game_state.player1.current_action_id + p2_action_id = env.envs[ + env_index + ].unwrapped.last_game_state.player2.current_action_id + p1_action_str = self.action_id_to_str[p1_action_id] + p2_action_str = self.action_id_to_str[p2_action_id] + + metrics_logger.log_value( + key=f"footsies/{stage}/actions/{p1_module}/{p1_action_str}", + value=1, + reduce="sum", + window=100, + clear_on_reduce=True, + ) + metrics_logger.log_value( + key=f"footsies/{stage}/actions/{p2_module}/{p2_action_str}", + value=1, + reduce="sum", + window=100, + clear_on_reduce=True, + ) + + def on_episode_end( + self, + *, + episode: MultiAgentEpisode, + env_runner: Optional[EnvRunner] = None, + metrics_logger: Optional[MetricsLogger] = None, + env: Optional[gym.Env] = None, + env_index: int, + **kwargs, + ) -> None: + """Log win rates + + Log win rates of the main policy against its opponent at the end of the (training or evaluation) episode. + """ + stage = "eval" if env_runner.config.in_evaluation else "train" + + # check status of "p1" and "p2" + last_game_state = env.envs[env_index].unwrapped.last_game_state + p1_dead = last_game_state.player1.is_dead + p2_dead = last_game_state.player2.is_dead + + # get the ModuleID for each agent + p1_module = episode.module_for("p1") + p2_module = episode.module_for("p2") + + if self.main_policy == p1_module: + opponent_id = p2_module + main_policy_win = p2_dead + elif self.main_policy == p2_module: + opponent_id = p1_module + main_policy_win = p1_dead + else: + logger.info( + f"RLlib {self.__class__.__name__}: Main policy: '{self.main_policy}' not found in this episode. " + f"Policies in this episode are: '{p1_module}' and '{p2_module}'. " + f"Check your multi_agent 'policy_mapping_fn'. " + f"Metrics logging for this episode will be skipped." + ) + return + + if p1_dead and p2_dead: + metrics_logger.log_value( + key=f"footsies/{stage}/both_dead/{self.main_policy}/vs_{opponent_id}", + value=1, + reduce="mean", + window=100, + clear_on_reduce=True, + ) + elif not p1_dead and not p2_dead: + metrics_logger.log_value( + key=f"footsies/{stage}/both_alive/{self.main_policy}/vs_{opponent_id}", + value=1, + reduce="mean", + window=100, + clear_on_reduce=True, + ) + else: + # log the win rate against the opponent with an 'opponent_id' + metrics_logger.log_value( + key=f"footsies/{stage}/win_rates/{self.main_policy}/vs_{opponent_id}", + value=int(main_policy_win), + reduce="mean", + window=100, + clear_on_reduce=True, + ) + + # log the win rate, without specifying the opponent + # this metric collected from the eval env runner + # will be used to decide whether to add + # a new opponent at the current level. + metrics_logger.log_value( + key=f"footsies/{stage}/win_rates/{self.main_policy}/vs_any", + value=int(main_policy_win), + reduce="mean", + window=100, + clear_on_reduce=True, + ) + + +class MixManagerCallback(RLlibCallback): + def __init__( + self, + win_rate_threshold: float, + main_policy: str, + target_mix_size: int, + starting_modules=list[str], # default is ["lstm", "noop"] + fixed_modules_progression_sequence=tuple[str], # default is ("noop", "back") + ) -> None: + """Track win rates and manage mix of opponents""" + super().__init__() + self.win_rate_threshold = win_rate_threshold + self.main_policy = main_policy + self.target_mix_size = target_mix_size + self.fixed_modules_progression_sequence = tuple( + fixed_modules_progression_sequence + ) # Order of RL modules to be added to the mix + self.modules_in_mix = list( + starting_modules + ) # RLModules that are currently in the mix + self._trained_policy_idx = ( + 0 # We will use this to create new opponents of the main policy + ) + + def on_evaluate_end( + self, + *, + algorithm: Algorithm, + metrics_logger: Optional[MetricsLogger] = None, + evaluation_metrics: dict, + **kwargs, + ) -> None: + """Check win rates and add new opponent if necessary. + + Check the win rate of the main policy against its current opponent. + If the win rate exceeds the specified threshold, add a new opponent to the mix, by modifying: + 1. update the policy_mapping_fn for (training and evaluation) env runners + 2. if the new policy is a trained one (not a fixed RL module), modify Algorithm's state (initialize the state of the newly added RLModule by using the main policy) + """ + _main_module = algorithm.get_module(self.main_policy) + new_module_id = None + new_module_spec = None + + win_rate = evaluation_metrics[ENV_RUNNER_RESULTS][ + f"footsies/eval/win_rates/{self.main_policy}/vs_any" + ] + + if win_rate > self.win_rate_threshold: + logger.info( + f"RLlib {self.__class__.__name__}: Win rate for main policy '{self.main_policy}' " + f"exceeded threshold ({win_rate} > {self.win_rate_threshold})." + f" Adding new RL Module to the mix..." + ) + + # check if fixed RL module should be added to the mix, + # and if so, create new_module_id and new_module_spec for it + for module_id in self.fixed_modules_progression_sequence: + if module_id not in self.modules_in_mix: + new_module_id = module_id + break + + # in case that all fixed RL Modules are already in the mix (together with the main policy), + # we will add a new RL Module by taking main policy and adding an instance of it to the mix + if new_module_id is None: + new_module_id = f"{self.main_policy}_v{self._trained_policy_idx}" + new_module_spec = RLModuleSpec.from_module(_main_module) + self._trained_policy_idx += 1 + + # create new policy mapping function, to ensure that the main policy plays against newly added policy + new_mapping_fn = Matchmaker( + [ + Matchup( + p1=self.main_policy, + p2=new_module_id, + prob=1.0, + ) + ] + ).agent_to_module_mapping_fn + + # update (training) env runners with the new mapping function + algorithm.env_runner_group.foreach_env_runner( + lambda er: er.config.multi_agent(policy_mapping_fn=new_mapping_fn), + local_env_runner=True, + ) + + # update (eval) env runners with the new mapping function + algorithm.eval_env_runner_group.foreach_env_runner( + lambda er: er.config.multi_agent(policy_mapping_fn=new_mapping_fn), + local_env_runner=True, + ) + + if new_module_id not in self.fixed_modules_progression_sequence: + algorithm.add_module( + module_id=new_module_id, + module_spec=new_module_spec, + new_agent_to_module_mapping_fn=new_mapping_fn, + ) + # newly added trained policy should be initialized with the state of the main policy + algorithm.set_state( + { + "learner_group": { + "learner": { + "rl_module": { + new_module_id: _main_module.get_state(), + } + } + }, + } + ) + # we added a new RL Module, so we need to update the current mix list. + self.modules_in_mix.append(new_module_id) + + else: + logger.info( + f"RLlib {self.__class__.__name__}: Win rate for main policy '{self.main_policy}' " + f"did not exceed threshold ({win_rate} <= {self.win_rate_threshold})." + ) + + def on_train_result( + self, + *, + algorithm: Algorithm, + metrics_logger: Optional[MetricsLogger] = None, + result: Dict, + **kwargs, + ) -> None: + """Report the current mix size at the end of training iteration. + + That will tell Ray Tune, whether to stop training (once the 'target_mix_size' has been reached). + """ + result["mix_size"] = len(self.modules_in_mix) diff --git a/rllib/examples/multi_agent/self_play_footsies.py b/rllib/examples/multi_agent/self_play_footsies.py new file mode 100644 index 000000000000..2cc5213eced2 --- /dev/null +++ b/rllib/examples/multi_agent/self_play_footsies.py @@ -0,0 +1,112 @@ +""" +Multi-agent RLlib Footsies Simplified Example (PPO) + +About: + - This example as a simplified version of "rllib/tuned_examples/ppo/multi_agent_footsies_ppo.py", + which has more detailed comments and instructions. Please refer to that example for more information. + - This example is created to test the self-play training progression with footsies. + - Simplified version runs with single learner (cpu), single env runner, and single eval env runner. +""" +from pathlib import Path + +from ray.rllib.tuned_examples.ppo.multi_agent_footsies_ppo import ( + config, + env_creator, + stop, +) +from ray.rllib.utils.test_utils import ( + add_rllib_example_script_args, +) +from ray.tune.registry import register_env + +parser = add_rllib_example_script_args( + default_iters=500, + default_timesteps=5_000_000, +) +parser.add_argument( + "--train-start-port", + type=int, + default=45001, + help="First port number for the Footsies training environment server (default: 45001). Each server gets its own port.", +) +parser.add_argument( + "--eval-start-port", + type=int, + default=55001, + help="First port number for the Footsies evaluation environment server (default: 55001) Each server gets its own port.", +) +parser.add_argument( + "--binary-download-dir", + type=Path, + default="/tmp/ray/binaries/footsies", + help="Directory to download Footsies binaries (default: /tmp/ray/binaries/footsies)", +) +parser.add_argument( + "--binary-extract-dir", + type=Path, + default="/tmp/ray/binaries/footsies", + help="Directory to extract Footsies binaries (default: /tmp/ray/binaries/footsies)", +) +parser.add_argument( + "--binary-to-download", + type=str, + choices=["linux_server", "linux_windowed", "mac_headless", "mac_windowed"], + default="linux_server", + help="Target binary for Footsies environment (default: linux_server). Linux and Mac machines are supported. " + "'linux_server' and 'mac_headless' choices are the default options for the training. Game will run in the batchmode, without initializing the graphics. " + "'linux_windowed' and 'mac_windowed' choices are for the local run only, because " + "game will be rendered in the OS window. To use this option effectively, set up: " + "--no-tune --num-env-runners 0 --evaluation-num-env-runners 0", +) +parser.add_argument( + "--win-rate-threshold", + type=float, + default=0.55, + help="The main policy should have at least 'win-rate-threshold' win rate against the " + "other policy to advance to the next level. Moving to the next level " + "means adding a new policy to the mix.", +) +parser.add_argument( + "--target-mix-size", + type=int, + default=4, + help="Target number of policies (RLModules) in the mix to consider the test passed. " + "The initial mix size is 2: 'main policy' vs. 'other'. " + "`--target-mix-size=4` means that 2 new policies will be added to the mix. " + "Whether to add new policy is decided by checking the '--win-rate-threshold' condition. ", +) +parser.add_argument( + "--rollout-fragment-length", + type=int, + default=256, + help="The length of each rollout fragment to be collected by the EnvRunners when sampling.", +) + +args = parser.parse_args() +register_env(name="FootsiesEnv", env_creator=env_creator) +stop["mix_size"] = args.target_mix_size + +config.environment( + env="FootsiesEnv", + env_config={ + "train_start_port": args.train_start_port, + "eval_start_port": args.eval_start_port, + "binary_download_dir": args.binary_download_dir, + "binary_extract_dir": args.binary_extract_dir, + "binary_to_download": args.binary_to_download, + }, +).training( + train_batch_size_per_learner=args.rollout_fragment_length + * (args.num_env_runners or 1), +) + + +if __name__ == "__main__": + from ray.rllib.utils.test_utils import run_rllib_example_script_experiment + + results = run_rllib_example_script_experiment( + base_config=config, + args=args, + stop=stop, + success_metric={"mix_size": args.target_mix_size}, + ) diff --git a/rllib/tuned_examples/ppo/multi_agent_footsies_ppo.py b/rllib/tuned_examples/ppo/multi_agent_footsies_ppo.py new file mode 100644 index 000000000000..e046c380933b --- /dev/null +++ b/rllib/tuned_examples/ppo/multi_agent_footsies_ppo.py @@ -0,0 +1,259 @@ +""" +Multi-agent RLlib Footsies Example (PPO) + +About: + - Example is based on the Footsies environment (https://github.com/chasemcd/FootsiesGym). + - Footsies is a two-player fighting game where each player controls a character and tries to hit the opponent while avoiding being hit. + - Footsies is a zero-sum game, when one player wins (+1 reward) the other loses (-1 reward). + +Summary: + - Main policy is an LSTM-based policy. + - Training algorithm is PPO. + +Training: + - Training is governed by adding new, more complex opponents to the mix as the main policy reaches a certain win rate threshold against the current opponent. + - Current opponent is always the newest opponent added to the mix. + - Training starts with a very simple opponent: "noop" (does nothing), then progresses to "back" (only moves backwards). These are the fixed (very simple) policies that are used to kick off the training. + - After "random", new opponents are frozen copies of the main policy at different training stages. They will be added to the mix as "lstm_v0", "lstm_v1", etc. + - In this way - after kick-starting the training with fixed simple opponents - the main policy will play against a version of itself from an earlier training stage. + - The main policy has to achieve the win rate threshold against the current opponent to add a new opponent to the mix. + - Training concludes when the target mix size is reached. + +Evaluation: + - Evaluation is performed against the current (newest) opponent. + - Evaluation runs for a fixed number of episodes at the end of each training iteration. + +""" +import functools +from pathlib import Path + +from ray.rllib.algorithms.ppo import PPOConfig +from ray.rllib.core.rl_module import RLModuleSpec, MultiRLModuleSpec +from ray.rllib.env.multi_agent_env_runner import MultiAgentEnvRunner +from ray.rllib.examples.envs.classes.multi_agent.footsies.fixed_rlmodules import ( + NoopFixedRLModule, + BackFixedRLModule, +) +from ray.rllib.examples.envs.classes.multi_agent.footsies.footsies_env import ( + env_creator, +) +from ray.rllib.examples.envs.classes.multi_agent.footsies.utils import ( + Matchup, + Matchmaker, + MetricsLoggerCallback, + MixManagerCallback, +) +from ray.rllib.examples.rl_modules.classes.lstm_containing_rlm import ( + LSTMContainingRLModule, +) +from ray.rllib.utils.metrics import NUM_ENV_STEPS_SAMPLED_LIFETIME +from ray.rllib.utils.test_utils import ( + add_rllib_example_script_args, +) +from ray.tune.registry import register_env +from ray.tune.result import TRAINING_ITERATION + +# setting two default stopping criteria: +# 1. training_iteration (via "stop_iters") +# 2. num_env_steps_sampled_lifetime (via "default_timesteps") +# ...values very high to make sure that the test passes by adding +# all required policies to the mix, not by hitting the iteration limit. +# Our main stopping criterion is "target_mix_size" (see an argument below). +parser = add_rllib_example_script_args( + default_iters=500, + default_timesteps=5_000_000, +) + +parser.add_argument( + "--train-start-port", + type=int, + default=45001, + help="First port number for the Footsies training environment server (default: 45001). Each server gets its own port.", +) +parser.add_argument( + "--eval-start-port", + type=int, + default=55001, + help="First port number for the Footsies evaluation environment server (default: 55001) Each server gets its own port.", +) +parser.add_argument( + "--binary-download-dir", + type=Path, + default="/tmp/ray/binaries/footsies", + help="Directory to download Footsies binaries (default: /tmp/ray/binaries/footsies)", +) +parser.add_argument( + "--binary-extract-dir", + type=Path, + default="/tmp/ray/binaries/footsies", + help="Directory to extract Footsies binaries (default: /tmp/ray/binaries/footsies)", +) +parser.add_argument( + "--binary-to-download", + type=str, + choices=["linux_server", "linux_windowed", "mac_headless", "mac_windowed"], + default="linux_server", + help="Target binary for Footsies environment (default: linux_server). Linux and Mac machines are supported. " + "'linux_server' and 'mac_headless' choices are the default options for the training. Game will run in the batchmode, without initializing the graphics. " + "'linux_windowed' and 'mac_windowed' choices are for the local run only, because " + "game will be rendered in the OS window. To use this option effectively, set up: " + "--no-tune --num-env-runners 0 --evaluation-num-env-runners 0", +) +parser.add_argument( + "--win-rate-threshold", + type=float, + default=0.8, + help="The main policy should have at least 'win-rate-threshold' win rate against the " + "other policy to advance to the next level. Moving to the next level " + "means adding a new policy to the mix.", +) +parser.add_argument( + "--target-mix-size", + type=int, + default=5, + help="Target number of policies (RLModules) in the mix to consider the test passed. " + "The initial mix size is 2: 'main policy' vs. 'other'. " + "`--target-mix-size=5` means that 3 new policies will be added to the mix. " + "Whether to add new policy is decided by checking the '--win-rate-threshold' condition. ", +) +parser.add_argument( + "--rollout-fragment-length", + type=int, + default=256, + help="The length of each rollout fragment to be collected by the EnvRunners when sampling.", +) + +main_policy = "lstm" +args = parser.parse_args() +register_env(name="FootsiesEnv", env_creator=env_creator) + +config = ( + PPOConfig() + .reporting( + min_time_s_per_iteration=30, + ) + .environment( + env="FootsiesEnv", + env_config={ + "max_t": 1000, + "frame_skip": 4, + "observation_delay": 16, + "train_start_port": args.train_start_port, + "eval_start_port": args.eval_start_port, + "host": "localhost", + "binary_download_dir": args.binary_download_dir, + "binary_extract_dir": args.binary_extract_dir, + "binary_to_download": args.binary_to_download, + }, + ) + .learners( + num_learners=1, + num_cpus_per_learner=1, + num_gpus_per_learner=0, + num_aggregator_actors_per_learner=0, + ) + .env_runners( + env_runner_cls=MultiAgentEnvRunner, + num_env_runners=args.num_env_runners or 1, + num_cpus_per_env_runner=0.5, + num_envs_per_env_runner=1, + batch_mode="truncate_episodes", + rollout_fragment_length=args.rollout_fragment_length, + episodes_to_numpy=False, + create_env_on_local_worker=True, + ) + .training( + train_batch_size_per_learner=args.rollout_fragment_length + * (args.num_env_runners or 1), + lr=1e-4, + entropy_coeff=0.01, + num_epochs=10, + minibatch_size=128, + ) + .multi_agent( + policies={ + main_policy, + "noop", + "back", + }, + # this is a starting policy_mapping_fn + # It will be updated by the MixManagerCallback during training. + policy_mapping_fn=Matchmaker( + [Matchup(main_policy, "noop", 1.0)] + ).agent_to_module_mapping_fn, + # we only train the main policy, this doesn't change during training. + policies_to_train=[main_policy], + ) + .rl_module( + rl_module_spec=MultiRLModuleSpec( + rl_module_specs={ + main_policy: RLModuleSpec( + module_class=LSTMContainingRLModule, + model_config={ + "lstm_cell_size": 128, + "dense_layers": [128, 128], + "max_seq_len": 64, + }, + ), + # for simplicity, all fixed RLModules are added to the config at the start. + # However, only "noop" is used at the start of training, + # the others are added to the mix later by the MixManagerCallback. + "noop": RLModuleSpec(module_class=NoopFixedRLModule), + "back": RLModuleSpec(module_class=BackFixedRLModule), + }, + ) + ) + .evaluation( + evaluation_num_env_runners=args.evaluation_num_env_runners or 1, + evaluation_sample_timeout_s=120, + evaluation_interval=1, + evaluation_duration=10, # 10 episodes is enough to get a good win rate estimate + evaluation_duration_unit="episodes", + evaluation_parallel_to_training=False, + # we may add new RLModules to the mix at the end of the evaluation stage. + # Running evaluation in parallel may result in training for one more iteration on the old mix. + evaluation_force_reset_envs_before_iteration=True, + evaluation_config={ + "env_config": {"env-for-evaluation": True}, + }, # evaluation_config is used to add an argument to the env creator. + ) + .callbacks( + [ + functools.partial( + MetricsLoggerCallback, + main_policy=main_policy, + ), + functools.partial( + MixManagerCallback, + win_rate_threshold=args.win_rate_threshold, + main_policy=main_policy, + target_mix_size=args.target_mix_size, + starting_modules=[main_policy, "noop"], + fixed_modules_progression_sequence=( + "noop", + "back", + ), + ), + ] + ) +) + +# stopping criteria to be passed to Ray Tune. The main stopping criterion is "mix_size". +# "mix_size" is reported at the end of each training iteration by the MixManagerCallback. +stop = { + NUM_ENV_STEPS_SAMPLED_LIFETIME: args.stop_timesteps, + TRAINING_ITERATION: args.stop_iters, + "mix_size": args.target_mix_size, +} + +if __name__ == "__main__": + from ray.rllib.utils.test_utils import run_rllib_example_script_experiment + + results = run_rllib_example_script_experiment( + base_config=config, + args=args, + stop=stop, + success_metric={ + "mix_size": args.target_mix_size + }, # pass the success metric for RLlib's testing framework + ) From 0ecb9ff6f810f613d49d734464ebfb90f9bd1b57 Mon Sep 17 00:00:00 2001 From: Srinath Krishnamachari <68668616+srinathk10@users.noreply.github.com> Date: Thu, 4 Sep 2025 15:12:49 -0700 Subject: [PATCH 457/634] [Data] Mark `test_groupby_e2e` as `data_non_parallel` (#56250) ## Why are these changes needed? ### Mark `test_groupby_e2e` as `data_non_parallel` `test_groupby_e2e` running into OOMs even though marked `enormous`. So tagging as `data_non_parallel`. https://buildkite.com/anyscale/rayturbo/builds/6127#0199137b-052c-404e-829b-4992e4d309e3 ``` [2025-09-04T07:54:10Z] Memory on the node (IP: 172.17.0.3, ID: 8141a363c5d7cf3ced6026777eaf254b1da0321a09a65b102a2e196e) where the lease (actor ID: NIL_IDlease ID: 9950000001000000ffffffffffffffffffffffffffffffffffffffffffffffff, name=_shuffle_block, pid=38068, memory used=0.13GB) was running was 14.68GB / 15.33GB (0.957355), which exceeds the memory usage threshold of 0.95. Ray killed this worker (ID: a6221da9f5db12ddd527e8edc718f3c279f7f8bfb094e38307ad2b3c) because it was the most recently scheduled task; to see more information about memory usage on this node, use `ray logs raylet.out -ip 172.17.0.3`. To see the logs of the worker, use `ray logs worker-a6221da9f5db12ddd527e8edc718f3c279f7f8bfb094e38307ad2b3c*out -ip 172.17.0.3. Top 10 memory users: ``` ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Srinath Krishnamachari --- python/ray/data/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ray/data/BUILD b/python/ray/data/BUILD index 396cb8501b2a..a3b44dadf21b 100644 --- a/python/ray/data/BUILD +++ b/python/ray/data/BUILD @@ -383,6 +383,7 @@ py_test( size = "enormous", srcs = ["tests/test_groupby_e2e.py"], tags = [ + "data_non_parallel", "exclusive", "team:data", ], From b6ab5423322911bbd1cd6ba2e851408931590934 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Thu, 4 Sep 2025 18:48:24 -0500 Subject: [PATCH 458/634] [core] Drop messages received after `ClientConnection::Close` (#56240) ## Problem Previously, we've made the assumption that after the Raylet calls [client->Close](https://github.com/ray-project/ray/blob/6f3689a909d85b431983ad68e006fb4f59259233/src/ray/raylet/node_manager.cc#L1410), no messages will ever be received from that client again. We have many `RAY_CHECK(worker)` calls that assert that the client that sent the messages is a "registered client" or a "registered driver" in the worker pool. This was assumed to be safe because the Raylet is single threaded and we remove the worker from the registered maps *and* call `client->Close()` when we [disconnect a worker](https://github.com/ray-project/ray/blob/6f3689a909d85b431983ad68e006fb4f59259233/src/ray/raylet/node_manager.cc#L1275). Therefore, if no messages can be received after `client->Close()`, we can assume all messages are from registered workers. This assumption was not completely safe. In most cases, outstanding [boost::asio_async_read](https://github.com/ray-project/ray/blob/6f3689a909d85b431983ad68e006fb4f59259233/src/ray/ipc/client_connection.cc#L377) calls will be canceled and call their callback with an error code when `socket_.close()` is called ([docs](https://www.boost.org/doc/libs/boost_1_42_0/doc/html/boost_asio/reference/basic_stream_socket/close/overload2.html)). Sometimes, presumably when the underyling poll syscall has already populated boost's internal data buffer, the `async_read` call _will_ actually call its callback with a fully populated data buffer. See https://github.com/ray-project/ray/pull/56205 for a manual reproduction of this behavior. ## Solution To handle this edge case, I've introduced a `closed_` flag that is set when `ClientConnection::Close()` is called. If this flag is set and `async_read` returns a message with no error, we drop the message. This provides the guarantee that we previously assumed. I've also moved `Close()` to `ClientConnection` instead of inheriting it from `ServerConnection` because it was never used in `ServerConnection`. --------- Signed-off-by: Edward Oakes --- src/ray/ipc/client_connection.cc | 24 ++++++++++++++++++++++-- src/ray/ipc/client_connection.h | 16 ++++++++++------ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/ray/ipc/client_connection.cc b/src/ray/ipc/client_connection.cc index 24de00f14c3f..fd6c7a7dc971 100644 --- a/src/ray/ipc/client_connection.cc +++ b/src/ray/ipc/client_connection.cc @@ -360,6 +360,12 @@ void ClientConnection::Register() { registered_ = true; } +void ClientConnection::Close() { + closed_ = true; + boost::system::error_code ec; + socket_.close(ec); +} + void ClientConnection::ProcessMessages() { // Wait for a message header from the client. The message header includes the // protocol version, the message type, and the length of the message. @@ -399,9 +405,16 @@ void ClientConnection::ProcessMessageHeader(const boost::system::error_code &err return; } - // If there was no error, make sure the ray cookie matches. + if (closed_) { + // In most cases all outstanding reads will have been canceled when the socket was. + // closed. However, if the boost async_read call has already received data into its + // buffer from the poll syscall, it may succeed. If this happens, drop the message. + return; + } + if (!CheckRayCookie()) { - ServerConnection::Close(); + RAY_LOG(WARNING) << "Mismatched Ray cookie, closing client connection."; + Close(); return; } @@ -470,6 +483,13 @@ void ClientConnection::ProcessMessage(const boost::system::error_code &error) { return connection_error_handler_(std::move(this_ptr), error); } + if (closed_) { + // In most cases all outstanding reads will have been canceled when the socket was. + // closed. However, if the boost async_read call has already received data into its + // buffer from the poll syscall, it may succeed. If this happens, drop the message. + return; + } + int64_t start_ms = current_time_ms(); message_handler_(std::move(this_ptr), read_type_, read_message_); int64_t interval = current_time_ms() - start_ms; diff --git a/src/ray/ipc/client_connection.h b/src/ray/ipc/client_connection.h index 530d117f7766..1f03a0863a45 100644 --- a/src/ray/ipc/client_connection.h +++ b/src/ray/ipc/client_connection.h @@ -105,12 +105,6 @@ class ServerConnection : public std::enable_shared_from_this { /// \return Status. virtual Status ReadBuffer(const std::vector &buffer); - /// Shuts down socket for this connection. - void Close() { - boost::system::error_code ec; - socket_.close(ec); - } - /// Get the native handle of the socket. int GetNativeHandle() { return socket_.native_handle(); } @@ -226,6 +220,14 @@ class ClientConnection : public ServerConnection { /// Register the client. void Register(); + /// Close the connection forcefully. + /// + /// - Clients will receive an error the next time they interact with the connection. + /// - No further messages will be processed from `ProcessMessages`. + /// - The `ConnectionErrorHandler` may be called with an error indicating that + /// outstanding reads failed. + void Close(); + /// Listen for and process messages from the client connection. Once a /// message has been fully received, the client manager's /// ProcessClientMessage handler will be called. @@ -266,6 +268,8 @@ class ClientConnection : public ServerConnection { /// Whether the client has sent us a registration message yet. bool registered_; + /// Whether the connection has been explicitly closed by the server. + bool closed_ = false; /// The handler for a message from the client. MessageHandler message_handler_; /// The handler for an unexpected connection error from this client. From 7bb2a1a99e0fbb8031e612bb30104cbbc1489417 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Thu, 4 Sep 2025 16:55:17 -0700 Subject: [PATCH 459/634] [serve] swap toolz for collections.defaultdict in tests (#56249) ## Why are these changes needed? Use `collections` instead of `toolz` in `test_metrics_utils` ## Related issue number https://github.com/ray-project/ray/issues/56227 Signed-off-by: Cindy Zhang --- python/ray/serve/tests/unit/test_metrics_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/serve/tests/unit/test_metrics_utils.py b/python/ray/serve/tests/unit/test_metrics_utils.py index 0e37d6b26d15..dd5e68c25d81 100644 --- a/python/ray/serve/tests/unit/test_metrics_utils.py +++ b/python/ray/serve/tests/unit/test_metrics_utils.py @@ -1,8 +1,8 @@ import asyncio import sys +from collections import defaultdict import pytest -from toolz.tests.test_dicttoolz import defaultdict from ray._common.test_utils import async_wait_for_condition from ray.serve._private.metrics_utils import ( From b78555d95a071163538a39306c36c0f80d1e4dca Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Thu, 4 Sep 2025 17:02:43 -0700 Subject: [PATCH 460/634] [core] Fix cancel race that leads to RAY_CHECK it->second.submitted_task_ref_count > 0 (#56123) Signed-off-by: hejialing.hjl Signed-off-by: dayshah Co-authored-by: hejialing.hjl --- src/ray/core_worker/core_worker.cc | 1 - src/ray/core_worker/core_worker.h | 2 - .../task_submission/actor_task_submitter.cc | 81 ++++++------ .../task_submission/actor_task_submitter.h | 39 +++--- src/ray/core_worker/tests/core_worker_test.cc | 117 ++++++++++++++---- .../core_worker/tests/memory_store_test.cc | 15 ++- 6 files changed, 165 insertions(+), 90 deletions(-) diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 0dc1ea211221..147e0fa58627 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -2287,7 +2287,6 @@ Status CoreWorker::SubmitActorTask( std::string err_msg = absl::StrFormat( "Can't find actor %s. It might be dead or it's from a different cluster", actor_id.Hex()); - // TODO(dayshah): make status take by value return Status::NotFound(err_msg); } /// Check whether backpressure may happen at the very beginning of submitting a task. diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index b5f07f4e8505..ba060a05b500 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -1888,8 +1888,6 @@ class CoreWorker { int64_t max_direct_call_object_size_; - friend class CoreWorkerTest; - TaskCounter task_counter_; /// Used to guarantee that submitting actor task is thread safe. diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.cc b/src/ray/core_worker/task_submission/actor_task_submitter.cc index 765697d75a1a..9df800d15e39 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.cc +++ b/src/ray/core_worker/task_submission/actor_task_submitter.cc @@ -173,6 +173,8 @@ void ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { bool task_queued = false; uint64_t send_pos = 0; { + // We must release mu_ before resolving the task dependencies since the callback that + // reacquires mu_ may get called in the same call stack. absl::MutexLock lock(&mu_); auto queue = client_queues_.find(actor_id); RAY_CHECK(queue != client_queues_.end()); @@ -193,37 +195,45 @@ void ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { } if (task_queued) { + { + absl::MutexLock resolver_lock(&resolver_mu_); + pending_dependency_resolution_.insert(task_id); + } io_service_.post( [task_spec, task_id, actor_id, send_pos, this]() mutable { - // We must release the lock before resolving the task dependencies since - // the callback may get called in the same call stack. - resolver_.ResolveDependencies( - task_spec, [this, send_pos, actor_id, task_id](Status status) { - task_manager_.MarkDependenciesResolved(task_id); - bool fail_or_retry_task = false; - { - absl::MutexLock lock(&mu_); - auto queue = client_queues_.find(actor_id); - RAY_CHECK(queue != client_queues_.end()); - auto &actor_submit_queue = queue->second.actor_submit_queue_; - // Only dispatch tasks if the submitted task is still queued. The task - // may have been dequeued if the actor has since failed. - if (actor_submit_queue->Contains(send_pos)) { - if (status.ok()) { - actor_submit_queue->MarkDependencyResolved(send_pos); - SendPendingTasks(actor_id); - } else { - fail_or_retry_task = true; - actor_submit_queue->MarkDependencyFailed(send_pos); + { + absl::MutexLock resolver_lock(&resolver_mu_); + if (pending_dependency_resolution_.erase(task_id) == 0) { + return; + } + resolver_.ResolveDependencies( + task_spec, [this, send_pos, actor_id, task_id](Status status) { + task_manager_.MarkDependenciesResolved(task_id); + bool fail_or_retry_task = false; + { + absl::MutexLock lock(&mu_); + auto queue = client_queues_.find(actor_id); + RAY_CHECK(queue != client_queues_.end()); + auto &actor_submit_queue = queue->second.actor_submit_queue_; + // Only dispatch tasks if the submitted task is still queued. The task + // may have been dequeued if the actor has since failed. + if (actor_submit_queue->Contains(send_pos)) { + if (status.ok()) { + actor_submit_queue->MarkDependencyResolved(send_pos); + SendPendingTasks(actor_id); + } else { + fail_or_retry_task = true; + actor_submit_queue->MarkDependencyFailed(send_pos); + } } } - } - if (fail_or_retry_task) { - GetTaskManagerWithoutMu().FailOrRetryPendingTask( - task_id, rpc::ErrorType::DEPENDENCY_RESOLUTION_FAILED, &status); - } - }); + if (fail_or_retry_task) { + GetTaskManagerWithoutMu().FailOrRetryPendingTask( + task_id, rpc::ErrorType::DEPENDENCY_RESOLUTION_FAILED, &status); + } + }); + } }, "ActorTaskSubmitter::SubmitTask"); } else { @@ -254,6 +264,12 @@ void ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { } } +void ActorTaskSubmitter::CancelDependencyResolution(const TaskID &task_id) { + absl::MutexLock resolver_lock(&resolver_mu_); + pending_dependency_resolution_.erase(task_id); + RAY_UNUSED(resolver_.CancelDependencyResolution(task_id)); +} + void ActorTaskSubmitter::DisconnectRpcClient(ClientQueue &queue) { queue.rpc_client_ = nullptr; core_worker_client_pool_.Disconnect(WorkerID::FromBinary(queue.worker_id_)); @@ -435,7 +451,7 @@ void ActorTaskSubmitter::DisconnectActor(const ActorID &actor_id, task_manager_.MarkTaskNoRetry(task_id); // This task may have been waiting for dependency resolution, so cancel // this first. - RAY_UNUSED(resolver_.CancelDependencyResolution(task_id)); + CancelDependencyResolution(task_id); bool fail_immediatedly = error_info.has_actor_died_error() && error_info.actor_died_error().has_oom_context() && @@ -707,7 +723,7 @@ void ActorTaskSubmitter::HandlePushTaskReply(const Status &status, // This task may have been waiting for dependency resolution, so cancel // this first. - RAY_UNUSED(resolver_.CancelDependencyResolution(task_id)); + CancelDependencyResolution(task_id); will_retry = GetTaskManagerWithoutMu().FailOrRetryPendingTask( task_id, @@ -886,13 +902,6 @@ void ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursive) task_queued = queue->second.actor_submit_queue_->Contains(send_pos); if (task_queued) { - auto dep_resolved = - queue->second.actor_submit_queue_->DependenciesResolved(send_pos); - if (!dep_resolved) { - RAY_LOG(DEBUG).WithField(task_id) - << "Task has been resolving dependencies. Cancel to resolve dependencies"; - RAY_UNUSED(resolver_.CancelDependencyResolution(task_id)); - } RAY_LOG(DEBUG).WithField(task_id) << "Task was queued. Mark a task is canceled from a queue."; queue->second.actor_submit_queue_->MarkTaskCanceled(send_pos); @@ -903,6 +912,8 @@ void ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursive) // The task won't be sent to an actor in this case. // We cannot hold a lock when calling `FailOrRetryPendingTask`. if (task_queued) { + // Could be in dependency resolution or ResolveDependencies call may be queued up + CancelDependencyResolution(task_id); rpc::RayErrorInfo error_info; std::ostringstream stream; stream << "The task " << task_id << " is canceled from an actor " << actor_id diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.h b/src/ray/core_worker/task_submission/actor_task_submitter.h index 64b809da29b9..a07591d582b7 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.h +++ b/src/ray/core_worker/task_submission/actor_task_submitter.h @@ -15,11 +15,8 @@ #pragma once #include -#include #include #include -#include -#include #include #include @@ -27,7 +24,6 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/synchronization/mutex.h" -#include "ray/common/asio/asio_util.h" #include "ray/common/id.h" #include "ray/core_worker/actor_creator.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" @@ -63,7 +59,7 @@ class ActorTaskSubmitterInterface { /// If called, preempted = true will be set in the death cause upon actor death. virtual void SetPreempted(const ActorID &actor_id) = 0; - virtual ~ActorTaskSubmitterInterface() {} + virtual ~ActorTaskSubmitterInterface() = default; }; // This class is thread-safe. @@ -81,14 +77,13 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { actor_creator_(actor_creator), resolver_(store, task_manager, actor_creator, tensor_transport_getter), task_manager_(task_manager), - warn_excess_queueing_(warn_excess_queueing), + warn_excess_queueing_(std::move(warn_excess_queueing)), + next_queueing_warn_threshold_( + ::RayConfig::instance().actor_excess_queueing_warn_threshold()), io_service_(io_service), - reference_counter_(reference_counter) { - next_queueing_warn_threshold_ = - ::RayConfig::instance().actor_excess_queueing_warn_threshold(); - } + reference_counter_(std::move(reference_counter)) {} - void SetPreempted(const ActorID &actor_id) { + void SetPreempted(const ActorID &actor_id) override { absl::MutexLock lock(&mu_); if (auto iter = client_queues_.find(actor_id); iter != client_queues_.end()) { iter->second.preempted_ = true; @@ -110,7 +105,7 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { int32_t max_pending_calls, bool allow_out_of_order_execution, bool fail_if_actor_unreachable, - bool owned); + bool owned) override; /// Submit a task to an actor for execution. void SubmitTask(TaskSpecification task_spec); @@ -127,7 +122,7 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { /// ignore the command to connect. void ConnectActor(const ActorID &actor_id, const rpc::Address &address, - int64_t num_restarts); + int64_t num_restarts) override; /// Disconnect from a failed actor. /// @@ -143,13 +138,13 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { int64_t num_restarts, bool dead, const rpc::ActorDeathCause &death_cause, - bool is_restartable); + bool is_restartable) override; /// Set the timerstamp for the caller. void SetCallerCreationTimestamp(int64_t timestamp); /// Check timeout tasks that are waiting for Death info. - void CheckTimeoutTasks(); + void CheckTimeoutTasks() override; /// If the number of tasks in requests is greater than or equal to /// max_pending_calls. @@ -303,7 +298,7 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { /// pending client callbacks. std::shared_ptr rpc_client_ = nullptr; /// The intended worker ID of the actor. - std::string worker_id_ = ""; + std::string worker_id_; /// The actor is out of scope but the death info is not published /// to this worker yet. bool pending_out_of_scope_death_ = false; @@ -360,6 +355,9 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { } }; + void CancelDependencyResolution(const TaskID &task_id) + ABSL_LOCKS_EXCLUDED(resolver_mu_); + /// Fail the task with the timeout error, or the preempted error. void FailTaskWithError(const PendingTaskWaitingForDeathInfo &task); @@ -418,6 +416,13 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { // Generators that are currently running and need to be resubmitted. absl::flat_hash_set generators_to_resubmit_ ABSL_GUARDED_BY(mu_); + // For when kicking off dependency resolution is still queued on the io_context. + // We need an extra mutex because the ResolveDependencies callback could be called + // immediately and it acquires mu_ and needs to call GetTaskManagerWithoutMu. + absl::Mutex resolver_mu_ ABSL_ACQUIRED_BEFORE(mu_); + absl::flat_hash_set pending_dependency_resolution_ + ABSL_GUARDED_BY(resolver_mu_); + /// Resolve object dependencies. LocalDependencyResolver resolver_; @@ -435,8 +440,6 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { instrumented_io_context &io_service_; std::shared_ptr reference_counter_; - - friend class CoreWorkerTest; }; } // namespace core diff --git a/src/ray/core_worker/tests/core_worker_test.cc b/src/ray/core_worker/tests/core_worker_test.cc index fe05b3f8d39e..635bbe4e19fa 100644 --- a/src/ray/core_worker/tests/core_worker_test.cc +++ b/src/ray/core_worker/tests/core_worker_test.cc @@ -50,9 +50,9 @@ using ::testing::_; using ::testing::InvokeWithoutArgs; using ::testing::Return; -class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { +class CoreWorkerTest : public ::testing::Test { public: - CoreWorkerHandleGetObjectStatusTest() + CoreWorkerTest() : io_work_(io_service_.get_executor()), task_execution_service_work_(task_execution_service_.get_executor()) { CoreWorkerOptions options; @@ -113,17 +113,16 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { false /* token_auth */); core_worker_server->Run(); - rpc::Address rpc_address; - rpc_address.set_ip_address(options.node_ip_address); - rpc_address.set_port(core_worker_server->GetPort()); - rpc_address.set_node_id(NodeID::FromRandom().Binary()); - rpc_address.set_worker_id(worker_context->GetWorkerID().Binary()); + rpc_address_.set_ip_address(options.node_ip_address); + rpc_address_.set_port(core_worker_server->GetPort()); + rpc_address_.set_node_id(NodeID::FromRandom().Binary()); + rpc_address_.set_worker_id(worker_context->GetWorkerID().Binary()); auto fake_object_info_publisher = std::make_unique(); auto fake_object_info_subscriber = std::make_unique(); reference_counter_ = std::make_shared( - rpc_address, + rpc_address_, fake_object_info_publisher.get(), fake_object_info_subscriber.get(), [](const NodeID &) { return false; }, @@ -139,14 +138,14 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { const absl::flat_hash_set &locations, uint64_t object_size) {}, core_worker_client_pool, - rpc_address); + rpc_address_); auto task_event_buffer = std::make_unique( std::make_unique(), std::make_unique(0, *client_call_manager), "test_session"); - auto task_manager = std::make_shared( + task_manager_ = std::make_shared( *memory_store_, *reference_counter_, [](const RayObject &object, const ObjectID &object_id) { return Status::OK(); }, @@ -164,31 +163,31 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { mock_gcs_client); auto object_recovery_manager = std::make_unique( - rpc_address, + rpc_address_, raylet_client_pool, [](const ObjectID &object_id, const ObjectLookupCallback &callback) { return Status::OK(); }, - *task_manager, + *task_manager_, *reference_counter_, *memory_store_, [](const ObjectID &object_id, rpc::ErrorType reason, bool pin_object) {}); auto lease_policy = std::unique_ptr( - std::make_unique(rpc_address)); + std::make_unique(rpc_address_)); auto lease_request_rate_limiter = std::make_shared(10); auto actor_creator = std::make_shared(mock_gcs_client); auto normal_task_submitter = std::make_unique( - rpc_address, + rpc_address_, fake_local_raylet_rpc_client, core_worker_client_pool, raylet_client_pool, std::move(lease_policy), memory_store_, - *task_manager, + *task_manager_, NodeID::Nil(), WorkerType::WORKER, 10000, @@ -201,13 +200,14 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { auto actor_task_submitter = std::make_unique( *core_worker_client_pool, *memory_store_, - *task_manager, + *task_manager_, *actor_creator, /*tensor_transport_getter=*/ [](const ObjectID &object_id) { return rpc::TensorTransport::OBJECT_STORE; }, [](const ActorID &actor_id, uint64_t num_queued) { return Status::OK(); }, io_service_, reference_counter_); + actor_task_submitter_ = actor_task_submitter.get(); auto actor_manager = std::make_unique( mock_gcs_client, *actor_task_submitter, *reference_counter_); @@ -224,7 +224,7 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { std::move(raylet_client_pool), std::move(periodical_runner), std::move(core_worker_server), - std::move(rpc_address), + std::move(rpc_address_), std::move(mock_gcs_client), std::move(fake_raylet_ipc_client), std::move(fake_local_raylet_rpc_client), @@ -234,7 +234,7 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { nullptr, // plasma_store_provider_ nullptr, // mutable_object_provider_ std::move(future_resolver), - std::move(task_manager), + task_manager_, std::move(actor_creator), std::move(actor_task_submitter), std::move(fake_object_info_publisher), @@ -249,18 +249,19 @@ class CoreWorkerHandleGetObjectStatusTest : public ::testing::Test { } protected: - instrumented_io_context io_service_{/*enable_lag_probe=*/false, - /*running_on_single_thread=*/true}; - instrumented_io_context task_execution_service_{/*enable_lag_probe=*/false, - /*running_on_single_thread=*/true}; + instrumented_io_context io_service_; + instrumented_io_context task_execution_service_; boost::asio::executor_work_guard io_work_; boost::asio::executor_work_guard task_execution_service_work_; boost::thread io_thread_; + rpc::Address rpc_address_; std::shared_ptr reference_counter_; std::shared_ptr memory_store_; + ActorTaskSubmitter *actor_task_submitter_; + std::shared_ptr task_manager_; std::shared_ptr core_worker_; }; @@ -277,7 +278,7 @@ std::shared_ptr MakeRayObject(const std::string &data_str, return std::make_shared(data, metadata, std::vector()); } -TEST_F(CoreWorkerHandleGetObjectStatusTest, IdempotencyTest) { +TEST_F(CoreWorkerTest, HandleGetObjectStatusIdempotency) { auto object_id = ObjectID::FromRandom(); auto ray_object = MakeRayObject("test_data", "meta"); @@ -327,7 +328,7 @@ TEST_F(CoreWorkerHandleGetObjectStatusTest, IdempotencyTest) { EXPECT_EQ("meta", reply2.object().metadata()); } -TEST_F(CoreWorkerHandleGetObjectStatusTest, ObjectPutAfterFirstRequest) { +TEST_F(CoreWorkerTest, HandleGetObjectStatusObjectPutAfterFirstRequest) { auto object_id = ObjectID::FromRandom(); auto ray_object = MakeRayObject("test_data", "meta"); @@ -382,7 +383,7 @@ TEST_F(CoreWorkerHandleGetObjectStatusTest, ObjectPutAfterFirstRequest) { EXPECT_EQ("meta", reply2.object().metadata()); } -TEST_F(CoreWorkerHandleGetObjectStatusTest, ObjectFreedBetweenRequests) { +TEST_F(CoreWorkerTest, HandleGetObjectStatusObjectFreedBetweenRequests) { auto object_id = ObjectID::FromRandom(); auto ray_object = MakeRayObject("test_data", "meta"); @@ -432,7 +433,7 @@ TEST_F(CoreWorkerHandleGetObjectStatusTest, ObjectFreedBetweenRequests) { ASSERT_FALSE(io_service_.poll_one()); } -TEST_F(CoreWorkerHandleGetObjectStatusTest, ObjectOutOfScope) { +TEST_F(CoreWorkerTest, HandleGetObjectStatusObjectOutOfScope) { auto object_id = ObjectID::FromRandom(); auto ray_object = MakeRayObject("test_data", "meta"); @@ -483,5 +484,69 @@ TEST_F(CoreWorkerHandleGetObjectStatusTest, ObjectOutOfScope) { EXPECT_EQ(reply2.status(), rpc::GetObjectStatusReply::OUT_OF_SCOPE); } +namespace { + +ObjectID CreateInlineObjectInMemoryStoreAndRefCounter(CoreWorkerMemoryStore &memory_store, + ReferenceCounter &reference_counter, + rpc::Address &rpc_address) { + auto inlined_dependency_id = ObjectID::FromRandom(); + std::string data = "hello"; + auto data_ptr = const_cast(reinterpret_cast(data.data())); + auto data_buffer = + std::make_shared(data_ptr, data.size(), /*copy_data=*/true); + RayObject memory_store_object(data_buffer, + /*metadata=*/nullptr, + std::vector(), + /*copy_data=*/true); + reference_counter.AddOwnedObject(inlined_dependency_id, + /*contained_ids=*/{}, + rpc_address, + "call_site", + /*object_size=*/100, + /*is_reconstructable=*/false, + /*add_local_ref=*/true); + memory_store.Put(memory_store_object, inlined_dependency_id); + return inlined_dependency_id; +} + +} // namespace + +TEST_F(CoreWorkerTest, ActorTaskCancelDuringDepResolution) { + /* + See https://github.com/ray-project/ray/pull/56123 for context. + 1. Put an inline object in the memory store + ref counter. + 2. Create an actor (just creating an actor queue in the submitter). + 3. Submit an actor task with the inline objects as dependencies. + 4. Cancel the actor task. + 5. Run the io context to completion to run the actual submission + dependency + resolution logic. + */ + + auto inlined_dependency_id = CreateInlineObjectInMemoryStoreAndRefCounter( + *memory_store_, *reference_counter_, rpc_address_); + + auto actor_id = ActorID::Of(JobID::FromInt(0), TaskID::Nil(), 0); + actor_task_submitter_->AddActorQueueIfNotExists(actor_id, + /*max_pending_calls=*/-1, + /*allow_out_of_order_execution=*/false, + /*fail_if_actor_unreachable=*/true, + /*owned=*/false); + + TaskSpecification task; + auto &task_message = task.GetMutableMessage(); + task_message.set_task_id(TaskID::FromRandom(actor_id.JobId()).Binary()); + task_message.set_type(TaskType::ACTOR_TASK); + task_message.mutable_actor_task_spec()->set_actor_id(actor_id.Binary()); + task_message.add_args()->mutable_object_ref()->set_object_id( + inlined_dependency_id.Binary()); + task_manager_->AddPendingTask(rpc_address_, task, "call_site"); + actor_task_submitter_->SubmitTask(task); + + actor_task_submitter_->CancelTask(task, /*recursive=*/false); + + while (io_service_.poll_one() > 0) { + } +} + } // namespace core } // namespace ray diff --git a/src/ray/core_worker/tests/memory_store_test.cc b/src/ray/core_worker/tests/memory_store_test.cc index fcee0b090698..5a90b26af481 100644 --- a/src/ray/core_worker/tests/memory_store_test.cc +++ b/src/ray/core_worker/tests/memory_store_test.cc @@ -31,18 +31,17 @@ namespace ray { namespace core { -inline std::shared_ptr MakeBufferFromString(const uint8_t *data, - size_t data_size) { - auto metadata = const_cast(data); +namespace { + +std::shared_ptr MakeLocalMemoryBufferFromString( + const std::string &str) { + auto metadata = const_cast(reinterpret_cast(str.data())); auto meta_buffer = - std::make_shared(metadata, data_size, /*copy_data=*/true); + std::make_shared(metadata, str.size(), /*copy_data=*/true); return meta_buffer; } -inline std::shared_ptr MakeLocalMemoryBufferFromString( - const std::string &str) { - return MakeBufferFromString(reinterpret_cast(str.data()), str.size()); -} +} // namespace TEST(TestMemoryStore, TestReportUnhandledErrors) { std::vector> results; From 88293d44c9659fc7ecd262bc0b487bcf04afebf1 Mon Sep 17 00:00:00 2001 From: kourosh hakhamaneshi <31483498+kouroshHakha@users.noreply.github.com> Date: Fri, 5 Sep 2025 02:37:34 +0200 Subject: [PATCH 461/634] [Serve.llm] Gracefully return timeouts as HTTPException (#56264) Signed-off-by: Kourosh Hakhamaneshi --- .../serve/deployments/routers/router.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/python/ray/llm/_internal/serve/deployments/routers/router.py b/python/ray/llm/_internal/serve/deployments/routers/router.py index d9af5b28e327..240fff0184e7 100644 --- a/python/ray/llm/_internal/serve/deployments/routers/router.py +++ b/python/ray/llm/_internal/serve/deployments/routers/router.py @@ -1,6 +1,7 @@ import asyncio import json import sys +from contextlib import asynccontextmanager from typing import ( Any, AsyncGenerator, @@ -218,6 +219,19 @@ async def _openai_json_wrapper( yield "data: [DONE]\n\n" +@asynccontextmanager +async def router_request_timeout(timeout_duration: float): + try: + async with timeout(timeout_duration): + yield + except asyncio.TimeoutError as e: + raise OpenAIHTTPException( + status_code=status.HTTP_408_REQUEST_TIMEOUT, + message="Request server side timeout", + internal_message=str(e), + ) + + class LLMRouter: def __init__( self, @@ -418,7 +432,7 @@ async def _process_llm_request( ) call_method = "chat" if is_chat else "completions" - async with timeout(DEFAULT_LLM_ROUTER_HTTP_TIMEOUT): + async with router_request_timeout(DEFAULT_LLM_ROUTER_HTTP_TIMEOUT): gen = self._get_response(body=body, call_method=call_method) @@ -476,7 +490,7 @@ async def embeddings(self, body: EmbeddingRequest) -> Response: Returns: A response object with embeddings. """ - async with timeout(DEFAULT_LLM_ROUTER_HTTP_TIMEOUT): + async with router_request_timeout(DEFAULT_LLM_ROUTER_HTTP_TIMEOUT): results = self._get_response(body=body, call_method="embeddings") result = await results.__anext__() if isinstance(result, ErrorResponse): @@ -502,7 +516,7 @@ async def score(self, body: ScoreRequest) -> Response: A response object with scores. """ - async with timeout(DEFAULT_LLM_ROUTER_HTTP_TIMEOUT): + async with router_request_timeout(DEFAULT_LLM_ROUTER_HTTP_TIMEOUT): results = self._get_response(body=body, call_method="score") result = await results.__anext__() if isinstance(result, ErrorResponse): From ce32813cfce480b0583e04a0f5dc92861d7afd46 Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Thu, 4 Sep 2025 23:26:45 -0700 Subject: [PATCH 462/634] [core][gpu-objects] Fix wrong skipif in test_gpu_objects_gloo (#56258) We should not skip it, since it doesn't need the tensordict dependency. --- python/ray/tests/gpu_objects/test_gpu_objects_gloo.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py b/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py index 1349dc09f4da..7ad5a4a3081d 100644 --- a/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py +++ b/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py @@ -517,10 +517,6 @@ def test_fetch_gpu_object_to_driver(ray_start_regular): assert result[2] == 7 -@pytest.mark.skipif( - not support_tensordict, - reason="tensordict is not supported on this platform", -) def test_invalid_tensor_transport(ray_start_regular): with pytest.raises(ValueError, match="Invalid tensor transport"): From 3303031e563ddab8217fd3ca131d582b138bb4ff Mon Sep 17 00:00:00 2001 From: Mao Yancan Date: Fri, 5 Sep 2025 23:01:46 +0800 Subject: [PATCH 463/634] gc collect from a gc_thread (#55838) Ray proactively triggers gc.collect() on idle workers to release Python objects that may still hold Plasma shared memory (shm) references. In the current implementation in (_raylet.pyx gc_collect()), Ray calls gc.collect() from Cython under a with gil block periodically. If the Python object graph is complex (e.g., cyclic references with finalizers), gc.collect() may take a long time. During this period, since the GIL is held for the entire collection, user code is completely frozen if gc.collect() time is longer than the periodic interval (e.g., 10s). We propose decoupling GC execution from the RPC call: gc_collect in Cython should not directly run gc.collect(). Instead, it should "signal an event" with minimum execution time (e.g., using a threading.Event or similar). A dedicated Python GC thread consumes this event and executes gc.collect() asynchronously, with a configurable GC interval. ## Related issue number Closes #55837 --------- Signed-off-by: Mao Yancan Co-authored-by: Mao Yancan Co-authored-by: Edward Oakes --- python/ray/_private/gc_collect_manager.py | 65 +++++++++++ python/ray/_private/ray_constants.py | 2 + python/ray/_raylet.pxd | 1 + python/ray/_raylet.pyx | 35 ++++-- python/ray/includes/ray_config.pxd | 2 + python/ray/includes/ray_config.pxi | 4 + python/ray/tests/test_basic.py | 2 +- python/ray/tests/test_global_gc.py | 134 ++++++++++++++++++++++ src/ray/common/ray_config_def.h | 3 + src/ray/raylet/worker_pool.cc | 1 + 10 files changed, 240 insertions(+), 9 deletions(-) create mode 100644 python/ray/_private/gc_collect_manager.py diff --git a/python/ray/_private/gc_collect_manager.py b/python/ray/_private/gc_collect_manager.py new file mode 100644 index 000000000000..d9bb723b88b0 --- /dev/null +++ b/python/ray/_private/gc_collect_manager.py @@ -0,0 +1,65 @@ +import gc +import logging +import threading +import time +from typing import Callable, Optional + +logger = logging.getLogger(__name__) + + +class PythonGCThread(threading.Thread): + """A background thread that triggers Python garbage collection. + + This thread waits for GC events from CoreWorker and triggers `gc.collect()` when + requested, ensuring that collections are spaced out by at least + `min_interval_s` seconds.""" + + def __init__( + self, *, min_interval_s: int = 5, gc_collect_func: Optional[Callable] = None + ): + logger.debug("Starting Python GC thread") + super().__init__(name="PythonGCThread", daemon=True) + self._should_exit = False + self._last_gc_time = float("-inf") + self._min_gc_interval = min_interval_s + self._gc_event = threading.Event() + # Set the gc_collect_func for UT, defaulting to gc.collect if None + self._gc_collect_func = gc_collect_func or gc.collect + + def trigger_gc(self) -> None: + self._gc_event.set() + + def run(self): + while not self._should_exit: + self._gc_event.wait() + self._gc_event.clear() + + if self._should_exit: + break + + time_since_last_gc = time.monotonic() - self._last_gc_time + if time_since_last_gc < self._min_gc_interval: + logger.debug( + f"Skipping GC, only {time_since_last_gc:.2f}s since last GC" + ) + continue + + try: + start = time.monotonic() + num_freed = self._gc_collect_func() + self._last_gc_time = time.monotonic() + if num_freed > 0: + logger.debug( + "gc.collect() freed {} refs in {} seconds".format( + num_freed, self._last_gc_time - start + ) + ) + except Exception as e: + logger.error(f"Error during GC: {e}") + self._last_gc_time = time.monotonic() + + def stop(self): + logger.debug("Stopping Python GC thread") + self._should_exit = True + self._gc_event.set() + self.join() diff --git a/python/ray/_private/ray_constants.py b/python/ray/_private/ray_constants.py index 33de100852ba..c161a95c38f9 100644 --- a/python/ray/_private/ray_constants.py +++ b/python/ray/_private/ray_constants.py @@ -584,3 +584,5 @@ def gcs_actor_scheduling_enabled(): FETCH_FAIL_TIMEOUT_SECONDS = ( env_integer("RAY_fetch_fail_timeout_milliseconds", 60000) / 1000 ) + +RAY_GC_MIN_COLLECT_INTERVAL = env_float("RAY_GC_MIN_COLLECT_INTERVAL_S", 5) diff --git a/python/ray/_raylet.pxd b/python/ray/_raylet.pxd index 89ef3261db2a..98b445fc3abb 100644 --- a/python/ray/_raylet.pxd +++ b/python/ray/_raylet.pxd @@ -140,6 +140,7 @@ cdef class CoreWorker: object _task_id_to_future_lock dict _task_id_to_future object event_loop_executor + object _gc_thread cdef unique_ptr[CAddress] _convert_python_address(self, address=*) cdef store_task_output( diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index a8c02bc4848d..edcb38d2c41c 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -248,6 +248,7 @@ from ray._private.utils import DeferSigint from ray._private.object_ref_generator import DynamicObjectRefGenerator from ray.util.annotations import PublicAPI from ray._private.custom_types import TensorTransportEnum +from ray._private.gc_collect_manager import PythonGCThread # Expose GCC & Clang macro to report # whether C++ optimizations were enabled during compilation. @@ -2496,14 +2497,21 @@ cdef CRayStatus check_signals() nogil: cdef void gc_collect(c_bool triggered_by_global_gc) nogil: - with gil: - start = time.perf_counter() - num_freed = gc.collect() - end = time.perf_counter() - if num_freed > 0: - logger.debug( - "gc.collect() freed {} refs in {} seconds".format( - num_freed, end - start)) + with gil: + if RayConfig.instance().start_python_gc_manager_thread(): + start = time.perf_counter() + worker = ray._private.worker.global_worker + worker.core_worker.trigger_gc() + end = time.perf_counter() + logger.debug("GC event triggered in {} seconds".format(end - start)) + else: + start = time.perf_counter() + num_freed = gc.collect() + end = time.perf_counter() + if num_freed > 0: + logger.debug( + "gc.collect() freed {} refs in {} seconds".format( + num_freed, end - start)) cdef c_vector[c_string] spill_objects_handler( @@ -3054,6 +3062,11 @@ cdef class CoreWorker: self._task_id_to_future = {} self.event_loop_executor = None + self._gc_thread = None + if RayConfig.instance().start_python_gc_manager_thread(): + self._gc_thread = PythonGCThread(min_interval_s=ray_constants.RAY_GC_MIN_COLLECT_INTERVAL) + self._gc_thread.start() + def shutdown_driver(self): # If it's a worker, the core worker process should have been # shutdown. So we can't call @@ -3061,6 +3074,9 @@ cdef class CoreWorker: # Instead, we use the cached `is_driver` flag to test if it's a # driver. assert self.is_driver + if self._gc_thread is not None: + self._gc_thread.stop() + self._gc_thread = None with nogil: CCoreWorkerProcess.Shutdown() @@ -4719,6 +4735,9 @@ cdef class CoreWorker: return self.current_runtime_env + def trigger_gc(self): + self._gc_thread.trigger_gc() + def get_pending_children_task_ids(self, parent_task_id: TaskID): cdef: CTaskID c_parent_task_id = parent_task_id.native() diff --git a/python/ray/includes/ray_config.pxd b/python/ray/includes/ray_config.pxd index 01e9827b98f9..729395a22ee3 100644 --- a/python/ray/includes/ray_config.pxd +++ b/python/ray/includes/ray_config.pxd @@ -86,3 +86,5 @@ cdef extern from "ray/common/ray_config.h" nogil: int maximum_gcs_destroyed_actor_cached_count() const c_bool record_task_actor_creation_sites() const + + c_bool start_python_gc_manager_thread() const diff --git a/python/ray/includes/ray_config.pxi b/python/ray/includes/ray_config.pxi index 26236f2cce2c..6915e4877962 100644 --- a/python/ray/includes/ray_config.pxi +++ b/python/ray/includes/ray_config.pxi @@ -140,3 +140,7 @@ cdef class Config: @staticmethod def maximum_gcs_destroyed_actor_cached_count(): return RayConfig.instance().maximum_gcs_destroyed_actor_cached_count() + + @staticmethod + def start_python_gc_manager_thread(): + return RayConfig.instance().start_python_gc_manager_thread() diff --git a/python/ray/tests/test_basic.py b/python/ray/tests/test_basic.py index c2bb0c662034..968005cf4980 100644 --- a/python/ray/tests/test_basic.py +++ b/python/ray/tests/test_basic.py @@ -267,7 +267,7 @@ def get_thread_count(self): ray.get(actor.get_thread_count.remote()) # Lowering these numbers in this assert should be celebrated, # increasing these numbers should be scrutinized - assert ray.get(actor.get_thread_count.remote()) in {24, 25} + assert ray.get(actor.get_thread_count.remote()) in {24, 25, 26} # https://github.com/ray-project/ray/issues/7287 diff --git a/python/ray/tests/test_global_gc.py b/python/ray/tests/test_global_gc.py index ad3aca696ebb..1e6fa4cf2606 100644 --- a/python/ray/tests/test_global_gc.py +++ b/python/ray/tests/test_global_gc.py @@ -2,7 +2,9 @@ import gc import logging import sys +import time import weakref +from unittest.mock import Mock import numpy as np import pytest @@ -10,6 +12,7 @@ import ray import ray.cluster_utils from ray._common.test_utils import wait_for_condition +from ray._private.gc_collect_manager import PythonGCThread from ray._private.internal_api import global_gc logger = logging.getLogger(__name__) @@ -216,5 +219,136 @@ def f(self): gc.enable() +def test_local_gc_called_once_per_interval(shutdown_only): + ray.init( + num_cpus=2, + _system_config={ + "local_gc_interval_s": 1, + "local_gc_min_interval_s": 0, + "global_gc_min_interval_s": 0, + }, + ) + + class ObjectWithCyclicRef: + def __init__(self): + self.loop = self + + @ray.remote(num_cpus=1) + class GarbageHolder: + def __init__(self): + gc.disable() + self.garbage = None + + def make_garbage(self): + x = ObjectWithCyclicRef() + self.garbage = weakref.ref(x) + return True + + def has_garbage(self): + return self.garbage() is not None + + def all_garbage_collected(local_ref): + return local_ref() is None and not any( + ray.get([a.has_garbage.remote() for a in actors]) + ) + + try: + gc.disable() + + # Round 1: first batch of garbage should be collected + # Local driver. + local_ref = weakref.ref(ObjectWithCyclicRef()) + # Remote workers. + actors = [GarbageHolder.remote() for _ in range(2)] + ray.get([a.make_garbage.remote() for a in actors]) + + assert local_ref() is not None + assert all(ray.get([a.has_garbage.remote() for a in actors])) + + wait_for_condition( + lambda: all_garbage_collected(local_ref), + ) + + # Round 2: second batch should NOT be collected within min_interval + local_ref = weakref.ref(ObjectWithCyclicRef()) + ray.get([a.make_garbage.remote() for a in actors]) + + with pytest.raises(RuntimeError): + wait_for_condition( + lambda: all_garbage_collected(local_ref), + timeout=2.0, # shorter than min_interval + retry_interval_ms=50, + ) + + # Round 3: after min_interval passes, garbage should be collected + wait_for_condition( + lambda: all_garbage_collected(local_ref), + timeout=10.0, + retry_interval_ms=50, + ) + + finally: + gc.enable() + + +def test_gc_manager_thread_basic_functionality(): + mock_gc_collect = Mock(return_value=10) + + gc_thread = PythonGCThread(min_interval_s=1, gc_collect_func=mock_gc_collect) + + try: + gc_thread.start() + assert gc_thread.is_alive() + + gc_thread.trigger_gc() + + wait_for_condition(lambda: mock_gc_collect.call_count == 1, timeout=2) + + mock_gc_collect.assert_called_once() + + finally: + gc_thread.stop() + assert not gc_thread.is_alive() + + +def test_gc_manager_thread_min_interval_throttling(): + mock_gc_collect = Mock(return_value=5) + + gc_thread = PythonGCThread(min_interval_s=2, gc_collect_func=mock_gc_collect) + + try: + gc_thread.start() + + for _ in range(3): + gc_thread.trigger_gc() + time.sleep(1) + + wait_for_condition(lambda: mock_gc_collect.call_count == 2, timeout=2) + + assert mock_gc_collect.call_count == 2 + + finally: + gc_thread.stop() + + +def test_gc_manager_thread_exception_handling(): + mock_gc_collect = Mock(side_effect=RuntimeError("GC failed")) + + gc_thread = PythonGCThread(min_interval_s=5, gc_collect_func=mock_gc_collect) + + try: + gc_thread.start() + + for _ in range(3): + gc_thread.trigger_gc() + time.sleep(0.1) + + assert gc_thread.is_alive() + mock_gc_collect.assert_called_once() + + finally: + gc_thread.stop() + + if __name__ == "__main__": sys.exit(pytest.main(["-sv", __file__])) diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index 6e89e27d4b25..d496e52fed7b 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -945,3 +945,6 @@ RAY_CONFIG(int32_t, raylet_rpc_server_reconnect_timeout_s, 60) // process getting spawned. Setting to zero or less maintains the default // number of threads grpc will spawn. RAY_CONFIG(int64_t, worker_num_grpc_internal_threads, 0) + +// Whether to start a background thread to manage Python GC in workers. +RAY_CONFIG(bool, start_python_gc_manager_thread, true) diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index 5f5719d206f1..f20274fbb5b8 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -461,6 +461,7 @@ WorkerPool::BuildProcessCommandArgs(const Language &language, // Support forking in gRPC. env.insert({"GRPC_ENABLE_FORK_SUPPORT", "True"}); env.insert({"GRPC_POLL_STRATEGY", "poll"}); + env.insert({"RAY_start_python_gc_manager_thread", "0"}); } return {std::move(worker_command_args), std::move(env)}; From 731c1796a10194c9a9c09f8092825a79f515b296 Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 5 Sep 2025 08:49:07 -0700 Subject: [PATCH 464/634] [Core] Remove the unnecessary redirection of get_protocols_provider (#56262) Signed-off-by: Jiajun Yao --- python/ray/_private/runtime_env/default_impl.py | 6 ------ python/ray/_private/runtime_env/protocol.py | 10 +++------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/python/ray/_private/runtime_env/default_impl.py b/python/ray/_private/runtime_env/default_impl.py index 331dc7fce01e..f0d1567530af 100644 --- a/python/ray/_private/runtime_env/default_impl.py +++ b/python/ray/_private/runtime_env/default_impl.py @@ -3,9 +3,3 @@ def get_image_uri_plugin_cls(): return ImageURIPlugin - - -def get_protocols_provider(): - from ray._private.runtime_env.protocol import ProtocolsProvider - - return ProtocolsProvider diff --git a/python/ray/_private/runtime_env/protocol.py b/python/ray/_private/runtime_env/protocol.py index 9d9887b409a5..d5db021f75c8 100644 --- a/python/ray/_private/runtime_env/protocol.py +++ b/python/ray/_private/runtime_env/protocol.py @@ -1,8 +1,6 @@ import enum import os -from ray._private.runtime_env.default_impl import get_protocols_provider - class ProtocolsProvider: _MISSING_DEPENDENCIES_WARNING = ( @@ -210,11 +208,9 @@ def open_file(uri, mode, *, transport_params=None): fout.write(fin.read()) -_protocols_provider = get_protocols_provider() - Protocol = enum.Enum( "Protocol", - {protocol.upper(): protocol for protocol in _protocols_provider.get_protocols()}, + {protocol.upper(): protocol for protocol in ProtocolsProvider.get_protocols()}, ) @@ -223,7 +219,7 @@ def _remote_protocols(cls): # Returns a list of protocols that support remote storage # These protocols should only be used with paths that end in ".zip" or ".whl" return [ - cls[protocol.upper()] for protocol in _protocols_provider.get_remote_protocols() + cls[protocol.upper()] for protocol in ProtocolsProvider.get_remote_protocols() ] @@ -231,7 +227,7 @@ def _remote_protocols(cls): def _download_remote_uri(self, source_uri, dest_file): - return _protocols_provider.download_remote_uri(self.value, source_uri, dest_file) + return ProtocolsProvider.download_remote_uri(self.value, source_uri, dest_file) Protocol.download_remote_uri = _download_remote_uri From 5278961d5a56d1a42d850dbdbab0894e3d38ae8e Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Fri, 5 Sep 2025 21:34:12 +0530 Subject: [PATCH 465/634] Enable ruff lint for `workers/`, `workflow/`, `setup-dev.py`, and `cloudpickle/` (#56081) Signed-off-by: Gagandeep Singh Co-authored-by: Edward Oakes --- .pre-commit-config.yaml | 2 +- pyproject.toml | 5 ----- python/ray/setup-dev.py | 3 ++- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 189a39b5d48e..89ba195ebe62 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: args: [ --fix, --exit-non-zero-on-fix ] - id: ruff args: [ --select, "I", --fix, --exit-non-zero-on-fix ] - files: '^python/ray/serve/|^python/ray/train|^python/ray/data|^python/ray/_private/|^python/ray/llm/|^python/ray/tune/|^python/ray/includes/|^python/ray/internal/|^python/ray/ray_operator/|^python/ray/scripts/|^python/ray/streaming/|^python/ray/dag/|^python/ray/tests/' + files: '^python/ray/serve/|^python/ray/train|^python/ray/data|^python/ray/_private/|^python/ray/llm/|^python/ray/tune/|^python/ray/includes/|^python/ray/internal/|^python/ray/ray_operator/|^python/ray/scripts/|^python/ray/streaming/|^python/ray/dag/|^python/ray/tests/|^python/ray/setup-dev.py|^python/ray/cloudpickle/|^python/ray/workers/|^python/ray/workflow/' - repo: https://github.com/jsh9/pydoclint rev: "0.6.6" diff --git a/pyproject.toml b/pyproject.toml index 63ab2c1660d6..5711be013c27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,18 +58,13 @@ afterray = ["psutil", "setproctitle"] # python/ray/cloudpickle/* # doc/* # python/ray/__init__.py -# python/ray/setup-dev.py # For the rest we will gradually remove them from the blacklist as we # reformat the code to follow the style guide. [tool.ruff.lint.per-file-ignores] "doc/*" = ["I"] "python/ray/__init__.py" = ["I"] -"python/ray/setup-dev.py" = ["I"] -"python/ray/cloudpickle/*" = ["I"] "python/ray/dag/__init__.py" = ["I"] "python/ray/util/*" = ["I"] -"python/ray/workers/*" = ["I"] -"python/ray/workflow/*" = ["I"] "rllib/*" = ["I"] "release/*" = ["I"] diff --git a/python/ray/setup-dev.py b/python/ray/setup-dev.py index 372ee95912ef..8f65e4e716ff 100755 --- a/python/ray/setup-dev.py +++ b/python/ray/setup-dev.py @@ -16,10 +16,11 @@ sys.path.append(this_dir) import argparse -import click import shutil import subprocess +import click + import ray From 9c7028b8a3fcb78209a34a644b72443e5b3b8c2f Mon Sep 17 00:00:00 2001 From: Potato Date: Sat, 6 Sep 2025 00:20:24 +0800 Subject: [PATCH 466/634] [CORE][DOC] Fix documentation typos, grammar, and formatting issues in ray-core directories (#56275) Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- doc/source/ray-core/actors/async_api.rst | 6 +++--- doc/source/ray-core/actors/concurrency_group_api.rst | 9 +++++---- doc/source/ray-core/actors/named-actors.rst | 4 ++-- doc/source/ray-core/actors/out-of-band-communication.rst | 4 ++-- doc/source/ray-core/actors/terminating-actors.rst | 8 ++++---- doc/source/ray-core/compiled-graph/profiling.rst | 2 +- .../ray-core/compiled-graph/ray-compiled-graph.rst | 2 +- 7 files changed, 18 insertions(+), 17 deletions(-) diff --git a/doc/source/ray-core/actors/async_api.rst b/doc/source/ray-core/actors/async_api.rst index 7a64a774c85a..50427459d433 100644 --- a/doc/source/ray-core/actors/async_api.rst +++ b/doc/source/ray-core/actors/async_api.rst @@ -22,7 +22,7 @@ AsyncIO for Actors Since Python 3.5, it is possible to write concurrent code using the ``async/await`` `syntax `__. -Ray natively integrates with asyncio. You can use ray alongside with popular +Ray natively integrates with asyncio. You can use Ray alongside popular async frameworks like aiohttp, aioredis, etc. .. testcode:: @@ -217,7 +217,7 @@ Please note that running blocking ``ray.get`` or ``ray.wait`` inside async actor method is not allowed, because ``ray.get`` will block the execution of the event loop. -In async actors, only one task can be running at any point in time (though tasks can be multi-plexed). There will be only one thread in AsyncActor! See :ref:`threaded-actors` if you want a threadpool. +In async actors, only one task can be running at any point in time (though tasks can be multiplexed). There will be only one thread in AsyncActor! See :ref:`threaded-actors` if you want a threadpool. Setting concurrency in Async Actors ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -284,7 +284,7 @@ Sometimes, asyncio is not an ideal solution for your actor. For example, you may have one method that performs some computation heavy task while blocking the event loop, not giving up control via ``await``. This would hurt the performance of an Async Actor because Async Actors can only execute 1 task at a time and rely on ``await`` to context switch. -Instead, you can use the ``max_concurrency`` Actor options without any async methods, allowng you to achieve threaded concurrency (like a thread pool). +Instead, you can use the ``max_concurrency`` Actor options without any async methods, allowing you to achieve threaded concurrency (like a thread pool). .. warning:: diff --git a/doc/source/ray-core/actors/concurrency_group_api.rst b/doc/source/ray-core/actors/concurrency_group_api.rst index 326f3b957bbc..29e9e6d788b7 100644 --- a/doc/source/ray-core/actors/concurrency_group_api.rst +++ b/doc/source/ray-core/actors/concurrency_group_api.rst @@ -1,7 +1,7 @@ Limiting Concurrency Per-Method with Concurrency Groups ======================================================= -Besides setting the max concurrency overall for an actor, Ray allows methods to be separated into *concurrency groups*, each with its own threads(s). This allows you to limit the concurrency per-method, e.g., allow a health-check method to be given its own concurrency quota separate from request serving methods. +Besides setting the max concurrency overall for an actor, Ray allows methods to be separated into *concurrency groups*, each with its own thread(s). This allows you to limit the concurrency per-method, e.g., allow a health-check method to be given its own concurrency quota separate from request serving methods. .. tip:: Concurrency groups work with both asyncio and threaded actors. The syntax is the same. @@ -14,7 +14,7 @@ This defines two concurrency groups, "io" with max concurrency = 2 and "compute" with max concurrency = 4. The methods ``f1`` and ``f2`` are placed in the "io" group, and the methods ``f3`` and ``f4`` are placed into the "compute" group. Note that there is always a default -concurrency group for actors, which has a default concurrency of 1000 +concurrency group for actors, which has a default concurrency of 1000 for AsyncIO actors and 1 otherwise. .. tab-set:: @@ -143,10 +143,11 @@ The concurrency of the default group can be changed by setting the ``max_concurr .. code-block:: java - class ConcurrentActor: + class ConcurrentActor { public long f1() { return Thread.currentThread().getId(); } + } ConcurrencyGroup group = new ConcurrencyGroupBuilder() @@ -156,7 +157,7 @@ The concurrency of the default group can be changed by setting the ``max_concurr .build(); ActorHandle myActor = Ray.actor(ConcurrentActor::new) - .setConcurrencyGroups(group1) + .setConcurrencyGroups(group) .setMaxConcurrency(10) .remote(); diff --git a/doc/source/ray-core/actors/named-actors.rst b/doc/source/ray-core/actors/named-actors.rst index 81ec48230010..980a2053c46f 100644 --- a/doc/source/ray-core/actors/named-actors.rst +++ b/doc/source/ray-core/actors/named-actors.rst @@ -52,7 +52,7 @@ exist. See :ref:`actor-lifetimes` for more details. // Retrieve the actor later somewhere boost::optional> counter = ray::GetGlobalActor("some_name"); - We also support non-global named actors in C++, which means that the actor name is only valid within the job and the actor cannot be accessed from another job + We also support non-global named actors in C++, which means that the actor name is only valid within the job and the actor cannot be accessed from another job. .. code-block:: c++ @@ -80,7 +80,7 @@ exist. See :ref:`actor-lifetimes` for more details. @ray.remote class Actor: - pass + pass # driver_1.py # Job 1 creates an actor, "orange" in the "colors" namespace. diff --git a/doc/source/ray-core/actors/out-of-band-communication.rst b/doc/source/ray-core/actors/out-of-band-communication.rst index 063b9a26f69d..ba6719046ecc 100644 --- a/doc/source/ray-core/actors/out-of-band-communication.rst +++ b/doc/source/ray-core/actors/out-of-band-communication.rst @@ -19,8 +19,8 @@ See :ref:`Ray Collective ` for more details. HTTP Server ----------- -You can start a http server inside the actor and expose http endpoints to clients -so users outside of the ray cluster can communicate with the actor. +You can start an HTTP server inside the actor and expose HTTP endpoints to clients +so users outside of the Ray cluster can communicate with the actor. .. tab-set:: diff --git a/doc/source/ray-core/actors/terminating-actors.rst b/doc/source/ray-core/actors/terminating-actors.rst index 3ac8dc22eefb..2422ffa88068 100644 --- a/doc/source/ray-core/actors/terminating-actors.rst +++ b/doc/source/ray-core/actors/terminating-actors.rst @@ -64,7 +64,7 @@ Ray to :ref:`automatically restart ` the actor, make sur flag ``no_restart=False`` to ``ray.kill``. For :ref:`named and detached actors `, calling ``ray.kill`` on -an actor handle destroys the actor and allow the name to be reused. +an actor handle destroys the actor and allows the name to be reused. Use `ray list actors --detail` from :ref:`State API ` to see the death cause of dead actors: @@ -133,7 +133,7 @@ This will kill the actor process and release resources associated/assigned to th Ray.exitActor(); - Garbage collection for actors haven't been implemented yet, so this is currently the + Garbage collection for actors hasn't been implemented yet, so this is currently the only way to terminate an actor gracefully. The ``ObjectRef`` resulting from the task can be waited on to wait for the actor to exit (calling ``ObjectRef::get`` on it will throw a ``RayActorException``). @@ -144,7 +144,7 @@ This will kill the actor process and release resources associated/assigned to th ray::ExitActor(); - Garbage collection for actors haven't been implemented yet, so this is currently the + Garbage collection for actors hasn't been implemented yet, so this is currently the only way to terminate an actor gracefully. The ``ObjectRef`` resulting from the task can be waited on to wait for the actor to exit (calling ``ObjectRef::Get`` on it will throw a ``RayActorException``). @@ -178,7 +178,7 @@ You could see the actor is dead as a result of the user's `exit_actor()` call: actor_died_error_context: error_message: 'The actor is dead because its worker process has died. Worker exit type: INTENDED_USER_EXIT Worker exit detail: Worker exits - by an user request. exit_actor() is called.' + by a user request. exit_actor() is called.' owner_id: 02000000ffffffffffffffffffffffffffffffffffffffffffffffff owner_ip_address: 127.0.0.1 node_ip_address: 127.0.0.1 diff --git a/doc/source/ray-core/compiled-graph/profiling.rst b/doc/source/ray-core/compiled-graph/profiling.rst index 8c3ac5d4bfb8..ddfae1313541 100644 --- a/doc/source/ray-core/compiled-graph/profiling.rst +++ b/doc/source/ray-core/compiled-graph/profiling.rst @@ -2,7 +2,7 @@ Profiling ========= Ray Compiled Graph provides both PyTorch-based and Nsight-based profiling functionalities to better understand the performance -of individual tasks, systems overhead, and performance bottlenecks. You can pick your favorite profiler based on your preference. +of individual tasks, system overhead, and performance bottlenecks. You can pick your favorite profiler based on your preference. PyTorch profiler ---------------- diff --git a/doc/source/ray-core/compiled-graph/ray-compiled-graph.rst b/doc/source/ray-core/compiled-graph/ray-compiled-graph.rst index 88cbd6bb4b11..b40956af2e94 100644 --- a/doc/source/ray-core/compiled-graph/ray-compiled-graph.rst +++ b/doc/source/ray-core/compiled-graph/ray-compiled-graph.rst @@ -12,7 +12,7 @@ As large language models (LLMs) become common, programming distributed systems w :ref:`Ray Core APIs ` facilitate using multiple GPUs but have limitations such as: * System overhead of ~1 ms per task launch, which is unsuitable for high-performance tasks like LLM inference. -* Lack support for direct GPU-to-GPU communication, requiring manual development with external libraries like NVIDIA Collective Communications Library (`NCCL `_). +* Lack of support for direct GPU-to-GPU communication, requiring manual development with external libraries like NVIDIA Collective Communications Library (`NCCL `_). Ray Compiled Graph gives you a Ray Core-like API but with: From d9709c8be45dd0e19e82ebbae246cd19b81df9d3 Mon Sep 17 00:00:00 2001 From: Rueian Date: Fri, 5 Sep 2025 09:48:29 -0700 Subject: [PATCH 467/634] [core][autoscaler] Reword `Total Demands` and `Total Constraints` to `Pending Demands` and `From request_resources` (#55787) Signed-off-by: Rueian Signed-off-by: Rueian Co-authored-by: Edward Oakes --- .../user-guides/configuring-autoscaling.md | 2 +- python/ray/autoscaler/_private/util.py | 6 ++-- python/ray/autoscaler/v2/tests/test_utils.py | 4 +-- python/ray/autoscaler/v2/utils.py | 6 ++-- python/ray/tests/test_autoscaler_e2e.py | 9 ++++-- .../test_cli_patterns/test_ray_status.txt | 6 ++-- .../test_ray_status_multinode.txt | 6 ++-- .../test_ray_status_multinode_v1.txt | 6 ++-- .../test_cli_patterns/test_ray_status_v1.txt | 6 ++-- .../tests/test_resource_demand_scheduler.py | 32 +++++++++---------- 10 files changed, 43 insertions(+), 40 deletions(-) diff --git a/doc/source/cluster/kubernetes/user-guides/configuring-autoscaling.md b/doc/source/cluster/kubernetes/user-guides/configuring-autoscaling.md index 1586124868a9..b24b891c7b45 100644 --- a/doc/source/cluster/kubernetes/user-guides/configuring-autoscaling.md +++ b/doc/source/cluster/kubernetes/user-guides/configuring-autoscaling.md @@ -431,7 +431,7 @@ Total Usage: 0B/72.63GiB memory 0B/33.53GiB object_store_memory -Total Demands: +Pending Demands: (no resource demands) Node: 40f427230584b2d9c9f113d8db51d10eaf914aa9bf61f81dc7fabc64 diff --git a/python/ray/autoscaler/_private/util.py b/python/ray/autoscaler/_private/util.py index 66a7e5a8b900..de5b9f506c3e 100644 --- a/python/ray/autoscaler/_private/util.py +++ b/python/ray/autoscaler/_private/util.py @@ -756,7 +756,7 @@ def get_constraint_report(request_demand: List[DictCount]): if len(constraint_lines) > 0: constraints_report = "\n".join(constraint_lines) else: - constraints_report = " (no request_resources() constraints)" + constraints_report = " (none)" return constraints_report @@ -948,9 +948,9 @@ def format_info_string( {separator} Total Usage: {usage_report} -Total Constraints: +From request_resources: {constraints_report} -Total Demands: +Pending Demands: {demand_report}""" if verbose: diff --git a/python/ray/autoscaler/v2/tests/test_utils.py b/python/ray/autoscaler/v2/tests/test_utils.py index 1322c5ee3b0d..2bec1c29e4e4 100644 --- a/python/ray/autoscaler/v2/tests/test_utils.py +++ b/python/ray/autoscaler/v2/tests/test_utils.py @@ -567,9 +567,9 @@ def test_cluster_status_formatter(): 0.0/4.0 GPU 5.42KiB/10.04KiB object_store_memory -Total Constraints: +From request_resources: {'GPU': 2, 'CPU': 100}: 2 from request_resources() -Total Demands: +Pending Demands: {'CPU': 1, 'GPU': 1}: 11+ pending tasks/actors {'CPU': 1, 'GPU': 1} * 1 (STRICT_SPREAD): 1+ pending placement groups {'GPU': 2} * 1 (STRICT_PACK): 2+ pending placement groups diff --git a/python/ray/autoscaler/v2/utils.py b/python/ray/autoscaler/v2/utils.py index d3128e961c63..8cf3dd13fc94 100644 --- a/python/ray/autoscaler/v2/utils.py +++ b/python/ray/autoscaler/v2/utils.py @@ -400,9 +400,9 @@ def format(cls, data: ClusterStatus, verbose: bool = False) -> str: separator, "Total Usage:", cluster_usage_report, - "Total Constraints:", + "From request_resources:", constraints_report, - "Total Demands:", + "Pending Demands:", demand_report, node_usage_report, ] @@ -631,7 +631,7 @@ def _constraint_report( constraint_lines.append(f" {bundle}: {count} from request_resources()") if constraint_lines: return "\n".join(constraint_lines) - return " (no request_resources() constraints)" + return " (none)" @staticmethod def _demand_report(data: ClusterStatus) -> str: diff --git a/python/ray/tests/test_autoscaler_e2e.py b/python/ray/tests/test_autoscaler_e2e.py index f3ea5dec3195..5585413e86b9 100644 --- a/python/ray/tests/test_autoscaler_e2e.py +++ b/python/ray/tests/test_autoscaler_e2e.py @@ -124,12 +124,15 @@ def ping(self): actor = Actor.remote() ray.get(actor.ping.remote()) - assert "Total Demands" in subprocess.check_output("ray status", shell=True).decode() assert ( - "Total Demands" in subprocess.check_output("ray status -v", shell=True).decode() + "Pending Demands" in subprocess.check_output("ray status", shell=True).decode() ) assert ( - "Total Demands" + "Pending Demands" + in subprocess.check_output("ray status -v", shell=True).decode() + ) + assert ( + "Pending Demands" in subprocess.check_output("ray status --verbose", shell=True).decode() ) diff --git a/python/ray/tests/test_cli_patterns/test_ray_status.txt b/python/ray/tests/test_cli_patterns/test_ray_status.txt index 5cdf2e0a220a..998eacc9c3f4 100644 --- a/python/ray/tests/test_cli_patterns/test_ray_status.txt +++ b/python/ray/tests/test_cli_patterns/test_ray_status.txt @@ -17,7 +17,7 @@ Total Usage: 0.+ 0.+ -Total Constraints: - \(no request_resources\(\) constraints\) -Total Demands: +From request_resources: + \(none\) +Pending Demands: \(no resource demands\) diff --git a/python/ray/tests/test_cli_patterns/test_ray_status_multinode.txt b/python/ray/tests/test_cli_patterns/test_ray_status_multinode.txt index c86f8cf00c89..b0ada8cd82c3 100644 --- a/python/ray/tests/test_cli_patterns/test_ray_status_multinode.txt +++ b/python/ray/tests/test_cli_patterns/test_ray_status_multinode.txt @@ -20,7 +20,7 @@ Total Usage: 0.+ 0.+ -Total Constraints: - \(no request_resources\(\) constraints\) -Total Demands: +From request_resources: + \(none\) +Pending Demands: \(no resource demands\) diff --git a/python/ray/tests/test_cli_patterns/test_ray_status_multinode_v1.txt b/python/ray/tests/test_cli_patterns/test_ray_status_multinode_v1.txt index cd228fbc591d..537cab7f8abc 100644 --- a/python/ray/tests/test_cli_patterns/test_ray_status_multinode_v1.txt +++ b/python/ray/tests/test_cli_patterns/test_ray_status_multinode_v1.txt @@ -18,7 +18,7 @@ Total Usage: 0.+ 0.+ -Total Constraints: - \(no request_resources\(\) constraints\) -Total Demands: +From request_resources: + \(none\) +Pending Demands: \(no resource demands\) diff --git a/python/ray/tests/test_cli_patterns/test_ray_status_v1.txt b/python/ray/tests/test_cli_patterns/test_ray_status_v1.txt index ec5125f5eb0e..8eac046f8444 100644 --- a/python/ray/tests/test_cli_patterns/test_ray_status_v1.txt +++ b/python/ray/tests/test_cli_patterns/test_ray_status_v1.txt @@ -15,7 +15,7 @@ Total Usage: 0.+ 0.+ -Total Constraints: - \(no request_resources\(\) constraints\) -Total Demands: +From request_resources: + \(none\) +Pending Demands: \(no resource demands\) diff --git a/python/ray/tests/test_resource_demand_scheduler.py b/python/ray/tests/test_resource_demand_scheduler.py index 2dbd2e459a14..2c8d921bf0a2 100644 --- a/python/ray/tests/test_resource_demand_scheduler.py +++ b/python/ray/tests/test_resource_demand_scheduler.py @@ -3284,9 +3284,9 @@ def test_info_string(): 2.00GiB/8.00GiB memory 3.14GiB/16.00GiB object_store_memory -Total Constraints: +From request_resources: {'CPU': 16}: 100 from request_resources() -Total Demands: +Pending Demands: {'CPU': 1}: 150+ pending tasks/actors {'CPU': 4} * 5 (PACK): 420+ pending placement groups """.strip() @@ -3341,10 +3341,10 @@ def test_info_string_multiple_constraints(): 2.00GiB/8.00GiB memory 3.14GiB/16.00GiB object_store_memory -Total Constraints: +From request_resources: {'CPU': 16}: 100 from request_resources() {'CPU': 1, 'GPU': 16}: 10 from request_resources() -Total Demands: +Pending Demands: {'CPU': 1}: 150+ pending tasks/actors {'CPU': 4} * 5 (PACK): 420+ pending placement groups """.strip() @@ -3433,9 +3433,9 @@ def test_info_string_verbose(): 2.00GiB/8.00GiB memory 3.14GiB/16.00GiB object_store_memory -Total Constraints: +From request_resources: {'CPU': 16}: 100 from request_resources() -Total Demands: +Pending Demands: {'CPU': 1}: 150+ pending tasks/actors {'CPU': 4} * 5 (PACK): 420+ pending placement groups @@ -3548,9 +3548,9 @@ def test_info_string_verbose_node_types(): 2.00GiB/8.00GiB memory 3.14GiB/16.00GiB object_store_memory -Total Constraints: +From request_resources: {'CPU': 16}: 100 from request_resources() -Total Demands: +Pending Demands: {'CPU': 1}: 150+ pending tasks/actors {'CPU': 4} * 5 (PACK): 420+ pending placement groups @@ -3640,9 +3640,9 @@ def test_info_string_verbose_no_breakdown(): 2.00GiB/8.00GiB memory 3.14GiB/16.00GiB object_store_memory -Total Constraints: +From request_resources: {'CPU': 16}: 100 from request_resources() -Total Demands: +Pending Demands: {'CPU': 1}: 150+ pending tasks/actors {'CPU': 4} * 5 (PACK): 420+ pending placement groups """.strip() @@ -3735,9 +3735,9 @@ def test_info_string_with_launch_failures(): 2.00GiB/8.00GiB memory 3.14GiB/16.00GiB object_store_memory -Total Constraints: +From request_resources: {'CPU': 16}: 100 from request_resources() -Total Demands: +Pending Demands: {'CPU': 1}: 150+ pending tasks/actors {'CPU': 4} * 5 (PACK): 420+ pending placement groups """.strip() @@ -3828,9 +3828,9 @@ def test_info_string_with_launch_failures_verbose(): 2.00GiB/8.00GiB memory 3.14GiB/16.00GiB object_store_memory -Total Constraints: +From request_resources: {'CPU': 16}: 100 from request_resources() -Total Demands: +Pending Demands: {'CPU': 1}: 150+ pending tasks/actors {'CPU': 4} * 5 (PACK): 420+ pending placement groups """.strip() @@ -3917,9 +3917,9 @@ def test_info_string_failed_node_cap(): 2.00GiB/8.00GiB memory 3.14GiB/16.00GiB object_store_memory -Total Constraints: +From request_resources: {'CPU': 16}: 100 from request_resources() -Total Demands: +Pending Demands: {'CPU': 2.0}: 153+ pending tasks/actors (3+ using placement groups) {'GPU': 0.5}: 100+ pending tasks/actors (100+ using placement groups) {'CPU': 4} * 5 (PACK): 420+ pending placement groups From 4a9c49c2c994197f709a8629ce85913c74e82393 Mon Sep 17 00:00:00 2001 From: Matthew Owen Date: Fri, 5 Sep 2025 09:54:10 -0700 Subject: [PATCH 468/634] [data] Adding in updated code to from uris release test (#56091) ## Why are these changes needed? Update the from uris release test to use the new code from https://github.com/ray-project/ray/pull/55824. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Matthew Owen --- .../logical/operators/map_operator.py | 10 ++++- .../dataset/read_from_uris_benchmark.py | 42 +++++++++++-------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/python/ray/data/_internal/logical/operators/map_operator.py b/python/ray/data/_internal/logical/operators/map_operator.py index 8afee463aef1..c6597ae084a2 100644 --- a/python/ray/data/_internal/logical/operators/map_operator.py +++ b/python/ray/data/_internal/logical/operators/map_operator.py @@ -372,7 +372,15 @@ def __init__( output_bytes_column_name: str, ray_remote_args: Optional[Dict[str, Any]] = None, ): - super().__init__("Download", input_op, ray_remote_args=ray_remote_args) + from ray.data._internal.compute import ActorPoolStrategy + + # Download operation uses CallableClass (PartitionActor) so needs ActorPoolStrategy + super().__init__( + "Download", + input_op, + ray_remote_args=ray_remote_args, + compute=ActorPoolStrategy(size=1), + ) self._uri_column_name = uri_column_name self._output_bytes_column_name = output_bytes_column_name diff --git a/release/nightly_tests/dataset/read_from_uris_benchmark.py b/release/nightly_tests/dataset/read_from_uris_benchmark.py index e1da9cb52142..eed591aaf502 100644 --- a/release/nightly_tests/dataset/read_from_uris_benchmark.py +++ b/release/nightly_tests/dataset/read_from_uris_benchmark.py @@ -1,11 +1,12 @@ import io -import boto3 import numpy as np +import pyarrow as pa +import pyarrow.compute as pc from PIL import Image import ray -from ray.data import ActorPoolStrategy +from ray.data.expressions import download from benchmark import Benchmark BUCKET = "anyscale-imagenet" @@ -21,22 +22,27 @@ def main(): def benchmark_fn(): metadata = ray.data.read_parquet(METADATA_PATH) - # Assuming there are 80 CPUs and 4 in-flight tasks per actor, we need at least 320 - # partitions to utilize all CPUs. - # TODO: This is a temporary workaround. We need to improve the default partitioning. - metadata = metadata.repartition(320) - - class LoadImage: - def __init__(self): - self._client = boto3.client("s3") - - def __call__(self, row): - data = io.BytesIO() - self._client.download_fileobj(BUCKET, row["key"], data) - image = Image.open(data).convert("RGB") - return {"image": np.array(image)} - - ds = metadata.map(LoadImage, compute=ActorPoolStrategy(min_size=1)) + + def decode_images(batch): + images = [] + for b in batch["image_bytes"]: + image = Image.open(io.BytesIO(b)).convert("RGB") + images.append(np.array(image)) + del batch["image_bytes"] + batch["image"] = np.array(images, dtype=object) + return batch + + def convert_key(table): + col = table["key"] + t = col.type + new_col = pc.binary_join_element_wise( + pa.scalar("s3://" + BUCKET, type=t), col, pa.scalar("/", type=t) + ) + return table.set_column(table.schema.get_field_index("key"), "key", new_col) + + ds = metadata.map_batches(convert_key, batch_format="pyarrow") + ds = ds.with_column("image_bytes", download("key")) + ds = ds.map_batches(decode_images) for _ in ds.iter_internal_ref_bundles(): pass From 1c637acf371df33a8c39131fe8510029387c8b58 Mon Sep 17 00:00:00 2001 From: harshit-anyscale Date: Fri, 5 Sep 2025 22:32:03 +0530 Subject: [PATCH 469/634] add tests and DLQ business logic (#55608) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary This pull request introduces Dead-Letter Queue (DLQ) functionality for async inference. Users can configure two DLQs: 1. `failed_task_queue` – for tasks that fail during normal execution. 2. `unprocessable_task_queue` – for tasks that cannot be processed (e.g., deserialization failures or missing handlers). All unprocessable tasks will automatically be routed to the unprocessable_task_queue, while other failures will go to the failed_task_queue. The detailed behavior is defined in the [RFC document](https://docs.google.com/document/d/1Ix7uKrP3Q5LCjJ5wZG47ncUi5ScbYzyrtFXsYSlGnwg/edit?tab=t.0). ### Changes in this PR 1. Integrated Celery signals (task_failure, task_unknown) to handle task failures. 2. Added helper functions for moving tasks into the correct DLQ. 3. Introduced tests to verify DLQ routing logic across different failure scenarios. 4. Added a persistence test to ensure tasks are retried at-least-once as per the [RFC’s NFR requirements](https://docs.google.com/document/d/1Ix7uKrP3Q5LCjJ5wZG47ncUi5ScbYzyrtFXsYSlGnwg/edit?tab=t.0#heading=h.4om3bw49w03x). ### Follow-up work (to be added in a separate PR) Additional tests will be added in the next PR to keep this one focused and manageable. These will cover: 1. Task processor metrics 2. Task processor health checks 3. Task cancellation (cancel_task) 4. Multiple task consumers in a single Serve application 5. Ensuring failed tasks are retried exactly max_retry + 1 times --------- Signed-off-by: harshit --- python/ray/serve/task_consumer.py | 4 +- python/ray/serve/task_processor.py | 203 ++++++++++-- python/ray/serve/tests/BUILD | 2 +- python/ray/serve/tests/test_task_processor.py | 288 +++++++++++++++++- .../serve/tests/unit/test_task_consumer.py | 2 +- 5 files changed, 471 insertions(+), 28 deletions(-) diff --git a/python/ray/serve/task_consumer.py b/python/ray/serve/task_consumer.py index 8210bc69ab31..da1266969980 100644 --- a/python/ray/serve/task_consumer.py +++ b/python/ray/serve/task_consumer.py @@ -71,7 +71,7 @@ def instantiate_adapter_from_config( ) try: - adapter_instance.initialize(config=task_processor_config) + adapter_instance.initialize() except Exception as e: raise RuntimeError(f"Failed to initialize {adapter_class.__name__}: {e}") @@ -158,7 +158,7 @@ def task_handler( Arguments: _func: The function to decorate. - name: The name of the task. + name: The name of the task. Default is the method name. Returns: A wrapper function that is marked as a task handler. diff --git a/python/ray/serve/task_processor.py b/python/ray/serve/task_processor.py index ed7435b75981..4d1d21ba6db6 100644 --- a/python/ray/serve/task_processor.py +++ b/python/ray/serve/task_processor.py @@ -1,3 +1,4 @@ +import json import logging import threading import time @@ -6,7 +7,9 @@ from typing import Any, Callable, Dict, List, Optional, Set from celery import Celery +from celery.signals import task_failure, task_unknown +from ray.serve import get_replica_context from ray.serve._private.constants import SERVE_LOGGER_NAME from ray.serve.schema import ( CeleryAdapterConfig, @@ -35,6 +38,18 @@ class AsyncCapability(Enum): HEALTH_CHECK = auto() # Ability to perform health checks asynchronously +def _json_dump(obj: Any) -> Any: + """Recursively make an object JSON serializable.""" + if isinstance(obj, dict): + return {k: _json_dump(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_json_dump(i) for i in obj] + try: + return json.dumps(obj) + except (TypeError, ValueError): + return str(obj) + + @PublicAPI(stability="alpha") class TaskProcessorAdapter(ABC): """ @@ -88,12 +103,9 @@ def supports_any_async(self) -> bool: return len(self._async_capabilities) > 0 @abstractmethod - def initialize(self, config: TaskProcessorConfig): + def initialize(self): """ - Initialize the task processor with the given configuration. - - Args: - config: TaskProcessorConfig containing adapter-specific configuration, queue names, retry settings, and other options. + Initialize the task processor. """ pass @@ -322,6 +334,7 @@ class CeleryTaskProcessorAdapter(TaskProcessorAdapter): _app: Celery _config: TaskProcessorConfig _worker_thread: Optional[threading.Thread] = None + _worker_hostname: Optional[str] = None def __init__(self, config: TaskProcessorConfig): super().__init__() @@ -336,33 +349,69 @@ def __init__(self, config: TaskProcessorConfig): # Celery adapter does not support any async capabilities # self._async_capabilities is already an empty set from parent class - def initialize(self, config: TaskProcessorConfig): + def initialize(self): self._app = Celery( - config.queue_name, - backend=config.adapter_config.backend_url, - broker=config.adapter_config.broker_url, + self._config.queue_name, + backend=self._config.adapter_config.backend_url, + broker=self._config.adapter_config.broker_url, ) self._app.conf.update( loglevel="info", worker_pool="threads", - worker_concurrency=config.adapter_config.worker_concurrency, - max_retries=config.max_retries, - task_default_queue=config.queue_name, + worker_concurrency=self._config.adapter_config.worker_concurrency, + max_retries=self._config.max_retries, + task_default_queue=self._config.queue_name, # Store task results so they can be retrieved after completion task_ignore_result=False, # Acknowledge tasks only after completion (not when received) for better reliability task_acks_late=True, # Reject and requeue tasks when worker is lost to prevent data loss - reject_on_worker_lost=True, + task_reject_on_worker_lost=True, + # Only prefetch 1 task at a time to match concurrency and prevent task hoarding + worker_prefetch_multiplier=1, + ) + + queue_config = { + self._config.queue_name: { + "exchange": self._config.queue_name, + "exchange_type": "direct", + "routing_key": self._config.queue_name, + }, + } + + if self._config.failed_task_queue_name: + queue_config[self._config.failed_task_queue_name] = { + "exchange": self._config.failed_task_queue_name, + "exchange_type": "direct", + "routing_key": self._config.failed_task_queue_name, + } + + if self._config.unprocessable_task_queue_name: + queue_config[self._config.unprocessable_task_queue_name] = { + "exchange": self._config.unprocessable_task_queue_name, + "exchange_type": "direct", + "routing_key": self._config.unprocessable_task_queue_name, + } + + self._app.conf.update( + task_queues=queue_config, + task_routes={ + # Default tasks go to main queue + "*": {"queue": self._config.queue_name}, + }, ) - if config.adapter_config.broker_transport_options is not None: + if self._config.adapter_config.broker_transport_options is not None: self._app.conf.update( - broker_transport_options=config.adapter_config.broker_transport_options, + broker_transport_options=self._config.adapter_config.broker_transport_options, ) - ### TODO(harshit|SERVE-987): add the failed_task_queue_name and unprocessable_task_queue_name business logic here + if self._config.failed_task_queue_name: + task_failure.connect(self._handle_task_failure) + + if self._config.unprocessable_task_queue_name: + task_unknown.connect(self._handle_unknown_task) def register_task_handle(self, func, name=None): task_options = { @@ -410,13 +459,25 @@ def start_consumer(self, **kwargs): logger.info("Celery worker thread is already running.") return + unique_id = get_replica_context().replica_tag + self._worker_hostname = f"{self._app.main}_{unique_id}" + + worker_args = [ + "worker", + f"--hostname={self._worker_hostname}", + "-Q", + self._config.queue_name, + ] + self._worker_thread = threading.Thread( target=self._app.worker_main, - args=(("worker", f"--hostname={self._app.main}"),), + args=(worker_args,), ) self._worker_thread.start() - logger.info(f"Celery worker thread started with hostname: {self._app.main}") + logger.info( + f"Celery worker thread started with hostname: {self._worker_hostname}" + ) def stop_consumer(self, timeout: float = 10.0): """Signals the Celery worker to shut down and waits for it to terminate.""" @@ -428,7 +489,7 @@ def stop_consumer(self, timeout: float = 10.0): # Use the worker's hostname for targeted shutdown self._app.control.broadcast( - "shutdown", destination=[f"celery@{self._app.main}"] + "shutdown", destination=[f"celery@{self._worker_hostname}"] ) self._worker_thread.join(timeout=timeout) @@ -440,7 +501,9 @@ def stop_consumer(self, timeout: float = 10.0): self._worker_thread = None def shutdown(self): + logger.info("Shutting down Celery worker...") self._app.control.shutdown() + logger.info("Celery worker shutdown complete...") def cancel_task_sync(self, task_id) -> bool: return self._app.AsyncResult(task_id).cancel() @@ -460,3 +523,105 @@ def health_check_sync(self) -> List[Dict]: More details can be found here: https://docs.celeryq.dev/en/stable/reference/celery.app.control.html#celery.app.control.Control.ping """ return self._app.control.ping() + + def _handle_task_failure( + self, + sender: Any = None, + task_id: str = None, + args: Any = None, + kwargs: Any = None, + einfo: Any = None, + **kw, + ): + """Handle task failures and route them to appropriate dead letter queues. + + This method is called when a task fails after all retry attempts have been + exhausted. It logs the failure and moves the task to failed_task_queue + + Args: + sender: The task object that failed + task_id: Unique identifier of the failed task + args: Positional arguments passed to the task + kwargs: Keyword arguments passed to the task + einfo: Exception info object containing exception details and traceback + **kw: Additional keyword arguments passed by Celery + """ + logger.info( + f"Task failure detected for task_id: {task_id}, args: {args}, kwargs: {kwargs}, einfo: {einfo}" + ) + + dlq_args = [ + task_id, + str(einfo.exception), + _json_dump(args), + _json_dump(kwargs), + str(einfo), + ] + + if self._config.failed_task_queue_name: + self._move_task_to_queue( + self._config.failed_task_queue_name, + sender.name, + dlq_args, + ) + + logger.error( + f"Task {task_id} failed after max retries. Exception: {einfo}. Moved it to the {self._config.failed_task_queue_name} queue." + ) + + def _handle_unknown_task( + self, + sender: Any = None, + name: str = None, + id: str = None, + message: Any = None, + exc: Any = None, + **kwargs, + ): + """Handle unknown or unregistered tasks received by Celery. + + This method is called when Celery receives a task that it doesn't recognize + (i.e., a task that hasn't been registered with the Celery app). These tasks + are moved to the unprocessable task queue if configured. + + Args: + sender: The Celery app or worker that detected the unknown task + name: Name of the unknown task + id: Task ID of the unknown task + message: The raw message received for the unknown task + exc: The exception raised when trying to process the unknown task + **kwargs: Additional context information from Celery + """ + logger.info( + f"Unknown task detected by Celery. Name: {name}, ID: {id}, Message: {message}" + ) + + if self._config.unprocessable_task_queue_name: + self._move_task_to_queue( + self._config.unprocessable_task_queue_name, + name, + [ + name, + id, + _json_dump(message), + str(exc), + _json_dump(kwargs), + ], + ) + + def _move_task_to_queue(self, queue_name: str, task_name: str, args: list): + """Helper function to move a task to a specified queue.""" + try: + logger.info( + f"Moving task: {task_name} to queue: {queue_name}, args: {args}" + ) + self._app.send_task( + name=task_name, + queue=queue_name, + args=args, + ) + except Exception as e: + logger.error( + f"Failed to move task: {task_name} to queue: {queue_name}, error: {e}" + ) + raise e diff --git a/python/ray/serve/tests/BUILD b/python/ray/serve/tests/BUILD index 174d782d925d..653b75363a2c 100644 --- a/python/ray/serve/tests/BUILD +++ b/python/ray/serve/tests/BUILD @@ -47,7 +47,6 @@ py_test_module_list( "test_persistence.py", "test_proxy_actor_wrapper.py", "test_replica_request_context.py", - "test_task_processor.py", "test_util.py", "test_websockets.py", ], @@ -94,6 +93,7 @@ py_test_module_list( "test_request_timeout.py", "test_streaming_response.py", "test_target_capacity.py", + "test_task_processor.py", "test_telemetry.py", ], tags = [ diff --git a/python/ray/serve/tests/test_task_processor.py b/python/ray/serve/tests/test_task_processor.py index d31192d2d0b5..1dc662325d69 100644 --- a/python/ray/serve/tests/test_task_processor.py +++ b/python/ray/serve/tests/test_task_processor.py @@ -1,12 +1,14 @@ +import json import sys import tempfile +from collections import defaultdict from pathlib import Path import pytest import ray from ray import serve -from ray._common.test_utils import wait_for_condition +from ray._common.test_utils import SignalActor, wait_for_condition from ray.serve.schema import CeleryAdapterConfig, TaskProcessorConfig from ray.serve.task_consumer import ( instantiate_adapter_from_config, @@ -16,11 +18,28 @@ @ray.remote -def send_request_to_queue(processor_config: TaskProcessorConfig, data): - adapter_instance = instantiate_adapter_from_config( +class ProcessedTasksTracker: + def __init__(self): + self.processed_tasks = set() + + def add_task(self, task_data): + self.processed_tasks.add(task_data) + + def get_processed_tasks(self): + return self.processed_tasks + + def get_count(self): + return len(self.processed_tasks) + + +@ray.remote +def send_request_to_queue( + processor_config: TaskProcessorConfig, data, task_name="process_request" +): + adapter_instance_global = instantiate_adapter_from_config( task_processor_config=processor_config ) - result = adapter_instance.enqueue_task_sync("process_request", args=[data]) + result = adapter_instance_global.enqueue_task_sync(task_name, args=[data]) assert result.id is not None return result.id @@ -71,7 +90,9 @@ def transport_options(temp_queue_directory): def create_processor_config(temp_queue_directory, transport_options): """Create a TaskProcessorConfig with common defaults.""" - def _create(**kwargs): + def _create( + failed_task_queue_name=None, unprocessable_task_queue_name=None, **kwargs + ): results_path = temp_queue_directory["results_path"] config_params = { @@ -80,8 +101,18 @@ def _create(**kwargs): broker_url="filesystem://", backend_url=f"file://{results_path}", broker_transport_options=transport_options, + worker_concurrency=1, ), } + + # Add dead letter queue names if provided + if failed_task_queue_name is not None: + config_params["failed_task_queue_name"] = failed_task_queue_name + if unprocessable_task_queue_name is not None: + config_params[ + "unprocessable_task_queue_name" + ] = unprocessable_task_queue_name + config_params.update(kwargs) return TaskProcessorConfig(**config_params) @@ -89,6 +120,33 @@ def _create(**kwargs): return _create +def _get_task_counts_by_routing_key(queue_path): + """Counts tasks in a queue directory by reading the routing key from each message.""" + counts = defaultdict(int) + if not queue_path.exists(): + return counts + + # Celery doesn't provide a way to get the queue size. + # so we've to levarage the broker's API to get the queue size. + # Since we are using the filesystem broker in tests, we can read the files in the queue directory to get the queue size. + for msg_file in queue_path.iterdir(): + if msg_file.is_file(): + try: + with open(msg_file, "r") as f: + data = json.load(f) + routing_key = ( + data.get("properties", {}) + .get("delivery_info", {}) + .get("routing_key") + ) + if routing_key: + counts[routing_key] += 1 + except (json.JSONDecodeError, IOError): + # Ignore files that aren't valid JSON or are otherwise unreadable + continue + return counts + + @pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.") class TestTaskConsumerWithRayServe: """Test task consumer integration with Ray Serve.""" @@ -177,6 +235,82 @@ def assert_result(): wait_for_condition(assert_result, timeout=10) + def test_task_consumer_persistence_across_restarts( + self, temp_queue_directory, serve_instance, create_processor_config + ): + """Test that tasks persist in queue and get executed after deployment restart.""" + # Setup + config = create_processor_config() + tracker = ProcessedTasksTracker.remote() + signal1 = SignalActor.remote() + + @serve.deployment(num_replicas=1, graceful_shutdown_timeout_s=60) + @task_consumer(task_processor_config=config) + class TaskConsumer: + def __init__(self, tracker_ref, signal_ref): + self.tracker, self.signal = tracker_ref, signal_ref + self.local_processed = [] + + @task_handler(name="process_request") + def process_request(self, data): + ray.get(self.signal.wait.remote()) # Block until signal + self.local_processed.append(data) + ray.get(self.tracker.add_task.remote(data)) + return f"Processed: {data}" + + def get_local_processed(self): + return self.local_processed + + # Deploy first version and send tasks + serve.run(TaskConsumer.bind(tracker, signal1), name="app_v1") + + num_tasks = 20 + for i in range(num_tasks): + ray.get(send_request_to_queue.remote(config, f"task_{i}")) + + # Process exactly 1 task, then restart deployment + wait_for_condition( + lambda: ray.get(signal1.cur_num_waiters.remote()) == 1, timeout=10 + ) + ray.get(signal1.send.remote(clear=True)) # Allow 1 task to complete + wait_for_condition(lambda: ray.get(tracker.get_count.remote()) == 1, timeout=10) + + # Shutdown first deployment + serve.delete("app_v1", _blocking=False) + ray.get(signal1.send.remote()) # Release any stuck tasks + wait_for_condition( + lambda: "app_v1" not in serve.status().applications, timeout=100 + ) + + tasks_before_restart = ray.get(tracker.get_count.remote()) + assert ( + tasks_before_restart >= 2 and tasks_before_restart < num_tasks + ), f"Expected at least 2 tasks processed and atleast one less than num_tasks, got {tasks_before_restart}" + + # Deploy second version and process remaining tasks + signal2 = SignalActor.remote() + handle = serve.run(TaskConsumer.bind(tracker, signal2), name="app_v2") + + wait_for_condition( + lambda: ray.get(signal2.cur_num_waiters.remote()) == 1, timeout=10 + ) + ray.get(signal2.send.remote()) # Process all remaining tasks + wait_for_condition( + lambda: ray.get(tracker.get_count.remote()) == num_tasks, timeout=100 + ) + + # Verify all tasks were processed and distributed correctly + expected_tasks = {f"task_{i}" for i in range(num_tasks)} + final_tasks = ray.get(tracker.get_processed_tasks.remote()) + second_deployment_tasks = handle.get_local_processed.remote().result() + + assert ( + final_tasks == expected_tasks + ), f"Missing tasks: {expected_tasks - final_tasks}" + assert ( + len(second_deployment_tasks) == num_tasks - tasks_before_restart + ), f"Second deployment processed {len(second_deployment_tasks)} tasks, expected {num_tasks - tasks_before_restart}" + def test_task_consumer_as_serve_deployment_with_async_task_handler( self, temp_queue_directory, serve_instance, create_processor_config ): @@ -203,5 +337,149 @@ async def process_request(self, data): self.data_received = data +@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.") +class TestTaskConsumerWithDLQsConfiguration: + """Test task consumer with dead letter queues.""" + + def _assert_queue_counts( + self, + temp_queue_directory, + processor_config, + expected_main=0, + expected_unprocessable=0, + expected_failed=0, + timeout=15, + ): + """Helper to assert expected task counts in different queues.""" + + def check_counts(): + queue_path = Path(temp_queue_directory["queue_path"]) + counts = _get_task_counts_by_routing_key(queue_path) + + main_count = counts.get(processor_config.queue_name, 0) + unprocessable_count = counts.get( + getattr(processor_config, "unprocessable_task_queue_name", ""), 0 + ) + failed_count = counts.get( + getattr(processor_config, "failed_task_queue_name", ""), 0 + ) + + return ( + main_count == expected_main + and unprocessable_count == expected_unprocessable + and failed_count == expected_failed + ) + + wait_for_condition(check_counts, timeout=timeout) + + def test_task_consumer_as_serve_deployment_with_unknown_task( + self, temp_queue_directory, serve_instance, create_processor_config + ): + """Test that unknown tasks are sent to the unprocessable task queue.""" + processor_config = create_processor_config( + unprocessable_task_queue_name="unprocessable_task_queue" + ) + + @serve.deployment + @task_consumer(task_processor_config=processor_config) + class ServeTaskConsumer: + @task_handler(name="process_request") + def process_request(self, data): + pass + + serve.run(ServeTaskConsumer.bind()) + + # Send a task with an unknown name + send_request_to_queue.remote( + processor_config, "test_data_1", task_name="unregistered_task" + ) + + self._assert_queue_counts( + temp_queue_directory, + processor_config, + expected_main=0, + expected_unprocessable=1, + timeout=10, + ) + + def test_task_consumer_as_serve_deployment_with_failed_task_and_dead_letter_queue( + self, temp_queue_directory, serve_instance, create_processor_config + ): + """Test that failed tasks are sent to the failed task queue.""" + processor_config = create_processor_config( + failed_task_queue_name="failed_task_queue" + ) + + @serve.deployment + @task_consumer(task_processor_config=processor_config) + class ServeTaskConsumer: + @task_handler(name="process_request") + def process_request(self, data): + raise ValueError("Task failed as expected") + + serve.run(ServeTaskConsumer.bind()) + send_request_to_queue.remote(processor_config, "test_data_1") + + self._assert_queue_counts( + temp_queue_directory, processor_config, expected_main=0, expected_failed=1 + ) + + def test_task_consumer_with_mismatched_arguments( + self, temp_queue_directory, serve_instance, create_processor_config + ): + """Test that tasks with mismatched arguments are sent to the unprocessable task queue.""" + processor_config = create_processor_config( + unprocessable_task_queue_name="unprocessable_task_queue", + failed_task_queue_name="failed_task_queue", + ) + + @serve.deployment + @task_consumer(task_processor_config=processor_config) + class ServeTaskConsumer: + @task_handler(name="process_request") + def process_request(self, arg1, arg2): # Expects two arguments + pass + + serve.run(ServeTaskConsumer.bind()) + + # Send a task with only one argument, which should cause a TypeError + send_request_to_queue.remote(processor_config, ["test_data_1"]) + + self._assert_queue_counts( + temp_queue_directory, + processor_config, + expected_main=0, + expected_failed=1, + ) + + def test_task_consumer_with_argument_type_mismatch( + self, temp_queue_directory, serve_instance, create_processor_config + ): + """Test that tasks with argument type mismatches are sent to the unprocessable task queue.""" + processor_config = create_processor_config( + unprocessable_task_queue_name="unprocessable_task_queue", + failed_task_queue_name="failed_task_queue", + ) + + @serve.deployment + @task_consumer(task_processor_config=processor_config) + class ServeTaskConsumer: + @task_handler(name="process_request") + def process_request(self, data: str): + return len(data) # This will fail if data is not a sequence + + serve.run(ServeTaskConsumer.bind()) + + # Send an integer, for which len() is undefined, causing a TypeError + send_request_to_queue.remote(processor_config, 12345) + + self._assert_queue_counts( + temp_queue_directory, + processor_config, + expected_main=0, + expected_failed=1, + ) + + if __name__ == "__main__": sys.exit(pytest.main(["-v", "-s", __file__])) diff --git a/python/ray/serve/tests/unit/test_task_consumer.py b/python/ray/serve/tests/unit/test_task_consumer.py index 4fe3c547ed01..f7e6435a0254 100644 --- a/python/ray/serve/tests/unit/test_task_consumer.py +++ b/python/ray/serve/tests/unit/test_task_consumer.py @@ -21,7 +21,7 @@ def __init__(self, config: TaskProcessorConfig): self._config = config self.register_task_handle_mock = MagicMock() - def initialize(self, config: TaskProcessorConfig): + def initialize(self): pass def register_task_handle(self, func, name=None): From e363caac8060f200d0f1cdfda3542a32b2e19192 Mon Sep 17 00:00:00 2001 From: Nikhil G Date: Fri, 5 Sep 2025 10:12:35 -0700 Subject: [PATCH 470/634] [llm] Vllm bump -> 0.10.1.1 (#56099) image --------- Signed-off-by: Linkun Signed-off-by: Nikhil Ghosh Signed-off-by: dayshah Signed-off-by: Edward Oakes Signed-off-by: omkar Signed-off-by: Omkar Kulkarni Signed-off-by: Cuong Nguyen Signed-off-by: zac Signed-off-by: Kourosh Hakhamaneshi Co-authored-by: Linkun Co-authored-by: Dhyey Shah Co-authored-by: ahao-anyscale Co-authored-by: Edward Oakes Co-authored-by: Omkar Kulkarni Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cindy Zhang Co-authored-by: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Co-authored-by: Zac Policzer Co-authored-by: Kourosh Hakhamaneshi Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> --- python/deplocks/llm/ray_py311_cpu.lock | 214 ++++++----- python/deplocks/llm/ray_py311_cu121.lock | 214 ++++++----- python/deplocks/llm/ray_py311_cu128.lock | 214 ++++++----- python/deplocks/llm/ray_test_py311_cpu.lock | 214 ++++++----- python/deplocks/llm/ray_test_py311_cu121.lock | 214 ++++++----- python/deplocks/llm/ray_test_py311_cu128.lock | 214 ++++++----- python/deplocks/llm/rayllm_py311_cpu.lock | 356 ++++++++++++------ python/deplocks/llm/rayllm_py311_cu121.lock | 356 ++++++++++++------ python/deplocks/llm/rayllm_py311_cu128.lock | 356 ++++++++++++------ .../deplocks/llm/rayllm_test_py311_cpu.lock | 352 +++++++++++------ .../deplocks/llm/rayllm_test_py311_cu121.lock | 352 +++++++++++------ .../deplocks/llm/rayllm_test_py311_cu128.lock | 352 +++++++++++------ .../batch/stages/vllm_engine_stage.py | 8 +- .../serve/configs/openai_api_models.py | 5 + .../serve/deployments/llm/vllm/vllm_engine.py | 13 +- .../serve/deployments/llm/vllm/vllm_models.py | 29 +- .../serve/deployments/routers/middleware.py | 10 +- .../serve/deployments/routers/router.py | 12 +- .../serve/deployments/utils/server_utils.py | 4 +- .../gpu/stages/test_vllm_engine_stage.py | 2 +- python/requirements/llm/llm-requirements.txt | 3 +- python/requirements_compiled.txt | 7 +- python/setup.py | 2 +- release/release_tests.yaml | 2 +- 24 files changed, 2142 insertions(+), 1363 deletions(-) diff --git a/python/deplocks/llm/ray_py311_cpu.lock b/python/deplocks/llm/ray_py311_cpu.lock index cd1fdcbbc79f..8cd0117266c0 100644 --- a/python/deplocks/llm/ray_py311_cpu.lock +++ b/python/deplocks/llm/ray_py311_cpu.lock @@ -1544,114 +1544,113 @@ pycparser==2.21 ; platform_python_implementation != 'PyPy' \ # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # cffi -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt # fastapi -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # pydantic @@ -1991,6 +1990,13 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer + # typing-inspection +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/deplocks/llm/ray_test_py311_cpu.lock + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 diff --git a/python/deplocks/llm/ray_py311_cu121.lock b/python/deplocks/llm/ray_py311_cu121.lock index 3417dd6ec83c..cdddd9e0dff0 100644 --- a/python/deplocks/llm/ray_py311_cu121.lock +++ b/python/deplocks/llm/ray_py311_cu121.lock @@ -1544,114 +1544,113 @@ pycparser==2.21 ; platform_python_implementation != 'PyPy' \ # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # cffi -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt # fastapi -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # pydantic @@ -1991,6 +1990,13 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer + # typing-inspection +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/deplocks/llm/ray_test_py311_cu121.lock + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 diff --git a/python/deplocks/llm/ray_py311_cu128.lock b/python/deplocks/llm/ray_py311_cu128.lock index 411cb5498708..5d6886ea1bf7 100644 --- a/python/deplocks/llm/ray_py311_cu128.lock +++ b/python/deplocks/llm/ray_py311_cu128.lock @@ -1508,114 +1508,113 @@ pycparser==2.21 ; platform_python_implementation != 'PyPy' \ # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # cffi -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt # fastapi -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # pydantic @@ -1954,6 +1953,13 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer + # typing-inspection +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/deplocks/llm/ray_test_py311_cu128.lock + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 diff --git a/python/deplocks/llm/ray_test_py311_cpu.lock b/python/deplocks/llm/ray_test_py311_cpu.lock index 4c7476294eaf..a8b64f17e94a 100644 --- a/python/deplocks/llm/ray_test_py311_cpu.lock +++ b/python/deplocks/llm/ray_test_py311_cpu.lock @@ -2403,114 +2403,113 @@ pycurl==7.45.3 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt # fastapi -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c /tmp/ray-deps/requirements_compiled.txt # pydantic @@ -3174,6 +3173,13 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer + # typing-inspection +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 diff --git a/python/deplocks/llm/ray_test_py311_cu121.lock b/python/deplocks/llm/ray_test_py311_cu121.lock index 92c04a8fb0ea..cf09d162e9a3 100644 --- a/python/deplocks/llm/ray_test_py311_cu121.lock +++ b/python/deplocks/llm/ray_test_py311_cu121.lock @@ -2403,114 +2403,113 @@ pycurl==7.45.3 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt # fastapi -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c /tmp/ray-deps/requirements_compiled.txt # pydantic @@ -3174,6 +3173,13 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer + # typing-inspection +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 diff --git a/python/deplocks/llm/ray_test_py311_cu128.lock b/python/deplocks/llm/ray_test_py311_cu128.lock index 04402a5fd426..f514c628272b 100644 --- a/python/deplocks/llm/ray_test_py311_cu128.lock +++ b/python/deplocks/llm/ray_test_py311_cu128.lock @@ -2403,114 +2403,113 @@ pycurl==7.45.3 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt # fastapi -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c /tmp/ray-deps/requirements_compiled.txt # pydantic @@ -3174,6 +3173,13 @@ typing-extensions==4.12.2 \ # pyopenssl # referencing # typer + # typing-inspection +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c /tmp/ray-deps/requirements_compiled.txt + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 diff --git a/python/deplocks/llm/rayllm_py311_cpu.lock b/python/deplocks/llm/rayllm_py311_cpu.lock index fafc05f0902e..a11f2db1e997 100644 --- a/python/deplocks/llm/rayllm_py311_cpu.lock +++ b/python/deplocks/llm/rayllm_py311_cpu.lock @@ -995,7 +995,7 @@ hf-transfer==0.1.9 \ # via # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt -hf-xet==1.1.5 \ +hf-xet==1.1.5 ; platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 \ --hash=sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245 \ --hash=sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a \ @@ -1074,7 +1074,6 @@ huggingface-hub==0.34.3 \ # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # tokenizers # transformers - # vllm idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 @@ -1768,9 +1767,27 @@ numpy==1.26.4 \ # transformers # vllm # xformers -openai==1.90.0 \ - --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ - --hash=sha256:e5dcb5498ea6b42fec47546d10f1bcc05fb854219a7d953a5ba766718b212a02 +openai==1.100.2 \ + --hash=sha256:54d3457b2c8d7303a1bc002a058de46bdd8f37a8117751c7cf4ed4438051f151 \ + --hash=sha256:787b4c3c8a65895182c58c424f790c25c790cc9a0330e34f73d55b6ee5a00e32 + # via + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock + # vllm +openai-harmony==0.0.4 \ + --hash=sha256:038f1d6772d1be5213b36ae76e5d042022395ec35c428a73ccb8b839b2cecf6a \ + --hash=sha256:15e6d53a66502491a3675a536df30e271f976e6c5efe68250a65191efcb85c4f \ + --hash=sha256:2d8d16d84702059833fb03b841b28c25600c54e83cadccef79af44e1c81166b1 \ + --hash=sha256:31e9bcac0902a309e2fc688e52f247eec7fffcd00d17e958b9a83a8fea6519c2 \ + --hash=sha256:3586d90c899cd41f8624e7b82a48c289f6e4be56c66304ecaf3a0ba88963a73f \ + --hash=sha256:3cf2344366f10981bbc0f6d9949a0b2bb87151d209ed295943ed6ad8eda37932 \ + --hash=sha256:567cc568b6bf7b4d041b0c9aa7d6b2c9394f8af6065bc87fa6d23f207b5af9a7 \ + --hash=sha256:5c67ac6df349236fb7b64f57c3dbb0273efcdca24314daa108f2a482c427106c \ + --hash=sha256:746f751de5033b3dbcfcd4a726a4c56ce452c593ad3d54472d8597ce8d8b6d44 \ + --hash=sha256:96a63199c0d81095b5d5d1ae8ca82b64c1c13d18d4e30323ae9e8ab31bc80a3d \ + --hash=sha256:97f1fe3909733212cc6b36f0f199b1421a9c57b79ec665f0322bd604cec47340 \ + --hash=sha256:b9ee9e9ab6a237cebbe16563c787a6e83f3fcc034075c3d321dab94448426282 \ + --hash=sha256:d38f2639f6bf7c3c34a5dfd79e29075811ae2fa9b895a63e76767f74a47a971e \ + --hash=sha256:ef21a1e2384a65c62d5ec5e1cded9fe026f1d032d5c5d725110d1a8d330d8f54 # via # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # vllm @@ -2425,9 +2442,9 @@ pycparser==2.21 \ # via # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # cffi -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt @@ -2436,110 +2453,110 @@ pydantic==2.10.0 \ # lm-format-enforcer # mistral-common # openai + # openai-harmony # pydantic-extra-types # vllm # xgrammar -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # pydantic @@ -3122,6 +3139,107 @@ sentencepiece==0.2.0 \ # gguf # mistral-common # vllm +setproctitle==1.3.6 \ + --hash=sha256:082413db8a96b1f021088e8ec23f0a61fec352e649aba20881895815388b66d3 \ + --hash=sha256:0dba8faee2e4a96e934797c9f0f2d093f8239bf210406a99060b3eabe549628e \ + --hash=sha256:0e6b5633c94c5111f7137f875e8f1ff48f53b991d5d5b90932f27dc8c1fa9ae4 \ + --hash=sha256:1065ed36bd03a3fd4186d6c6de5f19846650b015789f72e2dea2d77be99bdca1 \ + --hash=sha256:109fc07b1cd6cef9c245b2028e3e98e038283342b220def311d0239179810dbe \ + --hash=sha256:13624d9925bb481bc0ccfbc7f533da38bfbfe6e80652314f789abc78c2e513bd \ + --hash=sha256:156795b3db976611d09252fc80761fcdb65bb7c9b9581148da900851af25ecf4 \ + --hash=sha256:163dba68f979c61e4e2e779c4d643e968973bdae7c33c3ec4d1869f7a9ba8390 \ + --hash=sha256:17d7c833ed6545ada5ac4bb606b86a28f13a04431953d4beac29d3773aa00b1d \ + --hash=sha256:18d0667bafaaae4c1dee831e2e59841c411ff399b9b4766822ba2685d419c3be \ + --hash=sha256:1aa1935aa2195b76f377e5cb018290376b7bf085f0b53f5a95c0c21011b74367 \ + --hash=sha256:2156d55308431ac3b3ec4e5e05b1726d11a5215352d6a22bb933171dee292f8c \ + --hash=sha256:23a57d3b8f1549515c2dbe4a2880ebc1f27780dc126c5e064167563e015817f5 \ + --hash=sha256:2407955dc359d735a20ac6e797ad160feb33d529a2ac50695c11a1ec680eafab \ + --hash=sha256:2940cf13f4fc11ce69ad2ed37a9f22386bfed314b98d8aebfd4f55459aa59108 \ + --hash=sha256:2e51ec673513465663008ce402171192a053564865c2fc6dc840620871a9bd7c \ + --hash=sha256:3393859eb8f19f5804049a685bf286cb08d447e28ba5c6d8543c7bf5500d5970 \ + --hash=sha256:3884002b3a9086f3018a32ab5d4e1e8214dd70695004e27b1a45c25a6243ad0b \ + --hash=sha256:38ca045626af693da042ac35d7332e7b9dbd52e6351d6973b310612e3acee6d6 \ + --hash=sha256:391bb6a29c4fe7ccc9c30812e3744060802d89b39264cfa77f3d280d7f387ea5 \ + --hash=sha256:3cca16fd055316a48f0debfcbfb6af7cea715429fc31515ab3fcac05abd527d8 \ + --hash=sha256:3cde5b83ec4915cd5e6ae271937fd60d14113c8f7769b4a20d51769fe70d8717 \ + --hash=sha256:3f8194b4d631b003a1176a75d1acd545e04b1f54b821638e098a93e6e62830ef \ + --hash=sha256:3fc97805f9d74444b027babff710bf39df1541437a6a585a983d090ae00cedde \ + --hash=sha256:4431629c178193f23c538cb1de3da285a99ccc86b20ee91d81eb5f1a80e0d2ba \ + --hash=sha256:49498ebf68ca3e75321ffe634fcea5cc720502bfaa79bd6b03ded92ce0dc3c24 \ + --hash=sha256:4ac3eb04bcf0119aadc6235a2c162bae5ed5f740e3d42273a7228b915722de20 \ + --hash=sha256:4adf6a0013fe4e0844e3ba7583ec203ca518b9394c6cc0d3354df2bf31d1c034 \ + --hash=sha256:4efc91b437f6ff2578e89e3f17d010c0a0ff01736606473d082913ecaf7859ba \ + --hash=sha256:50706b9c0eda55f7de18695bfeead5f28b58aa42fd5219b3b1692d554ecbc9ec \ + --hash=sha256:5313a4e9380e46ca0e2c681ba739296f9e7c899e6f4d12a6702b2dc9fb846a31 \ + --hash=sha256:543f59601a4e32daf44741b52f9a23e0ee374f9f13b39c41d917302d98fdd7b0 \ + --hash=sha256:57bc54763bf741813a99fbde91f6be138c8706148b7b42d3752deec46545d470 \ + --hash=sha256:63cc10352dc6cf35a33951656aa660d99f25f574eb78132ce41a85001a638aa7 \ + --hash=sha256:6a1d3aa13acfe81f355b0ce4968facc7a19b0d17223a0f80c011a1dba8388f37 \ + --hash=sha256:6af330ddc2ec05a99c3933ab3cba9365357c0b8470a7f2fa054ee4b0984f57d1 \ + --hash=sha256:6d50bfcc1d1692dc55165b3dd2f0b9f8fb5b1f7b571a93e08d660ad54b9ca1a5 \ + --hash=sha256:70100e2087fe05359f249a0b5f393127b3a1819bf34dec3a3e0d4941138650c9 \ + --hash=sha256:74973aebea3543ad033b9103db30579ec2b950a466e09f9c2180089e8346e0ec \ + --hash=sha256:751ba352ed922e0af60458e961167fa7b732ac31c0ddd1476a2dfd30ab5958c5 \ + --hash=sha256:785cd210c0311d9be28a70e281a914486d62bfd44ac926fcd70cf0b4d65dff1c \ + --hash=sha256:7890e291bf4708e3b61db9069ea39b3ab0651e42923a5e1f4d78a7b9e4b18301 \ + --hash=sha256:793a23e8d9cb6c231aa3023d700008224c6ec5b8fd622d50f3c51665e3d0a190 \ + --hash=sha256:797f2846b546a8741413c57d9fb930ad5aa939d925c9c0fa6186d77580035af7 \ + --hash=sha256:7df5fcc48588f82b6cc8073db069609ddd48a49b1e9734a20d0efb32464753c4 \ + --hash=sha256:8050c01331135f77ec99d99307bfbc6519ea24d2f92964b06f3222a804a3ff1f \ + --hash=sha256:805bb33e92fc3d8aa05674db3068d14d36718e3f2c5c79b09807203f229bf4b5 \ + --hash=sha256:807796fe301b7ed76cf100113cc008c119daf4fea2f9f43c578002aef70c3ebf \ + --hash=sha256:81c443310831e29fabbd07b75ebbfa29d0740b56f5907c6af218482d51260431 \ + --hash=sha256:83066ffbf77a5f82b7e96e59bdccbdda203c8dccbfc3f9f0fdad3a08d0001d9c \ + --hash=sha256:8834ab7be6539f1bfadec7c8d12249bbbe6c2413b1d40ffc0ec408692232a0c6 \ + --hash=sha256:92df0e70b884f5da35f2e01489dca3c06a79962fb75636985f1e3a17aec66833 \ + --hash=sha256:9483aa336687463f5497dd37a070094f3dff55e2c888994f8440fcf426a1a844 \ + --hash=sha256:97a138fa875c6f281df7720dac742259e85518135cd0e3551aba1c628103d853 \ + --hash=sha256:9b50700785eccac0819bea794d968ed8f6055c88f29364776b7ea076ac105c5d \ + --hash=sha256:9b73cf0fe28009a04a35bb2522e4c5b5176cc148919431dcb73fdbdfaab15781 \ + --hash=sha256:9d5a369eb7ec5b2fdfa9927530b5259dd21893fa75d4e04a223332f61b84b586 \ + --hash=sha256:a094b7ce455ca341b59a0f6ce6be2e11411ba6e2860b9aa3dbb37468f23338f4 \ + --hash=sha256:a0d6252098e98129a1decb59b46920d4eca17b0395f3d71b0d327d086fefe77d \ + --hash=sha256:a1d856b0f4e4a33e31cdab5f50d0a14998f3a2d726a3fd5cb7c4d45a57b28d1b \ + --hash=sha256:a4ae2ea9afcfdd2b931ddcebf1cf82532162677e00326637b31ed5dff7d985ca \ + --hash=sha256:a5963b663da69ad25fa1559ee064584935570def665917918938c1f1289f5ebc \ + --hash=sha256:ad1c2c2baaba62823a7f348f469a967ece0062140ca39e7a48e4bbb1f20d54c4 \ + --hash=sha256:ae82507fe458f7c0c8227017f2158111a4c9e7ce94de05178894a7ea9fefc8a1 \ + --hash=sha256:af188f3305f0a65c3217c30c6d4c06891e79144076a91e8b454f14256acc7279 \ + --hash=sha256:af44bb7a1af163806bbb679eb8432fa7b4fb6d83a5d403b541b675dcd3798638 \ + --hash=sha256:b0174ca6f3018ddeaa49847f29b69612e590534c1d2186d54ab25161ecc42975 \ + --hash=sha256:b2b17855ed7f994f3f259cf2dfbfad78814538536fa1a91b50253d84d87fd88d \ + --hash=sha256:b2e54f4a2dc6edf0f5ea5b1d0a608d2af3dcb5aa8c8eeab9c8841b23e1b054fe \ + --hash=sha256:b6f4abde9a2946f57e8daaf1160b2351bcf64274ef539e6675c1d945dbd75e2a \ + --hash=sha256:b70c07409d465f3a8b34d52f863871fb8a00755370791d2bd1d4f82b3cdaf3d5 \ + --hash=sha256:bb465dd5825356c1191a038a86ee1b8166e3562d6e8add95eec04ab484cfb8a2 \ + --hash=sha256:c051f46ed1e13ba8214b334cbf21902102807582fbfaf0fef341b9e52f0fafbf \ + --hash=sha256:c1b20a5f4164cec7007be55c9cf18d2cd08ed7c3bf6769b3cd6d044ad888d74b \ + --hash=sha256:c86e9e82bfab579327dbe9b82c71475165fbc8b2134d24f9a3b2edaf200a5c3d \ + --hash=sha256:c9f32b96c700bb384f33f7cf07954bb609d35dd82752cef57fb2ee0968409169 \ + --hash=sha256:cce0ed8b3f64c71c140f0ec244e5fdf8ecf78ddf8d2e591d4a8b6aa1c1214235 \ + --hash=sha256:cdd7315314b0744a7dd506f3bd0f2cf90734181529cdcf75542ee35ad885cab7 \ + --hash=sha256:cf355fbf0d4275d86f9f57be705d8e5eaa7f8ddb12b24ced2ea6cbd68fdb14dc \ + --hash=sha256:d136fbf8ad4321716e44d6d6b3d8dffb4872626010884e07a1db54b7450836cf \ + --hash=sha256:d2c8e20487b3b73c1fa72c56f5c89430617296cd380373e7af3a538a82d4cd6d \ + --hash=sha256:d483cc23cc56ab32911ea0baa0d2d9ea7aa065987f47de847a0a93a58bf57905 \ + --hash=sha256:d5a6c4864bb6fa9fcf7b57a830d21aed69fd71742a5ebcdbafda476be673d212 \ + --hash=sha256:d714e002dd3638170fe7376dc1b686dbac9cb712cde3f7224440af722cc9866a \ + --hash=sha256:d73f14b86d0e2858ece6bf5807c9889670e392c001d414b4293d0d9b291942c3 \ + --hash=sha256:d88c63bd395c787b0aa81d8bbc22c1809f311032ce3e823a6517b711129818e4 \ + --hash=sha256:db608db98ccc21248370d30044a60843b3f0f3d34781ceeea67067c508cd5a28 \ + --hash=sha256:de004939fc3fd0c1200d26ea9264350bfe501ffbf46c8cf5dc7f345f2d87a7f1 \ + --hash=sha256:ded9e86397267732a0641d4776c7c663ea16b64d7dbc4d9cc6ad8536363a2d29 \ + --hash=sha256:e288f8a162d663916060beb5e8165a8551312b08efee9cf68302687471a6545d \ + --hash=sha256:e2a9e62647dc040a76d55563580bf3bb8fe1f5b6ead08447c2ed0d7786e5e794 \ + --hash=sha256:e3e44d08b61de0dd6f205528498f834a51a5c06689f8fb182fe26f3a3ce7dca9 \ + --hash=sha256:ea002088d5554fd75e619742cefc78b84a212ba21632e59931b3501f0cfc8f67 \ + --hash=sha256:eb7452849f6615871eabed6560ffedfe56bc8af31a823b6be4ce1e6ff0ab72c5 \ + --hash=sha256:ebcf34b69df4ca0eabaaaf4a3d890f637f355fed00ba806f7ebdd2d040658c26 \ + --hash=sha256:f24d5b9383318cbd1a5cd969377937d66cf0542f24aa728a4f49d9f98f9c0da8 \ + --hash=sha256:f33fbf96b52d51c23b6cff61f57816539c1c147db270cfc1cc3bc012f4a560a9 + # via + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock + # vllm shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de @@ -3338,12 +3456,11 @@ tqdm==4.67.1 \ # openai # transformers # vllm -transformers==4.53.2 \ - --hash=sha256:6c3ed95edfb1cba71c4245758f1b4878c93bf8cde77d076307dacb2cbbd72be2 \ - --hash=sha256:db8f4819bb34f000029c73c3c557e7d06fc1b8e612ec142eecdae3947a9c78bf +transformers==4.55.2 \ + --hash=sha256:097e3c2e2c0c9681db3da9d748d8f9d6a724c644514673d0030e8c5a1109f1f1 \ + --hash=sha256:a45ec60c03474fd67adbce5c434685051b7608b3f4f167c25aa6aeb1cad16d4f # via # -c python/deplocks/llm/rayllm_test_py311_cpu.lock - # -r python/requirements/llm/llm-requirements.txt # compressed-tensors # vllm # xgrammar @@ -3391,7 +3508,14 @@ typing-extensions==4.12.2 \ # referencing # torch # typer + # typing-inspection # vllm +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/deplocks/llm/rayllm_test_py311_cpu.lock + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 @@ -3467,9 +3591,9 @@ virtualenv==20.29.1 \ # via # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt -vllm==0.10.0 \ - --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ - --hash=sha256:a44e9013db26082a82c3931ed8772ac884d6d60566d36ecdb0e8dc01c65b241a +vllm==0.10.1.1 \ + --hash=sha256:3099824ee4bdaa14c4c4f7178a092101a0ec206d4c9371edf295849b2b730a39 \ + --hash=sha256:8ca0dd985e1ceac8540e7719c654f1553b3ba8a43c685ac8d3fa1366ffb6443a # via # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements/llm/llm-requirements.txt diff --git a/python/deplocks/llm/rayllm_py311_cu121.lock b/python/deplocks/llm/rayllm_py311_cu121.lock index 4f260cad18b3..76431a88ef1b 100644 --- a/python/deplocks/llm/rayllm_py311_cu121.lock +++ b/python/deplocks/llm/rayllm_py311_cu121.lock @@ -995,7 +995,7 @@ hf-transfer==0.1.9 \ # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt -hf-xet==1.1.5 \ +hf-xet==1.1.5 ; platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 \ --hash=sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245 \ --hash=sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a \ @@ -1074,7 +1074,6 @@ huggingface-hub==0.34.3 \ # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # tokenizers # transformers - # vllm idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 @@ -1884,9 +1883,27 @@ nvidia-nvtx-cu12==12.6.77 ; platform_machine == 'x86_64' and sys_platform == 'li # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # torch -openai==1.90.0 \ - --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ - --hash=sha256:e5dcb5498ea6b42fec47546d10f1bcc05fb854219a7d953a5ba766718b212a02 +openai==1.100.2 \ + --hash=sha256:54d3457b2c8d7303a1bc002a058de46bdd8f37a8117751c7cf4ed4438051f151 \ + --hash=sha256:787b4c3c8a65895182c58c424f790c25c790cc9a0330e34f73d55b6ee5a00e32 + # via + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock + # vllm +openai-harmony==0.0.4 \ + --hash=sha256:038f1d6772d1be5213b36ae76e5d042022395ec35c428a73ccb8b839b2cecf6a \ + --hash=sha256:15e6d53a66502491a3675a536df30e271f976e6c5efe68250a65191efcb85c4f \ + --hash=sha256:2d8d16d84702059833fb03b841b28c25600c54e83cadccef79af44e1c81166b1 \ + --hash=sha256:31e9bcac0902a309e2fc688e52f247eec7fffcd00d17e958b9a83a8fea6519c2 \ + --hash=sha256:3586d90c899cd41f8624e7b82a48c289f6e4be56c66304ecaf3a0ba88963a73f \ + --hash=sha256:3cf2344366f10981bbc0f6d9949a0b2bb87151d209ed295943ed6ad8eda37932 \ + --hash=sha256:567cc568b6bf7b4d041b0c9aa7d6b2c9394f8af6065bc87fa6d23f207b5af9a7 \ + --hash=sha256:5c67ac6df349236fb7b64f57c3dbb0273efcdca24314daa108f2a482c427106c \ + --hash=sha256:746f751de5033b3dbcfcd4a726a4c56ce452c593ad3d54472d8597ce8d8b6d44 \ + --hash=sha256:96a63199c0d81095b5d5d1ae8ca82b64c1c13d18d4e30323ae9e8ab31bc80a3d \ + --hash=sha256:97f1fe3909733212cc6b36f0f199b1421a9c57b79ec665f0322bd604cec47340 \ + --hash=sha256:b9ee9e9ab6a237cebbe16563c787a6e83f3fcc034075c3d321dab94448426282 \ + --hash=sha256:d38f2639f6bf7c3c34a5dfd79e29075811ae2fa9b895a63e76767f74a47a971e \ + --hash=sha256:ef21a1e2384a65c62d5ec5e1cded9fe026f1d032d5c5d725110d1a8d330d8f54 # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # vllm @@ -2541,9 +2558,9 @@ pycparser==2.21 \ # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # cffi -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt @@ -2552,110 +2569,110 @@ pydantic==2.10.0 \ # lm-format-enforcer # mistral-common # openai + # openai-harmony # pydantic-extra-types # vllm # xgrammar -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # pydantic @@ -3238,6 +3255,107 @@ sentencepiece==0.2.0 \ # gguf # mistral-common # vllm +setproctitle==1.3.6 \ + --hash=sha256:082413db8a96b1f021088e8ec23f0a61fec352e649aba20881895815388b66d3 \ + --hash=sha256:0dba8faee2e4a96e934797c9f0f2d093f8239bf210406a99060b3eabe549628e \ + --hash=sha256:0e6b5633c94c5111f7137f875e8f1ff48f53b991d5d5b90932f27dc8c1fa9ae4 \ + --hash=sha256:1065ed36bd03a3fd4186d6c6de5f19846650b015789f72e2dea2d77be99bdca1 \ + --hash=sha256:109fc07b1cd6cef9c245b2028e3e98e038283342b220def311d0239179810dbe \ + --hash=sha256:13624d9925bb481bc0ccfbc7f533da38bfbfe6e80652314f789abc78c2e513bd \ + --hash=sha256:156795b3db976611d09252fc80761fcdb65bb7c9b9581148da900851af25ecf4 \ + --hash=sha256:163dba68f979c61e4e2e779c4d643e968973bdae7c33c3ec4d1869f7a9ba8390 \ + --hash=sha256:17d7c833ed6545ada5ac4bb606b86a28f13a04431953d4beac29d3773aa00b1d \ + --hash=sha256:18d0667bafaaae4c1dee831e2e59841c411ff399b9b4766822ba2685d419c3be \ + --hash=sha256:1aa1935aa2195b76f377e5cb018290376b7bf085f0b53f5a95c0c21011b74367 \ + --hash=sha256:2156d55308431ac3b3ec4e5e05b1726d11a5215352d6a22bb933171dee292f8c \ + --hash=sha256:23a57d3b8f1549515c2dbe4a2880ebc1f27780dc126c5e064167563e015817f5 \ + --hash=sha256:2407955dc359d735a20ac6e797ad160feb33d529a2ac50695c11a1ec680eafab \ + --hash=sha256:2940cf13f4fc11ce69ad2ed37a9f22386bfed314b98d8aebfd4f55459aa59108 \ + --hash=sha256:2e51ec673513465663008ce402171192a053564865c2fc6dc840620871a9bd7c \ + --hash=sha256:3393859eb8f19f5804049a685bf286cb08d447e28ba5c6d8543c7bf5500d5970 \ + --hash=sha256:3884002b3a9086f3018a32ab5d4e1e8214dd70695004e27b1a45c25a6243ad0b \ + --hash=sha256:38ca045626af693da042ac35d7332e7b9dbd52e6351d6973b310612e3acee6d6 \ + --hash=sha256:391bb6a29c4fe7ccc9c30812e3744060802d89b39264cfa77f3d280d7f387ea5 \ + --hash=sha256:3cca16fd055316a48f0debfcbfb6af7cea715429fc31515ab3fcac05abd527d8 \ + --hash=sha256:3cde5b83ec4915cd5e6ae271937fd60d14113c8f7769b4a20d51769fe70d8717 \ + --hash=sha256:3f8194b4d631b003a1176a75d1acd545e04b1f54b821638e098a93e6e62830ef \ + --hash=sha256:3fc97805f9d74444b027babff710bf39df1541437a6a585a983d090ae00cedde \ + --hash=sha256:4431629c178193f23c538cb1de3da285a99ccc86b20ee91d81eb5f1a80e0d2ba \ + --hash=sha256:49498ebf68ca3e75321ffe634fcea5cc720502bfaa79bd6b03ded92ce0dc3c24 \ + --hash=sha256:4ac3eb04bcf0119aadc6235a2c162bae5ed5f740e3d42273a7228b915722de20 \ + --hash=sha256:4adf6a0013fe4e0844e3ba7583ec203ca518b9394c6cc0d3354df2bf31d1c034 \ + --hash=sha256:4efc91b437f6ff2578e89e3f17d010c0a0ff01736606473d082913ecaf7859ba \ + --hash=sha256:50706b9c0eda55f7de18695bfeead5f28b58aa42fd5219b3b1692d554ecbc9ec \ + --hash=sha256:5313a4e9380e46ca0e2c681ba739296f9e7c899e6f4d12a6702b2dc9fb846a31 \ + --hash=sha256:543f59601a4e32daf44741b52f9a23e0ee374f9f13b39c41d917302d98fdd7b0 \ + --hash=sha256:57bc54763bf741813a99fbde91f6be138c8706148b7b42d3752deec46545d470 \ + --hash=sha256:63cc10352dc6cf35a33951656aa660d99f25f574eb78132ce41a85001a638aa7 \ + --hash=sha256:6a1d3aa13acfe81f355b0ce4968facc7a19b0d17223a0f80c011a1dba8388f37 \ + --hash=sha256:6af330ddc2ec05a99c3933ab3cba9365357c0b8470a7f2fa054ee4b0984f57d1 \ + --hash=sha256:6d50bfcc1d1692dc55165b3dd2f0b9f8fb5b1f7b571a93e08d660ad54b9ca1a5 \ + --hash=sha256:70100e2087fe05359f249a0b5f393127b3a1819bf34dec3a3e0d4941138650c9 \ + --hash=sha256:74973aebea3543ad033b9103db30579ec2b950a466e09f9c2180089e8346e0ec \ + --hash=sha256:751ba352ed922e0af60458e961167fa7b732ac31c0ddd1476a2dfd30ab5958c5 \ + --hash=sha256:785cd210c0311d9be28a70e281a914486d62bfd44ac926fcd70cf0b4d65dff1c \ + --hash=sha256:7890e291bf4708e3b61db9069ea39b3ab0651e42923a5e1f4d78a7b9e4b18301 \ + --hash=sha256:793a23e8d9cb6c231aa3023d700008224c6ec5b8fd622d50f3c51665e3d0a190 \ + --hash=sha256:797f2846b546a8741413c57d9fb930ad5aa939d925c9c0fa6186d77580035af7 \ + --hash=sha256:7df5fcc48588f82b6cc8073db069609ddd48a49b1e9734a20d0efb32464753c4 \ + --hash=sha256:8050c01331135f77ec99d99307bfbc6519ea24d2f92964b06f3222a804a3ff1f \ + --hash=sha256:805bb33e92fc3d8aa05674db3068d14d36718e3f2c5c79b09807203f229bf4b5 \ + --hash=sha256:807796fe301b7ed76cf100113cc008c119daf4fea2f9f43c578002aef70c3ebf \ + --hash=sha256:81c443310831e29fabbd07b75ebbfa29d0740b56f5907c6af218482d51260431 \ + --hash=sha256:83066ffbf77a5f82b7e96e59bdccbdda203c8dccbfc3f9f0fdad3a08d0001d9c \ + --hash=sha256:8834ab7be6539f1bfadec7c8d12249bbbe6c2413b1d40ffc0ec408692232a0c6 \ + --hash=sha256:92df0e70b884f5da35f2e01489dca3c06a79962fb75636985f1e3a17aec66833 \ + --hash=sha256:9483aa336687463f5497dd37a070094f3dff55e2c888994f8440fcf426a1a844 \ + --hash=sha256:97a138fa875c6f281df7720dac742259e85518135cd0e3551aba1c628103d853 \ + --hash=sha256:9b50700785eccac0819bea794d968ed8f6055c88f29364776b7ea076ac105c5d \ + --hash=sha256:9b73cf0fe28009a04a35bb2522e4c5b5176cc148919431dcb73fdbdfaab15781 \ + --hash=sha256:9d5a369eb7ec5b2fdfa9927530b5259dd21893fa75d4e04a223332f61b84b586 \ + --hash=sha256:a094b7ce455ca341b59a0f6ce6be2e11411ba6e2860b9aa3dbb37468f23338f4 \ + --hash=sha256:a0d6252098e98129a1decb59b46920d4eca17b0395f3d71b0d327d086fefe77d \ + --hash=sha256:a1d856b0f4e4a33e31cdab5f50d0a14998f3a2d726a3fd5cb7c4d45a57b28d1b \ + --hash=sha256:a4ae2ea9afcfdd2b931ddcebf1cf82532162677e00326637b31ed5dff7d985ca \ + --hash=sha256:a5963b663da69ad25fa1559ee064584935570def665917918938c1f1289f5ebc \ + --hash=sha256:ad1c2c2baaba62823a7f348f469a967ece0062140ca39e7a48e4bbb1f20d54c4 \ + --hash=sha256:ae82507fe458f7c0c8227017f2158111a4c9e7ce94de05178894a7ea9fefc8a1 \ + --hash=sha256:af188f3305f0a65c3217c30c6d4c06891e79144076a91e8b454f14256acc7279 \ + --hash=sha256:af44bb7a1af163806bbb679eb8432fa7b4fb6d83a5d403b541b675dcd3798638 \ + --hash=sha256:b0174ca6f3018ddeaa49847f29b69612e590534c1d2186d54ab25161ecc42975 \ + --hash=sha256:b2b17855ed7f994f3f259cf2dfbfad78814538536fa1a91b50253d84d87fd88d \ + --hash=sha256:b2e54f4a2dc6edf0f5ea5b1d0a608d2af3dcb5aa8c8eeab9c8841b23e1b054fe \ + --hash=sha256:b6f4abde9a2946f57e8daaf1160b2351bcf64274ef539e6675c1d945dbd75e2a \ + --hash=sha256:b70c07409d465f3a8b34d52f863871fb8a00755370791d2bd1d4f82b3cdaf3d5 \ + --hash=sha256:bb465dd5825356c1191a038a86ee1b8166e3562d6e8add95eec04ab484cfb8a2 \ + --hash=sha256:c051f46ed1e13ba8214b334cbf21902102807582fbfaf0fef341b9e52f0fafbf \ + --hash=sha256:c1b20a5f4164cec7007be55c9cf18d2cd08ed7c3bf6769b3cd6d044ad888d74b \ + --hash=sha256:c86e9e82bfab579327dbe9b82c71475165fbc8b2134d24f9a3b2edaf200a5c3d \ + --hash=sha256:c9f32b96c700bb384f33f7cf07954bb609d35dd82752cef57fb2ee0968409169 \ + --hash=sha256:cce0ed8b3f64c71c140f0ec244e5fdf8ecf78ddf8d2e591d4a8b6aa1c1214235 \ + --hash=sha256:cdd7315314b0744a7dd506f3bd0f2cf90734181529cdcf75542ee35ad885cab7 \ + --hash=sha256:cf355fbf0d4275d86f9f57be705d8e5eaa7f8ddb12b24ced2ea6cbd68fdb14dc \ + --hash=sha256:d136fbf8ad4321716e44d6d6b3d8dffb4872626010884e07a1db54b7450836cf \ + --hash=sha256:d2c8e20487b3b73c1fa72c56f5c89430617296cd380373e7af3a538a82d4cd6d \ + --hash=sha256:d483cc23cc56ab32911ea0baa0d2d9ea7aa065987f47de847a0a93a58bf57905 \ + --hash=sha256:d5a6c4864bb6fa9fcf7b57a830d21aed69fd71742a5ebcdbafda476be673d212 \ + --hash=sha256:d714e002dd3638170fe7376dc1b686dbac9cb712cde3f7224440af722cc9866a \ + --hash=sha256:d73f14b86d0e2858ece6bf5807c9889670e392c001d414b4293d0d9b291942c3 \ + --hash=sha256:d88c63bd395c787b0aa81d8bbc22c1809f311032ce3e823a6517b711129818e4 \ + --hash=sha256:db608db98ccc21248370d30044a60843b3f0f3d34781ceeea67067c508cd5a28 \ + --hash=sha256:de004939fc3fd0c1200d26ea9264350bfe501ffbf46c8cf5dc7f345f2d87a7f1 \ + --hash=sha256:ded9e86397267732a0641d4776c7c663ea16b64d7dbc4d9cc6ad8536363a2d29 \ + --hash=sha256:e288f8a162d663916060beb5e8165a8551312b08efee9cf68302687471a6545d \ + --hash=sha256:e2a9e62647dc040a76d55563580bf3bb8fe1f5b6ead08447c2ed0d7786e5e794 \ + --hash=sha256:e3e44d08b61de0dd6f205528498f834a51a5c06689f8fb182fe26f3a3ce7dca9 \ + --hash=sha256:ea002088d5554fd75e619742cefc78b84a212ba21632e59931b3501f0cfc8f67 \ + --hash=sha256:eb7452849f6615871eabed6560ffedfe56bc8af31a823b6be4ce1e6ff0ab72c5 \ + --hash=sha256:ebcf34b69df4ca0eabaaaf4a3d890f637f355fed00ba806f7ebdd2d040658c26 \ + --hash=sha256:f24d5b9383318cbd1a5cd969377937d66cf0542f24aa728a4f49d9f98f9c0da8 \ + --hash=sha256:f33fbf96b52d51c23b6cff61f57816539c1c147db270cfc1cc3bc012f4a560a9 + # via + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock + # vllm shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de @@ -3483,12 +3601,11 @@ tqdm==4.67.1 \ # openai # transformers # vllm -transformers==4.53.2 \ - --hash=sha256:6c3ed95edfb1cba71c4245758f1b4878c93bf8cde77d076307dacb2cbbd72be2 \ - --hash=sha256:db8f4819bb34f000029c73c3c557e7d06fc1b8e612ec142eecdae3947a9c78bf +transformers==4.55.2 \ + --hash=sha256:097e3c2e2c0c9681db3da9d748d8f9d6a724c644514673d0030e8c5a1109f1f1 \ + --hash=sha256:a45ec60c03474fd67adbce5c434685051b7608b3f4f167c25aa6aeb1cad16d4f # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock - # -r python/requirements/llm/llm-requirements.txt # compressed-tensors # vllm # xgrammar @@ -3526,7 +3643,14 @@ typing-extensions==4.12.2 \ # referencing # torch # typer + # typing-inspection # vllm +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/deplocks/llm/rayllm_test_py311_cu121.lock + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 @@ -3602,9 +3726,9 @@ virtualenv==20.29.1 \ # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt -vllm==0.10.0 \ - --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ - --hash=sha256:a44e9013db26082a82c3931ed8772ac884d6d60566d36ecdb0e8dc01c65b241a +vllm==0.10.1.1 \ + --hash=sha256:3099824ee4bdaa14c4c4f7178a092101a0ec206d4c9371edf295849b2b730a39 \ + --hash=sha256:8ca0dd985e1ceac8540e7719c654f1553b3ba8a43c685ac8d3fa1366ffb6443a # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements/llm/llm-requirements.txt diff --git a/python/deplocks/llm/rayllm_py311_cu128.lock b/python/deplocks/llm/rayllm_py311_cu128.lock index 48efbd5be7dd..c0e933c02c02 100644 --- a/python/deplocks/llm/rayllm_py311_cu128.lock +++ b/python/deplocks/llm/rayllm_py311_cu128.lock @@ -995,7 +995,7 @@ hf-transfer==0.1.9 \ # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt -hf-xet==1.1.3 \ +hf-xet==1.1.3 ; platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:30c575a5306f8e6fda37edb866762140a435037365eba7a17ce7bd0bc0216a8b \ --hash=sha256:7c1a6aa6abed1f696f8099aa9796ca04c9ee778a58728a115607de9cc4638ff1 \ --hash=sha256:8203f52827e3df65981984936654a5b390566336956f65765a8aa58c362bb841 \ @@ -1074,7 +1074,6 @@ huggingface-hub==0.34.3 \ # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # tokenizers # transformers - # vllm idna==3.7 \ --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 @@ -1808,9 +1807,27 @@ nvidia-nvtx-cu12==12.8.55 ; platform_machine == 'x86_64' and sys_platform == 'li # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # torch -openai==1.90.0 \ - --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ - --hash=sha256:e5dcb5498ea6b42fec47546d10f1bcc05fb854219a7d953a5ba766718b212a02 +openai==1.100.2 \ + --hash=sha256:54d3457b2c8d7303a1bc002a058de46bdd8f37a8117751c7cf4ed4438051f151 \ + --hash=sha256:787b4c3c8a65895182c58c424f790c25c790cc9a0330e34f73d55b6ee5a00e32 + # via + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock + # vllm +openai-harmony==0.0.4 \ + --hash=sha256:038f1d6772d1be5213b36ae76e5d042022395ec35c428a73ccb8b839b2cecf6a \ + --hash=sha256:15e6d53a66502491a3675a536df30e271f976e6c5efe68250a65191efcb85c4f \ + --hash=sha256:2d8d16d84702059833fb03b841b28c25600c54e83cadccef79af44e1c81166b1 \ + --hash=sha256:31e9bcac0902a309e2fc688e52f247eec7fffcd00d17e958b9a83a8fea6519c2 \ + --hash=sha256:3586d90c899cd41f8624e7b82a48c289f6e4be56c66304ecaf3a0ba88963a73f \ + --hash=sha256:3cf2344366f10981bbc0f6d9949a0b2bb87151d209ed295943ed6ad8eda37932 \ + --hash=sha256:567cc568b6bf7b4d041b0c9aa7d6b2c9394f8af6065bc87fa6d23f207b5af9a7 \ + --hash=sha256:5c67ac6df349236fb7b64f57c3dbb0273efcdca24314daa108f2a482c427106c \ + --hash=sha256:746f751de5033b3dbcfcd4a726a4c56ce452c593ad3d54472d8597ce8d8b6d44 \ + --hash=sha256:96a63199c0d81095b5d5d1ae8ca82b64c1c13d18d4e30323ae9e8ab31bc80a3d \ + --hash=sha256:97f1fe3909733212cc6b36f0f199b1421a9c57b79ec665f0322bd604cec47340 \ + --hash=sha256:b9ee9e9ab6a237cebbe16563c787a6e83f3fcc034075c3d321dab94448426282 \ + --hash=sha256:d38f2639f6bf7c3c34a5dfd79e29075811ae2fa9b895a63e76767f74a47a971e \ + --hash=sha256:ef21a1e2384a65c62d5ec5e1cded9fe026f1d032d5c5d725110d1a8d330d8f54 # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # vllm @@ -2465,9 +2482,9 @@ pycparser==2.21 \ # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # cffi -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt @@ -2476,110 +2493,110 @@ pydantic==2.10.0 \ # lm-format-enforcer # mistral-common # openai + # openai-harmony # pydantic-extra-types # vllm # xgrammar -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # pydantic @@ -3162,6 +3179,107 @@ sentencepiece==0.2.0 \ # gguf # mistral-common # vllm +setproctitle==1.3.6 \ + --hash=sha256:082413db8a96b1f021088e8ec23f0a61fec352e649aba20881895815388b66d3 \ + --hash=sha256:0dba8faee2e4a96e934797c9f0f2d093f8239bf210406a99060b3eabe549628e \ + --hash=sha256:0e6b5633c94c5111f7137f875e8f1ff48f53b991d5d5b90932f27dc8c1fa9ae4 \ + --hash=sha256:1065ed36bd03a3fd4186d6c6de5f19846650b015789f72e2dea2d77be99bdca1 \ + --hash=sha256:109fc07b1cd6cef9c245b2028e3e98e038283342b220def311d0239179810dbe \ + --hash=sha256:13624d9925bb481bc0ccfbc7f533da38bfbfe6e80652314f789abc78c2e513bd \ + --hash=sha256:156795b3db976611d09252fc80761fcdb65bb7c9b9581148da900851af25ecf4 \ + --hash=sha256:163dba68f979c61e4e2e779c4d643e968973bdae7c33c3ec4d1869f7a9ba8390 \ + --hash=sha256:17d7c833ed6545ada5ac4bb606b86a28f13a04431953d4beac29d3773aa00b1d \ + --hash=sha256:18d0667bafaaae4c1dee831e2e59841c411ff399b9b4766822ba2685d419c3be \ + --hash=sha256:1aa1935aa2195b76f377e5cb018290376b7bf085f0b53f5a95c0c21011b74367 \ + --hash=sha256:2156d55308431ac3b3ec4e5e05b1726d11a5215352d6a22bb933171dee292f8c \ + --hash=sha256:23a57d3b8f1549515c2dbe4a2880ebc1f27780dc126c5e064167563e015817f5 \ + --hash=sha256:2407955dc359d735a20ac6e797ad160feb33d529a2ac50695c11a1ec680eafab \ + --hash=sha256:2940cf13f4fc11ce69ad2ed37a9f22386bfed314b98d8aebfd4f55459aa59108 \ + --hash=sha256:2e51ec673513465663008ce402171192a053564865c2fc6dc840620871a9bd7c \ + --hash=sha256:3393859eb8f19f5804049a685bf286cb08d447e28ba5c6d8543c7bf5500d5970 \ + --hash=sha256:3884002b3a9086f3018a32ab5d4e1e8214dd70695004e27b1a45c25a6243ad0b \ + --hash=sha256:38ca045626af693da042ac35d7332e7b9dbd52e6351d6973b310612e3acee6d6 \ + --hash=sha256:391bb6a29c4fe7ccc9c30812e3744060802d89b39264cfa77f3d280d7f387ea5 \ + --hash=sha256:3cca16fd055316a48f0debfcbfb6af7cea715429fc31515ab3fcac05abd527d8 \ + --hash=sha256:3cde5b83ec4915cd5e6ae271937fd60d14113c8f7769b4a20d51769fe70d8717 \ + --hash=sha256:3f8194b4d631b003a1176a75d1acd545e04b1f54b821638e098a93e6e62830ef \ + --hash=sha256:3fc97805f9d74444b027babff710bf39df1541437a6a585a983d090ae00cedde \ + --hash=sha256:4431629c178193f23c538cb1de3da285a99ccc86b20ee91d81eb5f1a80e0d2ba \ + --hash=sha256:49498ebf68ca3e75321ffe634fcea5cc720502bfaa79bd6b03ded92ce0dc3c24 \ + --hash=sha256:4ac3eb04bcf0119aadc6235a2c162bae5ed5f740e3d42273a7228b915722de20 \ + --hash=sha256:4adf6a0013fe4e0844e3ba7583ec203ca518b9394c6cc0d3354df2bf31d1c034 \ + --hash=sha256:4efc91b437f6ff2578e89e3f17d010c0a0ff01736606473d082913ecaf7859ba \ + --hash=sha256:50706b9c0eda55f7de18695bfeead5f28b58aa42fd5219b3b1692d554ecbc9ec \ + --hash=sha256:5313a4e9380e46ca0e2c681ba739296f9e7c899e6f4d12a6702b2dc9fb846a31 \ + --hash=sha256:543f59601a4e32daf44741b52f9a23e0ee374f9f13b39c41d917302d98fdd7b0 \ + --hash=sha256:57bc54763bf741813a99fbde91f6be138c8706148b7b42d3752deec46545d470 \ + --hash=sha256:63cc10352dc6cf35a33951656aa660d99f25f574eb78132ce41a85001a638aa7 \ + --hash=sha256:6a1d3aa13acfe81f355b0ce4968facc7a19b0d17223a0f80c011a1dba8388f37 \ + --hash=sha256:6af330ddc2ec05a99c3933ab3cba9365357c0b8470a7f2fa054ee4b0984f57d1 \ + --hash=sha256:6d50bfcc1d1692dc55165b3dd2f0b9f8fb5b1f7b571a93e08d660ad54b9ca1a5 \ + --hash=sha256:70100e2087fe05359f249a0b5f393127b3a1819bf34dec3a3e0d4941138650c9 \ + --hash=sha256:74973aebea3543ad033b9103db30579ec2b950a466e09f9c2180089e8346e0ec \ + --hash=sha256:751ba352ed922e0af60458e961167fa7b732ac31c0ddd1476a2dfd30ab5958c5 \ + --hash=sha256:785cd210c0311d9be28a70e281a914486d62bfd44ac926fcd70cf0b4d65dff1c \ + --hash=sha256:7890e291bf4708e3b61db9069ea39b3ab0651e42923a5e1f4d78a7b9e4b18301 \ + --hash=sha256:793a23e8d9cb6c231aa3023d700008224c6ec5b8fd622d50f3c51665e3d0a190 \ + --hash=sha256:797f2846b546a8741413c57d9fb930ad5aa939d925c9c0fa6186d77580035af7 \ + --hash=sha256:7df5fcc48588f82b6cc8073db069609ddd48a49b1e9734a20d0efb32464753c4 \ + --hash=sha256:8050c01331135f77ec99d99307bfbc6519ea24d2f92964b06f3222a804a3ff1f \ + --hash=sha256:805bb33e92fc3d8aa05674db3068d14d36718e3f2c5c79b09807203f229bf4b5 \ + --hash=sha256:807796fe301b7ed76cf100113cc008c119daf4fea2f9f43c578002aef70c3ebf \ + --hash=sha256:81c443310831e29fabbd07b75ebbfa29d0740b56f5907c6af218482d51260431 \ + --hash=sha256:83066ffbf77a5f82b7e96e59bdccbdda203c8dccbfc3f9f0fdad3a08d0001d9c \ + --hash=sha256:8834ab7be6539f1bfadec7c8d12249bbbe6c2413b1d40ffc0ec408692232a0c6 \ + --hash=sha256:92df0e70b884f5da35f2e01489dca3c06a79962fb75636985f1e3a17aec66833 \ + --hash=sha256:9483aa336687463f5497dd37a070094f3dff55e2c888994f8440fcf426a1a844 \ + --hash=sha256:97a138fa875c6f281df7720dac742259e85518135cd0e3551aba1c628103d853 \ + --hash=sha256:9b50700785eccac0819bea794d968ed8f6055c88f29364776b7ea076ac105c5d \ + --hash=sha256:9b73cf0fe28009a04a35bb2522e4c5b5176cc148919431dcb73fdbdfaab15781 \ + --hash=sha256:9d5a369eb7ec5b2fdfa9927530b5259dd21893fa75d4e04a223332f61b84b586 \ + --hash=sha256:a094b7ce455ca341b59a0f6ce6be2e11411ba6e2860b9aa3dbb37468f23338f4 \ + --hash=sha256:a0d6252098e98129a1decb59b46920d4eca17b0395f3d71b0d327d086fefe77d \ + --hash=sha256:a1d856b0f4e4a33e31cdab5f50d0a14998f3a2d726a3fd5cb7c4d45a57b28d1b \ + --hash=sha256:a4ae2ea9afcfdd2b931ddcebf1cf82532162677e00326637b31ed5dff7d985ca \ + --hash=sha256:a5963b663da69ad25fa1559ee064584935570def665917918938c1f1289f5ebc \ + --hash=sha256:ad1c2c2baaba62823a7f348f469a967ece0062140ca39e7a48e4bbb1f20d54c4 \ + --hash=sha256:ae82507fe458f7c0c8227017f2158111a4c9e7ce94de05178894a7ea9fefc8a1 \ + --hash=sha256:af188f3305f0a65c3217c30c6d4c06891e79144076a91e8b454f14256acc7279 \ + --hash=sha256:af44bb7a1af163806bbb679eb8432fa7b4fb6d83a5d403b541b675dcd3798638 \ + --hash=sha256:b0174ca6f3018ddeaa49847f29b69612e590534c1d2186d54ab25161ecc42975 \ + --hash=sha256:b2b17855ed7f994f3f259cf2dfbfad78814538536fa1a91b50253d84d87fd88d \ + --hash=sha256:b2e54f4a2dc6edf0f5ea5b1d0a608d2af3dcb5aa8c8eeab9c8841b23e1b054fe \ + --hash=sha256:b6f4abde9a2946f57e8daaf1160b2351bcf64274ef539e6675c1d945dbd75e2a \ + --hash=sha256:b70c07409d465f3a8b34d52f863871fb8a00755370791d2bd1d4f82b3cdaf3d5 \ + --hash=sha256:bb465dd5825356c1191a038a86ee1b8166e3562d6e8add95eec04ab484cfb8a2 \ + --hash=sha256:c051f46ed1e13ba8214b334cbf21902102807582fbfaf0fef341b9e52f0fafbf \ + --hash=sha256:c1b20a5f4164cec7007be55c9cf18d2cd08ed7c3bf6769b3cd6d044ad888d74b \ + --hash=sha256:c86e9e82bfab579327dbe9b82c71475165fbc8b2134d24f9a3b2edaf200a5c3d \ + --hash=sha256:c9f32b96c700bb384f33f7cf07954bb609d35dd82752cef57fb2ee0968409169 \ + --hash=sha256:cce0ed8b3f64c71c140f0ec244e5fdf8ecf78ddf8d2e591d4a8b6aa1c1214235 \ + --hash=sha256:cdd7315314b0744a7dd506f3bd0f2cf90734181529cdcf75542ee35ad885cab7 \ + --hash=sha256:cf355fbf0d4275d86f9f57be705d8e5eaa7f8ddb12b24ced2ea6cbd68fdb14dc \ + --hash=sha256:d136fbf8ad4321716e44d6d6b3d8dffb4872626010884e07a1db54b7450836cf \ + --hash=sha256:d2c8e20487b3b73c1fa72c56f5c89430617296cd380373e7af3a538a82d4cd6d \ + --hash=sha256:d483cc23cc56ab32911ea0baa0d2d9ea7aa065987f47de847a0a93a58bf57905 \ + --hash=sha256:d5a6c4864bb6fa9fcf7b57a830d21aed69fd71742a5ebcdbafda476be673d212 \ + --hash=sha256:d714e002dd3638170fe7376dc1b686dbac9cb712cde3f7224440af722cc9866a \ + --hash=sha256:d73f14b86d0e2858ece6bf5807c9889670e392c001d414b4293d0d9b291942c3 \ + --hash=sha256:d88c63bd395c787b0aa81d8bbc22c1809f311032ce3e823a6517b711129818e4 \ + --hash=sha256:db608db98ccc21248370d30044a60843b3f0f3d34781ceeea67067c508cd5a28 \ + --hash=sha256:de004939fc3fd0c1200d26ea9264350bfe501ffbf46c8cf5dc7f345f2d87a7f1 \ + --hash=sha256:ded9e86397267732a0641d4776c7c663ea16b64d7dbc4d9cc6ad8536363a2d29 \ + --hash=sha256:e288f8a162d663916060beb5e8165a8551312b08efee9cf68302687471a6545d \ + --hash=sha256:e2a9e62647dc040a76d55563580bf3bb8fe1f5b6ead08447c2ed0d7786e5e794 \ + --hash=sha256:e3e44d08b61de0dd6f205528498f834a51a5c06689f8fb182fe26f3a3ce7dca9 \ + --hash=sha256:ea002088d5554fd75e619742cefc78b84a212ba21632e59931b3501f0cfc8f67 \ + --hash=sha256:eb7452849f6615871eabed6560ffedfe56bc8af31a823b6be4ce1e6ff0ab72c5 \ + --hash=sha256:ebcf34b69df4ca0eabaaaf4a3d890f637f355fed00ba806f7ebdd2d040658c26 \ + --hash=sha256:f24d5b9383318cbd1a5cd969377937d66cf0542f24aa728a4f49d9f98f9c0da8 \ + --hash=sha256:f33fbf96b52d51c23b6cff61f57816539c1c147db270cfc1cc3bc012f4a560a9 + # via + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock + # vllm shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de @@ -3377,12 +3495,11 @@ tqdm==4.67.1 \ # openai # transformers # vllm -transformers==4.53.2 \ - --hash=sha256:6c3ed95edfb1cba71c4245758f1b4878c93bf8cde77d076307dacb2cbbd72be2 \ - --hash=sha256:db8f4819bb34f000029c73c3c557e7d06fc1b8e612ec142eecdae3947a9c78bf +transformers==4.55.2 \ + --hash=sha256:097e3c2e2c0c9681db3da9d748d8f9d6a724c644514673d0030e8c5a1109f1f1 \ + --hash=sha256:a45ec60c03474fd67adbce5c434685051b7608b3f4f167c25aa6aeb1cad16d4f # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock - # -r python/requirements/llm/llm-requirements.txt # compressed-tensors # vllm # xgrammar @@ -3419,7 +3536,14 @@ typing-extensions==4.12.2 \ # referencing # torch # typer + # typing-inspection # vllm +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/deplocks/llm/rayllm_test_py311_cu128.lock + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 @@ -3495,9 +3619,9 @@ virtualenv==20.29.1 \ # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt -vllm==0.10.0 \ - --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ - --hash=sha256:a44e9013db26082a82c3931ed8772ac884d6d60566d36ecdb0e8dc01c65b241a +vllm==0.10.1.1 \ + --hash=sha256:3099824ee4bdaa14c4c4f7178a092101a0ec206d4c9371edf295849b2b730a39 \ + --hash=sha256:8ca0dd985e1ceac8540e7719c654f1553b3ba8a43c685ac8d3fa1366ffb6443a # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements/llm/llm-requirements.txt diff --git a/python/deplocks/llm/rayllm_test_py311_cpu.lock b/python/deplocks/llm/rayllm_test_py311_cpu.lock index 04c9ddb2de7b..dc1ea6c32e2e 100644 --- a/python/deplocks/llm/rayllm_test_py311_cpu.lock +++ b/python/deplocks/llm/rayllm_test_py311_cpu.lock @@ -1374,7 +1374,7 @@ hf-transfer==0.1.9 \ --hash=sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f \ --hash=sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d # via -r python/requirements/llm/llm-requirements.txt -hf-xet==1.1.5 \ +hf-xet==1.1.5 ; platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 \ --hash=sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245 \ --hash=sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a \ @@ -1452,7 +1452,6 @@ huggingface-hub==0.34.3 \ # via # tokenizers # transformers - # vllm humanize==4.12.1 \ --hash=sha256:1338ba97415c96556758a6e2f65977ed406dddf4620d4c6db9bbdfd07f0f1232 \ --hash=sha256:86014ca5c52675dffa1d404491952f1f5bf03b07c175a51891a343daebf01fea @@ -2506,9 +2505,25 @@ oauth2client==4.1.3 \ # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt -openai==1.90.0 \ - --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ - --hash=sha256:e5dcb5498ea6b42fec47546d10f1bcc05fb854219a7d953a5ba766718b212a02 +openai==1.100.2 \ + --hash=sha256:54d3457b2c8d7303a1bc002a058de46bdd8f37a8117751c7cf4ed4438051f151 \ + --hash=sha256:787b4c3c8a65895182c58c424f790c25c790cc9a0330e34f73d55b6ee5a00e32 + # via vllm +openai-harmony==0.0.4 \ + --hash=sha256:038f1d6772d1be5213b36ae76e5d042022395ec35c428a73ccb8b839b2cecf6a \ + --hash=sha256:15e6d53a66502491a3675a536df30e271f976e6c5efe68250a65191efcb85c4f \ + --hash=sha256:2d8d16d84702059833fb03b841b28c25600c54e83cadccef79af44e1c81166b1 \ + --hash=sha256:31e9bcac0902a309e2fc688e52f247eec7fffcd00d17e958b9a83a8fea6519c2 \ + --hash=sha256:3586d90c899cd41f8624e7b82a48c289f6e4be56c66304ecaf3a0ba88963a73f \ + --hash=sha256:3cf2344366f10981bbc0f6d9949a0b2bb87151d209ed295943ed6ad8eda37932 \ + --hash=sha256:567cc568b6bf7b4d041b0c9aa7d6b2c9394f8af6065bc87fa6d23f207b5af9a7 \ + --hash=sha256:5c67ac6df349236fb7b64f57c3dbb0273efcdca24314daa108f2a482c427106c \ + --hash=sha256:746f751de5033b3dbcfcd4a726a4c56ce452c593ad3d54472d8597ce8d8b6d44 \ + --hash=sha256:96a63199c0d81095b5d5d1ae8ca82b64c1c13d18d4e30323ae9e8ab31bc80a3d \ + --hash=sha256:97f1fe3909733212cc6b36f0f199b1421a9c57b79ec665f0322bd604cec47340 \ + --hash=sha256:b9ee9e9ab6a237cebbe16563c787a6e83f3fcc034075c3d321dab94448426282 \ + --hash=sha256:d38f2639f6bf7c3c34a5dfd79e29075811ae2fa9b895a63e76767f74a47a971e \ + --hash=sha256:ef21a1e2384a65c62d5ec5e1cded9fe026f1d032d5c5d725110d1a8d330d8f54 # via vllm opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ @@ -3260,9 +3275,9 @@ pycurl==7.45.3 \ # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt @@ -3271,110 +3286,110 @@ pydantic==2.10.0 \ # lm-format-enforcer # mistral-common # openai + # openai-harmony # pydantic-extra-types # vllm # xgrammar -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # pydantic @@ -4040,6 +4055,105 @@ sentencepiece==0.2.0 \ # gguf # mistral-common # vllm +setproctitle==1.3.6 \ + --hash=sha256:082413db8a96b1f021088e8ec23f0a61fec352e649aba20881895815388b66d3 \ + --hash=sha256:0dba8faee2e4a96e934797c9f0f2d093f8239bf210406a99060b3eabe549628e \ + --hash=sha256:0e6b5633c94c5111f7137f875e8f1ff48f53b991d5d5b90932f27dc8c1fa9ae4 \ + --hash=sha256:1065ed36bd03a3fd4186d6c6de5f19846650b015789f72e2dea2d77be99bdca1 \ + --hash=sha256:109fc07b1cd6cef9c245b2028e3e98e038283342b220def311d0239179810dbe \ + --hash=sha256:13624d9925bb481bc0ccfbc7f533da38bfbfe6e80652314f789abc78c2e513bd \ + --hash=sha256:156795b3db976611d09252fc80761fcdb65bb7c9b9581148da900851af25ecf4 \ + --hash=sha256:163dba68f979c61e4e2e779c4d643e968973bdae7c33c3ec4d1869f7a9ba8390 \ + --hash=sha256:17d7c833ed6545ada5ac4bb606b86a28f13a04431953d4beac29d3773aa00b1d \ + --hash=sha256:18d0667bafaaae4c1dee831e2e59841c411ff399b9b4766822ba2685d419c3be \ + --hash=sha256:1aa1935aa2195b76f377e5cb018290376b7bf085f0b53f5a95c0c21011b74367 \ + --hash=sha256:2156d55308431ac3b3ec4e5e05b1726d11a5215352d6a22bb933171dee292f8c \ + --hash=sha256:23a57d3b8f1549515c2dbe4a2880ebc1f27780dc126c5e064167563e015817f5 \ + --hash=sha256:2407955dc359d735a20ac6e797ad160feb33d529a2ac50695c11a1ec680eafab \ + --hash=sha256:2940cf13f4fc11ce69ad2ed37a9f22386bfed314b98d8aebfd4f55459aa59108 \ + --hash=sha256:2e51ec673513465663008ce402171192a053564865c2fc6dc840620871a9bd7c \ + --hash=sha256:3393859eb8f19f5804049a685bf286cb08d447e28ba5c6d8543c7bf5500d5970 \ + --hash=sha256:3884002b3a9086f3018a32ab5d4e1e8214dd70695004e27b1a45c25a6243ad0b \ + --hash=sha256:38ca045626af693da042ac35d7332e7b9dbd52e6351d6973b310612e3acee6d6 \ + --hash=sha256:391bb6a29c4fe7ccc9c30812e3744060802d89b39264cfa77f3d280d7f387ea5 \ + --hash=sha256:3cca16fd055316a48f0debfcbfb6af7cea715429fc31515ab3fcac05abd527d8 \ + --hash=sha256:3cde5b83ec4915cd5e6ae271937fd60d14113c8f7769b4a20d51769fe70d8717 \ + --hash=sha256:3f8194b4d631b003a1176a75d1acd545e04b1f54b821638e098a93e6e62830ef \ + --hash=sha256:3fc97805f9d74444b027babff710bf39df1541437a6a585a983d090ae00cedde \ + --hash=sha256:4431629c178193f23c538cb1de3da285a99ccc86b20ee91d81eb5f1a80e0d2ba \ + --hash=sha256:49498ebf68ca3e75321ffe634fcea5cc720502bfaa79bd6b03ded92ce0dc3c24 \ + --hash=sha256:4ac3eb04bcf0119aadc6235a2c162bae5ed5f740e3d42273a7228b915722de20 \ + --hash=sha256:4adf6a0013fe4e0844e3ba7583ec203ca518b9394c6cc0d3354df2bf31d1c034 \ + --hash=sha256:4efc91b437f6ff2578e89e3f17d010c0a0ff01736606473d082913ecaf7859ba \ + --hash=sha256:50706b9c0eda55f7de18695bfeead5f28b58aa42fd5219b3b1692d554ecbc9ec \ + --hash=sha256:5313a4e9380e46ca0e2c681ba739296f9e7c899e6f4d12a6702b2dc9fb846a31 \ + --hash=sha256:543f59601a4e32daf44741b52f9a23e0ee374f9f13b39c41d917302d98fdd7b0 \ + --hash=sha256:57bc54763bf741813a99fbde91f6be138c8706148b7b42d3752deec46545d470 \ + --hash=sha256:63cc10352dc6cf35a33951656aa660d99f25f574eb78132ce41a85001a638aa7 \ + --hash=sha256:6a1d3aa13acfe81f355b0ce4968facc7a19b0d17223a0f80c011a1dba8388f37 \ + --hash=sha256:6af330ddc2ec05a99c3933ab3cba9365357c0b8470a7f2fa054ee4b0984f57d1 \ + --hash=sha256:6d50bfcc1d1692dc55165b3dd2f0b9f8fb5b1f7b571a93e08d660ad54b9ca1a5 \ + --hash=sha256:70100e2087fe05359f249a0b5f393127b3a1819bf34dec3a3e0d4941138650c9 \ + --hash=sha256:74973aebea3543ad033b9103db30579ec2b950a466e09f9c2180089e8346e0ec \ + --hash=sha256:751ba352ed922e0af60458e961167fa7b732ac31c0ddd1476a2dfd30ab5958c5 \ + --hash=sha256:785cd210c0311d9be28a70e281a914486d62bfd44ac926fcd70cf0b4d65dff1c \ + --hash=sha256:7890e291bf4708e3b61db9069ea39b3ab0651e42923a5e1f4d78a7b9e4b18301 \ + --hash=sha256:793a23e8d9cb6c231aa3023d700008224c6ec5b8fd622d50f3c51665e3d0a190 \ + --hash=sha256:797f2846b546a8741413c57d9fb930ad5aa939d925c9c0fa6186d77580035af7 \ + --hash=sha256:7df5fcc48588f82b6cc8073db069609ddd48a49b1e9734a20d0efb32464753c4 \ + --hash=sha256:8050c01331135f77ec99d99307bfbc6519ea24d2f92964b06f3222a804a3ff1f \ + --hash=sha256:805bb33e92fc3d8aa05674db3068d14d36718e3f2c5c79b09807203f229bf4b5 \ + --hash=sha256:807796fe301b7ed76cf100113cc008c119daf4fea2f9f43c578002aef70c3ebf \ + --hash=sha256:81c443310831e29fabbd07b75ebbfa29d0740b56f5907c6af218482d51260431 \ + --hash=sha256:83066ffbf77a5f82b7e96e59bdccbdda203c8dccbfc3f9f0fdad3a08d0001d9c \ + --hash=sha256:8834ab7be6539f1bfadec7c8d12249bbbe6c2413b1d40ffc0ec408692232a0c6 \ + --hash=sha256:92df0e70b884f5da35f2e01489dca3c06a79962fb75636985f1e3a17aec66833 \ + --hash=sha256:9483aa336687463f5497dd37a070094f3dff55e2c888994f8440fcf426a1a844 \ + --hash=sha256:97a138fa875c6f281df7720dac742259e85518135cd0e3551aba1c628103d853 \ + --hash=sha256:9b50700785eccac0819bea794d968ed8f6055c88f29364776b7ea076ac105c5d \ + --hash=sha256:9b73cf0fe28009a04a35bb2522e4c5b5176cc148919431dcb73fdbdfaab15781 \ + --hash=sha256:9d5a369eb7ec5b2fdfa9927530b5259dd21893fa75d4e04a223332f61b84b586 \ + --hash=sha256:a094b7ce455ca341b59a0f6ce6be2e11411ba6e2860b9aa3dbb37468f23338f4 \ + --hash=sha256:a0d6252098e98129a1decb59b46920d4eca17b0395f3d71b0d327d086fefe77d \ + --hash=sha256:a1d856b0f4e4a33e31cdab5f50d0a14998f3a2d726a3fd5cb7c4d45a57b28d1b \ + --hash=sha256:a4ae2ea9afcfdd2b931ddcebf1cf82532162677e00326637b31ed5dff7d985ca \ + --hash=sha256:a5963b663da69ad25fa1559ee064584935570def665917918938c1f1289f5ebc \ + --hash=sha256:ad1c2c2baaba62823a7f348f469a967ece0062140ca39e7a48e4bbb1f20d54c4 \ + --hash=sha256:ae82507fe458f7c0c8227017f2158111a4c9e7ce94de05178894a7ea9fefc8a1 \ + --hash=sha256:af188f3305f0a65c3217c30c6d4c06891e79144076a91e8b454f14256acc7279 \ + --hash=sha256:af44bb7a1af163806bbb679eb8432fa7b4fb6d83a5d403b541b675dcd3798638 \ + --hash=sha256:b0174ca6f3018ddeaa49847f29b69612e590534c1d2186d54ab25161ecc42975 \ + --hash=sha256:b2b17855ed7f994f3f259cf2dfbfad78814538536fa1a91b50253d84d87fd88d \ + --hash=sha256:b2e54f4a2dc6edf0f5ea5b1d0a608d2af3dcb5aa8c8eeab9c8841b23e1b054fe \ + --hash=sha256:b6f4abde9a2946f57e8daaf1160b2351bcf64274ef539e6675c1d945dbd75e2a \ + --hash=sha256:b70c07409d465f3a8b34d52f863871fb8a00755370791d2bd1d4f82b3cdaf3d5 \ + --hash=sha256:bb465dd5825356c1191a038a86ee1b8166e3562d6e8add95eec04ab484cfb8a2 \ + --hash=sha256:c051f46ed1e13ba8214b334cbf21902102807582fbfaf0fef341b9e52f0fafbf \ + --hash=sha256:c1b20a5f4164cec7007be55c9cf18d2cd08ed7c3bf6769b3cd6d044ad888d74b \ + --hash=sha256:c86e9e82bfab579327dbe9b82c71475165fbc8b2134d24f9a3b2edaf200a5c3d \ + --hash=sha256:c9f32b96c700bb384f33f7cf07954bb609d35dd82752cef57fb2ee0968409169 \ + --hash=sha256:cce0ed8b3f64c71c140f0ec244e5fdf8ecf78ddf8d2e591d4a8b6aa1c1214235 \ + --hash=sha256:cdd7315314b0744a7dd506f3bd0f2cf90734181529cdcf75542ee35ad885cab7 \ + --hash=sha256:cf355fbf0d4275d86f9f57be705d8e5eaa7f8ddb12b24ced2ea6cbd68fdb14dc \ + --hash=sha256:d136fbf8ad4321716e44d6d6b3d8dffb4872626010884e07a1db54b7450836cf \ + --hash=sha256:d2c8e20487b3b73c1fa72c56f5c89430617296cd380373e7af3a538a82d4cd6d \ + --hash=sha256:d483cc23cc56ab32911ea0baa0d2d9ea7aa065987f47de847a0a93a58bf57905 \ + --hash=sha256:d5a6c4864bb6fa9fcf7b57a830d21aed69fd71742a5ebcdbafda476be673d212 \ + --hash=sha256:d714e002dd3638170fe7376dc1b686dbac9cb712cde3f7224440af722cc9866a \ + --hash=sha256:d73f14b86d0e2858ece6bf5807c9889670e392c001d414b4293d0d9b291942c3 \ + --hash=sha256:d88c63bd395c787b0aa81d8bbc22c1809f311032ce3e823a6517b711129818e4 \ + --hash=sha256:db608db98ccc21248370d30044a60843b3f0f3d34781ceeea67067c508cd5a28 \ + --hash=sha256:de004939fc3fd0c1200d26ea9264350bfe501ffbf46c8cf5dc7f345f2d87a7f1 \ + --hash=sha256:ded9e86397267732a0641d4776c7c663ea16b64d7dbc4d9cc6ad8536363a2d29 \ + --hash=sha256:e288f8a162d663916060beb5e8165a8551312b08efee9cf68302687471a6545d \ + --hash=sha256:e2a9e62647dc040a76d55563580bf3bb8fe1f5b6ead08447c2ed0d7786e5e794 \ + --hash=sha256:e3e44d08b61de0dd6f205528498f834a51a5c06689f8fb182fe26f3a3ce7dca9 \ + --hash=sha256:ea002088d5554fd75e619742cefc78b84a212ba21632e59931b3501f0cfc8f67 \ + --hash=sha256:eb7452849f6615871eabed6560ffedfe56bc8af31a823b6be4ce1e6ff0ab72c5 \ + --hash=sha256:ebcf34b69df4ca0eabaaaf4a3d890f637f355fed00ba806f7ebdd2d040658c26 \ + --hash=sha256:f24d5b9383318cbd1a5cd969377937d66cf0542f24aa728a4f49d9f98f9c0da8 \ + --hash=sha256:f33fbf96b52d51c23b6cff61f57816539c1c147db270cfc1cc3bc012f4a560a9 + # via vllm shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de @@ -4405,11 +4519,10 @@ traitlets==5.14.3 \ # nbconvert # nbformat # notebook -transformers==4.53.2 \ - --hash=sha256:6c3ed95edfb1cba71c4245758f1b4878c93bf8cde77d076307dacb2cbbd72be2 \ - --hash=sha256:db8f4819bb34f000029c73c3c557e7d06fc1b8e612ec142eecdae3947a9c78bf +transformers==4.55.2 \ + --hash=sha256:097e3c2e2c0c9681db3da9d748d8f9d6a724c644514673d0030e8c5a1109f1f1 \ + --hash=sha256:a45ec60c03474fd67adbce5c434685051b7608b3f4f167c25aa6aeb1cad16d4f # via - # -r python/requirements/llm/llm-requirements.txt # compressed-tensors # vllm # xgrammar @@ -4464,7 +4577,14 @@ typing-extensions==4.12.2 \ # referencing # torch # typer + # typing-inspection # vllm +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/deplocks/llm/ray_test_py311_cpu.lock + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 @@ -4552,9 +4672,9 @@ virtualenv==20.29.1 \ # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt -vllm==0.10.0 \ - --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ - --hash=sha256:a44e9013db26082a82c3931ed8772ac884d6d60566d36ecdb0e8dc01c65b241a +vllm==0.10.1.1 \ + --hash=sha256:3099824ee4bdaa14c4c4f7178a092101a0ec206d4c9371edf295849b2b730a39 \ + --hash=sha256:8ca0dd985e1ceac8540e7719c654f1553b3ba8a43c685ac8d3fa1366ffb6443a # via -r python/requirements/llm/llm-requirements.txt watchfiles==0.19.0 \ --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ diff --git a/python/deplocks/llm/rayllm_test_py311_cu121.lock b/python/deplocks/llm/rayllm_test_py311_cu121.lock index 18d46b031806..eabbdaefa580 100644 --- a/python/deplocks/llm/rayllm_test_py311_cu121.lock +++ b/python/deplocks/llm/rayllm_test_py311_cu121.lock @@ -1374,7 +1374,7 @@ hf-transfer==0.1.9 \ --hash=sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f \ --hash=sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d # via -r python/requirements/llm/llm-requirements.txt -hf-xet==1.1.5 \ +hf-xet==1.1.5 ; platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:69ebbcfd9ec44fdc2af73441619eeb06b94ee34511bbcf57cd423820090f5694 \ --hash=sha256:73e167d9807d166596b4b2f0b585c6d5bd84a26dea32843665a8b58f6edba245 \ --hash=sha256:83088ecea236d5113de478acb2339f92c95b4fb0462acaa30621fac02f5a534a \ @@ -1452,7 +1452,6 @@ huggingface-hub==0.34.3 \ # via # tokenizers # transformers - # vllm humanize==4.12.1 \ --hash=sha256:1338ba97415c96556758a6e2f65977ed406dddf4620d4c6db9bbdfd07f0f1232 \ --hash=sha256:86014ca5c52675dffa1d404491952f1f5bf03b07c175a51891a343daebf01fea @@ -2597,9 +2596,25 @@ oauth2client==4.1.3 \ # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt -openai==1.90.0 \ - --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ - --hash=sha256:e5dcb5498ea6b42fec47546d10f1bcc05fb854219a7d953a5ba766718b212a02 +openai==1.100.2 \ + --hash=sha256:54d3457b2c8d7303a1bc002a058de46bdd8f37a8117751c7cf4ed4438051f151 \ + --hash=sha256:787b4c3c8a65895182c58c424f790c25c790cc9a0330e34f73d55b6ee5a00e32 + # via vllm +openai-harmony==0.0.4 \ + --hash=sha256:038f1d6772d1be5213b36ae76e5d042022395ec35c428a73ccb8b839b2cecf6a \ + --hash=sha256:15e6d53a66502491a3675a536df30e271f976e6c5efe68250a65191efcb85c4f \ + --hash=sha256:2d8d16d84702059833fb03b841b28c25600c54e83cadccef79af44e1c81166b1 \ + --hash=sha256:31e9bcac0902a309e2fc688e52f247eec7fffcd00d17e958b9a83a8fea6519c2 \ + --hash=sha256:3586d90c899cd41f8624e7b82a48c289f6e4be56c66304ecaf3a0ba88963a73f \ + --hash=sha256:3cf2344366f10981bbc0f6d9949a0b2bb87151d209ed295943ed6ad8eda37932 \ + --hash=sha256:567cc568b6bf7b4d041b0c9aa7d6b2c9394f8af6065bc87fa6d23f207b5af9a7 \ + --hash=sha256:5c67ac6df349236fb7b64f57c3dbb0273efcdca24314daa108f2a482c427106c \ + --hash=sha256:746f751de5033b3dbcfcd4a726a4c56ce452c593ad3d54472d8597ce8d8b6d44 \ + --hash=sha256:96a63199c0d81095b5d5d1ae8ca82b64c1c13d18d4e30323ae9e8ab31bc80a3d \ + --hash=sha256:97f1fe3909733212cc6b36f0f199b1421a9c57b79ec665f0322bd604cec47340 \ + --hash=sha256:b9ee9e9ab6a237cebbe16563c787a6e83f3fcc034075c3d321dab94448426282 \ + --hash=sha256:d38f2639f6bf7c3c34a5dfd79e29075811ae2fa9b895a63e76767f74a47a971e \ + --hash=sha256:ef21a1e2384a65c62d5ec5e1cded9fe026f1d032d5c5d725110d1a8d330d8f54 # via vllm opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ @@ -3351,9 +3366,9 @@ pycurl==7.45.3 \ # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt @@ -3362,110 +3377,110 @@ pydantic==2.10.0 \ # lm-format-enforcer # mistral-common # openai + # openai-harmony # pydantic-extra-types # vllm # xgrammar -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # pydantic @@ -4131,6 +4146,105 @@ sentencepiece==0.2.0 \ # gguf # mistral-common # vllm +setproctitle==1.3.6 \ + --hash=sha256:082413db8a96b1f021088e8ec23f0a61fec352e649aba20881895815388b66d3 \ + --hash=sha256:0dba8faee2e4a96e934797c9f0f2d093f8239bf210406a99060b3eabe549628e \ + --hash=sha256:0e6b5633c94c5111f7137f875e8f1ff48f53b991d5d5b90932f27dc8c1fa9ae4 \ + --hash=sha256:1065ed36bd03a3fd4186d6c6de5f19846650b015789f72e2dea2d77be99bdca1 \ + --hash=sha256:109fc07b1cd6cef9c245b2028e3e98e038283342b220def311d0239179810dbe \ + --hash=sha256:13624d9925bb481bc0ccfbc7f533da38bfbfe6e80652314f789abc78c2e513bd \ + --hash=sha256:156795b3db976611d09252fc80761fcdb65bb7c9b9581148da900851af25ecf4 \ + --hash=sha256:163dba68f979c61e4e2e779c4d643e968973bdae7c33c3ec4d1869f7a9ba8390 \ + --hash=sha256:17d7c833ed6545ada5ac4bb606b86a28f13a04431953d4beac29d3773aa00b1d \ + --hash=sha256:18d0667bafaaae4c1dee831e2e59841c411ff399b9b4766822ba2685d419c3be \ + --hash=sha256:1aa1935aa2195b76f377e5cb018290376b7bf085f0b53f5a95c0c21011b74367 \ + --hash=sha256:2156d55308431ac3b3ec4e5e05b1726d11a5215352d6a22bb933171dee292f8c \ + --hash=sha256:23a57d3b8f1549515c2dbe4a2880ebc1f27780dc126c5e064167563e015817f5 \ + --hash=sha256:2407955dc359d735a20ac6e797ad160feb33d529a2ac50695c11a1ec680eafab \ + --hash=sha256:2940cf13f4fc11ce69ad2ed37a9f22386bfed314b98d8aebfd4f55459aa59108 \ + --hash=sha256:2e51ec673513465663008ce402171192a053564865c2fc6dc840620871a9bd7c \ + --hash=sha256:3393859eb8f19f5804049a685bf286cb08d447e28ba5c6d8543c7bf5500d5970 \ + --hash=sha256:3884002b3a9086f3018a32ab5d4e1e8214dd70695004e27b1a45c25a6243ad0b \ + --hash=sha256:38ca045626af693da042ac35d7332e7b9dbd52e6351d6973b310612e3acee6d6 \ + --hash=sha256:391bb6a29c4fe7ccc9c30812e3744060802d89b39264cfa77f3d280d7f387ea5 \ + --hash=sha256:3cca16fd055316a48f0debfcbfb6af7cea715429fc31515ab3fcac05abd527d8 \ + --hash=sha256:3cde5b83ec4915cd5e6ae271937fd60d14113c8f7769b4a20d51769fe70d8717 \ + --hash=sha256:3f8194b4d631b003a1176a75d1acd545e04b1f54b821638e098a93e6e62830ef \ + --hash=sha256:3fc97805f9d74444b027babff710bf39df1541437a6a585a983d090ae00cedde \ + --hash=sha256:4431629c178193f23c538cb1de3da285a99ccc86b20ee91d81eb5f1a80e0d2ba \ + --hash=sha256:49498ebf68ca3e75321ffe634fcea5cc720502bfaa79bd6b03ded92ce0dc3c24 \ + --hash=sha256:4ac3eb04bcf0119aadc6235a2c162bae5ed5f740e3d42273a7228b915722de20 \ + --hash=sha256:4adf6a0013fe4e0844e3ba7583ec203ca518b9394c6cc0d3354df2bf31d1c034 \ + --hash=sha256:4efc91b437f6ff2578e89e3f17d010c0a0ff01736606473d082913ecaf7859ba \ + --hash=sha256:50706b9c0eda55f7de18695bfeead5f28b58aa42fd5219b3b1692d554ecbc9ec \ + --hash=sha256:5313a4e9380e46ca0e2c681ba739296f9e7c899e6f4d12a6702b2dc9fb846a31 \ + --hash=sha256:543f59601a4e32daf44741b52f9a23e0ee374f9f13b39c41d917302d98fdd7b0 \ + --hash=sha256:57bc54763bf741813a99fbde91f6be138c8706148b7b42d3752deec46545d470 \ + --hash=sha256:63cc10352dc6cf35a33951656aa660d99f25f574eb78132ce41a85001a638aa7 \ + --hash=sha256:6a1d3aa13acfe81f355b0ce4968facc7a19b0d17223a0f80c011a1dba8388f37 \ + --hash=sha256:6af330ddc2ec05a99c3933ab3cba9365357c0b8470a7f2fa054ee4b0984f57d1 \ + --hash=sha256:6d50bfcc1d1692dc55165b3dd2f0b9f8fb5b1f7b571a93e08d660ad54b9ca1a5 \ + --hash=sha256:70100e2087fe05359f249a0b5f393127b3a1819bf34dec3a3e0d4941138650c9 \ + --hash=sha256:74973aebea3543ad033b9103db30579ec2b950a466e09f9c2180089e8346e0ec \ + --hash=sha256:751ba352ed922e0af60458e961167fa7b732ac31c0ddd1476a2dfd30ab5958c5 \ + --hash=sha256:785cd210c0311d9be28a70e281a914486d62bfd44ac926fcd70cf0b4d65dff1c \ + --hash=sha256:7890e291bf4708e3b61db9069ea39b3ab0651e42923a5e1f4d78a7b9e4b18301 \ + --hash=sha256:793a23e8d9cb6c231aa3023d700008224c6ec5b8fd622d50f3c51665e3d0a190 \ + --hash=sha256:797f2846b546a8741413c57d9fb930ad5aa939d925c9c0fa6186d77580035af7 \ + --hash=sha256:7df5fcc48588f82b6cc8073db069609ddd48a49b1e9734a20d0efb32464753c4 \ + --hash=sha256:8050c01331135f77ec99d99307bfbc6519ea24d2f92964b06f3222a804a3ff1f \ + --hash=sha256:805bb33e92fc3d8aa05674db3068d14d36718e3f2c5c79b09807203f229bf4b5 \ + --hash=sha256:807796fe301b7ed76cf100113cc008c119daf4fea2f9f43c578002aef70c3ebf \ + --hash=sha256:81c443310831e29fabbd07b75ebbfa29d0740b56f5907c6af218482d51260431 \ + --hash=sha256:83066ffbf77a5f82b7e96e59bdccbdda203c8dccbfc3f9f0fdad3a08d0001d9c \ + --hash=sha256:8834ab7be6539f1bfadec7c8d12249bbbe6c2413b1d40ffc0ec408692232a0c6 \ + --hash=sha256:92df0e70b884f5da35f2e01489dca3c06a79962fb75636985f1e3a17aec66833 \ + --hash=sha256:9483aa336687463f5497dd37a070094f3dff55e2c888994f8440fcf426a1a844 \ + --hash=sha256:97a138fa875c6f281df7720dac742259e85518135cd0e3551aba1c628103d853 \ + --hash=sha256:9b50700785eccac0819bea794d968ed8f6055c88f29364776b7ea076ac105c5d \ + --hash=sha256:9b73cf0fe28009a04a35bb2522e4c5b5176cc148919431dcb73fdbdfaab15781 \ + --hash=sha256:9d5a369eb7ec5b2fdfa9927530b5259dd21893fa75d4e04a223332f61b84b586 \ + --hash=sha256:a094b7ce455ca341b59a0f6ce6be2e11411ba6e2860b9aa3dbb37468f23338f4 \ + --hash=sha256:a0d6252098e98129a1decb59b46920d4eca17b0395f3d71b0d327d086fefe77d \ + --hash=sha256:a1d856b0f4e4a33e31cdab5f50d0a14998f3a2d726a3fd5cb7c4d45a57b28d1b \ + --hash=sha256:a4ae2ea9afcfdd2b931ddcebf1cf82532162677e00326637b31ed5dff7d985ca \ + --hash=sha256:a5963b663da69ad25fa1559ee064584935570def665917918938c1f1289f5ebc \ + --hash=sha256:ad1c2c2baaba62823a7f348f469a967ece0062140ca39e7a48e4bbb1f20d54c4 \ + --hash=sha256:ae82507fe458f7c0c8227017f2158111a4c9e7ce94de05178894a7ea9fefc8a1 \ + --hash=sha256:af188f3305f0a65c3217c30c6d4c06891e79144076a91e8b454f14256acc7279 \ + --hash=sha256:af44bb7a1af163806bbb679eb8432fa7b4fb6d83a5d403b541b675dcd3798638 \ + --hash=sha256:b0174ca6f3018ddeaa49847f29b69612e590534c1d2186d54ab25161ecc42975 \ + --hash=sha256:b2b17855ed7f994f3f259cf2dfbfad78814538536fa1a91b50253d84d87fd88d \ + --hash=sha256:b2e54f4a2dc6edf0f5ea5b1d0a608d2af3dcb5aa8c8eeab9c8841b23e1b054fe \ + --hash=sha256:b6f4abde9a2946f57e8daaf1160b2351bcf64274ef539e6675c1d945dbd75e2a \ + --hash=sha256:b70c07409d465f3a8b34d52f863871fb8a00755370791d2bd1d4f82b3cdaf3d5 \ + --hash=sha256:bb465dd5825356c1191a038a86ee1b8166e3562d6e8add95eec04ab484cfb8a2 \ + --hash=sha256:c051f46ed1e13ba8214b334cbf21902102807582fbfaf0fef341b9e52f0fafbf \ + --hash=sha256:c1b20a5f4164cec7007be55c9cf18d2cd08ed7c3bf6769b3cd6d044ad888d74b \ + --hash=sha256:c86e9e82bfab579327dbe9b82c71475165fbc8b2134d24f9a3b2edaf200a5c3d \ + --hash=sha256:c9f32b96c700bb384f33f7cf07954bb609d35dd82752cef57fb2ee0968409169 \ + --hash=sha256:cce0ed8b3f64c71c140f0ec244e5fdf8ecf78ddf8d2e591d4a8b6aa1c1214235 \ + --hash=sha256:cdd7315314b0744a7dd506f3bd0f2cf90734181529cdcf75542ee35ad885cab7 \ + --hash=sha256:cf355fbf0d4275d86f9f57be705d8e5eaa7f8ddb12b24ced2ea6cbd68fdb14dc \ + --hash=sha256:d136fbf8ad4321716e44d6d6b3d8dffb4872626010884e07a1db54b7450836cf \ + --hash=sha256:d2c8e20487b3b73c1fa72c56f5c89430617296cd380373e7af3a538a82d4cd6d \ + --hash=sha256:d483cc23cc56ab32911ea0baa0d2d9ea7aa065987f47de847a0a93a58bf57905 \ + --hash=sha256:d5a6c4864bb6fa9fcf7b57a830d21aed69fd71742a5ebcdbafda476be673d212 \ + --hash=sha256:d714e002dd3638170fe7376dc1b686dbac9cb712cde3f7224440af722cc9866a \ + --hash=sha256:d73f14b86d0e2858ece6bf5807c9889670e392c001d414b4293d0d9b291942c3 \ + --hash=sha256:d88c63bd395c787b0aa81d8bbc22c1809f311032ce3e823a6517b711129818e4 \ + --hash=sha256:db608db98ccc21248370d30044a60843b3f0f3d34781ceeea67067c508cd5a28 \ + --hash=sha256:de004939fc3fd0c1200d26ea9264350bfe501ffbf46c8cf5dc7f345f2d87a7f1 \ + --hash=sha256:ded9e86397267732a0641d4776c7c663ea16b64d7dbc4d9cc6ad8536363a2d29 \ + --hash=sha256:e288f8a162d663916060beb5e8165a8551312b08efee9cf68302687471a6545d \ + --hash=sha256:e2a9e62647dc040a76d55563580bf3bb8fe1f5b6ead08447c2ed0d7786e5e794 \ + --hash=sha256:e3e44d08b61de0dd6f205528498f834a51a5c06689f8fb182fe26f3a3ce7dca9 \ + --hash=sha256:ea002088d5554fd75e619742cefc78b84a212ba21632e59931b3501f0cfc8f67 \ + --hash=sha256:eb7452849f6615871eabed6560ffedfe56bc8af31a823b6be4ce1e6ff0ab72c5 \ + --hash=sha256:ebcf34b69df4ca0eabaaaf4a3d890f637f355fed00ba806f7ebdd2d040658c26 \ + --hash=sha256:f24d5b9383318cbd1a5cd969377937d66cf0542f24aa728a4f49d9f98f9c0da8 \ + --hash=sha256:f33fbf96b52d51c23b6cff61f57816539c1c147db270cfc1cc3bc012f4a560a9 + # via vllm shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de @@ -4525,11 +4639,10 @@ traitlets==5.14.3 \ # nbconvert # nbformat # notebook -transformers==4.53.2 \ - --hash=sha256:6c3ed95edfb1cba71c4245758f1b4878c93bf8cde77d076307dacb2cbbd72be2 \ - --hash=sha256:db8f4819bb34f000029c73c3c557e7d06fc1b8e612ec142eecdae3947a9c78bf +transformers==4.55.2 \ + --hash=sha256:097e3c2e2c0c9681db3da9d748d8f9d6a724c644514673d0030e8c5a1109f1f1 \ + --hash=sha256:a45ec60c03474fd67adbce5c434685051b7608b3f4f167c25aa6aeb1cad16d4f # via - # -r python/requirements/llm/llm-requirements.txt # compressed-tensors # vllm # xgrammar @@ -4575,7 +4688,14 @@ typing-extensions==4.12.2 \ # referencing # torch # typer + # typing-inspection # vllm +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/deplocks/llm/ray_test_py311_cu121.lock + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 @@ -4663,9 +4783,9 @@ virtualenv==20.29.1 \ # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt -vllm==0.10.0 \ - --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ - --hash=sha256:a44e9013db26082a82c3931ed8772ac884d6d60566d36ecdb0e8dc01c65b241a +vllm==0.10.1.1 \ + --hash=sha256:3099824ee4bdaa14c4c4f7178a092101a0ec206d4c9371edf295849b2b730a39 \ + --hash=sha256:8ca0dd985e1ceac8540e7719c654f1553b3ba8a43c685ac8d3fa1366ffb6443a # via -r python/requirements/llm/llm-requirements.txt watchfiles==0.19.0 \ --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ diff --git a/python/deplocks/llm/rayllm_test_py311_cu128.lock b/python/deplocks/llm/rayllm_test_py311_cu128.lock index bea79b3cf3b6..713e00a75f78 100644 --- a/python/deplocks/llm/rayllm_test_py311_cu128.lock +++ b/python/deplocks/llm/rayllm_test_py311_cu128.lock @@ -1373,7 +1373,7 @@ hf-transfer==0.1.9 \ --hash=sha256:ee8b10afedcb75f71091bcc197c526a6ebf5c58bbbadb34fdeee6160f55f619f \ --hash=sha256:fc6bd19e1cc177c66bdef15ef8636ad3bde79d5a4f608c158021153b4573509d # via -r python/requirements/llm/llm-requirements.txt -hf-xet==1.1.3 \ +hf-xet==1.1.3 ; platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:30c575a5306f8e6fda37edb866762140a435037365eba7a17ce7bd0bc0216a8b \ --hash=sha256:7c1a6aa6abed1f696f8099aa9796ca04c9ee778a58728a115607de9cc4638ff1 \ --hash=sha256:8203f52827e3df65981984936654a5b390566336956f65765a8aa58c362bb841 \ @@ -1451,7 +1451,6 @@ huggingface-hub==0.34.3 \ # via # tokenizers # transformers - # vllm humanize==4.12.1 \ --hash=sha256:1338ba97415c96556758a6e2f65977ed406dddf4620d4c6db9bbdfd07f0f1232 \ --hash=sha256:86014ca5c52675dffa1d404491952f1f5bf03b07c175a51891a343daebf01fea @@ -2520,9 +2519,25 @@ oauth2client==4.1.3 \ # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt -openai==1.90.0 \ - --hash=sha256:9771982cdd5b6631af68c6a603da72ed44cd2caf73b49f717a72b71374bc565b \ - --hash=sha256:e5dcb5498ea6b42fec47546d10f1bcc05fb854219a7d953a5ba766718b212a02 +openai==1.100.2 \ + --hash=sha256:54d3457b2c8d7303a1bc002a058de46bdd8f37a8117751c7cf4ed4438051f151 \ + --hash=sha256:787b4c3c8a65895182c58c424f790c25c790cc9a0330e34f73d55b6ee5a00e32 + # via vllm +openai-harmony==0.0.4 \ + --hash=sha256:038f1d6772d1be5213b36ae76e5d042022395ec35c428a73ccb8b839b2cecf6a \ + --hash=sha256:15e6d53a66502491a3675a536df30e271f976e6c5efe68250a65191efcb85c4f \ + --hash=sha256:2d8d16d84702059833fb03b841b28c25600c54e83cadccef79af44e1c81166b1 \ + --hash=sha256:31e9bcac0902a309e2fc688e52f247eec7fffcd00d17e958b9a83a8fea6519c2 \ + --hash=sha256:3586d90c899cd41f8624e7b82a48c289f6e4be56c66304ecaf3a0ba88963a73f \ + --hash=sha256:3cf2344366f10981bbc0f6d9949a0b2bb87151d209ed295943ed6ad8eda37932 \ + --hash=sha256:567cc568b6bf7b4d041b0c9aa7d6b2c9394f8af6065bc87fa6d23f207b5af9a7 \ + --hash=sha256:5c67ac6df349236fb7b64f57c3dbb0273efcdca24314daa108f2a482c427106c \ + --hash=sha256:746f751de5033b3dbcfcd4a726a4c56ce452c593ad3d54472d8597ce8d8b6d44 \ + --hash=sha256:96a63199c0d81095b5d5d1ae8ca82b64c1c13d18d4e30323ae9e8ab31bc80a3d \ + --hash=sha256:97f1fe3909733212cc6b36f0f199b1421a9c57b79ec665f0322bd604cec47340 \ + --hash=sha256:b9ee9e9ab6a237cebbe16563c787a6e83f3fcc034075c3d321dab94448426282 \ + --hash=sha256:d38f2639f6bf7c3c34a5dfd79e29075811ae2fa9b895a63e76767f74a47a971e \ + --hash=sha256:ef21a1e2384a65c62d5ec5e1cded9fe026f1d032d5c5d725110d1a8d330d8f54 # via vllm opencensus==0.11.4 \ --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ @@ -3274,9 +3289,9 @@ pycurl==7.45.3 \ # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt @@ -3285,110 +3300,110 @@ pydantic==2.10.0 \ # lm-format-enforcer # mistral-common # openai + # openai-harmony # pydantic-extra-types # vllm # xgrammar -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # pydantic @@ -4054,6 +4069,105 @@ sentencepiece==0.2.0 \ # gguf # mistral-common # vllm +setproctitle==1.3.6 \ + --hash=sha256:082413db8a96b1f021088e8ec23f0a61fec352e649aba20881895815388b66d3 \ + --hash=sha256:0dba8faee2e4a96e934797c9f0f2d093f8239bf210406a99060b3eabe549628e \ + --hash=sha256:0e6b5633c94c5111f7137f875e8f1ff48f53b991d5d5b90932f27dc8c1fa9ae4 \ + --hash=sha256:1065ed36bd03a3fd4186d6c6de5f19846650b015789f72e2dea2d77be99bdca1 \ + --hash=sha256:109fc07b1cd6cef9c245b2028e3e98e038283342b220def311d0239179810dbe \ + --hash=sha256:13624d9925bb481bc0ccfbc7f533da38bfbfe6e80652314f789abc78c2e513bd \ + --hash=sha256:156795b3db976611d09252fc80761fcdb65bb7c9b9581148da900851af25ecf4 \ + --hash=sha256:163dba68f979c61e4e2e779c4d643e968973bdae7c33c3ec4d1869f7a9ba8390 \ + --hash=sha256:17d7c833ed6545ada5ac4bb606b86a28f13a04431953d4beac29d3773aa00b1d \ + --hash=sha256:18d0667bafaaae4c1dee831e2e59841c411ff399b9b4766822ba2685d419c3be \ + --hash=sha256:1aa1935aa2195b76f377e5cb018290376b7bf085f0b53f5a95c0c21011b74367 \ + --hash=sha256:2156d55308431ac3b3ec4e5e05b1726d11a5215352d6a22bb933171dee292f8c \ + --hash=sha256:23a57d3b8f1549515c2dbe4a2880ebc1f27780dc126c5e064167563e015817f5 \ + --hash=sha256:2407955dc359d735a20ac6e797ad160feb33d529a2ac50695c11a1ec680eafab \ + --hash=sha256:2940cf13f4fc11ce69ad2ed37a9f22386bfed314b98d8aebfd4f55459aa59108 \ + --hash=sha256:2e51ec673513465663008ce402171192a053564865c2fc6dc840620871a9bd7c \ + --hash=sha256:3393859eb8f19f5804049a685bf286cb08d447e28ba5c6d8543c7bf5500d5970 \ + --hash=sha256:3884002b3a9086f3018a32ab5d4e1e8214dd70695004e27b1a45c25a6243ad0b \ + --hash=sha256:38ca045626af693da042ac35d7332e7b9dbd52e6351d6973b310612e3acee6d6 \ + --hash=sha256:391bb6a29c4fe7ccc9c30812e3744060802d89b39264cfa77f3d280d7f387ea5 \ + --hash=sha256:3cca16fd055316a48f0debfcbfb6af7cea715429fc31515ab3fcac05abd527d8 \ + --hash=sha256:3cde5b83ec4915cd5e6ae271937fd60d14113c8f7769b4a20d51769fe70d8717 \ + --hash=sha256:3f8194b4d631b003a1176a75d1acd545e04b1f54b821638e098a93e6e62830ef \ + --hash=sha256:3fc97805f9d74444b027babff710bf39df1541437a6a585a983d090ae00cedde \ + --hash=sha256:4431629c178193f23c538cb1de3da285a99ccc86b20ee91d81eb5f1a80e0d2ba \ + --hash=sha256:49498ebf68ca3e75321ffe634fcea5cc720502bfaa79bd6b03ded92ce0dc3c24 \ + --hash=sha256:4ac3eb04bcf0119aadc6235a2c162bae5ed5f740e3d42273a7228b915722de20 \ + --hash=sha256:4adf6a0013fe4e0844e3ba7583ec203ca518b9394c6cc0d3354df2bf31d1c034 \ + --hash=sha256:4efc91b437f6ff2578e89e3f17d010c0a0ff01736606473d082913ecaf7859ba \ + --hash=sha256:50706b9c0eda55f7de18695bfeead5f28b58aa42fd5219b3b1692d554ecbc9ec \ + --hash=sha256:5313a4e9380e46ca0e2c681ba739296f9e7c899e6f4d12a6702b2dc9fb846a31 \ + --hash=sha256:543f59601a4e32daf44741b52f9a23e0ee374f9f13b39c41d917302d98fdd7b0 \ + --hash=sha256:57bc54763bf741813a99fbde91f6be138c8706148b7b42d3752deec46545d470 \ + --hash=sha256:63cc10352dc6cf35a33951656aa660d99f25f574eb78132ce41a85001a638aa7 \ + --hash=sha256:6a1d3aa13acfe81f355b0ce4968facc7a19b0d17223a0f80c011a1dba8388f37 \ + --hash=sha256:6af330ddc2ec05a99c3933ab3cba9365357c0b8470a7f2fa054ee4b0984f57d1 \ + --hash=sha256:6d50bfcc1d1692dc55165b3dd2f0b9f8fb5b1f7b571a93e08d660ad54b9ca1a5 \ + --hash=sha256:70100e2087fe05359f249a0b5f393127b3a1819bf34dec3a3e0d4941138650c9 \ + --hash=sha256:74973aebea3543ad033b9103db30579ec2b950a466e09f9c2180089e8346e0ec \ + --hash=sha256:751ba352ed922e0af60458e961167fa7b732ac31c0ddd1476a2dfd30ab5958c5 \ + --hash=sha256:785cd210c0311d9be28a70e281a914486d62bfd44ac926fcd70cf0b4d65dff1c \ + --hash=sha256:7890e291bf4708e3b61db9069ea39b3ab0651e42923a5e1f4d78a7b9e4b18301 \ + --hash=sha256:793a23e8d9cb6c231aa3023d700008224c6ec5b8fd622d50f3c51665e3d0a190 \ + --hash=sha256:797f2846b546a8741413c57d9fb930ad5aa939d925c9c0fa6186d77580035af7 \ + --hash=sha256:7df5fcc48588f82b6cc8073db069609ddd48a49b1e9734a20d0efb32464753c4 \ + --hash=sha256:8050c01331135f77ec99d99307bfbc6519ea24d2f92964b06f3222a804a3ff1f \ + --hash=sha256:805bb33e92fc3d8aa05674db3068d14d36718e3f2c5c79b09807203f229bf4b5 \ + --hash=sha256:807796fe301b7ed76cf100113cc008c119daf4fea2f9f43c578002aef70c3ebf \ + --hash=sha256:81c443310831e29fabbd07b75ebbfa29d0740b56f5907c6af218482d51260431 \ + --hash=sha256:83066ffbf77a5f82b7e96e59bdccbdda203c8dccbfc3f9f0fdad3a08d0001d9c \ + --hash=sha256:8834ab7be6539f1bfadec7c8d12249bbbe6c2413b1d40ffc0ec408692232a0c6 \ + --hash=sha256:92df0e70b884f5da35f2e01489dca3c06a79962fb75636985f1e3a17aec66833 \ + --hash=sha256:9483aa336687463f5497dd37a070094f3dff55e2c888994f8440fcf426a1a844 \ + --hash=sha256:97a138fa875c6f281df7720dac742259e85518135cd0e3551aba1c628103d853 \ + --hash=sha256:9b50700785eccac0819bea794d968ed8f6055c88f29364776b7ea076ac105c5d \ + --hash=sha256:9b73cf0fe28009a04a35bb2522e4c5b5176cc148919431dcb73fdbdfaab15781 \ + --hash=sha256:9d5a369eb7ec5b2fdfa9927530b5259dd21893fa75d4e04a223332f61b84b586 \ + --hash=sha256:a094b7ce455ca341b59a0f6ce6be2e11411ba6e2860b9aa3dbb37468f23338f4 \ + --hash=sha256:a0d6252098e98129a1decb59b46920d4eca17b0395f3d71b0d327d086fefe77d \ + --hash=sha256:a1d856b0f4e4a33e31cdab5f50d0a14998f3a2d726a3fd5cb7c4d45a57b28d1b \ + --hash=sha256:a4ae2ea9afcfdd2b931ddcebf1cf82532162677e00326637b31ed5dff7d985ca \ + --hash=sha256:a5963b663da69ad25fa1559ee064584935570def665917918938c1f1289f5ebc \ + --hash=sha256:ad1c2c2baaba62823a7f348f469a967ece0062140ca39e7a48e4bbb1f20d54c4 \ + --hash=sha256:ae82507fe458f7c0c8227017f2158111a4c9e7ce94de05178894a7ea9fefc8a1 \ + --hash=sha256:af188f3305f0a65c3217c30c6d4c06891e79144076a91e8b454f14256acc7279 \ + --hash=sha256:af44bb7a1af163806bbb679eb8432fa7b4fb6d83a5d403b541b675dcd3798638 \ + --hash=sha256:b0174ca6f3018ddeaa49847f29b69612e590534c1d2186d54ab25161ecc42975 \ + --hash=sha256:b2b17855ed7f994f3f259cf2dfbfad78814538536fa1a91b50253d84d87fd88d \ + --hash=sha256:b2e54f4a2dc6edf0f5ea5b1d0a608d2af3dcb5aa8c8eeab9c8841b23e1b054fe \ + --hash=sha256:b6f4abde9a2946f57e8daaf1160b2351bcf64274ef539e6675c1d945dbd75e2a \ + --hash=sha256:b70c07409d465f3a8b34d52f863871fb8a00755370791d2bd1d4f82b3cdaf3d5 \ + --hash=sha256:bb465dd5825356c1191a038a86ee1b8166e3562d6e8add95eec04ab484cfb8a2 \ + --hash=sha256:c051f46ed1e13ba8214b334cbf21902102807582fbfaf0fef341b9e52f0fafbf \ + --hash=sha256:c1b20a5f4164cec7007be55c9cf18d2cd08ed7c3bf6769b3cd6d044ad888d74b \ + --hash=sha256:c86e9e82bfab579327dbe9b82c71475165fbc8b2134d24f9a3b2edaf200a5c3d \ + --hash=sha256:c9f32b96c700bb384f33f7cf07954bb609d35dd82752cef57fb2ee0968409169 \ + --hash=sha256:cce0ed8b3f64c71c140f0ec244e5fdf8ecf78ddf8d2e591d4a8b6aa1c1214235 \ + --hash=sha256:cdd7315314b0744a7dd506f3bd0f2cf90734181529cdcf75542ee35ad885cab7 \ + --hash=sha256:cf355fbf0d4275d86f9f57be705d8e5eaa7f8ddb12b24ced2ea6cbd68fdb14dc \ + --hash=sha256:d136fbf8ad4321716e44d6d6b3d8dffb4872626010884e07a1db54b7450836cf \ + --hash=sha256:d2c8e20487b3b73c1fa72c56f5c89430617296cd380373e7af3a538a82d4cd6d \ + --hash=sha256:d483cc23cc56ab32911ea0baa0d2d9ea7aa065987f47de847a0a93a58bf57905 \ + --hash=sha256:d5a6c4864bb6fa9fcf7b57a830d21aed69fd71742a5ebcdbafda476be673d212 \ + --hash=sha256:d714e002dd3638170fe7376dc1b686dbac9cb712cde3f7224440af722cc9866a \ + --hash=sha256:d73f14b86d0e2858ece6bf5807c9889670e392c001d414b4293d0d9b291942c3 \ + --hash=sha256:d88c63bd395c787b0aa81d8bbc22c1809f311032ce3e823a6517b711129818e4 \ + --hash=sha256:db608db98ccc21248370d30044a60843b3f0f3d34781ceeea67067c508cd5a28 \ + --hash=sha256:de004939fc3fd0c1200d26ea9264350bfe501ffbf46c8cf5dc7f345f2d87a7f1 \ + --hash=sha256:ded9e86397267732a0641d4776c7c663ea16b64d7dbc4d9cc6ad8536363a2d29 \ + --hash=sha256:e288f8a162d663916060beb5e8165a8551312b08efee9cf68302687471a6545d \ + --hash=sha256:e2a9e62647dc040a76d55563580bf3bb8fe1f5b6ead08447c2ed0d7786e5e794 \ + --hash=sha256:e3e44d08b61de0dd6f205528498f834a51a5c06689f8fb182fe26f3a3ce7dca9 \ + --hash=sha256:ea002088d5554fd75e619742cefc78b84a212ba21632e59931b3501f0cfc8f67 \ + --hash=sha256:eb7452849f6615871eabed6560ffedfe56bc8af31a823b6be4ce1e6ff0ab72c5 \ + --hash=sha256:ebcf34b69df4ca0eabaaaf4a3d890f637f355fed00ba806f7ebdd2d040658c26 \ + --hash=sha256:f24d5b9383318cbd1a5cd969377937d66cf0542f24aa728a4f49d9f98f9c0da8 \ + --hash=sha256:f33fbf96b52d51c23b6cff61f57816539c1c147db270cfc1cc3bc012f4a560a9 + # via vllm shellingham==1.5.4 \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de @@ -4418,11 +4532,10 @@ traitlets==5.14.3 \ # nbconvert # nbformat # notebook -transformers==4.53.2 \ - --hash=sha256:6c3ed95edfb1cba71c4245758f1b4878c93bf8cde77d076307dacb2cbbd72be2 \ - --hash=sha256:db8f4819bb34f000029c73c3c557e7d06fc1b8e612ec142eecdae3947a9c78bf +transformers==4.55.2 \ + --hash=sha256:097e3c2e2c0c9681db3da9d748d8f9d6a724c644514673d0030e8c5a1109f1f1 \ + --hash=sha256:a45ec60c03474fd67adbce5c434685051b7608b3f4f167c25aa6aeb1cad16d4f # via - # -r python/requirements/llm/llm-requirements.txt # compressed-tensors # vllm # xgrammar @@ -4467,7 +4580,14 @@ typing-extensions==4.12.2 \ # referencing # torch # typer + # typing-inspection # vllm +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/deplocks/llm/ray_test_py311_cu128.lock + # pydantic tzdata==2025.2 \ --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 @@ -4555,9 +4675,9 @@ virtualenv==20.29.1 \ # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt -vllm==0.10.0 \ - --hash=sha256:8ca37559d82b43b5e8c8248d2e4a1ecb51d6d4e5d517491d656df6491ed93dab \ - --hash=sha256:a44e9013db26082a82c3931ed8772ac884d6d60566d36ecdb0e8dc01c65b241a +vllm==0.10.1.1 \ + --hash=sha256:3099824ee4bdaa14c4c4f7178a092101a0ec206d4c9371edf295849b2b730a39 \ + --hash=sha256:8ca0dd985e1ceac8540e7719c654f1553b3ba8a43c685ac8d3fa1366ffb6443a # via -r python/requirements/llm/llm-requirements.txt watchfiles==0.19.0 \ --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ diff --git a/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py b/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py index 092622d70bea..18606714869f 100644 --- a/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py +++ b/python/ray/llm/_internal/batch/stages/vllm_engine_stage.py @@ -11,6 +11,7 @@ from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Type import numpy as np +import torch from pydantic import BaseModel, Field, root_validator import ray @@ -109,6 +110,11 @@ def from_vllm_engine_output(cls, output: Any) -> "vLLMOutputData": data.num_generated_tokens = len(output.outputs[0].token_ids) elif isinstance(output, vllm.outputs.PoolingRequestOutput): data.embeddings = output.outputs.data.cpu() + if ( + isinstance(data.embeddings, torch.Tensor) + and data.embeddings.dtype == torch.bfloat16 + ): + data.embeddings = data.embeddings.to(torch.float32) else: raise ValueError(f"Unknown output type: {type(output)}") @@ -487,7 +493,7 @@ def __init__( model=self.model, model_source=model_source, idx_in_batch_column=self.IDX_IN_BATCH_COLUMN, - disable_log_requests=True, + enable_log_requests=False, max_pending_requests=self.max_pending_requests, dynamic_lora_loading_path=dynamic_lora_loading_path, **self.engine_kwargs, diff --git a/python/ray/llm/_internal/serve/configs/openai_api_models.py b/python/ray/llm/_internal/serve/configs/openai_api_models.py index 2d118ab4742e..78d4f4687e25 100644 --- a/python/ray/llm/_internal/serve/configs/openai_api_models.py +++ b/python/ray/llm/_internal/serve/configs/openai_api_models.py @@ -21,6 +21,7 @@ EmbeddingChatRequest as vLLMEmbeddingChatRequest, EmbeddingCompletionRequest as vLLMEmbeddingCompletionRequest, EmbeddingResponse as vLLMEmbeddingResponse, + ErrorInfo as vLLMErrorInfo, ErrorResponse as vLLMErrorResponse, ScoreRequest as vLLMScoreRequest, ScoreResponse as vLLMScoreResponse, @@ -43,6 +44,10 @@ class ChatCompletionStreamResponse(vLLMChatCompletionStreamResponse): model_config = ConfigDict(arbitrary_types_allowed=True) +class ErrorInfo(vLLMErrorInfo): + model_config = ConfigDict(arbitrary_types_allowed=True) + + class ErrorResponse(vLLMErrorResponse): model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py index b1e9457c8e74..d8eeea45801a 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py @@ -17,6 +17,7 @@ CompletionResponse, EmbeddingRequest, EmbeddingResponse, + ErrorInfo, ErrorResponse, ScoreRequest, ScoreResponse, @@ -355,7 +356,7 @@ async def resolve_lora(self, disk_lora_model: DiskMultiplexConfig): ) if isinstance(lora_request, VLLMErrorResponse): - raise ValueError(f"Failed to load lora model: {lora_request.message}") + raise ValueError(f"Failed to load lora model: {lora_request.error.message}") def _create_raw_request( self, @@ -397,7 +398,7 @@ async def chat( yield response else: if isinstance(chat_response, VLLMErrorResponse): - yield ErrorResponse(**chat_response.model_dump()) + yield ErrorResponse(error=ErrorInfo(**chat_response.error.model_dump())) else: yield ChatCompletionResponse(**chat_response.model_dump()) @@ -426,7 +427,9 @@ async def completions( yield response else: if isinstance(completion_response, VLLMErrorResponse): - yield ErrorResponse(**completion_response.model_dump()) + yield ErrorResponse( + error=ErrorInfo(**completion_response.error.model_dump()) + ) else: yield CompletionResponse(**completion_response.model_dump()) @@ -445,7 +448,9 @@ async def embeddings( ) if isinstance(embedding_response, VLLMErrorResponse): - yield ErrorResponse(**embedding_response.model_dump()) + yield ErrorResponse( + error=ErrorInfo(**embedding_response.error.model_dump()) + ) else: yield EmbeddingResponse(**embedding_response.model_dump()) diff --git a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py index 36a5444fc564..610205de86c4 100644 --- a/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py +++ b/python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_models.py @@ -93,26 +93,17 @@ def get_initialization_kwargs(self) -> dict: else: engine_kwargs["distributed_executor_backend"] = "ray" - # TODO(lk-chen): Remove the logic once we require vllm>=0.10.1 - # vLLM 0.10.1 replaces `disable_log_requests` with - # `enable_log_requests`. Here we are trying to be compatible with both. - if hasattr(AsyncEngineArgs, "enable_log_requests"): - if "disable_log_requests" in engine_kwargs: - logger.warning( - "disable_log_requests is set in engine_kwargs, but vLLM " - "does not support it. Converting to enable_log_requests." - ) - engine_kwargs["enable_log_requests"] = not engine_kwargs.pop( - "disable_log_requests" - ) - else: - engine_kwargs["enable_log_requests"] = False - elif "disable_log_requests" not in engine_kwargs: - logger.info( - "Disabling request logging by default. To enable, set to False" - " in engine_kwargs." + # TODO (Nikhil): Remove this once vLLM fully deprecates disable_log_requests. + if "disable_log_requests" in engine_kwargs: + logger.warning( + "disable_log_requests is set in engine_kwargs, but vLLM " + "does not support it. Converting to enable_log_requests." + ) + engine_kwargs["enable_log_requests"] = not engine_kwargs.pop( + "disable_log_requests" ) - engine_kwargs["disable_log_requests"] = True + elif "enable_log_requests" not in engine_kwargs: + engine_kwargs["enable_log_requests"] = False return engine_kwargs diff --git a/python/ray/llm/_internal/serve/deployments/routers/middleware.py b/python/ray/llm/_internal/serve/deployments/routers/middleware.py index 961e199332ff..6a2588d2a4dd 100644 --- a/python/ray/llm/_internal/serve/deployments/routers/middleware.py +++ b/python/ray/llm/_internal/serve/deployments/routers/middleware.py @@ -67,10 +67,10 @@ def _uncaught_exception_handler(request: Request, e: Exception): logger.error(f"Uncaught exception while handling request {request_id}", exc_info=e) - response_payload = get_response_for_error(e, request_id) + error_response = get_response_for_error(e, request_id) return JSONResponse( - content=response_payload.model_dump(), status_code=response_payload.code + content=error_response.model_dump(), status_code=error_response.error.code ) @@ -111,11 +111,11 @@ async def _handle_application_exceptions( return await _handle_validation_error(request, e) except Exception as e: request_id = get_request_id(request) - response_payload = get_response_for_error(e, request_id) + error_response = get_response_for_error(e, request_id) return JSONResponse( - content=response_payload.model_dump(), - status_code=response_payload.code, + content=error_response.model_dump(), + status_code=error_response.error.code, ) # This adds last-resort uncaught exception handler into Starlette diff --git a/python/ray/llm/_internal/serve/deployments/routers/router.py b/python/ray/llm/_internal/serve/deployments/routers/router.py index 240fff0184e7..5a859bc5e100 100644 --- a/python/ray/llm/_internal/serve/deployments/routers/router.py +++ b/python/ray/llm/_internal/serve/deployments/routers/router.py @@ -446,9 +446,9 @@ async def _process_llm_request( if isinstance(first_chunk, ErrorResponse): raise OpenAIHTTPException( - message=first_chunk.message, - status_code=first_chunk.code, - type=first_chunk.type, + message=first_chunk.error.message, + status_code=first_chunk.error.code, + type=first_chunk.error.type, ) if isinstance(first_chunk, NoneStreamingResponseType): @@ -495,9 +495,9 @@ async def embeddings(self, body: EmbeddingRequest) -> Response: result = await results.__anext__() if isinstance(result, ErrorResponse): raise OpenAIHTTPException( - message=result.message, - status_code=result.code, - type=result.type, + message=result.error.message, + status_code=result.error.code, + type=result.error.type, ) if isinstance(result, EmbeddingResponse): diff --git a/python/ray/llm/_internal/serve/deployments/utils/server_utils.py b/python/ray/llm/_internal/serve/deployments/utils/server_utils.py index 121b5ea68118..4d490ec329f2 100644 --- a/python/ray/llm/_internal/serve/deployments/utils/server_utils.py +++ b/python/ray/llm/_internal/serve/deployments/utils/server_utils.py @@ -9,6 +9,7 @@ from ray import serve from ray.llm._internal.serve.configs.openai_api_models import ( + ErrorInfo, ErrorResponse, OpenAIHTTPException, ) @@ -110,11 +111,12 @@ def get_response_for_error( if "(Request ID: " not in internal_message: internal_message += f" (Request ID: {request_id})" - error_response = ErrorResponse( + error_info = ErrorInfo( message=f"Message: {message}, Internal exception: {internal_message}, original exception: {str(e)}", code=status_code, type=exc_type, ) + error_response = ErrorResponse(error=error_info) return error_response diff --git a/python/ray/llm/tests/batch/gpu/stages/test_vllm_engine_stage.py b/python/ray/llm/tests/batch/gpu/stages/test_vllm_engine_stage.py index d05805f13eea..71971caf62e1 100644 --- a/python/ray/llm/tests/batch/gpu/stages/test_vllm_engine_stage.py +++ b/python/ray/llm/tests/batch/gpu/stages/test_vllm_engine_stage.py @@ -171,7 +171,7 @@ async def test_vllm_engine_udf_basic(mock_vllm_wrapper, model_llama_3_2_216M): task=vLLMTaskType.GENERATE, max_num_seqs=100, dynamic_lora_loading_path=None, - disable_log_requests=True, + enable_log_requests=False, ) diff --git a/python/requirements/llm/llm-requirements.txt b/python/requirements/llm/llm-requirements.txt index d8eb58a1fd7d..dde9a0e38905 100644 --- a/python/requirements/llm/llm-requirements.txt +++ b/python/requirements/llm/llm-requirements.txt @@ -2,7 +2,7 @@ # constraining to a maximum version (i.e. <=) to temporarily work around a bug. # Those pins for the sake of workarounds should not be advertised as constraints # on future releases in setup.py. -vllm>=0.10.0 +vllm>=0.10.1.1 # For json mode jsonref>=1.1.0 jsonschema @@ -13,7 +13,6 @@ typer meson pybind11 hf_transfer -transformers<4.54.0 # Due to https://github.com/vllm-project/vllm-ascend/issues/2046 # nixl version Needs to be in sync with the one in ray-llm/Dockerfile nixl==0.4.1 diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt index 03fbe19cc80f..466bf856d272 100644 --- a/python/requirements_compiled.txt +++ b/python/requirements_compiled.txt @@ -1619,7 +1619,7 @@ pycparser==2.21 # via cffi pycurl==7.45.3 # via -r python/requirements/cloud-requirements.txt -pydantic==2.10.0 +pydantic==2.11.7 # via # -r python/requirements.txt # -r python/requirements/test-requirements.txt @@ -1629,7 +1629,7 @@ pydantic==2.10.0 # gradio # mlflow-skinny # pyiceberg -pydantic-core==2.27.0 +pydantic-core==2.33.2 # via pydantic pydot==1.4.2 # via -r python/requirements/test-requirements.txt @@ -2416,6 +2416,9 @@ typing-extensions==4.12.2 # tensorflow # torch # typer + # typing-inspection +typing-inspection==0.4.1 + # via pydantic tzdata==2025.2 # via kombu tzlocal==5.3 diff --git a/python/setup.py b/python/setup.py index 7f65cf5ef2cd..d4770cd2f8c8 100644 --- a/python/setup.py +++ b/python/setup.py @@ -372,7 +372,7 @@ def get_packages(self): setup_spec.extras["llm"] = list( set( [ - "vllm>=0.10.0", + "vllm>=0.10.1.1", "jsonref>=1.1.0", "jsonschema", "ninja", diff --git a/release/release_tests.yaml b/release/release_tests.yaml index 1c7932f46d7d..110ebf61b755 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -4598,7 +4598,7 @@ script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_quantized_tp1_2p6d.yaml --skip-hf-token true - name: llm_serve_llama_3dot1_8B_quantized_tp1_2p6d_lmcache - frequency: nightly + frequency: manual # todo(ray-llm): fix this test with new/old lmcache version and new vllm version and re-enable it. python: "3.11" group: llm-serve team: llm From 91d9ac65d92d05b2791a6c2090c8054fd930bb4d Mon Sep 17 00:00:00 2001 From: Ricardo Decal Date: Fri, 5 Sep 2025 10:33:59 -0700 Subject: [PATCH 471/634] [CI] Add text embed benchmark tests to group: data-tests. (#56256) The text embedding benchmarks are currently ungrouped in buildkite. This PR adds them to the Data group Signed-off-by: Ricardo Decal --- release/release_tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release/release_tests.yaml b/release/release_tests.yaml index 110ebf61b755..e4ff69657d36 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -4666,6 +4666,7 @@ python: "3.11" # necessary for the llm-cu128 image working_dir: nightly_tests team: data + group: data-tests cluster: byod: @@ -4688,6 +4689,7 @@ python: "3.11" working_dir: nightly_tests team: data + group: data-tests cluster: byod: From 669c9385a1dcdeb640a48e51cb715a41864c2a7a Mon Sep 17 00:00:00 2001 From: Mengjin Yan Date: Fri, 5 Sep 2025 10:44:22 -0700 Subject: [PATCH 472/634] [Core] [TaskEvent] Fix Missing Events Issue in Task Events (#55916) Signed-off-by: Mengjin Yan --- src/ray/core_worker/task_event_buffer.cc | 39 ++-- src/ray/core_worker/task_event_buffer.h | 33 ++-- .../tests/task_event_buffer_test.cc | 171 +++++++++++------- 3 files changed, 143 insertions(+), 100 deletions(-) diff --git a/src/ray/core_worker/task_event_buffer.cc b/src/ray/core_worker/task_event_buffer.cc index 9187764b2768..f6f632b581c1 100644 --- a/src/ray/core_worker/task_event_buffer.cc +++ b/src/ray/core_worker/task_event_buffer.cc @@ -279,8 +279,9 @@ void TaskStatusEvent::PopulateRpcRayEventBaseFields( } } -void TaskStatusEvent::ToRpcRayEvents(RayEventsPair &ray_events_pair) { - auto &[task_definition_event_rpc, task_execution_event_rpc] = ray_events_pair; +void TaskStatusEvent::ToRpcRayEvents(RayEventsTuple &ray_events_tuple) { + auto &[task_definition_event_rpc, task_execution_event_rpc, task_profile_event_rpc] = + ray_events_tuple; google::protobuf::Timestamp timestamp = AbslTimeNanosToProtoTimestamp(timestamp_); @@ -356,16 +357,15 @@ void TaskProfileEvent::PopulateRpcRayEventBaseFields( ray_event.set_session_name(session_name_); } -void TaskProfileEvent::ToRpcRayEvents(RayEventsPair &ray_events_pair) { - auto &[task_profile_Event, null_event] = ray_events_pair; - // Second element of the RayEventsPair will always be empty for TaskProfileEvent - null_event = std::nullopt; +void TaskProfileEvent::ToRpcRayEvents(RayEventsTuple &ray_events_tuple) { + auto &[task_definition_event, task_execution_event, task_profile_event] = + ray_events_tuple; // Using profile start time as the event generation timestamp google::protobuf::Timestamp timestamp = AbslTimeNanosToProtoTimestamp(start_time_); // Populate Ray event base fields - auto &ray_event = task_profile_Event.emplace(); + auto &ray_event = task_profile_event.emplace(); PopulateRpcRayEventBaseFields(ray_event, timestamp); // Populate the task profile event @@ -622,21 +622,24 @@ std::unique_ptr TaskEventBufferImpl::CreateTaskEventDataToSe std::unique_ptr TaskEventBufferImpl::CreateRayEventsDataToSend( - absl::flat_hash_map &&agg_task_events, + absl::flat_hash_map &&agg_task_events, const absl::flat_hash_set &dropped_task_attempts_to_send) { auto data = std::make_unique(); // Move the ray events. - for (auto &[task_attempt, ray_events_pair] : agg_task_events) { - // For TaskStatusEvent: first = task definition event, second = task execution event - // For TaskProfileEvent: first = task profile event, second = nullopt (empty) - auto &[first_event, second_event] = ray_events_pair; - if (first_event) { + for (auto &[task_attempt, ray_events_tuple] : agg_task_events) { + auto &[task_definition_event, task_execution_event, task_profile_event] = + ray_events_tuple; + if (task_definition_event) { auto events = data->add_events(); - *events = std::move(first_event.value()); + *events = std::move(task_definition_event.value()); } - if (second_event) { + if (task_execution_event) { auto events = data->add_events(); - *events = std::move(second_event.value()); + *events = std::move(task_execution_event.value()); + } + if (task_profile_event) { + auto events = data->add_events(); + *events = std::move(task_profile_event.value()); } } @@ -657,8 +660,8 @@ TaskEventBuffer::TaskEventDataToSend TaskEventBufferImpl::CreateDataToSend( const absl::flat_hash_set &dropped_task_attempts_to_send) { // Aggregate the task events by TaskAttempt. absl::flat_hash_map agg_task_events; - // (task_attempt, (task_definition_event, task_execution_events)) - absl::flat_hash_map agg_ray_events; + // (task_attempt, (task_definition_event, task_execution_event, task_profile_event)) + absl::flat_hash_map agg_ray_events; auto to_rpc_event_fn = [this, &agg_task_events, &agg_ray_events, &dropped_task_attempts_to_send]( diff --git a/src/ray/core_worker/task_event_buffer.h b/src/ray/core_worker/task_event_buffer.h index 1f3d11283612..39030c3a7c0f 100644 --- a/src/ray/core_worker/task_event_buffer.h +++ b/src/ray/core_worker/task_event_buffer.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -41,13 +42,16 @@ namespace core { namespace worker { using TaskAttempt = std::pair; -/// A pair of rpc::events::RayEvent. -/// When converting the TaskStatusEvent, the pair will be populated with the -/// rpc::events::TaskDefinitionEvent and rpc::events::TaskExecutionEvent respectively. -/// When converting the TaskProfileEvent, only the first element of the pair will be -/// populated with rpc::events::TaskProfileEvents -using RayEventsPair = - std::pair, std::optional>; +/// A tuple of rpc::events::RayEvent. +/// When converting the TaskStatusEvent, the first 2 elements of the tuple will be +/// populated with rpc::events::TaskDefinitionEvent and rpc::events::TaskExecutionEvent +/// respectively. When converting the TaskProfileEvent, the last element of the tuple will +/// be populated with rpc::events::TaskProfileEvent. A tuple is needed because the +/// TaskProfileEvent, TaskDefinitionEvent and TaskExecutionEvent all can share the same +/// task_id and attempt_number. +using RayEventsTuple = std::tuple, + std::optional, + std::optional>; /// A wrapper class that will be converted to protobuf task events representation. /// @@ -84,7 +88,7 @@ class TaskEvent { /// Convert itself to a pair of RayEvent. /// /// \param[out] ray_events The pair of rpc::events::RayEvent - virtual void ToRpcRayEvents(RayEventsPair &ray_events) = 0; + virtual void ToRpcRayEvents(RayEventsTuple &ray_events_tuple) = 0; /// If it is a profile event. virtual bool IsProfileEvent() const = 0; @@ -169,9 +173,9 @@ class TaskStatusEvent : public TaskEvent { /// NOTE: this method will modify internal states by moving fields of task_spec_ to /// the rpc::events::RayEvent. /// - /// \param[out] ray_events The pair of rpc::events::RayEvent protobuf messages to be + /// \param[out] ray_events The tuple of rpc::events::RayEvent protobuf messages to be /// filled. - void ToRpcRayEvents(RayEventsPair &ray_events) override; + void ToRpcRayEvents(RayEventsTuple &ray_events_tuple) override; bool IsProfileEvent() const override { return false; } @@ -224,8 +228,8 @@ class TaskProfileEvent : public TaskEvent { std::shared_ptr rpc_task_export_event_data) override; /// Note: The extra data will be moved when this is called and will no longer be usable. - /// Second element of the RayEventsPair will always be empty for TaskProfileEvent. - void ToRpcRayEvents(RayEventsPair &ray_events) override; + /// Second element of the RayEventsTuple will always be empty for TaskProfileEvent. + void ToRpcRayEvents(RayEventsTuple &ray_events_tuple) override; bool IsProfileEvent() const override { return true; } @@ -464,7 +468,7 @@ class TaskEventBufferImpl : public TaskEventBuffer { /// status events being dropped. /// \return data The ray event data to be sent. std::unique_ptr CreateRayEventsDataToSend( - absl::flat_hash_map &&agg_task_events, + absl::flat_hash_map &&agg_task_events, const absl::flat_hash_set &dropped_task_attempts_to_send); /// Reset the metrics counters for flush. @@ -607,7 +611,8 @@ class TaskEventBufferImpl : public TaskEventBuffer { FRIEND_TEST(TaskEventBufferTestLimitProfileEvents, TestLimitProfileEventsPerTask); FRIEND_TEST(TaskEventTestWriteExport, TestWriteTaskExportEvents); FRIEND_TEST(TaskEventBufferTest, TestCreateRayEventsDataWithProfileEvents); - FRIEND_TEST(TaskEventBufferTest, TestMixedStatusAndProfileEventsToRayEvents); + FRIEND_TEST(TaskEventBufferTestDifferentDestination, + TestMixedStatusAndProfileEventsToRayEvents); }; } // namespace worker diff --git a/src/ray/core_worker/tests/task_event_buffer_test.cc b/src/ray/core_worker/tests/task_event_buffer_test.cc index 061ccdae97bf..6169b36b0176 100644 --- a/src/ray/core_worker/tests/task_event_buffer_test.cc +++ b/src/ray/core_worker/tests/task_event_buffer_test.cc @@ -433,9 +433,10 @@ TEST_P(TaskEventBufferTestDifferentDestination, TestFlushEvents) { auto event = expected_task_event_data.add_events_by_task(); task_event->ToRpcTaskEvents(event); - RayEventsPair ray_events_pair; - task_event->ToRpcRayEvents(ray_events_pair); - auto [task_definition_event, task_execution_event] = ray_events_pair; + RayEventsTuple ray_events_tuple; + task_event->ToRpcRayEvents(ray_events_tuple); + auto [task_definition_event, task_execution_event, task_profile_event] = + ray_events_tuple; if (task_definition_event) { auto new_event = expected_ray_events_data.add_events(); *new_event = std::move(task_definition_event.value()); @@ -444,6 +445,10 @@ TEST_P(TaskEventBufferTestDifferentDestination, TestFlushEvents) { auto new_event = expected_ray_events_data.add_events(); *new_event = std::move(task_execution_event.value()); } + if (task_profile_event) { + auto new_event = expected_ray_events_data.add_events(); + *new_event = std::move(task_profile_event.value()); + } } for (auto &task_event : task_events) { @@ -748,9 +753,10 @@ TEST_P(TaskEventBufferTestLimitBufferDifferentDestination, *static_cast(event_ptr.get())); event->ToRpcTaskEvents(expect_event); - RayEventsPair ray_events_pair; - event->ToRpcRayEvents(ray_events_pair); - auto [task_definition_event, task_execution_event] = ray_events_pair; + RayEventsTuple ray_events_tuple; + event->ToRpcRayEvents(ray_events_tuple); + auto [task_definition_event, task_execution_event, task_profile_event] = + ray_events_tuple; if (task_definition_event) { auto new_event = expected_ray_events_data.add_events(); *new_event = std::move(task_definition_event.value()); @@ -759,6 +765,10 @@ TEST_P(TaskEventBufferTestLimitBufferDifferentDestination, auto new_event = expected_ray_events_data.add_events(); *new_event = std::move(task_execution_event.value()); } + if (task_profile_event) { + auto new_event = expected_ray_events_data.add_events(); + *new_event = std::move(task_profile_event.value()); + } } // Add the data @@ -948,20 +958,23 @@ TEST_F(TaskEventBufferTest, TestTaskProfileEventToRpcRayEvents) { profile_event->SetEndTime(2000); profile_event->SetExtraData("test_extra_data"); - RayEventsPair ray_events_pair; - profile_event->ToRpcRayEvents(ray_events_pair); + RayEventsTuple ray_events_tuple; + profile_event->ToRpcRayEvents(ray_events_tuple); - auto &[first_event, second_event] = ray_events_pair; + auto &[task_definition_event, task_execution_event, task_profile_event] = + ray_events_tuple; // Verify that the second event is nullopt (empty) - EXPECT_FALSE(second_event.has_value()) - << "TaskProfileEvent should set second element of RayEventsPair to nullopt"; + EXPECT_FALSE(task_definition_event.has_value()) + << "TaskProfileEvent should be populated at the third element of RayEventsTuple"; + EXPECT_FALSE(task_execution_event.has_value()) + << "TaskProfileEvent should be populated at the third element of RayEventsTuple"; // Verify that the first event contains the profile event - ASSERT_TRUE(first_event.has_value()) - << "TaskProfileEvent should populate first element of RayEventsPair"; + ASSERT_TRUE(task_profile_event.has_value()) + << "TaskProfileEvent should populate third element of RayEventsTuple"; - const auto &ray_event = first_event.value(); + const auto &ray_event = task_profile_event.value(); // Verify base fields EXPECT_EQ(ray_event.source_type(), rpc::events::RayEvent::CORE_WORKER); @@ -1016,13 +1029,13 @@ TEST_F(TaskEventBufferTest, TestCreateRayEventsDataWithProfileEvents) { "test_session_name"); profile_event->SetEndTime(6000); - absl::flat_hash_map agg_ray_events; + absl::flat_hash_map agg_ray_events; TaskAttempt task_attempt = std::make_pair(task_id, attempt_number); // Populate the ray events pair - RayEventsPair ray_events_pair; - profile_event->ToRpcRayEvents(ray_events_pair); - agg_ray_events[task_attempt] = std::move(ray_events_pair); + RayEventsTuple ray_events_tuple; + profile_event->ToRpcRayEvents(ray_events_tuple); + agg_ray_events[task_attempt] = std::move(ray_events_tuple); // Create the data using the real implementation absl::flat_hash_set dropped_task_attempts; @@ -1044,17 +1057,20 @@ TEST_F(TaskEventBufferTest, TestCreateRayEventsDataWithProfileEvents) { EXPECT_EQ(task_profile_events.attempt_number(), attempt_number); } -TEST_F(TaskEventBufferTest, TestMixedStatusAndProfileEventsToRayEvents) { +TEST_P(TaskEventBufferTestDifferentDestination, + TestMixedStatusAndProfileEventsToRayEvents) { // Test that a mix of status events and profile events are correctly handled - auto task_id1 = RandomTaskId(); - auto task_id2 = RandomTaskId(); + const auto [to_gcs, to_aggregator] = GetParam(); + + // Generate the task id and job id + auto task_id = RandomTaskId(); auto job_id = JobID::FromInt(789); // Create a status event (should populate both elements of RayEventsPair) - auto status_event = GenStatusTaskEvent(task_id1, 1, 1000); + auto status_event = GenStatusTaskEvent(task_id, 1, 1000); // Create a profile event (should populate only first element) - auto profile_event = std::make_unique(task_id2, + auto profile_event = std::make_unique(task_id, job_id, 1, "core_worker", @@ -1063,56 +1079,75 @@ TEST_F(TaskEventBufferTest, TestMixedStatusAndProfileEventsToRayEvents) { "mixed_test", 7000, "test_session_name"); + // Expect data flushed match. Generate the expected data + rpc::TaskEventData expected_task_event_data; + rpc::events::RayEventsData expected_ray_events_data; + auto event = expected_task_event_data.add_events_by_task(); + status_event->ToRpcTaskEvents(event); + profile_event->ToRpcTaskEvents(event); + + RayEventsTuple ray_events_tuple; + status_event->ToRpcRayEvents(ray_events_tuple); + profile_event->ToRpcRayEvents(ray_events_tuple); + auto [task_definition_event, task_execution_event, task_profile_event] = + ray_events_tuple; + if (task_definition_event) { + auto new_event = expected_ray_events_data.add_events(); + *new_event = std::move(task_definition_event.value()); + } + if (task_execution_event) { + auto new_event = expected_ray_events_data.add_events(); + *new_event = std::move(task_execution_event.value()); + } + if (task_profile_event) { + auto new_event = expected_ray_events_data.add_events(); + *new_event = std::move(task_profile_event.value()); + } - // Create aggregated events - absl::flat_hash_map agg_ray_events; - - // Add status event - RayEventsPair status_ray_events_pair; - status_event->ToRpcRayEvents(status_ray_events_pair); - agg_ray_events[std::make_pair(task_id1, 1)] = std::move(status_ray_events_pair); - - // Add profile event - RayEventsPair profile_ray_events_pair; - profile_event->ToRpcRayEvents(profile_ray_events_pair); - agg_ray_events[std::make_pair(task_id2, 1)] = std::move(profile_ray_events_pair); + // Add Events to the task event buffer + task_event_buffer_->AddTaskEvent(std::move(status_event)); + task_event_buffer_->AddTaskEvent(std::move(profile_event)); + ASSERT_EQ(task_event_buffer_->GetNumTaskEventsStored(), 2); - // Create the data - absl::flat_hash_set dropped_task_attempts; - auto ray_events_data = task_event_buffer_->CreateRayEventsDataToSend( - std::move(agg_ray_events), dropped_task_attempts); + // Manually call flush should call GCS client's flushing grpc. + auto task_gcs_accessor = + static_cast(task_event_buffer_->GetGcsClient()) + ->mock_task_accessor; + if (to_gcs) { + EXPECT_CALL(*task_gcs_accessor, AsyncAddTaskEventData(_, _)) + .WillOnce([&](std::unique_ptr actual_data, + ray::gcs::StatusCallback callback) { + CompareTaskEventData(*actual_data, expected_task_event_data); + return Status::OK(); + }); + } else { + EXPECT_CALL(*task_gcs_accessor, AsyncAddTaskEventData(_, _)).Times(0); + } - // Should have 2 events: 1 from status event (execution only since no task_spec) + 1 - // from profile event - ASSERT_EQ(ray_events_data->events_size(), 2); - - // Count event types - int task_definition_events = 0; - int task_execution_events = 0; - int task_profile_events = 0; - - for (const auto &event : ray_events_data->events()) { - switch (event.event_type()) { - case rpc::events::RayEvent::TASK_DEFINITION_EVENT: - task_definition_events++; - break; - case rpc::events::RayEvent::TASK_EXECUTION_EVENT: - task_execution_events++; - break; - case rpc::events::RayEvent::TASK_PROFILE_EVENT: - task_profile_events++; - break; - default: - FAIL() << "Unexpected event type: " << event.event_type(); - } + // If ray events to aggregator is enabled, expect to call AddEvents grpc. + auto event_aggregator_client = static_cast( + task_event_buffer_->event_aggregator_client_.get()); + rpc::events::AddEventsRequest add_events_request; + if (to_aggregator) { + rpc::events::AddEventsReply reply; + Status status = Status::OK(); + EXPECT_CALL(*event_aggregator_client, AddEvents(_, _)) + .WillOnce(DoAll( + Invoke([&](const rpc::events::AddEventsRequest &request, + const rpc::ClientCallback &callback) { + CompareRayEventsData(request.events_data(), expected_ray_events_data); + }), + MakeAction( + new MockEventAggregatorAddEvents(std::move(status), std::move(reply))))); + } else { + EXPECT_CALL(*event_aggregator_client, AddEvents(_, _)).Times(0); } - EXPECT_EQ(task_definition_events, 0) - << "Should have 0 task definition events since GenStatusTaskEvent has no task_spec"; - EXPECT_EQ(task_execution_events, 1) - << "Should have 1 task execution event from status event"; - EXPECT_EQ(task_profile_events, 1) - << "Should have 1 task profile event from profile event"; + // Flush events + task_event_buffer_->FlushEvents(false); + + // Expect no more events. + ASSERT_EQ(task_event_buffer_->GetNumTaskEventsStored(), 0); } INSTANTIATE_TEST_SUITE_P(TaskEventBufferTest, From 5e85227978014975cea6e0bf825562c6adcbc50f Mon Sep 17 00:00:00 2001 From: Ibrahim Rabbani Date: Fri, 5 Sep 2025 10:57:50 -0700 Subject: [PATCH 473/634] [core] (cgroups 3/n) Creating CgroupManager to setup Ray's cgroup hierarchy and clean it up (#56186) Signed-off-by: irabbani Signed-off-by: Ibrahim Rabbani Signed-off-by: Ibrahim Rabbani Co-authored-by: Edward Oakes --- src/ray/common/cgroup2/BUILD.bazel | 48 ++++ src/ray/common/cgroup2/cgroup_manager.cc | 227 ++++++++++++++++++ src/ray/common/cgroup2/cgroup_manager.h | 134 +++++++++++ .../common/cgroup2/cgroup_manager_interface.h | 87 +++++++ src/ray/common/cgroup2/fake_cgroup_driver.h | 151 ++++++++++++ .../common/cgroup2/scoped_cgroup_operation.h | 54 +++++ src/ray/common/cgroup2/tests/BUILD.bazel | 19 ++ .../cgroup2/tests/cgroup_manager_test.cc | 101 ++++++++ src/ray/common/status_or.h | 1 + 9 files changed, 822 insertions(+) create mode 100644 src/ray/common/cgroup2/cgroup_manager.cc create mode 100644 src/ray/common/cgroup2/cgroup_manager.h create mode 100644 src/ray/common/cgroup2/cgroup_manager_interface.h create mode 100644 src/ray/common/cgroup2/fake_cgroup_driver.h create mode 100644 src/ray/common/cgroup2/scoped_cgroup_operation.h create mode 100644 src/ray/common/cgroup2/tests/cgroup_manager_test.cc diff --git a/src/ray/common/cgroup2/BUILD.bazel b/src/ray/common/cgroup2/BUILD.bazel index 2822450214c2..85cd2a9dc059 100644 --- a/src/ray/common/cgroup2/BUILD.bazel +++ b/src/ray/common/cgroup2/BUILD.bazel @@ -14,6 +14,40 @@ ray_cc_library( ], ) +ray_cc_library( + name = "cgroup_manager_interface", + hdrs = [ + "cgroup_manager_interface.h", + ], + target_compatible_with = [ + "@platforms//os:linux", + ], + deps = [ + "//src/ray/common:status", + "//src/ray/common:status_or", + ], +) + +ray_cc_library( + name = "cgroup_manager", + srcs = ["cgroup_manager.cc"], + hdrs = [ + "cgroup_manager.h", + "scoped_cgroup_operation.h", + ], + target_compatible_with = [ + "@platforms//os:linux", + ], + deps = [ + ":cgroup_driver_interface", + ":cgroup_manager_interface", + "//src/ray/common:status", + "//src/ray/common:status_or", + "//src/ray/util:logging", + "@com_google_absl//absl/strings", + ], +) + ray_cc_library( name = "sysfs_cgroup_driver", srcs = ["sysfs_cgroup_driver.cc"], @@ -32,6 +66,20 @@ ray_cc_library( ], ) +ray_cc_library( + name = "fake_cgroup_driver", + hdrs = [ + "fake_cgroup_driver.h", + ], + target_compatible_with = [ + "@platforms//os:linux", + ], + deps = [ + ":cgroup_driver_interface", + "//src/ray/common:status", + ], +) + ray_cc_library( name = "cgroup_test_utils", srcs = ["cgroup_test_utils.cc"], diff --git a/src/ray/common/cgroup2/cgroup_manager.cc b/src/ray/common/cgroup2/cgroup_manager.cc new file mode 100644 index 000000000000..f01741ea0d1d --- /dev/null +++ b/src/ray/common/cgroup2/cgroup_manager.cc @@ -0,0 +1,227 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/common/cgroup2/cgroup_manager.h" + +#include +#include +#include +#include +#include + +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "ray/common/cgroup2/cgroup_driver_interface.h" +#include "ray/common/cgroup2/scoped_cgroup_operation.h" +#include "ray/common/status_or.h" + +namespace ray { + +CgroupManager::CgroupManager(std::string base_cgroup_path, + const std::string &node_id, + std::unique_ptr cgroup_driver) + : base_cgroup_path_(std::move(base_cgroup_path)), + cgroup_driver_(std::move(cgroup_driver)) { + node_cgroup_path_ = base_cgroup_path_ + std::filesystem::path::preferred_separator + + absl::StrFormat("%s_%s", kNodeCgroupName, node_id); + system_cgroup_path_ = + node_cgroup_path_ + std::filesystem::path::preferred_separator + kSystemCgroupName; + + application_cgroup_path_ = node_cgroup_path_ + + std::filesystem::path::preferred_separator + + kApplicationCgroupName; +} + +CgroupManager::~CgroupManager() { + while (!cleanup_operations_.empty()) { + cleanup_operations_.pop_back(); + } +} + +StatusOr> CgroupManager::Create( + std::string base_cgroup_path, + const std::string &node_id, + const int64_t system_reserved_cpu_weight, + const int64_t system_reserved_memory_bytes, + std::unique_ptr cgroup_driver) { + // TODO(#54703): Add bounds checking for system_reserved_cpu_weight + // and system_reserved_memory_bytes. + RAY_RETURN_NOT_OK(cgroup_driver->CheckCgroupv2Enabled()); + RAY_RETURN_NOT_OK(cgroup_driver->CheckCgroup(base_cgroup_path)); + StatusOr> available_controllers = + cgroup_driver->GetAvailableControllers(base_cgroup_path); + + if (!available_controllers.ok()) { + return available_controllers.status(); + } + + std::string supported_controllers_str = + absl::StrCat("[", absl::StrJoin(supported_controllers_, ", "), "]"); + + for (const auto &ctrl : supported_controllers_) { + if (available_controllers->find(ctrl) == available_controllers->end()) { + std::string available_controllers_str = + absl::StrCat("[", absl::StrJoin(*available_controllers, ", "), "]"); + return Status::Invalid(absl::StrFormat( + "Failed to initialize resource isolation " + "because required controllers are not available in the cgroup %s. " + "To make controllers available in %s, you need to enable controllers for its " + "ancestor cgroups. See " + "https://docs.kernel.org/admin-guide/cgroup-v2.html#controlling-controllers " + "for more details. Available controllers: %s. Required controllers: " + "%s.", + base_cgroup_path, + base_cgroup_path, + available_controllers_str, + supported_controllers_str)); + } + } + + std::unique_ptr cgroup_manager = std::unique_ptr( + new CgroupManager(std::move(base_cgroup_path), node_id, std::move(cgroup_driver))); + + RAY_RETURN_NOT_OK(cgroup_manager->Initialize(system_reserved_cpu_weight, + system_reserved_memory_bytes)); + + return cgroup_manager; +} + +// TODO(#54703): This is a placeholder for cleanup. This will call +// CgroupDriver::DeleteCgroup. +void CgroupManager::RegisterDeleteCgroup(const std::string &cgroup_path) { + cleanup_operations_.emplace_back([cgroup = cgroup_path]() { + RAY_LOG(INFO) << absl::StrFormat("Deleting all cgroup %s.", cgroup); + }); +} + +// TODO(#54703): This is a placeholder for cleanup. This will call +// CgroupDriver::MoveAllProcesses. +void CgroupManager::RegisterMoveAllProcesses(const std::string &from, + const std::string &to) { + cleanup_operations_.emplace_back([from_cgroup = from, to_cgroup = to]() { + RAY_LOG(INFO) << absl::StrFormat( + "Moved All Processes from %s to %s.", from_cgroup, to_cgroup); + }); +} + +// TODO(#54703): This is a placeholder for cleanup. This will call +// CgroupDriver::AddConstraint(cgroup, constraint, default_value). +void CgroupManager::RegisterRemoveConstraint(const std::string &cgroup, + const std::string &constraint) { + cleanup_operations_.emplace_back( + [constrained_cgroup = cgroup, constraint_to_remove = constraint]() { + auto constraint_metadata = supported_constraints_.find(constraint_to_remove); + RAY_CHECK(constraint_metadata != supported_constraints_.end()); + RAY_LOG(INFO) << absl::StrFormat( + "Setting constraint %s to default value %lld for cgroup %s", + constraint_to_remove, + constraint_metadata->second.default_value, + constrained_cgroup); + }); +} + +// TODO(#54703): This is a placeholder for cleanup. This will call +// CgroupDriver::DisableController. +void CgroupManager::RegisterDisableController(const std::string &cgroup, + const std::string &controller) { + cleanup_operations_.emplace_back([cgroup_to_clean = cgroup, + controller_to_disable = controller]() { + RAY_LOG(INFO) << absl::StrFormat( + "Disabling controller %s for cgroup %s.", controller_to_disable, cgroup_to_clean); + }); +} + +Status CgroupManager::Initialize(int64_t system_reserved_cpu_weight, + int64_t system_reserved_memory_bytes) { + std::string supported_controllers = + absl::StrCat("[", absl::StrJoin(supported_controllers_, ", "), "]"); + + // The cpu.weight is distributed between the system and application cgroups. + // The application cgroup gets whatever is leftover from the system cgroup. + int64_t max_cpu_weight = supported_constraints_.at(kCPUWeightConstraint).Max(); + int64_t application_cgroup_cpu_weight = max_cpu_weight - system_reserved_cpu_weight; + + RAY_LOG(INFO) << absl::StrFormat( + "Initializing CgroupManager at base cgroup path at %s. Ray's cgroup " + "hierarchy will under the node cgroup %s. The %s controllers will be " + "enabled. " + "System cgroup %s will have constraints [%s=%lld, %s=%lld]. " + "Application cgroup %s will have constraints [%s=%lld].", + base_cgroup_path_, + node_cgroup_path_, + supported_controllers, + system_cgroup_path_, + kCPUWeightConstraint, + system_reserved_cpu_weight, + kMemoryMinConstraint, + system_reserved_memory_bytes, + application_cgroup_path_, + kCPUWeightConstraint, + application_cgroup_cpu_weight); + + // Create the cgroup heirarchy: + // base_cgroup_path (e.g. /sys/fs/cgroup) + // | + // ray_node_ + // | | + // system application + RAY_RETURN_NOT_OK(cgroup_driver_->CreateCgroup(node_cgroup_path_)); + RegisterDeleteCgroup(node_cgroup_path_); + + RAY_RETURN_NOT_OK(cgroup_driver_->CreateCgroup(system_cgroup_path_)); + RegisterDeleteCgroup(system_cgroup_path_); + + RAY_RETURN_NOT_OK(cgroup_driver_->CreateCgroup(application_cgroup_path_)); + RegisterDeleteCgroup(application_cgroup_path_); + + // Move all processes from the base_cgroup into the system_cgroup to make sure + // that the no internal process constraint is not violated. This is relevant + // when the base_cgroup_path is not a root cgroup for the system. This is likely + // the case if Ray is running inside a container. + RAY_RETURN_NOT_OK( + cgroup_driver_->MoveAllProcesses(base_cgroup_path_, system_cgroup_path_)); + RegisterMoveAllProcesses(system_cgroup_path_, base_cgroup_path_); + + for (const auto &ctrl : supported_controllers_) { + RAY_RETURN_NOT_OK(cgroup_driver_->EnableController(base_cgroup_path_, ctrl)); + RegisterDisableController(base_cgroup_path_, ctrl); + RAY_RETURN_NOT_OK(cgroup_driver_->EnableController(node_cgroup_path_, ctrl)); + RegisterDisableController(node_cgroup_path_, ctrl); + RAY_RETURN_NOT_OK(cgroup_driver_->EnableController(system_cgroup_path_, ctrl)); + RegisterDisableController(system_cgroup_path_, ctrl); + RAY_RETURN_NOT_OK(cgroup_driver_->EnableController(application_cgroup_path_, ctrl)); + RegisterDisableController(application_cgroup_path_, ctrl); + } + + RAY_RETURN_NOT_OK( + cgroup_driver_->AddConstraint(system_cgroup_path_, + kMemoryMinConstraint, + std::to_string(system_reserved_memory_bytes))); + RegisterRemoveConstraint(system_cgroup_path_, kMemoryMinConstraint); + + RAY_RETURN_NOT_OK( + cgroup_driver_->AddConstraint(system_cgroup_path_, + kCPUWeightConstraint, + std::to_string(system_reserved_cpu_weight))); + RegisterRemoveConstraint(system_cgroup_path_, kCPUWeightConstraint); + + RAY_RETURN_NOT_OK( + cgroup_driver_->AddConstraint(application_cgroup_path_, + kCPUWeightConstraint, + std::to_string(application_cgroup_cpu_weight))); + RegisterRemoveConstraint(application_cgroup_path_, kCPUWeightConstraint); + + return Status::OK(); +} +} // namespace ray diff --git a/src/ray/common/cgroup2/cgroup_manager.h b/src/ray/common/cgroup2/cgroup_manager.h new file mode 100644 index 000000000000..fa281e7376bf --- /dev/null +++ b/src/ray/common/cgroup2/cgroup_manager.h @@ -0,0 +1,134 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include + +#include "ray/common/cgroup2/cgroup_driver_interface.h" +#include "ray/common/cgroup2/cgroup_manager_interface.h" +#include "ray/common/cgroup2/scoped_cgroup_operation.h" +#include "ray/common/status.h" +#include "ray/common/status_or.h" + +namespace ray { +class CgroupManager : public CgroupManagerInterface { + public: + /** + + Creates a CgroupManager after checking for the following invariants: + + 1. cgroupv2 is mounted correctly in unified mode. For more details (@see + CgroupDriverInterface::CheckCgroupv2Enabled). + 2. the current process has permissions to read and write to the base_cgroup. + 3. supported cgroup controllers are available (@see supported_controllers_). + + The CgroupManager will be used to + 1. construct the cgroup hierarchy. + 2. move processes into the appropriate cgroups. + 3. enable controllers and resource constraints. + + @param base_cgroup the cgroup that the process will take ownership of. + @param node_id used to create a ray node cgroup. + @param system_reserved_cpu_weight a value between [1,10000] to assign to the cgroup + for system processes. The cgroup for application processes gets 10000 - + system_reserved_cpu_weight. + @param system_reserved_memory_bytes used to reserve memory for the system cgroup. + @param cgroup_driver used to perform cgroup operations. + + @return Status::OK with an instance of CgroupManager if everything succeeds. + @return Status::Invalid if cgroupv2 is not enabled correctly. + @return Status::InvalidArgument if base_cgroup is not a cgroup. + @return Status::NotFound if the base_cgroupd does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and + execute permissions. + */ + static StatusOr> Create( + std::string base_cgroup_path, + const std::string &node_id, + const int64_t system_reserved_cpu_weight, + const int64_t system_reserved_memory_bytes, + std::unique_ptr cgroup_driver); + + // Unmovable and uncopyable type. + CgroupManager(const CgroupManager &) = delete; + CgroupManager &operator=(const CgroupManager &) = delete; + CgroupManager(CgroupManager &&) = default; + CgroupManager &operator=(CgroupManager &&) = default; + + /** + Performs cleanup in reverse order from the Initialize function: + 1. remove resource constraints to the system and application cgroups. + 2. disable controllers on the base, system, and application cgroups respectively. + 3. move all processes from the system cgroup into the base cgroup. + 4. delete the node, system, and application cgroups respectively. + + Cleanup is best-effort. If any step fails, it will log a warning. + */ + ~CgroupManager() override; + + private: + CgroupManager(std::string base_cgroup_path, + const std::string &node_id, + std::unique_ptr cgroup_driver); + + /** + Performs the following operations: + + 1. create the node, system, and application cgroups respectively. + 2. move all processes from the base_cgroup into the system cgroup. + 3. enable controllers the base, node, system, and application cgroups respectively. + 4. add resource constraints to the system and application cgroups. + + @param system_reserved_cpu_weight a value between [1,10000] to assign to the cgroup + for system processes. The cgroup for application processes gets 10000 - + system_reserved_cpu_weight. + @param system_reserved_memory_bytes used to reserve memory for the system cgroup. + + @return Status::OK if no errors encountered. + @return Status::NotFound if base_cgroup does not exist. + @return Status::PermissionDenied if the process does not have enough permissions + to create a cgroup or write to it. + @return Status::Invalid if processes could not be moved between cgroups. + @return Status::InvalidArgument if base_cgroup_path_ is not a valid cgroup, + supported_controllers_ cannot be enabled, or a constraint is not supported. + @return Status::AlreadyExists if the the node, application, or system cgroup already + exists. + + */ + Status Initialize(const int64_t system_reserved_cpu_weight, + const int64_t system_reserved_memory_bytes); + + // TODO(#54703): This is a placeholder for cleanup. This will be implemented in the a + // future PR. + void RegisterDeleteCgroup(const std::string &cgroup_path); + void RegisterMoveAllProcesses(const std::string &from, const std::string &to); + void RegisterRemoveConstraint(const std::string &cgroup, const std::string &constraint); + void RegisterDisableController(const std::string &cgroup, + const std::string &controller); + + std::string base_cgroup_path_; + std::string node_cgroup_path_; + std::string system_cgroup_path_; + std::string application_cgroup_path_; + + // This will be popped in reverse order to clean up all side-effects performed + // during setup. + std::vector cleanup_operations_; + + std::unique_ptr cgroup_driver_; +}; +} // namespace ray diff --git a/src/ray/common/cgroup2/cgroup_manager_interface.h b/src/ray/common/cgroup2/cgroup_manager_interface.h new file mode 100644 index 000000000000..5b69405d3863 --- /dev/null +++ b/src/ray/common/cgroup2/cgroup_manager_interface.h @@ -0,0 +1,87 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "ray/common/cgroup2/cgroup_driver_interface.h" +#include "ray/common/status_or.h" + +namespace ray { + +/** + Sets up resource isolation for a Ray node using cgroup2 using the following + cgroup hierachy: + + base_cgroup_path (e.g. /sys/fs/cgroup) + | + ray_node_ + | | + system application +*/ +class CgroupManagerInterface { + // TODO(#54703): The Constraint struct, supported_constraints_, and + // supported_controllers_ are duplicated across CgroupManagerInterface and + // CgroupDriverInterface. It makes sense for these to be separated into two concerns: + // 1) Checking which controllers and constraints are supported in Ray should be in + // CgroupManagerInterface. + // 2) Checking what values are allowed for constraints should be inside + // CgroupDriverInterface. + // This will be done in a later PR. + struct Constraint { + std::pair range; + std::string controller; + int64_t default_value; + + int64_t Max() const { return range.second; } + int64_t Min() const { return range.first; } + }; + + public: + // TODO(#54703): These will be implemented in a later PR to move processes + // into a cgroup. + // virtual Status AddProcessToApplicationCgroup(int) = 0; + // virtual Status AddProcessToSystemCgroup(int) = 0; + + /** + Cleans up the cgroup hierarchy, disables all controllers and removes all + constraints. + */ + virtual ~CgroupManagerInterface() = default; + + protected: + inline static const std::string kNodeCgroupName = "ray_node"; + inline static const std::string kSystemCgroupName = "system"; + inline static const std::string kApplicationCgroupName = "application"; + inline static const std::string kCPUWeightConstraint = "cpu.weight"; + inline static const std::string kMemoryMinConstraint = "memory.min"; + + inline static const std::unordered_map supported_constraints_ = + {{kCPUWeightConstraint, {{1, 10000}, "cpu", 100}}, + { + kMemoryMinConstraint, + {{0, std::numeric_limits::max()}, "memory", 0}, + }}; + inline static const std::unordered_set supported_controllers_ = {"cpu", + "memory"}; +}; +} // namespace ray diff --git a/src/ray/common/cgroup2/fake_cgroup_driver.h b/src/ray/common/cgroup2/fake_cgroup_driver.h new file mode 100644 index 000000000000..d40235f71ee3 --- /dev/null +++ b/src/ray/common/cgroup2/fake_cgroup_driver.h @@ -0,0 +1,151 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "ray/common/cgroup2/cgroup_driver_interface.h" +#include "ray/common/cgroup2/cgroup_manager.h" +#include "ray/common/status.h" + +namespace ray { + +struct FakeCgroup { + std::string path_; + std::vector processes_; + std::vector> constraints_; + std::unordered_set available_controllers_; + std::unordered_set enabled_controllers_; + bool operator==(const FakeCgroup &other) const { + return path_ == other.path_ && processes_ == other.processes_ && + constraints_ == other.constraints_ && + available_controllers_ == other.available_controllers_ && + enabled_controllers_ == other.enabled_controllers_; + } +}; +class FakeCgroupDriver : public CgroupDriverInterface { + public: + explicit FakeCgroupDriver( + std::shared_ptr> cgroups) + : cgroups_(cgroups) {} + + explicit FakeCgroupDriver(std::string base_cgroup) + : cgroups_(std::make_shared>()) { + RAY_LOG(INFO) << "FakeCgroupDriver(std::string base_cgroup)"; + cgroups_->emplace(base_cgroup, FakeCgroup{base_cgroup}); + } + FakeCgroupDriver(std::string base_cgroup, + std::vector processes_in_base_cgroup, + std::unordered_set available_controllers) + : cgroups_(std::make_shared>()) { + cgroups_->emplace(base_cgroup, + FakeCgroup{base_cgroup, + std::move(processes_in_base_cgroup), + {}, + std::move(available_controllers), + {}}); + } + + std::shared_ptr> cgroups_; + + Status check_cgroup_enabled_s_ = Status::OK(); + Status check_cgroup_s_ = Status::OK(); + Status create_cgroup_s_ = Status::OK(); + Status move_all_processes_s_ = Status::OK(); + Status enable_controller_s_ = Status::OK(); + Status disable_controller_s_ = Status::OK(); + Status add_constraint_s_ = Status::OK(); + Status available_controllers_s_ = Status::OK(); + Status enabled_controllers_s_ = Status::OK(); + + // These have no side-effects. + Status CheckCgroupv2Enabled() override { return check_cgroup_enabled_s_; } + Status CheckCgroup(const std::string &cgroup) override { return check_cgroup_s_; } + + // These have side-effects made visible through the cgroups_ map. + // All of them can be short-circuited by setting the corresponding + // status to not ok. + Status CreateCgroup(const std::string &cgroup) override { + RAY_LOG(INFO) << "CreateCgroup " << cgroup; + if (!create_cgroup_s_.ok()) { + return create_cgroup_s_; + } + cgroups_->emplace(cgroup, FakeCgroup{cgroup}); + return create_cgroup_s_; + } + + Status MoveAllProcesses(const std::string &from, const std::string &to) override { + if (!move_all_processes_s_.ok()) { + return move_all_processes_s_; + } + FakeCgroup &from_cgroup = (*cgroups_)[from]; + FakeCgroup &to_cgroup = (*cgroups_)[to]; + while (!from_cgroup.processes_.empty()) { + to_cgroup.processes_.emplace_back(from_cgroup.processes_.back()); + from_cgroup.processes_.pop_back(); + } + return move_all_processes_s_; + } + + Status EnableController(const std::string &cgroup, + const std::string &controller) override { + if (!enable_controller_s_.ok()) { + return enable_controller_s_; + } + (*cgroups_)[cgroup].enabled_controllers_.emplace(controller); + return enable_controller_s_; + } + + Status DisableController(const std::string &cgroup, + const std::string &controller) override { + if (!disable_controller_s_.ok()) { + return disable_controller_s_; + } + (*cgroups_)[cgroup].enabled_controllers_.erase(controller); + return disable_controller_s_; + } + + Status AddConstraint(const std::string &cgroup, + const std::string &constraint, + const std::string &value) override { + if (!add_constraint_s_.ok()) { + return add_constraint_s_; + } + (*cgroups_)[cgroup].constraints_.emplace_back(constraint, value); + return add_constraint_s_; + } + + StatusOr> GetAvailableControllers( + const std::string &cgroup) override { + if (!available_controllers_s_.ok()) { + return available_controllers_s_; + } + return (*cgroups_)[cgroup].available_controllers_; + } + + StatusOr> GetEnabledControllers( + const std::string &cgroup) override { + if (!enabled_controllers_s_.ok()) { + return enabled_controllers_s_; + } + return (*cgroups_)[cgroup].enabled_controllers_; + } +}; + +} // namespace ray diff --git a/src/ray/common/cgroup2/scoped_cgroup_operation.h b/src/ray/common/cgroup2/scoped_cgroup_operation.h new file mode 100644 index 000000000000..4f8f26992ab2 --- /dev/null +++ b/src/ray/common/cgroup2/scoped_cgroup_operation.h @@ -0,0 +1,54 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include + +namespace ray { + +/** + A simple RAII style guard that calls the registered callback on destruction. + ScopedCgroupOperation instances can be moved, but they cannot be copied. + + Usage: + ScopedCgroupOperation say_hello_on_death([]() { + RAY_INFO(INFO) << "Hi, I'm dying!"; + }); +*/ +class ScopedCgroupOperation { + public: + explicit ScopedCgroupOperation(std::function cleanup_fcn) + : cleanup_fcn_(std::move(cleanup_fcn)) {} + + ~ScopedCgroupOperation() { cleanup_fcn_(); } + + ScopedCgroupOperation(const ScopedCgroupOperation &) = delete; + ScopedCgroupOperation &operator=(const ScopedCgroupOperation &other) = delete; + + ScopedCgroupOperation(ScopedCgroupOperation &&other) noexcept + : cleanup_fcn_(std::move(other.cleanup_fcn_)) { + other.cleanup_fcn_ = []() {}; + } + + ScopedCgroupOperation &operator=(ScopedCgroupOperation &&other) noexcept { + cleanup_fcn_ = std::move(other.cleanup_fcn_); + other.cleanup_fcn_ = []() {}; + return *this; + } + + private: + // Defaults to no cleanup. + std::function cleanup_fcn_ = []() {}; +}; +} // namespace ray diff --git a/src/ray/common/cgroup2/tests/BUILD.bazel b/src/ray/common/cgroup2/tests/BUILD.bazel index e9d165d5d49c..06d0ca6d1221 100644 --- a/src/ray/common/cgroup2/tests/BUILD.bazel +++ b/src/ray/common/cgroup2/tests/BUILD.bazel @@ -20,3 +20,22 @@ ray_cc_test( "@com_google_googletest//:gtest_main", ], ) + +ray_cc_test( + name = "cgroup_manager_test", + srcs = ["cgroup_manager_test.cc"], + tags = [ + "cgroup", + "no_windows", + "team:core", + ], + deps = [ + "//src/ray/common:status", + "//src/ray/common:status_or", + "//src/ray/common/cgroup2:cgroup_driver_interface", + "//src/ray/common/cgroup2:cgroup_manager", + "//src/ray/common/cgroup2:fake_cgroup_driver", + "@com_google_absl//absl/strings:str_format", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/common/cgroup2/tests/cgroup_manager_test.cc b/src/ray/common/cgroup2/tests/cgroup_manager_test.cc new file mode 100644 index 000000000000..1c7f5e154013 --- /dev/null +++ b/src/ray/common/cgroup2/tests/cgroup_manager_test.cc @@ -0,0 +1,101 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/common/cgroup2/cgroup_manager.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "ray/common/cgroup2/fake_cgroup_driver.h" +#include "ray/common/status.h" +namespace ray { + +TEST(CgroupManagerTest, CreateReturnsInvalidIfCgroupv2NotAvailable) { + std::shared_ptr> cgroups = + std::make_shared>(); + cgroups->emplace("/sys/fs/cgroup", FakeCgroup{"/sys/fs/cgroup"}); + FakeCgroup base_cgroup{"/sys/fs/cgroup"}; + FakeCgroupDriver *driver = new FakeCgroupDriver(cgroups); + driver->check_cgroup_enabled_s_ = Status::Invalid(""); + auto cgroup_manager_s = + CgroupManager::Create("/sys/fs/cgroup/ray", + "node_id_123", + 100, + 1000000, + std::unique_ptr(driver)); + ASSERT_TRUE(cgroup_manager_s.IsInvalid()) << cgroup_manager_s.ToString(); + // No visible side-effects + ASSERT_EQ(cgroups->size(), 1); + ASSERT_EQ(cgroups->begin()->second, base_cgroup); +} + +TEST(CgroupManagerTest, CreateReturnsNotFoundIfBaseCgroupDoesNotExist) { + std::shared_ptr> cgroups = + std::make_shared>(); + FakeCgroupDriver *driver = new FakeCgroupDriver(cgroups); + driver->check_cgroup_s_ = Status::NotFound(""); + auto cgroup_manager_s = + CgroupManager::Create("/sys/fs/cgroup/ray", + "node_id_123", + 100, + 1000000, + std::unique_ptr(driver)); + ASSERT_TRUE(cgroup_manager_s.IsNotFound()) << cgroup_manager_s.ToString(); + // No visible side-effects + ASSERT_EQ(cgroups->size(), 0); +} + +TEST(CgroupManagerTest, + CreateReturnsNotFoundIfProcessDoesNotHavePermissionsForBaseCgroup) { + std::shared_ptr> cgroups = + std::make_shared>(); + cgroups->emplace("/sys/fs/cgroup", FakeCgroup{"/sys/fs/cgroup"}); + FakeCgroup base_cgroup{"/sys/fs/cgroup"}; + FakeCgroupDriver *driver = new FakeCgroupDriver(cgroups); + driver->check_cgroup_s_ = Status::PermissionDenied(""); + auto cgroup_manager_s = + CgroupManager::Create("/sys/fs/cgroup/ray", + "node_id_123", + 100, + 1000000, + std::unique_ptr(driver)); + ASSERT_TRUE(cgroup_manager_s.IsPermissionDenied()) << cgroup_manager_s.ToString(); + // No visible side-effects + ASSERT_EQ(cgroups->size(), 1); + ASSERT_EQ(cgroups->begin()->second, base_cgroup); +} + +TEST(CgroupManagerTest, CreateReturnsInvalidIfSupportedControllersAreNotAvailable) { + std::shared_ptr> cgroups = + std::make_shared>(); + // By default no controllers are available. + cgroups->emplace("/sys/fs/cgroup", FakeCgroup{"/sys/fs/cgroup"}); + FakeCgroup base_cgroup{"/sys/fs/cgroup"}; + FakeCgroupDriver *driver = new FakeCgroupDriver(cgroups); + auto cgroup_manager_s = + CgroupManager::Create("/sys/fs/cgroup", + "node_id_123", + 100, + 1000000, + std::unique_ptr(driver)); + ASSERT_TRUE(cgroup_manager_s.IsInvalid()) << cgroup_manager_s.ToString(); + // No visible side-effects + ASSERT_EQ(cgroups->size(), 1); + ASSERT_EQ(cgroups->begin()->second, base_cgroup); +} + +} // namespace ray diff --git a/src/ray/common/status_or.h b/src/ray/common/status_or.h index 2a94bb4bec99..12c7ed8f7b44 100644 --- a/src/ray/common/status_or.h +++ b/src/ray/common/status_or.h @@ -155,6 +155,7 @@ class StatusOr { bool IsNotFound() const { return code() == StatusCode::NotFound; } bool IsInvalidArgument() const { return code() == StatusCode::InvalidArgument; } + bool IsInvalid() const { return code() == StatusCode::Invalid; } bool IsPermissionDenied() const { return code() == StatusCode::PermissionDenied; } // Returns a reference to the current `ray::Status` contained within the From 45baf10349ee3a8fb2395abf46ff6481fa775b60 Mon Sep 17 00:00:00 2001 From: Potato Date: Sat, 6 Sep 2025 02:30:47 +0800 Subject: [PATCH 474/634] [CORE][DOC] Fix grammar, spelling, and formatting issues in Ray Core documentation (#56278) Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../patterns/concurrent-operations-async-actor.rst | 4 ++-- .../ray-core/patterns/fork-new-processes.rst | 14 +++++++------- .../ray-core/patterns/limit-pending-tasks.rst | 2 +- .../out-of-band-object-ref-serialization.rst | 10 +++++----- doc/source/ray-core/scheduling/accelerators.rst | 4 ++-- doc/source/ray-core/scheduling/index.rst | 2 +- doc/source/ray-core/scheduling/placement-group.rst | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/doc/source/ray-core/patterns/concurrent-operations-async-actor.rst b/doc/source/ray-core/patterns/concurrent-operations-async-actor.rst index 656c3215243d..fc8e897569c3 100644 --- a/doc/source/ray-core/patterns/concurrent-operations-async-actor.rst +++ b/doc/source/ray-core/patterns/concurrent-operations-async-actor.rst @@ -22,7 +22,7 @@ With the default actor, the code will look like this: :start-after: __sync_actor_start__ :end-before: __sync_actor_end__ -This is problematic because ``TaskExecutor.run`` method runs forever and never yield the control to run other methods. +This is problematic because ``TaskExecutor.run`` method runs forever and never yields control to run other methods. We can solve this problem by using :ref:`async actors ` and use ``await`` to yield control: .. literalinclude:: ../doc_code/pattern_async_actor.py @@ -30,4 +30,4 @@ We can solve this problem by using :ref:`async actors ` and use `` :start-after: __async_actor_start__ :end-before: __async_actor_end__ -Here, instead of using the blocking :func:`ray.get() ` to get the value of an ObjectRef, we use ``await`` so it can yield the control while we are waiting for the object to be fetched. +Here, instead of using the blocking :func:`ray.get() ` to get the value of an ObjectRef, we use ``await`` so it can yield control while we are waiting for the object to be fetched. diff --git a/doc/source/ray-core/patterns/fork-new-processes.rst b/doc/source/ray-core/patterns/fork-new-processes.rst index 0ef83b88d274..0323d906dced 100644 --- a/doc/source/ray-core/patterns/fork-new-processes.rst +++ b/doc/source/ray-core/patterns/fork-new-processes.rst @@ -3,21 +3,21 @@ Anti-pattern: Forking new processes in application code ======================================================== -**Summary:** Don't fork new processes in Ray application code-for example, in -driver, tasks or actors. Instead, use "spawn" method to start new processes or use Ray +**Summary:** Don't fork new processes in Ray application code—for example, in +driver, tasks or actors. Instead, use the "spawn" method to start new processes or use Ray tasks and actors to parallelize your workload Ray manages the lifecycle of processes for you. Ray Objects, Tasks, and -Actors manages sockets to communicate with the Raylet and the GCS. If you fork new +Actors manage sockets to communicate with the Raylet and the GCS. If you fork new processes in your application code, the processes could share the same sockets without -any synchronization. This can lead to corrupted message and unexpected +any synchronization. This can lead to corrupted messages and unexpected behavior. The solution is to: -1. use "spawn" method to start new processes so that parent process's -memory space isn't copied to the child processes or +1. use the "spawn" method to start new processes so that the parent process's +memory space is not copied to the child processes or 2. use Ray tasks and -actors to parallelize your workload and let Ray to manage the lifecycle of the +actors to parallelize your workload and let Ray manage the lifecycle of the processes for you. Code example diff --git a/doc/source/ray-core/patterns/limit-pending-tasks.rst b/doc/source/ray-core/patterns/limit-pending-tasks.rst index 8a266990c4d4..6bdac69273aa 100644 --- a/doc/source/ray-core/patterns/limit-pending-tasks.rst +++ b/doc/source/ray-core/patterns/limit-pending-tasks.rst @@ -22,7 +22,7 @@ With ``ray.wait()``, we can apply backpressure and limit the number of pending t Example use case ---------------- -You have a worker actor that process tasks at a rate of X tasks per second and you want to submit tasks to it at a rate lower than X to avoid OOM. +You have a worker actor that processes tasks at a rate of X tasks per second and you want to submit tasks to it at a rate lower than X to avoid OOM. For example, Ray Serve uses this pattern to limit the number of pending queries for each worker. diff --git a/doc/source/ray-core/patterns/out-of-band-object-ref-serialization.rst b/doc/source/ray-core/patterns/out-of-band-object-ref-serialization.rst index 4b87f16d68fb..6a24ea50b1aa 100644 --- a/doc/source/ray-core/patterns/out-of-band-object-ref-serialization.rst +++ b/doc/source/ray-core/patterns/out-of-band-object-ref-serialization.rst @@ -6,14 +6,14 @@ Anti-pattern: Serialize ray.ObjectRef out of band **TLDR:** Avoid serializing ``ray.ObjectRef`` because Ray can't know when to garbage collect the underlying object. Ray's ``ray.ObjectRef`` is distributed reference counted. Ray pins the underlying object until the reference isn't used by the system anymore. -When all references are the pinned object gone, Ray garbage collects the pinned object and cleans it up from the system. -However, if user code serializes ``ray.objectRef``, Ray can't keep track of the reference. +When all references to the pinned object are gone, Ray garbage collects the pinned object and cleans it up from the system. +However, if user code serializes ``ray.ObjectRef``, Ray can't keep track of the reference. -To avoid incorrect behavior, if ``ray.cloudpickle`` serializes``ray.ObjectRef``, Ray pins the object for the lifetime of a worker. "Pin" means that object can't be evicted from the object store -until the corresponding owner worker dies. It's prone to Ray object leaks, which can lead disk spilling. See :ref:`this page ` for more details. +To avoid incorrect behavior, if ``ray.cloudpickle`` serializes ``ray.ObjectRef``, Ray pins the object for the lifetime of a worker. "Pin" means that object can't be evicted from the object store +until the corresponding owner worker dies. It's prone to Ray object leaks, which can lead to disk spilling. See :ref:`this page ` for more details. To detect if this pattern exists in your code, you can set an environment variable ``RAY_allow_out_of_band_object_ref_serialization=0``. If Ray detects -that ``ray.cloudpickle`` serialized``ray.ObjectRef``, it raises an exception with helpful messages. +that ``ray.cloudpickle`` serialized ``ray.ObjectRef``, it raises an exception with helpful messages. Code example ------------ diff --git a/doc/source/ray-core/scheduling/accelerators.rst b/doc/source/ray-core/scheduling/accelerators.rst index 707b5b75a053..47ef5cd80e6e 100644 --- a/doc/source/ray-core/scheduling/accelerators.rst +++ b/doc/source/ray-core/scheduling/accelerators.rst @@ -84,11 +84,11 @@ If you need to, you can :ref:`override ` this. .. tip:: You can set the ``NEURON_RT_VISIBLE_CORES`` environment variable before starting a Ray node - to limit the AWS Neuro Cores that are visible to Ray. + to limit the AWS Neuron Cores that are visible to Ray. For example, ``NEURON_RT_VISIBLE_CORES=1,3 ray start --head --resources='{"neuron_cores": 2}'`` lets Ray only see devices 1 and 3. - See the `Amazon documentation` for more examples of Ray on Neuron with EKS as an orchestration substrate. + See the `Amazon documentation `_ for more examples of Ray on Neuron with EKS as an orchestration substrate. .. tab-item:: Google TPU :sync: Google TPU diff --git a/doc/source/ray-core/scheduling/index.rst b/doc/source/ray-core/scheduling/index.rst index 9fb5bed4f712..e46203c21d3d 100644 --- a/doc/source/ray-core/scheduling/index.rst +++ b/doc/source/ray-core/scheduling/index.rst @@ -22,7 +22,7 @@ Given that, a node can be in one of the following states: - Infeasible: the node doesn't have the required resources. For example a CPU-only node is infeasible for a GPU task. Resource requirements are **hard** requirements meaning that only feasible nodes are eligible to run the task or actor. -If there are feasible nodes, Ray will either choose an available node or wait until a unavailable node to become available +If there are feasible nodes, Ray will either choose an available node or wait until an unavailable node to become available depending on other factors discussed below. If all nodes are infeasible, the task or actor cannot be scheduled until feasible nodes are added to the cluster. diff --git a/doc/source/ray-core/scheduling/placement-group.rst b/doc/source/ray-core/scheduling/placement-group.rst index beb00ee77cca..5a4245eeb005 100644 --- a/doc/source/ray-core/scheduling/placement-group.rst +++ b/doc/source/ray-core/scheduling/placement-group.rst @@ -37,7 +37,7 @@ Create a Placement Group (Reserve Resources) You can create a placement group using :func:`ray.util.placement_group`. Placement groups take in a list of bundles and a :ref:`placement strategy `. Note that each bundle must be able to fit on a single node on the Ray cluster. -For example, if you only have a 8 CPU node, and if you have a bundle that requires ``{"CPU": 9}``, +For example, if you only have an 8 CPU node, and if you have a bundle that requires ``{"CPU": 9}``, this bundle cannot be scheduled. Bundles are specified by a list of dictionaries, e.g., ``[{"CPU": 1}, {"CPU": 1, "GPU": 1}]``). From ba21a9b2e1952569600d7c014ddf03f1d1893600 Mon Sep 17 00:00:00 2001 From: Potato Date: Sat, 6 Sep 2025 02:39:49 +0800 Subject: [PATCH 475/634] [CORE][DOC] Fix typos and grammatical issues in Ray Core documentation (#56279) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR systematically reviews and fixes typos, grammatical errors, and syntax issues across all 21 `.rst` files in the `doc/source/ray-core/` directory. The changes are minimal and surgical, preserving the original intent and voice of the documentation while improving clarity and correctness. ## Issues Fixed ### Grammatical Errors - **configure.rst**: Fixed "various way" → "various ways" and improved consistency with "Look" → "See" - **cross-language.rst**: Fixed capitalization "ray call" → "Ray calls" - **ray-generator.rst**: Fixed article error "the an application" → "an application" - **user-spawn-processes.rst**: Fixed verb agreement "spawns" → "spawn" ### Typos - **handling-dependencies.rst**: - Fixed "what to compress" → "want to compress" - Fixed repository name typo "example_respository" → "example_repository" ### Syntax Issues - **actors.rst**: Fixed reference link spacing in async actors section - **fault-tolerance.rst**: Fixed missing underscore in code marker `_node_affinity_scheduling_strategy_start__` → `__node_affinity_scheduling_strategy_start__` - **namespaces.rst**: - Fixed C++ code block syntax `.. code-block::` → `.. code-block:: c++` - Removed stray backtick at end of code line ## Files Reviewed All 21 `.rst` files were manually reviewed following a systematic methodology to check for: - reStructuredText syntax errors - Grammatical issues (subject-verb agreement, tense consistency, article usage) - Spelling and typos - Technical content accuracy - Style and formatting consistency ## Impact These changes improve the overall quality and professionalism of the Ray Core documentation without altering any functional content or breaking existing links. The documentation now has better readability and fewer distractions from minor errors. --------- Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- doc/source/ray-core/actors.rst | 2 +- doc/source/ray-core/configure.rst | 4 ++-- doc/source/ray-core/cross-language.rst | 2 +- doc/source/ray-core/fault-tolerance.rst | 2 +- doc/source/ray-core/handling-dependencies.rst | 4 ++-- doc/source/ray-core/namespaces.rst | 6 +++--- doc/source/ray-core/ray-generator.rst | 2 +- doc/source/ray-core/user-spawn-processes.rst | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/source/ray-core/actors.rst b/doc/source/ray-core/actors.rst index ebbabda54342..50add30ab2c5 100644 --- a/doc/source/ray-core/actors.rst +++ b/doc/source/ray-core/actors.rst @@ -412,7 +412,7 @@ For tasks classified as a single-threaded Actor or a multi-threaded Actor, Ray offers no mechanism for interruption. **Running async actor tasks**: -For Tasks classified as `async Actors <_async-actors>`, Ray seeks to cancel the associated `asyncio.Task`. +For Tasks classified as :ref:`async Actors `, Ray seeks to cancel the associated `asyncio.Task`. This cancellation approach aligns with the standards presented in `asyncio task cancellation `__. Note that `asyncio.Task` won't be interrupted in the middle of execution if you don't `await` within the async function. diff --git a/doc/source/ray-core/configure.rst b/doc/source/ray-core/configure.rst index 85a960fee13c..060ec42eb79c 100644 --- a/doc/source/ray-core/configure.rst +++ b/doc/source/ray-core/configure.rst @@ -5,7 +5,7 @@ Configuring Ray .. note:: For running Java applications, see `Java Applications`_. -This page discusses the various way to configure Ray, both from the Python API +This page discusses the various ways to configure Ray, both from the Python API and from the command line. Take a look at the ``ray.init`` `documentation `__ for a complete overview of the configurations. @@ -96,7 +96,7 @@ Change the *root temporary directory* by passing ``--temp-dir={your temp path}`` There currently isn't a stable way to change the root temporary directory when calling ``ray.init()``, but if you need to, you can provide the ``_temp_dir`` argument to ``ray.init()``. -Look :ref:`Logging Directory Structure ` for more details. +See :ref:`Logging Directory Structure ` for more details. .. _ray-ports: diff --git a/doc/source/ray-core/cross-language.rst b/doc/source/ray-core/cross-language.rst index bde0c2a50cfc..c2bf5dba990c 100644 --- a/doc/source/ray-core/cross-language.rst +++ b/doc/source/ray-core/cross-language.rst @@ -148,7 +148,7 @@ from the preceding Python class. Cross-language data serialization --------------------------------- -Ray automatically serializes and deserializes the arguments and return values of ray call +Ray automatically serializes and deserializes the arguments and return values of Ray calls if their types are the following: - Primitive data types diff --git a/doc/source/ray-core/fault-tolerance.rst b/doc/source/ray-core/fault-tolerance.rst index d9f9f329e0b6..4de1bab34677 100644 --- a/doc/source/ray-core/fault-tolerance.rst +++ b/doc/source/ray-core/fault-tolerance.rst @@ -69,7 +69,7 @@ It allows you to specify the affinity as a soft constraint so even if the target .. literalinclude:: doc_code/fault_tolerance_tips.py :language: python - :start-after: _node_affinity_scheduling_strategy_start__ + :start-after: __node_affinity_scheduling_strategy_start__ :end-before: __node_affinity_scheduling_strategy_end__ diff --git a/doc/source/ray-core/handling-dependencies.rst b/doc/source/ray-core/handling-dependencies.rst index 272cb5aebe36..993ac51ff644 100644 --- a/doc/source/ray-core/handling-dependencies.rst +++ b/doc/source/ray-core/handling-dependencies.rst @@ -847,7 +847,7 @@ Your ``runtime_env`` dictionary should contain: Check for hidden files and metadata directories in zipped dependencies. You can inspect a zip file's contents by running the ``zipinfo -1 zip_file_name.zip`` command in the Terminal. Some zipping methods can cause hidden files or metadata directories to appear in the zip file at the top level. - To avoid this, use the ``zip -r`` command directly on the directory you want to compress from its parent's directory. For example, if you have a directory structure such as: ``a/b`` and you what to compress ``b``, issue the ``zip -r b`` command from the directory ``a.`` + To avoid this, use the ``zip -r`` command directly on the directory you want to compress from its parent's directory. For example, if you have a directory structure such as: ``a/b`` and you want to compress ``b``, issue the ``zip -r b`` command from the directory ``a.`` If Ray detects more than a single directory at the top level, it will use the entire zip file instead of the top-level directory, which may lead to unexpected behavior. Currently, four types of remote URIs are supported for hosting ``working_dir`` and ``py_modules`` packages: @@ -859,7 +859,7 @@ Currently, four types of remote URIs are supported for hosting ``working_dir`` a - Example: - - ``runtime_env = {"working_dir": "https://github.com/example_username/example_respository/archive/HEAD.zip"}`` + - ``runtime_env = {"working_dir": "https://github.com/example_username/example_repository/archive/HEAD.zip"}`` - ``S3``: ``S3`` refers to URIs starting with ``s3://`` that point to compressed packages stored in `AWS S3 `_. To use packages via ``S3`` URIs, you must have the ``smart_open`` and ``boto3`` libraries (you can install them using ``pip install smart_open`` and ``pip install boto3``). diff --git a/doc/source/ray-core/namespaces.rst b/doc/source/ray-core/namespaces.rst index 6a502534bda2..8f0c1a6ab70c 100644 --- a/doc/source/ray-core/namespaces.rst +++ b/doc/source/ray-core/namespaces.rst @@ -161,7 +161,7 @@ the specified namespace, no matter what namespace of the current job is. .. tab-item:: C++ - .. code-block:: + .. code-block:: c++ // `ray start --head` has been run to launch a local cluster. ray::RayConfig config; @@ -169,8 +169,8 @@ the specified namespace, no matter what namespace of the current job is. // Create an actor with specified namespace. ray::Actor(RAY_FUNC(Counter::FactoryCreate)).SetName("my_actor", "actor_namespace").Remote(); // It is accessible in its namespace. - ray::GetActor("orange"); - ray::Shutdown();` + ray::GetActor("my_actor", "actor_namespace"); + ray::Shutdown(); Anonymous namespaces diff --git a/doc/source/ray-core/ray-generator.rst b/doc/source/ray-core/ray-generator.rst index da81bc22f7ca..6aba716d4834 100644 --- a/doc/source/ray-core/ray-generator.rst +++ b/doc/source/ray-core/ray-generator.rst @@ -97,7 +97,7 @@ Ray raises the exception. :start-after: __streaming_generator_exception_start__ :end-before: __streaming_generator_exception_end__ -In the above example, if the an application fails the task, Ray returns the object reference with an exception +In the above example, if an application fails the task, Ray returns the object reference with an exception in a correct order. For example, if Ray raises the exception after the second yield, the third ``next(gen)`` returns an object reference with an exception all the time. If a system error fails the task, (e.g., a node failure or worker process failure), ``next(gen)`` returns the object reference that contains the system level exception diff --git a/doc/source/ray-core/user-spawn-processes.rst b/doc/source/ray-core/user-spawn-processes.rst index 1302f0a7b16a..af9dd7b0d072 100644 --- a/doc/source/ray-core/user-spawn-processes.rst +++ b/doc/source/ray-core/user-spawn-processes.rst @@ -1,7 +1,7 @@ Lifetimes of a User-Spawn Process ================================= -When you spawns child processes from Ray workers, you are responsible for managing the lifetime of child processes. However, it is not always possible, especially when worker crashes and child processes are spawned from libraries (torch dataloader). +When you spawn child processes from Ray workers, you are responsible for managing the lifetime of child processes. However, it is not always possible, especially when worker crashes and child processes are spawned from libraries (torch dataloader). To avoid leaking user-spawned processes, Ray provides mechanisms to kill all user-spawned processes when a worker that starts it exits. This feature prevents GPU memory leaks from child processes (e.g., torch). From c01c880f54a3ed7e115bbab98f3547bc8877f26e Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Fri, 5 Sep 2025 11:58:00 -0700 Subject: [PATCH 476/634] [ci] raydepsets: single depset compilation with dependencies (#56263) Raydepsets has a single depset compilation mode which compiles ONLY the depset referenced by name In some cases depsets have dependencies (ex. expand and subset depend on source depsets) To allow for the depset dependencies to be compiled: Subset the build graph by including the user specified target depset and all ancestors execute all of the depsets in build graph order Added unit tests --------- Signed-off-by: elliot-barn --- ci/raydepsets/cli.py | 19 +++++++++++++------ ci/raydepsets/tests/test_cli.py | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index f7aa52c316f5..77733f025943 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -5,7 +5,7 @@ import click import runfiles -from networkx import DiGraph, topological_sort +from networkx import DiGraph, topological_sort, ancestors as networkx_ancestors from ci.raydepsets.workspace import Depset, Workspace @@ -60,10 +60,7 @@ def build( workspace_dir=workspace_dir, uv_cache_dir=uv_cache_dir, ) - if name: - manager.execute_single(_get_depset(manager.config.depsets, name)) - else: - manager.execute() + manager.execute(name) class DependencySetManager: @@ -100,7 +97,17 @@ def _build(self): else: raise ValueError(f"Invalid operation: {depset.operation}") - def execute(self): + def subgraph_dependency_nodes(self, depset_name: str): + dependency_nodes = networkx_ancestors(self.build_graph, depset_name) + nodes = dependency_nodes | {depset_name} + self.build_graph = self.build_graph.subgraph(nodes).copy() + + def execute(self, single_depset_name: Optional[str] = None): + if single_depset_name: + # check if the depset exists + _get_depset(self.config.depsets, single_depset_name) + self.subgraph_dependency_nodes(single_depset_name) + for node in topological_sort(self.build_graph): depset = self.build_graph.nodes[node]["depset"] self.execute_single(depset) diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index dc57eda57e52..4a7fa2d5237b 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -437,6 +437,24 @@ def test_execute(self): with tempfile.TemporaryDirectory() as tmpdir: copy_data_to_tmpdir(tmpdir) + def test_execute_single_depset(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = _create_test_manager(tmpdir) + manager.execute(single_depset_name="general_depset__py311_cpu") + assert ( + manager.build_graph.nodes["general_depset__py311_cpu"]["operation"] + == "compile" + ) + assert len(manager.build_graph.nodes()) == 1 + + def test_execute_single_depset_that_does_not_exist(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = _create_test_manager(tmpdir) + with self.assertRaises(KeyError): + manager.execute(single_depset_name="fake_depset") + def test_expand(self): with tempfile.TemporaryDirectory() as tmpdir: copy_data_to_tmpdir(tmpdir) From ccb4ce9a5da019bb7a5f4a562a54d64e800bcbc1 Mon Sep 17 00:00:00 2001 From: Ping Dai Date: Sat, 6 Sep 2025 03:48:33 +0800 Subject: [PATCH 477/634] [core][gpu-objects] Add warning when GPU object refs passed back to the same actor (#55639) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #55511 Below is my complete test script. ```python import ray import torch from ray.experimental.collective import create_collective_group @ray.remote(enable_tensor_transport=True) class Actor: def __init__(self): self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @ray.method(tensor_transport="gloo") def tensor(self): return torch.zeros(1, device=self.device) def increment(self, t): t = t + 1 return t a = Actor.remote() b = Actor.remote() create_collective_group([a, b], backend="torch_gloo") t = a.tensor.remote() t1 = a.increment.remote(t) t2 = b.increment.remote(t) result1, result2 = ray.get([t1, t2]) try: assert torch.allclose(result1, result2), "The two tensor results are not equal" print("Assertion passed: The values of t1 and t2 are equal") except AssertionError as e: print(f"Assertion failed: {e}") ray.shutdown() ``` Log： image --------- Signed-off-by: daiping8 Signed-off-by: Ping Dai Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Stephanie Wang --- .../gpu_object_manager/gpu_object_manager.py | 38 +++++++++++++++++-- .../gpu_objects/test_gpu_objects_gloo.py | 24 ++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py index 5ce2a345b225..87d089880a98 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, List +import warnings +from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, List, Set import threading import ray @@ -14,7 +15,7 @@ import torch # GPUObjectMeta is a named tuple containing the source actor, tensor transport -# backend, and tensor metadata. +# backend, tensor metadata, and other information that needs to be recorded. # - The tensor transport backend is the backend used to transport the tensors. # Currently, the supported backends are "nccl" and "torch_gloo". # - The tensor metadata is a list of tuples, each containing the shape and dtype @@ -25,6 +26,10 @@ class GPUObjectMeta(NamedTuple): # `ray.util.collective.types.Backend`. tensor_transport_backend: str tensor_transport_meta: "TensorTransportMetadata" + # sent_dest_actors tracks the set of actor IDs that this object has been sent to. + sent_dest_actors: Set[str] + # sent_to_src_actor_and_others_warned indicates whether the object has already triggered a warning about being sent back to the source actor and other actors simultaneously. + sent_to_src_actor_and_others_warned: bool # TODO(swang): Uncomment and add an API docs page and example usage. @@ -124,6 +129,8 @@ def add_gpu_object_ref( src_actor=src_actor, tensor_transport_backend=tensor_transport_backend, tensor_transport_meta=tensor_meta, + sent_dest_actors=set(), + sent_to_src_actor_and_others_warned=False, ) def _get_gpu_object_metadata(self, obj_ref: ObjectRef) -> GPUObjectMeta: @@ -203,13 +210,38 @@ def trigger_out_of_band_tensor_transfer( src_actor = gpu_object_meta.src_actor tensor_transport_meta = gpu_object_meta.tensor_transport_meta + + obj_id = obj_ref.hex() + + # Update the set of destination actors for this object + # The set inside NamedTuple is mutable, so we can modify it directly + gpu_object_meta.sent_dest_actors.add(dst_actor._actor_id) + # Check if a warning should be triggered for this object: + # 1. object has not triggered a warning yet. + # 2. object is sent back to its source actor. + # 3. object is also sent to at least one other actor + if ( + not gpu_object_meta.sent_to_src_actor_and_others_warned + and src_actor._actor_id in gpu_object_meta.sent_dest_actors + and len(gpu_object_meta.sent_dest_actors) > 1 + ): + warnings.warn( + f"GPU ObjectRef({obj_id}) is being passed back to the actor that created it {src_actor}. " + "Note that GPU objects are mutable. If the tensor is modified, Ray's internal copy will also be updated, and subsequent passes to other actors " + "will receive the updated version instead of the original.", + UserWarning, + ) + # Mark the object as warned by creating a new NamedTuple instance + self.managed_gpu_object_metadata[obj_id] = gpu_object_meta._replace( + sent_to_src_actor_and_others_warned=True + ) + if src_actor._actor_id == dst_actor._actor_id: # If the source and destination actors are the same, the tensors can # be transferred intra-process, so we skip the out-of-band tensor # transfer. continue - obj_id = obj_ref.hex() tensor_transport_manager = get_tensor_transport_manager( gpu_object_meta.tensor_transport_backend ) diff --git a/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py b/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py index 7ad5a4a3081d..0d17c4c76c91 100644 --- a/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py +++ b/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py @@ -836,6 +836,30 @@ def gc(obj_id): assert not gpu_object_store.has_object(obj_id2) +def test_send_back_and_dst_warning(ray_start_regular): + # Test warning when object is sent back to the src actor and to dst actors + world_size = 2 + actors = [GPUTestActor.remote() for _ in range(world_size)] + create_collective_group(actors, backend="torch_gloo") + + src_actor, dst_actor = actors[0], actors[1] + + tensor = torch.tensor([1, 2, 3]) + + warning_message = r"GPU ObjectRef\(.+\)" + + with pytest.warns(UserWarning, match=warning_message): + t = src_actor.echo.remote(tensor) + t1 = src_actor.echo.remote(t) # Sent back to the source actor + t2 = dst_actor.echo.remote(t) # Also sent to another actor + ray.get([t1, t2]) + + # Second transmission of ObjectRef `t` to `dst_actor` should not trigger a warning + # Verify no `pytest.warns` context is used here because no warning should be raised + t3 = dst_actor.echo.remote(t) + ray.get(t3) + + def test_duplicate_objectref_transfer(ray_start_regular): world_size = 2 actors = [GPUTestActor.remote() for _ in range(world_size)] From 61ad346708213ca68e9d2bc5f0b92dfd842d6c38 Mon Sep 17 00:00:00 2001 From: Srinath Krishnamachari <68668616+srinathk10@users.noreply.github.com> Date: Fri, 5 Sep 2025 14:42:30 -0700 Subject: [PATCH 478/634] [Data] Bump test_json timeout (#56267) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? [Data] Bump test_json timeout Bump up test_json timeout to 6 minutes. In 3 minutes, it got 2/3rd of the way. https://buildkite.com/anyscale/rayturbo/builds/6172#0199175c-50ee-4076-a0dc-a6da0b207ecc ``` [2025-09-05T01:43:06Z] ================================================================================   | [2025-09-05T01:43:06Z] ==================== Test output for //python/ray/data:test_json:   | [2025-09-05T01:43:06Z] /opt/miniforge/lib/python3.9/site-packages/paramiko/pkey.py:82: CryptographyDeprecationWarning: TripleDES has been moved to cryptography.hazmat.decrepit.ciphers.algorithms.TripleDES and will be removed from cryptography.hazmat.primitives.ciphers.algorithms in 48.0.0.   | [2025-09-05T01:43:06Z] "cipher": algorithms.TripleDES,   | [2025-09-05T01:43:06Z] /opt/miniforge/lib/python3.9/site-packages/paramiko/transport.py:253: CryptographyDeprecationWarning: TripleDES has been moved to cryptography.hazmat.decrepit.ciphers.algorithms.TripleDES and will be removed from cryptography.hazmat.primitives.ciphers.algorithms in 48.0.0.   | [2025-09-05T01:43:06Z] "class": algorithms.TripleDES,   | [2025-09-05T01:43:06Z] ============================= test session starts ==============================   | [2025-09-05T01:43:06Z] platform linux -- Python 3.9.23, pytest-7.4.4, pluggy-1.3.0 -- /opt/miniforge/bin/python3   | [2025-09-05T01:43:06Z] cachedir: .pytest_cache   | [2025-09-05T01:43:06Z] rootdir: /root/.cache/bazel/_bazel_root/1df605deb6d24fc8068f6e25793ec703/execroot/io_ray   | [2025-09-05T01:43:06Z] configfile: pytest.ini   | [2025-09-05T01:43:06Z] plugins: repeat-0.9.3, anyio-3.7.1, fugue-0.8.7, aiohttp-1.1.0, asyncio-0.17.2, docker-tools-3.1.3, forked-1.4.0, pytest_httpserver-1.1.3, lazy-fixtures-1.1.2, mock-3.14.0, remotedata-0.3.2, rerunfailures-11.1.2, shutil-1.8.1, sphinx-0.5.1.dev0, sugar-0.9.5, timeout-2.1.0, virtualenv-1.8.1, typeguard-2.13.3   | [2025-09-05T01:43:06Z] asyncio: mode=auto   | [2025-09-05T01:43:06Z] timeout: 180.0s   | [2025-09-05T01:43:06Z] timeout method: signal   | [2025-09-05T01:43:06Z] timeout func_only: False   | [2025-09-05T01:43:06Z] collecting ... collected 102 items   | [2025-09-05T01:43:06Z]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioning[None] PASSED [ 0%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioning[134217728] PASSED [ 1%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read[None-None-local_path-None] PASSED [ 2%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read[None-local_fs-local_path-None] PASSED [ 3%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read[None-s3_fs-s3_path-s3_server] PASSED [ 4%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read[134217728-None-local_path-None] PASSED [ 5%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read[134217728-local_fs-local_path-None] PASSED [ 6%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read[134217728-s3_fs-s3_path-s3_server] PASSED [ 7%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_read_json_ignore_missing_paths[None-True] PASSED [ 8%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_read_json_ignore_missing_paths[None-False] PASSED [ 9%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_read_json_ignore_missing_paths[134217728-True] PASSED [ 10%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_read_json_ignore_missing_paths[134217728-False] PASSED [ 11%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_zipped_json_read[None] PASSED [ 12%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_zipped_json_read[134217728] PASSED [ 13%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_read_json_fallback_from_pyarrow_failure[None] PASSED [ 14%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_read_json_fallback_from_pyarrow_failure[134217728] PASSED [ 15%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_meta_provider[None-None-local_path-None] PASSED [ 16%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_meta_provider[None-local_fs-local_path-None] PASSED [ 17%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_meta_provider[None-s3_fs-s3_path-s3_server] PASSED [ 18%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_meta_provider[134217728-None-local_path-None] PASSED [ 19%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_meta_provider[134217728-local_fs-local_path-None] PASSED [ 20%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_meta_provider[134217728-s3_fs-s3_path-s3_server] PASSED [ 21%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_read_options[None-None-local_path-None] PASSED [ 22%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_read_options[None-local_fs-local_path-None] PASSED [ 23%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_read_options[None-s3_fs-s3_path-s3_server] PASSED [ 24%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_read_options[134217728-None-local_path-None] PASSED [ 25%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_read_options[134217728-local_fs-local_path-None] PASSED [ 26%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_read_options[134217728-s3_fs-s3_path-s3_server] PASSED [ 27%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_parse_options[None-None-local_path-None] PASSED [ 28%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_parse_options[None-local_fs-local_path-None] PASSED [ 29%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_parse_options[None-s3_fs-s3_path-s3_server] PASSED [ 30%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_parse_options[134217728-None-local_path-None] PASSED [ 31%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_parse_options[134217728-local_fs-local_path-None] PASSED [ 32%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_with_parse_options[134217728-s3_fs-s3_path-s3_server] PASSED [ 33%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[None-hive-None-local_path-None] PASSED [ 34%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[None-hive-local_fs-local_path-None] PASSED [ 35%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[None-hive-s3_fs-s3_path-s3_server] PASSED [ 36%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[None-dir-None-local_path-None] PASSED [ 37%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[None-dir-local_fs-local_path-None] PASSED [ 38%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[None-dir-s3_fs-s3_path-s3_server] PASSED [ 39%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[134217728-hive-None-local_path-None] PASSED [ 40%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[134217728-hive-local_fs-local_path-None] PASSED [ 41%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[134217728-hive-s3_fs-s3_path-s3_server] PASSED [ 42%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[134217728-dir-None-local_path-None] PASSED [ 43%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[134217728-dir-local_fs-local_path-None] PASSED [ 44%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_partitioned_with_filter[134217728-dir-s3_fs-s3_path-s3_server] PASSED [ 45%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_jsonl_lists[None-None] PASSED [ 46%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_jsonl_lists[None-1] PASSED [ 47%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_jsonl_lists[None-3] PASSED [ 48%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_jsonl_lists[134217728-None] PASSED [ 49%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_jsonl_lists[134217728-1] PASSED [ 50%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_jsonl_lists[134217728-3] PASSED [ 50%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_jsonl_mixed_types[None] PASSED [ 51%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_jsonl_mixed_types[134217728] PASSED [ 52%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_write[None] PASSED [ 53%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_write[134217728] PASSED [ 54%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_roundtrip[None-None] PASSED [ 55%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_roundtrip[None-2] PASSED [ 56%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_roundtrip[134217728-None] PASSED [ 57%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_roundtrip[134217728-2] PASSED [ 58%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_small_file_unit_block_size[None-None-local_path-None] PASSED [ 59%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_small_file_unit_block_size[None-local_fs-local_path-None] PASSED [ 60%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_small_file_unit_block_size[None-s3_fs-s3_path-s3_server] PASSED [ 61%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_small_file_unit_block_size[134217728-None-local_path-None] PASSED [ 62%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_small_file_unit_block_size[134217728-local_fs-local_path-None] PASSED [ 63%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_small_file_unit_block_size[134217728-s3_fs-s3_path-s3_server] PASSED [ 64%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_file_larger_than_block_size[None-None-local_path-None] PASSED [ 65%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_file_larger_than_block_size[None-local_fs-local_path-None] FAILED [ 66%]   | [2025-09-05T01:43:06Z] python/ray/data/tests/test_json.py::test_json_read_file_larger_than_block_size[None-s3_fs-s3_path-s3_server] -- Test timed out at 2025-09-05 01:43:03 UTC --   | [2025-09-05T01:43:06Z] ================================================================================   |   ``` ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Srinath Krishnamachari --- python/ray/data/tests/test_json.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/ray/data/tests/test_json.py b/python/ray/data/tests/test_json.py index 891f88ca46ea..7f5364a0f2b7 100644 --- a/python/ray/data/tests/test_json.py +++ b/python/ray/data/tests/test_json.py @@ -31,6 +31,9 @@ from ray.data.tests.test_partitioning import PathPartitionEncoder from ray.tests.conftest import * # noqa +# Set the test timeout to 6 minutes +pytestmark = pytest.mark.timeout(360) + def test_json_read_partitioning( ray_start_regular_shared, tmp_path, target_max_block_size_infinite_or_default From befc4baa671a31f412c61ed97e38be15deaed4e5 Mon Sep 17 00:00:00 2001 From: Potato Date: Sat, 6 Sep 2025 05:49:54 +0800 Subject: [PATCH 479/634] [DOC] Fix grammar, syntax, and formatting issues in ray-more-libs documentation (#56068) Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- doc/source/ray-more-libs/dask-on-ray.rst | 4 ++-- .../data_juicer_distributed_data_processing.md | 14 +++++++------- doc/source/ray-more-libs/joblib.rst | 4 ++-- doc/source/ray-more-libs/mars-on-ray.rst | 12 +++++++----- doc/source/ray-more-libs/modin/index.rst | 8 ++++---- doc/source/ray-more-libs/multiprocessing.rst | 2 +- doc/source/ray-more-libs/ray-collective.rst | 8 ++++---- doc/source/ray-more-libs/raydp.rst | 8 ++++---- 8 files changed, 31 insertions(+), 29 deletions(-) diff --git a/doc/source/ray-more-libs/dask-on-ray.rst b/doc/source/ray-more-libs/dask-on-ray.rst index 64dc2cdb6a5a..4bd68bce4abd 100644 --- a/doc/source/ray-more-libs/dask-on-ray.rst +++ b/doc/source/ray-more-libs/dask-on-ray.rst @@ -118,13 +118,13 @@ Best Practice for Large Scale workloads For Ray 1.3, the default scheduling policy is to pack tasks to the same node as much as possible. It is more desirable to spread tasks if you run a large scale / memory intensive Dask on Ray workloads. -In this case, there are two recommended setup. +In this case, there are two recommended setups. - Reducing the config flag `scheduler_spread_threshold` to tell the scheduler to prefer spreading tasks across the cluster instead of packing. - Setting the head node's `num-cpus` to 0 so that tasks are not scheduled on a head node. .. code-block:: bash - # Head node. Set `num_cpus=0` to avoid tasks are being scheduled on a head node. + # Head node. Set `num_cpus=0` to avoid tasks being scheduled on a head node. RAY_scheduler_spread_threshold=0.0 ray start --head --num-cpus=0 # Worker node. diff --git a/doc/source/ray-more-libs/data_juicer_distributed_data_processing.md b/doc/source/ray-more-libs/data_juicer_distributed_data_processing.md index 926368ffa65d..f48238127a7f 100644 --- a/doc/source/ray-more-libs/data_juicer_distributed_data_processing.md +++ b/doc/source/ray-more-libs/data_juicer_distributed_data_processing.md @@ -18,12 +18,12 @@ See the [Data-Juicer 2.0: Cloud-Scale Adaptive Data Processing for Foundation Mo ### Ray mode in Data-Juicer -- For most implementations of Data-Juicer [operators](https://github.com/modelscope/data-juicer/blob/main/docs/Operators.md), the core processing functions are engine-agnostic. Operators manage interoperability is primarily in [RayDataset](https://github.com/modelscope/data-juicer/blob/main/data_juicer/core/data/ray_dataset.py) and [RayExecutor](https://github.com/modelscope/data-juicer/blob/main/data_juicer/core/executor/ray_executor.py), which are subclasses of the base `DJDataset` and `BaseExecutor`, respectively, and support both Ray [Tasks](ray-remote-functions) and [Actors](actor-guide). -- The exception is the deduplication operators, which are challenging to scale in standalone mode. The names of these operators follow the pattern of [`ray_xx_deduplicator`](https://github.com/modelscope/data-juicer/blob/main//data_juicer/ops/deduplicator/). +- For most implementations of Data-Juicer [operators](https://github.com/modelscope/data-juicer/blob/main/docs/Operators.md), the core processing functions are engine-agnostic. Operator interoperability is managed primarily in [RayDataset](https://github.com/modelscope/data-juicer/blob/main/data_juicer/core/data/ray_dataset.py) and [RayExecutor](https://github.com/modelscope/data-juicer/blob/main/data_juicer/core/executor/ray_executor.py), which are subclasses of the base `DJDataset` and `BaseExecutor`, respectively, and support both Ray [Tasks](ray-remote-functions) and [Actors](actor-guide). +- The exception is the deduplication operators, which are challenging to scale in standalone mode. The names of these operators follow the pattern of [`ray_xx_deduplicator`](https://github.com/modelscope/data-juicer/blob/main/data_juicer/ops/deduplicator/). ### Subset splitting -When a cluster has tens of thousands of nodes but only a few dataset files, Ray splits the dataset files according to available resources and distribute the blocks across all nodes, incurring high network communication costs and reduced CPU utilization. For more details, see [Ray's `_autodetect_parallelism` function](https://github.com/ray-project/ray/blob/2dbd08a46f7f08ea614d8dd20fd0bca5682a3078/python/ray/data/_internal/util.py#L201-L205) and [tuning output blocks for Ray](read_output_blocks). +When a cluster has tens of thousands of nodes but only a few dataset files, Ray splits the dataset files according to available resources and distributes the blocks across all nodes, incurring high network communication costs and reducing CPU utilization. For more details, see [Ray's `_autodetect_parallelism` function](https://github.com/ray-project/ray/blob/2dbd08a46f7f08ea614d8dd20fd0bca5682a3078/python/ray/data/_internal/util.py#L201-L205) and [tuning output blocks for Ray](read_output_blocks). This default execution plan can be quite inefficient especially for scenarios with a large number of nodes. To optimize performance for such cases, Data-Juicer automatically splits the original dataset into smaller files in advance, taking into consideration the features of Ray and Arrow. When you encounter such performance issues, you can use this feature or split the dataset according to your own preferences. In this auto-split strategy, the single file size is about 128 MB, and the result should ensure that the number of sub-files after splitting is at least twice the total number of CPU cores available in the cluster. @@ -93,7 +93,7 @@ demos/process_on_ray ### Running Example of Ray Mode -In the `demo.yaml` config file, it sets the executor type to "ray" and specify an automatic Ray address. +In the `demo.yaml` config file, it sets the executor type to "ray" and specifies an automatic Ray address. ```yaml ... @@ -115,11 +115,11 @@ python tools/process_data.py --config demos/process_on_ray/configs/demo.yaml dj-process --config demos/process_on_ray/configs/demo.yaml ``` -Data-Juicer processes the demo dataset with the demo config file and export the result datasets to the directory specified by the `export_path` argument in the config file. +Data-Juicer processes the demo dataset with the demo config file and exports the result datasets to the directory specified by the `export_path` argument in the config file. ### Running Example of Distributed Deduplication -In the `dedup.yaml` config file, it sets the executor type to "ray" and specify an automatic Ray address. +In the `dedup.yaml` config file, it sets the executor type to "ray" and specifies an automatic Ray address. And it uses a dedicated distributed version of MinHash deduplication operator to deduplicate the dataset. ```yaml @@ -147,4 +147,4 @@ python tools/process_data.py --config demos/process_on_ray/configs/dedup.yaml dj-process --config demos/process_on_ray/configs/dedup.yaml ``` -Data-Juicer deduplicates the demo dataset with the demo config file and export the result datasets to the directory specified by the `export_path` argument in the config file. +Data-Juicer deduplicates the demo dataset with the demo config file and exports the result datasets to the directory specified by the `export_path` argument in the config file. diff --git a/doc/source/ray-more-libs/joblib.rst b/doc/source/ray-more-libs/joblib.rst index 3d9e5d6f818c..cb3f20c6d2a9 100644 --- a/doc/source/ray-more-libs/joblib.rst +++ b/doc/source/ray-more-libs/joblib.rst @@ -52,7 +52,7 @@ a multi-node Ray cluster instead. search.fit(digits.data, digits.target) You can also set the ``ray_remote_args`` argument in ``parallel_backend`` to :func:`configure -the Ray Actors ` making up the Pool. This can be used to eg. :ref:`assign resources +the Ray Actors ` making up the Pool. This can be used to e.g., :ref:`assign resources to Actors, such as GPUs `. .. code-block:: python @@ -67,7 +67,7 @@ Run on a Cluster This section assumes that you have a running Ray cluster. To start a Ray cluster, see the :ref:`cluster setup ` instructions. -To connect a scikit-learn to a running Ray cluster, you have to specify the address of the +To connect scikit-learn to a running Ray cluster, you have to specify the address of the head node by setting the ``RAY_ADDRESS`` environment variable. You can also start Ray manually by calling ``ray.init()`` (with any of its supported diff --git a/doc/source/ray-more-libs/mars-on-ray.rst b/doc/source/ray-more-libs/mars-on-ray.rst index 129fa94724f7..4baf6e38538d 100644 --- a/doc/source/ray-more-libs/mars-on-ray.rst +++ b/doc/source/ray-more-libs/mars-on-ray.rst @@ -6,11 +6,11 @@ Using Mars on Ray .. _`issue on GitHub`: https://github.com/mars-project/mars/issues -`Mars`_ is a tensor-based unified framework for large-scale data computation which scales Numpy, Pandas and Scikit-learn. +`Mars`_ is a tensor-based unified framework for large-scale data computation which scales NumPy, Pandas and Scikit-learn. Mars on Ray makes it easy to scale your programs with a Ray cluster. Currently Mars on Ray supports both Ray actors -and tasks as execution backend. The task will be scheduled by mars scheduler if Ray actors is used. This mode can reuse -all mars scheduler optimizations. If ray tasks mode is used, all tasks will be scheduled by ray, which can reuse failover and -pipeline capabilities provided by ray futures. +and tasks as an execution backend. The task will be scheduled by Mars scheduler if Ray actors are used. This mode can reuse +all Mars scheduler optimizations. If Ray tasks mode is used, all tasks will be scheduled by Ray, which can reuse failover and +pipeline capabilities provided by Ray futures. .. _`Mars`: https://mars-project.readthedocs.io/en/latest/ @@ -75,4 +75,6 @@ Interact with Dataset: df2 = ds.to_mars() print(df2.head(5).execute()) -Refer to _`Mars on Ray`: https://mars-project.readthedocs.io/en/latest/installation/ray.html#mars-ray for more information. +Refer to `Mars on Ray`_ for more information. + +.. _`Mars on Ray`: https://mars-project.readthedocs.io/en/latest/installation/ray.html#mars-ray diff --git a/doc/source/ray-more-libs/modin/index.rst b/doc/source/ray-more-libs/modin/index.rst index 04bfbcb82064..ad88c31d49c6 100644 --- a/doc/source/ray-more-libs/modin/index.rst +++ b/doc/source/ray-more-libs/modin/index.rst @@ -20,7 +20,7 @@ You can use Modin on Ray with your laptop or cluster. In this document, we show instructions for how to set up a Modin compatible Ray cluster and connect Modin to Ray. -.. note:: In previous versions of Modin, you had to initialize Ray before importing Modin. As of Modin 0.9.0, This is no longer the case. +.. note:: In previous versions of Modin, you had to initialize Ray before importing Modin. As of Modin 0.9.0, this is no longer the case. Using Modin with Ray's autoscaler --------------------------------- @@ -54,7 +54,7 @@ and operate on data with Ray. Dataframe operations '''''''''''''''''''' -The Modin Dataframe uses Ray tasks to perform data manipulations. Ray Tasks have +The Modin Dataframe uses Ray Tasks to perform data manipulations. Ray Tasks have a number of benefits over the actor model for data manipulation: - Multiple tasks may be manipulating the same objects simultaneously @@ -63,7 +63,7 @@ a number of benefits over the actor model for data manipulation: - As new workers come online the shuffling of data will happen as tasks are scheduled on the new node - Identical partitions need not be replicated, especially beneficial for operations - that selectively mutate the data (e.g. ``fillna``). + that selectively mutate the data (e.g., ``fillna``). - Finer grained parallelism with finer grained placement control Machine Learning @@ -71,7 +71,7 @@ Machine Learning Modin uses Ray Actors for the machine learning support it currently provides. Modin's implementation of XGBoost is able to spin up one actor for each node -and aggregate all of the partitions on that node to the XGBoost Actor. Modin +and aggregate all of the partitions on that node to the XGBoost actor. Modin is able to specify precisely the node IP for each actor on creation, giving fine-grained control over placement - a must for distributed training performance. diff --git a/doc/source/ray-more-libs/multiprocessing.rst b/doc/source/ray-more-libs/multiprocessing.rst index d57c4db54e0a..05c153c0472f 100644 --- a/doc/source/ray-more-libs/multiprocessing.rst +++ b/doc/source/ray-more-libs/multiprocessing.rst @@ -5,7 +5,7 @@ Distributed multiprocessing.Pool .. _`issue on GitHub`: https://github.com/ray-project/ray/issues -Ray supports running distributed python programs with the `multiprocessing.Pool API`_ +Ray supports running distributed Python programs with the `multiprocessing.Pool API`_ using `Ray Actors `__ instead of local processes. This makes it easy to scale existing applications that use ``multiprocessing.Pool`` from a single node to a cluster. diff --git a/doc/source/ray-more-libs/ray-collective.rst b/doc/source/ray-more-libs/ray-collective.rst index 46df357faa3f..c0c5d4f6afaf 100644 --- a/doc/source/ray-more-libs/ray-collective.rst +++ b/doc/source/ray-more-libs/ray-collective.rst @@ -185,19 +185,19 @@ remote actors. Refer to `APIs <#api-reference>`_ for the detailed descriptions o results = ray.get([w.compute.remote() for w in workers]) Note that for the same set of actors/task processes, multiple collective groups can be constructed, with ``group_name`` as their unique identifier. -This enables to specify complex communication patterns between different (sub)set of processes. +This enables specifying complex communication patterns between different (sub)set of processes. Collective Communication ^^^^^^^^^^^^^^^^^^^^^^^^ Check `the support matrix <#collective-primitives-support-matrix>`_ for the current status of supported collective calls and backends. -Note that the current set of collective communication API are imperative, and exhibit the following behaviours: +Note that the current set of collective communication APIs are imperative, and exhibit the following behaviours: * All the collective APIs are synchronous blocking calls * Since each API only specifies a part of the collective communication, the API is expected to be called by each participating process of the (pre-declared) collective group. - Upon all the processes have made the call and rendezvous with each other, the collective communication happens and proceeds. + Once all the processes have made the call and rendezvous with each other, the collective communication happens and proceeds. * The APIs are imperative and the communication happens out-of-band --- they need to be used inside the collective process (actor/task) code. An example of using ``ray.util.collective.allreduce`` is below: @@ -351,7 +351,7 @@ The following links provide helpful resources on how to efficiently leverage the * `More running examples `_ under ``ray.util.collective.examples``. -* `Scaling up the Spacy Name Entity Recognition (NER) pipeline `_ using Ray collective library. +* `Scaling up the spaCy Named Entity Recognition (NER) pipeline `_ using Ray collective library. * `Implementing the AllReduce strategy `_ for data-parallel distributed ML training. API References diff --git a/doc/source/ray-more-libs/raydp.rst b/doc/source/ray-more-libs/raydp.rst index 8ffcc70966d6..a227ef1f6046 100644 --- a/doc/source/ray-more-libs/raydp.rst +++ b/doc/source/ray-more-libs/raydp.rst @@ -8,7 +8,7 @@ RayDP combines your Spark and Ray clusters, making it easy to do large scale data processing using the PySpark API and seamlessly use that data to train your models using TensorFlow and PyTorch. -For more information and examples, see the RayDP Github page: +For more information and examples, see the RayDP GitHub page: https://github.com/oap-project/raydp ================ @@ -17,7 +17,7 @@ Installing RayDP RayDP can be installed from PyPI and supports PySpark 3.0 and 3.1. -.. code-block bash +.. code-block:: bash pip install raydp @@ -31,7 +31,7 @@ RayDP can be installed from PyPI and supports PySpark 3.0 and 3.1. Creating a Spark Session ======================== -To create a spark session, call ``raydp.init_spark`` +To create a Spark session, call ``raydp.init_spark`` For example, @@ -123,7 +123,7 @@ PyTorch. from raydp.utils import random_split train_df, test_df = random_split(df, [0.7, 0.3]) - # PyTorch Code + # PyTorch Code import torch class LinearModel(torch.nn.Module): def __init__(self): From beb70bfac3d785a1ce044bf1ed52ff95206d76e5 Mon Sep 17 00:00:00 2001 From: gangsf Date: Fri, 5 Sep 2025 15:26:08 -0700 Subject: [PATCH 480/634] [Core] Fix ABFSS (Azure Blob File System Secure) protocol support problems during E2E test in Anyscale environment (#56188) Signed-off-by: Gang Zhao Co-authored-by: Gang Zhao --- python/ray/_private/runtime_env/protocol.py | 58 +++++++++++++------ .../ray/tests/test_runtime_env_packaging.py | 43 ++++++++++++++ 2 files changed, 83 insertions(+), 18 deletions(-) diff --git a/python/ray/_private/runtime_env/protocol.py b/python/ray/_private/runtime_env/protocol.py index d5db021f75c8..12d562c6159a 100644 --- a/python/ray/_private/runtime_env/protocol.py +++ b/python/ray/_private/runtime_env/protocol.py @@ -1,5 +1,6 @@ import enum import os +from urllib.parse import urlparse class ProtocolsProvider: @@ -131,36 +132,57 @@ def _handle_abfss_protocol(cls): Raises: ImportError: If required dependencies are not installed. - ValueError: If required environment variables are not set. + ValueError: If the ABFSS URI format is invalid. """ try: + import adlfs from azure.identity import DefaultAzureCredential - from azure.storage.blob import BlobServiceClient # noqa: F401 - from smart_open import open as open_file except ImportError: raise ImportError( - "You must `pip install azure-storage-blob azure-identity smart_open[azure]` " + "You must `pip install adlfs azure-identity` " "to fetch URIs in Azure Blob File System Secure. " + cls._MISSING_DEPENDENCIES_WARNING ) - # Define authentication variable - azure_storage_account_name = os.getenv("AZURE_STORAGE_ACCOUNT") + def open_file(uri, mode, *, transport_params=None): + # Parse and validate the ABFSS URI + parsed = urlparse(uri) - if not azure_storage_account_name: - raise ValueError( - "Azure Blob File System Secure authentication requires " - "AZURE_STORAGE_ACCOUNT environment variable to be set." - ) + # Validate ABFSS URI format: abfss://container@account.dfs.core.windows.net/path + if not parsed.netloc or "@" not in parsed.netloc: + raise ValueError( + f"Invalid ABFSS URI format - missing container@account: {uri}" + ) - account_url = f"https://{azure_storage_account_name}.dfs.core.windows.net/" - transport_params = { - "client": BlobServiceClient( - account_url=account_url, credential=DefaultAzureCredential() + container_part, hostname_part = parsed.netloc.split("@", 1) + + # Validate container name (must be non-empty) + if not container_part: + raise ValueError( + f"Invalid ABFSS URI format - empty container name: {uri}" + ) + + # Validate hostname format + if not hostname_part or not hostname_part.endswith(".dfs.core.windows.net"): + raise ValueError( + f"Invalid ABFSS URI format - invalid hostname (must end with .dfs.core.windows.net): {uri}" + ) + + # Extract and validate account name + azure_storage_account_name = hostname_part.split(".")[0] + if not azure_storage_account_name: + raise ValueError( + f"Invalid ABFSS URI format - empty account name: {uri}" + ) + + # Handle ABFSS URI with adlfs + filesystem = adlfs.AzureBlobFileSystem( + account_name=azure_storage_account_name, + credential=DefaultAzureCredential(), ) - } + return filesystem.open(uri, mode) - return open_file, transport_params + return open_file, None @classmethod def download_remote_uri(cls, protocol: str, source_uri: str, dest_file: str): @@ -204,7 +226,7 @@ def open_file(uri, mode, *, transport_params=None): ) with open_file(source_uri, "rb", transport_params=tp) as fin: - with open_file(dest_file, "wb") as fout: + with open(dest_file, "wb") as fout: fout.write(fin.read()) diff --git a/python/ray/tests/test_runtime_env_packaging.py b/python/ray/tests/test_runtime_env_packaging.py index fdaf2450df92..98e72caf4f07 100644 --- a/python/ray/tests/test_runtime_env_packaging.py +++ b/python/ray/tests/test_runtime_env_packaging.py @@ -633,6 +633,49 @@ def test_parse_gcs_uri(self, gcs_uri): assert package_name == gcs_uri.split("/")[-1] +class TestAbfssProtocol: + """Test ABFSS protocol implementation.""" + + def test_abfss_protocol_handler_with_invalid_uris(self, tmp_path): + """Test that ABFSS protocol handler raises ValueError for invalid URIs.""" + import unittest.mock as mock + + invalid_uris = [ + "abfss://@account.dfs.core.windows.net/file.zip", # Empty container name + "abfss://container@.dfs.core.windows.net/file.zip", # Empty account name + "abfss://container@account.blob.core.windows.net/file.zip", # Wrong endpoint + "abfss://container@account.core.windows.net/file.zip", # Missing .dfs + "abfss://account.dfs.core.windows.net/file.zip", # Missing container@ + "abfss://container", # Missing @ and hostname + "abfss://", # Empty netloc + ] + + dest_file = tmp_path / "test_download.zip" + + # Mock adlfs and azure.identity modules in sys.modules to avoid import errors in CI + import sys + + mock_adlfs_module = mock.MagicMock() + mock_azure_identity_module = mock.MagicMock() + + with mock.patch.dict( + sys.modules, + { + "adlfs": mock_adlfs_module, + "azure": mock.MagicMock(), + "azure.identity": mock_azure_identity_module, + }, + ): + # Setup the mocks (though they won't be called due to validation failures) + mock_filesystem = mock.Mock() + mock_adlfs_module.AzureBlobFileSystem.return_value = mock_filesystem + mock_filesystem.open.return_value = mock.Mock() + + for invalid_uri in invalid_uris: + with pytest.raises(ValueError, match="Invalid ABFSS URI format"): + Protocol.ABFSS.download_remote_uri(invalid_uri, str(dest_file)) + + @pytest.mark.asyncio class TestDownloadAndUnpackPackage: async def test_download_and_unpack_package_with_gcs_uri_without_gcs_client( From e5743a07d4cfb112d154166163070276a0204f3f Mon Sep 17 00:00:00 2001 From: Potato Date: Sat, 6 Sep 2025 07:13:41 +0800 Subject: [PATCH 481/634] [DOC] Fix grammar, syntax, and formatting issues in ray-air and ray-contribute documentation (#56067) Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: angelinalg <122562471+angelinalg@users.noreply.github.com> --- doc/source/ray-air/deployment.rst | 4 ++-- doc/source/ray-air/getting-started.rst | 10 +++++----- doc/source/ray-contribute/ci.rst | 2 +- doc/source/ray-contribute/debugging.rst | 2 +- doc/source/ray-contribute/development.rst | 2 +- doc/source/ray-contribute/docs.md | 2 +- doc/source/ray-contribute/fake-autoscaler.rst | 2 +- doc/source/ray-contribute/getting-involved.rst | 2 +- doc/source/ray-contribute/involvement.rst | 8 ++++---- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/source/ray-air/deployment.rst b/doc/source/ray-air/deployment.rst index 812e7cf691fd..343618b97a1a 100644 --- a/doc/source/ray-air/deployment.rst +++ b/doc/source/ray-air/deployment.rst @@ -1,14 +1,14 @@ Deploying Ray for ML platforms ============================== -Here, we describe how you might use or deploy Ray in your infrastructure. There are two main deployment patterns -- pick and choose and within existing platforms. +This page describes how you might use or deploy Ray in your infrastructure. There are two main deployment patterns -- pick and choose, and within existing platforms. The core idea is that Ray can be **complementary** to your existing infrastructure and integration tools. Design Principles ----------------- -* Ray and its libraries handles the heavyweight compute aspects of AI apps and services. +* Ray and its libraries handle the heavyweight compute aspects of AI apps and services. * Ray relies on external integrations (e.g., Tecton, MLFlow, W&B) for Storage and Tracking. * Workflow Orchestrators (e.g., AirFlow) are an optional component that can be used for scheduling recurring jobs, launching new Ray clusters for jobs, and running non-Ray compute steps. * Lightweight orchestration of task graphs within a single Ray app can be handled using Ray tasks. diff --git a/doc/source/ray-air/getting-started.rst b/doc/source/ray-air/getting-started.rst index 047adbaf55f8..2ed5395fdd38 100644 --- a/doc/source/ray-air/getting-started.rst +++ b/doc/source/ray-air/getting-started.rst @@ -5,9 +5,9 @@ Ray for ML Infrastructure .. tip:: - We'd love to hear from you if you are using Ray to build a ML platform! Fill out `this short form `__ to get involved. + We'd love to hear from you if you are using Ray to build an ML platform! Fill out `this short form `__ to get involved. -Ray and its AI libraries provide unified compute runtime for teams looking to simplify their ML platform. +Ray and its AI libraries provide a unified compute runtime for teams looking to simplify their ML platform. Ray's libraries such as Ray Train, Ray Data, and Ray Serve can be used to compose end-to-end ML workflows, providing features and APIs for data preprocessing as part of training, and transitioning from training to serving. @@ -27,9 +27,9 @@ Ray's AI libraries simplify the ecosystem of machine learning frameworks, platfo .. https://docs.google.com/drawings/d/1oi_JwNHXVgtR_9iTdbecquesUd4hOk0dWgHaTaFj6gk/edit -**1. Seamless Dev to Prod**: Ray's AI libraries reduces friction going from development to production. With Ray and its libraries, the same Python code scales seamlessly from a laptop to a large cluster. +**1. Seamless Dev to Prod**: Ray's AI libraries reduce friction going from development to production. With Ray and its libraries, the same Python code scales seamlessly from a laptop to a large cluster. -**2. Unified ML API and Runtime**: Ray's APIs enables swapping between popular frameworks, such as XGBoost, PyTorch, and Hugging Face, with minimal code changes. Everything from training to serving runs on a single runtime (Ray + KubeRay). +**2. Unified ML API and Runtime**: Ray's APIs enable swapping between popular frameworks, such as XGBoost, PyTorch, and Hugging Face, with minimal code changes. Everything from training to serving runs on a single runtime (Ray + KubeRay). **3. Open and Extensible**: Ray is fully open-source and can run on any cluster, cloud, or Kubernetes. Build custom components and integrations on top of scalable developer APIs. @@ -46,7 +46,7 @@ Spotify `uses Ray for advanced applications `_ to discuss Ray! - Star and follow us on `on GitHub`_. -- To post questions or feature requests, check out the `Discussion Board`_! -- Follow us and spread the word on `Twitter`_! -- Join our `Meetup Group`_ to connect with others in the community! -- Use the `[ray]` tag on `StackOverflow`_ to ask and answer questions about Ray usage +- To post questions or feature requests, check out the `Discussion Board`_. +- Follow us and spread the word on `Twitter`_. +- Join our `Meetup Group`_ to connect with others in the community. +- Use the `[ray]` tag on `StackOverflow`_ to ask and answer questions about Ray usage. .. _`Discussion Board`: https://discuss.ray.io/ From ad906afeb5322b674ee230258ef42549c01e3dd2 Mon Sep 17 00:00:00 2001 From: Matthew Owen Date: Fri, 5 Sep 2025 16:54:14 -0700 Subject: [PATCH 482/634] [data] refactor download expression to use inheritance from `AbstractOneToOne` (#56294) ## Why are these changes needed? `Download` logical operator isn't really a map, we should use `AbstractOneToOne` as the base class instead. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Matthew Owen Signed-off-by: Matthew Owen Co-authored-by: Balaji Veeramani --- .../logical/operators/map_operator.py | 34 ------------------- .../logical/operators/one_to_one_operator.py | 33 +++++++++++++++++- .../_internal/planner/plan_download_op.py | 15 ++++---- python/ray/data/_internal/planner/planner.py | 3 +- python/ray/data/dataset.py | 3 +- 5 files changed, 42 insertions(+), 46 deletions(-) diff --git a/python/ray/data/_internal/logical/operators/map_operator.py b/python/ray/data/_internal/logical/operators/map_operator.py index c6597ae084a2..6b1bcefba5ad 100644 --- a/python/ray/data/_internal/logical/operators/map_operator.py +++ b/python/ray/data/_internal/logical/operators/map_operator.py @@ -360,37 +360,3 @@ def target_num_rows_per_block(self) -> int: def can_modify_num_rows(self) -> bool: return False - - -class Download(AbstractMap): - """Logical operator for download operation.""" - - def __init__( - self, - input_op: LogicalOperator, - uri_column_name: str, - output_bytes_column_name: str, - ray_remote_args: Optional[Dict[str, Any]] = None, - ): - from ray.data._internal.compute import ActorPoolStrategy - - # Download operation uses CallableClass (PartitionActor) so needs ActorPoolStrategy - super().__init__( - "Download", - input_op, - ray_remote_args=ray_remote_args, - compute=ActorPoolStrategy(size=1), - ) - self._uri_column_name = uri_column_name - self._output_bytes_column_name = output_bytes_column_name - - def can_modify_num_rows(self) -> bool: - return False - - @property - def uri_column_name(self) -> str: - return self._uri_column_name - - @property - def output_bytes_column_name(self) -> str: - return self._output_bytes_column_name diff --git a/python/ray/data/_internal/logical/operators/one_to_one_operator.py b/python/ray/data/_internal/logical/operators/one_to_one_operator.py index d9195aebcbdb..79da7094793a 100644 --- a/python/ray/data/_internal/logical/operators/one_to_one_operator.py +++ b/python/ray/data/_internal/logical/operators/one_to_one_operator.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional from ray.data._internal.logical.interfaces import LogicalOperator from ray.data.block import BlockMetadata @@ -83,3 +83,34 @@ def _input_files(self): assert len(self._input_dependencies) == 1, len(self._input_dependencies) assert isinstance(self._input_dependencies[0], LogicalOperator) return self._input_dependencies[0].infer_metadata().input_files + + +class Download(AbstractOneToOne): + """Logical operator for download operation.""" + + def __init__( + self, + input_op: LogicalOperator, + uri_column_name: str, + output_bytes_column_name: str, + ray_remote_args: Optional[Dict[str, Any]] = None, + ): + super().__init__("Download", input_op) + self._uri_column_name = uri_column_name + self._output_bytes_column_name = output_bytes_column_name + self._ray_remote_args = ray_remote_args or {} + + def can_modify_num_rows(self) -> bool: + return False + + @property + def uri_column_name(self) -> str: + return self._uri_column_name + + @property + def output_bytes_column_name(self) -> str: + return self._output_bytes_column_name + + @property + def ray_remote_args(self) -> Dict[str, Any]: + return self._ray_remote_args diff --git a/python/ray/data/_internal/planner/plan_download_op.py b/python/ray/data/_internal/planner/plan_download_op.py index 98617c4593b6..2e2756b6a886 100644 --- a/python/ray/data/_internal/planner/plan_download_op.py +++ b/python/ray/data/_internal/planner/plan_download_op.py @@ -7,14 +7,14 @@ import pyarrow as pa import ray -from ray.data._internal.compute import ActorPoolStrategy, get_compute +from ray.data._internal.compute import ActorPoolStrategy, TaskPoolStrategy from ray.data._internal.execution.interfaces import PhysicalOperator from ray.data._internal.execution.operators.map_operator import MapOperator from ray.data._internal.execution.operators.map_transformer import ( BlockMapTransformFn, MapTransformer, ) -from ray.data._internal.logical.operators.map_operator import Download +from ray.data._internal.logical.operators.one_to_one_operator import Download from ray.data._internal.util import RetryingPyFileSystem, make_async_gen from ray.data.block import BlockAccessor from ray.data.context import DataContext @@ -33,9 +33,9 @@ def plan_download_op( """Plan the download operation with partitioning and downloading stages.""" assert len(physical_children) == 1 input_physical_dag = physical_children[0] - compute = get_compute(op._compute) uri_column_name = op.uri_column_name output_bytes_column_name = op.output_bytes_column_name + ray_remote_args = op.ray_remote_args # Import _get_udf from the main planner file from ray.data._internal.planner.plan_udf_map_op import ( @@ -58,8 +58,7 @@ def plan_download_op( data_context, name="URIPartitioner", compute_strategy=partition_compute, # Use actor-based compute for callable class - ray_remote_args=op._ray_remote_args, - ray_remote_args_fn=op._ray_remote_args_fn, + ray_remote_args=ray_remote_args, ) fn, init_fn = _get_udf( @@ -74,14 +73,14 @@ def plan_download_op( BlockMapTransformFn(download_transform_fn), ] download_map_transformer = MapTransformer(transform_fns, init_fn) + download_compute = TaskPoolStrategy() download_map_operator = MapOperator.create( download_map_transformer, partition_map_operator, data_context, name="URIDownloader", - compute_strategy=compute, - ray_remote_args=op._ray_remote_args, - ray_remote_args_fn=op._ray_remote_args_fn, + compute_strategy=download_compute, + ray_remote_args=ray_remote_args, ) return download_map_operator diff --git a/python/ray/data/_internal/planner/planner.py b/python/ray/data/_internal/planner/planner.py index 371a238c33b5..5cc78fb7aa3a 100644 --- a/python/ray/data/_internal/planner/planner.py +++ b/python/ray/data/_internal/planner/planner.py @@ -26,13 +26,12 @@ from ray.data._internal.logical.operators.join_operator import Join from ray.data._internal.logical.operators.map_operator import ( AbstractUDFMap, - Download, Filter, Project, StreamingRepartition, ) from ray.data._internal.logical.operators.n_ary_operator import Union, Zip -from ray.data._internal.logical.operators.one_to_one_operator import Limit +from ray.data._internal.logical.operators.one_to_one_operator import Download, Limit from ray.data._internal.logical.operators.read_operator import Read from ray.data._internal.logical.operators.streaming_split_operator import StreamingSplit from ray.data._internal.logical.operators.write_operator import Write diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 33a628977bae..7fb1c46b7651 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -828,7 +828,8 @@ def with_column( A new dataset with the added column evaluated via the expression. """ # TODO: update schema based on the expression AST. - from ray.data._internal.logical.operators.map_operator import Download, Project + from ray.data._internal.logical.operators.map_operator import Project + from ray.data._internal.logical.operators.one_to_one_operator import Download # TODO: Once the expression API supports UDFs, we can clean up the code here. from ray.data.expressions import DownloadExpr From 9341bd4fd4b4ed82c90dd1f6ae8bff8540824bff Mon Sep 17 00:00:00 2001 From: minerharry <35383543+minerharry@users.noreply.github.com> Date: Fri, 5 Sep 2025 19:54:25 -0400 Subject: [PATCH 483/634] [core] Proper typing for ObjectRef (#55566) ## Why are these changes needed? Adds proper type-stubbing for ObjectRef (and its cousins, ActorID, TaskID, etc). See #53591. The ray/includes .pxi files (and the _raylet.pxi) are inaccessible to type-checkers, so core ray objects like ObjectRef don't have proper code-completion. While the issue extends to many ray objects defined in includes/* and _raylet.pxi, notably ObjectRefGenerator and similar, this PR focuses just on unique_ids.pxi and object_ref.pxi. By adding python stub files, .pyi, that correspond to the .pxi files, this PR allows type-checkers to understand the contents of the cython files. Each .pyi file contains the signature of every class and method in the .pxi files to expose (though I went ahead and added every method for completeness). Also, I added a `__class_getitem__` method to ObjectRef in object_ref.pxi which mirrors how typing.Generic works, as multiple inheritance (which would be required for generic subclassing of BaseID) is not well supported in cython. This allows for runtime type-subscripting (e.g. ObjectRef[int]) without error and returns a proper annotated type. ## Related issue number Limited-scope version of #55066, just focusing on unique_ids.pxi and object_ref.pxi. Partially satisfies #53591. ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [x] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [x] This PR is not tested :( --------- Signed-off-by: minerharry Signed-off-by: Philipp Moritz Co-authored-by: Philipp Moritz --- python/ray/_raylet.pyi | 45 +++++++-- python/ray/includes/object_ref.pxi | 5 +- python/ray/includes/object_ref.pyi | 74 +++++++++++++++ python/ray/includes/unique_ids.pyi | 146 +++++++++++++++++++++++++++++ 4 files changed, 262 insertions(+), 8 deletions(-) create mode 100644 python/ray/includes/object_ref.pyi create mode 100644 python/ray/includes/unique_ids.pyi diff --git a/python/ray/_raylet.pyi b/python/ray/_raylet.pyi index c28976409578..96c79d343b19 100644 --- a/python/ray/_raylet.pyi +++ b/python/ray/_raylet.pyi @@ -1,11 +1,42 @@ -from typing import Awaitable, TypeVar +from ray.includes.object_ref import ( + _set_future_helper, + ObjectRef +) -R = TypeVar("R") +from ray.includes.unique_ids import ( + check_id, + BaseID, + UniqueID, + TaskID, + NodeID, + JobID, + WorkerID, + ActorID, + FunctionID, + ActorClassID, + ClusterID, + ObjectID, + PlacementGroupID, +) -class ObjectRef(Awaitable[R]): # type: ignore - pass +__all__ = [ + # ray.includes.unique_ids + "ActorClassID", + "ActorID", + "BaseID", + "ClusterID", + "FunctionID", + "JobID", + "NodeID", + "ObjectID", + "PlacementGroupID", + "TaskID", + "UniqueID", + "WorkerID", + "check_id", - -class ObjectID(Awaitable[R]): # type: ignore - pass + # ray.includes.object_ref + "_set_future_helper", + "ObjectRef", +] diff --git a/python/ray/includes/object_ref.pxi b/python/ray/includes/object_ref.pxi index 829ec790c4cf..fa498c14bf98 100644 --- a/python/ray/includes/object_ref.pxi +++ b/python/ray/includes/object_ref.pxi @@ -6,6 +6,7 @@ import functools import logging import threading from typing import Callable, Any, Union +from _collections_abc import GenericAlias import ray import cython @@ -34,6 +35,7 @@ def _set_future_helper( cdef class ObjectRef(BaseID): + __class_getitem__ = classmethod(GenericAlias) # should match how typing.Generic works def __cinit__(self): self.in_core_worker = False @@ -97,7 +99,8 @@ cdef class ObjectRef(BaseID): def call_site(self): return decode(self.call_site_data) - def size(self): + @classmethod + def size(cls): return CObjectID.Size() def _set_id(self, id): diff --git a/python/ray/includes/object_ref.pyi b/python/ray/includes/object_ref.pyi new file mode 100644 index 000000000000..78a744e8a856 --- /dev/null +++ b/python/ray/includes/object_ref.pyi @@ -0,0 +1,74 @@ +# source: object_ref.pxi +import asyncio +import concurrent.futures +from typing import Any, Awaitable, Callable, Generator, TypeVar, Union + +from ray.includes.unique_ids import BaseID, JobID, TaskID + +_T = TypeVar("_T") +def _set_future_helper( + result: _T, + *, + py_future: Union[asyncio.Future[_T], concurrent.futures.Future[_T]], +) -> None: ... + + +_OR = TypeVar("_OR", bound=ObjectRef) +class ObjectRef(BaseID, Awaitable[_T]): + + + def __init__( + self, id: bytes, owner_addr: str = "", call_site_data: str = "", + skip_adding_local_ref: bool = False, tensor_transport_val = 0) -> None: ... + + def __dealloc__(self) -> None: ... + + + def task_id(self) -> TaskID: ... + + def job_id(self) -> JobID: ... + + def owner_address(self) -> str: ... + + def call_site(self) -> str: ... + + @classmethod + def size(cls) -> int: ... + + def _set_id(self, id: bytes) -> None: ... + + @classmethod + def nil(cls: type[_OR]) -> _OR: ... + + @classmethod + def from_random(cls: type[_OR]) -> _OR: ... + + def future(self) -> concurrent.futures.Future[_T]: + """Wrap ObjectRef with a concurrent.futures.Future + + Note that the future cancellation will not cancel the correspoding + task when the ObjectRef representing return object of a task. + Additionally, future.running() will always be ``False`` even if the + underlying task is running. + """ + ... + + def __await__(self) -> Generator[Any, None, _T]: ... + + def as_future(self, _internal=False) -> asyncio.Future[_T]: + """Wrap ObjectRef with an asyncio.Future. + + Note that the future cancellation will not cancel the correspoding + task when the ObjectRef representing return object of a task. + """ + ... + + def _on_completed(self, py_callback: Callable[[_T], None]): + """Register a callback that will be called after Object is ready. + If the ObjectRef is already ready, the callback will be called soon. + The callback should take the result as the only argument. The result + can be an exception object in case of task error. + """ + ... + + def tensor_transport(self) -> int: ... diff --git a/python/ray/includes/unique_ids.pyi b/python/ray/includes/unique_ids.pyi new file mode 100644 index 000000000000..5f04389f1aed --- /dev/null +++ b/python/ray/includes/unique_ids.pyi @@ -0,0 +1,146 @@ +from __future__ import annotations + +from typing import Tuple, TypeVar + +# backwards compatibility. Luckily circular references are fine in type stubs +from ray._raylet import ObjectRef + +ObjectID = ObjectRef + +# implementations are in unique_ids.pxi +def check_id(b: bytes, size: int = ...) -> None: ... + +_BID = TypeVar("_BID", bound=BaseID) +class BaseID: + + @classmethod + def from_binary(cls: type[_BID], id_bytes: bytes) -> _BID: ... + + @classmethod + def from_hex(cls: type[_BID], hex_id: str | bytes) -> _BID: ... + + def binary(self) -> bytes: ... + + @classmethod + def size(cls) -> int: ... + + def hex(self) -> str: ... + + def is_nil(self) -> bool: ... + + def __hash__(self) -> int: ... + + def __eq__(self, other: object) -> bool: ... + + def __ne__(self, other: object) -> bool: ... + + def __bytes__(self) -> bytes: ... + + def __hex__(self) -> str: ... + + def __repr__(self) -> str: ... + + def __str__(self) -> str: ... + + def __reduce__(self: _BID) -> Tuple[type[_BID], Tuple[bytes]]: ... + + def redis_shard_hash(self) -> int: ... + + +_UID = TypeVar("_UID", bound=UniqueID) +class UniqueID(BaseID): + + def __init__(self, id: bytes) -> None: ... + + @classmethod + def nil(cls: type[_UID]) -> _UID: ... + + @classmethod + def from_random(cls: type[_UID]) -> _UID: ... + + +_TID = TypeVar("_TID", bound=TaskID) +class TaskID(BaseID): + + def __init__(self, id: bytes) -> None: ... + + def actor_id(self) -> ActorID: ... + + def job_id(self) -> JobID: ... + + @classmethod + def nil(cls: type[_TID]) -> _TID: ... + + @classmethod + def for_fake_task(cls: type[_TID], job_id: JobID) -> _TID: ... + + @classmethod + def for_driver_task(cls: type[_TID], job_id: JobID) -> _TID: ... + + @classmethod + def for_actor_creation_task(cls: type[_TID], actor_id: ActorID) -> _TID: ... + + @classmethod + def for_actor_task(cls: type[_TID], job_id: JobID, parent_task_id: TaskID, + parent_task_counter: int, actor_id: ActorID) -> _TID: ... + + @classmethod + def for_normal_task(cls: type[_TID], job_id: JobID, parent_task_id: TaskID, parent_task_counter: int) -> _TID: ... + + +class NodeID(UniqueID): ... + +_JID = TypeVar("_JID", bound=JobID) +class JobID(BaseID): + + def __init__(self, id: bytes) -> None: ... + + @classmethod + def from_int(cls: type[_JID], value: int) -> _JID: ... + + @classmethod + def nil(cls: type[_JID]) -> _JID: ... + + def int(self) -> int: ... + + +class WorkerID(UniqueID): ... + +_AID = TypeVar("_AID", bound=ActorID) +class ActorID(BaseID): + + def __init__(self, id: bytes) -> None: ... + + @classmethod + def of(cls: type[_AID], job_id: JobID, parent_task_id: TaskID, parent_task_counter: int) -> _AID: ... + + @classmethod + def nil(cls: type[_AID]) -> _AID: ... + + @classmethod + def from_random(cls: type[_AID]) -> _AID: ... + + def _set_id(self, id: bytes) -> None: ... + + @property + def job_id(self) -> JobID: ... + + +class FunctionID(UniqueID): ... +class ActorClassID(UniqueID): ... +class ClusterID(UniqueID): ... + + +_PGID = TypeVar("_PGID", bound=PlacementGroupID) +class PlacementGroupID(BaseID): + + def __init__(self, id: bytes) -> None: ... + + @classmethod + def from_random(cls: type[_PGID]) -> _PGID: ... + + @classmethod + def of(cls: type[_PGID], job_id: JobID) -> _PGID: ... + + @classmethod + def nil(cls: type[_PGID]) -> _PGID: ... From fa2643c4b1eaddbea52a80233dcb99c915962644 Mon Sep 17 00:00:00 2001 From: Potato Date: Sat, 6 Sep 2025 08:00:13 +0800 Subject: [PATCH 484/634] [RLlib] [DOC] Fix documentation typos and grammatical issues in RLlib docs (#56130) Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- doc/source/rllib/algorithm-config.rst | 2 +- doc/source/rllib/external-envs.rst | 4 ++-- doc/source/rllib/getting-started.rst | 2 +- doc/source/rllib/metrics-logger.rst | 8 ++++---- doc/source/rllib/package_ref/index.rst | 2 +- doc/source/rllib/rllib-offline.rst | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/source/rllib/algorithm-config.rst b/doc/source/rllib/algorithm-config.rst index 800ff8de3d0e..76f1b2a7afbd 100644 --- a/doc/source/rllib/algorithm-config.rst +++ b/doc/source/rllib/algorithm-config.rst @@ -12,7 +12,7 @@ the auto-validated and type-safe gateway into configuring and building an RLlib :py:class:`~ray.rllib.algorithms.algorithm.Algorithm`. In essence, you first create an instance of :py:class:`~ray.rllib.algorithms.algorithm_config.AlgorithmConfig` -and then call some of its methods to set various configuration options. RLlib uses the following, `black `__ compliant format +and then call some of its methods to set various configuration options. RLlib uses the following `black `__-compliant format in all parts of its code. Note that you can chain together more than one method call, including the constructor: diff --git a/doc/source/rllib/external-envs.rst b/doc/source/rllib/external-envs.rst index 2307457447a8..467e9fdfb4da 100644 --- a/doc/source/rllib/external-envs.rst +++ b/doc/source/rllib/external-envs.rst @@ -68,7 +68,7 @@ Message Structure RLlink messages consist of a header and a body: - - **Header**: 8-byte length field indicating the size of the body, for example `00000016` for a body of length 16 (thus, in total, the message size ). + - **Header**: 8-byte length field indicating the size of the body, for example `00000016` for a body of length 16 (thus, in total, the message size). - **Body**: JSON-encoded content with a `type` field indicating the message type. Example Messages: PING and EPISODES_AND_GET_STATE @@ -153,7 +153,7 @@ Responses: Server → Client - **``SET_STATE``** - - Example: ``{"type": "PONG"}`` + - Example: ``{"type": "SET_STATE", "weights_seq_no": 123, "onnx_file": "... [base64 encoded ONNX file] ..."}`` - Purpose: Provide the client with the current state (for example, model weights). - Body: diff --git a/doc/source/rllib/getting-started.rst b/doc/source/rllib/getting-started.rst index 7cf14882a2fd..0958682c4765 100644 --- a/doc/source/rllib/getting-started.rst +++ b/doc/source/rllib/getting-started.rst @@ -77,7 +77,7 @@ method: ) -To scale your setup and define, how many :py:class:`~ray.rllib.env.env_runner.EnvRunner` actors you want to leverage, +To scale your setup and define how many :py:class:`~ray.rllib.env.env_runner.EnvRunner` actors you want to leverage, you can call the :py:meth:`~ray.rllib.algorithms.algorithm_config.AlgorithmConfig.env_runners` method. ``EnvRunners`` are used to collect samples for training updates from your :ref:`environment `. diff --git a/doc/source/rllib/metrics-logger.rst b/doc/source/rllib/metrics-logger.rst index 382ce6ad8596..2ef0b0d4c0b0 100644 --- a/doc/source/rllib/metrics-logger.rst +++ b/doc/source/rllib/metrics-logger.rst @@ -46,7 +46,7 @@ Features of MetricsLogger The :py:class:`~ray.rllib.utils.metrics.metrics_logger.MetricsLogger` API offers the following functionalities: - Log scalar values over time, such as losses, individual rewards, or episode returns. -- Configure different reduction types, in particular ``mean``, ``min``, ``max``, or ``sum``. Also, users can chose to not +- Configure different reduction types, in particular ``mean``, ``min``, ``max``, or ``sum``. Also, users can choose to not reduce at all through the ``reduce=None`` setting, leaving the logged values untouched. A separate ``clear_on_reduce=True`` setting allows for automatically clearing all logged values on each ``reduce`` event. - Specify sliding windows, over which reductions take place, for example ``window=100`` to average over the @@ -169,7 +169,7 @@ whenever reduction takes place or you peek at the current value: logger.peek("max_value") # Expect: 1000.0, which is the lifetime max (infinite window) -You can also chose to not reduce at all, but to simply collect individual values, for example a set of images you receive +You can also choose to not reduce at all, but to simply collect individual values, for example a set of images you receive from your environment over time and for which it doesn't make sense to reduce them in any way. Use the ``reduce=None`` argument for achieving this. However, it's strongly advised that you should also @@ -192,7 +192,7 @@ to :py:class:`~ray.rllib.algorithms.algorithm.Algorithm`: You should pass additional arguments like ``reduce=None`` and ``clear_on_reduce=True`` to the :py:meth:`~ray.rllib.utils.metrics.metrics_logger.MetricsLogger.log_value` method on each call. -Otherwise, MetricsLogger will emit warnings to ensure that it's behaviour is always as expected. +Otherwise, MetricsLogger will emit warnings to ensure that its behavior is always as expected. Logging a set of nested scalar values @@ -228,7 +228,7 @@ Logging non-scalar data :py:class:`~ray.rllib.utils.metrics.metrics_logger.MetricsLogger` isn't limited to scalar values. You can also use it to log images, videos, or any other complex data. -Normally, you would chose the previously described ``reduce=None`` argument. For example, to +Normally, you would choose the previously described ``reduce=None`` argument. For example, to log three consecutive image frames from a ``CartPole`` environment, do the following: .. testcode:: diff --git a/doc/source/rllib/package_ref/index.rst b/doc/source/rllib/package_ref/index.rst index 5411f2c213b1..7b88fbd8c8e9 100644 --- a/doc/source/rllib/package_ref/index.rst +++ b/doc/source/rllib/package_ref/index.rst @@ -10,7 +10,7 @@ Ray RLlib API .. tip:: We'd love to hear your feedback on using RLlib - `sign up to our forum and start asking questions `_! This section contains an overview of RLlib's package- and API reference. -If you think there is anything missing, please open an issue on `Github`_. +If you think there is anything missing, please open an issue on `GitHub`_. .. _`GitHub`: https://github.com/ray-project/ray/issues diff --git a/doc/source/rllib/rllib-offline.rst b/doc/source/rllib/rllib-offline.rst index 3d08c287f9c2..2320e1cfcc43 100644 --- a/doc/source/rllib/rllib-offline.rst +++ b/doc/source/rllib/rllib-offline.rst @@ -22,7 +22,7 @@ format. You should use the episode format when #. You need experiences grouped by their trajectory and ordered in time (for example, to train stateful modules). #. You want to use recorded experiences exclusively within RLlib (for example for offline RL or behavior cloning). -Contrary, you should prefer the table (columns) format, if +On the contrary, you should prefer the table (columns) format, if #. You need to read the data easily with other data tools or ML libraries. @@ -30,8 +30,8 @@ Contrary, you should prefer the table (columns) format, if :py:class:`~ray.rllib.env.single_agent_episode.SingleAgentEpisode` class is usable outside of an RLlib context. To enable faster access through external data tools (for example, for data transformations), it's recommended to use the table record format. -Most importantly, RLlib's offline RL API builds on top of :ref:`Ray Data ` and therefore features in general all read and -write methods supported by Ray Data (for example :py:class:`~ray.data.read_parquet`, :py:class:`~ray.data.read_json`, etc.) with +Most importantly, RLlib's offline RL API builds on top of :ref:`Ray Data ` and therefore supports all of its read and +write methods (for example :py:class:`~ray.data.read_parquet`, :py:class:`~ray.data.read_json`, etc.) with :py:class:`~ray.data.read_parquet` and :py:class:`~ray.data.Dataset.write_parquet` being the default read and write methods. A core design principle of the API is to apply as many data transformations as possible on-the-fly prior to engaging the learner, allowing the latter to focus exclusively on model updates. From 704c579e1c428aaf46d5b9d90dabf0bac9447c6a Mon Sep 17 00:00:00 2001 From: Ibrahim Rabbani Date: Fri, 5 Sep 2025 17:05:36 -0700 Subject: [PATCH 485/634] [core] (cgroups 4/n) adding constraint bounds checking to the CgroupManager. (#56246) This PR continues to implement the CgroupManager. CgroupManager will be used by the Raylet to manage the cgroup hierarchy. The implementation will be completed in subsequent PRs. This PR stacks on #56186. For more details about the resource isolation project see https://github.com/ray-project/ray/issues/54703. In this PR: * CgroupManager now bound checks constraints (e.g. cpu.weight is within [1,10000]. * CgroupDriver no longer bound checks constraints. --------- Signed-off-by: Ibrahim Rabbani Co-authored-by: Edward Oakes --- .../common/cgroup2/cgroup_driver_interface.h | 13 +---- src/ray/common/cgroup2/cgroup_manager.cc | 58 ++++++++++++------- src/ray/common/cgroup2/cgroup_manager.h | 5 +- .../common/cgroup2/cgroup_manager_interface.h | 52 +++++++++-------- src/ray/common/cgroup2/cgroup_test_utils.h | 1 - src/ray/common/cgroup2/fake_cgroup_driver.h | 1 + .../sysfs_cgroup_driver_integration_test.cc | 37 ++---------- src/ray/common/cgroup2/sysfs_cgroup_driver.cc | 51 +++------------- src/ray/common/cgroup2/sysfs_cgroup_driver.h | 18 +++--- .../cgroup2/tests/sysfs_cgroup_driver_test.cc | 2 +- 10 files changed, 94 insertions(+), 144 deletions(-) diff --git a/src/ray/common/cgroup2/cgroup_driver_interface.h b/src/ray/common/cgroup2/cgroup_driver_interface.h index 132000c79ff5..62e9f47f7b9a 100644 --- a/src/ray/common/cgroup2/cgroup_driver_interface.h +++ b/src/ray/common/cgroup2/cgroup_driver_interface.h @@ -157,6 +157,7 @@ class CgroupDriverInterface { supported or the value not correct. */ virtual Status AddConstraint(const std::string &cgroup, + const std::string &controller, const std::string &constraint, const std::string &value) = 0; /** @@ -190,17 +191,5 @@ class CgroupDriverInterface { */ virtual StatusOr> GetEnabledControllers( const std::string &cgroup) = 0; - - struct Constraint { - std::pair range; - std::string controller; - }; - - protected: - const std::unordered_map supported_constraints_ = { - {"cpu.weight", {{1, 10000}, "cpu"}}, - {"memory.min", {{0, std::numeric_limits::max()}, "memory"}}, - }; - const std::unordered_set supported_controllers_ = {"cpu", "memory"}; }; } // namespace ray diff --git a/src/ray/common/cgroup2/cgroup_manager.cc b/src/ray/common/cgroup2/cgroup_manager.cc index f01741ea0d1d..28402ba27617 100644 --- a/src/ray/common/cgroup2/cgroup_manager.cc +++ b/src/ray/common/cgroup2/cgroup_manager.cc @@ -55,8 +55,24 @@ StatusOr> CgroupManager::Create( const int64_t system_reserved_cpu_weight, const int64_t system_reserved_memory_bytes, std::unique_ptr cgroup_driver) { - // TODO(#54703): Add bounds checking for system_reserved_cpu_weight - // and system_reserved_memory_bytes. + if (!cpu_weight_constraint_.IsValid(system_reserved_cpu_weight)) { + return Status::InvalidArgument( + absl::StrFormat("Invalid constraint %s=%d. %s must be in the range [%d, %d].", + cpu_weight_constraint_.name_, + system_reserved_cpu_weight, + cpu_weight_constraint_.name_, + cpu_weight_constraint_.Min(), + cpu_weight_constraint_.Max())); + } + if (!memory_min_constraint_.IsValid(system_reserved_memory_bytes)) { + return Status::InvalidArgument( + absl::StrFormat("Invalid constraint %s=%d. %s must be in the range [%d, %d].", + memory_min_constraint_.name_, + system_reserved_memory_bytes, + memory_min_constraint_.name_, + memory_min_constraint_.Min(), + memory_min_constraint_.Max())); + } RAY_RETURN_NOT_OK(cgroup_driver->CheckCgroupv2Enabled()); RAY_RETURN_NOT_OK(cgroup_driver->CheckCgroup(base_cgroup_path)); StatusOr> available_controllers = @@ -117,16 +133,15 @@ void CgroupManager::RegisterMoveAllProcesses(const std::string &from, // TODO(#54703): This is a placeholder for cleanup. This will call // CgroupDriver::AddConstraint(cgroup, constraint, default_value). +template void CgroupManager::RegisterRemoveConstraint(const std::string &cgroup, - const std::string &constraint) { + const Constraint &constraint) { cleanup_operations_.emplace_back( [constrained_cgroup = cgroup, constraint_to_remove = constraint]() { - auto constraint_metadata = supported_constraints_.find(constraint_to_remove); - RAY_CHECK(constraint_metadata != supported_constraints_.end()); RAY_LOG(INFO) << absl::StrFormat( "Setting constraint %s to default value %lld for cgroup %s", - constraint_to_remove, - constraint_metadata->second.default_value, + constraint_to_remove.name_, + constraint_to_remove.default_value_, constrained_cgroup); }); } @@ -149,8 +164,8 @@ Status CgroupManager::Initialize(int64_t system_reserved_cpu_weight, // The cpu.weight is distributed between the system and application cgroups. // The application cgroup gets whatever is leftover from the system cgroup. - int64_t max_cpu_weight = supported_constraints_.at(kCPUWeightConstraint).Max(); - int64_t application_cgroup_cpu_weight = max_cpu_weight - system_reserved_cpu_weight; + int64_t application_cgroup_cpu_weight = + cpu_weight_constraint_.Max() - system_reserved_cpu_weight; RAY_LOG(INFO) << absl::StrFormat( "Initializing CgroupManager at base cgroup path at %s. Ray's cgroup " @@ -162,12 +177,12 @@ Status CgroupManager::Initialize(int64_t system_reserved_cpu_weight, node_cgroup_path_, supported_controllers, system_cgroup_path_, - kCPUWeightConstraint, + cpu_weight_constraint_.name_, system_reserved_cpu_weight, - kMemoryMinConstraint, + memory_min_constraint_.name_, system_reserved_memory_bytes, application_cgroup_path_, - kCPUWeightConstraint, + cpu_weight_constraint_.name_, application_cgroup_cpu_weight); // Create the cgroup heirarchy: @@ -206,21 +221,24 @@ Status CgroupManager::Initialize(int64_t system_reserved_cpu_weight, RAY_RETURN_NOT_OK( cgroup_driver_->AddConstraint(system_cgroup_path_, - kMemoryMinConstraint, - std::to_string(system_reserved_memory_bytes))); - RegisterRemoveConstraint(system_cgroup_path_, kMemoryMinConstraint); + cpu_weight_constraint_.controller_, + cpu_weight_constraint_.name_, + std::to_string(system_reserved_cpu_weight))); + RegisterRemoveConstraint(system_cgroup_path_, cpu_weight_constraint_); RAY_RETURN_NOT_OK( cgroup_driver_->AddConstraint(system_cgroup_path_, - kCPUWeightConstraint, - std::to_string(system_reserved_cpu_weight))); - RegisterRemoveConstraint(system_cgroup_path_, kCPUWeightConstraint); + memory_min_constraint_.controller_, + memory_min_constraint_.name_, + std::to_string(system_reserved_memory_bytes))); + RegisterRemoveConstraint(system_cgroup_path_, memory_min_constraint_); RAY_RETURN_NOT_OK( cgroup_driver_->AddConstraint(application_cgroup_path_, - kCPUWeightConstraint, + cpu_weight_constraint_.controller_, + cpu_weight_constraint_.name_, std::to_string(application_cgroup_cpu_weight))); - RegisterRemoveConstraint(application_cgroup_path_, kCPUWeightConstraint); + RegisterRemoveConstraint(application_cgroup_path_, cpu_weight_constraint_); return Status::OK(); } diff --git a/src/ray/common/cgroup2/cgroup_manager.h b/src/ray/common/cgroup2/cgroup_manager.h index fa281e7376bf..ba402d496c32 100644 --- a/src/ray/common/cgroup2/cgroup_manager.h +++ b/src/ray/common/cgroup2/cgroup_manager.h @@ -28,7 +28,6 @@ namespace ray { class CgroupManager : public CgroupManagerInterface { public: /** - Creates a CgroupManager after checking for the following invariants: 1. cgroupv2 is mounted correctly in unified mode. For more details (@see @@ -116,7 +115,9 @@ class CgroupManager : public CgroupManagerInterface { // future PR. void RegisterDeleteCgroup(const std::string &cgroup_path); void RegisterMoveAllProcesses(const std::string &from, const std::string &to); - void RegisterRemoveConstraint(const std::string &cgroup, const std::string &constraint); + template + void RegisterRemoveConstraint(const std::string &cgroup, + const Constraint &constraint); void RegisterDisableController(const std::string &cgroup, const std::string &controller); diff --git a/src/ray/common/cgroup2/cgroup_manager_interface.h b/src/ray/common/cgroup2/cgroup_manager_interface.h index 5b69405d3863..28b6f936932f 100644 --- a/src/ray/common/cgroup2/cgroup_manager_interface.h +++ b/src/ray/common/cgroup2/cgroup_manager_interface.h @@ -39,23 +39,6 @@ namespace ray { system application */ class CgroupManagerInterface { - // TODO(#54703): The Constraint struct, supported_constraints_, and - // supported_controllers_ are duplicated across CgroupManagerInterface and - // CgroupDriverInterface. It makes sense for these to be separated into two concerns: - // 1) Checking which controllers and constraints are supported in Ray should be in - // CgroupManagerInterface. - // 2) Checking what values are allowed for constraints should be inside - // CgroupDriverInterface. - // This will be done in a later PR. - struct Constraint { - std::pair range; - std::string controller; - int64_t default_value; - - int64_t Max() const { return range.second; } - int64_t Min() const { return range.first; } - }; - public: // TODO(#54703): These will be implemented in a later PR to move processes // into a cgroup. @@ -72,16 +55,35 @@ class CgroupManagerInterface { inline static const std::string kNodeCgroupName = "ray_node"; inline static const std::string kSystemCgroupName = "system"; inline static const std::string kApplicationCgroupName = "application"; - inline static const std::string kCPUWeightConstraint = "cpu.weight"; - inline static const std::string kMemoryMinConstraint = "memory.min"; - inline static const std::unordered_map supported_constraints_ = - {{kCPUWeightConstraint, {{1, 10000}, "cpu", 100}}, - { - kMemoryMinConstraint, - {{0, std::numeric_limits::max()}, "memory", 0}, - }}; + // Controllers that can be enabled in Ray. inline static const std::unordered_set supported_controllers_ = {"cpu", "memory"}; + /** + Metadata about constraints that can be used. + @tparam the type of value that the constraint can take. + */ + template + struct Constraint { + std::string name_; + std::string controller_; + std::pair range_; + T default_value_; + T Max() const { return range_.second; } + T Min() const { return range_.first; } + bool IsValid(T value) const { return value <= Max() && value >= Min(); } + }; + + // cpu.weight distributes a cgroup's cpu cycles between it's children. + // See https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu-interface-files + inline static const Constraint cpu_weight_constraint_{ + "cpu.weight", "cpu", {1, 10000}, 100}; + + // memory.min guarantees hard memory protection. If the memory usage of a cgroup + // is within its effective min boundary, the cgroup’s memory won’t be reclaimed under + // any conditions. + // See https://docs.kernel.org/admin-guide/cgroup-v2.html#memory-interface-files + inline static const Constraint memory_min_constraint_{ + "memory.min", "memory", {0, std::numeric_limits::max()}, 0}; }; } // namespace ray diff --git a/src/ray/common/cgroup2/cgroup_test_utils.h b/src/ray/common/cgroup2/cgroup_test_utils.h index 29cb8364eb84..beaa58c7de91 100644 --- a/src/ray/common/cgroup2/cgroup_test_utils.h +++ b/src/ray/common/cgroup2/cgroup_test_utils.h @@ -13,7 +13,6 @@ // limitations under the License. #pragma once -#include #include #include diff --git a/src/ray/common/cgroup2/fake_cgroup_driver.h b/src/ray/common/cgroup2/fake_cgroup_driver.h index d40235f71ee3..0fe1ad041c85 100644 --- a/src/ray/common/cgroup2/fake_cgroup_driver.h +++ b/src/ray/common/cgroup2/fake_cgroup_driver.h @@ -122,6 +122,7 @@ class FakeCgroupDriver : public CgroupDriverInterface { } Status AddConstraint(const std::string &cgroup, + const std::string &controller, const std::string &constraint, const std::string &value) override { if (!add_constraint_s_.ok()) { diff --git a/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc b/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc index 52dbf51eb72e..ab3312f27f25 100644 --- a/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc +++ b/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc @@ -11,9 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include #include -#include #include #include @@ -482,7 +480,7 @@ TEST_F(SysFsCgroupDriverIntegrationTest, AddResourceConstraintFailsIfCgroupDoesn std::string non_existent_path = test_cgroup_path_ + std::filesystem::path::preferred_separator + "nope"; SysFsCgroupDriver driver; - Status s = driver.AddConstraint(non_existent_path, "memory.min", "1"); + Status s = driver.AddConstraint(non_existent_path, "memory", "memory.min", "1"); ASSERT_TRUE(s.IsNotFound()) << s.ToString(); } @@ -492,7 +490,7 @@ TEST_F(SysFsCgroupDriverIntegrationTest, ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); auto cgroup = std::move(cgroup_or_status.value()); SysFsCgroupDriver driver; - Status s = driver.AddConstraint(cgroup->GetPath(), "memory.min", "1"); + Status s = driver.AddConstraint(cgroup->GetPath(), "memory", "memory.min", "1"); ASSERT_TRUE(s.IsPermissionDenied()) << s.ToString(); } @@ -503,20 +501,10 @@ TEST_F(SysFsCgroupDriverIntegrationTest, ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); auto cgroup = std::move(cgroup_or_status.value()); SysFsCgroupDriver driver; - Status s = driver.AddConstraint(cgroup->GetPath(), "memory.min", "1"); + Status s = driver.AddConstraint(cgroup->GetPath(), "memory", "memory.min", "1"); ASSERT_TRUE(s.IsPermissionDenied()) << s.ToString(); } -TEST_F(SysFsCgroupDriverIntegrationTest, - AddResourceConstraintFailsIfConstraintNotSupported) { - auto cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); - ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); - auto cgroup = std::move(cgroup_or_status.value()); - SysFsCgroupDriver driver; - // "memory.max" is not supported. - Status s = driver.AddConstraint(cgroup->GetPath(), "memory.max", "1"); - ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); -} TEST_F(SysFsCgroupDriverIntegrationTest, AddResourceConstraintFailsIfControllerNotEnabled) { auto cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); @@ -524,24 +512,9 @@ TEST_F(SysFsCgroupDriverIntegrationTest, auto cgroup = std::move(cgroup_or_status.value()); SysFsCgroupDriver driver; // Memory controller is not enabled. - Status s = driver.AddConstraint(cgroup->GetPath(), "memory.min", "1"); + Status s = driver.AddConstraint(cgroup->GetPath(), "memory", "memory.min", "1"); ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); } -TEST_F(SysFsCgroupDriverIntegrationTest, - AddResourceConstraintFailsIfInvalidConstraintValue) { - auto cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); - ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); - auto cgroup = std::move(cgroup_or_status.value()); - SysFsCgroupDriver driver; - // Enable the cpu controller first. - Status enable_controller_s = driver.EnableController(cgroup->GetPath(), "cpu"); - ASSERT_TRUE(enable_controller_s.ok()) << enable_controller_s.ToString(); - // cpu.weight must be between [1,10000] - Status s_too_low = driver.AddConstraint(cgroup->GetPath(), "cpu.weight", "0"); - ASSERT_TRUE(s_too_low.IsInvalidArgument()) << s_too_low.ToString(); - Status s_too_high = driver.AddConstraint(cgroup->GetPath(), "cpu.weight", "10001"); - ASSERT_TRUE(s_too_high.IsInvalidArgument()) << s_too_high.ToString(); -} TEST_F(SysFsCgroupDriverIntegrationTest, AddResourceConstraintSucceeds) { auto cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); @@ -552,7 +525,7 @@ TEST_F(SysFsCgroupDriverIntegrationTest, AddResourceConstraintSucceeds) { Status enable_controller_s = driver.EnableController(cgroup->GetPath(), "cpu"); ASSERT_TRUE(enable_controller_s.ok()) << enable_controller_s.ToString(); // cpu.weight must be between [1,10000] - Status s = driver.AddConstraint(cgroup->GetPath(), "cpu.weight", "500"); + Status s = driver.AddConstraint(cgroup->GetPath(), "cpu", "cpu.weight", "500"); ASSERT_TRUE(s.ok()) << s.ToString(); } } // namespace ray diff --git a/src/ray/common/cgroup2/sysfs_cgroup_driver.cc b/src/ray/common/cgroup2/sysfs_cgroup_driver.cc index 88a41d74af8a..bf29d8cafa66 100644 --- a/src/ray/common/cgroup2/sysfs_cgroup_driver.cc +++ b/src/ray/common/cgroup2/sysfs_cgroup_driver.cc @@ -28,9 +28,7 @@ #include #include #include -#include #include -#include #include #include @@ -297,61 +295,28 @@ Status SysFsCgroupDriver::DisableController(const std::string &cgroup_path, return Status::OK(); } -Status SysFsCgroupDriver::AddConstraint(const std::string &cgroup, +Status SysFsCgroupDriver::AddConstraint(const std::string &cgroup_path, + const std::string &controller, const std::string &constraint, const std::string &constraint_value) { - RAY_RETURN_NOT_OK(CheckCgroup(cgroup)); - auto constraint_it = supported_constraints_.find(constraint); - if (constraint_it == supported_constraints_.end()) { - std::string supported_constraint_names("["); - for (auto it = supported_constraints_.begin(); it != supported_constraints_.end(); - ++it) { - supported_constraint_names.append(it->first); - if (std::next(it) != supported_constraints_.end()) { - supported_constraint_names.append(", "); - } - } - supported_constraint_names.append("]"); - return Status::InvalidArgument(absl::StrFormat( - "Failed to apply constraint %s to cgroup %s. Ray only supports %s", - constraint, - cgroup, - supported_constraint_names)); - } - - // Check if the constraint value is out of range and therefore invalid. - auto [low, high] = constraint_it->second.range; - size_t value = static_cast(std::stoi(constraint_value)); - if (value < low || value > high) { - return Status::InvalidArgument(absl::StrFormat( - "Failed to apply constraint %s=%s to cgroup %s. %s can only have values " - "in the range[%i, %i].", - constraint, - constraint_value, - cgroup, - constraint, - low, - high)); - } - + RAY_RETURN_NOT_OK(CheckCgroup(cgroup_path)); // Check if the required controller for the constraint is enabled. - const std::string &controller = constraint_it->second.controller; StatusOr> available_controllers_s = - GetEnabledControllers(cgroup); + GetEnabledControllers(cgroup_path); RAY_RETURN_NOT_OK(available_controllers_s.status()); const auto &controllers = available_controllers_s.value(); if (controllers.find(controller) == controllers.end()) { return Status::InvalidArgument(absl::StrFormat( "Failed to apply %s to cgroup %s. To use %s, enable the %s controller.", constraint, - cgroup, + cgroup_path, constraint, controller)); } // Try to apply the constraint and propagate the appropriate failure error. std::string file_path = - cgroup + std::filesystem::path::preferred_separator + constraint; + cgroup_path + std::filesystem::path::preferred_separator + constraint; int fd = open(file_path.c_str(), O_RDWR); @@ -361,7 +326,7 @@ Status SysFsCgroupDriver::AddConstraint(const std::string &cgroup, "Error: %s", constraint, constraint_value, - cgroup, + cgroup_path, strerror(errno))); } @@ -374,7 +339,7 @@ Status SysFsCgroupDriver::AddConstraint(const std::string &cgroup, "Error: %s", constraint, constraint_value, - cgroup, + cgroup_path, strerror(errno))); } close(fd); diff --git a/src/ray/common/cgroup2/sysfs_cgroup_driver.h b/src/ray/common/cgroup2/sysfs_cgroup_driver.h index fd56d129617b..2f654115d282 100644 --- a/src/ray/common/cgroup2/sysfs_cgroup_driver.h +++ b/src/ray/common/cgroup2/sysfs_cgroup_driver.h @@ -188,8 +188,7 @@ class SysFsCgroupDriver : public CgroupDriverInterface { https://docs.kernel.org/admin-guide/cgroup-v2.html#controlling-controllers @param cgroup_path absolute path of the cgroup. - @param controller name of the controller i.e. "cpu" or "memory" from - @ref CgroupDriverInterface::supported_controllers_ "supported controllers". + @param controller name of the controller e.g. "cpu", "memory" etc. @return Status::OK if successful @return Status::NotFound if the cgroup does not exist. @@ -225,19 +224,22 @@ class SysFsCgroupDriver : public CgroupDriverInterface { const std::string &controller) override; /** - Adds a constraint to the respective cgroup file. See - @ref CgroupDriverInterface::supported_constraints_ "supported constraints" and valid - values. + Adds a constraint to the respective cgroup file. + + @param cgroup_path absolute path of the cgroup. + @param controller the name of the controller + @param constraint the name of the cgroup file to add the constraint to e.g. cpu.weight + @param constraint_value @return Status::OK if no errors are encounted. @return Status::NotFound if the cgroup does not exist. @return Status::PermissionDenied if current user doesn't have read, write, and execute permissions. - @return Status::InvalidArgument if the cgroup is not using cgroupv2, the constraint - is not supported in ray, the constraint value is out of range, or if cannot write - to the relevant constraint file. + @return Status::InvalidArgument if the cgroup is not using cgroupv2, controller is not + enabled, or cannot write to the constraint file. */ Status AddConstraint(const std::string &cgroup, + const std::string &controller, const std::string &constraint, const std::string &constraint_value) override; diff --git a/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc b/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc index a381c81f97de..70d123c5f5fb 100644 --- a/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc +++ b/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc @@ -125,7 +125,7 @@ TEST(SysFsCgroupDriver, AddConstraintFailsIfNotCgroupv2Path) { ASSERT_TRUE(temp_dir_or_status.ok()) << temp_dir_or_status.ToString(); std::unique_ptr temp_dir = std::move(temp_dir_or_status.value()); SysFsCgroupDriver driver; - Status s = driver.AddConstraint(temp_dir->GetPath(), "memory.min", "1"); + Status s = driver.AddConstraint(temp_dir->GetPath(), "memory", "memory.min", "1"); ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); } From a5591b5564e00289b0d5abbd533ada668c50f7ac Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Fri, 5 Sep 2025 18:21:35 -0700 Subject: [PATCH 486/634] [Core] Remove test_head_node_job_agent_always_used (#56292) Signed-off-by: Jiajun Yao --- .../dashboard/modules/job/tests/test_sdk.py | 49 ------------------- 1 file changed, 49 deletions(-) diff --git a/python/ray/dashboard/modules/job/tests/test_sdk.py b/python/ray/dashboard/modules/job/tests/test_sdk.py index cc7ec6bf0f90..f3c370b22b06 100644 --- a/python/ray/dashboard/modules/job/tests/test_sdk.py +++ b/python/ray/dashboard/modules/job/tests/test_sdk.py @@ -8,7 +8,6 @@ import pytest -import ray from ray._common.test_utils import wait_for_condition import ray.experimental.internal_kv as kv from ray._private.ray_constants import ( @@ -172,54 +171,6 @@ def get_register_agents_number(gcs_client): return len(keys) -@pytest.mark.parametrize( - "ray_start_cluster_head_with_env_vars", - [ - { - "include_dashboard": True, - "env_vars": { - RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR: "1", - "RAY_health_check_initial_delay_ms": "0", - "RAY_health_check_period_ms": "1000", - }, - } - ], - indirect=True, -) -def test_head_node_job_agent_always_used(ray_start_cluster_head_with_env_vars): - """Makes sure that job submission always uses the head node's job agent. - - 1. Create a cluster with a worker node and a head node. - 2. Submit 10 jobs. - 3. Make sure they all execute on the head node's job agent. - """ - cluster = ray_start_cluster_head_with_env_vars - assert wait_until_server_available(cluster.webui_url) is True - webui_url = cluster.webui_url - webui_url = format_web_url(webui_url) - client = JobSubmissionClient(webui_url) - - cluster_nodes = cluster.list_all_nodes() - assert len(cluster_nodes) == 1 and cluster_nodes[0].is_head - head_node_id = cluster_nodes[0].node_id - - # add a worker node. - cluster.add_node() - - job_ids = [client.submit_job(entrypoint="echo hello")] - - for job_id in job_ids: - wait_for_condition( - _check_job_succeeded, client=client, job_id=job_id, timeout=30 - ) - - actors = ray.state.actors() - - for _, actor_info in actors.items(): - if actor_info["Name"].startswith("_ray_internal_job_actor"): - assert actor_info["Address"]["NodeID"] == head_node_id - - @pytest.mark.parametrize( "ray_start_cluster_head_with_env_vars", [ From adcad134dac7dbd97db5d2fcbd1cd114688052d2 Mon Sep 17 00:00:00 2001 From: Haichuan Hu <74917084+KaisennHu@users.noreply.github.com> Date: Sat, 6 Sep 2025 12:10:13 +0800 Subject: [PATCH 487/634] [core] Add a warning when returning an object w/ num_returns=0 (#56213) Signed-off-by: Haichuan Hu Co-authored-by: Edward Oakes --- python/ray/_raylet.pyx | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index edcb38d2c41c..c9c87b70053d 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -262,6 +262,13 @@ GRPC_STATUS_CODE_UNIMPLEMENTED = CGrpcStatusCode.UNIMPLEMENTED logger = logging.getLogger(__name__) +import warnings +class NumReturnsWarning(UserWarning): + """Warning when num_returns=0 but the task returns a non-None value.""" + pass + +warnings.filterwarnings("once", category=NumReturnsWarning) + # The currently running task, if any. These are used to synchronize task # interruption for ray.cancel. current_task_id = None @@ -4413,6 +4420,17 @@ cdef class CoreWorker: num_returns = returns[0].size() if num_returns == 0: + if outputs is not None and len(outputs) > 0: + # Warn if num_returns=0 but the task returns a non-None value (likely unintended). + task_name = self.get_current_task_name() + obj_value = repr(outputs) + warnings.warn( + f"Task '{task_name}' has num_returns=0 but returned a non-None value '{obj_value}'. " + "The return value will be ignored.", + NumReturnsWarning, + stacklevel=2 + ) + return num_outputs_stored task_output_inlined_bytes = 0 From 7950cd1761aececf4f56ccc3e047149c7b6538b5 Mon Sep 17 00:00:00 2001 From: Alexey Kudinkin Date: Sat, 6 Sep 2025 00:17:42 -0400 Subject: [PATCH 488/634] [Data] Fixed `ParquetDatasource` encoding ratio estimation (#56268) ## Why are these changes needed? This change is a follow-up for https://github.com/ray-project/ray/pull/56105. Now dataset size estimation is based on listed file sizes. However, encoding ratio was still based on the file size estimates derived from the uncompressed data size obtained from Parquet metadata. This change is addressing that by: - Rebasing encoding ratio to relate estimated in-memory size to the listed file size - Cleaning up unused abstractions (like `ParquetMetadataProvider`) ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Alexey Kudinkin --- ci/lint/pydoclint-baseline.txt | 4 - doc/source/data/api/input_output.rst | 1 - .../datasource/parquet_datasource.py | 426 +++++++++--------- python/ray/data/datasource/__init__.py | 2 - .../ray/data/datasource/file_meta_provider.py | 1 - .../data/datasource/parquet_meta_provider.py | 236 ---------- python/ray/data/read_api.py | 8 +- .../ray/data/tests/test_metadata_provider.py | 55 --- python/ray/data/tests/test_parquet.py | 216 +-------- python/ray/data/tests/test_size_estimation.py | 2 +- 10 files changed, 230 insertions(+), 721 deletions(-) delete mode 100644 python/ray/data/datasource/parquet_meta_provider.py diff --git a/ci/lint/pydoclint-baseline.txt b/ci/lint/pydoclint-baseline.txt index a6ab3cb60a2b..8aa286605a2c 100644 --- a/ci/lint/pydoclint-baseline.txt +++ b/ci/lint/pydoclint-baseline.txt @@ -1332,10 +1332,6 @@ python/ray/data/datasource/filename_provider.py DOC201: Method `FilenameProvider.get_filename_for_block` does not have a return section in docstring DOC201: Method `FilenameProvider.get_filename_for_row` does not have a return section in docstring -------------------- -python/ray/data/datasource/parquet_meta_provider.py - DOC101: Method `ParquetMetadataProvider.prefetch_file_metadata`: Docstring contains fewer arguments than in function signature. - DOC103: Method `ParquetMetadataProvider.prefetch_file_metadata`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**ray_remote_args: ]. --------------------- python/ray/data/datasource/path_util.py DOC201: Function `_has_file_extension` does not have a return section in docstring DOC201: Function `_resolve_paths_and_filesystem` does not have a return section in docstring diff --git a/doc/source/data/api/input_output.rst b/doc/source/data/api/input_output.rst index 9c4ad3868bb2..0b32168c35f6 100644 --- a/doc/source/data/api/input_output.rst +++ b/doc/source/data/api/input_output.rst @@ -410,7 +410,6 @@ MetadataProvider API datasource.FileMetadataProvider datasource.BaseFileMetadataProvider datasource.DefaultFileMetadataProvider - datasource.ParquetMetadataProvider datasource.FastFileMetadataProvider Shuffling API diff --git a/python/ray/data/_internal/datasource/parquet_datasource.py b/python/ray/data/_internal/datasource/parquet_datasource.py index e893365a7e71..8cb6db7600ae 100644 --- a/python/ray/data/_internal/datasource/parquet_datasource.py +++ b/python/ray/data/_internal/datasource/parquet_datasource.py @@ -1,4 +1,5 @@ import logging +import math import warnings from dataclasses import dataclass from typing import ( @@ -15,9 +16,10 @@ ) import numpy as np +from packaging.version import parse as parse_version import ray -import ray.cloudpickle as cloudpickle +from ray._private.arrow_utils import get_pyarrow_version from ray.data._internal.progress_bar import ProgressBar from ray.data._internal.remote_fn import cached_remote_fn from ray.data._internal.util import ( @@ -26,19 +28,16 @@ _is_local_scheme, iterate_with_retry, ) -from ray.data.block import Block, BlockAccessor +from ray.data.block import Block, BlockAccessor, BlockMetadata from ray.data.context import DataContext from ray.data.datasource import Datasource from ray.data.datasource.datasource import ReadTask from ray.data.datasource.file_based_datasource import FileShuffleConfig from ray.data.datasource.file_meta_provider import ( + FileMetadataProvider, _handle_read_os_error, _list_files, ) -from ray.data.datasource.parquet_meta_provider import ( - ParquetFileMetadata, - ParquetMetadataProvider, -) from ray.data.datasource.partitioning import ( PartitionDataType, Partitioning, @@ -58,13 +57,17 @@ logger = logging.getLogger(__name__) + +MIN_PYARROW_TO_BATCHES_READAHEAD = parse_version("10.0.0") + + # The `num_cpus` for each metadata prefetching task. # Default to 0.5 instead of 1 because it is cheaper than normal read task. NUM_CPUS_FOR_META_FETCH_TASK = 0.5 # The number of rows to read per batch. This is sized to generate 10MiB batches # for rows about 1KiB in size. -PARQUET_READER_ROW_BATCH_SIZE = 10_000 +DEFAULT_PARQUET_READER_ROW_BATCH_SIZE = 10_000 FILE_READING_RETRY = 8 # The default size multiplier for reading Parquet data source in Arrow. @@ -97,43 +100,36 @@ PARQUET_ENCODING_RATIO_ESTIMATE_NUM_ROWS = 1024 -@dataclass(frozen=True) -class _SampleInfo: - actual_bytes_per_row: Optional[int] - estimated_bytes_per_row: Optional[int] - - -class _NoIOSerializableFragmentWrapper: - """This is a workaround to avoid utilizing `ParquetFileFragment` original +class _ParquetFragment: + """This wrapper class is created to avoid utilizing `ParquetFileFragment` original serialization protocol that actually does network RPCs during serialization - (to fetch metadata)""" + (to fetch actual parquet metadata)""" - def __init__(self, f: "ParquetFileFragment"): + def __init__(self, f: "ParquetFileFragment", file_size: int): self._fragment = f + self._file_size = file_size + + @property + def file_size(self) -> int: + return self._file_size + + @property + def original(self) -> "ParquetFileFragment": + return self._fragment def __reduce__(self): - return self._fragment.format.make_fragment, ( + return _ParquetFragment.make_fragment, ( + self._fragment.format, self._fragment.path, self._fragment.filesystem, self._fragment.partition_expression, + self._file_size, ) - def deserialize(self) -> "ParquetFileFragment": - # Implicitly trigger S3 subsystem initialization by importing - # pyarrow.fs. - import pyarrow.fs # noqa: F401 - - (file_format, path, filesystem, partition_expression) = cloudpickle.loads( - self._data - ) - return file_format.make_fragment(path, filesystem, partition_expression) - - -# Visible for test mocking. -def _deserialize_fragments( - serialized_fragments: List[_NoIOSerializableFragmentWrapper], -) -> List["pyarrow._dataset.ParquetFileFragment"]: - return [p.deserialize() for p in serialized_fragments] + @staticmethod + def make_fragment(format, path, filesystem, partition_expression, file_size): + fragment = format.make_fragment(path, filesystem, partition_expression) + return _ParquetFragment(fragment, file_size) def check_for_legacy_tensor_type(schema): @@ -181,7 +177,7 @@ def __init__( _block_udf: Optional[Callable[[Block], Block]] = None, filesystem: Optional["pyarrow.fs.FileSystem"] = None, schema: Optional[Union[type, "pyarrow.lib.Schema"]] = None, - meta_provider: ParquetMetadataProvider = ParquetMetadataProvider(), + meta_provider: Optional[FileMetadataProvider] = None, partition_filter: PathPartitionFilter = None, partitioning: Optional[Partitioning] = Partitioning("hive"), shuffle: Union[Literal["files"], None] = None, @@ -269,30 +265,6 @@ def __init__( columns, pq_ds.fragments[0], partitioning ) - try: - prefetch_remote_args = {} - prefetch_remote_args["num_cpus"] = NUM_CPUS_FOR_META_FETCH_TASK - if self._local_scheduling: - prefetch_remote_args["scheduling_strategy"] = self._local_scheduling - else: - # Use the scheduling strategy ("SPREAD" by default) provided in - # `DataContext``, to spread out prefetch tasks in cluster, avoid - # AWS S3 throttling error. - # Note: this is the same scheduling strategy used by read tasks. - prefetch_remote_args[ - "scheduling_strategy" - ] = DataContext.get_current().scheduling_strategy - - self._metadata = [ - ParquetFileMetadata( - num_bytes=num_bytes, - ) - for num_bytes in file_sizes - ] - - except OSError as e: - _handle_read_os_error(e, paths) - if to_batch_kwargs is None: to_batch_kwargs = {} @@ -300,10 +272,10 @@ def __init__( # network calls when `_ParquetDatasourceReader` is serialized. See # `_SerializedFragment()` implementation for more details. self._pq_fragments = [ - _NoIOSerializableFragmentWrapper(p) for p in pq_ds.fragments + _ParquetFragment(fragment, file_size) + for fragment, file_size in zip(pq_ds.fragments, file_sizes) ] self._pq_paths = [p.path for p in pq_ds.fragments] - self._meta_provider = meta_provider self._block_udf = _block_udf self._to_batches_kwargs = to_batch_kwargs self._data_columns = data_columns @@ -313,21 +285,35 @@ def __init__( self._file_metadata_shuffler = None self._include_paths = include_paths self._partitioning = partitioning + if shuffle == "files": self._file_metadata_shuffler = np.random.default_rng() elif isinstance(shuffle, FileShuffleConfig): self._file_metadata_shuffler = np.random.default_rng(shuffle.seed) - sample_infos = sample_fragments( + # Sample small number of parquet files to estimate + # - Encoding ratio: ratio of file size on disk to approximate expected + # size of the corresponding block in memory + # - Default batch-size: number of rows to be read from a file at a time, + # used to limit amount of memory pressure + sampled_fragments = _sample_fragments( self._pq_fragments, - to_batches_kwargs=to_batch_kwargs, - columns=data_columns, - schema=self._read_schema, + ) + + sampled_file_infos = _fetch_file_infos( + sampled_fragments, + columns=self._data_columns, + schema=schema, local_scheduling=self._local_scheduling, ) - self._encoding_ratio = estimate_files_encoding_ratio(sample_infos) - self._default_read_batch_size_rows = estimate_default_read_batch_size_rows( - sample_infos + + self._encoding_ratio = _estimate_files_encoding_ratio( + sampled_fragments, + sampled_file_infos, + ) + + self._default_batch_size = _estimate_reader_batch_size( + sampled_file_infos, DataContext.get_current().target_max_block_size ) if file_extensions is None: @@ -338,60 +324,41 @@ def __init__( emit_file_extensions_future_warning(self._FUTURE_FILE_EXTENSIONS) break - def estimate_inmemory_data_size(self) -> Optional[int]: - total_size = 0 - for file_metadata in self._metadata: - total_size += file_metadata.num_bytes - return total_size * self._encoding_ratio + def estimate_inmemory_data_size(self) -> int: + return self._estimate_in_mem_size(self._pq_fragments) def get_read_tasks(self, parallelism: int) -> List[ReadTask]: # NOTE: We override the base class FileBasedDatasource.get_read_tasks() # method in order to leverage pyarrow's ParquetDataset abstraction, # which simplifies partitioning logic. We still use # FileBasedDatasource's write side, however. - pq_metadata = self._metadata - if len(pq_metadata) < len(self._pq_fragments): - # Pad `pq_metadata` to be same length of `self._pq_fragments`. - # This can happen when no file metadata being prefetched. - pq_metadata += [None] * (len(self._pq_fragments) - len(pq_metadata)) - if self._file_metadata_shuffler is not None: - files_metadata = list(zip(self._pq_fragments, self._pq_paths, pq_metadata)) + files_metadata = list(zip(self._pq_fragments, self._pq_paths)) shuffled_files_metadata = [ files_metadata[i] for i in self._file_metadata_shuffler.permutation(len(files_metadata)) ] - pq_fragments, pq_paths, pq_metadata = list( - map(list, zip(*shuffled_files_metadata)) - ) + pq_fragments, pq_paths = list(map(list, zip(*shuffled_files_metadata))) else: - pq_fragments, pq_paths, pq_metadata = ( + pq_fragments, pq_paths = ( self._pq_fragments, self._pq_paths, - pq_metadata, ) read_tasks = [] - for fragments, paths, metadata in zip( + for fragments, paths in zip( np.array_split(pq_fragments, parallelism), np.array_split(pq_paths, parallelism), - np.array_split(pq_metadata, parallelism), ): if len(fragments) <= 0: continue - meta = self._meta_provider( - paths, - num_fragments=len(fragments), - prefetched_metadata=metadata, + meta = BlockMetadata( + num_rows=None, + size_bytes=self._estimate_in_mem_size(fragments), + input_files=paths, + exec_stats=None, ) - # If there is a filter operation, reset the calculated row count, - # since the resulting row count is unknown. - if self._to_batches_kwargs.get("filter") is not None: - meta.num_rows = None - - if meta.size_bytes is not None: - meta.size_bytes = int(meta.size_bytes * self._encoding_ratio) ( block_udf, @@ -405,7 +372,7 @@ def get_read_tasks(self, parallelism: int) -> List[ReadTask]: ) = ( self._block_udf, self._to_batches_kwargs, - self._default_read_batch_size_rows, + self._default_batch_size, self._data_columns, self._partition_columns, self._read_schema, @@ -444,6 +411,11 @@ def get_name(self): def supports_distributed_reads(self) -> bool: return self._supports_distributed_reads + def _estimate_in_mem_size(self, fragments: List[_ParquetFragment]) -> int: + in_mem_size = sum([f.file_size for f in fragments]) * self._encoding_ratio + + return round(in_mem_size) + def read_fragments( block_udf, @@ -452,7 +424,7 @@ def read_fragments( data_columns, partition_columns, schema, - fragments: List["ParquetFileFragment"], + fragments: List[_ParquetFragment], include_paths: bool, partitioning: Partitioning, ) -> Iterator["pyarrow.Table"]: @@ -465,13 +437,14 @@ def read_fragments( import pyarrow as pa logger.debug(f"Reading {len(fragments)} parquet fragments") + use_threads = to_batches_kwargs.pop("use_threads", False) batch_size = to_batches_kwargs.pop("batch_size", default_read_batch_size_rows) for fragment in fragments: partitions = {} if partitioning is not None: parse = PathPartitionParser(partitioning) - partitions = parse(fragment.path) + partitions = parse(fragment.original.path) # Filter out partitions that aren't in the user-specified columns list. if partition_columns is not None: @@ -485,7 +458,7 @@ def get_batch_iterable(): if batch_size is not None: to_batches_kwargs["batch_size"] = batch_size - return fragment.to_batches( + return fragment.original.to_batches( use_threads=use_threads, columns=data_columns, schema=schema, @@ -501,7 +474,7 @@ def get_batch_iterable(): table = pa.Table.from_batches([batch], schema=schema) if include_paths: table = BlockAccessor.for_block(table).fill_column( - "path", fragment.path + "path", fragment.original.path ) if partitions: table = _add_partitions_to_table(partitions, table) @@ -514,56 +487,77 @@ def get_batch_iterable(): yield table -def _sample_fragment( - to_batches_kwargs, - columns, - schema, - fragment: "ParquetFileFragment", -) -> _SampleInfo: +def _fetch_parquet_file_info( + fragment: _ParquetFragment, + *, + columns: Optional[List[str]], + schema: Optional["pyarrow.Schema"], +) -> Optional["_ParquetFileInfo"]: # If the fragment has no row groups, it's an empty or metadata-only file. # Skip it by returning empty sample info. - if fragment.metadata.num_row_groups == 0: - return _SampleInfo(actual_bytes_per_row=None, estimated_bytes_per_row=None) + # + # NOTE: Accessing `ParquetFileFragment.metadata` does fetch a parquet footer + # from storage + metadata = fragment.original.metadata + + if metadata.num_row_groups == 0: + return None # Only sample the first row group. - fragment = fragment.subset(row_group_ids=[0]) + row_group_fragment = fragment.original.subset(row_group_ids=[0]) batch_size = max( - min(fragment.metadata.num_rows, PARQUET_ENCODING_RATIO_ESTIMATE_NUM_ROWS), 1 + min( + row_group_fragment.metadata.num_rows, + PARQUET_ENCODING_RATIO_ESTIMATE_NUM_ROWS, + ), + 1, ) - # Use the batch_size calculated above, and ignore the one specified by user if set. - # This is to avoid sampling too few or too many rows. - to_batches_kwargs.pop("batch_size", None) - batches = fragment.to_batches( + + to_batches_kwargs = {} + + if get_pyarrow_version() >= MIN_PYARROW_TO_BATCHES_READAHEAD: + # Limit prefetching to just 1 batch + to_batches_kwargs["batch_readahead"] = 1 + + batches_iter = row_group_fragment.to_batches( columns=columns, schema=schema, batch_size=batch_size, **to_batches_kwargs, ) - # Use first batch in-memory size for estimation. - try: - batch = next(batches) - except StopIteration: - sample_data = _SampleInfo( - actual_bytes_per_row=None, estimated_bytes_per_row=None - ) - else: + + avg_row_size: Optional[int] = None + # Use first batch non-empty batch to estimate the avg size of the + # row in-memory + for batch in batches_iter: if batch.num_rows > 0: - metadata = fragment.metadata - total_size = 0 - for idx in range(metadata.num_row_groups): - total_size += metadata.row_group(idx).total_byte_size - sample_data = _SampleInfo( - actual_bytes_per_row=batch.nbytes / batch.num_rows, - estimated_bytes_per_row=total_size / metadata.num_rows, - ) - else: - sample_data = _SampleInfo( - actual_bytes_per_row=None, estimated_bytes_per_row=None - ) - return sample_data + avg_row_size = math.ceil(batch.nbytes / batch.num_rows) + break + return _ParquetFileInfo( + avg_row_in_mem_bytes=avg_row_size, + metadata=metadata, + ) -def estimate_files_encoding_ratio(sample_infos: List[_SampleInfo]) -> float: + +@dataclass +class _ParquetFileInfo: + # Estimated avg byte size of a row (in-memory) + avg_row_in_mem_bytes: Optional[int] + # Corresponding file metadata + metadata: "pyarrow._parquet.FileMetaData" + + def estimate_in_memory_bytes(self) -> Optional[int]: + if self.avg_row_in_mem_bytes is None: + return None + + return self.avg_row_in_mem_bytes * self.metadata.num_rows + + +def _estimate_files_encoding_ratio( + fragments: List[_ParquetFragment], + file_infos: List[_ParquetFileInfo], +) -> float: """Return an estimate of the Parquet files encoding ratio. To avoid OOMs, it is safer to return an over-estimate than an underestimate. @@ -571,46 +565,90 @@ def estimate_files_encoding_ratio(sample_infos: List[_SampleInfo]) -> float: if not DataContext.get_current().decoding_size_estimation: return PARQUET_ENCODING_RATIO_ESTIMATE_DEFAULT - def compute_encoding_ratio(sample_info: _SampleInfo) -> float: - if ( - sample_info.actual_bytes_per_row is None - or sample_info.estimated_bytes_per_row is None - ): - return PARQUET_ENCODING_RATIO_ESTIMATE_LOWER_BOUND - else: - return ( - sample_info.actual_bytes_per_row / sample_info.estimated_bytes_per_row + assert len(file_infos) == len(fragments) + + # Estimate size of the rows in a file in memory + estimated_in_mem_size_arr = [ + fi.estimate_in_memory_bytes() if fi is not None else None for fi in file_infos + ] + + file_size_arr = [f.file_size for f in fragments] + + estimated_encoding_ratios = [ + float(in_mem_size) / file_size + for in_mem_size, file_size in zip(estimated_in_mem_size_arr, file_size_arr) + if file_size > 0 and in_mem_size is not None + ] + + # Return default estimate of 5 if all sampled files turned out to be empty + if not estimated_encoding_ratios: + return PARQUET_ENCODING_RATIO_ESTIMATE_DEFAULT + + estimated_ratio = np.mean(estimated_encoding_ratios) + + logger.info(f"Estimated parquet encoding ratio is {estimated_ratio:.3f}.") + + return max(estimated_ratio, PARQUET_ENCODING_RATIO_ESTIMATE_LOWER_BOUND) + + +def _fetch_file_infos( + sampled_fragments: List[_ParquetFragment], + *, + columns: Optional[List[str]], + schema: Optional["pyarrow.Schema"], + local_scheduling: Optional[bool], +) -> List[Optional[_ParquetFileInfo]]: + fetc_file_info = cached_remote_fn(_fetch_parquet_file_info) + futures = [] + + for fragment in sampled_fragments: + # Sample the first rows batch in i-th file. + # Use SPREAD scheduling strategy to avoid packing many sampling tasks on + # same machine to cause OOM issue, as sampling can be memory-intensive. + futures.append( + fetc_file_info.options( + scheduling_strategy=local_scheduling + or DataContext.get_current().scheduling_strategy, + # Retry in case of transient errors during sampling. + retry_exceptions=[OSError], + ).remote( + fragment, + columns=columns, + schema=schema, ) + ) - ratio = np.mean(list(map(compute_encoding_ratio, sample_infos))) - logger.debug(f"Estimated Parquet encoding ratio from sampling is {ratio}.") - return max(ratio, PARQUET_ENCODING_RATIO_ESTIMATE_LOWER_BOUND) + sample_bar = ProgressBar("Parquet dataset sampling", len(futures), unit="file") + file_infos = sample_bar.fetch_until_complete(futures) + sample_bar.close() + return file_infos -def estimate_default_read_batch_size_rows( - sample_infos: List[_SampleInfo], + +def _estimate_reader_batch_size( + file_infos: List[Optional[_ParquetFileInfo]], target_block_size: Optional[int] ) -> Optional[int]: - ctx = DataContext.get_current() - if ctx.target_max_block_size is None: + if target_block_size is None: return None - def compute_batch_size_rows(sample_info: _SampleInfo) -> int: - # 'actual_bytes_per_row' is None if the sampled file was empty and 0 if the data - # was all null. - if not sample_info.actual_bytes_per_row: - return PARQUET_READER_ROW_BATCH_SIZE - else: - max_parquet_reader_row_batch_size_bytes = ctx.target_max_block_size // 10 - return max( - 1, - min( - PARQUET_READER_ROW_BATCH_SIZE, - max_parquet_reader_row_batch_size_bytes - // sample_info.actual_bytes_per_row, - ), - ) + avg_num_rows_per_block = [ + target_block_size / fi.avg_row_in_mem_bytes + for fi in file_infos + if ( + fi is not None + and fi.avg_row_in_mem_bytes is not None + and fi.avg_row_in_mem_bytes > 0 + ) + ] + + if not avg_num_rows_per_block: + return DEFAULT_PARQUET_READER_ROW_BATCH_SIZE - return np.mean(list(map(compute_batch_size_rows, sample_infos))) + estimated_batch_size: int = max(math.ceil(np.mean(avg_num_rows_per_block)), 1) + + logger.info(f"Estimated parquet reader batch size at {estimated_batch_size} rows") + + return estimated_batch_size def get_parquet_dataset(paths, filesystem, dataset_kwargs): @@ -634,19 +672,10 @@ def get_parquet_dataset(paths, filesystem, dataset_kwargs): return dataset -def sample_fragments( - serialized_fragments, - *, - to_batches_kwargs, - columns, - schema, - local_scheduling=None, -) -> List[_SampleInfo]: - # Sample a few rows from Parquet files to estimate the encoding ratio. - # Launch tasks to sample multiple files remotely in parallel. - # Evenly distributed to sample N rows in i-th row group in i-th file. - # TODO(ekl/cheng) take into account column pruning. - num_files = len(serialized_fragments) +def _sample_fragments( + fragments: List[_ParquetFragment], +) -> List[_ParquetFragment]: + num_files = len(fragments) num_samples = int(num_files * PARQUET_ENCODING_RATIO_ESTIMATE_SAMPLING_RATIO) min_num_samples = min(PARQUET_ENCODING_RATIO_ESTIMATE_MIN_NUM_SAMPLES, num_files) max_num_samples = min(PARQUET_ENCODING_RATIO_ESTIMATE_MAX_NUM_SAMPLES, num_files) @@ -654,36 +683,11 @@ def sample_fragments( # Evenly distributed to choose which file to sample, to avoid biased prediction # if data is skewed. - file_samples = [ - serialized_fragments[idx] + return [ + fragments[idx] for idx in np.linspace(0, num_files - 1, num_samples).astype(int).tolist() ] - sample_fragment = cached_remote_fn(_sample_fragment) - futures = [] - scheduling = local_scheduling or DataContext.get_current().scheduling_strategy - for sample in file_samples: - # Sample the first rows batch in i-th file. - # Use SPREAD scheduling strategy to avoid packing many sampling tasks on - # same machine to cause OOM issue, as sampling can be memory-intensive. - futures.append( - sample_fragment.options( - scheduling_strategy=scheduling, - # Retry in case of transient errors during sampling. - retry_exceptions=[OSError], - ).remote( - to_batches_kwargs, - columns, - schema, - sample, - ) - ) - sample_bar = ProgressBar("Parquet Files Sample", len(futures), unit="file") - sample_infos = sample_bar.fetch_until_complete(futures) - sample_bar.close() - - return sample_infos - def _add_partitions_to_table( partitions: Dict[str, PartitionDataType], table: "pyarrow.Table" diff --git a/python/ray/data/datasource/__init__.py b/python/ray/data/datasource/__init__.py index ef2eca5977ed..1e76f1bfd9d0 100644 --- a/python/ray/data/datasource/__init__.py +++ b/python/ray/data/datasource/__init__.py @@ -28,7 +28,6 @@ FileMetadataProvider, ) from ray.data.datasource.filename_provider import FilenameProvider -from ray.data.datasource.parquet_meta_provider import ParquetMetadataProvider from ray.data.datasource.partitioning import ( Partitioning, PartitionStyle, @@ -53,7 +52,6 @@ "FileShuffleConfig", "FileMetadataProvider", "FilenameProvider", - "ParquetMetadataProvider", "PartitionStyle", "PathPartitionFilter", "PathPartitionParser", diff --git a/python/ray/data/datasource/file_meta_provider.py b/python/ray/data/datasource/file_meta_provider.py index 354f761d9651..5d3b2b55cb45 100644 --- a/python/ray/data/datasource/file_meta_provider.py +++ b/python/ray/data/datasource/file_meta_provider.py @@ -37,7 +37,6 @@ class FileMetadataProvider: Current subclasses: - :class:`BaseFileMetadataProvider` - - :class:`ParquetMetadataProvider` """ def _get_block_metadata( diff --git a/python/ray/data/datasource/parquet_meta_provider.py b/python/ray/data/datasource/parquet_meta_provider.py deleted file mode 100644 index 25b33f27411e..000000000000 --- a/python/ray/data/datasource/parquet_meta_provider.py +++ /dev/null @@ -1,236 +0,0 @@ -import logging -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, List, Optional, Tuple - -from ray.data._internal.util import call_with_retry -from ray.data.block import BlockMetadata -from ray.data.datasource.file_meta_provider import ( - FileMetadataProvider, - _fetch_metadata_parallel, -) -from ray.util.annotations import DeveloperAPI - -if TYPE_CHECKING: - import pyarrow - from pyarrow.dataset import ParquetFileFragment - - -FRAGMENTS_PER_META_FETCH = 6 -PARALLELIZE_META_FETCH_THRESHOLD = 24 - -# The application-level exceptions to retry for metadata prefetching task. -# Default to retry on access denied and read timeout errors because AWS S3 would throw -# these transient errors when load is too high. -RETRY_EXCEPTIONS_FOR_META_FETCH_TASK = ["AWS Error ACCESS_DENIED", "Timeout"] -# Maximum number of retries for metadata prefetching task due to transient errors. -RETRY_MAX_ATTEMPTS_FOR_META_FETCH_TASK = 32 -# Maximum retry back-off interval in seconds for failed metadata prefetching task. -RETRY_MAX_BACKOFF_S_FOR_META_FETCH_TASK = 64 - - -logger = logging.getLogger(__name__) - - -@DeveloperAPI(stability="alpha") -@dataclass -class ParquetFileMetadata: - num_bytes: int - num_rows: Optional[int] = field(default=None) - - @classmethod - def from_(cls, pqm: "pyarrow.parquet.FileMetaData"): - return ParquetFileMetadata( - num_rows=pqm.num_rows, - num_bytes=_get_total_bytes(pqm), - ) - - -@DeveloperAPI -class ParquetMetadataProvider(FileMetadataProvider): - """Provides block metadata for Arrow Parquet file fragments.""" - - def _get_block_metadata( - self, - paths: List[str], - *, - num_fragments: int, - prefetched_metadata: Optional[List["ParquetFileMetadata"]], - ) -> BlockMetadata: - """Resolves and returns block metadata for files of a single dataset block. - - Args: - paths: The file paths for a single dataset block. - num_fragments: The number of Parquet file fragments derived from the input - file paths. - prefetched_metadata: Metadata previously returned from - `prefetch_file_metadata()` for each file fragment, where - `prefetched_metadata[i]` contains the metadata for `fragments[i]`. - - Returns: - BlockMetadata aggregated across the given file paths. - """ - if ( - prefetched_metadata is not None - and len(prefetched_metadata) == num_fragments - and all(m is not None for m in prefetched_metadata) - ): - total_bytes, total_rows = self._derive_totals(prefetched_metadata) - - # Fragment metadata was available, construct a normal - # BlockMetadata. - block_metadata = BlockMetadata( - num_rows=total_rows, - size_bytes=total_bytes, - input_files=paths, - exec_stats=None, - ) # Exec stats filled in later. - else: - # Fragment metadata was not available, construct an empty - # BlockMetadata. - block_metadata = BlockMetadata( - num_rows=None, - size_bytes=None, - input_files=paths, - exec_stats=None, - ) - return block_metadata - - @staticmethod - def _derive_totals( - prefetched_metadata: List["ParquetFileMetadata"], - ) -> Tuple[int, int]: - total_bytes = 0 - total_rows = 0 - - for m in prefetched_metadata: - total_bytes += m.num_bytes - - if total_rows is not None: - if m.num_rows is not None: - total_rows += m.num_rows - else: - total_rows = None - - return total_bytes, total_rows - - def prefetch_file_metadata( - self, - fragments: List["pyarrow.dataset.ParquetFileFragment"], - **ray_remote_args, - ) -> Optional[List[ParquetFileMetadata]]: - """Pre-fetches file metadata for all Parquet file fragments in a single batch. - - Subsets of the metadata returned will be provided as input to subsequent calls - to ``_get_block_metadata`` together with their corresponding Parquet file - fragments. - - Args: - fragments: The Parquet file fragments to fetch metadata for. - - Returns: - Metadata resolved for each input file fragment, or `None`. Metadata - must be returned in the same order as all input file fragments, such - that `metadata[i]` always contains the metadata for `fragments[i]`. - """ - from ray.data._internal.datasource.parquet_datasource import ( - _NoIOSerializableFragmentWrapper, - ) - - if len(fragments) > PARALLELIZE_META_FETCH_THRESHOLD: - # Wrap Parquet fragments in serialization workaround. - fragments = [ - _NoIOSerializableFragmentWrapper(fragment) for fragment in fragments - ] - # Fetch Parquet metadata in parallel using Ray tasks. - def _remote_fetch(fragments: List["ParquetFileFragment"]): - return _fetch_metadata_with_retry( - fragments, - # Ensure that retry settings are propagated to remote tasks. - retry_match=RETRY_EXCEPTIONS_FOR_META_FETCH_TASK, - retry_max_attempts=RETRY_MAX_ATTEMPTS_FOR_META_FETCH_TASK, - retry_max_interval=RETRY_MAX_BACKOFF_S_FOR_META_FETCH_TASK, - ) - - raw_metadata = list( - _fetch_metadata_parallel( - fragments, - _remote_fetch, - FRAGMENTS_PER_META_FETCH, - **ray_remote_args, - ) - ) - - return raw_metadata - - else: - # We don't deduplicate schemas in this branch because they're already - # deduplicated in `_fetch_metadata`. See - # https://github.com/ray-project/ray/pull/54821/files#r2265140929 for - # related discussion. - raw_metadata = _fetch_metadata(fragments) - return raw_metadata - - -def _fetch_metadata_with_retry( - fragments: List["ParquetFileFragment"], - retry_match: Optional[List[str]], - retry_max_attempts: int, - retry_max_interval: int, -) -> List["ParquetFileMetadata"]: - try: - metadata = call_with_retry( - lambda: _fetch_metadata(fragments), - description="fetch metdata", - match=retry_match, - max_attempts=retry_max_attempts, - max_backoff_s=retry_max_interval, - ) - except OSError as e: - raise RuntimeError( - f"Exceeded maximum number of attempts ({retry_max_attempts}) to retry " - "metadata fetching task. Metadata fetching tasks can fail due to transient " - "errors like rate limiting.\n" - "\n" - "To increase the maximum number of attempts, configure " - "`RETRY_MAX_ATTEMPTS_FOR_META_FETCH_TASK`. For example:\n" - "```\n" - "ray.data._internal.datasource.parquet_datasource.RETRY_MAX_ATTEMPTS_FOR_META_FETCH_TASK = 64\n" # noqa: E501 - "```\n" - "To increase the maximum retry backoff interval, configure " - "`RETRY_MAX_BACKOFF_S_FOR_META_FETCH_TASK`. For example:\n" - "```\n" - "ray.data._internal.datasource.parquet_datasource.RETRY_MAX_BACKOFF_S_FOR_META_FETCH_TASK = 128\n" # noqa: E501 - "```\n" - "If the error continues to occur, you can also try decresasing the " - "concurency of metadata fetching tasks by setting " - "`NUM_CPUS_FOR_META_FETCH_TASK` to a larger value. For example:\n" - "```\n" - "ray.data._internal.datasource.parquet_datasource.NUM_CPUS_FOR_META_FETCH_TASK = 4.\n" # noqa: E501 - "```\n" - "To change which exceptions to retry on, set " - "`RETRY_EXCEPTIONS_FOR_META_FETCH_TASK` to a list of error messages. For " - "example:\n" - "```\n" - 'ray.data._internal.datasource.parquet_datasource.RETRY_EXCEPTIONS_FOR_META_FETCH_TASK = ["AWS Error ACCESS_DENIED", "Timeout"]\n' # noqa: E501 - "```" - ) from e - return metadata - - -def _fetch_metadata( - fragments: List["pyarrow.dataset.ParquetFileFragment"], -) -> List["ParquetFileMetadata"]: - fragment_metadatas = [] - for f in fragments: - try: - # Convert directly to _ParquetFileFragmentMetaData - fragment_metadatas.append(ParquetFileMetadata.from_(f.metadata)) - except AttributeError as ae: - logger.warning(f"Failed to extract metadata from parquet file: {ae}") - break - # Deduplicate schemas to reduce memory usage - return fragment_metadatas - - -def _get_total_bytes(pqm: "pyarrow.parquet.FileMetaData") -> int: - return sum(pqm.row_group(i).total_byte_size for i in range(pqm.num_row_groups)) diff --git a/python/ray/data/read_api.py b/python/ray/data/read_api.py index 80ae80d4ff04..5ead3a4eead5 100644 --- a/python/ray/data/read_api.py +++ b/python/ray/data/read_api.py @@ -95,8 +95,8 @@ from ray.data.datasource.file_meta_provider import ( DefaultFileMetadataProvider, FastFileMetadataProvider, + FileMetadataProvider, ) -from ray.data.datasource.parquet_meta_provider import ParquetMetadataProvider from ray.data.datasource.partitioning import Partitioning from ray.types import ObjectRef from ray.util.annotations import Deprecated, DeveloperAPI, PublicAPI @@ -801,7 +801,7 @@ def read_parquet( parallelism: int = -1, ray_remote_args: Dict[str, Any] = None, tensor_column_schema: Optional[Dict[str, Tuple[np.dtype, Tuple[int, ...]]]] = None, - meta_provider: Optional[ParquetMetadataProvider] = None, + meta_provider: Optional[FileMetadataProvider] = None, partition_filter: Optional[PathPartitionFilter] = None, partitioning: Optional[Partitioning] = Partitioning("hive"), shuffle: Optional[Union[Literal["files"], FileShuffleConfig]] = None, @@ -937,8 +937,6 @@ def read_parquet( _emit_meta_provider_deprecation_warning(meta_provider) _validate_shuffle_arg(shuffle) - if meta_provider is None: - meta_provider = ParquetMetadataProvider() arrow_parquet_args = _resolve_parquet_args( tensor_column_schema, **arrow_parquet_args, @@ -3886,7 +3884,7 @@ def read_delta( columns: Optional[List[str]] = None, parallelism: int = -1, ray_remote_args: Optional[Dict[str, Any]] = None, - meta_provider: Optional[ParquetMetadataProvider] = None, + meta_provider: Optional[FileMetadataProvider] = None, partition_filter: Optional[PathPartitionFilter] = None, partitioning: Optional[Partitioning] = Partitioning("hive"), shuffle: Union[Literal["files"], None] = None, diff --git a/python/ray/data/tests/test_metadata_provider.py b/python/ray/data/tests/test_metadata_provider.py index c67e0890d506..b8d49544c9da 100644 --- a/python/ray/data/tests/test_metadata_provider.py +++ b/python/ray/data/tests/test_metadata_provider.py @@ -6,8 +6,6 @@ from unittest.mock import patch import pandas as pd -import pyarrow as pa -import pyarrow.parquet as pq import pytest from pyarrow.fs import LocalFileSystem from pytest_lazy_fixtures import lf as lazy_fixture @@ -17,7 +15,6 @@ DefaultFileMetadataProvider, FastFileMetadataProvider, FileMetadataProvider, - ParquetMetadataProvider, ) from ray.data.datasource.file_based_datasource import ( FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD, @@ -27,7 +24,6 @@ _get_file_infos_parallel, _get_file_infos_serial, ) -from ray.data.datasource.parquet_meta_provider import _get_total_bytes from ray.data.datasource.path_util import ( _resolve_paths_and_filesystem, _unwrap_protocol, @@ -65,57 +61,6 @@ def test_file_metadata_providers_not_implemented(): meta_provider.expand_paths(["/foo/bar.csv"], None) -@pytest.mark.parametrize( - "fs,data_path", - [ - (None, lazy_fixture("local_path")), - (lazy_fixture("local_fs"), lazy_fixture("local_path")), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path")), - ( - lazy_fixture("s3_fs_with_space"), - lazy_fixture("s3_path_with_space"), - ), # Path contains space. - ( - lazy_fixture("s3_fs_with_special_chars"), - lazy_fixture("s3_path_with_special_chars"), - ), - ], -) -def test_default_parquet_metadata_provider(fs, data_path): - path_module = os.path if urllib.parse.urlparse(data_path).scheme else posixpath - paths = [ - path_module.join(data_path, "test1.parquet"), - path_module.join(data_path, "test2.parquet"), - ] - paths, fs = _resolve_paths_and_filesystem(paths, fs) - - df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - table = pa.Table.from_pandas(df1) - pq.write_table(table, paths[0], filesystem=fs) - df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) - table = pa.Table.from_pandas(df2) - pq.write_table(table, paths[1], filesystem=fs) - - meta_provider = ParquetMetadataProvider() - pq_ds = pq.ParquetDataset(paths, filesystem=fs) - fragment_file_metas = meta_provider.prefetch_file_metadata(pq_ds.fragments) - - meta = meta_provider( - [p.path for p in pq_ds.fragments], - num_fragments=len(pq_ds.fragments), - prefetched_metadata=fragment_file_metas, - ) - - expected_meta_size_bytes = sum( - [_get_total_bytes(f.metadata) for f in pq_ds.fragments] - ) - - assert meta.size_bytes == expected_meta_size_bytes - assert meta.num_rows == 6 - assert len(paths) == 2 - assert all(path in meta.input_files for path in paths) - - @pytest.mark.parametrize( "fs,data_path,endpoint_url", [ diff --git a/python/ray/data/tests/test_parquet.py b/python/ray/data/tests/test_parquet.py index 67a517cfc002..08471a7653c3 100644 --- a/python/ray/data/tests/test_parquet.py +++ b/python/ray/data/tests/test_parquet.py @@ -21,17 +21,15 @@ from ray.data import FileShuffleConfig, Schema from ray.data._internal.datasource.parquet_bulk_datasource import ParquetBulkDatasource from ray.data._internal.datasource.parquet_datasource import ( - NUM_CPUS_FOR_META_FETCH_TASK, ParquetDatasource, ) from ray.data._internal.execution.interfaces.ref_bundle import ( _ref_bundles_iterator_to_block_refs_list, ) from ray.data._internal.util import rows_same -from ray.data.block import BlockAccessor, BlockMetadata +from ray.data.block import BlockAccessor from ray.data.context import DataContext -from ray.data.datasource import DefaultFileMetadataProvider, ParquetMetadataProvider -from ray.data.datasource.parquet_meta_provider import PARALLELIZE_META_FETCH_THRESHOLD +from ray.data.datasource import DefaultFileMetadataProvider from ray.data.datasource.partitioning import Partitioning, PathPartitionFilter from ray.data.datasource.path_util import _unwrap_protocol from ray.data.tests.conftest import * # noqa @@ -177,143 +175,6 @@ def test_parquet_read_basic( assert sorted(values) == [1, 2, 3, 4, 5, 6] -@pytest.mark.parametrize( - "fs,data_path", - [ - (None, lazy_fixture("local_path")), - (lazy_fixture("local_fs"), lazy_fixture("local_path")), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path")), - ( - lazy_fixture("s3_fs_with_anonymous_crendential"), - lazy_fixture("s3_path_with_anonymous_crendential"), - ), - ], -) -def test_parquet_read_meta_provider(ray_start_regular_shared, fs, data_path): - df1 = pd.DataFrame({"one": range(30_000), "two": ["a", "b", "c"] * 10_000}) - table = pa.Table.from_pandas(df1) - setup_data_path = _unwrap_protocol(data_path) - path1 = os.path.join(setup_data_path, "test1.parquet") - pq.write_table(table, path1, filesystem=fs) - df2 = pd.DataFrame({"one": range(30_000, 60_000), "two": ["e", "f", "g"] * 10000}) - table = pa.Table.from_pandas(df2) - path2 = os.path.join(setup_data_path, "test2.parquet") - pq.write_table(table, path2, filesystem=fs) - - expected_num_rows = len(df1) + len(df2) - # NOTE: Since we're testing against various Pyarrow versions size - # on disk could be varying slightly as it on top of data it also - # includes metadata - expected_byte_size = pytest.approx(463500, abs=500) - - # - # Case 1: Test metadata fetching happy path (obtaining, caching and propagating - # metadata) - # - - class AssertingMetadataProvider(ParquetMetadataProvider): - def prefetch_file_metadata(self, fragments, **ray_remote_args): - assert ray_remote_args["num_cpus"] == NUM_CPUS_FOR_META_FETCH_TASK - assert ( - ray_remote_args["scheduling_strategy"] - == DataContext.get_current().scheduling_strategy - ) - return super().prefetch_file_metadata(fragments, **ray_remote_args) - - ds = ray.data.read_parquet( - data_path, - filesystem=fs, - meta_provider=AssertingMetadataProvider(), - ) - - # Expect precomputed row counts and block sizes to be missing. - assert ds._meta_count() is None - - read_op = ds._plan._logical_plan.dag - - # Assert Read op metadata propagation - metadata = read_op.infer_metadata() - # NOTE: We assert on byte size separately, since we're using `pytest.approx` - # object for it - assert metadata.size_bytes == expected_byte_size - - assert metadata == BlockMetadata( - num_rows=None, - size_bytes=metadata.size_bytes, - exec_stats=None, - input_files=[path1, path2], - ) - - expected_schema = pa.schema({"one": pa.int64(), "two": pa.string()}) - - assert read_op.infer_schema().equals(expected_schema) - - # Expected - # - Fetched Parquet metadata to be reused - # - *No* dataset execution performed - assert ds.count() == expected_num_rows - assert ds.size_bytes() == expected_byte_size - assert ds.schema() == Schema(expected_schema) - assert set(ds.input_files()) == {path1, path2} - - assert not ds._plan.has_computed_output() - - expected_values = list( - zip(range(60_000), ["a", "b", "c"] * 10_000 + ["e", "f", "g"] * 10_000) - ) - - values = [(s["one"], s["two"]) for s in ds.take(60000)] - - exec_stats = ds._plan._snapshot_stats - read_stats = exec_stats.parents[0] - - # Assert that ref-bundles - # - Passed to ReadParquet hold metadata matching actual bundle - # - Produced by ReadParquet reflects actual amount of bytes read - assert read_stats.base_name == "ReadParquet" - # NOTE: Size of the task should be ~5kb, but could vary from platform to platform - # alas for different Python versions. However, it is substantially smaller - # than the dataset itself (~750kb) - assert read_stats.extra_metrics["average_bytes_inputs_per_task"] < 10_000 - - # TODO stats are broken for iteration-based executions due to the fact - # that returned stats object is obtained before iteration completes, - # hence not capturing the final state of the pipeline - # assert ( - # read_stats.extra_metrics["bytes_task_outputs_generated"] == expected_byte_size - # ) - - assert sorted(values) == expected_values - - # - # Case 2: Test metadata fetching *failing* (falling back to actually - # executing the dataset) - # - - class FailingMetadataProvider(ParquetMetadataProvider): - def prefetch_file_metadata(self, fragments, **ray_remote_args): - assert ray_remote_args["num_cpus"] == NUM_CPUS_FOR_META_FETCH_TASK - assert ( - ray_remote_args["scheduling_strategy"] - == DataContext.get_current().scheduling_strategy - ) - return None - - ds = ray.data.read_parquet( - data_path, - filesystem=fs, - meta_provider=FailingMetadataProvider(), - ) - - # Expected - # - Fetched Parquet metadata is not used (returns null), hence - # - Dataset execution has to be performed - assert ds.count() == expected_num_rows - assert ds.size_bytes() == expected_byte_size - assert ds.schema() == Schema(expected_schema) - assert set(ds.input_files()) == {path1, path2} - - @pytest.mark.parametrize( "fs,data_path", [ @@ -790,49 +651,6 @@ def _block_udf(block: pa.Table): np.testing.assert_array_equal(sorted(ones), np.array(one_data[:2]) + 1) -@pytest.mark.parametrize( - "fs,data_path", - [ - (None, lazy_fixture("local_path")), - (lazy_fixture("local_fs"), lazy_fixture("local_path")), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path")), - (lazy_fixture("s3_fs_with_space"), lazy_fixture("s3_path_with_space")), - ( - lazy_fixture("s3_fs_with_anonymous_crendential"), - lazy_fixture("s3_path_with_anonymous_crendential"), - ), - ], -) -def test_parquet_read_parallel_meta_fetch( - ray_start_regular_shared, fs, data_path, target_max_block_size_infinite_or_default -): - setup_data_path = _unwrap_protocol(data_path) - num_dfs = PARALLELIZE_META_FETCH_THRESHOLD + 1 - for idx in range(num_dfs): - df = pd.DataFrame({"one": list(range(3 * idx, 3 * (idx + 1)))}) - table = pa.Table.from_pandas(df) - path = os.path.join(setup_data_path, f"test_{idx}.parquet") - pq.write_table(table, path, filesystem=fs) - - parallelism = 8 - ds = ray.data.read_parquet( - data_path, filesystem=fs, override_num_blocks=parallelism - ) - - # Test metadata-only parquet ops. - assert ds.count() == num_dfs * 3 - assert ds.size_bytes() > 0 - # Schema information and input files are available from Parquet metadata, - # so we do not need to compute the first block. - assert ds.schema() is not None - input_files = ds.input_files() - assert len(input_files) == num_dfs, input_files - - # Forces a data read. - values = [s["one"] for s in ds.take(limit=3 * num_dfs)] - assert sorted(values) == list(range(3 * num_dfs)) - - def test_parquet_reader_estimate_data_size(shutdown_only, tmp_path): ctx = ray.data.context.DataContext.get_current() old_decoding_size_estimation = ctx.decoding_size_estimation @@ -844,9 +662,7 @@ def test_parquet_reader_estimate_data_size(shutdown_only, tmp_path): ray.data.range_tensor( 1000, shape=(1000,), override_num_blocks=10 ).write_parquet(tensor_output_path) - ds = ray.data.read_parquet( - tensor_output_path, meta_provider=ParquetMetadataProvider() - ) + ds = ray.data.read_parquet(tensor_output_path) assert ds._plan.initial_num_blocks() > 1 data_size = ds.size_bytes() assert ( @@ -857,9 +673,7 @@ def test_parquet_reader_estimate_data_size(shutdown_only, tmp_path): data_size >= 7_000_000 and data_size <= 10_000_000 ), "actual data size is out of expected bound" - datasource = ParquetDatasource( - tensor_output_path, meta_provider=ParquetMetadataProvider() - ) + datasource = ParquetDatasource(tensor_output_path) assert ( datasource._encoding_ratio >= 300 and datasource._encoding_ratio <= 600 ), "encoding ratio is out of expected bound" @@ -869,43 +683,35 @@ def test_parquet_reader_estimate_data_size(shutdown_only, tmp_path): ), "estimated data size is either out of expected bound" assert ( data_size - == ParquetDatasource( - tensor_output_path, meta_provider=ParquetMetadataProvider() - ).estimate_inmemory_data_size() + == ParquetDatasource(tensor_output_path).estimate_inmemory_data_size() ), "estimated data size is not deterministic in multiple calls." text_output_path = os.path.join(tmp_path, "text") ray.data.range(1000).map(lambda _: {"text": "a" * 1000}).write_parquet( text_output_path ) - ds = ray.data.read_parquet( - text_output_path, meta_provider=ParquetMetadataProvider() - ) + ds = ray.data.read_parquet(text_output_path) assert ds._plan.initial_num_blocks() > 1 data_size = ds.size_bytes() assert ( - data_size >= 800_000 and data_size <= 2_200_000 + data_size >= 700_000 and data_size <= 2_200_000 ), "estimated data size is out of expected bound" data_size = ds.materialize().size_bytes() assert ( data_size >= 1_000_000 and data_size <= 2_000_000 ), "actual data size is out of expected bound" - datasource = ParquetDatasource( - text_output_path, meta_provider=ParquetMetadataProvider() - ) + datasource = ParquetDatasource(text_output_path) assert ( - datasource._encoding_ratio >= 9 and datasource._encoding_ratio <= 300 + datasource._encoding_ratio >= 6 and datasource._encoding_ratio <= 300 ), "encoding ratio is out of expected bound" data_size = datasource.estimate_inmemory_data_size() assert ( - data_size >= 800_000 and data_size <= 2_200_000 + data_size >= 700_000 and data_size <= 2_200_000 ), "estimated data size is out of expected bound" assert ( data_size - == ParquetDatasource( - text_output_path, meta_provider=ParquetMetadataProvider() - ).estimate_inmemory_data_size() + == ParquetDatasource(text_output_path).estimate_inmemory_data_size() ), "estimated data size is not deterministic in multiple calls." finally: ctx.decoding_size_estimation = old_decoding_size_estimation diff --git a/python/ray/data/tests/test_size_estimation.py b/python/ray/data/tests/test_size_estimation.py index 7615b1a3beea..d23610b86261 100644 --- a/python/ray/data/tests/test_size_estimation.py +++ b/python/ray/data/tests/test_size_estimation.py @@ -146,7 +146,7 @@ def gen(name): nrow = ds2._block_num_rows() assert 2 < len(nrow) < 5, nrow for x in nrow[:-1]: - assert 50000 < x < 95000, (x, nrow) + assert 50000 < x < 96000, (x, nrow) # 1MiB ctx.target_max_block_size = 1_000_000 From 9b682416519acb58234e661170d8137b0c9f19ce Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Fri, 5 Sep 2025 23:11:58 -0700 Subject: [PATCH 489/634] [lint] change API annotations and policy check triggers (#56289) so that they are limited ot python related changes, rather than all changes. Signed-off-by: Lonnie Liu --- .buildkite/lint.rayci.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.buildkite/lint.rayci.yml b/.buildkite/lint.rayci.yml index f45c826374c4..9762ce7a8261 100644 --- a/.buildkite/lint.rayci.yml +++ b/.buildkite/lint.rayci.yml @@ -35,11 +35,21 @@ steps: commands: - ./ci/lint/lint.sh pre_commit_pydoclint - - label: ":lint-roller: lint: {{matrix}}" + - label: ":lint-roller: python API: {{matrix}}" tags: - oss - - lint - - always + - python + - dashboard + - ray_client + - data + - serve + - ml + - tune + - train + - llm + - rllib + - rllib_gpu + - doc key: lint-medium instance_type: medium depends_on: docbuild From 7095933406df35e8c7eec07df36d0ba6343f1623 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Fri, 5 Sep 2025 23:12:52 -0700 Subject: [PATCH 490/634] [deps] upgrade boto3 (#56291) upgrading boto3 --------- Signed-off-by: elliot-barn --- python/deplocks/llm/ray_py311_cpu.lock | 114 +++++++------ python/deplocks/llm/ray_py311_cu121.lock | 114 +++++++------ python/deplocks/llm/ray_py311_cu128.lock | 114 +++++++------ python/deplocks/llm/ray_test_py311_cpu.lock | 126 +++++++-------- python/deplocks/llm/ray_test_py311_cu121.lock | 126 +++++++-------- python/deplocks/llm/ray_test_py311_cu128.lock | 126 +++++++-------- python/deplocks/llm/rayllm_py311_cpu.lock | 114 +++++++------ python/deplocks/llm/rayllm_py311_cu121.lock | 114 +++++++------ python/deplocks/llm/rayllm_py311_cu128.lock | 114 +++++++------ .../deplocks/llm/rayllm_test_py311_cpu.lock | 126 +++++++-------- .../deplocks/llm/rayllm_test_py311_cu121.lock | 126 +++++++-------- .../deplocks/llm/rayllm_test_py311_cu128.lock | 126 +++++++-------- python/requirements/cloud-requirements.txt | 4 +- python/requirements/ml/core-requirements.txt | 2 +- python/requirements/ml/data-requirements.txt | 2 +- python/requirements/test-requirements.txt | 2 +- python/requirements_compiled.txt | 14 +- .../ray_release/byod/requirements_byod_3.9.in | 2 +- .../byod/requirements_byod_3.9.txt | 151 +++++++++--------- .../byod/requirements_ml_byod_3.9.in | 2 +- .../byod/requirements_ml_byod_3.9.txt | 24 +-- 21 files changed, 796 insertions(+), 847 deletions(-) diff --git a/python/deplocks/llm/ray_py311_cpu.lock b/python/deplocks/llm/ray_py311_cpu.lock index 8cd0117266c0..943eb4050549 100644 --- a/python/deplocks/llm/ray_py311_cpu.lock +++ b/python/deplocks/llm/ray_py311_cpu.lock @@ -649,9 +649,9 @@ frozenlist==1.4.1 \ # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt @@ -673,62 +673,58 @@ googleapis-common-protos==1.61.0 \ # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt diff --git a/python/deplocks/llm/ray_py311_cu121.lock b/python/deplocks/llm/ray_py311_cu121.lock index cdddd9e0dff0..e1d9eee4a8ef 100644 --- a/python/deplocks/llm/ray_py311_cu121.lock +++ b/python/deplocks/llm/ray_py311_cu121.lock @@ -649,9 +649,9 @@ frozenlist==1.4.1 \ # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt @@ -673,62 +673,58 @@ googleapis-common-protos==1.61.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt diff --git a/python/deplocks/llm/ray_py311_cu128.lock b/python/deplocks/llm/ray_py311_cu128.lock index 5d6886ea1bf7..a685eb63910c 100644 --- a/python/deplocks/llm/ray_py311_cu128.lock +++ b/python/deplocks/llm/ray_py311_cu128.lock @@ -649,9 +649,9 @@ frozenlist==1.4.1 \ # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt @@ -673,62 +673,58 @@ googleapis-common-protos==1.61.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt diff --git a/python/deplocks/llm/ray_test_py311_cpu.lock b/python/deplocks/llm/ray_test_py311_cpu.lock index a8b64f17e94a..c142079eff8f 100644 --- a/python/deplocks/llm/ray_test_py311_cpu.lock +++ b/python/deplocks/llm/ray_test_py311_cpu.lock @@ -272,16 +272,16 @@ bleach==6.1.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # nbconvert -boto3==1.26.76 \ - --hash=sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027 \ - --hash=sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729 +boto3==1.28.17 \ + --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ + --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.29.76 \ - --hash=sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7 \ - --hash=sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7 +botocore==1.31.17 \ + --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ + --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt @@ -871,9 +871,9 @@ frozenlist==1.4.1 \ # -c /tmp/ray-deps/requirements_compiled.txt # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt @@ -1005,62 +1005,58 @@ googleapis-common-protos==1.61.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt diff --git a/python/deplocks/llm/ray_test_py311_cu121.lock b/python/deplocks/llm/ray_test_py311_cu121.lock index cf09d162e9a3..87a35a550a95 100644 --- a/python/deplocks/llm/ray_test_py311_cu121.lock +++ b/python/deplocks/llm/ray_test_py311_cu121.lock @@ -272,16 +272,16 @@ bleach==6.1.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # nbconvert -boto3==1.26.76 \ - --hash=sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027 \ - --hash=sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729 +boto3==1.28.17 \ + --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ + --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.29.76 \ - --hash=sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7 \ - --hash=sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7 +botocore==1.31.17 \ + --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ + --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt @@ -871,9 +871,9 @@ frozenlist==1.4.1 \ # -c /tmp/ray-deps/requirements_compiled.txt # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt @@ -1005,62 +1005,58 @@ googleapis-common-protos==1.61.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt diff --git a/python/deplocks/llm/ray_test_py311_cu128.lock b/python/deplocks/llm/ray_test_py311_cu128.lock index f514c628272b..ec467bbc1f61 100644 --- a/python/deplocks/llm/ray_test_py311_cu128.lock +++ b/python/deplocks/llm/ray_test_py311_cu128.lock @@ -272,16 +272,16 @@ bleach==6.1.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # nbconvert -boto3==1.26.76 \ - --hash=sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027 \ - --hash=sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729 +boto3==1.28.17 \ + --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ + --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.29.76 \ - --hash=sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7 \ - --hash=sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7 +botocore==1.31.17 \ + --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ + --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt @@ -871,9 +871,9 @@ frozenlist==1.4.1 \ # -c /tmp/ray-deps/requirements_compiled.txt # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements.txt @@ -1005,62 +1005,58 @@ googleapis-common-protos==1.61.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt diff --git a/python/deplocks/llm/rayllm_py311_cpu.lock b/python/deplocks/llm/rayllm_py311_cpu.lock index a11f2db1e997..89da379ee9c5 100644 --- a/python/deplocks/llm/rayllm_py311_cpu.lock +++ b/python/deplocks/llm/rayllm_py311_cpu.lock @@ -862,9 +862,9 @@ frozenlist==1.4.1 \ # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt @@ -894,62 +894,58 @@ googleapis-common-protos==1.61.0 \ # via # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c python/deplocks/llm/rayllm_test_py311_cpu.lock # -r python/requirements.txt diff --git a/python/deplocks/llm/rayllm_py311_cu121.lock b/python/deplocks/llm/rayllm_py311_cu121.lock index 76431a88ef1b..7a291b4f9eac 100644 --- a/python/deplocks/llm/rayllm_py311_cu121.lock +++ b/python/deplocks/llm/rayllm_py311_cu121.lock @@ -862,9 +862,9 @@ frozenlist==1.4.1 \ # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt @@ -894,62 +894,58 @@ googleapis-common-protos==1.61.0 \ # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c python/deplocks/llm/rayllm_test_py311_cu121.lock # -r python/requirements.txt diff --git a/python/deplocks/llm/rayllm_py311_cu128.lock b/python/deplocks/llm/rayllm_py311_cu128.lock index c0e933c02c02..41dcc9dce7dc 100644 --- a/python/deplocks/llm/rayllm_py311_cu128.lock +++ b/python/deplocks/llm/rayllm_py311_cu128.lock @@ -862,9 +862,9 @@ frozenlist==1.4.1 \ # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt @@ -894,62 +894,58 @@ googleapis-common-protos==1.61.0 \ # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c python/deplocks/llm/rayllm_test_py311_cu128.lock # -r python/requirements.txt diff --git a/python/deplocks/llm/rayllm_test_py311_cpu.lock b/python/deplocks/llm/rayllm_test_py311_cpu.lock index dc1ea6c32e2e..a64a3ea650a1 100644 --- a/python/deplocks/llm/rayllm_test_py311_cpu.lock +++ b/python/deplocks/llm/rayllm_test_py311_cpu.lock @@ -376,16 +376,16 @@ bleach==6.1.0 \ # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbconvert -boto3==1.26.76 \ - --hash=sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027 \ - --hash=sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729 +boto3==1.28.17 \ + --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ + --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.29.76 \ - --hash=sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7 \ - --hash=sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7 +botocore==1.31.17 \ + --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ + --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt @@ -1075,9 +1075,9 @@ frozenlist==1.4.1 \ # -c python/deplocks/llm/ray_test_py311_cpu.lock # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements.txt @@ -1215,62 +1215,58 @@ googleapis-common-protos==1.61.0 \ # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt diff --git a/python/deplocks/llm/rayllm_test_py311_cu121.lock b/python/deplocks/llm/rayllm_test_py311_cu121.lock index eabbdaefa580..0f94bbb0514d 100644 --- a/python/deplocks/llm/rayllm_test_py311_cu121.lock +++ b/python/deplocks/llm/rayllm_test_py311_cu121.lock @@ -376,16 +376,16 @@ bleach==6.1.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbconvert -boto3==1.26.76 \ - --hash=sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027 \ - --hash=sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729 +boto3==1.28.17 \ + --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ + --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.29.76 \ - --hash=sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7 \ - --hash=sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7 +botocore==1.31.17 \ + --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ + --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt @@ -1075,9 +1075,9 @@ frozenlist==1.4.1 \ # -c python/deplocks/llm/ray_test_py311_cu121.lock # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements.txt @@ -1215,62 +1215,58 @@ googleapis-common-protos==1.61.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt diff --git a/python/deplocks/llm/rayllm_test_py311_cu128.lock b/python/deplocks/llm/rayllm_test_py311_cu128.lock index 713e00a75f78..e4a81a7eab1d 100644 --- a/python/deplocks/llm/rayllm_test_py311_cu128.lock +++ b/python/deplocks/llm/rayllm_test_py311_cu128.lock @@ -376,16 +376,16 @@ bleach==6.1.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbconvert -boto3==1.26.76 \ - --hash=sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027 \ - --hash=sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729 +boto3==1.28.17 \ + --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ + --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.29.76 \ - --hash=sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7 \ - --hash=sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7 +botocore==1.31.17 \ + --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ + --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt @@ -1074,9 +1074,9 @@ frozenlist==1.4.1 \ # -c python/deplocks/llm/ray_test_py311_cu128.lock # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements.txt @@ -1214,62 +1214,58 @@ googleapis-common-protos==1.61.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-api-core -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt diff --git a/python/requirements/cloud-requirements.txt b/python/requirements/cloud-requirements.txt index 6f8677797288..faccc9caa716 100644 --- a/python/requirements/cloud-requirements.txt +++ b/python/requirements/cloud-requirements.txt @@ -12,8 +12,8 @@ smart_open[s3,gcs,azure,http] adlfs[abfs] # Anyscale CLI requirements -boto3>=1.26.76 -botocore>=1.19.52 +boto3==1.28.17 +botocore>=1.31.17,<1.32.0 aiohttp>=3.7.4.post0 certifi>=2024.8.30 Click>=7.0 diff --git a/python/requirements/ml/core-requirements.txt b/python/requirements/ml/core-requirements.txt index 7f0b2caed3b0..2d6948ccef11 100644 --- a/python/requirements/ml/core-requirements.txt +++ b/python/requirements/ml/core-requirements.txt @@ -12,4 +12,4 @@ transformers==4.36.2 accelerate==0.28.0 # Cloud storage tools -s3fs==2023.5.0 +s3fs==2023.12.1 diff --git a/python/requirements/ml/data-requirements.txt b/python/requirements/ml/data-requirements.txt index 14ec80e7d00c..94f628a17cf9 100644 --- a/python/requirements/ml/data-requirements.txt +++ b/python/requirements/ml/data-requirements.txt @@ -6,7 +6,7 @@ dask[complete]==2023.6.1; python_version < '3.12' distributed==2023.6.1; python_version < '3.12' dask[complete]==2025.5.0; python_version >= '3.12' distributed==2025.5.0; python_version >= '3.12' -aioboto3==11.2.0 +aioboto3==11.3.0 crc32c==2.3 flask_cors bokeh==2.4.3; python_version < '3.12' diff --git a/python/requirements/test-requirements.txt b/python/requirements/test-requirements.txt index 480b1eab6eb4..b59f858e70a1 100644 --- a/python/requirements/test-requirements.txt +++ b/python/requirements/test-requirements.txt @@ -10,7 +10,7 @@ azure-mgmt-network==25.4.0 azure-mgmt-resource==23.1.1 msrestazure==0.6.4 beautifulsoup4==4.11.1 -boto3==1.26.76 +boto3==1.28.17 # Todo: investigate if we can get rid of this and exchange for ray.cloudpickle cloudpickle==2.2.0 ; python_version < "3.12" cloudpickle==3.0.0 ; python_version >= "3.12" diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt index 466bf856d272..ad16f111c782 100644 --- a/python/requirements_compiled.txt +++ b/python/requirements_compiled.txt @@ -34,9 +34,9 @@ aimrecords==0.0.7 # via aim aimrocks==0.5.2 # via aim -aioboto3==11.2.0 +aioboto3==11.3.0 # via -r python/requirements/ml/data-requirements.txt -aiobotocore==2.5.0 +aiobotocore==2.6.0 # via # aioboto3 # s3fs @@ -231,7 +231,7 @@ boltons==21.0.0 # semgrep boto==2.49.0 # via gcs-oauth2-boto-plugin -boto3==1.26.76 +boto3==1.28.17 # via # -r python/requirements/cloud-requirements.txt # -r python/requirements/test-requirements.txt @@ -241,7 +241,7 @@ boto3==1.26.76 # moto # smart-open # snowflake-connector-python -botocore==1.29.76 +botocore==1.31.17 # via # -r python/requirements/cloud-requirements.txt # aiobotocore @@ -573,7 +573,7 @@ frozenlist==1.4.1 # aiosignal fs==2.4.16 # via triad -fsspec==2023.5.0 +fsspec==2023.12.1 # via # -r python/requirements.txt # adlfs @@ -703,7 +703,7 @@ graphviz==0.20.3 # via -r python/requirements/test-requirements.txt greenlet==3.0.1 # via sqlalchemy -grpcio==1.66.2 ; python_version >= "3.10" +grpcio==1.74.0 ; python_version >= "3.10" # via # -r python/requirements.txt # -r python/requirements/cloud-requirements.txt @@ -1965,7 +1965,7 @@ ruamel-yaml==0.17.40 # yahp ruamel-yaml-clib==0.2.8 # via ruamel-yaml -s3fs==2023.5.0 +s3fs==2023.12.1 # via -r python/requirements/ml/core-requirements.txt s3transfer==0.6.2 # via boto3 diff --git a/release/ray_release/byod/requirements_byod_3.9.in b/release/ray_release/byod/requirements_byod_3.9.in index be42c3f1f682..515cdcbb8d61 100644 --- a/release/ray_release/byod/requirements_byod_3.9.in +++ b/release/ray_release/byod/requirements_byod_3.9.in @@ -7,7 +7,7 @@ cmake crc32c cython fastapi -gcsfs==2023.5.0 +gcsfs==2023.12.1 gsutil gymnasium gymnasium[atari] diff --git a/release/ray_release/byod/requirements_byod_3.9.txt b/release/ray_release/byod/requirements_byod_3.9.txt index 982e72c18612..1c291b2c3970 100644 --- a/release/ray_release/byod/requirements_byod_3.9.txt +++ b/release/ray_release/byod/requirements_byod_3.9.txt @@ -14,9 +14,9 @@ absl-py==1.4.0 \ # -c release/ray_release/byod/requirements_compiled.txt # tensorboard # tensorflow -aiobotocore==2.5.0 \ - --hash=sha256:6a5b397cddd4f81026aa91a14c7dd2650727425740a5af8ba75127ff663faf67 \ - --hash=sha256:9a2a022d7b78ec9a2af0de589916d2721cddbf96264401b78d7a73c1a1435f3b +aiobotocore==2.6.0 \ + --hash=sha256:0186e6a843364748cdbbf76ee98e9337c44f71a4e694ad1b110d5c516fbce909 \ + --hash=sha256:4805d0140bdfa17bfc2d0ba1243c8cc4273e927201fca5cf2e497c0004a9fab7 # via # -c release/ray_release/byod/requirements_compiled.txt # s3fs @@ -204,15 +204,15 @@ boto==2.49.0 \ # via # -c release/ray_release/byod/requirements_compiled.txt # gcs-oauth2-boto-plugin -boto3==1.26.76 \ - --hash=sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027 \ - --hash=sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729 +boto3==1.28.17 \ + --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ + --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_byod_3.9.in -botocore==1.29.76 \ - --hash=sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7 \ - --hash=sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7 +botocore==1.31.17 \ + --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ + --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b # via # -c release/ray_release/byod/requirements_compiled.txt # aiobotocore @@ -683,9 +683,9 @@ diskcache==5.6.3 \ --hash=sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc \ --hash=sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19 # via petastorm -exceptiongroup==1.2.1 \ - --hash=sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad \ - --hash=sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16 +exceptiongroup==1.3.0 \ + --hash=sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10 \ + --hash=sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88 # via # anyio # pytest @@ -814,9 +814,9 @@ frozenlist==1.4.1 \ # -c release/ray_release/byod/requirements_compiled.txt # aiohttp # aiosignal -fsspec==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c release/ray_release/byod/requirements_compiled.txt # gcsfs @@ -839,9 +839,9 @@ gcs-oauth2-boto-plugin==3.0 \ # via # -c release/ray_release/byod/requirements_compiled.txt # gsutil -gcsfs==2023.5.0 \ - --hash=sha256:02a815e1cf28197ab4f57335e89dc5df8744a065c7c956d42692b50a9e8f1625 \ - --hash=sha256:4f2ebc41814de3f566f85dec208704cf19823b9d04a55fd12b3142aef9046525 +gcsfs==2023.12.1 \ + --hash=sha256:c1ccfa9f84dca019cd334aaf7eb03cc1dc13c296717346927a9fd40255348f9c \ + --hash=sha256:e86cc583fdf879e5ea2f87bab61738d26ec7e8972762a1e6c6ab758b1e1af99c # via -r release/ray_release/byod/requirements_byod_3.9.in gevent==24.2.1 \ --hash=sha256:03aa5879acd6b7076f6a2a307410fb1e0d288b84b03cdfd8c74db8b4bc882fc5 \ @@ -1202,62 +1202,58 @@ greenlet==3.0.1 \ # via # -c release/ray_release/byod/requirements_compiled.txt # gevent -grpcio==1.66.2 \ - --hash=sha256:02697eb4a5cbe5a9639f57323b4c37bcb3ab2d48cec5da3dc2f13334d72790dd \ - --hash=sha256:03b0b307ba26fae695e067b94cbb014e27390f8bc5ac7a3a39b7723fed085604 \ - --hash=sha256:05bc2ceadc2529ab0b227b1310d249d95d9001cd106aa4d31e8871ad3c428d73 \ - --hash=sha256:06de8ec0bd71be123eec15b0e0d457474931c2c407869b6c349bd9bed4adbac3 \ - --hash=sha256:0be4e0490c28da5377283861bed2941d1d20ec017ca397a5df4394d1c31a9b50 \ - --hash=sha256:12fda97ffae55e6526825daf25ad0fa37483685952b5d0f910d6405c87e3adb6 \ - --hash=sha256:1caa38fb22a8578ab8393da99d4b8641e3a80abc8fd52646f1ecc92bcb8dee34 \ - --hash=sha256:2018b053aa15782db2541ca01a7edb56a0bf18c77efed975392583725974b249 \ - --hash=sha256:20657d6b8cfed7db5e11b62ff7dfe2e12064ea78e93f1434d61888834bc86d75 \ - --hash=sha256:2335c58560a9e92ac58ff2bc5649952f9b37d0735608242973c7a8b94a6437d8 \ - --hash=sha256:31fd163105464797a72d901a06472860845ac157389e10f12631025b3e4d0453 \ - --hash=sha256:38b68498ff579a3b1ee8f93a05eb48dc2595795f2f62716e797dc24774c1aaa8 \ - --hash=sha256:3b00efc473b20d8bf83e0e1ae661b98951ca56111feb9b9611df8efc4fe5d55d \ - --hash=sha256:3ed71e81782966ffead60268bbda31ea3f725ebf8aa73634d5dda44f2cf3fb9c \ - --hash=sha256:45a3d462826f4868b442a6b8fdbe8b87b45eb4f5b5308168c156b21eca43f61c \ - --hash=sha256:49f0ca7ae850f59f828a723a9064cadbed90f1ece179d375966546499b8a2c9c \ - --hash=sha256:4e504572433f4e72b12394977679161d495c4c9581ba34a88d843eaf0f2fbd39 \ - --hash=sha256:4ea1d062c9230278793820146c95d038dc0f468cbdd172eec3363e42ff1c7d01 \ - --hash=sha256:563588c587b75c34b928bc428548e5b00ea38c46972181a4d8b75ba7e3f24231 \ - --hash=sha256:6001e575b8bbd89eee11960bb640b6da6ae110cf08113a075f1e2051cc596cae \ - --hash=sha256:66a0cd8ba6512b401d7ed46bb03f4ee455839957f28b8d61e7708056a806ba6a \ - --hash=sha256:6851de821249340bdb100df5eacfecfc4e6075fa85c6df7ee0eb213170ec8e5d \ - --hash=sha256:728bdf36a186e7f51da73be7f8d09457a03061be848718d0edf000e709418987 \ - --hash=sha256:73e3b425c1e155730273f73e419de3074aa5c5e936771ee0e4af0814631fb30a \ - --hash=sha256:73fc8f8b9b5c4a03e802b3cd0c18b2b06b410d3c1dcbef989fdeb943bd44aff7 \ - --hash=sha256:78fa51ebc2d9242c0fc5db0feecc57a9943303b46664ad89921f5079e2e4ada7 \ - --hash=sha256:7b2c86457145ce14c38e5bf6bdc19ef88e66c5fee2c3d83285c5aef026ba93b3 \ - --hash=sha256:7d69ce1f324dc2d71e40c9261d3fdbe7d4c9d60f332069ff9b2a4d8a257c7b2b \ - --hash=sha256:802d84fd3d50614170649853d121baaaa305de7b65b3e01759247e768d691ddf \ - --hash=sha256:80fd702ba7e432994df208f27514280b4b5c6843e12a48759c9255679ad38db8 \ - --hash=sha256:8ac475e8da31484efa25abb774674d837b343afb78bb3bcdef10f81a93e3d6bf \ - --hash=sha256:950da58d7d80abd0ea68757769c9db0a95b31163e53e5bb60438d263f4bed7b7 \ - --hash=sha256:99a641995a6bc4287a6315989ee591ff58507aa1cbe4c2e70d88411c4dcc0839 \ - --hash=sha256:9c3a99c519f4638e700e9e3f83952e27e2ea10873eecd7935823dab0c1c9250e \ - --hash=sha256:9c509a4f78114cbc5f0740eb3d7a74985fd2eff022971bc9bc31f8bc93e66a3b \ - --hash=sha256:a18e20d8321c6400185b4263e27982488cb5cdd62da69147087a76a24ef4e7e3 \ - --hash=sha256:a917d26e0fe980b0ac7bfcc1a3c4ad6a9a4612c911d33efb55ed7833c749b0ee \ - --hash=sha256:a9539f01cb04950fd4b5ab458e64a15f84c2acc273670072abe49a3f29bbad54 \ - --hash=sha256:ad2efdbe90c73b0434cbe64ed372e12414ad03c06262279b104a029d1889d13e \ - --hash=sha256:b672abf90a964bfde2d0ecbce30f2329a47498ba75ce6f4da35a2f4532b7acbc \ - --hash=sha256:bbd27c24a4cc5e195a7f56cfd9312e366d5d61b86e36d46bbe538457ea6eb8dd \ - --hash=sha256:c400ba5675b67025c8a9f48aa846f12a39cf0c44df5cd060e23fda5b30e9359d \ - --hash=sha256:c408f5ef75cfffa113cacd8b0c0e3611cbfd47701ca3cdc090594109b9fcbaed \ - --hash=sha256:c806852deaedee9ce8280fe98955c9103f62912a5b2d5ee7e3eaa284a6d8d8e7 \ - --hash=sha256:ce89f5876662f146d4c1f695dda29d4433a5d01c8681fbd2539afff535da14d4 \ - --hash=sha256:d25a14af966438cddf498b2e338f88d1c9706f3493b1d73b93f695c99c5f0e2a \ - --hash=sha256:d8d4732cc5052e92cea2f78b233c2e2a52998ac40cd651f40e398893ad0d06ec \ - --hash=sha256:d9a9724a156c8ec6a379869b23ba3323b7ea3600851c91489b871e375f710bc8 \ - --hash=sha256:e636ce23273683b00410f1971d209bf3689238cf5538d960adc3cdfe80dd0dbd \ - --hash=sha256:e88264caad6d8d00e7913996030bac8ad5f26b7411495848cc218bd3a9040b6c \ - --hash=sha256:f145cc21836c332c67baa6fc81099d1d27e266401565bf481948010d6ea32d46 \ - --hash=sha256:fb57870449dfcfac428afbb5a877829fcb0d6db9d9baa1148705739e9083880e \ - --hash=sha256:fb70487c95786e345af5e854ffec8cb8cc781bcc5df7930c4fbb7feaa72e1cdf \ - --hash=sha256:fe96281713168a3270878255983d2cb1a97e034325c8c2c25169a69289d3ecfa \ - --hash=sha256:ff1f7882e56c40b0d33c4922c15dfa30612f05fb785074a012f7cda74d1c3679 +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e # via # tensorboard # tensorflow @@ -2637,9 +2633,9 @@ rsa==4.7.2 \ # gcs-oauth2-boto-plugin # google-auth # oauth2client -s3fs==2023.5.0 \ - --hash=sha256:0d82c4fa43d1214117f56b239c3e03c9a2886f41c31000c1c967ac6030d20362 \ - --hash=sha256:106b5d9a1000e6af413f918156ba4b96789ac832b7e08c99d186eb08164e6981 +s3fs==2023.12.1 \ + --hash=sha256:63e429bb6b5e814568cacd3f2a8551fc35493e8c418ddfcb44e6f86aa8696ccd \ + --hash=sha256:ed0b7df8cc20a2b5cefe607b1cf4e860d37c5ca4ac2d68f55464805d75d18710 # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_byod_3.9.in @@ -2915,6 +2911,7 @@ typing-extensions==4.12.2 \ # -r release/ray_release/byod/requirements_byod_3.9.in # aioitertools # ale-py + # exceptiongroup # fastapi # gymnasium # pydantic diff --git a/release/ray_release/byod/requirements_ml_byod_3.9.in b/release/ray_release/byod/requirements_ml_byod_3.9.in index 1a7ce561af71..6c373e1b1a2e 100644 --- a/release/ray_release/byod/requirements_ml_byod_3.9.in +++ b/release/ray_release/byod/requirements_ml_byod_3.9.in @@ -14,7 +14,7 @@ evaluate fairscale fastapi filelock -gcsfs==2023.5.0 +gcsfs==2023.12.1 gsutil ipywidgets jupytext diff --git a/release/ray_release/byod/requirements_ml_byod_3.9.txt b/release/ray_release/byod/requirements_ml_byod_3.9.txt index fe20bf45e379..687a325b9329 100644 --- a/release/ray_release/byod/requirements_ml_byod_3.9.txt +++ b/release/ray_release/byod/requirements_ml_byod_3.9.txt @@ -203,15 +203,15 @@ boto==2.49.0 \ # via # -c release/ray_release/byod/requirements_compiled.txt # gcs-oauth2-boto-plugin -boto3==1.26.76 \ - --hash=sha256:30c7d967ed1c6b5a05643e42cae9d4d36c3f1cb6782637ddc7007a104cfd9027 \ - --hash=sha256:b4c2969b7677762914394b8273cc1905dfe5b71f250741c1a575487ae357e729 +boto3==1.28.17 \ + --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ + --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_ml_byod_3.9.in -botocore==1.29.76 \ - --hash=sha256:70735b00cd529f152992231ca6757e458e5ec25db43767b3526e9a35b2f143b7 \ - --hash=sha256:c2f67b6b3f8acf2968eafca06526f07b9fb0d27bac4c68a635d51abb675134a7 +botocore==1.31.17 \ + --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ + --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b # via # -c release/ray_release/byod/requirements_compiled.txt # boto3 @@ -968,9 +968,9 @@ fs==2.4.16 \ # via # -c release/ray_release/byod/requirements_compiled.txt # triad -fsspec[http]==2023.5.0 \ - --hash=sha256:51a4ad01a5bb66fcc58036e288c0d53d3975a0df2a5dc59a93b59bade0391f2a \ - --hash=sha256:b3b56e00fb93ea321bc9e5d9cf6f8522a0198b20eb24e02774d329e9c6fb84ce +fsspec[http]==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 # via # -c release/ray_release/byod/requirements_compiled.txt # datasets @@ -1004,9 +1004,9 @@ gcs-oauth2-boto-plugin==3.0 \ # via # -c release/ray_release/byod/requirements_compiled.txt # gsutil -gcsfs==2023.5.0 \ - --hash=sha256:02a815e1cf28197ab4f57335e89dc5df8744a065c7c956d42692b50a9e8f1625 \ - --hash=sha256:4f2ebc41814de3f566f85dec208704cf19823b9d04a55fd12b3142aef9046525 +gcsfs==2023.12.1 \ + --hash=sha256:c1ccfa9f84dca019cd334aaf7eb03cc1dc13c296717346927a9fd40255348f9c \ + --hash=sha256:e86cc583fdf879e5ea2f87bab61738d26ec7e8972762a1e6c6ab758b1e1af99c # via -r release/ray_release/byod/requirements_ml_byod_3.9.in gevent==24.2.1 \ --hash=sha256:03aa5879acd6b7076f6a2a307410fb1e0d288b84b03cdfd8c74db8b4bc882fc5 \ From 1fa52d2e69bcee8ef3d18950faf991cd55fea6ed Mon Sep 17 00:00:00 2001 From: Ping Dai Date: Sun, 7 Sep 2025 04:13:56 +0800 Subject: [PATCH 491/634] [Data] Add total input/output row counts of Operator in the output of Dataset.stats() (#56040) --- python/ray/data/_internal/stats.py | 70 ++++++++++++++-- python/ray/data/tests/test_stats.py | 120 +++++++++++++++++++++++++++- 2 files changed, 180 insertions(+), 10 deletions(-) diff --git a/python/ray/data/_internal/stats.py b/python/ray/data/_internal/stats.py index cba592182e65..088acffc2e0f 100644 --- a/python/ray/data/_internal/stats.py +++ b/python/ray/data/_internal/stats.py @@ -1006,14 +1006,6 @@ def to_summary(self) -> "DatasetStatsSummary": object, which can be used to generate a summary string.""" operators_stats = [] is_sub_operator = len(self.metadata) > 1 - for name, stats in self.metadata.items(): - operators_stats.append( - OperatorStatsSummary.from_block_metadata( - name, - stats, - is_sub_operator=is_sub_operator, - ) - ) iter_stats = IterStatsSummary( self.iter_wait_s, @@ -1032,9 +1024,56 @@ def to_summary(self) -> "DatasetStatsSummary": self.iter_blocks_remote, self.iter_unknown_location, ) + stats_summary_parents = [] if self.parents is not None: stats_summary_parents = [p.to_summary() for p in self.parents] + + # Collect the sum of the final output row counts from all parent nodes + parent_total_output = 0 + for i, parent_summary in enumerate(stats_summary_parents): + if parent_summary.operators_stats: + # Get the last operator stats from the current parent summary + last_parent_op = parent_summary.operators_stats[-1] + # Extract output row count (handle dict type with "sum" key) + op_output = ( + last_parent_op.output_num_rows.get("sum", 0) + if isinstance(last_parent_op.output_num_rows, dict) + else 0 + ) + logger.debug( + f"Parent {i + 1} (operator: {last_parent_op.operator_name}) contributes {op_output} rows to input" + ) + parent_total_output += op_output + + # Create temporary operator stats objects from block metadata + op_stats = [ + OperatorStatsSummary.from_block_metadata( + name, stats, is_sub_operator=is_sub_operator + ) + for name, stats in self.metadata.items() + ] + + for i, op_stat in enumerate(op_stats): + # For sub-operators: inherit input based on the order in the current list + if is_sub_operator: + if i == 0: + # Input of the first sub-operator is the total output from parent nodes + op_stat.total_input_num_rows = parent_total_output + else: + # Input of subsequent sub-operators is the output of the previous sub-operator + prev_op = op_stats[i - 1] + op_stat.total_input_num_rows = ( + prev_op.output_num_rows["sum"] + if ( + prev_op.output_num_rows and "sum" in prev_op.output_num_rows + ) + else 0 + ) + else: + # Single operator scenario: input rows = total output from all parent nodes + op_stat.total_input_num_rows = parent_total_output + operators_stats.append(op_stat) streaming_exec_schedule_s = ( self.streaming_exec_schedule_s.get() if self.streaming_exec_schedule_s @@ -1336,6 +1375,8 @@ class OperatorStatsSummary: udf_time: Optional[Dict[str, float]] = None # memory: no "sum" stat memory: Optional[Dict[str, float]] = None + # Use the output_num_rows of the parent Operator as output_num_rows + total_input_num_rows: Optional[int] = None output_num_rows: Optional[Dict[str, float]] = None output_size_bytes: Optional[Dict[str, float]] = None # node_count: "count" stat instead of "sum" @@ -1470,6 +1511,9 @@ def from_block_metadata( "count": len(node_counts), } + # Assign a value in to_summary and initialize it as None. + total_input_num_rows = None + return OperatorStatsSummary( operator_name=operator_name, is_sub_operator=is_sub_operator, @@ -1481,6 +1525,7 @@ def from_block_metadata( cpu_time=cpu_stats, udf_time=udf_stats, memory=memory_stats, + total_input_num_rows=total_input_num_rows, output_num_rows=output_num_rows_stats, output_size_bytes=output_size_bytes_stats, node_count=node_counts_stats, @@ -1594,9 +1639,18 @@ def __str__(self) -> str: # total number of rows produced by the sum of the wall times across all # blocks of the operator. This assumes that on a single node the work done # would be equivalent, with no concurrency. + total_num_in_rows = ( + self.total_input_num_rows if self.total_input_num_rows else 0 + ) total_num_out_rows = output_num_rows_stats["sum"] out += indent out += "* Operator throughput:\n" + out += ( + indent + "\t* Total input num rows:" f" {total_num_in_rows} " "rows\n" + ) + out += ( + indent + "\t* Total output num rows:" f" {total_num_out_rows} " "rows\n" + ) out += ( indent + "\t* Ray Data throughput:" f" {total_num_out_rows / self.time_total_s} " diff --git a/python/ray/data/tests/test_stats.py b/python/ray/data/tests/test_stats.py index 71edcc8f94a4..4d7a5660a86c 100644 --- a/python/ray/data/tests/test_stats.py +++ b/python/ray/data/tests/test_stats.py @@ -296,6 +296,14 @@ def canonicalize(stats: str, filter_global_stats: bool = True) -> str: ) # Handle floats in (0, 1) canonicalized_stats = re.sub(r" (0\.0*[1-9][0-9]*)", " N", canonicalized_stats) + # Replace input rows value (0 or non-0) with 'N' while keeping key prefix + canonicalized_stats = re.sub( + r"(Total input num rows: )\d+(\.\d+)?", r"\g<1>N", canonicalized_stats + ) + # Replace output rows value (0 or non-0) with 'N' while keeping key prefix + canonicalized_stats = re.sub( + r"(Total output num rows: )\d+(\.\d+)?", r"\g<1>N", canonicalized_stats + ) # Handle zero values specially so we can check for missing values. canonicalized_stats = re.sub(r" [0]+(\.[0])?", " Z", canonicalized_stats) # Scientific notation for small or large numbers @@ -384,6 +392,8 @@ def test_streaming_split_stats(ray_start_regular_shared, restore_data_context): * Output rows per task: N min, N max, N mean, N tasks used * Tasks per node: N min, N max, N mean; N nodes used * Operator throughput: + * Total input num rows: N rows + * Total output num rows: N rows * Ray Data throughput: N rows/s * Estimated single node throughput: N rows/s * Extra metrics: {extra_metrics_1} @@ -448,6 +458,8 @@ def test_large_args_scheduling_strategy( f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"{read_extra_metrics}\n" @@ -461,6 +473,8 @@ def test_large_args_scheduling_strategy( f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"{map_extra_metrics}" @@ -505,6 +519,8 @@ def test_dataset_stats_basic( f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"{gen_extra_metrics_str(STANDARD_EXTRA_METRICS_TASK_BACKPRESSURE, verbose_stats_logs)}" # noqa: E501 @@ -530,6 +546,8 @@ def test_dataset_stats_basic( f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"{gen_extra_metrics_str(STANDARD_EXTRA_METRICS_TASK_BACKPRESSURE, verbose_stats_logs)}" # noqa: E501 @@ -560,6 +578,8 @@ def test_dataset_stats_basic( f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"{extra_metrics}\n" @@ -573,6 +593,8 @@ def test_dataset_stats_basic( f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"{extra_metrics}\n" @@ -615,6 +637,8 @@ def test_block_location_nums(ray_start_regular_shared, restore_data_context): f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"\n" @@ -1006,6 +1030,8 @@ def test_dataset_stats_shuffle(ray_start_regular_shared): * Output rows per task: N min, N max, N mean, N tasks used * Tasks per node: N min, N max, N mean; N nodes used * Operator throughput: + * Total input num rows: N rows + * Total output num rows: N rows * Ray Data throughput: N rows/s * Estimated single node throughput: N rows/s @@ -1019,6 +1045,8 @@ def test_dataset_stats_shuffle(ray_start_regular_shared): * Output rows per task: N min, N max, N mean, N tasks used * Tasks per node: N min, N max, N mean; N nodes used * Operator throughput: + * Total input num rows: N rows + * Total output num rows: N rows * Ray Data throughput: N rows/s * Estimated single node throughput: N rows/s @@ -1034,6 +1062,8 @@ def test_dataset_stats_shuffle(ray_start_regular_shared): * Output rows per task: N min, N max, N mean, N tasks used * Tasks per node: N min, N max, N mean; N nodes used * Operator throughput: + * Total input num rows: N rows + * Total output num rows: N rows * Ray Data throughput: N rows/s * Estimated single node throughput: N rows/s @@ -1047,6 +1077,8 @@ def test_dataset_stats_shuffle(ray_start_regular_shared): * Output rows per task: N min, N max, N mean, N tasks used * Tasks per node: N min, N max, N mean; N nodes used * Operator throughput: + * Total input num rows: N rows + * Total output num rows: N rows * Ray Data throughput: N rows/s * Estimated single node throughput: N rows/s @@ -1106,6 +1138,8 @@ def test_dataset_stats_range(ray_start_regular_shared, tmp_path): f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"\n" @@ -1135,6 +1169,8 @@ def test_dataset_split_stats(ray_start_regular_shared, tmp_path): f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"\n" @@ -1148,6 +1184,8 @@ def test_dataset_split_stats(ray_start_regular_shared, tmp_path): f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"\n" @@ -1161,6 +1199,8 @@ def test_dataset_split_stats(ray_start_regular_shared, tmp_path): f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"\n" @@ -1361,6 +1401,8 @@ def test_streaming_stats_full(ray_start_regular_shared, restore_data_context): * Output rows per task: N min, N max, N mean, N tasks used * Tasks per node: N min, N max, N mean; N nodes used * Operator throughput: + * Total input num rows: N rows + * Total output num rows: N rows * Ray Data throughput: N rows/s * Estimated single node throughput: N rows/s @@ -1399,6 +1441,8 @@ def test_write_ds_stats(ray_start_regular_shared, tmp_path): * Output rows per task: N min, N max, N mean, N tasks used * Tasks per node: N min, N max, N mean; N nodes used * Operator throughput: + * Total input num rows: N rows + * Total output num rows: N rows * Ray Data throughput: N rows/s * Estimated single node throughput: N rows/s @@ -1430,6 +1474,8 @@ def test_write_ds_stats(ray_start_regular_shared, tmp_path): * Output rows per task: N min, N max, N mean, N tasks used * Tasks per node: N min, N max, N mean; N nodes used * Operator throughput: + * Total input num rows: N rows + * Total output num rows: N rows * Ray Data throughput: N rows/s * Estimated single node throughput: N rows/s @@ -1443,6 +1489,8 @@ def test_write_ds_stats(ray_start_regular_shared, tmp_path): * Output rows per task: N min, N max, N mean, N tasks used * Tasks per node: N min, N max, N mean; N nodes used * Operator throughput: + * Total input num rows: N rows + * Total output num rows: N rows * Ray Data throughput: N rows/s * Estimated single node throughput: N rows/s @@ -1640,9 +1688,8 @@ def test_dataset_throughput(shutdown_only): f = dummy_map_batches_sleep(0.01) ds = ray.data.range(100).map(f).materialize().map(f).materialize() - # Pattern to match operator throughput operator_pattern = re.compile( - r"Operator (\d+).*?Ray Data throughput: (\d+\.\d+) rows/s.*?Estimated single node throughput: (\d+\.\d+) rows/s", # noqa: E501 + r"Operator (\d+).*?\* Operator throughput:\s*.*?\* Ray Data throughput: (\d+\.\d+) rows/s.*?\* Estimated single node throughput: (\d+\.\d+) rows/s", re.DOTALL, ) @@ -1661,6 +1708,73 @@ def test_dataset_throughput(shutdown_only): assert float(dataset_match[1]) >= float(dataset_match[2]) +def test_individual_operator_num_rows(shutdown_only): + # The input num rows of an individual operator should be the same as the output num rows of its parent operator. + ray.shutdown() + ray.init(num_cpus=2) + + data = [{"id": i, "value": i * 1.5, "category": i % 5} for i in range(500)] + ds = ( + ray.data.from_items(data) + .map(lambda x: {**x, "value_squared": x["value"] ** 2}) + .filter(lambda x: x["value_squared"] > 300) + ) + + stats_output = ds.materialize().stats() + re_op0_output = re.compile(r"Operator 0.*?Total output num rows: (\d+)", re.DOTALL) + re_op1_input = re.compile(r"Operator 1.*?Total input num rows: (\d+)", re.DOTALL) + + op0_output = int(re_op0_output.search(stats_output).group(1)) + op1_input = int(re_op1_input.search(stats_output).group(1)) + + assert op0_output == 500 + assert op0_output == op1_input + + +def test_sub_operator_num_rows(shutdown_only): + # The input num rows of sub operator: + # The first sub-operator: total output from all parent nodes + # Subsequent sub-operators: output of the previous sub-operator + ray.shutdown() + ray.init(num_cpus=2) + + data1 = [{"id": i, "value1": i * 1.5, "category1": i % 5} for i in range(500)] + ds1 = ray.data.from_items(data1) + data2 = [{"id": i, "value2": i * 1.5, "category2": i % 5} for i in range(300)] + ds2 = ray.data.from_items(data2) + ds = ds1.join(ds2, join_type="left_outer", num_partitions=2) + + stats_output = ds.materialize().stats() + + patterns = { + "operator0_output": re.compile( + r"Operator 0.*?Total output num rows: (\d+)", re.DOTALL + ), + "subop0_input": re.compile( + r"Suboperator 0.*?Total input num rows: (\d+)", re.DOTALL + ), + "subop0_output": re.compile( + r"Suboperator 0.*?Total output num rows: (\d+)", re.DOTALL + ), + "subop1_input": re.compile( + r"Suboperator 1.*?Total input num rows: (\d+)", re.DOTALL + ), + } + + extracted_data = {} + for key, pattern in patterns.items(): + match = pattern.search(stats_output) + if match: + extracted_data[key] = int(match.group(1)) + else: + extracted_data[key] = None + + assert extracted_data["operator0_output"] == 500 + assert extracted_data["subop0_output"] == 800 + assert extracted_data["operator0_output"] == extracted_data["subop0_input"] + assert extracted_data["subop0_output"] == extracted_data["subop1_input"] + + @pytest.mark.parametrize("verbose_stats_logs", [True, False]) def test_spilled_stats(shutdown_only, verbose_stats_logs, restore_data_context): context = DataContext.get_current() @@ -1686,6 +1800,8 @@ def test_spilled_stats(shutdown_only, verbose_stats_logs, restore_data_context): f"* Output rows per task: N min, N max, N mean, N tasks used\n" f"* Tasks per node: N min, N max, N mean; N nodes used\n" f"* Operator throughput:\n" + f" * Total input num rows: N rows\n" + f" * Total output num rows: N rows\n" f" * Ray Data throughput: N rows/s\n" f" * Estimated single node throughput: N rows/s\n" f"{extra_metrics}\n" From 369c780953239cb665655674d5aeb0bb0fdf863f Mon Sep 17 00:00:00 2001 From: Vaishnavi Panchavati <38342947+vaishdho1@users.noreply.github.com> Date: Sat, 6 Sep 2025 16:21:56 -0700 Subject: [PATCH 492/634] [serve] Fix buffered logging reusing request context (Fixes #55851) (#56094) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? Currently, when Serve file logs are buffered via a `MemoryHandler`, `ServeContextFilter` fetches the serve request context at flush time instead of when the log record is emitted. As a result, many log records flushed together can share the same request context, breaking per request tracing. This PR captures the request context at emit time when buffering is enabled and makes the filter idempotent so it won’t overwrite pre populated fields. This preserves correct per record context for buffered file logs without changing non buffered behavior. ## Related issue number Closes #55851 ## Performance Testing Manual Verification - Benchmarked both buffered and non buffered cases with and without the fix. Performance- Used Locust with 100 users for a duration of 3-4 mins Without buffering: With fix: `Avg: 396.69(ms), P99: 580(ms), RPS: 228.4` Without fix: `391.29(ms), P99: 560(ms), RPS: 239` With buffering: set `RAY_SERVE_REQUEST_PATH_LOG_BUFFER_SIZE` = 1000 With fix: `Avg(ms): 400.83, P99(ms): 620, RPS: 230.5` Without fix: `Avg(ms): 373.25, P99(ms): 610, RPS: 249.4` ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [x] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Vaishnavi Panchavati Signed-off-by: Vaishnavi Panchavati <38342947+vaishdho1@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Abrar Sheikh --- python/ray/serve/_private/logging_utils.py | 31 +++++++------- python/ray/serve/tests/test_logging.py | 48 ++++++++++++++++++++++ 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/python/ray/serve/_private/logging_utils.py b/python/ray/serve/_private/logging_utils.py index 4688e30e4089..1e44c059d583 100644 --- a/python/ray/serve/_private/logging_utils.py +++ b/python/ray/serve/_private/logging_utils.py @@ -93,6 +93,7 @@ class ServeContextFilter(logging.Filter): def filter(self, record): if should_skip_context_filter(record): return True + request_context = ray.serve.context._get_serve_request_context() if request_context.route: setattr(record, SERVE_LOG_ROUTE, request_context.route) @@ -369,15 +370,24 @@ def configure_component_logger( maxBytes=max_bytes, backupCount=backup_count, ) + # Create a memory handler that buffers log records and flushes to file handler + # Buffer capacity: buffer_size records + # Flush triggers: buffer full, ERROR messages, or explicit flush + memory_handler = logging.handlers.MemoryHandler( + capacity=buffer_size, + target=file_handler, + flushLevel=logging.ERROR, # Auto-flush on ERROR/CRITICAL + ) if RAY_SERVE_ENABLE_JSON_LOGGING: logger.warning( "'RAY_SERVE_ENABLE_JSON_LOGGING' is deprecated, please use " "'LoggingConfig' to enable json format." ) + # Add filters directly to the memory handler effective for both buffered and non buffered cases if RAY_SERVE_ENABLE_JSON_LOGGING or logging_config.encoding == EncodingType.JSON: - file_handler.addFilter(ServeCoreContextFilter()) - file_handler.addFilter(ServeContextFilter()) - file_handler.addFilter( + memory_handler.addFilter(ServeCoreContextFilter()) + memory_handler.addFilter(ServeContextFilter()) + memory_handler.addFilter( ServeComponentFilter(component_name, component_id, component_type) ) file_handler.setFormatter(json_formatter) @@ -385,12 +395,12 @@ def configure_component_logger( file_handler.setFormatter(serve_formatter) if logging_config.enable_access_log is False: - file_handler.addFilter(log_access_log_filter) + memory_handler.addFilter(log_access_log_filter) else: - file_handler.addFilter(ServeContextFilter()) + memory_handler.addFilter(ServeContextFilter()) # Remove unwanted attributes from the log record. - file_handler.addFilter(ServeLogAttributeRemovalFilter()) + memory_handler.addFilter(ServeLogAttributeRemovalFilter()) # Redirect print, stdout, and stderr to Serve logger, only when it's on the replica. if not RAY_SERVE_LOG_TO_STDERR and component_type == ServeComponentType.REPLICA: @@ -398,15 +408,6 @@ def configure_component_logger( sys.stdout = StreamToLogger(logger, logging.INFO, sys.stdout) sys.stderr = StreamToLogger(logger, logging.INFO, sys.stderr) - # Create a memory handler that buffers log records and flushes to file handler - # Buffer capacity: buffer_size records - # Flush triggers: buffer full, ERROR messages, or explicit flush - memory_handler = logging.handlers.MemoryHandler( - capacity=buffer_size, - target=file_handler, - flushLevel=logging.ERROR, # Auto-flush on ERROR/CRITICAL - ) - # Add the memory handler instead of the file handler directly logger.addHandler(memory_handler) diff --git a/python/ray/serve/tests/test_logging.py b/python/ray/serve/tests/test_logging.py index df6377a6acad..179f99c616f3 100644 --- a/python/ray/serve/tests/test_logging.py +++ b/python/ray/serve/tests/test_logging.py @@ -7,6 +7,7 @@ import sys import time import uuid +from collections import Counter from contextlib import redirect_stderr from pathlib import Path from typing import List, Tuple @@ -1360,5 +1361,52 @@ def test_configure_default_serve_logger_with_stderr_redirect( assert not isinstance(sys.stderr, StreamToLogger) +@pytest.mark.parametrize( + "ray_instance", + [ + {"RAY_SERVE_REQUEST_PATH_LOG_BUFFER_SIZE": "1"}, + {"RAY_SERVE_REQUEST_PATH_LOG_BUFFER_SIZE": "100"}, + ], + indirect=True, +) +def test_request_id_uniqueness_with_buffering(ray_instance): + """Test request IDs are unique when buffering is enabled.""" + + logger = logging.getLogger("ray.serve") + + @serve.deployment(logging_config={"encoding": "JSON"}) + class TestApp: + async def __call__(self): + logger.info("Processing request") + logger.info("Additional log entry") + return "OK" + + serve.run(TestApp.bind()) + for _ in range(200): + httpx.get("http://127.0.0.1:8000/") + + logs_dir = get_serve_logs_dir() + + def check_logs(): + for log_file in os.listdir(logs_dir): + if log_file.startswith("replica"): + with open(os.path.join(logs_dir, log_file)) as f: + log_request_ids = [] + for line in f: + log_entry = json.loads(line) + request_id = log_entry.get("request_id", None) + message = log_entry.get("message", None) + if request_id: + # Append the (request_id, message) pairs to the list + log_request_ids.append((request_id, message)) + # Check that there are no duplicate (request_id, message) pairs + request_id_counts = Counter(log_request_ids) + for _, count in request_id_counts.items(): + assert count == 1, "Request ID duplicates when buffering" + return True + + wait_for_condition(check_logs) + + if __name__ == "__main__": sys.exit(pytest.main(["-v", "-s", __file__])) From 7a3da1153c9e395d0dac3b8e00fb8d51e057240b Mon Sep 17 00:00:00 2001 From: Rueian Date: Sun, 7 Sep 2025 11:14:32 -0700 Subject: [PATCH 493/634] [core] replace node_manager_client with raylet_client_lib (#56261) Signed-off-by: Rueian --- BUILD.bazel | 2 +- src/fakes/ray/rpc/raylet/BUILD.bazel | 2 +- src/fakes/ray/rpc/raylet/raylet_client.h | 2 +- src/ray/core_worker/BUILD.bazel | 6 +- src/ray/core_worker/core_worker.h | 2 +- src/ray/core_worker/core_worker_process.cc | 15 +- .../experimental_mutable_object_provider.h | 2 +- .../io_ray_runtime_task_NativeTaskExecutor.cc | 2 +- src/ray/core_worker/object_recovery_manager.h | 4 +- .../core_worker/task_submission/BUILD.bazel | 2 +- .../task_submission/normal_task_submitter.h | 4 +- .../task_submission/tests/BUILD.bazel | 2 +- .../tests/normal_task_submitter_test.cc | 2 +- src/ray/core_worker/tests/BUILD.bazel | 2 +- .../tests/object_recovery_manager_test.cc | 2 +- src/ray/gcs/gcs_server/BUILD.bazel | 10 +- src/ray/gcs/gcs_server/gcs_actor_scheduler.h | 4 +- .../gcs_server/gcs_autoscaler_state_manager.h | 2 +- src/ray/gcs/gcs_server/gcs_node_manager.h | 2 +- .../gcs_placement_group_scheduler.h | 4 +- src/ray/gcs/gcs_server/gcs_server.cc | 4 +- src/ray/gcs/gcs_server/gcs_server.h | 2 +- src/ray/raylet/BUILD.bazel | 4 +- src/ray/raylet/main.cc | 4 +- src/ray/raylet/node_manager.h | 2 +- src/ray/raylet_client/BUILD.bazel | 56 ----- src/ray/raylet_client/node_manager_client.h | 207 ----------------- src/ray/rpc/BUILD.bazel | 46 +++- .../rpc/node_manager/node_manager_server.h | 2 +- .../raylet}/raylet_client.cc | 210 ++++++++++++++---- .../raylet}/raylet_client.h | 22 +- .../raylet}/raylet_client_interface.h | 0 .../raylet}/raylet_client_pool.cc | 2 +- .../raylet}/raylet_client_pool.h | 2 +- .../raylet}/tests/BUILD.bazel | 3 +- .../raylet}/tests/raylet_client_pool_test.cc | 2 +- src/ray/rpc/retryable_grpc_client.h | 31 ++- src/ray/rpc/worker/core_worker_client_pool.h | 4 +- 38 files changed, 300 insertions(+), 376 deletions(-) delete mode 100644 src/ray/raylet_client/BUILD.bazel delete mode 100644 src/ray/raylet_client/node_manager_client.h rename src/ray/{raylet_client => rpc/raylet}/raylet_client.cc (66%) rename src/ray/{raylet_client => rpc/raylet}/raylet_client.h (92%) rename src/ray/{raylet_client => rpc/raylet}/raylet_client_interface.h (100%) rename src/ray/{raylet_client => rpc/raylet}/raylet_client_pool.cc (98%) rename src/ray/{raylet_client => rpc/raylet}/raylet_client_pool.h (98%) rename src/ray/{raylet_client => rpc/raylet}/tests/BUILD.bazel (78%) rename src/ray/{raylet_client => rpc/raylet}/tests/raylet_client_pool_test.cc (99%) diff --git a/BUILD.bazel b/BUILD.bazel index 93a988b07b9d..c30d1b1f43ae 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -130,7 +130,7 @@ ray_cc_library( ), deps = [ "//src/ray/common:asio", - "//src/ray/raylet_client:raylet_client_interface", + "//src/ray/rpc:raylet_client_interface", ], ) diff --git a/src/fakes/ray/rpc/raylet/BUILD.bazel b/src/fakes/ray/rpc/raylet/BUILD.bazel index f6ced3e4fc2e..af2537ff4ab0 100644 --- a/src/fakes/ray/rpc/raylet/BUILD.bazel +++ b/src/fakes/ray/rpc/raylet/BUILD.bazel @@ -4,6 +4,6 @@ ray_cc_library( name = "fake_raylet_client", hdrs = ["raylet_client.h"], deps = [ - "//src/ray/raylet_client:raylet_client_interface", + "//src/ray/rpc:raylet_client_interface", ], ) diff --git a/src/fakes/ray/rpc/raylet/raylet_client.h b/src/fakes/ray/rpc/raylet/raylet_client.h index 044351f595f4..4b4e048cafd7 100644 --- a/src/fakes/ray/rpc/raylet/raylet_client.h +++ b/src/fakes/ray/rpc/raylet/raylet_client.h @@ -15,7 +15,7 @@ #pragma once #include "ray/common/scheduling/scheduling_ids.h" -#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/rpc/raylet/raylet_client_interface.h" namespace ray { diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 0c72ff982e03..10c55d1d387a 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -45,9 +45,9 @@ ray_cc_library( "//src/ray/protobuf:pubsub_cc_proto", "//src/ray/pubsub:publisher", "//src/ray/pubsub:subscriber", - "//src/ray/raylet_client:raylet_client_lib", "//src/ray/rpc:core_worker_client", "//src/ray/rpc:metrics_agent_client", + "//src/ray/rpc:raylet_client_lib", "//src/ray/stats:stats_lib", "//src/ray/util:container_util", "//src/ray/util:env", @@ -333,8 +333,8 @@ ray_cc_library( hdrs = ["experimental_mutable_object_provider.h"], deps = [ ":experimental_mutable_object_manager", - "//src/ray/raylet_client:raylet_client_interface", "//src/ray/rpc:client_call", + "//src/ray/rpc:raylet_client_interface", ], ) @@ -357,7 +357,7 @@ ray_cc_library( ":reference_count", ":task_manager", "//src/ray/common:id", - "//src/ray/raylet_client:raylet_client_pool", + "//src/ray/rpc:raylet_client_pool", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/synchronization", ], diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index ba060a05b500..ec5af5e24fa2 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -52,7 +52,7 @@ #include "ray/ipc/raylet_ipc_client_interface.h" #include "ray/pubsub/publisher.h" #include "ray/pubsub/subscriber.h" -#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/rpc/raylet/raylet_client_interface.h" #include "ray/util/process.h" #include "ray/util/shared_lru.h" #include "src/ray/protobuf/pubsub.pb.h" diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index 57c4e217a1a9..1c8d06f0f117 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -35,7 +35,7 @@ #include "ray/core_worker/core_worker_rpc_proxy.h" #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/ipc/raylet_ipc_client.h" -#include "ray/raylet_client/raylet_client.h" +#include "ray/rpc/raylet/raylet_client.h" #include "ray/stats/stats.h" #include "ray/util/container_util.h" #include "ray/util/env.h" @@ -244,10 +244,10 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( // instead of crashing. auto raylet_address = rpc::RayletClientPool::GenerateRayletAddress( local_node_id, options.node_ip_address, options.node_manager_port); - auto local_raylet_rpc_client = std::make_shared( - std::move(raylet_address), - *client_call_manager, - /*raylet_unavailable_timeout_callback=*/[] {}); + auto local_raylet_rpc_client = + std::make_shared(std::move(raylet_address), + *client_call_manager, + /*raylet_unavailable_timeout_callback=*/[] {}); auto core_worker_server = std::make_unique(WorkerTypeString(options.worker_type), assigned_port, @@ -284,7 +284,7 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( auto raylet_client_pool = std::make_shared([&](const rpc::Address &addr) { auto core_worker = GetCoreWorker(); - return std::make_shared( + return std::make_shared( addr, *core_worker->client_call_manager_, rpc::RayletClientPool::GetDefaultUnavailableTimeoutCallback( @@ -841,8 +841,7 @@ void CoreWorkerProcessImpl::InitializeSystemConfig() { // TODO(joshlee): This local raylet client has a custom retry policy below since its // likely the driver can start up before the raylet is ready. We want to move away // from this and will be fixed in https://github.com/ray-project/ray/issues/55200 - raylet::RayletClient local_raylet_rpc_client( - raylet_address, client_call_manager, [] {}); + rpc::RayletClient local_raylet_rpc_client(raylet_address, client_call_manager, [] {}); std::function get_once = [this, &get_once, diff --git a/src/ray/core_worker/experimental_mutable_object_provider.h b/src/ray/core_worker/experimental_mutable_object_provider.h index 16a9c11e2149..085f8994cfea 100644 --- a/src/ray/core_worker/experimental_mutable_object_provider.h +++ b/src/ray/core_worker/experimental_mutable_object_provider.h @@ -18,8 +18,8 @@ #include #include "ray/core_worker/experimental_mutable_object_manager.h" -#include "ray/raylet_client/raylet_client_interface.h" #include "ray/rpc/client_call.h" +#include "ray/rpc/raylet/raylet_client_interface.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc index 55ed42f580a5..b0077962cb2a 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_task_NativeTaskExecutor.cc @@ -20,7 +20,7 @@ #include "ray/common/id.h" #include "ray/core_worker/common.h" #include "ray/core_worker/core_worker.h" -#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/rpc/raylet/raylet_client_interface.h" #ifdef __cplusplus extern "C" { diff --git a/src/ray/core_worker/object_recovery_manager.h b/src/ray/core_worker/object_recovery_manager.h index 0e29b5d6ab7b..77cd9dac636c 100644 --- a/src/ray/core_worker/object_recovery_manager.h +++ b/src/ray/core_worker/object_recovery_manager.h @@ -25,8 +25,8 @@ #include "ray/core_worker/reference_count.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/core_worker/task_manager.h" -#include "ray/raylet_client/raylet_client_interface.h" -#include "ray/raylet_client/raylet_client_pool.h" +#include "ray/rpc/raylet/raylet_client_interface.h" +#include "ray/rpc/raylet/raylet_client_pool.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/task_submission/BUILD.bazel b/src/ray/core_worker/task_submission/BUILD.bazel index ce90126245e3..7054fc3e68cc 100644 --- a/src/ray/core_worker/task_submission/BUILD.bazel +++ b/src/ray/core_worker/task_submission/BUILD.bazel @@ -99,8 +99,8 @@ ray_cc_library( "//src/ray/core_worker:lease_policy", "//src/ray/core_worker:memory_store", "//src/ray/core_worker:task_manager_interface", - "//src/ray/raylet_client:raylet_client_interface", "//src/ray/rpc:core_worker_client", + "//src/ray/rpc:raylet_client_interface", "@com_google_absl//absl/base:core_headers", ], ) diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.h b/src/ray/core_worker/task_submission/normal_task_submitter.h index 1bd4b75f67ad..f7b385f05a36 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.h +++ b/src/ray/core_worker/task_submission/normal_task_submitter.h @@ -29,8 +29,8 @@ #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/core_worker/task_manager_interface.h" #include "ray/core_worker/task_submission/dependency_resolver.h" -#include "ray/raylet_client/raylet_client_interface.h" -#include "ray/raylet_client/raylet_client_pool.h" +#include "ray/rpc/raylet/raylet_client_interface.h" +#include "ray/rpc/raylet/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" diff --git a/src/ray/core_worker/task_submission/tests/BUILD.bazel b/src/ray/core_worker/task_submission/tests/BUILD.bazel index 5687a2de0698..4a8ff0ec0d95 100644 --- a/src/ray/core_worker/task_submission/tests/BUILD.bazel +++ b/src/ray/core_worker/task_submission/tests/BUILD.bazel @@ -70,8 +70,8 @@ ray_cc_test( "//src/ray/common:test_utils", "//src/ray/core_worker:memory_store", "//src/ray/core_worker/task_submission:normal_task_submitter", - "//src/ray/raylet_client:raylet_client_interface", "//src/ray/rpc:core_worker_client", + "//src/ray/rpc:raylet_client_interface", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc index fc2d400714c5..f3c741b5a5f4 100644 --- a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc @@ -30,7 +30,7 @@ #include "ray/common/task/task_util.h" #include "ray/common/test_utils.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" -#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/rpc/raylet/raylet_client_interface.h" #include "ray/rpc/worker/core_worker_client.h" namespace ray { diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index f8f357699e07..ab453f5da2c5 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -76,7 +76,7 @@ ray_cc_test( "//src/ray/core_worker:memory_store", "//src/ray/core_worker:object_recovery_manager", "//src/ray/object_manager:object_manager_common", - "//src/ray/raylet_client:raylet_client_interface", + "//src/ray/rpc:raylet_client_interface", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/core_worker/tests/object_recovery_manager_test.cc b/src/ray/core_worker/tests/object_recovery_manager_test.cc index 370aefebea00..814e886411c8 100644 --- a/src/ray/core_worker/tests/object_recovery_manager_test.cc +++ b/src/ray/core_worker/tests/object_recovery_manager_test.cc @@ -30,7 +30,7 @@ #include "ray/common/task/task_util.h" #include "ray/common/test_utils.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" -#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/rpc/raylet/raylet_client_interface.h" namespace ray { namespace core { diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 35ced96675f6..42b88c0c16f3 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -78,7 +78,7 @@ ray_cc_library( "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/protobuf:ray_syncer_cc_proto", "//src/ray/pubsub:gcs_publisher", - "//src/ray/raylet_client:raylet_client_pool", + "//src/ray/rpc:raylet_client_pool", "//src/ray/stats:stats_metric", "//src/ray/util:event", "//src/ray/util:logging", @@ -286,7 +286,7 @@ ray_cc_library( "//src/ray/common:task_common", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "//src/ray/raylet/scheduling:scheduling_context", - "//src/ray/raylet_client:raylet_client_interface", + "//src/ray/rpc:raylet_client_interface", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", ], @@ -393,8 +393,8 @@ ray_cc_library( "//src/ray/common:task_common", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/raylet/scheduling:cluster_lease_manager", - "//src/ray/raylet_client:raylet_client_interface", "//src/ray/rpc:core_worker_client", + "//src/ray/rpc:raylet_client_interface", "//src/ray/util:logging", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -516,11 +516,11 @@ ray_cc_library( "//src/ray/pubsub:gcs_publisher", "//src/ray/pubsub:publisher", "//src/ray/raylet/scheduling:scheduler", - "//src/ray/raylet_client:raylet_client_lib", - "//src/ray/raylet_client:raylet_client_pool", "//src/ray/rpc:core_worker_client", "//src/ray/rpc:grpc_server", "//src/ray/rpc:metrics_agent_client", + "//src/ray/rpc:raylet_client_lib", + "//src/ray/rpc:raylet_client_pool", "//src/ray/util:counter_map", "//src/ray/util:exponential_backoff", "//src/ray/util:network_util", diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h index e05fc10e46eb..bc12aa833b57 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h @@ -31,8 +31,8 @@ #include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" -#include "ray/raylet_client/raylet_client_interface.h" -#include "ray/raylet_client/raylet_client_pool.h" +#include "ray/rpc/raylet/raylet_client_interface.h" +#include "ray/rpc/raylet/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "src/ray/protobuf/gcs_service.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h index 61246ad0a6c9..9079b75bfbd8 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h +++ b/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h @@ -31,7 +31,7 @@ #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/gcs/gcs_server/state_util.h" #include "ray/pubsub/gcs_publisher.h" -#include "ray/raylet_client/raylet_client_pool.h" +#include "ray/rpc/raylet/raylet_client_pool.h" #include "ray/util/thread_checker.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.h b/src/ray/gcs/gcs_server/gcs_node_manager.h index 1189fd9ce521..65efb9a18cc2 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/ray/gcs/gcs_server/gcs_node_manager.h @@ -27,7 +27,7 @@ #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/pubsub/gcs_publisher.h" -#include "ray/raylet_client/raylet_client_pool.h" +#include "ray/rpc/raylet/raylet_client_pool.h" #include "ray/stats/metric_defs.h" #include "ray/util/event.h" #include "src/ray/protobuf/autoscaler.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h index 778a58f529dd..1f074da8c0a4 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h @@ -30,8 +30,8 @@ #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/scheduling/policy/scheduling_context.h" -#include "ray/raylet_client/raylet_client_interface.h" -#include "ray/raylet_client/raylet_client_pool.h" +#include "ray/rpc/raylet/raylet_client_interface.h" +#include "ray/rpc/raylet/raylet_client_pool.h" #include "src/ray/protobuf/gcs_service.pb.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index ee3e1a478acb..9adbcb2b9445 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -36,7 +36,7 @@ #include "ray/gcs/store_client/redis_store_client.h" #include "ray/gcs/store_client/store_client.h" #include "ray/pubsub/publisher.h" -#include "ray/raylet_client/raylet_client.h" +#include "ray/rpc/raylet/raylet_client.h" #include "ray/stats/stats.h" #include "ray/util/network_util.h" @@ -71,7 +71,7 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, ClusterID::Nil(), RayConfig::instance().gcs_server_rpc_client_thread_num()), raylet_client_pool_([this](const rpc::Address &addr) { - return std::make_shared( + return std::make_shared( addr, this->client_call_manager_, /*raylet_unavailable_timeout_callback=*/[this, addr]() { diff --git a/src/ray/gcs/gcs_server/gcs_server.h b/src/ray/gcs/gcs_server/gcs_server.h index 9db51a1e976d..4bbe81432e01 100644 --- a/src/ray/gcs/gcs_server/gcs_server.h +++ b/src/ray/gcs/gcs_server/gcs_server.h @@ -39,10 +39,10 @@ #include "ray/pubsub/gcs_publisher.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" -#include "ray/raylet_client/raylet_client_pool.h" #include "ray/rpc/client_call.h" #include "ray/rpc/grpc_server.h" #include "ray/rpc/metrics_agent_client.h" +#include "ray/rpc/raylet/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/throttler.h" diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index e4c6ca19b987..854a8d2478ee 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -286,9 +286,9 @@ ray_cc_binary( "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/object_manager:ownership_object_directory", "//src/ray/raylet/scheduling:cluster_lease_manager", - "//src/ray/raylet_client:raylet_client_lib", - "//src/ray/raylet_client:raylet_client_pool", "//src/ray/rpc:metrics_agent_client", + "//src/ray/rpc:raylet_client_lib", + "//src/ray/rpc:raylet_client_pool", "//src/ray/stats:stats_lib", "//src/ray/util:cmd_line_utils", "//src/ray/util:event", diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 85f29663e916..e96b52f195e9 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -35,7 +35,7 @@ #include "ray/raylet/local_object_manager.h" #include "ray/raylet/local_object_manager_interface.h" #include "ray/raylet/raylet.h" -#include "ray/raylet_client/raylet_client.h" +#include "ray/rpc/raylet/raylet_client.h" #include "ray/stats/stats.h" #include "ray/util/cmd_line_utils.h" #include "ray/util/event.h" @@ -562,7 +562,7 @@ int main(int argc, char *argv[]) { raylet_client_pool = std::make_unique([&](const ray::rpc::Address &addr) { - return std::make_shared( + return std::make_shared( addr, *client_call_manager, ray::rpc::RayletClientPool::GetDefaultUnavailableTimeoutCallback( diff --git a/src/ray/raylet/node_manager.h b/src/ray/raylet/node_manager.h index 2cfae2004fa5..da0b2182d200 100644 --- a/src/ray/raylet/node_manager.h +++ b/src/ray/raylet/node_manager.h @@ -48,8 +48,8 @@ #include "ray/raylet/wait_manager.h" #include "ray/raylet/worker_killing_policy.h" #include "ray/raylet/worker_pool.h" -#include "ray/raylet_client/raylet_client_pool.h" #include "ray/rpc/node_manager/node_manager_server.h" +#include "ray/rpc/raylet/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client_pool.h" #include "ray/util/throttler.h" diff --git a/src/ray/raylet_client/BUILD.bazel b/src/ray/raylet_client/BUILD.bazel deleted file mode 100644 index e8f6d4f05244..000000000000 --- a/src/ray/raylet_client/BUILD.bazel +++ /dev/null @@ -1,56 +0,0 @@ -load("//bazel:ray.bzl", "ray_cc_library") - -ray_cc_library( - name = "raylet_client_interface", - hdrs = [ - "raylet_client_interface.h", - ], - deps = [ - "//src/ray/protobuf:autoscaler_cc_proto", - "//src/ray/protobuf:common_cc_proto", - "//src/ray/protobuf:node_manager_cc_proto", - "//src/ray/rpc:client_call", - ], -) - -ray_cc_library( - name = "raylet_client_pool", - srcs = ["raylet_client_pool.cc"], - hdrs = [ - "raylet_client_pool.h", - ], - deps = [ - ":raylet_client_interface", - "//src/ray/gcs/gcs_client:gcs_client_lib", - ], -) - -ray_cc_library( - name = "node_manager_client", - hdrs = [ - "node_manager_client.h", - ], - visibility = [":__subpackages__"], - deps = [ - ":raylet_client_interface", - "//src/ray/common:id", - "//src/ray/gcs/gcs_client:gcs_client_lib", - "//src/ray/protobuf:node_manager_cc_grpc", - "//src/ray/rpc:client_call", - "//src/ray/rpc:grpc_client", - "//src/ray/util:network_util", - ], -) - -ray_cc_library( - name = "raylet_client_lib", - srcs = ["raylet_client.cc"], - hdrs = ["raylet_client.h"], - deps = [ - ":node_manager_client", - ":raylet_client_interface", - "//src/ray/flatbuffers:node_manager_generated", - "//src/ray/protobuf:common_cc_proto", - "//src/ray/util:logging", - ], -) diff --git a/src/ray/raylet_client/node_manager_client.h b/src/ray/raylet_client/node_manager_client.h deleted file mode 100644 index 60558226555b..000000000000 --- a/src/ray/raylet_client/node_manager_client.h +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright 2017 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include -#include -#include -#include -#include - -#include "ray/common/status.h" -#include "ray/rpc/grpc_client.h" -#include "ray/rpc/retryable_grpc_client.h" -#include "ray/util/logging.h" -#include "src/ray/protobuf/node_manager.grpc.pb.h" -#include "src/ray/protobuf/node_manager.pb.h" - -namespace ray { - -namespace raylet { -class RayletClient; -} - -namespace rpc { - -/// TODO(dayshah): https://github.com/ray-project/ray/issues/54816 Kill this completely. -/// This class is only used by the RayletClient which is just a wrapper around this. This -/// exists for the legacy reason that all the function definitions in RayletClient have to -/// change if you move the things in here into RayletClient. -class NodeManagerClient { - public: - friend class raylet::RayletClient; - - private: - /// Constructor. - /// - /// \param[in] address Address of the node manager server. - /// \param[in] port Port of the node manager server. - /// \param[in] client_call_manager The `ClientCallManager` used for managing requests. - /// \param[in] raylet_unavailable_timeout_callback The callback function that is used - /// by the retryable grpc to remove unresponsive raylet connections from the pool once - /// its been unavailable for more than server_unavailable_timeout_seconds. - NodeManagerClient(const rpc::Address &address, - ClientCallManager &client_call_manager, - std::function raylet_unavailable_timeout_callback) - : grpc_client_(std::make_shared>( - address.ip_address(), address.port(), client_call_manager)), - retryable_grpc_client_(RetryableGrpcClient::Create( - grpc_client_->Channel(), - client_call_manager.GetMainService(), - /*max_pending_requests_bytes=*/ - std::numeric_limits::max(), - /*check_channel_status_interval_milliseconds=*/ - ::RayConfig::instance() - .grpc_client_check_connection_status_interval_milliseconds(), - /*server_unavailable_timeout_seconds=*/ - ::RayConfig::instance().raylet_rpc_server_reconnect_timeout_s(), - /*server_unavailable_timeout_callback=*/ - std::move(raylet_unavailable_timeout_callback), - /*server_name=*/"Raylet " + address.ip_address())) {} - - std::shared_ptr Channel() const { return grpc_client_->Channel(); } - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - GetResourceLoad, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - CancelLeasesWithResourceShapes, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - NotifyGCSRestart, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - RequestWorkerLease, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - PrestartWorkers, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - ReportWorkerBacklog, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RETRYABLE_RPC_CLIENT_METHOD(retryable_grpc_client_, - NodeManagerService, - ReturnWorkerLease, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - ReleaseUnusedActorWorkers, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - ShutdownRaylet, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - DrainRaylet, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - IsLocalWorkerDead, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RETRYABLE_RPC_CLIENT_METHOD(retryable_grpc_client_, - NodeManagerService, - CancelWorkerLease, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - PrepareBundleResources, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - CommitBundleResources, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - CancelResourceReserve, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - PinObjectIDs, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - GlobalGC, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - ReleaseUnusedBundles, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - GetSystemConfig, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - /// Get all the object information from the node. - VOID_RPC_CLIENT_METHOD(NodeManagerService, - GetObjectsInfo, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - GetWorkerFailureCause, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - RegisterMutableObject, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - PushMutableObject, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - VOID_RPC_CLIENT_METHOD(NodeManagerService, - GetNodeStats, - grpc_client_, - /*method_timeout_ms*/ -1, ) - - std::shared_ptr> grpc_client_; - - std::shared_ptr retryable_grpc_client_; -}; - -} // namespace rpc -} // namespace ray diff --git a/src/ray/rpc/BUILD.bazel b/src/ray/rpc/BUILD.bazel index 439e72666618..1fb75da79340 100644 --- a/src/ray/rpc/BUILD.bazel +++ b/src/ray/rpc/BUILD.bazel @@ -185,6 +185,48 @@ ray_cc_library( ], ) +ray_cc_library( + name = "raylet_client_interface", + hdrs = [ + "raylet/raylet_client_interface.h", + ], + visibility = ["//visibility:public"], + deps = [ + ":client_call", + "//src/ray/protobuf:autoscaler_cc_proto", + "//src/ray/protobuf:common_cc_proto", + "//src/ray/protobuf:node_manager_cc_proto", + ], +) + +ray_cc_library( + name = "raylet_client_pool", + srcs = ["raylet/raylet_client_pool.cc"], + hdrs = [ + "raylet/raylet_client_pool.h", + ], + visibility = ["//visibility:public"], + deps = [ + ":raylet_client_interface", + "//src/ray/gcs/gcs_client:gcs_client_lib", + ], +) + +ray_cc_library( + name = "raylet_client_lib", + srcs = ["raylet/raylet_client.cc"], + hdrs = ["raylet/raylet_client.h"], + visibility = ["//visibility:public"], + deps = [ + ":raylet_client_interface", + ":retryable_grpc_client", + "//src/ray/common:ray_config", + "//src/ray/common:task_common", + "//src/ray/protobuf:node_manager_cc_grpc", + "//src/ray/util:logging", + ], +) + ray_cc_library( name = "core_worker_client", srcs = [ @@ -196,13 +238,13 @@ ray_cc_library( "worker/core_worker_client_pool.h", ], deps = [ + ":raylet_client_interface", + ":raylet_client_pool", "//src/ray/common:id", "//src/ray/common:status", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/protobuf:core_worker_cc_grpc", "//src/ray/pubsub:subscriber", - "//src/ray/raylet_client:raylet_client_interface", - "//src/ray/raylet_client:raylet_client_pool", "//src/ray/util:logging", "//src/ray/util:network_util", "@com_github_grpc_grpc//:grpc++", diff --git a/src/ray/rpc/node_manager/node_manager_server.h b/src/ray/rpc/node_manager/node_manager_server.h index 315507085459..262c72cb284b 100644 --- a/src/ray/rpc/node_manager/node_manager_server.h +++ b/src/ray/rpc/node_manager/node_manager_server.h @@ -65,7 +65,7 @@ class NodeManagerServiceHandler { /// Handlers. For all of the following handlers, the implementations can /// handle the request asynchronously. When handling is done, the /// `send_reply_callback` should be called. See - /// src/ray/raylet_client/node_manager_client.h and + /// src/ray/rpc/raylet/raylet_client.cc and /// src/ray/protobuf/node_manager.proto for a description of the /// functionality of each handler. /// diff --git a/src/ray/raylet_client/raylet_client.cc b/src/ray/rpc/raylet/raylet_client.cc similarity index 66% rename from src/ray/raylet_client/raylet_client.cc rename to src/ray/rpc/raylet/raylet_client.cc index a4b825f7ee7f..de28d5b8e09a 100644 --- a/src/ray/raylet_client/raylet_client.cc +++ b/src/ray/rpc/raylet/raylet_client.cc @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/raylet_client/raylet_client.h" +#include "ray/rpc/raylet/raylet_client.h" +#include #include #include #include @@ -22,18 +23,29 @@ #include "ray/common/bundle_spec.h" #include "ray/common/ray_config.h" -#include "ray/raylet_client/node_manager_client.h" #include "ray/util/logging.h" +#include "src/ray/protobuf/node_manager.grpc.pb.h" -namespace ray::raylet { +namespace ray { +namespace rpc { RayletClient::RayletClient(const rpc::Address &address, rpc::ClientCallManager &client_call_manager, std::function raylet_unavailable_timeout_callback) - : grpc_client_(std::shared_ptr( - new rpc::NodeManagerClient(address, - client_call_manager, - std::move(raylet_unavailable_timeout_callback)))) {} + : grpc_client_(std::make_shared>( + address.ip_address(), address.port(), client_call_manager)), + retryable_grpc_client_(rpc::RetryableGrpcClient::Create( + grpc_client_->Channel(), + client_call_manager.GetMainService(), + /*max_pending_requests_bytes=*/std::numeric_limits::max(), + /*check_channel_status_interval_milliseconds=*/ + ::RayConfig::instance() + .grpc_client_check_connection_status_interval_milliseconds(), + /*server_unavailable_timeout_seconds=*/ + ::RayConfig::instance().raylet_rpc_server_reconnect_timeout_s(), + /*server_unavailable_timeout_callback=*/ + std::move(raylet_unavailable_timeout_callback), + /*server_name=*/std::string("Raylet ") + address.ip_address())) {} void RayletClient::RequestWorkerLease( const rpc::LeaseSpec &lease_spec, @@ -53,13 +65,23 @@ void RayletClient::RequestWorkerLease( request->set_grant_or_reject(grant_or_reject); request->set_backlog_size(backlog_size); request->set_is_selected_based_on_locality(is_selected_based_on_locality); - grpc_client_->RequestWorkerLease(*request, callback); + INVOKE_RPC_CALL(NodeManagerService, + RequestWorkerLease, + *request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::PrestartWorkers( const rpc::PrestartWorkersRequest &request, const rpc::ClientCallback &callback) { - grpc_client_->PrestartWorkers(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + PrestartWorkers, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } std::shared_ptr RayletClient::GetChannel() const { @@ -72,12 +94,16 @@ void RayletClient::ReportWorkerBacklog( rpc::ReportWorkerBacklogRequest request; request.set_worker_id(worker_id.Binary()); request.mutable_backlog_reports()->Add(backlog_reports.begin(), backlog_reports.end()); - grpc_client_->ReportWorkerBacklog( + INVOKE_RPC_CALL( + NodeManagerService, + ReportWorkerBacklog, request, [](const Status &status, rpc::ReportWorkerBacklogReply &&reply /*unused*/) { RAY_LOG_IF_ERROR(INFO, status) << "Error reporting lease backlog information: " << status; - }); + }, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::ReturnWorkerLease(int worker_port, @@ -91,10 +117,16 @@ void RayletClient::ReturnWorkerLease(int worker_port, request.set_disconnect_worker(disconnect_worker); request.set_disconnect_worker_error_detail(disconnect_worker_error_detail); request.set_worker_exiting(worker_exiting); - grpc_client_->ReturnWorkerLease( - std::move(request), [](const Status &status, rpc::ReturnWorkerLeaseReply &&) { + INVOKE_RETRYABLE_RPC_CALL( + retryable_grpc_client_, + NodeManagerService, + ReturnWorkerLease, + request, + [](const Status &status, rpc::ReturnWorkerLeaseReply &&reply /*unused*/) { RAY_LOG_IF_ERROR(INFO, status) << "Error returning worker: " << status; - }); + }, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::GetWorkerFailureCause( @@ -102,11 +134,16 @@ void RayletClient::GetWorkerFailureCause( const ray::rpc::ClientCallback &callback) { rpc::GetWorkerFailureCauseRequest request; request.set_lease_id(lease_id.Binary()); - grpc_client_->GetWorkerFailureCause( - request, [callback](const Status &status, rpc::GetWorkerFailureCauseReply &&reply) { + INVOKE_RPC_CALL( + NodeManagerService, + GetWorkerFailureCause, + request, + [callback](const Status &status, rpc::GetWorkerFailureCauseReply &&reply) { RAY_LOG_IF_ERROR(INFO, status) << "Error getting task result: " << status; callback(status, std::move(reply)); - }); + }, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::RegisterMutableObjectReader( @@ -118,7 +155,12 @@ void RayletClient::RegisterMutableObjectReader( request.set_writer_object_id(writer_object_id.Binary()); request.set_num_readers(num_readers); request.set_reader_object_id(reader_object_id.Binary()); - grpc_client_->RegisterMutableObject(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + RegisterMutableObject, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::PushMutableObject( @@ -155,15 +197,20 @@ void RayletClient::PushMutableObject( request.set_metadata(static_cast(metadata), metadata_size); // TODO(jackhumphries): Add failure recovery, retries, and timeout. - grpc_client_->PushMutableObject( - request, [callback](const Status &status, rpc::PushMutableObjectReply &&reply) { + INVOKE_RPC_CALL( + NodeManagerService, + PushMutableObject, + request, + [callback](const Status &status, rpc::PushMutableObjectReply &&reply) { RAY_LOG_IF_ERROR(ERROR, status) << "Error pushing mutable object: " << status; if (reply.done()) { // The callback is only executed once the receiver node receives all chunks // for the mutable object write. callback(status, std::move(reply)); } - }); + }, + grpc_client_, + /*method_timeout_ms*/ -1); } } @@ -174,7 +221,9 @@ void RayletClient::ReleaseUnusedActorWorkers( for (auto &worker_id : workers_in_use) { request.add_worker_ids_in_use(worker_id.Binary()); } - grpc_client_->ReleaseUnusedActorWorkers( + INVOKE_RPC_CALL( + NodeManagerService, + ReleaseUnusedActorWorkers, request, [callback](const Status &status, rpc::ReleaseUnusedActorWorkersReply &&reply) { if (!status.ok()) { @@ -183,7 +232,9 @@ void RayletClient::ReleaseUnusedActorWorkers( << status; } callback(status, std::move(reply)); - }); + }, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::CancelWorkerLease( @@ -191,7 +242,13 @@ void RayletClient::CancelWorkerLease( const rpc::ClientCallback &callback) { rpc::CancelWorkerLeaseRequest request; request.set_lease_id(lease_id.Binary()); - grpc_client_->CancelWorkerLease(std::move(request), callback); + INVOKE_RETRYABLE_RPC_CALL(retryable_grpc_client_, + NodeManagerService, + CancelWorkerLease, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::PrepareBundleResources( @@ -205,7 +262,12 @@ void RayletClient::PrepareBundleResources( message_bundle->CopyFrom(bundle_spec->GetMessage()); } RAY_CHECK(nodes.size() == 1); - grpc_client_->PrepareBundleResources(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + PrepareBundleResources, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::CommitBundleResources( @@ -219,7 +281,12 @@ void RayletClient::CommitBundleResources( message_bundle->CopyFrom(bundle_spec->GetMessage()); } RAY_CHECK(nodes.size() == 1); - grpc_client_->CommitBundleResources(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + CommitBundleResources, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::CancelResourceReserve( @@ -227,7 +294,12 @@ void RayletClient::CancelResourceReserve( const ray::rpc::ClientCallback &callback) { rpc::CancelResourceReserveRequest request; request.mutable_bundle_spec()->CopyFrom(bundle_spec.GetMessage()); - grpc_client_->CancelResourceReserve(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + CancelResourceReserve, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::ReleaseUnusedBundles( @@ -237,15 +309,20 @@ void RayletClient::ReleaseUnusedBundles( for (auto &bundle : bundles_in_use) { request.add_bundles_in_use()->CopyFrom(bundle); } - grpc_client_->ReleaseUnusedBundles( - request, [callback](const Status &status, rpc::ReleaseUnusedBundlesReply &&reply) { + INVOKE_RPC_CALL( + NodeManagerService, + ReleaseUnusedBundles, + request, + [callback](const Status &status, rpc::ReleaseUnusedBundlesReply &&reply) { if (!status.ok()) { RAY_LOG(WARNING) << "Error releasing bundles from raylet, the raylet may have died:" << status; } callback(status, std::move(reply)); - }); + }, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::PinObjectIDs( @@ -267,7 +344,12 @@ void RayletClient::PinObjectIDs( pins_in_flight_--; callback(status, std::move(reply)); }; - grpc_client_->PinObjectIDs(request, rpc_callback); + INVOKE_RPC_CALL(NodeManagerService, + PinObjectIDs, + request, + rpc_callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::ShutdownRaylet( @@ -276,7 +358,12 @@ void RayletClient::ShutdownRaylet( const rpc::ClientCallback &callback) { rpc::ShutdownRayletRequest request; request.set_graceful(graceful); - grpc_client_->ShutdownRaylet(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + ShutdownRaylet, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::DrainRaylet( @@ -288,7 +375,12 @@ void RayletClient::DrainRaylet( request.set_reason(reason); request.set_reason_message(reason_message); request.set_deadline_timestamp_ms(deadline_timestamp_ms); - grpc_client_->DrainRaylet(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + DrainRaylet, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::IsLocalWorkerDead( @@ -296,18 +388,33 @@ void RayletClient::IsLocalWorkerDead( const rpc::ClientCallback &callback) { rpc::IsLocalWorkerDeadRequest request; request.set_worker_id(worker_id.Binary()); - grpc_client_->IsLocalWorkerDead(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + IsLocalWorkerDead, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::GlobalGC(const rpc::ClientCallback &callback) { rpc::GlobalGCRequest request; - grpc_client_->GlobalGC(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + GlobalGC, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::GetResourceLoad( const rpc::ClientCallback &callback) { rpc::GetResourceLoadRequest request; - grpc_client_->GetResourceLoad(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + GetResourceLoad, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::CancelLeasesWithResourceShapes( @@ -322,25 +429,46 @@ void RayletClient::CancelLeasesWithResourceShapes( resource_shape.end()); } - grpc_client_->CancelLeasesWithResourceShapes(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + CancelLeasesWithResourceShapes, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::NotifyGCSRestart( const rpc::ClientCallback &callback) { rpc::NotifyGCSRestartRequest request; - grpc_client_->NotifyGCSRestart(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + NotifyGCSRestart, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::GetSystemConfig( const rpc::ClientCallback &callback) { rpc::GetSystemConfigRequest request; - grpc_client_->GetSystemConfig(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + GetSystemConfig, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::GetNodeStats( const rpc::GetNodeStatsRequest &request, const rpc::ClientCallback &callback) { - grpc_client_->GetNodeStats(request, callback); + INVOKE_RPC_CALL(NodeManagerService, + GetNodeStats, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } -} // namespace ray::raylet +} // namespace rpc +} // namespace ray diff --git a/src/ray/raylet_client/raylet_client.h b/src/ray/rpc/raylet/raylet_client.h similarity index 92% rename from src/ray/raylet_client/raylet_client.h rename to src/ray/rpc/raylet/raylet_client.h index 5c7eedcfa194..ad8ea08cd37a 100644 --- a/src/ray/raylet_client/raylet_client.h +++ b/src/ray/rpc/raylet/raylet_client.h @@ -14,26 +14,26 @@ #pragma once +#include + #include #include #include #include #include -#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/rpc/grpc_client.h" +#include "ray/rpc/raylet/raylet_client_interface.h" +#include "ray/rpc/retryable_grpc_client.h" +#include "src/ray/protobuf/node_manager.grpc.pb.h" +#include "src/ray/protobuf/node_manager.pb.h" // Maps from resource name to its allocation. using ResourceMappingType = std::unordered_map>>; namespace ray { - -// Forward declaration. namespace rpc { -class NodeManagerClient; -} - -namespace raylet { /// Raylet client is responsible for communication with raylet. It implements /// [RayletClientInterface] and works on worker registration, lease management, etc. @@ -164,7 +164,10 @@ class RayletClient : public RayletClientInterface { private: /// gRPC client to the NodeManagerService. - std::shared_ptr grpc_client_; + std::shared_ptr> grpc_client_; + + /// Retryable gRPC client to monitor channel health and trigger timeout callbacks. + std::shared_ptr retryable_grpc_client_; /// A map from resource name to the resource IDs that are currently reserved /// for this worker. Each pair consists of the resource ID and the fraction @@ -175,6 +178,5 @@ class RayletClient : public RayletClientInterface { std::atomic pins_in_flight_ = 0; }; -} // namespace raylet - +} // namespace rpc } // namespace ray diff --git a/src/ray/raylet_client/raylet_client_interface.h b/src/ray/rpc/raylet/raylet_client_interface.h similarity index 100% rename from src/ray/raylet_client/raylet_client_interface.h rename to src/ray/rpc/raylet/raylet_client_interface.h diff --git a/src/ray/raylet_client/raylet_client_pool.cc b/src/ray/rpc/raylet/raylet_client_pool.cc similarity index 98% rename from src/ray/raylet_client/raylet_client_pool.cc rename to src/ray/rpc/raylet/raylet_client_pool.cc index f955d37a5a46..5283f73c88ff 100644 --- a/src/ray/raylet_client/raylet_client_pool.cc +++ b/src/ray/rpc/raylet/raylet_client_pool.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/raylet_client/raylet_client_pool.h" +#include "ray/rpc/raylet/raylet_client_pool.h" #include #include diff --git a/src/ray/raylet_client/raylet_client_pool.h b/src/ray/rpc/raylet/raylet_client_pool.h similarity index 98% rename from src/ray/raylet_client/raylet_client_pool.h rename to src/ray/rpc/raylet/raylet_client_pool.h index 2440ee543e45..d69edf0533ab 100644 --- a/src/ray/raylet_client/raylet_client_pool.h +++ b/src/ray/rpc/raylet/raylet_client_pool.h @@ -24,7 +24,7 @@ #include "absl/synchronization/mutex.h" #include "ray/common/id.h" #include "ray/gcs/gcs_client/gcs_client.h" -#include "ray/raylet_client/raylet_client_interface.h" +#include "ray/rpc/raylet/raylet_client_interface.h" namespace ray { namespace rpc { diff --git a/src/ray/raylet_client/tests/BUILD.bazel b/src/ray/rpc/raylet/tests/BUILD.bazel similarity index 78% rename from src/ray/raylet_client/tests/BUILD.bazel rename to src/ray/rpc/raylet/tests/BUILD.bazel index 775291eeb00d..280f5a3c3927 100644 --- a/src/ray/raylet_client/tests/BUILD.bazel +++ b/src/ray/rpc/raylet/tests/BUILD.bazel @@ -8,8 +8,7 @@ ray_cc_test( deps = [ "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/gcs/gcs_client:gcs_client_lib", - "//src/ray/raylet_client:node_manager_client", - "//src/ray/raylet_client:raylet_client_pool", + "//src/ray/rpc:raylet_client_pool", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/raylet_client/tests/raylet_client_pool_test.cc b/src/ray/rpc/raylet/tests/raylet_client_pool_test.cc similarity index 99% rename from src/ray/raylet_client/tests/raylet_client_pool_test.cc rename to src/ray/rpc/raylet/tests/raylet_client_pool_test.cc index 7c8a85855f85..5d711c92e5b3 100644 --- a/src/ray/raylet_client/tests/raylet_client_pool_test.cc +++ b/src/ray/rpc/raylet/tests/raylet_client_pool_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/raylet_client/raylet_client_pool.h" +#include "ray/rpc/raylet/raylet_client_pool.h" #include diff --git a/src/ray/rpc/retryable_grpc_client.h b/src/ray/rpc/retryable_grpc_client.h index 6e7558d118fb..270f101f83c4 100644 --- a/src/ray/rpc/retryable_grpc_client.h +++ b/src/ray/rpc/retryable_grpc_client.h @@ -31,18 +31,35 @@ namespace ray::rpc { +// This macro wraps the logic to call a specific RPC method of a service with the +// retryable grpc client, to make it easier to implement a new RPC client. +#define INVOKE_RETRYABLE_RPC_CALL(retryable_rpc_client, \ + SERVICE, \ + METHOD, \ + request, \ + callback, \ + rpc_client, \ + method_timeout_ms) \ + (retryable_rpc_client->CallMethod( \ + &SERVICE::Stub::PrepareAsync##METHOD, \ + rpc_client, \ + #SERVICE ".grpc_client." #METHOD, \ + std::move(request), \ + callback, \ + method_timeout_ms)) + // Define a void retryable RPC client method. #define VOID_RETRYABLE_RPC_CLIENT_METHOD( \ retryable_rpc_client, SERVICE, METHOD, rpc_client, method_timeout_ms, SPECS) \ void METHOD(METHOD##Request &&request, const ClientCallback &callback) \ SPECS { \ - retryable_rpc_client->CallMethod( \ - &SERVICE::Stub::PrepareAsync##METHOD, \ - rpc_client, \ - #SERVICE ".grpc_client." #METHOD, \ - std::move(request), \ - callback, \ - method_timeout_ms); \ + INVOKE_RETRYABLE_RPC_CALL(retryable_rpc_client, \ + SERVICE, \ + METHOD, \ + request, \ + callback, \ + rpc_client, \ + method_timeout_ms); \ } /** diff --git a/src/ray/rpc/worker/core_worker_client_pool.h b/src/ray/rpc/worker/core_worker_client_pool.h index cbbf4cc3d07c..b4008797fd26 100644 --- a/src/ray/rpc/worker/core_worker_client_pool.h +++ b/src/ray/rpc/worker/core_worker_client_pool.h @@ -24,8 +24,8 @@ #include "absl/synchronization/mutex.h" #include "ray/common/id.h" #include "ray/gcs/gcs_client/gcs_client.h" -#include "ray/raylet_client/raylet_client_interface.h" -#include "ray/raylet_client/raylet_client_pool.h" +#include "ray/rpc/raylet/raylet_client_interface.h" +#include "ray/rpc/raylet/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client.h" namespace ray { From d25f4ead3f91557b93fb9007c8883510a648ec61 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Sun, 7 Sep 2025 14:43:38 -0700 Subject: [PATCH 494/634] [core] Delete unnecessary plasma.cc + macro (#56308) Signed-off-by: dayshah --- src/ray/object_manager/object_buffer_pool.h | 3 +- src/ray/object_manager/plasma/BUILD.bazel | 3 -- src/ray/object_manager/plasma/common.h | 15 +++++----- src/ray/object_manager/plasma/plasma.cc | 28 ------------------- src/ray/object_manager/plasma/plasma.h | 10 ++----- src/ray/object_manager/plasma/shared_memory.h | 5 ++-- src/ray/util/macros.h | 7 ----- src/ray/util/subreaper.h | 7 +++-- 8 files changed, 19 insertions(+), 59 deletions(-) delete mode 100644 src/ray/object_manager/plasma/plasma.cc diff --git a/src/ray/object_manager/object_buffer_pool.h b/src/ray/object_manager/object_buffer_pool.h index 1a5345520120..108af01340bc 100644 --- a/src/ray/object_manager/object_buffer_pool.h +++ b/src/ray/object_manager/object_buffer_pool.h @@ -63,7 +63,8 @@ class ObjectBufferPool { ~ObjectBufferPool(); /// This object cannot be copied due to pool_mutex. - RAY_DISALLOW_COPY_AND_ASSIGN(ObjectBufferPool); + ObjectBufferPool(const ObjectBufferPool &) = delete; + ObjectBufferPool &operator=(const ObjectBufferPool &) = delete; /// Computes the number of chunks needed to transfer an object and its metadata. /// diff --git a/src/ray/object_manager/plasma/BUILD.bazel b/src/ray/object_manager/plasma/BUILD.bazel index e3c41627e1ec..6efca1100e7a 100644 --- a/src/ray/object_manager/plasma/BUILD.bazel +++ b/src/ray/object_manager/plasma/BUILD.bazel @@ -242,7 +242,6 @@ ray_cc_library( ray_cc_library( name = "object_manager_plasma_common", - srcs = ["plasma.cc"], hdrs = [ "common.h", "plasma.h", @@ -252,8 +251,6 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/object_manager:object_manager_common", "//src/ray/util:compat", - "//src/ray/util:macros", - "@boost//:asio", "@com_google_googletest//:gtest_prod", ], ) diff --git a/src/ray/object_manager/plasma/common.h b/src/ray/object_manager/plasma/common.h index 3796905b5d72..669aad1b7c3c 100644 --- a/src/ray/object_manager/plasma/common.h +++ b/src/ray/object_manager/plasma/common.h @@ -18,11 +18,8 @@ #pragma once #include -#include -#include -#include -#include +#include #include #include "ray/common/id.h" @@ -30,7 +27,6 @@ #include "ray/object_manager/plasma/plasma.h" #include "ray/object_manager/plasma/plasma_generated.h" #include "ray/util/compat.h" -#include "ray/util/macros.h" namespace plasma { @@ -73,7 +69,8 @@ struct Allocation { bool fallback_allocated_; // only allow moves. - RAY_DISALLOW_COPY_AND_ASSIGN(Allocation); + Allocation(const Allocation &) = delete; + Allocation &operator=(const Allocation &) = delete; Allocation(Allocation &&) noexcept = default; Allocation &operator=(Allocation &&) noexcept = default; @@ -116,9 +113,11 @@ struct Allocation { /// the eviction policy. class LocalObject { public: - explicit LocalObject(Allocation allocation); + explicit LocalObject(Allocation allocation) + : allocation_(std::move(allocation)), ref_count_(0) {} - RAY_DISALLOW_COPY_AND_ASSIGN(LocalObject); + LocalObject(const LocalObject &) = delete; + LocalObject &operator=(const LocalObject &) = delete; int64_t GetObjectSize() const { return object_info_.GetObjectSize(); } diff --git a/src/ray/object_manager/plasma/plasma.cc b/src/ray/object_manager/plasma/plasma.cc deleted file mode 100644 index 04bf03779d94..000000000000 --- a/src/ray/object_manager/plasma/plasma.cc +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "ray/object_manager/plasma/plasma.h" - -#include - -#include "ray/object_manager/plasma/common.h" - -namespace plasma { - -LocalObject::LocalObject(Allocation allocation) - : allocation_(std::move(allocation)), ref_count_(0) {} -} // namespace plasma diff --git a/src/ray/object_manager/plasma/plasma.h b/src/ray/object_manager/plasma/plasma.h index 6b2eecbf805f..3f162396dc90 100644 --- a/src/ray/object_manager/plasma/plasma.h +++ b/src/ray/object_manager/plasma/plasma.h @@ -17,14 +17,8 @@ #pragma once -#include -#include - -#include -#include -#include -#include -#include +#include +#include #include "ray/util/compat.h" diff --git a/src/ray/object_manager/plasma/shared_memory.h b/src/ray/object_manager/plasma/shared_memory.h index 8d597d538e1a..6623f25970b9 100644 --- a/src/ray/object_manager/plasma/shared_memory.h +++ b/src/ray/object_manager/plasma/shared_memory.h @@ -27,6 +27,9 @@ class ClientMmapTableEntry { public: ClientMmapTableEntry(MEMFD_TYPE fd, int64_t map_size); + ClientMmapTableEntry(const ClientMmapTableEntry &) = delete; + ClientMmapTableEntry &operator=(const ClientMmapTableEntry &) = delete; + ~ClientMmapTableEntry(); uint8_t *pointer() const { return reinterpret_cast(pointer_); } @@ -42,8 +45,6 @@ class ClientMmapTableEntry { size_t length_; void MaybeMadviseDontdump(); - - RAY_DISALLOW_COPY_AND_ASSIGN(ClientMmapTableEntry); }; } // namespace plasma diff --git a/src/ray/util/macros.h b/src/ray/util/macros.h index 0a81b92bc230..5e111c43fcb3 100644 --- a/src/ray/util/macros.h +++ b/src/ray/util/macros.h @@ -14,13 +14,6 @@ #pragma once -// From Google gutil -#ifndef RAY_DISALLOW_COPY_AND_ASSIGN -#define RAY_DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(const TypeName &) = delete; \ - void operator=(const TypeName &) = delete -#endif - #define RAY_UNUSED(x) (void)x // diff --git a/src/ray/util/subreaper.h b/src/ray/util/subreaper.h index ab3f060f00a9..94047a751373 100644 --- a/src/ray/util/subreaper.h +++ b/src/ray/util/subreaper.h @@ -105,10 +105,13 @@ class KnownChildrenTracker { std::vector ListUnknownChildren( std::function()> list_pids_fn); + KnownChildrenTracker(const KnownChildrenTracker &) = delete; + KnownChildrenTracker &operator=(const KnownChildrenTracker &) = delete; + + ~KnownChildrenTracker() = default; + private: KnownChildrenTracker() = default; - ~KnownChildrenTracker() = default; - RAY_DISALLOW_COPY_AND_ASSIGN(KnownChildrenTracker); bool enabled_ = false; absl::Mutex m_; From 32b778e51494b311bd82d0905c76bb7e7b2e7210 Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Mon, 8 Sep 2025 18:47:29 +0200 Subject: [PATCH 495/634] [serve] Require prefix `RAY_SERVE_` for env vars + value verification (#55864) ## Why are these changes needed? - Require prefix `RAY_SERVE_` for all environment variable names in the `serve` module (excluding existent). - Introduce deprecation warning and replacement for existing environment variable names without `RAY_SERVE_` prefix (deprecation in the next major `3.0.0` release): - the warning is logged only for explicitly defined variable - replacement contains `RAY_SERVE_` prefix (e.g. `RAY_SERVE_MAX_PER_REPLICA_RETRY_COUNT` for `MAX_PER_REPLICA_RETRY_COUNT`) Commands: ```bash ray start --head serve config # wrong name warning export MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT=1 export MAX_PER_REPLICA_RETRY_COUNT=1 export REQUEST_LATENCY_BUCKETS_MS=1 export MODEL_LOAD_LATENCY_BUCKETS_MS=1 export MAX_CACHED_HANDLES=1 export CONTROLLER_MAX_CONCURRENCY=1 export SERVE_REQUEST_PROCESSING_TIMEOUT_S=1 export RAY_SERVE_HANDLE_METRIC_PUSH_INTERVAL_S=1 ``` Output: ```bash bash$ serve config ~/ray/python/ray/serve/_private/constants.py:60: FutureWarning: Starting from version `3.0.0` environment variable `MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT` will be deprecated. Please use `RAY_SERVE_MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT` instead. get_env_int("MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT", 20), ~/ray/python/ray/serve/_private/constants.py:67: FutureWarning: Starting from version `3.0.0` environment variable `MAX_PER_REPLICA_RETRY_COUNT` will be deprecated. Please use `RAY_SERVE_MAX_PER_REPLICA_RETRY_COUNT` instead. get_env_int("MAX_PER_REPLICA_RETRY_COUNT", 3), ~/ray/python/ray/serve/_private/constants.py:112: FutureWarning: Starting from version `3.0.0` environment variable `REQUEST_LATENCY_BUCKETS_MS` will be deprecated. Please use `RAY_SERVE_REQUEST_LATENCY_BUCKETS_MS` instead. get_env_str("REQUEST_LATENCY_BUCKETS_MS", ""), ~/ray/python/ray/serve/_private/constants.py:120: FutureWarning: Starting from version `3.0.0` environment variable `MODEL_LOAD_LATENCY_BUCKETS_MS` will be deprecated. Please use `RAY_SERVE_MODEL_LOAD_LATENCY_BUCKETS_MS` instead. get_env_str("MODEL_LOAD_LATENCY_BUCKETS_MS", ""), ~/ray/python/ray/serve/_private/constants.py:136: FutureWarning: Starting from version `3.0.0` environment variable `MAX_CACHED_HANDLES` will be deprecated. Please use `RAY_SERVE_MAX_CACHED_HANDLES` instead. "RAY_SERVE_MAX_CACHED_HANDLES", get_env_int_positive("MAX_CACHED_HANDLES", 100) ~/ray/python/ray/serve/_private/constants.py:143: FutureWarning: Starting from version `3.0.0` environment variable `CONTROLLER_MAX_CONCURRENCY` will be deprecated. Please use `RAY_SERVE_CONTROLLER_MAX_CONCURRENCY` instead. get_env_int_positive("CONTROLLER_MAX_CONCURRENCY", 15_000), ~/ray/python/ray/serve/_private/constants.py:255: FutureWarning: Starting from version `3.0.0` environment variable `SERVE_REQUEST_PROCESSING_TIMEOUT_S` will be deprecated. Please use `RAY_SERVE_REQUEST_PROCESSING_TIMEOUT_S` instead. get_env_float_non_negative("SERVE_REQUEST_PROCESSING_TIMEOUT_S", 0.0), ~/ray/python/ray/serve/_private/constants.py:306: FutureWarning: Starting from version `3.0.0` environment variable `RAY_SERVE_HANDLE_METRIC_PUSH_INTERVAL_S` will be deprecated. Please use `RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S` instead. get_env_float("RAY_SERVE_HANDLE_METRIC_PUSH_INTERVAL_S", 10.0), ``` - Update `get_env_str` and `get_env_bool` to use internal `_get_env_value` + update type hints -- - Replace `get_env_float_non_zero_with_warning` with `get_env_float_positive` for the `2.50.0` release. See PR #55464 for details. Now the following environment variables require positive value: ``` RAY_SERVE_PROXY_HEALTH_CHECK_TIMEOUT_S RAY_SERVE_PROXY_HEALTH_CHECK_PERIOD_S RAY_SERVE_PROXY_READY_CHECK_TIMEOUT_S RAY_SERVE_PROXY_MIN_DRAINING_PERIOD_S RAY_SERVE_KV_TIMEOUT_S ``` - Require non negative value for the following environment variables: ``` RAY_SERVE_HTTP_KEEP_ALIVE_TIMEOUT_S RAY_SERVE_REQUEST_PROCESSING_TIMEOUT_S RAY_SERVE_MULTIPLEXED_MODEL_ID_MATCHING_TIMEOUT_S RAY_SERVE_QUEUE_LENGTH_CACHE_TIMEOUT_S RAY_SERVE_MIN_HANDLE_METRICS_TIMEOUT_S ``` -- For this change, the following could be added to `2.50.0` **release notes**: Ray Serve: Enhancements: Health check & env var safety: - Introduced `RAY_SERVE_` required prefix for environment variables. - Warnings for invalid environment variable names, with migration path planned for Ray `3.0.0`. - Added verification for timeout environment variables to be positive or non negative. ## Related issue number Closes #55453 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: axreldable --- python/ray/serve/_private/constants.py | 63 +++-- python/ray/serve/_private/constants_utils.py | 105 +++++--- .../serve/tests/unit/test_constants_utils.py | 225 +++++++----------- 3 files changed, 200 insertions(+), 193 deletions(-) diff --git a/python/ray/serve/_private/constants.py b/python/ray/serve/_private/constants.py index 168fcb718ecf..774aee0c86f8 100644 --- a/python/ray/serve/_private/constants.py +++ b/python/ray/serve/_private/constants.py @@ -4,8 +4,9 @@ get_env_bool, get_env_float, get_env_float_non_negative, - get_env_float_non_zero_with_warning, + get_env_float_positive, get_env_int, + get_env_int_non_negative, get_env_int_positive, get_env_str, parse_latency_buckets, @@ -52,15 +53,20 @@ #: Max retry count for allowing failures in replica constructor. #: If no replicas at target version is running by the time we're at -#: max construtor retry count, deploy() is considered failed. +#: max constructor retry count, deploy() is considered failed. #: By default we set threshold as min(num_replicas * 3, this value) MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT = get_env_int( - "MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT", 20 + "RAY_SERVE_MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT", + get_env_int("MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT", 20), ) # Max retry on deployment constructor is # min(num_replicas * MAX_PER_REPLICA_RETRY_COUNT, MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT) -MAX_PER_REPLICA_RETRY_COUNT = get_env_int("MAX_PER_REPLICA_RETRY_COUNT", 3) +MAX_PER_REPLICA_RETRY_COUNT = get_env_int( + "RAY_SERVE_MAX_PER_REPLICA_RETRY_COUNT", + get_env_int("MAX_PER_REPLICA_RETRY_COUNT", 3), +) + # If you are wondering why we are using histogram buckets, please refer to # https://prometheus.io/docs/practices/histograms/ @@ -101,11 +107,19 @@ # RAY_SERVE_MODEL_LOAD_LATENCY_BUCKET_MS="1,2,3,4" #: Histogram buckets for request latency. REQUEST_LATENCY_BUCKETS_MS = parse_latency_buckets( - get_env_str("REQUEST_LATENCY_BUCKETS_MS", ""), DEFAULT_LATENCY_BUCKET_MS + get_env_str( + "RAY_SERVE_REQUEST_LATENCY_BUCKETS_MS", + get_env_str("REQUEST_LATENCY_BUCKETS_MS", ""), + ), + DEFAULT_LATENCY_BUCKET_MS, ) #: Histogram buckets for model load/unload latency. MODEL_LOAD_LATENCY_BUCKETS_MS = parse_latency_buckets( - get_env_str("MODEL_LOAD_LATENCY_BUCKETS_MS", ""), DEFAULT_LATENCY_BUCKET_MS + get_env_str( + "RAY_SERVE_MODEL_LOAD_LATENCY_BUCKETS_MS", + get_env_str("MODEL_LOAD_LATENCY_BUCKETS_MS", ""), + ), + DEFAULT_LATENCY_BUCKET_MS, ) #: Name of deployment health check method implemented by user. @@ -118,11 +132,16 @@ #: Limit the number of cached handles because each handle has long poll #: overhead. See https://github.com/ray-project/ray/issues/18980 -MAX_CACHED_HANDLES = get_env_int_positive("MAX_CACHED_HANDLES", 100) +MAX_CACHED_HANDLES = get_env_int_positive( + "RAY_SERVE_MAX_CACHED_HANDLES", get_env_int_positive("MAX_CACHED_HANDLES", 100) +) #: Because ServeController will accept one long poll request per handle, its #: concurrency needs to scale as O(num_handles) -CONTROLLER_MAX_CONCURRENCY = get_env_int_positive("CONTROLLER_MAX_CONCURRENCY", 15_000) +CONTROLLER_MAX_CONCURRENCY = get_env_int_positive( + "RAY_SERVE_CONTROLLER_MAX_CONCURRENCY", + get_env_int_positive("CONTROLLER_MAX_CONCURRENCY", 15_000), +) DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT_S = 20 DEFAULT_GRACEFUL_SHUTDOWN_WAIT_LOOP_S = 2 @@ -132,14 +151,14 @@ DEFAULT_TARGET_ONGOING_REQUESTS = 2 # HTTP Proxy health check configs -PROXY_HEALTH_CHECK_TIMEOUT_S = get_env_float_non_zero_with_warning( +PROXY_HEALTH_CHECK_TIMEOUT_S = get_env_float_positive( "RAY_SERVE_PROXY_HEALTH_CHECK_TIMEOUT_S", 10.0 ) -PROXY_HEALTH_CHECK_PERIOD_S = get_env_float_non_zero_with_warning( +PROXY_HEALTH_CHECK_PERIOD_S = get_env_float_positive( "RAY_SERVE_PROXY_HEALTH_CHECK_PERIOD_S", 10.0 ) -PROXY_READY_CHECK_TIMEOUT_S = get_env_float_non_zero_with_warning( +PROXY_READY_CHECK_TIMEOUT_S = get_env_float_positive( "RAY_SERVE_PROXY_READY_CHECK_TIMEOUT_S", 5.0 ) @@ -148,7 +167,7 @@ PROXY_HEALTH_CHECK_UNHEALTHY_THRESHOLD = 3 # The minimum drain period for a HTTP proxy. -PROXY_MIN_DRAINING_PERIOD_S = get_env_float_non_zero_with_warning( +PROXY_MIN_DRAINING_PERIOD_S = get_env_float_positive( "RAY_SERVE_PROXY_MIN_DRAINING_PERIOD_S", 30.0 ) # The time in seconds that the http proxy state waits before @@ -167,9 +186,7 @@ CLIENT_CHECK_CREATION_POLLING_INTERVAL_S = 0.1 # Timeout for GCS internal KV service -RAY_SERVE_KV_TIMEOUT_S = get_env_float_non_zero_with_warning( - "RAY_SERVE_KV_TIMEOUT_S", None -) +RAY_SERVE_KV_TIMEOUT_S = get_env_float_positive("RAY_SERVE_KV_TIMEOUT_S", None) # Timeout for GCS RPC request RAY_GCS_RPC_TIMEOUT_S = 3.0 @@ -228,13 +245,15 @@ "skip_context_filter", } -RAY_SERVE_HTTP_KEEP_ALIVE_TIMEOUT_S = get_env_int( +RAY_SERVE_HTTP_KEEP_ALIVE_TIMEOUT_S = get_env_int_non_negative( "RAY_SERVE_HTTP_KEEP_ALIVE_TIMEOUT_S", 0 ) RAY_SERVE_REQUEST_PROCESSING_TIMEOUT_S = ( - get_env_float("RAY_SERVE_REQUEST_PROCESSING_TIMEOUT_S", 0.0) - or get_env_float("SERVE_REQUEST_PROCESSING_TIMEOUT_S", 0.0) + get_env_float_non_negative( + "RAY_SERVE_REQUEST_PROCESSING_TIMEOUT_S", + get_env_float_non_negative("SERVE_REQUEST_PROCESSING_TIMEOUT_S", 0.0), + ) or None ) @@ -289,11 +308,11 @@ # Serve multiplexed matching timeout. # This is the timeout for the matching process of multiplexed requests. To avoid -# thundering herd problem, the timeout value will be randomed between this value +# thundering herd problem, the timeout value will be randomized between this value # and this value * 2. The unit is second. # If the matching process takes longer than the timeout, the request will be # fallen to the default routing strategy. -RAY_SERVE_MULTIPLEXED_MODEL_ID_MATCHING_TIMEOUT_S = get_env_float( +RAY_SERVE_MULTIPLEXED_MODEL_ID_MATCHING_TIMEOUT_S = get_env_float_non_negative( "RAY_SERVE_MULTIPLEXED_MODEL_ID_MATCHING_TIMEOUT_S", 1.0 ) @@ -328,7 +347,7 @@ ) # Length of time to respect entries in the queue length cache when routing requests. -RAY_SERVE_QUEUE_LENGTH_CACHE_TIMEOUT_S = get_env_float( +RAY_SERVE_QUEUE_LENGTH_CACHE_TIMEOUT_S = get_env_float_non_negative( "RAY_SERVE_QUEUE_LENGTH_CACHE_TIMEOUT_S", 10.0 ) @@ -356,7 +375,7 @@ "RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE", "1" ) -RAY_SERVE_MIN_HANDLE_METRICS_TIMEOUT_S = get_env_float( +RAY_SERVE_MIN_HANDLE_METRICS_TIMEOUT_S = get_env_float_non_negative( "RAY_SERVE_MIN_HANDLE_METRICS_TIMEOUT_S", 10.0 ) diff --git a/python/ray/serve/_private/constants_utils.py b/python/ray/serve/_private/constants_utils.py index 0ac97ab3eabd..019df87d86b8 100644 --- a/python/ray/serve/_private/constants_utils.py +++ b/python/ray/serve/_private/constants_utils.py @@ -47,13 +47,38 @@ def parse_latency_buckets(bucket_str: str, default_buckets: List[float]) -> List T = TypeVar("T") +# todo: remove for the '3.0.0' release. +_wrong_names_white_list = { + "MAX_DEPLOYMENT_CONSTRUCTOR_RETRY_COUNT", + "MAX_PER_REPLICA_RETRY_COUNT", + "REQUEST_LATENCY_BUCKETS_MS", + "MODEL_LOAD_LATENCY_BUCKETS_MS", + "MAX_CACHED_HANDLES", + "CONTROLLER_MAX_CONCURRENCY", + "SERVE_REQUEST_PROCESSING_TIMEOUT_S", +} + + +def _validate_name(name: str) -> None: + """Validate Ray Serve environment variable name.""" + required_prefix = "RAY_SERVE_" + + if not name.startswith(required_prefix): + if name in _wrong_names_white_list: + return + + raise ValueError( + f"Got unexpected environment variable name `{name}`! " + f"Ray Serve environment variables require prefix `{required_prefix}`. " + ) + def _get_env_value( name: str, default: Optional[T], value_type: Type[T], validation_func: Optional[Callable[[T], bool]] = None, - expected_value_description: str = None, + expected_value_description: Optional[str] = None, ) -> Optional[T]: """Get environment variable with type conversion and validation. @@ -63,12 +88,13 @@ def _get_env_value( Args: name: The name of the environment variable. default: Default value to use if the environment variable is not set. - If None, the function will return None without validation. + If None, the function will return None without validation. value_type: Type to convert the environment variable value to (e.g., int, float, str). validation_func: Optional function that takes the converted value and returns - a boolean indicating whether the value is valid. + a boolean indicating whether the value is valid. expected_value_description: Description of the expected value characteristics - (e.g., "positive", "non negative") used in error messages. + (e.g., "positive", "non-negative") used in error messages. + Optional, expected only if validation_func is provided. Returns: The environment variable value converted to the specified type and validated, @@ -76,11 +102,19 @@ def _get_env_value( Raises: ValueError: If the environment variable value cannot be converted to the specified - type, or if it fails the optional validation check. + type, or if it fails the optional validation check. Also, if name validation fails. """ - raw = os.environ.get(name, default) - if raw is None: - return None + _validate_name(name) + + explicitly_defined_value = os.environ.get(name) + if explicitly_defined_value is None: + if default is None: + return None + else: + raw = default + else: + _deprecation_warning(name) + raw = explicitly_defined_value try: value = value_type(raw) @@ -194,7 +228,7 @@ def get_env_float_non_negative(name: str, default: Optional[float]) -> Optional[ return _get_env_value(name, default, float, lambda x: x >= 0, "non negative") -def get_env_str(name: str, default: Optional[str]) -> str: +def get_env_str(name: str, default: Optional[str]) -> Optional[str]: """Get environment variable as a string. Args: @@ -203,11 +237,12 @@ def get_env_str(name: str, default: Optional[str]) -> str: Returns: The environment variable value as a string. + Returns `None` if default is `None` and value not found. """ - return os.environ.get(name, default) + return _get_env_value(name, default, str) -def get_env_bool(name: str, default: Optional[str]) -> bool: +def get_env_bool(name: str, default: str) -> bool: """Get environment variable as a boolean. Environment variable values of "1" are interpreted as True, all others as False. @@ -215,38 +250,42 @@ def get_env_bool(name: str, default: Optional[str]) -> bool: Args: name: The name of the environment variable. default: Default value to use if the environment variable is not set. + Expects "0" or "1". Returns: True if the environment variable value is "1", False otherwise. """ - return os.environ.get(name, default) == "1" + env_value_str = _get_env_value(name, default, str) + return env_value_str == "1" -def get_env_float_non_zero_with_warning( - name: str, default: Optional[float] -) -> Optional[float]: - """Introduced for backward compatibility for constants: +def _deprecation_warning(name: str) -> None: + """Log replacement warning for wrong or legacy environment variables. - PROXY_HEALTH_CHECK_TIMEOUT_S - PROXY_HEALTH_CHECK_PERIOD_S - PROXY_READY_CHECK_TIMEOUT_S - PROXY_MIN_DRAINING_PERIOD_S - RAY_SERVE_KV_TIMEOUT_S + TODO: remove this function for the '3.0.0' release. - todo: replace this function with 'get_env_float_positive' for the '2.50.0' release. + :param name: environment variable name """ - removal_version = "2.50.0" - - env_value = get_env_float(name, default) - backward_compatible_result = env_value or default - if env_value is not None and env_value <= 0: - # warning message if unexpected value + def get_new_name(name: str) -> str: + if name == "RAY_SERVE_HANDLE_METRIC_PUSH_INTERVAL_S": + return "RAY_SERVE_HANDLE_AUTOSCALING_METRIC_PUSH_INTERVAL_S" + elif name == "SERVE_REQUEST_PROCESSING_TIMEOUT_S": + return "RAY_SERVE_REQUEST_PROCESSING_TIMEOUT_S" + else: + return f"{required_prefix}{name}" + + change_version = "3.0.0" + required_prefix = "RAY_SERVE_" + + if ( + name in _wrong_names_white_list + or name == "RAY_SERVE_HANDLE_METRIC_PUSH_INTERVAL_S" + ): + new_name = get_new_name(name) warnings.warn( - f"Got unexpected value `{env_value}` for `{name}` environment variable! " - f"Starting from version `{removal_version}`, the environment variable will require a positive value. " - f"Setting `{name}` to `{backward_compatible_result}`. ", + f"Starting from version `{change_version}` environment variable " + f"`{name}` will be deprecated. Please use `{new_name}` instead.", FutureWarning, - stacklevel=2, + stacklevel=4, ) - return backward_compatible_result diff --git a/python/ray/serve/tests/unit/test_constants_utils.py b/python/ray/serve/tests/unit/test_constants_utils.py index 96bcbcbd05e6..e5e2754b5190 100644 --- a/python/ray/serve/tests/unit/test_constants_utils.py +++ b/python/ray/serve/tests/unit/test_constants_utils.py @@ -2,13 +2,12 @@ from unittest.mock import patch import pytest -from testfixtures import mock from ray.serve._private.constants_utils import ( + _validate_name, get_env_bool, get_env_float, get_env_float_non_negative, - get_env_float_non_zero_with_warning, get_env_float_positive, get_env_int, get_env_int_non_negative, @@ -97,192 +96,142 @@ def mock_environ(): class TestEnvValueFunctions: def test_get_env_int(self, mock_environ): - assert get_env_int("TEST_VAR", 0) == 0 + assert get_env_int("RAY_SERVE_TEST_VAR", 0) == 0 - mock_environ["TEST_VAR"] = "42" - assert get_env_int("TEST_VAR", 0) == 42 + mock_environ["RAY_SERVE_TEST_VAR"] = "42" + assert get_env_int("RAY_SERVE_TEST_VAR", 0) == 42 - mock_environ["TEST_VAR"] = "-1" - assert get_env_int("TEST_VAR", 0) == -1 + mock_environ["RAY_SERVE_TEST_VAR"] = "-1" + assert get_env_int("RAY_SERVE_TEST_VAR", 0) == -1 - mock_environ["TEST_VAR"] = "0.1" + mock_environ["RAY_SERVE_TEST_VAR"] = "0.1" with pytest.raises(ValueError, match=".*`0.1` cannot be converted to `int`!*"): - get_env_int_positive("TEST_VAR", 5) + get_env_int_positive("RAY_SERVE_TEST_VAR", 5) - mock_environ["TEST_VAR"] = "abc" + mock_environ["RAY_SERVE_TEST_VAR"] = "abc" with pytest.raises(ValueError, match=".*`abc` cannot be converted to `int`!*"): - get_env_int_positive("TEST_VAR", 5) + get_env_int_positive("RAY_SERVE_TEST_VAR", 5) + + with pytest.raises(ValueError, match=".*require prefix `RAY_SERVE_`*"): + get_env_int_positive("NO_PREFIX", 5) def test_get_env_int_positive(self, mock_environ): - assert get_env_int_positive("TEST_VAR", 1) == 1 + assert get_env_int_positive("RAY_SERVE_TEST_VAR", 1) == 1 - mock_environ["TEST_VAR"] = "42" - assert get_env_int_positive("TEST_VAR", 1) == 42 + mock_environ["RAY_SERVE_TEST_VAR"] = "42" + assert get_env_int_positive("RAY_SERVE_TEST_VAR", 1) == 42 - mock_environ["TEST_VAR"] = "-1" + mock_environ["RAY_SERVE_TEST_VAR"] = "-1" with pytest.raises(ValueError, match=".*Expected positive `int`.*"): - get_env_int_positive("TEST_VAR", 5) + get_env_int_positive("RAY_SERVE_TEST_VAR", 5) def test_get_env_int_non_negative(self, mock_environ): - assert get_env_int_non_negative("TEST_VAR", 0) == 0 - assert get_env_int_non_negative("TEST_VAR", 1) == 1 + assert get_env_int_non_negative("RAY_SERVE_TEST_VAR", 0) == 0 + assert get_env_int_non_negative("RAY_SERVE_TEST_VAR", 1) == 1 - mock_environ["TEST_VAR"] = "42" - assert get_env_int_non_negative("TEST_VAR", 0) == 42 + mock_environ["RAY_SERVE_TEST_VAR"] = "42" + assert get_env_int_non_negative("RAY_SERVE_TEST_VAR", 0) == 42 - mock_environ["TEST_VAR"] = "-1" + mock_environ["RAY_SERVE_TEST_VAR"] = "-1" with pytest.raises(ValueError, match=".*Expected non negative `int`.*"): - get_env_int_non_negative("TEST_VAR", 5) + get_env_int_non_negative("RAY_SERVE_TEST_VAR", 5) with pytest.raises(ValueError, match=".*Expected non negative `int`.*"): - get_env_int_non_negative("TEST_VAR_FROM_DEFAULT", -1) + get_env_int_non_negative("RAY_SERVE_TEST_VAR_FROM_DEFAULT", -1) def test_get_env_float(self, mock_environ): - assert get_env_float("TEST_VAR", 0.0) == 0.0 + assert get_env_float("RAY_SERVE_TEST_VAR", 0.0) == 0.0 - mock_environ["TEST_VAR"] = "3.14" - assert get_env_float("TEST_VAR", 0.0) == 3.14 + mock_environ["RAY_SERVE_TEST_VAR"] = "3.14" + assert get_env_float("RAY_SERVE_TEST_VAR", 0.0) == 3.14 - mock_environ["TEST_VAR"] = "-2.5" - assert get_env_float("TEST_VAR", 0.0) == -2.5 + mock_environ["RAY_SERVE_TEST_VAR"] = "-2.5" + assert get_env_float("RAY_SERVE_TEST_VAR", 0.0) == -2.5 - mock_environ["TEST_VAR"] = "abc" + mock_environ["RAY_SERVE_TEST_VAR"] = "abc" with pytest.raises( ValueError, match=".*`abc` cannot be converted to `float`!*" ): - get_env_float("TEST_VAR", 0.0) + get_env_float("RAY_SERVE_TEST_VAR", 0.0) def test_get_env_float_positive(self, mock_environ): - assert get_env_float_positive("TEST_VAR", 1.5) == 1.5 - assert get_env_float_positive("TEST_VAR", None) is None + assert get_env_float_positive("RAY_SERVE_TEST_VAR", 1.5) == 1.5 + assert get_env_float_positive("RAY_SERVE_TEST_VAR", None) is None - mock_environ["TEST_VAR"] = "42.5" - assert get_env_float_positive("TEST_VAR", 1.0) == 42.5 + mock_environ["RAY_SERVE_TEST_VAR"] = "42.5" + assert get_env_float_positive("RAY_SERVE_TEST_VAR", 1.0) == 42.5 - mock_environ["TEST_VAR"] = "-1.2" + mock_environ["RAY_SERVE_TEST_VAR"] = "-1.2" with pytest.raises(ValueError, match=".*Expected positive `float`.*"): - get_env_float_positive("TEST_VAR", 5.0) + get_env_float_positive("RAY_SERVE_TEST_VAR", 5.0) with pytest.raises(ValueError, match=".*Expected positive `float`.*"): - get_env_float_positive("TEST_VAR_FROM_DEFAULT", 0.0) + get_env_float_positive("RAY_SERVE_TEST_VAR_FROM_DEFAULT", 0.0) with pytest.raises(ValueError, match=".*Expected positive `float`.*"): - get_env_float_positive("TEST_VAR_FROM_DEFAULT", -1) + get_env_float_positive("RAY_SERVE_TEST_VAR_FROM_DEFAULT", -1) def test_get_env_float_non_negative(self, mock_environ): - assert get_env_float_non_negative("TEST_VAR", 0.0) == 0.0 - assert get_env_float_non_negative("TEST_VAR", 1.5) == 1.5 + assert get_env_float_non_negative("RAY_SERVE_TEST_VAR", 0.0) == 0.0 + assert get_env_float_non_negative("RAY_SERVE_TEST_VAR", 1.5) == 1.5 - mock_environ["TEST_VAR"] = "42.5" - assert get_env_float_non_negative("TEST_VAR", 0.0) == 42.5 + mock_environ["RAY_SERVE_TEST_VAR"] = "42.5" + assert get_env_float_non_negative("RAY_SERVE_TEST_VAR", 0.0) == 42.5 - mock_environ["TEST_VAR"] = "-1.2" + mock_environ["RAY_SERVE_TEST_VAR"] = "-1.2" with pytest.raises(ValueError, match=".*Expected non negative `float`.*"): - get_env_float_non_negative("TEST_VAR", 5.0) + get_env_float_non_negative("RAY_SERVE_TEST_VAR", 5.0) def test_get_env_str(self, mock_environ): - mock_environ["TEST_STR"] = "hello" - assert get_env_str("TEST_STR", "default") == "hello" + mock_environ["RAY_SERVE_TEST_STR"] = "hello" + assert get_env_str("RAY_SERVE_TEST_STR", "default") == "hello" - assert get_env_str("NONEXISTENT_VAR", "default_str") == "default_str" + assert get_env_str("RAY_SERVE_NONEXISTENT_VAR", "default_str") == "default_str" - assert get_env_str("NONEXISTENT_VAR", None) is None + assert get_env_str("RAY_SERVE_NONEXISTENT_VAR", None) is None def test_get_env_bool(self, mock_environ): - mock_environ["TEST_BOOL_TRUE"] = "1" - assert get_env_bool("TEST_BOOL_TRUE", "0") is True + mock_environ["RAY_SERVE_TEST_BOOL_TRUE"] = "1" + assert get_env_bool("RAY_SERVE_TEST_BOOL_TRUE", "0") is True # Test with any other value (False) - mock_environ["TEST_BOOL_FALSE"] = "true" - assert get_env_bool("TEST_BOOL_FALSE", "0") is False - mock_environ["TEST_BOOL_FALSE2"] = "yes" - assert get_env_bool("TEST_BOOL_FALSE2", "0") is False + mock_environ["RAY_SERVE_TEST_BOOL_FALSE"] = "true" + assert get_env_bool("RAY_SERVE_TEST_BOOL_FALSE", "0") is False + mock_environ["RAY_SERVE_TEST_BOOL_FALSE2"] = "yes" + assert get_env_bool("RAY_SERVE_TEST_BOOL_FALSE2", "0") is False # Test with default when environment variable not set - assert get_env_bool("NONEXISTENT_VAR", "1") is True - assert get_env_bool("NONEXISTENT_VAR", "0") is False - - -class TestDeprecationFunctions: - def test_current_behavior(self, mock_environ): - mock_environ["OLD_VAR_NEG"] = "-1" - assert get_env_float("OLD_VAR_NEG", 10.0) or 10.0 == -1.0 - assert (get_env_float("OLD_VAR_NEG", 0.0) or None) == -1.0 - - mock_environ["OLD_VAR_ZERO"] = "0" - assert get_env_float("OLD_VAR_ZERO", 10.0) or 10.0 == 10.0 - - assert get_env_float("NOT_SET", 10.0) or 10.0 == 10.0 - - assert (get_env_float("NOT_SET", 0.0) or None) is None - - @mock.patch("ray.__version__", "2.49.0") # Version before 2.50.0 - def test_with_positive_value_before_250(self, mock_environ): - env_name = "TEST_POSITIVE_FLOAT" - mock_environ[env_name] = "5.5" - - result = get_env_float_non_zero_with_warning(env_name, 10.0) - - assert result == 5.5 - - @mock.patch("ray.__version__", "2.49.0") # Version before 2.50.0 - def test_with_non_positive_value_before_250(self, mock_environ): - env_name = "TEST_NON_POSITIVE_FLOAT" - mock_environ[env_name] = "-2.5" - - with pytest.warns(FutureWarning) as record: - result = get_env_float_non_zero_with_warning(env_name, 10.0) - - assert result == -2.5 - assert len(record) == 1 - assert "will require a positive value" in str(record[0].message) - - @mock.patch("ray.__version__", "2.49.0") # Version before 2.50.0 - def test_with_zero_value_before_250(self, mock_environ): - env_name = "TEST_ZERO_FLOAT" - mock_environ[env_name] = "0.0" - - with pytest.warns(FutureWarning) as record: - result = get_env_float_non_zero_with_warning(env_name, 10.0) - - assert result == 10.0 - assert len(record) == 1 - assert "will require a positive value" in str(record[0].message) - - @mock.patch("ray.__version__", "2.49.0") # Version before 2.50.0 - def test_with_no_env_value_before_250(self): - env_name = "TEST_MISSING_FLOAT" - # Don't set environment variable - - result = get_env_float_non_zero_with_warning(env_name, 1.0) - - assert result == 1.0 - - @mock.patch("ray.__version__", "2.50.0") # Version at 2.50.0 - def test_remain_the_same_behavior_at_2_50(self, mock_environ): - env_name = "TEST_FLOAT" - mock_environ[env_name] = "2.0" - - assert get_env_float_non_zero_with_warning(env_name, 1.0) == 2.0 - - mock_environ["TEST_VAR"] = "-1.2" - assert get_env_float_non_zero_with_warning("TEST_VAR", 5.0) == -1.2 - - mock_environ["TEST_VAR"] = "0.0" - assert get_env_float_non_zero_with_warning("TEST_VAR", 5.0) == 5.0 - - @mock.patch("ray.__version__", "2.51.0") # Version after 2.50.0 - def test_remain_the_same_behavior_after_2_50(self, mock_environ): - env_name = "TEST_FLOAT" - mock_environ[env_name] = "2.0" - - assert get_env_float_non_zero_with_warning(env_name, 1.0) == 2.0 - - mock_environ["TEST_VAR"] = "-1.2" - assert get_env_float_non_zero_with_warning("TEST_VAR", 5.0) == -1.2 - - mock_environ["TEST_VAR"] = "0.0" - assert get_env_float_non_zero_with_warning("TEST_VAR", 5.0) == 5.0 + assert get_env_bool("RAY_SERVE_NONEXISTENT_VAR", "1") is True + assert get_env_bool("RAY_SERVE_NONEXISTENT_VAR", "0") is False + + +class TestValidation: + @pytest.mark.parametrize( + "name", + [ + "RAY_SERVE_FOO", + "RAY_SERVE__DOUBLE_UNDERSCORE", + "RAY_SERVE_123", + "RAY_SERVE_VAR_NAME", + ], + ) + def test_validate_name_accepts_valid_prefix(self, name): + # Should not raise + assert _validate_name(name) is None + + @pytest.mark.parametrize( + "name", + [ + "", + "RAY_SERVE", # missing trailing underscore and name + "SERVE_VAR", + "ray_SERVE_BAR", + "RAY_service_VAR", + ], + ) + def test_validate_name_rejects_invalid_prefix(self, name): + with pytest.raises(ValueError, match=".*require prefix `RAY_SERVE_`*"): + _validate_name(name) if __name__ == "__main__": From 87a17dd5eeaca0bc64507950f661841c82e3be43 Mon Sep 17 00:00:00 2001 From: Aydin Abiar <62435714+Aydin-ab@users.noreply.github.com> Date: Mon, 8 Sep 2025 10:09:23 -0700 Subject: [PATCH 496/634] [docs] serve llm deployment examples refinement (#56287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? While testing the examples as Anyscale templates, I found few issues here are the fixes: * **URLs**: Change `latest` links to `master` because the links are not available in `latest` yet Example: `…/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html` → `…/en/master/serve/tutorials/deployment-serve-llm/small-size-llm/README.html` * **Wording**: Standardized “small size / medium size / large size” → “small-sized / medium-sized / large-sized.” * **HF token**: Added `export HF_TOKEN` to small/medium/large tutorials for easier sequential runs; updated comments. * **Service config**: Updated `service.yaml` examples to use Anyscale base images (faster startup); Dockerfile remains as an option in comments. * **Curl examples**: Fixed commands so they run both in terminal and Jupyter cells. * **Vision LLM**: Added a local image example and fixed a relative path error. * **nitpicks**: Improved consistency across docs (e.g. “Example curl:” and “Example python:” formatting for example). ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [x] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Aydin Abiar Co-authored-by: Aydin Abiar --- .../deployment-serve-llm/README.ipynb | 12 +++++------ .../tutorials/deployment-serve-llm/README.md | 12 +++++------ .../hybrid-reasoning-llm/README.md | 20 ++++++------------ .../hybrid-reasoning-llm/notebook.ipynb | 20 ++++++------------ .../large-size-llm/README.md | 11 ++++------ .../large-size-llm/notebook.ipynb | 11 ++++------ .../large-size-llm/service.yaml | 6 +++--- .../medium-size-llm/README.md | 18 +++++++--------- .../medium-size-llm/notebook.ipynb | 18 +++++++--------- .../medium-size-llm/serve_llama_3_1_70b.py | 2 +- .../medium-size-llm/service.yaml | 8 +++---- .../reasoning-llm/README.md | 11 ++++------ .../reasoning-llm/notebook.ipynb | 11 ++++------ .../small-size-llm/README.md | 18 +++++++--------- .../small-size-llm/notebook.ipynb | 18 +++++++--------- .../small-size-llm/serve_llama_3_1_8b.py | 2 +- .../small-size-llm/service.yaml | 8 +++---- .../deployment-serve-llm/vision-llm/README.md | 19 +++-------------- .../vision-llm/client_local_image.py | 2 +- .../vision-llm/example.jpg | Bin 0 -> 32458 bytes .../vision-llm/notebook.ipynb | 19 +++-------------- 21 files changed, 94 insertions(+), 152 deletions(-) create mode 100644 doc/source/serve/tutorials/deployment-serve-llm/vision-llm/example.jpg diff --git a/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb index d7f694270177..0b90c73aca73 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb @@ -13,32 +13,32 @@ "\n", "## Tutorial categories\n", "\n", - "**[Small-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/small-size-llm/README.html)** \n", + "**[Small-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html)** \n", "Deploy small-sized models on a single GPU, such as Llama 3 8 B, Mistral 7 B, or Phi-2. \n", "\n", "---\n", "\n", - "**[Medium-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/medium-size-llm/README.html)** \n", + "**[Medium-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html)** \n", "Deploy medium-sized models using tensor parallelism across 4—8 GPUs on a single node, such as Llama 3 70 B, Qwen 14 B, Mixtral 8x7 B. \n", "\n", "---\n", "\n", - "**[Large-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/large-size-llm/README.html)** \n", + "**[Large-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html)** \n", "Deploy massive models using pipeline parallelism across a multi-node cluster, such as Deepseek-R1 or Llama-Nemotron-253 B. \n", "\n", "---\n", "\n", - "**[Vision LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/vision-llm/README.html)** \n", + "**[Vision LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/vision-llm/README.html)** \n", "Deploy models with image and text input such as Qwen 2.5-VL-7 B-Instruct, MiniGPT-4, or Pixtral-12 B. \n", "\n", "---\n", "\n", - "**[Reasoning LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/reasoning-llm/README.html)** \n", + "**[Reasoning LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html)** \n", "Deploy models with reasoning capabilities designed for long-context tasks, coding, or tool use, such as QwQ-32 B. \n", "\n", "---\n", "\n", - "**[Hybrid thinking LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/hybrid-reasoning-llm/README.html)** \n", + "**[Hybrid thinking LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.html)** \n", "Deploy models that can switch between reasoning and non-reasoning modes for flexible usage, such as Qwen-3." ] } diff --git a/doc/source/serve/tutorials/deployment-serve-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/README.md index e4372a2a92b7..4aa5318b28cd 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/README.md @@ -12,30 +12,30 @@ Each tutorial includes development and production setups, tips for configuring y ## Tutorial categories -**[Small-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/small-size-llm/README.html)** +**[Small-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html)** Deploy small-sized models on a single GPU, such as Llama 3 8 B, Mistral 7 B, or Phi-2. --- -**[Medium-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/medium-size-llm/README.html)** +**[Medium-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html)** Deploy medium-sized models using tensor parallelism across 4—8 GPUs on a single node, such as Llama 3 70 B, Qwen 14 B, Mixtral 8x7 B. --- -**[Large-sized LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/large-size-llm/README.html)** +**[Large-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html)** Deploy massive models using pipeline parallelism across a multi-node cluster, such as Deepseek-R1 or Llama-Nemotron-253 B. --- -**[Vision LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/vision-llm/README.html)** +**[Vision LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/vision-llm/README.html)** Deploy models with image and text input such as Qwen 2.5-VL-7 B-Instruct, MiniGPT-4, or Pixtral-12 B. --- -**[Reasoning LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/reasoning-llm/README.html)** +**[Reasoning LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html)** Deploy models with reasoning capabilities designed for long-context tasks, coding, or tool use, such as QwQ-32 B. --- -**[Hybrid thinking LLM deployment](https://docs.ray.io/en/latest/ray-overview/examples/deployment-serve-llm/hybrid-reasoning-llm/README.html)** +**[Hybrid thinking LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.html)** Deploy models that can switch between reasoning and non-reasoning modes for flexible usage, such as Qwen-3. diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md index 6fdd089399d8..5a01ee73875c 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md @@ -27,7 +27,7 @@ This tutorial deploys a hybrid reasoning LLM using Ray Serve LLM. **Note:** Reasoning often benefits from long context windows (32K up to +1M tokens), high token throughput, low-temperature decoding (greedy sampling), and strong instruction tuning or scratchpad-style reasoning. -To see an example of deploying a purely reasoning model like *QwQ-32 B*, see [Deploying a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.html). +To see an example of deploying a purely reasoning model like *QwQ-32 B*, see [Deploying a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html). --- @@ -156,7 +156,7 @@ app = build_openai_app({"llm_configs": [llm_config]}) **Prerequisites** * Access to GPU compute. -* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`. +* (Optional) A **Hugging Face token** if using gated models like. Store it in `export HF_TOKEN=`. **Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks. @@ -191,7 +191,7 @@ Use the `model_id` defined in your config (here, `my-qwen-3-32b`) to query your You can disable thinking in Qwen-3 by either adding a `/no_think` tag in the prompt or by forwarding `enable_thinking: False` to the vLLM inference engine. -Example curl with `/no_think` +Example curl with `/no_think`: ```bash @@ -199,10 +199,7 @@ Example curl with `/no_think` curl -X POST http://localhost:8000/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer FAKE_KEY" \ - -d '{ \ - "model": "my-qwen-3-32b", \ - "messages": [{"role": "user", "content": "What is greater between 7.8 and 7.11 ? /no_think"}] \ - }' + -d '{ "model": "my-qwen-3-32b", "messages": [{"role": "user", "content": "What is greater between 7.8 and 7.11 ? /no_think"}] }' ``` Example Python with `enable_thinking: False`: @@ -240,7 +237,7 @@ Notice the `reasoning_content` is empty here. You can enable thinking in Qwen-3 by either adding a `/think` tag in the prompt or by forwarding `enable_thinking: True` to the vLLM inference engine. -Example curl with `/think` +Example curl with `/think`: ```bash @@ -248,10 +245,7 @@ Example curl with `/think` curl -X POST http://localhost:8000/v1/chat/completions \ -H "Content-Type: application/json" \ -H "Authorization: Bearer FAKE_KEY" \ - -d '{ \ - "model": "my-qwen-3-32b", \ - "messages": [{"role": "user", "content": "What is greater between 7.8 and 7.11 ? /think"}] \ - }' + -d '{ "model": "my-qwen-3-32b", "messages": [{"role": "user", "content": "What is greater between 7.8 and 7.11 ? /think"}] }' ``` Example Python with `enable_thinking: True`: @@ -299,7 +293,7 @@ serve shutdown -y ## Deploy to production with Anyscale services -For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without any code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#production-deployment-with-anyscale-service) for an example with a medium-sized model like the *Qwen-32b* from this tutorial. +For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without any code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *Qwen-32b* from this tutorial. --- diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb index 08d565da1443..221e991e88d1 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb @@ -24,7 +24,7 @@ "\n", "**Note:** Reasoning often benefits from long context windows (32K up to +1M tokens), high token throughput, low-temperature decoding (greedy sampling), and strong instruction tuning or scratchpad-style reasoning.\n", "\n", - "To see an example of deploying a purely reasoning model like *QwQ-32 B*, see [Deploying a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.html).\n", + "To see an example of deploying a purely reasoning model like *QwQ-32 B*, see [Deploying a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html).\n", "\n", "---\n", "\n", @@ -163,7 +163,7 @@ "**Prerequisites**\n", "\n", "* Access to GPU compute.\n", - "* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`.\n", + "* (Optional) A **Hugging Face token** if using gated models like. Store it in `export HF_TOKEN=`.\n", "\n", "**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks.\n", "\n", @@ -209,7 +209,7 @@ "\n", "You can disable thinking in Qwen-3 by either adding a `/no_think` tag in the prompt or by forwarding `enable_thinking: False` to the vLLM inference engine. \n", "\n", - "Example curl with `/no_think`" + "Example curl with `/no_think`:" ] }, { @@ -223,10 +223,7 @@ "curl -X POST http://localhost:8000/v1/chat/completions \\\n", " -H \"Content-Type: application/json\" \\\n", " -H \"Authorization: Bearer FAKE_KEY\" \\\n", - " -d '{ \\\n", - " \"model\": \"my-qwen-3-32b\", \\\n", - " \"messages\": [{\"role\": \"user\", \"content\": \"What is greater between 7.8 and 7.11 ? /no_think\"}] \\\n", - " }'" + " -d '{ \"model\": \"my-qwen-3-32b\", \"messages\": [{\"role\": \"user\", \"content\": \"What is greater between 7.8 and 7.11 ? /no_think\"}] }'" ] }, { @@ -280,7 +277,7 @@ " \n", "You can enable thinking in Qwen-3 by either adding a `/think` tag in the prompt or by forwarding `enable_thinking: True` to the vLLM inference engine. \n", "\n", - "Example curl with `/think`" + "Example curl with `/think`:" ] }, { @@ -294,10 +291,7 @@ "curl -X POST http://localhost:8000/v1/chat/completions \\\n", " -H \"Content-Type: application/json\" \\\n", " -H \"Authorization: Bearer FAKE_KEY\" \\\n", - " -d '{ \\\n", - " \"model\": \"my-qwen-3-32b\", \\\n", - " \"messages\": [{\"role\": \"user\", \"content\": \"What is greater between 7.8 and 7.11 ? /think\"}] \\\n", - " }'" + " -d '{ \"model\": \"my-qwen-3-32b\", \"messages\": [{\"role\": \"user\", \"content\": \"What is greater between 7.8 and 7.11 ? /think\"}] }'" ] }, { @@ -372,7 +366,7 @@ "\n", "## Deploy to production with Anyscale services\n", "\n", - "For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without any code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#production-deployment-with-anyscale-service) for an example with a medium-sized model like the *Qwen-32b* from this tutorial.\n", + "For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without any code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *Qwen-32b* from this tutorial.\n", "\n", "---\n", "\n", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md index f269a9ea8db5..e40fd9d80c8e 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md @@ -8,7 +8,7 @@ Modify notebook.ipynb instead, then regenerate this file with: jupyter nbconvert "$notebook.ipynb" --to markdown --output "README.md" --> -# Deploy a large size LLM +# Deploy a large-sized LLM A large LLM typically runs on multiple nodes with multiple GPUs, prioritizing peak quality and capability: stronger reasoning, broader knowledge, longer context windows, more robust generalization. When higher latency, complexity, and cost are acceptable trade-offs because you require state-of-the-art results. @@ -73,7 +73,7 @@ app = build_openai_app({"llm_configs": [llm_config]}) **Prerequisites** * Access to GPU compute. -* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`. +* (Optional) A **Hugging Face token** if using gated models. Store it in `export HF_TOKEN=`. **Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks. @@ -114,10 +114,7 @@ Example curl: curl -X POST http://localhost:8000/v1/chat/completions \ -H "Authorization: Bearer FAKE_KEY" \ -H "Content-Type: application/json" \ - -d '{ \ - "model": "my-deepseek-r1", \ - "messages": [{"role": "user", "content": "What is 2 + 2?"}] \ - }' + -d '{ "model": "my-deepseek-r1", "messages": [{"role": "user", "content": "What is 2 + 2?"}] }' ``` Example Python: @@ -191,7 +188,7 @@ Create your Anyscale service configuration in a new `service.yaml` file: ```yaml #service.yaml name: deploy-deepseek-r1 -image_uri: anyscale/ray-llm:2.49.0-py311-cu128 +image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image. Use `containerfile: ./Dockerfile` to use a custom Dockerfile. compute_config: auto_select_worker_config: true # Change default disk size to 1000GB diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb index e55d72774072..d4744d6f304f 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb @@ -5,7 +5,7 @@ "id": "f8f6fcbd", "metadata": {}, "source": [ - "# Deploy a large size LLM\n", + "# Deploy a large-sized LLM\n", "\n", "A large LLM typically runs on multiple nodes with multiple GPUs, prioritizing peak quality and capability: stronger reasoning, broader knowledge, longer context windows, more robust generalization. When higher latency, complexity, and cost are acceptable trade-offs because you require state-of-the-art results.\n", "\n", @@ -80,7 +80,7 @@ "**Prerequisites**\n", "\n", "* Access to GPU compute.\n", - "* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`.\n", + "* (Optional) A **Hugging Face token** if using gated models. Store it in `export HF_TOKEN=`.\n", "\n", "**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks.\n", "\n", @@ -138,10 +138,7 @@ "curl -X POST http://localhost:8000/v1/chat/completions \\\n", " -H \"Authorization: Bearer FAKE_KEY\" \\\n", " -H \"Content-Type: application/json\" \\\n", - " -d '{ \\\n", - " \"model\": \"my-deepseek-r1\", \\\n", - " \"messages\": [{\"role\": \"user\", \"content\": \"What is 2 + 2?\"}] \\\n", - " }'" + " -d '{ \"model\": \"my-deepseek-r1\", \"messages\": [{\"role\": \"user\", \"content\": \"What is 2 + 2?\"}] }'" ] }, { @@ -242,7 +239,7 @@ "```yaml\n", "#service.yaml\n", "name: deploy-deepseek-r1\n", - "image_uri: anyscale/ray-llm:2.49.0-py311-cu128\n", + "image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image. Use `containerfile: ./Dockerfile` to use a custom Dockerfile.\n", "compute_config:\n", " auto_select_worker_config: true \n", " # Change default disk size to 1000GB\n", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/service.yaml b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/service.yaml index fb53624fe496..9fb4e4e7130b 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/service.yaml +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/service.yaml @@ -1,6 +1,6 @@ #service.yaml name: deploy-deepseek-r1 -image_uri: anyscale/ray-llm:2.48.0-py311-cu128 +image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image. Use `containerfile: ./Dockerfile` to use a custom Dockerfile. compute_config: auto_select_worker_config: true # Change default disk size to 1000GB @@ -25,5 +25,5 @@ compute_config: working_dir: . cloud: applications: -# Point to your app in your Python module -- import_path: serve_deepseek_r1:app \ No newline at end of file + # Point to your app in your Python module + - import_path: serve_deepseek_r1:app \ No newline at end of file diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md index b93843b2cc90..b40f78e86b6b 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md @@ -8,7 +8,7 @@ Modify notebook.ipynb instead, then regenerate this file with: jupyter nbconvert "$notebook.ipynb" --to markdown --output "README.md" --> -# Deploying a medium size LLM +# Deploying a medium-sized LLM A medium LLM typically runs on a single node with 4-8 GPUs. It offers a balance between performance and efficiency. These models provide stronger accuracy and reasoning than small models while remaining more affordable and resource-friendly than very large ones. This makes them a solid choice for production workloads that need good quality at lower cost. They're also ideal for scaling applications where large models would be too slow or expensive. @@ -43,7 +43,7 @@ llm_config = LLMConfig( max_replicas=4, ) ), - ### If your model is not gated, you can skip `hf_token` + ### If your model is not gated, you can skip `HF_TOKEN` # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3. # Type `export HF_TOKEN=` in a terminal runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), @@ -87,6 +87,7 @@ In a terminal, run: ```bash %%bash +export HF_TOKEN= serve run serve_llama_3_1_70b:app --non-blocking ``` @@ -106,10 +107,7 @@ Example curl: curl -X POST http://localhost:8000/v1/chat/completions \ -H "Authorization: Bearer FAKE_KEY" \ -H "Content-Type: application/json" \ - -d '{ \ - "model": "my-llama-3.1-70b", \ - "messages": [{"role": "user", "content": "What is 2 + 2?"}] \ - }' + -d '{ "model": "my-llama-3.1-70b", "messages": [{"role": "user", "content": "What is 2 + 2?"}] }' ``` Example Python: @@ -165,16 +163,16 @@ Anyscale provides out-of-the-box images (`anyscale/ray-llm`), which come pre-loa Create your Anyscale service configuration in a new `service.yaml` file: ```yaml -#service.yaml +# service.yaml name: deploy-llama-3-70b -image_uri: anyscale/ray-llm:2.49.0-py311-cu128 +image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image. Use `containerfile: ./Dockerfile` to use a custom Dockerfile. compute_config: auto_select_worker_config: true working_dir: . cloud: applications: -# Point to your app in your Python module -- import_path: serve_llama_3_1_70b:app + # Point to your app in your Python module + - import_path: serve_llama_3_1_70b:app ``` Deploy your service. Make sure you forward your Hugging Face token to the command. diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb index 40d49aa0a749..fa284ed6f02b 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb @@ -5,7 +5,7 @@ "id": "f8f6fcbd", "metadata": {}, "source": [ - "# Deploying a medium size LLM\n", + "# Deploying a medium-sized LLM\n", "\n", "A medium LLM typically runs on a single node with 4-8 GPUs. It offers a balance between performance and efficiency. These models provide stronger accuracy and reasoning than small models while remaining more affordable and resource-friendly than very large ones. This makes them a solid choice for production workloads that need good quality at lower cost. They're also ideal for scaling applications where large models would be too slow or expensive.\n", "\n", @@ -46,7 +46,7 @@ " max_replicas=4,\n", " )\n", " ),\n", - " ### If your model is not gated, you can skip `hf_token`\n", + " ### If your model is not gated, you can skip `HF_TOKEN`\n", " # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3.\n", " # Type `export HF_TOKEN=` in a terminal\n", " runtime_env=dict(env_vars={\"HF_TOKEN\": os.environ.get(\"HF_TOKEN\")}),\n", @@ -100,6 +100,7 @@ "outputs": [], "source": [ "%%bash\n", + "export HF_TOKEN=\n", "serve run serve_llama_3_1_70b:app --non-blocking" ] }, @@ -130,10 +131,7 @@ "curl -X POST http://localhost:8000/v1/chat/completions \\\n", " -H \"Authorization: Bearer FAKE_KEY\" \\\n", " -H \"Content-Type: application/json\" \\\n", - " -d '{ \\\n", - " \"model\": \"my-llama-3.1-70b\", \\\n", - " \"messages\": [{\"role\": \"user\", \"content\": \"What is 2 + 2?\"}] \\\n", - " }'" + " -d '{ \"model\": \"my-llama-3.1-70b\", \"messages\": [{\"role\": \"user\", \"content\": \"What is 2 + 2?\"}] }'" ] }, { @@ -216,16 +214,16 @@ "\n", "Create your Anyscale service configuration in a new `service.yaml` file:\n", "```yaml\n", - "#service.yaml\n", + "# service.yaml\n", "name: deploy-llama-3-70b\n", - "image_uri: anyscale/ray-llm:2.49.0-py311-cu128\n", + "image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image. Use `containerfile: ./Dockerfile` to use a custom Dockerfile.\n", "compute_config:\n", " auto_select_worker_config: true \n", "working_dir: .\n", "cloud:\n", "applications:\n", - "# Point to your app in your Python module\n", - "- import_path: serve_llama_3_1_70b:app\n", + " # Point to your app in your Python module\n", + " - import_path: serve_llama_3_1_70b:app\n", "```\n", "\n", "Deploy your service. Make sure you forward your Hugging Face token to the command." diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/serve_llama_3_1_70b.py b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/serve_llama_3_1_70b.py index 650b4e2d6574..9e62adffb19a 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/serve_llama_3_1_70b.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/serve_llama_3_1_70b.py @@ -15,7 +15,7 @@ max_replicas=4, ) ), - ### If your model is not gated, you can skip `hf_token` + ### If your model is not gated, you can skip `HF_TOKEN` # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3. # Type `export HF_TOKEN=` in a terminal runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/service.yaml b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/service.yaml index c3828c619110..35388c72f961 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/service.yaml +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/service.yaml @@ -1,10 +1,10 @@ -#service.yaml +# service.yaml name: deploy-llama-3-70b -containerfile: ./Dockerfile +image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image. Use `containerfile: ./Dockerfile` to use a custom Dockerfile. compute_config: auto_select_worker_config: true working_dir: . cloud: applications: -# Point to your app in your Python module -- import_path: serve_llama_3_1_70b:app \ No newline at end of file + # Point to your app in your Python module + - import_path: serve_llama_3_1_70b:app \ No newline at end of file diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md index dcecf2ae8d0e..d7a3aa48f702 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md @@ -131,7 +131,7 @@ app = build_openai_app({"llm_configs": [llm_config]}) **Prerequisites** * Access to GPU compute. -* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=` +* (Optional) A **Hugging Face token** if using gated models. Store it in `export HF_TOKEN=` **Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks. @@ -170,13 +170,10 @@ Example curl: curl -X POST http://localhost:8000/v1/chat/completions \ -H "Authorization: Bearer FAKE_KEY" \ -H "Content-Type: application/json" \ - -d '{ \ - "model": "my-qwq-32B", \ - "messages": [{"role": "user", "content": "Pick three random words with 3 syllables each and count the number of R'\''s in each of them"}] \ - }' + -d '{ "model": "my-qwq-32B", "messages": [{"role": "user", "content": "Pick three random words with 3 syllables each and count the number of R'\''s in each of them"}] }' ``` -Example python: +Example Python: ```python @@ -219,7 +216,7 @@ serve shutdown -y ## Deploy to production with Anyscale services -For production, use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium size LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *QwQ-32 B* used here. +For production, use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *QwQ-32 B* used here. --- diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb index 938268090be5..7f60d2c396c2 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb @@ -138,7 +138,7 @@ "**Prerequisites**\n", "\n", "* Access to GPU compute.\n", - "* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=`\n", + "* (Optional) A **Hugging Face token** if using gated models. Store it in `export HF_TOKEN=`\n", "\n", "**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks.\n", "\n", @@ -194,10 +194,7 @@ "curl -X POST http://localhost:8000/v1/chat/completions \\\n", " -H \"Authorization: Bearer FAKE_KEY\" \\\n", " -H \"Content-Type: application/json\" \\\n", - " -d '{ \\\n", - " \"model\": \"my-qwq-32B\", \\\n", - " \"messages\": [{\"role\": \"user\", \"content\": \"Pick three random words with 3 syllables each and count the number of R'\\''s in each of them\"}] \\\n", - " }'" + " -d '{ \"model\": \"my-qwq-32B\", \"messages\": [{\"role\": \"user\", \"content\": \"Pick three random words with 3 syllables each and count the number of R'\\''s in each of them\"}] }'" ] }, { @@ -205,7 +202,7 @@ "id": "942e675c", "metadata": {}, "source": [ - "Example python:" + "Example Python:" ] }, { @@ -270,7 +267,7 @@ "\n", "## Deploy to production with Anyscale services\n", "\n", - "For production, use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium size LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *QwQ-32 B* used here.\n", + "For production, use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *QwQ-32 B* used here.\n", "\n", "---\n", "\n", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md index 503abeb49e94..3523ece616b3 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md @@ -8,7 +8,7 @@ Modify notebook.ipynb instead, then regenerate this file with: jupyter nbconvert "$notebook.ipynb" --to markdown --output "README.md" --> -# Deploy a small size LLM +# Deploy a small-sized LLM A small LLM runs on a single node with 1–2 GPUs, making it fast, inexpensive, and simple to use. It’s ideal for prototyping, lightweight applications, latency-critical use cases, cost-sensitive deployments, and environments with limited resources where efficiency matters more than peak accuracy. @@ -42,7 +42,7 @@ llm_config = LLMConfig( max_replicas=2, ) ), - ### If your model isn't gated, you can skip `hf_token` + ### If your model isn't gated, you can skip `HF_TOKEN` # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3 # Type `export HF_TOKEN=` in a terminal runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), @@ -82,6 +82,7 @@ In a terminal, run: ```bash %%bash +export HF_TOKEN= serve run serve_llama_3_1_8b:app --non-blocking ``` @@ -93,7 +94,7 @@ Deployment typically takes a few minutes as the cluster is provisioned, the vLLM Your endpoint is available locally at `http://localhost:8000`. You can use a placeholder authentication token for the OpenAI client, for example `"FAKE_KEY"`. -**Example curl:** +Example curl: ```bash @@ -101,13 +102,10 @@ Your endpoint is available locally at `http://localhost:8000`. You can use a pla curl -X POST http://localhost:8000/v1/chat/completions \ -H "Authorization: Bearer FAKE_KEY" \ -H "Content-Type: application/json" \ - -d '{ \ - "model": "my-llama-3.1-8b", \ - "messages": [{"role": "user", "content": "What is 2 + 2?"}] \ - }' + -d '{ "model": "my-llama-3.1-8b", "messages": [{"role": "user", "content": "What is 2 + 2?"}] }' ``` -**Example Python:** +Example Python: ```python @@ -163,7 +161,7 @@ Create your Anyscale Service configuration in a new `service.yaml` file: ```yaml # service.yaml name: deploy-llama-3-8b -image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image +image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image. Use `containerfile: ./Dockerfile` to use a custom Dockerfile. compute_config: auto_select_worker_config: true working_dir: . @@ -312,4 +310,4 @@ See this [Troubleshooting Guide](https://docs.anyscale.com/overview) for common ## Summary -In this tutorial, you deployed a small size LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and how to send requests. You also learned how to monitor your app and common troubleshooting issues. +In this tutorial, you deployed a small-sized LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and how to send requests. You also learned how to monitor your app and common troubleshooting issues. diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb index 09dfb2206b82..f4cd9ec72d34 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb @@ -5,7 +5,7 @@ "id": "6a51548b", "metadata": {}, "source": [ - "# Deploy a small size LLM\n", + "# Deploy a small-sized LLM\n", "\n", "A small LLM runs on a single node with 1–2 GPUs, making it fast, inexpensive, and simple to use. It’s ideal for prototyping, lightweight applications, latency-critical use cases, cost-sensitive deployments, and environments with limited resources where efficiency matters more than peak accuracy.\n", "\n", @@ -45,7 +45,7 @@ " max_replicas=2,\n", " )\n", " ),\n", - " ### If your model isn't gated, you can skip `hf_token`\n", + " ### If your model isn't gated, you can skip `HF_TOKEN`\n", " # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3\n", " # Type `export HF_TOKEN=` in a terminal\n", " runtime_env=dict(env_vars={\"HF_TOKEN\": os.environ.get(\"HF_TOKEN\")}),\n", @@ -95,6 +95,7 @@ "outputs": [], "source": [ "%%bash\n", + "export HF_TOKEN=\n", "serve run serve_llama_3_1_8b:app --non-blocking" ] }, @@ -111,7 +112,7 @@ "\n", "Your endpoint is available locally at `http://localhost:8000`. You can use a placeholder authentication token for the OpenAI client, for example `\"FAKE_KEY\"`.\n", "\n", - "**Example curl:**" + "Example curl:" ] }, { @@ -125,10 +126,7 @@ "curl -X POST http://localhost:8000/v1/chat/completions \\\n", " -H \"Authorization: Bearer FAKE_KEY\" \\\n", " -H \"Content-Type: application/json\" \\\n", - " -d '{ \\\n", - " \"model\": \"my-llama-3.1-8b\", \\\n", - " \"messages\": [{\"role\": \"user\", \"content\": \"What is 2 + 2?\"}] \\\n", - " }'" + " -d '{ \"model\": \"my-llama-3.1-8b\", \"messages\": [{\"role\": \"user\", \"content\": \"What is 2 + 2?\"}] }'" ] }, { @@ -136,7 +134,7 @@ "id": "d623a30f", "metadata": {}, "source": [ - "**Example Python:**" + "Example Python:" ] }, { @@ -214,7 +212,7 @@ "```yaml\n", "# service.yaml\n", "name: deploy-llama-3-8b\n", - "image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image\n", + "image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image. Use `containerfile: ./Dockerfile` to use a custom Dockerfile.\n", "compute_config:\n", " auto_select_worker_config: true \n", "working_dir: .\n", @@ -387,7 +385,7 @@ "\n", "## Summary\n", "\n", - "In this tutorial, you deployed a small size LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and how to send requests. You also learned how to monitor your app and common troubleshooting issues." + "In this tutorial, you deployed a small-sized LLM with Ray Serve LLM, from development to production. You learned how to configure Ray Serve LLM, deploy your service on your Ray cluster, and how to send requests. You also learned how to monitor your app and common troubleshooting issues." ] } ], diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/serve_llama_3_1_8b.py b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/serve_llama_3_1_8b.py index c5ed01f7b304..861a4f7aae7b 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/serve_llama_3_1_8b.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/serve_llama_3_1_8b.py @@ -15,7 +15,7 @@ max_replicas=2, ) ), - ### If your model isn't gated, you can skip `hf_token` + ### If your model isn't gated, you can skip `HF_TOKEN` # Share your Hugging Face token with the vllm engine so it can access the gated Llama 3 # Type `export HF_TOKEN=` in a terminal runtime_env=dict(env_vars={"HF_TOKEN": os.environ.get("HF_TOKEN")}), diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/service.yaml b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/service.yaml index 98d3ed37539b..4c12e613c0d0 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/service.yaml +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/service.yaml @@ -1,10 +1,10 @@ -#service.yaml +# service.yaml name: deploy-llama-3-8b -containerfile: ./Dockerfile +image_uri: anyscale/ray-llm:2.49.0-py311-cu128 # Anyscale Ray Serve LLM image. Use `containerfile: ./Dockerfile` to use a custom Dockerfile. compute_config: auto_select_worker_config: true working_dir: . cloud: applications: -# Point to your app in your Python module -- import_path: serve_llama_3_1_8b:app \ No newline at end of file + # Point to your app in your Python module + - import_path: serve_llama_3_1_8b:app \ No newline at end of file diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md index 050f7e32904b..352153951931 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md @@ -58,7 +58,7 @@ app = build_openai_app({"llm_configs": [llm_config]}) **Prerequisites** * Access to GPU compute. -* (Optional) A **Hugging Face token** if using gated models like Meta’s Llama. Store it in `export HF_TOKEN=` +* (Optional) A **Hugging Face token** if using gated models. Store it in `export HF_TOKEN=` **Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks. @@ -89,7 +89,7 @@ Deployment typically takes a few minutes as the cluster is provisioned, the vLLM Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `"FAKE_KEY"`. -Example cURL with image URL +Example curl with image URL: ```bash @@ -97,20 +97,7 @@ Example cURL with image URL curl -X POST http://localhost:8000/v1/chat/completions \ -H "Authorization: Bearer FAKE_KEY" \ -H "Content-Type: application/json" \ - -d '{ \ - "model": "my-qwen-VL", \ - "messages": [ \ - { \ - "role": "user", \ - "content": [ \ - {"type": "text", "text": "What do you see in this image?"}, \ - {"type": "image_url", "image_url": { \ - "url": "http://images.cocodataset.org/val2017/000000039769.jpg" \ - }} \ - ] \ - } \ - ] \ - }' + -d '{ "model": "my-qwen-VL", "messages": [ { "role": "user", "content": [ {"type": "text", "text": "What do you see in this image?"}, {"type": "image_url", "image_url": { "url": "http://images.cocodataset.org/val2017/000000039769.jpg" }} ] } ] }' ``` Example Python with image URL: diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py index 14cfcb2c9f0a..200d46c49ab9 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py @@ -10,7 +10,7 @@ ### From an image locally saved as `example.jpg` # Load and encode image as base64 -with open("example.jpg", "rb") as f: +with open("vision-llm/example.jpg", "rb") as f: img_base64 = base64.b64encode(f.read()).decode() response = client.chat.completions.create( diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/example.jpg b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/example.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4284d25ff336d18e23dd135759937c50bc12249c GIT binary patch literal 32458 zcmbrl2UL?y*C-kglp+wS^b$G*LJ7S|3B835QiKqS^biz~Cejj`^xi^8M2ZEe(tAfK zqI4;uARvMiPxSl#_k90<&t2=>b!R0j^UUnodv=+<_sl%`J^yFnbQ#-sRy<-dM^uM+3Eq0zqb z5Qw+GsH3xw6H*l6;{^$H^o59via`L1DuKRuTu3yh6Vlb)TM4w^+6m%xcUA&fN*Rb5 z_^Kn_+_i&INYh|LGeoclLe3eaqD-b3C?DwM>xD!+at3;Ndi%=@l2@uppgHJ!_z?gos>V?8SM9pFa1$eSZb+>h6O6Z}goJ|G@hOpgjLlbw)sto=7jGH`*T$D~>1iFX{Sv2KfI2 zjpr@EANi;0?-);K7YLrf__;iTtM(f$w&7clB2K zBPP5q@I2wAE3e^*b~JGJcK3GuM~?po+28j;I66Tz40J?{O^md4)O5^X8X_<&b(pc4 zDNNNwT^nYorDF)wG1P!r=@@E(IPt5;86l5AAsx{^DCNIv#1!fGuZpL;vZE(55b5rK z_ELoWF?D>)zbb!WApbcMUw8iB3*%o1R6WuEh3Ste$a^~enSG#$Gt$K|z!MEp{=au6 z|AYQt%zxE3YD4=n7IAHG(IwU+w0^<^q=qW19q7TkEFZBCTUUs^7x{icHCJgY<#{p*~d zsGNPPfJx%d9tk_+5&>Z&TftRt?69MIefgWu7^8VKm^R1#9*oYv&?bJFnO#S3<`oZ5 z`n=Z?(3TUQR)N-AkGF@d3pi2 zX2Tm6p&*~(X|$j=0evd@rrCSp+s(&o2<*7hTL3FNq>o&TMSwDTw2XxYWdLUmNbBxj59=`0w!WP^f%l3(hN+tUZ8Wn zD$rQiIPFBQl~`AJXX9MoWL8`(x~$G97G|%Nx3T*FbX;t%O61EhAY*oNk9p|YLSSMB z6E+@BZEWkVs931U82E6}>HOZB!d$ z^pe;fxj_J};)_Ow4h;RA%8%z8?6azMFVXmbMKU70=2-3XU8k(kzODY#wZX3JGfI_O z+*s_GuW(|}hrYc0;ITJK5b5KriTxsl57_SeGX-Cdygfuyh6K z#5V&>d$h%b#8huiuNJuFM&$CK?>%XTE4^QT#!40G*7?hI_)*IefApuIk;(J5q(h8d}&%aQTH1l{c0Gz@kQ}Bz;pAPAjx&k zV*H}^tDw~XMH4|nSo%#FQt!${pMn}5ie%7msW5ewu|4?J#1i?G8`bD%k|H1ZiP>pq zY};?nPB3sL&+~PDu=di2jO|T?PF;umvW$ZGqOhlZLZe*pxPyB9Tu$}$Z9+WW2mm%l z=r##JNV#+MS2cY~HG`p3Od9;IM_z@s`_`|9zAxi#i-$A%3eEuL=9!E(raJZ*sShS? z*7@(o=2$%qa)C!3Ano~~;v2jj}WDzm4PB zQdU=200_xI*x|zhPzJQN2Ie^vVg7MrS&NJ@Aos|1dVTS_9nZqMt@clPQxn$rP(MfS zS(v@qNwzN1u7ZdrG0zo@l#NvQFf?wjKTS9usNhDSYv)AP&DrwpU$jd%M~@H5m9J_y z+176#U8W_^PcaKoNJ6*h&4kRQ_Ad&T3S6lgefpL+U+0Z1#JU%L zW^1wz2>NyrAnkvWeUkFVx{h8bk02{80(4|=nHT7u6C~`n!>^V1TFGbpWN4=H*27o> z*NUl+JxYqtuUqvmq96r(14Vn?IEZwVFet+;*D=Vom09SvJLa`)6|lO<`tbb7$@iq> zW8W0(dsIMvA!cNIbf;3G0=yuf@?w4fPz{)5wP*G(UiS|EwqDnlKd$?EN8-~xyM~Sq z_kBelucxnF%Uw$o+n!E5PGe0E`qli<|MQRWJ0meKGc?{mOfzRwBu1SG0Vu?!dK0au zD20sb&O+C3ZbmOkHPNl^$UQWK*mE0*-?MJ8qOIbN#fLKZw(nd?uy4>=&E96mm|X&5 z^nH;0a8jX-zwM{|!(e_((E_^`^zr7N%lL!n>;;c3rEwPhAIF_?uHd;F;ZWNM*#hdI z0N<+h<6iM0oOsu1#utP?YHAw`j)@vEkr%Y9RUO+o4z&_ zwLUq&zdcbpbk?EN__6MpNYKVn8f7fAOZm!=tBSIJ`aX=p28bJgsry!cE}CA(-xU}LGb`N-I^q^j|XCefrp zmFuUAu;GEs_zeG?&A8N?GEzfx(x2JjZBL1S1l&p;SA}`)wOi}ct^Tq;R7G1`Qv;~7 zwFD@B_NB^Ali&AS95}wLH-s+KJ?YPsp*hqNJ=*!am6VaF`Tejp#eZg^ZpPeiE8qxq z{5j2bzDXp*OqR4FBEsSY_c2GxwuPBp6B8LV505o^+!Q0%;v~hd^TvWM>0923{4aT_ z-2S%OPh-t(D8>AspM0%yZguQ$dS2*d>6n}Z{a9Yuz7)Ohl^3`$QeEIT7Wi;L^lWn? zTx+E6^nLEc(8(69CFFVDNXQi=3?)otF?J1`8erT_e=T**n_ zFE_+Tnv1AT)^}}wQS-Ci?uY$QNXNq&)YNjArsFP z$z59$9|k>TDYQf79z7@m4P=$()yW6AT+Dg>m{q23JuX)vm?HB`}VEBd`jg%k7wxzSjK)X^ju+f`Z~Td-d`yb zxH|F*Vr0E@$sSIdxFggQ;7w4}HoVFJ@I zd^EGw;`FJob8_h2q3nBDVTF8X^aL!PIRQPvGHjKg6F+unYT)|0-QL_(HzJ0x3?SRq zuZhhkB)gQ^Iw1rUqbd6YX;U*D7bLp!U0r3W?Q_~@U9Imqik`JC6-p~snbibvYrJG* zfB0O!rM30--r45o+1=b~mKV?+Uxhnue5rtl7j%<`zHLP}FivQXS~fH6H^ zHgBhiFc^u-3nfDOXsnKm|pX%YZ2pjWI`LGFB^MvvD&v$((Q!lSZMwEeSz zBFjX98t3@!-qHT?JN(|y%{KR!-Jkg(G8M<)>>7C~-Q3D!o^2oVe7~ui1^&|g{pj=O z?XfwbXx6;K@CwA7><`CT^V=|nICX%E-uE8@Ky`x8sq#dwQ^}~0Pc)15c%6>?cQ)*m zKJwfA+%F7~npoYDc2@O+V^R%s@&XFjzlDpu2DdYP9QiiTDpBtHL#U>kwZtawdze?- z7e@^y7nF?YHi zp?8h|Jw3d_M>C*h*uPln7e{%8+-*h^HDW?GqLF+CHN-^7((}?0wlt|q;HE*4ZD+x~n9=4Yr(o*}dI4~6g>?%%rY z+&gSq58TlVe6N$)ROV$PoCO_?_$7Z`{w|47G%eFejJI`@X9niWR*{PnK@#@ngh~hW zD|F@|w1$q|tHXu>kQ{M>i0zQd+73DYwx6{0QP++-;wI5G{+MLxLF00?6&V3-d@ccZ zjfuFJKFw9EF+eRM$ILM;MW@=&^}ef#7@!^wATv2T&-v9bQ*mN_J3i*iL>VAtppt4P zX>fo1C6oH2+ayRU02!6^IC@M~AkLU%0=%LH=tnZrhzq`Z{or1Eo-T+8}w;I(dI6n>;6=To7ROR=pT-1150vbnHW(|I|*AmZ^<PtYzJoNZc8 zL`PIZ5*#g048YHjQ$o?c&O7JJ`%pINM)OAk)p{^`5`=(r4NsF@MFfil#KjhkFdTsWbNCules8;Q_hu`xHnZMvI8WptD5DGn;j zN^Jf!E+2(q8#lSzirpSm0I`(2g3nO?0v3%TNC{xnHt_*QX-#gx$PlWRIp9y7CKEeWnj{PR- z-U0x`#NsZIfD(p#c=`g=YO#ioWp#6g9ni5^CND{a#LJ4odDWcKHr?OfCL|J2{crR#cvBx zQW>K5i!@AdTsM&syV%*g1c03g+IVHkWK|MDRbpTyF_5uJY%Dbq+RiLboXQ4_K$lDk z+)%Xv7PA2ah_AcR;Wr2&ovtbd0nuPA4~5VkqJl62jklr%>$6LOT_A00#{4!0Vy=dr z6{xenZ*qGopdv$?Yq1RG)<^khiuhWwHNYa8fQ*=W%C^9N{a3>R#M8UViI1>0;Ia0|;X^lt_$? zpV7Cmci8(_=`FBiLv-Fb3Mddp{MGLxaTS$nKrz8PZ@hxw#7sfiZpJuLHd#)>&6iI_ zpX<&FPDdD%3&!di^E}ILd5`Dpf8Z9;GQiH+1?3vob zjv{cAhYajxbB>?<%YvHJVd~r0*kl1j5{`i_a@xi*N_1HB^v}`v6zJMW1t7{ir7xsj znt~e}r#xr0blHfUBL`C@?=6+VpLsP1Y3VZD`{{qw-gdlIySZpLQhS8F3*K!)(-&W3 zd<(eFbzSK2>jrp(zN#EuqOq1qBN)<}X^{Ut$wbqorM7%~MlU!MS5BQ&hsEc5a(vUA zlhXxqc34aBFTa7~z(a!o=O)ro4U6X6Telh%=l;8B>j_w}UcCmRbm34y6 z?5)i+|4&Pz&eOX*Jwi)nO#(X}&uu>48+HCUNiw<%Pe|qqqjBJA(G1rgr+#B7 zEk{&}M~NoQ0;aqGnGSVg90tAxkUk!Jxixhg_miJ+Dd>IPNzTzrk6HYgAs7fCBqSub zc8!RT5PwGarw|ZQU!wsMadOd$i4%ivtI|nu8`DePRl^@6lHd;)35W^Tuu{yz(LJjP zo}Q?+5owlH-!m8{#p%gH7|+GHJPQ zN;nhCyr#Z30*UU+P4h+_O)`&H%@EbZERP@Kt zWUK%#Mnc~+D>T!Z-C6Z}Ljz(9e%p`7(^D|WR0%%n>1ivY*^?!;cq_b>7{v1qW zje*1NH=aHQHyo5%72OQ6q&dxl&IjwepfYJ5?rje~v}ftO`al`i|~y~fjy{sGXC zb~=WE4|c?LGDqLNnGnet<9sUM7L|x?nm#J0pgFrBQt|HZ?fc+dCLP41jr3#iv^sYy zQJc^XOe92exUVeQhto)RLVpWf=(cvb?6 z!%<7IlJG8 zCl-#;7-jIh;Sx1tE|wc3o5TOqTG3Nl=gJm^wd`@OkjrB!EUpkF&7)1@sWs-NX?}91 zVM*azrl#3Hsl~#K71g=>j`{eUL@0xR5R$T8Gjc$8ybC%O*>{@I>zQBEihLbGKBX!2O!6g(6 z%ZX>lhjoi!MBMtYnuR9RzQ!(0w&;+eXv}jU3p{A>Mu#>6XHF}T{0h5g_oV~Vn<*jA zMQZ}*rO^%njr=rensIqyT&zYI$*RNkU5yjDi-;S_4RR5wGYRE7+3yZIZg$f2yQvU5=al)dwmB>{-AzbO-)PgQQK?~C z(T{K171_ZWO=o>%<6QbKds5z8)LQkV8Nt|((U>&GFY1LZii(Zi*CW63{80luT3yIf zj6^ucn-Dw7hf=iaDw*GT(1H0hr@G0@qZbxQh{i$5UGd)lIw9bP{RAt_iV5m*R20a8 z{j0wQPBxeFMui^J&Y9IfLOR)r?|@bK{RSr2YK)o7)iO~5umI~^rWYO0*m6?`_R_lsNjr?Qabd&6h?KA&({0k_%N$w^-* zREnR_m{nK!bwyv>5}eLpH|vbARFCJIPMR~3j7HmcgtH=bm>E(knK^PnI#ux~cfuez ztU^$W!5V!QiUfg+J|;N;RqXi4l}n03ilSlkV{~vBb2?Oq22;EOl^RZ(U_(TObVg?y z@w}Iz<$llptkbxWgETFSSF*)+;e9FA13B4zVAl9(GOQAvMopM?Z zJwds<0g0y=$hnEQkWuxZ%vrkSO(Z3E+?s4|Zg%~&v6Q59DJxpbQl&MX8#JWXhq11% zj=x#4WmTWYB1wBMFuZaI#_^Pf^~w5AwHos;FnE?o)>R}{N{UsB`t4+lii!yBlVk}8 z>cNwtg@zDOl?H4n>%?BJU#gm zdOaHO%wViC9SilWk<~-$5U(wEQ=V^3EKZn2;VzA!1HG>H%mF9*lz3OGUC{+~yL8KY zt5^DMJaHv0=C5?*-BL1X^4l4L^^X^Jnr7G~O`=Dwsf4SOaGpy$!Ev+xWG=mBm<_j9 z2gN|{4fwfy{)5;Uf%yjZl;SDgv+3NpZ!o{}g^KmfK+e};IXi{9N<6^=fw2jh;%{e5 z3iT;8neL|@kFn74B;Nr=$zwCWg4^#jJ1-^Zh#1r+eogco>U=HP)`m!q8|cO9&@lc6 zhzw~2u_h;(G&C@%y!j2#?AV;|hsx$Aal;rV-z5>o7HNw0;wpXv`Xc?)Hx3*ZcuG|) z;YLovqo4(PoO43+OjhNVI%|k(pj(b8$EF^aEnZVpU0FhysC4Nhfnjd+PZ?=xNQ`(Y z4dUFl_H@3=rguViqrYr=U}kI3aT{@Cm68GM=P|9;pGAc|Z;1;}m5q)%-qv=W;(9}#C!Vnr8^T6LZKwx0-d*<_FBJ=WvG$an{UdDO{4tRR{_mGElvbkiu^u6=T87eHdxthO$ND(H-sFSNgHD-29L# zCq2k)qkfW(YqYqhDV^1mDIUGVJ5VVp0XR1Sf)`VW*=CP@9h%TM7W-t;njK8Wy4Aya zT6}mnpWkWq!JKmkKew}x=3*s;s_Hj@<0xdJ#=xL0@%FK^)WW1l$xdO;PA+pUd-lP6 zi~kfm^$IUGnox|6nATL?)B!b?I2I#ElapLos3w+h%f(ZCxUz$r;SR9q^}8R>46PgW zd*p}S*@LGixO{4Q(lc*148`x9NhpSjD!No1uOYX;Z}1+EM*gx_4dpFR(YZk3(+aG> z>})B#H(fEozGgSK zl6)~#+5p&b(H-Nmls1nP>&{3I@ zL3Te&W_k3PPrO`dVm!gWz2qQaS;n(S~6B(7!X7&5KF zvU=KU8-03Zk@O3;4kEWs&EPOP%Pn0pO?@L$ftns^TzTj-x_BC4?1{&rXXmK9j!W-% zL#l7jY_bBv%W<9Zex^y@g%) znfNNrQM+;EVrhRbTy|^A<5t>$)G5`$b{$>(=aGxr{n2>0s5bC0e;uyj^L*#Ag~R>+ zzD&sd%hAq--++4uvSjRztHDu8CP(U&#Ljz}vx=P` zVz6#-LY_Et44*m(uZD#E2G~9?3=hYElsfSVHb4IMo`CQMKFj`hdW}!8sR@BJoLpk! zw~eoR2h$NYxywEO`?4*8> z@@&&JWj%dD(4o?xWnfZ*v zy2nh?^bh7sxw_#Wi28QdIs2wd>AF7UXATt|{sTHH2V~20-VY_`YpT9I%Uai&2=EEV zTd*MBf(Z$Tt^o+I{dxK1-&;iBwSSo~ttt@+E}>>j_b(eJ{AlDj zXasUAimTQD>4ZJl!Nn$4%(&NUr%I>Ta@?Sa9>y{I{HaOJt)xX$@fYcf6XEPB4jYEoXDOn5U|6Ch{X%S;xO~+> zSXP5lGYp+2iP`{RR9s1@$qaKh4V^*kI>VC{*+e=RHCiq;B`^+;7c83`sirnYqtX7| zC1D8KM>|=jwOdqzi!0&c0@Y7J^+@BXNYqASqj;dYa&gh|;3ezo{jAcUN(G*C8DcP9z(lAx1YYB#m|V`*`BtRmx-1|k#c)jb0CT+%thiN*{pRBXq0awEwf zk7dQn0H0LNKG_hY&MxDkpvAm^b5nyY<%OZf??qU~x-;*v^jUMPkDP=bl^$^IXcXop zhrmmRM^?_-rmRaRE99seqqa@Bxg2rW!`E6Hnui18jrGy-Q{=f zl{~Ui`_vW|%WA0y4ULerZh<sxxPM1Y4eGN@%$$!`a!rm zb8bVdW?^Yj14pgBJc~#o|DKN4l+&I~#>~2PCilU6IE`$GXX79B(8<4ox(_(9T&|w4 z^a;tZ_s5kU>Q<%h%|zI5W#(Dp#c75?P($t zXsK+YzItP{8bQZ&6dhoPtjA<1fL&f#{gufD$z=hejqh zz16ya?%ok_2QV=s(AoS%+GC`q7Y=HmNyH`6?zvqlr`Pp9rH{?HG%qkuK7VShwU=_v zqZ$&h@|-x6KVv=vZLgGE(g=SQCV_d_O|wtpFT-r}uvH$GDQ(T0K7#3JrsRg*X{MXX?v_r*P0iIJQ#j}@Cm{&)&BDndoxUTwe?-%CoLD~AGJ4<> zX^xRN-Gez<+2x|MZs8q3*Dcs@z$zR@-_1|3(tk56_zeqK?Zat&+!`& zoz*2wZx6NwcX;gRexy}{R@&O0Js5PA6TBUx-gixwPbnq~1Rq(*3 z3TGMYa&j8QP0wM~c87(9)ix_~tbbKpVoV;dP0w;9OY{PiCwSC6!85sbSFfQ#0vs3p zqv}mCcUe{kds2UY-@^90*vqU$i7}2(!W2KTolZJ8GccQcy{}3dQi`^NAJ6igk7~y` zn4vpo=JYZWPDJZ7nCm}4m|Y6i20$EN&oP;7hpFr41&~Q2G1{~ zM=qrG{^QeFL|@LQNv3>VTFqlhE%1HP_9D2GvcSQa?K5`Wx+{RgyCHb4oh85uWn)CLFeVVo^qPgsJXzx_~JKU=1j@(&16IKCvR}$ zskh7LJEyfM)>zO%`O5{SW!&jKbwBfd>?p!Bkw*>;$GdGDuT>k|>dNM7WXLHcTqG-| ze}Lx^n}Vna|5zrBg3gR6I+ z);5my1sByP%OpBET< zR9N)YZ?Z4L-UY*di0`n6Y(!dEhg4RE#pM2sa<6eT{5nnAMP%eKi8lV@AM66Ud{+5eV?Xk6lRgT4Toux3~&E=fE{X>S#O5 zPm&#wm(2c`DwQTm7jr+m!!>KJI#?iG$g`pzWF|=RZ0k3G=5zu1`Hz?Q4d{>UEQG(% zf%aaQDZ*i>##4uPWv#!Y*3Uo3U}=Qu^Ujw==;x+*-%v`$j!~?Q{K`QdG&J~4JItj* z*4{;*v`JedFXrMQRMD%Bj+q+vvBMpFDj_CgH)}y7E8mf)`~A&rZtmd=mnaPHC#z=T z;W5A3=g-yk9=`tab(Vv?$7NN4nZW`=lG-isoaM0GJDQ$eJN%B6A`N$J5k9<&0r4lk zMAvTo#{n<^pP+Jy(}=;Zae`Eh?>b7{_Is34LaQd}XRRZZ+AkaYV z!N-*YU=Ron6Bv>2LNA@+3=yA zqZ&*H-O{k5P#sGxf7v5xqnbLm?pudmf~y2=ea54oRH;j>f@;O6Da(q*ow$p%{qI&Q zIwb58_u5tzY%xp=NOkI>=9?pM^2msPT)w8@EtBIG=lgd|yEVBMnu$|F)dugc`FDrR@afa$ z6;)7sm5_Ko>_ED4X6z@Efey^Vko}u1oH#n@L{b)Z-J?8t?F?P6*)e2jj zQSLSK)tkP%1z$QY!f;Ly<3Rz z7Tny%jHxi#Ua4|2b1>ZWTc~bChPF9p0Pb_1YtJT0;($&Mv4uc0E+>-r;*Rdwi5b*^ z94O3?Mc!qWXb*Z}@iERVSbd;}%uU_<{aU}}7?z?yY$>CUNp*}?EzpUQ4lHLb3BPl0 zIb6oKv$hg@Qcbm3Z(&2PPFCyt{O~H}X{TnfW3|z4ylLu+%fOMaNJjNcpfobvDr}Bn zHjMs$XSrP&%!F*$pVftf>jsyZeDRK=m_}GelnSrkSJG2kF)HW1OT8GHFv4S*;k6`S z^Q#n&=A(C>-||D|{+ z(-;%!H#(`BVFdz%WIZ{!pkCD59@|!{SW+A? zHOFM7*adaD+|nNpz6UNs`>b+wP0|s!2b6KzisDZmtK~8$=o^0rRj;jd2ftky#0l2U zbJjz}+uk}h#>)3KsPtQT;vROz*^v_E(eiGjk^*DqXDD@Iqx%>@LAIm=5et>Ksv@l_ zZ1%D@HwI`v8Fg`ix*8L3bXFx|0y2@eM3@;G_75-=WJ(k(Icnevuz0kU2P>B&N*WV2 z^Y+omkVN9CNwR>xL>3iTy!3Iz<3K(>2bwvSFq+apu|ZWX(kP(O+a(WLwLR2#6wis~ zEz28N>tBR;x*^k(AG$**5VuJ5k)>A4^%HbGkzgw;1Q!vtugzYTx-*}K6l1rWzHk|| zuh7h^?6yc8L(0crMB=3f&on(%CnRsFb-R@cUVoX&;&=$oD1^c#sMJZ^zIzJSs)0;I z$R6CFQzf}YRifEVb;Rc|+~X1jo+! zkBYTi@;1UT{oQKvcQxAnAn{0soHPVQ%z&*=lq!)jR=UBlF+U09oT4g0VDC4ENqdrl zlUCDUe;6gc+oOAsJ$1#%8*dz>!9@a9+Z_WQ3#^Ay9iyZfqsQ+hC?DwZRK1tUO1_!N zK%1=8nxH__ zeLhWEAfS7NW;^aCrJuu0_K9@#;kexM&xNLG9$TX()a~qtWrX78rXD8*;1B4r?q z-Cd7SH$F+ND2WY5mzD%d)D-ly&QrEllqD5AOjO4?7?fVN- zTU!V*IplyY!38JJO}*-FNt<7N%KhrK1+>zrsD!=_C{SeyWA;dly2C{x!ZaDhMrT5r z)XmB~pdXCzd0rYe0Q%AOTJ06o$H_xWLyOV9?_9(7j!4;n^_j4GdS)|SKpmRjtxS$; z^@c`&93Pn7qz`y_^fvO2tn>VG?AwqVZzU&tB~=LR{YTFEqRZYmF}#m1Hn%;bWuMSS z$Ltxuf-IQ%#XZZQD)C~yUiGoy7rjt^l;G;!PF4F?JX<9)3J%vZ9`0>eegk8gie4oJ zU5znVm9KV5cTm?T;GBF*y)H>Qe$ABc=0|b9IIs$|>hNb3hmlc{t5%TWZWceP56nH7 zb2w_yNTuDjdX^VmGI|h7))h+H5>@FN>{)-$@_I_=-usB)t}*SY6aJZdJm*g8i>yrY zv5tXNPio84WP`UY9znKJOk&3Px$d8rym-D~b0@iS`ySth-uy6JybWxJ4CFB)Ady;w z(2NSHkR&<|SN3hPwGY)N-j;LQv7WE8P%ON@-v2Ttd4=L(*SGnIrVb?LX zuQL}B_PwesDvt~uT7;x3&ndo5+|PFo!RWkdUXiwRc-&%;Is22DdFF(UH}UTA+vU${ z@=T=JYQP-kF0Gt4_rT5=r>jBjSFpiPDQyd96wl<9x-OsCI5FSF_S91JF7bkFLZixh zncdT~K(9_zVqPhPlrd|&ZK+3VZL9gdK9Q3Dp~TQTZ~&*T*83t&&EAfg$?D9%bKzSk zbI=WZtWHI%D);j{WLC|VQ1$5@j?UkJ>n~Up63)DyWY;%LCV}ETpHH@P-GxIQtID&R znmHFZ*(@)iebu6#V`T=h+=@cna`R%#^S#n!zYgs2J*{ijFqTWr z(jwL0fR#AA-f*thI(uP{l@bCMUimgN2Xjw-rOzx^#pg%%9Guo~bz3CHx{OI+1X2w(Yt1Xxca@P%uR1Nu))8gO>`bswyBuL#SgS zMe13LeaxkF4Yd}D5o9vwHRW1jF!Q28GR`2Z+eyJK)K^*l(@9c#ch(9~3!|IEyO788 zJJStc$%)urJ=x$4e(FV|#J-*MymH&(Xj-$>goTw>-m_FUZc?g~5Sl7h^Ww5!-6HQ7 zEyAPfTv{UCJ;B5+pb63Wj1nkX-*&at&UNdW-eYH8cw2ld;*x{3R2^8Jc;-_7WC*{l znpQd2>Qff{ZWMb_#BwHd(#D>&c*(j3QI9UC2u*48Var}1_lYtuiyJFZM%#*C(=%LT zdB=06ezW|M*hl6h@iGOvfD(+2w(jAD?EQDLVJv7-ixXj=p>#|#BN8DxI)DDvy-Y&`>Agp*Zb*nyq zl37Y3X{c zvr!2@?neHegR0CUNG*z`aV7M?byt;#u3S*pc6zmVGVHNbizHSG<>R{j^+}LeV@!K5 zTDmOlO#6Oa_9tv+9%RK*2YQke2+?YdoFgY*ihj#C&9^`2aY9pARR0?=`yq3hyMVjZ zVMX(m)Q8mHfFBgQ^hqH%CokV|tcJ2yUJx>e5ILSgow_xmqQpulu0PhN1$NwaQNpOr zA@tHwwhELRT^ihKRi`Uw9n*m9F`XBYou z8qT*|ln*$luSPr6aLToI@G$Ur^`~-=k!zcv*K;4OfXhear9U*}!ksFAz**jftzOAf z{CK%-@wLR3_wC$<%~<|vLv)>RS2x+tw|LP9^A!RHx7G(DJJn961`nESGS3o?aS zpOtV4l0f+v^7qN{{x{HGKtIRz5<+DflB!atJMl|QWusiBT4e?dL256TvE2{l(j07Y zDU}v@?n;=_5jBLd+AL*~+9&PM$l?iIsiAk-6xE&Y1asZkSVKZ~XGXa4-Z?18o;brm z4Am9N#G?wO^I2nLbf`qf$!sqF%+jai6e?Pz*|tMh54UtpRyXM-zGPj+9;{|elDQ7~ z4Zccb1NPP4htO3<4PTOr-sFM2mlmzCyX4-YB>HmAm*2Ni7IiN*cId0K;OXK(KXJHta7b}kZCT-RM(gGqY%QBlbX1v!n8E$Ll0uFw<`u@~F* zMaf~N_Ckt0P&LooZfhKZol*>xqxnEkmE$AXt;bs5@?vz94Ci-8UEgSQ zchC)%!YOC0#A9d9FAZZ z3%`>StspGMR!-vtZI>u*!7fT_{Y+_fLB$|fwKN$Q?nGf~btl-qzbRbj$Feh*Aw|*K z*9~lI1*c#7FsG_%h?OWAtF~Zd35IKdQsadWlcH5)g{?9P7b_jbwh@BMgRPA(K205n zYR5ZD%2wMKo}G`K?v+5Z9ZEj1Lw^Gf+4tx37p4yGi;mQ&Xq9$S#=O>xvGp}cTq$6_ z6*CKc9XTo`=12k04WvT!u!uG?Ix6b35N&2kkVf^7-$y~I!e;XJ@aca}2rHKq%)?`{ z;7a#HrPidBUJ2^SVT-(itk!d)gT7V6=tQeirvdy9{qNtV@%woI00`+d(t6L^*ZxEn z)nT^v>o*{oedSI$Tz(+oCEe>M-}pkpwgc+-#3yf_xonQN#pOQi(78KM!Cu)1^3fUv z%H6%eZus1S@PGAm-a$=vPrMJkcMKrCw?IPgy$UEz>CywDM4Bidz4uNC(wovtzyMN1 zhfov|!5|$JM0#)DyuUm5CYk4tXC`OPb8=?)>~}xAk?(kcu=o7k{(hivaK5gJVLd|h z#qJTyDu;(UdOm*9nBr?6dFRKUg-`0$3=-isQp)G*qsNL~uRj`8&R_by#ebDyb_i|Y z&7|5mu)0%A-zB?J^+k64jesVoc#$&&8+1k_`_D+1J}f*RKs<`)VHHoJ+rt*K zWv0GoKOUdbZ1;f9QgukyuM z4&&E5K9zQ;NWMt{&g!*a7s=K;(hd;D#+jNq-(WSpAe4X2Sjd-}EG<5aP^E?8YYicB zC9@?<8E(nnRG>GBUqXnK18XUT6Ey9EtG*oO8rDUCVlKp++<`ndzVjFF&YPy5=u`x> zXSttFRc28}7nJ+e@(t>k?!bFy!_NqxUFEd91MjCtimTUu6`x`|PAO9?7p~cXFRPU_ zG=^N$`V^40zNH?00H=DEtiNEEqg3^Tn#gkP`#p4SY|E6NScyQV=|zbg{N_>IfM!`2MGz9VV^5Z?f-r-xlqN{^0zID0*&34fJO6t{O>C1dHQV^|gngTo3-D}-^J?#T(^~Z!I#+r#o6m21d*u=EI)s6qCT{W{ zpm^D>x{mA{1d!^fD)EipwBn`x`NpqHf~c{sgdDqk>q&Oyd+hP-d+L^s%unz$jj0$e zpKv##zkB61%$`4DajPgE^BJoAeJN5EO(<;@{R2;vd7>2|Bd$`l-fejg74>|dw?aIT zKh4IDc$Pn$(GXs$I`phNf7tJbz^{H?E@G>yy~aRPY`!88; zf}*p&SA$7z&8~l&LzA3;VnWR%)$UVRI$P;}Nc5-<8fYWLj*UhhG$E?q(B|_ksku=#7F_UxDa|fpu?0=OECN<)-61h!0UP5;~!FbbTwl{iW80n z1b}&WJ2l-+wI2v{!;ApJAW`yfx<)pYPVCTW@&F@*H^N@YrVJg^THp5EM|U(Y0)LWJ z7311eut=pBHeZ#(3TqJKc@y^O(Gf_v9=}M^+T$nfLu&>3W*8iQug9(XaO`wx{`I7F z@Rz6zlI4?M%d{$Hc!Ze}eA2(OWzI64DvQ;!GG|_&-=6_-%`R>H1DtStea4@5Xj5ty z*z_=>=IcO1@mt~2X3QkCcaTdx!-#4Br?`_sYHn!745>qQel?fr-(hS#X8!jATc%;{kIhgZ`1kF`D+`@$2R?6AI>CI2cr|Kp2${QT-6YwBwveT35t+ zbll&&yAIRNtnOoX9v-@;E85Wvn5d9+)nw8u{zH~)!9jm*yi=WStjt(7kXh>KDxuIDqi0+n}b@!B&&EL}+C~Sa|id58! zZ2*QhIjdIcO8`c(pj#FjIKSDJ1HZXMAGXj9TMx+U@n8nNf>!ebl=hXzJdp>onD-%B zXe=m|5(75+jTr+P!%)~DsXp&VG3nPESkKDhC}`0mmZFZ?)s<8gP)j5@SaQ1JMhi47 z^>&}~uz(6RZHJ^LgBD29Vug^8Naa1VRF^QG)3COj+}sq@QUrRzJre6U56YxalsadH z!m+sV1EFX_xC#Z=#tvpSHLrg(-?Qh}_yTmbEp~(zv_L|c^%;~Ek(w9^@X2ERhMLCm z#=!%bkq=f@^ID=7y(jd*B*{tgC#lt4Y#Oc;*9tJyh_E1`_cQ@Ie-;Km!KEJk(yi(G zuwxWlh`b~AqE1+r3nB0UrVlH12*a&O5Dh+cH0RPm2s({7O@XMu{*hGk!iCC$E1Q(m z>`nU-Eznx3PaBgiJN+k94dsC+aA!4I^mjEN;w;$OeV_nZz{V!UmxUg{3}d}qT4J*Z z^+Y{sec~cXKSHG@|3(w*#V)Maj<3?FdVLWi32sSg`$2DgytsNr;CV1{@X9zw?e!SR zE{)kA>J+%QXvji?nZ5?QB`OmOa zUSW@5s8cGrH2u()ijpq5+wbeLdf5$nff&J|=8VB#lz6U(ocx22ZpUl(8VsEw_c%T z+^TOG&W?h$R*+MD0eBLiY_6LE-I8)LozcbkjT=Ka+%>KfquH?hBb8@~lrJ-M??-dY zp0Z(29nF07z#7eL3ua}?miHGF;gxh?oJGceHP7dnPt=y3H#qO=j|IhnZkKnbFGp&B zwrk{Om(D$(H*`D_dyid)=1)3Q6C5obhBgw8KXq5Fw;F?XSk%33V~LuCTVvEbWtDB8 zFA&9~t5dy09krag9u8x`Y9t@x?bn@H0xPlOw%$@8ZeGycMkhy1%#Z_rRs`fRSrTe`HG>IY~6xf>7;lJK`qJu0X0*SxQqM} zKGTMGwF(J)+5sY}K3or=wnhzO6i8QpGlN=ci3-Uyt-`yA zk*d_bT726v-KZrXLoHeF-);hn!v>jH)(KDC*y{pp6L*LB)Oo|+oE#d^eEh+z+1mXY z*y?^(U&%5+WfI&1EkFAiK>Yga9v(EBvdW>3;f&{E3TNx0;!w=*y-^g3$!ZU_ z3t$zIgG0(*pXUOQQ&R6$*QXm;{%{f>NqSkHNMhulJy`3sxOK+YIOH#+)HgjO+3oTT zc5y%birvJ>D~y3!{_b9$OYI}6@<=e&#gP#d@kU{qm|cH+!epP+|@o zED=J`a-1=x-f|Mh{HDcypqCiysq8f#{D`BB;z-5y-NE^Z?*aC+;wJpn={WXFDots3 zD(e~Vhuy}Fi`6T_SJ*&>z(5#k$xziLd*zorsTLi3i1+b+`Coo0)p)*0cYrgiFZ|Wlu;@$=)(2&defvwfn(P=;Kz8Fr zz0V1a2v~^<*6RCRmub(R)vuH8Fw2IqB22hcOk_()`O6>*%E~xX)ddlp8o0$8nKr9_GJ70Rnq|d~PpCfYdIL#iD1 zi(6TTeL6U58ByYjmd8Rf9McEgv?78Jf+lVS2R!K}A1ZKD>Nrmu-%vIh0Yitap;18W z%sgzq0JQ$MRBnVfB{woYA?!Oq(%Pjdkdtede0gyYl9#MK4C@Sq)?Xh)9u ziwCONNYYUCTAO@XFrHyD!btYW+y>N!#r{SZNzBfSr9|ibkQERgMtq@X=&ZLp8}bkE z+iW@l8hT6=55kD+mL~@WjJ6Ql`@!wWHlf5vEM9nTmw2XL^QQA)y|kUud=9lTXXew^ zu&|r`-&t-UOLA@MwAuxFtWXG&E*a=Rcc%YBfgn&HJ}{~!ma4{39a>b+0~pEEPRa~G zs>MJNO6;&$tP=_^W;+ELQc23?>P#GJb1*vM62f$)tud_V_w;smLVAZXnJ*ab-3dBc z*t)YCKgm65vnlU{;YyXxFh*fwXA3@M9K@QpMXyOVvpWT^J^GKZjnsHfSyFMnT5c1W z0Zu{>9FED}JuM17+a~LnyuK@>1bxP$Jo+lcGBG}2e ztbw*0y?=mtJ6y0VR&T?`cLf{1YFh>qx5Z^LP1_%&1jh|d5k5qY$x{a#siX~0rt3+L zD&|AD=4hxx?Y;KIP+{Q&0gw%FuPwjHA5?)x;YIGCy#HH$48dlX+&_Thz?dP6do=6c z!?sejvxnKO8)pROQKHgO2lfuOoIw-bs~2QSMb>0I!}!B~b2IW_x%}mi2dHkD!QOqZ z^t6y0Wg0kYCdo31!htm8W?~s^`FXr9Bz%lnY9jA#u$5RCMz=AN{m3p-8fBN}Qah2U z_$rTU%jg6}nMOaqnhv`ymiG#){h3?8t%|%)ov~}3;B?jIrnTVal~1VExw*nA?{fX_ z29b><4sKhGYbzHsqffZ=5#@Q!pY5~d$9veDye@VVh(D0Pyk_>UtyAXW zQC39s5r5;KTy&4aPc^CVSo=~3SJM}phoM&81#WzP*Xwu2YdEB@mF6#w9)+Em87F70 zS+8==fsp$0WA5Ey6rMim{~gPQL(NWN*a$rDMH4WjWo{((GY@QTt7fJ(C(Tauq()^> z`%>uTo;>&m;B$uqIaqgZ3o@lZl=2Sq+!QQj)@(Us*0(j$FD5P12EQze6R*ueI7fzzt= z=B<~#t2ML`!-agnEd>hj1EIo9K`o7#-B9e-k4N<=z9l@>2O$o(C?p42aQE2q4i?Kx zB6rdglFJVFyJ{1&Fgm&r)YsaOxWtB0HnU?C;MVS%Qx#bkM!%(p;%QF%{{bv0?twRX zK$+-fWb##?n5ryGS3V4-lBz@%aJY9ocXwJz3rREE!g96HtlkM&Lklo!EI8_rKPx>w zK<{d5f%sZAWLwb!CtS~(43bjii$x30V$$g)f#Z!{SG&@vM*HFUFkiT*ji@mk+@vXm zg9otjr%0NczrTB!QbT$|^6H$5zumoND$d2ZlH)RNgUoU9fnf(Q*m8JSpl&_W7fu( z*Id1&xnp5Xie4Lg_Yqw^Mrkdz)L=ZkVdxoU0%!c#;{7e{RpbB+1+x9`)Q-9%NSm?5 zskbo)EXp&^xEpLCR>5%yQYtL(*gwG5%_d?2n(pU9iQ&u$wkd|89Qd(ZarsTMvQ4@j zXUq2PtGrC{({_*CRE_q|mR`u>2KaxUy@9ZhMl#FmxP!O+HrHPnq|`wF07egWWFKS( z_UT=;Uic388}5ol3(y?tW0_e&838U7f<)&M#hxu7RXtG{YIztp)G@T@5aICGgEpU# zW3?*ZGbkjx`Fb`h5}JjVKB#f-b!Xpk72ix@D3?af*JONofrV{Z<2iw^P7Ic_y;guJDOn|y~wzJ z@nVAJQt*mjl}(9?3Pl@Y!SX*o{Z>SzYZ-ycC&N3${D6i(9E)H0rL1itIG=WF2P`)C!P%TPHC1lB~nZ7!N4l6-ne1$ytWyYPIAHX4NjGG1Q258;y8P&lssqe@9D(hMdiW_(+b{=%c`09<*7S{7% z9(tRP(jdgye3!Z&TRN{_^0-5#=)pI6(`-L}5(>APx8Ksr}bj=Zm;} zPr1sqW1m*l5x>i6VrVk$(KHCjT<}|s*e~FY!(DO;ekWI3hNTSX0yn&r7@U|G+b!B{ z>JD4;(Wrp?qFu&-W%VZ}{nHu;n!#*))qVLtmb zzfRN{kNH-jmv?~gtp_e+<*87opm^9yv77ugn<2L=+KWXpVq@B`*fsU$&MD>$L!yis z?CxzycfHCQWk@-$mYNy9D2(NBEx0)xlDmHMvwRidfcpXxVM??IwsEQq&-gVK+- zUeoEyQ3WRNshAU!?sSRj^hbk|YXm||iKayY(+TBB%gy5as5f)-o{djnVrZWw74eeA z2hBR=QcYg0AnuKfTT8|#2l3xN8YX?llRS$|uu1AEiFNv%t({|Um-g6tksdUO|6H1f z#eQa9W>2=-TOwJb9^(UawhODftiBxid=*Owpgo(cL3EIaN$MADzxx!!Y7-T z)NbYvM{JAblYHlhhF(5_jBCM5S{cZ?Q5hgaq}P0N5pX+y(Zy#u4_fOr zDO#)qTuS2=VjnwyiLy@7p8~QB1{k=!v^ z<8d_GXtHKsL^6wue(np%OL03&$?RUrukv^jiO>_Ph+Li|gtrr8lyw{)X+$b*3y8;@ zATV4LxVjI1c~70TtD<5?MYlDe4H9ubs?52COvQ~7hX0lXki$S|Q#3lKQ#RK1qI$3J z_mhOY6fpqMqVgqw-V4#9Cw;Z43x?8G!W~{I@$^#9=Z$xEAqT@ZG zHc9?`1C~Sw(@D)7KRUy1gF-FWj@j2xF)e2N=skg_&0>qXihE)QR9Zq>CM>oz&sstO ztb<*h@QKdsIi_E024wakA6exG1ZhPh5}46B1L2@{g($-6)&OeqQc*Wk-83){QIVB@ z_+vETSMaR9fLl44AEUY?X0J@T<0z(ZVufErPHOFqq4O3c=|DS)PbcEJ&5#pC8f*5F z8y6xm!cn)4IAeyM{{`fU$&-f7txEl}b>>4oR-PjbiYVWC3o8ldtuCgRSzu*F*-M5+ zGjdmqQiKU@K!geP7Nx|9Z$@(FW;F}U*cCib z^H_PHMxP;swc19@g1OvthkBwb%(g!@uz^^6I*%hv za-yo#K!a}>gD_O;FThVAaX#skMh!sg*}|`a^r7jiw|F9!_e35W`IyQ#Q{)hv{-A(m zKYva86xp!;e7&Z9J~fuhCWpK#5r2i4>mr(#paF-2^Lq|3Fj{%?l1P5CGiVWYo#?;J zYw_?1s!ax&vqHH#^DHXaH69uQV%wD>Z5w#yL_? zxe-r%BoO15{O(;iWQAw?4oOZo#`4U0R=UrH3^54ejvD9@8Zp{+tVxytSa*1rvUos- zo-L$RfZN40TY1RWOZf)@MzOBEltPAy{IB{k{poWgRu3mL#+5ZK*$)nAlN@W!&0_|< zAM(l_l6{CoJeCp{M)5~J2TU-@CeFvxO)$rnB(^^@nA?(%mWlLAvRbE0=Ae!8F55FJ z6MacvX1=KN(!E(OzNUQv=oZX2V&9U)vco&13nzU!lmdk+#vAY7GY zmEQDYp7;~h_fUFvta*@#fOTaTaT7CeRhEEFqMJ~PjyyjH=&OklX-y!lsuLAvj(3sb zSr_m^nMy~7SdP^|{(!&8CjxsiMfv5?_nupZEbIwiQdm6O9zpNEX{6RB6gZ-)$C3{p^k3e_8u{LSeBk!9Q_i;Ud~>#_eYR zm>@~~R^x@jj_Fru`92=$X>8T{?|a6=9(fnrkyGp^?P5<<6%nF`>5NKXb47( z{Y|L9N*@8uR=IqrSJOofbLK`VH`30OgV-^Z)Xk}1*RCBhdml0CK))*L`w!sT0@iz| zqZm%dqoTVlvB+3DINks`wtJk6BifJE zB|hOhPJB9qG~ZWz=hOpPCFTB3N0m|jgdUL|(B3rFMq#jvj`j${11+>A>#V-#K1ypr zGe;JO^Jr>H4pw?l#4A%)$`KNy!5-PQ(8TRLrVib{*+(nL+@i|03Yt$!+{2Gas_+U- zp&Vt25~aRhN;fX}D`kDWOTK}Dy(X`czLArDQ+h$f-B2F>RdTX@W8G-7u4u zs>6h0geiERZumOdpAaz98uIyAkN1pv6Z^hC?(>LHiLyHt8b7{&i_*?8S7cttmte4> zNV>6Fuj+%@`x1foLz?v(^*Z7LP&z7}i=;zgsNceRZ;6IJINq_2O6U0;cSM`kvgdO4 z>*g9x%Cjj<`}Zh{?0Tb8V8crC_-|c7*0`(?x%Mu+p>~?O;}DTG3$$`_Q1#o6S6%}O zcWySUj$=34YUx$|szIS&PIxVlFuz(t)ou(DkGW)To~?VIgT-X`O~~o;wUp$iZlx=O zMa`qA9hqa+%n$l!p-W|ULCjZSkkVS)7r$uJFCR@tLe$N0`sR+(eWXJ`!pxgl-YSg-=gLfA{PmMyvtIg2qGz5}vLTlz`bY^ENXWL1c`R4-9Tp`K89pWCZwBz50Q z38;SB@aj|AXIPGbZ|(wHX7S6Hf^Isw`S5b^OXi2A!zS$m4xFt&m1IiKSw6C&L~nP5 zWRxS~oE+k{J%-qN4oS|VDMF^>>9fAHv75zq^4r-d18))^N80K`Y zTP7tj8zzl$KI(%|^i=4aS^iU1b^w$R6SUW~qr5-MQzyU^CfQ z`QAP#;gdrhtDbi?oP%;ZvfH1rammV2_(#vVB2n!^j`ofh@9FGws~C9NV+~r*pO0;D z(}iTVR|OFHG9TEHnN|9jaF1&zZ%is^1%;mq^M3uqQT2Da>NT@EXnR*8Mz{4&l_y80 z){AgFvnipvo>_0pdcX(a&0O71ZPu`0!soXUqaMvcz~@2JfG(Y_N`tjZgIk&YGHvx0OnS5U8%t=O zjP`HfiZaIRHi`4ffJAU4p`>P~)TX3xsP8KQT5QAF{`2ixYd?uktKI`4f6~Wq1&n)& zC!TEBx~KBvn781LMeDD^XZ57NPpY%Ukpf0NqSY&v4x@R-nR&e{?sh$fa z@LJ}n;5kKJDoH9N#a+MQ`^RqwP$lHnh2S{(%DGcb+M16<3%;I-938Yol1WtqmXM7WW7mH8#kvd*tJd+ywN@!f^TqfQfp+MT#_f7}EMm?*L%qL!8UVR&0}Y+X!Q<1 z{@!H_GgrqUq(sH&HWM%lRR|OLJ;&0`ZSKVT@S>!AY zXb5<5_hq8^)n!!D)*F#I_|XtYCAV4h(rSsDJb2KA`#FK9lxXBPCBHqz#X`w>p?5)i zd+ah<45~bmH{ukXImdvgR$IX+vgc2H-?f-crfTIgN85sVX@ej;J9jgm3-38hJjV#} ztFDY{etP#am)tGe(1!TQ&YfZG_yiQQO<1=Q`PF>sSE8CeITy)Tp(T2cPy(<+rC#k{ z_txo0KdGv~Q_VtOYTwU~DsK!64G$W93k~`%-u;PrVF#$oTD#9_D~T&ak$U;oJw5wV zwY#Uj=Es38Ar5C1<^`J&b(fC~?2Ria#!0X+kGD4C*ljWOGrGi)y9vbiC$zsNjN`q` zWh-BpOU8?qCVq)t73~M=TnRVd?D5-&Grc@MMwBv(6u0M;iT;&cNUd>_l~vyrkpFmK zXI|W%x_VNyT?lz!Lx4={LZg_l@msGry|JB;$qzlk1GM?!nV8riqPphlgWN;$Q`QRh z1bebU(v_If*fJ+{G*+_GS;5&hVpEvTEQeJfKPfS{*i;o73%&gK?F}RL?TpHYLs}ET z=*X2F&$6ms#zo$K*-}>GBNdIOtOHxzhK;R|?P@JLQ3fg*#dzQfbxE)~yPmi9C$@l~ zLdqT)=JSHt1>P+ul8=Ul#)}A&S^6=LJ-b+n-i=u=nU(LyoE0j?eK<}0*{oO{$>exK z;rgYckiZ=3;vXQCUmEzP{LiANr>*Ve9ev%`dy?}=AvQuJWMA?O)*p~nM+d6ju} zuGwALMZ63E&4HquRrb+tkC$9of@0!X(%5U9(ezHNJhB2Ri~PAy$Ux5S%g>($+Ltss zhK}9zA3(QBvVfV4q+dN!-=2c>Rl(ZjuNfv@1oaxBQg1O=%6SpFM`?bJz-UMK%*D0c4vm=4$`YH*lMhASc19r(~Z$2dUFM%)T1u zIw7vP#5Hh}lZ{9cE8?_%5?NnkQb|LaYfXXt6cP}B8OUHf_?|7-gK530KI3;Yv(=w} z0R5)cs5H(qg2Bdw2fyK;;wml%1L7h{28m|hn37h3=1Rph6ys>=3jl=&1D%ER6%>ma zixN2rZzHv1DHddP1t_%S?;TT`&nScLOd9Q7hOjrrAeQlj(Z|(Ffnk(3n@+MfKW`@;L>K0OFGRYHk602T@M%|2!RnF?QW= zq_K{i=Pv)ycunXyQ%n2JtkcW`|GV$)noYwV(Vz2#pc_5%ccMbJKZ~gYPpWzpEWe_^CL})(lV#coE-D?pBD1 zf>OWTr~iuC3ml1;w0p=+kMJGO;UJz_($fDF$(&|t)C&KU_8r3+&8GeHZsCp0@U_|? zmFNZD9j?jP-+ut;tZENAmwJxqy9Ck19FfP3UrhLo0f~+YI|CXQF{!b6@nn(=TwCk+ zyuV!@Qp1IuK}_xUNj$xfT_g?={x^glD7#6sb(R-)wcer@%mZK&02U93!lUKx&P`fYBK;2-j%*!M) zoQ4EQ>`zbFz(LI+PO0`3oePfLIDpuUJpr#^ye-CKK7+(J5$M}W>C42KQ6oIy4uB&V zJ@I*|M%pp>xyil~G$PtqGV&Au{Rc5i6X9MrrbS1NI+Evk5*)vOZcc0f+;*q=Pn5+gwjytg5#o2i26uH^g6V}P2T^cJF8@2 ztw_BH(K5Q4HgED8B=*_iQn^UIw-NrO#Z(?{CAWTqHc(fn>sz@V^edLY3)XSA(0vMx zJ(qnLt@G4hPEot0ob+{$XVMz#>;un^)-=;=zNyat^SRi z1LV~%oDcg)vY}ES*p2Z5{$`JA2z}W1rEl?>;2Uu;_khUbYKNNCM;9D=F(cv5m|Sr- zDc>?p`ZB|)D||lh$Md^Q z%VygpRua&pnJok-lmQiyN-VEw!mrdNbIgAIA=PJ(ZcF%J%51@9?(Z#TzF ze!>Y4H-3Kjr(-24ovWmr5%;;k=m)btd9B>g@5Dy!b?sjhSCw?NbJ<*U_Xrl%AkbOv z6<`w3iL#V}boi-8hj)^Zm4p-h7*3b3&TH3d8u;E$+I7EuLzc-Dh|ef!N$j4P zWj$EUb!DSJWDKz++9ERaS6Xd-e^KWz6LS*74Q7+!G#$UM&PO`8{+zo<)G+zW;*iuH zdx8&`;TaRUK?Jvr#rY4$=}d{APV%qGaAk%%eYk5Mm3ey6c9r6THvNO->HN}gtHCte zB+0pdsxb&@4LPMVP+N!SNhUp)o~hlzv%#{d?hdH*5uFSe?lag#?D$J}6cTKS88k&O z$^ix=$w7q=1!$j>F=qD=#6|y!qF;mKh4`*(T9VWnBpJTRNC6TozO0t)1>L1S@6n!D^sXq)t`egOW;m~^=@=#1hCH&)|$bPeO(MSoE`l%GD&g zElW#MJ=6X?xQA#Yk+$uQ%^$ewjK1*OT2+_u$^FgPm>)m&tbnl&jD2)T_YO3pa(r#y zPE3%<#fs$uKi6_VhlE^a36V17p(B?uE&D9+)y3h?YDb439@3A>IVGEAb%uGkIrNC= zBxKv6*Evw7 zQt~cyO5IqZ%&*o_^+UcH(U)IkW;I3i_VKJsz3Rp-EhbjJsy(wJD>ah%BObP_udjGx zkob*}(0TR##77V#Q86s|dpZc&5Z*$c=*0IAtk!E{KhHW!%Mr5VMu5;n{dzKZY{N0S!_G^aWIl*RO2&_i zid$w)UAJ5}yA6ZVg`F=}Pe~qf8Xgj&nOE~b)yoxr3hkuAs z4|zyD?(awNBlgTiF&FW-V;@i}9@j(IZK}qOj$30eOHBD6a<6LVA(MWSbfWvTt<0H} zYe-SQ*@0ByC*wT`e*vLBJ|g!7lqaQO`-XWLQ3U?WQBMI|CI=r0ug!$2Pu8u2(Cv+vM%|zyU}z71V;o5e`26TF2NOWQw_ZX`\n", + "* (Optional) A **Hugging Face token** if using gated models. Store it in `export HF_TOKEN=`\n", "\n", "**Note:** Depending on the organization, you can usually request access on the model's Hugging Face page. For example, Meta’s Llama models approval can take anywhere from a few hours to several weeks.\n", "\n", @@ -107,7 +107,7 @@ "\n", "Your endpoint is available locally at `http://localhost:8000` and you can use a placeholder authentication token for the OpenAI client, for example `\"FAKE_KEY\"`.\n", "\n", - "Example cURL with image URL" + "Example curl with image URL:" ] }, { @@ -121,20 +121,7 @@ "curl -X POST http://localhost:8000/v1/chat/completions \\\n", " -H \"Authorization: Bearer FAKE_KEY\" \\\n", " -H \"Content-Type: application/json\" \\\n", - " -d '{ \\\n", - " \"model\": \"my-qwen-VL\", \\\n", - " \"messages\": [ \\\n", - " { \\\n", - " \"role\": \"user\", \\\n", - " \"content\": [ \\\n", - " {\"type\": \"text\", \"text\": \"What do you see in this image?\"}, \\\n", - " {\"type\": \"image_url\", \"image_url\": { \\\n", - " \"url\": \"http://images.cocodataset.org/val2017/000000039769.jpg\" \\\n", - " }} \\\n", - " ] \\\n", - " } \\\n", - " ] \\\n", - " }'" + " -d '{ \"model\": \"my-qwen-VL\", \"messages\": [ { \"role\": \"user\", \"content\": [ {\"type\": \"text\", \"text\": \"What do you see in this image?\"}, {\"type\": \"image_url\", \"image_url\": { \"url\": \"http://images.cocodataset.org/val2017/000000039769.jpg\" }} ] } ] }'" ] }, { From bdac8cf20a71f2d099902318e5ab4871e958034e Mon Sep 17 00:00:00 2001 From: Potato Date: Tue, 9 Sep 2025 01:34:13 +0800 Subject: [PATCH 497/634] [CORE][DOC] Fix grammar, typos, and formatting issues in Ray Core documentation (#56277) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR addresses various documentation issues found during a comprehensive manual review of Ray Core documentation files in the `fault_tolerance`, `internals`, and `objects` directories. ## Issues Fixed **Grammar and Language Errors:** - Fixed "Sometime" → "Sometimes" in actor fault tolerance documentation - Added missing "to be" in serialization description: "will need to serialized" → "will need to be serialized" - Fixed grammar in tip section: "by using Actors hold objects" → "by using Actors to hold objects" - Added missing word "request" in task lifecycle workflow description **Formatting and Consistency:** - Standardized error name formatting from single backticks to double backticks for consistency with RestructuredText conventions - Fixed spacing issues: "logs-for example" → "logs--for example" - Added missing space after comma in default value formatting **Files Updated:** - `doc/source/ray-core/fault_tolerance/actors.rst` - `doc/source/ray-core/fault_tolerance/nodes.rst` - `doc/source/ray-core/fault_tolerance/objects.rst` - `doc/source/ray-core/internals/task-lifecycle.rst` - `doc/source/ray-core/objects/object-spilling.rst` - `doc/source/ray-core/objects/serialization.rst` All changes are minimal corrections focused on improving readability and consistency without altering the technical content or meaning of the documentation. --------- Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Edward Oakes Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- doc/source/ray-core/fault_tolerance/actors.rst | 4 ++-- doc/source/ray-core/fault_tolerance/nodes.rst | 2 +- doc/source/ray-core/fault_tolerance/objects.rst | 4 ++-- doc/source/ray-core/internals/task-lifecycle.rst | 2 +- doc/source/ray-core/objects/object-spilling.rst | 2 +- doc/source/ray-core/objects/serialization.rst | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/source/ray-core/fault_tolerance/actors.rst b/doc/source/ray-core/fault_tolerance/actors.rst index 6f46a6d67412..9dcbc6029a3b 100644 --- a/doc/source/ray-core/fault_tolerance/actors.rst +++ b/doc/source/ray-core/fault_tolerance/actors.rst @@ -163,7 +163,7 @@ If a task has ``max_task_retries > 0`` and it received ``ActorUnavailableError`` Actor method exceptions ----------------------- -Sometime you want to retry when an actor method raises exceptions. Use ``max_task_retries`` with ``retry_exceptions`` to retry. +Sometimes you want to retry when an actor method raises exceptions. Use ``max_task_retries`` with ``retry_exceptions`` to enable this. Note that by default, retrying on user raised exceptions is disabled. To enable it, make sure the method is **idempotent**, that is, invoking it multiple times should be equivalent to invoking it only once. @@ -180,6 +180,6 @@ Retry behavior depends on the value you set ``retry_exceptions`` to: - The method definition's value, for example, `@ray.method(max_task_retries=2)`. Ray ignores this value if you don't set it. - The actor creation call's value, for example, `Actor.options(max_task_retries=2)`. Ray ignores this value if you didn't set it. - The Actor class definition's value, for example, `@ray.remote(max_task_retries=2)` decorator. Ray ignores this value if you didn't set it. -- The default value,`0`. +- The default value, `0`. For example, if a method sets `max_task_retries=5` and `retry_exceptions=True`, and the actor sets `max_restarts=2`, Ray executes the method up to 6 times: once for the initial invocation, and 5 additional retries. The 6 invocations may include 2 actor crashes. After the 6th invocation, a `ray.get` call to the result Ray ObjectRef raises the exception raised in the last invocation, or `ray.exceptions.RayActorError` if the actor crashed in the last invocation. diff --git a/doc/source/ray-core/fault_tolerance/nodes.rst b/doc/source/ray-core/fault_tolerance/nodes.rst index 41d8959e3ae7..e106a962797a 100644 --- a/doc/source/ray-core/fault_tolerance/nodes.rst +++ b/doc/source/ray-core/fault_tolerance/nodes.rst @@ -24,6 +24,6 @@ so that when we start a new head node we still have all the cluster-level data. Raylet failure -------------- -When a raylet process fails, the corresponding node will be marked as dead and is treated the same as node failure. +When a raylet process fails, the corresponding node will be marked as dead and is treated the same as a node failure. Each raylet is associated with a unique id, so even if the raylet restarts on the same physical machine, it'll be treated as a new raylet/node to the Ray cluster. diff --git a/doc/source/ray-core/fault_tolerance/objects.rst b/doc/source/ray-core/fault_tolerance/objects.rst index 4c2987d8efd2..b363baebe9af 100644 --- a/doc/source/ray-core/fault_tolerance/objects.rst +++ b/doc/source/ray-core/fault_tolerance/objects.rst @@ -73,8 +73,8 @@ will clean up any remaining copies of the object's value to prevent a memory leak. Any workers that subsequently try to get the object's value will receive an ``OwnerDiedError`` exception, which can be handled manually. -Understanding `ObjectLostErrors` --------------------------------- +Understanding ``ObjectLostErrors`` +---------------------------------- Ray throws an ``ObjectLostError`` to the application when an object cannot be retrieved due to application or system error. This can occur during a diff --git a/doc/source/ray-core/internals/task-lifecycle.rst b/doc/source/ray-core/internals/task-lifecycle.rst index a3d47d9f7017..7c0393f09891 100644 --- a/doc/source/ray-core/internals/task-lifecycle.rst +++ b/doc/source/ray-core/internals/task-lifecycle.rst @@ -59,7 +59,7 @@ Once the task is submitted to ``NormalTaskSubmitter``, a worker process on some 2. Once all the arguments are available, ``NormalTaskSubmitter`` will try to find an idle worker to execute the task. ``NormalTaskSubmitter`` gets workers for task execution from raylet via a process called worker lease and this is where scheduling happens. Specifically, it will `send `__ a ``RequestWorkerLease`` RPC to a `selected `__ (it's either the local raylet or a data-locality-favored raylet) raylet for a worker lease. 3. Raylet `handles `__ the ``RequestWorkerLease`` RPC. -4. When the ``RequestWorkerLease`` RPC returns and a leased worker address is included in the response, a worker lease is granted to the caller to execute the task. If the ``RequestWorkerLease`` response contains another raylet address instead, ``NormalTaskSubmitter`` will then worker lease from the specified raylet. This process continues until a worker lease is obtained. +4. When the ``RequestWorkerLease`` RPC returns with a leased worker address in the response, a worker lease is granted to the caller to execute the task. If the ``RequestWorkerLease`` response contains another raylet address instead, ``NormalTaskSubmitter`` will then request a worker lease from the specified raylet. This process continues until a worker lease is obtained. Executing a task ---------------- diff --git a/doc/source/ray-core/objects/object-spilling.rst b/doc/source/ray-core/objects/object-spilling.rst index 1875a3229453..8f0ff5c7c857 100644 --- a/doc/source/ray-core/objects/object-spilling.rst +++ b/doc/source/ray-core/objects/object-spilling.rst @@ -31,7 +31,7 @@ For advanced usage and customizations, reach out to the `Ray team `_ to efficiently transfer objects across different processes and different nodes. Numpy arrays in the object store are shared between workers on the same node (zero-copy deserialization). +Since Ray processes do not share memory space, data transferred between workers and nodes will need to be **serialized** and **deserialized**. Ray uses the `Plasma object store `_ to efficiently transfer objects across different processes and different nodes. Numpy arrays in the object store are shared between workers on the same node (zero-copy deserialization). Overview -------- @@ -48,7 +48,7 @@ Numpy Arrays Ray optimizes for numpy arrays by using Pickle protocol 5 with out-of-band data. The numpy array is stored as a read-only object, and all Ray workers on the same node can read the numpy array in the object store without copying (zero-copy reads). Each numpy array object in the worker process holds a pointer to the relevant array held in shared memory. Any writes to the read-only object will require the user to first copy it into the local process memory. -.. tip:: You can often avoid serialization issues by using only native types (e.g., numpy arrays or lists/dicts of numpy arrays and other primitive types), or by using Actors hold objects that cannot be serialized. +.. tip:: You can often avoid serialization issues by using only native types (e.g., numpy arrays or lists/dicts of numpy arrays and other primitive types), or by using Actors to hold objects that cannot be serialized. Fixing "assignment destination is read-only" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 106761e30f4cdf638296752db779c2d28986dc36 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Mon, 8 Sep 2025 10:37:08 -0700 Subject: [PATCH 498/634] [core][event/04] node event: add proto schema (#56031) Add proto schema for node events. This contains all the field and not more from the existing https://github.com/ray-project/ray/blob/master/src/ray/protobuf/export_node_data.proto. It splits by the static vs dynamic state transition information, similar to other one event schema designs. The dynamic state transition information (captured by `StateTransition` in this PRs) is a vector of timestamp and State. It also includes metadata associated with the transition, such as the resource maps (which can change during a, future, to-be-added Resize state) and DeathInfo (which is set only when the node is dead). Test: - CI Signed-off-by: Cuong Nguyen --- src/ray/protobuf/public/BUILD.bazel | 25 +++++++++ .../protobuf/public/events_base_event.proto | 6 ++ .../public/events_node_definition_event.proto | 27 +++++++++ .../public/events_node_lifecycle_event.proto | 56 +++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100644 src/ray/protobuf/public/events_node_definition_event.proto create mode 100644 src/ray/protobuf/public/events_node_lifecycle_event.proto diff --git a/src/ray/protobuf/public/BUILD.bazel b/src/ray/protobuf/public/BUILD.bazel index 03a9f4b1c5f4..55f0364bb421 100644 --- a/src/ray/protobuf/public/BUILD.bazel +++ b/src/ray/protobuf/public/BUILD.bazel @@ -10,6 +10,8 @@ proto_library( ":events_actor_task_definition_event_proto", ":events_driver_job_definition_event_proto", ":events_driver_job_execution_event_proto", + ":events_node_definition_event_proto", + ":events_node_lifecycle_event_proto", ":events_task_definition_event_proto", ":events_task_execution_event_proto", "//src/ray/protobuf:events_task_profile_events_proto", @@ -92,6 +94,29 @@ cc_proto_library( deps = [":events_driver_job_execution_event_proto"], ) +proto_library( + name = "events_node_definition_event_proto", + srcs = ["events_node_definition_event.proto"], +) + +cc_proto_library( + name = "events_node_definition_event_cc_proto", + deps = [":events_node_definition_event_proto"], +) + +proto_library( + name = "events_node_lifecycle_event_proto", + srcs = ["events_node_lifecycle_event.proto"], + deps = [ + "@com_google_protobuf//:timestamp_proto", + ], +) + +cc_proto_library( + name = "events_node_lifecycle_event_cc_proto", + deps = [":events_node_lifecycle_event_proto"], +) + proto_library( name = "runtime_environment_proto", srcs = ["runtime_environment.proto"], diff --git a/src/ray/protobuf/public/events_base_event.proto b/src/ray/protobuf/public/events_base_event.proto index 859e498d3a10..af668a75c8a0 100644 --- a/src/ray/protobuf/public/events_base_event.proto +++ b/src/ray/protobuf/public/events_base_event.proto @@ -23,6 +23,8 @@ import "src/ray/protobuf/public/events_task_definition_event.proto"; import "src/ray/protobuf/public/events_task_execution_event.proto"; import "src/ray/protobuf/public/events_driver_job_definition_event.proto"; import "src/ray/protobuf/public/events_driver_job_execution_event.proto"; +import "src/ray/protobuf/public/events_node_definition_event.proto"; +import "src/ray/protobuf/public/events_node_lifecycle_event.proto"; // This is the base message for all ray events. message RayEvent { @@ -50,6 +52,8 @@ message RayEvent { TASK_PROFILE_EVENT = 4; DRIVER_JOB_DEFINITION_EVENT = 5; DRIVER_JOB_EXECUTION_EVENT = 6; + NODE_DEFINITION_EVENT = 7; + NODE_LIFECYCLE_EVENT = 8; } // The severities of events that can be generated. @@ -91,4 +95,6 @@ message RayEvent { TaskProfileEvents task_profile_events = 11; DriverJobDefinitionEvent driver_job_definition_event = 12; DriverJobExecutionEvent driver_job_execution_event = 13; + NodeDefinitionEvent node_definition_event = 14; + NodeLifecycleEvent node_lifecycle_event = 15; } diff --git a/src/ray/protobuf/public/events_node_definition_event.proto b/src/ray/protobuf/public/events_node_definition_event.proto new file mode 100644 index 000000000000..3d41a5ab27ed --- /dev/null +++ b/src/ray/protobuf/public/events_node_definition_event.proto @@ -0,0 +1,27 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ray.rpc; + +// Message containing the definition of a node, as observed via GCS. +// The message is expected to be emitted once per node creation. +// +// For runtime information associated with this event, see NodeLifecycleEvent. +message NodeDefinitionEvent { + bytes node_id = 1; + string node_ip_address = 2; + map labels = 3; +} diff --git a/src/ray/protobuf/public/events_node_lifecycle_event.proto b/src/ray/protobuf/public/events_node_lifecycle_event.proto new file mode 100644 index 000000000000..cd72ea6b5a97 --- /dev/null +++ b/src/ray/protobuf/public/events_node_lifecycle_event.proto @@ -0,0 +1,56 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +package ray.rpc; + +// Message containing the lifecycle information of a node, as observed via GCS. +// It can be used to capture the full state transition history. +// +// For static information associated with this event, see NodeDefinitionEvent. +message NodeLifecycleEvent { + enum State { + ALIVE = 0; + DEAD = 1; + } + + message DeathInfo { + enum Reason { + UNSPECIFIED = 0; + EXPECTED_TERMINATION = 1; + UNEXPECTED_TERMINATION = 2; + AUTOSCALER_DRAIN_PREEMPTED = 3; + AUTOSCALER_DRAIN_IDLE = 4; + } + Reason reason = 1; + string reason_message = 2; + } + + message StateTransition { + State state = 1; + google.protobuf.Timestamp timestamp = 2; + map resources = 3; // Resources (cpu, gpu, etc.) and their counts, + // available only in the ALIVE state. + DeathInfo death_info = 4; // Available only in the DEAD state + } + + bytes node_id = 1; + // This records the state transitions within each export interval. The consumer should + // concatenate these intervals over the node’s lifetime to reconstruct the complete + // state transition time series. + repeated StateTransition state_transitions = 2; +} From 452a575dd21f5339f9445c69d8bd880d9837467f Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Mon, 8 Sep 2025 10:37:18 -0700 Subject: [PATCH 499/634] [core][metric] Redefine STATS_tasks using Metric interface (#56015) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, we have two metric definition systems: the legacy STATS_ macros and the newer Metric/Gauge/etc. API. We only need one. This PR redefines STATS_tasks using the Metric interface. Over time, I’ll migrate all STATS_ metrics to the new system and eventually remove the old one. In this new design: - The tasks metric is defined at the `raylet/main.cc` and `core_worker_process.cc` top level. - It is passed down to sub-components through the MetricInterface. - Sub-component test cases use the newly added `MockMetricInterface` implementation. Pure refactoring, no functional changes. Test: - CI --------- Signed-off-by: Cuong Nguyen --- src/ray/core_worker/BUILD.bazel | 9 +++ src/ray/core_worker/core_worker.cc | 57 ++++++++------- src/ray/core_worker/core_worker.h | 13 +++- src/ray/core_worker/core_worker_process.cc | 6 +- src/ray/core_worker/core_worker_process.h | 3 + src/ray/core_worker/metrics.h | 45 ++++++++++++ src/ray/core_worker/task_manager.h | 27 +++++-- src/ray/core_worker/tests/BUILD.bazel | 2 + src/ray/core_worker/tests/core_worker_test.cc | 24 ++++++- .../core_worker/tests/task_manager_test.cc | 27 +++++-- src/ray/observability/BUILD.bazel | 10 +++ src/ray/observability/fake_metric.h | 70 +++++++++++++++++++ src/ray/raylet/BUILD.bazel | 1 + src/ray/raylet/lease_dependency_manager.h | 47 ++++++++----- src/ray/raylet/main.cc | 6 +- src/ray/raylet/tests/BUILD.bazel | 3 + .../tests/lease_dependency_manager_test.cc | 18 ++++- .../raylet/tests/local_lease_manager_test.cc | 5 +- src/ray/raylet/tests/node_manager_test.cc | 7 +- src/ray/stats/metric_defs.cc | 18 ----- src/ray/stats/metric_defs.h | 3 - 21 files changed, 314 insertions(+), 87 deletions(-) create mode 100644 src/ray/core_worker/metrics.h create mode 100644 src/ray/observability/fake_metric.h diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 10c55d1d387a..6cb8cff9b7f4 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -28,6 +28,7 @@ ray_cc_library( ":generator_waiter", ":grpc_service", ":memory_store", + ":metrics", ":object_recovery_manager", ":plasma_store_provider", ":profile_event", @@ -395,3 +396,11 @@ ray_cc_library( "@com_google_absl//absl/container:flat_hash_set", ], ) + +ray_cc_library( + name = "metrics", + hdrs = ["metrics.h"], + deps = [ + "//src/ray/stats:stats_lib", + ], +) diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 147e0fa58627..433baa6a0184 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -54,6 +54,8 @@ using MessageType = ray::protocol::MessageType; namespace ray::core { +using std::literals::operator""sv; + namespace { // Default capacity for serialization caches. constexpr size_t kDefaultSerializationCacheCap = 500; @@ -166,7 +168,8 @@ JobID GetProcessJobID(const CoreWorkerOptions &options) { return options.job_id; } -TaskCounter::TaskCounter() { +TaskCounter::TaskCounter(ray::observability::MetricInterface &task_by_state_counter) + : task_by_state_counter_(task_by_state_counter) { counter_.SetOnChangeCallback( [this](const std::tuple &key) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&mu_) mutable { @@ -181,37 +184,37 @@ TaskCounter::TaskCounter() { const auto is_retry_label = is_retry ? "1" : "0"; // RUNNING_IN_RAY_GET/WAIT are sub-states of RUNNING, so we need to subtract // them out to avoid double-counting. - ray::stats::STATS_tasks.Record( + task_by_state_counter_.Record( running_total - num_in_get - num_in_wait, - {{"State", rpc::TaskStatus_Name(rpc::TaskStatus::RUNNING)}, - {"Name", func_name}, - {"IsRetry", is_retry_label}, - {"JobId", job_id_}, - {"Source", "executor"}}); + {{"State"sv, rpc::TaskStatus_Name(rpc::TaskStatus::RUNNING)}, + {"Name"sv, func_name}, + {"IsRetry"sv, is_retry_label}, + {"JobId"sv, job_id_}, + {"Source"sv, "executor"}}); // Negate the metrics recorded from the submitter process for these tasks. - ray::stats::STATS_tasks.Record( + task_by_state_counter_.Record( -running_total, - {{"State", rpc::TaskStatus_Name(rpc::TaskStatus::SUBMITTED_TO_WORKER)}, - {"Name", func_name}, - {"IsRetry", is_retry_label}, - {"JobId", job_id_}, - {"Source", "executor"}}); + {{"State"sv, rpc::TaskStatus_Name(rpc::TaskStatus::SUBMITTED_TO_WORKER)}, + {"Name"sv, func_name}, + {"IsRetry"sv, is_retry_label}, + {"JobId"sv, job_id_}, + {"Source"sv, "executor"}}); // Record sub-state for get. - ray::stats::STATS_tasks.Record( + task_by_state_counter_.Record( num_in_get, - {{"State", rpc::TaskStatus_Name(rpc::TaskStatus::RUNNING_IN_RAY_GET)}, - {"Name", func_name}, - {"IsRetry", is_retry_label}, - {"JobId", job_id_}, - {"Source", "executor"}}); + {{"State"sv, rpc::TaskStatus_Name(rpc::TaskStatus::RUNNING_IN_RAY_GET)}, + {"Name"sv, func_name}, + {"IsRetry"sv, is_retry_label}, + {"JobId"sv, job_id_}, + {"Source"sv, "executor"}}); // Record sub-state for wait. - ray::stats::STATS_tasks.Record( + task_by_state_counter_.Record( num_in_wait, - {{"State", rpc::TaskStatus_Name(rpc::TaskStatus::RUNNING_IN_RAY_WAIT)}, - {"Name", func_name}, - {"IsRetry", is_retry_label}, - {"JobId", job_id_}, - {"Source", "executor"}}); + {{"State"sv, rpc::TaskStatus_Name(rpc::TaskStatus::RUNNING_IN_RAY_WAIT)}, + {"Name"sv, func_name}, + {"IsRetry"sv, is_retry_label}, + {"JobId"sv, job_id_}, + {"Source"sv, "executor"}}); }); } @@ -318,7 +321,8 @@ CoreWorker::CoreWorker( std::unique_ptr actor_manager, instrumented_io_context &task_execution_service, std::unique_ptr task_event_buffer, - uint32_t pid) + uint32_t pid, + ray::observability::MetricInterface &task_by_state_counter) : options_(std::move(options)), get_call_site_(RayConfig::instance().record_ref_creation_sites() ? options_.get_lang_stack @@ -357,6 +361,7 @@ CoreWorker::CoreWorker( task_execution_service_(task_execution_service), exiting_detail_(std::nullopt), max_direct_call_object_size_(RayConfig::instance().max_direct_call_object_size()), + task_counter_(task_by_state_counter), task_event_buffer_(std::move(task_event_buffer)), pid_(pid), actor_shutdown_callback_(std::move(options_.actor_shutdown_callback)), diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index ec5af5e24fa2..f1de5e739d7d 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -69,7 +69,7 @@ class TaskCounter { enum class TaskStatusType { kPending, kRunning, kFinished }; public: - TaskCounter(); + explicit TaskCounter(ray::observability::MetricInterface &task_by_state_counter); void BecomeActor(const std::string &actor_name) { absl::MutexLock l(&mu_); @@ -127,6 +127,14 @@ class TaskCounter { // Used for actor state tracking. std::string actor_name_ ABSL_GUARDED_BY(mu_); int64_t num_tasks_running_ ABSL_GUARDED_BY(mu_) = 0; + + // Metric to track the number of tasks by state. + // Expected tags: + // - State: the task state, as described by rpc::TaskState proto in common.proto + // - Name: the name of the function called + // - IsRetry: whether the task is a retry + // - Source: component reporting, e.g., "core_worker", "executor", or "pull_manager" + ray::observability::MetricInterface &task_by_state_counter_; }; struct TaskToRetry { @@ -190,7 +198,8 @@ class CoreWorker { std::unique_ptr actor_manager, instrumented_io_context &task_execution_service, std::unique_ptr task_event_buffer, - uint32_t pid); + uint32_t pid, + ray::observability::MetricInterface &task_by_state_counter); CoreWorker(CoreWorker const &) = delete; diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index 1c8d06f0f117..54db5f6fece9 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -456,7 +456,8 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( RAY_CHECK(addr.has_value()) << "Actor address not found for actor " << actor_id; return core_worker->core_worker_client_pool_->GetOrConnect(addr.value()); }, - gcs_client); + gcs_client, + task_by_state_counter_); auto on_excess_queueing = [this](const ActorID &actor_id, uint64_t num_queued) { auto timestamp = std::chrono::duration_cast( @@ -660,7 +661,8 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( std::move(actor_manager), task_execution_service_, std::move(task_event_buffer), - pid); + pid, + task_by_state_counter_); return core_worker; } diff --git a/src/ray/core_worker/core_worker_process.h b/src/ray/core_worker/core_worker_process.h index dd8536e3963b..9081360c02ed 100644 --- a/src/ray/core_worker/core_worker_process.h +++ b/src/ray/core_worker/core_worker_process.h @@ -20,6 +20,7 @@ #include "ray/core_worker/core_worker_options.h" #include "ray/core_worker/grpc_service.h" +#include "ray/core_worker/metrics.h" #include "ray/rpc/metrics_agent_client.h" #include "ray/util/mutex_protected.h" @@ -182,6 +183,8 @@ class CoreWorkerProcessImpl { /// The client to export metrics to the metrics agent. std::unique_ptr metrics_agent_client_; + + ray::stats::Gauge task_by_state_counter_{GetTaskMetric()}; }; } // namespace core } // namespace ray diff --git a/src/ray/core_worker/metrics.h b/src/ray/core_worker/metrics.h new file mode 100644 index 000000000000..e4a8b207ba25 --- /dev/null +++ b/src/ray/core_worker/metrics.h @@ -0,0 +1,45 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "ray/stats/metric.h" + +namespace ray { +namespace core { + +inline ray::stats::Gauge GetTaskMetric() { + /// Tracks tasks by state, including pending, running, and finished tasks. + /// This metric may be recorded from multiple components processing the task in Ray, + /// including the submitting core worker, executor core worker, and pull manager. + /// + /// To avoid metric collection conflicts between components reporting on the same task, + /// we use the "Source" required label. + return ray::stats::Gauge{ + /*name=*/"tasks", + /*description=*/"Current number of tasks currently in a particular state.", + /*unit=*/"", + // Expected tags: + // - State: the task state, as described by rpc::TaskState proto in common.proto + // - Name: the name of the function called (Keep this tag name in sync with the + // TASK_OR_ACTOR_NAME_TAG_KEY in + // python/ray/_private/telemetry/metric_cardinality.py) + // - IsRetry: whether the task is a retry + // - Source: component reporting, e.g., "core_worker", "executor", or "pull_manager" + /*tag_keys=*/{"State", "Name", "Source", "IsRetry", "JobId"}, + }; +} + +} // namespace core +} // namespace ray diff --git a/src/ray/core_worker/task_manager.h b/src/ray/core_worker/task_manager.h index 4b1c32e1e873..f4d85337a0d1 100644 --- a/src/ray/core_worker/task_manager.h +++ b/src/ray/core_worker/task_manager.h @@ -31,6 +31,7 @@ #include "ray/core_worker/task_event_buffer.h" #include "ray/core_worker/task_manager_interface.h" #include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/observability/metric_interface.h" #include "ray/stats/metric_defs.h" #include "ray/util/counter_map.h" #include "src/ray/protobuf/common.pb.h" @@ -40,6 +41,8 @@ namespace ray { namespace core { +using std::literals::operator""sv; + class ActorManager; using TaskStatusCounter = CounterMap>; @@ -182,7 +185,8 @@ class TaskManager : public TaskManagerInterface { worker::TaskEventBuffer &task_event_buffer, std::function(const ActorID &)> client_factory, - std::shared_ptr gcs_client) + std::shared_ptr gcs_client, + ray::observability::MetricInterface &task_by_state_counter) : in_memory_store_(in_memory_store), reference_counter_(reference_counter), put_in_local_plasma_callback_(std::move(put_in_local_plasma_callback)), @@ -192,16 +196,17 @@ class TaskManager : public TaskManagerInterface { max_lineage_bytes_(max_lineage_bytes), task_event_buffer_(task_event_buffer), get_actor_rpc_client_callback_(std::move(client_factory)), - gcs_client_(std::move(gcs_client)) { + gcs_client_(std::move(gcs_client)), + task_by_state_counter_(task_by_state_counter) { task_counter_.SetOnChangeCallback( [this](const std::tuple &key) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&mu_) { - ray::stats::STATS_tasks.Record( + task_by_state_counter_.Record( task_counter_.Get(key), - {{"State", rpc::TaskStatus_Name(std::get<1>(key))}, - {"Name", std::get<0>(key)}, - {"IsRetry", std::get<2>(key) ? "1" : "0"}, - {"Source", "owner"}}); + {{"State"sv, rpc::TaskStatus_Name(std::get<1>(key))}, + {"Name"sv, std::get<0>(key)}, + {"IsRetry"sv, std::get<2>(key) ? "1" : "0"}, + {"Source"sv, "owner"}}); }); reference_counter_.SetReleaseLineageCallback( [this](const ObjectID &object_id, std::vector *ids_to_release) { @@ -797,6 +802,14 @@ class TaskManager : public TaskManagerInterface { std::shared_ptr gcs_client_; + // Metric to track the number of tasks by state. + // Expected tags: + // - State: the task state, as described by rpc::TaskState proto in common.proto + // - Name: the name of the function called + // - IsRetry: whether the task is a retry + // - Source: component reporting, e.g., "core_worker", "executor", or "pull_manager" + observability::MetricInterface &task_by_state_counter_; + friend class TaskManagerTest; }; diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index ab453f5da2c5..6bd83a868c8b 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -98,6 +98,7 @@ ray_cc_test( "//src/ray/core_worker:task_event_buffer", "//src/ray/core_worker:task_manager", "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/observability:fake_metric", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], @@ -253,6 +254,7 @@ ray_cc_test( "//src/ray/core_worker:memory_store", "//src/ray/core_worker:reference_count", "//src/ray/ipc:fake_raylet_ipc_client", + "//src/ray/observability:fake_metric", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/core_worker/tests/core_worker_test.cc b/src/ray/core_worker/tests/core_worker_test.cc index 635bbe4e19fa..76535e79b74b 100644 --- a/src/ray/core_worker/tests/core_worker_test.cc +++ b/src/ray/core_worker/tests/core_worker_test.cc @@ -41,6 +41,7 @@ #include "ray/core_worker/task_submission/actor_task_submitter.h" #include "ray/core_worker/task_submission/normal_task_submitter.h" #include "ray/ipc/fake_raylet_ipc_client.h" +#include "ray/observability/fake_metric.h" #include "ray/rpc/worker/core_worker_client_pool.h" namespace ray { @@ -160,7 +161,8 @@ class CoreWorkerTest : public ::testing::Test { [](const ActorID &actor_id) { return std::make_shared(); }, - mock_gcs_client); + mock_gcs_client, + fake_task_by_state_counter_); auto object_recovery_manager = std::make_unique( rpc_address_, @@ -245,7 +247,8 @@ class CoreWorkerTest : public ::testing::Test { std::move(actor_manager), task_execution_service_, std::move(task_event_buffer), - getpid()); + getpid(), + fake_task_by_state_counter_); } protected: @@ -263,6 +266,7 @@ class CoreWorkerTest : public ::testing::Test { ActorTaskSubmitter *actor_task_submitter_; std::shared_ptr task_manager_; std::shared_ptr core_worker_; + ray::observability::FakeMetric fake_task_by_state_counter_; }; std::shared_ptr MakeRayObject(const std::string &data_str, @@ -278,6 +282,22 @@ std::shared_ptr MakeRayObject(const std::string &data_str, return std::make_shared(data, metadata, std::vector()); } +TEST_F(CoreWorkerTest, RecordMetrics) { + std::vector> results; + auto status = core_worker_->Get({}, -1, results); + ASSERT_TRUE(status.ok()); + // disconnect to trigger metric recording + core_worker_->Disconnect(rpc::WorkerExitType::SYSTEM_ERROR, "test", nullptr); + auto tag_to_value = fake_task_by_state_counter_.GetTagToValue(); + // 4 states: RUNNING, SUBMITTED_TO_WORKER, RUNNING_IN_RAY_GET and RUNNING_IN_RAY_WAIT + ASSERT_EQ(tag_to_value.size(), 4); + for (auto &[key, value] : tag_to_value) { + ASSERT_EQ(key.at("Name"), "Unknown task"); + ASSERT_EQ(key.at("Source"), "executor"); + ASSERT_EQ(key.at("IsRetry"), "0"); + } +} + TEST_F(CoreWorkerTest, HandleGetObjectStatusIdempotency) { auto object_id = ObjectID::FromRandom(); auto ray_object = MakeRayObject("test_data", "meta"); diff --git a/src/ray/core_worker/tests/task_manager_test.cc b/src/ray/core_worker/tests/task_manager_test.cc index 519a6020e9ef..d99915cbef1d 100644 --- a/src/ray/core_worker/tests/task_manager_test.cc +++ b/src/ray/core_worker/tests/task_manager_test.cc @@ -31,6 +31,7 @@ #include "ray/core_worker/reference_count.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/core_worker/task_event_buffer.h" +#include "ray/observability/fake_metric.h" namespace ray { namespace core { @@ -181,7 +182,8 @@ class TaskManagerTest : public ::testing::Test { -> std::shared_ptr { return nullptr; }, - mock_gcs_client_) {} + mock_gcs_client_, + fake_task_by_state_counter_) {} virtual void TearDown() { AssertNoLeaks(); } @@ -228,6 +230,7 @@ class TaskManagerTest : public ::testing::Test { uint32_t last_delay_ms_ = 0; bool last_object_recovery_ = false; std::unordered_set stored_in_plasma; + ray::observability::FakeMetric fake_task_by_state_counter_; }; class TaskManagerLineageTest : public TaskManagerTest { @@ -235,6 +238,19 @@ class TaskManagerLineageTest : public TaskManagerTest { TaskManagerLineageTest() : TaskManagerTest(true, /*max_lineage_bytes=*/10000) {} }; +TEST_F(TaskManagerTest, TestRecordMetrics) { + rpc::Address caller_address; + auto spec = CreateTaskHelper(1, {}); + manager_.AddPendingTask(caller_address, spec, ""); + manager_.RecordMetrics(); + auto tag_to_value = fake_task_by_state_counter_.GetTagToValue(); + ASSERT_EQ(tag_to_value.size(), 1); // one task state data point + ASSERT_EQ(tag_to_value.begin()->first.at("State"), + rpc::TaskStatus_Name(rpc::TaskStatus::PENDING_ARGS_AVAIL)); + ASSERT_EQ(tag_to_value.begin()->second, 1); // one task in the PENDING_ARGS_AVAIL state + manager_.FailPendingTask(spec.TaskId(), rpc::ErrorType::WORKER_DIED); +} + TEST_F(TaskManagerTest, TestTaskSuccess) { rpc::Address caller_address; ObjectID dep1 = ObjectID::FromRandom(); @@ -1396,7 +1412,8 @@ TEST_F(TaskManagerTest, PlasmaPut_ObjectStoreFull_FailsTaskAndWritesError) { [](const ActorID &) -> std::shared_ptr { return nullptr; }, - mock_gcs_client_); + mock_gcs_client_, + fake_task_by_state_counter_); rpc::Address caller_address; auto spec = CreateTaskHelper(1, {}); @@ -1459,7 +1476,8 @@ TEST_F(TaskManagerTest, PlasmaPut_TransientFull_RetriesThenSucceeds) { [](const ActorID &) -> std::shared_ptr { return nullptr; }, - mock_gcs_client_); + mock_gcs_client_, + fake_task_by_state_counter_); rpc::Address caller_address; auto spec = CreateTaskHelper(1, {}); @@ -1520,7 +1538,8 @@ TEST_F(TaskManagerTest, DynamicReturn_PlasmaPutFailure_FailsTaskImmediately) { [](const ActorID &) -> std::shared_ptr { return nullptr; }, - mock_gcs_client_); + mock_gcs_client_, + fake_task_by_state_counter_); auto spec = CreateTaskHelper(1, {}, /*dynamic_returns=*/true); dyn_mgr.AddPendingTask(addr_, spec, "", /*num_retries=*/0); diff --git a/src/ray/observability/BUILD.bazel b/src/ray/observability/BUILD.bazel index 623bd3ab6a53..ebdfb1aacaed 100644 --- a/src/ray/observability/BUILD.bazel +++ b/src/ray/observability/BUILD.bazel @@ -24,3 +24,13 @@ ray_cc_library( "@io_opencensus_cpp//opencensus/stats", ], ) + +ray_cc_library( + name = "fake_metric", + hdrs = [ + "fake_metric.h", + ], + deps = [ + ":metric_interface", + ], +) diff --git a/src/ray/observability/fake_metric.h b/src/ray/observability/fake_metric.h new file mode 100644 index 000000000000..8cafb45ded68 --- /dev/null +++ b/src/ray/observability/fake_metric.h @@ -0,0 +1,70 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "ray/observability/metric_interface.h" + +namespace ray { +namespace observability { + +class FakeMetric : public MetricInterface { + public: + FakeMetric() = default; + ~FakeMetric() = default; + + void Record(double value) override { Record(value, stats::TagsType{}); } + + void Record(double value, stats::TagsType tags) override { + absl::flat_hash_map tags_map; + for (const auto &tag : tags) { + tags_map[tag.first.name()] = tag.second; + } + tag_to_value_.emplace(std::move(tags_map), value); + } + + void Record(double value, + const std::unordered_map &tags) override { + stats::TagsType tags_pair_vec; + tags_pair_vec.reserve(tags.size()); + std::for_each(tags.begin(), tags.end(), [&tags_pair_vec](auto &tag) { + return tags_pair_vec.emplace_back(stats::TagKeyType::Register(tag.first), + std::move(tag.second)); + }); + Record(value, std::move(tags_pair_vec)); + } + + void Record(double value, + const std::unordered_map &tags) override { + stats::TagsType tags_pair_vec; + tags_pair_vec.reserve(tags.size()); + std::for_each(tags.begin(), tags.end(), [&tags_pair_vec](auto &tag) { + return tags_pair_vec.emplace_back(stats::TagKeyType::Register(tag.first), + std::move(tag.second)); + }); + Record(value, std::move(tags_pair_vec)); + } + + const absl::flat_hash_map, double> + &GetTagToValue() const { + return tag_to_value_; + } + + private: + absl::flat_hash_map, double> + tag_to_value_; +}; + +} // namespace observability +} // namespace ray diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index 854a8d2478ee..f8b4b2637aeb 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -283,6 +283,7 @@ ray_cc_binary( "//src/ray/common:ray_config", "//src/ray/common:status", "//src/ray/common/cgroup:cgroup_manager", + "//src/ray/core_worker:metrics", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/object_manager:ownership_object_directory", "//src/ray/raylet/scheduling:cluster_lease_manager", diff --git a/src/ray/raylet/lease_dependency_manager.h b/src/ray/raylet/lease_dependency_manager.h index cb72ebd93405..358a3e3cad0c 100644 --- a/src/ray/raylet/lease_dependency_manager.h +++ b/src/ray/raylet/lease_dependency_manager.h @@ -30,6 +30,8 @@ namespace ray { namespace raylet { +using std::literals::operator""sv; + /// Used for unit-testing the ClusterLeaseManager, which requests dependencies /// for queued leases. class LeaseDependencyManagerInterface { @@ -55,8 +57,10 @@ class LeaseDependencyManagerInterface { class LeaseDependencyManager : public LeaseDependencyManagerInterface { public: /// Create a lease dependency manager. - explicit LeaseDependencyManager(ObjectManagerInterface &object_manager) - : object_manager_(object_manager) { + explicit LeaseDependencyManager( + ObjectManagerInterface &object_manager, + ray::observability::MetricInterface &task_by_state_counter) + : object_manager_(object_manager), task_by_state_counter_(task_by_state_counter) { waiting_leases_counter_.SetOnChangeCallback( [this](std::pair key) mutable { int64_t num_total = waiting_leases_counter_.Get(key); @@ -66,25 +70,26 @@ class LeaseDependencyManager : public LeaseDependencyManagerInterface { int64_t num_inactive = std::min( num_total, object_manager_.PullManagerNumInactivePullsByTaskName(key)); // Offset the metric values recorded from the owner process. - ray::stats::STATS_tasks.Record( + task_by_state_counter_.Record( -num_total, - {{"State", rpc::TaskStatus_Name(rpc::TaskStatus::PENDING_NODE_ASSIGNMENT)}, - {"Name", key.first}, - {"IsRetry", key.second ? "1" : "0"}, - {"Source", "dependency_manager"}}); - ray::stats::STATS_tasks.Record( + {{"State"sv, + rpc::TaskStatus_Name(rpc::TaskStatus::PENDING_NODE_ASSIGNMENT)}, + {"Name"sv, key.first}, + {"IsRetry"sv, key.second ? "1" : "0"}, + {"Source"sv, "dependency_manager"}}); + task_by_state_counter_.Record( num_total - num_inactive, - {{"State", rpc::TaskStatus_Name(rpc::TaskStatus::PENDING_ARGS_FETCH)}, - {"Name", key.first}, - {"IsRetry", key.second ? "1" : "0"}, - {"Source", "dependency_manager"}}); - ray::stats::STATS_tasks.Record( + {{"State"sv, rpc::TaskStatus_Name(rpc::TaskStatus::PENDING_ARGS_FETCH)}, + {"Name"sv, key.first}, + {"IsRetry"sv, key.second ? "1" : "0"}, + {"Source"sv, "dependency_manager"}}); + task_by_state_counter_.Record( num_inactive, - {{"State", + {{"State"sv, rpc::TaskStatus_Name(rpc::TaskStatus::PENDING_OBJ_STORE_MEM_AVAIL)}, - {"Name", key.first}, - {"IsRetry", key.second ? "1" : "0"}, - {"Source", "dependency_manager"}}); + {"Name"sv, key.first}, + {"IsRetry"sv, key.second ? "1" : "0"}, + {"Source"sv, "dependency_manager"}}); }); } @@ -317,6 +322,14 @@ class LeaseDependencyManager : public LeaseDependencyManagerInterface { /// total will be less than or equal to the size of queued_lease_requests_. CounterMap waiting_leases_counter_; + // Metric to track the number of tasks by state. + // Expected tags: + // - State: the task state, as described by rpc::TaskState proto in common.proto + // - Name: the name of the function called + // - IsRetry: whether the task is a retry + // - Source: component reporting, e.g., "core_worker", "executor", or "pull_manager" + ray::observability::MetricInterface &task_by_state_counter_; + friend class LeaseDependencyManagerTest; }; diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index e96b52f195e9..9a433e473ebb 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -30,6 +30,7 @@ #include "ray/common/lease/lease.h" #include "ray/common/ray_config.h" #include "ray/common/status.h" +#include "ray/core_worker/metrics.h" #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/object_manager/ownership_object_directory.h" #include "ray/raylet/local_object_manager.h" @@ -262,6 +263,7 @@ int main(int argc, char *argv[]) { RAY_CHECK_OK(gcs_client->Connect(main_service)); std::unique_ptr raylet; + ray::stats::Gauge task_by_state_counter = ray::core::GetTaskMetric(); std::unique_ptr plasma_client; std::unique_ptr node_manager; std::unique_ptr client_call_manager; @@ -680,8 +682,8 @@ int main(int argc, char *argv[]) { /*core_worker_subscriber_=*/core_worker_subscriber.get(), object_directory.get()); - lease_dependency_manager = - std::make_unique(*object_manager); + lease_dependency_manager = std::make_unique( + *object_manager, task_by_state_counter); cluster_resource_scheduler = std::make_unique( main_service, diff --git a/src/ray/raylet/tests/BUILD.bazel b/src/ray/raylet/tests/BUILD.bazel index 25321d7d3fab..1577fce0758a 100644 --- a/src/ray/raylet/tests/BUILD.bazel +++ b/src/ray/raylet/tests/BUILD.bazel @@ -102,6 +102,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:lease", "//src/ray/common:test_utils", + "//src/ray/observability:fake_metric", "//src/ray/raylet:lease_dependency_manager", "@com_google_googletest//:gtest_main", ], @@ -119,6 +120,7 @@ ray_cc_test( "//src/ray/common:lease", "//src/ray/common:task_common", "//src/ray/common:test_utils", + "//src/ray/observability:fake_metric", "//src/ray/raylet:local_lease_manager", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "@com_google_googletest//:gtest_main", @@ -184,6 +186,7 @@ ray_cc_test( "//src/ray/common:ray_object", "//src/ray/common:task_common", "//src/ray/object_manager/plasma:plasma_client", + "//src/ray/observability:fake_metric", "//src/ray/raylet:local_object_manager_interface", "//src/ray/raylet:node_manager", "//src/ray/raylet/scheduling:cluster_lease_manager", diff --git a/src/ray/raylet/tests/lease_dependency_manager_test.cc b/src/ray/raylet/tests/lease_dependency_manager_test.cc index 906472c71d46..250032273c36 100644 --- a/src/ray/raylet/tests/lease_dependency_manager_test.cc +++ b/src/ray/raylet/tests/lease_dependency_manager_test.cc @@ -24,6 +24,7 @@ #include "gtest/gtest.h" #include "mock/ray/object_manager/object_manager.h" #include "ray/common/test_utils.h" +#include "ray/observability/fake_metric.h" namespace ray { @@ -69,7 +70,9 @@ class CustomMockObjectManager : public MockObjectManager { class LeaseDependencyManagerTest : public ::testing::Test { public: LeaseDependencyManagerTest() - : object_manager_mock_(), lease_dependency_manager_(object_manager_mock_) {} + : object_manager_mock_(), + fake_task_by_state_counter_(), + lease_dependency_manager_(object_manager_mock_, fake_task_by_state_counter_) {} int64_t NumWaiting(const std::string &lease_name) { return lease_dependency_manager_.waiting_leases_counter_.Get({lease_name, false}); @@ -92,9 +95,22 @@ class LeaseDependencyManagerTest : public ::testing::Test { } CustomMockObjectManager object_manager_mock_; + ray::observability::FakeMetric fake_task_by_state_counter_; LeaseDependencyManager lease_dependency_manager_; }; +TEST_F(LeaseDependencyManagerTest, TestRecordMetrics) { + auto obj_id = ObjectID::FromRandom(); + lease_dependency_manager_.RequestLeaseDependencies( + LeaseID::FromRandom(), ObjectIdsToRefs({obj_id}), {"foo", false}); + lease_dependency_manager_.HandleObjectLocal(obj_id); + lease_dependency_manager_.RecordMetrics(); + auto tag_to_value = fake_task_by_state_counter_.GetTagToValue(); + // 3 states: PENDING_NODE_ASSIGNMENT, PENDING_ARGS_FETCH, PENDING_OBJ_STORE_MEM_AVAIL + ASSERT_EQ(tag_to_value.size(), 3); + ASSERT_EQ(tag_to_value.begin()->first.at("Name"), "foo"); +} + /// Test requesting the dependencies for a lease. The dependency manager should /// return the lease ID as ready once all of its arguments are local. TEST_F(LeaseDependencyManagerTest, TestSimpleLease) { diff --git a/src/ray/raylet/tests/local_lease_manager_test.cc b/src/ray/raylet/tests/local_lease_manager_test.cc index faae316bfba9..4830b029c9ce 100644 --- a/src/ray/raylet/tests/local_lease_manager_test.cc +++ b/src/ray/raylet/tests/local_lease_manager_test.cc @@ -31,6 +31,7 @@ #include "ray/common/lease/lease.h" #include "ray/common/task/task_util.h" #include "ray/common/test_utils.h" +#include "ray/observability/fake_metric.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/tests/util.h" @@ -316,7 +317,8 @@ class LocalLeaseManagerTest : public ::testing::Test { id_(NodeID::FromRandom()), scheduler_(CreateSingleNodeScheduler(id_.Binary(), num_cpus, *gcs_client_)), object_manager_(), - lease_dependency_manager_(object_manager_), + fake_task_by_state_counter_(), + lease_dependency_manager_(object_manager_, fake_task_by_state_counter_), local_lease_manager_(std::make_shared( id_, *scheduler_, @@ -373,6 +375,7 @@ class LocalLeaseManagerTest : public ::testing::Test { absl::flat_hash_map node_info_; MockObjectManager object_manager_; + ray::observability::FakeMetric fake_task_by_state_counter_; LeaseDependencyManager lease_dependency_manager_; std::shared_ptr local_lease_manager_; }; diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index 5cda36052449..5dce1b2d3ec7 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -35,6 +35,7 @@ #include "ray/common/buffer.h" #include "ray/common/scheduling/cluster_resource_data.h" #include "ray/object_manager/plasma/client.h" +#include "ray/observability/fake_metric.h" #include "ray/raylet/local_object_manager_interface.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/tests/util.h" @@ -395,6 +396,7 @@ class NodeManagerTest : public ::testing::Test { core_worker_subscriber_ = std::make_unique(); mock_object_directory_ = std::make_unique(); mock_object_manager_ = std::make_unique(); + fake_task_by_state_counter_ = ray::observability::FakeMetric(); EXPECT_CALL(*mock_object_manager_, GetMemoryCapacity()).WillRepeatedly(Return(0)); @@ -418,8 +420,8 @@ class NodeManagerTest : public ::testing::Test { local_object_manager_ = std::make_unique(objects_pending_deletion_); - lease_dependency_manager_ = - std::make_unique(*mock_object_manager_); + lease_dependency_manager_ = std::make_unique( + *mock_object_manager_, fake_task_by_state_counter_); cluster_resource_scheduler_ = std::make_unique( io_service_, @@ -527,6 +529,7 @@ class NodeManagerTest : public ::testing::Test { MockWorkerPool mock_worker_pool_; absl::flat_hash_map> leased_workers_; std::shared_ptr> objects_pending_deletion_; + ray::observability::FakeMetric fake_task_by_state_counter_; }; TEST_F(NodeManagerTest, TestRegisterGcsAndCheckSelfAlive) { diff --git a/src/ray/stats/metric_defs.cc b/src/ray/stats/metric_defs.cc index d3dc780a84de..b42d661eec64 100644 --- a/src/ray/stats/metric_defs.cc +++ b/src/ray/stats/metric_defs.cc @@ -37,24 +37,6 @@ namespace ray::stats { /// =========== PUBLIC METRICS; keep in sync with ray-metrics.rst ================= /// =============================================================================== -/// Tracks tasks by state, including pending, running, and finished tasks. -/// This metric may be recorded from multiple components processing the task in Ray, -/// including the submitting core worker, executor core worker, and pull manager. -/// -/// To avoid metric collection conflicts between components reporting on the same task, -/// we use the "Source" required label. -DEFINE_stats( - tasks, - "Current number of tasks currently in a particular state.", - // State: the task state, as described by rpc::TaskState proto in common.proto. - // Name: the name of the function called (Keep in sync with the - // TASK_OR_ACTOR_NAME_TAG_KEY in python/ray/_private/telemetry/metric_cardinality.py) - // Source: component reporting, e.g., "core_worker", "executor", or "pull_manager". - // IsRetry: whether this task is a retry. - ("State", "Name", "Source", "IsRetry", "JobId"), - (), - ray::stats::GAUGE); - /// Tracks actors by state, including pending, running, and idle actors. /// /// To avoid metric collection conflicts between components reporting on the same task, diff --git a/src/ray/stats/metric_defs.h b/src/ray/stats/metric_defs.h index dfe22aa345c4..8a78603f3968 100644 --- a/src/ray/stats/metric_defs.h +++ b/src/ray/stats/metric_defs.h @@ -42,9 +42,6 @@ namespace stats { /// ray_[component]_[metrics_name]_total (e.g., ray_pull_manager_total) /// -/// Tasks stats, broken down by state. -DECLARE_stats(tasks); - /// Actor stats, broken down by state. DECLARE_stats(actors); From 9536fcafb6e6834c0bde6d65ba8a3ac0de14b8bf Mon Sep 17 00:00:00 2001 From: Ibrahim Rabbani Date: Mon, 8 Sep 2025 10:40:04 -0700 Subject: [PATCH 500/634] [core] (cgroups 5/n) Adding clean up methods to CgroupDriverInterface and SysFsCgroupDriver (#56255) This PR stacks on #56246. For more details about the resource isolation project see https://github.com/ray-project/ray/issues/54703. This PR adds the DeleteCgroup method (along with unit and integration tests) to the CgroupDriverInterface and the SysFsCgroupDriver to allow CgroupManager to clean up the cgroup hierarchy is graceful shutdown. --------- Signed-off-by: Ibrahim Rabbani Co-authored-by: Edward Oakes --- .../common/cgroup2/cgroup_driver_interface.h | 21 ++++- src/ray/common/cgroup2/cgroup_test_utils.cc | 17 +++- src/ray/common/cgroup2/fake_cgroup_driver.h | 10 +- .../sysfs_cgroup_driver_integration_test.cc | 94 +++++++++++++++++++ src/ray/common/cgroup2/sysfs_cgroup_driver.cc | 45 +++++++-- src/ray/common/cgroup2/sysfs_cgroup_driver.h | 19 ++++ .../cgroup2/tests/sysfs_cgroup_driver_test.cc | 17 ++++ 7 files changed, 206 insertions(+), 17 deletions(-) diff --git a/src/ray/common/cgroup2/cgroup_driver_interface.h b/src/ray/common/cgroup2/cgroup_driver_interface.h index 62e9f47f7b9a..01f61c68e0e7 100644 --- a/src/ray/common/cgroup2/cgroup_driver_interface.h +++ b/src/ray/common/cgroup2/cgroup_driver_interface.h @@ -68,19 +68,34 @@ class CgroupDriverInterface { /** Creates a new cgroup at the specified path. + Expects all cgroups on the path from root -> the new cgroup to already exist. Expects the user to have read, write, and execute privileges to parent cgroup. @param cgroup is an absolute path to the cgroup - @return Status::OK if no errors are encounted. Otherwise, one of the following errors + @return Status::OK if no errors are encounted. @return Status::NotFound if an ancestor cgroup does not exist. - @return Status::PermissionDenied if current user doesn't have read, write, and execute - permissions. + @return Status::PermissionDenied if the process doesn't have sufficient permissions. @return Status::AlreadyExists if the cgroup already exists. */ virtual Status CreateCgroup(const std::string &cgroup) = 0; + /** + Deletes the specified cgroup. + + Expects all cgroups from the root -> the specified cgroup to exist. + Expects the cgroup to have no children. + Expects the process to have adequate permissions for the parent cgroup. + + @param cgroup is an absolute path to the cgroup + + @return Status::OK if no errors are encounted. + @return Status::NotFound if an ancestor cgroup does not exist. + @return Status::PermissionDenied if the process doesn't have sufficient permissions. + */ + virtual Status DeleteCgroup(const std::string &cgroup) = 0; + /** Move all processes from one cgroup to another. The process must have read, write, and execute permissions for both cgroups and their lowest common ancestor. diff --git a/src/ray/common/cgroup2/cgroup_test_utils.cc b/src/ray/common/cgroup2/cgroup_test_utils.cc index 82452818eefc..49939b576153 100644 --- a/src/ray/common/cgroup2/cgroup_test_utils.cc +++ b/src/ray/common/cgroup2/cgroup_test_utils.cc @@ -62,11 +62,18 @@ ray::StatusOr> TempCgroupDirectory::Create( } TempCgroupDirectory::~TempCgroupDirectory() noexcept(false) { - RAY_CHECK(rmdir(path_.c_str()) != -1) << absl::StrFormat( - "Failed to delete a cgroup directory at %s with error %s. Please manually " - "delete it with rmdir.", - path_, - strerror(errno)); + // TODO(#54703): This can be refactored to disarm the destructor so that when you delete + // a cgroup created with TempCgroupDirectory and delete it outside the handler, this + // will not attempt to delete it. + if (rmdir(path_.c_str()) == -1) { + if (errno != ENOENT) { + RAY_LOG(WARNING) << absl::StrFormat( + "Failed to delete a cgroup directory at %s with error %s. Please manually " + "delete it with rmdir.", + path_, + strerror(errno)); + } + } } ray::StatusOr> TempDirectory::Create() { diff --git a/src/ray/common/cgroup2/fake_cgroup_driver.h b/src/ray/common/cgroup2/fake_cgroup_driver.h index 0fe1ad041c85..6245d0fc1f5b 100644 --- a/src/ray/common/cgroup2/fake_cgroup_driver.h +++ b/src/ray/common/cgroup2/fake_cgroup_driver.h @@ -67,6 +67,7 @@ class FakeCgroupDriver : public CgroupDriverInterface { Status check_cgroup_enabled_s_ = Status::OK(); Status check_cgroup_s_ = Status::OK(); Status create_cgroup_s_ = Status::OK(); + Status delete_cgroup_s_ = Status::OK(); Status move_all_processes_s_ = Status::OK(); Status enable_controller_s_ = Status::OK(); Status disable_controller_s_ = Status::OK(); @@ -82,7 +83,6 @@ class FakeCgroupDriver : public CgroupDriverInterface { // All of them can be short-circuited by setting the corresponding // status to not ok. Status CreateCgroup(const std::string &cgroup) override { - RAY_LOG(INFO) << "CreateCgroup " << cgroup; if (!create_cgroup_s_.ok()) { return create_cgroup_s_; } @@ -90,6 +90,14 @@ class FakeCgroupDriver : public CgroupDriverInterface { return create_cgroup_s_; } + Status DeleteCgroup(const std::string &cgroup) override { + if (!delete_cgroup_s_.ok()) { + return delete_cgroup_s_; + } + cgroups_->erase(cgroup); + return delete_cgroup_s_; + } + Status MoveAllProcesses(const std::string &from, const std::string &to) override { if (!move_all_processes_s_.ok()) { return move_all_processes_s_; diff --git a/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc b/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc index ab3312f27f25..3be47faaf5bd 100644 --- a/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc +++ b/src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. #include +#include #include #include @@ -154,6 +155,99 @@ TEST_F(SysFsCgroupDriverIntegrationTest, << "Error: " << strerror(errno); } +// Tests for DeleteCgroup +TEST_F(SysFsCgroupDriverIntegrationTest, DeleteCgroupFailsIfDoesNotExist) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup = std::move(cgroup_dir_or_status.value()); + std::string cgroup_to_delete = + cgroup->GetPath() + std::filesystem::path::preferred_separator + "cool_group"; + SysFsCgroupDriver driver; + Status s = driver.DeleteCgroup(cgroup_to_delete); + ASSERT_TRUE(s.IsNotFound()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, DeleteCgroupFailsIfAncestorCgroupDoesNotExist) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + std::string non_existent_path = cgroup_dir->GetPath() + + std::filesystem::path::preferred_separator + "no" + + std::filesystem::path::preferred_separator + "bueno"; + Status s = driver.DeleteCgroup(non_existent_path); + EXPECT_TRUE(s.IsNotFound()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, DeleteCgroupFailsIfOnlyReadPermissions) { + auto cgroup_dir_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + std::string child_cgroup_path = + cgroup_dir->GetPath() + std::filesystem::path::preferred_separator + "child"; + Status s = driver.DeleteCgroup(child_cgroup_path); + EXPECT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, DeleteCgroupFailsIfOnlyReadWritePermissions) { + auto cgroup_dir_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRUSR | S_IWUSR); + ASSERT_TRUE(cgroup_dir_or_status.ok()) << cgroup_dir_or_status.ToString(); + auto cgroup_dir = std::move(cgroup_dir_or_status.value()); + SysFsCgroupDriver driver; + std::string child_cgroup_path = + cgroup_dir->GetPath() + std::filesystem::path::preferred_separator + "child"; + Status s = driver.DeleteCgroup(child_cgroup_path); + EXPECT_TRUE(s.IsPermissionDenied()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, DeleteCgroupFailsIfCgroupHasChildren) { + auto parent_cgroup_dir_or_status = + TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(parent_cgroup_dir_or_status.ok()) << parent_cgroup_dir_or_status.ToString(); + std::unique_ptr parent_cgroup = + std::move(parent_cgroup_dir_or_status.value()); + auto child_cgroup_dir_or_status = + TempCgroupDirectory::Create(parent_cgroup->GetPath(), S_IRWXU); + ASSERT_TRUE(child_cgroup_dir_or_status.ok()) << child_cgroup_dir_or_status.ToString(); + SysFsCgroupDriver driver; + Status s = driver.DeleteCgroup(parent_cgroup->GetPath()); + EXPECT_TRUE(s.IsInvalidArgument()) << s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, DeleteCgroupFailsIfCgroupHasProcesses) { + auto cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); + auto cgroup = std::move(cgroup_or_status.value()); + StatusOr> child_process = + StartChildProcessInCgroup(cgroup->GetPath()); + ASSERT_TRUE(child_process.ok()) << child_process.ToString(); + auto [child_pid, child_pidfd] = *child_process; + SysFsCgroupDriver driver; + // Delete fails while process is alive. + Status failed_s = driver.DeleteCgroup(cgroup->GetPath()); + EXPECT_TRUE(failed_s.IsInvalidArgument()) << failed_s.ToString(); + Status terminate_child = + TerminateChildProcessAndWaitForTimeout(child_pid, child_pidfd, 5000); + ASSERT_TRUE(terminate_child.ok()) << terminate_child.ToString(); + // Delete succeeds after child process terminates. + Status succeeded_s = driver.DeleteCgroup(cgroup->GetPath()); + EXPECT_TRUE(succeeded_s.ok()) << succeeded_s.ToString(); +} + +TEST_F(SysFsCgroupDriverIntegrationTest, + DeleteCgroupSucceedsIfLeafCgroupExistsWithNoProcessesAndCorrectPermissions) { + auto cgroup_or_status = TempCgroupDirectory::Create(test_cgroup_path_, S_IRWXU); + ASSERT_TRUE(cgroup_or_status.ok()) << cgroup_or_status.ToString(); + auto cgroup = std::move(cgroup_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.DeleteCgroup(cgroup->GetPath()); + EXPECT_TRUE(s.ok()) << s.ToString(); +} + +// RemoveController tests + TEST_F(SysFsCgroupDriverIntegrationTest, GetAvailableControllersFailsIfCgroupDoesNotExist) { std::string non_existent_path = test_cgroup_path_ + diff --git a/src/ray/common/cgroup2/sysfs_cgroup_driver.cc b/src/ray/common/cgroup2/sysfs_cgroup_driver.cc index bf29d8cafa66..b36960c6127c 100644 --- a/src/ray/common/cgroup2/sysfs_cgroup_driver.cc +++ b/src/ray/common/cgroup2/sysfs_cgroup_driver.cc @@ -134,14 +134,14 @@ Status SysFsCgroupDriver::CreateCgroup(const std::string &cgroup_path) { strerror(errno))); } if (errno == EACCES) { - return Status::PermissionDenied(absl::StrFormat( - "Failed to create cgroup at path %s with permissions %#o. " - "The current user does not have read, write, execute permissions " - "for the parent cgroup.\n" - "Error: %s.", - cgroup_path, - S_IRWXU, - strerror(errno))); + return Status::PermissionDenied( + absl::StrFormat("Failed to create cgroup at path %s with permissions %#o. " + "The process does not have read, write, execute permissions " + "for the parent cgroup.\n" + "Error: %s.", + cgroup_path, + S_IRWXU, + strerror(errno))); } if (errno == EEXIST) { return Status::AlreadyExists( @@ -162,6 +162,35 @@ Status SysFsCgroupDriver::CreateCgroup(const std::string &cgroup_path) { return Status::OK(); } +Status SysFsCgroupDriver::DeleteCgroup(const std::string &cgroup_path) { + RAY_RETURN_NOT_OK(CheckCgroup(cgroup_path)); + if (rmdir(cgroup_path.c_str()) == -1) { + if (errno == ENOENT) { + return Status::NotFound(absl::StrFormat( + "Failed to delete cgroup at path %s. The parent cgroup does not exist.\n" + "Error: %s.", + cgroup_path, + strerror(errno))); + } + if (errno == EACCES) { + return Status::PermissionDenied( + absl::StrFormat("Failed to delete cgroup at path %s. " + "The process does not have read, write, execute permissions " + "for the parent cgroup.\n" + "Error: %s.", + cgroup_path, + strerror(errno))); + } + return Status::InvalidArgument( + absl::StrFormat("Failed to delete cgroup at path %s. To delete a cgroup, it must " + "have no children and it must not have any processes.\n" + "Error: %s.", + cgroup_path, + strerror(errno))); + } + return Status::OK(); +} + StatusOr> SysFsCgroupDriver::GetAvailableControllers( const std::string &cgroup_dir) { RAY_RETURN_NOT_OK(CheckCgroup(cgroup_dir)); diff --git a/src/ray/common/cgroup2/sysfs_cgroup_driver.h b/src/ray/common/cgroup2/sysfs_cgroup_driver.h index 2f654115d282..de5104caeec6 100644 --- a/src/ray/common/cgroup2/sysfs_cgroup_driver.h +++ b/src/ray/common/cgroup2/sysfs_cgroup_driver.h @@ -121,6 +121,25 @@ class SysFsCgroupDriver : public CgroupDriverInterface { */ Status CreateCgroup(const std::string &cgroup_path) override; + /** + To delete a cgroup using the cgroupv2 vfs, the current user needs to read, write, and + execute permissions for the parent cgroup. This can be achieved through cgroup + delegation. The cgroup must also have no processes or children. + + @see The relevant manpage section on delegation for more details + https://docs.kernel.org/admin-guide/cgroup-v2.html#delegation + + @param cgroup_path the absolute path of the cgroup directory to create. + + @return Status::OK if no errors are encounted. + @return Status::NotFound if an ancestor cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::InvalidArgument if the cgroup has children, processes, or for any + other reason. + */ + Status DeleteCgroup(const std::string &cgroup_path) override; + /** Parses the cgroup.controllers file which has a space separated list of all controllers available to the cgroup. diff --git a/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc b/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc index 70d123c5f5fb..0d712c1443c3 100644 --- a/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc +++ b/src/ray/common/cgroup2/tests/sysfs_cgroup_driver_test.cc @@ -87,6 +87,23 @@ TEST(SysFsCgroupDriver, CheckCgroupFailsIfCgroupDoesNotExist) { EXPECT_TRUE(s.IsNotFound()) << s.ToString(); } +TEST(SysFsCgroupDriver, DeleteCgroupFailsIfNotCgroup2Path) { + // This is not a directory on the cgroupv2 vfs. + auto temp_dir_or_status = TempDirectory::Create(); + ASSERT_TRUE(temp_dir_or_status.ok()) << temp_dir_or_status.ToString(); + std::unique_ptr temp_dir = std::move(temp_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.DeleteCgroup(temp_dir->GetPath()); + EXPECT_TRUE(s.IsInvalidArgument()) << s.ToString(); +} + +TEST(SysFsCgroupDriver, DeleteCgroupFailsIfCgroupDoesNotExist) { + // This is not a directory on the cgroupv2 vfs. + SysFsCgroupDriver driver; + Status s = driver.DeleteCgroup("/some/path/that/doesnt/exist"); + EXPECT_TRUE(s.IsNotFound()) << s.ToString(); +} + TEST(SysFsCgroupDriver, GetAvailableControllersFailsIfNotCgroup2Path) { auto temp_dir_or_status = TempDirectory::Create(); ASSERT_TRUE(temp_dir_or_status.ok()) << temp_dir_or_status.ToString(); From e740b365d16fb974e577671b0d93bbc50da4fdf8 Mon Sep 17 00:00:00 2001 From: Jun-Hao Wan Date: Tue, 9 Sep 2025 01:51:13 +0800 Subject: [PATCH 501/634] [Docs] Include CR UID in KubeRay metrics reference (#56312) Since https://github.com/ray-project/kuberay/pull/4003 added a new label `uid` to each metric, the documentation needs to be updated accordingly. image Result: https://anyscale-ray--56312.com.readthedocs.build/en/56312/cluster/kubernetes/k8s-ecosystem/metrics-references.html Signed-off-by: win5923 --- .../k8s-ecosystem/metrics-references.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/source/cluster/kubernetes/k8s-ecosystem/metrics-references.md b/doc/source/cluster/kubernetes/k8s-ecosystem/metrics-references.md index c3f95c771e14..f4a7a6c97ee5 100644 --- a/doc/source/cluster/kubernetes/k8s-ecosystem/metrics-references.md +++ b/doc/source/cluster/kubernetes/k8s-ecosystem/metrics-references.md @@ -28,24 +28,24 @@ curl localhost:8080/metrics | Metric name | Type | Description | Labels | |--------------------------------------------------|-------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------| -| `kuberay_cluster_info` | Gauge | Metadata information about RayCluster custom resources. | `namespace`: <RayCluster-namespace>
`name`: <RayCluster-name>
`owner_kind`: <RayJob\|RayService\|None> | -| `kuberay_cluster_condition_provisioned` | Gauge | Indicates whether the RayCluster is provisioned. See [RayClusterProvisioned](https://github.com/ray-project/kuberay/blob/7c6aedff5b4106281f50e87a7e9e177bf1237ec7/ray-operator/apis/ray/v1/raycluster_types.go#L214) for more information. | `namespace`: <RayCluster-namespace>
`name`: <RayCluster-name>
`condition`: <true\|false> | -| `kuberay_cluster_provisioned_duration_seconds` | Gauge | The time, in seconds, when a RayCluster's `RayClusterProvisioned` status transitions from false (or unset) to true. | `namespace`: <RayCluster-namespace>
`name`: <RayCluster-name> | +| `kuberay_cluster_info` | Gauge | Metadata information about RayCluster custom resources. | `namespace`: <RayCluster-namespace>
`name`: <RayCluster-name>
`owner_kind`: <RayJob\|RayService\|None>
`uid`: <RayCluster-uid> | +| `kuberay_cluster_condition_provisioned` | Gauge | Indicates whether the RayCluster is provisioned. See [RayClusterProvisioned](https://github.com/ray-project/kuberay/blob/7c6aedff5b4106281f50e87a7e9e177bf1237ec7/ray-operator/apis/ray/v1/raycluster_types.go#L214) for more information. | `namespace`: <RayCluster-namespace>
`name`: <RayCluster-name>
`condition`: <true\|false>
`uid`: <RayCluster-uid> | +| `kuberay_cluster_provisioned_duration_seconds` | Gauge | The time, in seconds, when a RayCluster's `RayClusterProvisioned` status transitions from false (or unset) to true. | `namespace`: <RayCluster-namespace>
`name`: <RayCluster-name>
`uid`: <RayCluster-uid> | ### RayService metrics | Metric name | Type | Description | Labels | |--------------------------------------------------|-------|------------------------------------------------------------|--------------------------------------------------------------------| -| `kuberay_service_info` | Gauge | Metadata information about RayService custom resources. | `namespace`: <RayService-namespace>
`name`: <RayService-name> | -| `kuberay_service_condition_ready` | Gauge | Describes whether the RayService is ready. Ready means users can send requests to the underlying cluster and the number of serve endpoints is greater than 0. See [RayServiceReady](https://github.com/ray-project/kuberay/blob/33ee6724ca2a429c77cb7ff5821ba9a3d63f7c34/ray-operator/apis/ray/v1/rayservice_types.go#L135) for more information. | `namespace`: <RayService-namespace>
`name`: <RayService-name> | -| `kuberay_service_condition_upgrade_in_progress` | Gauge | Describes whether the RayService is performing a zero-downtime upgrade. See [UpgradeInProgress](https://github.com/ray-project/kuberay/blob/33ee6724ca2a429c77cb7ff5821ba9a3d63f7c34/ray-operator/apis/ray/v1/rayservice_types.go#L137) for more information. | `namespace`: <RayService-namespace>
`name`: <RayService-name> | +| `kuberay_service_info` | Gauge | Metadata information about RayService custom resources. | `namespace`: <RayService-namespace>
`name`: <RayService-name>
`uid`: <RayService-uid> | +| `kuberay_service_condition_ready` | Gauge | Describes whether the RayService is ready. Ready means users can send requests to the underlying cluster and the number of serve endpoints is greater than 0. See [RayServiceReady](https://github.com/ray-project/kuberay/blob/33ee6724ca2a429c77cb7ff5821ba9a3d63f7c34/ray-operator/apis/ray/v1/rayservice_types.go#L135) for more information. | `namespace`: <RayService-namespace>
`name`: <RayService-name>
`uid`: <RayService-uid> | +| `kuberay_service_condition_upgrade_in_progress` | Gauge | Describes whether the RayService is performing a zero-downtime upgrade. See [UpgradeInProgress](https://github.com/ray-project/kuberay/blob/33ee6724ca2a429c77cb7ff5821ba9a3d63f7c34/ray-operator/apis/ray/v1/rayservice_types.go#L137) for more information. | `namespace`: <RayService-namespace>
`name`: <RayService-name>
`uid`: <RayService-uid> | ### RayJob metrics | Metric name | Type | Description | Labels | |--------------------------------------------------|-------|------------------------------------------------------------|---------------------------------------------------------------------------| -| `kuberay_job_info` | Gauge | Metadata information about RayJob custom resources. | `namespace`: <RayJob-namespace>
`name`: <RayJob-name> | -| `kuberay_job_deployment_status` | Gauge | The RayJob's current deployment status. | `namespace`: <RayJob-namespace>
`name`: <RayJob-name>
`deployment_status`: <New\|Initializing\|Running\|Complete\|Failed\|Suspending\|Suspended\|Retrying\|Waiting> | -| `kuberay_job_execution_duration_seconds` | Gauge | Duration of the RayJob CR’s JobDeploymentStatus transition from `Initializing` to either the `Retrying` state or a terminal state, such as `Complete` or `Failed`. The `Retrying` state indicates that the CR previously failed and that spec.backoffLimit is enabled. | `namespace`: <RayJob-namespace>
`name`: <RayJob-name>
`job_deployment_status`: <Complete\|Failed>
`retry_count`: <count> | +| `kuberay_job_info` | Gauge | Metadata information about RayJob custom resources. | `namespace`: <RayJob-namespace>
`name`: <RayJob-name>
`uid`: <RayJob-uid> | +| `kuberay_job_deployment_status` | Gauge | The RayJob's current deployment status. | `namespace`: <RayJob-namespace>
`name`: <RayJob-name>
`deployment_status`: <New\|Initializing\|Running\|Complete\|Failed\|Suspending\|Suspended\|Retrying\|Waiting>
`uid`: <RayJob-uid> | +| `kuberay_job_execution_duration_seconds` | Gauge | Duration of the RayJob CR’s JobDeploymentStatus transition from `Initializing` to either the `Retrying` state or a terminal state, such as `Complete` or `Failed`. The `Retrying` state indicates that the CR previously failed and that spec.backoffLimit is enabled. | `namespace`: <RayJob-namespace>
`name`: <RayJob-name>
`job_deployment_status`: <Complete\|Failed>
`retry_count`: <count>
`uid`: <RayJob-uid> | From f8e9645db22b1e5b168b66e542d3d0ba1f4468f5 Mon Sep 17 00:00:00 2001 From: Alexey Kudinkin Date: Mon, 8 Sep 2025 13:58:13 -0400 Subject: [PATCH 502/634] [Data] Fixing empty projection handling in `ParquetDataSource` (#56299) ## Why are these changes needed? 1. Fixing empty projection handling in `ParquetDataSource` 2. Adding tests ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Alexey Kudinkin --- .../datasource/parquet_datasource.py | 186 +++++++++++++----- python/ray/data/_internal/output_buffer.py | 4 +- python/ray/data/tests/test_parquet.py | 137 +++++++++++++ 3 files changed, 280 insertions(+), 47 deletions(-) diff --git a/python/ray/data/_internal/datasource/parquet_datasource.py b/python/ray/data/_internal/datasource/parquet_datasource.py index 8cb6db7600ae..ae66e7cab478 100644 --- a/python/ray/data/_internal/datasource/parquet_datasource.py +++ b/python/ray/data/_internal/datasource/parquet_datasource.py @@ -1,5 +1,6 @@ import logging import math +import os import warnings from dataclasses import dataclass from typing import ( @@ -7,6 +8,7 @@ Any, Callable, Dict, + Iterable, Iterator, List, Literal, @@ -20,6 +22,7 @@ import ray from ray._private.arrow_utils import get_pyarrow_version +from ray.data._internal.arrow_block import ArrowBlockAccessor from ray.data._internal.progress_bar import ProgressBar from ray.data._internal.remote_fn import cached_remote_fn from ray.data._internal.util import ( @@ -52,6 +55,7 @@ if TYPE_CHECKING: import pyarrow + from pyarrow import parquet as pq from pyarrow.dataset import ParquetFileFragment @@ -100,6 +104,9 @@ PARQUET_ENCODING_RATIO_ESTIMATE_NUM_ROWS = 1024 +_BATCH_SIZE_PRESERVING_STUB_COL_NAME = "__bsp_stub" + + class _ParquetFragment: """This wrapper class is created to avoid utilizing `ParquetFileFragment` original serialization protocol that actually does network RPCs during serialization @@ -434,51 +441,25 @@ def read_fragments( # Ensure that we're reading at least one dataset fragment. assert len(fragments) > 0 - import pyarrow as pa - logger.debug(f"Reading {len(fragments)} parquet fragments") - - use_threads = to_batches_kwargs.pop("use_threads", False) - batch_size = to_batches_kwargs.pop("batch_size", default_read_batch_size_rows) for fragment in fragments: - partitions = {} - if partitioning is not None: - parse = PathPartitionParser(partitioning) - partitions = parse(fragment.original.path) - - # Filter out partitions that aren't in the user-specified columns list. - if partition_columns is not None: - partitions = { - field_name: value - for field_name, value in partitions.items() - if field_name in partition_columns - } - - def get_batch_iterable(): - if batch_size is not None: - to_batches_kwargs["batch_size"] = batch_size - - return fragment.original.to_batches( - use_threads=use_threads, - columns=data_columns, - schema=schema, - **to_batches_kwargs, - ) - # S3 can raise transient errors during iteration, and PyArrow doesn't expose a # way to retry specific batches. ctx = ray.data.DataContext.get_current() - for batch in iterate_with_retry( - get_batch_iterable, "load batch", match=ctx.retried_io_errors + for table in iterate_with_retry( + lambda: _read_batches_from( + fragment.original, + schema=schema, + data_columns=data_columns, + partition_columns=partition_columns, + partitioning=partitioning, + include_path=include_paths, + batch_size=default_read_batch_size_rows, + to_batches_kwargs=to_batches_kwargs, + ), + "reading batches", + match=ctx.retried_io_errors, ): - table = pa.Table.from_batches([batch], schema=schema) - if include_paths: - table = BlockAccessor.for_block(table).fill_column( - "path", fragment.original.path - ) - if partitions: - table = _add_partitions_to_table(partitions, table) - # If the table is empty, drop it. if table.num_rows > 0: if block_udf is not None: @@ -487,6 +468,112 @@ def get_batch_iterable(): yield table +def _read_batches_from( + fragment: "ParquetFileFragment", + *, + schema: "pyarrow.Schema", + data_columns: Optional[List[str]], + partition_columns: Optional[List[str]], + partitioning: Partitioning, + filter_expr: Optional["pyarrow.dataset.Expression"] = None, + batch_size: Optional[int] = None, + include_path: bool = False, + use_threads: bool = False, + to_batches_kwargs: Optional[Dict[str, Any]] = None, +) -> Iterable["pyarrow.Table"]: + """Get an iterable of batches from a parquet fragment.""" + + import pyarrow as pa + + # Copy to avoid modifying passed in arg + to_batches_kwargs = dict(to_batches_kwargs or {}) + + # NOTE: Passed in kwargs overrides always take precedence + # TODO deprecate to_batches_kwargs + use_threads = to_batches_kwargs.pop("use_threads", use_threads) + filter_expr = to_batches_kwargs.pop("filter", filter_expr) + # NOTE: Arrow's ``to_batches`` expects ``batch_size`` as an int + if batch_size is not None: + to_batches_kwargs.setdefault("batch_size", batch_size) + + partition_col_values = _parse_partition_column_values( + fragment, partition_columns, partitioning + ) + + try: + for batch in fragment.to_batches( + columns=data_columns, + filter=filter_expr, + schema=schema, + use_threads=use_threads, + **to_batches_kwargs, + ): + table = pa.Table.from_batches([batch]) + + if include_path: + table = ArrowBlockAccessor.for_block(table).fill_column( + "path", fragment.path + ) + + if partition_col_values: + table = _add_partitions_to_table(partition_col_values, table) + + # ``ParquetFileFragment.to_batches`` returns ``RecordBatch``, + # which could have empty projection (ie ``num_columns`` == 0) + # while having non-empty rows (ie ``num_rows`` > 0), which + # could occur when list of requested columns is empty. + # + # However, when ``RecordBatches`` are concatenated using + # ``pyarrow.concat_tables`` it will return a single ``Table`` + # with 0 columns and therefore 0 rows (since ``Table``s number of + # rows is determined as the length of its columns). + # + # To avoid running into this pitfall, we introduce a stub column + # holding just nulls to maintain invariance of the number of rows. + # + # NOTE: There's no impact from this as the binary size of the + # extra column is basically 0 + if table.num_columns == 0 and table.num_rows > 0: + table = table.append_column( + _BATCH_SIZE_PRESERVING_STUB_COL_NAME, pa.nulls(table.num_rows) + ) + + yield table + + except pa.lib.ArrowInvalid as e: + error_message = str(e) + if "No match for FieldRef.Name" in error_message and filter_expr is not None: + filename = os.path.basename(fragment.path) + file_columns = set(fragment.physical_schema.names) + raise RuntimeError( + f"Filter expression: '{filter_expr}' failed on parquet " + f"file: '{filename}' with columns: {file_columns}" + ) + raise + + +def _parse_partition_column_values( + fragment: "ParquetFileFragment", + partition_columns: Optional[List[str]], + partitioning: Partitioning, +): + partitions = {} + + if partitioning is not None: + parse = PathPartitionParser(partitioning) + partitions = parse(fragment.path) + + # Filter out partitions that aren't in the user-specified columns list. + if partition_columns is not None: + partitions = { + field_name: value + for field_name, value in partitions.items() + if field_name in partition_columns + } + + return partitions + + def _fetch_parquet_file_info( fragment: _ParquetFragment, *, @@ -690,13 +777,18 @@ def _sample_fragments( def _add_partitions_to_table( - partitions: Dict[str, PartitionDataType], table: "pyarrow.Table" + partition_col_values: Dict[str, PartitionDataType], table: "pyarrow.Table" ) -> "pyarrow.Table": - for field_name, value in partitions.items(): - field_index = table.schema.get_field_index(field_name) + for partition_col, value in partition_col_values.items(): + field_index = table.schema.get_field_index(partition_col) if field_index == -1: - table = BlockAccessor.for_block(table).fill_column(field_name, value) + table = BlockAccessor.for_block(table).fill_column(partition_col, value) + elif log_once(f"duplicate_partition_field_{partition_col}"): + logger.warning( + f"The partition field '{partition_col}' also exists in the Parquet " + f"file. Ray Data will default to using the value in the Parquet file." + ) return table @@ -747,7 +839,11 @@ def emit_file_extensions_future_warning(future_file_extensions: List[str]): def _infer_schema( - parquet_dataset, schema, columns, partitioning, _block_udf + parquet_dataset: "pq.ParquetDataset", + schema: "pyarrow.Schema", + columns: Optional[List[str]], + partitioning, + _block_udf, ) -> "pyarrow.Schema": """Infer the schema of read data using the user-specified parameters.""" import pyarrow as pa @@ -760,7 +856,7 @@ def _infer_schema( partitioning, inferred_schema, parquet_dataset ) - if columns: + if columns is not None: inferred_schema = pa.schema( [inferred_schema.field(column) for column in columns], inferred_schema.metadata, diff --git a/python/ray/data/_internal/output_buffer.py b/python/ray/data/_internal/output_buffer.py index 20265ebd35b2..f8332b30d2b8 100644 --- a/python/ray/data/_internal/output_buffer.py +++ b/python/ray/data/_internal/output_buffer.py @@ -90,7 +90,7 @@ def has_next(self) -> bool: self._exceeded_buffer_row_limit() or self._exceeded_buffer_size_limit() ) - def _exceeded_block_size_slice_limit(self, block: Block) -> bool: + def _exceeded_block_size_slice_limit(self, block: BlockAccessor) -> bool: # Slice a block to respect the target max block size. We only do this if we are # more than 50% above the target block size, because this ensures that the last # block produced will be at least half the target block size. @@ -101,7 +101,7 @@ def _exceeded_block_size_slice_limit(self, block: Block) -> bool: * self._output_block_size_option.target_max_block_size ) - def _exceeded_block_row_slice_limit(self, block: Block) -> bool: + def _exceeded_block_row_slice_limit(self, block: BlockAccessor) -> bool: # Slice a block to respect the target max rows per block. We only do this if we # are more than 50% above the target rows per block, because this ensures that # the last block produced will be at least half the target row count. diff --git a/python/ray/data/tests/test_parquet.py b/python/ray/data/tests/test_parquet.py index 08471a7653c3..59de820688c8 100644 --- a/python/ray/data/tests/test_parquet.py +++ b/python/ray/data/tests/test_parquet.py @@ -601,6 +601,32 @@ def test_parquet_read_partitioned_explicit( ] +def test_proper_projection_for_partitioned_datasets(temp_dir): + ds = ray.data.read_parquet("example://iris.parquet").materialize() + + partitioned_ds_path = f"{temp_dir}/partitioned_iris" + # Write out partitioned dataset + ds.write_parquet(partitioned_ds_path, partition_cols=["variety"]) + + partitioned_ds = ray.data.read_parquet( + partitioned_ds_path, columns=["variety"] + ).materialize() + + print(partitioned_ds.schema()) + + assert [ + "sepal.length", + "sepal.width", + "petal.length", + "petal.width", + "variety", + ] == ds.take_batch(batch_format="pyarrow").column_names + + assert ["variety"] == partitioned_ds.take_batch(batch_format="pyarrow").column_names + + assert ds.count() == partitioned_ds.count() + + def test_parquet_read_with_udf( ray_start_regular_shared, tmp_path, target_max_block_size_infinite_or_default ): @@ -1984,6 +2010,117 @@ def test_read_parquet_with_none_partitioning_and_columns(tmp_path): assert ds.take_all() == [{"column": 42}] +def _create_test_data(num_rows: int) -> dict: + return { + "int_col": list(range(num_rows)), + "float_col": [float(i) for i in range(num_rows)], + "str_col": [f"str_{i}" for i in range(num_rows)], + } + + +@pytest.mark.parametrize( + "batch_size,filter_expr,expected_rows,description", + [ + # No batch size cases + (None, "int_col > 500", 499, "No batch size, int > 500"), + (None, "int_col < 200", 200, "No batch size, int < 200"), + ( + None, + "float_col == 42.0", + 1, + "No batch size, float == 42.0", + ), + ( + None, + "str_col == 'str_42'", + 1, + "No batch size, str == str_42", + ), + # Batch size cases + (100, "int_col > 500", 499, "Fixed batch size, int > 500"), + (200, "int_col < 200", 200, "Fixed batch size, int < 200"), + ( + 300, + "float_col == 42.0", + 1, + "Fixed batch size, float == 42.0", + ), + ( + 400, + "str_col == 'str_42'", + 1, + "Fixed batch size, str == str_42", + ), + ], +) +def test_read_parquet_with_filter_selectivity( + ray_start_regular_shared, + tmp_path, + batch_size, + filter_expr, + expected_rows, + description, +): + """Test reading parquet files with filter expressions and different batch sizes.""" + num_rows = 1000 + data = _create_test_data(num_rows) + table = pa.Table.from_pydict(data) + + file_path = os.path.join(tmp_path, "test.parquet") + pq.write_table(table, file_path, row_group_size=200) + + if batch_size is not None: + ray.data.DataContext.get_current().target_max_block_size = batch_size + ds = ray.data.read_parquet(file_path).filter(expr=filter_expr) + + assert ds.count() == expected_rows, ( + f"{description}: Filter '{filter_expr}' returned {ds.count()} rows, " + f"expected {expected_rows}" + ) + + # Verify schema has expected columns and types + assert ds.schema().base_schema == table.schema + + +@pytest.mark.parametrize("batch_size", [None, 100, 200, 10_000]) +@pytest.mark.parametrize( + "columns", + [ + # Empty projection + [], + ["int_col"], + ["int_col", "float_col", "str_col"], + ], +) +def test_read_parquet_with_columns_selectivity( + ray_start_regular_shared, + tmp_path, + batch_size, + columns, +): + """Test reading parquet files with different column selections and batch sizes.""" + num_rows = 1000 + data = _create_test_data(num_rows) + table = pa.Table.from_pydict(data) + + file_path = os.path.join(tmp_path, "test.parquet") + pq.write_table(table, file_path, row_group_size=200) + + if batch_size is not None: + ray.data.DataContext.get_current().target_max_block_size = batch_size + ds = ray.data.read_parquet(file_path, columns=columns) + + assert ds.count() == num_rows, ( + f"Column selection {columns} with batch_size={batch_size} " + f"returned {ds.count()} rows, expected {num_rows}" + ) + + assert set(ds.schema().names) == set(columns), ( + f"Column selection {columns} with batch_size={batch_size} " + f"returned columns {ds.schema().names}" + ) + + if __name__ == "__main__": import sys From a3254a3755cbb04266d20c48bedeee376b31cf71 Mon Sep 17 00:00:00 2001 From: harshit-anyscale Date: Mon, 8 Sep 2025 23:31:01 +0530 Subject: [PATCH 503/634] increase timeout for failed task test (#56328) we have set the retry backoff as true, and max retries as 3 for this test. so, actually the task gets consumed 4 times, and the difference between the task consumption time is 1 sec, 2 sec and 4 sec, which makes the total time to complete the test close of 10 seconds, hence the test was flaky increasing the timeout to 20 seconds Signed-off-by: harshit --- python/ray/serve/tests/test_task_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/serve/tests/test_task_processor.py b/python/ray/serve/tests/test_task_processor.py index 1dc662325d69..d194f3dca6cc 100644 --- a/python/ray/serve/tests/test_task_processor.py +++ b/python/ray/serve/tests/test_task_processor.py @@ -233,7 +233,7 @@ def assert_result(): else: return False - wait_for_condition(assert_result, timeout=10) + wait_for_condition(assert_result, timeout=20) def test_task_consumer_persistence_across_restarts( self, temp_queue_directory, serve_instance, create_processor_config From 57fd3b5137b4cc4eb820eda77620246039c731b6 Mon Sep 17 00:00:00 2001 From: Pavitra Bhalla Date: Mon, 8 Sep 2025 13:03:38 -0500 Subject: [PATCH 504/634] [Core] Add PID to structured logs for tasks and actors (#55176) Signed-off-by: pavitrabhalla Co-authored-by: Jiajun Yao --- python/ray/_private/ray_logging/constants.py | 1 + python/ray/_private/ray_logging/formatters.py | 1 + python/ray/tests/test_logging_2.py | 23 ++++++++++++++++--- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/python/ray/_private/ray_logging/constants.py b/python/ray/_private/ray_logging/constants.py index 6accad120006..a5bf5850a708 100644 --- a/python/ray/_private/ray_logging/constants.py +++ b/python/ray/_private/ray_logging/constants.py @@ -53,6 +53,7 @@ class LogKey(str, Enum): FILENAME = "filename" LINENO = "lineno" EXC_TEXT = "exc_text" + PROCESS = "process" # Ray logging context TIMESTAMP_NS = "timestamp_ns" diff --git a/python/ray/_private/ray_logging/formatters.py b/python/ray/_private/ray_logging/formatters.py index 9c1cc8a51e40..bf67a309bb5c 100644 --- a/python/ray/_private/ray_logging/formatters.py +++ b/python/ray/_private/ray_logging/formatters.py @@ -63,6 +63,7 @@ def generate_record_format_attrs( LogKey.MESSAGE.value: record.getMessage(), LogKey.FILENAME.value: record.filename, LogKey.LINENO.value: record.lineno, + LogKey.PROCESS.value: record.process, } ) if record.exc_info: diff --git a/python/ray/tests/test_logging_2.py b/python/ray/tests/test_logging_2.py index 11cca61fb1cf..a74f5025125b 100644 --- a/python/ray/tests/test_logging_2.py +++ b/python/ray/tests/test_logging_2.py @@ -18,9 +18,11 @@ def test_driver_process(self, shutdown_only): filter = CoreContextFilter() record = logging.makeLogRecord({}) assert filter.filter(record) - # Ray is not initialized so no context + # Ray is not initialized so no context except PID which should be available for attr in log_context: assert not hasattr(record, attr) + # PID should be available even when Ray is not initialized + assert hasattr(record, "process") assert hasattr(record, "_ray_timestamp_ns") ray.init() @@ -31,6 +33,7 @@ def test_driver_process(self, shutdown_only): "job_id": runtime_context.get_job_id(), "worker_id": runtime_context.get_worker_id(), "node_id": runtime_context.get_node_id(), + "process": record.process, } for attr in log_context: assert hasattr(record, attr) @@ -46,7 +49,7 @@ def f(): filter = CoreContextFilter() record = logging.makeLogRecord({}) assert filter.filter(record) - should_exist = ["job_id", "worker_id", "node_id", "task_id"] + should_exist = ["job_id", "worker_id", "node_id", "task_id", "process"] runtime_context = ray.get_runtime_context() expected_values = { "job_id": runtime_context.get_job_id(), @@ -55,6 +58,7 @@ def f(): "task_id": runtime_context.get_task_id(), "task_name": runtime_context.get_task_name(), "task_func_name": runtime_context.get_task_function_name(), + "process": record.process, } for attr in should_exist: assert hasattr(record, attr) @@ -73,7 +77,14 @@ def f(self): filter = CoreContextFilter() record = logging.makeLogRecord({}) assert filter.filter(record) - should_exist = ["job_id", "worker_id", "node_id", "actor_id", "task_id"] + should_exist = [ + "job_id", + "worker_id", + "node_id", + "actor_id", + "task_id", + "process", + ] runtime_context = ray.get_runtime_context() expected_values = { "job_id": runtime_context.get_job_id(), @@ -84,6 +95,7 @@ def f(self): "task_id": runtime_context.get_task_id(), "task_name": runtime_context.get_task_name(), "task_func_name": runtime_context.get_task_function_name(), + "process": record.process, } for attr in should_exist: assert hasattr(record, attr) @@ -102,6 +114,7 @@ def test_empty_record(self, shutdown_only): record_dict = json.loads(formatted) should_exist = [ + "process", "asctime", "levelname", "message", @@ -124,6 +137,7 @@ def test_record_with_exception(self, shutdown_only): formatted = formatter.format(record) record_dict = json.loads(formatted) should_exist = [ + "process", "asctime", "levelname", "message", @@ -143,6 +157,7 @@ def test_record_with_user_provided_context(self, shutdown_only): formatted = formatter.format(record) record_dict = json.loads(formatted) should_exist = [ + "process", "asctime", "levelname", "message", @@ -171,6 +186,7 @@ def test_record_with_flatten_keys_valid_dict(self, shutdown_only): formatted = formatter.format(record) record_dict = json.loads(formatted) should_exist = [ + "process", "asctime", "levelname", "message", @@ -196,6 +212,7 @@ def test_record_with_valid_additional_log_standard_attrs(self, shutdown_only): record_dict = json.loads(formatted) should_exist = [ + "process", "asctime", "levelname", "message", From 47eae759759c16d9d8aef9a62f2414b854751d8f Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Mon, 8 Sep 2025 11:40:32 -0700 Subject: [PATCH 505/634] [Data] Refactor `from_torch` unit tests (#56331) ## Why are these changes needed? This PR improves `ray.data.from_torch` tests by: * Moving them from test_formats.py to test_torch.py for clearer organization * Using a shared Ray cluster instead of shutdown_only to reduce overhead * Replacing FashionMNIST with a stub dataset to remove the external dependency * Splitting test_from_torch into separate map-style and iterable-style tests so each case is isolated ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Balaji Veeramani --- python/ray/data/tests/test_formats.py | 74 ------------------------ python/ray/data/tests/test_torch.py | 82 +++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 74 deletions(-) diff --git a/python/ray/data/tests/test_formats.py b/python/ray/data/tests/test_formats.py index 19d2d7aa36ae..c0ac1f8f4df7 100644 --- a/python/ray/data/tests/test_formats.py +++ b/python/ray/data/tests/test_formats.py @@ -5,7 +5,6 @@ import pyarrow as pa import pyarrow.parquet as pq import pytest -import torchvision from fsspec.implementations.http import HTTPFileSystem from fsspec.implementations.local import LocalFileSystem @@ -249,79 +248,6 @@ def test_from_tf(ray_start_regular_shared): tf.debugging.assert_equal(expected_label, actual_label) -@pytest.mark.parametrize("local_read", [True, False]) -def test_from_torch(shutdown_only, local_read, tmp_path): - torch_dataset = torchvision.datasets.FashionMNIST(tmp_path, download=True) - expected_data = list(torch_dataset) - - ray_dataset = ray.data.from_torch(torch_dataset, local_read=local_read) - - actual_data = extract_values("item", list(ray_dataset.take_all())) - assert actual_data == expected_data - - import torch - - class IterFashionMNIST(torch.utils.data.IterableDataset): - def __len__(self): - return len(torch_dataset) - - def __iter__(self): - return iter(torch_dataset) - - iter_torch_dataset = IterFashionMNIST() - ray_dataset = ray.data.from_torch(iter_torch_dataset) - - actual_data = extract_values("item", list(ray_dataset.take_all())) - assert actual_data == expected_data - - -@pytest.mark.parametrize("local_read", [True, False]) -def test_from_torch_boundary_conditions(shutdown_only, local_read): - """ - Tests that from_torch respects __len__ for map-style datasets - """ - from torch.utils.data import Dataset - - class BoundaryTestMapDataset(Dataset): - """A map-style dataset where __len__ is less than the underlying data size.""" - - def __init__(self, data, length): - super().__init__() - self._data = data - self._length = length - assert self._length <= len( - self._data - ), "Length must be <= data size to properly test boundary conditions" - - def __len__(self): - return self._length - - def __getitem__(self, index): - if not (0 <= index < self._length): - # Note: don't use IndexError because we want to fail clearly if - # Ray Data tries to access beyond __len__ - 1 - raise RuntimeError( - f"Index {index} out of bounds for dataset with length {self._length}" - ) - return self._data[index] - - source_data = list(range(10)) - dataset_len = 8 # Intentionally less than len(source_data) - - # --- Test MapDataset --- - map_ds = BoundaryTestMapDataset(source_data, dataset_len) - # Expected data only includes elements up to dataset_len - 1 - expected_items = source_data[:dataset_len] - - ray_ds_map = ray.data.from_torch(map_ds, local_read=local_read) - actual_items_map = extract_values("item", list(ray_ds_map.take_all())) - - # This assertion verifies that ray_ds_map didn't try to access index 8 or 9, - # which would have raised an IndexError in BoundaryTestMapDataset.__getitem__ - assert actual_items_map == expected_items - assert len(actual_items_map) == dataset_len - - def test_read_s3_file_error(shutdown_only, s3_path): dummy_path = s3_path + "_dummy" error_message = "Please check that file exists and has properly configured access." diff --git a/python/ray/data/tests/test_torch.py b/python/ray/data/tests/test_torch.py index 024a8f1044c3..5cae57075f8b 100644 --- a/python/ray/data/tests/test_torch.py +++ b/python/ray/data/tests/test_torch.py @@ -1,10 +1,12 @@ import numpy as np import pandas as pd import pytest +import torch import ray from ray.data.extensions.tensor_extension import TensorArray from ray.data.tests.conftest import * # noqa +from ray.data.tests.util import extract_values from ray.tests.conftest import * # noqa @@ -336,6 +338,86 @@ def train_loop_per_worker(): my_trainer.fit() +@pytest.mark.parametrize("local_read", [True, False]) +def test_from_torch_map_style_dataset(ray_start_10_cpus_shared, local_read): + class StubDataset(torch.utils.data.Dataset): + def __len__(self): + return 1 + + def __getitem__(self, index): + return index + + torch_dataset = StubDataset() + + ray_dataset = ray.data.from_torch(torch_dataset, local_read=local_read) + + actual_data = ray_dataset.take_all() + assert actual_data == [{"item": 0}] + + +def test_from_torch_iterable_style_dataset(ray_start_10_cpus_shared): + class StubIterableDataset(torch.utils.data.IterableDataset): + def __len__(self): + return 1 + + def __iter__(self): + return iter([0]) + + iter_torch_dataset = StubIterableDataset() + + ray_dataset = ray.data.from_torch(iter_torch_dataset) + + actual_data = ray_dataset.take_all() + assert actual_data == [{"item": 0}] + + +@pytest.mark.parametrize("local_read", [True, False]) +def test_from_torch_boundary_conditions(ray_start_10_cpus_shared, local_read): + """ + Tests that from_torch respects __len__ for map-style datasets + """ + from torch.utils.data import Dataset + + class BoundaryTestMapDataset(Dataset): + """A map-style dataset where __len__ is less than the underlying data size.""" + + def __init__(self, data, length): + super().__init__() + self._data = data + self._length = length + assert self._length <= len( + self._data + ), "Length must be <= data size to properly test boundary conditions" + + def __len__(self): + return self._length + + def __getitem__(self, index): + if not (0 <= index < self._length): + # Note: don't use IndexError because we want to fail clearly if + # Ray Data tries to access beyond __len__ - 1 + raise RuntimeError( + f"Index {index} out of bounds for dataset with length {self._length}" + ) + return self._data[index] + + source_data = list(range(10)) + dataset_len = 8 # Intentionally less than len(source_data) + + # --- Test MapDataset --- + map_ds = BoundaryTestMapDataset(source_data, dataset_len) + # Expected data only includes elements up to dataset_len - 1 + expected_items = source_data[:dataset_len] + + ray_ds_map = ray.data.from_torch(map_ds, local_read=local_read) + actual_items_map = extract_values("item", list(ray_ds_map.take_all())) + + # This assertion verifies that ray_ds_map didn't try to access index 8 or 9, + # which would have raised an IndexError in BoundaryTestMapDataset.__getitem__ + assert actual_items_map == expected_items + assert len(actual_items_map) == dataset_len + + if __name__ == "__main__": import sys From 33ce9e6d0cc0a03071a81a45387cce3622c9defc Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Mon, 8 Sep 2025 12:48:09 -0700 Subject: [PATCH 506/634] [core] Make RequestWorkerLease RPC Fault Tolerant (#56191) Signed-off-by: joshlee --- python/ray/tests/BUILD | 1 + python/ray/tests/test_failure_3.py | 14 +- python/ray/tests/test_failure_4.py | 13 +- python/ray/tests/test_object_manager.py | 2 + .../ray/tests/test_raylet_fault_tolerance.py | 48 +++++++ python/ray/tests/test_reconstruction.py | 2 + python/ray/tests/test_reconstruction_2.py | 2 + .../ray/tests/test_reconstruction_stress.py | 2 + .../tests/test_reconstruction_stress_spill.py | 2 + .../ray/tests/test_streaming_generator_2.py | 2 + .../ray/tests/test_streaming_generator_4.py | 2 + .../task_submission/normal_task_submitter.cc | 81 +++++------ .../task_submission/normal_task_submitter.h | 12 +- .../tests/normal_task_submitter_test.cc | 129 ++++++++++-------- src/ray/raylet/node_manager.cc | 17 ++- src/ray/raylet/tests/node_manager_test.cc | 80 +++++++++++ src/ray/rpc/raylet/raylet_client.cc | 30 ++-- 17 files changed, 310 insertions(+), 129 deletions(-) create mode 100644 python/ray/tests/test_raylet_fault_tolerance.py diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD index 96da51621016..075863d7df4f 100644 --- a/python/ray/tests/BUILD +++ b/python/ray/tests/BUILD @@ -883,6 +883,7 @@ py_test_module_list( "test_multi_node.py", "test_placement_group_3.py", "test_placement_group_5.py", + "test_raylet_fault_tolerance.py", "test_reconstruction.py", "test_reconstruction_2.py", "test_runtime_env_working_dir_uri.py", diff --git a/python/ray/tests/test_failure_3.py b/python/ray/tests/test_failure_3.py index 13facd165098..428d2f5b00cc 100644 --- a/python/ray/tests/test_failure_3.py +++ b/python/ray/tests/test_failure_3.py @@ -55,7 +55,19 @@ def test_plasma_store_operation_after_raylet_dies(ray_start_cluster): (RayletDiedError). """ cluster = ray_start_cluster - cluster.add_node(num_cpus=1) + # Required for reducing the retry time of RequestWorkerLease. The call to kill the raylet will also kill the plasma store on the raylet + # meaning the call to put will fail. This will trigger worker death, and the driver will try to queue the task again and request a new worker lease + # from the now dead raylet. + system_configs = { + "raylet_rpc_server_reconnect_timeout_s": 0, + "health_check_initial_delay_ms": 0, + "health_check_timeout_ms": 10, + "health_check_failure_threshold": 1, + } + cluster.add_node( + num_cpus=1, + _system_config=system_configs, + ) cluster.wait_for_nodes() ray.init(address=cluster.address) diff --git a/python/ray/tests/test_failure_4.py b/python/ray/tests/test_failure_4.py index 7c3dd9552caa..ed68031ff614 100644 --- a/python/ray/tests/test_failure_4.py +++ b/python/ray/tests/test_failure_4.py @@ -543,7 +543,18 @@ def task(): def test_task_failure_when_driver_local_raylet_dies(ray_start_cluster): cluster = ray_start_cluster - head = cluster.add_node(num_cpus=4, resources={"foo": 1}) + # Required for reducing the retry time of RequestWorkerLease + system_configs = { + "raylet_rpc_server_reconnect_timeout_s": 0, + "health_check_initial_delay_ms": 0, + "health_check_timeout_ms": 10, + "health_check_failure_threshold": 1, + } + head = cluster.add_node( + num_cpus=4, + resources={"foo": 1}, + _system_config=system_configs, + ) cluster.wait_for_nodes() ray.init(address=cluster.address) diff --git a/python/ray/tests/test_object_manager.py b/python/ray/tests/test_object_manager.py index 81a09c0783b4..e4b99f94bd0c 100644 --- a/python/ray/tests/test_object_manager.py +++ b/python/ray/tests/test_object_manager.py @@ -539,6 +539,8 @@ def test_object_directory_failure(ray_start_cluster): "health_check_period_ms": 500, "health_check_failure_threshold": 10, "object_timeout_milliseconds": 200, + # Required for reducing the retry time of RequestWorkerLease + "raylet_rpc_server_reconnect_timeout_s": 0, } # Add a head node. diff --git a/python/ray/tests/test_raylet_fault_tolerance.py b/python/ray/tests/test_raylet_fault_tolerance.py new file mode 100644 index 000000000000..21fcd1e84a5d --- /dev/null +++ b/python/ray/tests/test_raylet_fault_tolerance.py @@ -0,0 +1,48 @@ +import sys + +import pytest + +import ray +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy + + +@pytest.mark.parametrize("deterministic_failure", ["request", "response"]) +def test_request_worker_lease_idempotent( + monkeypatch, shutdown_only, deterministic_failure, ray_start_cluster +): + monkeypatch.setenv( + "RAY_testing_rpc_failure", + "NodeManagerService.grpc_client.RequestWorkerLease=1:" + + ("100:0" if deterministic_failure == "request" else "0:100"), + ) + + @ray.remote + def simple_task_1(): + return 0 + + @ray.remote + def simple_task_2(): + return 1 + + # Spin up a two-node cluster where we're targeting scheduling on the + # remote node via NodeAffinitySchedulingStrategy to test remote RequestWorkerLease + # calls. + cluster = ray_start_cluster + remote_node = cluster.add_node(num_cpus=1) + + result_ref1 = simple_task_1.options( + scheduling_strategy=NodeAffinitySchedulingStrategy( + node_id=remote_node.node_id, soft=False + ) + ).remote() + result_ref2 = simple_task_2.options( + scheduling_strategy=NodeAffinitySchedulingStrategy( + node_id=remote_node.node_id, soft=False + ) + ).remote() + + assert ray.get([result_ref1, result_ref2]) == [0, 1] + + +if __name__ == "__main__": + sys.exit(pytest.main(["-sv", __file__])) diff --git a/python/ray/tests/test_reconstruction.py b/python/ray/tests/test_reconstruction.py index f68d3c15d46c..6cf194cda0cc 100644 --- a/python/ray/tests/test_reconstruction.py +++ b/python/ray/tests/test_reconstruction.py @@ -22,6 +22,8 @@ def config(request): "health_check_period_ms": 100, "health_check_failure_threshold": 20, "object_timeout_milliseconds": 200, + # Required for reducing the retry time of RequestWorkerLease + "raylet_rpc_server_reconnect_timeout_s": 0, } yield config diff --git a/python/ray/tests/test_reconstruction_2.py b/python/ray/tests/test_reconstruction_2.py index 42ac99a8baef..d211aafb0ed5 100644 --- a/python/ray/tests/test_reconstruction_2.py +++ b/python/ray/tests/test_reconstruction_2.py @@ -25,6 +25,8 @@ def config(request): "health_check_period_ms": 100, "health_check_failure_threshold": 20, "object_timeout_milliseconds": 200, + # Required for reducing the retry time of RequestWorkerLease + "raylet_rpc_server_reconnect_timeout_s": 0, } yield config diff --git a/python/ray/tests/test_reconstruction_stress.py b/python/ray/tests/test_reconstruction_stress.py index 1dac4d58d9c7..e22360a9de36 100644 --- a/python/ray/tests/test_reconstruction_stress.py +++ b/python/ray/tests/test_reconstruction_stress.py @@ -16,6 +16,8 @@ def config(request): "health_check_period_ms": 100, "health_check_failure_threshold": 10, "object_timeout_milliseconds": 200, + # Required for reducing the retry time of RequestWorkerLease + "raylet_rpc_server_reconnect_timeout_s": 0, } yield config diff --git a/python/ray/tests/test_reconstruction_stress_spill.py b/python/ray/tests/test_reconstruction_stress_spill.py index 434553fe9e84..6d1322c23c08 100644 --- a/python/ray/tests/test_reconstruction_stress_spill.py +++ b/python/ray/tests/test_reconstruction_stress_spill.py @@ -16,6 +16,8 @@ def config(request): "health_check_period_ms": 100, "health_check_failure_threshold": 10, "object_timeout_milliseconds": 200, + # Required for reducing the retry time of RequestWorkerLease + "raylet_rpc_server_reconnect_timeout_s": 0, } yield config diff --git a/python/ray/tests/test_streaming_generator_2.py b/python/ray/tests/test_streaming_generator_2.py index 517c6f744c78..407dc83ddbdd 100644 --- a/python/ray/tests/test_streaming_generator_2.py +++ b/python/ray/tests/test_streaming_generator_2.py @@ -22,6 +22,8 @@ "task_retry_delay_ms": 100, "object_timeout_milliseconds": 200, "fetch_warn_timeout_milliseconds": 1000, + # Required for reducing the retry time of RequestWorkerLease + "raylet_rpc_server_reconnect_timeout_s": 0, } diff --git a/python/ray/tests/test_streaming_generator_4.py b/python/ray/tests/test_streaming_generator_4.py index 45c31c525e73..fe1455e7b4f2 100644 --- a/python/ray/tests/test_streaming_generator_4.py +++ b/python/ray/tests/test_streaming_generator_4.py @@ -23,6 +23,8 @@ "task_retry_delay_ms": 100, "object_timeout_milliseconds": 200, "fetch_warn_timeout_milliseconds": 1000, + # Required for reducing the retry time of RequestWorkerLease + "raylet_rpc_server_reconnect_timeout_s": 0, } diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.cc b/src/ray/core_worker/task_submission/normal_task_submitter.cc index 826e19904139..60f87076e069 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.cc +++ b/src/ray/core_worker/task_submission/normal_task_submitter.cc @@ -93,14 +93,14 @@ void NormalTaskSubmitter::SubmitTask(TaskSpecification task_spec) { void NormalTaskSubmitter::AddWorkerLeaseClient( const rpc::Address &addr, - std::shared_ptr raylet_client, + const NodeID &node_id, const google::protobuf::RepeatedPtrField &assigned_resources, const SchedulingKey &scheduling_key, const LeaseID &lease_id) { core_worker_client_pool_->GetOrConnect(addr); int64_t expiration = current_time_ms() + lease_timeout_ms_; LeaseEntry new_lease_entry{ - std::move(raylet_client), expiration, assigned_resources, scheduling_key, lease_id}; + node_id, expiration, assigned_resources, scheduling_key, lease_id}; worker_to_lease_entry_.emplace(addr, new_lease_entry); auto &scheduling_key_entry = scheduling_key_entries_[scheduling_key]; @@ -118,7 +118,7 @@ void NormalTaskSubmitter::ReturnWorkerLease(const rpc::Address &addr, auto &scheduling_key_entry = scheduling_key_entries_[scheduling_key]; RAY_CHECK(scheduling_key_entry.active_workers.size() >= 1); auto &lease_entry = worker_to_lease_entry_[addr]; - RAY_CHECK(lease_entry.raylet_client); + RAY_CHECK(!lease_entry.node_id.IsNil()); RAY_CHECK(!lease_entry.is_busy); // Decrement the number of active workers consuming tasks from the queue associated @@ -129,8 +129,9 @@ void NormalTaskSubmitter::ReturnWorkerLease(const rpc::Address &addr, // scheduling_key_entries_ hashmap. scheduling_key_entries_.erase(scheduling_key); } - - lease_entry.raylet_client->ReturnWorkerLease( + auto raylet_client = raylet_client_pool_->GetByID(lease_entry.node_id); + RAY_CHECK(raylet_client); + raylet_client->ReturnWorkerLease( addr.port(), lease_entry.lease_id, was_error, error_detail, worker_exiting); worker_to_lease_entry_.erase(addr); } @@ -143,7 +144,7 @@ void NormalTaskSubmitter::OnWorkerIdle( bool worker_exiting, const google::protobuf::RepeatedPtrField &assigned_resources) { auto &lease_entry = worker_to_lease_entry_[addr]; - if (!lease_entry.raylet_client) { + if (lease_entry.node_id.IsNil()) { return; } @@ -407,9 +408,8 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli << NodeID::FromBinary(reply.worker_address().node_id()) << " with worker " << WorkerID::FromBinary(reply.worker_address().worker_id()); - AddWorkerLeaseClient(reply.worker_address(), - std::move(raylet_lease_client), + NodeID::FromBinary(reply.worker_address().node_id()), reply.resource_mapping(), scheduling_key, lease_id); @@ -446,43 +446,30 @@ void NormalTaskSubmitter::RequestNewWorkerIfNeeded(const SchedulingKey &scheduli << status.ToString(); RequestNewWorkerIfNeeded(scheduling_key); - } else { - if (status.IsRpcError() && - status.rpc_code() == grpc::StatusCode::UNAVAILABLE) { - RAY_LOG(WARNING) - << "The worker failed to receive a response from the local " - << "raylet because the raylet is unavailable (crashed). " - << "Error: " << status; - if (worker_type_ == WorkerType::WORKER) { - // Exit the worker so that caller can retry somewhere else. - RAY_LOG(WARNING) << "Terminating the worker due to local raylet death"; - QuickExit(); - } - RAY_CHECK(worker_type_ == WorkerType::DRIVER); - error_type = rpc::ErrorType::LOCAL_RAYLET_DIED; - error_status = status; - // Grpc errors are not helpful at all. So we are overwriting it. - std::stringstream ss; - ss << "The worker failed to receive a response from the local raylet" - << "(id: " << NodeID::FromBinary(raylet_address.node_id()).Hex() - << " ,ip: " << raylet_address.ip_address() << ") " - << "because the raylet is " - "unavailable (crashed)."; - error_info.set_error_message(ss.str()); - tasks_to_fail = std::move(sched_entry.task_queue); - sched_entry.task_queue.clear(); - if (sched_entry.CanDelete()) { - scheduling_key_entries_.erase(scheduling_key); - } - } else { - RAY_LOG(WARNING) - << "The worker failed to receive a response from the local raylet, but " - "raylet is still alive. Try again on a local node. Error: " - << status; - // TODO(sang): Maybe we should raise FATAL error if it happens too many - // times. - RequestNewWorkerIfNeeded(scheduling_key); + RAY_LOG(WARNING) << "The worker failed to receive a response from the local " + << "raylet because the raylet is unavailable (crashed). " + << "Error: " << status; + if (worker_type_ == WorkerType::WORKER) { + // Exit the worker so that caller can retry somewhere else. + RAY_LOG(WARNING) << "Terminating the worker due to local raylet death"; + QuickExit(); + } + RAY_CHECK(worker_type_ == WorkerType::DRIVER); + error_type = rpc::ErrorType::LOCAL_RAYLET_DIED; + error_status = status; + // Grpc errors are not helpful at all. So we are overwriting it. + std::stringstream ss; + ss << "The worker failed to receive a response from the local raylet" + << "(id: " << NodeID::FromBinary(raylet_address.node_id()).Hex() + << " ,ip: " << raylet_address.ip_address() << ") " + << "because the raylet is " + "unavailable (crashed)."; + error_info.set_error_message(ss.str()); + tasks_to_fail = std::move(sched_entry.task_queue); + sched_entry.task_queue.clear(); + if (sched_entry.CanDelete()) { + scheduling_key_entries_.erase(scheduling_key); } } } @@ -584,9 +571,9 @@ void NormalTaskSubmitter::PushNormalTask( failed_tasks_pending_failure_cause_.erase(task_id); }; auto &cur_lease_entry = worker_to_lease_entry_[addr]; - RAY_CHECK(cur_lease_entry.raylet_client); - cur_lease_entry.raylet_client->GetWorkerFailureCause(cur_lease_entry.lease_id, - callback); + auto raylet_client = raylet_client_pool_->GetByID(cur_lease_entry.node_id); + RAY_CHECK(raylet_client); + raylet_client->GetWorkerFailureCause(cur_lease_entry.lease_id, callback); } OnWorkerIdle(addr, scheduling_key, diff --git a/src/ray/core_worker/task_submission/normal_task_submitter.h b/src/ray/core_worker/task_submission/normal_task_submitter.h index f7b385f05a36..86dfe685689f 100644 --- a/src/ray/core_worker/task_submission/normal_task_submitter.h +++ b/src/ray/core_worker/task_submission/normal_task_submitter.h @@ -194,7 +194,7 @@ class NormalTaskSubmitter { /// Set up client state for newly granted worker lease. void AddWorkerLeaseClient( const rpc::Address &addr, - std::shared_ptr raylet_client, + const NodeID &node_id, const google::protobuf::RepeatedPtrField &assigned_resources, const SchedulingKey &scheduling_key, const LeaseID &lease_id) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); @@ -276,14 +276,14 @@ class NormalTaskSubmitter { const JobID job_id_; /// A LeaseEntry struct is used to condense the metadata about a single executor: - /// (1) The lease client through which the worker should be returned + /// (1) The node id of the leased worker. /// (2) The expiration time of a worker's lease. /// (3) Whether the worker has assigned task to do. - /// (5) The resources assigned to the worker - /// (6) The SchedulingKey assigned to tasks that will be sent to the worker - /// (7) The task id used to obtain the worker lease. + /// (4) The resources assigned to the worker + /// (5) The SchedulingKey assigned to tasks that will be sent to the worker + /// (6) The task id used to obtain the worker lease. struct LeaseEntry { - std::shared_ptr raylet_client; + NodeID node_id; int64_t lease_expiration_time; google::protobuf::RepeatedPtrField assigned_resources; SchedulingKey scheduling_key; diff --git a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc index f3c741b5a5f4..ec3ae6d7b93f 100644 --- a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc @@ -311,7 +311,8 @@ class MockRayletClient : public FakeRayletClient { bool GrantWorkerLease( const std::string &address, int port, - const NodeID &retry_at_node_id, + const NodeID &granted_node_id, + const NodeID &retry_at_node_id = NodeID::Nil(), bool cancel = false, std::string worker_id = WorkerID::FromRandom().Binary(), bool reject = false, @@ -330,7 +331,7 @@ class MockRayletClient : public FakeRayletClient { } else { reply.mutable_worker_address()->set_ip_address(address); reply.mutable_worker_address()->set_port(port); - reply.mutable_worker_address()->set_node_id(retry_at_node_id.Binary()); + reply.mutable_worker_address()->set_node_id(granted_node_id.Binary()); reply.mutable_worker_address()->set_worker_id(worker_id); } rpc::ClientCallback callback = PopCallbackInLock(); @@ -563,7 +564,7 @@ TEST_F(NormalTaskSubmitterTest, TestLocalityAwareSubmitOneTask) { ASSERT_EQ(raylet_client->num_workers_returned, 0); ASSERT_EQ(worker_client->callbacks.size(), 0); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), 1); ASSERT_EQ(task_manager->num_tasks_complete, 0); ASSERT_EQ(task_manager->num_tasks_failed, 0); @@ -594,7 +595,7 @@ TEST_F(NormalTaskSubmitterTest, TestSubmitOneTask) { ASSERT_EQ(raylet_client->num_workers_returned, 0); ASSERT_EQ(worker_client->callbacks.size(), 0); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), 1); ASSERT_EQ(task_manager->num_tasks_complete, 0); ASSERT_EQ(task_manager->num_tasks_failed, 0); @@ -620,7 +621,7 @@ TEST_F(NormalTaskSubmitterTest, TestRetryTaskApplicationLevelError) { task.GetMutableMessage().set_retry_exceptions(true); submitter.SubmitTask(task); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, local_node_id)); // Simulate an application-level error. ASSERT_TRUE(worker_client->ReplyPushTask(Status::OK(), false, true)); ASSERT_EQ(raylet_client->num_workers_returned, 1); @@ -634,7 +635,7 @@ TEST_F(NormalTaskSubmitterTest, TestRetryTaskApplicationLevelError) { task.GetMutableMessage().set_retry_exceptions(false); submitter.SubmitTask(task); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, local_node_id)); // Simulate an application-level error. ASSERT_TRUE(worker_client->ReplyPushTask(Status::OK(), false, true)); ASSERT_EQ(raylet_client->num_workers_returned, 2); @@ -656,7 +657,7 @@ TEST_F(NormalTaskSubmitterTest, TestHandleTaskFailure) { TaskSpecification task = BuildEmptyTaskSpec(); submitter.SubmitTask(task); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, local_node_id)); // Simulate a system failure, i.e., worker died unexpectedly. ASSERT_TRUE(worker_client->ReplyPushTask(Status::IOError("oops"))); ASSERT_TRUE(raylet_client->ReplyGetWorkerFailureCause()); @@ -684,7 +685,7 @@ TEST_F(NormalTaskSubmitterTest, TestCancellationWhileHandlingTaskFailure) { TaskSpecification task = BuildEmptyTaskSpec(); submitter.SubmitTask(task); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, local_node_id)); // Simulate a system failure, i.e., worker died unexpectedly so that // GetWorkerFailureCause is called. ASSERT_TRUE(worker_client->ReplyPushTask(Status::IOError("oops"))); @@ -716,6 +717,7 @@ TEST_F(NormalTaskSubmitterTest, TestHandleUnschedulableTask) { ASSERT_TRUE(raylet_client->GrantWorkerLease( "", 0, + local_node_id, NodeID::Nil(), true, "", @@ -730,6 +732,7 @@ TEST_F(NormalTaskSubmitterTest, TestHandleUnschedulableTask) { ASSERT_TRUE(raylet_client->GrantWorkerLease( "", 0, + local_node_id, NodeID::Nil(), true, "", @@ -766,6 +769,7 @@ TEST_F(NormalTaskSubmitterTest, TestHandleRuntimeEnvSetupFailed) { ASSERT_TRUE(raylet_client->GrantWorkerLease( "", 0, + local_node_id, NodeID::Nil(), true, "", @@ -780,6 +784,7 @@ TEST_F(NormalTaskSubmitterTest, TestHandleRuntimeEnvSetupFailed) { ASSERT_TRUE(raylet_client->GrantWorkerLease( "", 0, + local_node_id, NodeID::Nil(), true, "", @@ -862,7 +867,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeases) { // Grant the first round of leases. for (int i = 0; i < concurrency; i++) { - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", i, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", i, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), i + 1); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, concurrency + i + 1); ASSERT_EQ(raylet_client->num_workers_requested, concurrency + i + 1); @@ -870,7 +875,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeases) { } for (int i = 0; i < concurrency; i++) { ASSERT_TRUE( - raylet_client->GrantWorkerLease("localhost", concurrency + i, NodeID::Nil())); + raylet_client->GrantWorkerLease("localhost", concurrency + i, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), concurrency + i + 1); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, tasks.size()); ASSERT_EQ(raylet_client->num_workers_requested, tasks.size()); @@ -917,14 +922,14 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamic) { ASSERT_EQ(raylet_client->reported_backlog_size, tasks.size() - 1); // Max concurrency is still 1. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, local_node_id)); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2); ASSERT_EQ(raylet_client->num_workers_requested, 2); ASSERT_EQ(raylet_client->reported_backlog_size, tasks.size() - 2); // Increase max concurrency. Should request leases up to the max concurrency. rateLimiter->limit_ = concurrency; - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, local_node_id)); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2 + concurrency); ASSERT_EQ(raylet_client->num_workers_requested, 2 + concurrency); ASSERT_EQ(raylet_client->reported_backlog_size, @@ -935,7 +940,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamic) { // concurrency. rateLimiter->limit_ = 1; for (int i = 0; i < concurrency - 1; i++) { - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", i, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", i, local_node_id)); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2 + concurrency); ASSERT_EQ(raylet_client->num_workers_requested, 2 + concurrency); ASSERT_EQ(raylet_client->reported_backlog_size, @@ -948,14 +953,14 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamic) { raylet_client->num_workers_requested = 0; for (int i = 0; i < num_tasks_remaining; i++) { ASSERT_TRUE( - raylet_client->GrantWorkerLease("localhost", concurrency + i, NodeID::Nil())); + raylet_client->GrantWorkerLease("localhost", concurrency + i, local_node_id)); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, i + 1); ASSERT_EQ(raylet_client->num_workers_requested, i + 1); } lease_policy_ptr->num_lease_policy_consults = 0; raylet_client->num_workers_requested = 0; - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 2000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 2000, local_node_id)); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 0); ASSERT_EQ(raylet_client->num_workers_requested, 0); @@ -1002,7 +1007,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamicWithSpillback) ASSERT_EQ(raylet_client->reported_backlog_size, tasks.size() - 1); // Max concurrency is still 1. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, local_node_id)); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2); ASSERT_EQ(raylet_client->num_workers_requested, 2); ASSERT_EQ(raylet_client->reported_backlog_size, tasks.size() - 2); @@ -1010,7 +1015,9 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamicWithSpillback) // Increase max concurrency. rateLimiter->limit_ = concurrency; // The outstanding lease request is spilled back to a remote raylet. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, NodeID::FromRandom())); + auto remote_node_id = NodeID::FromRandom(); + ASSERT_TRUE( + raylet_client->GrantWorkerLease("localhost", 1001, NodeID::Nil(), remote_node_id)); // We should request one lease request from the spillback raylet and then the // rest from the raylet returned by the lease policy. ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, concurrency + 1); @@ -1023,7 +1030,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamicWithSpillback) // concurrency. rateLimiter->limit_ = 1; for (int i = 0; i < concurrency - 1; i++) { - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", i, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", i, local_node_id)); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, concurrency + 1); ASSERT_EQ(raylet_client->num_workers_requested, 2 + concurrency); ASSERT_EQ(raylet_client->reported_backlog_size, @@ -1036,14 +1043,14 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentWorkerLeasesDynamicWithSpillback) raylet_client->num_workers_requested = 0; for (int i = 0; i < num_tasks_remaining; i++) { ASSERT_TRUE( - raylet_client->GrantWorkerLease("localhost", concurrency + i, NodeID::Nil())); + raylet_client->GrantWorkerLease("localhost", concurrency + i, local_node_id)); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, i + 1); ASSERT_EQ(raylet_client->num_workers_requested, i + 1); } lease_policy_ptr->num_lease_policy_consults = 0; raylet_client->num_workers_requested = 0; - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 2000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 2000, local_node_id)); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 0); ASSERT_EQ(raylet_client->num_workers_requested, 0); @@ -1079,21 +1086,21 @@ TEST_F(NormalTaskSubmitterTest, TestSubmitMultipleTasks) { ASSERT_EQ(raylet_client->reported_backlog_size, 0); // Task 1 is pushed; worker 2 is requested. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), 1); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2); ASSERT_EQ(raylet_client->num_workers_requested, 2); ASSERT_EQ(raylet_client->reported_backlog_size, 1); // Task 2 is pushed; worker 3 is requested. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), 2); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 3); ASSERT_EQ(raylet_client->num_workers_requested, 3); ASSERT_EQ(raylet_client->reported_backlog_size, 0); // Task 3 is pushed; no more workers requested. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1002, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1002, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), 3); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 3); ASSERT_EQ(raylet_client->num_workers_requested, 3); @@ -1129,7 +1136,7 @@ TEST_F(NormalTaskSubmitterTest, TestReuseWorkerLease) { ASSERT_EQ(raylet_client->num_workers_requested, 1); // Task 1 is pushed. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), 1); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2); ASSERT_EQ(raylet_client->num_workers_requested, 2); @@ -1152,7 +1159,7 @@ TEST_F(NormalTaskSubmitterTest, TestReuseWorkerLease) { ASSERT_EQ(raylet_client->num_workers_returned, 1); // The second lease request is returned immediately. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), 0); ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 2); ASSERT_EQ(raylet_client->num_workers_returned, 2); @@ -1180,7 +1187,7 @@ TEST_F(NormalTaskSubmitterTest, TestRetryLeaseCancellation) { ASSERT_EQ(raylet_client->num_workers_requested, 1); // Task 1 is pushed. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, local_node_id)); // Task 1 finishes, Task 2 is scheduled on the same worker. ASSERT_TRUE(worker_client->ReplyPushTask()); // Task 2 finishes, Task 3 is scheduled on the same worker. @@ -1202,7 +1209,8 @@ TEST_F(NormalTaskSubmitterTest, TestRetryLeaseCancellation) { ASSERT_EQ(raylet_client->num_leases_canceled, i); ASSERT_FALSE(raylet_client->ReplyCancelWorkerLease()); ASSERT_EQ(raylet_client->num_leases_canceled, i); - ASSERT_TRUE(raylet_client->GrantWorkerLease("", 0, NodeID::Nil(), /*cancel=*/true)); + ASSERT_TRUE(raylet_client->GrantWorkerLease( + "", 0, local_node_id, NodeID::Nil(), /*cancel=*/true)); ASSERT_EQ(worker_client->callbacks.size(), 0); // The canceled lease is not returned. ASSERT_EQ(raylet_client->num_workers_returned, 1); @@ -1226,7 +1234,7 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentCancellationAndSubmission) { submitter.SubmitTask(task2); // Task 1 is pushed. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, local_node_id)); ASSERT_EQ(raylet_client->num_workers_requested, 2); // Task 1 finishes, Task 2 is scheduled on the same worker. ASSERT_TRUE(worker_client->ReplyPushTask()); @@ -1245,11 +1253,12 @@ TEST_F(NormalTaskSubmitterTest, TestConcurrentCancellationAndSubmission) { // Task 2's lease request is canceled, a new worker is requested for task 3. ASSERT_TRUE(raylet_client->ReplyCancelWorkerLease()); ASSERT_EQ(raylet_client->num_workers_requested, 2); - ASSERT_TRUE(raylet_client->GrantWorkerLease("", 0, NodeID::Nil(), /*cancel=*/true)); + ASSERT_TRUE(raylet_client->GrantWorkerLease( + "", 0, local_node_id, NodeID::Nil(), /*cancel=*/true)); ASSERT_EQ(raylet_client->num_workers_requested, 3); // Task 3 finishes, all workers returned. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, local_node_id)); ASSERT_TRUE(worker_client->ReplyPushTask()); ASSERT_EQ(raylet_client->num_workers_returned, 2); ASSERT_FALSE(raylet_client->ReplyCancelWorkerLease()); @@ -1271,7 +1280,7 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerNotReusedOnError) { ASSERT_EQ(raylet_client->num_workers_requested, 1); // Task 1 is pushed. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), 1); ASSERT_EQ(raylet_client->num_workers_requested, 2); @@ -1283,7 +1292,7 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerNotReusedOnError) { ASSERT_EQ(raylet_client->num_workers_disconnected, 1); // Task 2 runs successfully on the second worker. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, local_node_id)); ASSERT_TRUE(worker_client->ReplyPushTask()); ASSERT_EQ(raylet_client->num_workers_returned, 1); ASSERT_EQ(raylet_client->num_workers_disconnected, 1); @@ -1306,7 +1315,7 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerNotReturnedOnExit) { ASSERT_EQ(raylet_client->num_workers_requested, 1); // Task 1 is pushed. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), 1); // Task 1 finishes with exit status; the worker is not returned. @@ -1347,15 +1356,16 @@ TEST_F(NormalTaskSubmitterTest, TestSpillback) { // Spillback to a remote node. auto remote_node_id = NodeID::FromRandom(); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 7777, remote_node_id)); + ASSERT_TRUE( + raylet_client->GrantWorkerLease("localhost", 7777, NodeID::Nil(), remote_node_id)); ASSERT_EQ(remote_raylet_clients.count(7777), 1); // Confirm that lease policy is not consulted on spillback. ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 1); // There should be no more callbacks on the local client. - ASSERT_FALSE(raylet_client->GrantWorkerLease("remote", 1234, NodeID::Nil())); + ASSERT_FALSE(raylet_client->GrantWorkerLease("remote", 1234, local_node_id)); // Trigger retry at the remote node. ASSERT_TRUE( - remote_raylet_clients[7777]->GrantWorkerLease("remote", 1234, NodeID::Nil())); + remote_raylet_clients[7777]->GrantWorkerLease("remote", 1234, remote_node_id)); // The worker is returned to the remote node, not the local one. ASSERT_TRUE(worker_client->ReplyPushTask()); @@ -1404,27 +1414,33 @@ TEST_F(NormalTaskSubmitterTest, TestSpillbackRoundTrip) { // Spillback to a remote node. auto remote_node_id = NodeID::FromRandom(); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 7777, remote_node_id)); + rpc::Address remote_address; + remote_address.set_node_id(remote_node_id.Binary()); + remote_address.set_ip_address("localhost"); + remote_address.set_port(7777); + raylet_client_pool->GetOrConnectByAddress(remote_address); + ASSERT_TRUE( + raylet_client->GrantWorkerLease("localhost", 7777, NodeID::Nil(), remote_node_id)); ASSERT_EQ(remote_raylet_clients.count(7777), 1); ASSERT_EQ(remote_raylet_clients[7777]->num_workers_requested, 1); // Confirm that the spillback lease request has grant_or_reject set to true. ASSERT_EQ(remote_raylet_clients[7777]->num_grant_or_reject_leases_requested, 1); // Confirm that lease policy is not consulted on spillback. ASSERT_EQ(lease_policy_ptr->num_lease_policy_consults, 1); - ASSERT_FALSE(raylet_client->GrantWorkerLease("remote", 1234, NodeID::Nil())); + ASSERT_FALSE(raylet_client->GrantWorkerLease("remote", 1234, local_node_id)); // Trigger a rejection back to the local node. ASSERT_TRUE(remote_raylet_clients[7777]->GrantWorkerLease( - "local", 1234, local_node_id, false, "", /*reject=*/true)); + "local", 1234, remote_node_id, NodeID::Nil(), false, "", /*reject=*/true)); // We should not have created another lease client to the local raylet. ASSERT_EQ(remote_raylet_clients.size(), 1); // There should be no more callbacks on the remote node. ASSERT_FALSE( - remote_raylet_clients[7777]->GrantWorkerLease("remote", 1234, NodeID::Nil())); + remote_raylet_clients[7777]->GrantWorkerLease("remote", 1234, remote_node_id)); // The worker is returned to the local node. ASSERT_EQ(raylet_client->num_grant_or_reject_leases_requested, 0); ASSERT_EQ(raylet_client->num_workers_requested, 2); - ASSERT_TRUE(raylet_client->GrantWorkerLease("local", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("local", 1234, local_node_id)); ASSERT_TRUE(worker_client->ReplyPushTask()); ASSERT_EQ(raylet_client->num_workers_returned, 1); ASSERT_EQ(remote_raylet_clients[7777]->num_workers_returned, 0); @@ -1451,6 +1467,7 @@ void TestSchedulingKey(const std::shared_ptr store, const TaskSpecification &same2, const TaskSpecification &different) { rpc::Address address; + auto local_node_id = NodeID::FromRandom(); auto raylet_client = std::make_shared(); auto raylet_client_pool = std::make_shared( [&](const rpc::Address &addr) { return raylet_client; }); @@ -1460,7 +1477,7 @@ void TestSchedulingKey(const std::shared_ptr store, auto task_manager = std::make_unique(); auto actor_creator = std::make_shared(); auto lease_policy = std::make_unique(); - lease_policy->SetNodeID(NodeID::FromRandom()); + lease_policy->SetNodeID(local_node_id); instrumented_io_context io_context; NormalTaskSubmitter submitter( address, @@ -1470,7 +1487,7 @@ void TestSchedulingKey(const std::shared_ptr store, std::move(lease_policy), store, *task_manager, - NodeID::Nil(), + local_node_id, WorkerType::WORKER, kLongTimeout, actor_creator, @@ -1488,7 +1505,7 @@ void TestSchedulingKey(const std::shared_ptr store, /*timeout_ms=*/1000); // same1 is pushed. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), 1); // Another worker is requested because same2 is pending. ASSERT_EQ(raylet_client->num_workers_requested, 3); @@ -1504,7 +1521,7 @@ void TestSchedulingKey(const std::shared_ptr store, ASSERT_TRUE(raylet_client->ReplyCancelWorkerLease()); // different is pushed. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, local_node_id)); ASSERT_EQ(worker_client->callbacks.size(), 2); ASSERT_EQ(raylet_client->num_workers_requested, 3); @@ -1521,7 +1538,8 @@ void TestSchedulingKey(const std::shared_ptr store, ASSERT_EQ(raylet_client->num_leases_canceled, 1); // Trigger reply to RequestWorkerLease to remove the canceled pending lease request - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1002, NodeID::Nil(), true)); + ASSERT_TRUE(raylet_client->GrantWorkerLease( + "localhost", 1002, local_node_id, NodeID::Nil(), true)); ASSERT_EQ(raylet_client->num_workers_returned, 2); // Check that there are no entries left in the scheduling_key_entries_ hashmap. These @@ -1694,7 +1712,7 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerLeaseTimeout) { ASSERT_EQ(raylet_client->num_workers_requested, 1); // Task 1 is pushed. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, local_node_id)); ASSERT_EQ(raylet_client->num_workers_requested, 2); // Task 1 finishes with failure; the worker is returned due to the error even though @@ -1706,7 +1724,7 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerLeaseTimeout) { // Task 2 runs successfully on the second worker; the worker is returned due to the // timeout. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1001, local_node_id)); std::this_thread::sleep_for( std::chrono::milliseconds(10)); // Sleep for 10ms, causing the lease to time out. ASSERT_TRUE(worker_client->ReplyPushTask()); @@ -1715,7 +1733,7 @@ TEST_F(NormalTaskSubmitterTest, TestWorkerLeaseTimeout) { // Task 3 runs successfully on the third worker; the worker is returned even though it // hasn't timed out. - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1002, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1002, local_node_id)); ASSERT_TRUE(worker_client->ReplyPushTask()); ASSERT_EQ(worker_client->callbacks.size(), 0); ASSERT_EQ(raylet_client->num_workers_returned, 2); @@ -1734,7 +1752,7 @@ TEST_F(NormalTaskSubmitterTest, TestKillExecutingTask) { TaskSpecification task = BuildEmptyTaskSpec(); submitter.SubmitTask(task); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, local_node_id)); // Try force kill, exiting the worker submitter.CancelTask(task, true, false); @@ -1751,7 +1769,7 @@ TEST_F(NormalTaskSubmitterTest, TestKillExecutingTask) { task.GetMutableMessage().set_task_id( TaskID::ForNormalTask(JobID::Nil(), TaskID::Nil(), 1).Binary()); submitter.SubmitTask(task); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, local_node_id)); // Try non-force kill, worker returns normally submitter.CancelTask(task, false, false); @@ -1787,7 +1805,8 @@ TEST_F(NormalTaskSubmitterTest, TestKillPendingTask) { ASSERT_TRUE(raylet_client->ReplyCancelWorkerLease()); // Trigger reply to RequestWorkerLease to remove the canceled pending lease request - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1000, NodeID::Nil(), true)); + ASSERT_TRUE(raylet_client->GrantWorkerLease( + "localhost", 1000, local_node_id, NodeID::Nil(), true)); // Check that there are no entries left in the scheduling_key_entries_ hashmap. These // would otherwise cause a memory leak. @@ -1824,7 +1843,7 @@ TEST_F(NormalTaskSubmitterTest, TestQueueGeneratorForResubmit) { CreateNormalTaskSubmitter(std::make_shared(1)); TaskSpecification task = BuildEmptyTaskSpec(); submitter.SubmitTask(task); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, local_node_id)); ASSERT_TRUE(submitter.QueueGeneratorForResubmit(task)); ASSERT_TRUE(worker_client->ReplyPushTask()); ASSERT_EQ(task_manager->num_tasks_complete, 0); @@ -1839,7 +1858,7 @@ TEST_F(NormalTaskSubmitterTest, TestCancelBeforeAfterQueueGeneratorForResubmit) CreateNormalTaskSubmitter(std::make_shared(1)); TaskSpecification task = BuildEmptyTaskSpec(); submitter.SubmitTask(task); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, local_node_id)); submitter.CancelTask(task, /*force_kill=*/false, /*recursive=*/true); ASSERT_FALSE(submitter.QueueGeneratorForResubmit(task)); worker_client->ReplyCancelTask(); @@ -1856,7 +1875,7 @@ TEST_F(NormalTaskSubmitterTest, TestCancelBeforeAfterQueueGeneratorForResubmit) // resubmit. TaskSpecification task2 = BuildEmptyTaskSpec(); submitter.SubmitTask(task2); - ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, NodeID::Nil())); + ASSERT_TRUE(raylet_client->GrantWorkerLease("localhost", 1234, local_node_id)); ASSERT_TRUE(submitter.QueueGeneratorForResubmit(task2)); submitter.CancelTask(task2, /*force_kill=*/false, /*recursive=*/true); ASSERT_TRUE(worker_client->ReplyPushTask()); diff --git a/src/ray/raylet/node_manager.cc b/src/ray/raylet/node_manager.cc index 3503f3d3d7f4..b48290b51adf 100644 --- a/src/ray/raylet/node_manager.cc +++ b/src/ray/raylet/node_manager.cc @@ -1627,6 +1627,21 @@ void NodeManager::HandleReportWorkerBacklog( void NodeManager::HandleRequestWorkerLease(rpc::RequestWorkerLeaseRequest request, rpc::RequestWorkerLeaseReply *reply, rpc::SendReplyCallback send_reply_callback) { + auto lease_id = LeaseID::FromBinary(request.lease_spec().lease_id()); + // If the lease is already granted, this is a retry and forward the address of the + // already leased worker to use. + if (leased_workers_.contains(lease_id)) { + const auto &worker = leased_workers_[lease_id]; + RAY_LOG(DEBUG) << "Lease " << lease_id + << " is already granted with worker: " << worker->WorkerId(); + reply->set_worker_pid(worker->GetProcess().GetId()); + reply->mutable_worker_address()->set_ip_address(worker->IpAddress()); + reply->mutable_worker_address()->set_port(worker->Port()); + reply->mutable_worker_address()->set_worker_id(worker->WorkerId().Binary()); + reply->mutable_worker_address()->set_node_id(self_node_id_.Binary()); + send_reply_callback(Status::OK(), nullptr, nullptr); + return; + } RayLease lease{std::move(*request.mutable_lease_spec())}; const auto caller_worker = WorkerID::FromBinary(lease.GetLeaseSpecification().CallerAddress().worker_id()); @@ -1671,7 +1686,7 @@ void NodeManager::HandleRequestWorkerLease(rpc::RequestWorkerLeaseRequest reques RAY_LOG(DEBUG).WithField(actor_id) << "Reject leasing as the raylet has no enough resources. " "normal_task_resources = " - << normal_task_resources.DebugString() << ", local_resoruce_view = " + << normal_task_resources.DebugString() << ", local_resource_view = " << cluster_resource_scheduler_.GetClusterResourceManager() .GetNodeResourceViewString( scheduling::NodeID(self_node_id_.Binary())); diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index 5dce1b2d3ec7..8e4217ea9dfd 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -1002,6 +1002,86 @@ INSTANTIATE_TEST_SUITE_P(NodeManagerReturnWorkerLeaseIdempotentVariations, NodeManagerReturnWorkerLeaseIdempotentTest, testing::Combine(testing::Bool(), testing::Bool())); +TEST_F(NodeManagerTest, TestHandleRequestWorkerLeaseIdempotent) { + auto lease_spec = BuildLeaseSpec({}); + rpc::RequestWorkerLeaseRequest request; + rpc::RequestWorkerLeaseReply reply1; + rpc::RequestWorkerLeaseReply reply2; + LeaseID lease_id = LeaseID::FromRandom(); + lease_spec.GetMutableMessage().set_lease_id(lease_id.Binary()); + request.mutable_lease_spec()->CopyFrom(lease_spec.GetMessage()); + request.set_backlog_size(1); + request.set_grant_or_reject(true); + request.set_is_selected_based_on_locality(true); + auto worker = std::make_shared(WorkerID::FromRandom(), 10); + PopWorkerCallback pop_worker_callback; + EXPECT_CALL(mock_worker_pool_, PopWorker(_, _)) + .Times(1) + .WillOnce([&](const LeaseSpecification &ls, const PopWorkerCallback &callback) { + pop_worker_callback = callback; + }); + node_manager_->HandleRequestWorkerLease( + request, + &reply1, + [](Status s, std::function success, std::function failure) { + ASSERT_TRUE(s.ok()); + }); + pop_worker_callback(worker, PopWorkerStatus::OK, ""); + ASSERT_EQ(leased_workers_.size(), 1); + ASSERT_EQ(leased_workers_[lease_id]->GetGrantedLeaseId(), lease_id); + request.mutable_lease_spec()->CopyFrom(lease_spec.GetMessage()); + node_manager_->HandleRequestWorkerLease( + request, + &reply2, + [](Status s, std::function success, std::function failure) { + ASSERT_TRUE(s.ok()); + }); + ASSERT_EQ(leased_workers_.size(), 1); + ASSERT_EQ(leased_workers_[lease_id]->GetGrantedLeaseId(), lease_id); + ASSERT_EQ(leased_workers_[lease_id]->WorkerId(), + WorkerID::FromBinary(reply1.worker_address().worker_id())); + ASSERT_EQ(reply1.worker_address(), reply2.worker_address()); +} + +TEST_F(NodeManagerTest, TestHandleRequestWorkerLeaseInfeasibleIdempotent) { + auto lease_spec = BuildLeaseSpec({{"CPU", 1}}); + lease_spec.GetMutableMessage() + .mutable_scheduling_strategy() + ->mutable_node_affinity_scheduling_strategy() + ->set_soft(false); // Hard constraint + + rpc::RequestWorkerLeaseRequest request; + rpc::RequestWorkerLeaseReply reply1; + rpc::RequestWorkerLeaseReply reply2; + LeaseID lease_id = LeaseID::FromRandom(); + lease_spec.GetMutableMessage().set_lease_id(lease_id.Binary()); + request.mutable_lease_spec()->CopyFrom(lease_spec.GetMessage()); + request.set_backlog_size(1); + request.set_grant_or_reject(true); + request.set_is_selected_based_on_locality(true); + node_manager_->HandleRequestWorkerLease( + request, + &reply1, + [](Status s, std::function success, std::function failure) { + ASSERT_TRUE(s.ok()); + }); + ASSERT_EQ(leased_workers_.size(), 0); + ASSERT_EQ(reply1.canceled(), true); + ASSERT_EQ(reply1.failure_type(), + rpc::RequestWorkerLeaseReply::SCHEDULING_CANCELLED_UNSCHEDULABLE); + request.mutable_lease_spec()->CopyFrom(lease_spec.GetMessage()); + node_manager_->HandleRequestWorkerLease( + request, + &reply2, + [](Status s, std::function success, std::function failure) { + ASSERT_TRUE(s.ok()); + }); + ASSERT_EQ(leased_workers_.size(), 0); + ASSERT_EQ(reply1.canceled(), reply2.canceled()); + ASSERT_EQ(reply1.failure_type(), reply2.failure_type()); + ASSERT_EQ(reply1.scheduling_failure_message(), reply2.scheduling_failure_message()); +} + size_t GetPendingLeaseWorkerCount(const LocalLeaseManager &local_lease_manager) { return local_lease_manager.waiting_lease_queue_.size() + local_lease_manager.leases_to_grant_.size(); diff --git a/src/ray/rpc/raylet/raylet_client.cc b/src/ray/rpc/raylet/raylet_client.cc index de28d5b8e09a..622271f4a30a 100644 --- a/src/ray/rpc/raylet/raylet_client.cc +++ b/src/ray/rpc/raylet/raylet_client.cc @@ -53,24 +53,18 @@ void RayletClient::RequestWorkerLease( const rpc::ClientCallback &callback, const int64_t backlog_size, const bool is_selected_based_on_locality) { - google::protobuf::Arena arena; - auto request = - google::protobuf::Arena::CreateMessage(&arena); - // The unsafe allocating here is actually safe because the life-cycle of - // lease_spec is longer than request. - // Request will be sent before the end of this call, and after that, it won't be - // used any more. - request->unsafe_arena_set_allocated_lease_spec( - const_cast(&lease_spec)); - request->set_grant_or_reject(grant_or_reject); - request->set_backlog_size(backlog_size); - request->set_is_selected_based_on_locality(is_selected_based_on_locality); - INVOKE_RPC_CALL(NodeManagerService, - RequestWorkerLease, - *request, - callback, - grpc_client_, - /*method_timeout_ms*/ -1); + rpc::RequestWorkerLeaseRequest request; + request.mutable_lease_spec()->CopyFrom(lease_spec); + request.set_grant_or_reject(grant_or_reject); + request.set_backlog_size(backlog_size); + request.set_is_selected_based_on_locality(is_selected_based_on_locality); + INVOKE_RETRYABLE_RPC_CALL(retryable_grpc_client_, + NodeManagerService, + RequestWorkerLease, + request, + callback, + grpc_client_, + /*method_timeout_ms*/ -1); } void RayletClient::PrestartWorkers( From 6cefec758d66e71324bbb29affd5ef208b9fd288 Mon Sep 17 00:00:00 2001 From: Potato Date: Tue, 9 Sep 2025 03:48:59 +0800 Subject: [PATCH 507/634] [DOC] Fix documentation issues in _includes and _templates directories (#56272) Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- doc/source/_includes/_help.rst | 2 +- doc/source/_templates/csat.html | 4 ++-- doc/source/_templates/template.ipynb | 14 +++++++------- doc/source/_templates/template.md | 14 +++++++------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/source/_includes/_help.rst b/doc/source/_includes/_help.rst index 05f46e7dcdf7..aecc526fd21c 100644 --- a/doc/source/_includes/_help.rst +++ b/doc/source/_includes/_help.rst @@ -3,7 +3,7 @@ You can post questions or issues or feedback through the following channels: 1. `Discussion Board`_: For **questions about Ray usage** or **feature requests**. 2. `GitHub Issues`_: For **bug reports**. 3. `Ray Slack`_: For **getting in touch** with Ray maintainers. -4. `StackOverflow`_: Use the [ray] tag **questions about Ray**. +4. `StackOverflow`_: Use the [ray] tag for **questions about Ray**. .. _`Discussion Board`: https://discuss.ray.io/ .. _`GitHub Issues`: https://github.com/ray-project/ray/issues diff --git a/doc/source/_templates/csat.html b/doc/source/_templates/csat.html index 368af0d322d2..852245d7bd50 100644 --- a/doc/source/_templates/csat.html +++ b/doc/source/_templates/csat.html @@ -8,13 +8,13 @@ - Yes + Yes
- No + No
diff --git a/doc/source/_templates/template.ipynb b/doc/source/_templates/template.ipynb index b1778ed9fb97..6ea5ba018ab4 100644 --- a/doc/source/_templates/template.ipynb +++ b/doc/source/_templates/template.ipynb @@ -15,9 +15,9 @@ "If you want to learn more about the MyST parser, see the\n", "[MyST documentation](https://myst-parser.readthedocs.io/en/latest/).\n", "\n", - "MyST is common markdown compliant, so if you can use plain markdown here.\n", - "In case you need to execute restructured text (`rSt`) directives, you can use `{eval-rst}` to execute the code.\n", - "For instance, a here's a note written in rSt:\n", + "MyST is CommonMark compliant, so you can use plain markdown here.\n", + "In case you need to execute restructured text (rST) directives, you can use `{eval-rst}` to execute the code.\n", + "For instance, here's a note written in rST:\n", "\n", "```{eval-rst}\n", ".. note::\n", @@ -69,10 +69,10 @@ "source": [ "## Hiding and removing cells\n", "\n", - "You can hide cells, so that they will toggle when you click on the cell header.\n", + "You can hide cells, so that they toggle when you click the cell header.\n", "You can use different `:tags:` like `hide-cell`, `hide-input`, or `hide-output` to hide cell content,\n", - "and you can use `remove-cell`, `remove-input`, or `remove-output` to remove the cell completely when rendered.\n", - "Those cells will still show up in the notebook itself." + "and you can use `remove-cell`, `remove-input`, or `remove-output` to completely remove the cell when rendered.\n", + "Those cells still show up in the notebook itself." ] }, { @@ -107,7 +107,7 @@ "And this is a note.\n", ":::\n", "\n", - "The following cell will be removed and not render:" + "The following cell doesn't render:" ] }, { diff --git a/doc/source/_templates/template.md b/doc/source/_templates/template.md index 8e1cc0c58655..cf330f14eba3 100644 --- a/doc/source/_templates/template.md +++ b/doc/source/_templates/template.md @@ -20,9 +20,9 @@ For more information on MyST notebooks, see the If you want to learn more about the MyST parser, see the [MyST documentation](https://myst-parser.readthedocs.io/en/latest/). -MyST is common markdown compliant, so if you can use plain markdown here. -In case you need to execute restructured text (`rSt`) directives, you can use `{eval-rst}` to execute the code. -For instance, a here's a note written in rSt: +MyST is CommonMark compliant, so you can use plain markdown here. +In case you need to execute restructured text (rST) directives, you can use `{eval-rst}` to execute the code. +For instance, here's a note written in rST: ```{eval-rst} .. note:: @@ -65,10 +65,10 @@ checkpoint_path = train_ppo_model() ## Hiding and removing cells -You can hide cells, so that they will toggle when you click on the cell header. +You can hide cells, so that they toggle when you click the cell header. You can use different `:tags:` like `hide-cell`, `hide-input`, or `hide-output` to hide cell content, -and you can use `remove-cell`, `remove-input`, or `remove-output` to remove the cell completely when rendered. -Those cells will still show up in the notebook itself. +and you can use `remove-cell`, `remove-input`, or `remove-output` to completely remove the cell when rendered. +Those cells still show up in the notebook itself. ```{code-cell} python3 :tags: [hide-cell] @@ -88,7 +88,7 @@ Here's a quick tip. And this is a note. ::: -The following cell will be removed and not render: +The following cell doesn't render: ```{code-cell} python3 :tags: [remove-cell] From d55b35da2c691b3ec80b949b05606fe4e606af82 Mon Sep 17 00:00:00 2001 From: Justin Yu Date: Mon, 8 Sep 2025 13:40:51 -0700 Subject: [PATCH 508/634] [release-test][data][train] Preload a subset of modules for torch dataloader forkserver multiprocessing (#56343) For the training ingest release test baseline using torch dataloader multiprocessing, we preload all imported modules and submodules. This can be brittle and run into issues if any of the imported modules cannot be forked safely. To make this more robust, I only preload modules in an allowlist of a few heavy imports that take a long time. --------- Signed-off-by: Justin Yu --- .../benchmark/torch_dataloader_factory.py | 95 ++++++------------- 1 file changed, 31 insertions(+), 64 deletions(-) diff --git a/release/train_tests/benchmark/torch_dataloader_factory.py b/release/train_tests/benchmark/torch_dataloader_factory.py index a4fd4d9e868c..079a1ecc5d40 100644 --- a/release/train_tests/benchmark/torch_dataloader_factory.py +++ b/release/train_tests/benchmark/torch_dataloader_factory.py @@ -1,7 +1,6 @@ from typing import Dict, Iterator, Tuple import logging from abc import ABC, abstractmethod -import sys import multiprocessing import torch @@ -100,26 +99,20 @@ def get_iterable_datasets(self) -> Dict[str, IterableDataset]: def _create_multiprocessing_context(self): # Importing libs in torch dataloader worker subprocesses is very slow. - # Preload all imported modules to speed up subprocess forking. - imported_modules = list(sys.modules.keys()) + # Preload some modules to speed up subprocess forking. ctx = multiprocessing.get_context("forkserver") - ctx.set_forkserver_preload(imported_modules) + modules = ["torch", "torchvision", "pandas", "numpy", "boto3", "fsspec"] + ctx.set_forkserver_preload(modules) return ctx - def get_train_dataloader(self) -> Iterator[Tuple[torch.Tensor, torch.Tensor]]: - """Create a DataLoader for training data. - - Returns: - An iterator that yields (image, label) tensors for training - """ + def _create_dataloader(self, dataset_key: DatasetKey, batch_size: int): worker_rank = ray.train.get_context().get_world_rank() - logger.info(f"Worker {worker_rank}: Creating train dataloader") - dataloader_config = self.get_dataloader_config() - device = self._get_device() # Create dataset and dataloader - train_ds = self.get_iterable_datasets()[DatasetKey.TRAIN] + ds = self.get_iterable_datasets()[dataset_key] + + device = self._get_device() # Adjust worker settings for 0 workers case num_workers = max(0, self.num_torch_workers) @@ -134,7 +127,6 @@ def get_train_dataloader(self) -> Iterator[Tuple[torch.Tensor, torch.Tensor]]: timeout = ( dataloader_config.torch_dataloader_timeout_seconds if num_workers > 0 else 0 ) - batch_size = dataloader_config.train_batch_size logger.info( f"Worker {worker_rank}: Creating train DataLoader with " @@ -143,17 +135,22 @@ def get_train_dataloader(self) -> Iterator[Tuple[torch.Tensor, torch.Tensor]]: f"timeout={timeout}, batch_size={batch_size}" ) + multiprocessing_args = {} + if num_workers > 0: + multiprocessing_args = dict( + multiprocessing_context=self._create_multiprocessing_context(), + worker_init_fn=self.worker_init_fn, + persistent_workers=persistent_workers, + ) dataloader = torch.utils.data.DataLoader( - dataset=train_ds, + dataset=ds, batch_size=batch_size, num_workers=num_workers, pin_memory=pin_memory, - persistent_workers=persistent_workers, prefetch_factor=prefetch_factor, timeout=timeout, drop_last=True, - worker_init_fn=self.worker_init_fn if num_workers > 0 else None, - multiprocessing_context=self._create_multiprocessing_context(), + **multiprocessing_args, ) # Add a DistributedSampler to the dataloader if possible (map-style datasets) dataloader = ray.train.torch.prepare_data_loader( @@ -162,6 +159,19 @@ def get_train_dataloader(self) -> Iterator[Tuple[torch.Tensor, torch.Tensor]]: return self.create_batch_iterator(dataloader, device) + def get_train_dataloader(self) -> Iterator[Tuple[torch.Tensor, torch.Tensor]]: + """Create a DataLoader for training data. + + Returns: + An iterator that yields (image, label) tensors for training + """ + worker_rank = ray.train.get_context().get_world_rank() + logger.info(f"Worker {worker_rank}: Creating train dataloader") + + return self._create_dataloader( + DatasetKey.TRAIN, self.get_dataloader_config().train_batch_size + ) + def get_val_dataloader(self) -> Iterator[Tuple[torch.Tensor, torch.Tensor]]: """Create a DataLoader for validation data. @@ -171,49 +181,6 @@ def get_val_dataloader(self) -> Iterator[Tuple[torch.Tensor, torch.Tensor]]: worker_rank = ray.train.get_context().get_world_rank() logger.info(f"Worker {worker_rank}: Creating validation dataloader") - dataloader_config = self.get_dataloader_config() - device = self._get_device() - - # Create dataset and dataloader with row limits - val_ds = self.get_iterable_datasets()[DatasetKey.VALID] - - # Adjust worker settings for 0 workers case - num_workers = max(0, self.num_torch_workers) - persistent_workers = num_workers > 0 - pin_memory = ( - dataloader_config.torch_pin_memory and torch.cuda.is_available() - ) # Use config setting - - if dataloader_config.torch_prefetch_factor >= 0: - prefetch_factor = dataloader_config.torch_prefetch_factor - else: - prefetch_factor = None - - timeout = ( - dataloader_config.torch_dataloader_timeout_seconds if num_workers > 0 else 0 + return self._create_dataloader( + DatasetKey.VALID, self.get_dataloader_config().validation_batch_size ) - batch_size = dataloader_config.validation_batch_size - - logger.info( - f"Worker {worker_rank}: Creating validation DataLoader with " - f"num_workers={num_workers}, pin_memory={pin_memory}, " - f"persistent_workers={persistent_workers}, prefetch_factor={prefetch_factor}, " - f"timeout={timeout}, batch_size={batch_size}" - ) - - dataloader = torch.utils.data.DataLoader( - dataset=val_ds, - batch_size=batch_size, - num_workers=num_workers, - pin_memory=pin_memory, - persistent_workers=persistent_workers, - prefetch_factor=prefetch_factor, - timeout=timeout, - drop_last=False, - worker_init_fn=self.worker_init_fn if num_workers > 0 else None, - multiprocessing_context=self._create_multiprocessing_context(), - ) - dataloader = ray.train.torch.prepare_data_loader( - dataloader, move_to_device=False - ) - return self.create_batch_iterator(dataloader, device) From 010791e56b596b41ec514c5347538ae58e7e5a7f Mon Sep 17 00:00:00 2001 From: Mengjin Yan Date: Mon, 8 Sep 2025 13:45:12 -0700 Subject: [PATCH 509/634] [Core] [Doc] Add OSS Document for Task Events (#56203) Signed-off-by: Mengjin Yan Co-authored-by: Jiajun Yao Co-authored-by: Dhyey Shah --- doc/source/ray-core/tasks.rst | 2 + .../images/ray-event-export.png | Bin 0 -> 46675 bytes .../ray-observability/user-guides/index.md | 2 + .../user-guides/ray-event-export.rst | 134 ++++++++++++++++++ 4 files changed, 138 insertions(+) create mode 100644 doc/source/ray-observability/images/ray-event-export.png create mode 100644 doc/source/ray-observability/user-guides/ray-event-export.rst diff --git a/doc/source/ray-core/tasks.rst b/doc/source/ray-core/tasks.rst index 76a7173e2100..81c0438a1b11 100644 --- a/doc/source/ray-core/tasks.rst +++ b/doc/source/ray-core/tasks.rst @@ -287,6 +287,8 @@ You can change this behavior by setting in :func:`ray.remote() ` and :meth:`.options() `. See :ref:`Ray fault tolerance ` for more details. +.. _task-events: + Task Events ----------- diff --git a/doc/source/ray-observability/images/ray-event-export.png b/doc/source/ray-observability/images/ray-event-export.png new file mode 100644 index 0000000000000000000000000000000000000000..1e81ff1974398b0c684c264c61d7f47d10f897d1 GIT binary patch literal 46675 zcmeGERa6~o(*}xSK@uzw+}+(ZxVyW%yGsZJcXubayF+jY?j9^S!Aa2FGg;rt`tQ%p z*<-^TgQjQqYh6`u)l*M(n1Y-*!fWi;U|?Vfk`f|HU|Hv78I}zhm%kpbN<*vebY^G%Zy$Tr^~5xQ*>?=?zWnjZEo1Y#l%sfbn{81BbSz zE{232wl;Ro+#Y6=g&A@EY1G*Was?Pw15dRfWBd1 zqGx3Iw{757UeH-?c_&L#pl49~{7k&Rulzsf{^^I80o3__2J>g8zs~|w<$uk~@b9wm zzsAzrf&l{)0Fx9ERPg{m$$F)uqKg_bbawXT?2P_o@Qd2X8StY9#Z${z+s?!9i*T}N zCpuvN{d~jK9sF%r<;zc;+rd_f?DpZcR_E`?f&ySr|KI2Tg)7kW2phztn4;&0Bqcbg)J3Rveik#Ptk zi9UkCYH$bCB=dt^6Z{`R`5FZlY2(~Za)1Q;0qlRD4+E$l;%qOPks*Q3{{1j=Krq&g zpGN1tCj^7|_k%7ECe}2tDFpvt0qj|W?HOio=K1*iw^3jOyx^G2=>Bh*!2UK0*fSC^ zlGU{{*n7yojS?DMKpB{5KhpIBD)M){|34UPk5$ykP6)5X^IYxqkJ+K;jl%hfE6e4M z!{=GT76TlYU;_ zFADp2HgCj89){di_j~zPpKMTJ1phNpKY7qNH<4B^C+zfgq=pbrYCEC}hW^uEAZUGh zWMl9+@I}M0lfp$QZMEvKfRp2U|^%wii4vVcwIF&Jeg1x2)LA2K>W(tdBokxaI4wuHt@nSOCLXq3%7O9Vt9m?{M!Ce{ndr zFP~#E*pjDGB%wB%Lr&jVrJh!$-&&$C()`qIol>ej=A}Zc?L-~Rj?d;=LlJ|g?Xq7B z{vT2JCQQgH@JLLU0NdVJAy45kV{+3LRpJ?Pc74$6tJURzp2g*tn%sI?U@=`#{zeSx zL<~orTe*=tszk4z-G!CaK-?2Bon=JIrV>T-LVjrU3RWaeTif z%)5xEa&>;~Xj$!g!S*!vYF)X#vbnC^Yu9pbXqiJ$HD^yYFsd8Fc~5wdul zK75zMk~NpdFIuL@q0;$zetoqsZ_1j(vgFUUIypE(F{9%CYRtT&U~)N z1**bvl`oaob!x-ndX!hAt;my@oEm&X-A|~$dZtZba3j`)nbDoVd9i&g*KV&>xt;9| zk1s5%<-UN&ppy^JR;?PHxi6O&-5F96NtU5TR}jv2G!sg7<+5kR=8_-&ip3ev+g0o~ zQ<7x<*TAv6qEowyRl8K>{jNqAvN`;(+;;ZdC`Q9CQ&VBiKifW1+n79UD8C~yqkYei z!S0oN+0owOsgR88se&z1^s)H|lY+Y4(ShJD z^Nacd8q=Wo&9Tt5aev{SHd1_0b!ey|z1VG{B4nw~eEK^MJqTCa_kUHnX8}&bw?)G7 z;q@pDUACn$T7jZIsjzszM4Ydusq=R_39LGk#ZO|SGMS0D`W^CtgU`$&McLK*U6g%y zvlIk7%g4tU+`ag_`9-5g@HSKwgO094gLPPxlF7tmuBY#nuzo#%it_OEE$Vx4Q|>h? za)oYMkVsg_qEyl>qeXLUH2ALH=w?A-RJ=;Py z!WrX`f&5=XMF64pW5jN~k-&5^YjX0z_i&BM>*0IjS>}GHzY4GGxDx+{{yN--$FFTJ zN!%uKlxN4-s5xbEq?y16_XDEiPJg@7S(I#|%OA58qhO8FS#qbxFA^C<_0MMn_KUVN z;k;`cv&pVBCA1+?@8}Kf9Zs*zViJ4IgGMdgxrXgO5Ec$}!yiQ=g&)14f+K`mH) zAye~R?3lcFm0m!-;IwvJ897m!IW&ejJUv!_M-&xLx?eu&$h_PA*X0u1 zyXUr@IwBgyuC3z)QfWg6T%P|d-Wb%QoAC-|-1aTnyopHjJUuiN{`+ecs&j6(@}g+z z)K3y!Uk2ujZ+Etr6G%o=RvxR>r97byg{CMENvoH9W{#}TukFuWYl_wjmLwF?xpc zElfCnj`t|QX*|hIPm;`wf34u;EgdSIw|(pK>(Nq+*E&CdLciml4?5~|fUdUU)yiI} zX|W!)*+b0`y;h@3)wW`t&R#d(>ng~<=CGW^pfTGfEtS#`?JdbY!9ymaqLS33K4s9_ za>;c5G>TlKdh6!`X8Zw0{Fws0fzY=R-&^cvuohp1Cl>PeQB0w-_n5|EG3BpwR2EIM zpeGc5NstVzbBG9)5zuLHgBkel5b1}PRbo7=MOwS*D=wq<6oJeXPrbj1bXZ`*av^Qv z!v_b|`PTW!C=WP<(BLx93K z?;ne9X!1fN{O6Jzg!`#QYCvF-xzUw;yG5MV$KM#7UCkk9>BjDhntZD<72 zdoq&t`rTi$UV-aF8C`r7fT7>GbH#=EyNeLEz-eT~(Lpzn(Ym0naiHA1lKIx%(FKkn z0{6EySYq&?2&lJ&^e}&A3i#F-@MWXI4F0b({80j!A2Rs1)8E^H!7u_tTup_&{ri*v zPmM4XG{OJBZ+0sR$zan5me+Gv@>mCBHca^8V6!+T>`PD{6dbBD)5GOljVMewDnnOJ z*3+2;rym`6LbfpNH@Q7j*HW5}N%B2wi+bSz}k}p+Bjt6U`XfHt$D(GFetIOf?6k*LvbFL4rC>tN^=rp-TZ0^HR?W5pE9}e22_xHBCP`h_w11kQ)6>rGJfc?uL2}*){;_%z_ilbXKR12>u>ycOFLCK0b`Gcq`{8O;3S~0OC&SsajldB zva7N?(r!VjqxSrwLoG8hO zOg>MQM(q}p<;^ArRMa;qlm#?bmjm6S5p)S?B%;yzJa!#%tgU&qne4)5)QX>p@h+Dn z=fn34!{@DO%*g1(6Mdrr=zYS)W;f2q2Nf-LFiioPWMzG0|ib z+vG7;Z?3@>!{dA9mYEU!KlagJ5UQfjiP^X(;B-a)zLXvgb-}&G2V(-8I05IT$l~yl z)p{9PZVW>f@A>&Dnj{epC2IKtnRpQPYw?QP==QdDkn`W~IXhL!6A)YJa!lgpb-gA$nyYB? zd%NzoT{vQS@|A2fkPb_-LNtFURFaPSw(xYQD)#(r`tw^?i<8`ry9H*kNXY*I20aac zjm}8!u3m6ihUxHsoMLy)*(+aAVp^ra!0~EzedS?1u)S<` zGyINfwyZCQzk_0A>@??X<)=!EEy{aW&HE@Td+!wTv(xV^h_v@AMsa6H$+uun0B9C5d4<&#O8i!5nOQ{k%ho`xh|IPRW0dE5#@ zhVm%up+#@xMT64_?)~k!bh}LD8f7`RR7GRZhuvXqfvj>MAo+xV5qt5H6$Oyo*H}L zc}H-x)bLqLjZg2RNFtTC-0QL6hk<&EdDnP#@B0~7QX2w09o5sxX3AvF zux%7g7U!)3h5YTy&yrYhzGsXRCKCrB0t&M1jOd#m=xyp8!eYjN2j; z+yD(18CMyljxU$|HC<$pUO)GRf7`{l52xpqJ=Zg!8d?Qqyab#kkasb zbjdxNn9C~$_PX6d5zF?Xm_kcZ7${YaGz9ZTr+wxcK6UT@bDas zAlB4v;Q#@W%Xi?rBZC;=P|qI(fY8HYt)l4q+XkK%yFKOhX6Y9StHzJZZ0H3`&{F_N@ejDC!3l!({2-@& zuLub((K8wj^VQ;L^_ox+juy1`m{U|f{0*a5K}&jj#wCXP5sEa$ElZ5f;ws>ZGy+m~ z$MhISjb0A_EJIR>QRv^X*aQOj!lX zM0bxmx-RJauM3(7#FuWAkmAlw;<+ZRqr2NpG0@RzVmutx1y%#jGi622lIS=e|HURX zD1aF(cP)etn7$IYV_tt%~Bec$149^5IK^6$7l3IRPR9BgSJ z|9k_HR=@vQ@=AT`OXHc#j~wljf8<|=3IOag z2(q!*yA{6vH7o8!nU*n#bSNs1@43?dlS$$qvt45bh+XBcGb0;xK|)Dpo+BP3lvX@m zkhfJ3tSU7Xi+_4>cA9}c?eBu}Z#NtP=Uqsew{JCZNt<=y$MB)N2#BF}+YKjB0L?rC ztajtUQo=$TZ6RQ_snnl^=IWFg;*n8d{}q=#WLV?;s%Ibs*$|N}&_u3*hluwz0Z;}f zO~wSJ;_=)_`!rrlIbx)SpStYDJSaZCa0eUD`(GL7H)FryP+CENJbPr~4-Rq25D`{_ zge6eKqR~PK>CMHJAq+|b*shk>AMEDldb1JF9FeT%&D=W|7a2gCTbs*m;_?jw9;g9O z8{I^}hHPK5g^dg=3IQMlxks)z)6g-qa}~1c>F-@>xsn+~c-9u;?_vn@mJ(vEt?rtl z=Gyd3oyt_vYY!qdFH5EAUH{DM!yXEt1Fd(6e9cS{09H}NoMb&Eb=Nm%`;Bs?#))d4 zGAU!BT0=-rPoFM*@O$GJm2@=kJ0c+wt^Ky8{2`RNiEf|gNRR=Z%;7_@Qf1W9}ES z6U)`$BmhdKNVtJP2>hc;0uDgnb3)t;{>t^kha%+Rv}QtC{kCX3kfkw~|6Jr;A7?zf zFxOLxm*&lm7IpExu?hq_QmeJ`lF252M7$L41CWK0JU`{i(6g(Zkh=!6We9H1T`=F5 z=cw1{bfRoWYVoCrwD&cQbGFSiU*zf8djZ7{*Gjqkg*cj%Py4wGk-d80{dS9;S zeCtc;30;Jl-0r(@#ZuinaFC08fAPd)tJ@uad%7k>@ZK+XDKkrKg;KREyz2eRU{{uQ z>&a`cyITY@+3a|hFW001oJ&c&xSqK>znk--TP$0iJUwomqEaH?aKKa|QR<{p?D!_W zCig|Ry+RE<=~Ddml=#95fEHs>STN#1IQQWe; zy-qqwZQUP^D-4e!dsHid*VIZE%jS<#2l0Y?)Cp%ZX#sLb5zFt)j?H0BMYp;u$iVqE zS8POw)pQpOk;?;R+ZLt>Jd6rMnBDhth19@@sJATWmD*=KCc8T!NON^|qUqz%jhIP? z%w~#vD{K{F+6$Gr-tKmXZFK;;eYEf;4&Q4-`G()~KvLFvY1V4r)5T*`PPcUj=5A&I zp}fSq{QgLA@8Fs5`gE1U)WCh~IOOhdc<|AEG_4q{)AL3tDTCW9_RMQJmCFzF#%4A) zo6oOs<+8&)16!S@6d$_KPd56(El_H}WBbTWDuEx>)Ij+8a3{nHJU`Bt$9mxLDbbCa z)h$<>qe=<7YcsAuBySm2yrk(`{mpu}HH~?Zrj#!a>(0;HmudbDBm(^BcmBQGzqf@T zWbnsH93cZmI@r{Y0^ur_7E>))b4}jpo5EST-3~LuXBX}}JQhUu4wJ|ZPifSYLwK&SW#rh#EJWw_fk@3_qoKC! zYkzCiXa+6C=O`?ZvG20QP+mY<|v08`F{5DwnHid}=T{)GJAM11w;jA?BM8$H}I%Q_}?Gu$;$4^{k ze-S>3VK~oWiz9_#irT7zJXAB~VA|!tkqYGkrnN+@iJC1o5O0?X$g~%M)!( zVOtxmnzq>QTQH(>>Se;ITYaA4Z?L(eu^UfHa1z`QbN=iodBR}XCsSLIY665RaHJB6 zL0$kHpMj*8`!fthnT1{C!PTm&qcLwK4h-9QXDY)6Rp|_r&TTf@`4DnbLhoN>7+sy6ZJmn2{90oXfdr9%TT&nZ3>@81ke#@ zUzkuSVPh^ib*S$1Q&$LY4~q74-rnARp70Ghgty_Me?XbYV=xktPjA*SHyGoBqZrBL z!|VR>Q9Wo7@or`9-{RYH+Q@*E-DO^w0o$ z=@n@L(#qW0&}OZUJ>nM!Z6$h3C!Rnf6l>cQ?#kX;Tu)yixm%k}5tIzymkC?f&LIEL z$Hzlk1wDW9dsH4fUj0*vG@I7TX`+~{UC+B@JJYFMs##3ToqJb9oZm*7&;>|8BH~S0 zyn>RE65wr-U2)0tlCw=paZ_ksNTifxZVWH<5SAr6v$LKf6`5>*0rpXTJ@p;GVUsVx zbleAZQK5^g8YW+IyL8Ua%(@S zuR>%@Sd+z(;A6I;K7|NEtlJ~*D3y(6>F-fXeAMz=2`L) z3ht2IyMwHvf62rD{%1fNt=?nMvu^axA0;9uG;js-T1)$Un;Bqa>8$P$t*1t*-WbA_ z{wQyBMtT)XX=l4UZb{Pz(x}tj4|*@Aqp8n0NqI;TuuZ?@K^AW&Fd`bi4XjP|_2jPc zZr8HYRk{L#-I80f>Lx$#HXs5(pihA(t5cX1lS?WtI!b^*Q6gGagJ7;JvU&bOcCaLf z?y5w;mU7;5CjK5VN1~{;@)SosZuQ9YNH#!+X1msyrPKFn#XQ7+W4Bgn zwD(fg8Vhr2dp=n*ia4HGVZ=Xy;-*jh5`(Vv{>N6Arfl6c)o`|pU+Pk~b;e~RMZI(p z-XO(qSLy>6GAv(n5*9qVF5u0a%l^j|{hWMNrN&#VHKPsR6N_?T6roZ&-?K?>g(>^t zWXT&=G!#>CK(g8`yBhv%<1~Hhd3_cBD7FGO-rH+qj)~RM<9(95wvz$U!P?s96k_0O zk7CzFzas_n;+4z<@9xX+96rY&AiY=^tzJE=;49SLB)whS!O40`ozbpz%h%%+h7$5_&sfGJCdl@3 zxZLfcl&|FLS_{SgIsD(hQj)-v>y?F{{|UK!3PHZmyZ6{I;8#S1gWH*VcC4nouNxtZ zlZcm9RyfmKO2TNHP*g7N??th$>UyL$ga#fft2lASwx^Rn>5=+~kfYFRbn1HA4j~;D zIjKpfigm{&sN&t%F2}c3t3gsEF=$N--t4XZ{3zd0vKS?q4dHON3^ru zVhwlFct*4qms&nxk7h}~CvbV&xu(f{%xv;NU{IQk1-xF9f@b^?lnWFylt2m*E8qC* zu=rC^a-M~{N%2cHQKdVwdg)T7J4w@hIebOeaPw}fQ)mGA=r7CJ+d)xd$r;K9Fs1l& zltcNx(SdoW(dk(XERSYYvY&r))&@=x{F1uD-8k=ssSAalJpep2F9wbdngZO1143Mx zS&KG{MU`%YID2l}lv*i7jLjnW$}1jsn%KWjWjj3Q>LXWOs??eXoZl*yrpex$;-(0U z_W@e(#3E(ihIu|;?6il=Cnuo|^wADhdF+uTW%a9CbCnO2#9)q?6B}GqjFcGEDv`D} ziT0DwMvO~5Hu^j59EskapJi!*i5r!%^=Xcdy-w81G10e{zE|fI7K$(09)MjC!B;X- z6@RO#s)W7~#~5dR5|!+AU7y+UO%;!9m(+3QU72kqBAj8|+V-?je=dAl-LK1u2y<=T ztqWB01^Qp*lON71GtHR*_rDt(mHxN;KSc^A?`AP0rcG!d2u1p8*w zrHhqX1b{hSFOtNCd86DT#|F?LNMKRXY^-P7Vk;&QYtHY!XCiKd`_S&Wg}$8PGfP~T zZ}i;YIA6D0SBCgBsg2=4Dl*lGlJbN_DsJ{9?--Fr>}&atxNyy){nLAy|`-!)3koXf+?ZuH%;(mLgh zvD0Z4dH+klEmE^YZYP_$rd#E3q+#C01y>km1I?Y$bqmQcKFAbOmugO4V@ocq)m6pH%ySNd;|rH z`m{9wnB4QN#Fc1r@4d>tu>`!6Bnk~HTfOk7uPK+?3q%7hua^qYgyD*hYeyJN5+mt|{lc4-y zCs@ipSCz%%*`Rf*2tR;FQGM)W9ijyTVF4lzHTZ1VfFQu6Zv%c`RZSdou%s8KYc}vo zWdqAtjYqhhPpLt93or;XVL}W4fN?X-16WuM{(x_YKVYfjRynFmi^hO}qPXfQxSSDO zlX1ZGia+Aq$p7gw&COO*zV#9<|Bboa6u%yp(xr7qLM9MIeU8DC*;(STIU`20At|0H zPTL8IQ?9K0zM*@K1e&4@u&THAcD=%r-VOfy=qSd*q;L$wifulsjkuTg^l#>sfw(>o z@FJ+3qu2brzJNGD@&tUj9|xpqjQd)g>92~*f384}uj1sqzJ9(L+drJz$WmFaeM=^3 zhH-FgyY~?Y`C$>Gz1P<*PI~a?tPeQLcE4FD6jg9qz!8sCN^e+=L{C>2pjK*kEuBQ# zNPn|f8v^|JQk`$cFgLNvbzi@~+F@S$lcgm57aXB7D(+7Mm z@7Y24&zJL7%IBEN7?ywsk=QK(m%&jdi?;FhfFXAJ)unIBCsntW$|A0@^{+<(GbQEV zX8`f6ESY{XtGnhtdW@Le-EKXX^^oOP+wBO6yjHCRVlj1xK%6Jsos+7SQk#sXW!ABf z)uuBt04C_Tj{~lGgX{Y1cn|v#H2MTF>ji78^OxR(XWdf9YTh(TNhPf=i>7ennBH;) zHFq1KTeR}hHj|>!dFlPqQ`f49gQWmIR-d}bt|}&$bq4@meP*e1cf`P{^GdZ;pQZzW zLyTAeW+!$)hU75W6m>Y|wt&4Zx9@j#^ zb%jX}&1QAfO-g{q`XsHxBgfX`nFa30e_>R7D zvCntcz6)SCB?LqI{yWz`G);J~Z9<*=dMzENlX&ud2 zcmF98C}uh^s1}`ckdcCsRH+l_FYhGMo#RGQ{AH=84sXn0R*WLAQLzmPlAL- ztEGf>Y7FC6{M6p$n0cv{mDlMzYn3#zH?ypq*e+n`cOa98e)YD`t3T0GrBaS0kYD3m z&i3rVOU==`y5XB+?lH*GMm_jMNc%iI39R!%8-ah)lbXw-58QtAo`3(=Q#OM_xf9<2az zsZ6!#k}!hk0-!vqk_T6CT`81L_)>~*K(sLfSevTlH1&LmFv+h*kCqz4(&GVb@VpT6=Q_Z`$yo*w^BDBI6y)r@ zXF{L9GI8JFaHYLQLwXK@kVg11rSWRdd2^zYbPItGI_5KVBQ{hKb$yGPkX}} z+(=q;Zk0S6F3xP>)|18lcULPSJT4+410#6pmP9jyF~?u)2^^XzgLxz9h;QM>42`oq zQA&6-t%wt-_Ah>%HX)S5N1j|iwpRj9nA&$Lli|!H3^&hwT`RgAX3HYjV>vaZhJo_a z@z>xWST_TpQ)vIV^?$w&tJ32wzC0wCBKD_wOg6ejE5THq%6bP`DVp_qRH90)n5dZJ zSPsZZt1x8)S+AlZoI6TkV8^rPc79~+^t#ZYJBj7;roaFB+{s+Ski#Qui=$Sg;kaG8 zILl--pfo6u^4-<)yd{mhKwzI}sK*aTh@4%# z4j4l6oHC5TOJo(LzJEM~6y5ijy|dpeyIP6SUaVGV7n_%4R*KAQ5&+N{Yws{m49F7Y zoID!|f3=8o)>6H2^y<~32*|DhRJk8@A+7+x&hAw=ZT%gc!p^7izWcs|rW^-)cAM6J z9-O63B2qCd4hQ{Sa!XqYB=0oBDP2&fMGcNWy9XLbpR>IurH`c&(^nN=YC($*{A8N?-T~yTc%`6#R6x}aQXL zjbcJkzs9X}e>=Ifeh!bT%siQ1(+&(D{C1@&%0fOCszhBJfV7FIH%#H}(|ZmvekasE9VO&IRLKHJ0C{$yf`F3XB3Jiv#Yb(LqXYp~UjjWX>$ zvTaMWf4RK;sw|GMjhw^`O0TaUIRdFt@}bL57S}y?Kz2x8J*I!*7LzqS9Y|i`WSs5l z+gw|$6AnkzKZ@3yxrgy$A|9-LRCg`gV_yri_^GQn{gcxQ}X3&ugNA^=pnK~Ve~g|-v(-I z8xBxbPUty|_ar+Ta={CmU5pXjkv#0#C%+IoE^lXb-A%rzq)pr@gCO$Z>q!QXt1zEu2-ifU~Zjf zpPgB=Fy><=%TZvob0s8Se?1Bbu${ym8*?voY>dbKk`VsAdd*4|`c_+yR=msjvc?0b zTVP1?W~lRPmB*BrdB+cZMdEkPmMorqhk!Wpk)2qWVnR_f)(X0*_fu2j&Oo1S zk5}{!WBE%p)GyNC;aNo*;KFvcmtzH1>vvA^V4r{PHz*!UU6v z8lpHJd*W%HuL=@Xp^Bdu5WhaHnBe=y?nm*76RBzV(=3IG_T{Kf$71^KaF+fjm(1r$ zh2*cqo}7qUbqS%vm?EUna~dTOCfB z7aHqo?Iq4g9$XHxr0}Bf4ie|coWsu0S_`Ga35MSBCd)PK1r9@>*>?AClFIH;we=!} zC3f?y78jyO%QeluX6o`0O0y%6OcaED&51R>CoZMh!}cVc$4HA&MK2SEfM-8ZF{I)9Qv3ym2;>SFoI=5GhG;!lKw{>2*d`Rq$X;@5CiC1* z7We|702`4MxmcL`eaIaWWOaof3rZgqzR8cA?&naegqH@VndHzQpTfyRSst5nD$9F6 z3LpSgK*eOp<`XQPF;$3gcsI>bH;_u*vqclXAS3k3POrUC?PB&Q(X?K)-`kFMg{d;q zjbA7zbmenzalb!@^#drQY~Chc9KS%&HM5R%Ro~!&Y*V!k%rn zo!av8h^JAxyEy&U8i1&- zlrn50VJe7D6^4r8sWc3~q7!ex8RKy@r_^F=f&%9d@rN%gc6 zTPYGJwH0-gdC@ck3G{0`e%~`Tz)qPZ2OeO2pqBfowSyL~)#YoQ z&2v`D+-4FY5@tu1G+nEk4}@ZkU7_aM%#tTNsefuDbe16*05N7!BemHo1wRA)lNQWua z&J2>7HZ}u$Xqm=7bBDvHP!pWE3N)1c3Whr5PcosjI(3oHhWe}VrXohQWX;u%!5^0s z#1ub4U?n-8919gdehN=fdSftb!XVj-0T5yXQPLIZmeF>TT7&8=lp0(-(N<~=2T)F9 z@822w?35Lao)2r+r1o;XxUo*^-vt!RlS*r5Kio*?AZG$)9i?LpNoNh*g;|JEcqKYr z*VJ<&JkgW6Z(|imM^=avboa-g=3chL1J%ks^=wn9@oe9_B?Zf*-mD@@S-p3s=U8=3 z@G40Ni3Pyn>11Jy!`;e#m&@BXjRuh(FQV5hD~02+X=--wMeJZEX1}nVHP|G;obq{# zd-mVptRFav?aeH_Cto!-qbFaadvFce+gdff@|$m(OXJjZTIZr!SsF5?n&gg}r+6L@ zhYrd6y{i6D-$vMpN#0Dr1H57uhK54uq;SP$YQ#hmd2qCLulobRmE*I|9VW)+bn~AQ z8cyaSZtdJjMs%k077ZG0>SgFN#>bS0qH$ul@9@m#!ndk|VkEK;0ide1MJ_BQiF3EF z<7oHt6U)5QL7j`U0CBd>1(ZIgvry3>3q1oWk?EHr8TxnC8D{X(Tq7yE?TT+2(@i~| z`}B*{_|R56oknz$!t!ADp^NIcgI^W}^b_^ZZ!(joPn6MYfcz0*l>`ZZJL;-B!H zZgHL3!$hdwPtSJe>i*_9=jsR1&?o@=2`@A;Tn6(NNei@hupdtz zUA^nW`Hc?Mhyg5$r2LNGKICiO&t<8#a13a;j8nd@0-!G?8&z zR+LJ%o-N~)=y8T(N~826Z^xAg1t1J4`ZpNgGsVd(J&Q)8eEKejUo_T7Q(NTJfw7o3 zAbN~_V=R16RNX?R{P8;zJ3To*50-rCi(207dj{y*psETT=8z`)#O97d^lMBOVwHrR!92fYoaqSs znsONjDk91+R6AUr4aOwS*$&WUbtVpBr7N=Rm}?Fq5qP9fSkBA#;vqX^-xIvK+K-`j zZwZo%$~ZlF`NWB;0TEv%3V%vhL0^I5yyFg!m-FGbY!#sbgb(n?#A_`gA|NYM(xBU` z+Kl;ilC!y9;-g5|{83PMx%L;njSfG_Fb}kW?(wzraEzv(Uw(OmYBH{pU#iAqIq!Dk zD7;@8mwj>)%J-|3(_{_OJ?h%-X}s$Fg|)UB_3cF4VV-QWn}}~^YHa^yrtsGo!VCqp zI?nx#2BdPDM~BOV%FQ{IeXhxft7dQ~!emVByo#+VL|d~{_MZK7u!K}WQpL1}b*!Js05WqZbpnvQUc`d-1D^Mb*Avx93(36 zHrl39jxcWm2y~>q7osYdjd>wy{&*bWU`BO+?vXGUIT>kKFeQ@JYzTMy{^ZN=s=XV? zl))A-z1A*X0i!|YvJ3UkT^|8IrMx8yGl&l5=i_0*sKnLvh;S0R9C??JKo0~Qr*jTj z%B|T#a>c5~pI!U{3Yp#rRE$|83W3BGD8mBd9ga={O{dqaY}!&QiilZ1j~qMar>ePM z-&B{bXh;amhaH4e%5M6>Z=k^8zTXv;Lo^+(wY0|OHgx-dWl!rEc~CEl!Z}_*KSDjW zf!HVsfyoLOZq>H}EupIGa`1yIaKp@-<;fF8VVT*J9-3a(1U!T0s8DuSlo54O!=a=j z_CwSOq<_Ze2mlV+YjnaPwsAcG8P7uFM|8*3lrEZoR_s-cut1L#+~e3zJk;K?#=6FF zt)J1KO+pmZm*_gZ%K|zaFMy(?J9>3V#1T(#-?V$g5o0bj{p72uW%A#>P&4=KJ?h2I z`SYOAMS+R^zHI?EleaF1f-^ZU{U&4jeB_GKs9&Nf7;L9e7h==%U|e%xI9X?(o8jr1`1Z9iTU zzOU39r_L3>Z3v?`U_-{A(enbE#J(M;OtR+No40vC`0v(v5|GJJUv+;~M%3xB&&Ct3 zu88X4#gC=!q%VEOu0J#UGha zoB+Atf{CjknlR39sWg-AIpSUR@U>moh66=uzqmbTi+<6*Oz?ww@j zgoR+2`-T~#&m0pf&VzOA@h5OgB?B*Ksk^veq3t$5PyvHdqfZl--j^u?u_&I2$sj1B zW(f^uq3q5NB5JIH-8eRGn=FYjc>yudM5*B>S*4{aP$v9_1P(CpElj$rTpk8hqq}c& zNk>i|slxgI8S{e5K5ycUvNO`{TM4>(^GCeD1_Mel{#`AX1Xw13sj{n5&g<#@LbxyM zZ2-&?18fI@I>22=kZi#oh=F$giGXZCZdQhNK=cwyKs`mY!YuZ!N~stgHlOcX(Qw>> zfBzQJ%Fwo1ccd+@dG(Dq;ke9td9r zkYOKpt-WjcjgmyIsq6Lrc@~l&bco82)lM3!qoZpEfF%zH=cOCIjt9mUNmxqv4QU7x z7Q)HwLF3n-nN|*Y#r}~!a*_`YZxiyw`=d+PyE!GUtwIQ}mGNmSgs-kzqS zjF{SVoPfo(Jg16ylG!Xa=syo$=+lRJ&$@q#c3z`-SoGJy_J^aU07^lSp9MRMRX)5uPO1pv;2a5h$_BOhudsZ4W_Ua}B<8-F_JU&*<_EXQ2o_@nK}Mo<+K!-z~uC74i; zMCVTr9M~Ks6hNul#}x5RX`t@lqL>sBAl;>=_GsAYqgQhEXhqy`{?mC7Ai%;g|gaYhZJYo&SQaa=ofI!4n zr6%<$SSGVMGhZyi>`mVwreF^??8&y^@a3_y=Bh^D2S#XM+A;)qSa&v%9Qa-R zY_lXwHCtH!$wzTJfh0c>%E2Y?8{WD!}_fJ3M=WETwrCE@}3 zMHT=U`i{=a1qYKY7%)A;-$-KS0&cL5`4^!6m`Yd_V32}eLLzJfdKPYIN+OhXk2su| z8&5ZKz68M;|5=d*Scy~#X|=R>o{4M-f5!Nn4lMl(HXeP08w5~iUo1)T08}L5H+VeB z0cAXajYwEKK&je!8rZK`L0pg(U=~FAZ6^ajMPyUM4gqw?9#5CAZtOOLK&3rn&G@_L zv9Y>zGw7*Cz`_VZp*!fC9=f_Oe6HAD4+eQY0=a;0kt(z(ETO~dbUOuLtI-L1zD6%9 zjacmIY5J5A&GhkuJ%skp*nUf%`tUpi5P}&X@b&`*59pp1IGfFu@XwLB5|Bf36c6F^ z6}~N)1VDM+54hpF<01|B8maK^+CPi(lLXj5USJbk-^F9@n9CC=N2b;$Y7LcqG=KNS z3@D=p5eKze7P4jvC*Sd+6`%tWB2D3t0HYA3%F@t&-qc)#^!f?N4#9dtz+%984w}t= z>7K+_2J+-1rq9>iN3*@1JO^{rkp8ZfT>97F=cSLPmIBV!TSb9%#m4~K363|ukl?e( zu|H2xDzG#cYsJid3vZ7-U<;q=@H6LNGkIJi+Zr8I-qczQRcu`WP0WSvRM+p1$g?C@ z6&x@bY4z2BBsbt2#d&sl#n<~ZS0roPKXUm~8rYTrj=NCKhahE*j1vd#a^59?c=lP& zH&|(@=BH~-lXv9B2LVRZpMy7ShDjQyBo%Id7Unzwcm`6BRh49=+Wve1>feF0EGqOpQJr4Kxs*|$GO zWbX`p`>BK7&=$u2aS;dsay#(i9k+gmYz|lWryuvG+%-d4{jl$L+Vp1GT~24XtTs!a zL2TT*-Qm8C1qYX_BT=nR7yQPvl@#u4T#*Y@(!#w6v+1zD2a7{7&xl{rP4ySFuB+PvyU{U(rL z==4=L0~g=u@{|Qwpal+(9|$iNnT!XTKIr{^ON5z(0C_X7+4`7P%njHd6?B0*_tOda za*eZ1cE;zzi`RY;nQVKhkiAO8P35v@BB;hJ zu&qgHfZAc559Yz7%4)f^_(;6vJo%T;LWz{)XgnXrxQiYVr1k^KH&kG^u!pPId4s2^ z{y#+o#{=vP_q)%Vv8L_EJimfM3nw#WT_tW|VlpJd5{jl=71Rh4r($CWHf9$Ym|;Q| zpZCrYFG7DTHHfL7L3?-DueXfuV;@Ez+J~k*0#$``xuV{qnRF>lBuCO_Odgr zn+``a{y_u5Z5p$kJsN;F3yZt!m}r_cys|De7-R?-5hzITF&A;@zMk>g*WDfCK-rwa z7Re$*j(Q$;yxAE$PL~#SC&7E4nMNtM$T{30P*M*~jCg0Y$q!8w$Vvf;pW}UpYTd5f z=*=DE*k8&BiyAH7+Dd&WPd`@3{5!vuefacL0+4P$-C(W*?|-QQgHUgY5!H0BAvaF) z;12t3$d(v^r7&~3DiDo1E9U@`C3$asd1mV=n1_NTI=%wT)2$INO;wN3=T~gF8B+~R zfd>F+eUdgI8PE6su=UnaS#4j`Fd!%hNOyOq(%s!10uo9~cPfo^Jks6WDWDS4B^}Zr zDIi_%e(?Uj@BQN)gK@6|kLT>O&pv0bHP@VT<$26SK!x#lrmY7tM>JG8$CABmf3K&I zmmKXRXw{^{ciU9vJ$?Q$Aa)Q1S6EU!4|`ibOodNHLgONnJg`fo9Y^_}o$x`wyg<<> zi|m1EI?|3nZ%UoTh5pp3y}sLIPa#SF%yKK}O}+Qt)6DeC)D15rs$&(?o^!cVMl5NWbA-~fc{qn_sM;cw^jVTaNG9UbENP6Lv9PeS9KG@+O4@?bQ2h93#jirEb#=4Zg zeCa-|6Q|;5_R>#HJla_22*}T%c;5h+#ar$Tr5)X5~q`@2gWAs-MuEd#Yryi`N~}^FMgJrK$ca~ z-ih}c=B0X_DfS$nKj}HqAJuNui~xRtn6*d?H$aLLDYXbU?YKv@G1T7Zd~de6;!$WR!{cd-d=MfDJW6TsF6Lj;?JV|FZES8zIj;50ly?0`x_p37J}r4y zo~FA{JJ2l)l7IkI^gl@TsCR+)C?z`TxB3C%rH5mE*q69Rcg~U4bK3>?d4;M=v=wm1 zfjGu*_I{x6#H_Cp!uav9LS47b%Ci+USU% z0~j8vKF|rQ+?Zj|oVT{sG~GNuSZt5(HhbOR)|Eqv%fbFnnN$8Tp!*!>-BN#Qx+^;L zW7`!0l(P%+_J>^2!$!Qmg1#OP(Y?d^16LKkw2b=wf7nsLK7(HU6wcifz1C8RV7j2( z!G!}J@MM_4yRPl@&Z6WGP(Mx_PCjTTR2uYG|Mw5L;c8DH_;Z{!Qr~29YsA5$DUSt^ z15;~BJVwx|nErSeJNDz5<>9hCG+&h0CRUk~%KdnJ$2g&Cqx}$5UT7Ww1BWZN^6a9P zYtL;uul)C8#-%L@b8MpR@{;566fm%nRP5>A%%le&K|m$^Sm3RO111y(UzNJip^E|r zo;kIP^6<;X*m5spnZ)C00-&R54#$W4N0RNqLVwcsbk9_q4jM`X1Vi)W;9sIH|D^d{ z)`;mVafzl&UsFH1zIvQDaip#UyBwYuOHUtFLe=tc59?}jtOxq#&U)V;ZSIY%7d`O(;!I(=nO;n_{8rq}HqcDDn{*e^kH#pd8{IN=kg=`c7$pKjrZ( zAyByZd)f!q{|@PTk7&5*ETx;FTT~tLmY&I#W1Wj@nNDhmayWtY03l{%NpHxZvA8 zspFTcc=kA^2g+w2R~#+12`1G-(7M#*zcmTXZ)i`tsI|Sx%JQ!TzCqV5I|gjGc^abN zfA(lV^FvO~S5H(UuY90Sp(;`e+?6gRz_NUxp#Nuf0Y->SvGVmU5h>0<536qEaXx}L z!F=SP5D))vQt8o~yX<5mIy1`q^`1Q<72U?R3b zfrW$10RWYoWt{Z?EM(BH;i;!`nZJ{!xf0p9K-ZQiJv3i@9y|U&edg!bu6OnZv|`Ke zgt}70p!0+N4gSwAeGqJ(*W+A*JA48s(u`)O_pFZ8qN3X?l8a=|Zdx?~WW>L(>gfC$ z@tE9*2V%LNZ8E5KkDliE{#h(Ba{(oR(wTD9o@gSr`m~EWAt;iwos8>%WV_z+z!oem#gbJ?L%?;3C@7n9c&^{ZpAA z@_D5+zvl~_JMHqDeGyXI9cZM&(2tuo#jiJjQJCd)&}zNoF9P`tRx?>1Jd?W^GB0Sp zxs$Uk6hXkmEw$ZK*XegB56!M+3;M?08{&J^~OTz@OTG}_R+w%b$pv&_znaJuZt=2jC-c!In*L0Ad&Q~c8% z=HP!yydA9Af*Gu8>9=-0j92watZOW!0JMinu+NVcO;7G+@uh)?F0pU+XU5f^dlG08 zzR#rnlD)dX7zA4dfOHlQnbDc=$)QF+g=PhUhflozXKI!sB}yyIZILt^n^nH5dsCAd zEE$RM@mnye4B0YKNTn=slZR8Jg`7AhoT{+@)6|z%)L{R!7&{{S)tLaC8-R;j7NZWY z8RhPBsjysea7Vg`Z@T_Wnc~$%x7y~e4x>DvApVUUOSM2Q20Q-WKliU&o%cfKjriS5 z#R%s3yw}3`n`{QXmiW*vhN*KnCT7}8Ow|TkapX+f&n(ueJAT$!v2cHyDRWF3%KYGF zceWt=0*jV87zryRQ`l2h#DBoh})=N5mvL<9T+QkNo z`S$}SE6oSC3?V5T4B<@bp)1u}2VM%t}f_5Q$A{QKFcC^3>?lK6q7b zm<~jRr2=e30H0OOs71hLyICxO1|S=)4uxU_HuHgTc1rG8W{)f)@PP?Hk}VFypO1Hx zWKPE`UmNB|*5iLy%9(zOn~42O_yqVt{Hzc8!a1MXuvhu1KVoWcES^uJm?u9ywnCi2@oop1F+@V&lNV=5J1YOz~YAL?kI7isRxnCC8+KKk{g z05C}zGC>ek1eo{n3)Xl`4Nmv4BGo866Y=T=aw^btx%qe_IKuTgD0rN9m}>s_Uj7k) z%eu_h_0D~*r%|(Pfr@hlEGtgr_jqVW^uk$iV;q66LGkO+z}CsNn){32X8K!=F}0fm zWzjZI3p@;a%=8P@U0Z>w_5>d4fA;j#{idTbaBu@ap(4SjXeIqg%MMx~4b#fPFQU}4 z;mNmWA+{PQhSkh`>H4Ev+^l4E_VXwjpmeG*N@$KiS$TqPwuS1OWwGWmo&X0@B6|55 zqzGcC+V%z2Z+vf4rmT5XGwpsKZGFUZ>9d$H-R}5aEY+9UJA5@y#%;wKjEtA*_I^Qj zTUkCHzr)cMQNiG3EKcU=t0bvlFUn5gH2?uSuMRZ*P=N$RUy zvZRW&Y~+@>P*jeQXE6!e0(-a~wG}t_rY`^g6t9irLbi@3eQzs6I(aW97YsZhG`*L= zHDDjfC4ugJgA6?~hqSi+E zu7F(QH)HieQ&;`EQ+&RkHoYlIBhI^R&osZ+AXzECj|~vrX@UtSLt1a`w|Uy)DnPAF8S0{b zh4%7;EO*?obMkoM-FD&|qijotOvJ0$=@wS?96}%<(y&wLxj0(P^}lz2Z9#-ld99w% z$Ix4YKXz@Jo{}#)7eP&Gb$2}QaR{(=QjE=9e-}6xO+AUNo&v%Q3N*YIXQ}LI7cLQ# zzoo7+>fLRQg6>+<*Q~AxSFoM>C5-qevrb9*v{>rFnm)n-E}m z2t(LXrHFJ`ry`0~*x0fz4`qm*_aU;DA3fgmBrLwWgtu^b|L&tf``6$HWSlq$#JFP# zuSv~wuf4${72!8{F{Fz+tISTypT>jkgTi)<%Q{(dh}W)glT>lTLo~5w8^dj$ir6#RvVOoc+-CqcOPc6UTp!4P9(-@w}I`dp6p`sU#x{ znEa-gfB{NA?6%asicM2&pdh2)ahA=>)}UyP-5*cYbn@wX>MN91`F$+|C+6=__rgsm zKZ~?!@_x%Lb1Fq$jM(9fT&S0lT>Fd+z|WRl!)xs43nK@@C^sAKBY0jr$ z`CC%Z%yV=K_w4x`43`yL}v5vo6#kt}ZxzA>gqnNy_tmeb;dvj7te=BVA7;HPd@Qk4ehWva2C ziFe5p?y0*aCw~8q^r(4n_(e86;c_EbIi9&CHIY8kwC$(WStPVtq%~B#Re^I<=jaQ# z_S(p8)N)&$MGd;YPe7KjY@4D=Fa7qcRX7K0B73VK$t5V>Ud1?5$g_Ku*!Y&K<7|-d zy+T-j{9MA-S8|@D62tG8_4>_pawpiWs)^l0;?yrh0`Oxr#CbCveheM;mT<)aPpc$K z)lYloZY5I&2rQbC1C|HW@~J~Q0w+(!Y2B@d8u;xtu{h=$or*Y=|9b&0rMfbB9xQO& zw>OMKmD;r0-Z(R==2O4!bP{8cruu@x_P%BU$AbU>jwxJ!rPL4QN_kl-G83^J*U^!D z<7SV}(s4Ezh+6ngaa9%a->)!JVk%UyLBS0;I_J1iujxY>OPVucRhb;6P7>Y^5F zV*_|BTG~Ry#xmn6P+-%v9Cb z9jqYO)}e?$EmNT0?s*V@|0ZoNB+l+-6XBD8$s7!^1;vA=2~bQWyQfwbK^ z{0|5-ise2`)Bs4DQKHgpGIb!0iGmRMouVp%Jg^572a5EPdxam8a|18xZI@pWiq!Lj zy6kIvex}LDG{<+`D5v6tt-)hkq-5Ahw@y~WVx_q?lZ|n+HmD8SU^2XI671ed)OG9^ zt2pXOj_ej#%#}b(!ZMhcKUnp!^u&)aY;dwY~6Fb251(fap zZ-AR-`>#W#!3ziLcCmVsYmX!`DqyBcZL|(zU4jjLQUROrRQP7ADNZP!Ewv$+%&Av{ z_Ks!TG==7i;U`^3%kfXg1I@{qKk}vZdcVAmfo+YJ<3^J^qjMa+rsS)OT+qn|n7>Bw zZo`>M$-#>+PnpaXg7J)hPj-~>)1Jvz?)FvyZ??X0qO9)<_lqm`^NnSxT`jiDF7Z7s z_2~;W>%DpUsOiN*MM8;!GpS)1Lhe0pQwK9``g;zBDR|m* z#~q1u+}LzoyAEfOCZsQZ6B8jCJgP+Pcie3v(m&W~`u(f1yt&^L(|d+G>iQX6-~Z<{ zC>|tAROyOt!>8TPyav(z6nCJr?T7+@OKBocMYKxkw}e6bKda0=3Ub)Nc)5J+J(B-- zk{K2NkI!Sq^7VR5(pR*O+#~_O3=11}ecAu6W*ZQYpHe)+_|$F1xpxjG`=QDG#UDpk zCE`rNlHs3?8bxod7aEre8|@YvD}EjtoR%HC|vg(QMAADgH+dlxRxk63_RuWOM&La zRZ50Qf2Tm`aFJl?=d^3l@t^*8n#+%(uPWO@2lZIvIq&fBp!gOJ2ACJA$iSTc?mB=B zv3raErX?(%<@bb83sk;R{t4VQlL0!)v;R<+dvVckrz9)25ehM#KfgoePsmu{EUO5j z7ynJniVU1w^RgK0%NCTA^_Rs@A14MkJb{GY<@3d_4&Nw~9KJk*f7_ubCL%8LER%}L z=uV*R)0mqKQTfW&$U%12$-8@eYcP+x5dl((wrbTvhn-D7)5qEVJsol+bH`|Uq9bxK z2~qWY1?)H@LtJrHT!G*Pg7P!J0;FfBu;rah4a%9d}0nzVPGa0$yX%57p`S7QDba?tw0a zS$tSYyqX$>?+*Xg>wc!FRb~$TtX8w=*T$R8UigC~Fl*kg^@fliXRQktNpnqXbaSnD zGnPde#teXke<)#ShsfKP-;+z*KV?)<;y#t)~| zT(WBT+?nrv>)UZm@c0T4P1f@Ye&j)mfY6^ml;o>d{7#OWPHB&;;r%v{=TEjV!b? zeVeH#vDQ|YX>M>ktyZr7QAgUSNY0FARL+uX3E4hLd3+KhNV5fPvX!^NmUHb^E{A_P zgsa|MM@nP63!B{ZwScV;&y}IZK zhG`bp+p%7OW`RLmQa+1;C_})a*qltLI#f_dNH3gKz6BY(5R{U}E{=X(wr$V=Lo>B! zEy%ZOwNk%H7xPGxMlzzEsH3E{A~z|I>!$q=zb-u#Tb)g~#4p(Vs%G3&!OP<3E?Lcs z3~5(FJy?oMouZe(8_k?(EyNqm+P&^i5_}4Lvmn|msTI9tTCHB=5@*i$Lr7=seca?&4{^frN1(*W2i0wjf^TySHo3BzHLBW#2g@I!=8vjcx|N2! z3LO0j{M=J4+ww9|Muzpz{_8cuVKP*f^R0QkIXUN1{IKRMiB6}HS zc!H-r;PT}p7!~^fv}%~F7S5n|Ylr`7*F`U%Mf1$nT01RM#Mg8zbf_vijR_Q-w0SzI z`!!0gqIPC_vRR>D5m7K(Kel?m7`|}85D`*&{4PABGJy!s)rPd3v`&;GQ6(BT(of~C zWCA&*yjF2xGB5B|lt;oHEThk%1WTo`yyZlFhrL!N*z#NT{oj)w=hQZ@XBGp=3a^=r zD_(><33%oUR2bC6mj1NQuN));4Jh)syp`29=$HLdV()KUY^B>8D~n+BisB7$t~SRM zdeIIPs*G-9aP8)0N+vDT@dmXjl#VfZnaVsrs1NU^K*PI@WwMB;-~5>jpruSQ^;F2|5Zo{D~KE$8n;Hb4Ge_cev0VIE*(gG8uG@YgJk(ZgjUZ+IC&N$jWuN&Nq25M`$zN zPOi7RlP7%T783Y@2q9|f9{3lcnURD-A%}W)ezgQz z8r2PH^S2v4Pe7mq1Lr16&M~@54&gJ*HhBr^h`gMzZ4Kt3&v8;~H-^ei2XS>UT9Sp_ zVVArjjT@o_w) z^3?H6#0?UZkMK z2tknoc!Orntz_U&L#YitN2LC*7ttj1eb3Kcx8x@Pqm*K$=<73A9$(&ASt?4-%%>~@ z^dgROFQ_O}XlPRVDb5|amHj(hmzu})5@K#SV(vxaX-jx zopd;=XR37bul0D%;81^0#jQDbw5OsPT$k8AXks?S!HlCgMwG|?VCkjjvQ z1lXApc0`&vvbH^kW6s%L(wP-Y&0Kb`n?5{sd!J_-^xm*Rvga8Z7I(T>+x6)}wNs6~ zpM;T027*+o88QMZ6Gv;f`LZ?Yp^YB=#_t5e7R&+E?kfN zO>>IMZ7q$ak-BmND~8NpDdtgUo%coKPrTKCMGsH0gy$V-x6n2mamltQ{peBMr(&_= zl?uy$Xy%V5`lZKkIgfg*b0}i{{A|l(?qx3;u<6?_{&`fVW1n{{aTDu3qg`6s9J{eu zbA~UdjaMUCzBi%L{6|^e+5ZlkspIDi^Y81Vo=vl^9jgHg$zMxUUY~U9P44Uo$E%p4 zh!$mNmM|jy19NFgoI6*k=cu@T&%Q&H`a~Seqo$niI*3b88nxh30HoQ)!_oR*fIWG_ z(EG*M<<>ub!owg0eDEE+APgfF z@$*nQ@_bUz5l;}d<>RY~xzX->8i8|)fB#}~+a?dTokC60$jN7qx1B;Wajml-yeLV= zEs_pbX}XiQ_uR^hAZ@=C2xK0K=SX)C5XjVAVNlst&m2VROA~$zIxwv0z$IUk#42j_N_a4?zghzfnA;OAB>8+GPjr~n;eaPRR#3nWy;pF#5U%*aQPlo6#~gYk0DjSP1XC;@2156O*I$tUJM^Q}}f_Qr`wTLuj+G$ZAP=KCSI|Jj|+lEa3#a-}G6vj(@cKPKuhu>K&e zJ$P^K@iamKLJo?`?Lg%eB)U;g7o92MsC6fcXG69%Bt3q zxONi$Z`Hyn!pOS?jIWCQn-c*F^pNeElMBOv<82**6ud%Ei8T>}la)3TyWAmsyy31j z_;06(+)h_R1)O;Fv(Q35Il2cJ?~_q<77%1Y<4oGXFu{F0LsI9G;mv2zTpmQ{L15Yv zOeCqGPY8V2q(yzI+#OrG3osyAhUCy+ns9=XD1~66{v#F4A_jgI904(zv(<1r|YZ+SOoCxGI*#+Ye6YV+#Ii*Pm z>%SFX8zbsOT6<{{vr0)o@~9Uu(T4le&&_1?>R-zb69o8)6Z~^Wf+2+GjX2DRAuw>! zqA+D_ZJLvBM&;#P5j$j}EK0pYQzAGUfk@Bp?wy6)vR2d9&S_wvYZu*NbKFPmB9+0rN zWhB2=j(|ZtT&+UedSHQ5*ujNaKDud(0(wy5b{zADdf=enV}y%djR5Eu(m!f zvBABspeN~yMyg+Haa~sr8&nd=q$`9jWeKp@7sgEA6#qNNGNQ{W6K3vc(%|d=M)4dx z_6k!1cErae7zl&DGdlWng>vz-H014vI$5c^K*`3a1tuW+`lyBB5WRYQ;Jbc+?+1I~U zfAxdLso(f>@@SSAR>)vgVlt$9^_xboE#L2f9%pK{XV*@LKOGB9=#-P=m5JwNMEq~Z zI+flj6H}yADFjkJopF48K-%)6_rs~fG*pn>|3c<~09PoiSPTHSg1PkTQHhR zl)gLEqHz4`B^}CPmIaLg^C`&HxtpW0pbMs29Bys7IVkd0rG*;r^ptEk9#6zAJl$~Bp(m^|mXfdQF7|u!MImHpb z^2~3mrM5}x@Vin}DuWaX8Ssq78j&QHNuLs0{D$VI1K>Z!fU=;;@6OMdPJ_f%rWO)S zvY01yxL2eg9Ih`#oowR^O1?Z6&iyJ^$dLS%uOBDl?m`Z!bIs}GG9yg;qN=?2e{^I} zn=z(x5hM*IrSfxC$U#Oj#+M);JtuqBKydIIbQ;@8W_yyv_BrL;cEM6rMxe#>fRn50 zxAc?NdQCRCdDlBX`Ih{gOgo~uOP9UD?l*Uq<`4FOG3>>dBv4`Mb@IdSj*%BYc@8LW zIi9KUv`gWUw&YjcP$U?<$x2{nSUw5M^+~}Q$Y)U30I}-dLwr(PdiD5SZW}1NqsG&{ z-J-U+CV!;}VPI<5T@2-XJ1hp4b&<4I#2)rRmWcoOZ?jO#>*kz#*9?f%?b#j#o52aY z;KwwpL4f3Og?bSNa8Flgaj!4Y3Fj_<^|yeV7?r_)9}$CUyk*iKGd=v_l11C$6q{LG zEmx}O6+taN{ao!Eb#9lfX=faaJ)g8|peIVx(d?A+E;Ve!0PWA>2(=jt%Ww2M-QT~~ z8SuZDMx__;{sBe1EI4KkZL%i(PE##@_vhXG%++`wkB1!x7&WDS!oJs{N(=5q*8A&z z$NO@n>)Nrv`7d8cnWxG)N20w%1N$1#kH?Et$QUT=K7fxeFLt1V-~y^)hjYo$Ch~u; zk`V0G73xV^R~;Q%(znm-j_Up#dc4kX&-Y(iP9f(yfe5?@IS#Aomi=2#sEJslN`G1@9uR65VBGO1*c@{Lk(;LL(j~BM zl28XNfb{x>D}Bm)5H@xKg?4$}VI2*qH!(TiomhGu-Aeut)xB^2 z9^p~9%eiIwDq$g#K%lbqqVt^!XbxgT&C05QS3yP4Cfl=nR;(YsIPguxf63le>E6TR zq0)3F+u|9QQOytNn?@WW3ZMD=E$c<|tF@E=wrSfBV6P`jmeXMOZQfj`aeN5F5}^Rb zUV|$gm#(7#xOB7Ex5+jK3OgBwS#`a{-UoXl4cx^L7J*%C+k;F=PK=r7NIg1*@j)NY z^qguYT7e6lTT64lT~U7CJ|X4(M(&Tu4OZR6q&u|%I2JZIQx3erT3mz;@|&E&-u`P~ zOAOL`-uOk*r$o*uK^R%D1b7 zg_^mttgQo==Z%7N)930qrQ_cSur#qD&~pCPMsSX4l-KSo_H#G1#tulCzJZ-dL&y%h zwJG;)XtQ*G6(#x&6*`~wt9gGT7lX*~Tib>x;6G8(a=Z2~C`_Y(p)q)0@(Q`OJLx<| zu-QG7r4Av9t?AeqGtG6%bJw9 z{*E(o_k$-23N6;IKNZ4D=H2VvB)`6}{h=sh_Lk}NjwE+f{d?3`lA{%;00lY)RJc26 zkR@t@h;J<#Xx0BBGYX%9lg2?=QHPH>lRlh6f{-zA9=k_H&(yN5W|RL3Q$Wk3nOc3HcuCE zPRFEEkpny|t!bNz=piX1_-;DG#JN2kR@DXjC-n=0zd-X9I-44|13J2malBv5_Y9M7 zRfU&Kxk#6*kU^I@or}-hGx3{c@d*jHnES6^&0w~e4GnnPJ-o)iBW9@WccHpzL&C|9nklZru^4SXw_FuWIRLD$+YzL!JZ(mITW9%Yz@LvYduc9c~fyjNa>7@ z;66Ozd974B9AX0*)nP-*<-L*0C&55+rDx^+?&Y2jNPwj_SI==Yvf{^NjLp~0_gZmy zB`i$6t|)gnsR0U?9|-vbiT99oreo-TgFq9Mt?~=@&T$q?{?p*tQ98 zq$I%pmrMzx(1_tV$2{x5fxWww!=S~MqI1O+`HfJ}1CihNHufat1=(_Vh)QOWGo969 zVPwSSVvQWCHE3xHTpK@7e+cUyj_$?~o3Cy7`<{{&d5wG+##r*bA)6JEyBDWrrj{(1Txq}zYB+t4_ZfD_Y^rO25H(#jInTYCa~{Lroyd)LBEoo!DX+>MWDg*VtKf~{ zT%+~aYCk+7<%IcL0;=g>say*}jyQn=;d6g--=JR-1^)DL#=~=Gt$WbMxOfbWw4r6S zHwE21I=R7$L@$FH9R&A4tW&I?i_(V6RDrF0$L_x-&GCMiRXKe^OAqjRnQ$j=ZAlJy zh2rOsgIVP8jn?w+>B&tPZQu!}@UzAuIG_6<~+)=lZqC zPT#&&Wf*)d)pDYpf^Hiq)G&z0qe~eF{zXd5XM>iOTCv#8o#PB$7? z4k~YaiCZpZ|(YUku5QwCk~8`4?Htq6T)|qj6@F+e-Jp*nvMzKf}0$Osr&K*o@0@ z*U>D;w&2(J{=f-oEV3sO9}2&}_N( z?o!Koyaw_-N|#oJKfXG+_E^(r9h$p#V8)V+<(kZFz#%N?C^kd0`$1@acC!0sAD1p* zhZ@%y4yEfHvH3U_&oSt}STN!ua`0C;>^0arn%psCrf&mnA$nEDeS-6$bVfVpm2t4`=hC3B3LvpJz@C-J9_rhe;*@UgDq% zvPBY?RgQz~%k?Ay7?@YD0sRBv)@INR(uepBMt8Ea@$-Jiefbmdd#GSpovIK zSp=V+5NvY-KI=GuM>QKKwOtvhczPN8II4o{# z@imLPS0)iEW&Z8ri6Sy^oZLuN{Z`RCk&dK|1woxYQQ{k|6$=iB=^9iRNtTx$GZTnaYU8j z)={2^4QQ8&ZI(_^sQqA}DKwdHHKO5n-=#m`_l4H>Byo@}@&+0R)w5PjQMeM%2Jt39 z$gG>=S7jgM?vz`2;58QX#us!qLDDR6qjJ52R&Je7=bt`*i_|pm7lgb!0!$;wGv1q` zR@!+e!6*a~;-N1p5*oFka`V)z+6;TSk5H7wt6|CgB_6&u%LBDdfn-&vn)U z7Q;z8I(=-<9`p^!CQ3tB-0+;jhYJKDG5_GRm1=mCez)0zzHYmy0HM}|u&Hz4p{tp% ziGUfbXN1rBc2Ewex_>fE%<}_+#1{h_P~;b=_;n-}S|zB)a6lJDoUNsnWlCWGx{6Zh z-KNKFI*!ZQG;-<}C!*JHnHqHcqk7hVa$J!oeC z8rYrGc)1-cPbTuo*b`0{VFBZ3Nuj(uisTD`45~lGX%x#XR3`GZMY;_dX63Z3CK*yX zGYld;=azsL!g;v^W6!zoeyHV*zekSzqJ(;iL=AgoKO1RzNDW%u>&Wi{0|(2sKlh16 zCTfg24csjoY>Krd6Z006y9LyTq?~V+26QSEK6=#`4BI5Don2#o{-ywuKB?Le;8aFa zgoyF8r*(LKJV6G{3Q#Kax#nA}P{V1w0sf&`t!w2fl%^m|$mNSM#h78NqauJV<+;<{ zl1QQ8c7O>w$j5&`KpTlTWsD%zK&jh*homit5unYMq5weo9MBeOa#9BE=VwVi zy#ACKttlJTnqq#4NOy*$p#Wn|aEE+GZ|h!V&!B1!=B7ubf|h0yJ7}y3Qv9EHAo*5n zhacJX=-)4~-SjWtQq8Z&4-b#0R*^D$$C~rJa?Wyh=j(tMDf8!8Of}E3w?L-+>zp*iY|sYJ19K98uC%5;u5-+k+|-7QvNBH$RB)*Pmy^n_L9k!=RHgUjOPM%cb=iKlQ#-a3tA$$Lv}zH9{<+2rw&)&dooX-0!_U zzUh~i49{SRXwZ$dpvp4Dday7HKVFz9e+@wmee*q7#xX7DzLPR#{aZr@23n51GXCWe2JZKrm&E>u_*;kT&zmY?1xEZCM%Z$TRp; za!aUtM6uTK+38f|gW9Xiw#yhIg`ru;se+j*qu4@onJC<)=G2xEcc<^Qb+t4Qrz zgU5h~1-eH@mW6P9gremi1p2RWB7ZVlzp@m2GfP>n=>{*2Sw?~Z*N3TyYb?}@cF0&t!HE!p`1it=LGIx=IsNkWx3j=YM1lGot4uh%2 z7ETe6m?EOzpcom-QA=h8m3lAv2i2NHqMck#_O&zAlS-!%Ouie*5kJfj@%>P{eZT&Z zW%>{3d#JIClJcqTB`=;$jc!m+6xEA1pPie=$h5xOlJ!JyIY26)fAaoYBD<#!t4DE* zC8`{D!ZX{7WPVE1xK77SN-o>ExcM{UVl9oVm?Q6&=J4FZG-@*o!+CT(tZwLcFn_4DuFGDGWsbm1>E_+4KcvWe(!|)HwbP;34y$f# ze!5v-Be8QHRkrICOn8=PFfg`EBDbh1xq%?=vt}VtM4Opu^!#sM3|YOxu7+ul0yQQ^ zI433MNz>UKaR1pgymwfS4ftk0l$|+XC*4LdnguvB5y_R=b*VNNL&prOC`398Vl;y~ zTr#JYz^eiVem^2$si(iOZ0f7IXx9+DBt{~-7iNa!n)n0wo~UU5^zTtWx4U#5Y=M&D zEVoODgs=4moiQ8oP;RXIHW3Mk&|a%5q3%q`2bV<>(=bY{QJTeHPUKpilL&|BxxZ39 zqh?TynMO?%C#3U(D}bSskN=R+wy5z2Zo6usVmBPQ570ua_G$)q4R0h8kIzxdO5nLn z?Nk6SX;~2l$u#u&{g7D?09&HT*M2r{Ts46ju_ z-lcI>@ zcfqU9C??3*sIgJ<`;-|??KPS&?Yd18@Mt-$*7F^0^7K?3HDp4~MTxu15_GW!Q~LbD z#*ErkztX>5V-`pf)%lCi40{jGZx$owYT9XPS$?zT!IEbM>XAK53ag=2C85tvUJ^~kY*%(Y85sVr5Xqu za#`;}Z6J2L8U_Y(gRFXd3l}a$AnD)TBEQgMK6@W#Pt5zsjJ2Wp(?@JpL50%QZ1|F9 z!onQ6Yu%$e8cCKDt6Tly0**+AjzFLoQcLXrhRFrs9$K_gjx0#>E$-X2b}Q9E1Vb=slqw@#S+uwK5@gDLo_-1Wq~TaWL)g2Ld61Y@&u|ysv&-x^k2z4) z*EP;Ka#jX;$P-&yU^I%)TuR=AI=f`PGzRwG`d7XSCx+~9k2se#Tym}W=c)J#J(CZF zwAXchaF_P*wo%mHt7lAj9(8N)rUHVhiTGS80NGBVfEZ1;?A}|x{Cn=ouMK!sOhPu^ zHnksbmBJe-e)lI7DCXIy8a>Ay%`6|G`4(zBn@~)Var2^p&~}x(Kvw@0_>+{N`$6ju zX}~A(g~XEH?`fZIr8#5O1{!~zjo`OkpRE!f+_@F*I^Oh87X_L@NBBn>WJm6_&8B_O zT+QE+##(yZw%^y6o&I&FdEQou*|XnI+t_bcW+!XEj?ws2JjpV*%dTZm=X@FA#40=& zZ*8Qom^;NWRdmU$kQfV`85y!{2@)JR2{q1eG!K6K!kMdQuv+uQ1aX7f*T+8Lml9p9 zOH%iWXfhxD}M1Zh;!{}9^iNQet2RrnBswv^Z68=%&0gYOgG7;Ka zj5+elJXay7=Y(2PE}zxuyu|>e!51;-&G*FVFOlTqVqE|usQ#zJeW2m;$AyHF`=U290yW0r|ypU$iHK`&7|!f}uQ@awzZOVspex z{y4{@{Hqj!+aq7e0J;B>q5}f_essVkoiQ&{B^ll2DK(vT%dk~fUD=pVLt4Fdzj8NC zH&6IcI%us9;T>J$RF8OhKi4{1^WM3~6?#|ThaCYxriIyI+`!WV(dXtosq(d8RV|Wy zi1=~Y3ldrsC}UEulx7X+cL`)~)9x5CG2icnX#LEPXaSs+P%~Pzzk7MO6-{Q7gh`x6 z;+W^8{L8s!#}&?UAyhQ5oj;(jIdn$}1p~g(QyA(NLy|RotCla7)6vCB&Qob*t`nn4 zUD8BD>%w!Pw*NeyT+~JK4cTl3i<}dXE1t&(6*C#urawQx&3^jvGDqqq-SKSI9%irZ z!j`1mm*sqY2Ao4Qc{g-_zOl!;`{xvJp_fmjPod6&@qwd*S=afs+6uJahvr_-${&b> zWa1r9tSvV8Vixu~?16FOg|d=0BZBWpsh#zDUOi1sf}I$3W+SzRRiE}nKh=`eaD2

w2>6eI*uy1S%9 zy6zc4Ki}`Vf583avStAbX3cxf`<{CCv-f@;90QQ)7YJn)IcYIbk$n;d)zSe#;`2!L z`t?t^fOPvv%T6+~R}c>L%Qem#=4ggl3A6Pu5#UZA?`|OJ`f)Abpp!%+z~; zCJnn-X|>_Rgq0!F^?0}NPmmsj|Y(re^g|N7Zu`d3VxY*cqf-28y;1)MLVcAIa+^ z3WJS;Uub)qq;QPCuRQErNCQef1EmsT>)&CN0%Lt2-sPJqHWBThIhe~?+*`;SgkiwS zY*8v?`L}7?M<$63zDGVmO6>AFS)VvRJI_eUwg{6FRDx@-DiKvCE^;s??JJa@p;We*{>~8lkqGJe+4ylC5VaD+V=A~gB)Fn zruv0teY+r`t6B5@8q>&rX-KR$7Oo@k5lWK#DHy_-vI3&~aGQvEllMA!ldpzB71qFN z48La9&OPLDjMCC7=plDK7Ve&_AIQVZ`Z2t*V?|HbCT5fi=PJSQw~En1gS5=nS$)0L zxfl^zvx^FY9tS6;6U+jhVYyV<+Zm<^rr1x4W^lkh%wNSX*IO&lm+z$El(-cg-bb{u z;tQN4v=kLrPUl*++}geltSF)95_nEXq__IPSHy$KzUY`1ldxf5cHtFrgppeZG-S1) zh2XJAAGeK#hNi{eyo;N^;LWa?aJOf7=^|MGmT(v8_|(n!I~7yqEyHRa9M50B0$@U& zl7bH1(&)s3TM4wxyxB`KamVedzz5Km+fI`^fFCU{T06afd9VFDuXt`JR+=+p-0x)MLC8VnzB$h z7Xmf;0mA9rDm9uA%V+IlXYueI*nG}UqvPJ-$&I6f=CnF~BgYi!eqbOK#rvb7XUyab z@J=2N;Slbk29I6Tx)GA8BSX_0Nuns!K^e@jsl#MOsbE5Z;e+G z#(S^Jjkgb*1Q-N9bEQmmFc^;=MNc{b}kK(jimHeMEKo$ z`GOG-U^8K`m$ng7ia(m|D!0<_a62kjzE2f)VyYgo?|QQ~!0$wK&uOxuZ%1RQ!3M)z z^V6#%h^eCUI~XUCgg`NC+6&*bs_i6WE?@TFP%o5=Pu;C#n#wK-bVZk9(bEL5E0Eul z;9)L3M_qDfBQBVN8nYV=4CKkyu|zGUBqfw^<~y9wmd$wh7E?Ca;-EA=FZx_mW;94$ z14vAst-k^NgF#26l`mLfk>5Wgj6>99!tZ-BPkM+AF@1Wi&&?M%bdo-QzoYs2G?zD$ zjLzlgE6M0n{T}#fuJcw_dH?;V;q-4RsPIR0SS$}o_?_bd)O8r=&WCseq{$W{w=ZCY zVud;H+L9;pz7@;i3y7+V>CL@S#m=Q`kWN}y(wobuZ|8IkZ{V)2ELyU|E~E-ro>CLE z@L!|qNwG^Q<`GyA8FLu zvZ%4kz<_;6nHu$AF{p5W;5;B}YYi43Inw++m5@*-3GzBwfGj<-vX_^+>YCu>xj@zPeDAG+$A% z3J%;sg7gB6H;@+U&xyl6D-{fgGNi zHCDxD0S?#&xmNDpl937)>^sx;(v%H>4MWGXity;;BCyp&fxM$Qh z6)8_lN!qu|rpc379$s)?MmN__fJ$%7cW=C0KS`1WhHxPy-*D0}8K~yATjp$U9SqMC~Mp|=?UoGXm^u^&+G23CLA3}}@Q6T@12&dB>) zDYoRzh#?1Jlqm)niaro>t`3;UH*Hy_FfDR1=}PZfQJW8Wi;oRsfP}}Cb&KzBW&xn{ zS2B^l;G+{veo?>FB7Z`y<_`ZGvjopwBn4DPE`)h^34k{YF};v(F8xeLg-zy%Q}4i! zS;H^=p5D(d1@iQmMHv#AJ+-nSQT;Htn({nxgy*Z%rV&^a^|25Avzt7NB?g4aoYvG6 zI;$?vCfV#gCf-15ev=ouBnF?{S+1_?@ZzLATkiYol0*T z;S6pky0F$(j2 zZo3%64I6C3UV*14GL6}Ppb`z4U0Qd5e>CLc%WGlIgfqKcNqsHC7{ZXAO4|Bxn4!-0023*r(13h*liw5 z+~z5LtyOKM=%9s78+J*(Kpw_JBuGS7R>k>Bk@FiE8-9t7J zB_#riQZK2sMVtcq8iGUJy&;92HcxtK)qWgnR$LU-*Aa#~?kgsC33;56#N^KS9X!MJ zFolYin4SH%My$J$(4f5phv^*VkqbVWqgGLbvX4$B&h`uu-2S@S-oeH&8c1`-rnU-E=~sufY&s33k1*_NG^N9ryEemgtKAt3)71kR z1C?tg@=iMf=%O3c5)IUt9AAsuJj5G%Z$E(+=61;LO5+S%kUKXkL8<%xGFOKQE1K(p zdQ*2>)de_&DOi=mijrwE1xOR9#ILu!4-p&k1X*7bMJx39#y1<7Cw%%BOl*YPcY+O6?C zeAh4}KP4Avx11Hn*;-hgr}m+pl7zHoJa&vwv0^MGRRNgJ!jemA9x{_tTOded8|$ZB zevzfOq}KrEWUD?TQt3!SooNT(_&K%Vq*v#~F|}HXXa`XY?j80RA8vR5L_>!=;WtDj z^JLD1sK-+2N0uCuL6+fwCf-K|VRh^XXA5+%s`p~H6Bx!(pJ%ir<4Dd|zt~^TNBAwx z$Dl%=8the*uh+W^5}S{2rmS#jASo#v=3TgZfu7Xro{n@B3GW@OV!s2l}N}4I={3Q zwDy{622HeuT&-X#-t^u*XJRp$!U)<{PvncjQq?Pq1H%=~*<$ISe)d-&Dw*i~@s z_vkZEzc17;_SA!nqlS~|I&X@g>*|;6nkq4dW+?Y$%TfoceFT+;P3?7(lsoO;rWGe0 znG(D<&m82;`J!_~RCucI{(V|EBpd7P9Z({@e%k2s5XAUZX|@BW5Ipi30G{P;*%z&e zQAWgPn7n*oe&;NkRR`9%+F1>-aQi$SG>vP6xKA+aegk1#>E=x}-$jW^$Eti2Wi0geLaWUF zD-M$h6;WJIuXS-Nq>%vHd9K_IjO=t#@Qck_2rH`aLqCv>p+ zw3l_#^D`YNgsi1jmO#>T#ieA=g7Lz1`^zN4zIumpXj~&RY<2iR>Pr~+^ChJIUY$;7~zxB;Io5dKD$Yl%?kd>U!Uc`zBn7I@cZAD;FR=r%b&2eMxQ zX4gDZX>ply_)jNNvfIuM3|9F9Z6Hq-PIQa~$_&xkzSrk}B}pKJbYRB#WY`~4&yJSb z^r^zF9^VuszZ5j%%B)N%V8}Z&?+1H}5g(LKD3>Yc@l)FjD;X z^oJ%PIluc%M9ut@;~vSk#BoZ|r0-nDap_8%D+{%r19sWt3uAwGjpo7#i@vNfn&?yh z$|6|Djsg~ej(;6<+I5=}1@%Z{=)C^SaTEpvYPnt*0Ul#rm9M&>yV&Dhe18aHQDXT8 zw8d@#u(NAoCr0-?+~Rj?@qRps-Qhhd*8{~^%VIKyUd7f1%n$jb5;)J@k6bILD|Rm| z95*cOZ=f6u!9kWZ06w~BIK2juW-j7ufsG9WXln86fZqZ<1jRa+G~7J50{O#pmkec| zLA$>Wj-3pRigZ-$5q>=c%UBmuF|$Y;wEw&j^A@$7;g`0M3VKLL_2#AwXcH7A@q226SOzj_VN+BY@y@*t8M}iNX zzFf#Crolzf5+5ety8h|z;nXL({(6`lVYBry0*X3=-#uAP&NmE_^yDUK=zqjQfB->; zlvL*ummjdnv2 z*AeiP`ZJx3?F{z=3S~=NwLeH*Vu}>nQs6VkdQILG*h@@*jS7^nRDaW{m2(T}Cn@54 z$24ZQCG!#-5^fB?4U8^s>m2a)Dh<2alvfNloH$sdf&9no9!lL@qhILf3796AF+6C4 zGBeUH0k3w*q`~rDESV?JX8i0#VK?4s=8U9$K~%@j^Y=F4me}2}&(#|{PS-x&hfagC z?q+os;*~{OY2GCJK=Et6?%W)EDbxb@o&xlXe>z>aGy2ii@dAbktS=ZC{J#X+P+u?e zYPfLE-<~NVM??T|@OzNo1|}Q}SRqA`-8AIEi4|eZJ{T8Ev>{g=L*`IH7I4jk_sM-z z8Y|%Gurid30&4Uo0K66=;Oe=Y zsXEt>quhCt-cmKNv5gIaXFU7F7tZ>!f2hcy^=pAhSCn*Z=W%ySEk^H=*bl-)OMbM* zZIxouq?6)xqg+}0b*M0~zq#i7{3-AK2XL~NZe)ovFVE3=E1^b5@5N#+)j$hMou(8S zUV=Pf{MX)VcpkMcJxXRM#c502jzr?0^(Og9Crg1~+g6Vtmj7>CJy^l?IJtou^Sm*( zu~@6Y==4Xutx*qGe60Z;Kw1pcQLeywWD8QkGyh6_{nn}QG_bA&ZuU2i_t#07$4f$~ z!~)iQvt|F9TMIYB?ljVk!Ia2$_t`;d=S{gUlA@1+?FNVbj4J)G8?En^WRPOI3S6Im zV^nw1jlv2To|wp%(ktW*ggN0nl%W)vZ6g=benEKEDJ#btKkkv9GhVw^k2j34zaRXt zKLJ%HRuss+whJFzfosS?=fMqTMjd4nia6agASb5#u8n_=HYgs=n$_m#yW4f?hmIV? z?#RxE5}SvHueNQotw zo`kF+-$tRSiUO`GGt(Pb8P%>`j@^^;An4-Ys0 zJPQ!lz`7qvWyjaY85I5RZKUx4+*d3+U4I~-hEx-_%$#*_X0X??5~S0CLKt{tKvJTj zCIuvAYZ4i+QHGc}s>J!7j%KTCE@qG9OjPSQx)Zp*j|G9%I)B5zb&$InK(5HJA;17C zhQdyrX~5a}19&gk4d85yd_OB!TKcvqqqJVeHwRD##U6N}SahEkcgB=W3e@xQbQ&0E&Penk5!E`t9-WsWoj%w401ZV<`z3EnMQO&Dg!~kVxAp z#eQ5+PK*`&>~6@W zyj@JzUx?M}cRw)NOfZMa^rg2XCyDH?ss%@xgE)h1S=s9q#sTasE`dcGW-*|0Vl!%s zmR^ZweFf53>0ixjhI>n%DE?h3u9G0e4pGdC0P2eLV*Y}UqNWEHDh~H`mLxCcA-A(* z(eQLg)C*9RJ>!^3A?<-T_CueFB8{T&3pk}AfweEI^XzPge&eucZOI-m_U9DKl`Obz z;Zo+9OwdYt3Ryd*oIm1nABRs_(GPPq1jPQ`zm>U>`DZiNSwM9%hv>lFK;F=N9t*Ht z)j{>3jUP#BL(6^QtP83H7UAs$wzWj8FMu7e{!sYAJOL$@fBs8HIywf`8m~(Gh32{o zi=0)dC22T-neO!;?XM^Bxmz7nG});MJ${{+SSWLS?(Y_Z3tS0+pxDe)1bB{^LqM1T zGp-j^`5s0?2iQ0tO*l;?vWH`w&H_YH_W-KU>$y2u<0%QMjMRdHNmOtKdeO1e+0BpS zQ|>vBVotpugEFXs3fluQV{M&Px3AV|`Td zi+LcIt{7pFBVJKIVW2)!vY^0sg^WLcpzyd!aD=>nd8)zHoDE}S{t(oMBRuX2E*u_3 zm3%MRAk%^Z2l)5eoNJPM)+$E(70+x{;dBS<>*X zk@^wT^H3ioc^!X<51VoBB0?3u`%$jNkHfTe&|QJEiZx(gNg8rDfp7(va=k}?O0$6k zHPG)a?(G{lb{za$kuqi4jFFb$6){A?1L>ugdc3zwJ8YPGf|*cpzhEIl&)0hO48Shd%P}D(B1p&V9&T z6C3$VzKFI){`Z`KD@__wZrU-vlUMN+_(*Vm3j;?) zes%d*|Dp!umHvy4{`=ID7ohX&RKLHjE8*w@;3 z+8r)EoGx$DA}u&@?w{`-bai9>Em?56KzS73nr4}0d@z35=&SFZ9>BfhA)AFGIBIr%5^ rfY9+@zuo`YrT_o?|C2jFLbc_-yO2W>BN$6?4SY$6N<)f7w7vch8)kHi literal 0 HcmV?d00001 diff --git a/doc/source/ray-observability/user-guides/index.md b/doc/source/ray-observability/user-guides/index.md index d50e4b8b0ae8..a39788f1c569 100644 --- a/doc/source/ray-observability/user-guides/index.md +++ b/doc/source/ray-observability/user-guides/index.md @@ -11,6 +11,7 @@ configure-logging profiling add-app-metrics ray-tracing +ray-event-export ``` These guides help you monitor and debug your Ray applications and clusters. @@ -21,3 +22,4 @@ The guides include: * {ref}`configure-logging` * {ref}`application-level-metrics` * {ref}`ray-tracing` +* {ref}`ray-event-export` diff --git a/doc/source/ray-observability/user-guides/ray-event-export.rst b/doc/source/ray-observability/user-guides/ray-event-export.rst new file mode 100644 index 000000000000..4c44021062c2 --- /dev/null +++ b/doc/source/ray-observability/user-guides/ray-event-export.rst @@ -0,0 +1,134 @@ +.. _ray-event-export: + +Ray Event Export +================ + +Starting from 2.49, Ray supports exporting structured events to a configured HTTP +endpoint. Each node sends events to the endpoint through an HTTP POST request. + +Ray 2.49 supports exporting task events. Future releases include support for other +event types, such as actor events, node events, job events, and more. + +Previously, Ray's :ref:`task events ` were only used internally by the Ray Dashboard +and :ref:`State API ` for monitoring and debugging. With the new event +export feature, you can now send these raw events to external systems for custom analytics, +monitoring, and integration with third-party tools. + +.. note:: + Ray Event Export is still in alpha. The way to configure event + reporting and the format of the events is subject to change. + +Enable event reporting +---------------------- +To enable event reporting, you need to set the ``RAY_enable_core_worker_ray_event_to_aggregator`` environment +variable to ``1`` when starting each Ray worker node. + +To set the target HTTP endpoint, set the ``RAY_events_export_addr`` +environment variable to a valid HTTP URL with the ``http://`` URL scheme. + +Event format +------------ + +Events are JSON objects in the POST request body. + +All events contain the same base fields and different event specific fields. +See `src/ray/protobuf/public/events_base_event.proto `_ for the base fields. + +Task events +^^^^^^^^^^^ + +For each task, Ray exports two types of events: Task Definition Event and Task Execution Event. + +* Each task attempt generates one Task Definition Event which contains the metadata of the task. + See `src/ray/protobuf/public/events_task_definition_event.proto `_ + and `src/ray/protobuf/public/events_actor_task_definition_event.proto `_ for the event formats for normal tasks + and actor tasks respectively. +* Task Execution Events contain task state transition information and metadata + generated during task execution. + See `src/ray/protobuf/public/events_task_execution_event.proto `_ for the event format. + +An example of a Task Definition Event and a Task Execution Event: + +.. code-block:: json + + // task definition event + { + "eventId":"N5n229xkwyjlZRFJDF2G1sh6ZNYlqChwJ4WPEQ==", + "sourceType":"CORE_WORKER", + "eventType":"TASK_DEFINITION_EVENT", + "timestamp":"2025-09-03T18:52:14.467290Z", + "severity":"INFO", + "sessionName":"session_2025-09-03_11-52-12_635210_85618", + "taskDefinitionEvent":{ + "taskId":"yO9FzNARJXH///////////////8BAAAA", + "taskFunc":{ + "pythonFunctionDescriptor":{ + "moduleName":"test-tasks", + "functionName":"test_task", + "functionHash":"37ddb110c0514b049bd4db5ab934627b", + "className":"" + } + }, + "taskName":"test_task", + "requiredResources":{ + "CPU":1.0 + }, + "runtimeEnvInfo":{ + "serializedRuntimeEnv":"{}", + "runtimeEnvConfig":{ + "setupTimeoutSeconds":600, + "eagerInstall":true, + "logFiles":[ + + ] + } + }, + "jobId":"AQAAAA==", + "parentTaskId":"//////////////////////////8BAAAA", + "placementGroupId":"////////////////////////", + "taskAttempt":0, + "taskType":"NORMAL_TASK", + "language":"PYTHON", + "refIds":{ + + } + }, + "message":"" + } + + // task execution event + { + "eventId":"vkIaAHlQC5KoppGosqs2kBq5k2WzsAAbawDDbQ==", + "sourceType":"CORE_WORKER", + "eventType":"TASK_EXECUTION_EVENT", + "timestamp":"2025-09-03T18:52:14.469074Z", + "severity":"INFO", + "sessionName":"session_2025-09-03_11-52-12_635210_85618", + "taskExecutionEvent":{ + "taskId":"yO9FzNARJXH///////////////8BAAAA", + "taskState":{ + // key is the integer value of TaskStatus enum in common.proto at + // https://github.com/ray-project/ray/blob/master/src/ray/protobuf/common.proto + "2":"2025-09-03T18:52:14.467402Z", // PENDING_NODE_ASSIGNMENT + "1":"2025-09-03T18:52:14.467290Z", // PENDING_ARGS_AVAIL + "5":"2025-09-03T18:52:14.469074Z" // SUBMITTED_TO_WORKER + }, + "nodeId":"ZvxTI6x9dlMFqMlIHErJpg5UEGK1INsKhW2zyg==", + "workerId":"hMybCNYIFi+/yInYYhdc+qH8yMF65j/8+uCTmw==", + "jobId":"AQAAAA==", + "taskAttempt":0, + "workerPid":0 + }, + "message":"" + } + +High-level Architecture +----------------------- + +The following diagram shows the high-level architecture of Ray Event Export. + +.. image:: ../images/ray-event-export.png + +All Ray components send events to an aggregator agent through gRPC. There is an aggregator +agent on each node. The aggregator agent collects all events on that node and sends the +events to the configured HTTP endpoint. \ No newline at end of file From fc73181245bf5fa39c62558a2f9c3a342bebb59c Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Mon, 8 Sep 2025 14:07:41 -0700 Subject: [PATCH 510/634] [serve] deflake test pow 2 router (#56342) We only back off after the first "all replica set" has been tried once, aka only after the first retry of the "all replicas set". So the order goes like: 1. replicas on same node 2. replicas in same AZ 3. all replicas -- no backoff after 4. all replicas (first retry) -- backoff after So r3 could get selected on (3), or (4), or neither (3) or (4) (for this last case we'd run into the 10s timeout which the test swallows bc it's expected to sometimes happen). These 3 cases give that we could have gone through either 3 or 4 sets of chosen replicas. https://github.com/ray-project/ray/issues/55617 --------- Signed-off-by: Cindy Zhang --- python/ray/serve/tests/unit/test_pow_2_request_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/serve/tests/unit/test_pow_2_request_router.py b/python/ray/serve/tests/unit/test_pow_2_request_router.py index 54ac8d612202..a594b94045b7 100644 --- a/python/ray/serve/tests/unit/test_pow_2_request_router.py +++ b/python/ray/serve/tests/unit/test_pow_2_request_router.py @@ -1918,7 +1918,7 @@ def fake_sample(seq, k): assert done.pop().result() == r3 # assert that we tried local node, followed by local AZ, followed by all replicas - assert len(chosen_replicas) == 3 + assert len(chosen_replicas) in (3, 4) assert set(chosen_replicas[0]) == {r1.replica_id} assert set(chosen_replicas[1]) == {r1.replica_id, r2.replica_id} # assert intersection of chosen_replicas[2] and {r1.replica_id, r2.replica_id, r3.replica_id} is not empty From 5b70e3a888fb9cc23f5ca05a9a4a04496c2ca5b0 Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Mon, 8 Sep 2025 15:56:12 -0700 Subject: [PATCH 511/634] [Data] Remove deprecated `Dataset.to_torch` (#56333) ## Why are these changes needed? `Dataset.to_torch` has been deprecated for 10 months, and has been slated for removal since May 2025. This PR actually removes it. ## Related issue number https://github.com/ray-project/ray/pull/48692 ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Balaji Veeramani --- doc/source/data/api/dataset.rst | 1 - python/ray/data/dataset.py | 131 ------------ python/ray/data/tests/test_object_gc.py | 13 -- python/ray/data/tests/test_raydp.py | 14 -- python/ray/data/tests/test_torch.py | 259 ------------------------ 5 files changed, 418 deletions(-) diff --git a/doc/source/data/api/dataset.rst b/doc/source/data/api/dataset.rst index 472ffa02b2f2..4e4fc16a04ad 100644 --- a/doc/source/data/api/dataset.rst +++ b/doc/source/data/api/dataset.rst @@ -43,4 +43,3 @@ Deprecated API :toctree: doc/ Dataset.iter_tf_batches - Dataset.to_torch diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 7fb1c46b7651..9d655547e66b 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -5214,137 +5214,6 @@ def iter_tf_batches( local_shuffle_seed=local_shuffle_seed, ) - @ConsumptionAPI(pattern="Time complexity:") - @Deprecated - def to_torch( - self, - *, - label_column: Optional[str] = None, - feature_columns: Optional[ - Union[List[str], List[List[str]], Dict[str, List[str]]] - ] = None, - label_column_dtype: Optional["torch.dtype"] = None, - feature_column_dtypes: Optional[ - Union["torch.dtype", List["torch.dtype"], Dict[str, "torch.dtype"]] - ] = None, - batch_size: int = 1, - prefetch_batches: int = 1, - drop_last: bool = False, - local_shuffle_buffer_size: Optional[int] = None, - local_shuffle_seed: Optional[int] = None, - unsqueeze_label_tensor: bool = True, - unsqueeze_feature_tensors: bool = True, - ) -> "torch.utils.data.IterableDataset": - """Return a - `Torch IterableDataset `_ - over this :class:`~ray.data.Dataset`. - - This is only supported for datasets convertible to Arrow records. - - It is recommended to use the returned ``IterableDataset`` directly - instead of passing it into a torch ``DataLoader``. - - Each element in ``IterableDataset`` is a tuple consisting of 2 - elements. The first item contains the feature tensor(s), and the - second item is the label tensor. Those can take on different - forms, depending on the specified arguments. - - For the features tensor (N is the ``batch_size`` and n, m, k - are the number of features per tensor): - - * If ``feature_columns`` is a ``List[str]``, the features is - a tensor of shape (N, n), with columns corresponding to - ``feature_columns`` - - * If ``feature_columns`` is a ``List[List[str]]``, the features is - a list of tensors of shape [(N, m),...,(N, k)], with columns of each - tensor corresponding to the elements of ``feature_columns`` - - * If ``feature_columns`` is a ``Dict[str, List[str]]``, the features - is a dict of key-tensor pairs of shape - {key1: (N, m),..., keyN: (N, k)}, with columns of each - tensor corresponding to the value of ``feature_columns`` under the - key. - - If ``unsqueeze_label_tensor=True`` (default), the label tensor is - of shape (N, 1). Otherwise, it is of shape (N,). - If ``label_column`` is specified as ``None``, then no column from the - ``Dataset`` is treated as the label, and the output label tensor - is ``None``. - - Note that you probably want to call :meth:`Dataset.split` on this dataset if - there are to be multiple Torch workers consuming the data. - - Time complexity: O(1) - - Args: - label_column: The name of the column used as the - label (second element of the output list). Can be None for - prediction, in which case the second element of returned - tuple will also be None. - feature_columns: The names of the columns - to use as the features. Can be a list of lists or - a dict of string-list pairs for multi-tensor output. - If ``None``, then use all columns except the label column as - the features. - label_column_dtype: The torch dtype to - use for the label column. If ``None``, then automatically infer - the dtype. - feature_column_dtypes: The dtypes to use for the feature - tensors. This should match the format of ``feature_columns``, - or be a single dtype, in which case it is applied to - all tensors. If ``None``, then automatically infer the dtype. - batch_size: How many samples per batch to yield at a time. - Defaults to 1. - prefetch_batches: The number of batches to fetch ahead of the current batch - to fetch. If set to greater than 0, a separate threadpool is used - to fetch the objects to the local node, format the batches, and apply - the collate_fn. Defaults to 1. - drop_last: Set to True to drop the last incomplete batch, - if the dataset size is not divisible by the batch size. If - False and the size of the stream is not divisible by the batch - size, then the last batch is smaller. Defaults to False. - local_shuffle_buffer_size: If non-None, the data is randomly shuffled - using a local in-memory shuffle buffer, and this value will serve as the - minimum number of rows that must be in the local in-memory shuffle - buffer in order to yield a batch. When there are no more rows to add to - the buffer, the remaining rows in the buffer are drained. This - buffer size must be greater than or equal to ``batch_size``, and - therefore ``batch_size`` must also be specified when using local - shuffling. - local_shuffle_seed: The seed to use for the local random shuffle. - unsqueeze_label_tensor: If set to True, the label tensor - is unsqueezed (reshaped to (N, 1)). Otherwise, it will - be left as is, that is (N, ). In general, regression loss - functions expect an unsqueezed tensor, while classification - loss functions expect a squeezed one. Defaults to True. - unsqueeze_feature_tensors: If set to True, the features tensors - are unsqueezed (reshaped to (N, 1)) before being concatenated into - the final features tensor. Otherwise, they are left as is, that is - (N, ). Defaults to True. - - Returns: - A `Torch IterableDataset`_. - """ # noqa: E501 - warnings.warn( - "`to_torch` is deprecated and will be removed after May 2025. Use " - "`iter_torch_batches` instead.", - DeprecationWarning, - ) - return self.iterator().to_torch( - label_column=label_column, - feature_columns=feature_columns, - label_column_dtype=label_column_dtype, - feature_column_dtypes=feature_column_dtypes, - batch_size=batch_size, - prefetch_batches=prefetch_batches, - drop_last=drop_last, - local_shuffle_buffer_size=local_shuffle_buffer_size, - local_shuffle_seed=local_shuffle_seed, - unsqueeze_label_tensor=unsqueeze_label_tensor, - unsqueeze_feature_tensors=unsqueeze_feature_tensors, - ) - @ConsumptionAPI @PublicAPI(api_group=IOC_API_GROUP) def to_tf( diff --git a/python/ray/data/tests/test_object_gc.py b/python/ray/data/tests/test_object_gc.py index fd6b5e3f49a4..2994a57bcf77 100644 --- a/python/ray/data/tests/test_object_gc.py +++ b/python/ray/data/tests/test_object_gc.py @@ -33,17 +33,6 @@ def _all_executor_threads_exited(): wait_for_condition(_all_executor_threads_exited, timeout=10, retry_interval_ms=1000) -def check_to_torch_no_spill(ctx, dataset): - # Iterate over the dataset for 10 epochs to stress test that - # no spilling will happen. - max_epoch = 10 - for _ in range(max_epoch): - for _ in dataset.to_torch(batch_size=None): - pass - meminfo = memory_summary(ctx.address_info["address"], stats_only=True) - assert "Spilled" not in meminfo, meminfo - - def check_iter_torch_batches_no_spill(ctx, dataset): # Iterate over the dataset for 10 epochs to stress test that # no spilling will happen. @@ -93,8 +82,6 @@ def test_torch_iteration(shutdown_only): # The size of dataset is 500*(80*80*4)*8B, about 100MB. ds = ray.data.range_tensor(500, shape=(80, 80, 4), override_num_blocks=100) - # to_torch - check_to_torch_no_spill(ctx, ds) # iter_torch_batches check_iter_torch_batches_no_spill(ctx, ds) diff --git a/python/ray/data/tests/test_raydp.py b/python/ray/data/tests/test_raydp.py index 84576aedaed9..633018798182 100644 --- a/python/ray/data/tests/test_raydp.py +++ b/python/ray/data/tests/test_raydp.py @@ -1,7 +1,6 @@ import pandas import pytest import raydp -import torch import ray from ray.data.tests.conftest import * # noqa @@ -58,19 +57,6 @@ def test_from_spark_e2e(spark): _check_usage_record(["FromArrow"]) -def test_raydp_to_torch_iter(spark): - spark_df = spark.createDataFrame([(1, 0), (2, 0), (3, 1)], ["feature", "label"]) - data_size = spark_df.count() - features = [r["feature"] for r in spark_df.take(data_size)] - features = torch.tensor(features).reshape(data_size, 1) - labels = [r["label"] for r in spark_df.take(data_size)] - labels = torch.tensor(labels).reshape(data_size, 1) - ds = ray.data.from_spark(spark_df) - dataset = ds.to_torch(label_column="label", batch_size=3) - data_features, data_labels = next(dataset.__iter__()) - assert torch.equal(data_features, features) and torch.equal(data_labels, labels) - - def test_to_pandas(spark): df = spark.range(100) ds = ray.data.from_spark(df) diff --git a/python/ray/data/tests/test_torch.py b/python/ray/data/tests/test_torch.py index 5cae57075f8b..41e69ec2a293 100644 --- a/python/ray/data/tests/test_torch.py +++ b/python/ray/data/tests/test_torch.py @@ -4,270 +4,11 @@ import torch import ray -from ray.data.extensions.tensor_extension import TensorArray from ray.data.tests.conftest import * # noqa from ray.data.tests.util import extract_values from ray.tests.conftest import * # noqa -def test_to_torch_emits_deprecation_warning(ray_start_10_cpus_shared): - with pytest.warns(DeprecationWarning): - ray.data.range(1).to_torch() - - -def test_to_torch(ray_start_10_cpus_shared): - import torch - - df1 = pd.DataFrame( - {"one": [1, 2, 3], "two": [1.0, 2.0, 3.0], "label": [1.0, 2.0, 3.0]} - ) - df2 = pd.DataFrame( - {"one": [4, 5, 6], "two": [4.0, 5.0, 6.0], "label": [4.0, 5.0, 6.0]} - ) - df3 = pd.DataFrame({"one": [7, 8], "two": [7.0, 8.0], "label": [7.0, 8.0]}) - df = pd.concat([df1, df2, df3]) - ds = ray.data.from_pandas([df1, df2, df3]) - torchd = ds.to_torch(label_column="label", batch_size=3) - - num_epochs = 2 - for _ in range(num_epochs): - iterations = [] - for batch in iter(torchd): - iterations.append(torch.cat((batch[0], batch[1]), dim=1).numpy()) - combined_iterations = np.concatenate(iterations) - np.testing.assert_array_equal(np.sort(df.values), np.sort(combined_iterations)) - - -@pytest.mark.parametrize("input", ["single", "list", "dict"]) -@pytest.mark.parametrize("force_dtype", [False, True]) -@pytest.mark.parametrize("label_type", [None, "squeezed", "unsqueezed"]) -def test_to_torch_feature_columns( - ray_start_10_cpus_shared, input, force_dtype, label_type -): - import torch - - df1 = pd.DataFrame( - { - "one": [1, 2, 3], - "two": [1.0, 2.0, 3.0], - "three": [4.0, 5.0, 6.0], - "label": [1.0, 2.0, 3.0], - } - ) - df2 = pd.DataFrame( - { - "one": [4, 5, 6], - "two": [4.0, 5.0, 6.0], - "three": [7.0, 8.0, 9.0], - "label": [4.0, 5.0, 6.0], - } - ) - df3 = pd.DataFrame( - {"one": [7, 8], "two": [7.0, 8.0], "three": [10.0, 11.0], "label": [7.0, 8.0]} - ) - df = pd.concat([df1, df2, df3]).drop("three", axis=1) - ds = ray.data.from_pandas([df1, df2, df3]) - - feature_column_dtypes = None - label_column_dtype = None - if force_dtype: - label_column_dtype = torch.long - if input == "single": - feature_columns = ["one", "two"] - if force_dtype: - feature_column_dtypes = torch.long - elif input == "list": - feature_columns = [["one"], ["two"]] - if force_dtype: - feature_column_dtypes = [torch.long, torch.long] - elif input == "dict": - feature_columns = {"X1": ["one"], "X2": ["two"]} - if force_dtype: - feature_column_dtypes = {"X1": torch.long, "X2": torch.long} - - label_column = None if label_type is None else "label" - unsqueeze_label_tensor = label_type == "unsqueezed" - - torchd = ds.to_torch( - label_column=label_column, - feature_columns=feature_columns, - feature_column_dtypes=feature_column_dtypes, - label_column_dtype=label_column_dtype, - unsqueeze_label_tensor=unsqueeze_label_tensor, - batch_size=3, - ) - iterations = [] - - for batch in iter(torchd): - features, label = batch - - if input == "single": - assert isinstance(features, torch.Tensor) - if force_dtype: - assert features.dtype == torch.long - data = features - elif input == "list": - assert isinstance(features, list) - assert all(isinstance(item, torch.Tensor) for item in features) - if force_dtype: - assert all(item.dtype == torch.long for item in features) - data = torch.cat(tuple(features), dim=1) - elif input == "dict": - assert isinstance(features, dict) - assert all(isinstance(item, torch.Tensor) for item in features.values()) - if force_dtype: - assert all(item.dtype == torch.long for item in features.values()) - data = torch.cat(tuple(features.values()), dim=1) - - if not label_type: - assert label is None - else: - assert isinstance(label, torch.Tensor) - if force_dtype: - assert label.dtype == torch.long - if unsqueeze_label_tensor: - assert label.dim() == 2 - else: - assert label.dim() == 1 - label = label.view(-1, 1) - data = torch.cat((data, label), dim=1) - iterations.append(data.numpy()) - - combined_iterations = np.concatenate(iterations) - if not label_type: - df.drop("label", axis=1, inplace=True) - np.testing.assert_array_equal(df.values, combined_iterations) - - -def test_tensors_in_tables_to_torch(ray_start_10_cpus_shared): - outer_dim = 3 - inner_shape = (2, 2, 2) - shape = (outer_dim,) + inner_shape - num_items = np.prod(np.array(shape)) - arr = np.arange(num_items).reshape(shape) - df1 = pd.DataFrame( - {"one": TensorArray(arr), "two": TensorArray(arr + 1), "label": [1.0, 2.0, 3.0]} - ) - arr2 = np.arange(num_items, 2 * num_items).reshape(shape) - df2 = pd.DataFrame( - { - "one": TensorArray(arr2), - "two": TensorArray(arr2 + 1), - "label": [4.0, 5.0, 6.0], - } - ) - df = pd.concat([df1, df2]) - ds = ray.data.from_pandas([df1, df2]) - torchd = ds.to_torch( - label_column="label", batch_size=2, unsqueeze_label_tensor=False - ) - - num_epochs = 2 - for _ in range(num_epochs): - features, labels = [], [] - for batch in iter(torchd): - features.append(batch[0].numpy()) - labels.append(batch[1].numpy()) - features, labels = np.concatenate(features), np.concatenate(labels) - values = np.stack([df["one"].to_numpy(), df["two"].to_numpy()], axis=1) - np.testing.assert_array_equal(values, features) - np.testing.assert_array_equal(df["label"].to_numpy(), labels) - - -def test_tensors_in_tables_to_torch_mix(ray_start_10_cpus_shared): - outer_dim = 3 - inner_shape = (2, 2, 2) - shape = (outer_dim,) + inner_shape - num_items = np.prod(np.array(shape)) - arr = np.arange(num_items).reshape(shape) - df1 = pd.DataFrame( - { - "one": TensorArray(arr), - "two": [1, 2, 3], - "label": [1.0, 2.0, 3.0], - } - ) - arr2 = np.arange(num_items, 2 * num_items).reshape(shape) - df2 = pd.DataFrame( - { - "one": TensorArray(arr2), - "two": [4, 5, 6], - "label": [4.0, 5.0, 6.0], - } - ) - df = pd.concat([df1, df2]) - ds = ray.data.from_pandas([df1, df2]) - torchd = ds.to_torch( - label_column="label", - feature_columns=[["one"], ["two"]], - batch_size=2, - unsqueeze_label_tensor=False, - unsqueeze_feature_tensors=False, - ) - - num_epochs = 2 - for _ in range(num_epochs): - col1, col2, labels = [], [], [] - for batch in iter(torchd): - col1.append(batch[0][0].numpy()) - col2.append(batch[0][1].numpy()) - labels.append(batch[1].numpy()) - col1, col2 = np.concatenate(col1), np.concatenate(col2) - labels = np.concatenate(labels) - np.testing.assert_array_equal(col1, np.sort(df["one"].to_numpy())) - np.testing.assert_array_equal(col2, np.sort(df["two"].to_numpy())) - np.testing.assert_array_equal(labels, np.sort(df["label"].to_numpy())) - - -@pytest.mark.skip( - reason=( - "Waiting for Torch to support unsqueezing and concatenating nested tensors." - ) -) -def test_tensors_in_tables_to_torch_variable_shaped(ray_start_10_cpus_shared): - shapes = [(2, 2), (3, 3), (4, 4)] - cumsum_sizes = np.cumsum([0] + [np.prod(shape) for shape in shapes[:-1]]) - arrs1 = [ - np.arange(offset, offset + np.prod(shape)).reshape(shape) - for offset, shape in zip(cumsum_sizes, shapes) - ] - df1 = pd.DataFrame( - { - "one": TensorArray(arrs1), - "two": TensorArray([a + 1 for a in arrs1]), - "label": [1.0, 2.0, 3.0], - } - ) - base = cumsum_sizes[-1] - arrs2 = [ - np.arange(base + offset, base + offset + np.prod(shape)).reshape(shape) - for offset, shape in zip(cumsum_sizes, shapes) - ] - df2 = pd.DataFrame( - { - "one": TensorArray(arrs2), - "two": TensorArray([a + 1 for a in arrs2]), - "label": [4.0, 5.0, 6.0], - } - ) - df = pd.concat([df1, df2]) - ds = ray.data.from_pandas([df1, df2]) - torchd = ds.to_torch( - label_column="label", batch_size=2, unsqueeze_label_tensor=False - ) - - num_epochs = 2 - for _ in range(num_epochs): - features, labels = [], [] - for batch in iter(torchd): - features.append(batch[0].numpy()) - labels.append(batch[1].numpy()) - features, labels = np.concatenate(features), np.concatenate(labels) - values = np.stack([df["one"].to_numpy(), df["two"].to_numpy()], axis=1) - np.testing.assert_array_equal(values, features) - np.testing.assert_array_equal(df["label"].to_numpy(), labels) - - def test_iter_torch_batches(ray_start_10_cpus_shared): import torch From 4d458596168e1ea74b9c42f1a922e719c8efd6aa Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Mon, 8 Sep 2025 16:08:33 -0700 Subject: [PATCH 512/634] [core] disable test db for cpp tests (#56348) Disable RAYCI_TEST_DB for core cpp tests. What this really means is not tracking and skipping flaky cpp tests on PRs. Chat offline with @edoakes and we think cpp tests are not flaky so it makes sense to do this for cpp tests. Test: - CI Signed-off-by: Cuong Nguyen --- .buildkite/core.rayci.yml | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/.buildkite/core.rayci.yml b/.buildkite/core.rayci.yml index c1291858d276..71ed4084c0e2 100644 --- a/.buildkite/core.rayci.yml +++ b/.buildkite/core.rayci.yml @@ -291,19 +291,21 @@ steps: - "3.11" - "3.12" - "3.13" + - label: ":ray: core: cgroup tests" tags: core_cpp instance_type: medium commands: - - bazel run //ci/ray_ci:test_in_docker -- //:all //src/ray/common/cgroup2/tests/... core --build-type clang --cache-test-results + - RAYCI_DISABLE_TEST_DB=1 bazel run //ci/ray_ci:test_in_docker -- //:all //src/ray/common/cgroup2/tests/... core --build-type clang --cache-test-results - docker run --privileged -i --rm --volume /tmp/artifacts:/artifact-mount --shm-size=2.5gb "$${RAYCI_WORK_REPO}":"$${RAYCI_BUILD_ID}"-corebuild /bin/bash "./src/ray/common/cgroup2/integration_tests/sysfs_cgroup_driver_integration_test_entrypoint.sh" + - label: ":ray: core: cpp tests" tags: core_cpp instance_type: medium commands: - - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --except-tags=cgroup --build-type clang + - RAYCI_DISABLE_TEST_DB=1 bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --except-tags=cgroup --build-type clang --cache-test-results --parallelism-per-worker 2 # block on premerge and microcheck @@ -316,7 +318,7 @@ steps: tags: core_cpp instance_type: medium commands: - - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --except-tags=cgroup + - RAYCI_DISABLE_TEST_DB=1 bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --except-tags=cgroup --build-type asan-clang --cache-test-results --parallelism-per-worker 2 depends_on: - block-core-cpp-sanitizer-tests @@ -326,7 +328,7 @@ steps: tags: core_cpp instance_type: large commands: - - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core + - RAYCI_DISABLE_TEST_DB=1 bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --build-type ubsan --except-tags no_ubsan,cgroup --cache-test-results --parallelism-per-worker 2 depends_on: @@ -337,27 +339,13 @@ steps: tags: core_cpp instance_type: medium commands: - - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core + - RAYCI_DISABLE_TEST_DB=1 bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --build-type tsan-clang --except-tags no_tsan,cgroup --cache-test-results --parallelism-per-worker 2 depends_on: - block-core-cpp-sanitizer-tests - corebuild - - label: ":ray: core: flaky cpp tests" - key: core_flaky_cpp_tests - tags: - - python - - flaky - - skip-on-premerge - instance_type: large - soft_fail: true - commands: - - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core - --run-flaky-tests --build-type clang - depends_on: - - corebuild - - label: ":ray: core: flaky tests" key: core_flaky_tests tags: From ed8ccd62737f66e9542128aedc48517388c0959f Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Mon, 8 Sep 2025 16:15:07 -0700 Subject: [PATCH 513/634] [Data] Remove redundant `ignore_missing_paths` and partitioning tests (#56341) ## Why are these changes needed? ## Related issue number This PR removes redundant tests for `ignore_missing_paths` and partitioning in format-specific datasources. These behaviors are implemented by the `FileBasedDatasource` base classes, so the subclass tests add little value while increasing runtime and complexity. ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Balaji Veeramani --- python/ray/data/tests/conftest.py | 60 ---- python/ray/data/tests/test_binary.py | 94 ------ python/ray/data/tests/test_csv.py | 309 ------------------ .../data/tests/test_file_based_datasource.py | 155 ++++++++- python/ray/data/tests/test_image.py | 44 --- python/ray/data/tests/test_json.py | 114 ------- python/ray/data/tests/test_numpy.py | 95 ------ python/ray/data/tests/test_text.py | 98 +----- python/ray/data/tests/test_tfrecords.py | 30 -- 9 files changed, 154 insertions(+), 845 deletions(-) diff --git a/python/ray/data/tests/conftest.py b/python/ray/data/tests/conftest.py index 53145fa06e98..5a14253918bb 100644 --- a/python/ray/data/tests/conftest.py +++ b/python/ray/data/tests/conftest.py @@ -16,7 +16,6 @@ from ray._private.internal_api import get_memory_info_reply, get_state_from_address from ray.air.constants import TENSOR_COLUMN_NAME from ray.air.util.tensor_extensions.arrow import ArrowTensorArray -from ray.data import Schema from ray.data.block import BlockExecStats, BlockMetadata from ray.data.context import DEFAULT_TARGET_MAX_BLOCK_SIZE, DataContext, ShuffleStrategy from ray.data.tests.mock_server import * # noqa @@ -213,65 +212,6 @@ def _write_partitioned_df( yield _write_partitioned_df -@pytest.fixture(scope="function") -def write_base_partitioned_df(base_partitioned_df, write_partitioned_df): - def _write_base_partitioned_df( - partition_keys, - partition_path_encoder, - file_writer_fn, - ): - write_partitioned_df( - base_partitioned_df, - partition_keys, - partition_path_encoder, - file_writer_fn, - ) - - yield _write_base_partitioned_df - - -@pytest.fixture(scope="function") -def assert_base_partitioned_ds(): - def _assert_base_partitioned_ds( - ds, - count=6, - num_input_files=2, - num_rows=6, - schema=Schema(pa.schema([("one", pa.int64()), ("two", pa.string())])), - sorted_values=None, - ds_take_transform_fn=None, - sorted_values_transform_fn=None, - ): - if ds_take_transform_fn is None: - ds_take_transform_fn = lambda taken: [ # noqa: E731 - [s["one"], s["two"]] for s in taken - ] - - if sorted_values_transform_fn is None: - sorted_values_transform_fn = ( # noqa: E731 - lambda sorted_values: sorted_values - ) - - if sorted_values is None: - sorted_values = [[1, "a"], [1, "b"], [1, "c"], [3, "e"], [3, "f"], [3, "g"]] - # Test metadata ops. - assert not ds._plan.has_started_execution - assert ds.count() == count, f"{ds.count()} != {count}" - assert ds.size_bytes() > 0, f"{ds.size_bytes()} <= 0" - assert ds.schema() == schema - actual_input_files = ds.input_files() - assert len(actual_input_files) == num_input_files, actual_input_files - - # Force a data read. - values = ds_take_transform_fn(ds.take_all()) - actual_sorted_values = sorted_values_transform_fn(sorted(values)) - assert ( - actual_sorted_values == sorted_values - ), f"{actual_sorted_values} != {sorted_values}" - - yield _assert_base_partitioned_ds - - @pytest.fixture def restore_data_context(request): """Restore any DataContext changes after the test runs""" diff --git a/python/ray/data/tests/test_binary.py b/python/ray/data/tests/test_binary.py index f1735da802f7..18e07200306a 100644 --- a/python/ray/data/tests/test_binary.py +++ b/python/ray/data/tests/test_binary.py @@ -1,45 +1,22 @@ import os from io import BytesIO -import pandas as pd import pyarrow as pa import pytest import requests import snappy import ray -from ray.data import Schema from ray.data.datasource import ( BaseFileMetadataProvider, FastFileMetadataProvider, - Partitioning, - PartitionStyle, - PathPartitionFilter, ) from ray.data.tests.conftest import * # noqa from ray.data.tests.mock_http_server import * # noqa -from ray.data.tests.test_partitioning import PathPartitionEncoder from ray.data.tests.util import extract_values, gen_bin_files from ray.tests.conftest import * # noqa -def test_read_binary_files_partitioning(ray_start_regular_shared, tmp_path): - os.mkdir(os.path.join(tmp_path, "country=us")) - path = os.path.join(tmp_path, "country=us", "file.bin") - with open(path, "wb") as f: - f.write(b"foo") - - ds = ray.data.read_binary_files(path, partitioning=Partitioning("hive")) - - assert ds.take() == [{"bytes": b"foo", "country": "us"}] - - ds = ray.data.read_binary_files( - path, include_paths=True, partitioning=Partitioning("hive") - ) - - assert ds.take() == [{"bytes": b"foo", "path": path, "country": "us"}] - - def test_read_binary_files(ray_start_regular_shared): with gen_bin_files(10) as (_, paths): ds = ray.data.read_binary_files(paths) @@ -52,24 +29,6 @@ def test_read_binary_files(ray_start_regular_shared): assert "bytes" in str(ds), ds -@pytest.mark.parametrize("ignore_missing_paths", [True, False]) -def test_read_binary_files_ignore_missing_paths( - ray_start_regular_shared, ignore_missing_paths -): - with gen_bin_files(1) as (_, paths): - paths = paths + ["missing_file"] - if ignore_missing_paths: - ds = ray.data.read_binary_files( - paths, ignore_missing_paths=ignore_missing_paths - ) - assert ds.input_files() == [paths[0]] - else: - with pytest.raises(FileNotFoundError): - ds = ray.data.read_binary_files( - paths, ignore_missing_paths=ignore_missing_paths - ).materialize() - - def test_read_binary_files_with_fs(ray_start_regular_shared): with gen_bin_files(10) as (tempdir, paths): # All the paths are absolute, so we want the root file system. @@ -142,59 +101,6 @@ def test_read_binary_meta_provider( ) -@pytest.mark.parametrize("style", [PartitionStyle.HIVE, PartitionStyle.DIRECTORY]) -def test_read_binary_snappy_partitioned_with_filter( - style, - ray_start_regular_shared, - tmp_path, - write_base_partitioned_df, - assert_base_partitioned_ds, -): - def df_to_binary(dataframe, path, **kwargs): - with open(path, "wb") as f: - df_string = dataframe.to_string(index=False, header=False, **kwargs) - byte_str = df_string.encode() - bytes = BytesIO(byte_str) - snappy.stream_compress(bytes, f) - - partition_keys = ["one"] - - def skip_unpartitioned(kv_dict): - return bool(kv_dict) - - base_dir = os.path.join(tmp_path, style.value) - partition_path_encoder = PathPartitionEncoder.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - ) - write_base_partitioned_df( - partition_keys, - partition_path_encoder, - df_to_binary, - ) - df_to_binary(pd.DataFrame({"1": [1]}), os.path.join(base_dir, "test.snappy")) - partition_path_filter = PathPartitionFilter.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - filter_fn=skip_unpartitioned, - ) - ds = ray.data.read_binary_files( - base_dir, - partition_filter=partition_path_filter, - arrow_open_stream_args=dict(compression="snappy"), - ) - assert_base_partitioned_ds( - ds, - count=2, - num_rows=2, - schema=Schema(pa.schema([("bytes", pa.binary())])), - sorted_values=[b"1 a\n1 b\n1 c", b"3 e\n3 f\n3 g"], - ds_take_transform_fn=lambda t: extract_values("bytes", t), - ) - - if __name__ == "__main__": import sys diff --git a/python/ray/data/tests/test_csv.py b/python/ray/data/tests/test_csv.py index 93ef35261e8f..9ba6e63bb60b 100644 --- a/python/ray/data/tests/test_csv.py +++ b/python/ray/data/tests/test_csv.py @@ -1,7 +1,5 @@ -import itertools import os import shutil -from functools import partial import pandas as pd import pyarrow as pa @@ -17,8 +15,6 @@ from ray.data.datasource import ( BaseFileMetadataProvider, FastFileMetadataProvider, - PartitionStyle, - PathPartitionFilter, ) from ray.data.datasource.file_based_datasource import ( FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD, @@ -26,7 +22,6 @@ from ray.data.datasource.path_util import _unwrap_protocol from ray.data.tests.conftest import * # noqa from ray.data.tests.mock_http_server import * # noqa -from ray.data.tests.test_partitioning import PathPartitionEncoder from ray.tests.conftest import * # noqa @@ -34,21 +29,6 @@ def df_to_csv(dataframe, path, **kwargs): dataframe.to_csv(path, **kwargs) -def test_csv_read_partitioning(ray_start_regular_shared, tmp_path): - path = os.path.join(tmp_path, "country=us", "file.csv") - os.mkdir(os.path.dirname(path)) - df = pd.DataFrame({"numbers": [1, 2, 3], "letters": ["a", "b", "c"]}) - df.to_csv(path, index=False) - - ds = ray.data.read_csv(path) - - assert ds.take() == [ - {"numbers": 1, "letters": "a", "country": "us"}, - {"numbers": 2, "letters": "b", "country": "us"}, - {"numbers": 3, "letters": "c", "country": "us"}, - ] - - @pytest.mark.parametrize( "fs,data_path,endpoint_url", [ @@ -303,75 +283,6 @@ def test_csv_read_many_files_basic( pd.testing.assert_frame_equal(df, dsdf) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) -def test_csv_read_many_files_partitioned( - ray_start_regular_shared, - fs, - data_path, - endpoint_url, - write_partitioned_df, - assert_base_partitioned_ds, -): - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) - - partition_keys = ["one"] - partition_path_encoder = PathPartitionEncoder.of( - base_dir=data_path, - field_names=partition_keys, - filesystem=fs, - ) - paths = [] - dfs = [] - num_dfs = FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD - num_rows = 6 * num_dfs - num_files = 2 * num_dfs - for i in range(num_dfs): - df = pd.DataFrame( - {"one": [1, 1, 1, 3, 3, 3], "two": list(range(6 * i, 6 * (i + 1)))} - ) - df_paths = write_partitioned_df( - df, - partition_keys, - partition_path_encoder, - partial(df_to_csv, storage_options=storage_options, index=False), - file_name_suffix=i, - ) - dfs.append(df) - paths.extend(df_paths) - - ds = ray.data.read_csv( - paths, - filesystem=fs, - partitioning=partition_path_encoder.scheme, - override_num_blocks=num_files, - ) - - assert_base_partitioned_ds( - ds, - count=num_rows, - num_input_files=num_files, - schema=Schema(pa.schema([("one", pa.int64()), ("two", pa.int64())])), - sorted_values=sorted( - itertools.chain.from_iterable( - list( - map(list, zip([1, 1, 1, 3, 3, 3], list(range(6 * i, 6 * (i + 1))))) - ) - for i in range(num_dfs) - ) - ), - ) - - @pytest.mark.parametrize( "fs,data_path,endpoint_url", [ @@ -417,226 +328,6 @@ def test_csv_read_many_files_diff_dirs( pd.testing.assert_frame_equal(df, dsdf) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ( - lazy_fixture("s3_fs_with_anonymous_crendential"), - lazy_fixture("s3_path_with_anonymous_crendential"), - lazy_fixture("s3_server"), - ), - ], -) -def test_csv_read_partitioned_hive_implicit( - ray_start_regular_shared, - fs, - data_path, - endpoint_url, - write_base_partitioned_df, - assert_base_partitioned_ds, -): - storage_options = ( - {} - if endpoint_url is None - else dict(client_kwargs=dict(endpoint_url=endpoint_url)) - ) - partition_keys = ["one"] - partition_path_encoder = PathPartitionEncoder.of( - base_dir=data_path, - field_names=partition_keys, - filesystem=fs, - ) - write_base_partitioned_df( - partition_keys, - partition_path_encoder, - partial(df_to_csv, storage_options=storage_options, index=False), - ) - ds = ray.data.read_csv( - data_path, - partition_filter=PathPartitionFilter.of(None, filesystem=fs), - filesystem=fs, - ) - assert_base_partitioned_ds(ds) - - -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ( - lazy_fixture("s3_fs_with_anonymous_crendential"), - lazy_fixture("s3_path_with_anonymous_crendential"), - lazy_fixture("s3_server"), - ), - ], -) -def test_csv_read_partitioned_styles_explicit( - ray_start_regular_shared, - fs, - data_path, - endpoint_url, - write_base_partitioned_df, - assert_base_partitioned_ds, -): - storage_options = ( - {} - if endpoint_url is None - else dict(client_kwargs=dict(endpoint_url=endpoint_url)) - ) - partition_keys = ["one"] - for style in [PartitionStyle.HIVE, PartitionStyle.DIRECTORY]: - base_dir = os.path.join(data_path, style.value) - partition_path_encoder = PathPartitionEncoder.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - filesystem=fs, - ) - write_base_partitioned_df( - partition_keys, - partition_path_encoder, - partial(df_to_csv, storage_options=storage_options, index=False), - ) - partition_path_filter = PathPartitionFilter.of( - None, - style=style, - base_dir=base_dir, - field_names=partition_keys, - filesystem=fs, - ) - ds = ray.data.read_csv( - base_dir, - partition_filter=partition_path_filter, - filesystem=fs, - ) - assert_base_partitioned_ds(ds) - - -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) -@pytest.mark.parametrize("style", [PartitionStyle.HIVE, PartitionStyle.DIRECTORY]) -def test_csv_read_partitioned_with_filter( - style, - ray_start_regular_shared, - fs, - data_path, - endpoint_url, - write_base_partitioned_df, - assert_base_partitioned_ds, -): - storage_options = ( - {} - if endpoint_url is None - else dict(client_kwargs=dict(endpoint_url=endpoint_url)) - ) - partition_keys = ["one"] - file_writer_fn = partial(df_to_csv, storage_options=storage_options, index=False) - - def skip_unpartitioned(kv_dict): - return bool(kv_dict) - - base_dir = os.path.join(data_path, style.value) - partition_path_encoder = PathPartitionEncoder.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - filesystem=fs, - ) - write_base_partitioned_df( - partition_keys, - partition_path_encoder, - file_writer_fn, - ) - file_writer_fn(pd.DataFrame({"1": [1]}), os.path.join(base_dir, "test.csv")) - partition_path_filter = PathPartitionFilter.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - filesystem=fs, - filter_fn=skip_unpartitioned, - ) - ds = ray.data.read_csv( - base_dir, - partition_filter=partition_path_filter, - filesystem=fs, - ) - assert_base_partitioned_ds(ds) - - -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) -@pytest.mark.parametrize("style", [PartitionStyle.HIVE, PartitionStyle.DIRECTORY]) -def test_csv_read_partitioned_with_filter_multikey( - style, - ray_start_regular_shared, - fs, - data_path, - endpoint_url, - write_base_partitioned_df, - assert_base_partitioned_ds, -): - storage_options = ( - {} - if endpoint_url is None - else dict(client_kwargs=dict(endpoint_url=endpoint_url)) - ) - partition_keys = ["one", "two"] - file_writer_fn = partial(df_to_csv, storage_options=storage_options, index=False) - - def keep_expected_partitions(kv_dict): - keep = bool(kv_dict) and ( - (kv_dict["one"] == "1" and kv_dict["two"] in {"a", "b", "c"}) - or (kv_dict["one"] == "3" and kv_dict["two"] in {"e", "f", "g"}) - ) - return keep - - base_dir = os.path.join(data_path, style.value) - partition_path_encoder = PathPartitionEncoder.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - filesystem=fs, - ) - write_base_partitioned_df( - partition_keys, - partition_path_encoder, - file_writer_fn, - ) - df = pd.DataFrame({"1": [1]}) - file_writer_fn(df, os.path.join(data_path, "test0.csv")) - partition_path_filter = PathPartitionFilter.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - filesystem=fs, - filter_fn=keep_expected_partitions, - ) - ds = ray.data.read_csv( - data_path, - partition_filter=partition_path_filter, - filesystem=fs, - override_num_blocks=6, - ) - assert_base_partitioned_ds(ds, num_input_files=6) - - def test_csv_write( ray_start_regular_shared, tmp_path, target_max_block_size_infinite_or_default ): diff --git a/python/ray/data/tests/test_file_based_datasource.py b/python/ray/data/tests/test_file_based_datasource.py index a73d72fbee5b..5d2df56f3cf2 100644 --- a/python/ray/data/tests/test_file_based_datasource.py +++ b/python/ray/data/tests/test_file_based_datasource.py @@ -1,13 +1,19 @@ import os -from typing import Iterator +from typing import Any, Dict, Iterator, List import pyarrow import pytest import ray from ray.data._internal.delegating_block_builder import DelegatingBlockBuilder -from ray.data.block import Block +from ray.data.block import Block, BlockAccessor +from ray.data.datasource.datasource import ReadTask from ray.data.datasource.file_based_datasource import FileBasedDatasource +from ray.data.datasource.partitioning import ( + Partitioning, + PartitionStyle, + PathPartitionFilter, +) class MockFileBasedDatasource(FileBasedDatasource): @@ -17,6 +23,151 @@ def _read_stream(self, f: "pyarrow.NativeFile", path: str) -> Iterator[Block]: yield builder.build() +def execute_read_tasks(tasks: List[ReadTask]) -> List[Dict[str, Any]]: + """Execute the read tasks and return the resulting rows. + + The motivation for this utility function is so that we can test datasources without + scheduling Ray tasks. + """ + builder = DelegatingBlockBuilder() + for task in tasks: + for block in task(): + builder.add_block(block) + block = builder.build() + + block_accessor = BlockAccessor.for_block(block) + rows = list(block_accessor.iter_rows(public_row_format=True)) + + return rows + + +def test_partitioning_hive(ray_start_regular_shared, tmp_path): + path = os.path.join(tmp_path, "country=us") + os.mkdir(path) + with open(os.path.join(path, "file.txt"), "wb") as file: + file.write(b"") + + datasource = MockFileBasedDatasource(tmp_path, partitioning=Partitioning("hive")) + + tasks = datasource.get_read_tasks(1) + rows = execute_read_tasks(tasks) + + assert rows == [{"data": b"", "country": "us"}] + + +def test_partition_filter_hive(ray_start_regular_shared, tmp_path): + for country in ["us", "jp"]: + path = os.path.join(tmp_path, f"country={country}") + os.mkdir(path) + with open(os.path.join(path, "file.txt"), "wb") as file: + file.write(b"") + + filter = PathPartitionFilter.of( + style=PartitionStyle.HIVE, + filter_fn=lambda partitions: partitions["country"] == "us", + ) + datasource = MockFileBasedDatasource( + tmp_path, partitioning=Partitioning("hive"), partition_filter=filter + ) + + tasks = datasource.get_read_tasks(1) + rows = execute_read_tasks(tasks) + + assert rows == [{"data": b"", "country": "us"}] + + +def test_partitioning_dir(ray_start_regular_shared, tmp_path): + path = os.path.join(tmp_path, "us") + os.mkdir(path) + with open(os.path.join(path, "file.txt"), "wb") as file: + file.write(b"") + + datasource = MockFileBasedDatasource( + tmp_path, + partitioning=Partitioning("dir", field_names=["country"], base_dir=tmp_path), + ) + + tasks = datasource.get_read_tasks(1) + rows = execute_read_tasks(tasks) + + assert rows == [{"data": b"", "country": "us"}] + + +def test_partition_filter_dir(ray_start_regular_shared, tmp_path): + for country in ["us", "jp"]: + path = os.path.join(tmp_path, country) + os.mkdir(path) + with open(os.path.join(path, "file.txt"), "wb") as file: + file.write(b"") + + filter = PathPartitionFilter.of( + style=PartitionStyle.DIRECTORY, + base_dir=tmp_path, + field_names=["country"], + filter_fn=lambda partitions: partitions["country"] == "us", + ) + partitioning = Partitioning("dir", field_names=["country"], base_dir=tmp_path) + datasource = MockFileBasedDatasource( + tmp_path, partitioning=partitioning, partition_filter=filter + ) + + tasks = datasource.get_read_tasks(1) + rows = execute_read_tasks(tasks) + + assert rows == [{"data": b"", "country": "us"}] + + +def test_partitioning_raises_on_mismatch(ray_start_regular_shared, tmp_path): + """Test when the partition key already exists in the data.""" + + class StubDatasource(FileBasedDatasource): + def _read_stream(self, f: "pyarrow.NativeFile", path: str) -> Iterator[Block]: + builder = DelegatingBlockBuilder() + builder.add({"country": f.readall()}) + yield builder.build() + + path = os.path.join(tmp_path, "country=us") + os.mkdir(path) + with open(os.path.join(path, "file.txt"), "wb") as file: + file.write(b"jp") + + datasource = StubDatasource(tmp_path, partitioning=Partitioning("hive")) + + # The data is `jp`, but the path contains `us`. Since the values are different, + # the datasource should raise a ValueError. + with pytest.raises(ValueError): + tasks = datasource.get_read_tasks(1) + execute_read_tasks(tasks) + + +def test_ignore_missing_paths_true(ray_start_regular_shared, tmp_path): + path = os.path.join(tmp_path, "file.txt") + with open(path, "wb") as file: + file.write(b"") + + datasource = MockFileBasedDatasource( + [path, "missing.txt"], ignore_missing_paths=True + ) + + tasks = datasource.get_read_tasks(1) + rows = execute_read_tasks(tasks) + + assert rows == [{"data": b""}] + + +def test_ignore_missing_paths_false(ray_start_regular_shared, tmp_path): + path = os.path.join(tmp_path, "file.txt") + with open(path, "wb") as file: + file.write(b"") + + with pytest.raises(FileNotFoundError): + datasource = MockFileBasedDatasource( + [path, "missing.txt"], ignore_missing_paths=False + ) + tasks = datasource.get_read_tasks(1) + execute_read_tasks(tasks) + + def test_local_paths(ray_start_regular_shared, tmp_path): path = os.path.join(tmp_path, "test.txt") with open(path, "w"): diff --git a/python/ray/data/tests/test_image.py b/python/ray/data/tests/test_image.py index a7e72b10081c..535c3d6ab64a 100644 --- a/python/ray/data/tests/test_image.py +++ b/python/ray/data/tests/test_image.py @@ -3,7 +3,6 @@ from typing import Dict import numpy as np -import pyarrow as pa import pytest from fsspec.implementations.local import LocalFileSystem from PIL import Image @@ -16,7 +15,6 @@ ImageDatasource, ImageFileMetadataProvider, ) -from ray.data.datasource import Partitioning from ray.data.datasource.file_meta_provider import FastFileMetadataProvider from ray.data.tests.conftest import * # noqa from ray.data.tests.mock_http_server import * # noqa @@ -71,27 +69,6 @@ def test_file_metadata_provider(self, ray_start_regular_shared): ) assert ds.count() == 3 - @pytest.mark.parametrize("ignore_missing_paths", [True, False]) - def test_ignore_missing_paths(self, ray_start_regular_shared, ignore_missing_paths): - paths = [ - "example://image-datasets/simple/image1.jpg", - "example://missing.jpg", - "example://image-datasets/missing/", - ] - - if ignore_missing_paths: - ds = ray.data.read_images(paths, ignore_missing_paths=ignore_missing_paths) - # example:// directive redirects to /ray/python/ray/data/examples/data - assert len(ds.input_files()) == 1 and ds.input_files()[0].endswith( - "ray/data/examples/data/image-datasets/simple/image1.jpg", - ) - else: - with pytest.raises(FileNotFoundError): - ds = ray.data.read_images( - paths, ignore_missing_paths=ignore_missing_paths - ) - ds.materialize() - def test_filtering(self, ray_start_regular_shared): # "different-extensions" contains three images and two non-images. ds = ray.data.read_images("example://image-datasets/different-extensions") @@ -130,27 +107,6 @@ def test_mode( ds = ray.data.read_images("example://image-datasets/different-modes", mode=mode) assert all([record["image"].shape == expected_shape for record in ds.take()]) - def test_partitioning( - self, ray_start_regular_shared, enable_automatic_tensor_extension_cast - ): - root = "example://image-datasets/dir-partitioned" - partitioning = Partitioning("dir", base_dir=root, field_names=["label"]) - - ds = ray.data.read_images(root, partitioning=partitioning) - - assert ds.schema().names == ["image", "label"] - - image_type, label_type = ds.schema().types - assert isinstance(image_type, get_arrow_extension_fixed_shape_tensor_types()) - assert pa.types.is_string(label_type) - - df = ds.to_pandas() - assert sorted(df["label"]) == ["cat", "cat", "dog"] - if enable_automatic_tensor_extension_cast: - assert all(tensor.shape == (32, 32, 3) for tensor in df["image"]) - else: - assert all(tensor.numpy_shape == (32, 32, 3) for tensor in df["image"]) - def test_random_shuffle(self, ray_start_regular_shared, restore_data_context): # NOTE: set preserve_order to True to allow consistent output behavior. context = ray.data.DataContext.get_current() diff --git a/python/ray/data/tests/test_json.py b/python/ray/data/tests/test_json.py index 7f5364a0f2b7..a7336f04ce51 100644 --- a/python/ray/data/tests/test_json.py +++ b/python/ray/data/tests/test_json.py @@ -2,7 +2,6 @@ import json import os import shutil -from functools import partial import pandas as pd import pyarrow as pa @@ -20,39 +19,18 @@ from ray.data.datasource import ( BaseFileMetadataProvider, FastFileMetadataProvider, - PartitionStyle, - PathPartitionFilter, ) from ray.data.datasource.file_based_datasource import ( FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD, ) from ray.data.datasource.path_util import _unwrap_protocol from ray.data.tests.conftest import * # noqa -from ray.data.tests.test_partitioning import PathPartitionEncoder from ray.tests.conftest import * # noqa # Set the test timeout to 6 minutes pytestmark = pytest.mark.timeout(360) -def test_json_read_partitioning( - ray_start_regular_shared, tmp_path, target_max_block_size_infinite_or_default -): - path = os.path.join(tmp_path, "country=us") - os.mkdir(path) - with open(os.path.join(path, "file1.json"), "w") as file: - json.dump({"number": 0, "string": "foo"}, file) - with open(os.path.join(path, "file2.json"), "w") as file: - json.dump({"number": 1, "string": "bar"}, file) - - ds = ray.data.read_json(path) - - assert sorted(ds.take(), key=lambda row: row["number"]) == [ - {"number": 0, "string": "foo", "country": "us"}, - {"number": 1, "string": "bar", "country": "us"}, - ] - - @pytest.mark.parametrize( "fs,data_path,endpoint_url", [ @@ -214,31 +192,6 @@ def test_json_read( fs.delete_dir(_unwrap_protocol(path)) -@pytest.mark.parametrize("ignore_missing_paths", [True, False]) -def test_read_json_ignore_missing_paths( - ray_start_regular_shared, - local_path, - ignore_missing_paths, - target_max_block_size_infinite_or_default, -): - df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - path1 = os.path.join(local_path, "test1.json") - df1.to_json(path1, orient="records", lines=True) - - paths = [ - path1, - "missing.json", - ] - - if ignore_missing_paths: - ds = ray.data.read_json(paths, ignore_missing_paths=ignore_missing_paths) - assert ds.input_files() == [path1] - else: - with pytest.raises(FileNotFoundError): - ds = ray.data.read_json(paths, ignore_missing_paths=ignore_missing_paths) - ds.materialize() - - def test_zipped_json_read( ray_start_regular_shared, tmp_path, target_max_block_size_infinite_or_default ): @@ -427,73 +380,6 @@ def test_json_read_with_parse_options( assert ds.schema() == Schema(pa.schema([("two", pa.string())])) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) -@pytest.mark.parametrize("style", [PartitionStyle.HIVE, PartitionStyle.DIRECTORY]) -def test_json_read_partitioned_with_filter( - style, - ray_start_regular_shared, - fs, - data_path, - endpoint_url, - write_base_partitioned_df, - assert_base_partitioned_ds, - target_max_block_size_infinite_or_default, -): - def df_to_json(dataframe, path, **kwargs): - dataframe.to_json(path, **kwargs) - - storage_options = ( - {} - if endpoint_url is None - else dict(client_kwargs=dict(endpoint_url=endpoint_url)) - ) - file_writer_fn = partial( - df_to_json, - orient="records", - lines=True, - storage_options=storage_options, - ) - partition_keys = ["one"] - - def skip_unpartitioned(kv_dict): - return bool(kv_dict) - - base_dir = os.path.join(data_path, style.value) - partition_path_encoder = PathPartitionEncoder.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - filesystem=fs, - ) - write_base_partitioned_df( - partition_keys, - partition_path_encoder, - file_writer_fn, - ) - file_writer_fn(pd.DataFrame({"1": [1]}), os.path.join(base_dir, "test.json")) - partition_path_filter = PathPartitionFilter.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - filter_fn=skip_unpartitioned, - filesystem=fs, - ) - ds = ray.data.read_json( - base_dir, - partition_filter=partition_path_filter, - file_extensions=None, - filesystem=fs, - ) - assert_base_partitioned_ds(ds) - - @pytest.mark.parametrize("override_num_blocks", [None, 1, 3]) def test_jsonl_lists( ray_start_regular_shared, diff --git a/python/ray/data/tests/test_numpy.py b/python/ray/data/tests/test_numpy.py index a4240c18041a..ee03ddb4315b 100644 --- a/python/ray/data/tests/test_numpy.py +++ b/python/ray/data/tests/test_numpy.py @@ -12,14 +12,10 @@ from ray.data.datasource import ( BaseFileMetadataProvider, FastFileMetadataProvider, - Partitioning, - PartitionStyle, - PathPartitionFilter, ) from ray.data.extensions.tensor_extension import ArrowTensorType from ray.data.tests.conftest import * # noqa from ray.data.tests.mock_http_server import * # noqa -from ray.data.tests.test_partitioning import PathPartitionEncoder from ray.data.tests.util import extract_values from ray.tests.conftest import * # noqa @@ -32,17 +28,6 @@ def _get_tensor_type(): ) -def test_numpy_read_partitioning(ray_start_regular_shared, tmp_path): - path = os.path.join(tmp_path, "country=us", "data.npy") - os.mkdir(os.path.dirname(path)) - np.save(path, np.arange(4).reshape([2, 2])) - - ds = ray.data.read_numpy(path, partitioning=Partitioning("hive")) - - assert ds.schema().names == ["data", "country"] - assert [r["country"] for r in ds.take()] == ["us", "us"] - - @pytest.mark.parametrize("from_ref", [False, True]) def test_from_numpy(ray_start_regular_shared, from_ref): arr1 = np.expand_dims(np.arange(0, 4), axis=1) @@ -158,28 +143,6 @@ def test_numpy_read_x(ray_start_regular_shared, tmp_path): assert [v["data"].item() for v in ds.take(2)] == [0, 1] -@pytest.mark.parametrize("ignore_missing_paths", [True, False]) -def test_numpy_read_ignore_missing_paths( - ray_start_regular_shared, tmp_path, ignore_missing_paths -): - path = os.path.join(tmp_path, "test_np_dir") - os.mkdir(path) - np.save(os.path.join(path, "test.npy"), np.expand_dims(np.arange(0, 10), 1)) - - paths = [ - os.path.join(path, "test.npy"), - "missing.npy", - ] - - if ignore_missing_paths: - ds = ray.data.read_numpy(paths, ignore_missing_paths=ignore_missing_paths) - assert ds.input_files() == [paths[0]] - else: - with pytest.raises(FileNotFoundError): - ds = ray.data.read_numpy(paths, ignore_missing_paths=ignore_missing_paths) - ds.materialize() - - def test_numpy_read_meta_provider(ray_start_regular_shared, tmp_path): tensor_type = _get_tensor_type() @@ -203,64 +166,6 @@ def test_numpy_read_meta_provider(ray_start_regular_shared, tmp_path): ) -@pytest.mark.parametrize("style", [PartitionStyle.HIVE, PartitionStyle.DIRECTORY]) -def test_numpy_read_partitioned_with_filter( - style, - ray_start_regular_shared, - tmp_path, - write_partitioned_df, - assert_base_partitioned_ds, -): - tensor_type = _get_tensor_type() - - def df_to_np(dataframe, path, **kwargs): - np.save(path, dataframe.to_numpy(dtype=np.dtype(np.int8)), **kwargs) - - df = pd.DataFrame({"one": [1, 1, 1, 3, 3, 3], "two": [0, 1, 2, 3, 4, 5]}) - partition_keys = ["one"] - - def skip_unpartitioned(kv_dict): - return bool(kv_dict) - - base_dir = os.path.join(tmp_path, style.value) - partition_path_encoder = PathPartitionEncoder.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - ) - write_partitioned_df( - df, - partition_keys, - partition_path_encoder, - df_to_np, - ) - df_to_np(df, os.path.join(base_dir, "test.npy")) - partition_path_filter = PathPartitionFilter.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - filter_fn=skip_unpartitioned, - ) - ds = ray.data.read_numpy(base_dir, partition_filter=partition_path_filter) - - def sorted_values_transform_fn(sorted_values): - # HACK: `assert_base_partitioned_ds` doesn't properly sort the values. This is a - # hack to make the test pass. - # TODO(@bveeramani): Clean this up. - actually_sorted_values = sorted(sorted_values[0], key=lambda item: tuple(item)) - return str([actually_sorted_values]) - - vals = [[1, 0], [1, 1], [1, 2], [3, 3], [3, 4], [3, 5]] - val_str = "".join(f"array({v}, dtype=int8), " for v in vals)[:-2] - assert_base_partitioned_ds( - ds, - schema=Schema(pa.schema([("data", tensor_type((2,), pa.int8()))])), - sorted_values=f"[[{val_str}]]", - ds_take_transform_fn=lambda taken: [extract_values("data", taken)], - sorted_values_transform_fn=sorted_values_transform_fn, - ) - - def test_numpy_write(ray_start_regular_shared, tmp_path): ds = ray.data.range_tensor(1) diff --git a/python/ray/data/tests/test_text.py b/python/ray/data/tests/test_text.py index af7f142085db..3a56fc513dda 100644 --- a/python/ray/data/tests/test_text.py +++ b/python/ray/data/tests/test_text.py @@ -1,25 +1,17 @@ import os -import pandas as pd -import pyarrow as pa import pytest import ray -from ray.data import Schema from ray.data._internal.execution.interfaces.ref_bundle import ( _ref_bundles_iterator_to_block_refs_list, ) from ray.data.datasource import ( BaseFileMetadataProvider, FastFileMetadataProvider, - Partitioning, - PartitionStyle, - PathPartitionFilter, ) from ray.data.tests.conftest import * # noqa from ray.data.tests.mock_http_server import * # noqa -from ray.data.tests.test_partitioning import PathPartitionEncoder -from ray.data.tests.util import Counter from ray.tests.conftest import * # noqa @@ -27,20 +19,6 @@ def _to_lines(rows): return [row["text"] for row in rows] -def test_read_text_partitioning(ray_start_regular_shared, tmp_path): - path = os.path.join(tmp_path, "country=us") - os.mkdir(path) - with open(os.path.join(path, "file.txt"), "w") as f: - f.write("foo\nbar\nbaz") - - ds = ray.data.read_text(path, partitioning=Partitioning("hive")) - - df = ds.to_pandas() - assert list(df.columns) == ["text", "country"] - assert sorted(df["text"]) == ["bar", "baz", "foo"] - assert list(df["country"]) == ["us", "us", "us"] - - def test_empty_text_files(ray_start_regular_shared, tmp_path): path = os.path.join(tmp_path, "test_text") os.mkdir(path) @@ -69,30 +47,6 @@ def test_read_text(ray_start_regular_shared, tmp_path): assert ds.count() == 4 -@pytest.mark.parametrize("ignore_missing_paths", [True, False]) -def test_read_text_ignore_missing_paths( - ray_start_regular_shared, tmp_path, ignore_missing_paths -): - path = os.path.join(tmp_path, "test_text") - os.mkdir(path) - with open(os.path.join(path, "file1.txt"), "w") as f: - f.write("hello\n") - f.write("world") - - paths = [ - path, - "missing.txt", - ] - - if ignore_missing_paths: - ds = ray.data.read_text(paths, ignore_missing_paths=ignore_missing_paths) - assert ds.input_files() == [os.path.join(path, "file1.txt")] - else: - with pytest.raises(FileNotFoundError): - ds = ray.data.read_text(paths, ignore_missing_paths=ignore_missing_paths) - ds.materialize() - - def test_read_text_meta_provider( ray_start_regular_shared, tmp_path, @@ -117,57 +71,6 @@ def test_read_text_meta_provider( ) -def test_read_text_partitioned_with_filter( - shutdown_only, - tmp_path, - write_base_partitioned_df, - assert_base_partitioned_ds, -): - def df_to_text(dataframe, path, **kwargs): - dataframe.to_string(path, index=False, header=False, **kwargs) - - partition_keys = ["one"] - kept_file_counter = Counter.remote() - skipped_file_counter = Counter.remote() - - def skip_unpartitioned(kv_dict): - keep = bool(kv_dict) - counter = kept_file_counter if keep else skipped_file_counter - ray.get(counter.increment.remote()) - return keep - - for style in [PartitionStyle.HIVE, PartitionStyle.DIRECTORY]: - base_dir = os.path.join(tmp_path, style.value) - partition_path_encoder = PathPartitionEncoder.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - ) - write_base_partitioned_df( - partition_keys, - partition_path_encoder, - df_to_text, - ) - df_to_text(pd.DataFrame({"1": [1]}), os.path.join(base_dir, "test.txt")) - partition_path_filter = PathPartitionFilter.of( - style=style, - base_dir=base_dir, - field_names=partition_keys, - filter_fn=skip_unpartitioned, - ) - ds = ray.data.read_text(base_dir, partition_filter=partition_path_filter) - assert_base_partitioned_ds( - ds, - schema=Schema(pa.schema([("text", pa.string())])), - sorted_values=["1 a", "1 b", "1 c", "3 e", "3 f", "3 g"], - ds_take_transform_fn=_to_lines, - ) - assert ray.get(kept_file_counter.get.remote()) == 2 - assert ray.get(skipped_file_counter.get.remote()) == 1 - ray.get(kept_file_counter.reset.remote()) - ray.get(skipped_file_counter.reset.remote()) - - def test_read_text_remote_args(ray_start_cluster, tmp_path): cluster = ray_start_cluster cluster.add_node( @@ -177,6 +80,7 @@ def test_read_text_remote_args(ray_start_cluster, tmp_path): ) cluster.add_node(resources={"bar": 100}, num_cpus=1) + ray.shutdown() ray.init(cluster.address) @ray.remote diff --git a/python/ray/data/tests/test_tfrecords.py b/python/ray/data/tests/test_tfrecords.py index 3d39cd588fdc..31c355f95db3 100644 --- a/python/ray/data/tests/test_tfrecords.py +++ b/python/ray/data/tests/test_tfrecords.py @@ -489,36 +489,6 @@ def test_read_tfrecords_ray_remote_args( assert kwargs["ray_remote_args"] == ray_remote_args -@pytest.mark.parametrize("ignore_missing_paths", [True, False]) -def test_read_tfrecords_ignore_missing_paths( - ray_start_regular_shared, tmp_path, ignore_missing_paths -): - import tensorflow as tf - - example = tf_records_empty()[0] - - path = os.path.join(tmp_path, "data.tfrecords") - with tf.io.TFRecordWriter(path=path) as writer: - writer.write(example.SerializeToString()) - - paths = [ - path, - "missing.tfrecords", - ] - - if ignore_missing_paths: - ds = read_tfrecords_with_tfx_read_override( - path, ignore_missing_paths=ignore_missing_paths - ) - assert ds.input_files() == [path] - else: - with pytest.raises(FileNotFoundError): - ds = read_tfrecords_with_tfx_read_override( - paths, ignore_missing_paths=ignore_missing_paths - ) - ds.materialize() - - @pytest.mark.parametrize("with_tf_schema", (True, False)) def test_write_tfrecords( with_tf_schema, From 83bb39f952d8bec8258460b5dfd60c963e1f80b2 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 8 Sep 2025 16:22:35 -0700 Subject: [PATCH 514/634] [cpp] performs rename when generating (#56338) this avoid the template files being treated as source files. also cleans up a bit the cpp template generation logic. Signed-off-by: Lonnie Liu --- cpp/BUILD.bazel | 5 ----- python/ray/scripts/scripts.py | 32 ++++++++++++++------------------ 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index 745316e11a32..9ab6348cac9b 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -140,11 +140,6 @@ pkg_files( name = "example_files", srcs = glob(["example/*"]), prefix = "ray/cpp/example/", - renames = { - "example/_WORKSPACE": "WORKSPACE", - "example/_BUILD.bazel": "BUILD.bazel", - "example/_.bazelrc": ".bazelrc", - }, visibility = ["//visibility:private"], ) diff --git a/python/ray/scripts/scripts.py b/python/ray/scripts/scripts.py index e05657a5d3c2..b1aa3e6440bb 100644 --- a/python/ray/scripts/scripts.py +++ b/python/ray/scripts/scripts.py @@ -2619,33 +2619,29 @@ def cpp(show_library_path, generate_bazel_project_template_to): cli_logger.print("Ray C++ include path {} ", cf.bold(f"{include_dir}")) cli_logger.print("Ray C++ library path {} ", cf.bold(f"{lib_dir}")) if generate_bazel_project_template_to: + out_dir = generate_bazel_project_template_to # copytree expects that the dst dir doesn't exist # so we manually delete it if it exists. - if os.path.exists(generate_bazel_project_template_to): - shutil.rmtree(generate_bazel_project_template_to) - shutil.copytree(cpp_templete_dir, generate_bazel_project_template_to) - out_include_dir = os.path.join( - generate_bazel_project_template_to, "thirdparty/include" - ) - if os.path.exists(out_include_dir): - shutil.rmtree(out_include_dir) + if os.path.exists(out_dir): + shutil.rmtree(out_dir) + + shutil.copytree(cpp_templete_dir, out_dir) + for filename in ["_WORKSPACE", "_BUILD.bazel", "_.bazelrc"]: + # Renames the bazel related files by removing the leading underscore. + dest_name = os.path.join(out_dir, filename[1:]) + shutil.move(os.path.join(out_dir, filename), dest_name) + + out_include_dir = os.path.join(out_dir, "thirdparty/include") shutil.copytree(include_dir, out_include_dir) - out_lib_dir = os.path.join(generate_bazel_project_template_to, "thirdparty/lib") - if os.path.exists(out_lib_dir): - shutil.rmtree(out_lib_dir) + out_lib_dir = os.path.join(out_dir, "thirdparty/lib") shutil.copytree(lib_dir, out_lib_dir) cli_logger.print( "Project template generated to {}", - cf.bold(f"{os.path.abspath(generate_bazel_project_template_to)}"), + cf.bold(f"{os.path.abspath(out_dir)}"), ) cli_logger.print("To build and run this template, run") - cli_logger.print( - cf.bold( - f" cd {os.path.abspath(generate_bazel_project_template_to)}" - " && bash run.sh" - ) - ) + cli_logger.print(cf.bold(f" cd {os.path.abspath(out_dir)} && bash run.sh")) @cli.command(hidden=True) From 1fb696561f159ba922ef6140aab2bd00a8d54dca Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Mon, 8 Sep 2025 16:47:19 -0700 Subject: [PATCH 515/634] [ci] raydepsets check lock files (#55856) Including lock file checking as part of raydepsets --------- Signed-off-by: elliot-barn Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> --- ci/raydepsets/cli.py | 83 +++++++++++++++++- ci/raydepsets/tests/test_cli.py | 147 ++++++++++++++++++++++++++------ 2 files changed, 199 insertions(+), 31 deletions(-) diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index 77733f025943..aae4b6de5c83 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -2,9 +2,12 @@ import subprocess from pathlib import Path from typing import List, Optional - +import shutil import click import runfiles +import tempfile +import difflib +import sys from networkx import DiGraph, topological_sort, ancestors as networkx_ancestors from ci.raydepsets.workspace import Depset, Workspace @@ -44,11 +47,17 @@ def cli(): @click.option( "--uv-cache-dir", default=None, help="The directory to cache uv dependencies" ) +@click.option( + "--check", + is_flag=True, + help="Check the the compiled dependencies are valid. Only compatible with generating all dependency sets.", +) def build( config_path: str, workspace_dir: Optional[str], name: Optional[str], uv_cache_dir: Optional[str], + check: Optional[bool], ): """ Build dependency sets from a config file. @@ -59,8 +68,17 @@ def build( config_path=config_path, workspace_dir=workspace_dir, uv_cache_dir=uv_cache_dir, + check=check, ) manager.execute(name) + if check: + try: + manager.diff_lock_files() + except RuntimeError as e: + click.echo(e, err=True) + sys.exit(1) + finally: + manager.cleanup() class DependencySetManager: @@ -69,14 +87,63 @@ def __init__( config_path: str = None, workspace_dir: Optional[str] = None, uv_cache_dir: Optional[str] = None, + check: Optional[bool] = False, ): self.workspace = Workspace(workspace_dir) self.config = self.workspace.load_config(config_path) + if check: + self.temp_dir = tempfile.mkdtemp() + self.output_paths = self.get_output_paths() + self.copy_to_temp_dir() self.build_graph = DiGraph() self._build() self._uv_binary = _uv_binary() self._uv_cache_dir = uv_cache_dir + def get_output_paths(self) -> List[Path]: + output_paths = [] + for depset in self.config.depsets: + output_paths.append(Path(depset.output)) + return output_paths + + def copy_to_temp_dir(self): + """Copy the lock files from source file paths to temp dir.""" + for output_path in self.output_paths: + source_fp, target_fp = self.get_source_and_dest(output_path) + target_fp.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2( + source_fp, + target_fp, + ) + + def get_diffs(self) -> List[str]: + diffs = [] + for output_path in self.output_paths: + new_lock_file_fp, old_lock_file_fp = self.get_source_and_dest(output_path) + old_lock_file_contents = self.read_lock_file(old_lock_file_fp) + new_lock_file_contents = self.read_lock_file(new_lock_file_fp) + for diff in difflib.unified_diff( + old_lock_file_contents, + new_lock_file_contents, + fromfile=new_lock_file_fp.as_posix(), + tofile=old_lock_file_fp.as_posix(), + lineterm="", + ): + diffs.append(diff) + return diffs + + def diff_lock_files(self): + diffs = self.get_diffs() + if len(diffs) > 0: + raise RuntimeError( + "Lock files are not up to date. Please update lock files and push the changes.\n" + + "".join(diffs) + ) + click.echo("Lock files are up to date.") + + def get_source_and_dest(self, output_path: str) -> tuple[Path, Path]: + return (self.get_path(output_path), (Path(self.temp_dir) / output_path)) + def _build(self): for depset in self.config.depsets: if depset.operation == "compile": @@ -235,8 +302,14 @@ def expand( override_flags=override_flags, ) - def get_path(self, path: str) -> str: - return (Path(self.workspace.dir) / path).as_posix() + def read_lock_file(self, file_path: Path) -> List[str]: + if not file_path.exists(): + raise RuntimeError(f"Lock file {file_path} does not exist") + with open(file_path, "r") as f: + return f.readlines() + + def get_path(self, path: str) -> Path: + return Path(self.workspace.dir) / path def check_subset_exists(self, source_depset: Depset, requirements: List[str]): for req in requirements: @@ -245,6 +318,10 @@ def check_subset_exists(self, source_depset: Depset, requirements: List[str]): f"Requirement {req} is not a subset of {source_depset.name}" ) + def cleanup(self): + if self.temp_dir: + shutil.rmtree(self.temp_dir) + def _get_bytes(packages: List[str]) -> bytes: return ("\n".join(packages) + "\n").encode("utf-8") diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 4a7fa2d5237b..70ff7f2d89e6 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -2,10 +2,9 @@ import sys import tempfile import unittest -from pathlib import Path from typing import Optional - import pytest +from pathlib import Path import runfiles from click.testing import CliRunner from networkx import topological_sort @@ -35,7 +34,7 @@ def _create_test_manager( - tmpdir: str, config_path: Optional[str] = None + tmpdir: str, config_path: Optional[str] = None, check: bool = False ) -> DependencySetManager: if config_path is None: config_path = "test.depsets.yaml" @@ -44,22 +43,41 @@ def _create_test_manager( config_path=config_path, workspace_dir=tmpdir, uv_cache_dir=uv_cache_dir.as_posix(), + check=check, ) +def _overwrite_config_file(tmpdir: str, depset: Depset): + with open(Path(tmpdir) / "test.depsets.yaml", "w") as f: + f.write( + f""" +depsets: + - name: {depset.name} + operation: {depset.operation} + constraints: + - {depset.constraints} + requirements: + - {depset.requirements} + output: {depset.output} + """ + ) + + class TestCli(unittest.TestCase): def test_cli_load_fail_no_config(self): - result = CliRunner().invoke( - build, - [ - "fake_path/test.depsets.yaml", - "--workspace-dir", - "/ci/raydepsets/test_data", - ], - ) - assert result.exit_code == 1 - assert isinstance(result.exception, FileNotFoundError) - assert "No such file or directory" in str(result.exception) + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + result = CliRunner().invoke( + build, + [ + "fake_path/test.depsets.yaml", + "--workspace-dir", + tmpdir, + ], + ) + assert result.exit_code == 1 + assert isinstance(result.exception, FileNotFoundError) + assert "No such file or directory" in str(result.exception) def test_dependency_set_manager_init(self): with tempfile.TemporaryDirectory() as tmpdir: @@ -166,7 +184,6 @@ def test_compile_by_depset_name(self): with tempfile.TemporaryDirectory() as tmpdir: copy_data_to_tmpdir(tmpdir) uv_cache_dir = Path(tmpdir) / "uv_cache" - result = CliRunner().invoke( build, [ @@ -179,7 +196,6 @@ def test_compile_by_depset_name(self): uv_cache_dir.as_posix(), ], ) - output_fp = Path(tmpdir) / "requirements_compiled.txt" assert output_fp.is_file() assert result.exit_code == 0 @@ -284,7 +300,7 @@ def test_get_path(self): manager = _create_test_manager(tmpdir) assert ( manager.get_path("requirements_test.txt") - == f"{tmpdir}/requirements_test.txt" + == Path(tmpdir) / "requirements_test.txt" ) def test_append_uv_flags_exist_in_output(self): @@ -419,17 +435,13 @@ def test_build_graph_predecessors(self): def test_build_graph_bad_operation(self): with tempfile.TemporaryDirectory() as tmpdir: copy_data_to_tmpdir(tmpdir) - with open(Path(tmpdir) / "test.depsets.yaml", "w") as f: - f.write( - """ -depsets: - - name: invalid_op_depset - operation: invalid_op - requirements: - - requirements_test.txt - output: requirements_compiled_invalid_op.txt - """ - ) + depset = Depset( + name="invalid_op_depset", + operation="invalid_op", + requirements=["requirements_test.txt"], + output="requirements_compiled_invalid_op.txt", + ) + _overwrite_config_file(tmpdir, depset) with self.assertRaises(ValueError): _create_test_manager(tmpdir) @@ -568,6 +580,85 @@ def test_get_depset_with_build_arg_set_and_no_build_arg_set_provided(self): with self.assertRaises(KeyError): _get_depset(manager.config.depsets, "build_args_test_depset_py311") + def test_copy_lock_files_to_temp_dir(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + depset = Depset( + name="check_depset", + operation="compile", + constraints=["requirement_constraints_test.txt"], + requirements=["requirements_test.txt"], + output="requirements_compiled_test.txt", + ) + _overwrite_config_file(tmpdir, depset) + manager = _create_test_manager(tmpdir, check=True) + manager.compile( + constraints=["requirement_constraints_test.txt"], + requirements=["requirements_test.txt"], + append_flags=["--no-annotate", "--no-header"], + name="check_depset", + output="requirements_compiled_test.txt", + ) + assert ( + Path(manager.workspace.dir) / "requirements_compiled_test.txt" + ).exists() + assert (Path(manager.temp_dir) / "requirements_compiled_test.txt").exists() + + def test_diff_lock_files_out_of_date(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + depset = Depset( + name="check_depset", + operation="compile", + constraints=["requirement_constraints_test.txt"], + requirements=["requirements_test.txt"], + output="requirements_compiled_test.txt", + ) + _overwrite_config_file(tmpdir, depset) + manager = _create_test_manager(tmpdir, check=True) + manager.compile( + constraints=["requirement_constraints_test.txt"], + requirements=["requirements_test.txt"], + append_flags=["--no-annotate", "--no-header"], + name="check_depset", + output="requirements_compiled_test.txt", + ) + replace_in_file( + Path(manager.workspace.dir) / "requirements_compiled_test.txt", + "emoji==2.9.0", + "emoji==2.8.0", + ) + + with self.assertRaises(RuntimeError) as e: + manager.diff_lock_files() + assert ( + "Lock files are not up to date. Please update lock files and push the changes." + in str(e.exception) + ) + assert "+emoji==2.8.0" in str(e.exception) + assert "-emoji==2.9.0" in str(e.exception) + + def test_diff_lock_files_up_to_date(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + depset = Depset( + name="check_depset", + operation="compile", + constraints=["requirement_constraints_test.txt"], + requirements=["requirements_test.txt"], + output="requirements_compiled_test.txt", + ) + _overwrite_config_file(tmpdir, depset) + manager = _create_test_manager(tmpdir, check=True) + manager.compile( + constraints=["requirement_constraints_test.txt"], + requirements=["requirements_test.txt"], + append_flags=["--no-annotate", "--no-header"], + name="check_depset", + output="requirements_compiled_test.txt", + ) + manager.diff_lock_files() + def test_compile_with_packages(self): with tempfile.TemporaryDirectory() as tmpdir: copy_data_to_tmpdir(tmpdir) From c67cec8b6603275cc6aef0fbe3580919a17acb77 Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Mon, 8 Sep 2025 17:02:20 -0700 Subject: [PATCH 516/634] [data] Rename env var for enforce schemas (#56254) ## Why are these changes needed? As titled ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- python/ray/data/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/data/context.py b/python/ray/data/context.py index f39c5cd700c1..59bebbaa3ede 100644 --- a/python/ray/data/context.py +++ b/python/ray/data/context.py @@ -140,7 +140,7 @@ class ShuffleStrategy(str, enum.Enum): "RAY_DATA_ENABLE_PROGRESS_BAR_NAME_TRUNCATION", True ) -DEFAULT_ENFORCE_SCHEMAS = env_bool("RAY_DATA_ALLOW_ENFORCE_SCHEMAS", False) +DEFAULT_ENFORCE_SCHEMAS = env_bool("RAY_DATA_ENFORCE_SCHEMAS", False) DEFAULT_ENABLE_GET_OBJECT_LOCATIONS_FOR_METRICS = False From 668c4e9fae28dfcf8361911412a8322b3b20dfdc Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 8 Sep 2025 20:02:20 -0700 Subject: [PATCH 517/634] [codeowner] remove aslonnie individual from many ownerships (#56364) delegating to ray-core and ray-ci teams and moving `_common` to core team leads Signed-off-by: Lonnie Liu --- .github/CODEOWNERS | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 38e78b8a4cad..8c4dbb3c0fa3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -39,7 +39,7 @@ # Common directory shared by core and the libraries. # @edoakes is the czar for now because the pattern is new. -/python/ray/_common/ @edoakes @aslonnie +/python/ray/_common/ @edoakes @jjyao # Ray data. /python/ray/data/ @ray-project/ray-data @@ -78,8 +78,8 @@ /python/requirements/ml/dl-gpu-requirements.txt @richardliaw @matthewdeng # Ray symbol export -/src/ray/ray_version_script.lds @aslonnie -/src/ray/ray_exported_symbols.lds @aslonnie +/src/ray/ray_version_script.lds @ray-project/ray-core +/src/ray/ray_exported_symbols.lds @ray-project/ray-core # Ray usage stats /python/ray/_private/usage/ @edoakes @richardliaw @jjyao @@ -111,8 +111,8 @@ # on their own. /release/ray_release/byod/*.sh -/.github/ISSUE_TEMPLATE/ @aslonnie +/.github/ISSUE_TEMPLATE/ @ray-project/ray-ci /.github/workflows/ @ray-project/ray-ci -/.gemini/ @edoakes @aslonnie +/.gemini/ @edoakes @ray-project/ray-ci From 907be9d35b90f5b89b07427b20bd44fa0aa23678 Mon Sep 17 00:00:00 2001 From: Potato Date: Tue, 9 Sep 2025 12:37:10 +0800 Subject: [PATCH 518/634] [TRAIN][DOC] Fix documentation issues in train/examples directory: typos, grammar, and formatting errors (#56273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR addresses various documentation issues found across the `doc/source/train/examples/` directory through a comprehensive manual review of all 30 files (RST, Jupyter notebooks, and Markdown). ## Issues Fixed ### RST Files (4 fixes) - **Reference formatting**: Fixed extra space in reference label `.. _transformers_torch_trainer_basic_example :` - **Title grammar**: Corrected "Fine-tune of Stable Diffusion" → "Fine-tuning of Stable Diffusion" - **URL errors**: Fixed duplicate protocol in URL `https://https://console.anyscale.com/...` - **Missing words**: Added missing verb "is" in "The Ray cluster now ready to handle workloads" ### Jupyter Notebooks (4 fixes) - **Grammar**: Fixed "the your machine" → "your machine" in text descriptions - **Word repetition**: Corrected "for for images" → "for images" in multiple instances - **Hyphenation**: Fixed "fine tunes" → "fine-tunes" for consistency - **Typos**: Corrected "dependecies" → "dependencies" ### Markdown Files - All files reviewed with no issues found ## Review Process The review followed a systematic approach: 1. Created comprehensive file listing and review guidelines 2. Manually reviewed each file for typos, syntax errors, and grammatical issues 3. Applied minimal, surgical fixes preserving original content structure 4. Maintained technical accuracy and existing writing style 5. Ensured consistency in terminology usage All changes are minimal and focused on fixing clear errors without optimizing or restructuring content. The documentation now has improved readability and professionalism while maintaining its technical accuracy. --------- Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> --- doc/source/train/examples/accelerate/accelerate_example.rst | 2 +- doc/source/train/examples/aws-trainium/llama3.rst | 2 +- doc/source/train/examples/intel_gaudi/bert.ipynb | 2 +- .../pytorch/convert_existing_pytorch_code_to_ray_train.ipynb | 2 +- doc/source/train/examples/pytorch/dreambooth_finetuning.rst | 4 ++-- .../train/examples/pytorch/pytorch_resnet_finetune.ipynb | 4 ++-- .../transformers/huggingface_text_classification.ipynb | 2 +- .../transformers/transformers_torch_trainer_basic.rst | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/source/train/examples/accelerate/accelerate_example.rst b/doc/source/train/examples/accelerate/accelerate_example.rst index d9e84c48d267..e3b941444615 100644 --- a/doc/source/train/examples/accelerate/accelerate_example.rst +++ b/doc/source/train/examples/accelerate/accelerate_example.rst @@ -5,7 +5,7 @@ Distributed Training with Hugging Face Accelerate .. raw:: html - + Run on Anyscale

diff --git a/doc/source/train/examples/aws-trainium/llama3.rst b/doc/source/train/examples/aws-trainium/llama3.rst index 92af58efac8c..ff20e9b6e916 100644 --- a/doc/source/train/examples/aws-trainium/llama3.rst +++ b/doc/source/train/examples/aws-trainium/llama3.rst @@ -89,7 +89,7 @@ Run it in the background with the following command: Launching Ray Jobs ------------------ -The Ray cluster now ready to handle workloads. Initiate the data preparation and fine-tuning Ray jobs: +The Ray cluster is now ready to handle workloads. Initiate the data preparation and fine-tuning Ray jobs: 1. Launch the Ray job for downloading the dolly-15k dataset and the Llama3.1 8B model artifacts: diff --git a/doc/source/train/examples/intel_gaudi/bert.ipynb b/doc/source/train/examples/intel_gaudi/bert.ipynb index c45532960111..c48d34476af3 100644 --- a/doc/source/train/examples/intel_gaudi/bert.ipynb +++ b/doc/source/train/examples/intel_gaudi/bert.ipynb @@ -30,7 +30,7 @@ "docker run -it --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --net=host --ipc=host vault.habana.ai/gaudi-docker/1.20.0/ubuntu22.04/habanalabs/pytorch-installer-2.6.0:latest\n", "```\n", "\n", - "Inside the container, install the following dependecies to run this notebook.\n", + "Inside the container, install the following dependencies to run this notebook.\n", "```bash\n", "pip install ray[train] notebook transformers datasets evaluate\n", "```" diff --git a/doc/source/train/examples/pytorch/convert_existing_pytorch_code_to_ray_train.ipynb b/doc/source/train/examples/pytorch/convert_existing_pytorch_code_to_ray_train.ipynb index 6a5666412cb8..66170aacdd00 100644 --- a/doc/source/train/examples/pytorch/convert_existing_pytorch_code_to_ray_train.ipynb +++ b/doc/source/train/examples/pytorch/convert_existing_pytorch_code_to_ray_train.ipynb @@ -77,7 +77,7 @@ "source": [ "Then we download the data: \n", "\n", - "This tutorial assumes that your existing code is using the `torch.utils.data.Dataset` native to PyTorch. It continues to use `torch.utils.data.Dataset` to allow you to make as few code changes as possible. **This tutorial also runs with Ray Data, which gives you the benefits of efficient parallel preprocessing.** For more details on using Ray Data for for images, see the {doc}`Working with Images ` Ray Data user guide." + "This tutorial assumes that your existing code is using the `torch.utils.data.Dataset` native to PyTorch. It continues to use `torch.utils.data.Dataset` to allow you to make as few code changes as possible. **This tutorial also runs with Ray Data, which gives you the benefits of efficient parallel preprocessing.** For more details on using Ray Data for images, see the {doc}`Working with Images ` Ray Data user guide." ] }, { diff --git a/doc/source/train/examples/pytorch/dreambooth_finetuning.rst b/doc/source/train/examples/pytorch/dreambooth_finetuning.rst index b8da33e517dc..6d88a556f8fa 100644 --- a/doc/source/train/examples/pytorch/dreambooth_finetuning.rst +++ b/doc/source/train/examples/pytorch/dreambooth_finetuning.rst @@ -1,7 +1,7 @@ :orphan: -Fine-tune of Stable Diffusion with DreamBooth and Ray Train -=========================================================== +Fine-tuning of Stable Diffusion with DreamBooth and Ray Train +============================================================= .. raw:: html diff --git a/doc/source/train/examples/pytorch/pytorch_resnet_finetune.ipynb b/doc/source/train/examples/pytorch/pytorch_resnet_finetune.ipynb index 83dec754ac0c..12a7fc47b088 100644 --- a/doc/source/train/examples/pytorch/pytorch_resnet_finetune.ipynb +++ b/doc/source/train/examples/pytorch/pytorch_resnet_finetune.ipynb @@ -11,7 +11,7 @@ "\n", "

\n", "\n", - "This example fine tunes a pre-trained ResNet model with Ray Train. \n", + "This example fine-tunes a pre-trained ResNet model with Ray Train. \n", "\n", "For this example, the network architecture consists of the intermediate layer output of a pre-trained ResNet model, which feeds into a randomly initialized linear layer that outputs classification logits for our new task.\n", "\n", @@ -211,7 +211,7 @@ "The `train_loop_per_worker` function defines the fine-tuning procedure for each worker.\n", "\n", "**1. Prepare dataloaders for each worker**:\n", - "- This tutorial assumes you are using PyTorch's native `torch.utils.data.Dataset` for data input. {meth}`train.torch.prepare_data_loader() ` prepares your dataLoader for distributed execution. You can also use Ray Data for more efficient preprocessing. For more details on using Ray Data for for images, see the {doc}`Working with Images ` Ray Data user guide.\n", + "- This tutorial assumes you are using PyTorch's native `torch.utils.data.Dataset` for data input. {meth}`train.torch.prepare_data_loader() ` prepares your dataLoader for distributed execution. You can also use Ray Data for more efficient preprocessing. For more details on using Ray Data for images, see the {doc}`Working with Images ` Ray Data user guide.\n", "\n", "**2. Prepare your model**:\n", "- {meth}`train.torch.prepare_model() ` prepares the model for distributed training. Under the hood, it converts your torch model to `DistributedDataParallel` model, which synchronize its weights across all workers.\n", diff --git a/doc/source/train/examples/transformers/huggingface_text_classification.ipynb b/doc/source/train/examples/transformers/huggingface_text_classification.ipynb index c193f2b41686..ace610bbd88b 100644 --- a/doc/source/train/examples/transformers/huggingface_text_classification.ipynb +++ b/doc/source/train/examples/transformers/huggingface_text_classification.ipynb @@ -87,7 +87,7 @@ "id": "oJiSdWy2hYbR" }, "source": [ - "Check the resources our cluster is composed of. If you are running this notebook on your local machine or Google Colab, you should see the number of CPU cores and GPUs available on the your machine." + "Check the resources our cluster is composed of. If you are running this notebook on your local machine or Google Colab, you should see the number of CPU cores and GPUs available on your machine." ] }, { diff --git a/doc/source/train/examples/transformers/transformers_torch_trainer_basic.rst b/doc/source/train/examples/transformers/transformers_torch_trainer_basic.rst index 795ca47d6664..4e0f4a0db892 100644 --- a/doc/source/train/examples/transformers/transformers_torch_trainer_basic.rst +++ b/doc/source/train/examples/transformers/transformers_torch_trainer_basic.rst @@ -1,6 +1,6 @@ :orphan: -.. _transformers_torch_trainer_basic_example : +.. _transformers_torch_trainer_basic_example: Fine-tune a Text Classifier with Hugging Face Transformers ========================================================== From 6e265ba03384c645472740722d23bb88537d4d63 Mon Sep 17 00:00:00 2001 From: Potato Date: Tue, 9 Sep 2025 12:38:24 +0800 Subject: [PATCH 519/634] [TRAIN][DOC] Fix typos, grammar, and formatting issues in Ray Train documentation (#56274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR addresses multiple documentation quality issues across 29 files in the `doc/source/train/` directory (excluding examples). The changes focus on improving readability and consistency while maintaining the existing content structure. ## Issues Fixed ### Spelling and Grammar - Fixed "stablizes" → "stabilizes" in experiment tracking guide - Fixed "an Pytorch" → "a PyTorch" in benchmarks - Fixed "secure group" → "security group" (AWS terminology) ### Capitalization Consistency - Standardized "Pytorch" → "PyTorch" throughout all files - Standardized "Tensorflow" → "TensorFlow" throughout all files - Fixed "Deepspeed" → "DeepSpeed" (proper product name) - Fixed "HuggingFace" → "Hugging Face" (company name with space) ### reStructuredText Formatting - Fixed escaped backslashes in "``Trainer``\s" → "``Trainer``s" and "``RayActorError``\s" → "``RayActorError``s" - Removed malformed directive ".. _:: ../doc_code:" from fault tolerance guide - Fixed extra backtick in Ray Data documentation reference - Corrected spacing and line breaks in DeepSpeed guide ### Reference Links - Updated benchmark reference links to match corrected framework names - Ensured consistency between text references and actual link targets ## Files Modified - **API docs (2 files)**: Fixed TensorFlow capitalization - **Core docs (1 file)**: Multiple benchmark fixes - **Framework guides (3 files)**: DeepSpeed, TensorFlow, and Horovod formatting - **User guides (4 files)**: Checkpoints, data loading, experiment tracking, fault tolerance - **Deprecated guides (1 file)**: Fault tolerance formatting ## Scope All changes are minimal and surgical, focusing only on clear typos and formatting errors. No content was rewritten or restructured. The fixes improve documentation quality without changing functionality or breaking existing links. Total: 35+ individual corrections across 12 files, with 17 files reviewed and found to have no issues. --------- Signed-off-by: Potato Signed-off-by: Jiajun Yao Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Jiajun Yao Co-authored-by: angelinalg <122562471+angelinalg@users.noreply.github.com> --- doc/source/train/api/api.rst | 2 +- doc/source/train/api/deprecated.rst | 2 +- doc/source/train/benchmarks.rst | 52 +++++++++---------- doc/source/train/deepspeed.rst | 4 +- .../train/distributed-tensorflow-keras.rst | 2 +- doc/source/train/user-guides/checkpoints.rst | 2 +- .../data-loading-preprocessing.rst | 2 +- .../train/user-guides/experiment-tracking.rst | 2 +- .../train/user-guides/fault-tolerance.rst | 2 - 9 files changed, 34 insertions(+), 36 deletions(-) diff --git a/doc/source/train/api/api.rst b/doc/source/train/api/api.rst index 344682bd66f5..9c0ed0af5d33 100644 --- a/doc/source/train/api/api.rst +++ b/doc/source/train/api/api.rst @@ -72,7 +72,7 @@ Hugging Face Transformers More Frameworks --------------- -Tensorflow/Keras +TensorFlow/Keras ~~~~~~~~~~~~~~~~ .. autosummary:: diff --git a/doc/source/train/api/deprecated.rst b/doc/source/train/api/deprecated.rst index 4c016577ee92..7a51cb4b48dd 100644 --- a/doc/source/train/api/deprecated.rst +++ b/doc/source/train/api/deprecated.rst @@ -66,7 +66,7 @@ Hugging Face Transformers More Frameworks --------------- -Tensorflow/Keras +TensorFlow/Keras ~~~~~~~~~~~~~~~~ .. autosummary:: diff --git a/doc/source/train/benchmarks.rst b/doc/source/train/benchmarks.rst index 050d56081029..94702b98916f 100644 --- a/doc/source/train/benchmarks.rst +++ b/doc/source/train/benchmarks.rst @@ -11,7 +11,7 @@ GPU image training ------------------ This task uses the TorchTrainer module to train different amounts of data -using an Pytorch ResNet model. +using a PyTorch ResNet model. We test out the performance across different cluster sizes and data sizes. @@ -22,7 +22,7 @@ We test out the performance across different cluster sizes and data sizes. .. note:: For multi-host distributed training, on AWS we need to ensure ec2 instances are in the same VPC and - all ports are open in the secure group. + all ports are open in the security group. .. list-table:: @@ -46,10 +46,10 @@ We test out the performance across different cluster sizes and data sizes. .. _pytorch-training-parity: -Pytorch Training Parity +PyTorch training parity ----------------------- -This task checks the performance parity between native Pytorch Distributed and +This task checks the performance parity between native PyTorch Distributed and Ray Train's distributed TorchTrainer. We demonstrate that the performance is similar (within 2.5\%) between the two frameworks. @@ -58,9 +58,9 @@ Performance may vary greatly across different model, hardware, and cluster confi The reported times are for the raw training times. There is an unreported constant setup overhead of a few seconds for both methods that is negligible for longer training runs. -- `Pytorch comparison training script`_ -- `Pytorch comparison CPU cluster configuration`_ -- `Pytorch comparison GPU cluster configuration`_ +- `PyTorch comparison training script`_ +- `PyTorch comparison CPU cluster configuration`_ +- `PyTorch comparison GPU cluster configuration`_ .. list-table:: @@ -70,24 +70,24 @@ overhead of a few seconds for both methods that is negligible for longer trainin - **Command** * - 4 m5.2xlarge nodes (4 workers) - FashionMNIST - - 196.64 s (vs 194.90 s Pytorch) + - 196.64 s (vs 194.90 s PyTorch) - `python workloads/torch_benchmark.py run --num-runs 3 --num-epochs 20 --num-workers 4 --cpus-per-worker 8` * - 4 m5.2xlarge nodes (16 workers) - FashionMNIST - - 430.88 s (vs 475.97 s Pytorch) + - 430.88 s (vs 475.97 s PyTorch) - `python workloads/torch_benchmark.py run --num-runs 3 --num-epochs 20 --num-workers 16 --cpus-per-worker 2` - * - 4 g4dn.12xlarge node (16 workers) + * - 4 g4dn.12xlarge nodes (16 workers) - FashionMNIST - - 149.80 s (vs 146.46 s Pytorch) + - 149.80 s (vs 146.46 s PyTorch) - `python workloads/torch_benchmark.py run --num-runs 3 --num-epochs 20 --num-workers 16 --cpus-per-worker 4 --use-gpu` .. _tf-training-parity: -Tensorflow Training Parity +TensorFlow training parity -------------------------- -This task checks the performance parity between native Tensorflow Distributed and +This task checks the performance parity between native TensorFlow Distributed and Ray Train's distributed TensorflowTrainer. We demonstrate that the performance is similar (within 1\%) between the two frameworks. @@ -98,9 +98,9 @@ overhead of a few seconds for both methods that is negligible for longer trainin .. note:: The batch size and number of epochs is different for the GPU benchmark, resulting in a longer runtime. -- `Tensorflow comparison training script`_ -- `Tensorflow comparison CPU cluster configuration`_ -- `Tensorflow comparison GPU cluster configuration`_ +- `TensorFlow comparison training script`_ +- `TensorFlow comparison CPU cluster configuration`_ +- `TensorFlow comparison GPU cluster configuration`_ .. list-table:: @@ -110,15 +110,15 @@ overhead of a few seconds for both methods that is negligible for longer trainin - **Command** * - 4 m5.2xlarge nodes (4 workers) - FashionMNIST - - 78.81 s (vs 79.67 s Tensorflow) + - 78.81 s (versus 79.67 s TensorFlow) - `python workloads/tensorflow_benchmark.py run --num-runs 3 --num-epochs 20 --num-workers 4 --cpus-per-worker 8` * - 4 m5.2xlarge nodes (16 workers) - FashionMNIST - - 64.57 s (vs 67.45 s Tensorflow) + - 64.57 s (versus 67.45 s TensorFlow) - `python workloads/tensorflow_benchmark.py run --num-runs 3 --num-epochs 20 --num-workers 16 --cpus-per-worker 2` - * - 4 g4dn.12xlarge node (16 workers) + * - 4 g4dn.12xlarge nodes (16 workers) - FashionMNIST - - 465.16 s (vs 461.74 s Tensorflow) + - 465.16 s (versus 461.74 s TensorFlow) - `python workloads/tensorflow_benchmark.py run --num-runs 3 --num-epochs 200 --num-workers 16 --cpus-per-worker 4 --batch-size 64 --use-gpu` .. _xgboost-benchmark: @@ -157,11 +157,11 @@ XGBoost parameters were kept as defaults for ``xgboost==1.7.6`` this task. .. _`GPU image training script`: https://github.com/ray-project/ray/blob/cec82a1ced631525a4d115e4dc0c283fa4275a7f/release/air_tests/air_benchmarks/workloads/pytorch_training_e2e.py#L95-L106 .. _`GPU training small cluster configuration`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/compute_gpu_1_aws.yaml#L6-L24 .. _`GPU training large cluster configuration`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml#L5-L25 -.. _`Pytorch comparison training script`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/workloads/torch_benchmark.py -.. _`Pytorch comparison CPU cluster configuration`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/compute_cpu_4_aws.yaml -.. _`Pytorch comparison GPU cluster configuration`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml -.. _`Tensorflow comparison training script`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/workloads/tensorflow_benchmark.py -.. _`Tensorflow comparison CPU cluster configuration`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/compute_cpu_4_aws.yaml -.. _`Tensorflow comparison GPU cluster configuration`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml +.. _`PyTorch comparison training script`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/workloads/torch_benchmark.py +.. _`PyTorch comparison CPU cluster configuration`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/compute_cpu_4_aws.yaml +.. _`PyTorch comparison GPU cluster configuration`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml +.. _`TensorFlow comparison training script`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/workloads/tensorflow_benchmark.py +.. _`TensorFlow comparison CPU cluster configuration`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/compute_cpu_4_aws.yaml +.. _`TensorFlow comparison GPU cluster configuration`: https://github.com/ray-project/ray/blob/master/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml .. _`XGBoost Training Script`: https://github.com/ray-project/ray/blob/9ac58f4efc83253fe63e280106f959fe317b1104/release/train_tests/xgboost_lightgbm/train_batch_inference_benchmark.py .. _`XGBoost Cluster Configuration`: https://github.com/ray-project/ray/tree/9ac58f4efc83253fe63e280106f959fe317b1104/release/train_tests/xgboost_lightgbm diff --git a/doc/source/train/deepspeed.rst b/doc/source/train/deepspeed.rst index 570892e5a594..44fdda3d1d13 100644 --- a/doc/source/train/deepspeed.rst +++ b/doc/source/train/deepspeed.rst @@ -71,7 +71,7 @@ Complete Examples ----------------- Below are complete examples of ZeRO-3 training with DeepSpeed. Each example shows a full implementation of fine-tuning - a Bidirectional Encoder Representations from Transformers (BERT) model on the Microsoft Research Paraphrase Corpus (MRPC) dataset. +a Bidirectional Encoder Representations from Transformers (BERT) model on the Microsoft Research Paraphrase Corpus (MRPC) dataset. Install the requirements: @@ -119,7 +119,7 @@ Check the below examples for more details: * - Framework - Example * - Accelerate (:ref:`User Guide `) - - `Fine-tune Llama-2 series models with Deepspeed, Accelerate, and Ray Train. `_ + - `Fine-tune Llama-2 series models with DeepSpeed, Accelerate, and Ray Train. `_ * - Transformers (:ref:`User Guide `) - :doc:`Fine-tune GPT-J-6b with DeepSpeed and Hugging Face Transformers ` * - Lightning (:ref:`User Guide `) diff --git a/doc/source/train/distributed-tensorflow-keras.rst b/doc/source/train/distributed-tensorflow-keras.rst index 5ab690bdbec7..34db90b6b37b 100644 --- a/doc/source/train/distributed-tensorflow-keras.rst +++ b/doc/source/train/distributed-tensorflow-keras.rst @@ -78,7 +78,7 @@ Create a TensorflowTrainer -------------------------- ``Trainer``\s are the primary Ray Train classes for managing state and -execute training. For distributed Tensorflow, +execute training. For distributed TensorFlow, use a :class:`~ray.train.tensorflow.TensorflowTrainer` that you can setup like this: diff --git a/doc/source/train/user-guides/checkpoints.rst b/doc/source/train/user-guides/checkpoints.rst index 595a3d7b2b7c..cd837cdccf13 100644 --- a/doc/source/train/user-guides/checkpoints.rst +++ b/doc/source/train/user-guides/checkpoints.rst @@ -120,7 +120,7 @@ Here are a few examples of saving checkpoints with different training frameworks .. tab-item:: Hugging Face Transformers - Ray Train leverages HuggingFace Transformers Trainer's ``Callback`` interface + Ray Train leverages Hugging Face Transformers Trainer's ``Callback`` interface to report metrics and checkpoints. **Option 1: Use Ray Train's default report callback** diff --git a/doc/source/train/user-guides/data-loading-preprocessing.rst b/doc/source/train/user-guides/data-loading-preprocessing.rst index 9e95dd1445e9..3835a697fb25 100644 --- a/doc/source/train/user-guides/data-loading-preprocessing.rst +++ b/doc/source/train/user-guides/data-loading-preprocessing.rst @@ -11,7 +11,7 @@ Key advantages include: - Automatic and fast failure recovery. - Automatic on-the-fly data splitting across distributed training workers. -For more details about Ray Data, check out the :ref:`Ray Data documentation`.` +For more details about Ray Data, check out the :ref:`Ray Data documentation`. .. note:: diff --git a/doc/source/train/user-guides/experiment-tracking.rst b/doc/source/train/user-guides/experiment-tracking.rst index e80d67bb79c5..424aff50f798 100644 --- a/doc/source/train/user-guides/experiment-tracking.rst +++ b/doc/source/train/user-guides/experiment-tracking.rst @@ -242,7 +242,7 @@ Refer to the tracking libraries' documentation for semantics. def train_func(): if ray.train.get_context().get_world_rank() == 0: - wandb.init(..., config={"ray_train_persistent_storage_path": "TODO: fill in when API stablizes"}) + wandb.init(..., config={"ray_train_persistent_storage_path": "TODO: fill in when API stabilizes"}) .. tip:: diff --git a/doc/source/train/user-guides/fault-tolerance.rst b/doc/source/train/user-guides/fault-tolerance.rst index 81533ef29e94..ab25902ce54e 100644 --- a/doc/source/train/user-guides/fault-tolerance.rst +++ b/doc/source/train/user-guides/fault-tolerance.rst @@ -1,5 +1,3 @@ -.. _:: ../doc_code: - .. _train-fault-tolerance: Handling Failures and Node Preemption From 3c4230e70e913b2ff070c678e677eb1b07abcb64 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Tue, 9 Sep 2025 00:27:25 -0500 Subject: [PATCH 520/634] [core] Clean up `actor_creator` targets (#56336) Signed-off-by: Edward Oakes --- src/mock/ray/core_worker/actor_creator.h | 55 ------------ src/ray/core_worker/BUILD.bazel | 12 ++- src/ray/core_worker/actor_creator.cc | 86 +++++++++++++++++++ src/ray/core_worker/actor_creator.h | 64 +++----------- src/ray/core_worker/core_worker_process.cc | 2 +- src/ray/core_worker/fake_actor_creator.h | 63 ++++++++++++++ .../task_submission/tests/BUILD.bazel | 3 + .../tests/actor_task_submitter_test.cc | 4 +- .../tests/dependency_resolver_test.cc | 60 +++---------- .../tests/direct_actor_transport_test.cc | 4 +- .../tests/normal_task_submitter_test.cc | 46 +--------- .../core_worker/tests/actor_creator_test.cc | 14 +-- src/ray/core_worker/tests/core_worker_test.cc | 2 +- 13 files changed, 204 insertions(+), 211 deletions(-) delete mode 100644 src/mock/ray/core_worker/actor_creator.h create mode 100644 src/ray/core_worker/actor_creator.cc create mode 100644 src/ray/core_worker/fake_actor_creator.h diff --git a/src/mock/ray/core_worker/actor_creator.h b/src/mock/ray/core_worker/actor_creator.h deleted file mode 100644 index 95deb2808a5a..000000000000 --- a/src/mock/ray/core_worker/actor_creator.h +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2021 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#pragma once - -#include "gmock/gmock.h" -namespace ray { -namespace core { - -class MockActorCreatorInterface : public ActorCreatorInterface { - public: - MOCK_METHOD(Status, - RegisterActor, - (const TaskSpecification &task_spec), - (const, override)); - MOCK_METHOD(void, - AsyncRegisterActor, - (const TaskSpecification &task_spec, gcs::StatusCallback callback), - (override)); - MOCK_METHOD(void, - AsyncCreateActor, - (const TaskSpecification &task_spec, - const rpc::ClientCallback &callback), - (override)); - MOCK_METHOD(void, - AsyncRestartActorForLineageReconstruction, - (const ActorID &actor_id, - uint64_t num_restarts, - gcs::StatusCallback callback), - (override)); - MOCK_METHOD(void, - AsyncReportActorOutOfScope, - (const ActorID &actor_id, - uint64_t num_restarts_due_to_lineage_reconstruction, - gcs::StatusCallback callback), - (override)); - MOCK_METHOD(void, - AsyncWaitForActorRegisterFinish, - (const ActorID &actor_id, gcs::StatusCallback callback), - (override)); - MOCK_METHOD(bool, IsActorInRegistering, (const ActorID &actor_id), (const, override)); -}; - -} // namespace core -} // namespace ray diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 6cb8cff9b7f4..608f66964e61 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -158,13 +158,23 @@ ray_cc_library( ray_cc_library( name = "actor_creator", + srcs = ["actor_creator.cc"], hdrs = ["actor_creator.h"], + visibility = [":__subpackages__"], deps = [ - "//src/ray/common:ray_config", "//src/ray/gcs/gcs_client:gcs_client_lib", ], ) +ray_cc_library( + name = "fake_actor_creator", + hdrs = ["fake_actor_creator.h"], + visibility = [":__subpackages__"], + deps = [ + ":actor_creator", + ], +) + ray_cc_library( name = "actor_manager", srcs = ["actor_manager.cc"], diff --git a/src/ray/core_worker/actor_creator.cc b/src/ray/core_worker/actor_creator.cc new file mode 100644 index 000000000000..b5d9e10c99a3 --- /dev/null +++ b/src/ray/core_worker/actor_creator.cc @@ -0,0 +1,86 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/core_worker/actor_creator.h" + +#include +#include +#include + +namespace ray { +namespace core { + +Status ActorCreator::RegisterActor(const TaskSpecification &task_spec) const { + const auto status = actor_client_.SyncRegisterActor(task_spec); + if (status.IsTimedOut()) { + std::ostringstream stream; + stream << "There was timeout in registering an actor. It is probably " + "because GCS server is dead or there's a high load there."; + return Status::TimedOut(stream.str()); + } + return status; +} + +void ActorCreator::AsyncRegisterActor(const TaskSpecification &task_spec, + gcs::StatusCallback callback) { + auto actor_id = task_spec.ActorCreationId(); + (*registering_actors_)[actor_id] = {}; + if (callback != nullptr) { + (*registering_actors_)[actor_id].emplace_back(std::move(callback)); + } + actor_client_.AsyncRegisterActor(task_spec, [actor_id, this](Status status) { + std::vector cbs; + cbs = std::move((*registering_actors_)[actor_id]); + registering_actors_->erase(actor_id); + for (auto &cb : cbs) { + cb(status); + } + }); +} + +void ActorCreator::AsyncRestartActorForLineageReconstruction( + const ActorID &actor_id, + uint64_t num_restarts_due_to_lineage_reconstructions, + gcs::StatusCallback callback) { + actor_client_.AsyncRestartActorForLineageReconstruction( + actor_id, num_restarts_due_to_lineage_reconstructions, callback); +} + +void ActorCreator::AsyncReportActorOutOfScope( + const ActorID &actor_id, + uint64_t num_restarts_due_to_lineage_reconstruction, + gcs::StatusCallback callback) { + actor_client_.AsyncReportActorOutOfScope( + actor_id, num_restarts_due_to_lineage_reconstruction, callback); +} + +bool ActorCreator::IsActorInRegistering(const ActorID &actor_id) const { + return registering_actors_->find(actor_id) != registering_actors_->end(); +} + +void ActorCreator::AsyncWaitForActorRegisterFinish(const ActorID &actor_id, + gcs::StatusCallback callback) { + auto iter = registering_actors_->find(actor_id); + RAY_CHECK(iter != registering_actors_->end()); + iter->second.emplace_back(std::move(callback)); +} + +void ActorCreator::AsyncCreateActor( + const TaskSpecification &task_spec, + const rpc::ClientCallback &callback) { + actor_client_.AsyncCreateActor(task_spec, callback); +} + +} // namespace core +} // namespace ray diff --git a/src/ray/core_worker/actor_creator.h b/src/ray/core_worker/actor_creator.h index fb92ce99ac94..8673dc57154e 100644 --- a/src/ray/core_worker/actor_creator.h +++ b/src/ray/core_worker/actor_creator.h @@ -18,8 +18,7 @@ #include #include -#include "ray/common/ray_config.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs/gcs_client/accessor.h" namespace ray { namespace core { @@ -72,73 +71,36 @@ class ActorCreatorInterface { virtual bool IsActorInRegistering(const ActorID &actor_id) const = 0; }; -class DefaultActorCreator : public ActorCreatorInterface { +class ActorCreator : public ActorCreatorInterface { public: - explicit DefaultActorCreator(std::shared_ptr gcs_client) - : gcs_client_(std::move(gcs_client)) {} - - Status RegisterActor(const TaskSpecification &task_spec) const override { - const auto status = gcs_client_->Actors().SyncRegisterActor(task_spec); - if (status.IsTimedOut()) { - std::ostringstream stream; - stream << "There was timeout in registering an actor. It is probably " - "because GCS server is dead or there's a high load there."; - return Status::TimedOut(stream.str()); - } - return status; - } + explicit ActorCreator(gcs::ActorInfoAccessor &actor_client) + : actor_client_(actor_client) {} + + Status RegisterActor(const TaskSpecification &task_spec) const override; void AsyncRegisterActor(const TaskSpecification &task_spec, - gcs::StatusCallback callback) override { - auto actor_id = task_spec.ActorCreationId(); - (*registering_actors_)[actor_id] = {}; - if (callback != nullptr) { - (*registering_actors_)[actor_id].emplace_back(std::move(callback)); - } - gcs_client_->Actors().AsyncRegisterActor(task_spec, [actor_id, this](Status status) { - std::vector cbs; - cbs = std::move((*registering_actors_)[actor_id]); - registering_actors_->erase(actor_id); - for (auto &cb : cbs) { - cb(status); - } - }); - } + gcs::StatusCallback callback) override; void AsyncRestartActorForLineageReconstruction( const ActorID &actor_id, uint64_t num_restarts_due_to_lineage_reconstructions, - gcs::StatusCallback callback) override { - gcs_client_->Actors().AsyncRestartActorForLineageReconstruction( - actor_id, num_restarts_due_to_lineage_reconstructions, callback); - } + gcs::StatusCallback callback) override; void AsyncReportActorOutOfScope(const ActorID &actor_id, uint64_t num_restarts_due_to_lineage_reconstruction, - gcs::StatusCallback callback) override { - gcs_client_->Actors().AsyncReportActorOutOfScope( - actor_id, num_restarts_due_to_lineage_reconstruction, callback); - } + gcs::StatusCallback callback) override; - bool IsActorInRegistering(const ActorID &actor_id) const override { - return registering_actors_->find(actor_id) != registering_actors_->end(); - } + bool IsActorInRegistering(const ActorID &actor_id) const override; void AsyncWaitForActorRegisterFinish(const ActorID &actor_id, - gcs::StatusCallback callback) override { - auto iter = registering_actors_->find(actor_id); - RAY_CHECK(iter != registering_actors_->end()); - iter->second.emplace_back(std::move(callback)); - } + gcs::StatusCallback callback) override; void AsyncCreateActor( const TaskSpecification &task_spec, - const rpc::ClientCallback &callback) override { - gcs_client_->Actors().AsyncCreateActor(task_spec, callback); - } + const rpc::ClientCallback &callback) override; private: - std::shared_ptr gcs_client_; + gcs::ActorInfoAccessor &actor_client_; using RegisteringActorType = absl::flat_hash_map>; ThreadPrivate registering_actors_; diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index 54db5f6fece9..3750dc5954c5 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -475,7 +475,7 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( timestamp)); }; - auto actor_creator = std::make_shared(gcs_client); + auto actor_creator = std::make_shared(gcs_client->Actors()); auto actor_task_submitter = std::make_unique( *core_worker_client_pool, diff --git a/src/ray/core_worker/fake_actor_creator.h b/src/ray/core_worker/fake_actor_creator.h new file mode 100644 index 000000000000..08deb9bf6cda --- /dev/null +++ b/src/ray/core_worker/fake_actor_creator.h @@ -0,0 +1,63 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "ray/core_worker/actor_creator.h" + +namespace ray { +namespace core { + +class FakeActorCreator : public ActorCreatorInterface { + public: + Status RegisterActor(const TaskSpecification &task_spec) const override { + return Status::OK(); + }; + + void AsyncRegisterActor(const TaskSpecification &task_spec, + gcs::StatusCallback callback) override {} + + void AsyncRestartActorForLineageReconstruction( + const ActorID &actor_id, + uint64_t num_restarts_due_to_lineage_reconstructions, + gcs::StatusCallback callback) override {} + + void AsyncReportActorOutOfScope(const ActorID &actor_id, + uint64_t num_restarts_due_to_lineage_reconstruction, + gcs::StatusCallback callback) override {} + + void AsyncCreateActor( + const TaskSpecification &task_spec, + const rpc::ClientCallback &callback) override {} + + void AsyncWaitForActorRegisterFinish(const ActorID &, + gcs::StatusCallback callback) override { + callbacks.push_back(callback); + } + + [[nodiscard]] bool IsActorInRegistering(const ActorID &actor_id) const override { + return actor_pending; + } + + std::list callbacks; + bool actor_pending = false; +}; + +} // namespace core +} // namespace ray diff --git a/src/ray/core_worker/task_submission/tests/BUILD.bazel b/src/ray/core_worker/task_submission/tests/BUILD.bazel index 4a8ff0ec0d95..b2cf928329af 100644 --- a/src/ray/core_worker/task_submission/tests/BUILD.bazel +++ b/src/ray/core_worker/task_submission/tests/BUILD.bazel @@ -9,6 +9,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:task_common", "//src/ray/common:test_utils", + "//src/ray/core_worker:fake_actor_creator", "//src/ray/core_worker/task_submission:dependency_resolver", "@com_google_googletest//:gtest", ], @@ -49,6 +50,7 @@ ray_cc_test( "//src/ray/common:task_common", "//src/ray/common:test_utils", "//src/ray/core_worker:actor_creator", + "//src/ray/core_worker:fake_actor_creator", "//src/ray/core_worker:reference_count", "//src/ray/core_worker:task_manager", "//src/ray/rpc:core_worker_client", @@ -68,6 +70,7 @@ ray_cc_test( "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/common:task_common", "//src/ray/common:test_utils", + "//src/ray/core_worker:fake_actor_creator", "//src/ray/core_worker:memory_store", "//src/ray/core_worker/task_submission:normal_task_submitter", "//src/ray/rpc:core_worker_client", diff --git a/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc index 96044df6722e..3708ad2274bb 100644 --- a/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc @@ -19,10 +19,10 @@ #include #include "gtest/gtest.h" -#include "mock/ray/core_worker/actor_creator.h" #include "mock/ray/core_worker/reference_count.h" #include "mock/ray/core_worker/task_manager_interface.h" #include "ray/common/test_utils.h" +#include "ray/core_worker/fake_actor_creator.h" #include "ray/rpc/worker/core_worker_client.h" namespace ray::core { @@ -111,7 +111,7 @@ class ActorTaskSubmitterTest : public ::testing::TestWithParam { int num_clients_connected_ = 0; int64_t last_queue_warning_ = 0; - MockActorCreatorInterface actor_creator_; + FakeActorCreator actor_creator_; std::shared_ptr client_pool_; std::shared_ptr worker_client_; std::shared_ptr store_; diff --git a/src/ray/core_worker/task_submission/tests/dependency_resolver_test.cc b/src/ray/core_worker/task_submission/tests/dependency_resolver_test.cc index 21aad49fa165..e9766aec1281 100644 --- a/src/ray/core_worker/task_submission/tests/dependency_resolver_test.cc +++ b/src/ray/core_worker/task_submission/tests/dependency_resolver_test.cc @@ -27,6 +27,7 @@ #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" #include "ray/common/test_utils.h" +#include "ray/core_worker/fake_actor_creator.h" namespace ray { namespace core { @@ -134,49 +135,10 @@ class MockTaskManager : public MockTaskManagerInterface { int num_fail_pending_task_calls = 0; }; -class MockActorCreator : public ActorCreatorInterface { - public: - MockActorCreator() = default; - - Status RegisterActor(const TaskSpecification &task_spec) const override { - return Status::OK(); - }; - - void AsyncRegisterActor(const TaskSpecification &task_spec, - gcs::StatusCallback callback) override {} - - void AsyncCreateActor( - const TaskSpecification &task_spec, - const rpc::ClientCallback &callback) override {} - - void AsyncRestartActorForLineageReconstruction( - const ActorID &actor_id, - uint64_t num_restarts_due_to_lineage_reconstructions, - gcs::StatusCallback callback) override {} - - void AsyncReportActorOutOfScope(const ActorID &actor_id, - uint64_t num_restarts_due_to_lineage_reconstruction, - gcs::StatusCallback callback) override {} - - void AsyncWaitForActorRegisterFinish(const ActorID &, - gcs::StatusCallback callback) override { - callbacks.push_back(callback); - } - - [[nodiscard]] bool IsActorInRegistering(const ActorID &actor_id) const override { - return actor_pending; - } - - ~MockActorCreator() {} - - std::list callbacks; - bool actor_pending = false; -}; - TEST(LocalDependencyResolverTest, TestNoDependencies) { auto store = DefaultCoreWorkerMemoryStoreWithThread::Create(); auto task_manager = std::make_shared(); - MockActorCreator actor_creator; + FakeActorCreator actor_creator; LocalDependencyResolver resolver( *store, *task_manager, actor_creator, [](const ObjectID &object_id) { return rpc::TensorTransport::OBJECT_STORE; @@ -192,7 +154,7 @@ TEST(LocalDependencyResolverTest, TestActorAndObjectDependencies1) { // Actor dependency resolved first. auto store = DefaultCoreWorkerMemoryStoreWithThread::Create(); auto task_manager = std::make_shared(); - MockActorCreator actor_creator; + FakeActorCreator actor_creator; LocalDependencyResolver resolver( *store, *task_manager, actor_creator, [](const ObjectID &object_id) { return rpc::TensorTransport::OBJECT_STORE; @@ -237,7 +199,7 @@ TEST(LocalDependencyResolverTest, TestActorAndObjectDependencies2) { // Object dependency resolved first. auto store = DefaultCoreWorkerMemoryStoreWithThread::Create(); auto task_manager = std::make_shared(); - MockActorCreator actor_creator; + FakeActorCreator actor_creator; LocalDependencyResolver resolver( *store, *task_manager, actor_creator, [](const ObjectID &object_id) { return rpc::TensorTransport::OBJECT_STORE; @@ -281,7 +243,7 @@ TEST(LocalDependencyResolverTest, TestActorAndObjectDependencies2) { TEST(LocalDependencyResolverTest, TestHandlePlasmaPromotion) { auto store = DefaultCoreWorkerMemoryStoreWithThread::Create(); auto task_manager = std::make_shared(); - MockActorCreator actor_creator; + FakeActorCreator actor_creator; LocalDependencyResolver resolver( *store, *task_manager, actor_creator, [](const ObjectID &object_id) { return rpc::TensorTransport::OBJECT_STORE; @@ -311,7 +273,7 @@ TEST(LocalDependencyResolverTest, TestHandlePlasmaPromotion) { TEST(LocalDependencyResolverTest, TestInlineLocalDependencies) { auto store = DefaultCoreWorkerMemoryStoreWithThread::Create(); auto task_manager = std::make_shared(); - MockActorCreator actor_creator; + FakeActorCreator actor_creator; LocalDependencyResolver resolver( *store, *task_manager, actor_creator, [](const ObjectID &object_id) { return rpc::TensorTransport::OBJECT_STORE; @@ -345,7 +307,7 @@ TEST(LocalDependencyResolverTest, TestInlineLocalDependencies) { TEST(LocalDependencyResolverTest, TestInlinePendingDependencies) { auto store = DefaultCoreWorkerMemoryStoreWithThread::Create(); auto task_manager = std::make_shared(); - MockActorCreator actor_creator; + FakeActorCreator actor_creator; LocalDependencyResolver resolver( *store, *task_manager, actor_creator, [](const ObjectID &object_id) { return rpc::TensorTransport::OBJECT_STORE; @@ -383,7 +345,7 @@ TEST(LocalDependencyResolverTest, TestInlinePendingDependencies) { TEST(LocalDependencyResolverTest, TestInlinedObjectIds) { auto store = DefaultCoreWorkerMemoryStoreWithThread::Create(); auto task_manager = std::make_shared(); - MockActorCreator actor_creator; + FakeActorCreator actor_creator; LocalDependencyResolver resolver( *store, *task_manager, actor_creator, [](const ObjectID &object_id) { return rpc::TensorTransport::OBJECT_STORE; @@ -423,7 +385,7 @@ TEST(LocalDependencyResolverTest, TestCancelDependencyResolution) { InstrumentedIOContextWithThread io_context("TestCancelDependencyResolution"); auto store = std::make_shared(io_context.GetIoService()); auto task_manager = std::make_shared(); - MockActorCreator actor_creator; + FakeActorCreator actor_creator; LocalDependencyResolver resolver( *store, *task_manager, actor_creator, [](const ObjectID &object_id) { return rpc::TensorTransport::OBJECT_STORE; @@ -458,7 +420,7 @@ TEST(LocalDependencyResolverTest, TestCancelDependencyResolution) { TEST(LocalDependencyResolverTest, TestDependenciesAlreadyLocal) { auto store = DefaultCoreWorkerMemoryStoreWithThread::Create(); auto task_manager = std::make_shared(); - MockActorCreator actor_creator; + FakeActorCreator actor_creator; LocalDependencyResolver resolver( *store, *task_manager, actor_creator, [](const ObjectID &object_id) { return rpc::TensorTransport::OBJECT_STORE; @@ -494,7 +456,7 @@ TEST(LocalDependencyResolverTest, TestMixedTensorTransport) { // there will be performance regression in some edge cases. auto store = DefaultCoreWorkerMemoryStoreWithThread::Create(); auto task_manager = std::make_shared(); - MockActorCreator actor_creator; + FakeActorCreator actor_creator; // `obj1` is a GPU object, and `obj2` is a normal object. ObjectID obj1 = ObjectID::FromRandom(); diff --git a/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc b/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc index eb52aa92bef5..a2e58b392145 100644 --- a/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc +++ b/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc @@ -33,7 +33,7 @@ class DirectTaskTransportTest : public ::testing::Test { void SetUp() override { gcs_client = std::make_shared(); - actor_creator = std::make_unique(gcs_client); + actor_creator = std::make_unique(gcs_client->Actors()); task_manager = std::make_shared(); client_pool = std::make_shared( @@ -83,7 +83,7 @@ class DirectTaskTransportTest : public ::testing::Test { std::shared_ptr client_pool; std::unique_ptr memory_store; std::shared_ptr task_manager; - std::unique_ptr actor_creator; + std::unique_ptr actor_creator; std::shared_ptr gcs_client; std::shared_ptr reference_counter; }; diff --git a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc index ec3ae6d7b93f..e3eeab9a0a3b 100644 --- a/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/normal_task_submitter_test.cc @@ -29,6 +29,7 @@ #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" #include "ray/common/test_utils.h" +#include "ray/core_worker/fake_actor_creator.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/rpc/raylet/raylet_client_interface.h" #include "ray/rpc/worker/core_worker_client.h" @@ -404,45 +405,6 @@ class MockRayletClient : public FakeRayletClient { get_task_failure_cause_callbacks = {}; }; -class MockActorCreator : public ActorCreatorInterface { - public: - MockActorCreator() {} - - Status RegisterActor(const TaskSpecification &task_spec) const override { - return Status::OK(); - }; - - void AsyncRegisterActor(const TaskSpecification &task_spec, - gcs::StatusCallback callback) override {} - - void AsyncRestartActorForLineageReconstruction( - const ActorID &actor_id, - uint64_t num_restarts_due_to_lineage_reconstructions, - gcs::StatusCallback callback) override {} - - void AsyncReportActorOutOfScope(const ActorID &actor_id, - uint64_t num_restarts_due_to_lineage_reconstruction, - gcs::StatusCallback callback) override {} - - void AsyncCreateActor( - const TaskSpecification &task_spec, - const rpc::ClientCallback &callback) override {} - - void AsyncWaitForActorRegisterFinish(const ActorID &, - gcs::StatusCallback callback) override { - callbacks.push_back(callback); - } - - [[nodiscard]] bool IsActorInRegistering(const ActorID &actor_id) const override { - return actor_pending; - } - - ~MockActorCreator() {} - - std::list callbacks; - bool actor_pending = false; -}; - class MockLeasePolicy : public LeasePolicyInterface { public: void SetNodeID(NodeID node_id) { fallback_rpc_address_.set_node_id(node_id.Binary()); } @@ -484,7 +446,7 @@ class NormalTaskSubmitterTest : public testing::Test { client_pool(std::make_shared( [&](const rpc::Address &) { return worker_client; })), task_manager(std::make_unique()), - actor_creator(std::make_shared()), + actor_creator(std::make_shared()), lease_policy(std::make_unique()), lease_policy_ptr(lease_policy.get()) { address.set_node_id(local_node_id.Binary()); @@ -542,7 +504,7 @@ class NormalTaskSubmitterTest : public testing::Test { std::shared_ptr store; std::shared_ptr client_pool; std::unique_ptr task_manager; - std::shared_ptr actor_creator; + std::shared_ptr actor_creator; // Note: Use lease_policy_ptr in tests, not lease_policy since it has to be moved into // the submitter. std::unique_ptr lease_policy; @@ -1475,7 +1437,7 @@ void TestSchedulingKey(const std::shared_ptr store, auto client_pool = std::make_shared( [&](const rpc::Address &addr) { return worker_client; }); auto task_manager = std::make_unique(); - auto actor_creator = std::make_shared(); + auto actor_creator = std::make_shared(); auto lease_policy = std::make_unique(); lease_policy->SetNodeID(local_node_id); instrumented_io_context io_context; diff --git a/src/ray/core_worker/tests/actor_creator_test.cc b/src/ray/core_worker/tests/actor_creator_test.cc index 7d749d011c47..fc9996cd07cf 100644 --- a/src/ray/core_worker/tests/actor_creator_test.cc +++ b/src/ray/core_worker/tests/actor_creator_test.cc @@ -32,7 +32,7 @@ class ActorCreatorTest : public ::testing::Test { ActorCreatorTest() {} void SetUp() override { gcs_client = std::make_shared(); - actor_creator = std::make_unique(gcs_client); + actor_creator = std::make_unique(gcs_client->Actors()); } TaskSpecification GetTaskSpec(const ActorID &actor_id) { rpc::TaskSpec task_spec; @@ -43,7 +43,7 @@ class ActorCreatorTest : public ::testing::Test { return TaskSpecification(task_spec); } std::shared_ptr gcs_client; - std::unique_ptr actor_creator; + std::unique_ptr actor_creator; }; TEST_F(ActorCreatorTest, IsRegister) { @@ -67,19 +67,19 @@ TEST_F(ActorCreatorTest, AsyncWaitForFinish) { EXPECT_CALL(*gcs_client->mock_actor_accessor, AsyncRegisterActor(::testing::_, ::testing::_, ::testing::_)) .WillRepeatedly(::testing::DoAll(::testing::SaveArg<1>(&cb))); - int cnt = 0; - auto per_finish_cb = [&cnt](Status status) { + int count = 0; + auto per_finish_cb = [&count](Status status) { ASSERT_TRUE(status.ok()); - cnt++; + count++; }; actor_creator->AsyncRegisterActor(task_spec, per_finish_cb); ASSERT_TRUE(actor_creator->IsActorInRegistering(actor_id)); - for (int i = 0; i < 100; ++i) { + for (int i = 0; i < 10; ++i) { actor_creator->AsyncWaitForActorRegisterFinish(actor_id, per_finish_cb); } cb(Status::OK()); ASSERT_FALSE(actor_creator->IsActorInRegistering(actor_id)); - ASSERT_EQ(101, cnt); + ASSERT_EQ(11, count); } } // namespace core diff --git a/src/ray/core_worker/tests/core_worker_test.cc b/src/ray/core_worker/tests/core_worker_test.cc index 76535e79b74b..6770eaaac5a4 100644 --- a/src/ray/core_worker/tests/core_worker_test.cc +++ b/src/ray/core_worker/tests/core_worker_test.cc @@ -180,7 +180,7 @@ class CoreWorkerTest : public ::testing::Test { auto lease_request_rate_limiter = std::make_shared(10); - auto actor_creator = std::make_shared(mock_gcs_client); + auto actor_creator = std::make_shared(mock_gcs_client->Actors()); auto normal_task_submitter = std::make_unique( rpc_address_, From c8f027926a747ccf8a2d3260f01a69007a63722b Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Mon, 8 Sep 2025 22:38:55 -0700 Subject: [PATCH 521/634] [Data] Remove filesystem parameterizations of `test_csv` tests (#56345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? Many `test_csv` tests include variants for different PyArrow filesystems, but this is unnecessary. Assuming the PyArrow filesystem abstraction works correctly, the underlying filesystem shouldn’t affect behavior. Also, `FileBasedDatasource` subclasses don’t interact with the filesystem directly—they just receive a stream and define how to deserialize it. So, it’s cleaner and simpler to test filesystem variants at the `FileBasedDatasource` base class level rather than in every subclass, reducing test duplication and complexity. ## Related issue number See also https://github.com/ray-project/ray/pull/56341 ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Balaji Veeramani --- python/ray/data/tests/test_csv.py | 219 +++++------------- .../data/tests/test_file_based_datasource.py | 49 ++++ 2 files changed, 101 insertions(+), 167 deletions(-) diff --git a/python/ray/data/tests/test_csv.py b/python/ray/data/tests/test_csv.py index 9ba6e63bb60b..6beca52fe113 100644 --- a/python/ray/data/tests/test_csv.py +++ b/python/ray/data/tests/test_csv.py @@ -6,7 +6,6 @@ import pyarrow.parquet as pq import pytest from packaging.version import Version -from pytest_lazy_fixtures import lf as lazy_fixture import ray from ray.data import Schema @@ -29,40 +28,14 @@ def df_to_csv(dataframe, path, **kwargs): dataframe.to_csv(path, **kwargs) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ( - lazy_fixture("s3_fs_with_space"), - lazy_fixture("s3_path_with_space"), - lazy_fixture("s3_server"), - ), - ( - lazy_fixture("s3_fs_with_special_chars"), - lazy_fixture("s3_path_with_special_chars"), - lazy_fixture("s3_server"), - ), - ], -) def test_csv_read( - ray_start_regular_shared, - fs, - data_path, - endpoint_url, - target_max_block_size_infinite_or_default, + ray_start_regular_shared, tmp_path, target_max_block_size_infinite_or_default ): - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) # Single file. df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - path1 = os.path.join(data_path, "test1.csv") - df1.to_csv(path1, index=False, storage_options=storage_options) - ds = ray.data.read_csv(path1, filesystem=fs, partitioning=None) + path1 = os.path.join(tmp_path, "test1.csv") + df1.to_csv(path1, index=False) + ds = ray.data.read_csv(path1, partitioning=None) dsdf = ds.to_pandas().sort_values(by=["one", "two"]).reset_index(drop=True) assert df1.equals(dsdf) # Test metadata ops. @@ -72,11 +45,9 @@ def test_csv_read( # Two files, override_num_blocks=2. df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) - path2 = os.path.join(data_path, "test2.csv") - df2.to_csv(path2, index=False, storage_options=storage_options) - ds = ray.data.read_csv( - [path1, path2], override_num_blocks=2, filesystem=fs, partitioning=None - ) + path2 = os.path.join(tmp_path, "test2.csv") + df2.to_csv(path2, index=False) + ds = ray.data.read_csv([path1, path2], override_num_blocks=2, partitioning=None) dsdf = ds.to_pandas().sort_values(by=["one", "two"]).reset_index(drop=True) df = pd.concat([df1, df2], ignore_index=True) assert df.equals(dsdf) @@ -86,12 +57,11 @@ def test_csv_read( # Three files, override_num_blocks=2. df3 = pd.DataFrame({"one": [7, 8, 9], "two": ["h", "i", "j"]}) - path3 = os.path.join(data_path, "test3.csv") - df3.to_csv(path3, index=False, storage_options=storage_options) + path3 = os.path.join(tmp_path, "test3.csv") + df3.to_csv(path3, index=False) ds = ray.data.read_csv( [path1, path2, path3], override_num_blocks=2, - filesystem=fs, partitioning=None, ) df = pd.concat([df1, df2, df3], ignore_index=True) @@ -99,136 +69,89 @@ def test_csv_read( assert df.equals(dsdf) # Directory, two files. - path = os.path.join(data_path, "test_csv_dir") - if fs is None: - os.mkdir(path) - else: - fs.create_dir(_unwrap_protocol(path)) + path = os.path.join(tmp_path, "test_csv_dir") + os.mkdir(path) df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) path1 = os.path.join(path, "data0.csv") - df1.to_csv(path1, index=False, storage_options=storage_options) + df1.to_csv(path1, index=False) df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) path2 = os.path.join(path, "data1.csv") - df2.to_csv(path2, index=False, storage_options=storage_options) - ds = ray.data.read_csv(path, filesystem=fs, partitioning=None) + df2.to_csv(path2, index=False) + ds = ray.data.read_csv(path, partitioning=None) df = pd.concat([df1, df2], ignore_index=True) dsdf = ds.to_pandas().sort_values(by=["one", "two"]).reset_index(drop=True) pd.testing.assert_frame_equal(df, dsdf) - if fs is None: - shutil.rmtree(path) - else: - fs.delete_dir(_unwrap_protocol(path)) + shutil.rmtree(path) # Two directories, three files. - path1 = os.path.join(data_path, "test_csv_dir1") - path2 = os.path.join(data_path, "test_csv_dir2") - if fs is None: - os.mkdir(path1) - os.mkdir(path2) - else: - fs.create_dir(_unwrap_protocol(path1)) - fs.create_dir(_unwrap_protocol(path2)) + path1 = os.path.join(tmp_path, "test_csv_dir1") + path2 = os.path.join(tmp_path, "test_csv_dir2") + os.mkdir(path1) + os.mkdir(path2) df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) file_path1 = os.path.join(path1, "data0.csv") - df1.to_csv(file_path1, index=False, storage_options=storage_options) + df1.to_csv(file_path1, index=False) df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) file_path2 = os.path.join(path2, "data1.csv") - df2.to_csv(file_path2, index=False, storage_options=storage_options) + df2.to_csv(file_path2, index=False) df3 = pd.DataFrame({"one": [7, 8, 9], "two": ["h", "i", "j"]}) file_path3 = os.path.join(path2, "data2.csv") - df3.to_csv(file_path3, index=False, storage_options=storage_options) - ds = ray.data.read_csv([path1, path2], filesystem=fs, partitioning=None) + df3.to_csv(file_path3, index=False) + ds = ray.data.read_csv([path1, path2], partitioning=None) df = pd.concat([df1, df2, df3], ignore_index=True) dsdf = ds.to_pandas().sort_values(by=["one", "two"]).reset_index(drop=True) assert df.equals(dsdf) - if fs is None: - shutil.rmtree(path1) - shutil.rmtree(path2) - else: - fs.delete_dir(_unwrap_protocol(path1)) - fs.delete_dir(_unwrap_protocol(path2)) + shutil.rmtree(path1) + shutil.rmtree(path2) # Directory and file, two files. - dir_path = os.path.join(data_path, "test_csv_dir") - if fs is None: - os.mkdir(dir_path) - else: - fs.create_dir(_unwrap_protocol(dir_path)) + dir_path = os.path.join(tmp_path, "test_csv_dir") + os.mkdir(dir_path) df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) path1 = os.path.join(dir_path, "data0.csv") - df1.to_csv(path1, index=False, storage_options=storage_options) + df1.to_csv(path1, index=False) df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) - path2 = os.path.join(data_path, "data1.csv") - df2.to_csv(path2, index=False, storage_options=storage_options) - ds = ray.data.read_csv([dir_path, path2], filesystem=fs, partitioning=None) + path2 = os.path.join(tmp_path, "data1.csv") + df2.to_csv(path2, index=False) + ds = ray.data.read_csv([dir_path, path2], partitioning=None) df = pd.concat([df1, df2], ignore_index=True) dsdf = ds.to_pandas().sort_values(by=["one", "two"]).reset_index(drop=True) assert df.equals(dsdf) - if fs is None: - shutil.rmtree(dir_path) - else: - fs.delete_dir(_unwrap_protocol(dir_path)) + shutil.rmtree(dir_path) # Directory, two files and non-csv file (test extension-based path filtering). - path = os.path.join(data_path, "test_csv_dir") - if fs is None: - os.mkdir(path) - else: - fs.create_dir(_unwrap_protocol(path)) + path = os.path.join(tmp_path, "test_csv_dir") + os.mkdir(path) df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) path1 = os.path.join(path, "data0.csv") - df1.to_csv(path1, index=False, storage_options=storage_options) + df1.to_csv(path1, index=False) df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) path2 = os.path.join(path, "data1.csv") - df2.to_csv(path2, index=False, storage_options=storage_options) + df2.to_csv(path2, index=False) # Add a file with a non-matching file extension. This file should be ignored. df_txt = pd.DataFrame({"foobar": [1, 2, 3]}) df_txt.to_json( os.path.join(path, "foo.txt"), - storage_options=storage_options, ) ds = ray.data.read_csv( path, - filesystem=fs, file_extensions=["csv"], partitioning=None, ) df = pd.concat([df1, df2], ignore_index=True) dsdf = ds.to_pandas().sort_values(by=["one", "two"]).reset_index(drop=True) assert df.equals(dsdf) - if fs is None: - shutil.rmtree(path) - else: - fs.delete_dir(_unwrap_protocol(path)) - - -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) -def test_csv_read_meta_provider( - ray_start_regular_shared, - fs, - data_path, - endpoint_url, -): - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) + shutil.rmtree(path) + +def test_csv_read_meta_provider(ray_start_regular_shared, tmp_path): df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - path1 = os.path.join(data_path, "test1.csv") - df1.to_csv(path1, index=False, storage_options=storage_options) + path1 = os.path.join(tmp_path, "test1.csv") + df1.to_csv(path1, index=False) ds = ray.data.read_csv( path1, - filesystem=fs, meta_provider=FastFileMetadataProvider(), ) @@ -243,73 +166,35 @@ def test_csv_read_meta_provider( with pytest.raises(NotImplementedError): ray.data.read_csv( path1, - filesystem=fs, meta_provider=BaseFileMetadataProvider(), ) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) -def test_csv_read_many_files_basic( - ray_start_regular_shared, - fs, - data_path, - endpoint_url, -): - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) - +def test_csv_read_many_files_basic(ray_start_regular_shared, tmp_path): paths = [] dfs = [] num_dfs = 4 * FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD for i in range(num_dfs): df = pd.DataFrame({"one": list(range(i * 3, (i + 1) * 3))}) dfs.append(df) - path = os.path.join(data_path, f"test_{i}.csv") + path = os.path.join(tmp_path, f"test_{i}.csv") paths.append(path) - df.to_csv(path, index=False, storage_options=storage_options) - ds = ray.data.read_csv(paths, filesystem=fs) + df.to_csv(path, index=False) + ds = ray.data.read_csv(paths) dsdf = ds.to_pandas() df = pd.concat(dfs).reset_index(drop=True) pd.testing.assert_frame_equal(df, dsdf) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) def test_csv_read_many_files_diff_dirs( ray_start_regular_shared, - fs, - data_path, - endpoint_url, + tmp_path, ): - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) - - dir1 = os.path.join(data_path, "dir1") - dir2 = os.path.join(data_path, "dir2") - if fs is None: - os.mkdir(dir1) - os.mkdir(dir2) - else: - fs.create_dir(_unwrap_protocol(dir1)) - fs.create_dir(_unwrap_protocol(dir2)) + dir1 = os.path.join(tmp_path, "dir1") + dir2 = os.path.join(tmp_path, "dir2") + os.mkdir(dir1) + os.mkdir(dir2) paths = [] dfs = [] @@ -320,8 +205,8 @@ def test_csv_read_many_files_diff_dirs( dfs.append(df) path = os.path.join(dir_path, f"test_{j}.csv") paths.append(path) - df.to_csv(path, index=False, storage_options=storage_options) - ds = ray.data.read_csv([dir1, dir2], filesystem=fs) + df.to_csv(path, index=False) + ds = ray.data.read_csv([dir1, dir2]) dsdf = ds.to_pandas().sort_values(by=["one"]).reset_index(drop=True) df = pd.concat(dfs).reset_index(drop=True) diff --git a/python/ray/data/tests/test_file_based_datasource.py b/python/ray/data/tests/test_file_based_datasource.py index 5d2df56f3cf2..fbf8c548bb44 100644 --- a/python/ray/data/tests/test_file_based_datasource.py +++ b/python/ray/data/tests/test_file_based_datasource.py @@ -1,8 +1,10 @@ import os from typing import Any, Dict, Iterator, List +from urllib.parse import urlparse import pyarrow import pytest +from pytest_lazy_fixtures import lf as lazy_fixture import ray from ray.data._internal.delegating_block_builder import DelegatingBlockBuilder @@ -41,6 +43,53 @@ def execute_read_tasks(tasks: List[ReadTask]) -> List[Dict[str, Any]]: return rows +def strip_scheme(uri): + """Remove scheme from a URI, if it exists.""" + parsed = urlparse(uri) + if parsed.scheme: + return uri.split("://", 1)[1] # remove scheme + return uri # no scheme, return as-is + + +@pytest.mark.parametrize( + "filesystem,dir_path,endpoint_url", + [ + (None, lazy_fixture("local_path"), None), + (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), + (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), + ( + lazy_fixture("s3_fs_with_space"), + lazy_fixture("s3_path_with_space"), + lazy_fixture("s3_server"), + ), + ( + lazy_fixture("s3_fs_with_special_chars"), + lazy_fixture("s3_path_with_special_chars"), + lazy_fixture("s3_server"), + ), + ], +) +def test_read_single_file(ray_start_regular_shared, filesystem, dir_path, endpoint_url): + # `FileBasedDatasource` should read from the local filesystem if you don't specify + # one. + write_filesystem = filesystem + if write_filesystem is None: + write_filesystem = pyarrow.fs.LocalFileSystem() + + # PyArrow filesystems expect paths without schemes. `FileBasedDatasource` handles + # this internally, but we need to manually strip the scheme for the test setup. + write_path = strip_scheme(os.path.join(dir_path, "file.txt")) + with write_filesystem.open_output_stream(write_path) as f: + f.write(b"spam") + + datasource = MockFileBasedDatasource(dir_path, filesystem=filesystem) + tasks = datasource.get_read_tasks(1) + + rows = execute_read_tasks(tasks) + + assert rows == [{"data": b"spam"}] + + def test_partitioning_hive(ray_start_regular_shared, tmp_path): path = os.path.join(tmp_path, "country=us") os.mkdir(path) From e8d87afd5bbf85cab02061859fc537e821afe670 Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Mon, 8 Sep 2025 23:48:06 -0700 Subject: [PATCH 522/634] [core] Banning implementation deps (#56359) Signed-off-by: joshlee --- src/ray/core_worker/BUILD.bazel | 4 +-- .../core_worker/task_submission/BUILD.bazel | 8 ++--- src/ray/gcs/gcs_server/BUILD.bazel | 35 +++++++------------ src/ray/pubsub/BUILD.bazel | 6 ++-- 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index 608f66964e61..e2c3ec5b77d9 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -13,9 +13,6 @@ ray_cc_library( "core_worker_rpc_proxy.h", "core_worker_shutdown_executor.h", ], - implementation_deps = [ - "//src/ray/util:time", - ], deps = [ ":actor_handle", ":actor_manager", @@ -60,6 +57,7 @@ ray_cc_library( "//src/ray/util:shared_lru", "//src/ray/util:stream_redirection", "//src/ray/util:stream_redirection_options", + "//src/ray/util:time", "@com_google_absl//absl/cleanup", "@com_google_absl//absl/strings", "@com_google_googletest//:gtest_prod", diff --git a/src/ray/core_worker/task_submission/BUILD.bazel b/src/ray/core_worker/task_submission/BUILD.bazel index 7054fc3e68cc..53fcf8b306d1 100644 --- a/src/ray/core_worker/task_submission/BUILD.bazel +++ b/src/ray/core_worker/task_submission/BUILD.bazel @@ -57,9 +57,6 @@ ray_cc_library( name = "actor_task_submitter", srcs = ["actor_task_submitter.cc"], hdrs = ["actor_task_submitter.h"], - implementation_deps = [ - "//src/ray/util:time", - ], visibility = [ ":__subpackages__", "//src/ray/core_worker:__pkg__", @@ -74,6 +71,7 @@ ray_cc_library( "//src/ray/common:protobuf_utils", "//src/ray/core_worker:actor_creator", "//src/ray/rpc:core_worker_client", + "//src/ray/util:time", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", @@ -84,9 +82,6 @@ ray_cc_library( name = "normal_task_submitter", srcs = ["normal_task_submitter.cc"], hdrs = ["normal_task_submitter.h"], - implementation_deps = [ - "//src/ray/util:time", - ], visibility = [ ":__subpackages__", "//src/ray/core_worker:__pkg__", @@ -101,6 +96,7 @@ ray_cc_library( "//src/ray/core_worker:task_manager_interface", "//src/ray/rpc:core_worker_client", "//src/ray/rpc:raylet_client_interface", + "//src/ray/util:time", "@com_google_absl//absl/base:core_headers", ], ) diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 42b88c0c16f3..f34da0076caf 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -357,9 +357,6 @@ ray_cc_library( hdrs = [ "gcs_actor.h", ], - implementation_deps = [ - "//src/ray/util:logging", - ], deps = [ "//src/ray/common:id", "//src/ray/common:lease", @@ -369,6 +366,7 @@ ray_cc_library( "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/util:counter_map", "//src/ray/util:event", + "//src/ray/util:logging", ], ) @@ -380,22 +378,20 @@ ray_cc_library( hdrs = [ "gcs_actor_scheduler.h", ], - implementation_deps = [ - "//src/ray/common:ray_config", - "//src/ray/util:time", - ], deps = [ ":gcs_actor", ":gcs_node_manager", ":gcs_table_storage", "//src/ray/common:asio", "//src/ray/common:id", + "//src/ray/common:ray_config", "//src/ray/common:task_common", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/raylet/scheduling:cluster_lease_manager", "//src/ray/rpc:core_worker_client", "//src/ray/rpc:raylet_client_interface", "//src/ray/util:logging", + "//src/ray/util:time", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_googletest//:gtest", @@ -410,14 +406,6 @@ ray_cc_library( hdrs = [ "gcs_actor_manager.h", ], - implementation_deps = [ - "//src/ray/common:protobuf_utils", - "//src/ray/common:ray_config", - "//src/ray/common:task_common", - "//src/ray/stats:stats_lib", - "//src/ray/util:logging", - "//src/ray/util:time", - ], deps = [ ":gcs_actor", ":gcs_actor_scheduler", @@ -428,12 +416,17 @@ ray_cc_library( ":grpc_service_interfaces", "//src/ray/common:asio", "//src/ray/common:id", + "//src/ray/common:protobuf_utils", + "//src/ray/common:ray_config", + "//src/ray/common:task_common", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/pubsub:gcs_publisher", "//src/ray/rpc:core_worker_client", + "//src/ray/stats:stats_lib", "//src/ray/util:counter_map", "//src/ray/util:logging", "//src/ray/util:thread_checker", + "//src/ray/util:time", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_googletest//:gtest", @@ -448,13 +441,6 @@ ray_cc_library( hdrs = [ "gcs_autoscaler_state_manager.h", ], - implementation_deps = [ - "//src/ray/common:protobuf_utils", - "//src/ray/common:ray_config", - "//src/ray/util:logging", - "//src/ray/util:string_utils", - "//src/ray/util:time", - ], deps = [ ":gcs_actor_manager", ":gcs_init_data", @@ -465,9 +451,14 @@ ray_cc_library( ":grpc_service_interfaces", "//src/ray/common:asio", "//src/ray/common:id", + "//src/ray/common:protobuf_utils", + "//src/ray/common:ray_config", "//src/ray/protobuf:gcs_cc_proto", "//src/ray/pubsub:gcs_publisher", + "//src/ray/util:logging", + "//src/ray/util:string_utils", "//src/ray/util:thread_checker", + "//src/ray/util:time", "@com_google_absl//absl/container:flat_hash_map", "@com_google_googletest//:gtest", ], diff --git a/src/ray/pubsub/BUILD.bazel b/src/ray/pubsub/BUILD.bazel index 2fce880c573b..01552be0f3da 100644 --- a/src/ray/pubsub/BUILD.bazel +++ b/src/ray/pubsub/BUILD.bazel @@ -80,15 +80,13 @@ ray_cc_library( name = "python_gcs_subscriber", srcs = ["python_gcs_subscriber.cc"], hdrs = ["python_gcs_subscriber.h"], - implementation_deps = [ - "//src/ray/rpc:gcs_client", - "@com_github_grpc_grpc//:grpc++", - ], deps = [ "//src/ray/common:status", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/protobuf:pubsub_cc_proto", + "//src/ray/rpc:gcs_client", "//src/ray/util:visibility", + "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/synchronization", ], ) From 0fbe0176cc8124b2083a886d6d26b0da2a1a1060 Mon Sep 17 00:00:00 2001 From: Lehui Liu Date: Tue, 9 Sep 2025 06:13:42 -0700 Subject: [PATCH 523/634] [RLlib][release test] Upgrade g3 to g4 machine for aws release test (#56248) ## Why are these changes needed? There are a few release test in core daily test that are using g3 machines: [RLlib rllib_learning_tests_poing_impala_torch](https://github.com/ray-project/ray/blob/master/release/release_tests.yaml#L2754-L2777) [rllib_learning_tests_pendulum_dreamerv3_torch](https://github.com/ray-project/ray/blob/master/release/release_tests.yaml#L2714C9-L2733) Since G3 machine are old and have lower availability, update to use newer machine, e.g. g4dn.8xlarge can reduce the stock out flakiness. A successful RLlib release test run: https://buildkite.com/ray-project/release/builds/56710 ## Related issue number Closes #56178 ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [x] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Lehui Liu --- release/rllib_tests/2gpus_32cpus.yaml | 2 +- release/rllib_tests/2gpus_64cpus.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/release/rllib_tests/2gpus_32cpus.yaml b/release/rllib_tests/2gpus_32cpus.yaml index 02065ef9dc8f..06739cff5739 100644 --- a/release/rllib_tests/2gpus_32cpus.yaml +++ b/release/rllib_tests/2gpus_32cpus.yaml @@ -5,7 +5,7 @@ max_workers: 0 head_node_type: name: head_node - instance_type: g3.8xlarge + instance_type: g4dn.12xlarge worker_node_types: [] diff --git a/release/rllib_tests/2gpus_64cpus.yaml b/release/rllib_tests/2gpus_64cpus.yaml index bd7f534c1fdf..d1a1d0b54dca 100644 --- a/release/rllib_tests/2gpus_64cpus.yaml +++ b/release/rllib_tests/2gpus_64cpus.yaml @@ -5,7 +5,7 @@ max_workers: 1 head_node_type: name: head_node - instance_type: g3.8xlarge + instance_type: g4dn.12xlarge worker_node_types: - name: worker_node From f17ba98f1c3c6badaee996ff84661753fd8d7679 Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Tue, 9 Sep 2025 19:08:57 +0530 Subject: [PATCH 524/634] [core] Fix error handling for plasma put errors (#56070) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? Followup to https://github.com/ray-project/ray/pull/55367#pullrequestreview-3153215777 - map Status -> ErrorType via MapStatusToErrorType (IOError->LOCAL_RAYLET_DIED, ObjectStoreFull/Transient->OUT_OF_MEMORY, OutOfDisk->OUT_OF_DISK_ERROR; default->WORKER_DIED). - no longer log‑and‑forget on any HandleTaskReturn error. - For streaming generator returns we now always put the error into the in‑memory store first (unblocks waiters) and only then best‑effort put to plasma; failures there are just warnings. --------- Signed-off-by: Sagar Sumit --- src/ray/core_worker/core_worker_process.cc | 24 +++++--- src/ray/core_worker/task_manager.cc | 64 +++++++++++++++++----- src/ray/core_worker/task_manager.h | 6 +- 3 files changed, 69 insertions(+), 25 deletions(-) diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index 3750dc5954c5..f514ce9f211c 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -425,14 +425,24 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( /*put_in_local_plasma_callback=*/ [this](const RayObject &object, const ObjectID &object_id) { auto core_worker = GetCoreWorker(); - auto put_status = - core_worker->PutInLocalPlasmaStore(object, object_id, /*pin_object=*/true); - if (!put_status.ok()) { - RAY_LOG(WARNING).WithField(object_id) - << "Failed to put object in plasma store: " << put_status; - return put_status; + constexpr int max_retries = 3; + int attempt = 0; + int64_t backoff_ms = 10; + Status put_status; + while (attempt++ < max_retries) { + put_status = + core_worker->PutInLocalPlasmaStore(object, object_id, /*pin_object=*/true); + if (put_status.ok()) { + return Status::OK(); + } + // Backoff before retrying. + std::this_thread::sleep_for(std::chrono::milliseconds(backoff_ms)); + backoff_ms *= 2; } - return Status::OK(); + RAY_LOG(WARNING).WithField(object_id) + << "Exhausted plasma put retries (attempts=" << attempt + << ") with status: " << put_status; + return put_status; }, /* retry_task_callback= */ [this](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) { diff --git a/src/ray/core_worker/task_manager.cc b/src/ray/core_worker/task_manager.cc index c058ec121bbd..a78b481a76e8 100644 --- a/src/ray/core_worker/task_manager.cc +++ b/src/ray/core_worker/task_manager.cc @@ -38,6 +38,31 @@ constexpr int64_t kTaskFailureThrottlingThreshold = 50; // Throttle task failure logs to once this interval. constexpr int64_t kTaskFailureLoggingFrequencyMillis = 5000; +namespace { + +rpc::ErrorType MapPlasmaPutStatusToErrorType(const Status &status) { + // Only the following should be returned from plasma put paths today. + RAY_DCHECK(status.IsObjectStoreFull() || status.IsTransientObjectStoreFull() || + status.IsOutOfDisk() || status.IsIOError()) + << "Unexpected status from plasma put: " << status; + + if (status.IsObjectStoreFull() || status.IsTransientObjectStoreFull()) { + // TODO(codope): add a dedicated OBJECT_STORE_FULL error type and map to it. + // https://github.com/ray-project/ray/pull/56070 + return rpc::ErrorType::OUT_OF_MEMORY; + } + if (status.IsOutOfDisk()) { + return rpc::ErrorType::OUT_OF_DISK_ERROR; + } + if (status.IsIOError()) { + // Local IPC failure to plasma/raylet; attribute to local control-plane failure. + return rpc::ErrorType::LOCAL_RAYLET_DIED; + } + return rpc::ErrorType::WORKER_DIED; +} + +} // namespace + absl::flat_hash_set ObjectRefStream::GetItemsUnconsumed() const { absl::flat_hash_set result; for (int64_t index = 0; index <= max_index_seen_; index++) { @@ -580,11 +605,6 @@ StatusOr TaskManager::HandleTaskReturn(const ObjectID &object_id, tensor_transport.value_or(rpc::TensorTransport::OBJECT_STORE)); if (store_in_plasma) { Status s = put_in_local_plasma_callback_(object, object_id); - int retry_count = 0; - while (!s.ok() && s.IsTransientObjectStoreFull() && retry_count < 3) { - retry_count++; - s = put_in_local_plasma_callback_(object, object_id); - } if (!s.ok()) { return s; } @@ -922,10 +942,7 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, RAY_LOG(WARNING).WithField(object_id) << "Failed to handle dynamic task return: " << direct_or.status(); Status st = direct_or.status(); - rpc::ErrorType err_type = rpc::ErrorType::WORKER_DIED; - if (st.IsObjectStoreFull() || st.IsTransientObjectStoreFull()) { - err_type = rpc::ErrorType::OUT_OF_MEMORY; - } + rpc::ErrorType err_type = MapPlasmaPutStatusToErrorType(st); rpc::RayErrorInfo err_info; err_info.set_error_message(st.ToString()); FailOrRetryPendingTask(task_id, @@ -953,10 +970,13 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, // If storing return in plasma failed, treat as system failure for this attempt. // Do not proceed with normal completion. Mark task failed immediately. Status st = direct_or.status(); + rpc::ErrorType err_type = MapPlasmaPutStatusToErrorType(st); + rpc::RayErrorInfo err_info; + err_info.set_error_message(st.ToString()); FailOrRetryPendingTask(task_id, - rpc::ErrorType::WORKER_DIED, + err_type, &st, - /*ray_error_info=*/nullptr, + /*ray_error_info=*/&err_info, /*mark_task_object_failed=*/true, /*fail_immediately=*/true); return; @@ -1093,6 +1113,17 @@ void TaskManager::CompletePendingTask(const TaskID &task_id, RAY_LOG(WARNING).WithField(generator_return_id) << "Failed to handle generator return during app error propagation: " << res.status(); + Status st = res.status(); + rpc::ErrorType err_type = MapPlasmaPutStatusToErrorType(st); + rpc::RayErrorInfo err_info; + err_info.set_error_message(st.ToString()); + FailOrRetryPendingTask(spec.TaskId(), + err_type, + &st, + /*ray_error_info=*/&err_info, + /*mark_task_object_failed=*/true, + /*fail_immediately=*/true); + return; } } } @@ -1504,26 +1535,28 @@ void TaskManager::MarkTaskReturnObjectsFailed( int64_t num_returns = spec.NumReturns(); for (int i = 0; i < num_returns; i++) { const auto object_id = ObjectID::FromIndex(task_id, /*index=*/i + 1); - // Always place an error marker in local memory to unblock waiters quickly. - in_memory_store_.Put(error, object_id); - // Best-effort plasma put if the object was meant to be in plasma. if (store_in_plasma_ids.contains(object_id)) { Status s = put_in_local_plasma_callback_(error, object_id); if (!s.ok()) { RAY_LOG(WARNING).WithField(object_id) << "Failed to put error object in plasma: " << s; + in_memory_store_.Put(error, object_id); } + } else { + in_memory_store_.Put(error, object_id); } } if (spec.ReturnsDynamic()) { for (const auto &dynamic_return_id : spec.DynamicReturnIds()) { - in_memory_store_.Put(error, dynamic_return_id); if (store_in_plasma_ids.contains(dynamic_return_id)) { Status s = put_in_local_plasma_callback_(error, dynamic_return_id); if (!s.ok()) { RAY_LOG(WARNING).WithField(dynamic_return_id) << "Failed to put error object in plasma: " << s; + in_memory_store_.Put(error, dynamic_return_id); } + } else { + in_memory_store_.Put(error, dynamic_return_id); } } } @@ -1550,6 +1583,7 @@ void TaskManager::MarkTaskReturnObjectsFailed( if (!s.ok()) { RAY_LOG(WARNING).WithField(generator_return_id) << "Failed to put error object in plasma: " << s; + in_memory_store_.Put(error, generator_return_id); } } else { in_memory_store_.Put(error, generator_return_id); diff --git a/src/ray/core_worker/task_manager.h b/src/ray/core_worker/task_manager.h index f4d85337a0d1..8da151499087 100644 --- a/src/ray/core_worker/task_manager.h +++ b/src/ray/core_worker/task_manager.h @@ -614,9 +614,9 @@ class TaskManager : public TaskManagerInterface { void MarkTaskNoRetryInternal(const TaskID &task_id, bool canceled) ABSL_LOCKS_EXCLUDED(mu_); - /// Update nested ref count info and store the in-memory value for a task's - /// return object. On success, sets direct_return_out to true if the object's value - /// was returned directly by value (not stored in plasma). + /// Update nested ref count info and store the task's return object. + /// Returns StatusOr where the bool indicates the object was returned + /// directly in-memory (not stored in plasma) when true. StatusOr HandleTaskReturn(const ObjectID &object_id, const rpc::ReturnObject &return_object, const NodeID &worker_node_id, From 3d082299e531741f619f29ec54e90f6b607686fe Mon Sep 17 00:00:00 2001 From: Haichuan Hu <74917084+KaisennHu@users.noreply.github.com> Date: Tue, 9 Sep 2025 21:43:28 +0800 Subject: [PATCH 525/634] [Tech-debt] Unify the Deprecation APIs across Ray libraries (#56326) It seems like that different serve libraries have different utilities for deprecation of classes and functions, it would be good to unify these utilities and put them under `ray/_common/deprecation.py` and remove the overhead of maintaining these individually across different libraries. Unit tests pass: image ## Related issue number Closes #56202 --------- Signed-off-by: Haichuan Hu --- .../ray/_common}/deprecation.py | 2 +- python/ray/_common/tests/BUILD | 1 + python/ray/_common/tests/test_deprecation.py | 95 ++++++++++++ .../llm/_internal/common/utils/deprecation.py | 136 ------------------ python/ray/serve/llm/__init__.py | 3 +- rllib/algorithms/algorithm.py | 2 +- rllib/algorithms/algorithm_config.py | 2 +- rllib/algorithms/appo/appo.py | 2 +- rllib/algorithms/appo/appo_rl_module.py | 2 +- .../appo/torch/appo_torch_rl_module.py | 2 +- rllib/algorithms/cql/cql.py | 2 +- rllib/algorithms/dqn/dqn.py | 2 +- rllib/algorithms/impala/impala.py | 2 +- rllib/algorithms/marwil/marwil.py | 2 +- rllib/algorithms/ppo/ppo.py | 2 +- rllib/algorithms/ppo/ppo_rl_module.py | 2 +- .../ppo/torch/ppo_torch_rl_module.py | 2 +- rllib/algorithms/sac/sac.py | 2 +- rllib/core/learner/learner.py | 2 +- rllib/core/learner/learner_group.py | 2 +- rllib/core/models/catalog.py | 2 +- rllib/core/models/specs/specs_base.py | 2 +- rllib/core/models/specs/specs_dict.py | 2 +- rllib/core/rl_module/multi_rl_module.py | 2 +- rllib/core/rl_module/rl_module.py | 2 +- rllib/env/env_runner_group.py | 2 +- rllib/env/external_env.py | 2 +- rllib/env/multi_agent_env.py | 2 +- rllib/env/multi_agent_env_runner.py | 2 +- rllib/env/multi_agent_episode.py | 2 +- rllib/env/single_agent_env_runner.py | 2 +- rllib/env/single_agent_episode.py | 2 +- rllib/env/utils/external_env_protocol.py | 2 +- rllib/evaluation/sample_batch_builder.py | 2 +- rllib/evaluation/sampler.py | 2 +- rllib/evaluation/worker_set.py | 2 +- .../utils/self_play_callback_old_api_stack.py | 2 +- ...lay_league_based_callback_old_api_stack.py | 2 +- rllib/execution/multi_gpu_learner_thread.py | 2 +- rllib/execution/train_ops.py | 2 +- rllib/models/catalog.py | 2 +- rllib/models/distributions.py | 2 +- rllib/models/modelv2.py | 2 +- rllib/models/tf/attention_net.py | 2 +- rllib/models/tf/layers/gru_gate.py | 2 +- .../models/tf/layers/multi_head_attention.py | 2 +- rllib/models/tf/layers/noisy_layer.py | 2 +- .../layers/relative_multi_head_attention.py | 2 +- rllib/models/tf/layers/skip_connection.py | 2 +- rllib/models/tf/recurrent_net.py | 2 +- rllib/models/tf/tf_modelv2.py | 2 +- rllib/models/torch/attention_net.py | 2 +- rllib/models/torch/mingpt.py | 2 +- rllib/models/torch/recurrent_net.py | 2 +- rllib/models/torch/torch_distributions.py | 2 +- .../offline/estimators/feature_importance.py | 2 +- .../estimators/off_policy_estimator.py | 2 +- rllib/offline/is_estimator.py | 2 +- rllib/offline/off_policy_estimator.py | 2 +- rllib/offline/wis_estimator.py | 2 +- rllib/policy/dynamic_tf_policy.py | 2 +- rllib/policy/eager_tf_policy.py | 2 +- rllib/policy/policy.py | 2 +- rllib/policy/policy_map.py | 2 +- rllib/policy/sample_batch.py | 2 +- rllib/policy/tf_policy.py | 2 +- rllib/policy/tf_policy_template.py | 2 +- rllib/tests/run_regression_tests.py | 2 +- rllib/utils/__init__.py | 2 +- rllib/utils/annotations.py | 2 +- rllib/utils/filter.py | 4 +- rllib/utils/framework.py | 2 +- rllib/utils/memory.py | 2 +- rllib/utils/metrics/metrics_logger.py | 2 +- rllib/utils/numpy.py | 2 +- rllib/utils/policy.py | 2 +- .../multi_agent_replay_buffer.py | 2 +- rllib/utils/replay_buffers/utils.py | 4 +- rllib/utils/tests/run_memory_leak_tests.py | 2 +- 79 files changed, 174 insertions(+), 215 deletions(-) rename {rllib/utils => python/ray/_common}/deprecation.py (98%) create mode 100644 python/ray/_common/tests/test_deprecation.py delete mode 100644 python/ray/llm/_internal/common/utils/deprecation.py diff --git a/rllib/utils/deprecation.py b/python/ray/_common/deprecation.py similarity index 98% rename from rllib/utils/deprecation.py rename to python/ray/_common/deprecation.py index 7f5dd0e78b8b..fb89526d87a9 100644 --- a/rllib/utils/deprecation.py +++ b/python/ray/_common/deprecation.py @@ -71,7 +71,7 @@ def Deprecated(old=None, *, new=None, help=None, error): .. testcode:: :skipif: True - from ray.rllib.utils.deprecation import Deprecated + from ray._common.deprecation import Deprecated # Deprecated class: Patches the constructor to warn if the class is # used. @Deprecated(new="NewAndMuchCoolerClass", error=False) diff --git a/python/ray/_common/tests/BUILD b/python/ray/_common/tests/BUILD index 8535dd58cb07..1cc36ef348a8 100644 --- a/python/ray/_common/tests/BUILD +++ b/python/ray/_common/tests/BUILD @@ -14,6 +14,7 @@ py_library( py_test_module_list( size = "small", files = [ + "test_deprecation.py", "test_network_utils.py", "test_ray_option_utils.py", "test_signal_semaphore_utils.py", diff --git a/python/ray/_common/tests/test_deprecation.py b/python/ray/_common/tests/test_deprecation.py new file mode 100644 index 000000000000..9f0b23efde3c --- /dev/null +++ b/python/ray/_common/tests/test_deprecation.py @@ -0,0 +1,95 @@ +import pytest +import sys +from ray._common.deprecation import ( + DEPRECATED_VALUE, + Deprecated, + deprecation_warning, +) +from unittest.mock import patch + + +def test_deprecation_warning_warn(): + with patch("ray._common.deprecation.logger.warning") as mock_warning: + deprecation_warning("old_feature", "new_feature") + + mock_warning.assert_called_once() + args, _ = mock_warning.call_args + assert ( + "DeprecationWarning: `old_feature` has been deprecated. Use `new_feature` instead." + in args[0] + ) + + +def test_deprecation_warning_error(): + with pytest.raises(ValueError) as excinfo: + deprecation_warning("old_feature", error=True) + assert "`old_feature` has been deprecated." in str(excinfo.value) + + +def test_deprecated_decorator_function(): + with patch("ray._common.deprecation.logger.warning") as mock_warning, patch( + "ray._common.deprecation.log_once" + ) as mock_log_once: + mock_log_once.return_value = True + + @Deprecated(old="old_func", new="new_func", error=False) + def old_func(): + return "result" + + result = old_func() + assert result == "result" + mock_warning.assert_called_once() + + +def test_deprecated_decorator_class(): + with patch("ray._common.deprecation.logger.warning") as mock_warning, patch( + "ray._common.deprecation.log_once" + ) as mock_log_once: + mock_log_once.return_value = True + + @Deprecated(old="OldClass", new="NewClass", error=False) + class OldClass: + pass + + instance = OldClass() + assert isinstance(instance, OldClass) + mock_warning.assert_called_once() + + +def test_deprecated_decorator_method(): + with patch("ray._common.deprecation.logger.warning") as mock_warning, patch( + "ray._common.deprecation.log_once" + ) as mock_log_once: + mock_log_once.return_value = True + + class MyClass: + @Deprecated(old="old_method", new="new_method", error=False) + def old_method(self): + return "method_result" + + instance = MyClass() + result = instance.old_method() + assert result == "method_result" + mock_warning.assert_called_once() + + +def test_deprecated_decorator_error(): + with patch("ray._common.deprecation.log_once") as mock_log_once: + mock_log_once.return_value = True + + @Deprecated(old="old_func", error=True) + def old_func(): + pass + + with pytest.raises(ValueError): + old_func() + + +def test_deprecated_value_constant(): + assert ( + DEPRECATED_VALUE == -1 + ), f"DEPRECATED_VALUE should be -1, but got {DEPRECATED_VALUE}" + + +if __name__ == "__main__": + sys.exit(pytest.main(["-sv", __file__])) diff --git a/python/ray/llm/_internal/common/utils/deprecation.py b/python/ray/llm/_internal/common/utils/deprecation.py deleted file mode 100644 index f0a4cfbae186..000000000000 --- a/python/ray/llm/_internal/common/utils/deprecation.py +++ /dev/null @@ -1,136 +0,0 @@ -# Using Deprecated copied from ray.rllib.utils.deprecation since they are returning better messages. - -import inspect -import logging -from typing import Optional, Union - -from ray.util import log_once -from ray.util.annotations import _mark_annotated - -logger = logging.getLogger(__name__) - -# A constant to use for any configuration that should be deprecated -# (to check, whether this config has actually been assigned a proper value or -# not). -DEPRECATED_VALUE = -1 - - -def deprecation_warning( - old: str, - new: Optional[str] = None, - *, - help: Optional[str] = None, - error: Optional[Union[bool, Exception]] = None, -) -> None: - """Warns (via the `logger` object) or throws a deprecation warning/error. - - Args: - old: A description of the "thing" that is to be deprecated. - new: A description of the new "thing" that replaces it. - help: An optional help text to tell the user, what to - do instead of using `old`. - error: Whether or which exception to raise. If True, raise ValueError. - If False, just warn. If `error` is-a subclass of Exception, - raise that Exception. - - Raises: - ValueError: If `error=True`. - Exception: Of type `error`, iff `error` is a sub-class of `Exception`. - """ - msg = "`{}` has been deprecated.{}".format( - old, (" Use `{}` instead.".format(new) if new else f" {help}" if help else "") - ) - - if error: - if not isinstance(error, bool) and issubclass(error, Exception): - # error is an Exception - raise error(msg) - else: - # error is a boolean, construct ValueError ourselves - raise ValueError(msg) - else: - logger.warning( - "DeprecationWarning: " + msg + " This will raise an error in the future!" - ) - - -def Deprecated(old=None, *, new=None, help=None, error): - """Decorator for documenting a deprecated class, method, or function. - - Automatically adds a `deprecation.deprecation_warning(old=..., - error=False)` to not break existing code at this point to the decorated - class' constructor, method, or function. - - In a next major release, this warning should then be made an error - (by setting error=True), which means at this point that the - class/method/function is no longer supported, but will still inform - the user about the deprecation event. - - In a further major release, the class, method, function should be erased - entirely from the codebase. - - - .. testcode:: - :skipif: True - - from ray.rllib.utils.deprecation import Deprecated - # Deprecated class: Patches the constructor to warn if the class is - # used. - @Deprecated(new="NewAndMuchCoolerClass", error=False) - class OldAndUncoolClass: - ... - - # Deprecated class method: Patches the method to warn if called. - class StillCoolClass: - ... - @Deprecated(new="StillCoolClass.new_and_much_cooler_method()", - error=False) - def old_and_uncool_method(self, uncool_arg): - ... - - # Deprecated function: Patches the function to warn if called. - @Deprecated(new="new_and_much_cooler_function", error=False) - def old_and_uncool_function(*uncool_args): - ... - """ - - def _inner(obj): - # A deprecated class. - if inspect.isclass(obj): - # Patch the class' init method to raise the warning/error. - obj_init = obj.__init__ - - def patched_init(*args, **kwargs): - if log_once(old or obj.__name__): - deprecation_warning( - old=old or obj.__name__, - new=new, - help=help, - error=error, - ) - return obj_init(*args, **kwargs) - - obj.__init__ = patched_init - _mark_annotated(obj) - # Return the patched class (with the warning/error when - # instantiated). - return obj - - # A deprecated class method or function. - # Patch with the warning/error at the beginning. - def _ctor(*args, **kwargs): - if log_once(old or obj.__name__): - deprecation_warning( - old=old or obj.__name__, - new=new, - help=help, - error=error, - ) - # Call the deprecated method/function. - return obj(*args, **kwargs) - - # Return the patched class method/function. - return _ctor - - # Return the prepared decorator. - return _inner diff --git a/python/ray/serve/llm/__init__.py b/python/ray/serve/llm/__init__.py index 02db3545bd70..178268965db5 100644 --- a/python/ray/serve/llm/__init__.py +++ b/python/ray/serve/llm/__init__.py @@ -1,7 +1,6 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union -# TODO (ahao): Ray core should inherit deprecation utility. -from ray.llm._internal.common.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.llm._internal.serve.configs.server_models import ( CloudMirrorConfig as _CloudMirrorConfig, LLMConfig as _LLMConfig, diff --git a/rllib/algorithms/algorithm.py b/rllib/algorithms/algorithm.py index d0ac2675ae11..5b3f56c8d96d 100644 --- a/rllib/algorithms/algorithm.py +++ b/rllib/algorithms/algorithm.py @@ -109,7 +109,7 @@ try_import_msgpack, ) from ray.rllib.utils.debug import update_global_seed_if_necessary -from ray.rllib.utils.deprecation import ( +from ray._common.deprecation import ( DEPRECATED_VALUE, Deprecated, deprecation_warning, diff --git a/rllib/algorithms/algorithm_config.py b/rllib/algorithms/algorithm_config.py index 26067dc840ce..f58470da92ed 100644 --- a/rllib/algorithms/algorithm_config.py +++ b/rllib/algorithms/algorithm_config.py @@ -48,7 +48,7 @@ OldAPIStack, OverrideToImplementCustomLogic_CallToSuperRecommended, ) -from ray.rllib.utils.deprecation import ( +from ray._common.deprecation import ( DEPRECATED_VALUE, Deprecated, deprecation_warning, diff --git a/rllib/algorithms/appo/appo.py b/rllib/algorithms/appo/appo.py index a59636df752d..c3bc4c0031bb 100644 --- a/rllib/algorithms/appo/appo.py +++ b/rllib/algorithms/appo/appo.py @@ -18,7 +18,7 @@ from ray.rllib.core.rl_module.rl_module import RLModuleSpec from ray.rllib.policy.policy import Policy from ray.rllib.utils.annotations import override -from ray.rllib.utils.deprecation import DEPRECATED_VALUE, deprecation_warning +from ray._common.deprecation import DEPRECATED_VALUE, deprecation_warning from ray.rllib.utils.metrics import ( LAST_TARGET_UPDATE_TS, NUM_AGENT_STEPS_SAMPLED, diff --git a/rllib/algorithms/appo/appo_rl_module.py b/rllib/algorithms/appo/appo_rl_module.py index 5a2f59f9f201..de9b862a92ab 100644 --- a/rllib/algorithms/appo/appo_rl_module.py +++ b/rllib/algorithms/appo/appo_rl_module.py @@ -2,7 +2,7 @@ from ray.rllib.algorithms.appo.default_appo_rl_module import ( # noqa DefaultAPPORLModule as APPORLModule, ) -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning deprecation_warning( old="ray.rllib.algorithms.appo.appo_rl_module.APPORLModule", diff --git a/rllib/algorithms/appo/torch/appo_torch_rl_module.py b/rllib/algorithms/appo/torch/appo_torch_rl_module.py index ae60657b2c95..3bb3f0ba7f40 100644 --- a/rllib/algorithms/appo/torch/appo_torch_rl_module.py +++ b/rllib/algorithms/appo/torch/appo_torch_rl_module.py @@ -2,7 +2,7 @@ from ray.rllib.algorithms.appo.torch.default_appo_torch_rl_module import ( # noqa DefaultAPPOTorchRLModule as APPOTorchRLModule, ) -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning deprecation_warning( diff --git a/rllib/algorithms/cql/cql.py b/rllib/algorithms/cql/cql.py index e2f3ac2eff44..6d3b95cad746 100644 --- a/rllib/algorithms/cql/cql.py +++ b/rllib/algorithms/cql/cql.py @@ -25,7 +25,7 @@ ) from ray.rllib.policy.policy import Policy from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.utils.deprecation import ( +from ray._common.deprecation import ( DEPRECATED_VALUE, deprecation_warning, ) diff --git a/rllib/algorithms/dqn/dqn.py b/rllib/algorithms/dqn/dqn.py index 02014e72554c..6bc65698a56b 100644 --- a/rllib/algorithms/dqn/dqn.py +++ b/rllib/algorithms/dqn/dqn.py @@ -59,7 +59,7 @@ TD_ERROR_KEY, TIMERS, ) -from ray.rllib.utils.deprecation import DEPRECATED_VALUE +from ray._common.deprecation import DEPRECATED_VALUE from ray.rllib.utils.replay_buffers.utils import sample_min_n_steps_from_buffer from ray.rllib.utils.typing import ( LearningRateOrSchedule, diff --git a/rllib/algorithms/impala/impala.py b/rllib/algorithms/impala/impala.py index e3c2abfa1f37..c183b9e2f653 100644 --- a/rllib/algorithms/impala/impala.py +++ b/rllib/algorithms/impala/impala.py @@ -21,7 +21,7 @@ from ray.rllib.policy.policy import Policy from ray.rllib.policy.sample_batch import concat_samples from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.utils.deprecation import DEPRECATED_VALUE, deprecation_warning +from ray._common.deprecation import DEPRECATED_VALUE, deprecation_warning from ray.rllib.utils.metrics import ( AGGREGATOR_ACTOR_RESULTS, ALL_MODULES, diff --git a/rllib/algorithms/marwil/marwil.py b/rllib/algorithms/marwil/marwil.py index 4ebf1d9333a4..7dfb1e1dcde6 100644 --- a/rllib/algorithms/marwil/marwil.py +++ b/rllib/algorithms/marwil/marwil.py @@ -20,7 +20,7 @@ ) from ray.rllib.policy.policy import Policy from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.utils.metrics import ( LEARNER_RESULTS, LEARNER_UPDATE_TIMER, diff --git a/rllib/algorithms/ppo/ppo.py b/rllib/algorithms/ppo/ppo.py index 2b28c5bd91c8..7ffa74477928 100644 --- a/rllib/algorithms/ppo/ppo.py +++ b/rllib/algorithms/ppo/ppo.py @@ -25,7 +25,7 @@ ) from ray.rllib.policy.policy import Policy from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.utils.deprecation import DEPRECATED_VALUE +from ray._common.deprecation import DEPRECATED_VALUE from ray.rllib.utils.metrics import ( ENV_RUNNER_RESULTS, ENV_RUNNER_SAMPLING_TIMER, diff --git a/rllib/algorithms/ppo/ppo_rl_module.py b/rllib/algorithms/ppo/ppo_rl_module.py index 78f1ccef9fbd..631bf29fdd62 100644 --- a/rllib/algorithms/ppo/ppo_rl_module.py +++ b/rllib/algorithms/ppo/ppo_rl_module.py @@ -2,7 +2,7 @@ from ray.rllib.algorithms.ppo.default_ppo_rl_module import ( # noqa DefaultPPORLModule as PPORLModule, ) -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning deprecation_warning( old="ray.rllib.algorithms.ppo.ppo_rl_module.PPORLModule", diff --git a/rllib/algorithms/ppo/torch/ppo_torch_rl_module.py b/rllib/algorithms/ppo/torch/ppo_torch_rl_module.py index 60370a150497..66acb9e5fb5a 100644 --- a/rllib/algorithms/ppo/torch/ppo_torch_rl_module.py +++ b/rllib/algorithms/ppo/torch/ppo_torch_rl_module.py @@ -2,7 +2,7 @@ from ray.rllib.algorithms.ppo.torch.default_ppo_torch_rl_module import ( # noqa DefaultPPOTorchRLModule as PPOTorchRLModule, ) -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning deprecation_warning( diff --git a/rllib/algorithms/sac/sac.py b/rllib/algorithms/sac/sac.py index 581434e03ed9..6a0c2375153a 100644 --- a/rllib/algorithms/sac/sac.py +++ b/rllib/algorithms/sac/sac.py @@ -15,7 +15,7 @@ from ray.rllib.policy.policy import Policy from ray.rllib.utils import deep_update from ray.rllib.utils.annotations import override -from ray.rllib.utils.deprecation import DEPRECATED_VALUE, deprecation_warning +from ray._common.deprecation import DEPRECATED_VALUE, deprecation_warning from ray.rllib.utils.framework import try_import_tf, try_import_tfp from ray.rllib.utils.replay_buffers.episode_replay_buffer import EpisodeReplayBuffer from ray.rllib.utils.typing import LearningRateOrSchedule, RLModuleSpecType diff --git a/rllib/core/learner/learner.py b/rllib/core/learner/learner.py index 3eab6072ff6f..e2c6578c9173 100644 --- a/rllib/core/learner/learner.py +++ b/rllib/core/learner/learner.py @@ -47,7 +47,7 @@ ) from ray.rllib.utils.checkpoints import Checkpointable from ray.rllib.utils.debug import update_global_seed_if_necessary -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.framework import try_import_torch from ray.rllib.utils.metrics import ( ALL_MODULES, diff --git a/rllib/core/learner/learner_group.py b/rllib/core/learner/learner_group.py index e1d816ff9e19..b1010950e746 100644 --- a/rllib/core/learner/learner_group.py +++ b/rllib/core/learner/learner_group.py @@ -34,7 +34,7 @@ ) from ray.rllib.utils.annotations import override from ray.rllib.utils.checkpoints import Checkpointable -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.typing import ( EpisodeType, ModuleID, diff --git a/rllib/core/models/catalog.py b/rllib/core/models/catalog.py index f1bf8c6a3ea5..e4f9abe53b88 100644 --- a/rllib/core/models/catalog.py +++ b/rllib/core/models/catalog.py @@ -19,7 +19,7 @@ from ray.rllib.core.distribution.distribution import Distribution from ray.rllib.models.preprocessors import get_preprocessor, Preprocessor from ray.rllib.models.utils import get_filter_config -from ray.rllib.utils.deprecation import deprecation_warning, DEPRECATED_VALUE +from ray._common.deprecation import deprecation_warning, DEPRECATED_VALUE from ray.rllib.utils.error import UnsupportedSpaceException from ray.rllib.utils.spaces.simplex import Simplex from ray.rllib.utils.spaces.space_utils import flatten_space diff --git a/rllib/core/models/specs/specs_base.py b/rllib/core/models/specs/specs_base.py index 722267b3dc6d..2274fdd73641 100644 --- a/rllib/core/models/specs/specs_base.py +++ b/rllib/core/models/specs/specs_base.py @@ -1,4 +1,4 @@ -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated @Deprecated( diff --git a/rllib/core/models/specs/specs_dict.py b/rllib/core/models/specs/specs_dict.py index 7d944688eb0e..9c60b46fe67d 100644 --- a/rllib/core/models/specs/specs_dict.py +++ b/rllib/core/models/specs/specs_dict.py @@ -1,4 +1,4 @@ -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated @Deprecated( diff --git a/rllib/core/rl_module/multi_rl_module.py b/rllib/core/rl_module/multi_rl_module.py index c444f411b45a..5ba41d10931c 100644 --- a/rllib/core/rl_module/multi_rl_module.py +++ b/rllib/core/rl_module/multi_rl_module.py @@ -27,7 +27,7 @@ OverrideToImplementCustomLogic, ) from ray.rllib.utils.checkpoints import Checkpointable -from ray.rllib.utils.deprecation import ( +from ray._common.deprecation import ( Deprecated, DEPRECATED_VALUE, deprecation_warning, diff --git a/rllib/core/rl_module/rl_module.py b/rllib/core/rl_module/rl_module.py index 247fa0b79160..eeb75a1cd680 100644 --- a/rllib/core/rl_module/rl_module.py +++ b/rllib/core/rl_module/rl_module.py @@ -14,7 +14,7 @@ OverrideToImplementCustomLogic, ) from ray.rllib.utils.checkpoints import Checkpointable -from ray.rllib.utils.deprecation import ( +from ray._common.deprecation import ( Deprecated, DEPRECATED_VALUE, deprecation_warning, diff --git a/rllib/env/env_runner_group.py b/rllib/env/env_runner_group.py index 7d49910598dc..6974c1d30187 100644 --- a/rllib/env/env_runner_group.py +++ b/rllib/env/env_runner_group.py @@ -37,7 +37,7 @@ from ray.rllib.policy.policy import Policy, PolicyState from ray.rllib.utils.actor_manager import FaultTolerantActorManager from ray.rllib.utils.annotations import OldAPIStack -from ray.rllib.utils.deprecation import ( +from ray._common.deprecation import ( Deprecated, deprecation_warning, DEPRECATED_VALUE, diff --git a/rllib/env/external_env.py b/rllib/env/external_env.py index 41eb89d6c471..783ae256cb99 100644 --- a/rllib/env/external_env.py +++ b/rllib/env/external_env.py @@ -13,7 +13,7 @@ EnvType, MultiEnvDict, ) -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning if TYPE_CHECKING: from ray.rllib.models.preprocessors import Preprocessor diff --git a/rllib/env/multi_agent_env.py b/rllib/env/multi_agent_env.py index 843169306dce..a09f2cd93f97 100644 --- a/rllib/env/multi_agent_env.py +++ b/rllib/env/multi_agent_env.py @@ -8,7 +8,7 @@ from ray.rllib.env.base_env import BaseEnv from ray.rllib.env.env_context import EnvContext from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.typing import ( AgentID, EnvCreator, diff --git a/rllib/env/multi_agent_env_runner.py b/rllib/env/multi_agent_env_runner.py index 2610d1f1ba2d..48d7e1a6a9af 100644 --- a/rllib/env/multi_agent_env_runner.py +++ b/rllib/env/multi_agent_env_runner.py @@ -28,7 +28,7 @@ from ray.rllib.utils import force_list from ray.rllib.utils.annotations import override from ray.rllib.utils.checkpoints import Checkpointable -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.framework import get_device, try_import_torch from ray.rllib.utils.metrics import ( ENV_TO_MODULE_CONNECTOR, diff --git a/rllib/env/multi_agent_episode.py b/rllib/env/multi_agent_episode.py index 76b078ef69ff..3e21bac0cb4e 100644 --- a/rllib/env/multi_agent_episode.py +++ b/rllib/env/multi_agent_episode.py @@ -20,7 +20,7 @@ from ray.rllib.env.utils.infinite_lookback_buffer import InfiniteLookbackBuffer from ray.rllib.policy.sample_batch import MultiAgentBatch from ray.rllib.utils import force_list -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.error import MultiAgentEnvError from ray.rllib.utils.spaces.space_utils import batch from ray.rllib.utils.typing import AgentID, ModuleID, MultiAgentDict diff --git a/rllib/env/single_agent_env_runner.py b/rllib/env/single_agent_env_runner.py index 05db62932cc3..d032f3a8d245 100644 --- a/rllib/env/single_agent_env_runner.py +++ b/rllib/env/single_agent_env_runner.py @@ -29,7 +29,7 @@ from ray.rllib.utils import force_list from ray.rllib.utils.annotations import override from ray.rllib.utils.checkpoints import Checkpointable -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.framework import get_device from ray.rllib.utils.metrics import ( ENV_TO_MODULE_CONNECTOR, diff --git a/rllib/env/single_agent_episode.py b/rllib/env/single_agent_episode.py index 7056ff5c43b3..03906ff3d692 100644 --- a/rllib/env/single_agent_episode.py +++ b/rllib/env/single_agent_episode.py @@ -13,7 +13,7 @@ from ray.rllib.env.utils.infinite_lookback_buffer import InfiniteLookbackBuffer from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.utils.serialization import gym_space_from_dict, gym_space_to_dict -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.typing import AgentID, ModuleID from ray.util.annotations import PublicAPI diff --git a/rllib/env/utils/external_env_protocol.py b/rllib/env/utils/external_env_protocol.py index 7fedebedcd26..3356a87da30a 100644 --- a/rllib/env/utils/external_env_protocol.py +++ b/rllib/env/utils/external_env_protocol.py @@ -1,5 +1,5 @@ from ray.rllib.env.external.rllink import RLlink # noqa -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning deprecation_warning( old="ray.rllib.env.utils.external_env_protocol", diff --git a/rllib/evaluation/sample_batch_builder.py b/rllib/evaluation/sample_batch_builder.py index c4c748fe3bce..e42242e375a2 100644 --- a/rllib/evaluation/sample_batch_builder.py +++ b/rllib/evaluation/sample_batch_builder.py @@ -8,7 +8,7 @@ from ray.rllib.policy.sample_batch import SampleBatch, MultiAgentBatch from ray.rllib.utils.annotations import OldAPIStack from ray.rllib.utils.debug import summarize -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.utils.typing import PolicyID, AgentID from ray.util.debug import log_once diff --git a/rllib/evaluation/sampler.py b/rllib/evaluation/sampler.py index c6b4ce937e6b..9fb2a3700029 100644 --- a/rllib/evaluation/sampler.py +++ b/rllib/evaluation/sampler.py @@ -19,7 +19,7 @@ from ray.rllib.offline import InputReader from ray.rllib.policy.sample_batch import concat_samples from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.utils.deprecation import deprecation_warning, DEPRECATED_VALUE +from ray._common.deprecation import deprecation_warning, DEPRECATED_VALUE from ray.rllib.utils.framework import try_import_tf from ray.rllib.utils.typing import SampleBatchType from ray.util.debug import log_once diff --git a/rllib/evaluation/worker_set.py b/rllib/evaluation/worker_set.py index 0eeea1ea2c8f..1f0beba433c2 100644 --- a/rllib/evaluation/worker_set.py +++ b/rllib/evaluation/worker_set.py @@ -1,4 +1,4 @@ -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated @Deprecated( diff --git a/rllib/examples/multi_agent/utils/self_play_callback_old_api_stack.py b/rllib/examples/multi_agent/utils/self_play_callback_old_api_stack.py index 42b05b945017..eb19e57b02e4 100644 --- a/rllib/examples/multi_agent/utils/self_play_callback_old_api_stack.py +++ b/rllib/examples/multi_agent/utils/self_play_callback_old_api_stack.py @@ -1,7 +1,7 @@ import numpy as np from ray.rllib.callbacks.callbacks import RLlibCallback -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS diff --git a/rllib/examples/multi_agent/utils/self_play_league_based_callback_old_api_stack.py b/rllib/examples/multi_agent/utils/self_play_league_based_callback_old_api_stack.py index dc39fa8fac9a..e33a208be488 100644 --- a/rllib/examples/multi_agent/utils/self_play_league_based_callback_old_api_stack.py +++ b/rllib/examples/multi_agent/utils/self_play_league_based_callback_old_api_stack.py @@ -3,7 +3,7 @@ import numpy as np from ray.rllib.callbacks.callbacks import RLlibCallback -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.metrics import ENV_RUNNER_RESULTS diff --git a/rllib/execution/multi_gpu_learner_thread.py b/rllib/execution/multi_gpu_learner_thread.py index aacf797b32b8..556586e88f58 100644 --- a/rllib/execution/multi_gpu_learner_thread.py +++ b/rllib/execution/multi_gpu_learner_thread.py @@ -7,7 +7,7 @@ from ray.rllib.execution.minibatch_buffer import MinibatchBuffer from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.utils.framework import try_import_tf from ray.rllib.utils.metrics.learner_info import LearnerInfoBuilder from ray.rllib.evaluation.rollout_worker import RolloutWorker diff --git a/rllib/execution/train_ops.py b/rllib/execution/train_ops.py index 732beb92e7c4..ebed28078b51 100644 --- a/rllib/execution/train_ops.py +++ b/rllib/execution/train_ops.py @@ -6,7 +6,7 @@ from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID from ray.rllib.utils.annotations import OldAPIStack from ray.rllib.utils.framework import try_import_tf -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.utils.metrics import ( NUM_ENV_STEPS_TRAINED, NUM_AGENT_STEPS_TRAINED, diff --git a/rllib/models/catalog.py b/rllib/models/catalog.py index 0b2d393c0d11..59180bcc8691 100644 --- a/rllib/models/catalog.py +++ b/rllib/models/catalog.py @@ -31,7 +31,7 @@ TorchMultiCategorical, ) from ray.rllib.utils.annotations import DeveloperAPI, PublicAPI -from ray.rllib.utils.deprecation import ( +from ray._common.deprecation import ( DEPRECATED_VALUE, deprecation_warning, ) diff --git a/rllib/models/distributions.py b/rllib/models/distributions.py index dac7b108d610..05d9670a8c7f 100644 --- a/rllib/models/distributions.py +++ b/rllib/models/distributions.py @@ -1,4 +1,4 @@ -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.core.distribution.distribution import Distribution # noqa deprecation_warning( diff --git a/rllib/models/modelv2.py b/rllib/models/modelv2.py index df07150e57ba..c3eda53c171d 100644 --- a/rllib/models/modelv2.py +++ b/rllib/models/modelv2.py @@ -11,7 +11,7 @@ from ray.rllib.policy.view_requirement import ViewRequirement from ray.rllib.utils import NullContextManager from ray.rllib.utils.annotations import OldAPIStack -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.framework import try_import_tf, try_import_torch, TensorType from ray.rllib.utils.spaces.repeated import Repeated from ray.rllib.utils.typing import ModelConfigDict, ModelInputDict, TensorStructType diff --git a/rllib/models/tf/attention_net.py b/rllib/models/tf/attention_net.py index 886580fce177..3a250bf897c1 100644 --- a/rllib/models/tf/attention_net.py +++ b/rllib/models/tf/attention_net.py @@ -29,7 +29,7 @@ from ray.rllib.utils.spaces.space_utils import get_base_struct_from_space from ray.rllib.utils.tf_utils import flatten_inputs_to_1d_tensor, one_hot from ray.rllib.utils.typing import ModelConfigDict, TensorType, List -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.util import log_once tf1, tf, tfv = try_import_tf() diff --git a/rllib/models/tf/layers/gru_gate.py b/rllib/models/tf/layers/gru_gate.py index a41b23bbf534..4a3fc0ad5303 100644 --- a/rllib/models/tf/layers/gru_gate.py +++ b/rllib/models/tf/layers/gru_gate.py @@ -1,6 +1,6 @@ from ray.rllib.utils.framework import try_import_tf from ray.rllib.utils.typing import TensorType, TensorShape -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.util import log_once tf1, tf, tfv = try_import_tf() diff --git a/rllib/models/tf/layers/multi_head_attention.py b/rllib/models/tf/layers/multi_head_attention.py index 595608989f0b..d1372c59903e 100644 --- a/rllib/models/tf/layers/multi_head_attention.py +++ b/rllib/models/tf/layers/multi_head_attention.py @@ -5,7 +5,7 @@ """ from ray.rllib.utils.framework import try_import_tf from ray.rllib.utils.typing import TensorType -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.util import log_once tf1, tf, tfv = try_import_tf() diff --git a/rllib/models/tf/layers/noisy_layer.py b/rllib/models/tf/layers/noisy_layer.py index 5bc149d5de13..b1ade2acf1fc 100644 --- a/rllib/models/tf/layers/noisy_layer.py +++ b/rllib/models/tf/layers/noisy_layer.py @@ -7,7 +7,7 @@ TensorType, TensorShape, ) -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.util import log_once tf1, tf, tfv = try_import_tf() diff --git a/rllib/models/tf/layers/relative_multi_head_attention.py b/rllib/models/tf/layers/relative_multi_head_attention.py index f88486ff2051..d0dfd3a20e40 100644 --- a/rllib/models/tf/layers/relative_multi_head_attention.py +++ b/rllib/models/tf/layers/relative_multi_head_attention.py @@ -2,7 +2,7 @@ from ray.rllib.utils.framework import try_import_tf from ray.rllib.utils.typing import TensorType -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.util import log_once tf1, tf, tfv = try_import_tf() diff --git a/rllib/models/tf/layers/skip_connection.py b/rllib/models/tf/layers/skip_connection.py index 3ee1751caf36..1ae2525e997b 100644 --- a/rllib/models/tf/layers/skip_connection.py +++ b/rllib/models/tf/layers/skip_connection.py @@ -2,7 +2,7 @@ from ray.rllib.utils.framework import try_import_tf from ray.rllib.utils.typing import TensorType -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.util import log_once tf1, tf, tfv = try_import_tf() diff --git a/rllib/models/tf/recurrent_net.py b/rllib/models/tf/recurrent_net.py index 2010d4a90118..cd4d721a2967 100644 --- a/rllib/models/tf/recurrent_net.py +++ b/rllib/models/tf/recurrent_net.py @@ -15,7 +15,7 @@ from ray.rllib.utils.spaces.space_utils import get_base_struct_from_space from ray.rllib.utils.tf_utils import flatten_inputs_to_1d_tensor, one_hot from ray.rllib.utils.typing import ModelConfigDict, TensorType -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.util.debug import log_once tf1, tf, tfv = try_import_tf() diff --git a/rllib/models/tf/tf_modelv2.py b/rllib/models/tf/tf_modelv2.py index 743879694424..f1ad20c3b65e 100644 --- a/rllib/models/tf/tf_modelv2.py +++ b/rllib/models/tf/tf_modelv2.py @@ -6,7 +6,7 @@ from ray.util import log_once from ray.rllib.models.modelv2 import ModelV2 from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.utils.framework import try_import_tf from ray.rllib.utils.typing import ModelConfigDict, TensorType diff --git a/rllib/models/torch/attention_net.py b/rllib/models/torch/attention_net.py index 2382a4da1381..d2624da0a5a2 100644 --- a/rllib/models/torch/attention_net.py +++ b/rllib/models/torch/attention_net.py @@ -30,7 +30,7 @@ from ray.rllib.utils.spaces.space_utils import get_base_struct_from_space from ray.rllib.utils.torch_utils import flatten_inputs_to_1d_tensor, one_hot from ray.rllib.utils.typing import ModelConfigDict, TensorType, List -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.util import log_once torch, nn = try_import_torch() diff --git a/rllib/models/torch/mingpt.py b/rllib/models/torch/mingpt.py index 7e24cfdc730a..f64ea12419b8 100644 --- a/rllib/models/torch/mingpt.py +++ b/rllib/models/torch/mingpt.py @@ -20,7 +20,7 @@ from torch.nn import functional as F from ray.rllib.utils.annotations import DeveloperAPI -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated @DeveloperAPI diff --git a/rllib/models/torch/recurrent_net.py b/rllib/models/torch/recurrent_net.py index 01fbab223e29..d4afc688ea8e 100644 --- a/rllib/models/torch/recurrent_net.py +++ b/rllib/models/torch/recurrent_net.py @@ -15,7 +15,7 @@ from ray.rllib.utils.spaces.space_utils import get_base_struct_from_space from ray.rllib.utils.torch_utils import flatten_inputs_to_1d_tensor, one_hot from ray.rllib.utils.typing import ModelConfigDict, TensorType -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.util.debug import log_once torch, nn = try_import_torch() diff --git a/rllib/models/torch/torch_distributions.py b/rllib/models/torch/torch_distributions.py index afba9a9a16a6..d9f77b975f4a 100644 --- a/rllib/models/torch/torch_distributions.py +++ b/rllib/models/torch/torch_distributions.py @@ -1,4 +1,4 @@ -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.core.distribution.torch.torch_distribution import ( # noqa TorchDistribution, TorchCategorical, diff --git a/rllib/offline/estimators/feature_importance.py b/rllib/offline/estimators/feature_importance.py index a5d4d1718932..148426aefb9b 100644 --- a/rllib/offline/estimators/feature_importance.py +++ b/rllib/offline/estimators/feature_importance.py @@ -2,7 +2,7 @@ __all__ = ["FeatureImportance"] -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning deprecation_warning( "ray.rllib.offline.estimators.feature_importance.FeatureImportance", diff --git a/rllib/offline/estimators/off_policy_estimator.py b/rllib/offline/estimators/off_policy_estimator.py index 7c6ef95eb78b..9abee46c1a12 100644 --- a/rllib/offline/estimators/off_policy_estimator.py +++ b/rllib/offline/estimators/off_policy_estimator.py @@ -13,7 +13,7 @@ ExperimentalAPI, OverrideToImplementCustomLogic, ) -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.numpy import convert_to_numpy from ray.rllib.utils.typing import TensorType, SampleBatchType from ray.rllib.offline.offline_evaluator import OfflineEvaluator diff --git a/rllib/offline/is_estimator.py b/rllib/offline/is_estimator.py index 58c8da3e0c72..d395e3f9a356 100644 --- a/rllib/offline/is_estimator.py +++ b/rllib/offline/is_estimator.py @@ -1,5 +1,5 @@ from ray.rllib.offline.estimators.importance_sampling import ImportanceSampling -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated @Deprecated( diff --git a/rllib/offline/off_policy_estimator.py b/rllib/offline/off_policy_estimator.py index c8a08fb4a1df..9d2b90195a57 100644 --- a/rllib/offline/off_policy_estimator.py +++ b/rllib/offline/off_policy_estimator.py @@ -1,7 +1,7 @@ from ray.rllib.offline.estimators.off_policy_estimator import ( # noqa: F401 OffPolicyEstimator, ) -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning deprecation_warning( old="ray.rllib.offline.off_policy_estimator", diff --git a/rllib/offline/wis_estimator.py b/rllib/offline/wis_estimator.py index 128b50e24b2a..95c7e3bcec09 100644 --- a/rllib/offline/wis_estimator.py +++ b/rllib/offline/wis_estimator.py @@ -1,7 +1,7 @@ from ray.rllib.offline.estimators.weighted_importance_sampling import ( WeightedImportanceSampling, ) -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated @Deprecated( diff --git a/rllib/policy/dynamic_tf_policy.py b/rllib/policy/dynamic_tf_policy.py index 9645faf6e08f..180f3059e6db 100644 --- a/rllib/policy/dynamic_tf_policy.py +++ b/rllib/policy/dynamic_tf_policy.py @@ -16,7 +16,7 @@ from ray.rllib.utils import force_list from ray.rllib.utils.annotations import OldAPIStack, override from ray.rllib.utils.debug import summarize -from ray.rllib.utils.deprecation import ( +from ray._common.deprecation import ( deprecation_warning, DEPRECATED_VALUE, ) diff --git a/rllib/policy/eager_tf_policy.py b/rllib/policy/eager_tf_policy.py index c2e4fa33f159..c4f43c6d4ee0 100644 --- a/rllib/policy/eager_tf_policy.py +++ b/rllib/policy/eager_tf_policy.py @@ -17,7 +17,7 @@ from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.utils import add_mixins, force_list from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.utils.deprecation import ( +from ray._common.deprecation import ( DEPRECATED_VALUE, deprecation_warning, ) diff --git a/rllib/policy/policy.py b/rllib/policy/policy.py index 0b1db3653a8c..859e1d5847ac 100644 --- a/rllib/policy/policy.py +++ b/rllib/policy/policy.py @@ -40,7 +40,7 @@ get_checkpoint_info, try_import_msgpack, ) -from ray.rllib.utils.deprecation import ( +from ray._common.deprecation import ( DEPRECATED_VALUE, deprecation_warning, ) diff --git a/rllib/policy/policy_map.py b/rllib/policy/policy_map.py index b14b2a27056e..1627d3788939 100644 --- a/rllib/policy/policy_map.py +++ b/rllib/policy/policy_map.py @@ -6,7 +6,7 @@ import ray from ray.rllib.policy.policy import Policy from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.utils.framework import try_import_tf from ray.rllib.utils.threading import with_lock from ray.rllib.utils.typing import PolicyID diff --git a/rllib/policy/sample_batch.py b/rllib/policy/sample_batch.py index 558140da8239..25bc6f313b09 100644 --- a/rllib/policy/sample_batch.py +++ b/rllib/policy/sample_batch.py @@ -12,7 +12,7 @@ from ray.rllib.core.columns import Columns from ray.rllib.utils.annotations import DeveloperAPI, ExperimentalAPI, PublicAPI from ray.rllib.utils.compression import pack, unpack, is_compressed -from ray.rllib.utils.deprecation import Deprecated, deprecation_warning +from ray._common.deprecation import Deprecated, deprecation_warning from ray.rllib.utils.framework import try_import_tf, try_import_torch from ray.rllib.utils.torch_utils import convert_to_torch_tensor from ray.rllib.utils.typing import ( diff --git a/rllib/policy/tf_policy.py b/rllib/policy/tf_policy.py index ff68aeed8a46..377ce00727ff 100644 --- a/rllib/policy/tf_policy.py +++ b/rllib/policy/tf_policy.py @@ -14,7 +14,7 @@ from ray.rllib.utils import force_list from ray.rllib.utils.annotations import OldAPIStack, override from ray.rllib.utils.debug import summarize -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.error import ERR_MSG_TF_POLICY_CANNOT_SAVE_KERAS_MODEL from ray.rllib.utils.framework import try_import_tf from ray.rllib.utils.metrics import ( diff --git a/rllib/policy/tf_policy_template.py b/rllib/policy/tf_policy_template.py index d82e0691b362..dea3687f5526 100644 --- a/rllib/policy/tf_policy_template.py +++ b/rllib/policy/tf_policy_template.py @@ -10,7 +10,7 @@ from ray.rllib.policy.tf_policy import TFPolicy from ray.rllib.utils import add_mixins, force_list from ray.rllib.utils.annotations import OldAPIStack, override -from ray.rllib.utils.deprecation import ( +from ray._common.deprecation import ( deprecation_warning, DEPRECATED_VALUE, ) diff --git a/rllib/tests/run_regression_tests.py b/rllib/tests/run_regression_tests.py index e0a82f00499a..8fc62da78c23 100644 --- a/rllib/tests/run_regression_tests.py +++ b/rllib/tests/run_regression_tests.py @@ -16,7 +16,7 @@ from ray import air from ray.air.integrations.wandb import WandbLoggerCallback from ray.rllib import _register_all -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.utils.metrics import ( ENV_RUNNER_RESULTS, EPISODE_RETURN_MEAN, diff --git a/rllib/utils/__init__.py b/rllib/utils/__init__.py index 7adcf6f7ca51..ff95e19a155f 100644 --- a/rllib/utils/__init__.py +++ b/rllib/utils/__init__.py @@ -5,7 +5,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union from ray.rllib.utils.annotations import override, PublicAPI, DeveloperAPI -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.utils.filter import Filter from ray.rllib.utils.filter_manager import FilterManager from ray.rllib.utils.framework import ( diff --git a/rllib/utils/annotations.py b/rllib/utils/annotations.py index 6824412b354f..286c541e0f12 100644 --- a/rllib/utils/annotations.py +++ b/rllib/utils/annotations.py @@ -1,4 +1,4 @@ -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.util.annotations import _mark_annotated diff --git a/rllib/utils/filter.py b/rllib/utils/filter.py index 5f1418cfd2d1..8b4e6ffcd827 100644 --- a/rllib/utils/filter.py +++ b/rllib/utils/filter.py @@ -5,13 +5,13 @@ import tree # pip install dm_tree from ray.rllib.utils.annotations import OldAPIStack -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.numpy import ( SMALL_NUMBER, ) # Assuming SMALL_NUMBER is a small float like 1e-8 from ray.rllib.utils.typing import TensorStructType from ray.rllib.utils.serialization import _serialize_ndarray, _deserialize_ndarray -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning logger = logging.getLogger(__name__) diff --git a/rllib/utils/framework.py b/rllib/utils/framework.py index c0b9a28fa472..ba2280488d69 100644 --- a/rllib/utils/framework.py +++ b/rllib/utils/framework.py @@ -8,7 +8,7 @@ import ray from ray.rllib.utils.annotations import DeveloperAPI, PublicAPI -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.typing import ( TensorShape, TensorStructType, diff --git a/rllib/utils/memory.py b/rllib/utils/memory.py index fe739cc0f99b..323bec70c50f 100644 --- a/rllib/utils/memory.py +++ b/rllib/utils/memory.py @@ -1,4 +1,4 @@ -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.utils.numpy import aligned_array, concat_aligned # noqa deprecation_warning( diff --git a/rllib/utils/metrics/metrics_logger.py b/rllib/utils/metrics/metrics_logger.py index 9d3ba1c4e8f9..d941db5599c3 100644 --- a/rllib/utils/metrics/metrics_logger.py +++ b/rllib/utils/metrics/metrics_logger.py @@ -4,7 +4,7 @@ from ray.rllib.utils import force_tuple, deep_update from ray.rllib.utils.metrics.stats import Stats, merge_stats -from ray.rllib.utils.deprecation import Deprecated, deprecation_warning +from ray._common.deprecation import Deprecated, deprecation_warning from ray.rllib.utils.framework import try_import_tf, try_import_torch from ray.util.annotations import PublicAPI from ray.util import log_once diff --git a/rllib/utils/numpy.py b/rllib/utils/numpy.py index b0970ad51427..f1bb8f2ff32b 100644 --- a/rllib/utils/numpy.py +++ b/rllib/utils/numpy.py @@ -7,7 +7,7 @@ from ray.rllib.utils.annotations import PublicAPI -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.framework import try_import_tf, try_import_torch from ray.rllib.utils.typing import SpaceStruct, TensorType, TensorStructType, Union diff --git a/rllib/utils/policy.py b/rllib/utils/policy.py index a5b6b2ccfda6..0cb149ced5bc 100644 --- a/rllib/utils/policy.py +++ b/rllib/utils/policy.py @@ -20,7 +20,7 @@ from ray.rllib.policy.policy import PolicySpec from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.utils.annotations import OldAPIStack -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.framework import try_import_tf from ray.rllib.utils.typing import ( ActionConnectorDataType, diff --git a/rllib/utils/replay_buffers/multi_agent_replay_buffer.py b/rllib/utils/replay_buffers/multi_agent_replay_buffer.py index ac3af0125b27..5fcfd75365c6 100644 --- a/rllib/utils/replay_buffers/multi_agent_replay_buffer.py +++ b/rllib/utils/replay_buffers/multi_agent_replay_buffer.py @@ -7,7 +7,7 @@ from ray.rllib.policy.rnn_sequencing import timeslice_along_seq_lens_with_overlap from ray.rllib.policy.sample_batch import MultiAgentBatch, SampleBatch from ray.rllib.utils.annotations import override -from ray.rllib.utils.deprecation import Deprecated +from ray._common.deprecation import Deprecated from ray.rllib.utils.from_config import from_config from ray.rllib.utils.replay_buffers.replay_buffer import ( _ALL_POLICIES, diff --git a/rllib/utils/replay_buffers/utils.py b/rllib/utils/replay_buffers/utils.py index 16fa37d0626f..baf24b6874c7 100644 --- a/rllib/utils/replay_buffers/utils.py +++ b/rllib/utils/replay_buffers/utils.py @@ -4,9 +4,9 @@ import numpy as np -from ray.rllib.utils import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.rllib.utils.annotations import OldAPIStack -from ray.rllib.utils.deprecation import DEPRECATED_VALUE +from ray._common.deprecation import DEPRECATED_VALUE from ray.rllib.utils.from_config import from_config from ray.rllib.utils.metrics import ALL_MODULES, TD_ERROR_KEY from ray.rllib.utils.metrics.learner_info import LEARNER_STATS_KEY diff --git a/rllib/utils/tests/run_memory_leak_tests.py b/rllib/utils/tests/run_memory_leak_tests.py index 4fc509fd7c88..36d5c96c2b93 100644 --- a/rllib/utils/tests/run_memory_leak_tests.py +++ b/rllib/utils/tests/run_memory_leak_tests.py @@ -26,7 +26,7 @@ from ray.rllib.common import SupportedFileType from ray.rllib.train import load_experiments_from_file from ray.rllib.utils.debug.memory import check_memory_leaks -from ray.rllib.utils.deprecation import deprecation_warning +from ray._common.deprecation import deprecation_warning from ray.tune.registry import get_trainable_cls parser = argparse.ArgumentParser() From 24834f34e67280792f7dd9737ce5c073b5dde34e Mon Sep 17 00:00:00 2001 From: Sagar Sumit Date: Tue, 9 Sep 2025 19:14:07 +0530 Subject: [PATCH 526/634] [core] Fix batching logic in `CoreWorkerPlasmaStoreProvider::Get` (#56041) This PR fixes a bug in `CoreWorkerPlasmaStoreProvider::Get`. The [batching logic](https://github.com/ray-project/ray/blob/235b43fd4ab81aab92010261d775dd48983ae983/src/ray/core_worker/store_provider/plasma_store_provider.cc#L287-L300) in `CoreWorkerPlasmaStoreProvider::Get` had two critical issues: 1. Incorrect loop condition: `i < batch_size` instead of `i < start + batch_size` 2. Incorrect array access: `id_vector[start + i]` instead of `id_vector[i]` Before fix: Only the first batch of objects was processed correctly. Subsequent batches were empty, causing incomplete object retrieval. For e.g., with batch_size = 2 and 5 objects, only objects 0-1 were processed, objects 2-4 were ignored. --------- Signed-off-by: Sagar Sumit --- .../ray/object_manager/plasma/BUILD.bazel | 12 ++ .../plasma/fake_plasma_client.h | 106 ++++++++++++++++++ src/mock/ray/object_manager/plasma/client.h | 2 + src/ray/core_worker/core_worker_process.cc | 10 ++ .../store_provider/plasma_store_provider.cc | 31 +++-- .../store_provider/plasma_store_provider.h | 7 +- src/ray/core_worker/tests/BUILD.bazel | 1 + src/ray/core_worker/tests/core_worker_test.cc | 73 ++++++++++++ src/ray/object_manager/plasma/BUILD.bazel | 13 +++ src/ray/object_manager/plasma/client.h | 7 +- src/ray/raylet/tests/node_manager_test.cc | 2 + 11 files changed, 244 insertions(+), 20 deletions(-) create mode 100644 src/fakes/ray/object_manager/plasma/BUILD.bazel create mode 100644 src/fakes/ray/object_manager/plasma/fake_plasma_client.h diff --git a/src/fakes/ray/object_manager/plasma/BUILD.bazel b/src/fakes/ray/object_manager/plasma/BUILD.bazel new file mode 100644 index 000000000000..86e70b439f83 --- /dev/null +++ b/src/fakes/ray/object_manager/plasma/BUILD.bazel @@ -0,0 +1,12 @@ +load("//bazel:ray.bzl", "ray_cc_library") + +ray_cc_library( + name = "fake_plasma_client", + hdrs = ["fake_plasma_client.h"], + deps = [ + "//src/ray/common:buffer", + "//src/ray/common:id", + "//src/ray/common:status", + "//src/ray/object_manager/plasma:plasma_client_interface", + ], +) diff --git a/src/fakes/ray/object_manager/plasma/fake_plasma_client.h b/src/fakes/ray/object_manager/plasma/fake_plasma_client.h new file mode 100644 index 000000000000..74ab301fd259 --- /dev/null +++ b/src/fakes/ray/object_manager/plasma/fake_plasma_client.h @@ -0,0 +1,106 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +// A simple fake implementation of PlasmaClientInterface for use in unit tests. +// +// This base fake does nothing (returns OK for most methods, empty results for Get). +// Extend it in test files to add behavior (recording batches, timeouts, missing objects). + +#include "ray/common/buffer.h" +#include "ray/common/id.h" +#include "ray/common/status.h" +#include "ray/object_manager/plasma/client.h" + +namespace plasma { + +class FakePlasmaClient : public PlasmaClientInterface { + public: + FakePlasmaClient(std::vector> *observed_batches = nullptr) + : observed_batches_(observed_batches) {} + + Status Connect(const std::string &, const std::string &, int) override { + return Status::OK(); + } + + Status Release(const ObjectID &) override { return Status::OK(); } + + Status Contains(const ObjectID &, bool *) override { return Status::OK(); } + + Status Disconnect() override { return Status::OK(); } + + Status Get(const std::vector &object_ids, + int64_t /*timeout_ms*/, + std::vector *object_buffers) override { + if (observed_batches_ != nullptr) { + observed_batches_->push_back(object_ids); + } + // Return non-null buffers to simulate presence for tests. + object_buffers->resize(object_ids.size()); + for (size_t i = 0; i < object_ids.size(); i++) { + uint8_t byte = 0; + auto parent = + std::make_shared(&byte, 1, /*copy_data=*/true); + (*object_buffers)[i].data = SharedMemoryBuffer::Slice(parent, 0, 1); + (*object_buffers)[i].metadata = SharedMemoryBuffer::Slice(parent, 0, 1); + } + return Status::OK(); + } + + Status GetExperimentalMutableObject(const ObjectID &, + std::unique_ptr *) override { + return Status::OK(); + } + + Status Seal(const ObjectID &) override { return Status::OK(); } + + Status Abort(const ObjectID &) override { return Status::OK(); } + + Status CreateAndSpillIfNeeded(const ObjectID &, + const ray::rpc::Address &, + bool, + int64_t, + const uint8_t *, + int64_t, + std::shared_ptr *, + flatbuf::ObjectSource, + int) override { + return Status::OK(); + } + + Status TryCreateImmediately(const ObjectID &, + const ray::rpc::Address &, + int64_t, + const uint8_t *, + int64_t, + std::shared_ptr *, + flatbuf::ObjectSource, + int) override { + return Status::OK(); + } + + Status Delete(const std::vector &) override { return Status::OK(); } + + StatusOr GetMemoryUsage() override { return std::string("fake"); } + + private: + std::vector> *observed_batches_; +}; + +} // namespace plasma diff --git a/src/mock/ray/object_manager/plasma/client.h b/src/mock/ray/object_manager/plasma/client.h index 8e5905c73463..dd7617b58641 100644 --- a/src/mock/ray/object_manager/plasma/client.h +++ b/src/mock/ray/object_manager/plasma/client.h @@ -79,6 +79,8 @@ class MockPlasmaClient : public PlasmaClientInterface { (override)); MOCK_METHOD(Status, Delete, (const std::vector &object_ids), (override)); + + MOCK_METHOD(StatusOr, GetMemoryUsage, (), (override)); }; } // namespace plasma diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index f514ce9f211c..5aa2545c8dbf 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -35,6 +35,7 @@ #include "ray/core_worker/core_worker_rpc_proxy.h" #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/ipc/raylet_ipc_client.h" +#include "ray/object_manager/plasma/client.h" #include "ray/rpc/raylet/raylet_client.h" #include "ray/stats/stats.h" #include "ray/util/container_util.h" @@ -353,6 +354,13 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( /*min_concurrent_lease_cap_*/ 10); } + // We can turn on exit_on_connection_failure on for the core worker plasma + // client to early exit core worker after the raylet's death because on the + // raylet side, we never proactively close the plasma store connection even + // during shutdown. So any error from the raylet side should be a sign of raylet + // death. + auto plasma_client = std::shared_ptr( + new plasma::PlasmaClient(/*exit_on_connection_failure*/ true)); auto plasma_store_provider = std::make_shared( options.store_socket, raylet_ipc_client, @@ -361,6 +369,8 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( /*warmup=*/ (options.worker_type != WorkerType::SPILL_WORKER && options.worker_type != WorkerType::RESTORE_WORKER), + /*store_client=*/plasma_client, + /*fetch_batch_size=*/RayConfig::instance().worker_fetch_request_size(), /*get_current_call_site=*/[this]() { auto core_worker = GetCoreWorker(); return core_worker->CurrentCallSite(); diff --git a/src/ray/core_worker/store_provider/plasma_store_provider.cc b/src/ray/core_worker/store_provider/plasma_store_provider.cc index 98da568c13b5..3323a924ec50 100644 --- a/src/ray/core_worker/store_provider/plasma_store_provider.cc +++ b/src/ray/core_worker/store_provider/plasma_store_provider.cc @@ -67,17 +67,14 @@ CoreWorkerPlasmaStoreProvider::CoreWorkerPlasmaStoreProvider( ReferenceCounter &reference_counter, std::function check_signals, bool warmup, + std::shared_ptr store_client, + int64_t fetch_batch_size, std::function get_current_call_site) : raylet_ipc_client_(raylet_ipc_client), - // We can turn on exit_on_connection_failure on for the core worker plasma - // client to early exit core worker after the raylet's death because on the - // raylet side, we never proactively close the plasma store connection even - // during shutdown. So any error from the raylet side should be a sign of raylet - // death. - store_client_( - std::make_shared(/*exit_on_connection_failure*/ true)), + store_client_(std::move(store_client)), reference_counter_(reference_counter), - check_signals_(std::move(check_signals)) { + check_signals_(std::move(check_signals)), + fetch_batch_size_(fetch_batch_size) { if (get_current_call_site != nullptr) { get_current_call_site_ = get_current_call_site; } else { @@ -85,7 +82,10 @@ CoreWorkerPlasmaStoreProvider::CoreWorkerPlasmaStoreProvider( } object_store_full_delay_ms_ = RayConfig::instance().object_store_full_delay_ms(); buffer_tracker_ = std::make_shared(); - RAY_CHECK_OK(store_client_->Connect(store_socket)); + if (!store_socket.empty()) { + RAY_CHECK(store_client_ != nullptr) << "Plasma client must be provided"; + RAY_CHECK_OK(store_client_->Connect(store_socket)); + } if (warmup) { RAY_CHECK_OK(WarmupStore()); } @@ -224,9 +224,7 @@ Status CoreWorkerPlasmaStoreProvider::GetIfLocal( const std::vector &object_ids, absl::flat_hash_map> *results) { std::vector plasma_results; - RAY_RETURN_NOT_OK(store_client_->Get(object_ids, - /*timeout_ms=*/0, - &plasma_results)); + RAY_RETURN_NOT_OK(store_client_->Get(object_ids, /*timeout_ms=*/0, &plasma_results)); for (size_t i = 0; i < object_ids.size(); i++) { if (plasma_results[i].data != nullptr || plasma_results[i].metadata != nullptr) { @@ -278,17 +276,16 @@ Status CoreWorkerPlasmaStoreProvider::Get( const WorkerContext &ctx, absl::flat_hash_map> *results, bool *got_exception) { - int64_t batch_size = RayConfig::instance().worker_fetch_request_size(); std::vector batch_ids; absl::flat_hash_set remaining(object_ids.begin(), object_ids.end()); // Send initial requests to pull all objects in parallel. std::vector id_vector(object_ids.begin(), object_ids.end()); int64_t total_size = static_cast(object_ids.size()); - for (int64_t start = 0; start < total_size; start += batch_size) { + for (int64_t start = 0; start < total_size; start += fetch_batch_size_) { batch_ids.clear(); - for (int64_t i = start; i < batch_size && i < total_size; i++) { - batch_ids.push_back(id_vector[start + i]); + for (int64_t i = start; i < start + fetch_batch_size_ && i < total_size; i++) { + batch_ids.push_back(id_vector[i]); } RAY_RETURN_NOT_OK( PullObjectsAndGetFromPlasmaStore(remaining, @@ -315,7 +312,7 @@ Status CoreWorkerPlasmaStoreProvider::Get( while (!remaining.empty() && !should_break) { batch_ids.clear(); for (const auto &id : remaining) { - if (static_cast(batch_ids.size()) == batch_size) { + if (static_cast(batch_ids.size()) == fetch_batch_size_) { break; } batch_ids.push_back(id); diff --git a/src/ray/core_worker/store_provider/plasma_store_provider.h b/src/ray/core_worker/store_provider/plasma_store_provider.h index c0997f5c0222..0da5aac4437a 100644 --- a/src/ray/core_worker/store_provider/plasma_store_provider.h +++ b/src/ray/core_worker/store_provider/plasma_store_provider.h @@ -100,6 +100,8 @@ class CoreWorkerPlasmaStoreProvider { ReferenceCounter &reference_counter, std::function check_signals, bool warmup, + std::shared_ptr store_client, + int64_t fetch_batch_size, std::function get_current_call_site = nullptr); ~CoreWorkerPlasmaStoreProvider(); @@ -201,7 +203,7 @@ class CoreWorkerPlasmaStoreProvider { StatusOr GetMemoryUsage(); - std::shared_ptr &store_client() { return store_client_; } + std::shared_ptr &store_client() { return store_client_; } private: /// Ask the raylet to pull a set of objects and then attempt to get them @@ -236,7 +238,7 @@ class CoreWorkerPlasmaStoreProvider { Status WarmupStore(); const std::shared_ptr raylet_ipc_client_; - std::shared_ptr store_client_; + std::shared_ptr store_client_; /// Used to look up a plasma object's owner. ReferenceCounter &reference_counter_; std::function check_signals_; @@ -244,6 +246,7 @@ class CoreWorkerPlasmaStoreProvider { uint32_t object_store_full_delay_ms_; // Pointer to the shared buffer tracker. std::shared_ptr buffer_tracker_; + int64_t fetch_batch_size_ = 0; }; } // namespace core diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index 6bd83a868c8b..6adf3162fa47 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -247,6 +247,7 @@ ray_cc_test( deps = [ "//:ray_fakes", "//:ray_mock", + "//src/fakes/ray/object_manager/plasma:fake_plasma_client", "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/common:test_utils", "//src/ray/core_worker:core_worker_lib", diff --git a/src/ray/core_worker/tests/core_worker_test.cc b/src/ray/core_worker/tests/core_worker_test.cc index 6770eaaac5a4..7923893880c9 100644 --- a/src/ray/core_worker/tests/core_worker_test.cc +++ b/src/ray/core_worker/tests/core_worker_test.cc @@ -24,11 +24,16 @@ #include #include +#include "absl/container/flat_hash_set.h" #include "fakes/ray/common/asio/fake_periodical_runner.h" +#include "fakes/ray/object_manager/plasma/fake_plasma_client.h" #include "fakes/ray/pubsub/publisher.h" #include "fakes/ray/pubsub/subscriber.h" #include "fakes/ray/rpc/raylet/raylet_client.h" #include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/object_manager/plasma/client.h" +#include "ray/common/buffer.h" +#include "ray/common/ray_config.h" #include "ray/core_worker/actor_creator.h" #include "ray/core_worker/actor_manager.h" #include "ray/core_worker/context.h" @@ -38,6 +43,7 @@ #include "ray/core_worker/object_recovery_manager.h" #include "ray/core_worker/reference_count.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" +#include "ray/core_worker/store_provider/plasma_store_provider.h" #include "ray/core_worker/task_submission/actor_task_submitter.h" #include "ray/core_worker/task_submission/normal_task_submitter.h" #include "ray/ipc/fake_raylet_ipc_client.h" @@ -568,5 +574,72 @@ TEST_F(CoreWorkerTest, ActorTaskCancelDuringDepResolution) { } } +TEST(BatchingPassesTwoTwoOneIntoPlasmaGet, CallsPlasmaGetInCorrectBatches) { + auto fake_raylet = std::make_shared(); + // Build a ReferenceCounter with minimal dependencies. + rpc::Address addr; + addr.set_ip_address("127.0.0.1"); + auto is_node_dead = [](const NodeID &) { return false; }; + ReferenceCounter ref_counter(addr, + /*object_info_publisher=*/nullptr, + /*object_info_subscriber=*/nullptr, + is_node_dead); + + // Fake plasma client that records Get calls. + std::vector> observed_batches; + class RecordingPlasmaGetClient : public plasma::FakePlasmaClient { + public: + explicit RecordingPlasmaGetClient(std::vector> *observed) + : observed_(observed) {} + Status Get(const std::vector &object_ids, + int64_t timeout_ms, + std::vector *object_buffers) override { + if (observed_ != nullptr) { + observed_->push_back(object_ids); + } + object_buffers->resize(object_ids.size()); + for (size_t i = 0; i < object_ids.size(); i++) { + uint8_t byte = 0; + auto parent = std::make_shared(&byte, 1, /*copy_data=*/true); + (*object_buffers)[i].data = SharedMemoryBuffer::Slice(parent, 0, 1); + (*object_buffers)[i].metadata = SharedMemoryBuffer::Slice(parent, 0, 1); + } + return Status::OK(); + } + + private: + std::vector> *observed_; + }; + + auto fake_plasma = std::make_shared(&observed_batches); + + CoreWorkerPlasmaStoreProvider provider( + /*store_socket=*/"", + fake_raylet, + ref_counter, + /*check_signals=*/[] { return Status::OK(); }, + /*warmup=*/false, + /*store_client=*/fake_plasma, + /*fetch_batch_size=*/2, + /*get_current_call_site=*/nullptr); + + // Build a set of 5 object ids. + std::vector ids; + for (int i = 0; i < 5; i++) ids.push_back(ObjectID::FromRandom()); + absl::flat_hash_set idset(ids.begin(), ids.end()); + + absl::flat_hash_map> results; + bool got_exception = false; + WorkerContext ctx(WorkerType::WORKER, WorkerID::FromRandom(), JobID::FromInt(0)); + + ASSERT_TRUE(provider.Get(idset, /*timeout_ms=*/-1, ctx, &results, &got_exception).ok()); + + // Assert: batches seen by plasma Get are [2,2,1]. + ASSERT_EQ(observed_batches.size(), 3U); + EXPECT_EQ(observed_batches[0].size(), 2U); + EXPECT_EQ(observed_batches[1].size(), 2U); + EXPECT_EQ(observed_batches[2].size(), 1U); +} + } // namespace core } // namespace ray diff --git a/src/ray/object_manager/plasma/BUILD.bazel b/src/ray/object_manager/plasma/BUILD.bazel index 6efca1100e7a..ea88fcbe1501 100644 --- a/src/ray/object_manager/plasma/BUILD.bazel +++ b/src/ray/object_manager/plasma/BUILD.bazel @@ -51,6 +51,19 @@ ray_cc_library( ], ) +ray_cc_library( + name = "plasma_client_interface", + hdrs = ["client.h"], + deps = [ + "//src/ray/common:buffer", + "//src/ray/common:id", + "//src/ray/common:status", + "//src/ray/object_manager:object_manager_common", + "//src/ray/protobuf:common_cc_proto", + "@com_google_absl//absl/container:flat_hash_map", + ], +) + ray_cc_library( name = "plasma_shared_memory", srcs = ["shared_memory.cc"], diff --git a/src/ray/object_manager/plasma/client.h b/src/ray/object_manager/plasma/client.h index dcf4bdfef3f1..befba9e60f58 100644 --- a/src/ray/object_manager/plasma/client.h +++ b/src/ray/object_manager/plasma/client.h @@ -229,6 +229,11 @@ class PlasmaClientInterface { /// \param object_ids The list of IDs of the objects to delete. /// \return The return status. If all the objects are non-existent, return OK. virtual Status Delete(const std::vector &object_ids) = 0; + + /// Get the current debug string from the plasma store server. + /// + /// \return the debug string if successful, otherwise return an error status. + virtual StatusOr GetMemoryUsage() = 0; }; class PlasmaClient : public PlasmaClientInterface { @@ -282,7 +287,7 @@ class PlasmaClient : public PlasmaClientInterface { /// Get the current debug string from the plasma store server. /// /// \return the debug string if successful, otherwise return an error status. - StatusOr GetMemoryUsage(); + StatusOr GetMemoryUsage() override; /// Get the memory capacity of the store. /// diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index 8e4217ea9dfd..6b9469be35c5 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -178,6 +178,8 @@ class FakePlasmaClient : public plasma::PlasmaClientInterface { int64_t store_capacity() { return 1; } + StatusOr GetMemoryUsage() override { return std::string("fake"); } + private: absl::flat_hash_set objects_ids_in_plasma_; absl::flat_hash_map, std::vector>> From 646c4dc73bac4330299de2679400d84409851fcb Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Tue, 9 Sep 2025 08:47:03 -0700 Subject: [PATCH 527/634] [deps] upgrade boto3 to 1.29.x (#56363) upgrading boto3==2.28.17 -> boto3==1.29.7 --------- Signed-off-by: elliot-barn --- python/deplocks/llm/ray_test_py311_cpu.lock | 18 +- python/deplocks/llm/ray_test_py311_cu121.lock | 18 +- python/deplocks/llm/ray_test_py311_cu128.lock | 18 +- .../deplocks/llm/rayllm_test_py311_cpu.lock | 18 +- .../deplocks/llm/rayllm_test_py311_cu121.lock | 18 +- .../deplocks/llm/rayllm_test_py311_cu128.lock | 18 +- python/requirements/cloud-requirements.txt | 4 +- python/requirements/ml/data-requirements.txt | 2 +- python/requirements/test-requirements.txt | 2 +- python/requirements_compiled.txt | 10 +- .../byod/requirements_byod_3.9.txt | 238 +++++++++--------- .../byod/requirements_ml_byod_3.9.txt | 232 ++++++++--------- 12 files changed, 304 insertions(+), 292 deletions(-) diff --git a/python/deplocks/llm/ray_test_py311_cpu.lock b/python/deplocks/llm/ray_test_py311_cpu.lock index c142079eff8f..34f24980a9f5 100644 --- a/python/deplocks/llm/ray_test_py311_cpu.lock +++ b/python/deplocks/llm/ray_test_py311_cpu.lock @@ -272,16 +272,16 @@ bleach==6.1.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # nbconvert -boto3==1.28.17 \ - --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ - --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b +boto3==1.29.7 \ + --hash=sha256:1eb4c548118b5fc5e018dee956fd33e6fb249cd1f2def85f1bba816aef4d9f3e \ + --hash=sha256:96e9890ebe7cd823b5f4976dd676e112c000c6528c28e20a2f274590589dd18b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.31.17 \ - --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ - --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b +botocore==1.32.7 \ + --hash=sha256:58b33d02cafa23461c8a9d211b30e8cded992380a84de409379fd02811fa3e11 \ + --hash=sha256:c6795c731b04c8e3635588c44cfd1a4462fc5987859195522c96812cf3eceff9 # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt @@ -2892,9 +2892,9 @@ rsa==4.7.2 \ # -c /tmp/ray-deps/requirements_compiled.txt # google-auth # oauth2client -s3transfer==0.6.2 \ - --hash=sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084 \ - --hash=sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861 +s3transfer==0.8.0 \ + --hash=sha256:baa479dc2e63e5c2ed51611b4d46cdf0295e2070d8d0b86b22f335ee5b954986 \ + --hash=sha256:e8d6bd52ffd99841e3a57b34370a54841f12d3aab072af862cdcc50955288002 # via # -c /tmp/ray-deps/requirements_compiled.txt # boto3 diff --git a/python/deplocks/llm/ray_test_py311_cu121.lock b/python/deplocks/llm/ray_test_py311_cu121.lock index 87a35a550a95..833d864f0280 100644 --- a/python/deplocks/llm/ray_test_py311_cu121.lock +++ b/python/deplocks/llm/ray_test_py311_cu121.lock @@ -272,16 +272,16 @@ bleach==6.1.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # nbconvert -boto3==1.28.17 \ - --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ - --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b +boto3==1.29.7 \ + --hash=sha256:1eb4c548118b5fc5e018dee956fd33e6fb249cd1f2def85f1bba816aef4d9f3e \ + --hash=sha256:96e9890ebe7cd823b5f4976dd676e112c000c6528c28e20a2f274590589dd18b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.31.17 \ - --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ - --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b +botocore==1.32.7 \ + --hash=sha256:58b33d02cafa23461c8a9d211b30e8cded992380a84de409379fd02811fa3e11 \ + --hash=sha256:c6795c731b04c8e3635588c44cfd1a4462fc5987859195522c96812cf3eceff9 # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt @@ -2892,9 +2892,9 @@ rsa==4.7.2 \ # -c /tmp/ray-deps/requirements_compiled.txt # google-auth # oauth2client -s3transfer==0.6.2 \ - --hash=sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084 \ - --hash=sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861 +s3transfer==0.8.0 \ + --hash=sha256:baa479dc2e63e5c2ed51611b4d46cdf0295e2070d8d0b86b22f335ee5b954986 \ + --hash=sha256:e8d6bd52ffd99841e3a57b34370a54841f12d3aab072af862cdcc50955288002 # via # -c /tmp/ray-deps/requirements_compiled.txt # boto3 diff --git a/python/deplocks/llm/ray_test_py311_cu128.lock b/python/deplocks/llm/ray_test_py311_cu128.lock index ec467bbc1f61..19466b6fc222 100644 --- a/python/deplocks/llm/ray_test_py311_cu128.lock +++ b/python/deplocks/llm/ray_test_py311_cu128.lock @@ -272,16 +272,16 @@ bleach==6.1.0 \ # via # -c /tmp/ray-deps/requirements_compiled.txt # nbconvert -boto3==1.28.17 \ - --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ - --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b +boto3==1.29.7 \ + --hash=sha256:1eb4c548118b5fc5e018dee956fd33e6fb249cd1f2def85f1bba816aef4d9f3e \ + --hash=sha256:96e9890ebe7cd823b5f4976dd676e112c000c6528c28e20a2f274590589dd18b # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.31.17 \ - --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ - --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b +botocore==1.32.7 \ + --hash=sha256:58b33d02cafa23461c8a9d211b30e8cded992380a84de409379fd02811fa3e11 \ + --hash=sha256:c6795c731b04c8e3635588c44cfd1a4462fc5987859195522c96812cf3eceff9 # via # -c /tmp/ray-deps/requirements_compiled.txt # -r python/requirements/cloud-requirements.txt @@ -2892,9 +2892,9 @@ rsa==4.7.2 \ # -c /tmp/ray-deps/requirements_compiled.txt # google-auth # oauth2client -s3transfer==0.6.2 \ - --hash=sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084 \ - --hash=sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861 +s3transfer==0.8.0 \ + --hash=sha256:baa479dc2e63e5c2ed51611b4d46cdf0295e2070d8d0b86b22f335ee5b954986 \ + --hash=sha256:e8d6bd52ffd99841e3a57b34370a54841f12d3aab072af862cdcc50955288002 # via # -c /tmp/ray-deps/requirements_compiled.txt # boto3 diff --git a/python/deplocks/llm/rayllm_test_py311_cpu.lock b/python/deplocks/llm/rayllm_test_py311_cpu.lock index a64a3ea650a1..27b0f5a357e6 100644 --- a/python/deplocks/llm/rayllm_test_py311_cpu.lock +++ b/python/deplocks/llm/rayllm_test_py311_cpu.lock @@ -376,16 +376,16 @@ bleach==6.1.0 \ # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # nbconvert -boto3==1.28.17 \ - --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ - --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b +boto3==1.29.7 \ + --hash=sha256:1eb4c548118b5fc5e018dee956fd33e6fb249cd1f2def85f1bba816aef4d9f3e \ + --hash=sha256:96e9890ebe7cd823b5f4976dd676e112c000c6528c28e20a2f274590589dd18b # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.31.17 \ - --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ - --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b +botocore==1.32.7 \ + --hash=sha256:58b33d02cafa23461c8a9d211b30e8cded992380a84de409379fd02811fa3e11 \ + --hash=sha256:c6795c731b04c8e3635588c44cfd1a4462fc5987859195522c96812cf3eceff9 # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # -r python/requirements/cloud-requirements.txt @@ -3906,9 +3906,9 @@ rsa==4.7.2 \ # -c python/deplocks/llm/ray_test_py311_cpu.lock # google-auth # oauth2client -s3transfer==0.6.2 \ - --hash=sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084 \ - --hash=sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861 +s3transfer==0.8.0 \ + --hash=sha256:baa479dc2e63e5c2ed51611b4d46cdf0295e2070d8d0b86b22f335ee5b954986 \ + --hash=sha256:e8d6bd52ffd99841e3a57b34370a54841f12d3aab072af862cdcc50955288002 # via # -c python/deplocks/llm/ray_test_py311_cpu.lock # boto3 diff --git a/python/deplocks/llm/rayllm_test_py311_cu121.lock b/python/deplocks/llm/rayllm_test_py311_cu121.lock index 0f94bbb0514d..be1ce34fce70 100644 --- a/python/deplocks/llm/rayllm_test_py311_cu121.lock +++ b/python/deplocks/llm/rayllm_test_py311_cu121.lock @@ -376,16 +376,16 @@ bleach==6.1.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # nbconvert -boto3==1.28.17 \ - --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ - --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b +boto3==1.29.7 \ + --hash=sha256:1eb4c548118b5fc5e018dee956fd33e6fb249cd1f2def85f1bba816aef4d9f3e \ + --hash=sha256:96e9890ebe7cd823b5f4976dd676e112c000c6528c28e20a2f274590589dd18b # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.31.17 \ - --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ - --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b +botocore==1.32.7 \ + --hash=sha256:58b33d02cafa23461c8a9d211b30e8cded992380a84de409379fd02811fa3e11 \ + --hash=sha256:c6795c731b04c8e3635588c44cfd1a4462fc5987859195522c96812cf3eceff9 # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # -r python/requirements/cloud-requirements.txt @@ -3997,9 +3997,9 @@ rsa==4.7.2 \ # -c python/deplocks/llm/ray_test_py311_cu121.lock # google-auth # oauth2client -s3transfer==0.6.2 \ - --hash=sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084 \ - --hash=sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861 +s3transfer==0.8.0 \ + --hash=sha256:baa479dc2e63e5c2ed51611b4d46cdf0295e2070d8d0b86b22f335ee5b954986 \ + --hash=sha256:e8d6bd52ffd99841e3a57b34370a54841f12d3aab072af862cdcc50955288002 # via # -c python/deplocks/llm/ray_test_py311_cu121.lock # boto3 diff --git a/python/deplocks/llm/rayllm_test_py311_cu128.lock b/python/deplocks/llm/rayllm_test_py311_cu128.lock index e4a81a7eab1d..df88832ae1be 100644 --- a/python/deplocks/llm/rayllm_test_py311_cu128.lock +++ b/python/deplocks/llm/rayllm_test_py311_cu128.lock @@ -376,16 +376,16 @@ bleach==6.1.0 \ # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # nbconvert -boto3==1.28.17 \ - --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ - --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b +boto3==1.29.7 \ + --hash=sha256:1eb4c548118b5fc5e018dee956fd33e6fb249cd1f2def85f1bba816aef4d9f3e \ + --hash=sha256:96e9890ebe7cd823b5f4976dd676e112c000c6528c28e20a2f274590589dd18b # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt # smart-open -botocore==1.31.17 \ - --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ - --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b +botocore==1.32.7 \ + --hash=sha256:58b33d02cafa23461c8a9d211b30e8cded992380a84de409379fd02811fa3e11 \ + --hash=sha256:c6795c731b04c8e3635588c44cfd1a4462fc5987859195522c96812cf3eceff9 # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # -r python/requirements/cloud-requirements.txt @@ -3920,9 +3920,9 @@ rsa==4.7.2 \ # -c python/deplocks/llm/ray_test_py311_cu128.lock # google-auth # oauth2client -s3transfer==0.6.2 \ - --hash=sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084 \ - --hash=sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861 +s3transfer==0.8.0 \ + --hash=sha256:baa479dc2e63e5c2ed51611b4d46cdf0295e2070d8d0b86b22f335ee5b954986 \ + --hash=sha256:e8d6bd52ffd99841e3a57b34370a54841f12d3aab072af862cdcc50955288002 # via # -c python/deplocks/llm/ray_test_py311_cu128.lock # boto3 diff --git a/python/requirements/cloud-requirements.txt b/python/requirements/cloud-requirements.txt index faccc9caa716..755a9bf44428 100644 --- a/python/requirements/cloud-requirements.txt +++ b/python/requirements/cloud-requirements.txt @@ -12,8 +12,8 @@ smart_open[s3,gcs,azure,http] adlfs[abfs] # Anyscale CLI requirements -boto3==1.28.17 -botocore>=1.31.17,<1.32.0 +boto3==1.29.7 +botocore==1.32.7 aiohttp>=3.7.4.post0 certifi>=2024.8.30 Click>=7.0 diff --git a/python/requirements/ml/data-requirements.txt b/python/requirements/ml/data-requirements.txt index 94f628a17cf9..931dea6f9143 100644 --- a/python/requirements/ml/data-requirements.txt +++ b/python/requirements/ml/data-requirements.txt @@ -6,7 +6,7 @@ dask[complete]==2023.6.1; python_version < '3.12' distributed==2023.6.1; python_version < '3.12' dask[complete]==2025.5.0; python_version >= '3.12' distributed==2025.5.0; python_version >= '3.12' -aioboto3==11.3.0 +aioboto3==12.1.0 crc32c==2.3 flask_cors bokeh==2.4.3; python_version < '3.12' diff --git a/python/requirements/test-requirements.txt b/python/requirements/test-requirements.txt index b59f858e70a1..7fa619651a3f 100644 --- a/python/requirements/test-requirements.txt +++ b/python/requirements/test-requirements.txt @@ -10,7 +10,7 @@ azure-mgmt-network==25.4.0 azure-mgmt-resource==23.1.1 msrestazure==0.6.4 beautifulsoup4==4.11.1 -boto3==1.28.17 +boto3==1.29.7 # Todo: investigate if we can get rid of this and exchange for ray.cloudpickle cloudpickle==2.2.0 ; python_version < "3.12" cloudpickle==3.0.0 ; python_version >= "3.12" diff --git a/python/requirements_compiled.txt b/python/requirements_compiled.txt index ad16f111c782..4b530883bb8a 100644 --- a/python/requirements_compiled.txt +++ b/python/requirements_compiled.txt @@ -34,9 +34,9 @@ aimrecords==0.0.7 # via aim aimrocks==0.5.2 # via aim -aioboto3==11.3.0 +aioboto3==12.1.0 # via -r python/requirements/ml/data-requirements.txt -aiobotocore==2.6.0 +aiobotocore==2.8.0 # via # aioboto3 # s3fs @@ -231,7 +231,7 @@ boltons==21.0.0 # semgrep boto==2.49.0 # via gcs-oauth2-boto-plugin -boto3==1.28.17 +boto3==1.29.7 # via # -r python/requirements/cloud-requirements.txt # -r python/requirements/test-requirements.txt @@ -241,7 +241,7 @@ boto3==1.28.17 # moto # smart-open # snowflake-connector-python -botocore==1.31.17 +botocore==1.32.7 # via # -r python/requirements/cloud-requirements.txt # aiobotocore @@ -1967,7 +1967,7 @@ ruamel-yaml-clib==0.2.8 # via ruamel-yaml s3fs==2023.12.1 # via -r python/requirements/ml/core-requirements.txt -s3transfer==0.6.2 +s3transfer==0.8.0 # via boto3 safetensors==0.4.3 # via diff --git a/release/ray_release/byod/requirements_byod_3.9.txt b/release/ray_release/byod/requirements_byod_3.9.txt index 1c291b2c3970..870a5236d18d 100644 --- a/release/ray_release/byod/requirements_byod_3.9.txt +++ b/release/ray_release/byod/requirements_byod_3.9.txt @@ -14,9 +14,9 @@ absl-py==1.4.0 \ # -c release/ray_release/byod/requirements_compiled.txt # tensorboard # tensorflow -aiobotocore==2.6.0 \ - --hash=sha256:0186e6a843364748cdbbf76ee98e9337c44f71a4e694ad1b110d5c516fbce909 \ - --hash=sha256:4805d0140bdfa17bfc2d0ba1243c8cc4273e927201fca5cf2e497c0004a9fab7 +aiobotocore==2.8.0 \ + --hash=sha256:32e632fea387acd45416c2bbc03828ee2c2a66a7dc4bd3a9bcb808dea249c469 \ + --hash=sha256:f160497cef21cfffc1a8d4219eeb27bb7b243389c2d021a812b9c0e3fb8e2bd1 # via # -c release/ray_release/byod/requirements_compiled.txt # s3fs @@ -204,15 +204,15 @@ boto==2.49.0 \ # via # -c release/ray_release/byod/requirements_compiled.txt # gcs-oauth2-boto-plugin -boto3==1.28.17 \ - --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ - --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b +boto3==1.29.7 \ + --hash=sha256:1eb4c548118b5fc5e018dee956fd33e6fb249cd1f2def85f1bba816aef4d9f3e \ + --hash=sha256:96e9890ebe7cd823b5f4976dd676e112c000c6528c28e20a2f274590589dd18b # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_byod_3.9.in -botocore==1.31.17 \ - --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ - --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b +botocore==1.32.7 \ + --hash=sha256:58b33d02cafa23461c8a9d211b30e8cded992380a84de409379fd02811fa3e11 \ + --hash=sha256:c6795c731b04c8e3635588c44cfd1a4462fc5987859195522c96812cf3eceff9 # via # -c release/ray_release/byod/requirements_compiled.txt # aiobotocore @@ -2169,114 +2169,113 @@ pycparser==2.21 \ # via # -c release/ray_release/byod/requirements_compiled.txt # cffi -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_byod_3.9.in # fastapi -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c release/ray_release/byod/requirements_compiled.txt # pydantic @@ -2639,9 +2638,9 @@ s3fs==2023.12.1 \ # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_byod_3.9.in -s3transfer==0.6.2 \ - --hash=sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084 \ - --hash=sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861 +s3transfer==0.8.0 \ + --hash=sha256:baa479dc2e63e5c2ed51611b4d46cdf0295e2070d8d0b86b22f335ee5b954986 \ + --hash=sha256:e8d6bd52ffd99841e3a57b34370a54841f12d3aab072af862cdcc50955288002 # via # -c release/ray_release/byod/requirements_compiled.txt # boto3 @@ -2921,6 +2920,13 @@ typing-extensions==4.12.2 \ # starlette # tensorflow # typer + # typing-inspection +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c release/ray_release/byod/requirements_compiled.txt + # pydantic urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 diff --git a/release/ray_release/byod/requirements_ml_byod_3.9.txt b/release/ray_release/byod/requirements_ml_byod_3.9.txt index 687a325b9329..abe636e92761 100644 --- a/release/ray_release/byod/requirements_ml_byod_3.9.txt +++ b/release/ray_release/byod/requirements_ml_byod_3.9.txt @@ -203,15 +203,15 @@ boto==2.49.0 \ # via # -c release/ray_release/byod/requirements_compiled.txt # gcs-oauth2-boto-plugin -boto3==1.28.17 \ - --hash=sha256:90f7cfb5e1821af95b1fc084bc50e6c47fa3edc99f32de1a2591faa0c546bea7 \ - --hash=sha256:bca0526f819e0f19c0f1e6eba3e2d1d6b6a92a45129f98c0d716e5aab6d9444b +boto3==1.29.7 \ + --hash=sha256:1eb4c548118b5fc5e018dee956fd33e6fb249cd1f2def85f1bba816aef4d9f3e \ + --hash=sha256:96e9890ebe7cd823b5f4976dd676e112c000c6528c28e20a2f274590589dd18b # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_ml_byod_3.9.in -botocore==1.31.17 \ - --hash=sha256:396459065dba4339eb4da4ec8b4e6599728eb89b7caaceea199e26f7d824a41c \ - --hash=sha256:6ac34a1d34aa3750e78b77b8596617e2bab938964694d651939dba2cbde2c12b +botocore==1.32.7 \ + --hash=sha256:58b33d02cafa23461c8a9d211b30e8cded992380a84de409379fd02811fa3e11 \ + --hash=sha256:c6795c731b04c8e3635588c44cfd1a4462fc5987859195522c96812cf3eceff9 # via # -c release/ray_release/byod/requirements_compiled.txt # boto3 @@ -2815,116 +2815,115 @@ pycparser==2.21 \ # via # -c release/ray_release/byod/requirements_compiled.txt # cffi -pydantic==2.10.0 \ - --hash=sha256:0aca0f045ff6e2f097f1fe89521115335f15049eeb8a7bef3dafe4b19a74e289 \ - --hash=sha256:5e7807ba9201bdf61b1b58aa6eb690916c40a47acfb114b1b4fef3e7fd5b30fc +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b # via # -c release/ray_release/byod/requirements_compiled.txt # -r release/ray_release/byod/requirements_ml_byod_3.9.in # albumentations # deepspeed # fastapi -pydantic-core==2.27.0 \ - --hash=sha256:0aa4d1b2eba9a325897308b3124014a142cdccb9f3e016f31d3ebee6b5ea5e75 \ - --hash=sha256:0d06b667e53320332be2bf6f9461f4a9b78092a079b8ce8634c9afaa7e10cd9f \ - --hash=sha256:153017e3d6cd3ce979de06d84343ca424bb6092727375eba1968c8b4693c6ecb \ - --hash=sha256:15e350efb67b855cd014c218716feea4986a149ed1f42a539edd271ee074a196 \ - --hash=sha256:185ef205256cd8b38431205698531026979db89a79587725c1e55c59101d64e9 \ - --hash=sha256:1da0c98a85a6c6ed702d5556db3b09c91f9b0b78de37b7593e2de8d03238807a \ - --hash=sha256:225bfff5d425c34e1fd562cef52d673579d59b967d9de06178850c4802af9039 \ - --hash=sha256:24f984fc7762ed5f806d9e8c4c77ea69fdb2afd987b4fd319ef06c87595a8c55 \ - --hash=sha256:25a7fd4de38f7ff99a37e18fa0098c3140286451bc823d1746ba80cec5b433a1 \ - --hash=sha256:2883b260f7a93235488699d39cbbd94fa7b175d3a8063fbfddd3e81ad9988cb2 \ - --hash=sha256:2a51ce96224eadd1845150b204389623c8e129fde5a67a84b972bd83a85c6c40 \ - --hash=sha256:2be0ad541bb9f059954ccf8877a49ed73877f862529575ff3d54bf4223e4dd61 \ - --hash=sha256:31a2cae5f059329f9cfe3d8d266d3da1543b60b60130d186d9b6a3c20a346361 \ - --hash=sha256:333c840a1303d1474f491e7be0b718226c730a39ead0f7dab2c7e6a2f3855555 \ - --hash=sha256:33d14369739c5d07e2e7102cdb0081a1fa46ed03215e07f097b34e020b83b1ae \ - --hash=sha256:35380671c3c921fe8adf31ad349dc6f7588b7e928dbe44e1093789734f607399 \ - --hash=sha256:359e7951f04ad35111b5ddce184db3391442345d0ab073aa63a95eb8af25a5ef \ - --hash=sha256:36aa167f69d8807ba7e341d67ea93e50fcaaf6bc433bb04939430fa3dab06f31 \ - --hash=sha256:395e3e1148fa7809016231f8065f30bb0dc285a97b4dc4360cd86e17bab58af7 \ - --hash=sha256:3e8d89c276234579cd3d095d5fa2a44eb10db9a218664a17b56363cddf226ff3 \ - --hash=sha256:3eb8849445c26b41c5a474061032c53e14fe92a11a5db969f722a2716cd12206 \ - --hash=sha256:3fd8bc2690e7c39eecdf9071b6a889ce7b22b72073863940edc2a0a23750ca90 \ - --hash=sha256:400bf470e4327e920883b51e255617dfe4496d4e80c3fea0b5a5d0bf2c404dd4 \ - --hash=sha256:4148dc9184ab79e356dc00a4199dc0ee8647973332cb385fc29a7cced49b9f9c \ - --hash=sha256:433689845288f9a1ee5714444e65957be26d30915f7745091ede4a83cfb2d7bb \ - --hash=sha256:43b61989068de9ce62296cde02beffabcadb65672207fc51e7af76dca75e6636 \ - --hash=sha256:4523c4009c3f39d948e01962223c9f5538602e7087a628479b723c939fab262d \ - --hash=sha256:483c2213a609e7db2c592bbc015da58b6c75af7360ca3c981f178110d9787bcf \ - --hash=sha256:49633583eb7dc5cba61aaf7cdb2e9e662323ad394e543ee77af265736bcd3eaa \ - --hash=sha256:4b51f964fcbb02949fc546022e56cdb16cda457af485e9a3e8b78ac2ecf5d77e \ - --hash=sha256:4bf1340ae507f6da6360b24179c2083857c8ca7644aab65807023cf35404ea8d \ - --hash=sha256:4fb49cfdb53af5041aba909be00cccfb2c0d0a2e09281bf542371c5fd36ad04c \ - --hash=sha256:510b11e9c3b1a852876d1ccd8d5903684336d635214148637ceb27366c75a467 \ - --hash=sha256:513cb14c0cc31a4dfd849a4674b20c46d87b364f997bbcb02282306f5e187abf \ - --hash=sha256:58560828ee0951bb125c6f2862fbc37f039996d19ceb6d8ff1905abf7da0bf3d \ - --hash=sha256:58ab0d979c969983cdb97374698d847a4acffb217d543e172838864636ef10d9 \ - --hash=sha256:5982048129f40b082c2654de10c0f37c67a14f5ff9d37cf35be028ae982f26df \ - --hash=sha256:5ab325fc86fbc077284c8d7f996d904d30e97904a87d6fb303dce6b3de7ebba9 \ - --hash=sha256:5cc822ab90a70ea3a91e6aed3afac570b276b1278c6909b1d384f745bd09c714 \ - --hash=sha256:5f2b19b8d6fca432cb3acf48cf5243a7bf512988029b6e6fd27e9e8c0a204d85 \ - --hash=sha256:5fc72fbfebbf42c0856a824b8b0dc2b5cd2e4a896050281a21cfa6fed8879cb1 \ - --hash=sha256:6354e18a9be37bfa124d6b288a87fb30c673745806c92956f1a25e3ae6e76b96 \ - --hash=sha256:678f66462058dd978702db17eb6a3633d634f7aa0deaea61e0a674152766d3fc \ - --hash=sha256:68950bc08f9735306322bfc16a18391fcaac99ded2509e1cc41d03ccb6013cfe \ - --hash=sha256:68ef5377eb582fa4343c9d0b57a5b094046d447b4c73dd9fbd9ffb216f829e7d \ - --hash=sha256:6b4c19525c3538fbc0bbda6229f9682fb8199ce9ac37395880e6952798e00373 \ - --hash=sha256:6bb69bf3b6500f195c3deb69c1205ba8fc3cb21d1915f1f158a10d6b1ef29b6a \ - --hash=sha256:6e19401742ed7b69e51d8e4df3c03ad5ec65a83b36244479fd70edde2828a5d9 \ - --hash=sha256:6f4a53af9e81d757756508b57cae1cf28293f0f31b9fa2bfcb416cc7fb230f9d \ - --hash=sha256:6fda87808429c520a002a85d6e7cdadbf58231d60e96260976c5b8f9a12a8e13 \ - --hash=sha256:78f841523729e43e3928a364ec46e2e3f80e6625a4f62aca5c345f3f626c6e8a \ - --hash=sha256:7a6ebfac28fd51890a61df36ef202adbd77d00ee5aca4a3dadb3d9ed49cfb929 \ - --hash=sha256:7b0202ebf2268954090209a84f9897345719e46a57c5f2c9b7b250ca0a9d3e63 \ - --hash=sha256:8117839a9bdbba86e7f9df57018fe3b96cec934c3940b591b0fd3fbfb485864a \ - --hash=sha256:82e1ad4ca170e8af4c928b67cff731b6296e6a0a0981b97b2eb7c275cc4e15bd \ - --hash=sha256:836a4bfe0cc6d36dc9a9cc1a7b391265bf6ce9d1eb1eac62ac5139f5d8d9a6fa \ - --hash=sha256:84af1cf7bfdcbc6fcf5a5f70cc9896205e0350306e4dd73d54b6a18894f79386 \ - --hash=sha256:84e35afd9e10b2698e6f2f32256678cb23ca6c1568d02628033a837638b3ed12 \ - --hash=sha256:884f1806609c2c66564082540cffc96868c5571c7c3cf3a783f63f2fb49bd3cd \ - --hash=sha256:8a150392102c402c538190730fda06f3bce654fc498865579a9f2c1d2b425833 \ - --hash=sha256:8e21d927469d04b39386255bf00d0feedead16f6253dcc85e9e10ddebc334084 \ - --hash=sha256:8e96ca781e0c01e32115912ebdf7b3fb0780ce748b80d7d28a0802fa9fbaf44e \ - --hash=sha256:8ee4c2a75af9fe21269a4a0898c5425afb01af1f5d276063f57e2ae1bc64e191 \ - --hash=sha256:91bc66f878557313c2a6bcf396e7befcffe5ab4354cfe4427318968af31143c3 \ - --hash=sha256:951e71da6c89d354572098bada5ba5b5dc3a9390c933af8a614e37755d3d1840 \ - --hash=sha256:99b2863c1365f43f74199c980a3d40f18a218fbe683dd64e470199db426c4d6a \ - --hash=sha256:9a8fbf506fde1529a1e3698198fe64bfbe2e0c09557bc6a7dcf872e7c01fec40 \ - --hash=sha256:9ce048deb1e033e7a865ca384770bccc11d44179cf09e5193a535c4c2f497bdc \ - --hash=sha256:9fe94d9d2a2b4edd7a4b22adcd45814b1b59b03feb00e56deb2e89747aec7bfe \ - --hash=sha256:a291d0b4243a259c8ea7e2b84eb9ccb76370e569298875a7c5e3e71baf49057a \ - --hash=sha256:a5c022bb0d453192426221605efc865373dde43b17822a264671c53b068ac20c \ - --hash=sha256:abb4785894936d7682635726613c44578c420a096729f1978cd061a7e72d5275 \ - --hash=sha256:b872c86d8d71827235c7077461c502feb2db3f87d9d6d5a9daa64287d75e4fa0 \ - --hash=sha256:bf37b72834e7239cf84d4a0b2c050e7f9e48bced97bad9bdf98d26b8eb72e846 \ - --hash=sha256:c0c431e4be5c1a0c6654e0c31c661cd89e0ca956ef65305c3c3fd96f4e72ca39 \ - --hash=sha256:c5726eec789ee38f2c53b10b1821457b82274f81f4f746bb1e666d8741fcfadb \ - --hash=sha256:c6fcb3fa3855d583aa57b94cf146f7781d5d5bc06cb95cb3afece33d31aac39b \ - --hash=sha256:c86679f443e7085ea55a7376462553996c688395d18ef3f0d3dbad7838f857a2 \ - --hash=sha256:c91e3c04f5191fd3fb68764bddeaf02025492d5d9f23343b283870f6ace69708 \ - --hash=sha256:c921ad596ff1a82f9c692b0758c944355abc9f0de97a4c13ca60ffc6d8dc15d4 \ - --hash=sha256:c9ed88b398ba7e3bad7bd64d66cc01dcde9cfcb7ec629a6fd78a82fa0b559d78 \ - --hash=sha256:cd2ac6b919f7fed71b17fe0b4603c092a4c9b5bae414817c9c81d3c22d1e1bcc \ - --hash=sha256:d28ca7066d6cdd347a50d8b725dc10d9a1d6a1cce09836cf071ea6a2d4908be0 \ - --hash=sha256:d29e235ce13c91902ef3efc3d883a677655b3908b1cbc73dee816e5e1f8f7739 \ - --hash=sha256:d8b5ee4ae9170e2775d495b81f414cc20268041c42571530513496ba61e94ba3 \ - --hash=sha256:db72e40628967f6dc572020d04b5f800d71264e0531c6da35097e73bdf38b003 \ - --hash=sha256:df45c4073bed486ea2f18757057953afed8dd77add7276ff01bccb79982cf46c \ - --hash=sha256:dfa5f5c0a4c8fced1422dc2ca7eefd872d5d13eb33cf324361dbf1dbfba0a9fe \ - --hash=sha256:e015833384ca3e1a0565a79f5d953b0629d9138021c27ad37c92a9fa1af7623c \ - --hash=sha256:e15315691fe2253eb447503153acef4d7223dfe7e7702f9ed66539fcd0c43801 \ - --hash=sha256:e65466b31be1070b4a5b7dbfbd14b247884cb8e8b79c64fb0f36b472912dbaea \ - --hash=sha256:e7820bb0d65e3ce1e3e70b6708c2f66143f55912fa02f4b618d0f08b61575f12 \ - --hash=sha256:e851a051f7260e6d688267eb039c81f05f23a19431bd7dfa4bf5e3cb34c108cd \ - --hash=sha256:e9f9feee7f334b72ceae46313333d002b56f325b5f04271b4ae2aadd9e993ae4 \ - --hash=sha256:eb40f828bc2f73f777d1eb8fee2e86cd9692a4518b63b6b5aa8af915dfd3207b \ - --hash=sha256:eb704155e73b833801c247f39d562229c0303f54770ca14fb1c053acb376cf10 \ - --hash=sha256:edb1bfd45227dec8d50bc7c7d86463cd8728bcc574f9b07de7369880de4626a3 \ - --hash=sha256:ee7d9d5537daf6d5c74a83b38a638cc001b648096c1cae8ef695b0c919d9d379 \ - --hash=sha256:f57783fbaf648205ac50ae7d646f27582fc706be3977e87c3c124e7a92407b10 \ - --hash=sha256:ff63a92f6e249514ef35bc795de10745be0226eaea06eb48b4bbeaa0c8850a4a +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d # via # -c release/ray_release/byod/requirements_compiled.txt # pydantic @@ -3414,9 +3413,9 @@ rsa==4.7.2 \ # gcs-oauth2-boto-plugin # google-auth # oauth2client -s3transfer==0.6.2 \ - --hash=sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084 \ - --hash=sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861 +s3transfer==0.8.0 \ + --hash=sha256:baa479dc2e63e5c2ed51611b4d46cdf0295e2070d8d0b86b22f335ee5b954986 \ + --hash=sha256:e8d6bd52ffd99841e3a57b34370a54841f12d3aab072af862cdcc50955288002 # via # -c release/ray_release/byod/requirements_compiled.txt # boto3 @@ -4479,7 +4478,14 @@ typing-extensions==4.12.2 \ # starlette # torch # typer + # typing-inspection # wandb +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c release/ray_release/byod/requirements_compiled.txt + # pydantic urllib3==1.26.19 \ --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 From 7dab7772685ff5fa0eaed8a19c0719f90e84bc4f Mon Sep 17 00:00:00 2001 From: czgdp1807 Date: Tue, 9 Sep 2025 21:44:01 +0530 Subject: [PATCH 528/634] Enable ruff lint for the entire code base (#56080) Signed-off-by: czgdp1807 Signed-off-by: Gagandeep Singh Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> --- .pre-commit-config.yaml | 1 - bazel/gen_extract.py | 2 +- ci/lint/git-clang-format | 1 + ci/ray_ci/anyscale_docker_container.py | 3 +- ci/ray_ci/test_anyscale_docker_container.py | 2 +- ci/raydepsets/cli.py | 11 ++-- ci/raydepsets/tests/test_cli.py | 3 +- ci/raydepsets/tests/test_workspace.py | 2 +- pyproject.toml | 2 +- python/ray/_common/network_utils.py | 3 +- python/ray/_common/test_utils.py | 11 ++-- .../ray/_common/tests/test_network_utils.py | 3 +- .../_common/tests/test_ray_option_utils.py | 13 +++-- .../tests/test_signal_semaphore_utils.py | 9 +-- python/ray/_common/tests/test_signature.py | 9 +-- python/ray/_common/tests/test_usage_stats.py | 16 ++--- python/ray/_common/tests/test_utils.py | 10 ++-- .../_common/tests/test_wait_for_condition.py | 5 +- python/ray/_common/usage/usage_lib.py | 2 +- python/ray/_common/utils.py | 5 +- python/ray/_raylet.pyi | 25 ++++---- .../ray/air/_internal/device_manager/npu.py | 2 +- python/ray/air/_internal/torch_utils.py | 15 +++-- python/ray/air/config.py | 4 +- python/ray/air/tests/test_air_usage.py | 4 +- python/ray/air/tests/test_arrow.py | 2 +- .../ray/air/tests/test_integration_wandb.py | 2 +- .../ray/air/util/object_extensions/arrow.py | 2 +- .../ray/air/util/tensor_extensions/arrow.py | 16 +++-- .../ray/air/util/tensor_extensions/utils.py | 2 +- python/ray/air/util/torch_dist.py | 2 +- python/ray/autoscaler/_private/gcp/config.py | 3 +- .../autoscaler/_private/gcp/node_provider.py | 2 +- .../_private/kuberay/node_provider.py | 2 +- .../_private/kuberay/run_autoscaler.py | 8 +-- python/ray/autoscaler/_private/monitor.py | 6 +- .../_private/spark/node_provider.py | 2 +- python/ray/autoscaler/_private/util.py | 2 +- .../autoscaler/local/coordinator_server.py | 4 +- python/ray/autoscaler/sdk/sdk.py | 6 +- python/ray/autoscaler/v2/autoscaler.py | 2 +- .../autoscaler/v2/instance_manager/config.py | 2 +- .../v2/instance_manager/ray_installer.py | 4 +- .../v2/instance_manager/reconciler.py | 8 +-- .../subscribers/threaded_ray_installer.py | 4 +- python/ray/autoscaler/v2/monitor.py | 8 +-- python/ray/autoscaler/v2/scheduler.py | 2 +- python/ray/autoscaler/v2/tests/test_e2e.py | 4 +- .../ray/autoscaler/v2/tests/test_scheduler.py | 2 +- .../v2/tests/test_threaded_ray_installer.py | 2 +- python/ray/autoscaler/v2/utils.py | 4 +- python/ray/client_builder.py | 3 +- python/ray/dashboard/agent.py | 2 +- python/ray/dashboard/dashboard.py | 8 +-- python/ray/dashboard/head.py | 4 +- python/ray/dashboard/http_server_agent.py | 2 +- python/ray/dashboard/http_server_head.py | 5 +- .../modules/aggregator/aggregator_agent.py | 20 ++++--- .../aggregator/tests/test_aggregator_agent.py | 58 +++++++++---------- .../ray/dashboard/modules/event/event_head.py | 2 +- .../modules/event/tests/test_event.py | 4 +- python/ray/dashboard/modules/job/cli.py | 2 +- python/ray/dashboard/modules/job/job_head.py | 4 +- .../ray/dashboard/modules/job/job_manager.py | 2 +- .../dashboard/modules/job/job_supervisor.py | 4 +- .../modules/job/tests/test_common.py | 2 +- .../job/tests/test_component_activities.py | 2 +- .../modules/job/tests/test_http_job_server.py | 2 +- .../modules/job/tests/test_job_agent.py | 4 +- .../modules/job/tests/test_job_manager.py | 13 +++-- .../dashboard/modules/job/tests/test_sdk.py | 2 +- .../dashboards/data_dashboard_panels.py | 2 +- .../dashboards/train_dashboard_panels.py | 3 +- .../dashboard/modules/node/tests/test_node.py | 2 +- .../modules/reporter/gpu_providers.py | 2 +- .../modules/reporter/healthz_agent.py | 2 +- .../modules/reporter/reporter_agent.py | 25 ++++---- .../modules/reporter/reporter_head.py | 4 +- .../modules/reporter/tests/test_actors.py | 7 ++- .../reporter/tests/test_gpu_providers.py | 8 +-- .../modules/reporter/tests/test_healthz.py | 2 +- .../modules/reporter/tests/test_reporter.py | 6 +- .../serve/tests/test_serve_dashboard.py | 2 +- .../serve/tests/test_serve_dashboard_2.py | 2 +- .../ray/dashboard/modules/state/state_head.py | 2 +- .../modules/usage_stats/usage_stats_head.py | 2 +- .../dashboard/subprocesses/tests/test_e2e.py | 7 +-- python/ray/dashboard/tests/test_dashboard.py | 10 ++-- python/ray/dashboard/utils.py | 2 +- python/ray/exceptions.py | 1 - python/ray/experimental/__init__.py | 2 +- .../channel/accelerator_context.py | 9 +-- python/ray/experimental/channel/common.py | 2 +- .../channel/communicator_handle.py | 1 + python/ray/experimental/channel/nccl_group.py | 2 +- .../channel/serialization_context.py | 2 +- .../torch_tensor_accelerator_channel.py | 14 ++--- .../ray/experimental/collective/__init__.py | 13 ++--- .../ray/experimental/collective/collective.py | 9 ++- .../collective/collective_tensor_transport.py | 9 ++- .../experimental/collective/communicator.py | 2 +- .../collective/nixl_tensor_transport.py | 10 ++-- .../ray/experimental/collective/operations.py | 4 +- .../collective/tensor_transport_manager.py | 11 ++-- python/ray/experimental/collective/util.py | 11 ++-- .../gpu_object_manager/gpu_object_manager.py | 11 ++-- .../gpu_object_manager/gpu_object_store.py | 9 ++- python/ray/remote_function.py | 2 +- python/ray/runtime_context.py | 2 +- python/ray/runtime_env/runtime_env.py | 2 +- python/ray/util/__init__.py | 7 +-- python/ray/util/accelerators/__init__.py | 32 +++++----- python/ray/util/accelerators/tpu.py | 1 + python/ray/util/actor_group.py | 6 +- python/ray/util/annotations.py | 4 +- python/ray/util/check_open_ports.py | 12 ++-- python/ray/util/check_serialize.py | 3 +- python/ray/util/client/__init__.py | 3 +- python/ray/util/client/client_app.py | 3 +- python/ray/util/client/client_pickler.py | 29 +++++----- python/ray/util/client/common.py | 2 +- python/ray/util/client/dataclient.py | 8 +-- python/ray/util/client/examples/run_tune.py | 3 +- python/ray/util/client/logsclient.py | 7 +-- python/ray/util/client/options.py | 4 +- python/ray/util/client/ray_client_helpers.py | 6 +- python/ray/util/client/runtime_context.py | 2 +- python/ray/util/client/server/dataservicer.py | 20 +++---- python/ray/util/client/server/proxier.py | 12 ++-- python/ray/util/client/server/server.py | 4 +- .../ray/util/client/server/server_pickler.py | 14 ++--- python/ray/util/client/server/server_stubs.py | 3 +- python/ray/util/client_connect.py | 4 +- python/ray/util/collective/__init__.py | 30 +++++----- .../collective_group/base_collective_group.py | 9 ++- .../collective_group/cuda_stream.py | 1 + .../collective_group/gloo_collective_group.py | 2 +- .../collective_group/nccl_collective_group.py | 17 +++--- .../collective/collective_group/nccl_util.py | 18 +++--- .../collective_group/nixl_backend.py | 6 +- .../torch_gloo_collective_group.py | 13 +++-- .../examples/nccl_allreduce_example.py | 2 +- ...reduce_example_declare_collective_group.py | 2 +- .../nccl_allreduce_multigpu_example.py | 4 +- .../examples/nccl_p2p_example_multigpu.py | 4 +- python/ray/util/collective/tests/conftest.py | 1 + python/ray/util/collective/tests/cpu_util.py | 6 +- .../test_distributed_allgather.py | 10 ++-- .../test_distributed_allreduce.py | 11 ++-- .../test_distributed_basic_apis.py | 10 ++-- .../test_distributed_broadcast.py | 9 +-- .../test_distributed_reduce.py | 9 +-- .../test_distributed_reducescatter.py | 10 ++-- .../test_distributed_sendrecv.py | 7 ++- .../test_distributed_allgather.py | 8 +-- .../test_distributed_allreduce.py | 7 +-- .../test_distributed_basic_apis.py | 8 ++- .../test_distributed_broadcast.py | 4 +- .../test_distributed_reduce.py | 6 +- .../test_distributed_reducescatter.py | 8 +-- .../test_distributed_sendrecv.py | 2 +- .../test_distributed_multigpu_allgather.py | 8 +-- .../test_distributed_multigpu_allreduce.py | 4 +- .../test_distributed_multigpu_basic_apis.py | 8 ++- .../test_distributed_multigpu_broadcast.py | 4 +- .../test_distributed_multigpu_reduce.py | 6 +- ...test_distributed_multigpu_reducescatter.py | 8 +-- .../test_distributed_multigpu_sendrecv.py | 2 +- .../single_node_cpu_tests/test_allgather.py | 6 +- .../single_node_cpu_tests/test_allreduce.py | 6 +- .../single_node_cpu_tests/test_basic_apis.py | 7 ++- .../single_node_cpu_tests/test_broadcast.py | 7 ++- .../test_gloo_group_isolation.py | 9 ++- .../single_node_cpu_tests/test_reduce.py | 9 +-- .../test_reducescatter.py | 10 ++-- .../single_node_cpu_tests/test_sendrecv.py | 9 +-- .../single_node_gpu_tests/test_allgather.py | 8 +-- .../single_node_gpu_tests/test_allreduce.py | 6 +- .../single_node_gpu_tests/test_basic_apis.py | 5 +- .../single_node_gpu_tests/test_broadcast.py | 4 +- .../single_node_gpu_tests/test_reduce.py | 6 +- .../test_reducescatter.py | 8 +-- .../single_node_gpu_tests/test_sendrecv.py | 4 +- python/ray/util/collective/tests/util.py | 8 +-- python/ray/util/collective/types.py | 4 +- python/ray/util/collective/util.py | 3 +- python/ray/util/dask/__init__.py | 14 ++--- python/ray/util/dask/callbacks.py | 3 +- python/ray/util/dask/common.py | 11 ++-- python/ray/util/dask/optimizations.py | 3 +- python/ray/util/dask/scheduler.py | 14 ++--- python/ray/util/dask/scheduler_utils.py | 2 +- .../ray/util/dask/tests/test_dask_callback.py | 3 +- .../util/dask/tests/test_dask_multi_node.py | 2 +- .../util/dask/tests/test_dask_optimization.py | 5 +- python/ray/util/debug.py | 5 +- python/ray/util/debugpy.py | 2 +- python/ray/util/helpers.py | 1 + python/ray/util/metrics.py | 9 ++- python/ray/util/multiprocessing/__init__.py | 2 +- python/ray/util/placement_group.py | 4 +- python/ray/util/queue.py | 2 +- python/ray/util/rpdb.py | 2 +- python/ray/util/scheduling_strategies.py | 3 +- python/ray/util/spark/__init__.py | 4 +- python/ray/util/spark/cluster_init.py | 55 +++++++++--------- python/ray/util/spark/databricks_hook.py | 6 +- python/ray/util/spark/start_ray_node.py | 13 ++--- python/ray/util/spark/utils.py | 14 ++--- python/ray/util/state/__init__.py | 13 ++--- python/ray/util/state/common.py | 22 +++---- python/ray/util/state/state_cli.py | 4 +- python/ray/util/state/state_manager.py | 10 ++-- python/ray/util/state/util.py | 1 + .../util/tracing/setup_local_tmp_tracing.py | 1 + .../ray/util/tracing/setup_tempo_tracing.py | 2 +- 216 files changed, 721 insertions(+), 712 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 89ba195ebe62..d404d2c92c99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,6 @@ repos: args: [ --fix, --exit-non-zero-on-fix ] - id: ruff args: [ --select, "I", --fix, --exit-non-zero-on-fix ] - files: '^python/ray/serve/|^python/ray/train|^python/ray/data|^python/ray/_private/|^python/ray/llm/|^python/ray/tune/|^python/ray/includes/|^python/ray/internal/|^python/ray/ray_operator/|^python/ray/scripts/|^python/ray/streaming/|^python/ray/dag/|^python/ray/tests/|^python/ray/setup-dev.py|^python/ray/cloudpickle/|^python/ray/workers/|^python/ray/workflow/' - repo: https://github.com/jsh9/pydoclint rev: "0.6.6" diff --git a/bazel/gen_extract.py b/bazel/gen_extract.py index 80402bc9f39c..a635922011ee 100644 --- a/bazel/gen_extract.py +++ b/bazel/gen_extract.py @@ -1,7 +1,7 @@ -from typing import List, Optional import os import shutil import subprocess +from typing import List, Optional import runfiles diff --git a/ci/lint/git-clang-format b/ci/lint/git-clang-format index 46b466ee191b..6972b1bf7c6e 100755 --- a/ci/lint/git-clang-format +++ b/ci/lint/git-clang-format @@ -25,6 +25,7 @@ Requires Python 2.7 or Python 3 """ from __future__ import absolute_import, division, print_function + import argparse import collections import contextlib diff --git a/ci/ray_ci/anyscale_docker_container.py b/ci/ray_ci/anyscale_docker_container.py index ff8499233daa..112b520cd6eb 100644 --- a/ci/ray_ci/anyscale_docker_container.py +++ b/ci/ray_ci/anyscale_docker_container.py @@ -1,6 +1,7 @@ +from ray_release.configs.global_config import get_global_config + from ci.ray_ci.container import _DOCKER_ECR_REPO, _DOCKER_GCP_REGISTRY from ci.ray_ci.docker_container import DockerContainer -from ray_release.configs.global_config import get_global_config class AnyscaleDockerContainer(DockerContainer): diff --git a/ci/ray_ci/test_anyscale_docker_container.py b/ci/ray_ci/test_anyscale_docker_container.py index 5214b2c6a056..6d96cab52cb7 100644 --- a/ci/ray_ci/test_anyscale_docker_container.py +++ b/ci/ray_ci/test_anyscale_docker_container.py @@ -4,11 +4,11 @@ from unittest import mock import pytest +from ray_release.configs.global_config import get_global_config from ci.ray_ci.anyscale_docker_container import AnyscaleDockerContainer from ci.ray_ci.container import _DOCKER_ECR_REPO, _DOCKER_GCP_REGISTRY from ci.ray_ci.test_base import RayCITestBase -from ray_release.configs.global_config import get_global_config class TestAnyscaleDockerContainer(RayCITestBase): diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index aae4b6de5c83..e614d5a3a179 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -1,14 +1,15 @@ +import difflib import platform +import shutil import subprocess +import sys +import tempfile from pathlib import Path from typing import List, Optional -import shutil + import click import runfiles -import tempfile -import difflib -import sys -from networkx import DiGraph, topological_sort, ancestors as networkx_ancestors +from networkx import DiGraph, ancestors as networkx_ancestors, topological_sort from ci.raydepsets.workspace import Depset, Workspace diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 70ff7f2d89e6..a05932336f9c 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -2,9 +2,10 @@ import sys import tempfile import unittest +from pathlib import Path from typing import Optional + import pytest -from pathlib import Path import runfiles from click.testing import CliRunner from networkx import topological_sort diff --git a/ci/raydepsets/tests/test_workspace.py b/ci/raydepsets/tests/test_workspace.py index 318fc07c52c4..50ca28aedf65 100644 --- a/ci/raydepsets/tests/test_workspace.py +++ b/ci/raydepsets/tests/test_workspace.py @@ -5,7 +5,7 @@ import pytest from ci.raydepsets.tests.utils import copy_data_to_tmpdir -from ci.raydepsets.workspace import Workspace, _substitute_build_args, BuildArgSet +from ci.raydepsets.workspace import BuildArgSet, Workspace, _substitute_build_args def test_workspace_init(): diff --git a/pyproject.toml b/pyproject.toml index 5711be013c27..c8c2219e451f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ afterray = ["psutil", "setproctitle"] "doc/*" = ["I"] "python/ray/__init__.py" = ["I"] "python/ray/dag/__init__.py" = ["I"] -"python/ray/util/*" = ["I"] +"python/ray/air/__init__.py" = ["I"] "rllib/*" = ["I"] "release/*" = ["I"] diff --git a/python/ray/_common/network_utils.py b/python/ray/_common/network_utils.py index 9664c53a94a2..b97eb55042d8 100644 --- a/python/ray/_common/network_utils.py +++ b/python/ray/_common/network_utils.py @@ -1,7 +1,6 @@ from typing import Optional, Tuple, Union -from ray._raylet import build_address as _build_address -from ray._raylet import parse_address as _parse_address +from ray._raylet import build_address as _build_address, parse_address as _parse_address def parse_address(address: str) -> Optional[Tuple[str, str]]: diff --git a/python/ray/_common/test_utils.py b/python/ray/_common/test_utils.py index 957a73be0158..c5e6020b1c98 100644 --- a/python/ray/_common/test_utils.py +++ b/python/ray/_common/test_utils.py @@ -6,21 +6,20 @@ """ import asyncio -from collections.abc import Awaitable -from contextlib import contextmanager import inspect import os import time import traceback -from typing import Any, Callable, Dict, Iterator, List, Optional, Set import uuid +from collections.abc import Awaitable +from contextlib import contextmanager from enum import Enum - +from typing import Any, Callable, Dict, Iterator, List, Optional, Set import ray -from ray._common.network_utils import build_address -import ray._private.utils import ray._common.usage.usage_lib as ray_usage_lib +import ray._private.utils +from ray._common.network_utils import build_address @ray.remote(num_cpus=0) diff --git a/python/ray/_common/tests/test_network_utils.py b/python/ray/_common/tests/test_network_utils.py index 347eed26055a..8aac0e1be420 100644 --- a/python/ray/_common/tests/test_network_utils.py +++ b/python/ray/_common/tests/test_network_utils.py @@ -1,6 +1,7 @@ -import pytest import sys +import pytest + from ray._common.network_utils import is_localhost diff --git a/python/ray/_common/tests/test_ray_option_utils.py b/python/ray/_common/tests/test_ray_option_utils.py index 48d48f385927..5cf52057a1f7 100644 --- a/python/ray/_common/tests/test_ray_option_utils.py +++ b/python/ray/_common/tests/test_ray_option_utils.py @@ -1,20 +1,21 @@ -import pytest import re import sys from unittest.mock import patch -from ray.util.placement_group import PlacementGroup +import pytest + from ray._common.ray_option_utils import ( Option, + _check_deprecate_placement_group, _counting_option, - _validate_resource_quantity, _resource_option, + _validate_resource_quantity, _validate_resources, - validate_task_options, - validate_actor_options, update_options, - _check_deprecate_placement_group, + validate_actor_options, + validate_task_options, ) +from ray.util.placement_group import PlacementGroup class TestOptionValidation: diff --git a/python/ray/_common/tests/test_signal_semaphore_utils.py b/python/ray/_common/tests/test_signal_semaphore_utils.py index 3c798c783678..dec2a21800b5 100644 --- a/python/ray/_common/tests/test_signal_semaphore_utils.py +++ b/python/ray/_common/tests/test_signal_semaphore_utils.py @@ -5,13 +5,14 @@ and synchronization in Ray tests. """ -import pytest import sys -import ray -from ray._common.test_utils import SignalActor, Semaphore -from ray._common.test_utils import wait_for_condition import time +import pytest + +import ray +from ray._common.test_utils import Semaphore, SignalActor, wait_for_condition + @pytest.fixture(scope="module") def ray_init(): diff --git a/python/ray/_common/tests/test_signature.py b/python/ray/_common/tests/test_signature.py index 8e0173fc38e6..e4691eebeae7 100644 --- a/python/ray/_common/tests/test_signature.py +++ b/python/ray/_common/tests/test_signature.py @@ -6,18 +6,19 @@ """ import inspect -import pytest import sys from typing import Any, Optional from unittest.mock import Mock, patch +import pytest + from ray._common.signature import ( - get_signature, + DUMMY_TYPE, extract_signature, - validate_args, flatten_args, + get_signature, recover_args, - DUMMY_TYPE, + validate_args, ) diff --git a/python/ray/_common/tests/test_usage_stats.py b/python/ray/_common/tests/test_usage_stats.py index fc871a2e9638..99fa1b1d6cc7 100644 --- a/python/ray/_common/tests/test_usage_stats.py +++ b/python/ray/_common/tests/test_usage_stats.py @@ -2,34 +2,34 @@ import os import pathlib import sys -import time import threading +import time from dataclasses import asdict +from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path from unittest.mock import Mock, patch -from ray._common.test_utils import wait_for_condition -from ray._raylet import GcsClient -from ray.tests.conftest import * # noqa: F403 -import requests import pytest +import requests from jsonschema import validate -from http.server import BaseHTTPRequestHandler, HTTPServer import ray import ray._common.usage.usage_constants as usage_constants import ray._common.usage.usage_lib as ray_usage_lib +from ray._common.test_utils import wait_for_condition +from ray._common.usage.usage_lib import ClusterConfigToReport, UsageStatsEnabledness +from ray._private.accelerators import NvidiaGPUAcceleratorManager from ray._private.test_utils import ( format_web_url, run_string_as_driver, wait_until_server_available, ) -from ray._common.usage.usage_lib import ClusterConfigToReport, UsageStatsEnabledness +from ray._raylet import GcsClient from ray.autoscaler._private.cli_logger import cli_logger +from ray.tests.conftest import * # noqa: F403 from ray.util.placement_group import ( placement_group, ) -from ray._private.accelerators import NvidiaGPUAcceleratorManager schema = { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/python/ray/_common/tests/test_utils.py b/python/ray/_common/tests/test_utils.py index c3d437bb7586..491781924df9 100644 --- a/python/ray/_common/tests/test_utils.py +++ b/python/ray/_common/tests/test_utils.py @@ -6,20 +6,20 @@ """ import asyncio -import warnings -import sys import os +import sys import tempfile +import warnings import pytest from ray._common.utils import ( + _BACKGROUND_TASKS, get_or_create_event_loop, + get_system_memory, + load_class, run_background_task, - _BACKGROUND_TASKS, try_to_create_directory, - load_class, - get_system_memory, ) # Optional imports for testing diff --git a/python/ray/_common/tests/test_wait_for_condition.py b/python/ray/_common/tests/test_wait_for_condition.py index 52cb8c9cd2b0..045817ca1aa2 100644 --- a/python/ray/_common/tests/test_wait_for_condition.py +++ b/python/ray/_common/tests/test_wait_for_condition.py @@ -1,9 +1,10 @@ import asyncio -import time import sys +import time + import pytest -from ray._common.test_utils import wait_for_condition, async_wait_for_condition +from ray._common.test_utils import async_wait_for_condition, wait_for_condition class TestWaitForCondition: diff --git a/python/ray/_common/usage/usage_lib.py b/python/ray/_common/usage/usage_lib.py index c2e7f2345f33..cb1536721186 100644 --- a/python/ray/_common/usage/usage_lib.py +++ b/python/ray/_common/usage/usage_lib.py @@ -57,8 +57,8 @@ import yaml import ray -import ray._private.ray_constants as ray_constants import ray._common.usage.usage_constants as usage_constant +import ray._private.ray_constants as ray_constants from ray._raylet import GcsClient from ray.core.generated import gcs_pb2, usage_pb2 from ray.experimental.internal_kv import ( diff --git a/python/ray/_common/utils.py b/python/ray/_common/utils.py index 103c40397801..28a05a356549 100644 --- a/python/ray/_common/utils.py +++ b/python/ray/_common/utils.py @@ -3,15 +3,16 @@ import errno import importlib import inspect -from inspect import signature import os -import psutil import random import string import sys import tempfile +from inspect import signature from typing import Any, Coroutine, Dict, Optional +import psutil + def import_attr(full_path: str, *, reload_module: bool = False): """Given a full import path to a module attr, return the imported attr. diff --git a/python/ray/_raylet.pyi b/python/ray/_raylet.pyi index 96c79d343b19..fff69c451b67 100644 --- a/python/ray/_raylet.pyi +++ b/python/ray/_raylet.pyi @@ -1,25 +1,20 @@ -from ray.includes.object_ref import ( - _set_future_helper, - ObjectRef -) - +from ray.includes.object_ref import ObjectRef, _set_future_helper from ray.includes.unique_ids import ( - check_id, - BaseID, - UniqueID, - TaskID, - NodeID, - JobID, - WorkerID, - ActorID, - FunctionID, ActorClassID, + ActorID, + BaseID, ClusterID, + FunctionID, + JobID, + NodeID, ObjectID, PlacementGroupID, + TaskID, + UniqueID, + WorkerID, + check_id, ) - __all__ = [ # ray.includes.unique_ids "ActorClassID", diff --git a/python/ray/air/_internal/device_manager/npu.py b/python/ray/air/_internal/device_manager/npu.py index 3a3c554da44f..0a40594e14f1 100644 --- a/python/ray/air/_internal/device_manager/npu.py +++ b/python/ray/air/_internal/device_manager/npu.py @@ -6,8 +6,8 @@ import ray import ray._private.ray_constants as ray_constants -from ray.air._internal.device_manager.torch_device_manager import TorchDeviceManager from ray._private.accelerators.npu import ASCEND_RT_VISIBLE_DEVICES_ENV_VAR +from ray.air._internal.device_manager.torch_device_manager import TorchDeviceManager def is_package_present(package_name: str) -> bool: diff --git a/python/ray/air/_internal/torch_utils.py b/python/ray/air/_internal/torch_utils.py index 2264e8c17e96..96fe7bd84c74 100644 --- a/python/ray/air/_internal/torch_utils.py +++ b/python/ray/air/_internal/torch_utils.py @@ -1,24 +1,23 @@ import warnings -from typing import Any, Dict, List, Optional, Union, Sequence +from typing import Any, Dict, List, Optional, Sequence, Union import numpy as np import pandas as pd -import torch import pyarrow +import torch +from ray._private.ray_constants import env_bool from ray.air._internal.device_manager import get_torch_device_manager_by_context from ray.air.util.data_batch_conversion import _unwrap_ndarray_object_type_if_needed from ray.data.collate_fn import ( - TensorBatchType, TensorBatchReturnType, - _is_tensor, - _is_tensor_sequence, + TensorBatchType, _is_nested_tensor_sequence, + _is_tensor, _is_tensor_mapping, + _is_tensor_sequence, _is_tensor_sequence_mapping, ) -from ray._private.ray_constants import env_bool - # Default non-blocking transfer for tensors. DEFAULT_TENSOR_NON_BLOCKING_TRANSFER = env_bool( @@ -385,8 +384,8 @@ def arrow_batch_to_tensors( A dictionary of column name to list of tensors. For non-chunked columns, the list will contain a single tensor. """ - from ray.data._internal.arrow_ops import transform_pyarrow from ray.data._internal.arrow_block import ArrowBlockAccessor + from ray.data._internal.arrow_ops import transform_pyarrow if combine_chunks: numpy_batch = ArrowBlockAccessor(batch).to_batch_format("numpy") diff --git a/python/ray/air/config.py b/python/ray/air/config.py index 727ceb5c8cfb..01c93e3c354c 100644 --- a/python/ray/air/config.py +++ b/python/ray/air/config.py @@ -1,7 +1,8 @@ import logging +import os +import warnings from collections import Counter, defaultdict from dataclasses import _MISSING_TYPE, dataclass, fields -import os from pathlib import Path from typing import ( TYPE_CHECKING, @@ -14,7 +15,6 @@ Tuple, Union, ) -import warnings import pyarrow.fs diff --git a/python/ray/air/tests/test_air_usage.py b/python/ray/air/tests/test_air_usage.py index 6a1d65b96ac3..bc6bbc194bd4 100644 --- a/python/ray/air/tests/test_air_usage.py +++ b/python/ray/air/tests/test_air_usage.py @@ -210,10 +210,10 @@ def test_tag_air_entrypoint(ray_start_4_cpus, mock_record, entrypoint, tuner, tr ) def test_tag_train_entrypoint(mock_record): """Test that Train v2 entrypoints are recorded correctly.""" - from ray.train.v2.torch.torch_trainer import TorchTrainer + from ray.train.v2.lightgbm.lightgbm_trainer import LightGBMTrainer from ray.train.v2.tensorflow.tensorflow_trainer import TensorflowTrainer + from ray.train.v2.torch.torch_trainer import TorchTrainer from ray.train.v2.xgboost.xgboost_trainer import XGBoostTrainer - from ray.train.v2.lightgbm.lightgbm_trainer import LightGBMTrainer trainer_classes = [ TorchTrainer, diff --git a/python/ray/air/tests/test_arrow.py b/python/ray/air/tests/test_arrow.py index 31d533155c3a..4c2ebc3099e9 100644 --- a/python/ray/air/tests/test_arrow.py +++ b/python/ray/air/tests/test_arrow.py @@ -9,10 +9,10 @@ from ray._private.arrow_utils import get_pyarrow_version from ray.air.util.tensor_extensions.arrow import ( ArrowConversionError, + ArrowTensorArray, _convert_to_pyarrow_native_array, _infer_pyarrow_type, convert_to_pyarrow_array, - ArrowTensorArray, ) from ray.air.util.tensor_extensions.utils import create_ragged_ndarray from ray.data import DataContext diff --git a/python/ray/air/tests/test_integration_wandb.py b/python/ray/air/tests/test_integration_wandb.py index 04228162d2fd..05a64ee82d34 100644 --- a/python/ray/air/tests/test_integration_wandb.py +++ b/python/ray/air/tests/test_integration_wandb.py @@ -50,10 +50,10 @@ WANDB_POPULATE_RUN_LOCATION_HOOK, WANDB_PROJECT_ENV_VAR, WANDB_SETUP_API_KEY_HOOK, + RunDisabled, WandbLoggerCallback, _QueueItem, _WandbLoggingActor, - RunDisabled, setup_wandb, ) from ray.air.tests.mocked_wandb_integration import ( diff --git a/python/ray/air/util/object_extensions/arrow.py b/python/ray/air/util/object_extensions/arrow.py index b7e2e569c61b..47867e54f5a3 100644 --- a/python/ray/air/util/object_extensions/arrow.py +++ b/python/ray/air/util/object_extensions/arrow.py @@ -6,8 +6,8 @@ from packaging.version import parse as parse_version import ray.air.util.object_extensions.pandas -from ray._private.serialization import pickle_dumps from ray._private.arrow_utils import get_pyarrow_version +from ray._private.serialization import pickle_dumps from ray.util.annotations import PublicAPI MIN_PYARROW_VERSION_SCALAR_SUBCLASS = parse_version("9.0.0") diff --git a/python/ray/air/util/tensor_extensions/arrow.py b/python/ray/air/util/tensor_extensions/arrow.py index 0c1772b46d78..712b2af080c0 100644 --- a/python/ray/air/util/tensor_extensions/arrow.py +++ b/python/ray/air/util/tensor_extensions/arrow.py @@ -1,34 +1,32 @@ import abc -from datetime import datetime - import itertools import json import logging import sys +from datetime import datetime +from enum import Enum from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union import numpy as np import pyarrow as pa from packaging.version import parse as parse_version -import ray.cloudpickle as cloudpickle -from enum import Enum +import ray.cloudpickle as cloudpickle from ray._private.arrow_utils import get_pyarrow_version +from ray._private.ray_constants import env_integer from ray.air.util.tensor_extensions.utils import ( + ArrayLike, _is_ndarray_variable_shaped_tensor, - create_ragged_ndarray, _should_convert_to_tensor, - ArrayLike, + create_ragged_ndarray, ) from ray.data._internal.numpy_support import ( - convert_to_numpy, _convert_datetime_to_np_datetime, + convert_to_numpy, ) from ray.util import log_once from ray.util.annotations import DeveloperAPI, PublicAPI from ray.util.common import INT32_MAX -from ray._private.ray_constants import env_integer - PYARROW_VERSION = get_pyarrow_version() # Minimum version of Arrow that supports subclassable ExtensionScalars. diff --git a/python/ray/air/util/tensor_extensions/utils.py b/python/ray/air/util/tensor_extensions/utils.py index 8468f721751e..142814285ffd 100644 --- a/python/ray/air/util/tensor_extensions/utils.py +++ b/python/ray/air/util/tensor_extensions/utils.py @@ -1,5 +1,5 @@ import warnings -from typing import TYPE_CHECKING, Any, Sequence, Union, List, Protocol +from typing import TYPE_CHECKING, Any, List, Protocol, Sequence, Union import numpy as np diff --git a/python/ray/air/util/torch_dist.py b/python/ray/air/util/torch_dist.py index acc7d78a47f8..c133d8133425 100644 --- a/python/ray/air/util/torch_dist.py +++ b/python/ray/air/util/torch_dist.py @@ -15,10 +15,10 @@ import torch.distributed as dist import ray +from ray._common.network_utils import build_address from ray.actor import ActorHandle from ray.air._internal.torch_utils import get_devices from ray.train._internal.utils import get_address_and_port -from ray._common.network_utils import build_address class TorchDistributedWorker(ABC): diff --git a/python/ray/autoscaler/_private/gcp/config.py b/python/ray/autoscaler/_private/gcp/config.py index e527e21ab556..2e646526cb34 100644 --- a/python/ray/autoscaler/_private/gcp/config.py +++ b/python/ray/autoscaler/_private/gcp/config.py @@ -13,8 +13,7 @@ from google.oauth2.credentials import Credentials as OAuthCredentials from googleapiclient import discovery, errors -from ray._private.accelerators import TPUAcceleratorManager -from ray._private.accelerators import tpu +from ray._private.accelerators import TPUAcceleratorManager, tpu from ray.autoscaler._private.gcp.node import MAX_POLLS, POLL_INTERVAL, GCPNodeType from ray.autoscaler._private.util import ( check_legacy_fields, diff --git a/python/ray/autoscaler/_private/gcp/node_provider.py b/python/ray/autoscaler/_private/gcp/node_provider.py index 2d7147a60619..56398433f624 100644 --- a/python/ray/autoscaler/_private/gcp/node_provider.py +++ b/python/ray/autoscaler/_private/gcp/node_provider.py @@ -18,8 +18,8 @@ # The logic has been abstracted away here to allow for different GCP resources # (API endpoints), which can differ widely, making it impossible to use # the same logic for everything. -from ray.autoscaler._private.gcp.node import GCPTPU # noqa from ray.autoscaler._private.gcp.node import ( + GCPTPU, # noqa GCPCompute, GCPNode, GCPNodeType, diff --git a/python/ray/autoscaler/_private/kuberay/node_provider.py b/python/ray/autoscaler/_private/kuberay/node_provider.py index b62b5ca78fa2..b3715f0dd9fa 100644 --- a/python/ray/autoscaler/_private/kuberay/node_provider.py +++ b/python/ray/autoscaler/_private/kuberay/node_provider.py @@ -8,6 +8,7 @@ import requests +from ray._common.network_utils import build_address from ray.autoscaler._private.constants import WORKER_LIVENESS_CHECK_KEY from ray.autoscaler._private.util import NodeID, NodeIP, NodeKind, NodeStatus, NodeType from ray.autoscaler.batching_node_provider import ( @@ -22,7 +23,6 @@ STATUS_UPDATE_FAILED, TAG_RAY_USER_NODE_TYPE, ) -from ray._common.network_utils import build_address # Key for KubeRay label that identifies a Ray pod as head or worker. KUBERAY_LABEL_KEY_KIND = "ray.io/node-type" diff --git a/python/ray/autoscaler/_private/kuberay/run_autoscaler.py b/python/ray/autoscaler/_private/kuberay/run_autoscaler.py index dcc810073797..37b09db1f46a 100644 --- a/python/ray/autoscaler/_private/kuberay/run_autoscaler.py +++ b/python/ray/autoscaler/_private/kuberay/run_autoscaler.py @@ -4,15 +4,15 @@ import time import ray -from ray._private import ray_constants +from ray._common.network_utils import build_address from ray._common.ray_constants import ( - LOGGING_ROTATE_BYTES, LOGGING_ROTATE_BACKUP_COUNT, + LOGGING_ROTATE_BYTES, ) +from ray._common.utils import try_to_create_directory +from ray._private import ray_constants from ray._private.ray_logging import setup_component_logger from ray._private.services import get_node_ip_address -from ray._common.network_utils import build_address -from ray._common.utils import try_to_create_directory from ray._raylet import GcsClient from ray.autoscaler._private.kuberay.autoscaling_config import AutoscalingConfigProducer from ray.autoscaler._private.monitor import Monitor diff --git a/python/ray/autoscaler/_private/monitor.py b/python/ray/autoscaler/_private/monitor.py index d62886e3a669..55a922293fca 100644 --- a/python/ray/autoscaler/_private/monitor.py +++ b/python/ray/autoscaler/_private/monitor.py @@ -14,14 +14,15 @@ import ray import ray._private.ray_constants as ray_constants +from ray._common.network_utils import build_address, parse_address from ray._common.ray_constants import ( - LOGGING_ROTATE_BYTES, LOGGING_ROTATE_BACKUP_COUNT, + LOGGING_ROTATE_BYTES, ) +from ray._private import logging_utils from ray._private.event.event_logger import get_event_logger from ray._private.ray_logging import setup_component_logger from ray._raylet import GcsClient -from ray._common.network_utils import parse_address, build_address from ray.autoscaler._private.autoscaler import StandardAutoscaler from ray.autoscaler._private.commands import teardown_cluster from ray.autoscaler._private.constants import ( @@ -44,7 +45,6 @@ _internal_kv_initialized, _internal_kv_put, ) -from ray._private import logging_utils try: import prometheus_client diff --git a/python/ray/autoscaler/_private/spark/node_provider.py b/python/ray/autoscaler/_private/spark/node_provider.py index 9fbea1f525ab..9c59ba4ed9fd 100644 --- a/python/ray/autoscaler/_private/spark/node_provider.py +++ b/python/ray/autoscaler/_private/spark/node_provider.py @@ -6,6 +6,7 @@ import requests +from ray._common.network_utils import build_address from ray.autoscaler.node_launch_exception import NodeLaunchException from ray.autoscaler.node_provider import NodeProvider from ray.autoscaler.tags import ( @@ -18,7 +19,6 @@ TAG_RAY_NODE_STATUS, TAG_RAY_USER_NODE_TYPE, ) -from ray._common.network_utils import build_address logger = logging.getLogger(__name__) diff --git a/python/ray/autoscaler/_private/util.py b/python/ray/autoscaler/_private/util.py index de5b9f506c3e..396dd409cdf4 100644 --- a/python/ray/autoscaler/_private/util.py +++ b/python/ray/autoscaler/_private/util.py @@ -13,8 +13,8 @@ from typing import Any, Dict, List, Optional, Tuple, Union import ray -from ray._common.utils import PLACEMENT_GROUP_BUNDLE_RESOURCE_NAME import ray._private.services as services +from ray._common.utils import PLACEMENT_GROUP_BUNDLE_RESOURCE_NAME from ray._private.utils import ( PLACEMENT_GROUP_INDEXED_BUNDLED_RESOURCE_PATTERN, PLACEMENT_GROUP_WILDCARD_RESOURCE_PATTERN, diff --git a/python/ray/autoscaler/local/coordinator_server.py b/python/ray/autoscaler/local/coordinator_server.py index 6ea69d71857b..7cca12645631 100644 --- a/python/ray/autoscaler/local/coordinator_server.py +++ b/python/ray/autoscaler/local/coordinator_server.py @@ -6,12 +6,12 @@ import argparse import json import logging -import threading import socket +import threading from http.server import HTTPServer, SimpleHTTPRequestHandler -from ray.autoscaler._private.local.node_provider import LocalNodeProvider from ray._common.network_utils import build_address +from ray.autoscaler._private.local.node_provider import LocalNodeProvider logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) diff --git a/python/ray/autoscaler/sdk/sdk.py b/python/ray/autoscaler/sdk/sdk.py index 276e85892c0e..437538bbeb3b 100644 --- a/python/ray/autoscaler/sdk/sdk.py +++ b/python/ray/autoscaler/sdk/sdk.py @@ -8,8 +8,10 @@ from ray.autoscaler._private import commands from ray.autoscaler._private.cli_logger import cli_logger -from ray.autoscaler._private.event_system import CreateClusterEvent # noqa: F401 -from ray.autoscaler._private.event_system import global_event_system # noqa: F401 +from ray.autoscaler._private.event_system import ( + CreateClusterEvent, # noqa: F401 + global_event_system, # noqa: F401 +) from ray.util.annotations import DeveloperAPI diff --git a/python/ray/autoscaler/v2/autoscaler.py b/python/ray/autoscaler/v2/autoscaler.py index 82de748a1f5f..cdc8620bc2b7 100644 --- a/python/ray/autoscaler/v2/autoscaler.py +++ b/python/ray/autoscaler/v2/autoscaler.py @@ -1,6 +1,7 @@ import logging from queue import Queue from typing import List, Optional +from urllib.parse import urlsplit from ray._raylet import GcsClient from ray.autoscaler._private.providers import _get_node_provider @@ -39,7 +40,6 @@ from ray.autoscaler.v2.scheduler import ResourceDemandScheduler from ray.autoscaler.v2.sdk import get_cluster_resource_state from ray.core.generated.autoscaler_pb2 import AutoscalingState -from urllib.parse import urlsplit logger = logging.getLogger(__name__) diff --git a/python/ray/autoscaler/v2/instance_manager/config.py b/python/ray/autoscaler/v2/instance_manager/config.py index ef329f804e71..d94d157c86fa 100644 --- a/python/ray/autoscaler/v2/instance_manager/config.py +++ b/python/ray/autoscaler/v2/instance_manager/config.py @@ -8,8 +8,8 @@ import yaml -from ray._private.ray_constants import env_integer from ray._common.utils import binary_to_hex +from ray._private.ray_constants import env_integer from ray._raylet import GcsClient from ray.autoscaler._private.constants import ( AUTOSCALER_MAX_CONCURRENT_LAUNCHES, diff --git a/python/ray/autoscaler/v2/instance_manager/ray_installer.py b/python/ray/autoscaler/v2/instance_manager/ray_installer.py index 3daf5d3e0b71..e99b2b1492ca 100644 --- a/python/ray/autoscaler/v2/instance_manager/ray_installer.py +++ b/python/ray/autoscaler/v2/instance_manager/ray_installer.py @@ -2,9 +2,9 @@ import subprocess from ray.autoscaler._private.updater import ( - NodeUpdater, - TAG_RAY_NODE_STATUS, STATUS_UP_TO_DATE, + TAG_RAY_NODE_STATUS, + NodeUpdater, ) from ray.autoscaler._private.util import with_envs, with_head_node_ip from ray.autoscaler.node_provider import NodeProvider as NodeProviderV1 diff --git a/python/ray/autoscaler/v2/instance_manager/reconciler.py b/python/ray/autoscaler/v2/instance_manager/reconciler.py index f274854eca6a..b403803e577b 100644 --- a/python/ray/autoscaler/v2/instance_manager/reconciler.py +++ b/python/ray/autoscaler/v2/instance_manager/reconciler.py @@ -21,10 +21,10 @@ LaunchNodeError, TerminateNodeError, ) +from ray.autoscaler.v2.instance_manager.subscribers.ray_stopper import RayStopError from ray.autoscaler.v2.instance_manager.subscribers.threaded_ray_installer import ( RayInstallError, ) -from ray.autoscaler.v2.instance_manager.subscribers.ray_stopper import RayStopError from ray.autoscaler.v2.metrics_reporter import AutoscalerMetricsReporter from ray.autoscaler.v2.scheduler import IResourceScheduler, SchedulingRequest from ray.autoscaler.v2.schema import AutoscalerInstance, NodeType @@ -38,12 +38,10 @@ PendingInstance, PendingInstanceRequest, ) -from ray.core.generated.instance_manager_pb2 import GetInstanceManagerStateRequest -from ray.core.generated.instance_manager_pb2 import Instance as IMInstance from ray.core.generated.instance_manager_pb2 import ( + GetInstanceManagerStateRequest, + Instance as IMInstance, InstanceUpdateEvent as IMInstanceUpdateEvent, -) -from ray.core.generated.instance_manager_pb2 import ( NodeKind, StatusCode, UpdateInstanceManagerStateRequest, diff --git a/python/ray/autoscaler/v2/instance_manager/subscribers/threaded_ray_installer.py b/python/ray/autoscaler/v2/instance_manager/subscribers/threaded_ray_installer.py index d364ccea07e0..d525b1aeccaa 100644 --- a/python/ray/autoscaler/v2/instance_manager/subscribers/threaded_ray_installer.py +++ b/python/ray/autoscaler/v2/instance_manager/subscribers/threaded_ray_installer.py @@ -2,8 +2,8 @@ import logging import time from concurrent.futures import ThreadPoolExecutor -from typing import List from queue import Queue +from typing import List from ray.autoscaler.v2.instance_manager.instance_manager import ( InstanceUpdatedSubscriber, @@ -11,9 +11,9 @@ from ray.autoscaler.v2.instance_manager.instance_storage import InstanceStorage from ray.autoscaler.v2.instance_manager.ray_installer import RayInstaller from ray.core.generated.instance_manager_pb2 import ( - NodeKind, Instance, InstanceUpdateEvent, + NodeKind, ) logger = logging.getLogger(__name__) diff --git a/python/ray/autoscaler/v2/monitor.py b/python/ray/autoscaler/v2/monitor.py index e771d6e7e404..34e31e7ac649 100644 --- a/python/ray/autoscaler/v2/monitor.py +++ b/python/ray/autoscaler/v2/monitor.py @@ -13,16 +13,17 @@ import ray import ray._private.ray_constants as ray_constants +from ray._common.network_utils import build_address, parse_address from ray._common.ray_constants import ( - LOGGING_ROTATE_BYTES, LOGGING_ROTATE_BACKUP_COUNT, + LOGGING_ROTATE_BYTES, ) +from ray._common.usage.usage_lib import record_extra_usage_tag +from ray._private import logging_utils from ray._private.event.event_logger import get_event_logger from ray._private.ray_logging import setup_component_logger -from ray._common.usage.usage_lib import record_extra_usage_tag from ray._private.worker import SCRIPT_MODE from ray._raylet import GcsClient -from ray._common.network_utils import parse_address, build_address from ray.autoscaler._private.constants import ( AUTOSCALER_METRIC_PORT, AUTOSCALER_UPDATE_INTERVAL_S, @@ -39,7 +40,6 @@ from ray.core.generated.autoscaler_pb2 import AutoscalingState from ray.core.generated.event_pb2 import Event as RayEvent from ray.core.generated.usage_pb2 import TagKey -from ray._private import logging_utils try: import prometheus_client diff --git a/python/ray/autoscaler/v2/scheduler.py b/python/ray/autoscaler/v2/scheduler.py index 6c87dc4e85f5..6dbacd893619 100644 --- a/python/ray/autoscaler/v2/scheduler.py +++ b/python/ray/autoscaler/v2/scheduler.py @@ -20,13 +20,13 @@ from ray.autoscaler.v2.instance_manager.config import NodeTypeConfig from ray.autoscaler.v2.schema import AutoscalerInstance, NodeType from ray.autoscaler.v2.utils import ProtobufUtil, ResourceRequestUtil -from ray.core.generated.common_pb2 import LabelSelectorOperator from ray.core.generated.autoscaler_pb2 import ( ClusterResourceConstraint, GangResourceRequest, ResourceRequest, ResourceRequestByCount, ) +from ray.core.generated.common_pb2 import LabelSelectorOperator from ray.core.generated.instance_manager_pb2 import ( Instance, LaunchRequest, diff --git a/python/ray/autoscaler/v2/tests/test_e2e.py b/python/ray/autoscaler/v2/tests/test_e2e.py index 4283af09a091..afa5b67baa1c 100644 --- a/python/ray/autoscaler/v2/tests/test_e2e.py +++ b/python/ray/autoscaler/v2/tests/test_e2e.py @@ -7,10 +7,10 @@ import pytest import ray -from ray._common.test_utils import wait_for_condition from ray._common.constants import HEAD_NODE_RESOURCE_NAME -from ray._private.test_utils import run_string_as_driver_nonblocking +from ray._common.test_utils import wait_for_condition from ray._common.usage.usage_lib import get_extra_usage_tags_to_report +from ray._private.test_utils import run_string_as_driver_nonblocking from ray._raylet import GcsClient from ray.autoscaler.v2.sdk import get_cluster_status from ray.cluster_utils import AutoscalingCluster diff --git a/python/ray/autoscaler/v2/tests/test_scheduler.py b/python/ray/autoscaler/v2/tests/test_scheduler.py index 1f95f83df3b2..a1ca0a9f0944 100644 --- a/python/ray/autoscaler/v2/tests/test_scheduler.py +++ b/python/ray/autoscaler/v2/tests/test_scheduler.py @@ -22,7 +22,6 @@ from ray.autoscaler.v2.schema import AutoscalerInstance, NodeType from ray.autoscaler.v2.tests.util import MockEventLogger, make_autoscaler_instance from ray.autoscaler.v2.utils import ResourceRequestUtil -from ray.core.generated.common_pb2 import LabelSelectorOperator from ray.core.generated.autoscaler_pb2 import ( ClusterResourceConstraint, GangResourceRequest, @@ -30,6 +29,7 @@ NodeStatus, ResourceRequest, ) +from ray.core.generated.common_pb2 import LabelSelectorOperator from ray.core.generated.instance_manager_pb2 import ( Instance, NodeKind, diff --git a/python/ray/autoscaler/v2/tests/test_threaded_ray_installer.py b/python/ray/autoscaler/v2/tests/test_threaded_ray_installer.py index 79cd43092d36..12594562d678 100644 --- a/python/ray/autoscaler/v2/tests/test_threaded_ray_installer.py +++ b/python/ray/autoscaler/v2/tests/test_threaded_ray_installer.py @@ -2,8 +2,8 @@ import os import sys import unittest -from unittest.mock import patch from queue import Queue +from unittest.mock import patch import pytest # noqa diff --git a/python/ray/autoscaler/v2/utils.py b/python/ray/autoscaler/v2/utils.py index 8cf3dd13fc94..bca10ddd1786 100644 --- a/python/ray/autoscaler/v2/utils.py +++ b/python/ray/autoscaler/v2/utils.py @@ -39,13 +39,11 @@ NodeStatus, PlacementConstraint, ResourceRequest, -) -from ray.core.generated.autoscaler_pb2 import ( ResourceRequestByCount as ResourceRequestByCountProto, ) from ray.core.generated.common_pb2 import ( - LabelSelectorConstraint, LabelSelector, + LabelSelectorConstraint, ) from ray.experimental.internal_kv import internal_kv_get_gcs_client diff --git a/python/ray/client_builder.py b/python/ray/client_builder.py index 714a941699c3..48d7f4cd0718 100644 --- a/python/ray/client_builder.py +++ b/python/ray/client_builder.py @@ -15,8 +15,7 @@ RAY_RUNTIME_ENV_ENVIRONMENT_VARIABLE, ) from ray._private.utils import get_ray_client_dependency_error, split_address -from ray._private.worker import BaseContext -from ray._private.worker import init as ray_driver_init +from ray._private.worker import BaseContext, init as ray_driver_init from ray.job_config import JobConfig from ray.util.annotations import Deprecated, PublicAPI diff --git a/python/ray/dashboard/agent.py b/python/ray/dashboard/agent.py index 624efc5742bb..6b95ad4d1444 100644 --- a/python/ray/dashboard/agent.py +++ b/python/ray/dashboard/agent.py @@ -9,9 +9,9 @@ import ray._private.ray_constants as ray_constants import ray.dashboard.consts as dashboard_consts import ray.dashboard.utils as dashboard_utils +from ray._common.network_utils import build_address, is_localhost from ray._common.utils import get_or_create_event_loop from ray._private import logging_utils -from ray._common.network_utils import build_address, is_localhost from ray._private.process_watcher import create_check_raylet_task from ray._private.ray_constants import AGENT_GRPC_MAX_MESSAGE_LENGTH from ray._private.ray_logging import setup_component_logger diff --git a/python/ray/dashboard/dashboard.py b/python/ray/dashboard/dashboard.py index 921a6069c88e..f774e22300d5 100644 --- a/python/ray/dashboard/dashboard.py +++ b/python/ray/dashboard/dashboard.py @@ -9,13 +9,13 @@ import ray import ray._private.ray_constants as ray_constants -from ray._common.ray_constants import ( - LOGGING_ROTATE_BYTES, - LOGGING_ROTATE_BACKUP_COUNT, -) import ray.dashboard.consts as dashboard_consts import ray.dashboard.head as dashboard_head import ray.dashboard.utils as dashboard_utils +from ray._common.ray_constants import ( + LOGGING_ROTATE_BACKUP_COUNT, + LOGGING_ROTATE_BYTES, +) from ray._common.utils import get_or_create_event_loop from ray._private import logging_utils from ray._private.ray_logging import setup_component_logger diff --git a/python/ray/dashboard/head.py b/python/ray/dashboard/head.py index 90469fd5c94d..94a8bb3cf380 100644 --- a/python/ray/dashboard/head.py +++ b/python/ray/dashboard/head.py @@ -9,10 +9,11 @@ import ray.dashboard.consts as dashboard_consts import ray.dashboard.utils as dashboard_utils import ray.experimental.internal_kv as internal_kv +from ray._common.network_utils import build_address +from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag from ray._private import ray_constants from ray._private.async_utils import enable_monitor_loop_lag from ray._private.ray_constants import env_integer -from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag from ray._raylet import GcsClient from ray.dashboard.consts import ( AVAILABLE_COMPONENT_NAMES_FOR_METRICS, @@ -24,7 +25,6 @@ DashboardHeadModuleConfig, async_loop_forever, ) -from ray._common.network_utils import build_address import psutil diff --git a/python/ray/dashboard/http_server_agent.py b/python/ray/dashboard/http_server_agent.py index cc3e20d85b87..b9146066933f 100644 --- a/python/ray/dashboard/http_server_agent.py +++ b/python/ray/dashboard/http_server_agent.py @@ -6,8 +6,8 @@ from packaging.version import Version import ray.dashboard.optional_utils as dashboard_optional_utils -from ray._common.utils import get_or_create_event_loop from ray._common.network_utils import build_address, is_localhost +from ray._common.utils import get_or_create_event_loop from ray.dashboard.optional_deps import aiohttp, aiohttp_cors, hdrs logger = logging.getLogger(__name__) diff --git a/python/ray/dashboard/http_server_head.py b/python/ray/dashboard/http_server_head.py index ffac41f4d7d1..49f748309271 100644 --- a/python/ray/dashboard/http_server_head.py +++ b/python/ray/dashboard/http_server_head.py @@ -17,10 +17,9 @@ import ray.dashboard.timezone_utils as timezone_utils import ray.dashboard.utils as dashboard_utils from ray import ray_constants -from ray._common.utils import get_or_create_event_loop -from ray._common.network_utils import build_address +from ray._common.network_utils import build_address, parse_address from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag -from ray._common.network_utils import parse_address +from ray._common.utils import get_or_create_event_loop from ray.dashboard.dashboard_metrics import DashboardPrometheusMetrics from ray.dashboard.head import DashboardHeadModule diff --git a/python/ray/dashboard/modules/aggregator/aggregator_agent.py b/python/ray/dashboard/modules/aggregator/aggregator_agent.py index d7782436ac22..584a6f6375ec 100644 --- a/python/ray/dashboard/modules/aggregator/aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/aggregator_agent.py @@ -1,15 +1,17 @@ import asyncio -import signal -import time -import os import json +import logging +import os import queue -from concurrent.futures import ThreadPoolExecutor +import signal import threading -import logging -from urllib3.util import Retry +import time +from concurrent.futures import ThreadPoolExecutor + from requests import Session from requests.adapters import HTTPAdapter +from urllib3.util import Retry + from ray._private.protobuf_compat import message_to_json try: @@ -19,14 +21,14 @@ prometheus_client = None import ray +import ray.dashboard.consts as dashboard_consts +import ray.dashboard.utils as dashboard_utils from ray._common.utils import get_or_create_event_loop from ray._private import ray_constants -import ray.dashboard.utils as dashboard_utils -import ray.dashboard.consts as dashboard_consts from ray.core.generated import ( + events_base_event_pb2, events_event_aggregator_service_pb2, events_event_aggregator_service_pb2_grpc, - events_base_event_pb2, ) logger = logging.getLogger(__name__) diff --git a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py index 8ab41de85aeb..31b3ed26b6cf 100644 --- a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py @@ -1,62 +1,58 @@ -import sys -import json import base64 +import json +import sys from unittest.mock import MagicMock import pytest from google.protobuf.timestamp_pb2 import Timestamp -from ray.dashboard.tests.conftest import * # noqa - -from ray._private import ray_constants -from ray._private.utils import init_grpc_channel -from ray._private.test_utils import wait_for_condition -from ray._raylet import GcsClient import ray.dashboard.consts as dashboard_consts +from ray._private import ray_constants from ray._private.test_utils import ( find_free_port, + wait_for_condition, ) - -from ray.core.generated.events_event_aggregator_service_pb2_grpc import ( - EventAggregatorServiceStub, +from ray._private.utils import init_grpc_channel +from ray._raylet import GcsClient +from ray.core.generated.common_pb2 import ( + ErrorType, + FunctionDescriptor, + Language, + PythonFunctionDescriptor, + RayErrorInfo, + TaskStatus, + TaskType, +) +from ray.core.generated.events_base_event_pb2 import RayEvent +from ray.core.generated.events_driver_job_definition_event_pb2 import ( + DriverJobDefinitionEvent, +) +from ray.core.generated.events_driver_job_execution_event_pb2 import ( + DriverJobExecutionEvent, ) from ray.core.generated.events_event_aggregator_service_pb2 import ( AddEventsRequest, RayEventsData, TaskEventsMetadata, ) -from ray.core.generated.events_base_event_pb2 import RayEvent +from ray.core.generated.events_event_aggregator_service_pb2_grpc import ( + EventAggregatorServiceStub, +) from ray.core.generated.events_task_definition_event_pb2 import ( TaskDefinitionEvent, ) from ray.core.generated.events_task_execution_event_pb2 import ( TaskExecutionEvent, ) -from ray.core.generated.profile_events_pb2 import ProfileEvents, ProfileEventEntry from ray.core.generated.events_task_profile_events_pb2 import TaskProfileEvents -from ray.core.generated.events_driver_job_definition_event_pb2 import ( - DriverJobDefinitionEvent, -) -from ray.core.generated.events_driver_job_execution_event_pb2 import ( - DriverJobExecutionEvent, -) +from ray.core.generated.profile_events_pb2 import ProfileEventEntry, ProfileEvents from ray.core.generated.runtime_environment_pb2 import ( + RuntimeEnvConfig, RuntimeEnvInfo, RuntimeEnvUris, - RuntimeEnvConfig, ) -from ray.core.generated.common_pb2 import ( - TaskType, - Language, - FunctionDescriptor, - PythonFunctionDescriptor, - TaskStatus, - ErrorType, - RayErrorInfo, -) - from ray.dashboard.modules.aggregator.aggregator_agent import AggregatorAgent - +from ray.dashboard.tests.conftest import * # noqa _EVENT_AGGREGATOR_AGENT_TARGET_PORT = find_free_port() _EVENT_AGGREGATOR_AGENT_TARGET_IP = "127.0.0.1" diff --git a/python/ray/dashboard/modules/event/event_head.py b/python/ray/dashboard/modules/event/event_head.py index 0a8a9712992e..b9e3d90eebba 100644 --- a/python/ray/dashboard/modules/event/event_head.py +++ b/python/ray/dashboard/modules/event/event_head.py @@ -13,9 +13,9 @@ import ray import ray.dashboard.optional_utils as dashboard_optional_utils import ray.dashboard.utils as dashboard_utils +from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag from ray._common.utils import get_or_create_event_loop from ray._private.ray_constants import env_integer -from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag from ray.dashboard.consts import ( RAY_STATE_SERVER_MAX_HTTP_REQUEST, RAY_STATE_SERVER_MAX_HTTP_REQUEST_ALLOWED, diff --git a/python/ray/dashboard/modules/event/tests/test_event.py b/python/ray/dashboard/modules/event/tests/test_event.py index 87200781a9f5..afff1116de8d 100644 --- a/python/ray/dashboard/modules/event/tests/test_event.py +++ b/python/ray/dashboard/modules/event/tests/test_event.py @@ -14,10 +14,11 @@ import numpy as np import pytest -from ray._common.test_utils import wait_for_condition import requests import ray +from ray._common.test_utils import wait_for_condition +from ray._common.utils import binary_to_hex from ray._private.event.event_logger import ( filter_event_by_level, get_event_id, @@ -33,7 +34,6 @@ format_web_url, wait_until_server_available, ) -from ray._common.utils import binary_to_hex from ray.cluster_utils import AutoscalingCluster from ray.core.generated import ( event_pb2, diff --git a/python/ray/dashboard/modules/job/cli.py b/python/ray/dashboard/modules/job/cli.py index 37e93c6cc50b..f6219ad1cca9 100644 --- a/python/ray/dashboard/modules/job/cli.py +++ b/python/ray/dashboard/modules/job/cli.py @@ -8,10 +8,10 @@ import click -from ray._common.utils import load_class import ray._private.ray_constants as ray_constants from ray._common.utils import ( get_or_create_event_loop, + load_class, ) from ray._private.utils import ( parse_metadata_json, diff --git a/python/ray/dashboard/modules/job/job_head.py b/python/ray/dashboard/modules/job/job_head.py index 06bc1cfe37d1..e91fda0fd3aa 100644 --- a/python/ray/dashboard/modules/job/job_head.py +++ b/python/ray/dashboard/modules/job/job_head.py @@ -15,15 +15,15 @@ import ray from ray import NodeID -from ray._common.utils import get_or_create_event_loop, load_class +from ray._common.network_utils import build_address from ray._common.pydantic_compat import BaseModel, Extra, Field, validator +from ray._common.utils import get_or_create_event_loop, load_class from ray._private.ray_constants import KV_NAMESPACE_DASHBOARD from ray._private.runtime_env.packaging import ( package_exists, pin_runtime_env_uri, upload_package_to_gcs, ) -from ray._common.network_utils import build_address from ray.dashboard.consts import ( DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX, GCS_RPC_TIMEOUT_SECONDS, diff --git a/python/ray/dashboard/modules/job/job_manager.py b/python/ray/dashboard/modules/job/job_manager.py index 78aff4850168..142d8d521fb0 100644 --- a/python/ray/dashboard/modules/job/job_manager.py +++ b/python/ray/dashboard/modules/job/job_manager.py @@ -33,7 +33,7 @@ from ray.dashboard.modules.job.utils import get_head_node_id from ray.dashboard.utils import close_logger_file_descriptor from ray.exceptions import ActorDiedError, ActorUnschedulableError, RuntimeEnvSetupError -from ray.job_submission import JobStatus, JobErrorType +from ray.job_submission import JobErrorType, JobStatus from ray.runtime_env import RuntimeEnvConfig from ray.util.scheduling_strategies import ( NodeAffinitySchedulingStrategy, diff --git a/python/ray/dashboard/modules/job/job_supervisor.py b/python/ray/dashboard/modules/job/job_supervisor.py index 1ffaad752f9c..846fc19eefd6 100644 --- a/python/ray/dashboard/modules/job/job_supervisor.py +++ b/python/ray/dashboard/modules/job/job_supervisor.py @@ -11,6 +11,7 @@ import ray import ray._private.ray_constants as ray_constants +from ray._common.network_utils import build_address from ray._private.accelerators.nvidia_gpu import NOSET_CUDA_VISIBLE_DEVICES_ENV_VAR from ray._private.ray_logging.filters import CoreContextFilter from ray._private.ray_logging.formatters import JSONFormatter, TextFormatter @@ -24,8 +25,7 @@ JobInfoStorageClient, ) from ray.dashboard.modules.job.job_log_storage_client import JobLogStorageClient -from ray.job_submission import JobStatus, JobErrorType -from ray._common.network_utils import build_address +from ray.job_submission import JobErrorType, JobStatus import psutil diff --git a/python/ray/dashboard/modules/job/tests/test_common.py b/python/ray/dashboard/modules/job/tests/test_common.py index 03ca01dc5282..036ad19386a3 100644 --- a/python/ray/dashboard/modules/job/tests/test_common.py +++ b/python/ray/dashboard/modules/job/tests/test_common.py @@ -5,9 +5,9 @@ from ray.core.generated.gcs_pb2 import JobsAPIInfo from ray.dashboard.modules.job.common import ( + JobErrorType, JobInfo, JobStatus, - JobErrorType, JobSubmitRequest, http_uri_components_to_uri, uri_to_http_components, diff --git a/python/ray/dashboard/modules/job/tests/test_component_activities.py b/python/ray/dashboard/modules/job/tests/test_component_activities.py index c0bbabca2086..9aac1651b116 100644 --- a/python/ray/dashboard/modules/job/tests/test_component_activities.py +++ b/python/ray/dashboard/modules/job/tests/test_component_activities.py @@ -5,9 +5,9 @@ import jsonschema import pytest -from ray._common.test_utils import wait_for_condition import requests +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( format_web_url, run_string_as_driver, diff --git a/python/ray/dashboard/modules/job/tests/test_http_job_server.py b/python/ray/dashboard/modules/job/tests/test_http_job_server.py index b66df1178bfa..84d3d09d319f 100644 --- a/python/ray/dashboard/modules/job/tests/test_http_job_server.py +++ b/python/ray/dashboard/modules/job/tests/test_http_job_server.py @@ -11,11 +11,11 @@ from unittest.mock import patch import pytest -from ray._common.test_utils import wait_for_condition import requests import yaml import ray +from ray._common.test_utils import wait_for_condition from ray._private.runtime_env.packaging import ( create_package, download_and_unpack_package, diff --git a/python/ray/dashboard/modules/job/tests/test_job_agent.py b/python/ray/dashboard/modules/job/tests/test_job_agent.py index 73229cc261a1..3a1c012d7030 100644 --- a/python/ray/dashboard/modules/job/tests/test_job_agent.py +++ b/python/ray/dashboard/modules/job/tests/test_job_agent.py @@ -9,11 +9,12 @@ import pytest import pytest_asyncio -from ray._common.test_utils import async_wait_for_condition, wait_for_condition import requests import yaml import ray +from ray._common.network_utils import build_address +from ray._common.test_utils import async_wait_for_condition, wait_for_condition from ray._common.utils import get_or_create_event_loop from ray._private.ray_constants import DEFAULT_DASHBOARD_AGENT_LISTEN_PORT from ray._private.runtime_env.py_modules import upload_py_modules_if_needed @@ -25,7 +26,6 @@ run_string_as_driver_nonblocking, wait_until_server_available, ) -from ray._common.network_utils import build_address from ray.dashboard.modules.job.common import ( JOB_ACTOR_NAME_TEMPLATE, SUPERVISOR_ACTOR_RAY_NAMESPACE, diff --git a/python/ray/dashboard/modules/job/tests/test_job_manager.py b/python/ray/dashboard/modules/job/tests/test_job_manager.py index fb13cbb53df9..e106ef8cf0a2 100644 --- a/python/ray/dashboard/modules/job/tests/test_job_manager.py +++ b/python/ray/dashboard/modules/job/tests/test_job_manager.py @@ -11,17 +11,18 @@ import pytest import ray -from ray._common.test_utils import SignalActor, wait_for_condition +from ray._common.network_utils import build_address +from ray._common.test_utils import ( + SignalActor, + async_wait_for_condition, + wait_for_condition, +) from ray._private.ray_constants import ( DEFAULT_DASHBOARD_AGENT_LISTEN_PORT, KV_HEAD_NODE_ID_KEY, KV_NAMESPACE_JOB, RAY_ADDRESS_ENVIRONMENT_VARIABLE, ) -from ray._common.network_utils import build_address -from ray._common.test_utils import ( - async_wait_for_condition, -) from ray.dashboard.consts import ( RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR, RAY_JOB_START_TIMEOUT_SECONDS_ENV_VAR, @@ -40,7 +41,7 @@ create_ray_cluster, ) from ray.exceptions import RpcError -from ray.job_submission import JobStatus, JobErrorType +from ray.job_submission import JobErrorType, JobStatus from ray.tests.conftest import call_ray_start # noqa: F401 from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy # noqa: F401 from ray.util.state import list_tasks diff --git a/python/ray/dashboard/modules/job/tests/test_sdk.py b/python/ray/dashboard/modules/job/tests/test_sdk.py index f3c370b22b06..fdbffad0654b 100644 --- a/python/ray/dashboard/modules/job/tests/test_sdk.py +++ b/python/ray/dashboard/modules/job/tests/test_sdk.py @@ -8,8 +8,8 @@ import pytest -from ray._common.test_utils import wait_for_condition import ray.experimental.internal_kv as kv +from ray._common.test_utils import wait_for_condition from ray._private.ray_constants import ( KV_NAMESPACE_DASHBOARD, ) diff --git a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py index 5ae50ac361ee..10cb946ae41b 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py @@ -3,8 +3,8 @@ from ray.dashboard.modules.metrics.dashboards.common import ( DashboardConfig, Panel, - Target, Row, + Target, ) # When adding a new panels for an OpRuntimeMetric, follow this format: diff --git a/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py index f6f88729f7ed..d8f0be9a5f1d 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/train_dashboard_panels.py @@ -2,11 +2,10 @@ from ray.dashboard.modules.metrics.dashboards.common import ( DashboardConfig, Panel, - Target, Row, + Target, ) - # Ray Train Metrics (Controller) CONTROLLER_STATE_PANEL = Panel( id=1, diff --git a/python/ray/dashboard/modules/node/tests/test_node.py b/python/ray/dashboard/modules/node/tests/test_node.py index 3016d65491d0..df4f980f4125 100644 --- a/python/ray/dashboard/modules/node/tests/test_node.py +++ b/python/ray/dashboard/modules/node/tests/test_node.py @@ -8,10 +8,10 @@ from datetime import datetime, timedelta import pytest -from ray._common.test_utils import wait_for_condition import requests import ray +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import ( format_web_url, wait_until_server_available, diff --git a/python/ray/dashboard/modules/reporter/gpu_providers.py b/python/ray/dashboard/modules/reporter/gpu_providers.py index 431bc9f10beb..d26ea4597fa6 100644 --- a/python/ray/dashboard/modules/reporter/gpu_providers.py +++ b/python/ray/dashboard/modules/reporter/gpu_providers.py @@ -8,8 +8,8 @@ import enum import logging import subprocess -from typing import Dict, List, Optional, Union, TypedDict from collections import defaultdict +from typing import Dict, List, Optional, TypedDict, Union from ray._private.ray_constants import RAY_METRIC_ENABLE_GPU_NVSMI diff --git a/python/ray/dashboard/modules/reporter/healthz_agent.py b/python/ray/dashboard/modules/reporter/healthz_agent.py index 09581852404d..cff4edaf33d1 100644 --- a/python/ray/dashboard/modules/reporter/healthz_agent.py +++ b/python/ray/dashboard/modules/reporter/healthz_agent.py @@ -3,8 +3,8 @@ import ray.dashboard.optional_utils as optional_utils import ray.dashboard.utils as dashboard_utils import ray.exceptions -from ray.dashboard.modules.reporter.utils import HealthChecker from ray._raylet import NodeID +from ray.dashboard.modules.reporter.utils import HealthChecker routes = optional_utils.DashboardAgentRouteTable diff --git a/python/ray/dashboard/modules/reporter/reporter_agent.py b/python/ray/dashboard/modules/reporter/reporter_agent.py index 8430bff607b6..663a897794e1 100644 --- a/python/ray/dashboard/modules/reporter/reporter_agent.py +++ b/python/ray/dashboard/modules/reporter/reporter_agent.py @@ -3,7 +3,6 @@ import json import logging import os -import requests import socket import sys import traceback @@ -11,32 +10,26 @@ from concurrent.futures import ThreadPoolExecutor from typing import List, Optional, Tuple +import requests +from grpc.aio import ServicerContext from opencensus.stats import stats as stats_module -from prometheus_client.core import REGISTRY -from prometheus_client.parser import text_string_to_metric_families from opentelemetry.proto.collector.metrics.v1 import ( metrics_service_pb2, metrics_service_pb2_grpc, ) from opentelemetry.proto.metrics.v1.metrics_pb2 import Metric -from grpc.aio import ServicerContext - +from prometheus_client.core import REGISTRY +from prometheus_client.parser import text_string_to_metric_families import ray import ray._private.prometheus_exporter as prometheus_exporter import ray.dashboard.modules.reporter.reporter_consts as reporter_consts import ray.dashboard.utils as dashboard_utils +from ray._common.network_utils import parse_address from ray._common.utils import ( get_or_create_event_loop, get_user_temp_dir, ) -from ray._common.network_utils import parse_address -from ray._private.utils import get_system_memory -from ray.dashboard.modules.reporter.gpu_providers import ( - GpuMetricProvider, - GpuUtilizationInfo, - TpuUtilizationInfo, -) from ray._private import utils from ray._private.metrics_agent import Gauge, MetricsAgent, Record from ray._private.ray_constants import ( @@ -47,6 +40,7 @@ from ray._private.telemetry.open_telemetry_metric_recorder import ( OpenTelemetryMetricRecorder, ) +from ray._private.utils import get_system_memory from ray._raylet import GCS_PID_KEY, WorkerID from ray.core.generated import reporter_pb2, reporter_pb2_grpc from ray.dashboard import k8s_utils @@ -56,10 +50,15 @@ COMPONENT_METRICS_TAG_KEYS, GCS_RPC_TIMEOUT_SECONDS, GPU_TAG_KEYS, - TPU_TAG_KEYS, NODE_TAG_KEYS, + TPU_TAG_KEYS, ) from ray.dashboard.modules.reporter.gpu_profile_manager import GpuProfilingManager +from ray.dashboard.modules.reporter.gpu_providers import ( + GpuMetricProvider, + GpuUtilizationInfo, + TpuUtilizationInfo, +) from ray.dashboard.modules.reporter.profile_manager import ( CpuProfilingManager, MemoryProfilingManager, diff --git a/python/ray/dashboard/modules/reporter/reporter_head.py b/python/ray/dashboard/modules/reporter/reporter_head.py index bbcaef0a863c..8971991c4dea 100644 --- a/python/ray/dashboard/modules/reporter/reporter_head.py +++ b/python/ray/dashboard/modules/reporter/reporter_head.py @@ -12,8 +12,9 @@ import ray.dashboard.optional_utils as dashboard_optional_utils import ray.dashboard.utils as dashboard_utils from ray import ActorID, NodeID -from ray._private.metrics_agent import PrometheusServiceDiscoveryWriter from ray._common.network_utils import build_address +from ray._common.usage.usage_constants import CLUSTER_METADATA_KEY +from ray._private.metrics_agent import PrometheusServiceDiscoveryWriter from ray._private.ray_constants import ( DEBUG_AUTOSCALING_ERROR, DEBUG_AUTOSCALING_STATUS, @@ -23,7 +24,6 @@ KV_NAMESPACE_DASHBOARD, env_integer, ) -from ray._common.usage.usage_constants import CLUSTER_METADATA_KEY from ray._private.utils import init_grpc_channel from ray.autoscaler._private.commands import debug_status from ray.core.generated import reporter_pb2, reporter_pb2_grpc diff --git a/python/ray/dashboard/modules/reporter/tests/test_actors.py b/python/ray/dashboard/modules/reporter/tests/test_actors.py index 763c66d866b4..ea1693430da0 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_actors.py +++ b/python/ray/dashboard/modules/reporter/tests/test_actors.py @@ -3,15 +3,16 @@ import sys import time -import psutil import pytest import requests import ray -from ray._private.test_utils import format_web_url, wait_until_server_available -from ray.dashboard.tests.conftest import * # noqa from ray._common.test_utils import wait_for_condition from ray._private.state_api_test_utils import _is_actor_task_running +from ray._private.test_utils import format_web_url, wait_until_server_available +from ray.dashboard.tests.conftest import * # noqa + +import psutil logger = logging.getLogger(__name__) diff --git a/python/ray/dashboard/modules/reporter/tests/test_gpu_providers.py b/python/ray/dashboard/modules/reporter/tests/test_gpu_providers.py index a49e97d421ab..516be688a746 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_gpu_providers.py +++ b/python/ray/dashboard/modules/reporter/tests/test_gpu_providers.py @@ -4,14 +4,14 @@ from unittest.mock import Mock, patch from ray.dashboard.modules.reporter.gpu_providers import ( + MB, + AmdGpuProvider, + GpuMetricProvider, GpuProvider, GpuProviderType, + GpuUtilizationInfo, NvidiaGpuProvider, - AmdGpuProvider, - GpuMetricProvider, ProcessGPUInfo, - GpuUtilizationInfo, - MB, ) diff --git a/python/ray/dashboard/modules/reporter/tests/test_healthz.py b/python/ray/dashboard/modules/reporter/tests/test_healthz.py index fd613e0d59e1..aed2c9515ab3 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_healthz.py +++ b/python/ray/dashboard/modules/reporter/tests/test_healthz.py @@ -1,10 +1,10 @@ import sys import pytest -from ray._common.test_utils import wait_for_condition import requests import ray._private.ray_constants as ray_constants +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import find_free_port from ray.tests.conftest import * # noqa: F401 F403 diff --git a/python/ray/dashboard/modules/reporter/tests/test_reporter.py b/python/ray/dashboard/modules/reporter/tests/test_reporter.py index 658b23dfa9ee..8fbc7218d51a 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_reporter.py +++ b/python/ray/dashboard/modules/reporter/tests/test_reporter.py @@ -9,22 +9,22 @@ import numpy as np import pytest -from ray._common.test_utils import wait_for_condition import requests from google.protobuf import text_format import ray import ray._common.usage.usage_lib as ray_usage_lib +from ray._common.network_utils import build_address +from ray._common.test_utils import wait_for_condition from ray._private import ray_constants from ray._private.metrics_agent import fix_grpc_metric -from ray._common.network_utils import build_address from ray._private.test_utils import ( fetch_prometheus, format_web_url, wait_until_server_available, ) from ray.core.generated.metrics_pb2 import Metric -from ray.dashboard.modules.reporter.gpu_providers import NvidiaGpuProvider, MB +from ray.dashboard.modules.reporter.gpu_providers import MB, NvidiaGpuProvider from ray.dashboard.modules.reporter.reporter_agent import ( ReporterAgent, TpuUtilizationInfo, diff --git a/python/ray/dashboard/modules/serve/tests/test_serve_dashboard.py b/python/ray/dashboard/modules/serve/tests/test_serve_dashboard.py index f0d31bdf3618..ac5f7e7a7998 100644 --- a/python/ray/dashboard/modules/serve/tests/test_serve_dashboard.py +++ b/python/ray/dashboard/modules/serve/tests/test_serve_dashboard.py @@ -6,9 +6,9 @@ from typing import Dict import pytest -from ray._common.test_utils import wait_for_condition import requests +from ray._common.test_utils import wait_for_condition from ray.serve._private.common import ( DeploymentStatus, DeploymentStatusTrigger, diff --git a/python/ray/dashboard/modules/serve/tests/test_serve_dashboard_2.py b/python/ray/dashboard/modules/serve/tests/test_serve_dashboard_2.py index 17e25c6c5b52..a21cad86ad32 100644 --- a/python/ray/dashboard/modules/serve/tests/test_serve_dashboard_2.py +++ b/python/ray/dashboard/modules/serve/tests/test_serve_dashboard_2.py @@ -7,12 +7,12 @@ import grpc import pytest -from ray._common.test_utils import wait_for_condition import requests import ray import ray._private.ray_constants as ray_constants from ray import serve +from ray._common.test_utils import wait_for_condition from ray._private.test_utils import generate_system_config_map from ray.serve.generated import serve_pb2, serve_pb2_grpc from ray.serve.schema import HTTPOptionsSchema, ServeInstanceDetails diff --git a/python/ray/dashboard/modules/state/state_head.py b/python/ray/dashboard/modules/state/state_head.py index 7ef52b3a7cdf..4fa2755d5839 100644 --- a/python/ray/dashboard/modules/state/state_head.py +++ b/python/ray/dashboard/modules/state/state_head.py @@ -10,8 +10,8 @@ import ray from ray import ActorID -from ray._private.ray_constants import env_integer from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag +from ray._private.ray_constants import env_integer from ray.core.generated.gcs_pb2 import ActorTableData from ray.dashboard.consts import ( RAY_STATE_SERVER_MAX_HTTP_REQUEST, diff --git a/python/ray/dashboard/modules/usage_stats/usage_stats_head.py b/python/ray/dashboard/modules/usage_stats/usage_stats_head.py index 0cecf0fba77a..91c500fb1f6f 100644 --- a/python/ray/dashboard/modules/usage_stats/usage_stats_head.py +++ b/python/ray/dashboard/modules/usage_stats/usage_stats_head.py @@ -9,9 +9,9 @@ import ray import ray._common.usage.usage_lib as ray_usage_lib import ray.dashboard.utils as dashboard_utils +from ray._common.network_utils import build_address from ray._common.utils import get_or_create_event_loop from ray.dashboard.utils import async_loop_forever -from ray._common.network_utils import build_address logger = logging.getLogger(__name__) diff --git a/python/ray/dashboard/subprocesses/tests/test_e2e.py b/python/ray/dashboard/subprocesses/tests/test_e2e.py index 05d3cd0a39cb..1f1567f70320 100644 --- a/python/ray/dashboard/subprocesses/tests/test_e2e.py +++ b/python/ray/dashboard/subprocesses/tests/test_e2e.py @@ -6,14 +6,13 @@ import pytest -from ray._common.test_utils import wait_for_condition import ray._private.ray_constants as ray_constants +import ray.dashboard.consts as dashboard_consts from ray._common.ray_constants import ( - LOGGING_ROTATE_BYTES, LOGGING_ROTATE_BACKUP_COUNT, + LOGGING_ROTATE_BYTES, ) -import ray.dashboard.consts as dashboard_consts -from ray._common.test_utils import async_wait_for_condition +from ray._common.test_utils import async_wait_for_condition, wait_for_condition from ray.dashboard.optional_deps import aiohttp from ray.dashboard.subprocesses.handle import SubprocessModuleHandle from ray.dashboard.subprocesses.module import SubprocessModule, SubprocessModuleConfig diff --git a/python/ray/dashboard/tests/test_dashboard.py b/python/ray/dashboard/tests/test_dashboard.py index 1f7036651465..3219606b96bc 100644 --- a/python/ray/dashboard/tests/test_dashboard.py +++ b/python/ray/dashboard/tests/test_dashboard.py @@ -14,23 +14,23 @@ from urllib.parse import quote_plus import pytest -from ray._common.test_utils import wait_for_condition import requests from click.testing import CliRunner from requests.exceptions import ConnectionError, HTTPError import ray +import ray._private.ray_constants as ray_constants import ray.dashboard.consts as dashboard_consts import ray.dashboard.modules import ray.dashboard.utils as dashboard_utils import ray.scripts.scripts as scripts -from ray._common.utils import get_or_create_event_loop -import ray._private.ray_constants as ray_constants +from ray._common.network_utils import build_address, parse_address from ray._common.ray_constants import ( - LOGGING_ROTATE_BYTES, LOGGING_ROTATE_BACKUP_COUNT, + LOGGING_ROTATE_BYTES, ) -from ray._common.network_utils import build_address, parse_address +from ray._common.test_utils import wait_for_condition +from ray._common.utils import get_or_create_event_loop from ray._private.ray_constants import ( DEBUG_AUTOSCALING_ERROR, DEBUG_AUTOSCALING_STATUS_LEGACY, diff --git a/python/ray/dashboard/utils.py b/python/ray/dashboard/utils.py index 73cfcd298ec8..4cb60681abeb 100644 --- a/python/ray/dashboard/utils.py +++ b/python/ray/dashboard/utils.py @@ -26,9 +26,9 @@ import ray._private.ray_constants as ray_constants import ray._private.services as services import ray.experimental.internal_kv as internal_kv +from ray._common.network_utils import parse_address from ray._common.utils import get_or_create_event_loop from ray._private.gcs_utils import GcsChannel -from ray._common.network_utils import parse_address from ray._private.utils import ( get_dashboard_dependency_error, split_address, diff --git a/python/ray/exceptions.py b/python/ray/exceptions.py index f628ed6549bd..c1c96ad00e1c 100644 --- a/python/ray/exceptions.py +++ b/python/ray/exceptions.py @@ -19,7 +19,6 @@ ) from ray.util.annotations import DeveloperAPI, PublicAPI - logger = logging.getLogger(__name__) diff --git a/python/ray/experimental/__init__.py b/python/ray/experimental/__init__.py index c6c5403a4ea0..37cb09a1513b 100644 --- a/python/ray/experimental/__init__.py +++ b/python/ray/experimental/__init__.py @@ -1,6 +1,6 @@ from ray.experimental.dynamic_resources import set_resource -from ray.experimental.locations import get_local_object_locations, get_object_locations from ray.experimental.gpu_object_manager import GPUObjectManager, wait_tensor_freed +from ray.experimental.locations import get_local_object_locations, get_object_locations __all__ = [ "get_object_locations", diff --git a/python/ray/experimental/channel/accelerator_context.py b/python/ray/experimental/channel/accelerator_context.py index 9acc1ad67d33..f4aa622af2b5 100644 --- a/python/ray/experimental/channel/accelerator_context.py +++ b/python/ray/experimental/channel/accelerator_context.py @@ -1,10 +1,11 @@ -import threading import importlib -import ray -from typing import TYPE_CHECKING, Optional, Type, ContextManager, List +import threading from contextlib import nullcontext -from ray.experimental.channel.communicator import Communicator +from typing import TYPE_CHECKING, ContextManager, List, Optional, Type + +import ray from ray._private.accelerators import get_accelerator_manager_for_resource +from ray.experimental.channel.communicator import Communicator if TYPE_CHECKING: import torch diff --git a/python/ray/experimental/channel/common.py b/python/ray/experimental/channel/common.py index 8de5f5642312..0f1b916a7224 100644 --- a/python/ray/experimental/channel/common.py +++ b/python/ray/experimental/channel/common.py @@ -18,9 +18,9 @@ import ray import ray.exceptions +from ray.experimental.channel.accelerator_context import AcceleratorContext from ray.experimental.channel.communicator import Communicator from ray.experimental.channel.communicator_handle import CommunicatorHandle -from ray.experimental.channel.accelerator_context import AcceleratorContext from ray.experimental.channel.serialization_context import _SerializationContext from ray.util.annotations import DeveloperAPI, PublicAPI diff --git a/python/ray/experimental/channel/communicator_handle.py b/python/ray/experimental/channel/communicator_handle.py index 26dd000ad98c..c6d8865bfc44 100644 --- a/python/ray/experimental/channel/communicator_handle.py +++ b/python/ray/experimental/channel/communicator_handle.py @@ -1,4 +1,5 @@ from typing import List + import ray diff --git a/python/ray/experimental/channel/nccl_group.py b/python/ray/experimental/channel/nccl_group.py index f81c8c4bcadb..64b640818833 100644 --- a/python/ray/experimental/channel/nccl_group.py +++ b/python/ray/experimental/channel/nccl_group.py @@ -4,9 +4,9 @@ import ray from ray.exceptions import RayChannelError +from ray.experimental.channel.accelerator_context import AcceleratorContext from ray.experimental.channel.communicator import Communicator, TorchTensorAllocator from ray.experimental.util.types import ReduceOp -from ray.experimental.channel.accelerator_context import AcceleratorContext if TYPE_CHECKING: import torch diff --git a/python/ray/experimental/channel/serialization_context.py b/python/ray/experimental/channel/serialization_context.py index d06a2d9e098b..548d36301f6b 100644 --- a/python/ray/experimental/channel/serialization_context.py +++ b/python/ray/experimental/channel/serialization_context.py @@ -174,8 +174,8 @@ def deserialize_from_numpy_or_scalar( tensor_device_type: str, target_device: Device, ): - import torch import numpy as np + import torch if target_device == Device.DEFAULT: target_device_type = tensor_device_type diff --git a/python/ray/experimental/channel/torch_tensor_accelerator_channel.py b/python/ray/experimental/channel/torch_tensor_accelerator_channel.py index ae1cb3772b6c..08149406951a 100644 --- a/python/ray/experimental/channel/torch_tensor_accelerator_channel.py +++ b/python/ray/experimental/channel/torch_tensor_accelerator_channel.py @@ -3,24 +3,24 @@ import uuid from dataclasses import dataclass from types import ModuleType -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, Type +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union import ray import ray.util.serialization from ray.experimental.channel import ChannelContext, utils +from ray.experimental.channel.accelerator_context import ( + AcceleratorContext, + is_accelerator_context_registered, + register_accelerator_context, +) from ray.experimental.channel.common import ChannelInterface from ray.experimental.channel.communicator import Communicator +from ray.experimental.channel.communicator_handle import CommunicatorHandle from ray.experimental.channel.cpu_communicator import CPUCommunicator from ray.experimental.channel.intra_process_channel import IntraProcessChannel -from ray.experimental.channel.communicator_handle import CommunicatorHandle from ray.experimental.channel.shared_memory_channel import SharedMemoryType from ray.experimental.channel.torch_tensor_type import TorchTensorType from ray.util.annotations import DeveloperAPI -from ray.experimental.channel.accelerator_context import ( - AcceleratorContext, - register_accelerator_context, - is_accelerator_context_registered, -) if TYPE_CHECKING: import torch diff --git a/python/ray/experimental/collective/__init__.py b/python/ray/experimental/collective/__init__.py index 8a526e691684..42289cee1653 100644 --- a/python/ray/experimental/collective/__init__.py +++ b/python/ray/experimental/collective/__init__.py @@ -1,17 +1,16 @@ +from ray.experimental.collective.collective import ( + create_collective_group, + destroy_all_collective_groups, + destroy_collective_group, + get_collective_groups, +) from ray.experimental.collective.operations import ( allgather, allreduce, reducescatter, ) -from ray.experimental.collective.collective import ( - get_collective_groups, - create_collective_group, - destroy_collective_group, - destroy_all_collective_groups, -) from ray.experimental.collective.util import get_tensor_transport_manager - __all__ = [ "allgather", "allreduce", diff --git a/python/ray/experimental/collective/collective.py b/python/ray/experimental/collective/collective.py index b2b05a931d39..db60fd500dab 100644 --- a/python/ray/experimental/collective/collective.py +++ b/python/ray/experimental/collective/collective.py @@ -1,17 +1,16 @@ -from typing import Dict, List, Optional, Union import threading import uuid +from typing import Dict, List, Optional, Union import ray +import ray.experimental.internal_kv as internal_kv from ray.experimental.collective.communicator import CommunicatorHandle from ray.experimental.collective.util import get_address_and_port -import ray.experimental.internal_kv as internal_kv -from ray.util.collective.types import Backend +from ray.util.annotations import PublicAPI from ray.util.collective.collective_group.torch_gloo_collective_group import ( get_master_address_metadata_key, ) -from ray.util.annotations import PublicAPI - +from ray.util.collective.types import Backend _remote_communicator_manager: "Optional[RemoteCommunicatorManager]" = None _remote_communicator_manager_lock = threading.Lock() diff --git a/python/ray/experimental/collective/collective_tensor_transport.py b/python/ray/experimental/collective/collective_tensor_transport.py index 1c0f84e05511..33155886f071 100644 --- a/python/ray/experimental/collective/collective_tensor_transport.py +++ b/python/ray/experimental/collective/collective_tensor_transport.py @@ -1,15 +1,14 @@ -from typing import Optional, List, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional import ray -from ray.util.collective.types import Backend from ray.experimental.collective.tensor_transport_manager import ( - TensorTransportManager, TensorTransportEnum, + TensorTransportManager, ) - from ray.util.collective.types import ( - CollectiveTransportMetadata, + Backend, CollectiveCommunicatorMetadata, + CollectiveTransportMetadata, ) if TYPE_CHECKING: diff --git a/python/ray/experimental/collective/communicator.py b/python/ray/experimental/collective/communicator.py index ba4aaecb87ae..2379bf220389 100644 --- a/python/ray/experimental/collective/communicator.py +++ b/python/ray/experimental/collective/communicator.py @@ -1,5 +1,5 @@ -from typing import List from dataclasses import dataclass +from typing import List import ray from ray.util.collective.types import Backend diff --git a/python/ray/experimental/collective/nixl_tensor_transport.py b/python/ray/experimental/collective/nixl_tensor_transport.py index b9938ccb8507..2046f55ac9eb 100644 --- a/python/ray/experimental/collective/nixl_tensor_transport.py +++ b/python/ray/experimental/collective/nixl_tensor_transport.py @@ -1,16 +1,16 @@ -from typing import Optional, List, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional import ray -from ray.util.collective.types import Backend from ray.experimental.collective.tensor_transport_manager import ( - TensorTransportManager, TensorTransportEnum, + TensorTransportManager, ) from ray.util.collective.collective import get_group_handle from ray.util.collective.types import ( NIXL_GROUP_NAME, - NixlTransportMetadata, + Backend, NixlCommunicatorMetadata, + NixlTransportMetadata, ) if TYPE_CHECKING: @@ -120,8 +120,8 @@ def recv_multiple_tensors( tensor_transport_metadata: NixlTransportMetadata, communicator_metadata: NixlCommunicatorMetadata, ): - from ray.util.collective.collective import get_group_handle from ray.util.collective import types + from ray.util.collective.collective import get_group_handle if tensors: g = get_group_handle(communicator_metadata.communicator_name) diff --git a/python/ray/experimental/collective/operations.py b/python/ray/experimental/collective/operations.py index 31edbcb9eb28..aa8ba37ecc1f 100644 --- a/python/ray/experimental/collective/operations.py +++ b/python/ray/experimental/collective/operations.py @@ -6,14 +6,14 @@ from ray.dag.constants import ( BIND_INDEX_KEY, COLLECTIVE_OPERATION_KEY, - PARENT_CLASS_NODE_KEY, IS_CLASS_METHOD_OUTPUT_KEY, + PARENT_CLASS_NODE_KEY, ) from ray.experimental.channel.torch_tensor_type import Communicator, TorchTensorType from ray.experimental.util.types import ( - ReduceOp, AllGatherOp, AllReduceOp, + ReduceOp, ReduceScatterOp, _CollectiveOp, ) diff --git a/python/ray/experimental/collective/tensor_transport_manager.py b/python/ray/experimental/collective/tensor_transport_manager.py index 5daf604f640f..8a8b29dae316 100644 --- a/python/ray/experimental/collective/tensor_transport_manager.py +++ b/python/ray/experimental/collective/tensor_transport_manager.py @@ -1,10 +1,13 @@ from abc import ABC, abstractmethod -from typing import List, Optional, TYPE_CHECKING -from ray.util.collective.types import TensorTransportMetadata, CommunicatorMetadata -from ray.util.collective.types import Backend -from ray._private.custom_types import TensorTransportEnum +from typing import TYPE_CHECKING, List, Optional import ray +from ray._private.custom_types import TensorTransportEnum +from ray.util.collective.types import ( + Backend, + CommunicatorMetadata, + TensorTransportMetadata, +) if TYPE_CHECKING: import torch diff --git a/python/ray/experimental/collective/util.py b/python/ray/experimental/collective/util.py index 3539e5780766..5b09697247f3 100644 --- a/python/ray/experimental/collective/util.py +++ b/python/ray/experimental/collective/util.py @@ -1,15 +1,14 @@ -from typing import Tuple, TYPE_CHECKING -from contextlib import closing import socket +from contextlib import closing +from typing import TYPE_CHECKING, Tuple import ray - -from ray.util.collective.types import Backend -from ray.experimental.collective.tensor_transport_manager import TensorTransportManager -from ray.experimental.collective.nixl_tensor_transport import NixlTensorTransport from ray.experimental.collective.collective_tensor_transport import ( CollectiveTensorTransport, ) +from ray.experimental.collective.nixl_tensor_transport import NixlTensorTransport +from ray.experimental.collective.tensor_transport_manager import TensorTransportManager +from ray.util.collective.types import Backend if TYPE_CHECKING: import torch diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py index 87d089880a98..f0a851386895 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py @@ -1,18 +1,19 @@ -import warnings -from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, List, Set import threading +import warnings +from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Set, Tuple import ray +from ray._private import ray_constants from ray._private.custom_types import TensorTransportEnum from ray._raylet import ObjectRef -from ray._private import ray_constants if TYPE_CHECKING: + import torch + from ray.experimental.gpu_object_manager.gpu_object_store import ( GPUObjectStore, ) from ray.util.collective.types import TensorTransportMetadata - import torch # GPUObjectMeta is a named tuple containing the source actor, tensor transport # backend, tensor metadata, and other information that needs to be recorded. @@ -110,10 +111,10 @@ def add_gpu_object_ref( src_actor: The actor that executes the task and that creates the GPU object. tensor_transport: The tensor transport protocol to use for the GPU object. """ + from ray.experimental.collective import get_tensor_transport_manager from ray.experimental.gpu_object_manager.gpu_object_store import ( _tensor_transport_to_collective_backend, ) - from ray.experimental.collective import get_tensor_transport_manager tensor_transport_backend = _tensor_transport_to_collective_backend( tensor_transport diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_store.py b/python/ray/experimental/gpu_object_manager/gpu_object_store.py index 7f6309048d8b..d526a8b4ff6f 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_store.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_store.py @@ -1,19 +1,18 @@ -from dataclasses import dataclass -from typing import Dict, List, Optional, Set import threading from collections import defaultdict, deque +from dataclasses import dataclass +from typing import Dict, List, Optional, Set import ray.util.collective as collective from ray._private.custom_types import TensorTransportEnum +from ray.experimental.collective import get_tensor_transport_manager +from ray.experimental.collective.util import device_match_transport from ray.util.collective.types import ( Backend, CommunicatorMetadata, TensorTransportMetadata, ) -from ray.experimental.collective import get_tensor_transport_manager -from ray.experimental.collective.util import device_match_transport - try: import torch except ImportError: diff --git a/python/ray/remote_function.py b/python/ray/remote_function.py index ac661dde1a23..0d6d20e44579 100644 --- a/python/ray/remote_function.py +++ b/python/ray/remote_function.py @@ -9,12 +9,12 @@ import ray._common.signature from ray import Language, cross_language from ray._common import ray_option_utils +from ray._common.ray_option_utils import _warn_if_using_deprecated_placement_group from ray._private.auto_init_hook import wrap_auto_init from ray._private.client_mode_hook import ( client_mode_convert_function, client_mode_should_convert, ) -from ray._common.ray_option_utils import _warn_if_using_deprecated_placement_group from ray._private.serialization import pickle_dumps from ray._private.utils import get_runtime_env_info, parse_runtime_env_for_task_or_actor from ray._raylet import ( diff --git a/python/ray/runtime_context.py b/python/ray/runtime_context.py index 5257c9688a80..df669f3447f5 100644 --- a/python/ray/runtime_context.py +++ b/python/ray/runtime_context.py @@ -1,6 +1,6 @@ import logging -from typing import Any, Dict, List, Optional import threading +from typing import Any, Dict, List, Optional import ray._private.worker from ray._private.client_mode_hook import client_mode_hook diff --git a/python/ray/runtime_env/runtime_env.py b/python/ray/runtime_env/runtime_env.py index c43685a921d0..b1ef6c87a743 100644 --- a/python/ray/runtime_env/runtime_env.py +++ b/python/ray/runtime_env/runtime_env.py @@ -13,8 +13,8 @@ from ray._private.runtime_env.plugin_schema_manager import RuntimeEnvPluginSchemaManager from ray._private.runtime_env.uv import get_uri as get_uv_uri from ray._private.runtime_env.validation import ( - OPTION_TO_VALIDATION_FN, OPTION_TO_NO_PATH_VALIDATION_FN, + OPTION_TO_VALIDATION_FN, ) from ray._private.thirdparty.dacite import from_dict from ray.core.generated.runtime_environment_pb2 import ( diff --git a/python/ray/util/__init__.py b/python/ray/util/__init__.py index 81feec19bcda..19d58a0dd318 100644 --- a/python/ray/util/__init__.py +++ b/python/ray/util/__init__.py @@ -1,14 +1,11 @@ from typing import List import ray -from ray._private.client_mode_hook import client_mode_hook from ray._private.auto_init_hook import wrap_auto_init +from ray._private.client_mode_hook import client_mode_hook from ray._private.services import get_node_instance_id, get_node_ip_address -from ray.util import iter -from ray.util import rpdb as pdb -from ray.util import debugpy as ray_debugpy +from ray.util import accelerators, debugpy as ray_debugpy, iter, rpdb as pdb from ray.util.actor_pool import ActorPool -from ray.util import accelerators from ray.util.annotations import PublicAPI from ray.util.check_serialize import inspect_serializability from ray.util.client_connect import connect, disconnect diff --git a/python/ray/util/accelerators/__init__.py b/python/ray/util/accelerators/__init__.py index 62888bc9de51..6c757121207b 100644 --- a/python/ray/util/accelerators/__init__.py +++ b/python/ray/util/accelerators/__init__.py @@ -2,32 +2,32 @@ from ray.util.accelerators import tpu from ray.util.accelerators.accelerators import ( - NVIDIA_TESLA_V100, - NVIDIA_TESLA_P100, - NVIDIA_TESLA_T4, - NVIDIA_TESLA_P4, - NVIDIA_TESLA_K80, - NVIDIA_TESLA_A10G, - NVIDIA_L4, - NVIDIA_A100, - NVIDIA_H100, - INTEL_MAX_1550, - INTEL_MAX_1100, - INTEL_GAUDI, AMD_INSTINCT_MI100, AMD_INSTINCT_MI210, AMD_INSTINCT_MI250, - AMD_INSTINCT_MI250x, - AMD_INSTINCT_MI300x, - AMD_RADEON_R9_200_HD_7900, AMD_RADEON_HD_7900, + AMD_RADEON_R9_200_HD_7900, AWS_NEURON_CORE, GOOGLE_TPU_V2, GOOGLE_TPU_V3, GOOGLE_TPU_V4, - GOOGLE_TPU_V5P, GOOGLE_TPU_V5LITEPOD, + GOOGLE_TPU_V5P, GOOGLE_TPU_V6E, + INTEL_GAUDI, + INTEL_MAX_1100, + INTEL_MAX_1550, + NVIDIA_A100, + NVIDIA_H100, + NVIDIA_L4, + NVIDIA_TESLA_A10G, + NVIDIA_TESLA_K80, + NVIDIA_TESLA_P4, + NVIDIA_TESLA_P100, + NVIDIA_TESLA_T4, + NVIDIA_TESLA_V100, + AMD_INSTINCT_MI250x, + AMD_INSTINCT_MI300x, ) __all__ = [ diff --git a/python/ray/util/accelerators/tpu.py b/python/ray/util/accelerators/tpu.py index 01dfbcf4a02f..ff31581d7ca5 100644 --- a/python/ray/util/accelerators/tpu.py +++ b/python/ray/util/accelerators/tpu.py @@ -1,4 +1,5 @@ from typing import Optional + from ray._private.accelerators import TPUAcceleratorManager from ray.util.annotations import PublicAPI diff --git a/python/ray/util/actor_group.py b/python/ray/util/actor_group.py index 53fe83285a72..5cd343f1b17d 100644 --- a/python/ray/util/actor_group.py +++ b/python/ray/util/actor_group.py @@ -1,12 +1,12 @@ +import logging import weakref from dataclasses import dataclass -import logging -from typing import List, TypeVar, Optional, Dict, Type, Tuple +from typing import Dict, List, Optional, Tuple, Type, TypeVar import ray +from ray._private.utils import get_ray_doc_version from ray.actor import ActorHandle from ray.util.annotations import Deprecated -from ray._private.utils import get_ray_doc_version T = TypeVar("T") ActorMetadata = TypeVar("ActorMetadata") diff --git a/python/ray/util/annotations.py b/python/ray/util/annotations.py index 206c02b36d26..a2e3fc664d55 100644 --- a/python/ray/util/annotations.py +++ b/python/ray/util/annotations.py @@ -1,9 +1,9 @@ -from enum import Enum -from typing import Optional import inspect import sys import warnings +from enum import Enum from functools import wraps +from typing import Optional class AnnotationType(Enum): diff --git a/python/ray/util/check_open_ports.py b/python/ray/util/check_open_ports.py index 29c9e03e4740..67f5e1fd87a5 100644 --- a/python/ray/util/check_open_ports.py +++ b/python/ray/util/check_open_ports.py @@ -3,19 +3,21 @@ See https://www.anyscale.com/blog/update-on-ray-cve-2023-48022-new-verification-tooling-available # noqa: E501 for more details. """ -from typing import List, Tuple +import json import subprocess -import click -import psutil import urllib -import json +from typing import List, Tuple + +import click import ray -from ray.util.annotations import PublicAPI from ray.autoscaler._private.cli_logger import add_click_logging_options, cli_logger from ray.autoscaler._private.constants import RAY_PROCESSES +from ray.util.annotations import PublicAPI from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy +import psutil + def _get_ray_ports() -> List[int]: unique_ports = set() diff --git a/python/ray/util/check_serialize.py b/python/ray/util/check_serialize.py index a9a8377b3a77..04e1c9633f26 100644 --- a/python/ray/util/check_serialize.py +++ b/python/ray/util/check_serialize.py @@ -3,9 +3,10 @@ from contextlib import contextmanager from typing import Any, Optional, Set, Tuple +import colorama + # Import ray first to use the bundled colorama import ray # noqa: F401 -import colorama import ray.cloudpickle as cp from ray.util.annotations import DeveloperAPI diff --git a/python/ray/util/client/__init__.py b/python/ray/util/client/__init__.py index e3d009f172a5..97f4bf2802bc 100644 --- a/python/ray/util/client/__init__.py +++ b/python/ray/util/client/__init__.py @@ -9,10 +9,9 @@ _explicitly_enable_client_mode, ) from ray._private.ray_logging import setup_logger +from ray._private.utils import check_version_info from ray.job_config import JobConfig from ray.util.annotations import DeveloperAPI -from ray._private.utils import check_version_info - logger = logging.getLogger(__name__) diff --git a/python/ray/util/client/client_app.py b/python/ray/util/client/client_app.py index ec0a37021298..612700147f4f 100644 --- a/python/ray/util/client/client_app.py +++ b/python/ray/util/client/client_app.py @@ -1,6 +1,7 @@ -from ray.util.client import ray from typing import Tuple +from ray.util.client import ray + ray.connect("localhost:50051") diff --git a/python/ray/util/client/client_pickler.py b/python/ray/util/client/client_pickler.py index 4971c0e11f96..39a025f1efac 100644 --- a/python/ray/util/client/client_pickler.py +++ b/python/ray/util/client/client_pickler.py @@ -22,25 +22,22 @@ """ import io - -from typing import NamedTuple -from typing import Any -from typing import Dict -from typing import Optional +import pickle # noqa: F401 +from typing import Any, Dict, NamedTuple, Optional import ray.cloudpickle as cloudpickle -from ray.util.client import RayAPIStub -from ray.util.client.common import ClientObjectRef -from ray.util.client.common import ClientActorHandle -from ray.util.client.common import ClientActorRef -from ray.util.client.common import ClientActorClass -from ray.util.client.common import ClientRemoteFunc -from ray.util.client.common import ClientRemoteMethod -from ray.util.client.common import OptionWrapper -from ray.util.client.common import InProgressSentinel import ray.core.generated.ray_client_pb2 as ray_client_pb2 - -import pickle # noqa: F401 +from ray.util.client import RayAPIStub +from ray.util.client.common import ( + ClientActorClass, + ClientActorHandle, + ClientActorRef, + ClientObjectRef, + ClientRemoteFunc, + ClientRemoteMethod, + InProgressSentinel, + OptionWrapper, +) # NOTE(barakmich): These PickleStubs are really close to diff --git a/python/ray/util/client/common.py b/python/ray/util/client/common.py index 7d027d2b0386..80435bc5c4fd 100644 --- a/python/ray/util/client/common.py +++ b/python/ray/util/client/common.py @@ -14,6 +14,7 @@ import ray._raylet as raylet import ray.core.generated.ray_client_pb2 as ray_client_pb2 import ray.core.generated.ray_client_pb2_grpc as ray_client_pb2_grpc +from ray._common.signature import extract_signature, get_signature from ray._private import ray_constants from ray._private.inspect_util import ( is_class_method, @@ -21,7 +22,6 @@ is_function_or_method, is_static_method, ) -from ray._common.signature import extract_signature, get_signature from ray._private.utils import check_oversized_function from ray.util.client import ray from ray.util.client.options import validate_options diff --git a/python/ray/util/client/dataclient.py b/python/ray/util/client/dataclient.py index 5ce08117087d..6ef6f29c190b 100644 --- a/python/ray/util/client/dataclient.py +++ b/python/ray/util/client/dataclient.py @@ -1,15 +1,15 @@ """This file implements a threaded stream controller to abstract a data stream back to the ray clientserver. """ -import math import logging +import math import queue import threading import warnings -import grpc - from collections import OrderedDict -from typing import Any, Callable, Dict, TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union + +import grpc import ray.core.generated.ray_client_pb2 as ray_client_pb2 import ray.core.generated.ray_client_pb2_grpc as ray_client_pb2_grpc diff --git a/python/ray/util/client/examples/run_tune.py b/python/ray/util/client/examples/run_tune.py index d7b76b778f4c..048c7de299be 100644 --- a/python/ray/util/client/examples/run_tune.py +++ b/python/ray/util/client/examples/run_tune.py @@ -1,6 +1,5 @@ -from ray.util.client import ray - from ray.tune import tune +from ray.util.client import ray ray.connect("localhost:50051") diff --git a/python/ray/util/client/logsclient.py b/python/ray/util/client/logsclient.py index b4d9a6af9928..34ad3f9f6ce9 100644 --- a/python/ray/util/client/logsclient.py +++ b/python/ray/util/client/logsclient.py @@ -1,18 +1,17 @@ """This file implements a threaded stream controller to return logs back from the ray clientserver. """ -import sys import logging import queue +import sys import threading import time -import grpc - from typing import TYPE_CHECKING +import grpc + import ray.core.generated.ray_client_pb2 as ray_client_pb2 import ray.core.generated.ray_client_pb2_grpc as ray_client_pb2_grpc - from ray.util.debug import log_once if TYPE_CHECKING: diff --git a/python/ray/util/client/options.py b/python/ray/util/client/options.py index 57f00109af5d..bd0946fa1975 100644 --- a/python/ray/util/client/options.py +++ b/python/ray/util/client/options.py @@ -1,6 +1,4 @@ -from typing import Any -from typing import Dict -from typing import Optional +from typing import Any, Dict, Optional from ray._common import ray_option_utils from ray.util.placement_group import PlacementGroup, check_placement_group_index diff --git a/python/ray/util/client/ray_client_helpers.py b/python/ray/util/client/ray_client_helpers.py index f1ff0eab01e8..1554bd5e1c23 100644 --- a/python/ray/util/client/ray_client_helpers.py +++ b/python/ray/util/client/ray_client_helpers.py @@ -1,12 +1,12 @@ -from contextlib import contextmanager import time +from contextlib import contextmanager from typing import Any, Dict import ray as real_ray -from ray.job_config import JobConfig import ray.util.client.server.server as ray_client_server -from ray.util.client import ray from ray._private.client_mode_hook import disable_client_hook +from ray.job_config import JobConfig +from ray.util.client import ray @contextmanager diff --git a/python/ray/util/client/runtime_context.py b/python/ray/util/client/runtime_context.py index 0fe9f33935cf..ea28055361d8 100644 --- a/python/ray/util/client/runtime_context.py +++ b/python/ray/util/client/runtime_context.py @@ -1,5 +1,5 @@ -from typing import TYPE_CHECKING from types import SimpleNamespace +from typing import TYPE_CHECKING if TYPE_CHECKING: from ray import JobID, NodeID diff --git a/python/ray/util/client/server/dataservicer.py b/python/ray/util/client/server/dataservicer.py index af06b8902785..0e9363ea3640 100644 --- a/python/ray/util/client/server/dataservicer.py +++ b/python/ray/util/client/server/dataservicer.py @@ -1,24 +1,24 @@ -from collections import defaultdict -from ray.util.client.server.server_pickler import loads_from_client -import ray import logging -import grpc -from queue import Queue import sys - -from typing import Any, Dict, Iterator, TYPE_CHECKING, Union -from threading import Event, Lock, Thread import time +from collections import defaultdict +from queue import Queue +from threading import Event, Lock, Thread +from typing import TYPE_CHECKING, Any, Dict, Iterator, Union + +import grpc +import ray import ray.core.generated.ray_client_pb2 as ray_client_pb2 import ray.core.generated.ray_client_pb2_grpc as ray_client_pb2_grpc +from ray._private.client_mode_hook import disable_client_hook from ray.util.client.common import ( CLIENT_SERVER_MAX_THREADS, - _propagate_error_in_context, OrderedResponseCache, + _propagate_error_in_context, ) +from ray.util.client.server.server_pickler import loads_from_client from ray.util.debug import log_once -from ray._private.client_mode_hook import disable_client_hook if TYPE_CHECKING: from ray.util.client.server.server import RayletServicer diff --git a/python/ray/util/client/server/proxier.py b/python/ray/util/client/server/proxier.py index 8be870c566d7..7bc959e3df17 100644 --- a/python/ray/util/client/server/proxier.py +++ b/python/ray/util/client/server/proxier.py @@ -5,30 +5,27 @@ import sys import time import traceback +import urllib from concurrent import futures from dataclasses import dataclass from itertools import chain -import urllib from threading import Event, Lock, RLock, Thread from typing import Callable, Dict, List, Optional, Tuple import grpc -# Import psutil after ray so the packaged version is used. -import psutil - import ray import ray.core.generated.ray_client_pb2 as ray_client_pb2 import ray.core.generated.ray_client_pb2_grpc as ray_client_pb2_grpc import ray.core.generated.runtime_env_agent_pb2 as runtime_env_agent_pb2 +from ray._common.network_utils import build_address, is_localhost from ray._private.client_mode_hook import disable_client_hook -from ray._raylet import GcsClient from ray._private.parameter import RayParams from ray._private.runtime_env.context import RuntimeEnvContext from ray._private.services import ProcessInfo, start_ray_client_server from ray._private.tls_utils import add_port_to_grpc_server from ray._private.utils import detect_fate_sharing_support -from ray._common.network_utils import build_address, is_localhost +from ray._raylet import GcsClient from ray.cloudpickle.compat import pickle from ray.job_config import JobConfig from ray.util.client.common import ( @@ -40,6 +37,9 @@ ) from ray.util.client.server.dataservicer import _get_reconnecting_from_context +# Import psutil after ray so the packaged version is used. +import psutil + logger = logging.getLogger(__name__) CHECK_PROCESS_INTERVAL_S = 30 diff --git a/python/ray/util/client/server/server.py b/python/ray/util/client/server/server.py index f4194ae83fa0..c4e5d897e09d 100644 --- a/python/ray/util/client/server/server.py +++ b/python/ray/util/client/server/server.py @@ -20,14 +20,14 @@ import ray.core.generated.ray_client_pb2 as ray_client_pb2 import ray.core.generated.ray_client_pb2_grpc as ray_client_pb2_grpc from ray import cloudpickle +from ray._common.network_utils import build_address, is_localhost from ray._private import ray_constants from ray._private.client_mode_hook import disable_client_hook -from ray._raylet import GcsClient from ray._private.ray_constants import env_integer from ray._private.ray_logging import setup_logger from ray._private.services import canonicalize_bootstrap_address_or_die from ray._private.tls_utils import add_port_to_grpc_server -from ray._common.network_utils import build_address, is_localhost +from ray._raylet import GcsClient from ray.job_config import JobConfig from ray.util.client.common import ( CLIENT_SERVER_MAX_THREADS, diff --git a/python/ray/util/client/server/server_pickler.py b/python/ray/util/client/server/server_pickler.py index a0d91f400baa..5211a7991a86 100644 --- a/python/ray/util/client/server/server_pickler.py +++ b/python/ray/util/client/server/server_pickler.py @@ -12,16 +12,16 @@ in the server instance. """ import io -import ray - -from typing import Any -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any -from ray._private.client_mode_hook import disable_client_hook +import ray import ray.cloudpickle as cloudpickle +from ray._private.client_mode_hook import disable_client_hook from ray.util.client.client_pickler import PickleStub -from ray.util.client.server.server_stubs import ClientReferenceActor -from ray.util.client.server.server_stubs import ClientReferenceFunction +from ray.util.client.server.server_stubs import ( + ClientReferenceActor, + ClientReferenceFunction, +) if TYPE_CHECKING: from ray.util.client.server.server import RayletServicer diff --git a/python/ray/util/client/server/server_stubs.py b/python/ray/util/client/server/server_stubs.py index e19cbb3134a4..020ebf2aeb2c 100644 --- a/python/ray/util/client/server/server_stubs.py +++ b/python/ray/util/client/server/server_stubs.py @@ -1,6 +1,5 @@ +from abc import ABC, abstractmethod from contextlib import contextmanager -from abc import ABC -from abc import abstractmethod _current_server = None diff --git a/python/ray/util/client_connect.py b/python/ray/util/client_connect.py index c88b86457b0a..8c64459a1436 100644 --- a/python/ray/util/client_connect.py +++ b/python/ray/util/client_connect.py @@ -1,14 +1,14 @@ -from typing import Any, Dict, List, Optional, Tuple import logging +from typing import Any, Dict, List, Optional, Tuple from ray._private.client_mode_hook import ( _explicitly_enable_client_mode, _set_client_hook_status, ) +from ray._private.utils import get_ray_doc_version from ray.job_config import JobConfig from ray.util.annotations import Deprecated from ray.util.client import ray -from ray._private.utils import get_ray_doc_version logger = logging.getLogger(__name__) diff --git a/python/ray/util/collective/__init__.py b/python/ray/util/collective/__init__.py index ad7bcde93e58..09423ad37c11 100644 --- a/python/ray/util/collective/__init__.py +++ b/python/ray/util/collective/__init__.py @@ -1,28 +1,28 @@ from ray.util.collective.collective import ( - nccl_available, - gloo_available, - is_group_initialized, - init_collective_group, - destroy_collective_group, - create_collective_group, - get_rank, - get_collective_group_size, + allgather, + allgather_multigpu, allreduce, allreduce_multigpu, barrier, - reduce, - reduce_multigpu, broadcast, broadcast_multigpu, - allgather, - allgather_multigpu, + create_collective_group, + destroy_collective_group, + get_collective_group_size, + get_group_handle, + get_rank, + gloo_available, + init_collective_group, + is_group_initialized, + nccl_available, + recv, + recv_multigpu, + reduce, + reduce_multigpu, reducescatter, reducescatter_multigpu, send, send_multigpu, - recv, - recv_multigpu, - get_group_handle, ) __all__ = [ diff --git a/python/ray/util/collective/collective_group/base_collective_group.py b/python/ray/util/collective/collective_group/base_collective_group.py index cfb6ebfa8725..eff07fb16c67 100644 --- a/python/ray/util/collective/collective_group/base_collective_group.py +++ b/python/ray/util/collective/collective_group/base_collective_group.py @@ -1,16 +1,15 @@ """Abstract class for collective groups.""" -from abc import ABCMeta -from abc import abstractmethod +from abc import ABCMeta, abstractmethod from ray.util.collective.types import ( + AllGatherOptions, AllReduceOptions, BarrierOptions, - ReduceOptions, - AllGatherOptions, BroadcastOptions, + RecvOptions, + ReduceOptions, ReduceScatterOptions, SendOptions, - RecvOptions, ) diff --git a/python/ray/util/collective/collective_group/cuda_stream.py b/python/ray/util/collective/collective_group/cuda_stream.py index d5496755f82b..dbccb00c1a17 100644 --- a/python/ray/util/collective/collective_group/cuda_stream.py +++ b/python/ray/util/collective/collective_group/cuda_stream.py @@ -2,6 +2,7 @@ import threading import cupy + from ray.util.collective.collective_group import nccl_util from ray.util.collective.const import ENV diff --git a/python/ray/util/collective/collective_group/gloo_collective_group.py b/python/ray/util/collective/collective_group/gloo_collective_group.py index 6782a8e38f72..5809c12b44b3 100644 --- a/python/ray/util/collective/collective_group/gloo_collective_group.py +++ b/python/ray/util/collective/collective_group/gloo_collective_group.py @@ -8,11 +8,11 @@ import pygloo import ray +from ray._common.network_utils import parse_address from ray._private import ray_constants from ray.util.collective.collective_group import gloo_util from ray.util.collective.collective_group.base_collective_group import BaseGroup from ray.util.collective.const import get_store_name -from ray._common.network_utils import parse_address from ray.util.collective.types import ( AllGatherOptions, AllReduceOptions, diff --git a/python/ray/util/collective/collective_group/nccl_collective_group.py b/python/ray/util/collective/collective_group/nccl_collective_group.py index 9c21b936d898..5f7de1cbc318 100644 --- a/python/ray/util/collective/collective_group/nccl_collective_group.py +++ b/python/ray/util/collective/collective_group/nccl_collective_group.py @@ -1,27 +1,26 @@ -import logging import datetime +import logging import time -import ray import cupy -from ray.util.collective.const import ENV +import ray from ray.util.collective.collective_group import nccl_util from ray.util.collective.collective_group.base_collective_group import BaseGroup -from ray.util.collective.const import get_store_name +from ray.util.collective.collective_group.cuda_stream import get_stream_pool +from ray.util.collective.const import ENV, get_store_name from ray.util.collective.types import ( + AllGatherOptions, AllReduceOptions, - BarrierOptions, Backend, - ReduceOptions, + BarrierOptions, BroadcastOptions, - AllGatherOptions, + RecvOptions, + ReduceOptions, ReduceScatterOptions, SendOptions, - RecvOptions, torch_available, ) -from ray.util.collective.collective_group.cuda_stream import get_stream_pool logger = logging.getLogger(__name__) diff --git a/python/ray/util/collective/collective_group/nccl_util.py b/python/ray/util/collective/collective_group/nccl_util.py index 221d5885c411..7f68d8430208 100644 --- a/python/ray/util/collective/collective_group/nccl_util.py +++ b/python/ray/util/collective/collective_group/nccl_util.py @@ -3,13 +3,17 @@ try: import cupy - from cupy.cuda import nccl - from cupy.cuda import Device # noqa: F401 - from cupy.cuda.nccl import get_version - from cupy.cuda.nccl import get_build_version - from cupy.cuda.nccl import NcclCommunicator - from cupy.cuda.nccl import groupStart # noqa: F401 - from cupy.cuda.nccl import groupEnd # noqa: F401 + from cupy.cuda import ( + Device, # noqa: F401 + nccl, + ) + from cupy.cuda.nccl import ( + NcclCommunicator, + get_build_version, + get_version, + groupEnd, # noqa: F401 + groupStart, # noqa: F401 + ) except ImportError: raise ImportError("NCCL in Ray requires Cupy being available!") diff --git a/python/ray/util/collective/collective_group/nixl_backend.py b/python/ray/util/collective/collective_group/nixl_backend.py index 4861a4818301..ea7fd3d7ab22 100644 --- a/python/ray/util/collective/collective_group/nixl_backend.py +++ b/python/ray/util/collective/collective_group/nixl_backend.py @@ -1,8 +1,10 @@ +import time +from typing import TYPE_CHECKING, List, Tuple + from nixl._api import nixl_agent, nixl_agent_config + import ray from ray.util.collective.types import Backend -from typing import TYPE_CHECKING, List, Tuple -import time if TYPE_CHECKING: import torch diff --git a/python/ray/util/collective/collective_group/torch_gloo_collective_group.py b/python/ray/util/collective/collective_group/torch_gloo_collective_group.py index 5ec743c673f6..51e7f6482b6f 100644 --- a/python/ray/util/collective/collective_group/torch_gloo_collective_group.py +++ b/python/ray/util/collective/collective_group/torch_gloo_collective_group.py @@ -1,22 +1,23 @@ -from typing import TYPE_CHECKING, List, Optional import os +from typing import TYPE_CHECKING, List, Optional + import torch import torch.distributed as dist import ray.experimental.internal_kv as internal_kv -from ray.util.collective.collective_group.base_collective_group import BaseGroup from ray._common.network_utils import parse_address +from ray.util.collective.collective_group.base_collective_group import BaseGroup from ray.util.collective.types import ( + AllGatherOptions, AllReduceOptions, - BarrierOptions, Backend, + BarrierOptions, + BroadcastOptions, + RecvOptions, ReduceOp, ReduceOptions, - BroadcastOptions, - AllGatherOptions, ReduceScatterOptions, SendOptions, - RecvOptions, ) if TYPE_CHECKING: diff --git a/python/ray/util/collective/examples/nccl_allreduce_example.py b/python/ray/util/collective/examples/nccl_allreduce_example.py index dd8a9f83d171..ec812843a3f8 100644 --- a/python/ray/util/collective/examples/nccl_allreduce_example.py +++ b/python/ray/util/collective/examples/nccl_allreduce_example.py @@ -1,6 +1,6 @@ -import ray import cupy as cp +import ray import ray.util.collective as collective diff --git a/python/ray/util/collective/examples/nccl_allreduce_example_declare_collective_group.py b/python/ray/util/collective/examples/nccl_allreduce_example_declare_collective_group.py index 276843ff6da9..df378785dffb 100644 --- a/python/ray/util/collective/examples/nccl_allreduce_example_declare_collective_group.py +++ b/python/ray/util/collective/examples/nccl_allreduce_example_declare_collective_group.py @@ -1,6 +1,6 @@ import cupy as cp -import ray +import ray import ray.util.collective as collective diff --git a/python/ray/util/collective/examples/nccl_allreduce_multigpu_example.py b/python/ray/util/collective/examples/nccl_allreduce_multigpu_example.py index 89282811a4e7..5a70976ae5ab 100644 --- a/python/ray/util/collective/examples/nccl_allreduce_multigpu_example.py +++ b/python/ray/util/collective/examples/nccl_allreduce_multigpu_example.py @@ -1,8 +1,8 @@ -import ray import cupy as cp +from cupy.cuda import Device +import ray import ray.util.collective as collective -from cupy.cuda import Device @ray.remote(num_gpus=2) diff --git a/python/ray/util/collective/examples/nccl_p2p_example_multigpu.py b/python/ray/util/collective/examples/nccl_p2p_example_multigpu.py index 10fe07928f67..1ef3e26ee428 100644 --- a/python/ray/util/collective/examples/nccl_p2p_example_multigpu.py +++ b/python/ray/util/collective/examples/nccl_p2p_example_multigpu.py @@ -1,8 +1,8 @@ -import ray import cupy as cp +from cupy.cuda import Device +import ray import ray.util.collective as collective -from cupy.cuda import Device @ray.remote(num_gpus=2) diff --git a/python/ray/util/collective/tests/conftest.py b/python/ray/util/collective/tests/conftest.py index 0c8fef090184..e4ec1df88675 100644 --- a/python/ray/util/collective/tests/conftest.py +++ b/python/ray/util/collective/tests/conftest.py @@ -2,6 +2,7 @@ import logging import pytest + import ray from ray.util.collective.collective_group.nccl_collective_group import ( _get_comm_key_from_devices, diff --git a/python/ray/util/collective/tests/cpu_util.py b/python/ray/util/collective/tests/cpu_util.py index f4951900dd20..1196afd86fad 100644 --- a/python/ray/util/collective/tests/cpu_util.py +++ b/python/ray/util/collective/tests/cpu_util.py @@ -1,12 +1,12 @@ -import numpy as np import logging +import numpy as np +import torch + import ray import ray.util.collective as col from ray.util.collective.types import Backend, ReduceOp -import torch - logger = logging.getLogger(__name__) diff --git a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_allgather.py b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_allgather.py index bdf32432f0ab..f48a41604405 100644 --- a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_allgather.py +++ b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_allgather.py @@ -1,15 +1,14 @@ """Test the allgather API on a distributed Ray cluster.""" -import pytest -import ray - import numpy as np +import pytest import torch -from ray.util.collective.types import Backend +import ray from ray.util.collective.tests.cpu_util import ( create_collective_workers, init_tensors_for_gather_scatter, ) +from ray.util.collective.types import Backend @pytest.mark.parametrize("backend", [Backend.GLOO]) @@ -137,7 +136,8 @@ def test_allgather_torch_numpy(ray_start_distributed_2_nodes, backend): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_allreduce.py b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_allreduce.py index 43be7b620fc0..d9d6df92f68c 100644 --- a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_allreduce.py +++ b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_allreduce.py @@ -1,13 +1,11 @@ """Test the collective allreduice API on a distributed Ray cluster.""" -import pytest -import ray -from ray.util.collective.types import ReduceOp - import numpy as np +import pytest import torch -from ray.util.collective.types import Backend +import ray from ray.util.collective.tests.cpu_util import create_collective_workers +from ray.util.collective.types import Backend, ReduceOp @pytest.mark.parametrize("backend", [Backend.GLOO]) @@ -174,7 +172,8 @@ def test_allreduce_torch_numpy(ray_start_distributed_2_nodes, backend): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_basic_apis.py b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_basic_apis.py index 1824cda807af..774a70f0a36b 100644 --- a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_basic_apis.py +++ b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_basic_apis.py @@ -1,10 +1,11 @@ """Test the collective group APIs.""" -import pytest -import ray from random import shuffle -from ray.util.collective.types import Backend +import pytest + +import ray from ray.util.collective.tests.cpu_util import Worker, create_collective_workers +from ray.util.collective.types import Backend @pytest.mark.parametrize("backend", [Backend.GLOO]) @@ -130,7 +131,8 @@ def test_destroy_group(ray_start_distributed_2_nodes, backend): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_broadcast.py b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_broadcast.py index d344d1894e8f..b00b92edf3ac 100644 --- a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_broadcast.py +++ b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_broadcast.py @@ -1,10 +1,10 @@ """Test the broadcast API.""" -import pytest import numpy as np -import ray +import pytest -from ray.util.collective.types import Backend +import ray from ray.util.collective.tests.cpu_util import create_collective_workers +from ray.util.collective.types import Backend @pytest.mark.parametrize("backend", [Backend.GLOO]) @@ -89,7 +89,8 @@ def test_broadcast_invalid_rank(ray_start_distributed_2_nodes, backend, src_rank if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_reduce.py b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_reduce.py index 901e773ca757..2df1d27b1e2c 100644 --- a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_reduce.py +++ b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_reduce.py @@ -1,10 +1,10 @@ """Test the reduce API.""" -import pytest import numpy as np -import ray -from ray.util.collective.types import Backend, ReduceOp +import pytest +import ray from ray.util.collective.tests.cpu_util import create_collective_workers +from ray.util.collective.types import Backend, ReduceOp @pytest.mark.parametrize("backend", [Backend.GLOO]) @@ -140,7 +140,8 @@ def test_reduce_invalid_rank(ray_start_distributed_2_nodes, backend, dst_rank=9) if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_reducescatter.py b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_reducescatter.py index fb5d37556fae..47d05b6965ae 100644 --- a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_reducescatter.py +++ b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_reducescatter.py @@ -1,15 +1,14 @@ """Test the collective reducescatter API on a distributed Ray cluster.""" -import pytest -import ray - import numpy as np +import pytest import torch -from ray.util.collective.types import Backend +import ray from ray.util.collective.tests.cpu_util import ( create_collective_workers, init_tensors_for_gather_scatter, ) +from ray.util.collective.types import Backend @pytest.mark.parametrize("backend", [Backend.GLOO]) @@ -125,7 +124,8 @@ def test_reducescatter_torch_numpy(ray_start_distributed_2_nodes, backend): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_sendrecv.py b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_sendrecv.py index 4d2285fcae7e..68aadc067adf 100644 --- a/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_sendrecv.py +++ b/python/ray/util/collective/tests/distributed_cpu_tests/test_distributed_sendrecv.py @@ -1,10 +1,10 @@ """Test the send/recv API.""" import numpy as np import pytest -import ray -from ray.util.collective.types import Backend +import ray from ray.util.collective.tests.cpu_util import create_collective_workers +from ray.util.collective.types import Backend @pytest.mark.parametrize("backend", [Backend.GLOO]) @@ -45,7 +45,8 @@ def test_sendrecv( if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_allgather.py b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_allgather.py index 6bdac60833b7..82afc324af49 100644 --- a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_allgather.py +++ b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_allgather.py @@ -1,10 +1,9 @@ """Test the allgather API on a distributed Ray cluster.""" -import pytest -import ray - import cupy as cp +import pytest import torch +import ray from ray.util.collective.tests.util import ( create_collective_workers, init_tensors_for_gather_scatter, @@ -132,7 +131,8 @@ def test_allgather_torch_cupy(ray_start_distributed_2_nodes_4_gpus): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_allreduce.py b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_allreduce.py index 580b6436e73c..f915db200851 100644 --- a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_allreduce.py +++ b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_allreduce.py @@ -1,12 +1,11 @@ """Test the collective allreduice API on a distributed Ray cluster.""" -import pytest -import ray -from ray.util.collective.types import ReduceOp - import cupy as cp +import pytest import torch +import ray from ray.util.collective.tests.util import create_collective_workers +from ray.util.collective.types import ReduceOp @pytest.mark.parametrize("group_name", ["default", "test", "123?34!"]) diff --git a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_basic_apis.py b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_basic_apis.py index ef61d7450611..bcd7b8c3808b 100644 --- a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_basic_apis.py +++ b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_basic_apis.py @@ -1,8 +1,9 @@ """Test the collective group APIs.""" -import pytest -import ray from random import shuffle +import pytest + +import ray from ray.util.collective.tests.util import Worker, create_collective_workers @@ -114,7 +115,8 @@ def test_destroy_group(ray_start_distributed_2_nodes_4_gpus): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_broadcast.py b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_broadcast.py index 4a8b9779d085..ad5055a7c826 100644 --- a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_broadcast.py +++ b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_broadcast.py @@ -1,8 +1,8 @@ """Test the broadcast API.""" -import pytest import cupy as cp -import ray +import pytest +import ray from ray.util.collective.tests.util import create_collective_workers diff --git a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_reduce.py b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_reduce.py index f7e68b85e1da..969647e78d7d 100644 --- a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_reduce.py +++ b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_reduce.py @@ -1,10 +1,10 @@ """Test the reduce API.""" -import pytest import cupy as cp -import ray -from ray.util.collective.types import ReduceOp +import pytest +import ray from ray.util.collective.tests.util import create_collective_workers +from ray.util.collective.types import ReduceOp @pytest.mark.parametrize("group_name", ["default", "test", "123?34!"]) diff --git a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_reducescatter.py b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_reducescatter.py index ea200f861416..99f7beb6d526 100644 --- a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_reducescatter.py +++ b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_reducescatter.py @@ -1,10 +1,9 @@ """Test the collective reducescatter API on a distributed Ray cluster.""" -import pytest -import ray - import cupy as cp +import pytest import torch +import ray from ray.util.collective.tests.util import ( create_collective_workers, init_tensors_for_gather_scatter, @@ -124,7 +123,8 @@ def test_reducescatter_torch_cupy(ray_start_distributed_2_nodes_4_gpus): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_sendrecv.py b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_sendrecv.py index 692159d223f9..9fb20cf06287 100644 --- a/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_sendrecv.py +++ b/python/ray/util/collective/tests/distributed_gpu_tests/test_distributed_sendrecv.py @@ -1,8 +1,8 @@ """Test the send/recv API.""" import cupy as cp import pytest -import ray +import ray from ray.util.collective.tests.util import create_collective_workers diff --git a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_allgather.py b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_allgather.py index 74ea2ebc11df..dea31ff53953 100644 --- a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_allgather.py +++ b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_allgather.py @@ -1,10 +1,9 @@ """Test the allgather API on a distributed Ray cluster.""" -import pytest -import ray - import cupy as cp +import pytest import torch +import ray from ray.util.collective.tests.util import ( create_collective_multigpu_workers, init_tensors_for_gather_scatter_multigpu, @@ -81,7 +80,8 @@ def test_allgather_torch_cupy(ray_start_distributed_multigpu_2_nodes_4_gpus): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_allreduce.py b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_allreduce.py index 1616e1c2e9d3..aa34cc4a6efb 100644 --- a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_allreduce.py +++ b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_allreduce.py @@ -1,12 +1,12 @@ """Test the collective allreduice API on a distributed Ray cluster.""" -import pytest import logging import cupy as cp +import pytest import ray -from ray.util.collective.types import ReduceOp from ray.util.collective.tests.util import create_collective_multigpu_workers +from ray.util.collective.types import ReduceOp logger = logging.getLogger(__name__) logger.setLevel("DEBUG") diff --git a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_basic_apis.py b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_basic_apis.py index ed6ad137d384..4b0c861f039d 100644 --- a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_basic_apis.py +++ b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_basic_apis.py @@ -1,8 +1,9 @@ """Test the collective group APIs.""" -import pytest -import ray from random import shuffle +import pytest + +import ray from ray.util.collective.tests.util import create_collective_multigpu_workers @@ -95,7 +96,8 @@ def test_destroy_group(ray_start_distributed_multigpu_2_nodes_4_gpus): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_broadcast.py b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_broadcast.py index 3b90c2568cb9..8cd52a962f5f 100644 --- a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_broadcast.py +++ b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_broadcast.py @@ -1,8 +1,8 @@ """Test the broadcast API.""" -import pytest import cupy as cp -import ray +import pytest +import ray from ray.util.collective.tests.util import create_collective_multigpu_workers diff --git a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_reduce.py b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_reduce.py index c584806eedc2..4a15fc4c40df 100644 --- a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_reduce.py +++ b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_reduce.py @@ -1,10 +1,10 @@ """Test the reduce API.""" -import pytest import cupy as cp -import ray -from ray.util.collective.types import ReduceOp +import pytest +import ray from ray.util.collective.tests.util import create_collective_multigpu_workers +from ray.util.collective.types import ReduceOp @pytest.mark.parametrize("group_name", ["default", "test", "123?34!"]) diff --git a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_reducescatter.py b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_reducescatter.py index 67a2b8b738a8..98cd51360ae4 100644 --- a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_reducescatter.py +++ b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_reducescatter.py @@ -1,10 +1,9 @@ """Test the collective reducescatter API on a distributed Ray cluster.""" -import pytest -import ray - import cupy as cp +import pytest import torch +import ray from ray.util.collective.tests.util import ( create_collective_multigpu_workers, init_tensors_for_gather_scatter_multigpu, @@ -84,7 +83,8 @@ def test_reducescatter_torch_cupy(ray_start_distributed_multigpu_2_nodes_4_gpus) if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_sendrecv.py b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_sendrecv.py index c7371343ba56..0fa18ddaf390 100644 --- a/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_sendrecv.py +++ b/python/ray/util/collective/tests/distributed_multigpu_tests/test_distributed_multigpu_sendrecv.py @@ -1,8 +1,8 @@ """Test the send/recv API.""" import cupy as cp import pytest -import ray +import ray from ray.util.collective.tests.util import create_collective_multigpu_workers diff --git a/python/ray/util/collective/tests/single_node_cpu_tests/test_allgather.py b/python/ray/util/collective/tests/single_node_cpu_tests/test_allgather.py index 67d9ddb01e9b..70026b88ddaa 100644 --- a/python/ray/util/collective/tests/single_node_cpu_tests/test_allgather.py +++ b/python/ray/util/collective/tests/single_node_cpu_tests/test_allgather.py @@ -1,8 +1,9 @@ """Test the collective allgather API.""" import numpy as np import pytest -import ray import torch + +import ray from ray.util.collective.tests.cpu_util import ( create_collective_workers, init_tensors_for_gather_scatter, @@ -135,7 +136,8 @@ def test_allgather_torch_numpy(ray_start_single_node, backend): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_cpu_tests/test_allreduce.py b/python/ray/util/collective/tests/single_node_cpu_tests/test_allreduce.py index 22ebcfeb6e1b..4791ed2ea388 100644 --- a/python/ray/util/collective/tests/single_node_cpu_tests/test_allreduce.py +++ b/python/ray/util/collective/tests/single_node_cpu_tests/test_allreduce.py @@ -1,8 +1,9 @@ """Test the collective allreduice API.""" import numpy as np import pytest -import ray import torch + +import ray from ray.util.collective.tests.cpu_util import create_collective_workers from ray.util.collective.types import Backend, ReduceOp @@ -158,7 +159,8 @@ def test_allreduce_torch_numpy(ray_start_single_node, backend): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_cpu_tests/test_basic_apis.py b/python/ray/util/collective/tests/single_node_cpu_tests/test_basic_apis.py index f8bd8dff63b3..0701f40f4eb5 100644 --- a/python/ray/util/collective/tests/single_node_cpu_tests/test_basic_apis.py +++ b/python/ray/util/collective/tests/single_node_cpu_tests/test_basic_apis.py @@ -1,9 +1,9 @@ """Test the collective group APIs.""" import pytest -import ray -from ray.util.collective.types import Backend +import ray from ray.util.collective.tests.cpu_util import Worker, create_collective_workers +from ray.util.collective.types import Backend @pytest.mark.parametrize("backend", [Backend.GLOO]) @@ -123,7 +123,8 @@ def test_destroy_group(ray_start_single_node, backend): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_cpu_tests/test_broadcast.py b/python/ray/util/collective/tests/single_node_cpu_tests/test_broadcast.py index f785c450c142..263f832ee280 100644 --- a/python/ray/util/collective/tests/single_node_cpu_tests/test_broadcast.py +++ b/python/ray/util/collective/tests/single_node_cpu_tests/test_broadcast.py @@ -1,8 +1,8 @@ """Test the broadcast API.""" -import pytest import numpy as np -import ray +import pytest +import ray from ray.util.collective.tests.cpu_util import create_collective_workers from ray.util.collective.types import Backend @@ -87,7 +87,8 @@ def test_broadcast_invalid_rank(ray_start_single_node, backend, src_rank=3): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_cpu_tests/test_gloo_group_isolation.py b/python/ray/util/collective/tests/single_node_cpu_tests/test_gloo_group_isolation.py index bc41e341bcc6..7d0d4888aca0 100644 --- a/python/ray/util/collective/tests/single_node_cpu_tests/test_gloo_group_isolation.py +++ b/python/ray/util/collective/tests/single_node_cpu_tests/test_gloo_group_isolation.py @@ -1,8 +1,10 @@ -from python.ray.util.collective.types import Backend +import time + from python.ray.util.collective.collective_group.gloo_collective_group import GLOOGroup +from python.ray.util.collective.types import Backend + import ray import ray.util.collective as col -import time @ray.remote @@ -57,7 +59,8 @@ def test_failure_when_initializing(shutdown_only): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_cpu_tests/test_reduce.py b/python/ray/util/collective/tests/single_node_cpu_tests/test_reduce.py index d7977b2c32e6..4a125b24b82a 100644 --- a/python/ray/util/collective/tests/single_node_cpu_tests/test_reduce.py +++ b/python/ray/util/collective/tests/single_node_cpu_tests/test_reduce.py @@ -1,10 +1,10 @@ """Test the reduce API.""" -import pytest import numpy as np -import ray -from ray.util.collective.types import Backend, ReduceOp +import pytest +import ray from ray.util.collective.tests.cpu_util import create_collective_workers +from ray.util.collective.types import Backend, ReduceOp @pytest.mark.parametrize("backend", [Backend.GLOO]) @@ -160,7 +160,8 @@ def test_reduce_invalid_rank(ray_start_single_node, backend, dst_rank=3): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_cpu_tests/test_reducescatter.py b/python/ray/util/collective/tests/single_node_cpu_tests/test_reducescatter.py index 245c84ed9e8a..22d0e56da733 100644 --- a/python/ray/util/collective/tests/single_node_cpu_tests/test_reducescatter.py +++ b/python/ray/util/collective/tests/single_node_cpu_tests/test_reducescatter.py @@ -1,15 +1,14 @@ """Test the collective reducescatter API.""" -import pytest -import ray - import numpy as np +import pytest import torch -from ray.util.collective.types import Backend +import ray from ray.util.collective.tests.cpu_util import ( create_collective_workers, init_tensors_for_gather_scatter, ) +from ray.util.collective.types import Backend @pytest.mark.parametrize("backend", [Backend.GLOO]) @@ -125,7 +124,8 @@ def test_reducescatter_torch_numpy(ray_start_single_node, backend): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_cpu_tests/test_sendrecv.py b/python/ray/util/collective/tests/single_node_cpu_tests/test_sendrecv.py index aae3440b7cde..e4bd841d7a10 100644 --- a/python/ray/util/collective/tests/single_node_cpu_tests/test_sendrecv.py +++ b/python/ray/util/collective/tests/single_node_cpu_tests/test_sendrecv.py @@ -1,10 +1,10 @@ """Test the send/recv API.""" -import pytest import numpy as np -import ray +import pytest -from ray.util.collective.types import Backend +import ray from ray.util.collective.tests.cpu_util import create_collective_workers +from ray.util.collective.types import Backend @pytest.mark.parametrize("backend", [Backend.GLOO]) @@ -85,7 +85,8 @@ def test_sendrecv_invalid_rank(ray_start_single_node, backend, dst_rank=3): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_gpu_tests/test_allgather.py b/python/ray/util/collective/tests/single_node_gpu_tests/test_allgather.py index eee8d48313f8..e7f78e6ac6a0 100644 --- a/python/ray/util/collective/tests/single_node_gpu_tests/test_allgather.py +++ b/python/ray/util/collective/tests/single_node_gpu_tests/test_allgather.py @@ -1,10 +1,9 @@ """Test the collective allgather API.""" -import pytest -import ray - import cupy as cp +import pytest import torch +import ray from ray.util.collective.tests.util import ( create_collective_workers, init_tensors_for_gather_scatter, @@ -132,7 +131,8 @@ def test_allgather_torch_cupy(ray_start_single_node_2_gpus): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_gpu_tests/test_allreduce.py b/python/ray/util/collective/tests/single_node_gpu_tests/test_allreduce.py index 0acab8c73077..1894adfc295d 100644 --- a/python/ray/util/collective/tests/single_node_gpu_tests/test_allreduce.py +++ b/python/ray/util/collective/tests/single_node_gpu_tests/test_allreduce.py @@ -1,8 +1,9 @@ """Test the collective allreduice API.""" import cupy as cp import pytest -import ray import torch + +import ray from ray.util.collective.tests.util import create_collective_workers from ray.util.collective.types import ReduceOp @@ -162,7 +163,8 @@ def test_allreduce_torch_cupy(ray_start_single_node_2_gpus): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_gpu_tests/test_basic_apis.py b/python/ray/util/collective/tests/single_node_gpu_tests/test_basic_apis.py index 00136b7a8523..892b13288689 100644 --- a/python/ray/util/collective/tests/single_node_gpu_tests/test_basic_apis.py +++ b/python/ray/util/collective/tests/single_node_gpu_tests/test_basic_apis.py @@ -1,7 +1,7 @@ """Test the collective group APIs.""" import pytest -import ray +import ray from ray.util.collective.tests.util import Worker, create_collective_workers @@ -111,7 +111,8 @@ def test_destroy_group(ray_start_single_node_2_gpus): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_gpu_tests/test_broadcast.py b/python/ray/util/collective/tests/single_node_gpu_tests/test_broadcast.py index e00f355053e9..85623ebdfa34 100644 --- a/python/ray/util/collective/tests/single_node_gpu_tests/test_broadcast.py +++ b/python/ray/util/collective/tests/single_node_gpu_tests/test_broadcast.py @@ -1,8 +1,8 @@ """Test the broadcast API.""" -import pytest import cupy as cp -import ray +import pytest +import ray from ray.util.collective.tests.util import create_collective_workers diff --git a/python/ray/util/collective/tests/single_node_gpu_tests/test_reduce.py b/python/ray/util/collective/tests/single_node_gpu_tests/test_reduce.py index 17fb446c871d..2439c30726d7 100644 --- a/python/ray/util/collective/tests/single_node_gpu_tests/test_reduce.py +++ b/python/ray/util/collective/tests/single_node_gpu_tests/test_reduce.py @@ -1,10 +1,10 @@ """Test the reduce API.""" -import pytest import cupy as cp -import ray -from ray.util.collective.types import ReduceOp +import pytest +import ray from ray.util.collective.tests.util import create_collective_workers +from ray.util.collective.types import ReduceOp @pytest.mark.parametrize("group_name", ["default", "test", "123?34!"]) diff --git a/python/ray/util/collective/tests/single_node_gpu_tests/test_reducescatter.py b/python/ray/util/collective/tests/single_node_gpu_tests/test_reducescatter.py index 122ef1a1faef..83c64f948fb4 100644 --- a/python/ray/util/collective/tests/single_node_gpu_tests/test_reducescatter.py +++ b/python/ray/util/collective/tests/single_node_gpu_tests/test_reducescatter.py @@ -1,10 +1,9 @@ """Test the collective reducescatter API.""" -import pytest -import ray - import cupy as cp +import pytest import torch +import ray from ray.util.collective.tests.util import ( create_collective_workers, init_tensors_for_gather_scatter, @@ -124,7 +123,8 @@ def test_reducescatter_torch_cupy(ray_start_single_node_2_gpus): if __name__ == "__main__": - import pytest import sys + import pytest + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/util/collective/tests/single_node_gpu_tests/test_sendrecv.py b/python/ray/util/collective/tests/single_node_gpu_tests/test_sendrecv.py index 2f79f1fb25b9..cce862ca230b 100644 --- a/python/ray/util/collective/tests/single_node_gpu_tests/test_sendrecv.py +++ b/python/ray/util/collective/tests/single_node_gpu_tests/test_sendrecv.py @@ -1,8 +1,8 @@ """Test the send/recv API.""" -import pytest import cupy as cp -import ray +import pytest +import ray from ray.util.collective.tests.util import create_collective_workers diff --git a/python/ray/util/collective/tests/util.py b/python/ray/util/collective/tests/util.py index 69eac6438224..e3dfd63adc54 100644 --- a/python/ray/util/collective/tests/util.py +++ b/python/ray/util/collective/tests/util.py @@ -1,12 +1,12 @@ -import cupy as cp import logging +import cupy as cp +import torch + import ray import ray.util.collective as col -from ray.util.collective.types import Backend, ReduceOp from ray.util.collective.collective_group.nccl_util import get_num_gpus - -import torch +from ray.util.collective.types import Backend, ReduceOp logger = logging.getLogger(__name__) diff --git a/python/ray/util/collective/types.py b/python/ray/util/collective/types.py index e46737c5a033..90ed5d6d86bd 100644 --- a/python/ray/util/collective/types.py +++ b/python/ray/util/collective/types.py @@ -1,9 +1,9 @@ """Types conversion between different backends.""" -from enum import Enum from dataclasses import dataclass from datetime import timedelta -from typing import List, Tuple, TYPE_CHECKING, Optional +from enum import Enum +from typing import TYPE_CHECKING, List, Optional, Tuple from numpy import int32 diff --git a/python/ray/util/collective/util.py b/python/ray/util/collective/util.py index 6acabf82de3e..84257fcbbfeb 100644 --- a/python/ray/util/collective/util.py +++ b/python/ray/util/collective/util.py @@ -1,7 +1,8 @@ """Some utility class for Collectives.""" -import ray import logging +import ray + logger = logging.getLogger(__name__) diff --git a/python/ray/util/dask/__init__.py b/python/ray/util/dask/__init__.py index 3376b5f8eaca..f9e4ac0cb1af 100644 --- a/python/ray/util/dask/__init__.py +++ b/python/ray/util/dask/__init__.py @@ -11,19 +11,19 @@ "Please upgrade your Dask installation." ) -from .scheduler import ( - ray_dask_get, - ray_dask_get_sync, - enable_dask_on_ray, - disable_dask_on_ray, -) from .callbacks import ( + ProgressBarCallback, RayDaskCallback, local_ray_callbacks, unpack_ray_callbacks, - ProgressBarCallback, ) from .optimizations import dataframe_optimize +from .scheduler import ( + disable_dask_on_ray, + enable_dask_on_ray, + ray_dask_get, + ray_dask_get_sync, +) dask_persist = dask.persist diff --git a/python/ray/util/dask/callbacks.py b/python/ray/util/dask/callbacks.py index 82c6ca1cf717..770d2208b504 100644 --- a/python/ray/util/dask/callbacks.py +++ b/python/ray/util/dask/callbacks.py @@ -1,6 +1,5 @@ import contextlib - -from collections import namedtuple, defaultdict +from collections import defaultdict, namedtuple from datetime import datetime from typing import Any, List, Optional diff --git a/python/ray/util/dask/common.py b/python/ray/util/dask/common.py index b041b7ff4676..47ec12d79a1b 100644 --- a/python/ray/util/dask/common.py +++ b/python/ray/util/dask/common.py @@ -1,16 +1,15 @@ +import uuid from collections import OrderedDict from collections.abc import Iterator from operator import getitem -import uuid - -import ray -from dask.core import quote -from dask.core import get as get_sync +from dask.core import get as get_sync, quote from dask.utils import apply +import ray + try: - from dataclasses import is_dataclass, fields as dataclass_fields + from dataclasses import fields as dataclass_fields, is_dataclass except ImportError: # Python < 3.7 def is_dataclass(x): diff --git a/python/ray/util/dask/optimizations.py b/python/ray/util/dask/optimizations.py index 096a6096d48f..e88416774c6d 100644 --- a/python/ray/util/dask/optimizations.py +++ b/python/ray/util/dask/optimizations.py @@ -8,9 +8,8 @@ from .scheduler import MultipleReturnFunc, multiple_return_get try: - from dask.dataframe.shuffle import SimpleShuffleLayer from dask.dataframe.optimize import optimize - from dask.dataframe.shuffle import shuffle_group + from dask.dataframe.shuffle import SimpleShuffleLayer, shuffle_group except ImportError: # SimpleShuffleLayer doesn't exist in this version of Dask. # This is the case for dask>=2025.1.0. diff --git a/python/ray/util/dask/scheduler.py b/python/ray/util/dask/scheduler.py index daa4449e6de3..0fa94706187f 100644 --- a/python/ray/util/dask/scheduler.py +++ b/python/ray/util/dask/scheduler.py @@ -1,10 +1,8 @@ -import warnings - import atexit import threading import time -from collections import defaultdict -from collections import OrderedDict +import warnings +from collections import OrderedDict, defaultdict from collections.abc import Mapping from dataclasses import dataclass from multiprocessing.pool import ThreadPool @@ -12,22 +10,22 @@ from typing import Optional import dask -from dask.core import istask, ishashable +from dask.core import ishashable, istask try: - from dask._task_spec import Task, Alias, DataNode, TaskRef, convert_legacy_graph + from dask._task_spec import Alias, DataNode, Task, TaskRef, convert_legacy_graph except ImportError: warnings.warn( "Dask on Ray is available only on dask>=2024.11.0, " f"you are on version {dask.__version__}." ) from dask.system import CPU_COUNT -from dask.threaded import pack_exception, _thread_get_id +from dask.threaded import _thread_get_id, pack_exception import ray from ray.util.dask.callbacks import local_ray_callbacks, unpack_ray_callbacks from ray.util.dask.common import unpack_object_refs -from ray.util.dask.scheduler_utils import get_async, apply_sync +from ray.util.dask.scheduler_utils import apply_sync, get_async main_thread = threading.current_thread() default_pool = None diff --git a/python/ray/util/dask/scheduler_utils.py b/python/ray/util/dask/scheduler_utils.py index bb7feca4ae8b..b4c840c6b896 100644 --- a/python/ray/util/dask/scheduler_utils.py +++ b/python/ray/util/dask/scheduler_utils.py @@ -5,7 +5,7 @@ import os import warnings -from queue import Queue, Empty +from queue import Empty, Queue import dask from dask import config diff --git a/python/ray/util/dask/tests/test_dask_callback.py b/python/ray/util/dask/tests/test_dask_callback.py index 99c59d791b33..d58c7dc3c130 100644 --- a/python/ray/util/dask/tests/test_dask_callback.py +++ b/python/ray/util/dask/tests/test_dask_callback.py @@ -1,12 +1,11 @@ import sys - import dask import pytest import ray from ray.tests.conftest import * # noqa: F403, F401 -from ray.util.dask import ray_dask_get, RayDaskCallback +from ray.util.dask import RayDaskCallback, ray_dask_get @dask.delayed diff --git a/python/ray/util/dask/tests/test_dask_multi_node.py b/python/ray/util/dask/tests/test_dask_multi_node.py index a3dc5f7effa0..44e8ace38b6e 100644 --- a/python/ray/util/dask/tests/test_dask_multi_node.py +++ b/python/ray/util/dask/tests/test_dask_multi_node.py @@ -1,10 +1,10 @@ import sys import dask -import pytest import dask.dataframe as dd import numpy as np import pandas as pd +import pytest import ray from ray.tests.conftest import * # noqa: F403, F401 diff --git a/python/ray/util/dask/tests/test_dask_optimization.py b/python/ray/util/dask/tests/test_dask_optimization.py index 858f0009a363..b09c62fbca39 100644 --- a/python/ray/util/dask/tests/test_dask_optimization.py +++ b/python/ray/util/dask/tests/test_dask_optimization.py @@ -1,8 +1,8 @@ import sys +from unittest import mock import dask import dask.dataframe as dd -from unittest import mock import numpy as np import pandas as pd import pytest @@ -21,9 +21,10 @@ if Version(dask.__version__) < Version("2025.1") and not DASK_EXPR_INSTALLED: from dask.dataframe.shuffle import SimpleShuffleLayer + from ray.util.dask.optimizations import ( - rewrite_simple_shuffle_layer, MultipleReturnSimpleShuffleLayer, + rewrite_simple_shuffle_layer, ) pytestmark = pytest.mark.skipif( diff --git a/python/ray/util/debug.py b/python/ray/util/debug.py index e5482c7b6d8c..29d9a3e1d497 100644 --- a/python/ray/util/debug.py +++ b/python/ray/util/debug.py @@ -1,10 +1,11 @@ -from collections import defaultdict, namedtuple import gc import os import re import time import tracemalloc +from collections import defaultdict, namedtuple from typing import Callable, List, Optional + from ray.util.annotations import DeveloperAPI _logged = set() @@ -210,8 +211,8 @@ def _take_snapshot(table, suspicious=None): def _find_memory_leaks_in_table(table): - import scipy.stats import numpy as np + import scipy.stats suspects = [] diff --git a/python/ray/util/debugpy.py b/python/ray/util/debugpy.py index 3513f2100fc2..1f5a0157f2b6 100644 --- a/python/ray/util/debugpy.py +++ b/python/ray/util/debugpy.py @@ -1,8 +1,8 @@ +import importlib import logging import os import sys import threading -import importlib import ray from ray._common.network_utils import build_address diff --git a/python/ray/util/helpers.py b/python/ray/util/helpers.py index b53c3c970a74..bfc400f2ffe2 100644 --- a/python/ray/util/helpers.py +++ b/python/ray/util/helpers.py @@ -1,4 +1,5 @@ from typing import TYPE_CHECKING, Any, Iterable, Iterator, Optional, Sequence, Union + import ray from ray.util.annotations import PublicAPI diff --git a/python/ray/util/metrics.py b/python/ray/util/metrics.py index 614dd34d41fa..7942f7f279c6 100644 --- a/python/ray/util/metrics.py +++ b/python/ray/util/metrics.py @@ -1,15 +1,14 @@ import logging +import os import re import warnings -import os - -from typing import Dict, Any, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from ray._raylet import ( Count as CythonCount, - Sum as CythonSum, - Histogram as CythonHistogram, Gauge as CythonGauge, + Histogram as CythonHistogram, + Sum as CythonSum, ) # noqa: E402 # Sum is used for CythonCount because it allows incrementing by positive diff --git a/python/ray/util/multiprocessing/__init__.py b/python/ray/util/multiprocessing/__init__.py index 5b390439f5e1..75c07d911814 100644 --- a/python/ray/util/multiprocessing/__init__.py +++ b/python/ray/util/multiprocessing/__init__.py @@ -1,4 +1,4 @@ -from multiprocessing import TimeoutError, JoinableQueue +from multiprocessing import JoinableQueue, TimeoutError from .pool import Pool diff --git a/python/ray/util/placement_group.py b/python/ray/util/placement_group.py index 286036118934..d2e29b81c536 100644 --- a/python/ray/util/placement_group.py +++ b/python/ray/util/placement_group.py @@ -1,15 +1,15 @@ import warnings from typing import Dict, List, Optional, Union -from ray._common.utils import hex_to_binary, PLACEMENT_GROUP_BUNDLE_RESOURCE_NAME import ray +from ray._common.utils import PLACEMENT_GROUP_BUNDLE_RESOURCE_NAME, hex_to_binary from ray._private.auto_init_hook import auto_init_ray from ray._private.client_mode_hook import client_mode_should_convert, client_mode_wrap +from ray._private.label_utils import validate_label_selector from ray._private.utils import get_ray_doc_version from ray._raylet import PlacementGroupID from ray.util.annotations import DeveloperAPI, PublicAPI from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy -from ray._private.label_utils import validate_label_selector bundle_reservation_check = None diff --git a/python/ray/util/queue.py b/python/ray/util/queue.py index 8bd205f972c9..b18075c801ac 100644 --- a/python/ray/util/queue.py +++ b/python/ray/util/queue.py @@ -1,7 +1,7 @@ import asyncio import queue -from typing import Optional, Any, List, Dict from collections.abc import Iterable +from typing import Any, Dict, List, Optional import ray from ray.util.annotations import PublicAPI diff --git a/python/ray/util/rpdb.py b/python/ray/util/rpdb.py index ae102c96120b..865980ffb2ea 100644 --- a/python/ray/util/rpdb.py +++ b/python/ray/util/rpdb.py @@ -3,7 +3,6 @@ # (BSD 2-Clause "Simplified" License) import errno -from ray._common.network_utils import build_address import inspect import json import logging @@ -19,6 +18,7 @@ from typing import Callable import ray +from ray._common.network_utils import build_address from ray._private import ray_constants from ray.experimental.internal_kv import _internal_kv_del, _internal_kv_put from ray.util.annotations import DeveloperAPI diff --git a/python/ray/util/scheduling_strategies.py b/python/ray/util/scheduling_strategies.py index b9953094a9c1..6f86622a8be3 100644 --- a/python/ray/util/scheduling_strategies.py +++ b/python/ray/util/scheduling_strategies.py @@ -1,4 +1,5 @@ -from typing import Dict, Union, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Optional, Union + from ray.util.annotations import PublicAPI if TYPE_CHECKING: diff --git a/python/ray/util/spark/__init__.py b/python/ray/util/spark/__init__.py index edded13240a1..69d68172eb19 100644 --- a/python/ray/util/spark/__init__.py +++ b/python/ray/util/spark/__init__.py @@ -1,8 +1,8 @@ from ray.util.spark.cluster_init import ( - setup_ray_cluster, - shutdown_ray_cluster, MAX_NUM_WORKER_NODES, setup_global_ray_cluster, + setup_ray_cluster, + shutdown_ray_cluster, ) __all__ = [ diff --git a/python/ray/util/spark/cluster_init.py b/python/ray/util/spark/cluster_init.py index 27649f278a29..fabbd51a9f76 100644 --- a/python/ray/util/spark/cluster_init.py +++ b/python/ray/util/spark/cluster_init.py @@ -1,49 +1,47 @@ import copy -import signal - -import yaml import json +import logging import os +import signal import socket import sys -import time import threading -import logging +import time import uuid import warnings +from threading import Event +from typing import Dict, Optional, Tuple, Type + import requests +import yaml from packaging.version import Version -from typing import Optional, Dict, Tuple, Type import ray import ray._private.services -from ray.autoscaler._private.spark.node_provider import HEAD_NODE_ID -from ray.util.annotations import DeveloperAPI, PublicAPI -from ray._common.utils import load_class -from ray._common.network_utils import build_address, parse_address - +from .databricks_hook import DefaultDatabricksRayOnSparkStartHook +from .start_hook_base import RayOnSparkStartHook from .utils import ( + _get_cpu_cores, + _get_local_ray_node_slots, + _get_num_physical_gpus, + _wait_service_up, + calc_mem_ray_head_node, exec_cmd, - is_port_in_use, + gen_cmd_exec_failure_msg, + get_avail_mem_per_ray_worker_node, + get_configured_spark_executor_memory_bytes, + get_max_num_concurrent_tasks, get_random_unused_port, - get_spark_session, get_spark_application_driver_host, - is_in_databricks_runtime, + get_spark_session, get_spark_task_assigned_physical_gpus, - get_avail_mem_per_ray_worker_node, - get_max_num_concurrent_tasks, - gen_cmd_exec_failure_msg, - calc_mem_ray_head_node, - _wait_service_up, - _get_local_ray_node_slots, - get_configured_spark_executor_memory_bytes, - _get_cpu_cores, - _get_num_physical_gpus, + is_in_databricks_runtime, + is_port_in_use, ) -from .start_hook_base import RayOnSparkStartHook -from .databricks_hook import DefaultDatabricksRayOnSparkStartHook -from threading import Event - +from ray._common.network_utils import build_address, parse_address +from ray._common.utils import load_class +from ray.autoscaler._private.spark.node_provider import HEAD_NODE_ID +from ray.util.annotations import DeveloperAPI, PublicAPI _logger = logging.getLogger("ray.util.spark") _logger.setLevel(logging.INFO) @@ -318,9 +316,10 @@ def _preallocate_ray_worker_port_range(): Returns: Allocated port range for current worker ports """ - import psutil import fcntl + import psutil + def acquire_lock(file_path): mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC try: diff --git a/python/ray/util/spark/databricks_hook.py b/python/ray/util/spark/databricks_hook.py index 8558c309f398..491f35c1419f 100644 --- a/python/ray/util/spark/databricks_hook.py +++ b/python/ray/util/spark/databricks_hook.py @@ -1,10 +1,10 @@ +import logging import os +import threading +import time from .start_hook_base import RayOnSparkStartHook from .utils import get_spark_session -import logging -import threading -import time _logger = logging.getLogger(__name__) diff --git a/python/ray/util/spark/start_ray_node.py b/python/ray/util/spark/start_ray_node.py index 76489b15b9e5..e03c99d74e55 100644 --- a/python/ray/util/spark/start_ray_node.py +++ b/python/ray/util/spark/start_ray_node.py @@ -1,20 +1,19 @@ +import fcntl +import logging import os.path -import subprocess -import sys -import time import shutil -import fcntl import signal import socket -import logging +import subprocess +import sys import threading +import time +from ray._private.ray_process_reaper import SIGTERM_GRACE_PERIOD_SECONDS from ray.util.spark.cluster_init import ( RAY_ON_SPARK_COLLECT_LOG_TO_PATH, RAY_ON_SPARK_START_RAY_PARENT_PID, ) -from ray._private.ray_process_reaper import SIGTERM_GRACE_PERIOD_SECONDS - # Spark on ray implementation does not directly invoke `ray start ...` script to create # ray node subprocess, instead, it creates a subprocess to run this diff --git a/python/ray/util/spark/utils.py b/python/ray/util/spark/utils.py index 65bfa4a52f2b..9aa3465881fe 100644 --- a/python/ray/util/spark/utils.py +++ b/python/ray/util/spark/utils.py @@ -1,14 +1,13 @@ -import subprocess -import os -import sys -import random -import threading import collections import logging +import os +import random import shutil +import subprocess +import sys +import threading import time - _logger = logging.getLogger("ray.util.spark.utils") @@ -199,9 +198,10 @@ def _get_spark_worker_total_shared_memory(): def calc_mem_ray_head_node(configured_heap_memory_bytes, configured_object_store_bytes): - import psutil import shutil + import psutil + if RAY_ON_SPARK_DRIVER_PHYSICAL_MEMORY_BYTES in os.environ: available_physical_mem = int( os.environ[RAY_ON_SPARK_DRIVER_PHYSICAL_MEMORY_BYTES] diff --git a/python/ray/util/state/__init__.py b/python/ray/util/state/__init__.py index d74f9b650df3..b8bb885e5408 100644 --- a/python/ray/util/state/__init__.py +++ b/python/ray/util/state/__init__.py @@ -1,29 +1,28 @@ from ray.util.state.api import ( + StateApiClient, get_actor, + get_job, get_log, get_node, get_objects, get_placement_group, get_task, get_worker, - get_job, list_actors, + list_cluster_events, list_jobs, + list_logs, list_nodes, + list_objects, list_placement_groups, + list_runtime_envs, list_tasks, list_workers, - list_objects, - list_runtime_envs, - list_logs, - list_cluster_events, summarize_actors, summarize_objects, summarize_tasks, - StateApiClient, ) - __all__ = [ "get_actor", "get_log", diff --git a/python/ray/util/state/common.py b/python/ray/util/state/common.py index 18fb6eeafc75..9e4e7000eec3 100644 --- a/python/ray/util/state/common.py +++ b/python/ray/util/state/common.py @@ -9,9 +9,13 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Union import ray.dashboard.utils as dashboard_utils -from ray._private.ray_constants import env_integer -from ray.core.generated.common_pb2 import TaskStatus, TaskType -from ray.core.generated.gcs_pb2 import TaskEvents + +# TODO(aguo): Instead of a version check, modify the below models +# to use pydantic BaseModel instead of dataclass. +# In pydantic 2, dataclass no longer needs the `init=True` kwarg to +# generate an __init__ method. Additionally, it will raise an error if +# it detects `init=True` to be set. +from ray._common.pydantic_compat import IS_PYDANTIC_2 from ray._private.custom_types import ( TypeActorStatus, TypeNodeStatus, @@ -22,15 +26,11 @@ TypeWorkerExitType, TypeWorkerType, ) -from ray.util.state.exception import RayStateApiException +from ray._private.ray_constants import env_integer +from ray.core.generated.common_pb2 import TaskStatus, TaskType +from ray.core.generated.gcs_pb2 import TaskEvents from ray.dashboard.modules.job.pydantic_models import JobDetails - -# TODO(aguo): Instead of a version check, modify the below models -# to use pydantic BaseModel instead of dataclass. -# In pydantic 2, dataclass no longer needs the `init=True` kwarg to -# generate an __init__ method. Additionally, it will raise an error if -# it detects `init=True` to be set. -from ray._common.pydantic_compat import IS_PYDANTIC_2 +from ray.util.state.exception import RayStateApiException try: from pydantic.dataclasses import dataclass diff --git a/python/ray/util/state/state_cli.py b/python/ray/util/state/state_cli.py index b76544d8d316..16ab4f34e2ea 100644 --- a/python/ray/util/state/state_cli.py +++ b/python/ray/util/state/state_cli.py @@ -8,8 +8,9 @@ import yaml import ray._private.services as services -from ray._private.thirdparty.tabulate.tabulate import tabulate from ray._common.network_utils import parse_address +from ray._private.thirdparty.tabulate.tabulate import tabulate +from ray.util.annotations import PublicAPI from ray.util.state import ( StateApiClient, get_log, @@ -31,7 +32,6 @@ resource_to_schema, ) from ray.util.state.exception import RayStateApiException -from ray.util.annotations import PublicAPI logger = logging.getLogger(__name__) diff --git a/python/ray/util/state/state_manager.py b/python/ray/util/state/state_manager.py index 16c570865007..b22ba784e8c2 100644 --- a/python/ray/util/state/state_manager.py +++ b/python/ray/util/state/state_manager.py @@ -1,20 +1,21 @@ import dataclasses import inspect +import json import logging from functools import wraps from typing import List, Optional, Tuple -import json import aiohttp import grpc from grpc.aio._call import UnaryStreamCall import ray -import ray.dashboard.modules.log.log_consts as log_consts import ray.dashboard.consts as dashboard_consts -from ray._private import ray_constants +import ray.dashboard.modules.log.log_consts as log_consts +from ray._common.network_utils import build_address from ray._common.utils import hex_to_binary -from ray._raylet import GcsClient, ActorID, JobID, TaskID, NodeID +from ray._private import ray_constants +from ray._raylet import ActorID, GcsClient, JobID, NodeID, TaskID from ray.core.generated import gcs_service_pb2_grpc from ray.core.generated.gcs_pb2 import ActorTableData, GcsNodeInfo from ray.core.generated.gcs_service_pb2 import ( @@ -54,7 +55,6 @@ SupportedFilterType, ) from ray.util.state.exception import DataSourceUnavailable -from ray._common.network_utils import build_address logger = logging.getLogger(__name__) diff --git a/python/ray/util/state/util.py b/python/ray/util/state/util.py index dd62076bc1dc..77894289d9fb 100644 --- a/python/ray/util/state/util.py +++ b/python/ray/util/state/util.py @@ -49,6 +49,7 @@ def convert_string_to_type( def record_deprecated_state_api_import(): import warnings + from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag warnings.warn( diff --git a/python/ray/util/tracing/setup_local_tmp_tracing.py b/python/ray/util/tracing/setup_local_tmp_tracing.py index f53579a9d9c6..94523ea8bff7 100644 --- a/python/ray/util/tracing/setup_local_tmp_tracing.py +++ b/python/ray/util/tracing/setup_local_tmp_tracing.py @@ -1,4 +1,5 @@ import os + from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ( diff --git a/python/ray/util/tracing/setup_tempo_tracing.py b/python/ray/util/tracing/setup_tempo_tracing.py index 12e310c612a0..e2bb3102b09d 100644 --- a/python/ray/util/tracing/setup_tempo_tracing.py +++ b/python/ray/util/tracing/setup_tempo_tracing.py @@ -1,9 +1,9 @@ # This file is intended for examples exporting traces to a local OTLP listener from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter, ) # noqa +from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ( ConsoleSpanExporter, SimpleSpanProcessor, From d2ed9182d2860616b3c71b6d1548de535ed1611b Mon Sep 17 00:00:00 2001 From: harshit-anyscale Date: Tue, 9 Sep 2025 22:09:12 +0530 Subject: [PATCH 529/634] fixing deployment scoped custom autoscaling (#56192) `attrbitutes` that start with `_` in pydantic are not seen by `validator`, `.dict()`, hence the previous implementation always resorted to the default policy. Changing `_policy` to `policy` fixes this problem. Also add a test to ensure this works --------- Signed-off-by: abrar Co-authored-by: abrar --- python/ray/serve/config.py | 22 +++---- .../serve/tests/test_autoscaling_policy.py | 61 ++++++++++++++++++- python/ray/serve/tests/test_controller.py | 3 + python/ray/serve/tests/test_deploy_2.py | 2 + python/ray/serve/tests/test_deploy_app_2.py | 2 + src/ray/protobuf/serve.proto | 2 +- 6 files changed, 75 insertions(+), 17 deletions(-) diff --git a/python/ray/serve/config.py b/python/ray/serve/config.py index 31b3ba84e825..650f62f71b44 100644 --- a/python/ray/serve/config.py +++ b/python/ray/serve/config.py @@ -233,11 +233,10 @@ class AutoscalingConfig(BaseModel): _serialized_policy_def: bytes = PrivateAttr(default=b"") # Autoscaling policy. This policy is deployment scoped. Defaults to the request-based autoscaler. - _policy: AutoscalingPolicy = Field(default_factory=AutoscalingPolicy) - - # This is to make `_policy` a normal field until its GA ready. - class Config: - underscore_attrs_are_private = True + policy: AutoscalingPolicy = Field( + default_factory=AutoscalingPolicy, + description="The autoscaling policy for the deployment. This option is experimental.", + ) @validator("max_replicas", always=True) def replicas_settings_valid(cls, max_replicas, values): @@ -285,23 +284,16 @@ def serialize_policy(self) -> None: Import the policy if it's passed in as a string import path. Then cloudpickle the policy and set `serialized_policy_def` if it's empty. """ - values = self.dict() - policy = values.get("_policy") - - policy_name = None - if isinstance(policy, dict): - policy_name = policy.get("name") + policy = self.policy + policy_name = policy.name if isinstance(policy_name, Callable): policy_name = f"{policy_name.__module__}.{policy_name.__name__}" - if not policy_name: - policy_name = DEFAULT_AUTOSCALING_POLICY_NAME - if not self._serialized_policy_def: self._serialized_policy_def = cloudpickle.dumps(import_attr(policy_name)) - self._policy = AutoscalingPolicy(name=policy_name) + self.policy = AutoscalingPolicy(name=policy_name) @classmethod def default(cls): diff --git a/python/ray/serve/tests/test_autoscaling_policy.py b/python/ray/serve/tests/test_autoscaling_policy.py index b3bbe85b06e8..98120b3e1626 100644 --- a/python/ray/serve/tests/test_autoscaling_policy.py +++ b/python/ray/serve/tests/test_autoscaling_policy.py @@ -15,6 +15,7 @@ import ray.util.state as state_api from ray import serve from ray._common.test_utils import SignalActor, wait_for_condition +from ray.serve._private.autoscaling_state import AutoscalingContext from ray.serve._private.common import ( DeploymentID, DeploymentStatus, @@ -36,7 +37,7 @@ get_num_alive_replicas, tlog, ) -from ray.serve.config import AutoscalingConfig +from ray.serve.config import AutoscalingConfig, AutoscalingPolicy from ray.serve.handle import DeploymentHandle from ray.serve.schema import ApplicationStatus, ServeDeploySchema from ray.util.state import list_actors @@ -1520,6 +1521,64 @@ def check_expected_statuses( print("Statuses are as expected.") +def custom_autoscaling_policy(ctx: AutoscalingContext): + if ctx.total_num_requests > 50: + return 3, {} + else: + return 2, {} + + +@pytest.mark.parametrize( + "policy", + [ + {"name": "ray.serve.tests.test_autoscaling_policy.custom_autoscaling_policy"}, + AutoscalingPolicy( + name="ray.serve.tests.test_autoscaling_policy.custom_autoscaling_policy" + ), + AutoscalingPolicy(name=custom_autoscaling_policy), + ], +) +def test_e2e_scale_up_down_basic_with_custom_policy(serve_instance_with_signal, policy): + """Send 100 requests and check that we autoscale up, and then back down.""" + + _, signal = serve_instance_with_signal + + @serve.deployment( + autoscaling_config={ + "min_replicas": 1, + "max_replicas": 4, + "downscale_delay_s": 0.5, + "upscale_delay_s": 0, + "policy": policy, + "metrics_interval_s": 0.1, + }, + # We will send over a lot of queries. This will make sure replicas are + # killed quickly during cleanup. + graceful_shutdown_timeout_s=1, + max_ongoing_requests=1000, + ) + class A: + async def __call__(self): + await signal.wait.remote() + + handle = serve.run(A.bind()) + wait_for_condition( + check_deployment_status, name="A", expected_status=DeploymentStatus.HEALTHY + ) + + [handle.remote() for _ in range(40)] + + # scale up one more replica from min_replicas + wait_for_condition(check_num_replicas_eq, name="A", target=2) + print("Scaled up to 2 replicas.") + + ray.get(signal.send.remote(clear=True)) + wait_for_condition(lambda: ray.get(signal.cur_num_waiters.remote()) == 0) + [handle.remote() for _ in range(70)] + wait_for_condition(check_num_replicas_eq, name="A", target=3) + ray.get(signal.send.remote(clear=True)) + + if __name__ == "__main__": import sys diff --git a/python/ray/serve/tests/test_controller.py b/python/ray/serve/tests/test_controller.py index 18ca6ce16ab2..f9765663eda0 100644 --- a/python/ray/serve/tests/test_controller.py +++ b/python/ray/serve/tests/test_controller.py @@ -177,6 +177,9 @@ def autoscaling_app(): "downscaling_factor": None, "downscale_delay_s": 600.0, "upscale_delay_s": 30.0, + "policy": { + "name": "ray.serve.autoscaling_policy:default_autoscaling_policy" + }, }, "graceful_shutdown_wait_loop_s": 2.0, "graceful_shutdown_timeout_s": 20.0, diff --git a/python/ray/serve/tests/test_deploy_2.py b/python/ray/serve/tests/test_deploy_2.py index 62abad6082e0..1dc32c7a23a0 100644 --- a/python/ray/serve/tests/test_deploy_2.py +++ b/python/ray/serve/tests/test_deploy_2.py @@ -331,6 +331,7 @@ async def __call__(self): "downscaling_factor": None, "smoothing_factor": 1.0, "initial_replicas": None, + "policy": {"name": "ray.serve.autoscaling_policy:default_autoscaling_policy"}, } @@ -384,6 +385,7 @@ async def __call__(self): "downscaling_factor": None, "smoothing_factor": 1.0, "initial_replicas": None, + "policy": {"name": "ray.serve.autoscaling_policy:default_autoscaling_policy"}, } for i in range(3): diff --git a/python/ray/serve/tests/test_deploy_app_2.py b/python/ray/serve/tests/test_deploy_app_2.py index 5d04dc45d60d..1b5acec54b4c 100644 --- a/python/ray/serve/tests/test_deploy_app_2.py +++ b/python/ray/serve/tests/test_deploy_app_2.py @@ -616,6 +616,7 @@ def test_num_replicas_auto_api(serve_instance): "downscaling_factor": None, "smoothing_factor": 1.0, "initial_replicas": None, + "policy": {"name": "ray.serve.autoscaling_policy:default_autoscaling_policy"}, } @@ -668,6 +669,7 @@ def test_num_replicas_auto_basic(serve_instance): "downscaling_factor": None, "smoothing_factor": 1.0, "initial_replicas": None, + "policy": {"name": "ray.serve.autoscaling_policy:default_autoscaling_policy"}, } h = serve.get_app_handle(SERVE_DEFAULT_APP_NAME) diff --git a/src/ray/protobuf/serve.proto b/src/ray/protobuf/serve.proto index a4d7202fb866..420bb7258b23 100644 --- a/src/ray/protobuf/serve.proto +++ b/src/ray/protobuf/serve.proto @@ -70,7 +70,7 @@ message AutoscalingConfig { bytes _serialized_policy_def = 11; // The autoscaling policy definition. - AutoscalingPolicy _policy = 12; + AutoscalingPolicy policy = 12; // Target number of in flight requests per replica. This is the primary configuration // knob for replica autoscaler. Lower the number, the more rapidly the replicas From f8d32c35b03fbfe7f425fbbdc4534f0bda45483b Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:26:27 -0700 Subject: [PATCH 530/634] [core] initialize opentelemetry.metrics once (#56347) Signed-off-by: Cuong Nguyen --- .../open_telemetry_metric_recorder.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/python/ray/_private/telemetry/open_telemetry_metric_recorder.py b/python/ray/_private/telemetry/open_telemetry_metric_recorder.py index 3f8e64fbfe08..a5cb561df1e8 100644 --- a/python/ray/_private/telemetry/open_telemetry_metric_recorder.py +++ b/python/ray/_private/telemetry/open_telemetry_metric_recorder.py @@ -23,17 +23,29 @@ class OpenTelemetryMetricRecorder: It uses OpenTelemetry's Prometheus exporter to export metrics. """ + _metrics_initialized = False + _metrics_initialized_lock = threading.Lock() + def __init__(self): self._lock = threading.Lock() self._registered_instruments = {} self._observations_by_name = defaultdict(dict) self._histogram_bucket_midpoints = defaultdict(list) - - prometheus_reader = PrometheusMetricReader() - provider = MeterProvider(metric_readers=[prometheus_reader]) - metrics.set_meter_provider(provider) + self._init_metrics() self.meter = metrics.get_meter(__name__) + def _init_metrics(self): + # Initialize the global metrics provider and meter. We only do this once on + # the first initialization of the class, because re-setting the meter provider + # can result in loss of metrics. + with self._metrics_initialized_lock: + if self._metrics_initialized: + return + prometheus_reader = PrometheusMetricReader() + provider = MeterProvider(metric_readers=[prometheus_reader]) + metrics.set_meter_provider(provider) + self._metrics_initialized = True + def register_gauge_metric(self, name: str, description: str) -> None: with self._lock: if name in self._registered_instruments: From d72d509a166ec6df3b87c4666d48c2b3b2cf8746 Mon Sep 17 00:00:00 2001 From: Eric Higgins <87031925+eric-higgins-ai@users.noreply.github.com> Date: Tue, 9 Sep 2025 13:41:47 -0400 Subject: [PATCH 531/634] [dashboard] fix grafana dashboard generation bug (#56346) If global_filters_str is empty then some queries in the dashboard contain ,, in their filters, which is invalid. Signed-off-by: eric-higgins-ai --- .../dashboard/modules/metrics/grafana_dashboard_factory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/ray/dashboard/modules/metrics/grafana_dashboard_factory.py b/python/ray/dashboard/modules/metrics/grafana_dashboard_factory.py index 2d57abe853ce..0ffa6b1cec15 100644 --- a/python/ray/dashboard/modules/metrics/grafana_dashboard_factory.py +++ b/python/ray/dashboard/modules/metrics/grafana_dashboard_factory.py @@ -66,7 +66,10 @@ def _read_configs_for_dashboard( ) or "" ) - global_filters = global_filters_str.split(",") + if global_filters_str == "": + global_filters = [] + else: + global_filters = global_filters_str.split(",") return uid, global_filters From d107d71b52b72948025fe5ce8b2b22f82f2c4f3e Mon Sep 17 00:00:00 2001 From: Srinath Krishnamachari <68668616+srinathk10@users.noreply.github.com> Date: Tue, 9 Sep 2025 10:54:34 -0700 Subject: [PATCH 532/634] [Data]: Fix mock_server cleanup on error (#56330) ## Why are these changes needed? ### [Data]: Fix mock_server cleanup on error - Better error handling in _s3_fs - Better error handling in mock_server stop_process ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Srinath Krishnamachari --- python/ray/data/tests/conftest.py | 45 ++++++++++++++------ python/ray/data/tests/mock_server.py | 63 ++++++++++++++++++++++++---- 2 files changed, 86 insertions(+), 22 deletions(-) diff --git a/python/ray/data/tests/conftest.py b/python/ray/data/tests/conftest.py index 5a14253918bb..6ee6e4df6bfc 100644 --- a/python/ray/data/tests/conftest.py +++ b/python/ray/data/tests/conftest.py @@ -144,19 +144,38 @@ def _s3_fs(aws_credentials, s3_server, s3_path): kwargs["allow_bucket_creation"] = True kwargs["allow_bucket_deletion"] = True - fs = pa.fs.S3FileSystem( - region="us-west-2", - endpoint_override=s3_server, - **kwargs, - ) - if s3_path.startswith("s3://"): - if "@" in s3_path: - s3_path = s3_path.split("@")[-1] - else: - s3_path = s3_path[len("s3://") :] - s3_path = urllib.parse.quote(s3_path) - fs.create_dir(s3_path) - yield fs + fs = None + try: + fs = pa.fs.S3FileSystem( + region="us-west-2", + endpoint_override=s3_server, + **kwargs, + ) + if s3_path.startswith("s3://"): + if "@" in s3_path: + s3_path = s3_path.split("@")[-1] + else: + s3_path = s3_path[len("s3://") :] + s3_path = urllib.parse.quote(s3_path) + fs.create_dir(s3_path) + yield fs + + finally: + # Explicit cleanup for S3FileSystem resources + if fs is not None: + try: + # Clean up test directory if it exists + try: + file_info = fs.get_file_info(s3_path) + if file_info.type != pa.fs.FileType.NotFound: + fs.delete_dir(s3_path) + except (OSError, pa.lib.ArrowIOError): + # Directory doesn't exist or can't be deleted, that's fine + pass + except Exception as e: + print(f"Warning: S3 filesystem cleanup error: {e}") + finally: + fs = None @pytest.fixture(scope="function") diff --git a/python/ray/data/tests/mock_server.py b/python/ray/data/tests/mock_server.py index f8a5e22bfa12..337aa1f87a72 100644 --- a/python/ray/data/tests/mock_server.py +++ b/python/ray/data/tests/mock_server.py @@ -1,5 +1,6 @@ import shutil import signal +import socket import subprocess as sp import time @@ -16,10 +17,51 @@ } +def _is_port_available(host, port): + """Check if a port is available for use.""" + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host, port)) + return True + except OSError: + return False + + +def _find_available_port(host, preferred_port, max_attempts=10): + """Find an available port starting from preferred_port.""" + + # Try the preferred port first + if _is_port_available(host, preferred_port): + return preferred_port + + # Try a wider range if preferred port is busy + for i in range(1, max_attempts): + port = preferred_port + i + if _is_port_available(host, port): + return port + + # If all else fails, let the OS pick a port + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host, 0)) # Let OS pick port + _, port = s.getsockname() + return port + except OSError as e: + raise RuntimeError( + f"Could not find any available port starting from " f"{preferred_port}: {e}" + ) from e + + def start_service(service_name, host, port): moto_svr_path = shutil.which("moto_server") if not moto_svr_path: pytest.skip("moto not installed") + + # Always use port conflict resolution to be safe + port = _find_available_port(host, port) + args = [moto_svr_path, service_name, "-H", host, "-p", str(port)] # For debugging # args = '{0} {1} -H {2} -p {3} 2>&1 | \ @@ -48,21 +90,25 @@ def start_service(service_name, host, port): stop_process(process) # pytest.fail doesn't call stop_process pytest.fail("Can not start service: {}".format(service_name)) - return process + return process, url def stop_process(process): + """Stop process with shorter timeout to prevent test hangs.""" + if process is None or process.poll() is not None: + return # Already stopped + try: process.send_signal(signal.SIGTERM) process.communicate(timeout=20) except sp.TimeoutExpired: process.kill() - outs, errors = process.communicate(timeout=20) - exit_code = process.returncode - msg = "Child process finished {} not in clean way: {} {}".format( - exit_code, outs, errors - ) - raise RuntimeError(msg) + try: + process.communicate(timeout=5) # Short timeout for kill + except sp.TimeoutExpired: + print("Warning: Process cleanup timed out") + except Exception as e: + print(f"Warning: Error during process cleanup: {e}") # TODO(Clark): We should be able to use "session" scope here, but we've found @@ -75,7 +121,6 @@ def stop_process(process): def s3_server(): host = "localhost" port = 5002 - url = f"http://{build_address(host, port)}" - process = start_service("s3", host, port) + process, url = start_service("s3", host, port) yield url stop_process(process) From 6d0a650a4a464fb7768c2a834c1e5668b5defe96 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 9 Sep 2025 11:02:50 -0700 Subject: [PATCH 533/634] [core] fix lint on test_deprecation (#56386) on isort enabling. Signed-off-by: Lonnie Liu --- python/ray/_common/tests/test_deprecation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/ray/_common/tests/test_deprecation.py b/python/ray/_common/tests/test_deprecation.py index 9f0b23efde3c..a6d9d7a13f54 100644 --- a/python/ray/_common/tests/test_deprecation.py +++ b/python/ray/_common/tests/test_deprecation.py @@ -1,11 +1,13 @@ -import pytest import sys +from unittest.mock import patch + +import pytest + from ray._common.deprecation import ( DEPRECATED_VALUE, Deprecated, deprecation_warning, ) -from unittest.mock import patch def test_deprecation_warning_warn(): From 4efdd3580c5e25bbdad2e2e95fba6f0ea06ac594 Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Tue, 9 Sep 2025 11:06:12 -0700 Subject: [PATCH 534/634] [serve] Add a test to ensure calling await multiple times on response doesn't result in multiple calls to replica (#56362) ## Why are these changes needed? Add handle test for fetching deployment response result multiple times. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: akyang-anyscale --- .../ray/serve/tests/test_handle_same_loop.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/python/ray/serve/tests/test_handle_same_loop.py b/python/ray/serve/tests/test_handle_same_loop.py index 056703fe4365..c3086f71ae86 100644 --- a/python/ray/serve/tests/test_handle_same_loop.py +++ b/python/ray/serve/tests/test_handle_same_loop.py @@ -220,5 +220,27 @@ async def check_num_waiters(): await signal_actor.send.remote(clear=True) +@pytest.mark.asyncio +async def test_multiple_awaits(serve_instance_async): + """Test that multiple awaits doesn't call replica multiple times.""" + a = 0 + + @serve.deployment + async def foo(): + nonlocal a + a += 1 + return a + + app = serve.run(foo.bind()) + + response = app.remote() + assert await response == 1 + assert await response == 1 + + response = app.remote() + assert await response == 2 + assert await response == 2 + + if __name__ == "__main__": sys.exit(pytest.main(["-v", "-s", __file__])) From 248d069405b76d714377e078d875e65782e1b682 Mon Sep 17 00:00:00 2001 From: Ibrahim Rabbani Date: Tue, 9 Sep 2025 11:26:16 -0700 Subject: [PATCH 535/634] Fixing broken CI due to linter issues (#56385) Signed-off-by: irabbani From b080aa1605e0d1d4e54ea7324d207079af4b845c Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Tue, 9 Sep 2025 12:55:09 -0700 Subject: [PATCH 536/634] [core] Allow task manager access with submitter mutex + unify retry (#56216) Signed-off-by: dayshah Signed-off-by: Dhyey Shah --- src/ray/core_worker/core_worker.cc | 27 ++---- src/ray/core_worker/core_worker.h | 4 +- src/ray/core_worker/core_worker_process.cc | 6 +- src/ray/core_worker/task_manager.cc | 10 +-- src/ray/core_worker/task_manager.h | 10 +-- .../task_submission/actor_task_submitter.cc | 86 +++++++++---------- .../task_submission/actor_task_submitter.h | 8 -- src/ray/core_worker/tests/core_worker_test.cc | 2 +- .../core_worker/tests/task_manager_test.cc | 24 +----- 9 files changed, 68 insertions(+), 109 deletions(-) diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 433baa6a0184..04db6d5398f7 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -4548,28 +4548,13 @@ void CoreWorker::UpdateTaskIsDebuggerPaused(const TaskID &task_id, worker::TaskStatusEvent::TaskStateUpdate(is_debugger_paused))); } -void CoreWorker::TaskManagerRetryTask(TaskSpecification &spec, - bool object_recovery, - uint32_t delay_ms) { +void CoreWorker::AsyncRetryTask(TaskSpecification &spec, uint32_t delay_ms) { spec.GetMutableMessage().set_attempt_number(spec.AttemptNumber() + 1); - if (!object_recovery) { - // Retry after a delay to emulate the existing Raylet reconstruction - // behaviour. TODO(ekl) backoff exponentially. - RAY_LOG(INFO) << "Will resubmit task after a " << delay_ms - << "ms delay: " << spec.DebugString(); - absl::MutexLock lock(&mutex_); - TaskToRetry task_to_retry{current_time_ms() + delay_ms, spec}; - to_resubmit_.push(std::move(task_to_retry)); - } else { - if (spec.IsActorTask()) { - auto actor_handle = actor_manager_->GetActorHandle(spec.ActorId()); - actor_handle->SetResubmittedActorTaskSpec(spec); - actor_task_submitter_->SubmitTask(spec); - } else { - RAY_CHECK(spec.IsNormalTask()); - normal_task_submitter_->SubmitTask(spec); - } - } + absl::MutexLock lock(&mutex_); + TaskToRetry task_to_retry{current_time_ms() + delay_ms, spec}; + RAY_LOG(INFO) << "Will resubmit task after a " << delay_ms + << "ms delay: " << spec.DebugString(); + to_resubmit_.push(std::move(task_to_retry)); } } // namespace ray::core diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index f1de5e739d7d..58e0cd65cf8a 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -1340,9 +1340,7 @@ class CoreWorker { const std::shared_ptr &creation_task_exception_pb_bytes = nullptr); - void TaskManagerRetryTask(TaskSpecification &spec, - bool object_recovery, - uint32_t delay_ms); + void AsyncRetryTask(TaskSpecification &spec, uint32_t delay_ms); private: static nlohmann::json OverrideRuntimeEnv(const nlohmann::json &child, diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index 5aa2545c8dbf..aaa3515264a2 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -454,10 +454,10 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( << ") with status: " << put_status; return put_status; }, - /* retry_task_callback= */ - [this](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) { + /* async_retry_task_callback=*/ + [this](TaskSpecification &spec, uint32_t delay_ms) { auto core_worker = GetCoreWorker(); - core_worker->TaskManagerRetryTask(spec, object_recovery, delay_ms); + core_worker->AsyncRetryTask(spec, delay_ms); }, /*queue_generator_resubmit=*/ [this](const TaskSpecification &spec) { diff --git a/src/ray/core_worker/task_manager.cc b/src/ray/core_worker/task_manager.cc index a78b481a76e8..ffd18eb96a18 100644 --- a/src/ray/core_worker/task_manager.cc +++ b/src/ray/core_worker/task_manager.cc @@ -416,7 +416,7 @@ std::optional TaskManager::ResubmitTask( // issue #54260. RAY_LOG(INFO) << "Resubmitting task that produced lost plasma object, attempt #" << spec.AttemptNumber() << ": " << spec.DebugString(); - retry_task_callback_(spec, /*object_recovery*/ true, /*delay_ms*/ 0); + async_retry_task_callback_(spec, /*delay_ms=*/0); return std::nullopt; } @@ -424,8 +424,8 @@ std::optional TaskManager::ResubmitTask( void TaskManager::SetupTaskEntryForResubmit(TaskEntry &task_entry) { task_entry.MarkRetry(); // NOTE(rickyx): We only increment the AttemptNumber on the task spec when - // `retry_task_callback_` is invoked. In order to record the correct status change for - // the new task attempt, we pass the attempt number explicitly. + // `async_retry_task_callback_` is invoked. In order to record the correct status change + // for the new task attempt, we pass the attempt number explicitly. SetTaskStatus(task_entry, rpc::TaskStatus::PENDING_ARGS_AVAIL, /* state_update */ std::nullopt, @@ -503,7 +503,7 @@ void TaskManager::MarkGeneratorFailedAndResubmit(const TaskID &task_id) { // Note: Don't need to call UpdateReferencesForResubmit because CompletePendingTask or // FailPendingTask are not called when this is. Therefore, RemoveFinishedTaskReferences // never happened for this task. - retry_task_callback_(spec, /*object_recovery*/ true, /*delay_ms*/ 0); + async_retry_task_callback_(spec, /*delay_ms*/ 0); } void TaskManager::DrainAndShutdown(std::function shutdown) { @@ -1225,7 +1225,7 @@ bool TaskManager::RetryTaskIfPossible(const TaskID &task_id, spec.AttemptNumber(), RayConfig::instance().task_oom_retry_delay_base_ms()) : RayConfig::instance().task_retry_delay_ms(); - retry_task_callback_(spec, /*object_recovery*/ false, delay_ms); + async_retry_task_callback_(spec, delay_ms); return true; } else { RAY_LOG(INFO) << "No retries left for task " << spec.TaskId() diff --git a/src/ray/core_worker/task_manager.h b/src/ray/core_worker/task_manager.h index 8da151499087..9d334eb226b7 100644 --- a/src/ray/core_worker/task_manager.h +++ b/src/ray/core_worker/task_manager.h @@ -48,8 +48,8 @@ class ActorManager; using TaskStatusCounter = CounterMap>; using PutInLocalPlasmaCallback = std::function; -using RetryTaskCallback = - std::function; +using AsyncRetryTaskCallback = + std::function; using ReconstructObjectCallback = std::function; using PushErrorCallback = std::function queue_generator_resubmit, PushErrorCallback push_error_callback, int64_t max_lineage_bytes, @@ -190,7 +190,7 @@ class TaskManager : public TaskManagerInterface { : in_memory_store_(in_memory_store), reference_counter_(reference_counter), put_in_local_plasma_callback_(std::move(put_in_local_plasma_callback)), - retry_task_callback_(std::move(retry_task_callback)), + async_retry_task_callback_(std::move(async_retry_task_callback)), queue_generator_resubmit_(std::move(queue_generator_resubmit)), push_error_callback_(std::move(push_error_callback)), max_lineage_bytes_(max_lineage_bytes), @@ -748,7 +748,7 @@ class TaskManager : public TaskManagerInterface { const PutInLocalPlasmaCallback put_in_local_plasma_callback_; /// Called when a task should be retried. - const RetryTaskCallback retry_task_callback_; + const AsyncRetryTaskCallback async_retry_task_callback_; /// For when a streaming generator task currently in progress needs to be resubmitted. std::function queue_generator_resubmit_; diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.cc b/src/ray/core_worker/task_submission/actor_task_submitter.cc index 9df800d15e39..3cf16396d1c2 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.cc +++ b/src/ray/core_worker/task_submission/actor_task_submitter.cc @@ -229,7 +229,7 @@ void ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { } if (fail_or_retry_task) { - GetTaskManagerWithoutMu().FailOrRetryPendingTask( + task_manager_.FailOrRetryPendingTask( task_id, rpc::ErrorType::DEPENDENCY_RESOLUTION_FAILED, &status); } }); @@ -255,12 +255,12 @@ void ActorTaskSubmitter::SubmitTask(TaskSpecification task_spec) { error_info.has_actor_died_error() && error_info.actor_died_error().has_oom_context() && error_info.actor_died_error().oom_context().fail_immediately(); - GetTaskManagerWithoutMu().FailOrRetryPendingTask(task_id, - error_type, - &status, - &error_info, - /*mark_task_object_failed*/ true, - fail_immediately); + task_manager_.FailOrRetryPendingTask(task_id, + error_type, + &status, + &error_info, + /*mark_task_object_failed*/ true, + fail_immediately); } } @@ -280,8 +280,8 @@ void ActorTaskSubmitter::FailInflightTasksOnRestart( const absl::flat_hash_map> &inflight_task_callbacks) { // NOTE(kfstorm): We invoke the callbacks with a bad status to act like there's a - // network issue. We don't call `task_manager_.FailOrRetryPendingTask` directly because - // there's much more work to do in the callback. + // network issue. We don't call `task_manager_.FailOrRetryPendingTask` directly + // because there's much more work to do in the callback. auto status = Status::IOError("The actor was restarted"); for (const auto &[_, callback] : inflight_task_callbacks) { callback(status, rpc::PushTaskReply()); @@ -456,18 +456,18 @@ void ActorTaskSubmitter::DisconnectActor(const ActorID &actor_id, error_info.has_actor_died_error() && error_info.actor_died_error().has_oom_context() && error_info.actor_died_error().oom_context().fail_immediately(); - GetTaskManagerWithoutMu().FailOrRetryPendingTask(task_id, - error_type, - &status, - &error_info, - /*mark_task_object_failed*/ true, - fail_immediatedly); + task_manager_.FailOrRetryPendingTask(task_id, + error_type, + &status, + &error_info, + /*mark_task_object_failed*/ true, + fail_immediatedly); } if (!wait_for_death_info_tasks.empty()) { RAY_LOG(DEBUG).WithField(actor_id) << "Failing tasks waiting for death info, size=" << wait_for_death_info_tasks.size(); for (auto &task : wait_for_death_info_tasks) { - GetTaskManagerWithoutMu().FailPendingTask( + task_manager_.FailPendingTask( task->task_spec_.TaskId(), error_type, &task->status_, &error_info); } } @@ -495,15 +495,15 @@ void ActorTaskSubmitter::FailTaskWithError(const PendingTaskWaitingForDeathInfo error_info.set_error_type(rpc::ErrorType::ACTOR_DIED); error_info.set_error_message("Actor died by preemption."); } - GetTaskManagerWithoutMu().FailPendingTask( + task_manager_.FailPendingTask( task.task_spec_.TaskId(), error_info.error_type(), &task.status_, &error_info); } void ActorTaskSubmitter::CheckTimeoutTasks() { // For each task in `wait_for_death_info_tasks`, if it times out, fail it with // timeout_error_info. But operating on the queue requires the mu_ lock; while calling - // FailPendingTask requires the opposite. So we copy the tasks out from the queue within - // the lock. This requires putting the data into shared_ptr. + // FailPendingTask requires the opposite. So we copy the tasks out from the queue + // within the lock. This requires putting the data into shared_ptr. std::vector> timeout_tasks; int64_t now = current_time_ms(); { @@ -655,7 +655,7 @@ void ActorTaskSubmitter::HandlePushTaskReply(const Status &status, } } if (resubmit_generator) { - GetTaskManagerWithoutMu().MarkGeneratorFailedAndResubmit(task_id); + task_manager_.MarkGeneratorFailedAndResubmit(task_id); return; } @@ -676,10 +676,10 @@ void ActorTaskSubmitter::HandlePushTaskReply(const Status &status, rpc::RayErrorInfo error_info; error_info.set_error_message(msg); error_info.set_error_type(rpc::ErrorType::TASK_CANCELLED); - GetTaskManagerWithoutMu().FailPendingTask(task_spec.TaskId(), - rpc::ErrorType::TASK_CANCELLED, - /*status*/ nullptr, - &error_info); + task_manager_.FailPendingTask(task_spec.TaskId(), + rpc::ErrorType::TASK_CANCELLED, + /*status*/ nullptr, + &error_info); } else { bool is_actor_dead = false; bool fail_immediately = false; @@ -698,8 +698,8 @@ void ActorTaskSubmitter::HandlePushTaskReply(const Status &status, auto &queue = queue_pair->second; // If the actor is already dead, immediately mark the task object as failed. - // Otherwise, start the grace period, waiting for the actor death reason. Before the - // deadline: + // Otherwise, start the grace period, waiting for the actor death reason. Before + // the deadline: // - If we got the death reason: mark the object as failed with that reason. // - If we did not get the death reason: raise ACTOR_UNAVAILABLE with the status. // - If we did not get the death reason, but *the actor is preempted*: raise @@ -712,8 +712,8 @@ void ActorTaskSubmitter::HandlePushTaskReply(const Status &status, error_info.actor_died_error().has_oom_context() && error_info.actor_died_error().oom_context().fail_immediately(); } else { - // The actor may or may not be dead, but the request failed. Consider the failure - // temporary. May recognize retry, so fail_immediately = false. + // The actor may or may not be dead, but the request failed. Consider the + // failure temporary. May recognize retry, so fail_immediately = false. error_info.set_error_message("The actor is temporarily unavailable: " + status.ToString()); error_info.set_error_type(rpc::ErrorType::ACTOR_UNAVAILABLE); @@ -725,25 +725,25 @@ void ActorTaskSubmitter::HandlePushTaskReply(const Status &status, // this first. CancelDependencyResolution(task_id); - will_retry = GetTaskManagerWithoutMu().FailOrRetryPendingTask( - task_id, - error_info.error_type(), - &status, - &error_info, - /*mark_task_object_failed*/ is_actor_dead, - fail_immediately); + will_retry = + task_manager_.FailOrRetryPendingTask(task_id, + error_info.error_type(), + &status, + &error_info, + /*mark_task_object_failed*/ is_actor_dead, + fail_immediately); if (!is_actor_dead && !will_retry) { // Ran out of retries, last failure = either user exception or actor death. if (status.ok()) { // last failure = user exception, just complete it with failure. RAY_CHECK(reply.is_retryable_error()); - GetTaskManagerWithoutMu().CompletePendingTask( + task_manager_.CompletePendingTask( task_id, reply, addr, reply.is_application_error()); } else if (RayConfig::instance().timeout_ms_task_wait_for_death_info() != 0) { - // last failure = Actor death, but we still see the actor "alive" so we optionally - // wait for a grace period for the death info. + // last failure = Actor death, but we still see the actor "alive" so we + // optionally wait for a grace period for the death info. int64_t death_info_grace_period_ms = current_time_ms() + @@ -767,7 +767,7 @@ void ActorTaskSubmitter::HandlePushTaskReply(const Status &status, auto queue_pair = client_queues_.find(actor_id); RAY_CHECK(queue_pair != client_queues_.end()); } - GetTaskManagerWithoutMu().FailPendingTask( + task_manager_.FailPendingTask( task_spec.TaskId(), error_info.error_type(), &status, &error_info); } } @@ -879,8 +879,8 @@ void ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursive) // Shouldn't hold a lock while accessing task_manager_. // Task is already canceled or finished. - GetTaskManagerWithoutMu().MarkTaskCanceled(task_id); - if (!GetTaskManagerWithoutMu().IsTaskPending(task_id)) { + task_manager_.MarkTaskCanceled(task_id); + if (!task_manager_.IsTaskPending(task_id)) { RAY_LOG(DEBUG).WithField(task_id) << "Task is already finished or canceled"; return; } @@ -920,7 +920,7 @@ void ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursive) << " before it executes."; error_info.set_error_message(stream.str()); error_info.set_error_type(rpc::ErrorType::TASK_CANCELLED); - GetTaskManagerWithoutMu().FailOrRetryPendingTask( + task_manager_.FailOrRetryPendingTask( task_id, rpc::ErrorType::TASK_CANCELLED, /*status*/ nullptr, &error_info); return; } @@ -958,7 +958,7 @@ void ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursive) // Keep retrying every 2 seconds until a task is officially // finished. - if (!GetTaskManagerWithoutMu().GetTaskSpec(task_id)) { + if (!task_manager_.GetTaskSpec(task_id)) { // Task is already finished. RAY_LOG(DEBUG).WithField(task_spec.TaskId()) << "Task is finished. Stop a cancel request."; diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.h b/src/ray/core_worker/task_submission/actor_task_submitter.h index a07591d582b7..9eb41a90298b 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.h +++ b/src/ray/core_worker/task_submission/actor_task_submitter.h @@ -254,14 +254,6 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { status_(std::move(status)), timeout_error_info_(std::move(timeout_error_info)) {} }; - /// A helper function to get task manager without holding mu_ - /// We should use this function when access - /// - FailOrRetryPendingTask - /// - FailPendingTask - TaskManagerInterface &GetTaskManagerWithoutMu() { - mu_.AssertNotHeld(); - return task_manager_; - } struct ClientQueue { ClientQueue(bool allow_out_of_order_execution, diff --git a/src/ray/core_worker/tests/core_worker_test.cc b/src/ray/core_worker/tests/core_worker_test.cc index 7923893880c9..d0837518bfa9 100644 --- a/src/ray/core_worker/tests/core_worker_test.cc +++ b/src/ray/core_worker/tests/core_worker_test.cc @@ -156,7 +156,7 @@ class CoreWorkerTest : public ::testing::Test { *memory_store_, *reference_counter_, [](const RayObject &object, const ObjectID &object_id) { return Status::OK(); }, - [](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) {}, + [](TaskSpecification &spec, uint32_t delay_ms) {}, [](const TaskSpecification &spec) { return false; }, [](const JobID &job_id, const std::string &type, diff --git a/src/ray/core_worker/tests/task_manager_test.cc b/src/ray/core_worker/tests/task_manager_test.cc index d99915cbef1d..63215f0d1eb6 100644 --- a/src/ray/core_worker/tests/task_manager_test.cc +++ b/src/ray/core_worker/tests/task_manager_test.cc @@ -163,11 +163,9 @@ class TaskManagerTest : public ::testing::Test { stored_in_plasma.insert(object_id); return Status::OK(); }, - [this](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) { + [this](TaskSpecification &spec, uint32_t delay_ms) { num_retries_++; last_delay_ms_ = delay_ms; - last_object_recovery_ = object_recovery; - return Status::OK(); }, [this](const TaskSpecification &spec) { return this->did_queue_generator_resubmit_; @@ -228,7 +226,6 @@ class TaskManagerTest : public ::testing::Test { TaskManager manager_; int num_retries_ = 0; uint32_t last_delay_ms_ = 0; - bool last_object_recovery_ = false; std::unordered_set stored_in_plasma; ray::observability::FakeMetric fake_task_by_state_counter_; }; @@ -446,7 +443,6 @@ TEST_F(TaskManagerTest, TestTaskReconstruction) { ASSERT_FALSE(store_->Get({return_id}, 1, 0, ctx, false, &results).ok()); ASSERT_EQ(num_retries_, i + 1); ASSERT_EQ(last_delay_ms_, RayConfig::instance().task_retry_delay_ms()); - ASSERT_EQ(last_object_recovery_, false); } manager_.FailOrRetryPendingTask(spec.TaskId(), error); @@ -581,13 +577,11 @@ TEST_F(TaskManagerTest, TestTaskOomAndNonOomKillReturnsLastError) { manager_.FailOrRetryPendingTask(spec.TaskId(), error); ASSERT_EQ(num_retries_, 1); ASSERT_EQ(last_delay_ms_, RayConfig::instance().task_oom_retry_delay_base_ms()); - ASSERT_EQ(last_object_recovery_, false); error = rpc::ErrorType::WORKER_DIED; manager_.FailOrRetryPendingTask(spec.TaskId(), error); ASSERT_EQ(num_retries_, 2); ASSERT_EQ(last_delay_ms_, RayConfig::instance().task_retry_delay_ms()); - ASSERT_EQ(last_object_recovery_, false); error = rpc::ErrorType::WORKER_DIED; manager_.FailOrRetryPendingTask(spec.TaskId(), error); @@ -1089,7 +1083,6 @@ TEST_F(TaskManagerLineageTest, TestResubmitTask) { ASSERT_EQ(resubmitted_task_deps, spec.GetDependencyIds()); ASSERT_EQ(num_retries_, 1); ASSERT_EQ(last_delay_ms_, 0); - ASSERT_EQ(last_object_recovery_, true); resubmitted_task_deps.clear(); // The return ID goes out of scope. @@ -1153,7 +1146,6 @@ TEST_F(TaskManagerLineageTest, TestResubmittedTaskNondeterministicReturns) { ASSERT_EQ(manager_.ResubmitTask(spec.TaskId(), &resubmitted_task_deps), std::nullopt); ASSERT_EQ(num_retries_, 1); ASSERT_EQ(last_delay_ms_, 0); - ASSERT_EQ(last_object_recovery_, true); // The re-executed task completes again. One of the return objects is now // returned directly. @@ -1218,7 +1210,6 @@ TEST_F(TaskManagerLineageTest, TestResubmittedTaskFails) { ASSERT_EQ(manager_.ResubmitTask(spec.TaskId(), &resubmitted_task_deps), std::nullopt); ASSERT_EQ(num_retries_, 1); ASSERT_EQ(last_delay_ms_, 0); - ASSERT_EQ(last_object_recovery_, true); // The re-executed task fails due to worker crashed. { @@ -1339,7 +1330,6 @@ TEST_F(TaskManagerLineageTest, TestResubmittedDynamicReturnsTaskFails) { ASSERT_EQ(manager_.ResubmitTask(spec.TaskId(), &resubmitted_task_deps), std::nullopt); ASSERT_EQ(num_retries_, 1); ASSERT_EQ(last_delay_ms_, 0); - ASSERT_EQ(last_object_recovery_, true); // Dereference the generator to a list of its internal ObjectRefs. for (const auto &dynamic_return_id : dynamic_return_ids) { @@ -1395,11 +1385,9 @@ TEST_F(TaskManagerTest, PlasmaPut_ObjectStoreFull_FailsTaskAndWritesError) { [](const RayObject &, const ObjectID &) { return Status::ObjectStoreFull("simulated"); }, - [this](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) { + [this](TaskSpecification &spec, uint32_t delay_ms) { num_retries_++; last_delay_ms_ = delay_ms; - last_object_recovery_ = object_recovery; - return Status::OK(); }, [this](const TaskSpecification &spec) { return this->did_queue_generator_resubmit_; @@ -1459,11 +1447,9 @@ TEST_F(TaskManagerTest, PlasmaPut_TransientFull_RetriesThenSucceeds) { } return Status::OK(); }, - [this](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) { + [this](TaskSpecification &spec, uint32_t delay_ms) { num_retries_++; last_delay_ms_ = delay_ms; - last_object_recovery_ = object_recovery; - return Status::OK(); }, [this](const TaskSpecification &spec) { return this->did_queue_generator_resubmit_; @@ -1521,11 +1507,9 @@ TEST_F(TaskManagerTest, DynamicReturn_PlasmaPutFailure_FailsTaskImmediately) { } return Status::OK(); }, - [this](TaskSpecification &spec, bool object_recovery, uint32_t delay_ms) { + [this](TaskSpecification &spec, uint32_t delay_ms) { num_retries_++; last_delay_ms_ = delay_ms; - last_object_recovery_ = object_recovery; - return Status::OK(); }, [this](const TaskSpecification &spec) { return this->did_queue_generator_resubmit_; From 82b026af2c961fdca3099420415fac6e27e4dfdb Mon Sep 17 00:00:00 2001 From: akyang-anyscale Date: Tue, 9 Sep 2025 13:04:15 -0700 Subject: [PATCH 537/634] [serve] Explicitly close `choose_replicas_with_backoff` async generator (#56357) ## Why are these changes needed? This async generator isn't closed/consumed because it's infinitely looping by design. Therefore a new async task will be created to close it if we don't explicitly call `.aclose()` to close it. Let's call `.aclose()` to minimize the number of async tasks created. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: akyang-anyscale --- .../_private/request_router/request_router.py | 76 ++++++++++--------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/python/ray/serve/_private/request_router/request_router.py b/python/ray/serve/_private/request_router/request_router.py index 2d95689d5b41..4002ee4540c0 100644 --- a/python/ray/serve/_private/request_router/request_router.py +++ b/python/ray/serve/_private/request_router/request_router.py @@ -979,46 +979,52 @@ async def _fulfill_pending_requests(self): backoff_index = 0 pending_request = self._get_next_pending_request_to_route() request_metadata = pending_request.metadata if pending_request else None - async for candidates in self._choose_replicas_with_backoff( + gen_choose_replicas_with_backoff = self._choose_replicas_with_backoff( pending_request - ): - # Clear out pending requests at the front of the - # queue that have been cancelled, then reevaluate - # if we need to continue this routing task. - while ( - len(self._pending_requests_to_fulfill) > 0 - and self._pending_requests_to_fulfill[0].future.done() - ): - self._pending_requests_to_fulfill.popleft() - - if len(self._routing_tasks) > self.target_num_routing_tasks: - break - - replica = await self._select_from_candidate_replicas( - candidates, backoff_index - ) - if replica is not None: - self._fulfill_next_pending_request(replica, request_metadata) - break - - backoff_index += 1 - if backoff_index >= 50 and backoff_index % 50 == 0: - routing_time_elapsed = time.time() - start_time - warning_log = ( - "Failed to route request after " - f"{backoff_index} attempts over " - f"{routing_time_elapsed:.2f}s. Retrying." + ) + try: + async for candidates in gen_choose_replicas_with_backoff: + # Clear out pending requests at the front of the + # queue that have been cancelled, then reevaluate + # if we need to continue this routing task. + while ( + len(self._pending_requests_to_fulfill) > 0 + and self._pending_requests_to_fulfill[0].future.done() + ): + self._pending_requests_to_fulfill.popleft() + + if len(self._routing_tasks) > self.target_num_routing_tasks: + break + + replica = await self._select_from_candidate_replicas( + candidates, backoff_index ) - if request_metadata is not None: - warning_log += ( - f" Request ID: {request_metadata.request_id}." + if replica is not None: + self._fulfill_next_pending_request( + replica, request_metadata + ) + break + + backoff_index += 1 + if backoff_index >= 50 and backoff_index % 50 == 0: + routing_time_elapsed = time.time() - start_time + warning_log = ( + "Failed to route request after " + f"{backoff_index} attempts over " + f"{routing_time_elapsed:.2f}s. Retrying." ) - if request_metadata.multiplexed_model_id: + if request_metadata is not None: warning_log += ( - " Multiplexed model ID: " - f"{request_metadata.multiplexed_model_id}." + f" Request ID: {request_metadata.request_id}." ) - logger.warning(warning_log) + if request_metadata.multiplexed_model_id: + warning_log += ( + " Multiplexed model ID: " + f"{request_metadata.multiplexed_model_id}." + ) + logger.warning(warning_log) + finally: + await gen_choose_replicas_with_backoff.aclose() except Exception: logger.exception("Unexpected error in _fulfill_pending_requests.") From cd767bdd0cf7ca8d847f1b588cd47081f78f1b4e Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Tue, 9 Sep 2025 13:51:06 -0700 Subject: [PATCH 538/634] [Data] Add `max_task_concurrency`, `min_scheduling_resources`, and `per_task_resource_allocation` (#56381) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? Adds three new methods to the PhysicalOperator interface to clarify resource usage and scheduling semantics: * `per_task_resource_allocation` – logical resources required per task (task-level granularity). * `max_task_concurrency` – maximum number of tasks allowed to run concurrently, if the operator enforces one. * `min_scheduling_resources` – minimum resource bundle required to schedule a worker (e.g., task vs. actor). These methods provide a clearer contract for how operators declare resource requirements, making scheduling behavior easier to reason about and extend. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Balaji Veeramani --- .../execution/interfaces/physical_operator.py | 34 +++++++++++++++++++ .../operators/actor_pool_map_operator.py | 16 +++++++++ .../operators/task_pool_map_operator.py | 13 +++++++ 3 files changed, 63 insertions(+) diff --git a/python/ray/data/_internal/execution/interfaces/physical_operator.py b/python/ray/data/_internal/execution/interfaces/physical_operator.py index 12f7242aa3c8..c3a7b429813e 100644 --- a/python/ray/data/_internal/execution/interfaces/physical_operator.py +++ b/python/ray/data/_internal/execution/interfaces/physical_operator.py @@ -402,6 +402,40 @@ def _get_logical_args(self) -> Dict[str, Dict[str, Any]]: res[logical_op_id] = logical_op._get_args() return res + # TODO(@balaji): Disambiguate this with `incremental_resource_usage`. + def per_task_resource_allocation( + self: "PhysicalOperator", + ) -> ExecutionResources: + """The amount of logical resources used by each task. + + For regular tasks, these are the resources required to schedule a task. For + actor tasks, these are the resources required to schedule an actor divided by + the number of actor threads (i.e., `max_concurrency`). + + Returns: + The resource requirement per task. + """ + return ExecutionResources.zero() + + def max_task_concurrency(self: "PhysicalOperator") -> Optional[int]: + """The maximum number of tasks that can be run concurrently. + + Some operators manually configure a maximum concurrency. For example, if you + specify `concurrency` in `map_batches`. + """ + return None + + # TODO(@balaji): Disambiguate this with `base_resource_usage`. + def min_scheduling_resources( + self: "PhysicalOperator", + ) -> ExecutionResources: + """The minimum resource bundle required to schedule a worker. + + For regular tasks, this is the resources required to schedule a task. For actor + tasks, this is the resources required to schedule an actor. + """ + return ExecutionResources.zero() + def progress_str(self) -> str: """Return any extra status to be displayed in the operator progress bar. diff --git a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py index c79350dcedfe..3080f460bb6d 100644 --- a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py @@ -478,6 +478,22 @@ def _apply_default_remote_args( def get_autoscaling_actor_pools(self) -> List[AutoscalingActorPool]: return [self._actor_pool] + def per_task_resource_allocation( + self: "PhysicalOperator", + ) -> ExecutionResources: + max_concurrency = self._ray_remote_args.get("max_concurrency", 1) + per_actor_resource_usage = self._actor_pool.per_actor_resource_usage() + return per_actor_resource_usage.scale(1 / max_concurrency) + + def max_task_concurrency(self: "PhysicalOperator") -> Optional[int]: + max_concurrency = self._ray_remote_args.get("max_concurrency", 1) + return max_concurrency * self._actor_pool.max_size() + + def min_scheduling_resources( + self: "PhysicalOperator", + ) -> ExecutionResources: + return self._actor_pool.per_actor_resource_usage() + def update_resource_usage(self) -> None: """Updates resources usage.""" for actor in self._actor_pool.get_running_actor_refs(): diff --git a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py index 3ec90eb32f71..a46b44cf2bbe 100644 --- a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py @@ -140,6 +140,19 @@ def incremental_resource_usage(self) -> ExecutionResources: or 0, ) + def per_task_resource_allocation( + self: "PhysicalOperator", + ) -> ExecutionResources: + return self.incremental_resource_usage() + + def max_task_concurrency(self: "PhysicalOperator") -> Optional[int]: + return self._concurrency + + def min_scheduling_resources( + self: "PhysicalOperator", + ) -> ExecutionResources: + return self.incremental_resource_usage() + def get_concurrency(self) -> Optional[int]: return self._concurrency From 3193145af57db3e5186b9b5e66bfcfc111c4d0c6 Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Tue, 9 Sep 2025 13:51:24 -0700 Subject: [PATCH 539/634] [Data] Add hash and `to_resource_dict` to `ExecutionResources` (#56383) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? This PR adds two utility methods to `ExecutionResources`: * `__hash__` – allows `ExecutionResources` to be effectively stored in sets or used as dictionary keys. * `to_resource_dict` – converts the object into a resource dictionary for interoperability with the autoscaler SDK and other components that expect that resource format. These improvements make `ExecutionResources` easier to use in scheduling and autoscaling components. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Balaji Veeramani --- .../execution/interfaces/execution_options.py | 19 +++++++++++++++++++ .../test_executor_resource_management.py | 10 ++++++++++ 2 files changed, 29 insertions(+) diff --git a/python/ray/data/_internal/execution/interfaces/execution_options.py b/python/ray/data/_internal/execution/interfaces/execution_options.py index 0485a12ec68c..3edfa2dceda5 100644 --- a/python/ray/data/_internal/execution/interfaces/execution_options.py +++ b/python/ray/data/_internal/execution/interfaces/execution_options.py @@ -49,6 +49,15 @@ def from_resource_dict( memory=resource_dict.get("memory", None), ) + def to_resource_dict(self) -> Dict[str, float]: + """Convert this ExecutionResources object to a resource dict.""" + return { + "CPU": self.cpu, + "GPU": self.gpu, + "object_store_memory": self.object_store_memory, + "memory": self.memory, + } + @classmethod def for_limits( cls, @@ -102,6 +111,16 @@ def __eq__(self, other: "ExecutionResources") -> bool: and self.memory == other.memory ) + def __hash__(self) -> int: + return hash( + ( + self.cpu, + self.gpu, + self.object_store_memory, + self.memory, + ) + ) + @classmethod def zero(cls) -> "ExecutionResources": """Returns an ExecutionResources object with zero resources.""" diff --git a/python/ray/data/tests/test_executor_resource_management.py b/python/ray/data/tests/test_executor_resource_management.py index 73e5a23326fb..3a2836cce545 100644 --- a/python/ray/data/tests/test_executor_resource_management.py +++ b/python/ray/data/tests/test_executor_resource_management.py @@ -581,6 +581,16 @@ def test_output_splitter_resource_reporting(ray_start_10_cpus_shared): assert op.metrics.obj_store_mem_internal_outqueue == 0 +def test_execution_resources_to_resource_dict(): + resources = ExecutionResources(cpu=1, gpu=2, object_store_memory=3, memory=4) + assert resources.to_resource_dict() == { + "CPU": 1, + "GPU": 2, + "object_store_memory": 3, + "memory": 4, + } + + if __name__ == "__main__": import sys From b5a6ad9300ce5f3da9fab9d5f875babc2cc7be2e Mon Sep 17 00:00:00 2001 From: Jiajun Yao Date: Tue, 9 Sep 2025 13:58:59 -0700 Subject: [PATCH 540/634] [Core] Rewrite JobManager _monitor_job_internal to fix hanging issue and wrong job failure reason issue (#56296) Signed-off-by: Jiajun Yao --- python/ray/dashboard/modules/job/common.py | 17 ++- .../ray/dashboard/modules/job/job_manager.py | 99 ++++++-------- .../modules/job/tests/test_job_manager.py | 128 ++++++++++-------- 3 files changed, 127 insertions(+), 117 deletions(-) diff --git a/python/ray/dashboard/modules/job/common.py b/python/ray/dashboard/modules/job/common.py index 3f205e6478d2..4466740f0ed8 100644 --- a/python/ray/dashboard/modules/job/common.py +++ b/python/ray/dashboard/modules/job/common.py @@ -260,7 +260,11 @@ def __init__( ) async def put_info( - self, job_id: str, job_info: JobInfo, overwrite: bool = True + self, + job_id: str, + job_info: JobInfo, + overwrite: bool = True, + timeout: Optional[int] = 30, ) -> bool: """Put job info to the internal kv store. @@ -268,6 +272,7 @@ async def put_info( job_id: The job id. job_info: The job info. overwrite: Whether to overwrite the existing job info. + timeout: The timeout in seconds for the GCS operation. Returns: True if a new key is added. @@ -277,6 +282,7 @@ async def put_info( json.dumps(job_info.to_json()).encode(), overwrite, namespace=ray_constants.KV_NAMESPACE_JOB, + timeout=timeout, ) if added_num == 1 or overwrite: # Write export event if data was updated in the KV store @@ -353,10 +359,11 @@ async def put_status( driver_exit_code: Optional[int] = None, error_type: Optional[JobErrorType] = None, jobinfo_replace_kwargs: Optional[Dict[str, Any]] = None, + timeout: Optional[int] = 30, ): """Puts or updates job status. Sets end_time if status is terminal.""" - old_info = await self.get_info(job_id) + old_info = await self.get_info(job_id, timeout=timeout) if jobinfo_replace_kwargs is None: jobinfo_replace_kwargs = dict() @@ -378,10 +385,10 @@ async def put_status( if status.is_terminal(): new_info.end_time = int(time.time() * 1000) - await self.put_info(job_id, new_info) + await self.put_info(job_id, new_info, timeout=timeout) - async def get_status(self, job_id: str) -> Optional[JobStatus]: - job_info = await self.get_info(job_id) + async def get_status(self, job_id: str, timeout: int = 30) -> Optional[JobStatus]: + job_info = await self.get_info(job_id, timeout) if job_info is None: return None else: diff --git a/python/ray/dashboard/modules/job/job_manager.py b/python/ray/dashboard/modules/job/job_manager.py index 142d8d521fb0..22692757e4eb 100644 --- a/python/ray/dashboard/modules/job/job_manager.py +++ b/python/ray/dashboard/modules/job/job_manager.py @@ -43,11 +43,6 @@ logger = logging.getLogger(__name__) -RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES = ray_constants.env_integer( - "RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES", 5 -) - - def generate_job_id() -> str: """Returns a job_id of the form 'raysubmit_XYZ'. @@ -166,12 +161,11 @@ async def _monitor_job_internal( ) ) - should_monitor = True - num_consecutive_failures = 0 - job_status = None + job_info = None + ping_obj_ref = None - while should_monitor: + while True: try: # NOTE: Job monitoring loop sleeps before proceeding with monitoring # sequence to consolidate the control-flow of the pacing @@ -179,12 +173,17 @@ async def _monitor_job_internal( # many branches await asyncio.sleep(self.JOB_MONITOR_LOOP_PERIOD_S) - job_status = await self._job_info_client.get_status(job_id) + job_status = await self._job_info_client.get_status( + job_id, timeout=None + ) if job_status == JobStatus.PENDING: # Compare the current time with the job start time. # If the job is still pending, we will set the status # to FAILED. - job_info = await self._job_info_client.get_info(job_id) + if job_info is None: + job_info = await self._job_info_client.get_info( + job_id, timeout=None + ) if time.time() - job_info.start_time / 1000 > timeout: err_msg = ( @@ -226,10 +225,10 @@ async def _monitor_job_internal( JobStatus.FAILED, message=err_msg, error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_TIMEOUT, + timeout=None, ) - should_monitor = False logger.error(err_msg) - continue + break if job_supervisor is None: job_supervisor = self._get_actor_for_job(job_id) @@ -253,22 +252,31 @@ async def _monitor_job_internal( "failed to get job supervisor." ), error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_FAILURE, + timeout=None, ) - should_monitor = False - continue - - # Verify `JobSupervisor` is alive and reachable - await job_supervisor.ping.remote() - # Reset consecutive failures counter - num_consecutive_failures = 0 + break + + # Check to see if `JobSupervisor` is alive and reachable + if ping_obj_ref is None: + ping_obj_ref = job_supervisor.ping.options( + max_task_retries=-1 + ).remote() + ready, _ = ray.wait([ping_obj_ref], timeout=0) + if ready: + ray.get(ping_obj_ref) + ping_obj_ref = None + else: + continue except Exception as e: + job_status = await self._job_info_client.get_status( + job_id, timeout=None + ) target_job_error_message = "" target_job_error_type: Optional[JobErrorType] = None if job_status is not None and job_status.is_terminal(): # If the job is already in a terminal state, then the actor # exiting is expected. - should_monitor = False pass else: if isinstance(e, RuntimeEnvSetupError): @@ -296,46 +304,27 @@ async def _monitor_job_internal( target_job_error_type = JobErrorType.JOB_SUPERVISOR_ACTOR_DIED else: - logger.warning( + logger.error( f"Job monitoring for job {job_id} failed " f"unexpectedly: {e}.", exc_info=e, ) - if ( - num_consecutive_failures - < RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES - ): - num_consecutive_failures += 1 - continue - else: - logger.error( - f"Job monitoring failed more than " - f"{RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES} " - f"times, marking job as failed", - exc_info=e, - ) - - target_job_error_message = f"Unexpected error occurred: {e}" - target_job_error_type = ( - JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE - ) - - # If target job error message is set it entails that the job ought - # to be marked as failed (and terminated) - if target_job_error_message: - # Terminate monitoring loop - should_monitor = False - - job_status = JobStatus.FAILED - await self._job_info_client.put_status( - job_id, - job_status, - message=target_job_error_message, - error_type=target_job_error_type - or JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE, + target_job_error_message = f"Unexpected error occurred: {e}" + target_job_error_type = ( + JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE ) + job_status = JobStatus.FAILED + await self._job_info_client.put_status( + job_id, + job_status, + message=target_job_error_message, + error_type=target_job_error_type + or JobErrorType.JOB_SUPERVISOR_ACTOR_UNKNOWN_FAILURE, + timeout=None, + ) + # Log error message to the job driver file for easy access. if target_job_error_message: log_path = self._log_client.get_log_file_path(job_id) @@ -354,6 +343,8 @@ async def _monitor_job_internal( else: self.event_logger.info(event_log, submission_id=job_id) + break + # Kill the actor defensively to avoid leaking actors in unexpected error cases. if job_supervisor is not None: ray.kill(job_supervisor, no_restart=True) diff --git a/python/ray/dashboard/modules/job/tests/test_job_manager.py b/python/ray/dashboard/modules/job/tests/test_job_manager.py index e106ef8cf0a2..8f6c37d73d13 100644 --- a/python/ray/dashboard/modules/job/tests/test_job_manager.py +++ b/python/ray/dashboard/modules/job/tests/test_job_manager.py @@ -5,7 +5,6 @@ import tempfile import time import urllib.request -from unittest.mock import AsyncMock from uuid import uuid4 import pytest @@ -29,7 +28,6 @@ ) from ray.dashboard.modules.job.common import JOB_ID_METADATA_KEY, JOB_NAME_METADATA_KEY from ray.dashboard.modules.job.job_manager import ( - RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES, JobLogStorageClient, JobManager, JobSupervisor, @@ -40,7 +38,6 @@ create_job_manager, create_ray_cluster, ) -from ray.exceptions import RpcError from ray.job_submission import JobErrorType, JobStatus from ray.tests.conftest import call_ray_start # noqa: F401 from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy # noqa: F401 @@ -354,6 +351,37 @@ async def test_runtime_env_setup_logged_to_job_driver_logs( assert start_message in logs +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_ray_start", + [ + { + "cmd": "ray start --head", + "env": { + "RAY_testing_rpc_failure": "ray::rpc::InternalKVGcsService.grpc_client.InternalKVGet=2:50:50,CoreWorkerService.grpc_client.PushTask=3:50:50" + }, + }, + ], + indirect=True, +) +async def test_job_manager_network_fault_tolerance( + call_ray_start, tmp_path # noqa: F811 +): + """Test that the job manager is tolerant to transient network failures + when making RPCs to GCS and supervisor actor.""" + + ray.init(address=call_ray_start) + gcs_client = ray._private.worker.global_worker.gcs_client + job_manager = JobManager(gcs_client, tmp_path) + + job_id = await job_manager.submit_job( + entrypoint="echo hello 1", + ) + await async_wait_for_condition( + check_job_succeeded, job_manager=job_manager, job_id=job_id + ) + + @pytest.fixture def shared_ray_instance(): # Remove ray address for test ray cluster in case we have @@ -1355,6 +1383,44 @@ async def test_monitor_job_pending(job_manager): ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_ray_start", + ["ray start --head --num-cpus=1"], + indirect=True, +) +async def test_job_timeout_lack_of_entrypoint_resources( + call_ray_start, tmp_path, monkeypatch # noqa: F811 +): + """Test the timeout when there are not enough resources to schedule the supervisor actor)""" + + monkeypatch.setenv(RAY_JOB_START_TIMEOUT_SECONDS_ENV_VAR, "1") + + ray.init(address=call_ray_start) + gcs_client = ray._private.worker.global_worker.gcs_client + job_manager = JobManager(gcs_client, tmp_path) + + # Submit a job with unsatisfied resource. + job_id = await job_manager.submit_job( + entrypoint="echo 'hello world'", + entrypoint_num_cpus=2, + ) + + # Wait for the job to timeout. + await async_wait_for_condition( + check_job_failed, + job_manager=job_manager, + job_id=job_id, + expected_error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_START_TIMEOUT, + ) + + # Check that the job timed out. + job_info = await job_manager.get_job_info(job_id) + assert job_info.status == JobStatus.FAILED + assert "Job supervisor actor failed to start within" in job_info.message + assert job_info.driver_exit_code is None + + @pytest.mark.asyncio async def test_job_pending_timeout(job_manager, monkeypatch): """Test the timeout for pending jobs.""" @@ -1444,6 +1510,7 @@ async def test_actor_creation_error_not_overwritten(shared_ray_instance, tmp_pat assert data.driver_exit_code is None +@pytest.mark.asyncio async def test_no_task_events_exported(shared_ray_instance, tmp_path): """Verify that no task events are exported by the JobSupervisor.""" job_manager = create_job_manager(shared_ray_instance, tmp_path) @@ -1460,60 +1527,5 @@ async def test_no_task_events_exported(shared_ray_instance, tmp_path): assert "JobSupervisor" not in t.name -@pytest.mark.parametrize( - "max_failures,expected_job_status", - [ - (RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES - 1, JobStatus.SUCCEEDED), - (RAY_JOB_MANAGER_MONITOR_MAX_CONSECUTIVE_FAILURES + 1, JobStatus.FAILED), - ], -) -async def test_job_manager_tolerates_gcs_failures( - job_manager, max_failures, expected_job_status -): - """Test driver exit code from finished task that failed""" - - original_get_info = job_manager._job_info_client.get_info - - num_failures = 0 - - async def _failing_get_info(*args, **kwargs): - nonlocal num_failures - - if num_failures < max_failures: - num_failures += 1 - raise RpcError("deadline exceeded") - else: - return await original_get_info(*args, **kwargs) - - # Mock out `JobManager._job_info_client` - job_manager._job_info_client.get_info = AsyncMock(side_effect=_failing_get_info) - - # Override `JobManager`s monitoring frequency to 100ms - job_manager.JOB_MONITOR_LOOP_PERIOD_S = 0.1 - - # Simulate job running for 5 seconds - job_id = await job_manager.submit_job(entrypoint="sleep 3; echo 'hello world'") - - if expected_job_status == JobStatus.FAILED: - expected_job_state_check = _check_job_failed - elif expected_job_status == JobStatus.SUCCEEDED: - expected_job_state_check = _check_job_succeeded - else: - raise NotImplementedError(f"unexpected job status: {expected_job_status}") - - # Wait for the job to reach expected target state - await async_wait_for_condition( - expected_job_state_check, - timeout=10, - get_job_info=original_get_info, - job_id=job_id, - ) - - # Check that the job failed - job_info = await job_manager.get_job_info(job_id) - - assert job_info.status == expected_job_status - - if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) From 4ac91b3f088047369d0a4084764006d645fd4522 Mon Sep 17 00:00:00 2001 From: gangsf Date: Tue, 9 Sep 2025 16:35:04 -0700 Subject: [PATCH 541/634] [Core] Add S3 public bucket fallback to handle NoCredentialsError (#56334) Signed-off-by: Gang Zhao Co-authored-by: Gang Zhao --- python/ray/_private/runtime_env/protocol.py | 15 ++- .../ray/tests/test_runtime_env_packaging.py | 93 +++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/python/ray/_private/runtime_env/protocol.py b/python/ray/_private/runtime_env/protocol.py index 12d562c6159a..663e9d4366da 100644 --- a/python/ray/_private/runtime_env/protocol.py +++ b/python/ray/_private/runtime_env/protocol.py @@ -58,7 +58,20 @@ def _handle_s3_protocol(cls): "to fetch URIs in s3 bucket. " + cls._MISSING_DEPENDENCIES_WARNING ) - transport_params = {"client": boto3.client("s3")} + # Create S3 client, falling back to unsigned for public buckets + session = boto3.Session() + # session.get_credentials() will return None if no credentials can be found. + if session.get_credentials(): + # If credentials are found, use a standard signed client. + s3_client = session.client("s3") + else: + # No credentials found, fall back to an unsigned client for public buckets. + from botocore import UNSIGNED + from botocore.config import Config + + s3_client = boto3.client("s3", config=Config(signature_version=UNSIGNED)) + + transport_params = {"client": s3_client} return open_file, transport_params @classmethod diff --git a/python/ray/tests/test_runtime_env_packaging.py b/python/ray/tests/test_runtime_env_packaging.py index 98e72caf4f07..8c015f8bf76e 100644 --- a/python/ray/tests/test_runtime_env_packaging.py +++ b/python/ray/tests/test_runtime_env_packaging.py @@ -676,6 +676,99 @@ def test_abfss_protocol_handler_with_invalid_uris(self, tmp_path): Protocol.ABFSS.download_remote_uri(invalid_uri, str(dest_file)) +class TestS3Protocol: + """Test S3 protocol implementation with public bucket fallback.""" + + def test_s3_client_creation_with_credentials(self): + """Test S3 client creation when credentials are available.""" + import sys + import unittest.mock as mock + + # Mock boto3 and smart_open modules + mock_boto3 = mock.MagicMock() + mock_smart_open = mock.MagicMock() + + # Setup successful credential scenario + mock_session = mock.MagicMock() + mock_s3_client = mock.MagicMock() + mock_credentials = mock.MagicMock() # Non-None credentials + + mock_boto3.Session.return_value = mock_session + mock_session.get_credentials.return_value = mock_credentials + mock_session.client.return_value = mock_s3_client + + with mock.patch.dict( + sys.modules, + { + "boto3": mock_boto3, + "smart_open": mock_smart_open, + }, + ): + mock_smart_open.open = mock.MagicMock() + + from ray._private.runtime_env.protocol import ProtocolsProvider + + open_file, transport_params = ProtocolsProvider._handle_s3_protocol() + + # Verify that Session was created and get_credentials was called + mock_boto3.Session.assert_called_once() + mock_session.get_credentials.assert_called_once() + # Verify that session.client was called to create signed S3 client + mock_session.client.assert_called_with("s3") + # Verify that the signed client is returned + assert transport_params["client"] == mock_s3_client + + def test_s3_client_creation_without_credentials(self): + """Test S3 client creation falls back to unsigned when no credentials.""" + import sys + import unittest.mock as mock + + # Mock boto3 and botocore modules + mock_boto3 = mock.MagicMock() + mock_botocore = mock.MagicMock() + mock_smart_open = mock.MagicMock() + + # Setup no credentials scenario + mock_session = mock.MagicMock() + mock_unsigned_client = mock.MagicMock() + + mock_boto3.Session.return_value = mock_session + mock_session.get_credentials.return_value = None # No credentials found + mock_boto3.client.return_value = mock_unsigned_client + + # Mock Config and UNSIGNED + mock_config_class = mock.MagicMock() + mock_config = mock.MagicMock() + mock_config_class.return_value = mock_config + mock_botocore.config.Config = mock_config_class + mock_botocore.UNSIGNED = "UNSIGNED" + + with mock.patch.dict( + sys.modules, + { + "boto3": mock_boto3, + "botocore": mock_botocore, + "botocore.config": mock_botocore.config, + "smart_open": mock_smart_open, + }, + ): + mock_smart_open.open = mock.MagicMock() + + from ray._private.runtime_env.protocol import ProtocolsProvider + + open_file, transport_params = ProtocolsProvider._handle_s3_protocol() + + # Verify that Session was created and get_credentials was called + mock_boto3.Session.assert_called_once() + mock_session.get_credentials.assert_called_once() + # Verify that boto3.client was called for unsigned client with config + mock_boto3.client.assert_called_with("s3", config=mock_config) + # Verify Config was created with UNSIGNED signature + mock_config_class.assert_called_with(signature_version="UNSIGNED") + # Verify that the unsigned client is returned + assert transport_params["client"] == mock_unsigned_client + + @pytest.mark.asyncio class TestDownloadAndUnpackPackage: async def test_download_and_unpack_package_with_gcs_uri_without_gcs_client( From 6ff42bff8743bab5795dd5cb88757267ab946fd2 Mon Sep 17 00:00:00 2001 From: Justin Yu Date: Tue, 9 Sep 2025 17:40:22 -0700 Subject: [PATCH 542/634] [release-test] Disable `drop_last` flag to fix division by zero in torch dataloader baselines (#56395) https://github.com/ray-project/ray/pull/56343 refactored some code for torch dataloader creation but introduced a bug when it came to the validation dataset throughput calculation. This happened because `drop_last=True` became the default setting, which would cause the validation dataset to be empty since it's small enough and spread across enough workers so that we couldn't form a single full batch. This PR fixes the issue by setting `drop_last=False`. --------- Signed-off-by: Justin Yu --- release/train_tests/benchmark/runner.py | 1 + release/train_tests/benchmark/torch_dataloader_factory.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/release/train_tests/benchmark/runner.py b/release/train_tests/benchmark/runner.py index ab20c9205a40..5c87f41674bf 100644 --- a/release/train_tests/benchmark/runner.py +++ b/release/train_tests/benchmark/runner.py @@ -201,6 +201,7 @@ def _validate_epoch(self) -> Dict[str, float]: self._metrics["validation/rows_processed"].add( self.benchmark_config.dataloader_config.validation_batch_size ) + assert num_rows > 0, "Validation dataset yielded no batches." return {"validation/loss": total_loss.item() / num_rows} diff --git a/release/train_tests/benchmark/torch_dataloader_factory.py b/release/train_tests/benchmark/torch_dataloader_factory.py index 079a1ecc5d40..733c15d497ca 100644 --- a/release/train_tests/benchmark/torch_dataloader_factory.py +++ b/release/train_tests/benchmark/torch_dataloader_factory.py @@ -149,7 +149,7 @@ def _create_dataloader(self, dataset_key: DatasetKey, batch_size: int): pin_memory=pin_memory, prefetch_factor=prefetch_factor, timeout=timeout, - drop_last=True, + drop_last=False, **multiprocessing_args, ) # Add a DistributedSampler to the dataloader if possible (map-style datasets) From 818a93a176e8fcd266728f6ab9d099a5bb98c5e4 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 9 Sep 2025 19:01:31 -0700 Subject: [PATCH 543/634] [ci] remove old wheel building logic (#56375) only keep parts that are still being used in macos wheel building Signed-off-by: Lonnie Liu --- ci/ci.sh | 109 +++++++----------------------- ci/ray_ci/macos/macos_ci_build.sh | 3 +- 2 files changed, 25 insertions(+), 87 deletions(-) diff --git a/ci/ci.sh b/ci/ci.sh index 8afa3ccc78a6..c228b0e3aa67 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -242,25 +242,20 @@ install_ray() { ) } -validate_wheels_commit_str() { - if [ "${OSTYPE}" = msys ]; then - echo "Windows builds do not set the commit string, skipping wheel commit validity check." - return 0 - fi - - if [ -n "${BUILDKITE_COMMIT}" ]; then - EXPECTED_COMMIT=${BUILDKITE_COMMIT:-} +_validate_macos_wheels_commit_str() { + if [[ -n "${BUILDKITE_COMMIT}" ]]; then + EXPECTED_COMMIT="${BUILDKITE_COMMIT:-}" else - EXPECTED_COMMIT=${TRAVIS_COMMIT:-} + EXPECTED_COMMIT="$(git rev-parse HEAD)" fi - if [ -z "$EXPECTED_COMMIT" ]; then - echo "Could not validate expected wheel commits: TRAVIS_COMMIT is empty." - return 0 + if [[ -z "$EXPECTED_COMMIT" ]]; then + echo "Could not validate expected wheel commits: BUILDKITE_COMMIT is empty." >&2 + exit 1 fi for whl in .whl/*.whl; do - basename=${whl##*/} + basename="${whl##*/}" if [[ "$basename" =~ "_cpp" ]]; then # cpp wheels cannot be checked this way @@ -281,85 +276,29 @@ validate_wheels_commit_str() { echo "All wheels passed the sanity check and have the correct wheel commit set." } -build_wheels_and_jars() { +build_macos_wheels_and_jars() { + if [[ "${OSTYPE}" != darwin* ]]; then + echo "Not on macOS" + exit 1 + fi + _bazel_build_before_install # Create wheel output directory and empty contents # If buildkite runners are re-used, wheels from previous builds might be here, so we delete them. + rm -rf .whl mkdir -p .whl - rm -rf .whl/* || true - - case "${OSTYPE}" in - linux*) - # Mount bazel cache dir to the docker container. - # For the linux wheel build, we use a shared cache between all - # wheels, but not between different travis runs, because that - # caused timeouts in the past. See the "cache: false" line below. - local MOUNT_BAZEL_CACHE=( - -e "TRAVIS=true" - -e "TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST:-false}" - -e "TRAVIS_COMMIT=${TRAVIS_COMMIT}" - -e "CI=${CI}" - -e "RAY_INSTALL_JAVA=${RAY_INSTALL_JAVA:-1}" - -e "BUILDKITE=${BUILDKITE:-}" - -e "BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST:-}" - -e "BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL:-}" - -e "RAY_DEBUG_BUILD=${RAY_DEBUG_BUILD:-}" - -e "BUILD_ONE_PYTHON_ONLY=${BUILD_ONE_PYTHON_ONLY:-}" - ) - - IMAGE_NAME="quay.io/pypa/manylinux2014_${HOSTTYPE}" - IMAGE_TAG="2022-12-20-b4884d9" - - local MOUNT_ENV=() - if [[ "${LINUX_JARS-}" == "1" ]]; then - MOUNT_ENV+=(-e "BUILD_JAR=1") - fi - if [[ -z "${BUILDKITE-}" ]]; then - # This command should be kept in sync with ray/python/README-building-wheels.md, - # except the "${MOUNT_BAZEL_CACHE[@]}" part. - docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \ - "${MOUNT_ENV[@]}" "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh - else - rm -rf /ray-mount/* - rm -rf /ray-mount/.whl || true - rm -rf /ray/.whl || true - cp -rT /ray /ray-mount - ls -a /ray-mount - docker run --rm -w /ray -v /ray:/ray "${MOUNT_BAZEL_CACHE[@]}" \ - "${MOUNT_ENV[@]}" "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh - cp -rT /ray-mount /ray # copy new files back here - find . | grep whl # testing - - # Sync the directory to buildkite artifacts - rm -rf /artifact-mount/.whl || true - - if [ "${UPLOAD_WHEELS_AS_ARTIFACTS-}" = "1" ]; then - cp -r .whl /artifact-mount/.whl - chmod -R 777 /artifact-mount/.whl - fi - - validate_wheels_commit_str - fi - ;; - darwin*) - # This command should be kept in sync with ray/python/README-building-wheels.md. - "${WORKSPACE_DIR}"/python/build-wheel-macos.sh - mkdir -p /tmp/artifacts/.whl - rm -rf /tmp/artifacts/.whl || true - - if [[ "${UPLOAD_WHEELS_AS_ARTIFACTS-}" == "1" ]]; then - cp -r .whl /tmp/artifacts/.whl - chmod -R 777 /tmp/artifacts/.whl - fi + # This command should be kept in sync with ray/python/README-building-wheels.md. + "${WORKSPACE_DIR}"/python/build-wheel-macos.sh + + mkdir -p /tmp/artifacts + rm -rf /tmp/artifacts/.whl + cp -r .whl /tmp/artifacts/.whl + chmod 755 /tmp/artifacts/.whl + chmod 644 /tmp/artifacts/.whl/* - validate_wheels_commit_str - ;; - msys*) - "${WORKSPACE_DIR}"/python/build-wheel-windows.sh - ;; - esac + _validate_macos_wheels_commit_str } configure_system() { diff --git a/ci/ray_ci/macos/macos_ci_build.sh b/ci/ray_ci/macos/macos_ci_build.sh index 810b9cc540af..242dbcb618ee 100755 --- a/ci/ray_ci/macos/macos_ci_build.sh +++ b/ci/ray_ci/macos/macos_ci_build.sh @@ -32,14 +32,13 @@ build() { export JAVA_HOME=/Library/Java/JavaVirtualMachines/temurin-8.jdk/Contents/Home java -version # Build wheels - export UPLOAD_WHEELS_AS_ARTIFACTS=1 export MAC_WHEELS=1 export MAC_JARS=1 export RAY_INSTALL_JAVA=1 export RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER=1 . ./ci/ci.sh init && source ~/.zshenv source ~/.zshrc - ./ci/ci.sh build_wheels_and_jars + ./ci/ci.sh build_macos_wheels_and_jars # Test wheels ./ci/ci.sh test_macos_wheels # Build jars From 7bb884c8a066166d56bb8596cb9a630331da855b Mon Sep 17 00:00:00 2001 From: Aydin Abiar <62435714+Aydin-ab@users.noreply.github.com> Date: Tue, 9 Sep 2025 20:56:55 -0700 Subject: [PATCH 544/634] [docs] [serve] [llm] Fix serve llm examples (#56382) --- doc/source/serve/examples.yml | 12 ++--- .../deployment-serve-llm/README.ipynb | 12 ++--- .../tutorials/deployment-serve-llm/README.md | 12 ++--- .../hybrid-reasoning-llm/README.md | 45 ++++++------------- .../hybrid-reasoning-llm/client_streaming.py | 6 +-- .../client_thinking_disabled.py | 6 +-- .../client_thinking_enabled.py | 6 +-- .../hybrid-reasoning-llm/notebook.ipynb | 45 ++++++------------- .../large-size-llm/README.md | 10 ++--- .../large-size-llm/client.py | 6 +-- .../large-size-llm/notebook.ipynb | 10 ++--- .../medium-size-llm/README.md | 12 ++--- .../medium-size-llm/client.py | 6 +-- .../medium-size-llm/notebook.ipynb | 12 ++--- .../reasoning-llm/README.md | 18 ++++---- .../reasoning-llm/client.py | 6 +-- .../reasoning-llm/client_streaming.py | 6 +-- .../reasoning-llm/notebook.ipynb | 18 ++++---- .../small-size-llm/README.md | 10 ++--- .../small-size-llm/client.py | 6 +-- .../small-size-llm/notebook.ipynb | 10 ++--- .../deployment-serve-llm/vision-llm/README.md | 18 ++++---- .../vision-llm/client_local_image.py | 6 +-- .../vision-llm/client_url_image.py | 6 +-- .../vision-llm/notebook.ipynb | 18 ++++---- 25 files changed, 142 insertions(+), 180 deletions(-) diff --git a/doc/source/serve/examples.yml b/doc/source/serve/examples.yml index e1077becdf6d..56416a1e0b4c 100644 --- a/doc/source/serve/examples.yml +++ b/doc/source/serve/examples.yml @@ -74,7 +74,7 @@ examples: - natural language processing link: tutorials/serve-deepseek related_technology: llm applications - - title: Deploying a small-sized LLM + - title: Deploy a small-sized LLM skill_level: beginner use_cases: - generative ai @@ -82,7 +82,7 @@ examples: - natural language processing link: tutorials/deployment-serve-llm/small-size-llm/README related_technology: llm applications - - title: Deploying a medium-sized LLM + - title: Deploy a medium-sized LLM skill_level: beginner use_cases: - generative ai @@ -90,7 +90,7 @@ examples: - natural language processing link: tutorials/deployment-serve-llm/medium-size-llm/README related_technology: llm applications - - title: Deploying a large-sized LLM + - title: Deploy a large-sized LLM skill_level: beginner use_cases: - generative ai @@ -98,7 +98,7 @@ examples: - natural language processing link: tutorials/deployment-serve-llm/large-size-llm/README related_technology: llm applications - - title: Deploying a vision LLM + - title: Deploy a vision LLM skill_level: beginner use_cases: - generative ai @@ -106,7 +106,7 @@ examples: - natural language processing link: tutorials/deployment-serve-llm/vision-llm/README related_technology: llm applications - - title: Deploying a reasoning LLM + - title: Deploy a reasoning LLM skill_level: beginner use_cases: - generative ai @@ -114,7 +114,7 @@ examples: - natural language processing link: tutorials/deployment-serve-llm/reasoning-llm/README related_technology: llm applications - - title: Deploying a hybrid reasoning LLM + - title: Deploy a hybrid reasoning LLM skill_level: beginner use_cases: - generative ai diff --git a/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb index 0b90c73aca73..9dfa5c6f1980 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/README.ipynb @@ -13,32 +13,32 @@ "\n", "## Tutorial categories\n", "\n", - "**[Small-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html)** \n", + "**[Deploy a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html)** \n", "Deploy small-sized models on a single GPU, such as Llama 3 8 B, Mistral 7 B, or Phi-2. \n", "\n", "---\n", "\n", - "**[Medium-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html)** \n", + "**[Deploy a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html)** \n", "Deploy medium-sized models using tensor parallelism across 4—8 GPUs on a single node, such as Llama 3 70 B, Qwen 14 B, Mixtral 8x7 B. \n", "\n", "---\n", "\n", - "**[Large-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html)** \n", + "**[Deploy a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html)** \n", "Deploy massive models using pipeline parallelism across a multi-node cluster, such as Deepseek-R1 or Llama-Nemotron-253 B. \n", "\n", "---\n", "\n", - "**[Vision LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/vision-llm/README.html)** \n", + "**[Deploy a vision LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/vision-llm/README.html)** \n", "Deploy models with image and text input such as Qwen 2.5-VL-7 B-Instruct, MiniGPT-4, or Pixtral-12 B. \n", "\n", "---\n", "\n", - "**[Reasoning LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html)** \n", + "**[Deploy a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html)** \n", "Deploy models with reasoning capabilities designed for long-context tasks, coding, or tool use, such as QwQ-32 B. \n", "\n", "---\n", "\n", - "**[Hybrid thinking LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.html)** \n", + "**[Deploy a hybrid reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.html)** \n", "Deploy models that can switch between reasoning and non-reasoning modes for flexible usage, such as Qwen-3." ] } diff --git a/doc/source/serve/tutorials/deployment-serve-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/README.md index 4aa5318b28cd..666a99af10c5 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/README.md @@ -12,30 +12,30 @@ Each tutorial includes development and production setups, tips for configuring y ## Tutorial categories -**[Small-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html)** +**[Deploy a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html)** Deploy small-sized models on a single GPU, such as Llama 3 8 B, Mistral 7 B, or Phi-2. --- -**[Medium-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html)** +**[Deploy a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html)** Deploy medium-sized models using tensor parallelism across 4—8 GPUs on a single node, such as Llama 3 70 B, Qwen 14 B, Mixtral 8x7 B. --- -**[Large-sized LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html)** +**[Deploy a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html)** Deploy massive models using pipeline parallelism across a multi-node cluster, such as Deepseek-R1 or Llama-Nemotron-253 B. --- -**[Vision LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/vision-llm/README.html)** +**[Deploy a vision LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/vision-llm/README.html)** Deploy models with image and text input such as Qwen 2.5-VL-7 B-Instruct, MiniGPT-4, or Pixtral-12 B. --- -**[Reasoning LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html)** +**[Deploy a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html)** Deploy models with reasoning capabilities designed for long-context tasks, coding, or tool use, such as QwQ-32 B. --- -**[Hybrid thinking LLM deployment](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.html)** +**[Deploy a hybrid reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.html)** Deploy models that can switch between reasoning and non-reasoning modes for flexible usage, such as Qwen-3. diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md index 5a01ee73875c..cdd0ecf74f03 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/README.md @@ -27,7 +27,7 @@ This tutorial deploys a hybrid reasoning LLM using Ray Serve LLM. **Note:** Reasoning often benefits from long context windows (32K up to +1M tokens), high token throughput, low-temperature decoding (greedy sampling), and strong instruction tuning or scratchpad-style reasoning. -To see an example of deploying a purely reasoning model like *QwQ-32 B*, see [Deploying a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html). +To see an example of deploying a purely reasoning model like *QwQ-32 B*, see [Deploy a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html). --- @@ -70,7 +70,7 @@ See [Send request with thinking enabled](#send-request-with-thinking-enabled) or In thinking mode, hybrid models often separate _reasoning_ from the _final answer_ using tags like `...`. Without a proper parser, this reasoning may end up in the `content` field instead of the dedicated `reasoning_content` field. -To ensure that Ray Serve LLM correctly parses the reasoning output, configure a `reasoning_parser` in your Ray Serve LLM deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output. +To ensure that Ray Serve LLM correctly parses the reasoning output, configure a `reasoning_parser` in your Ray Serve LLM deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output. **Note:** For example, *Qwen-3* uses the `qwen3` parser. See the [vLLM docs](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#supported-models) or your model's documentation to find a supported parser, or [build your own](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#how-to-support-a-new-reasoning-model) if needed. ```yaml @@ -147,7 +147,7 @@ app = build_openai_app({"llm_configs": [llm_config]}) ``` -**Note:** Before moving to a production setup, migrate your settings to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. +**Note:** Before moving to a production setup, migrate your settings to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example. --- @@ -210,10 +210,10 @@ Example Python with `enable_thinking: False`: from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) # Example: Complex query with thinking process response = client.chat.completions.create( @@ -256,10 +256,10 @@ curl -X POST http://localhost:8000/v1/chat/completions \ from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) # Example: Complex query with thinking process response = client.chat.completions.create( @@ -293,7 +293,7 @@ serve shutdown -y ## Deploy to production with Anyscale services -For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without any code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *Qwen-32b* from this tutorial. +For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without any code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploy a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *Qwen-32b* from this tutorial. --- @@ -307,29 +307,10 @@ In thinking mode, hybrid reasoning models may take longer to begin generating th from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) - -# Example: Complex query with thinking process -response = client.chat.completions.create( - model="my-qwen-3-32b", - messages=[ - {"role": "user", "content": "What's the capital of France ?"} - ], - extra_body={"chat_template_kwargs": {"enable_thinking": True}} -) - -print(f"Reasoning: \n{response.choices[0].message.reasoning_content}\n\n") -print(f"Answer: \n {response.choices[0].message.content}") -from urllib.parse import urljoin -from openai import OpenAI - -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" - -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) # Example: Complex query with thinking process response = client.chat.completions.create( diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_streaming.py b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_streaming.py index 7e51b794445c..ea383b2649f0 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_streaming.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_streaming.py @@ -2,10 +2,10 @@ from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) # Example: Complex query with thinking process response = client.chat.completions.create( diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_disabled.py b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_disabled.py index 46aa914441ea..fcacfe43166f 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_disabled.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_disabled.py @@ -2,10 +2,10 @@ from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) # Example: Complex query with thinking process response = client.chat.completions.create( diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_enabled.py b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_enabled.py index f09b5868648d..f1ea4070ec3f 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_enabled.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/client_thinking_enabled.py @@ -2,10 +2,10 @@ from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) # Example: Complex query with thinking process response = client.chat.completions.create( diff --git a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb index 221e991e88d1..97e7bb17834e 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/hybrid-reasoning-llm/notebook.ipynb @@ -24,7 +24,7 @@ "\n", "**Note:** Reasoning often benefits from long context windows (32K up to +1M tokens), high token throughput, low-temperature decoding (greedy sampling), and strong instruction tuning or scratchpad-style reasoning.\n", "\n", - "To see an example of deploying a purely reasoning model like *QwQ-32 B*, see [Deploying a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html).\n", + "To see an example of deploying a purely reasoning model like *QwQ-32 B*, see [Deploy a reasoning LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/reasoning-llm/README.html).\n", "\n", "---\n", "\n", @@ -67,7 +67,7 @@ "\n", "In thinking mode, hybrid models often separate _reasoning_ from the _final answer_ using tags like `...`. Without a proper parser, this reasoning may end up in the `content` field instead of the dedicated `reasoning_content` field. \n", "\n", - "To ensure that Ray Serve LLM correctly parses the reasoning output, configure a `reasoning_parser` in your Ray Serve LLM deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output.\n", + "To ensure that Ray Serve LLM correctly parses the reasoning output, configure a `reasoning_parser` in your Ray Serve LLM deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output. \n", "**Note:** For example, *Qwen-3* uses the `qwen3` parser. See the [vLLM docs](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#supported-models) or your model's documentation to find a supported parser, or [build your own](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#how-to-support-a-new-reasoning-model) if needed.\n", "\n", "```yaml\n", @@ -154,7 +154,7 @@ "id": "32272280", "metadata": {}, "source": [ - "**Note:** Before moving to a production setup, migrate your settings to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "**Note:** Before moving to a production setup, migrate your settings to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example.\n", "\n", "---\n", "\n", @@ -245,10 +245,10 @@ "from urllib.parse import urljoin\n", "from openai import OpenAI\n", "\n", - "api_key = \"FAKE_KEY\"\n", - "base_url = \"http://localhost:8000\"\n", + "API_KEY = \"FAKE_KEY\"\n", + "BASE_URL = \"http://localhost:8000\"\n", "\n", - "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "client = OpenAI(BASE_URL=urljoin(BASE_URL, \"v1\"), API_KEY=API_KEY)\n", "\n", "# Example: Complex query with thinking process\n", "response = client.chat.completions.create(\n", @@ -313,10 +313,10 @@ "from urllib.parse import urljoin\n", "from openai import OpenAI\n", "\n", - "api_key = \"FAKE_KEY\"\n", - "base_url = \"http://localhost:8000\"\n", + "API_KEY = \"FAKE_KEY\"\n", + "BASE_URL = \"http://localhost:8000\"\n", "\n", - "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "client = OpenAI(BASE_URL=urljoin(BASE_URL, \"v1\"), API_KEY=API_KEY)\n", "\n", "# Example: Complex query with thinking process\n", "response = client.chat.completions.create(\n", @@ -366,7 +366,7 @@ "\n", "## Deploy to production with Anyscale services\n", "\n", - "For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without any code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *Qwen-32b* from this tutorial.\n", + "For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without any code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploy a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *Qwen-32b* from this tutorial.\n", "\n", "---\n", "\n", @@ -386,29 +386,10 @@ "from urllib.parse import urljoin\n", "from openai import OpenAI\n", "\n", - "api_key = \"FAKE_KEY\"\n", - "base_url = \"http://localhost:8000\"\n", + "API_KEY = \"FAKE_KEY\"\n", + "BASE_URL = \"http://localhost:8000\"\n", "\n", - "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", - "\n", - "# Example: Complex query with thinking process\n", - "response = client.chat.completions.create(\n", - " model=\"my-qwen-3-32b\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"What's the capital of France ?\"}\n", - " ],\n", - " extra_body={\"chat_template_kwargs\": {\"enable_thinking\": True}}\n", - ")\n", - "\n", - "print(f\"Reasoning: \\n{response.choices[0].message.reasoning_content}\\n\\n\")\n", - "print(f\"Answer: \\n {response.choices[0].message.content}\")\n", - "from urllib.parse import urljoin\n", - "from openai import OpenAI\n", - "\n", - "api_key = \"FAKE_KEY\"\n", - "base_url = \"http://localhost:8000\"\n", - "\n", - "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "client = OpenAI(BASE_URL=urljoin(BASE_URL, \"v1\"), API_KEY=API_KEY)\n", "\n", "# Example: Complex query with thinking process\n", "response = client.chat.completions.create(\n", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md index e40fd9d80c8e..379e4c0d0ba4 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/README.md @@ -64,7 +64,7 @@ app = build_openai_app({"llm_configs": [llm_config]}) ``` -**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. +**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example. --- @@ -125,10 +125,10 @@ Example Python: from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) response = client.chat.completions.create( model="my-deepseek-r1", @@ -176,7 +176,7 @@ For production deployment, use Anyscale services to deploy the Ray Serve app to The following template runs only on H100 GPUs in your self-hosted Anyscale cloud, as H100s aren't available in Anyscale’s public cloud. This example uses two nodes of type *8xH100-80 GB:208CPU-1830 GB* on an AWS cloud. -To provision nodes with 1000 GB of disk capacity, see [Changing the default disk size for GCP clusters](https://docs.anyscale.com/configuration/compute/gcp/#changing-the-default-disk-size) for Google Cloud Platform (GCP) or [Changing the default disk size for AWS clusters](https://docs.anyscale.com/configuration/compute/aws/#changing-the-default-disk-size) for Amazon Web Services (AWS). +To provision nodes with 1000 GB of disk capacity, see [Changing the default disk size for GCP clusters](https://docs.anyscale.com/configuration/compute/gcp#disk-size) for Google Cloud Platform (GCP) or [Changing the default disk size for AWS clusters](https://docs.anyscale.com/configuration/compute/aws#disk-size) for Amazon Web Services (AWS). --- diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/client.py b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/client.py index 0f76eb43e6b9..839f17958d3b 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/client.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/client.py @@ -2,10 +2,10 @@ from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) response = client.chat.completions.create( model="my-deepseek-r1", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb index d4744d6f304f..6764565d998e 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/large-size-llm/notebook.ipynb @@ -71,7 +71,7 @@ "id": "6b2231a5", "metadata": {}, "source": [ - "**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example.\n", "\n", "---\n", "\n", @@ -160,10 +160,10 @@ "from urllib.parse import urljoin\n", "from openai import OpenAI\n", "\n", - "api_key = \"FAKE_KEY\"\n", - "base_url = \"http://localhost:8000\"\n", + "API_KEY = \"FAKE_KEY\"\n", + "BASE_URL = \"http://localhost:8000\"\n", "\n", - "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "client = OpenAI(BASE_URL=urljoin(BASE_URL, \"v1\"), API_KEY=API_KEY)\n", "\n", "response = client.chat.completions.create(\n", " model=\"my-deepseek-r1\",\n", @@ -227,7 +227,7 @@ "\n", "The following template runs only on H100 GPUs in your self-hosted Anyscale cloud, as H100s aren't available in Anyscale’s public cloud. This example uses two nodes of type *8xH100-80 GB:208CPU-1830 GB* on an AWS cloud.\n", "\n", - "To provision nodes with 1000 GB of disk capacity, see [Changing the default disk size for GCP clusters](https://docs.anyscale.com/configuration/compute/gcp/#changing-the-default-disk-size) for Google Cloud Platform (GCP) or [Changing the default disk size for AWS clusters](https://docs.anyscale.com/configuration/compute/aws/#changing-the-default-disk-size) for Amazon Web Services (AWS). \n", + "To provision nodes with 1000 GB of disk capacity, see [Changing the default disk size for GCP clusters](https://docs.anyscale.com/configuration/compute/gcp#disk-size) for Google Cloud Platform (GCP) or [Changing the default disk size for AWS clusters](https://docs.anyscale.com/configuration/compute/aws#disk-size) for Amazon Web Services (AWS). \n", "\n", "---\n", "\n", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md index b40f78e86b6b..271080a43e9b 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/README.md @@ -8,11 +8,11 @@ Modify notebook.ipynb instead, then regenerate this file with: jupyter nbconvert "$notebook.ipynb" --to markdown --output "README.md" --> -# Deploying a medium-sized LLM +# Deploy a medium-sized LLM A medium LLM typically runs on a single node with 4-8 GPUs. It offers a balance between performance and efficiency. These models provide stronger accuracy and reasoning than small models while remaining more affordable and resource-friendly than very large ones. This makes them a solid choice for production workloads that need good quality at lower cost. They're also ideal for scaling applications where large models would be too slow or expensive. -This tutorial deploys a medium-sized LLM using Ray Serve LLM. For smaller models, see [Deploying a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html), and for larger models, see [Deploying a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html). +This tutorial deploys a medium-sized LLM using Ray Serve LLM. For smaller models, see [Deploy a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html), and for larger models, see [Deploy a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html). --- @@ -58,7 +58,7 @@ app = build_openai_app({"llm_configs": [llm_config]}) ``` -**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. +**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example. --- @@ -118,10 +118,10 @@ Example Python: from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) response = client.chat.completions.create( model="my-llama-3.1-70b", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/client.py b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/client.py index 4f51d2f67a46..6715ef57b451 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/client.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/client.py @@ -2,10 +2,10 @@ from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) response = client.chat.completions.create( model="my-llama-3.1-70b", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb index fa284ed6f02b..b6dd436002b8 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/medium-size-llm/notebook.ipynb @@ -5,11 +5,11 @@ "id": "f8f6fcbd", "metadata": {}, "source": [ - "# Deploying a medium-sized LLM\n", + "# Deploy a medium-sized LLM\n", "\n", "A medium LLM typically runs on a single node with 4-8 GPUs. It offers a balance between performance and efficiency. These models provide stronger accuracy and reasoning than small models while remaining more affordable and resource-friendly than very large ones. This makes them a solid choice for production workloads that need good quality at lower cost. They're also ideal for scaling applications where large models would be too slow or expensive.\n", "\n", - "This tutorial deploys a medium-sized LLM using Ray Serve LLM. For smaller models, see [Deploying a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html), and for larger models, see [Deploying a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html).\n", + "This tutorial deploys a medium-sized LLM using Ray Serve LLM. For smaller models, see [Deploy a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html), and for larger models, see [Deploy a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html).\n", "\n", "---\n", "\n", @@ -65,7 +65,7 @@ "id": "6b2231a5", "metadata": {}, "source": [ - "**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example.\n", "\n", "---\n", "\n", @@ -153,10 +153,10 @@ "from urllib.parse import urljoin\n", "from openai import OpenAI\n", "\n", - "api_key = \"FAKE_KEY\"\n", - "base_url = \"http://localhost:8000\"\n", + "API_KEY = \"FAKE_KEY\"\n", + "BASE_URL = \"http://localhost:8000\"\n", "\n", - "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "client = OpenAI(BASE_URL=urljoin(BASE_URL, \"v1\"), API_KEY=API_KEY)\n", "\n", "response = client.chat.completions.create(\n", " model=\"my-llama-3.1-70b\",\n", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md index d7a3aa48f702..ef7a943d6410 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/README.md @@ -43,7 +43,7 @@ If your input is clear and complete, a standard model is usually faster and more Reasoning models often separate *reasoning* from the *final answer* using tags like `...`. Without a proper parser, this reasoning may end up in the `content` field instead of the dedicated `reasoning_content` field. -To extract reasoning correctly, configure a `reasoning_parser` in your Ray Serve deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output. +To extract reasoning correctly, configure a `reasoning_parser` in your Ray Serve deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output. **Note:** For example, *QwQ* uses the `deepseek-r1` parser. Other models may require different parsers. See the [vLLM docs](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#supported-models) or your model's documentation to find a supported parser, or [build your own](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#how-to-support-a-new-reasoning-model) if needed. ```yaml @@ -122,7 +122,7 @@ app = build_openai_app({"llm_configs": [llm_config]}) ``` -**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. +**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example. --- @@ -181,10 +181,10 @@ Example Python: from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) response = client.chat.completions.create( model="my-qwq-32B", @@ -216,7 +216,7 @@ serve shutdown -y ## Deploy to production with Anyscale services -For production, use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *QwQ-32 B* used here. +For production, use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploy a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *QwQ-32 B* used here. --- @@ -230,10 +230,10 @@ Reasoning models may take longer to begin generating the main content. You can s from urllib.parse import urljoin from openai import OpenAI -api_key = -base_url = +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) # Example: Complex query with thinking process response = client.chat.completions.create( diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client.py b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client.py index 9b5d768acc65..e7ba801365a3 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client.py @@ -2,10 +2,10 @@ from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) response = client.chat.completions.create( model="my-qwq-32B", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client_streaming.py b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client_streaming.py index f5c896593a3c..d522e0867603 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client_streaming.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/client_streaming.py @@ -2,10 +2,10 @@ from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) # Example: Complex query with thinking process response = client.chat.completions.create( diff --git a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb index 7f60d2c396c2..5c29cfb6856f 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/reasoning-llm/notebook.ipynb @@ -40,7 +40,7 @@ "\n", "Reasoning models often separate *reasoning* from the *final answer* using tags like `...`. Without a proper parser, this reasoning may end up in the `content` field instead of the dedicated `reasoning_content` field.\n", "\n", - "To extract reasoning correctly, configure a `reasoning_parser` in your Ray Serve deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output.\n", + "To extract reasoning correctly, configure a `reasoning_parser` in your Ray Serve deployment. This tells vLLM how to isolate the model’s thought process from the rest of the output. \n", "**Note:** For example, *QwQ* uses the `deepseek-r1` parser. Other models may require different parsers. See the [vLLM docs](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#supported-models) or your model's documentation to find a supported parser, or [build your own](https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#how-to-support-a-new-reasoning-model) if needed.\n", "\n", "```yaml\n", @@ -129,7 +129,7 @@ "id": "d515e268", "metadata": {}, "source": [ - "**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example.\n", "\n", "---\n", "\n", @@ -216,10 +216,10 @@ "from urllib.parse import urljoin\n", "from openai import OpenAI\n", "\n", - "api_key = \"FAKE_KEY\"\n", - "base_url = \"http://localhost:8000\"\n", + "API_KEY = \"FAKE_KEY\"\n", + "BASE_URL = \"http://localhost:8000\"\n", "\n", - "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "client = OpenAI(BASE_URL=urljoin(BASE_URL, \"v1\"), API_KEY=API_KEY)\n", "\n", "response = client.chat.completions.create(\n", " model=\"my-qwq-32B\",\n", @@ -267,7 +267,7 @@ "\n", "## Deploy to production with Anyscale services\n", "\n", - "For production, use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *QwQ-32 B* used here.\n", + "For production, use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploy a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a medium-sized model like the *QwQ-32 B* used here.\n", "\n", "---\n", "\n", @@ -287,10 +287,10 @@ "from urllib.parse import urljoin\n", "from openai import OpenAI\n", "\n", - "api_key = \n", - "base_url = \n", + "API_KEY = \"FAKE_KEY\"\n", + "BASE_URL = \"http://localhost:8000\"\n", "\n", - "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "client = OpenAI(BASE_URL=urljoin(BASE_URL, \"v1\"), API_KEY=API_KEY)\n", "\n", "# Example: Complex query with thinking process\n", "response = client.chat.completions.create(\n", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md index 3523ece616b3..88c25c7c121f 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/README.md @@ -13,7 +13,7 @@ jupyter nbconvert "$notebook.ipynb" --to markdown --output "README.md" A small LLM runs on a single node with 1–2 GPUs, making it fast, inexpensive, and simple to use. It’s ideal for prototyping, lightweight applications, latency-critical use cases, cost-sensitive deployments, and environments with limited resources where efficiency matters more than peak accuracy. -For larger models, see [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html) or [Deploying a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html). +For larger models, see [Deploy a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html) or [Deploy a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html). --- @@ -52,7 +52,7 @@ app = build_openai_app({"llm_configs": [llm_config]}) ``` -**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. +**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example. --- @@ -113,10 +113,10 @@ Example Python: from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) response = client.chat.completions.create( model="my-llama-3.1-8b", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/client.py b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/client.py index 25025a73dbcf..397cda670371 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/client.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/client.py @@ -1,10 +1,10 @@ from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) response = client.chat.completions.create( model="my-llama-3.1-8b", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb index f4cd9ec72d34..b1e7796913f5 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/small-size-llm/notebook.ipynb @@ -10,7 +10,7 @@ "A small LLM runs on a single node with 1–2 GPUs, making it fast, inexpensive, and simple to use. It’s ideal for prototyping, lightweight applications, latency-critical use cases, cost-sensitive deployments, and environments with limited resources where efficiency matters more than peak accuracy.\n", "\n", "\n", - "For larger models, see [Deploying a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html) or [Deploying a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html).\n", + "For larger models, see [Deploy a medium-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/medium-size-llm/README.html) or [Deploy a large-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/large-size-llm/README.html).\n", "\n", "---\n", "\n", @@ -59,7 +59,7 @@ "id": "b17a7140", "metadata": {}, "source": [ - "**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: Production Guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "**Note:** Before moving to a production setup, migrate to using a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example.\n", "\n", "---\n", "\n", @@ -148,10 +148,10 @@ "from urllib.parse import urljoin\n", "from openai import OpenAI\n", "\n", - "api_key = \"FAKE_KEY\"\n", - "base_url = \"http://localhost:8000\"\n", + "API_KEY = \"FAKE_KEY\"\n", + "BASE_URL = \"http://localhost:8000\"\n", "\n", - "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "client = OpenAI(BASE_URL=urljoin(BASE_URL, \"v1\"), API_KEY=API_KEY)\n", "\n", "response = client.chat.completions.create(\n", " model=\"my-llama-3.1-8b\",\n", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md index 352153951931..1a8c2654464f 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md +++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/README.md @@ -8,7 +8,7 @@ Modify notebook.ipynb instead, then regenerate this file with: jupyter nbconvert "$notebook.ipynb" --to markdown --output "README.md" --> -# Deploying a vision LLM +# Deploy a vision LLM A vision LLM can interpret images as well as text, enabling tasks like answering questions about charts, analyzing photos, or combining visuals with instructions. It extends LLMs beyond language to support multimodal reasoning and richer applications. @@ -49,7 +49,7 @@ app = build_openai_app({"llm_configs": [llm_config]}) ``` -**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example. +**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example. --- @@ -108,10 +108,10 @@ Example Python with image URL: from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) response = client.chat.completions.create( model="my-qwen-VL", @@ -143,10 +143,10 @@ from urllib.parse import urljoin import base64 from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) ### From an image locally saved as `example.jpg` # Load and encode image as base64 @@ -192,7 +192,7 @@ serve shutdown -y ## Deploy to production with Anyscale services -For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a small-sized model like the *Qwen2.5-VL-7 B-Instruct* used in this tutorial. +For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploy a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a small-sized model like the *Qwen2.5-VL-7 B-Instruct* used in this tutorial. --- diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py index 200d46c49ab9..ac6d86f18be9 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_local_image.py @@ -3,10 +3,10 @@ import base64 from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) ### From an image locally saved as `example.jpg` # Load and encode image as base64 diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_url_image.py b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_url_image.py index c976e460f2dc..0d093af6169b 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_url_image.py +++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/client_url_image.py @@ -2,10 +2,10 @@ from urllib.parse import urljoin from openai import OpenAI -api_key = "FAKE_KEY" -base_url = "http://localhost:8000" +API_KEY = "FAKE_KEY" +BASE_URL = "http://localhost:8000" -client = OpenAI(base_url=urljoin(base_url, "v1"), api_key=api_key) +client = OpenAI(BASE_URL=urljoin(BASE_URL, "v1"), API_KEY=API_KEY) response = client.chat.completions.create( model="my-qwen-VL", diff --git a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb index 5c1c482a2e4c..fa340e74ab20 100644 --- a/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb +++ b/doc/source/serve/tutorials/deployment-serve-llm/vision-llm/notebook.ipynb @@ -5,7 +5,7 @@ "id": "23243c2e", "metadata": {}, "source": [ - "# Deploying a vision LLM\n", + "# Deploy a vision LLM\n", "\n", "A vision LLM can interpret images as well as text, enabling tasks like answering questions about charts, analyzing photos, or combining visuals with instructions. It extends LLMs beyond language to support multimodal reasoning and richer applications. \n", "\n", @@ -56,7 +56,7 @@ "id": "c76a6362", "metadata": {}, "source": [ - "**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs: production guide](https://docs.ray.io/en/latest/serve/llm/serving-llms.html#production-deployment) for an example.\n", + "**Note:** Before moving to a production setup, migrate to a [Serve config file](https://docs.ray.io/en/latest/serve/production-guide/config.html) to make your deployment version-controlled, reproducible, and easier to maintain for CI/CD pipelines. See [Serving LLMs - Quickstart Examples: Production Guide](https://docs.ray.io/en/latest/serve/llm/quick-start.html#production-deployment) for an example.\n", "\n", "---\n", "\n", @@ -143,10 +143,10 @@ "from urllib.parse import urljoin\n", "from openai import OpenAI\n", "\n", - "api_key = \"FAKE_KEY\"\n", - "base_url = \"http://localhost:8000\"\n", + "API_KEY = \"FAKE_KEY\"\n", + "BASE_URL = \"http://localhost:8000\"\n", "\n", - "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "client = OpenAI(BASE_URL=urljoin(BASE_URL, \"v1\"), API_KEY=API_KEY)\n", "\n", "response = client.chat.completions.create(\n", " model=\"my-qwen-VL\",\n", @@ -189,10 +189,10 @@ "import base64\n", "from openai import OpenAI\n", "\n", - "api_key = \"FAKE_KEY\"\n", - "base_url = \"http://localhost:8000\"\n", + "API_KEY = \"FAKE_KEY\"\n", + "BASE_URL = \"http://localhost:8000\"\n", "\n", - "client = OpenAI(base_url=urljoin(base_url, \"v1\"), api_key=api_key)\n", + "client = OpenAI(BASE_URL=urljoin(BASE_URL, \"v1\"), API_KEY=API_KEY)\n", "\n", "### From an image locally saved as `example.jpg`\n", "# Load and encode image as base64\n", @@ -254,7 +254,7 @@ "\n", "## Deploy to production with Anyscale services\n", "\n", - "For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploying a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a small-sized model like the *Qwen2.5-VL-7 B-Instruct* used in this tutorial.\n", + "For production, it's recommended to use Anyscale services to deploy your Ray Serve app on a dedicated cluster without code changes. Anyscale provides scalability, fault tolerance, and load balancing, ensuring resilience against node failures, high traffic, and rolling updates. See [Deploy a small-sized LLM](https://docs.ray.io/en/latest/serve/tutorials/deployment-serve-llm/small-size-llm/README.html#deploy-to-production-with-anyscale-services) for an example with a small-sized model like the *Qwen2.5-VL-7 B-Instruct* used in this tutorial.\n", "\n", "---\n", "\n", From 119aa6c9bdeb993e1a2f462df1b1db0b57a47fee Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Tue, 9 Sep 2025 21:28:22 -0700 Subject: [PATCH 545/634] Revert "[core] Correct bytes in flight when objects <5mb (#54349)" (#56387) Signed-off-by: dayshah --- src/ray/common/ray_config_def.h | 4 +- src/ray/object_manager/chunk_object_reader.h | 5 - src/ray/object_manager/object_manager.cc | 21 +- src/ray/object_manager/object_manager.h | 2 +- src/ray/object_manager/push_manager.cc | 26 +- src/ray/object_manager/push_manager.h | 46 ++-- .../object_manager/tests/push_manager_test.cc | 255 ++++++++---------- src/ray/raylet/main.cc | 2 - 8 files changed, 157 insertions(+), 204 deletions(-) diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index d496e52fed7b..14a7f9de4d5f 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -334,7 +334,9 @@ RAY_CONFIG(uint64_t, object_manager_default_chunk_size, 5 * 1024 * 1024) /// The maximum number of outbound bytes to allow to be outstanding. This avoids /// excessive memory usage during object broadcast to many receivers. -RAY_CONFIG(int64_t, object_manager_max_bytes_in_flight, (int64_t)2 * 1024 * 1024 * 1024) +RAY_CONFIG(uint64_t, + object_manager_max_bytes_in_flight, + ((uint64_t)2) * 1024 * 1024 * 1024) /// Maximum number of ids in one batch to send to GCS to delete keys. RAY_CONFIG(uint32_t, maximum_gcs_deletion_batch_size, 1000) diff --git a/src/ray/object_manager/chunk_object_reader.h b/src/ray/object_manager/chunk_object_reader.h index b70df5c1a9f5..097d2c84e863 100644 --- a/src/ray/object_manager/chunk_object_reader.h +++ b/src/ray/object_manager/chunk_object_reader.h @@ -14,7 +14,6 @@ #pragma once -#include #include #include @@ -42,10 +41,6 @@ class ChunkObjectReader { const IObjectReader &GetObject() const { return *object_; } - uint64_t ChunkSize() const { - return std::min(chunk_size_, object_->GetDataSize() + object_->GetMetadataSize()); - } - private: const std::shared_ptr object_; const uint64_t chunk_size_; diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index 4291637d3fd7..e43af1acc6a4 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -116,7 +116,10 @@ ObjectManager::ObjectManager( get_spilled_object_url_(std::move(get_spilled_object_url)), pull_retry_timer_(*main_service_, boost::posix_time::milliseconds(config.timer_freq_ms)), - push_manager_(std::make_unique(config_.max_bytes_in_flight)) { + push_manager_(std::make_unique(/* max_chunks_in_flight= */ std::max( + static_cast(1L), + static_cast(config_.max_bytes_in_flight / + config_.object_chunk_size)))) { RAY_CHECK_GT(config_.rpc_service_threads_number, 0); pull_retry_timer_.async_wait([this](const boost::system::error_code &e) { Tick(e); }); @@ -488,13 +491,8 @@ void ObjectManager::PushObjectInternal(const ObjectID &object_id, << ", total data size: " << chunk_reader->GetObject().GetObjectSize(); auto push_id = UniqueID::FromRandom(); - uint64_t push_max_chunk_size = chunk_reader->ChunkSize(); push_manager_->StartPush( - node_id, - object_id, - chunk_reader->GetNumChunks(), - push_max_chunk_size, - [=](int64_t chunk_id) { + node_id, object_id, chunk_reader->GetNumChunks(), [=](int64_t chunk_id) { rpc_service_.post( [=]() { // Post to the multithreaded RPC event loop so that data is copied @@ -505,14 +503,11 @@ void ObjectManager::PushObjectInternal(const ObjectID &object_id, node_id, chunk_id, rpc_client, - [this, push_max_chunk_size](const Status &status) { + [=](const Status &status) { // Post back to the main event loop because the // PushManager is not thread-safe. - this->main_service_->post( - [this, push_max_chunk_size]() { - this->push_manager_->OnChunkComplete(push_max_chunk_size); - }, - "ObjectManager.Push"); + main_service_->post([this]() { push_manager_->OnChunkComplete(); }, + "ObjectManager.Push"); }, chunk_reader, from_disk); diff --git a/src/ray/object_manager/object_manager.h b/src/ray/object_manager/object_manager.h index 4a1b3f4535fa..593f2d2b1455 100644 --- a/src/ray/object_manager/object_manager.h +++ b/src/ray/object_manager/object_manager.h @@ -53,7 +53,7 @@ struct ObjectManagerConfig { /// Object chunk size, in bytes uint64_t object_chunk_size; /// Max object push bytes in flight. - int64_t max_bytes_in_flight; + uint64_t max_bytes_in_flight; /// The store socket name. std::string store_socket_name; /// The time in milliseconds to wait until a Push request diff --git a/src/ray/object_manager/push_manager.cc b/src/ray/object_manager/push_manager.cc index 25701fe32c5c..1487c6d5b563 100644 --- a/src/ray/object_manager/push_manager.cc +++ b/src/ray/object_manager/push_manager.cc @@ -24,7 +24,6 @@ namespace ray { void PushManager::StartPush(const NodeID &dest_id, const ObjectID &obj_id, int64_t num_chunks, - int64_t max_chunk_size, std::function send_chunk_fn) { auto push_id = std::make_pair(dest_id, obj_id); RAY_CHECK(num_chunks > 0); @@ -38,7 +37,6 @@ void PushManager::StartPush(const NodeID &dest_id, dest_id, obj_id, num_chunks, - max_chunk_size, std::move(send_chunk_fn)); } else { RAY_LOG(DEBUG) << "Duplicate push request " << push_id.first << ", " << push_id.second @@ -49,8 +47,8 @@ void PushManager::StartPush(const NodeID &dest_id, ScheduleRemainingPushes(); } -void PushManager::OnChunkComplete(int64_t push_max_chunk_size) { - bytes_in_flight_ -= push_max_chunk_size; +void PushManager::OnChunkComplete() { + chunks_in_flight_ -= 1; chunks_remaining_ -= 1; ScheduleRemainingPushes(); } @@ -64,17 +62,17 @@ void PushManager::ScheduleRemainingPushes() { // Loop over all active pushes for approximate round-robin prioritization. bool keep_looping = true; - while (bytes_in_flight_ < max_bytes_in_flight_ && keep_looping) { + while (chunks_in_flight_ < max_chunks_in_flight_ && keep_looping) { // Loop over each active push and try to send another chunk. - // If we could push out a chunk and haven't reached the max_bytes_in_flight_ limit, + // If we could push out a chunk and haven't reached the chunks_in_flight_ limit, // we'll loop again to try to send more chunks. keep_looping = false; auto iter = push_requests_with_chunks_to_send_.begin(); while (iter != push_requests_with_chunks_to_send_.end() && - bytes_in_flight_ < max_bytes_in_flight_) { + chunks_in_flight_ < max_chunks_in_flight_) { auto &push_state = *iter; push_state.SendOneChunk(); - bytes_in_flight_ += push_state.max_chunk_size_; + chunks_in_flight_ += 1; if (push_state.num_chunks_to_send_ == 0) { auto push_state_map_iter = push_state_map_.find(push_state.node_id_); RAY_CHECK(push_state_map_iter != push_state_map_.end()); @@ -109,16 +107,18 @@ void PushManager::HandleNodeRemoved(const NodeID &node_id) { void PushManager::RecordMetrics() const { ray::stats::STATS_push_manager_num_pushes_remaining.Record( - push_requests_with_chunks_to_send_.size()); - ray::stats::STATS_push_manager_chunks.Record(chunks_remaining_, "Remaining"); + NumPushRequestsWithChunksToSend()); + ray::stats::STATS_push_manager_chunks.Record(NumChunksInFlight(), "InFlight"); + ray::stats::STATS_push_manager_chunks.Record(NumChunksRemaining(), "Remaining"); } std::string PushManager::DebugString() const { std::stringstream result; result << "PushManager:"; - result << "\n- num pushes remaining: " << push_requests_with_chunks_to_send_.size(); - result << "\n- num chunks remaining: " << chunks_remaining_; - result << "\n- max bytes allowed: " << max_bytes_in_flight_; + result << "\n- num pushes remaining: " << NumPushRequestsWithChunksToSend(); + result << "\n- num chunks in flight: " << NumChunksInFlight(); + result << "\n- num chunks remaining: " << NumChunksRemaining(); + result << "\n- max chunks allowed: " << max_chunks_in_flight_; return result.str(); } diff --git a/src/ray/object_manager/push_manager.h b/src/ray/object_manager/push_manager.h index b61903e9123f..4149d8f29c30 100644 --- a/src/ray/object_manager/push_manager.h +++ b/src/ray/object_manager/push_manager.h @@ -28,10 +28,12 @@ class PushManager { public: /// Create a push manager. /// - /// \param max_bytes_in_flight Max number of bytes allowed to be in flight + /// \param max_chunks_in_flight Max number of chunks allowed to be in flight /// from this PushManager (this raylet). - explicit PushManager(int64_t max_bytes_in_flight) - : max_bytes_in_flight_(max_bytes_in_flight){}; + explicit PushManager(int64_t max_chunks_in_flight) + : max_chunks_in_flight_(max_chunks_in_flight) { + RAY_CHECK_GT(max_chunks_in_flight_, 0); + }; /// Start pushing an object subject to max chunks in flight limit. /// @@ -40,39 +42,40 @@ class PushManager { /// \param dest_id The node to send to. /// \param obj_id The object to send. /// \param num_chunks The total number of chunks to send. - /// \param max_chunk_size See comment for max_chunk_size_ in PushState. /// \param send_chunk_fn This function will be called with args 0...{num_chunks-1}. /// The caller promises to call PushManager::OnChunkComplete() /// once a call to send_chunk_fn finishes. void StartPush(const NodeID &dest_id, const ObjectID &obj_id, int64_t num_chunks, - int64_t max_chunk_size, std::function send_chunk_fn); /// Called every time a chunk completes to trigger additional sends. /// TODO(ekl) maybe we should cancel the entire push on error. - void OnChunkComplete(int64_t push_max_chunk_size); + void OnChunkComplete(); /// Cancel all pushes that have not yet been sent to the removed node. void HandleNodeRemoved(const NodeID &node_id); - void RecordMetrics() const; - - int64_t BytesInFlight() const { return bytes_in_flight_; } - - int64_t ChunksRemaining() const { return chunks_remaining_; } + /// Return the number of chunks currently in flight. For metrics and testing. + int64_t NumChunksInFlight() const { return chunks_in_flight_; }; - int64_t PushesInFlight() const { return push_state_map_.size(); } + /// Return the number of chunks remaining. For metrics and testing. + int64_t NumChunksRemaining() const { return chunks_remaining_; } - int64_t PushRequestsRemaining() const { + /// Return the number of push requests with remaining chunks. For metrics and testing. + int64_t NumPushRequestsWithChunksToSend() const { return push_requests_with_chunks_to_send_.size(); - } + }; + + /// Record the internal metrics. + void RecordMetrics() const; std::string DebugString() const; private: FRIEND_TEST(TestPushManager, TestPushState); + FRIEND_TEST(TestPushManager, TestNodeRemoved); /// Tracks the state of an active object push to another node. struct PushState { @@ -81,13 +84,8 @@ class PushManager { /// total number of chunks of this object. int64_t num_chunks_; - /// the max size of a chunk for this object in bytes, used to count bytes_in_flight_ - /// and assure it stays under max_bytes_in_flight_. This means we can overcount for - /// the last chunk but we're accepting that to keep the code simpler. - int64_t max_chunk_size_; /// The function to send chunks with. std::function chunk_send_fn_; - /// The index of the next chunk to send. int64_t next_chunk_id_ = 0; /// The number of chunks remaining to send. @@ -96,12 +94,10 @@ class PushManager { PushState(NodeID node_id, ObjectID object_id, int64_t num_chunks, - int64_t max_chunk_size, std::function chunk_send_fn) : node_id_(node_id), object_id_(object_id), num_chunks_(num_chunks), - max_chunk_size_(max_chunk_size), chunk_send_fn_(std::move(chunk_send_fn)), num_chunks_to_send_(num_chunks) {} @@ -126,11 +122,11 @@ class PushManager { /// Called on completion events to trigger additional pushes. void ScheduleRemainingPushes(); - /// Max number of bytes in flight allowed. - const int64_t max_bytes_in_flight_; + /// Max number of chunks in flight allowed. + const int64_t max_chunks_in_flight_; - /// Running count of bytes in flight - int64_t bytes_in_flight_ = 0; + /// Running count of chunks in flight, used to limit progress of in_flight_pushes_. + int64_t chunks_in_flight_ = 0; /// Remaining count of chunks to push to other nodes. int64_t chunks_remaining_ = 0; diff --git a/src/ray/object_manager/tests/push_manager_test.cc b/src/ray/object_manager/tests/push_manager_test.cc index 084d9078184b..ae87892bacaf 100644 --- a/src/ray/object_manager/tests/push_manager_test.cc +++ b/src/ray/object_manager/tests/push_manager_test.cc @@ -27,36 +27,30 @@ TEST(TestPushManager, TestSingleTransfer) { results.resize(10); auto node_id = NodeID::FromRandom(); auto obj_id = ObjectID::FromRandom(); - PushManager pm(25); - int64_t push_max_chunk_size = 5; - pm.StartPush(node_id, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { - results[chunk_id] = 1; - }); - ASSERT_EQ(pm.BytesInFlight(), 25); - ASSERT_EQ(pm.ChunksRemaining(), 10); - ASSERT_EQ(pm.PushRequestsRemaining(), 1); + PushManager pm(5); + pm.StartPush(node_id, obj_id, 10, [&](int64_t chunk_id) { results[chunk_id] = 1; }); + ASSERT_EQ(pm.NumChunksInFlight(), 5); + ASSERT_EQ(pm.NumChunksRemaining(), 10); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); for (int i = 0; i < 10; i++) { - pm.OnChunkComplete(push_max_chunk_size); + pm.OnChunkComplete(); } - ASSERT_EQ(pm.BytesInFlight(), 0); - ASSERT_EQ(pm.ChunksRemaining(), 0); - ASSERT_EQ(pm.PushRequestsRemaining(), 0); + ASSERT_EQ(pm.NumChunksInFlight(), 0); + ASSERT_EQ(pm.NumChunksRemaining(), 0); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); for (int i = 0; i < 10; i++) { ASSERT_EQ(results[i], 1); } } TEST(TestPushManager, TestPushState) { - int64_t push_max_chunk_size = 5; // normal sending. { std::vector sent_chunks; PushManager::PushState state{ - NodeID::FromRandom(), - ObjectID::FromRandom(), - 2, - push_max_chunk_size, - [&](int64_t chunk_id) { sent_chunks.push_back(chunk_id); }}; + NodeID::FromRandom(), ObjectID::FromRandom(), 2, [&](int64_t chunk_id) { + sent_chunks.push_back(chunk_id); + }}; ASSERT_EQ(state.num_chunks_, 2); ASSERT_EQ(state.next_chunk_id_, 0); ASSERT_EQ(state.num_chunks_to_send_, 2); @@ -79,11 +73,9 @@ TEST(TestPushManager, TestPushState) { { std::vector sent_chunks; PushManager::PushState state{ - NodeID::FromRandom(), - ObjectID::FromRandom(), - 3, - push_max_chunk_size, - [&](int64_t chunk_id) { sent_chunks.push_back(chunk_id); }}; + NodeID::FromRandom(), ObjectID::FromRandom(), 3, [&](int64_t chunk_id) { + sent_chunks.push_back(chunk_id); + }}; state.SendOneChunk(); ASSERT_EQ(state.num_chunks_, 3); ASSERT_EQ(state.next_chunk_id_, 1); @@ -115,42 +107,37 @@ TEST(TestPushManager, TestRetryDuplicates) { results.resize(10); auto node_id = NodeID::FromRandom(); auto obj_id = ObjectID::FromRandom(); - PushManager pm(25); + PushManager pm(5); // First push request. - int64_t push_max_chunk_size = 5; - pm.StartPush(node_id, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { - results[chunk_id] = 1; - }); - ASSERT_EQ(pm.BytesInFlight(), 25); - ASSERT_EQ(pm.ChunksRemaining(), 10); - ASSERT_EQ(pm.PushRequestsRemaining(), 1); + pm.StartPush(node_id, obj_id, 10, [&](int64_t chunk_id) { results[chunk_id] = 1; }); + ASSERT_EQ(pm.NumChunksInFlight(), 5); + ASSERT_EQ(pm.NumChunksRemaining(), 10); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); // Second push request will resent the full chunks. - pm.StartPush(node_id, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { - results[chunk_id] = 2; - }); - ASSERT_EQ(pm.BytesInFlight(), 25); - ASSERT_EQ(pm.ChunksRemaining(), 15); - ASSERT_EQ(pm.PushRequestsRemaining(), 1); + pm.StartPush(node_id, obj_id, 10, [&](int64_t chunk_id) { results[chunk_id] = 2; }); + ASSERT_EQ(pm.NumChunksInFlight(), 5); + ASSERT_EQ(pm.NumChunksRemaining(), 15); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); // first 5 chunks will be sent by first push request. for (int i = 0; i < 5; i++) { - pm.OnChunkComplete(push_max_chunk_size); + pm.OnChunkComplete(); } for (int i = 0; i < 5; i++) { ASSERT_EQ(results[i], 1); } - ASSERT_EQ(pm.BytesInFlight(), 25); - ASSERT_EQ(pm.ChunksRemaining(), 10); + ASSERT_EQ(pm.NumChunksInFlight(), 5); + ASSERT_EQ(pm.NumChunksRemaining(), 10); // we will resend all chunks by second push request. for (int i = 0; i < 10; i++) { - pm.OnChunkComplete(push_max_chunk_size); + pm.OnChunkComplete(); } for (int i = 0; i < 10; i++) { ASSERT_EQ(results[i], 2); } - ASSERT_EQ(pm.BytesInFlight(), 0); - ASSERT_EQ(pm.ChunksRemaining(), 0); - ASSERT_EQ(pm.PushRequestsRemaining(), 0); + ASSERT_EQ(pm.NumChunksInFlight(), 0); + ASSERT_EQ(pm.NumChunksRemaining(), 0); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); } TEST(TestPushManager, TestResendWholeObject) { @@ -158,39 +145,34 @@ TEST(TestPushManager, TestResendWholeObject) { results.resize(10); auto node_id = NodeID::FromRandom(); auto obj_id = ObjectID::FromRandom(); - PushManager pm(25); - int64_t push_max_chunk_size = 5; - pm.StartPush(node_id, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { - results[chunk_id] = 1; - }); - ASSERT_EQ(pm.BytesInFlight(), 25); - ASSERT_EQ(pm.ChunksRemaining(), 10); - ASSERT_EQ(pm.PushRequestsRemaining(), 1); + PushManager pm(5); + pm.StartPush(node_id, obj_id, 10, [&](int64_t chunk_id) { results[chunk_id] = 1; }); + ASSERT_EQ(pm.NumChunksInFlight(), 5); + ASSERT_EQ(pm.NumChunksRemaining(), 10); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); for (int i = 0; i < 5; i++) { - pm.OnChunkComplete(push_max_chunk_size); + pm.OnChunkComplete(); } // All chunks have been sent out - ASSERT_EQ(pm.PushRequestsRemaining(), 0); - ASSERT_EQ(pm.ChunksRemaining(), 5); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); + ASSERT_EQ(pm.NumChunksRemaining(), 5); // resend this object, and it needs to be added to the traversal list. - pm.StartPush(node_id, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { - results[chunk_id] = 2; - }); - ASSERT_EQ(pm.BytesInFlight(), 25); - ASSERT_EQ(pm.ChunksRemaining(), 15); - ASSERT_EQ(pm.PushRequestsRemaining(), 1); + pm.StartPush(node_id, obj_id, 10, [&](int64_t chunk_id) { results[chunk_id] = 2; }); + ASSERT_EQ(pm.NumChunksInFlight(), 5); + ASSERT_EQ(pm.NumChunksRemaining(), 15); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); // we will resend all chunks by second push request. for (int i = 0; i < 15; i++) { - pm.OnChunkComplete(push_max_chunk_size); + pm.OnChunkComplete(); } for (int i = 0; i < 10; i++) { ASSERT_EQ(results[i], 2); } - ASSERT_EQ(pm.BytesInFlight(), 0); - ASSERT_EQ(pm.ChunksRemaining(), 0); - ASSERT_EQ(pm.PushRequestsRemaining(), 0); + ASSERT_EQ(pm.NumChunksInFlight(), 0); + ASSERT_EQ(pm.NumChunksRemaining(), 0); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); } TEST(TestPushManager, TestMultipleTransfers) { @@ -203,31 +185,30 @@ TEST(TestPushManager, TestMultipleTransfers) { auto obj_id = ObjectID::FromRandom(); int num_active1 = 0; int num_active2 = 0; - PushManager pm(25); - int64_t push_max_chunk_size = 5; - pm.StartPush(node1, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { + PushManager pm(5); + pm.StartPush(node1, obj_id, 10, [&](int64_t chunk_id) { results1[chunk_id] = 1; num_active1++; }); - pm.StartPush(node2, obj_id, 10, push_max_chunk_size, [&](int64_t chunk_id) { + pm.StartPush(node2, obj_id, 10, [&](int64_t chunk_id) { results2[chunk_id] = 2; num_active2++; }); - ASSERT_EQ(pm.BytesInFlight(), 25); - ASSERT_EQ(pm.ChunksRemaining(), 20); - ASSERT_EQ(pm.PushRequestsRemaining(), 2); + ASSERT_EQ(pm.NumChunksInFlight(), 5); + ASSERT_EQ(pm.NumChunksRemaining(), 20); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 2); for (int i = 0; i < 20; i++) { if (num_active1 > 0) { - pm.OnChunkComplete(push_max_chunk_size); + pm.OnChunkComplete(); num_active1--; } else if (num_active2 > 0) { - pm.OnChunkComplete(push_max_chunk_size); + pm.OnChunkComplete(); num_active2--; } } - ASSERT_EQ(pm.BytesInFlight(), 0); - ASSERT_EQ(pm.ChunksRemaining(), 0); - ASSERT_EQ(pm.PushRequestsRemaining(), 0); + ASSERT_EQ(pm.NumChunksInFlight(), 0); + ASSERT_EQ(pm.NumChunksRemaining(), 0); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); for (int i = 0; i < 10; i++) { ASSERT_EQ(results1[i], 1); } @@ -241,55 +222,42 @@ TEST(TestPushManager, TestPushMultipleObject) { auto obj_id_1 = ObjectID::FromRandom(); auto obj_id_2 = ObjectID::FromRandom(); auto obj_id_3 = ObjectID::FromRandom(); - PushManager pm(15); + PushManager pm(3); absl::flat_hash_map> result; - int64_t push_max_chunk_size = 5; - pm.StartPush(node_id, - obj_id_1, - 4, - push_max_chunk_size, - [&, obj_id = obj_id_1](int64_t chunk_id) { - ASSERT_FALSE(result[obj_id].contains(chunk_id)); - result[obj_id].insert(chunk_id); - }); - pm.StartPush(node_id, - obj_id_2, - 1, - push_max_chunk_size, - [&, obj_id = obj_id_2](int64_t chunk_id) { - ASSERT_FALSE(result[obj_id].contains(chunk_id)); - result[obj_id].insert(chunk_id); - }); - pm.StartPush(node_id, - obj_id_3, - 2, - push_max_chunk_size, - [&, obj_id = obj_id_3](int64_t chunk_id) { - ASSERT_FALSE(result[obj_id].contains(chunk_id)); - result[obj_id].insert(chunk_id); - }); - ASSERT_EQ(pm.PushRequestsRemaining(), 3); - ASSERT_EQ(pm.BytesInFlight(), 15); - ASSERT_EQ(pm.ChunksRemaining(), 7); - ASSERT_EQ(pm.PushRequestsRemaining(), 3); - - pm.OnChunkComplete(push_max_chunk_size); - ASSERT_EQ(pm.PushRequestsRemaining(), 2); - pm.OnChunkComplete(push_max_chunk_size); - ASSERT_EQ(pm.PushRequestsRemaining(), 1); - pm.OnChunkComplete(push_max_chunk_size); - ASSERT_EQ(pm.PushRequestsRemaining(), 1); - pm.OnChunkComplete(push_max_chunk_size); - ASSERT_EQ(pm.PushRequestsRemaining(), 0); - - pm.OnChunkComplete(push_max_chunk_size); - pm.OnChunkComplete(push_max_chunk_size); - pm.OnChunkComplete(push_max_chunk_size); - - ASSERT_EQ(pm.BytesInFlight(), 0); - ASSERT_EQ(pm.ChunksRemaining(), 0); - ASSERT_EQ(pm.PushRequestsRemaining(), 0); + pm.StartPush(node_id, obj_id_1, 4, [&, obj_id = obj_id_1](int64_t chunk_id) { + ASSERT_FALSE(result[obj_id].contains(chunk_id)); + result[obj_id].insert(chunk_id); + }); + pm.StartPush(node_id, obj_id_2, 1, [&, obj_id = obj_id_2](int64_t chunk_id) { + ASSERT_FALSE(result[obj_id].contains(chunk_id)); + result[obj_id].insert(chunk_id); + }); + pm.StartPush(node_id, obj_id_3, 2, [&, obj_id = obj_id_3](int64_t chunk_id) { + ASSERT_FALSE(result[obj_id].contains(chunk_id)); + result[obj_id].insert(chunk_id); + }); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 3); + ASSERT_EQ(pm.NumChunksInFlight(), 3); + ASSERT_EQ(pm.NumChunksRemaining(), 7); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 3); + + pm.OnChunkComplete(); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 2); + pm.OnChunkComplete(); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); + pm.OnChunkComplete(); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); + pm.OnChunkComplete(); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); + + pm.OnChunkComplete(); + pm.OnChunkComplete(); + pm.OnChunkComplete(); + + ASSERT_EQ(pm.NumChunksInFlight(), 0); + ASSERT_EQ(pm.NumChunksRemaining(), 0); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); ASSERT_EQ(result[obj_id_1].size(), 4); ASSERT_EQ(result[obj_id_2].size(), 1); @@ -297,49 +265,48 @@ TEST(TestPushManager, TestPushMultipleObject) { } TEST(TestPushManager, TestNodeRemoved) { - PushManager pm(15); + PushManager pm(3); // Start pushing two objects to node 1. auto node_id_1 = NodeID::FromRandom(); auto obj_id_1 = ObjectID::FromRandom(); auto obj_id_2 = ObjectID::FromRandom(); - int64_t push_max_chunk_size = 5; - pm.StartPush(node_id_1, obj_id_1, 4, push_max_chunk_size, [](int64_t) {}); - pm.StartPush(node_id_1, obj_id_2, 2, push_max_chunk_size, [](int64_t) {}); + pm.StartPush(node_id_1, obj_id_1, 4, [](int64_t) {}); + pm.StartPush(node_id_1, obj_id_2, 2, [](int64_t) {}); // Start pushing one object to node 2. auto node_id_2 = NodeID::FromRandom(); auto obj_id_3 = ObjectID::FromRandom(); - pm.StartPush(node_id_2, obj_id_3, 3, push_max_chunk_size, [](int64_t) {}); + pm.StartPush(node_id_2, obj_id_3, 3, [](int64_t) {}); // 3 chunks in flight for 3 objects to two nodes. - ASSERT_EQ(pm.PushRequestsRemaining(), 3); - ASSERT_EQ(pm.BytesInFlight(), 15); - ASSERT_EQ(pm.PushesInFlight(), 2); - ASSERT_EQ(pm.PushRequestsRemaining(), 3); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 3); + ASSERT_EQ(pm.NumChunksInFlight(), 3); + ASSERT_EQ(pm.push_state_map_.size(), 2); + ASSERT_EQ(pm.push_requests_with_chunks_to_send_.size(), 3); // Remove Node 1. This should cause its associated push requests to be cleaned up. pm.HandleNodeRemoved(node_id_1); - ASSERT_EQ(pm.PushRequestsRemaining(), 1); - ASSERT_EQ(pm.BytesInFlight(), 15); - ASSERT_EQ(pm.PushesInFlight(), 1); - ASSERT_EQ(pm.PushRequestsRemaining(), 1); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 1); + ASSERT_EQ(pm.NumChunksInFlight(), 3); + ASSERT_EQ(pm.push_state_map_.size(), 1); + ASSERT_EQ(pm.push_requests_with_chunks_to_send_.size(), 1); // All 3 in flight chunks finish. // All pushes should be done with chunks to node 2 in flight. for (int i = 0; i < 3; i++) { - pm.OnChunkComplete(push_max_chunk_size); + pm.OnChunkComplete(); } - ASSERT_EQ(pm.PushRequestsRemaining(), 0); - ASSERT_EQ(pm.BytesInFlight(), 15); - ASSERT_EQ(pm.PushesInFlight(), 0); - ASSERT_EQ(pm.PushRequestsRemaining(), 0); + ASSERT_EQ(pm.NumPushRequestsWithChunksToSend(), 0); + ASSERT_EQ(pm.NumChunksInFlight(), 3); + ASSERT_EQ(pm.push_state_map_.size(), 0); + ASSERT_EQ(pm.push_requests_with_chunks_to_send_.size(), 0); // The in flight chunks complete. for (int i = 0; i < 3; i++) { - pm.OnChunkComplete(push_max_chunk_size); + pm.OnChunkComplete(); } - ASSERT_EQ(pm.BytesInFlight(), 0); + ASSERT_EQ(pm.NumChunksInFlight(), 0); } } // namespace ray diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 9a433e473ebb..c2421afe83b8 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -483,8 +483,6 @@ int main(int argc, char *argv[]) { object_manager_config.object_store_memory = object_store_memory; object_manager_config.max_bytes_in_flight = RayConfig::instance().object_manager_max_bytes_in_flight(); - RAY_CHECK_GT(object_manager_config.max_bytes_in_flight, 0) - << "object_manager_max_bytes_in_flight must be greater than 0"; object_manager_config.plasma_directory = plasma_directory; object_manager_config.fallback_directory = fallback_directory; object_manager_config.huge_pages = huge_pages; From 427100daf3143c774c9412385888e1437ebc62d8 Mon Sep 17 00:00:00 2001 From: Sampan S Nayak Date: Wed, 10 Sep 2025 10:16:49 +0530 Subject: [PATCH 546/634] [core] Improve docs for custom serialization for exceptions + add test (#56156) Signed-off-by: sampan Signed-off-by: Sampan S Nayak Co-authored-by: sampan --- doc/source/ray-core/objects/serialization.rst | 57 +++++++++++++++++++ python/ray/exceptions.py | 4 +- python/ray/tests/test_traceback.py | 39 ++++++++++++- 3 files changed, 97 insertions(+), 3 deletions(-) diff --git a/doc/source/ray-core/objects/serialization.rst b/doc/source/ray-core/objects/serialization.rst index 8d183b7c10e3..096c0dc20b35 100644 --- a/doc/source/ray-core/objects/serialization.rst +++ b/doc/source/ray-core/objects/serialization.rst @@ -202,6 +202,63 @@ There are at least 3 ways to define your custom serialization process: except TypeError: pass +.. _custom-exception-serializer: + +Custom Serializers for Exceptions +---------------------------------- + +When Ray tasks raise exceptions that cannot be serialized with the default pickle mechanism, you can register custom serializers to handle them (Note: the serializer must be registered in the driver and all workers). + +.. testcode:: + + import ray + import threading + + class CustomError(Exception): + def __init__(self, message, data): + self.message = message + self.data = data + self.lock = threading.Lock() # Cannot be serialized + + def custom_serializer(exc): + return {"message": exc.message, "data": str(exc.data)} + + def custom_deserializer(state): + return CustomError(state["message"], state["data"]) + + # Register in the driver + ray.util.register_serializer( + CustomError, + serializer=custom_serializer, + deserializer=custom_deserializer + ) + + @ray.remote + def task_that_registers_serializer_and_raises(): + # Register the custom serializer in the worker + ray.util.register_serializer( + CustomError, + serializer=custom_serializer, + deserializer=custom_deserializer + ) + + # Now raise the custom exception + raise CustomError("Something went wrong", {"complex": "data"}) + + # The custom exception will be properly serialized across worker boundaries + try: + ray.get(task_that_registers_serializer_and_raises.remote()) + except ray.exceptions.RayTaskError as e: + print(f"Caught exception: {e.cause}") # This will be our CustomError + +When a custom exception is raised in a remote task, Ray will: + +1. Serialize the exception using your custom serializer +2. Wrap it in a :class:`RayTaskError ` +3. The deserialized exception will be available as ``ray_task_error.cause`` + +Whenever serialization fails, Ray throws an :class:`UnserializableException ` containing the string representation of the original stack trace. + Troubleshooting --------------- diff --git a/python/ray/exceptions.py b/python/ray/exceptions.py index c1c96ad00e1c..2ea042b515c0 100644 --- a/python/ray/exceptions.py +++ b/python/ray/exceptions.py @@ -915,7 +915,7 @@ class UnserializableException(RayError): the original exception along with its stack trace that was captured at the time of serialization. - reference for more details: https://docs.ray.io/en/latest/ray-core/objects/serialization.html + For more details and how to handle this with custom serializers, :ref:`configuring custom exeception serializers ` Args: original_stack_trace: The string representation and stack trace of the @@ -927,7 +927,7 @@ def __init__(self, original_stack_trace: str): def __str__(self): return ( - "Failed to deserialize exception. Refer to https://docs.ray.io/en/latest/ray-core/objects/serialization.html#troubleshooting to troubleshoot.\n" + "Failed to deserialize exception. Refer to https://docs.ray.io/en/latest/ray-core/objects/serialization.html#custom-serializers-for-exceptions for more information.\n" "Original exception:\n" f"{self._original_stack_trace}" ) diff --git a/python/ray/tests/test_traceback.py b/python/ray/tests/test_traceback.py index 4921181d24d3..f9bef5dc74ff 100644 --- a/python/ray/tests/test_traceback.py +++ b/python/ray/tests/test_traceback.py @@ -301,7 +301,7 @@ def __repr__(self): def test_unpickleable_stacktrace(shutdown_only): - expected_output = """Failed to deserialize exception. Refer to https://docs.ray.io/en/latest/ray-core/objects/serialization.html#troubleshooting to troubleshoot. + expected_output = """Failed to deserialize exception. Refer to https://docs.ray.io/en/latest/ray-core/objects/serialization.html#custom-serializers-for-exceptions for more information. Original exception: ray.exceptions.RayTaskError: ray::f() (pid=XXX, ip=YYY) File "FILE", line ZZ, in f @@ -330,6 +330,43 @@ def f(): assert clean_noqa(expected_output) == scrub_traceback(str(excinfo.value)) +def test_exception_with_registered_serializer(shutdown_only): + class NoPickleError(OSError): + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return f"message: {self.msg}" + + def _serializer(e: NoPickleError): + return {"msg": e.msg} + + def _deserializer(state): + return NoPickleError(state["msg"] + " deserialized") + + @ray.remote + def raise_custom_exception(): + ray.util.register_serializer( + NoPickleError, serializer=_serializer, deserializer=_deserializer + ) + raise NoPickleError("message") + + try: + with pytest.raises(NoPickleError) as exc_info: + ray.get(raise_custom_exception.remote()) + + # Ensure dual-typed exception and message propagation + assert isinstance(exc_info.value, RayTaskError) + # if custom serializer was not registered, this would be an instance of UnserializableException() + assert isinstance(exc_info.value, NoPickleError) + assert "message" in str(exc_info.value) + # modified message should not be in the exception string, only in the cause + assert "deserialized" not in str(exc_info.value) + assert "message deserialized" in str(exc_info.value.cause) + finally: + ray.util.deregister_serializer(NoPickleError) + + def test_serialization_error_message(shutdown_only): expected_output_ray_put = """Could not serialize the put value :\nINSPECT_SERIALIZABILITY""" # noqa expected_output_task = """Could not serialize the argument for a task or actor test_traceback.test_serialization_error_message..task_with_unserializable_arg:\nINSPECT_SERIALIZABILITY""" # noqa From 4f93b8fddc3047e2d154e227d0a0b60d4c9b0526 Mon Sep 17 00:00:00 2001 From: Stephanie Wang Date: Tue, 9 Sep 2025 22:48:50 -0700 Subject: [PATCH 547/634] [core][gpu objects] Rename GPU objects -> RDT objects in user-facing exceptions (#56396) Rename GPU objects to RDT (Ray Direct Transport) in user-facing error messages. --------- Signed-off-by: Stephanie wang Signed-off-by: Stephanie Wang Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python/ray/_private/serialization.py | 4 ++-- .../experimental/collective/collective_tensor_transport.py | 4 ++-- python/ray/experimental/collective/nixl_tensor_transport.py | 2 +- .../ray/experimental/gpu_object_manager/gpu_object_store.py | 4 ++-- python/ray/tests/gpu_objects/test_gpu_objects_gloo.py | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/ray/_private/serialization.py b/python/ray/_private/serialization.py index 81b502882e28..5ae15b5a8442 100644 --- a/python/ray/_private/serialization.py +++ b/python/ray/_private/serialization.py @@ -168,8 +168,8 @@ def object_ref_reducer(obj): and worker.gpu_object_manager.is_managed_object(obj.hex()) ): raise ValueError( - "Passing GPU ObjectRefs inside data structures is not yet supported. " - "Pass GPU ObjectRefs directly as task arguments instead. For example, use `foo.remote(ref)` instead of `foo.remote([ref])`." + "Passing RDT ObjectRefs inside data structures is not yet supported. " + "Pass RDT ObjectRefs directly as task arguments instead. For example, use `foo.remote(ref)` instead of `foo.remote([ref])`." ) self.add_contained_object_ref( diff --git a/python/ray/experimental/collective/collective_tensor_transport.py b/python/ray/experimental/collective/collective_tensor_transport.py index 33155886f071..2fff70737bd7 100644 --- a/python/ray/experimental/collective/collective_tensor_transport.py +++ b/python/ray/experimental/collective/collective_tensor_transport.py @@ -62,7 +62,7 @@ def __ray_get_tensor_transport_metadata__( for t in gpu_object: if t.device.type != device.type: raise ValueError( - "All tensors in one GPU object must be the same device type." + "All tensors in an RDT object must have the same device type." ) tensor_meta.append((t.shape, t.dtype)) return CollectiveTransportMetadata( @@ -105,7 +105,7 @@ def get_communicator_metadata( elif len(communicators) > 1: raise ValueError( f"There are {len(communicators)} possible communicators that contain actors {src_actor} and {dst_actor}. " - "Currently, GPU objects only support one communicator. Please make sure only " + "Currently, RDT objects only support one communicator. Please make sure only " "one communicator exists." ) communicator = communicators[0] diff --git a/python/ray/experimental/collective/nixl_tensor_transport.py b/python/ray/experimental/collective/nixl_tensor_transport.py index 2046f55ac9eb..45149a0b0608 100644 --- a/python/ray/experimental/collective/nixl_tensor_transport.py +++ b/python/ray/experimental/collective/nixl_tensor_transport.py @@ -78,7 +78,7 @@ def __ray_get_tensor_transport_metadata__( for t in gpu_object: if t.device.type != device.type: raise ValueError( - "All tensors in one GPU object must be the same device type." + "All tensors in an RDT object must have the same device type." ) tensor_meta.append((t.shape, t.dtype)) else: diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_store.py b/python/ray/experimental/gpu_object_manager/gpu_object_store.py index d526a8b4ff6f..3bd9f532ad1d 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_store.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_store.py @@ -253,7 +253,7 @@ def _wait_object(self, obj_id: str, timeout: Optional[float] = None) -> None: timeout=timeout, ): raise TimeoutError( - f"ObjectRef({obj_id}) not found in GPU object store after {timeout}s, transfer may have failed. Please report this issue on GitHub: https://github.com/ray-project/ray/issues/new/choose" + f"ObjectRef({obj_id}) not found in RDT object store after {timeout}s, transfer may have failed. Please report this issue on GitHub: https://github.com/ray-project/ray/issues/new/choose" ) def pop_object(self, obj_id: str) -> List["torch.Tensor"]: @@ -283,7 +283,7 @@ def wait_tensor_freed( lambda: tensor not in self._tensor_to_object_ids, timeout=timeout ): raise TimeoutError( - f"Tensor {tensor} not freed from GPU object store after {timeout}s. The tensor will not be freed until all ObjectRefs containing the tensor have gone out of scope." + f"Tensor {tensor} not freed from RDT object store after {timeout}s. The tensor will not be freed until all ObjectRefs containing the tensor have gone out of scope." ) def get_num_objects(self) -> int: diff --git a/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py b/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py index 0d17c4c76c91..cc66641dccf4 100644 --- a/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py +++ b/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py @@ -664,7 +664,7 @@ def test_gpu_object_ref_in_list_throws_exception(ray_start_regular): # Test: GPU ref inside a list should fail during task submission with pytest.raises( ValueError, - match="Passing GPU ObjectRefs inside data structures is not yet supported", + match="Passing RDT ObjectRefs inside data structures is not yet supported", ): actor.double.remote([gpu_ref]) @@ -672,7 +672,7 @@ def test_gpu_object_ref_in_list_throws_exception(ray_start_regular): normal_ref = ray.put("normal_data") with pytest.raises( ValueError, - match="Passing GPU ObjectRefs inside data structures is not yet supported", + match="Passing RDT ObjectRefs inside data structures is not yet supported", ): actor.double.remote([gpu_ref, normal_ref]) From 4bfee4b062d8b1ac3a873592f1b298a0dd0b4558 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 9 Sep 2025 22:55:40 -0700 Subject: [PATCH 548/634] [bazel] change core BUILD files to BUILD.bazel (#56406) unify the naming across the repo Signed-off-by: Lonnie Liu --- python/ray/_common/tests/{BUILD => BUILD.bazel} | 0 python/ray/_private/{BUILD => BUILD.bazel} | 0 python/ray/_private/runtime_env/{BUILD => BUILD.bazel} | 0 python/ray/autoscaler/{BUILD => BUILD.bazel} | 0 python/ray/autoscaler/aws/{BUILD => BUILD.bazel} | 0 python/ray/autoscaler/azure/{BUILD => BUILD.bazel} | 0 python/ray/autoscaler/gcp/{BUILD => BUILD.bazel} | 0 python/ray/autoscaler/local/{BUILD => BUILD.bazel} | 0 python/ray/autoscaler/v2/{BUILD => BUILD.bazel} | 0 python/ray/dashboard/{BUILD => BUILD.bazel} | 0 python/ray/runtime_env/{BUILD => BUILD.bazel} | 0 python/ray/scripts/{BUILD => BUILD.bazel} | 0 python/ray/util/{BUILD => BUILD.bazel} | 0 python/ray/util/dask/{BUILD => BUILD.bazel} | 0 python/ray/util/dask/tests/{BUILD => BUILD.bazel} | 0 python/setup.py | 2 +- 16 files changed, 1 insertion(+), 1 deletion(-) rename python/ray/_common/tests/{BUILD => BUILD.bazel} (100%) rename python/ray/_private/{BUILD => BUILD.bazel} (100%) rename python/ray/_private/runtime_env/{BUILD => BUILD.bazel} (100%) rename python/ray/autoscaler/{BUILD => BUILD.bazel} (100%) rename python/ray/autoscaler/aws/{BUILD => BUILD.bazel} (100%) rename python/ray/autoscaler/azure/{BUILD => BUILD.bazel} (100%) rename python/ray/autoscaler/gcp/{BUILD => BUILD.bazel} (100%) rename python/ray/autoscaler/local/{BUILD => BUILD.bazel} (100%) rename python/ray/autoscaler/v2/{BUILD => BUILD.bazel} (100%) rename python/ray/dashboard/{BUILD => BUILD.bazel} (100%) rename python/ray/runtime_env/{BUILD => BUILD.bazel} (100%) rename python/ray/scripts/{BUILD => BUILD.bazel} (100%) rename python/ray/util/{BUILD => BUILD.bazel} (100%) rename python/ray/util/dask/{BUILD => BUILD.bazel} (100%) rename python/ray/util/dask/tests/{BUILD => BUILD.bazel} (100%) diff --git a/python/ray/_common/tests/BUILD b/python/ray/_common/tests/BUILD.bazel similarity index 100% rename from python/ray/_common/tests/BUILD rename to python/ray/_common/tests/BUILD.bazel diff --git a/python/ray/_private/BUILD b/python/ray/_private/BUILD.bazel similarity index 100% rename from python/ray/_private/BUILD rename to python/ray/_private/BUILD.bazel diff --git a/python/ray/_private/runtime_env/BUILD b/python/ray/_private/runtime_env/BUILD.bazel similarity index 100% rename from python/ray/_private/runtime_env/BUILD rename to python/ray/_private/runtime_env/BUILD.bazel diff --git a/python/ray/autoscaler/BUILD b/python/ray/autoscaler/BUILD.bazel similarity index 100% rename from python/ray/autoscaler/BUILD rename to python/ray/autoscaler/BUILD.bazel diff --git a/python/ray/autoscaler/aws/BUILD b/python/ray/autoscaler/aws/BUILD.bazel similarity index 100% rename from python/ray/autoscaler/aws/BUILD rename to python/ray/autoscaler/aws/BUILD.bazel diff --git a/python/ray/autoscaler/azure/BUILD b/python/ray/autoscaler/azure/BUILD.bazel similarity index 100% rename from python/ray/autoscaler/azure/BUILD rename to python/ray/autoscaler/azure/BUILD.bazel diff --git a/python/ray/autoscaler/gcp/BUILD b/python/ray/autoscaler/gcp/BUILD.bazel similarity index 100% rename from python/ray/autoscaler/gcp/BUILD rename to python/ray/autoscaler/gcp/BUILD.bazel diff --git a/python/ray/autoscaler/local/BUILD b/python/ray/autoscaler/local/BUILD.bazel similarity index 100% rename from python/ray/autoscaler/local/BUILD rename to python/ray/autoscaler/local/BUILD.bazel diff --git a/python/ray/autoscaler/v2/BUILD b/python/ray/autoscaler/v2/BUILD.bazel similarity index 100% rename from python/ray/autoscaler/v2/BUILD rename to python/ray/autoscaler/v2/BUILD.bazel diff --git a/python/ray/dashboard/BUILD b/python/ray/dashboard/BUILD.bazel similarity index 100% rename from python/ray/dashboard/BUILD rename to python/ray/dashboard/BUILD.bazel diff --git a/python/ray/runtime_env/BUILD b/python/ray/runtime_env/BUILD.bazel similarity index 100% rename from python/ray/runtime_env/BUILD rename to python/ray/runtime_env/BUILD.bazel diff --git a/python/ray/scripts/BUILD b/python/ray/scripts/BUILD.bazel similarity index 100% rename from python/ray/scripts/BUILD rename to python/ray/scripts/BUILD.bazel diff --git a/python/ray/util/BUILD b/python/ray/util/BUILD.bazel similarity index 100% rename from python/ray/util/BUILD rename to python/ray/util/BUILD.bazel diff --git a/python/ray/util/dask/BUILD b/python/ray/util/dask/BUILD.bazel similarity index 100% rename from python/ray/util/dask/BUILD rename to python/ray/util/dask/BUILD.bazel diff --git a/python/ray/util/dask/tests/BUILD b/python/ray/util/dask/tests/BUILD.bazel similarity index 100% rename from python/ray/util/dask/tests/BUILD rename to python/ray/util/dask/tests/BUILD.bazel diff --git a/python/setup.py b/python/setup.py index d4770cd2f8c8..60024b4b8c1f 100644 --- a/python/setup.py +++ b/python/setup.py @@ -805,7 +805,7 @@ def has_ext_modules(self): exclude_package_data={ # Empty string means "any package". # Therefore, exclude BUILD from every package: - "": ["BUILD"], + "": ["BUILD", "BUILD.bazel"], }, zip_safe=False, license="Apache 2.0", From cf3389cb79d323ecf81479ded3901122a84dbf8e Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Tue, 9 Sep 2025 23:52:13 -0700 Subject: [PATCH 549/634] [wheel] remove nvm from manylinux build (#56376) use node directly. Signed-off-by: Lonnie Liu --- ci/build/build-manylinux-forge.sh | 61 +++++++++++++++++++---------- ci/build/build-manylinux-ray.sh | 10 +++-- ci/ci.sh | 20 +++++----- python/build-wheel-manylinux2014.sh | 1 - 4 files changed, 59 insertions(+), 33 deletions(-) diff --git a/ci/build/build-manylinux-forge.sh b/ci/build/build-manylinux-forge.sh index 8c98069343f7..3553cdac7509 100755 --- a/ci/build/build-manylinux-forge.sh +++ b/ci/build/build-manylinux-forge.sh @@ -5,9 +5,22 @@ set -exuo pipefail BAZELISK_VERSION="v1.26.0" -platform="linux" +ARCH="$(uname -m)" -echo "Architecture(HOSTTYPE) is ${HOSTTYPE}" +case "$ARCH" in + x86_64|amd64) + ARCH="x86_64" + ;; + aarch64|arm64) + ARCH="aarch64" + ;; + *) + echo "Unsupported arch: $ARCH" >&2 + exit 1 + ;; +esac + +echo "Architecture is ${ARCH}" if [[ ! -e /usr/bin/nproc ]]; then echo -e '#!/bin/bash\necho 10' > "/usr/bin/nproc" @@ -16,7 +29,7 @@ fi # Install ray cpp dependencies. sudo yum -y install unzip zip sudo openssl xz -if [[ "${HOSTTYPE-}" == "x86_64" ]]; then +if [[ "${ARCH}" == "x86_64" ]]; then sudo yum -y install libasan-4.8.5-44.el7.x86_64 libubsan-7.3.1-5.10.el7.x86_64 \ devtoolset-8-libasan-devel.x86_64 fi @@ -30,28 +43,36 @@ if [[ "${RAYCI_DISABLE_JAVA:-false}" != "true" && "${RAY_INSTALL_JAVA:-1}" == "1 export JAVA_HOME="${JAVA_BIN%jre/bin/java}" fi -# Install ray dashboard dependencies. -curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash -source "$HOME"/.nvm/nvm.sh +# Install nodejs +NODE_VERSION_FULL="${NODE_VERSION_FULL:-14.21.3}" + +if [[ "${ARCH}" == "x86_64" ]]; then + NODE_URL="https://nodejs.org/dist/v${NODE_VERSION_FULL}/node-v${NODE_VERSION_FULL}-linux-x64.tar.xz" + NODE_SHA256="05c08a107c50572ab39ce9e8663a2a2d696b5d262d5bd6f98d84b997ce932d9a" +else # aarch64 + NODE_URL="https://nodejs.org/dist/v${NODE_VERSION_FULL}/node-v${NODE_VERSION_FULL}-linux-arm64.tar.xz" + NODE_SHA256="f06642bfcf0b8cc50231624629bec58b183954641b638e38ed6f94cd39e8a6ef" +fi -NODE_VERSION="14" -nvm install "$NODE_VERSION" -nvm use "$NODE_VERSION" +NODE_DIR="/usr/local/node" +curl -fsSL "${NODE_URL}" -o /tmp/node.tar.xz +echo "$NODE_SHA256 /tmp/node.tar.xz" | sha256sum -c - +sudo mkdir -p "$NODE_DIR" +sudo tar -xf /tmp/node.tar.xz -C "$NODE_DIR" --strip-components=1 +rm /tmp/node.tar.xz # Install bazel mkdir -p "$HOME"/bin -if [[ "${HOSTTYPE}" == "aarch64" || "${HOSTTYPE}" = "arm64" ]]; then - # architecture is "aarch64", but the bazel tag is "arm64" - BAZELISK_URL="https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-${platform}-arm64" -elif [[ "${HOSTTYPE}" == "x86_64" ]]; then - BAZELISK_URL="https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-${platform}-amd64" -else - echo "Could not found matching bazelisk URL for platform ${platform} and architecture ${HOSTTYPE}" - exit 1 +if [[ "${ARCH}" == "x86_64" ]]; then + BAZELISK_URL="https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64" +else # aarch64 + BAZELISK_URL="https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-arm64" fi -curl -sSfL -o "$HOME"/bin/bazelisk "${BAZELISK_URL}" -chmod +x "$HOME"/bin/bazelisk -sudo ln -sf "$HOME"/bin/bazelisk /usr/local/bin/bazel + +curl -sSfL -o /tmp/bazelisk "${BAZELISK_URL}" +chmod +x /tmp/bazelisk +sudo mv /tmp/bazelisk /usr/local/bin/bazelisk +sudo ln -sf /usr/local/bin/bazelisk /usr/local/bin/bazel # Use python3.9 as default python3 sudo ln -sf /usr/local/bin/python3.9 /usr/local/bin/python3 diff --git a/ci/build/build-manylinux-ray.sh b/ci/build/build-manylinux-ray.sh index e81eb1da9ea8..c32d23ac6347 100755 --- a/ci/build/build-manylinux-ray.sh +++ b/ci/build/build-manylinux-ray.sh @@ -11,7 +11,11 @@ if [[ "${RAY_INSTALL_JAVA}" == "1" ]]; then bazel build //java:ray_java_pkg fi +export PATH="/usr/local/node/bin:$PATH" + # Build ray dashboard -cd python/ray/dashboard/client -npm ci -npm run build +( + cd python/ray/dashboard/client + npm ci + npm run build +) diff --git a/ci/ci.sh b/ci/ci.sh index c228b0e3aa67..cea4b38f5e00 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -151,8 +151,8 @@ test_macos_wheels() { return "${TEST_WHEEL_RESULT}" } -install_npm_project() { - if [ "${OSTYPE}" = msys ]; then +_install_npm_project() { + if [[ "${OSTYPE}" == msys ]]; then # Not Windows-compatible: https://github.com/npm/cli/issues/558#issuecomment-584673763 { echo "WARNING: Skipping NPM due to module incompatibilities with Windows"; } 2> /dev/null else @@ -170,14 +170,16 @@ build_dashboard_front_end() { cd ray/dashboard/client # skip nvm activation on buildkite linux instances. - if [ -z "${BUILDKITE-}" ] || [[ "${OSTYPE}" != linux* ]]; then - set +x # suppress set -x since it'll get very noisy here - . "${HOME}/.nvm/nvm.sh" - NODE_VERSION="14" - nvm install $NODE_VERSION - nvm use --silent $NODE_VERSION + if [[ -z "${BUILDKITE-}" || "${OSTYPE}" != linux* ]]; then + if [[ -d "${HOME}/.nvm" ]]; then + set +x # suppress set -x since it'll get very noisy here + . "${HOME}/.nvm/nvm.sh" + NODE_VERSION="14" + nvm install $NODE_VERSION + nvm use --silent $NODE_VERSION + fi fi - install_npm_project + _install_npm_project npm run build ) fi diff --git a/python/build-wheel-manylinux2014.sh b/python/build-wheel-manylinux2014.sh index 9444765cddbc..d065fb3b6255 100755 --- a/python/build-wheel-manylinux2014.sh +++ b/python/build-wheel-manylinux2014.sh @@ -37,7 +37,6 @@ PYTHON_VERSIONS=( # Setup runtime environment ./ci/build/build-manylinux-forge.sh -source "$HOME"/.nvm/nvm.sh # Compile ray ./ci/build/build-manylinux-ray.sh From d9b61ae928a1525d827745fabc0536d8f171a087 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Wed, 10 Sep 2025 00:16:55 -0700 Subject: [PATCH 550/634] [bazel] rename train/tune/air BUILD files to BUILD.bazel (#56410) unify naming convention across the repo Signed-off-by: Lonnie Liu --- python/ray/air/{BUILD => BUILD.bazel} | 0 python/ray/train/{BUILD => BUILD.bazel} | 0 python/ray/train/v2/{BUILD => BUILD.bazel} | 0 python/ray/tune/{BUILD => BUILD.bazel} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename python/ray/air/{BUILD => BUILD.bazel} (100%) rename python/ray/train/{BUILD => BUILD.bazel} (100%) rename python/ray/train/v2/{BUILD => BUILD.bazel} (100%) rename python/ray/tune/{BUILD => BUILD.bazel} (100%) diff --git a/python/ray/air/BUILD b/python/ray/air/BUILD.bazel similarity index 100% rename from python/ray/air/BUILD rename to python/ray/air/BUILD.bazel diff --git a/python/ray/train/BUILD b/python/ray/train/BUILD.bazel similarity index 100% rename from python/ray/train/BUILD rename to python/ray/train/BUILD.bazel diff --git a/python/ray/train/v2/BUILD b/python/ray/train/v2/BUILD.bazel similarity index 100% rename from python/ray/train/v2/BUILD rename to python/ray/train/v2/BUILD.bazel diff --git a/python/ray/tune/BUILD b/python/ray/tune/BUILD.bazel similarity index 100% rename from python/ray/tune/BUILD rename to python/ray/tune/BUILD.bazel From 66ca1c503d1a0e49fd0464340fa630835fb40759 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Wed, 10 Sep 2025 01:41:45 -0700 Subject: [PATCH 551/634] [core][rdt] Fix check crash on gpu obj free if driver knows actor is dead (#56404) Signed-off-by: dayshah --- src/ray/core_worker/core_worker_process.cc | 9 ++++++--- src/ray/core_worker/task_manager.cc | 10 +++++++--- src/ray/core_worker/task_manager.h | 8 ++++---- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index aaa3515264a2..b8c751b578b6 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -470,11 +470,14 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( RayConfig::instance().max_lineage_bytes(), *task_event_buffer, /*get_actor_rpc_client_callback=*/ - [this](const ActorID &actor_id) { + [this](const ActorID &actor_id) + -> std::optional> { auto core_worker = GetCoreWorker(); auto addr = core_worker->actor_task_submitter_->GetActorAddress(actor_id); - RAY_CHECK(addr.has_value()) << "Actor address not found for actor " << actor_id; - return core_worker->core_worker_client_pool_->GetOrConnect(addr.value()); + if (!addr.has_value()) { + return std::nullopt; + } + return core_worker->core_worker_client_pool_->GetOrConnect(*addr); }, gcs_client, task_by_state_counter_); diff --git a/src/ray/core_worker/task_manager.cc b/src/ray/core_worker/task_manager.cc index ffd18eb96a18..ca5a08b64dc0 100644 --- a/src/ray/core_worker/task_manager.cc +++ b/src/ray/core_worker/task_manager.cc @@ -311,11 +311,15 @@ std::vector TaskManager::AddPendingTask( return_object_id, [this](const ObjectID &object_id) { auto actor_id = ObjectID::ToActorID(object_id); auto rpc_client = get_actor_rpc_client_callback_(actor_id); - auto request = rpc::FreeActorObjectRequest(); + if (!rpc_client.has_value()) { + // ActorTaskSubmitter already knows the actor is already dead. + return; + } + rpc::FreeActorObjectRequest request; request.set_object_id(object_id.Binary()); - rpc_client->FreeActorObject( + rpc_client.value()->FreeActorObject( request, - [object_id, actor_id](Status status, + [object_id, actor_id](const Status &status, const rpc::FreeActorObjectReply &reply) { if (!status.ok()) { RAY_LOG(ERROR).WithField(object_id).WithField(actor_id) diff --git a/src/ray/core_worker/task_manager.h b/src/ray/core_worker/task_manager.h index 9d334eb226b7..31f010bca283 100644 --- a/src/ray/core_worker/task_manager.h +++ b/src/ray/core_worker/task_manager.h @@ -183,8 +183,8 @@ class TaskManager : public TaskManagerInterface { PushErrorCallback push_error_callback, int64_t max_lineage_bytes, worker::TaskEventBuffer &task_event_buffer, - std::function(const ActorID &)> - client_factory, + std::function>( + const ActorID &)> get_actor_rpc_client_callback, std::shared_ptr gcs_client, ray::observability::MetricInterface &task_by_state_counter) : in_memory_store_(in_memory_store), @@ -195,7 +195,7 @@ class TaskManager : public TaskManagerInterface { push_error_callback_(std::move(push_error_callback)), max_lineage_bytes_(max_lineage_bytes), task_event_buffer_(task_event_buffer), - get_actor_rpc_client_callback_(std::move(client_factory)), + get_actor_rpc_client_callback_(std::move(get_actor_rpc_client_callback)), gcs_client_(std::move(gcs_client)), task_by_state_counter_(task_by_state_counter) { task_counter_.SetOnChangeCallback( @@ -796,7 +796,7 @@ class TaskManager : public TaskManagerInterface { worker::TaskEventBuffer &task_event_buffer_; /// Callback to get the actor RPC client. - std::function( + std::function>( const ActorID &actor_id)> get_actor_rpc_client_callback_; From a8ba8a62ab835dc3b8245fc4a9d7cecef94827fc Mon Sep 17 00:00:00 2001 From: Aleksei Starikov Date: Wed, 10 Sep 2025 17:39:40 +0200 Subject: [PATCH 552/634] [data.llm][API] Allow tuple for concurrency arg (#55867) Signed-off-by: Aleksei Starikov Signed-off-by: Nikhil Ghosh --- ci/lint/pydoclint-baseline.txt | 4 - python/ray/data/llm.py | 23 +++++- .../ray/llm/_internal/batch/processor/base.py | 80 +++++++++++++++++-- .../batch/processor/sglang_engine_proc.py | 10 +-- .../batch/processor/vllm_engine_proc.py | 23 ++---- .../cpu/processor/test_processor_base.py | 80 +++++++++++++++++-- release/llm_tests/batch/test_batch_vllm.py | 7 +- 7 files changed, 186 insertions(+), 41 deletions(-) diff --git a/ci/lint/pydoclint-baseline.txt b/ci/lint/pydoclint-baseline.txt index 8aa286605a2c..9ee8a2b39ad0 100644 --- a/ci/lint/pydoclint-baseline.txt +++ b/ci/lint/pydoclint-baseline.txt @@ -1480,10 +1480,6 @@ python/ray/llm/_internal/batch/processor/base.py DOC101: Method `ProcessorBuilder.build`: Docstring contains fewer arguments than in function signature. DOC103: Method `ProcessorBuilder.build`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**kwargs: ]. -------------------- -python/ray/llm/_internal/batch/processor/vllm_engine_proc.py - DOC101: Function `build_vllm_engine_processor`: Docstring contains fewer arguments than in function signature. - DOC103: Function `build_vllm_engine_processor`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [telemetry_agent: Optional[TelemetryAgent]]. --------------------- python/ray/llm/_internal/batch/stages/base.py DOC405: Method `StatefulStageUDF.__call__` has both "return" and "yield" statements. Please use Generator[YieldType, SendType, ReturnType] as the return type annotation, and put your yield type in YieldType and return type in ReturnType. More details in https://jsh9.github.io/pydoclint/notes_generator_vs_iterator.html -------------------- diff --git a/python/ray/data/llm.py b/python/ray/data/llm.py index 0e8055771795..a0718c597e98 100644 --- a/python/ray/data/llm.py +++ b/python/ray/data/llm.py @@ -28,6 +28,11 @@ class ProcessorConfig(_ProcessorConfig): accelerator_type: The accelerator type used by the LLM stage in a processor. Default to None, meaning that only the CPU will be used. concurrency: The number of workers for data parallelism. Default to 1. + If ``concurrency`` is a ``tuple`` ``(m, n)``, Ray creates an autoscaling + actor pool that scales between ``m`` and ``n`` workers (``1 <= m <= n``). + If ``concurrency`` is an ``int`` ``n``, Ray uses either a fixed pool of ``n`` + workers or an autoscaling pool from ``1`` to ``n`` workers, depending on + the processor and stage. """ pass @@ -41,7 +46,9 @@ class HttpRequestProcessorConfig(_HttpRequestProcessorConfig): batch_size: The batch size to send to the HTTP request. url: The URL to send the HTTP request to. headers: The headers to send with the HTTP request. - concurrency: The number of concurrent requests to send. + concurrency: The number of concurrent requests to send. Default to 1. + If ``concurrency`` is a ``tuple`` ``(m, n)``, + autoscaling strategy is used (``1 <= m <= n``). Examples: .. testcode:: @@ -116,6 +123,10 @@ class vLLMEngineProcessorConfig(_vLLMEngineProcessorConfig): accelerator_type: The accelerator type used by the LLM stage in a processor. Default to None, meaning that only the CPU will be used. concurrency: The number of workers for data parallelism. Default to 1. + If ``concurrency`` is a tuple ``(m, n)``, Ray creates an autoscaling + actor pool that scales between ``m`` and ``n`` workers (``1 <= m <= n``). + If ``concurrency`` is an ``int`` ``n``, CPU stages use an autoscaling + pool from ``(1, n)``, while GPU stages use a fixed pool of ``n`` workers. Examples: @@ -177,7 +188,7 @@ class SGLangEngineProcessorConfig(_SGLangEngineProcessorConfig): Args: model_source: The model source to use for the SGLang engine. - batch_size: The batch size to send to the vLLM engine. Large batch sizes are + batch_size: The batch size to send to the SGLang engine. Large batch sizes are likely to saturate the compute resources and could achieve higher throughput. On the other hand, small batch sizes are more fault-tolerant and could reduce bubbles in the data pipeline. You can tune the batch size to balance @@ -197,12 +208,16 @@ class SGLangEngineProcessorConfig(_SGLangEngineProcessorConfig): apply_chat_template: Whether to apply chat template. chat_template: The chat template to use. This is usually not needed if the model checkpoint already contains the chat template. - tokenize: Whether to tokenize the input before passing it to the vLLM engine. - If not, vLLM will tokenize the prompt in the engine. + tokenize: Whether to tokenize the input before passing it to the SGLang engine. + If not, SGLang will tokenize the prompt in the engine. detokenize: Whether to detokenize the output. accelerator_type: The accelerator type used by the LLM stage in a processor. Default to None, meaning that only the CPU will be used. concurrency: The number of workers for data parallelism. Default to 1. + If ``concurrency`` is a tuple ``(m, n)``, Ray creates an autoscaling + actor pool that scales between ``m`` and ``n`` workers (``1 <= m <= n``). + If ``concurrency`` is an ``int`` ``n``, CPU stages use an autoscaling + pool from ``(1, n)``, while GPU stages use a fixed pool of ``n`` workers. Examples: .. testcode:: diff --git a/python/ray/llm/_internal/batch/processor/base.py b/python/ray/llm/_internal/batch/processor/base.py index 049e5c2685f5..2029a1a0d33b 100644 --- a/python/ray/llm/_internal/batch/processor/base.py +++ b/python/ray/llm/_internal/batch/processor/base.py @@ -1,8 +1,8 @@ import logging from collections import OrderedDict -from typing import Any, Callable, Dict, List, Optional, Type +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union -from pydantic import Field +from pydantic import Field, field_validator import ray from ray.data import Dataset @@ -45,9 +45,14 @@ class ProcessorConfig(BaseModelExtended): description="The accelerator type used by the LLM stage in a processor. " "Default to None, meaning that only the CPU will be used.", ) - concurrency: Optional[int] = Field( + concurrency: Union[int, Tuple[int, int]] = Field( default=1, - description="The number of workers for data parallelism. Default to 1.", + description="The number of workers for data parallelism. Default to 1. " + "If ``concurrency`` is a ``tuple`` ``(m, n)``, Ray creates an autoscaling " + "actor pool that scales between ``m`` and ``n`` workers (``1 <= m <= n``). " + "If ``concurrency`` is an ``int`` ``n``, Ray uses either a fixed pool of ``n`` " + "workers or an autoscaling pool from ``1`` to ``n`` workers, depending on " + "the processor and stage.", ) experimental: Dict[str, Any] = Field( @@ -57,6 +62,71 @@ class ProcessorConfig(BaseModelExtended): "`max_tasks_in_flight_per_actor`: The maximum number of tasks in flight per actor. Default to 4.", ) + @field_validator("concurrency") + def validate_concurrency( + cls, concurrency: Union[int, Tuple[int, int]] + ) -> Union[int, Tuple[int, int]]: + """Validate that `concurrency` is either: + - a positive int, or + - a 2-tuple `(min, max)` of positive ints with `min <= max`. + """ + + def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + if isinstance(concurrency, int): + require( + concurrency > 0, + f"A positive integer for `concurrency` is expected! Got: `{concurrency}`.", + ) + elif isinstance(concurrency, tuple): + require( + all(c > 0 for c in concurrency), + f"`concurrency` tuple items must be positive integers! Got: `{concurrency}`.", + ) + + min_concurrency, max_concurrency = concurrency + require( + min_concurrency <= max_concurrency, + f"min > max in the concurrency tuple `{concurrency}`!", + ) + return concurrency + + def get_concurrency(self, autoscaling_enabled: bool = True) -> Tuple[int, int]: + """Return a normalized `(min, max)` worker range from `self.concurrency`. + + Behavior: + - If `concurrency` is an int `n`: + - `autoscaling_enabled` is True -> return `(1, n)` (autoscaling). + - `autoscaling_enabled` is False -> return `(n, n)` (fixed-size pool). + - If `concurrency` is a 2-tuple `(m, n)`, return it unchanged + (the `autoscaling_enabled` flag is ignored). + + Args: + autoscaling_enabled: When False, treat an integer `concurrency` as fixed `(n, n)`; + otherwise treat it as a range `(1, n)`. Defaults to True. + + Returns: + tuple[int, int]: The allowed worker range `(min, max)`. + + Examples: + >>> self.concurrency = (2, 4) + >>> self.get_concurrency() + (2, 4) + >>> self.concurrency = 4 + >>> self.get_concurrency() + (1, 4) + >>> self.get_concurrency(autoscaling_enabled=False) + (4, 4) + """ + if isinstance(self.concurrency, int): + if autoscaling_enabled: + return 1, self.concurrency + else: + return self.concurrency, self.concurrency + return self.concurrency + class Config: validate_assignment = True arbitrary_types_allowed = True @@ -263,7 +333,7 @@ class ProcessorBuilder: @classmethod def register(cls, config_type: Type[ProcessorConfig], builder: Callable) -> None: - """A decorator to assoicate a particular pipeline config + """A decorator to associate a particular pipeline config with its build function. """ type_name = config_type.__name__ diff --git a/python/ray/llm/_internal/batch/processor/sglang_engine_proc.py b/python/ray/llm/_internal/batch/processor/sglang_engine_proc.py index 1156e830177d..602536fc0ad5 100644 --- a/python/ray/llm/_internal/batch/processor/sglang_engine_proc.py +++ b/python/ray/llm/_internal/batch/processor/sglang_engine_proc.py @@ -85,7 +85,7 @@ def build_sglang_engine_processor( ), map_batches_kwargs=dict( zero_copy_batch=True, - concurrency=(1, config.concurrency), + concurrency=config.get_concurrency(), batch_size=config.batch_size, runtime_env=config.runtime_env, ), @@ -100,7 +100,7 @@ def build_sglang_engine_processor( ), map_batches_kwargs=dict( zero_copy_batch=True, - concurrency=(1, config.concurrency), + concurrency=config.get_concurrency(), batch_size=config.batch_size, runtime_env=config.runtime_env, ), @@ -123,8 +123,8 @@ def build_sglang_engine_processor( # which initiates enough many overlapping UDF calls per actor, to # saturate `max_concurrency`. compute=ray.data.ActorPoolStrategy( - min_size=config.concurrency, - max_size=config.concurrency, + min_size=config.get_concurrency(autoscaling_enabled=False)[0], + max_size=config.get_concurrency(autoscaling_enabled=False)[1], max_tasks_in_flight_per_actor=config.experimental.get( "max_tasks_in_flight_per_actor", DEFAULT_MAX_TASKS_IN_FLIGHT ), @@ -148,7 +148,7 @@ def build_sglang_engine_processor( ), map_batches_kwargs=dict( zero_copy_batch=True, - concurrency=(1, config.concurrency), + concurrency=config.get_concurrency(), batch_size=config.batch_size, runtime_env=config.runtime_env, ), diff --git a/python/ray/llm/_internal/batch/processor/vllm_engine_proc.py b/python/ray/llm/_internal/batch/processor/vllm_engine_proc.py index 2fdd80c641d4..da88c482feb8 100644 --- a/python/ray/llm/_internal/batch/processor/vllm_engine_proc.py +++ b/python/ray/llm/_internal/batch/processor/vllm_engine_proc.py @@ -80,6 +80,7 @@ def build_vllm_engine_processor( required fields for the following processing stages. postprocess: An optional lambda function that takes a row (dict) as input and returns a postprocessed row (dict). + telemetry_agent: An optional telemetry agent for collecting usage telemetry. Returns: The constructed processor. @@ -87,21 +88,13 @@ def build_vllm_engine_processor( ray.init(runtime_env=config.runtime_env, ignore_reinit_error=True) stages = [] - if isinstance(config.concurrency, int): - # For CPU-only stages, we leverage auto-scaling to recycle resources. - processor_concurrency = (1, config.concurrency) - else: - raise ValueError( - "``concurrency`` is expected to be set as an integer," - f" but got: {config.concurrency}." - ) if config.has_image: stages.append( PrepareImageStage( map_batches_kwargs=dict( zero_copy_batch=True, - concurrency=processor_concurrency, + concurrency=config.get_concurrency(), batch_size=config.batch_size, ), ) @@ -115,7 +108,7 @@ def build_vllm_engine_processor( ), map_batches_kwargs=dict( zero_copy_batch=True, - concurrency=processor_concurrency, + concurrency=config.get_concurrency(), batch_size=config.batch_size, runtime_env=config.runtime_env, ), @@ -130,7 +123,7 @@ def build_vllm_engine_processor( ), map_batches_kwargs=dict( zero_copy_batch=True, - concurrency=processor_concurrency, + concurrency=config.get_concurrency(), batch_size=config.batch_size, runtime_env=config.runtime_env, ), @@ -157,10 +150,8 @@ def build_vllm_engine_processor( # which initiates enough many overlapping UDF calls per actor, to # saturate `max_concurrency`. compute=ray.data.ActorPoolStrategy( - # vLLM start up time is significant, so if user give fixed - # concurrency, start all instances without auto-scaling. - min_size=config.concurrency, - max_size=config.concurrency, + min_size=config.get_concurrency(autoscaling_enabled=False)[0], + max_size=config.get_concurrency(autoscaling_enabled=False)[1], max_tasks_in_flight_per_actor=config.experimental.get( "max_tasks_in_flight_per_actor", DEFAULT_MAX_TASKS_IN_FLIGHT ), @@ -184,7 +175,7 @@ def build_vllm_engine_processor( ), map_batches_kwargs=dict( zero_copy_batch=True, - concurrency=processor_concurrency, + concurrency=config.get_concurrency(), batch_size=config.batch_size, runtime_env=config.runtime_env, ), diff --git a/python/ray/llm/tests/batch/cpu/processor/test_processor_base.py b/python/ray/llm/tests/batch/cpu/processor/test_processor_base.py index 4e2b64323c64..ba93421d8252 100644 --- a/python/ray/llm/tests/batch/cpu/processor/test_processor_base.py +++ b/python/ray/llm/tests/batch/cpu/processor/test_processor_base.py @@ -190,17 +190,87 @@ def overrider(name: str, stage: StatefulStage): class TestProcessorConfig: def test_valid_concurrency(self): + config = vLLMEngineProcessorConfig( + model_source="unsloth/Llama-3.2-1B-Instruct", + concurrency=(1, 2), + ) + assert config.concurrency == (1, 2) - with pytest.raises(pydantic.ValidationError, match="should be a valid integer"): - config = vLLMEngineProcessorConfig( - model_source="unsloth/Llama-3.2-1B-Instruct", - concurrency=(1, 2), - ) config = vLLMEngineProcessorConfig( model_source="unsloth/Llama-3.2-1B-Instruct", ) assert config.concurrency == 1 + def test_invalid_concurrency(self): + with pytest.raises(pydantic.ValidationError): + vLLMEngineProcessorConfig( + model_source="unsloth/Llama-3.2-1B-Instruct", + concurrency=1.1, + ) + + with pytest.raises(pydantic.ValidationError): + vLLMEngineProcessorConfig( + model_source="unsloth/Llama-3.2-1B-Instruct", + concurrency=[1, 2, 3], + ) + + @pytest.mark.parametrize("n", [1, 2, 10]) + def test_positive_int_not_fail(self, n): + conf = ProcessorConfig(concurrency=n) + assert conf.concurrency == n + + def test_positive_int_unusual_not_fail(self): + assert ProcessorConfig(concurrency="1").concurrency == 1 + assert ProcessorConfig(concurrency=1.0).concurrency == 1 + assert ProcessorConfig(concurrency="1.0").concurrency == 1 + + @pytest.mark.parametrize("pair", [(1, 1), (1, 2), (2, 8)]) + def test_valid_tuple_not_fail(self, pair): + conf = ProcessorConfig(concurrency=pair) + assert conf.concurrency == pair + + def test_valid_tuple_unusual_not_fail(self): + assert ProcessorConfig(concurrency=("1", 2)).concurrency == (1, 2) + assert ProcessorConfig(concurrency=(1, "2")).concurrency == (1, 2) + assert ProcessorConfig(concurrency=[1, "2"]).concurrency == (1, 2) + + @pytest.mark.parametrize( + "bad,msg_part", + [ + (0, "positive integer"), + (-5, "positive integer"), + ((1, 2, 3), "at most 2 items"), + ((0, 1), "positive integers"), + ((1, 0), "positive integers"), + ((-1, 2), "positive integers"), + ((1, -2), "positive integers"), + ((1, 2.5), "a number with a fractional part"), + ("2.1", "unable to parse string"), + ((5, 2), "min > max"), + ], + ) + def test_invalid_inputs_raise(self, bad, msg_part): + with pytest.raises(pydantic.ValidationError) as e: + ProcessorConfig(concurrency=bad) + assert msg_part in str(e.value) + + @pytest.mark.parametrize( + "n,expected", [(1, (1, 1)), (4, (1, 4)), (10, (1, 10)), ("10", (1, 10))] + ) + def test_with_int_concurrency_scaling(self, n, expected): + conf = ProcessorConfig(concurrency=n) + assert conf.get_concurrency() == expected + + @pytest.mark.parametrize("n,expected", [(1, (1, 1)), (4, (4, 4)), (10, (10, 10))]) + def test_with_int_concurrency_fixed(self, n, expected): + conf = ProcessorConfig(concurrency=n) + assert conf.get_concurrency(autoscaling_enabled=False) == expected + + @pytest.mark.parametrize("pair", [(1, 1), (1, 3), (2, 8)]) + def test_with_tuple_concurrency(self, pair): + conf = ProcessorConfig(concurrency=pair) + assert conf.get_concurrency() == pair + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/release/llm_tests/batch/test_batch_vllm.py b/release/llm_tests/batch/test_batch_vllm.py index 231153dedeb1..dea0adca7367 100644 --- a/release/llm_tests/batch/test_batch_vllm.py +++ b/release/llm_tests/batch/test_batch_vllm.py @@ -27,9 +27,12 @@ def add_buffer_time_between_tests(): """Add buffer time after each test to avoid resource conflicts, which cause flakiness. """ - yield # Test runs here + # yield # test runs + # time.sleep(10) + import gc - time.sleep(10) + gc.collect() + time.sleep(15) def test_chat_template_with_vllm(): From e88926cff452cdcbeee8bd3a598b05a8f7021dc6 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Wed, 10 Sep 2025 08:49:15 -0700 Subject: [PATCH 553/634] [bazel] change all BUILD files from BUILD to BUILD.bazel (#56337) unify naming conventions across the entire repository Signed-off-by: Lonnie Liu --- python/ray/dag/{BUILD => BUILD.bazel} | 0 python/ray/data/{BUILD => BUILD.bazel} | 0 python/ray/experimental/{BUILD => BUILD.bazel} | 0 python/ray/llm/tests/{BUILD => BUILD.bazel} | 0 python/ray/serve/{BUILD => BUILD.bazel} | 0 python/ray/serve/tests/{BUILD => BUILD.bazel} | 0 python/ray/serve/tests/unit/{BUILD => BUILD.bazel} | 0 python/ray/tests/{BUILD => BUILD.bazel} | 0 python/ray/tests/horovod/{BUILD => BUILD.bazel} | 0 python/ray/tests/ludwig/{BUILD => BUILD.bazel} | 0 python/ray/tests/modin/{BUILD => BUILD.bazel} | 0 python/ray/tests/unit/{BUILD => BUILD.bazel} | 0 rllib/{BUILD => BUILD.bazel} | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename python/ray/dag/{BUILD => BUILD.bazel} (100%) rename python/ray/data/{BUILD => BUILD.bazel} (100%) rename python/ray/experimental/{BUILD => BUILD.bazel} (100%) rename python/ray/llm/tests/{BUILD => BUILD.bazel} (100%) rename python/ray/serve/{BUILD => BUILD.bazel} (100%) rename python/ray/serve/tests/{BUILD => BUILD.bazel} (100%) rename python/ray/serve/tests/unit/{BUILD => BUILD.bazel} (100%) rename python/ray/tests/{BUILD => BUILD.bazel} (100%) rename python/ray/tests/horovod/{BUILD => BUILD.bazel} (100%) rename python/ray/tests/ludwig/{BUILD => BUILD.bazel} (100%) rename python/ray/tests/modin/{BUILD => BUILD.bazel} (100%) rename python/ray/tests/unit/{BUILD => BUILD.bazel} (100%) rename rllib/{BUILD => BUILD.bazel} (100%) diff --git a/python/ray/dag/BUILD b/python/ray/dag/BUILD.bazel similarity index 100% rename from python/ray/dag/BUILD rename to python/ray/dag/BUILD.bazel diff --git a/python/ray/data/BUILD b/python/ray/data/BUILD.bazel similarity index 100% rename from python/ray/data/BUILD rename to python/ray/data/BUILD.bazel diff --git a/python/ray/experimental/BUILD b/python/ray/experimental/BUILD.bazel similarity index 100% rename from python/ray/experimental/BUILD rename to python/ray/experimental/BUILD.bazel diff --git a/python/ray/llm/tests/BUILD b/python/ray/llm/tests/BUILD.bazel similarity index 100% rename from python/ray/llm/tests/BUILD rename to python/ray/llm/tests/BUILD.bazel diff --git a/python/ray/serve/BUILD b/python/ray/serve/BUILD.bazel similarity index 100% rename from python/ray/serve/BUILD rename to python/ray/serve/BUILD.bazel diff --git a/python/ray/serve/tests/BUILD b/python/ray/serve/tests/BUILD.bazel similarity index 100% rename from python/ray/serve/tests/BUILD rename to python/ray/serve/tests/BUILD.bazel diff --git a/python/ray/serve/tests/unit/BUILD b/python/ray/serve/tests/unit/BUILD.bazel similarity index 100% rename from python/ray/serve/tests/unit/BUILD rename to python/ray/serve/tests/unit/BUILD.bazel diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD.bazel similarity index 100% rename from python/ray/tests/BUILD rename to python/ray/tests/BUILD.bazel diff --git a/python/ray/tests/horovod/BUILD b/python/ray/tests/horovod/BUILD.bazel similarity index 100% rename from python/ray/tests/horovod/BUILD rename to python/ray/tests/horovod/BUILD.bazel diff --git a/python/ray/tests/ludwig/BUILD b/python/ray/tests/ludwig/BUILD.bazel similarity index 100% rename from python/ray/tests/ludwig/BUILD rename to python/ray/tests/ludwig/BUILD.bazel diff --git a/python/ray/tests/modin/BUILD b/python/ray/tests/modin/BUILD.bazel similarity index 100% rename from python/ray/tests/modin/BUILD rename to python/ray/tests/modin/BUILD.bazel diff --git a/python/ray/tests/unit/BUILD b/python/ray/tests/unit/BUILD.bazel similarity index 100% rename from python/ray/tests/unit/BUILD rename to python/ray/tests/unit/BUILD.bazel diff --git a/rllib/BUILD b/rllib/BUILD.bazel similarity index 100% rename from rllib/BUILD rename to rllib/BUILD.bazel From 1a6c5a9947b38b10444f857c0518dd889729d269 Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Wed, 10 Sep 2025 09:11:22 -0700 Subject: [PATCH 554/634] Make `ray-llm` code owner for `ray.data.llm` (#56420) ## Why are these changes needed? The LLM APIs under `ray.data.llm` are maintained by @ray-project/ray-llm, not the Data team. This PR updates the `CODEOWNERS` to reflect that responsibility, assigning ownership to the LLM team even though the code lives in the Data directory. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Balaji Veeramani --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8c4dbb3c0fa3..da43c939515c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -67,6 +67,7 @@ # LLM /python/ray/llm/ @ray-project/ray-llm +/python/ray/data/llm.py @ray-project/ray-llm # Ray Serve /python/ray/serve/ @ray-project/ray-serve From 212367df32b16c9ec29f3ee02035f9cc9aa13eff Mon Sep 17 00:00:00 2001 From: Abrar Sheikh Date: Wed, 10 Sep 2025 09:39:18 -0700 Subject: [PATCH 555/634] foundation work for aggregating metrics on controller (#56295) This PR is part of the larger plan outlined [[here](https://gist.github.com/abrarsheikh/ac23e10a61925e0db04097c59e694add)]. ### Scope of this PR * **No functional changes introduced.** * Begin transition toward controller-side aggregation by sending **unaggregated `num_ongoing_requests`** from both replica and handle to the controller, **in addition to** existing aggregated values. * In future PRs, pre-aggregated values will be removed once controller-side aggregation is fully enabled. * **Refactor `HandleMetricReport` and `ReplicaMetricReport`:** * Moved from `autoscaling_state` to `commons`. * Objects are now created at the source instead of in `autoscaling_state`, simplifying code and avoiding passing individual attributes through the controller. * These reports will be extended in the future to carry **custom metrics**. * **Controller method renames** for improved clarity. * **New feature flag:** `RAY_SERVE_RPC_LATENCY_WARNING_THRESHOLD_MS` * Emits warnings when replicas/handles are slow to send metrics to the controller. * Helps users operating large clusters identify and debug RPC communication issues earlier. ### Next Steps (Future PRs) 1. Introduce `RAY_SERVE_AGGREGATE_METRICS_AT_CONTROLLER`. 2. Add user-defined aggregation functions in autoscaling config. 3. Perform aggregation on the controller (replacing pre-aggregated values from handle/replica), gated by the new flag. https://github.com/ray-project/ray/pull/56306 https://github.com/ray-project/ray/pull/56311 --------- Signed-off-by: abrar --- .../serve/advanced-guides/performance.md | 2 +- .../ray/serve/_private/autoscaling_state.py | 133 +++----------- python/ray/serve/_private/common.py | 82 +++++++++ python/ray/serve/_private/constants.py | 5 + python/ray/serve/_private/controller.py | 54 +++--- python/ray/serve/_private/metrics_utils.py | 9 +- python/ray/serve/_private/replica.py | 19 +- python/ray/serve/_private/router.py | 52 ++++-- .../serve/tests/test_autoscaling_policy.py | 2 +- .../serve/tests/unit/test_deployment_state.py | 173 +++++++++++++----- python/ray/serve/tests/unit/test_router.py | 11 +- 11 files changed, 332 insertions(+), 210 deletions(-) diff --git a/doc/source/serve/advanced-guides/performance.md b/doc/source/serve/advanced-guides/performance.md index 634847dc5bc2..1cdff7ab7862 100644 --- a/doc/source/serve/advanced-guides/performance.md +++ b/doc/source/serve/advanced-guides/performance.md @@ -80,7 +80,7 @@ Ray Serve allows you to fine-tune the backoff behavior of the request router, wh The Serve Controller runs on the Ray head node and is responsible for a variety of tasks, including receiving autoscaling metrics from other Ray Serve components. If the Serve Controller becomes overloaded -(symptoms might include high CPU usage and a large number of pending `ServeController.record_handle_metrics` tasks), +(symptoms might include high CPU usage and a large number of pending `ServeController.record_autoscaling_metrics_from_handle` tasks), you can increase the interval between cycles of the control loop by setting the `RAY_SERVE_CONTROL_LOOP_INTERVAL_S` environment variable (defaults to `0.1` seconds). This setting gives the Controller more time to process requests and may help alleviate the overload. diff --git a/python/ray/serve/_private/autoscaling_state.py b/python/ray/serve/_private/autoscaling_state.py index c95abfa6cb5c..e7eace67b2d2 100644 --- a/python/ray/serve/_private/autoscaling_state.py +++ b/python/ray/serve/_private/autoscaling_state.py @@ -4,9 +4,11 @@ from typing import Any, Dict, List, Optional, Set from ray.serve._private.common import ( - DeploymentHandleSource, + RUNNING_REQUESTS_KEY, DeploymentID, + HandleMetricReport, ReplicaID, + ReplicaMetricReport, TargetCapacityDirection, ) from ray.serve._private.constants import ( @@ -19,65 +21,6 @@ logger = logging.getLogger(SERVE_LOGGER_NAME) -@dataclass -class HandleMetricReport: - """Report from a deployment handle on queued and ongoing requests. - - Args: - actor_id: If the deployment handle (from which this metric was - sent) lives on an actor, the actor ID of that actor. - handle_source: Describes what kind of entity holds this - deployment handle: a Serve proxy, a Serve replica, or - unknown. - queued_requests: The current number of queued requests at the - handle, i.e. requests that haven't been assigned to any - replica yet. - running_requests: A map of replica ID to the average number of - requests, assigned through the handle, running at that - replica. - timestamp: The time at which this report was received. - """ - - actor_id: Optional[str] - handle_source: DeploymentHandleSource - queued_requests: float - running_requests: Dict[ReplicaID, float] - timestamp: float - - @property - def total_requests(self) -> float: - """Total number of queued and running requests.""" - return self.queued_requests + sum(self.running_requests.values()) - - @property - def is_serve_component_source(self) -> bool: - """Whether the handle source is a Serve actor. - - More specifically, this returns whether a Serve actor tracked - by the controller holds the deployment handle that sent this - report. If the deployment handle lives on a driver, a Ray task, - or an actor that's not a Serve replica, then this returns False. - """ - return self.handle_source in [ - DeploymentHandleSource.PROXY, - DeploymentHandleSource.REPLICA, - ] - - -@dataclass -class ReplicaMetricReport: - """Report from a replica on ongoing requests. - - Args: - running_requests: Average number of running requests at the - replica. - timestamp: The time at which this report was received. - """ - - running_requests: float - timestamp: float - - @dataclass class AutoscalingContext: """Rich context provided to custom autoscaling policies.""" @@ -214,47 +157,32 @@ def apply_bounds(self, num_replicas: int) -> int: ) def record_request_metrics_for_replica( - self, replica_id: ReplicaID, window_avg: Optional[float], send_timestamp: float + self, replica_metric_report: ReplicaMetricReport ) -> None: """Records average number of ongoing requests at a replica.""" - if window_avg is None: - return - + replica_id = replica_metric_report.replica_id + send_timestamp = replica_metric_report.timestamp if ( replica_id not in self._replica_requests or send_timestamp > self._replica_requests[replica_id].timestamp ): - self._replica_requests[replica_id] = ReplicaMetricReport( - running_requests=window_avg, - timestamp=send_timestamp, - ) + self._replica_requests[replica_id] = replica_metric_report def record_request_metrics_for_handle( self, - *, - handle_id: str, - actor_id: Optional[str], - handle_source: DeploymentHandleSource, - queued_requests: float, - running_requests: Dict[ReplicaID, float], - send_timestamp: float, + handle_metric_report: HandleMetricReport, ) -> None: """Records average number of queued and running requests at a handle for this deployment. """ - + handle_id = handle_metric_report.handle_id + send_timestamp = handle_metric_report.timestamp if ( handle_id not in self._handle_requests or send_timestamp > self._handle_requests[handle_id].timestamp ): - self._handle_requests[handle_id] = HandleMetricReport( - actor_id=actor_id, - handle_source=handle_source, - queued_requests=queued_requests, - running_requests=running_requests, - timestamp=send_timestamp, - ) + self._handle_requests[handle_id] = handle_metric_report def drop_stale_handle_metrics(self, alive_serve_actor_ids: Set[str]) -> None: """Drops handle metrics that are no longer valid. @@ -353,16 +281,22 @@ def get_total_num_requests(self) -> float: for id in self._running_replicas: if id in self._replica_requests: - total_requests += self._replica_requests[id].running_requests + total_requests += self._replica_requests[id].aggregated_metrics.get( + RUNNING_REQUESTS_KEY + ) metrics_collected_on_replicas = total_requests > 0 for handle_metric in self._handle_requests.values(): total_requests += handle_metric.queued_requests if not metrics_collected_on_replicas: - for id in self._running_replicas: - if id in handle_metric.running_requests: - total_requests += handle_metric.running_requests[id] + for replica_id in self._running_replicas: + if replica_id in handle_metric.aggregated_metrics.get( + RUNNING_REQUESTS_KEY + ): + total_requests += handle_metric.aggregated_metrics.get( + RUNNING_REQUESTS_KEY + ).get(replica_id) return total_requests @@ -431,39 +365,26 @@ def is_within_bounds( ) def record_request_metrics_for_replica( - self, replica_id: ReplicaID, window_avg: Optional[float], send_timestamp: float + self, replica_metric_report: ReplicaMetricReport ) -> None: - deployment_id = replica_id.deployment_id + deployment_id = replica_metric_report.replica_id.deployment_id # Defensively guard against delayed replica metrics arriving # after the deployment's been deleted if deployment_id in self._autoscaling_states: self._autoscaling_states[deployment_id].record_request_metrics_for_replica( - replica_id=replica_id, - window_avg=window_avg, - send_timestamp=send_timestamp, + replica_metric_report ) def record_request_metrics_for_handle( self, - *, - deployment_id: str, - handle_id: str, - actor_id: Optional[str], - handle_source: DeploymentHandleSource, - queued_requests: float, - running_requests: Dict[ReplicaID, float], - send_timestamp: float, + handle_metric_report: HandleMetricReport, ) -> None: """Update request metric for a specific handle.""" + deployment_id = handle_metric_report.deployment_id if deployment_id in self._autoscaling_states: self._autoscaling_states[deployment_id].record_request_metrics_for_handle( - handle_id=handle_id, - actor_id=actor_id, - handle_source=handle_source, - queued_requests=queued_requests, - running_requests=running_requests, - send_timestamp=send_timestamp, + handle_metric_report ) def drop_stale_handle_metrics(self, alive_serve_actor_ids: Set[str]) -> None: diff --git a/python/ray/serve/_private/common.py b/python/ray/serve/_private/common.py index 682b186a6d7f..5493560ac988 100644 --- a/python/ray/serve/_private/common.py +++ b/python/ray/serve/_private/common.py @@ -754,3 +754,85 @@ class CreatePlacementGroupRequest: "Converting by-value DeploymentResponses to ObjectRefs is not supported. " "Use handle.options(_by_reference=True) to enable it." ) + +RUNNING_REQUESTS_KEY = "running_requests" + + +@dataclass(order=True) +class TimeStampedValue: + timestamp: float + value: float = field(compare=False) + + +@dataclass +class HandleMetricReport: + """Report from a deployment handle on queued and ongoing requests. + + Args: + deployment_id: The deployment ID of the deployment handle. + handle_id: The handle ID of the deployment handle. + actor_id: If the deployment handle (from which this metric was + sent) lives on an actor, the ID of that actor. + handle_source: Describes what kind of entity holds this + deployment handle: a Serve proxy, a Serve replica, or + unknown. + queued_requests: The current number of queued requests at the + handle, i.e. requests that haven't been assigned to any + replica yet. + aggregated_metrics: A map of metric name to the aggregated value over the past + look_back_period_s seconds at the handle for each replica. + metrics: A map of metric name to the list of values running at that handle for each replica + over the past look_back_period_s seconds. This is a list because + we take multiple measurements over time. + timestamp: The time at which this report was created. + """ + + deployment_id: DeploymentID + handle_id: str + actor_id: str + handle_source: DeploymentHandleSource + queued_requests: float + aggregated_metrics: Dict[str, Dict[ReplicaID, float]] + metrics: Dict[str, Dict[ReplicaID, List[float]]] + timestamp: float + + @property + def total_requests(self) -> float: + """Total number of queued and running requests.""" + return self.queued_requests + sum( + self.aggregated_metrics.get(RUNNING_REQUESTS_KEY, {}).values() + ) + + @property + def is_serve_component_source(self) -> bool: + """Whether the handle source is a Serve actor. + + More specifically, this returns whether a Serve actor tracked + by the controller holds the deployment handle that sent this + report. If the deployment handle lives on a driver, a Ray task, + or an actor that's not a Serve replica, then this returns False. + """ + return self.handle_source in [ + DeploymentHandleSource.PROXY, + DeploymentHandleSource.REPLICA, + ] + + +@dataclass +class ReplicaMetricReport: + """Report from a replica on ongoing requests. + + Args: + replica_id: The replica ID of the replica. + aggregated_metrics: A map of metric name to the aggregated value over the past + look_back_period_s seconds at the replica. + metrics: A map of metric name to the list of values running at that replica + over the past look_back_period_s seconds. This is a list because + we take multiple measurements over time. + timestamp: The time at which this report was created. + """ + + replica_id: ReplicaID + aggregated_metrics: Dict[str, float] + metrics: Dict[str, List[float]] + timestamp: float diff --git a/python/ray/serve/_private/constants.py b/python/ray/serve/_private/constants.py index 774aee0c86f8..dc99f63ef838 100644 --- a/python/ray/serve/_private/constants.py +++ b/python/ray/serve/_private/constants.py @@ -496,3 +496,8 @@ RAY_SERVE_REQUEST_PATH_LOG_BUFFER_SIZE = 1000 RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP = False RAY_SERVE_LOG_TO_STDERR = False + +# The maximum allowed RPC latency in milliseconds. +# This is used to detect and warn about long RPC latencies +# between the controller and the replicas. +RAY_SERVE_RPC_LATENCY_WARNING_THRESHOLD_MS = 2000 diff --git a/python/ray/serve/_private/controller.py b/python/ray/serve/_private/controller.py index 5d53906cecf5..fa946121b37a 100644 --- a/python/ray/serve/_private/controller.py +++ b/python/ray/serve/_private/controller.py @@ -13,9 +13,11 @@ from ray.serve._private.application_state import ApplicationStateManager, StatusOverview from ray.serve._private.autoscaling_state import AutoscalingStateManager from ray.serve._private.common import ( - DeploymentHandleSource, + RUNNING_REQUESTS_KEY, DeploymentID, + HandleMetricReport, NodeId, + ReplicaMetricReport, RequestProtocol, RequestRoutingInfo, RunningReplicaInfo, @@ -25,6 +27,7 @@ from ray.serve._private.constants import ( CONTROL_LOOP_INTERVAL_S, RAY_SERVE_CONTROLLER_CALLBACK_IMPORT_PATH, + RAY_SERVE_RPC_LATENCY_WARNING_THRESHOLD_MS, RECOVERING_LONG_POLL_BROADCAST_TIMEOUT_S, SERVE_CONTROLLER_NAME, SERVE_DEFAULT_APP_NAME, @@ -259,38 +262,43 @@ def check_alive(self) -> None: def get_pid(self) -> int: return os.getpid() - def record_autoscaling_metrics( - self, replica_id: str, window_avg: Optional[float], send_timestamp: float + def record_autoscaling_metrics_from_replica( + self, replica_metric_report: ReplicaMetricReport ): logger.debug( - f"Received metrics from replica {replica_id}: {window_avg} running requests" + f"Received metrics from replica {replica_metric_report.replica_id}: {replica_metric_report.aggregated_metrics.get(RUNNING_REQUESTS_KEY)} running requests" ) + latency = time.time() - replica_metric_report.timestamp + latency_ms = latency * 1000 + if latency_ms > RAY_SERVE_RPC_LATENCY_WARNING_THRESHOLD_MS: + logger.warning( + f"Received autoscaling metrics from replica {replica_metric_report.replica_id} with timestamp {replica_metric_report.timestamp} " + f"which is {latency_ms}ms ago. " + f"This is greater than the warning threshold RPC latency of {RAY_SERVE_RPC_LATENCY_WARNING_THRESHOLD_MS}ms. " + "This may indicate a performance issue with the controller try increasing the RAY_SERVE_RPC_LATENCY_WARNING_THRESHOLD_MS environment variable." + ) self.autoscaling_state_manager.record_request_metrics_for_replica( - replica_id, window_avg, send_timestamp + replica_metric_report ) - def record_handle_metrics( - self, - deployment_id: str, - handle_id: str, - actor_id: Optional[str], - handle_source: DeploymentHandleSource, - queued_requests: float, - running_requests: Dict[str, float], - send_timestamp: float, + def record_autoscaling_metrics_from_handle( + self, handle_metric_report: HandleMetricReport ): logger.debug( - f"Received metrics from handle {handle_id} for deployment {deployment_id}: " - f"{queued_requests} queued requests and {running_requests} running requests" + f"Received metrics from handle {handle_metric_report.handle_id} for deployment {handle_metric_report.deployment_id}: " + f"{handle_metric_report.queued_requests} queued requests and {handle_metric_report.aggregated_metrics[RUNNING_REQUESTS_KEY]} running requests" ) + latency = time.time() - handle_metric_report.timestamp + latency_ms = latency * 1000 + if latency_ms > RAY_SERVE_RPC_LATENCY_WARNING_THRESHOLD_MS: + logger.warning( + f"Received autoscaling metrics from handle {handle_metric_report.handle_id} for deployment {handle_metric_report.deployment_id} with timestamp {handle_metric_report.timestamp} " + f"which is {latency_ms}ms ago. " + f"This is greater than the warning threshold RPC latency of {RAY_SERVE_RPC_LATENCY_WARNING_THRESHOLD_MS}ms. " + "This may indicate a performance issue with the controller try increasing the RAY_SERVE_RPC_LATENCY_WARNING_THRESHOLD_MS environment variable." + ) self.autoscaling_state_manager.record_request_metrics_for_handle( - deployment_id=deployment_id, - handle_id=handle_id, - actor_id=actor_id, - handle_source=handle_source, - queued_requests=queued_requests, - running_requests=running_requests, - send_timestamp=send_timestamp, + handle_metric_report ) def _dump_autoscaling_metrics_for_testing(self): diff --git a/python/ray/serve/_private/metrics_utils.py b/python/ray/serve/_private/metrics_utils.py index 509ed48ed65c..10d493979c61 100644 --- a/python/ray/serve/_private/metrics_utils.py +++ b/python/ray/serve/_private/metrics_utils.py @@ -3,7 +3,7 @@ import logging import statistics from collections import defaultdict -from dataclasses import dataclass, field +from dataclasses import dataclass from itertools import chain from typing import ( Callable, @@ -16,6 +16,7 @@ Tuple, ) +from ray.serve._private.common import TimeStampedValue from ray.serve._private.constants import ( METRICS_PUSHER_GRACEFUL_SHUTDOWN_TIMEOUT_S, SERVE_LOGGER_NAME, @@ -123,12 +124,6 @@ async def graceful_shutdown(self): self._async_tasks.clear() -@dataclass(order=True) -class TimeStampedValue: - timestamp: float - value: float = field(compare=False) - - class InMemoryMetricsStore: """A very simple, in memory time series database""" diff --git a/python/ray/serve/_private/replica.py b/python/ray/serve/_private/replica.py index 1ca38d1a37f2..4a94a326006a 100644 --- a/python/ray/serve/_private/replica.py +++ b/python/ray/serve/_private/replica.py @@ -39,8 +39,10 @@ from ray.remote_function import RemoteFunction from ray.serve import metrics from ray.serve._private.common import ( + RUNNING_REQUESTS_KEY, DeploymentID, ReplicaID, + ReplicaMetricReport, ReplicaQueueLengthInfo, RequestMetadata, ServeComponentType, @@ -329,10 +331,21 @@ def record_request_metrics(self, *, route: str, latency_ms: float, was_error: bo def _push_autoscaling_metrics(self) -> Dict[str, Any]: look_back_period = self._autoscaling_config.look_back_period_s self._metrics_store.prune_keys_and_compact_data(time.time() - look_back_period) - self._controller_handle.record_autoscaling_metrics.remote( + replica_metric_report = ReplicaMetricReport( replica_id=self._replica_id, - window_avg=self._metrics_store.aggregate_avg([self._replica_id])[0], - send_timestamp=time.time(), + timestamp=time.time(), + aggregated_metrics={ + RUNNING_REQUESTS_KEY: self._metrics_store.aggregate_avg( + [self._replica_id] + )[0] + or 0.0 + }, + metrics={ + RUNNING_REQUESTS_KEY: self._metrics_store.data.get(self._replica_id, []) + }, + ) + self._controller_handle.record_autoscaling_metrics_from_replica.remote( + replica_metric_report ) def _add_autoscaling_metrics_point(self) -> None: diff --git a/python/ray/serve/_private/router.py b/python/ray/serve/_private/router.py index 7acc79f17fac..02af2ce0a53c 100644 --- a/python/ray/serve/_private/router.py +++ b/python/ray/serve/_private/router.py @@ -25,9 +25,11 @@ from ray.actor import ActorHandle from ray.exceptions import ActorDiedError, ActorUnavailableError, RayError from ray.serve._private.common import ( + RUNNING_REQUESTS_KEY, DeploymentHandleSource, DeploymentID, DeploymentTargetInfo, + HandleMetricReport, ReplicaID, RequestMetadata, RunningReplicaInfo, @@ -46,6 +48,7 @@ QUEUED_REQUESTS_KEY, InMemoryMetricsStore, MetricsPusher, + TimeStampedValue, ) from ray.serve._private.replica_result import ReplicaResult from ray.serve._private.request_router import PendingRequest, RequestRouter @@ -360,14 +363,8 @@ def push_autoscaling_metrics_to_controller(self): These metrics are used by the controller for autoscaling. """ - - self._controller_handle.record_handle_metrics.remote( - send_timestamp=time.time(), - deployment_id=self._deployment_id, - handle_id=self._handle_id, - actor_id=self._self_actor_id, - handle_source=self._handle_source, - **self._get_aggregated_requests(), + self._controller_handle.record_autoscaling_metrics_from_handle.remote( + self._get_metrics_report() ) def _add_autoscaling_metrics_point(self): @@ -389,25 +386,40 @@ def _add_autoscaling_metrics_point(self): start_timestamp = time.time() - self.autoscaling_config.look_back_period_s self.metrics_store.prune_keys_and_compact_data(start_timestamp) - def _get_aggregated_requests(self): + def _get_metrics_report(self) -> HandleMetricReport: running_requests = dict() + avg_running_requests = dict() + timestamp = time.time() if RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE and self.autoscaling_config: look_back_period = self.autoscaling_config.look_back_period_s self.metrics_store.prune_keys_and_compact_data( time.time() - look_back_period ) - running_requests = { - replica_id: self.metrics_store.aggregate_avg([replica_id])[0] - # If data hasn't been recorded yet, return current - # number of queued and ongoing requests. - or num_requests - for replica_id, num_requests in self.num_requests_sent_to_replicas.items() # noqa: E501 - } + for replica_id, num_requests in self.num_requests_sent_to_replicas.items(): + # Calculate avg running requests + avg_running_requests[replica_id] = ( + self.metrics_store.aggregate_avg([replica_id])[0] + # If data hasn't been recorded yet, return current + # number of queued and ongoing requests. + or num_requests + ) + # Get running requests data + running_requests[replica_id] = self.metrics_store.data.get( + replica_id, [TimeStampedValue(timestamp, num_requests)] + ) - return { - "queued_requests": self.num_queued_requests, - "running_requests": running_requests, - } + handle_metric_report = HandleMetricReport( + deployment_id=self._deployment_id, + handle_id=self._handle_id, + actor_id=self._self_actor_id, + handle_source=self._handle_source, + queued_requests=self.num_queued_requests, + aggregated_metrics={RUNNING_REQUESTS_KEY: avg_running_requests}, + metrics={RUNNING_REQUESTS_KEY: running_requests}, + timestamp=timestamp, + ) + + return handle_metric_report async def shutdown(self): """Shutdown metrics manager gracefully.""" diff --git a/python/ray/serve/tests/test_autoscaling_policy.py b/python/ray/serve/tests/test_autoscaling_policy.py index 98120b3e1626..707cc5179b14 100644 --- a/python/ray/serve/tests/test_autoscaling_policy.py +++ b/python/ray/serve/tests/test_autoscaling_policy.py @@ -378,7 +378,7 @@ async def call(self): # Wait for deployment A to scale up wait_for_condition(check_num_requests_eq, client=client, id=dep_id, expected=20) - wait_for_condition(check_num_replicas_eq, name="A", target=5) + wait_for_condition(check_num_replicas_eq, name="A", target=5, timeout=20) print("Confirmed deployment scaled to 5 replicas.") # Kill CallerActor diff --git a/python/ray/serve/tests/unit/test_deployment_state.py b/python/ray/serve/tests/unit/test_deployment_state.py index ad093c08ab3d..ea910ee826d7 100644 --- a/python/ray/serve/tests/unit/test_deployment_state.py +++ b/python/ray/serve/tests/unit/test_deployment_state.py @@ -8,13 +8,17 @@ from ray._common.ray_constants import DEFAULT_MAX_CONCURRENCY_ASYNC from ray.serve._private.autoscaling_state import AutoscalingStateManager from ray.serve._private.common import ( + RUNNING_REQUESTS_KEY, DeploymentHandleSource, DeploymentID, DeploymentStatus, DeploymentStatusTrigger, + HandleMetricReport, ReplicaID, + ReplicaMetricReport, ReplicaState, TargetCapacityDirection, + TimeStampedValue, ) from ray.serve._private.config import DeploymentConfig, ReplicaConfig from ray.serve._private.constants import ( @@ -2814,24 +2818,42 @@ def test_basic_autoscaling( req_per_replica = 2 if target_capacity_direction == "up" else 0 replicas = ds._replicas.get() if RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE: - asm.record_request_metrics_for_handle( + handle_metric_report = HandleMetricReport( deployment_id=TEST_DEPLOYMENT_ID, handle_id="random", - actor_id=None, + actor_id="actor_id", handle_source=DeploymentHandleSource.UNKNOWN, queued_requests=0, - running_requests={ - replica._actor.replica_id: req_per_replica for replica in replicas + aggregated_metrics={ + RUNNING_REQUESTS_KEY: { + replica._actor.replica_id: req_per_replica + for replica in replicas + } }, - send_timestamp=timer.time(), + metrics={ + RUNNING_REQUESTS_KEY: { + replica._actor.replica_id: [ + TimeStampedValue(timer.time(), req_per_replica) + ] + for replica in replicas + } + }, + timestamp=timer.time(), ) + asm.record_request_metrics_for_handle(handle_metric_report) else: for replica in replicas: - asm.record_request_metrics_for_replica( + replica_metric_report = ReplicaMetricReport( replica_id=replica._actor.replica_id, - window_avg=req_per_replica, - send_timestamp=timer.time(), + aggregated_metrics={RUNNING_REQUESTS_KEY: req_per_replica}, + metrics={ + RUNNING_REQUESTS_KEY: [ + TimeStampedValue(timer.time(), req_per_replica) + ] + }, + timestamp=timer.time(), ) + asm.record_request_metrics_for_replica(replica_metric_report) # status=UPSCALING/DOWNSCALING, status_trigger=AUTOSCALE dsm.update() @@ -2972,20 +2994,35 @@ def test_downscaling_reclaiming_starting_replicas_first( running_replicas = ds._replicas.get(states=[ReplicaState.RUNNING]) replicas = ds._replicas.get() if RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE: - asm.record_request_metrics_for_handle( + handle_metric_report = HandleMetricReport( deployment_id=TEST_DEPLOYMENT_ID, handle_id="random", - actor_id=None, + actor_id="actor_id", handle_source=DeploymentHandleSource.UNKNOWN, queued_requests=0, - running_requests={replica._actor.replica_id: 2 for replica in replicas}, - send_timestamp=timer.time(), + aggregated_metrics={ + RUNNING_REQUESTS_KEY: { + replica._actor.replica_id: 2 for replica in replicas + } + }, + metrics={ + RUNNING_REQUESTS_KEY: { + replica._actor.replica_id: [TimeStampedValue(timer.time(), 2)] + for replica in replicas + } + }, + timestamp=timer.time(), ) + asm.record_request_metrics_for_handle(handle_metric_report) else: for replica in replicas: - asm.record_request_metrics_for_replica( - replica._actor.replica_id, 2, timer.time() + replica_metric_report = ReplicaMetricReport( + replica_id=replica._actor.replica_id, + aggregated_metrics={RUNNING_REQUESTS_KEY: 2}, + metrics={RUNNING_REQUESTS_KEY: [TimeStampedValue(timer.time(), 2)]}, + timestamp=timer.time(), ) + asm.record_request_metrics_for_replica(replica_metric_report) # status=UPSCALING, status_trigger=AUTOSCALE dsm.update() @@ -3049,20 +3086,35 @@ def test_downscaling_reclaiming_starting_replicas_first( # Now, trigger downscaling attempting to reclaim half (3) of the replicas replicas = ds._replicas.get(states=[ReplicaState.RUNNING]) if RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE: - asm.record_request_metrics_for_handle( + handle_metric_report = HandleMetricReport( deployment_id=TEST_DEPLOYMENT_ID, handle_id="random", - actor_id=None, + actor_id="actor_id", handle_source=DeploymentHandleSource.UNKNOWN, queued_requests=0, - running_requests={replica._actor.replica_id: 1 for replica in replicas}, - send_timestamp=timer.time(), + aggregated_metrics={ + RUNNING_REQUESTS_KEY: { + replica._actor.replica_id: 1 for replica in replicas + } + }, + metrics={ + RUNNING_REQUESTS_KEY: { + replica._actor.replica_id: [TimeStampedValue(timer.time(), 1)] + for replica in replicas + } + }, + timestamp=timer.time(), ) + asm.record_request_metrics_for_handle(handle_metric_report) else: for replica in replicas: - asm.record_request_metrics_for_replica( - replica._actor.replica_id, 1, timer.time() + replica_metric_report = ReplicaMetricReport( + replica_id=replica._actor.replica_id, + aggregated_metrics={RUNNING_REQUESTS_KEY: 1}, + metrics={RUNNING_REQUESTS_KEY: [TimeStampedValue(timer.time(), 1)]}, + timestamp=timer.time(), ) + asm.record_request_metrics_for_replica(replica_metric_report) # status=DOWNSCALING, status_trigger=AUTOSCALE dsm.update() @@ -3139,20 +3191,35 @@ def test_update_autoscaling_config(self, mock_deployment_state_manager): # Num ongoing requests = 1, status should remain HEALTHY replicas = ds._replicas.get() if RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE: - asm.record_request_metrics_for_handle( + handle_metric_report = HandleMetricReport( deployment_id=TEST_DEPLOYMENT_ID, handle_id="random", - actor_id=None, + actor_id="actor_id", handle_source=DeploymentHandleSource.UNKNOWN, queued_requests=0, - running_requests={replica._actor.replica_id: 1 for replica in replicas}, - send_timestamp=timer.time(), + aggregated_metrics={ + RUNNING_REQUESTS_KEY: { + replica._actor.replica_id: 1 for replica in replicas + } + }, + metrics={ + RUNNING_REQUESTS_KEY: { + replica._actor.replica_id: [TimeStampedValue(timer.time(), 1)] + for replica in replicas + } + }, + timestamp=timer.time(), ) + asm.record_request_metrics_for_handle(handle_metric_report) else: for replica in replicas: - asm.record_request_metrics_for_replica( - replica._actor.replica_id, 1, timer.time() + replica_metric_report = ReplicaMetricReport( + replica_id=replica._actor.replica_id, + aggregated_metrics={RUNNING_REQUESTS_KEY: 1}, + metrics={RUNNING_REQUESTS_KEY: [TimeStampedValue(timer.time(), 1)]}, + timestamp=timer.time(), ) + asm.record_request_metrics_for_replica(replica_metric_report) check_counts(ds, total=3, by_state=[(ReplicaState.RUNNING, 3, None)]) assert ds.curr_status_info.status == DeploymentStatus.HEALTHY @@ -3237,15 +3304,17 @@ def test_replicas_fail_during_initial_scale_from_zero( ds: DeploymentState = dsm._deployment_states[TEST_DEPLOYMENT_ID] # Send request metrics to controller to make the deployment upscale - asm.record_request_metrics_for_handle( + handle_metric_report = HandleMetricReport( deployment_id=TEST_DEPLOYMENT_ID, handle_id="random", - actor_id=None, + actor_id="actor_id", handle_source=DeploymentHandleSource.UNKNOWN, queued_requests=1, - running_requests={}, - send_timestamp=timer.time(), + aggregated_metrics={}, + metrics={}, + timestamp=timer.time(), ) + asm.record_request_metrics_for_handle(handle_metric_report) # The controller should try to start a new replica. If that replica repeatedly # fails to start, the deployment should transition to UNHEALTHY and NOT retry @@ -3351,15 +3420,17 @@ def test_replicas_fail_during_subsequent_scale_from_zero( check_counts(ds, total=0) # Send request metrics to controller to make the deployment upscale - asm.record_request_metrics_for_handle( + handle_metric_report = HandleMetricReport( deployment_id=TEST_DEPLOYMENT_ID, handle_id="random", - actor_id=None, + actor_id="actor_id", handle_source=DeploymentHandleSource.UNKNOWN, queued_requests=1, - running_requests={}, - send_timestamp=timer.time(), + aggregated_metrics={}, + metrics={}, + timestamp=timer.time(), ) + asm.record_request_metrics_for_handle(handle_metric_report) # The controller should try to start a new replica. If that replica repeatedly # fails to start, the deployment should transition to UNHEALTHY. Meanwhile @@ -3425,15 +3496,25 @@ def test_handle_metrics_timeout(self, mock_deployment_state_manager): check_counts(ds, total=1, by_state=[(ReplicaState.RUNNING, 1, None)]) # Record 2 requests/replica -> trigger upscale - asm.record_request_metrics_for_handle( + handle_metric_report = HandleMetricReport( deployment_id=TEST_DEPLOYMENT_ID, handle_id="random", - actor_id=None, + actor_id="actor_id", handle_source=DeploymentHandleSource.UNKNOWN, queued_requests=0, - running_requests={ds._replicas.get()[0]._actor.replica_id: 2}, - send_timestamp=timer.time(), + aggregated_metrics={ + RUNNING_REQUESTS_KEY: {ds._replicas.get()[0]._actor.replica_id: 2} + }, + metrics={ + RUNNING_REQUESTS_KEY: { + ds._replicas.get()[0]._actor.replica_id: [ + TimeStampedValue(timer.time(), 2) + ] + } + }, + timestamp=timer.time(), ) + asm.record_request_metrics_for_handle(handle_metric_report) asm.drop_stale_handle_metrics(dsm.get_alive_replica_actor_ids()) dsm.update() check_counts( @@ -3511,15 +3592,25 @@ def test_handle_metrics_on_dead_serve_actor(self, mock_deployment_state_manager) check_counts(ds2, total=1, by_state=[(ReplicaState.RUNNING, 1, None)]) # Record 2 requests/replica (sent from d2 replica) -> trigger upscale - asm.record_request_metrics_for_handle( + handle_metric_report = HandleMetricReport( deployment_id=d_id1, handle_id="random", actor_id="d2_replica_actor_id", handle_source=DeploymentHandleSource.REPLICA, queued_requests=0, - running_requests={ds1._replicas.get()[0]._actor.replica_id: 2}, - send_timestamp=timer.time(), + aggregated_metrics={ + RUNNING_REQUESTS_KEY: {ds1._replicas.get()[0]._actor.replica_id: 2} + }, + metrics={ + RUNNING_REQUESTS_KEY: { + ds1._replicas.get()[0]._actor.replica_id: [ + TimeStampedValue(timer.time(), 2) + ] + } + }, + timestamp=timer.time(), ) + asm.record_request_metrics_for_handle(handle_metric_report) asm.drop_stale_handle_metrics(dsm.get_alive_replica_actor_ids()) dsm.update() check_counts( diff --git a/python/ray/serve/tests/unit/test_router.py b/python/ray/serve/tests/unit/test_router.py index 5b849c965f78..e08ffe3a5e65 100644 --- a/python/ray/serve/tests/unit/test_router.py +++ b/python/ray/serve/tests/unit/test_router.py @@ -1006,14 +1006,9 @@ async def test_push_autoscaling_metrics_to_controller(self): # Check metrics are pushed correctly metrics_manager.push_autoscaling_metrics_to_controller() - mock_controller_handle.record_handle_metrics.remote.assert_called_with( - deployment_id=deployment_id, - handle_id=handle_id, - actor_id=self_actor_id, - handle_source=DeploymentHandleSource.PROXY, - queued_requests=n, - running_requests=running_requests, - send_timestamp=start, + handle_metric_report = metrics_manager._get_metrics_report() + mock_controller_handle.record_autoscaling_metrics_from_handle.remote.assert_called_with( + handle_metric_report ) @pytest.mark.skipif( From dc954edf81bad9b1f9f1860d035cfbc9aea20b66 Mon Sep 17 00:00:00 2001 From: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Date: Wed, 10 Sep 2025 10:11:48 -0700 Subject: [PATCH 556/634] [Serve] Allow access to request context of each request in the batch (#56344) This is resolved by keeping a temporary list of request context of each request for the batch as part of the `ContextVar`. This temporary list will exist while the batching method is executed and will be reset after the method completes executing. Users will be able to access to the list of request contexts, which corresponds to the batch, through `_get_serve_batch_request_context`. **Output example of `get_serve_batch_request_context` when there was 10 requests in a single batch:** `[_RequestContext(route='/', request_id='b99a517d-f813-4833-8365-73626c78a144', _internal_request_id='c224b33d-93ae-44b1-a16a-bffe87adc098', app_name='default', multiplexed_model_id='', grpc_context=None, is_http_request=False, cancel_on_parent_request_cancel=False), _RequestContext(route='/', request_id='e7dd2b1d-18b2-4001-b0c8-3d50ebd78a4f', _internal_request_id='ff15eb47-e428-49f8-a5a4-3187b32aad3e', app_name='default', multiplexed_model_id='', grpc_context=None, is_http_request=False, cancel_on_parent_request_cancel=False), _RequestContext(route='/', request_id='63de5f3a-7814-4714-8544-bc4ef5e53740', _internal_request_id='005ff49e-0ec2-482c-b9fd-629dcb5db3e8', app_name='default', multiplexed_model_id='', grpc_context=None, is_http_request=False, cancel_on_parent_request_cancel=False), _RequestContext(route='/', request_id='46d9ebb3-7368-4b06-ae5b-aba57dc9a797', _internal_request_id='6624244b-efc7-43a0-be98-633cb4ce6161', app_name='default', multiplexed_model_id='', grpc_context=None, is_http_request=False, cancel_on_parent_request_cancel=False), _RequestContext(route='/', request_id='bb3adaac-2580-446e-b1b7-9d057400dec7', _internal_request_id='3d25a586-63bf-46e2-861e-2947af9c9100', app_name='default', multiplexed_model_id='', grpc_context=None, is_http_request=False, cancel_on_parent_request_cancel=False), _RequestContext(route='/', request_id='d805322c-12bc-4c84-aaee-bbf9c8e77482', _internal_request_id='77dbf78d-5716-4a38-9145-d6855404ecf0', app_name='default', multiplexed_model_id='', grpc_context=None, is_http_request=False, cancel_on_parent_request_cancel=False), _RequestContext(route='/', request_id='551dab8a-637f-4804-8f51-ed5c946be5bf', _internal_request_id='e8d362d7-186d-4c76-b79b-819bbd9891ae', app_name='default', multiplexed_model_id='', grpc_context=None, is_http_request=False, cancel_on_parent_request_cancel=False), _RequestContext(route='/', request_id='60a866f7-e2aa-40ec-bb9e-deb583c9bed5', _internal_request_id='89e5e32f-fc09-4c26-9c3d-a22b5b4da683', app_name='default', multiplexed_model_id='', grpc_context=None, is_http_request=False, cancel_on_parent_request_cancel=False), _RequestContext(route='/', request_id='61a482c6-c383-4bc1-abca-ba5f8f0156c7', _internal_request_id='1853d2c8-3676-4ad8-822d-75ec58578d8c', app_name='default', multiplexed_model_id='', grpc_context=None, is_http_request=False, cancel_on_parent_request_cancel=False), _RequestContext(route='/', request_id='988d5aa0-54d1-4311-be17-2ab48fb017f3', _internal_request_id='fd84a3e0-7aa1-4491-98be-65cf55de42e5', app_name='default', multiplexed_model_id='', grpc_context=None, is_http_request=False, cancel_on_parent_request_cancel=False)]` --------- Signed-off-by: doyoung --- python/ray/serve/batching.py | 16 ++++- python/ray/serve/context.py | 23 +++++- python/ray/serve/tests/test_batching.py | 94 ++++++++++++++++++++++++- 3 files changed, 130 insertions(+), 3 deletions(-) diff --git a/python/ray/serve/batching.py b/python/ray/serve/batching.py index 41c1b64e516a..ab16fe47e962 100644 --- a/python/ray/serve/batching.py +++ b/python/ray/serve/batching.py @@ -45,6 +45,7 @@ class _SingleRequest: self_arg: Any flattened_args: List[Any] future: asyncio.Future + request_context: serve.context._RequestContext @dataclass @@ -306,6 +307,9 @@ async def _assign_func_results( async def _process_batches(self, func: Callable) -> None: """Loops infinitely and processes queued request batches.""" + # When asyncio task is created, the task will inherit the request context from the current context. + # So we unset the request context so the current context is not inherited by the task, _process_batch. + serve.context._unset_request_context() while not self._loop.is_closed(): batch = await self.wait_for_batch() promise = self._process_batch(func, batch) @@ -343,6 +347,11 @@ async def _process_batch(self, func: Callable, batch: List[_SingleRequest]) -> N else: func_future_or_generator = func(*args, **kwargs) + # Add individual request context to the batch request context + serve.context._set_batch_request_context( + [req.request_context for req in batch] + ) + if isasyncgenfunction(func): func_generator = func_future_or_generator await self._consume_func_generator( @@ -352,6 +361,8 @@ async def _process_batch(self, func: Callable, batch: List[_SingleRequest]) -> N func_future = func_future_or_generator await self._assign_func_results(func_future, futures, len(batch)) + # Reset the batch request context after the batch is processed + serve.context._set_batch_request_context([]) except Exception as e: logger.exception("_process_batch ran into an unexpected exception.") @@ -690,7 +701,10 @@ def enqueue_request(args, kwargs) -> asyncio.Future: batch_queue = lazy_batch_queue_wrapper.queue future = get_or_create_event_loop().create_future() - batch_queue.put(_SingleRequest(self, flattened_args, future)) + request_context = serve.context._get_serve_request_context() + batch_queue.put( + _SingleRequest(self, flattened_args, future, request_context) + ) return future @wraps(_func) diff --git a/python/ray/serve/context.py b/python/ray/serve/context.py index 99b8e5f0d9e8..b6ad7bb2a685 100644 --- a/python/ray/serve/context.py +++ b/python/ray/serve/context.py @@ -8,7 +8,7 @@ import logging from collections import defaultdict from dataclasses import dataclass -from typing import Callable, Dict, Optional +from typing import Callable, Dict, List, Optional import ray from ray.exceptions import RayActorError @@ -195,6 +195,10 @@ class _RequestContext: "Serve internal request context variable", default=None ) +_serve_batch_request_context = contextvars.ContextVar( + "Serve internal batching request context variable", default=None +) + def _get_serve_request_context(): """Get the current request context. @@ -208,6 +212,13 @@ def _get_serve_request_context(): return _serve_request_context.get() +def _get_serve_batch_request_context(): + """Get the list of request contexts for the current batch.""" + if _serve_batch_request_context.get() is None: + _serve_batch_request_context.set([]) + return _serve_batch_request_context.get() + + def _set_request_context( route: str = "", request_id: str = "", @@ -233,6 +244,16 @@ def _set_request_context( ) +def _unset_request_context(): + """Unset the request context.""" + _serve_request_context.set(_RequestContext()) + + +def _set_batch_request_context(request_contexts: List[_RequestContext]): + """Add the request context to the batch request context.""" + _serve_batch_request_context.set(request_contexts) + + # `_requests_pending_assignment` is a map from request ID to a # dictionary of asyncio tasks. # The request ID points to an ongoing request that is executing on the diff --git a/python/ray/serve/tests/test_batching.py b/python/ray/serve/tests/test_batching.py index 80f9f35eb10c..ac149e79ad13 100644 --- a/python/ray/serve/tests/test_batching.py +++ b/python/ray/serve/tests/test_batching.py @@ -3,6 +3,7 @@ from collections.abc import Callable from concurrent.futures.thread import ThreadPoolExecutor from functools import partial +from threading import Thread from typing import List, Optional import httpx @@ -13,6 +14,10 @@ from ray._common.test_utils import SignalActor, async_wait_for_condition from ray.serve._private.test_utils import get_application_url from ray.serve.batching import _RuntimeSummaryStatistics +from ray.serve.context import ( + _get_serve_batch_request_context, + _get_serve_request_context, +) def test_batching(serve_instance): @@ -214,7 +219,7 @@ async def __call__(self, request): @pytest.mark.parametrize("max_batch_size", [1, 10]) @pytest.mark.parametrize("n_requests", [1, 10]) async def test_observability_helpers( - n_requests: int, max_batch_size: int, max_concurrent_batches: int + serve_instance, n_requests: int, max_batch_size: int, max_concurrent_batches: int ) -> None: """Checks observability helper methods that are used for batching. @@ -310,6 +315,93 @@ async def poll() -> bool: return await async_wait_for_condition(poll) +def test_batching_request_context(serve_instance): + """Test that _get_serve_batch_request_context() works correctly with batching. + + With 6 requests and max_batch_size=3, Serve should create 2 batches processed in parallel. + Each batch should have access to the request contexts of all requests in that batch, + and context should be properly unset after processing. + """ + + @serve.deployment(max_ongoing_requests=10) + class BatchContextTester: + def __init__(self): + self.batch_results = [] + + @serve.batch( + max_batch_size=3, batch_wait_timeout_s=1.0, max_concurrent_batches=2 + ) + async def handle_batch(self, batch): + # Store results for verification + batch_result = { + "batch_size": len(batch), + "batch_request_contexts": _get_serve_batch_request_context(), + "current_request_context": _get_serve_request_context(), + } + self.batch_results.append(batch_result) + + return ["ok" for _ in range(len(batch))] + + async def __call__(self, request): + return await self.handle_batch(1) + + async def get_results(self): + return self.batch_results + + handle = serve.run(BatchContextTester.bind()) + + def do_request(): + """Make a request with a specific request ID.""" + url = get_application_url() + r = httpx.post(f"{url}/") + r.raise_for_status() + + # Launch 6 requests. Expect 2 batches of 3 requests each. + threads = [Thread(target=do_request) for _ in range(6)] + + for t in threads: + t.start() + for t in threads: + t.join() + + # Get results from the deployment + batch_results = handle.get_results.remote().result() + + # Verify each batch has correct size and context + total_requests_processed = 0 + request_ids_in_batch_context = set() + + for result in batch_results: + # Batch context should contain all 3 request contexts + assert ( + len(result["batch_request_contexts"]) == 3 + ), f"Expected 3 contexts in batch, got {result['batch_request_contexts']}" + req_ids_in_batch_context = [ + ctx.request_id for ctx in result["batch_request_contexts"] + ] + assert ( + len(req_ids_in_batch_context) == 3 + ), f"Expected 3 batch request IDs, got {len(req_ids_in_batch_context)}" + request_ids_in_batch_context.update(req_ids_in_batch_context) + + # Current request context read within the batcher should be a default empty context. + current_request_context = result["current_request_context"] + assert current_request_context.request_id == "" + assert current_request_context.route == "" + assert current_request_context.app_name == "" + assert current_request_context.multiplexed_model_id == "" + + total_requests_processed += result["batch_size"] + + # Verify all 6 requests were processed + assert ( + total_requests_processed == 6 + ), f"Expected 6 total requests processed, got {total_requests_processed}" + assert ( + len(request_ids_in_batch_context) == 6 + ), f"Expected 6 unique request IDs, got {len(request_ids_in_batch_context)}" + + if __name__ == "__main__": import sys From a6ee029da86d48e9dea9bc9258c2aec2ed15b479 Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Wed, 10 Sep 2025 10:47:25 -0700 Subject: [PATCH 557/634] [data] fix metrics query for iteration + scheduling loop (#56390) ## Why are these changes needed? Im dumb, ctrl+h to find and replace and replaced a bit too much :'( doubled-checked these are the only places ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- .../modules/metrics/dashboards/data_dashboard_panels.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py index 10cb946ae41b..bc185d3854cf 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py @@ -680,7 +680,7 @@ unit="seconds", targets=[ Target( - expr='sum(ray_data_iter_initialize_seconds{{{global_filters}, operator=~"$Operator"}}) by (dataset)', + expr="sum(ray_data_iter_initialize_seconds{{{global_filters}}}) by (dataset)", legend="Seconds: {{dataset}}, {{operator}}", ) ], @@ -695,7 +695,7 @@ unit="seconds", targets=[ Target( - expr='sum(ray_data_iter_total_blocked_seconds{{{global_filters}, operator=~"$Operator"}}) by (dataset)', + expr="sum(ray_data_iter_total_blocked_seconds{{{global_filters}}}) by (dataset)", legend="Seconds: {{dataset}}", ) ], @@ -710,7 +710,7 @@ unit="seconds", targets=[ Target( - expr='sum(ray_data_iter_user_seconds{{{global_filters}, operator=~"$Operator"}}) by (dataset)', + expr="sum(ray_data_iter_user_seconds{{{global_filters}}}) by (dataset)", legend="Seconds: {{dataset}}", ) ], @@ -726,7 +726,7 @@ unit="seconds", targets=[ Target( - expr='sum(ray_data_sched_loop_duration_s{{{global_filters}, operator=~"$Operator"}}) by (dataset)', + expr="sum(ray_data_sched_loop_duration_s{{{global_filters}}}) by (dataset)", legend="Scheduling Loop Duration: {{dataset}}", ) ], From 08a7068b98fef676a7486da21c7865ffcb5412df Mon Sep 17 00:00:00 2001 From: coqian Date: Wed, 10 Sep 2025 10:47:41 -0700 Subject: [PATCH 558/634] [Data] Emit events rather than just logs for detected issues (#55717) ## Why are these changes needed? In Ray Data, we have an issue detection feature, but they are only available as printed logs. We also want to show these insights at our Data Dashboard for users to debug their code, by exporting these events. Example export event: ``` { "event_id": "Ce4Ee1B7AFd9Cb13B3", "timestamp": 1756423457, "source_type": "EXPORT_DATASET_OPERATOR_EVENT", "event_data": { "dataset_id": "dataset_6_0", "operator_id": "ReadImage->Map(preprocess_image)_1", "operator_name": "ReadImage->Map(preprocess_image)", "event_time": 1756423457.9333386, "event_type": "ISSUE_DETECTION_HIGH_MEMORY", "message": "\n\nOperator 'ReadImage->Map(preprocess_image)' uses xxx of memory per\ntask on average, but Ray only requests xxx per task at the start of\nthe pipeline.\n\nTo avoid out-of-memory errors, consider setting `memory=xxx` in the\nappropriate function or method call. (This might be unnecessary if the\nnumber of concurrent tasks is low.)\n\nTo change the frequency of this warning, set\n`DataContext.get_current().issue_detectors_config.high_memory_detector_config.detection_time_interval_s`,\nor disable the warning by setting value to -1. (current value: xxx)\n" } }, { "event_id": "bE831Ee49c4AadAcFB", "timestamp": 1756423458, "source_type": "EXPORT_DATASET_OPERATOR_EVENT", "event_data": { "dataset_id": "dataset_6_0", "operator_id": "MapBatches(ResnetModel)_2", "operator_name": "MapBatches(ResnetModel)", "event_time": 1756423458.1420121, "event_type": "ISSUE_DETECTION_HANGING", "message": "A task of operator MapBatches(ResnetModel) with task index xxx has been running for xxxs, which is longer than the average task duration of this operator (xxxs). If this message persists, please check the stack trace of the task for potential hanging issues." } } ``` ## Related issue number ## Checks - [X] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [X] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: cong.qian --- .../ray/_private/event/export_event_logger.py | 12 ++ .../issue_detection/issue_detector_manager.py | 31 +++- .../data/_internal/operator_event_exporter.py | 164 ++++++++++++++++++ .../tests/test_issue_detection_manager.py | 45 ++++- src/ray/protobuf/BUILD.bazel | 11 ++ .../export_dataset_operator_event.proto | 46 +++++ src/ray/protobuf/export_event.proto | 3 + 7 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 python/ray/data/_internal/operator_event_exporter.py create mode 100644 src/ray/protobuf/export_dataset_operator_event.proto diff --git a/python/ray/_private/event/export_event_logger.py b/python/ray/_private/event/export_event_logger.py index 4d47c68fb833..4e77ca1421ce 100644 --- a/python/ray/_private/event/export_event_logger.py +++ b/python/ray/_private/event/export_event_logger.py @@ -13,6 +13,9 @@ from ray.core.generated.export_dataset_metadata_pb2 import ( ExportDatasetMetadata, ) +from ray.core.generated.export_dataset_operator_event_pb2 import ( + ExportDatasetOperatorEventData, +) from ray.core.generated.export_event_pb2 import ExportEvent from ray.core.generated.export_submission_job_event_pb2 import ( ExportSubmissionJobEventData, @@ -31,6 +34,7 @@ ExportTrainRunEventData, ExportTrainRunAttemptEventData, ExportDatasetMetadata, + ExportDatasetOperatorEventData, ] @@ -43,6 +47,7 @@ class EventLogType(Enum): TRAIN_STATE: Export events related to training state, supporting train run and attempt events. SUBMISSION_JOB: Export events related to job submissions. DATASET_METADATA: Export events related to dataset metadata. + DATASET_OPERATOR_EVENT: Export events related to Ray Data operator. """ TRAIN_STATE = ( @@ -51,6 +56,10 @@ class EventLogType(Enum): ) SUBMISSION_JOB = ("EXPORT_SUBMISSION_JOB", {ExportSubmissionJobEventData}) DATASET_METADATA = ("EXPORT_DATASET_METADATA", {ExportDatasetMetadata}) + DATASET_OPERATOR_EVENT = ( + "EXPORT_DATASET_OPERATOR_EVENT", + {ExportDatasetOperatorEventData}, + ) def __init__(self, log_type_name: str, event_types: set[ExportEventDataType]): """Initialize an EventLogType enum value. @@ -119,6 +128,9 @@ def _create_export_event(self, event_data: ExportEventDataType) -> ExportEvent: elif isinstance(event_data, ExportDatasetMetadata): event.dataset_metadata.CopyFrom(event_data) event.source_type = ExportEvent.SourceType.EXPORT_DATASET_METADATA + elif isinstance(event_data, ExportDatasetOperatorEventData): + event.dataset_operator_event_data.CopyFrom(event_data) + event.source_type = ExportEvent.SourceType.EXPORT_DATASET_OPERATOR_EVENT else: raise TypeError(f"Invalid event_data type: {type(event_data)}") if not self.log_type.supports_event_type(event_data): diff --git a/python/ray/data/_internal/issue_detection/issue_detector_manager.py b/python/ray/data/_internal/issue_detection/issue_detector_manager.py index 91569e16deac..33ebbc69dafe 100644 --- a/python/ray/data/_internal/issue_detection/issue_detector_manager.py +++ b/python/ray/data/_internal/issue_detection/issue_detector_manager.py @@ -2,11 +2,19 @@ import time from typing import TYPE_CHECKING, Dict, List +from ray.core.generated.export_dataset_operator_event_pb2 import ( + ExportDatasetOperatorEventData as ProtoOperatorEventData, +) from ray.data._internal.issue_detection.issue_detector import ( Issue, IssueDetector, IssueType, ) +from ray.data._internal.operator_event_exporter import ( + OperatorEvent, + format_export_issue_event_name, + get_operator_event_exporter, +) if TYPE_CHECKING: from ray.data._internal.execution.interfaces.physical_operator import ( @@ -27,6 +35,7 @@ def __init__(self, executor: "StreamingExecutor"): detector: time.perf_counter() for detector in self._issue_detectors } self.executor = executor + self._operator_event_exporter = get_operator_event_exporter() def invoke_detectors(self) -> None: curr_time = time.perf_counter() @@ -47,8 +56,10 @@ def invoke_detectors(self) -> None: def _report_issues(self, issues: List[Issue]) -> None: operators: Dict[str, "PhysicalOperator"] = {} - for operator in self.executor._topology.keys(): + op_to_id: Dict["PhysicalOperator", str] = {} + for i, operator in enumerate(self.executor._topology.keys()): operators[operator.id] = operator + op_to_id[operator] = self.executor._get_operator_id(operator, i) # Reset issue detector metrics for each operator so that previous issues # don't affect the current ones. operator.metrics._issue_detector_hanging = 0 @@ -59,6 +70,24 @@ def _report_issues(self, issues: List[Issue]) -> None: operator = operators.get(issue.operator_id) if not operator: continue + + issue_event_type = format_export_issue_event_name(issue.issue_type) + if ( + self._operator_event_exporter is not None + and issue_event_type + in ProtoOperatorEventData.DatasetOperatorEventType.keys() + ): + event_time = time.time() + operator_event = OperatorEvent( + dataset_id=issue.dataset_name, + operator_id=op_to_id[operator], + operator_name=operator.name, + event_time=event_time, + event_type=issue_event_type, + message=issue.message, + ) + self._operator_event_exporter.export_operator_event(operator_event) + if issue.issue_type == IssueType.HANGING: operator.metrics._issue_detector_hanging += 1 if issue.issue_type == IssueType.HIGH_MEMORY: diff --git a/python/ray/data/_internal/operator_event_exporter.py b/python/ray/data/_internal/operator_event_exporter.py new file mode 100644 index 000000000000..5ee60f2131b9 --- /dev/null +++ b/python/ray/data/_internal/operator_event_exporter.py @@ -0,0 +1,164 @@ +"""Exporter API for Ray Data operator events.""" + +import logging +import os +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, Optional + +import ray +from ray._private.event.export_event_logger import ( + EventLogType, + check_export_api_enabled, + get_export_event_logger, +) + +logger = logging.getLogger(__name__) + + +@dataclass +class OperatorEvent: + """Represents an Ray Data operator event, such as issue detection + + Attributes: + dataset_id: The id of the dataset. + operator_id: The id of the operator within the DAG structure, typically + incorporating a position or index (e.g., "ReadParquet_0") + operator_name: The name of the operator. + event_time: The timestamp when the event is emitted (in seconds since epoch). + event_type: The type of the event. + message: The content of the event message. + """ + + dataset_id: str + operator_id: str + operator_name: str + event_time: float + event_type: str + message: str + + +def operator_event_to_proto(operator_event: OperatorEvent) -> Any: + """Convert the operator event to a protobuf message. + + Args: + operator_event: OperatorEvent object containing the event details + + Returns: + The protobuf message representing the operator event. + """ + + from ray.core.generated.export_dataset_operator_event_pb2 import ( + ExportDatasetOperatorEventData as ProtoOperatorEventData, + ) + + # Create the protobuf message + proto_operator_event_data = ProtoOperatorEventData( + dataset_id=operator_event.dataset_id, + operator_id=operator_event.operator_id, + operator_name=operator_event.operator_name, + event_time=operator_event.event_time, + event_type=ProtoOperatorEventData.DatasetOperatorEventType.Value( + operator_event.event_type + ), + message=operator_event.message, + ) + + return proto_operator_event_data + + +def format_export_issue_event_name(issue_name: str) -> str: + return "ISSUE_DETECTION_" + issue_name.upper().replace(" ", "_") + + +def get_operator_event_exporter() -> "OperatorEventExporter": + """Get the operator event exporter instance. + + Returns: + The operator event exporter instance. + """ + return LoggerOperatorEventExporter.create_if_enabled() + + +class OperatorEventExporter(ABC): + """Abstract base class for operator event exporters. + + Implementations of this interface can export Ray Data operator event to various + destinations like log files, databases, or monitoring systems. + """ + + @abstractmethod + def export_operator_event(self, operator_event: OperatorEvent) -> None: + """Export operator event to the destination. + + Args: + operator_event: OperatorEvent object containing operator event details. + """ + pass + + @classmethod + @abstractmethod + def create_if_enabled(cls) -> Optional["OperatorEventExporter"]: + """Create an event exporter instance if the export functionality is enabled. + + Returns: + An event exporter instance if enabled, none otherwise. + """ + pass + + +class LoggerOperatorEventExporter(OperatorEventExporter): + """Operator event exporter implementation that uses the Ray export event logger. + + This exporter writes operator event to log files using Ray's export event system. + """ + + def __init__(self, logger: logging.Logger): + """Initialize with a configured export event logger. + + Args: + logger: The export event logger to use for writing events. + """ + self._export_logger = logger + + def export_operator_event(self, operator_event: OperatorEvent) -> None: + """Export operator event using the export event logger. + + Args: + operator_event: OperatorEvent object containing operator event details. + """ + operator_event_proto = operator_event_to_proto(operator_event) + self._export_logger.send_event(operator_event_proto) + + @classmethod + def create_if_enabled(cls) -> Optional["LoggerOperatorEventExporter"]: + """Create a logger-based exporter if the export API is enabled. + + Returns: + A LoggerOperatorEventExporter instance, none otherwise. + """ + from ray.core.generated.export_event_pb2 import ExportEvent + + is_operator_event_export_api_enabled = check_export_api_enabled( + ExportEvent.SourceType.EXPORT_DATASET_OPERATOR_EVENT + ) + if not is_operator_event_export_api_enabled: + # The export API is not enabled, so we shouldn't create an exporter + return None + + log_directory = os.path.join( + ray._private.worker._global_node.get_session_dir_path(), "logs" + ) + + try: + logger = get_export_event_logger( + EventLogType.DATASET_OPERATOR_EVENT, + log_directory, + ) + return LoggerOperatorEventExporter(logger) + except Exception: + logger.exception( + "Unable to initialize the export event logger, so no operator export " + "events will be written." + ) + return None diff --git a/python/ray/data/tests/test_issue_detection_manager.py b/python/ray/data/tests/test_issue_detection_manager.py index 3dc1712aaeff..fbcd392bb2a2 100644 --- a/python/ray/data/tests/test_issue_detection_manager.py +++ b/python/ray/data/tests/test_issue_detection_manager.py @@ -1,14 +1,19 @@ +import json +import os import sys from unittest.mock import MagicMock import pytest +import ray +from ray._private import ray_constants from ray.data._internal.execution.operators.input_data_buffer import ( InputDataBuffer, ) from ray.data._internal.execution.operators.task_pool_map_operator import ( MapOperator, ) +from ray.data._internal.execution.streaming_executor import StreamingExecutor from ray.data._internal.issue_detection.issue_detector import ( Issue, IssueType, @@ -16,10 +21,30 @@ from ray.data._internal.issue_detection.issue_detector_manager import ( IssueDetectorManager, ) +from ray.data._internal.operator_event_exporter import ( + format_export_issue_event_name, +) from ray.data.context import DataContext +def _get_exported_data(): + exported_file = os.path.join( + ray._private.worker._global_node.get_session_dir_path(), + "logs", + "export_events", + "event_EXPORT_DATASET_OPERATOR_EVENT.log", + ) + assert os.path.isfile(exported_file) + + with open(exported_file, "r") as f: + data = f.readlines() + + return [json.loads(line) for line in data] + + def test_report_issues(): + ray.init() + ray_constants.RAY_ENABLE_EXPORT_API_WRITE_CONFIG = "EXPORT_DATASET_OPERATOR_EVENT" ctx = DataContext.get_current() input_operator = InputDataBuffer(ctx, input_data=[]) map_operator = MapOperator.create( @@ -29,7 +54,8 @@ def test_report_issues(): ray_remote_args={}, ) topology = {input_operator: MagicMock(), map_operator: MagicMock()} - executor = MagicMock(_topology=topology) + executor = StreamingExecutor(ctx) + executor._topology = topology detector = IssueDetectorManager(executor) detector._report_issues( @@ -53,6 +79,23 @@ def test_report_issues(): assert map_operator.metrics.issue_detector_hanging == 0 assert map_operator.metrics.issue_detector_high_memory == 1 + data = _get_exported_data() + assert len(data) == 2 + assert data[0]["event_data"]["dataset_id"] == "dataset" + assert data[0]["event_data"]["operator_id"] == f"{input_operator.name}_0" + assert data[0]["event_data"]["operator_name"] == input_operator.name + assert data[0]["event_data"]["event_type"] == format_export_issue_event_name( + IssueType.HANGING + ) + assert data[0]["event_data"]["message"] == "Hanging detected" + assert data[1]["event_data"]["dataset_id"] == "dataset" + assert data[1]["event_data"]["operator_id"] == f"{map_operator.name}_1" + assert data[1]["event_data"]["operator_name"] == map_operator.name + assert data[1]["event_data"]["event_type"] == format_export_issue_event_name( + IssueType.HIGH_MEMORY + ) + assert data[1]["event_data"]["message"] == "High memory usage detected" + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/src/ray/protobuf/BUILD.bazel b/src/ray/protobuf/BUILD.bazel index 5448fa1d071f..804988e7714b 100644 --- a/src/ray/protobuf/BUILD.bazel +++ b/src/ray/protobuf/BUILD.bazel @@ -254,6 +254,7 @@ proto_library( deps = [ ":export_actor_event_proto", ":export_dataset_metadata_proto", + ":export_dataset_operator_event_proto", ":export_driver_job_event_proto", ":export_node_event_proto", ":export_submission_job_event_proto", @@ -347,6 +348,16 @@ cc_proto_library( deps = [":export_train_state_proto"], ) +proto_library( + name = "export_dataset_operator_event_proto", + srcs = ["export_dataset_operator_event.proto"], +) + +cc_proto_library( + name = "export_dataset_operator_event_cc_proto", + deps = [":export_dataset_operator_event_proto"], +) + proto_library( name = "export_dataset_metadata_proto", srcs = ["export_dataset_metadata.proto"], diff --git a/src/ray/protobuf/export_dataset_operator_event.proto b/src/ray/protobuf/export_dataset_operator_event.proto new file mode 100644 index 000000000000..0c82133346b2 --- /dev/null +++ b/src/ray/protobuf/export_dataset_operator_event.proto @@ -0,0 +1,46 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +option cc_enable_arenas = true; +package ray.rpc; + +// This message defines the event_data stored by the export API for +// EXPORT_DATASET_OPERATOR type events from Ray Data operators. +message ExportDatasetOperatorEventData { + enum DatasetOperatorEventType { + UNSPECIFIED = 0; + ISSUE_DETECTION_HANGING = 1; + ISSUE_DETECTION_HIGH_MEMORY = 2; + } + + // The dataset ID + string dataset_id = 1; + + // The operator ID + string operator_id = 2; + + // The operator name + string operator_name = 3; + + // The timestamp when event is emitted (in seconds since epoch) + double event_time = 4; + + // The type of the event + DatasetOperatorEventType event_type = 5; + + // The content of the event message + string message = 6; +} diff --git a/src/ray/protobuf/export_event.proto b/src/ray/protobuf/export_event.proto index 5c0c56fc4dc0..fdc9281915f0 100644 --- a/src/ray/protobuf/export_event.proto +++ b/src/ray/protobuf/export_event.proto @@ -23,6 +23,7 @@ import "src/ray/protobuf/export_driver_job_event.proto"; import "src/ray/protobuf/export_submission_job_event.proto"; import "src/ray/protobuf/export_train_state.proto"; +import "src/ray/protobuf/export_dataset_operator_event.proto"; import "src/ray/protobuf/export_dataset_metadata.proto"; // ExportEvent defines events stored by the export API. This @@ -37,6 +38,7 @@ message ExportEvent { EXPORT_TRAIN_RUN = 5; EXPORT_TRAIN_RUN_ATTEMPT = 6; EXPORT_DATASET_METADATA = 7; + EXPORT_DATASET_OPERATOR_EVENT = 8; } // event_id is the unique ID of this event @@ -56,5 +58,6 @@ message ExportEvent { ExportTrainRunEventData train_run_event_data = 9; ExportTrainRunAttemptEventData train_run_attempt_event_data = 10; ExportDatasetMetadata dataset_metadata = 11; + ExportDatasetOperatorEventData dataset_operator_event_data = 12; } } From dd7b78c60f90452c97b9ace2229a59ffb859b4f4 Mon Sep 17 00:00:00 2001 From: "Owen Lin (You-Cheng Lin)" <106612301+owenowenisme@users.noreply.github.com> Date: Thu, 11 Sep 2025 01:55:56 +0800 Subject: [PATCH 559/634] [Data] Support initial concurrency value (#56370) ## Why are these changes needed? Support setting the initial actor pool size(concurrency) and update methods that accept tuple of concurrency. ## Related issue number ## Checks Closes #54648 - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: You-Cheng Lin (Owen) --- python/ray/data/_internal/compute.py | 26 +++++++++++- .../operators/actor_pool_map_operator.py | 13 +++++- python/ray/data/_internal/util.py | 40 +++++++++++-------- python/ray/data/dataset.py | 24 ++++++++--- python/ray/data/grouped_data.py | 5 ++- .../tests/test_actor_pool_map_operator.py | 25 +++++++++++- python/ray/data/tests/test_map.py | 6 +-- 7 files changed, 108 insertions(+), 31 deletions(-) diff --git a/python/ray/data/_internal/compute.py b/python/ray/data/_internal/compute.py index 333662c35f9f..6188dab3f899 100644 --- a/python/ray/data/_internal/compute.py +++ b/python/ray/data/_internal/compute.py @@ -76,6 +76,7 @@ def __init__( size: Optional[int] = None, min_size: Optional[int] = None, max_size: Optional[int] = None, + initial_size: Optional[int] = None, max_tasks_in_flight_per_actor: Optional[int] = None, ): """Construct ActorPoolStrategy for a Dataset transform. @@ -85,6 +86,8 @@ def __init__( specify both `size` and `min_size` or `max_size`. min_size: The minimum size of the actor pool. max_size: The maximum size of the actor pool. + initial_size: The initial number of actors to start with. If not specified, + defaults to min_size. Must be between min_size and max_size. max_tasks_in_flight_per_actor: The maximum number of tasks to concurrently send to a single actor worker. Increasing this will increase opportunities for pipelining task dependency prefetching with @@ -94,12 +97,13 @@ def __init__( if size is not None: if size < 1: raise ValueError("size must be >= 1", size) - if max_size is not None or min_size is not None: + if max_size is not None or min_size is not None or initial_size is not None: raise ValueError( - "min_size and max_size cannot be set at the same time as `size`" + "min_size, max_size, and initial_size cannot be set at the same time as `size`" ) min_size = size max_size = size + initial_size = size if min_size is not None and min_size < 1: raise ValueError("min_size must be >= 1", min_size) if max_size is not None: @@ -115,8 +119,24 @@ def __init__( "max_tasks_in_flight_per_actor must be >= 1, got: ", max_tasks_in_flight_per_actor, ) + self.min_size = min_size or 1 self.max_size = max_size or float("inf") + + # Validate and set initial_size + if initial_size is not None: + if initial_size < 1: + raise ValueError("initial_size must be >= 1", initial_size) + if initial_size < self.min_size: + raise ValueError( + f"initial_size ({initial_size}) must be >= min_size ({self.min_size})" + ) + if self.max_size != float("inf") and initial_size > self.max_size: + raise ValueError( + f"initial_size ({initial_size}) must be <= max_size ({self.max_size})" + ) + + self.initial_size = initial_size or self.min_size self.max_tasks_in_flight_per_actor = max_tasks_in_flight_per_actor self.num_workers = 0 self.ready_to_total_workers_ratio = 0.8 @@ -125,6 +145,7 @@ def __eq__(self, other: Any) -> bool: return isinstance(other, ActorPoolStrategy) and ( self.min_size == other.min_size and self.max_size == other.max_size + and self.initial_size == other.initial_size and self.max_tasks_in_flight_per_actor == other.max_tasks_in_flight_per_actor ) @@ -133,6 +154,7 @@ def __repr__(self) -> str: return ( f"ActorPoolStrategy(min_size={self.min_size}, " f"max_size={self.max_size}, " + f"initial_size={self.initial_size}, " f"max_tasks_in_flight_per_actor={self.max_tasks_in_flight_per_actor})" f"num_workers={self.num_workers}, " f"ready_to_total_workers_ratio={self.ready_to_total_workers_ratio})" diff --git a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py index 3080f460bb6d..a526b982ef45 100644 --- a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py @@ -150,6 +150,7 @@ def __init__( per_actor_resource_usage, min_size=compute_strategy.min_size, max_size=compute_strategy.max_size, + initial_size=compute_strategy.initial_size, max_actor_concurrency=max_actor_concurrency, max_tasks_in_flight_per_actor=( # NOTE: Unless explicitly configured by the user, max tasks-in-flight config @@ -203,7 +204,7 @@ def start(self, options: ExecutionOptions): self._actor_cls = ray.remote(**self._ray_remote_args)(self._map_worker_cls) self._actor_pool.scale( ActorPoolScalingRequest( - delta=self._actor_pool.min_size(), reason="scaling to min size" + delta=self._actor_pool.initial_size(), reason="scaling to initial size" ) ) @@ -729,6 +730,7 @@ def __init__( *, min_size: int, max_size: int, + initial_size: int, max_actor_concurrency: int, max_tasks_in_flight_per_actor: int, _enable_actor_pool_on_exit_hook: bool = False, @@ -744,8 +746,9 @@ def __init__( in the pool. Note, that this constraint could be violated when no new work is available for scheduling in the actor pool (ie when operator completes execution). - max_size: The minimum number of running actors to be maintained + max_size: The maximum number of running actors to be maintained in the pool. + initial_size: The initial number of actors to start with. max_actor_concurrency: The maximum number of concurrent tasks a single actor can execute (derived from `ray_remote_args` passed to the operator). @@ -757,6 +760,7 @@ def __init__( self._min_size: int = min_size self._max_size: int = max_size + self._initial_size: int = initial_size self._max_actor_concurrency: int = max_actor_concurrency self._max_tasks_in_flight: int = max_tasks_in_flight_per_actor self._create_actor_fn = create_actor_fn @@ -764,6 +768,8 @@ def __init__( assert self._min_size >= 1 assert self._max_size >= self._min_size + assert self._initial_size <= self._max_size + assert self._initial_size >= self._min_size assert self._max_tasks_in_flight >= 1 assert self._create_actor_fn is not None @@ -820,6 +826,9 @@ def max_actor_concurrency(self) -> int: def num_tasks_in_flight(self) -> int: return self._total_num_tasks_in_flight + def initial_size(self) -> int: + return self._initial_size + def _can_apply(self, config: ActorPoolScalingRequest) -> bool: """Returns whether Actor Pool is able to execute scaling request""" diff --git a/python/ray/data/_internal/util.py b/python/ray/data/_internal/util.py index d5220441f9f7..3db54be3d547 100644 --- a/python/ray/data/_internal/util.py +++ b/python/ray/data/_internal/util.py @@ -571,7 +571,7 @@ def get_compute_strategy( fn: "UserDefinedFunction", fn_constructor_args: Optional[Iterable[Any]] = None, compute: Optional[Union[str, "ComputeStrategy"]] = None, - concurrency: Optional[Union[int, Tuple[int, int]]] = None, + concurrency: Optional[Union[int, Tuple[int, int], Tuple[int, int, int]]] = None, ) -> "ComputeStrategy": """Get `ComputeStrategy` based on the function or class, and concurrency information. @@ -630,26 +630,34 @@ def get_compute_strategy( return compute elif concurrency is not None: if isinstance(concurrency, tuple): - if ( - len(concurrency) == 2 - and isinstance(concurrency[0], int) - and isinstance(concurrency[1], int) + # Validate tuple length and that all elements are integers + if len(concurrency) not in (2, 3) or not all( + isinstance(c, int) for c in concurrency ): - if is_callable_class: - return ActorPoolStrategy( - min_size=concurrency[0], max_size=concurrency[1] - ) - else: - raise ValueError( - "``concurrency`` is set as a tuple of integers, but ``fn`` " - f"is not a callable class: {fn}. Use ``concurrency=n`` to " - "control maximum number of workers to use." - ) - else: raise ValueError( "``concurrency`` is expected to be set as a tuple of " f"integers, but got: {concurrency}." ) + + # Check if function is callable class (common validation) + if not is_callable_class: + raise ValueError( + "``concurrency`` is set as a tuple of integers, but ``fn`` " + f"is not a callable class: {fn}. Use ``concurrency=n`` to " + "control maximum number of workers to use." + ) + + # Create ActorPoolStrategy based on tuple length + if len(concurrency) == 2: + return ActorPoolStrategy( + min_size=concurrency[0], max_size=concurrency[1] + ) + else: # len(concurrency) == 3 + return ActorPoolStrategy( + min_size=concurrency[0], + max_size=concurrency[1], + initial_size=concurrency[2], + ) elif isinstance(concurrency, int): if is_callable_class: return ActorPoolStrategy(size=concurrency) diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 9d655547e66b..53fcc3b0ef73 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -284,7 +284,7 @@ def map( num_cpus: Optional[float] = None, num_gpus: Optional[float] = None, memory: Optional[float] = None, - concurrency: Optional[Union[int, Tuple[int, int]]] = None, + concurrency: Optional[Union[int, Tuple[int, int], Tuple[int, int, int]]] = None, ray_remote_args_fn: Optional[Callable[[], Dict[str, Any]]] = None, **ray_remote_args, ) -> "Dataset": @@ -371,6 +371,9 @@ def parse_filename(row: Dict[str, Any]) -> Dict[str, Any]: * If ``fn`` is a class and ``concurrency`` is a tuple ``(m, n)``, Ray Data uses an autoscaling actor pool from ``m`` to ``n`` workers. + * If ``fn`` is a class and ``concurrency`` is a tuple ``(m, n, initial)``, Ray + Data uses an autoscaling actor pool from ``m`` to ``n`` workers, with an initial size of ``initial``. + * If ``fn`` is a class and ``concurrency`` isn't set (default), this method raises an error. @@ -467,7 +470,7 @@ def map_batches( num_cpus: Optional[float] = None, num_gpus: Optional[float] = None, memory: Optional[float] = None, - concurrency: Optional[Union[int, Tuple[int, int]]] = None, + concurrency: Optional[Union[int, Tuple[int, int], Tuple[int, int, int]]] = None, ray_remote_args_fn: Optional[Callable[[], Dict[str, Any]]] = None, **ray_remote_args, ) -> "Dataset": @@ -632,6 +635,9 @@ def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: * If ``fn`` is a class and ``concurrency`` is a tuple ``(m, n)``, Ray Data uses an autoscaling actor pool from ``m`` to ``n`` workers. + * If ``fn`` is a class and ``concurrency`` is a tuple ``(m, n, initial)``, Ray + Data uses an autoscaling actor pool from ``m`` to ``n`` workers, with an initial size of ``initial``. + * If ``fn`` is a class and ``concurrency`` isn't set (default), this method raises an error. @@ -722,7 +728,7 @@ def _map_batches_without_batch_size_validation( num_cpus: Optional[float], num_gpus: Optional[float], memory: Optional[float], - concurrency: Optional[Union[int, Tuple[int, int]]], + concurrency: Optional[Union[int, Tuple[int, int], Tuple[int, int, int]]], ray_remote_args_fn: Optional[Callable[[], Dict[str, Any]]], **ray_remote_args, ): @@ -1117,7 +1123,7 @@ def rename_columns( self, names: Union[List[str], Dict[str, str]], *, - concurrency: Optional[Union[int, Tuple[int, int]]] = None, + concurrency: Optional[Union[int, Tuple[int, int], Tuple[int, int, int]]] = None, **ray_remote_args, ): """Rename columns in the dataset. @@ -1251,7 +1257,7 @@ def flat_map( num_cpus: Optional[float] = None, num_gpus: Optional[float] = None, memory: Optional[float] = None, - concurrency: Optional[Union[int, Tuple[int, int]]] = None, + concurrency: Optional[Union[int, Tuple[int, int], Tuple[int, int, int]]] = None, ray_remote_args_fn: Optional[Callable[[], Dict[str, Any]]] = None, **ray_remote_args, ) -> "Dataset": @@ -1332,6 +1338,9 @@ def duplicate_row(row: Dict[str, Any]) -> List[Dict[str, Any]]: * If ``fn`` is a class and ``concurrency`` is a tuple ``(m, n)``, Ray Data uses an autoscaling actor pool from ``m`` to ``n`` workers. + * If ``fn`` is a class and ``concurrency`` is a tuple ``(m, n, initial)``, Ray + Data uses an autoscaling actor pool from ``m`` to ``n`` workers, with an initial size of ``initial``. + * If ``fn`` is a class and ``concurrency`` isn't set (default), this method raises an error. @@ -1394,7 +1403,7 @@ def filter( fn_kwargs: Optional[Dict[str, Any]] = None, fn_constructor_args: Optional[Iterable[Any]] = None, fn_constructor_kwargs: Optional[Dict[str, Any]] = None, - concurrency: Optional[Union[int, Tuple[int, int]]] = None, + concurrency: Optional[Union[int, Tuple[int, int], Tuple[int, int, int]]] = None, ray_remote_args_fn: Optional[Callable[[], Dict[str, Any]]] = None, **ray_remote_args, ) -> "Dataset": @@ -1450,6 +1459,9 @@ def filter( * If ``fn`` is a class and ``concurrency`` is a tuple ``(m, n)``, Ray Data uses an autoscaling actor pool from ``m`` to ``n`` workers. + * If ``fn`` is a class and ``concurrency`` is a tuple ``(m, n, initial)``, Ray + Data uses an autoscaling actor pool from ``m`` to ``n`` workers, with an initial size of ``initial``. + * If ``fn`` is a class and ``concurrency`` isn't set (default), this method raises an error. diff --git a/python/ray/data/grouped_data.py b/python/ray/data/grouped_data.py index 0771c6bac3f6..0d7fb0ed5e2b 100644 --- a/python/ray/data/grouped_data.py +++ b/python/ray/data/grouped_data.py @@ -108,7 +108,7 @@ def map_groups( num_cpus: Optional[float] = None, num_gpus: Optional[float] = None, memory: Optional[float] = None, - concurrency: Optional[Union[int, Tuple[int, int]]] = None, + concurrency: Optional[Union[int, Tuple[int, int], Tuple[int, int, int]]] = None, ray_remote_args_fn: Optional[Callable[[], Dict[str, Any]]] = None, **ray_remote_args, ) -> "Dataset": @@ -201,6 +201,9 @@ def map_groups( * If ``fn`` is a class and ``concurrency`` is a tuple ``(m, n)``, Ray Data uses an autoscaling actor pool from ``m`` to ``n`` workers. + * If ``fn`` is a class and ``concurrency`` is a tuple ``(m, n, initial)``, Ray + Data uses an autoscaling actor pool from ``m`` to ``n`` workers, with an initial size of ``initial``. + * If ``fn`` is a class and ``concurrency`` isn't set (default), this method raises an error. diff --git a/python/ray/data/tests/test_actor_pool_map_operator.py b/python/ray/data/tests/test_actor_pool_map_operator.py index f12f8da8df40..66dcad11f0db 100644 --- a/python/ray/data/tests/test_actor_pool_map_operator.py +++ b/python/ray/data/tests/test_actor_pool_map_operator.py @@ -16,7 +16,7 @@ from ray.actor import ActorHandle from ray.data._internal.actor_autoscaler import ActorPoolScalingRequest from ray.data._internal.execution.bundle_queue import FIFOBundleQueue -from ray.data._internal.execution.interfaces import ExecutionResources +from ray.data._internal.execution.interfaces import ExecutionOptions, ExecutionResources from ray.data._internal.execution.interfaces.physical_operator import _ActorPoolInfo from ray.data._internal.execution.interfaces.ref_bundle import RefBundle from ray.data._internal.execution.operators.actor_pool_map_operator import ( @@ -92,11 +92,13 @@ def _create_actor_pool( self, min_size=1, max_size=4, + initial_size=1, max_tasks_in_flight=4, ): pool = _ActorPool( min_size=min_size, max_size=max_size, + initial_size=initial_size, max_actor_concurrency=1, max_tasks_in_flight_per_actor=max_tasks_in_flight, create_actor_fn=self._create_actor_fn, @@ -591,6 +593,27 @@ def test_locality_based_actor_ranking_no_locations(self): assert res5 is None +def test_setting_initial_size_for_actor_pool(): + data_context = ray.data.DataContext.get_current() + op = ActorPoolMapOperator( + map_transformer=MagicMock(), + input_op=InputDataBuffer(data_context, input_data=MagicMock()), + data_context=data_context, + target_max_block_size=None, + compute_strategy=ray.data.ActorPoolStrategy( + min_size=1, max_size=4, initial_size=2 + ), + ray_remote_args={"num_cpus": 1}, + ) + + op.start(ExecutionOptions()) + + assert op._actor_pool.get_actor_info() == _ActorPoolInfo( + running=0, pending=2, restarting=0 + ) + ray.shutdown() + + def test_min_max_resource_requirements(restore_data_context): data_context = ray.data.DataContext.get_current() op = ActorPoolMapOperator( diff --git a/python/ray/data/tests/test_map.py b/python/ray/data/tests/test_map.py index 4bf7550d7a1d..439d8d991170 100644 --- a/python/ray/data/tests/test_map.py +++ b/python/ray/data/tests/test_map.py @@ -301,8 +301,8 @@ def __call__(self, x): # Test function and class. for fn in [udf, UDFClass]: # Test concurrency with None, single integer and a tuple of integers. - for concurrency in [2, (2, 4)]: - if fn == udf and concurrency == (2, 4): + for concurrency in [2, (2, 4), (2, 6, 4)]: + if fn == udf and (concurrency == (2, 4) or concurrency == (2, 6, 4)): error_message = "``concurrency`` is set as a tuple of integers" with pytest.raises(ValueError, match=error_message): ds.map(fn, concurrency=concurrency).take_all() @@ -312,7 +312,7 @@ def __call__(self, x): # Test concurrency with an illegal value. error_message = "``concurrency`` is expected to be set a" - for concurrency in ["dummy", (1, 3, 5)]: + for concurrency in ["dummy", (1, 3, 5, 7)]: with pytest.raises(ValueError, match=error_message): ds.map(UDFClass, concurrency=concurrency).take_all() From f4dc12e4aa38face0c834579bd3cd41e8c5fe094 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Wed, 10 Sep 2025 11:55:49 -0700 Subject: [PATCH 560/634] [ci] raydepsets: adding pre hooks for depsets (#56180) adding pre hooks for depsets - pre hooks can be defined on each depset - executed - replacing compile_llm_requirements with a pre hook (remove_compiled_headers.sh) - Adding unit tests for pre_hooks in test_cli and test_workspace --------- Signed-off-by: elliot-barn Signed-off-by: Elliot Barnwell Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> --- ci/raydepsets/BUILD.bazel | 2 ++ ci/raydepsets/cli.py | 34 +++++++++++++++---- .../pre_hooks/remove-compiled-headers.sh | 17 ++++++++++ ci/raydepsets/rayllm.depsets.yaml | 2 ++ ci/raydepsets/tests/test_cli.py | 26 ++++++++++++-- .../tests/test_data/pre-hook-error-test.sh | 7 ++++ .../tests/test_data/pre-hook-test.sh | 5 +++ .../tests/test_data/test.depsets.yaml | 7 ++++ ci/raydepsets/tests/test_workspace.py | 11 +++++- ci/raydepsets/tests/utils.py | 6 ++++ ci/raydepsets/workspace.py | 2 ++ ci/test_compile_llm_requirements.sh | 2 +- 12 files changed, 111 insertions(+), 10 deletions(-) create mode 100755 ci/raydepsets/pre_hooks/remove-compiled-headers.sh create mode 100755 ci/raydepsets/tests/test_data/pre-hook-error-test.sh create mode 100755 ci/raydepsets/tests/test_data/pre-hook-test.sh diff --git a/ci/raydepsets/BUILD.bazel b/ci/raydepsets/BUILD.bazel index b253519ec9dd..9802526e950a 100644 --- a/ci/raydepsets/BUILD.bazel +++ b/ci/raydepsets/BUILD.bazel @@ -35,6 +35,8 @@ py_test( name = "test_cli", srcs = ["tests/test_cli.py"], data = [ + "tests/test_data/pre-hook-error-test.sh", + "tests/test_data/pre-hook-test.sh", "tests/test_data/requirement_constraints_test.txt", "tests/test_data/requirements_compiled_test.txt", "tests/test_data/requirements_compiled_test_expand.txt", diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index e614d5a3a179..0f9aa59e66e2 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -149,21 +149,31 @@ def _build(self): for depset in self.config.depsets: if depset.operation == "compile": self.build_graph.add_node( - depset.name, operation="compile", depset=depset + depset.name, operation="compile", depset=depset, node_type="depset" ) elif depset.operation == "subset": self.build_graph.add_node( - depset.name, operation="subset", depset=depset + depset.name, operation="subset", depset=depset, node_type="depset" ) self.build_graph.add_edge(depset.source_depset, depset.name) elif depset.operation == "expand": self.build_graph.add_node( - depset.name, operation="expand", depset=depset + depset.name, operation="expand", depset=depset, node_type="depset" ) for depset_name in depset.depsets: self.build_graph.add_edge(depset_name, depset.name) else: raise ValueError(f"Invalid operation: {depset.operation}") + if depset.pre_hooks: + for ind, hook in enumerate(depset.pre_hooks): + hook_name = f"{depset.name}_pre_hook_{ind+1}" + self.build_graph.add_node( + hook_name, + operation="pre_hook", + pre_hook=hook, + node_type="pre_hook", + ) + self.build_graph.add_edge(hook_name, depset.name) def subgraph_dependency_nodes(self, depset_name: str): dependency_nodes = networkx_ancestors(self.build_graph, depset_name) @@ -177,8 +187,13 @@ def execute(self, single_depset_name: Optional[str] = None): self.subgraph_dependency_nodes(single_depset_name) for node in topological_sort(self.build_graph): - depset = self.build_graph.nodes[node]["depset"] - self.execute_single(depset) + node_type = self.build_graph.nodes[node]["node_type"] + if node_type == "pre_hook": + pre_hook = self.build_graph.nodes[node]["pre_hook"] + self.execute_pre_hook(pre_hook) + elif node_type == "depset": + depset = self.build_graph.nodes[node]["depset"] + self.execute_depset(depset) def exec_uv_cmd( self, cmd: str, args: List[str], stdin: Optional[bytes] = None @@ -190,7 +205,14 @@ def exec_uv_cmd( raise RuntimeError(f"Failed to execute command: {cmd}") return status.stdout - def execute_single(self, depset: Depset): + def execute_pre_hook(self, pre_hook: str): + status_code = subprocess.call(pre_hook, cwd=self.workspace.dir) + if status_code != 0: + raise RuntimeError(f"Failed to execute pre-hook: {pre_hook}") + click.echo(f"Executed pre-hook: {pre_hook}") + return status_code + + def execute_depset(self, depset: Depset): if depset.operation == "compile": self.compile( constraints=depset.constraints, diff --git a/ci/raydepsets/pre_hooks/remove-compiled-headers.sh b/ci/raydepsets/pre_hooks/remove-compiled-headers.sh new file mode 100755 index 000000000000..37f32d3808f9 --- /dev/null +++ b/ci/raydepsets/pre_hooks/remove-compiled-headers.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +PYTHON_CODE="$(python -c "import sys; v=sys.version_info; print(f'py{v.major}{v.minor}')")" +if [[ "${PYTHON_CODE}" != "py311" ]]; then + echo "--- Python version is not 3.11" + echo "--- Current Python version: ${PYTHON_CODE}" + exit 1 +fi + +mkdir -p /tmp/ray-deps + +# Remove the GPU constraints +cp python/requirements_compiled.txt /tmp/ray-deps/requirements_compiled.txt +sed -e '/^--extra-index-url /d' -e '/^--find-links /d' /tmp/ray-deps/requirements_compiled.txt > /tmp/ray-deps/requirements_compiled.txt.tmp +mv /tmp/ray-deps/requirements_compiled.txt.tmp /tmp/ray-deps/requirements_compiled.txt diff --git a/ci/raydepsets/rayllm.depsets.yaml b/ci/raydepsets/rayllm.depsets.yaml index d96df6168dc7..e34be3bd104c 100644 --- a/ci/raydepsets/rayllm.depsets.yaml +++ b/ci/raydepsets/rayllm.depsets.yaml @@ -34,6 +34,8 @@ depsets: - /tmp/ray-deps/requirements_compiled.txt output: python/deplocks/llm/ray_test_${PYTHON_VERSION}_${CUDA_CODE}.lock operation: compile + pre_hooks: + - ci/raydepsets/pre_hooks/remove-compiled-headers.sh # Second, expand it into LLM test dependencies. - name: compiled_ray_llm_test_depset_${PYTHON_VERSION}_${CUDA_CODE} diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index a05932336f9c..3dd24d9ee87a 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -387,8 +387,8 @@ def test_build_graph(self): copy_data_to_tmpdir(tmpdir) manager = _create_test_manager(tmpdir) assert manager.build_graph is not None - assert len(manager.build_graph.nodes()) == 6 - assert len(manager.build_graph.edges()) == 3 + assert len(manager.build_graph.nodes()) == 8 + assert len(manager.build_graph.edges()) == 4 # assert that the compile depsets are first assert ( manager.build_graph.nodes["general_depset__py311_cpu"]["operation"] @@ -581,6 +581,28 @@ def test_get_depset_with_build_arg_set_and_no_build_arg_set_provided(self): with self.assertRaises(KeyError): _get_depset(manager.config.depsets, "build_args_test_depset_py311") + def test_execute_single_pre_hook(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = _create_test_manager(tmpdir) + manager.execute_pre_hook("pre-hook-test.sh") + + def test_execute_single_invalid_pre_hook(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = _create_test_manager(tmpdir) + with self.assertRaises(RuntimeError): + manager.execute_pre_hook("pre-hook-error-test.sh") + + def test_execute_pre_hooks_failure_in_middle(self): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + manager = _create_test_manager(tmpdir) + with self.assertRaises(RuntimeError): + manager.execute_pre_hook("pre-hook-test.sh") + manager.execute_pre_hook("pre-hook-error-test.sh") + manager.execute_pre_hook("pre-hook-test.sh") + def test_copy_lock_files_to_temp_dir(self): with tempfile.TemporaryDirectory() as tmpdir: copy_data_to_tmpdir(tmpdir) diff --git a/ci/raydepsets/tests/test_data/pre-hook-error-test.sh b/ci/raydepsets/tests/test_data/pre-hook-error-test.sh new file mode 100755 index 000000000000..4196354f3deb --- /dev/null +++ b/ci/raydepsets/tests/test_data/pre-hook-error-test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail + +echo "Pre-hook test error" + +exit 1 diff --git a/ci/raydepsets/tests/test_data/pre-hook-test.sh b/ci/raydepsets/tests/test_data/pre-hook-test.sh new file mode 100755 index 000000000000..bd86a37a2e34 --- /dev/null +++ b/ci/raydepsets/tests/test_data/pre-hook-test.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +echo "Pre-hook test" diff --git a/ci/raydepsets/tests/test_data/test.depsets.yaml b/ci/raydepsets/tests/test_data/test.depsets.yaml index 2508025e2524..97950a046a68 100644 --- a/ci/raydepsets/tests/test_data/test.depsets.yaml +++ b/ci/raydepsets/tests/test_data/test.depsets.yaml @@ -51,3 +51,10 @@ depsets: output: requirements_compiled_expand_general.txt build_arg_sets: - py311_cpu + - name: pre_hook_test_depset + operation: compile + requirements: + - requirements_test.txt + output: requirements_compiled_pre_hook.txt + pre_hooks: + - pre-hook-test.sh diff --git a/ci/raydepsets/tests/test_workspace.py b/ci/raydepsets/tests/test_workspace.py index 50ca28aedf65..a79ce82d4b60 100644 --- a/ci/raydepsets/tests/test_workspace.py +++ b/ci/raydepsets/tests/test_workspace.py @@ -4,7 +4,7 @@ import pytest -from ci.raydepsets.tests.utils import copy_data_to_tmpdir +from ci.raydepsets.tests.utils import copy_data_to_tmpdir, get_depset_by_name from ci.raydepsets.workspace import BuildArgSet, Workspace, _substitute_build_args @@ -68,5 +68,14 @@ def test_invalid_build_arg_set(): workspace.load_config(path=Path(tmpdir) / "test.depsets.yaml") +def test_parse_pre_hooks(): + with tempfile.TemporaryDirectory() as tmpdir: + copy_data_to_tmpdir(tmpdir) + workspace = Workspace(dir=tmpdir) + config = workspace.load_config(path=Path(tmpdir) / "test.depsets.yaml") + pre_hook_depset = get_depset_by_name(config.depsets, "pre_hook_test_depset") + assert pre_hook_depset.pre_hooks == ["pre-hook-test.sh"] + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/ci/raydepsets/tests/utils.py b/ci/raydepsets/tests/utils.py index 6e3ca310b7d7..5f531cf7c3c0 100644 --- a/ci/raydepsets/tests/utils.py +++ b/ci/raydepsets/tests/utils.py @@ -43,3 +43,9 @@ def save_file_as(input_file, output_file): def append_to_file(filepath, new): with open(filepath, "a") as f: f.write(new + "\n") + + +def get_depset_by_name(depsets, name): + for depset in depsets: + if depset.name == name: + return depset diff --git a/ci/raydepsets/workspace.py b/ci/raydepsets/workspace.py index 46d9f74223f0..cbebc79890f8 100644 --- a/ci/raydepsets/workspace.py +++ b/ci/raydepsets/workspace.py @@ -23,6 +23,7 @@ class Depset: packages: Optional[List[str]] = None source_depset: Optional[str] = None depsets: Optional[List[str]] = None + pre_hooks: Optional[List[str]] = None def _substitute_build_args(obj: Any, build_arg_set: BuildArgSet): @@ -50,6 +51,7 @@ def _dict_to_depset(depset: dict) -> Depset: depsets=depset.get("depsets", []), override_flags=depset.get("override_flags", []), append_flags=depset.get("append_flags", []), + pre_hooks=depset.get("pre_hooks", []), packages=depset.get("packages", []), ) diff --git a/ci/test_compile_llm_requirements.sh b/ci/test_compile_llm_requirements.sh index 0843b7bcaac4..ee2ae90126f4 100755 --- a/ci/test_compile_llm_requirements.sh +++ b/ci/test_compile_llm_requirements.sh @@ -22,7 +22,7 @@ for LOCK_TYPE in "${LOCK_TYPES[@]}"; do done done -./ci/compile_llm_requirements.sh ci/raydepsets/rayllm.depsets.yaml +bazel run //ci/raydepsets:raydepsets -- build ci/raydepsets/rayllm.depsets.yaml # Copy files to artifact mount on Buildkite for LOCK_TYPE in "${LOCK_TYPES[@]}"; do From 48a6e7fafe48ca98f7670e4587b756a7cece736b Mon Sep 17 00:00:00 2001 From: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Date: Wed, 10 Sep 2025 12:36:17 -0700 Subject: [PATCH 561/634] [Data] Fix resource reservation by excluding completed operators' usages (#56319) ## Problem The `ReservationOpResourceAllocator` was incorrectly accounting for resource usage when calculating available resources for reservation. Specifically, it wasn't properly handling completed operators who have blocks in the output queue. The `ReadFiles` operator below consumes 50 GB of object store memory and should be excluded from reservation, but it is currently not. image ## Solution Added logic to identify and subtract resource usage specifically from completed physical operators: ## Testing results Before the fix image After the fix image ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: xgui Signed-off-by: Xinyuan <43737116+xinyuangui2@users.noreply.github.com> Co-authored-by: Alexey Kudinkin --- .../_internal/execution/resource_manager.py | 42 ++- .../ray/data/tests/test_resource_manager.py | 353 ++++++++++++++++++ 2 files changed, 393 insertions(+), 2 deletions(-) diff --git a/python/ray/data/_internal/execution/resource_manager.py b/python/ray/data/_internal/execution/resource_manager.py index f18e3a3f9771..4b2bfe4af666 100644 --- a/python/ray/data/_internal/execution/resource_manager.py +++ b/python/ray/data/_internal/execution/resource_manager.py @@ -493,18 +493,56 @@ def _get_eligible_ops(self) -> List[PhysicalOperator]: op for op in self._resource_manager._topology if self._is_op_eligible(op) ] + def _get_ineligible_ops_with_usage(self) -> List[PhysicalOperator]: + """ + Resource reservation is based on the number of eligible operators. + However, there might be completed operators that still have blocks in their output queue, which we need to exclude them from the reservation. + And we also need to exclude the downstream ineligible operators. + + E.g., for the following pipeline: + ``` + map1 (completed, but still has blocks in its output queue) -> limit1 (ineligible, not completed) -> map2 (not completed) -> limit2 -> map3 + ``` + + The reservation is based on the number of eligible operators (map2 and map3), but we need to exclude map1 and limit1 from the reservation. + """ + last_completed_ops = [] + ops_to_exclude_from_reservation = [] + # Traverse operator tree collecting all operators that have already finished + for op in self._resource_manager._topology: + if not op.execution_finished(): + for dep in op.input_dependencies: + if dep.execution_finished(): + last_completed_ops.append(dep) + + # In addition to completed operators, + # filter out downstream ineligible operators since they are omitted from reservation calculations. + for op in last_completed_ops: + ops_to_exclude_from_reservation.extend( + list(self._get_downstream_ineligible_ops(op)) + ) + ops_to_exclude_from_reservation.append(op) + return list(set(ops_to_exclude_from_reservation)) + def _update_reservation(self): - global_limits = self._resource_manager.get_global_limits() + global_limits = self._resource_manager.get_global_limits().copy() eligible_ops = self._get_eligible_ops() self._op_reserved.clear() self._reserved_for_op_outputs.clear() self._reserved_min_resources.clear() - remaining = global_limits.copy() if len(eligible_ops) == 0: return + op_to_exclude_from_reservation = self._get_ineligible_ops_with_usage() + for completed_op in op_to_exclude_from_reservation: + global_limits = global_limits.subtract( + self._resource_manager.get_op_usage(completed_op) + ) + global_limits = global_limits.max(ExecutionResources.zero()) + remaining = global_limits.copy() + # Reserve `reservation_ratio * global_limits / num_ops` resources for each # operator. default_reserved = global_limits.scale( diff --git a/python/ray/data/tests/test_resource_manager.py b/python/ray/data/tests/test_resource_manager.py index 90b69361e51b..fe03b8d1d3be 100644 --- a/python/ray/data/tests/test_resource_manager.py +++ b/python/ray/data/tests/test_resource_manager.py @@ -10,8 +10,10 @@ ExecutionResources, ) from ray.data._internal.execution.operators.input_data_buffer import InputDataBuffer +from ray.data._internal.execution.operators.join import JoinOperator from ray.data._internal.execution.operators.limit_operator import LimitOperator from ray.data._internal.execution.operators.map_operator import MapOperator +from ray.data._internal.execution.operators.union_operator import UnionOperator from ray.data._internal.execution.resource_manager import ( ReservationOpResourceAllocator, ResourceManager, @@ -45,6 +47,45 @@ def mock_map_op( return op +def mock_union_op( + input_ops, + incremental_resource_usage=None, +): + op = UnionOperator( + DataContext.get_current(), + *input_ops, + ) + op.start = MagicMock(side_effect=lambda _: None) + if incremental_resource_usage is not None: + op.incremental_resource_usage = MagicMock( + return_value=incremental_resource_usage + ) + return op + + +def mock_join_op( + left_input_op, + right_input_op, + incremental_resource_usage=None, +): + op = JoinOperator( + DataContext.get_current(), + left_input_op, + right_input_op, + ("id",), + ("id",), + "inner", + num_partitions=1, + ) + + op.start = MagicMock(side_effect=lambda _: None) + if incremental_resource_usage is not None: + op.incremental_resource_usage = MagicMock( + return_value=incremental_resource_usage + ) + return op + + class TestResourceManager: """Unit tests for ResourceManager.""" @@ -740,6 +781,318 @@ def test_gpu_usage_exceeds_global_limits(self, restore_data_context): assert allocator._op_budgets[o2].gpu == 0 + def test_get_ineligible_ops_with_usage(self, restore_data_context): + DataContext.get_current().op_resource_reservation_enabled = True + + o1 = InputDataBuffer(DataContext.get_current(), []) + o2 = mock_map_op( + o1, + ) + o3 = LimitOperator(1, o2, DataContext.get_current()) + o4 = mock_map_op( + o3, + ) + o5 = mock_map_op( + o4, + ) + o1.mark_execution_finished() + o2.mark_execution_finished() + + topo, _ = build_streaming_topology(o5, ExecutionOptions()) + + resource_manager = ResourceManager( + topo, ExecutionOptions(), MagicMock(), DataContext.get_current() + ) + + allocator = resource_manager._op_resource_allocator + + ops_to_exclude = allocator._get_ineligible_ops_with_usage() + assert len(ops_to_exclude) == 2 + assert set(ops_to_exclude) == {o2, o3} + + def test_get_ineligible_ops_with_usage_complex_graph(self, restore_data_context): + """ + o1 (InputDataBuffer) + | + v + o2 (MapOperator, completed) + | + v + o3 (LimitOperator) + | + v o4 (InputDataBuffer) + | | + | v + | o5 (MapOperator, completed) + | | + v v + o6 (UnionOperator) <-- + | + v + o8 (JoinOperator) <-- o7 (InputDataBuffer, completed) + """ + DataContext.get_current().op_resource_reservation_enabled = True + + o1 = InputDataBuffer(DataContext.get_current(), []) + o2 = mock_map_op( + o1, + ) + o3 = LimitOperator(1, o2, DataContext.get_current()) + o4 = InputDataBuffer(DataContext.get_current(), []) + o5 = mock_map_op( + o4, + ) + o6 = mock_union_op([o3, o5]) + o7 = InputDataBuffer(DataContext.get_current(), []) + o8 = mock_join_op(o7, o6) + + o1.mark_execution_finished() + o2.mark_execution_finished() + o4.mark_execution_finished() + o5.mark_execution_finished() + o7.mark_execution_finished() + + topo, _ = build_streaming_topology(o8, ExecutionOptions()) + + resource_manager = ResourceManager( + topo, ExecutionOptions(), MagicMock(), DataContext.get_current() + ) + + allocator = resource_manager._op_resource_allocator + + ops_to_exclude = allocator._get_ineligible_ops_with_usage() + assert len(ops_to_exclude) == 4 + assert set(ops_to_exclude) == {o2, o3, o5, o7} + + def test_reservation_accounts_for_completed_ops(self, restore_data_context): + """Test that resource reservation properly accounts for completed ops.""" + DataContext.get_current().op_resource_reservation_enabled = True + DataContext.get_current().op_resource_reservation_ratio = 0.5 + + o1 = InputDataBuffer(DataContext.get_current(), []) + o2 = mock_map_op(o1, incremental_resource_usage=ExecutionResources(1, 0, 10)) + o3 = mock_map_op(o2, incremental_resource_usage=ExecutionResources(1, 0, 10)) + o4 = mock_map_op(o3, incremental_resource_usage=ExecutionResources(1, 0, 10)) + o1.mark_execution_finished() + o2.mark_execution_finished() + + op_usages = { + o1: ExecutionResources.zero(), + o2: ExecutionResources(cpu=2, object_store_memory=50), + o3: ExecutionResources.zero(), + o4: ExecutionResources.zero(), + } + op_internal_usage = dict.fromkeys([o1, o2, o3, o4], 0) + op_outputs_usages = dict.fromkeys([o1, o2, o3, o4], 0) + + topo, _ = build_streaming_topology(o4, ExecutionOptions()) + + global_limits = ExecutionResources(cpu=10, object_store_memory=250) + + resource_manager = ResourceManager( + topo, ExecutionOptions(), MagicMock(), DataContext.get_current() + ) + resource_manager.get_op_usage = MagicMock(side_effect=lambda op: op_usages[op]) + resource_manager._mem_op_internal = op_internal_usage + resource_manager._mem_op_outputs = op_outputs_usages + resource_manager.get_global_limits = MagicMock(return_value=global_limits) + + allocator = resource_manager._op_resource_allocator + allocator.update_usages() + + # Check that o2's usage was subtracted from remaining resources + # global_limits (10 CPU, 250 mem) - o1 usage (0) - o2 usage (2 CPU, 50 mem) = remaining (8 CPU, 200 mem) + # With 2 eligible ops (o3, o4) and 50% reservation ratio: + # Each op gets reserved: (8 CPU, 200 mem) * 0.5 / 2 = (2 CPU, 50 mem) + + # Verify that reservations are calculated correctly + assert allocator._op_reserved[o3].cpu == 2.0 + assert allocator._op_reserved[o4].cpu == 2.0 + + # The total reserved memory should account for o2's usage being subtracted + total_reserved_memory = ( + allocator._op_reserved[o3].object_store_memory + + allocator._reserved_for_op_outputs[o3] + + allocator._op_reserved[o4].object_store_memory + + allocator._reserved_for_op_outputs[o4] + ) + + assert abs(total_reserved_memory - 100) < 1.0 + + def test_reservation_accounts_for_completed_ops_complex_graph( + self, restore_data_context + ): + """ + o1 (InputDataBuffer) + | + v + o2 (MapOperator, completed) + | + v + o3 (LimitOperator) + | + v o4 (InputDataBuffer) + | | + | v + | o5 (MapOperator, completed) + | | + v v + o6 (UnionOperator) <-- + | + v + o8 (JoinOperator) <-- o7 (InputDataBuffer, completed) + """ + DataContext.get_current().op_resource_reservation_enabled = True + DataContext.get_current().op_resource_reservation_ratio = 0.5 + + o1 = InputDataBuffer(DataContext.get_current(), []) + o2 = mock_map_op(o1, incremental_resource_usage=ExecutionResources(1, 0, 15)) + o3 = LimitOperator(1, o2, DataContext.get_current()) + o4 = InputDataBuffer(DataContext.get_current(), []) + o5 = mock_map_op(o4, incremental_resource_usage=ExecutionResources(1, 0, 10)) + o6 = mock_union_op( + [o3, o5], incremental_resource_usage=ExecutionResources(1, 0, 20) + ) + o7 = InputDataBuffer(DataContext.get_current(), []) + o8 = mock_join_op( + o7, o6, incremental_resource_usage=ExecutionResources(1, 0, 30) + ) + + o1.mark_execution_finished() + o2.mark_execution_finished() + o4.mark_execution_finished() + o5.mark_execution_finished() + o7.mark_execution_finished() + + op_usages = { + o1: ExecutionResources.zero(), + o2: ExecutionResources(cpu=2, object_store_memory=150), + o3: ExecutionResources(cpu=2, object_store_memory=50), + o4: ExecutionResources.zero(), + o5: ExecutionResources(cpu=3, object_store_memory=100), + o6: ExecutionResources.zero(), + o7: ExecutionResources(cpu=1, object_store_memory=100), + o8: ExecutionResources.zero(), + } + op_internal_usage = dict.fromkeys([o1, o2, o3, o4, o5, o6, o7, o8], 0) + op_outputs_usages = dict.fromkeys([o1, o2, o3, o4, o5, o6, o7, o8], 0) + + topo, _ = build_streaming_topology(o8, ExecutionOptions()) + + global_limits = ExecutionResources.zero() + + def mock_get_global_limits(): + nonlocal global_limits + return global_limits + + resource_manager = ResourceManager( + topo, ExecutionOptions(), MagicMock(), DataContext.get_current() + ) + resource_manager.get_op_usage = MagicMock(side_effect=lambda op: op_usages[op]) + resource_manager.get_global_limits = MagicMock( + side_effect=mock_get_global_limits + ) + resource_manager._mem_op_internal = op_internal_usage + resource_manager._mem_op_outputs = op_outputs_usages + + allocator = resource_manager._op_resource_allocator + global_limits = ExecutionResources(cpu=20, object_store_memory=2000) + allocator.update_usages() + """ + global_limits (20 CPU, 2000 mem) - o2 usage (2 CPU, 150 mem) - o3 usage (2 CPU, 50 mem) - o5 usage (3 CPU, 100 mem) - o7 usage (1 CPU, 100 mem) = remaining (12 CPU, 1600 mem) + +-----+------------------+------------------+--------------+ + | | _op_reserved | _reserved_for | used shared | + | | (used/remaining) | _op_outputs | resources | + | | | (used/remaining) | | + +-----+------------------+------------------+--------------+ + | op6 | 0/200 | 0/200 | 0 | + +-----+------------------+------------------+--------------+ + | op8 | 0/200 | 0/200 | 0 | + +-----+------------------+------------------+--------------+ + """ + assert set(allocator._op_budgets.keys()) == {o6, o8} + assert set(allocator._op_reserved.keys()) == {o6, o8} + assert allocator._op_reserved[o6] == ExecutionResources( + cpu=3, object_store_memory=200 + ) + assert allocator._op_reserved[o8] == ExecutionResources( + cpu=3, object_store_memory=200 + ) + assert allocator._reserved_for_op_outputs[o6] == 200 + assert allocator._reserved_for_op_outputs[o8] == 200 + assert allocator._total_shared == ExecutionResources( + cpu=6, object_store_memory=800 + ) + assert allocator._op_budgets[o6] == ExecutionResources( + cpu=6, object_store_memory=600 + ) + assert allocator._op_budgets[o8] == ExecutionResources( + cpu=6, object_store_memory=600 + ) + + # Test when resources are used. + op_usages[o6] = ExecutionResources(2, 0, 500) + op_internal_usage[o6] = 300 + op_outputs_usages[o6] = 200 + op_usages[o8] = ExecutionResources(2, 0, 100) + op_internal_usage[o8] = 50 + op_outputs_usages[o8] = 50 + """ + +-----+------------------+------------------+--------------+ + | | _op_reserved | _reserved_for | used shared | + | | (used/remaining) | _op_outputs | resources | + | | | (used/remaining) | | + +-----+------------------+------------------+--------------+ + | op6 | 200/0 | 200/0 | 100 | + +-----+------------------+------------------+--------------+ + | op8 | 50/150 | 50/150 | 0 | + +-----+------------------+------------------+--------------+ + """ + allocator.update_usages() + assert allocator._op_budgets[o6] == ExecutionResources( + cpu=4, object_store_memory=350 + ) + assert allocator._op_budgets[o8] == ExecutionResources( + cpu=4, object_store_memory=500 + ) + + # Test when completed ops update the usage. + op_usages[o5] = ExecutionResources.zero() + allocator.update_usages() + """ + global_limits (20 CPU, 2000 mem) - o2 usage (2 CPU, 150 mem) - o3 usage (2 CPU, 50 mem) - o5 usage (0 CPU, 0 mem) - o7 usage (1 CPU, 100 mem) = remaining (15 CPU, 1700 mem) + +-----+------------------+------------------+--------------+ + | | _op_reserved | _reserved_for | used shared | + | | (used/remaining) | _op_outputs | resources | + | | | (used/remaining) | | + +-----+------------------+------------------+--------------+ + | op6 | 213/0 | 200/13 | 300-213=87 | + +-----+------------------+------------------+--------------+ + | op8 | 50/163 | 50/163 | 0 | + +-----+------------------+------------------+--------------+ + """ + assert set(allocator._op_budgets.keys()) == {o6, o8} + assert set(allocator._op_reserved.keys()) == {o6, o8} + assert allocator._op_reserved[o6] == ExecutionResources( + cpu=3.75, object_store_memory=213 + ) + assert allocator._op_reserved[o8] == ExecutionResources( + cpu=3.75, object_store_memory=213 + ) + assert allocator._reserved_for_op_outputs[o6] == 212 + assert allocator._reserved_for_op_outputs[o8] == 212 + assert allocator._total_shared == ExecutionResources( + cpu=7.5, object_store_memory=850 + ) + # object_store_memory budget = 0 + (850 - 87) / 2 = 381 (rounded down) + assert allocator._op_budgets[o6] == ExecutionResources( + cpu=5.5, object_store_memory=381 + ) + # object_store_memory budget = 163 + (850 - 87) / 2 = 545 (rounded up) + assert allocator._op_budgets[o8] == ExecutionResources( + cpu=5.5, object_store_memory=545 + ) + if __name__ == "__main__": import sys From 62cf604587e286a4dec8f29be48b38a8dcfe3868 Mon Sep 17 00:00:00 2001 From: Abrar Sheikh Date: Wed, 10 Sep 2025 12:44:55 -0700 Subject: [PATCH 562/634] integrate deployment ranks with deployment state (#55829) Integrate `DeploymentRankManager` with `DeploymentState`'s state machine. 1. Ranks are assigned when replica go to the Running from the Starting state 2. Ranks are released when replicas are fully stopped. 3. We check of consistency and minimally reassign rank in each update cycle. This is only applicable when there are holes in ranks after a downscaling event. 4. Rank manager updates its state about the replica rank if previous state was recovering and the replicas' startup succeeded. This is only applicable in the case of a controller restart. Next PR https://github.com/ray-project/ray/pull/56120 --------- Signed-off-by: abrar --- python/ray/serve/_private/deployment_state.py | 156 +++++++- python/ray/serve/tests/unit/BUILD.bazel | 10 +- .../serve/tests/unit/test_deployment_state.py | 367 +++++++++++++++++- 3 files changed, 514 insertions(+), 19 deletions(-) diff --git a/python/ray/serve/_private/deployment_state.py b/python/ray/serve/_private/deployment_state.py index 96658abd2cb9..04d5519aacf1 100644 --- a/python/ray/serve/_private/deployment_state.py +++ b/python/ray/serve/_private/deployment_state.py @@ -424,12 +424,15 @@ def initialization_latency_s(self) -> Optional[float]: return self._initialization_latency_s - def start(self, deployment_info: DeploymentInfo) -> ReplicaSchedulingRequest: + def start( + self, deployment_info: DeploymentInfo, rank: int + ) -> ReplicaSchedulingRequest: """Start the current DeploymentReplica instance. The replica will be in the STARTING and PENDING_ALLOCATION states until the deployment scheduler schedules the underlying actor. """ + self._rank = rank # Store the rank assigned to this replica self._actor_resources = deployment_info.replica_config.resource_dict self._ingress = deployment_info.ingress # it is currently not possible to create a placement group @@ -460,8 +463,6 @@ def start(self, deployment_info: DeploymentInfo) -> ReplicaSchedulingRequest: if self._deployment_is_cross_language else deployment_info.replica_config.serialized_init_args ) - # TODO(abrar): Fill in the correct rank - rank = 0 init_args = ( self.replica_id, cloudpickle.dumps(deployment_info.replica_config.deployment_def) @@ -591,7 +592,11 @@ def _format_user_config(self, user_config: Any): temp = msgpack_deserialize(temp) return temp - def reconfigure(self, version: DeploymentVersion) -> bool: + def reconfigure( + self, + version: DeploymentVersion, + rank: int, + ) -> bool: """ Update replica version. Also, updates the deployment config on the actor behind this DeploymentReplica instance if necessary. @@ -599,16 +604,22 @@ def reconfigure(self, version: DeploymentVersion) -> bool: Returns: whether the actor is being updated. """ updating = False - if self._version.requires_actor_reconfigure(version): + + # Determine if we need heavyweight reconfiguration + # vs lightweight updates + needs_actor_reconfigure = self._version.requires_actor_reconfigure(version) + has_rank_changes = self._rank != rank + + if needs_actor_reconfigure or has_rank_changes: # Call into replica actor reconfigure() with updated user config and # graceful_shutdown_wait_loop_s + # Setting updating=True because we want to transition to UPDATING state + # when rank is updated or deployment config changes. updating = True deployment_config = copy(version.deployment_config) deployment_config.user_config = self._format_user_config( deployment_config.user_config ) - # TODO(abrar): FIll in the correct rank - rank = 0 self._ready_obj_ref = self._actor_handle.reconfigure.remote( deployment_config, rank, @@ -616,6 +627,7 @@ def reconfigure(self, version: DeploymentVersion) -> bool: ) self._version = version + self._rank = rank return updating def recover(self) -> bool: @@ -1125,24 +1137,30 @@ def initialization_latency_s(self) -> Optional[float]: return self._actor.initialization_latency_s - def start(self, deployment_info: DeploymentInfo) -> ReplicaSchedulingRequest: + def start( + self, deployment_info: DeploymentInfo, rank: int + ) -> ReplicaSchedulingRequest: """ Start a new actor for current DeploymentReplica instance. """ - replica_scheduling_request = self._actor.start(deployment_info) + replica_scheduling_request = self._actor.start(deployment_info, rank=rank) self._start_time = time.time() self._logged_shutdown_message = False self.update_actor_details(start_time_s=self._start_time) return replica_scheduling_request - def reconfigure(self, version: DeploymentVersion) -> bool: + def reconfigure( + self, + version: DeploymentVersion, + rank: int, + ) -> bool: """ Update replica version. Also, updates the deployment config on the actor behind this DeploymentReplica instance if necessary. Returns: whether the actor is being updated. """ - return self._actor.reconfigure(version) + return self._actor.reconfigure(version, rank=rank) def recover(self) -> bool: """ @@ -1160,6 +1178,11 @@ def recover(self) -> bool: self.update_actor_details(start_time_s=self._start_time) return True + @property + def rank(self) -> Optional[int]: + """Get the rank assigned to the replica.""" + return self._actor.rank + def check_started( self, ) -> Tuple[ReplicaStartupStatus, Optional[str], Optional[float]]: @@ -1706,6 +1729,8 @@ def __init__( DeploymentStatusTrigger.CONFIG_UPDATE_STARTED, ) + self._rank_manager = DeploymentRankManager() + self.replica_average_ongoing_requests: Dict[str, float] = {} self.health_check_gauge = metrics.Gauge( @@ -2204,7 +2229,13 @@ def _stop_or_update_outdated_version_replicas(self, max_to_stop=math.inf) -> boo self._target_state.version ): replicas_changed = True - actor_updating = replica.reconfigure(self._target_state.version) + # Get current rank for the replica + current_rank = self._rank_manager.get_replica_rank( + replica.replica_id.unique_id + ) + actor_updating = replica.reconfigure( + self._target_state.version, rank=current_rank + ) if actor_updating: self._replicas.add(ReplicaState.UPDATING, replica) else: @@ -2319,14 +2350,23 @@ def scale_deployment_replicas( logger.info(f"Adding {to_add} replica{'s' * (to_add>1)} to {self._id}.") for _ in range(to_add): replica_id = ReplicaID(get_random_string(), deployment_id=self._id) + + # Assign rank during replica creation (startup process) + assigned_rank = self._rank_manager.assign_rank(replica_id.unique_id) + + logger.info( + f"Assigned rank {assigned_rank} to new replica {replica_id.unique_id} during startup" + ) new_deployment_replica = DeploymentReplica( replica_id, self._target_state.version, ) - upscale.append( - new_deployment_replica.start(self._target_state.info) + scheduling_request = new_deployment_replica.start( + self._target_state.info, rank=assigned_rank ) + upscale.append(scheduling_request) + self._replicas.add(ReplicaState.STARTING, new_deployment_replica) elif delta_replicas < 0: @@ -2432,6 +2472,16 @@ def _check_startup_replicas( for replica in self._replicas.pop(states=[original_state]): start_status, error_msg = replica.check_started() if start_status == ReplicaStartupStatus.SUCCEEDED: + if original_state == ReplicaState.RECOVERING: + # If the previous state was RECOVERING, that mean the replica + # crashed and is now starting up again. We need to recover the rank + # from the replica actor. The invariant is that the rank is assigned + # during startup and before the replica is added to the replicas + # data structure with RUNNING state. + # Recover rank from the replica actor during controller restart + replica_id = replica.replica_id.unique_id + recovered_rank = replica.rank + self._rank_manager.recover_rank(replica_id, recovered_rank) # This replica should be now be added to handle's replica # set. self._replicas.add(ReplicaState.RUNNING, replica) @@ -2661,8 +2711,73 @@ def check_and_update_replicas(self): self._replicas.add(ReplicaState.STOPPING, replica) else: logger.info(f"{replica.replica_id} is stopped.") + # Release rank only after replica is successfully stopped + # This ensures rank is available during draining/graceful shutdown + replica_id = replica.replica_id.unique_id + self._rank_manager.release_rank(replica_id) + logger.info( + f"Released rank from replica {replica_id} in deployment {self._id}" + ) self._autoscaling_state_manager.on_replica_stopped(replica.replica_id) + # After replica state updates, check rank consistency and perform minimal reassignment if needed + # This ensures ranks are continuous after lifecycle events + # Only do consistency check when deployment is stable (not during active updates) + # maybe this constraint need to be relaxed in the future. The implication is that + # if we delay the rank reassignment, the rank system will be in an invalid state + # for a longer period of time. Abrar made this decision because he is not confident + # about how rollouts work in the deployment state machine. + active_replicas = self._replicas.get() + if ( + active_replicas + and self._curr_status_info.status == DeploymentStatus.HEALTHY + ): + replicas_to_reconfigure = ( + self._rank_manager.check_rank_consistency_and_reassign_minimally( + active_replicas, + ) + ) + + # Reconfigure replicas that had their ranks reassigned + self._reconfigure_replicas_with_new_ranks(replicas_to_reconfigure) + + def _reconfigure_replicas_with_new_ranks( + self, replicas_to_reconfigure: List["DeploymentReplica"] + ): + """Reconfigure replicas with their new ranks after reassignment. + This uses the reconfigure() mechanism to update replicas with their new ranks. + """ + if not replicas_to_reconfigure: + return + + logger.info( + f"Reconfiguring {len(replicas_to_reconfigure)} replicas with rank changes in deployment {self._id}" + ) + + updated_count = 0 + for replica in replicas_to_reconfigure: + replica_id = replica.replica_id.unique_id + new_rank = self._rank_manager.get_replica_rank(replica_id) + + # Use reconfigure() to update rank + # World size is calculated automatically from deployment config + _ = replica.reconfigure( + self._target_state.version, + rank=new_rank, + ) + updated_count += 1 + + logger.info( + f"Successfully reconfigured {updated_count} replicas with new ranks in deployment {self._id}" + ) + + def _get_replica_ranks_mapping(self) -> Dict[str, int]: + """Get the current mapping of replica IDs to ranks. + Returns: + Dictionary mapping replica_id to rank. + """ + return self._rank_manager.get_replica_ranks_mapping() + def _choose_pending_migration_replicas_to_stop( self, replicas: List[DeploymentReplica], @@ -3305,3 +3420,16 @@ def get_active_node_ids(self) -> Set[str]: for deployment_state in self._deployment_states.values(): node_ids.update(deployment_state.get_active_node_ids()) return node_ids + + def _get_replica_ranks_mapping(self, deployment_id: DeploymentID) -> Dict[str, int]: + """Get the current rank mapping for all replicas in a deployment. + Args: + deployment_id: The deployment ID to get ranks for. + Returns: + Dictionary mapping replica_id to rank. + """ + deployment_state = self._deployment_states.get(deployment_id) + if deployment_state is None: + return {} + + return deployment_state._get_replica_ranks_mapping() diff --git a/python/ray/serve/tests/unit/BUILD.bazel b/python/ray/serve/tests/unit/BUILD.bazel index 032ce7de5c5b..7e3510045199 100644 --- a/python/ray/serve/tests/unit/BUILD.bazel +++ b/python/ray/serve/tests/unit/BUILD.bazel @@ -21,7 +21,10 @@ py_test_run_all_subdirectory( py_test_module_list( size = "medium", - env = {"RAY_SERVE_USE_COMPACT_SCHEDULING_STRATEGY": "1"}, + env = { + "RAY_SERVE_USE_COMPACT_SCHEDULING_STRATEGY": "1", + "RAY_SERVE_FAIL_ON_RANK_ERROR": "1", + }, files = [ "test_deployment_scheduler.py", "test_deployment_state.py", @@ -40,7 +43,10 @@ py_test_module_list( py_test_module_list( size = "medium", - env = {"RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0"}, + env = { + "RAY_SERVE_COLLECT_AUTOSCALING_METRICS_ON_HANDLE": "0", + "RAY_SERVE_FAIL_ON_RANK_ERROR": "1", + }, files = [ "test_autoscaling_policy.py", "test_deployment_state.py", diff --git a/python/ray/serve/tests/unit/test_deployment_state.py b/python/ray/serve/tests/unit/test_deployment_state.py index ea910ee826d7..aab09ebdbbaa 100644 --- a/python/ray/serve/tests/unit/test_deployment_state.py +++ b/python/ray/serve/tests/unit/test_deployment_state.py @@ -62,6 +62,7 @@ # loop, so we can't "mark" a replica dead through a method. This global # state is cleared after each test that uses the fixtures in this file. dead_replicas_context = set() +replica_rank_context = {} TEST_DEPLOYMENT_ID = DeploymentID(name="test_deployment", app_name="test_app") TEST_DEPLOYMENT_ID_2 = DeploymentID(name="test_deployment_2", app_name="test_app") @@ -103,6 +104,7 @@ def __init__( self._pg_bundles = None self._initialization_latency_s = -1 self._docs_path = None + self._rank = replica_rank_context.get(replica_id.unique_id, None) @property def is_cross_language(self) -> bool: @@ -221,8 +223,10 @@ def set_node_id(self, node_id: str): def set_actor_id(self, actor_id: str): self._actor_id = actor_id - def start(self, deployment_info: DeploymentInfo): + def start(self, deployment_info: DeploymentInfo, rank: int): self.started = True + self._rank = rank + replica_rank_context[self._replica_id.unique_id] = rank def _on_scheduled_stub(*args, **kwargs): pass @@ -239,10 +243,20 @@ def _on_scheduled_stub(*args, **kwargs): on_scheduled=_on_scheduled_stub, ) - def reconfigure(self, version: DeploymentVersion): + @property + def rank(self) -> Optional[int]: + return self._rank + + def reconfigure( + self, + version: DeploymentVersion, + rank: int = None, + ): self.started = True updating = self.version.requires_actor_reconfigure(version) self.version = version + self._rank = rank + replica_rank_context[self._replica_id.unique_id] = rank return updating def recover(self): @@ -251,6 +265,7 @@ def recover(self): self.recovering = True self.started = False + self._rank = replica_rank_context.get(self._replica_id.unique_id, None) return True def check_ready(self) -> ReplicaStartupStatus: @@ -383,6 +398,7 @@ def create_deployment_state_manager( ) dead_replicas_context.clear() + replica_rank_context.clear() @pytest.fixture @@ -2659,7 +2675,7 @@ def test_max_concurrency_override(self): ) max_ongoing_requests = DEFAULT_MAX_CONCURRENCY_ASYNC + 1 d_info, _ = deployment_info(max_ongoing_requests=max_ongoing_requests) - replica_scheduling_request = actor_replica.start(d_info) + replica_scheduling_request = actor_replica.start(d_info, rank=0) assert ( "max_concurrency" in replica_scheduling_request.actor_options and replica_scheduling_request.actor_options["max_concurrency"] @@ -4981,5 +4997,350 @@ def test_docs_path_not_updated_for_different_version(mock_deployment_state_manag assert ds.docs_path is None +class TestDeploymentRankManagerIntegrationE2E: + """End-to-end integration tests for rank functionality through deployment state manager.""" + + def _set_replicas_ready( + self, ds: DeploymentState, replica_states: List[ReplicaState] + ): + """Helper to set replicas in given states to ready.""" + for replica in ds._replicas.get(replica_states): + replica._actor.set_ready() + + def _set_replicas_done_stopping(self, ds: DeploymentState): + """Helper to set stopping replicas as done stopping.""" + for replica in ds._replicas.get([ReplicaState.STOPPING]): + replica._actor.set_done_stopping() + + def test_scaling_up_and_down_scenario(self, mock_deployment_state_manager): + """Test a realistic scaling scenario through deployment state manager.""" + create_dsm, _, _, _ = mock_deployment_state_manager + dsm: DeploymentStateManager = create_dsm() + + # Start with 3 replicas + info_1, v1 = deployment_info(num_replicas=3, version="1") + dsm.deploy(TEST_DEPLOYMENT_ID, info_1) + ds: DeploymentState = dsm._deployment_states[TEST_DEPLOYMENT_ID] + + # Create initial replicas + dsm.update() + check_counts(ds, total=3, by_state=[(ReplicaState.STARTING, 3, v1)]) + + # Set replicas ready + self._set_replicas_ready(ds, [ReplicaState.STARTING]) + dsm.update() + check_counts(ds, total=3, by_state=[(ReplicaState.RUNNING, 3, v1)]) + assert ds.curr_status_info.status == DeploymentStatus.HEALTHY + + # Check initial ranks are 0, 1, 2 + ranks_mapping = ds._get_replica_ranks_mapping() + ranks = sorted(ranks_mapping.values()) + assert ranks == [0, 1, 2], f"Expected ranks [0, 1, 2], got {ranks}" + + # Scale down to 2 replicas - this should trigger rank reassignment + info_2, _ = deployment_info(num_replicas=2, version="1") + dsm.deploy(TEST_DEPLOYMENT_ID, info_2) + dsm.update() + + # One replica should be stopping + check_counts( + ds, + total=3, + by_state=[(ReplicaState.RUNNING, 2, v1), (ReplicaState.STOPPING, 1, v1)], + ) + + # Complete the scale down + self._set_replicas_done_stopping(ds) + dsm.update() + check_counts(ds, total=2, by_state=[(ReplicaState.RUNNING, 2, v1)]) + assert ds.curr_status_info.status == DeploymentStatus.HEALTHY + + # Trigger rank consistency check with one more update + dsm.update() + + # After scaling down and reaching healthy status, ranks should be contiguous [0, 1] + ranks_mapping = ds._get_replica_ranks_mapping() + ranks = sorted(ranks_mapping.values()) + assert ranks == [0, 1], f"Expected ranks [0, 1] after scale down, got {ranks}" + + # Scale back up to 3 replicas - new replica should reuse available rank + info_3, _ = deployment_info(num_replicas=3, version="1") + dsm.deploy(TEST_DEPLOYMENT_ID, info_3) + dsm.update() + + # Should have one new starting replica + check_counts( + ds, + total=3, + by_state=[(ReplicaState.RUNNING, 2, v1), (ReplicaState.STARTING, 1, v1)], + ) + + # Set new replica ready + self._set_replicas_ready(ds, [ReplicaState.STARTING]) + dsm.update() + check_counts(ds, total=3, by_state=[(ReplicaState.RUNNING, 3, v1)]) + assert ds.curr_status_info.status == DeploymentStatus.HEALTHY + + # Trigger rank consistency check with one more update + dsm.update() + + # Final ranks should be contiguous [0, 1, 2] + ranks_mapping = ds._get_replica_ranks_mapping() + ranks = sorted(ranks_mapping.values()) + assert ranks == [0, 1, 2], f"Expected final ranks [0, 1, 2], got {ranks}" + + def test_controller_recovery_with_scattered_ranks( + self, mock_deployment_state_manager + ): + """Test controller recovery with existing replica ranks through deployment state manager.""" + create_dsm, _, _, _ = mock_deployment_state_manager + dsm: DeploymentStateManager = create_dsm() + + # Deploy with 3 replicas + info_1, v1 = deployment_info(num_replicas=3, version="1") + target_state_changed = dsm.deploy(TEST_DEPLOYMENT_ID, info_1) + assert target_state_changed + dsm.save_checkpoint() + ds: DeploymentState = dsm._deployment_states[TEST_DEPLOYMENT_ID] + + # Create replicas and get them running + dsm.update() + check_counts(ds, total=3, by_state=[(ReplicaState.STARTING, 3, v1)]) + self._set_replicas_ready(ds, [ReplicaState.STARTING]) + dsm.update() + check_counts(ds, total=3, by_state=[(ReplicaState.RUNNING, 3, v1)]) + + # Get the actual replica objects (not just IDs) + replicas = ds._replicas.get([ReplicaState.RUNNING]) + replica_ids = [replica.replica_id for replica in replicas] + + # Simulate controller crashed! Create a new deployment state manager + # with the existing replica IDs to trigger recovery + new_dsm: DeploymentStateManager = create_dsm( + [replica_id.to_full_id_str() for replica_id in replica_ids] + ) + + # New deployment state should be created and replicas should be RECOVERING + new_ds = new_dsm._deployment_states[TEST_DEPLOYMENT_ID] + check_counts(new_ds, total=3, by_state=[(ReplicaState.RECOVERING, 3, v1)]) + + # Complete recovery - set replicas ready + self._set_replicas_ready(new_ds, [ReplicaState.RECOVERING]) + new_dsm.update() + check_counts(new_ds, total=3, by_state=[(ReplicaState.RUNNING, 3, v1)]) + assert new_ds.curr_status_info.status == DeploymentStatus.HEALTHY + + # At this point ranks should be scattered but all values [0, 1, 2] should be present + ranks_mapping = new_ds._get_replica_ranks_mapping() + ranks = sorted(ranks_mapping.values()) + assert ranks == [0, 1, 2], "Should have recovered scattered ranks" + + # Trigger rank consistency check with one more update - this should reorder if needed + new_dsm.update() + + # After rank consistency check, ranks should still be [0, 1, 2] + final_ranks_mapping = new_ds._get_replica_ranks_mapping() + final_ranks = sorted(final_ranks_mapping.values()) + assert final_ranks == [ + 0, + 1, + 2, + ], f"Expected contiguous ranks [0, 1, 2] after consistency check, got {final_ranks}" + + # Clean up + replica_rank_context.clear() + + def test_complex_reassignment_scenario(self, mock_deployment_state_manager): + """Test complex reassignment with many gaps through deployment state manager.""" + create_dsm, _, _, _ = mock_deployment_state_manager + dsm: DeploymentStateManager = create_dsm() + + # Deploy with 4 replicas + info_1, v1 = deployment_info(num_replicas=4, version="1") + target_state_changed = dsm.deploy(TEST_DEPLOYMENT_ID, info_1) + assert target_state_changed + dsm.save_checkpoint() + ds: DeploymentState = dsm._deployment_states[TEST_DEPLOYMENT_ID] + + # Create replicas and get them running + dsm.update() + check_counts(ds, total=4, by_state=[(ReplicaState.STARTING, 4, v1)]) + self._set_replicas_ready(ds, [ReplicaState.STARTING]) + dsm.update() + check_counts(ds, total=4, by_state=[(ReplicaState.RUNNING, 4, v1)]) + + # Get the actual replica objects + replicas = ds._replicas.get([ReplicaState.RUNNING]) + replica_ids = [replica.replica_id for replica in replicas] + + # Simulate very scattered ranks in global context: 0, 3, 7, 10 + global replica_rank_context + replica_rank_context.clear() + replica_rank_context[replica_ids[0].unique_id] = 0 + replica_rank_context[replica_ids[1].unique_id] = 3 + replica_rank_context[replica_ids[2].unique_id] = 7 + replica_rank_context[replica_ids[3].unique_id] = 10 + + # Simulate controller crashed! Create a new deployment state manager + # with the existing replica IDs to trigger recovery + new_dsm: DeploymentStateManager = create_dsm( + [replica_id.to_full_id_str() for replica_id in replica_ids] + ) + + # New deployment state should be created and replicas should be RECOVERING + new_ds = new_dsm._deployment_states[TEST_DEPLOYMENT_ID] + check_counts(new_ds, total=4, by_state=[(ReplicaState.RECOVERING, 4, v1)]) + + # Complete recovery - set replicas ready + self._set_replicas_ready(new_ds, [ReplicaState.RECOVERING]) + new_dsm.update() + check_counts(new_ds, total=4, by_state=[(ReplicaState.RUNNING, 4, v1)]) + assert new_ds.curr_status_info.status == DeploymentStatus.HEALTHY + + # Trigger rank consistency check with one more update + new_dsm.update() + + # After reassignment, ranks should be contiguous [0, 1, 2, 3] + ranks_mapping = new_ds._get_replica_ranks_mapping() + ranks = sorted(ranks_mapping.values()) + assert ranks == [ + 0, + 1, + 2, + 3, + ], f"Expected reassigned ranks [0, 1, 2, 3], got {ranks}" + + def test_rank_consistency_during_version_rollout( + self, mock_deployment_state_manager + ): + """Test that rank consistency is maintained during version rollouts.""" + create_dsm, _, _, _ = mock_deployment_state_manager + dsm: DeploymentStateManager = create_dsm() + + # Start with 3 replicas of version 1 + info_1, v1 = deployment_info(num_replicas=3, version="1") + dsm.deploy(TEST_DEPLOYMENT_ID, info_1) + ds: DeploymentState = dsm._deployment_states[TEST_DEPLOYMENT_ID] + + # Create and ready initial replicas + dsm.update() + check_counts(ds, total=3, by_state=[(ReplicaState.STARTING, 3, v1)]) + self._set_replicas_ready(ds, [ReplicaState.STARTING]) + dsm.update() + check_counts(ds, total=3, by_state=[(ReplicaState.RUNNING, 3, v1)]) + assert ds.curr_status_info.status == DeploymentStatus.HEALTHY + + # Verify initial ranks are contiguous + ranks_mapping = ds._get_replica_ranks_mapping() + initial_ranks = sorted(ranks_mapping.values()) + assert initial_ranks == [0, 1, 2] + + # Deploy version 2 - this should trigger rolling update + info_2, v2 = deployment_info(num_replicas=3, version="2") + dsm.deploy(TEST_DEPLOYMENT_ID, info_2) + dsm.update() + + # Complete the rolling update step by step + while True: + # Set any new starting replicas ready + starting_replicas = ds._replicas.get([ReplicaState.STARTING]) + if starting_replicas: + self._set_replicas_ready(ds, [ReplicaState.STARTING]) + + # Complete any stopping replicas + stopping_replicas = ds._replicas.get([ReplicaState.STOPPING]) + if stopping_replicas: + self._set_replicas_done_stopping(ds) + + dsm.update() + + # Check if rolling update is complete + running_replicas = ds._replicas.get([ReplicaState.RUNNING]) + if len(running_replicas) == 3 and all( + r.version == v2 for r in running_replicas + ): + break + + # After rolling update is complete, deployment should be healthy + assert ds.curr_status_info.status == DeploymentStatus.HEALTHY + + # Trigger rank consistency check with one more update + dsm.update() + + # After rolling update, verify ranks are still contiguous + final_ranks_mapping = ds._get_replica_ranks_mapping() + final_ranks = sorted(final_ranks_mapping.values()) + assert final_ranks == [ + 0, + 1, + 2, + ], f"Expected contiguous ranks [0, 1, 2] after rollout, got {final_ranks}" + + def test_rank_assignment_with_replica_failures(self, mock_deployment_state_manager): + """Test rank handling when replicas fail during startup.""" + create_dsm, _, _, _ = mock_deployment_state_manager + dsm: DeploymentStateManager = create_dsm() + + # Deploy with 3 replicas + info_1, v1 = deployment_info(num_replicas=3, version="1") + dsm.deploy(TEST_DEPLOYMENT_ID, info_1) + ds: DeploymentState = dsm._deployment_states[TEST_DEPLOYMENT_ID] + + # Create initial replicas + dsm.update() + check_counts(ds, total=3, by_state=[(ReplicaState.STARTING, 3, v1)]) + + # Make first two replicas ready, but let the third fail + starting_replicas = ds._replicas.get([ReplicaState.STARTING]) + starting_replicas[0]._actor.set_ready() + starting_replicas[1]._actor.set_ready() + starting_replicas[2]._actor.set_failed_to_start() + + dsm.update() + + running_count = ds._replicas.count(states=[ReplicaState.RUNNING]) + stopping_count = ds._replicas.count(states=[ReplicaState.STOPPING]) + assert running_count == 2, "Should have 2 running replicas" + assert stopping_count == 1, "Should have 1 stopping replica" + + self._set_replicas_done_stopping(ds) + dsm.update() + + starting_count = ds._replicas.count(states=[ReplicaState.STARTING]) + assert starting_count == 1, "Should have 1 starting replica" + + self._set_replicas_ready(ds, [ReplicaState.STARTING]) + + dsm.update() + # second update to reassign ranks + dsm.update() + + # Final verification - should have 3 running replicas (ignore failed/stopping replicas) + running_replicas = ds._replicas.get([ReplicaState.RUNNING]) + assert ( + len(running_replicas) == 3 + ), f"Expected 3 running replicas, got {len(running_replicas)}" + + # Verify that ranks are properly assigned and unique for running replicas + ranks_mapping = ds._get_replica_ranks_mapping() + + # Filter ranks to only include those for running replicas + running_replica_ids = [ + replica.replica_id.unique_id for replica in running_replicas + ] + running_replica_ranks = [ + ranks_mapping[replica_id] + for replica_id in running_replica_ids + if replica_id in ranks_mapping + ] + + # The ranks should be assigned to all running replicas + assert set(running_replica_ranks) == { + 0, + 1, + 2, + }, f"Expected ranks [0, 1, 2], got {ranks_mapping.values()}" + + if __name__ == "__main__": sys.exit(pytest.main(["-v", "-s", __file__])) From 752f8940d77d27aefb91f04a8863ed5b6da8dfd8 Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Wed, 10 Sep 2025 13:02:15 -0700 Subject: [PATCH 563/634] [Data] Remove filesystem variants from JSON, NumPy, and Delta tests (#56377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? Many our format-specific tests include variants for different PyArrow filesystems, but this is unnecessary. Assuming the PyArrow filesystem abstraction works correctly, the underlying filesystem shouldn’t affect behavior. Also, `FileBasedDatasource` subclasses don’t interact with the filesystem directly—they just receive a stream and define how to deserialize it. So, it’s cleaner and simpler to test filesystem variants at the `FileBasedDatasource` base class level rather than in every subclass, reducing test duplication and complexity. ## Related issue number This PR is a follow up https://github.com/ray-project/ray/pull/56345. ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Balaji Veeramani --- python/ray/data/tests/test_delta.py | 14 +- python/ray/data/tests/test_json.py | 306 +++++++--------------------- python/ray/data/tests/test_numpy.py | 19 +- 3 files changed, 76 insertions(+), 263 deletions(-) diff --git a/python/ray/data/tests/test_delta.py b/python/ray/data/tests/test_delta.py index c00882b1bda9..60851c2f6573 100644 --- a/python/ray/data/tests/test_delta.py +++ b/python/ray/data/tests/test_delta.py @@ -2,23 +2,14 @@ import pyarrow as pa import pytest -from pytest_lazy_fixtures import lf as lazy_fixture import ray from ray.data import Schema -from ray.data.datasource.path_util import _unwrap_protocol from ray.data.tests.conftest import * # noqa from ray.data.tests.mock_http_server import * # noqa from ray.tests.conftest import * # noqa -@pytest.mark.parametrize( - "data_path", - [ - lazy_fixture("local_path"), - lazy_fixture("s3_path"), - ], -) @pytest.mark.parametrize( "batch_size", [1, 100], @@ -27,13 +18,12 @@ "write_mode", ["append", "overwrite"], ) -def test_delta_read_basic(data_path, batch_size, write_mode): +def test_delta_read_basic(tmp_path, batch_size, write_mode): import pandas as pd from deltalake import write_deltalake # Parse the data path. - setup_data_path = _unwrap_protocol(data_path) - path = os.path.join(setup_data_path, "tmp_test_delta") + path = os.path.join(tmp_path, "tmp_test_delta") # Create a sample Delta Lake table df = pd.DataFrame( diff --git a/python/ray/data/tests/test_json.py b/python/ray/data/tests/test_json.py index a7336f04ce51..a10dc93658b1 100644 --- a/python/ray/data/tests/test_json.py +++ b/python/ray/data/tests/test_json.py @@ -8,7 +8,6 @@ import pyarrow.fs as fs import pyarrow.json as pajson import pytest -from pytest_lazy_fixtures import lf as lazy_fixture import ray from ray.data import Schema @@ -23,7 +22,6 @@ from ray.data.datasource.file_based_datasource import ( FILE_SIZE_FETCH_PARALLELIZATION_THRESHOLD, ) -from ray.data.datasource.path_util import _unwrap_protocol from ray.data.tests.conftest import * # noqa from ray.tests.conftest import * # noqa @@ -31,42 +29,26 @@ pytestmark = pytest.mark.timeout(360) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) def test_json_read( - ray_start_regular_shared, - fs, - data_path, - endpoint_url, - target_max_block_size_infinite_or_default, + ray_start_regular_shared, target_max_block_size_infinite_or_default, tmp_path ): - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) # Single file. df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - path1 = os.path.join(data_path, "test1.json") - df1.to_json(path1, orient="records", lines=True, storage_options=storage_options) - ds = ray.data.read_json(path1, filesystem=fs) + path1 = os.path.join(tmp_path, "test1.json") + df1.to_json(path1, orient="records", lines=True) + ds = ray.data.read_json(path1) dsdf = ds.to_pandas() assert df1.equals(dsdf) # Test metadata ops. assert ds.count() == 3 - assert ds.input_files() == [_unwrap_protocol(path1)] + assert ds.input_files() == [path1] assert ds.schema() == Schema(pa.schema([("one", pa.int64()), ("two", pa.string())])) # Two files, override_num_blocks=2. df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) - path2 = os.path.join(data_path, "test2.json") - df2.to_json(path2, orient="records", lines=True, storage_options=storage_options) - ds = ray.data.read_json([path1, path2], override_num_blocks=2, filesystem=fs) + path2 = os.path.join(tmp_path, "test2.json") + df2.to_json(path2, orient="records", lines=True) + ds = ray.data.read_json([path1, path2], override_num_blocks=2) dsdf = ds.to_pandas() df = pd.concat([df1, df2], ignore_index=True) assert df.equals(dsdf) @@ -76,102 +58,74 @@ def test_json_read( # Three files, override_num_blocks=2. df3 = pd.DataFrame({"one": [7, 8, 9], "two": ["h", "i", "j"]}) - path3 = os.path.join(data_path, "test3.json") - df3.to_json(path3, orient="records", lines=True, storage_options=storage_options) - ds = ray.data.read_json([path1, path2, path3], override_num_blocks=2, filesystem=fs) + path3 = os.path.join(tmp_path, "test3.json") + df3.to_json(path3, orient="records", lines=True) + ds = ray.data.read_json([path1, path2, path3], override_num_blocks=2) df = pd.concat([df1, df2, df3], ignore_index=True) dsdf = ds.to_pandas() assert df.equals(dsdf) # Directory, two files. - path = os.path.join(data_path, "test_json_dir") - if fs is None: - os.mkdir(path) - else: - fs.create_dir(_unwrap_protocol(path)) + path = os.path.join(tmp_path, "test_json_dir") + os.mkdir(path) + df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) path1 = os.path.join(path, "data0.json") - df1.to_json(path1, orient="records", lines=True, storage_options=storage_options) + df1.to_json(path1, orient="records", lines=True) df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) path2 = os.path.join(path, "data1.json") - df2.to_json(path2, orient="records", lines=True, storage_options=storage_options) - ds = ray.data.read_json(path, filesystem=fs) + df2.to_json(path2, orient="records", lines=True) + ds = ray.data.read_json(path) df = pd.concat([df1, df2], ignore_index=True) dsdf = ds.to_pandas().sort_values(by=["one", "two"]).reset_index(drop=True) assert df.equals(dsdf) - if fs is None: - shutil.rmtree(path) - else: - fs.delete_dir(_unwrap_protocol(path)) + shutil.rmtree(path) # Two directories, three files. - path1 = os.path.join(data_path, "test_json_dir1") - path2 = os.path.join(data_path, "test_json_dir2") - if fs is None: - os.mkdir(path1) - os.mkdir(path2) - else: - fs.create_dir(_unwrap_protocol(path1)) - fs.create_dir(_unwrap_protocol(path2)) + path1 = os.path.join(tmp_path, "test_json_dir1") + path2 = os.path.join(tmp_path, "test_json_dir2") + os.mkdir(path1) + os.mkdir(path2) df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) file_path1 = os.path.join(path1, "data0.json") - df1.to_json( - file_path1, orient="records", lines=True, storage_options=storage_options - ) + df1.to_json(file_path1, orient="records", lines=True) df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) file_path2 = os.path.join(path2, "data1.json") - df2.to_json( - file_path2, orient="records", lines=True, storage_options=storage_options - ) + df2.to_json(file_path2, orient="records", lines=True) df3 = pd.DataFrame({"one": [7, 8, 9], "two": ["h", "i", "j"]}) file_path3 = os.path.join(path2, "data2.json") - df3.to_json( - file_path3, orient="records", lines=True, storage_options=storage_options - ) - ds = ray.data.read_json([path1, path2], filesystem=fs) + df3.to_json(file_path3, orient="records", lines=True) + ds = ray.data.read_json([path1, path2]) df = pd.concat([df1, df2, df3], ignore_index=True) dsdf = ds.to_pandas().sort_values(by=["one", "two"]).reset_index(drop=True) assert df.equals(dsdf) - if fs is None: - shutil.rmtree(path1) - shutil.rmtree(path2) - else: - fs.delete_dir(_unwrap_protocol(path1)) - fs.delete_dir(_unwrap_protocol(path2)) + shutil.rmtree(path1) + shutil.rmtree(path2) # Directory and file, two files. - dir_path = os.path.join(data_path, "test_json_dir") - if fs is None: - os.mkdir(dir_path) - else: - fs.create_dir(_unwrap_protocol(dir_path)) + dir_path = os.path.join(tmp_path, "test_json_dir") + os.mkdir(dir_path) df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) path1 = os.path.join(dir_path, "data0.json") - df1.to_json(path1, orient="records", lines=True, storage_options=storage_options) + df1.to_json(path1, orient="records", lines=True) df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) - path2 = os.path.join(data_path, "data1.json") - df2.to_json(path2, orient="records", lines=True, storage_options=storage_options) - ds = ray.data.read_json([dir_path, path2], filesystem=fs) + path2 = os.path.join(tmp_path, "data1.json") + df2.to_json(path2, orient="records", lines=True) + ds = ray.data.read_json([dir_path, path2]) df = pd.concat([df1, df2], ignore_index=True) dsdf = ds.to_pandas().sort_values(by=["one", "two"]).reset_index(drop=True) assert df.equals(dsdf) - if fs is None: - shutil.rmtree(dir_path) - else: - fs.delete_dir(_unwrap_protocol(dir_path)) + shutil.rmtree(dir_path) # Directory, two files and non-json file (test default extension-based filtering). - path = os.path.join(data_path, "test_json_dir") - if fs is None: - os.mkdir(path) - else: - fs.create_dir(_unwrap_protocol(path)) + path = os.path.join(tmp_path, "test_json_dir") + os.mkdir(path) df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) path1 = os.path.join(path, "data0.json") - df1.to_json(path1, orient="records", lines=True, storage_options=storage_options) + df1.to_json(path1, orient="records", lines=True) df2 = pd.DataFrame({"one": [4, 5, 6], "two": ["e", "f", "g"]}) path2 = os.path.join(path, "data1.json") - df2.to_json(path2, orient="records", lines=True, storage_options=storage_options) + df2.to_json(path2, orient="records", lines=True) # Add a file with a non-matching file extension. This file should be ignored. df_txt = pd.DataFrame({"foobar": [1, 2, 3]}) @@ -179,17 +133,13 @@ def test_json_read( os.path.join(path, "foo.txt"), orient="records", lines=True, - storage_options=storage_options, ) - ds = ray.data.read_json(path, filesystem=fs) + ds = ray.data.read_json(path) df = pd.concat([df1, df2], ignore_index=True) dsdf = ds.to_pandas().sort_values(by=["one", "two"]).reset_index(drop=True) assert df.equals(dsdf) - if fs is None: - shutil.rmtree(path) - else: - fs.delete_dir(_unwrap_protocol(path)) + shutil.rmtree(path) def test_zipped_json_read( @@ -254,118 +204,71 @@ def test_read_json_fallback_from_pyarrow_failure( assert ds.take_all() == data -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) def test_json_read_meta_provider( ray_start_regular_shared, - fs, - data_path, - endpoint_url, + tmp_path, target_max_block_size_infinite_or_default, ): - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) - df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - path1 = os.path.join(data_path, "test1.json") - df1.to_json(path1, orient="records", lines=True, storage_options=storage_options) + path1 = os.path.join(tmp_path, "test1.json") + df1.to_json(path1, orient="records", lines=True) ds = ray.data.read_json( path1, - filesystem=fs, meta_provider=FastFileMetadataProvider(), ) # Expect to lazily compute all metadata correctly. assert ds.count() == 3 - assert ds.input_files() == [_unwrap_protocol(path1)] + assert ds.input_files() == [path1] assert ds.schema() == Schema(pa.schema([("one", pa.int64()), ("two", pa.string())])) with pytest.raises(NotImplementedError): ray.data.read_json( path1, - filesystem=fs, meta_provider=BaseFileMetadataProvider(), ) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) def test_json_read_with_read_options( ray_start_regular_shared, - fs, - data_path, - endpoint_url, + tmp_path, target_max_block_size_infinite_or_default, ): # Arrow's JSON ReadOptions isn't serializable in pyarrow < 8.0.0, so this test # covers our custom ReadOptions serializer. # TODO(Clark): Remove this test and our custom serializer once we require # pyarrow >= 8.0.0. - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - path1 = os.path.join(data_path, "test1.json") - df1.to_json(path1, orient="records", lines=True, storage_options=storage_options) + path1 = os.path.join(tmp_path, "test1.json") + df1.to_json(path1, orient="records", lines=True) ds = ray.data.read_json( path1, - filesystem=fs, read_options=pajson.ReadOptions(use_threads=False, block_size=2**30), ) dsdf = ds.to_pandas() assert df1.equals(dsdf) # Test metadata ops. assert ds.count() == 3 - assert ds.input_files() == [_unwrap_protocol(path1)] + assert ds.input_files() == [path1] assert ds.schema() == Schema(pa.schema([("one", pa.int64()), ("two", pa.string())])) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) def test_json_read_with_parse_options( ray_start_regular_shared, - fs, - data_path, - endpoint_url, + tmp_path, target_max_block_size_infinite_or_default, ): # Arrow's JSON ParseOptions isn't serializable in pyarrow < 8.0.0, so this test # covers our custom ParseOptions serializer, similar to ReadOptions in above test. # TODO(chengsu): Remove this test and our custom serializer once we require # pyarrow >= 8.0.0. - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - path1 = os.path.join(data_path, "test1.json") - df1.to_json(path1, orient="records", lines=True, storage_options=storage_options) + path1 = os.path.join(tmp_path, "test1.json") + df1.to_json(path1, orient="records", lines=True) ds = ray.data.read_json( path1, - filesystem=fs, parse_options=pajson.ParseOptions( explicit_schema=pa.schema([("two", pa.string())]), unexpected_field_behavior="ignore", @@ -376,7 +279,7 @@ def test_json_read_with_parse_options( assert (df1["two"]).equals(dsdf["two"]) # Test metadata ops. assert ds.count() == 3 - assert ds.input_files() == [_unwrap_protocol(path1)] + assert ds.input_files() == [path1] assert ds.schema() == Schema(pa.schema([("two", pa.string())])) @@ -469,62 +372,31 @@ def test_json_roundtrip( assert BlockAccessor.for_block(ray.get(block)).size_bytes() == meta.size_bytes -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) def test_json_read_small_file_unit_block_size( ray_start_regular_shared, - fs, - data_path, - endpoint_url, + tmp_path, target_max_block_size_infinite_or_default, ): """Test reading a small JSON file with unit block_size.""" - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) df1 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - path1 = os.path.join(data_path, "test1.json") - df1.to_json(path1, orient="records", lines=True, storage_options=storage_options) - ds = ray.data.read_json( - path1, filesystem=fs, read_options=pajson.ReadOptions(block_size=1) - ) + path1 = os.path.join(tmp_path, "test1.json") + df1.to_json(path1, orient="records", lines=True) + ds = ray.data.read_json(path1, read_options=pajson.ReadOptions(block_size=1)) dsdf = ds.to_pandas() assert df1.equals(dsdf) # Test metadata ops. assert ds.count() == 3 - assert ds.input_files() == [_unwrap_protocol(path1)] + assert ds.input_files() == [path1] assert ds.schema() == Schema(pa.schema([("one", pa.int64()), ("two", pa.string())])) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) def test_json_read_file_larger_than_block_size( ray_start_regular_shared, - fs, - data_path, - endpoint_url, + tmp_path, target_max_block_size_infinite_or_default, ): """Test reading a JSON file larger than the block size.""" - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) - block_size = 1024 num_chars = 2500 num_rows = 3 @@ -534,84 +406,48 @@ def test_json_read_file_larger_than_block_size( "two": ["b" * num_chars for _ in range(num_rows)], } ) - path2 = os.path.join(data_path, "test2.json") - df2.to_json(path2, orient="records", lines=True, storage_options=storage_options) + path2 = os.path.join(tmp_path, "test2.json") + df2.to_json(path2, orient="records", lines=True) ds = ray.data.read_json( - path2, filesystem=fs, read_options=pajson.ReadOptions(block_size=block_size) + path2, read_options=pajson.ReadOptions(block_size=block_size) ) dsdf = ds.to_pandas() assert df2.equals(dsdf) # Test metadata ops. assert ds.count() == num_rows - assert ds.input_files() == [_unwrap_protocol(path2)] + assert ds.input_files() == [path2] assert ds.schema() == Schema( pa.schema([("one", pa.string()), ("two", pa.string())]) ) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) def test_json_read_negative_block_size_fallback( - ray_start_regular_shared, - fs, - data_path, - endpoint_url, - target_max_block_size_infinite_or_default, + ray_start_regular_shared, tmp_path, target_max_block_size_infinite_or_default ): """Test reading JSON with negative block_size triggers fallback to json.load().""" - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) df3 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - path3 = os.path.join(data_path, "test3.json") - df3.to_json(path3, orient="records", lines=True, storage_options=storage_options) + path3 = os.path.join(tmp_path, "test3.json") + df3.to_json(path3, orient="records", lines=True) # Negative Buffer Size, fails with arrow but succeeds in fallback to json.load() - ds = ray.data.read_json( - path3, filesystem=fs, read_options=pajson.ReadOptions(block_size=-1) - ) + ds = ray.data.read_json(path3, read_options=pajson.ReadOptions(block_size=-1)) dsdf = ds.to_pandas() assert df3.equals(dsdf) -@pytest.mark.parametrize( - "fs,data_path,endpoint_url", - [ - (None, lazy_fixture("local_path"), None), - (lazy_fixture("local_fs"), lazy_fixture("local_path"), None), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path"), lazy_fixture("s3_server")), - ], -) def test_json_read_zero_block_size_failure( - ray_start_regular_shared, - fs, - data_path, - endpoint_url, - target_max_block_size_infinite_or_default, + ray_start_regular_shared, tmp_path, target_max_block_size_infinite_or_default ): """Test reading JSON with zero block_size fails in both arrow and fallback.""" - if endpoint_url is None: - storage_options = {} - else: - storage_options = dict(client_kwargs=dict(endpoint_url=endpoint_url)) df3 = pd.DataFrame({"one": [1, 2, 3], "two": ["a", "b", "c"]}) - path3 = os.path.join(data_path, "test3.json") - df3.to_json(path3, orient="records", lines=True, storage_options=storage_options) + path3 = os.path.join(tmp_path, "test3.json") + df3.to_json(path3, orient="records", lines=True) # Zero Buffer Size, fails with arrow and fails in fallback to json.load() with pytest.raises(json.decoder.JSONDecodeError, match="Extra data"): - ds = ray.data.read_json( - path3, filesystem=fs, read_options=pajson.ReadOptions(block_size=0) - ) + ds = ray.data.read_json(path3, read_options=pajson.ReadOptions(block_size=0)) dsdf = ds.to_pandas() assert dsdf.equals(df3) diff --git a/python/ray/data/tests/test_numpy.py b/python/ray/data/tests/test_numpy.py index ee03ddb4315b..fa6a26fd404a 100644 --- a/python/ray/data/tests/test_numpy.py +++ b/python/ray/data/tests/test_numpy.py @@ -4,7 +4,6 @@ import pandas as pd import pyarrow as pa import pytest -from pytest_lazy_fixtures import lf as lazy_fixture import ray from ray.air.util.tensor_extensions.arrow import ArrowTensorTypeV2 @@ -94,24 +93,12 @@ def test_to_numpy_refs(ray_start_regular_shared): ) -@pytest.mark.parametrize( - "fs,data_path", - [ - (None, lazy_fixture("local_path")), - (lazy_fixture("local_fs"), lazy_fixture("local_path")), - (lazy_fixture("s3_fs"), lazy_fixture("s3_path")), - ( - lazy_fixture("s3_fs_with_anonymous_crendential"), - lazy_fixture("s3_path_with_anonymous_crendential"), - ), - ], -) -def test_numpy_roundtrip(ray_start_regular_shared, fs, data_path): +def test_numpy_roundtrip(ray_start_regular_shared, tmp_path): tensor_type = _get_tensor_type() ds = ray.data.range_tensor(10, override_num_blocks=2) - ds.write_numpy(data_path, filesystem=fs, column="data") - ds = ray.data.read_numpy(data_path, filesystem=fs) + ds.write_numpy(tmp_path, column="data") + ds = ray.data.read_numpy(tmp_path) assert ds.count() == 10 assert ds.schema() == Schema(pa.schema([("data", tensor_type((1,), pa.int64()))])) assert sorted(ds.take_all(), key=lambda row: row["data"]) == [ From d28b3f4a6d174bef423e962a52c7b0f7a28fa038 Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Wed, 10 Sep 2025 13:02:21 -0700 Subject: [PATCH 564/634] [Data] Add `average_num_inputs_per_task` and `num_output_blocks_per_task_s` metrics (#56379) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? This PR adds two task-level metrics for better visibility into operator performance: * `average_num_inputs_per_task` – average input blocks per task. * `num_output_blocks_per_task_s` – average output blocks per task per second. Both return `None` if no tasks have finished or no output exists, avoiding misleading values. These metrics can be used to help making scheduling decisions. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Balaji Veeramani --- .../interfaces/op_runtime_metrics.py | 25 +++++++++++++++++++ python/ray/data/tests/test_stats.py | 10 ++++++++ 2 files changed, 35 insertions(+) diff --git a/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py b/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py index a7066a979f15..eeffc116216c 100644 --- a/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py +++ b/python/ray/data/_internal/execution/interfaces/op_runtime_metrics.py @@ -522,6 +522,31 @@ def average_num_outputs_per_task(self) -> Optional[float]: else: return self.num_outputs_of_finished_tasks / self.num_tasks_finished + @metric_property( + description="Average number of blocks generated per task.", + metrics_group=MetricsGroup.INPUTS, + ) + def average_num_inputs_per_task(self) -> Optional[float]: + """Average number of input blocks per task, or None if no task has finished.""" + if self.num_tasks_finished == 0: + return None + else: + return self.num_task_inputs_processed / self.num_tasks_finished + + @metric_property( + description="Average number of output blocks per task per second.", + metrics_group=MetricsGroup.OUTPUTS, + ) + def num_output_blocks_per_task_s(self) -> Optional[float]: + """Average number of output blocks per task per second. + + If the operator hasn't produced any output yet, this metric returns `None`. + """ + if self.block_generation_time == 0: + return None + else: + return self.num_task_outputs_generated / self.block_generation_time + @metric_property( description="Average size of task output in bytes.", metrics_group=MetricsGroup.OUTPUTS, diff --git a/python/ray/data/tests/test_stats.py b/python/ray/data/tests/test_stats.py index 4d7a5660a86c..07cb5c359fd2 100644 --- a/python/ray/data/tests/test_stats.py +++ b/python/ray/data/tests/test_stats.py @@ -75,6 +75,8 @@ def gen_expected_metrics( if is_map: metrics = [ "'average_num_outputs_per_task': N", + "'average_num_inputs_per_task': N", + "'num_output_blocks_per_task_s': N", "'average_bytes_per_output': N", "'obj_store_mem_internal_inqueue': Z", "'obj_store_mem_internal_outqueue': Z", @@ -136,6 +138,8 @@ def gen_expected_metrics( else: metrics = [ "'average_num_outputs_per_task': None", + "'average_num_inputs_per_task': None", + "'num_output_blocks_per_task_s': None", "'average_bytes_per_output': None", "'obj_store_mem_internal_inqueue': Z", "'obj_store_mem_internal_outqueue': Z", @@ -678,6 +682,8 @@ def test_dataset__repr__(ray_start_regular_shared, restore_data_context): " number=N,\n" " extra_metrics={\n" " average_num_outputs_per_task: N,\n" + " average_num_inputs_per_task: N,\n" + " num_output_blocks_per_task_s: N,\n" " average_bytes_per_output: N,\n" " obj_store_mem_internal_inqueue: Z,\n" " obj_store_mem_internal_outqueue: Z,\n" @@ -808,6 +814,8 @@ def check_stats(): " number=N,\n" " extra_metrics={\n" " average_num_outputs_per_task: N,\n" + " average_num_inputs_per_task: N,\n" + " num_output_blocks_per_task_s: N,\n" " average_bytes_per_output: N,\n" " obj_store_mem_internal_inqueue: Z,\n" " obj_store_mem_internal_outqueue: Z,\n" @@ -893,6 +901,8 @@ def check_stats(): " number=N,\n" " extra_metrics={\n" " average_num_outputs_per_task: N,\n" + " average_num_inputs_per_task: N,\n" + " num_output_blocks_per_task_s: N,\n" " average_bytes_per_output: N,\n" " obj_store_mem_internal_inqueue: Z,\n" " obj_store_mem_internal_outqueue: Z,\n" From 37ad9f56b8c7a50044623066897be508a173e310 Mon Sep 17 00:00:00 2001 From: Vaishnavi Panchavati <38342947+vaishdho1@users.noreply.github.com> Date: Wed, 10 Sep 2025 13:07:10 -0700 Subject: [PATCH 565/634] [core] Migrate ray_logging from _private to _common[#53478] (#56368) Signed-off-by: Vaishnavi Panchavati --- .../ray_logging => _common}/filters.py | 0 .../ray_logging => _common}/formatters.py | 0 python/ray/_common/tests/BUILD.bazel | 3 + python/ray/_common/tests/conftest.py | 2 + python/ray/_common/tests/test_filters.py | 106 ++++++++ python/ray/_common/tests/test_formatters.py | 162 ++++++++++++ .../_private/ray_logging/logging_config.py | 4 +- .../dashboard/modules/job/job_supervisor.py | 4 +- python/ray/data/_internal/logging.py | 6 +- .../batch/observability/logging/__init__.py | 2 +- .../batch/observability/logging/setup.py | 4 +- .../common/observability/logging/__init__.py | 2 +- .../serve/observability/logging/__init__.py | 2 +- .../serve/observability/logging/setup.py | 4 +- python/ray/serve/_private/logging_utils.py | 4 +- python/ray/serve/_private/proxy.py | 2 +- python/ray/serve/_private/replica.py | 2 +- python/ray/serve/tests/test_logging.py | 2 +- python/ray/tests/test_logging_2.py | 249 ------------------ .../ray/train/v2/_internal/logging/logging.py | 4 +- 20 files changed, 293 insertions(+), 271 deletions(-) rename python/ray/{_private/ray_logging => _common}/filters.py (100%) rename python/ray/{_private/ray_logging => _common}/formatters.py (100%) create mode 100644 python/ray/_common/tests/conftest.py create mode 100644 python/ray/_common/tests/test_filters.py create mode 100644 python/ray/_common/tests/test_formatters.py diff --git a/python/ray/_private/ray_logging/filters.py b/python/ray/_common/filters.py similarity index 100% rename from python/ray/_private/ray_logging/filters.py rename to python/ray/_common/filters.py diff --git a/python/ray/_private/ray_logging/formatters.py b/python/ray/_common/formatters.py similarity index 100% rename from python/ray/_private/ray_logging/formatters.py rename to python/ray/_common/formatters.py diff --git a/python/ray/_common/tests/BUILD.bazel b/python/ray/_common/tests/BUILD.bazel index 1cc36ef348a8..d9aba47b5eb9 100644 --- a/python/ray/_common/tests/BUILD.bazel +++ b/python/ray/_common/tests/BUILD.bazel @@ -15,6 +15,8 @@ py_test_module_list( size = "small", files = [ "test_deprecation.py", + "test_filters.py", + "test_formatters.py", "test_network_utils.py", "test_ray_option_utils.py", "test_signal_semaphore_utils.py", @@ -27,6 +29,7 @@ py_test_module_list( "team:core", ], deps = [ + ":conftest", "//:ray_lib", ], ) diff --git a/python/ray/_common/tests/conftest.py b/python/ray/_common/tests/conftest.py new file mode 100644 index 000000000000..07810c3694dc --- /dev/null +++ b/python/ray/_common/tests/conftest.py @@ -0,0 +1,2 @@ +# Imports for filters and formatters tests +pytest_plugins = ["ray.tests.conftest"] diff --git a/python/ray/_common/tests/test_filters.py b/python/ray/_common/tests/test_filters.py new file mode 100644 index 000000000000..330cc69a1696 --- /dev/null +++ b/python/ray/_common/tests/test_filters.py @@ -0,0 +1,106 @@ +import logging +import logging.config +import sys + +import pytest + +import ray +from ray._common.filters import CoreContextFilter + + +class TestCoreContextFilter: + def test_driver_process(self, shutdown_only): + log_context = ["job_id", "worker_id", "node_id"] + filter = CoreContextFilter() + record = logging.makeLogRecord({}) + assert filter.filter(record) + # Ray is not initialized so no context except PID which should be available + for attr in log_context: + assert not hasattr(record, attr) + # PID should be available even when Ray is not initialized + assert hasattr(record, "process") + assert hasattr(record, "_ray_timestamp_ns") + + ray.init() + record = logging.makeLogRecord({}) + assert filter.filter(record) + runtime_context = ray.get_runtime_context() + expected_values = { + "job_id": runtime_context.get_job_id(), + "worker_id": runtime_context.get_worker_id(), + "node_id": runtime_context.get_node_id(), + "process": record.process, + } + for attr in log_context: + assert hasattr(record, attr) + assert getattr(record, attr) == expected_values[attr] + # This is not a worker process, so actor_id and task_id should not exist. + for attr in ["actor_id", "task_id"]: + assert not hasattr(record, attr) + assert hasattr(record, "_ray_timestamp_ns") + + def test_task_process(self, shutdown_only): + @ray.remote + def f(): + filter = CoreContextFilter() + record = logging.makeLogRecord({}) + assert filter.filter(record) + should_exist = ["job_id", "worker_id", "node_id", "task_id", "process"] + runtime_context = ray.get_runtime_context() + expected_values = { + "job_id": runtime_context.get_job_id(), + "worker_id": runtime_context.get_worker_id(), + "node_id": runtime_context.get_node_id(), + "task_id": runtime_context.get_task_id(), + "task_name": runtime_context.get_task_name(), + "task_func_name": runtime_context.get_task_function_name(), + "process": record.process, + } + for attr in should_exist: + assert hasattr(record, attr) + assert getattr(record, attr) == expected_values[attr] + assert not hasattr(record, "actor_id") + assert not hasattr(record, "actor_name") + assert hasattr(record, "_ray_timestamp_ns") + + obj_ref = f.remote() + ray.get(obj_ref) + + def test_actor_process(self, shutdown_only): + @ray.remote + class A: + def f(self): + filter = CoreContextFilter() + record = logging.makeLogRecord({}) + assert filter.filter(record) + should_exist = [ + "job_id", + "worker_id", + "node_id", + "actor_id", + "task_id", + "process", + ] + runtime_context = ray.get_runtime_context() + expected_values = { + "job_id": runtime_context.get_job_id(), + "worker_id": runtime_context.get_worker_id(), + "node_id": runtime_context.get_node_id(), + "actor_id": runtime_context.get_actor_id(), + "actor_name": runtime_context.get_actor_name(), + "task_id": runtime_context.get_task_id(), + "task_name": runtime_context.get_task_name(), + "task_func_name": runtime_context.get_task_function_name(), + "process": record.process, + } + for attr in should_exist: + assert hasattr(record, attr) + assert getattr(record, attr) == expected_values[attr] + assert hasattr(record, "_ray_timestamp_ns") + + actor = A.remote() + ray.get(actor.f.remote()) + + +if __name__ == "__main__": + sys.exit(pytest.main(["-sv", __file__])) diff --git a/python/ray/_common/tests/test_formatters.py b/python/ray/_common/tests/test_formatters.py new file mode 100644 index 000000000000..f81dcdffe84d --- /dev/null +++ b/python/ray/_common/tests/test_formatters.py @@ -0,0 +1,162 @@ +import json +import logging +import logging.config +import sys + +import pytest + +from ray._common.formatters import JSONFormatter, TextFormatter + + +class TestJSONFormatter: + def test_empty_record(self, shutdown_only): + formatter = JSONFormatter() + record = logging.makeLogRecord({}) + formatted = formatter.format(record) + + record_dict = json.loads(formatted) + should_exist = [ + "process", + "asctime", + "levelname", + "message", + "filename", + "lineno", + "timestamp_ns", + ] + for key in should_exist: + assert key in record_dict + assert len(record_dict) == len(should_exist) + assert "exc_text" not in record_dict + + def test_record_with_exception(self, shutdown_only): + formatter = JSONFormatter() + record = logging.makeLogRecord({}) + try: + raise ValueError("test") + except ValueError: + record.exc_info = sys.exc_info() + formatted = formatter.format(record) + record_dict = json.loads(formatted) + should_exist = [ + "process", + "asctime", + "levelname", + "message", + "filename", + "lineno", + "exc_text", + "timestamp_ns", + ] + for key in should_exist: + assert key in record_dict + assert "Traceback (most recent call last):" in record_dict["exc_text"] + assert len(record_dict) == len(should_exist) + + def test_record_with_user_provided_context(self, shutdown_only): + formatter = JSONFormatter() + record = logging.makeLogRecord({"user": "ray"}) + formatted = formatter.format(record) + record_dict = json.loads(formatted) + should_exist = [ + "process", + "asctime", + "levelname", + "message", + "filename", + "lineno", + "user", + "timestamp_ns", + ] + for key in should_exist: + assert key in record_dict + assert record_dict["user"] == "ray" + assert len(record_dict) == len(should_exist) + assert "exc_text" not in record_dict + + def test_record_with_flatten_keys_invalid_value(self, shutdown_only): + formatter = JSONFormatter() + record = logging.makeLogRecord({"ray_serve_extra_fields": "not_a_dict"}) + with pytest.raises(ValueError): + formatter.format(record) + + def test_record_with_flatten_keys_valid_dict(self, shutdown_only): + formatter = JSONFormatter() + record = logging.makeLogRecord( + {"ray_serve_extra_fields": {"key1": "value1", "key2": 2}} + ) + formatted = formatter.format(record) + record_dict = json.loads(formatted) + should_exist = [ + "process", + "asctime", + "levelname", + "message", + "filename", + "lineno", + "key1", + "key2", + "timestamp_ns", + ] + for key in should_exist: + assert key in record_dict + assert record_dict["key1"] == "value1", record_dict + assert record_dict["key2"] == 2 + assert "ray_serve_extra_fields" not in record_dict + assert len(record_dict) == len(should_exist) + assert "exc_text" not in record_dict + + def test_record_with_valid_additional_log_standard_attrs(self, shutdown_only): + formatter = JSONFormatter() + formatter.set_additional_log_standard_attrs(["name"]) + record = logging.makeLogRecord({}) + formatted = formatter.format(record) + + record_dict = json.loads(formatted) + should_exist = [ + "process", + "asctime", + "levelname", + "message", + "filename", + "lineno", + "timestamp_ns", + "name", + ] + for key in should_exist: + assert key in record_dict + assert len(record_dict) == len(should_exist) + + +class TestTextFormatter: + def test_record_with_user_provided_context(self): + formatter = TextFormatter() + record = logging.makeLogRecord({"user": "ray"}) + formatted = formatter.format(record) + assert "user=ray" in formatted + + def test_record_with_exception(self): + formatter = TextFormatter() + record = logging.LogRecord( + name="test_logger", + level=logging.INFO, + pathname="test.py", + lineno=1000, + msg="Test message", + args=None, + exc_info=None, + ) + formatted = formatter.format(record) + for s in ["INFO", "Test message", "test.py:1000", "--"]: + assert s in formatted + + def test_record_with_valid_additional_log_standard_attrs(self, shutdown_only): + formatter = TextFormatter() + formatter.set_additional_log_standard_attrs(["name"]) + record = logging.makeLogRecord({}) + formatted = formatter.format(record) + assert "name=" in formatted + + +if __name__ == "__main__": + sys.exit(pytest.main(["-sv", __file__])) diff --git a/python/ray/_private/ray_logging/logging_config.py b/python/ray/_private/ray_logging/logging_config.py index 843935d6b415..67453163a027 100644 --- a/python/ray/_private/ray_logging/logging_config.py +++ b/python/ray/_private/ray_logging/logging_config.py @@ -3,10 +3,10 @@ from dataclasses import dataclass, field from typing import Set +from ray._common.filters import CoreContextFilter +from ray._common.formatters import JSONFormatter, TextFormatter from ray._private.ray_logging import default_impl from ray._private.ray_logging.constants import LOGRECORD_STANDARD_ATTRS -from ray._private.ray_logging.filters import CoreContextFilter -from ray._private.ray_logging.formatters import JSONFormatter, TextFormatter from ray.util.annotations import PublicAPI diff --git a/python/ray/dashboard/modules/job/job_supervisor.py b/python/ray/dashboard/modules/job/job_supervisor.py index 846fc19eefd6..60766ee93935 100644 --- a/python/ray/dashboard/modules/job/job_supervisor.py +++ b/python/ray/dashboard/modules/job/job_supervisor.py @@ -11,10 +11,10 @@ import ray import ray._private.ray_constants as ray_constants +from ray._common.filters import CoreContextFilter +from ray._common.formatters import JSONFormatter, TextFormatter from ray._common.network_utils import build_address from ray._private.accelerators.nvidia_gpu import NOSET_CUDA_VISIBLE_DEVICES_ENV_VAR -from ray._private.ray_logging.filters import CoreContextFilter -from ray._private.ray_logging.formatters import JSONFormatter, TextFormatter from ray._private.runtime_env.constants import RAY_JOB_CONFIG_JSON_ENV_VAR from ray._private.utils import remove_ray_internal_flags_from_env from ray._raylet import GcsClient diff --git a/python/ray/data/_internal/logging.py b/python/ray/data/_internal/logging.py index 0184ac58e5d6..9c3b5abc0301 100644 --- a/python/ray/data/_internal/logging.py +++ b/python/ray/data/_internal/logging.py @@ -10,7 +10,7 @@ DEFAULT_TEXT_FORMATTER = ( "%(asctime)s\t%(levelname)s %(filename)s:%(lineno)s -- %(message)s" # noqa: E501 ) -DEFAULT_JSON_FORMATTER = ray._private.ray_logging.formatters.JSONFormatter +DEFAULT_JSON_FORMATTER = ray._common.formatters.JSONFormatter DEFAULT_CONFIG = { "version": 1, "disable_existing_loggers": False, @@ -22,9 +22,7 @@ }, "filters": { "console_filter": {"()": "ray.data._internal.logging.HiddenRecordFilter"}, - "core_context_filter": { - "()": "ray._private.ray_logging.filters.CoreContextFilter" - }, + "core_context_filter": {"()": "ray._common.filters.CoreContextFilter"}, }, "handlers": { "file": { diff --git a/python/ray/llm/_internal/batch/observability/logging/__init__.py b/python/ray/llm/_internal/batch/observability/logging/__init__.py index 4a81025a613c..04cd4d26101f 100644 --- a/python/ray/llm/_internal/batch/observability/logging/__init__.py +++ b/python/ray/llm/_internal/batch/observability/logging/__init__.py @@ -1,7 +1,7 @@ import logging from typing import Optional -from ray._private.ray_logging.filters import CoreContextFilter +from ray._common.filters import CoreContextFilter def _setup_logger(logger_name: str): diff --git a/python/ray/llm/_internal/batch/observability/logging/setup.py b/python/ray/llm/_internal/batch/observability/logging/setup.py index 0c547e4a6305..75edff664939 100644 --- a/python/ray/llm/_internal/batch/observability/logging/setup.py +++ b/python/ray/llm/_internal/batch/observability/logging/setup.py @@ -1,7 +1,7 @@ import logging -from ray._private.ray_logging.filters import CoreContextFilter -from ray._private.ray_logging.formatters import JSONFormatter +from ray._common.filters import CoreContextFilter +from ray._common.formatters import JSONFormatter def _configure_stdlib_logging(): diff --git a/python/ray/llm/_internal/common/observability/logging/__init__.py b/python/ray/llm/_internal/common/observability/logging/__init__.py index cc1e3ce04cfd..789ae4e09e9c 100644 --- a/python/ray/llm/_internal/common/observability/logging/__init__.py +++ b/python/ray/llm/_internal/common/observability/logging/__init__.py @@ -1,7 +1,7 @@ import logging from typing import Optional -from ray._private.ray_logging.filters import CoreContextFilter +from ray._common.filters import CoreContextFilter def _setup_logger(logger_name: str): diff --git a/python/ray/llm/_internal/serve/observability/logging/__init__.py b/python/ray/llm/_internal/serve/observability/logging/__init__.py index 6e684874f33e..914e2a8dce9f 100644 --- a/python/ray/llm/_internal/serve/observability/logging/__init__.py +++ b/python/ray/llm/_internal/serve/observability/logging/__init__.py @@ -1,7 +1,7 @@ import logging from typing import Optional -from ray._private.ray_logging.filters import CoreContextFilter +from ray._common.filters import CoreContextFilter from ray.serve._private.logging_utils import ServeContextFilter diff --git a/python/ray/llm/_internal/serve/observability/logging/setup.py b/python/ray/llm/_internal/serve/observability/logging/setup.py index b57f7e149484..3b1915fd2ac6 100644 --- a/python/ray/llm/_internal/serve/observability/logging/setup.py +++ b/python/ray/llm/_internal/serve/observability/logging/setup.py @@ -1,7 +1,7 @@ import logging -from ray._private.ray_logging.filters import CoreContextFilter -from ray._private.ray_logging.formatters import JSONFormatter +from ray._common.filters import CoreContextFilter +from ray._common.formatters import JSONFormatter from ray.serve._private.logging_utils import ServeContextFilter diff --git a/python/ray/serve/_private/logging_utils.py b/python/ray/serve/_private/logging_utils.py index 1e44c059d583..521b675610a2 100644 --- a/python/ray/serve/_private/logging_utils.py +++ b/python/ray/serve/_private/logging_utils.py @@ -6,9 +6,9 @@ from typing import Any, Optional import ray +from ray._common.filters import CoreContextFilter +from ray._common.formatters import JSONFormatter, TextFormatter from ray._common.ray_constants import LOGGING_ROTATE_BACKUP_COUNT, LOGGING_ROTATE_BYTES -from ray._private.ray_logging.filters import CoreContextFilter -from ray._private.ray_logging.formatters import JSONFormatter, TextFormatter from ray.serve._private.common import ServeComponentType from ray.serve._private.constants import ( RAY_SERVE_ENABLE_JSON_LOGGING, diff --git a/python/ray/serve/_private/proxy.py b/python/ray/serve/_private/proxy.py index c9d65fe4afbf..767c16ba6d50 100644 --- a/python/ray/serve/_private/proxy.py +++ b/python/ray/serve/_private/proxy.py @@ -15,8 +15,8 @@ from starlette.types import Receive import ray +from ray._common.filters import CoreContextFilter from ray._common.utils import get_or_create_event_loop -from ray._private.ray_logging.filters import CoreContextFilter from ray.serve._private.common import ( DeploymentID, EndpointInfo, diff --git a/python/ray/serve/_private/replica.py b/python/ray/serve/_private/replica.py index 4a94a326006a..c8728d973ef3 100644 --- a/python/ray/serve/_private/replica.py +++ b/python/ray/serve/_private/replica.py @@ -33,8 +33,8 @@ import ray from ray import cloudpickle +from ray._common.filters import CoreContextFilter from ray._common.utils import get_or_create_event_loop -from ray._private.ray_logging.filters import CoreContextFilter from ray.actor import ActorClass, ActorHandle from ray.remote_function import RemoteFunction from ray.serve import metrics diff --git a/python/ray/serve/tests/test_logging.py b/python/ray/serve/tests/test_logging.py index 179f99c616f3..096bbe610c96 100644 --- a/python/ray/serve/tests/test_logging.py +++ b/python/ray/serve/tests/test_logging.py @@ -22,8 +22,8 @@ import ray import ray.util.state as state_api from ray import serve +from ray._common.formatters import JSONFormatter from ray._common.test_utils import wait_for_condition -from ray._private.ray_logging.formatters import JSONFormatter from ray.serve._private.common import DeploymentID, ReplicaID, ServeComponentType from ray.serve._private.constants import SERVE_LOG_EXTRA_FIELDS, SERVE_LOGGER_NAME from ray.serve._private.logging_utils import ( diff --git a/python/ray/tests/test_logging_2.py b/python/ray/tests/test_logging_2.py index a74f5025125b..86c496c66532 100644 --- a/python/ray/tests/test_logging_2.py +++ b/python/ray/tests/test_logging_2.py @@ -1,261 +1,12 @@ -import json -import logging -import logging.config import sys import pytest import ray -from ray._private.ray_logging.filters import CoreContextFilter -from ray._private.ray_logging.formatters import JSONFormatter, TextFormatter from ray._private.ray_logging.logging_config import LoggingConfig from ray._private.test_utils import run_string_as_driver -class TestCoreContextFilter: - def test_driver_process(self, shutdown_only): - log_context = ["job_id", "worker_id", "node_id"] - filter = CoreContextFilter() - record = logging.makeLogRecord({}) - assert filter.filter(record) - # Ray is not initialized so no context except PID which should be available - for attr in log_context: - assert not hasattr(record, attr) - # PID should be available even when Ray is not initialized - assert hasattr(record, "process") - assert hasattr(record, "_ray_timestamp_ns") - - ray.init() - record = logging.makeLogRecord({}) - assert filter.filter(record) - runtime_context = ray.get_runtime_context() - expected_values = { - "job_id": runtime_context.get_job_id(), - "worker_id": runtime_context.get_worker_id(), - "node_id": runtime_context.get_node_id(), - "process": record.process, - } - for attr in log_context: - assert hasattr(record, attr) - assert getattr(record, attr) == expected_values[attr] - # This is not a worker process, so actor_id and task_id should not exist. - for attr in ["actor_id", "task_id"]: - assert not hasattr(record, attr) - assert hasattr(record, "_ray_timestamp_ns") - - def test_task_process(self, shutdown_only): - @ray.remote - def f(): - filter = CoreContextFilter() - record = logging.makeLogRecord({}) - assert filter.filter(record) - should_exist = ["job_id", "worker_id", "node_id", "task_id", "process"] - runtime_context = ray.get_runtime_context() - expected_values = { - "job_id": runtime_context.get_job_id(), - "worker_id": runtime_context.get_worker_id(), - "node_id": runtime_context.get_node_id(), - "task_id": runtime_context.get_task_id(), - "task_name": runtime_context.get_task_name(), - "task_func_name": runtime_context.get_task_function_name(), - "process": record.process, - } - for attr in should_exist: - assert hasattr(record, attr) - assert getattr(record, attr) == expected_values[attr] - assert not hasattr(record, "actor_id") - assert not hasattr(record, "actor_name") - assert hasattr(record, "_ray_timestamp_ns") - - obj_ref = f.remote() - ray.get(obj_ref) - - def test_actor_process(self, shutdown_only): - @ray.remote - class A: - def f(self): - filter = CoreContextFilter() - record = logging.makeLogRecord({}) - assert filter.filter(record) - should_exist = [ - "job_id", - "worker_id", - "node_id", - "actor_id", - "task_id", - "process", - ] - runtime_context = ray.get_runtime_context() - expected_values = { - "job_id": runtime_context.get_job_id(), - "worker_id": runtime_context.get_worker_id(), - "node_id": runtime_context.get_node_id(), - "actor_id": runtime_context.get_actor_id(), - "actor_name": runtime_context.get_actor_name(), - "task_id": runtime_context.get_task_id(), - "task_name": runtime_context.get_task_name(), - "task_func_name": runtime_context.get_task_function_name(), - "process": record.process, - } - for attr in should_exist: - assert hasattr(record, attr) - assert getattr(record, attr) == expected_values[attr] - assert hasattr(record, "_ray_timestamp_ns") - - actor = A.remote() - ray.get(actor.f.remote()) - - -class TestJSONFormatter: - def test_empty_record(self, shutdown_only): - formatter = JSONFormatter() - record = logging.makeLogRecord({}) - formatted = formatter.format(record) - - record_dict = json.loads(formatted) - should_exist = [ - "process", - "asctime", - "levelname", - "message", - "filename", - "lineno", - "timestamp_ns", - ] - for key in should_exist: - assert key in record_dict - assert len(record_dict) == len(should_exist) - assert "exc_text" not in record_dict - - def test_record_with_exception(self, shutdown_only): - formatter = JSONFormatter() - record = logging.makeLogRecord({}) - try: - raise ValueError("test") - except ValueError: - record.exc_info = sys.exc_info() - formatted = formatter.format(record) - record_dict = json.loads(formatted) - should_exist = [ - "process", - "asctime", - "levelname", - "message", - "filename", - "lineno", - "exc_text", - "timestamp_ns", - ] - for key in should_exist: - assert key in record_dict - assert "Traceback (most recent call last):" in record_dict["exc_text"] - assert len(record_dict) == len(should_exist) - - def test_record_with_user_provided_context(self, shutdown_only): - formatter = JSONFormatter() - record = logging.makeLogRecord({"user": "ray"}) - formatted = formatter.format(record) - record_dict = json.loads(formatted) - should_exist = [ - "process", - "asctime", - "levelname", - "message", - "filename", - "lineno", - "user", - "timestamp_ns", - ] - for key in should_exist: - assert key in record_dict - assert record_dict["user"] == "ray" - assert len(record_dict) == len(should_exist) - assert "exc_text" not in record_dict - - def test_record_with_flatten_keys_invalid_value(self, shutdown_only): - formatter = JSONFormatter() - record = logging.makeLogRecord({"ray_serve_extra_fields": "not_a_dict"}) - with pytest.raises(ValueError): - formatter.format(record) - - def test_record_with_flatten_keys_valid_dict(self, shutdown_only): - formatter = JSONFormatter() - record = logging.makeLogRecord( - {"ray_serve_extra_fields": {"key1": "value1", "key2": 2}} - ) - formatted = formatter.format(record) - record_dict = json.loads(formatted) - should_exist = [ - "process", - "asctime", - "levelname", - "message", - "filename", - "lineno", - "key1", - "key2", - "timestamp_ns", - ] - for key in should_exist: - assert key in record_dict - assert record_dict["key1"] == "value1", record_dict - assert record_dict["key2"] == 2 - assert "ray_serve_extra_fields" not in record_dict - assert len(record_dict) == len(should_exist) - assert "exc_text" not in record_dict - - def test_record_with_valid_additional_log_standard_attrs(self, shutdown_only): - formatter = JSONFormatter() - formatter.set_additional_log_standard_attrs(["name"]) - record = logging.makeLogRecord({}) - formatted = formatter.format(record) - - record_dict = json.loads(formatted) - should_exist = [ - "process", - "asctime", - "levelname", - "message", - "filename", - "lineno", - "timestamp_ns", - "name", - ] - for key in should_exist: - assert key in record_dict - assert len(record_dict) == len(should_exist) - - -class TestTextFormatter: - def test_record_with_user_provided_context(self): - formatter = TextFormatter() - record = logging.makeLogRecord({"user": "ray"}) - formatted = formatter.format(record) - assert "user=ray" in formatted - - def test_record_with_exception(self): - formatter = TextFormatter() - record = logging.LogRecord( - name="test_logger", - level=logging.INFO, - pathname="test.py", - lineno=1000, - msg="Test message", - args=None, - exc_info=None, - ) - formatted = formatter.format(record) - for s in ["INFO", "Test message", "test.py:1000", "--"]: - assert s in formatted - - def test_record_with_valid_additional_log_standard_attrs(self, shutdown_only): - formatter = TextFormatter() - formatter.set_additional_log_standard_attrs(["name"]) - record = logging.makeLogRecord({}) - formatted = formatter.format(record) - assert "name=" in formatted - - def test_invalid_encoding(): with pytest.raises(ValueError): LoggingConfig(encoding="INVALID") diff --git a/python/ray/train/v2/_internal/logging/logging.py b/python/ray/train/v2/_internal/logging/logging.py index 053dd52e63fe..8fb645df3b5e 100644 --- a/python/ray/train/v2/_internal/logging/logging.py +++ b/python/ray/train/v2/_internal/logging/logging.py @@ -4,9 +4,9 @@ from typing import Optional, Union import ray +from ray._common.filters import CoreContextFilter +from ray._common.formatters import JSONFormatter from ray._private.log import PlainRayHandler -from ray._private.ray_logging.filters import CoreContextFilter -from ray._private.ray_logging.formatters import JSONFormatter from ray.train.v2._internal.execution.context import TrainContext, TrainRunContext from ray.train.v2._internal.util import get_module_name From 274a9ddd7c4ed319dc0625fe7d2978f2f161bf2b Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Wed, 10 Sep 2025 16:04:54 -0700 Subject: [PATCH 566/634] [core][1eventx/02] job event: add an interface for ray event recorder (#55065) This is part of a series of PRs to support JobEvent in the oneevent framework. The full effort will include adding the JobEvent schema, introducing a generic interface for exporting different types of events to the Event Aggregator, and implementing the necessary integration logic. ---- In this PR, we implement: - A base class for RayEvent. This base class implements common logic for `merging` and `serialize` into an proto object. Its implementation includes DriverJobDefinition and DriverJobExecution. - See DriverJobExecution as an example for what type of merging we want to perform - RayEventRecorder serves as both (i) a buffer of RayEvent, and (ii) grpc client to send these events to the EventAggregator (component of a DashboardAgent) Test: - CI --------- Signed-off-by: Cuong Nguyen --- src/ray/common/ray_config_def.h | 5 + src/ray/observability/BUILD.bazel | 82 ++++++++++++ .../ray_driver_job_definition_event.cc | 67 ++++++++++ .../ray_driver_job_definition_event.h | 38 ++++++ .../ray_driver_job_execution_event.cc | 56 ++++++++ .../ray_driver_job_execution_event.h | 39 ++++++ src/ray/observability/ray_event.h | 81 ++++++++++++ src/ray/observability/ray_event_interface.h | 64 ++++++++++ src/ray/observability/ray_event_recorder.cc | 74 +++++++++++ src/ray/observability/ray_event_recorder.h | 69 ++++++++++ .../ray_event_recorder_interface.h | 40 ++++++ src/ray/observability/tests/BUILD.bazel | 24 ++++ .../ray_driver_job_execution_event_test.cc | 41 ++++++ .../tests/ray_event_recorder_test.cc | 120 ++++++++++++++++++ 14 files changed, 800 insertions(+) create mode 100644 src/ray/observability/ray_driver_job_definition_event.cc create mode 100644 src/ray/observability/ray_driver_job_definition_event.h create mode 100644 src/ray/observability/ray_driver_job_execution_event.cc create mode 100644 src/ray/observability/ray_driver_job_execution_event.h create mode 100644 src/ray/observability/ray_event.h create mode 100644 src/ray/observability/ray_event_interface.h create mode 100644 src/ray/observability/ray_event_recorder.cc create mode 100644 src/ray/observability/ray_event_recorder.h create mode 100644 src/ray/observability/ray_event_recorder_interface.h create mode 100644 src/ray/observability/tests/ray_driver_job_execution_event_test.cc create mode 100644 src/ray/observability/tests/ray_event_recorder_test.cc diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index 14a7f9de4d5f..b9c2e7e6b979 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -458,6 +458,11 @@ RAY_CONFIG(bool, task_events_skip_driver_for_test, false) /// Setting the value to 0 disables the task event recording and reporting. RAY_CONFIG(int64_t, task_events_report_interval_ms, 1000) +/// The interval duration for which ray events will be reported to the event aggregator. +/// The reported data should only be used for observability. +/// Setting the value to 0 disables the ray event recording and reporting. +RAY_CONFIG(int64_t, ray_events_report_interval_ms, 1000) + /// The number of tasks tracked in GCS for task state events. Any additional events /// from new tasks will evict events of tasks reported earlier. /// Setting the value to -1 allows for unlimited task events stored in GCS. diff --git a/src/ray/observability/BUILD.bazel b/src/ray/observability/BUILD.bazel index ebdfb1aacaed..4da0cd2a5c83 100644 --- a/src/ray/observability/BUILD.bazel +++ b/src/ray/observability/BUILD.bazel @@ -34,3 +34,85 @@ ray_cc_library( ":metric_interface", ], ) + +ray_cc_library( + name = "ray_event_interface", + hdrs = [ + "ray_event_interface.h", + ], + deps = [ + "//src/ray/protobuf/public:events_base_event_cc_proto", + ], +) + +ray_cc_library( + name = "ray_event", + hdrs = [ + "ray_event.h", + ], + deps = [ + ":ray_event_interface", + "//src/ray/common:grpc_util", + "//src/ray/common:id", + "//src/ray/protobuf:gcs_cc_proto", + "@com_google_absl//absl/time", + ], +) + +ray_cc_library( + name = "ray_driver_job_definition_event", + srcs = [ + "ray_driver_job_definition_event.cc", + ], + hdrs = [ + "ray_driver_job_definition_event.h", + ], + deps = [ + ":ray_event", + "//src/ray/protobuf/public:events_driver_job_definition_event_cc_proto", + ], +) + +ray_cc_library( + name = "ray_driver_job_execution_event", + srcs = [ + "ray_driver_job_execution_event.cc", + ], + hdrs = [ + "ray_driver_job_execution_event.h", + ], + deps = [ + ":ray_event", + "//src/ray/protobuf/public:events_driver_job_execution_event_cc_proto", + ], +) + +ray_cc_library( + name = "ray_event_recorder_interface", + hdrs = [ + "ray_event_recorder_interface.h", + ], + deps = [ + ":ray_event", + ], +) + +ray_cc_library( + name = "ray_event_recorder", + srcs = [ + "ray_event_recorder.cc", + ], + hdrs = [ + "ray_event_recorder.h", + ], + deps = [ + ":ray_event", + ":ray_event_recorder_interface", + "//src/ray/common:asio", + "//src/ray/protobuf:events_event_aggregator_service_cc_proto", + "//src/ray/protobuf:gcs_cc_proto", + "//src/ray/rpc:event_aggregator_client", + "//src/ray/util:logging", + "@com_google_absl//absl/time", + ], +) diff --git a/src/ray/observability/ray_driver_job_definition_event.cc b/src/ray/observability/ray_driver_job_definition_event.cc new file mode 100644 index 000000000000..11bfd03730a5 --- /dev/null +++ b/src/ray/observability/ray_driver_job_definition_event.cc @@ -0,0 +1,67 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/observability/ray_driver_job_definition_event.h" + +namespace ray { +namespace observability { + +RayDriverJobDefinitionEvent::RayDriverJobDefinitionEvent(const rpc::JobTableData &data, + const std::string &session_name) + : RayEvent( + rpc::events::RayEvent::GCS, + rpc::events::RayEvent::DRIVER_JOB_DEFINITION_EVENT, + rpc::events::RayEvent::INFO, + "", + session_name) { + data_.set_job_id(data.job_id()); + data_.set_driver_pid(data.driver_pid()); + data_.set_driver_node_id(data.driver_address().node_id()); + data_.set_entrypoint(data.entrypoint()); + data_.mutable_config()->mutable_metadata()->insert(data.config().metadata().begin(), + data.config().metadata().end()); + + auto runtime_env_info = data_.mutable_config()->mutable_runtime_env_info(); + runtime_env_info->set_serialized_runtime_env( + data.config().runtime_env_info().serialized_runtime_env()); + auto runtime_env_uris = runtime_env_info->mutable_uris(); + runtime_env_uris->set_working_dir_uri( + data.config().runtime_env_info().uris().working_dir_uri()); + runtime_env_uris->mutable_py_modules_uris()->CopyFrom( + data.config().runtime_env_info().uris().py_modules_uris()); + auto runtime_env_config = runtime_env_info->mutable_runtime_env_config(); + runtime_env_config->set_setup_timeout_seconds( + data.config().runtime_env_info().runtime_env_config().setup_timeout_seconds()); + runtime_env_config->set_eager_install( + data.config().runtime_env_info().runtime_env_config().eager_install()); + runtime_env_config->mutable_log_files()->CopyFrom( + data.config().runtime_env_info().runtime_env_config().log_files()); +} + +std::string RayDriverJobDefinitionEvent::GetEntityId() const { return data_.job_id(); } + +void RayDriverJobDefinitionEvent::MergeData( + RayEvent &&other) { + RAY_LOG(WARNING) << "Merge should not be called for driver job definition event."; + return; +} + +ray::rpc::events::RayEvent RayDriverJobDefinitionEvent::SerializeData() && { + ray::rpc::events::RayEvent event; + event.mutable_driver_job_definition_event()->Swap(&data_); + return event; +} + +} // namespace observability +} // namespace ray diff --git a/src/ray/observability/ray_driver_job_definition_event.h b/src/ray/observability/ray_driver_job_definition_event.h new file mode 100644 index 000000000000..6ff80ba48c54 --- /dev/null +++ b/src/ray/observability/ray_driver_job_definition_event.h @@ -0,0 +1,38 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/observability/ray_event.h" +#include "src/ray/protobuf/gcs.pb.h" +#include "src/ray/protobuf/public/events_driver_job_definition_event.pb.h" + +namespace ray { +namespace observability { + +template class RayEvent; + +class RayDriverJobDefinitionEvent + : public RayEvent { + public: + RayDriverJobDefinitionEvent(const rpc::JobTableData &data, + const std::string &session_name); + + std::string GetEntityId() const override; + + protected: + ray::rpc::events::RayEvent SerializeData() && override; + void MergeData(RayEvent &&other) override; +}; + +} // namespace observability +} // namespace ray diff --git a/src/ray/observability/ray_driver_job_execution_event.cc b/src/ray/observability/ray_driver_job_execution_event.cc new file mode 100644 index 000000000000..fba7b274499a --- /dev/null +++ b/src/ray/observability/ray_driver_job_execution_event.cc @@ -0,0 +1,56 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/observability/ray_driver_job_execution_event.h" + +namespace ray { +namespace observability { + +RayDriverJobExecutionEvent::RayDriverJobExecutionEvent( + const rpc::JobTableData &data, + rpc::events::DriverJobExecutionEvent::State state, + const std::string &session_name) + : RayEvent( + rpc::events::RayEvent::GCS, + rpc::events::RayEvent::DRIVER_JOB_EXECUTION_EVENT, + rpc::events::RayEvent::INFO, + "", + session_name) { + ray::rpc::events::DriverJobExecutionEvent::StateTimestamp state_timestamp; + state_timestamp.set_state(state); + state_timestamp.mutable_timestamp()->CopyFrom(AbslTimeNanosToProtoTimestamp( + absl::ToInt64Nanoseconds(absl::Now() - absl::UnixEpoch()))); + + data_.mutable_states()->Add(std::move(state_timestamp)); + data_.set_job_id(data.job_id()); +} + +std::string RayDriverJobExecutionEvent::GetEntityId() const { return data_.job_id(); } + +void RayDriverJobExecutionEvent::MergeData( + RayEvent &&other) { + auto &&other_event = static_cast(other); + for (auto &state : *other_event.data_.mutable_states()) { + data_.mutable_states()->Add(std::move(state)); + } +} + +ray::rpc::events::RayEvent RayDriverJobExecutionEvent::SerializeData() && { + ray::rpc::events::RayEvent event; + event.mutable_driver_job_execution_event()->Swap(&data_); + return event; +} + +} // namespace observability +} // namespace ray diff --git a/src/ray/observability/ray_driver_job_execution_event.h b/src/ray/observability/ray_driver_job_execution_event.h new file mode 100644 index 000000000000..fd3d34ffe078 --- /dev/null +++ b/src/ray/observability/ray_driver_job_execution_event.h @@ -0,0 +1,39 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/common/grpc_util.h" +#include "ray/observability/ray_event.h" +#include "src/ray/protobuf/gcs.pb.h" +#include "src/ray/protobuf/public/events_driver_job_execution_event.pb.h" + +namespace ray { +namespace observability { + +template class RayEvent; + +class RayDriverJobExecutionEvent : public RayEvent { + public: + RayDriverJobExecutionEvent(const rpc::JobTableData &data, + rpc::events::DriverJobExecutionEvent::State state, + const std::string &session_name); + + std::string GetEntityId() const override; + + protected: + ray::rpc::events::RayEvent SerializeData() && override; + void MergeData(RayEvent &&other) override; +}; + +} // namespace observability +} // namespace ray diff --git a/src/ray/observability/ray_event.h b/src/ray/observability/ray_event.h new file mode 100644 index 000000000000..32711740723b --- /dev/null +++ b/src/ray/observability/ray_event.h @@ -0,0 +1,81 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "absl/time/time.h" +#include "ray/common/grpc_util.h" +#include "ray/common/id.h" +#include "ray/observability/ray_event_interface.h" +#include "src/ray/protobuf/public/events_base_event.pb.h" + +namespace ray { +namespace observability { + +// RayEvent is a base class for all Ray events. It is used to serialize the event data +// to a RayEvent proto before sending it to the aggregator. +template +class RayEvent : public RayEventInterface { + public: + void Merge(RayEventInterface &&other) override { + RAY_CHECK_EQ(GetEntityId(), other.GetEntityId()); + RAY_CHECK_EQ(GetEventType(), other.GetEventType()); + MergeData(static_cast &&>(other)); + } + + ray::rpc::events::RayEvent Serialize() && override { + ray::rpc::events::RayEvent event = std::move(*this).SerializeData(); + event.set_event_id(UniqueID::FromRandom().Binary()); + event.set_source_type(source_type_); + event.set_event_type(event_type_); + event.set_severity(severity_); + event.set_message(message_); + event.set_session_name(session_name_); + event.mutable_timestamp()->CopyFrom(AbslTimeNanosToProtoTimestamp( + absl::ToInt64Nanoseconds(event_timestamp_ - absl::UnixEpoch()))); + + return event; + } + + ray::rpc::events::RayEvent::EventType GetEventType() const override { + return event_type_; + } + + protected: + RayEvent(ray::rpc::events::RayEvent::SourceType source_type, + ray::rpc::events::RayEvent::EventType event_type, + ray::rpc::events::RayEvent::Severity severity, + const std::string &message, + const std::string &session_name) + : source_type_(source_type), + event_type_(event_type), + severity_(severity), + message_(message), + session_name_(session_name) { + event_timestamp_ = absl::Now(); + } + + T data_; // The nested event message within the RayEvent proto. + absl::Time event_timestamp_; + ray::rpc::events::RayEvent::SourceType source_type_; + ray::rpc::events::RayEvent::EventType event_type_; + ray::rpc::events::RayEvent::Severity severity_; + std::string message_; + std::string session_name_; + virtual void MergeData(RayEvent &&other) = 0; + virtual ray::rpc::events::RayEvent SerializeData() && = 0; +}; + +} // namespace observability +} // namespace ray diff --git a/src/ray/observability/ray_event_interface.h b/src/ray/observability/ray_event_interface.h new file mode 100644 index 000000000000..fc2a358ef311 --- /dev/null +++ b/src/ray/observability/ray_event_interface.h @@ -0,0 +1,64 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "src/ray/protobuf/public/events_base_event.pb.h" + +namespace ray { +namespace observability { + +class RayEventInterface { + public: + virtual ~RayEventInterface() = default; + + // Entity ID is a concept in Ray Event framework that captures the unique identifier + // of the entity that the event is associated with. For example, the entity ID of + // a task is the pair of task ID and task attempt ID, for a driver job, it is the + // driver job ID. + // + // Entity ID is used for two purposes: + // 1. To associate the execution event with the definition event. + // 2. To merge the individual execution events into a single execution event (single + // data point to a time series). + virtual std::string GetEntityId() const = 0; + + // Merge with another data point to form a time series. Merge is meant as an + // optimization for the data size. + // + // For example, given three events: + // + // 1. event 1: {entity_id: "1", type: "task", state_transitions: [("started", 1000)]} + // 2. event 2: {entity_id: "1", type: "task", state_transitions: [("running", 1001)]} + // 3. event 3: {entity_id: "1", type: "task", state_transitions: [("completed", 1002)]} + // + // The merged event will be: + // + // {entity_id: "1", type: "task", state_transitions: [("started", 1000), ("running", + // 1001), + // ("completed", 1002)]} + // + // This function assumes that the two events have the same type and entity ID. + virtual void Merge(RayEventInterface &&other) = 0; + + // Serialize the event data to a RayEvent proto. + virtual ray::rpc::events::RayEvent Serialize() && = 0; + + virtual ray::rpc::events::RayEvent::EventType GetEventType() const = 0; +}; + +} // namespace observability +} // namespace ray diff --git a/src/ray/observability/ray_event_recorder.cc b/src/ray/observability/ray_event_recorder.cc new file mode 100644 index 000000000000..0d214a445a87 --- /dev/null +++ b/src/ray/observability/ray_event_recorder.cc @@ -0,0 +1,74 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/observability/ray_event_recorder.h" + +#include "src/ray/protobuf/gcs.pb.h" + +namespace ray { +namespace observability { + +RayEventRecorder::RayEventRecorder(rpc::EventAggregatorClient &event_aggregator_client, + instrumented_io_context &io_service) + : event_aggregator_client_(event_aggregator_client), + periodical_runner_(PeriodicalRunner::Create(io_service)) {} + +void RayEventRecorder::StartExportingEvents() { + absl::MutexLock lock(&mutex_); + RAY_CHECK(!exporting_started_) + << "RayEventRecorder::StartExportingEvents() should be called only once."; + exporting_started_ = true; + periodical_runner_->RunFnPeriodically( + [this]() { ExportEvents(); }, + RayConfig::instance().ray_events_report_interval_ms(), + "RayEventRecorder.ExportEvents"); +} + +void RayEventRecorder::ExportEvents() { + absl::MutexLock lock(&mutex_); + if (buffer_.empty()) { + return; + } + rpc::events::AddEventsRequest request; + rpc::events::RayEventsData ray_event_data; + // TODO(#56391): To further optimize the performance, we can merge multiple + // events with the same resource ID into a single event. + for (auto &event : buffer_) { + rpc::events::RayEvent ray_event = std::move(*event).Serialize(); + *ray_event_data.mutable_events()->Add() = std::move(ray_event); + } + *request.mutable_events_data() = std::move(ray_event_data); + buffer_.clear(); + + event_aggregator_client_.AddEvents( + request, [](Status status, rpc::events::AddEventsReply reply) { + if (!status.ok()) { + // TODO(#56391): Add a metric to track the number of failed events. Also + // add logic for error recovery. + RAY_LOG(ERROR) << "Failed to record ray event: " << status.ToString(); + } + }); +} + +void RayEventRecorder::AddEvents( + std::vector> &&data_list) { + absl::MutexLock lock(&mutex_); + buffer_.reserve(buffer_.size() + data_list.size()); + for (auto &data : data_list) { + buffer_.emplace_back(std::move(data)); + } +} + +} // namespace observability +} // namespace ray diff --git a/src/ray/observability/ray_event_recorder.h b/src/ray/observability/ray_event_recorder.h new file mode 100644 index 000000000000..2650f99378c8 --- /dev/null +++ b/src/ray/observability/ray_event_recorder.h @@ -0,0 +1,69 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "absl/synchronization/mutex.h" +#include "absl/time/time.h" +#include "google/protobuf/timestamp.pb.h" +#include "ray/common/asio/periodical_runner.h" +#include "ray/common/ray_config.h" +#include "ray/observability/ray_event_interface.h" +#include "ray/observability/ray_event_recorder_interface.h" +#include "ray/rpc/event_aggregator_client.h" +#include "ray/util/logging.h" +#include "src/ray/protobuf/public/events_base_event.pb.h" + +namespace ray { +namespace observability { + +// RayEventRecorder is a class for recording different types of Ray +// events (e.g. task events, job events, etc.). Internal buffer is used to store events +// before sending to the event aggregator. Events are converted to RayEvent proto and +// added to the internal buffer. PeriodicalRunner is used to send events to the event +// aggregator periodically. +// +// This class is thread safe. +class RayEventRecorder : public RayEventRecorderInterface { + public: + RayEventRecorder(rpc::EventAggregatorClient &event_aggregator_client, + instrumented_io_context &io_service); + virtual ~RayEventRecorder() = default; + + // Start exporting events to the event aggregator by periodically sending events to + // the event aggregator. This should be called only once. Subsequent calls will be + // ignored. + void StartExportingEvents(); + + // Add a vector of data to the internal buffer. Data in the buffer will be sent to + // the event aggregator periodically. + void AddEvents(std::vector> &&data_list); + + private: + rpc::EventAggregatorClient &event_aggregator_client_; + std::shared_ptr periodical_runner_; + // Lock for thread safety when modifying the buffer. + absl::Mutex mutex_; + // Buffer to store events before sending to the event aggregator. + // TODO(#56391): Add a max size for the buffer and overflow recovery logic. + std::vector> buffer_ ABSL_GUARDED_BY(mutex_); + // Flag to track if exporting has been started + bool exporting_started_ ABSL_GUARDED_BY(mutex_) = false; + // Export events to the event aggregator. This is called periodically by the + // PeriodicalRunner. + void ExportEvents(); +}; + +} // namespace observability +} // namespace ray diff --git a/src/ray/observability/ray_event_recorder_interface.h b/src/ray/observability/ray_event_recorder_interface.h new file mode 100644 index 000000000000..f6e80e38eae2 --- /dev/null +++ b/src/ray/observability/ray_event_recorder_interface.h @@ -0,0 +1,40 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "ray/observability/ray_event_interface.h" + +namespace ray { +namespace observability { + +class RayEventRecorderInterface { + public: + virtual ~RayEventRecorderInterface() = default; + + // Start exporting events to the event aggregator by periodically sending events to + // the event aggregator. This should be called only once. Subsequent calls will be + // ignored. + virtual void StartExportingEvents() = 0; + + // Add a vector of data to the internal buffer. Data in the buffer will be sent to + // the event aggregator periodically. + virtual void AddEvents(std::vector> &&data_list) = 0; +}; + +} // namespace observability +} // namespace ray diff --git a/src/ray/observability/tests/BUILD.bazel b/src/ray/observability/tests/BUILD.bazel index e150e80aab35..db97a32197fa 100644 --- a/src/ray/observability/tests/BUILD.bazel +++ b/src/ray/observability/tests/BUILD.bazel @@ -10,3 +10,27 @@ ray_cc_test( "@com_google_googletest//:gtest_main", ], ) + +ray_cc_test( + name = "ray_event_recorder_test", + size = "small", + srcs = ["ray_event_recorder_test.cc"], + tags = ["team:core"], + deps = [ + "//src/ray/observability:ray_driver_job_definition_event", + "//src/ray/observability:ray_driver_job_execution_event", + "//src/ray/observability:ray_event_recorder", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "ray_driver_job_execution_event_test", + size = "small", + srcs = ["ray_driver_job_execution_event_test.cc"], + tags = ["team:core"], + deps = [ + "//src/ray/observability:ray_driver_job_execution_event", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/observability/tests/ray_driver_job_execution_event_test.cc b/src/ray/observability/tests/ray_driver_job_execution_event_test.cc new file mode 100644 index 000000000000..d6a2b1ad4d4b --- /dev/null +++ b/src/ray/observability/tests/ray_driver_job_execution_event_test.cc @@ -0,0 +1,41 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/observability/ray_driver_job_execution_event.h" + +#include "gtest/gtest.h" + +namespace ray { +namespace observability { + +class RayDriverJobExecutionEventTest : public ::testing::Test {}; + +TEST_F(RayDriverJobExecutionEventTest, TestMerge) { + rpc::JobTableData data; + data.set_job_id("test_job_id_1"); + auto event1 = std::make_unique( + data, rpc::events::DriverJobExecutionEvent::SUCCESS, "test_session_name_1"); + auto event2 = std::make_unique( + data, rpc::events::DriverJobExecutionEvent::FAILURE, "test_session_name_1"); + event1->Merge(std::move(*event2)); + auto serialized_event = std::move(*event1).Serialize(); + ASSERT_EQ(serialized_event.driver_job_execution_event().states_size(), 2); + ASSERT_EQ(serialized_event.driver_job_execution_event().states(0).state(), + rpc::events::DriverJobExecutionEvent::SUCCESS); + ASSERT_EQ(serialized_event.driver_job_execution_event().states(1).state(), + rpc::events::DriverJobExecutionEvent::FAILURE); +} + +} // namespace observability +} // namespace ray diff --git a/src/ray/observability/tests/ray_event_recorder_test.cc b/src/ray/observability/tests/ray_event_recorder_test.cc new file mode 100644 index 000000000000..4f825b3ee95a --- /dev/null +++ b/src/ray/observability/tests/ray_event_recorder_test.cc @@ -0,0 +1,120 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/observability/ray_event_recorder.h" + +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/ray_config.h" +#include "ray/observability/ray_driver_job_definition_event.h" +#include "ray/observability/ray_driver_job_execution_event.h" +#include "src/ray/protobuf/gcs.pb.h" +#include "src/ray/protobuf/public/events_base_event.pb.h" +#include "src/ray/protobuf/public/events_driver_job_execution_event.pb.h" + +namespace ray { +namespace observability { + +class FakeEventAggregatorClient : public rpc::EventAggregatorClient { + public: + FakeEventAggregatorClient() {} + + void AddEvents( + const rpc::events::AddEventsRequest &request, + const rpc::ClientCallback &callback) override { + absl::MutexLock lock(&mutex_); + for (const auto &event : request.events_data().events()) { + recorded_events_.push_back(event); + } + callback(Status::OK(), rpc::events::AddEventsReply{}); + } + + std::vector GetRecordedEvents() { + absl::MutexLock lock(&mutex_); + return recorded_events_; + } + + private: + std::vector recorded_events_ ABSL_GUARDED_BY(mutex_); + absl::Mutex mutex_; +}; + +class RayEventRecorderTest : public ::testing::Test { + public: + RayEventRecorderTest() { + fake_client_ = std::make_unique(); + recorder_ = std::make_unique(*fake_client_, io_service_); + recorder_->StartExportingEvents(); + } + + instrumented_io_context io_service_; + std::unique_ptr fake_client_; + std::unique_ptr recorder_; +}; + +TEST_F(RayEventRecorderTest, TestRecordEvents) { + rpc::JobTableData data1; + data1.set_job_id("test_job_id_1"); + data1.set_is_dead(false); + data1.set_driver_pid(12345); + data1.set_start_time(absl::ToUnixSeconds(absl::Now())); + data1.set_end_time(0); + data1.set_entrypoint("python test_script.py"); + data1.mutable_driver_address()->set_ip_address("127.0.0.1"); + + rpc::JobTableData data2; + data2.set_job_id("test_job_id_2"); + data2.set_is_dead(true); + data2.set_driver_pid(67890); + data2.set_start_time(absl::ToUnixSeconds(absl::Now()) - 3600); // 1 hour ago + data2.set_end_time(absl::ToUnixSeconds(absl::Now())); + data2.set_entrypoint("python another_script.py"); + data2.mutable_driver_address()->set_ip_address("192.168.1.100"); + + std::vector> events; + events.push_back( + std::make_unique(data1, "test_session_name_1")); + events.push_back(std::make_unique( + data2, rpc::events::DriverJobExecutionEvent::SUCCESS, "test_session_name_2")); + recorder_->AddEvents(std::move(events)); + io_service_.run_one(); + + std::vector recorded_events = fake_client_->GetRecordedEvents(); + // Verify first event + ASSERT_EQ(recorded_events.size(), 2); + ASSERT_EQ(recorded_events[0].source_type(), rpc::events::RayEvent::GCS); + ASSERT_EQ(recorded_events[0].session_name(), "test_session_name_1"); + ASSERT_EQ(recorded_events[0].event_type(), + rpc::events::RayEvent::DRIVER_JOB_DEFINITION_EVENT); + ASSERT_EQ(recorded_events[0].severity(), rpc::events::RayEvent::INFO); + ASSERT_TRUE(recorded_events[0].has_driver_job_definition_event()); + ASSERT_EQ(recorded_events[0].driver_job_definition_event().job_id(), "test_job_id_1"); + + // Verify second event + ASSERT_EQ(recorded_events[1].source_type(), rpc::events::RayEvent::GCS); + ASSERT_EQ(recorded_events[1].session_name(), "test_session_name_2"); + ASSERT_EQ(recorded_events[1].event_type(), + rpc::events::RayEvent::DRIVER_JOB_EXECUTION_EVENT); + ASSERT_EQ(recorded_events[1].severity(), rpc::events::RayEvent::INFO); + ASSERT_TRUE(recorded_events[1].has_driver_job_execution_event()); + ASSERT_EQ(recorded_events[1].driver_job_execution_event().job_id(), "test_job_id_2"); +} + +} // namespace observability +} // namespace ray From c1251d9224f967b2ee2ace25080ad50003932ce8 Mon Sep 17 00:00:00 2001 From: Sampan S Nayak Date: Thu, 11 Sep 2025 04:55:53 +0530 Subject: [PATCH 567/634] [core] [actor-event-01] Actor event: add proto schema (#56221) Add proto schema for actor events. This contains all the field and not more from the existing https://github.com/ray-project/ray/blob/master/src/ray/protobuf/export_actor_data.proto. It splits by the static vs dynamic state transition information, similar to other one event schema designs. Test: - CI --------- Signed-off-by: sampan Co-authored-by: sampan --- .../modules/aggregator/aggregator_agent.py | 3 +- src/ray/protobuf/public/BUILD.bazel | 26 +++++++++ .../events_actor_definition_event.proto | 43 +++++++++++++++ .../public/events_actor_lifecycle_event.proto | 54 +++++++++++++++++++ .../protobuf/public/events_base_event.proto | 6 +++ 5 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 src/ray/protobuf/public/events_actor_definition_event.proto create mode 100644 src/ray/protobuf/public/events_actor_lifecycle_event.proto diff --git a/python/ray/dashboard/modules/aggregator/aggregator_agent.py b/python/ray/dashboard/modules/aggregator/aggregator_agent.py index 584a6f6375ec..c5dbde841470 100644 --- a/python/ray/dashboard/modules/aggregator/aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/aggregator_agent.py @@ -81,7 +81,8 @@ DEFAULT_EXPOSABLE_EVENT_TYPES = ( "TASK_DEFINITION_EVENT,TASK_EXECUTION_EVENT," "ACTOR_TASK_DEFINITION_EVENT,ACTOR_TASK_EXECUTION_EVENT," - "DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_EXECUTION_EVENT" + "DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_EXECUTION_EVENT," + "ACTOR_DEFINITION_EVENT,ACTOR_LIFECYCLE_EVENT" ) EXPOSABLE_EVENT_TYPES = os.environ.get( f"{env_var_prefix}_EXPOSABLE_EVENT_TYPES", DEFAULT_EXPOSABLE_EVENT_TYPES diff --git a/src/ray/protobuf/public/BUILD.bazel b/src/ray/protobuf/public/BUILD.bazel index 55f0364bb421..25a484225caf 100644 --- a/src/ray/protobuf/public/BUILD.bazel +++ b/src/ray/protobuf/public/BUILD.bazel @@ -7,6 +7,8 @@ proto_library( name = "events_base_event_proto", srcs = ["events_base_event.proto"], deps = [ + ":events_actor_definition_event_proto", + ":events_actor_lifecycle_event_proto", ":events_actor_task_definition_event_proto", ":events_driver_job_definition_event_proto", ":events_driver_job_execution_event_proto", @@ -94,6 +96,25 @@ cc_proto_library( deps = [":events_driver_job_execution_event_proto"], ) +proto_library( + name = "events_actor_definition_event_proto", + srcs = ["events_actor_definition_event.proto"], +) + +cc_proto_library( + name = "events_actor_definition_event_cc_proto", + deps = [":events_actor_definition_event_proto"], +) + +proto_library( + name = "events_actor_lifecycle_event_proto", + srcs = ["events_actor_lifecycle_event.proto"], + deps = [ + "//src/ray/protobuf:common_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + proto_library( name = "events_node_definition_event_proto", srcs = ["events_node_definition_event.proto"], @@ -112,6 +133,11 @@ proto_library( ], ) +cc_proto_library( + name = "events_actor_lifecycle_event_cc_proto", + deps = [":events_actor_lifecycle_event_proto"], +) + cc_proto_library( name = "events_node_lifecycle_event_cc_proto", deps = [":events_node_lifecycle_event_proto"], diff --git a/src/ray/protobuf/public/events_actor_definition_event.proto b/src/ray/protobuf/public/events_actor_definition_event.proto new file mode 100644 index 000000000000..63ce89045cca --- /dev/null +++ b/src/ray/protobuf/public/events_actor_definition_event.proto @@ -0,0 +1,43 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ray.rpc.events; + +message ActorDefinitionEvent { + // The ID of the actor that was created. + bytes actor_id = 1; + // The ID of the job that created the actor. + bytes job_id = 2; + // Whether the actor is persistent. + bool is_detached = 3; + // Name of the actor. + string name = 4; + // The actor's namespace. Named `ray_namespace` to avoid conflicting with c++ keyword. + string ray_namespace = 5; + // Serialized runtime_env used to report in the dashboard snapshot. We need to populate + // it here instead of grabbing it from the task spec because the task spec is cleared + // for deleted actors: https://github.com/ray-project/ray/pull/11149. + string serialized_runtime_env = 6; + // The actor's class name. This is necessary because the task spec's lifetime + // is shorter than the ActorTableData. + string class_name = 7; + // Quantities of the different resources required by this actor. + map required_resources = 8; + // Placement group ID if the actor requires a placement group. + bytes placement_group_id = 9; + // The label selector for the actor. + map label_selector = 11; +} diff --git a/src/ray/protobuf/public/events_actor_lifecycle_event.proto b/src/ray/protobuf/public/events_actor_lifecycle_event.proto new file mode 100644 index 000000000000..debdf844bcf3 --- /dev/null +++ b/src/ray/protobuf/public/events_actor_lifecycle_event.proto @@ -0,0 +1,54 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package ray.rpc.events; + +import "src/ray/protobuf/common.proto"; +import "google/protobuf/timestamp.proto"; + +message ActorLifecycleEvent { + enum State{ + // Actor info is registered in GCS. But its dependencies are not ready. + DEPENDENCIES_UNREADY = 0; + // Actor local dependencies are ready. This actor is being created. + PENDING_CREATION = 1; + // Actor is alive. + ALIVE = 2; + // Actor is dead, now being restarted. + // After reconstruction finishes, the state will become alive again. + RESTARTING = 3; + // Actor is already dead and won't be restarted. + DEAD = 4; + } + + message StateTransition { + State state = 1; + google.protobuf.Timestamp timestamp = 2; + // The node id of the actor once it is created. + // available when state is ALIVE updated when the actor is restarted. + bytes node_id = 3; + // The worker id of the worker on which this actor is running. available when state is ALIVE. + // The worker id can change when the actor is restarted. + bytes worker_id = 4; + // Contains metadata about why the actor is dead. available when state is DEAD. + ActorDeathCause death_cause = 6; + } + + // The ID of the actor that was created. + bytes actor_id = 1; + // Current state of this actor. + repeated StateTransition state_transitions = 2; +} diff --git a/src/ray/protobuf/public/events_base_event.proto b/src/ray/protobuf/public/events_base_event.proto index af668a75c8a0..5adbf9757f62 100644 --- a/src/ray/protobuf/public/events_base_event.proto +++ b/src/ray/protobuf/public/events_base_event.proto @@ -23,6 +23,8 @@ import "src/ray/protobuf/public/events_task_definition_event.proto"; import "src/ray/protobuf/public/events_task_execution_event.proto"; import "src/ray/protobuf/public/events_driver_job_definition_event.proto"; import "src/ray/protobuf/public/events_driver_job_execution_event.proto"; +import "src/ray/protobuf/public/events_actor_definition_event.proto"; +import "src/ray/protobuf/public/events_actor_lifecycle_event.proto"; import "src/ray/protobuf/public/events_node_definition_event.proto"; import "src/ray/protobuf/public/events_node_lifecycle_event.proto"; @@ -54,6 +56,8 @@ message RayEvent { DRIVER_JOB_EXECUTION_EVENT = 6; NODE_DEFINITION_EVENT = 7; NODE_LIFECYCLE_EVENT = 8; + ACTOR_DEFINITION_EVENT = 9; + ACTOR_LIFECYCLE_EVENT = 10; } // The severities of events that can be generated. @@ -97,4 +101,6 @@ message RayEvent { DriverJobExecutionEvent driver_job_execution_event = 13; NodeDefinitionEvent node_definition_event = 14; NodeLifecycleEvent node_lifecycle_event = 15; + ActorDefinitionEvent actor_definition_event = 16; + ActorLifecycleEvent actor_lifecycle_event = 17; } From 0047e724b8a48ccaa2fb960401418c4efcfc95c7 Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Wed, 10 Sep 2025 18:00:27 -0700 Subject: [PATCH 568/634] [data] ignore metadata for pandas block (#56402) ## Why are these changes needed? Consider the following the code ```python import ray # Read File (1) source_path = "file_that_contains_tensor_strings.parquet" ds = ray.data.read_parquet(source_path) # Write File (2) dest_path = "/tmp" ds.map_batches(..., batch_format="pandas").write_parquet(dest_path) # Read File Again (3) new_ds = ray.data.read_parquet(dest_path).map_bataches(..., batch_format="pandas") ``` At a high level we read, write, read. On a lower-level, we convert arrow blocks -> pandas -> arrow blocks -> pandas. We have connectors and registered extension types in `python/ray/air/util/tensor_extensions/`, however we special case handle tensor types by converting them to `TensorArrays` [here](https://github.com/iamjustinhsu/ray/blob/1f7dcec413bf9aba3ac39c0a14d7d4b734a1939f/python/ray/data/_internal/pandas_block.py#L238) when we convert pandas -> arrow. During this process, however, pyarrow will store metadata about the pandas block, which will look something like this: ```json { "name": "feature1", "field_name": "feature1", "pandas_type": "object", "numpy_type": "numpy.ndarray(shape=(8, 2), dtype= ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- python/ray/air/tests/test_tensor_extension.py | 26 +++++++++++++++++++ python/ray/data/_internal/arrow_block.py | 4 ++- python/ray/data/context.py | 6 +++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/python/ray/air/tests/test_tensor_extension.py b/python/ray/air/tests/test_tensor_extension.py index fb5b6bbd43ab..1f0c8ff08756 100644 --- a/python/ray/air/tests/test_tensor_extension.py +++ b/python/ray/air/tests/test_tensor_extension.py @@ -800,6 +800,32 @@ def test_large_arrow_tensor_array(restore_data_context, tensor_format): assert np.asarray(arr).shape == (1000, 550) +@pytest.mark.parametrize("tensor_format", ["v1", "v2"]) +def test_tensor_array_string_tensors_simple(restore_data_context, tensor_format): + """Simple test for fixed-shape string tensor arrays with pandas/arrow roundtrip.""" + DataContext.get_current().use_arrow_tensor_v2 = tensor_format == "v2" + + # Create fixed-shape string tensor + string_tensors = np.array( + [["hello", "world"], ["arrow", "pandas"], ["tensor", "string"]] + ) + + # Create pandas DataFrame with TensorArray + df_pandas = pd.DataFrame({"id": [1, 2, 3], "strings": TensorArray(string_tensors)}) + # Convert to Arrow table + arrow_table = pa.Table.from_pandas(df_pandas) + + # Convert back to pandas + df_roundtrip = arrow_table.to_pandas(ignore_metadata=True) + + # Verify the roundtrip preserves the data + original_strings = df_pandas["strings"].to_numpy() + roundtrip_strings = df_roundtrip["strings"].to_numpy() + + np.testing.assert_array_equal(original_strings, roundtrip_strings) + np.testing.assert_array_equal(roundtrip_strings, string_tensors) + + if __name__ == "__main__": import sys diff --git a/python/ray/data/_internal/arrow_block.py b/python/ray/data/_internal/arrow_block.py index 9714ecaab709..14a11b8b0fab 100644 --- a/python/ray/data/_internal/arrow_block.py +++ b/python/ray/data/_internal/arrow_block.py @@ -263,8 +263,10 @@ def schema(self) -> "pyarrow.lib.Schema": def to_pandas(self) -> "pandas.DataFrame": from ray.air.util.data_batch_conversion import _cast_tensor_columns_to_ndarrays - df = self._table.to_pandas() + # We specify ignore_metadata=True because pyarrow will use the metadata + # to build the Table. This is handled incorrectly for older pyarrow versions ctx = DataContext.get_current() + df = self._table.to_pandas(ignore_metadata=ctx.pandas_block_ignore_metadata) if ctx.enable_tensor_extension_casting: df = _cast_tensor_columns_to_ndarrays(df) return df diff --git a/python/ray/data/context.py b/python/ray/data/context.py index 59bebbaa3ede..480192bfb986 100644 --- a/python/ray/data/context.py +++ b/python/ray/data/context.py @@ -70,6 +70,10 @@ class ShuffleStrategy(str, enum.Enum): DEFAULT_ENABLE_PANDAS_BLOCK = True +DEFAULT_PANDAS_BLOCK_IGNORE_METADATA = bool( + os.environ.get("RAY_DATA_PANDAS_BLOCK_IGNORE_METADATA", 0) +) + DEFAULT_READ_OP_MIN_NUM_BLOCKS = 200 DEFAULT_ACTOR_PREFETCHER_ENABLED = False @@ -541,6 +545,8 @@ class DataContext: enforce_schemas: bool = DEFAULT_ENFORCE_SCHEMAS + pandas_block_ignore_metadata: bool = DEFAULT_PANDAS_BLOCK_IGNORE_METADATA + def __post_init__(self): # The additonal ray remote args that should be added to # the task-pool-based data tasks. From ef7169a77d254a3c8e0779f80d61b2ba5c8a0198 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Wed, 10 Sep 2025 18:23:01 -0700 Subject: [PATCH 569/634] [core][otel] clear gauge metric cache at export time (#56405) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `self._observations_by_name` is a mapping of all time series for a given gauge metric to their current values. Currently, we do not clean up this map at each export interval, which can lead to issues where dead time series (e.g., series created by workers that are no longer alive) continue to emit their last value. Since we use sum aggregation for most gauge metrics, the sum ends up including contributions from these dead workers. This PR introduces cleanup at each export interval, which also improves memory usage. It relies on the active processes to emit the relevant, up-to-date information at every interval. Test: - CI - Test e2e on anyscale platform (previously the number of live actors remained as 6 after resizing, now they go back to 3 Screenshot 2025-09-10 at 10 28 57 AM Signed-off-by: Cuong Nguyen --- .../_private/telemetry/open_telemetry_metric_recorder.py | 2 ++ .../ray/dashboard/modules/reporter/tests/test_reporter.py | 7 ++----- python/ray/tests/test_metrics_agent.py | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/ray/_private/telemetry/open_telemetry_metric_recorder.py b/python/ray/_private/telemetry/open_telemetry_metric_recorder.py index a5cb561df1e8..1076ed5c0857 100644 --- a/python/ray/_private/telemetry/open_telemetry_metric_recorder.py +++ b/python/ray/_private/telemetry/open_telemetry_metric_recorder.py @@ -59,6 +59,8 @@ def callback(options): # Take snapshot of current observations. with self._lock: observations = self._observations_by_name[name] + # Clear the observations to avoid emitting dead observations. + self._observations_by_name[name] = {} # Drop high cardinality from tag_set and sum up the value for # same tag set after dropping aggregated_observations = defaultdict(float) diff --git a/python/ray/dashboard/modules/reporter/tests/test_reporter.py b/python/ray/dashboard/modules/reporter/tests/test_reporter.py index 8fbc7218d51a..8bcb52e76156 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_reporter.py +++ b/python/ray/dashboard/modules/reporter/tests/test_reporter.py @@ -270,11 +270,8 @@ def test_case_ip_correct(): break return str(raylet_proc.process.pid) == str(raylet_pid) - wait_for_condition( - lambda: test_case_stats_exist() and test_case_ip_correct(), - timeout=30, - retry_interval_ms=1000, - ) + wait_for_condition(test_case_stats_exist, timeout=30, retry_interval_ms=1000) + wait_for_condition(test_case_ip_correct, timeout=30, retry_interval_ms=1000) @pytest.mark.skipif( diff --git a/python/ray/tests/test_metrics_agent.py b/python/ray/tests/test_metrics_agent.py index 3dadc7015666..21907971c3bc 100644 --- a/python/ray/tests/test_metrics_agent.py +++ b/python/ray/tests/test_metrics_agent.py @@ -195,6 +195,7 @@ def _setup_cluster_for_test(request, ray_start_cluster): # Add a head node. cluster.add_node( _system_config={ + "metrics_report_interval_ms": 1000, "event_stats_print_interval_ms": 500, "event_stats": True, "enable_metrics_collection": enable_metrics_collection, From 65c983922d779875dcbf3f1838f9c7adab83272d Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Wed, 10 Sep 2025 20:29:09 -0700 Subject: [PATCH 570/634] [Data] Make operator `target_max_block_size` optional and rename as override (#56423) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? This PR: * Makes the `target_max_block_size` parameter of physical operators optional * Renames them to `target_max_block_size_override` No behavior changes are introduced. This is purely renaming and removing unnecessary keyword arguments. The goal is to clarify the parameter’s intent and simplify constructor calls. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Balaji Veeramani --- ci/lint/pydoclint-baseline.txt | 2 -- .../execution/interfaces/physical_operator.py | 22 +++++++++---------- .../execution/interfaces/task_context.py | 4 ++-- .../operators/actor_pool_map_operator.py | 10 ++++----- .../execution/operators/aggregate_num_rows.py | 1 - .../operators/base_physical_operator.py | 19 ++++++++-------- .../execution/operators/hash_shuffle.py | 1 - .../execution/operators/input_data_buffer.py | 2 +- .../execution/operators/limit_operator.py | 2 +- .../execution/operators/map_operator.py | 4 ++-- .../execution/operators/map_transformer.py | 8 +++---- .../execution/operators/output_splitter.py | 1 - .../operators/task_pool_map_operator.py | 2 +- .../execution/operators/zip_operator.py | 1 - .../rules/inherit_target_max_block_size.py | 6 ++--- .../logical/rules/operator_fusion.py | 10 ++++----- .../_internal/planner/plan_all_to_all_op.py | 1 - .../data/_internal/planner/random_shuffle.py | 4 ++-- .../ray/data/_internal/planner/repartition.py | 8 ++++--- .../tests/test_actor_pool_map_operator.py | 2 -- python/ray/data/tests/test_operators.py | 4 ++-- .../ray/data/tests/test_streaming_executor.py | 1 - 22 files changed, 54 insertions(+), 61 deletions(-) diff --git a/ci/lint/pydoclint-baseline.txt b/ci/lint/pydoclint-baseline.txt index 9ee8a2b39ad0..4455dd16b381 100644 --- a/ci/lint/pydoclint-baseline.txt +++ b/ci/lint/pydoclint-baseline.txt @@ -1098,8 +1098,6 @@ python/ray/data/_internal/execution/interfaces/task_context.py python/ray/data/_internal/execution/operators/base_physical_operator.py DOC101: Method `OneToOneOperator.__init__`: Docstring contains fewer arguments than in function signature. DOC103: Method `OneToOneOperator.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [data_context: DataContext]. - DOC101: Method `AllToAllOperator.__init__`: Docstring contains fewer arguments than in function signature. - DOC103: Method `AllToAllOperator.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [data_context: DataContext, target_max_block_size: Optional[int]]. DOC103: Method `NAryOperator.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [*input_ops: LogicalOperator, data_context: DataContext]. Arguments in the docstring but not in the function signature: [input_op: , name: ]. -------------------- python/ray/data/_internal/execution/operators/hash_shuffle.py diff --git a/python/ray/data/_internal/execution/interfaces/physical_operator.py b/python/ray/data/_internal/execution/interfaces/physical_operator.py index c3a7b429813e..769f7f967812 100644 --- a/python/ray/data/_internal/execution/interfaces/physical_operator.py +++ b/python/ray/data/_internal/execution/interfaces/physical_operator.py @@ -251,15 +251,15 @@ def __init__( name: str, input_dependencies: List["PhysicalOperator"], data_context: DataContext, - target_max_block_size: Optional[int], + target_max_block_size_override: Optional[int] = None, ): super().__init__(name, input_dependencies) for x in input_dependencies: assert isinstance(x, PhysicalOperator), x self._inputs_complete = not input_dependencies - self._output_block_size_option = None - self.set_target_max_block_size(target_max_block_size) + self._output_block_size_option_override = None + self.override_target_max_block_size(target_max_block_size_override) self._started = False self._shutdown = False self._in_task_submission_backpressure = False @@ -307,15 +307,15 @@ def set_logical_operators( self._logical_operators = list(logical_ops) @property - def target_max_block_size(self) -> Optional[int]: + def target_max_block_size_override(self) -> Optional[int]: """ Target max block size output by this operator. If this returns None, then the default from DataContext should be used. """ - if self._output_block_size_option is None: + if self._output_block_size_option_override is None: return None else: - return self._output_block_size_option.target_max_block_size + return self._output_block_size_option_override.target_max_block_size @property def actual_target_max_block_size(self) -> Optional[int]: @@ -325,18 +325,18 @@ def actual_target_max_block_size(self) -> Optional[int]: `None` if the target max block size is not set, otherwise the target max block size. `None` means the block size is infinite. """ - target_max_block_size = self.target_max_block_size + target_max_block_size = self.target_max_block_size_override if target_max_block_size is None: target_max_block_size = self.data_context.target_max_block_size return target_max_block_size - def set_target_max_block_size(self, target_max_block_size: Optional[int]): + def override_target_max_block_size(self, target_max_block_size: Optional[int]): if target_max_block_size is not None: - self._output_block_size_option = OutputBlockSizeOption( + self._output_block_size_option_override = OutputBlockSizeOption( target_max_block_size=target_max_block_size ) - elif self._output_block_size_option is not None: - self._output_block_size_option = None + elif self._output_block_size_option_override is not None: + self._output_block_size_option_override = None def mark_execution_finished(self): """Manually mark that this operator has finished execution.""" diff --git a/python/ray/data/_internal/execution/interfaces/task_context.py b/python/ray/data/_internal/execution/interfaces/task_context.py index 9fb4ffe6e20f..7ff0f60f9670 100644 --- a/python/ray/data/_internal/execution/interfaces/task_context.py +++ b/python/ray/data/_internal/execution/interfaces/task_context.py @@ -44,8 +44,8 @@ class TaskContext: # This should be set if upstream_map_transformer is set. upstream_map_ray_remote_args: Optional[Dict[str, Any]] = None - # The target maximum number of bytes to include in the task's output block. - target_max_block_size: Optional[int] = None + # Override of the target max-block-size for the task + target_max_block_size_override: Optional[int] = None # Additional keyword arguments passed to the task. kwargs: Dict[str, Any] = field(default_factory=dict) diff --git a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py index a526b982ef45..231046c984e3 100644 --- a/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/actor_pool_map_operator.py @@ -65,7 +65,6 @@ def __init__( map_transformer: MapTransformer, input_op: PhysicalOperator, data_context: DataContext, - target_max_block_size: Optional[int], compute_strategy: ActorPoolStrategy, name: str = "ActorPoolMap", min_rows_per_bundle: Optional[int] = None, @@ -73,6 +72,7 @@ def __init__( map_task_kwargs: Optional[Dict[str, Any]] = None, ray_remote_args_fn: Optional[Callable[[], Dict[str, Any]]] = None, ray_remote_args: Optional[Dict[str, Any]] = None, + target_max_block_size_override: Optional[int] = None, ): """Create an ActorPoolMapOperator instance. @@ -81,8 +81,6 @@ def __init__( to each ref bundle input. input_op: Operator generating input data for this op. data_context: The DataContext instance containing configuration settings. - target_max_block_size: The target maximum number of bytes to - include in an output block. compute_strategy: `ComputeStrategy` used for this operator. name: The name of this operator. min_rows_per_bundle: The number of rows to gather per batch passed to the @@ -100,13 +98,15 @@ def __init__( advanced, experimental feature. ray_remote_args: Customize the ray remote args for this op's tasks. See :func:`ray.remote` for details. + target_max_block_size_override: The target maximum number of bytes to + include in an output block. """ super().__init__( map_transformer, input_op, data_context, name, - target_max_block_size, + target_max_block_size_override, min_rows_per_bundle, supports_fusion, map_task_kwargs, @@ -301,7 +301,7 @@ def _dispatch_tasks(self): ctx = TaskContext( task_idx=self._next_data_task_idx, op_name=self.name, - target_max_block_size=self.actual_target_max_block_size, + target_max_block_size_override=self.actual_target_max_block_size, ) gen = actor.submit.options( num_returns="streaming", diff --git a/python/ray/data/_internal/execution/operators/aggregate_num_rows.py b/python/ray/data/_internal/execution/operators/aggregate_num_rows.py index 674012b00990..68084d2d0ad7 100644 --- a/python/ray/data/_internal/execution/operators/aggregate_num_rows.py +++ b/python/ray/data/_internal/execution/operators/aggregate_num_rows.py @@ -23,7 +23,6 @@ def __init__( "AggregateNumRows", input_dependencies, data_context, - target_max_block_size=None, ) self._column_name = column_name diff --git a/python/ray/data/_internal/execution/operators/base_physical_operator.py b/python/ray/data/_internal/execution/operators/base_physical_operator.py index df934ae323f6..948a34d92fef 100644 --- a/python/ray/data/_internal/execution/operators/base_physical_operator.py +++ b/python/ray/data/_internal/execution/operators/base_physical_operator.py @@ -31,16 +31,16 @@ def __init__( name: str, input_op: PhysicalOperator, data_context: DataContext, - target_max_block_size: Optional[int], + target_max_block_size_override: Optional[int] = None, ): """Create a OneToOneOperator. Args: input_op: Operator generating input data for this op. name: The name of this operator. - target_max_block_size: The target maximum number of bytes to + target_max_block_size_override: The target maximum number of bytes to include in an output block. """ - super().__init__(name, [input_op], data_context, target_max_block_size) + super().__init__(name, [input_op], data_context, target_max_block_size_override) @property def input_dependency(self) -> PhysicalOperator: @@ -58,7 +58,7 @@ def __init__( bulk_fn: AllToAllTransformFn, input_op: PhysicalOperator, data_context: DataContext, - target_max_block_size: Optional[int], + target_max_block_size_override: Optional[int] = None, num_outputs: Optional[int] = None, sub_progress_bar_names: Optional[List[str]] = None, name: str = "AllToAll", @@ -69,6 +69,9 @@ def __init__( list of input ref bundles, and the outputs are the output ref bundles and a stats dict. input_op: Operator generating input data for this op. + data_context: The DataContext instance containing configuration settings. + target_max_block_size_override: The target maximum number of bytes to + include in an output block. num_outputs: The number of expected output bundles for progress bar. sub_progress_bar_names: The names of internal sub progress bars. name: The name of this operator. @@ -82,7 +85,7 @@ def __init__( self._input_buffer: List[RefBundle] = [] self._output_buffer: List[RefBundle] = [] self._stats: StatsDict = {} - super().__init__(name, [input_op], data_context, target_max_block_size) + super().__init__(name, [input_op], data_context, target_max_block_size_override) def num_outputs_total(self) -> Optional[int]: return ( @@ -112,7 +115,7 @@ def all_inputs_done(self) -> None: task_idx=self._next_task_index, op_name=self.name, sub_progress_bar_dict=self._sub_progress_bar_dict, - target_max_block_size=self.actual_target_max_block_size, + target_max_block_size_override=self.actual_target_max_block_size, ) # NOTE: We don't account object store memory use from intermediate `bulk_fn` # outputs (e.g., map outputs for map-reduce). @@ -191,6 +194,4 @@ def __init__( """ input_names = ", ".join([op._name for op in input_ops]) op_name = f"{self.__class__.__name__}({input_names})" - super().__init__( - op_name, list(input_ops), data_context, target_max_block_size=None - ) + super().__init__(op_name, list(input_ops), data_context) diff --git a/python/ray/data/_internal/execution/operators/hash_shuffle.py b/python/ray/data/_internal/execution/operators/hash_shuffle.py index 400bd9e04573..3ada3a7bd2ad 100644 --- a/python/ray/data/_internal/execution/operators/hash_shuffle.py +++ b/python/ray/data/_internal/execution/operators/hash_shuffle.py @@ -426,7 +426,6 @@ def __init__( name=name, input_dependencies=input_ops, data_context=data_context, - target_max_block_size=None, ) if shuffle_progress_bar_name is None: diff --git a/python/ray/data/_internal/execution/operators/input_data_buffer.py b/python/ray/data/_internal/execution/operators/input_data_buffer.py index 4aa6c4a63688..b2213f9321d0 100644 --- a/python/ray/data/_internal/execution/operators/input_data_buffer.py +++ b/python/ray/data/_internal/execution/operators/input_data_buffer.py @@ -33,7 +33,7 @@ def __init__( num_output_blocks: The number of output blocks. If not specified, progress bars total will be set based on num output bundles instead. """ - super().__init__("Input", [], data_context, target_max_block_size=None) + super().__init__("Input", [], data_context) if input_data is not None: assert input_data_factory is None # Copy the input data to avoid mutating the original list. diff --git a/python/ray/data/_internal/execution/operators/limit_operator.py b/python/ray/data/_internal/execution/operators/limit_operator.py index c4702323d565..246f34b82453 100644 --- a/python/ray/data/_internal/execution/operators/limit_operator.py +++ b/python/ray/data/_internal/execution/operators/limit_operator.py @@ -29,7 +29,7 @@ def __init__( self._name = f"limit={limit}" self._output_blocks_stats: List[BlockStats] = [] self._cur_output_bundles = 0 - super().__init__(self._name, input_op, data_context, target_max_block_size=None) + super().__init__(self._name, input_op, data_context) if self._limit <= 0: self.mark_execution_finished() diff --git a/python/ray/data/_internal/execution/operators/map_operator.py b/python/ray/data/_internal/execution/operators/map_operator.py index 3efa53e28bb1..9af913687c5f 100644 --- a/python/ray/data/_internal/execution/operators/map_operator.py +++ b/python/ray/data/_internal/execution/operators/map_operator.py @@ -239,7 +239,7 @@ def create( map_transformer, input_op, data_context, - target_max_block_size=target_max_block_size, + target_max_block_size_override=target_max_block_size, compute_strategy=compute_strategy, name=name, min_rows_per_bundle=min_rows_per_bundle, @@ -551,7 +551,7 @@ def _map_task( ctx.kwargs.update(kwargs) TaskContext.set_current(ctx) stats = BlockExecStats.builder() - map_transformer.set_target_max_block_size(ctx.target_max_block_size) + map_transformer.override_target_max_block_size(ctx.target_max_block_size_override) with MemoryProfiler(data_context.memory_usage_poll_interval_s) as profiler: for b_out in map_transformer.apply_transform(iter(blocks), ctx): # TODO(Clark): Add input file propagation from input blocks. diff --git a/python/ray/data/_internal/execution/operators/map_transformer.py b/python/ray/data/_internal/execution/operators/map_transformer.py index db3e5fc5353e..4d8870c4d3b8 100644 --- a/python/ray/data/_internal/execution/operators/map_transformer.py +++ b/python/ray/data/_internal/execution/operators/map_transformer.py @@ -89,7 +89,7 @@ def category(self) -> MapTransformFnCategory: def output_block_size_option(self): return self._output_block_size_option - def set_target_max_block_size(self, target_max_block_size: Optional[int]): + def override_target_max_block_size(self, target_max_block_size: Optional[int]): self._output_block_size_option = OutputBlockSizeOption( target_max_block_size=target_max_block_size ) @@ -163,7 +163,7 @@ def get_transform_fns(self) -> List[MapTransformFn]: """Get the transform functions.""" return self._transform_fns - def set_target_max_block_size(self, target_max_block_size: int): + def override_target_max_block_size(self, target_max_block_size: Optional[int]): if target_max_block_size is not None: self._output_block_size_option = OutputBlockSizeOption( target_max_block_size=target_max_block_size @@ -221,7 +221,7 @@ def apply_transform( """Apply the transform functions to the input blocks.""" for transform_fn in self._transform_fns: if not transform_fn.output_block_size_option: - transform_fn.set_target_max_block_size(self.target_max_block_size) + transform_fn.override_target_max_block_size(self.target_max_block_size) iter = input_blocks # Apply the transform functions sequentially to the input iterable. @@ -251,7 +251,7 @@ def fused_init_fn(): fused_transform_fns = self._transform_fns + other._transform_fns transformer = MapTransformer(fused_transform_fns, init_fn=fused_init_fn) - transformer.set_target_max_block_size(target_max_block_size) + transformer.override_target_max_block_size(target_max_block_size) return transformer def udf_time(self) -> float: diff --git a/python/ray/data/_internal/execution/operators/output_splitter.py b/python/ray/data/_internal/execution/operators/output_splitter.py index 160ab0558ebd..df8c7e0ec962 100644 --- a/python/ray/data/_internal/execution/operators/output_splitter.py +++ b/python/ray/data/_internal/execution/operators/output_splitter.py @@ -47,7 +47,6 @@ def __init__( f"split({n}, equal={equal})", [input_op], data_context, - target_max_block_size=None, ) self._equal = equal # Buffer of bundles not yet assigned to output splits. diff --git a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py index a46b44cf2bbe..fd402d73cd05 100644 --- a/python/ray/data/_internal/execution/operators/task_pool_map_operator.py +++ b/python/ray/data/_internal/execution/operators/task_pool_map_operator.py @@ -85,7 +85,7 @@ def _add_bundled_input(self, bundle: RefBundle): ctx = TaskContext( task_idx=self._next_data_task_idx, op_name=self.name, - target_max_block_size=self.actual_target_max_block_size, + target_max_block_size_override=self.actual_target_max_block_size, ) dynamic_ray_remote_args = self._get_runtime_ray_remote_args(input_bundle=bundle) diff --git a/python/ray/data/_internal/execution/operators/zip_operator.py b/python/ray/data/_internal/execution/operators/zip_operator.py index 01f2a94774d9..d37ecd59a821 100644 --- a/python/ray/data/_internal/execution/operators/zip_operator.py +++ b/python/ray/data/_internal/execution/operators/zip_operator.py @@ -52,7 +52,6 @@ def __init__( "Zip", [left_input_op, right_input_op], data_context, - target_max_block_size=None, ) def num_outputs_total(self) -> Optional[int]: diff --git a/python/ray/data/_internal/logical/rules/inherit_target_max_block_size.py b/python/ray/data/_internal/logical/rules/inherit_target_max_block_size.py index 298ff6c4edbf..a7d55ccb0ead 100644 --- a/python/ray/data/_internal/logical/rules/inherit_target_max_block_size.py +++ b/python/ray/data/_internal/logical/rules/inherit_target_max_block_size.py @@ -16,13 +16,13 @@ def apply(self, plan: PhysicalPlan) -> PhysicalPlan: def _propagate_target_max_block_size_to_upstream_ops( self, dag: PhysicalOperator, target_max_block_size: Optional[int] = None ): - if dag.target_max_block_size is not None: + if dag.target_max_block_size_override is not None: # Set the target block size to inherit for # upstream ops. - target_max_block_size = dag.target_max_block_size + target_max_block_size = dag.target_max_block_size_override elif target_max_block_size is not None: # Inherit from downstream op. - dag.set_target_max_block_size(target_max_block_size) + dag.override_target_max_block_size(target_max_block_size) for upstream_op in dag.input_dependencies: self._propagate_target_max_block_size_to_upstream_ops( diff --git a/python/ray/data/_internal/logical/rules/operator_fusion.py b/python/ray/data/_internal/logical/rules/operator_fusion.py index 13104de04be8..ba1e07797d4c 100644 --- a/python/ray/data/_internal/logical/rules/operator_fusion.py +++ b/python/ray/data/_internal/logical/rules/operator_fusion.py @@ -212,8 +212,8 @@ def _can_fuse(self, down_op: PhysicalOperator, up_op: PhysicalOperator) -> bool: return False if not self._can_merge_target_max_block_size( - up_op.target_max_block_size, - down_op.target_max_block_size, + up_op.target_max_block_size_override, + down_op.target_max_block_size_override, up_op.data_context, ): return False @@ -302,7 +302,7 @@ def _get_fused_map_operator( ) target_max_block_size = self._get_merged_target_max_block_size( - up_op.target_max_block_size, down_op.target_max_block_size + up_op.target_max_block_size_override, down_op.target_max_block_size_override ) compute = self._fuse_compute_strategy( @@ -436,7 +436,7 @@ def fused_all_to_all_transform_fn( input_op = input_deps[0] target_max_block_size = self._get_merged_target_max_block_size( - up_op.target_max_block_size, down_op.target_max_block_size + up_op.target_max_block_size_override, down_op.target_max_block_size_override ) assert up_op.data_context is down_op.data_context @@ -444,7 +444,7 @@ def fused_all_to_all_transform_fn( fused_all_to_all_transform_fn, input_op, up_op.data_context, - target_max_block_size=target_max_block_size, + target_max_block_size_override=target_max_block_size, num_outputs=down_op._num_outputs, # Transfer over the existing sub-progress bars from # the AllToAllOperator (if any) into the fused operator. diff --git a/python/ray/data/_internal/planner/plan_all_to_all_op.py b/python/ray/data/_internal/planner/plan_all_to_all_op.py index 0fbef1188079..d3c4c0ae74dc 100644 --- a/python/ray/data/_internal/planner/plan_all_to_all_op.py +++ b/python/ray/data/_internal/planner/plan_all_to_all_op.py @@ -161,7 +161,6 @@ def plan_all_to_all_op( fn, input_physical_dag, data_context, - target_max_block_size=None, num_outputs=op._num_outputs, sub_progress_bar_names=op._sub_progress_bar_names, name=op.name, diff --git a/python/ray/data/_internal/planner/random_shuffle.py b/python/ray/data/_internal/planner/random_shuffle.py index 8a78fa587840..b698a3ecc91b 100644 --- a/python/ray/data/_internal/planner/random_shuffle.py +++ b/python/ray/data/_internal/planner/random_shuffle.py @@ -54,7 +54,7 @@ def fn( # overhead. This can be removed once dynamic block splitting is # supported for all-to-all ops. # See https://github.com/ray-project/ray/issues/40518. - map_transformer.set_target_max_block_size(float("inf")) + map_transformer.override_target_max_block_size(float("inf")) def upstream_map_fn(blocks): return map_transformer.apply_transform(blocks, ctx) @@ -64,7 +64,7 @@ def upstream_map_fn(blocks): ray_remote_args = ctx.upstream_map_ray_remote_args shuffle_spec = ShuffleTaskSpec( - ctx.target_max_block_size, + ctx.target_max_block_size_override, random_shuffle=True, random_seed=seed, upstream_map_fn=upstream_map_fn, diff --git a/python/ray/data/_internal/planner/repartition.py b/python/ray/data/_internal/planner/repartition.py index 6ba3afd0e147..5a119f540c83 100644 --- a/python/ray/data/_internal/planner/repartition.py +++ b/python/ray/data/_internal/planner/repartition.py @@ -48,13 +48,13 @@ def shuffle_repartition_fn( # overhead. This can be removed once dynamic block splitting is # supported for all-to-all ops. # See https://github.com/ray-project/ray/issues/40518. - map_transformer.set_target_max_block_size(float("inf")) + map_transformer.override_target_max_block_size(float("inf")) def upstream_map_fn(blocks): return map_transformer.apply_transform(blocks, ctx) shuffle_spec = ShuffleTaskSpec( - ctx.target_max_block_size, + ctx.target_max_block_size_override, random_shuffle=False, upstream_map_fn=upstream_map_fn, ) @@ -77,7 +77,9 @@ def split_repartition_fn( refs: List[RefBundle], ctx: TaskContext, ) -> AllToAllTransformFnResult: - shuffle_spec = ShuffleTaskSpec(ctx.target_max_block_size, random_shuffle=False) + shuffle_spec = ShuffleTaskSpec( + ctx.target_max_block_size_override, random_shuffle=False + ) scheduler = SplitRepartitionTaskScheduler(shuffle_spec) return scheduler.execute(refs, num_outputs, ctx) diff --git a/python/ray/data/tests/test_actor_pool_map_operator.py b/python/ray/data/tests/test_actor_pool_map_operator.py index 66dcad11f0db..e6b149c153a5 100644 --- a/python/ray/data/tests/test_actor_pool_map_operator.py +++ b/python/ray/data/tests/test_actor_pool_map_operator.py @@ -599,7 +599,6 @@ def test_setting_initial_size_for_actor_pool(): map_transformer=MagicMock(), input_op=InputDataBuffer(data_context, input_data=MagicMock()), data_context=data_context, - target_max_block_size=None, compute_strategy=ray.data.ActorPoolStrategy( min_size=1, max_size=4, initial_size=2 ), @@ -620,7 +619,6 @@ def test_min_max_resource_requirements(restore_data_context): map_transformer=MagicMock(), input_op=InputDataBuffer(data_context, input_data=MagicMock()), data_context=data_context, - target_max_block_size=None, compute_strategy=ray.data.ActorPoolStrategy( min_size=1, max_size=2, diff --git a/python/ray/data/tests/test_operators.py b/python/ray/data/tests/test_operators.py index 92e5f0ab5741..8ef0f2267386 100644 --- a/python/ray/data/tests/test_operators.py +++ b/python/ray/data/tests/test_operators.py @@ -121,7 +121,7 @@ def dummy_all_transform(bundles: List[RefBundle], ctx): dummy_all_transform, input_op, DataContext.get_current(), - target_max_block_size=DataContext.get_current().target_max_block_size, + target_max_block_size_override=DataContext.get_current().target_max_block_size, num_outputs=2, sub_progress_bar_names=["Test1", "Test2"], name="TestAll", @@ -172,7 +172,7 @@ def dummy_all_transform(bundles: List[RefBundle]): dummy_all_transform, input_op=op1, data_context=DataContext.get_current(), - target_max_block_size=DataContext.get_current().target_max_block_size, + target_max_block_size_override=DataContext.get_current().target_max_block_size, name="TestAll", ) assert op2.num_outputs_total() is None diff --git a/python/ray/data/tests/test_streaming_executor.py b/python/ray/data/tests/test_streaming_executor.py index 4bb996a280cf..38344b4f4134 100644 --- a/python/ray/data/tests/test_streaming_executor.py +++ b/python/ray/data/tests/test_streaming_executor.py @@ -142,7 +142,6 @@ def test_disallow_non_unique_operators(): "test_combine", [o2, o3], DataContext.get_current(), - target_max_block_size=None, ) with pytest.raises(ValueError): build_streaming_topology(o4, ExecutionOptions(verbose_progress=True)) From 857bea408f2c371dbfa73b16618ed1abd3644566 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Wed, 10 Sep 2025 21:26:55 -0700 Subject: [PATCH 571/634] [core] Introduce env var to set rpc failure prob for all rpc's (#56413) Signed-off-by: dayshah --- src/ray/common/ray_config_def.h | 12 +++-- src/ray/rpc/rpc_chaos.cc | 77 ++++++++++++++++++++--------- src/ray/rpc/rpc_chaos.h | 2 +- src/ray/rpc/tests/rpc_chaos_test.cc | 55 ++++++++++++++------- 4 files changed, 100 insertions(+), 46 deletions(-) diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index b9c2e7e6b979..f780c9b6868e 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -833,9 +833,15 @@ RAY_CONFIG(std::string, REDIS_SERVER_NAME, "") // it will apply to all methods. RAY_CONFIG(std::string, testing_asio_delay_us, "") -/// To use this, simply do -/// export -/// RAY_testing_rpc_failure="method1=max_num_failures:req_failure_prob:resp_failure_prob,method2=max_num_failures:req_failure_prob:resp_failure_prob" +/// To use this, simply do +/// export +/// RAY_testing_rpc_failure="method1=max_num_failures:req_failure_prob:resp_failure_prob,method2=max_num_failures:req_failure_prob:resp_failure_prob" +/// If you want to test all rpc failures you can use * as the method name and you can set +/// -1 max_num_failures to have unlimited failures. +/// Ex. unlimited failures for all rpc's with 25% request failures and 50% response +/// failures. +/// export RAY_testing_rpc_failure="*=-1:25:50" +/// NOTE: Setting the wildcard will override any configuration for other methods. RAY_CONFIG(std::string, testing_rpc_failure, "") /// The following are configs for the health check. They are borrowed diff --git a/src/ray/rpc/rpc_chaos.cc b/src/ray/rpc/rpc_chaos.cc index b56738945637..d8c1b2bd47bb 100644 --- a/src/ray/rpc/rpc_chaos.cc +++ b/src/ray/rpc/rpc_chaos.cc @@ -25,16 +25,22 @@ namespace ray { namespace rpc { namespace testing { -namespace { // RpcFailureManager is a simple chaos testing framework. Before starting ray, users // should set up os environment to use this feature for testing purposes. -// To use this, simply do + +// You can use this to set probabilities for specific rpc's. // export RAY_testing_rpc_failure="method1=3:25:50,method2=5:25:25" // Key is the RPC call name and value is a three part colon separated structure. It // contains the max number of failures to inject + probability of req failure + // probability of reply failure. +// You can also use a wildcard to set probabilities for all rpc's and -1 as num_failures +// to have unlimited failures. +// export RAY_testing_rpc_failure="*=-1:25:50" +// This will set the probabilities for all rpc's to 25% for request failures and 50% for +// reply failures. + class RpcFailureManager { public: RpcFailureManager() { Init(); } @@ -42,7 +48,10 @@ class RpcFailureManager { void Init() { absl::MutexLock lock(&mu_); + // Clear old state failable_methods_.clear(); + wildcard_set_ = false; + has_failures_ = false; if (!RayConfig::instance().testing_rpc_failure().empty()) { for (const auto &item : @@ -52,33 +61,67 @@ class RpcFailureManager { std::vector colon_split = absl::StrSplit(equal_split[1], ':'); RAY_CHECK_EQ(colon_split.size(), 3UL); auto [iter, _] = failable_methods_.emplace(equal_split[0], - Failable{std::stoul(colon_split[0]), + Failable{std::stol(colon_split[0]), std::stoul(colon_split[1]), std::stoul(colon_split[2])}); const auto &failable = iter->second; RAY_CHECK_LE(failable.req_failure_prob + failable.resp_failure_prob, 100UL); + if (equal_split[0] == "*") { + wildcard_set_ = true; + // The wildcard overrides all other method configurations. + break; + } } std::random_device rd; auto seed = rd(); RAY_LOG(INFO) << "Setting RpcFailureManager seed to " << seed; gen_.seed(seed); + has_failures_ = true; } } RpcFailure GetRpcFailure(const std::string &name) { + if (!has_failures_) { + return RpcFailure::None; + } + absl::MutexLock lock(&mu_); + // Wildcard overrides any other method configurations. + if (wildcard_set_) { + return GetFailureTypeFromFailable(failable_methods_["*"]); + } + auto iter = failable_methods_.find(name); if (iter == failable_methods_.end()) { return RpcFailure::None; } + return GetFailureTypeFromFailable(iter->second); + } - auto &failable = iter->second; + private: + absl::Mutex mu_; + std::mt19937 gen_; + std::atomic_bool has_failures_ = false; + + // If we're testing all rpc failures, we'll use these probabilites instead of + // failable_methods_ + bool wildcard_set_ = false; + + // call name -> (num_remaining_failures, req_failure_prob, resp_failure_prob) + struct Failable { + int64_t num_remaining_failures; + size_t req_failure_prob; + size_t resp_failure_prob; + }; + absl::flat_hash_map failable_methods_ ABSL_GUARDED_BY(&mu_); + + RpcFailure GetFailureTypeFromFailable(Failable &failable) { if (failable.num_remaining_failures == 0) { + // If < 0, unlimited failures. return RpcFailure::None; } - std::uniform_int_distribution dist(1ul, 100ul); const size_t random_number = dist(gen_); if (random_number <= failable.req_failure_prob) { @@ -91,34 +134,22 @@ class RpcFailureManager { } return RpcFailure::None; } - - private: - absl::Mutex mu_; - std::mt19937 gen_; - struct Failable { - size_t num_remaining_failures; - size_t req_failure_prob; - size_t resp_failure_prob; - }; - // call name -> (num_remaining_failures, req_failure_prob, resp_failure_prob) - absl::flat_hash_map failable_methods_ ABSL_GUARDED_BY(&mu_); }; -auto &rpc_failure_manager = []() -> RpcFailureManager & { +namespace { + +RpcFailureManager &GetRpcFailureManager() { static auto *manager = new RpcFailureManager(); return *manager; -}(); +} } // namespace RpcFailure GetRpcFailure(const std::string &name) { - if (RayConfig::instance().testing_rpc_failure().empty()) { - return RpcFailure::None; - } - return rpc_failure_manager.GetRpcFailure(name); + return GetRpcFailureManager().GetRpcFailure(name); } -void Init() { rpc_failure_manager.Init(); } +void Init() { GetRpcFailureManager().Init(); } } // namespace testing } // namespace rpc diff --git a/src/ray/rpc/rpc_chaos.h b/src/ray/rpc/rpc_chaos.h index f839fad39e6c..68a41aa9a4a0 100644 --- a/src/ray/rpc/rpc_chaos.h +++ b/src/ray/rpc/rpc_chaos.h @@ -20,7 +20,7 @@ namespace ray { namespace rpc { namespace testing { -enum class RpcFailure { +enum class RpcFailure : uint8_t { None, // Failure before server receives the request Request, diff --git a/src/ray/rpc/tests/rpc_chaos_test.cc b/src/ray/rpc/tests/rpc_chaos_test.cc index 021a139dd990..f3e6d6b7b8e5 100644 --- a/src/ray/rpc/tests/rpc_chaos_test.cc +++ b/src/ray/rpc/tests/rpc_chaos_test.cc @@ -17,30 +17,47 @@ #include "gtest/gtest.h" #include "ray/common/ray_config.h" -TEST(RpcChaosTest, Basic) { - RayConfig::instance().testing_rpc_failure() = "method1=0:25:25,method2=1:25:25"; - ray::rpc::testing::Init(); - ASSERT_EQ(ray::rpc::testing::GetRpcFailure("unknown"), - ray::rpc::testing::RpcFailure::None); - ASSERT_EQ(ray::rpc::testing::GetRpcFailure("method1"), - ray::rpc::testing::RpcFailure::None); +namespace ray::rpc::testing { + +TEST(RpcChaosTest, MethodRpcFailure) { + RayConfig::instance().testing_rpc_failure() = "method1=0:25:25,method2=1:100:0"; + Init(); + ASSERT_EQ(GetRpcFailure("unknown"), RpcFailure::None); + ASSERT_EQ(GetRpcFailure("method1"), RpcFailure::None); // At most one failure. - ASSERT_FALSE(ray::rpc::testing::GetRpcFailure("method2") != - ray::rpc::testing::RpcFailure::None && - ray::rpc::testing::GetRpcFailure("method2") != - ray::rpc::testing::RpcFailure::None); + ASSERT_TRUE(GetRpcFailure("method2") == RpcFailure::Request); + ASSERT_TRUE(GetRpcFailure("method2") == RpcFailure::None); } -TEST(RpcChaosTest, EdgeCaseProbability) { +TEST(RpcChaosTest, MethodRpcFailureEdgeCase) { RayConfig::instance().testing_rpc_failure() = "method1=1000:100:0,method2=1000:0:100,method3=1000:0:0"; - ray::rpc::testing::Init(); + Init(); for (int i = 0; i < 1000; i++) { - ASSERT_EQ(ray::rpc::testing::GetRpcFailure("method1"), - ray::rpc::testing::RpcFailure::Request); - ASSERT_EQ(ray::rpc::testing::GetRpcFailure("method2"), - ray::rpc::testing::RpcFailure::Response); - ASSERT_EQ(ray::rpc::testing::GetRpcFailure("method3"), - ray::rpc::testing::RpcFailure::None); + ASSERT_EQ(GetRpcFailure("method1"), RpcFailure::Request); + ASSERT_EQ(GetRpcFailure("method2"), RpcFailure::Response); + ASSERT_EQ(GetRpcFailure("method3"), RpcFailure::None); } } + +TEST(RpcChaosTest, WildcardRpcFailure) { + RayConfig::instance().testing_rpc_failure() = "*=-1:100:0"; + Init(); + for (int i = 0; i < 100; i++) { + ASSERT_EQ(GetRpcFailure("method"), RpcFailure::Request); + } + + RayConfig::instance().testing_rpc_failure() = "*=-1:0:100"; + Init(); + for (int i = 0; i < 100; i++) { + ASSERT_EQ(GetRpcFailure("method"), RpcFailure::Response); + } + + RayConfig::instance().testing_rpc_failure() = "*=-1:0:0"; + Init(); + for (int i = 0; i < 100; i++) { + ASSERT_EQ(GetRpcFailure("method"), RpcFailure::None); + } +} + +} // namespace ray::rpc::testing From 7d5a29c4c5980b925503146b8e7345a64e671492 Mon Sep 17 00:00:00 2001 From: Potato Date: Thu, 11 Sep 2025 13:13:51 +0800 Subject: [PATCH 572/634] [DOC] Fix documentation issues in ray-observability directory (#56069) Signed-off-by: Potato Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: angelinalg <122562471+angelinalg@users.noreply.github.com> Co-authored-by: Douglas Strodtman Co-authored-by: Alan Guo --- .../ray-observability/getting-started.rst | 4 ++-- .../{post-moretem.gif => post-mortem.gif} | Bin doc/source/ray-observability/key-concepts.rst | 2 +- .../ray-distributed-debugger.rst | 2 +- .../reference/system-metrics.rst | 4 ++-- .../ray-observability/user-guides/cli-sdk.rst | 7 ++++-- .../user-guides/configure-logging.md | 2 +- .../user-guides/debug-apps/debug-failures.rst | 8 +++---- .../user-guides/profiling.md | 20 +++++++++--------- 9 files changed, 26 insertions(+), 23 deletions(-) rename doc/source/ray-observability/images/{post-moretem.gif => post-mortem.gif} (100%) diff --git a/doc/source/ray-observability/getting-started.rst b/doc/source/ray-observability/getting-started.rst index 69ddbe76e7cb..ba66257146e9 100644 --- a/doc/source/ray-observability/getting-started.rst +++ b/doc/source/ray-observability/getting-started.rst @@ -130,7 +130,7 @@ Task Timeline First, download the chrome tracing file by clicking the download button. Alternatively, you can :ref:`use CLI or SDK to export the tracing file `. -Second, use tools like ``chrome://tracing`` or the `Perfetto UI `_ and drop the downloaded chrome tracing file. We will use the Perfetto as it is the recommendation way to visualize chrome tracing files. +Second, use tools like ``chrome://tracing`` or the `Perfetto UI `_ and drop the downloaded chrome tracing file. We will use Perfetto as it is the recommended way to visualize chrome tracing files. In the timeline visualization of Ray Tasks and Actors, there are Node rows (hardware) and Worker rows (processes). Each Worker rows display a list of Task events (e.g., Task scheduled, Task running, input/output deserialization, etc.) happening from that Worker over time. @@ -311,7 +311,7 @@ Additionally, users can see a snapshot of hardware utilization from the :ref:`Cl View the resource utilization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Ray requires users to specify the number of :ref:`resources ` their Tasks and Actors to use through arguments such as ``num_cpus``, ``num_gpus``, ``memory``, and ``resource``. +Ray requires users to specify the number of :ref:`resources ` their Tasks and Actors use through arguments such as ``num_cpus``, ``num_gpus``, ``memory``, and ``resource``. These values are used for scheduling, but may not always match the actual resource utilization (physical resource utilization). - See the logical and physical resource utilization over time from the :ref:`Metrics view `. diff --git a/doc/source/ray-observability/images/post-moretem.gif b/doc/source/ray-observability/images/post-mortem.gif similarity index 100% rename from doc/source/ray-observability/images/post-moretem.gif rename to doc/source/ray-observability/images/post-mortem.gif diff --git a/doc/source/ray-observability/key-concepts.rst b/doc/source/ray-observability/key-concepts.rst index 00723f3056aa..0273a0132791 100644 --- a/doc/source/ray-observability/key-concepts.rst +++ b/doc/source/ray-observability/key-concepts.rst @@ -73,7 +73,7 @@ View :ref:`Ray Debugger ` for more details. Profiling --------- -Profiling is way of analyzing the performance of an application by sampling the resource usage of it. Ray supports various profiling tools: +Profiling is a way of analyzing the performance of an application by sampling the resource usage of it. Ray supports various profiling tools: - CPU profiling for Driver and Worker processes, including integration with :ref:`py-spy ` and :ref:`cProfile ` - Memory profiling for Driver and Worker processes with :ref:`memray ` diff --git a/doc/source/ray-observability/ray-distributed-debugger.rst b/doc/source/ray-observability/ray-distributed-debugger.rst index ca1a545deb7b..461fd7db91a1 100644 --- a/doc/source/ray-observability/ray-distributed-debugger.rst +++ b/doc/source/ray-observability/ray-distributed-debugger.rst @@ -199,7 +199,7 @@ When the app throws an exception: - The paused task is listed in the Ray Debugger extension. - Click the play icon next to the name of the paused task to attach the debugger and start debugging. -.. image:: ./images/post-moretem.gif +.. image:: ./images/post-mortem.gif :align: center diff --git a/doc/source/ray-observability/reference/system-metrics.rst b/doc/source/ray-observability/reference/system-metrics.rst index 7cacfe249df5..b7ba9d889f27 100644 --- a/doc/source/ray-observability/reference/system-metrics.rst +++ b/doc/source/ray-observability/reference/system-metrics.rst @@ -6,7 +6,7 @@ Ray exports a number of system metrics, which provide introspection into the sta .. note:: - Certain labels are common across all metrics, such as `SessionName` (uniquely identifies a Ray cluster instance), `instance` (per-node label applied by Prometheus, and `JobId` (Ray job id, as applicable). + Certain labels are common across all metrics, such as `SessionName` (uniquely identifies a Ray cluster instance), `instance` (per-node label applied by Prometheus), and `JobId` (Ray job ID, as applicable). .. list-table:: Ray System Metrics :header-rows: 1 @@ -22,7 +22,7 @@ Ray exports a number of system metrics, which provide introspection into the sta - Current number of actors in a particular state. The State label is described by `rpc::ActorTableData `_ proto in gcs.proto. The actor class name is available in the Name label. * - `ray_resources` - `Name`, `State`, `InstanceId` - - Logical resource usage for each node of the cluster. Each resource has some quantity that is `in either `_ USED state vs AVAILABLE state. The Name label defines the resource name (e.g., CPU, GPU). + - Logical resource usage for each node of the cluster. Each resource has some quantity that is in either `USED or AVAILABLE state `_. The Name label defines the resource name (e.g., CPU, GPU). * - `ray_object_store_memory` - `Location`, `ObjectState`, `InstanceId` - Object store memory usage in bytes, `broken down `_ by logical Location (SPILLED, MMAP_DISK, MMAP_SHM, and WORKER_HEAP). Definitions are as follows. SPILLED--Objects that have spilled to disk or a remote Storage solution (for example, AWS S3). The default is the disk. MMAP_DISK--Objects stored on a memory-mapped page on disk. This mode very slow and only happens under severe memory pressure. MMAP_SHM--Objects store on a memory-mapped page in Shared Memory. This mode is the default, in the absence of memory pressure. WORKER_HEAP--Objects, usually smaller, stored in the memory of the Ray Worker process itself. Small objects are stored in the worker heap. diff --git a/doc/source/ray-observability/user-guides/cli-sdk.rst b/doc/source/ray-observability/user-guides/cli-sdk.rst index 11d34759fa81..5be71ed9f913 100644 --- a/doc/source/ray-observability/user-guides/cli-sdk.rst +++ b/doc/source/ray-observability/user-guides/cli-sdk.rst @@ -317,8 +317,11 @@ you can use ``list`` or ``get`` APIs to get more details for an individual abnor .. note:: By default, objects are summarized by callsite. However, callsite is not recorded by Ray by default. - To get callsite info, set env variable `RAY_record_ref_creation_sites=1` when starting the Ray Cluster - RAY_record_ref_creation_sites=1 ray start --head + To get callsite info, set env variable `RAY_record_ref_creation_sites=1` when starting the Ray cluster: + + .. code-block:: bash + + RAY_record_ref_creation_sites=1 ray start --head .. tab-set:: diff --git a/doc/source/ray-observability/user-guides/configure-logging.md b/doc/source/ray-observability/user-guides/configure-logging.md index 0d932be05fd3..d74daf7c442c 100644 --- a/doc/source/ray-observability/user-guides/configure-logging.md +++ b/doc/source/ray-observability/user-guides/configure-logging.md @@ -594,4 +594,4 @@ The max size of a log file, including its backup, is `RAY_ROTATION_MAX_BYTES * R ## Log persistence -To process and export logs to external storage or management systems, view {ref}`log persistence on Kubernetes ` see {ref}`log persistence on VMs ` for more details. +To process and export logs to external storage or management systems, see {ref}`log persistence on Kubernetes ` and {ref}`log persistence on VMs ` for more details. diff --git a/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst b/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst index b1b12d46b0e6..6ac6a695a26a 100644 --- a/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst +++ b/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst @@ -61,8 +61,8 @@ Many Python developers use a debugger to debug Python programs, and `Python pdb Ray has native integration to ``pdb``. You can simply add ``breakpoint()`` to Actors and Tasks code to enable ``pdb``. View :ref:`Ray Debugger ` for more details. -Running out of file descriptors (``Too may open files``) --------------------------------------------------------- +Running out of file descriptors (``Too many open files``) +--------------------------------------------------------- In a Ray cluster, arbitrary two system components can communicate with each other and make 1 or more connections. For example, some workers may need to communicate with GCS to schedule Actors (worker <-> GCS connection). @@ -76,7 +76,7 @@ more than 1024 connections to the component, it can raise error messages below. .. code-block:: bash - Too may open files + Too many open files It is especially common for the head node GCS process because it is a centralized component that many other components in Ray communicate with. When you see this error message, @@ -119,4 +119,4 @@ View :ref:`debugging memory issues ` for more details. This document discusses some common problems that people run into when using Ray as well as some known problems. If you encounter other problems, `let us know`_. -.. _`let us know`: https://github.com/ray-project/ray/issues \ No newline at end of file +.. _`let us know`: https://github.com/ray-project/ray/issues diff --git a/doc/source/ray-observability/user-guides/profiling.md b/doc/source/ray-observability/user-guides/profiling.md index 6068a97fa1f3..a5814e309163 100644 --- a/doc/source/ray-observability/user-guides/profiling.md +++ b/doc/source/ray-observability/user-guides/profiling.md @@ -102,12 +102,12 @@ ray.init() @ray.remote(num_gpus=1, runtime_env={ "nsight": "default"}) class RayActor: - def run(): - a = torch.tensor([1.0, 2.0, 3.0]).cuda() - b = torch.tensor([4.0, 5.0, 6.0]).cuda() - c = a * b + def run(self): + a = torch.tensor([1.0, 2.0, 3.0]).cuda() + b = torch.tensor([4.0, 5.0, 6.0]).cuda() + c = a * b - print("Result on GPU:", c) + print("Result on GPU:", c) ray_actor = RayActor.remote() # The Actor or Task process runs with : "nsys profile [default options] ..." @@ -135,12 +135,12 @@ runtime_env={ "nsight": { "cuda-graph-trace": "graph", }}) class RayActor: - def run(): - a = torch.tensor([1.0, 2.0, 3.0]).cuda() - b = torch.tensor([4.0, 5.0, 6.0]).cuda() - c = a * b + def run(self): + a = torch.tensor([1.0, 2.0, 3.0]).cuda() + b = torch.tensor([4.0, 5.0, 6.0]).cuda() + c = a * b - print("Result on GPU:", c) + print("Result on GPU:", c) ray_actor = RayActor.remote() From 6e0e96cc285aadaf7597bfab25ca393d2cb77aa9 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 11 Sep 2025 07:06:15 -0700 Subject: [PATCH 573/634] [release] Add base image build step to release configs (#56438) Separated from https://github.com/ray-project/ray/pull/56233/files. Added these base image step keys so they can be referenced in release test launching. --------- Signed-off-by: kevin --- ci/ray_ci/oss_config.yaml | 4 ++++ release/ray_release/configs/global_config.py | 10 ++++++++++ release/ray_release/tests/test_global_config.py | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/ci/ray_ci/oss_config.yaml b/ci/ray_ci/oss_config.yaml index 1eb2fa0d0a38..ffaa50fa7158 100644 --- a/ci/ray_ci/oss_config.yaml +++ b/ci/ray_ci/oss_config.yaml @@ -23,3 +23,7 @@ state_machine: aws_bucket: ray-ci-pr-results branch: aws_bucket: ray-ci-results +release_image_step: + ray: anyscalebuild + ray_ml: anyscalemlbuild + ray_llm: anyscalellmbuild diff --git a/release/ray_release/configs/global_config.py b/release/ray_release/configs/global_config.py index c3a07375a2d2..88eb6656c73a 100644 --- a/release/ray_release/configs/global_config.py +++ b/release/ray_release/configs/global_config.py @@ -21,6 +21,9 @@ class GlobalConfig(TypedDict): ci_pipeline_premerge: List[str] ci_pipeline_postmerge: List[str] ci_pipeline_buildkite_secret: str + release_image_step_ray: str + release_image_step_ray_ml: str + release_image_step_ray_llm: str config = None @@ -106,6 +109,13 @@ def _init_global_config(config_file: str): "buildkite_secret" ), kuberay_disabled=config_content.get("kuberay", {}).get("disabled", 0) == 1, + release_image_step_ray=config_content.get("release_image_step", {}).get("ray"), + release_image_step_ray_ml=config_content.get("release_image_step", {}).get( + "ray_ml" + ), + release_image_step_ray_llm=config_content.get("release_image_step", {}).get( + "ray_llm" + ), ) # setup GCP workload identity federation os.environ[ diff --git a/release/ray_release/tests/test_global_config.py b/release/ray_release/tests/test_global_config.py index 439dc25c3b14..ce60f26d3175 100644 --- a/release/ray_release/tests/test_global_config.py +++ b/release/ray_release/tests/test_global_config.py @@ -33,6 +33,10 @@ postmerge: - hi - three +release_image_step: + ray: anyscalebuild + ray_ml: anyscalemlbuild + ray_llm: anyscalellmbuild """ @@ -56,6 +60,9 @@ def test_init_global_config() -> None: assert config["byod_ray_cr_repo"] == "ray" assert config["byod_ray_ml_cr_repo"] == "ray-ml" assert config["byod_ray_llm_cr_repo"] == "ray-llm" + assert config["release_image_step_ray"] == "anyscalebuild" + assert config["release_image_step_ray_ml"] == "anyscalemlbuild" + assert config["release_image_step_ray_llm"] == "anyscalellmbuild" if __name__ == "__main__": From aadd7940b7990d51d237d2b1fa4bf22f9d9a0fc2 Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Thu, 11 Sep 2025 08:02:25 -0700 Subject: [PATCH 574/634] [core] Breaking up task_common + lease target no longer depends on task_common (#56371) Closes #55922 --------- Signed-off-by: joshlee --- src/fakes/ray/rpc/raylet/BUILD.bazel | 3 + src/fakes/ray/rpc/raylet/raylet_client.h | 2 + src/ray/common/BUILD.bazel | 96 ++++++-- src/ray/common/bundle_spec.cc | 57 +---- src/ray/common/bundle_spec.h | 35 +-- src/ray/common/function_descriptor.cc | 2 + src/ray/common/lease/lease_spec.cc | 14 +- src/ray/common/lease/lease_spec.h | 6 +- src/ray/common/placement_group.cc | 2 +- src/ray/common/placement_group.h | 5 +- src/ray/common/scheduling/BUILD.bazel | 110 +++++++++ .../scheduling/cluster_resource_data.cc | 9 +- .../common/scheduling/cluster_resource_data.h | 7 +- src/ray/common/scheduling/label_selector.cc | 1 - .../common/scheduling/placement_group_util.cc | 77 +++++++ .../common/scheduling/placement_group_util.h | 50 ++++ .../scheduling/resource_instance_set.cc | 5 +- .../common/scheduling/resource_instance_set.h | 1 + src/ray/common/scheduling/resource_set.cc | 3 - .../scheduling/scheduling_class_util.cc | 169 ++++++++++++++ .../common/scheduling/scheduling_class_util.h | 170 ++++++++++++++ src/ray/common/scheduling/scheduling_ids.cc | 4 + src/ray/common/scheduling/scheduling_ids.h | 4 +- src/ray/common/scheduling/tests/BUILD.bazel | 67 ++++++ .../tests/label_selector_test.cc | 0 .../tests/resource_instance_set_test.cc | 0 .../tests/resource_request_test.cc | 0 .../tests/resource_set_test.cc | 0 .../tests/scheduling_ids_test.cc | 3 +- src/ray/common/task/task_spec.cc | 56 +---- src/ray/common/task/task_spec.h | 216 +----------------- src/ray/common/tests/BUILD.bazel | 68 +----- src/ray/common/tests/task_spec_test.cc | 40 ++-- src/ray/core_worker/BUILD.bazel | 3 - src/ray/core_worker/common.cc | 1 - src/ray/core_worker/common.h | 1 - src/ray/core_worker/core_worker_options.h | 1 - .../store_provider/plasma_store_provider.h | 1 - .../core_worker/task_execution/BUILD.bazel | 3 - .../concurrency_group_manager.cc | 1 - .../concurrency_group_manager.h | 1 + .../out_of_order_actor_scheduling_queue.h | 1 - .../task_execution/task_receiver.h | 6 - src/ray/core_worker/tests/BUILD.bazel | 4 - .../core_worker/tests/actor_manager_test.cc | 1 - .../tests/object_recovery_manager_test.cc | 2 - .../task_event_buffer_export_event_test.cc | 3 - src/ray/gcs/gcs_client/BUILD.bazel | 1 + src/ray/gcs/gcs_server/BUILD.bazel | 16 +- src/ray/gcs/gcs_server/gcs_actor_scheduler.h | 5 - .../gcs_server/gcs_placement_group_manager.cc | 3 +- .../gcs_server/gcs_placement_group_manager.h | 4 - .../gcs_placement_group_scheduler.h | 1 - src/ray/gcs/gcs_server/gcs_resource_manager.h | 2 - src/ray/internal/internal.h | 2 +- src/ray/raylet/BUILD.bazel | 3 +- src/ray/raylet/local_lease_manager.cc | 13 +- src/ray/raylet/scheduling/BUILD.bazel | 24 +- .../scheduling/cluster_lease_manager.cc | 3 +- .../raylet/scheduling/cluster_lease_manager.h | 1 - src/ray/raylet/scheduling/internal.h | 1 - .../local_lease_manager_interface.h | 3 +- .../scheduling/local_resource_manager.cc | 5 +- .../scheduling/local_resource_manager.h | 3 - .../policy/bundle_scheduling_policy.h | 2 - .../scheduling/policy/scheduling_context.h | 3 - src/ray/raylet/scheduling/policy/scorer.cc | 2 - src/ray/raylet/scheduling/policy/scorer.h | 1 - .../scheduling/scheduler_resource_reporter.cc | 6 +- .../scheduling/scheduler_resource_reporter.h | 2 - src/ray/raylet/scheduling/scheduler_stats.h | 4 - src/ray/raylet/scheduling/scheduling_policy.h | 1 - src/ray/raylet/scheduling/tests/BUILD.bazel | 1 + src/ray/raylet/tests/BUILD.bazel | 2 +- .../tests/lease_dependency_manager_test.cc | 1 - .../placement_group_resource_manager_test.cc | 1 + src/ray/raylet/worker.cc | 4 +- src/ray/raylet/worker.h | 8 +- src/ray/rpc/BUILD.bazel | 3 +- src/ray/rpc/raylet/raylet_client_interface.h | 15 +- 80 files changed, 852 insertions(+), 605 deletions(-) create mode 100644 src/ray/common/scheduling/BUILD.bazel create mode 100644 src/ray/common/scheduling/placement_group_util.cc create mode 100644 src/ray/common/scheduling/placement_group_util.h create mode 100644 src/ray/common/scheduling/scheduling_class_util.cc create mode 100644 src/ray/common/scheduling/scheduling_class_util.h create mode 100644 src/ray/common/scheduling/tests/BUILD.bazel rename src/ray/common/{ => scheduling}/tests/label_selector_test.cc (100%) rename src/ray/common/{ => scheduling}/tests/resource_instance_set_test.cc (100%) rename src/ray/common/{ => scheduling}/tests/resource_request_test.cc (100%) rename src/ray/common/{ => scheduling}/tests/resource_set_test.cc (100%) rename src/ray/common/{ => scheduling}/tests/scheduling_ids_test.cc (97%) diff --git a/src/fakes/ray/rpc/raylet/BUILD.bazel b/src/fakes/ray/rpc/raylet/BUILD.bazel index af2537ff4ab0..fc6b5a141289 100644 --- a/src/fakes/ray/rpc/raylet/BUILD.bazel +++ b/src/fakes/ray/rpc/raylet/BUILD.bazel @@ -4,6 +4,9 @@ ray_cc_library( name = "fake_raylet_client", hdrs = ["raylet_client.h"], deps = [ + "//src/ray/common:id", + "//src/ray/common:status", + "//src/ray/common/scheduling:scheduling_ids", "//src/ray/rpc:raylet_client_interface", ], ) diff --git a/src/fakes/ray/rpc/raylet/raylet_client.h b/src/fakes/ray/rpc/raylet/raylet_client.h index 4b4e048cafd7..3680819c04fe 100644 --- a/src/fakes/ray/rpc/raylet/raylet_client.h +++ b/src/fakes/ray/rpc/raylet/raylet_client.h @@ -16,6 +16,8 @@ #include "ray/common/scheduling/scheduling_ids.h" #include "ray/rpc/raylet/raylet_client_interface.h" +#include "src/ray/common/id.h" +#include "src/ray/common/status.h" namespace ray { diff --git a/src/ray/common/BUILD.bazel b/src/ray/common/BUILD.bazel index 63c10cc49e43..f23c1e24ad6b 100644 --- a/src/ray/common/BUILD.bazel +++ b/src/ray/common/BUILD.bazel @@ -17,6 +17,7 @@ ray_cc_library( deps = [ ":asio", ":id", + ":placement_group", ":ray_object", ":task_common", "//src/ray/protobuf:autoscaler_cc_grpc", @@ -150,53 +151,94 @@ ray_cc_library( ], ) -# TODO(#55922): split out the scheduling dependencies into a separate bazel target -# and have lease and task bazel targets depend on this - ray_cc_library( - name = "task_common", + name = "bundle_spec", srcs = [ - "bundle_location_index.cc", "bundle_spec.cc", - "function_descriptor.cc", - "placement_group.cc", - "scheduling/cluster_resource_data.cc", - "scheduling/fixed_point.cc", - "scheduling/label_selector.cc", - "scheduling/resource_instance_set.cc", - "scheduling/resource_set.cc", - "scheduling/scheduling_ids.cc", - "task/task_spec.cc", ], hdrs = [ - "bundle_location_index.h", "bundle_spec.h", - "function_descriptor.h", + ], + deps = [ + ":grpc_util", + ":id", + "//src/ray/common/scheduling:cluster_resource_data", + "//src/ray/common/scheduling:label_selector", + "//src/ray/common/scheduling:placement_group_util", + "//src/ray/common/scheduling:scheduling_ids", + "//src/ray/protobuf:common_cc_proto", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_protobuf//:protobuf", + ], +) + +ray_cc_library( + name = "placement_group", + srcs = [ + "placement_group.cc", + ], + hdrs = [ "placement_group.h", - "scheduling/cluster_resource_data.h", - "scheduling/fixed_point.h", - "scheduling/label_selector.h", - "scheduling/resource_instance_set.h", - "scheduling/resource_set.h", - "scheduling/scheduling_ids.h", + ], + deps = [ + ":bundle_spec", + ":id", + "//src/ray/protobuf:common_cc_proto", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_protobuf//:protobuf", + ], +) + +ray_cc_library( + name = "function_descriptor", + srcs = ["function_descriptor.cc"], + hdrs = ["function_descriptor.h"], + deps = [ + ":grpc_util", + "//src/ray/protobuf:common_cc_proto", + "//src/ray/util:logging", + "@com_google_absl//absl/strings:str_format", + ], +) + +ray_cc_library( + name = "bundle_location_index", + srcs = ["bundle_location_index.cc"], + hdrs = ["bundle_location_index.h"], + deps = [ + ":id", + ":placement_group", + "//src/ray/protobuf:gcs_cc_proto", + "@com_google_absl//absl/container:flat_hash_map", + ], +) + +ray_cc_library( + name = "task_common", + srcs = [ + "task/task_spec.cc", + ], + hdrs = [ "task/task_common.h", "task/task_spec.h", "task/task_util.h", ], deps = [ ":event_stats", + ":function_descriptor", ":grpc_util", - ":id", ":ray_config", ":ray_object", ":runtime_env", + "//src/ray/common/scheduling:label_selector", + "//src/ray/common/scheduling:resource_set", + "//src/ray/common/scheduling:scheduling_class_util", "//src/ray/flatbuffers:node_manager_generated", "//src/ray/util:container_util", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/synchronization", ], ) @@ -210,8 +252,12 @@ ray_cc_library( "lease/lease_spec.h", ], deps = [ + ":function_descriptor", ":id", - ":task_common", + ":runtime_env", + "//src/ray/common/scheduling:label_selector", + "//src/ray/common/scheduling:resource_set", + "//src/ray/common/scheduling:scheduling_class_util", "//src/ray/protobuf:common_cc_proto", ], ) diff --git a/src/ray/common/bundle_spec.cc b/src/ray/common/bundle_spec.cc index 111765363b63..336f8906ab11 100644 --- a/src/ray/common/bundle_spec.cc +++ b/src/ray/common/bundle_spec.cc @@ -14,6 +14,10 @@ #include "ray/common/bundle_spec.h" +#include "ray/common/scheduling/label_selector.h" +#include "ray/common/scheduling/placement_group_util.h" +#include "ray/common/scheduling/scheduling_ids.h" + namespace ray { void BundleSpecification::ComputeResources() { @@ -142,59 +146,6 @@ std::string GetOriginalResourceNameFromWildcardResource(const std::string &resou } } -bool IsCPUOrPlacementGroupCPUResource(ResourceID resource_id) { - // Check whether the resource is CPU resource or CPU resource inside PG. - if (resource_id == ResourceID::CPU()) { - return true; - } - - auto possible_pg_resource = ParsePgFormattedResource(resource_id.Binary(), - /*for_wildcard_resource*/ true, - /*for_indexed_resource*/ true); - if (possible_pg_resource.has_value() && - possible_pg_resource->original_resource == ResourceID::CPU().Binary()) { - return true; - } - - return false; -} - -std::optional ParsePgFormattedResource( - const std::string &resource, bool for_wildcard_resource, bool for_indexed_resource) { - // Check if it is a wildcard pg resource. - PgFormattedResourceData data; - std::smatch match_groups; - RAY_CHECK(for_wildcard_resource || for_indexed_resource) - << "Either one of for_wildcard_resource or for_indexed_resource must be true"; - - if (for_wildcard_resource) { - static const std::regex wild_card_resource_pattern("^(.*)_group_([0-9a-f]+)$"); - - if (std::regex_match(resource, match_groups, wild_card_resource_pattern) && - match_groups.size() == 3) { - data.original_resource = match_groups[1].str(); - data.bundle_index = -1; - data.group_id = match_groups[2].str(); - return data; - } - } - - // Check if it is a regular pg resource. - if (for_indexed_resource) { - static const std::regex pg_resource_pattern("^(.+)_group_(\\d+)_([0-9a-zA-Z]+)"); - if (std::regex_match(resource, match_groups, pg_resource_pattern) && - match_groups.size() == 4) { - data.original_resource = match_groups[1].str(); - data.bundle_index = stoi(match_groups[2].str()); - data.group_id = match_groups[3].str(); - return data; - } - } - - // If it is not a wildcard or pg formatted resource, return nullopt. - return {}; -} - std::string GetDebugStringForBundles( const std::vector> &bundles) { std::ostringstream debug_info; diff --git a/src/ray/common/bundle_spec.h b/src/ray/common/bundle_spec.h index 63c9eea454b9..4b59d1895fe3 100644 --- a/src/ray/common/bundle_spec.h +++ b/src/ray/common/bundle_spec.h @@ -14,23 +14,22 @@ #pragma once -#include -#include +#include #include +#include #include -#include "absl/synchronization/mutex.h" -#include "ray/common/function_descriptor.h" +#include "absl/container/flat_hash_map.h" #include "ray/common/grpc_util.h" #include "ray/common/id.h" #include "ray/common/scheduling/cluster_resource_data.h" -#include "ray/common/task/task_common.h" +#include "src/ray/protobuf/common.pb.h" namespace ray { /// Arguments are the node ID to spill back to, the raylet's /// address and the raylet's port. -typedef std::function SpillbackBundleCallback; +using SpillbackBundleCallback = std::function; const std::string kGroupKeyword = "_group_"; const size_t kGroupKeywordSize = kGroupKeyword.size(); @@ -93,13 +92,6 @@ class BundleSpecification : public MessageWrapper { absl::flat_hash_map bundle_resource_labels_; }; -struct PgFormattedResourceData { - std::string original_resource; - /// -1 if it is a wildcard resource. - int64_t bundle_index; - std::string group_id; -}; - /// Format a placement group resource with provided parameters. /// /// \param original_resource_name The original resource name of the pg resource. @@ -126,23 +118,6 @@ std::string GetOriginalResourceName(const std::string &resource); // Returns "" if the resource is not a wildcard resource. std::string GetOriginalResourceNameFromWildcardResource(const std::string &resource); -/// Return whether the resource specified by the resource_id is a CPU resource -/// or CPU resource inside a placement group. -bool IsCPUOrPlacementGroupCPUResource(ResourceID resource_id); - -/// Parse the given resource and get the pg related information. -/// -/// \param resource name of the resource. -/// \param for_wildcard_resource if true, it parses wildcard pg resources. -/// E.g., [resource]_group_[pg_id] -/// \param for_indexed_resource if true, it parses indexed pg resources. -/// E.g., [resource]_group_[index]_[pg_id] -/// \return nullopt if it is not a pg resource. Otherwise, it returns the -/// struct with pg information parsed from the resource. -/// If a returned bundle index is -1, it means the resource is the wildcard resource. -std::optional ParsePgFormattedResource( - const std::string &resource, bool for_wildcard_resource, bool for_indexed_resource); - /// Generate debug information of given bundles. std::string GetDebugStringForBundles( const std::vector> &bundles); diff --git a/src/ray/common/function_descriptor.cc b/src/ray/common/function_descriptor.cc index 22a997932266..8df6c3e1ee1f 100644 --- a/src/ray/common/function_descriptor.cc +++ b/src/ray/common/function_descriptor.cc @@ -14,6 +14,8 @@ #include "ray/common/function_descriptor.h" +#include "ray/util/logging.h" + namespace ray { FunctionDescriptor FunctionDescriptorBuilder::Empty() { static ray::FunctionDescriptor empty = diff --git a/src/ray/common/lease/lease_spec.cc b/src/ray/common/lease/lease_spec.cc index 967ce54b284a..7d84ebd92144 100644 --- a/src/ray/common/lease/lease_spec.cc +++ b/src/ray/common/lease/lease_spec.cc @@ -19,10 +19,12 @@ namespace ray { +using SchedulingClass = int; + LeaseSpecification::LeaseSpecification(const rpc::TaskSpec &task_spec) : MessageWrapper(std::make_shared()) { - RAY_CHECK(task_spec.type() == TaskType::NORMAL_TASK || - task_spec.type() == TaskType::ACTOR_CREATION_TASK); + RAY_CHECK(task_spec.type() == rpc::TaskType::NORMAL_TASK || + task_spec.type() == rpc::TaskType::ACTOR_CREATION_TASK); message_->set_job_id(task_spec.job_id()); message_->mutable_caller_address()->CopyFrom(task_spec.caller_address()); message_->mutable_required_resources()->insert(task_spec.required_resources().begin(), @@ -73,14 +75,14 @@ const rpc::Address &LeaseSpecification::CallerAddress() const { return message_->caller_address(); } -Language LeaseSpecification::GetLanguage() const { return message_->language(); } +rpc::Language LeaseSpecification::GetLanguage() const { return message_->language(); } bool LeaseSpecification::IsNormalTask() const { - return message_->type() == TaskType::NORMAL_TASK; + return message_->type() == rpc::TaskType::NORMAL_TASK; } bool LeaseSpecification::IsActorCreationTask() const { - return message_->type() == TaskType::ACTOR_CREATION_TASK; + return message_->type() == rpc::TaskType::ACTOR_CREATION_TASK; } bool LeaseSpecification::IsNodeAffinitySchedulingStrategy() const { @@ -321,7 +323,7 @@ void LeaseSpecification::ComputeResources() { auto sched_cls_desc = SchedulingClassDescriptor( resource_set, label_selector, function_descriptor, depth, GetSchedulingStrategy()); // Map the scheduling class descriptor to an integer for performance. - sched_cls_id_ = TaskSpecification::GetSchedulingClass(sched_cls_desc); + sched_cls_id_ = SchedulingClassToIds::GetSchedulingClass(sched_cls_desc); RAY_CHECK_GT(sched_cls_id_, 0); runtime_env_hash_ = CalculateRuntimeEnvHash(SerializedRuntimeEnv()); diff --git a/src/ray/common/lease/lease_spec.h b/src/ray/common/lease/lease_spec.h index 6a2c566b795f..ab507a4e5544 100644 --- a/src/ray/common/lease/lease_spec.h +++ b/src/ray/common/lease/lease_spec.h @@ -15,17 +15,15 @@ #pragma once #include -#include #include #include #include -#include "absl/types/optional.h" #include "ray/common/grpc_util.h" #include "ray/common/id.h" #include "ray/common/scheduling/label_selector.h" #include "ray/common/scheduling/resource_set.h" -#include "ray/common/task/task_spec.h" +#include "ray/common/scheduling/scheduling_class_util.h" #include "src/ray/protobuf/common.pb.h" namespace ray { @@ -75,7 +73,7 @@ class LeaseSpecification : public MessageWrapper { bool IsDetachedActor() const; std::string DebugString() const; int GetRuntimeEnvHash() const; - Language GetLanguage() const; + rpc::Language GetLanguage() const; bool HasRuntimeEnv() const; const rpc::RuntimeEnvInfo &RuntimeEnvInfo() const; const std::string &SerializedRuntimeEnv() const; diff --git a/src/ray/common/placement_group.cc b/src/ray/common/placement_group.cc index 15c1d825aa15..a0ec994088f3 100644 --- a/src/ray/common/placement_group.cc +++ b/src/ray/common/placement_group.cc @@ -17,7 +17,7 @@ namespace ray { void PlacementGroupSpecification::ConstructBundles() { for (int i = 0; i < message_->bundles_size(); i++) { - bundles_.push_back(BundleSpecification(message_->bundles(i))); + bundles_.emplace_back(message_->bundles(i)); } } diff --git a/src/ray/common/placement_group.h b/src/ray/common/placement_group.h index f053a917ac70..c3d0057d88b2 100644 --- a/src/ray/common/placement_group.h +++ b/src/ray/common/placement_group.h @@ -14,6 +14,7 @@ #pragma once +#include "absl/container/flat_hash_map.h" #include "ray/common/bundle_spec.h" #include "ray/common/grpc_util.h" #include "ray/common/id.h" @@ -40,14 +41,14 @@ class PlacementGroupSpecification : public MessageWrapper message) - : MessageWrapper(message) { + : MessageWrapper(std::move(message)) { ConstructBundles(); } /// Return the placement group id. diff --git a/src/ray/common/scheduling/BUILD.bazel b/src/ray/common/scheduling/BUILD.bazel new file mode 100644 index 000000000000..baac27e1096f --- /dev/null +++ b/src/ray/common/scheduling/BUILD.bazel @@ -0,0 +1,110 @@ +load("//bazel:ray.bzl", "ray_cc_library") + +ray_cc_library( + name = "scheduling_ids", + srcs = ["scheduling_ids.cc"], + hdrs = ["scheduling_ids.h"], + deps = [ + "//src/ray/common:constants", + "//src/ray/common:ray_config", + "//src/ray/util:logging", + "@boost//:algorithm", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + ], +) + +ray_cc_library( + name = "label_selector", + srcs = ["label_selector.cc"], + hdrs = ["label_selector.h"], + deps = [ + "//src/ray/protobuf:common_cc_proto", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/strings", + "@com_google_protobuf//:protobuf", + ], +) + +ray_cc_library( + name = "fixed_point", + srcs = ["fixed_point.cc"], + hdrs = ["fixed_point.h"], + deps = [ + "//src/ray/common:constants", + ], +) + +ray_cc_library( + name = "placement_group_util", + srcs = ["placement_group_util.cc"], + hdrs = ["placement_group_util.h"], + deps = [ + ":scheduling_ids", + "//src/ray/util:logging", + ], +) + +ray_cc_library( + name = "resource_set", + srcs = ["resource_set.cc"], + hdrs = ["resource_set.h"], + deps = [ + ":fixed_point", + ":scheduling_ids", + "@boost//:range", + "@com_google_absl//absl/container:flat_hash_map", + ], +) + +ray_cc_library( + name = "cluster_resource_data", + srcs = ["cluster_resource_data.cc"], + hdrs = ["cluster_resource_data.h"], + deps = [ + ":fixed_point", + ":label_selector", + ":resource_instance_set", + ":resource_set", + ":scheduling_ids", + "//src/ray/util:logging", + "@boost//:range", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/time", + ], +) + +ray_cc_library( + name = "scheduling_class_util", + srcs = ["scheduling_class_util.cc"], + hdrs = ["scheduling_class_util.h"], + deps = [ + ":label_selector", + ":resource_set", + "//src/ray/common:function_descriptor", + "//src/ray/common:runtime_env", + "//src/ray/protobuf:common_cc_proto", + "//src/ray/util:logging", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/synchronization", + "@com_google_protobuf//:protobuf", + ], +) + +ray_cc_library( + name = "resource_instance_set", + srcs = ["resource_instance_set.cc"], + hdrs = ["resource_instance_set.h"], + deps = [ + ":fixed_point", + ":placement_group_util", + ":resource_set", + ":scheduling_ids", + "//src/ray/util:container_util", + "//src/ray/util:logging", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", + ], +) diff --git a/src/ray/common/scheduling/cluster_resource_data.cc b/src/ray/common/scheduling/cluster_resource_data.cc index 7555a329879e..4028de3ee4c8 100644 --- a/src/ray/common/scheduling/cluster_resource_data.cc +++ b/src/ray/common/scheduling/cluster_resource_data.cc @@ -17,9 +17,6 @@ #include #include -#include "ray/common/bundle_spec.h" -#include "ray/common/scheduling/resource_set.h" - namespace ray { /// Convert a map of resources to a ResourceRequest data structure. @@ -27,7 +24,7 @@ ResourceRequest ResourceMapToResourceRequest( const absl::flat_hash_map &resource_map, bool requires_object_store_memory) { ResourceRequest res({}, requires_object_store_memory); - for (auto entry : resource_map) { + for (const auto &entry : resource_map) { res.Set(ResourceID(entry.first), FixedPoint(entry.second)); } return res; @@ -116,7 +113,7 @@ bool NodeResources::IsFeasible(const ResourceRequest &resource_request) const { bool NodeResources::HasRequiredLabels(const LabelSelector &label_selector) const { // Check if node labels satisfy all label constraints - const auto constraints = label_selector.GetConstraints(); + const auto &constraints = label_selector.GetConstraints(); for (const auto &constraint : constraints) { if (!NodeLabelMatchesConstraint(constraint)) { return false; @@ -175,7 +172,7 @@ std::string NodeResources::DebugString() const { std::string NodeResources::DictString() const { return DebugString(); } -bool NodeResourceInstances::operator==(const NodeResourceInstances &other) { +bool NodeResourceInstances::operator==(const NodeResourceInstances &other) const { return this->total == other.total && this->available == other.available; } diff --git a/src/ray/common/scheduling/cluster_resource_data.h b/src/ray/common/scheduling/cluster_resource_data.h index cb3b4e8028a4..4ed7b77a79b5 100644 --- a/src/ray/common/scheduling/cluster_resource_data.h +++ b/src/ray/common/scheduling/cluster_resource_data.h @@ -15,15 +15,14 @@ #pragma once #include -#include +#include #include #include #include #include #include "absl/container/flat_hash_map.h" -#include "absl/container/flat_hash_set.h" -#include "ray/common/id.h" +#include "absl/time/time.h" #include "ray/common/scheduling/fixed_point.h" #include "ray/common/scheduling/label_selector.h" #include "ray/common/scheduling/resource_instance_set.h" @@ -376,7 +375,7 @@ class NodeResourceInstances { const NodeResourceInstanceSet &GetAvailableResourceInstances() const; const NodeResourceInstanceSet &GetTotalResourceInstances() const; /// Returns if this equals another node resources. - bool operator==(const NodeResourceInstances &other); + bool operator==(const NodeResourceInstances &other) const; /// Returns human-readable string for these resources. [[nodiscard]] std::string DebugString() const; }; diff --git a/src/ray/common/scheduling/label_selector.cc b/src/ray/common/scheduling/label_selector.cc index 4b27d25955c3..a2255549d080 100644 --- a/src/ray/common/scheduling/label_selector.cc +++ b/src/ray/common/scheduling/label_selector.cc @@ -18,7 +18,6 @@ #include #include "absl/strings/match.h" -#include "ray/util/logging.h" namespace ray { diff --git a/src/ray/common/scheduling/placement_group_util.cc b/src/ray/common/scheduling/placement_group_util.cc new file mode 100644 index 000000000000..7fab5a56efff --- /dev/null +++ b/src/ray/common/scheduling/placement_group_util.cc @@ -0,0 +1,77 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/common/scheduling/placement_group_util.h" + +#include +#include + +#include "ray/util/logging.h" + +namespace ray { + +bool IsCPUOrPlacementGroupCPUResource(ResourceID resource_id) { + // Check whether the resource is CPU resource or CPU resource inside PG. + if (resource_id == ResourceID::CPU()) { + return true; + } + + auto possible_pg_resource = ParsePgFormattedResource(resource_id.Binary(), + /*for_wildcard_resource*/ true, + /*for_indexed_resource*/ true); + if (possible_pg_resource.has_value() && + possible_pg_resource->original_resource == ResourceID::CPU().Binary()) { + return true; + } + + return false; +} + +std::optional ParsePgFormattedResource( + const std::string &resource, bool for_wildcard_resource, bool for_indexed_resource) { + // Check if it is a wildcard pg resource. + PgFormattedResourceData data; + std::smatch match_groups; + RAY_CHECK(for_wildcard_resource || for_indexed_resource) + << "Either one of for_wildcard_resource or for_indexed_resource must be true"; + + if (for_wildcard_resource) { + static const std::regex wild_card_resource_pattern("^(.*)_group_([0-9a-f]+)$"); + + if (std::regex_match(resource, match_groups, wild_card_resource_pattern) && + match_groups.size() == 3) { + data.original_resource = match_groups[1].str(); + data.bundle_index = -1; + data.group_id = match_groups[2].str(); + return data; + } + } + + // Check if it is a regular pg resource. + if (for_indexed_resource) { + static const std::regex pg_resource_pattern("^(.+)_group_(\\d+)_([0-9a-zA-Z]+)"); + if (std::regex_match(resource, match_groups, pg_resource_pattern) && + match_groups.size() == 4) { + data.original_resource = match_groups[1].str(); + data.bundle_index = stoi(match_groups[2].str()); + data.group_id = match_groups[3].str(); + return data; + } + } + + // If it is not a wildcard or pg formatted resource, return nullopt. + return std::nullopt; +} + +} // namespace ray diff --git a/src/ray/common/scheduling/placement_group_util.h b/src/ray/common/scheduling/placement_group_util.h new file mode 100644 index 000000000000..56c2137c5cd9 --- /dev/null +++ b/src/ray/common/scheduling/placement_group_util.h @@ -0,0 +1,50 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "ray/common/scheduling/scheduling_ids.h" + +namespace ray { + +using scheduling::ResourceID; + +struct PgFormattedResourceData { + std::string original_resource; + /// -1 if it is a wildcard resource. + int64_t bundle_index; + std::string group_id; +}; + +/// Return whether the resource specified by the resource_id is a CPU resource +/// or CPU resource inside a placement group. +bool IsCPUOrPlacementGroupCPUResource(ResourceID resource_id); + +/// Parse the given resource and get the pg related information. +/// +/// \param resource name of the resource. +/// \param for_wildcard_resource if true, it parses wildcard pg resources. +/// E.g., [resource]_group_[pg_id] +/// \param for_indexed_resource if true, it parses indexed pg resources. +/// E.g., [resource]_group_[index]_[pg_id] +/// \return nullopt if it is not a pg resource. Otherwise, it returns the +/// struct with pg information parsed from the resource. +/// If a returned bundle index is -1, it means the resource is the wildcard resource. +std::optional ParsePgFormattedResource( + const std::string &resource, bool for_wildcard_resource, bool for_indexed_resource); + +} // namespace ray diff --git a/src/ray/common/scheduling/resource_instance_set.cc b/src/ray/common/scheduling/resource_instance_set.cc index ed07a13fe24b..e64d9b329aab 100644 --- a/src/ray/common/scheduling/resource_instance_set.cc +++ b/src/ray/common/scheduling/resource_instance_set.cc @@ -20,7 +20,7 @@ #include #include -#include "ray/common/bundle_spec.h" +#include "ray/common/scheduling/placement_group_util.h" #include "ray/util/container_util.h" #include "ray/util/logging.h" @@ -191,8 +191,7 @@ NodeResourceInstanceSet::TryAllocate(const ResourceSet &resource_demands) { if (data) { // Aggregate based on resource type ResourceID original_resource_id{data->original_resource}; - pg_resource_map[original_resource_id].push_back( - std::make_pair(resource_id, data.value())); + pg_resource_map[original_resource_id].emplace_back(resource_id, data.value()); } else { // Directly allocate the resources if the resource is not with a placement group auto allocation = TryAllocate(resource_id, demand); diff --git a/src/ray/common/scheduling/resource_instance_set.h b/src/ray/common/scheduling/resource_instance_set.h index 61ad263a59fb..f49b2d01fccf 100644 --- a/src/ray/common/scheduling/resource_instance_set.h +++ b/src/ray/common/scheduling/resource_instance_set.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include diff --git a/src/ray/common/scheduling/resource_set.cc b/src/ray/common/scheduling/resource_set.cc index b33e82e97907..871b6655ff2d 100644 --- a/src/ray/common/scheduling/resource_set.cc +++ b/src/ray/common/scheduling/resource_set.cc @@ -14,14 +14,11 @@ #include "ray/common/scheduling/resource_set.h" -#include #include #include #include #include -#include "ray/util/logging.h" - namespace ray { ResourceSet::ResourceSet( diff --git a/src/ray/common/scheduling/scheduling_class_util.cc b/src/ray/common/scheduling/scheduling_class_util.cc new file mode 100644 index 000000000000..0e1248fe569e --- /dev/null +++ b/src/ray/common/scheduling/scheduling_class_util.cc @@ -0,0 +1,169 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/common/scheduling/scheduling_class_util.h" + +#include +#include +#include + +#include "google/protobuf/util/message_differencer.h" +#include "ray/common/runtime_env_common.h" +#include "ray/util/logging.h" + +namespace ray { + +SchedulingClassDescriptor::SchedulingClassDescriptor( + ResourceSet rs, + LabelSelector ls, + FunctionDescriptor fd, + int64_t d, + rpc::SchedulingStrategy sched_strategy) + : resource_set(std::move(rs)), + label_selector(std::move(ls)), + function_descriptor(std::move(fd)), + depth(d), + scheduling_strategy(std::move(sched_strategy)) {} + +bool operator==(const ray::rpc::SchedulingStrategy &lhs, + const ray::rpc::SchedulingStrategy &rhs) { + if (lhs.scheduling_strategy_case() != rhs.scheduling_strategy_case()) { + return false; + } + + switch (lhs.scheduling_strategy_case()) { + case ray::rpc::SchedulingStrategy::kNodeAffinitySchedulingStrategy: { + return (lhs.node_affinity_scheduling_strategy().node_id() == + rhs.node_affinity_scheduling_strategy().node_id()) && + (lhs.node_affinity_scheduling_strategy().soft() == + rhs.node_affinity_scheduling_strategy().soft()) && + (lhs.node_affinity_scheduling_strategy().spill_on_unavailable() == + rhs.node_affinity_scheduling_strategy().spill_on_unavailable()) && + (lhs.node_affinity_scheduling_strategy().fail_on_unavailable() == + rhs.node_affinity_scheduling_strategy().fail_on_unavailable()); + } + case ray::rpc::SchedulingStrategy::kPlacementGroupSchedulingStrategy: { + return (lhs.placement_group_scheduling_strategy().placement_group_id() == + rhs.placement_group_scheduling_strategy().placement_group_id()) && + (lhs.placement_group_scheduling_strategy().placement_group_bundle_index() == + rhs.placement_group_scheduling_strategy().placement_group_bundle_index()) && + (lhs.placement_group_scheduling_strategy() + .placement_group_capture_child_tasks() == + rhs.placement_group_scheduling_strategy() + .placement_group_capture_child_tasks()); + } + case ray::rpc::SchedulingStrategy::kNodeLabelSchedulingStrategy: { + return google::protobuf::util::MessageDifferencer::Equivalent( + lhs.node_label_scheduling_strategy(), rhs.node_label_scheduling_strategy()); + } + default: + return true; + } +} + +// SchedulingClassDescriptor methods +bool SchedulingClassDescriptor::operator==(const SchedulingClassDescriptor &other) const { + return depth == other.depth && resource_set == other.resource_set && + label_selector == other.label_selector && + function_descriptor == other.function_descriptor && + scheduling_strategy == other.scheduling_strategy; +} + +std::string SchedulingClassDescriptor::DebugString() const { + std::stringstream buffer; + buffer << "{" + << "depth=" << depth << " " + << "function_descriptor=" << function_descriptor->ToString() << " " + << "scheduling_strategy=" << scheduling_strategy.DebugString() << " " + << "resource_set=" + << "{"; + for (const auto &pair : resource_set.GetResourceMap()) { + buffer << pair.first << " : " << pair.second << ", "; + } + buffer << "}"; + + buffer << "label_selector={"; + for (const auto &constraint : label_selector.GetConstraints()) { + buffer << constraint.GetLabelKey() << " " + << (constraint.GetOperator() == ray::LabelSelectorOperator::LABEL_IN ? "in" + : "!in") + << " ("; + for (const auto &val : constraint.GetLabelValues()) { + buffer << val << ", "; + } + buffer << "), "; + } + buffer << "}}"; + + return buffer.str(); +} + +std::string SchedulingClassDescriptor::ResourceSetStr() const { + std::stringstream buffer; + buffer << "{"; + for (const auto &pair : resource_set.GetResourceMap()) { + buffer << pair.first << " : " << pair.second << ", "; + } + buffer << "}"; + return buffer.str(); +} + +// Static member definitions +absl::Mutex SchedulingClassToIds::mutex_; +absl::flat_hash_map + SchedulingClassToIds::sched_cls_to_id_; +absl::flat_hash_map + SchedulingClassToIds::sched_id_to_cls_; +int SchedulingClassToIds::next_sched_id_; + +SchedulingClassDescriptor &SchedulingClassToIds::GetSchedulingClassDescriptor( + SchedulingClass id) { + absl::MutexLock lock(&mutex_); + auto it = sched_id_to_cls_.find(id); + RAY_CHECK(it != sched_id_to_cls_.end()) << "invalid id: " << id; + return it->second; +} + +SchedulingClass SchedulingClassToIds::GetSchedulingClass( + const SchedulingClassDescriptor &sched_cls) { + SchedulingClass sched_cls_id = 0; + absl::MutexLock lock(&mutex_); + auto it = sched_cls_to_id_.find(sched_cls); + if (it == sched_cls_to_id_.end()) { + sched_cls_id = ++next_sched_id_; + // TODO(ekl) we might want to try cleaning up task types in these cases + if (sched_cls_id > 100) { + RAY_LOG_EVERY_MS(WARNING, 1000) + << "More than " << sched_cls_id + << " types of tasks seen, this may reduce performance."; + } + sched_cls_to_id_[sched_cls] = sched_cls_id; + sched_id_to_cls_.emplace(sched_cls_id, sched_cls); + } else { + sched_cls_id = it->second; + } + return sched_cls_id; +} + +int CalculateRuntimeEnvHash(const std::string &serialized_runtime_env) { + if (IsRuntimeEnvEmpty(serialized_runtime_env)) { + // It's useful to have the same predetermined value for both unspecified and empty + // runtime envs. + return 0; + } + size_t hash = std::hash()(serialized_runtime_env); + return static_cast(hash); +} + +} // namespace ray diff --git a/src/ray/common/scheduling/scheduling_class_util.h b/src/ray/common/scheduling/scheduling_class_util.h new file mode 100644 index 000000000000..37b56736667d --- /dev/null +++ b/src/ray/common/scheduling/scheduling_class_util.h @@ -0,0 +1,170 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/synchronization/mutex.h" +#include "ray/common/function_descriptor.h" +#include "ray/common/scheduling/label_selector.h" +#include "ray/common/scheduling/resource_set.h" +#include "src/ray/protobuf/common.pb.h" + +namespace ray { + +bool operator==(const ray::rpc::SchedulingStrategy &lhs, + const ray::rpc::SchedulingStrategy &rhs); + +struct SchedulingClassDescriptor { + public: + explicit SchedulingClassDescriptor(ResourceSet rs, + LabelSelector ls, + FunctionDescriptor fd, + int64_t d, + rpc::SchedulingStrategy sched_strategy); + ResourceSet resource_set; + LabelSelector label_selector; + FunctionDescriptor function_descriptor; + int64_t depth; + rpc::SchedulingStrategy scheduling_strategy; + + bool operator==(const SchedulingClassDescriptor &other) const; + std::string DebugString() const; + std::string ResourceSetStr() const; +}; + +template +H AbslHashValue(H h, const SchedulingClassDescriptor &sched_cls) { + return H::combine(std::move(h), + sched_cls.resource_set, + sched_cls.function_descriptor->Hash(), + sched_cls.depth, + sched_cls.scheduling_strategy, + sched_cls.label_selector); +} + +using SchedulingClass = int; + +struct SchedulingClassToIds { + /// Below static fields could be mutated in `ComputeResources` concurrently due to + /// multi-threading, we need a mutex to protect it. + static absl::Mutex mutex_; + /// Keep global static id mappings for SchedulingClass for performance. + static absl::flat_hash_map sched_cls_to_id_ + ABSL_GUARDED_BY(mutex_); + static absl::flat_hash_map sched_id_to_cls_ + ABSL_GUARDED_BY(mutex_); + static int next_sched_id_ ABSL_GUARDED_BY(mutex_); + + /// Gets the scheduling class descriptor for the given id. + static SchedulingClassDescriptor &GetSchedulingClassDescriptor(SchedulingClass id); + + /// Gets or creates a scheduling class id for the given descriptor. + static SchedulingClass GetSchedulingClass(const SchedulingClassDescriptor &sched_cls); +}; + +// Get a Hash for the runtime environment string. +// "" and "{}" have the same hash. +// Other than that, only compare literal strings. i.e. '{"a": 1, "b": 2}' and '{"b": 2, +// "a": 1}' have different hashes. +int CalculateRuntimeEnvHash(const std::string &serialized_runtime_env); +} // namespace ray + +// Template specializations for std::hash +namespace std { + +template <> +struct hash { + size_t operator()(const ray::rpc::LabelOperator &label_operator) const { + size_t hash_value = std::hash()(label_operator.label_operator_case()); + if (label_operator.has_label_in()) { + for (const auto &value : label_operator.label_in().values()) { + hash_value ^= std::hash()(value); + } + } else if (label_operator.has_label_not_in()) { + for (const auto &value : label_operator.label_not_in().values()) { + hash_value ^= std::hash()(value); + } + } + return hash_value; + } +}; + +template <> +struct hash { + size_t operator()(const ray::rpc::LabelMatchExpression &expression) const { + size_t hash_val = std::hash()(expression.key()); + hash_val ^= std::hash()(expression.operator_()); + return hash_val; + } +}; + +template <> +struct hash { + size_t operator()(const ray::rpc::LabelMatchExpressions &expressions) const { + size_t hash_val = 0; + for (const auto &expression : expressions.expressions()) { + hash_val ^= std::hash()(expression); + } + return hash_val; + } +}; + +template <> +struct hash { + size_t operator()(const ray::rpc::SchedulingStrategy &scheduling_strategy) const { + size_t hash_val = std::hash()(scheduling_strategy.scheduling_strategy_case()); + if (scheduling_strategy.scheduling_strategy_case() == + ray::rpc::SchedulingStrategy::kNodeAffinitySchedulingStrategy) { + hash_val ^= std::hash()( + scheduling_strategy.node_affinity_scheduling_strategy().node_id()); + // soft returns a bool + hash_val ^= static_cast( + scheduling_strategy.node_affinity_scheduling_strategy().soft()); + hash_val ^= static_cast( + scheduling_strategy.node_affinity_scheduling_strategy().spill_on_unavailable()); + hash_val ^= static_cast( + scheduling_strategy.node_affinity_scheduling_strategy().fail_on_unavailable()); + } else if (scheduling_strategy.scheduling_strategy_case() == + ray::rpc::SchedulingStrategy::kPlacementGroupSchedulingStrategy) { + hash_val ^= std::hash()( + scheduling_strategy.placement_group_scheduling_strategy().placement_group_id()); + hash_val ^= scheduling_strategy.placement_group_scheduling_strategy() + .placement_group_bundle_index(); + // placement_group_capture_child_tasks returns a bool + hash_val ^= + static_cast(scheduling_strategy.placement_group_scheduling_strategy() + .placement_group_capture_child_tasks()); + } else if (scheduling_strategy.has_node_label_scheduling_strategy()) { + if (scheduling_strategy.node_label_scheduling_strategy().hard().expressions_size() > + 0) { + hash_val ^= std::hash()("hard"); + hash_val ^= std::hash()( + scheduling_strategy.node_label_scheduling_strategy().hard()); + } + if (scheduling_strategy.node_label_scheduling_strategy().soft().expressions_size() > + 0) { + hash_val ^= std::hash()("soft"); + hash_val ^= std::hash()( + scheduling_strategy.node_label_scheduling_strategy().soft()); + } + } + return hash_val; + } +}; + +} // namespace std diff --git a/src/ray/common/scheduling/scheduling_ids.cc b/src/ray/common/scheduling/scheduling_ids.cc index d1d128c82f02..87dbb86abdc5 100644 --- a/src/ray/common/scheduling/scheduling_ids.cc +++ b/src/ray/common/scheduling/scheduling_ids.cc @@ -14,9 +14,13 @@ #include "ray/common/scheduling/scheduling_ids.h" +#include #include #include +#include "ray/common/ray_config.h" +#include "ray/util/logging.h" + namespace ray { int64_t StringIdMap::Get(const std::string &string_id) const { diff --git a/src/ray/common/scheduling/scheduling_ids.h b/src/ray/common/scheduling/scheduling_ids.h index e054bc5c9c60..ce97202130cc 100644 --- a/src/ray/common/scheduling/scheduling_ids.h +++ b/src/ray/common/scheduling/scheduling_ids.h @@ -14,17 +14,15 @@ #pragma once -#include #include #include #include #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" +#include "absl/strings/match.h" #include "absl/synchronization/mutex.h" #include "ray/common/constants.h" -#include "ray/common/ray_config.h" -#include "ray/util/logging.h" namespace ray { diff --git a/src/ray/common/scheduling/tests/BUILD.bazel b/src/ray/common/scheduling/tests/BUILD.bazel new file mode 100644 index 000000000000..2833b8227f76 --- /dev/null +++ b/src/ray/common/scheduling/tests/BUILD.bazel @@ -0,0 +1,67 @@ +load("//bazel:ray.bzl", "ray_cc_test") + +ray_cc_test( + name = "resource_request_test", + size = "small", + srcs = [ + "resource_request_test.cc", + ], + tags = ["team:core"], + deps = [ + "//src/ray/common/scheduling:cluster_resource_data", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "resource_set_test", + size = "small", + srcs = [ + "resource_set_test.cc", + ], + tags = ["team:core"], + deps = [ + "//src/ray/common/scheduling:resource_set", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "resource_instance_set_test", + size = "small", + srcs = [ + "resource_instance_set_test.cc", + ], + tags = ["team:core"], + deps = [ + "//src/ray/common/scheduling:resource_instance_set", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "scheduling_ids_test", + size = "small", + srcs = [ + "scheduling_ids_test.cc", + ], + tags = ["team:core"], + deps = [ + "//src/ray/common:ray_config", + "//src/ray/common/scheduling:scheduling_ids", + "@com_google_googletest//:gtest_main", + ], +) + +ray_cc_test( + name = "label_selector_test", + size = "small", + srcs = [ + "label_selector_test.cc", + ], + tags = ["team:core"], + deps = [ + "//src/ray/common/scheduling:label_selector", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/common/tests/label_selector_test.cc b/src/ray/common/scheduling/tests/label_selector_test.cc similarity index 100% rename from src/ray/common/tests/label_selector_test.cc rename to src/ray/common/scheduling/tests/label_selector_test.cc diff --git a/src/ray/common/tests/resource_instance_set_test.cc b/src/ray/common/scheduling/tests/resource_instance_set_test.cc similarity index 100% rename from src/ray/common/tests/resource_instance_set_test.cc rename to src/ray/common/scheduling/tests/resource_instance_set_test.cc diff --git a/src/ray/common/tests/resource_request_test.cc b/src/ray/common/scheduling/tests/resource_request_test.cc similarity index 100% rename from src/ray/common/tests/resource_request_test.cc rename to src/ray/common/scheduling/tests/resource_request_test.cc diff --git a/src/ray/common/tests/resource_set_test.cc b/src/ray/common/scheduling/tests/resource_set_test.cc similarity index 100% rename from src/ray/common/tests/resource_set_test.cc rename to src/ray/common/scheduling/tests/resource_set_test.cc diff --git a/src/ray/common/tests/scheduling_ids_test.cc b/src/ray/common/scheduling/tests/scheduling_ids_test.cc similarity index 97% rename from src/ray/common/tests/scheduling_ids_test.cc rename to src/ray/common/scheduling/tests/scheduling_ids_test.cc index 762436910b26..eabd09d3fe54 100644 --- a/src/ray/common/tests/scheduling_ids_test.cc +++ b/src/ray/common/scheduling/tests/scheduling_ids_test.cc @@ -18,6 +18,7 @@ #include #include "gtest/gtest.h" +#include "ray/common/ray_config.h" namespace ray { @@ -27,7 +28,7 @@ TEST_F(SchedulingIDsTest, BasicTest) { std::vector string_ids = {"hello", "whaaat", "yes"}; std::vector node_ids; for (auto &string_id : string_ids) { - node_ids.emplace_back(scheduling::NodeID(string_id)); + node_ids.emplace_back(string_id); ASSERT_EQ(node_ids.back().Binary(), string_id); } ASSERT_EQ(node_ids[0], scheduling::NodeID(string_ids[0])); diff --git a/src/ray/common/task/task_spec.cc b/src/ray/common/task/task_spec.cc index dd4ea7cf7e93..94c1199a7d27 100644 --- a/src/ray/common/task/task_spec.cc +++ b/src/ray/common/task/task_spec.cc @@ -28,42 +28,6 @@ namespace ray { -absl::Mutex TaskSpecification::mutex_; -absl::flat_hash_map - TaskSpecification::sched_cls_to_id_; -absl::flat_hash_map - TaskSpecification::sched_id_to_cls_; -int TaskSpecification::next_sched_id_; - -SchedulingClassDescriptor &TaskSpecification::GetSchedulingClassDescriptor( - SchedulingClass id) { - absl::MutexLock lock(&mutex_); - auto it = sched_id_to_cls_.find(id); - RAY_CHECK(it != sched_id_to_cls_.end()) << "invalid id: " << id; - return it->second; -} - -SchedulingClass TaskSpecification::GetSchedulingClass( - const SchedulingClassDescriptor &sched_cls) { - SchedulingClass sched_cls_id; - absl::MutexLock lock(&mutex_); - auto it = sched_cls_to_id_.find(sched_cls); - if (it == sched_cls_to_id_.end()) { - sched_cls_id = ++next_sched_id_; - // TODO(ekl) we might want to try cleaning up task types in these cases - if (sched_cls_id > 100) { - RAY_LOG_EVERY_MS(WARNING, 1000) - << "More than " << sched_cls_id - << " types of tasks seen, this may reduce performance."; - } - sched_cls_to_id_[sched_cls] = sched_cls_id; - sched_id_to_cls_.emplace(sched_cls_id, sched_cls); - } else { - sched_cls_id = it->second; - } - return sched_cls_id; -} - const BundleID TaskSpecification::PlacementGroupBundleId() const { if (message_->scheduling_strategy().scheduling_strategy_case() == rpc::SchedulingStrategy::SchedulingStrategyCase:: @@ -137,7 +101,7 @@ void TaskSpecification::ComputeResources() { depth, GetSchedulingStrategy()); // Map the scheduling class descriptor to an integer for performance. - sched_cls_id_ = GetSchedulingClass(sched_cls_desc); + sched_cls_id_ = SchedulingClassToIds::GetSchedulingClass(sched_cls_desc); } runtime_env_hash_ = CalculateRuntimeEnvHash(SerializedRuntimeEnv()); @@ -666,16 +630,6 @@ std::string TaskSpecification::CallSiteString() const { return stream.str(); } -int CalculateRuntimeEnvHash(const std::string &serialized_runtime_env) { - if (IsRuntimeEnvEmpty(serialized_runtime_env)) { - // It's useful to have the same predetermined value for both unspecified and empty - // runtime envs. - return 0; - } - size_t hash = std::hash()(serialized_runtime_env); - return static_cast(hash); -} - std::vector TaskSpecification::ConcurrencyGroups() const { RAY_CHECK(IsActorCreationTask()); std::vector concurrency_groups; @@ -692,10 +646,10 @@ std::vector TaskSpecification::ConcurrencyGroups() const { curr_group_message.function_descriptors(j))); } - concurrency_groups.push_back( - {std::string{curr_group_message.name()}, - static_cast(curr_group_message.max_concurrency()), - function_descriptors}); + concurrency_groups.emplace_back( + std::string{curr_group_message.name()}, + static_cast(curr_group_message.max_concurrency()), + function_descriptors); } return concurrency_groups; diff --git a/src/ray/common/task/task_spec.h b/src/ray/common/task/task_spec.h index 18ad83f80a44..9fab88cc7438 100644 --- a/src/ray/common/task/task_spec.h +++ b/src/ray/common/task/task_spec.h @@ -19,220 +19,21 @@ #include #include #include -#include #include #include -#include "absl/hash/hash.h" -#include "absl/synchronization/mutex.h" #include "ray/common/function_descriptor.h" #include "ray/common/grpc_util.h" #include "ray/common/id.h" #include "ray/common/scheduling/label_selector.h" #include "ray/common/scheduling/resource_set.h" +#include "ray/common/scheduling/scheduling_class_util.h" #include "ray/common/task/task_common.h" extern "C" { #include "ray/thirdparty/sha256.h" } -namespace ray { -inline bool operator==(const ray::rpc::SchedulingStrategy &lhs, - const ray::rpc::SchedulingStrategy &rhs) { - if (lhs.scheduling_strategy_case() != rhs.scheduling_strategy_case()) { - return false; - } - - switch (lhs.scheduling_strategy_case()) { - case ray::rpc::SchedulingStrategy::kNodeAffinitySchedulingStrategy: { - return (lhs.node_affinity_scheduling_strategy().node_id() == - rhs.node_affinity_scheduling_strategy().node_id()) && - (lhs.node_affinity_scheduling_strategy().soft() == - rhs.node_affinity_scheduling_strategy().soft()) && - (lhs.node_affinity_scheduling_strategy().spill_on_unavailable() == - rhs.node_affinity_scheduling_strategy().spill_on_unavailable()) && - (lhs.node_affinity_scheduling_strategy().fail_on_unavailable() == - rhs.node_affinity_scheduling_strategy().fail_on_unavailable()); - } - case ray::rpc::SchedulingStrategy::kPlacementGroupSchedulingStrategy: { - return (lhs.placement_group_scheduling_strategy().placement_group_id() == - rhs.placement_group_scheduling_strategy().placement_group_id()) && - (lhs.placement_group_scheduling_strategy().placement_group_bundle_index() == - rhs.placement_group_scheduling_strategy().placement_group_bundle_index()) && - (lhs.placement_group_scheduling_strategy() - .placement_group_capture_child_tasks() == - rhs.placement_group_scheduling_strategy() - .placement_group_capture_child_tasks()); - } - case ray::rpc::SchedulingStrategy::kNodeLabelSchedulingStrategy: { - return google::protobuf::util::MessageDifferencer::Equivalent( - lhs.node_label_scheduling_strategy(), rhs.node_label_scheduling_strategy()); - } - default: - return true; - } -} - -typedef int SchedulingClass; - -struct SchedulingClassDescriptor { - public: - explicit SchedulingClassDescriptor(ResourceSet rs, - LabelSelector ls, - FunctionDescriptor fd, - int64_t d, - rpc::SchedulingStrategy sched_strategy) - : resource_set(std::move(rs)), - label_selector(std::move(ls)), - function_descriptor(std::move(fd)), - depth(d), - scheduling_strategy(std::move(sched_strategy)) {} - ResourceSet resource_set; - LabelSelector label_selector; - FunctionDescriptor function_descriptor; - int64_t depth; - rpc::SchedulingStrategy scheduling_strategy; - - bool operator==(const SchedulingClassDescriptor &other) const { - return depth == other.depth && resource_set == other.resource_set && - label_selector == other.label_selector && - function_descriptor == other.function_descriptor && - scheduling_strategy == other.scheduling_strategy; - } - - std::string DebugString() const { - std::stringstream buffer; - buffer << "{" - << "depth=" << depth << " " - << "function_descriptor=" << function_descriptor->ToString() << " " - << "scheduling_strategy=" << scheduling_strategy.DebugString() << " " - << "resource_set=" - << "{"; - for (const auto &pair : resource_set.GetResourceMap()) { - buffer << pair.first << " : " << pair.second << ", "; - } - buffer << "}"; - - buffer << "label_selector={"; - for (const auto &constraint : label_selector.GetConstraints()) { - buffer << constraint.GetLabelKey() << " " - << (constraint.GetOperator() == ray::LabelSelectorOperator::LABEL_IN ? "in" - : "!in") - << " ("; - for (const auto &val : constraint.GetLabelValues()) { - buffer << val << ", "; - } - buffer << "), "; - } - buffer << "}}"; - - return buffer.str(); - } - - std::string ResourceSetStr() const { - std::stringstream buffer; - buffer << "{"; - for (const auto &pair : resource_set.GetResourceMap()) { - buffer << pair.first << " : " << pair.second << ", "; - } - buffer << "}"; - return buffer.str(); - } -}; - -template -H AbslHashValue(H h, const SchedulingClassDescriptor &sched_cls) { - return H::combine(std::move(h), - sched_cls.resource_set, - sched_cls.function_descriptor->Hash(), - sched_cls.depth, - sched_cls.scheduling_strategy, - sched_cls.label_selector); -} -} // namespace ray - -namespace std { -template <> -struct hash { - size_t operator()(const ray::rpc::LabelOperator &label_operator) const { - size_t hash_value = std::hash()(label_operator.label_operator_case()); - if (label_operator.has_label_in()) { - for (const auto &value : label_operator.label_in().values()) { - hash_value ^= std::hash()(value); - } - } else if (label_operator.has_label_not_in()) { - for (const auto &value : label_operator.label_not_in().values()) { - hash_value ^= std::hash()(value); - } - } - return hash_value; - } -}; - -template <> -struct hash { - size_t operator()(const ray::rpc::LabelMatchExpression &expression) const { - size_t hash_val = std::hash()(expression.key()); - hash_val ^= std::hash()(expression.operator_()); - return hash_val; - } -}; - -template <> -struct hash { - size_t operator()(const ray::rpc::LabelMatchExpressions &expressions) const { - size_t hash_val = 0; - for (const auto &expression : expressions.expressions()) { - hash_val ^= std::hash()(expression); - } - return hash_val; - } -}; - -template <> -struct hash { - size_t operator()(const ray::rpc::SchedulingStrategy &scheduling_strategy) const { - size_t hash_val = std::hash()(scheduling_strategy.scheduling_strategy_case()); - if (scheduling_strategy.scheduling_strategy_case() == - ray::rpc::SchedulingStrategy::kNodeAffinitySchedulingStrategy) { - hash_val ^= std::hash()( - scheduling_strategy.node_affinity_scheduling_strategy().node_id()); - // soft returns a bool - hash_val ^= static_cast( - scheduling_strategy.node_affinity_scheduling_strategy().soft()); - hash_val ^= static_cast( - scheduling_strategy.node_affinity_scheduling_strategy().spill_on_unavailable()); - hash_val ^= static_cast( - scheduling_strategy.node_affinity_scheduling_strategy().fail_on_unavailable()); - } else if (scheduling_strategy.scheduling_strategy_case() == - ray::rpc::SchedulingStrategy::kPlacementGroupSchedulingStrategy) { - hash_val ^= std::hash()( - scheduling_strategy.placement_group_scheduling_strategy().placement_group_id()); - hash_val ^= scheduling_strategy.placement_group_scheduling_strategy() - .placement_group_bundle_index(); - // placement_group_capture_child_tasks returns a bool - hash_val ^= - static_cast(scheduling_strategy.placement_group_scheduling_strategy() - .placement_group_capture_child_tasks()); - } else if (scheduling_strategy.has_node_label_scheduling_strategy()) { - if (scheduling_strategy.node_label_scheduling_strategy().hard().expressions_size() > - 0) { - hash_val ^= std::hash()("hard"); - hash_val ^= std::hash()( - scheduling_strategy.node_label_scheduling_strategy().hard()); - } - if (scheduling_strategy.node_label_scheduling_strategy().soft().expressions_size() > - 0) { - hash_val ^= std::hash()("soft"); - hash_val ^= std::hash()( - scheduling_strategy.node_label_scheduling_strategy().soft()); - } - } - return hash_val; - } -}; -} // namespace std - namespace ray { /// ConcurrencyGroup is a group of actor methods that shares @@ -579,21 +380,6 @@ class TaskSpecification : public MessageWrapper { // Field storing label selector for scheduling Task on a node. Initialized in constuctor // in ComputeResources() call. std::shared_ptr label_selector_; - /// Below static fields could be mutated in `ComputeResources` concurrently due to - /// multi-threading, we need a mutex to protect it. - static absl::Mutex mutex_; - /// Keep global static id mappings for SchedulingClass for performance. - static absl::flat_hash_map sched_cls_to_id_ - ABSL_GUARDED_BY(mutex_); - static absl::flat_hash_map sched_id_to_cls_ - ABSL_GUARDED_BY(mutex_); - static int next_sched_id_ ABSL_GUARDED_BY(mutex_); }; -// Get a Hash for the runtime environment string. -// "" and "{}" have the same hash. -// Other than that, only compare literal strings. i.e. '{"a": 1, "b": 2}' and '{"b": 2, -// "a": 1}' have different hashes. -int CalculateRuntimeEnvHash(const std::string &serialized_runtime_env); - } // namespace ray diff --git a/src/ray/common/tests/BUILD.bazel b/src/ray/common/tests/BUILD.bazel index 4822452c0d8b..840a4a2b4f5a 100644 --- a/src/ray/common/tests/BUILD.bazel +++ b/src/ray/common/tests/BUILD.bazel @@ -1,44 +1,5 @@ load("//bazel:ray.bzl", "ray_cc_binary", "ray_cc_library", "ray_cc_test") -ray_cc_test( - name = "resource_request_test", - size = "small", - srcs = [ - "resource_request_test.cc", - ], - tags = ["team:core"], - deps = [ - "//src/ray/common:task_common", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "resource_set_test", - size = "small", - srcs = [ - "resource_set_test.cc", - ], - tags = ["team:core"], - deps = [ - "//src/ray/common:task_common", - "@com_google_googletest//:gtest_main", - ], -) - -ray_cc_test( - name = "resource_instance_set_test", - size = "small", - srcs = [ - "resource_instance_set_test.cc", - ], - tags = ["team:core"], - deps = [ - "//src/ray/common:task_common", - "@com_google_googletest//:gtest_main", - ], -) - ray_cc_test( name = "ray_syncer_test", srcs = ["ray_syncer_test.cc"], @@ -132,6 +93,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/ray/common:task_common", + "//src/ray/common/scheduling:scheduling_class_util", "@com_google_googletest//:gtest_main", ], ) @@ -143,7 +105,7 @@ ray_cc_test( ], tags = ["team:core"], deps = [ - "//src/ray/common:task_common", + "//src/ray/common:bundle_location_index", "@com_google_googletest//:gtest_main", ], ) @@ -204,19 +166,6 @@ ray_cc_test( ], ) -ray_cc_test( - name = "scheduling_ids_test", - size = "small", - srcs = [ - "scheduling_ids_test.cc", - ], - tags = ["team:core"], - deps = [ - "//src/ray/common:task_common", - "@com_google_googletest//:gtest_main", - ], -) - ray_cc_test( name = "grpc_util_test", size = "small", @@ -231,19 +180,6 @@ ray_cc_test( ], ) -ray_cc_test( - name = "label_selector_test", - size = "small", - srcs = [ - "label_selector_test.cc", - ], - tags = ["team:core"], - deps = [ - "//src/ray/common:task_common", - "@com_google_googletest//:gtest_main", - ], -) - ray_cc_test( name = "source_location_test", size = "small", diff --git a/src/ray/common/tests/task_spec_test.cc b/src/ray/common/tests/task_spec_test.cc index a3b0cb5e05d1..bb8a64636ffe 100644 --- a/src/ray/common/tests/task_spec_test.cc +++ b/src/ray/common/tests/task_spec_test.cc @@ -66,62 +66,62 @@ TEST(TaskSpecTest, TestSchedulingClassDescriptor) { ASSERT_TRUE(descriptor1 == descriptor1); ASSERT_TRUE(absl::Hash()(descriptor1) == absl::Hash()(descriptor1)); - ASSERT_TRUE(TaskSpecification::GetSchedulingClass(descriptor1) == - TaskSpecification::GetSchedulingClass(descriptor1)); + ASSERT_TRUE(SchedulingClassToIds::GetSchedulingClass(descriptor1) == + SchedulingClassToIds::GetSchedulingClass(descriptor1)); ASSERT_FALSE(descriptor1 == descriptor2); ASSERT_FALSE(absl::Hash()(descriptor1) == absl::Hash()(descriptor2)); - ASSERT_FALSE(TaskSpecification::GetSchedulingClass(descriptor1) == - TaskSpecification::GetSchedulingClass(descriptor2)); + ASSERT_FALSE(SchedulingClassToIds::GetSchedulingClass(descriptor1) == + SchedulingClassToIds::GetSchedulingClass(descriptor2)); ASSERT_FALSE(descriptor1 == descriptor3); ASSERT_FALSE(absl::Hash()(descriptor1) == absl::Hash()(descriptor3)); - ASSERT_FALSE(TaskSpecification::GetSchedulingClass(descriptor1) == - TaskSpecification::GetSchedulingClass(descriptor3)); + ASSERT_FALSE(SchedulingClassToIds::GetSchedulingClass(descriptor1) == + SchedulingClassToIds::GetSchedulingClass(descriptor3)); ASSERT_FALSE(descriptor1 == descriptor4); ASSERT_FALSE(absl::Hash()(descriptor1) == absl::Hash()(descriptor4)); - ASSERT_FALSE(TaskSpecification::GetSchedulingClass(descriptor1) == - TaskSpecification::GetSchedulingClass(descriptor4)); + ASSERT_FALSE(SchedulingClassToIds::GetSchedulingClass(descriptor1) == + SchedulingClassToIds::GetSchedulingClass(descriptor4)); ASSERT_FALSE(descriptor4 == descriptor5); ASSERT_FALSE(absl::Hash()(descriptor4) == absl::Hash()(descriptor5)); - ASSERT_FALSE(TaskSpecification::GetSchedulingClass(descriptor4) == - TaskSpecification::GetSchedulingClass(descriptor5)); + ASSERT_FALSE(SchedulingClassToIds::GetSchedulingClass(descriptor4) == + SchedulingClassToIds::GetSchedulingClass(descriptor5)); ASSERT_TRUE(descriptor5 == descriptor6); ASSERT_TRUE(absl::Hash()(descriptor5) == absl::Hash()(descriptor6)); - ASSERT_TRUE(TaskSpecification::GetSchedulingClass(descriptor5) == - TaskSpecification::GetSchedulingClass(descriptor6)); + ASSERT_TRUE(SchedulingClassToIds::GetSchedulingClass(descriptor5) == + SchedulingClassToIds::GetSchedulingClass(descriptor6)); ASSERT_FALSE(descriptor6 == descriptor10); ASSERT_FALSE(absl::Hash()(descriptor6) == absl::Hash()(descriptor10)); - ASSERT_FALSE(TaskSpecification::GetSchedulingClass(descriptor6) == - TaskSpecification::GetSchedulingClass(descriptor10)); + ASSERT_FALSE(SchedulingClassToIds::GetSchedulingClass(descriptor6) == + SchedulingClassToIds::GetSchedulingClass(descriptor10)); ASSERT_FALSE(descriptor6 == descriptor7); ASSERT_FALSE(absl::Hash()(descriptor6) == absl::Hash()(descriptor7)); - ASSERT_FALSE(TaskSpecification::GetSchedulingClass(descriptor6) == - TaskSpecification::GetSchedulingClass(descriptor7)); + ASSERT_FALSE(SchedulingClassToIds::GetSchedulingClass(descriptor6) == + SchedulingClassToIds::GetSchedulingClass(descriptor7)); ASSERT_FALSE(descriptor7 == descriptor8); ASSERT_FALSE(absl::Hash()(descriptor7) == absl::Hash()(descriptor8)); - ASSERT_FALSE(TaskSpecification::GetSchedulingClass(descriptor7) == - TaskSpecification::GetSchedulingClass(descriptor8)); + ASSERT_FALSE(SchedulingClassToIds::GetSchedulingClass(descriptor7) == + SchedulingClassToIds::GetSchedulingClass(descriptor8)); ASSERT_TRUE(descriptor7 == descriptor9); ASSERT_TRUE(absl::Hash()(descriptor7) == absl::Hash()(descriptor9)); - ASSERT_TRUE(TaskSpecification::GetSchedulingClass(descriptor7) == - TaskSpecification::GetSchedulingClass(descriptor9)); + ASSERT_TRUE(SchedulingClassToIds::GetSchedulingClass(descriptor7) == + SchedulingClassToIds::GetSchedulingClass(descriptor9)); } TEST(TaskSpecTest, TestActorSchedulingClass) { diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index e2c3ec5b77d9..f62d6bed3ace 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -105,7 +105,6 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:ray_object", "//src/ray/common:status", - "//src/ray/common:task_common", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/util:process", ], @@ -314,7 +313,6 @@ ray_cc_library( "//src/ray/common:ray_config", "//src/ray/common:ray_object", "//src/ray/common:status", - "//src/ray/common:task_common", "//src/ray/object_manager:object_manager_common", "//src/ray/object_manager/plasma:plasma_client", "//src/ray/util:time", @@ -395,7 +393,6 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:ray_config", "//src/ray/common:status", - "//src/ray/common:task_common", "//src/ray/ipc:raylet_ipc_client_interface", "//src/ray/object_manager/plasma:plasma_client", "//src/ray/protobuf:common_cc_proto", diff --git a/src/ray/core_worker/common.cc b/src/ray/core_worker/common.cc index a28a54053051..e372c925b9bd 100644 --- a/src/ray/core_worker/common.cc +++ b/src/ray/core_worker/common.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include "ray/util/process.h" diff --git a/src/ray/core_worker/common.h b/src/ray/core_worker/common.h index afd98d20a568..3e0bd5c06379 100644 --- a/src/ray/core_worker/common.h +++ b/src/ray/core_worker/common.h @@ -22,7 +22,6 @@ #include "ray/common/id.h" #include "ray/common/ray_object.h" -#include "ray/common/scheduling/label_selector.h" #include "ray/common/task/task_spec.h" #include "src/ray/protobuf/common.pb.h" diff --git a/src/ray/core_worker/core_worker_options.h b/src/ray/core_worker/core_worker_options.h index 5e139e9758e6..c91cc81a3a2e 100644 --- a/src/ray/core_worker/core_worker_options.h +++ b/src/ray/core_worker/core_worker_options.h @@ -25,7 +25,6 @@ #include "ray/common/ray_object.h" #include "ray/common/status.h" #include "ray/common/task/task_common.h" -#include "ray/common/task/task_spec.h" #include "ray/core_worker/common.h" #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/util/process.h" diff --git a/src/ray/core_worker/store_provider/plasma_store_provider.h b/src/ray/core_worker/store_provider/plasma_store_provider.h index 0da5aac4437a..448dc5e6f40c 100644 --- a/src/ray/core_worker/store_provider/plasma_store_provider.h +++ b/src/ray/core_worker/store_provider/plasma_store_provider.h @@ -25,7 +25,6 @@ #include "ray/common/id.h" #include "ray/common/status.h" #include "ray/common/status_or.h" -#include "ray/core_worker/common.h" #include "ray/core_worker/context.h" #include "ray/core_worker/reference_count.h" #include "ray/ipc/raylet_ipc_client_interface.h" diff --git a/src/ray/core_worker/task_execution/BUILD.bazel b/src/ray/core_worker/task_execution/BUILD.bazel index 6f432bd339ca..49b0dc755cb4 100644 --- a/src/ray/core_worker/task_execution/BUILD.bazel +++ b/src/ray/core_worker/task_execution/BUILD.bazel @@ -113,7 +113,6 @@ ray_cc_library( "//src/ray/rpc:server_call", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/synchronization", ], ) @@ -142,7 +141,5 @@ ray_cc_library( "//src/ray/rpc:server_call", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/synchronization", ], ) diff --git a/src/ray/core_worker/task_execution/concurrency_group_manager.cc b/src/ray/core_worker/task_execution/concurrency_group_manager.cc index 12f0dfe63232..ce58694d06c4 100644 --- a/src/ray/core_worker/task_execution/concurrency_group_manager.cc +++ b/src/ray/core_worker/task_execution/concurrency_group_manager.cc @@ -15,7 +15,6 @@ #include "ray/core_worker/task_execution/concurrency_group_manager.h" #include -#include #include #include #include diff --git a/src/ray/core_worker/task_execution/concurrency_group_manager.h b/src/ray/core_worker/task_execution/concurrency_group_manager.h index c976523a56b4..4aa3bd16c6a1 100644 --- a/src/ray/core_worker/task_execution/concurrency_group_manager.h +++ b/src/ray/core_worker/task_execution/concurrency_group_manager.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include diff --git a/src/ray/core_worker/task_execution/out_of_order_actor_scheduling_queue.h b/src/ray/core_worker/task_execution/out_of_order_actor_scheduling_queue.h index 24ef6e1d505c..46d144e5fe97 100644 --- a/src/ray/core_worker/task_execution/out_of_order_actor_scheduling_queue.h +++ b/src/ray/core_worker/task_execution/out_of_order_actor_scheduling_queue.h @@ -20,7 +20,6 @@ #include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" -#include "absl/container/flat_hash_set.h" #include "absl/synchronization/mutex.h" #include "ray/common/id.h" #include "ray/common/task/task_spec.h" diff --git a/src/ray/core_worker/task_execution/task_receiver.h b/src/ray/core_worker/task_execution/task_receiver.h index 9ae7ce3c4f7b..157e597fd8e7 100644 --- a/src/ray/core_worker/task_execution/task_receiver.h +++ b/src/ray/core_worker/task_execution/task_receiver.h @@ -14,19 +14,13 @@ #pragma once -#include #include -#include -#include #include #include #include #include -#include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" -#include "absl/container/flat_hash_set.h" -#include "absl/synchronization/mutex.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" #include "ray/common/ray_object.h" diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index 6adf3162fa47..6e8df62c414f 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -71,7 +71,6 @@ ray_cc_test( "//:ray_mock", "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/mock/ray/pubsub:mock_publisher", - "//src/ray/common:task_common", "//src/ray/common:test_utils", "//src/ray/core_worker:memory_store", "//src/ray/core_worker:object_recovery_manager", @@ -134,13 +133,11 @@ ray_cc_test( ], deps = [ "//:ray_mock", - "//src/ray/common:task_common", "//src/ray/common:test_utils", "//src/ray/core_worker:task_event_buffer", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/util:event", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/synchronization", "@com_google_absl//absl/types:optional", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", @@ -187,7 +184,6 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", - "//src/ray/common:task_common", "//src/ray/common:test_utils", "//src/ray/core_worker:actor_manager", "//src/ray/gcs/gcs_client:gcs_client_lib", diff --git a/src/ray/core_worker/tests/actor_manager_test.cc b/src/ray/core_worker/tests/actor_manager_test.cc index cd5d28bb51d4..6acc46473a0f 100644 --- a/src/ray/core_worker/tests/actor_manager_test.cc +++ b/src/ray/core_worker/tests/actor_manager_test.cc @@ -21,7 +21,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "mock/ray/core_worker/reference_count.h" -#include "ray/common/task/task_spec.h" #include "ray/common/test_utils.h" #include "ray/gcs/gcs_client/accessor.h" #include "ray/gcs/gcs_client/gcs_client.h" diff --git a/src/ray/core_worker/tests/object_recovery_manager_test.cc b/src/ray/core_worker/tests/object_recovery_manager_test.cc index 814e886411c8..68ecc7bedf6d 100644 --- a/src/ray/core_worker/tests/object_recovery_manager_test.cc +++ b/src/ray/core_worker/tests/object_recovery_manager_test.cc @@ -26,8 +26,6 @@ #include "gtest/gtest.h" #include "mock/ray/core_worker/task_manager_interface.h" #include "mock/ray/pubsub/publisher.h" -#include "ray/common/task/task_spec.h" -#include "ray/common/task/task_util.h" #include "ray/common/test_utils.h" #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/rpc/raylet/raylet_client_interface.h" diff --git a/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc b/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc index 066ad5e4c965..5162892ae292 100644 --- a/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc +++ b/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc @@ -21,13 +21,10 @@ #include #include -#include "absl/base/thread_annotations.h" -#include "absl/synchronization/mutex.h" #include "absl/types/optional.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include "mock/ray/gcs/gcs_client/gcs_client.h" -#include "ray/common/task/task_spec.h" #include "ray/common/test_utils.h" #include "ray/core_worker/task_event_buffer.h" #include "ray/util/event.h" diff --git a/src/ray/gcs/gcs_client/BUILD.bazel b/src/ray/gcs/gcs_client/BUILD.bazel index bd6dabcc8ae7..ce1b1e861ff9 100644 --- a/src/ray/gcs/gcs_client/BUILD.bazel +++ b/src/ray/gcs/gcs_client/BUILD.bazel @@ -13,6 +13,7 @@ ray_cc_library( deps = [ "//src/ray/common:asio", "//src/ray/common:id", + "//src/ray/common:placement_group", "//src/ray/common:protobuf_utils", "//src/ray/gcs/store_client:redis_store_client", "//src/ray/protobuf:usage_cc_proto", diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index f34da0076caf..35fb38ae328f 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -101,12 +101,10 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:ray_config", "//src/ray/common:ray_syncer", - "//src/ray/common:task_common", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/protobuf:ray_syncer_cc_proto", "//src/ray/raylet/scheduling:cluster_lease_manager", "//src/ray/raylet/scheduling:cluster_resource_manager", - "//src/ray/stats:stats_metric", "//src/ray/util:logging", "@com_google_absl//absl/container:flat_hash_map", ], @@ -264,8 +262,8 @@ ray_cc_library( srcs = ["gcs_placement_group.cc"], hdrs = ["gcs_placement_group.h"], deps = [ + "//src/ray/common:bundle_spec", "//src/ray/common:id", - "//src/ray/common:task_common", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/stats:stats_lib", "//src/ray/util:counter_map", @@ -283,7 +281,6 @@ ray_cc_library( ":gcs_table_storage", "//src/ray/common:asio", "//src/ray/common:id", - "//src/ray/common:task_common", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "//src/ray/raylet/scheduling:scheduling_context", "//src/ray/rpc:raylet_client_interface", @@ -298,19 +295,20 @@ ray_cc_library( hdrs = ["gcs_placement_group_manager.h"], deps = [ ":gcs_init_data", - ":gcs_node_manager", ":gcs_placement_group", ":gcs_placement_group_scheduler", ":gcs_resource_manager", ":gcs_table_storage", ":gcs_usage_stats_client", + ":grpc_service_interfaces", "//src/ray/common:asio", + "//src/ray/common:bundle_spec", "//src/ray/common:id", - "//src/ray/common:task_common", + "//src/ray/common:ray_config", "//src/ray/protobuf:gcs_cc_proto", + "//src/ray/stats:stats_lib", "//src/ray/util:counter_map", "//src/ray/util:exponential_backoff", - "//src/ray/util:time", "@com_google_absl//absl/container:flat_hash_map", ], ) @@ -361,6 +359,7 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:lease", "//src/ray/common:task_common", + "//src/ray/common/scheduling:cluster_resource_data", "//src/ray/protobuf:core_worker_cc_proto", "//src/ray/protobuf:export_event_cc_proto", "//src/ray/protobuf:gcs_service_cc_proto", @@ -385,11 +384,10 @@ ray_cc_library( "//src/ray/common:asio", "//src/ray/common:id", "//src/ray/common:ray_config", - "//src/ray/common:task_common", - "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/raylet/scheduling:cluster_lease_manager", "//src/ray/rpc:core_worker_client", "//src/ray/rpc:raylet_client_interface", + "//src/ray/rpc:raylet_client_pool", "//src/ray/util:logging", "//src/ray/util:time", "@com_google_absl//absl/container:flat_hash_map", diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h index bc12aa833b57..1def8f69db62 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_actor_scheduler.h @@ -16,7 +16,6 @@ #include #include -#include #include #include #include @@ -25,17 +24,13 @@ #include "absl/container/flat_hash_set.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" -#include "ray/common/scheduling/scheduling_ids.h" -#include "ray/common/task/task_spec.h" #include "ray/gcs/gcs_server/gcs_actor.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/rpc/raylet/raylet_client_interface.h" #include "ray/rpc/raylet/raylet_client_pool.h" -#include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" -#include "src/ray/protobuf/gcs_service.pb.h" namespace ray { using raylet::ClusterLeaseManager; diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc b/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc index 6580f5872327..1a67ce39518b 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc @@ -20,8 +20,7 @@ #include #include "ray/common/asio/asio_util.h" -#include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/protobuf_utils.h" +#include "ray/common/bundle_spec.h" #include "ray/common/ray_config.h" #include "ray/stats/metric_defs.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_manager.h b/src/ray/gcs/gcs_server/gcs_placement_group_manager.h index 24130ccc7df0..b9a7450fddde 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_manager.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_manager.h @@ -24,11 +24,8 @@ #include "absl/container/flat_hash_map.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/bundle_spec.h" #include "ray/common/id.h" -#include "ray/common/task/task_spec.h" #include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/gcs_server/gcs_placement_group.h" #include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" #include "ray/gcs/gcs_server/gcs_resource_manager.h" @@ -37,7 +34,6 @@ #include "ray/gcs/gcs_server/usage_stats_client.h" #include "ray/util/counter_map.h" #include "ray/util/exponential_backoff.h" -#include "ray/util/time.h" #include "src/ray/protobuf/gcs_service.pb.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h index 1f074da8c0a4..a031c594c306 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h +++ b/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h @@ -24,7 +24,6 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/bundle_location_index.h" #include "ray/common/id.h" -#include "ray/common/scheduling/scheduling_ids.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/gcs_server/gcs_placement_group.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" diff --git a/src/ray/gcs/gcs_server/gcs_resource_manager.h b/src/ray/gcs/gcs_server/gcs_resource_manager.h index 96242cb291a7..78ced7dd01ab 100644 --- a/src/ray/gcs/gcs_server/gcs_resource_manager.h +++ b/src/ray/gcs/gcs_server/gcs_resource_manager.h @@ -23,13 +23,11 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" #include "ray/common/ray_syncer/ray_syncer.h" -#include "ray/common/scheduling/cluster_resource_data.h" #include "ray/gcs/gcs_server/gcs_init_data.h" #include "ray/gcs/gcs_server/gcs_node_manager.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" -#include "ray/stats/metric_defs.h" #include "src/ray/protobuf/gcs.pb.h" #include "src/ray/protobuf/ray_syncer.pb.h" diff --git a/src/ray/internal/internal.h b/src/ray/internal/internal.h index 20c89a4cc6c6..e9353150d998 100644 --- a/src/ray/internal/internal.h +++ b/src/ray/internal/internal.h @@ -20,7 +20,7 @@ #include "ray/common/buffer.h" #include "ray/common/id.h" -#include "ray/core_worker/core_worker.h" +#include "ray/core_worker/common.h" #include "ray/stats/metric.h" // This header is used to warp some internal code so we can reduce suspicious diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index f8b4b2637aeb..ec69ba17b874 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -24,7 +24,6 @@ ray_cc_library( visibility = [":__subpackages__"], deps = [ "//src/ray/common:id", - "//src/ray/common:lease", "//src/ray/object_manager", "//src/ray/util:counter_map", "@com_google_absl//absl/container:flat_hash_map", @@ -44,6 +43,8 @@ ray_cc_library( ":worker_pool", "//src/ray/common:lease", "//src/ray/common:ray_object", + "//src/ray/common/scheduling:cluster_resource_data", + "//src/ray/common/scheduling:placement_group_util", "//src/ray/object_manager:object_manager_common", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "//src/ray/raylet/scheduling:local_lease_manager_interface", diff --git a/src/ray/raylet/local_lease_manager.cc b/src/ray/raylet/local_lease_manager.cc index 7a497a04993f..402bfd6e1358 100644 --- a/src/ray/raylet/local_lease_manager.cc +++ b/src/ray/raylet/local_lease_manager.cc @@ -25,6 +25,7 @@ #include #include "ray/common/scheduling/cluster_resource_data.h" +#include "ray/common/scheduling/placement_group_util.h" #include "ray/stats/metric_defs.h" #include "ray/util/logging.h" @@ -193,7 +194,7 @@ void LocalLeaseManager::GrantScheduledLeasesToWorkers() { } } const auto &sched_cls_desc = - TaskSpecification::GetSchedulingClassDescriptor(scheduling_class); + SchedulingClassToIds::GetSchedulingClassDescriptor(scheduling_class); double total_cpus = cluster_resource_scheduler_.GetLocalResourceManager().GetNumCpus(); @@ -210,7 +211,7 @@ void LocalLeaseManager::GrantScheduledLeasesToWorkers() { for (auto &entry : info_by_sched_cls_) { // Only consider CPU requests const auto &cur_sched_cls_desc = - TaskSpecification::GetSchedulingClassDescriptor(entry.first); + SchedulingClassToIds::GetSchedulingClassDescriptor(entry.first); if (cur_sched_cls_desc.resource_set.Get(scheduling::ResourceID::CPU()).Double() > 0) { total_cpu_granted_leases += entry.second.granted_leases.size(); @@ -1163,7 +1164,7 @@ ResourceSet LocalLeaseManager::CalcNormalTaskResources() const { uint64_t LocalLeaseManager::MaxGrantedLeasesPerSchedulingClass( SchedulingClass sched_cls_id) const { - auto sched_cls = TaskSpecification::GetSchedulingClassDescriptor(sched_cls_id); + auto sched_cls = SchedulingClassToIds::GetSchedulingClassDescriptor(sched_cls_id); double cpu_req = sched_cls.resource_set.Get(ResourceID::CPU()).Double(); uint64_t total_cpus = cluster_resource_scheduler_.GetLocalResourceManager().GetNumCpus(); @@ -1228,7 +1229,8 @@ void LocalLeaseManager::DebugStr(std::stringstream &buffer) const { buffer << "}\n"; buffer << "Backlog Size per scheduling descriptor :{workerId: num backlogs}:\n"; for (const auto &[sched_cls, worker_to_backlog_size] : backlog_tracker_) { - const auto &descriptor = TaskSpecification::GetSchedulingClassDescriptor(sched_cls); + const auto &descriptor = + SchedulingClassToIds::GetSchedulingClassDescriptor(sched_cls); buffer << "\t" << descriptor.ResourceSetStr() << ": {\n"; for (const auto &[worker_id, backlog_size] : worker_to_backlog_size) { buffer << "\t\t" << worker_id << ": " << backlog_size << "\n"; @@ -1241,7 +1243,8 @@ void LocalLeaseManager::DebugStr(std::stringstream &buffer) const { for (const auto &pair : info_by_sched_cls_) { const auto &sched_cls = pair.first; const auto &info = pair.second; - const auto &descriptor = TaskSpecification::GetSchedulingClassDescriptor(sched_cls); + const auto &descriptor = + SchedulingClassToIds::GetSchedulingClassDescriptor(sched_cls); buffer << " - " << descriptor.DebugString() << ": " << info.granted_leases.size() << "/" << info.capacity << "\n"; } diff --git a/src/ray/raylet/scheduling/BUILD.bazel b/src/ray/raylet/scheduling/BUILD.bazel index ee89aba03ab6..6d00fd1c3205 100644 --- a/src/ray/raylet/scheduling/BUILD.bazel +++ b/src/ray/raylet/scheduling/BUILD.bazel @@ -34,7 +34,7 @@ ray_cc_library( hdrs = ["internal.h"], deps = [ "//src/ray/common:lease", - "//src/ray/common:ray_object", + "//src/ray/common/scheduling:cluster_resource_data", "//src/ray/protobuf:node_manager_cc_proto", ], ) @@ -45,9 +45,11 @@ ray_cc_library( hdrs = ["cluster_resource_manager.h"], deps = [ ":local_resource_manager", + "//src/ray/common:bundle_location_index", "//src/ray/common:grpc_util", "//src/ray/common:lease", "//src/ray/common:ray_config", + "//src/ray/common/scheduling:cluster_resource_data", "//src/ray/protobuf:gcs_cc_proto", "//src/ray/util:container_util", "//src/ray/util:logging", @@ -89,7 +91,6 @@ ray_cc_library( ":scheduler_resource_reporter", "//src/ray/common:lease", "//src/ray/common:ray_config", - "//src/ray/common:ray_object", "//src/ray/stats:stats_lib", "//src/ray/util:logging", "@com_google_absl//absl/container:flat_hash_map", @@ -110,7 +111,6 @@ ray_cc_library( hdrs = ["local_lease_manager_interface.h"], deps = [ ":scheduler_internal", - "//src/ray/common:lease", "@com_google_absl//absl/container:flat_hash_map", ], ) @@ -120,10 +120,9 @@ ray_cc_library( srcs = ["local_resource_manager.cc"], hdrs = ["local_resource_manager.h"], deps = [ - "//src/ray/common:grpc_util", - "//src/ray/common:lease", - "//src/ray/common:ray_config", "//src/ray/common:ray_syncer", + "//src/ray/common/scheduling:cluster_resource_data", + "//src/ray/common/scheduling:placement_group_util", "//src/ray/protobuf:gcs_cc_proto", "//src/ray/protobuf:node_manager_cc_proto", "//src/ray/stats:stats_metric", @@ -140,7 +139,6 @@ ray_cc_library( deps = [ ":local_lease_manager_interface", ":scheduler_internal", - "//src/ray/common:lease", "//src/ray/common:ray_config", "@com_google_absl//absl/container:flat_hash_map", ], @@ -160,8 +158,7 @@ ray_cc_library( hdrs = ["policy/scheduling_context.h"], deps = [ "//src/ray/common:id", - "//src/ray/common:lease", - "@com_google_absl//absl/container:flat_hash_map", + "//src/ray/common:placement_group", ], ) @@ -171,7 +168,7 @@ ray_cc_library( hdrs = ["policy/affinity_with_bundle_scheduling_policy.h"], deps = [ ":scheduling_policy", - "//src/ray/common:lease", + "//src/ray/common:bundle_location_index", ], ) @@ -184,7 +181,6 @@ ray_cc_library( ":scheduling_context", ":scheduling_policy", ":scorer", - "//src/ray/common:lease", ], ) @@ -258,7 +254,9 @@ ray_cc_library( name = "scorer", srcs = ["policy/scorer.cc"], hdrs = ["policy/scorer.h"], - deps = ["//src/ray/common:lease"], + deps = [ + "//src/ray/common/scheduling:cluster_resource_data", + ], ) ray_cc_library( @@ -266,6 +264,6 @@ ray_cc_library( hdrs = ["policy/scheduling_policy.h"], deps = [ ":scheduling_options", - "//src/ray/common:lease", + "//src/ray/common/scheduling:cluster_resource_data", ], ) diff --git a/src/ray/raylet/scheduling/cluster_lease_manager.cc b/src/ray/raylet/scheduling/cluster_lease_manager.cc index f96973da9574..61893fc29de8 100644 --- a/src/ray/raylet/scheduling/cluster_lease_manager.cc +++ b/src/ray/raylet/scheduling/cluster_lease_manager.cc @@ -21,7 +21,6 @@ #include #include -#include "ray/stats/metric_defs.h" #include "ray/util/logging.h" #include "ray/util/string_utils.h" @@ -161,7 +160,7 @@ bool ClusterLeaseManager::IsWorkWithResourceShape( SchedulingClass scheduling_class = work->lease_.GetLeaseSpecification().GetSchedulingClass(); ResourceSet resource_set = - TaskSpecification::GetSchedulingClassDescriptor(scheduling_class).resource_set; + SchedulingClassToIds::GetSchedulingClassDescriptor(scheduling_class).resource_set; for (const auto &target_resource_shape : target_resource_shapes) { if (resource_set == target_resource_shape) { return true; diff --git a/src/ray/raylet/scheduling/cluster_lease_manager.h b/src/ray/raylet/scheduling/cluster_lease_manager.h index 89f15ba62417..2e0efaa58a0e 100644 --- a/src/ray/raylet/scheduling/cluster_lease_manager.h +++ b/src/ray/raylet/scheduling/cluster_lease_manager.h @@ -20,7 +20,6 @@ #include "absl/container/flat_hash_map.h" #include "ray/common/lease/lease.h" -#include "ray/common/ray_object.h" #include "ray/raylet/scheduling/cluster_lease_manager_interface.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/scheduling/local_lease_manager_interface.h" diff --git a/src/ray/raylet/scheduling/internal.h b/src/ray/raylet/scheduling/internal.h index dfb6d130a618..19f2f587be56 100644 --- a/src/ray/raylet/scheduling/internal.h +++ b/src/ray/raylet/scheduling/internal.h @@ -18,7 +18,6 @@ #include #include "ray/common/lease/lease.h" -#include "ray/common/ray_object.h" #include "ray/common/scheduling/cluster_resource_data.h" #include "src/ray/protobuf/node_manager.pb.h" diff --git a/src/ray/raylet/scheduling/local_lease_manager_interface.h b/src/ray/raylet/scheduling/local_lease_manager_interface.h index dedf7ef53b98..8017efb1be13 100644 --- a/src/ray/raylet/scheduling/local_lease_manager_interface.h +++ b/src/ray/raylet/scheduling/local_lease_manager_interface.h @@ -19,10 +19,11 @@ #include #include "absl/container/flat_hash_map.h" -#include "ray/common/lease/lease.h" #include "ray/raylet/scheduling/internal.h" namespace ray { +class RayLease; + namespace raylet { // Forward declaration diff --git a/src/ray/raylet/scheduling/local_resource_manager.cc b/src/ray/raylet/scheduling/local_resource_manager.cc index f30a8997920d..f0e5064a511b 100644 --- a/src/ray/raylet/scheduling/local_resource_manager.cc +++ b/src/ray/raylet/scheduling/local_resource_manager.cc @@ -22,9 +22,10 @@ #include #include -#include "ray/common/grpc_util.h" -#include "ray/common/ray_config.h" +#include "ray/common/scheduling/placement_group_util.h" +#include "ray/common/scheduling/resource_set.h" #include "ray/stats/metric_defs.h" +#include "ray/util/logging.h" namespace ray { diff --git a/src/ray/raylet/scheduling/local_resource_manager.h b/src/ray/raylet/scheduling/local_resource_manager.h index 72c15ceb6514..3d3329c2fc0e 100644 --- a/src/ray/raylet/scheduling/local_resource_manager.h +++ b/src/ray/raylet/scheduling/local_resource_manager.h @@ -21,12 +21,9 @@ #include #include "absl/container/flat_hash_map.h" -#include "ray/common/bundle_spec.h" #include "ray/common/ray_syncer/ray_syncer.h" #include "ray/common/scheduling/cluster_resource_data.h" #include "ray/common/scheduling/fixed_point.h" -#include "ray/common/scheduling/resource_set.h" -#include "ray/util/logging.h" #include "src/ray/protobuf/gcs.pb.h" #include "src/ray/protobuf/node_manager.pb.h" diff --git a/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.h b/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.h index fe82f9fd55e5..4159b1a5c468 100644 --- a/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.h +++ b/src/ray/raylet/scheduling/policy/bundle_scheduling_policy.h @@ -16,8 +16,6 @@ #include -#include "ray/common/bundle_spec.h" -#include "ray/common/scheduling/fixed_point.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" #include "ray/raylet/scheduling/policy/scheduling_context.h" #include "ray/raylet/scheduling/policy/scheduling_policy.h" diff --git a/src/ray/raylet/scheduling/policy/scheduling_context.h b/src/ray/raylet/scheduling/policy/scheduling_context.h index 7dc71956d018..5d98a387cded 100644 --- a/src/ray/raylet/scheduling/policy/scheduling_context.h +++ b/src/ray/raylet/scheduling/policy/scheduling_context.h @@ -14,9 +14,6 @@ #pragma once -#include "absl/container/flat_hash_map.h" -#include "ray/common/bundle_location_index.h" -#include "ray/common/bundle_spec.h" #include "ray/common/id.h" #include "ray/common/placement_group.h" diff --git a/src/ray/raylet/scheduling/policy/scorer.cc b/src/ray/raylet/scheduling/policy/scorer.cc index 6bb07b8007ac..c53812b0abc2 100644 --- a/src/ray/raylet/scheduling/policy/scorer.cc +++ b/src/ray/raylet/scheduling/policy/scorer.cc @@ -14,8 +14,6 @@ #include "ray/raylet/scheduling/policy/scorer.h" -#include - namespace ray { namespace raylet_scheduling_policy { diff --git a/src/ray/raylet/scheduling/policy/scorer.h b/src/ray/raylet/scheduling/policy/scorer.h index cfc22a040958..e2bd1cfb2c72 100644 --- a/src/ray/raylet/scheduling/policy/scorer.h +++ b/src/ray/raylet/scheduling/policy/scorer.h @@ -13,7 +13,6 @@ // limitations under the License. #pragma once -#include #include "ray/common/scheduling/cluster_resource_data.h" diff --git a/src/ray/raylet/scheduling/scheduler_resource_reporter.cc b/src/ray/raylet/scheduling/scheduler_resource_reporter.cc index 35be2849e6da..a728bd69e214 100644 --- a/src/ray/raylet/scheduling/scheduler_resource_reporter.cc +++ b/src/ray/raylet/scheduling/scheduler_resource_reporter.cc @@ -22,6 +22,8 @@ #include #include +#include "ray/common/ray_config.h" + namespace ray { namespace raylet { @@ -78,7 +80,7 @@ void SchedulerResourceReporter::FillResourceUsage(rpc::ResourcesData &data) cons } const auto &scheduling_class_descriptor = - TaskSpecification::GetSchedulingClassDescriptor(scheduling_class); + SchedulingClassToIds::GetSchedulingClassDescriptor(scheduling_class); if ((scheduling_class_descriptor.scheduling_strategy.scheduling_strategy_case() == rpc::SchedulingStrategy::SchedulingStrategyCase:: kNodeAffinitySchedulingStrategy) && @@ -184,7 +186,7 @@ void SchedulerResourceReporter::FillPendingActorCountByShape( for (const auto &shape_entry : pending_count_by_shape) { auto by_shape_entry = resource_load_by_shape->Add(); for (const auto &resource_entry : - TaskSpecification::GetSchedulingClassDescriptor(shape_entry.first) + SchedulingClassToIds::GetSchedulingClassDescriptor(shape_entry.first) .resource_set.GetResourceMap()) { (*by_shape_entry->mutable_shape())[resource_entry.first] = resource_entry.second; } diff --git a/src/ray/raylet/scheduling/scheduler_resource_reporter.h b/src/ray/raylet/scheduling/scheduler_resource_reporter.h index 3357bd484a3f..5bc12c5d4139 100644 --- a/src/ray/raylet/scheduling/scheduler_resource_reporter.h +++ b/src/ray/raylet/scheduling/scheduler_resource_reporter.h @@ -17,8 +17,6 @@ #include #include "absl/container/flat_hash_map.h" -#include "ray/common/ray_config.h" -#include "ray/common/task/task_spec.h" #include "ray/raylet/scheduling/internal.h" #include "ray/raylet/scheduling/local_lease_manager_interface.h" diff --git a/src/ray/raylet/scheduling/scheduler_stats.h b/src/ray/raylet/scheduling/scheduler_stats.h index 3974ba6f0a2e..21b2ef86738a 100644 --- a/src/ray/raylet/scheduling/scheduler_stats.h +++ b/src/ray/raylet/scheduling/scheduler_stats.h @@ -16,10 +16,6 @@ #include -#include "absl/container/flat_hash_map.h" -#include "ray/common/ray_config.h" -#include "ray/common/task/task_spec.h" -#include "ray/raylet/scheduling/internal.h" #include "ray/raylet/scheduling/local_lease_manager_interface.h" #include "ray/stats/metric.h" diff --git a/src/ray/raylet/scheduling/scheduling_policy.h b/src/ray/raylet/scheduling/scheduling_policy.h index fe689355ac76..5fd17b7bf137 100644 --- a/src/ray/raylet/scheduling/scheduling_policy.h +++ b/src/ray/raylet/scheduling/scheduling_policy.h @@ -18,7 +18,6 @@ #include "ray/common/ray_config.h" #include "ray/common/scheduling/cluster_resource_data.h" -#include "ray/gcs/gcs_client/gcs_client.h" namespace ray { namespace raylet_scheduling_policy { diff --git a/src/ray/raylet/scheduling/tests/BUILD.bazel b/src/ray/raylet/scheduling/tests/BUILD.bazel index fa88ef1be1cb..15e6820c6873 100644 --- a/src/ray/raylet/scheduling/tests/BUILD.bazel +++ b/src/ray/raylet/scheduling/tests/BUILD.bazel @@ -11,6 +11,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:lease", "//src/ray/common:ray_config", + "//src/ray/common:task_common", "//src/ray/common:test_utils", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/raylet/scheduling:cluster_resource_scheduler", diff --git a/src/ray/raylet/tests/BUILD.bazel b/src/ray/raylet/tests/BUILD.bazel index 1577fce0758a..bf264b24fecc 100644 --- a/src/ray/raylet/tests/BUILD.bazel +++ b/src/ray/raylet/tests/BUILD.bazel @@ -70,6 +70,7 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:id", + "//src/ray/common/scheduling:placement_group_util", "//src/ray/raylet:placement_group_resource_manager", "@com_google_googletest//:gtest_main", ], @@ -100,7 +101,6 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", - "//src/ray/common:lease", "//src/ray/common:test_utils", "//src/ray/observability:fake_metric", "//src/ray/raylet:lease_dependency_manager", diff --git a/src/ray/raylet/tests/lease_dependency_manager_test.cc b/src/ray/raylet/tests/lease_dependency_manager_test.cc index 250032273c36..73b41cd0d688 100644 --- a/src/ray/raylet/tests/lease_dependency_manager_test.cc +++ b/src/ray/raylet/tests/lease_dependency_manager_test.cc @@ -14,7 +14,6 @@ #include "ray/raylet/lease_dependency_manager.h" -#include #include #include #include diff --git a/src/ray/raylet/tests/placement_group_resource_manager_test.cc b/src/ray/raylet/tests/placement_group_resource_manager_test.cc index c37fdc477cc0..f3cb445677fe 100644 --- a/src/ray/raylet/tests/placement_group_resource_manager_test.cc +++ b/src/ray/raylet/tests/placement_group_resource_manager_test.cc @@ -23,6 +23,7 @@ #include "mock/ray/gcs/gcs_client/gcs_client.h" #include "ray/common/bundle_spec.h" #include "ray/common/id.h" +#include "ray/common/scheduling/placement_group_util.h" #include "ray/common/scheduling/resource_set.h" namespace ray { diff --git a/src/ray/raylet/worker.cc b/src/ray/raylet/worker.cc index 8f5a59ef399d..b4f908e114e6 100644 --- a/src/ray/raylet/worker.cc +++ b/src/ray/raylet/worker.cc @@ -31,7 +31,7 @@ namespace raylet { Worker::Worker(const JobID &job_id, int runtime_env_hash, const WorkerID &worker_id, - const Language &language, + const rpc::Language &language, rpc::WorkerType worker_type, const std::string &ip_address, std::shared_ptr connection, @@ -120,7 +120,7 @@ void Worker::SetStartupToken(StartupToken startup_token) { startup_token_ = startup_token; } -Language Worker::GetLanguage() const { return language_; } +rpc::Language Worker::GetLanguage() const { return language_; } const std::string Worker::IpAddress() const { return ip_address_; } diff --git a/src/ray/raylet/worker.h b/src/ray/raylet/worker.h index 115ff53ce11d..d7466dac569e 100644 --- a/src/ray/raylet/worker.h +++ b/src/ray/raylet/worker.h @@ -56,7 +56,7 @@ class WorkerInterface { /// Return the worker process's startup token virtual StartupToken GetStartupToken() const = 0; virtual void SetProcess(Process proc) = 0; - virtual Language GetLanguage() const = 0; + virtual rpc::Language GetLanguage() const = 0; virtual const std::string IpAddress() const = 0; virtual void AsyncNotifyGCSRestart() = 0; /// Connect this worker's gRPC client. @@ -144,7 +144,7 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa Worker(const JobID &job_id, int runtime_env_hash, const WorkerID &worker_id, - const Language &language, + const rpc::Language &language, rpc::WorkerType worker_type, const std::string &ip_address, std::shared_ptr connection, @@ -169,7 +169,7 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa /// Return the worker process's startup token StartupToken GetStartupToken() const; void SetProcess(Process proc); - Language GetLanguage() const; + rpc::Language GetLanguage() const; const std::string IpAddress() const; void AsyncNotifyGCSRestart(); /// Connect this worker's gRPC client. @@ -269,7 +269,7 @@ class Worker : public std::enable_shared_from_this, public WorkerInterfa /// The worker's process's startup_token StartupToken startup_token_; /// The language type of this worker. - Language language_; + rpc::Language language_; /// The type of the worker. rpc::WorkerType worker_type_; /// IP address of this worker. diff --git a/src/ray/rpc/BUILD.bazel b/src/ray/rpc/BUILD.bazel index 1fb75da79340..584020804e6a 100644 --- a/src/ray/rpc/BUILD.bazel +++ b/src/ray/rpc/BUILD.bazel @@ -192,7 +192,6 @@ ray_cc_library( ], visibility = ["//visibility:public"], deps = [ - ":client_call", "//src/ray/protobuf:autoscaler_cc_proto", "//src/ray/protobuf:common_cc_proto", "//src/ray/protobuf:node_manager_cc_proto", @@ -220,8 +219,8 @@ ray_cc_library( deps = [ ":raylet_client_interface", ":retryable_grpc_client", + "//src/ray/common:bundle_spec", "//src/ray/common:ray_config", - "//src/ray/common:task_common", "//src/ray/protobuf:node_manager_cc_grpc", "//src/ray/util:logging", ], diff --git a/src/ray/rpc/raylet/raylet_client_interface.h b/src/ray/rpc/raylet/raylet_client_interface.h index 3480f86be848..f0d72452f547 100644 --- a/src/ray/rpc/raylet/raylet_client_interface.h +++ b/src/ray/rpc/raylet/raylet_client_interface.h @@ -16,13 +16,17 @@ #include #include +#include #include -#include "ray/rpc/client_call.h" #include "src/ray/protobuf/autoscaler.pb.h" #include "src/ray/protobuf/common.pb.h" #include "src/ray/protobuf/node_manager.pb.h" +// Maps from resource name to its allocation. +using ResourceMappingType = + std::unordered_map>>; + namespace grpc { class Channel; } @@ -37,6 +41,11 @@ class LeaseID; class NodeID; class BundleSpecification; +namespace rpc { +template +using ClientCallback = std::function; +} + class RayletClientInterface { public: /// Request to a raylet to pin a plasma object. The callback will be sent via gRPC. @@ -44,7 +53,7 @@ class RayletClientInterface { const rpc::Address &caller_address, const std::vector &object_ids, const ObjectID &generator_id, - const ray::rpc::ClientCallback &callback) = 0; + const rpc::ClientCallback &callback) = 0; /// Requests a worker from the raylet. The callback will be sent via gRPC. /// \param lease_spec Lease that is requested by the owner. @@ -56,7 +65,7 @@ class RayletClientInterface { virtual void RequestWorkerLease( const rpc::LeaseSpec &lease_spec, bool grant_or_reject, - const ray::rpc::ClientCallback &callback, + const rpc::ClientCallback &callback, const int64_t backlog_size = -1, const bool is_selected_based_on_locality = false) = 0; From 0409101165cb98263fbdd4945c75fb9e93bb311f Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Thu, 11 Sep 2025 09:39:46 -0700 Subject: [PATCH 575/634] [serve] fix release tests that use locust (#56354) Locust's mass monkey patching doesn't play well with ray. I don't know the exact reason but importing locust will hang indefinitely when inside a ray worker, which is what caused failures like https://buildkite.com/ray-project/release/builds/57072#019928b0-e84b-4b6e-8760-adb0d4ac7728. This specific behavior (that importing locust inside a ray worker will hang indefinitely) is new, probably due to new changes in ray because we pin locust versions for release tests, but the fact that there's issues between ray and locust has always been the case. Previous to this we also had to delay importing locust until inside the worker because importing in the driver meant the driver couldn't connect to the ray cluster. I found that turning off locust's monkey patching fixes this. Still need to figure out side effects --------- Signed-off-by: Cindy Zhang --- .../serve/_private/benchmarks/locust_utils.py | 279 +++++++++++++ release/serve_tests/workloads/locust_utils.py | 375 ++++++------------ 2 files changed, 404 insertions(+), 250 deletions(-) create mode 100644 python/ray/serve/_private/benchmarks/locust_utils.py diff --git a/python/ray/serve/_private/benchmarks/locust_utils.py b/python/ray/serve/_private/benchmarks/locust_utils.py new file mode 100644 index 000000000000..7949b69ea52e --- /dev/null +++ b/python/ray/serve/_private/benchmarks/locust_utils.py @@ -0,0 +1,279 @@ +import argparse +import logging +import time +from dataclasses import asdict, dataclass +from typing import Any, Dict, List + +from ray.serve._private.utils import generate_request_id + +logger = logging.getLogger(__file__) +logging.basicConfig(level=logging.INFO) + +MASTER_PORT = 5557 + + +@dataclass +class LocustStage: + duration_s: int + users: int + spawn_rate: float + + +@dataclass +class PerformanceStats: + p50_latency: float + p90_latency: float + p99_latency: float + rps: float + + +@dataclass +class LocustTestResults: + history: List[Dict] + total_requests: int + num_failures: int + avg_latency: float + p50_latency: float + p90_latency: float + p99_latency: float + avg_rps: float + stats_in_stages: List[PerformanceStats] + + +@dataclass +class FailedRequest: + request_id: str + status_code: int + exception: str + response_time_ms: float + start_time_s: float + + +class LocustClient: + def __init__( + self, + host_url: str, + token: str, + data: Dict[str, Any] = None, + ): + from locust import FastHttpUser, constant, events, task + from locust.contrib.fasthttp import FastResponse + + self.errors = [] + self.stats_in_stages: List[PerformanceStats] = [] + + class EndpointUser(FastHttpUser): + wait_time = constant(0) + failed_requests = [] + host = host_url + + @task + def test(self): + request_id = generate_request_id() + headers = ( + {"Authorization": f"Bearer {token}", "X-Request-ID": request_id} + if token + else None + ) + with self.client.get( + "", headers=headers, json=data, catch_response=True + ) as r: + r.request_meta["context"]["request_id"] = request_id + + @events.request.add_listener + def on_request( + response: FastResponse, + exception, + context, + start_time: float, + response_time: float, + **kwargs, + ): + if exception and response.status_code != 0: + request_id = context["request_id"] + print( + f"Request '{request_id}' failed with exception:\n" + f"{exception}\n{response.text}" + ) + + if response.status_code != 0: + response.encoding = "utf-8" + err = FailedRequest( + request_id=request_id, + status_code=response.status_code, + exception=response.text, + response_time_ms=response_time, + start_time_s=start_time, + ) + self.errors.append(err) + print( + f"Request '{request_id}' failed with exception:\n" + f"{exception}\n{response.text}" + ) + + self.user_class = EndpointUser + + +def on_stage_finished(master_runner, stats_in_stages): + stats_entry_key = ("", "GET") + stats_entry = master_runner.stats.entries.get(stats_entry_key) + + stats_in_stages.append( + PerformanceStats( + p50_latency=stats_entry.get_current_response_time_percentile(0.5), + p90_latency=stats_entry.get_current_response_time_percentile(0.9), + p99_latency=stats_entry.get_current_response_time_percentile(0.99), + rps=stats_entry.current_rps, + ) + ) + + +def run_locust_worker( + master_address: str, host_url: str, token: str, data: Dict[str, Any] +): + import locust + from locust.env import Environment + from locust.log import setup_logging + + setup_logging("INFO") + client = LocustClient(host_url=host_url, token=token, data=data) + env = Environment(user_classes=[client.user_class], events=locust.events) + + runner = env.create_worker_runner( + master_host=master_address, master_port=MASTER_PORT + ) + runner.greenlet.join() + + if client.errors: + raise RuntimeError(f"There were {len(client.errors)} errors: {client.errors}") + + +def run_locust_master( + host_url: str, + token: str, + expected_num_workers: int, + stages: List[LocustStage], + wait_for_workers_timeout_s: float, +): + import gevent + import locust + from locust import LoadTestShape + from locust.env import Environment + from locust.stats import ( + get_error_report_summary, + get_percentile_stats_summary, + get_stats_summary, + stats_history, + stats_printer, + ) + + client = LocustClient(host_url, token) + + class StagesShape(LoadTestShape): + curr_stage_ix = 0 + + def tick(cls): + run_time = cls.get_run_time() + prefix_time = 0 + for i, stage in enumerate(stages): + prefix_time += stage.duration_s + + if run_time < prefix_time: + if i != cls.curr_stage_ix: + on_stage_finished(master_runner, client.stats_in_stages) + cls.curr_stage_ix = i + + current_stage = stages[cls.curr_stage_ix] + return current_stage.users, current_stage.spawn_rate + + # End of stage test + on_stage_finished(master_runner, client.stats_in_stages) + + master_env = Environment( + user_classes=[client.user_class], + shape_class=StagesShape(), + events=locust.events, + ) + master_runner = master_env.create_master_runner("*", MASTER_PORT) + + start = time.time() + while len(master_runner.clients.ready) < expected_num_workers: + if time.time() - start > wait_for_workers_timeout_s: + raise RuntimeError( + f"Timed out waiting for {expected_num_workers} workers to " + "connect to Locust master." + ) + + print( + f"Waiting for workers to be ready, " + f"{len(master_runner.clients.ready)} " + f"of {expected_num_workers} ready." + ) + time.sleep(1) + + # Periodically output current stats (each entry is aggregated + # stats over the past 10 seconds, by default) + gevent.spawn(stats_printer(master_env.stats)) + gevent.spawn(stats_history, master_runner) + + # Start test & wait for the shape test to finish + master_runner.start_shape() + master_runner.shape_greenlet.join() + # Send quit signal to all locust workers + master_runner.quit() + + # Print stats + for line in get_stats_summary(master_runner.stats, current=False): + print(line) + # Print percentile stats + for line in get_percentile_stats_summary(master_runner.stats): + print(line) + # Print error report + if master_runner.stats.errors: + for line in get_error_report_summary(master_runner.stats): + print(line) + + stats_entry_key = ("", "GET") + stats_entry = master_runner.stats.entries.get(stats_entry_key) + results = LocustTestResults( + history=master_runner.stats.history, + total_requests=master_runner.stats.num_requests, + num_failures=master_runner.stats.num_failures, + avg_latency=stats_entry.avg_response_time, + p50_latency=stats_entry.get_response_time_percentile(0.5), + p90_latency=stats_entry.get_response_time_percentile(0.9), + p99_latency=stats_entry.get_response_time_percentile(0.99), + avg_rps=stats_entry.total_rps, + stats_in_stages=client.stats_in_stages, + ) + return asdict(results) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--worker-type", type=str, required=True) + parser.add_argument("--host-url", type=str, required=True) + parser.add_argument("--token", type=str, required=True) + parser.add_argument("--master-address", type=str, required=False) + parser.add_argument("--expected-num-workers", type=int, required=False) + parser.add_argument("--stages", type=str, required=False) + parser.add_argument("--wait-for-workers-timeout-s", type=float, required=False) + args = parser.parse_args() + host_url = args.host_url + token = args.token + if args.worker_type == "master": + results = run_locust_master( + host_url, + token, + args.expected_num_workers, + args.stages, + args.wait_for_workers_timeout_s, + ) + else: + results = run_locust_worker(args.master_address, host_url, token, args.data) + + print(results) + + +if __name__ == "__main__": + main() diff --git a/release/serve_tests/workloads/locust_utils.py b/release/serve_tests/workloads/locust_utils.py index 59c65e4e7e5c..6242b5e1dc0a 100644 --- a/release/serve_tests/workloads/locust_utils.py +++ b/release/serve_tests/workloads/locust_utils.py @@ -1,26 +1,23 @@ from dataclasses import asdict, dataclass -from itertools import chain +import os +import sys +import subprocess import json import logging -import time -from tqdm import tqdm -from typing import Any, Dict, List +from typing import Any, List import ray -from ray.serve._private.utils import generate_request_id +from ray.serve._private.benchmarks.locust_utils import ( + LocustStage, + LocustTestResults, + PerformanceStats, +) logger = logging.getLogger(__file__) logging.basicConfig(level=logging.INFO) -@dataclass -class LocustStage: - duration_s: int - users: int - spawn_rate: float - - @dataclass class LocustLoadTestConfig: num_workers: int @@ -31,247 +28,116 @@ class LocustLoadTestConfig: wait_for_workers_timeout_s: float = 600 -@dataclass -class PerformanceStats: - p50_latency: float - p90_latency: float - p99_latency: float - rps: float - - -@dataclass -class LocustTestResults: - history: List[Dict] - total_requests: int - num_failures: int - avg_latency: float - p50_latency: float - p90_latency: float - p99_latency: float - avg_rps: float - stats_in_stages: List[PerformanceStats] - - -@dataclass -class FailedRequest: - request_id: str - status_code: int - exception: str - response_time_ms: float - start_time_s: float - - -class LocustClient: - def __init__( - self, - host_url: str, - token: str, - data: Dict[str, Any] = None, - ): - from locust import task, constant, events, FastHttpUser - from locust.contrib.fasthttp import FastResponse - - self.errors = [] - - class EndpointUser(FastHttpUser): - wait_time = constant(0) - failed_requests = [] - host = host_url - - @task - def test(self): - request_id = generate_request_id() - headers = ( - {"Authorization": f"Bearer {token}", "X-Request-ID": request_id} - if token - else None - ) - with self.client.get( - "", headers=headers, json=data, catch_response=True - ) as r: - r.request_meta["context"]["request_id"] = request_id - - @events.request.add_listener - def on_request( - response: FastResponse, - exception, - context, - start_time: float, - response_time: float, - **kwargs, - ): - if exception: - request_id = context["request_id"] - response.encoding = "utf-8" - err = FailedRequest( - request_id=request_id, - status_code=response.status_code, - exception=response.text, - response_time_ms=response_time, - start_time_s=start_time, - ) - self.errors.append(err) - print( - f"Request '{request_id}' failed with exception: {response.text}" - ) - - self.user_class = EndpointUser - - -@ray.remote(num_cpus=1) -class LocustWorker(LocustClient): - def __init__( - self, - host_url: str, - token: str, - master_address: str, - data: Dict[str, Any] = None, - ): - # NOTE(zcin): We need to lazily import locust because the driver - # script won't connect to ray properly otherwise. - import locust - from locust.env import Environment - from locust.log import setup_logging - - super().__init__(host_url=host_url, token=token, data=data) - setup_logging("INFO") - self.env = Environment(user_classes=[self.user_class], events=locust.events) - self.master_address = master_address - - def run(self) -> List[Dict]: - runner = self.env.create_worker_runner( - master_host=self.master_address, master_port=5557 - ) - runner.greenlet.join() - return self.errors - - @ray.remote(num_cpus=1) -class LocustMaster(LocustClient): +class LocustProcess: def __init__( self, + worker_type: str, host_url: str, token: str, - expected_num_workers: int, - stages: List[LocustStage], - wait_for_workers_timeout_s: float, + expected_num_workers: int = None, + stages: List[LocustStage] = None, + wait_for_workers_timeout_s: float = None, + data: Any = None, + master_address: str = None, ): - # NOTE(zcin): We need to lazily import locust because the driver - # script won't connect to ray properly otherwise. - import locust - from locust import LoadTestShape - from locust.env import Environment - from locust.log import setup_logging - - super().__init__(host_url=host_url, token=token) - setup_logging("INFO") - - self.stats_in_stages: List[PerformanceStats] = [] - - class StagesShape(LoadTestShape): - curr_stage_ix = 0 - - def tick(cls): - run_time = cls.get_run_time() - prefix_time = 0 - for i, stage in enumerate(stages): - prefix_time += stage.duration_s - - if run_time < prefix_time: - if i != cls.curr_stage_ix: - self.on_stage_finished() - cls.curr_stage_ix = i - - current_stage = stages[cls.curr_stage_ix] - return current_stage.users, current_stage.spawn_rate - - # End of stage test - self.on_stage_finished() - - self.master_env = Environment( - user_classes=[self.user_class], - shape_class=StagesShape(), - events=locust.events, - ) + self.worker_type = worker_type + self.host_url = host_url + self.token = token self.expected_num_workers = expected_num_workers + self.stages = stages self.wait_for_workers_timeout_s = wait_for_workers_timeout_s - self.master_runner = None + self.data = data + self.master_address = master_address - def on_stage_finished(self): - stats_entry_key = ("", "GET") - stats_entry = self.master_runner.stats.entries.get(stats_entry_key) + def run(self): + # Create a temporary file for results + import tempfile - self.stats_in_stages.append( - PerformanceStats( - p50_latency=stats_entry.get_current_response_time_percentile(0.5), - p90_latency=stats_entry.get_current_response_time_percentile(0.9), - p99_latency=stats_entry.get_current_response_time_percentile(0.99), - rps=stats_entry.current_rps, - ) + results_file = tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".json" ) + results_file.close() - def run(self): - import gevent - from locust.stats import ( - get_stats_summary, - get_percentile_stats_summary, - get_error_report_summary, - stats_history, - stats_printer, + # Prepare the subprocess script + if self.worker_type == "master": + script = f""" +import sys +import json +from ray.serve._private.benchmarks.locust_utils import run_locust_master, run_locust_worker, LocustStage + +stages = json.loads(sys.argv[1]) +stages = [LocustStage(**stage) for stage in stages] +results = run_locust_master( + host_url="{self.host_url}", + token="{self.token}", + expected_num_workers={self.expected_num_workers}, + stages=stages, + wait_for_workers_timeout_s={self.wait_for_workers_timeout_s} +) + +with open("{results_file.name}", 'w') as f: + json.dump(results, f) +""" + stages = json.dumps([asdict(stage) for stage in self.stages]) + cmd_args = [sys.executable, "-c", script, stages] + else: + script = f""" +import sys +import json +from ray.serve._private.benchmarks.locust_utils import run_locust_master, run_locust_worker, LocustStage + +data = sys.argv[1] +results = run_locust_worker( + master_address="{self.master_address}", + host_url="{self.host_url}", + token="{self.token}", + data=data, +) +""" + data = json.dumps(self.data) + cmd_args = [sys.executable, "-c", script, data] + + # Start the Locust process + self.process = subprocess.Popen( + cmd_args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, ) - - self.master_runner = self.master_env.create_master_runner("*", 5557) - - start = time.time() - while len(self.master_runner.clients.ready) < self.expected_num_workers: - if time.time() - start > self.wait_for_workers_timeout_s: - raise RuntimeError( - f"Timed out waiting for {self.expected_num_workers} workers to " - "connect to Locust master." + print(f"Started {self.worker_type} subprocess ({self.process.pid})") + + try: + # Wait for the process to complete first + for line in self.process.stdout: # yields as the child prints + sys.stdout.write(line) # stream to our stdout + + return_code = self.process.wait() + if return_code != 0: + # Clean up the results file on error + try: + os.unlink(results_file.name) + except OSError: + pass + raise RuntimeError(f"Subprocess failed with return code {return_code}.") + + # Read the result from the results file + with open(results_file.name, "r") as f: + result_data = f.read() + + if result_data: + result_data = json.loads(result_data) + stats_in_stages = [ + PerformanceStats(**stage) + for stage in result_data.pop("stats_in_stages") + ] + result = LocustTestResults( + **result_data, stats_in_stages=stats_in_stages ) - - print( - f"Waiting for workers to be ready, " - f"{len(self.master_runner.clients.ready)} " - f"of {self.expected_num_workers} ready." - ) - time.sleep(1) - - # Periodically output current stats (each entry is aggregated - # stats over the past 10 seconds, by default) - gevent.spawn(stats_printer(self.master_env.stats)) - gevent.spawn(stats_history, self.master_runner) - - # Start test & wait for the shape test to finish - self.master_runner.start_shape() - self.master_runner.shape_greenlet.join() - # Send quit signal to all locust workers - self.master_runner.quit() - - # Print stats - for line in get_stats_summary(self.master_runner.stats, current=False): - print(line) - # Print percentile stats - for line in get_percentile_stats_summary(self.master_runner.stats): - print(line) - # Print error report - if self.master_runner.stats.errors: - for line in get_error_report_summary(self.master_runner.stats): - print(line) - - stats_entry_key = ("", "GET") - stats_entry = self.master_runner.stats.entries.get(stats_entry_key) - return LocustTestResults( - history=self.master_runner.stats.history, - total_requests=self.master_runner.stats.num_requests, - num_failures=self.master_runner.stats.num_failures, - avg_latency=stats_entry.avg_response_time, - p50_latency=stats_entry.get_response_time_percentile(0.5), - p90_latency=stats_entry.get_response_time_percentile(0.9), - p99_latency=stats_entry.get_response_time_percentile(0.99), - avg_rps=stats_entry.total_rps, - stats_in_stages=self.stats_in_stages, - ) + return result + finally: + os.unlink(results_file.name) def run_locust_load_test(config: LocustLoadTestConfig) -> LocustTestResults: @@ -288,17 +154,20 @@ def run_locust_load_test(config: LocustLoadTestConfig) -> LocustTestResults: worker_refs = [] # Start Locust workers - for _ in tqdm(range(config.num_workers)): - locust_worker = LocustWorker.remote( + for i in range(config.num_workers): + locust_worker = LocustProcess.options(name=f"LocustWorker-{i}").remote( + worker_type="worker", host_url=config.host_url, token=config.auth_token, master_address=master_address, data=config.data, ) worker_refs.append(locust_worker.run.remote()) + print(f"Started worker {i}") # Start Locust master - master_worker = LocustMaster.remote( + master_worker = LocustProcess.options(name="LocustMaster").remote( + worker_type="master", host_url=config.host_url, token=config.auth_token, expected_num_workers=config.num_workers, @@ -309,13 +178,19 @@ def run_locust_load_test(config: LocustLoadTestConfig) -> LocustTestResults: # Collect results and metrics stats: LocustTestResults = ray.get(master_ref) - errors = sorted(chain(*ray.get(worker_refs)), key=lambda e: e.start_time_s) + ray.get(worker_refs) + return stats - # If there were any requests that failed, raise error. - if stats.num_failures > 0: - errors_json = [asdict(err) for err in errors] - raise RuntimeError( - f"There were failed requests: {json.dumps(errors_json, indent=4)}" - ) - return stats +if __name__ == "__main__": + ray.init(address="auto") + results = run_locust_load_test( + LocustLoadTestConfig( + num_workers=9, + host_url="https://services-canary-pinger-aws-zugs7.cld-kvedzwag2qa8i5bj.s.anyscaleuserdata.com/info", + auth_token="v9M8jb3tBbHOGoWrg7X1fCwF8wYn7gqZR5VZ1_h4t50", + data=None, + stages=[LocustStage(duration_s=10, users=10, spawn_rate=1)], + ) + ) + print(results) From 5e8107bdf87f8522a59b35709b3c37b6e1a73993 Mon Sep 17 00:00:00 2001 From: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> Date: Thu, 11 Sep 2025 10:52:06 -0700 Subject: [PATCH 576/634] [Serve] Refactor test_deploy_app_2.py and add port attributes (#55748) This PR refactors test_deploy_app_2.py and adds additional attributes to `deployment_state.py` and `replica.py` --------- Signed-off-by: doyoung Signed-off-by: Doyoung Kim <34902420+landscapepainter@users.noreply.github.com> --- python/ray/serve/_private/deployment_state.py | 50 ++++++- python/ray/serve/_private/replica.py | 8 +- python/ray/serve/_private/test_utils.py | 23 ++++ .../serve/tests/test_controller_recovery.py | 4 +- python/ray/serve/tests/test_deploy_app_2.py | 130 ++++++++---------- .../serve/tests/unit/test_deployment_state.py | 2 +- 6 files changed, 133 insertions(+), 84 deletions(-) diff --git a/python/ray/serve/_private/deployment_state.py b/python/ray/serve/_private/deployment_state.py index 04d5519aacf1..dd9a136632b4 100644 --- a/python/ray/serve/_private/deployment_state.py +++ b/python/ray/serve/_private/deployment_state.py @@ -248,7 +248,7 @@ def __init__( self._last_health_check_time: float = 0.0 self._consecutive_health_check_failures = 0 self._initialization_latency_s: Optional[float] = None - self._port: Optional[int] = None + self._internal_grpc_port: Optional[int] = None self._docs_path: Optional[str] = None # Rank assigned to the replica. self._rank: Optional[int] = None @@ -264,6 +264,8 @@ def __init__( self._node_ip: str = None self._node_instance_id: str = None self._log_file_path: str = None + self._http_port: int = None + self._grpc_port: int = None # Populated in self.stop(). self._graceful_shutdown_ref: ObjectRef = None @@ -365,6 +367,14 @@ def health_check_period_s(self) -> float: def health_check_timeout_s(self) -> float: return self.deployment_config.health_check_timeout_s + @property + def http_port(self) -> Optional[int]: + return self._http_port + + @property + def grpc_port(self) -> Optional[int]: + return self._grpc_port + @property def request_routing_stats_period_s(self) -> float: return ( @@ -752,8 +762,10 @@ def check_ready(self) -> Tuple[ReplicaStartupStatus, Optional[str]]: _, self._version, self._initialization_latency_s, - self._port, + self._internal_grpc_port, self._docs_path, + self._http_port, + self._grpc_port, self._rank, ) = ray.get(self._ready_obj_ref) except RayTaskError as e: @@ -1065,7 +1077,7 @@ def get_running_replica_info( is_cross_language=self._actor.is_cross_language, multiplexed_model_ids=self.multiplexed_model_ids, routing_stats=self.routing_stats, - port=self._actor._port, + port=self._actor._internal_grpc_port, ) def record_multiplexed_model_ids(self, multiplexed_model_ids: List[str]): @@ -1126,6 +1138,14 @@ def actor_node_id(self) -> Optional[str]: """Returns the node id of the actor, None if not placed.""" return self._actor.node_id + @property + def actor_http_port(self) -> Optional[int]: + return self._actor.http_port + + @property + def actor_grpc_port(self) -> Optional[int]: + return self._actor.grpc_port + @property def actor_pid(self) -> Optional[int]: """Returns the node id of the actor, None if not placed.""" @@ -2902,6 +2922,9 @@ def _stop_one_running_replica_for_testing(self): for replica in running_replicas: self._replicas.add(ReplicaState.RUNNING, replica) + def is_ingress(self) -> bool: + return self._target_state.info.ingress + class DeploymentStateManager: """Manages all state for deployments in the system. @@ -3421,6 +3444,27 @@ def get_active_node_ids(self) -> Set[str]: node_ids.update(deployment_state.get_active_node_ids()) return node_ids + def get_ingress_replicas_info(self) -> List[Tuple[str, str, int, int]]: + """Get all ingress replicas info for all deployments.""" + ingress_replicas_list = [ + deployment_state._replicas.get() + for deployment_state in self._deployment_states.values() + if deployment_state.is_ingress() + ] + + ingress_replicas_info = [] + for replicas in ingress_replicas_list: + for replica in replicas: + ingress_replicas_info.append( + ( + replica.actor_node_id, + replica.replica_id.unique_id, + replica.actor_http_port, + replica.actor_grpc_port, + ) + ) + return ingress_replicas_info + def _get_replica_ranks_mapping(self, deployment_id: DeploymentID) -> Dict[str, int]: """Get the current rank mapping for all replicas in a deployment. Args: diff --git a/python/ray/serve/_private/replica.py b/python/ray/serve/_private/replica.py index c8728d973ef3..ff1b8425c888 100644 --- a/python/ray/serve/_private/replica.py +++ b/python/ray/serve/_private/replica.py @@ -425,8 +425,10 @@ def __init__( ingress=ingress, ) - self._port: Optional[int] = None + self._internal_grpc_port: Optional[int] = None self._docs_path: Optional[str] = None + self._http_port: Optional[int] = None + self._grpc_port: Optional[int] = None @property def max_ongoing_requests(self) -> int: @@ -441,8 +443,10 @@ def get_metadata(self) -> ReplicaMetadata: self._version.deployment_config, self._version, self._initialization_latency, - self._port, + self._internal_grpc_port, self._docs_path, + self._http_port, + self._grpc_port, current_rank, ) diff --git a/python/ray/serve/_private/test_utils.py b/python/ray/serve/_private/test_utils.py index c3c9142ab475..acdff7669a51 100644 --- a/python/ray/serve/_private/test_utils.py +++ b/python/ray/serve/_private/test_utils.py @@ -717,6 +717,29 @@ def tlog(s: str, level: str = "INFO"): print(f"[{level}] {now} {s}") +def check_target_groups_ready( + client: ServeControllerClient, + app_name: str, + protocol: Union[str, RequestProtocol] = RequestProtocol.HTTP, +): + """Wait for target groups to be ready for the given app and protocol. + + Target groups are ready when there are at least one target for the given protocol. And it's + possible that target groups are not ready immediately. An example is when the controller + is recovering from a crash. + """ + target_groups = ray.get(client._controller.get_target_groups.remote(app_name)) + target_groups = [ + target_group + for target_group in target_groups + if target_group.protocol == protocol + ] + all_targets = [ + target for target_group in target_groups for target in target_group.targets + ] + return len(all_targets) > 0 + + def get_application_urls( protocol: Union[str, RequestProtocol] = RequestProtocol.HTTP, app_name: str = SERVE_DEFAULT_APP_NAME, diff --git a/python/ray/serve/tests/test_controller_recovery.py b/python/ray/serve/tests/test_controller_recovery.py index 52f405153a1e..535483411e71 100644 --- a/python/ray/serve/tests/test_controller_recovery.py +++ b/python/ray/serve/tests/test_controller_recovery.py @@ -65,7 +65,7 @@ def __call__(self, *args): replica_version_hash = None for replica in deployment_dict[id]: ref = replica.actor_handle.initialize_and_get_metadata.remote() - _, version, _, _, _, _ = ray.get(ref) + _, version, _, _, _, _, _, _ = ray.get(ref) if replica_version_hash is None: replica_version_hash = hash(version) assert replica_version_hash == hash(version), ( @@ -118,7 +118,7 @@ def __call__(self, *args): for replica_name in recovered_replica_names: actor_handle = ray.get_actor(replica_name, namespace=SERVE_NAMESPACE) ref = actor_handle.initialize_and_get_metadata.remote() - _, version, _, _, _, _ = ray.get(ref) + _, version, _, _, _, _, _, _ = ray.get(ref) assert replica_version_hash == hash( version ), "Replica version hash should be the same after recover from actor names" diff --git a/python/ray/serve/tests/test_deploy_app_2.py b/python/ray/serve/tests/test_deploy_app_2.py index 1b5acec54b4c..28ce0a1adb92 100644 --- a/python/ray/serve/tests/test_deploy_app_2.py +++ b/python/ray/serve/tests/test_deploy_app_2.py @@ -20,6 +20,8 @@ ) from ray.serve._private.test_utils import ( check_num_replicas_eq, + check_running, + check_target_groups_ready, get_application_url, ) from ray.serve.schema import ( @@ -28,7 +30,6 @@ ServeDeploySchema, ServeInstanceDetails, ) -from ray.serve.tests.test_deploy_app import check_running from ray.tests.conftest import call_ray_stop_only # noqa: F401 from ray.util.state import list_actors @@ -408,18 +409,9 @@ def test_deploy_does_not_affect_dynamic_apps(serve_instance): ], ) client.deploy_apps(config) - - def check_application_running( - name: str, route_prefix: str, *, msg: str = "wonderful world" - ): - status = serve.status().applications[name] - assert status.status == "RUNNING" - assert httpx.post(f"http://localhost:8000{route_prefix}/").text == msg - return True - - wait_for_condition( - check_application_running, name="declarative-app-1", route_prefix="/app-1" - ) + wait_for_condition(check_running, app_name="declarative-app-1") + url = get_application_url(app_name="declarative-app-1") + assert httpx.post(url).text == "wonderful world" # Now `serve.run` a dynamic app. @serve.deployment @@ -428,12 +420,9 @@ def __call__(self, *args) -> str: return "Hello!" serve.run(D.bind(), name="dynamic-app", route_prefix="/dynamic") - wait_for_condition( - check_application_running, - name="dynamic-app", - route_prefix="/dynamic", - msg="Hello!", - ) + wait_for_condition(check_running, app_name="dynamic-app") + url = get_application_url(app_name="dynamic-app") + assert httpx.post(url).text == "Hello!" # Add a new app via declarative API. # Existing declarative app and dynamic app should not be affected. @@ -445,46 +434,35 @@ def __call__(self, *args) -> str: ), ) client.deploy_apps(config) + wait_for_condition(check_running, app_name="declarative-app-2") + url = get_application_url(app_name="declarative-app-2") + assert httpx.post(url).text == "wonderful world" - wait_for_condition( - check_application_running, name="declarative-app-2", route_prefix="/app-2" - ) - wait_for_condition( - check_application_running, name="declarative-app-1", route_prefix="/app-1" - ) - wait_for_condition( - check_application_running, - name="dynamic-app", - route_prefix="/dynamic", - msg="Hello!", - ) + url = get_application_url(app_name="declarative-app-1") + assert httpx.post(url).text == "wonderful world" + + url = get_application_url(app_name="dynamic-app") + assert httpx.post(url).text == "Hello!" # Delete one of the apps via declarative API. # Other declarative app and dynamic app should not be affected. config.applications.pop(0) client.deploy_apps(config) + wait_for_condition(check_running, app_name="declarative-app-2") + url = get_application_url(app_name="declarative-app-2") + assert httpx.post(url).text == "wonderful world" - wait_for_condition( - check_application_running, name="declarative-app-2", route_prefix="/app-2" - ) - wait_for_condition( - check_application_running, - name="dynamic-app", - route_prefix="/dynamic", - msg="Hello!", - ) + url = get_application_url(app_name="dynamic-app") + assert httpx.post(url).text == "Hello!" wait_for_condition(lambda: "declarative-app-1" not in serve.status().applications) # Now overwrite the declarative app with a dynamic app with the same name. # On subsequent declarative apply, that app should not be affected. serve.run(D.bind(), name="declarative-app-2", route_prefix="/app-2") - wait_for_condition( - check_application_running, - name="declarative-app-2", - route_prefix="/app-2", - msg="Hello!", - ) + wait_for_condition(check_running, app_name="declarative-app-2") + url = get_application_url(app_name="declarative-app-2") + assert httpx.post(url).text == "Hello!" config.applications = [ ServeApplicationSchema( @@ -494,39 +472,41 @@ def __call__(self, *args) -> str: ), ] client.deploy_apps(config) + wait_for_condition(check_running, app_name="declarative-app-1") + url = get_application_url(app_name="declarative-app-1") + assert httpx.post(url).text == "wonderful world" - wait_for_condition( - check_application_running, - name="declarative-app-1", - route_prefix="/app-1", - ) - wait_for_condition( - check_application_running, - name="dynamic-app", - route_prefix="/dynamic", - msg="Hello!", - ) - wait_for_condition( - check_application_running, - name="declarative-app-2", - route_prefix="/app-2", - msg="Hello!", - ) + wait_for_condition(check_running, app_name="dynamic-app") + url = get_application_url(app_name="dynamic-app") + assert httpx.post(url).text == "Hello!" + + wait_for_condition(check_running, app_name="declarative-app-2") + url = get_application_url(app_name="declarative-app-2") + assert httpx.post(url).text == "Hello!" # Verify that the controller does not delete the dynamic apps on recovery. ray.kill(client._controller, no_restart=False) + + wait_for_condition(check_running, app_name="declarative-app-1") + # It takes some time for the target groups to be ready after controller recovery. + # So we make sure the target groups are ready before obtaining the URL. wait_for_condition( - check_application_running, - name="dynamic-app", - route_prefix="/dynamic", - msg="Hello!", + check_target_groups_ready, client=client, app_name="declarative-app-1" ) + url = get_application_url(app_name="declarative-app-1") + assert httpx.post(url).text == "wonderful world" + + wait_for_condition(check_running, app_name="dynamic-app") + wait_for_condition(check_target_groups_ready, client=client, app_name="dynamic-app") + url = get_application_url(app_name="dynamic-app") + assert httpx.post(url).text == "Hello!" + + wait_for_condition(check_running, app_name="declarative-app-2") wait_for_condition( - check_application_running, - name="declarative-app-2", - route_prefix="/app-2", - msg="Hello!", + check_target_groups_ready, client=client, app_name="declarative-app-2" ) + url = get_application_url(app_name="declarative-app-2") + assert httpx.post(url).text == "Hello!" # Now overwrite the dynamic app with a declarative one and check that it gets # deleted upon another apply that doesn't include it. @@ -538,11 +518,9 @@ def __call__(self, *args) -> str: ), ] client.deploy_apps(config) - wait_for_condition( - check_application_running, - name="declarative-app-2", - route_prefix="/app-2", - ) + wait_for_condition(check_running, app_name="declarative-app-2") + url = get_application_url(app_name="declarative-app-2") + assert httpx.post(url).text == "wonderful world" config.applications = [] client.deploy_apps(config) diff --git a/python/ray/serve/tests/unit/test_deployment_state.py b/python/ray/serve/tests/unit/test_deployment_state.py index aab09ebdbbaa..9bafe7b4f9b3 100644 --- a/python/ray/serve/tests/unit/test_deployment_state.py +++ b/python/ray/serve/tests/unit/test_deployment_state.py @@ -100,7 +100,7 @@ def __init__( self._node_instance_id = None self._node_id_is_set = False self._actor_id = None - self._port = None + self._internal_grpc_port = None self._pg_bundles = None self._initialization_latency_s = -1 self._docs_path = None From 9f713212abba86e3f7b2c6d9e35152f7a597225e Mon Sep 17 00:00:00 2001 From: Mengjin Yan Date: Thu, 11 Sep 2025 11:01:53 -0700 Subject: [PATCH 577/634] [Doc][Core] Fix the Environment Variable Name in the Ray Event Export Doc (#56455) Signed-off-by: Mengjin Yan --- doc/source/ray-observability/user-guides/ray-event-export.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/ray-observability/user-guides/ray-event-export.rst b/doc/source/ray-observability/user-guides/ray-event-export.rst index 4c44021062c2..89ef27bcc46c 100644 --- a/doc/source/ray-observability/user-guides/ray-event-export.rst +++ b/doc/source/ray-observability/user-guides/ray-event-export.rst @@ -23,7 +23,7 @@ Enable event reporting To enable event reporting, you need to set the ``RAY_enable_core_worker_ray_event_to_aggregator`` environment variable to ``1`` when starting each Ray worker node. -To set the target HTTP endpoint, set the ``RAY_events_export_addr`` +To set the target HTTP endpoint, set the ``RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR`` environment variable to a valid HTTP URL with the ``http://`` URL scheme. Event format From bdeb3687395dde48080baf260cc047106b58c672 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Thu, 11 Sep 2025 11:30:44 -0700 Subject: [PATCH 578/634] [core] deflake darwin://python/ray/tests:test_metric_cardinality_otel (#56456) The commit https://github.com/ray-project/ray/commit/ef7169a77d254a3c8e0779f80d61b2ba5c8a0198 fixed a bug where Ray continued emitting metrics from dead workers. However, this change made the test `darwin://python/ray/tests:test_metric_cardinality_otel` flaky. The test assumed that all metrics would appear in every Prometheus poll, but in reality, data points for different metrics can arrive at different intervals. This PR fixes the issue by checking for the presence of each metric independently. Test: - CI - Run test locally Closes https://github.com/ray-project/ray/issues/56449. Signed-off-by: Cuong Nguyen --- python/ray/tests/test_metric_cardinality.py | 91 +++++++++++---------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/python/ray/tests/test_metric_cardinality.py b/python/ray/tests/test_metric_cardinality.py index c7b429cdef9d..e44b37b3e3f0 100644 --- a/python/ray/tests/test_metric_cardinality.py +++ b/python/ray/tests/test_metric_cardinality.py @@ -75,7 +75,7 @@ async def run(self): ray.get(obj_refs) -def _cardinality_level_test(_setup_cluster_for_test, cardinality_level): +def _cardinality_level_test(_setup_cluster_for_test, cardinality_level, metric): """ Test that the ray_tasks and ray_actors metric are reported with the expected cardinality level """ @@ -84,43 +84,42 @@ def _cardinality_level_test(_setup_cluster_for_test, cardinality_level): def _validate(): metric_samples = fetch_prometheus_metrics(prom_addresses) - for metric in _TO_TEST_METRICS: - samples = metric_samples.get(metric) - assert samples, f"Metric {metric} not found in samples" - for sample in samples: - if cardinality_level == "recommended": - # If the cardinality level is recommended, the WorkerId tag should - # be removed - assert ( - sample.labels.get(WORKER_ID_TAG_KEY) is None - ), f"Sample {sample} contains WorkerId tag" - elif cardinality_level == "legacy": - # If the cardinality level is legacy, the WorkerId tag should be - # present - assert ( - sample.labels.get(WORKER_ID_TAG_KEY) is not None - ), f"Sample {sample} does not contain WorkerId tag" - if metric == "ray_tasks" or metric == "ray_actors": - assert ( - sample.labels.get(TASK_OR_ACTOR_NAME_TAG_KEY) is not None - ), f"Sample {sample} does not contain Name tag" - elif cardinality_level == "low": - # If the cardinality level is low, the WorkerId and Name tags should - # be removed + samples = metric_samples.get(metric) + assert samples, f"Metric {metric} not found in samples" + for sample in samples: + if cardinality_level == "recommended": + # If the cardinality level is recommended, the WorkerId tag should + # be removed + assert ( + sample.labels.get(WORKER_ID_TAG_KEY) is None + ), f"Sample {sample} contains WorkerId tag" + elif cardinality_level == "legacy": + # If the cardinality level is legacy, the WorkerId tag should be + # present + assert ( + sample.labels.get(WORKER_ID_TAG_KEY) is not None + ), f"Sample {sample} does not contain WorkerId tag" + if metric == "ray_tasks" or metric == "ray_actors": assert ( - sample.labels.get(WORKER_ID_TAG_KEY) is None - ), f"Sample {sample} contains WorkerId tag" - if metric == "ray_tasks" or metric == "ray_actors": - assert ( - sample.labels.get(TASK_OR_ACTOR_NAME_TAG_KEY) is None - ), f"Sample {sample} contains Name tag" - else: - raise ValueError(f"Unknown cardinality level: {cardinality_level}") - - # The Component tag should be present on all cardinality levels + sample.labels.get(TASK_OR_ACTOR_NAME_TAG_KEY) is not None + ), f"Sample {sample} does not contain Name tag" + elif cardinality_level == "low": + # If the cardinality level is low, the WorkerId and Name tags should + # be removed assert ( - sample.labels.get(_COMPONENT_TAG_KEY) is not None - ), f"Sample {sample} does not contain Component tag" + sample.labels.get(WORKER_ID_TAG_KEY) is None + ), f"Sample {sample} contains WorkerId tag" + if metric == "ray_tasks" or metric == "ray_actors": + assert ( + sample.labels.get(TASK_OR_ACTOR_NAME_TAG_KEY) is None + ), f"Sample {sample} contains Name tag" + else: + raise ValueError(f"Unknown cardinality level: {cardinality_level}") + + # The Component tag should be present on all cardinality levels + assert ( + sample.labels.get(_COMPONENT_TAG_KEY) is not None + ), f"Sample {sample} does not contain Component tag" wait_for_assertion( _validate, @@ -131,14 +130,18 @@ def _validate(): @pytest.mark.skipif(prometheus_client is None, reason="Prometheus not installed") @pytest.mark.parametrize( - "_setup_cluster_for_test,cardinality_level", - [("recommended", "recommended"), ("legacy", "legacy")], + "_setup_cluster_for_test,cardinality_level,metric", + [ + (cardinality, cardinality, metric) + for cardinality in ["recommended", "legacy"] + for metric in _TO_TEST_METRICS + ], indirect=["_setup_cluster_for_test"], ) def test_cardinality_recommended_and_legacy_levels( - _setup_cluster_for_test, cardinality_level + _setup_cluster_for_test, cardinality_level, metric ): - _cardinality_level_test(_setup_cluster_for_test, cardinality_level) + _cardinality_level_test(_setup_cluster_for_test, cardinality_level, metric) # We only enable low cardinality test for open telemetry because the legacy opencensus @@ -149,12 +152,12 @@ def test_cardinality_recommended_and_legacy_levels( reason="OpenTelemetry is not enabled", ) @pytest.mark.parametrize( - "_setup_cluster_for_test,cardinality_level", - [("low", "low")], + "_setup_cluster_for_test,cardinality_level,metric", + [("low", "low", metric) for metric in _TO_TEST_METRICS], indirect=["_setup_cluster_for_test"], ) -def test_cardinality_low_levels(_setup_cluster_for_test, cardinality_level): - _cardinality_level_test(_setup_cluster_for_test, cardinality_level) +def test_cardinality_low_levels(_setup_cluster_for_test, cardinality_level, metric): + _cardinality_level_test(_setup_cluster_for_test, cardinality_level, metric) if __name__ == "__main__": From f3f75d58ac213e5e3a00aaf043929c0e9fd78929 Mon Sep 17 00:00:00 2001 From: Alan Guo Date: Thu, 11 Sep 2025 12:25:08 -0700 Subject: [PATCH 579/634] Add operator panels id list to dataset export (#56428) ## Why are these changes needed? Adds a new resource utilization graph that combines CPU and GPU resources into a single graph. Exports the panel ids for OPERATOR_PANELS with the ray dataset metadata. This can be used by importers of the ray dataset data to determine which panels are relevant to show for dataset operators. This follows a similar patter to Train export (#53072) ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Alan Guo --- .../modules/metrics/dashboards/common.py | 9 ++++++ .../dashboards/data_dashboard_panels.py | 30 +++++++++++++++++++ .../ray/data/_internal/metadata_exporter.py | 20 +++++++++++++ .../protobuf/export_dataset_metadata.proto | 11 +++++++ 4 files changed, 70 insertions(+) diff --git a/python/ray/dashboard/modules/metrics/dashboards/common.py b/python/ray/dashboard/modules/metrics/dashboards/common.py index 4e6b82a21330..a4a1e4b4787e 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/common.py +++ b/python/ray/dashboard/modules/metrics/dashboards/common.py @@ -2,7 +2,10 @@ from enum import Enum from typing import List, Optional +from ray.util.annotations import DeveloperAPI + +@DeveloperAPI @dataclass class GridPos: x: int @@ -30,11 +33,13 @@ class GridPos: } +@DeveloperAPI class TargetTemplate(Enum): GRAPH = GRAPH_TARGET_TEMPLATE HEATMAP = HEATMAP_TARGET_TEMPLATE +@DeveloperAPI @dataclass class Target: """Defines a Grafana target (time-series query) within a panel. @@ -360,6 +365,7 @@ class Target: } +@DeveloperAPI class PanelTemplate(Enum): GRAPH = GRAPH_PANEL_TEMPLATE HEATMAP = HEATMAP_TEMPLATE @@ -368,6 +374,7 @@ class PanelTemplate(Enum): GAUGE = GAUGE_PANEL_TEMPLATE +@DeveloperAPI @dataclass class Panel: """Defines a Grafana panel (graph) for the Ray dashboard page. @@ -397,6 +404,7 @@ class Panel: template: Optional[PanelTemplate] = PanelTemplate.GRAPH +@DeveloperAPI @dataclass class Row: """Defines a Grafana row that can contain multiple panels. @@ -413,6 +421,7 @@ class Row: collapsed: bool = False +@DeveloperAPI @dataclass class DashboardConfig: # This dashboard name is an internal key used to determine which env vars diff --git a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py index bc185d3854cf..4b3dcf374536 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py @@ -810,6 +810,29 @@ stack=False, ) +ALL_RESOURCES_UTILIZATION_PANEL = Panel( + id=57, + title="All logical resources utilization", + description=( + "Shows all logical resources utilization on a single graph. Filtering by operator is recommended." + ), + unit="cores", + targets=[ + Target( + expr='sum(ray_data_cpu_usage_cores{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', + legend="CPU: {{dataset}}, {{operator}}", + ), + Target( + expr='sum(ray_data_gpu_usage_cores{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', + legend="GPU: {{dataset}}, {{operator}}", + ), + ], + fill=0, + stack=False, +) + +OPERATOR_PANELS = [TASK_THROUGHPUT_BY_NODE_PANEL, ALL_RESOURCES_UTILIZATION_PANEL] + DATA_GRAFANA_ROWS = [ # Overview Row Row( @@ -934,6 +957,13 @@ ], collapsed=True, ), + # Operator Panels Row (these graphs should only be viewed when filtering down to a single operator) + Row( + title="Operator Panels", + id=108, + panels=[ALL_RESOURCES_UTILIZATION_PANEL], + collapsed=True, + ), ] # Get all panel IDs from both top-level panels and panels within rows diff --git a/python/ray/data/_internal/metadata_exporter.py b/python/ray/data/_internal/metadata_exporter.py index 3ce64c3aef6f..59f90db33164 100644 --- a/python/ray/data/_internal/metadata_exporter.py +++ b/python/ray/data/_internal/metadata_exporter.py @@ -12,6 +12,13 @@ check_export_api_enabled, get_export_event_logger, ) +from ray.core.generated.export_dataset_metadata_pb2 import ( + ExportDatasetMetadata as ProtoDatasetMetadata, +) +from ray.dashboard.modules.metrics.dashboards.common import Panel +from ray.dashboard.modules.metrics.dashboards.data_dashboard_panels import ( + OPERATOR_PANELS, +) from ray.data._internal.execution.dataset_state import DatasetState from ray.data.context import DataContext @@ -259,12 +266,25 @@ def dataset_metadata_to_proto(dataset_metadata: DatasetMetadata) -> Any: execution_start_time=dataset_metadata.execution_start_time, execution_end_time=dataset_metadata.execution_end_time, state=ProtoDatasetMetadata.DatasetState.Value(dataset_metadata.state), + operator_panels=[_to_proto_dashboard_panel(p) for p in OPERATOR_PANELS], ) proto_dataset_metadata.topology.CopyFrom(proto_topology) return proto_dataset_metadata +def _to_proto_dashboard_panel( + panel: Panel, +) -> ProtoDatasetMetadata.DashboardPanelMetadata: + """Convert Dashboard Panel to protobuf format.""" + proto_panel = ProtoDatasetMetadata.DashboardPanelMetadata( + id=str(panel.id), + title=panel.title, + ) + + return proto_panel + + def get_dataset_metadata_exporter() -> "DatasetMetadataExporter": """Get the dataset metadata exporter instance. diff --git a/src/ray/protobuf/export_dataset_metadata.proto b/src/ray/protobuf/export_dataset_metadata.proto index a508b5c3d186..e9135441f7e1 100644 --- a/src/ray/protobuf/export_dataset_metadata.proto +++ b/src/ray/protobuf/export_dataset_metadata.proto @@ -88,6 +88,13 @@ message ExportDatasetMetadata { PENDING = 4; } + message DashboardPanelMetadata { + // Unique identifier for the panel + string id = 1; + // Display name of the panel + string title = 2; + } + // The operator DAG structure Topology topology = 1; @@ -111,4 +118,8 @@ message ExportDatasetMetadata { // The state of the dataset DatasetState state = 8; + + // List of metric panels to show for operators + // When showing these panels, it is expected to filter the metrics by operator ID. + repeated DashboardPanelMetadata operator_panels = 9; } From a1a2bf257fb2de23e3d0a8df80ed6c389ae7ba90 Mon Sep 17 00:00:00 2001 From: Sven Mika Date: Thu, 11 Sep 2025 21:44:47 +0200 Subject: [PATCH 580/634] [RLlib] Fix Metrics/Stats lifetime count and throughput measurement for async remote actors. (#56047) Fix Metrics/Stats lifetime count and throughput measurement for async remote actors. ## Why are these changes needed? ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [x] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: sven1977 Signed-off-by: simonsays1980 Co-authored-by: simonsays1980 --- rllib/utils/metrics/stats.py | 165 ++++++++++++------------ rllib/utils/metrics/tests/test_stats.py | 21 +-- 2 files changed, 92 insertions(+), 94 deletions(-) diff --git a/rllib/utils/metrics/stats.py b/rllib/utils/metrics/stats.py index d4f77aeee342..5ec957580c34 100644 --- a/rllib/utils/metrics/stats.py +++ b/rllib/utils/metrics/stats.py @@ -1,58 +1,21 @@ from collections import defaultdict, deque -import time import copy -import threading import heapq +import threading +import time from typing import Any, Dict, List, Union, Optional, Tuple +import uuid import numpy as np from ray.rllib.utils import force_list -from ray.rllib.utils.framework import try_import_tf, try_import_torch +from ray.rllib.utils.framework import try_import_torch from ray.rllib.utils.numpy import convert_to_numpy from ray.util.annotations import DeveloperAPI -_, tf, _ = try_import_tf() torch, _ = try_import_torch() -@DeveloperAPI -def compute_percentiles(sorted_list, percentiles): - """Compute percentiles from an already sorted list. - - Note that this will not raise an error if the list is not sorted to avoid overhead. - - Args: - sorted_list: A list of numbers sorted in ascending order - percentiles: A list of percentile values (0-100) - - Returns: - A dictionary mapping percentile values to their corresponding data values - """ - n = len(sorted_list) - - if n == 0: - return {p: None for p in percentiles} - - results = {} - - for p in percentiles: - index = (p / 100) * (n - 1) - - if index.is_integer(): - results[p] = sorted_list[int(index)] - else: - lower_index = int(index) - upper_index = lower_index + 1 - weight = index - lower_index - results[p] = ( - sorted_list[lower_index] * (1 - weight) - + sorted_list[upper_index] * weight - ) - - return results - - @DeveloperAPI class Stats: """A container class holding a number of values and executing reductions over them. @@ -172,9 +135,9 @@ def __init__( "A window must be specified when reduce is 'percentiles'!" ) if reduce_per_index_on_aggregate is not False: - print(reduce_per_index_on_aggregate) raise ValueError( - "`reduce_per_index_on_aggregate` must be `False` when `percentiles` is not `False`!" + f"`reduce_per_index_on_aggregate` ({reduce_per_index_on_aggregate})" + f" must be `False` when `percentiles` is not `False`!" ) if percentiles is True: @@ -234,10 +197,11 @@ def __init__( self._has_returned_zero = False # On each `.reduce()` call, we store the result of this call in - # reduce_history[0] and the previous `reduce()` result in reduce_history[1]. - self._reduce_history: deque[List[Any]] = deque( - [[np.nan], [np.nan], [np.nan]], maxlen=3 - ) + # self._last_reduce. + self._last_reduced = [np.nan] + # The ID of this Stats instance. + self.id_ = str(uuid.uuid4()) + self._prev_merge_values = defaultdict(int) self._throughput_ema_coeff = throughput_ema_coeff self._throughput_stats = None @@ -283,7 +247,7 @@ def check_value(self, value: Any) -> None: if self._reduce_method is not None: if isinstance(value, np.ndarray) and value.shape == (): return - elif (torch and torch.is_tensor(value)) or (tf and tf.is_tensor(value)): + elif torch and torch.is_tensor(value): self._is_tensor = True if tuple(value.shape) == (): return @@ -371,27 +335,13 @@ def peek(self, compile: bool = True) -> Union[Any, List[Any]]: return compute_percentiles(reduced_values, self._percentiles) return reduced_value else: - return_value = self.get_reduce_history()[-1].copy() + return_value = self._last_reduced if compile: # We don't need to check for self._reduce_method or percentiles here # because we only store the reduced value if there is a reduce method. return_value = return_value[0] return return_value - def get_reduce_history(self) -> List[Any]: - """Returns the history of reduced values as a list. - - The history contains the most recent reduced values, with the most recent value - at the end of the list. The length of the history is limited by the maxlen of - the internal history deque. - - Returns: - A list containing the history of reduced values. - """ - # Turning the reduce history into a deque avoids mutating the original reduce - # history's elements. - return list(self._reduce_history) - @property def throughput(self) -> float: """Returns the current throughput estimate per second. @@ -445,7 +395,7 @@ class for details on the reduction logic applied to the values list, based on self._set_values(reduced_internal_values_list) else: reduced_internal_values_list = None - reduced = self.get_reduce_history()[-1] + reduced = self._last_reduced reduced = self._numpy_if_necessary(reduced) @@ -454,7 +404,7 @@ class for details on the reduction logic applied to the values list, based on # It only makes sense to extend the history if we are reducing to a single # value. We need to make a copy here because the new_values_list is a # reference to the internal values list - self._reduce_history.append(force_list(reduced.copy())) + self._last_reduced = force_list(reduced.copy()) else: # If there is a window and no reduce method, we don't want to use the reduce # history to return reduced values in other methods @@ -607,7 +557,7 @@ def merge_in_parallel(self, *others: "Stats") -> None: # Mark that we have new values since we modified the values list self._has_new_values = True - def _clear_throughput(self) -> None: + def clear_throughput(self) -> None: """Clears the throughput Stats, if applicable and `self` has throughput. Also resets `self._last_throughput_measure_time` to -1 such that the Stats @@ -751,7 +701,7 @@ def get_state(self) -> Dict[str, Any]: "window": self._window, "ema_coeff": self._ema_coeff, "clear_on_reduce": self._clear_on_reduce, - "_hist": list(self.get_reduce_history()), + "_last_reduced": self._last_reduced, "_is_tensor": self._is_tensor, } if self._throughput_stats is not None: @@ -810,13 +760,13 @@ def from_state(state: Dict[str, Any]) -> "Stats": # Compatibility to old checkpoints where a reduce sometimes resulted in a single # values instead of a list such that the history would be a list of integers # instead of a list of lists. - # TODO(Artur): Remove this after a few Ray releases. - if not isinstance(state["_hist"][0], list): - state["_hist"] = list(map(lambda x: [x], state["_hist"])) - - stats._reduce_history = deque( - state["_hist"], maxlen=stats._reduce_history.maxlen - ) + if "_hist" in state: + # TODO(Artur): Remove this after a few Ray releases. + if not isinstance(state["_hist"][0], list): + state["_hist"] = list(map(lambda x: [x], state["_hist"])) + stats._last_reduced = state["_hist"][-1] + else: + stats._last_reduced = state.get("_last_reduced", [np.nan]) return stats @staticmethod @@ -851,7 +801,8 @@ def similar_to( else False, throughput_ema_coeff=other._throughput_ema_coeff, ) - stats._reduce_history = other._reduce_history + stats.id_ = other.id_ + stats._last_reduced = other._last_reduced return stats def _set_values(self, new_values): @@ -936,8 +887,6 @@ def _reduced_values(self, values=None) -> Tuple[Any, Any]: def safe_isnan(value): if torch and isinstance(value, torch.Tensor): return torch.isnan(value) - if tf and tf.is_tensor(value): - return tf.math.is_nan(value) return np.isnan(value) # Convert from numpy to primitive python types, if original `values` are @@ -966,6 +915,43 @@ def safe_isnan(value): return [reduced], values +@DeveloperAPI +def compute_percentiles(sorted_list, percentiles): + """Compute percentiles from an already sorted list. + + Note that this will not raise an error if the list is not sorted to avoid overhead. + + Args: + sorted_list: A list of numbers sorted in ascending order + percentiles: A list of percentile values (0-100) + + Returns: + A dictionary mapping percentile values to their corresponding data values + """ + n = len(sorted_list) + + if n == 0: + return {p: None for p in percentiles} + + results = {} + + for p in percentiles: + index = (p / 100) * (n - 1) + + if index.is_integer(): + results[p] = sorted_list[int(index)] + else: + lower_index = int(index) + upper_index = lower_index + 1 + weight = index - lower_index + results[p] = ( + sorted_list[lower_index] * (1 - weight) + + sorted_list[upper_index] * weight + ) + + return results + + @DeveloperAPI def merge_stats(base_stats: Optional[Stats], incoming_stats: List[Stats]) -> Stats: """Merges Stats objects. @@ -991,13 +977,21 @@ def merge_stats(base_stats: Optional[Stats], incoming_stats: List[Stats]) -> Sta if new_root_stats: # We need to deepcopy here first because stats from incoming_stats may be altered in the future base_stats = copy.deepcopy(incoming_stats[0]) - base_stats._clear_throughput() + base_stats.clear_throughput() # Note that we may take a mean of means here, which is not the same as a # mean of all values. In the future, we could implement a weighted mean # of means here by introducing a new Stats object that counts samples # for each mean Stats object. if len(incoming_stats) > 1: base_stats.merge_in_parallel(*incoming_stats[1:]) + if ( + base_stats._reduce_method == "sum" + and base_stats._inf_window + and base_stats._clear_on_reduce is False + ): + for stat in incoming_stats: + base_stats._prev_merge_values[stat.id_] = stat.peek() + elif len(incoming_stats) > 0: # Special case: `base_stats` is a lifetime sum (reduce=sum, # clear_on_reduce=False) -> We subtract the previous value (from 2 @@ -1016,29 +1010,28 @@ def merge_stats(base_stats: Optional[Stats], incoming_stats: List[Stats]) -> Sta for stat in incoming_stats: # Subtract "lifetime counts" from the Stat's values to not count # older "lifetime counts" more than once. - _hist = stat.get_reduce_history() - prev_reduction, new_reduction = _hist[-2][0], _hist[-1][0] - # This may not be populated yet -> use 0 then. - if np.isnan(prev_reduction): - prev_reduction = 0 + prev_reduction = base_stats._prev_merge_values[stat.id_] + new_reduction = stat.peek(compile=True) base_stats.values[-1] -= prev_reduction # Keep track of how many counts we actually gained (for throughput # recomputation). added_sum += new_reduction - prev_reduction + base_stats._prev_merge_values[stat.id_] = new_reduction + parallel_merged_stat = copy.deepcopy(incoming_stats[0]) if len(incoming_stats) > 1: # There are more than one incoming parallel others -> Merge all of # them in parallel (equal importance). - incoming_stats[0].merge_in_parallel(*incoming_stats[1:]) + parallel_merged_stat.merge_in_parallel(*incoming_stats[1:]) # Merge incoming Stats object into base Stats object on time axis # (giving incoming ones priority). if base_stats._reduce_method == "mean" and not base_stats._clear_on_reduce: # If we don't clear values, values that are not cleared would contribute # to the mean multiple times. - base_stats._set_values(incoming_stats[0].values.copy()) + base_stats._set_values(parallel_merged_stat.values.copy()) else: - base_stats.merge_on_time_axis(incoming_stats[0]) + base_stats.merge_on_time_axis(parallel_merged_stat) # Keep track of throughput through the sum of added counts. if base_stats.has_throughput: base_stats._recompute_throughput(added_sum) diff --git a/rllib/utils/metrics/tests/test_stats.py b/rllib/utils/metrics/tests/test_stats.py index e85783a238f1..ec4f36700533 100644 --- a/rllib/utils/metrics/tests/test_stats.py +++ b/rllib/utils/metrics/tests/test_stats.py @@ -1,6 +1,7 @@ import pytest import time import numpy as np +import re from ray.rllib.utils.metrics.stats import Stats, merge_stats from ray.rllib.utils.metrics.metrics_logger import MetricsLogger @@ -343,7 +344,7 @@ def test_similar_to(): # Test that adding to the similar stats does not affect the original stats similar.push(10) check(original.peek(), 3) - check(original.get_reduce_history(), [[np.nan], [np.nan], [3]]) + check(original._last_reduced, [3]) def test_reduce_history(): @@ -359,19 +360,19 @@ def test_reduce_history(): ) # Initially history should contain NaN values - check(stats.get_reduce_history(), [[np.nan], [np.nan], [np.nan]]) + check(stats._last_reduced, [np.nan]) # Push values and reduce stats.push(1) stats.push(2) check(stats.reduce(), 3) - check(stats.get_reduce_history(), [[np.nan], [np.nan], [3]]) + check(stats._last_reduced, [3]) # Push more values and reduce stats.push(3) stats.push(4) check(stats.reduce(), 10) - check(stats.get_reduce_history(), [[np.nan], [3], [10]]) + check(stats._last_reduced, [10]) def test_reduce_history_with_clear(): @@ -390,13 +391,13 @@ def test_reduce_history_with_clear(): stats.push(1) stats.push(2) check(stats.reduce(), 3) - check(stats.get_reduce_history(), [[np.nan], [np.nan], [3]]) + check(stats._last_reduced, [3]) check(len(stats), 0) # Values should be cleared stats.push(3) stats.push(4) check(stats.reduce(), 7) - check(stats.get_reduce_history(), [[np.nan], [3], [7]]) + check(stats._last_reduced, [7]) check(len(stats), 0) @@ -1179,12 +1180,16 @@ def test_percentiles(): # Test validation - percentiles must be None for other reduce methods with pytest.raises( - ValueError, match="`reduce` must be `None` when `percentiles` is not `False" + ValueError, match="`reduce` must be `None` when `percentiles` is not `False`" ): Stats(reduce="mean", window=5, percentiles=[50]) with pytest.raises( - ValueError, match="`reduce_per_index_on_aggregate` must be `False`" + ValueError, + match=re.escape( + "`reduce_per_index_on_aggregate` (True) must be `False` " + "when `percentiles` is not `False`!" + ), ): Stats( reduce=None, reduce_per_index_on_aggregate=True, percentiles=True, window=5 From e88b3f8a57a5fb26cd56a72c820f534a162210f9 Mon Sep 17 00:00:00 2001 From: Timothy Seah Date: Thu, 11 Sep 2025 13:10:58 -0700 Subject: [PATCH 581/634] [train][checkpoint] Add checkpoint_upload_mode to ray.train.report (#55637) Implement async checkpoint uploads in ray.train.report(..., checkpoint_upload_mode), supporting SYNC (default), ASYNC, and NO_UPLOAD. * Introduce per-worker checkpoint counters to preserve report order. * Use a thread pool to limit concurrent uploads and avoid OOM. * Wrap the training function to wait for pending uploads before exiting. * Add delete_local_checkpoint_after_upload to control temporary local directory cleanup. --------- Signed-off-by: Timothy Seah Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python/ray/train/__init__.py | 3 + python/ray/train/v2/BUILD.bazel | 32 +++ .../checkpoint/checkpoint_manager.py | 16 +- .../train/v2/_internal/execution/context.py | 161 ++++++++++--- .../execution/controller/controller.py | 4 +- .../train/v2/_internal/execution/storage.py | 3 +- .../v2/_internal/execution/train_fn_utils.py | 17 +- .../execution/worker_group/thread_runner.py | 7 +- .../execution/worker_group/worker.py | 8 +- python/ray/train/v2/api/report_config.py | 21 ++ python/ray/train/v2/api/train_fn_utils.py | 17 +- .../v2/tests/test_async_checkpointing.py | 223 ++++++++++++++++++ .../train/v2/tests/test_checkpoint_manager.py | 4 +- .../ray/train/v2/tests/test_thread_runner.py | 16 +- python/ray/train/v2/tests/test_worker.py | 76 ++++++ 15 files changed, 553 insertions(+), 55 deletions(-) create mode 100644 python/ray/train/v2/api/report_config.py create mode 100644 python/ray/train/v2/tests/test_async_checkpointing.py create mode 100644 python/ray/train/v2/tests/test_worker.py diff --git a/python/ray/train/__init__.py b/python/ray/train/__init__.py index 5b72413f79bf..a7cd2aad13af 100644 --- a/python/ray/train/__init__.py +++ b/python/ray/train/__init__.py @@ -34,6 +34,7 @@ RunConfig, ScalingConfig, ) + from ray.train.v2.api.report_config import CheckpointUploadMode # noqa: F811 from ray.train.v2.api.reported_checkpoint import ReportedCheckpoint # noqa: F811 from ray.train.v2.api.result import Result # noqa: F811 from ray.train.v2.api.train_fn_utils import ( # noqa: F811 @@ -82,6 +83,8 @@ if is_v2_enabled(): __all__.append("UserCallback") UserCallback.__module__ = "ray.train" + __all__.append("CheckpointUploadMode") + CheckpointUploadMode.__module__ = "ray.train" __all__.append("get_all_reported_checkpoints") get_all_reported_checkpoints.__module__ = "ray.train" __all__.append("ReportedCheckpoint") diff --git a/python/ray/train/v2/BUILD.bazel b/python/ray/train/v2/BUILD.bazel index 7ec0bed9018e..c19c054dec37 100644 --- a/python/ray/train/v2/BUILD.bazel +++ b/python/ray/train/v2/BUILD.bazel @@ -21,6 +21,22 @@ py_test( ], ) +py_test( + name = "test_async_checkpointing", + size = "medium", + srcs = ["tests/test_async_checkpointing.py"], + env = {"RAY_TRAIN_V2_ENABLED": "1"}, + tags = [ + "exclusive", + "team:ml", + "train_v2", + ], + deps = [ + ":conftest", + "//:ray_lib", + ], +) + py_test( name = "test_checkpoint_manager", size = "small", @@ -469,6 +485,22 @@ py_test( ], ) +py_test( + name = "test_worker", + size = "small", + srcs = ["tests/test_worker.py"], + env = {"RAY_TRAIN_V2_ENABLED": "1"}, + tags = [ + "exclusive", + "team:ml", + "train_v2", + ], + deps = [ + ":conftest", + "//:ray_lib", + ], +) + py_test( name = "test_worker_group", size = "medium", diff --git a/python/ray/train/v2/_internal/execution/checkpoint/checkpoint_manager.py b/python/ray/train/v2/_internal/execution/checkpoint/checkpoint_manager.py index bb0ed40e5503..a4a086ca7f84 100644 --- a/python/ray/train/v2/_internal/execution/checkpoint/checkpoint_manager.py +++ b/python/ray/train/v2/_internal/execution/checkpoint/checkpoint_manager.py @@ -15,7 +15,7 @@ WorkerGroupCallback, ) from ray.train.v2._internal.execution.context import StorageContext -from ray.train.v2._internal.execution.storage import _delete_fs_path, _exists_at_fs_path +from ray.train.v2._internal.execution.storage import _exists_at_fs_path, delete_fs_path from ray.train.v2._internal.execution.worker_group import Worker from ray.train.v2.api.reported_checkpoint import ReportedCheckpoint @@ -86,7 +86,7 @@ def __init__( # This tracks the number of report calls that have been processed # for the current worker group. - self._num_report_calls = 0 + self._current_report_index = 0 self._condition = asyncio.Condition() super().__init__(checkpoint_config) @@ -145,9 +145,9 @@ def register_checkpoint(self, checkpoint_result: _TrainingResult): for checkpoint_result in results_to_delete: checkpoint = checkpoint_result.checkpoint logger.debug("Deleting checkpoint: ", checkpoint) - _delete_fs_path(fs=checkpoint.filesystem, fs_path=checkpoint.path) + delete_fs_path(fs=checkpoint.filesystem, fs_path=checkpoint.path) - self._num_report_calls += 1 + self._current_report_index += 1 async def async_notify(): async with self._condition: @@ -283,7 +283,7 @@ def after_report( self, metrics: List[Dict[str, Any]], checkpoint: Optional[Checkpoint] ): if not checkpoint: - self._num_report_calls += 1 + self._current_report_index += 1 return rank_0_metrics = metrics[0] @@ -296,7 +296,7 @@ def after_report( # -------------------------- def before_init_train_context(self, workers: List[Worker]) -> Dict[str, List[Any]]: - self._num_report_calls = 0 + self._current_report_index = 0 latest_checkpoint = ( self.latest_checkpoint_result.checkpoint if self.latest_checkpoint_result @@ -308,12 +308,12 @@ def before_init_train_context(self, workers: List[Worker]) -> Dict[str, List[Any return train_context_args async def get_all_reported_checkpoints( - self, expected_num_report_calls: int + self, current_report_index: int ) -> List[ReportedCheckpoint]: """Once expected_num_checkpoints are reported, return the ReportedCheckpoints.""" async with self._condition: await self._condition.wait_for( - lambda: self._num_report_calls == expected_num_report_calls + lambda: self._current_report_index == current_report_index ) # TODO: might be nice for CheckpointManager to manage ReportedCheckpoint # instead of _TrainingResult but that is a large refactor. diff --git a/python/ray/train/v2/_internal/execution/context.py b/python/ray/train/v2/_internal/execution/context.py index d21678b307b6..b5965d97219e 100644 --- a/python/ray/train/v2/_internal/execution/context.py +++ b/python/ray/train/v2/_internal/execution/context.py @@ -2,6 +2,7 @@ import sys import threading import uuid +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from queue import Queue from typing import TYPE_CHECKING, Any, Dict, List, Optional @@ -12,9 +13,14 @@ from ray.train._internal import session from ray.train._internal.session import _TrainingResult from ray.train.v2._internal.execution.checkpoint.sync_actor import SynchronizationActor -from ray.train.v2._internal.execution.storage import StorageContext -from ray.train.v2._internal.util import _copy_doc, invoke_context_managers +from ray.train.v2._internal.execution.storage import StorageContext, delete_fs_path +from ray.train.v2._internal.util import ( + _copy_doc, + construct_user_exception_with_traceback, + invoke_context_managers, +) from ray.train.v2.api.config import RunConfig, ScalingConfig +from ray.train.v2.api.report_config import CheckpointUploadMode if TYPE_CHECKING: from ray.train import BackendConfig, Checkpoint, DataConfig @@ -30,6 +36,10 @@ logger = logging.getLogger(__file__) +# TODO: make this value manually or automatically configurable. +MAX_CHECKPOINT_UPLOAD_THREADS = 1 + + @dataclass(frozen=True) class TrainRunContext: """Holds the metadata and context for the current training run.""" @@ -101,8 +111,15 @@ class TrainContext: controller_actor: ActorHandle dataset_shard_provider: "DatasetShardProvider" + + # TODO: consolidate into CheckpointContext checkpoint: Optional["Checkpoint"] = None - num_report_calls: int = 0 + current_report_index: int = 0 + report_call_index: int = 0 + report_order_condition: threading.Condition = threading.Condition() + checkpoint_upload_threadpool: ThreadPoolExecutor = ThreadPoolExecutor( + max_workers=MAX_CHECKPOINT_UPLOAD_THREADS + ) @_copy_doc(session.get_experiment_name) def get_experiment_name(self) -> str: @@ -140,12 +157,13 @@ def get_synchronization_actor(self): return self.execution_context.synchronization_actor def get_checkpoint(self): - return self.checkpoint + with self.report_order_condition: + return self.checkpoint def get_all_reported_checkpoints(self) -> List["ReportedCheckpoint"]: return ray.get( self.controller_actor.get_all_reported_checkpoints.remote( - self.num_report_calls + self.current_report_index ) ) @@ -197,11 +215,12 @@ def _sync_checkpoint_dir_name_across_ranks( ) ) - def _save_checkpoint( + def _upload_checkpoint( self, checkpoint_dir_name: str, metrics: Dict[str, Any], checkpoint: Optional["Checkpoint"] = None, + delete_local_checkpoint_after_upload: bool = False, ) -> _TrainingResult: """Save the checkpoint to remote storage. @@ -209,6 +228,7 @@ def _save_checkpoint( checkpoint_dir_name: The checkpoint dir to persist to. metrics: The metrics to report. checkpoint: The checkpoint to report. + delete_local_checkpoint_after_upload: Whether to delete the checkpoint after it is uploaded. Returns: The training result object containing the persisted checkpoint. @@ -218,32 +238,68 @@ def _save_checkpoint( return _TrainingResult(checkpoint=None, metrics=metrics) # Persist the checkpoint to the remote storage path. - persisted_checkpoint = self.storage_context.persist_current_checkpoint( - checkpoint, checkpoint_dir_name - ) - # Update latest checkpoint as the persisted checkpoint. - self.checkpoint = persisted_checkpoint + try: + persisted_checkpoint = self.storage_context.persist_current_checkpoint( + checkpoint, checkpoint_dir_name + ) + except FileNotFoundError: + logger.exception( + f"Failed to find local checkpoint {checkpoint} when attempting to upload it. " + "This could be caused by multiple workers on a node attempting to upload the " + "same directory, and then one of the workers deletes the directory before the " + "others finish." + ) + raise + # TODO: consider deleting local checkpoint as async callback instead + if delete_local_checkpoint_after_upload: + try: + delete_fs_path(checkpoint.filesystem, checkpoint.path) + except Exception: + logger.exception( + f"Failed to delete the local checkpoint after a successful upload: {checkpoint}" + ) return _TrainingResult(checkpoint=persisted_checkpoint, metrics=metrics) + def _wait_then_report( + self, training_result: _TrainingResult, report_call_index: int + ) -> None: + """Thread waits for its turn before reporting training result to result queue. + + It does this in order to guarantee the FIFO processing of checkpoints. + + The queue size is set to 1 to avoid accumulating unprocessed results. + If the queue is full, the put operation blocks until a result is consumed. + + TODO: Add a metric to track the blocking time waiting for the + training result to be consumed by the controller. + """ + with self.report_order_condition: + self.report_order_condition.wait_for( + lambda: self.current_report_index == report_call_index - 1 + ) + logger.info( + f"Reporting training result {report_call_index}: {training_result}" + ) + # Update latest checkpoint as the persisted checkpoint. + if training_result.checkpoint: + self.checkpoint = training_result.checkpoint + self.get_result_queue().put(training_result) + self.current_report_index += 1 + self.report_order_condition.notify_all() + def report( self, metrics: Dict[str, Any], checkpoint: Optional["Checkpoint"] = None, checkpoint_dir_name: Optional[str] = None, + checkpoint_upload_mode: CheckpointUploadMode = CheckpointUploadMode.SYNC, + delete_local_checkpoint_after_upload: Optional[bool] = None, ) -> None: """ Upload checkpoint to remote storage and put a training result on the result queue of this worker process. - Args: - metrics: The metrics to report. - checkpoint: The checkpoint to report. - checkpoint_dir_name: The name of the checkpoint dir - in this iteration. Note: If not set, the checkpoint will - be stored in the default storage path. If set, make sure - this value is unique for each iteration. - TODO: the report function should be implemented in the worker instead of in the train context. The train context should only keep the train related information and not the worker related actions. This refactor @@ -267,22 +323,65 @@ def report( for callback in self.execution_context.train_context_callbacks ] ): - # Step 1: sync the checkpoint dir name across ranks. + self.report_call_index += 1 + report_call_index = self.report_call_index + + # Sync the checkpoint dir name across ranks. checkpoint_dir_name = self._sync_checkpoint_dir_name_across_ranks( checkpoint_dir_name ) - # Step 2: save the checkpoint to remote storage. - training_result = self._save_checkpoint( - checkpoint_dir_name, metrics, checkpoint - ) - # Step 3: Report the training result to the result queue. - # The queue size is set to 1 to avoid accumulating unprocessed results. - # If the queue is full, the put operation blocks until a result is consumed. - # TODO (hpguo): Add a metrics to track the blocking time waiting for the - # training result to be consumed by the controller. - self.get_result_queue().put(training_result) - self.num_report_calls += 1 + # Upload checkpoint, wait for turn, and report. + if checkpoint_upload_mode == CheckpointUploadMode.SYNC: + training_result = self._upload_checkpoint( + checkpoint_dir_name, + metrics, + checkpoint, + delete_local_checkpoint_after_upload, + ) + self._wait_then_report(training_result, report_call_index) + + elif checkpoint_upload_mode == CheckpointUploadMode.NO_UPLOAD: + training_result = _TrainingResult( + checkpoint=checkpoint, metrics=metrics + ) + self._wait_then_report(training_result, report_call_index) + + elif checkpoint_upload_mode == CheckpointUploadMode.ASYNC: + + def _upload_checkpoint_and_report( + checkpoint_dir_name: str, + metrics: Dict[str, Any], + checkpoint: Optional["Checkpoint"], + report_call_index: int, + ) -> None: + try: + training_result = self._upload_checkpoint( + checkpoint_dir_name, + metrics, + checkpoint, + delete_local_checkpoint_after_upload, + ) + self._wait_then_report(training_result, report_call_index) + except Exception as e: + logger.exception( + "Async checkpoint upload failed - shutting down workers" + ) + self.execution_context.training_thread_runner.get_exception_queue().put( + construct_user_exception_with_traceback(e) + ) + + self.checkpoint_upload_threadpool.submit( + _upload_checkpoint_and_report, + checkpoint_dir_name, + metrics, + checkpoint, + report_call_index, + ) + else: + raise ValueError( + f"Invalid checkpoint upload mode: {checkpoint_upload_mode}" + ) # The global variable holding the current TrainContext diff --git a/python/ray/train/v2/_internal/execution/controller/controller.py b/python/ray/train/v2/_internal/execution/controller/controller.py index 53074db40858..3d84796c4340 100644 --- a/python/ray/train/v2/_internal/execution/controller/controller.py +++ b/python/ray/train/v2/_internal/execution/controller/controller.py @@ -557,8 +557,8 @@ def get_training_failed_error(self) -> Optional[TrainingFailedError]: return None async def get_all_reported_checkpoints( - self, expected_num_report_calls: int + self, current_report_index: int ) -> List["ReportedCheckpoint"]: return await self._checkpoint_manager.get_all_reported_checkpoints( - expected_num_report_calls + current_report_index ) diff --git a/python/ray/train/v2/_internal/execution/storage.py b/python/ray/train/v2/_internal/execution/storage.py index 4ffc740c50af..abf80697da36 100644 --- a/python/ray/train/v2/_internal/execution/storage.py +++ b/python/ray/train/v2/_internal/execution/storage.py @@ -120,7 +120,8 @@ def _pyarrow_fs_copy_files( # TODO(justinvyu): Add unit tests for all these utils. -def _delete_fs_path(fs: pyarrow.fs.FileSystem, fs_path: str): +def delete_fs_path(fs: pyarrow.fs.FileSystem, fs_path: str): + """Deletes (fs, fs_path) or raises FileNotFoundError if it doesn't exist.""" is_dir = _is_directory(fs, fs_path) try: diff --git a/python/ray/train/v2/_internal/execution/train_fn_utils.py b/python/ray/train/v2/_internal/execution/train_fn_utils.py index b05f7dae0631..932bc69dd973 100644 --- a/python/ray/train/v2/_internal/execution/train_fn_utils.py +++ b/python/ray/train/v2/_internal/execution/train_fn_utils.py @@ -14,6 +14,7 @@ LocalTrainContext, TrainContext as ExternalTrainContext, ) +from ray.train.v2.api.report_config import CheckpointUploadMode logger = logging.getLogger(__name__) @@ -36,6 +37,8 @@ def report( metrics: Dict[str, Any], checkpoint: Optional["Checkpoint"] = None, checkpoint_dir_name: Optional[str] = None, + checkpoint_upload_mode: CheckpointUploadMode = CheckpointUploadMode.SYNC, + delete_local_checkpoint_after_upload: Optional[bool] = None, ) -> None: """Upload checkpoint to remote storage and put a training result on the result queue. @@ -46,6 +49,10 @@ def report( in this iteration. Note: If not set, the checkpoint will be stored in the default storage path. If set, make sure this value is unique for each iteration. + checkpoint_upload_mode: The manner in which we want to upload the checkpoint. + Defaults to uploading the checkpoint synchronously. + This works when no checkpoint is provided but is not useful in that case. + delete_local_checkpoint_after_upload: Whether to delete the checkpoint after it is uploaded. """ pass @@ -121,9 +128,15 @@ def report( metrics: Dict[str, Any], checkpoint: Optional["Checkpoint"] = None, checkpoint_dir_name: Optional[str] = None, + checkpoint_upload_mode: CheckpointUploadMode = CheckpointUploadMode.SYNC, + delete_local_checkpoint_after_upload: Optional[bool] = None, ) -> None: return get_internal_train_context().report( - metrics, checkpoint, checkpoint_dir_name + metrics, + checkpoint, + checkpoint_dir_name, + checkpoint_upload_mode, + delete_local_checkpoint_after_upload, ) def get_checkpoint(self): @@ -166,6 +179,8 @@ def report( metrics: Dict[str, Any], checkpoint: Optional["Checkpoint"] = None, checkpoint_dir_name: Optional[str] = None, + checkpoint_upload_mode: CheckpointUploadMode = CheckpointUploadMode.SYNC, + delete_local_checkpoint_after_upload: Optional[bool] = None, ) -> None: self._last_metrics = metrics self._last_checkpoint = checkpoint diff --git a/python/ray/train/v2/_internal/execution/worker_group/thread_runner.py b/python/ray/train/v2/_internal/execution/worker_group/thread_runner.py index 460b2f59c18a..df0e4cb28e65 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/thread_runner.py +++ b/python/ray/train/v2/_internal/execution/worker_group/thread_runner.py @@ -44,10 +44,11 @@ def _run_target(): self._ret = result self._exc_queue.put(None) except BaseException as e: - # Exclude the first 2 frames from the traceback, which are - # the `ThreadRunner._run_target` and `construct_train_func` calls. + # Exclude the first 3 frames from the traceback, which are + # the `ThreadRunner._run_target`, `construct_train_func`, and + # train_fn_with_final_checkpoint_flush calls. self._exc_queue.put( - construct_user_exception_with_traceback(e, exclude_frames=2) + construct_user_exception_with_traceback(e, exclude_frames=3) ) with self._lock: diff --git a/python/ray/train/v2/_internal/execution/worker_group/worker.py b/python/ray/train/v2/_internal/execution/worker_group/worker.py index 2735a3991bbd..c890ad5c8a88 100644 --- a/python/ray/train/v2/_internal/execution/worker_group/worker.py +++ b/python/ray/train/v2/_internal/execution/worker_group/worker.py @@ -138,8 +138,14 @@ def run_train_fn(self, train_fn_ref: ObjectRefWrapper[Callable[[], None]]): logger.error(f"Error deserializing the training function: {e}") raise + def train_fn_with_final_checkpoint_flush(): + train_fn() + get_train_context().checkpoint_upload_threadpool.shutdown() + # Create and start the training thread. - get_train_context().execution_context.training_thread_runner.run(train_fn) + get_train_context().execution_context.training_thread_runner.run( + train_fn_with_final_checkpoint_flush + ) def get_metadata(self) -> ActorMetadata: return ActorMetadata( diff --git a/python/ray/train/v2/api/report_config.py b/python/ray/train/v2/api/report_config.py new file mode 100644 index 000000000000..bcd4393da287 --- /dev/null +++ b/python/ray/train/v2/api/report_config.py @@ -0,0 +1,21 @@ +from enum import Enum + +from ray.util.annotations import PublicAPI + + +@PublicAPI(stability="alpha") +class CheckpointUploadMode(Enum): + """The manner in which we want to upload the checkpoint. + + Args: + ASYNC: Upload checkpoint asynchronously. + SYNC: Upload checkpoint synchronously. + NO_UPLOAD: Do not upload checkpoint. + """ + + ASYNC = "ASYNC" + SYNC = "SYNC" + NO_UPLOAD = "NO_UPLOAD" + + def _default_delete_local_checkpoint_after_upload(self) -> bool: + return self == CheckpointUploadMode.ASYNC diff --git a/python/ray/train/v2/api/train_fn_utils.py b/python/ray/train/v2/api/train_fn_utils.py index 639867e29257..eec8274367c2 100644 --- a/python/ray/train/v2/api/train_fn_utils.py +++ b/python/ray/train/v2/api/train_fn_utils.py @@ -3,6 +3,7 @@ from ray.train.v2._internal.data_integration.interfaces import DatasetShardMetadata from ray.train.v2._internal.execution.train_fn_utils import get_train_fn_utils from ray.train.v2.api.context import TrainContext +from ray.train.v2.api.report_config import CheckpointUploadMode from ray.util.annotations import PublicAPI if TYPE_CHECKING: @@ -16,6 +17,8 @@ def report( metrics: Dict[str, Any], checkpoint: Optional["Checkpoint"] = None, checkpoint_dir_name: Optional[str] = None, + checkpoint_upload_mode: CheckpointUploadMode = CheckpointUploadMode.SYNC, + delete_local_checkpoint_after_upload: Optional[bool] = None, ): """Report metrics and optionally save a checkpoint. @@ -88,10 +91,22 @@ def train_func(config): If provided, it must be unique across all checkpoints per worker to avoid naming collisions. Consider including identifiers such as the epoch or batch index in the name. + checkpoint_upload_mode: The manner in which we want to upload the checkpoint. + Defaults to uploading the checkpoint synchronously. + This works when no checkpoint is provided but is not useful in that case. + delete_local_checkpoint_after_upload: Whether to delete the checkpoint after it is uploaded. """ + if delete_local_checkpoint_after_upload is None: + delete_local_checkpoint_after_upload = ( + checkpoint_upload_mode._default_delete_local_checkpoint_after_upload() + ) get_train_fn_utils().report( - metrics=metrics, checkpoint=checkpoint, checkpoint_dir_name=checkpoint_dir_name + metrics=metrics, + checkpoint=checkpoint, + checkpoint_dir_name=checkpoint_dir_name, + checkpoint_upload_mode=checkpoint_upload_mode, + delete_local_checkpoint_after_upload=delete_local_checkpoint_after_upload, ) diff --git a/python/ray/train/v2/tests/test_async_checkpointing.py b/python/ray/train/v2/tests/test_async_checkpointing.py new file mode 100644 index 000000000000..6e2251e8510c --- /dev/null +++ b/python/ray/train/v2/tests/test_async_checkpointing.py @@ -0,0 +1,223 @@ +import os +from unittest.mock import create_autospec + +import pytest + +import ray +import ray.cloudpickle as ray_pickle +from ray.train import Checkpoint, RunConfig, ScalingConfig +from ray.train.v2.api.data_parallel_trainer import DataParallelTrainer +from ray.train.v2.api.exceptions import WorkerGroupError +from ray.train.v2.api.report_config import CheckpointUploadMode + + +def test_report_mixed_checkpoint_upload_modes(ray_start_4_cpus, tmp_path): + """Run all 10 possible pairs (e.g. (SYNC, ASYNC)) of checkpoint upload modes between 2 workers.""" + + def get_checkpoint_iteration(checkpoint): + if not checkpoint: + return -1 + return int(checkpoint.path.split("_")[-1]) + + def train_fn(): + # When reporting with async checkpointing, write the checkpoint to + # tmp_path, which stays alive for the duration of the test, instead of + # tempfile.TemporaryDirectory(), which might get deleted before the + # async checkpoint upload completes. + + # Run all 10 possible pairs of checkpoint upload modes + rank = ray.train.get_context().get_world_rank() + if rank == 0: + ASYNC_ITERATIONS = [0, 1, 2, 3] + SYNC_ITERATIONS = [4, 5, 6] + NO_UPLOAD_ITERATIONS = [7, 8] + NO_CHECKPOINT_ITERATIONS = [9] + else: + ASYNC_ITERATIONS = [0] + SYNC_ITERATIONS = [1, 4] + NO_UPLOAD_ITERATIONS = [2, 5, 7] + NO_CHECKPOINT_ITERATIONS = [3, 6, 8, 9] + + prev_latest_checkpoint_iteration = -1 + for i in range(10): + # Set variables + if i in ASYNC_ITERATIONS: + checkpoint_upload_mode = CheckpointUploadMode.ASYNC + elif i in SYNC_ITERATIONS: + checkpoint_upload_mode = CheckpointUploadMode.SYNC + else: + checkpoint_upload_mode = CheckpointUploadMode.NO_UPLOAD + metrics = {"metric": f"iteration_{i}_shard_{rank}"} + + # Create and report checkpoint + if i in NO_CHECKPOINT_ITERATIONS: + ray.train.report( + metrics=metrics, + checkpoint=None, + ) + assert prev_latest_checkpoint_iteration <= get_checkpoint_iteration( + ray.train.get_checkpoint() + ) + else: + # Create remote or local checkpoint_dir + checkpoint_dir_name = f"checkpoint_iteration_{i}" + if i in NO_UPLOAD_ITERATIONS: + checkpoint_dir = ( + ray.train.get_context() + .get_storage() + .build_checkpoint_path_from_name(checkpoint_dir_name) + ) + else: + checkpoint_dir = os.path.join( + tmp_path, checkpoint_dir_name, f"_{rank}" + ) + + # Create and report that remote or local checkpoint + os.makedirs(checkpoint_dir, exist_ok=True) + with open(os.path.join(checkpoint_dir, f"shard_{rank}"), "wb") as f: + ray_pickle.dump(f"iteration_{i}_shard_{rank}", f) + checkpoint = Checkpoint(checkpoint_dir) + ray.train.report( + metrics=metrics, + checkpoint=checkpoint, + checkpoint_upload_mode=checkpoint_upload_mode, + checkpoint_dir_name=checkpoint_dir_name, + ) + + # Check the status of latest_checkpoint + latest_checkpoint = ray.train.get_checkpoint() + if i in NO_UPLOAD_ITERATIONS: + assert latest_checkpoint == checkpoint + elif i in SYNC_ITERATIONS: + assert checkpoint_dir_name in latest_checkpoint.path + else: + assert prev_latest_checkpoint_iteration <= get_checkpoint_iteration( + latest_checkpoint + ) + + prev_latest_checkpoint_iteration = get_checkpoint_iteration( + latest_checkpoint + ) + + trainer = DataParallelTrainer( + train_fn, + scaling_config=ScalingConfig(num_workers=2), + run_config=RunConfig(storage_path=str(tmp_path)), + ) + result = trainer.fit() + # Note that the (checkpoint=None, checkpoint=None) pair does not produce any checkpoint + assert len(result.best_checkpoints) == 9 + for i, (checkpoint, metrics) in enumerate(result.best_checkpoints): + assert checkpoint.path.endswith(f"checkpoint_iteration_{i}") + assert metrics["metric"] == f"iteration_{i}_shard_0" + + +@pytest.mark.parametrize( + "delete_local_checkpoint_after_upload,checkpoint_upload_mode", + [ + (True, CheckpointUploadMode.ASYNC), + (False, CheckpointUploadMode.ASYNC), + (True, CheckpointUploadMode.SYNC), + (False, CheckpointUploadMode.SYNC), + (True, CheckpointUploadMode.NO_UPLOAD), + (False, CheckpointUploadMode.NO_UPLOAD), + ], +) +def test_report_delete_local_checkpoint_after_upload( + ray_start_4_cpus, + tmp_path, + delete_local_checkpoint_after_upload, + checkpoint_upload_mode, +): + """Check that the local checkpoint is deleted after upload.""" + + def train_fn(): + rank = ray.train.get_context().get_world_rank() + if rank == 0: + if checkpoint_upload_mode == CheckpointUploadMode.NO_UPLOAD: + checkpoint_dir = ( + ray.train.get_context() + .get_storage() + .build_checkpoint_path_from_name("my_checkpoint_dir") + ) + else: + checkpoint_dir = os.path.join( + tmp_path, + "my_checkpoint_dir", + ) + os.makedirs(checkpoint_dir, exist_ok=True) + with open(os.path.join(checkpoint_dir, "shard_0"), "wb") as f: + ray_pickle.dump("some_checkpoint_contents", f) + checkpoint = Checkpoint(checkpoint_dir) + ray.train.report( + {}, + checkpoint, + checkpoint_upload_mode=checkpoint_upload_mode, + delete_local_checkpoint_after_upload=delete_local_checkpoint_after_upload, + ) + else: + ray.train.report( + {}, + None, + ) + + trainer = DataParallelTrainer( + train_fn, + scaling_config=ScalingConfig(num_workers=2), + run_config=RunConfig(storage_path=str(tmp_path)), + ) + trainer.fit() + if ( + delete_local_checkpoint_after_upload + or checkpoint_upload_mode == CheckpointUploadMode.NO_UPLOAD + ): + assert not os.path.exists(os.path.join(tmp_path, "my_checkpoint_dir")) + else: + assert os.path.exists(os.path.join(tmp_path, "my_checkpoint_dir")) + + +def test_report_checkpoint_upload_error(ray_start_4_cpus, monkeypatch, tmp_path): + """Check that the trainer shuts down when an error occurs during checkpoint upload.""" + + def train_fn(): + + if ray.train.get_context().get_world_rank() == 0: + + # Mock persist_current_checkpoint to raise an error + mock_persist_current_checkpoint = create_autospec( + ray.train.get_context().get_storage().persist_current_checkpoint + ) + mock_persist_current_checkpoint.side_effect = ValueError("error") + monkeypatch.setattr( + ray.train.get_context().get_storage(), + "persist_current_checkpoint", + mock_persist_current_checkpoint, + ) + + # Report minimal valid checkpoint + local_checkpoint_dir = os.path.join(tmp_path, "local_checkpoint_dir") + os.makedirs(local_checkpoint_dir, exist_ok=True) + ray.train.report( + {}, + Checkpoint.from_directory(local_checkpoint_dir), + checkpoint_upload_mode=CheckpointUploadMode.ASYNC, + ) + else: + ray.train.report( + {}, None, checkpoint_upload_mode=CheckpointUploadMode.ASYNC + ) + + trainer = DataParallelTrainer( + train_fn, + scaling_config=ScalingConfig(num_workers=2), + run_config=RunConfig(storage_path=str(tmp_path)), + ) + with pytest.raises(WorkerGroupError) as exc_info: + trainer.fit() + assert isinstance(exc_info.value.worker_failures[0], ValueError) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", "-x", __file__])) diff --git a/python/ray/train/v2/tests/test_checkpoint_manager.py b/python/ray/train/v2/tests/test_checkpoint_manager.py index 3debda020e7e..2d9304a112b6 100644 --- a/python/ray/train/v2/tests/test_checkpoint_manager.py +++ b/python/ray/train/v2/tests/test_checkpoint_manager.py @@ -97,7 +97,7 @@ async def test_save_load_state_equivalence( # Mock the delete function as we don't want report checkpoints to be deleted. monkeypatch.setattr( ray.train.v2._internal.execution.checkpoint.checkpoint_manager, - "_delete_fs_path", + "delete_fs_path", lambda *args, **kwargs: None, ) exp_name = f"checkpoint_manager_test-{uuid.uuid4().hex}" @@ -117,7 +117,7 @@ async def test_save_load_state_equivalence( # Register the training results into checkpoint manager for i, tr in enumerate(training_results): checkpoint_manager.register_checkpoint(tr) - assert checkpoint_manager._num_report_calls == i + 1 + assert checkpoint_manager._current_report_index == i + 1 loaded_checkpoint_manager = CheckpointManager( storage_context=storage_context, checkpoint_config=checkpoint_config, diff --git a/python/ray/train/v2/tests/test_thread_runner.py b/python/ray/train/v2/tests/test_thread_runner.py index 9d0c7f3d730b..2a5038e2b7d8 100644 --- a/python/ray/train/v2/tests/test_thread_runner.py +++ b/python/ray/train/v2/tests/test_thread_runner.py @@ -46,13 +46,16 @@ def target(): def test_error(thread_runner): """Checks that an exception can be captured from the target function.""" - def target(): - def nested(): - raise ValueError + def wrapped_train_func(): + def train_fn_with_final_checkpoint_flush(): + def train_func(): + raise ValueError - nested() + train_func() - thread_runner.run(target) + train_fn_with_final_checkpoint_flush() + + thread_runner.run(wrapped_train_func) assert not thread_runner.join() assert thread_runner.get_return_value() is None @@ -64,6 +67,9 @@ def nested(): assert isinstance(error._base_exc, ValueError) print(error._traceback_str) assert "_run_target" not in error._traceback_str + assert "wrapped_train_func" not in error._traceback_str + assert "train_fn_with_final_checkpoint_flush" not in error._traceback_str + assert "train_func" in error._traceback_str def test_nested_thread_error(thread_runner): diff --git a/python/ray/train/v2/tests/test_worker.py b/python/ray/train/v2/tests/test_worker.py new file mode 100644 index 000000000000..75fdf07c3ea0 --- /dev/null +++ b/python/ray/train/v2/tests/test_worker.py @@ -0,0 +1,76 @@ +import queue +import time +from unittest.mock import create_autospec + +import pytest + +from ray.actor import ActorHandle +from ray.train.v2._internal.constants import ENABLE_WORKER_STRUCTURED_LOGGING_ENV_VAR +from ray.train.v2._internal.execution.context import ( + DistributedContext, + TrainRunContext, + get_train_context, +) +from ray.train.v2._internal.execution.storage import StorageContext +from ray.train.v2._internal.execution.worker_group.worker import RayTrainWorker +from ray.train.v2._internal.util import ObjectRefWrapper + + +@pytest.mark.parametrize("created_nested_threads", [True, False]) +def test_worker_finished_after_all_threads_finish(monkeypatch, created_nested_threads): + # Disable this to avoid TypeError from logging MagicMock + monkeypatch.setenv(ENABLE_WORKER_STRUCTURED_LOGGING_ENV_VAR, False) + + # Initialize RayTrainWorker state + worker = RayTrainWorker() + worker.init_train_context( + train_run_context=create_autospec(TrainRunContext, instance=True), + distributed_context=DistributedContext( + world_rank=0, + world_size=1, + local_rank=0, + local_world_size=1, + node_rank=0, + ), + synchronization_actor=create_autospec(ActorHandle, instance=True), + storage_context=create_autospec(StorageContext, instance=True), + worker_callbacks=[], + controller_actor=create_autospec(ActorHandle, instance=True), + ) + global_queue = queue.Queue() + + def train_fn(): + tc = get_train_context() + + def target(): + # Intentionally sleep longer than poll interval to test that we wait + # for nested threads to finish + time.sleep(0.1) + global_queue.put("nested") + + if created_nested_threads: + tc.checkpoint_upload_threadpool.submit(target) + else: + global_queue.put("main") + + # Run train fn and wait for it to finish + train_fn_ref = create_autospec(ObjectRefWrapper, instance=True) + train_fn_ref.get.return_value = train_fn + worker.run_train_fn(train_fn_ref) + while worker.poll_status().running: + time.sleep(0.01) + + # Verify queue contents + queue_contents = [] + while not global_queue.empty(): + queue_contents.append(global_queue.get()) + if created_nested_threads: + assert queue_contents == ["nested"] + else: + assert queue_contents == ["main"] + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main(["-v", "-x", __file__])) From d92dcf6ece879d0b09e8682c85067a653c96435c Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Thu, 11 Sep 2025 13:11:59 -0700 Subject: [PATCH 582/634] Support ray.put() and ray.get() with nixl in gpu objects (#56146) Support ray.put() and ray.get() with nixl in gpu objects. Example ```python @ray.remote(num_gpus=1, num_cpus=0, enable_tensor_transport=True) class GPUTestActor: def produce(self, tensors): refs = [] for t in tensors: refs.append(ray.put(t, tensor_transport="nixl")) return refs def consume(self, refs): tensors = [ray.get(ref) for ref in refs] sum = 0 for t in tensors: assert t.device.type == "cuda" sum += t.sum().item() return sum actors = [GPUTestActor.remote() for _ in range(2)] src_actor, dst_actor = actors[0], actors[1] tensor1 = torch.tensor([1, 2, 3]).to("cuda") tensor2 = torch.tensor([4, 5, 6, 0]).to("cuda") tensor3 = torch.tensor([7, 8, 9, 0, 0]).to("cuda") tensors = [tensor1, tensor2, tensor3] ref = src_actor.produce.remote(tensors) ref1 = dst_actor.consume.remote(ref) result = ray.get(ref1) assert result == 45 time.sleep(5) ``` --- ci/lint/pydoclint-baseline.txt | 4 + python/ray/_private/serialization.py | 102 ++++++++++++---- python/ray/_private/worker.py | 94 ++++++++++++-- python/ray/_raylet.pyx | 15 ++- .../collective/collective_tensor_transport.py | 40 +++--- .../collective/nixl_tensor_transport.py | 64 +++++----- .../collective/tensor_transport_manager.py | 18 ++- .../gpu_object_manager/gpu_object_manager.py | 115 ++++++++++++++++-- python/ray/includes/libcoreworker.pxd | 3 +- .../gpu_objects/test_gpu_objects_gloo.py | 30 ----- .../gpu_objects/test_gpu_objects_nixl.py | 73 ++++++++++- .../collective_group/nixl_backend.py | 8 +- src/ray/core_worker/core_worker.cc | 12 +- src/ray/core_worker/core_worker.h | 4 +- 14 files changed, 440 insertions(+), 142 deletions(-) diff --git a/ci/lint/pydoclint-baseline.txt b/ci/lint/pydoclint-baseline.txt index 4455dd16b381..602d58a3e274 100644 --- a/ci/lint/pydoclint-baseline.txt +++ b/ci/lint/pydoclint-baseline.txt @@ -2815,3 +2815,7 @@ python/ray/widgets/util.py DOC103: Function `_has_missing`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [*deps: Iterable[Union[str, Optional[str]]]]. Arguments in the docstring but not in the function signature: [deps: ]. DOC103: Function `repr_with_fallback`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [*notebook_deps: Iterable[Union[str, Optional[str]]]]. Arguments in the docstring but not in the function signature: [notebook_deps: ]. -------------------- +python/ray/_private/serialization.py + DOC106: Function `_gpu_object_ref_deserializer`: The option `--arg-type-hints-in-signature` is `True` but there are no argument type hints in the signature + DOC107: Function `_gpu_object_ref_deserializer`: The option `--arg-type-hints-in-signature` is `True` but not all args in the signature have type hints +-------------------- diff --git a/python/ray/_private/serialization.py b/python/ray/_private/serialization.py index 5ae15b5a8442..e2729ac38ab8 100644 --- a/python/ray/_private/serialization.py +++ b/python/ray/_private/serialization.py @@ -82,7 +82,9 @@ def pickle_dumps(obj: Any, error_msg: str): raise ray.exceptions.OufOfBandObjectRefSerializationException(msg) -def _object_ref_deserializer(binary, call_site, owner_address, object_status): +def _object_ref_deserializer( + binary, call_site, owner_address, object_status, tensor_transport_val +): # NOTE(suquark): This function should be a global function so # cloudpickle can access it directly. Otherwise cloudpickle # has to dump the whole function definition, which is inefficient. @@ -91,7 +93,9 @@ def _object_ref_deserializer(binary, call_site, owner_address, object_status): # the core worker to resolve the value. This is to make sure # that the ref count for the ObjectRef is greater than 0 by the # time the core worker resolves the value of the object. - obj_ref = ray.ObjectRef(binary, owner_address, call_site) + obj_ref = ray.ObjectRef( + binary, owner_address, call_site, tensor_transport_val=tensor_transport_val + ) # TODO(edoakes): we should be able to just capture a reference # to 'self' here instead, but this function is itself pickled @@ -111,6 +115,40 @@ def _object_ref_deserializer(binary, call_site, owner_address, object_status): return obj_ref +def _gpu_object_ref_deserializer( + binary, + call_site, + owner_address, + object_status, + tensor_transport_val, + gpu_object_meta, +): + """ + Deserialize a GPU object ref. When the GPU object ref is deserialized, + it firstly deserialize the normal object ref, and then add metadata of + the GPU object to the GPU object manager, which will be used to fetch + the GPU object later. + + Args: + binary: The binary data of the object ref. + call_site: The call site of the object ref. + owner_address: The owner address of the object ref. + object_status: The object status of the object ref. + tensor_transport_val: The tensor transport value of the GPU object ref. + gpu_object_meta: The GPU object metadata. This is used to fetch the GPU object later. + + Returns: + The deserialized GPU object ref. + """ + obj_ref = _object_ref_deserializer( + binary, call_site, owner_address, object_status, tensor_transport_val + ) + gpu_object_manager = ray._private.worker.global_worker.gpu_object_manager + gpu_object_manager.add_gpu_object_metadata(obj_ref, gpu_object_meta) + + return obj_ref + + def _actor_handle_deserializer(serialized_obj, weak_ref): # If this actor handle was stored in another object, then tell the # core worker. @@ -162,16 +200,6 @@ def object_ref_reducer(obj): worker = ray._private.worker.global_worker worker.check_connected() - # Check if this is a GPU ObjectRef being serialized inside a collection - if ( - self.is_in_band_serialization() - and worker.gpu_object_manager.is_managed_object(obj.hex()) - ): - raise ValueError( - "Passing RDT ObjectRefs inside data structures is not yet supported. " - "Pass RDT ObjectRefs directly as task arguments instead. For example, use `foo.remote(ref)` instead of `foo.remote([ref])`." - ) - self.add_contained_object_ref( obj, allow_out_of_band_serialization=( @@ -179,14 +207,35 @@ def object_ref_reducer(obj): ), call_site=obj.call_site(), ) + obj, owner_address, object_status = worker.core_worker.serialize_object_ref( obj ) + # Check if this is a GPU ObjectRef being serialized inside a collection + if ( + self.is_in_band_serialization() + and worker.gpu_object_manager.is_managed_object(obj.hex()) + ): + + gpu_object_manager = ( + ray._private.worker.global_worker.gpu_object_manager + ) + gpu_object_meta = gpu_object_manager._get_gpu_object_metadata(obj) + return _gpu_object_ref_deserializer, ( + obj.binary(), + obj.call_site(), + owner_address, + object_status, + obj.tensor_transport(), + gpu_object_meta, + ) + return _object_ref_deserializer, ( obj.binary(), obj.call_site(), owner_address, object_status, + obj.tensor_transport(), ) self._register_cloudpickle_reducer(ray.ObjectRef, object_ref_reducer) @@ -623,24 +672,19 @@ def _python_serializer(o): metadata, msgpack_data, contained_object_refs, pickle5_serialized_object ) - def serialize_and_store_gpu_objects( + def serialize_gpu_objects( self, value: Any, - obj_id: bytes, - ) -> MessagePackSerializedObject: + ) -> Tuple[MessagePackSerializedObject, List["torch.Tensor"]]: """Retrieve GPU data from `value` and store it in the GPU object store. Then, return the serialized value. Args: value: The value to serialize. - obj_id: The object ID of the value. `obj_id` is required, and the GPU data (e.g. tensors) in `value` - will be stored in the GPU object store with the key `obj_id`. Returns: Serialized value. """ - assert ( - obj_id is not None - ), "`obj_id` is required, and it is the key to retrieve corresponding tensors from the GPU object store." + if not self._torch_custom_serializer_registered: # Register a custom serializer for torch.Tensor. If the method is # decorated with `@ray.method(tensor_transport="xxx")`, it will @@ -653,16 +697,28 @@ def serialize_and_store_gpu_objects( self._torch_custom_serializer_registered = True serialized_val, tensors = self._serialize_and_retrieve_tensors(value) + + return serialized_val, tensors + + def store_gpu_objects(self, obj_id: str, tensors: List["torch.Tensor"]): + """ + Store GPU objects in the GPU object store. + + Args: + obj_id: The object ID of the value. `obj_id` is required, and the GPU data (e.g. tensors) in `value` + will be stored in the GPU object store with the key `obj_id`. + tensors: The tensors to store in the GPU object store. + """ + assert ( + obj_id is not None + ), "`obj_id` is required, and it is the key to retrieve corresponding tensors from the GPU object store." # Regardless of whether `tensors` is empty, we always store the GPU object # in the GPU object store. This ensures that `_get_tensor_meta` is not # blocked indefinitely. - obj_id = obj_id.decode("ascii") worker = ray._private.worker.global_worker gpu_object_manager = worker.gpu_object_manager gpu_object_manager.gpu_object_store.add_object(obj_id, tensors, is_primary=True) - return serialized_val - def serialize( self, value: Any ) -> Union[RawSerializedObject, MessagePackSerializedObject]: diff --git a/python/ray/_private/worker.py b/python/ray/_private/worker.py index ebb15ee817e1..6657a9246033 100644 --- a/python/ray/_private/worker.py +++ b/python/ray/_private/worker.py @@ -797,6 +797,7 @@ def put_object( value: Any, owner_address: Optional[str] = None, _is_experimental_channel: bool = False, + _tensor_transport: str = "object_store", ): """Put value in the local object store. @@ -813,7 +814,7 @@ def put_object( objects. If True, then the returned object will not have a valid value. The object must be written to using the ray.experimental.channel API before readers can read. - + _tensor_transport: [Alpha] The tensor transport backend to use. Currently, this supports "object_store" and "nixl". Returns: ObjectRef: The object ref the object was put under. @@ -829,9 +830,25 @@ def put_object( "If you really want to do this, you can wrap the " "ray.ObjectRef in a list and call 'put' on it." ) - + tensors = None + tensor_transport: TensorTransportEnum = TensorTransportEnum.from_str( + _tensor_transport + ) + if tensor_transport not in [ + TensorTransportEnum.OBJECT_STORE, + TensorTransportEnum.NIXL, + ]: + raise ValueError( + "Currently, Ray Direct Transport only supports 'object_store' and 'nixl' for tensor transport in ray.put()." + ) try: - serialized_value = self.get_serialization_context().serialize(value) + if tensor_transport != TensorTransportEnum.OBJECT_STORE: + ( + serialized_value, + tensors, + ) = self.get_serialization_context().serialize_gpu_objects(value) + else: + serialized_value = self.get_serialization_context().serialize(value) except TypeError as e: sio = io.StringIO() ray.util.inspect_serializability(value, print_file=sio) @@ -852,13 +869,17 @@ def put_object( # reference will be created. If another reference is created and # removed before this one, it will corrupt the state in the # reference counter. - return self.core_worker.put_object( + ret = self.core_worker.put_object( serialized_value, pin_object=pin_object, owner_address=owner_address, inline_small_object=True, _is_experimental_channel=_is_experimental_channel, + tensor_transport_val=tensor_transport.value, ) + if tensors: + self.gpu_object_manager.put_object(ret, tensor_transport, tensors) + return ret def raise_errors(self, serialized_objects, object_refs): out = self.deserialize_objects(serialized_objects, object_refs) @@ -867,21 +888,47 @@ def raise_errors(self, serialized_objects, object_refs): for e in out: _unhandled_error_handler(e) - def deserialize_objects(self, serialized_objects, object_refs): + def deserialize_objects( + self, + serialized_objects, + object_refs, + tensor_transport_hint: Optional[TensorTransportEnum] = None, + ): gpu_objects: Dict[str, List["torch.Tensor"]] = {} for obj_ref, (_, _, tensor_transport) in zip(object_refs, serialized_objects): - # If using a non-object store transport, then tensors will be sent - # out-of-band. Get them before deserializing the object store data. + # TODO: Here tensor_transport_hint is set by the user in ray.get(), tensor_transport is set + # in serialize_objects by ray.method(tensor_transport="xxx"), and obj_ref.tensor_transport() + # is set by ray.put(). We may clean up this logic in the future. if ( tensor_transport is None or tensor_transport == TensorTransportEnum.OBJECT_STORE + ) and ( + obj_ref is None + or obj_ref.tensor_transport() == TensorTransportEnum.OBJECT_STORE.value ): + # The object is not a gpu object, so we cannot use other external transport to + # fetch it. continue + # If the object is a gpu object, we can choose to use the object store or other external + # transport to fetch it. The `tensor_transport_hint` has the highest priority, then the + # tensor_transport in obj_ref.tensor_transport(), then the tensor_transport in serialize_objects, + # then the default value `OBJECT_STORE`. + chosen_tensor_transport = ( + tensor_transport_hint + or ( + TensorTransportEnum(obj_ref.tensor_transport()) if obj_ref else None + ) + or tensor_transport + or TensorTransportEnum.OBJECT_STORE + ) + object_id = obj_ref.hex() if object_id not in gpu_objects: + # If using a non-object store transport, then tensors will be sent + # out-of-band. Get them before deserializing the object store data. gpu_objects[object_id] = self.gpu_object_manager.get_gpu_object( - object_id + object_id, tensor_transport == chosen_tensor_transport ) # Function actor manager or the import thread may call pickle.loads @@ -900,6 +947,7 @@ def get_objects( timeout: Optional[float] = None, return_exceptions: bool = False, skip_deserialization: bool = False, + _tensor_transport: Optional[str] = None, ) -> Tuple[List[serialization.SerializedRayObject], bytes]: """Get the values in the object store associated with the IDs. @@ -918,6 +966,7 @@ def get_objects( raised. skip_deserialization: If true, only the buffer will be released and the object associated with the buffer will not be deserialized. + _tensor_transport: [Alpha] The tensor transport to use to fetch `torch.Tensors` found in the Ray Direct Transport object. Currently, this supports "object_store" and "nixl". Returns: list: List of deserialized objects or None if skip_deserialization is True. bytes: UUID of the debugger breakpoint we should drop @@ -930,7 +979,16 @@ def get_objects( f"Attempting to call `get` on the value {object_ref}, " "which is not an ray.ObjectRef." ) - + tensor_transport: TensorTransportEnum = ( + TensorTransportEnum.from_str(_tensor_transport) + if _tensor_transport is not None + else None + ) + assert tensor_transport in [ + TensorTransportEnum.OBJECT_STORE, + TensorTransportEnum.NIXL, + None, + ], "Currently, RDT only supports 'object_store' and 'nixl' for tensor transport in ray.get()." timeout_ms = ( int(timeout * 1000) if timeout is not None and timeout != -1 else -1 ) @@ -954,7 +1012,9 @@ def get_objects( if skip_deserialization: return None, debugger_breakpoint - values = self.deserialize_objects(serialized_objects, object_refs) + values = self.deserialize_objects( + serialized_objects, object_refs, tensor_transport_hint=tensor_transport + ) if not return_exceptions: # Raise exceptions instead of returning them to the user. for i, value in enumerate(values): @@ -2797,6 +2857,7 @@ def get( ], *, timeout: Optional[float] = None, + _tensor_transport: Optional[str] = None, ) -> Union[Any, List[Any]]: """Get a remote object or a list of remote objects from the object store. @@ -2832,6 +2893,7 @@ def get( corresponding object becomes available. Setting ``timeout=0`` will return the object immediately if it's available, else raise GetTimeoutError in accordance with the above docstring. + _tensor_transport: [Alpha] The tensor transport to use to fetch `torch.Tensors` found in the Ray Direct Transport object. Currently, this supports "object_store" and "nixl". Returns: A Python object or a list of Python objects. @@ -2891,7 +2953,9 @@ def get( "'object_refs' must either be an ObjectRef or a list of ObjectRefs. " ) - values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout) + values, debugger_breakpoint = worker.get_objects( + object_refs, timeout=timeout, _tensor_transport=_tensor_transport + ) for i, value in enumerate(values): if isinstance(value, RayError): if isinstance(value, ray.exceptions.ObjectLostError): @@ -2927,6 +2991,7 @@ def put( value: Any, *, _owner: Optional["ray.actor.ActorHandle"] = None, + _tensor_transport: str = "object_store", ) -> "ray.ObjectRef": """Store an object in the object store. @@ -2946,6 +3011,7 @@ def put( object prior to the object creator exiting, otherwise the reference will still be lost. *Note that this argument is an experimental API and should be avoided if possible.* + _tensor_transport: [Alpha] The tensor transport to use for the GPU object. Currently, this supports "object_store" and "nixl" for tensor transport in ray.put(). Returns: The object ref assigned to this value. @@ -2972,7 +3038,11 @@ def put( with profiling.profile("ray.put"): try: - object_ref = worker.put_object(value, owner_address=serialize_owner_address) + object_ref = worker.put_object( + value, + owner_address=serialize_owner_address, + _tensor_transport=_tensor_transport, + ) except ObjectStoreFullError: logger.info( "Put failed since the value was either too large or the " diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index c9c87b70053d..ef082647473c 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -3422,11 +3422,12 @@ cdef class CoreWorker: owner_address, c_bool inline_small_object, c_bool _is_experimental_channel, + int tensor_transport_val=0 ): """Create an object reference with the current worker as the owner. """ created_object = self.put_serialized_object_and_increment_local_ref( - serialized_object, pin_object, owner_address, inline_small_object, _is_experimental_channel) + serialized_object, pin_object, owner_address, inline_small_object, _is_experimental_channel, tensor_transport_val) if owner_address is None: owner_address = CCoreWorkerProcess.GetCoreWorker().GetRpcAddress().SerializeAsString() @@ -3435,7 +3436,8 @@ cdef class CoreWorker: return ObjectRef( created_object, owner_address, - skip_adding_local_ref=True + skip_adding_local_ref=True, + tensor_transport_val=tensor_transport_val ) def put_serialized_object_and_increment_local_ref( @@ -3445,6 +3447,7 @@ cdef class CoreWorker: owner_address=None, c_bool inline_small_object=True, c_bool _is_experimental_channel=False, + int tensor_transport_val=0 ): cdef: CObjectID c_object_id @@ -3458,6 +3461,7 @@ cdef class CoreWorker: serialized_object.contained_object_refs) size_t total_bytes = serialized_object.total_bytes + c_tensor_transport_val = tensor_transport_val with nogil: check_status(CCoreWorkerProcess.GetCoreWorker() .CreateOwnedAndIncrementLocalRef( @@ -3468,7 +3472,8 @@ cdef class CoreWorker: &c_object_id, &data, c_owner_address, - inline_small_object)) + inline_small_object, + c_tensor_transport_val)) if (data.get() == NULL): # Object already exists @@ -4470,7 +4475,9 @@ cdef class CoreWorker: if c_tensor_transport != TENSOR_TRANSPORT_OBJECT_STORE: # `output` contains tensors. We need to retrieve these tensors from `output` # and store them in the GPUObjectManager. - serialized_object = context.serialize_and_store_gpu_objects(output, return_id.Hex()) + serialized_object, tensors = context.serialize_gpu_objects(output) + context.store_gpu_objects(return_id.Hex().decode("ascii"), tensors) + else: serialized_object = context.serialize(output) data_size = serialized_object.total_bytes diff --git a/python/ray/experimental/collective/collective_tensor_transport.py b/python/ray/experimental/collective/collective_tensor_transport.py index 2fff70737bd7..61996d608d68 100644 --- a/python/ray/experimental/collective/collective_tensor_transport.py +++ b/python/ray/experimental/collective/collective_tensor_transport.py @@ -2,7 +2,6 @@ import ray from ray.experimental.collective.tensor_transport_manager import ( - TensorTransportEnum, TensorTransportManager, ) from ray.util.collective.types import ( @@ -35,39 +34,44 @@ def actor_has_tensor_transport(self, actor: "ray.actor.ActorHandle") -> bool: ) return len(communicators) > 0 + @staticmethod + def extract_tensor_transport_metadata( + gpu_object: List["torch.Tensor"], + ) -> CollectiveTransportMetadata: + tensor_meta = [] + device = None + if gpu_object: + device = gpu_object[0].device + for t in gpu_object: + if t.device.type != device.type: + raise ValueError( + "All tensors in an RDT object must have the same device type." + ) + tensor_meta.append((t.shape, t.dtype)) + return CollectiveTransportMetadata( + tensor_meta=tensor_meta, + tensor_device=device, + ) + @staticmethod def get_tensor_transport_metadata( src_actor: "ray.actor.ActorHandle", obj_id: str, - tensor_transport: TensorTransportEnum, ) -> CollectiveTransportMetadata: def __ray_get_tensor_transport_metadata__( self: "ray.actor.ActorHandle", obj_id: str, - tensor_transport: TensorTransportEnum, ) -> CollectiveTransportMetadata: from ray._private.worker import global_worker - from ray.util.collective.types import CollectiveTransportMetadata gpu_object_store = global_worker.gpu_object_manager.gpu_object_store # NOTE: We do not specify a timeout here because the user task that returns # it could take arbitrarily long and we don't want to trigger a spurious # timeout. gpu_object = gpu_object_store.wait_and_get_object(obj_id) - tensor_meta = [] - device = None - if gpu_object: - device = gpu_object[0].device - for t in gpu_object: - if t.device.type != device.type: - raise ValueError( - "All tensors in an RDT object must have the same device type." - ) - tensor_meta.append((t.shape, t.dtype)) - return CollectiveTransportMetadata( - tensor_meta=tensor_meta, - tensor_device=device, + return CollectiveTensorTransport.extract_tensor_transport_metadata( + gpu_object ) # Submit a Ray actor task to the source actor to get the tensor metadata. @@ -78,7 +82,7 @@ def __ray_get_tensor_transport_metadata__( # executing on the main thread blocking this task. return src_actor.__ray_call__.options(concurrency_group="_ray_system").remote( - __ray_get_tensor_transport_metadata__, obj_id, tensor_transport + __ray_get_tensor_transport_metadata__, obj_id ) @staticmethod diff --git a/python/ray/experimental/collective/nixl_tensor_transport.py b/python/ray/experimental/collective/nixl_tensor_transport.py index 45149a0b0608..40701590e9d0 100644 --- a/python/ray/experimental/collective/nixl_tensor_transport.py +++ b/python/ray/experimental/collective/nixl_tensor_transport.py @@ -2,7 +2,6 @@ import ray from ray.experimental.collective.tensor_transport_manager import ( - TensorTransportEnum, TensorTransportManager, ) from ray.util.collective.collective import get_group_handle @@ -42,53 +41,54 @@ def __ray_actor_has_tensor_transport__( ) ) + @staticmethod + def extract_tensor_transport_metadata( + gpu_object: List["torch.Tensor"], + ) -> NixlTransportMetadata: + from ray.util.collective.collective_group.nixl_backend import NixlBackend + from ray.util.collective.types import NixlTransportMetadata + + nixl_backend: NixlBackend = get_group_handle(NIXL_GROUP_NAME) + device = None + tensor_meta = [] + if gpu_object: + serialized_descs, agent_meta = nixl_backend.get_nixl_metadata(gpu_object) + # We assume all tensors in one GPU object have the same device type. + device = gpu_object[0].device + for t in gpu_object: + if t.device.type != device.type: + raise ValueError( + "All tensors in an RDT object must have the same device type." + ) + tensor_meta.append((t.shape, t.dtype)) + else: + serialized_descs, agent_meta = None, None + return NixlTransportMetadata( + tensor_meta=tensor_meta, + tensor_device=device, + nixl_serialized_descs=serialized_descs, + nixl_agent_meta=agent_meta, + ) + @staticmethod def get_tensor_transport_metadata( src_actor: "ray.actor.ActorHandle", obj_id: str, - tensor_transport: TensorTransportEnum, ) -> NixlTransportMetadata: - from ray.util.collective.collective_group.nixl_backend import NixlBackend - def __ray_get_tensor_transport_metadata__( self: "ray.actor.ActorHandle", obj_id: str, - tensor_transport: TensorTransportEnum, ) -> NixlTransportMetadata: from ray._private.worker import global_worker - from ray.util.collective.types import NixlTransportMetadata gpu_object_store = global_worker.gpu_object_manager.gpu_object_store # NOTE: We do not specify a timeout here because the user task that returns # it could take arbitrarily long and we don't want to trigger a spurious # timeout. gpu_object = gpu_object_store.wait_and_get_object(obj_id) - from ray.util.collective.collective import get_group_handle - - nixl_backend: NixlBackend = get_group_handle(NIXL_GROUP_NAME) - device = None - tensor_meta = [] - if gpu_object: - serialized_descs, agent_meta = nixl_backend.get_nixl_metadata( - gpu_object - ) - # We assume all tensors in one GPU object have the same device type. - device = gpu_object[0].device - for t in gpu_object: - if t.device.type != device.type: - raise ValueError( - "All tensors in an RDT object must have the same device type." - ) - tensor_meta.append((t.shape, t.dtype)) - else: - serialized_descs, agent_meta = None, None - return NixlTransportMetadata( - tensor_meta=tensor_meta, - tensor_device=device, - nixl_serialized_descs=serialized_descs, - nixl_agent_meta=agent_meta, - ) + + return NixlTensorTransport.extract_tensor_transport_metadata(gpu_object) # Submit a Ray actor task to the source actor to get the tensor metadata. # The metadata is a list of tuples, where each tuple contains the shape and dtype @@ -98,7 +98,7 @@ def __ray_get_tensor_transport_metadata__( # executing on the main thread blocking this task. return src_actor.__ray_call__.options(concurrency_group="_ray_system").remote( - __ray_get_tensor_transport_metadata__, obj_id, tensor_transport + __ray_get_tensor_transport_metadata__, obj_id ) @staticmethod diff --git a/python/ray/experimental/collective/tensor_transport_manager.py b/python/ray/experimental/collective/tensor_transport_manager.py index 8a8b29dae316..c86dc4554f17 100644 --- a/python/ray/experimental/collective/tensor_transport_manager.py +++ b/python/ray/experimental/collective/tensor_transport_manager.py @@ -2,7 +2,6 @@ from typing import TYPE_CHECKING, List, Optional import ray -from ray._private.custom_types import TensorTransportEnum from ray.util.collective.types import ( Backend, CommunicatorMetadata, @@ -48,7 +47,6 @@ def actor_has_tensor_transport(self, actor: "ray.actor.ActorHandle") -> bool: def get_tensor_transport_metadata( src_actor: "ray.actor.ActorHandle", obj_id: str, - tensor_transport: TensorTransportEnum, ) -> TensorTransportMetadata: """ Get the tensor transport metadata for the GPU object. @@ -58,12 +56,26 @@ def get_tensor_transport_metadata( Args: src_actor: The actor that runs this function. obj_id: The ID of the GPU object to get metadata for - tensor_transport: The tensor transport protocol to use for the GPU object. Returns: TensorTransportMetadata: A named tuple containing the tensor metadata. """ + @staticmethod + @abstractmethod + def extract_tensor_transport_metadata( + gpu_object: List["torch.Tensor"], + ) -> TensorTransportMetadata: + """ + Extract the tensor transport metadata from the GPU object. + + Args: + gpu_object: The GPU object to extract the tensor transport metadata from. + + Returns: + TensorTransportMetadata: The tensor transport metadata. + """ + @staticmethod @abstractmethod def get_communicator_metadata( diff --git a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py index f0a851386895..73ac927e355e 100644 --- a/python/ray/experimental/gpu_object_manager/gpu_object_manager.py +++ b/python/ray/experimental/gpu_object_manager/gpu_object_manager.py @@ -64,6 +64,7 @@ def __init__(self): # This dictionary is hosted on the "driver" process of the actors that # store and send/receive GPU objects. self.managed_gpu_object_metadata: Dict[str, GPUObjectMeta] = {} + # Per-actor local storage for GPU objects. We create the GPU object # store lazily, if a user specifies a non-default tensor_transport, to # avoid circular import and because it imports third-party dependencies @@ -96,11 +97,25 @@ def is_managed_object(self, obj_id: str) -> bool: """ return obj_id in self.managed_gpu_object_metadata + def add_gpu_object_metadata( + self, obj_ref: ObjectRef, gpu_object_meta: GPUObjectMeta + ): + """ + Add the GPU object metadata to the GPU object manager. + + Args: + obj_ref: The ObjectRef of the GPU object. + gpu_object_meta: The GPU object metadata. + """ + obj_id = obj_ref.hex() + self.managed_gpu_object_metadata[obj_id] = gpu_object_meta + def add_gpu_object_ref( self, obj_ref: ObjectRef, src_actor: "ray.actor.ActorHandle", tensor_transport: TensorTransportEnum, + tensor_transport_meta: Optional["TensorTransportMetadata"] = None, ): """Add a GPU object reference to the GPU object manager. This should be called whenever the current process calls a task that is annotated with @@ -110,6 +125,7 @@ def add_gpu_object_ref( obj_ref: The ObjectRef of the task output. src_actor: The actor that executes the task and that creates the GPU object. tensor_transport: The tensor transport protocol to use for the GPU object. + tensor_transport_meta: The tensor transport metadata that is pre-computed. """ from ray.experimental.collective import get_tensor_transport_manager from ray.experimental.gpu_object_manager.gpu_object_store import ( @@ -123,9 +139,12 @@ def add_gpu_object_ref( tensor_transport_manager = get_tensor_transport_manager( tensor_transport_backend ) - tensor_meta = tensor_transport_manager.get_tensor_transport_metadata( - src_actor, obj_id, tensor_transport - ) + if not tensor_transport_meta: + tensor_meta = tensor_transport_manager.get_tensor_transport_metadata( + src_actor, obj_id + ) + else: + tensor_meta = tensor_transport_meta self.managed_gpu_object_metadata[obj_id] = GPUObjectMeta( src_actor=src_actor, tensor_transport_backend=tensor_transport_backend, @@ -138,7 +157,11 @@ def _get_gpu_object_metadata(self, obj_ref: ObjectRef) -> GPUObjectMeta: obj_id = obj_ref.hex() return self.managed_gpu_object_metadata[obj_id] - def fetch_object(self, obj_id: str): + def fetch_object( + self, + obj_id: str, + tensor_transport: TensorTransportEnum = TensorTransportEnum.OBJECT_STORE, + ): """ Fetches the GPU object from the source actor's GPU object store via the object store instead of out-of-band tensor transfer and stores the tensors in the local GPU object store. @@ -149,25 +172,45 @@ def fetch_object(self, obj_id: str): Args: obj_id: The object ID of the GPU object. + tensor_transport: The tensor transport to use to fetch the GPU object. Returns: None """ + from ray.experimental.collective import get_tensor_transport_manager from ray.experimental.gpu_object_manager.gpu_object_store import ( __ray_fetch_gpu_object__, ) if self.gpu_object_store.has_object(obj_id): return - gpu_object_meta = self.managed_gpu_object_metadata[obj_id] src_actor = gpu_object_meta.src_actor - tensors = ray.get( - src_actor.__ray_call__.options(concurrency_group="_ray_system").remote( - __ray_fetch_gpu_object__, obj_id - ) + tensor_transport_backend = gpu_object_meta.tensor_transport_backend + tensor_transport_manager = get_tensor_transport_manager( + tensor_transport_backend + ) + tensor_transport_meta = gpu_object_meta.tensor_transport_meta + use_object_store = ( + tensor_transport == TensorTransportEnum.OBJECT_STORE + or isinstance(tensor_transport_meta, ObjectRef) ) - self.gpu_object_store.add_object(obj_id, tensors) + if use_object_store: + tensors = ray.get( + src_actor.__ray_call__.options(concurrency_group="_ray_system").remote( + __ray_fetch_gpu_object__, obj_id + ) + ) + self.gpu_object_store.add_object(obj_id, tensors) + else: + from ray.experimental.gpu_object_manager.gpu_object_store import ( + __ray_recv__, + ) + + communicator_meta = tensor_transport_manager.get_communicator_metadata( + None, None, tensor_transport_backend + ) + __ray_recv__(None, obj_id, tensor_transport_meta, communicator_meta) def trigger_out_of_band_tensor_transfer( self, dst_actor: "ray.actor.ActorHandle", task_args: Tuple[Any, ...] @@ -265,13 +308,24 @@ def trigger_out_of_band_tensor_transfer( communicator_meta, ) - def get_gpu_object(self, object_id: str) -> List["torch.Tensor"]: + def get_gpu_object( + self, + object_id: str, + tensor_transport: TensorTransportEnum = TensorTransportEnum.OBJECT_STORE, + ) -> List["torch.Tensor"]: """ Get the GPU object for a given object ID. + + Args: + object_id: The object ID of the GPU object. + tensor_transport: The tensor transport to use to fetch the GPU object. + + Returns: + The GPU object. """ gpu_object_store = self.gpu_object_store if self.is_managed_object(object_id): - self.fetch_object(object_id) + self.fetch_object(object_id, tensor_transport) # If the GPU object is the primary copy, it means the transfer is intra-actor. # In this case, we should not remove the GPU object after it is consumed once, @@ -315,3 +369,40 @@ def actor_has_tensor_transport( tensor_transport_backend ) return tensor_transport_manager.actor_has_tensor_transport(actor) + + def put_object( + self, + obj_ref: ObjectRef, + tensor_transport: TensorTransportEnum, + tensors: List["torch.Tensor"], + ): + """ + Put the GPU object into the GPU object manager. + + Args: + obj_ref: The object ref of the GPU object. + tensor_transport: The tensor transport backend to use. + tensors: The tensors to put into the GPU object manager. + + """ + from ray.experimental.collective import get_tensor_transport_manager + from ray.experimental.gpu_object_manager.gpu_object_store import ( + _tensor_transport_to_collective_backend, + ) + + tensor_transport_backend = _tensor_transport_to_collective_backend( + tensor_transport + ) + transport_manager = get_tensor_transport_manager(tensor_transport_backend) + tensor_transport_meta = transport_manager.extract_tensor_transport_metadata( + tensors + ) + + src_actor = ray.get_runtime_context().current_actor + self.gpu_object_store.add_object(obj_ref.hex(), tensors, is_primary=True) + self.add_gpu_object_ref( + obj_ref, + src_actor, + tensor_transport, + tensor_transport_meta=tensor_transport_meta, + ) diff --git a/python/ray/includes/libcoreworker.pxd b/python/ray/includes/libcoreworker.pxd index 2b401369f777..f010f3a13c0b 100644 --- a/python/ray/includes/libcoreworker.pxd +++ b/python/ray/includes/libcoreworker.pxd @@ -260,7 +260,8 @@ cdef extern from "ray/core_worker/core_worker.h" nogil: const c_vector[CObjectID] &contained_object_ids, CObjectID *object_id, shared_ptr[CBuffer] *data, const unique_ptr[CAddress] &owner_address, - c_bool inline_small_object) + c_bool inline_small_object, + CTensorTransport tensor_transport) CRayStatus CreateExisting(const shared_ptr[CBuffer] &metadata, const size_t data_size, const CObjectID &object_id, diff --git a/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py b/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py index cc66641dccf4..ce0a2e4e9a90 100644 --- a/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py +++ b/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py @@ -647,36 +647,6 @@ def double(self, data): ref = sender.tensor_method.options(tensor_transport="gloo").remote() -def test_gpu_object_ref_in_list_throws_exception(ray_start_regular): - """Test that passing GPU ObjectRefs inside lists as task arguments raises an error.""" - - print("loc2") - actor = GPUTestActor.remote() - create_collective_group([actor], backend="torch_gloo") - - tensor = torch.randn((1,)) - - # Test: GPU ref passed directly to task should work - gpu_ref = actor.echo.remote(tensor) - result = actor.double.remote(gpu_ref) - assert ray.get(result) == pytest.approx(tensor * 2) - - # Test: GPU ref inside a list should fail during task submission - with pytest.raises( - ValueError, - match="Passing RDT ObjectRefs inside data structures is not yet supported", - ): - actor.double.remote([gpu_ref]) - - # Test: Mixed list with GPU ref and normal data should also fail - normal_ref = ray.put("normal_data") - with pytest.raises( - ValueError, - match="Passing RDT ObjectRefs inside data structures is not yet supported", - ): - actor.double.remote([gpu_ref, normal_ref]) - - def test_app_error_inter_actor(ray_start_regular): world_size = 2 actors = [GPUTestActor.remote() for _ in range(world_size)] diff --git a/python/ray/tests/gpu_objects/test_gpu_objects_nixl.py b/python/ray/tests/gpu_objects/test_gpu_objects_nixl.py index 835a0ef4c059..1d57b8c675a9 100644 --- a/python/ray/tests/gpu_objects/test_gpu_objects_nixl.py +++ b/python/ray/tests/gpu_objects/test_gpu_objects_nixl.py @@ -16,6 +16,38 @@ def sum(self, data, device): assert data.device.type == device return data.sum().item() + def produce(self, tensors): + refs = [] + for t in tensors: + refs.append(ray.put(t, _tensor_transport="nixl")) + return refs + + def consume_with_nixl(self, refs): + tensors = [ray.get(ref) for ref in refs] + sum = 0 + for t in tensors: + assert t.device.type == "cuda" + sum += t.sum().item() + return sum + + def consume_with_object_store(self, refs): + tensors = [ray.get(ref, _tensor_transport="object_store") for ref in refs] + sum = 0 + for t in tensors: + assert t.device.type == "cuda" + sum += t.sum().item() + return sum + + def gc(self): + tensor = torch.tensor([1, 2, 3]).to("cuda") + ref = ray.put(tensor, _tensor_transport="nixl") + gpu_manager = ray._private.worker.global_worker.gpu_object_manager + assert gpu_manager.gpu_object_store.has_tensor(tensor) + del ref + gpu_manager.gpu_object_store.wait_tensor_freed(tensor, timeout=10) + assert not gpu_manager.gpu_object_store.has_tensor(tensor) + return "Success" + @pytest.mark.parametrize("ray_start_regular", [{"num_gpus": 2}], indirect=True) def test_p2p(ray_start_regular): @@ -34,12 +66,12 @@ def test_p2p(ray_start_regular): # Trigger tensor transfer from src to dst actor result = dst_actor.sum.remote(ref, "cuda") - assert tensor.sum().item() == ray.get(result) + assert tensor.sum().item() == ray.get(result, _tensor_transport="object_store") # Test CPU to CPU transfer ref1 = src_actor.echo.remote(tensor1, "cpu") result1 = dst_actor.sum.remote(ref1, "cpu") - assert tensor1.sum().item() == ray.get(result1) + assert tensor1.sum().item() == ray.get(result1, _tensor_transport="object_store") @pytest.mark.parametrize("ray_start_regular", [{"num_gpus": 1}], indirect=True) @@ -51,7 +83,42 @@ def test_intra_gpu_tensor_transfer(ray_start_regular): # Intra-actor communication for pure GPU tensors ref = actor.echo.remote(tensor, "cuda") result = actor.sum.remote(ref, "cuda") - assert tensor.sum().item() == ray.get(result) + assert tensor.sum().item() == ray.get(result, _tensor_transport="object_store") + + +@pytest.mark.parametrize("ray_start_regular", [{"num_gpus": 2}], indirect=True) +def test_put_and_get_object_with_nixl(ray_start_regular): + actors = [GPUTestActor.remote() for _ in range(2)] + src_actor, dst_actor = actors[0], actors[1] + tensor1 = torch.tensor([1, 2, 3]).to("cuda") + tensor2 = torch.tensor([4, 5, 6, 0]).to("cuda") + tensor3 = torch.tensor([7, 8, 9, 0, 0]).to("cuda") + tensors = [tensor1, tensor2, tensor3] + ref = src_actor.produce.remote(tensors) + ref1 = dst_actor.consume_with_nixl.remote(ref) + result1 = ray.get(ref1) + assert result1 == 45 + + +@pytest.mark.parametrize("ray_start_regular", [{"num_gpus": 2}], indirect=True) +def test_put_and_get_object_with_object_store(ray_start_regular): + actors = [GPUTestActor.remote() for _ in range(2)] + src_actor, dst_actor = actors[0], actors[1] + tensor1 = torch.tensor([1, 2, 3]).to("cuda") + tensor2 = torch.tensor([4, 5, 6, 0]).to("cuda") + tensor3 = torch.tensor([7, 8, 9, 0, 0]).to("cuda") + tensors = [tensor1, tensor2, tensor3] + ref = src_actor.produce.remote(tensors) + ref1 = dst_actor.consume_with_object_store.remote(ref) + result1 = ray.get(ref1) + assert result1 == 45 + + +@pytest.mark.parametrize("ray_start_regular", [{"num_gpus": 1}], indirect=True) +def test_put_gc(ray_start_regular): + actor = GPUTestActor.remote() + ref = actor.gc.remote() + assert ray.get(ref) == "Success" if __name__ == "__main__": diff --git a/python/ray/util/collective/collective_group/nixl_backend.py b/python/ray/util/collective/collective_group/nixl_backend.py index ea7fd3d7ab22..1950f952d4ef 100644 --- a/python/ray/util/collective/collective_group/nixl_backend.py +++ b/python/ray/util/collective/collective_group/nixl_backend.py @@ -58,7 +58,13 @@ def recv( remote_name = nixl_agent.add_remote_agent(remote_nixl_agent_meta) xfer_handle = nixl_agent.initialize_xfer( - "READ", local_descs.trim(), remote_descs, remote_name + # "UUID" here is just a placeholder, can be any bytes, but without it, + # nixl will fail to transfer multiple times. + "READ", + local_descs.trim(), + remote_descs, + remote_name, + "UUID", ) state = nixl_agent.transfer(xfer_handle) diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 04db6d5398f7..c8922e067f16 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -1017,7 +1017,8 @@ Status CoreWorker::CreateOwnedAndIncrementLocalRef( ObjectID *object_id, std::shared_ptr *data, const std::unique_ptr &owner_address, - bool inline_small_object) { + bool inline_small_object, + rpc::TensorTransport tensor_transport) { auto status = WaitForActorRegistered(contained_object_ids); if (!status.ok()) { return status; @@ -1036,7 +1037,14 @@ Status CoreWorker::CreateOwnedAndIncrementLocalRef( data_size + metadata->Size(), /*is_reconstructable=*/false, /*add_local_ref=*/true, - NodeID::FromBinary(rpc_address_.node_id())); + NodeID::FromBinary(rpc_address_.node_id()), + /*tensor_transport=*/tensor_transport); + + // Register the callback to free the GPU object when it is out of scope. + if (tensor_transport != rpc::TensorTransport::OBJECT_STORE) { + reference_counter_->AddObjectOutOfScopeOrFreedCallback( + *object_id, options_.free_actor_object_callback); + } } else { // Because in the remote worker's `HandleAssignObjectOwner`, // a `WaitForRefRemoved` RPC request will be sent back to diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index 58e0cd65cf8a..4091ec28609b 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -516,6 +516,7 @@ class CoreWorker { /// defaults to this worker. /// \param[in] inline_small_object Whether to inline create this object if it's /// small. + /// \param[in] tensor_transport The tensor transport to use for the object. /// \return Status. Status CreateOwnedAndIncrementLocalRef( bool is_experimental_mutable_object, @@ -525,7 +526,8 @@ class CoreWorker { ObjectID *object_id, std::shared_ptr *data, const std::unique_ptr &owner_address = nullptr, - bool inline_small_object = true); + bool inline_small_object = true, + rpc::TensorTransport tensor_transport = rpc::TensorTransport::OBJECT_STORE); /// Create and return a buffer in the object store that can be directly written /// into, for an object ID that already exists. After writing to the buffer, the From 2a2a4921884980ed0fb9970ae7b2fc66d5794b3b Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Thu, 11 Sep 2025 13:21:38 -0700 Subject: [PATCH 583/634] [core] Make Object Manager Unit Testable (#56315) Signed-off-by: joshlee --- .../plasma/fake_plasma_client.h | 124 ++++++++------- src/mock/ray/object_manager/object_manager.h | 2 + src/ray/object_manager/BUILD.bazel | 4 +- src/ray/object_manager/object_manager.cc | 92 ++++-------- src/ray/object_manager/object_manager.h | 52 ++++--- src/ray/object_manager/tests/BUILD.bazel | 18 +++ .../tests/object_manager_test.cc | 142 ++++++++++++++++++ src/ray/raylet/BUILD.bazel | 1 + src/ray/raylet/main.cc | 107 +++++++++---- src/ray/raylet/tests/BUILD.bazel | 2 +- src/ray/raylet/tests/node_manager_test.cc | 95 +----------- src/ray/rpc/BUILD.bazel | 31 ---- src/ray/rpc/object_manager/BUILD.bazel | 51 +++++++ .../fake_object_manager_client.h | 110 ++++++++++++++ .../object_manager/object_manager_client.h | 14 +- .../object_manager_client_interface.h | 52 +++++++ 16 files changed, 600 insertions(+), 297 deletions(-) create mode 100644 src/ray/object_manager/tests/object_manager_test.cc create mode 100644 src/ray/rpc/object_manager/BUILD.bazel create mode 100644 src/ray/rpc/object_manager/fake_object_manager_client.h create mode 100644 src/ray/rpc/object_manager/object_manager_client_interface.h diff --git a/src/fakes/ray/object_manager/plasma/fake_plasma_client.h b/src/fakes/ray/object_manager/plasma/fake_plasma_client.h index 74ab301fd259..42030c1ab68a 100644 --- a/src/fakes/ray/object_manager/plasma/fake_plasma_client.h +++ b/src/fakes/ray/object_manager/plasma/fake_plasma_client.h @@ -16,13 +16,10 @@ #include #include +#include #include -// A simple fake implementation of PlasmaClientInterface for use in unit tests. -// -// This base fake does nothing (returns OK for most methods, empty results for Get). -// Extend it in test files to add behavior (recording batches, timeouts, missing objects). - +#include "absl/container/flat_hash_map.h" #include "ray/common/buffer.h" #include "ray/common/id.h" #include "ray/common/status.h" @@ -32,75 +29,98 @@ namespace plasma { class FakePlasmaClient : public PlasmaClientInterface { public: - FakePlasmaClient(std::vector> *observed_batches = nullptr) - : observed_batches_(observed_batches) {} - - Status Connect(const std::string &, const std::string &, int) override { + Status Connect(const std::string &store_socket_name, + const std::string &manager_socket_name = "", + int num_retries = -1) override { + return Status::OK(); + }; + + Status CreateAndSpillIfNeeded(const ObjectID &object_id, + const ray::rpc::Address &owner_address, + bool is_mutable, + int64_t data_size, + const uint8_t *metadata, + int64_t metadata_size, + std::shared_ptr *data, + plasma::flatbuf::ObjectSource source, + int device_num = 0) override { return Status::OK(); } - Status Release(const ObjectID &) override { return Status::OK(); } - - Status Contains(const ObjectID &, bool *) override { return Status::OK(); } - - Status Disconnect() override { return Status::OK(); } - - Status Get(const std::vector &object_ids, - int64_t /*timeout_ms*/, - std::vector *object_buffers) override { - if (observed_batches_ != nullptr) { - observed_batches_->push_back(object_ids); + Status TryCreateImmediately(const ObjectID &object_id, + const ray::rpc::Address &owner_address, + int64_t data_size, + const uint8_t *metadata, + int64_t metadata_size, + std::shared_ptr *data, + plasma::flatbuf::ObjectSource source, + int device_num = 0) override { + std::vector data_vec(data_size); + if (data != nullptr && data_size > 0) { + data_vec.assign(data->get()->Data(), data->get()->Data() + data_size); } - // Return non-null buffers to simulate presence for tests. - object_buffers->resize(object_ids.size()); - for (size_t i = 0; i < object_ids.size(); i++) { - uint8_t byte = 0; - auto parent = - std::make_shared(&byte, 1, /*copy_data=*/true); - (*object_buffers)[i].data = SharedMemoryBuffer::Slice(parent, 0, 1); - (*object_buffers)[i].metadata = SharedMemoryBuffer::Slice(parent, 0, 1); + std::vector metadata_vec; + if (metadata != nullptr && metadata_size > 0) { + metadata_vec.assign(metadata, metadata + metadata_size); } + objects_in_plasma_.emplace( + object_id, std::make_pair(std::move(data_vec), std::move(metadata_vec))); return Status::OK(); } - Status GetExperimentalMutableObject(const ObjectID &, - std::unique_ptr *) override { + Status Get(const std::vector &object_ids, + int64_t timeout_ms, + std::vector *object_buffers) override { + for (const auto &id : object_ids) { + auto &buffers = objects_in_plasma_[id]; + plasma::ObjectBuffer shm_buffer{std::make_shared( + buffers.first.data(), buffers.first.size()), + std::make_shared( + buffers.second.data(), buffers.second.size())}; + object_buffers->emplace_back(shm_buffer); + } return Status::OK(); } - Status Seal(const ObjectID &) override { return Status::OK(); } + Status GetExperimentalMutableObject( + const ObjectID &object_id, + std::unique_ptr *mutable_object) override { + return Status::OK(); + } - Status Abort(const ObjectID &) override { return Status::OK(); } + Status Release(const ObjectID &object_id) override { + objects_in_plasma_.erase(object_id); + return Status::OK(); + } - Status CreateAndSpillIfNeeded(const ObjectID &, - const ray::rpc::Address &, - bool, - int64_t, - const uint8_t *, - int64_t, - std::shared_ptr *, - flatbuf::ObjectSource, - int) override { + Status Contains(const ObjectID &object_id, bool *has_object) override { + *has_object = objects_in_plasma_.contains(object_id); return Status::OK(); } - Status TryCreateImmediately(const ObjectID &, - const ray::rpc::Address &, - int64_t, - const uint8_t *, - int64_t, - std::shared_ptr *, - flatbuf::ObjectSource, - int) override { + Status Abort(const ObjectID &object_id) override { return Status::OK(); } + + Status Seal(const ObjectID &object_id) override { return Status::OK(); } + + Status Delete(const std::vector &object_ids) override { + num_free_objects_requests++; + for (const auto &id : object_ids) { + objects_in_plasma_.erase(id); + } return Status::OK(); } - Status Delete(const std::vector &) override { return Status::OK(); } + Status Disconnect() override { return Status::OK(); }; + + std::string DebugString() { return ""; } + + int64_t store_capacity() { return 0; } StatusOr GetMemoryUsage() override { return std::string("fake"); } - private: - std::vector> *observed_batches_; + absl::flat_hash_map, std::vector>> + objects_in_plasma_; + uint32_t num_free_objects_requests = 0; }; } // namespace plasma diff --git a/src/mock/ray/object_manager/object_manager.h b/src/mock/ray/object_manager/object_manager.h index 67813e247ca2..3f16bb85b3f5 100644 --- a/src/mock/ray/object_manager/object_manager.h +++ b/src/mock/ray/object_manager/object_manager.h @@ -54,6 +54,8 @@ class MockObjectManager : public ObjectManagerInterface { MOCK_METHOD(void, Stop, (), (override)); MOCK_METHOD(void, RecordMetrics, (), (override)); MOCK_METHOD(void, HandleNodeRemoved, (const NodeID &node_id), (override)); + MOCK_METHOD(void, HandleObjectAdded, (const ObjectInfo &object_info), (override)); + MOCK_METHOD(void, HandleObjectDeleted, (const ObjectID &object_id), (override)); }; } // namespace ray diff --git a/src/ray/object_manager/BUILD.bazel b/src/ray/object_manager/BUILD.bazel index 5ffc9b9081ed..ebe575525e53 100644 --- a/src/ray/object_manager/BUILD.bazel +++ b/src/ray/object_manager/BUILD.bazel @@ -19,8 +19,8 @@ ray_cc_library( "//src/ray/object_manager/plasma:plasma_store_server_lib", "//src/ray/protobuf:common_cc_proto", "//src/ray/protobuf:node_manager_cc_proto", - "//src/ray/rpc:object_manager_client", - "//src/ray/rpc:object_manager_server", + "//src/ray/rpc/object_manager:object_manager_client_interface", + "//src/ray/rpc/object_manager:object_manager_server", "@com_google_absl//absl/container:flat_hash_map", ], ) diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index e43af1acc6a4..4c2dafed127b 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -66,44 +66,24 @@ ObjectManager::ObjectManager( IObjectDirectory *object_directory, RestoreSpilledObjectCallback restore_spilled_object, std::function get_spilled_object_url, - SpillObjectsCallback spill_objects_callback, - std::function object_store_full_callback, - AddObjectCallback add_object_callback, - DeleteObjectCallback delete_object_callback, std::function(const ObjectID &object_id)> pin_object, - const std::function fail_pull_request) + std::function fail_pull_request, + const std::shared_ptr &buffer_pool_store_client, + std::unique_ptr object_store_internal, + std::function( + const std::string &address, + const int port, + rpc::ClientCallManager &client_call_manager)> object_manager_client_factory, + instrumented_io_context &rpc_service) : main_service_(&main_service), self_node_id_(self_node_id), config_(config), gcs_client_(gcs_client), object_directory_(object_directory), - object_store_internal_(std::make_unique( - config, - spill_objects_callback, - object_store_full_callback, - /*add_object_callback=*/ - [this, add_object_callback = std::move(add_object_callback)]( - const ObjectInfo &object_info) { - main_service_->post( - [this, object_info, &add_object_callback]() { - HandleObjectAdded(object_info); - add_object_callback(object_info); - }, - "ObjectManager.ObjectAdded"); - }, - /*delete_object_callback=*/ - [this, delete_object_callback = std::move(delete_object_callback)]( - const ObjectID &object_id) { - main_service_->post( - [this, object_id, &delete_object_callback]() { - HandleObjectDeleted(object_id); - delete_object_callback(object_id); - }, - "ObjectManager.ObjectDeleted"); - })), - buffer_pool_store_client_(std::make_shared()), + object_store_internal_(std::move(object_store_internal)), + buffer_pool_store_client_(buffer_pool_store_client), buffer_pool_(buffer_pool_store_client_, config_.object_chunk_size), - rpc_work_(rpc_service_.get_executor()), + rpc_service_(rpc_service), object_manager_server_("ObjectManager", config_.object_manager_port, config_.object_manager_address == "127.0.0.1", @@ -119,7 +99,8 @@ ObjectManager::ObjectManager( push_manager_(std::make_unique(/* max_chunks_in_flight= */ std::max( static_cast(1L), static_cast(config_.max_bytes_in_flight / - config_.object_chunk_size)))) { + config_.object_chunk_size)))), + object_manager_client_factory_(std::move(object_manager_client_factory)) { RAY_CHECK_GT(config_.rpc_service_threads_number, 0); pull_retry_timer_.async_wait([this](const boost::system::error_code &e) { Tick(e); }); @@ -173,16 +154,7 @@ bool ObjectManager::IsPlasmaObjectSpillable(const ObjectID &object_id) { return plasma::plasma_store_runner->IsPlasmaObjectSpillable(object_id); } -void ObjectManager::RunRpcService(int index) { - SetThreadName(absl::StrFormat("rpc.obj.mgr.%d", index)); - rpc_service_.run(); -} - void ObjectManager::StartRpcService() { - rpc_threads_.resize(config_.rpc_service_threads_number); - for (int i = 0; i < config_.rpc_service_threads_number; i++) { - rpc_threads_[i] = std::thread(&ObjectManager::RunRpcService, this, i); - } object_manager_server_.RegisterService( std::make_unique(rpc_service_, *this), false /* token_auth */); @@ -191,11 +163,6 @@ void ObjectManager::StartRpcService() { void ObjectManager::StopRpcService() { rpc_service_.stop(); - for (int i = 0; i < config_.rpc_service_threads_number; i++) { - if (rpc_threads_[i].joinable()) { - rpc_threads_[i].join(); - } - } object_manager_server_.Shutdown(); } @@ -516,14 +483,15 @@ void ObjectManager::PushObjectInternal(const ObjectID &object_id, }); } -void ObjectManager::SendObjectChunk(const UniqueID &push_id, - const ObjectID &object_id, - const NodeID &node_id, - uint64_t chunk_index, - std::shared_ptr rpc_client, - std::function on_complete, - std::shared_ptr chunk_reader, - bool from_disk) { +void ObjectManager::SendObjectChunk( + const UniqueID &push_id, + const ObjectID &object_id, + const NodeID &node_id, + uint64_t chunk_index, + std::shared_ptr rpc_client, + std::function on_complete, + std::shared_ptr chunk_reader, + bool from_disk) { double start_time = absl::GetCurrentTimeNanos() / 1e9; rpc::PushRequest push_request; // Set request header @@ -671,9 +639,11 @@ void ObjectManager::FreeObjects(const std::vector &object_ids, bool local_only) { buffer_pool_.FreeObjects(object_ids); if (!local_only) { - std::vector> rpc_clients; + std::vector> rpc_clients; + // TODO(#56414): optimize this so we don't have to send a free objects request for + // every object to every node const auto &node_info_map = gcs_client_.Nodes().GetAll(); - for (const auto &[node_id, node_info] : node_info_map) { + for (const auto &[node_id, _] : node_info_map) { if (node_id == self_node_id_) { continue; } @@ -692,7 +662,7 @@ void ObjectManager::FreeObjects(const std::vector &object_ids, void ObjectManager::SpreadFreeObjectsRequest( const std::vector &object_ids, - const std::vector> &rpc_clients) { + const std::vector> &rpc_clients) { // This code path should be called from node manager. rpc::FreeObjectsRequest free_objects_request; for (const auto &e : object_ids) { @@ -711,7 +681,7 @@ void ObjectManager::SpreadFreeObjectsRequest( } } -std::shared_ptr ObjectManager::GetRpcClient( +std::shared_ptr ObjectManager::GetRpcClient( const NodeID &node_id) { auto it = remote_object_manager_clients_.find(node_id); if (it != remote_object_manager_clients_.end()) { @@ -722,9 +692,9 @@ std::shared_ptr ObjectManager::GetRpcClient( return nullptr; } auto object_manager_client = - std::make_shared(node_info->node_manager_address(), - node_info->object_manager_port(), - client_call_manager_); + object_manager_client_factory_(node_info->node_manager_address(), + node_info->object_manager_port(), + client_call_manager_); RAY_LOG(DEBUG) << "Get rpc client, address: " << node_info->node_manager_address() << ", port: " << node_info->object_manager_port() diff --git a/src/ray/object_manager/object_manager.h b/src/ray/object_manager/object_manager.h index 593f2d2b1455..3a8658393a12 100644 --- a/src/ray/object_manager/object_manager.h +++ b/src/ray/object_manager/object_manager.h @@ -30,7 +30,7 @@ #include "ray/object_manager/object_directory.h" #include "ray/object_manager/pull_manager.h" #include "ray/object_manager/push_manager.h" -#include "ray/rpc/object_manager/object_manager_client.h" +#include "ray/rpc/object_manager/object_manager_client_interface.h" #include "ray/rpc/object_manager/object_manager_server.h" #include "ray/stats/metric.h" #include "src/ray/protobuf/common.pb.h" @@ -78,6 +78,7 @@ struct LocalObjectInfo { /// Information from the object store about the object. ObjectInfo object_info; }; + class ObjectStoreRunner { public: ObjectStoreRunner(const ObjectManagerConfig &config, @@ -108,10 +109,12 @@ class ObjectManagerInterface { virtual bool PullManagerHasPullsQueued() const = 0; virtual int64_t GetMemoryCapacity() const = 0; virtual std::string DebugString() const = 0; - virtual void FillObjectStoreStats(rpc::GetNodeStatsReply *reply) const = 0; + virtual void FillObjectStoreStats(rpc::GetNodeStatsReply *repOly) const = 0; virtual double GetUsedMemoryPercentage() const = 0; virtual void Stop() = 0; virtual void RecordMetrics() = 0; + virtual void HandleObjectAdded(const ObjectInfo &object_info) = 0; + virtual void HandleObjectDeleted(const ObjectID &object_id) = 0; virtual ~ObjectManagerInterface() = default; }; @@ -164,7 +167,6 @@ class ObjectManager : public ObjectManagerInterface, return pull_manager_->NumInactivePulls(task_key); } - public: /// Takes user-defined IObjectDirectory implementation. /// When this constructor is used, the ObjectManager assumes ownership of /// the given ObjectDirectory instance. @@ -180,12 +182,15 @@ class ObjectManager : public ObjectManagerInterface, IObjectDirectory *object_directory, RestoreSpilledObjectCallback restore_spilled_object, std::function get_spilled_object_url, - SpillObjectsCallback spill_objects_callback, - std::function object_store_full_callback, - AddObjectCallback add_object_callback, - DeleteObjectCallback delete_object_callback, std::function(const ObjectID &object_id)> pin_object, - std::function fail_pull_request); + std::function fail_pull_request, + const std::shared_ptr &buffer_pool_store_client, + std::unique_ptr object_store_internal, + std::function( + const std::string &address, + const int port, + rpc::ClientCallManager &client_call_manager)> object_manager_client_factory, + instrumented_io_context &rpc_service); ~ObjectManager() override; @@ -268,13 +273,14 @@ class ObjectManager : public ObjectManagerInterface, private: friend class TestObjectManager; + friend uint32_t NumRemoteFreeObjectsRequests(const ObjectManager &object_manager); /// Spread the Free request to all objects managers. /// /// \param object_ids the The list of ObjectIDs to be deleted. void SpreadFreeObjectsRequest( const std::vector &object_ids, - const std::vector> &rpc_clients); + const std::vector> &rpc_clients); /// Pushing a known local object to a remote object manager. /// @@ -320,7 +326,7 @@ class ObjectManager : public ObjectManagerInterface, const ObjectID &object_id, const NodeID &node_id, uint64_t chunk_index, - std::shared_ptr rpc_client, + std::shared_ptr rpc_client, std::function on_complete, std::shared_ptr chunk_reader, bool from_disk); @@ -333,12 +339,12 @@ class ObjectManager : public ObjectManagerInterface, /// Handle an object being added to this node. This adds the object to the /// directory, pushes the object to other nodes if necessary, and cancels any /// outstanding Pull requests for the object. - void HandleObjectAdded(const ObjectInfo &object_info); + void HandleObjectAdded(const ObjectInfo &object_info) override; /// Handle an object being deleted from this node. This registers object remove /// with directory. This also asks the pull manager to fetch this object again /// as soon as possible. - void HandleObjectDeleted(const ObjectID &object_id); + void HandleObjectDeleted(const ObjectID &object_id) override; /// This is used to notify the main thread that the sending of a chunk has /// completed. @@ -398,7 +404,7 @@ class ObjectManager : public ObjectManagerInterface, /// Get the rpc client according to the node ID /// /// \param node_id Remote node id, will send rpc request to it - std::shared_ptr GetRpcClient(const NodeID &node_id); + std::shared_ptr GetRpcClient(const NodeID &node_id); /// Weak reference to main service. We ensure this object is destroyed before /// main_service_ is stopped. @@ -418,20 +424,13 @@ class ObjectManager : public ObjectManagerInterface, /// Used by the buffer pool to read and write objects in the local store /// during object transfers. - std::shared_ptr buffer_pool_store_client_; + std::shared_ptr buffer_pool_store_client_; /// Manages accesses to local objects for object transfers. ObjectBufferPool buffer_pool_; /// Multi-thread asio service, deal with all outgoing and incoming RPC request. - instrumented_io_context rpc_service_; - - /// Keep rpc service running when no task in rpc service. - boost::asio::executor_work_guard rpc_work_; - - /// The thread pool used for running `rpc_service`. - /// Data copy operations during request are done in this thread pool. - std::vector rpc_threads_; + instrumented_io_context &rpc_service_; /// Mapping from locally available objects to information about those objects /// including when the object was last pushed to other object managers. @@ -456,7 +455,7 @@ class ObjectManager : public ObjectManagerInterface, rpc::ClientCallManager client_call_manager_; /// Client id - object manager gRPC client. - absl::flat_hash_map> + absl::flat_hash_map> remote_object_manager_clients_; /// Callback to trigger direct restoration of an object. @@ -475,6 +474,13 @@ class ObjectManager : public ObjectManagerInterface, /// Object pull manager. std::unique_ptr pull_manager_; + /// Factory function to create object manager client. + std::function( + const std::string &address, + const int port, + rpc::ClientCallManager &client_call_manager)> + object_manager_client_factory_; + /// Running sum of the amount of memory used in the object store. int64_t used_memory_ = 0; diff --git a/src/ray/object_manager/tests/BUILD.bazel b/src/ray/object_manager/tests/BUILD.bazel index c59944be6ddc..ac154895c962 100644 --- a/src/ray/object_manager/tests/BUILD.bazel +++ b/src/ray/object_manager/tests/BUILD.bazel @@ -103,3 +103,21 @@ ray_cc_test( "@com_google_googletest//:gtest_main", ], ) + +ray_cc_test( + name = "object_manager_test", + size = "medium", + srcs = [ + "object_manager_test.cc", + ], + tags = ["team:core"], + deps = [ + "//:ray_mock", + "//src/fakes/ray/object_manager/plasma:fake_plasma_client", + "//src/ray/common:test_utils", + "//src/ray/object_manager", + "//src/ray/rpc/object_manager:fake_object_manager_client", + "//src/ray/util:temporary_directory", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/object_manager/tests/object_manager_test.cc b/src/ray/object_manager/tests/object_manager_test.cc new file mode 100644 index 000000000000..4a6f99f8a1b0 --- /dev/null +++ b/src/ray/object_manager/tests/object_manager_test.cc @@ -0,0 +1,142 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/object_manager/object_manager.h" + +#include + +#include +#include +#include +#include +#include + +#include "fakes/ray/object_manager/plasma/fake_plasma_client.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/object_manager/object_directory.h" +#include "ray/common/asio/instrumented_io_context.h" +#include "ray/common/buffer.h" +#include "ray/common/id.h" +#include "ray/common/ray_object.h" +#include "ray/common/status.h" +#include "ray/object_manager/common.h" +#include "ray/rpc/object_manager/fake_object_manager_client.h" +#include "ray/util/temporary_directory.h" + +namespace ray { + +using ::testing::_; +using ::testing::Invoke; +using ::testing::Return; + +class ObjectManagerTest : public ::testing::Test { + protected: + ObjectManagerTest() + : io_work_(boost::asio::make_work_guard(io_context_.get_executor())), + rpc_work_(boost::asio::make_work_guard(rpc_context_.get_executor())) { + ObjectManagerConfig config_; + config_.object_manager_address = "127.0.0.1"; + config_.object_manager_port = 0; + config_.store_socket_name = "test_store_socket"; + config_.rpc_service_threads_number = 1; + + local_node_id_ = NodeID::FromRandom(); + mock_gcs_client_ = std::make_unique(); + mock_object_directory_ = std::make_unique(); + fake_plasma_client_ = std::make_shared(); + + object_manager_ = std::make_unique( + io_context_, + local_node_id_, + config_, + *mock_gcs_client_, + mock_object_directory_.get(), + // RestoreSpilledObjectCallback + [](const ObjectID &object_id, + int64_t object_size, + const std::string &object_url, + std::function callback) {}, + // get_spilled_object_url + [](const ObjectID &object_id) -> std::string { return ""; }, + // pin_object + [](const ObjectID &object_id) -> std::unique_ptr { return nullptr; }, + // fail_pull_request + [](const ObjectID &object_id, rpc::ErrorType error_type) {}, + fake_plasma_client_, + nullptr, + [](const std::string &address, + const int port, + ray::rpc::ClientCallManager &client_call_manager) { + return std::make_shared( + address, port, client_call_manager); + }, + rpc_context_); + } + + NodeID local_node_id_; + + std::unique_ptr mock_gcs_client_; + std::unique_ptr mock_object_directory_; + std::unique_ptr object_manager_; + std::shared_ptr fake_plasma_client_; + + instrumented_io_context io_context_{/*enable_lag_probe=*/false, + /*running_on_single_thread=*/true}; + instrumented_io_context rpc_context_{/*enable_lag_probe=*/false, + /*running_on_single_thread=*/true}; + boost::asio::executor_work_guard io_work_; + boost::asio::executor_work_guard rpc_work_; +}; + +uint32_t NumRemoteFreeObjectsRequests(const ObjectManager &object_manager) { + uint32_t num_free_objects_requests = 0; + for (const auto &[node_id, rpc_client] : + object_manager.remote_object_manager_clients_) { + auto fake_rpc_client = + std::dynamic_pointer_cast(rpc_client); + num_free_objects_requests += fake_rpc_client->num_free_objects_requests; + } + return num_free_objects_requests; +} + +TEST_F(ObjectManagerTest, TestFreeObjectsLocalOnlyFalse) { + auto object_id = ObjectID::FromRandom(); + + absl::flat_hash_map node_info_map_; + rpc::GcsNodeInfo self_node_info; + self_node_info.set_node_id(local_node_id_.Binary()); + node_info_map_[local_node_id_] = self_node_info; + NodeID remote_node_id_ = NodeID::FromRandom(); + rpc::GcsNodeInfo remote_node_info; + remote_node_info.set_node_id(remote_node_id_.Binary()); + node_info_map_[remote_node_id_] = remote_node_info; + + EXPECT_CALL(*mock_gcs_client_->mock_node_accessor, GetAll()) + .WillOnce(::testing::ReturnRef(node_info_map_)); + EXPECT_CALL(*mock_gcs_client_->mock_node_accessor, Get(remote_node_id_, _)) + .WillOnce(::testing::Return(&remote_node_info)); + + fake_plasma_client_->objects_in_plasma_[object_id] = + std::make_pair(std::vector(1), std::vector(1)); + object_manager_->FreeObjects({object_id}, false); + ASSERT_EQ(fake_plasma_client_->num_free_objects_requests, 1); + ASSERT_TRUE(!fake_plasma_client_->objects_in_plasma_.contains(object_id)); + ASSERT_EQ(NumRemoteFreeObjectsRequests(*object_manager_), 0); + ASSERT_EQ(rpc_context_.poll_one(), 1); + ASSERT_EQ(NumRemoteFreeObjectsRequests(*object_manager_), 1); +} + +} // namespace ray diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index ec69ba17b874..0625c5fba1d9 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -291,6 +291,7 @@ ray_cc_binary( "//src/ray/rpc:metrics_agent_client", "//src/ray/rpc:raylet_client_lib", "//src/ray/rpc:raylet_client_pool", + "//src/ray/rpc/object_manager:object_manager_client", "//src/ray/stats:stats_lib", "//src/ray/util:cmd_line_utils", "//src/ray/util:event", diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index c2421afe83b8..a440a64e6e79 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -36,6 +36,7 @@ #include "ray/raylet/local_object_manager.h" #include "ray/raylet/local_object_manager_interface.h" #include "ray/raylet/raylet.h" +#include "ray/rpc/object_manager/object_manager_client.h" #include "ray/rpc/raylet/raylet_client.h" #include "ray/stats/stats.h" #include "ray/util/cmd_line_utils.h" @@ -252,6 +253,16 @@ int main(int argc, char *argv[]) { boost::asio::executor_work_guard main_service_work(main_service.get_executor()); + instrumented_io_context object_manager_rpc_service{/*emit_metrics=*/false, + /*running_on_single_thread=*/false, + "object_manager_rpc_io_context"}; + boost::asio::executor_work_guard + object_manager_rpc_work(object_manager_rpc_service.get_executor()); + + /// The thread pool used for running `rpc_service`. + /// Data copy operations during request are done in this thread pool. + std::vector object_manager_rpc_threads; + // Initialize gcs client std::unique_ptr gcs_client; ray::gcs::GcsClientOptions client_options(FLAGS_gcs_address, @@ -325,15 +336,23 @@ int main(int argc, char *argv[]) { auto shutted_down = std::make_shared>(false); - auto shutdown_raylet_after_unregistration = - [&main_service, &raylet_socket_name, &raylet, &gcs_client]() { - // We should stop the service and remove the local socket file. - raylet->Stop(); - gcs_client->Disconnect(); - ray::stats::Shutdown(); - main_service.stop(); - remove(raylet_socket_name.c_str()); - }; + auto shutdown_raylet_after_unregistration = [&main_service, + &raylet_socket_name, + &raylet, + &gcs_client, + &object_manager_rpc_threads]() { + // We should stop the service and remove the local socket file. + raylet->Stop(); + gcs_client->Disconnect(); + ray::stats::Shutdown(); + main_service.stop(); + for (size_t i = 0; i < object_manager_rpc_threads.size(); i++) { + if (object_manager_rpc_threads[i].joinable()) { + object_manager_rpc_threads[i].join(); + } + } + remove(raylet_socket_name.c_str()); + }; // Shut down raylet gracefully, in a synchronous fashion. // This is an internal method and should only be run on the main_service. @@ -594,24 +613,8 @@ int main(int argc, char *argv[]) { node_manager->MarkObjectsAsFailed(error_type, {ref}, ray::JobID::Nil()); }); - object_manager = std::make_unique( - main_service, - raylet_node_id, + auto object_store_runner = std::make_unique( object_manager_config, - *gcs_client, - object_directory.get(), - /*restore_spilled_object=*/ - [&](const ray::ObjectID &object_id, - int64_t object_size, - const std::string &object_url, - std::function callback) { - local_object_manager->AsyncRestoreSpilledObject( - object_id, object_size, object_url, std::move(callback)); - }, - /*get_spilled_object_url=*/ - [&](const ray::ObjectID &object_id) { - return local_object_manager->GetLocalSpilledObjectURL(object_id); - }, /*spill_objects_callback=*/ [&]() { // This callback is called from the plasma store thread. @@ -631,11 +634,48 @@ int main(int argc, char *argv[]) { }, /*add_object_callback=*/ [&](const ray::ObjectInfo &object_info) { - node_manager->HandleObjectLocal(object_info); + main_service.post( + [&object_manager, &node_manager, object_info]() { + object_manager->HandleObjectAdded(object_info); + node_manager->HandleObjectLocal(object_info); + }, + "ObjectManager.ObjectAdded"); }, /*delete_object_callback=*/ [&](const ray::ObjectID &object_id) { - node_manager->HandleObjectMissing(object_id); + main_service.post( + [&object_manager, &node_manager, object_id]() { + object_manager->HandleObjectDeleted(object_id); + node_manager->HandleObjectMissing(object_id); + }, + "ObjectManager.ObjectDeleted"); + }); + + object_manager_rpc_threads.resize(object_manager_config.rpc_service_threads_number); + for (int i = 0; i < object_manager_config.rpc_service_threads_number; i++) { + object_manager_rpc_threads[i] = std::thread([&object_manager_rpc_service, i] { + SetThreadName(absl::StrFormat("rpc.obj.mgr.%d", i)); + object_manager_rpc_service.run(); + }); + } + + object_manager = std::make_unique( + main_service, + raylet_node_id, + object_manager_config, + *gcs_client, + object_directory.get(), + /*restore_spilled_object=*/ + [&](const ray::ObjectID &object_id, + int64_t object_size, + const std::string &object_url, + std::function callback) { + local_object_manager->AsyncRestoreSpilledObject( + object_id, object_size, object_url, std::move(callback)); + }, + /*get_spilled_object_url=*/ + [&](const ray::ObjectID &object_id) { + return local_object_manager->GetLocalSpilledObjectURL(object_id); }, /*pin_object=*/ [&](const ray::ObjectID &object_id) { @@ -653,7 +693,16 @@ int main(int argc, char *argv[]) { ray::rpc::ObjectReference ref; ref.set_object_id(object_id.Binary()); node_manager->MarkObjectsAsFailed(error_type, {ref}, ray::JobID::Nil()); - }); + }, + std::make_shared(), + std::move(object_store_runner), + [&](const std::string &address, + const int port, + ray::rpc::ClientCallManager &call_manager) { + return std::make_shared( + address, port, call_manager); + }, + object_manager_rpc_service); local_object_manager = std::make_unique( raylet_node_id, diff --git a/src/ray/raylet/tests/BUILD.bazel b/src/ray/raylet/tests/BUILD.bazel index bf264b24fecc..746660fbeca6 100644 --- a/src/ray/raylet/tests/BUILD.bazel +++ b/src/ray/raylet/tests/BUILD.bazel @@ -181,11 +181,11 @@ ray_cc_test( ":util", "//:ray_fakes", "//:ray_mock", + "//src/fakes/ray/object_manager/plasma:fake_plasma_client", "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/ray/common:lease", "//src/ray/common:ray_object", "//src/ray/common:task_common", - "//src/ray/object_manager/plasma:plasma_client", "//src/ray/observability:fake_metric", "//src/ray/raylet:local_object_manager_interface", "//src/ray/raylet:node_manager", diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index 6b9469be35c5..b05d4c5ac0bd 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -21,6 +21,7 @@ #include #include +#include "fakes/ray/object_manager/plasma/fake_plasma_client.h" #include "fakes/ray/pubsub/subscriber.h" #include "fakes/ray/rpc/raylet/raylet_client.h" #include "gmock/gmock.h" @@ -28,13 +29,11 @@ #include "mock/ray/gcs/gcs_client/gcs_client.h" #include "mock/ray/object_manager/object_directory.h" #include "mock/ray/object_manager/object_manager.h" -#include "mock/ray/object_manager/plasma/client.h" #include "mock/ray/raylet/local_lease_manager.h" #include "mock/ray/raylet/worker_pool.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/buffer.h" #include "ray/common/scheduling/cluster_resource_data.h" -#include "ray/object_manager/plasma/client.h" #include "ray/observability/fake_metric.h" #include "ray/raylet/local_object_manager_interface.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" @@ -96,96 +95,6 @@ class FakeLocalObjectManager : public LocalObjectManagerInterface { std::shared_ptr> objects_pending_deletion_; }; -class FakePlasmaClient : public plasma::PlasmaClientInterface { - public: - Status Connect(const std::string &store_socket_name, - const std::string &manager_socket_name = "", - int num_retries = -1) override { - return Status::OK(); - }; - - Status CreateAndSpillIfNeeded(const ObjectID &object_id, - const ray::rpc::Address &owner_address, - bool is_mutable, - int64_t data_size, - const uint8_t *metadata, - int64_t metadata_size, - std::shared_ptr *data, - plasma::flatbuf::ObjectSource source, - int device_num = 0) override { - return TryCreateImmediately( - object_id, owner_address, data_size, metadata, metadata_size, data, source); - } - - Status TryCreateImmediately(const ObjectID &object_id, - const ray::rpc::Address &owner_address, - int64_t data_size, - const uint8_t *metadata, - int64_t metadata_size, - std::shared_ptr *data, - plasma::flatbuf::ObjectSource source, - int device_num = 0) override { - objects_ids_in_plasma_.emplace(object_id); - objects_in_plasma_.emplace( - object_id, std::make_pair(std::vector{}, std::vector{})); - return Status::OK(); - } - - Status Get(const std::vector &object_ids, - int64_t timeout_ms, - std::vector *object_buffers) override { - for (const auto &id : object_ids) { - auto &buffers = objects_in_plasma_[id]; - plasma::ObjectBuffer shm_buffer{std::make_shared( - buffers.first.data(), buffers.first.size()), - std::make_shared( - buffers.second.data(), buffers.second.size())}; - object_buffers->emplace_back(shm_buffer); - } - return Status::OK(); - } - - Status GetExperimentalMutableObject( - const ObjectID &object_id, - std::unique_ptr *mutable_object) override { - return Status::OK(); - } - - Status Release(const ObjectID &object_id) override { - objects_ids_in_plasma_.erase(object_id); - return Status::OK(); - } - - Status Contains(const ObjectID &object_id, bool *has_object) override { - *has_object = objects_ids_in_plasma_.find(object_id) != objects_ids_in_plasma_.end(); - return Status::OK(); - } - - Status Abort(const ObjectID &object_id) override { return Status::OK(); } - - Status Seal(const ObjectID &object_id) override { return Status::OK(); } - - Status Delete(const std::vector &object_ids) override { - for (const auto &id : object_ids) { - objects_ids_in_plasma_.erase(id); - } - return Status::OK(); - } - - Status Disconnect() override { return Status::OK(); }; - - std::string DebugString() { return ""; } - - int64_t store_capacity() { return 1; } - - StatusOr GetMemoryUsage() override { return std::string("fake"); } - - private: - absl::flat_hash_set objects_ids_in_plasma_; - absl::flat_hash_map, std::vector>> - objects_in_plasma_; -}; - LeaseSpecification BuildLeaseSpec( const std::unordered_map &resources) { TaskSpecBuilder builder; @@ -525,7 +434,7 @@ class NodeManagerTest : public ::testing::Test { std::unique_ptr mock_object_manager_; core::experimental::MockMutableObjectProvider *mock_mutable_object_provider_; std::shared_ptr mock_store_client_ = - std::make_shared(); + std::make_shared(); std::unique_ptr node_manager_; MockWorkerPool mock_worker_pool_; diff --git a/src/ray/rpc/BUILD.bazel b/src/ray/rpc/BUILD.bazel index 584020804e6a..3b663ebf23f9 100644 --- a/src/ray/rpc/BUILD.bazel +++ b/src/ray/rpc/BUILD.bazel @@ -154,37 +154,6 @@ ray_cc_library( ], ) -ray_cc_library( - name = "object_manager_client", - hdrs = [ - "object_manager/object_manager_client.h", - ], - visibility = ["//visibility:public"], - deps = [ - "//src/ray/object_manager:object_manager_grpc_client_manager", - "//src/ray/protobuf:object_manager_cc_grpc", - "//src/ray/util:logging", - "@com_github_grpc_grpc//:grpc++", - ], -) - -ray_cc_library( - name = "object_manager_server", - hdrs = [ - "object_manager/object_manager_server.h", - ], - visibility = ["//visibility:public"], - deps = [ - ":grpc_server", - ":server_call", - "//src/ray/common:asio", - "//src/ray/object_manager:object_manager_grpc_client_manager", - "//src/ray/protobuf:object_manager_cc_grpc", - "@boost//:asio", - "@com_github_grpc_grpc//:grpc++", - ], -) - ray_cc_library( name = "raylet_client_interface", hdrs = [ diff --git a/src/ray/rpc/object_manager/BUILD.bazel b/src/ray/rpc/object_manager/BUILD.bazel new file mode 100644 index 000000000000..d6bf8135ac43 --- /dev/null +++ b/src/ray/rpc/object_manager/BUILD.bazel @@ -0,0 +1,51 @@ +load("//bazel:ray.bzl", "ray_cc_library") + +ray_cc_library( + name = "object_manager_client", + hdrs = [ + "object_manager_client.h", + ], + visibility = ["//visibility:public"], + deps = [ + ":object_manager_client_interface", + "//src/ray/object_manager:object_manager_grpc_client_manager", + "//src/ray/protobuf:object_manager_cc_grpc", + "//src/ray/util:logging", + "@com_github_grpc_grpc//:grpc++", + ], +) + +ray_cc_library( + name = "object_manager_client_interface", + hdrs = ["object_manager_client_interface.h"], + deps = [ + "//src/ray/protobuf:object_manager_cc_proto", + ], +) + +ray_cc_library( + name = "object_manager_server", + hdrs = [ + "object_manager_server.h", + ], + visibility = ["//visibility:public"], + deps = [ + "//src/ray/common:asio", + "//src/ray/object_manager:object_manager_grpc_client_manager", + "//src/ray/protobuf:object_manager_cc_grpc", + "//src/ray/rpc:grpc_server", + "//src/ray/rpc:server_call", + "@boost//:asio", + "@com_github_grpc_grpc//:grpc++", + ], +) + +ray_cc_library( + name = "fake_object_manager_client", + hdrs = ["fake_object_manager_client.h"], + deps = [ + ":object_manager_client_interface", + "//src/ray/common:status", + "//src/ray/protobuf:object_manager_cc_proto", + ], +) diff --git a/src/ray/rpc/object_manager/fake_object_manager_client.h b/src/ray/rpc/object_manager/fake_object_manager_client.h new file mode 100644 index 000000000000..97b0af8fc848 --- /dev/null +++ b/src/ray/rpc/object_manager/fake_object_manager_client.h @@ -0,0 +1,110 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "ray/common/status.h" +#include "ray/rpc/object_manager/object_manager_client_interface.h" +#include "src/ray/protobuf/object_manager.pb.h" + +namespace ray { +namespace rpc { + +template +using ClientCallback = std::function; + +class FakeObjectManagerClient : public ObjectManagerClientInterface { + public: + FakeObjectManagerClient(const std::string &address, + const int port, + ClientCallManager &client_call_manager) + : address_(address), port_(port) {} + + void Push(const PushRequest &request, + const ClientCallback &callback) override { + num_push_requests++; + push_callbacks.push_back(callback); + } + + void Pull(const PullRequest &request, + const ClientCallback &callback) override { + num_pull_requests++; + pull_callbacks.push_back(callback); + } + + void FreeObjects(const FreeObjectsRequest &request, + const ClientCallback &callback) override { + num_free_objects_requests++; + free_objects_callbacks.push_back(callback); + } + + bool ReplyPush(const Status &status = Status::OK()) { + if (push_callbacks.empty()) { + return false; + } + PushReply reply; + auto callback = push_callbacks.front(); + push_callbacks.pop_front(); + callback(status, std::move(reply)); + return true; + } + + bool ReplyPull(const Status &status = Status::OK()) { + if (pull_callbacks.empty()) { + return false; + } + PullReply reply; + auto callback = pull_callbacks.front(); + pull_callbacks.pop_front(); + callback(status, std::move(reply)); + return true; + } + + bool ReplyFreeObjects(const Status &status = Status::OK()) { + if (free_objects_callbacks.empty()) { + return false; + } + FreeObjectsReply reply; + auto callback = free_objects_callbacks.front(); + free_objects_callbacks.pop_front(); + callback(status, std::move(reply)); + return true; + } + + const std::string &GetAddress() const { return address_; } + + int GetPort() const { return port_; } + + uint32_t num_push_requests = 0; + uint32_t num_pull_requests = 0; + uint32_t num_free_objects_requests = 0; + + std::list> push_callbacks; + std::list> pull_callbacks; + std::list> free_objects_callbacks; + + std::string address_; + int port_; +}; + +} // namespace rpc +} // namespace ray diff --git a/src/ray/rpc/object_manager/object_manager_client.h b/src/ray/rpc/object_manager/object_manager_client.h index 1adadb9d72bd..121363961346 100644 --- a/src/ray/rpc/object_manager/object_manager_client.h +++ b/src/ray/rpc/object_manager/object_manager_client.h @@ -25,6 +25,7 @@ #include "ray/common/status.h" #include "ray/object_manager/grpc_client_manager.h" #include "ray/rpc/grpc_client.h" +#include "ray/rpc/object_manager/object_manager_client_interface.h" #include "ray/util/logging.h" #include "src/ray/protobuf/object_manager.grpc.pb.h" #include "src/ray/protobuf/object_manager.pb.h" @@ -32,8 +33,8 @@ namespace ray { namespace rpc { -/// Client used for communicating with a remote node manager server. -class ObjectManagerClient { +/// Client used for communicating with a remote object manager server. +class ObjectManagerClient : public ObjectManagerClientInterface { public: /// Constructor. /// @@ -54,7 +55,8 @@ class ObjectManagerClient { VOID_RPC_CLIENT_METHOD(ObjectManagerService, Push, grpc_client_manager_->GetGrpcClient(), - /*method_timeout_ms*/ -1, ) + /*method_timeout_ms*/ -1, + override) /// Pull object from remote object manager /// @@ -63,7 +65,8 @@ class ObjectManagerClient { VOID_RPC_CLIENT_METHOD(ObjectManagerService, Pull, grpc_client_manager_->GetGrpcClient(), - /*method_timeout_ms*/ -1, ) + /*method_timeout_ms*/ -1, + override) /// Tell remote object manager to free objects /// @@ -72,7 +75,8 @@ class ObjectManagerClient { VOID_RPC_CLIENT_METHOD(ObjectManagerService, FreeObjects, grpc_client_manager_->GetGrpcClient(), - /*method_timeout_ms*/ -1, ) + /*method_timeout_ms*/ -1, + override) private: std::unique_ptr> grpc_client_manager_; diff --git a/src/ray/rpc/object_manager/object_manager_client_interface.h b/src/ray/rpc/object_manager/object_manager_client_interface.h new file mode 100644 index 000000000000..ad74fcf5ffdd --- /dev/null +++ b/src/ray/rpc/object_manager/object_manager_client_interface.h @@ -0,0 +1,52 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "src/ray/protobuf/object_manager.pb.h" + +namespace ray { +namespace rpc { + +template +using ClientCallback = std::function; +/// Abstract client interface for object manager clients. +class ObjectManagerClientInterface { + public: + virtual ~ObjectManagerClientInterface() = default; + + /// Push object to remote object manager + /// + /// \param request The request message. + /// \param callback The callback function that handles reply from server + virtual void Push(const PushRequest &request, + const ClientCallback &callback) = 0; + + /// Pull object from remote object manager + /// + /// \param request The request message + /// \param callback The callback function that handles reply from server + virtual void Pull(const PullRequest &request, + const ClientCallback &callback) = 0; + + /// Tell remote object manager to free objects + /// + /// \param request The request message + /// \param callback The callback function that handles reply + virtual void FreeObjects(const FreeObjectsRequest &request, + const ClientCallback &callback) = 0; +}; + +} // namespace rpc +} // namespace ray From f900567b902022d34026e336e99fe88728fc8158 Mon Sep 17 00:00:00 2001 From: Ibrahim Rabbani Date: Thu, 11 Sep 2025 15:22:04 -0700 Subject: [PATCH 584/634] [core] (cgroups 6/n) CgroupManager cleans up the entire cgroup hierarchy in reverse order when destroyed. (#56260) Signed-off-by: irabbani Signed-off-by: Ibrahim Rabbani Signed-off-by: Ibrahim Rabbani Co-authored-by: Edward Oakes --- python/ray/_private/worker.py | 4 - python/ray/_private/workers/default_worker.py | 1 - python/ray/_raylet.pyx | 3 +- python/ray/includes/libcoreworker.pxd | 1 - src/ray/common/cgroup2/BUILD.bazel | 62 +++-- src/ray/common/cgroup2/cgroup_manager.cc | 75 +++--- src/ray/common/cgroup2/cgroup_manager.h | 7 +- src/ray/common/cgroup2/fake_cgroup_driver.h | 109 ++++++-- src/ray/common/cgroup2/noop_cgroup_manager.cc | 39 +++ .../cgroup2/tests/cgroup_manager_test.cc | 233 +++++++++++++++--- src/ray/core_worker/core_worker_options.h | 7 +- src/ray/core_worker/core_worker_process.cc | 7 - src/ray/raylet/BUILD.bazel | 3 +- src/ray/raylet/main.cc | 75 +++++- src/ray/raylet/node_manager.h | 3 - src/ray/raylet/tests/worker_pool_test.cc | 3 +- src/ray/raylet/worker_pool.cc | 12 +- src/ray/raylet/worker_pool.h | 8 +- 18 files changed, 493 insertions(+), 159 deletions(-) create mode 100644 src/ray/common/cgroup2/noop_cgroup_manager.cc diff --git a/python/ray/_private/worker.py b/python/ray/_private/worker.py index 6657a9246033..f40a8c4f0464 100644 --- a/python/ray/_private/worker.py +++ b/python/ray/_private/worker.py @@ -2440,7 +2440,6 @@ def is_initialized() -> bool: return ray._private.worker.global_worker.connected -# TODO(hjiang): Add cgroup path along with [enable_resource_isolation]. @with_connect_or_shutdown_lock def connect( node, @@ -2459,7 +2458,6 @@ def connect( worker_launch_time_ms: int = -1, worker_launched_time_ms: int = -1, debug_source: str = "", - enable_resource_isolation: bool = False, ): """Connect this worker to the raylet, to Plasma, and to GCS. @@ -2488,7 +2486,6 @@ def connect( finshes launching. If the worker is not launched by raylet (e.g., driver), this must be -1 (default value). debug_source: Source information for `CoreWorker`, used for debugging and informational purpose, rather than functional purpose. - enable_resource_isolation: If true, core worker enables resource isolation by adding itself into appropriate cgroup. """ # Do some basic checking to make sure we didn't call ray.init twice. error_message = "Perhaps you called ray.init twice by accident?" @@ -2667,7 +2664,6 @@ def connect( worker_launch_time_ms, worker_launched_time_ms, debug_source, - enable_resource_isolation, ) if mode == SCRIPT_MODE: diff --git a/python/ray/_private/workers/default_worker.py b/python/ray/_private/workers/default_worker.py index cb6bce0043f6..12cf83040574 100644 --- a/python/ray/_private/workers/default_worker.py +++ b/python/ray/_private/workers/default_worker.py @@ -271,7 +271,6 @@ ray_debugger_external=args.ray_debugger_external, worker_launch_time_ms=args.worker_launch_time_ms, worker_launched_time_ms=worker_launched_time_ms, - enable_resource_isolation=args.enable_resource_isolation, ) worker = ray._private.worker.global_worker diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index ef082647473c..f5548d8edc40 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -3002,7 +3002,7 @@ cdef class CoreWorker: local_mode, driver_name, serialized_job_config, metrics_agent_port, runtime_env_hash, startup_token, session_name, cluster_id, entrypoint, - worker_launch_time_ms, worker_launched_time_ms, debug_source, enable_resource_isolation): + worker_launch_time_ms, worker_launched_time_ms, debug_source): self.is_local_mode = local_mode cdef CCoreWorkerOptions options = CCoreWorkerOptions() @@ -3058,7 +3058,6 @@ cdef class CoreWorker: options.worker_launch_time_ms = worker_launch_time_ms options.worker_launched_time_ms = worker_launched_time_ms options.debug_source = debug_source - options.enable_resource_isolation = enable_resource_isolation CCoreWorkerProcess.Initialize(options) self.cgname_to_eventloop_dict = None diff --git a/python/ray/includes/libcoreworker.pxd b/python/ray/includes/libcoreworker.pxd index f010f3a13c0b..5a65b6b3cc8b 100644 --- a/python/ray/includes/libcoreworker.pxd +++ b/python/ray/includes/libcoreworker.pxd @@ -439,7 +439,6 @@ cdef extern from "ray/core_worker/core_worker.h" nogil: int64_t worker_launch_time_ms int64_t worker_launched_time_ms c_string debug_source - c_bool enable_resource_isolation cdef cppclass CCoreWorkerProcess "ray::core::CoreWorkerProcess": @staticmethod diff --git a/src/ray/common/cgroup2/BUILD.bazel b/src/ray/common/cgroup2/BUILD.bazel index 85cd2a9dc059..a3f39c7040ad 100644 --- a/src/ray/common/cgroup2/BUILD.bazel +++ b/src/ray/common/cgroup2/BUILD.bazel @@ -1,13 +1,40 @@ load("//bazel:ray.bzl", "ray_cc_library") +config_setting( + name = "is_linux", + constraint_values = ["@platforms//os:linux"], +) + +# Public targets. +ray_cc_library( + name = "cgroup_manager", + srcs = select({ + ":is_linux": ["cgroup_manager.cc"], + "//conditions:default": ["noop_cgroup_manager.cc"], + }), + hdrs = ["cgroup_manager.h"], + visibility = ["//visibility:public"], + deps = [ + ":cgroup_driver_interface", + ":cgroup_manager_interface", + "//src/ray/common:status", + "//src/ray/common:status_or", + ] + select({ + ":is_linux": [ + ":scoped_cgroup_operation", + "//src/ray/util:logging", + "@com_google_absl//absl/strings", + ], + "//conditions:default": [], + }), +) + ray_cc_library( name = "cgroup_driver_interface", hdrs = [ "cgroup_driver_interface.h", ], - target_compatible_with = [ - "@platforms//os:linux", - ], + visibility = ["//visibility:public"], deps = [ "//src/ray/common:status", "//src/ray/common:status_or", @@ -19,9 +46,7 @@ ray_cc_library( hdrs = [ "cgroup_manager_interface.h", ], - target_compatible_with = [ - "@platforms//os:linux", - ], + visibility = ["//visibility:public"], deps = [ "//src/ray/common:status", "//src/ray/common:status_or", @@ -29,18 +54,17 @@ ray_cc_library( ) ray_cc_library( - name = "cgroup_manager", - srcs = ["cgroup_manager.cc"], + name = "sysfs_cgroup_driver", + srcs = ["sysfs_cgroup_driver.cc"], hdrs = [ - "cgroup_manager.h", - "scoped_cgroup_operation.h", + "sysfs_cgroup_driver.h", ], target_compatible_with = [ "@platforms//os:linux", ], + visibility = ["//visibility:public"], deps = [ ":cgroup_driver_interface", - ":cgroup_manager_interface", "//src/ray/common:status", "//src/ray/common:status_or", "//src/ray/util:logging", @@ -48,22 +72,16 @@ ray_cc_library( ], ) +# Private Targets. ray_cc_library( - name = "sysfs_cgroup_driver", - srcs = ["sysfs_cgroup_driver.cc"], + name = "scoped_cgroup_operation", hdrs = [ - "sysfs_cgroup_driver.h", + "scoped_cgroup_operation.h", ], target_compatible_with = [ "@platforms//os:linux", ], - deps = [ - ":cgroup_driver_interface", - "//src/ray/common:status", - "//src/ray/common:status_or", - "//src/ray/util:logging", - "@com_google_absl//absl/strings", - ], + visibility = [":__subpackages__"], ) ray_cc_library( @@ -74,6 +92,7 @@ ray_cc_library( target_compatible_with = [ "@platforms//os:linux", ], + visibility = [":__subpackages__"], deps = [ ":cgroup_driver_interface", "//src/ray/common:status", @@ -87,6 +106,7 @@ ray_cc_library( target_compatible_with = [ "@platforms//os:linux", ], + visibility = [":__subpackages__"], deps = [ "//src/ray/common:id", "//src/ray/common:status", diff --git a/src/ray/common/cgroup2/cgroup_manager.cc b/src/ray/common/cgroup2/cgroup_manager.cc index 28402ba27617..e210191565d2 100644 --- a/src/ray/common/cgroup2/cgroup_manager.cc +++ b/src/ray/common/cgroup2/cgroup_manager.cc @@ -113,48 +113,65 @@ StatusOr> CgroupManager::Create( return cgroup_manager; } -// TODO(#54703): This is a placeholder for cleanup. This will call -// CgroupDriver::DeleteCgroup. void CgroupManager::RegisterDeleteCgroup(const std::string &cgroup_path) { - cleanup_operations_.emplace_back([cgroup = cgroup_path]() { - RAY_LOG(INFO) << absl::StrFormat("Deleting all cgroup %s.", cgroup); + cleanup_operations_.emplace_back([this, cgroup = cgroup_path]() { + Status s = this->cgroup_driver_->DeleteCgroup(cgroup); + if (!s.ok()) { + RAY_LOG(WARNING) << absl::StrFormat( + "Failed to delete cgroup %s with error %s.", cgroup, s.ToString()); + } }); } -// TODO(#54703): This is a placeholder for cleanup. This will call -// CgroupDriver::MoveAllProcesses. void CgroupManager::RegisterMoveAllProcesses(const std::string &from, const std::string &to) { - cleanup_operations_.emplace_back([from_cgroup = from, to_cgroup = to]() { - RAY_LOG(INFO) << absl::StrFormat( - "Moved All Processes from %s to %s.", from_cgroup, to_cgroup); + cleanup_operations_.emplace_back([this, from_cgroup = from, to_cgroup = to]() { + Status s = this->cgroup_driver_->MoveAllProcesses(from_cgroup, to_cgroup); + if (!s.ok()) { + RAY_LOG(WARNING) << absl::StrFormat( + "Failed to move all processes from %s to %s with error %s", + from_cgroup, + to_cgroup, + s.ToString()); + } }); } -// TODO(#54703): This is a placeholder for cleanup. This will call -// CgroupDriver::AddConstraint(cgroup, constraint, default_value). template void CgroupManager::RegisterRemoveConstraint(const std::string &cgroup, const Constraint &constraint) { cleanup_operations_.emplace_back( - [constrained_cgroup = cgroup, constraint_to_remove = constraint]() { - RAY_LOG(INFO) << absl::StrFormat( - "Setting constraint %s to default value %lld for cgroup %s", - constraint_to_remove.name_, - constraint_to_remove.default_value_, - constrained_cgroup); + [this, constrained_cgroup = cgroup, constraint_to_remove = constraint]() { + std::string default_value = std::to_string(constraint_to_remove.default_value_); + Status s = this->cgroup_driver_->AddConstraint(constrained_cgroup, + constraint_to_remove.controller_, + constraint_to_remove.name_, + default_value); + if (!s.ok()) { + RAY_LOG(WARNING) << absl::StrFormat( + "Failed to set constraint %s=%s to default value for cgroup %s with error " + "%s.", + constraint_to_remove.name_, + default_value, + constrained_cgroup, + s.ToString()); + } }); } -// TODO(#54703): This is a placeholder for cleanup. This will call -// CgroupDriver::DisableController. -void CgroupManager::RegisterDisableController(const std::string &cgroup, +void CgroupManager::RegisterDisableController(const std::string &cgroup_path, const std::string &controller) { - cleanup_operations_.emplace_back([cgroup_to_clean = cgroup, - controller_to_disable = controller]() { - RAY_LOG(INFO) << absl::StrFormat( - "Disabling controller %s for cgroup %s.", controller_to_disable, cgroup_to_clean); - }); + cleanup_operations_.emplace_back( + [this, cgroup = cgroup_path, controller_to_disable = controller]() { + Status s = this->cgroup_driver_->DisableController(cgroup, controller_to_disable); + if (!s.ok()) { + RAY_LOG(WARNING) << absl::StrFormat( + "Failed to disable controller %s for cgroup %s with error %s", + controller_to_disable, + cgroup, + s.ToString()); + } + }); } Status CgroupManager::Initialize(int64_t system_reserved_cpu_weight, @@ -168,11 +185,11 @@ Status CgroupManager::Initialize(int64_t system_reserved_cpu_weight, cpu_weight_constraint_.Max() - system_reserved_cpu_weight; RAY_LOG(INFO) << absl::StrFormat( - "Initializing CgroupManager at base cgroup path at %s. Ray's cgroup " - "hierarchy will under the node cgroup %s. The %s controllers will be " + "Initializing CgroupManager at base cgroup at '%s'. Ray's cgroup " + "hierarchy will under the node cgroup at '%s'. The %s controllers will be " "enabled. " - "System cgroup %s will have constraints [%s=%lld, %s=%lld]. " - "Application cgroup %s will have constraints [%s=%lld].", + "The system cgroup at '%s' will have constraints [%s=%lld, %s=%lld]. " + "The application cgroup '%s' will have constraints [%s=%lld].", base_cgroup_path_, node_cgroup_path_, supported_controllers, diff --git a/src/ray/common/cgroup2/cgroup_manager.h b/src/ray/common/cgroup2/cgroup_manager.h index ba402d496c32..466abe6e1257 100644 --- a/src/ray/common/cgroup2/cgroup_manager.h +++ b/src/ray/common/cgroup2/cgroup_manager.h @@ -111,9 +111,10 @@ class CgroupManager : public CgroupManagerInterface { Status Initialize(const int64_t system_reserved_cpu_weight, const int64_t system_reserved_memory_bytes); - // TODO(#54703): This is a placeholder for cleanup. This will be implemented in the a - // future PR. - void RegisterDeleteCgroup(const std::string &cgroup_path); + // The Register* methods register a callback that will execute in the destructor + // in FILO order. All callbacks required the cgroup_driver_ to be available to + // remove the cgroup hierarchy. + void RegisterDeleteCgroup(const std::string &cgroup); void RegisterMoveAllProcesses(const std::string &from, const std::string &to); template void RegisterRemoveConstraint(const std::string &cgroup, diff --git a/src/ray/common/cgroup2/fake_cgroup_driver.h b/src/ray/common/cgroup2/fake_cgroup_driver.h index 6245d0fc1f5b..e49e63429670 100644 --- a/src/ray/common/cgroup2/fake_cgroup_driver.h +++ b/src/ray/common/cgroup2/fake_cgroup_driver.h @@ -29,7 +29,7 @@ namespace ray { struct FakeCgroup { std::string path_; std::vector processes_; - std::vector> constraints_; + std::unordered_map constraints_; std::unordered_set available_controllers_; std::unordered_set enabled_controllers_; bool operator==(const FakeCgroup &other) const { @@ -39,31 +39,83 @@ struct FakeCgroup { enabled_controllers_ == other.enabled_controllers_; } }; + +struct FakeConstraint { + std::string cgroup_; + std::string name_; +}; + +struct FakeController { + std::string cgroup_; + std::string name_; +}; + +struct FakeMoveProcesses { + std::string from_; + std::string to_; +}; + +// Intended to be used only in unit tests. This class is not thread-safe. class FakeCgroupDriver : public CgroupDriverInterface { public: - explicit FakeCgroupDriver( - std::shared_ptr> cgroups) - : cgroups_(cgroups) {} - - explicit FakeCgroupDriver(std::string base_cgroup) - : cgroups_(std::make_shared>()) { - RAY_LOG(INFO) << "FakeCgroupDriver(std::string base_cgroup)"; - cgroups_->emplace(base_cgroup, FakeCgroup{base_cgroup}); - } - FakeCgroupDriver(std::string base_cgroup, - std::vector processes_in_base_cgroup, - std::unordered_set available_controllers) - : cgroups_(std::make_shared>()) { - cgroups_->emplace(base_cgroup, - FakeCgroup{base_cgroup, - std::move(processes_in_base_cgroup), - {}, - std::move(available_controllers), - {}}); + static std::unique_ptr Create( + std::shared_ptr> cgroups = nullptr, + std::shared_ptr>> deleted_cgroups = nullptr, + std::shared_ptr>> constraints_disabled = + nullptr, + std::shared_ptr>> controllers_disabled = + nullptr, + std::shared_ptr>> processes_moved = + nullptr) { + if (!cgroups) { + cgroups = std::make_shared>(); + } + if (!deleted_cgroups) { + deleted_cgroups = std::make_shared>>(); + } + if (!constraints_disabled) { + constraints_disabled = + std::make_shared>>(); + } + if (!controllers_disabled) { + controllers_disabled = + std::make_shared>>(); + } + if (!processes_moved) { + processes_moved = + std::make_shared>>(); + } + return std::unique_ptr(new FakeCgroupDriver(cgroups, + deleted_cgroups, + constraints_disabled, + controllers_disabled, + processes_moved)); } + FakeCgroupDriver( + std::shared_ptr> cgroups, + std::shared_ptr>> deleted_cgroups, + std::shared_ptr>> constraints_disabled, + std::shared_ptr>> controllers_disabled, + std::shared_ptr>> processes_moved) + : cgroups_(cgroups), + deleted_cgroups_(deleted_cgroups), + constraints_disabled_(constraints_disabled), + controllers_disabled_(controllers_disabled), + processes_moved_(processes_moved) {} + std::shared_ptr> cgroups_; + // Cgroup cleanup order can be recorded by setting cleanup_mode_ to true. + bool cleanup_mode_ = false; + // cleanup_counter_ is incremented with each cleanup operation to capture + // the order of operations. + int cleanup_counter_ = 0; + std::shared_ptr>> deleted_cgroups_; + std::shared_ptr>> constraints_disabled_; + std::shared_ptr>> controllers_disabled_; + std::shared_ptr>> processes_moved_; + Status check_cgroup_enabled_s_ = Status::OK(); Status check_cgroup_s_ = Status::OK(); Status create_cgroup_s_ = Status::OK(); @@ -95,6 +147,9 @@ class FakeCgroupDriver : public CgroupDriverInterface { return delete_cgroup_s_; } cgroups_->erase(cgroup); + if (cleanup_mode_) { + deleted_cgroups_->emplace_back(std::make_pair(++cleanup_counter_, cgroup)); + } return delete_cgroup_s_; } @@ -108,6 +163,10 @@ class FakeCgroupDriver : public CgroupDriverInterface { to_cgroup.processes_.emplace_back(from_cgroup.processes_.back()); from_cgroup.processes_.pop_back(); } + if (cleanup_mode_) { + processes_moved_->emplace_back( + std::make_pair(++cleanup_counter_, FakeMoveProcesses{from, to})); + } return move_all_processes_s_; } @@ -125,6 +184,10 @@ class FakeCgroupDriver : public CgroupDriverInterface { if (!disable_controller_s_.ok()) { return disable_controller_s_; } + if (cleanup_mode_) { + controllers_disabled_->emplace_back( + std::make_pair(++cleanup_counter_, FakeController{cgroup, controller})); + } (*cgroups_)[cgroup].enabled_controllers_.erase(controller); return disable_controller_s_; } @@ -136,7 +199,11 @@ class FakeCgroupDriver : public CgroupDriverInterface { if (!add_constraint_s_.ok()) { return add_constraint_s_; } - (*cgroups_)[cgroup].constraints_.emplace_back(constraint, value); + (*cgroups_)[cgroup].constraints_.emplace(constraint, value); + if (cleanup_mode_) { + constraints_disabled_->emplace_back( + std::make_pair(++cleanup_counter_, FakeConstraint{cgroup, constraint})); + } return add_constraint_s_; } diff --git a/src/ray/common/cgroup2/noop_cgroup_manager.cc b/src/ray/common/cgroup2/noop_cgroup_manager.cc new file mode 100644 index 000000000000..1accae4827df --- /dev/null +++ b/src/ray/common/cgroup2/noop_cgroup_manager.cc @@ -0,0 +1,39 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include +#include + +#include "ray/common/cgroup2/cgroup_driver_interface.h" +#include "ray/common/cgroup2/cgroup_manager.h" +#include "ray/common/status_or.h" + +namespace ray { + +CgroupManager::CgroupManager(std::string base_cgroup_path, + const std::string &node_id, + std::unique_ptr cgroup_driver) {} + +CgroupManager::~CgroupManager() {} + +StatusOr> CgroupManager::Create( + std::string base_cgroup_path, + const std::string &node_id, + const int64_t system_reserved_cpu_weight, + const int64_t system_reserved_memory_bytes, + std::unique_ptr cgroup_driver) { + return std::unique_ptr( + new CgroupManager(base_cgroup_path, node_id, std::move(cgroup_driver))); +} +} // namespace ray diff --git a/src/ray/common/cgroup2/tests/cgroup_manager_test.cc b/src/ray/common/cgroup2/tests/cgroup_manager_test.cc index 1c7f5e154013..aa83dfa64828 100644 --- a/src/ray/common/cgroup2/tests/cgroup_manager_test.cc +++ b/src/ray/common/cgroup2/tests/cgroup_manager_test.cc @@ -29,14 +29,12 @@ TEST(CgroupManagerTest, CreateReturnsInvalidIfCgroupv2NotAvailable) { std::make_shared>(); cgroups->emplace("/sys/fs/cgroup", FakeCgroup{"/sys/fs/cgroup"}); FakeCgroup base_cgroup{"/sys/fs/cgroup"}; - FakeCgroupDriver *driver = new FakeCgroupDriver(cgroups); + + std::unique_ptr driver = FakeCgroupDriver::Create(cgroups); + driver->check_cgroup_enabled_s_ = Status::Invalid(""); - auto cgroup_manager_s = - CgroupManager::Create("/sys/fs/cgroup/ray", - "node_id_123", - 100, - 1000000, - std::unique_ptr(driver)); + auto cgroup_manager_s = CgroupManager::Create( + "/sys/fs/cgroup/ray", "node_id_123", 100, 1000000, std::move(driver)); ASSERT_TRUE(cgroup_manager_s.IsInvalid()) << cgroup_manager_s.ToString(); // No visible side-effects ASSERT_EQ(cgroups->size(), 1); @@ -46,14 +44,10 @@ TEST(CgroupManagerTest, CreateReturnsInvalidIfCgroupv2NotAvailable) { TEST(CgroupManagerTest, CreateReturnsNotFoundIfBaseCgroupDoesNotExist) { std::shared_ptr> cgroups = std::make_shared>(); - FakeCgroupDriver *driver = new FakeCgroupDriver(cgroups); + std::unique_ptr driver = FakeCgroupDriver::Create(cgroups); driver->check_cgroup_s_ = Status::NotFound(""); - auto cgroup_manager_s = - CgroupManager::Create("/sys/fs/cgroup/ray", - "node_id_123", - 100, - 1000000, - std::unique_ptr(driver)); + auto cgroup_manager_s = CgroupManager::Create( + "/sys/fs/cgroup/ray", "node_id_123", 100, 1000000, std::move(driver)); ASSERT_TRUE(cgroup_manager_s.IsNotFound()) << cgroup_manager_s.ToString(); // No visible side-effects ASSERT_EQ(cgroups->size(), 0); @@ -65,14 +59,10 @@ TEST(CgroupManagerTest, std::make_shared>(); cgroups->emplace("/sys/fs/cgroup", FakeCgroup{"/sys/fs/cgroup"}); FakeCgroup base_cgroup{"/sys/fs/cgroup"}; - FakeCgroupDriver *driver = new FakeCgroupDriver(cgroups); + std::unique_ptr driver = FakeCgroupDriver::Create(cgroups); driver->check_cgroup_s_ = Status::PermissionDenied(""); - auto cgroup_manager_s = - CgroupManager::Create("/sys/fs/cgroup/ray", - "node_id_123", - 100, - 1000000, - std::unique_ptr(driver)); + auto cgroup_manager_s = CgroupManager::Create( + "/sys/fs/cgroup/ray", "node_id_123", 100, 1000000, std::move(driver)); ASSERT_TRUE(cgroup_manager_s.IsPermissionDenied()) << cgroup_manager_s.ToString(); // No visible side-effects ASSERT_EQ(cgroups->size(), 1); @@ -82,20 +72,205 @@ TEST(CgroupManagerTest, TEST(CgroupManagerTest, CreateReturnsInvalidIfSupportedControllersAreNotAvailable) { std::shared_ptr> cgroups = std::make_shared>(); - // By default no controllers are available. cgroups->emplace("/sys/fs/cgroup", FakeCgroup{"/sys/fs/cgroup"}); FakeCgroup base_cgroup{"/sys/fs/cgroup"}; - FakeCgroupDriver *driver = new FakeCgroupDriver(cgroups); - auto cgroup_manager_s = - CgroupManager::Create("/sys/fs/cgroup", - "node_id_123", - 100, - 1000000, - std::unique_ptr(driver)); + std::unique_ptr driver = FakeCgroupDriver::Create(cgroups); + auto cgroup_manager_s = CgroupManager::Create( + "/sys/fs/cgroup", "node_id_123", 100, 1000000, std::move(driver)); ASSERT_TRUE(cgroup_manager_s.IsInvalid()) << cgroup_manager_s.ToString(); // No visible side-effects ASSERT_EQ(cgroups->size(), 1); ASSERT_EQ(cgroups->begin()->second, base_cgroup); } +TEST(CgroupManagerTest, CreateReturnsInvalidArgumentIfConstraintValuesOutOfBounds) { + std::shared_ptr> cgroups = + std::make_shared>(); + cgroups->emplace("/sys/fs/cgroup", FakeCgroup{"/sys/fs/cgroup"}); + FakeCgroup base_cgroup{"/sys/fs/cgroup"}; + std::unique_ptr driver = FakeCgroupDriver::Create(cgroups); + auto cgroup_manager_s = + CgroupManager::Create("/sys/fs/cgroup", "node_id_123", -1, -1, std::move(driver)); + ASSERT_TRUE(cgroup_manager_s.IsInvalidArgument()) << cgroup_manager_s.ToString(); + // No visible side-effects + ASSERT_EQ(cgroups->size(), 1); + ASSERT_EQ(cgroups->begin()->second, base_cgroup); +} + +TEST(CgroupManagerTest, CreateSucceedsWithCleanupInOrder) { + std::shared_ptr> cgroups = + std::make_shared>(); + + cgroups->emplace("/sys/fs/cgroup", + FakeCgroup{"/sys/fs/cgroup", {5}, {}, {"cpu", "memory"}, {}}); + + auto deleted_cgroups = std::make_shared>>(); + auto constraints_disabled = + std::make_shared>>(); + auto controllers_disabled = + std::make_shared>>(); + auto processes_moved = + std::make_shared>>(); + + std::unique_ptr owned_driver = + FakeCgroupDriver::Create(cgroups, + deleted_cgroups, + constraints_disabled, + controllers_disabled, + processes_moved); + + FakeCgroupDriver *driver = owned_driver.get(); + + // node, system, and application cgroups were created in the fake + std::string node_id = "id_123"; + std::string base_cgroup_path = "/sys/fs/cgroup"; + std::string node_cgroup_path = "/sys/fs/cgroup/ray_node_id_123"; + std::string system_cgroup_path = "/sys/fs/cgroup/ray_node_id_123/system"; + std::string application_cgroup_path = "/sys/fs/cgroup/ray_node_id_123/application"; + int64_t system_reserved_cpu_weight = 1000; + int64_t system_reserved_memory_bytes = 1024 * 1024 * 1024; + + auto cgroup_manager_s = CgroupManager::Create(base_cgroup_path, + node_id, + system_reserved_cpu_weight, + system_reserved_memory_bytes, + std::move(owned_driver)); + + // The cgroup hierarchy was created correctly. + ASSERT_EQ(cgroups->size(), 4); + ASSERT_NE(cgroups->find(base_cgroup_path), cgroups->end()); + ASSERT_NE(cgroups->find(node_cgroup_path), cgroups->end()); + ASSERT_NE(cgroups->find(system_cgroup_path), cgroups->end()); + ASSERT_NE(cgroups->find(application_cgroup_path), cgroups->end()); + + std::array created_cgroups{&cgroups->at(base_cgroup_path), + &cgroups->at(node_cgroup_path), + &cgroups->at(system_cgroup_path), + &cgroups->at(application_cgroup_path)}; + + // Controllers are enabled on base, node, application, and system cgroups. + for (const FakeCgroup *cg : created_cgroups) { + ASSERT_EQ(cg->enabled_controllers_.size(), 2); + ASSERT_NE(cg->enabled_controllers_.find("cpu"), cg->enabled_controllers_.end()); + ASSERT_NE(cg->enabled_controllers_.find("memory"), cg->enabled_controllers_.end()); + } + + // Processes were moved out of the base cgroup into the system cgroup. + const FakeCgroup &base_cgroup = cgroups->find(base_cgroup_path)->second; + const FakeCgroup &system_cgroup = cgroups->find(system_cgroup_path)->second; + ASSERT_TRUE(base_cgroup.processes_.empty()); + ASSERT_EQ(system_cgroup.processes_.size(), 1); + + // Check to see that the memory and cpu constraints were enabled correctly + // for the system and application cgroups. + ASSERT_EQ(system_cgroup.constraints_.size(), 2); + ASSERT_NE(system_cgroup.constraints_.find("cpu.weight"), + system_cgroup.constraints_.end()); + ASSERT_EQ(system_cgroup.constraints_.at("cpu.weight"), + std::to_string(system_reserved_cpu_weight)); + ASSERT_EQ(system_cgroup.constraints_.at("memory.min"), + std::to_string(system_reserved_memory_bytes)); + + const FakeCgroup &app_cgroup = cgroups->find(application_cgroup_path)->second; + ASSERT_EQ(app_cgroup.constraints_.size(), 1); + ASSERT_NE(app_cgroup.constraints_.find("cpu.weight"), app_cgroup.constraints_.end()); + ASSERT_EQ(app_cgroup.constraints_.at("cpu.weight"), + std::to_string(10000 - system_reserved_cpu_weight)); + + // Switching the mode of the FakeCgroupDriver to cleanup to record cleanup + // operations + driver->cleanup_mode_ = true; + // Destroying the cgroup manager triggers automatic cleanup. + std::unique_ptr cgroup_manager = std::move(cgroup_manager_s.value()); + cgroup_manager.reset(); + + // Only the base cgroup is left after the cgroup_manager is destroyed. + ASSERT_EQ(cgroups->size(), 1); + ASSERT_NE(cgroups->find(base_cgroup_path), cgroups->end()); + + // Since the order of operation matters during cleanup for cgroups, we're going + // to have to check the fake for side-effects extensively: + // + // Constraints have to be disabled before controllers are disabled. + ASSERT_EQ(constraints_disabled->size(), 3); + // Since constraints were only enabled on leaf nodes, the order does not matter. + ASSERT_EQ( + std::count_if(constraints_disabled->begin(), + constraints_disabled->end(), + [&system_cgroup_path](const std::pair &item) { + return item.second.cgroup_ == system_cgroup_path && + item.second.name_ == "cpu.weight"; + }), + 1); + ASSERT_EQ( + std::count_if(constraints_disabled->begin(), + constraints_disabled->end(), + [&system_cgroup_path](const std::pair &item) { + return item.second.cgroup_ == system_cgroup_path && + item.second.name_ == "memory.min"; + }), + 1); + ASSERT_EQ(std::count_if( + constraints_disabled->begin(), + constraints_disabled->end(), + [&application_cgroup_path](const std::pair &item) { + return item.second.cgroup_ == application_cgroup_path && + item.second.name_ == "cpu.weight"; + }), + 1); + + // Controllers were disabled second. + ASSERT_EQ(controllers_disabled->size(), 8); + // Controllers must be disabled after the constraints are removed. + ASSERT_LT(constraints_disabled->back().first, controllers_disabled->front().first); + // Check to see controllers are disabled on all cgroups from the leaves to + // the root. + ASSERT_EQ((*controllers_disabled)[0].second.cgroup_, application_cgroup_path); + ASSERT_EQ((*controllers_disabled)[1].second.cgroup_, system_cgroup_path); + ASSERT_EQ((*controllers_disabled)[2].second.cgroup_, node_cgroup_path); + ASSERT_EQ((*controllers_disabled)[3].second.cgroup_, base_cgroup_path); + ASSERT_EQ((*controllers_disabled)[4].second.cgroup_, application_cgroup_path); + ASSERT_EQ((*controllers_disabled)[5].second.cgroup_, system_cgroup_path); + ASSERT_EQ((*controllers_disabled)[6].second.cgroup_, node_cgroup_path); + ASSERT_EQ((*controllers_disabled)[7].second.cgroup_, base_cgroup_path); + + // The memory and cpu controller are both disabled for each cgroup + std::array cgroup_names{ + base_cgroup_path, + node_cgroup_path, + system_cgroup_path, + application_cgroup_path, + }; + + for (const auto &cgroup_name : cgroup_names) { + ASSERT_EQ(std::count_if(controllers_disabled->begin(), + controllers_disabled->end(), + [&cgroup_name](const std::pair &item) { + return item.second.cgroup_ == cgroup_name && + item.second.name_ == "cpu"; + }), + 1); + ASSERT_EQ(std::count_if(controllers_disabled->begin(), + controllers_disabled->end(), + [&cgroup_name](const std::pair &item) { + return item.second.cgroup_ == cgroup_name && + item.second.name_ == "memory"; + }), + 1); + } + + // Processes were moved third. + ASSERT_EQ(processes_moved->size(), 1); + ASSERT_EQ((*processes_moved)[0].second.from_, system_cgroup_path); + ASSERT_EQ((*processes_moved)[0].second.to_, base_cgroup_path); + ASSERT_LT(constraints_disabled->back().first, processes_moved->front().first); + + // Cgroups were deleted last and in reverse order i.e. application, system, node. + ASSERT_EQ(deleted_cgroups->size(), 3); + ASSERT_LT(processes_moved->back().first, deleted_cgroups->front().first); + ASSERT_EQ((*deleted_cgroups)[0].second, application_cgroup_path); + ASSERT_EQ((*deleted_cgroups)[1].second, system_cgroup_path); + ASSERT_EQ((*deleted_cgroups)[2].second, node_cgroup_path); +} + } // namespace ray diff --git a/src/ray/core_worker/core_worker_options.h b/src/ray/core_worker/core_worker_options.h index c91cc81a3a2e..3f9b72f48b7d 100644 --- a/src/ray/core_worker/core_worker_options.h +++ b/src/ray/core_worker/core_worker_options.h @@ -106,8 +106,7 @@ struct CoreWorkerOptions { entrypoint(""), worker_launch_time_ms(-1), worker_launched_time_ms(-1), - debug_source(""), - enable_resource_isolation(false) {} + debug_source("") {} /// Type of this worker (i.e., DRIVER or WORKER). WorkerType worker_type; @@ -211,10 +210,6 @@ struct CoreWorkerOptions { // Source information for `CoreWorker`, used for debugging and informational purpose, // rather than functional purpose. std::string debug_source; - - // If true, core worker enables resource isolation through cgroupv2 by reserving - // resources for ray system processes. - bool enable_resource_isolation = false; }; } // namespace core } // namespace ray diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index b8c751b578b6..ab7db114366c 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -144,13 +144,6 @@ std::shared_ptr CoreWorkerProcessImpl::CreateCoreWorker( options.worker_type, worker_id, GetProcessJobID(options)); auto pid = getpid(); - // Move worker process into cgroup on startup. - AppProcCgroupMetadata app_cgroup_metadata; - app_cgroup_metadata.pid = pid; - app_cgroup_metadata.max_memory = kUnlimitedCgroupMemory; - GetCgroupSetup(options.enable_resource_isolation) - .ApplyCgroupContext(app_cgroup_metadata); - RAY_LOG(DEBUG) << "Creating core worker with debug source: " << options.debug_source; RAY_LOG(DEBUG).WithField(worker_id) << "Constructing CoreWorker"; diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index 0625c5fba1d9..c617eb84fc47 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -283,7 +283,8 @@ ray_cc_binary( "//src/ray/common:lease", "//src/ray/common:ray_config", "//src/ray/common:status", - "//src/ray/common/cgroup:cgroup_manager", + "//src/ray/common/cgroup2:cgroup_manager", + "//src/ray/common/cgroup2:sysfs_cgroup_driver", "//src/ray/core_worker:metrics", "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/object_manager:ownership_object_directory", diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index a440a64e6e79..5cb7cd35975a 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -24,12 +24,14 @@ #include "gflags/gflags.h" #include "nlohmann/json.hpp" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/cgroup/cgroup_manager.h" +#include "ray/common/cgroup2/cgroup_manager.h" +#include "ray/common/cgroup2/sysfs_cgroup_driver.h" #include "ray/common/constants.h" #include "ray/common/id.h" #include "ray/common/lease/lease.h" #include "ray/common/ray_config.h" #include "ray/common/status.h" +#include "ray/common/status_or.h" #include "ray/core_worker/metrics.h" #include "ray/gcs/gcs_client/gcs_client.h" #include "ray/object_manager/ownership_object_directory.h" @@ -98,12 +100,6 @@ DEFINE_int64(object_store_memory, -1, "The initial memory of the object store.") DEFINE_string(node_name, "", "The user-provided identifier or name for this node."); DEFINE_string(session_name, "", "The current Ray session name."); DEFINE_string(cluster_id, "", "ID of the cluster, separate from observability."); -// TODO(hjiang): At the moment only enablement flag is added, I will add other flags for -// CPU and memory resource reservation in the followup PR. -DEFINE_bool(enable_resource_isolation, - false, - "Enable resource isolation through cgroupv2 by reserving resources for ray " - "system processes."); #ifdef __linux__ DEFINE_string(plasma_directory, @@ -119,6 +115,30 @@ DEFINE_bool(huge_pages, false, "Enable huge pages."); DEFINE_string(labels, "", "Define the key-value format of node labels, which is a serialized JSON."); +DEFINE_bool( + enable_resource_isolation, + false, + "Enables resource isolation through cgroupv2. The raylet will create and " + "manage a cgroup hierarchy that separates system processes and worker processes " + "into separate cgroups."); +DEFINE_string( + cgroup_path, + "", + "Path of the cgroup that the raylet will take ownership of to create its cgorup " + "hierarchy. The raylet process must have read, write, and execute permission for " + "this path. If enable_resource_isolation is true, then this cannot be empty."); +DEFINE_int64( + system_reserved_cpu_weight, + -1, + "The amount of cores reserved for ray system processes. It will be applied " + "as a cpu.weight constraint to the system cgroup. 10000 - " + "system_reserved_cpu_weight will be applied as a constraint to the " + "application cgroup. If enable resource isolation is true, then this cannot be -1."); +DEFINE_int64(system_reserved_memory_bytes, + -1, + "The amount of memory in bytes reserved for ray system processes. It will " + "be applied as a memory.min constraint to the sytem cgroup. If enable " + "resource isolation is true, then this cannot be -1"); absl::flat_hash_map parse_node_labels( const std::string &labels_json_str) { @@ -226,18 +246,50 @@ int main(int argc, char *argv[]) { const std::string session_name = FLAGS_session_name; const bool is_head_node = FLAGS_head; const std::string labels_json_str = FLAGS_labels; + const bool enable_resource_isolation = FLAGS_enable_resource_isolation; + const std::string cgroup_path = FLAGS_cgroup_path; + const int64_t system_reserved_cpu_weight = FLAGS_system_reserved_cpu_weight; + const int64_t system_reserved_memory_bytes = FLAGS_system_reserved_memory_bytes; RAY_CHECK_NE(FLAGS_cluster_id, "") << "Expected cluster ID."; ray::ClusterID cluster_id = ray::ClusterID::FromHex(FLAGS_cluster_id); RAY_LOG(INFO) << "Setting cluster ID to: " << cluster_id; gflags::ShutDownCommandLineFlags(); - // Get cgroup setup instance and perform necessary resource setup. - ray::GetCgroupSetup(FLAGS_enable_resource_isolation); + // TODO(#54703): Link OSS documentation once it's available in the error messages. + if (enable_resource_isolation) { + RAY_CHECK(!cgroup_path.empty()) + << "Failed to start up raylet. If enable_resource_isolation is set to true, " + "cgroup_path cannot be empty."; + RAY_CHECK_NE(system_reserved_cpu_weight, -1) + << "Failed to start up raylet. If enable_resource_isolation is set to true, " + "system_reserved_cpu_weight must be set to a value between [1,10000]"; + RAY_CHECK_NE(system_reserved_memory_bytes, -1) + << "Failed to start up raylet. If enable_resource_isolation is set to true, " + "system_reserved_memory_byres must be set to a value > 0"; + + std::unique_ptr cgroup_driver; + ray::StatusOr> cgroup_manager = + ray::CgroupManager::Create(std::move(cgroup_path), + node_id, + system_reserved_cpu_weight, + system_reserved_memory_bytes, + std::move(cgroup_driver)); + + // TODO(#54703) - Link to OSS documentation once available. + RAY_CHECK(cgroup_manager.ok()) + << "Failed to start raylet. Could not create CgroupManager because of " + << cgroup_manager.ToString(); + +#ifndef __linux__ + RAY_LOG(WARNING) + << "Resource isolation with cgroups is only supported in linux. Please set " + "enable_resource_isolation to false. This is likely a misconfiguration."; +#endif + } // Configuration for the node manager. ray::raylet::NodeManagerConfig node_manager_config; - node_manager_config.enable_resource_isolation = FLAGS_enable_resource_isolation; absl::flat_hash_map static_resource_conf; @@ -561,8 +613,7 @@ int main(int argc, char *argv[]) { /*starting_worker_timeout_callback=*/ [&] { cluster_lease_manager->ScheduleAndGrantLeases(); }, node_manager_config.ray_debugger_external, - /*get_time=*/[]() { return absl::Now(); }, - node_manager_config.enable_resource_isolation); + /*get_time=*/[]() { return absl::Now(); }); client_call_manager = std::make_unique( main_service, /*record_stats=*/true); diff --git a/src/ray/raylet/node_manager.h b/src/ray/raylet/node_manager.h index da0b2182d200..38665194bd8b 100644 --- a/src/ray/raylet/node_manager.h +++ b/src/ray/raylet/node_manager.h @@ -115,9 +115,6 @@ struct NodeManagerConfig { int max_io_workers; // The key-value labels of this node. absl::flat_hash_map labels; - // If true, core worker enables resource isolation by adding itself into appropriate - // cgroup. - bool enable_resource_isolation = false; }; class NodeManager : public rpc::NodeManagerServiceHandler, diff --git a/src/ray/raylet/tests/worker_pool_test.cc b/src/ray/raylet/tests/worker_pool_test.cc index 41c275d8ba5f..a75cee7603ca 100644 --- a/src/ray/raylet/tests/worker_pool_test.cc +++ b/src/ray/raylet/tests/worker_pool_test.cc @@ -153,8 +153,7 @@ class WorkerPoolMock : public WorkerPool { "", []() {}, 0, - [this]() { return absl::FromUnixMillis(current_time_ms_); }, - /*enable_resource_isolation=*/false), + [this]() { return absl::FromUnixMillis(current_time_ms_); }), last_worker_process_(), instrumented_io_service_(io_service), client_call_manager_(instrumented_io_service_, false), diff --git a/src/ray/raylet/worker_pool.cc b/src/ray/raylet/worker_pool.cc index f20274fbb5b8..32c6d7530e45 100644 --- a/src/ray/raylet/worker_pool.cc +++ b/src/ray/raylet/worker_pool.cc @@ -101,8 +101,7 @@ WorkerPool::WorkerPool(instrumented_io_context &io_service, std::string native_library_path, std::function starting_worker_timeout_callback, int ray_debugger_external, - std::function get_time, - bool enable_resource_isolation) + std::function get_time) : worker_startup_token_counter_(0), io_service_(&io_service), node_id_(node_id), @@ -123,8 +122,7 @@ WorkerPool::WorkerPool(instrumented_io_context &io_service, std::min(num_prestarted_python_workers, maximum_startup_concurrency_)), num_prestart_python_workers(num_prestarted_python_workers), periodical_runner_(PeriodicalRunner::Create(io_service)), - get_time_(std::move(get_time)), - enable_resource_isolation_(enable_resource_isolation) { + get_time_(std::move(get_time)) { RAY_CHECK_GT(maximum_startup_concurrency_, 0); // We need to record so that the metric exists. This way, we report that 0 // processes have started before a task runs on the node (as opposed to the @@ -443,12 +441,6 @@ WorkerPool::BuildProcessCommandArgs(const Language &language, serialized_preload_python_modules); } - // Pass resource isolation flag to python worker. - if (language == Language::PYTHON && worker_type == rpc::WorkerType::WORKER) { - worker_command_args.emplace_back(absl::StrFormat( - "--enable-resource-isolation=%s", enable_resource_isolation_ ? "true" : "false")); - } - // We use setproctitle to change python worker process title, // causing the process's /proc/PID/environ being empty. // Add `SPT_NOENV` env to prevent setproctitle breaking /proc/PID/environ. diff --git a/src/ray/raylet/worker_pool.h b/src/ray/raylet/worker_pool.h index 30cdb2dc82d7..1d6e43a23497 100644 --- a/src/ray/raylet/worker_pool.h +++ b/src/ray/raylet/worker_pool.h @@ -304,7 +304,6 @@ class WorkerPool : public WorkerPoolInterface { /// \param ray_debugger_external Ray debugger in workers will be started in a way /// that they are accessible from outside the node. /// \param get_time A callback to get the current time in milliseconds. - /// \param enable_resource_isolation If true, core worker enables resource isolation by /// adding itself into appropriate cgroup. WorkerPool(instrumented_io_context &io_service, const NodeID &node_id, @@ -320,8 +319,7 @@ class WorkerPool : public WorkerPoolInterface { std::string native_library_path, std::function starting_worker_timeout_callback, int ray_debugger_external, - std::function get_time, - bool enable_resource_isolation); + std::function get_time); /// Destructor responsible for freeing a set of workers owned by this class. ~WorkerPool() override; @@ -912,10 +910,6 @@ class WorkerPool : public WorkerPoolInterface { int64_t process_failed_pending_registration_ = 0; int64_t process_failed_runtime_env_setup_failed_ = 0; - // If true, core worker enables resource isolation by adding itself into appropriate - // cgroup after it is created. - bool enable_resource_isolation_ = false; - /// Ray metrics ray::stats::Sum ray_metric_num_workers_started_{ /*name=*/"internal_num_processes_started", From 4c0ec7f284be9ed1ef1fd3fe67e2f8c8a1829002 Mon Sep 17 00:00:00 2001 From: Lehui Liu Date: Thu, 11 Sep 2025 15:38:40 -0700 Subject: [PATCH 585/634] [release] Change back to g4dn.12x for 4 GPU (#56469) 1. previously when I cleaning up the legacy g3 test, accidentally changed it https://github.com/ray-project/ray/pull/56175 to became g4dn.8xlarge machine which only has 1 GPU per host, this is to change it back. ## Why are these changes needed? ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [x] Release tests - [ ] This PR is not tested :( Signed-off-by: Lehui Liu --- release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml b/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml index 897fb4ed728e..e56edf8bbf28 100644 --- a/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml +++ b/release/air_tests/air_benchmarks/compute_gpu_4x4_aws.yaml @@ -5,11 +5,11 @@ max_workers: 3 head_node_type: name: head_node - instance_type: g4dn.8xlarge + instance_type: g4dn.12xlarge worker_node_types: - name: worker_node - instance_type: g4dn.8xlarge + instance_type: g4dn.12xlarge max_workers: 3 min_workers: 3 use_spot: false From a9a57a614ab1f9ea51c3a8d710d0491c9364ccca Mon Sep 17 00:00:00 2001 From: Matthew Owen Date: Thu, 11 Sep 2025 16:21:23 -0700 Subject: [PATCH 586/634] [data] Download op fusion / removal of interleaved partitioners (#56462) ## Why are these changes needed? If we have multiple chained downloads e.g. `ds.with_column("bytes_1", download("uri_1")).with_column("bytes_2", download("uri_2")).with_column("bytes_3", download("uri_3"))`, then we would have an operator structure like `URIPartitioner->URIDownloader->URIPartitioner->URIDownloader->URIPartitioner->URIDownloader`. Each of the `URIPartitioner` operators will be implemented with an ActorPoolMapOperator with concurrency of 1. In these chained downloads, these become bottlenecks and scaling the concurrency of these up will result in additional resource usage that will take resources away from other operators. This solves the problem by deferring some of the partitioning to the `URIDownloader` so we can remove the interleaved partitioners. The result is an operator structure like `URIPartitioner->URIDownloader->URIDownloader->URIDownloader` which delivers much better performance for these cases. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Matthew Owen --- .../_internal/planner/plan_download_op.py | 100 +++++++++++------- .../data/tests/test_download_expression.py | 50 +++++++++ 2 files changed, 114 insertions(+), 36 deletions(-) diff --git a/python/ray/data/_internal/planner/plan_download_op.py b/python/ray/data/_internal/planner/plan_download_op.py index 2e2756b6a886..279e90ab9470 100644 --- a/python/ray/data/_internal/planner/plan_download_op.py +++ b/python/ray/data/_internal/planner/plan_download_op.py @@ -1,7 +1,7 @@ import logging import math from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import List +from typing import Iterator, List from urllib.parse import urlparse import pyarrow as pa @@ -33,6 +33,13 @@ def plan_download_op( """Plan the download operation with partitioning and downloading stages.""" assert len(physical_children) == 1 input_physical_dag = physical_children[0] + + upstream_op_is_download = False + if len(input_physical_dag._logical_operators) == 1 and isinstance( + input_physical_dag._logical_operators[0], Download + ): + upstream_op_is_download = True + uri_column_name = op.uri_column_name output_bytes_column_name = op.output_bytes_column_name ray_remote_args = op.ray_remote_args @@ -43,23 +50,33 @@ def plan_download_op( _get_udf, ) - # PartitionActor is a callable class, so we need ActorPoolStrategy - partition_compute = ActorPoolStrategy(size=1) # Use single actor for partitioning - - fn, init_fn = _get_udf(PartitionActor, (), {}, (uri_column_name, data_context), {}) - block_fn = _generate_transform_fn_for_map_batches(fn) - partition_transform_fns = [ - BlockMapTransformFn(block_fn), - ] - partition_map_transformer = MapTransformer(partition_transform_fns, init_fn) - partition_map_operator = MapOperator.create( - partition_map_transformer, - input_physical_dag, - data_context, - name="URIPartitioner", - compute_strategy=partition_compute, # Use actor-based compute for callable class - ray_remote_args=ray_remote_args, - ) + # If we have multiple download operators in a row, we should only include the partition actor + # at the start of the chain. This is primarily done to prevent partition actors from bottlenecking + # the chain becuase the interleaved operators would be a single actor. As a result, the + # URIDownloader physical operator is responsible for outputting appropriately sized blocks. + partition_map_operator = None + if not upstream_op_is_download: + # PartitionActor is a callable class, so we need ActorPoolStrategy + partition_compute = ActorPoolStrategy( + size=1 + ) # Use single actor for partitioning + + fn, init_fn = _get_udf( + PartitionActor, (), {}, (uri_column_name, data_context), {} + ) + block_fn = _generate_transform_fn_for_map_batches(fn) + partition_transform_fns = [ + BlockMapTransformFn(block_fn), + ] + partition_map_transformer = MapTransformer(partition_transform_fns, init_fn) + partition_map_operator = MapOperator.create( + partition_map_transformer, + input_physical_dag, + data_context, + name="URIPartitioner", + compute_strategy=partition_compute, # Use actor-based compute for callable class + ray_remote_args=ray_remote_args, + ) fn, init_fn = _get_udf( download_bytes_threaded, @@ -76,7 +93,7 @@ def plan_download_op( download_compute = TaskPoolStrategy() download_map_operator = MapOperator.create( download_map_transformer, - partition_map_operator, + partition_map_operator if partition_map_operator else input_physical_dag, data_context, name="URIDownloader", compute_strategy=download_compute, @@ -95,12 +112,22 @@ def uri_to_path(uri: str) -> str: return parsed.netloc + parsed.path +def _arrow_batcher(table: pa.Table, output_batch_size: int): + """Batch a PyArrow table into smaller tables of size n using zero-copy slicing.""" + num_rows = table.num_rows + for i in range(0, num_rows, output_batch_size): + end_idx = min(i + output_batch_size, num_rows) + # Use PyArrow's zero-copy slice operation + batch_table = table.slice(i, end_idx - i) + yield batch_table + + def download_bytes_threaded( - block, - uri_column_name, + block: pa.Table, + uri_column_name: str, output_bytes_column_name, data_context: DataContext, -): +) -> Iterator[pa.Table]: """Optimized version that uses make_async_gen for concurrent downloads.""" if not isinstance(block, pa.Table): block = BlockAccessor.for_block(block).to_arrow() @@ -109,7 +136,8 @@ def download_bytes_threaded( uris = block.column(uri_column_name).to_pylist() if len(uris) == 0: - return block + yield block + return paths, fs = _resolve_paths_and_filesystem(uris) fs = RetryingPyFileSystem.wrap(fs, retryable_errors=data_context.retried_io_errors) @@ -132,9 +160,18 @@ def load_uri_bytes(uri_path_iterator): ) # Add the new column to the PyArrow table - return block.add_column( + output_block = block.add_column( len(block.column_names), output_bytes_column_name, pa.array(uri_bytes) ) + output_block_size = output_block.nbytes + ctx = ray.data.context.DatasetContext.get_current() + max_bytes = ctx.target_max_block_size + if max_bytes is not None and output_block_size > max_bytes: + num_blocks = math.ceil(output_block_size / max_bytes) + num_rows = output_block.num_rows + yield from _arrow_batcher(output_block, int(math.ceil(num_rows / num_blocks))) + else: + yield output_block class PartitionActor: @@ -147,7 +184,7 @@ def __init__(self, uri_column_name: str, data_context: DataContext): self._data_context = data_context self._batch_size_estimate = None - def _sample_sizes(self, uris): + def _sample_sizes(self, uris: List[str]) -> List[int]: """Fetch file sizes in parallel using ThreadPoolExecutor.""" def get_file_size(uri_path, fs): @@ -185,16 +222,7 @@ def get_file_size(uri_path, fs): return file_sizes - def _arrow_batcher(self, table, n): - """Batch a PyArrow table into smaller tables of size n using zero-copy slicing.""" - num_rows = table.num_rows - for i in range(0, num_rows, n): - end_idx = min(i + n, num_rows) - # Use PyArrow's zero-copy slice operation - batch_table = table.slice(i, end_idx - i) - yield batch_table - - def __call__(self, block): + def __call__(self, block: pa.Table) -> Iterator[pa.Table]: if not isinstance(block, pa.Table): block = BlockAccessor.for_block(block).to_arrow() @@ -216,4 +244,4 @@ def __call__(self, block): max_bytes = ctx.target_max_block_size self._batch_size_estimate = math.floor(max_bytes / file_size_estimate) - yield from self._arrow_batcher(block, self._batch_size_estimate) + yield from _arrow_batcher(block, self._batch_size_estimate) diff --git a/python/ray/data/tests/test_download_expression.py b/python/ray/data/tests/test_download_expression.py index 53aa28318cd4..2c593190de11 100644 --- a/python/ray/data/tests/test_download_expression.py +++ b/python/ray/data/tests/test_download_expression.py @@ -154,6 +154,56 @@ def test_download_expression_with_different_file_types(self, tmp_path): assert downloaded_image.size == (8, 8) assert downloaded_image.mode == "RGB" + def test_chained_download_expressions(self, tmp_path): + """Test chained download expressions functionality.""" + # Create sample files with different content + sample_data = [ + b"Content for file 1", + b"Content for file 2", + b"Content for file 3", + ] + + file_paths = [] + for i, data in enumerate(sample_data): + file_path = tmp_path / f"test_file_{i}.txt" + file_path.write_bytes(data) + file_paths.append(str(file_path)) + + # Create dataset with file URIs + table = pa.Table.from_arrays( + [ + pa.array([f"local://{path}" for path in file_paths]), + pa.array([f"id_{i}" for i in range(len(file_paths))]), + ], + names=["file_uri", "file_id"], + ) + + ds = ray.data.from_arrow(table) + + # Chain multiple download expressions from the same URI column + ds_with_chained_downloads = ( + ds.with_column("file_bytes_1", download("file_uri")) + .with_column("file_bytes_2", download("file_uri")) + .with_column("file_bytes_3", download("file_uri")) + ) + + # Verify results + results = ds_with_chained_downloads.take_all() + assert len(results) == len(sample_data) + + for i, result in enumerate(results): + # All download columns should have the same content + assert "file_bytes_1" in result + assert "file_bytes_2" in result + assert "file_bytes_3" in result + assert result["file_bytes_1"] == sample_data[i] + assert result["file_bytes_2"] == sample_data[i] + assert result["file_bytes_3"] == sample_data[i] + + # Original columns should be preserved + assert result["file_id"] == f"id_{i}" + assert result["file_uri"] == f"local://{file_paths[i]}" + class TestDownloadExpressionErrors: """Test error conditions and edge cases for download expressions.""" From d27f4037b0d6ee0c8bcb3852ac8ea1b579027471 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Thu, 11 Sep 2025 16:55:22 -0700 Subject: [PATCH 587/634] [core] Fix HandleRefRemoved thread safety (#56445) Signed-off-by: dayshah --- src/ray/core_worker/core_worker.cc | 12 ++---- src/ray/core_worker/reference_count.cc | 43 +++++++++++-------- src/ray/core_worker/reference_count.h | 34 +++++++-------- .../core_worker/tests/reference_count_test.cc | 5 +-- 4 files changed, 45 insertions(+), 49 deletions(-) diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index c8922e067f16..eee7cba66534 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -3796,24 +3796,20 @@ void CoreWorker::ProcessSubscribeForRefRemoved( const rpc::WorkerRefRemovedSubMessage &message) { const ObjectID &object_id = ObjectID::FromBinary(message.reference().object_id()); - // Set a callback to publish the message when the requested object ID's ref count - // goes to 0. - auto ref_removed_callback = - boost::bind(&ReferenceCounter::HandleRefRemoved, reference_counter_, object_id); - const auto intended_worker_id = WorkerID::FromBinary(message.intended_worker_id()); if (intended_worker_id != worker_context_->GetWorkerID()) { RAY_LOG(INFO) << "The ProcessSubscribeForRefRemoved message is for worker " << intended_worker_id << ", but the current worker is " << worker_context_->GetWorkerID() << ". The RPC will be no-op."; - ref_removed_callback(object_id); + reference_counter_->PublishRefRemoved(object_id); return; } const auto owner_address = message.reference().owner_address(); ObjectID contained_in_id = ObjectID::FromBinary(message.contained_in_id()); - reference_counter_->SetRefRemovedCallback( - object_id, contained_in_id, owner_address, ref_removed_callback); + // So it will call PublishRefRemovedInternal to publish a message when the requested + // object ID's ref count goes to 0. + reference_counter_->SubscribeRefRemoved(object_id, contained_in_id, owner_address); } void CoreWorker::HandleRemoteCancelTask(rpc::RemoteCancelTaskRequest request, diff --git a/src/ray/core_worker/reference_count.cc b/src/ray/core_worker/reference_count.cc index 11c237904d2b..2f51b14467da 100644 --- a/src/ray/core_worker/reference_count.cc +++ b/src/ray/core_worker/reference_count.cc @@ -558,7 +558,7 @@ int64_t ReferenceCounter::ReleaseLineageReferences(ReferenceTable::iterator ref) OnObjectOutOfScopeOrFreed(arg_it); } if (arg_it->second.ShouldDelete(lineage_pinning_enabled_)) { - RAY_CHECK(arg_it->second.on_ref_removed == nullptr); + RAY_CHECK(!arg_it->second.publish_ref_removed); lineage_bytes_evicted += ReleaseLineageReferences(arg_it); EraseReference(arg_it); } @@ -683,10 +683,10 @@ void ReferenceCounter::DeleteReferenceInternal(ReferenceTable::iterator it, std::vector *deleted) { const ObjectID id = it->first; RAY_LOG(DEBUG) << "Attempting to delete object " << id; - if (it->second.RefCount() == 0 && it->second.on_ref_removed) { - RAY_LOG(DEBUG) << "Calling on_ref_removed for object " << id; - it->second.on_ref_removed(id); - it->second.on_ref_removed = nullptr; + if (it->second.RefCount() == 0 && it->second.publish_ref_removed) { + RAY_LOG(DEBUG) << "Calling PublishRefRemoved for object " << id; + PublishRefRemovedInternal(id); + it->second.publish_ref_removed = false; } PRINT_REF_COUNT(it); @@ -1253,8 +1253,13 @@ void ReferenceCounter::AddNestedObjectIdsInternal(const ObjectID &object_id, } } -void ReferenceCounter::HandleRefRemoved(const ObjectID &object_id) { - RAY_LOG(DEBUG).WithField(object_id) << "HandleRefRemoved "; +void ReferenceCounter::PublishRefRemoved(const ObjectID &object_id) { + absl::MutexLock lock(&mutex_); + PublishRefRemovedInternal(object_id); +} + +void ReferenceCounter::PublishRefRemovedInternal(const ObjectID &object_id) { + RAY_LOG(DEBUG).WithField(object_id) << "PublishRefRemoved "; auto it = object_id_refs_.find(object_id); if (it != object_id_refs_.end()) { PRINT_REF_COUNT(it); @@ -1284,11 +1289,9 @@ void ReferenceCounter::HandleRefRemoved(const ObjectID &object_id) { object_info_publisher_->Publish(std::move(pub_message)); } -void ReferenceCounter::SetRefRemovedCallback( - const ObjectID &object_id, - const ObjectID &contained_in_id, - const rpc::Address &owner_address, - const ReferenceCounter::ReferenceRemovedCallback &ref_removed_callback) { +void ReferenceCounter::SubscribeRefRemoved(const ObjectID &object_id, + const ObjectID &contained_in_id, + const rpc::Address &owner_address) { absl::MutexLock lock(&mutex_); RAY_LOG(DEBUG).WithField(object_id) << "Received WaitForRefRemoved object contained in " << contained_in_id; @@ -1298,6 +1301,8 @@ void ReferenceCounter::SetRefRemovedCallback( it = object_id_refs_.emplace(object_id, Reference()).first; } + auto &reference = it->second; + // If we are borrowing the ID because we own an object that contains it, then // add the outer object to the inner ID's ref count. We will not respond to // the owner of the inner ID until the outer object ID goes out of scope. @@ -1305,28 +1310,28 @@ void ReferenceCounter::SetRefRemovedCallback( AddNestedObjectIdsInternal(contained_in_id, {object_id}, rpc_address_); } - if (it->second.RefCount() == 0) { + if (reference.RefCount() == 0) { RAY_LOG(DEBUG).WithField(object_id) << "Ref count for borrowed object is already 0, responding to WaitForRefRemoved"; // We already stopped borrowing the object ID. Respond to the owner // immediately. - ref_removed_callback(object_id); + PublishRefRemovedInternal(object_id); DeleteReferenceInternal(it, nullptr); } else { // We are still borrowing the object ID. Respond to the owner once we have // stopped borrowing it. - if (it->second.on_ref_removed != nullptr) { + if (reference.publish_ref_removed) { // TODO(swang): If the owner of an object dies and and is re-executed, it // is possible that we will receive a duplicate request to set - // on_ref_removed. If messages are delayed and we overwrite the + // publish_ref_removed. If messages are delayed and we overwrite the // callback here, it's possible we will drop the request that was sent by // the more recent owner. We should fix this by setting multiple // callbacks or by versioning the owner requests. RAY_LOG(WARNING).WithField(object_id) - << "on_ref_removed already set for object. The owner task must have died and " - "been re-executed."; + << "publish_ref_removed already set for object. The owner task must have " + "died and been re-executed."; } - it->second.on_ref_removed = ref_removed_callback; + reference.publish_ref_removed = true; } } diff --git a/src/ray/core_worker/reference_count.h b/src/ray/core_worker/reference_count.h index d168f749e42f..75e8423ca86e 100644 --- a/src/ray/core_worker/reference_count.h +++ b/src/ray/core_worker/reference_count.h @@ -345,8 +345,8 @@ class ReferenceCounter : public ReferenceCounterInterface, const std::function callback) override ABSL_LOCKS_EXCLUDED(mutex_); - /// Set a callback for when we are no longer borrowing this object (when our - /// ref count goes to 0). + /// So we call PublishRefRemovedInternal when we are no longer borrowing this object + /// (when our ref count goes to 0). /// /// \param[in] object_id The object ID to set the callback for. /// \param[in] contained_in_id The object ID that contains object_id, if any. @@ -354,13 +354,9 @@ class ReferenceCounter : public ReferenceCounterInterface, /// submitted. Then, as long as we have contained_in_id in scope, we are /// borrowing object_id. /// \param[in] owner_address The owner of object_id's address. - /// \param[in] ref_removed_callback The callback to call when we are no - /// longer borrowing the object. - void SetRefRemovedCallback(const ObjectID &object_id, - const ObjectID &contained_in_id, - const rpc::Address &owner_address, - const ReferenceRemovedCallback &ref_removed_callback) - ABSL_LOCKS_EXCLUDED(mutex_); + void SubscribeRefRemoved(const ObjectID &object_id, + const ObjectID &contained_in_id, + const rpc::Address &owner_address) ABSL_LOCKS_EXCLUDED(mutex_); /// Set a callback to call whenever a Reference that we own is deleted. A /// Reference can only be deleted if: @@ -371,12 +367,8 @@ class ReferenceCounter : public ReferenceCounterInterface, /// \param[in] callback The callback to call. void SetReleaseLineageCallback(const LineageReleasedCallback &callback); - /// Respond to the object's owner once we are no longer borrowing it. The - /// sender is the owner of the object ID. We will send the reply when our - /// RefCount() for the object ID goes to 0. - /// - /// \param[in] object_id The object that we were borrowing. - void HandleRefRemoved(const ObjectID &object_id) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + /// Just calls PublishRefRemovedInternal with a lock. + void PublishRefRemoved(const ObjectID &object_id) ABSL_LOCKS_EXCLUDED(mutex_); /// Returns the total number of ObjectIDs currently in scope. size_t NumObjectIDsInScope() const ABSL_LOCKS_EXCLUDED(mutex_); @@ -815,9 +807,9 @@ class ReferenceCounter : public ReferenceCounterInterface, /// Callback that will be called when the object ref is deleted /// from the reference table (all refs including lineage ref count go to 0). std::function on_object_ref_delete; - /// Callback that is called when this process is no longer a borrower - /// (RefCount() == 0). - std::function on_ref_removed; + /// If this is set, we'll call PublishRefRemovedInternal when this process is no + /// longer a borrower (RefCount() == 0). + bool publish_ref_removed = false; /// For objects that have been spilled to external storage, the URL from which /// they can be retrieved. @@ -990,6 +982,12 @@ class ReferenceCounter : public ReferenceCounterInterface, std::vector *deleted) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + /// To respond to the object's owner once we are no longer borrowing it. The + /// sender is the owner of the object ID. We will send the reply when our + /// RefCount() for the object ID goes to 0. + void PublishRefRemovedInternal(const ObjectID &object_id) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + /// Erase the Reference from the table. Assumes that the entry has no more /// references, normal or lineage. void EraseReference(ReferenceTable::iterator entry) diff --git a/src/ray/core_worker/tests/reference_count_test.cc b/src/ray/core_worker/tests/reference_count_test.cc index 72f7dc6a1364..0686db4837f9 100644 --- a/src/ray/core_worker/tests/reference_count_test.cc +++ b/src/ray/core_worker/tests/reference_count_test.cc @@ -319,10 +319,7 @@ class MockWorkerClient : public MockCoreWorkerClientInterface { auto r = num_requests_; auto borrower_callback = [=]() { - auto ref_removed_callback = - absl::bind_front(&ReferenceCounter::HandleRefRemoved, &rc_); - rc_.SetRefRemovedCallback( - object_id, contained_in_id, owner_address, ref_removed_callback); + rc_.SubscribeRefRemoved(object_id, contained_in_id, owner_address); }; borrower_callbacks_[r] = borrower_callback; From 8b5c8983a40b5d94af0140fda95bf7e8dc2a223d Mon Sep 17 00:00:00 2001 From: Ibrahim Rabbani Date: Thu, 11 Sep 2025 17:02:37 -0700 Subject: [PATCH 588/634] [core] Starting log_monitor before starting the raylet. (#56471) Signed-off-by: irabbani --- python/ray/_private/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ray/_private/node.py b/python/ray/_private/node.py index 74e13ae883f3..49a50e2ff4bb 100644 --- a/python/ray/_private/node.py +++ b/python/ray/_private/node.py @@ -1422,10 +1422,11 @@ def start_ray_processes(self): if self.resource_isolation_config.is_enabled(): self.resource_isolation_config.add_object_store_memory(object_store_memory) - self.start_raylet(plasma_directory, fallback_directory, object_store_memory) if self._ray_params.include_log_monitor: self.start_log_monitor() + self.start_raylet(plasma_directory, fallback_directory, object_store_memory) + def _kill_process_type( self, process_type, From e829eafcac96d56c54aa16f8998472a00b6d57be Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Thu, 11 Sep 2025 17:41:09 -0700 Subject: [PATCH 589/634] [core][1eventx/03] job event: send job events to the aggregator (#55213) This is part of a series of PRs to support JobEvent in the oneevent framework. The full effort will include adding the JobEvent schema, introducing a generic interface for exporting different types of events to the Event Aggregator, and implementing the necessary integration logic. ---- In this PR, we implement: - The integration of RayEventRecorder to the GcsServer to export job event - Add an e2e test to check that the expected job event is exported Test: - CI --------- Signed-off-by: Cuong Nguyen --- .../aggregator/tests/test_aggregator_agent.py | 4 +- .../aggregator/tests/test_ray_job_events.py | 69 +++++++++++++++++++ src/ray/common/ray_config_def.h | 4 ++ src/ray/gcs/gcs_server/BUILD.bazel | 4 ++ src/ray/gcs/gcs_server/gcs_job_manager.cc | 31 +++++++-- src/ray/gcs/gcs_server/gcs_job_manager.h | 12 +++- src/ray/gcs/gcs_server/gcs_server.cc | 17 ++++- src/ray/gcs/gcs_server/gcs_server.h | 6 ++ .../gcs_server/gcs_server_io_context_policy.h | 14 ++-- src/ray/gcs/gcs_server/gcs_server_main.cc | 1 + src/ray/gcs/gcs_server/tests/BUILD.bazel | 2 + .../gcs_job_manager_export_event_test.cc | 45 +++++++++++- .../gcs_server/tests/gcs_job_manager_test.cc | 11 ++- src/ray/observability/BUILD.bazel | 9 +++ .../observability/fake_ray_event_recorder.h | 48 +++++++++++++ .../ray_driver_job_execution_event_test.cc | 8 +-- .../tests/ray_event_recorder_test.cc | 2 +- .../events_driver_job_execution_event.proto | 3 +- 18 files changed, 264 insertions(+), 26 deletions(-) create mode 100644 python/ray/dashboard/modules/aggregator/tests/test_ray_job_events.py create mode 100644 src/ray/observability/fake_ray_event_recorder.h diff --git a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py index 31b3ed26b6cf..4d5bf15ed4ba 100644 --- a/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py +++ b/python/ray/dashboard/modules/aggregator/tests/test_aggregator_agent.py @@ -839,7 +839,7 @@ def test_aggregator_agent_receive_driver_job_execution_event( timestamp=Timestamp(seconds=1234567890), ), DriverJobExecutionEvent.StateTimestamp( - state=DriverJobExecutionEvent.State.FAILURE, + state=DriverJobExecutionEvent.State.FINISHED, timestamp=Timestamp(seconds=1234567890), ), ], @@ -862,7 +862,7 @@ def test_aggregator_agent_receive_driver_job_execution_event( ) assert len(req_json[0]["driverJobExecutionEvent"]["states"]) == 2 assert req_json[0]["driverJobExecutionEvent"]["states"][0]["state"] == "CREATED" - assert req_json[0]["driverJobExecutionEvent"]["states"][1]["state"] == "FAILURE" + assert req_json[0]["driverJobExecutionEvent"]["states"][1]["state"] == "FINISHED" if __name__ == "__main__": diff --git a/python/ray/dashboard/modules/aggregator/tests/test_ray_job_events.py b/python/ray/dashboard/modules/aggregator/tests/test_ray_job_events.py new file mode 100644 index 000000000000..006bccd2f19d --- /dev/null +++ b/python/ray/dashboard/modules/aggregator/tests/test_ray_job_events.py @@ -0,0 +1,69 @@ +import base64 +import json +import sys + +import pytest + +import ray +import ray.dashboard.consts as dashboard_consts +from ray._private import ray_constants +from ray._private.test_utils import wait_for_condition +from ray._raylet import GcsClient +from ray.dashboard.tests.conftest import * # noqa + +_RAY_EVENT_PORT = 12345 + + +@pytest.fixture(scope="session") +def httpserver_listen_address(): + return ("127.0.0.1", _RAY_EVENT_PORT) + + +def wait_for_dashboard_agent_available(cluster): + gcs_client = GcsClient(address=cluster.address) + + def get_dashboard_agent_address(): + return gcs_client.internal_kv_get( + f"{dashboard_consts.DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX}{cluster.head_node.node_id}".encode(), + namespace=ray_constants.KV_NAMESPACE_DASHBOARD, + timeout=dashboard_consts.GCS_RPC_TIMEOUT_SECONDS, + ) + + wait_for_condition(lambda: get_dashboard_agent_address() is not None) + + +def test_ray_job_events(ray_start_cluster, httpserver): + cluster = ray_start_cluster + cluster.add_node( + env_vars={ + "RAY_DASHBOARD_AGGREGATOR_AGENT_EVENTS_EXPORT_ADDR": f"http://127.0.0.1:{_RAY_EVENT_PORT}", + "RAY_DASHBOARD_AGGREGATOR_AGENT_EXPOSABLE_EVENT_TYPES": "DRIVER_JOB_DEFINITION_EVENT,DRIVER_JOB_EXECUTION_EVENT", + }, + _system_config={ + "enable_ray_event": True, + }, + ) + cluster.wait_for_nodes() + ray.init(address=cluster.address) + wait_for_dashboard_agent_available(cluster) + + # Submit a ray job + @ray.remote + def f(): + return 1 + + ray.get(f.remote()) + + # Check that a driver job event with the correct job id is published. + httpserver.expect_request("/", method="POST").respond_with_data("", status=200) + wait_for_condition(lambda: len(httpserver.log) >= 1) + req, _ = httpserver.log[0] + req_json = json.loads(req.data) + assert ( + base64.b64decode(req_json[0]["driverJobDefinitionEvent"]["jobId"]).hex() + == ray.get_runtime_context().get_job_id() + ) + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", __file__])) diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index f780c9b6868e..4f166f891816 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -530,6 +530,10 @@ RAY_CONFIG(std::string, metric_cardinality_level, "legacy") /// using OpenCensus. RAY_CONFIG(bool, enable_open_telemetry, false) +/// Whether to enable Ray Event as the event collection backend. The default is +/// using the Export API. +RAY_CONFIG(bool, enable_ray_event, false) + /// Comma separated list of components we enable grpc metrics collection for. /// Only effective if `enable_metrics_collection` is also true. Will have some performance /// degredations. diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/gcs_server/BUILD.bazel index 35fb38ae328f..695ddad003b1 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/gcs_server/BUILD.bazel @@ -231,6 +231,7 @@ ray_cc_library( deps = [ ":gcs_task_manager", "//src/ray/common:ray_syncer", + "//src/ray/observability:ray_event_recorder", "//src/ray/pubsub:gcs_publisher", "//src/ray/util:array", "//src/ray/util:type_traits", @@ -248,6 +249,9 @@ ray_cc_library( ":grpc_service_interfaces", "//src/ray/common:protobuf_utils", "//src/ray/common:runtime_env", + "//src/ray/observability:ray_driver_job_definition_event", + "//src/ray/observability:ray_driver_job_execution_event", + "//src/ray/observability:ray_event_recorder_interface", "//src/ray/pubsub:gcs_publisher", "//src/ray/rpc:core_worker_client", "//src/ray/stats:stats_metric", diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.cc b/src/ray/gcs/gcs_server/gcs_job_manager.cc index 0ea008ad7ee9..89af057e6832 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.cc +++ b/src/ray/gcs/gcs_server/gcs_job_manager.cc @@ -23,6 +23,8 @@ #include "absl/strings/match.h" #include "ray/common/protobuf_utils.h" +#include "ray/observability/ray_driver_job_definition_event.h" +#include "ray/observability/ray_driver_job_execution_event.h" #include "ray/stats/metric.h" #include "ray/util/time.h" @@ -42,18 +44,33 @@ void GcsJobManager::Initialize(const GcsInitData &gcs_init_data) { } } -void GcsJobManager::WriteDriverJobExportEvent(rpc::JobTableData job_data) const { +void GcsJobManager::WriteDriverJobExportEvent( + rpc::JobTableData job_data, rpc::events::DriverJobExecutionEvent::State state) const { /// Write job_data as a export driver job event if /// enable_export_api_write() is enabled and if this job is /// not in the _ray_internal_ namespace. - if (!export_event_write_enabled_) { - return; - } if (absl::StartsWith(job_data.config().ray_namespace(), kRayInternalNamespacePrefix)) { // Namespace of this job starts with _ray_internal_ so // don't write export event. return; } + if (RayConfig::instance().enable_ray_event()) { + std::vector> events; + if (state == rpc::events::DriverJobExecutionEvent::CREATED) { + // Job definition event is emitted once when the job is created. + events.push_back(std::make_unique( + job_data, session_name_)); + } + events.push_back(std::make_unique( + job_data, state, session_name_)); + ray_event_recorder_.AddEvents(std::move(events)); + return; + } + + // TODO(#56391): to be deprecated once the Ray Event system is stable. + if (!export_event_write_enabled_) { + return; + } std::shared_ptr export_driver_job_data_ptr = std::make_shared(); export_driver_job_data_ptr->set_job_id(job_data.job_id()); @@ -105,7 +122,8 @@ void GcsJobManager::HandleAddJob(rpc::AddJobRequest request, reply, send_reply_callback = std::move(send_reply_callback)](const Status &status) mutable { - WriteDriverJobExportEvent(job_table_data); + WriteDriverJobExportEvent(job_table_data, + rpc::events::DriverJobExecutionEvent::CREATED); if (!status.ok()) { RAY_LOG(ERROR).WithField(job_id).WithField("driver_pid", job_table_data.driver_pid()) @@ -155,7 +173,8 @@ void GcsJobManager::MarkJobAsFinished(rpc::JobTableData job_table_data, RAY_LOG(DEBUG).WithField(job_id) << "Marked job as finished."; } function_manager_.RemoveJobReference(job_id); - WriteDriverJobExportEvent(job_table_data); + WriteDriverJobExportEvent(job_table_data, + rpc::events::DriverJobExecutionEvent::FINISHED); // Update running job status. // Note: This operation must be idempotent since MarkJobFinished can be called diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.h b/src/ray/gcs/gcs_server/gcs_job_manager.h index 095393c960c0..f6c534eb5c70 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.h +++ b/src/ray/gcs/gcs_server/gcs_job_manager.h @@ -27,6 +27,7 @@ #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "ray/gcs/gcs_server/gcs_table_storage.h" #include "ray/gcs/gcs_server/grpc_service_interfaces.h" +#include "ray/observability/ray_event_recorder_interface.h" #include "ray/pubsub/gcs_publisher.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" @@ -57,7 +58,9 @@ class GcsJobManager : public rpc::JobInfoGcsServiceHandler { GCSFunctionManager &function_manager, InternalKVInterface &internal_kv, instrumented_io_context &io_context, - rpc::CoreWorkerClientPool &worker_client_pool) + rpc::CoreWorkerClientPool &worker_client_pool, + observability::RayEventRecorderInterface &ray_event_recorder, + const std::string &session_name) : gcs_table_storage_(gcs_table_storage), gcs_publisher_(gcs_publisher), runtime_env_manager_(runtime_env_manager), @@ -65,6 +68,8 @@ class GcsJobManager : public rpc::JobInfoGcsServiceHandler { internal_kv_(internal_kv), io_context_(io_context), worker_client_pool_(worker_client_pool), + ray_event_recorder_(ray_event_recorder), + session_name_(session_name), export_event_write_enabled_(IsExportAPIEnabledDriverJob()) {} void Initialize(const GcsInitData &gcs_init_data); @@ -99,7 +104,8 @@ class GcsJobManager : public rpc::JobInfoGcsServiceHandler { /// \param node_id The specified node id. void OnNodeDead(const NodeID &node_id); - void WriteDriverJobExportEvent(rpc::JobTableData job_data) const; + void WriteDriverJobExportEvent(rpc::JobTableData job_data, + rpc::events::DriverJobExecutionEvent::State state) const; // Verify if export events should be written for EXPORT_DRIVER_JOB source types bool IsExportAPIEnabledDriverJob() const { @@ -145,6 +151,8 @@ class GcsJobManager : public rpc::JobInfoGcsServiceHandler { InternalKVInterface &internal_kv_; instrumented_io_context &io_context_; rpc::CoreWorkerClientPool &worker_client_pool_; + observability::RayEventRecorderInterface &ray_event_recorder_; + std::string session_name_; /// If true, driver job events are exported for Export API bool export_event_write_enabled_ = false; diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server/gcs_server.cc index 9adbcb2b9445..1249b6298d7f 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server/gcs_server.cc @@ -113,6 +113,16 @@ GcsServer::GcsServer(const ray::gcs::GcsServerConfig &config, }); }); }), + event_aggregator_client_call_manager_( + io_context_provider_.GetIOContext(), + /*record_stats=*/true, + ClusterID::Nil(), + RayConfig::instance().gcs_server_rpc_client_thread_num()), + event_aggregator_client_(std::make_unique( + config_.metrics_agent_port, event_aggregator_client_call_manager_)), + ray_event_recorder_(std::make_unique( + *event_aggregator_client_, + io_context_provider_.GetIOContext())), pubsub_periodical_runner_(PeriodicalRunner::Create( io_context_provider_.GetIOContext())), periodical_runner_( @@ -255,9 +265,10 @@ void GcsServer::DoStart(const GcsInitData &gcs_init_data) { InitGcsAutoscalerStateManager(gcs_init_data); InitUsageStatsClient(); - // Init OpenTelemetry exporter. + // Init metrics and event exporter. metrics_agent_client_->WaitForServerReady([this](const Status &server_status) { stats::InitOpenTelemetryExporter(config_.metrics_agent_port, server_status); + ray_event_recorder_->StartExportingEvents(); }); // Start RPC server when all tables have finished loading initial @@ -446,7 +457,9 @@ void GcsServer::InitGcsJobManager(const GcsInitData &gcs_init_data) { *function_manager_, kv_manager_->GetInstance(), io_context_provider_.GetDefaultIOContext(), - worker_client_pool_); + worker_client_pool_, + *ray_event_recorder_, + config_.session_name); gcs_job_manager_->Initialize(gcs_init_data); rpc_server_.RegisterService(std::make_unique( diff --git a/src/ray/gcs/gcs_server/gcs_server.h b/src/ray/gcs/gcs_server/gcs_server.h index 4bbe81432e01..31c3f3a09fb4 100644 --- a/src/ray/gcs/gcs_server/gcs_server.h +++ b/src/ray/gcs/gcs_server/gcs_server.h @@ -36,6 +36,7 @@ #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/store_client/observable_store_client.h" #include "ray/gcs/store_client/redis_store_client.h" +#include "ray/observability/ray_event_recorder.h" #include "ray/pubsub/gcs_publisher.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" @@ -267,6 +268,11 @@ class GcsServer { std::unique_ptr kv_manager_; /// Job info handler. std::unique_ptr gcs_job_manager_; + /// The Ray event recorder that is used to record events (e.g. job events, node events, + /// etc.). + rpc::ClientCallManager event_aggregator_client_call_manager_; + std::unique_ptr event_aggregator_client_; + std::unique_ptr ray_event_recorder_; /// Ray Syncer related fields. std::unique_ptr ray_syncer_; diff --git a/src/ray/gcs/gcs_server/gcs_server_io_context_policy.h b/src/ray/gcs/gcs_server/gcs_server_io_context_policy.h index d43dc1086657..95e5e550634f 100644 --- a/src/ray/gcs/gcs_server/gcs_server_io_context_policy.h +++ b/src/ray/gcs/gcs_server/gcs_server_io_context_policy.h @@ -20,6 +20,7 @@ #include "ray/common/ray_syncer/ray_syncer.h" #include "ray/gcs/gcs_server/gcs_task_manager.h" +#include "ray/observability/ray_event_recorder.h" #include "ray/pubsub/gcs_publisher.h" #include "ray/util/array.h" #include "ray/util/type_traits.h" @@ -41,6 +42,8 @@ struct GcsServerIOContextPolicy { return IndexOf("pubsub_io_context"); } else if constexpr (std::is_same_v) { return IndexOf("ray_syncer_io_context"); + } else if constexpr (std::is_same_v) { + return IndexOf("ray_event_io_context"); } else if constexpr (std::is_same_v) { // default io context return -1; @@ -54,10 +57,13 @@ struct GcsServerIOContextPolicy { // This list must be unique and complete set of names returned from // GetDedicatedIOContextIndex. Or you can get runtime crashes when accessing a missing // name, or get leaks by creating unused threads. - constexpr static std::array kAllDedicatedIOContextNames{ - "task_io_context", "pubsub_io_context", "ray_syncer_io_context"}; - constexpr static std::array kAllDedicatedIOContextEnableLagProbe{ - true, true, true}; + constexpr static std::array kAllDedicatedIOContextNames{ + "task_io_context", + "pubsub_io_context", + "ray_syncer_io_context", + "ray_event_io_context"}; + constexpr static std::array kAllDedicatedIOContextEnableLagProbe{ + true, true, true, true}; constexpr static size_t IndexOf(std::string_view name) { return ray::IndexOf(kAllDedicatedIOContextNames, name); diff --git a/src/ray/gcs/gcs_server/gcs_server_main.cc b/src/ray/gcs/gcs_server/gcs_server_main.cc index 9cf428dbbae7..712acb8d9768 100644 --- a/src/ray/gcs/gcs_server/gcs_server_main.cc +++ b/src/ray/gcs/gcs_server/gcs_server_main.cc @@ -150,6 +150,7 @@ int main(int argc, char *argv[]) { gcs_server_config.grpc_server_port = gcs_server_port; gcs_server_config.grpc_server_thread_num = RayConfig::instance().gcs_server_rpc_server_thread_num(); + gcs_server_config.metrics_agent_port = metrics_agent_port; gcs_server_config.redis_address = redis_address; gcs_server_config.redis_port = redis_port; gcs_server_config.enable_redis_ssl = FLAGS_redis_enable_ssl; diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/gcs_server/tests/BUILD.bazel index 7d5ecaf74534..feae8305f242 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/gcs_server/tests/BUILD.bazel @@ -128,6 +128,7 @@ ray_cc_test( "//src/ray/gcs/gcs_server:gcs_job_manager", "//src/ray/gcs/gcs_server:gcs_kv_manager", "//src/ray/gcs/store_client:in_memory_store_client", + "//src/ray/observability:fake_ray_event_recorder", "@com_google_googletest//:gtest_main", ], ) @@ -395,6 +396,7 @@ ray_cc_test( "//src/ray/gcs/gcs_server:gcs_job_manager", "//src/ray/gcs/gcs_server:gcs_kv_manager", "//src/ray/gcs/store_client:in_memory_store_client", + "//src/ray/observability:fake_ray_event_recorder", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc index 498ba1754e4c..9a310d1f923e 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc +++ b/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc @@ -26,6 +26,7 @@ #include "ray/gcs/gcs_server/gcs_job_manager.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" +#include "ray/observability/fake_ray_event_recorder.h" using json = nlohmann::json; @@ -59,6 +60,7 @@ class GcsJobManagerTest : public ::testing::Test { return std::make_shared( address.port()); }); + fake_ray_event_recorder_ = std::make_unique(); log_dir_ = "event_12345"; } @@ -78,11 +80,50 @@ class GcsJobManagerTest : public ::testing::Test { std::unique_ptr kv_; std::unique_ptr fake_kv_; std::unique_ptr worker_client_pool_; + std::unique_ptr fake_ray_event_recorder_; RuntimeEnvManager runtime_env_manager_; const std::chrono::milliseconds timeout_ms_{5000}; std::string log_dir_; }; +TEST_F(GcsJobManagerTest, TestRayEventDriverJobEvents) { + RayConfig::instance().initialize( + R"( +{ + "enable_ray_event": true +} + )"); + gcs::GcsJobManager gcs_job_manager(*gcs_table_storage_, + *gcs_publisher_, + runtime_env_manager_, + *function_manager_, + *fake_kv_, + io_service_, + *worker_client_pool_, + *fake_ray_event_recorder_, + "test_session_name"); + gcs::GcsInitData gcs_init_data(*gcs_table_storage_); + gcs_job_manager.Initialize(gcs_init_data); + auto job_api_job_id = JobID::FromInt(100); + std::string submission_id = "submission_id_100"; + auto add_job_request = GenAddJobRequest(job_api_job_id, "namespace_100", submission_id); + rpc::AddJobReply empty_reply; + std::promise promise; + gcs_job_manager.HandleAddJob( + *add_job_request, + &empty_reply, + [&promise](Status, std::function, std::function) { + promise.set_value(true); + }); + promise.get_future().get(); + auto buffer = fake_ray_event_recorder_->FlushBuffer(); + + ASSERT_EQ(buffer.size(), 2); + ASSERT_EQ(buffer[0]->GetEventType(), + rpc::events::RayEvent::DRIVER_JOB_DEFINITION_EVENT); + ASSERT_EQ(buffer[1]->GetEventType(), rpc::events::RayEvent::DRIVER_JOB_EXECUTION_EVENT); +} + TEST_F(GcsJobManagerTest, TestExportDriverJobEvents) { // Test adding and marking a driver job as finished, and that corresponding // export events are written. @@ -105,7 +146,9 @@ TEST_F(GcsJobManagerTest, TestExportDriverJobEvents) { *function_manager_, *fake_kv_, io_service_, - *worker_client_pool_); + *worker_client_pool_, + *fake_ray_event_recorder_, + "test_session_name"); gcs::GcsInitData gcs_init_data(*gcs_table_storage_); gcs_job_manager.Initialize(gcs_init_data); diff --git a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc index ee9f6a320e9a..4c4227f5d929 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc @@ -24,6 +24,7 @@ #include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_kv_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" +#include "ray/observability/fake_ray_event_recorder.h" namespace ray { @@ -55,13 +56,16 @@ class GcsJobManagerTest : public ::testing::Test { return std::make_shared( address.port()); }); + fake_ray_event_recorder_ = std::make_unique(); gcs_job_manager_ = std::make_unique(*gcs_table_storage_, *gcs_publisher_, runtime_env_manager_, *function_manager_, *fake_kv_, io_service_, - *worker_client_pool_); + *worker_client_pool_, + *fake_ray_event_recorder_, + "test_session_name"); } ~GcsJobManagerTest() { @@ -82,6 +86,7 @@ class GcsJobManagerTest : public ::testing::Test { RuntimeEnvManager runtime_env_manager_; const std::chrono::milliseconds timeout_ms_{5000}; std::unique_ptr gcs_job_manager_; + std::unique_ptr fake_ray_event_recorder_; }; TEST_F(GcsJobManagerTest, TestFakeInternalKV) { @@ -607,7 +612,9 @@ TEST_F(GcsJobManagerTest, TestMarkJobFinishedIdempotency) { *function_manager_, *fake_kv_, io_service_, - *worker_client_pool_); + *worker_client_pool_, + *fake_ray_event_recorder_, + "test_session_name"); auto job_id = JobID::FromInt(1); gcs::GcsInitData gcs_init_data(*gcs_table_storage_); diff --git a/src/ray/observability/BUILD.bazel b/src/ray/observability/BUILD.bazel index 4da0cd2a5c83..ac19b4f6eea2 100644 --- a/src/ray/observability/BUILD.bazel +++ b/src/ray/observability/BUILD.bazel @@ -116,3 +116,12 @@ ray_cc_library( "@com_google_absl//absl/time", ], ) + +ray_cc_library( + name = "fake_ray_event_recorder", + hdrs = ["fake_ray_event_recorder.h"], + deps = [ + ":ray_event_interface", + ":ray_event_recorder_interface", + ], +) diff --git a/src/ray/observability/fake_ray_event_recorder.h b/src/ray/observability/fake_ray_event_recorder.h new file mode 100644 index 000000000000..bbbecdf0d69a --- /dev/null +++ b/src/ray/observability/fake_ray_event_recorder.h @@ -0,0 +1,48 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "ray/observability/ray_event_interface.h" +#include "ray/observability/ray_event_recorder_interface.h" + +namespace ray { +namespace observability { + +class FakeRayEventRecorder : public RayEventRecorderInterface { + public: + void StartExportingEvents() override {} + void AddEvents(std::vector> &&data_list) override { + absl::MutexLock lock(&mutex_); + buffer_.insert(buffer_.end(), + std::make_move_iterator(data_list.begin()), + std::make_move_iterator(data_list.end())); + } + + const std::vector> FlushBuffer() { + absl::MutexLock lock(&mutex_); + auto buffer = std::move(buffer_); + buffer_.clear(); + return buffer; + } + + private: + std::vector> buffer_ ABSL_GUARDED_BY(mutex_); + absl::Mutex mutex_; +}; + +} // namespace observability +} // namespace ray diff --git a/src/ray/observability/tests/ray_driver_job_execution_event_test.cc b/src/ray/observability/tests/ray_driver_job_execution_event_test.cc index d6a2b1ad4d4b..1e6a26a0ea53 100644 --- a/src/ray/observability/tests/ray_driver_job_execution_event_test.cc +++ b/src/ray/observability/tests/ray_driver_job_execution_event_test.cc @@ -25,16 +25,16 @@ TEST_F(RayDriverJobExecutionEventTest, TestMerge) { rpc::JobTableData data; data.set_job_id("test_job_id_1"); auto event1 = std::make_unique( - data, rpc::events::DriverJobExecutionEvent::SUCCESS, "test_session_name_1"); + data, rpc::events::DriverJobExecutionEvent::CREATED, "test_session_name_1"); auto event2 = std::make_unique( - data, rpc::events::DriverJobExecutionEvent::FAILURE, "test_session_name_1"); + data, rpc::events::DriverJobExecutionEvent::FINISHED, "test_session_name_1"); event1->Merge(std::move(*event2)); auto serialized_event = std::move(*event1).Serialize(); ASSERT_EQ(serialized_event.driver_job_execution_event().states_size(), 2); ASSERT_EQ(serialized_event.driver_job_execution_event().states(0).state(), - rpc::events::DriverJobExecutionEvent::SUCCESS); + rpc::events::DriverJobExecutionEvent::CREATED); ASSERT_EQ(serialized_event.driver_job_execution_event().states(1).state(), - rpc::events::DriverJobExecutionEvent::FAILURE); + rpc::events::DriverJobExecutionEvent::FINISHED); } } // namespace observability diff --git a/src/ray/observability/tests/ray_event_recorder_test.cc b/src/ray/observability/tests/ray_event_recorder_test.cc index 4f825b3ee95a..13a64f8aa4bf 100644 --- a/src/ray/observability/tests/ray_event_recorder_test.cc +++ b/src/ray/observability/tests/ray_event_recorder_test.cc @@ -91,7 +91,7 @@ TEST_F(RayEventRecorderTest, TestRecordEvents) { events.push_back( std::make_unique(data1, "test_session_name_1")); events.push_back(std::make_unique( - data2, rpc::events::DriverJobExecutionEvent::SUCCESS, "test_session_name_2")); + data2, rpc::events::DriverJobExecutionEvent::FINISHED, "test_session_name_2")); recorder_->AddEvents(std::move(events)); io_service_.run_one(); diff --git a/src/ray/protobuf/public/events_driver_job_execution_event.proto b/src/ray/protobuf/public/events_driver_job_execution_event.proto index 2d6c58f1c760..4c9dd611140c 100644 --- a/src/ray/protobuf/public/events_driver_job_execution_event.proto +++ b/src/ray/protobuf/public/events_driver_job_execution_event.proto @@ -27,8 +27,7 @@ message DriverJobExecutionEvent { enum State { UNSPECIFIED = 0; CREATED = 1; - FAILURE = 2; - SUCCESS = 3; + FINISHED = 2; } message StateTimestamp { From 3f124af6277d3291201cb42ee32590c68239f9d5 Mon Sep 17 00:00:00 2001 From: kourosh hakhamaneshi <31483498+kouroshHakha@users.noreply.github.com> Date: Fri, 12 Sep 2025 02:54:23 +0200 Subject: [PATCH 590/634] [data.llm] Fix sglang byod on release (#55885) Signed-off-by: Kourosh Hakhamaneshi Co-authored-by: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> --- release/ray_release/byod/byod_llm_sglang_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/ray_release/byod/byod_llm_sglang_test.sh b/release/ray_release/byod/byod_llm_sglang_test.sh index 43a43b4a7ed6..6b75a5306b2d 100755 --- a/release/ray_release/byod/byod_llm_sglang_test.sh +++ b/release/ray_release/byod/byod_llm_sglang_test.sh @@ -4,4 +4,4 @@ set -exo pipefail -pip3 install "sglang[all]==0.4.5.post1" +pip3 install "sglang[all]==0.5.1.post2" From c05037e5df0393ab6781aa3599888af8918c5290 Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Thu, 11 Sep 2025 18:31:30 -0700 Subject: [PATCH 591/634] [Data] Update image embedding benchmark to use `download` (#56245) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? The existing image embedding release test downloads images from URIs in a pandas DataFrame. Previously, it used a boto-based UDF, but now it’s updated to use the built-in Ray API introduced in https://github.com/ray-project/ray/pull/55824. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Balaji Veeramani --- .../autoscaling_cluster_compute.yaml | 21 ++++ .../fixed_size_cluster_compute.yaml | 21 ++++ .../main.py} | 100 +++++++++++------- release/release_data_tests.yaml | 53 ++++------ 4 files changed, 126 insertions(+), 69 deletions(-) create mode 100644 release/nightly_tests/dataset/image_embedding_from_uris/autoscaling_cluster_compute.yaml create mode 100644 release/nightly_tests/dataset/image_embedding_from_uris/fixed_size_cluster_compute.yaml rename release/nightly_tests/dataset/{batch_inference_mock_image_pipeline.py => image_embedding_from_uris/main.py} (71%) diff --git a/release/nightly_tests/dataset/image_embedding_from_uris/autoscaling_cluster_compute.yaml b/release/nightly_tests/dataset/image_embedding_from_uris/autoscaling_cluster_compute.yaml new file mode 100644 index 000000000000..ff9f39f3cc5a --- /dev/null +++ b/release/nightly_tests/dataset/image_embedding_from_uris/autoscaling_cluster_compute.yaml @@ -0,0 +1,21 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +advanced_configurations_json: + IamInstanceProfile: {"Name": "ray-autoscaler-v1"} + +head_node_type: + name: head-node + instance_type: m5.2xlarge + resources: + cpu: 0 + +worker_node_types: + - name: worker-node + instance_type: g4dn.2xlarge + min_workers: 0 + max_workers: 100 + use_spot: false + +flags: + allow-cross-zone-autoscaling: true diff --git a/release/nightly_tests/dataset/image_embedding_from_uris/fixed_size_cluster_compute.yaml b/release/nightly_tests/dataset/image_embedding_from_uris/fixed_size_cluster_compute.yaml new file mode 100644 index 000000000000..199da1873dc3 --- /dev/null +++ b/release/nightly_tests/dataset/image_embedding_from_uris/fixed_size_cluster_compute.yaml @@ -0,0 +1,21 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +advanced_configurations_json: + IamInstanceProfile: {"Name": "ray-autoscaler-v1"} + +head_node_type: + name: head-node + instance_type: m5.2xlarge + resources: + cpu: 0 + +worker_node_types: + - name: worker-node + instance_type: g4dn.2xlarge + min_workers: 100 + max_workers: 100 + use_spot: false + +flags: + allow-cross-zone-autoscaling: true diff --git a/release/nightly_tests/dataset/batch_inference_mock_image_pipeline.py b/release/nightly_tests/dataset/image_embedding_from_uris/main.py similarity index 71% rename from release/nightly_tests/dataset/batch_inference_mock_image_pipeline.py rename to release/nightly_tests/dataset/image_embedding_from_uris/main.py index 41aa12f10845..48fcf8fc83f3 100644 --- a/release/nightly_tests/dataset/batch_inference_mock_image_pipeline.py +++ b/release/nightly_tests/dataset/image_embedding_from_uris/main.py @@ -3,7 +3,6 @@ import uuid from typing import Any, Dict -import boto3 import numpy as np import pandas as pd import torch @@ -12,19 +11,22 @@ from torchvision.models import vit_b_16, ViT_B_16_Weights import albumentations as A import ray -from ray.data import ActorPoolStrategy, DataContext import copy import itertools from typing import List import string import random import time +from ray.data.expressions import download +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy +from ray._private.test_utils import EC2InstanceTerminatorWithGracePeriod + WRITE_PATH = f"s3://ray-data-write-benchmark/{uuid.uuid4().hex}" BUCKET = "ray-benchmark-data-internal-us-west-2" # Assumptions: homogenously shaped images, homogenous images -# Each iamge is 2048 * 2048 * 3 = 12.58 MB -> 11 images / block. 8 blocks per task, so ~88 images per task. +# Each image is 2048 * 2048 * 3 = 12.58 MB -> 11 images / block. 8 blocks per task, so ~88 images per task. IMAGES_PER_BLOCK = 11 BLOCKS_PER_TASK = 8 NUM_UNITS = 1380 @@ -42,6 +44,13 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() + parser.add_argument( + "--inference-concurrency", + nargs=2, + type=int, + required=True, + help="The minimum and maximum concurrency for the inference operator.", + ) parser.add_argument( "--sf", dest="scale_factor", @@ -52,6 +61,14 @@ def parse_args() -> argparse.Namespace: "dataset." ), ) + parser.add_argument( + "--chaos", + action="store_true", + help=( + "Whether to enable chaos. If set, this script terminates one worker node " + "every minute with a grace period." + ), + ) return parser.parse_args() @@ -70,10 +87,9 @@ def create_metadata(scale_factor: int): "metadata_6": "".join(random.choices(string.ascii_letters, k=16)), "container_order_read_id": f"{i:04d}_{j:04d}", "container_id": i, - "channel_keys": [ - f"15TiB-high-resolution-images/group={i:04d}/{j:04d}_{k}.png" - for k in range(3) - ], + "channel0_uris": f"s3://{BUCKET}/15TiB-high-resolution-images/group={i:04d}/{j:04d}_{0}.png", + "channel1_uris": f"s3://{BUCKET}/15TiB-high-resolution-images/group={i:04d}/{j:04d}_{1}.png", + "channel2_uris": f"s3://{BUCKET}/15TiB-high-resolution-images/group={i:04d}/{j:04d}_{2}.png", "applied_scale": 1, } for j in range(NUM_UNITS) @@ -82,20 +98,16 @@ def create_metadata(scale_factor: int): ) -class LoadImage: - def __init__(self): - self._client = boto3.client("s3") +def combine_channels(row: Dict[str, Any]) -> Dict[str, np.ndarray]: + channels = [] + for i in range(3): + data = io.BytesIO(row.pop(f"channel{i}")) + image = Image.open(data) + channels.append(np.array(image)) - def __call__(self, row): - channels = [] - for key in row["channel_keys"]: - data = io.BytesIO() - self._client.download_fileobj(BUCKET, key, data) - image = Image.open(data) - channels.append(np.array(image)) + row["image"] = np.dstack(channels) - row["image"] = np.dstack(channels) - return row + return row def process_image(row: Dict[str, Any]) -> Dict[str, np.ndarray]: @@ -177,11 +189,14 @@ def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: return batch -def main(scale_factor: int): +def main(args: argparse.Namespace): benchmark = Benchmark() + if args.chaos: + start_chaos() + print("Creating metadata") - metadata = create_metadata(scale_factor=scale_factor) + metadata = create_metadata(scale_factor=args.scale_factor) def benchmark_fn(): weights = ViT_B_16_Weights.DEFAULT @@ -189,28 +204,21 @@ def benchmark_fn(): transform = weights.transforms() model_ref = ray.put(model) - # Toggle on features that are required for the pipeline to work. - ctx = DataContext.get_current() - ctx.enable_fallback_to_arrow_object_ext_type = True - ctx.execution_options.actor_locality_enabled = True - - print(f"Starting pipeline with {OVERRIDE_NUM_BLOCKS} blocks") ( - ray.data.from_pandas(metadata, override_num_blocks=OVERRIDE_NUM_BLOCKS) - .map( - LoadImage, - # TODO(mowen): When we fix the deadlocking bug we should increase this to 800. - compute=ActorPoolStrategy(min_size=1, max_size=700), - max_concurrency=4, # needed to prevent image loading from becoming the bottleneck - ) + ray.data.from_pandas(metadata) + .with_column("channel0", download("channel0_uris")) + .with_column("channel1", download("channel1_uris")) + .with_column("channel2", download("channel2_uris")) + .map(combine_channels) .filter(lambda row: row["image"].size != 0) .map(process_image) .flat_map(patch_image) .map_batches(ProcessPatches(transform)) .map_batches( - FakeEmbedPatches, + EmbedPatches, + num_gpus=1, batch_size=BATCH_SIZE, - compute=ActorPoolStrategy(min_size=1, max_size=100), + concurrency=tuple(args.inference_concurrency), fn_constructor_kwargs={"model": model_ref, "device": "cuda"}, ) .write_parquet(WRITE_PATH) @@ -220,7 +228,23 @@ def benchmark_fn(): benchmark.write_result() +def start_chaos(): + assert ray.is_initialized() + + head_node_id = ray.get_runtime_context().get_node_id() + scheduling_strategy = NodeAffinitySchedulingStrategy( + node_id=head_node_id, soft=False + ) + resource_killer = EC2InstanceTerminatorWithGracePeriod.options( + scheduling_strategy=scheduling_strategy + ).remote(head_node_id, max_to_kill=None) + + ray.get(resource_killer.ready.remote()) + + resource_killer.run.remote() + + if __name__ == "__main__": args = parse_args() - scale_factor = args.scale_factor - main(scale_factor) + ray.init() + main(args) diff --git a/release/release_data_tests.yaml b/release/release_data_tests.yaml index ce36281a8e50..9572ae6b2840 100644 --- a/release/release_data_tests.yaml +++ b/release/release_data_tests.yaml @@ -518,45 +518,36 @@ python dataset/gpu_batch_inference.py --data-directory 300G-image-data-synthetic-raw-parquet --data-format parquet --chaos-test -- name: batch_inference_mock_image_pipeline - frequency: manual - working_dir: nightly_tests - - cluster: - cluster_compute: dataset/autoscaling_100_cpu_compute.yaml - - run: - timeout: 3600 - script: > - python dataset/batch_inference_mock_image_pipeline.py - variations: - - __suffix__: regular - - __suffix__: chaos - run: - prepare: > - python setup_chaos.py --chaos TerminateEC2InstanceWithGracePeriod - --batch-size-to-kill 10 --max-to-kill 100 --kill-delay 120 - -- name: batch_inference_mock_image_pipeline_fixed +- name: image_embedding_from_uris_{{case}} frequency: manual - working_dir: nightly_tests + + matrix: + setup: + case: [] + cluster_type: [] + args: [] + adjustments: + - with: + case: fixed_size + cluster_type: fixed_size + args: --inference-concurrency 100 100 + - with: + case: autoscaling + cluster_type: autoscaling + args: --inference-concurrency 1 100 + - with: + case: fixed_size_chaos + cluster_type: fixed_size + args: --inference-concurrency 100 100 --chaos cluster: - cluster_compute: dataset/fixed_size_100_cpu_compute.yaml + cluster_compute: image_embedding_from_uris/{{cluster_type}}_cluster_compute.yaml run: timeout: 3600 - script: > - python dataset/batch_inference_mock_image_pipeline.py + script: python image_embedding_from_uris/main.py {{args}} - variations: - - __suffix__: regular - - __suffix__: chaos - run: - prepare: > - python setup_chaos.py --chaos TerminateEC2InstanceWithGracePeriod - --batch-size-to-kill 10 --max-to-kill 100 --kill-delay 120 - name: batch_inference_hetero_{{case}} frequency: manual From d8be1b228f5208b4831e36f1ba8a02042447de92 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen <128072568+can-anyscale@users.noreply.github.com> Date: Thu, 11 Sep 2025 22:14:13 -0700 Subject: [PATCH 592/634] [core][otel] fix default value for missing metric tags (#56467) When a metric emits a data point with a missing tag, the existing OpenCensus implementation records the tag value as an empty string, whereas the new OpenTelemetry implementation omits the tag entirely. Update the OpenTelemetry implementation to default missing tags to an empty string, ensuring consistent behavior between the two systems. Test: - CI with the test_task_metrics.py test Signed-off-by: Cuong Nguyen --- src/ray/stats/metric.cc | 9 ++-- src/ray/stats/metric.h | 23 +++------ .../tests/metric_with_open_telemetry_test.cc | 47 ++++++++++--------- 3 files changed, 37 insertions(+), 42 deletions(-) diff --git a/src/ray/stats/metric.cc b/src/ray/stats/metric.cc index 4903352ab0d2..89ba9e21303e 100644 --- a/src/ray/stats/metric.cc +++ b/src/ray/stats/metric.cc @@ -116,15 +116,16 @@ void Metric::Record(double value, TagsType tags) { if (::RayConfig::instance().enable_open_telemetry()) { // Collect tags from both the metric-specific tags and the global tags. absl::flat_hash_map open_telemetry_tags; - std::unordered_set tag_keys_set; + // Add default values for missing tag keys. for (const auto &tag_key : tag_keys_) { - tag_keys_set.insert(tag_key.name()); + open_telemetry_tags[tag_key.name()] = ""; } // Insert metric-specific tags that match the expected keys. for (const auto &tag : tags) { const std::string &key = tag.first.name(); - if (tag_keys_set.count(key)) { - open_telemetry_tags[key] = tag.second; + auto it = open_telemetry_tags.find(key); + if (it != open_telemetry_tags.end()) { + it->second = tag.second; } } // Add global tags, overwriting any existing tag keys. diff --git a/src/ray/stats/metric.h b/src/ray/stats/metric.h index cb1d3d83702a..4ae0c4ec3f4a 100644 --- a/src/ray/stats/metric.h +++ b/src/ray/stats/metric.h @@ -334,16 +334,6 @@ inline std::vector convert_tags( return ret; } -inline std::unordered_set build_tag_key_set( - const std::vector &tag_keys) { - std::unordered_set tag_keys_set; - tag_keys_set.reserve(tag_keys.size()); - for (const auto &tag_key : tag_keys) { - tag_keys_set.insert(tag_key); - } - return tag_keys_set; -} - /* This is a helper class to define a metrics. With this class we'll be able to define a multi-view-single-measure metric for @@ -366,9 +356,7 @@ class Stats { const std::string, const std::vector, const std::vector &buckets)> register_func) - : name_(measure), - tag_keys_(convert_tags(tag_keys)), - tag_keys_set_(build_tag_key_set(tag_keys)) { + : name_(measure), tag_keys_(convert_tags(tag_keys)) { auto stats_init = [register_func, measure, description, buckets, this]() { measure_ = std::make_unique(Measure::Register(measure, description, "")); register_func(measure, description, tag_keys_, buckets); @@ -398,10 +386,14 @@ class Stats { absl::flat_hash_map open_telemetry_tags; // Insert metric-specific tags that match the expected keys. + for (const auto &tag_key : tag_keys_) { + open_telemetry_tags[tag_key.name()] = ""; + } for (const auto &tag : open_census_tags) { const std::string &key = tag.first.name(); - if (tag_keys_set_.count(key) != 0) { - open_telemetry_tags[key] = tag.second; + auto it = open_telemetry_tags.find(key); + if (it != open_telemetry_tags.end()) { + it->second = tag.second; } } // Add global tags, overwriting any existing tag keys. @@ -479,7 +471,6 @@ class Stats { const std::string name_; // TODO: Depricate `tag_keys_` once we have fully migrated away from opencensus const std::vector tag_keys_; - const std::unordered_set tag_keys_set_; std::unique_ptr> measure_; }; diff --git a/src/ray/stats/tests/metric_with_open_telemetry_test.cc b/src/ray/stats/tests/metric_with_open_telemetry_test.cc index 4c337d47fb76..0ff403e939c9 100644 --- a/src/ray/stats/tests/metric_with_open_telemetry_test.cc +++ b/src/ray/stats/tests/metric_with_open_telemetry_test.cc @@ -176,14 +176,15 @@ INSTANTIATE_TEST_SUITE_P( GaugeMetricTest, ::testing::Values( // Gauge metric without global tags - GaugeMetricCase{/*metric_name=*/"metric_gauge_test", - /*record_value=*/42.0, - /*record_tags=*/ - {{stats::TagKeyType::Register("Tag1"), "Value1"}, - {stats::TagKeyType::Register("Tag2"), "Value1"}}, - /*global_tags=*/{}, // no global tags - /*expected_tags=*/{{"Tag1", "Value1"}, {"Tag2", "Value1"}}, - /*expected_value=*/42.0}, + GaugeMetricCase{ + /*metric_name=*/"metric_gauge_test", + /*record_value=*/42.0, + /*record_tags=*/ + {{stats::TagKeyType::Register("Tag1"), "Value1"}, + {stats::TagKeyType::Register("Tag2"), "Value1"}}, + /*global_tags=*/{}, // no global tags + /*expected_tags=*/{{"Tag1", "Value1"}, {"Tag2", "Value1"}, {"Tag3", ""}}, + /*expected_value=*/42.0}, // Gauge metric with a single global tag that is metric-specific GaugeMetricCase{/*metric_name=*/"metric_gauge_test", /*record_value=*/52.0, @@ -195,19 +196,20 @@ INSTANTIATE_TEST_SUITE_P( {{"Tag1", "Value2"}, {"Tag2", "Value2"}, {"Tag3", "Global"}}, /*expected_value=*/52.0}, // Gauge metric with a non-metric-specific global tag - GaugeMetricCase{/*metric_name=*/"metric_gauge_test", - /*record_value=*/62.0, - /*record_tags=*/ - {{stats::TagKeyType::Register("Tag1"), "Value3"}, - {stats::TagKeyType::Register("Tag2"), "Value3"}}, - /*global_tags=*/ - { - {stats::TagKeyType::Register("Tag4"), - "Global"} // Tag4 not registered in metric definition - }, - /*expected_tags=*/ - {{"Tag1", "Value3"}, {"Tag2", "Value3"}, {"Tag4", "Global"}}, - /*expected_value=*/62.0}, + GaugeMetricCase{ + /*metric_name=*/"metric_gauge_test", + /*record_value=*/62.0, + /*record_tags=*/ + {{stats::TagKeyType::Register("Tag1"), "Value3"}, + {stats::TagKeyType::Register("Tag2"), "Value3"}}, + /*global_tags=*/ + { + {stats::TagKeyType::Register("Tag4"), + "Global"} // Tag4 not registered in metric definition + }, + /*expected_tags=*/ + {{"Tag1", "Value3"}, {"Tag2", "Value3"}, {"Tag3", ""}, {"Tag4", "Global"}}, + /*expected_value=*/62.0}, // Gauge metric where global tags overwrite record tags GaugeMetricCase{/*metric_name=*/"metric_gauge_test", /*record_value=*/72.0, @@ -230,7 +232,8 @@ INSTANTIATE_TEST_SUITE_P( /*global_tags=*/{}, // no global tags /*expected_tags=*/ {{"Tag1", "Value5"}, // unsupported tag dropped - {"Tag2", "Value5"}}, + {"Tag2", "Value5"}, + {"Tag3", ""}}, /*expected_value=*/82.0})); } // namespace observability From e9670ed2a3b6792e1e9fb2ccb90713602bd2728f Mon Sep 17 00:00:00 2001 From: Yen Hong <89289858+wyhong3103@users.noreply.github.com> Date: Fri, 12 Sep 2025 14:25:27 +0800 Subject: [PATCH 593/634] [train] Add hf trainer support for dictionary of datasets (#56484) To provide support for evaluating model via HuggingFace trainer using a dictionary of Ray datasets. This is to align with the `eval_dataset` argument type in `get_eval_dataloader` in [transformers](https://github.com/huggingface/transformers/blob/v4.56.1/src/transformers/trainer.py#L1196). Signed-off-by: yenhong.wong Co-authored-by: yenhong.wong --- .../transformers/_transformers_utils.py | 13 ++- .../tests/test_torch_transformers_train.py | 68 +++++++++++- .../v2/tests/test_torch_transformers_train.py | 102 ++++++++++++++++++ 3 files changed, 175 insertions(+), 8 deletions(-) diff --git a/python/ray/train/huggingface/transformers/_transformers_utils.py b/python/ray/train/huggingface/transformers/_transformers_utils.py index b195a869f304..7f3eaeefac4a 100644 --- a/python/ray/train/huggingface/transformers/_transformers_utils.py +++ b/python/ray/train/huggingface/transformers/_transformers_utils.py @@ -2,7 +2,7 @@ import shutil from pathlib import Path from tempfile import TemporaryDirectory -from typing import Iterator, Optional, Type +from typing import Iterator, Optional, Type, Union from torch.utils.data import DataLoader, Dataset, IterableDataset @@ -126,12 +126,19 @@ def get_train_dataloader(self) -> DataLoader: return super().get_train_dataloader() def get_eval_dataloader( - self, eval_dataset: Optional[Dataset] = None + self, eval_dataset: Optional[Union[str, Dataset]] = None ) -> DataLoader: if eval_dataset is None: eval_dataset = self.eval_dataset - if isinstance(eval_dataset, _IterableFromIterator): + if ( + isinstance(eval_dataset, str) + and isinstance(self.eval_dataset, dict) + and isinstance(self.eval_dataset[eval_dataset], _IterableFromIterator) + ): + dataset = RayTorchIterableDataset(self.eval_dataset[eval_dataset]) + return DataLoader(dataset, batch_size=1, collate_fn=lambda x: x[0]) + elif isinstance(eval_dataset, _IterableFromIterator): dataset = RayTorchIterableDataset(eval_dataset) return DataLoader(dataset, batch_size=1, collate_fn=lambda x: x[0]) else: diff --git a/python/ray/train/tests/test_torch_transformers_train.py b/python/ray/train/tests/test_torch_transformers_train.py index 94eb03715dea..70b67ec3883e 100644 --- a/python/ray/train/tests/test_torch_transformers_train.py +++ b/python/ray/train/tests/test_torch_transformers_train.py @@ -55,6 +55,7 @@ def ray_start_8_cpus(): "save_steps": None, "logging_steps": None, "no_cuda": False, + "use_dict_eval_datasets": False, }, "steps_gpu": { "evaluation_strategy": "steps", @@ -64,6 +65,7 @@ def ray_start_8_cpus(): "save_steps": STEPS_PER_EPOCH * 2, "logging_steps": 1, "no_cuda": False, + "use_dict_eval_datasets": False, }, "steps_cpu": { "evaluation_strategy": "steps", @@ -73,6 +75,7 @@ def ray_start_8_cpus(): "save_steps": STEPS_PER_EPOCH, "logging_steps": 1, "no_cuda": True, + "use_dict_eval_datasets": False, }, } @@ -81,14 +84,27 @@ def train_func(config): # Datasets if config["use_ray_data"]: train_ds_shard = ray.train.get_dataset_shard("train") - eval_ds_shard = ray.train.get_dataset_shard("eval") - train_dataset = train_ds_shard.iter_torch_batches( batch_size=BATCH_SIZE_PER_WORKER ) - eval_dataset = eval_ds_shard.iter_torch_batches( - batch_size=BATCH_SIZE_PER_WORKER - ) + if config["use_dict_eval_datasets"]: + eval_ds_shard_1 = ray.train.get_dataset_shard("eval_1") + eval_ds_shard_2 = ray.train.get_dataset_shard("eval_2") + + eval_dataset = { + "eval_1": eval_ds_shard_1.iter_torch_batches( + batch_size=BATCH_SIZE_PER_WORKER + ), + "eval_2": eval_ds_shard_2.iter_torch_batches( + batch_size=BATCH_SIZE_PER_WORKER + ), + } + else: + eval_ds_shard = ray.train.get_dataset_shard("eval") + + eval_dataset = eval_ds_shard.iter_torch_batches( + batch_size=BATCH_SIZE_PER_WORKER + ) else: train_df = pd.read_json(train_data) validation_df = pd.read_json(validation_data) @@ -201,6 +217,48 @@ def test_e2e_ray_data(ray_start_6_cpus_2_gpus, config_id): assert "eval_loss" in result.metrics +@pytest.mark.parametrize("config_id", ["steps_gpu", "steps_cpu"]) +def test_e2e_dict_eval_ray_data(ray_start_6_cpus_2_gpus, config_id): + train_loop_config = CONFIGURATIONS[config_id] + + # Must specify `max_steps` for Iterable Dataset + train_loop_config["use_ray_data"] = True + train_loop_config["use_dict_eval_datasets"] = True + train_loop_config["max_steps"] = MAX_STEPS + + # Calculate the num of Ray training iterations + num_iterations = MAX_STEPS // train_loop_config["save_steps"] + + train_df = pd.read_json(train_data) + validation_df = pd.read_json(validation_data) + + ray_train_ds = ray.data.from_pandas(train_df) + ray_eval_ds_1 = ray.data.from_pandas(validation_df) + ray_eval_ds_2 = ray.data.from_pandas(validation_df) + + use_gpu = not train_loop_config["no_cuda"] + + trainer = TorchTrainer( + train_func, + train_loop_config=train_loop_config, + scaling_config=ScalingConfig(num_workers=NUM_WORKERS, use_gpu=use_gpu), + datasets={ + "train": ray_train_ds, + "eval_1": ray_eval_ds_1, + "eval_2": ray_eval_ds_2, + }, + ) + result = trainer.fit() + + assert result.metrics["step"] == MAX_STEPS + assert result.metrics["training_iteration"] == num_iterations + assert result.checkpoint + assert isinstance(result.checkpoint, Checkpoint) + assert len(result.best_checkpoints) == num_iterations + assert "eval_eval_1_loss" in result.metrics + assert "eval_eval_2_loss" in result.metrics + + # Tests if Ray Tune works correctly. def test_tune(ray_start_8_cpus): train_loop_config = CONFIGURATIONS["steps_cpu"] diff --git a/python/ray/train/v2/tests/test_torch_transformers_train.py b/python/ray/train/v2/tests/test_torch_transformers_train.py index 1484aef66893..fad84fe4f693 100644 --- a/python/ray/train/v2/tests/test_torch_transformers_train.py +++ b/python/ray/train/v2/tests/test_torch_transformers_train.py @@ -303,6 +303,108 @@ def train_func(config): assert "eval_loss" in result.metrics +@pytest.mark.parametrize("config_id", ["steps_cpu"]) +def test_e2e_dict_eval_ray_data(ray_start_6_cpus_2_gpus, config_id): + def train_func(config): + # Datasets + if config["use_ray_data"]: + train_ds_shard = ray.train.get_dataset_shard("train") + eval_ds_shard_1 = ray.train.get_dataset_shard("eval_1") + eval_ds_shard_2 = ray.train.get_dataset_shard("eval_2") + + train_dataset = train_ds_shard.iter_torch_batches( + batch_size=BATCH_SIZE_PER_WORKER + ) + eval_dataset = { + "eval_1": eval_ds_shard_1.iter_torch_batches( + batch_size=BATCH_SIZE_PER_WORKER + ), + "eval_2": eval_ds_shard_2.iter_torch_batches( + batch_size=BATCH_SIZE_PER_WORKER + ), + } + else: + train_df = pd.read_json(train_data) + validation_df = pd.read_json(validation_data) + + train_dataset = Dataset.from_pandas(train_df) + eval_dataset = Dataset.from_pandas(validation_df) + + # Model + model_config = AutoConfig.from_pretrained(MODEL_NAME) + model = AutoModelForCausalLM.from_config(model_config) + + # HF Transformers Trainer + training_args = TrainingArguments( + f"{MODEL_NAME}-wikitext2", + evaluation_strategy=config["evaluation_strategy"], + logging_strategy=config["logging_strategy"], + save_strategy=config["save_strategy"], + eval_steps=config["eval_steps"], + save_steps=config["save_steps"], + logging_steps=config["logging_steps"], + num_train_epochs=config.get("num_train_epochs", MAX_EPOCHS), + max_steps=config.get("max_steps", -1), + learning_rate=config.get("learning_rate", 2e-5), + per_device_train_batch_size=BATCH_SIZE_PER_WORKER, + per_device_eval_batch_size=BATCH_SIZE_PER_WORKER, + weight_decay=0.01, + disable_tqdm=True, + no_cuda=config["no_cuda"], + report_to="none", + ) + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + ) + + # Report to Ray Train + trainer.add_callback(RayTrainReportCallback()) + trainer = prepare_trainer(trainer) + + # Start Training + trainer.train() + + train_loop_config = CONFIGURATIONS[config_id] + + # Must specify `max_steps` for Iterable Dataset + train_loop_config["use_ray_data"] = True + train_loop_config["max_steps"] = MAX_STEPS + + # Calculate the num of Ray training iterations + num_iterations = MAX_STEPS // train_loop_config["save_steps"] + + train_df = pd.read_json(train_data) + validation_df = pd.read_json(validation_data) + + ray_train_ds = ray.data.from_pandas(train_df) + ray_eval_ds_1 = ray.data.from_pandas(validation_df) + ray_eval_ds_2 = ray.data.from_pandas(validation_df) + + use_gpu = not train_loop_config["no_cuda"] + + trainer = TorchTrainer( + train_func, + train_loop_config=train_loop_config, + scaling_config=ScalingConfig(num_workers=NUM_WORKERS, use_gpu=use_gpu), + datasets={ + "train": ray_train_ds, + "eval_1": ray_eval_ds_1, + "eval_2": ray_eval_ds_2, + }, + ) + result = trainer.fit() + + assert result.metrics["step"] == MAX_STEPS + assert result.checkpoint + assert isinstance(result.checkpoint, Checkpoint) + assert len(result.best_checkpoints) == num_iterations + assert "eval_eval_1_loss" in result.metrics + assert "eval_eval_2_loss" in result.metrics + + if __name__ == "__main__": import sys From df5951e09e329e48a1b85e00ecf526fdf7da3d2d Mon Sep 17 00:00:00 2001 From: goutamvenkat-anyscale Date: Thu, 11 Sep 2025 23:36:49 -0700 Subject: [PATCH 594/634] [Data] - Improve performance for `unify_schemas` (#55880) ## Why are these changes needed? Find all diverging schemas, coalesce them if possible, and do so recursively in the presence of structs. Perform a single pass to gather stats for all columns across all schemas. ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Goutam V Signed-off-by: Goutam V. --- .../_internal/arrow_ops/transform_pyarrow.py | 330 ++++++++++-------- .../ray/data/tests/test_transform_pyarrow.py | 40 +-- 2 files changed, 188 insertions(+), 182 deletions(-) diff --git a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py index ae0c72389c07..d52f97eb9d41 100644 --- a/python/ray/data/_internal/arrow_ops/transform_pyarrow.py +++ b/python/ray/data/_internal/arrow_ops/transform_pyarrow.py @@ -1,5 +1,6 @@ import logging -from typing import TYPE_CHECKING, Dict, List, Optional, Union +from collections import defaultdict +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union import numpy as np from packaging.version import parse as parse_version @@ -10,6 +11,7 @@ from ray.air.util.tensor_extensions.arrow import ( MIN_PYARROW_VERSION_CHUNKED_ARRAY_TO_NUMPY_ZERO_COPY_ONLY, PYARROW_VERSION, + get_arrow_extension_tensor_types, ) try: @@ -155,180 +157,210 @@ def take_table( return table -def unify_schemas( - schemas: List["pyarrow.Schema"], *, promote_types: bool = False -) -> "pyarrow.Schema": - """Version of `pyarrow.unify_schemas()` which also handles checks for - variable-shaped tensors in the given schemas. - - This function scans all input schemas to identify columns that contain - variable-shaped tensors or objects. For tensor columns, it ensures the - use of appropriate tensor types (including variable-shaped tensor types). - For object columns, it uses a specific object type to accommodate any - objects present. Additionally, it handles columns with null-typed lists - by determining their actual types from the given schemas. - - Currently, it disallows the concatenation of tensor columns and - pickled object columsn for performance reasons. +def _reconcile_diverging_fields( + unique_schemas: List["pyarrow.Schema"], + promote_types: bool, +) -> Dict[str, Any]: """ - import pyarrow as pa - - from ray.air.util.object_extensions.arrow import ArrowPythonObjectType - from ray.air.util.tensor_extensions.arrow import ( - ArrowTensorType, - ArrowVariableShapedTensorType, - ) + Identify and reconcile fields whose presence or types differ across the provided schemas. - # The schema metadata might be unhashable. - # We need schemas to be hashable for unification - schemas = [schema.remove_metadata() for schema in schemas] - try: - if len(set(schemas)) == 1: - # Early exit because unifying can be expensive - return schemas.pop() - except Exception as e: - # Unsure if there are cases where schemas are NOT hashable - logger.debug(f"Failed to hash the schemas (for deduplication): {e}") + Args: + unique_schemas: List of PyArrow schemas to find diverging fields in. + promote_types: Whether to promote types. - schemas_to_unify = [] - schema_field_overrides = {} + Returns: + A dictionary of diverging fields with their reconciled types. + """ + from ray.air.util.object_extensions.arrow import ArrowPythonObjectType - # Rollup columns with opaque (null-typed) lists, to override types in - # the following for-loop. - cols_with_null_list = set() + reconciled_fields = {} + field_types = defaultdict(set) # field_name -> set of types seen so far + field_flags = defaultdict( + lambda: defaultdict(bool) + ) # field_name -> dict of boolean flags + + # Process schemas and reconcile on-the-fly + for schema in unique_schemas: + for field_name in schema.names: + if field_name in reconciled_fields: + # If the field has already been reconciled, skip it. + continue + + field_type = schema.field(field_name).type + field_types[field_name].add(field_type) + flags = field_flags[field_name] + + # Update flags + flags["has_object"] |= isinstance(field_type, ArrowPythonObjectType) + flags["has_tensor"] |= isinstance( + field_type, get_arrow_extension_tensor_types() + ) + flags["has_list"] |= pyarrow.types.is_list(field_type) + flags["has_null"] |= pyarrow.types.is_null(field_type) + flags["has_struct"] |= pyarrow.types.is_struct(field_type) - all_columns = set() - for schema in schemas: - for col_name in schema.names: - # Check for duplicate field names in this schema - if schema.names.count(col_name) > 1: - # This is broken for Pandas blocks and broken with the logic here + # Check for object-tensor conflict + if flags["has_object"] and flags["has_tensor"]: raise ValueError( - f"Schema {schema} has multiple fields with the same name: {col_name}" + f"Found columns with both objects and tensors: {field_name}" ) - col_type = schema.field(col_name).type - if pa.types.is_list(col_type) and pa.types.is_null(col_type.value_type): - cols_with_null_list.add(col_name) - all_columns.add(col_name) + # Reconcile immediately if it's a special type and if it's divergent. + if any(flags.values()) and len(field_types[field_name]) > 1: + reconciled_value = _reconcile_field( + non_null_types=field_types[field_name], + promote_types=promote_types, + ) + if reconciled_value is not None: + reconciled_fields[field_name] = reconciled_value + + return reconciled_fields + + +def _reconcile_field( + non_null_types: List[pyarrow.DataType], + promote_types: bool = False, +) -> Optional[pyarrow.DataType]: + """ + Reconcile a single divergent field across schemas. + + Returns reconciled type or None if default PyArrow handling is sufficient. + """ + from ray.air.util.object_extensions.arrow import ArrowPythonObjectType from ray.air.util.tensor_extensions.arrow import ( - get_arrow_extension_fixed_shape_tensor_types, + ArrowTensorType, + ArrowVariableShapedTensorType, get_arrow_extension_tensor_types, ) - arrow_tensor_types = get_arrow_extension_tensor_types() - arrow_fixed_shape_tensor_types = get_arrow_extension_fixed_shape_tensor_types() - - columns_with_objects = set() - columns_with_tensor_array = set() - columns_with_struct = set() - for col_name in all_columns: - for s in schemas: - if col_name in s.names: - if isinstance(s.field(col_name).type, ArrowPythonObjectType): - columns_with_objects.add(col_name) - if isinstance(s.field(col_name).type, arrow_tensor_types): - columns_with_tensor_array.add(col_name) - if isinstance(s.field(col_name).type, pa.StructType): - columns_with_struct.add(col_name) - - if len(columns_with_objects.intersection(columns_with_tensor_array)) > 0: - # This is supportable if we use object type, but it will be expensive - raise ValueError( - "Found columns with both objects and tensors: " - f"{columns_with_tensor_array.intersection(columns_with_objects)}" + if not non_null_types: + return None + + tensor_types = get_arrow_extension_tensor_types() + + # Handle special cases in priority order + + # 1. Tensor fields + tensor_field_types = [t for t in non_null_types if isinstance(t, tensor_types)] + if tensor_field_types: + needs_variable_shape = ArrowTensorType._need_variable_shaped_tensor_array( + tensor_field_types ) - for col_name in columns_with_tensor_array: - tensor_array_types = [ - s.field(col_name).type - for s in schemas - if col_name in s.names - and isinstance(s.field(col_name).type, arrow_tensor_types) - ] - # Check if we have missing tensor fields (some schemas don't have this field) - has_missing_fields = len(tensor_array_types) < len(schemas) - - # Convert to variable-shaped if needed or if we have missing fields - if ( - ArrowTensorType._need_variable_shaped_tensor_array(tensor_array_types) - or has_missing_fields - ): - if isinstance(tensor_array_types[0], ArrowVariableShapedTensorType): - new_type = tensor_array_types[0] - elif isinstance(tensor_array_types[0], arrow_fixed_shape_tensor_types): - new_type = ArrowVariableShapedTensorType( - dtype=tensor_array_types[0].scalar_type, - ndim=len(tensor_array_types[0].shape), - ) + if needs_variable_shape: + first_tensor = tensor_field_types[0] + if isinstance(first_tensor, ArrowVariableShapedTensorType): + return first_tensor else: - raise ValueError( - "Detected need for variable shaped tensor representation, " - f"but schema is not ArrayTensorType: {tensor_array_types[0]}" + # Convert fixed-shape to variable-shape + return ArrowVariableShapedTensorType( + dtype=first_tensor.scalar_type, ndim=len(first_tensor.shape) ) - schema_field_overrides[col_name] = new_type - - for col_name in columns_with_objects: - schema_field_overrides[col_name] = ArrowPythonObjectType() - for col_name in columns_with_struct: - field_types = [s.field(col_name).type for s in schemas] + # 2. Object fields + if any(isinstance(t, ArrowPythonObjectType) for t in non_null_types): + return ArrowPythonObjectType() - # Unify struct schemas + # 3. Struct fields (recursive unification) + struct_types = [t for t in non_null_types if pyarrow.types.is_struct(t)] + if struct_types: + # Convert struct types to schemas struct_schemas = [] - for t in field_types: - if t is not None and pa.types.is_struct(t): - struct_schemas.append(pa.schema(list(t))) - else: - struct_schemas.append(pa.schema([])) + for t in non_null_types: + if pyarrow.types.is_struct(t): + struct_schemas.append(pyarrow.schema(list(t))) + # Recursively unify + unified_struct = unify_schemas(struct_schemas, promote_types=promote_types) + return pyarrow.struct(list(unified_struct)) + + # 4. Null-typed list fields (Need this pyarrow < 14.0.0) + null_lists = [ + t + for t in non_null_types + if pyarrow.types.is_list(t) and pyarrow.types.is_null(t.value_type) + ] + if null_lists: + # Find first non-null list type + for t in non_null_types: + if not (pyarrow.types.is_list(t) and pyarrow.types.is_null(t.value_type)): + return t + # At this phase, we have no special types to reconcile, so return None. Arrow will fail to unify. + return None + + +def _unify_schemas_pyarrow( + schemas: List["pyarrow.Schema"], promote_types: bool = False +) -> "pyarrow.Schema": + """Wrapper for pyarrow.unify_schemas with version compatibility.""" + if get_pyarrow_version() < MIN_PYARROW_VERSION_TYPE_PROMOTION: + return pyarrow.unify_schemas(schemas) - unified_struct_schema = unify_schemas( - struct_schemas, promote_types=promote_types - ) + promote_options = "permissive" if promote_types else "default" + return pyarrow.unify_schemas(schemas, promote_options=promote_options) - schema_field_overrides[col_name] = pa.struct(list(unified_struct_schema)) - - if cols_with_null_list: - # For each opaque list column, iterate through all schemas until we find - # a valid value_type that can be used to override the column types in - # the following for-loop. - for col_name in cols_with_null_list: - for schema in schemas: - col_type = schema.field(col_name).type - if not pa.types.is_list(col_type) or not pa.types.is_null( - col_type.value_type - ): - schema_field_overrides[col_name] = col_type - break - - if schema_field_overrides: - # Go through all schemas and update the types of columns from the above loop. - for schema in schemas: - for col_name, col_new_type in schema_field_overrides.items(): - if col_name in schema.names: - var_shaped_col = schema.field(col_name).with_type(col_new_type) - col_idx = schema.get_field_index(col_name) - schema = schema.set(col_idx, var_shaped_col) - schemas_to_unify.append(schema) - else: - schemas_to_unify = schemas - try: - if get_pyarrow_version() < MIN_PYARROW_VERSION_TYPE_PROMOTION: - return pyarrow.unify_schemas(schemas_to_unify) +def unify_schemas( + schemas: List["pyarrow.Schema"], *, promote_types: bool = False +) -> "pyarrow.Schema": + """ + Unify schemas handling Ray-specific types (tensors, objects, etc.). - # NOTE: By default type promotion (from "smaller" to "larger" types) is disabled, - # allowing only promotion b/w nullable and non-nullable ones - arrow_promote_types_mode = "permissive" if promote_types else "default" + Falls back to PyArrow's unify_schemas when possible, with custom + handling for tensor arrays, object types, and recursive struct unification. + """ + if not schemas: + raise ValueError("No schemas provided for unify_schemas") - return pyarrow.unify_schemas( - schemas_to_unify, promote_options=arrow_promote_types_mode - ) - except Exception as e: - schemas_str = "\n-----\n".join([str(s) for s in schemas_to_unify]) + # Deduplicate schemas. Calling this before PyArrow's unify_schemas is more efficient (100x faster). - logger.error(f"Failed to unify schemas: {schemas_str}", exc_info=e) + # Remove metadata for hashability + schemas[0].remove_metadata() + schemas_to_unify = [schemas[0]] + for schema in schemas[1:]: + schema.remove_metadata() + if not schema.equals(schemas[0]): + schemas_to_unify.append(schema) + pyarrow_exception = None + # If there is only one schema, return it + if len(schemas_to_unify) == 1: + return schemas_to_unify[0] + + # Try PyArrow's unification first, only reconcile for tensor fields + try: + return _unify_schemas_pyarrow(schemas_to_unify, promote_types) + except (pyarrow.lib.ArrowTypeError, pyarrow.lib.ArrowInvalid) as e: + # If we raise only on non tensor errors, it fails to unify PythonObjectType and pyarrow primitives. + # Look at test_pyarrow_conversion_error_handling for an example. + pyarrow_exception = e + pass + + # Reconcile diverging fields + overrides = _reconcile_diverging_fields(schemas_to_unify, promote_types) + + # At this point, we're not able to reconcile the fields, so raise the original exception. + if not overrides: + raise pyarrow_exception + + # Apply overrides to schemas + updated_schemas = [] + for schema in schemas_to_unify: + for name, new_type in overrides.items(): + try: + idx = schema.get_field_index(name) + field = schema.field(name).with_type(new_type) + schema = schema.set(idx, field) + except KeyError: + pass + updated_schemas.append(schema) + schemas_to_unify = updated_schemas + + # Final unification with overrides applied + try: + return _unify_schemas_pyarrow(schemas_to_unify, promote_types) + except Exception as e: + schemas_str = "\n-----\n".join(str(s) for s in schemas_to_unify) + logger.error(f"Failed to unify schemas: {schemas_str}", exc_info=e) raise diff --git a/python/ray/data/tests/test_transform_pyarrow.py b/python/ray/data/tests/test_transform_pyarrow.py index 9c253e4ea4d0..6cefc62ce113 100644 --- a/python/ray/data/tests/test_transform_pyarrow.py +++ b/python/ray/data/tests/test_transform_pyarrow.py @@ -572,22 +572,6 @@ def test_unify_schemas(unify_schemas_basic_schemas, unify_schemas_multicol_schem ) -def test_unify_schemas_null_typed_lists(unify_schemas_null_typed_lists_schemas): - """Test handling of null-typed lists (cols_with_null_list functionality).""" - schemas = unify_schemas_null_typed_lists_schemas - - # Should find valid value_type from schema2 and override - result = unify_schemas([schemas["null_list"], schemas["int_list"]]) - assert result == schemas["expected"] - - # Test with multiple schemas, some with null types - result = unify_schemas( - [schemas["null_list"], schemas["int_list"], schemas["string_list"]] - ) - # Should use the first non-null type found (int32) - assert result == schemas["expected"] - - def test_unify_schemas_object_types(unify_schemas_object_types_schemas): """Test handling of object types (columns_with_objects functionality).""" schemas = unify_schemas_object_types_schemas @@ -626,6 +610,10 @@ def test_unify_schemas_objects_and_tensors(unify_schemas_objects_and_tensors_sch unify_schemas(unify_schemas_objects_and_tensors_schemas) +@pytest.mark.skipif( + get_pyarrow_version() < parse_version("17.0.0"), + reason="Requires PyArrow version 17 or higher", +) def test_unify_schemas_missing_tensor_fields( unify_schemas_missing_tensor_fields_schemas, ): @@ -2224,7 +2212,7 @@ def struct_with_null_tensor_values_expected(): "struct", pa.struct( [ - ("tensor", ArrowVariableShapedTensorType(pa.float32(), 2)), + ("tensor", ArrowTensorTypeV2((2,), pa.float32())), ("value", pa.int64()), ] ), @@ -2750,20 +2738,6 @@ def struct_variable_shaped_tensor_expected(): } -@pytest.fixture -def unify_schemas_null_typed_lists_schemas(): - """Fixture for null typed lists unify schemas test data.""" - schema1 = pa.schema([("list_col", pa.list_(pa.null()))]) - schema2 = pa.schema([("list_col", pa.list_(pa.int32()))]) - schema3 = pa.schema([("list_col", pa.list_(pa.string()))]) - return { - "null_list": schema1, - "int_list": schema2, - "string_list": schema3, - "expected": pa.schema([("list_col", pa.list_(pa.int32()))]), - } - - @pytest.fixture def unify_schemas_object_types_schemas(): """Fixture for object types unify schemas test data.""" @@ -2825,7 +2799,7 @@ def unify_schemas_missing_tensor_fields_schemas(): "struct", pa.struct( [ - ("tensor", ArrowVariableShapedTensorType(pa.int32(), 2)), + ("tensor", ArrowTensorType((2, 2), pa.int32())), ("value", pa.int64()), ] ), @@ -2887,7 +2861,7 @@ def unify_schemas_nested_struct_tensors_schemas(): [ ( "tensor", - ArrowVariableShapedTensorType(pa.float32(), 2), + ArrowTensorType((3, 3), pa.float32()), ), ("data", pa.string()), ] From f2d046bdcd4df4520b7f7fba52a2b2ff681e747f Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Fri, 12 Sep 2025 01:02:25 -0700 Subject: [PATCH 595/634] [Data] Add text embedding release test (#56459) ## Why are these changes needed? This PR adds a release test based on a real-user text embedding workload. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Balaji Veeramani --- .../autoscaling_cluster_compute.yaml | 21 +++ .../dataset/text_embedding/create_dataset.py | 102 ++++++++++++ .../fixed_size_cluster_compute.yaml | 21 +++ .../dataset/text_embedding/main.py | 149 ++++++++++++++++++ .../byod/byod_install_text_embedding.sh | 6 + release/release_data_tests.yaml | 31 ++++ 6 files changed, 330 insertions(+) create mode 100644 release/nightly_tests/dataset/text_embedding/autoscaling_cluster_compute.yaml create mode 100644 release/nightly_tests/dataset/text_embedding/create_dataset.py create mode 100644 release/nightly_tests/dataset/text_embedding/fixed_size_cluster_compute.yaml create mode 100644 release/nightly_tests/dataset/text_embedding/main.py create mode 100755 release/ray_release/byod/byod_install_text_embedding.sh diff --git a/release/nightly_tests/dataset/text_embedding/autoscaling_cluster_compute.yaml b/release/nightly_tests/dataset/text_embedding/autoscaling_cluster_compute.yaml new file mode 100644 index 000000000000..b601a66dc843 --- /dev/null +++ b/release/nightly_tests/dataset/text_embedding/autoscaling_cluster_compute.yaml @@ -0,0 +1,21 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +advanced_configurations_json: + IamInstanceProfile: {"Name": "ray-autoscaler-v1"} + +head_node_type: + name: head-node + instance_type: r6a.8xlarge + resources: + cpu: 0 + +worker_node_types: + - name: gpu-node + instance_type: g5.xlarge + min_workers: 1 + max_workers: 100 + use_spot: false + +flags: + allow-cross-zone-autoscaling: true diff --git a/release/nightly_tests/dataset/text_embedding/create_dataset.py b/release/nightly_tests/dataset/text_embedding/create_dataset.py new file mode 100644 index 000000000000..e8e619d88e19 --- /dev/null +++ b/release/nightly_tests/dataset/text_embedding/create_dataset.py @@ -0,0 +1,102 @@ +import pyarrow as pa +import uuid +import random +import string +import ray +import pyarrow.parquet as pq +from tqdm import tqdm + +STRING_PLACEHOLDER = "" +UUID_PLACEHOLDER = uuid.UUID(int=0) +INT_PLACEHOLDER = 0 + +TARGET_SIZE_BYTES = 4096 +NUM_FILES = 50 + +SCHEMA = pa.schema( + [ + ("metadata00", pa.string()), + ("metadata01", pa.list_(pa.binary(16))), + ("metadata02", pa.string()), + ("metadata03", pa.uint64()), + ("metadata04", pa.list_(pa.binary(16))), + ("metadata05", pa.list_(pa.binary(16))), + ("metadata06", pa.binary(16)), + ("metadata07", pa.string()), + ("metadata08", pa.binary(16)), + ("metadata09", pa.uint64()), + ("metadata10", pa.binary(16)), + ("metadata11", pa.list_(pa.binary(16))), + ("metadata12", pa.uint64()), + ("metadata13", pa.uint64()), + ("metadata14", pa.list_(pa.binary(16))), + ("span_text", pa.string()), + ("metadata15", pa.binary(16)), + ("metadata16", pa.string()), + ("metadata17", pa.list_(pa.binary(16))), + ("metadata18", pa.list_(pa.binary(16))), + ] +) + + +def random_word(min_len=3, max_len=8): + length = random.randint(min_len, max_len) + return "".join(random.choices(string.ascii_lowercase, k=length)) + + +def create_random_sentence(): + sentence = "" + while len(sentence.encode("utf-8")) < TARGET_SIZE_BYTES: + word = random_word() + sentence += word + " " # space between words + + # Trim to exact size + sentence_bytes = sentence.encode("utf-8")[:TARGET_SIZE_BYTES] + return sentence_bytes.decode("utf-8", errors="ignore") + + +def create_row(): + return { + "metadata00": STRING_PLACEHOLDER, + "metadata01": [UUID_PLACEHOLDER.bytes], + "metadata02": STRING_PLACEHOLDER, + "metadata03": INT_PLACEHOLDER, + "metadata04": [UUID_PLACEHOLDER.bytes], + "metadata05": [UUID_PLACEHOLDER.bytes], + "metadata06": UUID_PLACEHOLDER.bytes, + "metadata07": STRING_PLACEHOLDER, + "metadata08": UUID_PLACEHOLDER.bytes, + "metadata09": INT_PLACEHOLDER, + "metadata10": UUID_PLACEHOLDER.bytes, + "metadata11": [UUID_PLACEHOLDER.bytes], + "metadata12": INT_PLACEHOLDER, + "metadata13": None if random.random() < 0.01 else INT_PLACEHOLDER, + "metadata14": [UUID_PLACEHOLDER.bytes], + "span_text": create_random_sentence(), + "metadata15": UUID_PLACEHOLDER.bytes, + "metadata16": STRING_PLACEHOLDER, + "metadata17": [UUID_PLACEHOLDER.bytes], + "metadata18": [UUID_PLACEHOLDER.bytes], + } + + +@ray.remote +def write_table(i: int): + rows = [] + for _ in range(20_000): + rows.append(create_row()) + + table = pa.Table.from_pylist(rows, schema=SCHEMA) + pq.write_table( + table, f"s3://ray-benchmark-data-internal-us-west-2/text-spans/{i}.parquet" + ) + + +refs = [write_table.remote(i) for i in range(NUM_FILES)] + +pbar = tqdm(total=len(refs)) +while refs: + ready, refs = ray.wait(refs, num_returns=1) + pbar.update(len(ready)) + +pbar.close() diff --git a/release/nightly_tests/dataset/text_embedding/fixed_size_cluster_compute.yaml b/release/nightly_tests/dataset/text_embedding/fixed_size_cluster_compute.yaml new file mode 100644 index 000000000000..eb51bba4b5ab --- /dev/null +++ b/release/nightly_tests/dataset/text_embedding/fixed_size_cluster_compute.yaml @@ -0,0 +1,21 @@ +cloud_id: {{env["ANYSCALE_CLOUD_ID"]}} +region: us-west-2 + +advanced_configurations_json: + IamInstanceProfile: {"Name": "ray-autoscaler-v1"} + +head_node_type: + name: head-node + instance_type: r6a.8xlarge + resources: + cpu: 0 + +worker_node_types: + - name: gpu-node + instance_type: g5.xlarge + min_workers: 100 + max_workers: 100 + use_spot: false + +flags: + allow-cross-zone-autoscaling: true diff --git a/release/nightly_tests/dataset/text_embedding/main.py b/release/nightly_tests/dataset/text_embedding/main.py new file mode 100644 index 000000000000..a74e02657003 --- /dev/null +++ b/release/nightly_tests/dataset/text_embedding/main.py @@ -0,0 +1,149 @@ +import argparse +from typing import Dict +import uuid +import boto3 +import json + +import numpy as np +import pyarrow as pa +from sentence_transformers import SentenceTransformer +import torch + +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy +from ray._private.test_utils import EC2InstanceTerminatorWithGracePeriod +import ray + +from benchmark import Benchmark + +BATCH_SIZE = 128 + +# This dataset has 50 files, each with 20,000 rows of <1024-token text spans. It +# includes one empty Parquet file and some nulls. See `create_dataset.py` for details. +INPUT_PREFIX = "s3://ray-benchmark-data-internal-us-west-2/text-spans" +# Add a random prefix to avoid conflicts between different runs. +OUTPUT_PREFIX = f"s3://ray-data-write-benchmark/{uuid.uuid4().hex}" + +# These are used to fetch the HF token from AWS Secrets Manager. +SECRET_REGION_NAME = "us-west-2" +SECRET_ID = ( + "arn:aws:secretsmanager:us-west-2:188439194153:secret:release_test_hf_token-p3Lcqy" +) + +# FIXME: We need to explicitly define the schema and specify lists of variable-size +# binaries because Ray Data can't handle lists of fixed-size binaries. +SCHEMA = pa.schema( + [ + ("metadata00", pa.string()), + ("metadata01", pa.list_(pa.binary())), + ("metadata02", pa.string()), + ("metadata03", pa.uint64()), + ("metadata04", pa.list_(pa.binary())), + ("metadata05", pa.list_(pa.binary())), + ("metadata06", pa.binary()), + ("metadata07", pa.string()), + ("metadata08", pa.binary()), + ("metadata09", pa.uint64()), + ("metadata10", pa.binary()), + ("metadata11", pa.list_(pa.binary())), + ("metadata12", pa.uint64()), + ("metadata13", pa.uint64()), + ("metadata14", pa.list_(pa.binary())), + ("span_text", pa.string()), + ("metadata15", pa.binary()), + ("metadata16", pa.string()), + ("metadata17", pa.list_(pa.binary())), + ("metadata18", pa.list_(pa.binary())), + ] +) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--inference-concurrency", + nargs=2, + type=int, + required=True, + help="The minimum and maximum concurrency for the inference operator.", + ) + parser.add_argument( + "--chaos", + action="store_true", + help=( + "Whether to enable chaos. If set, this script terminates one worker node " + "every minute with a grace period." + ), + ) + return parser.parse_args() + + +def main(args: argparse.Namespace): + benchmark = Benchmark() + + if args.chaos: + start_chaos() + + def benchmark_fn(): + ( + ray.data.read_parquet(INPUT_PREFIX, schema=SCHEMA) + .repartition(target_num_rows_per_block=256) + .map_batches( + EncodingUDF, + concurrency=tuple(args.inference_concurrency), + num_gpus=1, + batch_size=BATCH_SIZE, + fn_constructor_kwargs={"model": "BAAI/bge-m3", "token": get_hf_token()}, + ) + .write_parquet(OUTPUT_PREFIX, mode="overwrite") + ) + + benchmark.run_fn("main", benchmark_fn) + benchmark.write_result() + + +def start_chaos(): + assert ray.is_initialized() + + head_node_id = ray.get_runtime_context().get_node_id() + scheduling_strategy = NodeAffinitySchedulingStrategy( + node_id=head_node_id, soft=False + ) + resource_killer = EC2InstanceTerminatorWithGracePeriod.options( + scheduling_strategy=scheduling_strategy + ).remote(head_node_id, max_to_kill=None) + + ray.get(resource_killer.ready.remote()) + + resource_killer.run.remote() + + +class EncodingUDF: + def __init__(self, model: str, token: str): + device = "cuda" if torch.cuda.is_available() else "cpu" + self._model = SentenceTransformer( + model, + device=device, + token=token, + model_kwargs={"torch_dtype": torch.bfloat16}, + ) + + def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: + batch["vector"] = self._model.encode( + batch["span_text"], batch_size=BATCH_SIZE, convert_to_numpy=True + ) + return batch + + +def get_hf_token() -> str: + session = boto3.session.Session() + client = session.client( + service_name="secretsmanager", region_name=SECRET_REGION_NAME + ) + secret_string = client.get_secret_value(SecretId=SECRET_ID)["SecretString"] + return json.loads(secret_string)["HF_TOKEN"] + + +if __name__ == "__main__": + ray.init() + args = parse_args() + main(args) diff --git a/release/ray_release/byod/byod_install_text_embedding.sh b/release/ray_release/byod/byod_install_text_embedding.sh new file mode 100755 index 000000000000..7bb9a5d9f4fa --- /dev/null +++ b/release/ray_release/byod/byod_install_text_embedding.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# shellcheck disable=SC2102 + +set -exo pipefail + +pip3 install --no-cache-dir --upgrade-strategy only-if-needed sentence-transformers==5.1.0 torch==2.8.0 diff --git a/release/release_data_tests.yaml b/release/release_data_tests.yaml index 9572ae6b2840..c9c8cc442456 100644 --- a/release/release_data_tests.yaml +++ b/release/release_data_tests.yaml @@ -580,6 +580,37 @@ timeout: 3600 script: python batch_inference_hetero/main.py {{args}} +- name: text_embedding_{{case}} + frequency: manual + + matrix: + setup: + case: [] + cluster_type: [] + args: [] + adjustments: + - with: + case: fixed_size + cluster_type: fixed_size + args: --inference-concurrency 100 100 + - with: + case: autoscaling + cluster_type: autoscaling + args: --inference-concurrency 1 100 + - with: + case: fixed_size_chaos + cluster_type: fixed_size + args: --inference-concurrency 100 100 --chaos + + cluster: + cluster_compute: text_embedding/{{cluster_type}}_cluster_compute.yaml + byod: + type: cu123 + post_build_script: byod_install_text_embedding.sh + + run: + timeout: 3600 + script: python text_embedding/main.py {{args}} ############## # TPCH Queries From fe0c6eb64e161511c5f44d2aad3cbcf7382a421e Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Fri, 12 Sep 2025 06:34:07 -0700 Subject: [PATCH 596/634] [deps] changing compile llm requirements image (#56172) Using manylinux image with all python versions installed for llm compilation This will be used for raydepsets building passing build here: https://buildkite.com/ray-project/microcheck/builds/25418 --------- Signed-off-by: elliot-barn --- .buildkite/dependencies.rayci.yml | 5 +++-- ci/compile_llm_requirements.sh | 7 ------- ci/test_compile_llm_requirements.sh | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.buildkite/dependencies.rayci.yml b/.buildkite/dependencies.rayci.yml index 97f96c3e752b..002b0899d4ed 100644 --- a/.buildkite/dependencies.rayci.yml +++ b/.buildkite/dependencies.rayci.yml @@ -24,5 +24,6 @@ steps: tags: always instance_type: small command: ./ci/test_compile_llm_requirements.sh - job_env: oss-ci-base_test-py3.11 - depends_on: oss-ci-base_test-multipy + job_env: manylinux + depends_on: + - manylinux diff --git a/ci/compile_llm_requirements.sh b/ci/compile_llm_requirements.sh index 2b3c0de1cb04..cf71563bf47c 100755 --- a/ci/compile_llm_requirements.sh +++ b/ci/compile_llm_requirements.sh @@ -4,13 +4,6 @@ set -euo pipefail CONFIG_PATH="${1:-ci/raydepsets/rayllm.depsets.yaml}" -PYTHON_CODE="$(python -c "import sys; v=sys.version_info; print(f'py{v.major}{v.minor}')")" -if [[ "${PYTHON_CODE}" != "py311" ]]; then - echo "--- Python version is not 3.11" - echo "--- Current Python version: ${PYTHON_CODE}" - exit 1 -fi - mkdir -p /tmp/ray-deps # Remove the GPU constraints diff --git a/ci/test_compile_llm_requirements.sh b/ci/test_compile_llm_requirements.sh index ee2ae90126f4..7be0634145b8 100755 --- a/ci/test_compile_llm_requirements.sh +++ b/ci/test_compile_llm_requirements.sh @@ -35,7 +35,7 @@ done FAILED=0 for LOCK_TYPE in "${LOCK_TYPES[@]}"; do for VARIANT in "${VARIANTS[@]}"; do - diff --color -u ./python/deplocks/llm/"${LOCK_TYPE}"_py311_"${VARIANT}".lock "$TEMP_DIR/${LOCK_TYPE}_py311_${VARIANT}_backup.lock" || { + diff -u ./python/deplocks/llm/"${LOCK_TYPE}"_py311_"${VARIANT}".lock "$TEMP_DIR/${LOCK_TYPE}_py311_${VARIANT}_backup.lock" || { echo "${LOCK_TYPE}_py311_${VARIANT}.lock is not up to date. Please download it from Artifacts tab and git push the changes." FAILED=1 } From 35f3a6910df256821062ebad91a70cc9b5de6ee0 Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Fri, 12 Sep 2025 09:50:13 -0700 Subject: [PATCH 597/634] [data] large schema release test (#56353) ## Why are these changes needed? Ray data has been experiencing slowdowns with large schemas. In practice, we unify schemas from blocks into a one big schema. This schema unification can take a lot of time, so here's release test so that this doesn't happen again. ### Dataset Basically there is one paramter: datatype, which will unify across either - tensors - primitives (ie, ints) - objects - nested structs Each dataset contains about 500-600Mbs of data, except for objects, which contain about 150Mb (this is because their pickle bloat is big). Furthermore, each column contains 500 character. ### Stats - tensors: 45 seconds - primitives: 15 seconds - objects: 30 seconds - nested structs: 30 seconds wide schema generation script: https://gist.github.com/iamjustinhsu/3352100cd18f720f32f2ca58dd9b6108 ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- .../dataset/wide_schema_pipeline_benchmark.py | 57 +++++++++++++++++++ release/release_data_tests.yaml | 19 +++++++ 2 files changed, 76 insertions(+) create mode 100644 release/nightly_tests/dataset/wide_schema_pipeline_benchmark.py diff --git a/release/nightly_tests/dataset/wide_schema_pipeline_benchmark.py b/release/nightly_tests/dataset/wide_schema_pipeline_benchmark.py new file mode 100644 index 000000000000..373afb23e55f --- /dev/null +++ b/release/nightly_tests/dataset/wide_schema_pipeline_benchmark.py @@ -0,0 +1,57 @@ +import argparse +from typing import Dict, Any + +import ray +from benchmark import Benchmark + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Wide schema pipeline benchmark") + parser.add_argument( + "--data-type", + choices=["primitives", "tensors", "objects", "nested_structs"], + default="primitives", + help="Type of pre-generated dataset to benchmark", + ) + + return parser.parse_args() + + +def main(args: argparse.Namespace) -> None: + benchmark = Benchmark() + + # Each dataset contains about 500-600Mbs of data, except for objects, + # which contain about 150Mb (this is because their pickle bloat is big). + # Furthermore, the schema contains 5000 fields, and each column contains + # 500 characters. + input_path = ( + f"s3://ray-benchmark-data-internal-us-west-2/wide_schema/{args.data_type}" + ) + + print(f"Using pre-generated dataset: {input_path}") + + # Run the pipeline benchmark (TIMED) + def run_pipeline() -> Dict[str, Any]: + """Run the data pipeline: read -> map_batches -> write""" + ds = ray.data.read_parquet(input_path) + + for _ in ds.iter_internal_ref_bundles(): + pass + + # Get dataset stats for reporting + actual_num_columns = len(ds.schema().base_schema) + + return { + "num_columns": actual_num_columns, + "data_type": args.data_type, + "input_path": input_path, + } + + # Run the timed benchmark + benchmark.run_fn("wide_schema_pipeline", run_pipeline) + benchmark.write_result() + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/release/release_data_tests.yaml b/release/release_data_tests.yaml index c9c8cc442456..9906afcad56f 100644 --- a/release/release_data_tests.yaml +++ b/release/release_data_tests.yaml @@ -240,6 +240,25 @@ --join_type {{join_type}} --num_partitions 50 +############### +# Wide Schema tests +############### + +- name: wide_schema_pipeline_{{data_type}} + + cluster: + cluster_compute: fixed_size_cpu_compute.yaml + + matrix: + setup: + data_type: [primitives, tensors, objects, nested_structs] + + run: + timeout: 300 + script: > + python wide_schema_pipeline_benchmark.py + --data-type {{data_type}} + ####################### # Streaming split tests ####################### From b535134d2b151e9c762ac37d13455edc3fa62d97 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Fri, 12 Sep 2025 10:01:11 -0700 Subject: [PATCH 598/634] [core][rdt] Wait on nccl id with event (#56322) I'd see concerning logs saying `The NCCL ID has not been set yet` when running gpu objects code with nccl. Looks like we have a loop with a 1 second sleep to try to get the nccl id. Creating a new `wait_and_get_id` with an asyncio condition so we don't have to have that sleep loop with the scary log. I'd also see logs that say `Failed to free actor object` but it's always because the actor's often dead by the time the driver tries to free at the end. We're not handling rpc failures right now for gpu objects, so just turning it down to an info log so it doesn't look as scary to a user. --------- Signed-off-by: dayshah --- .../collective_group/nccl_collective_group.py | 19 ++++++------------- python/ray/util/collective/util.py | 12 ++++++++++-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/python/ray/util/collective/collective_group/nccl_collective_group.py b/python/ray/util/collective/collective_group/nccl_collective_group.py index 5f7de1cbc318..f866b70a9c1e 100644 --- a/python/ray/util/collective/collective_group/nccl_collective_group.py +++ b/python/ray/util/collective/collective_group/nccl_collective_group.py @@ -108,19 +108,12 @@ def get_nccl_id(self, timeout_s=180): """ if not self._store: raise ValueError("Rendezvous store is not setup.") - uid = None - timeout_delta = datetime.timedelta(seconds=timeout_s) - elapsed = datetime.timedelta(seconds=0) - start_time = datetime.datetime.now() - while elapsed < timeout_delta: - uid = ray.get(self._store.get_id.remote()) - if not uid: - time.sleep(1) - elapsed = datetime.datetime.now() - start_time - continue - break - if not uid: - raise RuntimeError("Unable to get the NCCLUniqueID from the store.") + try: + uid = ray.get(self._store.wait_and_get_id.remote(), timeout=timeout_s) + except ray.exceptions.GetTimeoutError: + raise RuntimeError( + f"Unable to get the NCCLUniqueID from the store within {timeout_s} seconds." + ) from None return uid diff --git a/python/ray/util/collective/util.py b/python/ray/util/collective/util.py index 84257fcbbfeb..e46d374d8f85 100644 --- a/python/ray/util/collective/util.py +++ b/python/ray/util/collective/util.py @@ -1,5 +1,6 @@ """Some utility class for Collectives.""" import logging +import asyncio import ray @@ -21,8 +22,9 @@ class NCCLUniqueIDStore: def __init__(self, name): self.name = name self.nccl_id = None + self.event = asyncio.Event() - def set_id(self, uid): + async def set_id(self, uid): """ Initialize the NCCL unique ID for this store. @@ -30,9 +32,15 @@ def set_id(self, uid): uid: the unique ID generated via the NCCL generate_communicator_id API. Returns: - None + The NCCL unique ID set. """ self.nccl_id = uid + self.event.set() + return uid + + async def wait_and_get_id(self): + """Wait for the NCCL unique ID to be set and return it.""" + await self.event.wait() return self.nccl_id def get_id(self): From 30899ba37a48ef81c070a68cc5b5629b5375ec27 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Fri, 12 Sep 2025 11:01:04 -0700 Subject: [PATCH 599/634] [core] Fix ruff for #56322 (#56488) Signed-off-by: dayshah --- python/ray/util/collective/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/util/collective/util.py b/python/ray/util/collective/util.py index e46d374d8f85..02221995fd60 100644 --- a/python/ray/util/collective/util.py +++ b/python/ray/util/collective/util.py @@ -1,6 +1,6 @@ """Some utility class for Collectives.""" -import logging import asyncio +import logging import ray From 6e7c96b7708676b6179b08d9f90565c0dad74fe4 Mon Sep 17 00:00:00 2001 From: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Date: Fri, 12 Sep 2025 11:34:49 -0700 Subject: [PATCH 600/634] Revert "[llm] disable sglang release test (#55884)" (#56475) Signed-off-by: Seiji Eicher --- release/release_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/release_tests.yaml b/release/release_tests.yaml index e4ff69657d36..bf7146fb92f9 100644 --- a/release/release_tests.yaml +++ b/release/release_tests.yaml @@ -4643,7 +4643,7 @@ pytest -sv test_batch_vllm.py - name: llm_batch_sglang_llama - frequency: manual # TODO(ray-llm): fix this test and re-enable it. + frequency: nightly python: "3.11" group: llm-batch team: llm From baa70c249d554c139b6e203e26c5cea7c27e5189 Mon Sep 17 00:00:00 2001 From: Abrar Sheikh Date: Fri, 12 Sep 2025 13:17:00 -0700 Subject: [PATCH 601/634] add tests for replica ranks (#56120) Part 4 of https://github.com/ray-project/ray/pull/54938 Add integration tests for replica ranks --------- Signed-off-by: abrar --- python/ray/serve/_private/controller.py | 9 + python/ray/serve/tests/BUILD.bazel | 4 + python/ray/serve/tests/test_replica_ranks.py | 399 +++++++++++++++++++ 3 files changed, 412 insertions(+) create mode 100644 python/ray/serve/tests/test_replica_ranks.py diff --git a/python/ray/serve/_private/controller.py b/python/ray/serve/_private/controller.py index fa946121b37a..446406026976 100644 --- a/python/ray/serve/_private/controller.py +++ b/python/ray/serve/_private/controller.py @@ -1111,6 +1111,15 @@ def record_request_routing_info(self, info: RequestRoutingInfo): """ self.deployment_state_manager.record_request_routing_info(info) + def _get_replica_ranks_mapping(self, deployment_id: DeploymentID) -> Dict[str, int]: + """Get the current rank mapping for all replicas in a deployment. + Args: + deployment_id: The deployment ID to get ranks for. + Returns: + Dictionary mapping replica_id to rank. + """ + return self.deployment_state_manager._get_replica_ranks_mapping(deployment_id) + async def graceful_shutdown(self, wait: bool = True): """Set the shutting down flag on controller to signal shutdown in run_control_loop(). diff --git a/python/ray/serve/tests/BUILD.bazel b/python/ray/serve/tests/BUILD.bazel index 653b75363a2c..3a1cd3fde236 100644 --- a/python/ray/serve/tests/BUILD.bazel +++ b/python/ray/serve/tests/BUILD.bazel @@ -110,10 +110,14 @@ py_test_module_list( # Medium tests, don't run on windows. py_test_module_list( size = "medium", + env = { + "RAY_SERVE_FAIL_ON_RANK_ERROR": "1", + }, files = [ "test_fastapi.py", "test_gcs_failure.py", "test_gradio.py", + "test_replica_ranks.py", ], tags = [ "exclusive", diff --git a/python/ray/serve/tests/test_replica_ranks.py b/python/ray/serve/tests/test_replica_ranks.py new file mode 100644 index 000000000000..c2f26b0754c7 --- /dev/null +++ b/python/ray/serve/tests/test_replica_ranks.py @@ -0,0 +1,399 @@ +import random +import sys +from typing import Dict, List + +import pytest + +import ray +from ray import serve +from ray._common.test_utils import SignalActor, wait_for_condition +from ray.serve._private.common import ( + DeploymentID, + DeploymentStatus, + ReplicaState, +) +from ray.serve._private.constants import ( + SERVE_CONTROLLER_NAME, + SERVE_DEFAULT_APP_NAME, + SERVE_NAMESPACE, +) +from ray.serve._private.controller import ServeController +from ray.serve._private.test_utils import ( + check_deployment_status, + check_num_replicas_eq, +) + + +def get_controller() -> ServeController: + """Get the current ServeController actor.""" + return ray.get_actor(SERVE_CONTROLLER_NAME, namespace=SERVE_NAMESPACE) + + +def get_replica_ranks(deployment_name: str) -> Dict[str, int]: + """Get the current rank mapping for all replicas in a deployment.""" + controller = get_controller() + deployment_id = DeploymentID(name=deployment_name, app_name=SERVE_DEFAULT_APP_NAME) + + # Use the public API method on the controller + return ray.get(controller._get_replica_ranks_mapping.remote(deployment_id)) + + +def get_running_replica_ids(deployment_name: str) -> List[str]: + """Get the replica IDs of running replicas for given deployment.""" + controller = get_controller() + deployment_id = DeploymentID(name=deployment_name, app_name=SERVE_DEFAULT_APP_NAME) + + replicas = ray.get( + controller._dump_replica_states_for_testing.remote(deployment_id) + ) + running_replicas = replicas.get([ReplicaState.RUNNING]) + return [replica.replica_id.unique_id for replica in running_replicas] + + +def check_rank_contiguity(ranks: Dict[str, int]) -> bool: + """Check that ranks form a contiguous sequence from 0 to N-1.""" + if not ranks: + return True + + rank_values = sorted(ranks.values()) + expected = list(range(len(rank_values))) + assert rank_values == expected, f"Expected {expected}, got {rank_values}" + return True + + +def check_rank_assignment_complete(deployment_name: str, expected_count: int) -> bool: + """Check that all replicas have been assigned ranks and they are contiguous.""" + try: + replica_ids = get_running_replica_ids(deployment_name) + ranks = get_replica_ranks(deployment_name) + + # Check all running replicas have ranks + for replica_id in replica_ids: + if replica_id not in ranks: + print(f"Replica {replica_id} not found in ranks: {ranks}") + return False + + # Check we have expected number of ranks + if len(ranks) != expected_count: + print(f"Expected {expected_count} ranks, got {len(ranks)}: {ranks}") + return False + + # Check ranks are contiguous + return check_rank_contiguity(ranks) + except Exception as e: + print(f"Error checking rank assignment: {e}") + return False + + +@pytest.mark.parametrize("num_replicas", [1, 3, 5]) +def test_basic_rank_assignment(serve_instance, num_replicas): + """Test basic rank assignment for different numbers of replicas.""" + + @serve.deployment(num_replicas=num_replicas) + class RankTracker: + def __init__(self): + self.replica_rank = None + self.world_size = None + + def __call__(self): + context = serve.get_replica_context() + self.replica_rank = context.rank + self.world_size = context.world_size + return { + "rank": self.replica_rank, + "world_size": self.world_size, + } + + handle = serve.run(RankTracker.bind()) + + # Wait for all replicas to be running and have ranks assigned + wait_for_condition( + lambda: check_rank_assignment_complete("RankTracker", num_replicas), + ) + + # Verify ranks are correctly assigned + ranks = get_replica_ranks("RankTracker") + assert len(ranks) == num_replicas + assert check_rank_contiguity(ranks) + + # Verify replicas can access their ranks via API + responses = [] + for _ in range(10): # Make multiple requests to hit different replicas + response = handle.remote().result() + responses.append(response) + + # Check that we got responses from all replicas + seen_ranks = set() + for response in responses: + assert response["world_size"] == num_replicas + if response["rank"] is not None: + seen_ranks.add(response["rank"]) + + # We should eventually see all ranks (though it might take multiple requests) + assert len(seen_ranks) <= num_replicas + for rank in seen_ranks: + assert 0 <= rank < num_replicas + + +def test_rank_assignment_with_autoscaling(serve_instance): + """Test rank assignment and reassignment during autoscaling.""" + signal_actor = SignalActor.remote() + + @serve.deployment( + autoscaling_config={ + "target_ongoing_requests": 1, + "metrics_interval_s": 0.1, + "min_replicas": 2, + "max_replicas": 4, + "upscale_delay_s": 1, + "downscale_delay_s": 1, + "look_back_period_s": 10, + }, + max_ongoing_requests=10, + ) + class AutoscalingRankTracker: + async def __call__(self): + await signal_actor.wait.remote() + context = serve.get_replica_context() + return { + "rank": context.rank, + "world_size": context.world_size, + } + + handle = serve.run(AutoscalingRankTracker.bind()) + + # Wait for initial replicas + wait_for_condition( + lambda: check_rank_assignment_complete("AutoscalingRankTracker", 2), + ) + + initial_ranks = get_replica_ranks("AutoscalingRankTracker") + assert len(initial_ranks) == 2 + assert check_rank_contiguity(initial_ranks) + + # Send concurrent requests to trigger autoscaling + _ = [handle.remote() for _ in range(10)] + + # Wait for scale-up to happen and ranks to be reassigned + wait_for_condition( + lambda: check_num_replicas_eq("AutoscalingRankTracker", 4, use_controller=True), + timeout=20, + ) + + # Check that ranks are still contiguous after scale-up + wait_for_condition( + lambda: check_rank_assignment_complete("AutoscalingRankTracker", 4), + ) + + scaled_ranks = get_replica_ranks("AutoscalingRankTracker") + assert len(scaled_ranks) == 4 + assert check_rank_contiguity(scaled_ranks) + + signal_actor.send.remote() + + # Wait for scale-down (no more load) + wait_for_condition( + lambda: check_num_replicas_eq("AutoscalingRankTracker", 2, use_controller=True), + ) + + # Check that ranks are reassigned and contiguous after scale-down + wait_for_condition( + lambda: check_rank_assignment_complete("AutoscalingRankTracker", 2), + ) + + final_ranks = get_replica_ranks("AutoscalingRankTracker") + assert len(final_ranks) == 2 + assert check_rank_contiguity(final_ranks) + + +def test_rank_persistence_across_controller_restart(serve_instance): + """Test that ranks are preserved across controller failures.""" + + @serve.deployment(num_replicas=3) + class PersistentRankTracker: + def __call__(self): + context = serve.get_replica_context() + return { + "rank": context.rank, + "world_size": context.world_size, + } + + serve.run(PersistentRankTracker.bind()) + + # Wait for all replicas to be running + wait_for_condition( + lambda: check_rank_assignment_complete("PersistentRankTracker", 3), + ) + + # Record initial ranks + initial_ranks = get_replica_ranks("PersistentRankTracker") + + assert len(initial_ranks) == 3 + assert check_rank_contiguity(initial_ranks) + + # Kill the controller to simulate failure + controller = get_controller() + ray.kill(controller, no_restart=False) + + # Wait for controller to be restarted and deployment to be recovered + wait_for_condition( + lambda: check_deployment_status( + "PersistentRankTracker", DeploymentStatus.HEALTHY + ), + ) + + # Wait for rank assignment to be restored + wait_for_condition( + lambda: check_rank_assignment_complete("PersistentRankTracker", 3), + ) + + # Check that ranks are preserved for surviving replicas + recovered_ranks = get_replica_ranks("PersistentRankTracker") + + assert len(recovered_ranks) == 3 + assert check_rank_contiguity(recovered_ranks) + + # Check that the recovered ranks are the same as the initial ranks + assert recovered_ranks == initial_ranks + + +def test_single_replica_deployment(serve_instance): + """Test rank assignment for single replica deployment.""" + + @serve.deployment(num_replicas=1) + class SingleReplicaTracker: + def __call__(self): + context = serve.get_replica_context() + return { + "rank": context.rank, + "world_size": context.world_size, + } + + handle = serve.run(SingleReplicaTracker.bind()) + + # Wait for deployment + wait_for_condition( + lambda: check_rank_assignment_complete("SingleReplicaTracker", 1), + ) + + # Verify single replica has rank 0 + ranks = get_replica_ranks("SingleReplicaTracker") + assert len(ranks) == 1 + assert 0 in ranks.values() + + # Verify API returns correct values + response = handle.remote().result() + assert response["rank"] == 0 + assert response["world_size"] == 1 + + +def test_multiple_deployments_independent_ranks(serve_instance): + """Test that different deployments have independent rank spaces.""" + + @serve.deployment(name="deployment1", num_replicas=2) + class RankTracker1: + def __call__(self): + context = serve.get_replica_context() + return { + "deployment": "deployment1", + "rank": context.rank, + "world_size": context.world_size, + } + + @serve.deployment(name="deployment2", num_replicas=3) + class RankTracker2: + def __init__(self, rank_tracker1): + self.rank_tracker1 = rank_tracker1 + + def __call__(self): + context = serve.get_replica_context() + return { + "deployment": "deployment2", + "rank": context.rank, + "world_size": context.world_size, + } + + serve.run(RankTracker2.bind(RankTracker1.bind())) + # Wait for both deployments + wait_for_condition( + lambda: check_rank_assignment_complete("deployment1", 2), + ) + wait_for_condition( + lambda: check_rank_assignment_complete("deployment2", 3), + ) + + # Check ranks are independent + ranks1 = get_replica_ranks("deployment1") + ranks2 = get_replica_ranks("deployment2") + + assert len(ranks1) == 2 + assert len(ranks2) == 3 + assert check_rank_contiguity(ranks1) + assert check_rank_contiguity(ranks2) + + # Both should have rank 0 (in their own space) + assert 0 in ranks1.values() + assert 0 in ranks2.values() + assert 1 in ranks1.values() + assert 1 in ranks2.values() + assert 2 in ranks2.values() # Only deployment2 should have rank 2 + + handle1 = serve.get_deployment_handle("deployment1", SERVE_DEFAULT_APP_NAME) + handle2 = serve.get_deployment_handle("deployment2", SERVE_DEFAULT_APP_NAME) + + response1 = handle1.remote().result() + response2 = handle2.remote().result() + assert response1["world_size"] == 2 + assert response2["world_size"] == 3 + + +def test_rank_stability_on_replica_death(serve_instance): + """Test that when one replica dies, other replicas keep their ranks.""" + + @serve.deployment(num_replicas=4) + class StableRankTracker: + def __call__(self): + return "hello" + + serve.run(StableRankTracker.bind()) + + # Wait for all replicas to be running and have ranks + wait_for_condition( + lambda: check_rank_assignment_complete("StableRankTracker", 4), + ) + + # get_replica_ranks + initial_ranks = get_replica_ranks("StableRankTracker") + initial_replica_ids = get_running_replica_ids("StableRankTracker") + assert len(initial_ranks) == 4 + assert check_rank_contiguity(initial_ranks) + + # kill the replica with rank 1 + random_replica_id_idx = random.choice(range(len(initial_replica_ids))) + killed_replica_id = initial_replica_ids[random_replica_id_idx] + replica_handle = ray.get_actor( + f"SERVE_REPLICA::default#StableRankTracker#{killed_replica_id}", + namespace=SERVE_NAMESPACE, + ) + ray.kill(replica_handle, no_restart=False) + + def _check(): + new_running_replica_ids = get_running_replica_ids("StableRankTracker") + assert len(new_running_replica_ids) == 4 + assert new_running_replica_ids != initial_replica_ids + return True + + wait_for_condition(_check, timeout=20) + + # get_replica_ranks + final_ranks = get_replica_ranks("StableRankTracker") + assert len(final_ranks) == 4 + assert check_rank_contiguity(final_ranks) + # for all replicas that is not killed, their ranks should be the same as before + for replica_id in initial_replica_ids: + if replica_id != killed_replica_id: + assert final_ranks[replica_id] == initial_ranks[replica_id] + + +if __name__ == "__main__": + sys.exit(pytest.main(["-v", "-s", __file__])) From 08afa53ccf20e114b8d0b4f68a4ecaf19dffaafd Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Fri, 12 Sep 2025 13:25:24 -0700 Subject: [PATCH 602/634] [data] add missing doc strings for DataContext (#56460) ## Why are these changes needed? https://github.com/iamjustinhsu/ray/pull/4 old PR ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- python/ray/data/context.py | 39 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/python/ray/data/context.py b/python/ray/data/context.py index 480192bfb986..30daaf92249a 100644 --- a/python/ray/data/context.py +++ b/python/ray/data/context.py @@ -364,6 +364,8 @@ class DataContext: to use. use_ray_tqdm: Whether to enable distributed tqdm. enable_progress_bars: Whether to enable progress bars. + enable_operator_progress_bars: Whether to enable progress bars for individual + operators during execution. enable_progress_bar_name_truncation: If True, the name of the progress bar (often the operator name) will be truncated if it exceeds `ProgressBar.MAX_NAME_LENGTH`. Otherwise, the full operator name is shown. @@ -379,7 +381,8 @@ class DataContext: retry. This follows same format as :ref:`retry_exceptions ` in Ray Core. Default to `False` to not retry on any errors. Set to `True` to retry all errors, or set to a list of errors to retry. - enable_op_resource_reservation: Whether to reserve resources for each operator. + op_resource_reservation_enabled: Whether to enable resource reservation for + operators to prevent resource contention. op_resource_reservation_ratio: The ratio of the total resources to reserve for each operator. max_errored_blocks: Max number of blocks that are allowed to have errors, @@ -409,10 +412,42 @@ class DataContext: retried_io_errors: A list of substrings of error messages that should trigger a retry when reading or writing files. This is useful for handling transient errors when reading from remote storage systems. + default_hash_shuffle_parallelism: Default parallelism level for hash-based + shuffle operations if the number of partitions is unspecifed. + max_hash_shuffle_aggregators: Maximum number of aggregating actors that can be + provisioned for hash-shuffle aggregations. + min_hash_shuffle_aggregator_wait_time_in_s: Minimum time to wait for hash + shuffle aggregators to become available, in seconds. + hash_shuffle_aggregator_health_warning_interval_s: Interval for health warning + checks on hash shuffle aggregators, in seconds. + max_hash_shuffle_finalization_batch_size: Maximum batch size for concurrent + hash-shuffle finalization tasks. If `None`, defaults to + `max_hash_shuffle_aggregators`. + join_operator_actor_num_cpus_per_partition_override: Override CPU allocation + per partition for join operator actors. + hash_shuffle_operator_actor_num_cpus_per_partition_override: Override CPU + allocation per partition for hash shuffle operator actors. + hash_aggregate_operator_actor_num_cpus_per_partition_override: Override CPU + allocation per partition for hash aggregate operator actors. + use_polars_sort: Whether to use Polars for tabular dataset sorting operations. enable_per_node_metrics: Enable per node metrics reporting for Ray Data, disabled by default. + override_object_store_memory_limit_fraction: Override the fraction of object + store memory limit. If `None`, uses Ray's default. memory_usage_poll_interval_s: The interval to poll the USS of map tasks. If `None`, map tasks won't record memory stats. + dataset_logger_id: Optional logger ID for dataset operations. If `None`, uses + default logging configuration. + issue_detectors_config: Configuration for issue detection and monitoring during + dataset operations. + downstream_capacity_backpressure_ratio: Ratio for downstream capacity + backpressure control. A higher ratio causes backpressure to kick-in + later. If `None`, this type of backpressure is disabled. + downstream_capacity_backpressure_max_queued_bundles: Maximum number of queued + bundles before applying backpressure. If `None`, no limit is applied. + enforce_schemas: Whether to enforce schema consistency across dataset operations. + pandas_block_ignore_metadata: Whether to ignore pandas metadata when converting + between Arrow and pandas formats for better type inference. """ # `None` means the block size is infinite. @@ -440,7 +475,7 @@ class DataContext: # Default hash-shuffle parallelism level (will be used when not # provided explicitly) - default_hash_shuffle_parallelism = DEFAULT_MIN_PARALLELISM + default_hash_shuffle_parallelism: int = DEFAULT_MIN_PARALLELISM # Max number of aggregating actors that could be provisioned # to perform aggregations on partitions produced during hash-shuffling From 268634f73cc43cf249d1260dc758acfcbbd86454 Mon Sep 17 00:00:00 2001 From: iamjustinhsu <140442892+iamjustinhsu@users.noreply.github.com> Date: Fri, 12 Sep 2025 13:25:44 -0700 Subject: [PATCH 603/634] [data] Wrong name for Inqueue Panel (#56463) ## Why are these changes needed? as titled, verified these are the only ones misaligned, it should be inqueue, not outqueue ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: iamjustinhsu --- .../metrics/dashboards/data_dashboard_panels.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py index 4b3dcf374536..fd50e1ba2312 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py @@ -583,8 +583,8 @@ EXTERNAL_INQUEUE_BLOCKS_PANEL = Panel( id=2, - title="Operator External OutQueue Size (Blocks)", - description="Number of blocks in operator's external output queue", + title="Operator External InQueue Size (Blocks)", + description="Number of blocks in operator's external input queue", unit="blocks", targets=[ Target( @@ -598,12 +598,12 @@ EXTERNAL_INQUEUE_BYTES_PANEL = Panel( id=27, - title="Operator External OutQueue Size (bytes)", - description="Byte size of blocks in operator's external output queue", + title="Operator External InQueue Size (bytes)", + description="Byte size of blocks in operator's external input queue", unit="bytes", targets=[ Target( - expr='sum(ray_data_num_external_inqueue_blocks{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', + expr='sum(ray_data_num_external_inqueue_bytes{{{global_filters}, operator=~"$Operator"}}) by (dataset, operator)', legend="Number of Bytes: {{dataset}}, {{operator}}", ) ], From dd43876604422ddd90c6404690de7a1d39d4782b Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Fri, 12 Sep 2025 14:33:49 -0700 Subject: [PATCH 604/634] [core][rdt] Support tensor transfer from outside owners of actors (#56485) Signed-off-by: dayshah --- python/ray/_raylet.pyx | 20 ++++++++++--------- python/ray/actor.py | 4 ++++ python/ray/includes/common.pxd | 5 +++-- python/ray/includes/libcoreworker.pxd | 1 + .../gpu_objects/test_gpu_objects_gloo.py | 20 +++++++++++++++++++ src/ray/common/task/task_spec.h | 8 +++++--- src/ray/core_worker/actor_handle.cc | 6 +++++- src/ray/core_worker/actor_handle.h | 3 +++ src/ray/core_worker/common.h | 3 +++ src/ray/core_worker/core_worker.cc | 1 + src/ray/protobuf/core_worker.proto | 2 ++ 11 files changed, 58 insertions(+), 15 deletions(-) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index f5548d8edc40..995a417b472f 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -786,6 +786,7 @@ cdef int prepare_labels( if label_dict is None: return 0 + label_map[0].reserve(len(label_dict)) for key, value in label_dict.items(): if not isinstance(key, str): raise ValueError(f"Label key must be string, but got {type(key)}") @@ -802,6 +803,7 @@ cdef int prepare_label_selector( if label_selector_dict is None: return 0 + label_selector[0].reserve(len(label_selector_dict)) for key, value in label_selector_dict.items(): if not isinstance(key, str): raise ValueError(f"Label selector key type must be string, but got {type(key)}") @@ -829,6 +831,7 @@ cdef int prepare_resources( if resource_dict is None: raise ValueError("Must provide resource map.") + resource_map[0].reserve(len(resource_dict)) for key, value in resource_dict.items(): if not (isinstance(value, int) or isinstance(value, float)): raise ValueError("Resource quantities may only be ints or floats.") @@ -855,6 +858,7 @@ cdef c_vector[CFunctionDescriptor] prepare_function_descriptors(pyfd_list): c_vector[CFunctionDescriptor] fd_list CRayFunction ray_function + fd_list.reserve(len(pyfd_list)) for pyfd in pyfd_list: fd_list.push_back(CFunctionDescriptorBuilder.BuildPython( pyfd.module_name, pyfd.class_name, pyfd.function_name, b"")) @@ -866,17 +870,16 @@ cdef int prepare_actor_concurrency_groups( c_vector[CConcurrencyGroup] *concurrency_groups): cdef: - CConcurrencyGroup cg c_vector[CFunctionDescriptor] c_fd_list if concurrency_groups_dict is None: raise ValueError("Must provide it...") + concurrency_groups.reserve(len(concurrency_groups_dict)) for key, value in concurrency_groups_dict.items(): c_fd_list = prepare_function_descriptors(value["function_descriptors"]) - cg = CConcurrencyGroup( - key.encode("ascii"), value["max_concurrency"], c_fd_list) - concurrency_groups.push_back(cg) + concurrency_groups.push_back(CConcurrencyGroup( + key.encode("ascii"), value["max_concurrency"], move(c_fd_list))) return 1 @@ -3836,6 +3839,7 @@ cdef class CoreWorker: labels, label_selector, c_bool allow_out_of_order_execution, + c_bool enable_tensor_transport, ): cdef: CRayFunction ray_function @@ -3890,6 +3894,7 @@ cdef class CoreWorker: c_concurrency_groups, allow_out_of_order_execution, max_pending_calls, + enable_tensor_transport, enable_task_events, c_labels, c_label_selector), @@ -4169,6 +4174,7 @@ cdef class CoreWorker: max_task_retries = dereference(c_actor_handle).MaxTaskRetries() enable_task_events = dereference(c_actor_handle).EnableTaskEvents() allow_out_of_order_execution = dereference(c_actor_handle).AllowOutOfOrderExecution() + enable_tensor_transport = dereference(c_actor_handle).EnableTensorTransport() if language == Language.PYTHON: assert isinstance(actor_creation_function_descriptor, PythonFunctionDescriptor) @@ -4192,11 +4198,7 @@ cdef class CoreWorker: method_meta.retry_exceptions, method_meta.generator_backpressure_num_objects, # noqa method_meta.enable_task_events, - # TODO(swang): Pass - # enable_tensor_transport when - # serializing an ActorHandle and - # sending to another actor. - False, # enable_tensor_transport + enable_tensor_transport, method_meta.method_name_to_tensor_transport, actor_method_cpu, actor_creation_function_descriptor, diff --git a/python/ray/actor.py b/python/ray/actor.py index 64058ee0b728..de960f9c29aa 100644 --- a/python/ray/actor.py +++ b/python/ray/actor.py @@ -894,6 +894,7 @@ def __getstate__(self): "is_generator": self._is_generator, "generator_backpressure_num_objects": self._generator_backpressure_num_objects, # noqa "enable_task_events": self._enable_task_events, + "_tensor_transport": self._tensor_transport, } def __setstate__(self, state): @@ -907,6 +908,7 @@ def __setstate__(self, state): state["generator_backpressure_num_objects"], state["enable_task_events"], state["decorator"], + state["_tensor_transport"], ) @@ -1795,6 +1797,7 @@ def _remote(self, args=None, kwargs=None, **actor_options) -> ActorProxy[T]: labels=actor_options.get("_labels"), label_selector=actor_options.get("label_selector"), allow_out_of_order_execution=allow_out_of_order_execution, + enable_tensor_transport=meta.enable_tensor_transport, ) if _actor_launch_hook: @@ -1892,6 +1895,7 @@ class ActorHandle(Generic[T]): _ray_actor_creation_function_descriptor: The function descriptor of the actor creation task. _ray_allow_out_of_order_execution: Whether the actor can execute tasks out of order. + _ray_enable_tensor_transport: Whether tensor transport is enabled for this actor. """ def __init__( diff --git a/python/ray/includes/common.pxd b/python/ray/includes/common.pxd index 95ca2488e59a..6493a100c47a 100644 --- a/python/ray/includes/common.pxd +++ b/python/ray/includes/common.pxd @@ -365,6 +365,7 @@ cdef extern from "ray/core_worker/common.h" nogil: const c_vector[CConcurrencyGroup] &concurrency_groups, c_bool allow_out_of_order_execution, int32_t max_pending_calls, + c_bool enable_tensor_transport, c_bool enable_task_events, const unordered_map[c_string, c_string] &labels, const unordered_map[c_string, c_string] &label_selector) @@ -765,9 +766,9 @@ cdef extern from "src/ray/protobuf/autoscaler.pb.h" nogil: cdef extern from "ray/common/task/task_spec.h" nogil: cdef cppclass CConcurrencyGroup "ray::ConcurrencyGroup": CConcurrencyGroup( - const c_string &name, + c_string name, uint32_t max_concurrency, - const c_vector[CFunctionDescriptor] &c_fds) + c_vector[CFunctionDescriptor] c_fds) CConcurrencyGroup() c_string GetName() const uint32_t GetMaxConcurrency() const diff --git a/python/ray/includes/libcoreworker.pxd b/python/ray/includes/libcoreworker.pxd index 5a65b6b3cc8b..98ec283bbe81 100644 --- a/python/ray/includes/libcoreworker.pxd +++ b/python/ray/includes/libcoreworker.pxd @@ -117,6 +117,7 @@ cdef extern from "ray/core_worker/core_worker.h" nogil: int MaxTaskRetries() const c_bool EnableTaskEvents() const c_bool AllowOutOfOrderExecution() const + c_bool EnableTensorTransport() const cdef cppclass CCoreWorker "ray::core::CoreWorker": CWorkerType GetWorkerType() diff --git a/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py b/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py index ce0a2e4e9a90..ce723104736a 100644 --- a/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py +++ b/python/ray/tests/gpu_objects/test_gpu_objects_gloo.py @@ -865,5 +865,25 @@ def test_duplicate_objectref_transfer(ray_start_regular): ), f"Results differ: result1={val1}, result2={val2}" +def test_transfer_from_not_actor_creator(ray_start_regular): + @ray.remote + class Actor: + @ray.method(tensor_transport="gloo") + def create(self): + return torch.tensor([1, 2, 3]) + + def consume(self, obj): + return obj + + def do_transfer(self, a1, a2): + create_collective_group([a1, a2], backend="torch_gloo") + return ray.get(a1.consume.remote(a2.create.remote())) + + actor = [Actor.remote() for _ in range(3)] + assert ray.get(actor[2].do_transfer.remote(actor[0], actor[1])) == pytest.approx( + torch.tensor([1, 2, 3]) + ) + + if __name__ == "__main__": sys.exit(pytest.main(["-sv", __file__])) diff --git a/src/ray/common/task/task_spec.h b/src/ray/common/task/task_spec.h index 9fab88cc7438..3ead82c55128 100644 --- a/src/ray/common/task/task_spec.h +++ b/src/ray/common/task/task_spec.h @@ -48,10 +48,12 @@ struct ConcurrencyGroup { ConcurrencyGroup() = default; - ConcurrencyGroup(const std::string &name, + ConcurrencyGroup(std::string name, uint32_t max_concurrency, - const std::vector &fds) - : name_(name), max_concurrency_(max_concurrency), function_descriptors_(fds) {} + std::vector fds) + : name_(std::move(name)), + max_concurrency_(max_concurrency), + function_descriptors_(std::move(fds)) {} std::string GetName() const { return name_; } diff --git a/src/ray/core_worker/actor_handle.cc b/src/ray/core_worker/actor_handle.cc index b11065c8bffc..4f3e04875345 100644 --- a/src/ray/core_worker/actor_handle.cc +++ b/src/ray/core_worker/actor_handle.cc @@ -35,6 +35,7 @@ rpc::ActorHandle CreateInnerActorHandle( const std::string &ray_namespace, int32_t max_pending_calls, bool allow_out_of_order_execution, + bool enable_tensor_transport, std::optional enable_task_events, const std::unordered_map &labels) { rpc::ActorHandle inner; @@ -50,8 +51,9 @@ rpc::ActorHandle CreateInnerActorHandle( inner.set_max_task_retries(max_task_retries); inner.set_name(name); inner.set_ray_namespace(ray_namespace); - inner.set_allow_out_of_order_execution(allow_out_of_order_execution); inner.set_max_pending_calls(max_pending_calls); + inner.set_allow_out_of_order_execution(allow_out_of_order_execution); + inner.set_enable_tensor_transport(enable_tensor_transport); inner.set_enable_task_events(enable_task_events.value_or(kDefaultTaskEventEnabled)); inner.mutable_labels()->insert(labels.begin(), labels.end()); return inner; @@ -105,6 +107,7 @@ ActorHandle::ActorHandle( const std::string &ray_namespace, int32_t max_pending_calls, bool allow_out_of_order_execution, + bool enable_tensor_transport, std::optional enable_task_events, const std::unordered_map &labels) : ActorHandle(CreateInnerActorHandle(actor_id, @@ -120,6 +123,7 @@ ActorHandle::ActorHandle( ray_namespace, max_pending_calls, allow_out_of_order_execution, + enable_tensor_transport, enable_task_events, labels)) {} diff --git a/src/ray/core_worker/actor_handle.h b/src/ray/core_worker/actor_handle.h index b3ec2294befb..d9aa163f7dfa 100644 --- a/src/ray/core_worker/actor_handle.h +++ b/src/ray/core_worker/actor_handle.h @@ -49,6 +49,7 @@ class ActorHandle { const std::string &ray_namespace, int32_t max_pending_calls, bool allow_out_of_order_execution = false, + bool enable_tensor_transport = false, std::optional enable_task_events = absl::nullopt, const std::unordered_map &labels = {}); @@ -110,6 +111,8 @@ class ActorHandle { bool AllowOutOfOrderExecution() const { return inner_.allow_out_of_order_execution(); } + bool EnableTensorTransport() const { return inner_.enable_tensor_transport(); } + const ::google::protobuf::Map &GetLabels() const { return inner_.labels(); } diff --git a/src/ray/core_worker/common.h b/src/ray/core_worker/common.h index 3e0bd5c06379..c68bed12d18e 100644 --- a/src/ray/core_worker/common.h +++ b/src/ray/core_worker/common.h @@ -129,6 +129,7 @@ struct ActorCreationOptions { std::vector concurrency_groups_p = {}, bool allow_out_of_order_execution_p = false, int32_t max_pending_calls_p = -1, + bool enable_tensor_transport_p = false, bool enable_task_events_p = kDefaultTaskEventEnabled, std::unordered_map labels_p = {}, std::unordered_map label_selector_p = {}) @@ -147,6 +148,7 @@ struct ActorCreationOptions { concurrency_groups(std::move(concurrency_groups_p)), allow_out_of_order_execution(allow_out_of_order_execution_p), max_pending_calls(max_pending_calls_p), + enable_tensor_transport(enable_tensor_transport_p), scheduling_strategy(std::move(scheduling_strategy_p)), enable_task_events(enable_task_events_p), labels(std::move(labels_p)), @@ -200,6 +202,7 @@ struct ActorCreationOptions { const bool allow_out_of_order_execution = false; /// The maximum actor call pending count. const int max_pending_calls = -1; + const bool enable_tensor_transport = false; // The strategy about how to schedule this actor. rpc::SchedulingStrategy scheduling_strategy; /// True if task events (worker::TaskEvent) from this creation task should be reported diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index eee7cba66534..62ef51ae0bdc 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -2104,6 +2104,7 @@ Status CoreWorker::CreateActor(const RayFunction &function, ray_namespace, actor_creation_options.max_pending_calls, actor_creation_options.allow_out_of_order_execution, + actor_creation_options.enable_tensor_transport, actor_creation_options.enable_task_events, actor_creation_options.labels); std::string serialized_actor_handle; diff --git a/src/ray/protobuf/core_worker.proto b/src/ray/protobuf/core_worker.proto index 38383ce11172..401d1fc7a424 100644 --- a/src/ray/protobuf/core_worker.proto +++ b/src/ray/protobuf/core_worker.proto @@ -72,6 +72,8 @@ message ActorHandle { // The key-value labels for actor. map labels = 15; + + bool enable_tensor_transport = 16; } message PushTaskRequest { From 2aec3708f1785048d42b3cbec5881f4408103def Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Fri, 12 Sep 2025 14:35:37 -0700 Subject: [PATCH 605/634] [ci] 2 different pydoclints (1 for ci and 1 for local) (#56464) creating 2 pydoclint pre-commit jobs due to the following issue: https://anyscaleteam.slack.com/archives/C02J07KKF2A/p1757377523554369 Issue: pydoclint-baseline.txt gets overwritten during precommit because pydoclint is only running on updated files When manually running `pre-commit run pydoclint --all-files --show-diff-on-failure` all files are scanned and the pydoctlint-baseline is respected Solution: Create 2 pre-commit jobs 1 for local (**pydoclintlocal**): setting `--auto-regenerate-baseline=False` so its not overwritten but will surface errors 1 for ci (**pydoclintci**): Leaving current setup `--auto-regenerate-baseline=True` and adding **manual** stage to manually trigger the hook This step is manually run in ci and will error out if theres a diff in the baseline --------- Signed-off-by: elliot-barn --- .pre-commit-config.yaml | 28 ++++++++++++++++++++++++++++ ci/lint/lint.sh | 6 +++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d404d2c92c99..fb79122d44b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,10 +45,38 @@ repos: - id: ruff args: [ --select, "I", --fix, --exit-non-zero-on-fix ] + # pydoclint-local is for local commits only due to pre-commit-hook only passing + # updated files to the hook and overwriting the baseline text file - repo: https://github.com/jsh9/pydoclint rev: "0.6.6" hooks: - id: pydoclint + name: pydoclint-local + stages: [pre-commit, pre-push] + args: [ + --style=google, + --baseline=ci/lint/pydoclint-baseline.txt, + --exclude=thirdparty|^python/ray/serve/tests/test_config_files/syntax_error\.py$|^python/ray/_private/parameter\.py$, + --auto-regenerate-baseline=False, + # Current settings (not because we think they're right, but because we + # don't want a baseline the size of the codebase) + --arg-type-hints-in-docstring=False, + --skip-checking-raises=True, + --check-return-types=False, + --allow-init-docstring=True, + --check-class-attributes=False, + # --check-style-mismatch=True, # Bring this back once things are a bit cleaner + ] + types: [python] + files: '^python/ray/' + + # pydoclint-ci is for CI, overwrites the baseline text file, and is run with the manual stage flag + - repo: https://github.com/jsh9/pydoclint + rev: "0.6.6" + hooks: + - id: pydoclint + name: pydoclint-ci + stages: [manual] args: [ --style=google, --baseline=ci/lint/pydoclint-baseline.txt, diff --git a/ci/lint/lint.sh b/ci/lint/lint.sh index e8355dda12ba..53d7156e86c8 100755 --- a/ci/lint/lint.sh +++ b/ci/lint/lint.sh @@ -48,7 +48,11 @@ pre_commit() { pre_commit_pydoclint() { # Run pre-commit pydoclint on all files pip install -c python/requirements_compiled.txt pre-commit clang-format - pre-commit run pydoclint --all-files --show-diff-on-failure + pre-commit run pydoclint --hook-stage manual --all-files --show-diff-on-failure + git diff --quiet -- ci/lint/pydoclint-baseline.txt || { + echo "Baseline needs update. Run the CI-style hook: \"pre-commit run pydoclint --hook-stage manual --all-files --show-diff-on-failure\" locally and commit the baseline." + exit 1 + } } code_format() { From ef9168e824c56d05e16883d1ab87a9d7329e064a Mon Sep 17 00:00:00 2001 From: lkchen Date: Fri, 12 Sep 2025 14:46:56 -0700 Subject: [PATCH 606/634] [LLM][Serve] Allow setting `data_parallel_size=1` in engine_kwargs (#55750) --- .../_internal/serve/configs/server_models.py | 14 +++-- .../tests/serve/cpu/configs/test_models.py | 51 +++++++++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/python/ray/llm/_internal/serve/configs/server_models.py b/python/ray/llm/_internal/serve/configs/server_models.py index 3be3e246929a..d7a7b9af9f75 100644 --- a/python/ray/llm/_internal/serve/configs/server_models.py +++ b/python/ray/llm/_internal/serve/configs/server_models.py @@ -560,18 +560,22 @@ def get_serve_options( # Configure DP deployment options. # TODO(rui): move the following to DPServer, e.g., # deployment_options = DPServer.get_deployment_options(llm_config) - dp_size = self.engine_kwargs.get("data_parallel_size", None) - if dp_size is not None: + dp_size = self.engine_kwargs.get("data_parallel_size", 1) + if not (isinstance(dp_size, int) and dp_size > 0): + raise ValueError( + f"Invalid data_parallel_size: {dp_size}, expecting " "positive integer." + ) + if dp_size != 1: if "num_replicas" in deployment_options: raise ValueError( "num_replicas should not be specified for DP deployment, " - "use engine_kwargs.data_parallel_size instead." + f"use engine_kwargs.data_parallel_size={dp_size} instead." ) if "autoscaling_config" in deployment_options: raise ValueError( "autoscaling_config is not supported for DP deployment, " - "use engine_kwargs.data_parallel_size to set a fixed number " - "of replicas instead." + f"use engine_kwargs.data_parallel_size={dp_size} to set a " + "fixed number of replicas instead." ) deployment_options["num_replicas"] = dp_size deployment_options["max_ongoing_requests"] = DEFAULT_MAX_ONGOING_REQUESTS diff --git a/python/ray/llm/tests/serve/cpu/configs/test_models.py b/python/ray/llm/tests/serve/cpu/configs/test_models.py index b88ca0524029..2ae92508c9e9 100644 --- a/python/ray/llm/tests/serve/cpu/configs/test_models.py +++ b/python/ray/llm/tests/serve/cpu/configs/test_models.py @@ -1,4 +1,5 @@ import sys +from copy import deepcopy from pathlib import Path import pydantic @@ -302,6 +303,56 @@ def test_log_engine_metrics_disable_log_stats_validation(self): engine_kwargs={"disable_log_stats": True}, ) + @pytest.mark.parametrize( + "data_parallel_size,num_replica,allowed", + [ + (None, 1, True), + (None, 2, True), + (None, 3, True), + (1, 1, True), + (1, 2, True), + (1, 3, True), + (2, 2, False), + (2, 3, False), + (4, 2, False), + (2, None, True), + (None, None, True), + ], + ) + def test_multi_replica_dp_validation( + self, data_parallel_size, num_replica, allowed + ): + """Test that multi-replica and DP size are mutually exclusive. + + Ray.llm's implementation does not yet support multi-replica + deployment along with DP. + """ + engine_kwargs = ( + {} + if data_parallel_size is None + else {"data_parallel_size": data_parallel_size} + ) + deployment_config = {} if num_replica is None else {"num_replicas": num_replica} + + def get_serve_options_with_num_replica(): + return LLMConfig( + model_loading_config=dict(model_id="test_model"), + engine_kwargs=deepcopy(engine_kwargs), + deployment_config=deepcopy(deployment_config), + ).get_serve_options(name_prefix="Test:") + + if allowed: + serve_options = get_serve_options_with_num_replica() + actual_num_replicas = serve_options.get("num_replicas", 1) + expected_num_replicas = (data_parallel_size or 1) * (num_replica or 1) + assert actual_num_replicas == expected_num_replicas + else: + with pytest.raises( + ValueError, + match="use engine_kwargs.data_parallel_size", + ): + get_serve_options_with_num_replica() + class TestFieldValidators: """Test the field validators for dict validation.""" From 3054b71bd5efc07e158012d9e6ead874e6ca526c Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Fri, 12 Sep 2025 15:37:46 -0700 Subject: [PATCH 607/634] [ci] removing python ver check for llm lockfile compile (#56495) removing python ver check for llm compilation already use --python flag on compilation Signed-off-by: elliot-barn --- ci/raydepsets/pre_hooks/remove-compiled-headers.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ci/raydepsets/pre_hooks/remove-compiled-headers.sh b/ci/raydepsets/pre_hooks/remove-compiled-headers.sh index 37f32d3808f9..109563fd2be2 100755 --- a/ci/raydepsets/pre_hooks/remove-compiled-headers.sh +++ b/ci/raydepsets/pre_hooks/remove-compiled-headers.sh @@ -2,13 +2,6 @@ set -euo pipefail -PYTHON_CODE="$(python -c "import sys; v=sys.version_info; print(f'py{v.major}{v.minor}')")" -if [[ "${PYTHON_CODE}" != "py311" ]]; then - echo "--- Python version is not 3.11" - echo "--- Current Python version: ${PYTHON_CODE}" - exit 1 -fi - mkdir -p /tmp/ray-deps # Remove the GPU constraints From 1028dcc128d7d9739d423171b566474ce81ab332 Mon Sep 17 00:00:00 2001 From: Nikhil G Date: Fri, 12 Sep 2025 17:16:13 -0700 Subject: [PATCH 608/634] [Data.llm] Fix multimodal image extraction when no system prompt is present (#56435) Signed-off-by: Nikhil Ghosh --- .../batch/stages/prepare_image_stage.py | 17 +++- .../cpu/stages/test_prepare_image_stage.py | 98 +++++++++++++++++++ 2 files changed, 111 insertions(+), 4 deletions(-) diff --git a/python/ray/llm/_internal/batch/stages/prepare_image_stage.py b/python/ray/llm/_internal/batch/stages/prepare_image_stage.py index 8b1989863c42..bb5a31db131d 100644 --- a/python/ray/llm/_internal/batch/stages/prepare_image_stage.py +++ b/python/ray/llm/_internal/batch/stages/prepare_image_stage.py @@ -322,12 +322,21 @@ def extract_image_info(self, messages: List[Dict]) -> List[_ImageType]: image_info: List[_ImageType] = [] for message in messages: - if not isinstance(message["content"], list): + content = message["content"] + + # Convert PyArrow objects to Python objects if needed (like ChatTemplateStage). + # This handles the case where unform content types are serialized with PyArrow + # instead of pickle- happens when all messages have the same content structure + # (e.g., no system prompt + string content mixed with user messages with list content). + if hasattr(content, "tolist"): + content = content.tolist() + + if not isinstance(content, list): continue - for content in message["content"]: - if content["type"] not in ("image", "image_url"): + for content_item in content: + if content_item["type"] not in ("image", "image_url"): continue - image = content[content["type"]] + image = content_item[content_item["type"]] if not isinstance(image, str) and not isinstance( image, self.Image.Image ): diff --git a/python/ray/llm/tests/batch/cpu/stages/test_prepare_image_stage.py b/python/ray/llm/tests/batch/cpu/stages/test_prepare_image_stage.py index 07b5182dfbcd..2ee2f5e8bd3d 100644 --- a/python/ray/llm/tests/batch/cpu/stages/test_prepare_image_stage.py +++ b/python/ray/llm/tests/batch/cpu/stages/test_prepare_image_stage.py @@ -164,5 +164,103 @@ async def test_prepare_image_udf_invalid_image_type(mock_image_processor): pass +# Test that image extraction works consistently with both uniform content types +# (no system prompt) and mixed content types (with system prompt) + + +@pytest.mark.parametrize( + "messages,expected_images,test_description", + [ + # Test with system prompt + ( + [ + {"role": "system", "content": "You are an assistant"}, + { + "role": "user", + "content": [ + { + "type": "image", + "image": "https://example.com/test-image.jpg", + }, + { + "type": "text", + "text": "Can you describe this image in 1 words?", + }, + ], + }, + ], + ["https://example.com/test-image.jpg"], + "with_system_prompt", + ), + # Test without system prompt + ( + [ + { + "role": "user", + "content": [ + { + "type": "image", + "image": "https://example.com/test-image.jpg", + }, + { + "type": "text", + "text": "Can you describe this image in 1 words?", + }, + ], + } + ], + ["https://example.com/test-image.jpg"], + "without_system_prompt", + ), + # Test multiple images without system prompt + ( + [ + { + "role": "user", + "content": [ + {"type": "image", "image": "https://example.com/image1.jpg"}, + {"type": "text", "text": "Describe this image"}, + ], + }, + { + "role": "user", + "content": [ + {"type": "image", "image": "https://example.com/image2.jpg"}, + {"type": "text", "text": "What do you see?"}, + ], + }, + ], + ["https://example.com/image1.jpg", "https://example.com/image2.jpg"], + "multiple_images_no_system_prompt", + ), + # Test image_url format without system prompt + ( + [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": "https://example.com/image.jpg", + }, + {"type": "text", "text": "Describe this image"}, + ], + } + ], + ["https://example.com/image.jpg"], + "image_url_format_no_system_prompt", + ), + ], + ids=lambda x: x if isinstance(x, str) else None, +) +def test_extract_image_info(messages, expected_images, test_description): + """Test image extraction with various message structures and formats.""" + udf = PrepareImageUDF(data_column="__data", expected_input_keys=["messages"]) + + image_info = udf.extract_image_info(messages) + assert len(image_info) == len(expected_images) + assert image_info == expected_images + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) From bae1d4598f193777a3b1ca556d0236c41d87100e Mon Sep 17 00:00:00 2001 From: Stephanie Wang Date: Fri, 12 Sep 2025 17:57:52 -0700 Subject: [PATCH 609/634] [core][gpu-objects] Add initial docs (#55981) Initial user guide for GPU objects. Missing a couple things that we can add in follow-ups: - installation instructions - full API reference - performance numbers --------- Signed-off-by: Stephanie wang Signed-off-by: Stephanie Wang Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Edward Oakes Co-authored-by: Qiaolin Yu Co-authored-by: Dhyey Shah --- doc/BUILD.bazel | 37 +++ doc/source/ray-core/api/direct-transport.rst | 28 ++ doc/source/ray-core/api/index.rst | 1 + doc/source/ray-core/direct-transport.rst | 250 ++++++++++++++++++ .../doc_code/direct_transport_gloo.py | 136 ++++++++++ .../doc_code/direct_transport_nccl.py | 27 ++ .../doc_code/direct_transport_nixl.py | 34 +++ doc/source/ray-core/user-guide.rst | 1 + python/ray/actor.py | 17 +- 9 files changed, 525 insertions(+), 6 deletions(-) create mode 100644 doc/source/ray-core/api/direct-transport.rst create mode 100644 doc/source/ray-core/direct-transport.rst create mode 100644 doc/source/ray-core/doc_code/direct_transport_gloo.py create mode 100644 doc/source/ray-core/doc_code/direct_transport_nccl.py create mode 100644 doc/source/ray-core/doc_code/direct_transport_nixl.py diff --git a/doc/BUILD.bazel b/doc/BUILD.bazel index 018aa5185495..2dcca3b6ac1e 100644 --- a/doc/BUILD.bazel +++ b/doc/BUILD.bazel @@ -202,6 +202,41 @@ py_test( ], ) +py_test( + name = "doc_code_direct_transport_gloo", + size = "small", + srcs = ["source/ray-core/doc_code/direct_transport_gloo.py"], + main = "source/ray-core/doc_code/direct_transport_gloo.py", + tags = [ + "exclusive", + "team:core", + ], +) + +py_test( + name = "doc_code_direct_transport_nccl", + size = "small", + srcs = ["source/ray-core/doc_code/direct_transport_nccl.py"], + main = "source/ray-core/doc_code/direct_transport_nccl.py", + tags = [ + "exclusive", + "multi_gpu", + "team:core", + ], +) + +py_test( + name = "doc_code_direct_transport_nixl", + size = "small", + srcs = ["source/ray-core/doc_code/direct_transport_nixl.py"], + main = "source/ray-core/doc_code/direct_transport_nixl.py", + tags = [ + "exclusive", + "multi_gpu", + "team:core", + ], +) + py_test_run_all_subdirectory( size = "medium", include = ["source/ray-core/doc_code/*.py"], @@ -214,6 +249,8 @@ py_test_run_all_subdirectory( "source/ray-core/doc_code/cgraph_overlap.py", # not testing this as it purposefully segfaults "source/ray-core/doc_code/cgraph_troubleshooting.py", + "source/ray-core/doc_code/direct_transport_nccl.py", + "source/ray-core/doc_code/direct_transport_nixl.py", ], extra_srcs = [], tags = [ diff --git a/doc/source/ray-core/api/direct-transport.rst b/doc/source/ray-core/api/direct-transport.rst new file mode 100644 index 000000000000..07815b1f9758 --- /dev/null +++ b/doc/source/ray-core/api/direct-transport.rst @@ -0,0 +1,28 @@ +Ray Direct Transport (RDT) API +============================== + +Usage with Core APIs +-------------------- +Enable RDT for actor tasks with the :func:`@ray.method ` decorator, or pass `_tensor_transport` to :func:`ray.put`. You can then pass the resulting `ray.ObjectRef` to other actor tasks, or use :func:`ray.get` to retrieve the result. See :ref:`Ray Direct Transport (RDT) ` for more details on usage. + + +.. autosummary:: + :nosignatures: + :toctree: doc/ + + ray.method + ray.put + ray.get + +Collective tensor transports +---------------------------- +Collective tensor transports require a collective group to be created before RDT objects can be used. Use these methods to create and manage collective groups for the `gloo` and `nccl` tensor transports. + + +.. autosummary:: + :nosignatures: + :toctree: doc/ + + ray.experimental.collective.create_collective_group + ray.experimental.collective.get_collective_groups + ray.experimental.collective.destroy_collective_group \ No newline at end of file diff --git a/doc/source/ray-core/api/index.rst b/doc/source/ray-core/api/index.rst index 2845ebe892ef..25b15b766bf1 100644 --- a/doc/source/ray-core/api/index.rst +++ b/doc/source/ray-core/api/index.rst @@ -12,3 +12,4 @@ Ray Core API cli.rst ../../ray-observability/reference/cli.rst ../../ray-observability/reference/api.rst + direct-transport.rst \ No newline at end of file diff --git a/doc/source/ray-core/direct-transport.rst b/doc/source/ray-core/direct-transport.rst new file mode 100644 index 000000000000..d8d08a7c163c --- /dev/null +++ b/doc/source/ray-core/direct-transport.rst @@ -0,0 +1,250 @@ +.. _direct-transport: + +.. TODO: asyncio not yet supported. +.. TODO: wait_tensor_freed + +************************** +Ray Direct Transport (RDT) +************************** + +Ray objects are normally stored in Ray's CPU-based object store and copied and deserialized when accessed by a Ray task or actor. +For GPU data specifically, this can lead to unnecessary and expensive data transfers. +For example, passing a CUDA ``torch.Tensor`` from one Ray task to another would require a copy from GPU to CPU memory, then back again to GPU memory. + +*Ray Direct Transport (RDT)* is a new feature that allows Ray to store and pass objects directly between Ray actors. +This feature augments the familiar Ray :class:`ObjectRef ` API by: + +- Keeping GPU data in GPU memory until a transfer is needed +- Avoiding expensive serialization and copies to and from the Ray object store +- Using efficient data transports like collective communication libraries (`Gloo `__ or `NCCL `__) or point-to-point RDMA (via `NVIDIA's NIXL `__) to transfer data directly between devices, including both CPU and GPUs + +.. note:: + RDT is currently in **alpha**. Not all Ray Core APIs are supported yet. Future releases may introduce breaking API changes. See the :ref:`limitations ` section for more details. + +Getting started +=============== + +.. tip:: + RDT currently supports ``torch.Tensor`` objects created by Ray actor tasks. Other datatypes and Ray non-actor tasks may be supported in future releases. + +This walkthrough will show how to create and use RDT with different *tensor transports*, i.e. the mechanism used to transfer the tensor between actors. +Currently, RDT supports the following tensor transports: + +1. `Gloo `__: A collective communication library for PyTorch and CPUs. +2. `NVIDIA NCCL `__: A collective communication library for NVIDIA GPUs. +3. `NVIDIA NIXL `__ (backed by `UCX `__): A library for accelerating point-to-point transfers via RDMA, especially between various types of memory and NVIDIA GPUs. + +For ease of following along, we'll start with the `Gloo `__ transport, which can be used without any physical GPUs. + +.. _direct-transport-gloo: + +Usage with Gloo (CPUs only) +--------------------------- + +Installation +^^^^^^^^^^^^ + +.. note:: + Under construction. + +Walkthrough +^^^^^^^^^^^ + +To get started, define an actor class and a task that returns a ``torch.Tensor``: + +.. literalinclude:: doc_code/direct_transport_gloo.py + :language: python + :start-after: __normal_example_start__ + :end-before: __normal_example_end__ + +As written, when the ``torch.Tensor`` is returned, it will be copied into Ray's CPU-based object store. +For CPU-based tensors, this can require an expensive step to copy and serialize the object, while GPU-based tensors additionally require a copy to and from CPU memory. + +To enable RDT, use the ``tensor_transport`` option in the :func:`@ray.method ` decorator. + +.. literalinclude:: doc_code/direct_transport_gloo.py + :language: python + :start-after: __gloo_example_start__ + :end-before: __gloo_example_end__ + +This decorator can be added to any actor tasks that return a ``torch.Tensor``, or that return ``torch.Tensors`` nested inside other Python objects. +Adding this decorator will change Ray's behavior in the following ways: + +1. When returning the tensor, Ray will store a *reference* to the tensor instead of copying it to CPU memory. +2. When the :class:`ray.ObjectRef` is passed to another task, Ray will use Gloo to transfer the tensor to the destination task. + +Note that for (2) to work, the :func:`@ray.method(tensor_transport) ` decorator only needs to be added to the actor task that *returns* the tensor. It should not be added to actor tasks that *consume* the tensor (unless those tasks also return tensors). + +Also, for (2) to work, we must first create a *collective group* of actors. + +Creating a collective group +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To create a collective group for use with RDT: + +1. Create multiple Ray actors. +2. Create a collective group on the actors using the :func:`ray.experimental.collective.create_collective_group ` function. The `backend` specified must match the `tensor_transport` used in the :func:`@ray.method ` decorator. + +Here is an example: + +.. literalinclude:: doc_code/direct_transport_gloo.py + :language: python + :start-after: __gloo_group_start__ + :end-before: __gloo_group_end__ + +The actors can now communicate directly via gloo. +The group can also be destroyed using the :func:`ray.experimental.collective.destroy_collective_group ` function. +After calling this function, a new collective group can be created on the same actors. + +Passing objects to other actors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now that we have a collective group, we can create and pass RDT objects between the actors. +Here is a full example: + +.. literalinclude:: doc_code/direct_transport_gloo.py + :language: python + :start-after: __gloo_full_example_start__ + :end-before: __gloo_full_example_end__ + +When the :class:`ray.ObjectRef` is passed to another task, Ray will use Gloo to transfer the tensor directly from the source actor to the destination actor instead of the default object store. +Note that the :func:`@ray.method(tensor_transport) ` decorator is only added to the actor task that *returns* the tensor; once this hint has been added, the receiving actor task `receiver.sum` will automatically use Gloo to receive the tensor. +In this example, because `MyActor.sum` does not have the :func:`@ray.method(tensor_transport) ` decorator, it will use the default Ray object store transport to return `torch.sum(tensor)`. + +RDT also supports passing tensors nested inside Python data structures, as well as actor tasks that return multiple tensors, like in this example: + +.. literalinclude:: doc_code/direct_transport_gloo.py + :language: python + :start-after: __gloo_multiple_tensors_example_start__ + :end-before: __gloo_multiple_tensors_example_end__ + +Passing RDT objects to the actor that produced them +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +RDT :class:`ray.ObjectRefs ` can also be passed to the actor that produced them. +This avoids any copies and just provides a reference to the same ``torch.Tensor`` that was previously created. +For example: + +.. literalinclude:: doc_code/direct_transport_gloo.py + :language: python + :start-after: __gloo_intra_actor_start__ + :end-before: __gloo_intra_actor_end__ + + +.. note:: + Ray only keeps a reference to the tensor created by the user, so the tensor objects are *mutable*. + If ``sender.sum`` were to modify the tensor in the above example, the changes would also be seen by ``receiver.sum``. + This differs from the normal Ray Core API, which always makes an immutable copy of data returned by actors. + + +``ray.get`` +^^^^^^^^^^^ + +The :func:`ray.get ` function can also be used as usual to retrieve the result of an RDT object, via Ray's object store. + +.. TODO: This example needs to be updated once we change the default transport for ray.get to match the ray.method transport. + +.. literalinclude:: doc_code/direct_transport_gloo.py + :language: python + :start-after: __gloo_get_start__ + :end-before: __gloo_get_end__ + +Usage with NCCL (NVIDIA GPUs only) +---------------------------------- + +RDT requires just a few lines of code change to switch tensor transports. Here is the :ref:`Gloo example `, modified to use NVIDIA GPUs and the `NCCL `__ library for collective GPU communication. + +.. literalinclude:: doc_code/direct_transport_nccl.py + :language: python + :start-after: __nccl_full_example_start__ + :end-before: __nccl_full_example_end__ + +The main code differences are: + +1. The :func:`@ray.method ` uses ``tensor_transport="nccl"`` instead of ``tensor_transport="gloo"``. +2. The :func:`ray.experimental.collective.create_collective_group ` function is used to create a collective group. +3. The tensor is created on the GPU using the ``.cuda()`` method. + +Usage with NIXL (CPUs or NVIDIA GPUs) +------------------------------------- + +NIXL can transfer data between different devices, including CPUs and NVIDIA GPUs, but doesn't require a collective group to be created ahead of time. +This means that any actor that has NIXL installed in its environment can be used to create and pass an RDT object. + +Otherwise, the usage is the same as in the :ref:`Gloo example `. + +Here is an example showing how to use NIXL to transfer an RDT object between two actors: + +.. literalinclude:: doc_code/direct_transport_nixl.py + :language: python + :start-after: __nixl_full_example_start__ + :end-before: __nixl_full_example_end__ + +Compared to the :ref:`Gloo example `, the main code differences are: + +1. The :func:`@ray.method ` uses ``tensor_transport="nixl"`` instead of ``tensor_transport="gloo"``. +2. No collective group is needed. + +.. TODO: ray.get with NIXL + ``ray.get`` + ^^^^^^^^^^^ + + Unlike the collective-based tensor transports (Gloo and NCCL), the :func:`ray.get ` function can use NIXL or the Ray object store to retrieve a copy of the result. + By default, the tensor transport for :func:`ray.get ` will be the one specified in the :func:`@ray.method ` decorator. + + .. literalinclude:: doc_code/direct_transport_nixl.py + :language: python + :start-after: __nixl_get_start__ + :end-before: __nixl_get_end__ + +Summary +------- + +RDT allows Ray to store and pass objects directly between Ray actors, using accelerated transports like GLOO, NCCL, and NIXL. +Here are the main points to keep in mind: + +* If using a collective-based tensor transport (Gloo or NCCL), a collective group must be created ahead of time. NIXL just requires all involved actors to have NIXL installed. +* Unlike objects in the Ray object store, RDT objects are *mutable*, meaning that Ray only holds a reference, not a copy, to the stored tensor(s). +* Otherwise, actors can be used as normal. + +For a full list of limitations, see the :ref:`limitations ` section. + + +Microbenchmarks +=============== + +.. note:: + Under construction. + +.. _limitations: + +Limitations +=========== + +RDT is currently in alpha and currently has the following limitations, which may be addressed in future releases: + +* Support for ``torch.Tensor`` objects only. +* Support for Ray actors only, not Ray tasks. +* Support for the following transports: Gloo, NCCL, and NIXL. +* Support for CPUs and NVIDIA GPUs only. +* RDT objects are *mutable*. This means that Ray only holds a reference to the tensor, and will not copy it until a transfer is requested. Thus, if the application code also keeps a reference to a tensor before returning it, and modifies the tensor in place, then some or all of the changes may be seen by the receiving actor. + +For collective-based tensor transports (Gloo and NCCL): + +* Only the process that created the collective group can submit actor tasks that return and pass RDT objects. If the creating process passes the actor handles to other processes, those processes can submit actor tasks as usual, but will not be able to use RDT objects. +* Similarly, the process that created the collective group cannot serialize and pass RDT :class:`ray.ObjectRefs ` to other Ray tasks or actors. Instead, the :class:`ray.ObjectRef`\s can only be passed as direct arguments to other actor tasks, and those actors must be in the same collective group. +* Each actor can only be in one collective group per tensor transport at a time. +* No support for :func:`ray.put `. +* If a system-level error occurs during a collective operation, the collective group will be destroyed and the actors will no longer be able to communicate via the collective group. Note that application-level errors, i.e. exceptions raised by user code, will not destroy the collective group and will instead be propagated to any dependent task(s), as for non-RDT Ray objects. System-level errors include: + + * Errors internal to the third-party transport, e.g., NCCL network errors + * Actor and node failure + * Tensors returned by the user that are located on an unsupported device, e.g., a CPU tensor when using NCCL + * Any unexpected system bugs + + +Advanced: RDT Internals +======================= + +.. note:: + Under construction. diff --git a/doc/source/ray-core/doc_code/direct_transport_gloo.py b/doc/source/ray-core/doc_code/direct_transport_gloo.py new file mode 100644 index 000000000000..282ee55a8d63 --- /dev/null +++ b/doc/source/ray-core/doc_code/direct_transport_gloo.py @@ -0,0 +1,136 @@ +# flake8: noqa + +# __normal_example_start__ +import torch +import ray + + +@ray.remote +class MyActor: + def random_tensor(self): + return torch.randn(1000, 1000) + + +# __normal_example_end__ + +# __gloo_example_start__ +@ray.remote +class MyActor: + @ray.method(tensor_transport="gloo") + def random_tensor(self): + return torch.randn(1000, 1000) + + +# __gloo_example_end__ + +# __gloo_group_start__ +import torch +import ray +from ray.experimental.collective import create_collective_group + + +@ray.remote +class MyActor: + @ray.method(tensor_transport="gloo") + def random_tensor(self): + return torch.randn(1000, 1000) + + def sum(self, tensor: torch.Tensor): + return torch.sum(tensor) + + +sender, receiver = MyActor.remote(), MyActor.remote() +# The tensor_transport specified here must match the one used in the @ray.method +# decorator. +group = create_collective_group([sender, receiver], backend="torch_gloo") +# __gloo_group_end__ + +# __gloo_group_destroy_start__ +from ray.experimental.collective import destroy_collective_group + +destroy_collective_group(group) +# __gloo_group_destroy_end__ + +# __gloo_full_example_start__ +import torch +import ray +from ray.experimental.collective import create_collective_group + + +@ray.remote +class MyActor: + @ray.method(tensor_transport="gloo") + def random_tensor(self): + return torch.randn(1000, 1000) + + def sum(self, tensor: torch.Tensor): + return torch.sum(tensor) + + +sender, receiver = MyActor.remote(), MyActor.remote() +group = create_collective_group([sender, receiver], backend="torch_gloo") + +# The tensor will be stored by the `sender` actor instead of in Ray's object +# store. +tensor = sender.random_tensor.remote() +result = receiver.sum.remote(tensor) +print(ray.get(result)) +# __gloo_full_example_end__ + +# __gloo_multiple_tensors_example_start__ +import torch +import ray +from ray.experimental.collective import create_collective_group + + +@ray.remote +class MyActor: + @ray.method(tensor_transport="gloo") + def random_tensor_dict(self): + return {"tensor1": torch.randn(1000, 1000), "tensor2": torch.randn(1000, 1000)} + + def sum(self, tensor_dict: dict): + return torch.sum(tensor_dict["tensor1"]) + torch.sum(tensor_dict["tensor2"]) + + +sender, receiver = MyActor.remote(), MyActor.remote() +group = create_collective_group([sender, receiver], backend="torch_gloo") + +# Both tensor values in the dictionary will be stored by the `sender` actor +# instead of in Ray's object store. +tensor_dict = sender.random_tensor_dict.remote() +result = receiver.sum.remote(tensor_dict) +print(ray.get(result)) +# __gloo_multiple_tensors_example_end__ + +# __gloo_intra_actor_start__ +import torch +import ray +from ray.experimental.collective import create_collective_group + + +@ray.remote +class MyActor: + @ray.method(tensor_transport="gloo") + def random_tensor(self): + return torch.randn(1000, 1000) + + def sum(self, tensor: torch.Tensor): + return torch.sum(tensor) + + +sender, receiver = MyActor.remote(), MyActor.remote() +group = create_collective_group([sender, receiver], backend="torch_gloo") + +tensor = sender.random_tensor.remote() +# Pass the ObjectRef back to the actor that produced it. The tensor will be +# passed back to the same actor without copying. +sum1 = sender.sum.remote(tensor) +sum2 = receiver.sum.remote(tensor) +assert torch.allclose(*ray.get([sum1, sum2])) +# __gloo_intra_actor_end__ + +# __gloo_get_start__ +print(ray.get(tensor)) +# torch.Tensor(...) +# __gloo_get_end__ diff --git a/doc/source/ray-core/doc_code/direct_transport_nccl.py b/doc/source/ray-core/doc_code/direct_transport_nccl.py new file mode 100644 index 000000000000..2296a073ce6b --- /dev/null +++ b/doc/source/ray-core/doc_code/direct_transport_nccl.py @@ -0,0 +1,27 @@ +# flake8: noqa + +# __nccl_full_example_start__ +import torch +import ray +from ray.experimental.collective import create_collective_group + + +@ray.remote(num_gpus=1) +class MyActor: + @ray.method(tensor_transport="nccl") + def random_tensor(self): + return torch.randn(1000, 1000).cuda() + + def sum(self, tensor: torch.Tensor): + return torch.sum(tensor) + + +sender, receiver = MyActor.remote(), MyActor.remote() +group = create_collective_group([sender, receiver], backend="nccl") + +# The tensor will be stored by the `sender` actor instead of in Ray's object +# store. +tensor = sender.random_tensor.remote() +result = receiver.sum.remote(tensor) +ray.get(result) +# __nccl_full_example_end__ diff --git a/doc/source/ray-core/doc_code/direct_transport_nixl.py b/doc/source/ray-core/doc_code/direct_transport_nixl.py new file mode 100644 index 000000000000..7952c7f9fb87 --- /dev/null +++ b/doc/source/ray-core/doc_code/direct_transport_nixl.py @@ -0,0 +1,34 @@ +# flake8: noqa + +# __nixl_full_example_start__ +import torch +import ray + + +@ray.remote(num_gpus=1) +class MyActor: + @ray.method(tensor_transport="nixl") + def random_tensor(self): + return torch.randn(1000, 1000).cuda() + + def sum(self, tensor: torch.Tensor): + return torch.sum(tensor) + + +# No collective group is needed. The two actors just need to have NIXL +# installed. +sender, receiver = MyActor.remote(), MyActor.remote() + +# The tensor will be stored by the `sender` actor instead of in Ray's object +# store. +tensor = sender.random_tensor.remote() +result = receiver.sum.remote(tensor) +ray.get(result) +# __nixl_full_example_end__ + +# __nixl_get_start__ +# The :func:`ray.get ` function will also use NIXL to retrieve the +# result. +print(ray.get(tensor)) +# torch.Tensor(...) +# __nixl_get_end__ diff --git a/doc/source/ray-core/user-guide.rst b/doc/source/ray-core/user-guide.rst index 89c40d108548..6befdfc519ba 100644 --- a/doc/source/ray-core/user-guide.rst +++ b/doc/source/ray-core/user-guide.rst @@ -17,5 +17,6 @@ If you’re brand new to Ray, we recommend starting with the :ref:`walkthrough < scheduling/index.rst fault-tolerance patterns/index.rst + direct-transport compiled-graph/ray-compiled-graph advanced-topics diff --git a/python/ray/actor.py b/python/ray/actor.py index de960f9c29aa..07f808c90352 100644 --- a/python/ray/actor.py +++ b/python/ray/actor.py @@ -434,13 +434,18 @@ def bar(self): to use for the actor method. By default, the actor is single-threaded and runs all actor tasks on the same thread. See :ref:`Defining Concurrency Groups `. - tensor_transport: [Experimental] The tensor transport protocol to + tensor_transport: [Alpha] The tensor transport protocol to use for the actor method. The valid values are "OBJECT_STORE" - (default), "NCCL", or "GLOO" (case-insensitive). torch.Tensors - returned by this task will be sent to other tasks using the - specified transport. NCCL and GLOO transports require first creating - a collective with the involved actors using - `ray.experimental.collective.create_collective_group`. + (default), "NCCL", "GLOO", or "NIXL" (case-insensitive). If a + non-object store transport is specified, Ray will store a + *reference* instead of a copy of any torch.Tensors found inside + values returned by this task, and the tensors will be sent directly + to other tasks using the specified transport. NCCL and GLOO + transports require first creating a collective with the involved + actors using + :func:`ray.experimental.collective.create_collective_group`. + See :ref:`Ray Direct Transport (RDT) ` for more + details. """ valid_kwargs = [ "num_returns", From a900ae172afe02ab2108384729643e1dee14bed3 Mon Sep 17 00:00:00 2001 From: Balaji Veeramani Date: Fri, 12 Sep 2025 20:50:05 -0700 Subject: [PATCH 610/634] [Data] Refactor batch inference release test definitions (#56489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why are these changes needed? This PR updates the batch inference release tests to make them easier to run and clearer: * Sets the group name to `batch-inference`, removing the need to list each test individually. * Renames batch_inference_hetero → image_embedding_from_jsonl and batch_inference → image_classification for clarity. * Sets the image and text embedding workloads to run weekly for consistent signal. ## Related issue number ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: Balaji Veeramani --- .../autoscaling_cluster_compute.yaml | 0 .../fixed_size_cluster_compute.yaml | 0 .../main.py | 0 release/release_data_tests.yaml | 24 ++++++++++++------- 4 files changed, 15 insertions(+), 9 deletions(-) rename release/nightly_tests/dataset/{batch_inference_hetero => image_embedding_from_jsonl}/autoscaling_cluster_compute.yaml (100%) rename release/nightly_tests/dataset/{batch_inference_hetero => image_embedding_from_jsonl}/fixed_size_cluster_compute.yaml (100%) rename release/nightly_tests/dataset/{batch_inference_hetero => image_embedding_from_jsonl}/main.py (100%) diff --git a/release/nightly_tests/dataset/batch_inference_hetero/autoscaling_cluster_compute.yaml b/release/nightly_tests/dataset/image_embedding_from_jsonl/autoscaling_cluster_compute.yaml similarity index 100% rename from release/nightly_tests/dataset/batch_inference_hetero/autoscaling_cluster_compute.yaml rename to release/nightly_tests/dataset/image_embedding_from_jsonl/autoscaling_cluster_compute.yaml diff --git a/release/nightly_tests/dataset/batch_inference_hetero/fixed_size_cluster_compute.yaml b/release/nightly_tests/dataset/image_embedding_from_jsonl/fixed_size_cluster_compute.yaml similarity index 100% rename from release/nightly_tests/dataset/batch_inference_hetero/fixed_size_cluster_compute.yaml rename to release/nightly_tests/dataset/image_embedding_from_jsonl/fixed_size_cluster_compute.yaml diff --git a/release/nightly_tests/dataset/batch_inference_hetero/main.py b/release/nightly_tests/dataset/image_embedding_from_jsonl/main.py similarity index 100% rename from release/nightly_tests/dataset/batch_inference_hetero/main.py rename to release/nightly_tests/dataset/image_embedding_from_jsonl/main.py diff --git a/release/release_data_tests.yaml b/release/release_data_tests.yaml index 9906afcad56f..52839de05d80 100644 --- a/release/release_data_tests.yaml +++ b/release/release_data_tests.yaml @@ -492,7 +492,8 @@ # 300 GB image classification parquet data up to 10 GPUs # 10 g4dn.12xlarge. -- name: "batch_inference_{{scaling}}" +- name: "image_classification_{{scaling}}" + group: batch-inference cluster: cluster_compute: "{{scaling}}_gpu_compute.yaml" @@ -507,11 +508,12 @@ python gpu_batch_inference.py --data-directory 300G-image-data-synthetic-raw-parquet --data-format parquet -- name: batch_inference_chaos +- name: image_classification_chaos stable: False # Don't use 'nightly_tests/dataset' as the working directory because we need to run # the 'setup_chaos.py' script. working_dir: nightly_tests + group: batch-inference cluster: cluster_compute: dataset/autoscaling_gpu_compute.yaml @@ -523,9 +525,10 @@ python dataset/gpu_batch_inference.py --data-directory 300G-image-data-synthetic-raw-parquet --data-format parquet --chaos-test -- name: batch_inference_chaos_no_scale_back +- name: image_classification_chaos_no_scale_back stable: False working_dir: nightly_tests + group: batch-inference cluster: cluster_compute: dataset/autoscaling_gpu_compute.yaml @@ -539,7 +542,8 @@ - name: image_embedding_from_uris_{{case}} - frequency: manual + frequency: weekly + group: batch-inference matrix: setup: @@ -568,8 +572,9 @@ script: python image_embedding_from_uris/main.py {{args}} -- name: batch_inference_hetero_{{case}} - frequency: manual +- name: image_embedding_from_jsonl_{{case}} + frequency: weekly + group: batch-inference matrix: setup: @@ -591,16 +596,17 @@ args: --inference-concurrency 40 40 --chaos cluster: - cluster_compute: batch_inference_hetero/{{cluster_type}}_cluster_compute.yaml + cluster_compute: image_embedding_from_jsonl/{{cluster_type}}_cluster_compute.yaml byod: post_build_script: byod_install_pybase64.sh run: timeout: 3600 - script: python batch_inference_hetero/main.py {{args}} + script: python image_embedding_from_jsonl/main.py {{args}} - name: text_embedding_{{case}} - frequency: manual + frequency: weekly + group: batch-inference matrix: setup: From e26d21a8ad7224a37b9e0bb23efe49ec7113b19d Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Fri, 12 Sep 2025 21:43:02 -0700 Subject: [PATCH 611/634] [ci] upgrading uv ver 0.8.17 (latest) (#56494) upgrade uv binary --------- Signed-off-by: elliot-barn --- WORKSPACE | 8 ++++---- ci/raydepsets/tests/test_cli.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 20fd81787491..666b38d06797 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -125,8 +125,8 @@ filegroup( visibility = ["//visibility:public"], ) """, - sha256 = "2c4392591fe9469d006452ef22f32712f35087d87fb1764ec03e23544eb8770d", - urls = ["https://github.com/astral-sh/uv/releases/download/0.8.10/uv-x86_64-unknown-linux-gnu.tar.gz"], + sha256 = "920cbcaad514cc185634f6f0dcd71df5e8f4ee4456d440a22e0f8c0f142a8203", + urls = ["https://github.com/astral-sh/uv/releases/download/0.8.17/uv-x86_64-unknown-linux-gnu.tar.gz"], ) http_archive( @@ -138,8 +138,8 @@ filegroup( visibility = ["//visibility:public"], ) """, - sha256 = "5200278ae00b5c0822a7db7a99376b2167e8e9391b29c3de22f9e4fdebc9c0e8", - urls = ["https://github.com/astral-sh/uv/releases/download/0.8.10/uv-aarch64-apple-darwin.tar.gz"], + sha256 = "e4d4859d7726298daa4c12e114f269ff282b2cfc2b415dc0b2ca44ae2dbd358e", + urls = ["https://github.com/astral-sh/uv/releases/download/0.8.17/uv-aarch64-apple-darwin.tar.gz"], ) http_archive( diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 3dd24d9ee87a..49943b1f7127 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -111,7 +111,7 @@ def test_uv_version(self): stderr=subprocess.PIPE, ) assert result.returncode == 0 - assert "uv 0.8.10" in result.stdout.decode("utf-8") + assert "uv 0.8.17" in result.stdout.decode("utf-8") assert result.stderr.decode("utf-8") == "" def test_compile(self): From ce4e4735351aa5c1f35f65eb2d1174e0ffe8a2a8 Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Sat, 13 Sep 2025 04:41:57 -0700 Subject: [PATCH 612/634] [core] Fix ASAN issues in object manager test (#56492) Signed-off-by: joshlee --- src/mock/ray/gcs/gcs_client/gcs_client.h | 1 + src/ray/object_manager/tests/BUILD.bazel | 2 -- .../object_manager/tests/object_manager_test.cc | 15 +++++---------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/mock/ray/gcs/gcs_client/gcs_client.h b/src/mock/ray/gcs/gcs_client/gcs_client.h index a798ef77760d..8c4d15189033 100644 --- a/src/mock/ray/gcs/gcs_client/gcs_client.h +++ b/src/mock/ray/gcs/gcs_client/gcs_client.h @@ -58,6 +58,7 @@ class MockGcsClient : public GcsClient { GcsClient::error_accessor_.reset(mock_error_accessor); GcsClient::worker_accessor_.reset(mock_worker_accessor); GcsClient::placement_group_accessor_.reset(mock_placement_group_accessor); + GcsClient::internal_kv_accessor_.reset(mock_internal_kv_accessor); GcsClient::task_accessor_.reset(mock_task_accessor); } MockActorInfoAccessor *mock_actor_accessor; diff --git a/src/ray/object_manager/tests/BUILD.bazel b/src/ray/object_manager/tests/BUILD.bazel index ac154895c962..437d4c81b4af 100644 --- a/src/ray/object_manager/tests/BUILD.bazel +++ b/src/ray/object_manager/tests/BUILD.bazel @@ -114,10 +114,8 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/fakes/ray/object_manager/plasma:fake_plasma_client", - "//src/ray/common:test_utils", "//src/ray/object_manager", "//src/ray/rpc/object_manager:fake_object_manager_client", - "//src/ray/util:temporary_directory", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/object_manager/tests/object_manager_test.cc b/src/ray/object_manager/tests/object_manager_test.cc index 4a6f99f8a1b0..72fe55f7ad2d 100644 --- a/src/ray/object_manager/tests/object_manager_test.cc +++ b/src/ray/object_manager/tests/object_manager_test.cc @@ -14,9 +14,6 @@ #include "ray/object_manager/object_manager.h" -#include - -#include #include #include #include @@ -28,13 +25,11 @@ #include "mock/ray/gcs/gcs_client/gcs_client.h" #include "mock/ray/object_manager/object_directory.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/common/buffer.h" #include "ray/common/id.h" #include "ray/common/ray_object.h" #include "ray/common/status.h" #include "ray/object_manager/common.h" #include "ray/rpc/object_manager/fake_object_manager_client.h" -#include "ray/util/temporary_directory.h" namespace ray { @@ -88,17 +83,17 @@ class ObjectManagerTest : public ::testing::Test { NodeID local_node_id_; - std::unique_ptr mock_gcs_client_; - std::unique_ptr mock_object_directory_; - std::unique_ptr object_manager_; - std::shared_ptr fake_plasma_client_; - instrumented_io_context io_context_{/*enable_lag_probe=*/false, /*running_on_single_thread=*/true}; instrumented_io_context rpc_context_{/*enable_lag_probe=*/false, /*running_on_single_thread=*/true}; boost::asio::executor_work_guard io_work_; boost::asio::executor_work_guard rpc_work_; + + std::unique_ptr mock_gcs_client_; + std::unique_ptr mock_object_directory_; + std::unique_ptr object_manager_; + std::shared_ptr fake_plasma_client_; }; uint32_t NumRemoteFreeObjectsRequests(const ObjectManager &object_manager) { From 895d78b30dc37647ce0a25d7abe0882c325d6c5e Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Sat, 13 Sep 2025 20:10:07 -0700 Subject: [PATCH 613/634] [core] Don't hold shared ptr to client in actor submitter queues (#56448) Signed-off-by: dayshah --- .../task_submission/actor_task_submitter.cc | 38 ++++++++----------- .../task_submission/actor_task_submitter.h | 5 +-- .../tests/actor_task_submitter_test.cc | 17 +-------- src/ray/rpc/worker/core_worker_client_pool.cc | 3 +- 4 files changed, 20 insertions(+), 43 deletions(-) diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.cc b/src/ray/core_worker/task_submission/actor_task_submitter.cc index 3cf16396d1c2..59c9797dbec2 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.cc +++ b/src/ray/core_worker/task_submission/actor_task_submitter.cc @@ -271,7 +271,7 @@ void ActorTaskSubmitter::CancelDependencyResolution(const TaskID &task_id) { } void ActorTaskSubmitter::DisconnectRpcClient(ClientQueue &queue) { - queue.rpc_client_ = nullptr; + queue.client_address_ = std::nullopt; core_worker_client_pool_.Disconnect(WorkerID::FromBinary(queue.worker_id_)); queue.worker_id_.clear(); } @@ -310,9 +310,9 @@ void ActorTaskSubmitter::ConnectActor(const ActorID &actor_id, return; } - if (queue->second.rpc_client_ && - queue->second.rpc_client_->Addr().ip_address() == address.ip_address() && - queue->second.rpc_client_->Addr().port() == address.port()) { + if (queue->second.client_address_.has_value() && + queue->second.client_address_->ip_address() == address.ip_address() && + queue->second.client_address_->port() == address.port()) { RAY_LOG(DEBUG).WithField(actor_id) << "Skip actor that has already been connected"; return; } @@ -324,7 +324,7 @@ void ActorTaskSubmitter::ConnectActor(const ActorID &actor_id, } queue->second.num_restarts_ = num_restarts; - if (queue->second.rpc_client_) { + if (queue->second.client_address_.has_value()) { // Clear the client to the old version of the actor. DisconnectRpcClient(queue->second); inflight_task_callbacks = std::move(queue->second.inflight_task_callbacks_); @@ -332,10 +332,9 @@ void ActorTaskSubmitter::ConnectActor(const ActorID &actor_id, } queue->second.state_ = rpc::ActorTableData::ALIVE; - // Update the mapping so new RPCs go out with the right intended worker id. + // So new RPCs go out with the right intended worker id to the right address. queue->second.worker_id_ = address.worker_id(); - // Create a new connection to the actor. - queue->second.rpc_client_ = core_worker_client_pool_.GetOrConnect(address); + queue->second.client_address_ = address; SendPendingTasks(actor_id); } @@ -538,7 +537,7 @@ void ActorTaskSubmitter::SendPendingTasks(const ActorID &actor_id) { // and pending tasks will be sent at that time. return; } - if (!client_queue.rpc_client_) { + if (!client_queue.client_address_.has_value()) { if (client_queue.state_ == rpc::ActorTableData::RESTARTING && client_queue.fail_if_actor_unreachable_) { // When `fail_if_actor_unreachable` is true, tasks submitted while the actor is in @@ -599,7 +598,7 @@ void ActorTaskSubmitter::PushActorTask(ClientQueue &queue, next_queueing_warn_threshold_ *= 2; } - rpc::Address addr(queue.rpc_client_->Addr()); + auto &addr = queue.client_address_.value(); rpc::ClientCallback reply_callback = [this, addr, task_spec](const Status &status, const rpc::PushTaskReply &reply) { HandlePushTaskReply(status, reply, addr, task_spec); @@ -630,7 +629,7 @@ void ActorTaskSubmitter::PushActorTask(ClientQueue &queue, task_manager_.MarkTaskWaitingForExecution(task_id, NodeID::FromBinary(addr.node_id()), WorkerID::FromBinary(addr.worker_id())); - queue.rpc_client_->PushActorTask( + core_worker_client_pool_.GetOrConnect(addr)->PushActorTask( std::move(request), skip_queue, std::move(wrapped_callback)); } @@ -797,24 +796,17 @@ bool ActorTaskSubmitter::IsActorAlive(const ActorID &actor_id) const { absl::MutexLock lock(&mu_); auto iter = client_queues_.find(actor_id); - return (iter != client_queues_.end() && iter->second.rpc_client_); + return (iter != client_queues_.end() && iter->second.client_address_.has_value()); } std::optional ActorTaskSubmitter::GetActorAddress( const ActorID &actor_id) const { absl::MutexLock lock(&mu_); - auto iter = client_queues_.find(actor_id); if (iter == client_queues_.end()) { return std::nullopt; } - - const auto &rpc_client = iter->second.rpc_client_; - if (rpc_client == nullptr) { - return std::nullopt; - } - - return iter->second.rpc_client_->Addr(); + return iter->second.client_address_; } bool ActorTaskSubmitter::PendingTasksFull(const ActorID &actor_id) const { @@ -938,17 +930,17 @@ void ActorTaskSubmitter::CancelTask(TaskSpecification task_spec, bool recursive) RAY_LOG(DEBUG).WithField(task_id) << "Task was sent to an actor. Send a cancel RPC."; auto queue = client_queues_.find(actor_id); RAY_CHECK(queue != client_queues_.end()); - if (!queue->second.rpc_client_) { + if (!queue->second.client_address_.has_value()) { RetryCancelTask(task_spec, recursive, 1000); return; } - const auto &client = queue->second.rpc_client_; - auto request = rpc::CancelTaskRequest(); + rpc::CancelTaskRequest request; request.set_intended_task_id(task_spec.TaskIdBinary()); request.set_force_kill(force_kill); request.set_recursive(recursive); request.set_caller_worker_id(task_spec.CallerWorkerIdBinary()); + auto client = core_worker_client_pool_.GetOrConnect(*queue->second.client_address_); client->CancelTask(request, [this, task_spec = std::move(task_spec), recursive, task_id]( const Status &status, const rpc::CancelTaskReply &reply) { diff --git a/src/ray/core_worker/task_submission/actor_task_submitter.h b/src/ray/core_worker/task_submission/actor_task_submitter.h index 9eb41a90298b..b077d7edb77c 100644 --- a/src/ray/core_worker/task_submission/actor_task_submitter.h +++ b/src/ray/core_worker/task_submission/actor_task_submitter.h @@ -286,9 +286,8 @@ class ActorTaskSubmitter : public ActorTaskSubmitterInterface { int64_t num_restarts_due_to_lineage_reconstructions_ = 0; /// Whether this actor exits by spot preemption. bool preempted_ = false; - /// The RPC client. We use shared_ptr to enable shared_from_this for - /// pending client callbacks. - std::shared_ptr rpc_client_ = nullptr; + /// The RPC client address. + std::optional client_address_; /// The intended worker ID of the actor. std::string worker_id_; /// The actor is out of scope but the death info is not published diff --git a/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc b/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc index 3708ad2274bb..0d49e7f8369b 100644 --- a/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc +++ b/src/ray/core_worker/task_submission/tests/actor_task_submitter_test.cc @@ -85,11 +85,8 @@ class MockWorkerClient : public rpc::CoreWorkerClientInterface { class ActorTaskSubmitterTest : public ::testing::TestWithParam { public: ActorTaskSubmitterTest() - : client_pool_( - std::make_shared([&](const rpc::Address &addr) { - num_clients_connected_++; - return worker_client_; - })), + : client_pool_(std::make_shared( + [&](const rpc::Address &addr) { return worker_client_; })), worker_client_(std::make_shared()), store_(std::make_shared(io_context)), task_manager_(std::make_shared()), @@ -109,7 +106,6 @@ class ActorTaskSubmitterTest : public ::testing::TestWithParam { void TearDown() override { io_context.stop(); } - int num_clients_connected_ = 0; int64_t last_queue_warning_ = 0; FakeActorCreator actor_creator_; std::shared_ptr client_pool_; @@ -548,7 +544,6 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderGcs) { addr.set_port(0); submitter_.ConnectActor(actor_id, addr, 0); ASSERT_EQ(worker_client_->callbacks.size(), 0); - ASSERT_EQ(num_clients_connected_, 1); // Create four tasks for the actor. auto task1 = CreateActorTaskHelper(actor_id, worker_id, 0); @@ -561,7 +556,6 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderGcs) { // Actor restarts, but we don't receive the disconnect message until later. addr.set_port(1); submitter_.ConnectActor(actor_id, addr, 1); - ASSERT_EQ(num_clients_connected_, 2); // Submit a task. auto task2 = CreateActorTaskHelper(actor_id, worker_id, 1); submitter_.SubmitTask(task2); @@ -573,7 +567,6 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderGcs) { const auto death_cause = CreateMockDeathCause(); submitter_.DisconnectActor( actor_id, 1, /*dead=*/false, death_cause, /*is_restartable=*/true); - ASSERT_EQ(num_clients_connected_, 2); // Submit a task. auto task3 = CreateActorTaskHelper(actor_id, worker_id, 2); submitter_.SubmitTask(task3); @@ -584,7 +577,6 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderGcs) { // The actor dies twice. We receive the last RESTART message first. submitter_.DisconnectActor( actor_id, 3, /*dead=*/false, death_cause, /*is_restartable=*/true); - ASSERT_EQ(num_clients_connected_, 2); // Submit a task. auto task4 = CreateActorTaskHelper(actor_id, worker_id, 3); submitter_.SubmitTask(task4); @@ -601,19 +593,16 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartOutOfOrderGcs) { submitter_.ConnectActor(actor_id, addr, 2); submitter_.DisconnectActor( actor_id, 2, /*dead=*/false, death_cause, /*is_restartable=*/true); - ASSERT_EQ(num_clients_connected_, 2); // The actor dies permanently. submitter_.DisconnectActor( actor_id, 3, /*dead=*/true, death_cause, /*is_restartable=*/false); - ASSERT_EQ(num_clients_connected_, 2); // We receive more late messages. Nothing happens because the actor is dead. submitter_.DisconnectActor( actor_id, 4, /*dead=*/false, death_cause, /*is_restartable=*/true); addr.set_port(3); submitter_.ConnectActor(actor_id, addr, 4); - ASSERT_EQ(num_clients_connected_, 2); // Submit a task. auto task5 = CreateActorTaskHelper(actor_id, worker_id, 4); EXPECT_CALL(*task_manager_, FailOrRetryPendingTask(task5.TaskId(), _, _, _, _, _)) @@ -635,7 +624,6 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartFailInflightTasks) { /*owned*/ false); submitter_.ConnectActor(actor_id, actor_addr1, 0); ASSERT_EQ(worker_client_->callbacks.size(), 0); - ASSERT_EQ(num_clients_connected_, 1); // Create 3 tasks for the actor. auto task1_first_attempt = CreateActorTaskHelper(actor_id, caller_worker_id, 0); @@ -748,7 +736,6 @@ TEST_P(ActorTaskSubmitterTest, TestActorRestartFastFail) { addr.set_port(0); submitter_.ConnectActor(actor_id, addr, 0); ASSERT_EQ(worker_client_->callbacks.size(), 0); - ASSERT_EQ(num_clients_connected_, 1); auto task1 = CreateActorTaskHelper(actor_id, worker_id, 0); // Submit a task. diff --git a/src/ray/rpc/worker/core_worker_client_pool.cc b/src/ray/rpc/worker/core_worker_client_pool.cc index c66b860daaaf..33a1c673ccf5 100644 --- a/src/ray/rpc/worker/core_worker_client_pool.cc +++ b/src/ray/rpc/worker/core_worker_client_pool.cc @@ -39,8 +39,7 @@ std::function CoreWorkerClientPool::GetDefaultUnavailableTimeoutCallback node_id](const rpc::GcsNodeInfo &node_info) { auto raylet_addr = RayletClientPool::GenerateRayletAddress( node_id, node_info.node_manager_address(), node_info.node_manager_port()); - auto raylet_client = - raylet_client_pool->GetOrConnectByAddress(std::move(raylet_addr)); + auto raylet_client = raylet_client_pool->GetOrConnectByAddress(raylet_addr); raylet_client->IsLocalWorkerDead( worker_id, [worker_client_pool, worker_id, node_id](const Status &status, From 25bb6248fdef334f732b8c8c73f0c575842b622f Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Sun, 14 Sep 2025 01:38:40 -0700 Subject: [PATCH 614/634] [core] Fixing timeout in test_object_spilling_3.py (#56512) Signed-off-by: joshlee --- python/ray/tests/test_object_spilling_3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/ray/tests/test_object_spilling_3.py b/python/ray/tests/test_object_spilling_3.py index a74596088eee..7b53228e0dc4 100644 --- a/python/ray/tests/test_object_spilling_3.py +++ b/python/ray/tests/test_object_spilling_3.py @@ -330,6 +330,8 @@ def test_spill_reconstruction_errors(ray_start_cluster, object_spilling_config): "max_direct_call_object_size": 100, "task_retry_delay_ms": 100, "object_timeout_milliseconds": 200, + # Required for reducing the retry time of RequestWorkerLease + "raylet_rpc_server_reconnect_timeout_s": 0, } cluster = ray_start_cluster # Head node with no resources. From 8259540a7127c3f9df0c46a6b78ba13f5e4ed286 Mon Sep 17 00:00:00 2001 From: Joshua Lee <73967497+Sparks0219@users.noreply.github.com> Date: Sun, 14 Sep 2025 22:44:29 -0700 Subject: [PATCH 615/634] [core] Fix UBSAN errors in object_manager_test (#56521) Signed-off-by: joshlee --- src/ray/object_manager/tests/object_manager_test.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ray/object_manager/tests/object_manager_test.cc b/src/ray/object_manager/tests/object_manager_test.cc index 72fe55f7ad2d..7010ee53a7b8 100644 --- a/src/ray/object_manager/tests/object_manager_test.cc +++ b/src/ray/object_manager/tests/object_manager_test.cc @@ -26,6 +26,7 @@ #include "mock/ray/object_manager/object_directory.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" +#include "ray/common/ray_config.h" #include "ray/common/ray_object.h" #include "ray/common/status.h" #include "ray/object_manager/common.h" @@ -45,8 +46,15 @@ class ObjectManagerTest : public ::testing::Test { ObjectManagerConfig config_; config_.object_manager_address = "127.0.0.1"; config_.object_manager_port = 0; + config_.timer_freq_ms = RayConfig::instance().object_manager_timer_freq_ms(); + config_.pull_timeout_ms = RayConfig::instance().object_manager_pull_timeout_ms(); + config_.object_chunk_size = RayConfig::instance().object_manager_default_chunk_size(); + config_.max_bytes_in_flight = + RayConfig::instance().object_manager_max_bytes_in_flight(); config_.store_socket_name = "test_store_socket"; + config_.push_timeout_ms = RayConfig::instance().object_manager_push_timeout_ms(); config_.rpc_service_threads_number = 1; + config_.huge_pages = false; local_node_id_ = NodeID::FromRandom(); mock_gcs_client_ = std::make_unique(); From f896b686f52fa46ec2ab2d75d4660b99b3f54c3f Mon Sep 17 00:00:00 2001 From: Ibrahim Rabbani Date: Mon, 15 Sep 2025 02:18:51 -0700 Subject: [PATCH 616/634] [core] Creating non-linux implementation for sysfs_cgroup_driver. (#56483) Signed-off-by: Ibrahim Rabbani Signed-off-by: israbbani Co-authored-by: Ibrahim Rabbani Co-authored-by: Jiajun Yao --- src/ray/common/cgroup2/BUILD.bazel | 35 ++++----- .../cgroup2/noop_sysfs_cgroup_driver.cc | 74 +++++++++++++++++++ src/ray/common/cgroup2/sysfs_cgroup_driver.cc | 6 ++ src/ray/common/cgroup2/sysfs_cgroup_driver.h | 18 ++--- 4 files changed, 101 insertions(+), 32 deletions(-) create mode 100644 src/ray/common/cgroup2/noop_sysfs_cgroup_driver.cc diff --git a/src/ray/common/cgroup2/BUILD.bazel b/src/ray/common/cgroup2/BUILD.bazel index a3f39c7040ad..b2becaa0575b 100644 --- a/src/ray/common/cgroup2/BUILD.bazel +++ b/src/ray/common/cgroup2/BUILD.bazel @@ -12,7 +12,10 @@ ray_cc_library( ":is_linux": ["cgroup_manager.cc"], "//conditions:default": ["noop_cgroup_manager.cc"], }), - hdrs = ["cgroup_manager.h"], + hdrs = [ + "cgroup_manager.h", + "scoped_cgroup_operation.h", + ], visibility = ["//visibility:public"], deps = [ ":cgroup_driver_interface", @@ -21,7 +24,6 @@ ray_cc_library( "//src/ray/common:status_or", ] + select({ ":is_linux": [ - ":scoped_cgroup_operation", "//src/ray/util:logging", "@com_google_absl//absl/strings", ], @@ -55,35 +57,28 @@ ray_cc_library( ray_cc_library( name = "sysfs_cgroup_driver", - srcs = ["sysfs_cgroup_driver.cc"], + srcs = select({ + ":is_linux": ["sysfs_cgroup_driver.cc"], + "//conditions:default": ["noop_sysfs_cgroup_driver.cc"], + }), hdrs = [ "sysfs_cgroup_driver.h", ], - target_compatible_with = [ - "@platforms//os:linux", - ], visibility = ["//visibility:public"], deps = [ ":cgroup_driver_interface", "//src/ray/common:status", "//src/ray/common:status_or", - "//src/ray/util:logging", - "@com_google_absl//absl/strings", - ], + ] + select({ + ":is_linux": [ + "//src/ray/util:logging", + "@com_google_absl//absl/strings", + ], + "//conditions:default": [], + }), ) # Private Targets. -ray_cc_library( - name = "scoped_cgroup_operation", - hdrs = [ - "scoped_cgroup_operation.h", - ], - target_compatible_with = [ - "@platforms//os:linux", - ], - visibility = [":__subpackages__"], -) - ray_cc_library( name = "fake_cgroup_driver", hdrs = [ diff --git a/src/ray/common/cgroup2/noop_sysfs_cgroup_driver.cc b/src/ray/common/cgroup2/noop_sysfs_cgroup_driver.cc new file mode 100644 index 000000000000..b448f021a8ad --- /dev/null +++ b/src/ray/common/cgroup2/noop_sysfs_cgroup_driver.cc @@ -0,0 +1,74 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "ray/common/cgroup2/sysfs_cgroup_driver.h" +#include "ray/common/status.h" +#include "ray/common/status_or.h" + +namespace ray { +Status SysFsCgroupDriver::CheckCgroupv2Enabled() { return Status::OK(); } + +Status SysFsCgroupDriver::CheckCgroup(const std::string &cgroup_path) { + return Status::OK(); +} + +Status SysFsCgroupDriver::CreateCgroup(const std::string &cgroup_path) { + return Status::OK(); +} + +Status SysFsCgroupDriver::DeleteCgroup(const std::string &cgroup_path) { + return Status::OK(); +} + +StatusOr> SysFsCgroupDriver::GetAvailableControllers( + const std::string &cgroup_dir) { + return std::unordered_set{}; +} + +StatusOr> SysFsCgroupDriver::GetEnabledControllers( + const std::string &cgroup_dir) { + return std::unordered_set{}; +} + +Status SysFsCgroupDriver::MoveAllProcesses(const std::string &from, + const std::string &to) { + return Status::OK(); +} + +Status SysFsCgroupDriver::EnableController(const std::string &cgroup_path, + const std::string &controller) { + return Status::OK(); +} + +Status SysFsCgroupDriver::DisableController(const std::string &cgroup_path, + const std::string &controller) { + return Status::OK(); +} + +Status SysFsCgroupDriver::AddConstraint(const std::string &cgroup_path, + const std::string &controller, + const std::string &constraint, + const std::string &constraint_value) { + return Status::OK(); +} + +StatusOr> SysFsCgroupDriver::ReadControllerFile( + const std::string &controller_file_path) { + return std::unordered_set{}; +} + +} // namespace ray diff --git a/src/ray/common/cgroup2/sysfs_cgroup_driver.cc b/src/ray/common/cgroup2/sysfs_cgroup_driver.cc index b36960c6127c..afa5be6ce544 100644 --- a/src/ray/common/cgroup2/sysfs_cgroup_driver.cc +++ b/src/ray/common/cgroup2/sysfs_cgroup_driver.cc @@ -37,6 +37,12 @@ #include "ray/common/status.h" #include "ray/common/status_or.h" +// Used to identify if a filesystem is mounted using cgroupv2. +// See: https://docs.kernel.org/admin-guide/cgroup-v2.html#mounting +#ifndef CGROUP2_SUPER_MAGIC +#define CGROUP2_SUPER_MAGIC 0x63677270 +#endif + namespace ray { Status SysFsCgroupDriver::CheckCgroupv2Enabled() { FILE *fp = setmntent(mount_file_path_.c_str(), "r"); diff --git a/src/ray/common/cgroup2/sysfs_cgroup_driver.h b/src/ray/common/cgroup2/sysfs_cgroup_driver.h index de5104caeec6..6b01fbe4886f 100644 --- a/src/ray/common/cgroup2/sysfs_cgroup_driver.h +++ b/src/ray/common/cgroup2/sysfs_cgroup_driver.h @@ -13,8 +13,10 @@ // limitations under the License. #pragma once -#include -#include +// TODO(#54703): SysFsCgroupDriver should not be a public target. +// It will be hidden behind a CgroupManagerFactory which will create +// an appropriate depending on configuration and platform. +// #include #include #include @@ -24,12 +26,6 @@ #include "ray/common/status.h" #include "ray/common/status_or.h" -// Used to identify if a filesystem is mounted using cgroupv2. -// See: https://docs.kernel.org/admin-guide/cgroup-v2.html#mounting -#ifndef CGROUP2_SUPER_MAGIC -#define CGROUP2_SUPER_MAGIC 0x63677270 -#endif - namespace ray { /** @@ -46,12 +42,9 @@ namespace ray { class SysFsCgroupDriver : public CgroupDriverInterface { public: /** - * MOUNTED is defined in mntent.h (and typically refers to /etc/mtab) - * @see https://www.gnu.org/software/libc/manual/2.24/html_node/Mount-Information.html - * * @param mount_file_path only used for testing. */ - explicit SysFsCgroupDriver(std::string mount_file_path = MOUNTED) + explicit SysFsCgroupDriver(std::string mount_file_path = kMountFilePath) : mount_file_path_(std::move(mount_file_path)) {} ~SysFsCgroupDriver() override = default; @@ -280,5 +273,6 @@ class SysFsCgroupDriver : public CgroupDriverInterface { static constexpr std::string_view kCgroupSubtreeControlFilename = "cgroup.subtree_control"; static constexpr std::string_view kCgroupControllersFilename = "cgroup.controllers"; + static inline std::string kMountFilePath = "/etc/mtab"; }; } // namespace ray From 33200ddb635cb54c92a591b55467248fff3dfe42 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 15 Sep 2025 08:42:05 -0700 Subject: [PATCH 617/634] [image] add label for ray version and commit (#56493) so that it is easier to detect the ray version in the image Signed-off-by: Lonnie Liu --- ci/build/build-ray-docker.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/build/build-ray-docker.sh b/ci/build/build-ray-docker.sh index 59857533d5ee..86325491d97f 100755 --- a/ci/build/build-ray-docker.sh +++ b/ci/build/build-ray-docker.sh @@ -7,6 +7,9 @@ CONSTRAINTS_FILE="$3" DEST_IMAGE="$4" PIP_FREEZE_FILE="$5" +RAY_VERSION="$(python python/ray/_version.py | cut -d' ' -f1)" +RAY_COMMIT="$(git rev-parse HEAD)" + CPU_TMP="$(mktemp -d)" cp -r .whl "${CPU_TMP}/.whl" @@ -20,6 +23,8 @@ tar --mtime="UTC 2020-01-01" -c -f - . \ --build-arg FULL_BASE_IMAGE="$SOURCE_IMAGE" \ --build-arg WHEEL_PATH=".whl/${WHEEL_NAME}" \ --build-arg CONSTRAINTS_FILE="$CONSTRAINTS_FILE" \ + --label "io.ray.ray-version=$RAY_VERSION" \ + --label "io.ray.ray-commit=$RAY_COMMIT" \ -t "$DEST_IMAGE" -f Dockerfile - # Copy the pip freeze file to the artifact mount. From 7845f4ca7eb1c80618694aa8a4a0700a4ee98667 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 15 Sep 2025 11:04:00 -0500 Subject: [PATCH 618/634] [core] Fix `clang-format` pre-commit step (#56534) The existing one seemed to do nothing... swapped to using the recommendation from this [stack overflow post](https://stackoverflow.com/questions/55965712/how-do-i-add-clang-formatting-to-pre-commit-hook). --------- Signed-off-by: Edward Oakes --- .pre-commit-config.yaml | 7 ++++--- ci/lint/format.sh | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb79122d44b8..4029869cda16 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -175,11 +175,12 @@ repos: # 1091: Not following {file} due to some error # 2207: Prefer mapfile or read -a to split command output (or quote to avoid splitting). -- these aren't compatible with macOS's old Bash - - repo: https://github.com/pocc/pre-commit-hooks - rev: v1.3.5 + - repo: https://github.com/pre-commit/mirrors-clang-format + # `rev` specifies a tag on the above repo that mirrors the corresponding clang-format version. + # The version should be kept in sync with the version in `ci/lint/format.sh`. + rev: v12.0.1 hooks: - id: clang-format - args: [--version=12.0.1] - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks rev: v2.11.0 diff --git a/ci/lint/format.sh b/ci/lint/format.sh index a540b3d65e21..b2a98d1527bb 100755 --- a/ci/lint/format.sh +++ b/ci/lint/format.sh @@ -88,6 +88,7 @@ else fi if command -v clang-format >/dev/null; then + # This version should be kept in sync with the clang-format version tag in `.pre-commit-config.yaml`. CLANG_FORMAT_VERSION=$(clang-format --version | awk '{print $3}') tool_version_check "clang-format" "$CLANG_FORMAT_VERSION" "12.0.1" else From f7ddcbe191d8b81d0349b2ffdf6ed4cef79b2767 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Mon, 15 Sep 2025 09:06:43 -0700 Subject: [PATCH 619/634] [ci][deps] raydepsets: building ray img lockfiles (#56444) building ray img lockfiles for all supported python versions --------- Signed-off-by: elliot-barn Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> --- .buildkite/build.rayci.yml | 25 - .buildkite/dependencies.rayci.yml | 15 +- ci/build/build-placeholder-wheel.sh | 11 + ci/docker/manylinux.Dockerfile | 3 + ci/raydepsets/cli.py | 1 - ci/raydepsets/rayimg.depsets.yaml | 31 + ci/raydepsets/rayllm.depsets.yaml | 1 + ci/raydepsets/tests/test_cli.py | 2 - python/deplocks/llm/ray_py311_cpu.lock | 2 +- python/deplocks/llm/ray_py311_cu121.lock | 2 +- python/deplocks/llm/ray_py311_cu128.lock | 2 +- python/deplocks/llm/ray_test_py311_cpu.lock | 2 +- python/deplocks/llm/ray_test_py311_cu121.lock | 2 +- python/deplocks/llm/ray_test_py311_cu128.lock | 2 +- python/deplocks/llm/rayllm_py311_cpu.lock | 2 +- python/deplocks/llm/rayllm_py311_cu121.lock | 4 +- python/deplocks/llm/rayllm_py311_cu128.lock | 4 +- .../deplocks/llm/rayllm_test_py311_cpu.lock | 4 +- .../deplocks/llm/rayllm_test_py311_cu121.lock | 4 +- .../deplocks/llm/rayllm_test_py311_cu128.lock | 4 +- python/deplocks/ray_img/ray_img_py310.lock | 2173 +++++++++++++++++ python/deplocks/ray_img/ray_img_py311.lock | 2162 ++++++++++++++++ python/deplocks/ray_img/ray_img_py312.lock | 2172 ++++++++++++++++ python/deplocks/ray_img/ray_img_py39.lock | 2173 +++++++++++++++++ 24 files changed, 8756 insertions(+), 47 deletions(-) create mode 100755 ci/build/build-placeholder-wheel.sh create mode 100644 ci/raydepsets/rayimg.depsets.yaml create mode 100644 python/deplocks/ray_img/ray_img_py310.lock create mode 100644 python/deplocks/ray_img/ray_img_py311.lock create mode 100644 python/deplocks/ray_img/ray_img_py312.lock create mode 100644 python/deplocks/ray_img/ray_img_py39.lock diff --git a/.buildkite/build.rayci.yml b/.buildkite/build.rayci.yml index 3ca111f3ad16..01256a3b63c8 100644 --- a/.buildkite/build.rayci.yml +++ b/.buildkite/build.rayci.yml @@ -19,31 +19,6 @@ steps: - manylinux - forge - - label: ":tapioca: build & test placeholder wheel {{matrix}}" - key: placeholder_wheels - tags: - - python - instance_type: medium - commands: - # validate minimal installation - - python ./ci/env/check_minimal_install.py --expected-python-version {{matrix}} - - python -m pip install --upgrade pip - # build placeholder wheel - - export RAY_DEBUG_BUILD=deps-only - - mkdir -p .whl - - pip wheel python/ --no-deps -w .whl/ --use-pep517 - - ls -a .whl - # test placeholder wheel - - ./ci/build/test-linux-placeholder-wheel.sh {{matrix}} - depends_on: - - minbuild-core - job_env: minbuild-core-py{{matrix}} - matrix: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - label: ":tapioca: build: jar" key: java_wheels tags: diff --git a/.buildkite/dependencies.rayci.yml b/.buildkite/dependencies.rayci.yml index 002b0899d4ed..2620a7ee399c 100644 --- a/.buildkite/dependencies.rayci.yml +++ b/.buildkite/dependencies.rayci.yml @@ -25,5 +25,16 @@ steps: instance_type: small command: ./ci/test_compile_llm_requirements.sh job_env: manylinux - depends_on: - - manylinux + depends_on: manylinux + + - label: ":tapioca: build: raydepsets: compile ray img dependencies" + key: raydepsets_compile_rayimg_dependencies + tags: always + instance_type: medium + commands: + # build placeholder wheel for all python versions + - bash ci/build/build-placeholder-wheel.sh + # compile rayimg dependencies + - bazel run //ci/raydepsets:raydepsets -- build ci/raydepsets/rayimg.depsets.yaml --check + job_env: manylinux + depends_on: manylinux diff --git a/ci/build/build-placeholder-wheel.sh b/ci/build/build-placeholder-wheel.sh new file mode 100755 index 000000000000..effa117c3da8 --- /dev/null +++ b/ci/build/build-placeholder-wheel.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -exuo pipefail + +export RAY_DEBUG_BUILD=deps-only + +PYTHON_VERSIONS=("3.9" "3.10" "3.11" "3.12") + +for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do + uv build --wheel --directory python/ -o ../.whl/ --force-pep517 --python "$PYTHON_VERSION" +done diff --git a/ci/docker/manylinux.Dockerfile b/ci/docker/manylinux.Dockerfile index c9c3111b3077..7a243e5033f1 100644 --- a/ci/docker/manylinux.Dockerfile +++ b/ci/docker/manylinux.Dockerfile @@ -13,6 +13,9 @@ ENV BUILDKITE_BAZEL_CACHE_URL=$BUILDKITE_BAZEL_CACHE_URL RUN yum -y install sudo +RUN curl -LsSf https://astral.sh/uv/0.8.17/install.sh | \ + env UV_INSTALL_DIR=/usr/local/bin sh + COPY ci/build/build-manylinux-forge.sh /tmp/build-manylinux-forge.sh RUN ./tmp/build-manylinux-forge.sh diff --git a/ci/raydepsets/cli.py b/ci/raydepsets/cli.py index 0f9aa59e66e2..6e73b7b5b13e 100644 --- a/ci/raydepsets/cli.py +++ b/ci/raydepsets/cli.py @@ -16,7 +16,6 @@ DEFAULT_UV_FLAGS = """ --generate-hashes --strip-extras - --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/ci/raydepsets/rayimg.depsets.yaml b/ci/raydepsets/rayimg.depsets.yaml new file mode 100644 index 000000000000..88b0a3d341de --- /dev/null +++ b/ci/raydepsets/rayimg.depsets.yaml @@ -0,0 +1,31 @@ +build_arg_sets: + py39: + PYTHON_VERSION: "3.9" + PYTHON_SHORT: "39" + py310: + PYTHON_VERSION: "3.10" + PYTHON_SHORT: "310" + py311: + PYTHON_VERSION: "3.11" + PYTHON_SHORT: "311" + py312: + PYTHON_VERSION: "3.12" + PYTHON_SHORT: "312" + + +depsets: + - name: ray_img_depset_${PYTHON_SHORT} + packages: + - ray[all]==100.0.0-dev + constraints: + - python/requirements_compiled.txt + output: python/deplocks/ray_img/ray_img_py${PYTHON_SHORT}.lock + operation: compile + append_flags: + - --python-version=${PYTHON_VERSION} + - --find-links=.whl/ + build_arg_sets: + - py39 + - py310 + - py311 + - py312 diff --git a/ci/raydepsets/rayllm.depsets.yaml b/ci/raydepsets/rayllm.depsets.yaml index e34be3bd104c..b6ed8fb51cec 100644 --- a/ci/raydepsets/rayllm.depsets.yaml +++ b/ci/raydepsets/rayllm.depsets.yaml @@ -15,6 +15,7 @@ build_arg_sets: - --extra-index-url https://download.pytorch.org/whl/${CUDA_CODE} append_flags: - --python-version=3.11 + - --unsafe-package ray - --python-platform=linux build_arg_sets: - cpu diff --git a/ci/raydepsets/tests/test_cli.py b/ci/raydepsets/tests/test_cli.py index 49943b1f7127..d44ce195711f 100644 --- a/ci/raydepsets/tests/test_cli.py +++ b/ci/raydepsets/tests/test_cli.py @@ -352,8 +352,6 @@ def test_override_uv_flag_single_flag(self): def test_override_uv_flag_multiple_flags(self): expected_flags = DEFAULT_UV_FLAGS.copy() expected_flags.remove("--unsafe-package") - expected_flags.remove("ray") - expected_flags.remove("--unsafe-package") expected_flags.remove("setuptools") expected_flags.extend(["--unsafe-package", "dummy"]) assert ( diff --git a/python/deplocks/llm/ray_py311_cpu.lock b/python/deplocks/llm/ray_py311_cpu.lock index 943eb4050549..9e76d9282fb9 100644 --- a/python/deplocks/llm/ray_py311_cpu.lock +++ b/python/deplocks/llm/ray_py311_cpu.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cpu.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cpu.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --unsafe-package ray --python-platform=linux -c python/deplocks/llm/ray_test_py311_cpu.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cpu.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/deplocks/llm/ray_py311_cu121.lock b/python/deplocks/llm/ray_py311_cu121.lock index e1d9eee4a8ef..e9a88445ed84 100644 --- a/python/deplocks/llm/ray_py311_cu121.lock +++ b/python/deplocks/llm/ray_py311_cu121.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu121.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cu121.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --unsafe-package ray --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu121.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cu121.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/deplocks/llm/ray_py311_cu128.lock b/python/deplocks/llm/ray_py311_cu128.lock index a685eb63910c..d65b8adf1934 100644 --- a/python/deplocks/llm/ray_py311_cu128.lock +++ b/python/deplocks/llm/ray_py311_cu128.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu128.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cu128.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --unsafe-package ray --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu128.lock python/requirements.txt -o python/deplocks/llm/ray_py311_cu128.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/python/deplocks/llm/ray_test_py311_cpu.lock b/python/deplocks/llm/ray_test_py311_cpu.lock index 34f24980a9f5..2681827678b0 100644 --- a/python/deplocks/llm/ray_test_py311_cpu.lock +++ b/python/deplocks/llm/ray_test_py311_cpu.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/deplocks/llm/ray_test_py311_cpu.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --unsafe-package ray --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/deplocks/llm/ray_test_py311_cpu.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/deplocks/llm/ray_test_py311_cu121.lock b/python/deplocks/llm/ray_test_py311_cu121.lock index 833d864f0280..d61e27a0a6c9 100644 --- a/python/deplocks/llm/ray_test_py311_cu121.lock +++ b/python/deplocks/llm/ray_test_py311_cu121.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/deplocks/llm/ray_test_py311_cu121.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --unsafe-package ray --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/deplocks/llm/ray_test_py311_cu121.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/python/deplocks/llm/ray_test_py311_cu128.lock b/python/deplocks/llm/ray_test_py311_cu128.lock index 19466b6fc222..058fe256012f 100644 --- a/python/deplocks/llm/ray_test_py311_cu128.lock +++ b/python/deplocks/llm/ray_test_py311_cu128.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/deplocks/llm/ray_test_py311_cu128.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --unsafe-package ray --python-platform=linux -c /tmp/ray-deps/requirements_compiled.txt python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt -o python/deplocks/llm/ray_test_py311_cu128.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/python/deplocks/llm/rayllm_py311_cpu.lock b/python/deplocks/llm/rayllm_py311_cpu.lock index 89da379ee9c5..de5b155c04c0 100644 --- a/python/deplocks/llm/rayllm_py311_cpu.lock +++ b/python/deplocks/llm/rayllm_py311_cpu.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/deplocks/llm/rayllm_test_py311_cpu.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cpu.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --unsafe-package ray --python-platform=linux -c python/deplocks/llm/rayllm_test_py311_cpu.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cpu.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/python/deplocks/llm/rayllm_py311_cu121.lock b/python/deplocks/llm/rayllm_py311_cu121.lock index 7a291b4f9eac..777b41bc8ccd 100644 --- a/python/deplocks/llm/rayllm_py311_cu121.lock +++ b/python/deplocks/llm/rayllm_py311_cu121.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/rayllm_test_py311_cu121.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cu121.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --unsafe-package ray --python-platform=linux -c python/deplocks/llm/rayllm_test_py311_cu121.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cu121.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 @@ -3964,5 +3964,5 @@ zipp==3.19.2 \ # importlib-metadata # The following packages were excluded from the output: -# ray # setuptools +# ray diff --git a/python/deplocks/llm/rayllm_py311_cu128.lock b/python/deplocks/llm/rayllm_py311_cu128.lock index 41dcc9dce7dc..1af647c3d386 100644 --- a/python/deplocks/llm/rayllm_py311_cu128.lock +++ b/python/deplocks/llm/rayllm_py311_cu128.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/rayllm_test_py311_cu128.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cu128.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --unsafe-package ray --python-platform=linux -c python/deplocks/llm/rayllm_test_py311_cu128.lock python/requirements.txt python/requirements/llm/llm-requirements.txt -o python/deplocks/llm/rayllm_py311_cu128.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 @@ -3855,5 +3855,5 @@ zipp==3.19.2 \ # importlib-metadata # The following packages were excluded from the output: -# ray # setuptools +# ray diff --git a/python/deplocks/llm/rayllm_test_py311_cpu.lock b/python/deplocks/llm/rayllm_test_py311_cpu.lock index 27b0f5a357e6..acbc353b49cb 100644 --- a/python/deplocks/llm/rayllm_test_py311_cpu.lock +++ b/python/deplocks/llm/rayllm_test_py311_cpu.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cpu.lock python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/deplocks/llm/rayllm_test_py311_cpu.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cpu --python-version=3.11 --unsafe-package ray --python-platform=linux -c python/deplocks/llm/ray_test_py311_cpu.lock python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/deplocks/llm/rayllm_test_py311_cpu.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu @@ -5092,5 +5092,5 @@ zipp==3.19.2 \ # importlib-metadata # The following packages were excluded from the output: -# ray # setuptools +# ray diff --git a/python/deplocks/llm/rayllm_test_py311_cu121.lock b/python/deplocks/llm/rayllm_test_py311_cu121.lock index be1ce34fce70..17c70071f3fb 100644 --- a/python/deplocks/llm/rayllm_test_py311_cu121.lock +++ b/python/deplocks/llm/rayllm_test_py311_cu121.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu121.lock python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/deplocks/llm/rayllm_test_py311_cu121.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu121 --python-version=3.11 --unsafe-package ray --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu121.lock python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/deplocks/llm/rayllm_test_py311_cu121.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu121 @@ -5203,5 +5203,5 @@ zipp==3.19.2 \ # importlib-metadata # The following packages were excluded from the output: -# ray # setuptools +# ray diff --git a/python/deplocks/llm/rayllm_test_py311_cu128.lock b/python/deplocks/llm/rayllm_test_py311_cu128.lock index df88832ae1be..51ce2f2ab445 100644 --- a/python/deplocks/llm/rayllm_test_py311_cu128.lock +++ b/python/deplocks/llm/rayllm_test_py311_cu128.lock @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --strip-extras --unsafe-package ray --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu128.lock python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/deplocks/llm/rayllm_test_py311_cu128.lock +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --extra-index-url https://download.pytorch.org/whl/cu128 --python-version=3.11 --unsafe-package ray --python-platform=linux -c python/deplocks/llm/ray_test_py311_cu128.lock python/requirements.txt python/requirements/cloud-requirements.txt python/requirements/base-test-requirements.txt python/requirements/llm/llm-requirements.txt python/requirements/llm/llm-test-requirements.txt -o python/deplocks/llm/rayllm_test_py311_cu128.lock --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cu128 @@ -5093,5 +5093,5 @@ zipp==3.19.2 \ # importlib-metadata # The following packages were excluded from the output: -# ray # setuptools +# ray diff --git a/python/deplocks/ray_img/ray_img_py310.lock b/python/deplocks/ray_img/ray_img_py310.lock new file mode 100644 index 000000000000..c6d9fab2624b --- /dev/null +++ b/python/deplocks/ray_img/ray_img_py310.lock @@ -0,0 +1,2173 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.10 --find-links=.whl/ -c python/requirements_compiled.txt - -o python/deplocks/ray_img/ray_img_py310.lock +--index-url https://pypi.org/simple +--extra-index-url https://download.pytorch.org/whl/cpu +--find-links .whl/ +--find-links https://data.pyg.org/whl/torch-2.3.0+cpu.html + +aiohappyeyeballs==2.6.1 \ + --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ + --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 + # via + # -c python/requirements_compiled.txt + # aiohttp +aiohttp==3.11.16 \ + --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ + --hash=sha256:0902e887b0e1d50424112f200eb9ae3dfed6c0d0a19fc60f633ae5a57c809656 \ + --hash=sha256:09b00dd520d88eac9d1768439a59ab3d145065c91a8fab97f900d1b5f802895e \ + --hash=sha256:0a2f451849e6b39e5c226803dcacfa9c7133e9825dcefd2f4e837a2ec5a3bb98 \ + --hash=sha256:0a950c2eb8ff17361abd8c85987fd6076d9f47d040ebffce67dce4993285e973 \ + --hash=sha256:0ad1fb47da60ae1ddfb316f0ff16d1f3b8e844d1a1e154641928ea0583d486ed \ + --hash=sha256:13ceac2c5cdcc3f64b9015710221ddf81c900c5febc505dbd8f810e770011540 \ + --hash=sha256:14461157d8426bcb40bd94deb0450a6fa16f05129f7da546090cebf8f3123b0f \ + --hash=sha256:16f8a2c9538c14a557b4d309ed4d0a7c60f0253e8ed7b6c9a2859a7582f8b1b8 \ + --hash=sha256:17ae4664031aadfbcb34fd40ffd90976671fa0c0286e6c4113989f78bebab37a \ + --hash=sha256:1ce63ae04719513dd2651202352a2beb9f67f55cb8490c40f056cea3c5c355ce \ + --hash=sha256:23a15727fbfccab973343b6d1b7181bfb0b4aa7ae280f36fd2f90f5476805682 \ + --hash=sha256:2540ddc83cc724b13d1838026f6a5ad178510953302a49e6d647f6e1de82bc34 \ + --hash=sha256:37dcee4906454ae377be5937ab2a66a9a88377b11dd7c072df7a7c142b63c37c \ + --hash=sha256:38bea84ee4fe24ebcc8edeb7b54bf20f06fd53ce4d2cc8b74344c5b9620597fd \ + --hash=sha256:3ab3367bb7f61ad18793fea2ef71f2d181c528c87948638366bf1de26e239183 \ + --hash=sha256:3ad1d59fd7114e6a08c4814983bb498f391c699f3c78712770077518cae63ff7 \ + --hash=sha256:3b4e6db8dc4879015b9955778cfb9881897339c8fab7b3676f8433f849425913 \ + --hash=sha256:3e061b09f6fa42997cf627307f220315e313ece74907d35776ec4373ed718b86 \ + --hash=sha256:42864e70a248f5f6a49fdaf417d9bc62d6e4d8ee9695b24c5916cb4bb666c802 \ + --hash=sha256:493910ceb2764f792db4dc6e8e4b375dae1b08f72e18e8f10f18b34ca17d0979 \ + --hash=sha256:4d0c970c0d602b1017e2067ff3b7dac41c98fef4f7472ec2ea26fd8a4e8c2149 \ + --hash=sha256:54eb3aead72a5c19fad07219acd882c1643a1027fbcdefac9b502c267242f955 \ + --hash=sha256:56a3443aca82abda0e07be2e1ecb76a050714faf2be84256dae291182ba59049 \ + --hash=sha256:576f5ca28d1b3276026f7df3ec841ae460e0fc3aac2a47cbf72eabcfc0f102e1 \ + --hash=sha256:58ede86453a6cf2d6ce40ef0ca15481677a66950e73b0a788917916f7e35a0bb \ + --hash=sha256:61c721764e41af907c9d16b6daa05a458f066015abd35923051be8705108ed17 \ + --hash=sha256:634d96869be6c4dc232fc503e03e40c42d32cfaa51712aee181e922e61d74814 \ + --hash=sha256:696ef00e8a1f0cec5e30640e64eca75d8e777933d1438f4facc9c0cdf288a810 \ + --hash=sha256:69a2cbd61788d26f8f1e626e188044834f37f6ae3f937bd9f08b65fc9d7e514e \ + --hash=sha256:6a792ce34b999fbe04a7a71a90c74f10c57ae4c51f65461a411faa70e154154e \ + --hash=sha256:6ac13b71761e49d5f9e4d05d33683bbafef753e876e8e5a7ef26e937dd766713 \ + --hash=sha256:6fdec0213244c39973674ca2a7f5435bf74369e7d4e104d6c7473c81c9bcc8c4 \ + --hash=sha256:72b1b03fb4655c1960403c131740755ec19c5898c82abd3961c364c2afd59fe7 \ + --hash=sha256:745f1ed5e2c687baefc3c5e7b4304e91bf3e2f32834d07baaee243e349624b24 \ + --hash=sha256:776c8e959a01e5e8321f1dec77964cb6101020a69d5a94cd3d34db6d555e01f7 \ + --hash=sha256:780df0d837276276226a1ff803f8d0fa5f8996c479aeef52eb040179f3156cbd \ + --hash=sha256:78e6e23b954644737e385befa0deb20233e2dfddf95dd11e9db752bdd2a294d3 \ + --hash=sha256:7951decace76a9271a1ef181b04aa77d3cc309a02a51d73826039003210bdc86 \ + --hash=sha256:7ba92a2d9ace559a0a14b03d87f47e021e4fa7681dc6970ebbc7b447c7d4b7cd \ + --hash=sha256:7f6428fee52d2bcf96a8aa7b62095b190ee341ab0e6b1bcf50c615d7966fd45b \ + --hash=sha256:87944bd16b7fe6160607f6a17808abd25f17f61ae1e26c47a491b970fb66d8cb \ + --hash=sha256:87a6e922b2b2401e0b0cf6b976b97f11ec7f136bfed445e16384fbf6fd5e8602 \ + --hash=sha256:8cb0688a8d81c63d716e867d59a9ccc389e97ac7037ebef904c2b89334407180 \ + --hash=sha256:8df6612df74409080575dca38a5237282865408016e65636a76a2eb9348c2567 \ + --hash=sha256:911a6e91d08bb2c72938bc17f0a2d97864c531536b7832abee6429d5296e5b27 \ + --hash=sha256:92b7ee222e2b903e0a4b329a9943d432b3767f2d5029dbe4ca59fb75223bbe2e \ + --hash=sha256:938f756c2b9374bbcc262a37eea521d8a0e6458162f2a9c26329cc87fdf06534 \ + --hash=sha256:9756d9b9d4547e091f99d554fbba0d2a920aab98caa82a8fb3d3d9bee3c9ae85 \ + --hash=sha256:98b88a2bf26965f2015a771381624dd4b0839034b70d406dc74fd8be4cc053e3 \ + --hash=sha256:9b751a6306f330801665ae69270a8a3993654a85569b3469662efaad6cf5cc50 \ + --hash=sha256:a2a450bcce4931b295fc0848f384834c3f9b00edfc2150baafb4488c27953de6 \ + --hash=sha256:a3814760a1a700f3cfd2f977249f1032301d0a12c92aba74605cfa6ce9f78489 \ + --hash=sha256:a5abcbba9f4b463a45c8ca8b7720891200658f6f46894f79517e6cd11f3405ca \ + --hash=sha256:a6db7458ab89c7d80bc1f4e930cc9df6edee2200127cfa6f6e080cf619eddfbd \ + --hash=sha256:ad497f38a0d6c329cb621774788583ee12321863cd4bd9feee1effd60f2ad133 \ + --hash=sha256:ad9509ffb2396483ceacb1eee9134724443ee45b92141105a4645857244aecc8 \ + --hash=sha256:bbcba75fe879ad6fd2e0d6a8d937f34a571f116a0e4db37df8079e738ea95c71 \ + --hash=sha256:c10d85e81d0b9ef87970ecbdbfaeec14a361a7fa947118817fcea8e45335fa46 \ + --hash=sha256:c15b2271c44da77ee9d822552201180779e5e942f3a71fb74e026bf6172ff287 \ + --hash=sha256:ca37057625693d097543bd88076ceebeb248291df9d6ca8481349efc0b05dcd0 \ + --hash=sha256:cc3a145479a76ad0ed646434d09216d33d08eef0d8c9a11f5ae5cdc37caa3540 \ + --hash=sha256:ccf10f16ab498d20e28bc2b5c1306e9c1512f2840f7b6a67000a517a4b37d5ee \ + --hash=sha256:cd464ba806e27ee24a91362ba3621bfc39dbbb8b79f2e1340201615197370f7c \ + --hash=sha256:d007aa39a52d62373bd23428ba4a2546eed0e7643d7bf2e41ddcefd54519842c \ + --hash=sha256:d0666afbe984f6933fe72cd1f1c3560d8c55880a0bdd728ad774006eb4241ecd \ + --hash=sha256:d07502cc14ecd64f52b2a74ebbc106893d9a9717120057ea9ea1fd6568a747e7 \ + --hash=sha256:d489d9778522fbd0f8d6a5c6e48e3514f11be81cb0a5954bdda06f7e1594b321 \ + --hash=sha256:df7db76400bf46ec6a0a73192b14c8295bdb9812053f4fe53f4e789f3ea66bbb \ + --hash=sha256:e3538bc9fe1b902bef51372462e3d7c96fce2b566642512138a480b7adc9d508 \ + --hash=sha256:e87fd812899aa78252866ae03a048e77bd11b80fb4878ce27c23cade239b42b2 \ + --hash=sha256:ecdb8173e6c7aa09eee342ac62e193e6904923bd232e76b4157ac0bfa670609f \ + --hash=sha256:f244b8e541f414664889e2c87cac11a07b918cb4b540c36f7ada7bfa76571ea2 \ + --hash=sha256:f4065145bf69de124accdd17ea5f4dc770da0a6a6e440c53f6e0a8c27b3e635c \ + --hash=sha256:f420bfe862fb357a6d76f2065447ef6f484bc489292ac91e29bc65d2d7a2c84d \ + --hash=sha256:f6ddd90d9fb4b501c97a4458f1c1720e42432c26cb76d28177c5b5ad4e332601 \ + --hash=sha256:fa73e8c2656a3653ae6c307b3f4e878a21f87859a9afab228280ddccd7369d71 \ + --hash=sha256:fadbb8f1d4140825069db3fedbbb843290fd5f5bc0a5dbd7eaf81d91bf1b003b \ + --hash=sha256:fb3d0cc5cdb926090748ea60172fa8a213cec728bd6c54eae18b96040fcd6227 \ + --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ + --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb + # via + # -c python/requirements_compiled.txt + # aiohttp-cors + # ray +aiohttp-cors==0.7.0 \ + --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ + --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d + # via + # -c python/requirements_compiled.txt + # ray +aiosignal==1.3.1 \ + --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ + --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 + # via + # -c python/requirements_compiled.txt + # aiohttp +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled.txt + # kombu +annotated-types==0.6.0 \ + --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ + --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d + # via + # -c python/requirements_compiled.txt + # pydantic +anyio==3.7.1 \ + --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ + --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 + # via + # -c python/requirements_compiled.txt + # starlette + # watchfiles +async-timeout==4.0.3 ; python_full_version < '3.11' \ + --hash=sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f \ + --hash=sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028 + # via + # -c python/requirements_compiled.txt + # aiohttp +attrs==25.1.0 \ + --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ + --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a + # via + # -c python/requirements_compiled.txt + # aiohttp + # jsonschema + # referencing +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled.txt + # celery +cachetools==5.5.2 \ + --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ + --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a + # via + # -c python/requirements_compiled.txt + # google-auth +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled.txt + # ray +certifi==2025.1.31 \ + --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ + --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe + # via + # -c python/requirements_compiled.txt + # requests +cffi==1.16.0 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ + --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \ + --hash=sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417 \ + --hash=sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab \ + --hash=sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520 \ + --hash=sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36 \ + --hash=sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743 \ + --hash=sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8 \ + --hash=sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed \ + --hash=sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684 \ + --hash=sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56 \ + --hash=sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324 \ + --hash=sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d \ + --hash=sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235 \ + --hash=sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e \ + --hash=sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088 \ + --hash=sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000 \ + --hash=sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7 \ + --hash=sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e \ + --hash=sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673 \ + --hash=sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c \ + --hash=sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe \ + --hash=sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2 \ + --hash=sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098 \ + --hash=sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8 \ + --hash=sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a \ + --hash=sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0 \ + --hash=sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b \ + --hash=sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896 \ + --hash=sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e \ + --hash=sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9 \ + --hash=sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2 \ + --hash=sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b \ + --hash=sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6 \ + --hash=sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404 \ + --hash=sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f \ + --hash=sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0 \ + --hash=sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4 \ + --hash=sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc \ + --hash=sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936 \ + --hash=sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba \ + --hash=sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872 \ + --hash=sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb \ + --hash=sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614 \ + --hash=sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1 \ + --hash=sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d \ + --hash=sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969 \ + --hash=sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b \ + --hash=sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4 \ + --hash=sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627 \ + --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ + --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 + # via + # -c python/requirements_compiled.txt + # cryptography +charset-normalizer==3.3.2 \ + --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ + --hash=sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087 \ + --hash=sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786 \ + --hash=sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8 \ + --hash=sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09 \ + --hash=sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185 \ + --hash=sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574 \ + --hash=sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e \ + --hash=sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519 \ + --hash=sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898 \ + --hash=sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269 \ + --hash=sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3 \ + --hash=sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f \ + --hash=sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6 \ + --hash=sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8 \ + --hash=sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a \ + --hash=sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73 \ + --hash=sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc \ + --hash=sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714 \ + --hash=sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2 \ + --hash=sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc \ + --hash=sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce \ + --hash=sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d \ + --hash=sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e \ + --hash=sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6 \ + --hash=sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269 \ + --hash=sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96 \ + --hash=sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d \ + --hash=sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a \ + --hash=sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4 \ + --hash=sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77 \ + --hash=sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d \ + --hash=sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0 \ + --hash=sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed \ + --hash=sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068 \ + --hash=sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac \ + --hash=sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25 \ + --hash=sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8 \ + --hash=sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab \ + --hash=sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26 \ + --hash=sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2 \ + --hash=sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db \ + --hash=sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f \ + --hash=sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5 \ + --hash=sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99 \ + --hash=sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c \ + --hash=sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d \ + --hash=sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811 \ + --hash=sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa \ + --hash=sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a \ + --hash=sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03 \ + --hash=sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b \ + --hash=sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04 \ + --hash=sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c \ + --hash=sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001 \ + --hash=sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458 \ + --hash=sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389 \ + --hash=sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99 \ + --hash=sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985 \ + --hash=sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537 \ + --hash=sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238 \ + --hash=sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f \ + --hash=sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d \ + --hash=sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796 \ + --hash=sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a \ + --hash=sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143 \ + --hash=sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8 \ + --hash=sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c \ + --hash=sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5 \ + --hash=sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5 \ + --hash=sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711 \ + --hash=sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4 \ + --hash=sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6 \ + --hash=sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c \ + --hash=sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7 \ + --hash=sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4 \ + --hash=sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b \ + --hash=sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae \ + --hash=sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12 \ + --hash=sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c \ + --hash=sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae \ + --hash=sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8 \ + --hash=sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887 \ + --hash=sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b \ + --hash=sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4 \ + --hash=sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f \ + --hash=sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5 \ + --hash=sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33 \ + --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ + --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 + # via + # -c python/requirements_compiled.txt + # requests +click==8.1.7 \ + --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ + --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de + # via + # -c python/requirements_compiled.txt + # celery + # click-didyoumean + # click-plugins + # click-repl + # ray + # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled.txt + # celery +cloudpickle==2.2.0 \ + --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ + --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 + # via + # -c python/requirements_compiled.txt + # gymnasium +colorful==0.5.5 \ + --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ + --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d + # via + # -c python/requirements_compiled.txt + # ray +cryptography==44.0.3 \ + --hash=sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259 \ + --hash=sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43 \ + --hash=sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645 \ + --hash=sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8 \ + --hash=sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44 \ + --hash=sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d \ + --hash=sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f \ + --hash=sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d \ + --hash=sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54 \ + --hash=sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9 \ + --hash=sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137 \ + --hash=sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f \ + --hash=sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c \ + --hash=sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334 \ + --hash=sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c \ + --hash=sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b \ + --hash=sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2 \ + --hash=sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375 \ + --hash=sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88 \ + --hash=sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5 \ + --hash=sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647 \ + --hash=sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c \ + --hash=sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359 \ + --hash=sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5 \ + --hash=sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d \ + --hash=sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028 \ + --hash=sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01 \ + --hash=sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904 \ + --hash=sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d \ + --hash=sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93 \ + --hash=sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06 \ + --hash=sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff \ + --hash=sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76 \ + --hash=sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff \ + --hash=sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759 \ + --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ + --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 + # via + # -c python/requirements_compiled.txt + # pyopenssl +cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ + --hash=sha256:230f8a8e99c81a653baa0ed00819990c0ed1f0cf0298214786b5e323461dc61a \ + --hash=sha256:2d16eaa2d086e416ac13467d4ff3184b9a081fe76b761ce51d4a46ec1c4bd28a \ + --hash=sha256:432273fd4b61a284f7d705d08b8291403548fd422bcbd945635cc155bc6a923d \ + --hash=sha256:4c51a1062a3c5a826b0425952d229ffe73b1791656a31de95b318117e67a9576 \ + --hash=sha256:4c8e9fdb1f3ffc3151808f8bb8c871518d2783e1be8b53792b698a840543d60c \ + --hash=sha256:51b1d6cb83d82dfa306c9efaeb4d57f24bad3041ebd8716d61072676abbcf67b \ + --hash=sha256:52185a2cf95d3bac2c3fda95c9c8e06a985b5a00cd2e587d3caace337db33899 \ + --hash=sha256:5afb6658faa22f21479ae2c0a07254df31c0aebc36907a64a1f6be4ecc9e96da \ + --hash=sha256:d3dc91ef9c4104652195eea4b282d343ecad653021efe20d1c8dd8dfe8ccfd86 \ + --hash=sha256:d60d1e124592cb82a5f3f45b3e7bee7bda7b72a743029f275e9d6b125f338c60 \ + --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ + --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa + # via + # -c python/requirements_compiled.txt + # ray +distlib==0.3.7 \ + --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ + --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 + # via + # -c python/requirements_compiled.txt + # virtualenv +dm-tree==0.1.8 \ + --hash=sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6 \ + --hash=sha256:09964470f76a5201aff2e8f9b26842976de7889300676f927930f6285e256760 \ + --hash=sha256:0d3172394079a86c3a759179c65f64c48d1a42b89495fcf38976d11cc3bb952c \ + --hash=sha256:0e9620ccf06393eb6b613b5e366469304622d4ea96ae6540b28a33840e6c89cf \ + --hash=sha256:0fcaabbb14e7980377439e7140bd05552739ca5e515ecb3119f234acee4b9430 \ + --hash=sha256:1607ce49aa42f010d1e5e616d92ce899d66835d4d8bea49679582435285515de \ + --hash=sha256:181c35521d480d0365f39300542cb6cd7fd2b77351bb43d7acfda15aef63b317 \ + --hash=sha256:1d7c26e431fc93cc7e0cba867eb000db6a05f6f2b25af11ac4e9dada88fc5bca \ + --hash=sha256:1fe962015b2fe1282892b28ebe962faed53c7f98d942da9a4625cbf27baef913 \ + --hash=sha256:250b692fb75f45f02e2f58fbef9ab338904ef334b90557565621fa251df267cf \ + --hash=sha256:2869228d9c619074de501a3c10dc7f07c75422f8fab36ecdcb859b6f1b1ec3ef \ + --hash=sha256:28c52cbf4f8b3dbd0beaedf44f69fa85eec5e9dede612e08035e06ada6ec9426 \ + --hash=sha256:2f7915660f59c09068e428613c480150180df1060561fd0d1470684ae7007bd1 \ + --hash=sha256:343a4a4ebaa127451ff971254a4be4084eb4bdc0b2513c32b46f6f728fd03f9e \ + --hash=sha256:35cc164a79336bfcfafb47e5f297898359123bbd3330c1967f0c4994f9cf9f60 \ + --hash=sha256:378cc8ad93c5fe3590f405a309980721f021c790ca1bdf9b15bb1d59daec57f5 \ + --hash=sha256:39070ba268c0491af9fe7a58644d99e8b4f2cde6e5884ba3380bddc84ed43d5f \ + --hash=sha256:435227cf3c5dc63f4de054cf3d00183790bd9ead4c3623138c74dde7f67f521b \ + --hash=sha256:5483dca4d7eb1a0d65fe86d3b6a53ae717face83c1f17e0887b1a4a64ae5c410 \ + --hash=sha256:694c3654cfd2a81552c08ec66bb5c4a3d48fa292b9a181880fb081c36c5b9134 \ + --hash=sha256:75c5d528bb992981c20793b6b453e91560784215dffb8a5440ba999753c14ceb \ + --hash=sha256:803bfc53b4659f447ac694dbd04235f94a73ef7c1fd1e0df7c84ac41e0bc963b \ + --hash=sha256:81fce77f22a302d7a5968aebdf4efafef4def7ce96528719a354e6990dcd49c7 \ + --hash=sha256:83b7764de0d855338abefc6e3ee9fe40d301668310aa3baea3f778ff051f4393 \ + --hash=sha256:8c60a7eadab64c2278861f56bca320b2720f163dca9d7558103c3b77f2416571 \ + --hash=sha256:8ed3564abed97c806db122c2d3e1a2b64c74a63debe9903aad795167cc301368 \ + --hash=sha256:94d3f0826311f45ee19b75f5b48c99466e4218a0489e81c0f0167bda50cacf22 \ + --hash=sha256:96a548a406a6fb15fe58f6a30a57ff2f2aafbf25f05afab00c8f5e5977b6c715 \ + --hash=sha256:a5d819c38c03f0bb5b3b3703c60e4b170355a0fc6b5819325bf3d4ceb3ae7e80 \ + --hash=sha256:ad16ceba90a56ec47cf45b21856d14962ac314787975ef786efb5e6e9ca75ec7 \ + --hash=sha256:af4b3d372f2477dcd89a6e717e4a575ca35ccc20cc4454a8a4b6f8838a00672d \ + --hash=sha256:b095ba4f8ca1ba19350fd53cf1f8f3eb0bd406aa28af64a6dfc86707b32a810a \ + --hash=sha256:b9bd9b9ccb59409d33d51d84b7668010c04c2af7d4a371632874c1ca356cff3d \ + --hash=sha256:b9f89a454e98806b44fe9d40ec9eee61f848388f7e79ac2371a55679bd5a3ac6 \ + --hash=sha256:bb2d109f42190225112da899b9f3d46d0d5f26aef501c61e43529fe9322530b5 \ + --hash=sha256:c0a94aba18a35457a1b5cd716fd7b46c5dafdc4cf7869b4bae665b91c4682a8e \ + --hash=sha256:c5c8c12e3fda754ef6af94161bacdaeda816d941995fac415d6855c6c386af68 \ + --hash=sha256:d1612fcaecd79023dbc6a6ae48d51a80beb5c385d6f3f6d71688e57bc8d07de8 \ + --hash=sha256:d16e1f2a073604cfcc09f7131ae8d534674f43c3aef4c25742eae295bc60d04f \ + --hash=sha256:d20f2faa3672b52e5013f4077117bfb99c4cfc0b445d3bde1584c34032b57436 \ + --hash=sha256:d40fa4106ca6edc66760246a08f500ec0c85ef55c762fb4a363f6ee739ba02ee \ + --hash=sha256:de287fabc464b8734be251e46e06aa9aa1001f34198da2b6ce07bd197172b9cb \ + --hash=sha256:e4d714371bb08839e4e5e29024fc95832d9affe129825ef38836b143028bd144 \ + --hash=sha256:ea9e59e0451e7d29aece402d9f908f2e2a80922bcde2ebfd5dcb07750fcbfee8 \ + --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ + --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d + # via + # -c python/requirements_compiled.txt + # ray +exceptiongroup==1.3.0 ; python_full_version < '3.11' \ + --hash=sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10 \ + --hash=sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88 + # via anyio +farama-notifications==0.0.4 \ + --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ + --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae + # via + # -c python/requirements_compiled.txt + # gymnasium +fastapi==0.115.12 \ + --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ + --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d + # via + # -c python/requirements_compiled.txt + # ray +fastrlock==0.8.2 ; sys_platform != 'darwin' \ + --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ + --hash=sha256:07ed3c7b3867c05a3d6be4ced200c7767000f3431b9be6da66972822dd86e8be \ + --hash=sha256:08315bde19d0c2e6b06593d5a418be3dc8f9b1ee721afa96867b9853fceb45cf \ + --hash=sha256:11bbbbc526363955aeddb9eec4cee2a0012322b7b2f15b54f44454fcf4fd398a \ + --hash=sha256:17734e2e5af4c07ddb0fb10bd484e062c22de3be6b67940b9cc6ec2f18fa61ba \ + --hash=sha256:1b15430b93d7eb3d56f6ff690d2ebecb79ed0e58248427717eba150a508d1cd7 \ + --hash=sha256:1fed2f4797ad68e9982038423018cf08bec5f4ce9fed63a94a790773ed6a795c \ + --hash=sha256:2074548a335fcf7d19ebb18d9208da9e33b06f745754466a7e001d2b1c58dd19 \ + --hash=sha256:2587cedbb36c7988e707d83f0f1175c1f882f362b5ebbee25d70218ea33d220d \ + --hash=sha256:25945f962c7bd808415cfde3da624d4399d4ea71ed8918538375f16bceb79e1c \ + --hash=sha256:27786c62a400e282756ae1b090bcd7cfa35f28270cff65a9e7b27a5327a32561 \ + --hash=sha256:2c1719ddc8218b01e82fb2e82e8451bd65076cb96d7bef4477194bbb4305a968 \ + --hash=sha256:2d5595903444c854b99c42122b87edfe8a37cd698a4eae32f4fd1d2a7b6c115d \ + --hash=sha256:30bdbe4662992348132d03996700e1cf910d141d629179b967b146a22942264e \ + --hash=sha256:31a27a2edf482df72b91fe6c6438314d2c65290aa7becc55589d156c9b91f0da \ + --hash=sha256:320fd55bafee3eb069cfb5d6491f811a912758387ef2193840e2663e80e16f48 \ + --hash=sha256:33145acbad8317584cd64588131c7e1e286beef6280c0009b4544c91fce171d2 \ + --hash=sha256:43a241655e83e4603a152192cf022d5ca348c2f4e56dfb02e5c9c4c1a32f9cdb \ + --hash=sha256:4d63b6596368dab9e0cc66bf047e7182a56f33b34db141816a4f21f5bf958228 \ + --hash=sha256:4fb04442b6d1e2b36c774919c6bcbe3339c61b337261d4bd57e27932589095af \ + --hash=sha256:4fb2e77ff04bc4beb71d63c8e064f052ce5a6ea1e001d528d4d7f4b37d736f2e \ + --hash=sha256:5460c5ee6ced6d61ec8cd2324ebbe793a4960c4ffa2131ffff480e3b61c99ec5 \ + --hash=sha256:59344c1d46b7dec97d3f22f1cc930fafe8980b3c5bc9c9765c56738a5f1559e4 \ + --hash=sha256:5dfb78dd600a12f23fc0c3ec58f81336229fdc74501ecf378d1ce5b3f2f313ea \ + --hash=sha256:643e1e65b4f5b284427e61a894d876d10459820e93aa1e724dfb415117be24e0 \ + --hash=sha256:644ec9215cf9c4df8028d8511379a15d9c1af3e16d80e47f1b6fdc6ba118356a \ + --hash=sha256:66f2662c640bb71a1016a031eea6eef9d25c2bcdf7ffd1d1ddc5a58f9a1ced04 \ + --hash=sha256:685e656048b59d8dfde8c601f188ad53a4d719eb97080cafc8696cda6d75865e \ + --hash=sha256:7269bb3fc15587b0c191eecd95831d771a7d80f0c48929e560806b038ff3066c \ + --hash=sha256:73426f5eb2ecc10626c67cf86bd0af9e00d53e80e5c67d5ce8e18376d6abfa09 \ + --hash=sha256:75c07726c8b1a52147fd7987d6baaa318c5dced1416c3f25593e40f56e10755b \ + --hash=sha256:790fc19bccbd39426060047e53629f171a44745613bf360a045e9f9c8c4a2cea \ + --hash=sha256:7a2ccaf88ac0db153e84305d1ef0aa138cea82c6a88309066f6eaa3bc98636cd \ + --hash=sha256:87f4e01b042c84e6090dbc4fbe3415ddd69f6bc0130382323f9d3f1b8dd71b46 \ + --hash=sha256:88f079335e9da631efa64486c8207564a7bcd0c00526bb9e842e9d5b7e50a6cc \ + --hash=sha256:8c1c91a68926421f5ccbc82c85f83bd3ba593b121a46a1b9a554b3f0dd67a4bf \ + --hash=sha256:9121a894d74e65557e47e777060a495ab85f4b903e80dd73a3c940ba042920d7 \ + --hash=sha256:94e348c72a1fd1f8191f25ea056448e4f5a87b8fbf005b39d290dcb0581a48cd \ + --hash=sha256:98195866d3a9949915935d40a88e4f1c166e82e378f622c88025f2938624a90a \ + --hash=sha256:99dd6652bd6f730beadf74ef769d38c6bbd8ee6d1c15c8d138ea680b0594387f \ + --hash=sha256:9af691a9861027181d4de07ed74f0aee12a9650ac60d0a07f4320bff84b5d95f \ + --hash=sha256:a3b8b5d2935403f1b4b25ae324560e94b59593a38c0d2e7b6c9872126a9622ed \ + --hash=sha256:a3dcc876050b8f5cbc0ee84ef1e7f0c1dfe7c148f10098828bc4403683c33f10 \ + --hash=sha256:a74f5a92fa6e51c4f3c69b29c4662088b97be12f40652a21109605a175c81824 \ + --hash=sha256:ab91b0c36e95d42e1041a4907e3eefd06c482d53af3c7a77be7e214cc7cd4a63 \ + --hash=sha256:ad1bc61c7f6b0e58106aaab034916b6cb041757f708b07fbcdd9d6e1ac629225 \ + --hash=sha256:adcb9e77aa132cc6c9de2ffe7cf880a20aa8cdba21d367d1da1a412f57bddd5d \ + --hash=sha256:b22ea9bf5f9fad2b0077e944a7813f91593a4f61adf8faf734a70aed3f2b3a40 \ + --hash=sha256:b2a1c354f13f22b737621d914f3b4a8434ae69d3027a775e94b3e671756112f9 \ + --hash=sha256:b32fdf874868326351a75b1e4c02f97e802147119ae44c52d3d9da193ec34f5b \ + --hash=sha256:b3853ed4ce522598dc886160a7bab432a093051af85891fa2f5577c1dcac8ed6 \ + --hash=sha256:b443e73a4dfc7b6e0800ea4c13567b9694358e86f53bb2612a51c9e727cac67b \ + --hash=sha256:b4c9083ea89ab236b06e9ef2263971db3b4b507195fc7d5eecab95828dcae325 \ + --hash=sha256:b8ca0fe21458457077e4cb2d81e1ebdb146a00b3e9e2db6180a773f7ea905032 \ + --hash=sha256:c393af77c659a38bffbca215c0bcc8629ba4299568308dd7e4ff65d62cabed39 \ + --hash=sha256:c6bffa978793bea5e1b00e677062e53a62255439339591b70e209fa1552d5ee0 \ + --hash=sha256:ccf39ad5702e33e4d335b48ef9d56e21619b529b7f7471b5211419f380329b62 \ + --hash=sha256:cf81e0278b645004388873e0a1f9e3bc4c9ab8c18e377b14ed1a544be4b18c9a \ + --hash=sha256:d34546ad2e4a480b94b6797bcc5a322b3c705c4c74c3e4e545c4a3841c1b2d59 \ + --hash=sha256:d47713ffe6d4a627fbf078be9836a95ac106b4a0543e3841572c91e292a5d885 \ + --hash=sha256:d918dfe473291e8bfd8e13223ea5cb9b317bd9f50c280923776c377f7c64b428 \ + --hash=sha256:dbdce852e6bb66e1b8c36679d482971d69d93acf1785657522e51b7de30c3356 \ + --hash=sha256:dcc1bf0ac8a194313cf6e645e300a8a379674ceed8e0b1e910a2de3e3c28989e \ + --hash=sha256:dd961a32a7182c3891cdebca417fda67496d5d5de6ae636962254d22723bdf52 \ + --hash=sha256:ddf5d247f686aec853ddcc9a1234bfcc6f57b0a0670d2ad82fc25d8ae7e6a15f \ + --hash=sha256:e27c3cd27fbd25e5223c5c992b300cd4ee8f0a75c6f222ce65838138d853712c \ + --hash=sha256:e380ec4e6d8b26e389713995a43cb7fe56baea2d25fe073d4998c4821a026211 \ + --hash=sha256:e4bbde174a0aff5f6eeba75cf8c4c5d2a316316bc21f03a0bddca0fc3659a6f3 \ + --hash=sha256:e8b49b5743ede51e0bcf6805741f39f5e0e0fd6a172ba460cb39e3097ba803bb \ + --hash=sha256:e9904b5b37c3e5bb4a245c56bc4b7e497da57ffb8528f4fc39af9dcb168ee2e1 \ + --hash=sha256:ea96503b918fceaf40443182742b8964d47b65c5ebdea532893cb9479620000c \ + --hash=sha256:eb31fe390f03f7ae886dcc374f1099ec88526631a4cb891d399b68181f154ff0 \ + --hash=sha256:ebb32d776b61acd49f859a1d16b9e3d84e7b46d0d92aebd58acd54dc38e96664 \ + --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ + --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e + # via + # -c python/requirements_compiled.txt + # cupy-cuda12x +filelock==3.17.0 \ + --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ + --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e + # via + # -c python/requirements_compiled.txt + # ray + # virtualenv +frozenlist==1.4.1 \ + --hash=sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7 \ + --hash=sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98 \ + --hash=sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad \ + --hash=sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5 \ + --hash=sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae \ + --hash=sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e \ + --hash=sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a \ + --hash=sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701 \ + --hash=sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d \ + --hash=sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6 \ + --hash=sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6 \ + --hash=sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106 \ + --hash=sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75 \ + --hash=sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868 \ + --hash=sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a \ + --hash=sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0 \ + --hash=sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1 \ + --hash=sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826 \ + --hash=sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec \ + --hash=sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6 \ + --hash=sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950 \ + --hash=sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19 \ + --hash=sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0 \ + --hash=sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8 \ + --hash=sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a \ + --hash=sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09 \ + --hash=sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86 \ + --hash=sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c \ + --hash=sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5 \ + --hash=sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b \ + --hash=sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b \ + --hash=sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d \ + --hash=sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0 \ + --hash=sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea \ + --hash=sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776 \ + --hash=sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a \ + --hash=sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897 \ + --hash=sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7 \ + --hash=sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09 \ + --hash=sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9 \ + --hash=sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe \ + --hash=sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd \ + --hash=sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742 \ + --hash=sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09 \ + --hash=sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0 \ + --hash=sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932 \ + --hash=sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1 \ + --hash=sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a \ + --hash=sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49 \ + --hash=sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d \ + --hash=sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7 \ + --hash=sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480 \ + --hash=sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89 \ + --hash=sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e \ + --hash=sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b \ + --hash=sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82 \ + --hash=sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb \ + --hash=sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068 \ + --hash=sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8 \ + --hash=sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b \ + --hash=sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb \ + --hash=sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2 \ + --hash=sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11 \ + --hash=sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b \ + --hash=sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc \ + --hash=sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0 \ + --hash=sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497 \ + --hash=sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17 \ + --hash=sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0 \ + --hash=sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2 \ + --hash=sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439 \ + --hash=sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5 \ + --hash=sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac \ + --hash=sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825 \ + --hash=sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887 \ + --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ + --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 + # via + # -c python/requirements_compiled.txt + # aiohttp + # aiosignal +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 + # via + # -c python/requirements_compiled.txt + # ray +google-api-core==2.24.2 \ + --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ + --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 + # via + # -c python/requirements_compiled.txt + # opencensus +google-auth==2.23.4 \ + --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ + --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 + # via + # -c python/requirements_compiled.txt + # google-api-core +googleapis-common-protos==1.61.0 \ + --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ + --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b + # via + # -c python/requirements_compiled.txt + # google-api-core +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e + # via + # -c python/requirements_compiled.txt + # ray +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a + # via + # -c python/requirements_compiled.txt + # ray +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # -c python/requirements_compiled.txt + # uvicorn +httptools==0.6.4 \ + --hash=sha256:0614154d5454c21b6410fdf5262b4a3ddb0f53f1e1721cfd59d55f32138c578a \ + --hash=sha256:0e563e54979e97b6d13f1bbc05a96109923e76b901f786a5eae36e99c01237bd \ + --hash=sha256:16e603a3bff50db08cd578d54f07032ca1631450ceb972c2f834c2b860c28ea2 \ + --hash=sha256:288cd628406cc53f9a541cfaf06041b4c71d751856bab45e3702191f931ccd17 \ + --hash=sha256:28908df1b9bb8187393d5b5db91435ccc9c8e891657f9cbb42a2541b44c82fc8 \ + --hash=sha256:322d20ea9cdd1fa98bd6a74b77e2ec5b818abdc3d36695ab402a0de8ef2865a3 \ + --hash=sha256:342dd6946aa6bda4b8f18c734576106b8a31f2fe31492881a9a160ec84ff4bd5 \ + --hash=sha256:345c288418f0944a6fe67be8e6afa9262b18c7626c3ef3c28adc5eabc06a68da \ + --hash=sha256:3c73ce323711a6ffb0d247dcd5a550b8babf0f757e86a52558fe5b86d6fefcc0 \ + --hash=sha256:40a5ec98d3f49904b9fe36827dcf1aadfef3b89e2bd05b0e35e94f97c2b14721 \ + --hash=sha256:40b0f7fe4fd38e6a507bdb751db0379df1e99120c65fbdc8ee6c1d044897a636 \ + --hash=sha256:40dc6a8e399e15ea525305a2ddba998b0af5caa2566bcd79dcbe8948181eeaff \ + --hash=sha256:4b36913ba52008249223042dca46e69967985fb4051951f94357ea681e1f5dc0 \ + --hash=sha256:4d87b29bd4486c0093fc64dea80231f7c7f7eb4dc70ae394d70a495ab8436071 \ + --hash=sha256:4e93eee4add6493b59a5c514da98c939b244fce4a0d8879cd3f466562f4b7d5c \ + --hash=sha256:59e724f8b332319e2875efd360e61ac07f33b492889284a3e05e6d13746876f4 \ + --hash=sha256:69422b7f458c5af875922cdb5bd586cc1f1033295aa9ff63ee196a87519ac8e1 \ + --hash=sha256:703c346571fa50d2e9856a37d7cd9435a25e7fd15e236c397bf224afaa355fe9 \ + --hash=sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44 \ + --hash=sha256:856f4bc0478ae143bad54a4242fccb1f3f86a6e1be5548fecfd4102061b3a083 \ + --hash=sha256:85797e37e8eeaa5439d33e556662cc370e474445d5fab24dcadc65a8ffb04003 \ + --hash=sha256:90d96a385fa941283ebd231464045187a31ad932ebfa541be8edf5b3c2328959 \ + --hash=sha256:94978a49b8f4569ad607cd4946b759d90b285e39c0d4640c6b36ca7a3ddf2efc \ + --hash=sha256:aafe0f1918ed07b67c1e838f950b1c1fabc683030477e60b335649b8020e1076 \ + --hash=sha256:ab9ba8dcf59de5181f6be44a77458e45a578fc99c31510b8c65b7d5acc3cf490 \ + --hash=sha256:ade273d7e767d5fae13fa637f4d53b6e961fb7fd93c7797562663f0171c26660 \ + --hash=sha256:b799de31416ecc589ad79dd85a0b2657a8fe39327944998dea368c1d4c9e55e6 \ + --hash=sha256:c26f313951f6e26147833fc923f78f95604bbec812a43e5ee37f26dc9e5a686c \ + --hash=sha256:ca80b7485c76f768a3bc83ea58373f8db7b015551117375e4918e2aa77ea9b50 \ + --hash=sha256:d1ffd262a73d7c28424252381a5b854c19d9de5f56f075445d33919a637e3547 \ + --hash=sha256:d3f0d369e7ffbe59c4b6116a44d6a8eb4783aae027f2c0b366cf0aa964185dba \ + --hash=sha256:d54efd20338ac52ba31e7da78e4a72570cf729fac82bc31ff9199bedf1dc7440 \ + --hash=sha256:dacdd3d10ea1b4ca9df97a0a303cbacafc04b5cd375fa98732678151643d4988 \ + --hash=sha256:db353d22843cf1028f43c3651581e4bb49374d85692a85f95f7b9a130e1b2cab \ + --hash=sha256:db78cb9ca56b59b016e64b6031eda5653be0589dba2b1b43453f6e8b405a0970 \ + --hash=sha256:deee0e3343f98ee8047e9f4c5bc7cedbf69f5734454a94c38ee829fb2d5fa3c1 \ + --hash=sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2 \ + --hash=sha256:df959752a0c2748a65ab5387d08287abf6779ae9165916fe053e68ae1fbdc47f \ + --hash=sha256:ec4f178901fa1834d4a060320d2f3abc5c9e39766953d038f1458cb885f47e81 \ + --hash=sha256:f47f8ed67cc0ff862b84a1189831d1d33c963fb3ce1ee0c65d3b0cbe7b711069 \ + --hash=sha256:f8787367fbdfccae38e35abf7641dafc5310310a5987b689f4c32cc8cc3ee975 \ + --hash=sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f \ + --hash=sha256:fc411e1c0a7dcd2f902c7c48cf079947a7e65b5485dea9decb82b9105ca71a43 + # via uvicorn +idna==3.7 \ + --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ + --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 + # via + # -c python/requirements_compiled.txt + # anyio + # requests + # yarl +importlib-metadata==6.11.0 \ + --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ + --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b + # via + # -c python/requirements_compiled.txt + # opentelemetry-api +jinja2==3.1.6 ; sys_platform != 'win32' \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via + # -c python/requirements_compiled.txt + # memray +jsonschema==4.23.0 \ + --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ + --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 + # via + # -c python/requirements_compiled.txt + # ray +jsonschema-specifications==2024.10.1 \ + --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ + --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf + # via + # -c python/requirements_compiled.txt + # jsonschema +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled.txt + # celery +lz4==4.3.3 \ + --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ + --hash=sha256:054b4631a355606e99a42396f5db4d22046a3397ffc3269a348ec41eaebd69d2 \ + --hash=sha256:0a136e44a16fc98b1abc404fbabf7f1fada2bdab6a7e970974fb81cf55b636d0 \ + --hash=sha256:0e9c410b11a31dbdc94c05ac3c480cb4b222460faf9231f12538d0074e56c563 \ + --hash=sha256:222a7e35137d7539c9c33bb53fcbb26510c5748779364014235afc62b0ec797f \ + --hash=sha256:24b3206de56b7a537eda3a8123c644a2b7bf111f0af53bc14bed90ce5562d1aa \ + --hash=sha256:2b901c7784caac9a1ded4555258207d9e9697e746cc8532129f150ffe1f6ba0d \ + --hash=sha256:2f7b1839f795315e480fb87d9bc60b186a98e3e5d17203c6e757611ef7dcef61 \ + --hash=sha256:30e8c20b8857adef7be045c65f47ab1e2c4fabba86a9fa9a997d7674a31ea6b6 \ + --hash=sha256:31ea4be9d0059c00b2572d700bf2c1bc82f241f2c3282034a759c9a4d6ca4dc2 \ + --hash=sha256:337cb94488a1b060ef1685187d6ad4ba8bc61d26d631d7ba909ee984ea736be1 \ + --hash=sha256:33c9a6fd20767ccaf70649982f8f3eeb0884035c150c0b818ea660152cf3c809 \ + --hash=sha256:363ab65bf31338eb364062a15f302fc0fab0a49426051429866d71c793c23394 \ + --hash=sha256:43cf03059c0f941b772c8aeb42a0813d68d7081c009542301637e5782f8a33e2 \ + --hash=sha256:56f4fe9c6327adb97406f27a66420b22ce02d71a5c365c48d6b656b4aaeb7775 \ + --hash=sha256:5d35533bf2cee56f38ced91f766cd0038b6abf46f438a80d50c52750088be93f \ + --hash=sha256:6756212507405f270b66b3ff7f564618de0606395c0fe10a7ae2ffcbbe0b1fba \ + --hash=sha256:6cdc60e21ec70266947a48839b437d46025076eb4b12c76bd47f8e5eb8a75dcc \ + --hash=sha256:abc197e4aca8b63f5ae200af03eb95fb4b5055a8f990079b5bdf042f568469dd \ + --hash=sha256:b14d948e6dce389f9a7afc666d60dd1e35fa2138a8ec5306d30cd2e30d36b40c \ + --hash=sha256:b47839b53956e2737229d70714f1d75f33e8ac26e52c267f0197b3189ca6de24 \ + --hash=sha256:b6d9ec061b9eca86e4dcc003d93334b95d53909afd5a32c6e4f222157b50c071 \ + --hash=sha256:b891880c187e96339474af2a3b2bfb11a8e4732ff5034be919aa9029484cd201 \ + --hash=sha256:bca8fccc15e3add173da91be8f34121578dc777711ffd98d399be35487c934bf \ + --hash=sha256:c81703b12475da73a5d66618856d04b1307e43428a7e59d98cfe5a5d608a74c6 \ + --hash=sha256:d2507ee9c99dbddd191c86f0e0c8b724c76d26b0602db9ea23232304382e1f21 \ + --hash=sha256:e36cd7b9d4d920d3bfc2369840da506fa68258f7bb176b8743189793c055e43d \ + --hash=sha256:e7d84b479ddf39fe3ea05387f10b779155fc0990125f4fb35d636114e1c63a2e \ + --hash=sha256:eac9af361e0d98335a02ff12fb56caeb7ea1196cf1a49dbf6f17828a131da807 \ + --hash=sha256:edfd858985c23523f4e5a7526ca6ee65ff930207a7ec8a8f57a01eae506aaee7 \ + --hash=sha256:ee9ff50557a942d187ec85462bb0960207e7ec5b19b3b48949263993771c6205 \ + --hash=sha256:f0e822cd7644995d9ba248cb4b67859701748a93e2ab7fc9bc18c599a52e4604 \ + --hash=sha256:f180904f33bdd1e92967923a43c22899e303906d19b2cf8bb547db6653ea6e7d \ + --hash=sha256:f1d18718f9d78182c6b60f568c9a9cec8a7204d7cb6fad4e511a2ef279e4cb05 \ + --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ + --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 + # via + # -c python/requirements_compiled.txt + # ray +markdown-it-py==2.2.0 ; sys_platform != 'win32' \ + --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ + --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 + # via + # -c python/requirements_compiled.txt + # rich +markupsafe==2.1.3 ; sys_platform != 'win32' \ + --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ + --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ + --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ + --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ + --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ + --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ + --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ + --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ + --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ + --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ + --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ + --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ + --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ + --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ + --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ + --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ + --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ + --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ + --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ + --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ + --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ + --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ + --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ + --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ + --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 + # via + # -c python/requirements_compiled.txt + # jinja2 +mdurl==0.1.2 ; sys_platform != 'win32' \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via + # -c python/requirements_compiled.txt + # markdown-it-py +memray==1.10.0 ; sys_platform != 'win32' \ + --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ + --hash=sha256:22f2a47871c172a0539bd72737bb6b294fc10c510464066b825d90fcd3bb4916 \ + --hash=sha256:23e8c402625cfb32d0e9edb5ec0945f3e5e54bc6b0c5699f6284302082b80bd4 \ + --hash=sha256:2ce59ef485db3634de98b3a026d2450fc0a875e3a58a9ea85f7a89098841defe \ + --hash=sha256:322ed0b69014a0969b777768d461a785203f81f9864386b666b5b26645d9c294 \ + --hash=sha256:38322e052b882790993412f1840517a51818aa55c47037f69915b2007f2c4cee \ + --hash=sha256:38393c86ce6d0a08e6ec0eb1401d49803b7c0c950c2565386751cdc81568cba8 \ + --hash=sha256:391aac6c9f744528d3186bc82d708a1acc83525778f804045d7c96f860f8ec98 \ + --hash=sha256:3a8bb7fbd8303c4f0017ba7faef6b88f904cda2931ed667cbf3b98f024b3bc44 \ + --hash=sha256:3c401c57f49c4c5f1fecaee1e746f537cdc6680da05fb963dc143bd08ee109bf \ + --hash=sha256:4eba29179772b4a2e440a065b320b03bc2e73fe2648bdf7936aa3b9a086fab4a \ + --hash=sha256:53a8f66af18b1f3bcf5c9f3c95ae4134dd675903a38f9d0e6341b7bca01b63d0 \ + --hash=sha256:566602b2143e06b3d592901d98c52ce4599e71aa2555146eeb5cec03506f9498 \ + --hash=sha256:663d463e89a64bae4a6b2f8c837d11a3d094834442d536a4165e1d31899a3500 \ + --hash=sha256:68bd8df023c8a32f44c11d997e5c536837e27c0955daf557d3a377edd55a1dd3 \ + --hash=sha256:6937d7ef67d18ccc01c3250cdf3b4ef1445b859ee8756f09e3d11bd3ff0c7d67 \ + --hash=sha256:6b311e91203be71e1a0ce5e4f978137765bcb1045f3bf5646129c83c5b96ab3c \ + --hash=sha256:6fd13ef666c7fced9768d1cfabf71dc6dfa6724935a8dff463495ac2dc5e13a4 \ + --hash=sha256:8196c684f1be8fe423e5cdd2356d4255a2cb482a1f3e89612b70d2a2862cf5bb \ + --hash=sha256:843a688877691746f9d1835cfa8a65139948471bdd78720435808d20bc30a1cc \ + --hash=sha256:85c32d6613d81b075f740e398c4d653e0803cd48e82c33dcd584c109d6782666 \ + --hash=sha256:898acd60f57a10dc5aaf1fd64aa2f821f0420114f3f60c3058083788603f173a \ + --hash=sha256:8d56f37a34125684746c13d24bd7a3fb17549b0bb355eb50969eb11e05e3ba62 \ + --hash=sha256:92c372cb262eddd23049f945ca9527f0e4cc7c40a070aade1802d066f680885b \ + --hash=sha256:95e563d9c976e429ad597ad2720d95cebbe8bac891a3082465439143e2740772 \ + --hash=sha256:9627184c926252c8f719c301f1fefe970f0d033c643a6448b93fed2889d1ea94 \ + --hash=sha256:a9e985fb7646b0475c303919d19211d2aa54e5a9e2cd2a102472299be5dbebd3 \ + --hash=sha256:b681519357d94f5f0857fbc6029e7c44d3f41436109e955a14fd312d8317bc35 \ + --hash=sha256:b75040f28e8678d0e9c4907d55c95cf26db8ef5adc9941a228f1b280a9efd9c0 \ + --hash=sha256:c3a14960838d89a91747885897d34134afb65883cc3b0ed7ff30fe1af00f9fe6 \ + --hash=sha256:c7aeb47174c42e99740a8e2b3b6fe0932c95d987258d48a746974ead19176c26 \ + --hash=sha256:ce22a887a585ef5020896de89ffc793e531b65ccc81fbafcc7886010c2c562b3 \ + --hash=sha256:cf6d683c4f8d25c6ad06ae18715f218983c5eb86803953615e902d632fdf6ec1 \ + --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ + --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 + # via + # -c python/requirements_compiled.txt + # ray +msgpack==1.0.7 \ + --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ + --hash=sha256:0bfdd914e55e0d2c9e1526de210f6fe8ffe9705f2b1dfcc4aecc92a4cb4b533d \ + --hash=sha256:1dc93e8e4653bdb5910aed79f11e165c85732067614f180f70534f056da97db3 \ + --hash=sha256:1e2d69948e4132813b8d1131f29f9101bc2c915f26089a6d632001a5c1349672 \ + --hash=sha256:235a31ec7db685f5c82233bddf9858748b89b8119bf4538d514536c485c15fe0 \ + --hash=sha256:27dcd6f46a21c18fa5e5deed92a43d4554e3df8d8ca5a47bf0615d6a5f39dbc9 \ + --hash=sha256:28efb066cde83c479dfe5a48141a53bc7e5f13f785b92ddde336c716663039ee \ + --hash=sha256:3476fae43db72bd11f29a5147ae2f3cb22e2f1a91d575ef130d2bf49afd21c46 \ + --hash=sha256:36e17c4592231a7dbd2ed09027823ab295d2791b3b1efb2aee874b10548b7524 \ + --hash=sha256:384d779f0d6f1b110eae74cb0659d9aa6ff35aaf547b3955abf2ab4c901c4819 \ + --hash=sha256:38949d30b11ae5f95c3c91917ee7a6b239f5ec276f271f28638dec9156f82cfc \ + --hash=sha256:3967e4ad1aa9da62fd53e346ed17d7b2e922cba5ab93bdd46febcac39be636fc \ + --hash=sha256:3e7bf4442b310ff154b7bb9d81eb2c016b7d597e364f97d72b1acc3817a0fdc1 \ + --hash=sha256:3f0c8c6dfa6605ab8ff0611995ee30d4f9fcff89966cf562733b4008a3d60d82 \ + --hash=sha256:484ae3240666ad34cfa31eea7b8c6cd2f1fdaae21d73ce2974211df099a95d81 \ + --hash=sha256:4a7b4f35de6a304b5533c238bee86b670b75b03d31b7797929caa7a624b5dda6 \ + --hash=sha256:4cb14ce54d9b857be9591ac364cb08dc2d6a5c4318c1182cb1d02274029d590d \ + --hash=sha256:4e71bc4416de195d6e9b4ee93ad3f2f6b2ce11d042b4d7a7ee00bbe0358bd0c2 \ + --hash=sha256:52700dc63a4676669b341ba33520f4d6e43d3ca58d422e22ba66d1736b0a6e4c \ + --hash=sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87 \ + --hash=sha256:576eb384292b139821c41995523654ad82d1916da6a60cff129c715a6223ea84 \ + --hash=sha256:5b0bf0effb196ed76b7ad883848143427a73c355ae8e569fa538365064188b8e \ + --hash=sha256:5b6ccc0c85916998d788b295765ea0e9cb9aac7e4a8ed71d12e7d8ac31c23c95 \ + --hash=sha256:5ed82f5a7af3697b1c4786053736f24a0efd0a1b8a130d4c7bfee4b9ded0f08f \ + --hash=sha256:6d4c80667de2e36970ebf74f42d1088cc9ee7ef5f4e8c35eee1b40eafd33ca5b \ + --hash=sha256:730076207cb816138cf1af7f7237b208340a2c5e749707457d70705715c93b93 \ + --hash=sha256:7687e22a31e976a0e7fc99c2f4d11ca45eff652a81eb8c8085e9609298916dcf \ + --hash=sha256:822ea70dc4018c7e6223f13affd1c5c30c0f5c12ac1f96cd8e9949acddb48a61 \ + --hash=sha256:84b0daf226913133f899ea9b30618722d45feffa67e4fe867b0b5ae83a34060c \ + --hash=sha256:85765fdf4b27eb5086f05ac0491090fc76f4f2b28e09d9350c31aac25a5aaff8 \ + --hash=sha256:8dd178c4c80706546702c59529ffc005681bd6dc2ea234c450661b205445a34d \ + --hash=sha256:8f5b234f567cf76ee489502ceb7165c2a5cecec081db2b37e35332b537f8157c \ + --hash=sha256:98bbd754a422a0b123c66a4c341de0474cad4a5c10c164ceed6ea090f3563db4 \ + --hash=sha256:993584fc821c58d5993521bfdcd31a4adf025c7d745bbd4d12ccfecf695af5ba \ + --hash=sha256:a40821a89dc373d6427e2b44b572efc36a2778d3f543299e2f24eb1a5de65415 \ + --hash=sha256:b291f0ee7961a597cbbcc77709374087fa2a9afe7bdb6a40dbbd9b127e79afee \ + --hash=sha256:b573a43ef7c368ba4ea06050a957c2a7550f729c31f11dd616d2ac4aba99888d \ + --hash=sha256:b610ff0f24e9f11c9ae653c67ff8cc03c075131401b3e5ef4b82570d1728f8a9 \ + --hash=sha256:bdf38ba2d393c7911ae989c3bbba510ebbcdf4ecbdbfec36272abe350c454075 \ + --hash=sha256:bfef2bb6ef068827bbd021017a107194956918ab43ce4d6dc945ffa13efbc25f \ + --hash=sha256:cab3db8bab4b7e635c1c97270d7a4b2a90c070b33cbc00c99ef3f9be03d3e1f7 \ + --hash=sha256:cb70766519500281815dfd7a87d3a178acf7ce95390544b8c90587d76b227681 \ + --hash=sha256:cca1b62fe70d761a282496b96a5e51c44c213e410a964bdffe0928e611368329 \ + --hash=sha256:ccf9a39706b604d884d2cb1e27fe973bc55f2890c52f38df742bc1d79ab9f5e1 \ + --hash=sha256:dc43f1ec66eb8440567186ae2f8c447d91e0372d793dfe8c222aec857b81a8cf \ + --hash=sha256:dd632777ff3beaaf629f1ab4396caf7ba0bdd075d948a69460d13d44357aca4c \ + --hash=sha256:e45ae4927759289c30ccba8d9fdce62bb414977ba158286b5ddaf8df2cddb5c5 \ + --hash=sha256:e50ebce52f41370707f1e21a59514e3375e3edd6e1832f5e5235237db933c98b \ + --hash=sha256:ebbbba226f0a108a7366bf4b59bf0f30a12fd5e75100c630267d94d7f0ad20e5 \ + --hash=sha256:ec79ff6159dffcc30853b2ad612ed572af86c92b5168aa3fc01a67b0fa40665e \ + --hash=sha256:f0936e08e0003f66bfd97e74ee530427707297b0d0361247e9b4f59ab78ddc8b \ + --hash=sha256:f26a07a6e877c76a88e3cecac8531908d980d3d5067ff69213653649ec0f60ad \ + --hash=sha256:f64e376cd20d3f030190e8c32e1c64582eba56ac6dc7d5b0b49a9d44021b52fd \ + --hash=sha256:f6ffbc252eb0d229aeb2f9ad051200668fc3a9aaa8994e49f0cb2ffe2b7867e7 \ + --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ + --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc + # via + # -c python/requirements_compiled.txt + # ray +multidict==6.0.5 \ + --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ + --hash=sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c \ + --hash=sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29 \ + --hash=sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b \ + --hash=sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8 \ + --hash=sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7 \ + --hash=sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd \ + --hash=sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40 \ + --hash=sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6 \ + --hash=sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3 \ + --hash=sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c \ + --hash=sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9 \ + --hash=sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5 \ + --hash=sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae \ + --hash=sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442 \ + --hash=sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9 \ + --hash=sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc \ + --hash=sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c \ + --hash=sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea \ + --hash=sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5 \ + --hash=sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50 \ + --hash=sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182 \ + --hash=sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453 \ + --hash=sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e \ + --hash=sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600 \ + --hash=sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733 \ + --hash=sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda \ + --hash=sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241 \ + --hash=sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461 \ + --hash=sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e \ + --hash=sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e \ + --hash=sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b \ + --hash=sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e \ + --hash=sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7 \ + --hash=sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386 \ + --hash=sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd \ + --hash=sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9 \ + --hash=sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf \ + --hash=sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee \ + --hash=sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5 \ + --hash=sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a \ + --hash=sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271 \ + --hash=sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54 \ + --hash=sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4 \ + --hash=sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496 \ + --hash=sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb \ + --hash=sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319 \ + --hash=sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3 \ + --hash=sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f \ + --hash=sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527 \ + --hash=sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed \ + --hash=sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604 \ + --hash=sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef \ + --hash=sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8 \ + --hash=sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5 \ + --hash=sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5 \ + --hash=sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626 \ + --hash=sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c \ + --hash=sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d \ + --hash=sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c \ + --hash=sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc \ + --hash=sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc \ + --hash=sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b \ + --hash=sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38 \ + --hash=sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450 \ + --hash=sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1 \ + --hash=sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f \ + --hash=sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3 \ + --hash=sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755 \ + --hash=sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226 \ + --hash=sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a \ + --hash=sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046 \ + --hash=sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf \ + --hash=sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479 \ + --hash=sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e \ + --hash=sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1 \ + --hash=sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a \ + --hash=sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83 \ + --hash=sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929 \ + --hash=sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93 \ + --hash=sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a \ + --hash=sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c \ + --hash=sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44 \ + --hash=sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89 \ + --hash=sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba \ + --hash=sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e \ + --hash=sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da \ + --hash=sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24 \ + --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ + --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef + # via + # -c python/requirements_compiled.txt + # aiohttp + # yarl +numpy==1.26.4 \ + --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ + --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ + --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \ + --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \ + --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \ + --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \ + --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \ + --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \ + --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \ + --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \ + --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \ + --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \ + --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \ + --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \ + --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \ + --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \ + --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \ + --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \ + --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \ + --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \ + --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \ + --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \ + --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \ + --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \ + --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \ + --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \ + --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \ + --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \ + --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \ + --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \ + --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \ + --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \ + --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \ + --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \ + --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ + --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f + # via + # -c python/requirements_compiled.txt + # cupy-cuda12x + # gymnasium + # pandas + # ray + # scipy + # tensorboardx +opencensus==0.11.4 \ + --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ + --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 + # via + # -c python/requirements_compiled.txt + # ray +opencensus-context==0.1.3 \ + --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ + --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c + # via + # -c python/requirements_compiled.txt + # opencensus +opentelemetry-api==1.34.1 \ + --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ + --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-exporter-prometheus==0.55b1 \ + --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ + --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e + # via + # -c python/requirements_compiled.txt + # ray +opentelemetry-proto==1.27.0 \ + --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ + --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace + # via + # -c python/requirements_compiled.txt + # ray +opentelemetry-sdk==1.34.1 \ + --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ + --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # ray +opentelemetry-semantic-conventions==0.55b1 \ + --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ + --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 + # via + # -c python/requirements_compiled.txt + # opentelemetry-sdk +ormsgpack==1.7.0 \ + --hash=sha256:0d88307ab45d95416ce4071b1b99326ca31362af01c3d206f15a0551a7a874bd \ + --hash=sha256:22418a4d399027a72fb2e6b873559b1886cf2e63323ca7afc17b222c454413b7 \ + --hash=sha256:2c22c62a6bc93bcb194b7f91864ca0b39455b2cbbfc1538a3da0f9ec3c11d184 \ + --hash=sha256:3a6a97937d2cf21496d7689b90a43df83c5062bbe846aaa39197cc9ad73eaa7b \ + --hash=sha256:462089a419dbde654915ccb0b859c0dbe3c178b0ac580018e82befea6ccd73f4 \ + --hash=sha256:4b353204e99b56c1d33f1cf4767bd1fe1195596181a1cc789f25aa26c0b50f3d \ + --hash=sha256:5ec763096d978d35eedcef0af13991a10741717c2e236b26f4c2047b0740ea7b \ + --hash=sha256:5fefa1ca842dbba258401ea958113fe62c6b70a7a4d46edac440113f68dc431e \ + --hash=sha256:65525438b4a8b3b64ccfcda25e758ea3db392d1c206b5e09ef70efbbafa6dbf9 \ + --hash=sha256:6b4c98839cb7fc2a212037d2258f3a22857155249eb293d45c45cb974cfba834 \ + --hash=sha256:6d114652dadd81802b8a35a49e07a3e9ef2a47aed6123fb5031f2220d1c8e434 \ + --hash=sha256:77bc2ea387d85cfad045b9bcb8040bae43ad32dafe9363360f732cc19d489bbe \ + --hash=sha256:7e6ada21f5c7a20ff7cf9b061c44e3814352f819947a12022ad8cb52a9f2a809 \ + --hash=sha256:8d301e47565fe0e52a60052e730a9bb7669dfbd2a94643b8be925e3928c64c15 \ + --hash=sha256:90aabfd816db60dadab1100d583d061e0238209015bf684f8170c0fca4eb445a \ + --hash=sha256:91ebb7d3609db249cdff629ffef83ec3d025b1384749a297cf3b6a8240cf22ac \ + --hash=sha256:97723786755a7df85fcf6e68d7b5359dacea98d5c26b1d9af219a3cc05df4734 \ + --hash=sha256:9b0945523ccc75aa6907f38f2240d36818618baccb8633923bd7740a5a929e67 \ + --hash=sha256:a0ca6a64d47073f22ecc1dd96b384e44f98796d3f88ee383e92dfbcdf18c2efd \ + --hash=sha256:a5e12b51a590be47ccef67907905653e679fc2f920854b456edc216690ecc09c \ + --hash=sha256:a8fbe7bb50ee8381df030823d9366984fac718447947c2327969405d1d799b95 \ + --hash=sha256:c683071bf4527ffa7b6cfcf28f750d1a82eb77846d106743c09261ab1b79b193 \ + --hash=sha256:ca4d35b694f32112eb33ac0b733cb903dbbc59f019d05ca3d74f6ad2f587b0bf \ + --hash=sha256:e8385181bf195af80fc270e64fd477f1c414ffb05837320382e2ec9ca34be0ec \ + --hash=sha256:e86124cdbc8ed249806347c2fba96843e8941122b161b429139a0c973d270de4 \ + --hash=sha256:f9967a7f3647ad118751abf090f8397fda3e4bca6833340cab95a3f2bec598cd + # via + # -c python/requirements_compiled.txt + # ray +packaging==23.0 \ + --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ + --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 + # via + # -c python/requirements_compiled.txt + # kombu + # ray + # tensorboardx +pandas==1.5.3 \ + --hash=sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813 \ + --hash=sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792 \ + --hash=sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406 \ + --hash=sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373 \ + --hash=sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328 \ + --hash=sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996 \ + --hash=sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf \ + --hash=sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6 \ + --hash=sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7 \ + --hash=sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc \ + --hash=sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1 \ + --hash=sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23 \ + --hash=sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a \ + --hash=sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51 \ + --hash=sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572 \ + --hash=sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31 \ + --hash=sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5 \ + --hash=sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a \ + --hash=sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003 \ + --hash=sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d \ + --hash=sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354 \ + --hash=sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee \ + --hash=sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa \ + --hash=sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0 \ + --hash=sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9 \ + --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ + --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc + # via + # -c python/requirements_compiled.txt + # ray +platformdirs==3.11.0 \ + --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ + --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e + # via + # -c python/requirements_compiled.txt + # virtualenv +prometheus-client==0.19.0 \ + --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ + --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # ray +prompt-toolkit==3.0.41 \ + --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ + --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 + # via + # -c python/requirements_compiled.txt + # click-repl +propcache==0.3.0 \ + --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ + --hash=sha256:03acd9ff19021bd0567582ac88f821b66883e158274183b9e5586f678984f8fe \ + --hash=sha256:03c091bb752349402f23ee43bb2bff6bd80ccab7c9df6b88ad4322258d6960fc \ + --hash=sha256:07700939b2cbd67bfb3b76a12e1412405d71019df00ca5697ce75e5ef789d829 \ + --hash=sha256:0c3e893c4464ebd751b44ae76c12c5f5c1e4f6cbd6fbf67e3783cd93ad221863 \ + --hash=sha256:119e244ab40f70a98c91906d4c1f4c5f2e68bd0b14e7ab0a06922038fae8a20f \ + --hash=sha256:11ae6a8a01b8a4dc79093b5d3ca2c8a4436f5ee251a9840d7790dccbd96cb649 \ + --hash=sha256:15010f29fbed80e711db272909a074dc79858c6d28e2915704cfc487a8ac89c6 \ + --hash=sha256:19d36bb351ad5554ff20f2ae75f88ce205b0748c38b146c75628577020351e3c \ + --hash=sha256:1c8f7d896a16da9455f882870a507567d4f58c53504dc2d4b1e1d386dfe4588a \ + --hash=sha256:2383a17385d9800b6eb5855c2f05ee550f803878f344f58b6e194de08b96352c \ + --hash=sha256:24c04f8fbf60094c531667b8207acbae54146661657a1b1be6d3ca7773b7a545 \ + --hash=sha256:2578541776769b500bada3f8a4eeaf944530516b6e90c089aa368266ed70c49e \ + --hash=sha256:26a67e5c04e3119594d8cfae517f4b9330c395df07ea65eab16f3d559b7068fe \ + --hash=sha256:2b975528998de037dfbc10144b8aed9b8dd5a99ec547f14d1cb7c5665a43f075 \ + --hash=sha256:2d15bc27163cd4df433e75f546b9ac31c1ba7b0b128bfb1b90df19082466ff57 \ + --hash=sha256:2d913d36bdaf368637b4f88d554fb9cb9d53d6920b9c5563846555938d5450bf \ + --hash=sha256:3302c5287e504d23bb0e64d2a921d1eb4a03fb93a0a0aa3b53de059f5a5d737d \ + --hash=sha256:36ca5e9a21822cc1746023e88f5c0af6fce3af3b85d4520efb1ce4221bed75cc \ + --hash=sha256:3b812b3cb6caacd072276ac0492d249f210006c57726b6484a1e1805b3cfeea0 \ + --hash=sha256:3c6ec957025bf32b15cbc6b67afe233c65b30005e4c55fe5768e4bb518d712f1 \ + --hash=sha256:41de3da5458edd5678b0f6ff66691507f9885f5fe6a0fb99a5d10d10c0fd2d64 \ + --hash=sha256:42924dc0c9d73e49908e35bbdec87adedd651ea24c53c29cac103ede0ea1d340 \ + --hash=sha256:4544699674faf66fb6b4473a1518ae4999c1b614f0b8297b1cef96bac25381db \ + --hash=sha256:46ed02532cb66612d42ae5c3929b5e98ae330ea0f3900bc66ec5f4862069519b \ + --hash=sha256:49ea05212a529c2caffe411e25a59308b07d6e10bf2505d77da72891f9a05641 \ + --hash=sha256:4fa0e7c9c3cf7c276d4f6ab9af8adddc127d04e0fcabede315904d2ff76db626 \ + --hash=sha256:507c5357a8d8b4593b97fb669c50598f4e6cccbbf77e22fa9598aba78292b4d7 \ + --hash=sha256:549722908de62aa0b47a78b90531c022fa6e139f9166be634f667ff45632cc92 \ + --hash=sha256:58e6d2a5a7cb3e5f166fd58e71e9a4ff504be9dc61b88167e75f835da5764d07 \ + --hash=sha256:5a16167118677d94bb48bfcd91e420088854eb0737b76ec374b91498fb77a70e \ + --hash=sha256:5d62c4f6706bff5d8a52fd51fec6069bef69e7202ed481486c0bc3874912c787 \ + --hash=sha256:5fa159dcee5dba00c1def3231c249cf261185189205073bde13797e57dd7540a \ + --hash=sha256:6032231d4a5abd67c7f71168fd64a47b6b451fbcb91c8397c2f7610e67683810 \ + --hash=sha256:63f26258a163c34542c24808f03d734b338da66ba91f410a703e505c8485791d \ + --hash=sha256:65a37714b8ad9aba5780325228598a5b16c47ba0f8aeb3dc0514701e4413d7c0 \ + --hash=sha256:67054e47c01b7b349b94ed0840ccae075449503cf1fdd0a1fdd98ab5ddc2667b \ + --hash=sha256:67dda3c7325691c2081510e92c561f465ba61b975f481735aefdfc845d2cd043 \ + --hash=sha256:6985a593417cdbc94c7f9c3403747335e450c1599da1647a5af76539672464d3 \ + --hash=sha256:6a1948df1bb1d56b5e7b0553c0fa04fd0e320997ae99689488201f19fa90d2e7 \ + --hash=sha256:6b5b7fd6ee7b54e01759f2044f936dcf7dea6e7585f35490f7ca0420fe723c0d \ + --hash=sha256:6c929916cbdb540d3407c66f19f73387f43e7c12fa318a66f64ac99da601bcdf \ + --hash=sha256:6f4d7a7c0aff92e8354cceca6fe223973ddf08401047920df0fcb24be2bd5138 \ + --hash=sha256:728af36011bb5d344c4fe4af79cfe186729efb649d2f8b395d1572fb088a996c \ + --hash=sha256:742840d1d0438eb7ea4280f3347598f507a199a35a08294afdcc560c3739989d \ + --hash=sha256:75e872573220d1ee2305b35c9813626e620768248425f58798413e9c39741f46 \ + --hash=sha256:794c3dd744fad478b6232289c866c25406ecdfc47e294618bdf1697e69bd64a6 \ + --hash=sha256:7c0fdbdf6983526e269e5a8d53b7ae3622dd6998468821d660d0daf72779aefa \ + --hash=sha256:7c5f5290799a3f6539cc5e6f474c3e5c5fbeba74a5e1e5be75587746a940d51e \ + --hash=sha256:7c6e7e4f9167fddc438cd653d826f2222222564daed4116a02a184b464d3ef05 \ + --hash=sha256:7cedd25e5f678f7738da38037435b340694ab34d424938041aa630d8bac42663 \ + --hash=sha256:7e2e068a83552ddf7a39a99488bcba05ac13454fb205c847674da0352602082f \ + --hash=sha256:8319293e85feadbbfe2150a5659dbc2ebc4afdeaf7d98936fb9a2f2ba0d4c35c \ + --hash=sha256:8526b0941ec5a40220fc4dfde76aed58808e2b309c03e9fa8e2260083ef7157f \ + --hash=sha256:8884ba1a0fe7210b775106b25850f5e5a9dc3c840d1ae9924ee6ea2eb3acbfe7 \ + --hash=sha256:8cb625bcb5add899cb8ba7bf716ec1d3e8f7cdea9b0713fa99eadf73b6d4986f \ + --hash=sha256:8d663fd71491dde7dfdfc899d13a067a94198e90695b4321084c6e450743b8c7 \ + --hash=sha256:8ee1983728964d6070ab443399c476de93d5d741f71e8f6e7880a065f878e0b9 \ + --hash=sha256:997e7b8f173a391987df40f3b52c423e5850be6f6df0dcfb5376365440b56667 \ + --hash=sha256:9be90eebc9842a93ef8335291f57b3b7488ac24f70df96a6034a13cb58e6ff86 \ + --hash=sha256:9ddd49258610499aab83b4f5b61b32e11fce873586282a0e972e5ab3bcadee51 \ + --hash=sha256:9ecde3671e62eeb99e977f5221abcf40c208f69b5eb986b061ccec317c82ebd0 \ + --hash=sha256:9ff4e9ecb6e4b363430edf2c6e50173a63e0820e549918adef70515f87ced19a \ + --hash=sha256:a254537b9b696ede293bfdbc0a65200e8e4507bc9f37831e2a0318a9b333c85c \ + --hash=sha256:a2b9bf8c79b660d0ca1ad95e587818c30ccdb11f787657458d6f26a1ea18c568 \ + --hash=sha256:a61a68d630e812b67b5bf097ab84e2cd79b48c792857dc10ba8a223f5b06a2af \ + --hash=sha256:a7080b0159ce05f179cfac592cda1a82898ca9cd097dacf8ea20ae33474fbb25 \ + --hash=sha256:a8fd93de4e1d278046345f49e2238cdb298589325849b2645d4a94c53faeffc5 \ + --hash=sha256:a94ffc66738da99232ddffcf7910e0f69e2bbe3a0802e54426dbf0714e1c2ffe \ + --hash=sha256:aa806bbc13eac1ab6291ed21ecd2dd426063ca5417dd507e6be58de20e58dfcf \ + --hash=sha256:b0c1a133d42c6fc1f5fbcf5c91331657a1ff822e87989bf4a6e2e39b818d0ee9 \ + --hash=sha256:b58229a844931bca61b3a20efd2be2a2acb4ad1622fc026504309a6883686fbf \ + --hash=sha256:bb2f144c6d98bb5cbc94adeb0447cfd4c0f991341baa68eee3f3b0c9c0e83767 \ + --hash=sha256:be90c94570840939fecedf99fa72839aed70b0ced449b415c85e01ae67422c90 \ + --hash=sha256:bf0d9a171908f32d54f651648c7290397b8792f4303821c42a74e7805bfb813c \ + --hash=sha256:bf15fc0b45914d9d1b706f7c9c4f66f2b7b053e9517e40123e137e8ca8958b3d \ + --hash=sha256:bf4298f366ca7e1ad1d21bbb58300a6985015909964077afd37559084590c929 \ + --hash=sha256:c441c841e82c5ba7a85ad25986014be8d7849c3cfbdb6004541873505929a74e \ + --hash=sha256:cacea77ef7a2195f04f9279297684955e3d1ae4241092ff0cfcef532bb7a1c32 \ + --hash=sha256:cd54895e4ae7d32f1e3dd91261df46ee7483a735017dc6f987904f194aa5fd14 \ + --hash=sha256:d1323cd04d6e92150bcc79d0174ce347ed4b349d748b9358fd2e497b121e03c8 \ + --hash=sha256:d383bf5e045d7f9d239b38e6acadd7b7fdf6c0087259a84ae3475d18e9a2ae8b \ + --hash=sha256:d3e7420211f5a65a54675fd860ea04173cde60a7cc20ccfbafcccd155225f8bc \ + --hash=sha256:d8074c5dd61c8a3e915fa8fc04754fa55cfa5978200d2daa1e2d4294c1f136aa \ + --hash=sha256:df03cd88f95b1b99052b52b1bb92173229d7a674df0ab06d2b25765ee8404bce \ + --hash=sha256:e45377d5d6fefe1677da2a2c07b024a6dac782088e37c0b1efea4cfe2b1be19b \ + --hash=sha256:e53d19c2bf7d0d1e6998a7e693c7e87300dd971808e6618964621ccd0e01fe4e \ + --hash=sha256:e560fd75aaf3e5693b91bcaddd8b314f4d57e99aef8a6c6dc692f935cc1e6bbf \ + --hash=sha256:ec5060592d83454e8063e487696ac3783cc48c9a329498bafae0d972bc7816c9 \ + --hash=sha256:ecc2920630283e0783c22e2ac94427f8cca29a04cfdf331467d4f661f4072dac \ + --hash=sha256:ed7161bccab7696a473fe7ddb619c1d75963732b37da4618ba12e60899fefe4f \ + --hash=sha256:ee0bd3a7b2e184e88d25c9baa6a9dc609ba25b76daae942edfb14499ac7ec374 \ + --hash=sha256:ee25f1ac091def37c4b59d192bbe3a206298feeb89132a470325bf76ad122a1e \ + --hash=sha256:efa44f64c37cc30c9f05932c740a8b40ce359f51882c70883cc95feac842da4d \ + --hash=sha256:f47d52fd9b2ac418c4890aad2f6d21a6b96183c98021f0a48497a904199f006e \ + --hash=sha256:f857034dc68d5ceb30fb60afb6ff2103087aea10a01b613985610e007053a121 \ + --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ + --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 + # via + # -c python/requirements_compiled.txt + # aiohttp + # yarl +proto-plus==1.22.3 \ + --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ + --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b + # via + # -c python/requirements_compiled.txt + # google-api-core +protobuf==4.25.8 \ + --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ + --hash=sha256:15a0af558aa3b13efef102ae6e4f3efac06f1eea11afb3a57db2901447d9fb59 \ + --hash=sha256:27d498ffd1f21fb81d987a041c32d07857d1d107909f5134ba3350e1ce80a4af \ + --hash=sha256:504435d831565f7cfac9f0714440028907f1975e4bed228e58e72ecfff58a1e0 \ + --hash=sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd \ + --hash=sha256:83e6e54e93d2b696a92cad6e6efc924f3850f82b52e1563778dfab8b355101b0 \ + --hash=sha256:9ad7ef62d92baf5a8654fbb88dac7fa5594cfa70fd3440488a5ca3bfc6d795a7 \ + --hash=sha256:bd551eb1fe1d7e92c1af1d75bdfa572eff1ab0e5bf1736716814cdccdb2360f9 \ + --hash=sha256:ca809b42f4444f144f2115c4c1a747b9a404d590f18f37e9402422033e464e0f \ + --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ + --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 + # via + # -c python/requirements_compiled.txt + # google-api-core + # googleapis-common-protos + # opentelemetry-proto + # proto-plus + # ray + # tensorboardx +py-spy==0.4.0 ; python_full_version < '3.12' \ + --hash=sha256:47cdda4c34d9b6cb01f3aaeceb2e88faf57da880207fe72ff6ff97e9bb6cc8a9 \ + --hash=sha256:77d8f637ade38367d944874776f45b703b7ac5938b1f7be8891f3a5876ddbb96 \ + --hash=sha256:806602ce7972782cc9c1e383f339bfc27bfb822d42485e6a3e0530ae5040e1f0 \ + --hash=sha256:87573e64dbfdfc89ba2e0f5e2f525aa84e0299c7eb6454b47ea335fde583a7a0 \ + --hash=sha256:8bf2f3702cef367a489faa45177b41a6c31b2a3e5bd78c978d44e29340152f5a \ + --hash=sha256:c5f06ffce4c9c98b7fc9f5e67e5e7db591173f1351837633f3f23d9378b1d18a \ + --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ + --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 + # via + # -c python/requirements_compiled.txt + # ray +pyarrow==19.0.1 \ + --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ + --hash=sha256:0148bb4fc158bfbc3d6dfe5001d93ebeed253793fff4435167f6ce1dc4bddeae \ + --hash=sha256:1b93ef2c93e77c442c979b0d596af45e4665d8b96da598db145b0fec014b9136 \ + --hash=sha256:1c7556165bd38cf0cd992df2636f8bcdd2d4b26916c6b7e646101aff3c16f76f \ + --hash=sha256:335d170e050bcc7da867a1ed8ffb8b44c57aaa6e0843b156a501298657b1e972 \ + --hash=sha256:3bf266b485df66a400f282ac0b6d1b500b9d2ae73314a153dbe97d6d5cc8a99e \ + --hash=sha256:41f9706fbe505e0abc10e84bf3a906a1338905cbbcf1177b71486b03e6ea6608 \ + --hash=sha256:4982f8e2b7afd6dae8608d70ba5bd91699077323f812a0448d8b7abdff6cb5d3 \ + --hash=sha256:49a3aecb62c1be1d822f8bf629226d4a96418228a42f5b40835c1f10d42e4db6 \ + --hash=sha256:4d5d1ec7ec5324b98887bdc006f4d2ce534e10e60f7ad995e7875ffa0ff9cb14 \ + --hash=sha256:58d9397b2e273ef76264b45531e9d552d8ec8a6688b7390b5be44c02a37aade8 \ + --hash=sha256:5a9137cf7e1640dce4c190551ee69d478f7121b5c6f323553b319cac936395f6 \ + --hash=sha256:5bd1618ae5e5476b7654c7b55a6364ae87686d4724538c24185bbb2952679960 \ + --hash=sha256:65cf9feebab489b19cdfcfe4aa82f62147218558d8d3f0fc1e9dea0ab8e7905a \ + --hash=sha256:699799f9c80bebcf1da0983ba86d7f289c5a2a5c04b945e2f2bcf7e874a91911 \ + --hash=sha256:6c5941c1aac89a6c2f2b16cd64fe76bcdb94b2b1e99ca6459de4e6f07638d755 \ + --hash=sha256:6ebfb5171bb5f4a52319344ebbbecc731af3f021e49318c74f33d520d31ae0c4 \ + --hash=sha256:7a544ec12de66769612b2d6988c36adc96fb9767ecc8ee0a4d270b10b1c51e00 \ + --hash=sha256:7c1bca1897c28013db5e4c83944a2ab53231f541b9e0c3f4791206d0c0de389a \ + --hash=sha256:80b2ad2b193e7d19e81008a96e313fbd53157945c7be9ac65f44f8937a55427b \ + --hash=sha256:8464c9fbe6d94a7fe1599e7e8965f350fd233532868232ab2596a71586c5a429 \ + --hash=sha256:8f04d49a6b64cf24719c080b3c2029a3a5b16417fd5fd7c4041f94233af732f3 \ + --hash=sha256:96606c3ba57944d128e8a8399da4812f56c7f61de8c647e3470b417f795d0ef9 \ + --hash=sha256:99bc1bec6d234359743b01e70d4310d0ab240c3d6b0da7e2a93663b0158616f6 \ + --hash=sha256:ad76aef7f5f7e4a757fddcdcf010a8290958f09e3470ea458c80d26f4316ae89 \ + --hash=sha256:b4c4156a625f1e35d6c0b2132635a237708944eb41df5fbe7d50f20d20c17832 \ + --hash=sha256:b9766a47a9cb56fefe95cb27f535038b5a195707a08bf61b180e642324963b46 \ + --hash=sha256:c0fe3dbbf054a00d1f162fda94ce236a899ca01123a798c561ba307ca38af5f0 \ + --hash=sha256:c6cb2335a411b713fdf1e82a752162f72d4a7b5dbc588e32aa18383318b05866 \ + --hash=sha256:cc55d71898ea30dc95900297d191377caba257612f384207fe9f8293b5850f90 \ + --hash=sha256:d03c9d6f2a3dffbd62671ca070f13fc527bb1867b4ec2b98c7eeed381d4f389a \ + --hash=sha256:d383591f3dcbe545f6cc62daaef9c7cdfe0dff0fb9e1c8121101cabe9098cfa6 \ + --hash=sha256:d9d46e06846a41ba906ab25302cf0fd522f81aa2a85a71021826f34639ad31ef \ + --hash=sha256:d9dedeaf19097a143ed6da37f04f4051aba353c95ef507764d344229b2b740ae \ + --hash=sha256:e45274b20e524ae5c39d7fc1ca2aa923aab494776d2d4b316b49ec7572ca324c \ + --hash=sha256:ee8dec072569f43835932a3b10c55973593abc00936c202707a4ad06af7cb294 \ + --hash=sha256:f24faab6ed18f216a37870d8c5623f9c044566d75ec586ef884e13a02a9d62c5 \ + --hash=sha256:f2a21d39fbdb948857f67eacb5bbaaf36802de044ec36fbef7a1c8f0dd3a4ab2 \ + --hash=sha256:f3ad4c0eb4e2a9aeb990af6c09e6fa0b195c8c0e7b272ecc8d4d2b6574809d34 \ + --hash=sha256:fc28912a2dc924dddc2087679cc8b7263accc71b9ff025a1362b004711661a69 \ + --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ + --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 + # via + # -c python/requirements_compiled.txt + # ray +pyasn1==0.5.1 \ + --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ + --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c + # via + # -c python/requirements_compiled.txt + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 \ + --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ + --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d + # via + # -c python/requirements_compiled.txt + # google-auth +pycparser==2.21 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ + --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 + # via + # -c python/requirements_compiled.txt + # cffi +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b + # via + # -c python/requirements_compiled.txt + # fastapi + # ray +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d + # via + # -c python/requirements_compiled.txt + # pydantic +pygments==2.18.0 ; sys_platform != 'win32' \ + --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ + --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a + # via + # -c python/requirements_compiled.txt + # rich +pyopenssl==25.0.0 \ + --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ + --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 + # via + # -c python/requirements_compiled.txt + # ray +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via + # -c python/requirements_compiled.txt + # celery + # pandas +python-dotenv==1.1.1 \ + --hash=sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc \ + --hash=sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab + # via uvicorn +pytz==2022.7.1 \ + --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ + --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a + # via + # -c python/requirements_compiled.txt + # pandas +pyyaml==6.0.1 \ + --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ + --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ + --hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \ + --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ + --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \ + --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \ + --hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \ + --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \ + --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \ + --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \ + --hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \ + --hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \ + --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \ + --hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \ + --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \ + --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \ + --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \ + --hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \ + --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \ + --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \ + --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \ + --hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \ + --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \ + --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \ + --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \ + --hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \ + --hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \ + --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ + --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ + --hash=sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef \ + --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ + --hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \ + --hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \ + --hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \ + --hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \ + --hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \ + --hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \ + --hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \ + --hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \ + --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \ + --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \ + --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \ + --hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \ + --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \ + --hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \ + --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \ + --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \ + --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \ + --hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \ + --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ + --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f + # via + # -c python/requirements_compiled.txt + # ray + # uvicorn +ray==100.0.0.dev0 \ + --hash=sha256:9739ca053529f0ec60c6248748773470765550f34bb78502c55b913d65bb32eb +referencing==0.36.2 \ + --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ + --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 + # via + # -c python/requirements_compiled.txt + # jsonschema + # jsonschema-specifications +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 + # via + # -c python/requirements_compiled.txt + # google-api-core + # ray +rich==13.3.2 ; sys_platform != 'win32' \ + --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ + --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f + # via + # -c python/requirements_compiled.txt + # memray +rpds-py==0.22.3 \ + --hash=sha256:009de23c9c9ee54bf11303a966edf4d9087cd43a6003672e6aa7def643d06518 \ + --hash=sha256:02fbb9c288ae08bcb34fb41d516d5eeb0455ac35b5512d03181d755d80810059 \ + --hash=sha256:0a0461200769ab3b9ab7e513f6013b7a97fdeee41c29b9db343f3c5a8e2b9e61 \ + --hash=sha256:0b09865a9abc0ddff4e50b5ef65467cd94176bf1e0004184eb915cbc10fc05c5 \ + --hash=sha256:0b8db6b5b2d4491ad5b6bdc2bc7c017eec108acbf4e6785f42a9eb0ba234f4c9 \ + --hash=sha256:0c150c7a61ed4a4f4955a96626574e9baf1adf772c2fb61ef6a5027e52803543 \ + --hash=sha256:0f3cec041684de9a4684b1572fe28c7267410e02450f4561700ca5a3bc6695a2 \ + --hash=sha256:1352ae4f7c717ae8cba93421a63373e582d19d55d2ee2cbb184344c82d2ae55a \ + --hash=sha256:177c7c0fce2855833819c98e43c262007f42ce86651ffbb84f37883308cb0e7d \ + --hash=sha256:1978d0021e943aae58b9b0b196fb4895a25cc53d3956b8e35e0b7682eefb6d56 \ + --hash=sha256:1a60bce91f81ddaac922a40bbb571a12c1070cb20ebd6d49c48e0b101d87300d \ + --hash=sha256:1aef18820ef3e4587ebe8b3bc9ba6e55892a6d7b93bac6d29d9f631a3b4befbd \ + --hash=sha256:1e9663daaf7a63ceccbbb8e3808fe90415b0757e2abddbfc2e06c857bf8c5e2b \ + --hash=sha256:20070c65396f7373f5df4005862fa162db5d25d56150bddd0b3e8214e8ef45b4 \ + --hash=sha256:214b7a953d73b5e87f0ebece4a32a5bd83c60a3ecc9d4ec8f1dca968a2d91e99 \ + --hash=sha256:22bebe05a9ffc70ebfa127efbc429bc26ec9e9b4ee4d15a740033efda515cf3d \ + --hash=sha256:24e8abb5878e250f2eb0d7859a8e561846f98910326d06c0d51381fed59357bd \ + --hash=sha256:26fd7cac7dd51011a245f29a2cc6489c4608b5a8ce8d75661bb4a1066c52dfbe \ + --hash=sha256:27b1d3b3915a99208fee9ab092b8184c420f2905b7d7feb4aeb5e4a9c509b8a1 \ + --hash=sha256:27e98004595899949bd7a7b34e91fa7c44d7a97c40fcaf1d874168bb652ec67e \ + --hash=sha256:2b8f60e1b739a74bab7e01fcbe3dddd4657ec685caa04681df9d562ef15b625f \ + --hash=sha256:2de29005e11637e7a2361fa151f780ff8eb2543a0da1413bb951e9f14b699ef3 \ + --hash=sha256:2e8b55d8517a2fda8d95cb45d62a5a8bbf9dd0ad39c5b25c8833efea07b880ca \ + --hash=sha256:2fa4331c200c2521512595253f5bb70858b90f750d39b8cbfd67465f8d1b596d \ + --hash=sha256:3445e07bf2e8ecfeef6ef67ac83de670358abf2996916039b16a218e3d95e97e \ + --hash=sha256:3453e8d41fe5f17d1f8e9c383a7473cd46a63661628ec58e07777c2fff7196dc \ + --hash=sha256:378753b4a4de2a7b34063d6f95ae81bfa7b15f2c1a04a9518e8644e81807ebea \ + --hash=sha256:3af6e48651c4e0d2d166dc1b033b7042ea3f871504b6805ba5f4fe31581d8d38 \ + --hash=sha256:3dfcbc95bd7992b16f3f7ba05af8a64ca694331bd24f9157b49dadeeb287493b \ + --hash=sha256:3f21f0495edea7fdbaaa87e633a8689cd285f8f4af5c869f27bc8074638ad69c \ + --hash=sha256:4041711832360a9b75cfb11b25a6a97c8fb49c07b8bd43d0d02b45d0b499a4ff \ + --hash=sha256:44d61b4b7d0c2c9ac019c314e52d7cbda0ae31078aabd0f22e583af3e0d79723 \ + --hash=sha256:4617e1915a539a0d9a9567795023de41a87106522ff83fbfaf1f6baf8e85437e \ + --hash=sha256:4b232061ca880db21fa14defe219840ad9b74b6158adb52ddf0e87bead9e8493 \ + --hash=sha256:5246b14ca64a8675e0a7161f7af68fe3e910e6b90542b4bfb5439ba752191df6 \ + --hash=sha256:5725dd9cc02068996d4438d397e255dcb1df776b7ceea3b9cb972bdb11260a83 \ + --hash=sha256:583f6a1993ca3369e0f80ba99d796d8e6b1a3a2a442dd4e1a79e652116413091 \ + --hash=sha256:59259dc58e57b10e7e18ce02c311804c10c5a793e6568f8af4dead03264584d1 \ + --hash=sha256:593eba61ba0c3baae5bc9be2f5232430453fb4432048de28399ca7376de9c627 \ + --hash=sha256:59f4a79c19232a5774aee369a0c296712ad0e77f24e62cad53160312b1c1eaa1 \ + --hash=sha256:5f0e260eaf54380380ac3808aa4ebe2d8ca28b9087cf411649f96bad6900c728 \ + --hash=sha256:62d9cfcf4948683a18a9aff0ab7e1474d407b7bab2ca03116109f8464698ab16 \ + --hash=sha256:64607d4cbf1b7e3c3c8a14948b99345eda0e161b852e122c6bb71aab6d1d798c \ + --hash=sha256:655ca44a831ecb238d124e0402d98f6212ac527a0ba6c55ca26f616604e60a45 \ + --hash=sha256:666ecce376999bf619756a24ce15bb14c5bfaf04bf00abc7e663ce17c3f34fe7 \ + --hash=sha256:68049202f67380ff9aa52f12e92b1c30115f32e6895cd7198fa2a7961621fc5a \ + --hash=sha256:69803198097467ee7282750acb507fba35ca22cc3b85f16cf45fb01cb9097730 \ + --hash=sha256:6c7b99ca52c2c1752b544e310101b98a659b720b21db00e65edca34483259967 \ + --hash=sha256:6dd9412824c4ce1aca56c47b0991e65bebb7ac3f4edccfd3f156150c96a7bf25 \ + --hash=sha256:70eb60b3ae9245ddea20f8a4190bd79c705a22f8028aaf8bbdebe4716c3fab24 \ + --hash=sha256:70fb28128acbfd264eda9bf47015537ba3fe86e40d046eb2963d75024be4d055 \ + --hash=sha256:7b2513ba235829860b13faa931f3b6846548021846ac808455301c23a101689d \ + --hash=sha256:7ef9d9da710be50ff6809fed8f1963fecdfecc8b86656cadfca3bc24289414b0 \ + --hash=sha256:81e69b0a0e2537f26d73b4e43ad7bc8c8efb39621639b4434b76a3de50c6966e \ + --hash=sha256:8633e471c6207a039eff6aa116e35f69f3156b3989ea3e2d755f7bc41754a4a7 \ + --hash=sha256:8bd7c8cfc0b8247c8799080fbff54e0b9619e17cdfeb0478ba7295d43f635d7c \ + --hash=sha256:9253fc214112405f0afa7db88739294295f0e08466987f1d70e29930262b4c8f \ + --hash=sha256:99b37292234e61325e7a5bb9689e55e48c3f5f603af88b1642666277a81f1fbd \ + --hash=sha256:9bd7228827ec7bb817089e2eb301d907c0d9827a9e558f22f762bb690b131652 \ + --hash=sha256:9beeb01d8c190d7581a4d59522cd3d4b6887040dcfc744af99aa59fef3e041a8 \ + --hash=sha256:a63cbdd98acef6570c62b92a1e43266f9e8b21e699c363c0fef13bd530799c11 \ + --hash=sha256:a76e42402542b1fae59798fab64432b2d015ab9d0c8c47ba7addddbaf7952333 \ + --hash=sha256:ac0a03221cdb5058ce0167ecc92a8c89e8d0decdc9e99a2ec23380793c4dcb96 \ + --hash=sha256:b0b4136a252cadfa1adb705bb81524eee47d9f6aab4f2ee4fa1e9d3cd4581f64 \ + --hash=sha256:b25bc607423935079e05619d7de556c91fb6adeae9d5f80868dde3468657994b \ + --hash=sha256:b3d504047aba448d70cf6fa22e06cb09f7cbd761939fdd47604f5e007675c24e \ + --hash=sha256:bb47271f60660803ad11f4c61b42242b8c1312a31c98c578f79ef9387bbde21c \ + --hash=sha256:bbb232860e3d03d544bc03ac57855cd82ddf19c7a07651a7c0fdb95e9efea8b9 \ + --hash=sha256:bc27863442d388870c1809a87507727b799c8460573cfbb6dc0eeaef5a11b5ec \ + --hash=sha256:bc51abd01f08117283c5ebf64844a35144a0843ff7b2983e0648e4d3d9f10dbb \ + --hash=sha256:be2eb3f2495ba669d2a985f9b426c1797b7d48d6963899276d22f23e33d47e37 \ + --hash=sha256:bf9db5488121b596dbfc6718c76092fda77b703c1f7533a226a5a9f65248f8ad \ + --hash=sha256:c58e2339def52ef6b71b8f36d13c3688ea23fa093353f3a4fee2556e62086ec9 \ + --hash=sha256:cfbc454a2880389dbb9b5b398e50d439e2e58669160f27b60e5eca11f68ae17c \ + --hash=sha256:cff63a0272fcd259dcc3be1657b07c929c466b067ceb1c20060e8d10af56f5bf \ + --hash=sha256:d115bffdd417c6d806ea9069237a4ae02f513b778e3789a359bc5856e0404cc4 \ + --hash=sha256:d20cfb4e099748ea39e6f7b16c91ab057989712d31761d3300d43134e26e165f \ + --hash=sha256:d48424e39c2611ee1b84ad0f44fb3b2b53d473e65de061e3f460fc0be5f1939d \ + --hash=sha256:e0fa2d4ec53dc51cf7d3bb22e0aa0143966119f42a0c3e4998293a3dd2856b09 \ + --hash=sha256:e32fee8ab45d3c2db6da19a5323bc3362237c8b653c70194414b892fd06a080d \ + --hash=sha256:e35ba67d65d49080e8e5a1dd40101fccdd9798adb9b050ff670b7d74fa41c566 \ + --hash=sha256:e3fb866d9932a3d7d0c82da76d816996d1667c44891bd861a0f97ba27e84fc74 \ + --hash=sha256:e61b02c3f7a1e0b75e20c3978f7135fd13cb6cf551bf4a6d29b999a88830a338 \ + --hash=sha256:e67ba3c290821343c192f7eae1d8fd5999ca2dc99994114643e2f2d3e6138b15 \ + --hash=sha256:e79dd39f1e8c3504be0607e5fc6e86bb60fe3584bec8b782578c3b0fde8d932c \ + --hash=sha256:e89391e6d60251560f0a8f4bd32137b077a80d9b7dbe6d5cab1cd80d2746f648 \ + --hash=sha256:ea7433ce7e4bfc3a85654aeb6747babe3f66eaf9a1d0c1e7a4435bbdf27fea84 \ + --hash=sha256:eaf16ae9ae519a0e237a0f528fd9f0197b9bb70f40263ee57ae53c2b8d48aeb3 \ + --hash=sha256:eb0c341fa71df5a4595f9501df4ac5abfb5a09580081dffbd1ddd4654e6e9123 \ + --hash=sha256:f276b245347e6e36526cbd4a266a417796fc531ddf391e43574cf6466c492520 \ + --hash=sha256:f47ad3d5f3258bd7058d2d506852217865afefe6153a36eb4b6928758041d831 \ + --hash=sha256:f56a6b404f74ab372da986d240e2e002769a7d7102cc73eb238a4f72eec5284e \ + --hash=sha256:f5cf2a0c2bdadf3791b5c205d55a37a54025c6e18a71c71f82bb536cf9a454bf \ + --hash=sha256:f5d36399a1b96e1a5fdc91e0522544580dbebeb1f77f27b2b0ab25559e103b8b \ + --hash=sha256:f60bd8423be1d9d833f230fdbccf8f57af322d96bcad6599e5a771b151398eb2 \ + --hash=sha256:f612463ac081803f243ff13cccc648578e2279295048f2a8d5eb430af2bae6e3 \ + --hash=sha256:f73d3fef726b3243a811121de45193c0ca75f6407fe66f3f4e183c983573e130 \ + --hash=sha256:f82a116a1d03628a8ace4859556fb39fd1424c933341a08ea3ed6de1edb0283b \ + --hash=sha256:fb0ba113b4983beac1a2eb16faffd76cb41e176bf58c4afe3e14b9c681f702de \ + --hash=sha256:fb4f868f712b2dd4bcc538b0a0c1f63a2b1d584c925e69a224d759e7070a12d5 \ + --hash=sha256:fb6116dfb8d1925cbdb52595560584db42a7f664617a1f7d7f6e32f138cdf37d \ + --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ + --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e + # via + # -c python/requirements_compiled.txt + # jsonschema + # referencing +rsa==4.7.2 \ + --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ + --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 + # via + # -c python/requirements_compiled.txt + # google-auth +scipy==1.11.4 \ + --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ + --hash=sha256:028eccd22e654b3ea01ee63705681ee79933652b2d8f873e7949898dda6d11b6 \ + --hash=sha256:1b7c3dca977f30a739e0409fb001056484661cb2541a01aba0bb0029f7b68db8 \ + --hash=sha256:2c6ff6ef9cc27f9b3db93a6f8b38f97387e6e0591600369a297a50a8e96e835d \ + --hash=sha256:36750b7733d960d7994888f0d148d31ea3017ac15eef664194b4ef68d36a4a97 \ + --hash=sha256:530f9ad26440e85766509dbf78edcfe13ffd0ab7fec2560ee5c36ff74d6269ff \ + --hash=sha256:5e347b14fe01003d3b78e196e84bd3f48ffe4c8a7b8a1afbcb8f5505cb710993 \ + --hash=sha256:6550466fbeec7453d7465e74d4f4b19f905642c89a7525571ee91dd7adabb5a3 \ + --hash=sha256:6df1468153a31cf55ed5ed39647279beb9cfb5d3f84369453b49e4b8502394fd \ + --hash=sha256:6e619aba2df228a9b34718efb023966da781e89dd3d21637b27f2e54db0410d7 \ + --hash=sha256:8fce70f39076a5aa62e92e69a7f62349f9574d8405c0a5de6ed3ef72de07f446 \ + --hash=sha256:90a2b78e7f5733b9de748f589f09225013685f9b218275257f8a8168ededaeaa \ + --hash=sha256:91af76a68eeae0064887a48e25c4e616fa519fa0d38602eda7e0f97d65d57937 \ + --hash=sha256:933baf588daa8dc9a92c20a0be32f56d43faf3d1a60ab11b3f08c356430f6e56 \ + --hash=sha256:acf8ed278cc03f5aff035e69cb511741e0418681d25fbbb86ca65429c4f4d9cd \ + --hash=sha256:ad669df80528aeca5f557712102538f4f37e503f0c5b9541655016dd0932ca79 \ + --hash=sha256:b030c6674b9230d37c5c60ab456e2cf12f6784596d15ce8da9365e70896effc4 \ + --hash=sha256:b9999c008ccf00e8fbcce1236f85ade5c569d13144f77a1946bef8863e8f6eb4 \ + --hash=sha256:bc9a714581f561af0848e6b69947fda0614915f072dfd14142ed1bfe1b806710 \ + --hash=sha256:ce7fff2e23ab2cc81ff452a9444c215c28e6305f396b2ba88343a567feec9660 \ + --hash=sha256:cf00bd2b1b0211888d4dc75656c0412213a8b25e80d73898083f402b50f47e41 \ + --hash=sha256:d10e45a6c50211fe256da61a11c34927c68f277e03138777bdebedd933712fea \ + --hash=sha256:ee410e6de8f88fd5cf6eadd73c135020bfbbbdfcd0f6162c36a7638a1ea8cc65 \ + --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ + --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec + # via + # -c python/requirements_compiled.txt + # ray +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via + # -c python/requirements_compiled.txt + # opencensus + # python-dateutil +smart-open==6.2.0 \ + --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ + --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 + # via + # -c python/requirements_compiled.txt + # ray +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc + # via + # -c python/requirements_compiled.txt + # anyio +starlette==0.46.2 \ + --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ + --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 + # via + # -c python/requirements_compiled.txt + # fastapi + # ray +tensorboardx==2.6.2.2 \ + --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ + --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 + # via + # -c python/requirements_compiled.txt + # ray +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d + # via + # -c python/requirements_compiled.txt + # exceptiongroup + # fastapi + # gymnasium + # opentelemetry-api + # opentelemetry-sdk + # opentelemetry-semantic-conventions + # pydantic + # pydantic-core + # pyopenssl + # referencing + # typing-inspection +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/requirements_compiled.txt + # pydantic +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled.txt + # kombu +urllib3==1.26.19 \ + --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ + --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 + # via + # -c python/requirements_compiled.txt + # requests +uvicorn==0.22.0 \ + --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ + --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 + # via + # -c python/requirements_compiled.txt + # ray +uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32' \ + --hash=sha256:0878c2640cf341b269b7e128b1a5fed890adc4455513ca710d77d5e93aa6d6a0 \ + --hash=sha256:10d66943def5fcb6e7b37310eb6b5639fd2ccbc38df1177262b0640c3ca68c1f \ + --hash=sha256:10da8046cc4a8f12c91a1c39d1dd1585c41162a15caaef165c2174db9ef18bdc \ + --hash=sha256:17df489689befc72c39a08359efac29bbee8eee5209650d4b9f34df73d22e414 \ + --hash=sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f \ + --hash=sha256:196274f2adb9689a289ad7d65700d37df0c0930fd8e4e743fa4834e850d7719d \ + --hash=sha256:221f4f2a1f46032b403bf3be628011caf75428ee3cc204a22addf96f586b19fd \ + --hash=sha256:2d1f581393673ce119355d56da84fe1dd9d2bb8b3d13ce792524e1607139feff \ + --hash=sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c \ + --hash=sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3 \ + --hash=sha256:4509360fcc4c3bd2c70d87573ad472de40c13387f5fda8cb58350a1d7475e58d \ + --hash=sha256:460def4412e473896ef179a1671b40c039c7012184b627898eea5072ef6f017a \ + --hash=sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb \ + --hash=sha256:46923b0b5ee7fc0020bef24afe7836cb068f5050ca04caf6b487c513dc1a20b2 \ + --hash=sha256:53e420a3afe22cdcf2a0f4846e377d16e718bc70103d7088a4f7623567ba5fb0 \ + --hash=sha256:5ee4d4ef48036ff6e5cfffb09dd192c7a5027153948d85b8da7ff705065bacc6 \ + --hash=sha256:67dd654b8ca23aed0a8e99010b4c34aca62f4b7fce88f39d452ed7622c94845c \ + --hash=sha256:787ae31ad8a2856fc4e7c095341cccc7209bd657d0e71ad0dc2ea83c4a6fa8af \ + --hash=sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc \ + --hash=sha256:87c43e0f13022b998eb9b973b5e97200c8b90823454d4bc06ab33829e09fb9bb \ + --hash=sha256:88cb67cdbc0e483da00af0b2c3cdad4b7c61ceb1ee0f33fe00e09c81e3a6cb75 \ + --hash=sha256:8a375441696e2eda1c43c44ccb66e04d61ceeffcd76e4929e527b7fa401b90fb \ + --hash=sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553 \ + --hash=sha256:b9fb766bb57b7388745d8bcc53a359b116b8a04c83a2288069809d2b3466c37e \ + --hash=sha256:baa0e6291d91649c6ba4ed4b2f982f9fa165b5bbd50a9e203c416a2797bab3c6 \ + --hash=sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d \ + --hash=sha256:bc09f0ff191e61c2d592a752423c767b4ebb2986daa9ed62908e2b1b9a9ae206 \ + --hash=sha256:bd53ecc9a0f3d87ab847503c2e1552b690362e005ab54e8a48ba97da3924c0dc \ + --hash=sha256:bfd55dfcc2a512316e65f16e503e9e450cab148ef11df4e4e679b5e8253a5281 \ + --hash=sha256:c097078b8031190c934ed0ebfee8cc5f9ba9642e6eb88322b9958b649750f72b \ + --hash=sha256:c0f3fa6200b3108919f8bdabb9a7f87f20e7097ea3c543754cabc7d717d95cf8 \ + --hash=sha256:e678ad6fe52af2c58d2ae3c73dc85524ba8abe637f134bf3564ed07f555c5e79 \ + --hash=sha256:ec7e6b09a6fdded42403182ab6b832b71f4edaf7f37a9a0e371a01db5f0cb45f \ + --hash=sha256:f0ce1b49560b1d2d8a2977e3ba4afb2414fb46b86a1b64056bc4ab929efdafbe \ + --hash=sha256:f38b2e090258d051d68a5b14d1da7203a3c3677321cf32a95a6f4db4dd8b6f26 \ + --hash=sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816 \ + --hash=sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2 + # via + # -c python/requirements_compiled.txt + # uvicorn +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled.txt + # amqp + # celery + # kombu +virtualenv==20.29.1 \ + --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ + --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 + # via + # -c python/requirements_compiled.txt + # ray +watchfiles==0.19.0 \ + --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ + --hash=sha256:09ea3397aecbc81c19ed7f025e051a7387feefdb789cf768ff994c1228182fda \ + --hash=sha256:176a9a7641ec2c97b24455135d58012a5be5c6217fc4d5fef0b2b9f75dbf5154 \ + --hash=sha256:18b28f6ad871b82df9542ff958d0c86bb0d8310bb09eb8e87d97318a3b5273af \ + --hash=sha256:20b44221764955b1e703f012c74015306fb7e79a00c15370785f309b1ed9aa8d \ + --hash=sha256:3d7d267d27aceeeaa3de0dd161a0d64f0a282264d592e335fff7958cc0cbae7c \ + --hash=sha256:5471582658ea56fca122c0f0d0116a36807c63fefd6fdc92c71ca9a4491b6b48 \ + --hash=sha256:5569fc7f967429d4bc87e355cdfdcee6aabe4b620801e2cf5805ea245c06097c \ + --hash=sha256:68dce92b29575dda0f8d30c11742a8e2b9b8ec768ae414b54f7453f27bdf9545 \ + --hash=sha256:79c533ff593db861ae23436541f481ec896ee3da4e5db8962429b441bbaae16e \ + --hash=sha256:7f3920b1285a7d3ce898e303d84791b7bf40d57b7695ad549dc04e6a44c9f120 \ + --hash=sha256:91633e64712df3051ca454ca7d1b976baf842d7a3640b87622b323c55f3345e7 \ + --hash=sha256:945be0baa3e2440151eb3718fd8846751e8b51d8de7b884c90b17d271d34cae8 \ + --hash=sha256:9afd0d69429172c796164fd7fe8e821ade9be983f51c659a38da3faaaaac44dc \ + --hash=sha256:9c75eff897786ee262c9f17a48886f4e98e6cfd335e011c591c305e5d083c056 \ + --hash=sha256:b538014a87f94d92f98f34d3e6d2635478e6be6423a9ea53e4dd96210065e193 \ + --hash=sha256:b6577b8c6c8701ba8642ea9335a129836347894b666dd1ec2226830e263909d3 \ + --hash=sha256:c0376deac92377817e4fb8f347bf559b7d44ff556d9bc6f6208dd3f79f104aaf \ + --hash=sha256:cae3dde0b4b2078f31527acff6f486e23abed307ba4d3932466ba7cdd5ecec79 \ + --hash=sha256:cb5d45c4143c1dd60f98a16187fd123eda7248f84ef22244818c18d531a249d1 \ + --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ + --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 + # via + # -c python/requirements_compiled.txt + # ray + # uvicorn +wcwidth==0.2.13 \ + --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ + --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 + # via + # -c python/requirements_compiled.txt + # prompt-toolkit +websockets==11.0.3 \ + --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \ + --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \ + --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \ + --hash=sha256:0ee68fe502f9031f19d495dae2c268830df2760c0524cbac5d759921ba8c8e82 \ + --hash=sha256:1553cb82942b2a74dd9b15a018dce645d4e68674de2ca31ff13ebc2d9f283788 \ + --hash=sha256:1a073fc9ab1c8aff37c99f11f1641e16da517770e31a37265d2755282a5d28aa \ + --hash=sha256:1d2256283fa4b7f4c7d7d3e84dc2ece74d341bce57d5b9bf385df109c2a1a82f \ + --hash=sha256:1d5023a4b6a5b183dc838808087033ec5df77580485fc533e7dab2567851b0a4 \ + --hash=sha256:1fdf26fa8a6a592f8f9235285b8affa72748dc12e964a5518c6c5e8f916716f7 \ + --hash=sha256:2529338a6ff0eb0b50c7be33dc3d0e456381157a31eefc561771ee431134a97f \ + --hash=sha256:279e5de4671e79a9ac877427f4ac4ce93751b8823f276b681d04b2156713b9dd \ + --hash=sha256:2d903ad4419f5b472de90cd2d40384573b25da71e33519a67797de17ef849b69 \ + --hash=sha256:332d126167ddddec94597c2365537baf9ff62dfcc9db4266f263d455f2f031cb \ + --hash=sha256:34fd59a4ac42dff6d4681d8843217137f6bc85ed29722f2f7222bd619d15e95b \ + --hash=sha256:3580dd9c1ad0701169e4d6fc41e878ffe05e6bdcaf3c412f9d559389d0c9e016 \ + --hash=sha256:3ccc8a0c387629aec40f2fc9fdcb4b9d5431954f934da3eaf16cdc94f67dbfac \ + --hash=sha256:41f696ba95cd92dc047e46b41b26dd24518384749ed0d99bea0a941ca87404c4 \ + --hash=sha256:42cc5452a54a8e46a032521d7365da775823e21bfba2895fb7b77633cce031bb \ + --hash=sha256:4841ed00f1026dfbced6fca7d963c4e7043aa832648671b5138008dc5a8f6d99 \ + --hash=sha256:4b253869ea05a5a073ebfdcb5cb3b0266a57c3764cf6fe114e4cd90f4bfa5f5e \ + --hash=sha256:54c6e5b3d3a8936a4ab6870d46bdd6ec500ad62bde9e44462c32d18f1e9a8e54 \ + --hash=sha256:619d9f06372b3a42bc29d0cd0354c9bb9fb39c2cbc1a9c5025b4538738dbffaf \ + --hash=sha256:6505c1b31274723ccaf5f515c1824a4ad2f0d191cec942666b3d0f3aa4cb4007 \ + --hash=sha256:660e2d9068d2bedc0912af508f30bbeb505bbbf9774d98def45f68278cea20d3 \ + --hash=sha256:6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6 \ + --hash=sha256:68b977f21ce443d6d378dbd5ca38621755f2063d6fdb3335bda981d552cfff86 \ + --hash=sha256:69269f3a0b472e91125b503d3c0b3566bda26da0a3261c49f0027eb6075086d1 \ + --hash=sha256:6f1a3f10f836fab6ca6efa97bb952300b20ae56b409414ca85bff2ad241d2a61 \ + --hash=sha256:7622a89d696fc87af8e8d280d9b421db5133ef5b29d3f7a1ce9f1a7bf7fcfa11 \ + --hash=sha256:777354ee16f02f643a4c7f2b3eff8027a33c9861edc691a2003531f5da4f6bc8 \ + --hash=sha256:84d27a4832cc1a0ee07cdcf2b0629a8a72db73f4cf6de6f0904f6661227f256f \ + --hash=sha256:8531fdcad636d82c517b26a448dcfe62f720e1922b33c81ce695d0edb91eb931 \ + --hash=sha256:86d2a77fd490ae3ff6fae1c6ceaecad063d3cc2320b44377efdde79880e11526 \ + --hash=sha256:88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016 \ + --hash=sha256:8a34e13a62a59c871064dfd8ffb150867e54291e46d4a7cf11d02c94a5275bae \ + --hash=sha256:8c82f11964f010053e13daafdc7154ce7385ecc538989a354ccc7067fd7028fd \ + --hash=sha256:92b2065d642bf8c0a82d59e59053dd2fdde64d4ed44efe4870fa816c1232647b \ + --hash=sha256:97b52894d948d2f6ea480171a27122d77af14ced35f62e5c892ca2fae9344311 \ + --hash=sha256:9d9acd80072abcc98bd2c86c3c9cd4ac2347b5a5a0cae7ed5c0ee5675f86d9af \ + --hash=sha256:9f59a3c656fef341a99e3d63189852be7084c0e54b75734cde571182c087b152 \ + --hash=sha256:aa5003845cdd21ac0dc6c9bf661c5beddd01116f6eb9eb3c8e272353d45b3288 \ + --hash=sha256:b16fff62b45eccb9c7abb18e60e7e446998093cdcb50fed33134b9b6878836de \ + --hash=sha256:b30c6590146e53149f04e85a6e4fcae068df4289e31e4aee1fdf56a0dead8f97 \ + --hash=sha256:b58cbf0697721120866820b89f93659abc31c1e876bf20d0b3d03cef14faf84d \ + --hash=sha256:b67c6f5e5a401fc56394f191f00f9b3811fe843ee93f4a70df3c389d1adf857d \ + --hash=sha256:bceab846bac555aff6427d060f2fcfff71042dba6f5fca7dc4f75cac815e57ca \ + --hash=sha256:bee9fcb41db2a23bed96c6b6ead6489702c12334ea20a297aa095ce6d31370d0 \ + --hash=sha256:c114e8da9b475739dde229fd3bc6b05a6537a88a578358bc8eb29b4030fac9c9 \ + --hash=sha256:c1f0524f203e3bd35149f12157438f406eff2e4fb30f71221c8a5eceb3617b6b \ + --hash=sha256:c792ea4eabc0159535608fc5658a74d1a81020eb35195dd63214dcf07556f67e \ + --hash=sha256:c7f3cb904cce8e1be667c7e6fef4516b98d1a6a0635a58a57528d577ac18a128 \ + --hash=sha256:d67ac60a307f760c6e65dad586f556dde58e683fab03323221a4e530ead6f74d \ + --hash=sha256:dcacf2c7a6c3a84e720d1bb2b543c675bf6c40e460300b628bab1b1efc7c034c \ + --hash=sha256:de36fe9c02995c7e6ae6efe2e205816f5f00c22fd1fbf343d4d18c3d5ceac2f5 \ + --hash=sha256:def07915168ac8f7853812cc593c71185a16216e9e4fa886358a17ed0fd9fcf6 \ + --hash=sha256:df41b9bc27c2c25b486bae7cf42fccdc52ff181c8c387bfd026624a491c2671b \ + --hash=sha256:e052b8467dd07d4943936009f46ae5ce7b908ddcac3fda581656b1b19c083d9b \ + --hash=sha256:e063b1865974611313a3849d43f2c3f5368093691349cf3c7c8f8f75ad7cb280 \ + --hash=sha256:e1459677e5d12be8bbc7584c35b992eea142911a6236a3278b9b5ce3326f282c \ + --hash=sha256:e1a99a7a71631f0efe727c10edfba09ea6bee4166a6f9c19aafb6c0b5917d09c \ + --hash=sha256:e590228200fcfc7e9109509e4d9125eace2042fd52b595dd22bbc34bb282307f \ + --hash=sha256:e6316827e3e79b7b8e7d8e3b08f4e331af91a48e794d5d8b099928b6f0b85f20 \ + --hash=sha256:e7837cb169eca3b3ae94cc5787c4fed99eef74c0ab9506756eea335e0d6f3ed8 \ + --hash=sha256:e848f46a58b9fcf3d06061d17be388caf70ea5b8cc3466251963c8345e13f7eb \ + --hash=sha256:ed058398f55163a79bb9f06a90ef9ccc063b204bb346c4de78efc5d15abfe602 \ + --hash=sha256:f2e58f2c36cc52d41f2659e4c0cbf7353e28c8c9e63e30d8c6d3494dc9fdedcf \ + --hash=sha256:f467ba0050b7de85016b43f5a22b46383ef004c4f672148a8abf32bc999a87f0 \ + --hash=sha256:f61bdb1df43dc9c131791fbc2355535f9024b9a04398d3bd0684fc16ab07df74 \ + --hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \ + --hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564 + # via + # -c python/requirements_compiled.txt + # uvicorn +yarl==1.18.3 \ + --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ + --hash=sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193 \ + --hash=sha256:045b8482ce9483ada4f3f23b3774f4e1bf4f23a2d5c912ed5170f68efb053318 \ + --hash=sha256:09c7907c8548bcd6ab860e5f513e727c53b4a714f459b084f6580b49fa1b9cee \ + --hash=sha256:0b0cad37311123211dc91eadcb322ef4d4a66008d3e1bdc404808992260e1a0e \ + --hash=sha256:0b3c92fa08759dbf12b3a59579a4096ba9af8dd344d9a813fc7f5070d86bbab1 \ + --hash=sha256:0fb2171a4486bb075316ee754c6d8382ea6eb8b399d4ec62fde2b591f879778a \ + --hash=sha256:1a74a13a4c857a84a845505fd2d68e54826a2cd01935a96efb1e9d86c728e186 \ + --hash=sha256:1d407181cfa6e70077df3377938c08012d18893f9f20e92f7d2f314a437c30b1 \ + --hash=sha256:1dd4bdd05407ced96fed3d7f25dbbf88d2ffb045a0db60dbc247f5b3c5c25d50 \ + --hash=sha256:25b411eddcfd56a2f0cd6a384e9f4f7aa3efee14b188de13048c25b5e91f1640 \ + --hash=sha256:2d06d3005e668744e11ed80812e61efd77d70bb7f03e33c1598c301eea20efbb \ + --hash=sha256:2ec9bbba33b2d00999af4631a3397d1fd78290c48e2a3e52d8dd72db3a067ac8 \ + --hash=sha256:3236da9272872443f81fedc389bace88408f64f89f75d1bdb2256069a8730ccc \ + --hash=sha256:35098b24e0327fc4ebdc8ffe336cee0a87a700c24ffed13161af80124b7dc8e5 \ + --hash=sha256:41f7ce59d6ee7741af71d82020346af364949314ed3d87553763a2df1829cc58 \ + --hash=sha256:436c4fc0a4d66b2badc6c5fc5ef4e47bb10e4fd9bf0c79524ac719a01f3607c2 \ + --hash=sha256:4891ed92157e5430874dad17b15eb1fda57627710756c27422200c52d8a4e393 \ + --hash=sha256:4ac515b860c36becb81bb84b667466885096b5fc85596948548b667da3bf9f24 \ + --hash=sha256:5094d9206c64181d0f6e76ebd8fb2f8fe274950a63890ee9e0ebfd58bf9d787b \ + --hash=sha256:54d6921f07555713b9300bee9c50fb46e57e2e639027089b1d795ecd9f7fa910 \ + --hash=sha256:578e281c393af575879990861823ef19d66e2b1d0098414855dd367e234f5b3c \ + --hash=sha256:5a3f356548e34a70b0172d8890006c37be92995f62d95a07b4a42e90fba54272 \ + --hash=sha256:602d98f2c2d929f8e697ed274fbadc09902c4025c5a9963bf4e9edfc3ab6f7ed \ + --hash=sha256:61b1a825a13bef4a5f10b1885245377d3cd0bf87cba068e1d9a88c2ae36880e1 \ + --hash=sha256:61e5e68cb65ac8f547f6b5ef933f510134a6bf31bb178be428994b0cb46c2a04 \ + --hash=sha256:61ee62ead9b68b9123ec24bc866cbef297dd266175d53296e2db5e7f797f902d \ + --hash=sha256:6333c5a377c8e2f5fae35e7b8f145c617b02c939d04110c76f29ee3676b5f9a5 \ + --hash=sha256:6748dbf9bfa5ba1afcc7556b71cda0d7ce5f24768043a02a58846e4a443d808d \ + --hash=sha256:67a283dd2882ac98cc6318384f565bffc751ab564605959df4752d42483ad889 \ + --hash=sha256:75674776d96d7b851b6498f17824ba17849d790a44d282929c42dbb77d4f17ae \ + --hash=sha256:757e81cae69244257d125ff31663249b3013b5dc0a8520d73694aed497fb195b \ + --hash=sha256:77a6e85b90a7641d2e07184df5557132a337f136250caafc9ccaa4a2a998ca2c \ + --hash=sha256:7c33dd1931a95e5d9a772d0ac5e44cac8957eaf58e3c8da8c1414de7dd27c576 \ + --hash=sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34 \ + --hash=sha256:7e2ee16578af3b52ac2f334c3b1f92262f47e02cc6193c598502bd46f5cd1477 \ + --hash=sha256:80316a8bd5109320d38eef8833ccf5f89608c9107d02d2a7f985f98ed6876990 \ + --hash=sha256:82123d0c954dc58db301f5021a01854a85bf1f3bb7d12ae0c01afc414a882ca2 \ + --hash=sha256:84b2deecba4a3f1a398df819151eb72d29bfeb3b69abb145a00ddc8d30094512 \ + --hash=sha256:8503ad47387b8ebd39cbbbdf0bf113e17330ffd339ba1144074da24c545f0069 \ + --hash=sha256:877d209b6aebeb5b16c42cbb377f5f94d9e556626b1bfff66d7b0d115be88d0a \ + --hash=sha256:8874027a53e3aea659a6d62751800cf6e63314c160fd607489ba5c2edd753cf6 \ + --hash=sha256:88a19f62ff30117e706ebc9090b8ecc79aeb77d0b1f5ec10d2d27a12bc9f66d0 \ + --hash=sha256:8d39d351e7faf01483cc7ff7c0213c412e38e5a340238826be7e0e4da450fdc8 \ + --hash=sha256:90adb47ad432332d4f0bc28f83a5963f426ce9a1a8809f5e584e704b82685dcb \ + --hash=sha256:913829534200eb0f789d45349e55203a091f45c37a2674678744ae52fae23efa \ + --hash=sha256:93b2e109287f93db79210f86deb6b9bbb81ac32fc97236b16f7433db7fc437d8 \ + --hash=sha256:9d41beda9dc97ca9ab0b9888cb71f7539124bc05df02c0cff6e5acc5a19dcc6e \ + --hash=sha256:a440a2a624683108a1b454705ecd7afc1c3438a08e890a1513d468671d90a04e \ + --hash=sha256:a4bb030cf46a434ec0225bddbebd4b89e6471814ca851abb8696170adb163985 \ + --hash=sha256:a9ca04806f3be0ac6d558fffc2fdf8fcef767e0489d2684a21912cc4ed0cd1b8 \ + --hash=sha256:ac1801c45cbf77b6c99242eeff4fffb5e4e73a800b5c4ad4fc0be5def634d2e1 \ + --hash=sha256:ac36703a585e0929b032fbaab0707b75dc12703766d0b53486eabd5139ebadd5 \ + --hash=sha256:b1771de9944d875f1b98a745bc547e684b863abf8f8287da8466cf470ef52690 \ + --hash=sha256:b464c4ab4bfcb41e3bfd3f1c26600d038376c2de3297760dfe064d2cb7ea8e10 \ + --hash=sha256:b4f6450109834af88cb4cc5ecddfc5380ebb9c228695afc11915a0bf82116789 \ + --hash=sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b \ + --hash=sha256:b643562c12680b01e17239be267bc306bbc6aac1f34f6444d1bded0c5ce438ca \ + --hash=sha256:b958ddd075ddba5b09bb0be8a6d9906d2ce933aee81100db289badbeb966f54e \ + --hash=sha256:b9d60031cf568c627d028239693fd718025719c02c9f55df0a53e587aab951b5 \ + --hash=sha256:ba23302c0c61a9999784e73809427c9dbedd79f66a13d84ad1b1943802eaaf59 \ + --hash=sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9 \ + --hash=sha256:c017a3b6df3a1bd45b9fa49a0f54005e53fbcad16633870104b66fa1a30a29d8 \ + --hash=sha256:c1e1cc06da1491e6734f0ea1e6294ce00792193c463350626571c287c9a704db \ + --hash=sha256:c654d5207c78e0bd6d749f6dae1dcbbfde3403ad3a4b11f3c5544d9906969dde \ + --hash=sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7 \ + --hash=sha256:c7d79f7d9aabd6011004e33b22bc13056a3e3fb54794d138af57f5ee9d9032cb \ + --hash=sha256:ccaa3a4b521b780a7e771cc336a2dba389a0861592bbce09a476190bb0c8b4b3 \ + --hash=sha256:ccd17349166b1bee6e529b4add61727d3f55edb7babbe4069b5764c9587a8cc6 \ + --hash=sha256:ce1af883b94304f493698b00d0f006d56aea98aeb49d75ec7d98cd4a777e9285 \ + --hash=sha256:d0e883008013c0e4aef84dcfe2a0b172c4d23c2669412cf5b3371003941f72bb \ + --hash=sha256:d980e0325b6eddc81331d3f4551e2a333999fb176fd153e075c6d1c2530aa8a8 \ + --hash=sha256:e17c9361d46a4d5addf777c6dd5eab0715a7684c2f11b88c67ac37edfba6c482 \ + --hash=sha256:e2c08cc9b16f4f4bc522771d96734c7901e7ebef70c6c5c35dd0f10845270bcd \ + --hash=sha256:e35ef8683211db69ffe129a25d5634319a677570ab6b2eba4afa860f54eeaf75 \ + --hash=sha256:e3b9fd71836999aad54084906f8663dffcd2a7fb5cdafd6c37713b2e72be1760 \ + --hash=sha256:ef9f7768395923c3039055c14334ba4d926f3baf7b776c923c93d80195624782 \ + --hash=sha256:f52a265001d830bc425f82ca9eabda94a64a4d753b07d623a9f2863fde532b53 \ + --hash=sha256:f91c4803173928a25e1a55b943c81f55b8872f0018be83e3ad4938adffb77dd2 \ + --hash=sha256:fbd6748e8ab9b41171bb95c6142faf068f5ef1511935a0aa07025438dd9a9bc1 \ + --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ + --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 + # via + # -c python/requirements_compiled.txt + # aiohttp +zipp==3.19.2 \ + --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c + # via + # -c python/requirements_compiled.txt + # importlib-metadata diff --git a/python/deplocks/ray_img/ray_img_py311.lock b/python/deplocks/ray_img/ray_img_py311.lock new file mode 100644 index 000000000000..c01332ec3f9a --- /dev/null +++ b/python/deplocks/ray_img/ray_img_py311.lock @@ -0,0 +1,2162 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.11 --find-links=.whl/ -c python/requirements_compiled.txt - -o python/deplocks/ray_img/ray_img_py311.lock +--index-url https://pypi.org/simple +--extra-index-url https://download.pytorch.org/whl/cpu +--find-links .whl/ +--find-links https://data.pyg.org/whl/torch-2.3.0+cpu.html + +aiohappyeyeballs==2.6.1 \ + --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ + --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 + # via + # -c python/requirements_compiled.txt + # aiohttp +aiohttp==3.11.16 \ + --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ + --hash=sha256:0902e887b0e1d50424112f200eb9ae3dfed6c0d0a19fc60f633ae5a57c809656 \ + --hash=sha256:09b00dd520d88eac9d1768439a59ab3d145065c91a8fab97f900d1b5f802895e \ + --hash=sha256:0a2f451849e6b39e5c226803dcacfa9c7133e9825dcefd2f4e837a2ec5a3bb98 \ + --hash=sha256:0a950c2eb8ff17361abd8c85987fd6076d9f47d040ebffce67dce4993285e973 \ + --hash=sha256:0ad1fb47da60ae1ddfb316f0ff16d1f3b8e844d1a1e154641928ea0583d486ed \ + --hash=sha256:13ceac2c5cdcc3f64b9015710221ddf81c900c5febc505dbd8f810e770011540 \ + --hash=sha256:14461157d8426bcb40bd94deb0450a6fa16f05129f7da546090cebf8f3123b0f \ + --hash=sha256:16f8a2c9538c14a557b4d309ed4d0a7c60f0253e8ed7b6c9a2859a7582f8b1b8 \ + --hash=sha256:17ae4664031aadfbcb34fd40ffd90976671fa0c0286e6c4113989f78bebab37a \ + --hash=sha256:1ce63ae04719513dd2651202352a2beb9f67f55cb8490c40f056cea3c5c355ce \ + --hash=sha256:23a15727fbfccab973343b6d1b7181bfb0b4aa7ae280f36fd2f90f5476805682 \ + --hash=sha256:2540ddc83cc724b13d1838026f6a5ad178510953302a49e6d647f6e1de82bc34 \ + --hash=sha256:37dcee4906454ae377be5937ab2a66a9a88377b11dd7c072df7a7c142b63c37c \ + --hash=sha256:38bea84ee4fe24ebcc8edeb7b54bf20f06fd53ce4d2cc8b74344c5b9620597fd \ + --hash=sha256:3ab3367bb7f61ad18793fea2ef71f2d181c528c87948638366bf1de26e239183 \ + --hash=sha256:3ad1d59fd7114e6a08c4814983bb498f391c699f3c78712770077518cae63ff7 \ + --hash=sha256:3b4e6db8dc4879015b9955778cfb9881897339c8fab7b3676f8433f849425913 \ + --hash=sha256:3e061b09f6fa42997cf627307f220315e313ece74907d35776ec4373ed718b86 \ + --hash=sha256:42864e70a248f5f6a49fdaf417d9bc62d6e4d8ee9695b24c5916cb4bb666c802 \ + --hash=sha256:493910ceb2764f792db4dc6e8e4b375dae1b08f72e18e8f10f18b34ca17d0979 \ + --hash=sha256:4d0c970c0d602b1017e2067ff3b7dac41c98fef4f7472ec2ea26fd8a4e8c2149 \ + --hash=sha256:54eb3aead72a5c19fad07219acd882c1643a1027fbcdefac9b502c267242f955 \ + --hash=sha256:56a3443aca82abda0e07be2e1ecb76a050714faf2be84256dae291182ba59049 \ + --hash=sha256:576f5ca28d1b3276026f7df3ec841ae460e0fc3aac2a47cbf72eabcfc0f102e1 \ + --hash=sha256:58ede86453a6cf2d6ce40ef0ca15481677a66950e73b0a788917916f7e35a0bb \ + --hash=sha256:61c721764e41af907c9d16b6daa05a458f066015abd35923051be8705108ed17 \ + --hash=sha256:634d96869be6c4dc232fc503e03e40c42d32cfaa51712aee181e922e61d74814 \ + --hash=sha256:696ef00e8a1f0cec5e30640e64eca75d8e777933d1438f4facc9c0cdf288a810 \ + --hash=sha256:69a2cbd61788d26f8f1e626e188044834f37f6ae3f937bd9f08b65fc9d7e514e \ + --hash=sha256:6a792ce34b999fbe04a7a71a90c74f10c57ae4c51f65461a411faa70e154154e \ + --hash=sha256:6ac13b71761e49d5f9e4d05d33683bbafef753e876e8e5a7ef26e937dd766713 \ + --hash=sha256:6fdec0213244c39973674ca2a7f5435bf74369e7d4e104d6c7473c81c9bcc8c4 \ + --hash=sha256:72b1b03fb4655c1960403c131740755ec19c5898c82abd3961c364c2afd59fe7 \ + --hash=sha256:745f1ed5e2c687baefc3c5e7b4304e91bf3e2f32834d07baaee243e349624b24 \ + --hash=sha256:776c8e959a01e5e8321f1dec77964cb6101020a69d5a94cd3d34db6d555e01f7 \ + --hash=sha256:780df0d837276276226a1ff803f8d0fa5f8996c479aeef52eb040179f3156cbd \ + --hash=sha256:78e6e23b954644737e385befa0deb20233e2dfddf95dd11e9db752bdd2a294d3 \ + --hash=sha256:7951decace76a9271a1ef181b04aa77d3cc309a02a51d73826039003210bdc86 \ + --hash=sha256:7ba92a2d9ace559a0a14b03d87f47e021e4fa7681dc6970ebbc7b447c7d4b7cd \ + --hash=sha256:7f6428fee52d2bcf96a8aa7b62095b190ee341ab0e6b1bcf50c615d7966fd45b \ + --hash=sha256:87944bd16b7fe6160607f6a17808abd25f17f61ae1e26c47a491b970fb66d8cb \ + --hash=sha256:87a6e922b2b2401e0b0cf6b976b97f11ec7f136bfed445e16384fbf6fd5e8602 \ + --hash=sha256:8cb0688a8d81c63d716e867d59a9ccc389e97ac7037ebef904c2b89334407180 \ + --hash=sha256:8df6612df74409080575dca38a5237282865408016e65636a76a2eb9348c2567 \ + --hash=sha256:911a6e91d08bb2c72938bc17f0a2d97864c531536b7832abee6429d5296e5b27 \ + --hash=sha256:92b7ee222e2b903e0a4b329a9943d432b3767f2d5029dbe4ca59fb75223bbe2e \ + --hash=sha256:938f756c2b9374bbcc262a37eea521d8a0e6458162f2a9c26329cc87fdf06534 \ + --hash=sha256:9756d9b9d4547e091f99d554fbba0d2a920aab98caa82a8fb3d3d9bee3c9ae85 \ + --hash=sha256:98b88a2bf26965f2015a771381624dd4b0839034b70d406dc74fd8be4cc053e3 \ + --hash=sha256:9b751a6306f330801665ae69270a8a3993654a85569b3469662efaad6cf5cc50 \ + --hash=sha256:a2a450bcce4931b295fc0848f384834c3f9b00edfc2150baafb4488c27953de6 \ + --hash=sha256:a3814760a1a700f3cfd2f977249f1032301d0a12c92aba74605cfa6ce9f78489 \ + --hash=sha256:a5abcbba9f4b463a45c8ca8b7720891200658f6f46894f79517e6cd11f3405ca \ + --hash=sha256:a6db7458ab89c7d80bc1f4e930cc9df6edee2200127cfa6f6e080cf619eddfbd \ + --hash=sha256:ad497f38a0d6c329cb621774788583ee12321863cd4bd9feee1effd60f2ad133 \ + --hash=sha256:ad9509ffb2396483ceacb1eee9134724443ee45b92141105a4645857244aecc8 \ + --hash=sha256:bbcba75fe879ad6fd2e0d6a8d937f34a571f116a0e4db37df8079e738ea95c71 \ + --hash=sha256:c10d85e81d0b9ef87970ecbdbfaeec14a361a7fa947118817fcea8e45335fa46 \ + --hash=sha256:c15b2271c44da77ee9d822552201180779e5e942f3a71fb74e026bf6172ff287 \ + --hash=sha256:ca37057625693d097543bd88076ceebeb248291df9d6ca8481349efc0b05dcd0 \ + --hash=sha256:cc3a145479a76ad0ed646434d09216d33d08eef0d8c9a11f5ae5cdc37caa3540 \ + --hash=sha256:ccf10f16ab498d20e28bc2b5c1306e9c1512f2840f7b6a67000a517a4b37d5ee \ + --hash=sha256:cd464ba806e27ee24a91362ba3621bfc39dbbb8b79f2e1340201615197370f7c \ + --hash=sha256:d007aa39a52d62373bd23428ba4a2546eed0e7643d7bf2e41ddcefd54519842c \ + --hash=sha256:d0666afbe984f6933fe72cd1f1c3560d8c55880a0bdd728ad774006eb4241ecd \ + --hash=sha256:d07502cc14ecd64f52b2a74ebbc106893d9a9717120057ea9ea1fd6568a747e7 \ + --hash=sha256:d489d9778522fbd0f8d6a5c6e48e3514f11be81cb0a5954bdda06f7e1594b321 \ + --hash=sha256:df7db76400bf46ec6a0a73192b14c8295bdb9812053f4fe53f4e789f3ea66bbb \ + --hash=sha256:e3538bc9fe1b902bef51372462e3d7c96fce2b566642512138a480b7adc9d508 \ + --hash=sha256:e87fd812899aa78252866ae03a048e77bd11b80fb4878ce27c23cade239b42b2 \ + --hash=sha256:ecdb8173e6c7aa09eee342ac62e193e6904923bd232e76b4157ac0bfa670609f \ + --hash=sha256:f244b8e541f414664889e2c87cac11a07b918cb4b540c36f7ada7bfa76571ea2 \ + --hash=sha256:f4065145bf69de124accdd17ea5f4dc770da0a6a6e440c53f6e0a8c27b3e635c \ + --hash=sha256:f420bfe862fb357a6d76f2065447ef6f484bc489292ac91e29bc65d2d7a2c84d \ + --hash=sha256:f6ddd90d9fb4b501c97a4458f1c1720e42432c26cb76d28177c5b5ad4e332601 \ + --hash=sha256:fa73e8c2656a3653ae6c307b3f4e878a21f87859a9afab228280ddccd7369d71 \ + --hash=sha256:fadbb8f1d4140825069db3fedbbb843290fd5f5bc0a5dbd7eaf81d91bf1b003b \ + --hash=sha256:fb3d0cc5cdb926090748ea60172fa8a213cec728bd6c54eae18b96040fcd6227 \ + --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ + --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb + # via + # -c python/requirements_compiled.txt + # aiohttp-cors + # ray +aiohttp-cors==0.7.0 \ + --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ + --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d + # via + # -c python/requirements_compiled.txt + # ray +aiosignal==1.3.1 \ + --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ + --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 + # via + # -c python/requirements_compiled.txt + # aiohttp +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled.txt + # kombu +annotated-types==0.6.0 \ + --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ + --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d + # via + # -c python/requirements_compiled.txt + # pydantic +anyio==3.7.1 \ + --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ + --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 + # via + # -c python/requirements_compiled.txt + # starlette + # watchfiles +attrs==25.1.0 \ + --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ + --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a + # via + # -c python/requirements_compiled.txt + # aiohttp + # jsonschema + # referencing +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled.txt + # celery +cachetools==5.5.2 \ + --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ + --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a + # via + # -c python/requirements_compiled.txt + # google-auth +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled.txt + # ray +certifi==2025.1.31 \ + --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ + --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe + # via + # -c python/requirements_compiled.txt + # requests +cffi==1.16.0 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ + --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \ + --hash=sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417 \ + --hash=sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab \ + --hash=sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520 \ + --hash=sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36 \ + --hash=sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743 \ + --hash=sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8 \ + --hash=sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed \ + --hash=sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684 \ + --hash=sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56 \ + --hash=sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324 \ + --hash=sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d \ + --hash=sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235 \ + --hash=sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e \ + --hash=sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088 \ + --hash=sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000 \ + --hash=sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7 \ + --hash=sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e \ + --hash=sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673 \ + --hash=sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c \ + --hash=sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe \ + --hash=sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2 \ + --hash=sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098 \ + --hash=sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8 \ + --hash=sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a \ + --hash=sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0 \ + --hash=sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b \ + --hash=sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896 \ + --hash=sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e \ + --hash=sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9 \ + --hash=sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2 \ + --hash=sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b \ + --hash=sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6 \ + --hash=sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404 \ + --hash=sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f \ + --hash=sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0 \ + --hash=sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4 \ + --hash=sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc \ + --hash=sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936 \ + --hash=sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba \ + --hash=sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872 \ + --hash=sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb \ + --hash=sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614 \ + --hash=sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1 \ + --hash=sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d \ + --hash=sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969 \ + --hash=sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b \ + --hash=sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4 \ + --hash=sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627 \ + --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ + --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 + # via + # -c python/requirements_compiled.txt + # cryptography +charset-normalizer==3.3.2 \ + --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ + --hash=sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087 \ + --hash=sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786 \ + --hash=sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8 \ + --hash=sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09 \ + --hash=sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185 \ + --hash=sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574 \ + --hash=sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e \ + --hash=sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519 \ + --hash=sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898 \ + --hash=sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269 \ + --hash=sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3 \ + --hash=sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f \ + --hash=sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6 \ + --hash=sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8 \ + --hash=sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a \ + --hash=sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73 \ + --hash=sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc \ + --hash=sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714 \ + --hash=sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2 \ + --hash=sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc \ + --hash=sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce \ + --hash=sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d \ + --hash=sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e \ + --hash=sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6 \ + --hash=sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269 \ + --hash=sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96 \ + --hash=sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d \ + --hash=sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a \ + --hash=sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4 \ + --hash=sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77 \ + --hash=sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d \ + --hash=sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0 \ + --hash=sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed \ + --hash=sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068 \ + --hash=sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac \ + --hash=sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25 \ + --hash=sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8 \ + --hash=sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab \ + --hash=sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26 \ + --hash=sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2 \ + --hash=sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db \ + --hash=sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f \ + --hash=sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5 \ + --hash=sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99 \ + --hash=sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c \ + --hash=sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d \ + --hash=sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811 \ + --hash=sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa \ + --hash=sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a \ + --hash=sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03 \ + --hash=sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b \ + --hash=sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04 \ + --hash=sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c \ + --hash=sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001 \ + --hash=sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458 \ + --hash=sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389 \ + --hash=sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99 \ + --hash=sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985 \ + --hash=sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537 \ + --hash=sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238 \ + --hash=sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f \ + --hash=sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d \ + --hash=sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796 \ + --hash=sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a \ + --hash=sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143 \ + --hash=sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8 \ + --hash=sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c \ + --hash=sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5 \ + --hash=sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5 \ + --hash=sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711 \ + --hash=sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4 \ + --hash=sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6 \ + --hash=sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c \ + --hash=sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7 \ + --hash=sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4 \ + --hash=sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b \ + --hash=sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae \ + --hash=sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12 \ + --hash=sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c \ + --hash=sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae \ + --hash=sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8 \ + --hash=sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887 \ + --hash=sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b \ + --hash=sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4 \ + --hash=sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f \ + --hash=sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5 \ + --hash=sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33 \ + --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ + --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 + # via + # -c python/requirements_compiled.txt + # requests +click==8.1.7 \ + --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ + --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de + # via + # -c python/requirements_compiled.txt + # celery + # click-didyoumean + # click-plugins + # click-repl + # ray + # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled.txt + # celery +cloudpickle==2.2.0 \ + --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ + --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 + # via + # -c python/requirements_compiled.txt + # gymnasium +colorful==0.5.5 \ + --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ + --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d + # via + # -c python/requirements_compiled.txt + # ray +cryptography==44.0.3 \ + --hash=sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259 \ + --hash=sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43 \ + --hash=sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645 \ + --hash=sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8 \ + --hash=sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44 \ + --hash=sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d \ + --hash=sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f \ + --hash=sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d \ + --hash=sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54 \ + --hash=sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9 \ + --hash=sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137 \ + --hash=sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f \ + --hash=sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c \ + --hash=sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334 \ + --hash=sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c \ + --hash=sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b \ + --hash=sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2 \ + --hash=sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375 \ + --hash=sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88 \ + --hash=sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5 \ + --hash=sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647 \ + --hash=sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c \ + --hash=sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359 \ + --hash=sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5 \ + --hash=sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d \ + --hash=sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028 \ + --hash=sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01 \ + --hash=sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904 \ + --hash=sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d \ + --hash=sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93 \ + --hash=sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06 \ + --hash=sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff \ + --hash=sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76 \ + --hash=sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff \ + --hash=sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759 \ + --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ + --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 + # via + # -c python/requirements_compiled.txt + # pyopenssl +cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ + --hash=sha256:230f8a8e99c81a653baa0ed00819990c0ed1f0cf0298214786b5e323461dc61a \ + --hash=sha256:2d16eaa2d086e416ac13467d4ff3184b9a081fe76b761ce51d4a46ec1c4bd28a \ + --hash=sha256:432273fd4b61a284f7d705d08b8291403548fd422bcbd945635cc155bc6a923d \ + --hash=sha256:4c51a1062a3c5a826b0425952d229ffe73b1791656a31de95b318117e67a9576 \ + --hash=sha256:4c8e9fdb1f3ffc3151808f8bb8c871518d2783e1be8b53792b698a840543d60c \ + --hash=sha256:51b1d6cb83d82dfa306c9efaeb4d57f24bad3041ebd8716d61072676abbcf67b \ + --hash=sha256:52185a2cf95d3bac2c3fda95c9c8e06a985b5a00cd2e587d3caace337db33899 \ + --hash=sha256:5afb6658faa22f21479ae2c0a07254df31c0aebc36907a64a1f6be4ecc9e96da \ + --hash=sha256:d3dc91ef9c4104652195eea4b282d343ecad653021efe20d1c8dd8dfe8ccfd86 \ + --hash=sha256:d60d1e124592cb82a5f3f45b3e7bee7bda7b72a743029f275e9d6b125f338c60 \ + --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ + --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa + # via + # -c python/requirements_compiled.txt + # ray +distlib==0.3.7 \ + --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ + --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 + # via + # -c python/requirements_compiled.txt + # virtualenv +dm-tree==0.1.8 \ + --hash=sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6 \ + --hash=sha256:09964470f76a5201aff2e8f9b26842976de7889300676f927930f6285e256760 \ + --hash=sha256:0d3172394079a86c3a759179c65f64c48d1a42b89495fcf38976d11cc3bb952c \ + --hash=sha256:0e9620ccf06393eb6b613b5e366469304622d4ea96ae6540b28a33840e6c89cf \ + --hash=sha256:0fcaabbb14e7980377439e7140bd05552739ca5e515ecb3119f234acee4b9430 \ + --hash=sha256:1607ce49aa42f010d1e5e616d92ce899d66835d4d8bea49679582435285515de \ + --hash=sha256:181c35521d480d0365f39300542cb6cd7fd2b77351bb43d7acfda15aef63b317 \ + --hash=sha256:1d7c26e431fc93cc7e0cba867eb000db6a05f6f2b25af11ac4e9dada88fc5bca \ + --hash=sha256:1fe962015b2fe1282892b28ebe962faed53c7f98d942da9a4625cbf27baef913 \ + --hash=sha256:250b692fb75f45f02e2f58fbef9ab338904ef334b90557565621fa251df267cf \ + --hash=sha256:2869228d9c619074de501a3c10dc7f07c75422f8fab36ecdcb859b6f1b1ec3ef \ + --hash=sha256:28c52cbf4f8b3dbd0beaedf44f69fa85eec5e9dede612e08035e06ada6ec9426 \ + --hash=sha256:2f7915660f59c09068e428613c480150180df1060561fd0d1470684ae7007bd1 \ + --hash=sha256:343a4a4ebaa127451ff971254a4be4084eb4bdc0b2513c32b46f6f728fd03f9e \ + --hash=sha256:35cc164a79336bfcfafb47e5f297898359123bbd3330c1967f0c4994f9cf9f60 \ + --hash=sha256:378cc8ad93c5fe3590f405a309980721f021c790ca1bdf9b15bb1d59daec57f5 \ + --hash=sha256:39070ba268c0491af9fe7a58644d99e8b4f2cde6e5884ba3380bddc84ed43d5f \ + --hash=sha256:435227cf3c5dc63f4de054cf3d00183790bd9ead4c3623138c74dde7f67f521b \ + --hash=sha256:5483dca4d7eb1a0d65fe86d3b6a53ae717face83c1f17e0887b1a4a64ae5c410 \ + --hash=sha256:694c3654cfd2a81552c08ec66bb5c4a3d48fa292b9a181880fb081c36c5b9134 \ + --hash=sha256:75c5d528bb992981c20793b6b453e91560784215dffb8a5440ba999753c14ceb \ + --hash=sha256:803bfc53b4659f447ac694dbd04235f94a73ef7c1fd1e0df7c84ac41e0bc963b \ + --hash=sha256:81fce77f22a302d7a5968aebdf4efafef4def7ce96528719a354e6990dcd49c7 \ + --hash=sha256:83b7764de0d855338abefc6e3ee9fe40d301668310aa3baea3f778ff051f4393 \ + --hash=sha256:8c60a7eadab64c2278861f56bca320b2720f163dca9d7558103c3b77f2416571 \ + --hash=sha256:8ed3564abed97c806db122c2d3e1a2b64c74a63debe9903aad795167cc301368 \ + --hash=sha256:94d3f0826311f45ee19b75f5b48c99466e4218a0489e81c0f0167bda50cacf22 \ + --hash=sha256:96a548a406a6fb15fe58f6a30a57ff2f2aafbf25f05afab00c8f5e5977b6c715 \ + --hash=sha256:a5d819c38c03f0bb5b3b3703c60e4b170355a0fc6b5819325bf3d4ceb3ae7e80 \ + --hash=sha256:ad16ceba90a56ec47cf45b21856d14962ac314787975ef786efb5e6e9ca75ec7 \ + --hash=sha256:af4b3d372f2477dcd89a6e717e4a575ca35ccc20cc4454a8a4b6f8838a00672d \ + --hash=sha256:b095ba4f8ca1ba19350fd53cf1f8f3eb0bd406aa28af64a6dfc86707b32a810a \ + --hash=sha256:b9bd9b9ccb59409d33d51d84b7668010c04c2af7d4a371632874c1ca356cff3d \ + --hash=sha256:b9f89a454e98806b44fe9d40ec9eee61f848388f7e79ac2371a55679bd5a3ac6 \ + --hash=sha256:bb2d109f42190225112da899b9f3d46d0d5f26aef501c61e43529fe9322530b5 \ + --hash=sha256:c0a94aba18a35457a1b5cd716fd7b46c5dafdc4cf7869b4bae665b91c4682a8e \ + --hash=sha256:c5c8c12e3fda754ef6af94161bacdaeda816d941995fac415d6855c6c386af68 \ + --hash=sha256:d1612fcaecd79023dbc6a6ae48d51a80beb5c385d6f3f6d71688e57bc8d07de8 \ + --hash=sha256:d16e1f2a073604cfcc09f7131ae8d534674f43c3aef4c25742eae295bc60d04f \ + --hash=sha256:d20f2faa3672b52e5013f4077117bfb99c4cfc0b445d3bde1584c34032b57436 \ + --hash=sha256:d40fa4106ca6edc66760246a08f500ec0c85ef55c762fb4a363f6ee739ba02ee \ + --hash=sha256:de287fabc464b8734be251e46e06aa9aa1001f34198da2b6ce07bd197172b9cb \ + --hash=sha256:e4d714371bb08839e4e5e29024fc95832d9affe129825ef38836b143028bd144 \ + --hash=sha256:ea9e59e0451e7d29aece402d9f908f2e2a80922bcde2ebfd5dcb07750fcbfee8 \ + --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ + --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d + # via + # -c python/requirements_compiled.txt + # ray +farama-notifications==0.0.4 \ + --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ + --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae + # via + # -c python/requirements_compiled.txt + # gymnasium +fastapi==0.115.12 \ + --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ + --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d + # via + # -c python/requirements_compiled.txt + # ray +fastrlock==0.8.2 ; sys_platform != 'darwin' \ + --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ + --hash=sha256:07ed3c7b3867c05a3d6be4ced200c7767000f3431b9be6da66972822dd86e8be \ + --hash=sha256:08315bde19d0c2e6b06593d5a418be3dc8f9b1ee721afa96867b9853fceb45cf \ + --hash=sha256:11bbbbc526363955aeddb9eec4cee2a0012322b7b2f15b54f44454fcf4fd398a \ + --hash=sha256:17734e2e5af4c07ddb0fb10bd484e062c22de3be6b67940b9cc6ec2f18fa61ba \ + --hash=sha256:1b15430b93d7eb3d56f6ff690d2ebecb79ed0e58248427717eba150a508d1cd7 \ + --hash=sha256:1fed2f4797ad68e9982038423018cf08bec5f4ce9fed63a94a790773ed6a795c \ + --hash=sha256:2074548a335fcf7d19ebb18d9208da9e33b06f745754466a7e001d2b1c58dd19 \ + --hash=sha256:2587cedbb36c7988e707d83f0f1175c1f882f362b5ebbee25d70218ea33d220d \ + --hash=sha256:25945f962c7bd808415cfde3da624d4399d4ea71ed8918538375f16bceb79e1c \ + --hash=sha256:27786c62a400e282756ae1b090bcd7cfa35f28270cff65a9e7b27a5327a32561 \ + --hash=sha256:2c1719ddc8218b01e82fb2e82e8451bd65076cb96d7bef4477194bbb4305a968 \ + --hash=sha256:2d5595903444c854b99c42122b87edfe8a37cd698a4eae32f4fd1d2a7b6c115d \ + --hash=sha256:30bdbe4662992348132d03996700e1cf910d141d629179b967b146a22942264e \ + --hash=sha256:31a27a2edf482df72b91fe6c6438314d2c65290aa7becc55589d156c9b91f0da \ + --hash=sha256:320fd55bafee3eb069cfb5d6491f811a912758387ef2193840e2663e80e16f48 \ + --hash=sha256:33145acbad8317584cd64588131c7e1e286beef6280c0009b4544c91fce171d2 \ + --hash=sha256:43a241655e83e4603a152192cf022d5ca348c2f4e56dfb02e5c9c4c1a32f9cdb \ + --hash=sha256:4d63b6596368dab9e0cc66bf047e7182a56f33b34db141816a4f21f5bf958228 \ + --hash=sha256:4fb04442b6d1e2b36c774919c6bcbe3339c61b337261d4bd57e27932589095af \ + --hash=sha256:4fb2e77ff04bc4beb71d63c8e064f052ce5a6ea1e001d528d4d7f4b37d736f2e \ + --hash=sha256:5460c5ee6ced6d61ec8cd2324ebbe793a4960c4ffa2131ffff480e3b61c99ec5 \ + --hash=sha256:59344c1d46b7dec97d3f22f1cc930fafe8980b3c5bc9c9765c56738a5f1559e4 \ + --hash=sha256:5dfb78dd600a12f23fc0c3ec58f81336229fdc74501ecf378d1ce5b3f2f313ea \ + --hash=sha256:643e1e65b4f5b284427e61a894d876d10459820e93aa1e724dfb415117be24e0 \ + --hash=sha256:644ec9215cf9c4df8028d8511379a15d9c1af3e16d80e47f1b6fdc6ba118356a \ + --hash=sha256:66f2662c640bb71a1016a031eea6eef9d25c2bcdf7ffd1d1ddc5a58f9a1ced04 \ + --hash=sha256:685e656048b59d8dfde8c601f188ad53a4d719eb97080cafc8696cda6d75865e \ + --hash=sha256:7269bb3fc15587b0c191eecd95831d771a7d80f0c48929e560806b038ff3066c \ + --hash=sha256:73426f5eb2ecc10626c67cf86bd0af9e00d53e80e5c67d5ce8e18376d6abfa09 \ + --hash=sha256:75c07726c8b1a52147fd7987d6baaa318c5dced1416c3f25593e40f56e10755b \ + --hash=sha256:790fc19bccbd39426060047e53629f171a44745613bf360a045e9f9c8c4a2cea \ + --hash=sha256:7a2ccaf88ac0db153e84305d1ef0aa138cea82c6a88309066f6eaa3bc98636cd \ + --hash=sha256:87f4e01b042c84e6090dbc4fbe3415ddd69f6bc0130382323f9d3f1b8dd71b46 \ + --hash=sha256:88f079335e9da631efa64486c8207564a7bcd0c00526bb9e842e9d5b7e50a6cc \ + --hash=sha256:8c1c91a68926421f5ccbc82c85f83bd3ba593b121a46a1b9a554b3f0dd67a4bf \ + --hash=sha256:9121a894d74e65557e47e777060a495ab85f4b903e80dd73a3c940ba042920d7 \ + --hash=sha256:94e348c72a1fd1f8191f25ea056448e4f5a87b8fbf005b39d290dcb0581a48cd \ + --hash=sha256:98195866d3a9949915935d40a88e4f1c166e82e378f622c88025f2938624a90a \ + --hash=sha256:99dd6652bd6f730beadf74ef769d38c6bbd8ee6d1c15c8d138ea680b0594387f \ + --hash=sha256:9af691a9861027181d4de07ed74f0aee12a9650ac60d0a07f4320bff84b5d95f \ + --hash=sha256:a3b8b5d2935403f1b4b25ae324560e94b59593a38c0d2e7b6c9872126a9622ed \ + --hash=sha256:a3dcc876050b8f5cbc0ee84ef1e7f0c1dfe7c148f10098828bc4403683c33f10 \ + --hash=sha256:a74f5a92fa6e51c4f3c69b29c4662088b97be12f40652a21109605a175c81824 \ + --hash=sha256:ab91b0c36e95d42e1041a4907e3eefd06c482d53af3c7a77be7e214cc7cd4a63 \ + --hash=sha256:ad1bc61c7f6b0e58106aaab034916b6cb041757f708b07fbcdd9d6e1ac629225 \ + --hash=sha256:adcb9e77aa132cc6c9de2ffe7cf880a20aa8cdba21d367d1da1a412f57bddd5d \ + --hash=sha256:b22ea9bf5f9fad2b0077e944a7813f91593a4f61adf8faf734a70aed3f2b3a40 \ + --hash=sha256:b2a1c354f13f22b737621d914f3b4a8434ae69d3027a775e94b3e671756112f9 \ + --hash=sha256:b32fdf874868326351a75b1e4c02f97e802147119ae44c52d3d9da193ec34f5b \ + --hash=sha256:b3853ed4ce522598dc886160a7bab432a093051af85891fa2f5577c1dcac8ed6 \ + --hash=sha256:b443e73a4dfc7b6e0800ea4c13567b9694358e86f53bb2612a51c9e727cac67b \ + --hash=sha256:b4c9083ea89ab236b06e9ef2263971db3b4b507195fc7d5eecab95828dcae325 \ + --hash=sha256:b8ca0fe21458457077e4cb2d81e1ebdb146a00b3e9e2db6180a773f7ea905032 \ + --hash=sha256:c393af77c659a38bffbca215c0bcc8629ba4299568308dd7e4ff65d62cabed39 \ + --hash=sha256:c6bffa978793bea5e1b00e677062e53a62255439339591b70e209fa1552d5ee0 \ + --hash=sha256:ccf39ad5702e33e4d335b48ef9d56e21619b529b7f7471b5211419f380329b62 \ + --hash=sha256:cf81e0278b645004388873e0a1f9e3bc4c9ab8c18e377b14ed1a544be4b18c9a \ + --hash=sha256:d34546ad2e4a480b94b6797bcc5a322b3c705c4c74c3e4e545c4a3841c1b2d59 \ + --hash=sha256:d47713ffe6d4a627fbf078be9836a95ac106b4a0543e3841572c91e292a5d885 \ + --hash=sha256:d918dfe473291e8bfd8e13223ea5cb9b317bd9f50c280923776c377f7c64b428 \ + --hash=sha256:dbdce852e6bb66e1b8c36679d482971d69d93acf1785657522e51b7de30c3356 \ + --hash=sha256:dcc1bf0ac8a194313cf6e645e300a8a379674ceed8e0b1e910a2de3e3c28989e \ + --hash=sha256:dd961a32a7182c3891cdebca417fda67496d5d5de6ae636962254d22723bdf52 \ + --hash=sha256:ddf5d247f686aec853ddcc9a1234bfcc6f57b0a0670d2ad82fc25d8ae7e6a15f \ + --hash=sha256:e27c3cd27fbd25e5223c5c992b300cd4ee8f0a75c6f222ce65838138d853712c \ + --hash=sha256:e380ec4e6d8b26e389713995a43cb7fe56baea2d25fe073d4998c4821a026211 \ + --hash=sha256:e4bbde174a0aff5f6eeba75cf8c4c5d2a316316bc21f03a0bddca0fc3659a6f3 \ + --hash=sha256:e8b49b5743ede51e0bcf6805741f39f5e0e0fd6a172ba460cb39e3097ba803bb \ + --hash=sha256:e9904b5b37c3e5bb4a245c56bc4b7e497da57ffb8528f4fc39af9dcb168ee2e1 \ + --hash=sha256:ea96503b918fceaf40443182742b8964d47b65c5ebdea532893cb9479620000c \ + --hash=sha256:eb31fe390f03f7ae886dcc374f1099ec88526631a4cb891d399b68181f154ff0 \ + --hash=sha256:ebb32d776b61acd49f859a1d16b9e3d84e7b46d0d92aebd58acd54dc38e96664 \ + --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ + --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e + # via + # -c python/requirements_compiled.txt + # cupy-cuda12x +filelock==3.17.0 \ + --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ + --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e + # via + # -c python/requirements_compiled.txt + # ray + # virtualenv +frozenlist==1.4.1 \ + --hash=sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7 \ + --hash=sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98 \ + --hash=sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad \ + --hash=sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5 \ + --hash=sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae \ + --hash=sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e \ + --hash=sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a \ + --hash=sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701 \ + --hash=sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d \ + --hash=sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6 \ + --hash=sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6 \ + --hash=sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106 \ + --hash=sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75 \ + --hash=sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868 \ + --hash=sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a \ + --hash=sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0 \ + --hash=sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1 \ + --hash=sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826 \ + --hash=sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec \ + --hash=sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6 \ + --hash=sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950 \ + --hash=sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19 \ + --hash=sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0 \ + --hash=sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8 \ + --hash=sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a \ + --hash=sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09 \ + --hash=sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86 \ + --hash=sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c \ + --hash=sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5 \ + --hash=sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b \ + --hash=sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b \ + --hash=sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d \ + --hash=sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0 \ + --hash=sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea \ + --hash=sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776 \ + --hash=sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a \ + --hash=sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897 \ + --hash=sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7 \ + --hash=sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09 \ + --hash=sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9 \ + --hash=sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe \ + --hash=sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd \ + --hash=sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742 \ + --hash=sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09 \ + --hash=sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0 \ + --hash=sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932 \ + --hash=sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1 \ + --hash=sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a \ + --hash=sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49 \ + --hash=sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d \ + --hash=sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7 \ + --hash=sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480 \ + --hash=sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89 \ + --hash=sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e \ + --hash=sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b \ + --hash=sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82 \ + --hash=sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb \ + --hash=sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068 \ + --hash=sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8 \ + --hash=sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b \ + --hash=sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb \ + --hash=sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2 \ + --hash=sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11 \ + --hash=sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b \ + --hash=sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc \ + --hash=sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0 \ + --hash=sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497 \ + --hash=sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17 \ + --hash=sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0 \ + --hash=sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2 \ + --hash=sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439 \ + --hash=sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5 \ + --hash=sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac \ + --hash=sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825 \ + --hash=sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887 \ + --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ + --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 + # via + # -c python/requirements_compiled.txt + # aiohttp + # aiosignal +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 + # via + # -c python/requirements_compiled.txt + # ray +google-api-core==2.24.2 \ + --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ + --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 + # via + # -c python/requirements_compiled.txt + # opencensus +google-auth==2.23.4 \ + --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ + --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 + # via + # -c python/requirements_compiled.txt + # google-api-core +googleapis-common-protos==1.61.0 \ + --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ + --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b + # via + # -c python/requirements_compiled.txt + # google-api-core +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e + # via + # -c python/requirements_compiled.txt + # ray +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a + # via + # -c python/requirements_compiled.txt + # ray +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # -c python/requirements_compiled.txt + # uvicorn +httptools==0.6.4 \ + --hash=sha256:0614154d5454c21b6410fdf5262b4a3ddb0f53f1e1721cfd59d55f32138c578a \ + --hash=sha256:0e563e54979e97b6d13f1bbc05a96109923e76b901f786a5eae36e99c01237bd \ + --hash=sha256:16e603a3bff50db08cd578d54f07032ca1631450ceb972c2f834c2b860c28ea2 \ + --hash=sha256:288cd628406cc53f9a541cfaf06041b4c71d751856bab45e3702191f931ccd17 \ + --hash=sha256:28908df1b9bb8187393d5b5db91435ccc9c8e891657f9cbb42a2541b44c82fc8 \ + --hash=sha256:322d20ea9cdd1fa98bd6a74b77e2ec5b818abdc3d36695ab402a0de8ef2865a3 \ + --hash=sha256:342dd6946aa6bda4b8f18c734576106b8a31f2fe31492881a9a160ec84ff4bd5 \ + --hash=sha256:345c288418f0944a6fe67be8e6afa9262b18c7626c3ef3c28adc5eabc06a68da \ + --hash=sha256:3c73ce323711a6ffb0d247dcd5a550b8babf0f757e86a52558fe5b86d6fefcc0 \ + --hash=sha256:40a5ec98d3f49904b9fe36827dcf1aadfef3b89e2bd05b0e35e94f97c2b14721 \ + --hash=sha256:40b0f7fe4fd38e6a507bdb751db0379df1e99120c65fbdc8ee6c1d044897a636 \ + --hash=sha256:40dc6a8e399e15ea525305a2ddba998b0af5caa2566bcd79dcbe8948181eeaff \ + --hash=sha256:4b36913ba52008249223042dca46e69967985fb4051951f94357ea681e1f5dc0 \ + --hash=sha256:4d87b29bd4486c0093fc64dea80231f7c7f7eb4dc70ae394d70a495ab8436071 \ + --hash=sha256:4e93eee4add6493b59a5c514da98c939b244fce4a0d8879cd3f466562f4b7d5c \ + --hash=sha256:59e724f8b332319e2875efd360e61ac07f33b492889284a3e05e6d13746876f4 \ + --hash=sha256:69422b7f458c5af875922cdb5bd586cc1f1033295aa9ff63ee196a87519ac8e1 \ + --hash=sha256:703c346571fa50d2e9856a37d7cd9435a25e7fd15e236c397bf224afaa355fe9 \ + --hash=sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44 \ + --hash=sha256:856f4bc0478ae143bad54a4242fccb1f3f86a6e1be5548fecfd4102061b3a083 \ + --hash=sha256:85797e37e8eeaa5439d33e556662cc370e474445d5fab24dcadc65a8ffb04003 \ + --hash=sha256:90d96a385fa941283ebd231464045187a31ad932ebfa541be8edf5b3c2328959 \ + --hash=sha256:94978a49b8f4569ad607cd4946b759d90b285e39c0d4640c6b36ca7a3ddf2efc \ + --hash=sha256:aafe0f1918ed07b67c1e838f950b1c1fabc683030477e60b335649b8020e1076 \ + --hash=sha256:ab9ba8dcf59de5181f6be44a77458e45a578fc99c31510b8c65b7d5acc3cf490 \ + --hash=sha256:ade273d7e767d5fae13fa637f4d53b6e961fb7fd93c7797562663f0171c26660 \ + --hash=sha256:b799de31416ecc589ad79dd85a0b2657a8fe39327944998dea368c1d4c9e55e6 \ + --hash=sha256:c26f313951f6e26147833fc923f78f95604bbec812a43e5ee37f26dc9e5a686c \ + --hash=sha256:ca80b7485c76f768a3bc83ea58373f8db7b015551117375e4918e2aa77ea9b50 \ + --hash=sha256:d1ffd262a73d7c28424252381a5b854c19d9de5f56f075445d33919a637e3547 \ + --hash=sha256:d3f0d369e7ffbe59c4b6116a44d6a8eb4783aae027f2c0b366cf0aa964185dba \ + --hash=sha256:d54efd20338ac52ba31e7da78e4a72570cf729fac82bc31ff9199bedf1dc7440 \ + --hash=sha256:dacdd3d10ea1b4ca9df97a0a303cbacafc04b5cd375fa98732678151643d4988 \ + --hash=sha256:db353d22843cf1028f43c3651581e4bb49374d85692a85f95f7b9a130e1b2cab \ + --hash=sha256:db78cb9ca56b59b016e64b6031eda5653be0589dba2b1b43453f6e8b405a0970 \ + --hash=sha256:deee0e3343f98ee8047e9f4c5bc7cedbf69f5734454a94c38ee829fb2d5fa3c1 \ + --hash=sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2 \ + --hash=sha256:df959752a0c2748a65ab5387d08287abf6779ae9165916fe053e68ae1fbdc47f \ + --hash=sha256:ec4f178901fa1834d4a060320d2f3abc5c9e39766953d038f1458cb885f47e81 \ + --hash=sha256:f47f8ed67cc0ff862b84a1189831d1d33c963fb3ce1ee0c65d3b0cbe7b711069 \ + --hash=sha256:f8787367fbdfccae38e35abf7641dafc5310310a5987b689f4c32cc8cc3ee975 \ + --hash=sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f \ + --hash=sha256:fc411e1c0a7dcd2f902c7c48cf079947a7e65b5485dea9decb82b9105ca71a43 + # via uvicorn +idna==3.7 \ + --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ + --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 + # via + # -c python/requirements_compiled.txt + # anyio + # requests + # yarl +importlib-metadata==6.11.0 \ + --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ + --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b + # via + # -c python/requirements_compiled.txt + # opentelemetry-api +jinja2==3.1.6 ; sys_platform != 'win32' \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via + # -c python/requirements_compiled.txt + # memray +jsonschema==4.23.0 \ + --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ + --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 + # via + # -c python/requirements_compiled.txt + # ray +jsonschema-specifications==2024.10.1 \ + --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ + --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf + # via + # -c python/requirements_compiled.txt + # jsonschema +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled.txt + # celery +lz4==4.3.3 \ + --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ + --hash=sha256:054b4631a355606e99a42396f5db4d22046a3397ffc3269a348ec41eaebd69d2 \ + --hash=sha256:0a136e44a16fc98b1abc404fbabf7f1fada2bdab6a7e970974fb81cf55b636d0 \ + --hash=sha256:0e9c410b11a31dbdc94c05ac3c480cb4b222460faf9231f12538d0074e56c563 \ + --hash=sha256:222a7e35137d7539c9c33bb53fcbb26510c5748779364014235afc62b0ec797f \ + --hash=sha256:24b3206de56b7a537eda3a8123c644a2b7bf111f0af53bc14bed90ce5562d1aa \ + --hash=sha256:2b901c7784caac9a1ded4555258207d9e9697e746cc8532129f150ffe1f6ba0d \ + --hash=sha256:2f7b1839f795315e480fb87d9bc60b186a98e3e5d17203c6e757611ef7dcef61 \ + --hash=sha256:30e8c20b8857adef7be045c65f47ab1e2c4fabba86a9fa9a997d7674a31ea6b6 \ + --hash=sha256:31ea4be9d0059c00b2572d700bf2c1bc82f241f2c3282034a759c9a4d6ca4dc2 \ + --hash=sha256:337cb94488a1b060ef1685187d6ad4ba8bc61d26d631d7ba909ee984ea736be1 \ + --hash=sha256:33c9a6fd20767ccaf70649982f8f3eeb0884035c150c0b818ea660152cf3c809 \ + --hash=sha256:363ab65bf31338eb364062a15f302fc0fab0a49426051429866d71c793c23394 \ + --hash=sha256:43cf03059c0f941b772c8aeb42a0813d68d7081c009542301637e5782f8a33e2 \ + --hash=sha256:56f4fe9c6327adb97406f27a66420b22ce02d71a5c365c48d6b656b4aaeb7775 \ + --hash=sha256:5d35533bf2cee56f38ced91f766cd0038b6abf46f438a80d50c52750088be93f \ + --hash=sha256:6756212507405f270b66b3ff7f564618de0606395c0fe10a7ae2ffcbbe0b1fba \ + --hash=sha256:6cdc60e21ec70266947a48839b437d46025076eb4b12c76bd47f8e5eb8a75dcc \ + --hash=sha256:abc197e4aca8b63f5ae200af03eb95fb4b5055a8f990079b5bdf042f568469dd \ + --hash=sha256:b14d948e6dce389f9a7afc666d60dd1e35fa2138a8ec5306d30cd2e30d36b40c \ + --hash=sha256:b47839b53956e2737229d70714f1d75f33e8ac26e52c267f0197b3189ca6de24 \ + --hash=sha256:b6d9ec061b9eca86e4dcc003d93334b95d53909afd5a32c6e4f222157b50c071 \ + --hash=sha256:b891880c187e96339474af2a3b2bfb11a8e4732ff5034be919aa9029484cd201 \ + --hash=sha256:bca8fccc15e3add173da91be8f34121578dc777711ffd98d399be35487c934bf \ + --hash=sha256:c81703b12475da73a5d66618856d04b1307e43428a7e59d98cfe5a5d608a74c6 \ + --hash=sha256:d2507ee9c99dbddd191c86f0e0c8b724c76d26b0602db9ea23232304382e1f21 \ + --hash=sha256:e36cd7b9d4d920d3bfc2369840da506fa68258f7bb176b8743189793c055e43d \ + --hash=sha256:e7d84b479ddf39fe3ea05387f10b779155fc0990125f4fb35d636114e1c63a2e \ + --hash=sha256:eac9af361e0d98335a02ff12fb56caeb7ea1196cf1a49dbf6f17828a131da807 \ + --hash=sha256:edfd858985c23523f4e5a7526ca6ee65ff930207a7ec8a8f57a01eae506aaee7 \ + --hash=sha256:ee9ff50557a942d187ec85462bb0960207e7ec5b19b3b48949263993771c6205 \ + --hash=sha256:f0e822cd7644995d9ba248cb4b67859701748a93e2ab7fc9bc18c599a52e4604 \ + --hash=sha256:f180904f33bdd1e92967923a43c22899e303906d19b2cf8bb547db6653ea6e7d \ + --hash=sha256:f1d18718f9d78182c6b60f568c9a9cec8a7204d7cb6fad4e511a2ef279e4cb05 \ + --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ + --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 + # via + # -c python/requirements_compiled.txt + # ray +markdown-it-py==2.2.0 ; sys_platform != 'win32' \ + --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ + --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 + # via + # -c python/requirements_compiled.txt + # rich +markupsafe==2.1.3 ; sys_platform != 'win32' \ + --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ + --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ + --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ + --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ + --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ + --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ + --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ + --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ + --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ + --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ + --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ + --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ + --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ + --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ + --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ + --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ + --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ + --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ + --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ + --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ + --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ + --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ + --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ + --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ + --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 + # via + # -c python/requirements_compiled.txt + # jinja2 +mdurl==0.1.2 ; sys_platform != 'win32' \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via + # -c python/requirements_compiled.txt + # markdown-it-py +memray==1.10.0 ; sys_platform != 'win32' \ + --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ + --hash=sha256:22f2a47871c172a0539bd72737bb6b294fc10c510464066b825d90fcd3bb4916 \ + --hash=sha256:23e8c402625cfb32d0e9edb5ec0945f3e5e54bc6b0c5699f6284302082b80bd4 \ + --hash=sha256:2ce59ef485db3634de98b3a026d2450fc0a875e3a58a9ea85f7a89098841defe \ + --hash=sha256:322ed0b69014a0969b777768d461a785203f81f9864386b666b5b26645d9c294 \ + --hash=sha256:38322e052b882790993412f1840517a51818aa55c47037f69915b2007f2c4cee \ + --hash=sha256:38393c86ce6d0a08e6ec0eb1401d49803b7c0c950c2565386751cdc81568cba8 \ + --hash=sha256:391aac6c9f744528d3186bc82d708a1acc83525778f804045d7c96f860f8ec98 \ + --hash=sha256:3a8bb7fbd8303c4f0017ba7faef6b88f904cda2931ed667cbf3b98f024b3bc44 \ + --hash=sha256:3c401c57f49c4c5f1fecaee1e746f537cdc6680da05fb963dc143bd08ee109bf \ + --hash=sha256:4eba29179772b4a2e440a065b320b03bc2e73fe2648bdf7936aa3b9a086fab4a \ + --hash=sha256:53a8f66af18b1f3bcf5c9f3c95ae4134dd675903a38f9d0e6341b7bca01b63d0 \ + --hash=sha256:566602b2143e06b3d592901d98c52ce4599e71aa2555146eeb5cec03506f9498 \ + --hash=sha256:663d463e89a64bae4a6b2f8c837d11a3d094834442d536a4165e1d31899a3500 \ + --hash=sha256:68bd8df023c8a32f44c11d997e5c536837e27c0955daf557d3a377edd55a1dd3 \ + --hash=sha256:6937d7ef67d18ccc01c3250cdf3b4ef1445b859ee8756f09e3d11bd3ff0c7d67 \ + --hash=sha256:6b311e91203be71e1a0ce5e4f978137765bcb1045f3bf5646129c83c5b96ab3c \ + --hash=sha256:6fd13ef666c7fced9768d1cfabf71dc6dfa6724935a8dff463495ac2dc5e13a4 \ + --hash=sha256:8196c684f1be8fe423e5cdd2356d4255a2cb482a1f3e89612b70d2a2862cf5bb \ + --hash=sha256:843a688877691746f9d1835cfa8a65139948471bdd78720435808d20bc30a1cc \ + --hash=sha256:85c32d6613d81b075f740e398c4d653e0803cd48e82c33dcd584c109d6782666 \ + --hash=sha256:898acd60f57a10dc5aaf1fd64aa2f821f0420114f3f60c3058083788603f173a \ + --hash=sha256:8d56f37a34125684746c13d24bd7a3fb17549b0bb355eb50969eb11e05e3ba62 \ + --hash=sha256:92c372cb262eddd23049f945ca9527f0e4cc7c40a070aade1802d066f680885b \ + --hash=sha256:95e563d9c976e429ad597ad2720d95cebbe8bac891a3082465439143e2740772 \ + --hash=sha256:9627184c926252c8f719c301f1fefe970f0d033c643a6448b93fed2889d1ea94 \ + --hash=sha256:a9e985fb7646b0475c303919d19211d2aa54e5a9e2cd2a102472299be5dbebd3 \ + --hash=sha256:b681519357d94f5f0857fbc6029e7c44d3f41436109e955a14fd312d8317bc35 \ + --hash=sha256:b75040f28e8678d0e9c4907d55c95cf26db8ef5adc9941a228f1b280a9efd9c0 \ + --hash=sha256:c3a14960838d89a91747885897d34134afb65883cc3b0ed7ff30fe1af00f9fe6 \ + --hash=sha256:c7aeb47174c42e99740a8e2b3b6fe0932c95d987258d48a746974ead19176c26 \ + --hash=sha256:ce22a887a585ef5020896de89ffc793e531b65ccc81fbafcc7886010c2c562b3 \ + --hash=sha256:cf6d683c4f8d25c6ad06ae18715f218983c5eb86803953615e902d632fdf6ec1 \ + --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ + --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 + # via + # -c python/requirements_compiled.txt + # ray +msgpack==1.0.7 \ + --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ + --hash=sha256:0bfdd914e55e0d2c9e1526de210f6fe8ffe9705f2b1dfcc4aecc92a4cb4b533d \ + --hash=sha256:1dc93e8e4653bdb5910aed79f11e165c85732067614f180f70534f056da97db3 \ + --hash=sha256:1e2d69948e4132813b8d1131f29f9101bc2c915f26089a6d632001a5c1349672 \ + --hash=sha256:235a31ec7db685f5c82233bddf9858748b89b8119bf4538d514536c485c15fe0 \ + --hash=sha256:27dcd6f46a21c18fa5e5deed92a43d4554e3df8d8ca5a47bf0615d6a5f39dbc9 \ + --hash=sha256:28efb066cde83c479dfe5a48141a53bc7e5f13f785b92ddde336c716663039ee \ + --hash=sha256:3476fae43db72bd11f29a5147ae2f3cb22e2f1a91d575ef130d2bf49afd21c46 \ + --hash=sha256:36e17c4592231a7dbd2ed09027823ab295d2791b3b1efb2aee874b10548b7524 \ + --hash=sha256:384d779f0d6f1b110eae74cb0659d9aa6ff35aaf547b3955abf2ab4c901c4819 \ + --hash=sha256:38949d30b11ae5f95c3c91917ee7a6b239f5ec276f271f28638dec9156f82cfc \ + --hash=sha256:3967e4ad1aa9da62fd53e346ed17d7b2e922cba5ab93bdd46febcac39be636fc \ + --hash=sha256:3e7bf4442b310ff154b7bb9d81eb2c016b7d597e364f97d72b1acc3817a0fdc1 \ + --hash=sha256:3f0c8c6dfa6605ab8ff0611995ee30d4f9fcff89966cf562733b4008a3d60d82 \ + --hash=sha256:484ae3240666ad34cfa31eea7b8c6cd2f1fdaae21d73ce2974211df099a95d81 \ + --hash=sha256:4a7b4f35de6a304b5533c238bee86b670b75b03d31b7797929caa7a624b5dda6 \ + --hash=sha256:4cb14ce54d9b857be9591ac364cb08dc2d6a5c4318c1182cb1d02274029d590d \ + --hash=sha256:4e71bc4416de195d6e9b4ee93ad3f2f6b2ce11d042b4d7a7ee00bbe0358bd0c2 \ + --hash=sha256:52700dc63a4676669b341ba33520f4d6e43d3ca58d422e22ba66d1736b0a6e4c \ + --hash=sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87 \ + --hash=sha256:576eb384292b139821c41995523654ad82d1916da6a60cff129c715a6223ea84 \ + --hash=sha256:5b0bf0effb196ed76b7ad883848143427a73c355ae8e569fa538365064188b8e \ + --hash=sha256:5b6ccc0c85916998d788b295765ea0e9cb9aac7e4a8ed71d12e7d8ac31c23c95 \ + --hash=sha256:5ed82f5a7af3697b1c4786053736f24a0efd0a1b8a130d4c7bfee4b9ded0f08f \ + --hash=sha256:6d4c80667de2e36970ebf74f42d1088cc9ee7ef5f4e8c35eee1b40eafd33ca5b \ + --hash=sha256:730076207cb816138cf1af7f7237b208340a2c5e749707457d70705715c93b93 \ + --hash=sha256:7687e22a31e976a0e7fc99c2f4d11ca45eff652a81eb8c8085e9609298916dcf \ + --hash=sha256:822ea70dc4018c7e6223f13affd1c5c30c0f5c12ac1f96cd8e9949acddb48a61 \ + --hash=sha256:84b0daf226913133f899ea9b30618722d45feffa67e4fe867b0b5ae83a34060c \ + --hash=sha256:85765fdf4b27eb5086f05ac0491090fc76f4f2b28e09d9350c31aac25a5aaff8 \ + --hash=sha256:8dd178c4c80706546702c59529ffc005681bd6dc2ea234c450661b205445a34d \ + --hash=sha256:8f5b234f567cf76ee489502ceb7165c2a5cecec081db2b37e35332b537f8157c \ + --hash=sha256:98bbd754a422a0b123c66a4c341de0474cad4a5c10c164ceed6ea090f3563db4 \ + --hash=sha256:993584fc821c58d5993521bfdcd31a4adf025c7d745bbd4d12ccfecf695af5ba \ + --hash=sha256:a40821a89dc373d6427e2b44b572efc36a2778d3f543299e2f24eb1a5de65415 \ + --hash=sha256:b291f0ee7961a597cbbcc77709374087fa2a9afe7bdb6a40dbbd9b127e79afee \ + --hash=sha256:b573a43ef7c368ba4ea06050a957c2a7550f729c31f11dd616d2ac4aba99888d \ + --hash=sha256:b610ff0f24e9f11c9ae653c67ff8cc03c075131401b3e5ef4b82570d1728f8a9 \ + --hash=sha256:bdf38ba2d393c7911ae989c3bbba510ebbcdf4ecbdbfec36272abe350c454075 \ + --hash=sha256:bfef2bb6ef068827bbd021017a107194956918ab43ce4d6dc945ffa13efbc25f \ + --hash=sha256:cab3db8bab4b7e635c1c97270d7a4b2a90c070b33cbc00c99ef3f9be03d3e1f7 \ + --hash=sha256:cb70766519500281815dfd7a87d3a178acf7ce95390544b8c90587d76b227681 \ + --hash=sha256:cca1b62fe70d761a282496b96a5e51c44c213e410a964bdffe0928e611368329 \ + --hash=sha256:ccf9a39706b604d884d2cb1e27fe973bc55f2890c52f38df742bc1d79ab9f5e1 \ + --hash=sha256:dc43f1ec66eb8440567186ae2f8c447d91e0372d793dfe8c222aec857b81a8cf \ + --hash=sha256:dd632777ff3beaaf629f1ab4396caf7ba0bdd075d948a69460d13d44357aca4c \ + --hash=sha256:e45ae4927759289c30ccba8d9fdce62bb414977ba158286b5ddaf8df2cddb5c5 \ + --hash=sha256:e50ebce52f41370707f1e21a59514e3375e3edd6e1832f5e5235237db933c98b \ + --hash=sha256:ebbbba226f0a108a7366bf4b59bf0f30a12fd5e75100c630267d94d7f0ad20e5 \ + --hash=sha256:ec79ff6159dffcc30853b2ad612ed572af86c92b5168aa3fc01a67b0fa40665e \ + --hash=sha256:f0936e08e0003f66bfd97e74ee530427707297b0d0361247e9b4f59ab78ddc8b \ + --hash=sha256:f26a07a6e877c76a88e3cecac8531908d980d3d5067ff69213653649ec0f60ad \ + --hash=sha256:f64e376cd20d3f030190e8c32e1c64582eba56ac6dc7d5b0b49a9d44021b52fd \ + --hash=sha256:f6ffbc252eb0d229aeb2f9ad051200668fc3a9aaa8994e49f0cb2ffe2b7867e7 \ + --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ + --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc + # via + # -c python/requirements_compiled.txt + # ray +multidict==6.0.5 \ + --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ + --hash=sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c \ + --hash=sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29 \ + --hash=sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b \ + --hash=sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8 \ + --hash=sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7 \ + --hash=sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd \ + --hash=sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40 \ + --hash=sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6 \ + --hash=sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3 \ + --hash=sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c \ + --hash=sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9 \ + --hash=sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5 \ + --hash=sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae \ + --hash=sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442 \ + --hash=sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9 \ + --hash=sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc \ + --hash=sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c \ + --hash=sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea \ + --hash=sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5 \ + --hash=sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50 \ + --hash=sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182 \ + --hash=sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453 \ + --hash=sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e \ + --hash=sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600 \ + --hash=sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733 \ + --hash=sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda \ + --hash=sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241 \ + --hash=sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461 \ + --hash=sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e \ + --hash=sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e \ + --hash=sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b \ + --hash=sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e \ + --hash=sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7 \ + --hash=sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386 \ + --hash=sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd \ + --hash=sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9 \ + --hash=sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf \ + --hash=sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee \ + --hash=sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5 \ + --hash=sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a \ + --hash=sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271 \ + --hash=sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54 \ + --hash=sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4 \ + --hash=sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496 \ + --hash=sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb \ + --hash=sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319 \ + --hash=sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3 \ + --hash=sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f \ + --hash=sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527 \ + --hash=sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed \ + --hash=sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604 \ + --hash=sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef \ + --hash=sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8 \ + --hash=sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5 \ + --hash=sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5 \ + --hash=sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626 \ + --hash=sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c \ + --hash=sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d \ + --hash=sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c \ + --hash=sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc \ + --hash=sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc \ + --hash=sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b \ + --hash=sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38 \ + --hash=sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450 \ + --hash=sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1 \ + --hash=sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f \ + --hash=sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3 \ + --hash=sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755 \ + --hash=sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226 \ + --hash=sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a \ + --hash=sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046 \ + --hash=sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf \ + --hash=sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479 \ + --hash=sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e \ + --hash=sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1 \ + --hash=sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a \ + --hash=sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83 \ + --hash=sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929 \ + --hash=sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93 \ + --hash=sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a \ + --hash=sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c \ + --hash=sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44 \ + --hash=sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89 \ + --hash=sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba \ + --hash=sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e \ + --hash=sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da \ + --hash=sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24 \ + --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ + --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef + # via + # -c python/requirements_compiled.txt + # aiohttp + # yarl +numpy==1.26.4 \ + --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ + --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ + --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \ + --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \ + --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \ + --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \ + --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \ + --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \ + --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \ + --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \ + --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \ + --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \ + --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \ + --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \ + --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \ + --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \ + --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \ + --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \ + --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \ + --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \ + --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \ + --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \ + --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \ + --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \ + --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \ + --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \ + --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \ + --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \ + --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \ + --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \ + --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \ + --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \ + --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \ + --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \ + --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ + --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f + # via + # -c python/requirements_compiled.txt + # cupy-cuda12x + # gymnasium + # pandas + # ray + # scipy + # tensorboardx +opencensus==0.11.4 \ + --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ + --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 + # via + # -c python/requirements_compiled.txt + # ray +opencensus-context==0.1.3 \ + --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ + --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c + # via + # -c python/requirements_compiled.txt + # opencensus +opentelemetry-api==1.34.1 \ + --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ + --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-exporter-prometheus==0.55b1 \ + --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ + --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e + # via + # -c python/requirements_compiled.txt + # ray +opentelemetry-proto==1.27.0 \ + --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ + --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace + # via + # -c python/requirements_compiled.txt + # ray +opentelemetry-sdk==1.34.1 \ + --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ + --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # ray +opentelemetry-semantic-conventions==0.55b1 \ + --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ + --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 + # via + # -c python/requirements_compiled.txt + # opentelemetry-sdk +ormsgpack==1.7.0 \ + --hash=sha256:0d88307ab45d95416ce4071b1b99326ca31362af01c3d206f15a0551a7a874bd \ + --hash=sha256:22418a4d399027a72fb2e6b873559b1886cf2e63323ca7afc17b222c454413b7 \ + --hash=sha256:2c22c62a6bc93bcb194b7f91864ca0b39455b2cbbfc1538a3da0f9ec3c11d184 \ + --hash=sha256:3a6a97937d2cf21496d7689b90a43df83c5062bbe846aaa39197cc9ad73eaa7b \ + --hash=sha256:462089a419dbde654915ccb0b859c0dbe3c178b0ac580018e82befea6ccd73f4 \ + --hash=sha256:4b353204e99b56c1d33f1cf4767bd1fe1195596181a1cc789f25aa26c0b50f3d \ + --hash=sha256:5ec763096d978d35eedcef0af13991a10741717c2e236b26f4c2047b0740ea7b \ + --hash=sha256:5fefa1ca842dbba258401ea958113fe62c6b70a7a4d46edac440113f68dc431e \ + --hash=sha256:65525438b4a8b3b64ccfcda25e758ea3db392d1c206b5e09ef70efbbafa6dbf9 \ + --hash=sha256:6b4c98839cb7fc2a212037d2258f3a22857155249eb293d45c45cb974cfba834 \ + --hash=sha256:6d114652dadd81802b8a35a49e07a3e9ef2a47aed6123fb5031f2220d1c8e434 \ + --hash=sha256:77bc2ea387d85cfad045b9bcb8040bae43ad32dafe9363360f732cc19d489bbe \ + --hash=sha256:7e6ada21f5c7a20ff7cf9b061c44e3814352f819947a12022ad8cb52a9f2a809 \ + --hash=sha256:8d301e47565fe0e52a60052e730a9bb7669dfbd2a94643b8be925e3928c64c15 \ + --hash=sha256:90aabfd816db60dadab1100d583d061e0238209015bf684f8170c0fca4eb445a \ + --hash=sha256:91ebb7d3609db249cdff629ffef83ec3d025b1384749a297cf3b6a8240cf22ac \ + --hash=sha256:97723786755a7df85fcf6e68d7b5359dacea98d5c26b1d9af219a3cc05df4734 \ + --hash=sha256:9b0945523ccc75aa6907f38f2240d36818618baccb8633923bd7740a5a929e67 \ + --hash=sha256:a0ca6a64d47073f22ecc1dd96b384e44f98796d3f88ee383e92dfbcdf18c2efd \ + --hash=sha256:a5e12b51a590be47ccef67907905653e679fc2f920854b456edc216690ecc09c \ + --hash=sha256:a8fbe7bb50ee8381df030823d9366984fac718447947c2327969405d1d799b95 \ + --hash=sha256:c683071bf4527ffa7b6cfcf28f750d1a82eb77846d106743c09261ab1b79b193 \ + --hash=sha256:ca4d35b694f32112eb33ac0b733cb903dbbc59f019d05ca3d74f6ad2f587b0bf \ + --hash=sha256:e8385181bf195af80fc270e64fd477f1c414ffb05837320382e2ec9ca34be0ec \ + --hash=sha256:e86124cdbc8ed249806347c2fba96843e8941122b161b429139a0c973d270de4 \ + --hash=sha256:f9967a7f3647ad118751abf090f8397fda3e4bca6833340cab95a3f2bec598cd + # via + # -c python/requirements_compiled.txt + # ray +packaging==23.0 \ + --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ + --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 + # via + # -c python/requirements_compiled.txt + # kombu + # ray + # tensorboardx +pandas==1.5.3 \ + --hash=sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813 \ + --hash=sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792 \ + --hash=sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406 \ + --hash=sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373 \ + --hash=sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328 \ + --hash=sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996 \ + --hash=sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf \ + --hash=sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6 \ + --hash=sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7 \ + --hash=sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc \ + --hash=sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1 \ + --hash=sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23 \ + --hash=sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a \ + --hash=sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51 \ + --hash=sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572 \ + --hash=sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31 \ + --hash=sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5 \ + --hash=sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a \ + --hash=sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003 \ + --hash=sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d \ + --hash=sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354 \ + --hash=sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee \ + --hash=sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa \ + --hash=sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0 \ + --hash=sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9 \ + --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ + --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc + # via + # -c python/requirements_compiled.txt + # ray +platformdirs==3.11.0 \ + --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ + --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e + # via + # -c python/requirements_compiled.txt + # virtualenv +prometheus-client==0.19.0 \ + --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ + --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # ray +prompt-toolkit==3.0.41 \ + --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ + --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 + # via + # -c python/requirements_compiled.txt + # click-repl +propcache==0.3.0 \ + --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ + --hash=sha256:03acd9ff19021bd0567582ac88f821b66883e158274183b9e5586f678984f8fe \ + --hash=sha256:03c091bb752349402f23ee43bb2bff6bd80ccab7c9df6b88ad4322258d6960fc \ + --hash=sha256:07700939b2cbd67bfb3b76a12e1412405d71019df00ca5697ce75e5ef789d829 \ + --hash=sha256:0c3e893c4464ebd751b44ae76c12c5f5c1e4f6cbd6fbf67e3783cd93ad221863 \ + --hash=sha256:119e244ab40f70a98c91906d4c1f4c5f2e68bd0b14e7ab0a06922038fae8a20f \ + --hash=sha256:11ae6a8a01b8a4dc79093b5d3ca2c8a4436f5ee251a9840d7790dccbd96cb649 \ + --hash=sha256:15010f29fbed80e711db272909a074dc79858c6d28e2915704cfc487a8ac89c6 \ + --hash=sha256:19d36bb351ad5554ff20f2ae75f88ce205b0748c38b146c75628577020351e3c \ + --hash=sha256:1c8f7d896a16da9455f882870a507567d4f58c53504dc2d4b1e1d386dfe4588a \ + --hash=sha256:2383a17385d9800b6eb5855c2f05ee550f803878f344f58b6e194de08b96352c \ + --hash=sha256:24c04f8fbf60094c531667b8207acbae54146661657a1b1be6d3ca7773b7a545 \ + --hash=sha256:2578541776769b500bada3f8a4eeaf944530516b6e90c089aa368266ed70c49e \ + --hash=sha256:26a67e5c04e3119594d8cfae517f4b9330c395df07ea65eab16f3d559b7068fe \ + --hash=sha256:2b975528998de037dfbc10144b8aed9b8dd5a99ec547f14d1cb7c5665a43f075 \ + --hash=sha256:2d15bc27163cd4df433e75f546b9ac31c1ba7b0b128bfb1b90df19082466ff57 \ + --hash=sha256:2d913d36bdaf368637b4f88d554fb9cb9d53d6920b9c5563846555938d5450bf \ + --hash=sha256:3302c5287e504d23bb0e64d2a921d1eb4a03fb93a0a0aa3b53de059f5a5d737d \ + --hash=sha256:36ca5e9a21822cc1746023e88f5c0af6fce3af3b85d4520efb1ce4221bed75cc \ + --hash=sha256:3b812b3cb6caacd072276ac0492d249f210006c57726b6484a1e1805b3cfeea0 \ + --hash=sha256:3c6ec957025bf32b15cbc6b67afe233c65b30005e4c55fe5768e4bb518d712f1 \ + --hash=sha256:41de3da5458edd5678b0f6ff66691507f9885f5fe6a0fb99a5d10d10c0fd2d64 \ + --hash=sha256:42924dc0c9d73e49908e35bbdec87adedd651ea24c53c29cac103ede0ea1d340 \ + --hash=sha256:4544699674faf66fb6b4473a1518ae4999c1b614f0b8297b1cef96bac25381db \ + --hash=sha256:46ed02532cb66612d42ae5c3929b5e98ae330ea0f3900bc66ec5f4862069519b \ + --hash=sha256:49ea05212a529c2caffe411e25a59308b07d6e10bf2505d77da72891f9a05641 \ + --hash=sha256:4fa0e7c9c3cf7c276d4f6ab9af8adddc127d04e0fcabede315904d2ff76db626 \ + --hash=sha256:507c5357a8d8b4593b97fb669c50598f4e6cccbbf77e22fa9598aba78292b4d7 \ + --hash=sha256:549722908de62aa0b47a78b90531c022fa6e139f9166be634f667ff45632cc92 \ + --hash=sha256:58e6d2a5a7cb3e5f166fd58e71e9a4ff504be9dc61b88167e75f835da5764d07 \ + --hash=sha256:5a16167118677d94bb48bfcd91e420088854eb0737b76ec374b91498fb77a70e \ + --hash=sha256:5d62c4f6706bff5d8a52fd51fec6069bef69e7202ed481486c0bc3874912c787 \ + --hash=sha256:5fa159dcee5dba00c1def3231c249cf261185189205073bde13797e57dd7540a \ + --hash=sha256:6032231d4a5abd67c7f71168fd64a47b6b451fbcb91c8397c2f7610e67683810 \ + --hash=sha256:63f26258a163c34542c24808f03d734b338da66ba91f410a703e505c8485791d \ + --hash=sha256:65a37714b8ad9aba5780325228598a5b16c47ba0f8aeb3dc0514701e4413d7c0 \ + --hash=sha256:67054e47c01b7b349b94ed0840ccae075449503cf1fdd0a1fdd98ab5ddc2667b \ + --hash=sha256:67dda3c7325691c2081510e92c561f465ba61b975f481735aefdfc845d2cd043 \ + --hash=sha256:6985a593417cdbc94c7f9c3403747335e450c1599da1647a5af76539672464d3 \ + --hash=sha256:6a1948df1bb1d56b5e7b0553c0fa04fd0e320997ae99689488201f19fa90d2e7 \ + --hash=sha256:6b5b7fd6ee7b54e01759f2044f936dcf7dea6e7585f35490f7ca0420fe723c0d \ + --hash=sha256:6c929916cbdb540d3407c66f19f73387f43e7c12fa318a66f64ac99da601bcdf \ + --hash=sha256:6f4d7a7c0aff92e8354cceca6fe223973ddf08401047920df0fcb24be2bd5138 \ + --hash=sha256:728af36011bb5d344c4fe4af79cfe186729efb649d2f8b395d1572fb088a996c \ + --hash=sha256:742840d1d0438eb7ea4280f3347598f507a199a35a08294afdcc560c3739989d \ + --hash=sha256:75e872573220d1ee2305b35c9813626e620768248425f58798413e9c39741f46 \ + --hash=sha256:794c3dd744fad478b6232289c866c25406ecdfc47e294618bdf1697e69bd64a6 \ + --hash=sha256:7c0fdbdf6983526e269e5a8d53b7ae3622dd6998468821d660d0daf72779aefa \ + --hash=sha256:7c5f5290799a3f6539cc5e6f474c3e5c5fbeba74a5e1e5be75587746a940d51e \ + --hash=sha256:7c6e7e4f9167fddc438cd653d826f2222222564daed4116a02a184b464d3ef05 \ + --hash=sha256:7cedd25e5f678f7738da38037435b340694ab34d424938041aa630d8bac42663 \ + --hash=sha256:7e2e068a83552ddf7a39a99488bcba05ac13454fb205c847674da0352602082f \ + --hash=sha256:8319293e85feadbbfe2150a5659dbc2ebc4afdeaf7d98936fb9a2f2ba0d4c35c \ + --hash=sha256:8526b0941ec5a40220fc4dfde76aed58808e2b309c03e9fa8e2260083ef7157f \ + --hash=sha256:8884ba1a0fe7210b775106b25850f5e5a9dc3c840d1ae9924ee6ea2eb3acbfe7 \ + --hash=sha256:8cb625bcb5add899cb8ba7bf716ec1d3e8f7cdea9b0713fa99eadf73b6d4986f \ + --hash=sha256:8d663fd71491dde7dfdfc899d13a067a94198e90695b4321084c6e450743b8c7 \ + --hash=sha256:8ee1983728964d6070ab443399c476de93d5d741f71e8f6e7880a065f878e0b9 \ + --hash=sha256:997e7b8f173a391987df40f3b52c423e5850be6f6df0dcfb5376365440b56667 \ + --hash=sha256:9be90eebc9842a93ef8335291f57b3b7488ac24f70df96a6034a13cb58e6ff86 \ + --hash=sha256:9ddd49258610499aab83b4f5b61b32e11fce873586282a0e972e5ab3bcadee51 \ + --hash=sha256:9ecde3671e62eeb99e977f5221abcf40c208f69b5eb986b061ccec317c82ebd0 \ + --hash=sha256:9ff4e9ecb6e4b363430edf2c6e50173a63e0820e549918adef70515f87ced19a \ + --hash=sha256:a254537b9b696ede293bfdbc0a65200e8e4507bc9f37831e2a0318a9b333c85c \ + --hash=sha256:a2b9bf8c79b660d0ca1ad95e587818c30ccdb11f787657458d6f26a1ea18c568 \ + --hash=sha256:a61a68d630e812b67b5bf097ab84e2cd79b48c792857dc10ba8a223f5b06a2af \ + --hash=sha256:a7080b0159ce05f179cfac592cda1a82898ca9cd097dacf8ea20ae33474fbb25 \ + --hash=sha256:a8fd93de4e1d278046345f49e2238cdb298589325849b2645d4a94c53faeffc5 \ + --hash=sha256:a94ffc66738da99232ddffcf7910e0f69e2bbe3a0802e54426dbf0714e1c2ffe \ + --hash=sha256:aa806bbc13eac1ab6291ed21ecd2dd426063ca5417dd507e6be58de20e58dfcf \ + --hash=sha256:b0c1a133d42c6fc1f5fbcf5c91331657a1ff822e87989bf4a6e2e39b818d0ee9 \ + --hash=sha256:b58229a844931bca61b3a20efd2be2a2acb4ad1622fc026504309a6883686fbf \ + --hash=sha256:bb2f144c6d98bb5cbc94adeb0447cfd4c0f991341baa68eee3f3b0c9c0e83767 \ + --hash=sha256:be90c94570840939fecedf99fa72839aed70b0ced449b415c85e01ae67422c90 \ + --hash=sha256:bf0d9a171908f32d54f651648c7290397b8792f4303821c42a74e7805bfb813c \ + --hash=sha256:bf15fc0b45914d9d1b706f7c9c4f66f2b7b053e9517e40123e137e8ca8958b3d \ + --hash=sha256:bf4298f366ca7e1ad1d21bbb58300a6985015909964077afd37559084590c929 \ + --hash=sha256:c441c841e82c5ba7a85ad25986014be8d7849c3cfbdb6004541873505929a74e \ + --hash=sha256:cacea77ef7a2195f04f9279297684955e3d1ae4241092ff0cfcef532bb7a1c32 \ + --hash=sha256:cd54895e4ae7d32f1e3dd91261df46ee7483a735017dc6f987904f194aa5fd14 \ + --hash=sha256:d1323cd04d6e92150bcc79d0174ce347ed4b349d748b9358fd2e497b121e03c8 \ + --hash=sha256:d383bf5e045d7f9d239b38e6acadd7b7fdf6c0087259a84ae3475d18e9a2ae8b \ + --hash=sha256:d3e7420211f5a65a54675fd860ea04173cde60a7cc20ccfbafcccd155225f8bc \ + --hash=sha256:d8074c5dd61c8a3e915fa8fc04754fa55cfa5978200d2daa1e2d4294c1f136aa \ + --hash=sha256:df03cd88f95b1b99052b52b1bb92173229d7a674df0ab06d2b25765ee8404bce \ + --hash=sha256:e45377d5d6fefe1677da2a2c07b024a6dac782088e37c0b1efea4cfe2b1be19b \ + --hash=sha256:e53d19c2bf7d0d1e6998a7e693c7e87300dd971808e6618964621ccd0e01fe4e \ + --hash=sha256:e560fd75aaf3e5693b91bcaddd8b314f4d57e99aef8a6c6dc692f935cc1e6bbf \ + --hash=sha256:ec5060592d83454e8063e487696ac3783cc48c9a329498bafae0d972bc7816c9 \ + --hash=sha256:ecc2920630283e0783c22e2ac94427f8cca29a04cfdf331467d4f661f4072dac \ + --hash=sha256:ed7161bccab7696a473fe7ddb619c1d75963732b37da4618ba12e60899fefe4f \ + --hash=sha256:ee0bd3a7b2e184e88d25c9baa6a9dc609ba25b76daae942edfb14499ac7ec374 \ + --hash=sha256:ee25f1ac091def37c4b59d192bbe3a206298feeb89132a470325bf76ad122a1e \ + --hash=sha256:efa44f64c37cc30c9f05932c740a8b40ce359f51882c70883cc95feac842da4d \ + --hash=sha256:f47d52fd9b2ac418c4890aad2f6d21a6b96183c98021f0a48497a904199f006e \ + --hash=sha256:f857034dc68d5ceb30fb60afb6ff2103087aea10a01b613985610e007053a121 \ + --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ + --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 + # via + # -c python/requirements_compiled.txt + # aiohttp + # yarl +proto-plus==1.22.3 \ + --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ + --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b + # via + # -c python/requirements_compiled.txt + # google-api-core +protobuf==4.25.8 \ + --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ + --hash=sha256:15a0af558aa3b13efef102ae6e4f3efac06f1eea11afb3a57db2901447d9fb59 \ + --hash=sha256:27d498ffd1f21fb81d987a041c32d07857d1d107909f5134ba3350e1ce80a4af \ + --hash=sha256:504435d831565f7cfac9f0714440028907f1975e4bed228e58e72ecfff58a1e0 \ + --hash=sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd \ + --hash=sha256:83e6e54e93d2b696a92cad6e6efc924f3850f82b52e1563778dfab8b355101b0 \ + --hash=sha256:9ad7ef62d92baf5a8654fbb88dac7fa5594cfa70fd3440488a5ca3bfc6d795a7 \ + --hash=sha256:bd551eb1fe1d7e92c1af1d75bdfa572eff1ab0e5bf1736716814cdccdb2360f9 \ + --hash=sha256:ca809b42f4444f144f2115c4c1a747b9a404d590f18f37e9402422033e464e0f \ + --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ + --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 + # via + # -c python/requirements_compiled.txt + # google-api-core + # googleapis-common-protos + # opentelemetry-proto + # proto-plus + # ray + # tensorboardx +py-spy==0.4.0 ; python_full_version < '3.12' \ + --hash=sha256:47cdda4c34d9b6cb01f3aaeceb2e88faf57da880207fe72ff6ff97e9bb6cc8a9 \ + --hash=sha256:77d8f637ade38367d944874776f45b703b7ac5938b1f7be8891f3a5876ddbb96 \ + --hash=sha256:806602ce7972782cc9c1e383f339bfc27bfb822d42485e6a3e0530ae5040e1f0 \ + --hash=sha256:87573e64dbfdfc89ba2e0f5e2f525aa84e0299c7eb6454b47ea335fde583a7a0 \ + --hash=sha256:8bf2f3702cef367a489faa45177b41a6c31b2a3e5bd78c978d44e29340152f5a \ + --hash=sha256:c5f06ffce4c9c98b7fc9f5e67e5e7db591173f1351837633f3f23d9378b1d18a \ + --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ + --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 + # via + # -c python/requirements_compiled.txt + # ray +pyarrow==19.0.1 \ + --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ + --hash=sha256:0148bb4fc158bfbc3d6dfe5001d93ebeed253793fff4435167f6ce1dc4bddeae \ + --hash=sha256:1b93ef2c93e77c442c979b0d596af45e4665d8b96da598db145b0fec014b9136 \ + --hash=sha256:1c7556165bd38cf0cd992df2636f8bcdd2d4b26916c6b7e646101aff3c16f76f \ + --hash=sha256:335d170e050bcc7da867a1ed8ffb8b44c57aaa6e0843b156a501298657b1e972 \ + --hash=sha256:3bf266b485df66a400f282ac0b6d1b500b9d2ae73314a153dbe97d6d5cc8a99e \ + --hash=sha256:41f9706fbe505e0abc10e84bf3a906a1338905cbbcf1177b71486b03e6ea6608 \ + --hash=sha256:4982f8e2b7afd6dae8608d70ba5bd91699077323f812a0448d8b7abdff6cb5d3 \ + --hash=sha256:49a3aecb62c1be1d822f8bf629226d4a96418228a42f5b40835c1f10d42e4db6 \ + --hash=sha256:4d5d1ec7ec5324b98887bdc006f4d2ce534e10e60f7ad995e7875ffa0ff9cb14 \ + --hash=sha256:58d9397b2e273ef76264b45531e9d552d8ec8a6688b7390b5be44c02a37aade8 \ + --hash=sha256:5a9137cf7e1640dce4c190551ee69d478f7121b5c6f323553b319cac936395f6 \ + --hash=sha256:5bd1618ae5e5476b7654c7b55a6364ae87686d4724538c24185bbb2952679960 \ + --hash=sha256:65cf9feebab489b19cdfcfe4aa82f62147218558d8d3f0fc1e9dea0ab8e7905a \ + --hash=sha256:699799f9c80bebcf1da0983ba86d7f289c5a2a5c04b945e2f2bcf7e874a91911 \ + --hash=sha256:6c5941c1aac89a6c2f2b16cd64fe76bcdb94b2b1e99ca6459de4e6f07638d755 \ + --hash=sha256:6ebfb5171bb5f4a52319344ebbbecc731af3f021e49318c74f33d520d31ae0c4 \ + --hash=sha256:7a544ec12de66769612b2d6988c36adc96fb9767ecc8ee0a4d270b10b1c51e00 \ + --hash=sha256:7c1bca1897c28013db5e4c83944a2ab53231f541b9e0c3f4791206d0c0de389a \ + --hash=sha256:80b2ad2b193e7d19e81008a96e313fbd53157945c7be9ac65f44f8937a55427b \ + --hash=sha256:8464c9fbe6d94a7fe1599e7e8965f350fd233532868232ab2596a71586c5a429 \ + --hash=sha256:8f04d49a6b64cf24719c080b3c2029a3a5b16417fd5fd7c4041f94233af732f3 \ + --hash=sha256:96606c3ba57944d128e8a8399da4812f56c7f61de8c647e3470b417f795d0ef9 \ + --hash=sha256:99bc1bec6d234359743b01e70d4310d0ab240c3d6b0da7e2a93663b0158616f6 \ + --hash=sha256:ad76aef7f5f7e4a757fddcdcf010a8290958f09e3470ea458c80d26f4316ae89 \ + --hash=sha256:b4c4156a625f1e35d6c0b2132635a237708944eb41df5fbe7d50f20d20c17832 \ + --hash=sha256:b9766a47a9cb56fefe95cb27f535038b5a195707a08bf61b180e642324963b46 \ + --hash=sha256:c0fe3dbbf054a00d1f162fda94ce236a899ca01123a798c561ba307ca38af5f0 \ + --hash=sha256:c6cb2335a411b713fdf1e82a752162f72d4a7b5dbc588e32aa18383318b05866 \ + --hash=sha256:cc55d71898ea30dc95900297d191377caba257612f384207fe9f8293b5850f90 \ + --hash=sha256:d03c9d6f2a3dffbd62671ca070f13fc527bb1867b4ec2b98c7eeed381d4f389a \ + --hash=sha256:d383591f3dcbe545f6cc62daaef9c7cdfe0dff0fb9e1c8121101cabe9098cfa6 \ + --hash=sha256:d9d46e06846a41ba906ab25302cf0fd522f81aa2a85a71021826f34639ad31ef \ + --hash=sha256:d9dedeaf19097a143ed6da37f04f4051aba353c95ef507764d344229b2b740ae \ + --hash=sha256:e45274b20e524ae5c39d7fc1ca2aa923aab494776d2d4b316b49ec7572ca324c \ + --hash=sha256:ee8dec072569f43835932a3b10c55973593abc00936c202707a4ad06af7cb294 \ + --hash=sha256:f24faab6ed18f216a37870d8c5623f9c044566d75ec586ef884e13a02a9d62c5 \ + --hash=sha256:f2a21d39fbdb948857f67eacb5bbaaf36802de044ec36fbef7a1c8f0dd3a4ab2 \ + --hash=sha256:f3ad4c0eb4e2a9aeb990af6c09e6fa0b195c8c0e7b272ecc8d4d2b6574809d34 \ + --hash=sha256:fc28912a2dc924dddc2087679cc8b7263accc71b9ff025a1362b004711661a69 \ + --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ + --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 + # via + # -c python/requirements_compiled.txt + # ray +pyasn1==0.5.1 \ + --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ + --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c + # via + # -c python/requirements_compiled.txt + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 \ + --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ + --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d + # via + # -c python/requirements_compiled.txt + # google-auth +pycparser==2.21 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ + --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 + # via + # -c python/requirements_compiled.txt + # cffi +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b + # via + # -c python/requirements_compiled.txt + # fastapi + # ray +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d + # via + # -c python/requirements_compiled.txt + # pydantic +pygments==2.18.0 ; sys_platform != 'win32' \ + --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ + --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a + # via + # -c python/requirements_compiled.txt + # rich +pyopenssl==25.0.0 \ + --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ + --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 + # via + # -c python/requirements_compiled.txt + # ray +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via + # -c python/requirements_compiled.txt + # celery + # pandas +python-dotenv==1.1.1 \ + --hash=sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc \ + --hash=sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab + # via uvicorn +pytz==2022.7.1 \ + --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ + --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a + # via + # -c python/requirements_compiled.txt + # pandas +pyyaml==6.0.1 \ + --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ + --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ + --hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \ + --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ + --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \ + --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \ + --hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \ + --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \ + --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \ + --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \ + --hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \ + --hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \ + --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \ + --hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \ + --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \ + --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \ + --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \ + --hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \ + --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \ + --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \ + --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \ + --hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \ + --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \ + --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \ + --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \ + --hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \ + --hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \ + --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ + --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ + --hash=sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef \ + --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ + --hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \ + --hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \ + --hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \ + --hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \ + --hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \ + --hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \ + --hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \ + --hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \ + --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \ + --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \ + --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \ + --hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \ + --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \ + --hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \ + --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \ + --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \ + --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \ + --hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \ + --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ + --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f + # via + # -c python/requirements_compiled.txt + # ray + # uvicorn +ray==100.0.0.dev0 \ + --hash=sha256:287f652801352646b3d4ab6cf71d91763555ca2a714364d1a187fbdead96a122 +referencing==0.36.2 \ + --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ + --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 + # via + # -c python/requirements_compiled.txt + # jsonschema + # jsonschema-specifications +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 + # via + # -c python/requirements_compiled.txt + # google-api-core + # ray +rich==13.3.2 ; sys_platform != 'win32' \ + --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ + --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f + # via + # -c python/requirements_compiled.txt + # memray +rpds-py==0.22.3 \ + --hash=sha256:009de23c9c9ee54bf11303a966edf4d9087cd43a6003672e6aa7def643d06518 \ + --hash=sha256:02fbb9c288ae08bcb34fb41d516d5eeb0455ac35b5512d03181d755d80810059 \ + --hash=sha256:0a0461200769ab3b9ab7e513f6013b7a97fdeee41c29b9db343f3c5a8e2b9e61 \ + --hash=sha256:0b09865a9abc0ddff4e50b5ef65467cd94176bf1e0004184eb915cbc10fc05c5 \ + --hash=sha256:0b8db6b5b2d4491ad5b6bdc2bc7c017eec108acbf4e6785f42a9eb0ba234f4c9 \ + --hash=sha256:0c150c7a61ed4a4f4955a96626574e9baf1adf772c2fb61ef6a5027e52803543 \ + --hash=sha256:0f3cec041684de9a4684b1572fe28c7267410e02450f4561700ca5a3bc6695a2 \ + --hash=sha256:1352ae4f7c717ae8cba93421a63373e582d19d55d2ee2cbb184344c82d2ae55a \ + --hash=sha256:177c7c0fce2855833819c98e43c262007f42ce86651ffbb84f37883308cb0e7d \ + --hash=sha256:1978d0021e943aae58b9b0b196fb4895a25cc53d3956b8e35e0b7682eefb6d56 \ + --hash=sha256:1a60bce91f81ddaac922a40bbb571a12c1070cb20ebd6d49c48e0b101d87300d \ + --hash=sha256:1aef18820ef3e4587ebe8b3bc9ba6e55892a6d7b93bac6d29d9f631a3b4befbd \ + --hash=sha256:1e9663daaf7a63ceccbbb8e3808fe90415b0757e2abddbfc2e06c857bf8c5e2b \ + --hash=sha256:20070c65396f7373f5df4005862fa162db5d25d56150bddd0b3e8214e8ef45b4 \ + --hash=sha256:214b7a953d73b5e87f0ebece4a32a5bd83c60a3ecc9d4ec8f1dca968a2d91e99 \ + --hash=sha256:22bebe05a9ffc70ebfa127efbc429bc26ec9e9b4ee4d15a740033efda515cf3d \ + --hash=sha256:24e8abb5878e250f2eb0d7859a8e561846f98910326d06c0d51381fed59357bd \ + --hash=sha256:26fd7cac7dd51011a245f29a2cc6489c4608b5a8ce8d75661bb4a1066c52dfbe \ + --hash=sha256:27b1d3b3915a99208fee9ab092b8184c420f2905b7d7feb4aeb5e4a9c509b8a1 \ + --hash=sha256:27e98004595899949bd7a7b34e91fa7c44d7a97c40fcaf1d874168bb652ec67e \ + --hash=sha256:2b8f60e1b739a74bab7e01fcbe3dddd4657ec685caa04681df9d562ef15b625f \ + --hash=sha256:2de29005e11637e7a2361fa151f780ff8eb2543a0da1413bb951e9f14b699ef3 \ + --hash=sha256:2e8b55d8517a2fda8d95cb45d62a5a8bbf9dd0ad39c5b25c8833efea07b880ca \ + --hash=sha256:2fa4331c200c2521512595253f5bb70858b90f750d39b8cbfd67465f8d1b596d \ + --hash=sha256:3445e07bf2e8ecfeef6ef67ac83de670358abf2996916039b16a218e3d95e97e \ + --hash=sha256:3453e8d41fe5f17d1f8e9c383a7473cd46a63661628ec58e07777c2fff7196dc \ + --hash=sha256:378753b4a4de2a7b34063d6f95ae81bfa7b15f2c1a04a9518e8644e81807ebea \ + --hash=sha256:3af6e48651c4e0d2d166dc1b033b7042ea3f871504b6805ba5f4fe31581d8d38 \ + --hash=sha256:3dfcbc95bd7992b16f3f7ba05af8a64ca694331bd24f9157b49dadeeb287493b \ + --hash=sha256:3f21f0495edea7fdbaaa87e633a8689cd285f8f4af5c869f27bc8074638ad69c \ + --hash=sha256:4041711832360a9b75cfb11b25a6a97c8fb49c07b8bd43d0d02b45d0b499a4ff \ + --hash=sha256:44d61b4b7d0c2c9ac019c314e52d7cbda0ae31078aabd0f22e583af3e0d79723 \ + --hash=sha256:4617e1915a539a0d9a9567795023de41a87106522ff83fbfaf1f6baf8e85437e \ + --hash=sha256:4b232061ca880db21fa14defe219840ad9b74b6158adb52ddf0e87bead9e8493 \ + --hash=sha256:5246b14ca64a8675e0a7161f7af68fe3e910e6b90542b4bfb5439ba752191df6 \ + --hash=sha256:5725dd9cc02068996d4438d397e255dcb1df776b7ceea3b9cb972bdb11260a83 \ + --hash=sha256:583f6a1993ca3369e0f80ba99d796d8e6b1a3a2a442dd4e1a79e652116413091 \ + --hash=sha256:59259dc58e57b10e7e18ce02c311804c10c5a793e6568f8af4dead03264584d1 \ + --hash=sha256:593eba61ba0c3baae5bc9be2f5232430453fb4432048de28399ca7376de9c627 \ + --hash=sha256:59f4a79c19232a5774aee369a0c296712ad0e77f24e62cad53160312b1c1eaa1 \ + --hash=sha256:5f0e260eaf54380380ac3808aa4ebe2d8ca28b9087cf411649f96bad6900c728 \ + --hash=sha256:62d9cfcf4948683a18a9aff0ab7e1474d407b7bab2ca03116109f8464698ab16 \ + --hash=sha256:64607d4cbf1b7e3c3c8a14948b99345eda0e161b852e122c6bb71aab6d1d798c \ + --hash=sha256:655ca44a831ecb238d124e0402d98f6212ac527a0ba6c55ca26f616604e60a45 \ + --hash=sha256:666ecce376999bf619756a24ce15bb14c5bfaf04bf00abc7e663ce17c3f34fe7 \ + --hash=sha256:68049202f67380ff9aa52f12e92b1c30115f32e6895cd7198fa2a7961621fc5a \ + --hash=sha256:69803198097467ee7282750acb507fba35ca22cc3b85f16cf45fb01cb9097730 \ + --hash=sha256:6c7b99ca52c2c1752b544e310101b98a659b720b21db00e65edca34483259967 \ + --hash=sha256:6dd9412824c4ce1aca56c47b0991e65bebb7ac3f4edccfd3f156150c96a7bf25 \ + --hash=sha256:70eb60b3ae9245ddea20f8a4190bd79c705a22f8028aaf8bbdebe4716c3fab24 \ + --hash=sha256:70fb28128acbfd264eda9bf47015537ba3fe86e40d046eb2963d75024be4d055 \ + --hash=sha256:7b2513ba235829860b13faa931f3b6846548021846ac808455301c23a101689d \ + --hash=sha256:7ef9d9da710be50ff6809fed8f1963fecdfecc8b86656cadfca3bc24289414b0 \ + --hash=sha256:81e69b0a0e2537f26d73b4e43ad7bc8c8efb39621639b4434b76a3de50c6966e \ + --hash=sha256:8633e471c6207a039eff6aa116e35f69f3156b3989ea3e2d755f7bc41754a4a7 \ + --hash=sha256:8bd7c8cfc0b8247c8799080fbff54e0b9619e17cdfeb0478ba7295d43f635d7c \ + --hash=sha256:9253fc214112405f0afa7db88739294295f0e08466987f1d70e29930262b4c8f \ + --hash=sha256:99b37292234e61325e7a5bb9689e55e48c3f5f603af88b1642666277a81f1fbd \ + --hash=sha256:9bd7228827ec7bb817089e2eb301d907c0d9827a9e558f22f762bb690b131652 \ + --hash=sha256:9beeb01d8c190d7581a4d59522cd3d4b6887040dcfc744af99aa59fef3e041a8 \ + --hash=sha256:a63cbdd98acef6570c62b92a1e43266f9e8b21e699c363c0fef13bd530799c11 \ + --hash=sha256:a76e42402542b1fae59798fab64432b2d015ab9d0c8c47ba7addddbaf7952333 \ + --hash=sha256:ac0a03221cdb5058ce0167ecc92a8c89e8d0decdc9e99a2ec23380793c4dcb96 \ + --hash=sha256:b0b4136a252cadfa1adb705bb81524eee47d9f6aab4f2ee4fa1e9d3cd4581f64 \ + --hash=sha256:b25bc607423935079e05619d7de556c91fb6adeae9d5f80868dde3468657994b \ + --hash=sha256:b3d504047aba448d70cf6fa22e06cb09f7cbd761939fdd47604f5e007675c24e \ + --hash=sha256:bb47271f60660803ad11f4c61b42242b8c1312a31c98c578f79ef9387bbde21c \ + --hash=sha256:bbb232860e3d03d544bc03ac57855cd82ddf19c7a07651a7c0fdb95e9efea8b9 \ + --hash=sha256:bc27863442d388870c1809a87507727b799c8460573cfbb6dc0eeaef5a11b5ec \ + --hash=sha256:bc51abd01f08117283c5ebf64844a35144a0843ff7b2983e0648e4d3d9f10dbb \ + --hash=sha256:be2eb3f2495ba669d2a985f9b426c1797b7d48d6963899276d22f23e33d47e37 \ + --hash=sha256:bf9db5488121b596dbfc6718c76092fda77b703c1f7533a226a5a9f65248f8ad \ + --hash=sha256:c58e2339def52ef6b71b8f36d13c3688ea23fa093353f3a4fee2556e62086ec9 \ + --hash=sha256:cfbc454a2880389dbb9b5b398e50d439e2e58669160f27b60e5eca11f68ae17c \ + --hash=sha256:cff63a0272fcd259dcc3be1657b07c929c466b067ceb1c20060e8d10af56f5bf \ + --hash=sha256:d115bffdd417c6d806ea9069237a4ae02f513b778e3789a359bc5856e0404cc4 \ + --hash=sha256:d20cfb4e099748ea39e6f7b16c91ab057989712d31761d3300d43134e26e165f \ + --hash=sha256:d48424e39c2611ee1b84ad0f44fb3b2b53d473e65de061e3f460fc0be5f1939d \ + --hash=sha256:e0fa2d4ec53dc51cf7d3bb22e0aa0143966119f42a0c3e4998293a3dd2856b09 \ + --hash=sha256:e32fee8ab45d3c2db6da19a5323bc3362237c8b653c70194414b892fd06a080d \ + --hash=sha256:e35ba67d65d49080e8e5a1dd40101fccdd9798adb9b050ff670b7d74fa41c566 \ + --hash=sha256:e3fb866d9932a3d7d0c82da76d816996d1667c44891bd861a0f97ba27e84fc74 \ + --hash=sha256:e61b02c3f7a1e0b75e20c3978f7135fd13cb6cf551bf4a6d29b999a88830a338 \ + --hash=sha256:e67ba3c290821343c192f7eae1d8fd5999ca2dc99994114643e2f2d3e6138b15 \ + --hash=sha256:e79dd39f1e8c3504be0607e5fc6e86bb60fe3584bec8b782578c3b0fde8d932c \ + --hash=sha256:e89391e6d60251560f0a8f4bd32137b077a80d9b7dbe6d5cab1cd80d2746f648 \ + --hash=sha256:ea7433ce7e4bfc3a85654aeb6747babe3f66eaf9a1d0c1e7a4435bbdf27fea84 \ + --hash=sha256:eaf16ae9ae519a0e237a0f528fd9f0197b9bb70f40263ee57ae53c2b8d48aeb3 \ + --hash=sha256:eb0c341fa71df5a4595f9501df4ac5abfb5a09580081dffbd1ddd4654e6e9123 \ + --hash=sha256:f276b245347e6e36526cbd4a266a417796fc531ddf391e43574cf6466c492520 \ + --hash=sha256:f47ad3d5f3258bd7058d2d506852217865afefe6153a36eb4b6928758041d831 \ + --hash=sha256:f56a6b404f74ab372da986d240e2e002769a7d7102cc73eb238a4f72eec5284e \ + --hash=sha256:f5cf2a0c2bdadf3791b5c205d55a37a54025c6e18a71c71f82bb536cf9a454bf \ + --hash=sha256:f5d36399a1b96e1a5fdc91e0522544580dbebeb1f77f27b2b0ab25559e103b8b \ + --hash=sha256:f60bd8423be1d9d833f230fdbccf8f57af322d96bcad6599e5a771b151398eb2 \ + --hash=sha256:f612463ac081803f243ff13cccc648578e2279295048f2a8d5eb430af2bae6e3 \ + --hash=sha256:f73d3fef726b3243a811121de45193c0ca75f6407fe66f3f4e183c983573e130 \ + --hash=sha256:f82a116a1d03628a8ace4859556fb39fd1424c933341a08ea3ed6de1edb0283b \ + --hash=sha256:fb0ba113b4983beac1a2eb16faffd76cb41e176bf58c4afe3e14b9c681f702de \ + --hash=sha256:fb4f868f712b2dd4bcc538b0a0c1f63a2b1d584c925e69a224d759e7070a12d5 \ + --hash=sha256:fb6116dfb8d1925cbdb52595560584db42a7f664617a1f7d7f6e32f138cdf37d \ + --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ + --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e + # via + # -c python/requirements_compiled.txt + # jsonschema + # referencing +rsa==4.7.2 \ + --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ + --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 + # via + # -c python/requirements_compiled.txt + # google-auth +scipy==1.11.4 \ + --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ + --hash=sha256:028eccd22e654b3ea01ee63705681ee79933652b2d8f873e7949898dda6d11b6 \ + --hash=sha256:1b7c3dca977f30a739e0409fb001056484661cb2541a01aba0bb0029f7b68db8 \ + --hash=sha256:2c6ff6ef9cc27f9b3db93a6f8b38f97387e6e0591600369a297a50a8e96e835d \ + --hash=sha256:36750b7733d960d7994888f0d148d31ea3017ac15eef664194b4ef68d36a4a97 \ + --hash=sha256:530f9ad26440e85766509dbf78edcfe13ffd0ab7fec2560ee5c36ff74d6269ff \ + --hash=sha256:5e347b14fe01003d3b78e196e84bd3f48ffe4c8a7b8a1afbcb8f5505cb710993 \ + --hash=sha256:6550466fbeec7453d7465e74d4f4b19f905642c89a7525571ee91dd7adabb5a3 \ + --hash=sha256:6df1468153a31cf55ed5ed39647279beb9cfb5d3f84369453b49e4b8502394fd \ + --hash=sha256:6e619aba2df228a9b34718efb023966da781e89dd3d21637b27f2e54db0410d7 \ + --hash=sha256:8fce70f39076a5aa62e92e69a7f62349f9574d8405c0a5de6ed3ef72de07f446 \ + --hash=sha256:90a2b78e7f5733b9de748f589f09225013685f9b218275257f8a8168ededaeaa \ + --hash=sha256:91af76a68eeae0064887a48e25c4e616fa519fa0d38602eda7e0f97d65d57937 \ + --hash=sha256:933baf588daa8dc9a92c20a0be32f56d43faf3d1a60ab11b3f08c356430f6e56 \ + --hash=sha256:acf8ed278cc03f5aff035e69cb511741e0418681d25fbbb86ca65429c4f4d9cd \ + --hash=sha256:ad669df80528aeca5f557712102538f4f37e503f0c5b9541655016dd0932ca79 \ + --hash=sha256:b030c6674b9230d37c5c60ab456e2cf12f6784596d15ce8da9365e70896effc4 \ + --hash=sha256:b9999c008ccf00e8fbcce1236f85ade5c569d13144f77a1946bef8863e8f6eb4 \ + --hash=sha256:bc9a714581f561af0848e6b69947fda0614915f072dfd14142ed1bfe1b806710 \ + --hash=sha256:ce7fff2e23ab2cc81ff452a9444c215c28e6305f396b2ba88343a567feec9660 \ + --hash=sha256:cf00bd2b1b0211888d4dc75656c0412213a8b25e80d73898083f402b50f47e41 \ + --hash=sha256:d10e45a6c50211fe256da61a11c34927c68f277e03138777bdebedd933712fea \ + --hash=sha256:ee410e6de8f88fd5cf6eadd73c135020bfbbbdfcd0f6162c36a7638a1ea8cc65 \ + --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ + --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec + # via + # -c python/requirements_compiled.txt + # ray +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via + # -c python/requirements_compiled.txt + # opencensus + # python-dateutil +smart-open==6.2.0 \ + --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ + --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 + # via + # -c python/requirements_compiled.txt + # ray +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc + # via + # -c python/requirements_compiled.txt + # anyio +starlette==0.46.2 \ + --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ + --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 + # via + # -c python/requirements_compiled.txt + # fastapi + # ray +tensorboardx==2.6.2.2 \ + --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ + --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 + # via + # -c python/requirements_compiled.txt + # ray +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d + # via + # -c python/requirements_compiled.txt + # fastapi + # gymnasium + # opentelemetry-api + # opentelemetry-sdk + # opentelemetry-semantic-conventions + # pydantic + # pydantic-core + # pyopenssl + # referencing + # typing-inspection +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/requirements_compiled.txt + # pydantic +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled.txt + # kombu +urllib3==1.26.19 \ + --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ + --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 + # via + # -c python/requirements_compiled.txt + # requests +uvicorn==0.22.0 \ + --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ + --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 + # via + # -c python/requirements_compiled.txt + # ray +uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32' \ + --hash=sha256:0878c2640cf341b269b7e128b1a5fed890adc4455513ca710d77d5e93aa6d6a0 \ + --hash=sha256:10d66943def5fcb6e7b37310eb6b5639fd2ccbc38df1177262b0640c3ca68c1f \ + --hash=sha256:10da8046cc4a8f12c91a1c39d1dd1585c41162a15caaef165c2174db9ef18bdc \ + --hash=sha256:17df489689befc72c39a08359efac29bbee8eee5209650d4b9f34df73d22e414 \ + --hash=sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f \ + --hash=sha256:196274f2adb9689a289ad7d65700d37df0c0930fd8e4e743fa4834e850d7719d \ + --hash=sha256:221f4f2a1f46032b403bf3be628011caf75428ee3cc204a22addf96f586b19fd \ + --hash=sha256:2d1f581393673ce119355d56da84fe1dd9d2bb8b3d13ce792524e1607139feff \ + --hash=sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c \ + --hash=sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3 \ + --hash=sha256:4509360fcc4c3bd2c70d87573ad472de40c13387f5fda8cb58350a1d7475e58d \ + --hash=sha256:460def4412e473896ef179a1671b40c039c7012184b627898eea5072ef6f017a \ + --hash=sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb \ + --hash=sha256:46923b0b5ee7fc0020bef24afe7836cb068f5050ca04caf6b487c513dc1a20b2 \ + --hash=sha256:53e420a3afe22cdcf2a0f4846e377d16e718bc70103d7088a4f7623567ba5fb0 \ + --hash=sha256:5ee4d4ef48036ff6e5cfffb09dd192c7a5027153948d85b8da7ff705065bacc6 \ + --hash=sha256:67dd654b8ca23aed0a8e99010b4c34aca62f4b7fce88f39d452ed7622c94845c \ + --hash=sha256:787ae31ad8a2856fc4e7c095341cccc7209bd657d0e71ad0dc2ea83c4a6fa8af \ + --hash=sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc \ + --hash=sha256:87c43e0f13022b998eb9b973b5e97200c8b90823454d4bc06ab33829e09fb9bb \ + --hash=sha256:88cb67cdbc0e483da00af0b2c3cdad4b7c61ceb1ee0f33fe00e09c81e3a6cb75 \ + --hash=sha256:8a375441696e2eda1c43c44ccb66e04d61ceeffcd76e4929e527b7fa401b90fb \ + --hash=sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553 \ + --hash=sha256:b9fb766bb57b7388745d8bcc53a359b116b8a04c83a2288069809d2b3466c37e \ + --hash=sha256:baa0e6291d91649c6ba4ed4b2f982f9fa165b5bbd50a9e203c416a2797bab3c6 \ + --hash=sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d \ + --hash=sha256:bc09f0ff191e61c2d592a752423c767b4ebb2986daa9ed62908e2b1b9a9ae206 \ + --hash=sha256:bd53ecc9a0f3d87ab847503c2e1552b690362e005ab54e8a48ba97da3924c0dc \ + --hash=sha256:bfd55dfcc2a512316e65f16e503e9e450cab148ef11df4e4e679b5e8253a5281 \ + --hash=sha256:c097078b8031190c934ed0ebfee8cc5f9ba9642e6eb88322b9958b649750f72b \ + --hash=sha256:c0f3fa6200b3108919f8bdabb9a7f87f20e7097ea3c543754cabc7d717d95cf8 \ + --hash=sha256:e678ad6fe52af2c58d2ae3c73dc85524ba8abe637f134bf3564ed07f555c5e79 \ + --hash=sha256:ec7e6b09a6fdded42403182ab6b832b71f4edaf7f37a9a0e371a01db5f0cb45f \ + --hash=sha256:f0ce1b49560b1d2d8a2977e3ba4afb2414fb46b86a1b64056bc4ab929efdafbe \ + --hash=sha256:f38b2e090258d051d68a5b14d1da7203a3c3677321cf32a95a6f4db4dd8b6f26 \ + --hash=sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816 \ + --hash=sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2 + # via + # -c python/requirements_compiled.txt + # uvicorn +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled.txt + # amqp + # celery + # kombu +virtualenv==20.29.1 \ + --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ + --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 + # via + # -c python/requirements_compiled.txt + # ray +watchfiles==0.19.0 \ + --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ + --hash=sha256:09ea3397aecbc81c19ed7f025e051a7387feefdb789cf768ff994c1228182fda \ + --hash=sha256:176a9a7641ec2c97b24455135d58012a5be5c6217fc4d5fef0b2b9f75dbf5154 \ + --hash=sha256:18b28f6ad871b82df9542ff958d0c86bb0d8310bb09eb8e87d97318a3b5273af \ + --hash=sha256:20b44221764955b1e703f012c74015306fb7e79a00c15370785f309b1ed9aa8d \ + --hash=sha256:3d7d267d27aceeeaa3de0dd161a0d64f0a282264d592e335fff7958cc0cbae7c \ + --hash=sha256:5471582658ea56fca122c0f0d0116a36807c63fefd6fdc92c71ca9a4491b6b48 \ + --hash=sha256:5569fc7f967429d4bc87e355cdfdcee6aabe4b620801e2cf5805ea245c06097c \ + --hash=sha256:68dce92b29575dda0f8d30c11742a8e2b9b8ec768ae414b54f7453f27bdf9545 \ + --hash=sha256:79c533ff593db861ae23436541f481ec896ee3da4e5db8962429b441bbaae16e \ + --hash=sha256:7f3920b1285a7d3ce898e303d84791b7bf40d57b7695ad549dc04e6a44c9f120 \ + --hash=sha256:91633e64712df3051ca454ca7d1b976baf842d7a3640b87622b323c55f3345e7 \ + --hash=sha256:945be0baa3e2440151eb3718fd8846751e8b51d8de7b884c90b17d271d34cae8 \ + --hash=sha256:9afd0d69429172c796164fd7fe8e821ade9be983f51c659a38da3faaaaac44dc \ + --hash=sha256:9c75eff897786ee262c9f17a48886f4e98e6cfd335e011c591c305e5d083c056 \ + --hash=sha256:b538014a87f94d92f98f34d3e6d2635478e6be6423a9ea53e4dd96210065e193 \ + --hash=sha256:b6577b8c6c8701ba8642ea9335a129836347894b666dd1ec2226830e263909d3 \ + --hash=sha256:c0376deac92377817e4fb8f347bf559b7d44ff556d9bc6f6208dd3f79f104aaf \ + --hash=sha256:cae3dde0b4b2078f31527acff6f486e23abed307ba4d3932466ba7cdd5ecec79 \ + --hash=sha256:cb5d45c4143c1dd60f98a16187fd123eda7248f84ef22244818c18d531a249d1 \ + --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ + --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 + # via + # -c python/requirements_compiled.txt + # ray + # uvicorn +wcwidth==0.2.13 \ + --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ + --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 + # via + # -c python/requirements_compiled.txt + # prompt-toolkit +websockets==11.0.3 \ + --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \ + --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \ + --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \ + --hash=sha256:0ee68fe502f9031f19d495dae2c268830df2760c0524cbac5d759921ba8c8e82 \ + --hash=sha256:1553cb82942b2a74dd9b15a018dce645d4e68674de2ca31ff13ebc2d9f283788 \ + --hash=sha256:1a073fc9ab1c8aff37c99f11f1641e16da517770e31a37265d2755282a5d28aa \ + --hash=sha256:1d2256283fa4b7f4c7d7d3e84dc2ece74d341bce57d5b9bf385df109c2a1a82f \ + --hash=sha256:1d5023a4b6a5b183dc838808087033ec5df77580485fc533e7dab2567851b0a4 \ + --hash=sha256:1fdf26fa8a6a592f8f9235285b8affa72748dc12e964a5518c6c5e8f916716f7 \ + --hash=sha256:2529338a6ff0eb0b50c7be33dc3d0e456381157a31eefc561771ee431134a97f \ + --hash=sha256:279e5de4671e79a9ac877427f4ac4ce93751b8823f276b681d04b2156713b9dd \ + --hash=sha256:2d903ad4419f5b472de90cd2d40384573b25da71e33519a67797de17ef849b69 \ + --hash=sha256:332d126167ddddec94597c2365537baf9ff62dfcc9db4266f263d455f2f031cb \ + --hash=sha256:34fd59a4ac42dff6d4681d8843217137f6bc85ed29722f2f7222bd619d15e95b \ + --hash=sha256:3580dd9c1ad0701169e4d6fc41e878ffe05e6bdcaf3c412f9d559389d0c9e016 \ + --hash=sha256:3ccc8a0c387629aec40f2fc9fdcb4b9d5431954f934da3eaf16cdc94f67dbfac \ + --hash=sha256:41f696ba95cd92dc047e46b41b26dd24518384749ed0d99bea0a941ca87404c4 \ + --hash=sha256:42cc5452a54a8e46a032521d7365da775823e21bfba2895fb7b77633cce031bb \ + --hash=sha256:4841ed00f1026dfbced6fca7d963c4e7043aa832648671b5138008dc5a8f6d99 \ + --hash=sha256:4b253869ea05a5a073ebfdcb5cb3b0266a57c3764cf6fe114e4cd90f4bfa5f5e \ + --hash=sha256:54c6e5b3d3a8936a4ab6870d46bdd6ec500ad62bde9e44462c32d18f1e9a8e54 \ + --hash=sha256:619d9f06372b3a42bc29d0cd0354c9bb9fb39c2cbc1a9c5025b4538738dbffaf \ + --hash=sha256:6505c1b31274723ccaf5f515c1824a4ad2f0d191cec942666b3d0f3aa4cb4007 \ + --hash=sha256:660e2d9068d2bedc0912af508f30bbeb505bbbf9774d98def45f68278cea20d3 \ + --hash=sha256:6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6 \ + --hash=sha256:68b977f21ce443d6d378dbd5ca38621755f2063d6fdb3335bda981d552cfff86 \ + --hash=sha256:69269f3a0b472e91125b503d3c0b3566bda26da0a3261c49f0027eb6075086d1 \ + --hash=sha256:6f1a3f10f836fab6ca6efa97bb952300b20ae56b409414ca85bff2ad241d2a61 \ + --hash=sha256:7622a89d696fc87af8e8d280d9b421db5133ef5b29d3f7a1ce9f1a7bf7fcfa11 \ + --hash=sha256:777354ee16f02f643a4c7f2b3eff8027a33c9861edc691a2003531f5da4f6bc8 \ + --hash=sha256:84d27a4832cc1a0ee07cdcf2b0629a8a72db73f4cf6de6f0904f6661227f256f \ + --hash=sha256:8531fdcad636d82c517b26a448dcfe62f720e1922b33c81ce695d0edb91eb931 \ + --hash=sha256:86d2a77fd490ae3ff6fae1c6ceaecad063d3cc2320b44377efdde79880e11526 \ + --hash=sha256:88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016 \ + --hash=sha256:8a34e13a62a59c871064dfd8ffb150867e54291e46d4a7cf11d02c94a5275bae \ + --hash=sha256:8c82f11964f010053e13daafdc7154ce7385ecc538989a354ccc7067fd7028fd \ + --hash=sha256:92b2065d642bf8c0a82d59e59053dd2fdde64d4ed44efe4870fa816c1232647b \ + --hash=sha256:97b52894d948d2f6ea480171a27122d77af14ced35f62e5c892ca2fae9344311 \ + --hash=sha256:9d9acd80072abcc98bd2c86c3c9cd4ac2347b5a5a0cae7ed5c0ee5675f86d9af \ + --hash=sha256:9f59a3c656fef341a99e3d63189852be7084c0e54b75734cde571182c087b152 \ + --hash=sha256:aa5003845cdd21ac0dc6c9bf661c5beddd01116f6eb9eb3c8e272353d45b3288 \ + --hash=sha256:b16fff62b45eccb9c7abb18e60e7e446998093cdcb50fed33134b9b6878836de \ + --hash=sha256:b30c6590146e53149f04e85a6e4fcae068df4289e31e4aee1fdf56a0dead8f97 \ + --hash=sha256:b58cbf0697721120866820b89f93659abc31c1e876bf20d0b3d03cef14faf84d \ + --hash=sha256:b67c6f5e5a401fc56394f191f00f9b3811fe843ee93f4a70df3c389d1adf857d \ + --hash=sha256:bceab846bac555aff6427d060f2fcfff71042dba6f5fca7dc4f75cac815e57ca \ + --hash=sha256:bee9fcb41db2a23bed96c6b6ead6489702c12334ea20a297aa095ce6d31370d0 \ + --hash=sha256:c114e8da9b475739dde229fd3bc6b05a6537a88a578358bc8eb29b4030fac9c9 \ + --hash=sha256:c1f0524f203e3bd35149f12157438f406eff2e4fb30f71221c8a5eceb3617b6b \ + --hash=sha256:c792ea4eabc0159535608fc5658a74d1a81020eb35195dd63214dcf07556f67e \ + --hash=sha256:c7f3cb904cce8e1be667c7e6fef4516b98d1a6a0635a58a57528d577ac18a128 \ + --hash=sha256:d67ac60a307f760c6e65dad586f556dde58e683fab03323221a4e530ead6f74d \ + --hash=sha256:dcacf2c7a6c3a84e720d1bb2b543c675bf6c40e460300b628bab1b1efc7c034c \ + --hash=sha256:de36fe9c02995c7e6ae6efe2e205816f5f00c22fd1fbf343d4d18c3d5ceac2f5 \ + --hash=sha256:def07915168ac8f7853812cc593c71185a16216e9e4fa886358a17ed0fd9fcf6 \ + --hash=sha256:df41b9bc27c2c25b486bae7cf42fccdc52ff181c8c387bfd026624a491c2671b \ + --hash=sha256:e052b8467dd07d4943936009f46ae5ce7b908ddcac3fda581656b1b19c083d9b \ + --hash=sha256:e063b1865974611313a3849d43f2c3f5368093691349cf3c7c8f8f75ad7cb280 \ + --hash=sha256:e1459677e5d12be8bbc7584c35b992eea142911a6236a3278b9b5ce3326f282c \ + --hash=sha256:e1a99a7a71631f0efe727c10edfba09ea6bee4166a6f9c19aafb6c0b5917d09c \ + --hash=sha256:e590228200fcfc7e9109509e4d9125eace2042fd52b595dd22bbc34bb282307f \ + --hash=sha256:e6316827e3e79b7b8e7d8e3b08f4e331af91a48e794d5d8b099928b6f0b85f20 \ + --hash=sha256:e7837cb169eca3b3ae94cc5787c4fed99eef74c0ab9506756eea335e0d6f3ed8 \ + --hash=sha256:e848f46a58b9fcf3d06061d17be388caf70ea5b8cc3466251963c8345e13f7eb \ + --hash=sha256:ed058398f55163a79bb9f06a90ef9ccc063b204bb346c4de78efc5d15abfe602 \ + --hash=sha256:f2e58f2c36cc52d41f2659e4c0cbf7353e28c8c9e63e30d8c6d3494dc9fdedcf \ + --hash=sha256:f467ba0050b7de85016b43f5a22b46383ef004c4f672148a8abf32bc999a87f0 \ + --hash=sha256:f61bdb1df43dc9c131791fbc2355535f9024b9a04398d3bd0684fc16ab07df74 \ + --hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \ + --hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564 + # via + # -c python/requirements_compiled.txt + # uvicorn +yarl==1.18.3 \ + --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ + --hash=sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193 \ + --hash=sha256:045b8482ce9483ada4f3f23b3774f4e1bf4f23a2d5c912ed5170f68efb053318 \ + --hash=sha256:09c7907c8548bcd6ab860e5f513e727c53b4a714f459b084f6580b49fa1b9cee \ + --hash=sha256:0b0cad37311123211dc91eadcb322ef4d4a66008d3e1bdc404808992260e1a0e \ + --hash=sha256:0b3c92fa08759dbf12b3a59579a4096ba9af8dd344d9a813fc7f5070d86bbab1 \ + --hash=sha256:0fb2171a4486bb075316ee754c6d8382ea6eb8b399d4ec62fde2b591f879778a \ + --hash=sha256:1a74a13a4c857a84a845505fd2d68e54826a2cd01935a96efb1e9d86c728e186 \ + --hash=sha256:1d407181cfa6e70077df3377938c08012d18893f9f20e92f7d2f314a437c30b1 \ + --hash=sha256:1dd4bdd05407ced96fed3d7f25dbbf88d2ffb045a0db60dbc247f5b3c5c25d50 \ + --hash=sha256:25b411eddcfd56a2f0cd6a384e9f4f7aa3efee14b188de13048c25b5e91f1640 \ + --hash=sha256:2d06d3005e668744e11ed80812e61efd77d70bb7f03e33c1598c301eea20efbb \ + --hash=sha256:2ec9bbba33b2d00999af4631a3397d1fd78290c48e2a3e52d8dd72db3a067ac8 \ + --hash=sha256:3236da9272872443f81fedc389bace88408f64f89f75d1bdb2256069a8730ccc \ + --hash=sha256:35098b24e0327fc4ebdc8ffe336cee0a87a700c24ffed13161af80124b7dc8e5 \ + --hash=sha256:41f7ce59d6ee7741af71d82020346af364949314ed3d87553763a2df1829cc58 \ + --hash=sha256:436c4fc0a4d66b2badc6c5fc5ef4e47bb10e4fd9bf0c79524ac719a01f3607c2 \ + --hash=sha256:4891ed92157e5430874dad17b15eb1fda57627710756c27422200c52d8a4e393 \ + --hash=sha256:4ac515b860c36becb81bb84b667466885096b5fc85596948548b667da3bf9f24 \ + --hash=sha256:5094d9206c64181d0f6e76ebd8fb2f8fe274950a63890ee9e0ebfd58bf9d787b \ + --hash=sha256:54d6921f07555713b9300bee9c50fb46e57e2e639027089b1d795ecd9f7fa910 \ + --hash=sha256:578e281c393af575879990861823ef19d66e2b1d0098414855dd367e234f5b3c \ + --hash=sha256:5a3f356548e34a70b0172d8890006c37be92995f62d95a07b4a42e90fba54272 \ + --hash=sha256:602d98f2c2d929f8e697ed274fbadc09902c4025c5a9963bf4e9edfc3ab6f7ed \ + --hash=sha256:61b1a825a13bef4a5f10b1885245377d3cd0bf87cba068e1d9a88c2ae36880e1 \ + --hash=sha256:61e5e68cb65ac8f547f6b5ef933f510134a6bf31bb178be428994b0cb46c2a04 \ + --hash=sha256:61ee62ead9b68b9123ec24bc866cbef297dd266175d53296e2db5e7f797f902d \ + --hash=sha256:6333c5a377c8e2f5fae35e7b8f145c617b02c939d04110c76f29ee3676b5f9a5 \ + --hash=sha256:6748dbf9bfa5ba1afcc7556b71cda0d7ce5f24768043a02a58846e4a443d808d \ + --hash=sha256:67a283dd2882ac98cc6318384f565bffc751ab564605959df4752d42483ad889 \ + --hash=sha256:75674776d96d7b851b6498f17824ba17849d790a44d282929c42dbb77d4f17ae \ + --hash=sha256:757e81cae69244257d125ff31663249b3013b5dc0a8520d73694aed497fb195b \ + --hash=sha256:77a6e85b90a7641d2e07184df5557132a337f136250caafc9ccaa4a2a998ca2c \ + --hash=sha256:7c33dd1931a95e5d9a772d0ac5e44cac8957eaf58e3c8da8c1414de7dd27c576 \ + --hash=sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34 \ + --hash=sha256:7e2ee16578af3b52ac2f334c3b1f92262f47e02cc6193c598502bd46f5cd1477 \ + --hash=sha256:80316a8bd5109320d38eef8833ccf5f89608c9107d02d2a7f985f98ed6876990 \ + --hash=sha256:82123d0c954dc58db301f5021a01854a85bf1f3bb7d12ae0c01afc414a882ca2 \ + --hash=sha256:84b2deecba4a3f1a398df819151eb72d29bfeb3b69abb145a00ddc8d30094512 \ + --hash=sha256:8503ad47387b8ebd39cbbbdf0bf113e17330ffd339ba1144074da24c545f0069 \ + --hash=sha256:877d209b6aebeb5b16c42cbb377f5f94d9e556626b1bfff66d7b0d115be88d0a \ + --hash=sha256:8874027a53e3aea659a6d62751800cf6e63314c160fd607489ba5c2edd753cf6 \ + --hash=sha256:88a19f62ff30117e706ebc9090b8ecc79aeb77d0b1f5ec10d2d27a12bc9f66d0 \ + --hash=sha256:8d39d351e7faf01483cc7ff7c0213c412e38e5a340238826be7e0e4da450fdc8 \ + --hash=sha256:90adb47ad432332d4f0bc28f83a5963f426ce9a1a8809f5e584e704b82685dcb \ + --hash=sha256:913829534200eb0f789d45349e55203a091f45c37a2674678744ae52fae23efa \ + --hash=sha256:93b2e109287f93db79210f86deb6b9bbb81ac32fc97236b16f7433db7fc437d8 \ + --hash=sha256:9d41beda9dc97ca9ab0b9888cb71f7539124bc05df02c0cff6e5acc5a19dcc6e \ + --hash=sha256:a440a2a624683108a1b454705ecd7afc1c3438a08e890a1513d468671d90a04e \ + --hash=sha256:a4bb030cf46a434ec0225bddbebd4b89e6471814ca851abb8696170adb163985 \ + --hash=sha256:a9ca04806f3be0ac6d558fffc2fdf8fcef767e0489d2684a21912cc4ed0cd1b8 \ + --hash=sha256:ac1801c45cbf77b6c99242eeff4fffb5e4e73a800b5c4ad4fc0be5def634d2e1 \ + --hash=sha256:ac36703a585e0929b032fbaab0707b75dc12703766d0b53486eabd5139ebadd5 \ + --hash=sha256:b1771de9944d875f1b98a745bc547e684b863abf8f8287da8466cf470ef52690 \ + --hash=sha256:b464c4ab4bfcb41e3bfd3f1c26600d038376c2de3297760dfe064d2cb7ea8e10 \ + --hash=sha256:b4f6450109834af88cb4cc5ecddfc5380ebb9c228695afc11915a0bf82116789 \ + --hash=sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b \ + --hash=sha256:b643562c12680b01e17239be267bc306bbc6aac1f34f6444d1bded0c5ce438ca \ + --hash=sha256:b958ddd075ddba5b09bb0be8a6d9906d2ce933aee81100db289badbeb966f54e \ + --hash=sha256:b9d60031cf568c627d028239693fd718025719c02c9f55df0a53e587aab951b5 \ + --hash=sha256:ba23302c0c61a9999784e73809427c9dbedd79f66a13d84ad1b1943802eaaf59 \ + --hash=sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9 \ + --hash=sha256:c017a3b6df3a1bd45b9fa49a0f54005e53fbcad16633870104b66fa1a30a29d8 \ + --hash=sha256:c1e1cc06da1491e6734f0ea1e6294ce00792193c463350626571c287c9a704db \ + --hash=sha256:c654d5207c78e0bd6d749f6dae1dcbbfde3403ad3a4b11f3c5544d9906969dde \ + --hash=sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7 \ + --hash=sha256:c7d79f7d9aabd6011004e33b22bc13056a3e3fb54794d138af57f5ee9d9032cb \ + --hash=sha256:ccaa3a4b521b780a7e771cc336a2dba389a0861592bbce09a476190bb0c8b4b3 \ + --hash=sha256:ccd17349166b1bee6e529b4add61727d3f55edb7babbe4069b5764c9587a8cc6 \ + --hash=sha256:ce1af883b94304f493698b00d0f006d56aea98aeb49d75ec7d98cd4a777e9285 \ + --hash=sha256:d0e883008013c0e4aef84dcfe2a0b172c4d23c2669412cf5b3371003941f72bb \ + --hash=sha256:d980e0325b6eddc81331d3f4551e2a333999fb176fd153e075c6d1c2530aa8a8 \ + --hash=sha256:e17c9361d46a4d5addf777c6dd5eab0715a7684c2f11b88c67ac37edfba6c482 \ + --hash=sha256:e2c08cc9b16f4f4bc522771d96734c7901e7ebef70c6c5c35dd0f10845270bcd \ + --hash=sha256:e35ef8683211db69ffe129a25d5634319a677570ab6b2eba4afa860f54eeaf75 \ + --hash=sha256:e3b9fd71836999aad54084906f8663dffcd2a7fb5cdafd6c37713b2e72be1760 \ + --hash=sha256:ef9f7768395923c3039055c14334ba4d926f3baf7b776c923c93d80195624782 \ + --hash=sha256:f52a265001d830bc425f82ca9eabda94a64a4d753b07d623a9f2863fde532b53 \ + --hash=sha256:f91c4803173928a25e1a55b943c81f55b8872f0018be83e3ad4938adffb77dd2 \ + --hash=sha256:fbd6748e8ab9b41171bb95c6142faf068f5ef1511935a0aa07025438dd9a9bc1 \ + --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ + --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 + # via + # -c python/requirements_compiled.txt + # aiohttp +zipp==3.19.2 \ + --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c + # via + # -c python/requirements_compiled.txt + # importlib-metadata diff --git a/python/deplocks/ray_img/ray_img_py312.lock b/python/deplocks/ray_img/ray_img_py312.lock new file mode 100644 index 000000000000..4cc81338771e --- /dev/null +++ b/python/deplocks/ray_img/ray_img_py312.lock @@ -0,0 +1,2172 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.12 --find-links=.whl/ -c python/requirements_compiled.txt - -o python/deplocks/ray_img/ray_img_py312.lock +--index-url https://pypi.org/simple +--extra-index-url https://download.pytorch.org/whl/cpu +--find-links .whl/ +--find-links https://data.pyg.org/whl/torch-2.3.0+cpu.html + +aiohappyeyeballs==2.6.1 \ + --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ + --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 + # via + # -c python/requirements_compiled.txt + # aiohttp +aiohttp==3.11.16 \ + --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ + --hash=sha256:0902e887b0e1d50424112f200eb9ae3dfed6c0d0a19fc60f633ae5a57c809656 \ + --hash=sha256:09b00dd520d88eac9d1768439a59ab3d145065c91a8fab97f900d1b5f802895e \ + --hash=sha256:0a2f451849e6b39e5c226803dcacfa9c7133e9825dcefd2f4e837a2ec5a3bb98 \ + --hash=sha256:0a950c2eb8ff17361abd8c85987fd6076d9f47d040ebffce67dce4993285e973 \ + --hash=sha256:0ad1fb47da60ae1ddfb316f0ff16d1f3b8e844d1a1e154641928ea0583d486ed \ + --hash=sha256:13ceac2c5cdcc3f64b9015710221ddf81c900c5febc505dbd8f810e770011540 \ + --hash=sha256:14461157d8426bcb40bd94deb0450a6fa16f05129f7da546090cebf8f3123b0f \ + --hash=sha256:16f8a2c9538c14a557b4d309ed4d0a7c60f0253e8ed7b6c9a2859a7582f8b1b8 \ + --hash=sha256:17ae4664031aadfbcb34fd40ffd90976671fa0c0286e6c4113989f78bebab37a \ + --hash=sha256:1ce63ae04719513dd2651202352a2beb9f67f55cb8490c40f056cea3c5c355ce \ + --hash=sha256:23a15727fbfccab973343b6d1b7181bfb0b4aa7ae280f36fd2f90f5476805682 \ + --hash=sha256:2540ddc83cc724b13d1838026f6a5ad178510953302a49e6d647f6e1de82bc34 \ + --hash=sha256:37dcee4906454ae377be5937ab2a66a9a88377b11dd7c072df7a7c142b63c37c \ + --hash=sha256:38bea84ee4fe24ebcc8edeb7b54bf20f06fd53ce4d2cc8b74344c5b9620597fd \ + --hash=sha256:3ab3367bb7f61ad18793fea2ef71f2d181c528c87948638366bf1de26e239183 \ + --hash=sha256:3ad1d59fd7114e6a08c4814983bb498f391c699f3c78712770077518cae63ff7 \ + --hash=sha256:3b4e6db8dc4879015b9955778cfb9881897339c8fab7b3676f8433f849425913 \ + --hash=sha256:3e061b09f6fa42997cf627307f220315e313ece74907d35776ec4373ed718b86 \ + --hash=sha256:42864e70a248f5f6a49fdaf417d9bc62d6e4d8ee9695b24c5916cb4bb666c802 \ + --hash=sha256:493910ceb2764f792db4dc6e8e4b375dae1b08f72e18e8f10f18b34ca17d0979 \ + --hash=sha256:4d0c970c0d602b1017e2067ff3b7dac41c98fef4f7472ec2ea26fd8a4e8c2149 \ + --hash=sha256:54eb3aead72a5c19fad07219acd882c1643a1027fbcdefac9b502c267242f955 \ + --hash=sha256:56a3443aca82abda0e07be2e1ecb76a050714faf2be84256dae291182ba59049 \ + --hash=sha256:576f5ca28d1b3276026f7df3ec841ae460e0fc3aac2a47cbf72eabcfc0f102e1 \ + --hash=sha256:58ede86453a6cf2d6ce40ef0ca15481677a66950e73b0a788917916f7e35a0bb \ + --hash=sha256:61c721764e41af907c9d16b6daa05a458f066015abd35923051be8705108ed17 \ + --hash=sha256:634d96869be6c4dc232fc503e03e40c42d32cfaa51712aee181e922e61d74814 \ + --hash=sha256:696ef00e8a1f0cec5e30640e64eca75d8e777933d1438f4facc9c0cdf288a810 \ + --hash=sha256:69a2cbd61788d26f8f1e626e188044834f37f6ae3f937bd9f08b65fc9d7e514e \ + --hash=sha256:6a792ce34b999fbe04a7a71a90c74f10c57ae4c51f65461a411faa70e154154e \ + --hash=sha256:6ac13b71761e49d5f9e4d05d33683bbafef753e876e8e5a7ef26e937dd766713 \ + --hash=sha256:6fdec0213244c39973674ca2a7f5435bf74369e7d4e104d6c7473c81c9bcc8c4 \ + --hash=sha256:72b1b03fb4655c1960403c131740755ec19c5898c82abd3961c364c2afd59fe7 \ + --hash=sha256:745f1ed5e2c687baefc3c5e7b4304e91bf3e2f32834d07baaee243e349624b24 \ + --hash=sha256:776c8e959a01e5e8321f1dec77964cb6101020a69d5a94cd3d34db6d555e01f7 \ + --hash=sha256:780df0d837276276226a1ff803f8d0fa5f8996c479aeef52eb040179f3156cbd \ + --hash=sha256:78e6e23b954644737e385befa0deb20233e2dfddf95dd11e9db752bdd2a294d3 \ + --hash=sha256:7951decace76a9271a1ef181b04aa77d3cc309a02a51d73826039003210bdc86 \ + --hash=sha256:7ba92a2d9ace559a0a14b03d87f47e021e4fa7681dc6970ebbc7b447c7d4b7cd \ + --hash=sha256:7f6428fee52d2bcf96a8aa7b62095b190ee341ab0e6b1bcf50c615d7966fd45b \ + --hash=sha256:87944bd16b7fe6160607f6a17808abd25f17f61ae1e26c47a491b970fb66d8cb \ + --hash=sha256:87a6e922b2b2401e0b0cf6b976b97f11ec7f136bfed445e16384fbf6fd5e8602 \ + --hash=sha256:8cb0688a8d81c63d716e867d59a9ccc389e97ac7037ebef904c2b89334407180 \ + --hash=sha256:8df6612df74409080575dca38a5237282865408016e65636a76a2eb9348c2567 \ + --hash=sha256:911a6e91d08bb2c72938bc17f0a2d97864c531536b7832abee6429d5296e5b27 \ + --hash=sha256:92b7ee222e2b903e0a4b329a9943d432b3767f2d5029dbe4ca59fb75223bbe2e \ + --hash=sha256:938f756c2b9374bbcc262a37eea521d8a0e6458162f2a9c26329cc87fdf06534 \ + --hash=sha256:9756d9b9d4547e091f99d554fbba0d2a920aab98caa82a8fb3d3d9bee3c9ae85 \ + --hash=sha256:98b88a2bf26965f2015a771381624dd4b0839034b70d406dc74fd8be4cc053e3 \ + --hash=sha256:9b751a6306f330801665ae69270a8a3993654a85569b3469662efaad6cf5cc50 \ + --hash=sha256:a2a450bcce4931b295fc0848f384834c3f9b00edfc2150baafb4488c27953de6 \ + --hash=sha256:a3814760a1a700f3cfd2f977249f1032301d0a12c92aba74605cfa6ce9f78489 \ + --hash=sha256:a5abcbba9f4b463a45c8ca8b7720891200658f6f46894f79517e6cd11f3405ca \ + --hash=sha256:a6db7458ab89c7d80bc1f4e930cc9df6edee2200127cfa6f6e080cf619eddfbd \ + --hash=sha256:ad497f38a0d6c329cb621774788583ee12321863cd4bd9feee1effd60f2ad133 \ + --hash=sha256:ad9509ffb2396483ceacb1eee9134724443ee45b92141105a4645857244aecc8 \ + --hash=sha256:bbcba75fe879ad6fd2e0d6a8d937f34a571f116a0e4db37df8079e738ea95c71 \ + --hash=sha256:c10d85e81d0b9ef87970ecbdbfaeec14a361a7fa947118817fcea8e45335fa46 \ + --hash=sha256:c15b2271c44da77ee9d822552201180779e5e942f3a71fb74e026bf6172ff287 \ + --hash=sha256:ca37057625693d097543bd88076ceebeb248291df9d6ca8481349efc0b05dcd0 \ + --hash=sha256:cc3a145479a76ad0ed646434d09216d33d08eef0d8c9a11f5ae5cdc37caa3540 \ + --hash=sha256:ccf10f16ab498d20e28bc2b5c1306e9c1512f2840f7b6a67000a517a4b37d5ee \ + --hash=sha256:cd464ba806e27ee24a91362ba3621bfc39dbbb8b79f2e1340201615197370f7c \ + --hash=sha256:d007aa39a52d62373bd23428ba4a2546eed0e7643d7bf2e41ddcefd54519842c \ + --hash=sha256:d0666afbe984f6933fe72cd1f1c3560d8c55880a0bdd728ad774006eb4241ecd \ + --hash=sha256:d07502cc14ecd64f52b2a74ebbc106893d9a9717120057ea9ea1fd6568a747e7 \ + --hash=sha256:d489d9778522fbd0f8d6a5c6e48e3514f11be81cb0a5954bdda06f7e1594b321 \ + --hash=sha256:df7db76400bf46ec6a0a73192b14c8295bdb9812053f4fe53f4e789f3ea66bbb \ + --hash=sha256:e3538bc9fe1b902bef51372462e3d7c96fce2b566642512138a480b7adc9d508 \ + --hash=sha256:e87fd812899aa78252866ae03a048e77bd11b80fb4878ce27c23cade239b42b2 \ + --hash=sha256:ecdb8173e6c7aa09eee342ac62e193e6904923bd232e76b4157ac0bfa670609f \ + --hash=sha256:f244b8e541f414664889e2c87cac11a07b918cb4b540c36f7ada7bfa76571ea2 \ + --hash=sha256:f4065145bf69de124accdd17ea5f4dc770da0a6a6e440c53f6e0a8c27b3e635c \ + --hash=sha256:f420bfe862fb357a6d76f2065447ef6f484bc489292ac91e29bc65d2d7a2c84d \ + --hash=sha256:f6ddd90d9fb4b501c97a4458f1c1720e42432c26cb76d28177c5b5ad4e332601 \ + --hash=sha256:fa73e8c2656a3653ae6c307b3f4e878a21f87859a9afab228280ddccd7369d71 \ + --hash=sha256:fadbb8f1d4140825069db3fedbbb843290fd5f5bc0a5dbd7eaf81d91bf1b003b \ + --hash=sha256:fb3d0cc5cdb926090748ea60172fa8a213cec728bd6c54eae18b96040fcd6227 \ + --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ + --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb + # via + # -c python/requirements_compiled.txt + # aiohttp-cors + # ray +aiohttp-cors==0.7.0 \ + --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ + --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d + # via + # -c python/requirements_compiled.txt + # ray +aiosignal==1.3.1 \ + --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ + --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 + # via + # -c python/requirements_compiled.txt + # aiohttp +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled.txt + # kombu +annotated-types==0.6.0 \ + --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ + --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d + # via + # -c python/requirements_compiled.txt + # pydantic +anyio==3.7.1 \ + --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ + --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 + # via + # -c python/requirements_compiled.txt + # starlette + # watchfiles +attrs==25.1.0 \ + --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ + --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a + # via + # -c python/requirements_compiled.txt + # aiohttp + # jsonschema + # referencing +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled.txt + # celery +cachetools==5.5.2 \ + --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ + --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a + # via + # -c python/requirements_compiled.txt + # google-auth +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled.txt + # ray +certifi==2025.1.31 \ + --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ + --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe + # via + # -c python/requirements_compiled.txt + # requests +cffi==1.16.0 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ + --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \ + --hash=sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417 \ + --hash=sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab \ + --hash=sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520 \ + --hash=sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36 \ + --hash=sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743 \ + --hash=sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8 \ + --hash=sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed \ + --hash=sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684 \ + --hash=sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56 \ + --hash=sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324 \ + --hash=sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d \ + --hash=sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235 \ + --hash=sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e \ + --hash=sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088 \ + --hash=sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000 \ + --hash=sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7 \ + --hash=sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e \ + --hash=sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673 \ + --hash=sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c \ + --hash=sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe \ + --hash=sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2 \ + --hash=sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098 \ + --hash=sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8 \ + --hash=sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a \ + --hash=sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0 \ + --hash=sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b \ + --hash=sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896 \ + --hash=sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e \ + --hash=sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9 \ + --hash=sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2 \ + --hash=sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b \ + --hash=sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6 \ + --hash=sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404 \ + --hash=sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f \ + --hash=sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0 \ + --hash=sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4 \ + --hash=sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc \ + --hash=sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936 \ + --hash=sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba \ + --hash=sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872 \ + --hash=sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb \ + --hash=sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614 \ + --hash=sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1 \ + --hash=sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d \ + --hash=sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969 \ + --hash=sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b \ + --hash=sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4 \ + --hash=sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627 \ + --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ + --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 + # via + # -c python/requirements_compiled.txt + # cryptography +charset-normalizer==3.3.2 \ + --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ + --hash=sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087 \ + --hash=sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786 \ + --hash=sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8 \ + --hash=sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09 \ + --hash=sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185 \ + --hash=sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574 \ + --hash=sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e \ + --hash=sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519 \ + --hash=sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898 \ + --hash=sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269 \ + --hash=sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3 \ + --hash=sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f \ + --hash=sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6 \ + --hash=sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8 \ + --hash=sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a \ + --hash=sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73 \ + --hash=sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc \ + --hash=sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714 \ + --hash=sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2 \ + --hash=sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc \ + --hash=sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce \ + --hash=sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d \ + --hash=sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e \ + --hash=sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6 \ + --hash=sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269 \ + --hash=sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96 \ + --hash=sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d \ + --hash=sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a \ + --hash=sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4 \ + --hash=sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77 \ + --hash=sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d \ + --hash=sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0 \ + --hash=sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed \ + --hash=sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068 \ + --hash=sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac \ + --hash=sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25 \ + --hash=sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8 \ + --hash=sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab \ + --hash=sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26 \ + --hash=sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2 \ + --hash=sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db \ + --hash=sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f \ + --hash=sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5 \ + --hash=sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99 \ + --hash=sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c \ + --hash=sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d \ + --hash=sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811 \ + --hash=sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa \ + --hash=sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a \ + --hash=sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03 \ + --hash=sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b \ + --hash=sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04 \ + --hash=sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c \ + --hash=sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001 \ + --hash=sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458 \ + --hash=sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389 \ + --hash=sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99 \ + --hash=sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985 \ + --hash=sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537 \ + --hash=sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238 \ + --hash=sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f \ + --hash=sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d \ + --hash=sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796 \ + --hash=sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a \ + --hash=sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143 \ + --hash=sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8 \ + --hash=sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c \ + --hash=sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5 \ + --hash=sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5 \ + --hash=sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711 \ + --hash=sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4 \ + --hash=sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6 \ + --hash=sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c \ + --hash=sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7 \ + --hash=sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4 \ + --hash=sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b \ + --hash=sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae \ + --hash=sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12 \ + --hash=sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c \ + --hash=sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae \ + --hash=sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8 \ + --hash=sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887 \ + --hash=sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b \ + --hash=sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4 \ + --hash=sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f \ + --hash=sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5 \ + --hash=sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33 \ + --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ + --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 + # via + # -c python/requirements_compiled.txt + # requests +click==8.1.7 \ + --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ + --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de + # via + # -c python/requirements_compiled.txt + # celery + # click-didyoumean + # click-plugins + # click-repl + # ray + # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled.txt + # celery +cloudpickle==3.1.1 \ + --hash=sha256:b216fa8ae4019d5482a8ac3c95d8f6346115d8835911fd4aefd1a445e4242c64 \ + --hash=sha256:c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e + # via gymnasium +colorful==0.5.5 \ + --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ + --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d + # via + # -c python/requirements_compiled.txt + # ray +cryptography==44.0.3 \ + --hash=sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259 \ + --hash=sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43 \ + --hash=sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645 \ + --hash=sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8 \ + --hash=sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44 \ + --hash=sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d \ + --hash=sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f \ + --hash=sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d \ + --hash=sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54 \ + --hash=sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9 \ + --hash=sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137 \ + --hash=sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f \ + --hash=sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c \ + --hash=sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334 \ + --hash=sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c \ + --hash=sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b \ + --hash=sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2 \ + --hash=sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375 \ + --hash=sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88 \ + --hash=sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5 \ + --hash=sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647 \ + --hash=sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c \ + --hash=sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359 \ + --hash=sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5 \ + --hash=sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d \ + --hash=sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028 \ + --hash=sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01 \ + --hash=sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904 \ + --hash=sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d \ + --hash=sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93 \ + --hash=sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06 \ + --hash=sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff \ + --hash=sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76 \ + --hash=sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff \ + --hash=sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759 \ + --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ + --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 + # via + # -c python/requirements_compiled.txt + # pyopenssl +cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ + --hash=sha256:230f8a8e99c81a653baa0ed00819990c0ed1f0cf0298214786b5e323461dc61a \ + --hash=sha256:2d16eaa2d086e416ac13467d4ff3184b9a081fe76b761ce51d4a46ec1c4bd28a \ + --hash=sha256:432273fd4b61a284f7d705d08b8291403548fd422bcbd945635cc155bc6a923d \ + --hash=sha256:4c51a1062a3c5a826b0425952d229ffe73b1791656a31de95b318117e67a9576 \ + --hash=sha256:4c8e9fdb1f3ffc3151808f8bb8c871518d2783e1be8b53792b698a840543d60c \ + --hash=sha256:51b1d6cb83d82dfa306c9efaeb4d57f24bad3041ebd8716d61072676abbcf67b \ + --hash=sha256:52185a2cf95d3bac2c3fda95c9c8e06a985b5a00cd2e587d3caace337db33899 \ + --hash=sha256:5afb6658faa22f21479ae2c0a07254df31c0aebc36907a64a1f6be4ecc9e96da \ + --hash=sha256:d3dc91ef9c4104652195eea4b282d343ecad653021efe20d1c8dd8dfe8ccfd86 \ + --hash=sha256:d60d1e124592cb82a5f3f45b3e7bee7bda7b72a743029f275e9d6b125f338c60 \ + --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ + --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa + # via + # -c python/requirements_compiled.txt + # ray +distlib==0.3.7 \ + --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ + --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 + # via + # -c python/requirements_compiled.txt + # virtualenv +dm-tree==0.1.8 \ + --hash=sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6 \ + --hash=sha256:09964470f76a5201aff2e8f9b26842976de7889300676f927930f6285e256760 \ + --hash=sha256:0d3172394079a86c3a759179c65f64c48d1a42b89495fcf38976d11cc3bb952c \ + --hash=sha256:0e9620ccf06393eb6b613b5e366469304622d4ea96ae6540b28a33840e6c89cf \ + --hash=sha256:0fcaabbb14e7980377439e7140bd05552739ca5e515ecb3119f234acee4b9430 \ + --hash=sha256:1607ce49aa42f010d1e5e616d92ce899d66835d4d8bea49679582435285515de \ + --hash=sha256:181c35521d480d0365f39300542cb6cd7fd2b77351bb43d7acfda15aef63b317 \ + --hash=sha256:1d7c26e431fc93cc7e0cba867eb000db6a05f6f2b25af11ac4e9dada88fc5bca \ + --hash=sha256:1fe962015b2fe1282892b28ebe962faed53c7f98d942da9a4625cbf27baef913 \ + --hash=sha256:250b692fb75f45f02e2f58fbef9ab338904ef334b90557565621fa251df267cf \ + --hash=sha256:2869228d9c619074de501a3c10dc7f07c75422f8fab36ecdcb859b6f1b1ec3ef \ + --hash=sha256:28c52cbf4f8b3dbd0beaedf44f69fa85eec5e9dede612e08035e06ada6ec9426 \ + --hash=sha256:2f7915660f59c09068e428613c480150180df1060561fd0d1470684ae7007bd1 \ + --hash=sha256:343a4a4ebaa127451ff971254a4be4084eb4bdc0b2513c32b46f6f728fd03f9e \ + --hash=sha256:35cc164a79336bfcfafb47e5f297898359123bbd3330c1967f0c4994f9cf9f60 \ + --hash=sha256:378cc8ad93c5fe3590f405a309980721f021c790ca1bdf9b15bb1d59daec57f5 \ + --hash=sha256:39070ba268c0491af9fe7a58644d99e8b4f2cde6e5884ba3380bddc84ed43d5f \ + --hash=sha256:435227cf3c5dc63f4de054cf3d00183790bd9ead4c3623138c74dde7f67f521b \ + --hash=sha256:5483dca4d7eb1a0d65fe86d3b6a53ae717face83c1f17e0887b1a4a64ae5c410 \ + --hash=sha256:694c3654cfd2a81552c08ec66bb5c4a3d48fa292b9a181880fb081c36c5b9134 \ + --hash=sha256:75c5d528bb992981c20793b6b453e91560784215dffb8a5440ba999753c14ceb \ + --hash=sha256:803bfc53b4659f447ac694dbd04235f94a73ef7c1fd1e0df7c84ac41e0bc963b \ + --hash=sha256:81fce77f22a302d7a5968aebdf4efafef4def7ce96528719a354e6990dcd49c7 \ + --hash=sha256:83b7764de0d855338abefc6e3ee9fe40d301668310aa3baea3f778ff051f4393 \ + --hash=sha256:8c60a7eadab64c2278861f56bca320b2720f163dca9d7558103c3b77f2416571 \ + --hash=sha256:8ed3564abed97c806db122c2d3e1a2b64c74a63debe9903aad795167cc301368 \ + --hash=sha256:94d3f0826311f45ee19b75f5b48c99466e4218a0489e81c0f0167bda50cacf22 \ + --hash=sha256:96a548a406a6fb15fe58f6a30a57ff2f2aafbf25f05afab00c8f5e5977b6c715 \ + --hash=sha256:a5d819c38c03f0bb5b3b3703c60e4b170355a0fc6b5819325bf3d4ceb3ae7e80 \ + --hash=sha256:ad16ceba90a56ec47cf45b21856d14962ac314787975ef786efb5e6e9ca75ec7 \ + --hash=sha256:af4b3d372f2477dcd89a6e717e4a575ca35ccc20cc4454a8a4b6f8838a00672d \ + --hash=sha256:b095ba4f8ca1ba19350fd53cf1f8f3eb0bd406aa28af64a6dfc86707b32a810a \ + --hash=sha256:b9bd9b9ccb59409d33d51d84b7668010c04c2af7d4a371632874c1ca356cff3d \ + --hash=sha256:b9f89a454e98806b44fe9d40ec9eee61f848388f7e79ac2371a55679bd5a3ac6 \ + --hash=sha256:bb2d109f42190225112da899b9f3d46d0d5f26aef501c61e43529fe9322530b5 \ + --hash=sha256:c0a94aba18a35457a1b5cd716fd7b46c5dafdc4cf7869b4bae665b91c4682a8e \ + --hash=sha256:c5c8c12e3fda754ef6af94161bacdaeda816d941995fac415d6855c6c386af68 \ + --hash=sha256:d1612fcaecd79023dbc6a6ae48d51a80beb5c385d6f3f6d71688e57bc8d07de8 \ + --hash=sha256:d16e1f2a073604cfcc09f7131ae8d534674f43c3aef4c25742eae295bc60d04f \ + --hash=sha256:d20f2faa3672b52e5013f4077117bfb99c4cfc0b445d3bde1584c34032b57436 \ + --hash=sha256:d40fa4106ca6edc66760246a08f500ec0c85ef55c762fb4a363f6ee739ba02ee \ + --hash=sha256:de287fabc464b8734be251e46e06aa9aa1001f34198da2b6ce07bd197172b9cb \ + --hash=sha256:e4d714371bb08839e4e5e29024fc95832d9affe129825ef38836b143028bd144 \ + --hash=sha256:ea9e59e0451e7d29aece402d9f908f2e2a80922bcde2ebfd5dcb07750fcbfee8 \ + --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ + --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d + # via + # -c python/requirements_compiled.txt + # ray +farama-notifications==0.0.4 \ + --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ + --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae + # via + # -c python/requirements_compiled.txt + # gymnasium +fastapi==0.115.12 \ + --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ + --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d + # via + # -c python/requirements_compiled.txt + # ray +fastrlock==0.8.2 ; sys_platform != 'darwin' \ + --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ + --hash=sha256:07ed3c7b3867c05a3d6be4ced200c7767000f3431b9be6da66972822dd86e8be \ + --hash=sha256:08315bde19d0c2e6b06593d5a418be3dc8f9b1ee721afa96867b9853fceb45cf \ + --hash=sha256:11bbbbc526363955aeddb9eec4cee2a0012322b7b2f15b54f44454fcf4fd398a \ + --hash=sha256:17734e2e5af4c07ddb0fb10bd484e062c22de3be6b67940b9cc6ec2f18fa61ba \ + --hash=sha256:1b15430b93d7eb3d56f6ff690d2ebecb79ed0e58248427717eba150a508d1cd7 \ + --hash=sha256:1fed2f4797ad68e9982038423018cf08bec5f4ce9fed63a94a790773ed6a795c \ + --hash=sha256:2074548a335fcf7d19ebb18d9208da9e33b06f745754466a7e001d2b1c58dd19 \ + --hash=sha256:2587cedbb36c7988e707d83f0f1175c1f882f362b5ebbee25d70218ea33d220d \ + --hash=sha256:25945f962c7bd808415cfde3da624d4399d4ea71ed8918538375f16bceb79e1c \ + --hash=sha256:27786c62a400e282756ae1b090bcd7cfa35f28270cff65a9e7b27a5327a32561 \ + --hash=sha256:2c1719ddc8218b01e82fb2e82e8451bd65076cb96d7bef4477194bbb4305a968 \ + --hash=sha256:2d5595903444c854b99c42122b87edfe8a37cd698a4eae32f4fd1d2a7b6c115d \ + --hash=sha256:30bdbe4662992348132d03996700e1cf910d141d629179b967b146a22942264e \ + --hash=sha256:31a27a2edf482df72b91fe6c6438314d2c65290aa7becc55589d156c9b91f0da \ + --hash=sha256:320fd55bafee3eb069cfb5d6491f811a912758387ef2193840e2663e80e16f48 \ + --hash=sha256:33145acbad8317584cd64588131c7e1e286beef6280c0009b4544c91fce171d2 \ + --hash=sha256:43a241655e83e4603a152192cf022d5ca348c2f4e56dfb02e5c9c4c1a32f9cdb \ + --hash=sha256:4d63b6596368dab9e0cc66bf047e7182a56f33b34db141816a4f21f5bf958228 \ + --hash=sha256:4fb04442b6d1e2b36c774919c6bcbe3339c61b337261d4bd57e27932589095af \ + --hash=sha256:4fb2e77ff04bc4beb71d63c8e064f052ce5a6ea1e001d528d4d7f4b37d736f2e \ + --hash=sha256:5460c5ee6ced6d61ec8cd2324ebbe793a4960c4ffa2131ffff480e3b61c99ec5 \ + --hash=sha256:59344c1d46b7dec97d3f22f1cc930fafe8980b3c5bc9c9765c56738a5f1559e4 \ + --hash=sha256:5dfb78dd600a12f23fc0c3ec58f81336229fdc74501ecf378d1ce5b3f2f313ea \ + --hash=sha256:643e1e65b4f5b284427e61a894d876d10459820e93aa1e724dfb415117be24e0 \ + --hash=sha256:644ec9215cf9c4df8028d8511379a15d9c1af3e16d80e47f1b6fdc6ba118356a \ + --hash=sha256:66f2662c640bb71a1016a031eea6eef9d25c2bcdf7ffd1d1ddc5a58f9a1ced04 \ + --hash=sha256:685e656048b59d8dfde8c601f188ad53a4d719eb97080cafc8696cda6d75865e \ + --hash=sha256:7269bb3fc15587b0c191eecd95831d771a7d80f0c48929e560806b038ff3066c \ + --hash=sha256:73426f5eb2ecc10626c67cf86bd0af9e00d53e80e5c67d5ce8e18376d6abfa09 \ + --hash=sha256:75c07726c8b1a52147fd7987d6baaa318c5dced1416c3f25593e40f56e10755b \ + --hash=sha256:790fc19bccbd39426060047e53629f171a44745613bf360a045e9f9c8c4a2cea \ + --hash=sha256:7a2ccaf88ac0db153e84305d1ef0aa138cea82c6a88309066f6eaa3bc98636cd \ + --hash=sha256:87f4e01b042c84e6090dbc4fbe3415ddd69f6bc0130382323f9d3f1b8dd71b46 \ + --hash=sha256:88f079335e9da631efa64486c8207564a7bcd0c00526bb9e842e9d5b7e50a6cc \ + --hash=sha256:8c1c91a68926421f5ccbc82c85f83bd3ba593b121a46a1b9a554b3f0dd67a4bf \ + --hash=sha256:9121a894d74e65557e47e777060a495ab85f4b903e80dd73a3c940ba042920d7 \ + --hash=sha256:94e348c72a1fd1f8191f25ea056448e4f5a87b8fbf005b39d290dcb0581a48cd \ + --hash=sha256:98195866d3a9949915935d40a88e4f1c166e82e378f622c88025f2938624a90a \ + --hash=sha256:99dd6652bd6f730beadf74ef769d38c6bbd8ee6d1c15c8d138ea680b0594387f \ + --hash=sha256:9af691a9861027181d4de07ed74f0aee12a9650ac60d0a07f4320bff84b5d95f \ + --hash=sha256:a3b8b5d2935403f1b4b25ae324560e94b59593a38c0d2e7b6c9872126a9622ed \ + --hash=sha256:a3dcc876050b8f5cbc0ee84ef1e7f0c1dfe7c148f10098828bc4403683c33f10 \ + --hash=sha256:a74f5a92fa6e51c4f3c69b29c4662088b97be12f40652a21109605a175c81824 \ + --hash=sha256:ab91b0c36e95d42e1041a4907e3eefd06c482d53af3c7a77be7e214cc7cd4a63 \ + --hash=sha256:ad1bc61c7f6b0e58106aaab034916b6cb041757f708b07fbcdd9d6e1ac629225 \ + --hash=sha256:adcb9e77aa132cc6c9de2ffe7cf880a20aa8cdba21d367d1da1a412f57bddd5d \ + --hash=sha256:b22ea9bf5f9fad2b0077e944a7813f91593a4f61adf8faf734a70aed3f2b3a40 \ + --hash=sha256:b2a1c354f13f22b737621d914f3b4a8434ae69d3027a775e94b3e671756112f9 \ + --hash=sha256:b32fdf874868326351a75b1e4c02f97e802147119ae44c52d3d9da193ec34f5b \ + --hash=sha256:b3853ed4ce522598dc886160a7bab432a093051af85891fa2f5577c1dcac8ed6 \ + --hash=sha256:b443e73a4dfc7b6e0800ea4c13567b9694358e86f53bb2612a51c9e727cac67b \ + --hash=sha256:b4c9083ea89ab236b06e9ef2263971db3b4b507195fc7d5eecab95828dcae325 \ + --hash=sha256:b8ca0fe21458457077e4cb2d81e1ebdb146a00b3e9e2db6180a773f7ea905032 \ + --hash=sha256:c393af77c659a38bffbca215c0bcc8629ba4299568308dd7e4ff65d62cabed39 \ + --hash=sha256:c6bffa978793bea5e1b00e677062e53a62255439339591b70e209fa1552d5ee0 \ + --hash=sha256:ccf39ad5702e33e4d335b48ef9d56e21619b529b7f7471b5211419f380329b62 \ + --hash=sha256:cf81e0278b645004388873e0a1f9e3bc4c9ab8c18e377b14ed1a544be4b18c9a \ + --hash=sha256:d34546ad2e4a480b94b6797bcc5a322b3c705c4c74c3e4e545c4a3841c1b2d59 \ + --hash=sha256:d47713ffe6d4a627fbf078be9836a95ac106b4a0543e3841572c91e292a5d885 \ + --hash=sha256:d918dfe473291e8bfd8e13223ea5cb9b317bd9f50c280923776c377f7c64b428 \ + --hash=sha256:dbdce852e6bb66e1b8c36679d482971d69d93acf1785657522e51b7de30c3356 \ + --hash=sha256:dcc1bf0ac8a194313cf6e645e300a8a379674ceed8e0b1e910a2de3e3c28989e \ + --hash=sha256:dd961a32a7182c3891cdebca417fda67496d5d5de6ae636962254d22723bdf52 \ + --hash=sha256:ddf5d247f686aec853ddcc9a1234bfcc6f57b0a0670d2ad82fc25d8ae7e6a15f \ + --hash=sha256:e27c3cd27fbd25e5223c5c992b300cd4ee8f0a75c6f222ce65838138d853712c \ + --hash=sha256:e380ec4e6d8b26e389713995a43cb7fe56baea2d25fe073d4998c4821a026211 \ + --hash=sha256:e4bbde174a0aff5f6eeba75cf8c4c5d2a316316bc21f03a0bddca0fc3659a6f3 \ + --hash=sha256:e8b49b5743ede51e0bcf6805741f39f5e0e0fd6a172ba460cb39e3097ba803bb \ + --hash=sha256:e9904b5b37c3e5bb4a245c56bc4b7e497da57ffb8528f4fc39af9dcb168ee2e1 \ + --hash=sha256:ea96503b918fceaf40443182742b8964d47b65c5ebdea532893cb9479620000c \ + --hash=sha256:eb31fe390f03f7ae886dcc374f1099ec88526631a4cb891d399b68181f154ff0 \ + --hash=sha256:ebb32d776b61acd49f859a1d16b9e3d84e7b46d0d92aebd58acd54dc38e96664 \ + --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ + --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e + # via + # -c python/requirements_compiled.txt + # cupy-cuda12x +filelock==3.17.0 \ + --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ + --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e + # via + # -c python/requirements_compiled.txt + # ray + # virtualenv +frozenlist==1.4.1 \ + --hash=sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7 \ + --hash=sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98 \ + --hash=sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad \ + --hash=sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5 \ + --hash=sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae \ + --hash=sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e \ + --hash=sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a \ + --hash=sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701 \ + --hash=sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d \ + --hash=sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6 \ + --hash=sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6 \ + --hash=sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106 \ + --hash=sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75 \ + --hash=sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868 \ + --hash=sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a \ + --hash=sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0 \ + --hash=sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1 \ + --hash=sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826 \ + --hash=sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec \ + --hash=sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6 \ + --hash=sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950 \ + --hash=sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19 \ + --hash=sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0 \ + --hash=sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8 \ + --hash=sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a \ + --hash=sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09 \ + --hash=sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86 \ + --hash=sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c \ + --hash=sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5 \ + --hash=sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b \ + --hash=sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b \ + --hash=sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d \ + --hash=sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0 \ + --hash=sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea \ + --hash=sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776 \ + --hash=sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a \ + --hash=sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897 \ + --hash=sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7 \ + --hash=sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09 \ + --hash=sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9 \ + --hash=sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe \ + --hash=sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd \ + --hash=sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742 \ + --hash=sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09 \ + --hash=sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0 \ + --hash=sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932 \ + --hash=sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1 \ + --hash=sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a \ + --hash=sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49 \ + --hash=sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d \ + --hash=sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7 \ + --hash=sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480 \ + --hash=sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89 \ + --hash=sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e \ + --hash=sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b \ + --hash=sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82 \ + --hash=sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb \ + --hash=sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068 \ + --hash=sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8 \ + --hash=sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b \ + --hash=sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb \ + --hash=sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2 \ + --hash=sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11 \ + --hash=sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b \ + --hash=sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc \ + --hash=sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0 \ + --hash=sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497 \ + --hash=sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17 \ + --hash=sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0 \ + --hash=sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2 \ + --hash=sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439 \ + --hash=sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5 \ + --hash=sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac \ + --hash=sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825 \ + --hash=sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887 \ + --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ + --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 + # via + # -c python/requirements_compiled.txt + # aiohttp + # aiosignal +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 + # via + # -c python/requirements_compiled.txt + # ray +google-api-core==2.24.2 \ + --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ + --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 + # via + # -c python/requirements_compiled.txt + # opencensus +google-auth==2.23.4 \ + --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ + --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 + # via + # -c python/requirements_compiled.txt + # google-api-core +googleapis-common-protos==1.61.0 \ + --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ + --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b + # via + # -c python/requirements_compiled.txt + # google-api-core +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e + # via + # -c python/requirements_compiled.txt + # ray +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a + # via + # -c python/requirements_compiled.txt + # ray +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # -c python/requirements_compiled.txt + # uvicorn +httptools==0.6.4 \ + --hash=sha256:0614154d5454c21b6410fdf5262b4a3ddb0f53f1e1721cfd59d55f32138c578a \ + --hash=sha256:0e563e54979e97b6d13f1bbc05a96109923e76b901f786a5eae36e99c01237bd \ + --hash=sha256:16e603a3bff50db08cd578d54f07032ca1631450ceb972c2f834c2b860c28ea2 \ + --hash=sha256:288cd628406cc53f9a541cfaf06041b4c71d751856bab45e3702191f931ccd17 \ + --hash=sha256:28908df1b9bb8187393d5b5db91435ccc9c8e891657f9cbb42a2541b44c82fc8 \ + --hash=sha256:322d20ea9cdd1fa98bd6a74b77e2ec5b818abdc3d36695ab402a0de8ef2865a3 \ + --hash=sha256:342dd6946aa6bda4b8f18c734576106b8a31f2fe31492881a9a160ec84ff4bd5 \ + --hash=sha256:345c288418f0944a6fe67be8e6afa9262b18c7626c3ef3c28adc5eabc06a68da \ + --hash=sha256:3c73ce323711a6ffb0d247dcd5a550b8babf0f757e86a52558fe5b86d6fefcc0 \ + --hash=sha256:40a5ec98d3f49904b9fe36827dcf1aadfef3b89e2bd05b0e35e94f97c2b14721 \ + --hash=sha256:40b0f7fe4fd38e6a507bdb751db0379df1e99120c65fbdc8ee6c1d044897a636 \ + --hash=sha256:40dc6a8e399e15ea525305a2ddba998b0af5caa2566bcd79dcbe8948181eeaff \ + --hash=sha256:4b36913ba52008249223042dca46e69967985fb4051951f94357ea681e1f5dc0 \ + --hash=sha256:4d87b29bd4486c0093fc64dea80231f7c7f7eb4dc70ae394d70a495ab8436071 \ + --hash=sha256:4e93eee4add6493b59a5c514da98c939b244fce4a0d8879cd3f466562f4b7d5c \ + --hash=sha256:59e724f8b332319e2875efd360e61ac07f33b492889284a3e05e6d13746876f4 \ + --hash=sha256:69422b7f458c5af875922cdb5bd586cc1f1033295aa9ff63ee196a87519ac8e1 \ + --hash=sha256:703c346571fa50d2e9856a37d7cd9435a25e7fd15e236c397bf224afaa355fe9 \ + --hash=sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44 \ + --hash=sha256:856f4bc0478ae143bad54a4242fccb1f3f86a6e1be5548fecfd4102061b3a083 \ + --hash=sha256:85797e37e8eeaa5439d33e556662cc370e474445d5fab24dcadc65a8ffb04003 \ + --hash=sha256:90d96a385fa941283ebd231464045187a31ad932ebfa541be8edf5b3c2328959 \ + --hash=sha256:94978a49b8f4569ad607cd4946b759d90b285e39c0d4640c6b36ca7a3ddf2efc \ + --hash=sha256:aafe0f1918ed07b67c1e838f950b1c1fabc683030477e60b335649b8020e1076 \ + --hash=sha256:ab9ba8dcf59de5181f6be44a77458e45a578fc99c31510b8c65b7d5acc3cf490 \ + --hash=sha256:ade273d7e767d5fae13fa637f4d53b6e961fb7fd93c7797562663f0171c26660 \ + --hash=sha256:b799de31416ecc589ad79dd85a0b2657a8fe39327944998dea368c1d4c9e55e6 \ + --hash=sha256:c26f313951f6e26147833fc923f78f95604bbec812a43e5ee37f26dc9e5a686c \ + --hash=sha256:ca80b7485c76f768a3bc83ea58373f8db7b015551117375e4918e2aa77ea9b50 \ + --hash=sha256:d1ffd262a73d7c28424252381a5b854c19d9de5f56f075445d33919a637e3547 \ + --hash=sha256:d3f0d369e7ffbe59c4b6116a44d6a8eb4783aae027f2c0b366cf0aa964185dba \ + --hash=sha256:d54efd20338ac52ba31e7da78e4a72570cf729fac82bc31ff9199bedf1dc7440 \ + --hash=sha256:dacdd3d10ea1b4ca9df97a0a303cbacafc04b5cd375fa98732678151643d4988 \ + --hash=sha256:db353d22843cf1028f43c3651581e4bb49374d85692a85f95f7b9a130e1b2cab \ + --hash=sha256:db78cb9ca56b59b016e64b6031eda5653be0589dba2b1b43453f6e8b405a0970 \ + --hash=sha256:deee0e3343f98ee8047e9f4c5bc7cedbf69f5734454a94c38ee829fb2d5fa3c1 \ + --hash=sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2 \ + --hash=sha256:df959752a0c2748a65ab5387d08287abf6779ae9165916fe053e68ae1fbdc47f \ + --hash=sha256:ec4f178901fa1834d4a060320d2f3abc5c9e39766953d038f1458cb885f47e81 \ + --hash=sha256:f47f8ed67cc0ff862b84a1189831d1d33c963fb3ce1ee0c65d3b0cbe7b711069 \ + --hash=sha256:f8787367fbdfccae38e35abf7641dafc5310310a5987b689f4c32cc8cc3ee975 \ + --hash=sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f \ + --hash=sha256:fc411e1c0a7dcd2f902c7c48cf079947a7e65b5485dea9decb82b9105ca71a43 + # via uvicorn +idna==3.7 \ + --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ + --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 + # via + # -c python/requirements_compiled.txt + # anyio + # requests + # yarl +importlib-metadata==6.11.0 \ + --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ + --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b + # via + # -c python/requirements_compiled.txt + # opentelemetry-api +jinja2==3.1.6 ; sys_platform != 'win32' \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via + # -c python/requirements_compiled.txt + # memray +jsonschema==4.23.0 \ + --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ + --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 + # via + # -c python/requirements_compiled.txt + # ray +jsonschema-specifications==2024.10.1 \ + --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ + --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf + # via + # -c python/requirements_compiled.txt + # jsonschema +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled.txt + # celery +lz4==4.3.3 \ + --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ + --hash=sha256:054b4631a355606e99a42396f5db4d22046a3397ffc3269a348ec41eaebd69d2 \ + --hash=sha256:0a136e44a16fc98b1abc404fbabf7f1fada2bdab6a7e970974fb81cf55b636d0 \ + --hash=sha256:0e9c410b11a31dbdc94c05ac3c480cb4b222460faf9231f12538d0074e56c563 \ + --hash=sha256:222a7e35137d7539c9c33bb53fcbb26510c5748779364014235afc62b0ec797f \ + --hash=sha256:24b3206de56b7a537eda3a8123c644a2b7bf111f0af53bc14bed90ce5562d1aa \ + --hash=sha256:2b901c7784caac9a1ded4555258207d9e9697e746cc8532129f150ffe1f6ba0d \ + --hash=sha256:2f7b1839f795315e480fb87d9bc60b186a98e3e5d17203c6e757611ef7dcef61 \ + --hash=sha256:30e8c20b8857adef7be045c65f47ab1e2c4fabba86a9fa9a997d7674a31ea6b6 \ + --hash=sha256:31ea4be9d0059c00b2572d700bf2c1bc82f241f2c3282034a759c9a4d6ca4dc2 \ + --hash=sha256:337cb94488a1b060ef1685187d6ad4ba8bc61d26d631d7ba909ee984ea736be1 \ + --hash=sha256:33c9a6fd20767ccaf70649982f8f3eeb0884035c150c0b818ea660152cf3c809 \ + --hash=sha256:363ab65bf31338eb364062a15f302fc0fab0a49426051429866d71c793c23394 \ + --hash=sha256:43cf03059c0f941b772c8aeb42a0813d68d7081c009542301637e5782f8a33e2 \ + --hash=sha256:56f4fe9c6327adb97406f27a66420b22ce02d71a5c365c48d6b656b4aaeb7775 \ + --hash=sha256:5d35533bf2cee56f38ced91f766cd0038b6abf46f438a80d50c52750088be93f \ + --hash=sha256:6756212507405f270b66b3ff7f564618de0606395c0fe10a7ae2ffcbbe0b1fba \ + --hash=sha256:6cdc60e21ec70266947a48839b437d46025076eb4b12c76bd47f8e5eb8a75dcc \ + --hash=sha256:abc197e4aca8b63f5ae200af03eb95fb4b5055a8f990079b5bdf042f568469dd \ + --hash=sha256:b14d948e6dce389f9a7afc666d60dd1e35fa2138a8ec5306d30cd2e30d36b40c \ + --hash=sha256:b47839b53956e2737229d70714f1d75f33e8ac26e52c267f0197b3189ca6de24 \ + --hash=sha256:b6d9ec061b9eca86e4dcc003d93334b95d53909afd5a32c6e4f222157b50c071 \ + --hash=sha256:b891880c187e96339474af2a3b2bfb11a8e4732ff5034be919aa9029484cd201 \ + --hash=sha256:bca8fccc15e3add173da91be8f34121578dc777711ffd98d399be35487c934bf \ + --hash=sha256:c81703b12475da73a5d66618856d04b1307e43428a7e59d98cfe5a5d608a74c6 \ + --hash=sha256:d2507ee9c99dbddd191c86f0e0c8b724c76d26b0602db9ea23232304382e1f21 \ + --hash=sha256:e36cd7b9d4d920d3bfc2369840da506fa68258f7bb176b8743189793c055e43d \ + --hash=sha256:e7d84b479ddf39fe3ea05387f10b779155fc0990125f4fb35d636114e1c63a2e \ + --hash=sha256:eac9af361e0d98335a02ff12fb56caeb7ea1196cf1a49dbf6f17828a131da807 \ + --hash=sha256:edfd858985c23523f4e5a7526ca6ee65ff930207a7ec8a8f57a01eae506aaee7 \ + --hash=sha256:ee9ff50557a942d187ec85462bb0960207e7ec5b19b3b48949263993771c6205 \ + --hash=sha256:f0e822cd7644995d9ba248cb4b67859701748a93e2ab7fc9bc18c599a52e4604 \ + --hash=sha256:f180904f33bdd1e92967923a43c22899e303906d19b2cf8bb547db6653ea6e7d \ + --hash=sha256:f1d18718f9d78182c6b60f568c9a9cec8a7204d7cb6fad4e511a2ef279e4cb05 \ + --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ + --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 + # via + # -c python/requirements_compiled.txt + # ray +markdown-it-py==2.2.0 ; sys_platform != 'win32' \ + --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ + --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 + # via + # -c python/requirements_compiled.txt + # rich +markupsafe==2.1.3 ; sys_platform != 'win32' \ + --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ + --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ + --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ + --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ + --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ + --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ + --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ + --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ + --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ + --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ + --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ + --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ + --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ + --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ + --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ + --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ + --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ + --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ + --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ + --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ + --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ + --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ + --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ + --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ + --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 + # via + # -c python/requirements_compiled.txt + # jinja2 +mdurl==0.1.2 ; sys_platform != 'win32' \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via + # -c python/requirements_compiled.txt + # markdown-it-py +memray==1.10.0 ; sys_platform != 'win32' \ + --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ + --hash=sha256:22f2a47871c172a0539bd72737bb6b294fc10c510464066b825d90fcd3bb4916 \ + --hash=sha256:23e8c402625cfb32d0e9edb5ec0945f3e5e54bc6b0c5699f6284302082b80bd4 \ + --hash=sha256:2ce59ef485db3634de98b3a026d2450fc0a875e3a58a9ea85f7a89098841defe \ + --hash=sha256:322ed0b69014a0969b777768d461a785203f81f9864386b666b5b26645d9c294 \ + --hash=sha256:38322e052b882790993412f1840517a51818aa55c47037f69915b2007f2c4cee \ + --hash=sha256:38393c86ce6d0a08e6ec0eb1401d49803b7c0c950c2565386751cdc81568cba8 \ + --hash=sha256:391aac6c9f744528d3186bc82d708a1acc83525778f804045d7c96f860f8ec98 \ + --hash=sha256:3a8bb7fbd8303c4f0017ba7faef6b88f904cda2931ed667cbf3b98f024b3bc44 \ + --hash=sha256:3c401c57f49c4c5f1fecaee1e746f537cdc6680da05fb963dc143bd08ee109bf \ + --hash=sha256:4eba29179772b4a2e440a065b320b03bc2e73fe2648bdf7936aa3b9a086fab4a \ + --hash=sha256:53a8f66af18b1f3bcf5c9f3c95ae4134dd675903a38f9d0e6341b7bca01b63d0 \ + --hash=sha256:566602b2143e06b3d592901d98c52ce4599e71aa2555146eeb5cec03506f9498 \ + --hash=sha256:663d463e89a64bae4a6b2f8c837d11a3d094834442d536a4165e1d31899a3500 \ + --hash=sha256:68bd8df023c8a32f44c11d997e5c536837e27c0955daf557d3a377edd55a1dd3 \ + --hash=sha256:6937d7ef67d18ccc01c3250cdf3b4ef1445b859ee8756f09e3d11bd3ff0c7d67 \ + --hash=sha256:6b311e91203be71e1a0ce5e4f978137765bcb1045f3bf5646129c83c5b96ab3c \ + --hash=sha256:6fd13ef666c7fced9768d1cfabf71dc6dfa6724935a8dff463495ac2dc5e13a4 \ + --hash=sha256:8196c684f1be8fe423e5cdd2356d4255a2cb482a1f3e89612b70d2a2862cf5bb \ + --hash=sha256:843a688877691746f9d1835cfa8a65139948471bdd78720435808d20bc30a1cc \ + --hash=sha256:85c32d6613d81b075f740e398c4d653e0803cd48e82c33dcd584c109d6782666 \ + --hash=sha256:898acd60f57a10dc5aaf1fd64aa2f821f0420114f3f60c3058083788603f173a \ + --hash=sha256:8d56f37a34125684746c13d24bd7a3fb17549b0bb355eb50969eb11e05e3ba62 \ + --hash=sha256:92c372cb262eddd23049f945ca9527f0e4cc7c40a070aade1802d066f680885b \ + --hash=sha256:95e563d9c976e429ad597ad2720d95cebbe8bac891a3082465439143e2740772 \ + --hash=sha256:9627184c926252c8f719c301f1fefe970f0d033c643a6448b93fed2889d1ea94 \ + --hash=sha256:a9e985fb7646b0475c303919d19211d2aa54e5a9e2cd2a102472299be5dbebd3 \ + --hash=sha256:b681519357d94f5f0857fbc6029e7c44d3f41436109e955a14fd312d8317bc35 \ + --hash=sha256:b75040f28e8678d0e9c4907d55c95cf26db8ef5adc9941a228f1b280a9efd9c0 \ + --hash=sha256:c3a14960838d89a91747885897d34134afb65883cc3b0ed7ff30fe1af00f9fe6 \ + --hash=sha256:c7aeb47174c42e99740a8e2b3b6fe0932c95d987258d48a746974ead19176c26 \ + --hash=sha256:ce22a887a585ef5020896de89ffc793e531b65ccc81fbafcc7886010c2c562b3 \ + --hash=sha256:cf6d683c4f8d25c6ad06ae18715f218983c5eb86803953615e902d632fdf6ec1 \ + --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ + --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 + # via + # -c python/requirements_compiled.txt + # ray +msgpack==1.0.7 \ + --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ + --hash=sha256:0bfdd914e55e0d2c9e1526de210f6fe8ffe9705f2b1dfcc4aecc92a4cb4b533d \ + --hash=sha256:1dc93e8e4653bdb5910aed79f11e165c85732067614f180f70534f056da97db3 \ + --hash=sha256:1e2d69948e4132813b8d1131f29f9101bc2c915f26089a6d632001a5c1349672 \ + --hash=sha256:235a31ec7db685f5c82233bddf9858748b89b8119bf4538d514536c485c15fe0 \ + --hash=sha256:27dcd6f46a21c18fa5e5deed92a43d4554e3df8d8ca5a47bf0615d6a5f39dbc9 \ + --hash=sha256:28efb066cde83c479dfe5a48141a53bc7e5f13f785b92ddde336c716663039ee \ + --hash=sha256:3476fae43db72bd11f29a5147ae2f3cb22e2f1a91d575ef130d2bf49afd21c46 \ + --hash=sha256:36e17c4592231a7dbd2ed09027823ab295d2791b3b1efb2aee874b10548b7524 \ + --hash=sha256:384d779f0d6f1b110eae74cb0659d9aa6ff35aaf547b3955abf2ab4c901c4819 \ + --hash=sha256:38949d30b11ae5f95c3c91917ee7a6b239f5ec276f271f28638dec9156f82cfc \ + --hash=sha256:3967e4ad1aa9da62fd53e346ed17d7b2e922cba5ab93bdd46febcac39be636fc \ + --hash=sha256:3e7bf4442b310ff154b7bb9d81eb2c016b7d597e364f97d72b1acc3817a0fdc1 \ + --hash=sha256:3f0c8c6dfa6605ab8ff0611995ee30d4f9fcff89966cf562733b4008a3d60d82 \ + --hash=sha256:484ae3240666ad34cfa31eea7b8c6cd2f1fdaae21d73ce2974211df099a95d81 \ + --hash=sha256:4a7b4f35de6a304b5533c238bee86b670b75b03d31b7797929caa7a624b5dda6 \ + --hash=sha256:4cb14ce54d9b857be9591ac364cb08dc2d6a5c4318c1182cb1d02274029d590d \ + --hash=sha256:4e71bc4416de195d6e9b4ee93ad3f2f6b2ce11d042b4d7a7ee00bbe0358bd0c2 \ + --hash=sha256:52700dc63a4676669b341ba33520f4d6e43d3ca58d422e22ba66d1736b0a6e4c \ + --hash=sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87 \ + --hash=sha256:576eb384292b139821c41995523654ad82d1916da6a60cff129c715a6223ea84 \ + --hash=sha256:5b0bf0effb196ed76b7ad883848143427a73c355ae8e569fa538365064188b8e \ + --hash=sha256:5b6ccc0c85916998d788b295765ea0e9cb9aac7e4a8ed71d12e7d8ac31c23c95 \ + --hash=sha256:5ed82f5a7af3697b1c4786053736f24a0efd0a1b8a130d4c7bfee4b9ded0f08f \ + --hash=sha256:6d4c80667de2e36970ebf74f42d1088cc9ee7ef5f4e8c35eee1b40eafd33ca5b \ + --hash=sha256:730076207cb816138cf1af7f7237b208340a2c5e749707457d70705715c93b93 \ + --hash=sha256:7687e22a31e976a0e7fc99c2f4d11ca45eff652a81eb8c8085e9609298916dcf \ + --hash=sha256:822ea70dc4018c7e6223f13affd1c5c30c0f5c12ac1f96cd8e9949acddb48a61 \ + --hash=sha256:84b0daf226913133f899ea9b30618722d45feffa67e4fe867b0b5ae83a34060c \ + --hash=sha256:85765fdf4b27eb5086f05ac0491090fc76f4f2b28e09d9350c31aac25a5aaff8 \ + --hash=sha256:8dd178c4c80706546702c59529ffc005681bd6dc2ea234c450661b205445a34d \ + --hash=sha256:8f5b234f567cf76ee489502ceb7165c2a5cecec081db2b37e35332b537f8157c \ + --hash=sha256:98bbd754a422a0b123c66a4c341de0474cad4a5c10c164ceed6ea090f3563db4 \ + --hash=sha256:993584fc821c58d5993521bfdcd31a4adf025c7d745bbd4d12ccfecf695af5ba \ + --hash=sha256:a40821a89dc373d6427e2b44b572efc36a2778d3f543299e2f24eb1a5de65415 \ + --hash=sha256:b291f0ee7961a597cbbcc77709374087fa2a9afe7bdb6a40dbbd9b127e79afee \ + --hash=sha256:b573a43ef7c368ba4ea06050a957c2a7550f729c31f11dd616d2ac4aba99888d \ + --hash=sha256:b610ff0f24e9f11c9ae653c67ff8cc03c075131401b3e5ef4b82570d1728f8a9 \ + --hash=sha256:bdf38ba2d393c7911ae989c3bbba510ebbcdf4ecbdbfec36272abe350c454075 \ + --hash=sha256:bfef2bb6ef068827bbd021017a107194956918ab43ce4d6dc945ffa13efbc25f \ + --hash=sha256:cab3db8bab4b7e635c1c97270d7a4b2a90c070b33cbc00c99ef3f9be03d3e1f7 \ + --hash=sha256:cb70766519500281815dfd7a87d3a178acf7ce95390544b8c90587d76b227681 \ + --hash=sha256:cca1b62fe70d761a282496b96a5e51c44c213e410a964bdffe0928e611368329 \ + --hash=sha256:ccf9a39706b604d884d2cb1e27fe973bc55f2890c52f38df742bc1d79ab9f5e1 \ + --hash=sha256:dc43f1ec66eb8440567186ae2f8c447d91e0372d793dfe8c222aec857b81a8cf \ + --hash=sha256:dd632777ff3beaaf629f1ab4396caf7ba0bdd075d948a69460d13d44357aca4c \ + --hash=sha256:e45ae4927759289c30ccba8d9fdce62bb414977ba158286b5ddaf8df2cddb5c5 \ + --hash=sha256:e50ebce52f41370707f1e21a59514e3375e3edd6e1832f5e5235237db933c98b \ + --hash=sha256:ebbbba226f0a108a7366bf4b59bf0f30a12fd5e75100c630267d94d7f0ad20e5 \ + --hash=sha256:ec79ff6159dffcc30853b2ad612ed572af86c92b5168aa3fc01a67b0fa40665e \ + --hash=sha256:f0936e08e0003f66bfd97e74ee530427707297b0d0361247e9b4f59ab78ddc8b \ + --hash=sha256:f26a07a6e877c76a88e3cecac8531908d980d3d5067ff69213653649ec0f60ad \ + --hash=sha256:f64e376cd20d3f030190e8c32e1c64582eba56ac6dc7d5b0b49a9d44021b52fd \ + --hash=sha256:f6ffbc252eb0d229aeb2f9ad051200668fc3a9aaa8994e49f0cb2ffe2b7867e7 \ + --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ + --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc + # via + # -c python/requirements_compiled.txt + # ray +multidict==6.0.5 \ + --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ + --hash=sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c \ + --hash=sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29 \ + --hash=sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b \ + --hash=sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8 \ + --hash=sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7 \ + --hash=sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd \ + --hash=sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40 \ + --hash=sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6 \ + --hash=sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3 \ + --hash=sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c \ + --hash=sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9 \ + --hash=sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5 \ + --hash=sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae \ + --hash=sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442 \ + --hash=sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9 \ + --hash=sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc \ + --hash=sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c \ + --hash=sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea \ + --hash=sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5 \ + --hash=sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50 \ + --hash=sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182 \ + --hash=sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453 \ + --hash=sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e \ + --hash=sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600 \ + --hash=sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733 \ + --hash=sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda \ + --hash=sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241 \ + --hash=sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461 \ + --hash=sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e \ + --hash=sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e \ + --hash=sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b \ + --hash=sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e \ + --hash=sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7 \ + --hash=sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386 \ + --hash=sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd \ + --hash=sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9 \ + --hash=sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf \ + --hash=sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee \ + --hash=sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5 \ + --hash=sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a \ + --hash=sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271 \ + --hash=sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54 \ + --hash=sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4 \ + --hash=sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496 \ + --hash=sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb \ + --hash=sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319 \ + --hash=sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3 \ + --hash=sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f \ + --hash=sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527 \ + --hash=sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed \ + --hash=sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604 \ + --hash=sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef \ + --hash=sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8 \ + --hash=sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5 \ + --hash=sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5 \ + --hash=sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626 \ + --hash=sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c \ + --hash=sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d \ + --hash=sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c \ + --hash=sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc \ + --hash=sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc \ + --hash=sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b \ + --hash=sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38 \ + --hash=sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450 \ + --hash=sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1 \ + --hash=sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f \ + --hash=sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3 \ + --hash=sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755 \ + --hash=sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226 \ + --hash=sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a \ + --hash=sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046 \ + --hash=sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf \ + --hash=sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479 \ + --hash=sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e \ + --hash=sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1 \ + --hash=sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a \ + --hash=sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83 \ + --hash=sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929 \ + --hash=sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93 \ + --hash=sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a \ + --hash=sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c \ + --hash=sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44 \ + --hash=sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89 \ + --hash=sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba \ + --hash=sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e \ + --hash=sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da \ + --hash=sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24 \ + --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ + --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef + # via + # -c python/requirements_compiled.txt + # aiohttp + # yarl +numpy==1.26.4 \ + --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ + --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ + --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \ + --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \ + --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \ + --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \ + --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \ + --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \ + --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \ + --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \ + --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \ + --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \ + --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \ + --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \ + --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \ + --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \ + --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \ + --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \ + --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \ + --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \ + --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \ + --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \ + --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \ + --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \ + --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \ + --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \ + --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \ + --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \ + --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \ + --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \ + --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \ + --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \ + --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \ + --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \ + --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ + --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f + # via + # -c python/requirements_compiled.txt + # cupy-cuda12x + # gymnasium + # pandas + # ray + # scipy + # tensorboardx +opencensus==0.11.4 \ + --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ + --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 + # via + # -c python/requirements_compiled.txt + # ray +opencensus-context==0.1.3 \ + --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ + --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c + # via + # -c python/requirements_compiled.txt + # opencensus +opentelemetry-api==1.34.1 \ + --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ + --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-exporter-prometheus==0.55b1 \ + --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ + --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e + # via + # -c python/requirements_compiled.txt + # ray +opentelemetry-proto==1.27.0 \ + --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ + --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace + # via + # -c python/requirements_compiled.txt + # ray +opentelemetry-sdk==1.34.1 \ + --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ + --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # ray +opentelemetry-semantic-conventions==0.55b1 \ + --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ + --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 + # via + # -c python/requirements_compiled.txt + # opentelemetry-sdk +ormsgpack==1.7.0 \ + --hash=sha256:0d88307ab45d95416ce4071b1b99326ca31362af01c3d206f15a0551a7a874bd \ + --hash=sha256:22418a4d399027a72fb2e6b873559b1886cf2e63323ca7afc17b222c454413b7 \ + --hash=sha256:2c22c62a6bc93bcb194b7f91864ca0b39455b2cbbfc1538a3da0f9ec3c11d184 \ + --hash=sha256:3a6a97937d2cf21496d7689b90a43df83c5062bbe846aaa39197cc9ad73eaa7b \ + --hash=sha256:462089a419dbde654915ccb0b859c0dbe3c178b0ac580018e82befea6ccd73f4 \ + --hash=sha256:4b353204e99b56c1d33f1cf4767bd1fe1195596181a1cc789f25aa26c0b50f3d \ + --hash=sha256:5ec763096d978d35eedcef0af13991a10741717c2e236b26f4c2047b0740ea7b \ + --hash=sha256:5fefa1ca842dbba258401ea958113fe62c6b70a7a4d46edac440113f68dc431e \ + --hash=sha256:65525438b4a8b3b64ccfcda25e758ea3db392d1c206b5e09ef70efbbafa6dbf9 \ + --hash=sha256:6b4c98839cb7fc2a212037d2258f3a22857155249eb293d45c45cb974cfba834 \ + --hash=sha256:6d114652dadd81802b8a35a49e07a3e9ef2a47aed6123fb5031f2220d1c8e434 \ + --hash=sha256:77bc2ea387d85cfad045b9bcb8040bae43ad32dafe9363360f732cc19d489bbe \ + --hash=sha256:7e6ada21f5c7a20ff7cf9b061c44e3814352f819947a12022ad8cb52a9f2a809 \ + --hash=sha256:8d301e47565fe0e52a60052e730a9bb7669dfbd2a94643b8be925e3928c64c15 \ + --hash=sha256:90aabfd816db60dadab1100d583d061e0238209015bf684f8170c0fca4eb445a \ + --hash=sha256:91ebb7d3609db249cdff629ffef83ec3d025b1384749a297cf3b6a8240cf22ac \ + --hash=sha256:97723786755a7df85fcf6e68d7b5359dacea98d5c26b1d9af219a3cc05df4734 \ + --hash=sha256:9b0945523ccc75aa6907f38f2240d36818618baccb8633923bd7740a5a929e67 \ + --hash=sha256:a0ca6a64d47073f22ecc1dd96b384e44f98796d3f88ee383e92dfbcdf18c2efd \ + --hash=sha256:a5e12b51a590be47ccef67907905653e679fc2f920854b456edc216690ecc09c \ + --hash=sha256:a8fbe7bb50ee8381df030823d9366984fac718447947c2327969405d1d799b95 \ + --hash=sha256:c683071bf4527ffa7b6cfcf28f750d1a82eb77846d106743c09261ab1b79b193 \ + --hash=sha256:ca4d35b694f32112eb33ac0b733cb903dbbc59f019d05ca3d74f6ad2f587b0bf \ + --hash=sha256:e8385181bf195af80fc270e64fd477f1c414ffb05837320382e2ec9ca34be0ec \ + --hash=sha256:e86124cdbc8ed249806347c2fba96843e8941122b161b429139a0c973d270de4 \ + --hash=sha256:f9967a7f3647ad118751abf090f8397fda3e4bca6833340cab95a3f2bec598cd + # via + # -c python/requirements_compiled.txt + # ray +packaging==23.0 \ + --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ + --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 + # via + # -c python/requirements_compiled.txt + # kombu + # ray + # tensorboardx +pandas==2.3.2 \ + --hash=sha256:0064187b80a5be6f2f9c9d6bdde29372468751dfa89f4211a3c5871854cfbf7a \ + --hash=sha256:0bd281310d4f412733f319a5bc552f86d62cddc5f51d2e392c8787335c994175 \ + --hash=sha256:0c6ecbac99a354a051ef21c5307601093cb9e0f4b1855984a084bfec9302699e \ + --hash=sha256:0cee69d583b9b128823d9514171cabb6861e09409af805b54459bd0c821a35c2 \ + --hash=sha256:114c2fe4f4328cf98ce5716d1532f3ab79c5919f95a9cfee81d9140064a2e4d6 \ + --hash=sha256:12d039facec710f7ba305786837d0225a3444af7bbd9c15c32ca2d40d157ed8b \ + --hash=sha256:1333e9c299adcbb68ee89a9bb568fc3f20f9cbb419f1dd5225071e6cddb2a743 \ + --hash=sha256:13bd629c653856f00c53dc495191baa59bcafbbf54860a46ecc50d3a88421a96 \ + --hash=sha256:1b9b52693123dd234b7c985c68b709b0b009f4521000d0525f2b95c22f15944b \ + --hash=sha256:1d81573b3f7db40d020983f78721e9bfc425f411e616ef019a10ebf597aedb2e \ + --hash=sha256:213a5adf93d020b74327cb2c1b842884dbdd37f895f42dcc2f09d451d949f811 \ + --hash=sha256:21bb612d148bb5860b7eb2c10faacf1a810799245afd342cf297d7551513fbb6 \ + --hash=sha256:220cc5c35ffaa764dd5bb17cf42df283b5cb7fdf49e10a7b053a06c9cb48ee2b \ + --hash=sha256:2319656ed81124982900b4c37f0e0c58c015af9a7bbc62342ba5ad07ace82ba9 \ + --hash=sha256:36d627906fd44b5fd63c943264e11e96e923f8de77d6016dc2f667b9ad193438 \ + --hash=sha256:3fbb977f802156e7a3f829e9d1d5398f6192375a3e2d1a9ee0803e35fe70a2b9 \ + --hash=sha256:42c05e15111221384019897df20c6fe893b2f697d03c811ee67ec9e0bb5a3424 \ + --hash=sha256:45178cf09d1858a1509dc73ec261bf5b25a625a389b65be2e47b559905f0ab6a \ + --hash=sha256:48fa91c4dfb3b2b9bfdb5c24cd3567575f4e13f9636810462ffed8925352be5a \ + --hash=sha256:4ac8c320bded4718b298281339c1a50fb00a6ba78cb2a63521c39bec95b0209b \ + --hash=sha256:52bc29a946304c360561974c6542d1dd628ddafa69134a7131fdfd6a5d7a1a35 \ + --hash=sha256:76972bcbd7de8e91ad5f0ca884a9f2c477a2125354af624e022c49e5bd0dfff4 \ + --hash=sha256:77cefe00e1b210f9c76c697fedd8fdb8d3dd86563e9c8adc9fa72b90f5e9e4c2 \ + --hash=sha256:837248b4fc3a9b83b9c6214699a13f069dc13510a6a6d7f9ba33145d2841a012 \ + --hash=sha256:88080a0ff8a55eac9c84e3ff3c7665b3b5476c6fbc484775ca1910ce1c3e0b87 \ + --hash=sha256:8c13b81a9347eb8c7548f53fd9a4f08d4dfe996836543f805c987bafa03317ae \ + --hash=sha256:9467697b8083f9667b212633ad6aa4ab32436dcbaf4cd57325debb0ddef2012f \ + --hash=sha256:96d31a6b4354e3b9b8a2c848af75d31da390657e3ac6f30c05c82068b9ed79b9 \ + --hash=sha256:a9d7ec92d71a420185dec44909c32e9a362248c4ae2238234b76d5be37f208cc \ + --hash=sha256:ab7b58f8f82706890924ccdfb5f48002b83d2b5a3845976a9fb705d36c34dcdb \ + --hash=sha256:b37205ad6f00d52f16b6d09f406434ba928c1a1966e2771006a9033c736d30d2 \ + --hash=sha256:b62d586eb25cb8cb70a5746a378fc3194cb7f11ea77170d59f889f5dfe3cec7a \ + --hash=sha256:b98bdd7c456a05eef7cd21fd6b29e3ca243591fe531c62be94a2cc987efb5ac2 \ + --hash=sha256:c253828cb08f47488d60f43c5fc95114c771bbfff085da54bfc79cb4f9e3a372 \ + --hash=sha256:c624b615ce97864eb588779ed4046186f967374185c047070545253a52ab2d57 \ + --hash=sha256:c6f048aa0fd080d6a06cc7e7537c09b53be6642d330ac6f54a600c3ace857ee9 \ + --hash=sha256:cc03acc273c5515ab69f898df99d9d4f12c4d70dbfc24c3acc6203751d0804cf \ + --hash=sha256:d25c20a03e8870f6339bcf67281b946bd20b86f1a544ebbebb87e66a8d642cba \ + --hash=sha256:d2c3554bd31b731cd6490d94a28f3abb8dd770634a9e06eb6d2911b9827db370 \ + --hash=sha256:d4a558c7620340a0931828d8065688b3cc5b4c8eb674bcaf33d18ff4a6870b4a \ + --hash=sha256:df4df0b9d02bb873a106971bb85d448378ef14b86ba96f035f50bbd3688456b4 \ + --hash=sha256:e190b738675a73b581736cc8ec71ae113d6c3768d0bd18bffa5b9a0927b0b6ea + # via ray +platformdirs==3.11.0 \ + --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ + --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e + # via + # -c python/requirements_compiled.txt + # virtualenv +prometheus-client==0.19.0 \ + --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ + --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # ray +prompt-toolkit==3.0.41 \ + --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ + --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 + # via + # -c python/requirements_compiled.txt + # click-repl +propcache==0.3.0 \ + --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ + --hash=sha256:03acd9ff19021bd0567582ac88f821b66883e158274183b9e5586f678984f8fe \ + --hash=sha256:03c091bb752349402f23ee43bb2bff6bd80ccab7c9df6b88ad4322258d6960fc \ + --hash=sha256:07700939b2cbd67bfb3b76a12e1412405d71019df00ca5697ce75e5ef789d829 \ + --hash=sha256:0c3e893c4464ebd751b44ae76c12c5f5c1e4f6cbd6fbf67e3783cd93ad221863 \ + --hash=sha256:119e244ab40f70a98c91906d4c1f4c5f2e68bd0b14e7ab0a06922038fae8a20f \ + --hash=sha256:11ae6a8a01b8a4dc79093b5d3ca2c8a4436f5ee251a9840d7790dccbd96cb649 \ + --hash=sha256:15010f29fbed80e711db272909a074dc79858c6d28e2915704cfc487a8ac89c6 \ + --hash=sha256:19d36bb351ad5554ff20f2ae75f88ce205b0748c38b146c75628577020351e3c \ + --hash=sha256:1c8f7d896a16da9455f882870a507567d4f58c53504dc2d4b1e1d386dfe4588a \ + --hash=sha256:2383a17385d9800b6eb5855c2f05ee550f803878f344f58b6e194de08b96352c \ + --hash=sha256:24c04f8fbf60094c531667b8207acbae54146661657a1b1be6d3ca7773b7a545 \ + --hash=sha256:2578541776769b500bada3f8a4eeaf944530516b6e90c089aa368266ed70c49e \ + --hash=sha256:26a67e5c04e3119594d8cfae517f4b9330c395df07ea65eab16f3d559b7068fe \ + --hash=sha256:2b975528998de037dfbc10144b8aed9b8dd5a99ec547f14d1cb7c5665a43f075 \ + --hash=sha256:2d15bc27163cd4df433e75f546b9ac31c1ba7b0b128bfb1b90df19082466ff57 \ + --hash=sha256:2d913d36bdaf368637b4f88d554fb9cb9d53d6920b9c5563846555938d5450bf \ + --hash=sha256:3302c5287e504d23bb0e64d2a921d1eb4a03fb93a0a0aa3b53de059f5a5d737d \ + --hash=sha256:36ca5e9a21822cc1746023e88f5c0af6fce3af3b85d4520efb1ce4221bed75cc \ + --hash=sha256:3b812b3cb6caacd072276ac0492d249f210006c57726b6484a1e1805b3cfeea0 \ + --hash=sha256:3c6ec957025bf32b15cbc6b67afe233c65b30005e4c55fe5768e4bb518d712f1 \ + --hash=sha256:41de3da5458edd5678b0f6ff66691507f9885f5fe6a0fb99a5d10d10c0fd2d64 \ + --hash=sha256:42924dc0c9d73e49908e35bbdec87adedd651ea24c53c29cac103ede0ea1d340 \ + --hash=sha256:4544699674faf66fb6b4473a1518ae4999c1b614f0b8297b1cef96bac25381db \ + --hash=sha256:46ed02532cb66612d42ae5c3929b5e98ae330ea0f3900bc66ec5f4862069519b \ + --hash=sha256:49ea05212a529c2caffe411e25a59308b07d6e10bf2505d77da72891f9a05641 \ + --hash=sha256:4fa0e7c9c3cf7c276d4f6ab9af8adddc127d04e0fcabede315904d2ff76db626 \ + --hash=sha256:507c5357a8d8b4593b97fb669c50598f4e6cccbbf77e22fa9598aba78292b4d7 \ + --hash=sha256:549722908de62aa0b47a78b90531c022fa6e139f9166be634f667ff45632cc92 \ + --hash=sha256:58e6d2a5a7cb3e5f166fd58e71e9a4ff504be9dc61b88167e75f835da5764d07 \ + --hash=sha256:5a16167118677d94bb48bfcd91e420088854eb0737b76ec374b91498fb77a70e \ + --hash=sha256:5d62c4f6706bff5d8a52fd51fec6069bef69e7202ed481486c0bc3874912c787 \ + --hash=sha256:5fa159dcee5dba00c1def3231c249cf261185189205073bde13797e57dd7540a \ + --hash=sha256:6032231d4a5abd67c7f71168fd64a47b6b451fbcb91c8397c2f7610e67683810 \ + --hash=sha256:63f26258a163c34542c24808f03d734b338da66ba91f410a703e505c8485791d \ + --hash=sha256:65a37714b8ad9aba5780325228598a5b16c47ba0f8aeb3dc0514701e4413d7c0 \ + --hash=sha256:67054e47c01b7b349b94ed0840ccae075449503cf1fdd0a1fdd98ab5ddc2667b \ + --hash=sha256:67dda3c7325691c2081510e92c561f465ba61b975f481735aefdfc845d2cd043 \ + --hash=sha256:6985a593417cdbc94c7f9c3403747335e450c1599da1647a5af76539672464d3 \ + --hash=sha256:6a1948df1bb1d56b5e7b0553c0fa04fd0e320997ae99689488201f19fa90d2e7 \ + --hash=sha256:6b5b7fd6ee7b54e01759f2044f936dcf7dea6e7585f35490f7ca0420fe723c0d \ + --hash=sha256:6c929916cbdb540d3407c66f19f73387f43e7c12fa318a66f64ac99da601bcdf \ + --hash=sha256:6f4d7a7c0aff92e8354cceca6fe223973ddf08401047920df0fcb24be2bd5138 \ + --hash=sha256:728af36011bb5d344c4fe4af79cfe186729efb649d2f8b395d1572fb088a996c \ + --hash=sha256:742840d1d0438eb7ea4280f3347598f507a199a35a08294afdcc560c3739989d \ + --hash=sha256:75e872573220d1ee2305b35c9813626e620768248425f58798413e9c39741f46 \ + --hash=sha256:794c3dd744fad478b6232289c866c25406ecdfc47e294618bdf1697e69bd64a6 \ + --hash=sha256:7c0fdbdf6983526e269e5a8d53b7ae3622dd6998468821d660d0daf72779aefa \ + --hash=sha256:7c5f5290799a3f6539cc5e6f474c3e5c5fbeba74a5e1e5be75587746a940d51e \ + --hash=sha256:7c6e7e4f9167fddc438cd653d826f2222222564daed4116a02a184b464d3ef05 \ + --hash=sha256:7cedd25e5f678f7738da38037435b340694ab34d424938041aa630d8bac42663 \ + --hash=sha256:7e2e068a83552ddf7a39a99488bcba05ac13454fb205c847674da0352602082f \ + --hash=sha256:8319293e85feadbbfe2150a5659dbc2ebc4afdeaf7d98936fb9a2f2ba0d4c35c \ + --hash=sha256:8526b0941ec5a40220fc4dfde76aed58808e2b309c03e9fa8e2260083ef7157f \ + --hash=sha256:8884ba1a0fe7210b775106b25850f5e5a9dc3c840d1ae9924ee6ea2eb3acbfe7 \ + --hash=sha256:8cb625bcb5add899cb8ba7bf716ec1d3e8f7cdea9b0713fa99eadf73b6d4986f \ + --hash=sha256:8d663fd71491dde7dfdfc899d13a067a94198e90695b4321084c6e450743b8c7 \ + --hash=sha256:8ee1983728964d6070ab443399c476de93d5d741f71e8f6e7880a065f878e0b9 \ + --hash=sha256:997e7b8f173a391987df40f3b52c423e5850be6f6df0dcfb5376365440b56667 \ + --hash=sha256:9be90eebc9842a93ef8335291f57b3b7488ac24f70df96a6034a13cb58e6ff86 \ + --hash=sha256:9ddd49258610499aab83b4f5b61b32e11fce873586282a0e972e5ab3bcadee51 \ + --hash=sha256:9ecde3671e62eeb99e977f5221abcf40c208f69b5eb986b061ccec317c82ebd0 \ + --hash=sha256:9ff4e9ecb6e4b363430edf2c6e50173a63e0820e549918adef70515f87ced19a \ + --hash=sha256:a254537b9b696ede293bfdbc0a65200e8e4507bc9f37831e2a0318a9b333c85c \ + --hash=sha256:a2b9bf8c79b660d0ca1ad95e587818c30ccdb11f787657458d6f26a1ea18c568 \ + --hash=sha256:a61a68d630e812b67b5bf097ab84e2cd79b48c792857dc10ba8a223f5b06a2af \ + --hash=sha256:a7080b0159ce05f179cfac592cda1a82898ca9cd097dacf8ea20ae33474fbb25 \ + --hash=sha256:a8fd93de4e1d278046345f49e2238cdb298589325849b2645d4a94c53faeffc5 \ + --hash=sha256:a94ffc66738da99232ddffcf7910e0f69e2bbe3a0802e54426dbf0714e1c2ffe \ + --hash=sha256:aa806bbc13eac1ab6291ed21ecd2dd426063ca5417dd507e6be58de20e58dfcf \ + --hash=sha256:b0c1a133d42c6fc1f5fbcf5c91331657a1ff822e87989bf4a6e2e39b818d0ee9 \ + --hash=sha256:b58229a844931bca61b3a20efd2be2a2acb4ad1622fc026504309a6883686fbf \ + --hash=sha256:bb2f144c6d98bb5cbc94adeb0447cfd4c0f991341baa68eee3f3b0c9c0e83767 \ + --hash=sha256:be90c94570840939fecedf99fa72839aed70b0ced449b415c85e01ae67422c90 \ + --hash=sha256:bf0d9a171908f32d54f651648c7290397b8792f4303821c42a74e7805bfb813c \ + --hash=sha256:bf15fc0b45914d9d1b706f7c9c4f66f2b7b053e9517e40123e137e8ca8958b3d \ + --hash=sha256:bf4298f366ca7e1ad1d21bbb58300a6985015909964077afd37559084590c929 \ + --hash=sha256:c441c841e82c5ba7a85ad25986014be8d7849c3cfbdb6004541873505929a74e \ + --hash=sha256:cacea77ef7a2195f04f9279297684955e3d1ae4241092ff0cfcef532bb7a1c32 \ + --hash=sha256:cd54895e4ae7d32f1e3dd91261df46ee7483a735017dc6f987904f194aa5fd14 \ + --hash=sha256:d1323cd04d6e92150bcc79d0174ce347ed4b349d748b9358fd2e497b121e03c8 \ + --hash=sha256:d383bf5e045d7f9d239b38e6acadd7b7fdf6c0087259a84ae3475d18e9a2ae8b \ + --hash=sha256:d3e7420211f5a65a54675fd860ea04173cde60a7cc20ccfbafcccd155225f8bc \ + --hash=sha256:d8074c5dd61c8a3e915fa8fc04754fa55cfa5978200d2daa1e2d4294c1f136aa \ + --hash=sha256:df03cd88f95b1b99052b52b1bb92173229d7a674df0ab06d2b25765ee8404bce \ + --hash=sha256:e45377d5d6fefe1677da2a2c07b024a6dac782088e37c0b1efea4cfe2b1be19b \ + --hash=sha256:e53d19c2bf7d0d1e6998a7e693c7e87300dd971808e6618964621ccd0e01fe4e \ + --hash=sha256:e560fd75aaf3e5693b91bcaddd8b314f4d57e99aef8a6c6dc692f935cc1e6bbf \ + --hash=sha256:ec5060592d83454e8063e487696ac3783cc48c9a329498bafae0d972bc7816c9 \ + --hash=sha256:ecc2920630283e0783c22e2ac94427f8cca29a04cfdf331467d4f661f4072dac \ + --hash=sha256:ed7161bccab7696a473fe7ddb619c1d75963732b37da4618ba12e60899fefe4f \ + --hash=sha256:ee0bd3a7b2e184e88d25c9baa6a9dc609ba25b76daae942edfb14499ac7ec374 \ + --hash=sha256:ee25f1ac091def37c4b59d192bbe3a206298feeb89132a470325bf76ad122a1e \ + --hash=sha256:efa44f64c37cc30c9f05932c740a8b40ce359f51882c70883cc95feac842da4d \ + --hash=sha256:f47d52fd9b2ac418c4890aad2f6d21a6b96183c98021f0a48497a904199f006e \ + --hash=sha256:f857034dc68d5ceb30fb60afb6ff2103087aea10a01b613985610e007053a121 \ + --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ + --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 + # via + # -c python/requirements_compiled.txt + # aiohttp + # yarl +proto-plus==1.22.3 \ + --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ + --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b + # via + # -c python/requirements_compiled.txt + # google-api-core +protobuf==4.25.8 \ + --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ + --hash=sha256:15a0af558aa3b13efef102ae6e4f3efac06f1eea11afb3a57db2901447d9fb59 \ + --hash=sha256:27d498ffd1f21fb81d987a041c32d07857d1d107909f5134ba3350e1ce80a4af \ + --hash=sha256:504435d831565f7cfac9f0714440028907f1975e4bed228e58e72ecfff58a1e0 \ + --hash=sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd \ + --hash=sha256:83e6e54e93d2b696a92cad6e6efc924f3850f82b52e1563778dfab8b355101b0 \ + --hash=sha256:9ad7ef62d92baf5a8654fbb88dac7fa5594cfa70fd3440488a5ca3bfc6d795a7 \ + --hash=sha256:bd551eb1fe1d7e92c1af1d75bdfa572eff1ab0e5bf1736716814cdccdb2360f9 \ + --hash=sha256:ca809b42f4444f144f2115c4c1a747b9a404d590f18f37e9402422033e464e0f \ + --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ + --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 + # via + # -c python/requirements_compiled.txt + # google-api-core + # googleapis-common-protos + # opentelemetry-proto + # proto-plus + # ray + # tensorboardx +py-spy==0.4.1 \ + --hash=sha256:1fb8bf71ab8df95a95cc387deed6552934c50feef2cf6456bc06692a5508fd0c \ + --hash=sha256:4972c21890b6814017e39ac233c22572c4a61fd874524ebc5ccab0f2237aee0a \ + --hash=sha256:532d3525538254d1859b49de1fbe9744df6b8865657c9f0e444bf36ce3f19226 \ + --hash=sha256:6a80ec05eb8a6883863a367c6a4d4f2d57de68466f7956b6367d4edd5c61bb29 \ + --hash=sha256:809094208c6256c8f4ccadd31e9a513fe2429253f48e20066879239ba12cd8cc \ + --hash=sha256:d92e522bd40e9bf7d87c204033ce5bb5c828fca45fa28d970f58d71128069fdc \ + --hash=sha256:e53aa53daa2e47c2eef97dd2455b47bb3a7e7f962796a86cc3e7dbde8e6f4db4 \ + --hash=sha256:ee776b9d512a011d1ad3907ed53ae32ce2f3d9ff3e1782236554e22103b5c084 + # via ray +pyarrow==19.0.1 \ + --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ + --hash=sha256:0148bb4fc158bfbc3d6dfe5001d93ebeed253793fff4435167f6ce1dc4bddeae \ + --hash=sha256:1b93ef2c93e77c442c979b0d596af45e4665d8b96da598db145b0fec014b9136 \ + --hash=sha256:1c7556165bd38cf0cd992df2636f8bcdd2d4b26916c6b7e646101aff3c16f76f \ + --hash=sha256:335d170e050bcc7da867a1ed8ffb8b44c57aaa6e0843b156a501298657b1e972 \ + --hash=sha256:3bf266b485df66a400f282ac0b6d1b500b9d2ae73314a153dbe97d6d5cc8a99e \ + --hash=sha256:41f9706fbe505e0abc10e84bf3a906a1338905cbbcf1177b71486b03e6ea6608 \ + --hash=sha256:4982f8e2b7afd6dae8608d70ba5bd91699077323f812a0448d8b7abdff6cb5d3 \ + --hash=sha256:49a3aecb62c1be1d822f8bf629226d4a96418228a42f5b40835c1f10d42e4db6 \ + --hash=sha256:4d5d1ec7ec5324b98887bdc006f4d2ce534e10e60f7ad995e7875ffa0ff9cb14 \ + --hash=sha256:58d9397b2e273ef76264b45531e9d552d8ec8a6688b7390b5be44c02a37aade8 \ + --hash=sha256:5a9137cf7e1640dce4c190551ee69d478f7121b5c6f323553b319cac936395f6 \ + --hash=sha256:5bd1618ae5e5476b7654c7b55a6364ae87686d4724538c24185bbb2952679960 \ + --hash=sha256:65cf9feebab489b19cdfcfe4aa82f62147218558d8d3f0fc1e9dea0ab8e7905a \ + --hash=sha256:699799f9c80bebcf1da0983ba86d7f289c5a2a5c04b945e2f2bcf7e874a91911 \ + --hash=sha256:6c5941c1aac89a6c2f2b16cd64fe76bcdb94b2b1e99ca6459de4e6f07638d755 \ + --hash=sha256:6ebfb5171bb5f4a52319344ebbbecc731af3f021e49318c74f33d520d31ae0c4 \ + --hash=sha256:7a544ec12de66769612b2d6988c36adc96fb9767ecc8ee0a4d270b10b1c51e00 \ + --hash=sha256:7c1bca1897c28013db5e4c83944a2ab53231f541b9e0c3f4791206d0c0de389a \ + --hash=sha256:80b2ad2b193e7d19e81008a96e313fbd53157945c7be9ac65f44f8937a55427b \ + --hash=sha256:8464c9fbe6d94a7fe1599e7e8965f350fd233532868232ab2596a71586c5a429 \ + --hash=sha256:8f04d49a6b64cf24719c080b3c2029a3a5b16417fd5fd7c4041f94233af732f3 \ + --hash=sha256:96606c3ba57944d128e8a8399da4812f56c7f61de8c647e3470b417f795d0ef9 \ + --hash=sha256:99bc1bec6d234359743b01e70d4310d0ab240c3d6b0da7e2a93663b0158616f6 \ + --hash=sha256:ad76aef7f5f7e4a757fddcdcf010a8290958f09e3470ea458c80d26f4316ae89 \ + --hash=sha256:b4c4156a625f1e35d6c0b2132635a237708944eb41df5fbe7d50f20d20c17832 \ + --hash=sha256:b9766a47a9cb56fefe95cb27f535038b5a195707a08bf61b180e642324963b46 \ + --hash=sha256:c0fe3dbbf054a00d1f162fda94ce236a899ca01123a798c561ba307ca38af5f0 \ + --hash=sha256:c6cb2335a411b713fdf1e82a752162f72d4a7b5dbc588e32aa18383318b05866 \ + --hash=sha256:cc55d71898ea30dc95900297d191377caba257612f384207fe9f8293b5850f90 \ + --hash=sha256:d03c9d6f2a3dffbd62671ca070f13fc527bb1867b4ec2b98c7eeed381d4f389a \ + --hash=sha256:d383591f3dcbe545f6cc62daaef9c7cdfe0dff0fb9e1c8121101cabe9098cfa6 \ + --hash=sha256:d9d46e06846a41ba906ab25302cf0fd522f81aa2a85a71021826f34639ad31ef \ + --hash=sha256:d9dedeaf19097a143ed6da37f04f4051aba353c95ef507764d344229b2b740ae \ + --hash=sha256:e45274b20e524ae5c39d7fc1ca2aa923aab494776d2d4b316b49ec7572ca324c \ + --hash=sha256:ee8dec072569f43835932a3b10c55973593abc00936c202707a4ad06af7cb294 \ + --hash=sha256:f24faab6ed18f216a37870d8c5623f9c044566d75ec586ef884e13a02a9d62c5 \ + --hash=sha256:f2a21d39fbdb948857f67eacb5bbaaf36802de044ec36fbef7a1c8f0dd3a4ab2 \ + --hash=sha256:f3ad4c0eb4e2a9aeb990af6c09e6fa0b195c8c0e7b272ecc8d4d2b6574809d34 \ + --hash=sha256:fc28912a2dc924dddc2087679cc8b7263accc71b9ff025a1362b004711661a69 \ + --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ + --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 + # via + # -c python/requirements_compiled.txt + # ray +pyasn1==0.5.1 \ + --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ + --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c + # via + # -c python/requirements_compiled.txt + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 \ + --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ + --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d + # via + # -c python/requirements_compiled.txt + # google-auth +pycparser==2.21 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ + --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 + # via + # -c python/requirements_compiled.txt + # cffi +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b + # via + # -c python/requirements_compiled.txt + # fastapi + # ray +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d + # via + # -c python/requirements_compiled.txt + # pydantic +pygments==2.18.0 ; sys_platform != 'win32' \ + --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ + --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a + # via + # -c python/requirements_compiled.txt + # rich +pyopenssl==25.0.0 \ + --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ + --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 + # via + # -c python/requirements_compiled.txt + # ray +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via + # -c python/requirements_compiled.txt + # celery + # pandas +python-dotenv==1.1.1 \ + --hash=sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc \ + --hash=sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab + # via uvicorn +pytz==2022.7.1 \ + --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ + --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a + # via + # -c python/requirements_compiled.txt + # pandas +pyyaml==6.0.1 \ + --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ + --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ + --hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \ + --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ + --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \ + --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \ + --hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \ + --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \ + --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \ + --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \ + --hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \ + --hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \ + --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \ + --hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \ + --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \ + --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \ + --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \ + --hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \ + --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \ + --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \ + --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \ + --hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \ + --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \ + --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \ + --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \ + --hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \ + --hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \ + --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ + --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ + --hash=sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef \ + --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ + --hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \ + --hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \ + --hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \ + --hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \ + --hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \ + --hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \ + --hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \ + --hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \ + --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \ + --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \ + --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \ + --hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \ + --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \ + --hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \ + --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \ + --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \ + --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \ + --hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \ + --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ + --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f + # via + # -c python/requirements_compiled.txt + # ray + # uvicorn +ray==100.0.0.dev0 \ + --hash=sha256:4fef4f9d9cc8b516f22c2eea42b7fa244ef1dace261809d9f175875de0ec9fed +referencing==0.36.2 \ + --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ + --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 + # via + # -c python/requirements_compiled.txt + # jsonschema + # jsonschema-specifications +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 + # via + # -c python/requirements_compiled.txt + # google-api-core + # ray +rich==13.3.2 ; sys_platform != 'win32' \ + --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ + --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f + # via + # -c python/requirements_compiled.txt + # memray +rpds-py==0.22.3 \ + --hash=sha256:009de23c9c9ee54bf11303a966edf4d9087cd43a6003672e6aa7def643d06518 \ + --hash=sha256:02fbb9c288ae08bcb34fb41d516d5eeb0455ac35b5512d03181d755d80810059 \ + --hash=sha256:0a0461200769ab3b9ab7e513f6013b7a97fdeee41c29b9db343f3c5a8e2b9e61 \ + --hash=sha256:0b09865a9abc0ddff4e50b5ef65467cd94176bf1e0004184eb915cbc10fc05c5 \ + --hash=sha256:0b8db6b5b2d4491ad5b6bdc2bc7c017eec108acbf4e6785f42a9eb0ba234f4c9 \ + --hash=sha256:0c150c7a61ed4a4f4955a96626574e9baf1adf772c2fb61ef6a5027e52803543 \ + --hash=sha256:0f3cec041684de9a4684b1572fe28c7267410e02450f4561700ca5a3bc6695a2 \ + --hash=sha256:1352ae4f7c717ae8cba93421a63373e582d19d55d2ee2cbb184344c82d2ae55a \ + --hash=sha256:177c7c0fce2855833819c98e43c262007f42ce86651ffbb84f37883308cb0e7d \ + --hash=sha256:1978d0021e943aae58b9b0b196fb4895a25cc53d3956b8e35e0b7682eefb6d56 \ + --hash=sha256:1a60bce91f81ddaac922a40bbb571a12c1070cb20ebd6d49c48e0b101d87300d \ + --hash=sha256:1aef18820ef3e4587ebe8b3bc9ba6e55892a6d7b93bac6d29d9f631a3b4befbd \ + --hash=sha256:1e9663daaf7a63ceccbbb8e3808fe90415b0757e2abddbfc2e06c857bf8c5e2b \ + --hash=sha256:20070c65396f7373f5df4005862fa162db5d25d56150bddd0b3e8214e8ef45b4 \ + --hash=sha256:214b7a953d73b5e87f0ebece4a32a5bd83c60a3ecc9d4ec8f1dca968a2d91e99 \ + --hash=sha256:22bebe05a9ffc70ebfa127efbc429bc26ec9e9b4ee4d15a740033efda515cf3d \ + --hash=sha256:24e8abb5878e250f2eb0d7859a8e561846f98910326d06c0d51381fed59357bd \ + --hash=sha256:26fd7cac7dd51011a245f29a2cc6489c4608b5a8ce8d75661bb4a1066c52dfbe \ + --hash=sha256:27b1d3b3915a99208fee9ab092b8184c420f2905b7d7feb4aeb5e4a9c509b8a1 \ + --hash=sha256:27e98004595899949bd7a7b34e91fa7c44d7a97c40fcaf1d874168bb652ec67e \ + --hash=sha256:2b8f60e1b739a74bab7e01fcbe3dddd4657ec685caa04681df9d562ef15b625f \ + --hash=sha256:2de29005e11637e7a2361fa151f780ff8eb2543a0da1413bb951e9f14b699ef3 \ + --hash=sha256:2e8b55d8517a2fda8d95cb45d62a5a8bbf9dd0ad39c5b25c8833efea07b880ca \ + --hash=sha256:2fa4331c200c2521512595253f5bb70858b90f750d39b8cbfd67465f8d1b596d \ + --hash=sha256:3445e07bf2e8ecfeef6ef67ac83de670358abf2996916039b16a218e3d95e97e \ + --hash=sha256:3453e8d41fe5f17d1f8e9c383a7473cd46a63661628ec58e07777c2fff7196dc \ + --hash=sha256:378753b4a4de2a7b34063d6f95ae81bfa7b15f2c1a04a9518e8644e81807ebea \ + --hash=sha256:3af6e48651c4e0d2d166dc1b033b7042ea3f871504b6805ba5f4fe31581d8d38 \ + --hash=sha256:3dfcbc95bd7992b16f3f7ba05af8a64ca694331bd24f9157b49dadeeb287493b \ + --hash=sha256:3f21f0495edea7fdbaaa87e633a8689cd285f8f4af5c869f27bc8074638ad69c \ + --hash=sha256:4041711832360a9b75cfb11b25a6a97c8fb49c07b8bd43d0d02b45d0b499a4ff \ + --hash=sha256:44d61b4b7d0c2c9ac019c314e52d7cbda0ae31078aabd0f22e583af3e0d79723 \ + --hash=sha256:4617e1915a539a0d9a9567795023de41a87106522ff83fbfaf1f6baf8e85437e \ + --hash=sha256:4b232061ca880db21fa14defe219840ad9b74b6158adb52ddf0e87bead9e8493 \ + --hash=sha256:5246b14ca64a8675e0a7161f7af68fe3e910e6b90542b4bfb5439ba752191df6 \ + --hash=sha256:5725dd9cc02068996d4438d397e255dcb1df776b7ceea3b9cb972bdb11260a83 \ + --hash=sha256:583f6a1993ca3369e0f80ba99d796d8e6b1a3a2a442dd4e1a79e652116413091 \ + --hash=sha256:59259dc58e57b10e7e18ce02c311804c10c5a793e6568f8af4dead03264584d1 \ + --hash=sha256:593eba61ba0c3baae5bc9be2f5232430453fb4432048de28399ca7376de9c627 \ + --hash=sha256:59f4a79c19232a5774aee369a0c296712ad0e77f24e62cad53160312b1c1eaa1 \ + --hash=sha256:5f0e260eaf54380380ac3808aa4ebe2d8ca28b9087cf411649f96bad6900c728 \ + --hash=sha256:62d9cfcf4948683a18a9aff0ab7e1474d407b7bab2ca03116109f8464698ab16 \ + --hash=sha256:64607d4cbf1b7e3c3c8a14948b99345eda0e161b852e122c6bb71aab6d1d798c \ + --hash=sha256:655ca44a831ecb238d124e0402d98f6212ac527a0ba6c55ca26f616604e60a45 \ + --hash=sha256:666ecce376999bf619756a24ce15bb14c5bfaf04bf00abc7e663ce17c3f34fe7 \ + --hash=sha256:68049202f67380ff9aa52f12e92b1c30115f32e6895cd7198fa2a7961621fc5a \ + --hash=sha256:69803198097467ee7282750acb507fba35ca22cc3b85f16cf45fb01cb9097730 \ + --hash=sha256:6c7b99ca52c2c1752b544e310101b98a659b720b21db00e65edca34483259967 \ + --hash=sha256:6dd9412824c4ce1aca56c47b0991e65bebb7ac3f4edccfd3f156150c96a7bf25 \ + --hash=sha256:70eb60b3ae9245ddea20f8a4190bd79c705a22f8028aaf8bbdebe4716c3fab24 \ + --hash=sha256:70fb28128acbfd264eda9bf47015537ba3fe86e40d046eb2963d75024be4d055 \ + --hash=sha256:7b2513ba235829860b13faa931f3b6846548021846ac808455301c23a101689d \ + --hash=sha256:7ef9d9da710be50ff6809fed8f1963fecdfecc8b86656cadfca3bc24289414b0 \ + --hash=sha256:81e69b0a0e2537f26d73b4e43ad7bc8c8efb39621639b4434b76a3de50c6966e \ + --hash=sha256:8633e471c6207a039eff6aa116e35f69f3156b3989ea3e2d755f7bc41754a4a7 \ + --hash=sha256:8bd7c8cfc0b8247c8799080fbff54e0b9619e17cdfeb0478ba7295d43f635d7c \ + --hash=sha256:9253fc214112405f0afa7db88739294295f0e08466987f1d70e29930262b4c8f \ + --hash=sha256:99b37292234e61325e7a5bb9689e55e48c3f5f603af88b1642666277a81f1fbd \ + --hash=sha256:9bd7228827ec7bb817089e2eb301d907c0d9827a9e558f22f762bb690b131652 \ + --hash=sha256:9beeb01d8c190d7581a4d59522cd3d4b6887040dcfc744af99aa59fef3e041a8 \ + --hash=sha256:a63cbdd98acef6570c62b92a1e43266f9e8b21e699c363c0fef13bd530799c11 \ + --hash=sha256:a76e42402542b1fae59798fab64432b2d015ab9d0c8c47ba7addddbaf7952333 \ + --hash=sha256:ac0a03221cdb5058ce0167ecc92a8c89e8d0decdc9e99a2ec23380793c4dcb96 \ + --hash=sha256:b0b4136a252cadfa1adb705bb81524eee47d9f6aab4f2ee4fa1e9d3cd4581f64 \ + --hash=sha256:b25bc607423935079e05619d7de556c91fb6adeae9d5f80868dde3468657994b \ + --hash=sha256:b3d504047aba448d70cf6fa22e06cb09f7cbd761939fdd47604f5e007675c24e \ + --hash=sha256:bb47271f60660803ad11f4c61b42242b8c1312a31c98c578f79ef9387bbde21c \ + --hash=sha256:bbb232860e3d03d544bc03ac57855cd82ddf19c7a07651a7c0fdb95e9efea8b9 \ + --hash=sha256:bc27863442d388870c1809a87507727b799c8460573cfbb6dc0eeaef5a11b5ec \ + --hash=sha256:bc51abd01f08117283c5ebf64844a35144a0843ff7b2983e0648e4d3d9f10dbb \ + --hash=sha256:be2eb3f2495ba669d2a985f9b426c1797b7d48d6963899276d22f23e33d47e37 \ + --hash=sha256:bf9db5488121b596dbfc6718c76092fda77b703c1f7533a226a5a9f65248f8ad \ + --hash=sha256:c58e2339def52ef6b71b8f36d13c3688ea23fa093353f3a4fee2556e62086ec9 \ + --hash=sha256:cfbc454a2880389dbb9b5b398e50d439e2e58669160f27b60e5eca11f68ae17c \ + --hash=sha256:cff63a0272fcd259dcc3be1657b07c929c466b067ceb1c20060e8d10af56f5bf \ + --hash=sha256:d115bffdd417c6d806ea9069237a4ae02f513b778e3789a359bc5856e0404cc4 \ + --hash=sha256:d20cfb4e099748ea39e6f7b16c91ab057989712d31761d3300d43134e26e165f \ + --hash=sha256:d48424e39c2611ee1b84ad0f44fb3b2b53d473e65de061e3f460fc0be5f1939d \ + --hash=sha256:e0fa2d4ec53dc51cf7d3bb22e0aa0143966119f42a0c3e4998293a3dd2856b09 \ + --hash=sha256:e32fee8ab45d3c2db6da19a5323bc3362237c8b653c70194414b892fd06a080d \ + --hash=sha256:e35ba67d65d49080e8e5a1dd40101fccdd9798adb9b050ff670b7d74fa41c566 \ + --hash=sha256:e3fb866d9932a3d7d0c82da76d816996d1667c44891bd861a0f97ba27e84fc74 \ + --hash=sha256:e61b02c3f7a1e0b75e20c3978f7135fd13cb6cf551bf4a6d29b999a88830a338 \ + --hash=sha256:e67ba3c290821343c192f7eae1d8fd5999ca2dc99994114643e2f2d3e6138b15 \ + --hash=sha256:e79dd39f1e8c3504be0607e5fc6e86bb60fe3584bec8b782578c3b0fde8d932c \ + --hash=sha256:e89391e6d60251560f0a8f4bd32137b077a80d9b7dbe6d5cab1cd80d2746f648 \ + --hash=sha256:ea7433ce7e4bfc3a85654aeb6747babe3f66eaf9a1d0c1e7a4435bbdf27fea84 \ + --hash=sha256:eaf16ae9ae519a0e237a0f528fd9f0197b9bb70f40263ee57ae53c2b8d48aeb3 \ + --hash=sha256:eb0c341fa71df5a4595f9501df4ac5abfb5a09580081dffbd1ddd4654e6e9123 \ + --hash=sha256:f276b245347e6e36526cbd4a266a417796fc531ddf391e43574cf6466c492520 \ + --hash=sha256:f47ad3d5f3258bd7058d2d506852217865afefe6153a36eb4b6928758041d831 \ + --hash=sha256:f56a6b404f74ab372da986d240e2e002769a7d7102cc73eb238a4f72eec5284e \ + --hash=sha256:f5cf2a0c2bdadf3791b5c205d55a37a54025c6e18a71c71f82bb536cf9a454bf \ + --hash=sha256:f5d36399a1b96e1a5fdc91e0522544580dbebeb1f77f27b2b0ab25559e103b8b \ + --hash=sha256:f60bd8423be1d9d833f230fdbccf8f57af322d96bcad6599e5a771b151398eb2 \ + --hash=sha256:f612463ac081803f243ff13cccc648578e2279295048f2a8d5eb430af2bae6e3 \ + --hash=sha256:f73d3fef726b3243a811121de45193c0ca75f6407fe66f3f4e183c983573e130 \ + --hash=sha256:f82a116a1d03628a8ace4859556fb39fd1424c933341a08ea3ed6de1edb0283b \ + --hash=sha256:fb0ba113b4983beac1a2eb16faffd76cb41e176bf58c4afe3e14b9c681f702de \ + --hash=sha256:fb4f868f712b2dd4bcc538b0a0c1f63a2b1d584c925e69a224d759e7070a12d5 \ + --hash=sha256:fb6116dfb8d1925cbdb52595560584db42a7f664617a1f7d7f6e32f138cdf37d \ + --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ + --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e + # via + # -c python/requirements_compiled.txt + # jsonschema + # referencing +rsa==4.7.2 \ + --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ + --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 + # via + # -c python/requirements_compiled.txt + # google-auth +scipy==1.11.4 \ + --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ + --hash=sha256:028eccd22e654b3ea01ee63705681ee79933652b2d8f873e7949898dda6d11b6 \ + --hash=sha256:1b7c3dca977f30a739e0409fb001056484661cb2541a01aba0bb0029f7b68db8 \ + --hash=sha256:2c6ff6ef9cc27f9b3db93a6f8b38f97387e6e0591600369a297a50a8e96e835d \ + --hash=sha256:36750b7733d960d7994888f0d148d31ea3017ac15eef664194b4ef68d36a4a97 \ + --hash=sha256:530f9ad26440e85766509dbf78edcfe13ffd0ab7fec2560ee5c36ff74d6269ff \ + --hash=sha256:5e347b14fe01003d3b78e196e84bd3f48ffe4c8a7b8a1afbcb8f5505cb710993 \ + --hash=sha256:6550466fbeec7453d7465e74d4f4b19f905642c89a7525571ee91dd7adabb5a3 \ + --hash=sha256:6df1468153a31cf55ed5ed39647279beb9cfb5d3f84369453b49e4b8502394fd \ + --hash=sha256:6e619aba2df228a9b34718efb023966da781e89dd3d21637b27f2e54db0410d7 \ + --hash=sha256:8fce70f39076a5aa62e92e69a7f62349f9574d8405c0a5de6ed3ef72de07f446 \ + --hash=sha256:90a2b78e7f5733b9de748f589f09225013685f9b218275257f8a8168ededaeaa \ + --hash=sha256:91af76a68eeae0064887a48e25c4e616fa519fa0d38602eda7e0f97d65d57937 \ + --hash=sha256:933baf588daa8dc9a92c20a0be32f56d43faf3d1a60ab11b3f08c356430f6e56 \ + --hash=sha256:acf8ed278cc03f5aff035e69cb511741e0418681d25fbbb86ca65429c4f4d9cd \ + --hash=sha256:ad669df80528aeca5f557712102538f4f37e503f0c5b9541655016dd0932ca79 \ + --hash=sha256:b030c6674b9230d37c5c60ab456e2cf12f6784596d15ce8da9365e70896effc4 \ + --hash=sha256:b9999c008ccf00e8fbcce1236f85ade5c569d13144f77a1946bef8863e8f6eb4 \ + --hash=sha256:bc9a714581f561af0848e6b69947fda0614915f072dfd14142ed1bfe1b806710 \ + --hash=sha256:ce7fff2e23ab2cc81ff452a9444c215c28e6305f396b2ba88343a567feec9660 \ + --hash=sha256:cf00bd2b1b0211888d4dc75656c0412213a8b25e80d73898083f402b50f47e41 \ + --hash=sha256:d10e45a6c50211fe256da61a11c34927c68f277e03138777bdebedd933712fea \ + --hash=sha256:ee410e6de8f88fd5cf6eadd73c135020bfbbbdfcd0f6162c36a7638a1ea8cc65 \ + --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ + --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec + # via + # -c python/requirements_compiled.txt + # ray +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via + # -c python/requirements_compiled.txt + # opencensus + # python-dateutil +smart-open==6.2.0 \ + --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ + --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 + # via + # -c python/requirements_compiled.txt + # ray +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc + # via + # -c python/requirements_compiled.txt + # anyio +starlette==0.46.2 \ + --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ + --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 + # via + # -c python/requirements_compiled.txt + # fastapi + # ray +tensorboardx==2.6.2.2 \ + --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ + --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 + # via + # -c python/requirements_compiled.txt + # ray +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d + # via + # -c python/requirements_compiled.txt + # fastapi + # gymnasium + # opentelemetry-api + # opentelemetry-sdk + # opentelemetry-semantic-conventions + # pydantic + # pydantic-core + # pyopenssl + # referencing + # typing-inspection +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/requirements_compiled.txt + # pydantic +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled.txt + # kombu + # pandas +urllib3==1.26.19 \ + --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ + --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 + # via + # -c python/requirements_compiled.txt + # requests +uvicorn==0.22.0 \ + --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ + --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 + # via + # -c python/requirements_compiled.txt + # ray +uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32' \ + --hash=sha256:0878c2640cf341b269b7e128b1a5fed890adc4455513ca710d77d5e93aa6d6a0 \ + --hash=sha256:10d66943def5fcb6e7b37310eb6b5639fd2ccbc38df1177262b0640c3ca68c1f \ + --hash=sha256:10da8046cc4a8f12c91a1c39d1dd1585c41162a15caaef165c2174db9ef18bdc \ + --hash=sha256:17df489689befc72c39a08359efac29bbee8eee5209650d4b9f34df73d22e414 \ + --hash=sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f \ + --hash=sha256:196274f2adb9689a289ad7d65700d37df0c0930fd8e4e743fa4834e850d7719d \ + --hash=sha256:221f4f2a1f46032b403bf3be628011caf75428ee3cc204a22addf96f586b19fd \ + --hash=sha256:2d1f581393673ce119355d56da84fe1dd9d2bb8b3d13ce792524e1607139feff \ + --hash=sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c \ + --hash=sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3 \ + --hash=sha256:4509360fcc4c3bd2c70d87573ad472de40c13387f5fda8cb58350a1d7475e58d \ + --hash=sha256:460def4412e473896ef179a1671b40c039c7012184b627898eea5072ef6f017a \ + --hash=sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb \ + --hash=sha256:46923b0b5ee7fc0020bef24afe7836cb068f5050ca04caf6b487c513dc1a20b2 \ + --hash=sha256:53e420a3afe22cdcf2a0f4846e377d16e718bc70103d7088a4f7623567ba5fb0 \ + --hash=sha256:5ee4d4ef48036ff6e5cfffb09dd192c7a5027153948d85b8da7ff705065bacc6 \ + --hash=sha256:67dd654b8ca23aed0a8e99010b4c34aca62f4b7fce88f39d452ed7622c94845c \ + --hash=sha256:787ae31ad8a2856fc4e7c095341cccc7209bd657d0e71ad0dc2ea83c4a6fa8af \ + --hash=sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc \ + --hash=sha256:87c43e0f13022b998eb9b973b5e97200c8b90823454d4bc06ab33829e09fb9bb \ + --hash=sha256:88cb67cdbc0e483da00af0b2c3cdad4b7c61ceb1ee0f33fe00e09c81e3a6cb75 \ + --hash=sha256:8a375441696e2eda1c43c44ccb66e04d61ceeffcd76e4929e527b7fa401b90fb \ + --hash=sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553 \ + --hash=sha256:b9fb766bb57b7388745d8bcc53a359b116b8a04c83a2288069809d2b3466c37e \ + --hash=sha256:baa0e6291d91649c6ba4ed4b2f982f9fa165b5bbd50a9e203c416a2797bab3c6 \ + --hash=sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d \ + --hash=sha256:bc09f0ff191e61c2d592a752423c767b4ebb2986daa9ed62908e2b1b9a9ae206 \ + --hash=sha256:bd53ecc9a0f3d87ab847503c2e1552b690362e005ab54e8a48ba97da3924c0dc \ + --hash=sha256:bfd55dfcc2a512316e65f16e503e9e450cab148ef11df4e4e679b5e8253a5281 \ + --hash=sha256:c097078b8031190c934ed0ebfee8cc5f9ba9642e6eb88322b9958b649750f72b \ + --hash=sha256:c0f3fa6200b3108919f8bdabb9a7f87f20e7097ea3c543754cabc7d717d95cf8 \ + --hash=sha256:e678ad6fe52af2c58d2ae3c73dc85524ba8abe637f134bf3564ed07f555c5e79 \ + --hash=sha256:ec7e6b09a6fdded42403182ab6b832b71f4edaf7f37a9a0e371a01db5f0cb45f \ + --hash=sha256:f0ce1b49560b1d2d8a2977e3ba4afb2414fb46b86a1b64056bc4ab929efdafbe \ + --hash=sha256:f38b2e090258d051d68a5b14d1da7203a3c3677321cf32a95a6f4db4dd8b6f26 \ + --hash=sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816 \ + --hash=sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2 + # via + # -c python/requirements_compiled.txt + # uvicorn +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled.txt + # amqp + # celery + # kombu +virtualenv==20.29.1 \ + --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ + --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 + # via + # -c python/requirements_compiled.txt + # ray +watchfiles==0.19.0 \ + --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ + --hash=sha256:09ea3397aecbc81c19ed7f025e051a7387feefdb789cf768ff994c1228182fda \ + --hash=sha256:176a9a7641ec2c97b24455135d58012a5be5c6217fc4d5fef0b2b9f75dbf5154 \ + --hash=sha256:18b28f6ad871b82df9542ff958d0c86bb0d8310bb09eb8e87d97318a3b5273af \ + --hash=sha256:20b44221764955b1e703f012c74015306fb7e79a00c15370785f309b1ed9aa8d \ + --hash=sha256:3d7d267d27aceeeaa3de0dd161a0d64f0a282264d592e335fff7958cc0cbae7c \ + --hash=sha256:5471582658ea56fca122c0f0d0116a36807c63fefd6fdc92c71ca9a4491b6b48 \ + --hash=sha256:5569fc7f967429d4bc87e355cdfdcee6aabe4b620801e2cf5805ea245c06097c \ + --hash=sha256:68dce92b29575dda0f8d30c11742a8e2b9b8ec768ae414b54f7453f27bdf9545 \ + --hash=sha256:79c533ff593db861ae23436541f481ec896ee3da4e5db8962429b441bbaae16e \ + --hash=sha256:7f3920b1285a7d3ce898e303d84791b7bf40d57b7695ad549dc04e6a44c9f120 \ + --hash=sha256:91633e64712df3051ca454ca7d1b976baf842d7a3640b87622b323c55f3345e7 \ + --hash=sha256:945be0baa3e2440151eb3718fd8846751e8b51d8de7b884c90b17d271d34cae8 \ + --hash=sha256:9afd0d69429172c796164fd7fe8e821ade9be983f51c659a38da3faaaaac44dc \ + --hash=sha256:9c75eff897786ee262c9f17a48886f4e98e6cfd335e011c591c305e5d083c056 \ + --hash=sha256:b538014a87f94d92f98f34d3e6d2635478e6be6423a9ea53e4dd96210065e193 \ + --hash=sha256:b6577b8c6c8701ba8642ea9335a129836347894b666dd1ec2226830e263909d3 \ + --hash=sha256:c0376deac92377817e4fb8f347bf559b7d44ff556d9bc6f6208dd3f79f104aaf \ + --hash=sha256:cae3dde0b4b2078f31527acff6f486e23abed307ba4d3932466ba7cdd5ecec79 \ + --hash=sha256:cb5d45c4143c1dd60f98a16187fd123eda7248f84ef22244818c18d531a249d1 \ + --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ + --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 + # via + # -c python/requirements_compiled.txt + # ray + # uvicorn +wcwidth==0.2.13 \ + --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ + --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 + # via + # -c python/requirements_compiled.txt + # prompt-toolkit +websockets==11.0.3 \ + --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \ + --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \ + --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \ + --hash=sha256:0ee68fe502f9031f19d495dae2c268830df2760c0524cbac5d759921ba8c8e82 \ + --hash=sha256:1553cb82942b2a74dd9b15a018dce645d4e68674de2ca31ff13ebc2d9f283788 \ + --hash=sha256:1a073fc9ab1c8aff37c99f11f1641e16da517770e31a37265d2755282a5d28aa \ + --hash=sha256:1d2256283fa4b7f4c7d7d3e84dc2ece74d341bce57d5b9bf385df109c2a1a82f \ + --hash=sha256:1d5023a4b6a5b183dc838808087033ec5df77580485fc533e7dab2567851b0a4 \ + --hash=sha256:1fdf26fa8a6a592f8f9235285b8affa72748dc12e964a5518c6c5e8f916716f7 \ + --hash=sha256:2529338a6ff0eb0b50c7be33dc3d0e456381157a31eefc561771ee431134a97f \ + --hash=sha256:279e5de4671e79a9ac877427f4ac4ce93751b8823f276b681d04b2156713b9dd \ + --hash=sha256:2d903ad4419f5b472de90cd2d40384573b25da71e33519a67797de17ef849b69 \ + --hash=sha256:332d126167ddddec94597c2365537baf9ff62dfcc9db4266f263d455f2f031cb \ + --hash=sha256:34fd59a4ac42dff6d4681d8843217137f6bc85ed29722f2f7222bd619d15e95b \ + --hash=sha256:3580dd9c1ad0701169e4d6fc41e878ffe05e6bdcaf3c412f9d559389d0c9e016 \ + --hash=sha256:3ccc8a0c387629aec40f2fc9fdcb4b9d5431954f934da3eaf16cdc94f67dbfac \ + --hash=sha256:41f696ba95cd92dc047e46b41b26dd24518384749ed0d99bea0a941ca87404c4 \ + --hash=sha256:42cc5452a54a8e46a032521d7365da775823e21bfba2895fb7b77633cce031bb \ + --hash=sha256:4841ed00f1026dfbced6fca7d963c4e7043aa832648671b5138008dc5a8f6d99 \ + --hash=sha256:4b253869ea05a5a073ebfdcb5cb3b0266a57c3764cf6fe114e4cd90f4bfa5f5e \ + --hash=sha256:54c6e5b3d3a8936a4ab6870d46bdd6ec500ad62bde9e44462c32d18f1e9a8e54 \ + --hash=sha256:619d9f06372b3a42bc29d0cd0354c9bb9fb39c2cbc1a9c5025b4538738dbffaf \ + --hash=sha256:6505c1b31274723ccaf5f515c1824a4ad2f0d191cec942666b3d0f3aa4cb4007 \ + --hash=sha256:660e2d9068d2bedc0912af508f30bbeb505bbbf9774d98def45f68278cea20d3 \ + --hash=sha256:6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6 \ + --hash=sha256:68b977f21ce443d6d378dbd5ca38621755f2063d6fdb3335bda981d552cfff86 \ + --hash=sha256:69269f3a0b472e91125b503d3c0b3566bda26da0a3261c49f0027eb6075086d1 \ + --hash=sha256:6f1a3f10f836fab6ca6efa97bb952300b20ae56b409414ca85bff2ad241d2a61 \ + --hash=sha256:7622a89d696fc87af8e8d280d9b421db5133ef5b29d3f7a1ce9f1a7bf7fcfa11 \ + --hash=sha256:777354ee16f02f643a4c7f2b3eff8027a33c9861edc691a2003531f5da4f6bc8 \ + --hash=sha256:84d27a4832cc1a0ee07cdcf2b0629a8a72db73f4cf6de6f0904f6661227f256f \ + --hash=sha256:8531fdcad636d82c517b26a448dcfe62f720e1922b33c81ce695d0edb91eb931 \ + --hash=sha256:86d2a77fd490ae3ff6fae1c6ceaecad063d3cc2320b44377efdde79880e11526 \ + --hash=sha256:88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016 \ + --hash=sha256:8a34e13a62a59c871064dfd8ffb150867e54291e46d4a7cf11d02c94a5275bae \ + --hash=sha256:8c82f11964f010053e13daafdc7154ce7385ecc538989a354ccc7067fd7028fd \ + --hash=sha256:92b2065d642bf8c0a82d59e59053dd2fdde64d4ed44efe4870fa816c1232647b \ + --hash=sha256:97b52894d948d2f6ea480171a27122d77af14ced35f62e5c892ca2fae9344311 \ + --hash=sha256:9d9acd80072abcc98bd2c86c3c9cd4ac2347b5a5a0cae7ed5c0ee5675f86d9af \ + --hash=sha256:9f59a3c656fef341a99e3d63189852be7084c0e54b75734cde571182c087b152 \ + --hash=sha256:aa5003845cdd21ac0dc6c9bf661c5beddd01116f6eb9eb3c8e272353d45b3288 \ + --hash=sha256:b16fff62b45eccb9c7abb18e60e7e446998093cdcb50fed33134b9b6878836de \ + --hash=sha256:b30c6590146e53149f04e85a6e4fcae068df4289e31e4aee1fdf56a0dead8f97 \ + --hash=sha256:b58cbf0697721120866820b89f93659abc31c1e876bf20d0b3d03cef14faf84d \ + --hash=sha256:b67c6f5e5a401fc56394f191f00f9b3811fe843ee93f4a70df3c389d1adf857d \ + --hash=sha256:bceab846bac555aff6427d060f2fcfff71042dba6f5fca7dc4f75cac815e57ca \ + --hash=sha256:bee9fcb41db2a23bed96c6b6ead6489702c12334ea20a297aa095ce6d31370d0 \ + --hash=sha256:c114e8da9b475739dde229fd3bc6b05a6537a88a578358bc8eb29b4030fac9c9 \ + --hash=sha256:c1f0524f203e3bd35149f12157438f406eff2e4fb30f71221c8a5eceb3617b6b \ + --hash=sha256:c792ea4eabc0159535608fc5658a74d1a81020eb35195dd63214dcf07556f67e \ + --hash=sha256:c7f3cb904cce8e1be667c7e6fef4516b98d1a6a0635a58a57528d577ac18a128 \ + --hash=sha256:d67ac60a307f760c6e65dad586f556dde58e683fab03323221a4e530ead6f74d \ + --hash=sha256:dcacf2c7a6c3a84e720d1bb2b543c675bf6c40e460300b628bab1b1efc7c034c \ + --hash=sha256:de36fe9c02995c7e6ae6efe2e205816f5f00c22fd1fbf343d4d18c3d5ceac2f5 \ + --hash=sha256:def07915168ac8f7853812cc593c71185a16216e9e4fa886358a17ed0fd9fcf6 \ + --hash=sha256:df41b9bc27c2c25b486bae7cf42fccdc52ff181c8c387bfd026624a491c2671b \ + --hash=sha256:e052b8467dd07d4943936009f46ae5ce7b908ddcac3fda581656b1b19c083d9b \ + --hash=sha256:e063b1865974611313a3849d43f2c3f5368093691349cf3c7c8f8f75ad7cb280 \ + --hash=sha256:e1459677e5d12be8bbc7584c35b992eea142911a6236a3278b9b5ce3326f282c \ + --hash=sha256:e1a99a7a71631f0efe727c10edfba09ea6bee4166a6f9c19aafb6c0b5917d09c \ + --hash=sha256:e590228200fcfc7e9109509e4d9125eace2042fd52b595dd22bbc34bb282307f \ + --hash=sha256:e6316827e3e79b7b8e7d8e3b08f4e331af91a48e794d5d8b099928b6f0b85f20 \ + --hash=sha256:e7837cb169eca3b3ae94cc5787c4fed99eef74c0ab9506756eea335e0d6f3ed8 \ + --hash=sha256:e848f46a58b9fcf3d06061d17be388caf70ea5b8cc3466251963c8345e13f7eb \ + --hash=sha256:ed058398f55163a79bb9f06a90ef9ccc063b204bb346c4de78efc5d15abfe602 \ + --hash=sha256:f2e58f2c36cc52d41f2659e4c0cbf7353e28c8c9e63e30d8c6d3494dc9fdedcf \ + --hash=sha256:f467ba0050b7de85016b43f5a22b46383ef004c4f672148a8abf32bc999a87f0 \ + --hash=sha256:f61bdb1df43dc9c131791fbc2355535f9024b9a04398d3bd0684fc16ab07df74 \ + --hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \ + --hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564 + # via + # -c python/requirements_compiled.txt + # uvicorn +yarl==1.18.3 \ + --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ + --hash=sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193 \ + --hash=sha256:045b8482ce9483ada4f3f23b3774f4e1bf4f23a2d5c912ed5170f68efb053318 \ + --hash=sha256:09c7907c8548bcd6ab860e5f513e727c53b4a714f459b084f6580b49fa1b9cee \ + --hash=sha256:0b0cad37311123211dc91eadcb322ef4d4a66008d3e1bdc404808992260e1a0e \ + --hash=sha256:0b3c92fa08759dbf12b3a59579a4096ba9af8dd344d9a813fc7f5070d86bbab1 \ + --hash=sha256:0fb2171a4486bb075316ee754c6d8382ea6eb8b399d4ec62fde2b591f879778a \ + --hash=sha256:1a74a13a4c857a84a845505fd2d68e54826a2cd01935a96efb1e9d86c728e186 \ + --hash=sha256:1d407181cfa6e70077df3377938c08012d18893f9f20e92f7d2f314a437c30b1 \ + --hash=sha256:1dd4bdd05407ced96fed3d7f25dbbf88d2ffb045a0db60dbc247f5b3c5c25d50 \ + --hash=sha256:25b411eddcfd56a2f0cd6a384e9f4f7aa3efee14b188de13048c25b5e91f1640 \ + --hash=sha256:2d06d3005e668744e11ed80812e61efd77d70bb7f03e33c1598c301eea20efbb \ + --hash=sha256:2ec9bbba33b2d00999af4631a3397d1fd78290c48e2a3e52d8dd72db3a067ac8 \ + --hash=sha256:3236da9272872443f81fedc389bace88408f64f89f75d1bdb2256069a8730ccc \ + --hash=sha256:35098b24e0327fc4ebdc8ffe336cee0a87a700c24ffed13161af80124b7dc8e5 \ + --hash=sha256:41f7ce59d6ee7741af71d82020346af364949314ed3d87553763a2df1829cc58 \ + --hash=sha256:436c4fc0a4d66b2badc6c5fc5ef4e47bb10e4fd9bf0c79524ac719a01f3607c2 \ + --hash=sha256:4891ed92157e5430874dad17b15eb1fda57627710756c27422200c52d8a4e393 \ + --hash=sha256:4ac515b860c36becb81bb84b667466885096b5fc85596948548b667da3bf9f24 \ + --hash=sha256:5094d9206c64181d0f6e76ebd8fb2f8fe274950a63890ee9e0ebfd58bf9d787b \ + --hash=sha256:54d6921f07555713b9300bee9c50fb46e57e2e639027089b1d795ecd9f7fa910 \ + --hash=sha256:578e281c393af575879990861823ef19d66e2b1d0098414855dd367e234f5b3c \ + --hash=sha256:5a3f356548e34a70b0172d8890006c37be92995f62d95a07b4a42e90fba54272 \ + --hash=sha256:602d98f2c2d929f8e697ed274fbadc09902c4025c5a9963bf4e9edfc3ab6f7ed \ + --hash=sha256:61b1a825a13bef4a5f10b1885245377d3cd0bf87cba068e1d9a88c2ae36880e1 \ + --hash=sha256:61e5e68cb65ac8f547f6b5ef933f510134a6bf31bb178be428994b0cb46c2a04 \ + --hash=sha256:61ee62ead9b68b9123ec24bc866cbef297dd266175d53296e2db5e7f797f902d \ + --hash=sha256:6333c5a377c8e2f5fae35e7b8f145c617b02c939d04110c76f29ee3676b5f9a5 \ + --hash=sha256:6748dbf9bfa5ba1afcc7556b71cda0d7ce5f24768043a02a58846e4a443d808d \ + --hash=sha256:67a283dd2882ac98cc6318384f565bffc751ab564605959df4752d42483ad889 \ + --hash=sha256:75674776d96d7b851b6498f17824ba17849d790a44d282929c42dbb77d4f17ae \ + --hash=sha256:757e81cae69244257d125ff31663249b3013b5dc0a8520d73694aed497fb195b \ + --hash=sha256:77a6e85b90a7641d2e07184df5557132a337f136250caafc9ccaa4a2a998ca2c \ + --hash=sha256:7c33dd1931a95e5d9a772d0ac5e44cac8957eaf58e3c8da8c1414de7dd27c576 \ + --hash=sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34 \ + --hash=sha256:7e2ee16578af3b52ac2f334c3b1f92262f47e02cc6193c598502bd46f5cd1477 \ + --hash=sha256:80316a8bd5109320d38eef8833ccf5f89608c9107d02d2a7f985f98ed6876990 \ + --hash=sha256:82123d0c954dc58db301f5021a01854a85bf1f3bb7d12ae0c01afc414a882ca2 \ + --hash=sha256:84b2deecba4a3f1a398df819151eb72d29bfeb3b69abb145a00ddc8d30094512 \ + --hash=sha256:8503ad47387b8ebd39cbbbdf0bf113e17330ffd339ba1144074da24c545f0069 \ + --hash=sha256:877d209b6aebeb5b16c42cbb377f5f94d9e556626b1bfff66d7b0d115be88d0a \ + --hash=sha256:8874027a53e3aea659a6d62751800cf6e63314c160fd607489ba5c2edd753cf6 \ + --hash=sha256:88a19f62ff30117e706ebc9090b8ecc79aeb77d0b1f5ec10d2d27a12bc9f66d0 \ + --hash=sha256:8d39d351e7faf01483cc7ff7c0213c412e38e5a340238826be7e0e4da450fdc8 \ + --hash=sha256:90adb47ad432332d4f0bc28f83a5963f426ce9a1a8809f5e584e704b82685dcb \ + --hash=sha256:913829534200eb0f789d45349e55203a091f45c37a2674678744ae52fae23efa \ + --hash=sha256:93b2e109287f93db79210f86deb6b9bbb81ac32fc97236b16f7433db7fc437d8 \ + --hash=sha256:9d41beda9dc97ca9ab0b9888cb71f7539124bc05df02c0cff6e5acc5a19dcc6e \ + --hash=sha256:a440a2a624683108a1b454705ecd7afc1c3438a08e890a1513d468671d90a04e \ + --hash=sha256:a4bb030cf46a434ec0225bddbebd4b89e6471814ca851abb8696170adb163985 \ + --hash=sha256:a9ca04806f3be0ac6d558fffc2fdf8fcef767e0489d2684a21912cc4ed0cd1b8 \ + --hash=sha256:ac1801c45cbf77b6c99242eeff4fffb5e4e73a800b5c4ad4fc0be5def634d2e1 \ + --hash=sha256:ac36703a585e0929b032fbaab0707b75dc12703766d0b53486eabd5139ebadd5 \ + --hash=sha256:b1771de9944d875f1b98a745bc547e684b863abf8f8287da8466cf470ef52690 \ + --hash=sha256:b464c4ab4bfcb41e3bfd3f1c26600d038376c2de3297760dfe064d2cb7ea8e10 \ + --hash=sha256:b4f6450109834af88cb4cc5ecddfc5380ebb9c228695afc11915a0bf82116789 \ + --hash=sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b \ + --hash=sha256:b643562c12680b01e17239be267bc306bbc6aac1f34f6444d1bded0c5ce438ca \ + --hash=sha256:b958ddd075ddba5b09bb0be8a6d9906d2ce933aee81100db289badbeb966f54e \ + --hash=sha256:b9d60031cf568c627d028239693fd718025719c02c9f55df0a53e587aab951b5 \ + --hash=sha256:ba23302c0c61a9999784e73809427c9dbedd79f66a13d84ad1b1943802eaaf59 \ + --hash=sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9 \ + --hash=sha256:c017a3b6df3a1bd45b9fa49a0f54005e53fbcad16633870104b66fa1a30a29d8 \ + --hash=sha256:c1e1cc06da1491e6734f0ea1e6294ce00792193c463350626571c287c9a704db \ + --hash=sha256:c654d5207c78e0bd6d749f6dae1dcbbfde3403ad3a4b11f3c5544d9906969dde \ + --hash=sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7 \ + --hash=sha256:c7d79f7d9aabd6011004e33b22bc13056a3e3fb54794d138af57f5ee9d9032cb \ + --hash=sha256:ccaa3a4b521b780a7e771cc336a2dba389a0861592bbce09a476190bb0c8b4b3 \ + --hash=sha256:ccd17349166b1bee6e529b4add61727d3f55edb7babbe4069b5764c9587a8cc6 \ + --hash=sha256:ce1af883b94304f493698b00d0f006d56aea98aeb49d75ec7d98cd4a777e9285 \ + --hash=sha256:d0e883008013c0e4aef84dcfe2a0b172c4d23c2669412cf5b3371003941f72bb \ + --hash=sha256:d980e0325b6eddc81331d3f4551e2a333999fb176fd153e075c6d1c2530aa8a8 \ + --hash=sha256:e17c9361d46a4d5addf777c6dd5eab0715a7684c2f11b88c67ac37edfba6c482 \ + --hash=sha256:e2c08cc9b16f4f4bc522771d96734c7901e7ebef70c6c5c35dd0f10845270bcd \ + --hash=sha256:e35ef8683211db69ffe129a25d5634319a677570ab6b2eba4afa860f54eeaf75 \ + --hash=sha256:e3b9fd71836999aad54084906f8663dffcd2a7fb5cdafd6c37713b2e72be1760 \ + --hash=sha256:ef9f7768395923c3039055c14334ba4d926f3baf7b776c923c93d80195624782 \ + --hash=sha256:f52a265001d830bc425f82ca9eabda94a64a4d753b07d623a9f2863fde532b53 \ + --hash=sha256:f91c4803173928a25e1a55b943c81f55b8872f0018be83e3ad4938adffb77dd2 \ + --hash=sha256:fbd6748e8ab9b41171bb95c6142faf068f5ef1511935a0aa07025438dd9a9bc1 \ + --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ + --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 + # via + # -c python/requirements_compiled.txt + # aiohttp +zipp==3.19.2 \ + --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c + # via + # -c python/requirements_compiled.txt + # importlib-metadata diff --git a/python/deplocks/ray_img/ray_img_py39.lock b/python/deplocks/ray_img/ray_img_py39.lock new file mode 100644 index 000000000000..9787b8fc5971 --- /dev/null +++ b/python/deplocks/ray_img/ray_img_py39.lock @@ -0,0 +1,2173 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --generate-hashes --strip-extras --unsafe-package setuptools --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match --no-strip-markers --emit-index-url --emit-find-links --python-version=3.9 --find-links=.whl/ -c python/requirements_compiled.txt - -o python/deplocks/ray_img/ray_img_py39.lock +--index-url https://pypi.org/simple +--extra-index-url https://download.pytorch.org/whl/cpu +--find-links .whl/ +--find-links https://data.pyg.org/whl/torch-2.3.0+cpu.html + +aiohappyeyeballs==2.6.1 \ + --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ + --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 + # via + # -c python/requirements_compiled.txt + # aiohttp +aiohttp==3.11.16 \ + --hash=sha256:004511d3413737700835e949433536a2fe95a7d0297edd911a1e9705c5b5ea43 \ + --hash=sha256:0902e887b0e1d50424112f200eb9ae3dfed6c0d0a19fc60f633ae5a57c809656 \ + --hash=sha256:09b00dd520d88eac9d1768439a59ab3d145065c91a8fab97f900d1b5f802895e \ + --hash=sha256:0a2f451849e6b39e5c226803dcacfa9c7133e9825dcefd2f4e837a2ec5a3bb98 \ + --hash=sha256:0a950c2eb8ff17361abd8c85987fd6076d9f47d040ebffce67dce4993285e973 \ + --hash=sha256:0ad1fb47da60ae1ddfb316f0ff16d1f3b8e844d1a1e154641928ea0583d486ed \ + --hash=sha256:13ceac2c5cdcc3f64b9015710221ddf81c900c5febc505dbd8f810e770011540 \ + --hash=sha256:14461157d8426bcb40bd94deb0450a6fa16f05129f7da546090cebf8f3123b0f \ + --hash=sha256:16f8a2c9538c14a557b4d309ed4d0a7c60f0253e8ed7b6c9a2859a7582f8b1b8 \ + --hash=sha256:17ae4664031aadfbcb34fd40ffd90976671fa0c0286e6c4113989f78bebab37a \ + --hash=sha256:1ce63ae04719513dd2651202352a2beb9f67f55cb8490c40f056cea3c5c355ce \ + --hash=sha256:23a15727fbfccab973343b6d1b7181bfb0b4aa7ae280f36fd2f90f5476805682 \ + --hash=sha256:2540ddc83cc724b13d1838026f6a5ad178510953302a49e6d647f6e1de82bc34 \ + --hash=sha256:37dcee4906454ae377be5937ab2a66a9a88377b11dd7c072df7a7c142b63c37c \ + --hash=sha256:38bea84ee4fe24ebcc8edeb7b54bf20f06fd53ce4d2cc8b74344c5b9620597fd \ + --hash=sha256:3ab3367bb7f61ad18793fea2ef71f2d181c528c87948638366bf1de26e239183 \ + --hash=sha256:3ad1d59fd7114e6a08c4814983bb498f391c699f3c78712770077518cae63ff7 \ + --hash=sha256:3b4e6db8dc4879015b9955778cfb9881897339c8fab7b3676f8433f849425913 \ + --hash=sha256:3e061b09f6fa42997cf627307f220315e313ece74907d35776ec4373ed718b86 \ + --hash=sha256:42864e70a248f5f6a49fdaf417d9bc62d6e4d8ee9695b24c5916cb4bb666c802 \ + --hash=sha256:493910ceb2764f792db4dc6e8e4b375dae1b08f72e18e8f10f18b34ca17d0979 \ + --hash=sha256:4d0c970c0d602b1017e2067ff3b7dac41c98fef4f7472ec2ea26fd8a4e8c2149 \ + --hash=sha256:54eb3aead72a5c19fad07219acd882c1643a1027fbcdefac9b502c267242f955 \ + --hash=sha256:56a3443aca82abda0e07be2e1ecb76a050714faf2be84256dae291182ba59049 \ + --hash=sha256:576f5ca28d1b3276026f7df3ec841ae460e0fc3aac2a47cbf72eabcfc0f102e1 \ + --hash=sha256:58ede86453a6cf2d6ce40ef0ca15481677a66950e73b0a788917916f7e35a0bb \ + --hash=sha256:61c721764e41af907c9d16b6daa05a458f066015abd35923051be8705108ed17 \ + --hash=sha256:634d96869be6c4dc232fc503e03e40c42d32cfaa51712aee181e922e61d74814 \ + --hash=sha256:696ef00e8a1f0cec5e30640e64eca75d8e777933d1438f4facc9c0cdf288a810 \ + --hash=sha256:69a2cbd61788d26f8f1e626e188044834f37f6ae3f937bd9f08b65fc9d7e514e \ + --hash=sha256:6a792ce34b999fbe04a7a71a90c74f10c57ae4c51f65461a411faa70e154154e \ + --hash=sha256:6ac13b71761e49d5f9e4d05d33683bbafef753e876e8e5a7ef26e937dd766713 \ + --hash=sha256:6fdec0213244c39973674ca2a7f5435bf74369e7d4e104d6c7473c81c9bcc8c4 \ + --hash=sha256:72b1b03fb4655c1960403c131740755ec19c5898c82abd3961c364c2afd59fe7 \ + --hash=sha256:745f1ed5e2c687baefc3c5e7b4304e91bf3e2f32834d07baaee243e349624b24 \ + --hash=sha256:776c8e959a01e5e8321f1dec77964cb6101020a69d5a94cd3d34db6d555e01f7 \ + --hash=sha256:780df0d837276276226a1ff803f8d0fa5f8996c479aeef52eb040179f3156cbd \ + --hash=sha256:78e6e23b954644737e385befa0deb20233e2dfddf95dd11e9db752bdd2a294d3 \ + --hash=sha256:7951decace76a9271a1ef181b04aa77d3cc309a02a51d73826039003210bdc86 \ + --hash=sha256:7ba92a2d9ace559a0a14b03d87f47e021e4fa7681dc6970ebbc7b447c7d4b7cd \ + --hash=sha256:7f6428fee52d2bcf96a8aa7b62095b190ee341ab0e6b1bcf50c615d7966fd45b \ + --hash=sha256:87944bd16b7fe6160607f6a17808abd25f17f61ae1e26c47a491b970fb66d8cb \ + --hash=sha256:87a6e922b2b2401e0b0cf6b976b97f11ec7f136bfed445e16384fbf6fd5e8602 \ + --hash=sha256:8cb0688a8d81c63d716e867d59a9ccc389e97ac7037ebef904c2b89334407180 \ + --hash=sha256:8df6612df74409080575dca38a5237282865408016e65636a76a2eb9348c2567 \ + --hash=sha256:911a6e91d08bb2c72938bc17f0a2d97864c531536b7832abee6429d5296e5b27 \ + --hash=sha256:92b7ee222e2b903e0a4b329a9943d432b3767f2d5029dbe4ca59fb75223bbe2e \ + --hash=sha256:938f756c2b9374bbcc262a37eea521d8a0e6458162f2a9c26329cc87fdf06534 \ + --hash=sha256:9756d9b9d4547e091f99d554fbba0d2a920aab98caa82a8fb3d3d9bee3c9ae85 \ + --hash=sha256:98b88a2bf26965f2015a771381624dd4b0839034b70d406dc74fd8be4cc053e3 \ + --hash=sha256:9b751a6306f330801665ae69270a8a3993654a85569b3469662efaad6cf5cc50 \ + --hash=sha256:a2a450bcce4931b295fc0848f384834c3f9b00edfc2150baafb4488c27953de6 \ + --hash=sha256:a3814760a1a700f3cfd2f977249f1032301d0a12c92aba74605cfa6ce9f78489 \ + --hash=sha256:a5abcbba9f4b463a45c8ca8b7720891200658f6f46894f79517e6cd11f3405ca \ + --hash=sha256:a6db7458ab89c7d80bc1f4e930cc9df6edee2200127cfa6f6e080cf619eddfbd \ + --hash=sha256:ad497f38a0d6c329cb621774788583ee12321863cd4bd9feee1effd60f2ad133 \ + --hash=sha256:ad9509ffb2396483ceacb1eee9134724443ee45b92141105a4645857244aecc8 \ + --hash=sha256:bbcba75fe879ad6fd2e0d6a8d937f34a571f116a0e4db37df8079e738ea95c71 \ + --hash=sha256:c10d85e81d0b9ef87970ecbdbfaeec14a361a7fa947118817fcea8e45335fa46 \ + --hash=sha256:c15b2271c44da77ee9d822552201180779e5e942f3a71fb74e026bf6172ff287 \ + --hash=sha256:ca37057625693d097543bd88076ceebeb248291df9d6ca8481349efc0b05dcd0 \ + --hash=sha256:cc3a145479a76ad0ed646434d09216d33d08eef0d8c9a11f5ae5cdc37caa3540 \ + --hash=sha256:ccf10f16ab498d20e28bc2b5c1306e9c1512f2840f7b6a67000a517a4b37d5ee \ + --hash=sha256:cd464ba806e27ee24a91362ba3621bfc39dbbb8b79f2e1340201615197370f7c \ + --hash=sha256:d007aa39a52d62373bd23428ba4a2546eed0e7643d7bf2e41ddcefd54519842c \ + --hash=sha256:d0666afbe984f6933fe72cd1f1c3560d8c55880a0bdd728ad774006eb4241ecd \ + --hash=sha256:d07502cc14ecd64f52b2a74ebbc106893d9a9717120057ea9ea1fd6568a747e7 \ + --hash=sha256:d489d9778522fbd0f8d6a5c6e48e3514f11be81cb0a5954bdda06f7e1594b321 \ + --hash=sha256:df7db76400bf46ec6a0a73192b14c8295bdb9812053f4fe53f4e789f3ea66bbb \ + --hash=sha256:e3538bc9fe1b902bef51372462e3d7c96fce2b566642512138a480b7adc9d508 \ + --hash=sha256:e87fd812899aa78252866ae03a048e77bd11b80fb4878ce27c23cade239b42b2 \ + --hash=sha256:ecdb8173e6c7aa09eee342ac62e193e6904923bd232e76b4157ac0bfa670609f \ + --hash=sha256:f244b8e541f414664889e2c87cac11a07b918cb4b540c36f7ada7bfa76571ea2 \ + --hash=sha256:f4065145bf69de124accdd17ea5f4dc770da0a6a6e440c53f6e0a8c27b3e635c \ + --hash=sha256:f420bfe862fb357a6d76f2065447ef6f484bc489292ac91e29bc65d2d7a2c84d \ + --hash=sha256:f6ddd90d9fb4b501c97a4458f1c1720e42432c26cb76d28177c5b5ad4e332601 \ + --hash=sha256:fa73e8c2656a3653ae6c307b3f4e878a21f87859a9afab228280ddccd7369d71 \ + --hash=sha256:fadbb8f1d4140825069db3fedbbb843290fd5f5bc0a5dbd7eaf81d91bf1b003b \ + --hash=sha256:fb3d0cc5cdb926090748ea60172fa8a213cec728bd6c54eae18b96040fcd6227 \ + --hash=sha256:fb46bb0f24813e6cede6cc07b1961d4b04f331f7112a23b5e21f567da4ee50aa \ + --hash=sha256:fd36c119c5d6551bce374fcb5c19269638f8d09862445f85a5a48596fd59f4bb + # via + # -c python/requirements_compiled.txt + # aiohttp-cors + # ray +aiohttp-cors==0.7.0 \ + --hash=sha256:0451ba59fdf6909d0e2cd21e4c0a43752bc0703d33fc78ae94d9d9321710193e \ + --hash=sha256:4d39c6d7100fd9764ed1caf8cebf0eb01bf5e3f24e2e073fda6234bc48b19f5d + # via + # -c python/requirements_compiled.txt + # ray +aiosignal==1.3.1 \ + --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ + --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 + # via + # -c python/requirements_compiled.txt + # aiohttp +amqp==5.3.1 \ + --hash=sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2 \ + --hash=sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432 + # via + # -c python/requirements_compiled.txt + # kombu +annotated-types==0.6.0 \ + --hash=sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43 \ + --hash=sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d + # via + # -c python/requirements_compiled.txt + # pydantic +anyio==3.7.1 \ + --hash=sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780 \ + --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5 + # via + # -c python/requirements_compiled.txt + # starlette + # watchfiles +async-timeout==4.0.3 ; python_full_version < '3.11' \ + --hash=sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f \ + --hash=sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028 + # via + # -c python/requirements_compiled.txt + # aiohttp +attrs==25.1.0 \ + --hash=sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e \ + --hash=sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a + # via + # -c python/requirements_compiled.txt + # aiohttp + # jsonschema + # referencing +billiard==4.2.1 \ + --hash=sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f \ + --hash=sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb + # via + # -c python/requirements_compiled.txt + # celery +cachetools==5.5.2 \ + --hash=sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4 \ + --hash=sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a + # via + # -c python/requirements_compiled.txt + # google-auth +celery==5.5.3 \ + --hash=sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525 \ + --hash=sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5 + # via + # -c python/requirements_compiled.txt + # ray +certifi==2025.1.31 \ + --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ + --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe + # via + # -c python/requirements_compiled.txt + # requests +cffi==1.16.0 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ + --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \ + --hash=sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417 \ + --hash=sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab \ + --hash=sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520 \ + --hash=sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36 \ + --hash=sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743 \ + --hash=sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8 \ + --hash=sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed \ + --hash=sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684 \ + --hash=sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56 \ + --hash=sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324 \ + --hash=sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d \ + --hash=sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235 \ + --hash=sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e \ + --hash=sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088 \ + --hash=sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000 \ + --hash=sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7 \ + --hash=sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e \ + --hash=sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673 \ + --hash=sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c \ + --hash=sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe \ + --hash=sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2 \ + --hash=sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098 \ + --hash=sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8 \ + --hash=sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a \ + --hash=sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0 \ + --hash=sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b \ + --hash=sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896 \ + --hash=sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e \ + --hash=sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9 \ + --hash=sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2 \ + --hash=sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b \ + --hash=sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6 \ + --hash=sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404 \ + --hash=sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f \ + --hash=sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0 \ + --hash=sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4 \ + --hash=sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc \ + --hash=sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936 \ + --hash=sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba \ + --hash=sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872 \ + --hash=sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb \ + --hash=sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614 \ + --hash=sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1 \ + --hash=sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d \ + --hash=sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969 \ + --hash=sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b \ + --hash=sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4 \ + --hash=sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627 \ + --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ + --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 + # via + # -c python/requirements_compiled.txt + # cryptography +charset-normalizer==3.3.2 \ + --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ + --hash=sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087 \ + --hash=sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786 \ + --hash=sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8 \ + --hash=sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09 \ + --hash=sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185 \ + --hash=sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574 \ + --hash=sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e \ + --hash=sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519 \ + --hash=sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898 \ + --hash=sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269 \ + --hash=sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3 \ + --hash=sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f \ + --hash=sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6 \ + --hash=sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8 \ + --hash=sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a \ + --hash=sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73 \ + --hash=sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc \ + --hash=sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714 \ + --hash=sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2 \ + --hash=sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc \ + --hash=sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce \ + --hash=sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d \ + --hash=sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e \ + --hash=sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6 \ + --hash=sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269 \ + --hash=sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96 \ + --hash=sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d \ + --hash=sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a \ + --hash=sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4 \ + --hash=sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77 \ + --hash=sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d \ + --hash=sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0 \ + --hash=sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed \ + --hash=sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068 \ + --hash=sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac \ + --hash=sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25 \ + --hash=sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8 \ + --hash=sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab \ + --hash=sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26 \ + --hash=sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2 \ + --hash=sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db \ + --hash=sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f \ + --hash=sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5 \ + --hash=sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99 \ + --hash=sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c \ + --hash=sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d \ + --hash=sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811 \ + --hash=sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa \ + --hash=sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a \ + --hash=sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03 \ + --hash=sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b \ + --hash=sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04 \ + --hash=sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c \ + --hash=sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001 \ + --hash=sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458 \ + --hash=sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389 \ + --hash=sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99 \ + --hash=sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985 \ + --hash=sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537 \ + --hash=sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238 \ + --hash=sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f \ + --hash=sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d \ + --hash=sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796 \ + --hash=sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a \ + --hash=sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143 \ + --hash=sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8 \ + --hash=sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c \ + --hash=sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5 \ + --hash=sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5 \ + --hash=sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711 \ + --hash=sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4 \ + --hash=sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6 \ + --hash=sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c \ + --hash=sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7 \ + --hash=sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4 \ + --hash=sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b \ + --hash=sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae \ + --hash=sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12 \ + --hash=sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c \ + --hash=sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae \ + --hash=sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8 \ + --hash=sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887 \ + --hash=sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b \ + --hash=sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4 \ + --hash=sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f \ + --hash=sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5 \ + --hash=sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33 \ + --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ + --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 + # via + # -c python/requirements_compiled.txt + # requests +click==8.1.7 \ + --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ + --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de + # via + # -c python/requirements_compiled.txt + # celery + # click-didyoumean + # click-plugins + # click-repl + # ray + # uvicorn +click-didyoumean==0.3.1 \ + --hash=sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463 \ + --hash=sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c + # via + # -c python/requirements_compiled.txt + # celery +click-plugins==1.1.1.2 \ + --hash=sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6 \ + --hash=sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261 + # via + # -c python/requirements_compiled.txt + # celery +click-repl==0.3.0 \ + --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \ + --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812 + # via + # -c python/requirements_compiled.txt + # celery +cloudpickle==2.2.0 \ + --hash=sha256:3f4219469c55453cfe4737e564b67c2a149109dabf7f242478948b895f61106f \ + --hash=sha256:7428798d5926d8fcbfd092d18d01a2a03daf8237d8fcdc8095d256b8490796f0 + # via + # -c python/requirements_compiled.txt + # gymnasium +colorful==0.5.5 \ + --hash=sha256:62c187e27c1433db9463ff93b1451898d1e7e23a7e553583fd9daeb6325182e4 \ + --hash=sha256:66f8c1264b2a26f7293b96a03bb7a76c4bc8b9634369a0bffdcd12d618056a1d + # via + # -c python/requirements_compiled.txt + # ray +cryptography==44.0.3 \ + --hash=sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259 \ + --hash=sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43 \ + --hash=sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645 \ + --hash=sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8 \ + --hash=sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44 \ + --hash=sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d \ + --hash=sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f \ + --hash=sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d \ + --hash=sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54 \ + --hash=sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9 \ + --hash=sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137 \ + --hash=sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f \ + --hash=sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c \ + --hash=sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334 \ + --hash=sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c \ + --hash=sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b \ + --hash=sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2 \ + --hash=sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375 \ + --hash=sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88 \ + --hash=sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5 \ + --hash=sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647 \ + --hash=sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c \ + --hash=sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359 \ + --hash=sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5 \ + --hash=sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d \ + --hash=sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028 \ + --hash=sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01 \ + --hash=sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904 \ + --hash=sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d \ + --hash=sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93 \ + --hash=sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06 \ + --hash=sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff \ + --hash=sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76 \ + --hash=sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff \ + --hash=sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759 \ + --hash=sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4 \ + --hash=sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053 + # via + # -c python/requirements_compiled.txt + # pyopenssl +cupy-cuda12x==13.1.0 ; sys_platform != 'darwin' \ + --hash=sha256:230f8a8e99c81a653baa0ed00819990c0ed1f0cf0298214786b5e323461dc61a \ + --hash=sha256:2d16eaa2d086e416ac13467d4ff3184b9a081fe76b761ce51d4a46ec1c4bd28a \ + --hash=sha256:432273fd4b61a284f7d705d08b8291403548fd422bcbd945635cc155bc6a923d \ + --hash=sha256:4c51a1062a3c5a826b0425952d229ffe73b1791656a31de95b318117e67a9576 \ + --hash=sha256:4c8e9fdb1f3ffc3151808f8bb8c871518d2783e1be8b53792b698a840543d60c \ + --hash=sha256:51b1d6cb83d82dfa306c9efaeb4d57f24bad3041ebd8716d61072676abbcf67b \ + --hash=sha256:52185a2cf95d3bac2c3fda95c9c8e06a985b5a00cd2e587d3caace337db33899 \ + --hash=sha256:5afb6658faa22f21479ae2c0a07254df31c0aebc36907a64a1f6be4ecc9e96da \ + --hash=sha256:d3dc91ef9c4104652195eea4b282d343ecad653021efe20d1c8dd8dfe8ccfd86 \ + --hash=sha256:d60d1e124592cb82a5f3f45b3e7bee7bda7b72a743029f275e9d6b125f338c60 \ + --hash=sha256:dac0284fecb90b5731f514e569a6fcf6674a730ae95b9490781a713b60a34423 \ + --hash=sha256:e7a25ef1b44ae6276b5105affc2289edb34f1aa6676babd5bcd80907348c4cfa + # via + # -c python/requirements_compiled.txt + # ray +distlib==0.3.7 \ + --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 \ + --hash=sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8 + # via + # -c python/requirements_compiled.txt + # virtualenv +dm-tree==0.1.8 \ + --hash=sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6 \ + --hash=sha256:09964470f76a5201aff2e8f9b26842976de7889300676f927930f6285e256760 \ + --hash=sha256:0d3172394079a86c3a759179c65f64c48d1a42b89495fcf38976d11cc3bb952c \ + --hash=sha256:0e9620ccf06393eb6b613b5e366469304622d4ea96ae6540b28a33840e6c89cf \ + --hash=sha256:0fcaabbb14e7980377439e7140bd05552739ca5e515ecb3119f234acee4b9430 \ + --hash=sha256:1607ce49aa42f010d1e5e616d92ce899d66835d4d8bea49679582435285515de \ + --hash=sha256:181c35521d480d0365f39300542cb6cd7fd2b77351bb43d7acfda15aef63b317 \ + --hash=sha256:1d7c26e431fc93cc7e0cba867eb000db6a05f6f2b25af11ac4e9dada88fc5bca \ + --hash=sha256:1fe962015b2fe1282892b28ebe962faed53c7f98d942da9a4625cbf27baef913 \ + --hash=sha256:250b692fb75f45f02e2f58fbef9ab338904ef334b90557565621fa251df267cf \ + --hash=sha256:2869228d9c619074de501a3c10dc7f07c75422f8fab36ecdcb859b6f1b1ec3ef \ + --hash=sha256:28c52cbf4f8b3dbd0beaedf44f69fa85eec5e9dede612e08035e06ada6ec9426 \ + --hash=sha256:2f7915660f59c09068e428613c480150180df1060561fd0d1470684ae7007bd1 \ + --hash=sha256:343a4a4ebaa127451ff971254a4be4084eb4bdc0b2513c32b46f6f728fd03f9e \ + --hash=sha256:35cc164a79336bfcfafb47e5f297898359123bbd3330c1967f0c4994f9cf9f60 \ + --hash=sha256:378cc8ad93c5fe3590f405a309980721f021c790ca1bdf9b15bb1d59daec57f5 \ + --hash=sha256:39070ba268c0491af9fe7a58644d99e8b4f2cde6e5884ba3380bddc84ed43d5f \ + --hash=sha256:435227cf3c5dc63f4de054cf3d00183790bd9ead4c3623138c74dde7f67f521b \ + --hash=sha256:5483dca4d7eb1a0d65fe86d3b6a53ae717face83c1f17e0887b1a4a64ae5c410 \ + --hash=sha256:694c3654cfd2a81552c08ec66bb5c4a3d48fa292b9a181880fb081c36c5b9134 \ + --hash=sha256:75c5d528bb992981c20793b6b453e91560784215dffb8a5440ba999753c14ceb \ + --hash=sha256:803bfc53b4659f447ac694dbd04235f94a73ef7c1fd1e0df7c84ac41e0bc963b \ + --hash=sha256:81fce77f22a302d7a5968aebdf4efafef4def7ce96528719a354e6990dcd49c7 \ + --hash=sha256:83b7764de0d855338abefc6e3ee9fe40d301668310aa3baea3f778ff051f4393 \ + --hash=sha256:8c60a7eadab64c2278861f56bca320b2720f163dca9d7558103c3b77f2416571 \ + --hash=sha256:8ed3564abed97c806db122c2d3e1a2b64c74a63debe9903aad795167cc301368 \ + --hash=sha256:94d3f0826311f45ee19b75f5b48c99466e4218a0489e81c0f0167bda50cacf22 \ + --hash=sha256:96a548a406a6fb15fe58f6a30a57ff2f2aafbf25f05afab00c8f5e5977b6c715 \ + --hash=sha256:a5d819c38c03f0bb5b3b3703c60e4b170355a0fc6b5819325bf3d4ceb3ae7e80 \ + --hash=sha256:ad16ceba90a56ec47cf45b21856d14962ac314787975ef786efb5e6e9ca75ec7 \ + --hash=sha256:af4b3d372f2477dcd89a6e717e4a575ca35ccc20cc4454a8a4b6f8838a00672d \ + --hash=sha256:b095ba4f8ca1ba19350fd53cf1f8f3eb0bd406aa28af64a6dfc86707b32a810a \ + --hash=sha256:b9bd9b9ccb59409d33d51d84b7668010c04c2af7d4a371632874c1ca356cff3d \ + --hash=sha256:b9f89a454e98806b44fe9d40ec9eee61f848388f7e79ac2371a55679bd5a3ac6 \ + --hash=sha256:bb2d109f42190225112da899b9f3d46d0d5f26aef501c61e43529fe9322530b5 \ + --hash=sha256:c0a94aba18a35457a1b5cd716fd7b46c5dafdc4cf7869b4bae665b91c4682a8e \ + --hash=sha256:c5c8c12e3fda754ef6af94161bacdaeda816d941995fac415d6855c6c386af68 \ + --hash=sha256:d1612fcaecd79023dbc6a6ae48d51a80beb5c385d6f3f6d71688e57bc8d07de8 \ + --hash=sha256:d16e1f2a073604cfcc09f7131ae8d534674f43c3aef4c25742eae295bc60d04f \ + --hash=sha256:d20f2faa3672b52e5013f4077117bfb99c4cfc0b445d3bde1584c34032b57436 \ + --hash=sha256:d40fa4106ca6edc66760246a08f500ec0c85ef55c762fb4a363f6ee739ba02ee \ + --hash=sha256:de287fabc464b8734be251e46e06aa9aa1001f34198da2b6ce07bd197172b9cb \ + --hash=sha256:e4d714371bb08839e4e5e29024fc95832d9affe129825ef38836b143028bd144 \ + --hash=sha256:ea9e59e0451e7d29aece402d9f908f2e2a80922bcde2ebfd5dcb07750fcbfee8 \ + --hash=sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb \ + --hash=sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d + # via + # -c python/requirements_compiled.txt + # ray +exceptiongroup==1.3.0 ; python_full_version < '3.11' \ + --hash=sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10 \ + --hash=sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88 + # via anyio +farama-notifications==0.0.4 \ + --hash=sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18 \ + --hash=sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae + # via + # -c python/requirements_compiled.txt + # gymnasium +fastapi==0.115.12 \ + --hash=sha256:1e2c2a2646905f9e83d32f04a3f86aff4a286669c6c950ca95b5fd68c2602681 \ + --hash=sha256:e94613d6c05e27be7ffebdd6ea5f388112e5e430c8f7d6494a9d1d88d43e814d + # via + # -c python/requirements_compiled.txt + # ray +fastrlock==0.8.2 ; sys_platform != 'darwin' \ + --hash=sha256:067edb0a0805bf61e17a251d5046af59f6e9d2b8ad01222e0ef7a0b7937d5548 \ + --hash=sha256:07ed3c7b3867c05a3d6be4ced200c7767000f3431b9be6da66972822dd86e8be \ + --hash=sha256:08315bde19d0c2e6b06593d5a418be3dc8f9b1ee721afa96867b9853fceb45cf \ + --hash=sha256:11bbbbc526363955aeddb9eec4cee2a0012322b7b2f15b54f44454fcf4fd398a \ + --hash=sha256:17734e2e5af4c07ddb0fb10bd484e062c22de3be6b67940b9cc6ec2f18fa61ba \ + --hash=sha256:1b15430b93d7eb3d56f6ff690d2ebecb79ed0e58248427717eba150a508d1cd7 \ + --hash=sha256:1fed2f4797ad68e9982038423018cf08bec5f4ce9fed63a94a790773ed6a795c \ + --hash=sha256:2074548a335fcf7d19ebb18d9208da9e33b06f745754466a7e001d2b1c58dd19 \ + --hash=sha256:2587cedbb36c7988e707d83f0f1175c1f882f362b5ebbee25d70218ea33d220d \ + --hash=sha256:25945f962c7bd808415cfde3da624d4399d4ea71ed8918538375f16bceb79e1c \ + --hash=sha256:27786c62a400e282756ae1b090bcd7cfa35f28270cff65a9e7b27a5327a32561 \ + --hash=sha256:2c1719ddc8218b01e82fb2e82e8451bd65076cb96d7bef4477194bbb4305a968 \ + --hash=sha256:2d5595903444c854b99c42122b87edfe8a37cd698a4eae32f4fd1d2a7b6c115d \ + --hash=sha256:30bdbe4662992348132d03996700e1cf910d141d629179b967b146a22942264e \ + --hash=sha256:31a27a2edf482df72b91fe6c6438314d2c65290aa7becc55589d156c9b91f0da \ + --hash=sha256:320fd55bafee3eb069cfb5d6491f811a912758387ef2193840e2663e80e16f48 \ + --hash=sha256:33145acbad8317584cd64588131c7e1e286beef6280c0009b4544c91fce171d2 \ + --hash=sha256:43a241655e83e4603a152192cf022d5ca348c2f4e56dfb02e5c9c4c1a32f9cdb \ + --hash=sha256:4d63b6596368dab9e0cc66bf047e7182a56f33b34db141816a4f21f5bf958228 \ + --hash=sha256:4fb04442b6d1e2b36c774919c6bcbe3339c61b337261d4bd57e27932589095af \ + --hash=sha256:4fb2e77ff04bc4beb71d63c8e064f052ce5a6ea1e001d528d4d7f4b37d736f2e \ + --hash=sha256:5460c5ee6ced6d61ec8cd2324ebbe793a4960c4ffa2131ffff480e3b61c99ec5 \ + --hash=sha256:59344c1d46b7dec97d3f22f1cc930fafe8980b3c5bc9c9765c56738a5f1559e4 \ + --hash=sha256:5dfb78dd600a12f23fc0c3ec58f81336229fdc74501ecf378d1ce5b3f2f313ea \ + --hash=sha256:643e1e65b4f5b284427e61a894d876d10459820e93aa1e724dfb415117be24e0 \ + --hash=sha256:644ec9215cf9c4df8028d8511379a15d9c1af3e16d80e47f1b6fdc6ba118356a \ + --hash=sha256:66f2662c640bb71a1016a031eea6eef9d25c2bcdf7ffd1d1ddc5a58f9a1ced04 \ + --hash=sha256:685e656048b59d8dfde8c601f188ad53a4d719eb97080cafc8696cda6d75865e \ + --hash=sha256:7269bb3fc15587b0c191eecd95831d771a7d80f0c48929e560806b038ff3066c \ + --hash=sha256:73426f5eb2ecc10626c67cf86bd0af9e00d53e80e5c67d5ce8e18376d6abfa09 \ + --hash=sha256:75c07726c8b1a52147fd7987d6baaa318c5dced1416c3f25593e40f56e10755b \ + --hash=sha256:790fc19bccbd39426060047e53629f171a44745613bf360a045e9f9c8c4a2cea \ + --hash=sha256:7a2ccaf88ac0db153e84305d1ef0aa138cea82c6a88309066f6eaa3bc98636cd \ + --hash=sha256:87f4e01b042c84e6090dbc4fbe3415ddd69f6bc0130382323f9d3f1b8dd71b46 \ + --hash=sha256:88f079335e9da631efa64486c8207564a7bcd0c00526bb9e842e9d5b7e50a6cc \ + --hash=sha256:8c1c91a68926421f5ccbc82c85f83bd3ba593b121a46a1b9a554b3f0dd67a4bf \ + --hash=sha256:9121a894d74e65557e47e777060a495ab85f4b903e80dd73a3c940ba042920d7 \ + --hash=sha256:94e348c72a1fd1f8191f25ea056448e4f5a87b8fbf005b39d290dcb0581a48cd \ + --hash=sha256:98195866d3a9949915935d40a88e4f1c166e82e378f622c88025f2938624a90a \ + --hash=sha256:99dd6652bd6f730beadf74ef769d38c6bbd8ee6d1c15c8d138ea680b0594387f \ + --hash=sha256:9af691a9861027181d4de07ed74f0aee12a9650ac60d0a07f4320bff84b5d95f \ + --hash=sha256:a3b8b5d2935403f1b4b25ae324560e94b59593a38c0d2e7b6c9872126a9622ed \ + --hash=sha256:a3dcc876050b8f5cbc0ee84ef1e7f0c1dfe7c148f10098828bc4403683c33f10 \ + --hash=sha256:a74f5a92fa6e51c4f3c69b29c4662088b97be12f40652a21109605a175c81824 \ + --hash=sha256:ab91b0c36e95d42e1041a4907e3eefd06c482d53af3c7a77be7e214cc7cd4a63 \ + --hash=sha256:ad1bc61c7f6b0e58106aaab034916b6cb041757f708b07fbcdd9d6e1ac629225 \ + --hash=sha256:adcb9e77aa132cc6c9de2ffe7cf880a20aa8cdba21d367d1da1a412f57bddd5d \ + --hash=sha256:b22ea9bf5f9fad2b0077e944a7813f91593a4f61adf8faf734a70aed3f2b3a40 \ + --hash=sha256:b2a1c354f13f22b737621d914f3b4a8434ae69d3027a775e94b3e671756112f9 \ + --hash=sha256:b32fdf874868326351a75b1e4c02f97e802147119ae44c52d3d9da193ec34f5b \ + --hash=sha256:b3853ed4ce522598dc886160a7bab432a093051af85891fa2f5577c1dcac8ed6 \ + --hash=sha256:b443e73a4dfc7b6e0800ea4c13567b9694358e86f53bb2612a51c9e727cac67b \ + --hash=sha256:b4c9083ea89ab236b06e9ef2263971db3b4b507195fc7d5eecab95828dcae325 \ + --hash=sha256:b8ca0fe21458457077e4cb2d81e1ebdb146a00b3e9e2db6180a773f7ea905032 \ + --hash=sha256:c393af77c659a38bffbca215c0bcc8629ba4299568308dd7e4ff65d62cabed39 \ + --hash=sha256:c6bffa978793bea5e1b00e677062e53a62255439339591b70e209fa1552d5ee0 \ + --hash=sha256:ccf39ad5702e33e4d335b48ef9d56e21619b529b7f7471b5211419f380329b62 \ + --hash=sha256:cf81e0278b645004388873e0a1f9e3bc4c9ab8c18e377b14ed1a544be4b18c9a \ + --hash=sha256:d34546ad2e4a480b94b6797bcc5a322b3c705c4c74c3e4e545c4a3841c1b2d59 \ + --hash=sha256:d47713ffe6d4a627fbf078be9836a95ac106b4a0543e3841572c91e292a5d885 \ + --hash=sha256:d918dfe473291e8bfd8e13223ea5cb9b317bd9f50c280923776c377f7c64b428 \ + --hash=sha256:dbdce852e6bb66e1b8c36679d482971d69d93acf1785657522e51b7de30c3356 \ + --hash=sha256:dcc1bf0ac8a194313cf6e645e300a8a379674ceed8e0b1e910a2de3e3c28989e \ + --hash=sha256:dd961a32a7182c3891cdebca417fda67496d5d5de6ae636962254d22723bdf52 \ + --hash=sha256:ddf5d247f686aec853ddcc9a1234bfcc6f57b0a0670d2ad82fc25d8ae7e6a15f \ + --hash=sha256:e27c3cd27fbd25e5223c5c992b300cd4ee8f0a75c6f222ce65838138d853712c \ + --hash=sha256:e380ec4e6d8b26e389713995a43cb7fe56baea2d25fe073d4998c4821a026211 \ + --hash=sha256:e4bbde174a0aff5f6eeba75cf8c4c5d2a316316bc21f03a0bddca0fc3659a6f3 \ + --hash=sha256:e8b49b5743ede51e0bcf6805741f39f5e0e0fd6a172ba460cb39e3097ba803bb \ + --hash=sha256:e9904b5b37c3e5bb4a245c56bc4b7e497da57ffb8528f4fc39af9dcb168ee2e1 \ + --hash=sha256:ea96503b918fceaf40443182742b8964d47b65c5ebdea532893cb9479620000c \ + --hash=sha256:eb31fe390f03f7ae886dcc374f1099ec88526631a4cb891d399b68181f154ff0 \ + --hash=sha256:ebb32d776b61acd49f859a1d16b9e3d84e7b46d0d92aebd58acd54dc38e96664 \ + --hash=sha256:fb5363cf0fddd9b50525ddbf64a1e1b28ec4c6dfb28670a940cb1cf988a6786b \ + --hash=sha256:ff75c90663d6e8996610d435e71487daa853871ad1770dd83dc0f2fc4997241e + # via + # -c python/requirements_compiled.txt + # cupy-cuda12x +filelock==3.17.0 \ + --hash=sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338 \ + --hash=sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e + # via + # -c python/requirements_compiled.txt + # ray + # virtualenv +frozenlist==1.4.1 \ + --hash=sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7 \ + --hash=sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98 \ + --hash=sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad \ + --hash=sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5 \ + --hash=sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae \ + --hash=sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e \ + --hash=sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a \ + --hash=sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701 \ + --hash=sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d \ + --hash=sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6 \ + --hash=sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6 \ + --hash=sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106 \ + --hash=sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75 \ + --hash=sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868 \ + --hash=sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a \ + --hash=sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0 \ + --hash=sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1 \ + --hash=sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826 \ + --hash=sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec \ + --hash=sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6 \ + --hash=sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950 \ + --hash=sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19 \ + --hash=sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0 \ + --hash=sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8 \ + --hash=sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a \ + --hash=sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09 \ + --hash=sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86 \ + --hash=sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c \ + --hash=sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5 \ + --hash=sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b \ + --hash=sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b \ + --hash=sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d \ + --hash=sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0 \ + --hash=sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea \ + --hash=sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776 \ + --hash=sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a \ + --hash=sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897 \ + --hash=sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7 \ + --hash=sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09 \ + --hash=sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9 \ + --hash=sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe \ + --hash=sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd \ + --hash=sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742 \ + --hash=sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09 \ + --hash=sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0 \ + --hash=sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932 \ + --hash=sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1 \ + --hash=sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a \ + --hash=sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49 \ + --hash=sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d \ + --hash=sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7 \ + --hash=sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480 \ + --hash=sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89 \ + --hash=sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e \ + --hash=sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b \ + --hash=sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82 \ + --hash=sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb \ + --hash=sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068 \ + --hash=sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8 \ + --hash=sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b \ + --hash=sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb \ + --hash=sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2 \ + --hash=sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11 \ + --hash=sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b \ + --hash=sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc \ + --hash=sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0 \ + --hash=sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497 \ + --hash=sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17 \ + --hash=sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0 \ + --hash=sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2 \ + --hash=sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439 \ + --hash=sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5 \ + --hash=sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac \ + --hash=sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825 \ + --hash=sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887 \ + --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ + --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 + # via + # -c python/requirements_compiled.txt + # aiohttp + # aiosignal +fsspec==2023.12.1 \ + --hash=sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0 \ + --hash=sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990 + # via + # -c python/requirements_compiled.txt + # ray +google-api-core==2.24.2 \ + --hash=sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9 \ + --hash=sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696 + # via + # -c python/requirements_compiled.txt + # opencensus +google-auth==2.23.4 \ + --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \ + --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2 + # via + # -c python/requirements_compiled.txt + # google-api-core +googleapis-common-protos==1.61.0 \ + --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \ + --hash=sha256:8a64866a97f6304a7179873a465d6eee97b7a24ec6cfd78e0f575e96b821240b + # via + # -c python/requirements_compiled.txt + # google-api-core +grpcio==1.74.0 \ + --hash=sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f \ + --hash=sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc \ + --hash=sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7 \ + --hash=sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7 \ + --hash=sha256:1a2b06afe2e50ebfd46247ac3ba60cac523f54ec7792ae9ba6073c12daf26f0a \ + --hash=sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4 \ + --hash=sha256:22b834cef33429ca6cc28303c9c327ba9a3fafecbf62fae17e9a7b7163cc43ac \ + --hash=sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6 \ + --hash=sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89 \ + --hash=sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3 \ + --hash=sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49 \ + --hash=sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20 \ + --hash=sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f \ + --hash=sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc \ + --hash=sha256:4bc5fca10aaf74779081e16c2bcc3d5ec643ffd528d9e7b1c9039000ead73bae \ + --hash=sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82 \ + --hash=sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b \ + --hash=sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91 \ + --hash=sha256:5f251c355167b2360537cf17bea2cf0197995e551ab9da6a0a59b3da5e8704f9 \ + --hash=sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5 \ + --hash=sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362 \ + --hash=sha256:655726919b75ab3c34cdad39da5c530ac6fa32696fb23119e36b64adcfca174a \ + --hash=sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d \ + --hash=sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb \ + --hash=sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31 \ + --hash=sha256:6bab67d15ad617aff094c382c882e0177637da73cbc5532d52c07b4ee887a87b \ + --hash=sha256:7d95d71ff35291bab3f1c52f52f474c632db26ea12700c2ff0ea0532cb0b5854 \ + --hash=sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1 \ + --hash=sha256:834988b6c34515545b3edd13e902c1acdd9f2465d386ea5143fb558f153a7176 \ + --hash=sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8 \ + --hash=sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907 \ + --hash=sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11 \ + --hash=sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c \ + --hash=sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4 \ + --hash=sha256:8f7b5882fb50632ab1e48cb3122d6df55b9afabc265582808036b6e51b9fd6b7 \ + --hash=sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707 \ + --hash=sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5 \ + --hash=sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce \ + --hash=sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa \ + --hash=sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01 \ + --hash=sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9 \ + --hash=sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182 \ + --hash=sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b \ + --hash=sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486 \ + --hash=sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249 \ + --hash=sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3 \ + --hash=sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11 \ + --hash=sha256:ecde9ab49f58433abe02f9ed076c7b5be839cf0153883a6d23995937a82392fa \ + --hash=sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e \ + --hash=sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24 \ + --hash=sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e + # via ray +gymnasium==1.1.1 \ + --hash=sha256:8bd9ea9bdef32c950a444ff36afc785e1d81051ec32d30435058953c20d2456d \ + --hash=sha256:9c167ec0a2b388666e37f63b2849cd2552f7f5b71938574c637bb36487eb928a + # via + # -c python/requirements_compiled.txt + # ray +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # -c python/requirements_compiled.txt + # uvicorn +httptools==0.6.4 \ + --hash=sha256:0614154d5454c21b6410fdf5262b4a3ddb0f53f1e1721cfd59d55f32138c578a \ + --hash=sha256:0e563e54979e97b6d13f1bbc05a96109923e76b901f786a5eae36e99c01237bd \ + --hash=sha256:16e603a3bff50db08cd578d54f07032ca1631450ceb972c2f834c2b860c28ea2 \ + --hash=sha256:288cd628406cc53f9a541cfaf06041b4c71d751856bab45e3702191f931ccd17 \ + --hash=sha256:28908df1b9bb8187393d5b5db91435ccc9c8e891657f9cbb42a2541b44c82fc8 \ + --hash=sha256:322d20ea9cdd1fa98bd6a74b77e2ec5b818abdc3d36695ab402a0de8ef2865a3 \ + --hash=sha256:342dd6946aa6bda4b8f18c734576106b8a31f2fe31492881a9a160ec84ff4bd5 \ + --hash=sha256:345c288418f0944a6fe67be8e6afa9262b18c7626c3ef3c28adc5eabc06a68da \ + --hash=sha256:3c73ce323711a6ffb0d247dcd5a550b8babf0f757e86a52558fe5b86d6fefcc0 \ + --hash=sha256:40a5ec98d3f49904b9fe36827dcf1aadfef3b89e2bd05b0e35e94f97c2b14721 \ + --hash=sha256:40b0f7fe4fd38e6a507bdb751db0379df1e99120c65fbdc8ee6c1d044897a636 \ + --hash=sha256:40dc6a8e399e15ea525305a2ddba998b0af5caa2566bcd79dcbe8948181eeaff \ + --hash=sha256:4b36913ba52008249223042dca46e69967985fb4051951f94357ea681e1f5dc0 \ + --hash=sha256:4d87b29bd4486c0093fc64dea80231f7c7f7eb4dc70ae394d70a495ab8436071 \ + --hash=sha256:4e93eee4add6493b59a5c514da98c939b244fce4a0d8879cd3f466562f4b7d5c \ + --hash=sha256:59e724f8b332319e2875efd360e61ac07f33b492889284a3e05e6d13746876f4 \ + --hash=sha256:69422b7f458c5af875922cdb5bd586cc1f1033295aa9ff63ee196a87519ac8e1 \ + --hash=sha256:703c346571fa50d2e9856a37d7cd9435a25e7fd15e236c397bf224afaa355fe9 \ + --hash=sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44 \ + --hash=sha256:856f4bc0478ae143bad54a4242fccb1f3f86a6e1be5548fecfd4102061b3a083 \ + --hash=sha256:85797e37e8eeaa5439d33e556662cc370e474445d5fab24dcadc65a8ffb04003 \ + --hash=sha256:90d96a385fa941283ebd231464045187a31ad932ebfa541be8edf5b3c2328959 \ + --hash=sha256:94978a49b8f4569ad607cd4946b759d90b285e39c0d4640c6b36ca7a3ddf2efc \ + --hash=sha256:aafe0f1918ed07b67c1e838f950b1c1fabc683030477e60b335649b8020e1076 \ + --hash=sha256:ab9ba8dcf59de5181f6be44a77458e45a578fc99c31510b8c65b7d5acc3cf490 \ + --hash=sha256:ade273d7e767d5fae13fa637f4d53b6e961fb7fd93c7797562663f0171c26660 \ + --hash=sha256:b799de31416ecc589ad79dd85a0b2657a8fe39327944998dea368c1d4c9e55e6 \ + --hash=sha256:c26f313951f6e26147833fc923f78f95604bbec812a43e5ee37f26dc9e5a686c \ + --hash=sha256:ca80b7485c76f768a3bc83ea58373f8db7b015551117375e4918e2aa77ea9b50 \ + --hash=sha256:d1ffd262a73d7c28424252381a5b854c19d9de5f56f075445d33919a637e3547 \ + --hash=sha256:d3f0d369e7ffbe59c4b6116a44d6a8eb4783aae027f2c0b366cf0aa964185dba \ + --hash=sha256:d54efd20338ac52ba31e7da78e4a72570cf729fac82bc31ff9199bedf1dc7440 \ + --hash=sha256:dacdd3d10ea1b4ca9df97a0a303cbacafc04b5cd375fa98732678151643d4988 \ + --hash=sha256:db353d22843cf1028f43c3651581e4bb49374d85692a85f95f7b9a130e1b2cab \ + --hash=sha256:db78cb9ca56b59b016e64b6031eda5653be0589dba2b1b43453f6e8b405a0970 \ + --hash=sha256:deee0e3343f98ee8047e9f4c5bc7cedbf69f5734454a94c38ee829fb2d5fa3c1 \ + --hash=sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2 \ + --hash=sha256:df959752a0c2748a65ab5387d08287abf6779ae9165916fe053e68ae1fbdc47f \ + --hash=sha256:ec4f178901fa1834d4a060320d2f3abc5c9e39766953d038f1458cb885f47e81 \ + --hash=sha256:f47f8ed67cc0ff862b84a1189831d1d33c963fb3ce1ee0c65d3b0cbe7b711069 \ + --hash=sha256:f8787367fbdfccae38e35abf7641dafc5310310a5987b689f4c32cc8cc3ee975 \ + --hash=sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f \ + --hash=sha256:fc411e1c0a7dcd2f902c7c48cf079947a7e65b5485dea9decb82b9105ca71a43 + # via uvicorn +idna==3.7 \ + --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ + --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 + # via + # -c python/requirements_compiled.txt + # anyio + # requests + # yarl +importlib-metadata==6.11.0 \ + --hash=sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443 \ + --hash=sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b + # via + # -c python/requirements_compiled.txt + # gymnasium + # opentelemetry-api +jinja2==3.1.6 ; sys_platform != 'win32' \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via + # -c python/requirements_compiled.txt + # memray +jsonschema==4.23.0 \ + --hash=sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4 \ + --hash=sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566 + # via + # -c python/requirements_compiled.txt + # ray +jsonschema-specifications==2024.10.1 \ + --hash=sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272 \ + --hash=sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf + # via + # -c python/requirements_compiled.txt + # jsonschema +kombu==5.5.4 \ + --hash=sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363 \ + --hash=sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8 + # via + # -c python/requirements_compiled.txt + # celery +lz4==4.3.3 \ + --hash=sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e \ + --hash=sha256:054b4631a355606e99a42396f5db4d22046a3397ffc3269a348ec41eaebd69d2 \ + --hash=sha256:0a136e44a16fc98b1abc404fbabf7f1fada2bdab6a7e970974fb81cf55b636d0 \ + --hash=sha256:0e9c410b11a31dbdc94c05ac3c480cb4b222460faf9231f12538d0074e56c563 \ + --hash=sha256:222a7e35137d7539c9c33bb53fcbb26510c5748779364014235afc62b0ec797f \ + --hash=sha256:24b3206de56b7a537eda3a8123c644a2b7bf111f0af53bc14bed90ce5562d1aa \ + --hash=sha256:2b901c7784caac9a1ded4555258207d9e9697e746cc8532129f150ffe1f6ba0d \ + --hash=sha256:2f7b1839f795315e480fb87d9bc60b186a98e3e5d17203c6e757611ef7dcef61 \ + --hash=sha256:30e8c20b8857adef7be045c65f47ab1e2c4fabba86a9fa9a997d7674a31ea6b6 \ + --hash=sha256:31ea4be9d0059c00b2572d700bf2c1bc82f241f2c3282034a759c9a4d6ca4dc2 \ + --hash=sha256:337cb94488a1b060ef1685187d6ad4ba8bc61d26d631d7ba909ee984ea736be1 \ + --hash=sha256:33c9a6fd20767ccaf70649982f8f3eeb0884035c150c0b818ea660152cf3c809 \ + --hash=sha256:363ab65bf31338eb364062a15f302fc0fab0a49426051429866d71c793c23394 \ + --hash=sha256:43cf03059c0f941b772c8aeb42a0813d68d7081c009542301637e5782f8a33e2 \ + --hash=sha256:56f4fe9c6327adb97406f27a66420b22ce02d71a5c365c48d6b656b4aaeb7775 \ + --hash=sha256:5d35533bf2cee56f38ced91f766cd0038b6abf46f438a80d50c52750088be93f \ + --hash=sha256:6756212507405f270b66b3ff7f564618de0606395c0fe10a7ae2ffcbbe0b1fba \ + --hash=sha256:6cdc60e21ec70266947a48839b437d46025076eb4b12c76bd47f8e5eb8a75dcc \ + --hash=sha256:abc197e4aca8b63f5ae200af03eb95fb4b5055a8f990079b5bdf042f568469dd \ + --hash=sha256:b14d948e6dce389f9a7afc666d60dd1e35fa2138a8ec5306d30cd2e30d36b40c \ + --hash=sha256:b47839b53956e2737229d70714f1d75f33e8ac26e52c267f0197b3189ca6de24 \ + --hash=sha256:b6d9ec061b9eca86e4dcc003d93334b95d53909afd5a32c6e4f222157b50c071 \ + --hash=sha256:b891880c187e96339474af2a3b2bfb11a8e4732ff5034be919aa9029484cd201 \ + --hash=sha256:bca8fccc15e3add173da91be8f34121578dc777711ffd98d399be35487c934bf \ + --hash=sha256:c81703b12475da73a5d66618856d04b1307e43428a7e59d98cfe5a5d608a74c6 \ + --hash=sha256:d2507ee9c99dbddd191c86f0e0c8b724c76d26b0602db9ea23232304382e1f21 \ + --hash=sha256:e36cd7b9d4d920d3bfc2369840da506fa68258f7bb176b8743189793c055e43d \ + --hash=sha256:e7d84b479ddf39fe3ea05387f10b779155fc0990125f4fb35d636114e1c63a2e \ + --hash=sha256:eac9af361e0d98335a02ff12fb56caeb7ea1196cf1a49dbf6f17828a131da807 \ + --hash=sha256:edfd858985c23523f4e5a7526ca6ee65ff930207a7ec8a8f57a01eae506aaee7 \ + --hash=sha256:ee9ff50557a942d187ec85462bb0960207e7ec5b19b3b48949263993771c6205 \ + --hash=sha256:f0e822cd7644995d9ba248cb4b67859701748a93e2ab7fc9bc18c599a52e4604 \ + --hash=sha256:f180904f33bdd1e92967923a43c22899e303906d19b2cf8bb547db6653ea6e7d \ + --hash=sha256:f1d18718f9d78182c6b60f568c9a9cec8a7204d7cb6fad4e511a2ef279e4cb05 \ + --hash=sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0 \ + --hash=sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7 + # via + # -c python/requirements_compiled.txt + # ray +markdown-it-py==2.2.0 ; sys_platform != 'win32' \ + --hash=sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30 \ + --hash=sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1 + # via + # -c python/requirements_compiled.txt + # rich +markupsafe==2.1.3 ; sys_platform != 'win32' \ + --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ + --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ + --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ + --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ + --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ + --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ + --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ + --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ + --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ + --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ + --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ + --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ + --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ + --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ + --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ + --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ + --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ + --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ + --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ + --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ + --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ + --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ + --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ + --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ + --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 + # via + # -c python/requirements_compiled.txt + # jinja2 +mdurl==0.1.2 ; sys_platform != 'win32' \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via + # -c python/requirements_compiled.txt + # markdown-it-py +memray==1.10.0 ; sys_platform != 'win32' \ + --hash=sha256:0a21745fb516b7a6efcd40aa7487c59e9313fcfc782d0193fcfcf00b48426874 \ + --hash=sha256:22f2a47871c172a0539bd72737bb6b294fc10c510464066b825d90fcd3bb4916 \ + --hash=sha256:23e8c402625cfb32d0e9edb5ec0945f3e5e54bc6b0c5699f6284302082b80bd4 \ + --hash=sha256:2ce59ef485db3634de98b3a026d2450fc0a875e3a58a9ea85f7a89098841defe \ + --hash=sha256:322ed0b69014a0969b777768d461a785203f81f9864386b666b5b26645d9c294 \ + --hash=sha256:38322e052b882790993412f1840517a51818aa55c47037f69915b2007f2c4cee \ + --hash=sha256:38393c86ce6d0a08e6ec0eb1401d49803b7c0c950c2565386751cdc81568cba8 \ + --hash=sha256:391aac6c9f744528d3186bc82d708a1acc83525778f804045d7c96f860f8ec98 \ + --hash=sha256:3a8bb7fbd8303c4f0017ba7faef6b88f904cda2931ed667cbf3b98f024b3bc44 \ + --hash=sha256:3c401c57f49c4c5f1fecaee1e746f537cdc6680da05fb963dc143bd08ee109bf \ + --hash=sha256:4eba29179772b4a2e440a065b320b03bc2e73fe2648bdf7936aa3b9a086fab4a \ + --hash=sha256:53a8f66af18b1f3bcf5c9f3c95ae4134dd675903a38f9d0e6341b7bca01b63d0 \ + --hash=sha256:566602b2143e06b3d592901d98c52ce4599e71aa2555146eeb5cec03506f9498 \ + --hash=sha256:663d463e89a64bae4a6b2f8c837d11a3d094834442d536a4165e1d31899a3500 \ + --hash=sha256:68bd8df023c8a32f44c11d997e5c536837e27c0955daf557d3a377edd55a1dd3 \ + --hash=sha256:6937d7ef67d18ccc01c3250cdf3b4ef1445b859ee8756f09e3d11bd3ff0c7d67 \ + --hash=sha256:6b311e91203be71e1a0ce5e4f978137765bcb1045f3bf5646129c83c5b96ab3c \ + --hash=sha256:6fd13ef666c7fced9768d1cfabf71dc6dfa6724935a8dff463495ac2dc5e13a4 \ + --hash=sha256:8196c684f1be8fe423e5cdd2356d4255a2cb482a1f3e89612b70d2a2862cf5bb \ + --hash=sha256:843a688877691746f9d1835cfa8a65139948471bdd78720435808d20bc30a1cc \ + --hash=sha256:85c32d6613d81b075f740e398c4d653e0803cd48e82c33dcd584c109d6782666 \ + --hash=sha256:898acd60f57a10dc5aaf1fd64aa2f821f0420114f3f60c3058083788603f173a \ + --hash=sha256:8d56f37a34125684746c13d24bd7a3fb17549b0bb355eb50969eb11e05e3ba62 \ + --hash=sha256:92c372cb262eddd23049f945ca9527f0e4cc7c40a070aade1802d066f680885b \ + --hash=sha256:95e563d9c976e429ad597ad2720d95cebbe8bac891a3082465439143e2740772 \ + --hash=sha256:9627184c926252c8f719c301f1fefe970f0d033c643a6448b93fed2889d1ea94 \ + --hash=sha256:a9e985fb7646b0475c303919d19211d2aa54e5a9e2cd2a102472299be5dbebd3 \ + --hash=sha256:b681519357d94f5f0857fbc6029e7c44d3f41436109e955a14fd312d8317bc35 \ + --hash=sha256:b75040f28e8678d0e9c4907d55c95cf26db8ef5adc9941a228f1b280a9efd9c0 \ + --hash=sha256:c3a14960838d89a91747885897d34134afb65883cc3b0ed7ff30fe1af00f9fe6 \ + --hash=sha256:c7aeb47174c42e99740a8e2b3b6fe0932c95d987258d48a746974ead19176c26 \ + --hash=sha256:ce22a887a585ef5020896de89ffc793e531b65ccc81fbafcc7886010c2c562b3 \ + --hash=sha256:cf6d683c4f8d25c6ad06ae18715f218983c5eb86803953615e902d632fdf6ec1 \ + --hash=sha256:e356af93e3b031c83957e9ac1a653f5aaba5df1e357dd17142f5ed19bb3dc660 \ + --hash=sha256:f16c5c8730b616613dc8bafe32649ca6bd7252606251eb00148582011758d0b5 + # via + # -c python/requirements_compiled.txt + # ray +msgpack==1.0.7 \ + --hash=sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862 \ + --hash=sha256:0bfdd914e55e0d2c9e1526de210f6fe8ffe9705f2b1dfcc4aecc92a4cb4b533d \ + --hash=sha256:1dc93e8e4653bdb5910aed79f11e165c85732067614f180f70534f056da97db3 \ + --hash=sha256:1e2d69948e4132813b8d1131f29f9101bc2c915f26089a6d632001a5c1349672 \ + --hash=sha256:235a31ec7db685f5c82233bddf9858748b89b8119bf4538d514536c485c15fe0 \ + --hash=sha256:27dcd6f46a21c18fa5e5deed92a43d4554e3df8d8ca5a47bf0615d6a5f39dbc9 \ + --hash=sha256:28efb066cde83c479dfe5a48141a53bc7e5f13f785b92ddde336c716663039ee \ + --hash=sha256:3476fae43db72bd11f29a5147ae2f3cb22e2f1a91d575ef130d2bf49afd21c46 \ + --hash=sha256:36e17c4592231a7dbd2ed09027823ab295d2791b3b1efb2aee874b10548b7524 \ + --hash=sha256:384d779f0d6f1b110eae74cb0659d9aa6ff35aaf547b3955abf2ab4c901c4819 \ + --hash=sha256:38949d30b11ae5f95c3c91917ee7a6b239f5ec276f271f28638dec9156f82cfc \ + --hash=sha256:3967e4ad1aa9da62fd53e346ed17d7b2e922cba5ab93bdd46febcac39be636fc \ + --hash=sha256:3e7bf4442b310ff154b7bb9d81eb2c016b7d597e364f97d72b1acc3817a0fdc1 \ + --hash=sha256:3f0c8c6dfa6605ab8ff0611995ee30d4f9fcff89966cf562733b4008a3d60d82 \ + --hash=sha256:484ae3240666ad34cfa31eea7b8c6cd2f1fdaae21d73ce2974211df099a95d81 \ + --hash=sha256:4a7b4f35de6a304b5533c238bee86b670b75b03d31b7797929caa7a624b5dda6 \ + --hash=sha256:4cb14ce54d9b857be9591ac364cb08dc2d6a5c4318c1182cb1d02274029d590d \ + --hash=sha256:4e71bc4416de195d6e9b4ee93ad3f2f6b2ce11d042b4d7a7ee00bbe0358bd0c2 \ + --hash=sha256:52700dc63a4676669b341ba33520f4d6e43d3ca58d422e22ba66d1736b0a6e4c \ + --hash=sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87 \ + --hash=sha256:576eb384292b139821c41995523654ad82d1916da6a60cff129c715a6223ea84 \ + --hash=sha256:5b0bf0effb196ed76b7ad883848143427a73c355ae8e569fa538365064188b8e \ + --hash=sha256:5b6ccc0c85916998d788b295765ea0e9cb9aac7e4a8ed71d12e7d8ac31c23c95 \ + --hash=sha256:5ed82f5a7af3697b1c4786053736f24a0efd0a1b8a130d4c7bfee4b9ded0f08f \ + --hash=sha256:6d4c80667de2e36970ebf74f42d1088cc9ee7ef5f4e8c35eee1b40eafd33ca5b \ + --hash=sha256:730076207cb816138cf1af7f7237b208340a2c5e749707457d70705715c93b93 \ + --hash=sha256:7687e22a31e976a0e7fc99c2f4d11ca45eff652a81eb8c8085e9609298916dcf \ + --hash=sha256:822ea70dc4018c7e6223f13affd1c5c30c0f5c12ac1f96cd8e9949acddb48a61 \ + --hash=sha256:84b0daf226913133f899ea9b30618722d45feffa67e4fe867b0b5ae83a34060c \ + --hash=sha256:85765fdf4b27eb5086f05ac0491090fc76f4f2b28e09d9350c31aac25a5aaff8 \ + --hash=sha256:8dd178c4c80706546702c59529ffc005681bd6dc2ea234c450661b205445a34d \ + --hash=sha256:8f5b234f567cf76ee489502ceb7165c2a5cecec081db2b37e35332b537f8157c \ + --hash=sha256:98bbd754a422a0b123c66a4c341de0474cad4a5c10c164ceed6ea090f3563db4 \ + --hash=sha256:993584fc821c58d5993521bfdcd31a4adf025c7d745bbd4d12ccfecf695af5ba \ + --hash=sha256:a40821a89dc373d6427e2b44b572efc36a2778d3f543299e2f24eb1a5de65415 \ + --hash=sha256:b291f0ee7961a597cbbcc77709374087fa2a9afe7bdb6a40dbbd9b127e79afee \ + --hash=sha256:b573a43ef7c368ba4ea06050a957c2a7550f729c31f11dd616d2ac4aba99888d \ + --hash=sha256:b610ff0f24e9f11c9ae653c67ff8cc03c075131401b3e5ef4b82570d1728f8a9 \ + --hash=sha256:bdf38ba2d393c7911ae989c3bbba510ebbcdf4ecbdbfec36272abe350c454075 \ + --hash=sha256:bfef2bb6ef068827bbd021017a107194956918ab43ce4d6dc945ffa13efbc25f \ + --hash=sha256:cab3db8bab4b7e635c1c97270d7a4b2a90c070b33cbc00c99ef3f9be03d3e1f7 \ + --hash=sha256:cb70766519500281815dfd7a87d3a178acf7ce95390544b8c90587d76b227681 \ + --hash=sha256:cca1b62fe70d761a282496b96a5e51c44c213e410a964bdffe0928e611368329 \ + --hash=sha256:ccf9a39706b604d884d2cb1e27fe973bc55f2890c52f38df742bc1d79ab9f5e1 \ + --hash=sha256:dc43f1ec66eb8440567186ae2f8c447d91e0372d793dfe8c222aec857b81a8cf \ + --hash=sha256:dd632777ff3beaaf629f1ab4396caf7ba0bdd075d948a69460d13d44357aca4c \ + --hash=sha256:e45ae4927759289c30ccba8d9fdce62bb414977ba158286b5ddaf8df2cddb5c5 \ + --hash=sha256:e50ebce52f41370707f1e21a59514e3375e3edd6e1832f5e5235237db933c98b \ + --hash=sha256:ebbbba226f0a108a7366bf4b59bf0f30a12fd5e75100c630267d94d7f0ad20e5 \ + --hash=sha256:ec79ff6159dffcc30853b2ad612ed572af86c92b5168aa3fc01a67b0fa40665e \ + --hash=sha256:f0936e08e0003f66bfd97e74ee530427707297b0d0361247e9b4f59ab78ddc8b \ + --hash=sha256:f26a07a6e877c76a88e3cecac8531908d980d3d5067ff69213653649ec0f60ad \ + --hash=sha256:f64e376cd20d3f030190e8c32e1c64582eba56ac6dc7d5b0b49a9d44021b52fd \ + --hash=sha256:f6ffbc252eb0d229aeb2f9ad051200668fc3a9aaa8994e49f0cb2ffe2b7867e7 \ + --hash=sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002 \ + --hash=sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc + # via + # -c python/requirements_compiled.txt + # ray +multidict==6.0.5 \ + --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ + --hash=sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c \ + --hash=sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29 \ + --hash=sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b \ + --hash=sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8 \ + --hash=sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7 \ + --hash=sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd \ + --hash=sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40 \ + --hash=sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6 \ + --hash=sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3 \ + --hash=sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c \ + --hash=sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9 \ + --hash=sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5 \ + --hash=sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae \ + --hash=sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442 \ + --hash=sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9 \ + --hash=sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc \ + --hash=sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c \ + --hash=sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea \ + --hash=sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5 \ + --hash=sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50 \ + --hash=sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182 \ + --hash=sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453 \ + --hash=sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e \ + --hash=sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600 \ + --hash=sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733 \ + --hash=sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda \ + --hash=sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241 \ + --hash=sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461 \ + --hash=sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e \ + --hash=sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e \ + --hash=sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b \ + --hash=sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e \ + --hash=sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7 \ + --hash=sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386 \ + --hash=sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd \ + --hash=sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9 \ + --hash=sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf \ + --hash=sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee \ + --hash=sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5 \ + --hash=sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a \ + --hash=sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271 \ + --hash=sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54 \ + --hash=sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4 \ + --hash=sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496 \ + --hash=sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb \ + --hash=sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319 \ + --hash=sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3 \ + --hash=sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f \ + --hash=sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527 \ + --hash=sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed \ + --hash=sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604 \ + --hash=sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef \ + --hash=sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8 \ + --hash=sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5 \ + --hash=sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5 \ + --hash=sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626 \ + --hash=sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c \ + --hash=sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d \ + --hash=sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c \ + --hash=sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc \ + --hash=sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc \ + --hash=sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b \ + --hash=sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38 \ + --hash=sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450 \ + --hash=sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1 \ + --hash=sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f \ + --hash=sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3 \ + --hash=sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755 \ + --hash=sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226 \ + --hash=sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a \ + --hash=sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046 \ + --hash=sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf \ + --hash=sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479 \ + --hash=sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e \ + --hash=sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1 \ + --hash=sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a \ + --hash=sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83 \ + --hash=sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929 \ + --hash=sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93 \ + --hash=sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a \ + --hash=sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c \ + --hash=sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44 \ + --hash=sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89 \ + --hash=sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba \ + --hash=sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e \ + --hash=sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da \ + --hash=sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24 \ + --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ + --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef + # via + # -c python/requirements_compiled.txt + # aiohttp + # yarl +numpy==1.26.4 \ + --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ + --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ + --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \ + --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \ + --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \ + --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \ + --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \ + --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \ + --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \ + --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \ + --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \ + --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \ + --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \ + --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \ + --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \ + --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \ + --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \ + --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \ + --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \ + --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \ + --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \ + --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \ + --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \ + --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \ + --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \ + --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \ + --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \ + --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \ + --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \ + --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \ + --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \ + --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \ + --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \ + --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \ + --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ + --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f + # via + # -c python/requirements_compiled.txt + # cupy-cuda12x + # gymnasium + # pandas + # ray + # scipy + # tensorboardx +opencensus==0.11.4 \ + --hash=sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864 \ + --hash=sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2 + # via + # -c python/requirements_compiled.txt + # ray +opencensus-context==0.1.3 \ + --hash=sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039 \ + --hash=sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c + # via + # -c python/requirements_compiled.txt + # opencensus +opentelemetry-api==1.34.1 \ + --hash=sha256:64f0bd06d42824843731d05beea88d4d4b6ae59f9fe347ff7dfa2cc14233bbb3 \ + --hash=sha256:b7df4cb0830d5a6c29ad0c0691dbae874d8daefa934b8b1d642de48323d32a8c + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-exporter-prometheus==0.55b1 \ + --hash=sha256:d13ec0b22bf394113ff1ada5da98133a4b051779b803dae183188e26c4bd9ee0 \ + --hash=sha256:f364fbbff9e5de37a112ff104d1185fb1d7e2046c5ab5911e5afebc7ab3ddf0e + # via + # -c python/requirements_compiled.txt + # ray +opentelemetry-proto==1.27.0 \ + --hash=sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6 \ + --hash=sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace + # via + # -c python/requirements_compiled.txt + # ray +opentelemetry-sdk==1.34.1 \ + --hash=sha256:308effad4059562f1d92163c61c8141df649da24ce361827812c40abb2a1e96e \ + --hash=sha256:8091db0d763fcd6098d4781bbc80ff0971f94e260739aa6afe6fd379cdf3aa4d + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # ray +opentelemetry-semantic-conventions==0.55b1 \ + --hash=sha256:5da81dfdf7d52e3d37f8fe88d5e771e191de924cfff5f550ab0b8f7b2409baed \ + --hash=sha256:ef95b1f009159c28d7a7849f5cbc71c4c34c845bb514d66adfdf1b3fff3598b3 + # via + # -c python/requirements_compiled.txt + # opentelemetry-sdk +ormsgpack==1.7.0 \ + --hash=sha256:0d88307ab45d95416ce4071b1b99326ca31362af01c3d206f15a0551a7a874bd \ + --hash=sha256:22418a4d399027a72fb2e6b873559b1886cf2e63323ca7afc17b222c454413b7 \ + --hash=sha256:2c22c62a6bc93bcb194b7f91864ca0b39455b2cbbfc1538a3da0f9ec3c11d184 \ + --hash=sha256:3a6a97937d2cf21496d7689b90a43df83c5062bbe846aaa39197cc9ad73eaa7b \ + --hash=sha256:462089a419dbde654915ccb0b859c0dbe3c178b0ac580018e82befea6ccd73f4 \ + --hash=sha256:4b353204e99b56c1d33f1cf4767bd1fe1195596181a1cc789f25aa26c0b50f3d \ + --hash=sha256:5ec763096d978d35eedcef0af13991a10741717c2e236b26f4c2047b0740ea7b \ + --hash=sha256:5fefa1ca842dbba258401ea958113fe62c6b70a7a4d46edac440113f68dc431e \ + --hash=sha256:65525438b4a8b3b64ccfcda25e758ea3db392d1c206b5e09ef70efbbafa6dbf9 \ + --hash=sha256:6b4c98839cb7fc2a212037d2258f3a22857155249eb293d45c45cb974cfba834 \ + --hash=sha256:6d114652dadd81802b8a35a49e07a3e9ef2a47aed6123fb5031f2220d1c8e434 \ + --hash=sha256:77bc2ea387d85cfad045b9bcb8040bae43ad32dafe9363360f732cc19d489bbe \ + --hash=sha256:7e6ada21f5c7a20ff7cf9b061c44e3814352f819947a12022ad8cb52a9f2a809 \ + --hash=sha256:8d301e47565fe0e52a60052e730a9bb7669dfbd2a94643b8be925e3928c64c15 \ + --hash=sha256:90aabfd816db60dadab1100d583d061e0238209015bf684f8170c0fca4eb445a \ + --hash=sha256:91ebb7d3609db249cdff629ffef83ec3d025b1384749a297cf3b6a8240cf22ac \ + --hash=sha256:97723786755a7df85fcf6e68d7b5359dacea98d5c26b1d9af219a3cc05df4734 \ + --hash=sha256:9b0945523ccc75aa6907f38f2240d36818618baccb8633923bd7740a5a929e67 \ + --hash=sha256:a0ca6a64d47073f22ecc1dd96b384e44f98796d3f88ee383e92dfbcdf18c2efd \ + --hash=sha256:a5e12b51a590be47ccef67907905653e679fc2f920854b456edc216690ecc09c \ + --hash=sha256:a8fbe7bb50ee8381df030823d9366984fac718447947c2327969405d1d799b95 \ + --hash=sha256:c683071bf4527ffa7b6cfcf28f750d1a82eb77846d106743c09261ab1b79b193 \ + --hash=sha256:ca4d35b694f32112eb33ac0b733cb903dbbc59f019d05ca3d74f6ad2f587b0bf \ + --hash=sha256:e8385181bf195af80fc270e64fd477f1c414ffb05837320382e2ec9ca34be0ec \ + --hash=sha256:e86124cdbc8ed249806347c2fba96843e8941122b161b429139a0c973d270de4 \ + --hash=sha256:f9967a7f3647ad118751abf090f8397fda3e4bca6833340cab95a3f2bec598cd + # via + # -c python/requirements_compiled.txt + # ray +packaging==23.0 \ + --hash=sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2 \ + --hash=sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97 + # via + # -c python/requirements_compiled.txt + # kombu + # ray + # tensorboardx +pandas==1.5.3 \ + --hash=sha256:14e45300521902689a81f3f41386dc86f19b8ba8dd5ac5a3c7010ef8d2932813 \ + --hash=sha256:26d9c71772c7afb9d5046e6e9cf42d83dd147b5cf5bcb9d97252077118543792 \ + --hash=sha256:3749077d86e3a2f0ed51367f30bf5b82e131cc0f14260c4d3e499186fccc4406 \ + --hash=sha256:41179ce559943d83a9b4bbacb736b04c928b095b5f25dd2b7389eda08f46f373 \ + --hash=sha256:478ff646ca42b20376e4ed3fa2e8d7341e8a63105586efe54fa2508ee087f328 \ + --hash=sha256:50869a35cbb0f2e0cd5ec04b191e7b12ed688874bd05dd777c19b28cbea90996 \ + --hash=sha256:565fa34a5434d38e9d250af3c12ff931abaf88050551d9fbcdfafca50d62babf \ + --hash=sha256:5f2b952406a1588ad4cad5b3f55f520e82e902388a6d5a4a91baa8d38d23c7f6 \ + --hash=sha256:5fbcb19d6fceb9e946b3e23258757c7b225ba450990d9ed63ccceeb8cae609f7 \ + --hash=sha256:6973549c01ca91ec96199e940495219c887ea815b2083722821f1d7abfa2b4dc \ + --hash=sha256:74a3fd7e5a7ec052f183273dc7b0acd3a863edf7520f5d3a1765c04ffdb3b0b1 \ + --hash=sha256:7a0a56cef15fd1586726dace5616db75ebcfec9179a3a55e78f72c5639fa2a23 \ + --hash=sha256:7cec0bee9f294e5de5bbfc14d0573f65526071029d036b753ee6507d2a21480a \ + --hash=sha256:87bd9c03da1ac870a6d2c8902a0e1fd4267ca00f13bc494c9e5a9020920e1d51 \ + --hash=sha256:972d8a45395f2a2d26733eb8d0f629b2f90bebe8e8eddbb8829b180c09639572 \ + --hash=sha256:9842b6f4b8479e41968eced654487258ed81df7d1c9b7b870ceea24ed9459b31 \ + --hash=sha256:9f69c4029613de47816b1bb30ff5ac778686688751a5e9c99ad8c7031f6508e5 \ + --hash=sha256:a50d9a4336a9621cab7b8eb3fb11adb82de58f9b91d84c2cd526576b881a0c5a \ + --hash=sha256:bc4c368f42b551bf72fac35c5128963a171b40dce866fb066540eeaf46faa003 \ + --hash=sha256:c39a8da13cede5adcd3be1182883aea1c925476f4e84b2807a46e2775306305d \ + --hash=sha256:c3ac844a0fe00bfaeb2c9b51ab1424e5c8744f89860b138434a363b1f620f354 \ + --hash=sha256:c4c00e0b0597c8e4f59e8d461f797e5d70b4d025880516a8261b2817c47759ee \ + --hash=sha256:c74a62747864ed568f5a82a49a23a8d7fe171d0c69038b38cedf0976831296fa \ + --hash=sha256:dd05f7783b3274aa206a1af06f0ceed3f9b412cf665b7247eacd83be41cf7bf0 \ + --hash=sha256:dfd681c5dc216037e0b0a2c821f5ed99ba9f03ebcf119c7dac0e9a7b960b9ec9 \ + --hash=sha256:e474390e60ed609cec869b0da796ad94f420bb057d86784191eefc62b65819ae \ + --hash=sha256:f76d097d12c82a535fda9dfe5e8dd4127952b45fea9b0276cb30cca5ea313fbc + # via + # -c python/requirements_compiled.txt + # ray +platformdirs==3.11.0 \ + --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ + --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e + # via + # -c python/requirements_compiled.txt + # virtualenv +prometheus-client==0.19.0 \ + --hash=sha256:4585b0d1223148c27a225b10dbec5ae9bc4c81a99a3fa80774fa6209935324e1 \ + --hash=sha256:c88b1e6ecf6b41cd8fb5731c7ae919bf66df6ec6fafa555cd6c0e16ca169ae92 + # via + # -c python/requirements_compiled.txt + # opentelemetry-exporter-prometheus + # ray +prompt-toolkit==3.0.41 \ + --hash=sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0 \ + --hash=sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2 + # via + # -c python/requirements_compiled.txt + # click-repl +propcache==0.3.0 \ + --hash=sha256:02df07041e0820cacc8f739510078f2aadcfd3fc57eaeeb16d5ded85c872c89e \ + --hash=sha256:03acd9ff19021bd0567582ac88f821b66883e158274183b9e5586f678984f8fe \ + --hash=sha256:03c091bb752349402f23ee43bb2bff6bd80ccab7c9df6b88ad4322258d6960fc \ + --hash=sha256:07700939b2cbd67bfb3b76a12e1412405d71019df00ca5697ce75e5ef789d829 \ + --hash=sha256:0c3e893c4464ebd751b44ae76c12c5f5c1e4f6cbd6fbf67e3783cd93ad221863 \ + --hash=sha256:119e244ab40f70a98c91906d4c1f4c5f2e68bd0b14e7ab0a06922038fae8a20f \ + --hash=sha256:11ae6a8a01b8a4dc79093b5d3ca2c8a4436f5ee251a9840d7790dccbd96cb649 \ + --hash=sha256:15010f29fbed80e711db272909a074dc79858c6d28e2915704cfc487a8ac89c6 \ + --hash=sha256:19d36bb351ad5554ff20f2ae75f88ce205b0748c38b146c75628577020351e3c \ + --hash=sha256:1c8f7d896a16da9455f882870a507567d4f58c53504dc2d4b1e1d386dfe4588a \ + --hash=sha256:2383a17385d9800b6eb5855c2f05ee550f803878f344f58b6e194de08b96352c \ + --hash=sha256:24c04f8fbf60094c531667b8207acbae54146661657a1b1be6d3ca7773b7a545 \ + --hash=sha256:2578541776769b500bada3f8a4eeaf944530516b6e90c089aa368266ed70c49e \ + --hash=sha256:26a67e5c04e3119594d8cfae517f4b9330c395df07ea65eab16f3d559b7068fe \ + --hash=sha256:2b975528998de037dfbc10144b8aed9b8dd5a99ec547f14d1cb7c5665a43f075 \ + --hash=sha256:2d15bc27163cd4df433e75f546b9ac31c1ba7b0b128bfb1b90df19082466ff57 \ + --hash=sha256:2d913d36bdaf368637b4f88d554fb9cb9d53d6920b9c5563846555938d5450bf \ + --hash=sha256:3302c5287e504d23bb0e64d2a921d1eb4a03fb93a0a0aa3b53de059f5a5d737d \ + --hash=sha256:36ca5e9a21822cc1746023e88f5c0af6fce3af3b85d4520efb1ce4221bed75cc \ + --hash=sha256:3b812b3cb6caacd072276ac0492d249f210006c57726b6484a1e1805b3cfeea0 \ + --hash=sha256:3c6ec957025bf32b15cbc6b67afe233c65b30005e4c55fe5768e4bb518d712f1 \ + --hash=sha256:41de3da5458edd5678b0f6ff66691507f9885f5fe6a0fb99a5d10d10c0fd2d64 \ + --hash=sha256:42924dc0c9d73e49908e35bbdec87adedd651ea24c53c29cac103ede0ea1d340 \ + --hash=sha256:4544699674faf66fb6b4473a1518ae4999c1b614f0b8297b1cef96bac25381db \ + --hash=sha256:46ed02532cb66612d42ae5c3929b5e98ae330ea0f3900bc66ec5f4862069519b \ + --hash=sha256:49ea05212a529c2caffe411e25a59308b07d6e10bf2505d77da72891f9a05641 \ + --hash=sha256:4fa0e7c9c3cf7c276d4f6ab9af8adddc127d04e0fcabede315904d2ff76db626 \ + --hash=sha256:507c5357a8d8b4593b97fb669c50598f4e6cccbbf77e22fa9598aba78292b4d7 \ + --hash=sha256:549722908de62aa0b47a78b90531c022fa6e139f9166be634f667ff45632cc92 \ + --hash=sha256:58e6d2a5a7cb3e5f166fd58e71e9a4ff504be9dc61b88167e75f835da5764d07 \ + --hash=sha256:5a16167118677d94bb48bfcd91e420088854eb0737b76ec374b91498fb77a70e \ + --hash=sha256:5d62c4f6706bff5d8a52fd51fec6069bef69e7202ed481486c0bc3874912c787 \ + --hash=sha256:5fa159dcee5dba00c1def3231c249cf261185189205073bde13797e57dd7540a \ + --hash=sha256:6032231d4a5abd67c7f71168fd64a47b6b451fbcb91c8397c2f7610e67683810 \ + --hash=sha256:63f26258a163c34542c24808f03d734b338da66ba91f410a703e505c8485791d \ + --hash=sha256:65a37714b8ad9aba5780325228598a5b16c47ba0f8aeb3dc0514701e4413d7c0 \ + --hash=sha256:67054e47c01b7b349b94ed0840ccae075449503cf1fdd0a1fdd98ab5ddc2667b \ + --hash=sha256:67dda3c7325691c2081510e92c561f465ba61b975f481735aefdfc845d2cd043 \ + --hash=sha256:6985a593417cdbc94c7f9c3403747335e450c1599da1647a5af76539672464d3 \ + --hash=sha256:6a1948df1bb1d56b5e7b0553c0fa04fd0e320997ae99689488201f19fa90d2e7 \ + --hash=sha256:6b5b7fd6ee7b54e01759f2044f936dcf7dea6e7585f35490f7ca0420fe723c0d \ + --hash=sha256:6c929916cbdb540d3407c66f19f73387f43e7c12fa318a66f64ac99da601bcdf \ + --hash=sha256:6f4d7a7c0aff92e8354cceca6fe223973ddf08401047920df0fcb24be2bd5138 \ + --hash=sha256:728af36011bb5d344c4fe4af79cfe186729efb649d2f8b395d1572fb088a996c \ + --hash=sha256:742840d1d0438eb7ea4280f3347598f507a199a35a08294afdcc560c3739989d \ + --hash=sha256:75e872573220d1ee2305b35c9813626e620768248425f58798413e9c39741f46 \ + --hash=sha256:794c3dd744fad478b6232289c866c25406ecdfc47e294618bdf1697e69bd64a6 \ + --hash=sha256:7c0fdbdf6983526e269e5a8d53b7ae3622dd6998468821d660d0daf72779aefa \ + --hash=sha256:7c5f5290799a3f6539cc5e6f474c3e5c5fbeba74a5e1e5be75587746a940d51e \ + --hash=sha256:7c6e7e4f9167fddc438cd653d826f2222222564daed4116a02a184b464d3ef05 \ + --hash=sha256:7cedd25e5f678f7738da38037435b340694ab34d424938041aa630d8bac42663 \ + --hash=sha256:7e2e068a83552ddf7a39a99488bcba05ac13454fb205c847674da0352602082f \ + --hash=sha256:8319293e85feadbbfe2150a5659dbc2ebc4afdeaf7d98936fb9a2f2ba0d4c35c \ + --hash=sha256:8526b0941ec5a40220fc4dfde76aed58808e2b309c03e9fa8e2260083ef7157f \ + --hash=sha256:8884ba1a0fe7210b775106b25850f5e5a9dc3c840d1ae9924ee6ea2eb3acbfe7 \ + --hash=sha256:8cb625bcb5add899cb8ba7bf716ec1d3e8f7cdea9b0713fa99eadf73b6d4986f \ + --hash=sha256:8d663fd71491dde7dfdfc899d13a067a94198e90695b4321084c6e450743b8c7 \ + --hash=sha256:8ee1983728964d6070ab443399c476de93d5d741f71e8f6e7880a065f878e0b9 \ + --hash=sha256:997e7b8f173a391987df40f3b52c423e5850be6f6df0dcfb5376365440b56667 \ + --hash=sha256:9be90eebc9842a93ef8335291f57b3b7488ac24f70df96a6034a13cb58e6ff86 \ + --hash=sha256:9ddd49258610499aab83b4f5b61b32e11fce873586282a0e972e5ab3bcadee51 \ + --hash=sha256:9ecde3671e62eeb99e977f5221abcf40c208f69b5eb986b061ccec317c82ebd0 \ + --hash=sha256:9ff4e9ecb6e4b363430edf2c6e50173a63e0820e549918adef70515f87ced19a \ + --hash=sha256:a254537b9b696ede293bfdbc0a65200e8e4507bc9f37831e2a0318a9b333c85c \ + --hash=sha256:a2b9bf8c79b660d0ca1ad95e587818c30ccdb11f787657458d6f26a1ea18c568 \ + --hash=sha256:a61a68d630e812b67b5bf097ab84e2cd79b48c792857dc10ba8a223f5b06a2af \ + --hash=sha256:a7080b0159ce05f179cfac592cda1a82898ca9cd097dacf8ea20ae33474fbb25 \ + --hash=sha256:a8fd93de4e1d278046345f49e2238cdb298589325849b2645d4a94c53faeffc5 \ + --hash=sha256:a94ffc66738da99232ddffcf7910e0f69e2bbe3a0802e54426dbf0714e1c2ffe \ + --hash=sha256:aa806bbc13eac1ab6291ed21ecd2dd426063ca5417dd507e6be58de20e58dfcf \ + --hash=sha256:b0c1a133d42c6fc1f5fbcf5c91331657a1ff822e87989bf4a6e2e39b818d0ee9 \ + --hash=sha256:b58229a844931bca61b3a20efd2be2a2acb4ad1622fc026504309a6883686fbf \ + --hash=sha256:bb2f144c6d98bb5cbc94adeb0447cfd4c0f991341baa68eee3f3b0c9c0e83767 \ + --hash=sha256:be90c94570840939fecedf99fa72839aed70b0ced449b415c85e01ae67422c90 \ + --hash=sha256:bf0d9a171908f32d54f651648c7290397b8792f4303821c42a74e7805bfb813c \ + --hash=sha256:bf15fc0b45914d9d1b706f7c9c4f66f2b7b053e9517e40123e137e8ca8958b3d \ + --hash=sha256:bf4298f366ca7e1ad1d21bbb58300a6985015909964077afd37559084590c929 \ + --hash=sha256:c441c841e82c5ba7a85ad25986014be8d7849c3cfbdb6004541873505929a74e \ + --hash=sha256:cacea77ef7a2195f04f9279297684955e3d1ae4241092ff0cfcef532bb7a1c32 \ + --hash=sha256:cd54895e4ae7d32f1e3dd91261df46ee7483a735017dc6f987904f194aa5fd14 \ + --hash=sha256:d1323cd04d6e92150bcc79d0174ce347ed4b349d748b9358fd2e497b121e03c8 \ + --hash=sha256:d383bf5e045d7f9d239b38e6acadd7b7fdf6c0087259a84ae3475d18e9a2ae8b \ + --hash=sha256:d3e7420211f5a65a54675fd860ea04173cde60a7cc20ccfbafcccd155225f8bc \ + --hash=sha256:d8074c5dd61c8a3e915fa8fc04754fa55cfa5978200d2daa1e2d4294c1f136aa \ + --hash=sha256:df03cd88f95b1b99052b52b1bb92173229d7a674df0ab06d2b25765ee8404bce \ + --hash=sha256:e45377d5d6fefe1677da2a2c07b024a6dac782088e37c0b1efea4cfe2b1be19b \ + --hash=sha256:e53d19c2bf7d0d1e6998a7e693c7e87300dd971808e6618964621ccd0e01fe4e \ + --hash=sha256:e560fd75aaf3e5693b91bcaddd8b314f4d57e99aef8a6c6dc692f935cc1e6bbf \ + --hash=sha256:ec5060592d83454e8063e487696ac3783cc48c9a329498bafae0d972bc7816c9 \ + --hash=sha256:ecc2920630283e0783c22e2ac94427f8cca29a04cfdf331467d4f661f4072dac \ + --hash=sha256:ed7161bccab7696a473fe7ddb619c1d75963732b37da4618ba12e60899fefe4f \ + --hash=sha256:ee0bd3a7b2e184e88d25c9baa6a9dc609ba25b76daae942edfb14499ac7ec374 \ + --hash=sha256:ee25f1ac091def37c4b59d192bbe3a206298feeb89132a470325bf76ad122a1e \ + --hash=sha256:efa44f64c37cc30c9f05932c740a8b40ce359f51882c70883cc95feac842da4d \ + --hash=sha256:f47d52fd9b2ac418c4890aad2f6d21a6b96183c98021f0a48497a904199f006e \ + --hash=sha256:f857034dc68d5ceb30fb60afb6ff2103087aea10a01b613985610e007053a121 \ + --hash=sha256:fb91d20fa2d3b13deea98a690534697742029f4fb83673a3501ae6e3746508b5 \ + --hash=sha256:fddb8870bdb83456a489ab67c6b3040a8d5a55069aa6f72f9d872235fbc52f54 + # via + # -c python/requirements_compiled.txt + # aiohttp + # yarl +proto-plus==1.22.3 \ + --hash=sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df \ + --hash=sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b + # via + # -c python/requirements_compiled.txt + # google-api-core +protobuf==4.25.8 \ + --hash=sha256:077ff8badf2acf8bc474406706ad890466274191a48d0abd3bd6987107c9cde5 \ + --hash=sha256:15a0af558aa3b13efef102ae6e4f3efac06f1eea11afb3a57db2901447d9fb59 \ + --hash=sha256:27d498ffd1f21fb81d987a041c32d07857d1d107909f5134ba3350e1ce80a4af \ + --hash=sha256:504435d831565f7cfac9f0714440028907f1975e4bed228e58e72ecfff58a1e0 \ + --hash=sha256:6135cf8affe1fc6f76cced2641e4ea8d3e59518d1f24ae41ba97bcad82d397cd \ + --hash=sha256:83e6e54e93d2b696a92cad6e6efc924f3850f82b52e1563778dfab8b355101b0 \ + --hash=sha256:9ad7ef62d92baf5a8654fbb88dac7fa5594cfa70fd3440488a5ca3bfc6d795a7 \ + --hash=sha256:bd551eb1fe1d7e92c1af1d75bdfa572eff1ab0e5bf1736716814cdccdb2360f9 \ + --hash=sha256:ca809b42f4444f144f2115c4c1a747b9a404d590f18f37e9402422033e464e0f \ + --hash=sha256:d552c53d0415449c8d17ced5c341caba0d89dbf433698e1436c8fa0aae7808a3 \ + --hash=sha256:f4510b93a3bec6eba8fd8f1093e9d7fb0d4a24d1a81377c10c0e5bbfe9e4ed24 + # via + # -c python/requirements_compiled.txt + # google-api-core + # googleapis-common-protos + # opentelemetry-proto + # proto-plus + # ray + # tensorboardx +py-spy==0.4.0 ; python_full_version < '3.12' \ + --hash=sha256:47cdda4c34d9b6cb01f3aaeceb2e88faf57da880207fe72ff6ff97e9bb6cc8a9 \ + --hash=sha256:77d8f637ade38367d944874776f45b703b7ac5938b1f7be8891f3a5876ddbb96 \ + --hash=sha256:806602ce7972782cc9c1e383f339bfc27bfb822d42485e6a3e0530ae5040e1f0 \ + --hash=sha256:87573e64dbfdfc89ba2e0f5e2f525aa84e0299c7eb6454b47ea335fde583a7a0 \ + --hash=sha256:8bf2f3702cef367a489faa45177b41a6c31b2a3e5bd78c978d44e29340152f5a \ + --hash=sha256:c5f06ffce4c9c98b7fc9f5e67e5e7db591173f1351837633f3f23d9378b1d18a \ + --hash=sha256:eee3d0bde85ca5cf4f01f012d461180ca76c24835a96f7b5c4ded64eb6a008ab \ + --hash=sha256:f2cf3f7130e7d780471faa5957441d3b4e0ec39a79b2c00f4c33d494f7728428 + # via + # -c python/requirements_compiled.txt + # ray +pyarrow==19.0.1 \ + --hash=sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466 \ + --hash=sha256:0148bb4fc158bfbc3d6dfe5001d93ebeed253793fff4435167f6ce1dc4bddeae \ + --hash=sha256:1b93ef2c93e77c442c979b0d596af45e4665d8b96da598db145b0fec014b9136 \ + --hash=sha256:1c7556165bd38cf0cd992df2636f8bcdd2d4b26916c6b7e646101aff3c16f76f \ + --hash=sha256:335d170e050bcc7da867a1ed8ffb8b44c57aaa6e0843b156a501298657b1e972 \ + --hash=sha256:3bf266b485df66a400f282ac0b6d1b500b9d2ae73314a153dbe97d6d5cc8a99e \ + --hash=sha256:41f9706fbe505e0abc10e84bf3a906a1338905cbbcf1177b71486b03e6ea6608 \ + --hash=sha256:4982f8e2b7afd6dae8608d70ba5bd91699077323f812a0448d8b7abdff6cb5d3 \ + --hash=sha256:49a3aecb62c1be1d822f8bf629226d4a96418228a42f5b40835c1f10d42e4db6 \ + --hash=sha256:4d5d1ec7ec5324b98887bdc006f4d2ce534e10e60f7ad995e7875ffa0ff9cb14 \ + --hash=sha256:58d9397b2e273ef76264b45531e9d552d8ec8a6688b7390b5be44c02a37aade8 \ + --hash=sha256:5a9137cf7e1640dce4c190551ee69d478f7121b5c6f323553b319cac936395f6 \ + --hash=sha256:5bd1618ae5e5476b7654c7b55a6364ae87686d4724538c24185bbb2952679960 \ + --hash=sha256:65cf9feebab489b19cdfcfe4aa82f62147218558d8d3f0fc1e9dea0ab8e7905a \ + --hash=sha256:699799f9c80bebcf1da0983ba86d7f289c5a2a5c04b945e2f2bcf7e874a91911 \ + --hash=sha256:6c5941c1aac89a6c2f2b16cd64fe76bcdb94b2b1e99ca6459de4e6f07638d755 \ + --hash=sha256:6ebfb5171bb5f4a52319344ebbbecc731af3f021e49318c74f33d520d31ae0c4 \ + --hash=sha256:7a544ec12de66769612b2d6988c36adc96fb9767ecc8ee0a4d270b10b1c51e00 \ + --hash=sha256:7c1bca1897c28013db5e4c83944a2ab53231f541b9e0c3f4791206d0c0de389a \ + --hash=sha256:80b2ad2b193e7d19e81008a96e313fbd53157945c7be9ac65f44f8937a55427b \ + --hash=sha256:8464c9fbe6d94a7fe1599e7e8965f350fd233532868232ab2596a71586c5a429 \ + --hash=sha256:8f04d49a6b64cf24719c080b3c2029a3a5b16417fd5fd7c4041f94233af732f3 \ + --hash=sha256:96606c3ba57944d128e8a8399da4812f56c7f61de8c647e3470b417f795d0ef9 \ + --hash=sha256:99bc1bec6d234359743b01e70d4310d0ab240c3d6b0da7e2a93663b0158616f6 \ + --hash=sha256:ad76aef7f5f7e4a757fddcdcf010a8290958f09e3470ea458c80d26f4316ae89 \ + --hash=sha256:b4c4156a625f1e35d6c0b2132635a237708944eb41df5fbe7d50f20d20c17832 \ + --hash=sha256:b9766a47a9cb56fefe95cb27f535038b5a195707a08bf61b180e642324963b46 \ + --hash=sha256:c0fe3dbbf054a00d1f162fda94ce236a899ca01123a798c561ba307ca38af5f0 \ + --hash=sha256:c6cb2335a411b713fdf1e82a752162f72d4a7b5dbc588e32aa18383318b05866 \ + --hash=sha256:cc55d71898ea30dc95900297d191377caba257612f384207fe9f8293b5850f90 \ + --hash=sha256:d03c9d6f2a3dffbd62671ca070f13fc527bb1867b4ec2b98c7eeed381d4f389a \ + --hash=sha256:d383591f3dcbe545f6cc62daaef9c7cdfe0dff0fb9e1c8121101cabe9098cfa6 \ + --hash=sha256:d9d46e06846a41ba906ab25302cf0fd522f81aa2a85a71021826f34639ad31ef \ + --hash=sha256:d9dedeaf19097a143ed6da37f04f4051aba353c95ef507764d344229b2b740ae \ + --hash=sha256:e45274b20e524ae5c39d7fc1ca2aa923aab494776d2d4b316b49ec7572ca324c \ + --hash=sha256:ee8dec072569f43835932a3b10c55973593abc00936c202707a4ad06af7cb294 \ + --hash=sha256:f24faab6ed18f216a37870d8c5623f9c044566d75ec586ef884e13a02a9d62c5 \ + --hash=sha256:f2a21d39fbdb948857f67eacb5bbaaf36802de044ec36fbef7a1c8f0dd3a4ab2 \ + --hash=sha256:f3ad4c0eb4e2a9aeb990af6c09e6fa0b195c8c0e7b272ecc8d4d2b6574809d34 \ + --hash=sha256:fc28912a2dc924dddc2087679cc8b7263accc71b9ff025a1362b004711661a69 \ + --hash=sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec \ + --hash=sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8 + # via + # -c python/requirements_compiled.txt + # ray +pyasn1==0.5.1 \ + --hash=sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58 \ + --hash=sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c + # via + # -c python/requirements_compiled.txt + # pyasn1-modules + # rsa +pyasn1-modules==0.3.0 \ + --hash=sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c \ + --hash=sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d + # via + # -c python/requirements_compiled.txt + # google-auth +pycparser==2.21 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ + --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 + # via + # -c python/requirements_compiled.txt + # cffi +pydantic==2.11.7 \ + --hash=sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db \ + --hash=sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b + # via + # -c python/requirements_compiled.txt + # fastapi + # ray +pydantic-core==2.33.2 \ + --hash=sha256:0069c9acc3f3981b9ff4cdfaf088e98d83440a4c7ea1bc07460af3d4dc22e72d \ + --hash=sha256:031c57d67ca86902726e0fae2214ce6770bbe2f710dc33063187a68744a5ecac \ + --hash=sha256:0405262705a123b7ce9f0b92f123334d67b70fd1f20a9372b907ce1080c7ba02 \ + --hash=sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56 \ + --hash=sha256:09fb9dd6571aacd023fe6aaca316bd01cf60ab27240d7eb39ebd66a3a15293b4 \ + --hash=sha256:0a39979dcbb70998b0e505fb1556a1d550a0781463ce84ebf915ba293ccb7e22 \ + --hash=sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef \ + --hash=sha256:0e03262ab796d986f978f79c943fc5f620381be7287148b8010b4097f79a39ec \ + --hash=sha256:0e5b2671f05ba48b94cb90ce55d8bdcaaedb8ba00cc5359f6810fc918713983d \ + --hash=sha256:0e6116757f7959a712db11f3e9c0a99ade00a5bbedae83cb801985aa154f071b \ + --hash=sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a \ + --hash=sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f \ + --hash=sha256:1a8695a8d00c73e50bff9dfda4d540b7dee29ff9b8053e38380426a85ef10052 \ + --hash=sha256:1e063337ef9e9820c77acc768546325ebe04ee38b08703244c1309cccc4f1bab \ + --hash=sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916 \ + --hash=sha256:2058a32994f1fde4ca0480ab9d1e75a0e8c87c22b53a3ae66554f9af78f2fe8c \ + --hash=sha256:235f45e5dbcccf6bd99f9f472858849f73d11120d76ea8707115415f8e5ebebf \ + --hash=sha256:2807668ba86cb38c6817ad9bc66215ab8584d1d304030ce4f0887336f28a5e27 \ + --hash=sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a \ + --hash=sha256:2b3d326aaef0c0399d9afffeb6367d5e26ddc24d351dbc9c636840ac355dc5d8 \ + --hash=sha256:2bfb5112df54209d820d7bf9317c7a6c9025ea52e49f46b6a2060104bba37de7 \ + --hash=sha256:2f82865531efd18d6e07a04a17331af02cb7a651583c418df8266f17a63c6612 \ + --hash=sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1 \ + --hash=sha256:3a1c81334778f9e3af2f8aeb7a960736e5cab1dfebfb26aabca09afd2906c039 \ + --hash=sha256:3abcd9392a36025e3bd55f9bd38d908bd17962cc49bc6da8e7e96285336e2bca \ + --hash=sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 \ + --hash=sha256:3dc625f4aa79713512d1976fe9f0bc99f706a9dee21dfd1810b4bbbf228d0e8a \ + --hash=sha256:3eb3fe62804e8f859c49ed20a8451342de53ed764150cb14ca71357c765dc2a6 \ + --hash=sha256:44857c3227d3fb5e753d5fe4a3420d6376fa594b07b621e220cd93703fe21782 \ + --hash=sha256:4b25d91e288e2c4e0662b8038a28c6a07eaac3e196cfc4ff69de4ea3db992a1b \ + --hash=sha256:4c5b0a576fb381edd6d27f0a85915c6daf2f8138dc5c267a57c08a62900758c7 \ + --hash=sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025 \ + --hash=sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849 \ + --hash=sha256:53a57d2ed685940a504248187d5685e49eb5eef0f696853647bf37c418c538f7 \ + --hash=sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b \ + --hash=sha256:5c4aa4e82353f65e548c476b37e64189783aa5384903bfea4f41580f255fddfa \ + --hash=sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e \ + --hash=sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea \ + --hash=sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac \ + --hash=sha256:6368900c2d3ef09b69cb0b913f9f8263b03786e5b2a387706c5afb66800efd51 \ + --hash=sha256:64632ff9d614e5eecfb495796ad51b0ed98c453e447a76bcbeeb69615079fc7e \ + --hash=sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162 \ + --hash=sha256:6b99022f1d19bc32a4c2a0d544fc9a76e3be90f0b3f4af413f87d38749300e65 \ + --hash=sha256:6bdfe4b3789761f3bcb4b1ddf33355a71079858958e3a552f16d5af19768fef2 \ + --hash=sha256:6fa6dfc3e4d1f734a34710f391ae822e0a8eb8559a85c6979e14e65ee6ba2954 \ + --hash=sha256:73662edf539e72a9440129f231ed3757faab89630d291b784ca99237fb94db2b \ + --hash=sha256:73cf6373c21bc80b2e0dc88444f41ae60b2f070ed02095754eb5a01df12256de \ + --hash=sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc \ + --hash=sha256:7f92c15cd1e97d4b12acd1cc9004fa092578acfa57b67ad5e43a197175d01a64 \ + --hash=sha256:82f68293f055f51b51ea42fafc74b6aad03e70e191799430b90c13d643059ebb \ + --hash=sha256:83aa99b1285bc8f038941ddf598501a86f1536789740991d7d8756e34f1e74d9 \ + --hash=sha256:87acbfcf8e90ca885206e98359d7dca4bcbb35abdc0ff66672a293e1d7a19101 \ + --hash=sha256:87b31b6846e361ef83fedb187bb5b4372d0da3f7e28d85415efa92d6125d6e6d \ + --hash=sha256:881b21b5549499972441da4758d662aeea93f1923f953e9cbaff14b8b9565aef \ + --hash=sha256:8d55ab81c57b8ff8548c3e4947f119551253f4e3787a7bbc0b6b3ca47498a9d3 \ + --hash=sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1 \ + --hash=sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5 \ + --hash=sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88 \ + --hash=sha256:970919794d126ba8645f3837ab6046fb4e72bbc057b3709144066204c19a455d \ + --hash=sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290 \ + --hash=sha256:9fcd347d2cc5c23b06de6d3b7b8275be558a0c90549495c699e379a80bf8379e \ + --hash=sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d \ + --hash=sha256:a11c8d26a50bfab49002947d3d237abe4d9e4b5bdc8846a63537b6488e197808 \ + --hash=sha256:a144d4f717285c6d9234a66778059f33a89096dfb9b39117663fd8413d582dcc \ + --hash=sha256:a2b911a5b90e0374d03813674bf0a5fbbb7741570dcd4b4e85a2e48d17def29d \ + --hash=sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc \ + --hash=sha256:aa9d91b338f2df0508606f7009fde642391425189bba6d8c653afd80fd6bb64e \ + --hash=sha256:b0379a2b24882fef529ec3b4987cb5d003b9cda32256024e6fe1586ac45fc640 \ + --hash=sha256:bc7aee6f634a6f4a95676fcb5d6559a2c2a390330098dba5e5a5f28a2e4ada30 \ + --hash=sha256:bdc25f3681f7b78572699569514036afe3c243bc3059d3942624e936ec93450e \ + --hash=sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9 \ + --hash=sha256:c20c462aa4434b33a2661701b861604913f912254e441ab8d78d30485736115a \ + --hash=sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9 \ + --hash=sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f \ + --hash=sha256:c54c939ee22dc8e2d545da79fc5381f1c020d6d3141d3bd747eab59164dc89fb \ + --hash=sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5 \ + --hash=sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab \ + --hash=sha256:d3f26877a748dc4251cfcfda9dfb5f13fcb034f5308388066bcfe9031b63ae7d \ + --hash=sha256:d53b22f2032c42eaaf025f7c40c2e3b94568ae077a606f006d206a463bc69572 \ + --hash=sha256:d87c561733f66531dced0da6e864f44ebf89a8fba55f31407b00c2f7f9449593 \ + --hash=sha256:d946c8bf0d5c24bf4fe333af284c59a19358aa3ec18cb3dc4370080da1e8ad29 \ + --hash=sha256:dac89aea9af8cd672fa7b510e7b8c33b0bba9a43186680550ccf23020f32d535 \ + --hash=sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1 \ + --hash=sha256:dc46a01bf8d62f227d5ecee74178ffc448ff4e5197c756331f71efcc66dc980f \ + --hash=sha256:dd14041875d09cc0f9308e37a6f8b65f5585cf2598a53aa0123df8b129d481f8 \ + --hash=sha256:de4b83bb311557e439b9e186f733f6c645b9417c84e2eb8203f3f820a4b988bf \ + --hash=sha256:e799c050df38a639db758c617ec771fd8fb7a5f8eaaa4b27b101f266b216a246 \ + --hash=sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9 \ + --hash=sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011 \ + --hash=sha256:eb9b459ca4df0e5c87deb59d37377461a538852765293f9e6ee834f0435a93b9 \ + --hash=sha256:efec8db3266b76ef9607c2c4c419bdb06bf335ae433b80816089ea7585816f6a \ + --hash=sha256:f481959862f57f29601ccced557cc2e817bce7533ab8e01a797a48b49c9692b3 \ + --hash=sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6 \ + --hash=sha256:f889f7a40498cc077332c7ab6b4608d296d852182211787d4f3ee377aaae66e8 \ + --hash=sha256:f8de619080e944347f5f20de29a975c2d815d9ddd8be9b9b7268e2e3ef68605a \ + --hash=sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 \ + --hash=sha256:fa754d1850735a0b0e03bcffd9d4b4343eb417e47196e4485d9cca326073a42c \ + --hash=sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6 \ + --hash=sha256:fe5b32187cbc0c862ee201ad66c30cf218e5ed468ec8dc1cf49dec66e160cc4d + # via + # -c python/requirements_compiled.txt + # pydantic +pygments==2.18.0 ; sys_platform != 'win32' \ + --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \ + --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a + # via + # -c python/requirements_compiled.txt + # rich +pyopenssl==25.0.0 \ + --hash=sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90 \ + --hash=sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16 + # via + # -c python/requirements_compiled.txt + # ray +python-dateutil==2.8.2 \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 + # via + # -c python/requirements_compiled.txt + # celery + # pandas +python-dotenv==1.1.1 \ + --hash=sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc \ + --hash=sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab + # via uvicorn +pytz==2022.7.1 \ + --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ + --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a + # via + # -c python/requirements_compiled.txt + # pandas +pyyaml==6.0.1 \ + --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ + --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ + --hash=sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df \ + --hash=sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741 \ + --hash=sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206 \ + --hash=sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27 \ + --hash=sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595 \ + --hash=sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62 \ + --hash=sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98 \ + --hash=sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696 \ + --hash=sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290 \ + --hash=sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9 \ + --hash=sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d \ + --hash=sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6 \ + --hash=sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867 \ + --hash=sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47 \ + --hash=sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486 \ + --hash=sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6 \ + --hash=sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3 \ + --hash=sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007 \ + --hash=sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938 \ + --hash=sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0 \ + --hash=sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c \ + --hash=sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735 \ + --hash=sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d \ + --hash=sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28 \ + --hash=sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4 \ + --hash=sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba \ + --hash=sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8 \ + --hash=sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef \ + --hash=sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5 \ + --hash=sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd \ + --hash=sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3 \ + --hash=sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0 \ + --hash=sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515 \ + --hash=sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c \ + --hash=sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c \ + --hash=sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924 \ + --hash=sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34 \ + --hash=sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43 \ + --hash=sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859 \ + --hash=sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673 \ + --hash=sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54 \ + --hash=sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a \ + --hash=sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b \ + --hash=sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab \ + --hash=sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa \ + --hash=sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c \ + --hash=sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585 \ + --hash=sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d \ + --hash=sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f + # via + # -c python/requirements_compiled.txt + # ray + # uvicorn +ray==100.0.0.dev0 \ + --hash=sha256:09b6b63a28bde8dfce18d07c3316c1330ecb81d57d4e2831a4d3e83883b6267d +referencing==0.36.2 \ + --hash=sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa \ + --hash=sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0 + # via + # -c python/requirements_compiled.txt + # jsonschema + # jsonschema-specifications +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 + # via + # -c python/requirements_compiled.txt + # google-api-core + # ray +rich==13.3.2 ; sys_platform != 'win32' \ + --hash=sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001 \ + --hash=sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f + # via + # -c python/requirements_compiled.txt + # memray +rpds-py==0.22.3 \ + --hash=sha256:009de23c9c9ee54bf11303a966edf4d9087cd43a6003672e6aa7def643d06518 \ + --hash=sha256:02fbb9c288ae08bcb34fb41d516d5eeb0455ac35b5512d03181d755d80810059 \ + --hash=sha256:0a0461200769ab3b9ab7e513f6013b7a97fdeee41c29b9db343f3c5a8e2b9e61 \ + --hash=sha256:0b09865a9abc0ddff4e50b5ef65467cd94176bf1e0004184eb915cbc10fc05c5 \ + --hash=sha256:0b8db6b5b2d4491ad5b6bdc2bc7c017eec108acbf4e6785f42a9eb0ba234f4c9 \ + --hash=sha256:0c150c7a61ed4a4f4955a96626574e9baf1adf772c2fb61ef6a5027e52803543 \ + --hash=sha256:0f3cec041684de9a4684b1572fe28c7267410e02450f4561700ca5a3bc6695a2 \ + --hash=sha256:1352ae4f7c717ae8cba93421a63373e582d19d55d2ee2cbb184344c82d2ae55a \ + --hash=sha256:177c7c0fce2855833819c98e43c262007f42ce86651ffbb84f37883308cb0e7d \ + --hash=sha256:1978d0021e943aae58b9b0b196fb4895a25cc53d3956b8e35e0b7682eefb6d56 \ + --hash=sha256:1a60bce91f81ddaac922a40bbb571a12c1070cb20ebd6d49c48e0b101d87300d \ + --hash=sha256:1aef18820ef3e4587ebe8b3bc9ba6e55892a6d7b93bac6d29d9f631a3b4befbd \ + --hash=sha256:1e9663daaf7a63ceccbbb8e3808fe90415b0757e2abddbfc2e06c857bf8c5e2b \ + --hash=sha256:20070c65396f7373f5df4005862fa162db5d25d56150bddd0b3e8214e8ef45b4 \ + --hash=sha256:214b7a953d73b5e87f0ebece4a32a5bd83c60a3ecc9d4ec8f1dca968a2d91e99 \ + --hash=sha256:22bebe05a9ffc70ebfa127efbc429bc26ec9e9b4ee4d15a740033efda515cf3d \ + --hash=sha256:24e8abb5878e250f2eb0d7859a8e561846f98910326d06c0d51381fed59357bd \ + --hash=sha256:26fd7cac7dd51011a245f29a2cc6489c4608b5a8ce8d75661bb4a1066c52dfbe \ + --hash=sha256:27b1d3b3915a99208fee9ab092b8184c420f2905b7d7feb4aeb5e4a9c509b8a1 \ + --hash=sha256:27e98004595899949bd7a7b34e91fa7c44d7a97c40fcaf1d874168bb652ec67e \ + --hash=sha256:2b8f60e1b739a74bab7e01fcbe3dddd4657ec685caa04681df9d562ef15b625f \ + --hash=sha256:2de29005e11637e7a2361fa151f780ff8eb2543a0da1413bb951e9f14b699ef3 \ + --hash=sha256:2e8b55d8517a2fda8d95cb45d62a5a8bbf9dd0ad39c5b25c8833efea07b880ca \ + --hash=sha256:2fa4331c200c2521512595253f5bb70858b90f750d39b8cbfd67465f8d1b596d \ + --hash=sha256:3445e07bf2e8ecfeef6ef67ac83de670358abf2996916039b16a218e3d95e97e \ + --hash=sha256:3453e8d41fe5f17d1f8e9c383a7473cd46a63661628ec58e07777c2fff7196dc \ + --hash=sha256:378753b4a4de2a7b34063d6f95ae81bfa7b15f2c1a04a9518e8644e81807ebea \ + --hash=sha256:3af6e48651c4e0d2d166dc1b033b7042ea3f871504b6805ba5f4fe31581d8d38 \ + --hash=sha256:3dfcbc95bd7992b16f3f7ba05af8a64ca694331bd24f9157b49dadeeb287493b \ + --hash=sha256:3f21f0495edea7fdbaaa87e633a8689cd285f8f4af5c869f27bc8074638ad69c \ + --hash=sha256:4041711832360a9b75cfb11b25a6a97c8fb49c07b8bd43d0d02b45d0b499a4ff \ + --hash=sha256:44d61b4b7d0c2c9ac019c314e52d7cbda0ae31078aabd0f22e583af3e0d79723 \ + --hash=sha256:4617e1915a539a0d9a9567795023de41a87106522ff83fbfaf1f6baf8e85437e \ + --hash=sha256:4b232061ca880db21fa14defe219840ad9b74b6158adb52ddf0e87bead9e8493 \ + --hash=sha256:5246b14ca64a8675e0a7161f7af68fe3e910e6b90542b4bfb5439ba752191df6 \ + --hash=sha256:5725dd9cc02068996d4438d397e255dcb1df776b7ceea3b9cb972bdb11260a83 \ + --hash=sha256:583f6a1993ca3369e0f80ba99d796d8e6b1a3a2a442dd4e1a79e652116413091 \ + --hash=sha256:59259dc58e57b10e7e18ce02c311804c10c5a793e6568f8af4dead03264584d1 \ + --hash=sha256:593eba61ba0c3baae5bc9be2f5232430453fb4432048de28399ca7376de9c627 \ + --hash=sha256:59f4a79c19232a5774aee369a0c296712ad0e77f24e62cad53160312b1c1eaa1 \ + --hash=sha256:5f0e260eaf54380380ac3808aa4ebe2d8ca28b9087cf411649f96bad6900c728 \ + --hash=sha256:62d9cfcf4948683a18a9aff0ab7e1474d407b7bab2ca03116109f8464698ab16 \ + --hash=sha256:64607d4cbf1b7e3c3c8a14948b99345eda0e161b852e122c6bb71aab6d1d798c \ + --hash=sha256:655ca44a831ecb238d124e0402d98f6212ac527a0ba6c55ca26f616604e60a45 \ + --hash=sha256:666ecce376999bf619756a24ce15bb14c5bfaf04bf00abc7e663ce17c3f34fe7 \ + --hash=sha256:68049202f67380ff9aa52f12e92b1c30115f32e6895cd7198fa2a7961621fc5a \ + --hash=sha256:69803198097467ee7282750acb507fba35ca22cc3b85f16cf45fb01cb9097730 \ + --hash=sha256:6c7b99ca52c2c1752b544e310101b98a659b720b21db00e65edca34483259967 \ + --hash=sha256:6dd9412824c4ce1aca56c47b0991e65bebb7ac3f4edccfd3f156150c96a7bf25 \ + --hash=sha256:70eb60b3ae9245ddea20f8a4190bd79c705a22f8028aaf8bbdebe4716c3fab24 \ + --hash=sha256:70fb28128acbfd264eda9bf47015537ba3fe86e40d046eb2963d75024be4d055 \ + --hash=sha256:7b2513ba235829860b13faa931f3b6846548021846ac808455301c23a101689d \ + --hash=sha256:7ef9d9da710be50ff6809fed8f1963fecdfecc8b86656cadfca3bc24289414b0 \ + --hash=sha256:81e69b0a0e2537f26d73b4e43ad7bc8c8efb39621639b4434b76a3de50c6966e \ + --hash=sha256:8633e471c6207a039eff6aa116e35f69f3156b3989ea3e2d755f7bc41754a4a7 \ + --hash=sha256:8bd7c8cfc0b8247c8799080fbff54e0b9619e17cdfeb0478ba7295d43f635d7c \ + --hash=sha256:9253fc214112405f0afa7db88739294295f0e08466987f1d70e29930262b4c8f \ + --hash=sha256:99b37292234e61325e7a5bb9689e55e48c3f5f603af88b1642666277a81f1fbd \ + --hash=sha256:9bd7228827ec7bb817089e2eb301d907c0d9827a9e558f22f762bb690b131652 \ + --hash=sha256:9beeb01d8c190d7581a4d59522cd3d4b6887040dcfc744af99aa59fef3e041a8 \ + --hash=sha256:a63cbdd98acef6570c62b92a1e43266f9e8b21e699c363c0fef13bd530799c11 \ + --hash=sha256:a76e42402542b1fae59798fab64432b2d015ab9d0c8c47ba7addddbaf7952333 \ + --hash=sha256:ac0a03221cdb5058ce0167ecc92a8c89e8d0decdc9e99a2ec23380793c4dcb96 \ + --hash=sha256:b0b4136a252cadfa1adb705bb81524eee47d9f6aab4f2ee4fa1e9d3cd4581f64 \ + --hash=sha256:b25bc607423935079e05619d7de556c91fb6adeae9d5f80868dde3468657994b \ + --hash=sha256:b3d504047aba448d70cf6fa22e06cb09f7cbd761939fdd47604f5e007675c24e \ + --hash=sha256:bb47271f60660803ad11f4c61b42242b8c1312a31c98c578f79ef9387bbde21c \ + --hash=sha256:bbb232860e3d03d544bc03ac57855cd82ddf19c7a07651a7c0fdb95e9efea8b9 \ + --hash=sha256:bc27863442d388870c1809a87507727b799c8460573cfbb6dc0eeaef5a11b5ec \ + --hash=sha256:bc51abd01f08117283c5ebf64844a35144a0843ff7b2983e0648e4d3d9f10dbb \ + --hash=sha256:be2eb3f2495ba669d2a985f9b426c1797b7d48d6963899276d22f23e33d47e37 \ + --hash=sha256:bf9db5488121b596dbfc6718c76092fda77b703c1f7533a226a5a9f65248f8ad \ + --hash=sha256:c58e2339def52ef6b71b8f36d13c3688ea23fa093353f3a4fee2556e62086ec9 \ + --hash=sha256:cfbc454a2880389dbb9b5b398e50d439e2e58669160f27b60e5eca11f68ae17c \ + --hash=sha256:cff63a0272fcd259dcc3be1657b07c929c466b067ceb1c20060e8d10af56f5bf \ + --hash=sha256:d115bffdd417c6d806ea9069237a4ae02f513b778e3789a359bc5856e0404cc4 \ + --hash=sha256:d20cfb4e099748ea39e6f7b16c91ab057989712d31761d3300d43134e26e165f \ + --hash=sha256:d48424e39c2611ee1b84ad0f44fb3b2b53d473e65de061e3f460fc0be5f1939d \ + --hash=sha256:e0fa2d4ec53dc51cf7d3bb22e0aa0143966119f42a0c3e4998293a3dd2856b09 \ + --hash=sha256:e32fee8ab45d3c2db6da19a5323bc3362237c8b653c70194414b892fd06a080d \ + --hash=sha256:e35ba67d65d49080e8e5a1dd40101fccdd9798adb9b050ff670b7d74fa41c566 \ + --hash=sha256:e3fb866d9932a3d7d0c82da76d816996d1667c44891bd861a0f97ba27e84fc74 \ + --hash=sha256:e61b02c3f7a1e0b75e20c3978f7135fd13cb6cf551bf4a6d29b999a88830a338 \ + --hash=sha256:e67ba3c290821343c192f7eae1d8fd5999ca2dc99994114643e2f2d3e6138b15 \ + --hash=sha256:e79dd39f1e8c3504be0607e5fc6e86bb60fe3584bec8b782578c3b0fde8d932c \ + --hash=sha256:e89391e6d60251560f0a8f4bd32137b077a80d9b7dbe6d5cab1cd80d2746f648 \ + --hash=sha256:ea7433ce7e4bfc3a85654aeb6747babe3f66eaf9a1d0c1e7a4435bbdf27fea84 \ + --hash=sha256:eaf16ae9ae519a0e237a0f528fd9f0197b9bb70f40263ee57ae53c2b8d48aeb3 \ + --hash=sha256:eb0c341fa71df5a4595f9501df4ac5abfb5a09580081dffbd1ddd4654e6e9123 \ + --hash=sha256:f276b245347e6e36526cbd4a266a417796fc531ddf391e43574cf6466c492520 \ + --hash=sha256:f47ad3d5f3258bd7058d2d506852217865afefe6153a36eb4b6928758041d831 \ + --hash=sha256:f56a6b404f74ab372da986d240e2e002769a7d7102cc73eb238a4f72eec5284e \ + --hash=sha256:f5cf2a0c2bdadf3791b5c205d55a37a54025c6e18a71c71f82bb536cf9a454bf \ + --hash=sha256:f5d36399a1b96e1a5fdc91e0522544580dbebeb1f77f27b2b0ab25559e103b8b \ + --hash=sha256:f60bd8423be1d9d833f230fdbccf8f57af322d96bcad6599e5a771b151398eb2 \ + --hash=sha256:f612463ac081803f243ff13cccc648578e2279295048f2a8d5eb430af2bae6e3 \ + --hash=sha256:f73d3fef726b3243a811121de45193c0ca75f6407fe66f3f4e183c983573e130 \ + --hash=sha256:f82a116a1d03628a8ace4859556fb39fd1424c933341a08ea3ed6de1edb0283b \ + --hash=sha256:fb0ba113b4983beac1a2eb16faffd76cb41e176bf58c4afe3e14b9c681f702de \ + --hash=sha256:fb4f868f712b2dd4bcc538b0a0c1f63a2b1d584c925e69a224d759e7070a12d5 \ + --hash=sha256:fb6116dfb8d1925cbdb52595560584db42a7f664617a1f7d7f6e32f138cdf37d \ + --hash=sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00 \ + --hash=sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e + # via + # -c python/requirements_compiled.txt + # jsonschema + # referencing +rsa==4.7.2 \ + --hash=sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2 \ + --hash=sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9 + # via + # -c python/requirements_compiled.txt + # google-auth +scipy==1.11.4 \ + --hash=sha256:00150c5eae7b610c32589dda259eacc7c4f1665aedf25d921907f4d08a951b1c \ + --hash=sha256:028eccd22e654b3ea01ee63705681ee79933652b2d8f873e7949898dda6d11b6 \ + --hash=sha256:1b7c3dca977f30a739e0409fb001056484661cb2541a01aba0bb0029f7b68db8 \ + --hash=sha256:2c6ff6ef9cc27f9b3db93a6f8b38f97387e6e0591600369a297a50a8e96e835d \ + --hash=sha256:36750b7733d960d7994888f0d148d31ea3017ac15eef664194b4ef68d36a4a97 \ + --hash=sha256:530f9ad26440e85766509dbf78edcfe13ffd0ab7fec2560ee5c36ff74d6269ff \ + --hash=sha256:5e347b14fe01003d3b78e196e84bd3f48ffe4c8a7b8a1afbcb8f5505cb710993 \ + --hash=sha256:6550466fbeec7453d7465e74d4f4b19f905642c89a7525571ee91dd7adabb5a3 \ + --hash=sha256:6df1468153a31cf55ed5ed39647279beb9cfb5d3f84369453b49e4b8502394fd \ + --hash=sha256:6e619aba2df228a9b34718efb023966da781e89dd3d21637b27f2e54db0410d7 \ + --hash=sha256:8fce70f39076a5aa62e92e69a7f62349f9574d8405c0a5de6ed3ef72de07f446 \ + --hash=sha256:90a2b78e7f5733b9de748f589f09225013685f9b218275257f8a8168ededaeaa \ + --hash=sha256:91af76a68eeae0064887a48e25c4e616fa519fa0d38602eda7e0f97d65d57937 \ + --hash=sha256:933baf588daa8dc9a92c20a0be32f56d43faf3d1a60ab11b3f08c356430f6e56 \ + --hash=sha256:acf8ed278cc03f5aff035e69cb511741e0418681d25fbbb86ca65429c4f4d9cd \ + --hash=sha256:ad669df80528aeca5f557712102538f4f37e503f0c5b9541655016dd0932ca79 \ + --hash=sha256:b030c6674b9230d37c5c60ab456e2cf12f6784596d15ce8da9365e70896effc4 \ + --hash=sha256:b9999c008ccf00e8fbcce1236f85ade5c569d13144f77a1946bef8863e8f6eb4 \ + --hash=sha256:bc9a714581f561af0848e6b69947fda0614915f072dfd14142ed1bfe1b806710 \ + --hash=sha256:ce7fff2e23ab2cc81ff452a9444c215c28e6305f396b2ba88343a567feec9660 \ + --hash=sha256:cf00bd2b1b0211888d4dc75656c0412213a8b25e80d73898083f402b50f47e41 \ + --hash=sha256:d10e45a6c50211fe256da61a11c34927c68f277e03138777bdebedd933712fea \ + --hash=sha256:ee410e6de8f88fd5cf6eadd73c135020bfbbbdfcd0f6162c36a7638a1ea8cc65 \ + --hash=sha256:f313b39a7e94f296025e3cffc2c567618174c0b1dde173960cf23808f9fae4be \ + --hash=sha256:f3cd9e7b3c2c1ec26364856f9fbe78695fe631150f94cd1c22228456404cf1ec + # via + # -c python/requirements_compiled.txt + # ray +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via + # -c python/requirements_compiled.txt + # opencensus + # python-dateutil +smart-open==6.2.0 \ + --hash=sha256:088bf00f9327c71e549bc2f86567d3320df5d89667f009ce1c16568976068ef7 \ + --hash=sha256:1b4df5c8365218f3852c507451920ccad606c80b0acb4e67508e50ba9b5d2632 + # via + # -c python/requirements_compiled.txt + # ray +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc + # via + # -c python/requirements_compiled.txt + # anyio +starlette==0.46.2 \ + --hash=sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35 \ + --hash=sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5 + # via + # -c python/requirements_compiled.txt + # fastapi + # ray +tensorboardx==2.6.2.2 \ + --hash=sha256:160025acbf759ede23fd3526ae9d9bfbfd8b68eb16c38a010ebe326dc6395db8 \ + --hash=sha256:c6476d7cd0d529b0b72f4acadb1269f9ed8b22f441e87a84f2a3b940bb87b666 + # via + # -c python/requirements_compiled.txt + # ray +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d + # via + # -c python/requirements_compiled.txt + # exceptiongroup + # fastapi + # gymnasium + # opentelemetry-api + # opentelemetry-sdk + # opentelemetry-semantic-conventions + # pydantic + # pydantic-core + # pyopenssl + # referencing + # starlette + # typing-inspection +typing-inspection==0.4.1 \ + --hash=sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 \ + --hash=sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28 + # via + # -c python/requirements_compiled.txt + # pydantic +tzdata==2025.2 \ + --hash=sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 \ + --hash=sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9 + # via + # -c python/requirements_compiled.txt + # kombu +urllib3==1.26.19 \ + --hash=sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3 \ + --hash=sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429 + # via + # -c python/requirements_compiled.txt + # requests +uvicorn==0.22.0 \ + --hash=sha256:79277ae03db57ce7d9aa0567830bbb51d7a612f54d6e1e3e92da3ef24c2c8ed8 \ + --hash=sha256:e9434d3bbf05f310e762147f769c9f21235ee118ba2d2bf1155a7196448bd996 + # via + # -c python/requirements_compiled.txt + # ray +uvloop==0.21.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32' \ + --hash=sha256:0878c2640cf341b269b7e128b1a5fed890adc4455513ca710d77d5e93aa6d6a0 \ + --hash=sha256:10d66943def5fcb6e7b37310eb6b5639fd2ccbc38df1177262b0640c3ca68c1f \ + --hash=sha256:10da8046cc4a8f12c91a1c39d1dd1585c41162a15caaef165c2174db9ef18bdc \ + --hash=sha256:17df489689befc72c39a08359efac29bbee8eee5209650d4b9f34df73d22e414 \ + --hash=sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f \ + --hash=sha256:196274f2adb9689a289ad7d65700d37df0c0930fd8e4e743fa4834e850d7719d \ + --hash=sha256:221f4f2a1f46032b403bf3be628011caf75428ee3cc204a22addf96f586b19fd \ + --hash=sha256:2d1f581393673ce119355d56da84fe1dd9d2bb8b3d13ce792524e1607139feff \ + --hash=sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c \ + --hash=sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3 \ + --hash=sha256:4509360fcc4c3bd2c70d87573ad472de40c13387f5fda8cb58350a1d7475e58d \ + --hash=sha256:460def4412e473896ef179a1671b40c039c7012184b627898eea5072ef6f017a \ + --hash=sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb \ + --hash=sha256:46923b0b5ee7fc0020bef24afe7836cb068f5050ca04caf6b487c513dc1a20b2 \ + --hash=sha256:53e420a3afe22cdcf2a0f4846e377d16e718bc70103d7088a4f7623567ba5fb0 \ + --hash=sha256:5ee4d4ef48036ff6e5cfffb09dd192c7a5027153948d85b8da7ff705065bacc6 \ + --hash=sha256:67dd654b8ca23aed0a8e99010b4c34aca62f4b7fce88f39d452ed7622c94845c \ + --hash=sha256:787ae31ad8a2856fc4e7c095341cccc7209bd657d0e71ad0dc2ea83c4a6fa8af \ + --hash=sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc \ + --hash=sha256:87c43e0f13022b998eb9b973b5e97200c8b90823454d4bc06ab33829e09fb9bb \ + --hash=sha256:88cb67cdbc0e483da00af0b2c3cdad4b7c61ceb1ee0f33fe00e09c81e3a6cb75 \ + --hash=sha256:8a375441696e2eda1c43c44ccb66e04d61ceeffcd76e4929e527b7fa401b90fb \ + --hash=sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553 \ + --hash=sha256:b9fb766bb57b7388745d8bcc53a359b116b8a04c83a2288069809d2b3466c37e \ + --hash=sha256:baa0e6291d91649c6ba4ed4b2f982f9fa165b5bbd50a9e203c416a2797bab3c6 \ + --hash=sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d \ + --hash=sha256:bc09f0ff191e61c2d592a752423c767b4ebb2986daa9ed62908e2b1b9a9ae206 \ + --hash=sha256:bd53ecc9a0f3d87ab847503c2e1552b690362e005ab54e8a48ba97da3924c0dc \ + --hash=sha256:bfd55dfcc2a512316e65f16e503e9e450cab148ef11df4e4e679b5e8253a5281 \ + --hash=sha256:c097078b8031190c934ed0ebfee8cc5f9ba9642e6eb88322b9958b649750f72b \ + --hash=sha256:c0f3fa6200b3108919f8bdabb9a7f87f20e7097ea3c543754cabc7d717d95cf8 \ + --hash=sha256:e678ad6fe52af2c58d2ae3c73dc85524ba8abe637f134bf3564ed07f555c5e79 \ + --hash=sha256:ec7e6b09a6fdded42403182ab6b832b71f4edaf7f37a9a0e371a01db5f0cb45f \ + --hash=sha256:f0ce1b49560b1d2d8a2977e3ba4afb2414fb46b86a1b64056bc4ab929efdafbe \ + --hash=sha256:f38b2e090258d051d68a5b14d1da7203a3c3677321cf32a95a6f4db4dd8b6f26 \ + --hash=sha256:f3df876acd7ec037a3d005b3ab85a7e4110422e4d9c1571d4fc89b0fc41b6816 \ + --hash=sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2 + # via + # -c python/requirements_compiled.txt + # uvicorn +vine==5.1.0 \ + --hash=sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc \ + --hash=sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0 + # via + # -c python/requirements_compiled.txt + # amqp + # celery + # kombu +virtualenv==20.29.1 \ + --hash=sha256:4e4cb403c0b0da39e13b46b1b2476e505cb0046b25f242bee80f62bf990b2779 \ + --hash=sha256:b8b8970138d32fb606192cb97f6cd4bb644fa486be9308fb9b63f81091b5dc35 + # via + # -c python/requirements_compiled.txt + # ray +watchfiles==0.19.0 \ + --hash=sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911 \ + --hash=sha256:09ea3397aecbc81c19ed7f025e051a7387feefdb789cf768ff994c1228182fda \ + --hash=sha256:176a9a7641ec2c97b24455135d58012a5be5c6217fc4d5fef0b2b9f75dbf5154 \ + --hash=sha256:18b28f6ad871b82df9542ff958d0c86bb0d8310bb09eb8e87d97318a3b5273af \ + --hash=sha256:20b44221764955b1e703f012c74015306fb7e79a00c15370785f309b1ed9aa8d \ + --hash=sha256:3d7d267d27aceeeaa3de0dd161a0d64f0a282264d592e335fff7958cc0cbae7c \ + --hash=sha256:5471582658ea56fca122c0f0d0116a36807c63fefd6fdc92c71ca9a4491b6b48 \ + --hash=sha256:5569fc7f967429d4bc87e355cdfdcee6aabe4b620801e2cf5805ea245c06097c \ + --hash=sha256:68dce92b29575dda0f8d30c11742a8e2b9b8ec768ae414b54f7453f27bdf9545 \ + --hash=sha256:79c533ff593db861ae23436541f481ec896ee3da4e5db8962429b441bbaae16e \ + --hash=sha256:7f3920b1285a7d3ce898e303d84791b7bf40d57b7695ad549dc04e6a44c9f120 \ + --hash=sha256:91633e64712df3051ca454ca7d1b976baf842d7a3640b87622b323c55f3345e7 \ + --hash=sha256:945be0baa3e2440151eb3718fd8846751e8b51d8de7b884c90b17d271d34cae8 \ + --hash=sha256:9afd0d69429172c796164fd7fe8e821ade9be983f51c659a38da3faaaaac44dc \ + --hash=sha256:9c75eff897786ee262c9f17a48886f4e98e6cfd335e011c591c305e5d083c056 \ + --hash=sha256:b538014a87f94d92f98f34d3e6d2635478e6be6423a9ea53e4dd96210065e193 \ + --hash=sha256:b6577b8c6c8701ba8642ea9335a129836347894b666dd1ec2226830e263909d3 \ + --hash=sha256:c0376deac92377817e4fb8f347bf559b7d44ff556d9bc6f6208dd3f79f104aaf \ + --hash=sha256:cae3dde0b4b2078f31527acff6f486e23abed307ba4d3932466ba7cdd5ecec79 \ + --hash=sha256:cb5d45c4143c1dd60f98a16187fd123eda7248f84ef22244818c18d531a249d1 \ + --hash=sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b \ + --hash=sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0 + # via + # -c python/requirements_compiled.txt + # ray + # uvicorn +wcwidth==0.2.13 \ + --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ + --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 + # via + # -c python/requirements_compiled.txt + # prompt-toolkit +websockets==11.0.3 \ + --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \ + --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \ + --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \ + --hash=sha256:0ee68fe502f9031f19d495dae2c268830df2760c0524cbac5d759921ba8c8e82 \ + --hash=sha256:1553cb82942b2a74dd9b15a018dce645d4e68674de2ca31ff13ebc2d9f283788 \ + --hash=sha256:1a073fc9ab1c8aff37c99f11f1641e16da517770e31a37265d2755282a5d28aa \ + --hash=sha256:1d2256283fa4b7f4c7d7d3e84dc2ece74d341bce57d5b9bf385df109c2a1a82f \ + --hash=sha256:1d5023a4b6a5b183dc838808087033ec5df77580485fc533e7dab2567851b0a4 \ + --hash=sha256:1fdf26fa8a6a592f8f9235285b8affa72748dc12e964a5518c6c5e8f916716f7 \ + --hash=sha256:2529338a6ff0eb0b50c7be33dc3d0e456381157a31eefc561771ee431134a97f \ + --hash=sha256:279e5de4671e79a9ac877427f4ac4ce93751b8823f276b681d04b2156713b9dd \ + --hash=sha256:2d903ad4419f5b472de90cd2d40384573b25da71e33519a67797de17ef849b69 \ + --hash=sha256:332d126167ddddec94597c2365537baf9ff62dfcc9db4266f263d455f2f031cb \ + --hash=sha256:34fd59a4ac42dff6d4681d8843217137f6bc85ed29722f2f7222bd619d15e95b \ + --hash=sha256:3580dd9c1ad0701169e4d6fc41e878ffe05e6bdcaf3c412f9d559389d0c9e016 \ + --hash=sha256:3ccc8a0c387629aec40f2fc9fdcb4b9d5431954f934da3eaf16cdc94f67dbfac \ + --hash=sha256:41f696ba95cd92dc047e46b41b26dd24518384749ed0d99bea0a941ca87404c4 \ + --hash=sha256:42cc5452a54a8e46a032521d7365da775823e21bfba2895fb7b77633cce031bb \ + --hash=sha256:4841ed00f1026dfbced6fca7d963c4e7043aa832648671b5138008dc5a8f6d99 \ + --hash=sha256:4b253869ea05a5a073ebfdcb5cb3b0266a57c3764cf6fe114e4cd90f4bfa5f5e \ + --hash=sha256:54c6e5b3d3a8936a4ab6870d46bdd6ec500ad62bde9e44462c32d18f1e9a8e54 \ + --hash=sha256:619d9f06372b3a42bc29d0cd0354c9bb9fb39c2cbc1a9c5025b4538738dbffaf \ + --hash=sha256:6505c1b31274723ccaf5f515c1824a4ad2f0d191cec942666b3d0f3aa4cb4007 \ + --hash=sha256:660e2d9068d2bedc0912af508f30bbeb505bbbf9774d98def45f68278cea20d3 \ + --hash=sha256:6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6 \ + --hash=sha256:68b977f21ce443d6d378dbd5ca38621755f2063d6fdb3335bda981d552cfff86 \ + --hash=sha256:69269f3a0b472e91125b503d3c0b3566bda26da0a3261c49f0027eb6075086d1 \ + --hash=sha256:6f1a3f10f836fab6ca6efa97bb952300b20ae56b409414ca85bff2ad241d2a61 \ + --hash=sha256:7622a89d696fc87af8e8d280d9b421db5133ef5b29d3f7a1ce9f1a7bf7fcfa11 \ + --hash=sha256:777354ee16f02f643a4c7f2b3eff8027a33c9861edc691a2003531f5da4f6bc8 \ + --hash=sha256:84d27a4832cc1a0ee07cdcf2b0629a8a72db73f4cf6de6f0904f6661227f256f \ + --hash=sha256:8531fdcad636d82c517b26a448dcfe62f720e1922b33c81ce695d0edb91eb931 \ + --hash=sha256:86d2a77fd490ae3ff6fae1c6ceaecad063d3cc2320b44377efdde79880e11526 \ + --hash=sha256:88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016 \ + --hash=sha256:8a34e13a62a59c871064dfd8ffb150867e54291e46d4a7cf11d02c94a5275bae \ + --hash=sha256:8c82f11964f010053e13daafdc7154ce7385ecc538989a354ccc7067fd7028fd \ + --hash=sha256:92b2065d642bf8c0a82d59e59053dd2fdde64d4ed44efe4870fa816c1232647b \ + --hash=sha256:97b52894d948d2f6ea480171a27122d77af14ced35f62e5c892ca2fae9344311 \ + --hash=sha256:9d9acd80072abcc98bd2c86c3c9cd4ac2347b5a5a0cae7ed5c0ee5675f86d9af \ + --hash=sha256:9f59a3c656fef341a99e3d63189852be7084c0e54b75734cde571182c087b152 \ + --hash=sha256:aa5003845cdd21ac0dc6c9bf661c5beddd01116f6eb9eb3c8e272353d45b3288 \ + --hash=sha256:b16fff62b45eccb9c7abb18e60e7e446998093cdcb50fed33134b9b6878836de \ + --hash=sha256:b30c6590146e53149f04e85a6e4fcae068df4289e31e4aee1fdf56a0dead8f97 \ + --hash=sha256:b58cbf0697721120866820b89f93659abc31c1e876bf20d0b3d03cef14faf84d \ + --hash=sha256:b67c6f5e5a401fc56394f191f00f9b3811fe843ee93f4a70df3c389d1adf857d \ + --hash=sha256:bceab846bac555aff6427d060f2fcfff71042dba6f5fca7dc4f75cac815e57ca \ + --hash=sha256:bee9fcb41db2a23bed96c6b6ead6489702c12334ea20a297aa095ce6d31370d0 \ + --hash=sha256:c114e8da9b475739dde229fd3bc6b05a6537a88a578358bc8eb29b4030fac9c9 \ + --hash=sha256:c1f0524f203e3bd35149f12157438f406eff2e4fb30f71221c8a5eceb3617b6b \ + --hash=sha256:c792ea4eabc0159535608fc5658a74d1a81020eb35195dd63214dcf07556f67e \ + --hash=sha256:c7f3cb904cce8e1be667c7e6fef4516b98d1a6a0635a58a57528d577ac18a128 \ + --hash=sha256:d67ac60a307f760c6e65dad586f556dde58e683fab03323221a4e530ead6f74d \ + --hash=sha256:dcacf2c7a6c3a84e720d1bb2b543c675bf6c40e460300b628bab1b1efc7c034c \ + --hash=sha256:de36fe9c02995c7e6ae6efe2e205816f5f00c22fd1fbf343d4d18c3d5ceac2f5 \ + --hash=sha256:def07915168ac8f7853812cc593c71185a16216e9e4fa886358a17ed0fd9fcf6 \ + --hash=sha256:df41b9bc27c2c25b486bae7cf42fccdc52ff181c8c387bfd026624a491c2671b \ + --hash=sha256:e052b8467dd07d4943936009f46ae5ce7b908ddcac3fda581656b1b19c083d9b \ + --hash=sha256:e063b1865974611313a3849d43f2c3f5368093691349cf3c7c8f8f75ad7cb280 \ + --hash=sha256:e1459677e5d12be8bbc7584c35b992eea142911a6236a3278b9b5ce3326f282c \ + --hash=sha256:e1a99a7a71631f0efe727c10edfba09ea6bee4166a6f9c19aafb6c0b5917d09c \ + --hash=sha256:e590228200fcfc7e9109509e4d9125eace2042fd52b595dd22bbc34bb282307f \ + --hash=sha256:e6316827e3e79b7b8e7d8e3b08f4e331af91a48e794d5d8b099928b6f0b85f20 \ + --hash=sha256:e7837cb169eca3b3ae94cc5787c4fed99eef74c0ab9506756eea335e0d6f3ed8 \ + --hash=sha256:e848f46a58b9fcf3d06061d17be388caf70ea5b8cc3466251963c8345e13f7eb \ + --hash=sha256:ed058398f55163a79bb9f06a90ef9ccc063b204bb346c4de78efc5d15abfe602 \ + --hash=sha256:f2e58f2c36cc52d41f2659e4c0cbf7353e28c8c9e63e30d8c6d3494dc9fdedcf \ + --hash=sha256:f467ba0050b7de85016b43f5a22b46383ef004c4f672148a8abf32bc999a87f0 \ + --hash=sha256:f61bdb1df43dc9c131791fbc2355535f9024b9a04398d3bd0684fc16ab07df74 \ + --hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \ + --hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564 + # via + # -c python/requirements_compiled.txt + # uvicorn +yarl==1.18.3 \ + --hash=sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba \ + --hash=sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193 \ + --hash=sha256:045b8482ce9483ada4f3f23b3774f4e1bf4f23a2d5c912ed5170f68efb053318 \ + --hash=sha256:09c7907c8548bcd6ab860e5f513e727c53b4a714f459b084f6580b49fa1b9cee \ + --hash=sha256:0b0cad37311123211dc91eadcb322ef4d4a66008d3e1bdc404808992260e1a0e \ + --hash=sha256:0b3c92fa08759dbf12b3a59579a4096ba9af8dd344d9a813fc7f5070d86bbab1 \ + --hash=sha256:0fb2171a4486bb075316ee754c6d8382ea6eb8b399d4ec62fde2b591f879778a \ + --hash=sha256:1a74a13a4c857a84a845505fd2d68e54826a2cd01935a96efb1e9d86c728e186 \ + --hash=sha256:1d407181cfa6e70077df3377938c08012d18893f9f20e92f7d2f314a437c30b1 \ + --hash=sha256:1dd4bdd05407ced96fed3d7f25dbbf88d2ffb045a0db60dbc247f5b3c5c25d50 \ + --hash=sha256:25b411eddcfd56a2f0cd6a384e9f4f7aa3efee14b188de13048c25b5e91f1640 \ + --hash=sha256:2d06d3005e668744e11ed80812e61efd77d70bb7f03e33c1598c301eea20efbb \ + --hash=sha256:2ec9bbba33b2d00999af4631a3397d1fd78290c48e2a3e52d8dd72db3a067ac8 \ + --hash=sha256:3236da9272872443f81fedc389bace88408f64f89f75d1bdb2256069a8730ccc \ + --hash=sha256:35098b24e0327fc4ebdc8ffe336cee0a87a700c24ffed13161af80124b7dc8e5 \ + --hash=sha256:41f7ce59d6ee7741af71d82020346af364949314ed3d87553763a2df1829cc58 \ + --hash=sha256:436c4fc0a4d66b2badc6c5fc5ef4e47bb10e4fd9bf0c79524ac719a01f3607c2 \ + --hash=sha256:4891ed92157e5430874dad17b15eb1fda57627710756c27422200c52d8a4e393 \ + --hash=sha256:4ac515b860c36becb81bb84b667466885096b5fc85596948548b667da3bf9f24 \ + --hash=sha256:5094d9206c64181d0f6e76ebd8fb2f8fe274950a63890ee9e0ebfd58bf9d787b \ + --hash=sha256:54d6921f07555713b9300bee9c50fb46e57e2e639027089b1d795ecd9f7fa910 \ + --hash=sha256:578e281c393af575879990861823ef19d66e2b1d0098414855dd367e234f5b3c \ + --hash=sha256:5a3f356548e34a70b0172d8890006c37be92995f62d95a07b4a42e90fba54272 \ + --hash=sha256:602d98f2c2d929f8e697ed274fbadc09902c4025c5a9963bf4e9edfc3ab6f7ed \ + --hash=sha256:61b1a825a13bef4a5f10b1885245377d3cd0bf87cba068e1d9a88c2ae36880e1 \ + --hash=sha256:61e5e68cb65ac8f547f6b5ef933f510134a6bf31bb178be428994b0cb46c2a04 \ + --hash=sha256:61ee62ead9b68b9123ec24bc866cbef297dd266175d53296e2db5e7f797f902d \ + --hash=sha256:6333c5a377c8e2f5fae35e7b8f145c617b02c939d04110c76f29ee3676b5f9a5 \ + --hash=sha256:6748dbf9bfa5ba1afcc7556b71cda0d7ce5f24768043a02a58846e4a443d808d \ + --hash=sha256:67a283dd2882ac98cc6318384f565bffc751ab564605959df4752d42483ad889 \ + --hash=sha256:75674776d96d7b851b6498f17824ba17849d790a44d282929c42dbb77d4f17ae \ + --hash=sha256:757e81cae69244257d125ff31663249b3013b5dc0a8520d73694aed497fb195b \ + --hash=sha256:77a6e85b90a7641d2e07184df5557132a337f136250caafc9ccaa4a2a998ca2c \ + --hash=sha256:7c33dd1931a95e5d9a772d0ac5e44cac8957eaf58e3c8da8c1414de7dd27c576 \ + --hash=sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34 \ + --hash=sha256:7e2ee16578af3b52ac2f334c3b1f92262f47e02cc6193c598502bd46f5cd1477 \ + --hash=sha256:80316a8bd5109320d38eef8833ccf5f89608c9107d02d2a7f985f98ed6876990 \ + --hash=sha256:82123d0c954dc58db301f5021a01854a85bf1f3bb7d12ae0c01afc414a882ca2 \ + --hash=sha256:84b2deecba4a3f1a398df819151eb72d29bfeb3b69abb145a00ddc8d30094512 \ + --hash=sha256:8503ad47387b8ebd39cbbbdf0bf113e17330ffd339ba1144074da24c545f0069 \ + --hash=sha256:877d209b6aebeb5b16c42cbb377f5f94d9e556626b1bfff66d7b0d115be88d0a \ + --hash=sha256:8874027a53e3aea659a6d62751800cf6e63314c160fd607489ba5c2edd753cf6 \ + --hash=sha256:88a19f62ff30117e706ebc9090b8ecc79aeb77d0b1f5ec10d2d27a12bc9f66d0 \ + --hash=sha256:8d39d351e7faf01483cc7ff7c0213c412e38e5a340238826be7e0e4da450fdc8 \ + --hash=sha256:90adb47ad432332d4f0bc28f83a5963f426ce9a1a8809f5e584e704b82685dcb \ + --hash=sha256:913829534200eb0f789d45349e55203a091f45c37a2674678744ae52fae23efa \ + --hash=sha256:93b2e109287f93db79210f86deb6b9bbb81ac32fc97236b16f7433db7fc437d8 \ + --hash=sha256:9d41beda9dc97ca9ab0b9888cb71f7539124bc05df02c0cff6e5acc5a19dcc6e \ + --hash=sha256:a440a2a624683108a1b454705ecd7afc1c3438a08e890a1513d468671d90a04e \ + --hash=sha256:a4bb030cf46a434ec0225bddbebd4b89e6471814ca851abb8696170adb163985 \ + --hash=sha256:a9ca04806f3be0ac6d558fffc2fdf8fcef767e0489d2684a21912cc4ed0cd1b8 \ + --hash=sha256:ac1801c45cbf77b6c99242eeff4fffb5e4e73a800b5c4ad4fc0be5def634d2e1 \ + --hash=sha256:ac36703a585e0929b032fbaab0707b75dc12703766d0b53486eabd5139ebadd5 \ + --hash=sha256:b1771de9944d875f1b98a745bc547e684b863abf8f8287da8466cf470ef52690 \ + --hash=sha256:b464c4ab4bfcb41e3bfd3f1c26600d038376c2de3297760dfe064d2cb7ea8e10 \ + --hash=sha256:b4f6450109834af88cb4cc5ecddfc5380ebb9c228695afc11915a0bf82116789 \ + --hash=sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b \ + --hash=sha256:b643562c12680b01e17239be267bc306bbc6aac1f34f6444d1bded0c5ce438ca \ + --hash=sha256:b958ddd075ddba5b09bb0be8a6d9906d2ce933aee81100db289badbeb966f54e \ + --hash=sha256:b9d60031cf568c627d028239693fd718025719c02c9f55df0a53e587aab951b5 \ + --hash=sha256:ba23302c0c61a9999784e73809427c9dbedd79f66a13d84ad1b1943802eaaf59 \ + --hash=sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9 \ + --hash=sha256:c017a3b6df3a1bd45b9fa49a0f54005e53fbcad16633870104b66fa1a30a29d8 \ + --hash=sha256:c1e1cc06da1491e6734f0ea1e6294ce00792193c463350626571c287c9a704db \ + --hash=sha256:c654d5207c78e0bd6d749f6dae1dcbbfde3403ad3a4b11f3c5544d9906969dde \ + --hash=sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7 \ + --hash=sha256:c7d79f7d9aabd6011004e33b22bc13056a3e3fb54794d138af57f5ee9d9032cb \ + --hash=sha256:ccaa3a4b521b780a7e771cc336a2dba389a0861592bbce09a476190bb0c8b4b3 \ + --hash=sha256:ccd17349166b1bee6e529b4add61727d3f55edb7babbe4069b5764c9587a8cc6 \ + --hash=sha256:ce1af883b94304f493698b00d0f006d56aea98aeb49d75ec7d98cd4a777e9285 \ + --hash=sha256:d0e883008013c0e4aef84dcfe2a0b172c4d23c2669412cf5b3371003941f72bb \ + --hash=sha256:d980e0325b6eddc81331d3f4551e2a333999fb176fd153e075c6d1c2530aa8a8 \ + --hash=sha256:e17c9361d46a4d5addf777c6dd5eab0715a7684c2f11b88c67ac37edfba6c482 \ + --hash=sha256:e2c08cc9b16f4f4bc522771d96734c7901e7ebef70c6c5c35dd0f10845270bcd \ + --hash=sha256:e35ef8683211db69ffe129a25d5634319a677570ab6b2eba4afa860f54eeaf75 \ + --hash=sha256:e3b9fd71836999aad54084906f8663dffcd2a7fb5cdafd6c37713b2e72be1760 \ + --hash=sha256:ef9f7768395923c3039055c14334ba4d926f3baf7b776c923c93d80195624782 \ + --hash=sha256:f52a265001d830bc425f82ca9eabda94a64a4d753b07d623a9f2863fde532b53 \ + --hash=sha256:f91c4803173928a25e1a55b943c81f55b8872f0018be83e3ad4938adffb77dd2 \ + --hash=sha256:fbd6748e8ab9b41171bb95c6142faf068f5ef1511935a0aa07025438dd9a9bc1 \ + --hash=sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719 \ + --hash=sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62 + # via + # -c python/requirements_compiled.txt + # aiohttp +zipp==3.19.2 \ + --hash=sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c + # via + # -c python/requirements_compiled.txt + # importlib-metadata From 03e5cd9870f472f4d9d8345343d4baa84a14c16e Mon Sep 17 00:00:00 2001 From: harshit-anyscale Date: Mon, 15 Sep 2025 22:10:47 +0530 Subject: [PATCH 620/634] add more tests for async inf (#56408) added more tests for asynchronous inference for the below cases: - metrics - health checks - cancel tasks --------- Signed-off-by: harshit --- python/ray/serve/task_processor.py | 18 +-- python/ray/serve/tests/conftest.py | 6 +- python/ray/serve/tests/test_task_processor.py | 147 +++++++++++++++++- 3 files changed, 156 insertions(+), 15 deletions(-) diff --git a/python/ray/serve/task_processor.py b/python/ray/serve/task_processor.py index 4d1d21ba6db6..cd66d83ff354 100644 --- a/python/ray/serve/task_processor.py +++ b/python/ray/serve/task_processor.py @@ -180,15 +180,12 @@ def shutdown(self): pass @abstractmethod - def cancel_task_sync(self, task_id: str) -> bool: + def cancel_task_sync(self, task_id: str): """ Cancel a task synchronously. Args: task_id: Unique identifier of the task to cancel. - - Returns: - bool: True if cancellation was requested successfully. """ pass @@ -265,16 +262,13 @@ async def get_task_status_async(self, task_id: str) -> TaskResult: "Subclass must implement get_task_status_async function" ) - async def cancel_task_async(self, task_id: str) -> bool: + async def cancel_task_async(self, task_id: str): """ Cancel a task. Args: task_id: Unique identifier of the task to cancel. - Returns: - bool: True if cancellation was requested successfully. - Raises: NotImplementedError: If async task cancellation is not supported by this adapter. """ @@ -505,8 +499,12 @@ def shutdown(self): self._app.control.shutdown() logger.info("Celery worker shutdown complete...") - def cancel_task_sync(self, task_id) -> bool: - return self._app.AsyncResult(task_id).cancel() + def cancel_task_sync(self, task_id): + """ + Cancels a task synchronously. Only supported for Redis and RabbitMQ brokers by Celery. + More details can be found here: https://docs.celeryq.dev/en/stable/userguide/workers.html#revoke-revoking-tasks + """ + self._app.control.revoke(task_id) def get_metrics_sync(self) -> Dict[str, Any]: """ diff --git a/python/ray/serve/tests/conftest.py b/python/ray/serve/tests/conftest.py index 5de04057db06..69e7392e4b91 100644 --- a/python/ray/serve/tests/conftest.py +++ b/python/ray/serve/tests/conftest.py @@ -23,7 +23,11 @@ ) from ray.serve.config import HTTPOptions, gRPCOptions from ray.serve.context import _get_global_client -from ray.tests.conftest import propagate_logs, pytest_runtest_makereport # noqa +from ray.tests.conftest import ( # noqa + external_redis, + propagate_logs, + pytest_runtest_makereport, +) # https://tools.ietf.org/html/rfc6335#section-6 MIN_DYNAMIC_PORT = 49152 diff --git a/python/ray/serve/tests/test_task_processor.py b/python/ray/serve/tests/test_task_processor.py index d194f3dca6cc..b8892992712e 100644 --- a/python/ray/serve/tests/test_task_processor.py +++ b/python/ray/serve/tests/test_task_processor.py @@ -1,4 +1,5 @@ import json +import os import sys import tempfile from collections import defaultdict @@ -15,6 +16,7 @@ task_consumer, task_handler, ) +from ray.tests.conftest import external_redis # noqa: F401 @ray.remote @@ -76,13 +78,13 @@ def transport_options(temp_queue_directory): return { # Incoming message queue - where new task messages are written when sent to broker - "data_folder_in": queue_path, + "data_folder_in": str(queue_path), # Outgoing message storage - where task results and responses are written after completion - "data_folder_out": queue_path, + "data_folder_out": str(queue_path), # Processed message archive - where messages are moved after successful processing - "data_folder_processed": queue_path, + "data_folder_processed": str(queue_path), # Control message storage - where Celery management and control commands are stored - "control_folder": control_path, + "control_folder": str(control_path), } @@ -336,6 +338,143 @@ async def process_request(self, data): self.task_received = True self.data_received = data + def test_task_consumer_metrics( + self, temp_queue_directory, serve_instance, create_processor_config + ): + """Test that task processor metrics are collected and exposed correctly.""" + processor_config = create_processor_config() + + @serve.deployment + @task_consumer(task_processor_config=processor_config) + class ServeTaskConsumer: + def __init__(self): + self.task_received = False + + @task_handler(name="process_request") + def process_request(self, data): + self.task_received = True + + def get_task_received(self) -> bool: + return self.task_received + + handle = serve.run(ServeTaskConsumer.bind()) + send_request_to_queue.remote(processor_config, "test_data_1") + + def assert_task_received(): + return handle.get_task_received.remote().result() + + wait_for_condition(assert_task_received, timeout=20) + + adapter_instance = instantiate_adapter_from_config( + task_processor_config=processor_config + ) + metrics = adapter_instance.get_metrics_sync() + + assert len(metrics) == 1 + worker_name = next(iter(metrics)) + worker_stats = metrics[worker_name] + + # Check that the total number of processed tasks is correct. + assert worker_stats["pool"]["threads"] == 1 + assert worker_stats["pool"]["max-concurrency"] == 1 + assert worker_stats["total"]["process_request"] == 1 + assert worker_stats["broker"]["transport"] == "filesystem" + + def test_task_consumer_health_check( + self, temp_queue_directory, serve_instance, create_processor_config + ): + """Test that the health check for the task processor works correctly.""" + processor_config = create_processor_config() + + @serve.deployment + @task_consumer(task_processor_config=processor_config) + class ServeTaskConsumer: + pass + + serve.run(ServeTaskConsumer.bind()) + + adapter_instance = instantiate_adapter_from_config( + task_processor_config=processor_config + ) + + def check_health(): + health_status = adapter_instance.health_check_sync() + return len(health_status) > 0 + + # Wait for the worker to be ready + wait_for_condition(check_health, timeout=20) + + health_status = adapter_instance.health_check_sync() + assert len(health_status) == 1 + + worker_reply = health_status[0] + assert len(worker_reply) == 1 + worker_name = next(iter(worker_reply)) + assert worker_reply[worker_name] == {"ok": "pong"} + + def test_task_processor_with_cancel_tasks( + self, external_redis, serve_instance # noqa: F811 + ): + """Test the cancel task functionality with celery broker.""" + redis_address = os.environ.get("RAY_REDIS_ADDRESS") + + processor_config = TaskProcessorConfig( + queue_name="my_app_queue", + adapter_config=CeleryAdapterConfig( + broker_url=f"redis://{redis_address}/0", + backend_url=f"redis://{redis_address}/1", + worker_concurrency=1, + ), + ) + + signal = SignalActor.remote() + + @serve.deployment + @task_consumer(task_processor_config=processor_config) + class MyTaskConsumer: + def __init__(self, signal_actor): + self._signal = signal_actor + self.message_received = [] + + @task_handler(name="process") + def process(self, data): + ray.get(self._signal.wait.remote()) + self.message_received.append(data) + + def get_message_received(self): + return self.message_received + + handle = serve.run(MyTaskConsumer.bind(signal), name="app_v1") + + task_ids = [] + for i in range(2): + task_id_ref = send_request_to_queue.remote( + processor_config, f"test_data_{i}", task_name="process" + ) + task_ids.append(ray.get(task_id_ref)) + + wait_for_condition( + lambda: ray.get(signal.cur_num_waiters.remote()) == 1, timeout=10 + ) + + adapter_instance = instantiate_adapter_from_config( + task_processor_config=processor_config + ) + adapter_instance.cancel_task_sync(task_ids[1]) + + ray.get(signal.send.remote()) + + def check_revoked(): + status = adapter_instance.get_task_status_sync(task_ids[1]) + return status.status == "REVOKED" + + wait_for_condition(check_revoked, timeout=20) + + assert "test_data_0" in handle.get_message_received.remote().result() + assert "test_data_1" not in handle.get_message_received.remote().result() + + serve.delete("app_v1") + @pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.") class TestTaskConsumerWithDLQsConfiguration: From 97e2b324bbb3a65d717597ec05398b7c3ebcb932 Mon Sep 17 00:00:00 2001 From: Omkar Kulkarni Date: Mon, 15 Sep 2025 09:42:44 -0700 Subject: [PATCH 621/634] [SERVE] Proxy Actor Interface (#56288) Introduce proxy actor interface. Signed-off-by: Omkar Kulkarni Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python/ray/serve/_private/proxy.py | 118 +++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 5 deletions(-) diff --git a/python/ray/serve/_private/proxy.py b/python/ray/serve/_private/proxy.py index 767c16ba6d50..66ad787a2146 100644 --- a/python/ray/serve/_private/proxy.py +++ b/python/ray/serve/_private/proxy.py @@ -1016,8 +1016,113 @@ async def send_request_to_replica( yield status +class ProxyActorInterface(ABC): + """Abstract interface for proxy actors in Ray Serve. + + This interface defines the contract that all proxy actor implementations must follow, + allowing for different proxy backends (Ray HTTP/gRPC proxies, HAProxy, etc.). + """ + + def __init__( + self, + *, + node_id: NodeId, + node_ip_address: str, + logging_config: LoggingConfig, + ): + """Initialize the proxy actor. + + Args: + node_id: ID of the node this proxy is running on + node_ip_address: IP address of the node + logging_config: Logging configuration + """ + self._node_id = node_id + self._node_ip_address = node_ip_address + self._logging_config = logging_config + + @abstractmethod + async def ready(self) -> str: + """Blocks until the proxy is ready to serve requests. + + Returns: + JSON-serialized metadata containing proxy information (worker ID, log file path, etc.) + """ + pass + + @abstractmethod + async def update_draining( + self, draining: bool, _after: Optional[Any] = None + ) -> None: + """Update the draining status of the proxy. + + Args: + draining: Whether the proxy should be draining + _after: Optional ObjectRef for scheduling dependency + """ + pass + + @abstractmethod + async def is_drained(self, _after: Optional[Any] = None) -> bool: + """Check whether the proxy is drained. + + Args: + _after: Optional ObjectRef for scheduling dependency + + Returns: + True if the proxy is drained, False otherwise + """ + pass + + @abstractmethod + async def check_health(self) -> None: + """Check the health of the proxy. + + Raises: + Exception: if the proxy is unhealthy + """ + pass + + @abstractmethod + def pong(self) -> str: + """Respond to ping from replicas. + + Returns: + A response string + """ + pass + + @abstractmethod + async def receive_asgi_messages(self, request_metadata: RequestMetadata) -> bytes: + """Handle ASGI messages for HTTP requests. + + Args: + request_metadata: Metadata about the request + + Returns: + Serialized ASGI messages + """ + pass + + # Testing and debugging methods + @abstractmethod + def _get_http_options(self) -> HTTPOptions: + """Get HTTP options used by the proxy.""" + pass + + @abstractmethod + def _get_logging_config(self) -> Optional[str]: + """Get the file path for the logger (for testing purposes).""" + pass + + @abstractmethod + def _dump_ingress_replicas_for_testing(self, route: str) -> Set: + """Get replicas for a route (for testing).""" + pass + + @ray.remote(num_cpus=0) -class ProxyActor: +class ProxyActor(ProxyActorInterface): def __init__( self, http_options: HTTPOptions, @@ -1028,12 +1133,15 @@ def __init__( logging_config: LoggingConfig, long_poll_client: Optional[LongPollClient] = None, ): # noqa: F821 - self._node_id = node_id - self._node_ip_address = node_ip_address - self._http_options = configure_http_middlewares(http_options) + super().__init__( + node_id=node_id, + node_ip_address=node_ip_address, + logging_config=logging_config, + ) + self._grpc_options = grpc_options + self._http_options = configure_http_middlewares(http_options) grpc_enabled = is_grpc_enabled(self._grpc_options) - event_loop = get_or_create_event_loop() self.long_poll_client = long_poll_client or LongPollClient( ray.get_actor(SERVE_CONTROLLER_NAME, namespace=SERVE_NAMESPACE), From 237f7924e21b85b4a6b7ded3d5375f3cb9e98ed4 Mon Sep 17 00:00:00 2001 From: Abrar Sheikh Date: Mon, 15 Sep 2025 10:23:54 -0700 Subject: [PATCH 622/634] stop ray instance in serve test logging (#56480) Should fix the windows test, i am 90% sure. I could not manually test this because I am unsuccessfully in running test_logging on windows using this runbook https://www.notion.so/anyscale-hq/How-to-debug-Windows-tests-20e027c809cb803b92c8c796266b7852?source=copy_link. I am sure there is a way but not investing more time into this. --------- Signed-off-by: abrar --- python/ray/serve/tests/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/serve/tests/test_logging.py b/python/ray/serve/tests/test_logging.py index 096bbe610c96..c1c1b612f338 100644 --- a/python/ray/serve/tests/test_logging.py +++ b/python/ray/serve/tests/test_logging.py @@ -1369,7 +1369,7 @@ def test_configure_default_serve_logger_with_stderr_redirect( ], indirect=True, ) -def test_request_id_uniqueness_with_buffering(ray_instance): +def test_request_id_uniqueness_with_buffering(serve_and_ray_shutdown, ray_instance): """Test request IDs are unique when buffering is enabled.""" logger = logging.getLogger("ray.serve") From a78952eea6a52b9a6c758850fe8a0e109273d8e8 Mon Sep 17 00:00:00 2001 From: Dhyey Shah Date: Mon, 15 Sep 2025 10:47:49 -0700 Subject: [PATCH 623/634] [core][ci] Don't build cpp api in ci test container (#56517) The cpp api is only tested on`:ray: core: cpp worker tests` , but we still build it on most ci steps. Ex. this commit was only broken for the cpp api and nothing else, but almost every single ci step broke. https://buildkite.com/ray-project/premerge/builds/48767 This sets `RAY_DISABLE_EXTRA_CPP` in the test containers so the cpp api doesn't need to get rebuilt on every test step. This should make ci a bit faster when making core cpp changes that cause the cpp api to rebuild. It'll still get built when we build the wheels so any compilation errors for the cpp api will get verified there. Signed-off-by: dayshah --- ci/ray_ci/tests.env.Dockerfile | 1 + ci/ray_ci/windows/tests.env.Dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/ray_ci/tests.env.Dockerfile b/ci/ray_ci/tests.env.Dockerfile index 7ae17986e618..fb009afa59d4 100644 --- a/ci/ray_ci/tests.env.Dockerfile +++ b/ci/ray_ci/tests.env.Dockerfile @@ -9,6 +9,7 @@ ARG RAY_INSTALL_MASK= ENV CC=clang ENV CXX=clang++-12 +ENV RAY_DISABLE_EXTRA_CPP=1 RUN mkdir /rayci WORKDIR /rayci diff --git a/ci/ray_ci/windows/tests.env.Dockerfile b/ci/ray_ci/windows/tests.env.Dockerfile index 0e0cd9eea4ab..cce117b1fe7e 100644 --- a/ci/ray_ci/windows/tests.env.Dockerfile +++ b/ci/ray_ci/windows/tests.env.Dockerfile @@ -12,6 +12,7 @@ ENV PYTHON=3.9 ENV RAY_USE_RANDOM_PORTS=1 ENV RAY_DEFAULT_BUILD=1 ENV RAY_INSTALL_JAVA=0 +ENV RAY_DISABLE_EXTRA_CPP=1 ENV RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER=1 ENV LC_ALL=en_US.UTF-8 ENV LANG=en_US.UTF-8 From fba6fe7705a8d8a29a6b2b234466f2667f8a1aee Mon Sep 17 00:00:00 2001 From: "Owen Lin (You-Cheng Lin)" <106612301+owenowenisme@users.noreply.github.com> Date: Tue, 16 Sep 2025 02:54:21 +0900 Subject: [PATCH 624/634] [Data] Remove redundant check for initial size of actor pool (#56440) ## Why are these changes needed? The check is redundant here, since the `initial_size` can't be smaller than `min_size` (which must be bigger that 1) ## Related issue number https://github.com/ray-project/ray/pull/56370#discussion_r2337538808 ## Checks - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [ ] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: You-Cheng Lin (Owen) --- python/ray/data/_internal/compute.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/ray/data/_internal/compute.py b/python/ray/data/_internal/compute.py index 6188dab3f899..0644023bb58a 100644 --- a/python/ray/data/_internal/compute.py +++ b/python/ray/data/_internal/compute.py @@ -125,8 +125,6 @@ def __init__( # Validate and set initial_size if initial_size is not None: - if initial_size < 1: - raise ValueError("initial_size must be >= 1", initial_size) if initial_size < self.min_size: raise ValueError( f"initial_size ({initial_size}) must be >= min_size ({self.min_size})" From 72eb7a629911af3e8ca1c49728f9e479dc1261ed Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 15 Sep 2025 13:11:39 -0500 Subject: [PATCH 625/634] [core] Move `gcs_client` out of `gcs` directory (#56515) Making `gcs` contain only the GCS component's files. --------- Signed-off-by: Edward Oakes --- BUILD.bazel | 4 +-- cpp/BUILD.bazel | 2 +- cpp/src/ray/util/process_helper.h | 2 +- python/ray/includes/common.pxd | 10 +++---- python/ray/includes/gcs_client.pxi | 2 +- python/ray/includes/global_state_accessor.pxd | 2 +- src/mock/ray/core_worker/core_worker.h | 2 +- src/mock/ray/{gcs => }/gcs_client/accessor.h | 2 +- .../ray/{gcs => }/gcs_client/gcs_client.h | 4 +-- src/ray/core_worker/BUILD.bazel | 10 +++---- src/ray/core_worker/actor_creator.h | 2 +- src/ray/core_worker/actor_manager.h | 2 +- src/ray/core_worker/core_worker.cc | 2 +- src/ray/core_worker/core_worker.h | 2 +- src/ray/core_worker/core_worker_options.h | 2 +- src/ray/core_worker/core_worker_process.cc | 2 +- src/ray/core_worker/lib/java/BUILD.bazel | 2 +- .../io_ray_runtime_gcs_GlobalStateAccessor.cc | 2 +- ...io_ray_runtime_object_NativeObjectStore.cc | 2 +- src/ray/core_worker/task_event_buffer.h | 2 +- src/ray/core_worker/task_manager.h | 2 +- .../tests/direct_actor_transport_test.cc | 2 +- src/ray/core_worker/tests/BUILD.bazel | 12 ++++----- .../core_worker/tests/actor_creator_test.cc | 7 +++-- .../core_worker/tests/actor_manager_test.cc | 4 +-- src/ray/core_worker/tests/core_worker_test.cc | 2 +- .../task_event_buffer_export_event_test.cc | 2 +- .../tests/task_event_buffer_test.cc | 2 +- .../core_worker/tests/task_manager_test.cc | 2 +- .../gcs_server/tests/gcs_server_rpc_test.cc | 2 +- src/ray/{gcs => }/gcs_client/BUILD.bazel | 26 ++++++++++++++++--- src/ray/{gcs => }/gcs_client/accessor.cc | 4 +-- src/ray/{gcs => }/gcs_client/accessor.h | 0 src/ray/{gcs => }/gcs_client/gcs_client.cc | 4 +-- src/ray/{gcs => }/gcs_client/gcs_client.h | 4 +-- .../gcs_client/global_state_accessor.cc | 2 +- .../gcs_client/global_state_accessor.h | 2 +- .../{gcs => }/gcs_client/python_callbacks.h | 0 .../rpc_client.h} | 0 .../{gcs => }/gcs_client/tests/BUILD.bazel | 10 +++---- .../gcs_client/tests/accessor_test.cc | 2 +- .../tests/gcs_client_reconnection_test.cc | 6 ++--- .../gcs_client/tests/gcs_client_test.cc | 6 ++--- .../tests/global_state_accessor_test.cc | 4 +-- src/ray/object_manager/BUILD.bazel | 4 +-- src/ray/object_manager/object_directory.h | 2 +- .../ownership_object_directory.h | 2 +- .../tests/object_manager_test.cc | 2 +- .../tests/ownership_object_directory_test.cc | 4 +-- src/ray/pubsub/BUILD.bazel | 2 +- src/ray/pubsub/python_gcs_subscriber.cc | 2 +- src/ray/raylet/BUILD.bazel | 10 +++---- src/ray/raylet/local_object_manager.h | 2 +- src/ray/raylet/main.cc | 2 +- src/ray/raylet/scheduling/tests/BUILD.bazel | 2 +- .../tests/cluster_lease_manager_test.cc | 2 +- .../tests/cluster_resource_scheduler_test.cc | 2 +- src/ray/raylet/tests/BUILD.bazel | 2 +- .../raylet/tests/local_lease_manager_test.cc | 2 +- .../raylet/tests/local_object_manager_test.cc | 4 +-- src/ray/raylet/tests/node_manager_test.cc | 2 +- .../placement_group_resource_manager_test.cc | 2 +- src/ray/raylet/tests/worker_pool_test.cc | 2 +- src/ray/raylet/worker_pool.h | 2 +- src/ray/rpc/BUILD.bazel | 21 ++------------- src/ray/rpc/raylet/raylet_client_pool.h | 2 +- src/ray/rpc/raylet/tests/BUILD.bazel | 2 +- src/ray/rpc/worker/core_worker_client_pool.h | 2 +- 68 files changed, 125 insertions(+), 123 deletions(-) rename src/mock/ray/{gcs => }/gcs_client/accessor.h (99%) rename src/mock/ray/{gcs => }/gcs_client/gcs_client.h (96%) rename src/ray/{gcs => }/gcs_client/BUILD.bazel (62%) rename src/ray/{gcs => }/gcs_client/accessor.cc (99%) rename src/ray/{gcs => }/gcs_client/accessor.h (100%) rename src/ray/{gcs => }/gcs_client/gcs_client.cc (99%) rename src/ray/{gcs => }/gcs_client/gcs_client.h (99%) rename src/ray/{gcs => }/gcs_client/global_state_accessor.cc (99%) rename src/ray/{gcs => }/gcs_client/global_state_accessor.h (99%) rename src/ray/{gcs => }/gcs_client/python_callbacks.h (100%) rename src/ray/{rpc/gcs/gcs_rpc_client.h => gcs_client/rpc_client.h} (100%) rename src/ray/{gcs => }/gcs_client/tests/BUILD.bazel (88%) rename src/ray/{gcs => }/gcs_client/tests/accessor_test.cc (98%) rename src/ray/{gcs => }/gcs_client/tests/gcs_client_reconnection_test.cc (99%) rename src/ray/{gcs => }/gcs_client/tests/gcs_client_test.cc (99%) rename src/ray/{gcs => }/gcs_client/tests/global_state_accessor_test.cc (99%) diff --git a/BUILD.bazel b/BUILD.bazel index c30d1b1f43ae..5337472727fe 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -246,10 +246,10 @@ pyx_library( "//:src/ray/ray_exported_symbols.lds", "//:src/ray/ray_version_script.lds", "//src/ray/core_worker:core_worker_lib", - "//src/ray/gcs/gcs_client:gcs_python_callbacks", - "//src/ray/gcs/gcs_client:global_state_accessor_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", "//src/ray/gcs/store_client:redis_store_client", + "//src/ray/gcs_client:gcs_python_callbacks", + "//src/ray/gcs_client:global_state_accessor_lib", "//src/ray/protobuf:serialization_cc_proto", "//src/ray/pubsub:python_gcs_subscriber", "//src/ray/thirdparty/setproctitle", diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index 9ab6348cac9b..9174d0683dbd 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -66,7 +66,7 @@ cc_library( "//src/ray/common:ray_config", "//src/ray/common:task_common", "//src/ray/core_worker:core_worker_lib", - "//src/ray/gcs/gcs_client:global_state_accessor_lib", + "//src/ray/gcs_client:global_state_accessor_lib", "//src/ray/util:cmd_line_utils", "//src/ray/util:network_util", "//src/ray/util:process", diff --git a/cpp/src/ray/util/process_helper.h b/cpp/src/ray/util/process_helper.h index 084bbeda93a7..26dfc6ca108c 100644 --- a/cpp/src/ray/util/process_helper.h +++ b/cpp/src/ray/util/process_helper.h @@ -17,7 +17,7 @@ #include "../config_internal.h" #include "ray/core_worker/core_worker.h" -#include "ray/gcs/gcs_client/global_state_accessor.h" +#include "ray/gcs_client/global_state_accessor.h" #include "util.h" namespace ray { diff --git a/python/ray/includes/common.pxd b/python/ray/includes/common.pxd index 6493a100c47a..ec40d7c43f8c 100644 --- a/python/ray/includes/common.pxd +++ b/python/ray/includes/common.pxd @@ -391,7 +391,7 @@ cdef extern from "ray/core_worker/common.h" nogil: const CNodeID &GetSpilledNodeID() const const c_bool GetDidSpill() const -cdef extern from "ray/gcs/gcs_client/python_callbacks.h" namespace "ray::gcs": +cdef extern from "ray/gcs_client/python_callbacks.h" namespace "ray::gcs": cdef cppclass MultiItemPyCallback[T]: MultiItemPyCallback( object (*)(CRayStatus, c_vector[T]) nogil, @@ -410,7 +410,7 @@ cdef extern from "ray/gcs/gcs_client/python_callbacks.h" namespace "ray::gcs": void (object, object) nogil, object) nogil -cdef extern from "ray/gcs/gcs_client/accessor.h" nogil: +cdef extern from "ray/gcs_client/accessor.h" nogil: cdef cppclass CActorInfoAccessor "ray::gcs::ActorInfoAccessor": void AsyncGetAllByFilter( const optional[CActorID] &actor_id, @@ -616,7 +616,7 @@ cdef extern from "ray/gcs/gcs_client/accessor.h" nogil: ) -cdef extern from "ray/gcs/gcs_client/gcs_client.h" nogil: +cdef extern from "ray/gcs_client/gcs_client.h" nogil: cdef enum CGrpcStatusCode "grpc::StatusCode": UNAVAILABLE "grpc::StatusCode::UNAVAILABLE", UNKNOWN "grpc::StatusCode::UNKNOWN", @@ -646,7 +646,7 @@ cdef extern from "ray/gcs/gcs_client/gcs_client.h" nogil: cdef CRayStatus ConnectOnSingletonIoContext(CGcsClient &gcs_client, int timeout_ms) -cdef extern from "ray/gcs/gcs_client/gcs_client.h" namespace "ray::gcs" nogil: +cdef extern from "ray/gcs_client/gcs_client.h" namespace "ray::gcs" nogil: unordered_map[c_string, double] PythonGetResourcesTotal( const CGcsNodeInfo& node_info) @@ -672,7 +672,7 @@ cdef extern from "ray/pubsub/python_gcs_subscriber.h" nogil: cdef extern from "ray/pubsub/python_gcs_subscriber.h" namespace "ray::pubsub" nogil: c_vector[c_string] PythonGetLogBatchLines(CLogBatch log_batch) -cdef extern from "ray/gcs/gcs_client/gcs_client.h" namespace "ray::gcs" nogil: +cdef extern from "ray/gcs_client/gcs_client.h" namespace "ray::gcs" nogil: unordered_map[c_string, c_string] PythonGetNodeLabels( const CGcsNodeInfo& node_info) diff --git a/python/ray/includes/gcs_client.pxi b/python/ray/includes/gcs_client.pxi index 177bf48fbba3..bc47457b179b 100644 --- a/python/ray/includes/gcs_client.pxi +++ b/python/ray/includes/gcs_client.pxi @@ -14,7 +14,7 @@ Binding of C++ ray::gcs::GcsClient. # # We need to best-effort import everything we need. # -# For how async API are implemented, see src/ray/gcs/gcs_client/python_callbacks.h +# For how async API are implemented, see src/ray/gcs_client/python_callbacks.h from asyncio import Future from typing import List, Sequence from libcpp.utility cimport move diff --git a/python/ray/includes/global_state_accessor.pxd b/python/ray/includes/global_state_accessor.pxd index 38a6703cf25f..1eb54aeb2025 100644 --- a/python/ray/includes/global_state_accessor.pxd +++ b/python/ray/includes/global_state_accessor.pxd @@ -24,7 +24,7 @@ from ray.includes.optional cimport ( optional ) -cdef extern from "ray/gcs/gcs_client/global_state_accessor.h" nogil: +cdef extern from "ray/gcs_client/global_state_accessor.h" nogil: cdef cppclass CGlobalStateAccessor "ray::gcs::GlobalStateAccessor": CGlobalStateAccessor(const CGcsClientOptions&) c_bool Connect() diff --git a/src/mock/ray/core_worker/core_worker.h b/src/mock/ray/core_worker/core_worker.h index 905ecceddec9..403d97de0db0 100644 --- a/src/mock/ray/core_worker/core_worker.h +++ b/src/mock/ray/core_worker/core_worker.h @@ -13,7 +13,7 @@ // limitations under the License. #pragma once #include "gmock/gmock.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" namespace ray::core { diff --git a/src/mock/ray/gcs/gcs_client/accessor.h b/src/mock/ray/gcs_client/accessor.h similarity index 99% rename from src/mock/ray/gcs/gcs_client/accessor.h rename to src/mock/ray/gcs_client/accessor.h index 344729b2065b..ce66405ed34a 100644 --- a/src/mock/ray/gcs/gcs_client/accessor.h +++ b/src/mock/ray/gcs_client/accessor.h @@ -13,7 +13,7 @@ // limitations under the License. #pragma once #include "gmock/gmock.h" -#include "ray/gcs/gcs_client/accessor.h" +#include "ray/gcs_client/accessor.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_client/gcs_client.h b/src/mock/ray/gcs_client/gcs_client.h similarity index 96% rename from src/mock/ray/gcs/gcs_client/gcs_client.h rename to src/mock/ray/gcs_client/gcs_client.h index 8c4d15189033..1ad7d85b3ffc 100644 --- a/src/mock/ray/gcs/gcs_client/gcs_client.h +++ b/src/mock/ray/gcs_client/gcs_client.h @@ -14,8 +14,8 @@ #pragma once -#include "mock/ray/gcs/gcs_client/accessor.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/accessor.h" +#include "ray/gcs_client/gcs_client.h" namespace ray { namespace gcs { diff --git a/src/ray/core_worker/BUILD.bazel b/src/ray/core_worker/BUILD.bazel index f62d6bed3ace..429655af4fe1 100644 --- a/src/ray/core_worker/BUILD.bazel +++ b/src/ray/core_worker/BUILD.bazel @@ -38,7 +38,7 @@ ray_cc_library( "//src/ray/common/cgroup:constants", "//src/ray/core_worker/task_execution:task_receiver", "//src/ray/core_worker/task_submission:normal_task_submitter", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/ipc:raylet_ipc_client", "//src/ray/protobuf:pubsub_cc_proto", "//src/ray/pubsub:publisher", @@ -105,7 +105,7 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:ray_object", "//src/ray/common:status", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/util:process", ], ) @@ -159,7 +159,7 @@ ray_cc_library( hdrs = ["actor_creator.h"], visibility = [":__subpackages__"], deps = [ - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", ], ) @@ -186,7 +186,7 @@ ray_cc_library( "//src/ray/common:protobuf_utils", "//src/ray/common:task_common", "//src/ray/core_worker/task_submission:actor_task_submitter", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/protobuf:core_worker_cc_proto", "@com_google_absl//absl/container:flat_hash_map", "@com_google_googletest//:gtest_prod", @@ -236,7 +236,7 @@ ray_cc_library( "//src/ray/common:id", "//src/ray/common:protobuf_utils", "//src/ray/common:task_common", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/protobuf:export_task_event_cc_proto", "//src/ray/protobuf:gcs_cc_proto", "//src/ray/rpc:event_aggregator_client", diff --git a/src/ray/core_worker/actor_creator.h b/src/ray/core_worker/actor_creator.h index 8673dc57154e..e34751f1f116 100644 --- a/src/ray/core_worker/actor_creator.h +++ b/src/ray/core_worker/actor_creator.h @@ -18,7 +18,7 @@ #include #include -#include "ray/gcs/gcs_client/accessor.h" +#include "ray/gcs_client/accessor.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/actor_manager.h b/src/ray/core_worker/actor_manager.h index 3d91ca155dbc..ee9eaf798563 100644 --- a/src/ray/core_worker/actor_manager.h +++ b/src/ray/core_worker/actor_manager.h @@ -26,7 +26,7 @@ #include "ray/core_worker/actor_handle.h" #include "ray/core_worker/reference_count.h" #include "ray/core_worker/task_submission/actor_task_submitter.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/core_worker.cc b/src/ray/core_worker/core_worker.cc index 62ef51ae0bdc..c9b93f4e0d22 100644 --- a/src/ray/core_worker/core_worker.cc +++ b/src/ray/core_worker/core_worker.cc @@ -42,7 +42,7 @@ #include "ray/common/ray_config.h" #include "ray/common/runtime_env_common.h" #include "ray/common/task/task_util.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/rpc/event_aggregator_client.h" #include "ray/util/container_util.h" #include "ray/util/event.h" diff --git a/src/ray/core_worker/core_worker.h b/src/ray/core_worker/core_worker.h index 4091ec28609b..1f1ccda1d196 100644 --- a/src/ray/core_worker/core_worker.h +++ b/src/ray/core_worker/core_worker.h @@ -48,7 +48,7 @@ #include "ray/core_worker/task_event_buffer.h" #include "ray/core_worker/task_execution/task_receiver.h" #include "ray/core_worker/task_submission/normal_task_submitter.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/ipc/raylet_ipc_client_interface.h" #include "ray/pubsub/publisher.h" #include "ray/pubsub/subscriber.h" diff --git a/src/ray/core_worker/core_worker_options.h b/src/ray/core_worker/core_worker_options.h index 3f9b72f48b7d..fc59a55495e5 100644 --- a/src/ray/core_worker/core_worker_options.h +++ b/src/ray/core_worker/core_worker_options.h @@ -26,7 +26,7 @@ #include "ray/common/status.h" #include "ray/common/task/task_common.h" #include "ray/core_worker/common.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/util/process.h" namespace ray { diff --git a/src/ray/core_worker/core_worker_process.cc b/src/ray/core_worker/core_worker_process.cc index ab7db114366c..9513bbd6af05 100644 --- a/src/ray/core_worker/core_worker_process.cc +++ b/src/ray/core_worker/core_worker_process.cc @@ -33,7 +33,7 @@ #include "ray/common/task/task_util.h" #include "ray/core_worker/core_worker.h" #include "ray/core_worker/core_worker_rpc_proxy.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/ipc/raylet_ipc_client.h" #include "ray/object_manager/plasma/client.h" #include "ray/rpc/raylet/raylet_client.h" diff --git a/src/ray/core_worker/lib/java/BUILD.bazel b/src/ray/core_worker/lib/java/BUILD.bazel index 4e35f82a0490..470e5a775838 100644 --- a/src/ray/core_worker/lib/java/BUILD.bazel +++ b/src/ray/core_worker/lib/java/BUILD.bazel @@ -24,7 +24,7 @@ ray_cc_binary( "//:src/ray/ray_exported_symbols.lds", "//:src/ray/ray_version_script.lds", "//src/ray/core_worker:core_worker_lib", - "//src/ray/gcs/gcs_client:global_state_accessor_lib", + "//src/ray/gcs_client:global_state_accessor_lib", "//src/ray/stats:stats_lib", "//src/ray/util:time", "@bazel_tools//tools/jdk:jni", diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_gcs_GlobalStateAccessor.cc b/src/ray/core_worker/lib/java/io_ray_runtime_gcs_GlobalStateAccessor.cc index 258263f176b1..d1b5f56c4699 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_gcs_GlobalStateAccessor.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_gcs_GlobalStateAccessor.cc @@ -22,7 +22,7 @@ #include "jni_utils.h" // NOLINT(build/include_subdir) #include "ray/common/ray_config.h" #include "ray/core_worker/common.h" -#include "ray/gcs/gcs_client/global_state_accessor.h" +#include "ray/gcs_client/global_state_accessor.h" #ifdef __cplusplus extern "C" { diff --git a/src/ray/core_worker/lib/java/io_ray_runtime_object_NativeObjectStore.cc b/src/ray/core_worker/lib/java/io_ray_runtime_object_NativeObjectStore.cc index 2c0f5548ec0a..bb942786ab1b 100644 --- a/src/ray/core_worker/lib/java/io_ray_runtime_object_NativeObjectStore.cc +++ b/src/ray/core_worker/lib/java/io_ray_runtime_object_NativeObjectStore.cc @@ -25,7 +25,7 @@ #include "ray/common/id.h" #include "ray/core_worker/common.h" #include "ray/core_worker/core_worker.h" -#include "ray/gcs/gcs_client/global_state_accessor.h" +#include "ray/gcs_client/global_state_accessor.h" Status PutSerializedObject(JNIEnv *env, jobject obj, diff --git a/src/ray/core_worker/task_event_buffer.h b/src/ray/core_worker/task_event_buffer.h index 39030c3a7c0f..2e9ce16fc4dc 100644 --- a/src/ray/core_worker/task_event_buffer.h +++ b/src/ray/core_worker/task_event_buffer.h @@ -29,7 +29,7 @@ #include "ray/common/id.h" #include "ray/common/protobuf_utils.h" #include "ray/common/task/task_spec.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/rpc/event_aggregator_client.h" #include "ray/util/counter_map.h" #include "ray/util/event.h" diff --git a/src/ray/core_worker/task_manager.h b/src/ray/core_worker/task_manager.h index 31f010bca283..43f6fce25ea9 100644 --- a/src/ray/core_worker/task_manager.h +++ b/src/ray/core_worker/task_manager.h @@ -30,7 +30,7 @@ #include "ray/core_worker/store_provider/memory_store/memory_store.h" #include "ray/core_worker/task_event_buffer.h" #include "ray/core_worker/task_manager_interface.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/observability/metric_interface.h" #include "ray/stats/metric_defs.h" #include "ray/util/counter_map.h" diff --git a/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc b/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc index a2e58b392145..a4d50e583a61 100644 --- a/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc +++ b/src/ray/core_worker/task_submission/tests/direct_actor_transport_test.cc @@ -19,7 +19,7 @@ #include "mock/ray/core_worker/memory_store.h" #include "mock/ray/core_worker/reference_count.h" #include "mock/ray/core_worker/task_manager_interface.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "ray/core_worker/actor_creator.h" #include "ray/core_worker/task_submission/actor_task_submitter.h" diff --git a/src/ray/core_worker/tests/BUILD.bazel b/src/ray/core_worker/tests/BUILD.bazel index 6e8df62c414f..c4366aa3297c 100644 --- a/src/ray/core_worker/tests/BUILD.bazel +++ b/src/ray/core_worker/tests/BUILD.bazel @@ -96,7 +96,7 @@ ray_cc_test( "//src/ray/core_worker:reference_count", "//src/ray/core_worker:task_event_buffer", "//src/ray/core_worker:task_manager", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/observability:fake_metric", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", @@ -113,7 +113,7 @@ ray_cc_test( "//src/ray/common:task_common", "//src/ray/common:test_utils", "//src/ray/core_worker:task_event_buffer", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/util:event", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/synchronization", @@ -135,7 +135,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:test_utils", "//src/ray/core_worker:task_event_buffer", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/util:event", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/types:optional", @@ -153,7 +153,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:test_utils", "//src/ray/core_worker:actor_creator", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/util:path_utils", "//src/ray/util:raii", "@com_google_googletest//:gtest", @@ -171,7 +171,7 @@ ray_cc_test( "//src/ray/common:test_utils", "//src/ray/core_worker:common", "//src/ray/core_worker:generator_waiter", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], @@ -186,7 +186,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:test_utils", "//src/ray/core_worker:actor_manager", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/core_worker/tests/actor_creator_test.cc b/src/ray/core_worker/tests/actor_creator_test.cc index fc9996cd07cf..10d3b3574c3e 100644 --- a/src/ray/core_worker/tests/actor_creator_test.cc +++ b/src/ray/core_worker/tests/actor_creator_test.cc @@ -12,17 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -// clang-format off +#include "ray/core_worker/actor_creator.h" + #include #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "ray/core_worker/actor_creator.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "ray/common/test_utils.h" #include "ray/util/path_utils.h" #include "ray/util/raii.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" -// clang-format on namespace ray { namespace core { diff --git a/src/ray/core_worker/tests/actor_manager_test.cc b/src/ray/core_worker/tests/actor_manager_test.cc index 6acc46473a0f..1e2c644064ff 100644 --- a/src/ray/core_worker/tests/actor_manager_test.cc +++ b/src/ray/core_worker/tests/actor_manager_test.cc @@ -22,8 +22,8 @@ #include "gtest/gtest.h" #include "mock/ray/core_worker/reference_count.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_client/accessor.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/accessor.h" +#include "ray/gcs_client/gcs_client.h" namespace ray { namespace core { diff --git a/src/ray/core_worker/tests/core_worker_test.cc b/src/ray/core_worker/tests/core_worker_test.cc index d0837518bfa9..a972d19a47a5 100644 --- a/src/ray/core_worker/tests/core_worker_test.cc +++ b/src/ray/core_worker/tests/core_worker_test.cc @@ -30,7 +30,7 @@ #include "fakes/ray/pubsub/publisher.h" #include "fakes/ray/pubsub/subscriber.h" #include "fakes/ray/rpc/raylet/raylet_client.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "mock/ray/object_manager/plasma/client.h" #include "ray/common/buffer.h" #include "ray/common/ray_config.h" diff --git a/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc b/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc index 5162892ae292..cf2e6e7203f2 100644 --- a/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc +++ b/src/ray/core_worker/tests/task_event_buffer_export_event_test.cc @@ -24,7 +24,7 @@ #include "absl/types/optional.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "ray/common/test_utils.h" #include "ray/core_worker/task_event_buffer.h" #include "ray/util/event.h" diff --git a/src/ray/core_worker/tests/task_event_buffer_test.cc b/src/ray/core_worker/tests/task_event_buffer_test.cc index 6169b36b0176..58edefd13604 100644 --- a/src/ray/core_worker/tests/task_event_buffer_test.cc +++ b/src/ray/core_worker/tests/task_event_buffer_test.cc @@ -31,7 +31,7 @@ #include "absl/types/optional.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" #include "ray/common/test_utils.h" diff --git a/src/ray/core_worker/tests/task_manager_test.cc b/src/ray/core_worker/tests/task_manager_test.cc index 63215f0d1eb6..b68c1acbe6c6 100644 --- a/src/ray/core_worker/tests/task_manager_test.cc +++ b/src/ray/core_worker/tests/task_manager_test.cc @@ -23,7 +23,7 @@ #include "fakes/ray/pubsub/subscriber.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "mock/ray/pubsub/publisher.h" #include "ray/common/task/task_spec.h" #include "ray/common/task/task_util.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc b/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc index f9b208f0e33f..7cdc021f49a0 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc +++ b/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc @@ -21,7 +21,7 @@ #include "ray/common/ray_config.h" #include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/rpc/gcs/gcs_rpc_client.h" +#include "ray/gcs_client/rpc_client.h" namespace ray { diff --git a/src/ray/gcs/gcs_client/BUILD.bazel b/src/ray/gcs_client/BUILD.bazel similarity index 62% rename from src/ray/gcs/gcs_client/BUILD.bazel rename to src/ray/gcs_client/BUILD.bazel index ce1b1e861ff9..4b416ebb7026 100644 --- a/src/ray/gcs/gcs_client/BUILD.bazel +++ b/src/ray/gcs_client/BUILD.bazel @@ -1,7 +1,7 @@ load("//bazel:ray.bzl", "ray_cc_library") ray_cc_library( - name = "gcs_client_lib", + name = "gcs_client", srcs = [ "accessor.cc", "gcs_client.cc", @@ -11,6 +11,7 @@ ray_cc_library( "gcs_client.h", ], deps = [ + ":rpc_client", "//src/ray/common:asio", "//src/ray/common:id", "//src/ray/common:placement_group", @@ -19,7 +20,6 @@ ray_cc_library( "//src/ray/protobuf:usage_cc_proto", "//src/ray/pubsub:gcs_subscriber", "//src/ray/pubsub:subscriber", - "//src/ray/rpc:gcs_client", "//src/ray/util:container_util", "//src/ray/util:network_util", "//src/ray/util:sequencer", @@ -31,7 +31,7 @@ ray_cc_library( srcs = ["global_state_accessor.cc"], hdrs = ["global_state_accessor.h"], deps = [ - ":gcs_client_lib", + ":gcs_client", "//src/ray/util:time", ], ) @@ -42,3 +42,23 @@ ray_cc_library( "python_callbacks.h", ], ) + +ray_cc_library( + name = "rpc_client", + hdrs = [ + "rpc_client.h", + ], + visibility = [ + ":__pkg__", + "//src/ray/pubsub:__pkg__", + ], + deps = [ + "//src/ray/common:ray_config", + "//src/ray/protobuf:autoscaler_cc_grpc", + "//src/ray/protobuf:gcs_service_cc_grpc", + "//src/ray/rpc:client_call", + "//src/ray/rpc:retryable_grpc_client", + "//src/ray/util:network_util", + "@com_google_absl//absl/container:btree", + ], +) diff --git a/src/ray/gcs/gcs_client/accessor.cc b/src/ray/gcs_client/accessor.cc similarity index 99% rename from src/ray/gcs/gcs_client/accessor.cc rename to src/ray/gcs_client/accessor.cc index e2a3fbf39387..e035c2002e0c 100644 --- a/src/ray/gcs/gcs_client/accessor.cc +++ b/src/ray/gcs_client/accessor.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_client/accessor.h" +#include "ray/gcs_client/accessor.h" #include #include @@ -21,7 +21,7 @@ #include #include -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/util/container_util.h" namespace ray { diff --git a/src/ray/gcs/gcs_client/accessor.h b/src/ray/gcs_client/accessor.h similarity index 100% rename from src/ray/gcs/gcs_client/accessor.h rename to src/ray/gcs_client/accessor.h diff --git a/src/ray/gcs/gcs_client/gcs_client.cc b/src/ray/gcs_client/gcs_client.cc similarity index 99% rename from src/ray/gcs/gcs_client/gcs_client.cc rename to src/ray/gcs_client/gcs_client.cc index f3974a75e5d9..7d1d9e9bf6f3 100644 --- a/src/ray/gcs/gcs_client/gcs_client.cc +++ b/src/ray/gcs_client/gcs_client.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include #include @@ -24,7 +24,7 @@ #include "ray/common/asio/asio_util.h" #include "ray/common/ray_config.h" -#include "ray/gcs/gcs_client/accessor.h" +#include "ray/gcs_client/accessor.h" #include "ray/pubsub/subscriber.h" #include "ray/util/network_util.h" diff --git a/src/ray/gcs/gcs_client/gcs_client.h b/src/ray/gcs_client/gcs_client.h similarity index 99% rename from src/ray/gcs/gcs_client/gcs_client.h rename to src/ray/gcs_client/gcs_client.h index 913f41415908..b13eb5292e5d 100644 --- a/src/ray/gcs/gcs_client/gcs_client.h +++ b/src/ray/gcs_client/gcs_client.h @@ -28,9 +28,9 @@ #include "ray/common/asio/periodical_runner.h" #include "ray/common/id.h" #include "ray/common/status.h" -#include "ray/gcs/gcs_client/accessor.h" +#include "ray/gcs_client/accessor.h" +#include "ray/gcs_client/rpc_client.h" #include "ray/pubsub/gcs_subscriber.h" -#include "ray/rpc/gcs/gcs_rpc_client.h" #include "ray/util/logging.h" #include "ray/util/network_util.h" #include "src/ray/protobuf/autoscaler.grpc.pb.h" diff --git a/src/ray/gcs/gcs_client/global_state_accessor.cc b/src/ray/gcs_client/global_state_accessor.cc similarity index 99% rename from src/ray/gcs/gcs_client/global_state_accessor.cc rename to src/ray/gcs_client/global_state_accessor.cc index 89b3582150c1..4e80d9496e5e 100644 --- a/src/ray/gcs/gcs_client/global_state_accessor.cc +++ b/src/ray/gcs_client/global_state_accessor.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_client/global_state_accessor.h" +#include "ray/gcs_client/global_state_accessor.h" #include #include diff --git a/src/ray/gcs/gcs_client/global_state_accessor.h b/src/ray/gcs_client/global_state_accessor.h similarity index 99% rename from src/ray/gcs/gcs_client/global_state_accessor.h rename to src/ray/gcs_client/global_state_accessor.h index 7c2266a53b15..c525ab7c5d2c 100644 --- a/src/ray/gcs/gcs_client/global_state_accessor.h +++ b/src/ray/gcs_client/global_state_accessor.h @@ -23,7 +23,7 @@ #include "absl/base/thread_annotations.h" #include "absl/synchronization/mutex.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_client/python_callbacks.h b/src/ray/gcs_client/python_callbacks.h similarity index 100% rename from src/ray/gcs/gcs_client/python_callbacks.h rename to src/ray/gcs_client/python_callbacks.h diff --git a/src/ray/rpc/gcs/gcs_rpc_client.h b/src/ray/gcs_client/rpc_client.h similarity index 100% rename from src/ray/rpc/gcs/gcs_rpc_client.h rename to src/ray/gcs_client/rpc_client.h diff --git a/src/ray/gcs/gcs_client/tests/BUILD.bazel b/src/ray/gcs_client/tests/BUILD.bazel similarity index 88% rename from src/ray/gcs/gcs_client/tests/BUILD.bazel rename to src/ray/gcs_client/tests/BUILD.bazel index d94a8b143f23..c0b12afa33b5 100644 --- a/src/ray/gcs/gcs_client/tests/BUILD.bazel +++ b/src/ray/gcs_client/tests/BUILD.bazel @@ -9,7 +9,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "@com_google_googletest//:gtest_main", ], ) @@ -31,9 +31,9 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_client:gcs_client_lib", - "//src/ray/gcs/gcs_client:global_state_accessor_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs_client", + "//src/ray/gcs_client:global_state_accessor_lib", "//src/ray/util:path_utils", "//src/ray/util:raii", "@com_google_googletest//:gtest_main", @@ -61,8 +61,8 @@ ray_cc_test( ], deps = [ "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs_client", "//src/ray/util:network_util", "//src/ray/util:raii", "//src/ray/util:time", @@ -89,8 +89,8 @@ ray_cc_test( ], deps = [ "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_client:gcs_client_lib", "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs_client", "//src/ray/util:network_util", "//src/ray/util:path_utils", "//src/ray/util:raii", diff --git a/src/ray/gcs/gcs_client/tests/accessor_test.cc b/src/ray/gcs_client/tests/accessor_test.cc similarity index 98% rename from src/ray/gcs/gcs_client/tests/accessor_test.cc rename to src/ray/gcs_client/tests/accessor_test.cc index ff2c8f78e5c5..b8c1d19c4108 100644 --- a/src/ray/gcs/gcs_client/tests/accessor_test.cc +++ b/src/ray/gcs_client/tests/accessor_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_client/accessor.h" +#include "ray/gcs_client/accessor.h" #include "gtest/gtest.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc b/src/ray/gcs_client/tests/gcs_client_reconnection_test.cc similarity index 99% rename from src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc rename to src/ray/gcs_client/tests/gcs_client_reconnection_test.cc index 1167df2f0c53..11803e70019a 100644 --- a/src/ray/gcs/gcs_client/tests/gcs_client_reconnection_test.cc +++ b/src/ray/gcs_client/tests/gcs_client_reconnection_test.cc @@ -23,10 +23,10 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_client/accessor.h" -#include "ray/gcs/gcs_client/gcs_client.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/rpc/gcs/gcs_rpc_client.h" +#include "ray/gcs_client/accessor.h" +#include "ray/gcs_client/gcs_client.h" +#include "ray/gcs_client/rpc_client.h" #include "ray/util/network_util.h" #include "ray/util/path_utils.h" #include "ray/util/raii.h" diff --git a/src/ray/gcs/gcs_client/tests/gcs_client_test.cc b/src/ray/gcs_client/tests/gcs_client_test.cc similarity index 99% rename from src/ray/gcs/gcs_client/tests/gcs_client_test.cc rename to src/ray/gcs_client/tests/gcs_client_test.cc index 12fde5c25d50..e30edd401536 100644 --- a/src/ray/gcs/gcs_client/tests/gcs_client_test.cc +++ b/src/ray/gcs_client/tests/gcs_client_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include #include @@ -23,9 +23,9 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_client/accessor.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/rpc/gcs/gcs_rpc_client.h" +#include "ray/gcs_client/accessor.h" +#include "ray/gcs_client/rpc_client.h" #include "ray/util/network_util.h" #include "ray/util/path_utils.h" #include "ray/util/raii.h" diff --git a/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc b/src/ray/gcs_client/tests/global_state_accessor_test.cc similarity index 99% rename from src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc rename to src/ray/gcs_client/tests/global_state_accessor_test.cc index baa880a625a9..eb9b3b32b16f 100644 --- a/src/ray/gcs/gcs_client/tests/global_state_accessor_test.cc +++ b/src/ray/gcs_client/tests/global_state_accessor_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_client/global_state_accessor.h" +#include "ray/gcs_client/global_state_accessor.h" #include #include @@ -22,7 +22,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/test_utils.h" #include "ray/gcs/gcs_server/gcs_server.h" -#include "ray/rpc/gcs/gcs_rpc_client.h" +#include "ray/gcs_client/rpc_client.h" #include "ray/util/path_utils.h" #include "ray/util/raii.h" diff --git a/src/ray/object_manager/BUILD.bazel b/src/ray/object_manager/BUILD.bazel index ebe575525e53..ab2afb3a2679 100644 --- a/src/ray/object_manager/BUILD.bazel +++ b/src/ray/object_manager/BUILD.bazel @@ -66,7 +66,7 @@ ray_cc_library( ":object_directory", "//src/ray/common:asio", "//src/ray/common:id", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/pubsub:subscriber_interface", "//src/ray/rpc:core_worker_client", "@com_google_absl//absl/container:flat_hash_map", @@ -81,7 +81,7 @@ ray_cc_library( "//src/ray/common:asio", "//src/ray/common:id", "//src/ray/common:status", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", ], ) diff --git a/src/ray/object_manager/object_directory.h b/src/ray/object_manager/object_directory.h index 4eb636ec5e06..fa9130111ea0 100644 --- a/src/ray/object_manager/object_directory.h +++ b/src/ray/object_manager/object_directory.h @@ -24,7 +24,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" #include "ray/common/status.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/object_manager/common.h" namespace ray { diff --git a/src/ray/object_manager/ownership_object_directory.h b/src/ray/object_manager/ownership_object_directory.h index a054a7ef68f5..c1da711673cc 100644 --- a/src/ray/object_manager/ownership_object_directory.h +++ b/src/ray/object_manager/ownership_object_directory.h @@ -23,7 +23,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" #include "ray/common/status.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/object_manager/object_directory.h" #include "ray/pubsub/subscriber.h" #include "ray/rpc/worker/core_worker_client.h" diff --git a/src/ray/object_manager/tests/object_manager_test.cc b/src/ray/object_manager/tests/object_manager_test.cc index 7010ee53a7b8..63049ff4023f 100644 --- a/src/ray/object_manager/tests/object_manager_test.cc +++ b/src/ray/object_manager/tests/object_manager_test.cc @@ -22,7 +22,7 @@ #include "fakes/ray/object_manager/plasma/fake_plasma_client.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "mock/ray/object_manager/object_directory.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" diff --git a/src/ray/object_manager/tests/ownership_object_directory_test.cc b/src/ray/object_manager/tests/ownership_object_directory_test.cc index 1d152f0aadba..ce1a7d54ceda 100644 --- a/src/ray/object_manager/tests/ownership_object_directory_test.cc +++ b/src/ray/object_manager/tests/ownership_object_directory_test.cc @@ -26,8 +26,8 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/status.h" -#include "ray/gcs/gcs_client/accessor.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/accessor.h" +#include "ray/gcs_client/gcs_client.h" namespace ray { diff --git a/src/ray/pubsub/BUILD.bazel b/src/ray/pubsub/BUILD.bazel index 01552be0f3da..302f799ce1c3 100644 --- a/src/ray/pubsub/BUILD.bazel +++ b/src/ray/pubsub/BUILD.bazel @@ -82,9 +82,9 @@ ray_cc_library( hdrs = ["python_gcs_subscriber.h"], deps = [ "//src/ray/common:status", + "//src/ray/gcs_client:rpc_client", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/protobuf:pubsub_cc_proto", - "//src/ray/rpc:gcs_client", "//src/ray/util:visibility", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/synchronization", diff --git a/src/ray/pubsub/python_gcs_subscriber.cc b/src/ray/pubsub/python_gcs_subscriber.cc index d50628b4713d..995fd35d457b 100644 --- a/src/ray/pubsub/python_gcs_subscriber.cc +++ b/src/ray/pubsub/python_gcs_subscriber.cc @@ -21,7 +21,7 @@ #include #include -#include "ray/rpc/gcs/gcs_rpc_client.h" +#include "ray/gcs_client/rpc_client.h" namespace ray { namespace pubsub { diff --git a/src/ray/raylet/BUILD.bazel b/src/ray/raylet/BUILD.bazel index c617eb84fc47..5e70ab42e5f4 100644 --- a/src/ray/raylet/BUILD.bazel +++ b/src/ray/raylet/BUILD.bazel @@ -61,7 +61,7 @@ ray_cc_library( visibility = [":__subpackages__"], deps = [ "//src/ray/common:id", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "//src/ray/util:container_util", "@com_google_absl//absl/container:flat_hash_map", @@ -112,7 +112,7 @@ ray_cc_library( "//src/ray/common:ray_config", "//src/ray/common:runtime_env", "//src/ray/common:status", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/ipc:client_connection", "//src/ray/util:network_util", "//src/ray/util:time", @@ -163,7 +163,7 @@ ray_cc_library( ":worker_pool", "//src/ray/common:id", "//src/ray/common:ray_object", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/object_manager:object_directory", "//src/ray/object_manager:object_manager_common", "//src/ray/protobuf:node_manager_cc_proto", @@ -231,7 +231,7 @@ ray_cc_library( "//src/ray/common:memory_monitor", "//src/ray/core_worker:experimental_mutable_object_provider", "//src/ray/flatbuffers:node_manager_generated", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/object_manager", "//src/ray/object_manager:ownership_object_directory", "//src/ray/object_manager/plasma:plasma_client", @@ -286,7 +286,7 @@ ray_cc_binary( "//src/ray/common/cgroup2:cgroup_manager", "//src/ray/common/cgroup2:sysfs_cgroup_driver", "//src/ray/core_worker:metrics", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/object_manager:ownership_object_directory", "//src/ray/raylet/scheduling:cluster_lease_manager", "//src/ray/rpc:metrics_agent_client", diff --git a/src/ray/raylet/local_object_manager.h b/src/ray/raylet/local_object_manager.h index b04604eaa5b8..25597aeda64b 100644 --- a/src/ray/raylet/local_object_manager.h +++ b/src/ray/raylet/local_object_manager.h @@ -23,7 +23,7 @@ #include "ray/common/id.h" #include "ray/common/ray_object.h" -#include "ray/gcs/gcs_client/accessor.h" +#include "ray/gcs_client/accessor.h" #include "ray/object_manager/common.h" #include "ray/object_manager/object_directory.h" #include "ray/pubsub/subscriber_interface.h" diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 5cb7cd35975a..c556a1219e81 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -33,7 +33,7 @@ #include "ray/common/status.h" #include "ray/common/status_or.h" #include "ray/core_worker/metrics.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/object_manager/ownership_object_directory.h" #include "ray/raylet/local_object_manager.h" #include "ray/raylet/local_object_manager_interface.h" diff --git a/src/ray/raylet/scheduling/tests/BUILD.bazel b/src/ray/raylet/scheduling/tests/BUILD.bazel index 15e6820c6873..25483ef3b383 100644 --- a/src/ray/raylet/scheduling/tests/BUILD.bazel +++ b/src/ray/raylet/scheduling/tests/BUILD.bazel @@ -13,7 +13,7 @@ ray_cc_test( "//src/ray/common:ray_config", "//src/ray/common:task_common", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/raylet/scheduling:cluster_resource_scheduler", "@com_google_googletest//:gtest_main", ], diff --git a/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc b/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc index 40eeee6f1cf5..aabdab0c559e 100644 --- a/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc +++ b/src/ray/raylet/scheduling/tests/cluster_lease_manager_test.cc @@ -35,7 +35,7 @@ #include "ray/raylet/local_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/tests/util.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" // clang-format on namespace ray { diff --git a/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc b/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc index 3237971f00de..9614fa22a737 100644 --- a/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc +++ b/src/ray/raylet/scheduling/tests/cluster_resource_scheduler_test.cc @@ -28,7 +28,7 @@ #include "ray/common/test_utils.h" #include "ray/common/scheduling/resource_set.h" #include "ray/common/scheduling/scheduling_ids.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" // clang-format on using namespace std; // NOLINT diff --git a/src/ray/raylet/tests/BUILD.bazel b/src/ray/raylet/tests/BUILD.bazel index 746660fbeca6..615dae910b46 100644 --- a/src/ray/raylet/tests/BUILD.bazel +++ b/src/ray/raylet/tests/BUILD.bazel @@ -50,7 +50,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:asio", "//src/ray/common:id", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/object_manager:ownership_object_directory", "//src/ray/protobuf:core_worker_cc_grpc", "//src/ray/pubsub:subscriber", diff --git a/src/ray/raylet/tests/local_lease_manager_test.cc b/src/ray/raylet/tests/local_lease_manager_test.cc index 4830b029c9ce..c2755dfe0659 100644 --- a/src/ray/raylet/tests/local_lease_manager_test.cc +++ b/src/ray/raylet/tests/local_lease_manager_test.cc @@ -25,7 +25,7 @@ #include #include -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "mock/ray/object_manager/object_manager.h" #include "ray/common/id.h" #include "ray/common/lease/lease.h" diff --git a/src/ray/raylet/tests/local_object_manager_test.cc b/src/ray/raylet/tests/local_object_manager_test.cc index 9fd1fe7e2490..294b25cd40e4 100644 --- a/src/ray/raylet/tests/local_object_manager_test.cc +++ b/src/ray/raylet/tests/local_object_manager_test.cc @@ -25,10 +25,10 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" -#include "ray/gcs/gcs_client/accessor.h" +#include "ray/gcs_client/accessor.h" #include "ray/object_manager/ownership_object_directory.h" #include "ray/pubsub/subscriber.h" #include "ray/raylet/tests/util.h" diff --git a/src/ray/raylet/tests/node_manager_test.cc b/src/ray/raylet/tests/node_manager_test.cc index b05d4c5ac0bd..bd3780619e26 100644 --- a/src/ray/raylet/tests/node_manager_test.cc +++ b/src/ray/raylet/tests/node_manager_test.cc @@ -26,7 +26,7 @@ #include "fakes/ray/rpc/raylet/raylet_client.h" #include "gmock/gmock.h" #include "mock/ray/core_worker/experimental_mutable_object_provider.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "mock/ray/object_manager/object_directory.h" #include "mock/ray/object_manager/object_manager.h" #include "mock/ray/raylet/local_lease_manager.h" diff --git a/src/ray/raylet/tests/placement_group_resource_manager_test.cc b/src/ray/raylet/tests/placement_group_resource_manager_test.cc index f3cb445677fe..494e81941ee4 100644 --- a/src/ray/raylet/tests/placement_group_resource_manager_test.cc +++ b/src/ray/raylet/tests/placement_group_resource_manager_test.cc @@ -20,7 +20,7 @@ #include #include "gtest/gtest.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "ray/common/bundle_spec.h" #include "ray/common/id.h" #include "ray/common/scheduling/placement_group_util.h" diff --git a/src/ray/raylet/tests/worker_pool_test.cc b/src/ray/raylet/tests/worker_pool_test.cc index a75cee7603ca..b5d24485a7bf 100644 --- a/src/ray/raylet/tests/worker_pool_test.cc +++ b/src/ray/raylet/tests/worker_pool_test.cc @@ -26,7 +26,7 @@ #include #include "absl/time/time.h" -#include "mock/ray/gcs/gcs_client/gcs_client.h" +#include "mock/ray/gcs_client/gcs_client.h" #include "nlohmann/json.hpp" #include "ray/common/asio/asio_util.h" #include "ray/common/asio/instrumented_io_context.h" diff --git a/src/ray/raylet/worker_pool.h b/src/ray/raylet/worker_pool.h index 1d6e43a23497..f048ac2f6c29 100644 --- a/src/ray/raylet/worker_pool.h +++ b/src/ray/raylet/worker_pool.h @@ -36,7 +36,7 @@ #include "ray/common/asio/periodical_runner.h" #include "ray/common/lease/lease.h" #include "ray/common/runtime_env_manager.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/ipc/client_connection.h" #include "ray/raylet/runtime_env_agent_client.h" #include "ray/raylet/worker.h" diff --git a/src/ray/rpc/BUILD.bazel b/src/ray/rpc/BUILD.bazel index 3b663ebf23f9..2a83b60815e8 100644 --- a/src/ray/rpc/BUILD.bazel +++ b/src/ray/rpc/BUILD.bazel @@ -137,23 +137,6 @@ ray_cc_library( ], ) -ray_cc_library( - name = "gcs_client", - hdrs = [ - "gcs/gcs_rpc_client.h", - ], - visibility = ["//visibility:public"], - deps = [ - ":client_call", - ":retryable_grpc_client", - "//src/ray/common:ray_config", - "//src/ray/protobuf:autoscaler_cc_grpc", - "//src/ray/protobuf:gcs_service_cc_grpc", - "//src/ray/util:network_util", - "@com_google_absl//absl/container:btree", - ], -) - ray_cc_library( name = "raylet_client_interface", hdrs = [ @@ -176,7 +159,7 @@ ray_cc_library( visibility = ["//visibility:public"], deps = [ ":raylet_client_interface", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", ], ) @@ -210,7 +193,7 @@ ray_cc_library( ":raylet_client_pool", "//src/ray/common:id", "//src/ray/common:status", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/protobuf:core_worker_cc_grpc", "//src/ray/pubsub:subscriber", "//src/ray/util:logging", diff --git a/src/ray/rpc/raylet/raylet_client_pool.h b/src/ray/rpc/raylet/raylet_client_pool.h index d69edf0533ab..1ab520b0a519 100644 --- a/src/ray/rpc/raylet/raylet_client_pool.h +++ b/src/ray/rpc/raylet/raylet_client_pool.h @@ -23,7 +23,7 @@ #include "absl/strings/str_cat.h" #include "absl/synchronization/mutex.h" #include "ray/common/id.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/rpc/raylet/raylet_client_interface.h" namespace ray { diff --git a/src/ray/rpc/raylet/tests/BUILD.bazel b/src/ray/rpc/raylet/tests/BUILD.bazel index 280f5a3c3927..ac59bcc2f969 100644 --- a/src/ray/rpc/raylet/tests/BUILD.bazel +++ b/src/ray/rpc/raylet/tests/BUILD.bazel @@ -7,7 +7,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/fakes/ray/rpc/raylet:fake_raylet_client", - "//src/ray/gcs/gcs_client:gcs_client_lib", + "//src/ray/gcs_client", "//src/ray/rpc:raylet_client_pool", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", diff --git a/src/ray/rpc/worker/core_worker_client_pool.h b/src/ray/rpc/worker/core_worker_client_pool.h index b4008797fd26..4e33fd8ca2c4 100644 --- a/src/ray/rpc/worker/core_worker_client_pool.h +++ b/src/ray/rpc/worker/core_worker_client_pool.h @@ -23,7 +23,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/synchronization/mutex.h" #include "ray/common/id.h" -#include "ray/gcs/gcs_client/gcs_client.h" +#include "ray/gcs_client/gcs_client.h" #include "ray/rpc/raylet/raylet_client_interface.h" #include "ray/rpc/raylet/raylet_client_pool.h" #include "ray/rpc/worker/core_worker_client.h" From e0548b1697b8e89b2ce010058d561191dd7b6bc5 Mon Sep 17 00:00:00 2001 From: Alan Guo Date: Mon, 15 Sep 2025 11:23:33 -0700 Subject: [PATCH 626/634] Fix accidentally using task throughput instead of row throughput (#56503) ## Why are these changes needed? in #56428 I accidentally added the wrong throughput graph. This is row throughput I wanted. ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [ ] Unit tests - [ ] Release tests - [ ] This PR is not tested :( Signed-off-by: Alan Guo --- .../modules/metrics/dashboards/data_dashboard_panels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py index fd50e1ba2312..b9b9e14abc17 100644 --- a/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py +++ b/python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py @@ -831,7 +831,7 @@ stack=False, ) -OPERATOR_PANELS = [TASK_THROUGHPUT_BY_NODE_PANEL, ALL_RESOURCES_UTILIZATION_PANEL] +OPERATOR_PANELS = [ROWS_OUTPUT_PER_SECOND_PANEL, ALL_RESOURCES_UTILIZATION_PANEL] DATA_GRAFANA_ROWS = [ # Overview Row From ee89e8cbdc0945706bacb3e8d605024b40e27a88 Mon Sep 17 00:00:00 2001 From: Abrar Sheikh Date: Mon, 15 Sep 2025 11:33:14 -0700 Subject: [PATCH 627/634] use default gc frequency for proxy (#56511) ## script used for benchmarking ```python import time from typing import Optional from python.ray._common.test_utils import wait_for_condition from ray import serve from ray.util.state import list_actors import logging logger = logging.getLogger("ray.serve") @serve.deployment(max_ongoing_requests=1000) class MemoryLeakTest: async def __call__(self): logger.info("MemoryLeakTest") return "MemoryLeakTest" app = serve.run(MemoryLeakTest.bind(), logging_config={ "encoding": "JSON", }) def get_replica_pid() -> Optional[int]: all_current_actors = list_actors(filters=[("state", "=", "ALIVE")]) for actor in all_current_actors: if "MemoryLeakTest" in actor["name"]: return actor["pid"] return None wait_for_condition(get_replica_pid) print(get_replica_pid()) # track the memory of the replica in a loop in MB import psutil def track_memory(): pid = get_replica_pid() if pid is not None: process = psutil.Process(pid) return process.memory_info().rss / 1024 / 1024 return None while True: memory_mb = track_memory() print(f"\rMemory usage: {memory_mb:.2f} MB", end="", flush=True) time.sleep(.1) ``` simulating load using `ab -n 500 -c 1 http://127.0.0.1:8000/` used [memray](https://bloomberg.github.io/memray/tutorials/1.html) to profile the proxy process. Used instructions from [here](https://docs.ray.io/en/latest/ray-observability/user-guides/debug-apps/debug-memory.html#memory-profiling-ray-tasks-and-actors). ### On master image ### With fix image When we reduce the garbage collection (GC) frequency to every 10k allocations, proxy memory peaks at **1.3 GB** for my test workload. By contrast, under the default GC frequency (700 allocations), peak RSS memory is **700 MB**. The higher memory footprint with less frequent GC occurs because this workload involves large object transactions. With GC running only after 10k allocations, these large objects remain in RSS longer, inflating memory usage until a collection cycle is triggered. Importantly, I found no evidence of a memory leak under sustained load. With the fix, memory stabilizes at around **700 MB**, and even without the fix, usage plateaus at **1.3 GB** rather than growing unbounded. This feature was added in https://github.com/ray-project/ray/pull/49720 as a performance optimization. So we are taking slight hit in RPS for stable memory usage for larger payloads. --------- Signed-off-by: abrar --- python/ray/serve/_private/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/serve/_private/constants.py b/python/ray/serve/_private/constants.py index dc99f63ef838..a670fdd53616 100644 --- a/python/ray/serve/_private/constants.py +++ b/python/ray/serve/_private/constants.py @@ -437,7 +437,7 @@ ) # Used for gc.set_threshold() when proxy GC optimizations are enabled. -RAY_SERVE_PROXY_GC_THRESHOLD = get_env_int("RAY_SERVE_PROXY_GC_THRESHOLD", 10_000) +RAY_SERVE_PROXY_GC_THRESHOLD = get_env_int("RAY_SERVE_PROXY_GC_THRESHOLD", 700) # Interval at which cached metrics will be exported using the Ray metric API. # Set to `0` to disable caching entirely. From ab7266502982bb8d87b183ebbe453b2c079f4d52 Mon Sep 17 00:00:00 2001 From: Elliot Barnwell Date: Mon, 15 Sep 2025 11:48:47 -0700 Subject: [PATCH 628/634] [ci] updating raydepsets llm check (#56439) using `--check` feature to verify llm lock files are unchanged --------- Signed-off-by: elliot-barn Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> --- .buildkite/dependencies.rayci.yml | 5 ++++- ci/raydepsets/rayllm.depsets.yaml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.buildkite/dependencies.rayci.yml b/.buildkite/dependencies.rayci.yml index 2620a7ee399c..ab8217bfa214 100644 --- a/.buildkite/dependencies.rayci.yml +++ b/.buildkite/dependencies.rayci.yml @@ -23,7 +23,10 @@ steps: key: raydepsets_compile_llm_dependencies tags: always instance_type: small - command: ./ci/test_compile_llm_requirements.sh + commands: + - bazel run //ci/raydepsets:raydepsets -- build ci/raydepsets/rayllm.depsets.yaml --check + - chown -R 2000:100 /artifact-mount + - cp ./python/deplocks/llm/* /artifact-mount/ job_env: manylinux depends_on: manylinux diff --git a/ci/raydepsets/rayllm.depsets.yaml b/ci/raydepsets/rayllm.depsets.yaml index b6ed8fb51cec..ecb4fea950cf 100644 --- a/ci/raydepsets/rayllm.depsets.yaml +++ b/ci/raydepsets/rayllm.depsets.yaml @@ -26,6 +26,7 @@ depsets: # First, extract base test dependencies from the current compiled mono repo one. # This also expands to the indirect dependencies for this Python version & platform. - name: ray_base_test_depset_${PYTHON_VERSION}_${CUDA_CODE} + operation: compile <<: *common_settings requirements: - python/requirements.txt @@ -34,7 +35,6 @@ depsets: constraints: - /tmp/ray-deps/requirements_compiled.txt output: python/deplocks/llm/ray_test_${PYTHON_VERSION}_${CUDA_CODE}.lock - operation: compile pre_hooks: - ci/raydepsets/pre_hooks/remove-compiled-headers.sh From ac90fa0d420d48044cb56cf894f6cd7191f4baf8 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 15 Sep 2025 14:59:04 -0700 Subject: [PATCH 629/634] [image] allow using explicit base type (#56545) this allows using `base-extra` or `base-extra-testdeps` or other base variations for building ray images. Signed-off-by: Lonnie Liu --- ci/ray_ci/builder.py | 2 +- ci/ray_ci/ray_docker_container.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ci/ray_ci/builder.py b/ci/ray_ci/builder.py index e6d02b251c73..7e74a4906b98 100644 --- a/ci/ray_ci/builder.py +++ b/ci/ray_ci/builder.py @@ -172,7 +172,7 @@ def build_anyscale( for p in platform: RayDockerContainer( python_version, p, image_type, architecture, canonical_tag, upload=False - ).run(use_base_extra_testdeps=True) + ).run(base="base-extra-testdeps") AnyscaleDockerContainer( python_version, p, image_type, architecture, canonical_tag, upload ).run() diff --git a/ci/ray_ci/ray_docker_container.py b/ci/ray_ci/ray_docker_container.py index e58532b10692..b3f9e1758bb6 100644 --- a/ci/ray_ci/ray_docker_container.py +++ b/ci/ray_ci/ray_docker_container.py @@ -1,5 +1,5 @@ import os -from typing import List +from typing import List, Optional from ray_release.configs.global_config import get_global_config @@ -14,21 +14,25 @@ class RayDockerContainer(DockerContainer): Container for building and publishing ray docker images """ - def run(self, use_base_extra_testdeps: bool = False) -> None: + def run(self, base: Optional[str] = None) -> None: """ Build and publish ray docker images """ assert "RAYCI_BUILD_ID" in os.environ, "RAYCI_BUILD_ID not set" rayci_build_id = os.environ["RAYCI_BUILD_ID"] - base_name = "base" if not use_base_extra_testdeps else "base-extra-testdeps" + if base is None: + base = "base" + if self.architecture == DEFAULT_ARCHITECTURE: - suffix = base_name + suffix = base else: - suffix = f"{base_name}-{self.architecture}" + suffix = f"{base}-{self.architecture}" + + image_repo = self.image_type base_image = ( f"{_DOCKER_ECR_REPO}:{rayci_build_id}" - f"-{self.image_type}-py{self.python_version}-{self.platform}-{suffix}" + f"-{image_repo}-py{self.python_version}-{self.platform}-{suffix}" ) docker_pull(base_image) @@ -39,7 +43,7 @@ def run(self, use_base_extra_testdeps: bool = False) -> None: ) constraints_file = "requirements_compiled.txt" tag = self._get_canonical_tag() - ray_image = f"rayproject/{self.image_type}:{tag}" + ray_image = f"rayproject/{image_repo}:{tag}" pip_freeze = f"{self.image_type}:{tag}_pip-freeze.txt" cmds = [ From dc01c8a907fdb8ccb893e61449ab407a3ad90561 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Mon, 15 Sep 2025 17:27:12 -0500 Subject: [PATCH 630/634] [core] Remove `gcs_server` directory nesting (#56516) Non-GCS component files have been moved; no longer need the nesting. --------- Signed-off-by: Edward Oakes --- BUILD.bazel | 4 +- python/ray/includes/global_state_accessor.pxd | 2 +- .../gcs/{gcs_server => }/gcs_actor_manager.h | 2 +- .../{gcs_server => }/gcs_actor_scheduler.h | 2 +- .../gcs/{gcs_server => }/gcs_job_manager.h | 2 +- .../ray/gcs/{gcs_server => }/gcs_kv_manager.h | 2 +- .../gcs/{gcs_server => }/gcs_node_manager.h | 2 +- .../gcs_placement_group_manager.h | 2 +- .../gcs_placement_group_scheduler.h | 2 +- .../{gcs_server => }/gcs_resource_manager.h | 2 +- .../gcs/{gcs_server => }/gcs_task_manager.h | 2 +- .../gcs/{gcs_server => }/gcs_worker_manager.h | 2 +- src/mock/ray/gcs/pubsub/gcs_pub_sub.h | 30 ------- src/ray/gcs/{gcs_server => }/BUILD.bazel | 4 +- src/ray/gcs/{gcs_server => }/gcs_actor.cc | 2 +- src/ray/gcs/{gcs_server => }/gcs_actor.h | 0 .../gcs/{gcs_server => }/gcs_actor_manager.cc | 2 +- .../gcs/{gcs_server => }/gcs_actor_manager.h | 14 +-- .../{gcs_server => }/gcs_actor_scheduler.cc | 2 +- .../{gcs_server => }/gcs_actor_scheduler.h | 6 +- .../gcs_autoscaler_state_manager.cc | 2 +- .../gcs_autoscaler_state_manager.h | 14 +-- .../{gcs_server => }/gcs_function_manager.h | 2 +- .../gcs_health_check_manager.cc | 2 +- .../gcs_health_check_manager.h | 0 src/ray/gcs/{gcs_server => }/gcs_init_data.cc | 2 +- src/ray/gcs/{gcs_server => }/gcs_init_data.h | 2 +- .../gcs/{gcs_server => }/gcs_job_manager.cc | 2 +- .../gcs/{gcs_server => }/gcs_job_manager.h | 10 +-- .../gcs/{gcs_server => }/gcs_kv_manager.cc | 2 +- src/ray/gcs/{gcs_server => }/gcs_kv_manager.h | 2 +- .../gcs/{gcs_server => }/gcs_node_manager.cc | 2 +- .../gcs/{gcs_server => }/gcs_node_manager.h | 6 +- .../{gcs_server => }/gcs_placement_group.cc | 2 +- .../{gcs_server => }/gcs_placement_group.h | 0 .../gcs_placement_group_manager.cc | 2 +- .../gcs_placement_group_manager.h | 14 +-- .../gcs_placement_group_scheduler.cc | 2 +- .../gcs_placement_group_scheduler.h | 6 +- .../gcs_ray_event_converter.cc | 2 +- .../gcs_ray_event_converter.h | 0 .../{gcs_server => }/gcs_resource_manager.cc | 4 +- .../{gcs_server => }/gcs_resource_manager.h | 6 +- src/ray/gcs/{gcs_server => }/gcs_server.cc | 18 ++-- src/ray/gcs/{gcs_server => }/gcs_server.h | 22 ++--- .../gcs_server_io_context_policy.h | 2 +- .../gcs/{gcs_server => }/gcs_server_main.cc | 2 +- .../gcs/{gcs_server => }/gcs_table_storage.cc | 2 +- .../gcs/{gcs_server => }/gcs_table_storage.h | 0 .../gcs/{gcs_server => }/gcs_task_manager.cc | 2 +- .../gcs/{gcs_server => }/gcs_task_manager.h | 6 +- .../{gcs_server => }/gcs_worker_manager.cc | 2 +- .../gcs/{gcs_server => }/gcs_worker_manager.h | 8 +- .../grpc_service_interfaces.h | 0 src/ray/gcs/{gcs_server => }/grpc_services.cc | 2 +- src/ray/gcs/{gcs_server => }/grpc_services.h | 2 +- .../gcs/{gcs_server => }/pubsub_handler.cc | 2 +- src/ray/gcs/{gcs_server => }/pubsub_handler.h | 2 +- .../{gcs_server => }/runtime_env_handler.cc | 2 +- .../{gcs_server => }/runtime_env_handler.h | 2 +- src/ray/gcs/{gcs_server => }/state_util.cc | 2 +- src/ray/gcs/{gcs_server => }/state_util.h | 0 .../gcs/{gcs_server => }/store_client_kv.cc | 2 +- .../gcs/{gcs_server => }/store_client_kv.h | 2 +- .../gcs/{gcs_server => }/tests/BUILD.bazel | 88 +++++++++---------- .../gcs_actor_manager_export_event_test.cc | 10 +-- .../gcs_job_manager_export_event_test.cc | 6 +- .../gcs_node_manager_export_event_test.cc | 2 +- .../tests/gcs_actor_manager_test.cc | 12 +-- .../tests/gcs_actor_scheduler_mock_test.cc | 6 +- .../tests/gcs_actor_scheduler_test.cc | 8 +- .../gcs_autoscaler_state_manager_test.cc | 14 +-- .../tests/gcs_function_manager_test.cc | 4 +- .../tests/gcs_health_check_manager_test.cc | 2 +- .../tests/gcs_job_manager_test.cc | 6 +- .../tests/gcs_kv_manager_test.cc | 4 +- .../tests/gcs_node_manager_test.cc | 2 +- .../gcs_placement_group_manager_mock_test.cc | 8 +- .../tests/gcs_placement_group_manager_test.cc | 4 +- .../gcs_placement_group_scheduler_test.cc | 10 +-- .../tests/gcs_ray_event_converter_test.cc | 2 +- .../tests/gcs_resource_manager_test.cc | 4 +- .../tests/gcs_server_rpc_test.cc | 2 +- .../tests/gcs_server_test_util.h | 12 +-- .../tests/gcs_table_storage_test_base.h | 2 +- .../tests/gcs_task_manager_test.cc | 2 +- .../tests/gcs_worker_manager_test.cc | 4 +- .../tests/in_memory_gcs_table_storage_test.cc | 4 +- .../tests/redis_gcs_table_storage_test.cc | 4 +- .../tests/usage_stats_client_test.cc | 6 +- .../{gcs_server => }/usage_stats_client.cc | 2 +- .../gcs/{gcs_server => }/usage_stats_client.h | 2 +- src/ray/gcs_client/accessor.cc | 2 +- src/ray/gcs_client/tests/BUILD.bazel | 6 +- .../tests/gcs_client_reconnection_test.cc | 2 +- src/ray/gcs_client/tests/gcs_client_test.cc | 2 +- .../tests/global_state_accessor_test.cc | 2 +- 97 files changed, 235 insertions(+), 265 deletions(-) rename src/mock/ray/gcs/{gcs_server => }/gcs_actor_manager.h (98%) rename src/mock/ray/gcs/{gcs_server => }/gcs_actor_scheduler.h (98%) rename src/mock/ray/gcs/{gcs_server => }/gcs_job_manager.h (97%) rename src/mock/ray/gcs/{gcs_server => }/gcs_kv_manager.h (99%) rename src/mock/ray/gcs/{gcs_server => }/gcs_node_manager.h (97%) rename src/mock/ray/gcs/{gcs_server => }/gcs_placement_group_manager.h (97%) rename src/mock/ray/gcs/{gcs_server => }/gcs_placement_group_scheduler.h (98%) rename src/mock/ray/gcs/{gcs_server => }/gcs_resource_manager.h (97%) rename src/mock/ray/gcs/{gcs_server => }/gcs_task_manager.h (96%) rename src/mock/ray/gcs/{gcs_server => }/gcs_worker_manager.h (97%) delete mode 100644 src/mock/ray/gcs/pubsub/gcs_pub_sub.h rename src/ray/gcs/{gcs_server => }/BUILD.bazel (99%) rename src/ray/gcs/{gcs_server => }/gcs_actor.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_actor.h (100%) rename src/ray/gcs/{gcs_server => }/gcs_actor_manager.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_actor_manager.h (98%) rename src/ray/gcs/{gcs_server => }/gcs_actor_scheduler.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_actor_scheduler.h (99%) rename src/ray/gcs/{gcs_server => }/gcs_autoscaler_state_manager.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_autoscaler_state_manager.h (96%) rename src/ray/gcs/{gcs_server => }/gcs_function_manager.h (98%) rename src/ray/gcs/{gcs_server => }/gcs_health_check_manager.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_health_check_manager.h (100%) rename src/ray/gcs/{gcs_server => }/gcs_init_data.cc (98%) rename src/ray/gcs/{gcs_server => }/gcs_init_data.h (98%) rename src/ray/gcs/{gcs_server => }/gcs_job_manager.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_job_manager.h (96%) rename src/ray/gcs/{gcs_server => }/gcs_kv_manager.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_kv_manager.h (99%) rename src/ray/gcs/{gcs_server => }/gcs_node_manager.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_node_manager.h (98%) rename src/ray/gcs/{gcs_server => }/gcs_placement_group.cc (98%) rename src/ray/gcs/{gcs_server => }/gcs_placement_group.h (100%) rename src/ray/gcs/{gcs_server => }/gcs_placement_group_manager.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_placement_group_manager.h (97%) rename src/ray/gcs/{gcs_server => }/gcs_placement_group_scheduler.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_placement_group_scheduler.h (99%) rename src/ray/gcs/{gcs_server => }/gcs_ray_event_converter.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_ray_event_converter.h (100%) rename src/ray/gcs/{gcs_server => }/gcs_resource_manager.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_resource_manager.h (98%) rename src/ray/gcs/{gcs_server => }/gcs_server.cc (98%) rename src/ray/gcs/{gcs_server => }/gcs_server.h (94%) rename src/ray/gcs/{gcs_server => }/gcs_server_io_context_policy.h (98%) rename src/ray/gcs/{gcs_server => }/gcs_server_main.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_table_storage.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_table_storage.h (100%) rename src/ray/gcs/{gcs_server => }/gcs_task_manager.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_task_manager.h (99%) rename src/ray/gcs/{gcs_server => }/gcs_worker_manager.cc (99%) rename src/ray/gcs/{gcs_server => }/gcs_worker_manager.h (94%) rename src/ray/gcs/{gcs_server => }/grpc_service_interfaces.h (100%) rename src/ray/gcs/{gcs_server => }/grpc_services.cc (99%) rename src/ray/gcs/{gcs_server => }/grpc_services.h (99%) rename src/ray/gcs/{gcs_server => }/pubsub_handler.cc (99%) rename src/ray/gcs/{gcs_server => }/pubsub_handler.h (97%) rename src/ray/gcs/{gcs_server => }/runtime_env_handler.cc (96%) rename src/ray/gcs/{gcs_server => }/runtime_env_handler.h (96%) rename src/ray/gcs/{gcs_server => }/state_util.cc (97%) rename src/ray/gcs/{gcs_server => }/state_util.h (100%) rename src/ray/gcs/{gcs_server => }/store_client_kv.cc (99%) rename src/ray/gcs/{gcs_server => }/store_client_kv.h (97%) rename src/ray/gcs/{gcs_server => }/tests/BUILD.bazel (80%) rename src/ray/gcs/{gcs_server => }/tests/export_api/gcs_actor_manager_export_event_test.cc (98%) rename src/ray/gcs/{gcs_server => }/tests/export_api/gcs_job_manager_export_event_test.cc (98%) rename src/ray/gcs/{gcs_server => }/tests/export_api/gcs_node_manager_export_event_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_actor_manager_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_actor_scheduler_mock_test.cc (98%) rename src/ray/gcs/{gcs_server => }/tests/gcs_actor_scheduler_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_autoscaler_state_manager_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_function_manager_test.cc (97%) rename src/ray/gcs/{gcs_server => }/tests/gcs_health_check_manager_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_job_manager_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_kv_manager_test.cc (98%) rename src/ray/gcs/{gcs_server => }/tests/gcs_node_manager_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_placement_group_manager_mock_test.cc (97%) rename src/ray/gcs/{gcs_server => }/tests/gcs_placement_group_manager_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_placement_group_scheduler_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_ray_event_converter_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_resource_manager_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_server_rpc_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_server_test_util.h (97%) rename src/ray/gcs/{gcs_server => }/tests/gcs_table_storage_test_base.h (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_task_manager_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/gcs_worker_manager_test.cc (99%) rename src/ray/gcs/{gcs_server => }/tests/in_memory_gcs_table_storage_test.cc (91%) rename src/ray/gcs/{gcs_server => }/tests/redis_gcs_table_storage_test.cc (93%) rename src/ray/gcs/{gcs_server => }/tests/usage_stats_client_test.cc (93%) rename src/ray/gcs/{gcs_server => }/usage_stats_client.cc (96%) rename src/ray/gcs/{gcs_server => }/usage_stats_client.h (97%) diff --git a/BUILD.bazel b/BUILD.bazel index 5337472727fe..9224cf8f2373 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -246,7 +246,7 @@ pyx_library( "//:src/ray/ray_exported_symbols.lds", "//:src/ray/ray_version_script.lds", "//src/ray/core_worker:core_worker_lib", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs:gcs_server_lib", "//src/ray/gcs/store_client:redis_store_client", "//src/ray/gcs_client:gcs_python_callbacks", "//src/ray/gcs_client:global_state_accessor_lib", @@ -385,7 +385,7 @@ pkg_files( pkg_files( name = "gcs_server_files", - srcs = ["//src/ray/gcs/gcs_server"], + srcs = ["//src/ray/gcs:gcs_server"], attributes = pkg_attributes(mode = "755"), prefix = "ray/core/src/ray/gcs", visibility = ["//visibility:private"], diff --git a/python/ray/includes/global_state_accessor.pxd b/python/ray/includes/global_state_accessor.pxd index 1eb54aeb2025..44d2e3321c1c 100644 --- a/python/ray/includes/global_state_accessor.pxd +++ b/python/ray/includes/global_state_accessor.pxd @@ -70,7 +70,7 @@ cdef extern from "ray/gcs_client/global_state_accessor.h" nogil: cdef extern from * namespace "ray::gcs" nogil: """ #include - #include "ray/gcs/gcs_server/store_client_kv.h" + #include "ray/gcs/store_client_kv.h" #include "ray/gcs/store_client/redis_store_client.h" #include "ray/util/raii.h" namespace ray { diff --git a/src/mock/ray/gcs/gcs_server/gcs_actor_manager.h b/src/mock/ray/gcs/gcs_actor_manager.h similarity index 98% rename from src/mock/ray/gcs/gcs_server/gcs_actor_manager.h rename to src/mock/ray/gcs/gcs_actor_manager.h index b3984e0c36c1..fd5a5f4a8769 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_actor_manager.h +++ b/src/mock/ray/gcs/gcs_actor_manager.h @@ -16,7 +16,7 @@ #include -#include "ray/gcs/gcs_server/gcs_actor_manager.h" +#include "ray/gcs/gcs_actor_manager.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/mock/ray/gcs/gcs_actor_scheduler.h similarity index 98% rename from src/mock/ray/gcs/gcs_server/gcs_actor_scheduler.h rename to src/mock/ray/gcs/gcs_actor_scheduler.h index f4edac842542..7ada39f420d6 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_actor_scheduler.h +++ b/src/mock/ray/gcs/gcs_actor_scheduler.h @@ -16,7 +16,7 @@ #include -#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" +#include "ray/gcs/gcs_actor_scheduler.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_job_manager.h b/src/mock/ray/gcs/gcs_job_manager.h similarity index 97% rename from src/mock/ray/gcs/gcs_server/gcs_job_manager.h rename to src/mock/ray/gcs/gcs_job_manager.h index 9228c063e4f3..2a04a8e2b87a 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_job_manager.h +++ b/src/mock/ray/gcs/gcs_job_manager.h @@ -16,7 +16,7 @@ #include -#include "ray/gcs/gcs_server/gcs_job_manager.h" +#include "ray/gcs/gcs_job_manager.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_kv_manager.h b/src/mock/ray/gcs/gcs_kv_manager.h similarity index 99% rename from src/mock/ray/gcs/gcs_server/gcs_kv_manager.h rename to src/mock/ray/gcs/gcs_kv_manager.h index 5a5a224e0199..87df51b573db 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_kv_manager.h +++ b/src/mock/ray/gcs/gcs_kv_manager.h @@ -16,7 +16,7 @@ #include -#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_kv_manager.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_node_manager.h b/src/mock/ray/gcs/gcs_node_manager.h similarity index 97% rename from src/mock/ray/gcs/gcs_server/gcs_node_manager.h rename to src/mock/ray/gcs/gcs_node_manager.h index 67161c0a6456..ef81ef8a6d71 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/mock/ray/gcs/gcs_node_manager.h @@ -16,7 +16,7 @@ #include -#include "ray/gcs/gcs_server/gcs_node_manager.h" +#include "ray/gcs/gcs_node_manager.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_placement_group_manager.h b/src/mock/ray/gcs/gcs_placement_group_manager.h similarity index 97% rename from src/mock/ray/gcs/gcs_server/gcs_placement_group_manager.h rename to src/mock/ray/gcs/gcs_placement_group_manager.h index 433bc132bddb..ffd4ceee0cb0 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_placement_group_manager.h +++ b/src/mock/ray/gcs/gcs_placement_group_manager.h @@ -16,7 +16,7 @@ #include -#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" +#include "ray/gcs/gcs_placement_group_manager.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_placement_group_scheduler.h b/src/mock/ray/gcs/gcs_placement_group_scheduler.h similarity index 98% rename from src/mock/ray/gcs/gcs_server/gcs_placement_group_scheduler.h rename to src/mock/ray/gcs/gcs_placement_group_scheduler.h index e35bcd45940b..f6fb6ac3ff14 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_placement_group_scheduler.h +++ b/src/mock/ray/gcs/gcs_placement_group_scheduler.h @@ -16,7 +16,7 @@ #include -#include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" +#include "ray/gcs/gcs_placement_group_scheduler.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_resource_manager.h b/src/mock/ray/gcs/gcs_resource_manager.h similarity index 97% rename from src/mock/ray/gcs/gcs_server/gcs_resource_manager.h rename to src/mock/ray/gcs/gcs_resource_manager.h index a3865ab02968..0d5c83531cb3 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_resource_manager.h +++ b/src/mock/ray/gcs/gcs_resource_manager.h @@ -17,7 +17,7 @@ #include #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/gcs_resource_manager.h" +#include "ray/gcs/gcs_resource_manager.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_task_manager.h b/src/mock/ray/gcs/gcs_task_manager.h similarity index 96% rename from src/mock/ray/gcs/gcs_server/gcs_task_manager.h rename to src/mock/ray/gcs/gcs_task_manager.h index e3c8222a01d4..db633ba6e6b8 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_task_manager.h +++ b/src/mock/ray/gcs/gcs_task_manager.h @@ -16,7 +16,7 @@ #include -#include "ray/gcs/gcs_server/gcs_task_manager.h" +#include "ray/gcs/gcs_task_manager.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/gcs_server/gcs_worker_manager.h b/src/mock/ray/gcs/gcs_worker_manager.h similarity index 97% rename from src/mock/ray/gcs/gcs_server/gcs_worker_manager.h rename to src/mock/ray/gcs/gcs_worker_manager.h index deb459f53a65..e44259ed523f 100644 --- a/src/mock/ray/gcs/gcs_server/gcs_worker_manager.h +++ b/src/mock/ray/gcs/gcs_worker_manager.h @@ -16,7 +16,7 @@ #include -#include "ray/gcs/gcs_server/gcs_worker_manager.h" +#include "ray/gcs/gcs_worker_manager.h" namespace ray { namespace gcs { diff --git a/src/mock/ray/gcs/pubsub/gcs_pub_sub.h b/src/mock/ray/gcs/pubsub/gcs_pub_sub.h deleted file mode 100644 index 14252da567cc..000000000000 --- a/src/mock/ray/gcs/pubsub/gcs_pub_sub.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2021 The Ray Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace ray { -namespace gcs { - -class MockGcsPubSub : public GcsPubSub { - public: - MOCK_METHOD(Status, - Publish, - (const std::string &channel, - const std::string &id, - const std::string &data, - const StatusCallback &done), - (override)); -}; - -} // namespace gcs -} // namespace ray diff --git a/src/ray/gcs/gcs_server/BUILD.bazel b/src/ray/gcs/BUILD.bazel similarity index 99% rename from src/ray/gcs/gcs_server/BUILD.bazel rename to src/ray/gcs/BUILD.bazel index 695ddad003b1..5493169d3f6b 100644 --- a/src/ray/gcs/gcs_server/BUILD.bazel +++ b/src/ray/gcs/BUILD.bazel @@ -45,7 +45,7 @@ ray_cc_library( deps = [ "//src/ray/common:asio", "//src/ray/common:status", - "//src/ray/gcs/gcs_server:grpc_service_interfaces", + "//src/ray/gcs:grpc_service_interfaces", "//src/ray/protobuf:gcs_cc_proto", ], ) @@ -136,7 +136,7 @@ ray_cc_library( srcs = ["pubsub_handler.cc"], hdrs = ["pubsub_handler.h"], deps = [ - "//src/ray/gcs/gcs_server:grpc_service_interfaces", + "//src/ray/gcs:grpc_service_interfaces", "//src/ray/protobuf:gcs_service_cc_proto", "//src/ray/pubsub:gcs_publisher", "@com_google_absl//absl/container:flat_hash_map", diff --git a/src/ray/gcs/gcs_server/gcs_actor.cc b/src/ray/gcs/gcs_actor.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_actor.cc rename to src/ray/gcs/gcs_actor.cc index 2e9b6769b052..152294b05a94 100644 --- a/src/ray/gcs/gcs_server/gcs_actor.cc +++ b/src/ray/gcs/gcs_actor.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_actor.h" +#include "ray/gcs/gcs_actor.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_actor.h b/src/ray/gcs/gcs_actor.h similarity index 100% rename from src/ray/gcs/gcs_server/gcs_actor.h rename to src/ray/gcs/gcs_actor.h diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.cc b/src/ray/gcs/gcs_actor_manager.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_actor_manager.cc rename to src/ray/gcs/gcs_actor_manager.cc index dde37de4b3e0..8c5e69f0853e 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.cc +++ b/src/ray/gcs/gcs_actor_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_actor_manager.h" +#include "ray/gcs/gcs_actor_manager.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_actor_manager.h b/src/ray/gcs/gcs_actor_manager.h similarity index 98% rename from src/ray/gcs/gcs_server/gcs_actor_manager.h rename to src/ray/gcs/gcs_actor_manager.h index 1f95b8ccf31f..947c52f107bd 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_manager.h +++ b/src/ray/gcs/gcs_actor_manager.h @@ -27,13 +27,13 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" #include "ray/common/runtime_env_manager.h" -#include "ray/gcs/gcs_server/gcs_actor.h" -#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" -#include "ray/gcs/gcs_server/gcs_function_manager.h" -#include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" -#include "ray/gcs/gcs_server/usage_stats_client.h" +#include "ray/gcs/gcs_actor.h" +#include "ray/gcs/gcs_actor_scheduler.h" +#include "ray/gcs/gcs_function_manager.h" +#include "ray/gcs/gcs_init_data.h" +#include "ray/gcs/gcs_table_storage.h" +#include "ray/gcs/grpc_service_interfaces.h" +#include "ray/gcs/usage_stats_client.h" #include "ray/pubsub/gcs_publisher.h" #include "ray/rpc/worker/core_worker_client.h" #include "ray/rpc/worker/core_worker_client_pool.h" diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc b/src/ray/gcs/gcs_actor_scheduler.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_actor_scheduler.cc rename to src/ray/gcs/gcs_actor_scheduler.cc index d471ada70c15..79bfd2930770 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.cc +++ b/src/ray/gcs/gcs_actor_scheduler.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" +#include "ray/gcs/gcs_actor_scheduler.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h b/src/ray/gcs/gcs_actor_scheduler.h similarity index 99% rename from src/ray/gcs/gcs_server/gcs_actor_scheduler.h rename to src/ray/gcs/gcs_actor_scheduler.h index 1def8f69db62..432b93cf4c9a 100644 --- a/src/ray/gcs/gcs_server/gcs_actor_scheduler.h +++ b/src/ray/gcs/gcs_actor_scheduler.h @@ -24,9 +24,9 @@ #include "absl/container/flat_hash_set.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" -#include "ray/gcs/gcs_server/gcs_actor.h" -#include "ray/gcs/gcs_server/gcs_node_manager.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/gcs_actor.h" +#include "ray/gcs/gcs_node_manager.h" +#include "ray/gcs/gcs_table_storage.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/rpc/raylet/raylet_client_interface.h" #include "ray/rpc/raylet/raylet_client_pool.h" diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc b/src/ray/gcs/gcs_autoscaler_state_manager.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc rename to src/ray/gcs/gcs_autoscaler_state_manager.cc index 85ff43477a9d..6d0841745282 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc +++ b/src/ray/gcs/gcs_autoscaler_state_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_autoscaler_state_manager.h" +#include "ray/gcs/gcs_autoscaler_state_manager.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h b/src/ray/gcs/gcs_autoscaler_state_manager.h similarity index 96% rename from src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h rename to src/ray/gcs/gcs_autoscaler_state_manager.h index 9079b75bfbd8..2a459a7e51fe 100644 --- a/src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h +++ b/src/ray/gcs/gcs_autoscaler_state_manager.h @@ -23,13 +23,13 @@ #include "absl/container/flat_hash_map.h" #include "ray/common/asio/instrumented_io_context.h" -#include "ray/gcs/gcs_server/gcs_actor_manager.h" -#include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" -#include "ray/gcs/gcs_server/gcs_node_manager.h" -#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" -#include "ray/gcs/gcs_server/state_util.h" +#include "ray/gcs/gcs_actor_manager.h" +#include "ray/gcs/gcs_init_data.h" +#include "ray/gcs/gcs_kv_manager.h" +#include "ray/gcs/gcs_node_manager.h" +#include "ray/gcs/gcs_placement_group_manager.h" +#include "ray/gcs/grpc_service_interfaces.h" +#include "ray/gcs/state_util.h" #include "ray/pubsub/gcs_publisher.h" #include "ray/rpc/raylet/raylet_client_pool.h" #include "ray/util/thread_checker.h" diff --git a/src/ray/gcs/gcs_server/gcs_function_manager.h b/src/ray/gcs/gcs_function_manager.h similarity index 98% rename from src/ray/gcs/gcs_server/gcs_function_manager.h rename to src/ray/gcs/gcs_function_manager.h index 27380c052e83..3c861fe83f98 100644 --- a/src/ray/gcs/gcs_server/gcs_function_manager.h +++ b/src/ray/gcs/gcs_function_manager.h @@ -18,7 +18,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/constants.h" #include "ray/common/id.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_kv_manager.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/gcs_health_check_manager.cc b/src/ray/gcs/gcs_health_check_manager.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_health_check_manager.cc rename to src/ray/gcs/gcs_health_check_manager.cc index 109a313d5394..9cc54c945304 100644 --- a/src/ray/gcs/gcs_server/gcs_health_check_manager.cc +++ b/src/ray/gcs/gcs_health_check_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_health_check_manager.h" +#include "ray/gcs/gcs_health_check_manager.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_health_check_manager.h b/src/ray/gcs/gcs_health_check_manager.h similarity index 100% rename from src/ray/gcs/gcs_server/gcs_health_check_manager.h rename to src/ray/gcs/gcs_health_check_manager.h diff --git a/src/ray/gcs/gcs_server/gcs_init_data.cc b/src/ray/gcs/gcs_init_data.cc similarity index 98% rename from src/ray/gcs/gcs_server/gcs_init_data.cc rename to src/ray/gcs/gcs_init_data.cc index 1c7ed5389813..2f695f5e9188 100644 --- a/src/ray/gcs/gcs_server/gcs_init_data.cc +++ b/src/ray/gcs/gcs_init_data.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_init_data.h" +#include "ray/gcs/gcs_init_data.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_init_data.h b/src/ray/gcs/gcs_init_data.h similarity index 98% rename from src/ray/gcs/gcs_server/gcs_init_data.h rename to src/ray/gcs/gcs_init_data.h index d5c2c24c8f2c..1fcd02897346 100644 --- a/src/ray/gcs/gcs_server/gcs_init_data.h +++ b/src/ray/gcs/gcs_init_data.h @@ -17,7 +17,7 @@ #include "absl/container/flat_hash_map.h" #include "ray/common/asio/postable.h" #include "ray/common/id.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/gcs_table_storage.h" #include "src/ray/protobuf/gcs.pb.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.cc b/src/ray/gcs/gcs_job_manager.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_job_manager.cc rename to src/ray/gcs/gcs_job_manager.cc index 89af057e6832..a2eee9004c80 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.cc +++ b/src/ray/gcs/gcs_job_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_job_manager.h" +#include "ray/gcs/gcs_job_manager.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_job_manager.h b/src/ray/gcs/gcs_job_manager.h similarity index 96% rename from src/ray/gcs/gcs_server/gcs_job_manager.h rename to src/ray/gcs/gcs_job_manager.h index f6c534eb5c70..9a6db62a494b 100644 --- a/src/ray/gcs/gcs_server/gcs_job_manager.h +++ b/src/ray/gcs/gcs_job_manager.h @@ -22,11 +22,11 @@ #include "absl/container/flat_hash_map.h" #include "ray/common/runtime_env_manager.h" -#include "ray/gcs/gcs_server/gcs_function_manager.h" -#include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" +#include "ray/gcs/gcs_function_manager.h" +#include "ray/gcs/gcs_init_data.h" +#include "ray/gcs/gcs_kv_manager.h" +#include "ray/gcs/gcs_table_storage.h" +#include "ray/gcs/grpc_service_interfaces.h" #include "ray/observability/ray_event_recorder_interface.h" #include "ray/pubsub/gcs_publisher.h" #include "ray/rpc/worker/core_worker_client.h" diff --git a/src/ray/gcs/gcs_server/gcs_kv_manager.cc b/src/ray/gcs/gcs_kv_manager.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_kv_manager.cc rename to src/ray/gcs/gcs_kv_manager.cc index 26e77cf3bf1f..988604021f68 100644 --- a/src/ray/gcs/gcs_server/gcs_kv_manager.cc +++ b/src/ray/gcs/gcs_kv_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_kv_manager.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_kv_manager.h b/src/ray/gcs/gcs_kv_manager.h similarity index 99% rename from src/ray/gcs/gcs_server/gcs_kv_manager.h rename to src/ray/gcs/gcs_kv_manager.h index bacbf74a8768..6814b593e92e 100644 --- a/src/ray/gcs/gcs_server/gcs_kv_manager.h +++ b/src/ray/gcs/gcs_kv_manager.h @@ -22,7 +22,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/asio/postable.h" #include "ray/common/status.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" +#include "ray/gcs/grpc_service_interfaces.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.cc b/src/ray/gcs/gcs_node_manager.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_node_manager.cc rename to src/ray/gcs/gcs_node_manager.cc index 116c664c017c..ffa6a7fee49a 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.cc +++ b/src/ray/gcs/gcs_node_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_node_manager.h" +#include "ray/gcs/gcs_node_manager.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_node_manager.h b/src/ray/gcs/gcs_node_manager.h similarity index 98% rename from src/ray/gcs/gcs_server/gcs_node_manager.h rename to src/ray/gcs/gcs_node_manager.h index 65efb9a18cc2..fe463d6adf8d 100644 --- a/src/ray/gcs/gcs_server/gcs_node_manager.h +++ b/src/ray/gcs/gcs_node_manager.h @@ -23,9 +23,9 @@ #include "absl/container/flat_hash_map.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" -#include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" +#include "ray/gcs/gcs_init_data.h" +#include "ray/gcs/gcs_table_storage.h" +#include "ray/gcs/grpc_service_interfaces.h" #include "ray/pubsub/gcs_publisher.h" #include "ray/rpc/raylet/raylet_client_pool.h" #include "ray/stats/metric_defs.h" diff --git a/src/ray/gcs/gcs_server/gcs_placement_group.cc b/src/ray/gcs/gcs_placement_group.cc similarity index 98% rename from src/ray/gcs/gcs_server/gcs_placement_group.cc rename to src/ray/gcs/gcs_placement_group.cc index a238e5c51000..4de5b8fa229b 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group.cc +++ b/src/ray/gcs/gcs_placement_group.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_placement_group.h" +#include "ray/gcs/gcs_placement_group.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_placement_group.h b/src/ray/gcs/gcs_placement_group.h similarity index 100% rename from src/ray/gcs/gcs_server/gcs_placement_group.h rename to src/ray/gcs/gcs_placement_group.h diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc b/src/ray/gcs/gcs_placement_group_manager.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_placement_group_manager.cc rename to src/ray/gcs/gcs_placement_group_manager.cc index 1a67ce39518b..be3c8d21853a 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_manager.cc +++ b/src/ray/gcs/gcs_placement_group_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" +#include "ray/gcs/gcs_placement_group_manager.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_manager.h b/src/ray/gcs/gcs_placement_group_manager.h similarity index 97% rename from src/ray/gcs/gcs_server/gcs_placement_group_manager.h rename to src/ray/gcs/gcs_placement_group_manager.h index b9a7450fddde..aa3bfae4c6f4 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_manager.h +++ b/src/ray/gcs/gcs_placement_group_manager.h @@ -25,13 +25,13 @@ #include "absl/container/flat_hash_map.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" -#include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_placement_group.h" -#include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" -#include "ray/gcs/gcs_server/gcs_resource_manager.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" -#include "ray/gcs/gcs_server/usage_stats_client.h" +#include "ray/gcs/gcs_init_data.h" +#include "ray/gcs/gcs_placement_group.h" +#include "ray/gcs/gcs_placement_group_scheduler.h" +#include "ray/gcs/gcs_resource_manager.h" +#include "ray/gcs/gcs_table_storage.h" +#include "ray/gcs/grpc_service_interfaces.h" +#include "ray/gcs/usage_stats_client.h" #include "ray/util/counter_map.h" #include "ray/util/exponential_backoff.h" #include "src/ray/protobuf/gcs_service.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc b/src/ray/gcs/gcs_placement_group_scheduler.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc rename to src/ray/gcs/gcs_placement_group_scheduler.cc index 68e29aadcc3a..f14f380d6018 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.cc +++ b/src/ray/gcs/gcs_placement_group_scheduler.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" +#include "ray/gcs/gcs_placement_group_scheduler.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h b/src/ray/gcs/gcs_placement_group_scheduler.h similarity index 99% rename from src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h rename to src/ray/gcs/gcs_placement_group_scheduler.h index a031c594c306..6e43239d7967 100644 --- a/src/ray/gcs/gcs_server/gcs_placement_group_scheduler.h +++ b/src/ray/gcs/gcs_placement_group_scheduler.h @@ -24,9 +24,9 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/bundle_location_index.h" #include "ray/common/id.h" -#include "ray/gcs/gcs_server/gcs_node_manager.h" -#include "ray/gcs/gcs_server/gcs_placement_group.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/gcs_node_manager.h" +#include "ray/gcs/gcs_placement_group.h" +#include "ray/gcs/gcs_table_storage.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/raylet/scheduling/policy/scheduling_context.h" #include "ray/rpc/raylet/raylet_client_interface.h" diff --git a/src/ray/gcs/gcs_server/gcs_ray_event_converter.cc b/src/ray/gcs/gcs_ray_event_converter.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_ray_event_converter.cc rename to src/ray/gcs/gcs_ray_event_converter.cc index 43b76e8e001f..d07a51ca3557 100644 --- a/src/ray/gcs/gcs_server/gcs_ray_event_converter.cc +++ b/src/ray/gcs/gcs_ray_event_converter.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_ray_event_converter.h" +#include "ray/gcs/gcs_ray_event_converter.h" #include diff --git a/src/ray/gcs/gcs_server/gcs_ray_event_converter.h b/src/ray/gcs/gcs_ray_event_converter.h similarity index 100% rename from src/ray/gcs/gcs_server/gcs_ray_event_converter.h rename to src/ray/gcs/gcs_ray_event_converter.h diff --git a/src/ray/gcs/gcs_server/gcs_resource_manager.cc b/src/ray/gcs/gcs_resource_manager.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_resource_manager.cc rename to src/ray/gcs/gcs_resource_manager.cc index 0dca317fda20..b9303b58ee55 100644 --- a/src/ray/gcs/gcs_server/gcs_resource_manager.cc +++ b/src/ray/gcs/gcs_resource_manager.cc @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_resource_manager.h" +#include "ray/gcs/gcs_resource_manager.h" #include #include #include #include "ray/common/ray_config.h" -#include "ray/gcs/gcs_server/state_util.h" +#include "ray/gcs/state_util.h" #include "ray/util/logging.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/gcs_resource_manager.h b/src/ray/gcs/gcs_resource_manager.h similarity index 98% rename from src/ray/gcs/gcs_server/gcs_resource_manager.h rename to src/ray/gcs/gcs_resource_manager.h index 78ced7dd01ab..4d477ba1cb1f 100644 --- a/src/ray/gcs/gcs_server/gcs_resource_manager.h +++ b/src/ray/gcs/gcs_resource_manager.h @@ -23,9 +23,9 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" #include "ray/common/ray_syncer/ray_syncer.h" -#include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_node_manager.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" +#include "ray/gcs/gcs_init_data.h" +#include "ray/gcs/gcs_node_manager.h" +#include "ray/gcs/grpc_service_interfaces.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_server.cc b/src/ray/gcs/gcs_server.cc similarity index 98% rename from src/ray/gcs/gcs_server/gcs_server.cc rename to src/ray/gcs/gcs_server.cc index 1249b6298d7f..6dcd258465f2 100644 --- a/src/ray/gcs/gcs_server/gcs_server.cc +++ b/src/ray/gcs/gcs_server.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_server.h" +#include "ray/gcs/gcs_server.h" #include #include @@ -23,18 +23,18 @@ #include "ray/common/asio/asio_util.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/ray_config.h" -#include "ray/gcs/gcs_server/gcs_actor_manager.h" -#include "ray/gcs/gcs_server/gcs_autoscaler_state_manager.h" -#include "ray/gcs/gcs_server/gcs_job_manager.h" -#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" -#include "ray/gcs/gcs_server/gcs_resource_manager.h" -#include "ray/gcs/gcs_server/gcs_worker_manager.h" -#include "ray/gcs/gcs_server/grpc_services.h" -#include "ray/gcs/gcs_server/store_client_kv.h" +#include "ray/gcs/gcs_actor_manager.h" +#include "ray/gcs/gcs_autoscaler_state_manager.h" +#include "ray/gcs/gcs_job_manager.h" +#include "ray/gcs/gcs_placement_group_manager.h" +#include "ray/gcs/gcs_resource_manager.h" +#include "ray/gcs/gcs_worker_manager.h" +#include "ray/gcs/grpc_services.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/store_client/observable_store_client.h" #include "ray/gcs/store_client/redis_store_client.h" #include "ray/gcs/store_client/store_client.h" +#include "ray/gcs/store_client_kv.h" #include "ray/pubsub/publisher.h" #include "ray/rpc/raylet/raylet_client.h" #include "ray/stats/stats.h" diff --git a/src/ray/gcs/gcs_server/gcs_server.h b/src/ray/gcs/gcs_server.h similarity index 94% rename from src/ray/gcs/gcs_server/gcs_server.h rename to src/ray/gcs/gcs_server.h index 31c3f3a09fb4..c9541defe3c1 100644 --- a/src/ray/gcs/gcs_server/gcs_server.h +++ b/src/ray/gcs/gcs_server.h @@ -22,20 +22,20 @@ #include "ray/common/asio/postable.h" #include "ray/common/ray_syncer/ray_syncer.h" #include "ray/common/runtime_env_manager.h" -#include "ray/gcs/gcs_server/gcs_function_manager.h" -#include "ray/gcs/gcs_server/gcs_health_check_manager.h" -#include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" -#include "ray/gcs/gcs_server/gcs_resource_manager.h" -#include "ray/gcs/gcs_server/gcs_server_io_context_policy.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/gcs_server/gcs_task_manager.h" -#include "ray/gcs/gcs_server/pubsub_handler.h" -#include "ray/gcs/gcs_server/runtime_env_handler.h" -#include "ray/gcs/gcs_server/usage_stats_client.h" +#include "ray/gcs/gcs_function_manager.h" +#include "ray/gcs/gcs_health_check_manager.h" +#include "ray/gcs/gcs_init_data.h" +#include "ray/gcs/gcs_kv_manager.h" +#include "ray/gcs/gcs_resource_manager.h" +#include "ray/gcs/gcs_server_io_context_policy.h" +#include "ray/gcs/gcs_table_storage.h" +#include "ray/gcs/gcs_task_manager.h" +#include "ray/gcs/pubsub_handler.h" +#include "ray/gcs/runtime_env_handler.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/store_client/observable_store_client.h" #include "ray/gcs/store_client/redis_store_client.h" +#include "ray/gcs/usage_stats_client.h" #include "ray/observability/ray_event_recorder.h" #include "ray/pubsub/gcs_publisher.h" #include "ray/raylet/scheduling/cluster_lease_manager.h" diff --git a/src/ray/gcs/gcs_server/gcs_server_io_context_policy.h b/src/ray/gcs/gcs_server_io_context_policy.h similarity index 98% rename from src/ray/gcs/gcs_server/gcs_server_io_context_policy.h rename to src/ray/gcs/gcs_server_io_context_policy.h index 95e5e550634f..f8a504762162 100644 --- a/src/ray/gcs/gcs_server/gcs_server_io_context_policy.h +++ b/src/ray/gcs/gcs_server_io_context_policy.h @@ -19,7 +19,7 @@ #include #include "ray/common/ray_syncer/ray_syncer.h" -#include "ray/gcs/gcs_server/gcs_task_manager.h" +#include "ray/gcs/gcs_task_manager.h" #include "ray/observability/ray_event_recorder.h" #include "ray/pubsub/gcs_publisher.h" #include "ray/util/array.h" diff --git a/src/ray/gcs/gcs_server/gcs_server_main.cc b/src/ray/gcs/gcs_server_main.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_server_main.cc rename to src/ray/gcs/gcs_server_main.cc index 712acb8d9768..19d155e407b9 100644 --- a/src/ray/gcs/gcs_server/gcs_server_main.cc +++ b/src/ray/gcs/gcs_server_main.cc @@ -20,7 +20,7 @@ #include "gflags/gflags.h" #include "ray/common/ray_config.h" -#include "ray/gcs/gcs_server/gcs_server.h" +#include "ray/gcs/gcs_server.h" #include "ray/gcs/store_client/redis_store_client.h" #include "ray/stats/stats.h" #include "ray/util/event.h" diff --git a/src/ray/gcs/gcs_server/gcs_table_storage.cc b/src/ray/gcs/gcs_table_storage.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_table_storage.cc rename to src/ray/gcs/gcs_table_storage.cc index 203e8c0e4848..549a0c1733fc 100644 --- a/src/ray/gcs/gcs_server/gcs_table_storage.cc +++ b/src/ray/gcs/gcs_table_storage.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/gcs_table_storage.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_table_storage.h b/src/ray/gcs/gcs_table_storage.h similarity index 100% rename from src/ray/gcs/gcs_server/gcs_table_storage.h rename to src/ray/gcs/gcs_table_storage.h diff --git a/src/ray/gcs/gcs_server/gcs_task_manager.cc b/src/ray/gcs/gcs_task_manager.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_task_manager.cc rename to src/ray/gcs/gcs_task_manager.cc index bbea0fe45f37..250a88c9fe40 100644 --- a/src/ray/gcs/gcs_server/gcs_task_manager.cc +++ b/src/ray/gcs/gcs_task_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_task_manager.h" +#include "ray/gcs/gcs_task_manager.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_task_manager.h b/src/ray/gcs/gcs_task_manager.h similarity index 99% rename from src/ray/gcs/gcs_server/gcs_task_manager.h rename to src/ray/gcs/gcs_task_manager.h index 2e9b6f3dc877..ee0ebe3110f3 100644 --- a/src/ray/gcs/gcs_server/gcs_task_manager.h +++ b/src/ray/gcs/gcs_task_manager.h @@ -25,9 +25,9 @@ #include "absl/container/flat_hash_set.h" #include "absl/synchronization/mutex.h" #include "ray/common/protobuf_utils.h" -#include "ray/gcs/gcs_server/gcs_ray_event_converter.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" -#include "ray/gcs/gcs_server/usage_stats_client.h" +#include "ray/gcs/gcs_ray_event_converter.h" +#include "ray/gcs/grpc_service_interfaces.h" +#include "ray/gcs/usage_stats_client.h" #include "ray/stats/metric_defs.h" #include "ray/util/counter_map.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/gcs_worker_manager.cc b/src/ray/gcs/gcs_worker_manager.cc similarity index 99% rename from src/ray/gcs/gcs_server/gcs_worker_manager.cc rename to src/ray/gcs/gcs_worker_manager.cc index aa13eeda4819..3819112f75c6 100644 --- a/src/ray/gcs/gcs_server/gcs_worker_manager.cc +++ b/src/ray/gcs/gcs_worker_manager.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_worker_manager.h" +#include "ray/gcs/gcs_worker_manager.h" #include #include diff --git a/src/ray/gcs/gcs_server/gcs_worker_manager.h b/src/ray/gcs/gcs_worker_manager.h similarity index 94% rename from src/ray/gcs/gcs_server/gcs_worker_manager.h rename to src/ray/gcs/gcs_worker_manager.h index a5823283d9f7..062fb25d1d3d 100644 --- a/src/ray/gcs/gcs_server/gcs_worker_manager.h +++ b/src/ray/gcs/gcs_worker_manager.h @@ -16,10 +16,10 @@ #include -#include "ray/gcs/gcs_server/gcs_kv_manager.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" -#include "ray/gcs/gcs_server/usage_stats_client.h" +#include "ray/gcs/gcs_kv_manager.h" +#include "ray/gcs/gcs_table_storage.h" +#include "ray/gcs/grpc_service_interfaces.h" +#include "ray/gcs/usage_stats_client.h" #include "ray/pubsub/gcs_publisher.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/grpc_service_interfaces.h b/src/ray/gcs/grpc_service_interfaces.h similarity index 100% rename from src/ray/gcs/gcs_server/grpc_service_interfaces.h rename to src/ray/gcs/grpc_service_interfaces.h diff --git a/src/ray/gcs/gcs_server/grpc_services.cc b/src/ray/gcs/grpc_services.cc similarity index 99% rename from src/ray/gcs/gcs_server/grpc_services.cc rename to src/ray/gcs/grpc_services.cc index b868c93f9425..012f81537a55 100644 --- a/src/ray/gcs/gcs_server/grpc_services.cc +++ b/src/ray/gcs/grpc_services.cc @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/grpc_services.h" +#include "ray/gcs/grpc_services.h" #include #include diff --git a/src/ray/gcs/gcs_server/grpc_services.h b/src/ray/gcs/grpc_services.h similarity index 99% rename from src/ray/gcs/gcs_server/grpc_services.h rename to src/ray/gcs/grpc_services.h index 12e745196163..9a7862d334b5 100644 --- a/src/ray/gcs/gcs_server/grpc_services.h +++ b/src/ray/gcs/grpc_services.h @@ -27,7 +27,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/id.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" +#include "ray/gcs/grpc_service_interfaces.h" #include "ray/rpc/grpc_server.h" #include "ray/rpc/server_call.h" #include "src/ray/protobuf/autoscaler.grpc.pb.h" diff --git a/src/ray/gcs/gcs_server/pubsub_handler.cc b/src/ray/gcs/pubsub_handler.cc similarity index 99% rename from src/ray/gcs/gcs_server/pubsub_handler.cc rename to src/ray/gcs/pubsub_handler.cc index 14b0291c6126..1e281ba1b56c 100644 --- a/src/ray/gcs/gcs_server/pubsub_handler.cc +++ b/src/ray/gcs/pubsub_handler.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/pubsub_handler.h" +#include "ray/gcs/pubsub_handler.h" #include #include diff --git a/src/ray/gcs/gcs_server/pubsub_handler.h b/src/ray/gcs/pubsub_handler.h similarity index 97% rename from src/ray/gcs/gcs_server/pubsub_handler.h rename to src/ray/gcs/pubsub_handler.h index 6808c254ef2c..f69b8e2a50f1 100644 --- a/src/ray/gcs/gcs_server/pubsub_handler.h +++ b/src/ray/gcs/pubsub_handler.h @@ -18,7 +18,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" +#include "ray/gcs/grpc_service_interfaces.h" #include "ray/pubsub/gcs_publisher.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/runtime_env_handler.cc b/src/ray/gcs/runtime_env_handler.cc similarity index 96% rename from src/ray/gcs/gcs_server/runtime_env_handler.cc rename to src/ray/gcs/runtime_env_handler.cc index 83aa0c5c3538..b71604b9cecc 100644 --- a/src/ray/gcs/gcs_server/runtime_env_handler.cc +++ b/src/ray/gcs/runtime_env_handler.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/runtime_env_handler.h" +#include "ray/gcs/runtime_env_handler.h" #include diff --git a/src/ray/gcs/gcs_server/runtime_env_handler.h b/src/ray/gcs/runtime_env_handler.h similarity index 96% rename from src/ray/gcs/gcs_server/runtime_env_handler.h rename to src/ray/gcs/runtime_env_handler.h index 15eb88ee95ca..4211fb95030a 100644 --- a/src/ray/gcs/gcs_server/runtime_env_handler.h +++ b/src/ray/gcs/runtime_env_handler.h @@ -19,7 +19,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/runtime_env_manager.h" -#include "ray/gcs/gcs_server/grpc_service_interfaces.h" +#include "ray/gcs/grpc_service_interfaces.h" namespace ray { namespace gcs { diff --git a/src/ray/gcs/gcs_server/state_util.cc b/src/ray/gcs/state_util.cc similarity index 97% rename from src/ray/gcs/gcs_server/state_util.cc rename to src/ray/gcs/state_util.cc index b1f41b393682..64576c793687 100644 --- a/src/ray/gcs/gcs_server/state_util.cc +++ b/src/ray/gcs/state_util.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/state_util.h" +#include "ray/gcs/state_util.h" #include diff --git a/src/ray/gcs/gcs_server/state_util.h b/src/ray/gcs/state_util.h similarity index 100% rename from src/ray/gcs/gcs_server/state_util.h rename to src/ray/gcs/state_util.h diff --git a/src/ray/gcs/gcs_server/store_client_kv.cc b/src/ray/gcs/store_client_kv.cc similarity index 99% rename from src/ray/gcs/gcs_server/store_client_kv.cc rename to src/ray/gcs/store_client_kv.cc index 1586087246b5..31297c49536e 100644 --- a/src/ray/gcs/gcs_server/store_client_kv.cc +++ b/src/ray/gcs/store_client_kv.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/store_client_kv.h" +#include "ray/gcs/store_client_kv.h" #include #include diff --git a/src/ray/gcs/gcs_server/store_client_kv.h b/src/ray/gcs/store_client_kv.h similarity index 97% rename from src/ray/gcs/gcs_server/store_client_kv.h rename to src/ray/gcs/store_client_kv.h index 9d122b85184e..295ad387a8e6 100644 --- a/src/ray/gcs/gcs_server/store_client_kv.h +++ b/src/ray/gcs/store_client_kv.h @@ -20,7 +20,7 @@ #include #include "ray/common/asio/postable.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_kv_manager.h" #include "ray/gcs/store_client/store_client.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/BUILD.bazel b/src/ray/gcs/tests/BUILD.bazel similarity index 80% rename from src/ray/gcs/gcs_server/tests/BUILD.bazel rename to src/ray/gcs/tests/BUILD.bazel index feae8305f242..012661e15b01 100644 --- a/src/ray/gcs/gcs_server/tests/BUILD.bazel +++ b/src/ray/gcs/tests/BUILD.bazel @@ -6,7 +6,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//:ray_mock", - "//src/ray/gcs/gcs_server:gcs_function_manager", + "//src/ray/gcs:gcs_function_manager", "@com_google_googletest//:gtest_main", ], ) @@ -21,7 +21,7 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_placement_group_manager", + "//src/ray/gcs:gcs_placement_group_manager", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], @@ -48,7 +48,7 @@ ray_cc_test( ], deps = [ "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs:gcs_server_lib", "@com_google_googletest//:gtest", ], ) @@ -70,8 +70,8 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_kv_manager", - "//src/ray/gcs/gcs_server:gcs_store_client_kv", + "//src/ray/gcs:gcs_kv_manager", + "//src/ray/gcs:gcs_store_client_kv", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/store_client:redis_store_client", "@com_google_googletest//:gtest", @@ -89,7 +89,7 @@ ray_cc_test( "team:core", ], deps = [ - "//src/ray/gcs/gcs_server:gcs_health_check_manager", + "//src/ray/gcs:gcs_health_check_manager", "//src/ray/rpc:grpc_server", "//src/ray/util:network_util", "@boost//:thread", @@ -109,7 +109,7 @@ ray_cc_test( "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_node_manager", + "//src/ray/gcs:gcs_node_manager", "@com_google_googletest//:gtest_main", ], ) @@ -125,8 +125,8 @@ ray_cc_test( "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_job_manager", - "//src/ray/gcs/gcs_server:gcs_kv_manager", + "//src/ray/gcs:gcs_job_manager", + "//src/ray/gcs:gcs_kv_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/observability:fake_ray_event_recorder", "@com_google_googletest//:gtest_main", @@ -144,7 +144,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:protobuf_utils", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_task_manager", + "//src/ray/gcs:gcs_task_manager", "@com_google_googletest//:gtest_main", ], ) @@ -163,7 +163,7 @@ ray_cc_test( "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_placement_group_manager", + "//src/ray/gcs:gcs_placement_group_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", @@ -185,11 +185,11 @@ ray_cc_test( "//src/fakes/ray/rpc/worker:fake_core_worker_client", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_node_manager", - "//src/ray/gcs/gcs_server:gcs_placement_group", - "//src/ray/gcs/gcs_server:gcs_placement_group_scheduler", - "//src/ray/gcs/gcs_server:gcs_resource_manager", - "//src/ray/gcs/gcs_server:gcs_table_storage", + "//src/ray/gcs:gcs_node_manager", + "//src/ray/gcs:gcs_placement_group", + "//src/ray/gcs:gcs_placement_group_scheduler", + "//src/ray/gcs:gcs_resource_manager", + "//src/ray/gcs:gcs_table_storage", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", @@ -208,9 +208,9 @@ ray_cc_test( "//src/fakes/ray/rpc/worker:fake_core_worker_client", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_actor", - "//src/ray/gcs/gcs_server:gcs_actor_scheduler", - "//src/ray/gcs/gcs_server:gcs_resource_manager", + "//src/ray/gcs:gcs_actor", + "//src/ray/gcs:gcs_actor_scheduler", + "//src/ray/gcs:gcs_resource_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", @@ -227,8 +227,8 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_actor", - "//src/ray/gcs/gcs_server:gcs_actor_scheduler", + "//src/ray/gcs:gcs_actor", + "//src/ray/gcs:gcs_actor_scheduler", "//src/ray/util:counter_map", "@com_google_googletest//:gtest_main", ], @@ -248,10 +248,10 @@ ray_cc_test( "//src/ray/common:asio", "//src/ray/common:runtime_env", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_actor", - "//src/ray/gcs/gcs_server:gcs_actor_manager", - "//src/ray/gcs/gcs_server:gcs_actor_scheduler", - "//src/ray/gcs/gcs_server:gcs_function_manager", + "//src/ray/gcs:gcs_actor", + "//src/ray/gcs:gcs_actor_manager", + "//src/ray/gcs:gcs_actor_scheduler", + "//src/ray/gcs:gcs_function_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/pubsub:publisher", "@com_google_googletest//:gtest_main", @@ -268,8 +268,8 @@ ray_cc_test( deps = [ "//src/mock/ray/pubsub:mock_publisher", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_store_client_kv", - "//src/ray/gcs/gcs_server:gcs_worker_manager", + "//src/ray/gcs:gcs_store_client_kv", + "//src/ray/gcs:gcs_worker_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/util:process", "@com_google_googletest//:gtest_main", @@ -304,7 +304,7 @@ ray_cc_test( deps = [ ":gcs_table_storage_test_lib", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_table_storage", + "//src/ray/gcs:gcs_table_storage", "//src/ray/gcs/store_client/tests:store_client_test_lib", "@com_google_googletest//:gtest", ], @@ -318,7 +318,7 @@ ray_cc_test( deps = [ ":gcs_table_storage_test_lib", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_table_storage", + "//src/ray/gcs:gcs_table_storage", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/gcs/store_client/tests:store_client_test_lib", "@com_google_googletest//:gtest_main", @@ -339,10 +339,10 @@ ray_cc_test( "//src/ray/common:asio", "//src/ray/common:protobuf_utils", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_autoscaler_state_manager", - "//src/ray/gcs/gcs_server:gcs_init_data", - "//src/ray/gcs/gcs_server:gcs_resource_manager", - "//src/ray/gcs/gcs_server:gcs_store_client_kv", + "//src/ray/gcs:gcs_autoscaler_state_manager", + "//src/ray/gcs:gcs_init_data", + "//src/ray/gcs:gcs_resource_manager", + "//src/ray/gcs:gcs_store_client_kv", "//src/ray/raylet/scheduling:cluster_resource_manager", "@com_google_googletest//:gtest_main", ], @@ -358,8 +358,8 @@ ray_cc_test( deps = [ "//:ray_mock", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_node_manager", - "//src/ray/gcs/gcs_server:gcs_resource_manager", + "//src/ray/gcs:gcs_node_manager", + "//src/ray/gcs:gcs_resource_manager", "//src/ray/raylet/scheduling:cluster_resource_manager", "@com_google_googletest//:gtest_main", ], @@ -376,7 +376,7 @@ ray_cc_test( "//:ray_mock", "//src/ray/common:asio", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_usage_stats_client", + "//src/ray/gcs:gcs_usage_stats_client", "@com_google_googletest//:gtest_main", ], ) @@ -393,8 +393,8 @@ ray_cc_test( "//:ray_mock", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_job_manager", - "//src/ray/gcs/gcs_server:gcs_kv_manager", + "//src/ray/gcs:gcs_job_manager", + "//src/ray/gcs:gcs_kv_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/observability:fake_ray_event_recorder", "@com_google_googletest//:gtest_main", @@ -414,10 +414,10 @@ ray_cc_test( "//src/ray/common:asio", "//src/ray/common:runtime_env", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_actor", - "//src/ray/gcs/gcs_server:gcs_actor_manager", - "//src/ray/gcs/gcs_server:gcs_actor_scheduler", - "//src/ray/gcs/gcs_server:gcs_function_manager", + "//src/ray/gcs:gcs_actor", + "//src/ray/gcs:gcs_actor_manager", + "//src/ray/gcs:gcs_actor_scheduler", + "//src/ray/gcs:gcs_function_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/pubsub:publisher", "//src/ray/rpc:core_worker_client", @@ -438,7 +438,7 @@ ray_cc_test( "//src/fakes/ray/rpc/raylet:fake_raylet_client", "//src/mock/ray/pubsub:mock_publisher", "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_node_manager", + "//src/ray/gcs:gcs_node_manager", "//src/ray/gcs/store_client:in_memory_store_client", "//src/ray/util:string_utils", "@com_google_googletest//:gtest", @@ -451,7 +451,7 @@ ray_cc_test( srcs = ["gcs_ray_event_converter_test.cc"], tags = ["team:core"], deps = [ - "//src/ray/gcs/gcs_server:gcs_ray_event_converter", + "//src/ray/gcs:gcs_ray_event_converter", "@com_google_googletest//:gtest_main", ], ) diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc b/src/ray/gcs/tests/export_api/gcs_actor_manager_export_event_test.cc similarity index 98% rename from src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc rename to src/ray/gcs/tests/export_api/gcs_actor_manager_export_event_test.cc index 4622270460a2..0a154594859c 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_actor_manager_export_event_test.cc +++ b/src/ray/gcs/tests/export_api/gcs_actor_manager_export_event_test.cc @@ -22,14 +22,14 @@ #include #include -#include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" -#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" +#include "mock/ray/gcs/gcs_kv_manager.h" +#include "mock/ray/gcs/gcs_node_manager.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/runtime_env_manager.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_actor.h" -#include "ray/gcs/gcs_server/gcs_actor_manager.h" -#include "ray/gcs/gcs_server/gcs_function_manager.h" +#include "ray/gcs/gcs_actor.h" +#include "ray/gcs/gcs_actor_manager.h" +#include "ray/gcs/gcs_function_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/pubsub/publisher.h" #include "ray/rpc/worker/core_worker_client.h" diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc b/src/ray/gcs/tests/export_api/gcs_job_manager_export_event_test.cc similarity index 98% rename from src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc rename to src/ray/gcs/tests/export_api/gcs_job_manager_export_event_test.cc index 9a310d1f923e..afaf73eeac39 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_job_manager_export_event_test.cc +++ b/src/ray/gcs/tests/export_api/gcs_job_manager_export_event_test.cc @@ -19,12 +19,12 @@ #include #include -#include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" +#include "mock/ray/gcs/gcs_kv_manager.h" #include "mock/ray/pubsub/publisher.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_job_manager.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_job_manager.h" +#include "ray/gcs/gcs_kv_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/observability/fake_ray_event_recorder.h" diff --git a/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc b/src/ray/gcs/tests/export_api/gcs_node_manager_export_event_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc rename to src/ray/gcs/tests/export_api/gcs_node_manager_export_event_test.cc index aa9731d2823f..5c2ceb17f61b 100644 --- a/src/ray/gcs/gcs_server/tests/export_api/gcs_node_manager_export_event_test.cc +++ b/src/ray/gcs/tests/export_api/gcs_node_manager_export_event_test.cc @@ -24,7 +24,7 @@ #include "fakes/ray/rpc/raylet/raylet_client.h" #include "mock/ray/pubsub/publisher.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_node_manager.h" +#include "ray/gcs/gcs_node_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/util/event.h" #include "ray/util/string_utils.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc b/src/ray/gcs/tests/gcs_actor_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc rename to src/ray/gcs/tests/gcs_actor_manager_test.cc index 1c4a60bff97c..bcc7b74b5730 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_manager_test.cc +++ b/src/ray/gcs/tests/gcs_actor_manager_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_actor_manager.h" +#include "ray/gcs/gcs_actor_manager.h" #include @@ -22,14 +22,14 @@ #include #include -#include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" -#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" +#include "mock/ray/gcs/gcs_kv_manager.h" +#include "mock/ray/gcs/gcs_node_manager.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/runtime_env_manager.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_actor.h" -#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" -#include "ray/gcs/gcs_server/gcs_function_manager.h" +#include "ray/gcs/gcs_actor.h" +#include "ray/gcs/gcs_actor_scheduler.h" +#include "ray/gcs/gcs_function_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/pubsub/publisher.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc b/src/ray/gcs/tests/gcs_actor_scheduler_mock_test.cc similarity index 98% rename from src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc rename to src/ray/gcs/tests/gcs_actor_scheduler_mock_test.cc index 4aeb171c36d1..cbe43ab9f513 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_mock_test.cc +++ b/src/ray/gcs/tests/gcs_actor_scheduler_mock_test.cc @@ -18,13 +18,13 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" +#include "mock/ray/gcs/gcs_node_manager.h" #include "mock/ray/gcs/store_client/store_client.h" #include "mock/ray/raylet_client/raylet_client.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_actor.h" -#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" +#include "ray/gcs/gcs_actor.h" +#include "ray/gcs/gcs_actor_scheduler.h" #include "ray/util/counter_map.h" using namespace ::testing; // NOLINT diff --git a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc b/src/ray/gcs/tests/gcs_actor_scheduler_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc rename to src/ray/gcs/tests/gcs_actor_scheduler_test.cc index d891ce55caba..689007ffc67e 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_actor_scheduler_test.cc +++ b/src/ray/gcs/tests/gcs_actor_scheduler_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" +#include "ray/gcs/gcs_actor_scheduler.h" #include @@ -27,9 +27,9 @@ #include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/asio_util.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_actor.h" -#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" -#include "ray/gcs/gcs_server/gcs_resource_manager.h" +#include "ray/gcs/gcs_actor.h" +#include "ray/gcs/gcs_actor_scheduler.h" +#include "ray/gcs/gcs_resource_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/util/counter_map.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc b/src/ray/gcs/tests/gcs_autoscaler_state_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc rename to src/ray/gcs/tests/gcs_autoscaler_state_manager_test.cc index 52d244ae3a35..ca07d5a72ab5 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_autoscaler_state_manager_test.cc +++ b/src/ray/gcs/tests/gcs_autoscaler_state_manager_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_autoscaler_state_manager.h" +#include "ray/gcs/gcs_autoscaler_state_manager.h" #include #include @@ -26,17 +26,17 @@ #include #include "fakes/ray/rpc/raylet/raylet_client.h" -#include "mock/ray/gcs/gcs_server/gcs_actor_manager.h" -#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" -#include "mock/ray/gcs/gcs_server/gcs_placement_group_manager.h" +#include "mock/ray/gcs/gcs_actor_manager.h" +#include "mock/ray/gcs/gcs_node_manager.h" +#include "mock/ray/gcs/gcs_placement_group_manager.h" #include "mock/ray/gcs/store_client/store_client.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/protobuf_utils.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_init_data.h" -#include "ray/gcs/gcs_server/gcs_resource_manager.h" -#include "ray/gcs/gcs_server/store_client_kv.h" +#include "ray/gcs/gcs_init_data.h" +#include "ray/gcs/gcs_resource_manager.h" +#include "ray/gcs/store_client_kv.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc b/src/ray/gcs/tests/gcs_function_manager_test.cc similarity index 97% rename from src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc rename to src/ray/gcs/tests/gcs_function_manager_test.cc index 10b8409dc844..b24eb51a8a21 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_function_manager_test.cc +++ b/src/ray/gcs/tests/gcs_function_manager_test.cc @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_function_manager.h" +#include "ray/gcs/gcs_function_manager.h" #include #include -#include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" +#include "mock/ray/gcs/gcs_kv_manager.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc b/src/ray/gcs/tests/gcs_health_check_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc rename to src/ray/gcs/tests/gcs_health_check_manager_test.cc index 8ca66a12522f..8c6d5e485e8d 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_health_check_manager_test.cc +++ b/src/ray/gcs/tests/gcs_health_check_manager_test.cc @@ -34,7 +34,7 @@ using namespace boost::asio::ip; // NOLINT #include #include "gtest/gtest.h" -#include "ray/gcs/gcs_server/gcs_health_check_manager.h" +#include "ray/gcs/gcs_health_check_manager.h" #include "ray/util/network_util.h" int GetFreePort() { diff --git a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc b/src/ray/gcs/tests/gcs_job_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc rename to src/ray/gcs/tests/gcs_job_manager_test.cc index 4c4227f5d929..e2aae44e87d1 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_job_manager_test.cc +++ b/src/ray/gcs/tests/gcs_job_manager_test.cc @@ -12,17 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_job_manager.h" +#include "ray/gcs/gcs_job_manager.h" #include #include #include "gtest/gtest.h" -#include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" +#include "mock/ray/gcs/gcs_kv_manager.h" #include "mock/ray/pubsub/publisher.h" #include "mock/ray/rpc/worker/core_worker_client.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_kv_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/observability/fake_ray_event_recorder.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc b/src/ray/gcs/tests/gcs_kv_manager_test.cc similarity index 98% rename from src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc rename to src/ray/gcs/tests/gcs_kv_manager_test.cc index 26d9f1a56fe5..e33795bc0869 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_kv_manager_test.cc +++ b/src/ray/gcs/tests/gcs_kv_manager_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_kv_manager.h" #include #include @@ -22,9 +22,9 @@ #include "gtest/gtest.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/store_client_kv.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/gcs/store_client/redis_store_client.h" +#include "ray/gcs/store_client_kv.h" class GcsKVManagerTest : public ::testing::TestWithParam { public: diff --git a/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc b/src/ray/gcs/tests/gcs_node_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc rename to src/ray/gcs/tests/gcs_node_manager_test.cc index e1ec7cf7398a..0406017f31cb 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_node_manager_test.cc +++ b/src/ray/gcs/tests/gcs_node_manager_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_node_manager.h" +#include "ray/gcs/gcs_node_manager.h" #include diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc b/src/ray/gcs/tests/gcs_placement_group_manager_mock_test.cc similarity index 97% rename from src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc rename to src/ray/gcs/tests/gcs_placement_group_manager_mock_test.cc index ad8db572aa44..66987d0dae74 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_mock_test.cc +++ b/src/ray/gcs/tests/gcs_placement_group_manager_mock_test.cc @@ -18,12 +18,12 @@ #include #include -#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" -#include "mock/ray/gcs/gcs_server/gcs_placement_group_scheduler.h" -#include "mock/ray/gcs/gcs_server/gcs_resource_manager.h" +#include "mock/ray/gcs/gcs_node_manager.h" +#include "mock/ray/gcs/gcs_placement_group_scheduler.h" +#include "mock/ray/gcs/gcs_resource_manager.h" #include "mock/ray/gcs/store_client/store_client.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" +#include "ray/gcs/gcs_placement_group_manager.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" #include "ray/util/counter_map.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc b/src/ray/gcs/tests/gcs_placement_group_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc rename to src/ray/gcs/tests/gcs_placement_group_manager_test.cc index fc0a0dd953fa..bc0126e33e79 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_manager_test.cc +++ b/src/ray/gcs/tests/gcs_placement_group_manager_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_placement_group_manager.h" +#include "ray/gcs/gcs_placement_group_manager.h" #include @@ -20,7 +20,7 @@ #include #include -#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" +#include "mock/ray/gcs/gcs_node_manager.h" #include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/test_utils.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc b/src/ray/gcs/tests/gcs_placement_group_scheduler_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc rename to src/ray/gcs/tests/gcs_placement_group_scheduler_test.cc index f5d9d64cd84c..3a866866d72e 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_placement_group_scheduler_test.cc +++ b/src/ray/gcs/tests/gcs_placement_group_scheduler_test.cc @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" +#include "ray/gcs/gcs_placement_group_scheduler.h" #include @@ -26,10 +26,10 @@ #include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_node_manager.h" -#include "ray/gcs/gcs_server/gcs_placement_group.h" -#include "ray/gcs/gcs_server/gcs_resource_manager.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/gcs_node_manager.h" +#include "ray/gcs/gcs_placement_group.h" +#include "ray/gcs/gcs_resource_manager.h" +#include "ray/gcs/gcs_table_storage.h" #include "ray/gcs/store_client/in_memory_store_client.h" #include "ray/raylet/scheduling/cluster_resource_scheduler.h" #include "ray/util/counter_map.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_ray_event_converter_test.cc b/src/ray/gcs/tests/gcs_ray_event_converter_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_ray_event_converter_test.cc rename to src/ray/gcs/tests/gcs_ray_event_converter_test.cc index ffffa9ddea0b..89a10bebe4f2 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_ray_event_converter_test.cc +++ b/src/ray/gcs/tests/gcs_ray_event_converter_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_ray_event_converter.h" +#include "ray/gcs/gcs_ray_event_converter.h" #include diff --git a/src/ray/gcs/gcs_server/tests/gcs_resource_manager_test.cc b/src/ray/gcs/tests/gcs_resource_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_resource_manager_test.cc rename to src/ray/gcs/tests/gcs_resource_manager_test.cc index a652ec345edf..a9732014aadd 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_resource_manager_test.cc +++ b/src/ray/gcs/tests/gcs_resource_manager_test.cc @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_resource_manager.h" +#include "ray/gcs/gcs_resource_manager.h" #include #include #include #include "gtest/gtest.h" -#include "mock/ray/gcs/gcs_server/gcs_node_manager.h" +#include "mock/ray/gcs/gcs_node_manager.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/test_utils.h" #include "ray/raylet/scheduling/cluster_resource_manager.h" diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc b/src/ray/gcs/tests/gcs_server_rpc_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc rename to src/ray/gcs/tests/gcs_server_rpc_test.cc index 7cdc021f49a0..fff248c8b0f5 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_server_rpc_test.cc +++ b/src/ray/gcs/tests/gcs_server_rpc_test.cc @@ -20,7 +20,7 @@ #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/ray_config.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_server.h" +#include "ray/gcs/gcs_server.h" #include "ray/gcs_client/rpc_client.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h b/src/ray/gcs/tests/gcs_server_test_util.h similarity index 97% rename from src/ray/gcs/gcs_server/tests/gcs_server_test_util.h rename to src/ray/gcs/tests/gcs_server_test_util.h index b52445c5c4f5..1b4ff40714a4 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_server_test_util.h +++ b/src/ray/gcs/tests/gcs_server_test_util.h @@ -27,12 +27,12 @@ #include "ray/common/lease/lease.h" #include "ray/common/task/task_util.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_actor_manager.h" -#include "ray/gcs/gcs_server/gcs_actor_scheduler.h" -#include "ray/gcs/gcs_server/gcs_node_manager.h" -#include "ray/gcs/gcs_server/gcs_placement_group_mgr.h" -#include "ray/gcs/gcs_server/gcs_placement_group_scheduler.h" -#include "ray/gcs/gcs_server/gcs_resource_manager.h" +#include "ray/gcs/gcs_actor_manager.h" +#include "ray/gcs/gcs_actor_scheduler.h" +#include "ray/gcs/gcs_node_manager.h" +#include "ray/gcs/gcs_placement_group_mgr.h" +#include "ray/gcs/gcs_placement_group_scheduler.h" +#include "ray/gcs/gcs_resource_manager.h" #include "ray/gcs/store_client/in_memory_store_client.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h b/src/ray/gcs/tests/gcs_table_storage_test_base.h similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h rename to src/ray/gcs/tests/gcs_table_storage_test_base.h index 982327dffee1..7b5010f8ebaa 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h +++ b/src/ray/gcs/tests/gcs_table_storage_test_base.h @@ -20,7 +20,7 @@ #include "gtest/gtest.h" #include "ray/common/id.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" +#include "ray/gcs/gcs_table_storage.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc b/src/ray/gcs/tests/gcs_task_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc rename to src/ray/gcs/tests/gcs_task_manager_test.cc index b52232537d48..19249eee30d7 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_task_manager_test.cc +++ b/src/ray/gcs/tests/gcs_task_manager_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_task_manager.h" +#include "ray/gcs/gcs_task_manager.h" #include diff --git a/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc b/src/ray/gcs/tests/gcs_worker_manager_test.cc similarity index 99% rename from src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc rename to src/ray/gcs/tests/gcs_worker_manager_test.cc index 4adaa81169b8..6730cdd29283 100644 --- a/src/ray/gcs/gcs_server/tests/gcs_worker_manager_test.cc +++ b/src/ray/gcs/tests/gcs_worker_manager_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/gcs_worker_manager.h" +#include "ray/gcs/gcs_worker_manager.h" #include @@ -22,8 +22,8 @@ #include "mock/ray/pubsub/publisher.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/store_client_kv.h" #include "ray/gcs/store_client/in_memory_store_client.h" +#include "ray/gcs/store_client_kv.h" #include "ray/util/process.h" #include "src/ray/protobuf/common.pb.h" #include "src/ray/protobuf/gcs.pb.h" diff --git a/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc b/src/ray/gcs/tests/in_memory_gcs_table_storage_test.cc similarity index 91% rename from src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc rename to src/ray/gcs/tests/in_memory_gcs_table_storage_test.cc index 4b4ddcbcfa7b..ac20883d2e85 100644 --- a/src/ray/gcs/gcs_server/tests/in_memory_gcs_table_storage_test.cc +++ b/src/ray/gcs/tests/in_memory_gcs_table_storage_test.cc @@ -17,9 +17,9 @@ #include #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h" +#include "ray/gcs/gcs_table_storage.h" #include "ray/gcs/store_client/in_memory_store_client.h" +#include "ray/gcs/tests/gcs_table_storage_test_base.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc b/src/ray/gcs/tests/redis_gcs_table_storage_test.cc similarity index 93% rename from src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc rename to src/ray/gcs/tests/redis_gcs_table_storage_test.cc index f89124b3319e..fd9ec84352f9 100644 --- a/src/ray/gcs/gcs_server/tests/redis_gcs_table_storage_test.cc +++ b/src/ray/gcs/tests/redis_gcs_table_storage_test.cc @@ -16,9 +16,9 @@ #include "gtest/gtest.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_table_storage.h" -#include "ray/gcs/gcs_server/tests/gcs_table_storage_test_base.h" +#include "ray/gcs/gcs_table_storage.h" #include "ray/gcs/store_client/redis_store_client.h" +#include "ray/gcs/tests/gcs_table_storage_test_base.h" namespace ray { diff --git a/src/ray/gcs/gcs_server/tests/usage_stats_client_test.cc b/src/ray/gcs/tests/usage_stats_client_test.cc similarity index 93% rename from src/ray/gcs/gcs_server/tests/usage_stats_client_test.cc rename to src/ray/gcs/tests/usage_stats_client_test.cc index 7a0b7ffea376..15fcaa19674d 100644 --- a/src/ray/gcs/gcs_server/tests/usage_stats_client_test.cc +++ b/src/ray/gcs/tests/usage_stats_client_test.cc @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/usage_stats_client.h" +#include "ray/gcs/usage_stats_client.h" #include #include #include -#include "mock/ray/gcs/gcs_server/gcs_kv_manager.h" +#include "mock/ray/gcs/gcs_kv_manager.h" #include "ray/common/asio/asio_util.h" -#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_kv_manager.h" using namespace ray; // NOLINT diff --git a/src/ray/gcs/gcs_server/usage_stats_client.cc b/src/ray/gcs/usage_stats_client.cc similarity index 96% rename from src/ray/gcs/gcs_server/usage_stats_client.cc rename to src/ray/gcs/usage_stats_client.cc index 8f46eb6b4970..cdd1ae431496 100644 --- a/src/ray/gcs/gcs_server/usage_stats_client.cc +++ b/src/ray/gcs/usage_stats_client.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ray/gcs/gcs_server/usage_stats_client.h" +#include "ray/gcs/usage_stats_client.h" #include diff --git a/src/ray/gcs/gcs_server/usage_stats_client.h b/src/ray/gcs/usage_stats_client.h similarity index 97% rename from src/ray/gcs/gcs_server/usage_stats_client.h rename to src/ray/gcs/usage_stats_client.h index a79cb6bbc4e4..2ff37f70354a 100644 --- a/src/ray/gcs/gcs_server/usage_stats_client.h +++ b/src/ray/gcs/usage_stats_client.h @@ -17,7 +17,7 @@ #include #include -#include "ray/gcs/gcs_server/gcs_kv_manager.h" +#include "ray/gcs/gcs_kv_manager.h" #include "src/ray/protobuf/usage.pb.h" namespace ray { diff --git a/src/ray/gcs_client/accessor.cc b/src/ray/gcs_client/accessor.cc index e035c2002e0c..b698a6b14bc4 100644 --- a/src/ray/gcs_client/accessor.cc +++ b/src/ray/gcs_client/accessor.cc @@ -1165,7 +1165,7 @@ void InternalKVAccessor::AsyncInternalKVMultiGet( callback(status, map); } else { // TODO(ryw): reply.status() is not examined. It's never populated in - // src/ray/gcs/gcs_server/gcs_kv_manager.cc either anyway so it's ok for now. + // src/ray/gcs/gcs_kv_manager.cc either anyway so it's ok for now. // Investigate if we wanna remove that field. for (const auto &entry : reply.results()) { map[entry.key()] = entry.value(); diff --git a/src/ray/gcs_client/tests/BUILD.bazel b/src/ray/gcs_client/tests/BUILD.bazel index c0b12afa33b5..f12f17b71480 100644 --- a/src/ray/gcs_client/tests/BUILD.bazel +++ b/src/ray/gcs_client/tests/BUILD.bazel @@ -31,7 +31,7 @@ ray_cc_test( tags = ["team:core"], deps = [ "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs:gcs_server_lib", "//src/ray/gcs_client", "//src/ray/gcs_client:global_state_accessor_lib", "//src/ray/util:path_utils", @@ -61,7 +61,7 @@ ray_cc_test( ], deps = [ "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs:gcs_server_lib", "//src/ray/gcs_client", "//src/ray/util:network_util", "//src/ray/util:raii", @@ -89,7 +89,7 @@ ray_cc_test( ], deps = [ "//src/ray/common:test_utils", - "//src/ray/gcs/gcs_server:gcs_server_lib", + "//src/ray/gcs:gcs_server_lib", "//src/ray/gcs_client", "//src/ray/util:network_util", "//src/ray/util:path_utils", diff --git a/src/ray/gcs_client/tests/gcs_client_reconnection_test.cc b/src/ray/gcs_client/tests/gcs_client_reconnection_test.cc index 11803e70019a..ec02beadf82d 100644 --- a/src/ray/gcs_client/tests/gcs_client_reconnection_test.cc +++ b/src/ray/gcs_client/tests/gcs_client_reconnection_test.cc @@ -23,7 +23,7 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_server.h" +#include "ray/gcs/gcs_server.h" #include "ray/gcs_client/accessor.h" #include "ray/gcs_client/gcs_client.h" #include "ray/gcs_client/rpc_client.h" diff --git a/src/ray/gcs_client/tests/gcs_client_test.cc b/src/ray/gcs_client/tests/gcs_client_test.cc index e30edd401536..c788f08d2a47 100644 --- a/src/ray/gcs_client/tests/gcs_client_test.cc +++ b/src/ray/gcs_client/tests/gcs_client_test.cc @@ -23,7 +23,7 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_server.h" +#include "ray/gcs/gcs_server.h" #include "ray/gcs_client/accessor.h" #include "ray/gcs_client/rpc_client.h" #include "ray/util/network_util.h" diff --git a/src/ray/gcs_client/tests/global_state_accessor_test.cc b/src/ray/gcs_client/tests/global_state_accessor_test.cc index eb9b3b32b16f..1575c85766fa 100644 --- a/src/ray/gcs_client/tests/global_state_accessor_test.cc +++ b/src/ray/gcs_client/tests/global_state_accessor_test.cc @@ -21,7 +21,7 @@ #include "gtest/gtest.h" #include "ray/common/asio/instrumented_io_context.h" #include "ray/common/test_utils.h" -#include "ray/gcs/gcs_server/gcs_server.h" +#include "ray/gcs/gcs_server.h" #include "ray/gcs_client/rpc_client.h" #include "ray/util/path_utils.h" #include "ray/util/raii.h" From 212ce299048606763bcc8e9d7d961cce3a84a17c Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 15 Sep 2025 15:33:22 -0700 Subject: [PATCH 631/634] [image] change tag methods of container class to private (#56551) they are only used within the class Signed-off-by: Lonnie Liu --- ci/ray_ci/docker_container.py | 8 ++++---- ci/ray_ci/test_ray_docker_container.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ci/ray_ci/docker_container.py b/ci/ray_ci/docker_container.py index 9544cef62167..492c9217de6f 100644 --- a/ci/ray_ci/docker_container.py +++ b/ci/ray_ci/docker_container.py @@ -114,10 +114,10 @@ def _get_canonical_tag(self) -> str: # e.g. sha-pyversion-platform return self.canonical_tag if self.canonical_tag else self._get_image_tags()[0] - def get_python_version_tag(self) -> str: + def _get_python_version_tag(self) -> str: return f"-py{self.python_version.replace('.', '')}" # 3.x -> py3x - def get_platform_tag(self) -> str: + def _get_platform_tag(self) -> str: if self.platform == "cpu": return "-cpu" versions = self.platform.split(".") @@ -138,7 +138,7 @@ def _get_image_tags(self, external: bool = False) -> List[str]: versions = self._get_image_version_tags(external) - platforms = [self.get_platform_tag()] + platforms = [self._get_platform_tag()] if self.platform == "cpu" and self.image_type == RayType.RAY: # no tag is alias to cpu for ray image platforms.append("") @@ -149,7 +149,7 @@ def _get_image_tags(self, external: bool = False) -> List[str]: # no tag is alias to gpu for ray-ml image platforms.append("") - py_versions = [self.get_python_version_tag()] + py_versions = [self._get_python_version_tag()] if self.python_version == DEFAULT_PYTHON_VERSION: py_versions.append("") diff --git a/ci/ray_ci/test_ray_docker_container.py b/ci/ray_ci/test_ray_docker_container.py index e60528207c9b..96f14d3dc741 100644 --- a/ci/ray_ci/test_ray_docker_container.py +++ b/ci/ray_ci/test_ray_docker_container.py @@ -395,30 +395,30 @@ def test_get_python_version_tag(self) -> None: v = DEFAULT_PYTHON_VERSION pv = self.get_python_version(v) container = RayDockerContainer(v, "cpu", "ray") - assert container.get_python_version_tag() == f"-{pv}" + assert container._get_python_version_tag() == f"-{pv}" def test_get_platform_tag(self) -> None: v = DEFAULT_PYTHON_VERSION container = RayDockerContainer(v, "cpu", "ray") - assert container.get_platform_tag() == "-cpu" + assert container._get_platform_tag() == "-cpu" container = RayDockerContainer(v, "cu11.8.0-cudnn8", "ray") - assert container.get_platform_tag() == "-cu118" + assert container._get_platform_tag() == "-cu118" container = RayDockerContainer(v, "cu12.3.2-cudnn9", "ray") - assert container.get_platform_tag() == "-cu123" + assert container._get_platform_tag() == "-cu123" container = RayDockerContainer(v, "cu12.4.1-cudnn", "ray") - assert container.get_platform_tag() == "-cu124" + assert container._get_platform_tag() == "-cu124" container = RayDockerContainer(v, "cu12.5.1-cudnn", "ray") - assert container.get_platform_tag() == "-cu125" + assert container._get_platform_tag() == "-cu125" container = RayDockerContainer(v, "cu12.6.3-cudnn", "ray") - assert container.get_platform_tag() == "-cu126" + assert container._get_platform_tag() == "-cu126" container = RayDockerContainer(v, "cu12.8.1-cudnn", "ray") - assert container.get_platform_tag() == "-cu128" + assert container._get_platform_tag() == "-cu128" def test_should_upload(self) -> None: v = DEFAULT_PYTHON_VERSION From e5e4ae30afdd52b88d4fb0db9aca6670496f0734 Mon Sep 17 00:00:00 2001 From: Lonnie Liu <95255098+aslonnie@users.noreply.github.com> Date: Mon, 15 Sep 2025 15:56:14 -0700 Subject: [PATCH 632/634] [image] add ray-llm image type check (#56542) make the check stricter Signed-off-by: Lonnie Liu --- ci/ray_ci/docker_container.py | 15 ++++++++++--- ci/ray_ci/test_ray_docker_container.py | 30 ++++++++++++++------------ 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/ci/ray_ci/docker_container.py b/ci/ray_ci/docker_container.py index 492c9217de6f..9f739466d679 100644 --- a/ci/ray_ci/docker_container.py +++ b/ci/ray_ci/docker_container.py @@ -21,17 +21,21 @@ "cpu", "cu12.1.1-cudnn8", ] +PLATFORMS_RAY_LLM = ["cu12.8.1-cudnn"] GPU_PLATFORM = "cu12.1.1-cudnn8" PYTHON_VERSIONS_RAY = ["3.9", "3.10", "3.11", "3.12"] PYTHON_VERSIONS_RAY_ML = ["3.9", "3.10", "3.11"] +PYTHON_VERSIONS_RAY_LLM = ["3.11"] ARCHITECTURES_RAY = ["x86_64", "aarch64"] ARCHITECTURES_RAY_ML = ["x86_64"] +ARCHITECTURES_RAY_LLM = ["x86_64"] class RayType(str, Enum): RAY = "ray" RAY_ML = "ray-ml" + RAY_LLM = "ray-llm" class DockerContainer(LinuxContainer): @@ -50,13 +54,18 @@ def __init__( ) -> None: assert "RAYCI_CHECKOUT_DIR" in os.environ, "RAYCI_CHECKOUT_DIR not set" - assert python_version in PYTHON_VERSIONS_RAY - assert platform in PLATFORMS_RAY - assert architecture in ARCHITECTURES_RAY if image_type == RayType.RAY_ML: assert python_version in PYTHON_VERSIONS_RAY_ML assert platform in PLATFORMS_RAY_ML assert architecture in ARCHITECTURES_RAY_ML + elif image_type == RayType.RAY_LLM: + assert python_version in PYTHON_VERSIONS_RAY_LLM + assert platform in PLATFORMS_RAY_LLM + assert architecture in ARCHITECTURES_RAY_LLM + else: + assert python_version in PYTHON_VERSIONS_RAY + assert platform in PLATFORMS_RAY + assert architecture in ARCHITECTURES_RAY rayci_checkout_dir = os.environ["RAYCI_CHECKOUT_DIR"] self.python_version = python_version diff --git a/ci/ray_ci/test_ray_docker_container.py b/ci/ray_ci/test_ray_docker_container.py index 96f14d3dc741..767bfbdd228e 100644 --- a/ci/ray_ci/test_ray_docker_container.py +++ b/ci/ray_ci/test_ray_docker_container.py @@ -53,6 +53,7 @@ def _mock_run_script(input: List[str]) -> None: v = "3.11" cv = self.get_cpp_version(v) pv = self.get_python_version(v) + cuda = "cu12.8.1-cudnn" container = RayDockerContainer(v, cuda, "ray-llm") container.run() cmd = self.cmds[-1] @@ -61,14 +62,15 @@ def _mock_run_script(input: List[str]) -> None: f"ray-{RAY_VERSION}-{cv}-{cv}-manylinux2014_x86_64.whl " f"{_DOCKER_ECR_REPO}:{ray_ci_build_id}-ray-llm-py{v}-{cuda}-base " "requirements_compiled.txt " - f"rayproject/ray-llm:{sha}-{pv}-cu124 " - f"ray-llm:{sha}-{pv}-cu124_pip-freeze.txt" + f"rayproject/ray-llm:{sha}-{pv}-cu128 " + f"ray-llm:{sha}-{pv}-cu128_pip-freeze.txt" ) # Run with non-default python version and ray-ml image v = self.get_non_default_python() cv = self.get_cpp_version(v) pv = self.get_python_version(v) + cuda = "cu12.4.1-cudnn" container = RayDockerContainer(v, "cpu", "ray-ml") container.run() cmd = self.cmds[-1] @@ -132,7 +134,7 @@ def _mock_run_script(input: List[str]) -> None: v = "3.11" cv = self.get_cpp_version(v) pv = self.get_python_version(v) - cuda = "cu12.4.1-cudnn" + cuda = "cu12.8.1-cudnn" container = RayDockerContainer(v, cuda, "ray-llm") container.run() assert len(self.cmds) == 6 @@ -141,8 +143,8 @@ def _mock_run_script(input: List[str]) -> None: f"ray-{RAY_VERSION}-{cv}-{cv}-manylinux2014_x86_64.whl " f"{_DOCKER_ECR_REPO}:{ray_ci_build_id}-ray-llm-py{v}-{cuda}-base " "requirements_compiled.txt " - f"rayproject/ray-llm:{sha}-{pv}-cu124 " - f"ray-llm:{sha}-{pv}-cu124_pip-freeze.txt" + f"rayproject/ray-llm:{sha}-{pv}-cu128 " + f"ray-llm:{sha}-{pv}-cu128_pip-freeze.txt" ) assert ( self.cmds[1] @@ -217,8 +219,8 @@ def _mock_run_script(input: List[str]) -> None: # Run with specific python version and ray-llm image self.cmds = [] - v = DEFAULT_PYTHON_VERSION - cuda = "cu12.4.1-cudnn" + v = "3.11" + cuda = "cu12.8.1-cudnn" cv = self.get_cpp_version(v) pv = self.get_python_version(v) container = RayDockerContainer(v, cuda, "ray-llm") @@ -229,8 +231,8 @@ def _mock_run_script(input: List[str]) -> None: f"ray-{RAY_VERSION}-{cv}-{cv}-manylinux2014_x86_64.whl " f"{_DOCKER_ECR_REPO}:{ray_ci_build_id}-ray-llm-py{v}-{cuda}-base " "requirements_compiled.txt " - f"rayproject/ray-llm:{sha}-{pv}-cu124 " - f"ray-llm:{sha}-{pv}-cu124_pip-freeze.txt" + f"rayproject/ray-llm:{sha}-{pv}-cu128 " + f"ray-llm:{sha}-{pv}-cu128_pip-freeze.txt" ) # Run with non-default python version and ray-ml image @@ -341,17 +343,17 @@ def test_get_image_name(self) -> None: v = "3.11" pv = self.get_python_version(v) - container = RayDockerContainer(v, "cu12.4.1-cudnn", "ray-llm") + container = RayDockerContainer(v, "cu12.8.1-cudnn", "ray-llm") with mock.patch.dict(os.environ, {"RAYCI_SCHEDULE": "daytime"}): assert container._get_image_names() == [ - f"rayproject/ray-llm:{sha}-{pv}-cu124", - f"rayproject/ray-llm:{rayci_build_id}-{pv}-cu124", + f"rayproject/ray-llm:{sha}-{pv}-cu128", + f"rayproject/ray-llm:{rayci_build_id}-{pv}-cu128", ] with mock.patch.dict(os.environ, {"RAYCI_SCHEDULE": "nightly"}): assert container._get_image_names() == [ - f"rayproject/ray-llm:nightly.{formatted_date}.{sha}-{pv}-cu124", - f"rayproject/ray-llm:nightly-{pv}-cu124", + f"rayproject/ray-llm:nightly.{formatted_date}.{sha}-{pv}-cu128", + f"rayproject/ray-llm:nightly-{pv}-cu128", ] v = self.get_non_default_python() From 13af56ad1e5e4a152d4e1561ae98c14d52d0f5b2 Mon Sep 17 00:00:00 2001 From: Jugal Shah <47508441+jugalshah291@users.noreply.github.com> Date: Mon, 15 Sep 2025 16:05:06 -0700 Subject: [PATCH 633/634] Add optional APIType filter to /api/serve/applications/ endpoint (#56458) ## Why are these changes needed? As part of this PR I am trying to address Problem 2 raised in issue https://github.com/ray-project/ray/issues/44226. The main aim is to enable KubeRay to exclusively check the status of only DECLARATIVE Serve apps. The solution would be build on top of this https://github.com/ray-project/ray/pull/45522 Based on my current understanding, it seems KubeRay should only operate on the DECLARATIVE Serve apps Thus my solution will involve two key steps: This PR- Update the /api/serve/applications/ endpoint to read the APIType from the request body and pass it on to the controller controller.get_serve_instance_details Next modify KubeRay to explicitly pass Declarative as the APIType when calling the /api/serve/applications/ ## Related issue number ## Checks - [x] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR. - [x] I've run `scripts/format.sh` to lint the changes in this PR. - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/. - [ ] I've added any new APIs to the API Reference. For example, if I added a method in Tune, I've added it in `doc/source/tune/api/` under the corresponding `.rst` file. - [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/ - Testing Strategy - [x] Unit tests - [ ] Release tests - [ ] This PR is not tested :( --------- Signed-off-by: jugalshah291 Co-authored-by: Cindy Zhang --- .../ray/dashboard/modules/serve/serve_head.py | 27 ++- .../serve/tests/test_serve_dashboard.py | 166 ++++++++++++++++++ .../ray/serve/_private/application_state.py | 30 +++- python/ray/serve/_private/controller.py | 10 +- python/ray/serve/schema.py | 8 + 5 files changed, 231 insertions(+), 10 deletions(-) diff --git a/python/ray/dashboard/modules/serve/serve_head.py b/python/ray/dashboard/modules/serve/serve_head.py index 151a19908002..02e940bc7601 100644 --- a/python/ray/dashboard/modules/serve/serve_head.py +++ b/python/ray/dashboard/modules/serve/serve_head.py @@ -3,6 +3,7 @@ import json import logging from functools import wraps +from typing import Optional import aiohttp from aiohttp.web import Request, Response @@ -81,7 +82,27 @@ async def get_version(self, req: Request) -> Response: @dashboard_optional_utils.init_ray_and_catch_exceptions() @validate_endpoint() async def get_serve_instance_details(self, req: Request) -> Response: - from ray.serve.schema import ServeInstanceDetails + from ray.serve.schema import APIType, ServeInstanceDetails + + api_type: Optional[APIType] = None + api_type_str = req.query.get("api_type") + + if api_type_str: + api_type_lower = api_type_str.lower() + valid_values = APIType.get_valid_user_values() + + if api_type_lower not in valid_values: + # Explicitly check against valid user values (excludes 'unknown') + return Response( + status=400, + text=( + f"Invalid 'api_type' value: '{api_type_str}'. " + f"Must be one of: {', '.join(valid_values)}" + ), + content_type="text/plain", + ) + + api_type = APIType(api_type_lower) controller = await self.get_serve_controller() @@ -90,7 +111,9 @@ async def get_serve_instance_details(self, req: Request) -> Response: details = ServeInstanceDetails.get_empty_schema_dict() else: try: - details = await controller.get_serve_instance_details.remote() + details = await controller.get_serve_instance_details.remote( + source=api_type + ) except ray.exceptions.RayTaskError as e: # Task failure sometimes are due to GCS # failure. When GCS failed, we expect a longer time diff --git a/python/ray/dashboard/modules/serve/tests/test_serve_dashboard.py b/python/ray/dashboard/modules/serve/tests/test_serve_dashboard.py index ac5f7e7a7998..2d019f25dee2 100644 --- a/python/ray/dashboard/modules/serve/tests/test_serve_dashboard.py +++ b/python/ray/dashboard/modules/serve/tests/test_serve_dashboard.py @@ -572,5 +572,171 @@ def applications_running(): print("Finished checking application details.") +@pytest.mark.skipif( + sys.platform == "darwin" and not TEST_ON_DARWIN, reason="Flaky on OSX." +) +def test_get_serve_instance_details_api_type_filtering(ray_start_stop): + """ + Test the api_type query parameter for filtering applications by API type. + Tests both declarative and imperative applications. + """ + # First, deploy declarative applications + world_import_path = "ray.serve.tests.test_config_files.world.DagNode" + declarative_config = { + "applications": [ + { + "name": "declarative_app1", + "route_prefix": "/declarative1", + "import_path": world_import_path, + }, + { + "name": "declarative_app2", + "route_prefix": "/declarative2", + "import_path": world_import_path, + }, + ], + } + + deploy_config_multi_app(declarative_config, SERVE_HEAD_URL) + + # Wait for declarative apps to be running + def declarative_apps_running(): + response = requests.get(SERVE_HEAD_URL, timeout=15) + assert response.status_code == 200 + serve_details = ServeInstanceDetails(**response.json()) + return len(serve_details.applications) == 2 and all( + app.status == ApplicationStatus.RUNNING + for app in serve_details.applications.values() + ) + + wait_for_condition(declarative_apps_running, timeout=15) + print("Declarative applications are running.") + + # Deploy imperative applications using subprocess + deploy = subprocess.run( + [ + sys.executable, + str(Path(__file__).parent / "deploy_imperative_serve_apps.py"), + ], + capture_output=True, + universal_newlines=True, + ) + assert deploy.returncode == 0 + + # Wait for imperative apps to be running + def all_apps_running(): + response = requests.get(SERVE_HEAD_URL, timeout=15) + assert response.status_code == 200 + serve_details = ServeInstanceDetails(**response.json()) + return len( + serve_details.applications + ) == 4 and all( # 2 declarative + 2 imperative + app.status == ApplicationStatus.RUNNING + for app in serve_details.applications.values() + ) + + wait_for_condition(all_apps_running, timeout=15) + print("All applications (declarative + imperative) are running.") + + # Test 1: No api_type parameter - should return all applications + response = requests.get(SERVE_HEAD_URL, timeout=15) + assert response.status_code == 200 + serve_details = ServeInstanceDetails(**response.json()) + assert len(serve_details.applications) == 4 + app_names = set(serve_details.applications.keys()) + assert app_names == {"declarative_app1", "declarative_app2", "app1", "app2"} + + # Test 2: Filter by declarative applications + response = requests.get(SERVE_HEAD_URL + "?api_type=declarative", timeout=15) + assert response.status_code == 200 + serve_details = ServeInstanceDetails(**response.json()) + assert len(serve_details.applications) == 2 + app_names = set(serve_details.applications.keys()) + assert app_names == {"declarative_app1", "declarative_app2"} + for app in serve_details.applications.values(): + assert app.source == "declarative" + + # Test 3: Filter by imperative applications + response = requests.get(SERVE_HEAD_URL + "?api_type=imperative", timeout=15) + assert response.status_code == 200 + serve_details = ServeInstanceDetails(**response.json()) + assert len(serve_details.applications) == 2 + app_names = set(serve_details.applications.keys()) + assert app_names == {"app1", "app2"} + for app in serve_details.applications.values(): + assert app.source == "imperative" + + # Test 4: Filter by unknown - should return 400 error (unknown is not a valid user input) + response = requests.get(SERVE_HEAD_URL + "?api_type=unknown", timeout=15) + assert response.status_code == 400 + assert "Invalid 'api_type' value" in response.text + assert "Must be one of: imperative, declarative" in response.text + + +@pytest.mark.skipif( + sys.platform == "darwin" and not TEST_ON_DARWIN, reason="Flaky on OSX." +) +def test_get_serve_instance_details_invalid_api_type(ray_start_stop): + """ + Test that invalid api_type values return appropriate error responses. + """ + # Test with invalid api_type value + response = requests.get(SERVE_HEAD_URL + "?api_type=invalid_type", timeout=15) + assert response.status_code == 400 + assert "Invalid 'api_type' value" in response.text + assert "Must be one of: imperative, declarative" in response.text + + # Test with another invalid value + response = requests.get(SERVE_HEAD_URL + "?api_type=python", timeout=15) + assert response.status_code == 400 + assert "Invalid 'api_type' value" in response.text + + +@pytest.mark.skipif( + sys.platform == "darwin" and not TEST_ON_DARWIN, reason="Flaky on OSX." +) +def test_get_serve_instance_details_api_type_case_insensitive(ray_start_stop): + """ + Test that api_type parameter is case insensitive. + """ + # Deploy a declarative application + world_import_path = "ray.serve.tests.test_config_files.world.DagNode" + config = { + "applications": [ + { + "name": "test_app", + "route_prefix": "/test", + "import_path": world_import_path, + } + ], + } + + deploy_config_multi_app(config, SERVE_HEAD_URL) + + def app_running(): + response = requests.get(SERVE_HEAD_URL, timeout=15) + assert response.status_code == 200 + serve_details = ServeInstanceDetails(**response.json()) + return ( + len(serve_details.applications) == 1 + and serve_details.applications["test_app"].status + == ApplicationStatus.RUNNING + ) + + wait_for_condition(app_running, timeout=15) + + # Test case insensitive filtering + test_cases = ["DECLARATIVE", "Declarative", "declarative", "DeClArAtIvE"] + + for api_type_value in test_cases: + response = requests.get( + f"{SERVE_HEAD_URL}?api_type={api_type_value}", timeout=15 + ) + assert response.status_code == 200 + serve_details = ServeInstanceDetails(**response.json()) + assert len(serve_details.applications) == 1 + assert "test_app" in serve_details.applications + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) diff --git a/python/ray/serve/_private/application_state.py b/python/ray/serve/_private/application_state.py index 21123b9be039..6f5a36bb3d59 100644 --- a/python/ray/serve/_private/application_state.py +++ b/python/ray/serve/_private/application_state.py @@ -1104,12 +1104,30 @@ def get_ingress_deployment_name(self, name: str) -> Optional[str]: def get_app_source(self, name: str) -> APIType: return self._application_states[name].api_type - def list_app_statuses(self) -> Dict[str, ApplicationStatusInfo]: - """Return a dictionary with {app name: application info}""" - return { - name: self._application_states[name].get_application_status_info() - for name in self._application_states - } + def list_app_statuses( + self, source: Optional[APIType] = None + ) -> Dict[str, ApplicationStatusInfo]: + """Return a dictionary with {app name: application info} + + Args: + source: Optional API type filter. If provided, only returns apps + deployed via the specified API type. + + Returns: + Dict[str, ApplicationStatusInfo]: A dictionary mapping application names + to their corresponding status information. + """ + if source is None: + return { + name: self._application_states[name].get_application_status_info() + for name in self._application_states + } + else: + return { + name: self._application_states[name].get_application_status_info() + for name in self._application_states + if self.get_app_source(name) is source + } def list_deployment_details(self, name: str) -> Dict[str, DeploymentDetails]: """Gets detailed info on all deployments in specified application.""" diff --git a/python/ray/serve/_private/controller.py b/python/ray/serve/_private/controller.py index 446406026976..7e28d0d3f1ff 100644 --- a/python/ray/serve/_private/controller.py +++ b/python/ray/serve/_private/controller.py @@ -67,6 +67,7 @@ EndpointSet, ) from ray.serve.schema import ( + APIType, ApplicationDetails, DeploymentDetails, HTTPOptionsSchema, @@ -920,12 +921,17 @@ def list_deployment_ids(self) -> List[DeploymentID]: """Gets the current list of all deployments' identifiers.""" return self.deployment_state_manager._deployment_states.keys() - def get_serve_instance_details(self) -> Dict: + def get_serve_instance_details(self, source: Optional[APIType] = None) -> Dict: """Gets details on all applications on the cluster and system-level info. The information includes application and deployment statuses, config options, error messages, etc. + Args: + source: If provided, returns application + statuses for applications matching this API type. + Defaults to None, which means all applications are returned. + Returns: Dict that follows the format of the schema ServeInstanceDetails. """ @@ -934,7 +940,7 @@ def get_serve_instance_details(self) -> Dict: grpc_config = self.get_grpc_config() applications = {} - app_statuses = self.application_state_manager.list_app_statuses() + app_statuses = self.application_state_manager.list_app_statuses(source=source) # If there are no app statuses, there's no point getting the app configs. # Moreover, there might be no app statuses because the GCS is down, diff --git a/python/ray/serve/schema.py b/python/ray/serve/schema.py index 455ba06a9904..055420eff9a4 100644 --- a/python/ray/serve/schema.py +++ b/python/ray/serve/schema.py @@ -1089,6 +1089,14 @@ class APIType(str, Enum): IMPERATIVE = "imperative" DECLARATIVE = "declarative" + @classmethod + def get_valid_user_values(cls): + """Get list of valid APIType values that users can explicitly pass. + + Excludes 'unknown' which is for internal use only. + """ + return [cls.IMPERATIVE.value, cls.DECLARATIVE.value] + @PublicAPI(stability="stable") class ApplicationDetails(BaseModel, extra=Extra.forbid, frozen=True): From 9ab7a16ed0e7d36aa0f6fd10c6173068ee2feba9 Mon Sep 17 00:00:00 2001 From: Guy Stone Date: Mon, 15 Sep 2025 21:55:40 -0400 Subject: [PATCH 634/634] [Data][LLM] Support openai's nested image_url format in PrepareImageStage Signed-off-by: Guy Stone --- .../batch/stages/prepare_image_stage.py | 13 +++++++++++- .../cpu/stages/test_prepare_image_stage.py | 20 +++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/python/ray/llm/_internal/batch/stages/prepare_image_stage.py b/python/ray/llm/_internal/batch/stages/prepare_image_stage.py index bb5a31db131d..7c5c27571486 100644 --- a/python/ray/llm/_internal/batch/stages/prepare_image_stage.py +++ b/python/ray/llm/_internal/batch/stages/prepare_image_stage.py @@ -1,4 +1,5 @@ """Prepare Image Stage""" + import asyncio import base64 import importlib @@ -336,7 +337,17 @@ def extract_image_info(self, messages: List[Dict]) -> List[_ImageType]: for content_item in content: if content_item["type"] not in ("image", "image_url"): continue - image = content_item[content_item["type"]] + + image_data = content_item[content_item["type"]] + if content_item["type"] == "image_url" and isinstance(image_data, dict): + # OpenAI nested format: {"image_url": {"url": "..."}} + image = image_data.get("url") + if image is None: + raise ValueError("image_url dict must contain 'url' key") + else: + # Simple format: {"image": "..."} or {"image_url": "..."} + image = image_data + if not isinstance(image, str) and not isinstance( image, self.Image.Image ): diff --git a/python/ray/llm/tests/batch/cpu/stages/test_prepare_image_stage.py b/python/ray/llm/tests/batch/cpu/stages/test_prepare_image_stage.py index 2ee2f5e8bd3d..6e27850a7fd5 100644 --- a/python/ray/llm/tests/batch/cpu/stages/test_prepare_image_stage.py +++ b/python/ray/llm/tests/batch/cpu/stages/test_prepare_image_stage.py @@ -166,8 +166,6 @@ async def test_prepare_image_udf_invalid_image_type(mock_image_processor): # Test that image extraction works consistently with both uniform content types # (no system prompt) and mixed content types (with system prompt) - - @pytest.mark.parametrize( "messages,expected_images,test_description", [ @@ -250,6 +248,24 @@ async def test_prepare_image_udf_invalid_image_type(mock_image_processor): ["https://example.com/image.jpg"], "image_url_format_no_system_prompt", ), + # Test OpenAI nested format without system prompt + # https://github.com/openai/openai-openapi/blob/manual_spec/openapi.yaml#L1937-L1940 + ( + [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.jpg"}, + }, + {"type": "text", "text": "Describe this image"}, + ], + } + ], + ["https://example.com/image.jpg"], + "openai_image_url_format_no_system_prompt", + ), ], ids=lambda x: x if isinstance(x, str) else None, )